diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..500d065f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,11 @@ +# GPU and accelerator surfaces require maintainer review. +.github/CODEOWNERS @frames-sg/j2k-maintainers +.github/workflows/gpu-validation.yml @frames-sg/j2k-maintainers +crates/j2k-cuda-runtime/ @frames-sg/j2k-maintainers +crates/j2k-jpeg-cuda/ @frames-sg/j2k-maintainers +crates/j2k-cuda/ @frames-sg/j2k-maintainers +crates/j2k-transcode-cuda/ @frames-sg/j2k-maintainers +crates/j2k-metal-support/ @frames-sg/j2k-maintainers +crates/j2k-jpeg-metal/ @frames-sg/j2k-maintainers +crates/j2k-metal/ @frames-sg/j2k-maintainers +crates/j2k-transcode-metal/ @frames-sg/j2k-maintainers diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43c720b1..57fdcd16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,11 @@ on: push: branches: [main] pull_request: + schedule: + - cron: "17 8 * * *" + +permissions: + contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -15,34 +20,185 @@ env: RUSTDOCFLAGS: "-D warnings" jobs: + gpu-path-policy: + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + pull-requests: read + steps: + - name: Non-PR events do not require GPU path policy + if: ${{ github.event_name != 'pull_request' }} + run: echo "GPU path policy only applies to pull requests." + - name: Require GPU validation for GPU path changes + if: ${{ github.event_name == 'pull_request' }} + env: + GH_TOKEN: ${{ github.token }} + GITHUB_API_URL: ${{ github.api_url }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPOSITORY: ${{ github.repository }} + run: | + python3 <<'PY' + import json + import os + import sys + import urllib.parse + import urllib.request + + api_url = os.environ["GITHUB_API_URL"].rstrip("/") + repo = os.environ["REPOSITORY"] + token = os.environ["GH_TOKEN"] + pr_number = os.environ["PR_NUMBER"] + head_sha = os.environ["HEAD_SHA"] + + gpu_prefixes = ( + "crates/j2k-cuda-runtime/", + "crates/j2k-jpeg-cuda/", + "crates/j2k-cuda/", + "crates/j2k-transcode-cuda/", + "crates/j2k-metal-support/", + "crates/j2k-jpeg-metal/", + "crates/j2k-metal/", + "crates/j2k-transcode-metal/", + ) + gpu_exact_paths = { + ".github/CODEOWNERS", + ".github/workflows/gpu-validation.yml", + } + + def get_json(path): + request = urllib.request.Request( + f"{api_url}/repos/{repo}{path}", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + changed_files = [] + page = 1 + while True: + files = get_json(f"/pulls/{pr_number}/files?per_page=100&page={page}") + changed_files.extend(item["filename"] for item in files) + if len(files) < 100: + break + page += 1 + + gpu_changes = [ + path + for path in changed_files + if path in gpu_exact_paths or path.startswith(gpu_prefixes) + ] + if not gpu_changes: + print("No GPU path changes detected.") + sys.exit(0) + + encoded_sha = urllib.parse.quote(head_sha, safe="") + runs = get_json( + f"/actions/workflows/gpu-validation.yml/runs?head_sha={encoded_sha}&per_page=50" + ).get("workflow_runs", []) + for run in runs: + if run.get("head_sha") == head_sha and run.get("conclusion") == "success": + print( + "GPU path changes detected and gpu-validation.yml succeeded " + f"for head SHA {head_sha}." + ) + sys.exit(0) + + print("GPU path changes require a successful gpu-validation.yml run.") + print(f"Head SHA: {head_sha}") + print("Changed GPU paths:") + for path in gpu_changes: + print(f" - {path}") + sys.exit(1) + PY + fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - with: { components: rustfmt } + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: "1.96" + components: rustfmt - run: cargo xtask fmt clippy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - with: { components: clippy } + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + components: clippy - run: cargo xtask clippy + semver: + # Published Metal baselines include macOS-gated public API and must be + # rustdoc-built on macOS for cargo-semver-checks. + runs-on: macos-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + with: + fetch-depth: 0 + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: "1.96" + - run: cargo install cargo-semver-checks --version 0.48.0 --locked + - run: cargo xtask semver + docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable - run: cargo xtask doc + stable-api: + runs-on: macos-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: nightly + - uses: taiki-e/install-action@7a79fe8c3a13344501c80d99cae481c1c9085912 + with: + tool: cargo-public-api@0.52.0 + - run: cargo public-api --version || cargo install cargo-public-api --version 0.52.0 --locked + - run: cargo xtask stable-api + + release-integrity: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: cargo xtask release-integrity + + unsafe-audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: cargo xtask unsafe-audit + typos: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: crate-ci/typos@v1 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: crate-ci/typos@d80b8e26878e372a041833cd67163dbdb6a4336e test: strategy: @@ -51,15 +207,15 @@ jobs: include: - { os: ubuntu-latest, rust: stable, arch: x86_64 } - { os: ubuntu-latest, rust: beta, arch: x86_64 } - - { os: ubuntu-latest, rust: "1.94", arch: x86_64 } + - { os: ubuntu-latest, rust: "1.96", arch: x86_64 } - { os: ubuntu-24.04-arm, rust: stable, arch: aarch64 } - - { os: ubuntu-24.04-arm, rust: "1.94", arch: aarch64 } + - { os: ubuntu-24.04-arm, rust: "1.96", arch: aarch64 } - { os: macos-latest, rust: stable, arch: aarch64 } - - { os: macos-latest, rust: "1.94", arch: aarch64 } + - { os: macos-latest, rust: "1.96", arch: aarch64 } runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@master + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 with: toolchain: ${{ matrix.rust }} - run: cargo xtask test @@ -71,63 +227,122 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable - run: cargo xtask release-cpu release-metal: runs-on: macos-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable - run: cargo xtask release-metal no-std: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable - run: cargo xtask no-std + miri: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: nightly + components: miri + - run: cargo xtask miri + bench-build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable - run: cargo xtask bench-build fuzz-build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable - run: cargo xtask fuzz-build + fuzz-run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: nightly + - uses: taiki-e/install-action@7a79fe8c3a13344501c80d99cae481c1c9085912 + with: + tool: cargo-fuzz + - run: cargo xtask fuzz-run + env: + J2K_FUZZ_RUNS: "512" + J2K_FUZZ_MAX_TOTAL_TIME_SECONDS: "60" + + fuzz-run-long: + if: ${{ github.event_name == 'schedule' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: nightly + - uses: taiki-e/install-action@7a79fe8c3a13344501c80d99cae481c1c9085912 + with: + tool: cargo-fuzz + - run: cargo xtask fuzz-run + env: + J2K_FUZZ_RUNS: "20000" + J2K_FUZZ_MAX_TOTAL_TIME_SECONDS: "900" + package: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable - run: cargo xtask package coverage: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@cargo-llvm-cov + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - uses: taiki-e/install-action@91534edaf9fd796a162759d80d49cdff574bff2c + with: + tool: cargo-llvm-cov - run: cargo xtask coverage - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 if: always() with: - name: signinum-lcov + name: j2k-lcov path: lcov.info if-no-files-found: ignore deny: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: EmbarkStudios/cargo-deny-action@v2 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - uses: EmbarkStudios/cargo-deny-action@8f84122a46a358a27cb0625d85ad60ab436a1b87 with: { command: check licenses advisories bans sources } diff --git a/.github/workflows/gpu-validation.yml b/.github/workflows/gpu-validation.yml index c77f46ce..12b08562 100644 --- a/.github/workflows/gpu-validation.yml +++ b/.github/workflows/gpu-validation.yml @@ -12,12 +12,25 @@ on: description: "Run Apple Silicon Metal validation jobs" type: boolean required: false - default: true + # Default off: with no Metal runner online the job sits queued forever, + # which holds the whole run "in progress" and locks every job's logs + # (including the CUDA gate). Opt in explicitly when a Metal runner is up. + default: false run-linux-ci: description: "Run the Linux CI gate on the self-hosted CUDA runner" type: boolean required: false default: false + run-nvidia-baseline: + description: "Run the nvJPEG/nvJPEG2000 baseline comparison (requires nvJPEG2000 installed on the runner)" + type: boolean + required: false + default: false + run-cuda-htj2k-decode-profile: + description: "Collect CUDA HTJ2K decode RCA/profile log, trace, and Criterion artifacts" + type: boolean + required: false + default: false permissions: contents: read @@ -32,9 +45,11 @@ jobs: name: Linux CI on self-hosted CUDA runner runs-on: [self-hosted, Linux, X64, cuda] steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - with: { components: rustfmt, clippy } + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + components: rustfmt, clippy - name: Show runner diagnostics run: | uname -a @@ -46,7 +61,7 @@ jobs: - run: cargo xtask fuzz-build - name: Install CI helper tools run: | - cargo install typos-cli --locked + cargo install typos-cli --version 1.42.3 --locked cargo install cargo-deny --locked cargo install cargo-llvm-cov --locked - run: cargo xtask typos @@ -58,37 +73,44 @@ jobs: name: Metal validation on Apple Silicon runs-on: [self-hosted, macOS, ARM64, metal] steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: cargo test -p signinum-jpeg-metal --all-targets - - run: cargo test -p signinum-j2k-metal --all-targets - - run: cargo bench -p signinum-jpeg-metal --no-run - - run: cargo bench -p signinum-j2k-metal --no-run - - run: cargo bench -p signinum-j2k-metal --bench encode_stages --no-run + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: cargo test -p j2k-jpeg-metal --all-targets + - run: cargo test -p j2k-metal --all-targets + - run: cargo bench -p j2k-jpeg-metal --no-run - if: ${{ inputs.run-timed-benchmarks }} env: - SIGNINUM_GPU_BENCH_DIM: "1024" - SIGNINUM_GPU_BENCH_BATCH: "64" - SIGNINUM_GPU_BENCH_BATCH_DIM: "1024" - run: cargo bench -p signinum-jpeg-metal --bench device_upload -- --noplot + J2K_GPU_BENCH_DIM: "1024" + J2K_GPU_BENCH_BATCH: "64" + J2K_GPU_BENCH_BATCH_DIM: "1024" + run: cargo bench -p j2k-jpeg-metal --bench device_upload -- --noplot --sample-size 10 --warm-up-time 1 --measurement-time 2 - if: ${{ inputs.run-timed-benchmarks }} - run: cargo bench -p signinum-jpeg-metal --bench compare -- --noplot - - if: ${{ inputs.run-timed-benchmarks }} - run: cargo bench -p signinum-j2k-metal --bench compare - - if: ${{ inputs.run-timed-benchmarks }} - env: - SIGNINUM_REQUIRE_METAL_BENCH: "1" - run: cargo bench -p signinum-j2k-metal --bench encode_stages -- --noplot + run: cargo bench -p j2k-jpeg-metal --bench compare -- --noplot --sample-size 10 --warm-up-time 1 --measurement-time 2 + # cuda-x86_64-compatibility is THE authoritative CUDA parity gate. + # It must be run (via workflow_dispatch) before merging any change to the + # CUDA encode path (crates/j2k-cuda, crates/j2k-cuda-runtime). + # The job is workflow_dispatch-only; do not add push/pull_request triggers + # without a policy decision — these are cost-sensitive self-hosted GPU runners. cuda-x86_64-compatibility: name: CUDA API compatibility on x86_64 runs-on: [self-hosted, Linux, X64, cuda] env: - SIGNINUM_REQUIRE_CUDA_RUNTIME: "1" - SIGNINUM_REQUIRE_CUDA_JPEG_HARDWARE_DECODE: "1" + NVCC: /usr/local/cuda/bin/nvcc + J2K_REQUIRE_CUDA_RUNTIME: "1" + J2K_REQUIRE_CUDA_HTJ2K_STRICT: "1" + J2K_REQUIRE_CUDA_JPEG_HARDWARE_DECODE: "1" + RUST_TEST_THREADS: "1" steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - name: Assert CUDA runtime gate is set + run: | + test -n "${J2K_REQUIRE_CUDA_RUNTIME:-}" || { echo "J2K_REQUIRE_CUDA_RUNTIME not set — parity tests would silently skip"; exit 1; } - name: Show CUDA runner diagnostics run: | uname -a @@ -99,27 +121,224 @@ jobs: else echo "nvidia-smi not found; CUDA runtime validation requires a working CUDA driver" fi - if command -v nvcc >/dev/null 2>&1; then - nvcc --version - else - echo "nvcc not found; CUDA encode kernel build will use fallback PTX" + command -v nvcc + nvcc --version + - run: cargo test -p j2k-cuda-runtime --all-targets + - run: cargo test -p j2k-cuda-runtime --all-targets --features cuda-profiling + - run: cargo test -p j2k-jpeg-cuda --all-targets --features cuda-runtime + - run: cargo test -p j2k-cuda --all-targets --features cuda-runtime + - name: Run HTJ2K encode parity tests with executed-count floor + run: | + out=$(cargo test -p j2k-cuda --features cuda-runtime --test htj2k_encode_parity -- --nocapture 2>&1) + echo "$out" + passed=$(echo "$out" | sed -n 's/^test result: ok\. \([0-9]\+\) passed.*/\1/p' | head -n1) + echo "parity tests passed: ${passed:-0}" + test "${passed:-0}" -ge 8 || { echo "Expected at least 8 executed parity tests, got ${passed:-0} — tests may have silently skipped"; exit 1; } + - run: cargo test -p j2k-transcode-cuda --all-targets --features cuda-runtime + - name: Run JPEG->HTJ2K transcode parity tests with executed-count floor + run: | + out=$(cargo test -p j2k-transcode-cuda --features cuda-runtime --test reversible_dwt53_parity --test dwt97_parity --test dwt97_batch_parity --test htj2k97_codeblock_parity --test jpeg_to_htj2k -- --nocapture 2>&1) + echo "$out" + passed=$(echo "$out" | sed -n 's/^test result: ok\. \([0-9]\+\) passed.*/\1/p' | awk '{s+=$1} END{print s+0}') + echo "transcode parity tests passed: ${passed:-0}" + test "${passed:-0}" -ge 7 || { echo "Expected at least 7 executed transcode parity tests, got ${passed:-0} — tests may have silently skipped"; exit 1; } + - run: cargo test -p j2k-cuda --all-targets --features cuda-profiling + # Opt-in test-only NVIDIA codec comparator. Requires a separately-installed + # library (set NVJPEG2K_LIB_DIR / NVJPEG2K_INCLUDE_DIR if it is not under + # /usr/local/cuda). Set the repo variable J2K_BENCH_SVS to an .svs on + # the runner to extract real WSI tiles; or J2K_BENCH_JPEG_DIR to a + # pre-extracted tile directory. With neither, it runs the committed pancreas + # tile corpus and requires at least 100 benchmark tiles. + - name: NVIDIA baseline comparison + if: ${{ inputs.run-nvidia-baseline }} + env: + J2K_REQUIRE_CUDA_RUNTIME: "1" + J2K_REQUIRE_CUDA_HTJ2K_STRICT: "1" + J2K_REQUIRE_NV_BASELINE_BUILD: "1" + J2K_BENCH_SVS: ${{ vars.J2K_BENCH_SVS }} + J2K_BENCH_JPEG_DIR: ${{ vars.J2K_BENCH_JPEG_DIR }} + run: | + set -e + DIR="${J2K_BENCH_JPEG_DIR:-}" + if [ -z "$DIR" ] && [ -n "${J2K_BENCH_SVS:-}" ]; then + if [ ! -f "${J2K_BENCH_SVS}" ]; then + echo "Configured J2K_BENCH_SVS does not exist: ${J2K_BENCH_SVS}" >&2 + exit 1 + fi + echo "extracting benchmark tiles from ${J2K_BENCH_SVS}" + cargo run --release --manifest-path tests/nvidia-baseline/Cargo.toml --bin svs_extract -- \ + "${J2K_BENCH_SVS}" "${RUNNER_TEMP:-/tmp}/svs_bench_tiles" \ + --limit 256 --stride 51 --quality 85 --min-tissue 0.6 + DIR="${RUNNER_TEMP:-/tmp}/svs_bench_tiles" fi - if command -v ldconfig >/dev/null 2>&1; then - ldconfig -p | grep -i nvjpeg || true + # Default to the committed pancreas tissue tiles when nothing else is set. + if [ -z "$DIR" ]; then + DIR="tests/nvidia-baseline/benchtiles/pancreas" fi - - run: cargo test -p signinum-cuda-runtime --all-targets - - run: cargo test -p signinum-jpeg-cuda --all-targets --features cuda-runtime - - run: cargo test -p signinum-j2k-cuda --all-targets --features cuda-runtime - - run: cargo bench -p signinum-jpeg-cuda --bench device_decode --features cuda-runtime --no-run - - run: cargo bench -p signinum-j2k-cuda --bench encode_stages --features cuda-runtime --no-run + export J2K_BENCH_JPEG_DIR="$DIR" + echo "benchmark JPEG dir: ${J2K_BENCH_JPEG_DIR}" + cargo test --manifest-path tests/nvidia-baseline/Cargo.toml --features nvjpeg2000 --all-targets + cargo clippy --manifest-path tests/nvidia-baseline/Cargo.toml --features nvjpeg2000 --all-targets -- -D warnings + cargo run --release --manifest-path tests/nvidia-baseline/Cargo.toml --features nvjpeg2000 --bin transcode_compare -- \ + --decomposition-levels 1 \ + --quant-scales 1.90 \ + --match-nvidia-bytes \ + --match-tolerance 0.20 \ + --min-tiles 100 \ + --warmup 3 \ + --iterations 8 \ + --json target/transcode_compare_level1.json \ + --csv target/transcode_compare_level1.csv + python3 -m json.tool target/transcode_compare_level1.json >/dev/null + python3 tests/nvidia-baseline/scripts/assert_transcode_perf.py \ + --json target/transcode_compare_level1.json \ + --label cuda-htj2k-transcode-level1-rate-match \ + --min-cuda-ht-mps "${J2K_LEVEL1_CUDA_HT_MIN_MPS:-350}" \ + --min-cuda-ht-speedup-vs-nvidia "${J2K_LEVEL1_CUDA_HT_MIN_SPEEDUP_VS_NVIDIA:-4.0}" \ + --max-byte-delta-abs 0.20 \ + --min-ht-codeblock-dispatches 1 + cargo run --release --manifest-path tests/nvidia-baseline/Cargo.toml --features nvjpeg2000 --bin transcode_compare -- \ + --decomposition-levels 2 \ + --quant-scales 1.00 \ + --subband-scales 1.00,1.00,1.00,1.00 \ + --match-nvidia-bytes \ + --match-tolerance 0.20 \ + --min-tiles 100 \ + --warmup 3 \ + --iterations 8 \ + --json target/transcode_compare_level2.json \ + --csv target/transcode_compare_level2.csv + python3 -m json.tool target/transcode_compare_level2.json >/dev/null + python3 tests/nvidia-baseline/scripts/assert_transcode_perf.py \ + --json target/transcode_compare_level2.json \ + --label cuda-htj2k-transcode-level2-rate-match \ + --min-cuda-ht-mps "${J2K_LEVEL2_CUDA_HT_MIN_MPS:-60}" \ + --min-cuda-ht-speedup-vs-nvidia "${J2K_LEVEL2_CUDA_HT_MIN_SPEEDUP_VS_NVIDIA:-1.10}" \ + --max-byte-delta-abs 0.20 \ + --min-ht-codeblock-dispatches 1 + cp target/transcode_compare_level2.json target/transcode_compare.json + cp target/transcode_compare_level2.csv target/transcode_compare.csv + cargo run --release --manifest-path tests/nvidia-baseline/Cargo.toml --features nvjpeg2000 --bin decode_compare -- \ + --jpeg-dir "${J2K_BENCH_JPEG_DIR}" \ + --warmup 2 \ + --iterations 10 \ + --min-inputs 100 \ + --max-inputs 100 \ + --json target/decode_compare.json \ + --csv target/decode_compare.csv + python3 -m json.tool target/decode_compare.json >/dev/null + - name: Upload NVIDIA baseline comparison artifacts + if: ${{ inputs.run-nvidia-baseline && always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: nvidia-baseline-comparison + path: | + target/transcode_compare.json + target/transcode_compare.csv + target/transcode_compare_level1.json + target/transcode_compare_level1.csv + target/transcode_compare_level2.json + target/transcode_compare_level2.csv + target/decode_compare.json + target/decode_compare.csv + if-no-files-found: warn + - run: cargo clippy -p j2k-cuda-runtime --all-targets -- -D warnings + - run: cargo clippy -p j2k-cuda-runtime --all-targets --features cuda-profiling -- -D warnings + - run: cargo clippy -p j2k-cuda --all-targets --features cuda-runtime -- -D warnings + - run: cargo clippy -p j2k-transcode-cuda --all-targets --features cuda-runtime -- -D warnings + - run: cargo clippy -p j2k-cuda --all-targets --features cuda-profiling -- -D warnings + - run: cargo bench -p j2k-jpeg-cuda --bench device_decode --features cuda-runtime --no-run + - run: cargo bench -p j2k-cuda --bench encode_stages --features cuda-runtime --no-run + - run: cargo bench -p j2k-cuda --bench htj2k_decode --features cuda-runtime --no-run + - run: cargo bench -p j2k-cuda --bench htj2k_encode --features cuda-runtime --no-run + - if: ${{ inputs.run-timed-benchmarks }} + env: + J2K_REQUIRE_CUDA_BENCH: "1" + J2K_GPU_BENCH_DIM: "4096" + J2K_GPU_BENCH_BATCH: "64" + J2K_GPU_BENCH_BATCH_DIM: "1024" + run: cargo bench -p j2k-jpeg-cuda --bench device_decode --features cuda-runtime -- --noplot --sample-size 10 --warm-up-time 1 --measurement-time 2 + - if: ${{ inputs.run-timed-benchmarks }} + env: + J2K_REQUIRE_CUDA_BENCH: "1" + run: cargo bench -p j2k-cuda --bench encode_stages --features cuda-runtime -- --noplot --sample-size 10 --warm-up-time 1 --measurement-time 2 - if: ${{ inputs.run-timed-benchmarks }} env: - SIGNINUM_REQUIRE_CUDA_BENCH: "1" - SIGNINUM_GPU_BENCH_DIM: "4096" - SIGNINUM_GPU_BENCH_BATCH: "64" - SIGNINUM_GPU_BENCH_BATCH_DIM: "1024" - run: cargo bench -p signinum-jpeg-cuda --bench device_decode --features cuda-runtime -- --noplot + J2K_REQUIRE_CUDA_BENCH: "1" + run: cargo bench -p j2k-cuda --bench htj2k_decode --features cuda-runtime -- --noplot --sample-size 10 --warm-up-time 1 --measurement-time 2 + - name: CUDA HTJ2K decode RCA profile + if: ${{ inputs.run-cuda-htj2k-decode-profile }} + env: + J2K_REQUIRE_CUDA_RUNTIME: "1" + J2K_REQUIRE_CUDA_HTJ2K_STRICT: "1" + J2K_REQUIRE_CUDA_BENCH: "1" + J2K_PROFILE_STAGES: summary + J2K_CUDA_TRACE: ${{ github.workspace }}/target/cuda_htj2k_decode_trace.json + run: | + set -o pipefail + mkdir -p target + samply_status_file=target/cuda_htj2k_decode_samply_status.txt + bench_cmd=( + cargo bench -p j2k-cuda --bench htj2k_decode --features cuda-runtime,cuda-profiling -- + --noplot --sample-size 10 --warm-up-time 1 --measurement-time 2 + ) + if ! command -v samply >/dev/null 2>&1; then + RUSTFLAGS="" cargo install samply --version 0.13.1 --locked + fi + samply --version + run_samply=1 + if [ -r /proc/sys/kernel/perf_event_paranoid ]; then + current_perf_event_paranoid="$(cat /proc/sys/kernel/perf_event_paranoid)" + echo "perf_event_paranoid=${current_perf_event_paranoid}" + if [ "${current_perf_event_paranoid}" -gt 1 ] 2>/dev/null; then + perf_lowered=0 + if [ -w /proc/sys/kernel/perf_event_paranoid ]; then + if echo 1 > /proc/sys/kernel/perf_event_paranoid; then + perf_lowered=1 + fi + fi + if [ "${perf_lowered}" -ne 1 ] && command -v sudo >/dev/null 2>&1; then + if echo 1 | sudo -n tee /proc/sys/kernel/perf_event_paranoid >/dev/null; then + perf_lowered=1 + else + echo "passwordless sudo could not lower perf_event_paranoid for samply" + fi + fi + if [ "${perf_lowered}" -eq 1 ]; then + echo "perf_event_paranoid=$(cat /proc/sys/kernel/perf_event_paranoid)" + else + run_samply=0 + { + echo "samply_status=blocked" + echo "perf_event_paranoid=${current_perf_event_paranoid}" + echo "reason=passwordless sudo unavailable and /proc/sys/kernel/perf_event_paranoid is not writable" + echo "fallback=cargo bench with J2K_PROFILE_STAGES and J2K_CUDA_TRACE" + } | tee "${samply_status_file}" + fi + fi + fi + if [ "${run_samply}" -eq 1 ]; then + echo "samply_status=attempting" | tee "${samply_status_file}" + samply record --save-only -o target/cuda_htj2k_decode_samply.json.gz -- \ + "${bench_cmd[@]}" \ + 2>&1 | tee target/cuda_htj2k_decode_profile.log + echo "samply_status=recorded" > "${samply_status_file}" + else + "${bench_cmd[@]}" 2>&1 | tee target/cuda_htj2k_decode_profile.log + fi + - name: Upload CUDA HTJ2K decode RCA profile artifacts + if: ${{ inputs.run-cuda-htj2k-decode-profile && always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: cuda-htj2k-decode-rca-profile + path: | + target/cuda_htj2k_decode_profile.log + target/cuda_htj2k_decode_trace.json + target/cuda_htj2k_decode_samply.json.gz + target/cuda_htj2k_decode_samply_status.txt + target/criterion + if-no-files-found: warn - if: ${{ inputs.run-timed-benchmarks }} env: - SIGNINUM_REQUIRE_CUDA_BENCH: "1" - run: cargo bench -p signinum-j2k-cuda --bench encode_stages --features cuda-runtime -- --noplot + J2K_REQUIRE_CUDA_BENCH: "1" + run: cargo bench -p j2k-cuda --bench htj2k_encode --features cuda-runtime -- --noplot --sample-size 10 --warm-up-time 1 --measurement-time 2 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 25aa619f..d8513393 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -17,129 +17,236 @@ permissions: env: CARGO_TERM_COLOR: always - CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} CRATES_IO_INDEX_SETTLE_SECONDS: "130" DRY_RUN_ONLY: ${{ github.event_name == 'workflow_dispatch' && inputs.dry-run-only }} jobs: - publish-signinum-core: + preflight: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum-core - - publish-signinum-j2k-native: - needs: - - publish-signinum-profile + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: cargo xtask release-integrity + + publish-j2k-core: + needs: [preflight] runs-on: ubuntu-latest + environment: crates-io-publish steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum-j2k-native - - publish-signinum-cuda-runtime: - needs: - - publish-signinum-core + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-core + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-profile: + needs: [publish-j2k-core] runs-on: ubuntu-latest + environment: crates-io-publish steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum-cuda-runtime - - publish-signinum-profile: - needs: - - publish-signinum-cuda-runtime + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-profile + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-types: + needs: [publish-j2k-profile] runs-on: ubuntu-latest + environment: crates-io-publish steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum-profile - - publish-signinum-jpeg: - needs: - - publish-signinum-j2k-native + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-types + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-cuda-runtime: + needs: [publish-j2k-types] runs-on: ubuntu-latest + environment: crates-io-publish steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum-jpeg - - publish-signinum-tilecodec: - needs: - - publish-signinum-jpeg - - publish-signinum-j2k-native + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-cuda-runtime + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-metal-support: + needs: [publish-j2k-cuda-runtime] runs-on: ubuntu-latest + environment: crates-io-publish steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum-tilecodec - - publish-signinum-j2k: - needs: - - publish-signinum-tilecodec + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-metal-support + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-native: + needs: [publish-j2k-metal-support] runs-on: ubuntu-latest + environment: crates-io-publish steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum-j2k - - publish-signinum-jpeg-metal: - needs: - - publish-signinum-j2k + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-native + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-jpeg: + needs: [publish-j2k-native] runs-on: ubuntu-latest + environment: crates-io-publish steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum-jpeg-metal - - publish-signinum-j2k-metal: - needs: - - publish-signinum-jpeg-metal + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-jpeg + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-tilecodec: + needs: [publish-j2k-jpeg] runs-on: ubuntu-latest + environment: crates-io-publish steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum-j2k-metal - - publish-signinum-jpeg-cuda: - needs: - - publish-signinum-cuda-runtime - - publish-signinum-j2k-metal + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-tilecodec + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k: + needs: [publish-j2k-tilecodec] runs-on: ubuntu-latest + environment: crates-io-publish steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum-jpeg-cuda - - publish-signinum-j2k-cuda: - needs: - - publish-signinum-jpeg-cuda + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-transcode: + needs: [publish-j2k] runs-on: ubuntu-latest + environment: crates-io-publish steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum-j2k-cuda - - publish-signinum-cli: - needs: - - publish-signinum-j2k-cuda + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-transcode + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-transcode-cuda: + needs: [publish-j2k-transcode] runs-on: ubuntu-latest + environment: crates-io-publish steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum-cli - - publish-signinum: - needs: - - publish-signinum-tilecodec - - publish-signinum-jpeg-metal - - publish-signinum-j2k-metal - - publish-signinum-jpeg-cuda - - publish-signinum-j2k-cuda - - publish-signinum-cli + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-transcode-cuda + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-jpeg-metal: + needs: [publish-j2k-transcode-cuda] + runs-on: ubuntu-latest + environment: crates-io-publish + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-jpeg-metal + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-metal: + needs: [publish-j2k-jpeg-metal] + runs-on: ubuntu-latest + environment: crates-io-publish + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-metal + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-transcode-metal: + needs: [publish-j2k-metal] + runs-on: ubuntu-latest + environment: crates-io-publish + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-transcode-metal + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-jpeg-cuda: + needs: [publish-j2k-transcode-metal] + runs-on: ubuntu-latest + environment: crates-io-publish + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-jpeg-cuda + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-cuda: + needs: [publish-j2k-jpeg-cuda] + runs-on: ubuntu-latest + environment: crates-io-publish + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-cuda + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} + + publish-j2k-cli: + needs: [publish-j2k-cuda] runs-on: ubuntu-latest + environment: crates-io-publish steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: scripts/publish-crate.sh signinum + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - run: scripts/publish-crate.sh j2k-cli + env: + CRATES_IO_API_TOKEN: ${{ secrets.CRATES_IO_API_TOKEN }} diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml index e504a31f..b747a0f8 100644 --- a/.github/workflows/secret-scan.yml +++ b/.github/workflows/secret-scan.yml @@ -13,14 +13,19 @@ jobs: gitleaks: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd with: fetch-depth: 0 - name: Install Gitleaks env: GITLEAKS_VERSION: "8.30.1" + GITLEAKS_SHA256: "551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb" run: | - curl -sSfL "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" \ - | tar -xz gitleaks + archive="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + curl --proto '=https' --tlsv1.2 --fail --location --show-error \ + --output "${archive}" \ + "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/${archive}" + echo "${GITLEAKS_SHA256} ${archive}" | sha256sum -c - + tar -xzf "${archive}" gitleaks - name: Scan git history run: ./gitleaks detect --source . --redact --verbose diff --git a/.gitignore b/.gitignore index f1bf037b..4ade221b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ target/ +docs/superpowers/ +.codewhale/ +lcov.info **/*.rs.bk Cargo.lock.bak .vscode/ diff --git a/CHANGELOG.md b/CHANGELOG.md index e6c96b2b..9d8636e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,103 +1,92 @@ # Changelog -All notable changes to this project will be documented in this file. The -format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and this project adheres to [Semantic Versioning](https://semver.org/). - -## [0.4.2] - 2026-05-15 - -### Changed - -- Refactored shared core, profiling, JPEG adapter, and J2K adapter internals - toward focused crate-owned contracts without intended behavior changes. -- Split CUDA adapter implementations into focused modules while preserving - their public exports. -- Moved repeated benchmark and reference fixture generators into the dev-only - `signinum-test-support` crate. - -## [0.4.1] - 2026-05-12 - -### Fixed - -- Removed the unsupported Intel macOS CI gate from the public release matrix. -- Prevented CI tests from executing benchmark binaries as tests. -- Stabilized hosted macOS Metal CI by keeping J2K Metal runtime validation on - self-hosted GPU runners. - -## [0.4.0] - 2026-05-12 - -### Changed - -- Refreshed crates.io package metadata for the transfer to the - `frames-sg/signinum` repository. - -## [1.0.3] - 2026-05-06 - -### Changed - -- Updated the `signinum` facade Metal feature to depend on - `signinum-jpeg-metal` 0.2.2 for resident fast 4:4:4 JPEG Metal decode - outputs. - -## [0.2.2] - 2026-05-06 - -### Fixed - -- Marked fast 4:4:4 Metal JPEG decode outputs as Metal-resident instead of - CPU-staged Metal uploads, allowing strict device-decode consumers to use the - resident buffer path end to end. - -## [1.0.0] - 2026-05-01 - -CPU-first 1.0 release posture. - -### Changed - -- Promoted `signinum-core`, `signinum-jpeg`, `signinum-j2k`, `signinum-tilecodec`, - and `signinum-cli` to the stable CPU-first 1.0 release set. -- Kept `signinum-j2k-native` as a published pre-1.0 implementation dependency - for `signinum-j2k`. -- Excluded Metal, CUDA, and comparator crates from the 1.0 publish workflow. -- Clarified that CUDA crates can use `cuda-runtime` to return CUDA device memory - surfaces, with `signinum-jpeg-cuda` using nvJPEG for full-frame RGB8 JPEG - decode when the CUDA driver and `libnvjpeg` are available. NVIDIA performance - claims require recorded self-hosted GPU benchmark evidence. - -## [0.1.0] - 2026-04-25 - -Initial public-source checkpoint. The workspace remains pre-1.0 while the -JPEG 2000 / HTJ2K ROI and GPU adapter APIs settle. - -### Added - -- `signinum-core` shared trait/type crate: - - `ImageDecode`, `ImageDecodeRows`, `TileBatchDecode`, `TileDecompress` - - `PixelFormat`, `Downscale`, `Info`, `Rect`, `DecodeOutcome` - - `ScratchPool` and `DecoderContext` contracts -- `signinum-jpeg` as the WSI-oriented JPEG implementation with: - - borrowed parse/decode surfaces - - row-streaming decode - - region and scaled decode - - tile-batch/context/scratch reuse - - external-corpus and parity coverage -- `signinum-j2k` with: - - JP2 / raw codestream inspect - - full-frame, region, scaled, row-streaming, and tile-batch decode - - HTJ2K coverage - - OpenJPEG differential tests and compare bench -- `signinum-tilecodec` with: - - `DeflateCodec` - - `ZstdCodec` - - `LzwCodec` - - `UncompressedCodec` - - typed scratch pools and compare bench coverage -- `signinum-cli` inspect dispatch for JPEG and JPEG 2000 inputs -- workspace-level CI coverage for tests, clippy, bench build, fuzz-target - build, and `cargo deny` - -### Changed - -- Workspace version promoted to `0.1.0` -- Benchmark documentation now covers JPEG, JPEG 2000, and tile decompression -- Top-level README now documents the full pathology codec stack instead of the - original JPEG-only scope +This changelog tracks the current staged release line. Historical phase notes +and stale roadmap entries have been removed from the public documentation set. + +## [0.6.0] + +- Stages the `j2k` facade release. +- Keeps CPU decode as the portable correctness baseline. +- Treats Runtime backend selection defaults to `Auto` as the public backend + policy. +- Adds resident Metal and CUDA device memory surfaces for supported adapter + paths through cuda-runtime integration. +- Uses J2K-owned CUDA kernels for supported CUDA codec stages. +- Requires recorded benchmark evidence before NVIDIA performance claims. +- Consolidates shared J2K encode-stage, CUDA submit, Metal runtime, tilecodec, + JPEG output, and test-support helpers. + +### Breaking API Changes + +- Collapses the 26 `j2k-metal` lossless encode/submit permutations + (`{encode,submit}_lossless_from_{padded_,}metal_buffer{,s}` x + to-metal/with-report/with-config/to-metal-batch) into three request-based + entry points: `submit_lossless_batch` (host codestream bytes), + `submit_lossless_batch_to_metal` (Metal-backed codestreams with batch + stats), and `encode_lossless_batch_with_report` (host-byte timing + reports), all taking the new `MetalLosslessEncodeBatchRequest` (`tiles` + + now-public `MetalEncodeInputStaging` + `MetalLosslessEncodeConfig`). The + single-tile submit wrapper `SubmittedJ2kLosslessMetalEncode` is removed + (submit a one-tile batch and take the first result). Single-tile + `_with_report` callers now route through the batch report entry, which + may use the resident batch path where the removed wrapper always used + the per-tile staged pipeline; the report's stage-residency fields can + differ accordingly. +- Collapses the 24 `j2k-jpeg-metal` `Codec::decode_rgb8_*_with_session` + batch permutations (full/scaled/region-scaled x bytes/decoders x + reusable/resizable buffer/textures) into two request-based entry points: + `decode_rgb8_batch_into_buffer_with_session` and + `decode_rgb8_batch_into_textures_with_session`, taking the new + `Rgb8MetalBatchRequest` (`Rgb8MetalBatchSource` + `Rgb8MetalBatchOp`) and + `MetalBufferBatchTarget`/`MetalTextureBatchTarget` enums. The three hottest + permutations remain as convenience wrappers with unchanged signatures: + `decode_rgb8_decoder_batch_into_resizable_metal_textures_with_session` and + the two region-scaled resizable forms. Unsupported-batch error reasons no + longer distinguish byte vs decoder sources. +- Introduces the `j2k-types` contract crate: the 49 encode-stage + job/output/report types shared by `j2k` and `j2k-native` + are defined once there (both crates re-export them at their existing + paths), and the `j2k` adapter's `*_from_native`/`*_to_native` + encode-stage converter functions are removed since both sides now use the + same types. +- Widens transcode accelerator errors: `DctToWaveletStageAccelerator` methods + now return `TranscodeStageError` (`Unsupported`/`Backend`/`DeviceUnavailable`) + instead of `&'static str`, `JpegToHtj2kError::Accelerator` carries the new + type, and the Metal/CUDA transcode error `as_static_str` funnels are removed + so backend failures keep their diagnostic detail. +- Renames backend capability detection to compile-time defaults and makes + facade Metal/CUDA gating symmetric with those compile-time features. +- Renames the JPEG fast packet adapter surface from Metal-specific names to + backend-neutral `JpegFast*` packet/table/error names. +- Trims and documents the J2K adapter surface, removing legacy preference + aliases before the facade release. +- Replaces broad facade glob reexports with explicit export lists and adds the + missing root `TileBatchDecodeDevice` and `TileBatchDecodeSubmit` traits. +- Narrows CUDA runtime root exports to explicit public modules and types. +- Makes `ProfileSummary` drop emission explicit; cloned summaries no longer + inherit stderr side effects. +- Makes JPEG sampling-factor construction fallible instead of panicking on + caller input, and adds explicit max-byte JPEG output-buffer constructors for + callers that need to override the default allocation cap. + +### Maturity And Fixes + +- Fixes confirmed stale-cache, malformed-input, tile-grid, GPU validator, FFI + cleanup, unsafe-deposit, and test-helper drift findings from the release + audit. +- Enforces shared host allocation caps for codec-owned output scratch, aligns + lightweight J2K inspect SIZ validation with full decode, and caps bounded J2K + row-decode stripes by bytes as well as rows. +- Stops default CUDA builds from probing PATH `nvcc`; strict GPU validation now + requires an explicit absolute `NVCC`. +- Strengthens unsafe, fuzz, Miri, and dependency-advisory governance in CI. +- Moves repository policy checks into `cargo xtask repo-lint` and pins public + API, release-integrity, environment-variable, workflow, and packaging + invariants there. +- Documents supported `J2K_*` environment variables and removes the + experiment-only JPEG Metal fast420 split selector from the runtime surface. +- Routes env-gated Metal timing output through `j2k-profile`. +- Adds stricter J2K component-plane validation before output writes and removes + stale generated-table dead-code suppression. +- Refreshes the stable public API inventory after facade and profile surface + changes. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 48e18d42..d5240886 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,8 +14,8 @@ cargo doc --workspace --no-deps cargo clippy --workspace --all-targets -- -D warnings ``` -Comparator benchmarks may need optional system libraries. See -`docs/bench.md` for setup and skip behavior. +Comparator benchmarks may need optional system libraries. The workspace README +defines benchmark publication and no-silent-skip behavior. ## Pull Requests @@ -38,4 +38,4 @@ surface behavior should update: - README quick-start or examples when user-facing behavior changes - API docs for affected public items - integration tests covering caller-visible behavior -- `docs/bench.md` when benchmark methodology changes +- the README benchmark policy when benchmark methodology changes diff --git a/Cargo.lock b/Cargo.lock index 5f2543ff..98ef1aff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "anes" version = "0.1.6" @@ -37,9 +46,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "bit-set" @@ -58,15 +67,9 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" -version = "1.3.2" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "block" @@ -74,12 +77,6 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" -[[package]] -name = "bytemuck" -version = "1.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" - [[package]] name = "cast" version = "0.3.0" @@ -88,9 +85,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.61" +version = "1.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" dependencies = [ "find-msvc-tools", "jobserver", @@ -158,9 +155,9 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "core-foundation" -version = "0.9.4" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ "core-foundation-sys", "libc", @@ -174,11 +171,11 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core-graphics-types" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" +checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" dependencies = [ - "bitflags 1.3.2", + "bitflags", "core-foundation", "libc", ] @@ -194,23 +191,22 @@ dependencies = [ [[package]] name = "criterion" -version = "0.5.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" dependencies = [ + "alloca", "anes", "cast", "ciborium", "clap", "criterion-plot", - "is-terminal", "itertools", "num-traits", - "once_cell", "oorandom", + "page_size", "regex", "serde", - "serde_derive", "serde_json", "tinytemplate", "walkdir", @@ -218,9 +214,9 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.5.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", "itertools", @@ -259,9 +255,9 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "equivalent" @@ -287,12 +283,9 @@ checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "fearless_simd" -version = "0.3.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb2907d1f08b2b316b9223ced5b0e89d87028ba8deae9764741dba8ff7f3903" -dependencies = [ - "bytemuck", -] +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" [[package]] name = "find-msvc-tools" @@ -397,9 +390,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -407,12 +400,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - [[package]] name = "id-arena" version = "2.3.0" @@ -426,27 +413,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] -[[package]] -name = "is-terminal" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", -] - [[package]] name = "itertools" -version = "0.10.5" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ "either", ] @@ -457,6 +433,220 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "j2k" +version = "0.6.0" +dependencies = [ + "criterion", + "j2k-core", + "j2k-native", + "j2k-test-support", + "j2k-types", + "proptest", + "thiserror", +] + +[[package]] +name = "j2k-cli" +version = "0.6.0" +dependencies = [ + "j2k", + "j2k-jpeg", + "j2k-test-support", +] + +[[package]] +name = "j2k-compare" +version = "0.2.0" +dependencies = [ + "cc", + "j2k", + "j2k-core", + "j2k-test-support", + "openjpeg-sys", +] + +[[package]] +name = "j2k-core" +version = "0.6.0" +dependencies = [ + "serde_json", + "thiserror", +] + +[[package]] +name = "j2k-cuda" +version = "0.6.0" +dependencies = [ + "criterion", + "j2k", + "j2k-core", + "j2k-cuda-runtime", + "j2k-native", + "j2k-profile", + "j2k-test-support", + "thiserror", +] + +[[package]] +name = "j2k-cuda-runtime" +version = "0.6.0" +dependencies = [ + "j2k-core", + "libloading", + "thiserror", +] + +[[package]] +name = "j2k-jpeg" +version = "0.6.0" +dependencies = [ + "criterion", + "j2k-core", + "j2k-profile", + "j2k-test-support", + "jpeg-decoder", + "memchr", + "proptest", + "rayon", + "thiserror", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "j2k-jpeg-cuda" +version = "0.6.0" +dependencies = [ + "criterion", + "j2k-core", + "j2k-cuda-runtime", + "j2k-jpeg", + "j2k-profile", + "j2k-test-support", + "thiserror", +] + +[[package]] +name = "j2k-jpeg-metal" +version = "0.6.0" +dependencies = [ + "criterion", + "j2k-core", + "j2k-jpeg", + "j2k-metal-support", + "j2k-profile", + "j2k-test-support", + "jpeg-decoder", + "jpeg-encoder", + "metal", + "thiserror", +] + +[[package]] +name = "j2k-metal" +version = "0.6.0" +dependencies = [ + "cc", + "j2k", + "j2k-core", + "j2k-metal-support", + "j2k-native", + "j2k-profile", + "j2k-test-support", + "libc", + "metal", + "rayon", + "thiserror", +] + +[[package]] +name = "j2k-metal-support" +version = "0.6.0" +dependencies = [ + "j2k-core", + "metal", +] + +[[package]] +name = "j2k-native" +version = "0.6.0" +dependencies = [ + "criterion", + "fearless_simd", + "j2k-profile", + "j2k-types", + "libm", + "log", + "rayon", +] + +[[package]] +name = "j2k-profile" +version = "0.6.0" + +[[package]] +name = "j2k-test-support" +version = "0.6.0" +dependencies = [ + "j2k-native", +] + +[[package]] +name = "j2k-tilecodec" +version = "0.6.0" +dependencies = [ + "criterion", + "flate2", + "j2k-core", + "thiserror", + "weezl", + "zstd", +] + +[[package]] +name = "j2k-transcode" +version = "0.6.0" +dependencies = [ + "criterion", + "j2k", + "j2k-jpeg", + "j2k-native", + "j2k-test-support", + "proptest", + "rayon", +] + +[[package]] +name = "j2k-transcode-cuda" +version = "0.6.0" +dependencies = [ + "j2k-cuda-runtime", + "j2k-jpeg", + "j2k-native", + "j2k-test-support", + "j2k-transcode", +] + +[[package]] +name = "j2k-transcode-metal" +version = "0.6.0" +dependencies = [ + "criterion", + "j2k-core", + "j2k-jpeg", + "j2k-metal-support", + "j2k-native", + "j2k-test-support", + "j2k-transcode", + "metal", + "rayon", +] + +[[package]] +name = "j2k-types" +version = "0.6.0" + [[package]] name = "jobserver" version = "0.1.34" @@ -493,9 +683,9 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" -version = "0.8.9" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" dependencies = [ "cfg-if", "windows-link", @@ -509,9 +699,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libz-sys" -version = "1.1.28" +version = "1.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3a226e576f50782b3305c5ccf458698f92798987f551c6a02efe8276721e22" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" dependencies = [ "cc", "pkg-config", @@ -526,9 +716,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "malloc_buf" @@ -541,17 +731,17 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "metal" -version = "0.31.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f569fb946490b5743ad69813cb19629130ce9374034abe31614a36402d18f99e" +checksum = "c7047791b5bc903b8cd963014b355f71dc9864a9a0b727057676c1dcae5cbc15" dependencies = [ - "bitflags 2.11.1", + "bitflags", "block", "core-graphics-types", "foreign-types", @@ -610,6 +800,16 @@ dependencies = [ "libc", ] +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "paste" version = "1.0.15" @@ -658,7 +858,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.11.1", + "bitflags", "num-traits", "rand", "rand_chacha", @@ -756,9 +956,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -779,9 +979,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "rustix" @@ -789,7 +989,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.1", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -808,6 +1008,12 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "same-file" version = "1.0.6" @@ -855,9 +1061,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -867,182 +1073,23 @@ dependencies = [ ] [[package]] -name = "shlex" -version = "1.3.0" +name = "serde_yaml_ng" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signinum" -version = "0.4.2" -dependencies = [ - "criterion", - "signinum-core", - "signinum-j2k", - "signinum-j2k-cuda", - "signinum-j2k-metal", - "signinum-j2k-native", - "signinum-jpeg", - "signinum-jpeg-cuda", - "signinum-jpeg-metal", - "signinum-test-support", - "signinum-tilecodec", -] - -[[package]] -name = "signinum-cli" -version = "0.4.2" -dependencies = [ - "signinum-j2k", - "signinum-jpeg", -] - -[[package]] -name = "signinum-core" -version = "0.4.2" -dependencies = [ - "serde_json", - "thiserror", -] - -[[package]] -name = "signinum-cuda-runtime" -version = "0.4.2" -dependencies = [ - "libloading", - "thiserror", -] - -[[package]] -name = "signinum-j2k" -version = "0.4.2" -dependencies = [ - "criterion", - "proptest", - "signinum-core", - "signinum-j2k-native", - "signinum-test-support", - "thiserror", -] - -[[package]] -name = "signinum-j2k-compare" -version = "0.2.0" -dependencies = [ - "cc", - "openjpeg-sys", - "signinum-core", - "signinum-j2k", - "signinum-test-support", -] - -[[package]] -name = "signinum-j2k-cuda" -version = "0.4.2" -dependencies = [ - "criterion", - "signinum-core", - "signinum-cuda-runtime", - "signinum-j2k", - "signinum-j2k-native", - "signinum-profile", - "thiserror", -] - -[[package]] -name = "signinum-j2k-metal" -version = "0.4.2" -dependencies = [ - "criterion", - "libc", - "metal", - "rayon", - "signinum-core", - "signinum-j2k", - "signinum-j2k-compare", - "signinum-j2k-native", - "signinum-profile", - "signinum-test-support", - "thiserror", -] - -[[package]] -name = "signinum-j2k-native" -version = "0.4.2" -dependencies = [ - "criterion", - "fearless_simd", - "libm", - "log", - "rayon", - "signinum-profile", -] - -[[package]] -name = "signinum-jpeg" -version = "0.4.2" -dependencies = [ - "criterion", - "jpeg-decoder", - "memchr", - "proptest", - "rayon", - "signinum-core", - "signinum-profile", - "signinum-test-support", - "thiserror", - "zune-core", - "zune-jpeg", -] - -[[package]] -name = "signinum-jpeg-cuda" -version = "0.4.2" +checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" dependencies = [ - "criterion", - "jpeg-encoder", - "signinum-core", - "signinum-cuda-runtime", - "signinum-jpeg", - "signinum-profile", - "signinum-test-support", - "thiserror", -] - -[[package]] -name = "signinum-jpeg-metal" -version = "0.4.2" -dependencies = [ - "criterion", - "jpeg-decoder", - "jpeg-encoder", - "metal", - "signinum-core", - "signinum-jpeg", - "signinum-profile", - "signinum-test-support", - "thiserror", + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", ] [[package]] -name = "signinum-profile" -version = "0.4.2" - -[[package]] -name = "signinum-test-support" -version = "0.4.2" - -[[package]] -name = "signinum-tilecodec" -version = "0.4.2" -dependencies = [ - "criterion", - "flate2", - "signinum-core", - "thiserror", - "weezl", - "zstd", -] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "simd-adler32" @@ -1122,6 +1169,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "vcpkg" version = "0.2.15" @@ -1149,9 +1202,9 @@ dependencies = [ [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen 0.57.1", ] @@ -1193,7 +1246,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.1", + "bitflags", "hashbrown 0.15.5", "indexmap", "semver", @@ -1201,9 +1254,25 @@ dependencies = [ [[package]] name = "weezl" -version = "0.1.12" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2" + +[[package]] +name = "winapi" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" @@ -1214,6 +1283,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" @@ -1293,7 +1368,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.1", + "bitflags", "indexmap", "log", "serde", @@ -1326,21 +1401,25 @@ dependencies = [ [[package]] name = "xtask" version = "0.0.0" +dependencies = [ + "serde_json", + "serde_yaml_ng", +] [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index d87a1843..54930499 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,42 +1,68 @@ [workspace] resolver = "2" members = [ - "crates/signinum", - "crates/signinum-core", - "crates/signinum-cuda-runtime", - "crates/signinum-profile", - "crates/signinum-j2k-compare", - "crates/signinum-j2k-native", - "crates/signinum-j2k", - "crates/signinum-j2k-cuda", - "crates/signinum-j2k-metal", - "crates/signinum-jpeg", - "crates/signinum-jpeg-cuda", - "crates/signinum-jpeg-metal", - "crates/signinum-test-support", - "crates/signinum-tilecodec", - "crates/signinum-cli", + "crates/j2k", + "crates/j2k-core", + "crates/j2k-compare", + "crates/j2k-cuda", + "crates/j2k-cuda-runtime", + "crates/j2k-jpeg", + "crates/j2k-jpeg-cuda", + "crates/j2k-jpeg-metal", + "crates/j2k-metal", + "crates/j2k-metal-support", + "crates/j2k-native", + "crates/j2k-profile", + "crates/j2k-test-support", + "crates/j2k-tilecodec", + "crates/j2k-transcode", + "crates/j2k-transcode-cuda", + "crates/j2k-transcode-metal", + "crates/j2k-types", + "crates/j2k-cli", "xtask", ] [workspace.package] -version = "0.4.2" +version = "0.6.0" edition = "2021" -rust-version = "1.94" +rust-version = "1.96" license = "Apache-2.0" -repository = "https://github.com/frames-sg/signinum" -homepage = "https://github.com/frames-sg/signinum" -keywords = ["pathology", "wsi", "jpeg", "jpeg2000", "tiff"] +repository = "https://github.com/frames-sg/j2k" +homepage = "https://github.com/frames-sg/j2k" +keywords = ["jpeg", "jpeg2000", "htj2k", "gpu", "transcode"] categories = ["multimedia::images"] [workspace.dependencies] thiserror = { version = "2", default-features = false } proptest = { version = "1" } -criterion = { version = "0.5", default-features = false } +criterion = { version = "0.8", default-features = false } flate2 = { version = "1.1", default-features = false, features = ["zlib"] } zstd = { version = "0.13" } -weezl = { version = "0.1" } -libloading = "0.8" +weezl = { version = "0.2" } +libloading = "0.9" +metal = "0.33" +rayon = "1" +cc = "1.2" +libc = "0.2" +log = "0.4" +memchr = { version = "2.8", default-features = false } + +# Shared strict-lint policy for safe crates. Crates with genuinely different +# needs (SIMD/FFI crates that use `unsafe`, crates with documented temporary +# allow lists) keep their own [lints] tables until later cleanup phases +# narrow them. +[workspace.lints.rust] +unsafe_code = "forbid" +unsafe_op_in_unsafe_fn = "deny" +unreachable_pub = "warn" + +[workspace.lints.clippy] +pedantic = { level = "warn", priority = -1 } +module_name_repetitions = "allow" +must_use_candidate = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" [profile.release] lto = "fat" diff --git a/NOTICES.md b/NOTICES.md index d1762aab..cf3ab857 100644 --- a/NOTICES.md +++ b/NOTICES.md @@ -1,8 +1,6 @@ # Notices -`signinum` is an independent implementation. No source code is derived from -any other project. The following prior work was used as **reference material** -during design and verification: +`j2k` is an independent implementation. The following prior work was used as **reference material** during design and verification, or is included as small test/data assets where noted: - **ITU-T Recommendation T.81 (09/92)** — *Information technology — Digital compression and coding of continuous-tone still images — Requirements and @@ -15,4 +13,21 @@ during design and verification: `corpus/conformance/` are generated by linking against libjpeg-turbo on the maintainer's machine and are committed to the repo; the pinned version is recorded in `corpus/conformance/manifest.json`. No libjpeg-turbo source - code is incorporated into `signinum`. + code is incorporated into `j2k`. + +- **OpenHTJ2K conformance fixtures** — Tiny HTONLY codestream fixtures in + `crates/j2k-native/fixtures/htj2k/` are copied from OpenHTJ2K + commit `ffe5acf9f1eedb87c36c3fd2134fdc1ddea5e75f`. The retained upstream + BSD 3-Clause license is in + `crates/j2k-native/fixtures/htj2k/LICENSE.OpenHTJ2K`. + +- **Compact ICC Profiles** + — ICC profile assets in `crates/j2k-native/assets/` are used for + color-management tests. The profiles are available under CC0 1.0 Universal; + the retained license text is in + `crates/j2k-native/assets/LICENSE.txt`. + +- **OpenJPH** — HTJ2K lookup tables in + `crates/j2k-native/src/j2c/ht_tables.rs` and + `crates/j2k-native/src/j2c/ht_encode_tables.rs` are generated from + OpenJPH source tables. They are table data, not linked OpenJPH source files. diff --git a/README.md b/README.md index 9a8a8728..2e111261 100644 --- a/README.md +++ b/README.md @@ -1,295 +1,150 @@ -# signinum - -Rust codec primitives for pathology and whole-slide imaging workloads. - -`signinum` is for compressed tile payloads: JPEG, JPEG 2000 / HTJ2K, and -container tile compression such as Deflate, Zstd, LZW, and uncompressed tiles. -It is not a whole-slide container reader, pyramid manager, cache, or DICOM -writer. Use [`statumen`](https://github.com/frames-sg/statumen) for slide -container parsing and [`wsi-dicom`](https://github.com/frames-sg/wsi-dicom) -for DICOM VL Whole Slide Microscopy export. - -The current public-source target is the `signinum` facade release. -Runtime backend selection defaults to `Auto`: CPU decode is always available, -and compiled Metal or CUDA adapters are used only for supported workloads. CUDA -adapters expose CUDA device memory through `cuda-runtime` when a CUDA driver is -available. JPEG full-frame RGB8 CUDA requests can use nvJPEG; -NVIDIA performance claims require self-hosted GPU benchmark evidence. +# J2K + +J2K is a Rust image-codec workspace for JPEG 2000 / HTJ2K decode, encode, +GPU acceleration, and JPEG-to-J2K/HTJ2K transcoding. The public crate release +centers on `j2k`, with lower-level crates for native codec internals, device +adapters, JPEG input, and transcode pipelines. + +The APIs are general codec APIs. Whole-slide imaging and DICOM tile workloads +are the main public examples and benchmark fixtures because they stress +large tiled images, strict color handling, and high-throughput GPU paths, but +the decoder, encoder, and transcode crates are not WSI-only. + +Runtime backend selection defaults to `Auto`: CPU remains the portable baseline, +and Metal or CUDA paths are selected only for supported shapes with validation +and benchmark evidence. Explicit device requests are strict. Unsupported device +shapes return errors instead of silently changing the requested backend. +`Auto` is an optimization policy, not a promise to use a device whenever one is +available. + +CUDA paths use J2K-owned CUDA kernels and `cuda-runtime` support for CUDA +device memory surfaces where implemented. NVIDIA performance claims require +self-hosted benchmark evidence; hosted CI is not treated as NVIDIA performance +evidence. ## Which crate should I use? -| Task | Use | Install | -|------|-----|---------| -| Start a new application with one import surface | `signinum` facade | `cargo add signinum` | -| JPEG tile inspect/decode | `signinum-jpeg` or `signinum::jpeg` | `cargo add signinum-jpeg` | -| JPEG 2000 / HTJ2K inspect, decode, and lossless encode | `signinum-j2k` or `signinum::j2k` | `cargo add signinum-j2k` | -| TIFF/DICOM-style tile decompression | `signinum-tilecodec` or `signinum::tilecodec` | `cargo add signinum-tilecodec` | -| Shared pixel, backend, scratch, row, and passthrough traits | `signinum-core` | `cargo add signinum-core` | -| CLI header inspection | `signinum-cli` | `cargo install signinum-cli` | -| Apple Metal device-output surfaces | `signinum-jpeg-metal`, `signinum-j2k-metal`, or the facade `metal` feature | `cargo add signinum-jpeg-metal` | -| CUDA device-memory output | `signinum-jpeg-cuda`, `signinum-j2k-cuda`, plus the adapter `cuda-runtime` feature | `cargo add signinum-jpeg-cuda --features cuda-runtime` or `cargo add signinum-j2k-cuda --features cuda-runtime` | - -Most application code should start with the facade: - -```toml -[dependencies] -signinum = "0.4.2" -``` - -The facade exposes: - -- `signinum::jpeg` for JPEG tiles -- `signinum::j2k` for JPEG 2000 / HTJ2K -- `signinum::tilecodec` for container tile decompression -- shared `signinum-core` contracts at the crate root and under - `signinum::core` - -The default facade build includes portable CPU codecs and the Metal adapter. -Use `--no-default-features` when you want only the CPU-portable facade, or -`--features cuda` / `--features gpu` when the facade should expose CUDA adapter -modules. CUDA runtime allocation, copies, kernels, and nvJPEG loading are -enabled on the adapter crates with their `cuda-runtime` feature. - -## Quick start - -The snippets below assume they are inside a function that returns `Result`. +Use `cargo add j2k` for JPEG 2000 / HTJ2K application code. Lower-level +`j2k-*` crates remain public implementation and integration crates. -Inspect JPEG and JPEG 2000 headers without decoding pixels: +Use lower-level crates only when you need a specific integration point: -```rust -use signinum::jpeg::Decoder as JpegDecoder; -use signinum::j2k::J2kDecoder; +| Need | Crate | +| --- | --- | +| JPEG 2000 / HTJ2K inspect, decode, encode, and recode | `j2k` | +| Shared traits and backend types | `j2k-core` | +| JPEG inspect/decode and fixture/fallback encode | `j2k-jpeg` | +| Native JPEG 2000 and HTJ2K codec engine | `j2k-native` | +| JPEG to J2K/HTJ2K transcode | `j2k-transcode` | +| CUDA adapters | `j2k-jpeg-cuda`, `j2k-cuda`, `j2k-transcode-cuda` | +| Metal adapters | `j2k-jpeg-metal`, `j2k-metal`, `j2k-transcode-metal` | +| Tile compression codecs | `j2k-tilecodec` | +| Command-line inspection | `j2k-cli` | -let jpeg_bytes = std::fs::read("tile.jpg")?; -let jpeg_info = JpegDecoder::inspect(&jpeg_bytes)?; - -let j2k_bytes = std::fs::read("tile.jp2")?; -let j2k_info = J2kDecoder::inspect(&j2k_bytes)?; - -println!("JPEG: {:?}", jpeg_info.dimensions); -println!("J2K: {:?}", j2k_info.dimensions); -``` - -Decode a JPEG tile into caller-owned RGB output: - -```rust -use signinum::{jpeg::Decoder as JpegDecoder, PixelFormat}; - -let bytes = std::fs::read("tile.jpg")?; -let decoder = JpegDecoder::new(&bytes)?; -let (width, height) = decoder.info().dimensions; -let stride = width as usize * PixelFormat::Rgb8.bytes_per_pixel(); -let mut rgb = vec![0_u8; stride * height as usize]; - -decoder.decode_into(&mut rgb, stride, PixelFormat::Rgb8)?; -``` +The names `statumen` and `wsi-dicom` are not current package names. -Decode a JPEG 2000 / HTJ2K tile with the same caller-owned output model: - -```rust -use signinum::{j2k::J2kDecoder, j2k::J2kScratchPool, Downscale, PixelFormat}; - -let bytes = std::fs::read("tile.jp2")?; -let mut decoder = J2kDecoder::new(&bytes)?; -let (width, height) = decoder.info().dimensions; -let stride = width as usize * PixelFormat::Rgb8.bytes_per_pixel(); -let mut rgb = vec![0_u8; stride * height as usize]; -let mut scratch = J2kScratchPool::new(); - -decoder.decode_scaled_into( - &mut scratch, - &mut rgb, - stride, - PixelFormat::Rgb8, - Downscale::None, -)?; -``` - -Encode lossless JPEG 2000 / HTJ2K when compressed source bytes cannot be -passed through legally: - -```rust -use signinum::j2k::{ - encode_j2k_lossless, J2kLosslessEncodeOptions, J2kLosslessSamples, -}; - -let pixels = vec![0_u8; 256 * 256]; -let samples = J2kLosslessSamples::new(&pixels, 256, 256, 1, 8, false)?; -let encoded = encode_j2k_lossless(samples, &J2kLosslessEncodeOptions::default())?; - -assert!(encoded.codestream.starts_with(&[0xFF, 0x4F])); -``` - -Decompress a container tile payload: - -```rust -use signinum::{tilecodec::DeflateCodec, TileDecompress}; - -let compressed = std::fs::read("tile.deflate")?; -let mut pool = ::Pool::default(); -let mut out = vec![0_u8; 1 << 20]; -let written = DeflateCodec::decompress_into(&mut pool, &compressed, &mut out)?; - -println!("decoded {written} bytes"); -``` +## Fast Path For LLM-Assisted Use -Inspect from the command line: +For normal JPEG 2000 / HTJ2K work, start with the public codec crate: -```sh -cargo install signinum-cli -signinum inspect tile.jp2 +```bash +cargo add j2k ``` -## Supported workflows - -`signinum-jpeg` provides WSI-focused JPEG inspect and decode: - -- borrowed parse surfaces through `JpegView` -- baseline JPEG decode for WSI tiles -- ROI, scaled decode, row streaming, and tile-batch decode APIs -- passthrough candidates for baseline and extended sequential interchange - streams -- a small baseline JPEG encoder for fixtures, fallback, and derived output - -`signinum-j2k` provides JPEG 2000 / HTJ2K inspect, decode, and encode: - -- JP2 and raw codestream inspection -- borrowed passthrough candidates for JP2, JPEG 2000, and HTJ2K payloads -- full-frame, ROI, reduced-resolution, combined ROI+reduced-resolution, - row-bounded, and tile-batch decode -- pure-Rust in-repo JPEG 2000 / HTJ2K decode engine -- lossless JPEG 2000 / HTJ2K encode for new diagnostic codestreams -- parity and benchmark coverage against Grok and OpenJPEG where available - -`signinum-tilecodec` provides tile decompression primitives: - -- `DeflateCodec` -- `ZstdCodec` -- `LzwCodec` -- `UncompressedCodec` - -These codecs implement the shared `TileDecompress` trait from `signinum-core`. - -## Backend model - -The public API uses `BackendRequest` so callers state what kind of output they -need: - -- `BackendRequest::Cpu` requires host-backed output. -- `BackendRequest::Auto` lets an adapter use a validated device path for - supported workloads and otherwise fall back to CPU. -- `BackendRequest::Metal` requires resident Metal execution on macOS and - reports unsupported or unavailable requests as errors. -- `BackendRequest::Cuda` requires CUDA device-memory output when the - `cuda-runtime` feature and a CUDA driver are available. - -CPU decode is the portability baseline on native `x86_64` and `aarch64` -hosts. Device adapters are additive: removing Metal and CUDA crates leaves the -CPU codec stack functional. - -Metal adapters target Apple Silicon macOS. They return `MTLBuffer`-backed -`DeviceSurface`s for supported shapes and keep explicit Metal requests strict. -If a caller wants CPU-decoded bytes uploaded to Metal, use the adapter's -explicit CPU-staged upload APIs instead of `BackendRequest::Metal`. - -CUDA adapters expose CUDA device-memory output for explicit CUDA requests when -they are built with `cuda-runtime`. `signinum-jpeg-cuda` can use NVIDIA nvJPEG -for full-frame RGB8 JPEG requests when `cuda-runtime`, a CUDA driver, and -`libnvjpeg` are available; unsupported JPEG shapes use CPU decode plus CUDA -upload. `signinum-j2k-cuda` currently returns CUDA-backed output by uploading -CPU-decoded JPEG 2000 / HTJ2K pixels; it does not claim CUDA codestream decode -kernels yet. - -## Architecture at a glance - -The workspace is layered so container readers and viewers can bring their own -threading, I/O, cache, pyramid policy, and prefetch logic. - -```text -foundation -> codecs / codec engines -> device adapters -> facade / CLI +The shared decode traits live in `j2k-core` and are implemented by codec +crates: `ImageDecode`, `ImageDecodeRows`, `TileBatchDecode`, and +device-surface traits. + +Runnable examples: + +- `crates/j2k/examples/decode_generated.rs` +- `crates/j2k-jpeg/examples/inspect.rs` +- `crates/j2k-tilecodec/examples/decompress.rs` +- `crates/j2k-transcode/examples/jpeg_to_htj2k.rs` + +## Current backend posture + +CPU is the correctness baseline. `BackendRequest::Auto` may return CPU-backed +outputs when a device path is unavailable, unsupported, or not benchmarked for +the requested shape. + +GPU routing is intentionally selective. A Metal or CUDA path should be enabled +automatically only when the shape is supported, parity-covered, large or +regular enough to amortize dispatch and transfer costs, and backed by benchmark +evidence. Small tiles, irregular packet shapes, entropy-heavy stages, and +codestream assembly should remain CPU unless a measured resident path shows a +net win. + +Metal adapters are macOS-only and experimental. Explicit Metal requests return +resident Metal surfaces or encode-stage dispatches only for supported adapter +paths. Metal encode support is not a blanket end-to-end guarantee for every +public encode route; unsupported explicit Metal shapes fail clearly. + +CUDA adapters require a CUDA driver and adapter support. CUDA device memory +surfaces are available for supported paths; unsupported explicit CUDA requests +fail clearly. J2K-owned CUDA kernels are used for CUDA codec stages. NVIDIA +performance claims require recorded self-hosted benchmark output. + +## Public API and support policy + +Stable APIs are `j2k`, `j2k-core` traits and value types, `j2k-jpeg`, +and `j2k-tilecodec`. Experimental APIs are the Metal adapters, CUDA adapters, +transcode crates, and backend encode-stage adapter SPI. + +Codec contracts include `ImageDecode`, `decode_region_scaled_into`, +`decode_rows`, `TileBatchDecode`, `DeviceSurface`, `ScratchPool`, and +`DecoderContext`. `BackendRequest::Auto` may return CPU output. +`BackendRequest::Metal` and `BackendRequest::Cuda` are strict and fail for +unsupported shapes. + +Container and storage integrations should pass compatible compressed payloads +through when the payload kind, dimensions, component count, bit depth, +signedness, and color interpretation already match the destination +requirements. Decode and re-encode only when passthrough is invalid and the +source codec path is supported. + +Unsupported input must fail explicitly. Error messages must avoid sensitive +internal details. Unsafe Rust inventory is tracked in +[docs/unsafe-audit.md](docs/unsafe-audit.md). Fuzzing and malformed-input tests +are part of release hardening. MSRV is declared in the root manifest. + +Reference files: + +- [docs/architecture.md](docs/architecture.md) - workspace layer rules and crate + dependency graph +- [docs/env-vars.md](docs/env-vars.md) - supported `J2K_*` + environment variables +- [docs/release.md](docs/release.md) - release and package validation policy +- [docs/stable-api-1.0.md](docs/stable-api-1.0.md) - stable API snapshot policy +- [CHANGELOG.md](CHANGELOG.md) - current release notes + +## Benchmark and parity policy + +A published benchmark must identify: + +- host hardware and OS +- exact command +- input source +- whether input is j2k-generated or external +- comparator availability +- comparator version +- skipped paths and skip reason +- thread count, including `J2K_COMPARE_THREADS` when applicable + +Public OpenJPEG and Grok comparison claims require explicit comparator gates and +cannot silently skip: + +```bash +J2K_REQUIRE_OPENJPEG=1 +J2K_REQUIRE_GROK=1 ``` -| Layer | Crates | Responsibility | -|-------|--------|----------------| -| Foundation | `signinum-core` | Shared traits, pixel/sample types, backend selection, device surfaces, scratch/context contracts, passthrough contracts | -| Instrumentation helper | `signinum-profile` | Shared profiling row formatting, env parsing, and summary aggregation used by runtime crates | -| Codecs | `signinum-jpeg`, `signinum-j2k`, `signinum-tilecodec` | Format-specific inspect, decode, encode, row, ROI, scaled, batch, and decompression APIs | -| Engine | `signinum-j2k-native` | Published implementation dependency for the public J2K crate | -| Adapters | `signinum-jpeg-metal`, `signinum-j2k-metal`, `signinum-jpeg-cuda`, `signinum-j2k-cuda` | Device-output surfaces for downstream GPU pipelines | -| Runtime helper | `signinum-cuda-runtime` | CUDA Driver API allocation, copy, kernel, and nvJPEG loading used by CUDA adapters | -| Facade and CLI | `signinum`, `signinum-cli` | One import surface for application code and `signinum inspect ` | -| Reference tooling | `signinum-j2k-compare` | OpenJPEG/Grok comparison helpers for tests and benches; not a runtime dependency | - -The full system map, dependency rules, and current adapter routing policy live -in [docs/architecture.md](docs/architecture.md). - -## WSI and DICOM passthrough policy - -Container integrations should pass compressed tile bytes through unchanged -whenever the destination transfer syntax and frame metadata make that legal. -Codec views expose borrowed `PassthroughCandidate`s; container layers remain -responsible for DICOM-specific frame ordering, fragment writing, and metadata -validation. - -If new diagnostic codestream bytes are required, prefer lossless JPEG 2000 / -HTJ2K encode. Baseline JPEG encode is for explicit fallback, generated -fixtures, or non-diagnostic derived output. - -See: - -- [docs/wsi-decode-api.md](docs/wsi-decode-api.md) -- [docs/wsi-dicom-passthrough.md](docs/wsi-dicom-passthrough.md) - -## Fast Path For LLM-Assisted Use - -If you are asking an LLM to use this repository, give it this instruction: - -> Use `signinum` for JPEG, JPEG 2000 / HTJ2K, tile decompression, and -> device-output codec primitives. If the task says "open a whole-slide image", -> use `statumen` first. If the task says "convert a slide to DICOM", use -> `wsi-dicom`. - -## Benchmarks and parity - -Benchmark methodology and comparator policy live in [docs/bench.md](docs/bench.md). -Parity expectations live in [docs/parity.md](docs/parity.md). - -The repo carries compare benches for: - -- `signinum-jpeg` -- `signinum-j2k` -- `signinum-jpeg-metal` -- `signinum-j2k-metal` -- `signinum-tilecodec` - -Benchmark results are hardware-specific. GPU benchmark baselines should be -collected on self-hosted runners with the target device stack installed. - -## Project docs - -- [docs/architecture.md](docs/architecture.md) - workspace map and dependency rules -- [docs/wsi-decode-api.md](docs/wsi-decode-api.md) - public WSI decode API guide -- [docs/wsi-dicom-passthrough.md](docs/wsi-dicom-passthrough.md) - passthrough-first conversion policy -- [docs/bench.md](docs/bench.md) - benchmark methodology -- [docs/parity.md](docs/parity.md) - reference decoder parity expectations -- [docs/release.md](docs/release.md) - release staging notes -- [CHANGELOG.md](CHANGELOG.md) - release history - -Runnable crate examples are available under: - -- `crates/signinum-jpeg/examples` -- `crates/signinum-j2k/examples` -- `crates/signinum-tilecodec/examples` - -## Platform and MSRV - -- Rust edition: 2021 -- MSRV: Rust 1.94, pinned by [rust-toolchain.toml](rust-toolchain.toml) -- Decode hosts: native `x86_64` and `aarch64` -- Metal: Apple Silicon macOS for resident Metal device surfaces -- CUDA: hosts with a CUDA driver when CUDA adapters are built with - `cuda-runtime` +J2K-generated J2K/HTJ2K codestreams require native decoder round trips. +OpenJPEG and Grok comparisons are used where those tools support the feature. +Missing comparators cannot convert a parity signoff into a pass. -## License +## Security -Apache-2.0. See [LICENSE-APACHE](LICENSE-APACHE). +Report vulnerabilities according to [SECURITY.md](SECURITY.md). Codec errors +should be explicit, non-sensitive, and should not silently treat unsupported +input as successful decode. diff --git a/SECURITY.md b/SECURITY.md index c81ffbf6..39a45bd7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,25 +1,28 @@ # Security Policy -## Reporting a Vulnerability +## Supported versions -JPEG decoders ingest adversarial byte streams from the wild. If you find a -crash, memory-safety violation, or undefined behavior in `signinum`, please -report it privately rather than opening a public issue. +The current supported development line is `0.5.x`. -Use GitHub's private vulnerability reporting for the repository, or contact the -maintainer through the repository owner profile if private reporting is not yet -enabled. +## Reporting vulnerabilities -Please include: -- A minimal reproducer (input bytes + API call). -- Rust version, target triple, and cargo features used. -- Expected vs. observed behavior. +Report suspected vulnerabilities privately through GitHub private vulnerability +reporting: +(repository **Security** tab → **Report a vulnerability**). Do not open public +issues or publish proof-of-concept exploit details before triage. -Reports are acknowledged within 7 days. Patches are issued as soon as possible, -generally within 30 days for high-severity issues. +Response expectations: -## Supported versions +- Acknowledgment of a private report within **3 business days**. +- Triage decision (accepted / declined / needs more information) within + **14 days** of acknowledgment. +- Coordinated disclosure: we will agree on a publication date with the + reporter before any advisory or fix details are made public. + +## Baseline expectations -The supported stable line is CPU-first 1.x. Pre-1.0 adapter crates receive -security fixes in their latest minor line while they remain part of the -repository. +- Unsupported input must fail explicitly. +- Error responses must avoid sensitive internal details. +- Device backends must not silently substitute a different explicit backend. +- Unsafe Rust inventory is tracked in `docs/unsafe-audit.md`. +- Fuzzing and malformed-input tests are part of release hardening. diff --git a/corpus/README.md b/corpus/README.md index 8f331abe..752c91e5 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -1,49 +1,11 @@ -# signinum corpus +# Corpus -Test data committed to this repo. Size budget: ~30 MB total. +This directory stores small committed conformance fixtures and metadata for +larger external corpora used during validation. -## Directory policy +Committed fixtures are intentionally limited. Larger WSI, JPEG, J2K, and HTJ2K +inputs must be supplied through environment variables or external corpus +locations and must not be committed accidentally. -- `wsi-samples/` — trimmed JPEG payloads extracted from public WSI files. - Every sample has an entry in `wsi-samples/manifest.json` documenting the - source slide, level, tile coordinates, licensing, and expected tolerance. - License verification is mandatory — reject any source without a permissive - or public-domain license. -- `conformance/` — libjpeg-turbo-generated reference outputs. Generated by - `conformance/generate.sh` on the maintainer's machine; the libjpeg-turbo - version used is pinned in `conformance/manifest.json`. -- `regressions/` — per-bug repro files. Naming: `issue-NNN.jpg` keyed to the - issue tracker number. Each has a corresponding test in `tests/regressions.rs`. -- `fuzz-seeds/` — hand-crafted adversarial inputs (extreme dimensions, crafted - restart markers, truncated streams, zero-sized tables). Used to seed - `cargo-fuzz` targets. - -The committed corpus is intentionally small. `conformance/` currently carries -baseline RGB, grayscale, restart-coded, and sampling-variant JPEG fixtures with -raw reference outputs listed in `conformance/manifest.json`. - -Optional real-slide parity corpora stay out of the default test path. To fetch -entries from a public manifest with URLs and SHA-256s, run: - -```sh -scripts/parity-corpus-fetch.sh path/to/manifest.json corpus/wsi-samples -``` - -Then opt in to local WSI JPEG coverage with: - -```sh -SIGNINUM_WSI_ROOT=corpus/wsi-samples cargo test -p signinum-jpeg --test external_wsi -``` - -## Generating conformance fixtures - -Run `corpus/conformance/generate.sh` on a machine with libjpeg-turbo installed. -The script produces reference `.rgb`/`.gray` outputs and a `manifest.json` -recording the libjpeg-turbo version. Commit the results — CI does **not** -regenerate them. - -Regenerate on: -- First-time repo setup. -- Adding, removing, or replacing a committed fixture. -- libjpeg-turbo version bump recorded in `manifest.json` (tracks Risk #3 in - the design spec). +Fixture inventory should be derived from manifests and directory contents, not +from stale prose. diff --git a/corpus/j2k-conformance/README.md b/corpus/j2k-conformance/README.md new file mode 100644 index 00000000..6721a5b6 --- /dev/null +++ b/corpus/j2k-conformance/README.md @@ -0,0 +1,11 @@ +# J2K Conformance Corpus + +This directory holds metadata for JPEG 2000 conformance vectors. + +The manifest is the source of truth: + +- `manifest.tsv` + +Release signoff should classify vectors by shipped feature coverage, known +limitations, and investigation status. Narrative summaries must not override the +manifest. diff --git a/corpus/j2k-conformance/manifest.tsv b/corpus/j2k-conformance/manifest.tsv new file mode 100644 index 00000000..97b97db2 --- /dev/null +++ b/corpus/j2k-conformance/manifest.tsv @@ -0,0 +1,11 @@ +# id path classification features reason +part1_core_lossless_53 codestreams_profile0/p0_01.j2k blocking part1-core;lossless-5-3 Part 1 reversible core codestream support is shipped by the release scope. +part1_core_lossy_97_layers_precincts codestreams_profile0/p0_04.j2k blocking part1-core;lossy-9-7;quality-layers;precincts;progression-orders Part 1 irreversible lossy multi-layer codestream support, precincts, and progression-order support are shipped by the release scope. +part1_poc_tlm_sop codestreams_profile0/p0_03.j2k blocking part1-core;poc;progression-orders;tlm;sop POC packet iteration, TLM metadata, SOP markers, and progression-order support are shipped by the release scope. +part1_plt_sop_eph codestreams_profile0/p0_07.j2k blocking part1-core;poc;progression-orders;plt;sop;eph PLT packet-length metadata, SOP/EPH packet markers, and POC progression changes are shipped by the release scope. +openhtj2k_ds0_ht_12_b11 htj2k_bsets_profile0/p0_12_bset/ds0_ht_12_b11.j2k blocking part15-core;ht-refinement HTJ2K core refinement-pass conformance vector shipped by the release scope. +openhtj2k_ds0_ht_09_b11 htj2k_bsets_profile0/p0_09_bset/ds0_ht_09_b11.j2k blocking part15-core;ht-refinement HTJ2K core refinement-pass conformance vector shipped by the release scope. +plm_iso_vector_absent known-limitations/no-plm-vector-in-t803v3.txt known-limitation plm;conformance-coverage-gap The available T.803v3 extraction has no described PLM-bearing vector; PLM behavior is covered by unit and encode round-trip tests. +jpx_part2_deferred known-limitations/jpx-part2-placeholder.jp2 known-limitation jpx;part2 JPX and JPEG 2000 Part 2 extensions are explicitly deferred. +icc_roundtrip_deferred known-limitations/icc-roundtrip-placeholder.jp2 known-limitation icc-roundtrip ICC round-trip on encode is explicitly deferred. +encode_gt16_deferred known-limitations/gt16-placeholder.j2k known-limitation encode-17-38-bit 17-38 bit encode is explicitly unsupported in this release. diff --git a/crates/signinum-cli/Cargo.toml b/crates/j2k-cli/Cargo.toml similarity index 52% rename from crates/signinum-cli/Cargo.toml rename to crates/j2k-cli/Cargo.toml index 13bdf3d5..2e247949 100644 --- a/crates/signinum-cli/Cargo.toml +++ b/crates/j2k-cli/Cargo.toml @@ -1,21 +1,32 @@ [package] -name = "signinum-cli" -description = "Command-line tool for signinum: inspect" -version = "0.4.2" +name = "j2k-cli" +description = "Command-line JPEG 2000 and HTJ2K inspection tool for j2k" +version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true readme = "README.md" +[package.metadata.docs.rs] +all-features = true +targets = [] + [[bin]] -name = "signinum" +name = "j2k" path = "src/main.rs" +doc = false [dependencies] -signinum-j2k = { path = "../signinum-j2k", version = "=0.4.2" } -signinum-jpeg = { path = "../signinum-jpeg", version = "=0.4.2" } +j2k = { path = "../j2k", version = "=0.6.0" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.6.0" } + +[dev-dependencies] +j2k-test-support = { path = "../j2k-test-support" } # NOTE: `[profile.*]` blocks in package-level manifests are silently ignored # inside a workspace (see https://doc.rust-lang.org/cargo/reference/profiles.html#overrides). # Any codegen tweaks for the CLI must go in the root workspace `Cargo.toml`. + +[lints] +workspace = true diff --git a/crates/j2k-cli/README.md b/crates/j2k-cli/README.md new file mode 100644 index 00000000..ed61b8c4 --- /dev/null +++ b/crates/j2k-cli/README.md @@ -0,0 +1,6 @@ +# j2k-cli + +Command-line entry point for J2K. + +The current CLI focuses on inspection and smoke-test workflows. Full +compress/decompress CLI parity is not claimed by this package. diff --git a/crates/signinum-cli/src/main.rs b/crates/j2k-cli/src/main.rs similarity index 51% rename from crates/signinum-cli/src/main.rs rename to crates/j2k-cli/src/main.rs index 91ad9842..17a8a847 100644 --- a/crates/signinum-cli/src/main.rs +++ b/crates/j2k-cli/src/main.rs @@ -4,26 +4,23 @@ use std::io::{self, Read}; use std::path::Path; use std::process::ExitCode; +const INSPECT_READ_LIMIT: usize = 64 * 1024 * 1024; + fn main() -> ExitCode { let mut args = std::env::args().skip(1); let subcommand = args.next(); match subcommand.as_deref() { Some("inspect") => { - let path = match args.next() { - Some(p) => p, - None => { - eprintln!("usage: signinum inspect "); - return ExitCode::from(2); - } + let Some(path) = args.next() else { + eprintln!("usage: j2k inspect "); + return ExitCode::from(2); }; inspect(Path::new(&path)) } - Some("--help") | Some("-h") | Some("help") | None => { - eprintln!("signinum {}", env!("CARGO_PKG_VERSION")); + Some("--help" | "-h" | "help") | None => { + eprintln!("j2k {}", env!("CARGO_PKG_VERSION")); eprintln!("Usage:"); - eprintln!( - " signinum inspect Parse JPEG or JPEG 2000 headers and print Info" - ); + eprintln!(" j2k inspect Parse JPEG or JPEG 2000 headers and print Info"); ExitCode::SUCCESS } Some(other) => { @@ -34,25 +31,36 @@ fn main() -> ExitCode { } fn inspect(path: &Path) -> ExitCode { - let bytes = match read_file(path) { - Ok(b) => b, + let input = match read_inspect_input(path) { + Ok(input) => input, Err(e) => { eprintln!("error reading {}: {e}", path.display()); return ExitCode::from(1); } }; - match inspect_bytes(&bytes) { + match inspect_bytes(&input.bytes) { Ok(line) => { println!("{line}"); ExitCode::SUCCESS } Err(message) => { eprintln!("{message}"); + if input.truncated { + eprintln!( + "note: inspect read only the first {INSPECT_READ_LIMIT} bytes to avoid unbounded memory use" + ); + } ExitCode::from(1) } } } +#[derive(Debug, Clone, PartialEq, Eq)] +struct InspectInput { + bytes: Vec, + truncated: bool, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum InspectFormat { Jpeg, @@ -70,7 +78,7 @@ fn detect_inspect_format(bytes: &[u8]) -> InspectFormat { fn inspect_bytes(bytes: &[u8]) -> Result { match detect_inspect_format(bytes) { - InspectFormat::Jpeg => match signinum_jpeg::Decoder::inspect(bytes) { + InspectFormat::Jpeg => match j2k_jpeg::Decoder::inspect(bytes) { Ok(info) => Ok(format!( "{}×{} {:?} {:?} bit={} samp={:?} mcu={}x{} units={}x{} rst={:?} scans={}", info.dimensions.0, @@ -90,13 +98,13 @@ fn inspect_bytes(bytes: &[u8]) -> Result { let mut message = format!("error: {e}"); if e.is_unsupported() { message.push_str( - "\nhint: this file is not supported by signinum; try jpeg-decoder or openjpeg", + "\nhint: this file is not supported by j2k; try jpeg-decoder or openjpeg", ); } Err(message) } }, - InspectFormat::J2k => match signinum_j2k::J2kDecoder::inspect(bytes) { + InspectFormat::J2k => match j2k::J2kDecoder::inspect(bytes) { Ok(info) => Ok(format!( "{}×{} {:?} bit={} comps={} levels={} tiles={:?}", info.dimensions.0, @@ -112,72 +120,31 @@ fn inspect_bytes(bytes: &[u8]) -> Result { } } -fn read_file(path: &Path) -> io::Result> { +fn read_inspect_input(path: &Path) -> io::Result { + read_inspect_input_with_limit(path, INSPECT_READ_LIMIT) +} + +fn read_inspect_input_with_limit(path: &Path, limit: usize) -> io::Result { let mut file = std::fs::File::open(path)?; let mut buf = Vec::new(); - file.read_to_end(&mut buf)?; - Ok(buf) + let mut limited = file.by_ref().take((limit + 1) as u64); + limited.read_to_end(&mut buf)?; + let truncated = buf.len() > limit; + if truncated { + buf.truncate(limit); + } + Ok(InspectInput { + bytes: buf, + truncated, + }) } #[cfg(test)] mod tests { - use super::{detect_inspect_format, inspect_bytes, InspectFormat}; - - fn minimal_j2k_codestream() -> Vec { - let mut bytes = vec![0xFF, 0x4F]; - let mut siz = Vec::new(); - push_u16(&mut siz, 0); - push_u32(&mut siz, 128); - push_u32(&mut siz, 64); - push_u32(&mut siz, 0); - push_u32(&mut siz, 0); - push_u32(&mut siz, 64); - push_u32(&mut siz, 64); - push_u32(&mut siz, 0); - push_u32(&mut siz, 0); - push_u16(&mut siz, 3); - for _ in 0..3 { - siz.extend_from_slice(&[0x07, 0x01, 0x01]); - } - bytes.extend_from_slice(&[0xFF, 0x51]); - push_u16(&mut bytes, (siz.len() + 2) as u16); - bytes.extend_from_slice(&siz); - - let cod = [0x00, 0x00, 0x00, 0x01, 0x01, 0x05, 0x04, 0x04, 0x00, 0x01]; - bytes.extend_from_slice(&[0xFF, 0x52]); - push_u16(&mut bytes, (cod.len() + 2) as u16); - bytes.extend_from_slice(&cod); - bytes.extend_from_slice(&[0xFF, 0x90, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); - bytes - } - - fn minimal_jp2() -> Vec { - let codestream = minimal_j2k_codestream(); - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0D, 0x0A, 0x87, 0x0A]); - bytes.extend_from_slice(&[ - 0, 0, 0, 20, b'f', b't', b'y', b'p', b'j', b'p', b'2', b' ', 0, 0, 0, 0, b'j', b'p', - b'2', b' ', - ]); - bytes.extend_from_slice(&[ - 0, 0, 0, 45, b'j', b'p', b'2', b'h', 0, 0, 0, 22, b'i', b'h', b'd', b'r', 0, 0, 0, 64, - 0, 0, 0, 128, 0, 3, 7, 7, 0, 0, 0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0, 0, 0, 0, - 16, - ]); - let len = (8 + codestream.len()) as u32; - bytes.extend_from_slice(&len.to_be_bytes()); - bytes.extend_from_slice(b"jp2c"); - bytes.extend_from_slice(&codestream); - bytes - } - - fn push_u16(out: &mut Vec, value: u16) { - out.extend_from_slice(&value.to_be_bytes()); - } - - fn push_u32(out: &mut Vec, value: u32) { - out.extend_from_slice(&value.to_be_bytes()); - } + use super::{ + detect_inspect_format, inspect_bytes, read_inspect_input_with_limit, InspectFormat, + }; + use j2k_test_support::{minimal_j2k_codestream, minimal_jp2}; #[test] fn detects_j2k_codestream_magic() { @@ -192,6 +159,19 @@ mod tests { assert_eq!(detect_inspect_format(&minimal_jp2()), InspectFormat::J2k); } + #[test] + fn inspect_read_is_bounded() { + let path = + std::env::temp_dir().join(format!("j2k-cli-inspect-bounded-{}", std::process::id())); + std::fs::write(&path, [0xFF; 12]).expect("write bounded-read fixture"); + + let input = read_inspect_input_with_limit(&path, 8).expect("read bounded inspect input"); + let _ = std::fs::remove_file(&path); + + assert!(input.truncated); + assert_eq!(input.bytes.len(), 8); + } + #[test] fn inspect_bytes_dispatches_to_j2k() { let line = inspect_bytes(&minimal_jp2()).expect("jp2 inspect"); diff --git a/crates/j2k-cli/tests/inspect_cli.rs b/crates/j2k-cli/tests/inspect_cli.rs new file mode 100644 index 00000000..80a3e943 --- /dev/null +++ b/crates/j2k-cli/tests/inspect_cli.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + fs, + path::{Path, PathBuf}, + process::{Command, Output}, +}; + +use j2k_test_support::{minimal_gray8_jpeg, minimal_jp2}; + +fn j2k_bin() -> &'static str { + env!("CARGO_BIN_EXE_j2k") +} + +fn run_j2k(args: &[&str]) -> Output { + Command::new(j2k_bin()) + .args(args) + .output() + .expect("run j2k CLI") +} + +fn write_temp_file(name: &str, bytes: &[u8]) -> PathBuf { + let dir = std::env::temp_dir().join(format!("j2k-cli-tests-{}", std::process::id())); + fs::create_dir_all(&dir).expect("create CLI test temp dir"); + let path = dir.join(name); + fs::write(&path, bytes).expect("write CLI test input"); + path +} + +fn stdout(output: &Output) -> String { + String::from_utf8_lossy(&output.stdout).into_owned() +} + +fn stderr(output: &Output) -> String { + String::from_utf8_lossy(&output.stderr).into_owned() +} + +#[test] +fn inspect_cli_reports_jpeg_info() { + let jpeg = minimal_gray8_jpeg(); + let input = write_temp_file("grayscale_8x8.jpg", &jpeg); + + let output = run_j2k(&["inspect", path_str(&input)]); + + assert!(output.status.success(), "stderr: {}", stderr(&output)); + let stdout = stdout(&output); + assert!(stdout.contains('8')); + assert!(stdout.contains("Grayscale")); + assert!(stdout.contains("bit=8")); +} + +#[test] +fn inspect_cli_reports_jp2_info() { + let input = write_temp_file("minimal.jp2", &minimal_jp2()); + + let output = run_j2k(&["inspect", path_str(&input)]); + + assert!(output.status.success(), "stderr: {}", stderr(&output)); + let stdout = stdout(&output); + assert!(stdout.contains("128")); + assert!(stdout.contains("64")); + assert!(stdout.contains("levels=6")); +} + +#[test] +fn inspect_cli_rejects_unknown_subcommand() { + let output = run_j2k(&["unknown"]); + + assert_eq!(output.status.code(), Some(2)); + assert!(stderr(&output).contains("unknown subcommand: unknown")); +} + +#[test] +fn inspect_cli_reports_missing_file() { + let missing = std::env::temp_dir() + .join(format!("j2k-cli-tests-{}", std::process::id())) + .join("missing.jpg"); + + let output = run_j2k(&["inspect", path_str(&missing)]); + + assert_eq!(output.status.code(), Some(1)); + assert!(stderr(&output).contains("error reading")); +} + +#[test] +fn inspect_cli_reports_invalid_jpeg() { + let input = write_temp_file("invalid.jpg", b"not a jpeg"); + + let output = run_j2k(&["inspect", path_str(&input)]); + + assert_eq!(output.status.code(), Some(1)); + assert!(stderr(&output).contains("error:")); +} + +fn path_str(path: &Path) -> &str { + path.to_str().expect("test path is UTF-8") +} diff --git a/crates/signinum-j2k-compare/Cargo.toml b/crates/j2k-compare/Cargo.toml similarity index 61% rename from crates/signinum-j2k-compare/Cargo.toml rename to crates/j2k-compare/Cargo.toml index fc70492f..bc78afb1 100644 --- a/crates/signinum-j2k-compare/Cargo.toml +++ b/crates/j2k-compare/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "signinum-j2k-compare" -description = "JPEG 2000 oracle comparison helpers for signinum" +name = "j2k-compare" +description = "JPEG 2000 oracle comparison helpers for j2k" version = "0.2.0" edition.workspace = true rust-version.workspace = true @@ -9,20 +9,22 @@ repository.workspace = true publish = false [dependencies] -signinum-core = { path = "../signinum-core", version = "=0.4.2" } -signinum-j2k = { path = "../signinum-j2k", version = "=0.4.2" } +j2k-core = { path = "../j2k-core", version = "=0.6.0" } +j2k = { path = "../j2k", version = "=0.6.0" } +j2k-test-support = { path = "../j2k-test-support", version = "=0.6.0" } openjpeg-sys = "1.0.12" -[dev-dependencies] -signinum-test-support = { path = "../signinum-test-support" } - [build-dependencies] -cc = "1.2" +cc = { workspace = true } [[bin]] name = "jp2k_batch_compare" path = "src/bin/jp2k_batch_compare.rs" +[[bin]] +name = "jp2k_roi_batch_compare" +path = "src/bin/jp2k_roi_batch_compare.rs" + [lints.rust] unsafe_code = "allow" unsafe_op_in_unsafe_fn = "deny" @@ -30,6 +32,7 @@ unreachable_pub = "warn" [lints.clippy] pedantic = { level = "warn", priority = -1 } +undocumented_unsafe_blocks = "warn" module_name_repetitions = "allow" must_use_candidate = "allow" missing_errors_doc = "allow" diff --git a/crates/j2k-compare/build.rs b/crates/j2k-compare/build.rs new file mode 100644 index 00000000..56d7839c --- /dev/null +++ b/crates/j2k-compare/build.rs @@ -0,0 +1,234 @@ +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +fn main() { + println!("cargo:rustc-check-cfg=cfg(have_grok)"); + println!("cargo:rerun-if-env-changed=J2K_GROK_ROOT"); + println!("cargo:rerun-if-env-changed=J2K_GROK_SOURCE"); + println!("cargo:rerun-if-env-changed=PKG_CONFIG_PATH"); + println!("cargo:rerun-if-changed=src/grok_shim.c"); + + if let Some(config) = grok_config() { + let staged_lib_dir = stage_grok_runtime(&config.lib_dir) + .unwrap_or_else(|err| panic!("failed to stage Grok runtime libraries: {err}")); + let grok_version = grok_version(&config); + println!("cargo:rustc-cfg=have_grok"); + println!("cargo:rustc-env=J2K_GROK_VERSION={grok_version}"); + println!( + "cargo:rustc-env=J2K_GROK_LIB_DIR={}", + config.lib_dir.display() + ); + println!( + "cargo:rustc-link-search=native={}", + staged_lib_dir.display() + ); + println!( + "cargo:rustc-link-search=native={}", + config.lib_dir.display() + ); + println!("cargo:rustc-link-lib=dylib=grokj2k"); + #[cfg(target_os = "macos")] + println!( + "cargo:rustc-link-arg=-Wl,-rpath,{}", + config.lib_dir.display() + ); + #[cfg(target_os = "macos")] + println!( + "cargo:rustc-link-arg=-Wl,-rpath,{}", + staged_lib_dir.display() + ); + + cc::Build::new() + .file("src/grok_shim.c") + .include(config.source_include) + .include(config.build_include) + .warnings(false) + .compile("j2k_grok_shim"); + } +} + +fn stage_grok_runtime(lib_dir: &Path) -> Result { + let out_dir = + PathBuf::from(std::env::var_os("OUT_DIR").ok_or_else(|| "OUT_DIR missing".to_string())?); + #[cfg(target_os = "macos")] + { + stage_grok_dylib_family(lib_dir, &out_dir)?; + } + #[cfg(not(target_os = "macos"))] + { + let _ = lib_dir; + } + Ok(out_dir) +} + +#[cfg(target_os = "macos")] +fn stage_grok_dylib_family(lib_dir: &Path, out_dir: &Path) -> Result<(), String> { + let real_src = find_grok_real_dylib(lib_dir)?; + let real_name = real_src + .file_name() + .and_then(std::ffi::OsStr::to_str) + .ok_or_else(|| format!("invalid Grok dylib name: {}", real_src.display()))?; + let real_dst = out_dir.join(real_name); + if real_dst.exists() { + std::fs::remove_file(&real_dst) + .map_err(|err| format!("remove {}: {err}", real_dst.display()))?; + } + std::fs::copy(&real_src, &real_dst) + .map_err(|err| format!("copy {}: {err}", real_src.display()))?; + symlink_in_dir(real_name, &out_dir.join("libgrokj2k.1.dylib"))?; + symlink_in_dir("libgrokj2k.1.dylib", &out_dir.join("libgrokj2k.dylib"))?; + Ok(()) +} + +#[cfg(target_os = "macos")] +fn find_grok_real_dylib(lib_dir: &Path) -> Result { + let mut candidates = std::fs::read_dir(lib_dir) + .map_err(|err| format!("read Grok lib dir {}: {err}", lib_dir.display()))? + .map(|entry| entry.map(|entry| entry.path())) + .collect::, _>>() + .map_err(|err| format!("read Grok lib dir entry: {err}"))?; + candidates.retain(|path| { + path.file_name() + .and_then(std::ffi::OsStr::to_str) + .is_some_and(|name| { + name.starts_with("libgrokj2k.") + && path + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("dylib")) + && name != "libgrokj2k.dylib" + && name != "libgrokj2k.1.dylib" + && !name.contains("codec") + }) + }); + candidates.sort(); + candidates.pop().ok_or_else(|| { + format!( + "missing versioned libgrokj2k dylib in {}", + lib_dir.display() + ) + }) +} + +#[cfg(target_os = "macos")] +fn symlink_in_dir(target_name: &str, dst: &PathBuf) -> Result<(), String> { + if dst.exists() { + std::fs::remove_file(dst).map_err(|err| format!("remove {}: {err}", dst.display()))?; + } + std::os::unix::fs::symlink(target_name, dst) + .map_err(|err| format!("symlink {} -> {target_name}: {err}", dst.display())) +} + +struct GrokConfig { + lib_dir: PathBuf, + source_include: PathBuf, + build_include: PathBuf, +} + +fn grok_config() -> Option { + let source_root = std::env::var_os("J2K_GROK_SOURCE") + .map_or_else(|| PathBuf::from("/tmp/grok- j2k"), PathBuf::from); + let lib_dir = std::env::var_os("J2K_GROK_ROOT") + .map_or_else(|| source_root.join("build/bin"), PathBuf::from); + let source_include = source_root.join("src/lib/core"); + let build_include = source_root.join("build/src/lib/core"); + if has_grok_artifacts(&lib_dir, &source_include, &build_include) { + Some(GrokConfig { + lib_dir, + source_include, + build_include, + }) + } else { + pkg_config_grok_config() + } +} + +fn pkg_config_grok_config() -> Option { + let include_dir = pkg_config_variable("libgrokj2k", "includedir")?; + let lib_dir = pkg_config_variable("libgrokj2k", "libdir")?; + if has_grok_artifacts(&lib_dir, &include_dir, &include_dir) { + Some(GrokConfig { + lib_dir, + source_include: include_dir.clone(), + build_include: include_dir, + }) + } else { + None + } +} + +fn pkg_config_variable(package: &str, variable: &str) -> Option { + let output = Command::new("pkg-config") + .args(["--variable", variable, package]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let value = String::from_utf8(output.stdout).ok()?; + let value = value.trim(); + if value.is_empty() { + None + } else { + Some(PathBuf::from(value)) + } +} + +fn has_grok_artifacts(lib_dir: &Path, source_include: &Path, build_include: &Path) -> bool { + let header = source_include.join("grok.h"); + let config_header = build_include.join("grk_config.h"); + if !(header.exists() && config_header.exists()) { + return false; + } + [ + lib_dir.join("libgrokj2k.dylib"), + lib_dir.join("libgrokj2k.so"), + lib_dir.join("grokj2k.lib"), + ] + .into_iter() + .any(|path| path.exists()) +} + +fn grok_version(config: &GrokConfig) -> String { + pkg_config_modversion("libgrokj2k") + .or_else(|| source_tree_version(config)) + .unwrap_or_else(|| "unknown".to_string()) +} + +fn pkg_config_modversion(package: &str) -> Option { + let output = Command::new("pkg-config") + .args(["--modversion", package]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let value = String::from_utf8(output.stdout).ok()?; + let value = value.trim(); + if value.is_empty() { + None + } else { + Some(value.to_string()) + } +} + +fn source_tree_version(config: &GrokConfig) -> Option { + let source_root = config.source_include.parent()?.parent()?.parent()?; + let output = Command::new("git") + .args(["-C"]) + .arg(source_root) + .args(["describe", "--tags", "--always", "--dirty"]) + .output() + .ok()?; + if !output.status.success() { + return None; + } + let value = String::from_utf8(output.stdout).ok()?; + let value = value.trim(); + if value.is_empty() { + None + } else { + Some(value.to_string()) + } +} diff --git a/crates/signinum-j2k-compare/src/bin/jp2k_batch_compare.rs b/crates/j2k-compare/src/bin/jp2k_batch_compare.rs similarity index 78% rename from crates/signinum-j2k-compare/src/bin/jp2k_batch_compare.rs rename to crates/j2k-compare/src/bin/jp2k_batch_compare.rs index 7e5c6aca..2275b9eb 100644 --- a/crates/signinum-j2k-compare/src/bin/jp2k_batch_compare.rs +++ b/crates/j2k-compare/src/bin/jp2k_batch_compare.rs @@ -2,11 +2,12 @@ use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; -use std::time::Instant; -use signinum_core::tile_batch_worker_count; -use signinum_j2k::{decode_tiles_into, J2kDecoder, PixelFormat, TileBatchOptions, TileDecodeJob}; -use signinum_j2k_compare::{grok, openjpeg}; +use j2k::{decode_tiles_into, J2kDecoder, PixelFormat, TileBatchOptions, TileDecodeJob}; +use j2k_compare::{ + grok, measure_repeated, openjpeg, parse_positive_usize, sample_stats, usize_to_f64, +}; +use j2k_core::tile_batch_worker_count; const DEFAULT_BATCH_SIZES: &[usize] = &[1, 16, 512, 1024]; const DEFAULT_REPEATS: usize = 3; @@ -69,14 +70,14 @@ fn run() -> Result<(), String> { .copied() .max() .ok_or_else(|| "no batch sizes requested".to_string())?; - let repeats = std::env::var("SIGNINUM_J2K_BATCH_COMPARE_REPEATS") + let repeats = std::env::var("J2K_BATCH_COMPARE_REPEATS") .ok() - .map(|value| parse_positive_usize(&value, "SIGNINUM_J2K_BATCH_COMPARE_REPEATS")) + .map(|value| parse_positive_usize(&value, "J2K_BATCH_COMPARE_REPEATS")) .transpose()? .unwrap_or(DEFAULT_REPEATS); - let workers = std::env::var("SIGNINUM_J2K_BATCH_COMPARE_THREADS") + let workers = std::env::var("J2K_BATCH_COMPARE_THREADS") .ok() - .map(|value| parse_positive_usize(&value, "SIGNINUM_J2K_BATCH_COMPARE_THREADS")) + .map(|value| parse_positive_usize(&value, "J2K_BATCH_COMPARE_THREADS")) .transpose()? .map(|value| NonZeroUsize::new(value).expect("positive value was validated")); @@ -108,7 +109,7 @@ fn run() -> Result<(), String> { ); for batch_size in batch_sizes { - emit_measurement(measure_signinum(&tiles[..batch_size], repeats, workers)?); + emit_measurement(measure_j2k(&tiles[..batch_size], repeats, workers)?); emit_measurement(measure_external( &tiles[..batch_size], repeats, @@ -203,31 +204,24 @@ fn pixel_format(components: u8, bit_depth: u8) -> Option { } } -fn measure_signinum( +fn measure_j2k( tiles: &[TileInput], repeats: usize, workers: Option, ) -> Result { let format = tiles[0].format; - let mut samples = Vec::with_capacity(repeats); - let mut decoded_bytes_per_repeat = decode_signinum_once(tiles, format, workers)?; - std::hint::black_box(decoded_bytes_per_repeat); - for _ in 0..repeats { - let started = Instant::now(); - decoded_bytes_per_repeat = decode_signinum_once(tiles, format, workers)?; - samples.push(started.elapsed().as_secs_f64() * 1000.0); - std::hint::black_box(decoded_bytes_per_repeat); - } - Ok(measurement( - "signinum", + let (samples, decoded_bytes_per_repeat) = + measure_repeated(repeats, 1000.0, || decode_j2k_once(tiles, format, workers))?; + measurement( + "j2k", tiles.len(), repeats, samples, decoded_bytes_per_repeat, - )) + ) } -fn decode_signinum_once( +fn decode_j2k_once( tiles: &[TileInput], format: PixelFormat, workers: Option, @@ -246,7 +240,7 @@ fn decode_signinum_once( }) .collect::>(); decode_tiles_into(&mut jobs, format, TileBatchOptions { workers }) - .map_err(|err| format!("signinum batch decode failed: {err}"))?; + .map_err(|err| format!("j2k batch decode failed: {err}"))?; Ok(outputs.iter().map(Vec::len).sum()) } @@ -260,22 +254,16 @@ fn measure_external( ExternalDecoder::OpenJpeg => "openjpeg", ExternalDecoder::Grok => "grok", }; - let mut samples = Vec::with_capacity(repeats); - let mut decoded_bytes_per_repeat = decode_external_once(tiles, workers, decoder)?; - std::hint::black_box(decoded_bytes_per_repeat); - for _ in 0..repeats { - let started = Instant::now(); - decoded_bytes_per_repeat = decode_external_once(tiles, workers, decoder)?; - samples.push(started.elapsed().as_secs_f64() * 1000.0); - std::hint::black_box(decoded_bytes_per_repeat); - } - Ok(measurement( + let (samples, decoded_bytes_per_repeat) = measure_repeated(repeats, 1000.0, || { + decode_external_once(tiles, workers, decoder) + })?; + measurement( decoder_name, tiles.len(), repeats, samples, decoded_bytes_per_repeat, - )) + ) } fn decode_external_once( @@ -331,19 +319,18 @@ fn measurement( repeats: usize, samples: Vec, decoded_bytes_per_repeat: usize, -) -> Measurement { - let median_ms = median(samples.clone()); - let mean_ms = samples.iter().sum::() / usize_to_f64(samples.len()); - Measurement { +) -> Result { + let stats = sample_stats(&samples)?; + Ok(Measurement { decoder, batch_size, repeats, sample_ms: samples, - median_ms, - mean_ms, - tiles_per_second_median: usize_to_f64(batch_size) / (median_ms / 1000.0), + median_ms: stats.median, + mean_ms: stats.mean, + tiles_per_second_median: usize_to_f64(batch_size) / (stats.median / 1000.0), decoded_bytes_per_repeat, - } + }) } fn stride(tile: &TileInput, format: PixelFormat) -> usize { @@ -354,21 +341,23 @@ fn output_len(tile: &TileInput, format: PixelFormat) -> usize { stride(tile, format) * tile.dimensions.1 as usize } -fn parse_positive_usize(value: &str, label: &str) -> Result { - let parsed = value - .parse::() - .map_err(|err| format!("invalid {label} {value:?}: {err}"))?; - if parsed == 0 { - return Err(format!("{label} must be > 0")); - } - Ok(parsed) -} +#[cfg(test)] +mod tests { + use j2k_compare::{parse_positive_usize, sample_stats}; -fn median(mut samples: Vec) -> f64 { - samples.sort_by(f64::total_cmp); - samples[samples.len() / 2] -} + #[test] + fn shared_parse_positive_usize_rejects_zero() { + assert_eq!(parse_positive_usize("3", "threads"), Ok(3)); + assert_eq!( + parse_positive_usize("0", "threads"), + Err("threads must be > 0".to_string()) + ); + } -fn usize_to_f64(value: usize) -> f64 { - f64::from(u32::try_from(value).unwrap_or(u32::MAX)) + #[test] + fn shared_sample_stats_reports_mean_and_median() { + let stats = sample_stats(&[9.0, 1.0, 5.0]).expect("stats"); + assert!((stats.median - 5.0).abs() < f64::EPSILON); + assert!((stats.mean - 5.0).abs() < f64::EPSILON); + } } diff --git a/crates/j2k-compare/src/bin/jp2k_roi_batch_compare.rs b/crates/j2k-compare/src/bin/jp2k_roi_batch_compare.rs new file mode 100644 index 00000000..0bf7840c --- /dev/null +++ b/crates/j2k-compare/src/bin/jp2k_roi_batch_compare.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::num::NonZeroUsize; + +use j2k::{ + decode_tiles_region_scaled_into, encode_j2k_lossless, EncodeBackendPreference, + J2kBlockCodingMode, J2kEncodeValidation, J2kLosslessEncodeOptions, J2kLosslessSamples, + TileBatchOptions, TileRegionScaledDecodeJob, +}; +use j2k_compare::{grok, measure_repeated, parse_positive_usize, sample_stats, usize_to_f64}; +use j2k_core::{tile_batch_worker_count, Downscale, PixelFormat, Rect}; +use j2k_test_support::{patterned_rgb8, wrap_jp2_codestream}; + +const DEFAULT_REPEATS: usize = 9; +const DEFAULT_BATCH_SIZE: usize = 16; + +struct CompareCase { + name: &'static str, + bytes: Vec, + roi: Rect, + scale: Downscale, + batch_size: usize, +} + +struct Measurement { + decoder: &'static str, + case_name: &'static str, + repeats: usize, + batch_size: usize, + median_us: f64, + mean_us: f64, + tiles_per_second_median: f64, + decoded_bytes_per_repeat: usize, + samples_us: Vec, +} + +fn main() { + if let Err(error) = run() { + eprintln!("{error}"); + std::process::exit(1); + } +} + +fn run() -> Result<(), String> { + if !grok::is_available() { + return Err( + "in-process Grok is unavailable; install libgrokj2k or set J2K_GROK_SOURCE/J2K_GROK_ROOT" + .to_string(), + ); + } + + let repeats = std::env::var("J2K_ROI_COMPARE_REPEATS") + .ok() + .map(|value| parse_positive_usize(&value, "J2K_ROI_COMPARE_REPEATS")) + .transpose()? + .unwrap_or(DEFAULT_REPEATS); + let workers = std::env::var("J2K_ROI_COMPARE_THREADS") + .ok() + .map(|value| parse_positive_usize(&value, "J2K_ROI_COMPARE_THREADS")) + .transpose()? + .map(|value| NonZeroUsize::new(value).expect("positive value was validated")); + + let cases = compare_cases()?; + println!( + "repeats\t{repeats}\nworkers\t{}\ngrok_available\t{}", + workers.map_or_else(|| "auto".to_string(), |value| value.get().to_string()), + grok::is_available() + ); + println!( + "decoder\tcase\tbatch_size\trepeats\tmedian_us\tmean_us\ttiles_per_second_median\tdecoded_bytes_per_repeat\tsamples_us" + ); + + for case in &cases { + validate_case(case)?; + emit_measurement(&measure_j2k(case, repeats, workers)?); + emit_measurement(&measure_grok(case, repeats, workers)?); + } + Ok(()) +} + +fn compare_cases() -> Result, String> { + let raw_512 = encode_htj2k_rgb_codestream(512, 512)?; + let jp2_512 = wrap_jp2_codestream(&raw_512, 512, 512, 3, 8, 16); + let raw_256 = encode_htj2k_rgb_codestream(256, 256)?; + let jp2_256 = wrap_jp2_codestream(&raw_256, 256, 256, 3, 8, 16); + Ok(vec![ + CompareCase { + name: "htj2k_raw_rgb8_512_roi256_q4_repeated_batch16", + bytes: raw_512, + roi: Rect { + x: 128, + y: 128, + w: 256, + h: 256, + }, + scale: Downscale::Quarter, + batch_size: DEFAULT_BATCH_SIZE, + }, + CompareCase { + name: "htj2k_jp2_rgb8_512_roi256_q4_repeated_batch16", + bytes: jp2_512, + roi: Rect { + x: 128, + y: 128, + w: 256, + h: 256, + }, + scale: Downscale::Quarter, + batch_size: DEFAULT_BATCH_SIZE, + }, + CompareCase { + name: "htj2k_jp2_rgb8_256_roi128_q4_repeated_batch16", + bytes: jp2_256, + roi: Rect { + x: 64, + y: 64, + w: 128, + h: 128, + }, + scale: Downscale::Quarter, + batch_size: DEFAULT_BATCH_SIZE, + }, + ]) +} + +fn encode_htj2k_rgb_codestream(width: u32, height: u32) -> Result, String> { + let pixels = patterned_rgb8(width, height); + let samples = J2kLosslessSamples::new(&pixels, width, height, 3, 8, false) + .map_err(|error| error.to_string())?; + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(2)) + .with_validation(J2kEncodeValidation::External); + Ok(encode_j2k_lossless(samples, &options) + .map_err(|error| error.to_string())? + .codestream) +} + +fn validate_case(case: &CompareCase) -> Result<(), String> { + let ours = decode_j2k_once(case)?; + let theirs = decode_grok_once(case, None)?; + if ours != theirs { + return Err(format!( + "{}: j2k/Grok ROI+scale output mismatch: {} vs {} bytes", + case.name, + ours.len(), + theirs.len() + )); + } + Ok(()) +} + +fn measure_j2k( + case: &CompareCase, + repeats: usize, + workers: Option, +) -> Result { + let (samples, decoded) = measure_repeated(repeats, 1_000_000.0, || { + decode_j2k_once_with_workers(case, workers) + })?; + measurement("j2k", case, repeats, samples, decoded.len()) +} + +fn decode_j2k_once(case: &CompareCase) -> Result, String> { + decode_j2k_once_with_workers(case, None) +} + +fn decode_j2k_once_with_workers( + case: &CompareCase, + workers: Option, +) -> Result, String> { + let output_len = output_len(case); + let stride = output_stride(case); + let mut outputs = vec![vec![0_u8; output_len]; case.batch_size]; + let mut jobs = outputs + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: case.bytes.as_slice(), + out: out.as_mut_slice(), + stride, + roi: case.roi, + scale: case.scale, + }) + .collect::>(); + decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8, TileBatchOptions { workers }) + .map_err(|error| error.to_string())?; + Ok(outputs.into_iter().flatten().collect()) +} + +fn measure_grok( + case: &CompareCase, + repeats: usize, + workers: Option, +) -> Result { + let (samples, decoded) = + measure_repeated(repeats, 1_000_000.0, || decode_grok_once(case, workers))?; + measurement("grok", case, repeats, samples, decoded.len()) +} + +fn decode_grok_once(case: &CompareCase, workers: Option) -> Result, String> { + let worker_count = tile_batch_worker_count( + case.batch_size, + TileBatchOptions { workers }, + std::thread::available_parallelism().map_or(1, NonZeroUsize::get), + ); + let chunk_size = case.batch_size.div_ceil(worker_count); + let reduce = reduce_factor(case.scale)?; + let chunks = (0..case.batch_size) + .collect::>() + .chunks(chunk_size) + .map(<[_]>::to_vec) + .collect::>(); + + let outputs = std::thread::scope(|scope| { + let mut handles = Vec::with_capacity(chunks.len()); + for chunk in chunks { + handles.push(scope.spawn(move || { + chunk + .iter() + .map(|_| grok::decode_rgb_region_scaled(&case.bytes, case.roi, reduce)) + .collect::, _>>() + })); + } + + let mut outputs = Vec::with_capacity(case.batch_size); + for handle in handles { + match handle.join() { + Ok(Ok(mut chunk_outputs)) => outputs.append(&mut chunk_outputs), + Ok(Err(error)) => return Err(error), + Err(payload) => std::panic::resume_unwind(payload), + } + } + Ok(outputs) + })?; + Ok(outputs.into_iter().flatten().collect()) +} + +fn measurement( + decoder: &'static str, + case: &CompareCase, + repeats: usize, + samples: Vec, + decoded_bytes_per_repeat: usize, +) -> Result { + let stats = sample_stats(&samples)?; + Ok(Measurement { + decoder, + case_name: case.name, + repeats, + batch_size: case.batch_size, + median_us: stats.median, + mean_us: stats.mean, + tiles_per_second_median: usize_to_f64(case.batch_size) / (stats.median / 1_000_000.0), + decoded_bytes_per_repeat, + samples_us: samples, + }) +} + +fn emit_measurement(row: &Measurement) { + let samples = row + .samples_us + .iter() + .map(|value| format!("{value:.3}")) + .collect::>() + .join(","); + println!( + "{}\t{}\t{}\t{}\t{:.3}\t{:.3}\t{:.3}\t{}\t{}", + row.decoder, + row.case_name, + row.batch_size, + row.repeats, + row.median_us, + row.mean_us, + row.tiles_per_second_median, + row.decoded_bytes_per_repeat, + samples + ); +} + +fn output_stride(case: &CompareCase) -> usize { + case.roi.scaled_covering(case.scale).w as usize * PixelFormat::Rgb8.bytes_per_pixel() +} + +fn output_len(case: &CompareCase) -> usize { + let scaled = case.roi.scaled_covering(case.scale); + output_stride(case) * scaled.h as usize +} + +fn reduce_factor(scale: Downscale) -> Result { + match scale { + Downscale::None => Ok(0), + Downscale::Half => Ok(1), + Downscale::Quarter => Ok(2), + Downscale::Eighth => Ok(3), + _ => Err(format!("unsupported downscale for Grok compare: {scale:?}")), + } +} diff --git a/crates/signinum-j2k-compare/src/grok.rs b/crates/j2k-compare/src/grok.rs similarity index 54% rename from crates/signinum-j2k-compare/src/grok.rs rename to crates/j2k-compare/src/grok.rs index 7d64a56c..ef05bbea 100644 --- a/crates/signinum-j2k-compare/src/grok.rs +++ b/crates/j2k-compare/src/grok.rs @@ -1,65 +1,41 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_core::Rect; #[cfg(have_grok)] use std::{ffi::c_void, ptr}; +use crate::ExternalDecodeRequest; + pub fn is_available() -> bool { cfg!(have_grok) } -pub fn decode_rgb(bytes: &[u8]) -> Result, String> { - decode(bytes, 3, None, None) -} - -pub fn decode_gray(bytes: &[u8]) -> Result, String> { - decode(bytes, 1, None, None) -} - -pub fn decode_rgb_region(bytes: &[u8], roi: Rect) -> Result, String> { - decode(bytes, 3, None, Some(roi)) -} - -pub fn decode_gray_region(bytes: &[u8], roi: Rect) -> Result, String> { - decode(bytes, 1, None, Some(roi)) -} - -pub fn decode_rgb_region_scaled(bytes: &[u8], roi: Rect, reduce: u32) -> Result, String> { - decode(bytes, 3, Some(reduce), Some(roi)) +pub fn version() -> &'static str { + option_env!("J2K_GROK_VERSION").unwrap_or("unavailable") } -pub fn decode_gray_region_scaled(bytes: &[u8], roi: Rect, reduce: u32) -> Result, String> { - decode(bytes, 1, Some(reduce), Some(roi)) +pub fn library_path() -> &'static str { + option_env!("J2K_GROK_LIB_DIR").unwrap_or("unavailable") } -pub fn decode_rgb_scaled(bytes: &[u8], reduce: u32) -> Result, String> { - decode(bytes, 3, Some(reduce), None) -} - -pub fn decode_gray_scaled(bytes: &[u8], reduce: u32) -> Result, String> { - decode(bytes, 1, Some(reduce), None) -} +crate::external_decode_wrappers!(decode); -fn decode( - bytes: &[u8], - channels: u32, - reduce: Option, - region: Option, -) -> Result, String> { +fn decode(bytes: &[u8], request: ExternalDecodeRequest) -> Result, String> { #[cfg(have_grok)] + // SAFETY: Grok FFI pointers are checked for null and output buffers are bounded. unsafe { let mut out = ptr::null_mut(); let mut out_len = 0_usize; let mut out_width = 0_u32; let mut out_height = 0_u32; - let (has_region, x0, y0, x1, y1) = match region { + let channels = request.color.channels(); + let (has_region, x0, y0, x1, y1) = match request.region { Some(roi) => (1, roi.x, roi.y, roi.x + roi.w, roi.y + roi.h), None => (0, 0, 0, 0, 0), }; - let ok = signinum_grok_decode_u8( + let ok = j2k_grok_decode_u8( bytes.as_ptr(), bytes.len(), - reduce.unwrap_or(0), + request.reduce.unwrap_or(0), has_region, x0, y0, @@ -75,7 +51,7 @@ fn decode( return Err("grok: decode failed".to_string()); } let packed = std::slice::from_raw_parts(out, out_len).to_vec(); - signinum_grok_free(out.cast()); + j2k_grok_free(out.cast()); let expected = out_width as usize * out_height as usize * channels as usize; if packed.len() != expected { return Err(format!( @@ -89,14 +65,14 @@ fn decode( #[cfg(not(have_grok))] { - let _ = (bytes, channels, reduce, region); + let _ = (bytes, request); Err("grok: local library not available".to_string()) } } #[cfg(have_grok)] unsafe extern "C" { - fn signinum_grok_decode_u8( + fn j2k_grok_decode_u8( bytes: *const u8, len: usize, reduce: u32, @@ -111,5 +87,5 @@ unsafe extern "C" { out_width: *mut u32, out_height: *mut u32, ) -> i32; - fn signinum_grok_free(ptr: *mut c_void); + fn j2k_grok_free(ptr: *mut c_void); } diff --git a/crates/signinum-j2k-compare/src/grok_shim.c b/crates/j2k-compare/src/grok_shim.c similarity index 84% rename from crates/signinum-j2k-compare/src/grok_shim.c rename to crates/j2k-compare/src/grok_shim.c index 04fa71ad..45655805 100644 --- a/crates/signinum-j2k-compare/src/grok_shim.c +++ b/crates/j2k-compare/src/grok_shim.c @@ -4,7 +4,7 @@ #include #include -static void signinum_grok_init_once(void) { +static void j2k_grok_init_once(void) { static int initialized = 0; if (!initialized) { grk_initialize(NULL, 1, NULL); @@ -12,7 +12,7 @@ static void signinum_grok_init_once(void) { } } -static uint8_t signinum_clamp_u8(int32_t value) { +static uint8_t j2k_clamp_u8(int32_t value) { if (value < 0) { return 0; } @@ -22,7 +22,7 @@ static uint8_t signinum_clamp_u8(int32_t value) { return (uint8_t)value; } -static int32_t signinum_grok_component_sample(const grk_image_comp *component, +static int32_t j2k_grok_component_sample(const grk_image_comp *component, size_t index) { if (!component || !component->data) { return 0; @@ -48,7 +48,7 @@ static int32_t signinum_grok_component_sample(const grk_image_comp *component, } } -int signinum_grok_decode_u8(const uint8_t *bytes, size_t len, uint32_t reduce, +int j2k_grok_decode_u8(const uint8_t *bytes, size_t len, uint32_t reduce, int has_region, uint32_t x0, uint32_t y0, uint32_t x1, uint32_t y1, uint32_t channels, uint8_t **out_data, size_t *out_len, @@ -69,7 +69,7 @@ int signinum_grok_decode_u8(const uint8_t *bytes, size_t len, uint32_t reduce, *out_width = 0; *out_height = 0; - signinum_grok_init_once(); + j2k_grok_init_once(); memset(&stream_params, 0, sizeof(stream_params)); memset(¶ms, 0, sizeof(params)); memset(&header_info, 0, sizeof(header_info)); @@ -128,13 +128,13 @@ int signinum_grok_decode_u8(const uint8_t *bytes, size_t len, uint32_t reduce, size_t dst = ((size_t)row * width + col) * channels; grk_image_comp *c0 = &image->comps[0]; size_t c0_index = (size_t)row * c0->stride + col; - int32_t v0 = signinum_grok_component_sample(c0, c0_index); + int32_t v0 = j2k_grok_component_sample(c0, c0_index); if (channels == 1) { - packed[dst] = signinum_clamp_u8(v0); + packed[dst] = j2k_clamp_u8(v0); continue; } if (image->numcomps == 1) { - uint8_t gray = signinum_clamp_u8(v0); + uint8_t gray = j2k_clamp_u8(v0); packed[dst] = gray; packed[dst + 1] = gray; packed[dst + 2] = gray; @@ -144,11 +144,11 @@ int signinum_grok_decode_u8(const uint8_t *bytes, size_t len, uint32_t reduce, grk_image_comp *c2 = &image->comps[2]; size_t c1_index = (size_t)row * c1->stride + col; size_t c2_index = (size_t)row * c2->stride + col; - int32_t v1 = signinum_grok_component_sample(c1, c1_index); - int32_t v2 = signinum_grok_component_sample(c2, c2_index); - packed[dst] = signinum_clamp_u8(v0); - packed[dst + 1] = signinum_clamp_u8(v1); - packed[dst + 2] = signinum_clamp_u8(v2); + int32_t v1 = j2k_grok_component_sample(c1, c1_index); + int32_t v2 = j2k_grok_component_sample(c2, c2_index); + packed[dst] = j2k_clamp_u8(v0); + packed[dst + 1] = j2k_clamp_u8(v1); + packed[dst + 2] = j2k_clamp_u8(v2); } } @@ -160,4 +160,4 @@ int signinum_grok_decode_u8(const uint8_t *bytes, size_t len, uint32_t reduce, return 1; } -void signinum_grok_free(void *ptr) { free(ptr); } +void j2k_grok_free(void *ptr) { free(ptr); } diff --git a/crates/j2k-compare/src/lib.rs b/crates/j2k-compare/src/lib.rs new file mode 100644 index 00000000..3b9ffb48 --- /dev/null +++ b/crates/j2k-compare/src/lib.rs @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! In-process external JPEG 2000 comparators for benches and parity tests. + +use std::time::Instant; + +use j2k_core::Rect; + +/// Color shape requested from an external comparator decode. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ExternalDecodeColor { + /// Single-channel 8-bit grayscale output. + Gray, + /// Three-channel interleaved 8-bit RGB output. + Rgb, +} + +impl ExternalDecodeColor { + /// Number of interleaved output channels. + #[must_use] + pub const fn channels(self) -> u32 { + match self { + Self::Gray => 1, + Self::Rgb => 3, + } + } +} + +/// Decode request shared by `OpenJPEG` and `Grok` wrappers. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ExternalDecodeRequest { + /// Output color shape. + pub color: ExternalDecodeColor, + /// Optional OpenJPEG/Grok resolution reduction factor. + pub reduce: Option, + /// Optional source region. + pub region: Option, +} + +impl ExternalDecodeRequest { + /// Full-resolution RGB decode. + #[must_use] + pub const fn rgb() -> Self { + Self { + color: ExternalDecodeColor::Rgb, + reduce: None, + region: None, + } + } + + /// Full-resolution grayscale decode. + #[must_use] + pub const fn gray() -> Self { + Self { + color: ExternalDecodeColor::Gray, + reduce: None, + region: None, + } + } + + /// Region RGB decode. + #[must_use] + pub const fn rgb_region(region: Rect) -> Self { + Self { + region: Some(region), + ..Self::rgb() + } + } + + /// Region grayscale decode. + #[must_use] + pub const fn gray_region(region: Rect) -> Self { + Self { + region: Some(region), + ..Self::gray() + } + } + + /// Reduced-resolution RGB decode. + #[must_use] + pub const fn rgb_scaled(reduce: u32) -> Self { + Self { + reduce: Some(reduce), + ..Self::rgb() + } + } + + /// Reduced-resolution grayscale decode. + #[must_use] + pub const fn gray_scaled(reduce: u32) -> Self { + Self { + reduce: Some(reduce), + ..Self::gray() + } + } + + /// Region plus reduced-resolution RGB decode. + #[must_use] + pub const fn rgb_region_scaled(region: Rect, reduce: u32) -> Self { + Self { + reduce: Some(reduce), + region: Some(region), + ..Self::rgb() + } + } + + /// Region plus reduced-resolution grayscale decode. + #[must_use] + pub const fn gray_region_scaled(region: Rect, reduce: u32) -> Self { + Self { + reduce: Some(reduce), + region: Some(region), + ..Self::gray() + } + } +} + +macro_rules! external_decode_wrappers { + ($decode:ident) => { + pub fn decode_rgb(bytes: &[u8]) -> Result, String> { + $decode(bytes, crate::ExternalDecodeRequest::rgb()) + } + + pub fn decode_gray(bytes: &[u8]) -> Result, String> { + $decode(bytes, crate::ExternalDecodeRequest::gray()) + } + + pub fn decode_rgb_region(bytes: &[u8], roi: j2k_core::Rect) -> Result, String> { + $decode(bytes, crate::ExternalDecodeRequest::rgb_region(roi)) + } + + pub fn decode_gray_region(bytes: &[u8], roi: j2k_core::Rect) -> Result, String> { + $decode(bytes, crate::ExternalDecodeRequest::gray_region(roi)) + } + + pub fn decode_rgb_region_scaled( + bytes: &[u8], + roi: j2k_core::Rect, + reduce: u32, + ) -> Result, String> { + $decode( + bytes, + crate::ExternalDecodeRequest::rgb_region_scaled(roi, reduce), + ) + } + + pub fn decode_gray_region_scaled( + bytes: &[u8], + roi: j2k_core::Rect, + reduce: u32, + ) -> Result, String> { + $decode( + bytes, + crate::ExternalDecodeRequest::gray_region_scaled(roi, reduce), + ) + } + + pub fn decode_rgb_scaled(bytes: &[u8], reduce: u32) -> Result, String> { + $decode(bytes, crate::ExternalDecodeRequest::rgb_scaled(reduce)) + } + + pub fn decode_gray_scaled(bytes: &[u8], reduce: u32) -> Result, String> { + $decode(bytes, crate::ExternalDecodeRequest::gray_scaled(reduce)) + } + }; +} + +pub(crate) use external_decode_wrappers; + +pub mod grok; +pub mod openjpeg; + +/// Summary statistics for benchmark samples. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct SampleStats { + /// Median sample value. + pub median: f64, + /// Arithmetic mean sample value. + pub mean: f64, +} + +/// Parses a positive `usize` from command-line or environment input. +pub fn parse_positive_usize(value: &str, label: &str) -> Result { + let parsed = value + .parse::() + .map_err(|error| format!("invalid {label} {value:?}: {error}"))?; + if parsed == 0 { + return Err(format!("{label} must be > 0")); + } + Ok(parsed) +} + +/// Runs an untimed warmup followed by `repeats` timed samples. +pub fn measure_repeated( + repeats: usize, + seconds_to_units: f64, + mut run: impl FnMut() -> Result, +) -> Result<(Vec, T), String> { + let mut last = run()?; + std::hint::black_box(&last); + let mut samples = Vec::with_capacity(repeats); + for _ in 0..repeats { + let started = Instant::now(); + last = run()?; + samples.push(started.elapsed().as_secs_f64() * seconds_to_units); + std::hint::black_box(&last); + } + Ok((samples, last)) +} + +/// Computes mean and median for non-empty benchmark samples. +pub fn sample_stats(samples: &[f64]) -> Result { + if samples.is_empty() { + return Err("cannot summarize empty benchmark samples".to_string()); + } + let mut sorted = samples.to_vec(); + sorted.sort_by(f64::total_cmp); + Ok(SampleStats { + median: sorted[sorted.len() / 2], + mean: samples.iter().sum::() / usize_to_f64(samples.len()), + }) +} + +/// Lossy but bounded conversion for benchmark ratios. +#[must_use] +pub fn usize_to_f64(value: usize) -> f64 { + f64::from(u32::try_from(value).unwrap_or(u32::MAX)) +} diff --git a/crates/signinum-j2k-compare/src/openjpeg.rs b/crates/j2k-compare/src/openjpeg.rs similarity index 59% rename from crates/signinum-j2k-compare/src/openjpeg.rs rename to crates/j2k-compare/src/openjpeg.rs index 973497d7..813d3db0 100644 --- a/crates/signinum-j2k-compare/src/openjpeg.rs +++ b/crates/j2k-compare/src/openjpeg.rs @@ -6,100 +6,117 @@ use openjpeg_sys::{ opj_set_decode_area, opj_set_decoded_resolution_factor, opj_set_default_decoder_parameters, opj_setup_decoder, opj_stream_create, opj_stream_destroy, opj_stream_set_read_function, opj_stream_set_seek_function, opj_stream_set_skip_function, opj_stream_set_user_data, - opj_stream_set_user_data_length, opj_stream_t, OPJ_BOOL, OPJ_CODEC_FORMAT, OPJ_FALSE, - OPJ_OFF_T, OPJ_SIZE_T, OPJ_STREAM_READ, OPJ_TRUE, + opj_stream_set_user_data_length, opj_stream_t, opj_version, OPJ_BOOL, OPJ_CODEC_FORMAT, + OPJ_FALSE, OPJ_OFF_T, OPJ_SIZE_T, OPJ_STREAM_READ, OPJ_TRUE, }; -use signinum_core::Rect; -use std::{ffi::c_void, ptr, slice}; +use std::{ + ffi::{c_void, CStr}, + ptr, slice, +}; + +use crate::ExternalDecodeRequest; pub fn is_available() -> bool { true } -pub fn decode_rgb(bytes: &[u8]) -> Result, String> { - decode(bytes, 3, None, None) +pub fn version() -> String { + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. + unsafe { + let ptr = opj_version(); + if ptr.is_null() { + "unknown".to_string() + } else { + CStr::from_ptr(ptr).to_string_lossy().into_owned() + } + } } -pub fn decode_gray(bytes: &[u8]) -> Result, String> { - decode(bytes, 1, None, None) +pub fn library_path() -> &'static str { + "openjpeg-sys vendored openjp2" } -pub fn decode_rgb_region(bytes: &[u8], roi: Rect) -> Result, String> { - decode(bytes, 3, None, Some(roi)) -} +crate::external_decode_wrappers!(decode); -pub fn decode_gray_region(bytes: &[u8], roi: Rect) -> Result, String> { - decode(bytes, 1, None, Some(roi)) -} +/// Owns an `OpenJPEG` stream; never null. +struct StreamGuard(*mut opj_stream_t); -pub fn decode_rgb_region_scaled(bytes: &[u8], roi: Rect, reduce: u32) -> Result, String> { - decode(bytes, 3, Some(reduce), Some(roi)) +impl Drop for StreamGuard { + fn drop(&mut self) { + // SAFETY: the pointer came from a successful `opj_stream_create` and + // this guard is its single owner. + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. + unsafe { opj_stream_destroy(self.0) }; + } } -pub fn decode_gray_region_scaled(bytes: &[u8], roi: Rect, reduce: u32) -> Result, String> { - decode(bytes, 1, Some(reduce), Some(roi)) -} +/// Owns an `OpenJPEG` codec; never null. +struct CodecGuard(*mut openjpeg_sys::opj_codec_t); -pub fn decode_rgb_scaled(bytes: &[u8], reduce: u32) -> Result, String> { - decode(bytes, 3, Some(reduce), None) +impl Drop for CodecGuard { + fn drop(&mut self) { + // SAFETY: the pointer came from a successful `opj_create_decompress` + // and this guard is its single owner. + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. + unsafe { opj_destroy_codec(self.0) }; + } } -pub fn decode_gray_scaled(bytes: &[u8], reduce: u32) -> Result, String> { - decode(bytes, 1, Some(reduce), None) +/// Owns the image pointer `opj_read_header` fills in; null until then. +struct ImageGuard(*mut opj_image_t); + +impl Drop for ImageGuard { + fn drop(&mut self) { + if !self.0.is_null() { + // SAFETY: non-null only after OpenJPEG allocated the image, and + // this guard is its single owner. + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. + unsafe { opj_image_destroy(self.0) }; + } + } } -fn decode( - bytes: &[u8], - channels: usize, - reduce: Option, - region: Option, -) -> Result, String> { - let mut image = ptr::null_mut(); +fn decode(bytes: &[u8], request: ExternalDecodeRequest) -> Result, String> { let codec_format = codec_format(bytes)?; - let stream = create_stream(bytes)?; - let codec = create_codec(codec_format)?; - let result = unsafe { - if opj_read_header(stream, codec, &raw mut image) == bool_false() { - Err("openjpeg: failed to read header".to_string()) - } else { - if let Some(reduce) = reduce { - if opj_set_decoded_resolution_factor(codec, reduce) == bool_false() { - return Err("openjpeg: failed to set reduction factor".to_string()); - } - } - if let Some(roi) = region { - if opj_set_decode_area( - codec, - image, - roi.x as i32, - roi.y as i32, - (roi.x + roi.w) as i32, - (roi.y + roi.h) as i32, - ) == bool_false() - { - return Err("openjpeg: failed to set decode area".to_string()); - } + let channels = request.color.channels() as usize; + // Declaration order matters: drops run in reverse (image, codec, stream), + // and the RAII guards free the FFI resources on every exit path, + // including the early error returns below. + let stream = StreamGuard(create_stream(bytes)?); + let codec = CodecGuard(create_codec(codec_format)?); + let mut image = ImageGuard(ptr::null_mut()); + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. + unsafe { + if opj_read_header(stream.0, codec.0, &raw mut image.0) == bool_false() { + return Err("openjpeg: failed to read header".to_string()); + } + if let Some(reduce) = request.reduce { + if opj_set_decoded_resolution_factor(codec.0, reduce) == bool_false() { + return Err("openjpeg: failed to set reduction factor".to_string()); } - if opj_decode(codec, stream, image) == bool_false() { - Err("openjpeg: decode failed".to_string()) - } else { - let packed = pack_image(image, channels)?; - if opj_end_decompress(codec, stream) == bool_false() { - Err("openjpeg: end_decompress failed".to_string()) - } else { - Ok(packed) - } + } + if let Some(roi) = request.region { + if opj_set_decode_area( + codec.0, + image.0, + roi.x as i32, + roi.y as i32, + (roi.x + roi.w) as i32, + (roi.y + roi.h) as i32, + ) == bool_false() + { + return Err("openjpeg: failed to set decode area".to_string()); } } - }; - unsafe { - if !image.is_null() { - opj_image_destroy(image); + if opj_decode(codec.0, stream.0, image.0) == bool_false() { + return Err("openjpeg: decode failed".to_string()); + } + let packed = pack_image(image.0, channels)?; + if opj_end_decompress(codec.0, stream.0) == bool_false() { + return Err("openjpeg: end_decompress failed".to_string()); } - opj_destroy_codec(codec); - opj_stream_destroy(stream); + Ok(packed) } - result } fn codec_format(bytes: &[u8]) -> Result { @@ -113,11 +130,13 @@ fn codec_format(bytes: &[u8]) -> Result { } fn create_stream(bytes: &[u8]) -> Result<*mut opj_stream_t, String> { + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. let stream = unsafe { opj_stream_create(64 * 1024, OPJ_STREAM_READ as OPJ_BOOL) }; if stream.is_null() { return Err("openjpeg: failed to create stream".to_string()); } let user = Box::into_raw(Box::new(MemoryStream::new(bytes))); + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. unsafe { opj_stream_set_user_data(stream, user.cast(), Some(drop_memory_stream)); opj_stream_set_user_data_length(stream, bytes.len() as u64); @@ -129,19 +148,26 @@ fn create_stream(bytes: &[u8]) -> Result<*mut opj_stream_t, String> { } fn create_codec(codec_format: OPJ_CODEC_FORMAT) -> Result<*mut openjpeg_sys::opj_codec_t, String> { + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. let mut params = unsafe { std::mem::zeroed::() }; + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. unsafe { opj_set_default_decoder_parameters(&raw mut params) }; + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. let codec = unsafe { opj_create_decompress(codec_format) }; if codec.is_null() { return Err("openjpeg: failed to create codec".to_string()); } + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. let setup_ok = unsafe { opj_setup_decoder(codec, &raw mut params) }; if setup_ok == bool_false() { + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. unsafe { opj_destroy_codec(codec) }; return Err("openjpeg: setup_decoder failed".to_string()); } + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. let threading_ok = unsafe { opj_codec_set_threads(codec, 1) }; if threading_ok == bool_false() { + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. unsafe { opj_destroy_codec(codec) }; return Err("openjpeg: codec_set_threads failed".to_string()); } @@ -152,10 +178,12 @@ fn pack_image(image: *mut opj_image_t, channels: usize) -> Result, Strin if image.is_null() { return Err("openjpeg: null image".to_string()); } + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. let image_ref = unsafe { &*image }; if image_ref.numcomps == 0 || image_ref.comps.is_null() { return Err("openjpeg: image has no components".to_string()); } + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. let comp0 = unsafe { &*image_ref.comps }; let width = comp0.w as usize; let height = comp0.h as usize; @@ -190,6 +218,7 @@ fn read_component( full_width: usize, full_height: usize, ) -> Result { + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. let comp = unsafe { image .comps @@ -205,6 +234,7 @@ fn read_component( if stride == 0 || height == 0 { return Err("openjpeg: component has zero-sized output".to_string()); } + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. let data = unsafe { slice::from_raw_parts(comp.data, stride * height) }; let comp_col = col.saturating_mul(stride) / full_width.max(1); let comp_row = row.saturating_mul(height) / full_height.max(1); @@ -250,6 +280,7 @@ unsafe extern "C" fn read_memory( ) -> OPJ_SIZE_T { // SAFETY: OpenJPEG passes back the `MemoryStream` pointer registered in // `create_stream`; the stream owns it until `drop_memory_stream` runs. + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. let state = unsafe { &mut *user_data.cast::() }; let remaining = state.len.saturating_sub(state.offset); if remaining == 0 { @@ -258,6 +289,7 @@ unsafe extern "C" fn read_memory( let count = remaining.min(bytes); // SAFETY: `count` is bounded by the remaining source length, and OpenJPEG // provides a writable buffer of the requested size for the read callback. + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. unsafe { ptr::copy_nonoverlapping(state.ptr.add(state.offset), buffer.cast::(), count); } @@ -268,6 +300,7 @@ unsafe extern "C" fn read_memory( unsafe extern "C" fn skip_memory(bytes: OPJ_OFF_T, user_data: *mut c_void) -> OPJ_OFF_T { // SAFETY: OpenJPEG passes back the `MemoryStream` pointer registered in // `create_stream`; the stream owns it until `drop_memory_stream` runs. + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. let state = unsafe { &mut *user_data.cast::() }; if bytes < 0 { return -1; @@ -283,6 +316,7 @@ unsafe extern "C" fn skip_memory(bytes: OPJ_OFF_T, user_data: *mut c_void) -> OP unsafe extern "C" fn seek_memory(bytes: OPJ_OFF_T, user_data: *mut c_void) -> OPJ_BOOL { // SAFETY: OpenJPEG passes back the `MemoryStream` pointer registered in // `create_stream`; the stream owns it until `drop_memory_stream` runs. + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. let state = unsafe { &mut *user_data.cast::() }; if bytes < 0 || bytes as usize > state.len { return bool_false(); @@ -294,6 +328,7 @@ unsafe extern "C" fn seek_memory(bytes: OPJ_OFF_T, user_data: *mut c_void) -> OP unsafe extern "C" fn drop_memory_stream(user_data: *mut c_void) { // SAFETY: `create_stream` allocated this pointer with `Box::into_raw` and // registered this callback as the single owner responsible for freeing it. + // SAFETY: OpenJPEG FFI calls use checked handles and validated component buffers. unsafe { drop(Box::from_raw(user_data.cast::())) }; } diff --git a/crates/j2k-compare/tests/bench_harness.rs b/crates/j2k-compare/tests/bench_harness.rs new file mode 100644 index 00000000..272f9b90 --- /dev/null +++ b/crates/j2k-compare/tests/bench_harness.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[test] +fn roi_batch_compare_binary_exposes_grok_wsi_surfaces() { + let source = include_str!("../src/bin/jp2k_roi_batch_compare.rs"); + + for expected in [ + "htj2k_raw_rgb8_512_roi256_q4_repeated_batch16", + "htj2k_jp2_rgb8_512_roi256_q4_repeated_batch16", + "htj2k_jp2_rgb8_256_roi128_q4_repeated_batch16", + "j2k", + "grok", + ] { + assert!( + source.contains(expected), + "ROI batch compare binary is missing `{expected}`" + ); + } +} diff --git a/crates/j2k-compare/tests/in_process_parity.rs b/crates/j2k-compare/tests/in_process_parity.rs new file mode 100644 index 00000000..0223cd6a --- /dev/null +++ b/crates/j2k-compare/tests/in_process_parity.rs @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::J2kDecoder; +use j2k_core::{Downscale, PixelFormat, Rect}; +use j2k_test_support::{gradient_u8, write_pnm}; +use std::{ + fs, + path::PathBuf, + process::Command, + sync::{ + atomic::{AtomicUsize, Ordering}, + OnceLock, + }, +}; + +#[test] +fn openjpeg_in_process_matches_j2k_rgb_fixture() { + let Some(input) = bench_fixture_rgb() else { + return; + }; + let ours = j2k_rgb(&input); + let theirs = j2k_compare::openjpeg::decode_rgb(&input).expect("openjpeg"); + assert_eq!(ours, theirs); +} + +#[test] +fn openjpeg_in_process_region_matches_j2k_rgb_fixture() { + let Some(input) = bench_fixture_rgb() else { + return; + }; + let roi = Rect { + x: 16, + y: 24, + w: 64, + h: 64, + }; + let ours = j2k_rgb_region(&input, roi); + let theirs = j2k_compare::openjpeg::decode_rgb_region(&input, roi).expect("openjpeg"); + assert_eq!(ours, theirs); +} + +#[test] +fn openjpeg_in_process_scaled_matches_j2k_rgb_fixture() { + let Some(input) = bench_fixture_rgb() else { + return; + }; + let ours = j2k_rgb_scaled_q4(&input); + let theirs = j2k_compare::openjpeg::decode_rgb_scaled(&input, 2).expect("openjpeg"); + assert_eq!(ours, theirs); +} + +#[test] +fn openjpeg_in_process_region_scaled_matches_j2k_rgb_fixture() { + let Some(input) = bench_fixture_rgb() else { + return; + }; + let roi = Rect { + x: 16, + y: 24, + w: 64, + h: 64, + }; + let ours = j2k_rgb_region_scaled_q4(&input, roi); + let theirs = j2k_compare::openjpeg::decode_rgb_region_scaled(&input, roi, 2).expect("openjpeg"); + assert_eq!(ours, theirs); +} + +#[test] +fn grok_in_process_matches_j2k_rgb_fixture() { + if !j2k_compare::grok::is_available() { + assert!( + !require_grok(), + "J2K_REQUIRE_GROK is set but in-process Grok is unavailable" + ); + return; + } + let Some(input) = bench_fixture_rgb() else { + return; + }; + let ours = j2k_rgb(&input); + let theirs = j2k_compare::grok::decode_rgb(&input).expect("grok"); + assert_eq!(ours, theirs); +} + +#[test] +fn grok_in_process_region_matches_j2k_rgb_fixture() { + if !j2k_compare::grok::is_available() { + assert!( + !require_grok(), + "J2K_REQUIRE_GROK is set but in-process Grok is unavailable" + ); + return; + } + let Some(input) = bench_fixture_rgb() else { + return; + }; + let roi = Rect { + x: 16, + y: 24, + w: 64, + h: 64, + }; + let ours = j2k_rgb_region(&input, roi); + let theirs = j2k_compare::grok::decode_rgb_region(&input, roi).expect("grok"); + assert_eq!(ours, theirs); +} + +#[test] +fn grok_in_process_scaled_matches_j2k_rgb_fixture() { + if !j2k_compare::grok::is_available() { + assert!( + !require_grok(), + "J2K_REQUIRE_GROK is set but in-process Grok is unavailable" + ); + return; + } + let Some(input) = bench_fixture_rgb() else { + return; + }; + let ours = j2k_rgb_scaled_q4(&input); + let theirs = j2k_compare::grok::decode_rgb_scaled(&input, 2).expect("grok"); + assert_eq!(ours, theirs); +} + +#[test] +fn grok_in_process_region_scaled_matches_j2k_rgb_fixture() { + if !j2k_compare::grok::is_available() { + assert!( + !require_grok(), + "J2K_REQUIRE_GROK is set but in-process Grok is unavailable" + ); + return; + } + let Some(input) = bench_fixture_rgb() else { + return; + }; + let roi = Rect { + x: 16, + y: 24, + w: 64, + h: 64, + }; + let ours = j2k_rgb_region_scaled_q4(&input, roi); + let theirs = j2k_compare::grok::decode_rgb_region_scaled(&input, roi, 2).expect("grok"); + assert_eq!(ours, theirs); +} + +fn bench_fixture_rgb() -> Option> { + let pixels = gradient_u8(128, 128, 3); + openjpeg_encode_jp2("in_process_parity_rgb", &pixels, 128, 128) +} + +fn j2k_rgb(bytes: &[u8]) -> Vec { + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + let dims = decoder.info().dimensions; + let mut out = vec![0_u8; dims.0 as usize * dims.1 as usize * 3]; + decoder + .decode_into(&mut out, dims.0 as usize * 3, PixelFormat::Rgb8) + .expect("decode"); + out +} + +fn j2k_rgb_region(bytes: &[u8], roi: Rect) -> Vec { + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + let mut out = vec![0_u8; roi.w as usize * roi.h as usize * 3]; + decoder + .decode_region_into( + &mut j2k::J2kScratchPool::new(), + &mut out, + roi.w as usize * 3, + PixelFormat::Rgb8, + roi, + ) + .expect("region decode"); + out +} + +fn j2k_rgb_scaled_q4(bytes: &[u8]) -> Vec { + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + let dims = decoder.info().dimensions; + let scaled = (dims.0.div_ceil(4), dims.1.div_ceil(4)); + let mut out = vec![0_u8; scaled.0 as usize * scaled.1 as usize * 3]; + decoder + .decode_scaled_into( + &mut j2k::J2kScratchPool::new(), + &mut out, + scaled.0 as usize * 3, + PixelFormat::Rgb8, + Downscale::Quarter, + ) + .expect("scaled decode"); + out +} + +fn j2k_rgb_region_scaled_q4(bytes: &[u8], roi: Rect) -> Vec { + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + let scaled = roi.scaled_covering(Downscale::Quarter); + let mut out = vec![0_u8; scaled.w as usize * scaled.h as usize * 3]; + decoder + .decode_region_scaled_into( + &mut j2k::J2kScratchPool::new(), + &mut out, + scaled.w as usize * 3, + PixelFormat::Rgb8, + roi, + Downscale::Quarter, + ) + .expect("region scaled decode"); + out +} + +fn openjpeg_encode_jp2(name: &str, pixels: &[u8], width: u32, height: u32) -> Option> { + let Some(bin) = openjpeg_compress_bin() else { + assert!( + !require_openjpeg(), + "J2K_REQUIRE_OPENJPEG is set but opj_compress was not found" + ); + return None; + }; + let dir = openjpeg_temp_dir(); + let unique = next_temp_suffix(); + let src_path = dir.join(format!("{name}-{unique}.ppm")); + let out_path = dir.join(format!("{name}-{unique}.jp2")); + write_pnm(&src_path, pixels, width, height, 3).ok()?; + let status = Command::new(bin) + .arg("-i") + .arg(&src_path) + .arg("-o") + .arg(&out_path) + .status() + .ok()?; + if !status.success() { + assert!( + !require_openjpeg(), + "J2K_REQUIRE_OPENJPEG is set but opj_compress failed" + ); + return None; + } + fs::read(out_path).ok() +} + +fn next_temp_suffix() -> usize { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + COUNTER.fetch_add(1, Ordering::Relaxed) +} + +fn openjpeg_compress_bin() -> Option { + static OPENJPEG_COMPRESS: OnceLock> = OnceLock::new(); + OPENJPEG_COMPRESS + .get_or_init(|| { + if let Some(path) = std::env::var_os("J2K_OPENJPEG_COMPRESS_BIN") { + let path = PathBuf::from(path); + if path.exists() { + return Some(path); + } + } + let default = PathBuf::from("/opt/homebrew/bin/opj_compress"); + default.exists().then_some(default) + }) + .clone() +} + +fn require_openjpeg() -> bool { + std::env::var_os("J2K_REQUIRE_OPENJPEG").is_some() +} + +fn require_grok() -> bool { + std::env::var_os("J2K_REQUIRE_GROK").is_some() +} + +fn openjpeg_temp_dir() -> PathBuf { + static DIR: OnceLock = OnceLock::new(); + DIR.get_or_init(|| { + let dir = std::env::temp_dir().join("j2k-openjpeg-tests"); + fs::create_dir_all(&dir).expect("create openjpeg temp dir"); + dir + }) + .clone() +} diff --git a/crates/signinum-core/Cargo.toml b/crates/j2k-core/Cargo.toml similarity index 64% rename from crates/signinum-core/Cargo.toml rename to crates/j2k-core/Cargo.toml index cdd4cdf6..fe81e02d 100644 --- a/crates/signinum-core/Cargo.toml +++ b/crates/j2k-core/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "signinum-core" -description = "Shared decode contracts and types for signinum" +name = "j2k-core" +description = "Shared codec contracts and backend types for j2k" version.workspace = true edition.workspace = true rust-version.workspace = true @@ -10,8 +10,12 @@ keywords.workspace = true categories.workspace = true readme = "README.md" +[package.metadata.docs.rs] +all-features = true +targets = [] + [lib] -name = "signinum_core" +name = "j2k_core" path = "src/lib.rs" [dependencies] @@ -21,11 +25,15 @@ thiserror = { workspace = true } serde_json = "1" [lints.rust] +# `GpuAbi` provides the shared host/device byte-view contract used by adapter +# crates; every unsafe operation remains audited with `unsafe_op_in_unsafe_fn`. +unsafe_code = "allow" unsafe_op_in_unsafe_fn = "deny" unreachable_pub = "warn" [lints.clippy] pedantic = { level = "warn", priority = -1 } +undocumented_unsafe_blocks = "warn" module_name_repetitions = "allow" must_use_candidate = "allow" missing_errors_doc = "allow" diff --git a/crates/j2k-core/README.md b/crates/j2k-core/README.md new file mode 100644 index 00000000..fd74bd8d --- /dev/null +++ b/crates/j2k-core/README.md @@ -0,0 +1,9 @@ +# j2k-core + +Shared contracts for J2K codec crates. + +This crate defines backend requests, pixel formats, decode outcomes, device +surfaces, scratch-pool traits, row output traits, and tile-batch decode traits. +It does not implement a codec. + +Use it when implementing or integrating J2K codec adapters. diff --git a/crates/j2k-core/src/accelerator.rs b/crates/j2k-core/src/accelerator.rs new file mode 100644 index 00000000..9b4702b6 --- /dev/null +++ b/crates/j2k-core/src/accelerator.rs @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: Apache-2.0 + +use core::mem::{size_of, size_of_val}; +use core::slice; + +use crate::backend::BackendKind; + +/// Residency of an accelerator-visible surface or buffer. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum SurfaceResidency { + /// Host memory owned by CPU code. + Host, + /// Pixels were produced directly by a CUDA decode path. + CudaResidentDecode, + /// Pixels were decoded on CPU and uploaded into CUDA memory. + CpuStagedCudaUpload, + /// Pixels were produced directly by a Metal decode path. + MetalResidentDecode, + /// Pixels were decoded on CPU and uploaded into Metal memory. + CpuStagedMetalUpload, + /// Device-local memory owned by a backend. + Device(BackendKind), + /// Host/device shared memory for the backend. + Shared(BackendKind), +} + +impl SurfaceResidency { + /// Generic residency for a backend-produced surface. + #[must_use] + pub const fn for_backend(backend: BackendKind) -> Self { + match backend { + BackendKind::Cpu => Self::Host, + BackendKind::Metal => Self::Device(BackendKind::Metal), + BackendKind::Cuda => Self::Device(BackendKind::Cuda), + } + } +} + +/// Execution counters reported by accelerator sessions and surfaces. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +pub struct ExecutionStats { + /// Number of submitted backend command buffers, streams, or equivalent jobs. + pub submissions: u64, + /// Number of kernel or shader dispatches. + pub kernel_dispatches: u64, + /// Bytes uploaded from host to device. + pub upload_bytes: u64, + /// Bytes read back from device to host. + pub readback_bytes: u64, + /// Backend-reported execution time in microseconds, when available. + pub device_us: u128, +} + +impl ExecutionStats { + /// Construct empty execution statistics. + pub const fn new() -> Self { + Self { + submissions: 0, + kernel_dispatches: 0, + upload_bytes: 0, + readback_bytes: 0, + device_us: 0, + } + } + + /// Saturating sum of two execution-stat blocks. + #[must_use] + pub const fn saturating_add(self, other: Self) -> Self { + Self { + submissions: self.submissions.saturating_add(other.submissions), + kernel_dispatches: self + .kernel_dispatches + .saturating_add(other.kernel_dispatches), + upload_bytes: self.upload_bytes.saturating_add(other.upload_bytes), + readback_bytes: self.readback_bytes.saturating_add(other.readback_bytes), + device_us: self.device_us.saturating_add(other.device_us), + } + } +} + +/// Opaque byte range in accelerator-visible memory. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct DeviceMemoryRange { + /// Backend that owns the range. + pub backend: BackendKind, + /// Backend-local allocation identifier. Backends define its meaning. + pub allocation: u64, + /// Byte offset inside the allocation. + pub offset: usize, + /// Byte length of the range. + pub len: usize, +} + +impl DeviceMemoryRange { + /// Construct a backend-local memory range. + pub const fn new(backend: BackendKind, allocation: u64, offset: usize, len: usize) -> Self { + Self { + backend, + allocation, + offset, + len, + } + } +} + +/// Shared session contract for caller-owned accelerator runtime state. +pub trait AcceleratorSession { + /// Backend owned by this session. + fn backend_kind(&self) -> BackendKind; + + /// Execution statistics accumulated by this session. + fn execution_stats(&self) -> ExecutionStats { + ExecutionStats::default() + } +} + +/// Backend failure class used before mapping into codec-specific errors. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum BackendFailureKind { + /// Requested backend is not available on this host or build. + Unavailable, + /// Request shape is outside the backend implementation. + Unsupported, + /// Runtime/device/queue/context setup failed. + Runtime, + /// Kernel or shader execution failed. + Kernel, + /// Backend state lock or session state was poisoned. + StatePoisoned, + /// Backend rejected malformed caller input. + InvalidInput, + /// Backend allocation failed. + OutOfMemory, +} + +/// Marker trait for host-side values whose memory layout is part of a GPU ABI. +/// +/// # Safety +/// Implementers must guarantee that `Self` has a stable host/shader layout for +/// every backend that consumes it. In practice this means using `#[repr(C)]` or +/// an equivalent explicit layout, avoiding padding-sensitive interpretation +/// unless tests cover the exact byte layout, and keeping all fields plain data. +pub unsafe trait GpuAbi: Copy + 'static { + /// Human-readable ABI name used in layout-test failures. + const NAME: &'static str; + + /// View one value as bytes. + fn as_bytes(value: &Self) -> &[u8] { + // SAFETY: The trait contract requires `Self` to be plain GPU ABI data. + unsafe { slice::from_raw_parts(core::ptr::from_ref(value).cast::(), size_of::()) } + } + + /// View a slice of values as bytes. + fn slice_as_bytes(values: &[Self]) -> &[u8] { + // SAFETY: The trait contract requires `Self` to be plain GPU ABI data. + unsafe { slice::from_raw_parts(values.as_ptr().cast::(), size_of_val(values)) } + } + + /// Mutably view a slice of values as bytes. + fn slice_as_bytes_mut(values: &mut [Self]) -> &mut [u8] { + // SAFETY: The trait contract requires `Self` to be plain GPU ABI data. + unsafe { slice::from_raw_parts_mut(values.as_mut_ptr().cast::(), size_of_val(values)) } + } +} + +macro_rules! impl_gpu_abi_primitive { + ($($ty:ty),* $(,)?) => { + $( + // SAFETY: Primitive numeric types are plain data with stable Rust layouts + // accepted by the shader ABI helpers as scalar values. + unsafe impl GpuAbi for $ty { + const NAME: &'static str = stringify!($ty); + } + )* + }; +} + +impl_gpu_abi_primitive!(u8, i8, u16, i16, u32, i32, u64, i64, f32, f64); + +// SAFETY: Arrays preserve element order and contain no extra non-element state; +// the element type supplies the GPU ABI layout contract. +unsafe impl GpuAbi for [T; N] +where + T: GpuAbi, +{ + const NAME: &'static str = "array"; +} diff --git a/crates/signinum-core/src/backend.rs b/crates/j2k-core/src/backend.rs similarity index 57% rename from crates/signinum-core/src/backend.rs rename to crates/j2k-core/src/backend.rs index ef7ea715..66c17893 100644 --- a/crates/signinum-core/src/backend.rs +++ b/crates/j2k-core/src/backend.rs @@ -1,34 +1,57 @@ // SPDX-License-Identifier: Apache-2.0 -#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] -compile_error!("signinum-core only supports x86_64 and aarch64 targets"); - use core::sync::atomic::{AtomicU8, Ordering}; +/// Runtime backend that executes codec work. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum BackendKind { + /// Portable CPU implementation. Cpu, + /// Apple Metal implementation. Metal, + /// NVIDIA CUDA implementation. Cuda, } +/// Caller preference for backend selection. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum BackendRequest { + /// Let the codec choose the best available backend. #[default] Auto, + /// Force the portable CPU backend. Cpu, + /// Force Metal and fail if unavailable. Metal, + /// Force CUDA and fail if unavailable. Cuda, } +impl BackendRequest { + /// Adaptive accelerated route: let the codec choose CPU and device stages + /// for benchmark-approved workload shapes. + pub const ACCELERATED: Self = Self::Auto; + /// Explicit portable CPU route. + pub const CPU_ONLY: Self = Self::Cpu; + /// Strict Metal route; fail when Metal is unavailable or unsupported. + pub const STRICT_METAL: Self = Self::Metal; + /// Strict CUDA route; fail when CUDA is unavailable or unsupported. + pub const STRICT_CUDA: Self = Self::Cuda; +} + +/// CPU SIMD feature flags detected for the current host. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] pub struct CpuFeatures { + /// True when AVX2 is available and enabled by the OS. pub avx2: bool, + /// True when SSE4.1 is available. pub sse41: bool, + /// True when NEON is available. pub neon: bool, } impl CpuFeatures { + /// Detect CPU SIMD features once and reuse the cached result. pub fn detect() -> Self { static DETECTED: AtomicU8 = AtomicU8::new(0); @@ -61,6 +84,11 @@ impl CpuFeatures { neon: true, } } + + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + { + Self::default() + } } const fn to_cache_byte(self) -> u8 { @@ -110,20 +138,34 @@ fn detect_x86_avx2() -> bool { return false; } + let max_leaf = core::arch::x86_64::__cpuid(0).eax; + if max_leaf < 7 { + return false; + } + let leaf7 = core::arch::x86_64::__cpuid_count(7, 0); (leaf7.ebx & (1 << 5)) != 0 } +/// Backend availability for a codec/runtime combination. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct BackendCapabilities { + /// Host CPU feature set. pub cpu: CpuFeatures, + /// True when Metal is available to this crate. pub metal: bool, + /// True when CUDA is available to this crate. pub cuda: bool, } impl BackendCapabilities { + /// Return default capabilities implied by the current build target. + /// + /// This does not probe GPU devices or runtime libraries. Codec facades and + /// adapters must further gate the returned device flags by their compiled + /// features and runtime availability. #[must_use] - pub fn detect() -> Self { + pub fn compile_time_defaults() -> Self { Self { cpu: CpuFeatures::detect(), metal: cfg!(target_os = "macos"), @@ -131,6 +173,7 @@ impl BackendCapabilities { } } + /// Return whether a backend request can be satisfied. #[must_use] pub const fn supports(self, request: BackendRequest) -> bool { match request { @@ -140,22 +183,31 @@ impl BackendCapabilities { } } + /// Resolve a backend request to the concrete backend that should run. + /// + /// `Auto` resolves to CPU here. Workload-aware device promotion belongs in + /// codec-specific route planners that have benchmark evidence for the + /// requested operation. #[must_use] pub fn resolve(self, request: BackendRequest) -> Option { match request { - BackendRequest::Auto => { - if self.metal { - Some(BackendKind::Metal) - } else if self.cuda { - Some(BackendKind::Cuda) - } else { - Some(BackendKind::Cpu) - } - } - BackendRequest::Cpu => Some(BackendKind::Cpu), + BackendRequest::Auto | BackendRequest::Cpu => Some(BackendKind::Cpu), BackendRequest::Metal if self.metal => Some(BackendKind::Metal), BackendRequest::Cuda if self.cuda => Some(BackendKind::Cuda), BackendRequest::Metal | BackendRequest::Cuda => None, } } + + /// Return an available accelerator backend without implying it should be + /// selected for a workload. + #[must_use] + pub const fn first_available_accelerator(self) -> Option { + if self.metal { + Some(BackendKind::Metal) + } else if self.cuda { + Some(BackendKind::Cuda) + } else { + None + } + } } diff --git a/crates/signinum-core/src/batch.rs b/crates/j2k-core/src/batch.rs similarity index 50% rename from crates/signinum-core/src/batch.rs rename to crates/j2k-core/src/batch.rs index a5a6734e..0f120fd3 100644 --- a/crates/signinum-core/src/batch.rs +++ b/crates/j2k-core/src/batch.rs @@ -3,6 +3,8 @@ use alloc::vec::Vec; use core::num::NonZeroUsize; +use crate::{scale::Downscale, types::Rect}; + /// Worker configuration for CPU tile batches. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct TileBatchOptions { @@ -10,9 +12,85 @@ pub struct TileBatchOptions { pub workers: Option, } +impl TileBatchOptions { + /// Construct tile-batch options with an optional fixed worker count. + pub const fn new(workers: Option) -> Self { + Self { workers } + } +} + /// Indexed result produced by one tile-batch worker. pub type IndexedBatchResult = (usize, Result); +/// One full-tile decode request. +pub struct TileDecodeJob<'i, 'o> { + /// Compressed tile bytes. + pub input: &'i [u8], + /// Caller-owned output buffer for this tile. + pub out: &'o mut [u8], + /// Distance in bytes between output rows. + pub stride: usize, +} + +/// One region tile decode request. +pub struct TileRegionDecodeJob<'i, 'o> { + /// Compressed tile bytes. + pub input: &'i [u8], + /// Caller-owned output buffer for this tile. + pub out: &'o mut [u8], + /// Distance in bytes between output rows. + pub stride: usize, + /// Region of interest in source-image coordinates. + pub roi: Rect, +} + +/// One scaled tile decode request. +pub struct TileScaledDecodeJob<'i, 'o> { + /// Compressed tile bytes. + pub input: &'i [u8], + /// Caller-owned output buffer for this tile. + pub out: &'o mut [u8], + /// Distance in bytes between output rows. + pub stride: usize, + /// Downscale factor applied to the full-tile decode. + pub scale: Downscale, +} + +/// One region+scaled tile decode request. +pub struct TileRegionScaledDecodeJob<'i, 'o> { + /// Compressed tile bytes. + pub input: &'i [u8], + /// Caller-owned output buffer for this tile. + pub out: &'o mut [u8], + /// Distance in bytes between output rows. + pub stride: usize, + /// Region of interest in source-image coordinates. + pub roi: Rect, + /// Downscale factor applied to the region decode. + pub scale: Downscale, +} + +/// Error returned by tile batches, annotated with the failing input index. +#[derive(Debug)] +pub struct TileBatchError { + /// Index of the first failing tile in input order. + pub index: usize, + /// Decode error reported for that tile. + pub source: E, +} + +impl core::fmt::Display for TileBatchError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "tile {} decode failed: {}", self.index, self.source) + } +} + +impl core::error::Error for TileBatchError { + fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { + Some(&self.source) + } +} + /// Resolve the number of CPU workers for a tile batch. /// /// `available_workers` should be the host's available parallelism. Passing diff --git a/crates/signinum-core/src/buffer.rs b/crates/j2k-core/src/buffer.rs similarity index 77% rename from crates/signinum-core/src/buffer.rs rename to crates/j2k-core/src/buffer.rs index 42371a10..b9282a52 100644 --- a/crates/signinum-core/src/buffer.rs +++ b/crates/j2k-core/src/buffer.rs @@ -2,6 +2,25 @@ use crate::{error::BufferError, pixel::PixelFormat}; +/// Default cap for host-side codec-owned allocations. +pub const DEFAULT_MAX_HOST_ALLOCATION_BYTES: usize = 512 * 1024 * 1024; + +/// Returns `len` if it is at or below `cap`. +pub fn ensure_allocation_within_cap( + len: usize, + cap: usize, + what: &'static str, +) -> Result { + if len > cap { + return Err(BufferError::AllocationTooLarge { + requested: len, + cap, + what, + }); + } + Ok(len) +} + /// Returns the number of bytes required for a strided image output buffer. /// /// The returned length covers the last written byte of the final row and does @@ -24,6 +43,18 @@ pub fn strided_output_len( }) } +/// Returns the strided output byte length, rejecting requests over `cap`. +pub fn strided_output_len_capped( + dimensions: (u32, u32), + stride: usize, + fmt: PixelFormat, + cap: usize, + what: &'static str, +) -> Result { + let len = strided_output_len(dimensions, stride, fmt)?; + ensure_allocation_within_cap(len, cap, what) +} + /// Validates that `out_len` and `stride` can hold an image output. pub fn validate_strided_output_buffer( dimensions: (u32, u32), diff --git a/crates/j2k-core/src/context.rs b/crates/j2k-core/src/context.rs new file mode 100644 index 00000000..1959155a --- /dev/null +++ b/crates/j2k-core/src/context.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 + +/// Cache hit/miss counters reported by codec contexts. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct CacheStats { + /// Number of cache lookups that reused existing state. + pub hits: u64, + /// Number of cache lookups that had to build new state. + pub misses: u64, + /// Number of currently occupied cache slots. + pub occupied_slots: u64, + /// Number of cache entries evicted by later insertions. + pub evictions: u64, +} + +impl CacheStats { + /// Construct cache statistics from explicit counters. + pub const fn new(hits: u64, misses: u64) -> Self { + Self { + hits, + misses, + occupied_slots: 0, + evictions: 0, + } + } + + /// Construct cache statistics from full counters. + pub const fn with_slots(hits: u64, misses: u64, occupied_slots: u64, evictions: u64) -> Self { + Self { + hits, + misses, + occupied_slots, + evictions, + } + } +} + +/// Reusable codec state cached across decode calls. +pub trait CodecContext: Default + Send { + /// Drop cached state while keeping the context reusable. + fn clear(&mut self); + + /// Return current cache counters, when the codec tracks them. + fn cache_stats(&self) -> CacheStats { + CacheStats::default() + } +} + +/// Wrapper that owns codec context state for repeated decode calls. +#[derive(Debug, Default)] +pub struct DecoderContext { + codec: C, +} + +impl DecoderContext { + /// Construct an empty decoder context. + pub fn new() -> Self { + Self { + codec: C::default(), + } + } + + /// Borrow the codec-specific context. + pub fn codec(&self) -> &C { + &self.codec + } + + /// Mutably borrow the codec-specific context. + pub fn codec_mut(&mut self) -> &mut C { + &mut self.codec + } + + /// Clear cached codec state. + pub fn clear(&mut self) { + self.codec.clear(); + } + + /// Return cache counters from the codec-specific context. + pub fn cache_stats(&self) -> CacheStats { + self.codec.cache_stats() + } + + /// Consume the wrapper and return the codec-specific context. + pub fn into_inner(self) -> C { + self.codec + } +} diff --git a/crates/j2k-core/src/device.rs b/crates/j2k-core/src/device.rs new file mode 100644 index 00000000..a9253851 --- /dev/null +++ b/crates/j2k-core/src/device.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 + +use crate::backend::BackendRequest; + +/// Validate a backend request for adapters that support CPU fallback and CUDA output. +pub const fn validate_cuda_surface_backend_request( + request: BackendRequest, +) -> Result<(), BackendRequest> { + match request { + BackendRequest::Cpu | BackendRequest::Auto | BackendRequest::Cuda => Ok(()), + BackendRequest::Metal => Err(request), + } +} diff --git a/crates/j2k-core/src/error.rs b/crates/j2k-core/src/error.rs new file mode 100644 index 00000000..ac6e6cd8 --- /dev/null +++ b/crates/j2k-core/src/error.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 + +use crate::{pixel::PixelFormat, sample::SampleType}; + +/// Buffer validation and copy errors. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum BufferError { + /// Destination buffer is too small for the requested output. + #[error("output buffer too small: required {required} bytes, have {have}")] + OutputTooSmall { + /// Required destination length in bytes. + required: usize, + /// Actual destination length in bytes. + have: usize, + }, + /// Source buffer is too small for the requested copy/decode. + #[error("input buffer too small: required {required} bytes, have {have}")] + InputTooSmall { + /// Required source length in bytes. + required: usize, + /// Actual source length in bytes. + have: usize, + }, + /// A byte-count computation overflowed. + #[error("buffer size overflow while computing {what}")] + SizeOverflow { + /// Name of the size being computed. + what: &'static str, + }, + /// A requested allocation exceeds the configured safety cap. + #[error("{what} allocation too large: requested {requested} bytes, cap {cap}")] + AllocationTooLarge { + /// Requested byte count. + requested: usize, + /// Configured byte cap. + cap: usize, + /// Name of the allocation being checked. + what: &'static str, + }, + /// Output stride cannot hold one decoded row. + #[error("stride {stride} is smaller than row width {row_bytes}")] + StrideTooSmall { + /// Required row width in bytes. + row_bytes: usize, + /// Supplied stride in bytes. + stride: usize, + }, + /// Output stride does not meet backend alignment requirements. + #[error("stride {stride} is not aligned to {align}")] + StrideNotAligned { + /// Supplied stride in bytes. + stride: usize, + /// Required byte alignment. + align: usize, + }, + /// Requested pixel format uses a different sample width than the row type. + #[error("pixel format {fmt:?} does not match sample type {sample_type:?}")] + SampleTypeMismatch { + /// Requested output pixel format. + fmt: PixelFormat, + /// Sample type accepted by the current sink or buffer. + sample_type: SampleType, + }, +} + +/// Generic malformed or truncated input errors. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum InputError { + /// Input ended before a required fixed-size read. + #[error("input too short: need {need} bytes, have {have}")] + TooShort { + /// Required byte count. + need: usize, + /// Available byte count. + have: usize, + }, + /// Input ended while reading a named segment. + #[error("input truncated at offset {offset} while reading {segment}")] + TruncatedAt { + /// Byte offset where truncation was detected. + offset: usize, + /// Name of the segment being read. + segment: &'static str, + }, +} + +/// Error for a valid request that is not implemented yet. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("not yet implemented: {what}")] +pub struct NotImplemented { + /// Feature or path that is not implemented. + pub what: &'static str, +} + +/// Error for input or options unsupported by the current codec. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("unsupported: {what}")] +pub struct Unsupported { + /// Unsupported feature or option. + pub what: &'static str, +} + +/// Shared error classification used by facade traits. +pub trait CodecError: core::error::Error + Send + Sync + 'static { + /// True when the error indicates truncated input. + fn is_truncated(&self) -> bool; + /// True when the error indicates an unimplemented supported surface. + fn is_not_implemented(&self) -> bool; + /// True when the error indicates unsupported input or options. + fn is_unsupported(&self) -> bool; + /// True when the error indicates caller buffer sizing or layout problems. + fn is_buffer_error(&self) -> bool; +} diff --git a/crates/j2k-core/src/lib.rs b/crates/j2k-core/src/lib.rs new file mode 100644 index 00000000..57cfcab7 --- /dev/null +++ b/crates/j2k-core/src/lib.rs @@ -0,0 +1,76 @@ +//! Shared traits and value types for the `j2k` workspace. +//! +//! Codec crates use this crate to expose common pixel formats, decode +//! outcomes, row sinks, caller-owned scratch pools, and CPU/GPU backend +//! selection contracts without depending on each other. + +#![no_std] +#![warn(unreachable_pub)] + +extern crate alloc; + +/// Shared accelerator runtime contracts. +pub mod accelerator; +/// Backend selection and capability discovery. +pub mod backend; +/// Shared helpers for ordered tile-batch work. +pub mod batch; +mod buffer; +/// Reusable codec context wrappers. +pub mod context; +/// Shared device-output request policies. +pub mod device; +/// Common error classifications and helper error types. +pub mod error; +/// Compressed-byte passthrough eligibility checks. +pub mod passthrough; +/// Pixel layout and format descriptors. +pub mod pixel; +/// Row-streaming output sink trait. +pub mod row_sink; +/// Integer sample type descriptors. +pub mod sample; +/// Decode downscale options. +pub mod scale; +/// Caller-owned scratch pool trait. +pub mod scratch; +/// Facade traits implemented by codec crates. +pub mod traits; +/// Shared metadata and geometry types. +pub mod types; + +pub use accelerator::{ + AcceleratorSession, BackendFailureKind, DeviceMemoryRange, ExecutionStats, GpuAbi, + SurfaceResidency, +}; +pub use backend::{BackendCapabilities, BackendKind, BackendRequest, CpuFeatures}; +pub use batch::{ + collect_indexed_batch_results, tile_batch_worker_count, IndexedBatchResult, TileBatchError, + TileBatchOptions, TileDecodeJob, TileRegionDecodeJob, TileRegionScaledDecodeJob, + TileScaledDecodeJob, +}; +pub use buffer::{ + copy_tight_pixels_to_strided_output, ensure_allocation_within_cap, strided_output_len, + strided_output_len_capped, validate_strided_output_buffer, DEFAULT_MAX_HOST_ALLOCATION_BYTES, +}; +pub use context::{CacheStats, CodecContext, DecoderContext}; +pub use device::validate_cuda_surface_backend_request; +pub use error::{BufferError, CodecError, InputError, NotImplemented, Unsupported}; +pub use passthrough::{ + CompressedPayloadKind, CompressedTransferSyntax, PassthroughCandidate, PassthroughDecision, + PassthroughRejectReason, PassthroughRequirements, +}; +pub use pixel::{PixelFormat, PixelLayout}; +pub use row_sink::RowSink; +pub use sample::{Sample, SampleType}; +pub use scale::Downscale; +pub use scratch::ScratchPool; +pub use traits::{ + submit_ready_device, DecodeRowsError, DeviceSubmission, DeviceSubmitSession, DeviceSurface, + ImageCodec, ImageDecode, ImageDecodeDevice, ImageDecodeRows, ImageDecodeSubmit, + ReadySubmission, TileBatchDecode, TileBatchDecodeDevice, TileBatchDecodeManyDevice, + TileBatchDecodeSubmit, TileDecompress, +}; +pub use types::{ + CodedUnitLayout, Colorspace, DecodeOutcome, DecodeRequest, Info, Rect, TileLayout, WarningKind, +}; diff --git a/crates/signinum-core/src/passthrough.rs b/crates/j2k-core/src/passthrough.rs similarity index 84% rename from crates/signinum-core/src/passthrough.rs rename to crates/j2k-core/src/passthrough.rs index b95e37e6..2ab34816 100644 --- a/crates/signinum-core/src/passthrough.rs +++ b/crates/j2k-core/src/passthrough.rs @@ -185,12 +185,19 @@ impl<'a> PassthroughCandidate<'a> { /// Destination requirements for copying compressed bytes unchanged. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct PassthroughRequirements { + /// Required destination compressed syntax. pub transfer_syntax: CompressedTransferSyntax, + /// Required destination payload/container shape. pub payload_kind: CompressedPayloadKind, + /// Optional exact output dimensions. pub dimensions: Option<(u32, u32)>, + /// Optional exact component count. pub components: Option, + /// Optional exact bit depth. pub bit_depth: Option, + /// Optional exact colorspace. pub colorspace: Option, + /// Optional exact tile layout. pub tile_layout: Option, } @@ -252,42 +259,70 @@ impl PassthroughRequirements { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PassthroughDecision<'a> { /// Copy these compressed bytes unchanged. - Copy { bytes: &'a [u8] }, + Copy { + /// Borrowed source bytes to copy unchanged. + bytes: &'a [u8], + }, /// Decode/transcode instead, for the stated reason. - Transcode { reason: PassthroughRejectReason }, + Transcode { + /// Reason byte-preserving passthrough was rejected. + reason: PassthroughRejectReason, + }, } /// First reason a compressed payload was rejected for byte-preserving copy. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum PassthroughRejectReason { + /// The source compressed payload is empty. EmptyPayload, + /// Source and destination compressed syntaxes differ. TransferSyntaxMismatch { + /// Source syntax found in the candidate. source: CompressedTransferSyntax, + /// Required destination syntax. destination: CompressedTransferSyntax, }, + /// Source and destination payload/container shapes differ. PayloadKindMismatch { + /// Source payload shape found in the candidate. source: CompressedPayloadKind, + /// Required destination payload shape. destination: CompressedPayloadKind, }, + /// Source and destination dimensions differ. DimensionsMismatch { + /// Source dimensions found in the candidate. source: (u32, u32), + /// Required destination dimensions. destination: (u32, u32), }, + /// Source and destination component counts differ. ComponentsMismatch { + /// Source component count found in the candidate. source: u8, + /// Required destination component count. destination: u8, }, + /// Source and destination bit depths differ. BitDepthMismatch { + /// Source bit depth found in the candidate. source: u8, + /// Required destination bit depth. destination: u8, }, + /// Source and destination colorspaces differ. ColorspaceMismatch { + /// Source colorspace found in the candidate. source: Colorspace, + /// Required destination colorspace. destination: Colorspace, }, + /// Source and destination tile layouts differ. TileLayoutMismatch { + /// Source tile layout found in the candidate. source: Option, + /// Required destination tile layout. destination: TileLayout, }, } diff --git a/crates/signinum-core/src/pixel.rs b/crates/j2k-core/src/pixel.rs similarity index 65% rename from crates/signinum-core/src/pixel.rs rename to crates/j2k-core/src/pixel.rs index d32eafb6..20a170e2 100644 --- a/crates/signinum-core/src/pixel.rs +++ b/crates/j2k-core/src/pixel.rs @@ -2,26 +2,38 @@ use crate::sample::SampleType; +/// Channel layout independent of sample width. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum PixelLayout { + /// Three-channel red, green, blue layout. Rgb, + /// Four-channel red, green, blue, alpha layout. Rgba, + /// Single-channel grayscale layout. Gray, } +/// Concrete interleaved pixel format. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum PixelFormat { + /// Interleaved 8-bit RGB. Rgb8, + /// Interleaved 8-bit RGBA. Rgba8, + /// 8-bit grayscale. Gray8, + /// Interleaved 16-bit RGB. Rgb16, + /// Interleaved 16-bit RGBA. Rgba16, + /// 16-bit grayscale. Gray16, } impl PixelFormat { + /// Return the channel layout for this pixel format. pub const fn layout(self) -> PixelLayout { match self { Self::Rgb8 | Self::Rgb16 => PixelLayout::Rgb, @@ -30,6 +42,7 @@ impl PixelFormat { } } + /// Return the integer sample type for this pixel format. pub const fn sample(self) -> SampleType { match self { Self::Rgb8 | Self::Rgba8 | Self::Gray8 => SampleType::U8, @@ -37,6 +50,7 @@ impl PixelFormat { } } + /// Return the number of channels per pixel. pub const fn channels(self) -> usize { match self.layout() { PixelLayout::Rgb => 3, @@ -45,6 +59,7 @@ impl PixelFormat { } } + /// Return the number of bytes in one channel sample. pub const fn bytes_per_sample(self) -> usize { match self.sample() { SampleType::U8 => 1, @@ -52,6 +67,7 @@ impl PixelFormat { } } + /// Return the number of bytes in one interleaved pixel. pub const fn bytes_per_pixel(self) -> usize { self.channels() * self.bytes_per_sample() } diff --git a/crates/signinum-core/src/row_sink.rs b/crates/j2k-core/src/row_sink.rs similarity index 57% rename from crates/signinum-core/src/row_sink.rs rename to crates/j2k-core/src/row_sink.rs index 38b4ce82..fb7b71af 100644 --- a/crates/signinum-core/src/row_sink.rs +++ b/crates/j2k-core/src/row_sink.rs @@ -2,8 +2,11 @@ use crate::sample::Sample; +/// Destination for row-streaming decode output. pub trait RowSink { + /// Error returned by the sink when it cannot accept a row. type Error: core::error::Error + Send + Sync + 'static; + /// Write one decoded row at source/output row index `y`. fn write_row(&mut self, y: u32, row: &[S]) -> Result<(), Self::Error>; } diff --git a/crates/signinum-core/src/sample.rs b/crates/j2k-core/src/sample.rs similarity index 62% rename from crates/signinum-core/src/sample.rs rename to crates/j2k-core/src/sample.rs index b395d04c..4e5d0afb 100644 --- a/crates/signinum-core/src/sample.rs +++ b/crates/j2k-core/src/sample.rs @@ -1,14 +1,20 @@ // SPDX-License-Identifier: Apache-2.0 +/// Integer sample width used by a pixel format or row sink. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum SampleType { + /// Unsigned 8-bit samples. U8, + /// Unsigned 16-bit samples. U16, } +/// Supported integer sample type for row-oriented APIs. pub trait Sample: Copy + Default + Send + Sync + 'static { + /// Runtime sample type tag. const TYPE: SampleType; + /// Number of significant bits in the sample type. const BITS: u8; } diff --git a/crates/signinum-core/src/scale.rs b/crates/j2k-core/src/scale.rs similarity index 60% rename from crates/signinum-core/src/scale.rs rename to crates/j2k-core/src/scale.rs index 0611bb5d..3d06cefe 100644 --- a/crates/signinum-core/src/scale.rs +++ b/crates/j2k-core/src/scale.rs @@ -1,15 +1,22 @@ // SPDX-License-Identifier: Apache-2.0 -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +/// Power-of-two downscale requested during decode. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum Downscale { + /// Full-resolution output. + #[default] None, + /// Half-resolution output. Half, + /// Quarter-resolution output. Quarter, + /// Eighth-resolution output. Eighth, } impl Downscale { + /// Return the integer scale denominator. pub const fn denominator(self) -> u32 { match self { Self::None => 1, @@ -19,6 +26,7 @@ impl Downscale { } } + /// Return the decoded DCT block dimension after scaling. pub const fn output_block_size(self) -> u32 { match self { Self::None => 8, diff --git a/crates/j2k-core/src/scratch.rs b/crates/j2k-core/src/scratch.rs new file mode 100644 index 00000000..33e173ed --- /dev/null +++ b/crates/j2k-core/src/scratch.rs @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: Apache-2.0 + +/// Caller-owned reusable scratch allocations for codec implementations. +pub trait ScratchPool: Send { + /// Return the number of bytes currently retained by the pool. + fn bytes_allocated(&self) -> usize; + /// Clear reusable allocations or cached state held by the pool. + fn reset(&mut self); +} diff --git a/crates/signinum-core/src/traits.rs b/crates/j2k-core/src/traits.rs similarity index 79% rename from crates/signinum-core/src/traits.rs rename to crates/j2k-core/src/traits.rs index ac1aef62..9481773c 100644 --- a/crates/signinum-core/src/traits.rs +++ b/crates/j2k-core/src/traits.rs @@ -3,6 +3,7 @@ use alloc::vec::Vec; use crate::{ + accelerator::{DeviceMemoryRange, ExecutionStats, SurfaceResidency}, backend::{BackendKind, BackendRequest}, context::{CodecContext, DecoderContext}, error::CodecError, @@ -11,7 +12,7 @@ use crate::{ sample::Sample, scale::Downscale, scratch::ScratchPool, - types::{DecodeOutcome, Info, Rect}, + types::{DecodeOutcome, DecodeRequest, Info, Rect}, }; /// Error wrapper used by row-streaming decode when either the codec or the @@ -23,8 +24,10 @@ where E: core::error::Error + 'static, { #[error(transparent)] + /// Codec decode failure. Decode(D), #[error(transparent)] + /// Caller-provided row sink failure. Sink(E), } @@ -42,12 +45,24 @@ pub trait ImageCodec { pub trait DeviceSurface { /// Backend that owns or produced the surface. fn backend_kind(&self) -> BackendKind; + /// Memory residency of the surface. + fn residency(&self) -> SurfaceResidency { + SurfaceResidency::for_backend(self.backend_kind()) + } /// Surface dimensions in pixels. fn dimensions(&self) -> (u32, u32); /// Pixel format stored by the surface. fn pixel_format(&self) -> PixelFormat; /// Number of bytes represented by the surface. fn byte_len(&self) -> usize; + /// Execution statistics attached to the surface. + fn execution_stats(&self) -> ExecutionStats { + ExecutionStats::default() + } + /// Backend-visible memory range, when the backend can expose one safely. + fn memory_range(&self) -> Option { + None + } } /// Submitted device decode operation that can be waited on for completion. @@ -81,6 +96,24 @@ impl DeviceSubmission for ReadySubmission { } } +/// Mutable device session that tracks submitted backend work. +pub trait DeviceSubmitSession { + /// Record a submitted device operation. + fn record_submit(&mut self); +} + +/// Record a device submission and wrap an immediate result as ready. +pub fn submit_ready_device( + session: &mut S, + submit: impl FnOnce(&mut S) -> Result, +) -> ReadySubmission +where + S: DeviceSubmitSession + ?Sized, +{ + session.record_submit(); + ReadySubmission::from_result(submit(session)) +} + /// Borrowed-image decode API for codecs that parse compressed bytes directly. pub trait ImageDecode<'a>: ImageCodec + Sized + 'a { /// Borrowed parse product that can later construct a decoder. @@ -140,6 +173,25 @@ pub trait ImageDecode<'a>: ImageCodec + Sized + 'a { roi: Rect, scale: Downscale, ) -> Result, Self::Error>; + + /// Decode a normalized full/ROI/scaled request into caller-owned output. + fn decode_request_into( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + request: DecodeRequest, + ) -> Result, Self::Error> { + match (request.roi, request.scale) { + (None, Downscale::None) => self.decode_into_with_scratch(pool, out, stride, fmt), + (Some(roi), Downscale::None) => self.decode_region_into(pool, out, stride, fmt, roi), + (None, scale) => self.decode_scaled_into(pool, out, stride, fmt, scale), + (Some(roi), scale) => { + self.decode_region_scaled_into(pool, out, stride, fmt, roi, scale) + } + } + } } /// Decode API for implementations that can submit work to a device backend. @@ -186,6 +238,26 @@ pub trait ImageDecodeSubmit<'a>: ImageDecode<'a> { scale: Downscale, backend: BackendRequest, ) -> Result; + + /// Submit a normalized full/ROI/scaled decode request to a device backend. + fn submit_request_to_device( + &mut self, + session: &mut Self::Session, + fmt: PixelFormat, + backend: BackendRequest, + request: DecodeRequest, + ) -> Result { + match (request.roi, request.scale) { + (None, Downscale::None) => self.submit_to_device(session, fmt, backend), + (Some(roi), Downscale::None) => { + self.submit_region_to_device(session, fmt, roi, backend) + } + (None, scale) => self.submit_scaled_to_device(session, fmt, scale, backend), + (Some(roi), scale) => { + self.submit_region_scaled_to_device(session, fmt, roi, scale, backend) + } + } + } } /// Synchronous device-output decode API. @@ -330,6 +402,28 @@ pub trait TileBatchDecode: ImageCodec { roi: Rect, scale: Downscale, ) -> Result, Self::Error>; + + /// Decode one tile for a normalized full/ROI/scaled request. + fn decode_tile_request<'a>( + ctx: &mut DecoderContext, + pool: &mut Self::Pool, + input: &'a [u8], + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + request: DecodeRequest, + ) -> Result, Self::Error> { + match (request.roi, request.scale) { + (None, Downscale::None) => Self::decode_tile(ctx, pool, input, out, stride, fmt), + (Some(roi), Downscale::None) => { + Self::decode_tile_region(ctx, pool, input, out, stride, fmt, roi) + } + (None, scale) => Self::decode_tile_scaled(ctx, pool, input, out, stride, fmt, scale), + (Some(roi), scale) => { + Self::decode_tile_region_scaled(ctx, pool, input, out, stride, fmt, roi, scale) + } + } + } } /// Tile-batch helpers that return synchronous device surfaces. @@ -524,6 +618,32 @@ pub trait TileBatchDecodeSubmit: ImageCodec { scale: Downscale, backend: BackendRequest, ) -> Result; + + /// Submit one tile for a normalized full/ROI/scaled request. + fn submit_tile_request_to_device<'a>( + ctx: &mut DecoderContext, + session: &mut Self::Session, + pool: &mut Self::Pool, + input: &'a [u8], + fmt: PixelFormat, + backend: BackendRequest, + request: DecodeRequest, + ) -> Result { + match (request.roi, request.scale) { + (None, Downscale::None) => { + Self::submit_tile_to_device(ctx, session, pool, input, fmt, backend) + } + (Some(roi), Downscale::None) => { + Self::submit_tile_region_to_device(ctx, session, pool, input, fmt, roi, backend) + } + (None, scale) => { + Self::submit_tile_scaled_to_device(ctx, session, pool, input, fmt, scale, backend) + } + (Some(roi), scale) => Self::submit_tile_region_scaled_to_device( + ctx, session, pool, input, fmt, roi, scale, backend, + ), + } + } } /// Tile payload decompression API for container codecs such as Deflate, Zstd, diff --git a/crates/j2k-core/src/types.rs b/crates/j2k-core/src/types.rs new file mode 100644 index 00000000..f9b05113 --- /dev/null +++ b/crates/j2k-core/src/types.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: Apache-2.0 + +use alloc::vec::Vec; + +use crate::scale::Downscale; + +/// Color interpretation of decoded samples. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Colorspace { + /// Single-channel grayscale. + Grayscale, + /// JPEG-style luma/chroma color. + YCbCr, + /// Red, green, blue color. + Rgb, + /// Cyan, magenta, yellow, black color. + Cmyk, + /// Luma/chroma plus black color. + Ycck, + /// Standard RGB color. + SRgb, + /// Standard grayscale color. + SGray, + /// Color described by an embedded ICC profile. + IccTagged, + /// JPEG 2000 reversible color transform. + Rct, + /// JPEG 2000 irreversible color transform. + Ict, +} + +/// Regular tile grid layout for a compressed image. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct TileLayout { + /// Width of one tile in pixels. + pub tile_width: u32, + /// Height of one tile in pixels. + pub tile_height: u32, + /// Number of tiles across the image. + pub tiles_x: u32, + /// Number of tiles down the image. + pub tiles_y: u32, +} + +/// Regular coded-unit grid layout for formats with independently coded units. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct CodedUnitLayout { + /// Width of one coded unit in pixels. + pub unit_width: u32, + /// Height of one coded unit in pixels. + pub unit_height: u32, + /// Number of coded units across the image. + pub units_x: u32, + /// Number of coded units down the image. + pub units_y: u32, +} + +impl CodedUnitLayout { + /// Return the saturating total coded-unit count. + pub const fn unit_count(&self) -> u32 { + self.units_x.saturating_mul(self.units_y) + } +} + +/// Rectangle in source pixel coordinates. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Rect { + /// Left coordinate. + pub x: u32, + /// Top coordinate. + pub y: u32, + /// Rectangle width. + pub w: u32, + /// Rectangle height. + pub h: u32, +} + +impl Rect { + /// Return a rectangle covering the full dimensions. + pub const fn full(dims: (u32, u32)) -> Self { + Self { + x: 0, + y: 0, + w: dims.0, + h: dims.1, + } + } + + /// Return whether this rectangle is fully inside `dims`. + pub fn is_within(&self, dims: (u32, u32)) -> bool { + let (w, h) = dims; + self.x.checked_add(self.w).is_some_and(|r| r <= w) + && self.y.checked_add(self.h).is_some_and(|b| b <= h) + } + + /// Return the smallest scaled rectangle that covers this source rectangle. + #[must_use] + pub fn scaled_covering(&self, scale: Downscale) -> Self { + let denom = scale.denominator(); + let x_end = self.x.saturating_add(self.w); + let y_end = self.y.saturating_add(self.h); + let x0 = self.x / denom; + let y0 = self.y / denom; + let x1 = x_end.div_ceil(denom); + let y1 = y_end.div_ceil(denom); + Self { + x: x0, + y: y0, + w: x1.saturating_sub(x0), + h: y1.saturating_sub(y0), + } + } +} + +/// Source-region and reduced-resolution shape requested during decode. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub struct DecodeRequest { + /// Optional source-image region. `None` requests the full image. + pub roi: Option, + /// Reduced-resolution decode scale. + pub scale: Downscale, +} + +impl DecodeRequest { + /// Full-image, full-resolution decode request. + pub const FULL: Self = Self { + roi: None, + scale: Downscale::None, + }; + + /// Construct a full-image, full-resolution decode request. + pub const fn full() -> Self { + Self::FULL + } + + /// Construct a full-resolution region decode request. + pub const fn region(roi: Rect) -> Self { + Self { + roi: Some(roi), + scale: Downscale::None, + } + } + + /// Construct a full-image scaled decode request. + pub const fn scaled(scale: Downscale) -> Self { + Self { roi: None, scale } + } + + /// Construct a region scaled decode request. + pub const fn region_scaled(roi: Rect, scale: Downscale) -> Self { + Self { + roi: Some(roi), + scale, + } + } + + /// Return whether this request covers the full image at full resolution. + pub const fn is_full_resolution_full_image(self) -> bool { + matches!( + self, + Self { + roi: None, + scale: Downscale::None + } + ) + } + + /// Return the output rectangle covered by this request for image dimensions. + #[must_use] + pub fn decoded_rect(self, dimensions: (u32, u32)) -> Rect { + let roi = self.roi.unwrap_or_else(|| Rect::full(dimensions)); + roi.scaled_covering(self.scale) + } +} + +/// Basic image metadata returned by inspect/parse operations. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Info { + /// Image dimensions in pixels. + pub dimensions: (u32, u32), + /// Number of image components. + pub components: u8, + /// Color interpretation of the components. + pub colorspace: Colorspace, + /// Bits per component sample. + pub bit_depth: u8, + /// Optional compressed tile grid. + pub tile_layout: Option, + /// Optional coded-unit grid. + pub coded_unit_layout: Option, + /// Optional restart interval for formats that expose one. + pub restart_interval: Option, + /// Number of resolution levels available in the codestream. + pub resolution_levels: u8, +} + +/// Broad warning category for non-fatal decode issues. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum WarningKind { + /// Minor compliance issue that did not prevent decode. + MinorCompliance, + /// Non-fatal truncation tolerated by the codec. + NonFatalTruncation, + /// Unusual but accepted feature or structure. + UnusualFeature, +} + +/// Successful decode metadata plus non-fatal warnings. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecodeOutcome { + /// Source/output rectangle actually decoded. + pub decoded: Rect, + /// Non-fatal warnings observed during decode. + pub warnings: Vec, +} + +impl DecodeOutcome { + /// Construct a decode outcome from the decoded rectangle and warnings. + pub fn new(decoded: Rect, warnings: Vec) -> Self { + Self { decoded, warnings } + } +} diff --git a/crates/signinum-core/tests/api.rs b/crates/j2k-core/tests/api.rs similarity index 82% rename from crates/signinum-core/tests/api.rs rename to crates/j2k-core/tests/api.rs index fcfbd340..a5a774ad 100644 --- a/crates/signinum-core/tests/api.rs +++ b/crates/j2k-core/tests/api.rs @@ -1,13 +1,14 @@ -use signinum_core::{ - copy_tight_pixels_to_strided_output, strided_output_len, validate_strided_output_buffer, - BackendCapabilities, BackendKind, BackendRequest, BufferError, CodecContext, CodecError, - CpuFeatures, DecoderContext, DeviceSubmission, DeviceSurface, Downscale, ImageCodec, +use j2k_core::{ + copy_tight_pixels_to_strided_output, strided_output_len, submit_ready_device, + validate_cuda_surface_backend_request, validate_strided_output_buffer, BackendCapabilities, + BackendKind, BackendRequest, BufferError, CodecContext, CodecError, CpuFeatures, + DecoderContext, DeviceSubmission, DeviceSubmitSession, DeviceSurface, Downscale, ImageCodec, ImageDecode, ImageDecodeDevice, ImageDecodeSubmit, PassthroughCandidate, PassthroughDecision, PassthroughRejectReason, PassthroughRequirements, PixelFormat, PixelLayout, ReadySubmission, Rect, SampleType, ScratchPool, TileBatchDecodeDevice, TileBatchDecodeManyDevice, TileBatchDecodeSubmit, TileBatchOptions, }; -use signinum_core::{ +use j2k_core::{ CodedUnitLayout, Colorspace, CompressedPayloadKind, CompressedTransferSyntax, Info, TileLayout, }; use std::num::NonZeroUsize; @@ -19,6 +20,70 @@ fn pixel_format_reports_layout_and_sample_type() { assert_eq!(PixelFormat::Rgb16.sample(), SampleType::U16); } +#[derive(Default)] +struct SubmitCounter { + submissions: u64, +} + +impl DeviceSubmitSession for SubmitCounter { + fn record_submit(&mut self) { + self.submissions += 1; + } +} + +#[test] +fn submit_ready_device_records_and_returns_success() { + let mut session = SubmitCounter::default(); + + let surface = submit_ready_device(&mut session, |session| { + assert_eq!(session.submissions, 1); + Ok::<_, DummyError>(DummySurface { + backend: BackendKind::Cuda, + dims: (2, 3), + fmt: PixelFormat::Rgb8, + len: 18, + }) + }) + .wait() + .expect("ready submission succeeds"); + + assert_eq!(session.submissions, 1); + assert_eq!(surface.backend_kind(), BackendKind::Cuda); + assert_eq!(surface.dimensions(), (2, 3)); +} + +#[test] +fn submit_ready_device_records_and_returns_error() { + let mut session = SubmitCounter::default(); + + let error = submit_ready_device(&mut session, |_session| Err::(DummyError)) + .wait() + .expect_err("ready submission surfaces the error"); + + assert_eq!(error, DummyError); + assert_eq!(session.submissions, 1); +} + +#[test] +fn cuda_surface_backend_request_validation_rejects_metal_only() { + assert_eq!( + validate_cuda_surface_backend_request(BackendRequest::Cpu), + Ok(()) + ); + assert_eq!( + validate_cuda_surface_backend_request(BackendRequest::Auto), + Ok(()) + ); + assert_eq!( + validate_cuda_surface_backend_request(BackendRequest::Cuda), + Ok(()) + ); + assert_eq!( + validate_cuda_surface_backend_request(BackendRequest::Metal), + Err(BackendRequest::Metal) + ); +} + #[test] fn downscale_reports_expected_denominators() { assert_eq!(Downscale::None.denominator(), 1); @@ -229,7 +294,7 @@ fn validate_strided_output_buffer_reports_stride_and_output_errors() { #[test] fn tile_batch_worker_count_uses_available_workers_when_unspecified() { assert_eq!( - signinum_core::tile_batch_worker_count(8, TileBatchOptions::default(), 4), + j2k_core::tile_batch_worker_count(8, TileBatchOptions::default(), 4), 4 ); } @@ -240,13 +305,13 @@ fn tile_batch_worker_count_clamps_to_batch_size_and_at_least_one_worker() { workers: Some(NonZeroUsize::new(16).expect("nonzero")), }; - assert_eq!(signinum_core::tile_batch_worker_count(3, options, 8), 3); + assert_eq!(j2k_core::tile_batch_worker_count(3, options, 8), 3); assert_eq!( - signinum_core::tile_batch_worker_count(8, TileBatchOptions::default(), 0), + j2k_core::tile_batch_worker_count(8, TileBatchOptions::default(), 0), 1 ); assert_eq!( - signinum_core::tile_batch_worker_count(0, TileBatchOptions::default(), 8), + j2k_core::tile_batch_worker_count(0, TileBatchOptions::default(), 8), 1 ); } @@ -256,10 +321,8 @@ fn collect_indexed_batch_results_restores_input_order() { let results = vec![(2, Ok("two")), (0, Ok("zero")), (1, Ok("one"))]; let outcomes = - signinum_core::collect_indexed_batch_results(3, results, |index, source: &str| { - (index, source) - }) - .expect("ordered outcomes"); + j2k_core::collect_indexed_batch_results(3, results, |index, source: &str| (index, source)) + .expect("ordered outcomes"); assert_eq!(outcomes, ["zero", "one", "two"]); } @@ -273,9 +336,8 @@ fn collect_indexed_batch_results_returns_first_error_by_input_index() { (3, Ok("three")), ]; - let err = - signinum_core::collect_indexed_batch_results(4, results, |index, source| (index, source)) - .expect_err("first failing input index"); + let err = j2k_core::collect_indexed_batch_results(4, results, |index, source| (index, source)) + .expect_err("first failing input index"); assert_eq!(err, (1, "first")); } @@ -285,8 +347,7 @@ fn collect_indexed_batch_results_returns_first_error_by_input_index() { fn collect_indexed_batch_results_rejects_out_of_bounds_error_index() { let results = vec![(3, Err::<&str, _>("outside"))]; - let _ = - signinum_core::collect_indexed_batch_results(3, results, |index, source| (index, source)); + let _ = j2k_core::collect_indexed_batch_results(3, results, |index, source| (index, source)); } #[test] @@ -296,17 +357,79 @@ fn backend_capabilities_resolve_auto_and_explicit_requests() { metal: true, cuda: false, }; - assert_eq!(caps.resolve(BackendRequest::Auto), Some(BackendKind::Metal)); + assert_eq!(caps.resolve(BackendRequest::Auto), Some(BackendKind::Cpu)); assert_eq!(caps.resolve(BackendRequest::Cpu), Some(BackendKind::Cpu)); assert_eq!( caps.resolve(BackendRequest::Metal), Some(BackendKind::Metal) ); assert_eq!(caps.resolve(BackendRequest::Cuda), None); + assert_eq!(caps.first_available_accelerator(), Some(BackendKind::Metal)); assert!(caps.supports(BackendRequest::Metal)); assert!(!caps.supports(BackendRequest::Cuda)); } +#[test] +fn backend_request_exposes_adaptive_cpu_and_strict_aliases() { + assert_eq!(BackendRequest::ACCELERATED, BackendRequest::Auto); + assert_eq!(BackendRequest::CPU_ONLY, BackendRequest::Cpu); + assert_eq!(BackendRequest::STRICT_METAL, BackendRequest::Metal); + assert_eq!(BackendRequest::STRICT_CUDA, BackendRequest::Cuda); +} + +#[test] +fn decode_request_default_and_scaled_rects_are_stable() { + let full = j2k_core::DecodeRequest::default(); + assert!(full.is_full_resolution_full_image()); + assert_eq!(full.decoded_rect((17, 9)), Rect::full((17, 9))); + + let roi = Rect { + x: 3, + y: 5, + w: 9, + h: 7, + }; + let request = j2k_core::DecodeRequest::region_scaled(roi, Downscale::Quarter); + assert_eq!( + request.decoded_rect((99, 99)), + Rect { + x: 0, + y: 1, + w: 3, + h: 2 + } + ); +} + +#[test] +fn execution_stats_and_gpu_abi_helpers_are_stable() { + let combined = j2k_core::ExecutionStats { + submissions: 1, + kernel_dispatches: 2, + upload_bytes: 3, + readback_bytes: 4, + device_us: 5, + } + .saturating_add(j2k_core::ExecutionStats { + submissions: u64::MAX, + kernel_dispatches: 1, + upload_bytes: 1, + readback_bytes: 1, + device_us: 1, + }); + + assert_eq!(combined.submissions, u64::MAX); + assert_eq!(combined.kernel_dispatches, 3); + + let values = [1_u32, 2_u32]; + let bytes = ::slice_as_bytes(&values); + assert_eq!(bytes.len(), 8); + assert_eq!( + j2k_core::DeviceMemoryRange::new(j2k_core::BackendKind::Cuda, 7, 8, 9).len, + 9 + ); +} + #[derive(Debug, Clone, Copy)] struct DummySurface { backend: BackendKind, @@ -370,7 +493,7 @@ impl ScratchPool for DummyPool { fn reset(&mut self) {} } -#[derive(Debug, thiserror::Error)] +#[derive(Debug, PartialEq, Eq, thiserror::Error)] #[error("dummy decode error")] struct DummyError; @@ -438,7 +561,7 @@ impl<'a> ImageDecode<'a> for DummyImageDecoder { _out: &mut [u8], _stride: usize, _fmt: PixelFormat, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { Err(DummyError) } @@ -448,7 +571,7 @@ impl<'a> ImageDecode<'a> for DummyImageDecoder { _out: &mut [u8], _stride: usize, _fmt: PixelFormat, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { Err(DummyError) } @@ -459,7 +582,7 @@ impl<'a> ImageDecode<'a> for DummyImageDecoder { _stride: usize, _fmt: PixelFormat, _roi: Rect, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { Err(DummyError) } @@ -470,7 +593,7 @@ impl<'a> ImageDecode<'a> for DummyImageDecoder { _stride: usize, _fmt: PixelFormat, _scale: Downscale, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { Err(DummyError) } @@ -482,7 +605,7 @@ impl<'a> ImageDecode<'a> for DummyImageDecoder { _fmt: PixelFormat, _roi: Rect, _scale: Downscale, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { Err(DummyError) } } diff --git a/crates/signinum-cuda-runtime/Cargo.toml b/crates/j2k-cuda-runtime/Cargo.toml similarity index 52% rename from crates/signinum-cuda-runtime/Cargo.toml rename to crates/j2k-cuda-runtime/Cargo.toml index 677a2b99..38bfff01 100644 --- a/crates/signinum-cuda-runtime/Cargo.toml +++ b/crates/j2k-cuda-runtime/Cargo.toml @@ -1,18 +1,33 @@ [package] -name = "signinum-cuda-runtime" -description = "CUDA Driver API runtime helpers for signinum device adapters" -version = "0.4.2" +name = "j2k-cuda-runtime" +description = "CUDA codec engine and Driver API runtime for j2k device adapters" +version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true readme = "README.md" +[package.metadata.docs.rs] +all-features = true +targets = [] + [lib] -name = "signinum_cuda_runtime" +name = "j2k_cuda_runtime" path = "src/lib.rs" +[features] +default = [] +cuda-profiling = [] +cuda-oxide-copy-u8 = [] +cuda-oxide-j2k-encode = [] +cuda-oxide-j2k-decode-store = [] +cuda-oxide-j2k-dequantize = [] +cuda-oxide-j2k-idwt = [] +cuda-oxide-transcode = [] + [dependencies] +j2k-core = { path = "../j2k-core", version = "=0.6.0" } libloading = { workspace = true } thiserror = { workspace = true } diff --git a/crates/j2k-cuda-runtime/README.md b/crates/j2k-cuda-runtime/README.md new file mode 100644 index 00000000..957d7819 --- /dev/null +++ b/crates/j2k-cuda-runtime/README.md @@ -0,0 +1,10 @@ +# j2k-cuda-runtime + +CUDA codec engine and Driver API runtime crate for J2K CUDA adapters. + +It owns J2K CUDA kernel modules and the host-side launch logic for CUDA +codec stages, plus allocation, copy, stream, timing, and pooled resource +helpers used by the adapter crates. + +This crate is the shared CUDA engine layer, but not proof of NVIDIA performance. +CUDA benchmark claims require self-hosted benchmark evidence. diff --git a/crates/j2k-cuda-runtime/build.rs b/crates/j2k-cuda-runtime/build.rs new file mode 100644 index 00000000..840b0b4a --- /dev/null +++ b/crates/j2k-cuda-runtime/build.rs @@ -0,0 +1,487 @@ +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn main() { + emit_build_script_metadata(); + + let out_dir = PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR is set by cargo")); + let require_kernel_build = env::var_os("J2K_REQUIRE_CUDA_HTJ2K_STRICT").is_some() + || env::var_os("J2K_REQUIRE_CUDA_KERNEL_BUILD").is_some(); + let nvcc = configured_nvcc(require_kernel_build); + + let j2k_encode_ptx = out_dir.join("j2k_encode_kernels.ptx"); + let j2k_encode_compiled = compile_or_copy_ptx( + nvcc.as_deref(), + Path::new("src/j2k_encode_kernels.cu"), + Path::new("src/j2k_encode_kernels.ptx"), + &j2k_encode_ptx, + require_kernel_build, + ); + if j2k_encode_compiled { + println!("cargo:rustc-cfg= j2k_cuda_j2k_encode_ptx_built"); + } + let htj2k_decode_ptx = out_dir.join("htj2k_decode_kernels.ptx"); + compile_or_copy_ptx( + nvcc.as_deref(), + Path::new("src/htj2k_decode_kernels.cu"), + Path::new("src/htj2k_decode_kernels.ptx"), + &htj2k_decode_ptx, + require_kernel_build, + ); + let htj2k_encode_ptx = out_dir.join("htj2k_encode_kernels.ptx"); + let htj2k_encode_compiled = compile_or_copy_ptx( + nvcc.as_deref(), + Path::new("src/htj2k_encode_kernels.cu"), + Path::new("src/htj2k_encode_kernels.ptx"), + &htj2k_encode_ptx, + require_kernel_build, + ); + if htj2k_encode_compiled { + println!("cargo:rustc-cfg= j2k_cuda_htj2k_encode_ptx_built"); + } + + let jpeg_decode_ptx = out_dir.join("jpeg_decode_kernels.ptx"); + let jpeg_decode_compiled = compile_optional_ptx( + nvcc.as_deref(), + Path::new("src/jpeg_decode_kernels.cu"), + &jpeg_decode_ptx, + require_kernel_build, + ); + if jpeg_decode_compiled { + println!("cargo:rustc-cfg= j2k_cuda_jpeg_decode_ptx_built"); + } + + // Transcode (DCT->wavelet) kernels are new: there is no checked-in PTX + // fallback, so this is an OPTIONAL compile. On nvcc success it sets a cfg + // that gates the Rust dispatch; on a non-nvcc host it is skipped (the + // dispatch is cfg'd out). The runner sets the strict env, which requires + // nvcc to succeed. + let transcode_ptx = out_dir.join("transcode_kernels.ptx"); + let transcode_compiled = compile_optional_ptx( + nvcc.as_deref(), + Path::new("src/transcode_kernels.cu"), + &transcode_ptx, + require_kernel_build, + ); + if transcode_compiled { + println!("cargo:rustc-cfg= j2k_cuda_transcode_ptx_built"); + } + + compile_cuda_oxide_feature_projects(&out_dir); +} + +fn emit_build_script_metadata() { + println!("cargo:rerun-if-changed=src/j2k_encode_kernels.cu"); + println!("cargo:rerun-if-changed=src/j2k_encode_kernels.ptx"); + println!("cargo:rerun-if-changed=src/htj2k_decode_kernels.cu"); + println!("cargo:rerun-if-changed=src/htj2k_decode_kernels.ptx"); + println!("cargo:rerun-if-changed=src/htj2k_encode_kernels.cu"); + println!("cargo:rerun-if-changed=src/htj2k_encode_kernels.ptx"); + println!("cargo:rerun-if-changed=src/jpeg_decode_kernels.cu"); + println!("cargo:rerun-if-changed=src/transcode_kernels.cu"); + println!("cargo:rerun-if-changed=src/cuda_oxide_copy_u8/Cargo.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_copy_u8/rust-toolchain.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_copy_u8/src/main.rs"); + println!("cargo:rerun-if-changed=src/cuda_oxide_copy_u8/simt/Cargo.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_copy_u8/simt/src/main.rs"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_encode/Cargo.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_encode/rust-toolchain.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_encode/src/main.rs"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_encode/simt/Cargo.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_encode/simt/src/main.rs"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_decode_store/Cargo.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_decode_store/rust-toolchain.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_decode_store/src/main.rs"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_decode_store/simt/Cargo.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_decode_store/simt/src/main.rs"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_dequantize/Cargo.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_dequantize/rust-toolchain.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_dequantize/src/main.rs"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_dequantize/simt/Cargo.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_dequantize/simt/src/main.rs"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_idwt/Cargo.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_idwt/rust-toolchain.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_idwt/src/main.rs"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_idwt/simt/Cargo.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_j2k_idwt/simt/src/main.rs"); + println!("cargo:rerun-if-changed=src/cuda_oxide_transcode/Cargo.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_transcode/rust-toolchain.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_transcode/src/main.rs"); + println!("cargo:rerun-if-changed=src/cuda_oxide_transcode/simt/Cargo.toml"); + println!("cargo:rerun-if-changed=src/cuda_oxide_transcode/simt/src/main.rs"); + println!("cargo:rerun-if-env-changed=NVCC"); + println!("cargo:rerun-if-env-changed=J2K_CUDA_OXIDE_ARCH"); + println!("cargo:rerun-if-env-changed=J2K_REQUIRE_CUDA_OXIDE_COPY_U8"); + println!("cargo:rerun-if-env-changed=J2K_REQUIRE_CUDA_OXIDE_J2K_ENCODE"); + println!("cargo:rerun-if-env-changed=J2K_REQUIRE_CUDA_OXIDE_J2K_DECODE_STORE"); + println!("cargo:rerun-if-env-changed=J2K_REQUIRE_CUDA_OXIDE_J2K_DEQUANTIZE"); + println!("cargo:rerun-if-env-changed=J2K_REQUIRE_CUDA_OXIDE_J2K_IDWT"); + println!("cargo:rerun-if-env-changed=J2K_REQUIRE_CUDA_OXIDE_TRANSCODE"); + println!("cargo:rerun-if-env-changed=J2K_REQUIRE_CUDA_HTJ2K_STRICT"); + println!("cargo:rerun-if-env-changed=J2K_REQUIRE_CUDA_KERNEL_BUILD"); + println!("cargo:rustc-check-cfg=cfg( j2k_cuda_j2k_encode_ptx_built)"); + println!("cargo:rustc-check-cfg=cfg( j2k_cuda_htj2k_encode_ptx_built)"); + println!("cargo:rustc-check-cfg=cfg( j2k_cuda_jpeg_decode_ptx_built)"); + println!("cargo:rustc-check-cfg=cfg( j2k_cuda_transcode_ptx_built)"); + println!("cargo:rustc-check-cfg=cfg(j2k_cuda_oxide_copy_u8_built)"); + println!("cargo:rustc-check-cfg=cfg(j2k_cuda_oxide_j2k_encode_built)"); + println!("cargo:rustc-check-cfg=cfg(j2k_cuda_oxide_j2k_decode_store_built)"); + println!("cargo:rustc-check-cfg=cfg(j2k_cuda_oxide_j2k_dequantize_built)"); + println!("cargo:rustc-check-cfg=cfg(j2k_cuda_oxide_j2k_idwt_built)"); + println!("cargo:rustc-check-cfg=cfg(j2k_cuda_oxide_transcode_built)"); +} + +fn compile_cuda_oxide_feature_projects(out_dir: &Path) { + if env::var_os("CARGO_FEATURE_CUDA_OXIDE_COPY_U8").is_some() { + let require_cuda_oxide = env::var_os("J2K_REQUIRE_CUDA_OXIDE_COPY_U8").is_some(); + if compile_cuda_oxide_copy_u8(out_dir, require_cuda_oxide) { + println!("cargo:rustc-cfg=j2k_cuda_oxide_copy_u8_built"); + } + } + + if env::var_os("CARGO_FEATURE_CUDA_OXIDE_J2K_ENCODE").is_some() { + let require_cuda_oxide = env::var_os("J2K_REQUIRE_CUDA_OXIDE_J2K_ENCODE").is_some(); + if compile_cuda_oxide_j2k_encode(out_dir, require_cuda_oxide) { + println!("cargo:rustc-cfg=j2k_cuda_oxide_j2k_encode_built"); + } + } + + if env::var_os("CARGO_FEATURE_CUDA_OXIDE_J2K_DECODE_STORE").is_some() { + let require_cuda_oxide = env::var_os("J2K_REQUIRE_CUDA_OXIDE_J2K_DECODE_STORE").is_some(); + if compile_cuda_oxide_j2k_decode_store(out_dir, require_cuda_oxide) { + println!("cargo:rustc-cfg=j2k_cuda_oxide_j2k_decode_store_built"); + } + } + + if env::var_os("CARGO_FEATURE_CUDA_OXIDE_J2K_DEQUANTIZE").is_some() { + let require_cuda_oxide = env::var_os("J2K_REQUIRE_CUDA_OXIDE_J2K_DEQUANTIZE").is_some(); + if compile_cuda_oxide_j2k_dequantize(out_dir, require_cuda_oxide) { + println!("cargo:rustc-cfg=j2k_cuda_oxide_j2k_dequantize_built"); + } + } + + if env::var_os("CARGO_FEATURE_CUDA_OXIDE_J2K_IDWT").is_some() { + let require_cuda_oxide = env::var_os("J2K_REQUIRE_CUDA_OXIDE_J2K_IDWT").is_some(); + if compile_cuda_oxide_j2k_idwt(out_dir, require_cuda_oxide) { + println!("cargo:rustc-cfg=j2k_cuda_oxide_j2k_idwt_built"); + } + } + + if env::var_os("CARGO_FEATURE_CUDA_OXIDE_TRANSCODE").is_some() { + let require_cuda_oxide = env::var_os("J2K_REQUIRE_CUDA_OXIDE_TRANSCODE").is_some(); + if compile_cuda_oxide_transcode(out_dir, require_cuda_oxide) { + println!("cargo:rustc-cfg=j2k_cuda_oxide_transcode_built"); + } + } +} + +fn compile_cuda_oxide_copy_u8(out_dir: &Path, require_cuda_oxide: bool) -> bool { + compile_cuda_oxide_project( + out_dir, + CudaOxideProject { + source_dir: Path::new("src/cuda_oxide_copy_u8"), + output_name: "cuda_oxide_copy_u8.ptx", + artifact_name: "j2k_cuda_oxide_copy_u8.ptx", + display_name: "cuda-oxide CopyU8", + }, + require_cuda_oxide, + ) +} + +fn compile_cuda_oxide_j2k_encode(out_dir: &Path, require_cuda_oxide: bool) -> bool { + compile_cuda_oxide_project( + out_dir, + CudaOxideProject { + source_dir: Path::new("src/cuda_oxide_j2k_encode"), + output_name: "cuda_oxide_j2k_encode.ptx", + artifact_name: "j2k_cuda_oxide_j2k_encode.ptx", + display_name: "cuda-oxide J2K encode", + }, + require_cuda_oxide, + ) +} + +fn compile_cuda_oxide_j2k_decode_store(out_dir: &Path, require_cuda_oxide: bool) -> bool { + compile_cuda_oxide_project( + out_dir, + CudaOxideProject { + source_dir: Path::new("src/cuda_oxide_j2k_decode_store"), + output_name: "cuda_oxide_j2k_decode_store.ptx", + artifact_name: "j2k_cuda_oxide_j2k_decode_store.ptx", + display_name: "cuda-oxide J2K decode store", + }, + require_cuda_oxide, + ) +} + +fn compile_cuda_oxide_j2k_dequantize(out_dir: &Path, require_cuda_oxide: bool) -> bool { + compile_cuda_oxide_project( + out_dir, + CudaOxideProject { + source_dir: Path::new("src/cuda_oxide_j2k_dequantize"), + output_name: "cuda_oxide_j2k_dequantize.ptx", + artifact_name: "j2k_cuda_oxide_j2k_dequantize.ptx", + display_name: "cuda-oxide J2K dequantize", + }, + require_cuda_oxide, + ) +} + +fn compile_cuda_oxide_j2k_idwt(out_dir: &Path, require_cuda_oxide: bool) -> bool { + compile_cuda_oxide_project( + out_dir, + CudaOxideProject { + source_dir: Path::new("src/cuda_oxide_j2k_idwt"), + output_name: "cuda_oxide_j2k_idwt.ptx", + artifact_name: "j2k_cuda_oxide_j2k_idwt.ptx", + display_name: "cuda-oxide J2K IDWT", + }, + require_cuda_oxide, + ) +} + +fn compile_cuda_oxide_transcode(out_dir: &Path, require_cuda_oxide: bool) -> bool { + compile_cuda_oxide_project( + out_dir, + CudaOxideProject { + source_dir: Path::new("src/cuda_oxide_transcode"), + output_name: "cuda_oxide_transcode.ptx", + artifact_name: "j2k_cuda_oxide_transcode.ptx", + display_name: "cuda-oxide transcode", + }, + require_cuda_oxide, + ) +} + +#[derive(Clone, Copy)] +struct CudaOxideProject<'a> { + source_dir: &'a Path, + output_name: &'a str, + artifact_name: &'a str, + display_name: &'a str, +} + +fn compile_cuda_oxide_project( + out_dir: &Path, + project: CudaOxideProject<'_>, + require_cuda_oxide: bool, +) -> bool { + let output = out_dir.join(project.output_name); + let host = env::var("HOST").expect("HOST is set by cargo"); + if !host.contains("linux") { + return skip_cuda_oxide_project( + &output, + require_cuda_oxide, + project.display_name, + &format!( + "{} requires a Linux host; current HOST={host}", + project.display_name + ), + ); + } + + let project_dir = out_dir.join(project.output_name.trim_end_matches(".ptx")); + copy_cuda_oxide_project(project.source_dir, &project_dir); + + let arch = env::var("J2K_CUDA_OXIDE_ARCH").unwrap_or_else(|_| "sm_80".to_string()); + println!( + "cargo:warning=building {} with `cargo oxide build --arch {arch}`", + project.display_name + ); + // Use the rustup cargo proxy so the staged rust-toolchain.toml selects + // cuda-oxide's pinned nightly instead of the outer workspace toolchain. + let status = Command::new("cargo") + .args(["oxide", "build", "--arch"]) + .arg(&arch) + .env_remove("RUSTUP_TOOLCHAIN") + .env_remove("RUSTC") + .env_remove("RUSTDOC") + .env_remove("RUSTFLAGS") + .env_remove("CARGO_ENCODED_RUSTFLAGS") + .current_dir(&project_dir) + .status(); + let status = match status { + Ok(status) => status, + Err(error) => { + return skip_cuda_oxide_project( + &output, + require_cuda_oxide, + project.display_name, + &format!("failed to invoke cargo oxide build: {error}"), + ); + } + }; + if !status.success() { + return skip_cuda_oxide_project( + &output, + require_cuda_oxide, + project.display_name, + &format!("{} build failed with status {status}", project.display_name), + ); + } + + let generated = project_dir.join("ptx").join(project.artifact_name); + let mut bytes = fs::read(&generated).unwrap_or_else(|error| { + panic!( + "{} build did not produce {}: {error}", + project.display_name, + generated.display() + ) + }); + if bytes.last().copied() != Some(0) { + bytes.push(0); + } + fs::write(&output, bytes).unwrap_or_else(|error| { + panic!( + "failed to write {} PTX to {}: {error}", + project.display_name, + output.display() + ) + }); + true +} + +fn skip_cuda_oxide_project( + output: &Path, + required: bool, + display_name: &str, + message: &str, +) -> bool { + assert!(!required, "{message}"); + println!("cargo:warning=skipping {display_name} build: {message}"); + fs::write(output, b".version 7.0\n.target sm_52\n.address_size 64\n\0") + .unwrap_or_else(|error| panic!("write placeholder {display_name} PTX: {error}")); + false +} + +fn copy_cuda_oxide_project(source_dir: &Path, project_dir: &Path) { + copy_cuda_oxide_file(source_dir, project_dir, Path::new("Cargo.toml")); + copy_cuda_oxide_file(source_dir, project_dir, Path::new("rust-toolchain.toml")); + copy_cuda_oxide_file(source_dir, project_dir, Path::new("src/main.rs")); + copy_cuda_oxide_file(source_dir, project_dir, Path::new("simt/Cargo.toml")); + copy_cuda_oxide_file(source_dir, project_dir, Path::new("simt/src/main.rs")); +} + +fn copy_cuda_oxide_file(source_dir: &Path, project_dir: &Path, relative: &Path) { + let source = source_dir.join(relative); + let dest = project_dir.join(relative); + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent).unwrap_or_else(|error| { + panic!( + "failed to create cuda-oxide project dir {}: {error}", + parent.display() + ) + }); + } + fs::copy(&source, &dest).unwrap_or_else(|error| { + panic!( + "failed to stage cuda-oxide project file {} to {}: {error}", + source.display(), + dest.display() + ) + }); +} + +/// Compile a CUDA kernel to PTX with nvcc only (no checked-in fallback). +/// +/// Returns whether nvcc produced PTX. When `require_kernel_build` is set (the +/// runner), nvcc failure is a hard error; otherwise a non-nvcc host simply +/// skips the kernel and the Rust dispatch is cfg-gated off. +fn compile_optional_ptx( + nvcc: Option<&std::ffi::OsStr>, + source: &Path, + ptx: &Path, + require_kernel_build: bool, +) -> bool { + let compiled = nvcc.is_some_and(|nvcc| { + Command::new(nvcc) + .args(["--ptx", "-O3", "--std=c++14", "--fmad=false"]) + .arg(source) + .arg("-o") + .arg(ptx) + .status() + .is_ok_and(|status| status.success()) + }); + + if compiled { + let mut bytes = fs::read(ptx).expect("read generated CUDA transcode PTX"); + if bytes.last().copied() != Some(0) { + bytes.push(0); + fs::write(ptx, bytes).expect("NUL-terminate generated CUDA transcode PTX"); + } + true + } else { + assert!( + !require_kernel_build, + "strict CUDA kernel build required, but nvcc failed for {}", + source.display() + ); + // No checked-in fallback exists for this kernel. Write a placeholder + // empty PTX module so `include_bytes!(OUT_DIR/transcode_kernels.ptx)` + // always resolves on non-nvcc hosts and the Rust dispatch type-checks. + // It is never loaded at runtime: the dispatch first checks the + // `j2k_cuda_transcode_ptx_built` cfg and returns a typed error. + fs::write(ptx, b".version 7.0\n.target sm_52\n.address_size 64\n\0") + .expect("write placeholder transcode PTX"); + false + } +} + +fn compile_or_copy_ptx( + nvcc: Option<&std::ffi::OsStr>, + source: &Path, + fallback: &Path, + ptx: &Path, + require_kernel_build: bool, +) -> bool { + // --fmad=false: native (Rust/LLVM) does not contract a*b+c into a single-rounding + // FMA; nvcc does by default. Disabling it keeps the f32 DWT/RCT lossless path + // bit-identical to the native reference (byte-parity requirement). + let compiled = nvcc.is_some_and(|nvcc| { + Command::new(nvcc) + .args(["--ptx", "-O3", "--std=c++14", "--fmad=false"]) + .arg(source) + .arg("-o") + .arg(ptx) + .status() + .is_ok_and(|status| status.success()) + }); + + if compiled { + let mut bytes = fs::read(ptx).expect("read generated CUDA PTX"); + if bytes.last().copied() != Some(0) { + bytes.push(0); + fs::write(ptx, bytes).expect("NUL-terminate generated CUDA PTX"); + } + true + } else { + assert!( + !require_kernel_build, + "strict CUDA kernel build required, but nvcc failed for {}", + source.display() + ); + let mut bytes = fs::read(fallback).expect("read fallback CUDA PTX"); + if bytes.last().copied() != Some(0) { + bytes.push(0); + } + fs::write(ptx, bytes).expect("write fallback CUDA PTX"); + false + } +} + +fn configured_nvcc(strict: bool) -> Option { + let nvcc = env::var_os("NVCC"); + if strict { + let nvcc = nvcc.expect("strict CUDA kernel build requires absolute NVCC"); + assert!( + Path::new(&nvcc).is_absolute(), + "strict CUDA kernel build requires absolute NVCC, got {}", + Path::new(&nvcc).display() + ); + Some(nvcc) + } else { + nvcc + } +} diff --git a/crates/j2k-cuda-runtime/docs/cuda-oxide-copy-u8-pr-notes.md b/crates/j2k-cuda-runtime/docs/cuda-oxide-copy-u8-pr-notes.md new file mode 100644 index 00000000..fe1e4997 --- /dev/null +++ b/crates/j2k-cuda-runtime/docs/cuda-oxide-copy-u8-pr-notes.md @@ -0,0 +1,72 @@ +# cuda-oxide CopyU8 Spike Notes + +## Summary + +- Adds an opt-in `cuda-oxide-copy-u8` feature for a Rust-authored `j2k_copy_u8` + CUDA kernel. +- Keeps the existing checked-in PTX `CudaKernel::CopyU8` path as the default. +- Loads the cuda-oxide PTX through the existing `CudaContext` Driver API module + boundary with a separate module-cache key, so parity can compare both paths in + one context. + +## How To Enable + +```bash +cargo test -p j2k-cuda-runtime --features cuda-oxide-copy-u8 --all-targets +``` + +The build script stages the cuda-oxide device crate into `OUT_DIR`, runs +`cargo oxide build --arch ${J2K_CUDA_OXIDE_ARCH:-sm_80}`, then includes the +generated NUL-terminated PTX when cuda-oxide is available. Ordinary +`--all-features` builds on unsupported hosts write a placeholder PTX and do not +set the generated-PTX cfg; runtime dispatch returns a typed error before loading +that placeholder. Set `J2K_REQUIRE_CUDA_OXIDE_COPY_U8=1` on a Linux cuda-oxide +host to make a missing generated PTX a build failure. + +## Build Friction + +- cuda-oxide is documented as an early alpha with expected bugs and API + breakage. +- cuda-oxide is currently Linux-only. On this macOS host, ordinary + `--all-features` builds skip generation with a warning; strict validation uses + `J2K_REQUIRE_CUDA_OXIDE_COPY_U8=1`. +- The documented toolchain is heavier than the existing PTX fallback: pinned + Rust nightly, CUDA Toolkit, LLVM 21+, Clang 21+, and `cargo-oxide`. +- The nested device crate follows cuda-oxide's standalone project template with + git dependencies pinned to `NVlabs/cuda-oxide` commit + `a9f964a956f397dd0b3c8db88a3ca5824186c261`. Broader migration should move + that pin through the normal dependency review process once the spike + graduates. + +## Migration Viability + +CopyU8 is viable as an isolated opt-in spike because it has a simple raw-pointer +ABI, no shared memory, and an existing CPU/CUDA parity surface. Broader kernel +migration should remain gated until a Linux CUDA runner validates generated PTX, +records build time, and confirms the pinned cuda-oxide toolchain remains +destabilizing default builds. + +## Guidance Applied + +- cuda-oxide docs: used `#[kernel]` plus a `#[cuda_module]` device crate; kept + the function name `j2k_copy_u8` because cuda-oxide preserves the original + function name as the PTX entry point; defaulted the basic build target to + `sm_80` with `J2K_CUDA_OXIDE_ARCH` override. Sources: + [book quick start](https://nvlabs.github.io/cuda-oxide/index.html), + [installation](https://nvlabs.github.io/cuda-oxide/getting-started/installation.html), + [launch config](https://nvlabs.github.io/cuda-oxide/gpu-programming/launching-kernels.html). +- Cargo features: kept `cuda-oxide-copy-u8` additive and out of `default`; the + existing PTX path is not disabled. Source: + [Cargo feature unification](https://doc.rust-lang.org/cargo/reference/features.html#feature-unification). +- Rust API Guidelines: used a direct, meaningful feature name without `use-` or + `with-`, and made unsupported explicit builds fail at the boundary. Sources: + [C-FEATURE](https://rust-lang.github.io/api-guidelines/naming.html#c-feature), + [C-VALIDATE](https://rust-lang.github.io/api-guidelines/dependability.html#c-validate). +- Clippy docs: did not add blanket lint allowances; the code stays within the + crate's existing pedantic lint setup and would only use targeted allowances if + a specific lint needed justification. Source: + [Clippy lint groups](https://doc.rust-lang.org/clippy/lints.html). +- Unsafe Code Guidelines: kept raw pointer unsafety inside the Rust-authored GPU + kernel and existing Driver API boundary, while exposing the same safe + `CudaKernelOutput` API to callers. Source: + [UCG glossary: soundness and unsafe burden](https://rust-lang.github.io/unsafe-code-guidelines/glossary.html#soundness-of-code--of-a-library). diff --git a/crates/j2k-cuda-runtime/src/build_flags.rs b/crates/j2k-cuda-runtime/src/build_flags.rs new file mode 100644 index 00000000..1b3492e4 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/build_flags.rs @@ -0,0 +1,257 @@ +use crate::{driver::CuResult, error::CudaError, kernels::CudaKernel}; +use std::{os::raw::c_uint, sync::OnceLock}; + +pub(crate) const CUDA_SUCCESS: CuResult = 0; + +pub(crate) const PINNED_UPLOAD_STAGING_POOL_MAX: usize = 8; + +pub(crate) const PINNED_POOLED_I16_UPLOAD_MAX_BYTES: usize = 4 * 1024 * 1024; + +pub(crate) const DWT97_ROW_LIFT_MAX_WIDTH: i32 = 1024; + +pub(crate) const DWT97_ROW_LIFT_COOP_THREADS_X: c_uint = 128; + +pub(crate) const DWT97_ROW_LIFT_COOP_ROWS_PER_BLOCK: c_uint = 4; + +pub(crate) const CUDA_IDWT_TRACE_ENV_VAR: &str = "J2K_CUDA_IDWT_TRACE"; + +#[cfg(feature = "cuda-oxide-j2k-encode")] +pub(crate) const CUDA_OXIDE_J2K_ENCODE_ENV_VAR: &str = "J2K_CUDA_USE_OXIDE_J2K_ENCODE"; + +#[cfg(feature = "cuda-oxide-j2k-decode-store")] +pub(crate) const CUDA_OXIDE_J2K_DECODE_STORE_ENV_VAR: &str = "J2K_CUDA_USE_OXIDE_J2K_DECODE_STORE"; + +#[cfg(feature = "cuda-oxide-j2k-dequantize")] +pub(crate) const CUDA_OXIDE_J2K_DEQUANTIZE_ENV_VAR: &str = "J2K_CUDA_USE_OXIDE_J2K_DEQUANTIZE"; + +#[cfg(feature = "cuda-oxide-j2k-idwt")] +pub(crate) const CUDA_OXIDE_J2K_IDWT_ENV_VAR: &str = "J2K_CUDA_USE_OXIDE_J2K_IDWT"; + +#[cfg(feature = "cuda-oxide-transcode")] +pub(crate) const CUDA_OXIDE_TRANSCODE_ENV_VAR: &str = "J2K_CUDA_USE_OXIDE_TRANSCODE"; + +pub(crate) const DWT97_FUSED_COLUMN_QUANTIZE_DISABLE_ENV_VAR: &str = + "J2K_CUDA_DISABLE_DWT97_FUSED_COLUMN_QUANTIZE"; + +pub(crate) static CUDA_STAGE_TIMINGS_DISABLED: OnceLock = OnceLock::new(); + +pub(crate) static DWT97_FUSED_COLUMN_QUANTIZE_DISABLED: OnceLock = OnceLock::new(); + +#[cfg(feature = "cuda-oxide-j2k-encode")] +pub(crate) static CUDA_OXIDE_J2K_ENCODE_ENABLED: OnceLock = OnceLock::new(); + +#[cfg(feature = "cuda-oxide-j2k-decode-store")] +pub(crate) static CUDA_OXIDE_J2K_DECODE_STORE_ENABLED: OnceLock = OnceLock::new(); + +#[cfg(feature = "cuda-oxide-j2k-dequantize")] +pub(crate) static CUDA_OXIDE_J2K_DEQUANTIZE_ENABLED: OnceLock = OnceLock::new(); + +#[cfg(feature = "cuda-oxide-j2k-idwt")] +pub(crate) static CUDA_OXIDE_J2K_IDWT_ENABLED: OnceLock = OnceLock::new(); + +#[cfg(feature = "cuda-oxide-transcode")] +pub(crate) static CUDA_OXIDE_TRANSCODE_ENABLED: OnceLock = OnceLock::new(); + +pub(crate) fn cuda_stage_timings_disabled() -> bool { + *CUDA_STAGE_TIMINGS_DISABLED + .get_or_init(|| std::env::var_os("J2K_CUDA_DISABLE_STAGE_TIMINGS").is_some()) +} + +pub(crate) fn dwt97_fused_column_quantize_disabled() -> bool { + *DWT97_FUSED_COLUMN_QUANTIZE_DISABLED + .get_or_init(|| std::env::var_os(DWT97_FUSED_COLUMN_QUANTIZE_DISABLE_ENV_VAR).is_some()) +} + +#[cfg(feature = "cuda-oxide-j2k-encode")] +pub(crate) fn cuda_oxide_j2k_encode_enabled() -> bool { + *CUDA_OXIDE_J2K_ENCODE_ENABLED + .get_or_init(|| std::env::var_os(CUDA_OXIDE_J2K_ENCODE_ENV_VAR).is_some()) +} + +#[cfg(feature = "cuda-oxide-j2k-decode-store")] +pub(crate) fn cuda_oxide_j2k_decode_store_enabled() -> bool { + *CUDA_OXIDE_J2K_DECODE_STORE_ENABLED + .get_or_init(|| std::env::var_os(CUDA_OXIDE_J2K_DECODE_STORE_ENV_VAR).is_some()) +} + +#[cfg(feature = "cuda-oxide-j2k-dequantize")] +pub(crate) fn cuda_oxide_j2k_dequantize_enabled() -> bool { + *CUDA_OXIDE_J2K_DEQUANTIZE_ENABLED + .get_or_init(|| std::env::var_os(CUDA_OXIDE_J2K_DEQUANTIZE_ENV_VAR).is_some()) +} + +#[cfg(feature = "cuda-oxide-j2k-idwt")] +pub(crate) fn cuda_oxide_j2k_idwt_enabled() -> bool { + *CUDA_OXIDE_J2K_IDWT_ENABLED + .get_or_init(|| std::env::var_os(CUDA_OXIDE_J2K_IDWT_ENV_VAR).is_some()) +} + +#[cfg(feature = "cuda-oxide-transcode")] +pub(crate) fn cuda_oxide_transcode_enabled() -> bool { + *CUDA_OXIDE_TRANSCODE_ENABLED + .get_or_init(|| std::env::var_os(CUDA_OXIDE_TRANSCODE_ENV_VAR).is_some()) +} + +pub(crate) fn ensure_kernel_ptx_built(kernel: CudaKernel) -> Result<(), CudaError> { + let message = match kernel { + CudaKernel::J2kDeinterleaveToF32 + | CudaKernel::J2kForwardRct + | CudaKernel::J2kForwardIct + | CudaKernel::J2kForwardDwt53Horizontal + | CudaKernel::J2kForwardDwt53Vertical + | CudaKernel::J2kForwardDwt97Horizontal + | CudaKernel::J2kForwardDwt97Vertical + | CudaKernel::J2kQuantizeSubband + | CudaKernel::J2kQuantizeSubbandStrided + if !J2K_ENCODE_PTX_BUILT_FROM_CUDA => + { + Some("JPEG 2000 encode CUDA PTX was not built from j2k_encode_kernels.cu") + } + CudaKernel::Htj2kEncodeCodeblock + | CudaKernel::Htj2kEncodeCodeblocks + | CudaKernel::Htj2kPacketizeCleanup + if !HTJ2K_ENCODE_PTX_BUILT_FROM_CUDA => + { + Some("HTJ2K encode CUDA PTX was not built from htj2k_encode_kernels.cu") + } + CudaKernel::TranscodeReversible53Idct + | CudaKernel::TranscodeReversible53VerticalLow + | CudaKernel::TranscodeReversible53VerticalHigh + | CudaKernel::TranscodeReversible53HorizontalLow + | CudaKernel::TranscodeReversible53HorizontalHigh + | CudaKernel::TranscodeDwt97Idct + | CudaKernel::TranscodeDwt97RowLift + | CudaKernel::TranscodeDwt97ColumnLift + | CudaKernel::TranscodeDwt97IdctBatch + | CudaKernel::TranscodeDwt97IdctI16Batch + | CudaKernel::TranscodeDwt97RowLiftBatch + | CudaKernel::TranscodeDwt97RowLiftBatchCoop + | CudaKernel::TranscodeDwt97ColumnLiftBatch + | CudaKernel::TranscodeDwt97QuantizeCodeblocks + | CudaKernel::TranscodeDwt97ColumnLiftQuantizeCodeblocksBatch + if !TRANSCODE_PTX_BUILT_FROM_CUDA => + { + Some("transcode CUDA PTX was not built from transcode_kernels.cu") + } + _ => None, + }; + match message { + Some(message) => Err(CudaError::InvalidArgument { + message: message.to_string(), + }), + None => Ok(()), + } +} + +#[cfg(feature = "cuda-oxide-copy-u8")] +pub(crate) fn ensure_cuda_oxide_copy_u8_ptx_built() -> Result<(), CudaError> { + if CUDA_OXIDE_COPY_U8_PTX_BUILT { + Ok(()) + } else { + Err(CudaError::InvalidArgument { + message: "cuda-oxide CopyU8 PTX was not built; set J2K_REQUIRE_CUDA_OXIDE_COPY_U8 on a Linux cuda-oxide host to require it".to_string(), + }) + } +} + +#[cfg(feature = "cuda-oxide-j2k-encode")] +pub(crate) fn ensure_cuda_oxide_j2k_encode_ptx_built() -> Result<(), CudaError> { + if CUDA_OXIDE_J2K_ENCODE_PTX_BUILT { + Ok(()) + } else { + Err(CudaError::InvalidArgument { + message: "cuda-oxide J2K encode PTX was not built; set J2K_REQUIRE_CUDA_OXIDE_J2K_ENCODE on a Linux cuda-oxide host to require it".to_string(), + }) + } +} + +#[cfg(feature = "cuda-oxide-j2k-decode-store")] +pub(crate) fn ensure_cuda_oxide_j2k_decode_store_ptx_built() -> Result<(), CudaError> { + if CUDA_OXIDE_J2K_DECODE_STORE_PTX_BUILT { + Ok(()) + } else { + Err(CudaError::InvalidArgument { + message: "cuda-oxide J2K decode store PTX was not built; set J2K_REQUIRE_CUDA_OXIDE_J2K_DECODE_STORE on a Linux cuda-oxide host to require it".to_string(), + }) + } +} + +#[cfg(feature = "cuda-oxide-j2k-dequantize")] +pub(crate) fn ensure_cuda_oxide_j2k_dequantize_ptx_built() -> Result<(), CudaError> { + if CUDA_OXIDE_J2K_DEQUANTIZE_PTX_BUILT { + Ok(()) + } else { + Err(CudaError::InvalidArgument { + message: "cuda-oxide J2K dequantize PTX was not built; set J2K_REQUIRE_CUDA_OXIDE_J2K_DEQUANTIZE on a Linux cuda-oxide host to require it".to_string(), + }) + } +} + +#[cfg(feature = "cuda-oxide-j2k-idwt")] +pub(crate) fn ensure_cuda_oxide_j2k_idwt_ptx_built() -> Result<(), CudaError> { + if CUDA_OXIDE_J2K_IDWT_PTX_BUILT { + Ok(()) + } else { + Err(CudaError::InvalidArgument { + message: "cuda-oxide J2K IDWT PTX was not built; set J2K_REQUIRE_CUDA_OXIDE_J2K_IDWT on a Linux cuda-oxide host to require it".to_string(), + }) + } +} + +#[cfg(feature = "cuda-oxide-transcode")] +pub(crate) fn ensure_cuda_oxide_transcode_ptx_built() -> Result<(), CudaError> { + if CUDA_OXIDE_TRANSCODE_PTX_BUILT { + Ok(()) + } else { + Err(CudaError::InvalidArgument { + message: "cuda-oxide transcode PTX was not built; set J2K_REQUIRE_CUDA_OXIDE_TRANSCODE on a Linux cuda-oxide host to require it".to_string(), + }) + } +} + +pub(crate) const J2K_ENCODE_PTX_BUILT_FROM_CUDA: bool = cfg!(j2k_cuda_j2k_encode_ptx_built); + +pub(crate) const HTJ2K_ENCODE_PTX_BUILT_FROM_CUDA: bool = cfg!(j2k_cuda_htj2k_encode_ptx_built); + +/// True when the coefficient-domain transcode kernels were compiled by nvcc +/// (the runner). When false, build.rs wrote a placeholder PTX, so dispatch +/// returns a typed error instead of loading a non-existent kernel. +pub(crate) const TRANSCODE_PTX_BUILT_FROM_CUDA: bool = cfg!(j2k_cuda_transcode_ptx_built); + +#[cfg(feature = "cuda-oxide-copy-u8")] +pub(crate) const CUDA_OXIDE_COPY_U8_PTX_BUILT: bool = cfg!(j2k_cuda_oxide_copy_u8_built); + +#[cfg(feature = "cuda-oxide-j2k-encode")] +pub(crate) const CUDA_OXIDE_J2K_ENCODE_PTX_BUILT: bool = cfg!(j2k_cuda_oxide_j2k_encode_built); + +#[cfg(feature = "cuda-oxide-j2k-decode-store")] +pub(crate) const CUDA_OXIDE_J2K_DECODE_STORE_PTX_BUILT: bool = + cfg!(j2k_cuda_oxide_j2k_decode_store_built); + +#[cfg(feature = "cuda-oxide-j2k-dequantize")] +pub(crate) const CUDA_OXIDE_J2K_DEQUANTIZE_PTX_BUILT: bool = + cfg!(j2k_cuda_oxide_j2k_dequantize_built); + +#[cfg(feature = "cuda-oxide-j2k-idwt")] +pub(crate) const CUDA_OXIDE_J2K_IDWT_PTX_BUILT: bool = cfg!(j2k_cuda_oxide_j2k_idwt_built); + +#[cfg(feature = "cuda-oxide-transcode")] +pub(crate) const CUDA_OXIDE_TRANSCODE_PTX_BUILT: bool = cfg!(j2k_cuda_oxide_transcode_built); + +/// Whether the coefficient-domain transcode kernels were compiled (runner). +/// Backends check this to fall back to the scalar oracle when the kernels are +/// unavailable (e.g. a non-nvcc build) instead of attempting a device launch. +#[must_use] +pub fn transcode_kernels_built() -> bool { + if TRANSCODE_PTX_BUILT_FROM_CUDA { + return true; + } + #[cfg(feature = "cuda-oxide-transcode")] + { + cuda_oxide_transcode_enabled() && CUDA_OXIDE_TRANSCODE_PTX_BUILT + } + #[cfg(not(feature = "cuda-oxide-transcode"))] + { + false + } +} diff --git a/crates/j2k-cuda-runtime/src/bytes.rs b/crates/j2k-cuda-runtime/src/bytes.rs new file mode 100644 index 00000000..ca6cb505 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/bytes.rs @@ -0,0 +1,279 @@ +use crate::{ + error::CudaError, + htj2k_decode::{ + CudaHtj2kCleanupMultiKernelJob, CudaHtj2kCodeBlockKernelJob, CudaHtj2kDequantizeKernelJob, + CudaHtj2kStatus, + }, + htj2k_encode::{ + CudaHtj2kEncodeCompactJob, CudaHtj2kEncodeKernelJob, CudaHtj2kEncodeMultiInputKernelJob, + CudaHtj2kEncodeParams, CudaHtj2kEncodeStatus, + }, + htj2k_packetize::{ + CudaHtj2kPacketizationBlock, CudaHtj2kPacketizationKernelPacket, + CudaHtj2kPacketizationStatus, CudaHtj2kPacketizationSubband, + CudaHtj2kPacketizationSubbandTagState, CudaHtj2kPacketizationTagNodeState, + }, + j2k_decode::{ + CudaJ2kIdwtJob, CudaJ2kIdwtMultiKernelJob, CudaJ2kInverseMctJob, CudaJ2kStoreGray16Job, + CudaJ2kStoreGray8Job, CudaJ2kStoreRgb16Job, CudaJ2kStoreRgb16MctJob, CudaJ2kStoreRgb8Job, + CudaJ2kStoreRgb8MctBatchJob, + }, + jpeg::{ + CudaJpegDecodeStatus, CudaJpegEntropyCheckpoint, CudaJpegEntropyOverflowState, + CudaJpegEntropySyncState, CudaJpegHuffmanTable, + }, +}; +use j2k_core::GpuAbi; + +macro_rules! impl_cuda_gpu_abi { + ($($ty:ty),+ $(,)?) => { + $( + // SAFETY: These repr(C) structs are copied byte-for-byte to CUDA kernels with matching ABI tests. + unsafe impl GpuAbi for $ty { + const NAME: &'static str = stringify!($ty); + } + )+ + }; +} + +impl_cuda_gpu_abi! { + CudaJpegHuffmanTable, + CudaJpegEntropyCheckpoint, + CudaJpegDecodeStatus, + CudaJpegEntropySyncState, + CudaJpegEntropyOverflowState, + CudaHtj2kEncodeParams, + CudaHtj2kEncodeStatus, + CudaHtj2kEncodeKernelJob, + CudaHtj2kEncodeMultiInputKernelJob, + CudaHtj2kEncodeCompactJob, + CudaHtj2kPacketizationKernelPacket, + CudaHtj2kPacketizationSubband, + CudaHtj2kPacketizationBlock, + CudaHtj2kPacketizationSubbandTagState, + CudaHtj2kPacketizationTagNodeState, + CudaHtj2kPacketizationStatus, + CudaHtj2kCodeBlockKernelJob, + CudaHtj2kCleanupMultiKernelJob, + CudaHtj2kDequantizeKernelJob, + CudaHtj2kStatus, + CudaJ2kIdwtJob, + CudaJ2kIdwtMultiKernelJob, + CudaJ2kStoreGray8Job, + CudaJ2kStoreGray16Job, + CudaJ2kInverseMctJob, + CudaJ2kStoreRgb8Job, + CudaJ2kStoreRgb16Job, + CudaJ2kStoreRgb8MctBatchJob, + CudaJ2kStoreRgb16MctJob, +} + +pub(crate) fn f32_slice_as_bytes(samples: &[f32]) -> &[u8] { + ::slice_as_bytes(samples) +} + +pub(crate) fn f32_slice_as_bytes_mut(samples: &mut [f32]) -> &mut [u8] { + ::slice_as_bytes_mut(samples) +} + +pub(crate) fn i16_slice_as_bytes(samples: &[i16]) -> &[u8] { + ::slice_as_bytes(samples) +} + +pub(crate) fn i32_slice_as_bytes(samples: &[i32]) -> &[u8] { + ::slice_as_bytes(samples) +} + +pub(crate) fn i32_slice_as_bytes_mut(samples: &mut [i32]) -> &mut [u8] { + ::slice_as_bytes_mut(samples) +} + +pub(crate) fn u16_slice_as_bytes(samples: &[u16]) -> &[u8] { + ::slice_as_bytes(samples) +} + +#[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] +pub(crate) fn cuda_jpeg_huffman_table_as_bytes(table: &CudaJpegHuffmanTable) -> &[u8] { + ::as_bytes(table) +} + +#[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] +pub(crate) fn cuda_jpeg_entropy_checkpoints_as_bytes( + checkpoints: &[CudaJpegEntropyCheckpoint], +) -> &[u8] { + ::slice_as_bytes(checkpoints) +} + +#[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] +pub(crate) fn cuda_jpeg_decode_statuses_as_bytes(statuses: &[CudaJpegDecodeStatus]) -> &[u8] { + ::slice_as_bytes(statuses) +} + +#[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] +pub(crate) fn cuda_jpeg_decode_statuses_as_bytes_mut( + statuses: &mut [CudaJpegDecodeStatus], +) -> &mut [u8] { + ::slice_as_bytes_mut(statuses) +} + +#[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] +pub(crate) fn cuda_jpeg_entropy_sync_states_as_bytes(states: &[CudaJpegEntropySyncState]) -> &[u8] { + ::slice_as_bytes(states) +} + +#[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] +pub(crate) fn cuda_jpeg_entropy_sync_states_as_bytes_mut( + states: &mut [CudaJpegEntropySyncState], +) -> &mut [u8] { + ::slice_as_bytes_mut(states) +} + +#[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] +pub(crate) fn cuda_jpeg_entropy_overflow_states_as_bytes( + states: &[CudaJpegEntropyOverflowState], +) -> &[u8] { + ::slice_as_bytes(states) +} + +#[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] +pub(crate) fn cuda_jpeg_entropy_overflow_states_as_bytes_mut( + states: &mut [CudaJpegEntropyOverflowState], +) -> &mut [u8] { + ::slice_as_bytes_mut(states) +} + +pub(crate) fn store_gray8_job_as_bytes(job: &CudaJ2kStoreGray8Job) -> &[u8] { + ::as_bytes(job) +} + +pub(crate) fn store_gray16_job_as_bytes(job: &CudaJ2kStoreGray16Job) -> &[u8] { + ::as_bytes(job) +} + +pub(crate) fn inverse_mct_job_as_bytes(job: &CudaJ2kInverseMctJob) -> &[u8] { + ::as_bytes(job) +} + +pub(crate) fn store_rgb8_job_as_bytes(job: &CudaJ2kStoreRgb8Job) -> &[u8] { + ::as_bytes(job) +} + +pub(crate) fn store_rgb16_job_as_bytes(job: &CudaJ2kStoreRgb16Job) -> &[u8] { + ::as_bytes(job) +} + +pub(crate) fn store_rgb8_mct_batch_jobs_as_bytes(jobs: &[CudaJ2kStoreRgb8MctBatchJob]) -> &[u8] { + ::slice_as_bytes(jobs) +} + +pub(crate) fn store_rgb16_mct_job_as_bytes(job: &CudaJ2kStoreRgb16MctJob) -> &[u8] { + ::as_bytes(job) +} + +pub(crate) fn htj2k_encode_params_as_bytes(params: &CudaHtj2kEncodeParams) -> &[u8] { + ::as_bytes(params) +} + +pub(crate) fn htj2k_encode_status_as_bytes(status: &CudaHtj2kEncodeStatus) -> &[u8] { + ::as_bytes(status) +} + +pub(crate) fn htj2k_encode_status_as_bytes_mut(status: &mut CudaHtj2kEncodeStatus) -> &mut [u8] { + ::slice_as_bytes_mut(std::slice::from_mut(status)) +} + +pub(crate) fn htj2k_encode_statuses_byte_len(count: usize) -> Result { + count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: count }) +} + +pub(crate) fn htj2k_encode_jobs_as_bytes(jobs: &[CudaHtj2kEncodeKernelJob]) -> &[u8] { + ::slice_as_bytes(jobs) +} + +pub(crate) fn htj2k_encode_multi_input_jobs_as_bytes( + jobs: &[CudaHtj2kEncodeMultiInputKernelJob], +) -> &[u8] { + ::slice_as_bytes(jobs) +} + +pub(crate) fn htj2k_encode_compact_jobs_as_bytes(jobs: &[CudaHtj2kEncodeCompactJob]) -> &[u8] { + ::slice_as_bytes(jobs) +} + +pub(crate) fn htj2k_encode_statuses_as_bytes_mut( + statuses: &mut [CudaHtj2kEncodeStatus], +) -> &mut [u8] { + ::slice_as_bytes_mut(statuses) +} + +pub(crate) fn htj2k_packetization_packets_as_bytes( + packets: &[CudaHtj2kPacketizationKernelPacket], +) -> &[u8] { + ::slice_as_bytes(packets) +} + +pub(crate) fn htj2k_packetization_subbands_as_bytes( + subbands: &[CudaHtj2kPacketizationSubband], +) -> &[u8] { + ::slice_as_bytes(subbands) +} + +pub(crate) fn htj2k_packetization_blocks_as_bytes(blocks: &[CudaHtj2kPacketizationBlock]) -> &[u8] { + ::slice_as_bytes(blocks) +} + +pub(crate) fn htj2k_packetization_subband_tag_states_as_bytes( + states: &[CudaHtj2kPacketizationSubbandTagState], +) -> &[u8] { + ::slice_as_bytes(states) +} + +pub(crate) fn htj2k_packetization_tag_nodes_as_bytes( + nodes: &[CudaHtj2kPacketizationTagNodeState], +) -> &[u8] { + ::slice_as_bytes(nodes) +} + +pub(crate) fn htj2k_packetization_statuses_as_bytes( + statuses: &[CudaHtj2kPacketizationStatus], +) -> &[u8] { + ::slice_as_bytes(statuses) +} + +pub(crate) fn htj2k_packetization_statuses_as_bytes_mut( + statuses: &mut [CudaHtj2kPacketizationStatus], +) -> &mut [u8] { + ::slice_as_bytes_mut(statuses) +} + +pub(crate) fn htj2k_jobs_as_bytes(jobs: &[CudaHtj2kCodeBlockKernelJob]) -> &[u8] { + ::slice_as_bytes(jobs) +} + +pub(crate) fn htj2k_cleanup_multi_jobs_as_bytes(jobs: &[CudaHtj2kCleanupMultiKernelJob]) -> &[u8] { + ::slice_as_bytes(jobs) +} + +pub(crate) fn htj2k_dequantize_jobs_as_bytes(jobs: &[CudaHtj2kDequantizeKernelJob]) -> &[u8] { + ::slice_as_bytes(jobs) +} + +pub(crate) fn htj2k_statuses_byte_len(count: usize) -> Result { + count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: count }) +} + +pub(crate) fn htj2k_statuses_as_bytes_mut(statuses: &mut [CudaHtj2kStatus]) -> &mut [u8] { + ::slice_as_bytes_mut(statuses) +} + +pub(crate) fn idwt_job_as_bytes(job: &CudaJ2kIdwtJob) -> &[u8] { + ::as_bytes(job) +} + +pub(crate) fn idwt_multi_jobs_as_bytes(jobs: &[CudaJ2kIdwtMultiKernelJob]) -> &[u8] { + ::slice_as_bytes(jobs) +} diff --git a/crates/j2k-cuda-runtime/src/context.rs b/crates/j2k-cuda-runtime/src/context.rs new file mode 100644 index 00000000..f8274160 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/context.rs @@ -0,0 +1,872 @@ +#[cfg(feature = "cuda-oxide-copy-u8")] +use crate::build_flags::ensure_cuda_oxide_copy_u8_ptx_built; +#[cfg(feature = "cuda-oxide-j2k-decode-store")] +use crate::build_flags::ensure_cuda_oxide_j2k_decode_store_ptx_built; +#[cfg(feature = "cuda-oxide-j2k-dequantize")] +use crate::build_flags::ensure_cuda_oxide_j2k_dequantize_ptx_built; +#[cfg(feature = "cuda-oxide-j2k-encode")] +use crate::build_flags::ensure_cuda_oxide_j2k_encode_ptx_built; +#[cfg(feature = "cuda-oxide-j2k-idwt")] +use crate::build_flags::ensure_cuda_oxide_j2k_idwt_ptx_built; +#[cfg(feature = "cuda-oxide-transcode")] +use crate::build_flags::ensure_cuda_oxide_transcode_ptx_built; +#[cfg(any( + feature = "cuda-oxide-copy-u8", + feature = "cuda-oxide-j2k-encode", + feature = "cuda-oxide-j2k-decode-store", + feature = "cuda-oxide-j2k-dequantize", + feature = "cuda-oxide-j2k-idwt", + feature = "cuda-oxide-transcode" +))] +use crate::kernels; +use crate::{ + build_flags::{ensure_kernel_ptx_built, CUDA_IDWT_TRACE_ENV_VAR}, + bytes::{f32_slice_as_bytes_mut, i32_slice_as_bytes_mut}, + driver::{CuContext, CuFunction, CuModule, Driver}, + error::CudaError, + execution::{CudaExecutionStats, CudaLaunchMode}, + htj2k_decode::{ + htj2k_decode_needs_zero_fill, CudaHtj2kCodeBlockJob, CudaHtj2kDecodeOutput, + CudaHtj2kDecodeStageTimings, CudaQueuedHtj2kCleanup, + }, + htj2k_encode::{ + CudaHtj2kEncodeStageTimings, CudaHtj2kEncodeStatus, CudaHtj2kEncodedCodeBlock, + CudaHtj2kEncodedCodeBlocks, + }, + kernels::CudaKernel, + memory::{pooled_device_buffer, CudaDeviceBuffer, CudaPooledDeviceBuffer}, +}; +use std::{ + collections::HashMap, + ffi::{c_char, c_void}, + sync::{Arc, Mutex}, +}; + +pub(crate) struct ContextInner { + pub(crate) driver: Driver, + pub(crate) context: CuContext, + pub(crate) modules: Mutex>, + pub(crate) pinned_upload_staging: Mutex>, +} + +pub(crate) struct PinnedUploadStaging { + pub(crate) ptr: *mut u8, + pub(crate) len: usize, +} + +impl PinnedUploadStaging { + pub(crate) fn as_slice(&self) -> &[u8] { + if self.len == 0 { + &[] + } else { + // SAFETY: ptr is a live pinned allocation of len bytes. + unsafe { std::slice::from_raw_parts(self.ptr.cast_const(), self.len) } + } + } + + pub(crate) fn as_mut_slice(&mut self) -> &mut [u8] { + if self.len == 0 { + &mut [] + } else { + // SAFETY: ptr is uniquely borrowed through &mut self and covers len + // bytes allocated by CUDA. + unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) } + } + } + + pub(crate) fn free(self, driver: &Driver) -> Result<(), CudaError> { + if self.ptr.is_null() { + return Ok(()); + } + // SAFETY: ptr was returned by cuMemHostAlloc for this process. + driver.check("cuMemFreeHost", unsafe { + (driver.cu_mem_free_host)(self.ptr.cast()) + }) + } +} + +// SAFETY: The pinned allocation is owned by this value. Mutable access requires +// &mut self, and freeing is explicitly coordinated by the owning CudaContext. +unsafe impl Send for PinnedUploadStaging {} + +impl ContextInner { + pub(crate) fn set_current(&self) -> Result<(), CudaError> { + // SAFETY: context is created by cuCtxCreate_v2 and remains valid while + // ContextInner is alive. + self.driver.check("cuCtxSetCurrent", unsafe { + (self.driver.cu_ctx_set_current)(self.context) + }) + } + + pub(crate) fn kernel_function(&self, kernel: CudaKernel) -> Result { + self.kernel_function_from_key(CompiledKernelKey::Builtin(kernel)) + } + + #[cfg(feature = "cuda-oxide-copy-u8")] + pub(crate) fn cuda_oxide_copy_u8_kernel_function(&self) -> Result { + ensure_cuda_oxide_copy_u8_ptx_built()?; + self.kernel_function_from_key(CompiledKernelKey::CudaOxideCopyU8) + } + + #[cfg(feature = "cuda-oxide-j2k-encode")] + pub(crate) fn cuda_oxide_j2k_encode_kernel_function( + &self, + kernel: CudaKernel, + ) -> Result { + ensure_cuda_oxide_j2k_encode_ptx_built()?; + if !kernel.is_cuda_oxide_j2k_encode_stage() { + return Err(CudaError::InvalidArgument { + message: format!("kernel {kernel:?} is not a J2K encode cuda-oxide stage"), + }); + } + self.kernel_function_from_key(CompiledKernelKey::CudaOxideJ2kEncode(kernel)) + } + + #[cfg(feature = "cuda-oxide-j2k-decode-store")] + pub(crate) fn cuda_oxide_j2k_decode_store_kernel_function( + &self, + kernel: CudaKernel, + ) -> Result { + ensure_cuda_oxide_j2k_decode_store_ptx_built()?; + if !kernel.is_j2k_decode_store_stage() { + return Err(CudaError::InvalidArgument { + message: format!("kernel {kernel:?} is not a J2K decode-store cuda-oxide stage"), + }); + } + self.kernel_function_from_key(CompiledKernelKey::CudaOxideJ2kDecodeStore(kernel)) + } + + #[cfg(feature = "cuda-oxide-j2k-dequantize")] + pub(crate) fn cuda_oxide_j2k_dequantize_kernel_function( + &self, + kernel: CudaKernel, + ) -> Result { + ensure_cuda_oxide_j2k_dequantize_ptx_built()?; + if !kernel.is_j2k_dequantize_stage() { + return Err(CudaError::InvalidArgument { + message: format!("kernel {kernel:?} is not a J2K dequantize cuda-oxide stage"), + }); + } + self.kernel_function_from_key(CompiledKernelKey::CudaOxideJ2kDequantize(kernel)) + } + + #[cfg(feature = "cuda-oxide-j2k-idwt")] + pub(crate) fn cuda_oxide_j2k_idwt_kernel_function( + &self, + kernel: CudaKernel, + ) -> Result { + ensure_cuda_oxide_j2k_idwt_ptx_built()?; + if !kernel.is_j2k_idwt_stage() { + return Err(CudaError::InvalidArgument { + message: format!("kernel {kernel:?} is not a J2K IDWT cuda-oxide stage"), + }); + } + self.kernel_function_from_key(CompiledKernelKey::CudaOxideJ2kIdwt(kernel)) + } + + #[cfg(feature = "cuda-oxide-transcode")] + pub(crate) fn cuda_oxide_transcode_kernel_function( + &self, + kernel: CudaKernel, + ) -> Result { + ensure_cuda_oxide_transcode_ptx_built()?; + if !kernel.is_cuda_oxide_transcode_stage() { + return Err(CudaError::InvalidArgument { + message: format!("kernel {kernel:?} is not a supported transcode cuda-oxide stage"), + }); + } + self.kernel_function_from_key(CompiledKernelKey::CudaOxideTranscode(kernel)) + } + + fn kernel_function_from_key(&self, key: CompiledKernelKey) -> Result { + match key { + CompiledKernelKey::Builtin(kernel) => ensure_kernel_ptx_built(kernel)?, + #[cfg(feature = "cuda-oxide-copy-u8")] + CompiledKernelKey::CudaOxideCopyU8 => {} + #[cfg(feature = "cuda-oxide-j2k-encode")] + CompiledKernelKey::CudaOxideJ2kEncode(_) => {} + #[cfg(feature = "cuda-oxide-j2k-decode-store")] + CompiledKernelKey::CudaOxideJ2kDecodeStore(_) => {} + #[cfg(feature = "cuda-oxide-j2k-dequantize")] + CompiledKernelKey::CudaOxideJ2kDequantize(_) => {} + #[cfg(feature = "cuda-oxide-j2k-idwt")] + CompiledKernelKey::CudaOxideJ2kIdwt(_) => {} + #[cfg(feature = "cuda-oxide-transcode")] + CompiledKernelKey::CudaOxideTranscode(_) => {} + } + self.set_current()?; + let mut modules = self + .modules + .lock() + .map_err(|error| CudaError::StatePoisoned { + message: error.to_string(), + })?; + if let Some(compiled) = modules.get(&key) { + return Ok(compiled.function); + } + + let compiled = CompiledKernel::load(self, key)?; + let function = compiled.function; + modules.insert(key, compiled); + Ok(function) + } +} + +impl Drop for ContextInner { + fn drop(&mut self) { + if !self.context.is_null() { + let _ = self.set_current(); + let pinned_upload_staging = match self.pinned_upload_staging.get_mut() { + Ok(pinned_upload_staging) => pinned_upload_staging, + Err(poisoned) => poisoned.into_inner(), + }; + for staging in pinned_upload_staging.drain(..) { + let _ = staging.free(&self.driver); + } + let modules = match self.modules.get_mut() { + Ok(modules) => modules, + Err(poisoned) => poisoned.into_inner(), + }; + for compiled in modules.drain().map(|(_, compiled)| compiled) { + // SAFETY: modules were loaded into this CUDA context. Drop + // cannot surface errors, so cleanup failures are ignored. + let _ = unsafe { (self.driver.cu_module_unload)(compiled.module) }; + } + // SAFETY: context was created by this ContextInner and cached + // modules have already been unloaded. + let _ = unsafe { (self.driver.cu_ctx_destroy)(self.context) }; + } + } +} + +// SAFETY: ContextInner owns an opaque CUDA context handle and synchronizes its +// Rust-side mutable caches with mutexes. +unsafe impl Send for ContextInner {} + +// SAFETY: All shared Rust state is mutex-protected, and CUDA operations set the +// current context before touching context-owned resources. +unsafe impl Sync for ContextInner {} + +/// CUDA driver context shared by J2K CUDA adapter crates. +#[derive(Clone)] +pub struct CudaContext { + pub(crate) inner: Arc, +} + +/// Host-visible compact HTJ2K cleanup-pass encode metadata for one code block. +#[derive(Debug)] +pub struct CudaHtj2kCompactEncodedCodeBlock { + pub(crate) payload_range: std::ops::Range, + pub(crate) status: CudaHtj2kEncodeStatus, + pub(crate) execution: CudaExecutionStats, + pub(crate) stage_timings: CudaHtj2kEncodeStageTimings, +} + +impl CudaHtj2kCompactEncodedCodeBlock { + /// Encoded cleanup-pass payload range in the batch payload. + pub fn payload_range(&self) -> std::ops::Range { + self.payload_range.clone() + } + + /// HTJ2K cleanup segment length in bytes. + pub fn cleanup_length(&self) -> u32 { + if self.status.number_of_coding_passes <= 1 { + self.status.data_len + } else { + self.status.reserved0 + } + } + + /// HTJ2K refinement segment length in bytes. + pub fn refinement_length(&self) -> u32 { + if self.status.number_of_coding_passes <= 1 { + 0 + } else { + self.status.reserved1 + } + } + + /// Number of coding passes in the encoded payload. + pub fn num_coding_passes(&self) -> u8 { + u8::try_from(self.status.number_of_coding_passes).unwrap_or(u8::MAX) + } + + /// Number of missing most-significant bitplanes. + pub fn num_zero_bitplanes(&self) -> u8 { + u8::try_from(self.status.missing_bit_planes).unwrap_or(u8::MAX) + } + + /// Consume this code block and return its payload range plus segment metadata. + pub fn into_parts(self) -> (std::ops::Range, u32, u32, u8, u8) { + let cleanup_length = if self.status.number_of_coding_passes <= 1 { + self.status.data_len + } else { + self.status.reserved0 + }; + let refinement_length = if self.status.number_of_coding_passes <= 1 { + 0 + } else { + self.status.reserved1 + }; + ( + self.payload_range, + cleanup_length, + refinement_length, + u8::try_from(self.status.number_of_coding_passes).unwrap_or(u8::MAX), + u8::try_from(self.status.missing_bit_planes).unwrap_or(u8::MAX), + ) + } + + /// Kernel status row downloaded after dispatch. + pub fn status(&self) -> CudaHtj2kEncodeStatus { + self.status + } + + /// CUDA execution counters for the encode dispatch. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// CUDA event timings for the encode dispatch. + pub fn stage_timings(&self) -> CudaHtj2kEncodeStageTimings { + self.stage_timings + } +} + +/// Host-visible compact HTJ2K cleanup-pass encode batch produced by one CUDA +/// kernel dispatch. +#[derive(Debug)] +pub struct CudaHtj2kCompactEncodedCodeBlocks { + pub(crate) payload: Vec, + pub(crate) code_blocks: Vec, + pub(crate) execution: CudaExecutionStats, + pub(crate) stage_timings: CudaHtj2kEncodeStageTimings, +} + +impl CudaHtj2kCompactEncodedCodeBlocks { + /// Compact encoded payload shared by all code-block ranges. + pub fn payload(&self) -> &[u8] { + &self.payload + } + + /// Encoded cleanup code-block metadata, in submitted-job order. + pub fn code_blocks(&self) -> &[CudaHtj2kCompactEncodedCodeBlock] { + &self.code_blocks + } + + /// Consume the batch and return its payload plus per-code-block metadata. + pub fn into_payload_and_code_blocks(self) -> (Vec, Vec) { + (self.payload, self.code_blocks) + } + + /// CUDA execution counters for the batch encode dispatch. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// CUDA event timings for the batch encode dispatch. + pub fn stage_timings(&self) -> CudaHtj2kEncodeStageTimings { + self.stage_timings + } + + pub(crate) fn into_owned_code_blocks(self) -> Result { + let Self { + payload, + code_blocks, + execution, + stage_timings, + } = self; + let code_blocks = code_blocks + .into_iter() + .map(|block| { + let CudaHtj2kCompactEncodedCodeBlock { + payload_range, + status, + execution, + stage_timings, + } = block; + if payload_range.start > payload_range.end || payload_range.end > payload.len() { + return Err(CudaError::LengthTooLarge { + len: payload_range.end, + }); + } + Ok(CudaHtj2kEncodedCodeBlock { + data: payload[payload_range].to_vec(), + status, + execution, + stage_timings, + }) + }) + .collect::, CudaError>>()?; + + Ok(CudaHtj2kEncodedCodeBlocks { + code_blocks, + execution, + stage_timings, + }) + } +} + +pub(crate) const HTJ2K_UVLC_ENCODE_TABLE_BYTES: usize = 75 * 6; + +impl CudaContext { + /// Create a context for the system default CUDA device. + pub fn system_default() -> Result { + let driver = Driver::load()?; + + // SAFETY: cuInit is the CUDA Driver API process initializer. + driver.check("cuInit", unsafe { (driver.cu_init)(0) })?; + + let mut count = 0; + // SAFETY: CUDA writes one integer device count to the provided pointer. + driver.check("cuDeviceGetCount", unsafe { + (driver.cu_device_get_count)(&raw mut count) + })?; + if count <= 0 { + return Err(CudaError::Unavailable { + message: "no CUDA devices reported by driver".to_string(), + }); + } + + let mut device = 0; + // SAFETY: device 0 is valid when count is greater than zero. + driver.check("cuDeviceGet", unsafe { + (driver.cu_device_get)(&raw mut device, 0) + })?; + + let mut context = std::ptr::null_mut(); + // SAFETY: CUDA writes a newly-created context handle for a valid device. + driver.check("cuCtxCreate_v2", unsafe { + (driver.cu_ctx_create)(&raw mut context, 0, device) + })?; + + Ok(Self { + inner: Arc::new(ContextInner { + driver, + context, + modules: Mutex::new(HashMap::new()), + pinned_upload_staging: Mutex::new(Vec::new()), + }), + }) + } + + /// Dequantize HTJ2K cleanup outputs using the metadata buffer already held + /// live by a queued cleanup launch. + pub fn j2k_dequantize_queued_htj2k_cleanup_with_pool( + &self, + cleanup: &CudaQueuedHtj2kCleanup, + ) -> Result { + self.inner.set_current()?; + if cleanup.status_count == 0 { + return Ok(CudaExecutionStats::default()); + } + let Some(jobs_buffer) = cleanup.resources.first() else { + return Err(CudaError::InvalidArgument { + message: "queued HTJ2K cleanup has no metadata buffer".to_string(), + }); + }; + self.launch_j2k_dequantize_htj2k_cleanup_jobs_multi( + pooled_device_buffer(jobs_buffer)?, + cleanup.status_count, + CudaLaunchMode::Sync, + )?; + Ok(CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }) + } + + pub(crate) fn decode_empty_htj2k_codeblocks( + &self, + jobs: &[CudaHtj2kCodeBlockJob], + output_words: usize, + ) -> Result { + self.inner.set_current()?; + let output_bytes = output_words + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: output_words })?; + let coefficients = self.allocate(output_bytes)?; + if htj2k_decode_needs_zero_fill(jobs, output_words)? { + self.memset_d32(&coefficients, 0, output_words)?; + } + Ok(CudaHtj2kDecodeOutput { + coefficients, + execution: CudaExecutionStats::default(), + statuses: Vec::new(), + stage_timings: CudaHtj2kDecodeStageTimings::default(), + }) + } +} + +impl std::fmt::Debug for CudaContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CudaContext").finish_non_exhaustive() + } +} + +/// Bundled CUDA kernel identifiers that can be preloaded by adapters. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +#[non_exhaustive] +pub enum CudaKernelName { + /// Byte-wise device copy kernel. + CopyU8, + /// JPEG 2000 pixel deinterleave/level-shift kernel. + J2kDeinterleaveToF32, + /// JPEG 2000 forward reversible color transform kernel. + J2kForwardRct, + /// JPEG 2000 forward irreversible color transform kernel. + J2kForwardIct, + /// JPEG 2000 forward 5/3 horizontal DWT kernel. + J2kForwardDwt53Horizontal, + /// JPEG 2000 forward 5/3 vertical DWT kernel. + J2kForwardDwt53Vertical, + /// JPEG 2000 forward 9/7 horizontal DWT kernel. + J2kForwardDwt97Horizontal, + /// JPEG 2000 forward 9/7 vertical DWT kernel. + J2kForwardDwt97Vertical, + /// JPEG 2000 sub-band quantization kernel. + J2kQuantizeSubband, + /// JPEG 2000 strided sub-band quantization kernel. + J2kQuantizeSubbandStrided, + /// HTJ2K entropy code-block decode kernel. + Htj2kDecodeCodeblocks, + /// HTJ2K cleanup-only decode plus dequantization kernel. + Htj2kDecodeCodeblocksMultiCleanupDequantize, + /// JPEG 2000 HTJ2K coefficient dequantization kernel. + J2kDequantizeHtj2kCodeblocks, + /// JPEG 2000 HTJ2K multi-buffer coefficient dequantization kernel. + J2kDequantizeHtj2kCodeblocksMulti, + /// JPEG 2000 HTJ2K multi-buffer dequantization from cleanup metadata. + J2kDequantizeHtj2kCleanupJobsMulti, + /// JPEG 2000 inverse DWT band interleave kernel. + J2kIdwtInterleave, + /// JPEG 2000 fused band interleave and reversible 5/3 horizontal lifting kernel. + J2kIdwtInterleaveHorizontal53Multi, + /// JPEG 2000 fused band interleave and irreversible 9/7 horizontal lifting kernel. + J2kIdwtInterleaveHorizontal97Multi, + /// JPEG 2000 inverse DWT horizontal lifting kernel. + J2kIdwtHorizontal, + /// JPEG 2000 inverse 5/3 DWT horizontal lifting kernel. + J2kIdwtHorizontal53, + /// JPEG 2000 inverse 9/7 DWT horizontal lifting kernel. + J2kIdwtHorizontal97, + /// JPEG 2000 inverse DWT vertical lifting kernel. + J2kIdwtVertical, + /// JPEG 2000 reversible 5/3 vertical lifting multi-target kernel. + J2kIdwtVertical53Multi, + /// JPEG 2000 irreversible 9/7 vertical lifting multi-target kernel. + J2kIdwtVertical97Multi, + /// JPEG 2000 irreversible 9/7 vertical lifting multi-target 4-column kernel. + J2kIdwtVertical97MultiCols4, + /// JPEG 2000 inverse 5/3 DWT vertical lifting kernel. + J2kIdwtVertical53, + /// JPEG 2000 inverse 9/7 DWT vertical lifting kernel. + J2kIdwtVertical97, + /// JPEG 2000 inverse DWT single-decomposition kernel. + J2kInverseDwtSingle, + /// JPEG 2000 inverse RCT/ICT color transform kernel. + J2kInverseMct, + /// JPEG 2000 grayscale f32-to-Gray8 store kernel. + J2kStoreGray8, + /// JPEG 2000 grayscale f32-to-Gray16 store kernel. + J2kStoreGray16, + /// JPEG 2000 RGB/RGBA 8-bit store kernel. + J2kStoreRgb8, + /// JPEG 2000 fused inverse MCT and RGB/RGBA 8-bit store kernel. + J2kStoreRgb8Mct, + /// JPEG 2000 batched fused inverse MCT and RGB/RGBA 8-bit store kernel. + J2kStoreRgb8MctBatch, + /// JPEG 2000 RGB/RGBA 16-bit store kernel. + J2kStoreRgb16, + /// JPEG 2000 fused inverse MCT and RGB/RGBA 16-bit store kernel. + J2kStoreRgb16Mct, + /// HTJ2K single code-block encode kernel. + Htj2kEncodeCodeblock, + /// HTJ2K batched code-block encode kernel. + Htj2kEncodeCodeblocks, + /// HTJ2K batched multi-input code-block encode kernel. + Htj2kEncodeCodeblocksMultiInput, + /// HTJ2K cleanup-only batched multi-input code-block encode kernel. + Htj2kEncodeCodeblocksMultiInputCleanup, + /// HTJ2K cleanup-only batched multi-input 64x64 code-block encode kernel. + Htj2kEncodeCodeblocksMultiInputCleanup64, + /// HTJ2K batched code-block output compaction kernel. + Htj2kCompactCodeblocks, + /// HTJ2K packet header/body assembly kernel. + Htj2kPacketizeCleanup, +} + +impl CudaKernelName { + pub(crate) fn kernel(self) -> CudaKernel { + match self { + Self::CopyU8 => CudaKernel::CopyU8, + Self::J2kDeinterleaveToF32 => CudaKernel::J2kDeinterleaveToF32, + Self::J2kForwardRct => CudaKernel::J2kForwardRct, + Self::J2kForwardIct => CudaKernel::J2kForwardIct, + Self::J2kForwardDwt53Horizontal => CudaKernel::J2kForwardDwt53Horizontal, + Self::J2kForwardDwt53Vertical => CudaKernel::J2kForwardDwt53Vertical, + Self::J2kForwardDwt97Horizontal => CudaKernel::J2kForwardDwt97Horizontal, + Self::J2kForwardDwt97Vertical => CudaKernel::J2kForwardDwt97Vertical, + Self::J2kQuantizeSubband => CudaKernel::J2kQuantizeSubband, + Self::J2kQuantizeSubbandStrided => CudaKernel::J2kQuantizeSubbandStrided, + Self::Htj2kDecodeCodeblocks => CudaKernel::Htj2kDecodeCodeblocks, + Self::Htj2kDecodeCodeblocksMultiCleanupDequantize => { + CudaKernel::Htj2kDecodeCodeblocksMultiCleanupDequantize + } + Self::J2kDequantizeHtj2kCodeblocks => CudaKernel::J2kDequantizeHtj2kCodeblocks, + Self::J2kDequantizeHtj2kCodeblocksMulti => { + CudaKernel::J2kDequantizeHtj2kCodeblocksMulti + } + Self::J2kDequantizeHtj2kCleanupJobsMulti => { + CudaKernel::J2kDequantizeHtj2kCleanupJobsMulti + } + Self::J2kIdwtInterleave => CudaKernel::J2kIdwtInterleave, + Self::J2kIdwtInterleaveHorizontal53Multi => { + CudaKernel::J2kIdwtInterleaveHorizontal53Multi + } + Self::J2kIdwtInterleaveHorizontal97Multi => { + CudaKernel::J2kIdwtInterleaveHorizontal97Multi + } + Self::J2kIdwtHorizontal => CudaKernel::J2kIdwtHorizontal, + Self::J2kIdwtHorizontal53 => CudaKernel::J2kIdwtHorizontal53, + Self::J2kIdwtHorizontal97 => CudaKernel::J2kIdwtHorizontal97, + Self::J2kIdwtVertical => CudaKernel::J2kIdwtVertical, + Self::J2kIdwtVertical53Multi => CudaKernel::J2kIdwtVertical53Multi, + Self::J2kIdwtVertical97Multi => CudaKernel::J2kIdwtVertical97Multi, + Self::J2kIdwtVertical97MultiCols4 => CudaKernel::J2kIdwtVertical97MultiCols4, + Self::J2kIdwtVertical53 => CudaKernel::J2kIdwtVertical53, + Self::J2kIdwtVertical97 => CudaKernel::J2kIdwtVertical97, + Self::J2kInverseDwtSingle => CudaKernel::J2kInverseDwtSingle, + Self::J2kInverseMct => CudaKernel::J2kInverseMct, + Self::J2kStoreGray8 => CudaKernel::J2kStoreGray8, + Self::J2kStoreGray16 => CudaKernel::J2kStoreGray16, + Self::J2kStoreRgb8 => CudaKernel::J2kStoreRgb8, + Self::J2kStoreRgb8Mct => CudaKernel::J2kStoreRgb8Mct, + Self::J2kStoreRgb8MctBatch => CudaKernel::J2kStoreRgb8MctBatch, + Self::J2kStoreRgb16 => CudaKernel::J2kStoreRgb16, + Self::J2kStoreRgb16Mct => CudaKernel::J2kStoreRgb16Mct, + Self::Htj2kEncodeCodeblock => CudaKernel::Htj2kEncodeCodeblock, + Self::Htj2kEncodeCodeblocks => CudaKernel::Htj2kEncodeCodeblocks, + Self::Htj2kEncodeCodeblocksMultiInput => CudaKernel::Htj2kEncodeCodeblocksMultiInput, + Self::Htj2kEncodeCodeblocksMultiInputCleanup => { + CudaKernel::Htj2kEncodeCodeblocksMultiInputCleanup + } + Self::Htj2kEncodeCodeblocksMultiInputCleanup64 => { + CudaKernel::Htj2kEncodeCodeblocksMultiInputCleanup64 + } + Self::Htj2kCompactCodeblocks => CudaKernel::Htj2kCompactCodeblocks, + Self::Htj2kPacketizeCleanup => CudaKernel::Htj2kPacketizeCleanup, + } + } + + pub(crate) fn entrypoint(self) -> &'static str { + match self { + Self::CopyU8 => "j2k_copy_u8", + Self::J2kDeinterleaveToF32 => "j2k_deinterleave_to_f32", + Self::J2kForwardRct => "j2k_forward_rct", + Self::J2kForwardIct => "j2k_forward_ict", + Self::J2kForwardDwt53Horizontal => "j2k_forward_dwt53_horizontal", + Self::J2kForwardDwt53Vertical => "j2k_forward_dwt53_vertical", + Self::J2kForwardDwt97Horizontal => "j2k_forward_dwt97_horizontal", + Self::J2kForwardDwt97Vertical => "j2k_forward_dwt97_vertical", + Self::J2kQuantizeSubband => "j2k_quantize_subband", + Self::J2kQuantizeSubbandStrided => "j2k_quantize_subband_strided", + Self::Htj2kDecodeCodeblocks => "j2k_htj2k_decode_codeblocks", + Self::Htj2kDecodeCodeblocksMultiCleanupDequantize => { + "j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize" + } + Self::J2kDequantizeHtj2kCodeblocks => "j2k_dequantize_htj2k_codeblocks", + Self::J2kDequantizeHtj2kCodeblocksMulti => "j2k_dequantize_htj2k_codeblocks_multi", + Self::J2kDequantizeHtj2kCleanupJobsMulti => "j2k_dequantize_htj2k_cleanup_jobs_multi", + Self::J2kIdwtInterleave => "j2k_idwt_interleave", + Self::J2kIdwtInterleaveHorizontal53Multi => "j2k_idwt_interleave_horizontal_53_multi", + Self::J2kIdwtInterleaveHorizontal97Multi => "j2k_idwt_interleave_horizontal_97_multi", + Self::J2kIdwtHorizontal => "j2k_idwt_horizontal", + Self::J2kIdwtHorizontal53 => "j2k_idwt_horizontal_53", + Self::J2kIdwtHorizontal97 => "j2k_idwt_horizontal_97", + Self::J2kIdwtVertical => "j2k_idwt_vertical", + Self::J2kIdwtVertical53Multi => "j2k_idwt_vertical_53_multi", + Self::J2kIdwtVertical97Multi => "j2k_idwt_vertical_97_multi", + Self::J2kIdwtVertical97MultiCols4 => "j2k_idwt_vertical_97_multi_cols4", + Self::J2kIdwtVertical53 => "j2k_idwt_vertical_53", + Self::J2kIdwtVertical97 => "j2k_idwt_vertical_97", + Self::J2kInverseDwtSingle => "j2k_inverse_dwt_single", + Self::J2kInverseMct => "j2k_inverse_mct", + Self::J2kStoreGray8 => "j2k_store_gray8", + Self::J2kStoreGray16 => "j2k_store_gray16", + Self::J2kStoreRgb8 => "j2k_store_rgb8", + Self::J2kStoreRgb8Mct => "j2k_store_rgb8_mct", + Self::J2kStoreRgb8MctBatch => "j2k_store_rgb8_mct_batch", + Self::J2kStoreRgb16 => "j2k_store_rgb16", + Self::J2kStoreRgb16Mct => "j2k_store_rgb16_mct", + Self::Htj2kEncodeCodeblock => "j2k_htj2k_encode_codeblock", + Self::Htj2kEncodeCodeblocks => "j2k_htj2k_encode_codeblocks", + Self::Htj2kEncodeCodeblocksMultiInput => "j2k_htj2k_encode_codeblocks_multi_input", + Self::Htj2kEncodeCodeblocksMultiInputCleanup => { + "j2k_htj2k_encode_codeblocks_multi_input_cleanup" + } + Self::Htj2kEncodeCodeblocksMultiInputCleanup64 => { + "j2k_htj2k_encode_codeblocks_multi_input_cleanup_64" + } + Self::Htj2kCompactCodeblocks => "j2k_htj2k_compact_codeblocks", + Self::Htj2kPacketizeCleanup => "j2k_htj2k_packetize_cleanup", + } + } +} + +/// Metadata for a preloaded CUDA kernel module entry point. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaKernelModule { + pub(crate) kernel: CudaKernelName, + pub(crate) entrypoint: &'static str, +} + +impl CudaKernelModule { + /// Bundled kernel identifier. + pub fn kernel(&self) -> CudaKernelName { + self.kernel + } + + /// Kernel entry point name. + pub fn entrypoint(&self) -> &'static str { + self.entrypoint + } +} + +pub(crate) fn cuda_idwt_trace_enabled() -> bool { + std::env::var_os(CUDA_IDWT_TRACE_ENV_VAR).is_some() +} + +impl CudaContext { + pub(crate) fn download_i32_band( + buffer: &CudaDeviceBuffer, + count: usize, + ) -> Result, CudaError> { + let mut out = vec![0i32; count]; + if count != 0 { + buffer.copy_to_host(i32_slice_as_bytes_mut(&mut out))?; + } + Ok(out) + } +} + +impl CudaContext { + pub(crate) fn download_f32_band( + buffer: &CudaDeviceBuffer, + count: usize, + ) -> Result, CudaError> { + let mut out = vec![0f32; count]; + if count != 0 { + buffer.copy_to_host(f32_slice_as_bytes_mut(&mut out))?; + } + Ok(out) + } + + pub(crate) fn download_pooled_f32_band( + buffer: &CudaPooledDeviceBuffer, + count: usize, + ) -> Result, CudaError> { + let mut out = vec![0f32; count]; + if count != 0 { + buffer.copy_to_host(f32_slice_as_bytes_mut(&mut out))?; + } + Ok(out) + } +} + +#[derive(Debug)] +pub(crate) struct CompiledKernel { + pub(crate) module: CuModule, + pub(crate) function: CuFunction, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub(crate) enum CompiledKernelKey { + Builtin(CudaKernel), + #[cfg(feature = "cuda-oxide-copy-u8")] + CudaOxideCopyU8, + #[cfg(feature = "cuda-oxide-j2k-encode")] + CudaOxideJ2kEncode(CudaKernel), + #[cfg(feature = "cuda-oxide-j2k-decode-store")] + CudaOxideJ2kDecodeStore(CudaKernel), + #[cfg(feature = "cuda-oxide-j2k-dequantize")] + CudaOxideJ2kDequantize(CudaKernel), + #[cfg(feature = "cuda-oxide-j2k-idwt")] + CudaOxideJ2kIdwt(CudaKernel), + #[cfg(feature = "cuda-oxide-transcode")] + CudaOxideTranscode(CudaKernel), +} + +impl CompiledKernelKey { + pub(crate) fn kernel(self) -> CudaKernel { + match self { + Self::Builtin(kernel) => kernel, + #[cfg(feature = "cuda-oxide-copy-u8")] + Self::CudaOxideCopyU8 => CudaKernel::CopyU8, + #[cfg(feature = "cuda-oxide-j2k-encode")] + Self::CudaOxideJ2kEncode(kernel) => kernel, + #[cfg(feature = "cuda-oxide-j2k-decode-store")] + Self::CudaOxideJ2kDecodeStore(kernel) => kernel, + #[cfg(feature = "cuda-oxide-j2k-dequantize")] + Self::CudaOxideJ2kDequantize(kernel) => kernel, + #[cfg(feature = "cuda-oxide-j2k-idwt")] + Self::CudaOxideJ2kIdwt(kernel) => kernel, + #[cfg(feature = "cuda-oxide-transcode")] + Self::CudaOxideTranscode(kernel) => kernel, + } + } + + pub(crate) fn ptx(self) -> &'static [u8] { + match self { + Self::Builtin(kernel) => kernel.ptx(), + #[cfg(feature = "cuda-oxide-copy-u8")] + Self::CudaOxideCopyU8 => kernels::cuda_oxide_copy_u8_ptx(), + #[cfg(feature = "cuda-oxide-j2k-encode")] + Self::CudaOxideJ2kEncode(_) => kernels::cuda_oxide_j2k_encode_ptx(), + #[cfg(feature = "cuda-oxide-j2k-decode-store")] + Self::CudaOxideJ2kDecodeStore(_) => kernels::cuda_oxide_j2k_decode_store_ptx(), + #[cfg(feature = "cuda-oxide-j2k-dequantize")] + Self::CudaOxideJ2kDequantize(_) => kernels::cuda_oxide_j2k_dequantize_ptx(), + #[cfg(feature = "cuda-oxide-j2k-idwt")] + Self::CudaOxideJ2kIdwt(_) => kernels::cuda_oxide_j2k_idwt_ptx(), + #[cfg(feature = "cuda-oxide-transcode")] + Self::CudaOxideTranscode(_) => kernels::cuda_oxide_transcode_ptx(), + } + } + + pub(crate) fn entrypoint(self) -> &'static [u8] { + self.kernel().entrypoint() + } +} + +impl CompiledKernel { + pub(crate) fn load(context: &ContextInner, key: CompiledKernelKey) -> Result { + context.set_current()?; + let mut module = std::ptr::null_mut(); + // SAFETY: image is a NUL-terminated PTX string. CUDA copies or parses + // it during module load, and the context cache unloads the module on + // context drop. + context.driver.check("cuModuleLoadData", unsafe { + (context.driver.cu_module_load_data)( + &raw mut module, + key.ptx().as_ptr().cast::(), + ) + })?; + let mut function = std::ptr::null_mut(); + // SAFETY: name is a NUL-terminated kernel symbol in this module. + context.driver.check("cuModuleGetFunction", unsafe { + (context.driver.cu_module_get_function)( + &raw mut function, + module, + key.entrypoint().as_ptr().cast::(), + ) + })?; + Ok(Self { module, function }) + } +} + +// SAFETY: CompiledKernel stores opaque CUDA module/function handles. Lifetime +// and unloading are coordinated by ContextInner's module cache mutex. +unsafe impl Send for CompiledKernel {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/Cargo.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/Cargo.toml new file mode 100644 index 00000000..1e7e390e --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "j2k_cuda_oxide_copy_u8_host" +version = "0.1.0" +edition = "2024" + +[package.metadata.cuda-oxide.interop] +kind = "j2k-copy-u8" + +[[package.metadata.cuda-oxide.device-crates]] +manifest-path = "simt/Cargo.toml" +ptx-dir = "ptx" +artifact-name = "j2k_cuda_oxide_copy_u8" + +[workspace] +resolver = "2" diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/rust-toolchain.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/rust-toolchain.toml new file mode 100644 index 00000000..9f964011 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly-2026-04-03" +components = ["rust-src", "rustc-dev", "rust-analyzer", "clippy", "llvm-tools"] diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/simt/Cargo.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/simt/Cargo.toml new file mode 100644 index 00000000..bed64c30 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/simt/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "j2k_cuda_oxide_copy_u8" +version = "0.1.0" +edition = "2024" + +[workspace] +resolver = "2" + +[dependencies] +cuda-core = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } +cuda-device = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } +cuda-host = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/simt/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/simt/src/main.rs new file mode 100644 index 00000000..11009d82 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/simt/src/main.rs @@ -0,0 +1,19 @@ +use cuda_device::{kernel, thread}; +use cuda_host::cuda_module; + +#[cuda_module] +mod kernels { + use super::*; + + #[kernel] + pub unsafe fn j2k_copy_u8(dst: *mut u8, src: *const u8, len: u64) { + let index = thread::index_1d().get(); + if index < len as usize { + unsafe { + *dst.add(index) = *src.add(index); + } + } + } +} + +fn main() {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/src/main.rs new file mode 100644 index 00000000..f328e4d9 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/src/main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/Cargo.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/Cargo.toml new file mode 100644 index 00000000..d62c8d3a --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "j2k_cuda_oxide_j2k_decode_store_host" +version = "0.1.0" +edition = "2024" + +[package.metadata.cuda-oxide.interop] +kind = "j2k-decode-store" + +[[package.metadata.cuda-oxide.device-crates]] +manifest-path = "simt/Cargo.toml" +ptx-dir = "ptx" +artifact-name = "j2k_cuda_oxide_j2k_decode_store" + +[workspace] +resolver = "2" diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/rust-toolchain.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/rust-toolchain.toml new file mode 100644 index 00000000..9f964011 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly-2026-04-03" +components = ["rust-src", "rustc-dev", "rust-analyzer", "clippy", "llvm-tools"] diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/Cargo.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/Cargo.toml new file mode 100644 index 00000000..198eb818 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "j2k_cuda_oxide_j2k_decode_store" +version = "0.1.0" +edition = "2024" + +[workspace] +resolver = "2" + +[dependencies] +cuda-core = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } +cuda-device = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } +cuda-host = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/main.rs new file mode 100644 index 00000000..7b138dfe --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/main.rs @@ -0,0 +1,571 @@ +use cuda_device::{kernel, thread}; +use cuda_host::cuda_module; + +#[repr(C)] +#[derive(Clone, Copy)] +struct CudaJ2kStoreGray8Job { + input_width: u32, + source_x: u32, + source_y: u32, + copy_width: u32, + copy_height: u32, + output_width: u32, + output_height: u32, + output_x: u32, + output_y: u32, + addend: f32, + bit_depth: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct CudaJ2kStoreGray16Job { + input_width: u32, + source_x: u32, + source_y: u32, + copy_width: u32, + copy_height: u32, + output_width: u32, + output_height: u32, + output_x: u32, + output_y: u32, + addend: f32, + bit_depth: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct CudaJ2kInverseMctJob { + len: u32, + irreversible97: u32, + addend0: f32, + addend1: f32, + addend2: f32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct CudaJ2kStoreRgb8Job { + input_width0: u32, + input_width1: u32, + input_width2: u32, + source_x0: u32, + source_y0: u32, + source_x1: u32, + source_y1: u32, + source_x2: u32, + source_y2: u32, + copy_width: u32, + copy_height: u32, + output_width: u32, + output_height: u32, + output_x: u32, + output_y: u32, + addend0: f32, + addend1: f32, + addend2: f32, + bit_depth0: u32, + bit_depth1: u32, + bit_depth2: u32, + rgba: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct CudaJ2kStoreRgb16Job { + input_width0: u32, + input_width1: u32, + input_width2: u32, + source_x0: u32, + source_y0: u32, + source_x1: u32, + source_y1: u32, + source_x2: u32, + source_y2: u32, + copy_width: u32, + copy_height: u32, + output_width: u32, + output_height: u32, + output_x: u32, + output_y: u32, + addend0: f32, + addend1: f32, + addend2: f32, + bit_depth0: u32, + bit_depth1: u32, + bit_depth2: u32, + rgba: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct CudaJ2kStoreRgb8MctJob { + store: CudaJ2kStoreRgb8Job, + irreversible97: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct CudaJ2kStoreRgb8MctBatchJob { + plane0_ptr: u64, + plane1_ptr: u64, + plane2_ptr: u64, + output_ptr: u64, + job: CudaJ2kStoreRgb8MctJob, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct CudaJ2kStoreRgb16MctJob { + store: CudaJ2kStoreRgb16Job, + irreversible97: u32, +} + +#[inline(always)] +fn load_f32(ptr: *const f32, index: u32) -> f32 { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn load_job(ptr: *const T) -> T { + unsafe { *ptr } +} + +#[inline(always)] +fn store_f32(ptr: *mut f32, index: u32, value: f32) { + unsafe { + *ptr.add(index as usize) = value; + } +} + +#[inline(always)] +fn store_u8(ptr: *mut u8, index: u32, value: u8) { + unsafe { + *ptr.add(index as usize) = value; + } +} + +#[inline(always)] +fn store_u16(ptr: *mut u16, index: u32, value: u16) { + unsafe { + *ptr.add(index as usize) = value; + } +} + +#[inline(always)] +fn floor_f32(value: f32) -> f32 { + // f32::floor routes through libdevice in cuda-oxide, which emits NVVM IR + // instead of the PTX loaded by this runtime path. + let truncated = value as i32 as f32; + if truncated > value { + truncated - 1.0 + } else { + truncated + } +} + +#[inline(always)] +fn round_f32(value: f32) -> f32 { + if value >= 0.0 { + floor_f32(value + 0.5) + } else { + -floor_f32(-value + 0.5) + } +} + +#[inline(always)] +fn clamp_f32(value: f32, min: f32, max: f32) -> f32 { + if value < min { + min + } else if value > max { + max + } else { + value + } +} + +#[inline(always)] +fn max_int_for_bit_depth(bit_depth: u32) -> u32 { + if bit_depth == 0 { + 1 + } else { + (1_u32 << bit_depth) - 1 + } +} + +#[inline(always)] +fn sample_as_u8(sample: f32, bit_depth: u32) -> u8 { + let rounded = round_f32(sample); + if bit_depth >= 8 { + return clamp_f32(rounded, 0.0, 255.0) as u8; + } + + let max_int = (1_u32 << bit_depth) - 1; + let max_value = if max_int > 1 { max_int as f32 } else { 1.0 }; + round_f32((clamp_f32(rounded, 0.0, max_value) / max_value) * 255.0) as u8 +} + +#[inline(always)] +fn sample_as_u16(sample: f32, bit_depth: u32) -> u16 { + let rounded = round_f32(sample); + if bit_depth >= 16 { + return clamp_f32(rounded, 0.0, 65535.0) as u16; + } + + let max_int = max_int_for_bit_depth(bit_depth); + let max_value = if max_int > 1 { max_int as f32 } else { 1.0 }; + round_f32((clamp_f32(rounded, 0.0, max_value) / max_value) * 65535.0) as u16 +} + +#[inline(always)] +fn inverse_mct_sample(src0: f32, src1: f32, src2: f32, irreversible97: u32) -> (f32, f32, f32) { + if irreversible97 != 0 { + ( + src0 + 1.402 * src2, + src0 - 0.34413 * src1 - 0.71414 * src2, + src0 + 1.772 * src1, + ) + } else { + let green = src0 - floor_f32((src2 + src1) * 0.25); + (src2 + green, green, src1 + green) + } +} + +#[inline(always)] +fn source_index(input_width: u32, source_x: u32, source_y: u32, row: u32, col: u32) -> u32 { + (source_y + row) * input_width + source_x + col +} + +#[inline(always)] +fn output_pixel_index(output_width: u32, output_x: u32, output_y: u32, row: u32, col: u32) -> u32 { + (output_y + row) * output_width + output_x + col +} + +#[inline(always)] +fn pixel_coords(gid: u32, copy_width: u32) -> (u32, u32) { + let row = gid / copy_width; + (row, gid - row * copy_width) +} + +#[inline(always)] +fn store_rgb8_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u8, + job: CudaJ2kStoreRgb8Job, + gid: u32, +) { + let (row, col) = pixel_coords(gid, job.copy_width); + let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); + let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); + let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); + let channels = if job.rgba != 0 { 4 } else { 3 }; + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col) * channels; + + store_u8( + output, + dst, + sample_as_u8(load_f32(plane0, src0) + job.addend0, job.bit_depth0), + ); + store_u8( + output, + dst + 1, + sample_as_u8(load_f32(plane1, src1) + job.addend1, job.bit_depth1), + ); + store_u8( + output, + dst + 2, + sample_as_u8(load_f32(plane2, src2) + job.addend2, job.bit_depth2), + ); + if job.rgba != 0 { + store_u8(output, dst + 3, 255); + } +} + +#[inline(always)] +fn store_rgb16_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u16, + job: CudaJ2kStoreRgb16Job, + gid: u32, +) { + let (row, col) = pixel_coords(gid, job.copy_width); + let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); + let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); + let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); + let channels = if job.rgba != 0 { 4 } else { 3 }; + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col) * channels; + + store_u16( + output, + dst, + sample_as_u16(load_f32(plane0, src0) + job.addend0, job.bit_depth0), + ); + store_u16( + output, + dst + 1, + sample_as_u16(load_f32(plane1, src1) + job.addend1, job.bit_depth1), + ); + store_u16( + output, + dst + 2, + sample_as_u16(load_f32(plane2, src2) + job.addend2, job.bit_depth2), + ); + if job.rgba != 0 { + store_u16(output, dst + 3, 65535); + } +} + +#[inline(always)] +fn store_rgb8_mct_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u8, + mct_job: CudaJ2kStoreRgb8MctJob, + gid: u32, +) { + let job = mct_job.store; + let (row, col) = pixel_coords(gid, job.copy_width); + let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); + let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); + let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); + let channels = if job.rgba != 0 { 4 } else { 3 }; + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col) * channels; + let (out0, out1, out2) = inverse_mct_sample( + load_f32(plane0, src0), + load_f32(plane1, src1), + load_f32(plane2, src2), + mct_job.irreversible97, + ); + + store_u8( + output, + dst, + sample_as_u8(out0 + job.addend0, job.bit_depth0), + ); + store_u8( + output, + dst + 1, + sample_as_u8(out1 + job.addend1, job.bit_depth1), + ); + store_u8( + output, + dst + 2, + sample_as_u8(out2 + job.addend2, job.bit_depth2), + ); + if job.rgba != 0 { + store_u8(output, dst + 3, 255); + } +} + +#[inline(always)] +fn store_rgb16_mct_sample( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u16, + mct_job: CudaJ2kStoreRgb16MctJob, + gid: u32, +) { + let job = mct_job.store; + let (row, col) = pixel_coords(gid, job.copy_width); + let src0 = source_index(job.input_width0, job.source_x0, job.source_y0, row, col); + let src1 = source_index(job.input_width1, job.source_x1, job.source_y1, row, col); + let src2 = source_index(job.input_width2, job.source_x2, job.source_y2, row, col); + let channels = if job.rgba != 0 { 4 } else { 3 }; + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col) * channels; + let (out0, out1, out2) = inverse_mct_sample( + load_f32(plane0, src0), + load_f32(plane1, src1), + load_f32(plane2, src2), + mct_job.irreversible97, + ); + + store_u16( + output, + dst, + sample_as_u16(out0 + job.addend0, job.bit_depth0), + ); + store_u16( + output, + dst + 1, + sample_as_u16(out1 + job.addend1, job.bit_depth1), + ); + store_u16( + output, + dst + 2, + sample_as_u16(out2 + job.addend2, job.bit_depth2), + ); + if job.rgba != 0 { + store_u16(output, dst + 3, 65535); + } +} + +#[cuda_module] +mod kernels { + use super::*; + + #[kernel] + pub unsafe fn j2k_store_gray8( + input: *const f32, + output: *mut u8, + job_buffer: *const CudaJ2kStoreGray8Job, + ) { + let job = load_job(job_buffer); + let pixels = job.copy_width * job.copy_height; + let gid = thread::index_1d().get() as u32; + if gid >= pixels { + return; + } + + let (row, col) = pixel_coords(gid, job.copy_width); + let src = source_index(job.input_width, job.source_x, job.source_y, row, col); + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + store_u8( + output, + dst, + sample_as_u8(load_f32(input, src) + job.addend, job.bit_depth), + ); + } + + #[kernel] + pub unsafe fn j2k_store_gray16( + input: *const f32, + output: *mut u16, + job_buffer: *const CudaJ2kStoreGray16Job, + ) { + let job = load_job(job_buffer); + let pixels = job.copy_width * job.copy_height; + let gid = thread::index_1d().get() as u32; + if gid >= pixels { + return; + } + + let (row, col) = pixel_coords(gid, job.copy_width); + let src = source_index(job.input_width, job.source_x, job.source_y, row, col); + let dst = output_pixel_index(job.output_width, job.output_x, job.output_y, row, col); + store_u16( + output, + dst, + sample_as_u16(load_f32(input, src) + job.addend, job.bit_depth), + ); + } + + #[kernel] + pub unsafe fn j2k_inverse_mct( + plane0: *mut f32, + plane1: *mut f32, + plane2: *mut f32, + job_buffer: *const CudaJ2kInverseMctJob, + ) { + let job = load_job(job_buffer); + let gid = thread::index_1d().get() as u32; + if gid >= job.len { + return; + } + + let (out0, out1, out2) = inverse_mct_sample( + load_f32(plane0.cast_const(), gid), + load_f32(plane1.cast_const(), gid), + load_f32(plane2.cast_const(), gid), + job.irreversible97, + ); + store_f32(plane0, gid, out0 + job.addend0); + store_f32(plane1, gid, out1 + job.addend1); + store_f32(plane2, gid, out2 + job.addend2); + } + + #[kernel] + pub unsafe fn j2k_store_rgb8( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u8, + job_buffer: *const CudaJ2kStoreRgb8Job, + ) { + let job = load_job(job_buffer); + let pixels = job.copy_width * job.copy_height; + let gid = thread::index_1d().get() as u32; + if gid >= pixels { + return; + } + + store_rgb8_sample(plane0, plane1, plane2, output, job, gid); + } + + #[kernel] + pub unsafe fn j2k_store_rgb16( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u16, + job_buffer: *const CudaJ2kStoreRgb16Job, + ) { + let job = load_job(job_buffer); + let pixels = job.copy_width * job.copy_height; + let gid = thread::index_1d().get() as u32; + if gid >= pixels { + return; + } + + store_rgb16_sample(plane0, plane1, plane2, output, job, gid); + } + + #[kernel] + pub unsafe fn j2k_store_rgb8_mct( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u8, + job_buffer: *const CudaJ2kStoreRgb8MctJob, + ) { + let mct_job = load_job(job_buffer); + let pixels = mct_job.store.copy_width * mct_job.store.copy_height; + let gid = thread::index_1d().get() as u32; + if gid >= pixels { + return; + } + + store_rgb8_mct_sample(plane0, plane1, plane2, output, mct_job, gid); + } + + #[kernel] + pub unsafe fn j2k_store_rgb8_mct_batch(jobs: *const CudaJ2kStoreRgb8MctBatchJob) { + let item = load_job(unsafe { jobs.add(thread::index_2d_row() as usize) }); + let plane0 = item.plane0_ptr as usize as *const f32; + let plane1 = item.plane1_ptr as usize as *const f32; + let plane2 = item.plane2_ptr as usize as *const f32; + let output = item.output_ptr as usize as *mut u8; + let pixels = item.job.store.copy_width * item.job.store.copy_height; + let gid = thread::index_2d_col() as u32; + if gid >= pixels { + return; + } + + store_rgb8_mct_sample(plane0, plane1, plane2, output, item.job, gid); + } + + #[kernel] + pub unsafe fn j2k_store_rgb16_mct( + plane0: *const f32, + plane1: *const f32, + plane2: *const f32, + output: *mut u16, + job_buffer: *const CudaJ2kStoreRgb16MctJob, + ) { + let mct_job = load_job(job_buffer); + let pixels = mct_job.store.copy_width * mct_job.store.copy_height; + let gid = thread::index_1d().get() as u32; + if gid >= pixels { + return; + } + + store_rgb16_mct_sample(plane0, plane1, plane2, output, mct_job, gid); + } +} + +fn main() {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/src/main.rs new file mode 100644 index 00000000..f328e4d9 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/src/main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/Cargo.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/Cargo.toml new file mode 100644 index 00000000..134df247 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "j2k_cuda_oxide_j2k_dequantize_host" +version = "0.1.0" +edition = "2024" + +[package.metadata.cuda-oxide.interop] +kind = "j2k-dequantize" + +[[package.metadata.cuda-oxide.device-crates]] +manifest-path = "simt/Cargo.toml" +ptx-dir = "ptx" +artifact-name = "j2k_cuda_oxide_j2k_dequantize" + +[workspace] +resolver = "2" diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/rust-toolchain.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/rust-toolchain.toml new file mode 100644 index 00000000..9f964011 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly-2026-04-03" +components = ["rust-src", "rustc-dev", "rust-analyzer", "clippy", "llvm-tools"] diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/simt/Cargo.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/simt/Cargo.toml new file mode 100644 index 00000000..b517b90b --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/simt/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "j2k_cuda_oxide_j2k_dequantize" +version = "0.1.0" +edition = "2024" + +[workspace] +resolver = "2" + +[dependencies] +cuda-core = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } +cuda-device = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } +cuda-host = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/simt/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/simt/src/main.rs new file mode 100644 index 00000000..cd076fe1 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/simt/src/main.rs @@ -0,0 +1,167 @@ +use cuda_device::{kernel, thread}; +use cuda_host::cuda_module; + +#[repr(C)] +#[derive(Clone, Copy)] +struct J2kHtCleanupBatchJob { + coded_offset: u32, + width: u32, + height: u32, + coded_len: u32, + cleanup_length: u32, + refinement_length: u32, + missing_msbs: u32, + num_bitplanes: u32, + number_of_coding_passes: u32, + output_stride: u32, + output_offset: u32, + dequantization_step: f32, + stripe_causal: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct J2kHtDequantizeJob { + output_ptr: u64, + width: u32, + height: u32, + output_stride: u32, + output_offset: u32, + num_bitplanes: u32, + reserved: u32, + dequantization_step: f32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct J2kHtCleanupMultiBatchJob { + output_ptr: u64, + coded_offset: u32, + width: u32, + height: u32, + coded_len: u32, + cleanup_length: u32, + refinement_length: u32, + missing_msbs: u32, + num_bitplanes: u32, + number_of_coding_passes: u32, + output_stride: u32, + output_offset: u32, + dequantization_step: f32, + stripe_causal: u32, +} + +#[inline(always)] +fn load_u32(ptr: *const u32, index: u32) -> u32 { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn store_u32(ptr: *mut u32, index: u32, value: u32) { + unsafe { + *ptr.add(index as usize) = value; + } +} + +#[inline(always)] +fn load_job(ptr: *const T, index: u32) -> T { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn coefficient_to_i32(value: u32, k_max: u32) -> i32 { + let shift = 31 - k_max; + let magnitude = ((value & 0x7fff_ffff) >> shift) as i32; + if (value & 0x8000_0000) != 0 { + -magnitude + } else { + magnitude + } +} + +#[inline(always)] +fn coefficient_to_float_bits(value: u32, k_max: u32, scale: f32) -> u32 { + ((coefficient_to_i32(value, k_max) as f32) * scale).to_bits() +} + +#[inline(always)] +fn dequantize_codeblock( + decoded_data: *mut u32, + width: u32, + height: u32, + output_stride: u32, + output_offset: u32, + num_bitplanes: u32, + dequantization_step: f32, +) { + let sample_count = width * height; + let mut sample = thread::threadIdx_x(); + let step = thread::blockDim_x(); + while sample < sample_count { + let y = sample / width; + let x = sample - y * width; + let idx = output_offset + y * output_stride + x; + store_u32( + decoded_data, + idx, + coefficient_to_float_bits( + load_u32(decoded_data.cast_const(), idx), + num_bitplanes, + dequantization_step, + ), + ); + sample += step; + } +} + +#[cuda_module] +mod kernels { + use super::*; + + #[kernel] + pub unsafe fn j2k_dequantize_htj2k_codeblocks( + decoded_data: *mut u32, + jobs: *const J2kHtCleanupBatchJob, + ) { + let job = load_job(jobs, thread::blockIdx_x()); + dequantize_codeblock( + decoded_data, + job.width, + job.height, + job.output_stride, + job.output_offset, + job.num_bitplanes, + job.dequantization_step, + ); + } + + #[kernel] + pub unsafe fn j2k_dequantize_htj2k_codeblocks_multi(jobs: *const J2kHtDequantizeJob) { + let job = load_job(jobs, thread::blockIdx_x()); + dequantize_codeblock( + job.output_ptr as usize as *mut u32, + job.width, + job.height, + job.output_stride, + job.output_offset, + job.num_bitplanes, + job.dequantization_step, + ); + } + + #[kernel] + pub unsafe fn j2k_dequantize_htj2k_cleanup_jobs_multi(jobs: *const J2kHtCleanupMultiBatchJob) { + let job = load_job(jobs, thread::blockIdx_x()); + dequantize_codeblock( + job.output_ptr as usize as *mut u32, + job.width, + job.height, + job.output_stride, + job.output_offset, + job.num_bitplanes, + job.dequantization_step, + ); + } +} + +fn main() {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/src/main.rs new file mode 100644 index 00000000..f328e4d9 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/src/main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/Cargo.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/Cargo.toml new file mode 100644 index 00000000..e363f517 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "j2k_cuda_oxide_j2k_encode_host" +version = "0.1.0" +edition = "2024" + +[package.metadata.cuda-oxide.interop] +kind = "j2k-encode" + +[[package.metadata.cuda-oxide.device-crates]] +manifest-path = "simt/Cargo.toml" +ptx-dir = "ptx" +artifact-name = "j2k_cuda_oxide_j2k_encode" + +[workspace] +resolver = "2" diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/rust-toolchain.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/rust-toolchain.toml new file mode 100644 index 00000000..9f964011 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly-2026-04-03" +components = ["rust-src", "rustc-dev", "rust-analyzer", "clippy", "llvm-tools"] diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/Cargo.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/Cargo.toml new file mode 100644 index 00000000..7bf1d9cc --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "j2k_cuda_oxide_j2k_encode" +version = "0.1.0" +edition = "2024" + +[workspace] +resolver = "2" + +[dependencies] +cuda-core = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } +cuda-device = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } +cuda-host = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/main.rs new file mode 100644 index 00000000..1ce11e2c --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/main.rs @@ -0,0 +1,1500 @@ +#![allow( + clippy::manual_div_ceil, + clippy::manual_is_multiple_of, + clippy::too_many_arguments, + static_mut_refs +)] + +use cuda_device::{SharedArray, kernel, thread}; +use cuda_host::cuda_module; + +const J2K_FDWT97_ALPHA: f32 = -1.5861343; +const J2K_FDWT97_BETA: f32 = -0.052980117; +const J2K_FDWT97_GAMMA: f32 = 0.8829111; +const J2K_FDWT97_DELTA: f32 = 0.44350687; +const J2K_FDWT97_KAPPA: f32 = 1.2301741; +const J2K_FDWT97_INV_KAPPA: f32 = 1.0 / J2K_FDWT97_KAPPA; +const J2K_HT_MEL_SIZE: u32 = 192; +const J2K_HT_VLC_SIZE: u32 = 3072 - J2K_HT_MEL_SIZE; +const J2K_HT_MS_SIZE: u32 = ((16384 * 16) + 14) / 15; +const J2K_HT_MEL_OFFSET: u32 = J2K_HT_MS_SIZE; +const J2K_HT_VLC_OFFSET: u32 = J2K_HT_MS_SIZE + J2K_HT_MEL_SIZE; +const J2K_HT_COMPACT_ASSEMBLE_FLAG: u32 = 0x8000_0000; +const J2K_HT_COMPACT_LENGTH_MASK: u32 = 0x7fff; +const J2K_ENCODE_STATUS_OK: u32 = 0; +const J2K_ENCODE_STATUS_FAIL: u32 = 1; +const J2K_ENCODE_STATUS_UNSUPPORTED: u32 = 2; +const J2K_PACKET_TAG_INF: u32 = 0x7fff_ffff; +const J2K_PACKET_MAX_TAG_NODES: usize = 2048; +const J2K_PACKET_MAX_TAG_LEVELS: usize = 16; + +#[repr(C)] +#[derive(Clone, Copy)] +struct J2kHtEncodeCompactJob { + source_offset: u32, + compact_offset: u32, + data_len: u32, + reserved: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct J2kHtPacketJob { + block_start: u32, + block_count: u32, + subband_start: u32, + subband_count: u32, + output_offset: u32, + output_capacity: u32, + layer: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct J2kHtPacketSubband { + block_start: u32, + block_count: u32, + num_cbs_x: u32, + num_cbs_y: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct J2kHtPacketBlock { + data_offset: u32, + data_len: u32, + cleanup_length: u32, + refinement_length: u32, + num_coding_passes: u32, + num_zero_bitplanes: u32, + l_block: u32, + previously_included: u32, + inclusion_layer: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct J2kHtPacketSubbandTagState { + inclusion_node_start: u32, + zero_bitplane_node_start: u32, + node_count: u32, + reserved0: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct J2kHtPacketTagNodeState { + current: u32, + known: u32, +} + +#[repr(C)] +struct J2kHtPacketStatus { + code: u32, + detail: u32, + output_len: u32, + reserved0: u32, +} + +struct J2kPacketBitWriter { + out: *mut u8, + pos: u32, + capacity: u32, + buffer: u32, + bits_in_buffer: u32, + last_byte_was_ff: u32, + failed: u32, +} + +struct J2kPacketTagTree { + values: [u32; J2K_PACKET_MAX_TAG_NODES], + current: [u32; J2K_PACKET_MAX_TAG_NODES], + known: [u32; J2K_PACKET_MAX_TAG_NODES], + widths: [u32; J2K_PACKET_MAX_TAG_LEVELS], + heights: [u32; J2K_PACKET_MAX_TAG_LEVELS], + offsets: [u32; J2K_PACKET_MAX_TAG_LEVELS], + levels: u32, + total_nodes: u32, + failed: u32, +} + +struct J2kPacketHeaderResult { + code: u32, + detail: u32, + header_len: u32, + body_len: u32, + output_len: u32, +} + +#[inline(always)] +fn load_u8(ptr: *const u8, index: u64) -> u8 { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn load_u32(ptr: *const u32, index: u64) -> u32 { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn load_f32(ptr: *const f32, index: u32) -> f32 { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn load_f32_u64(ptr: *const f32, index: u64) -> f32 { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn store_f32(ptr: *mut f32, index: u32, value: f32) { + unsafe { + *ptr.add(index as usize) = value; + } +} + +#[inline(always)] +fn store_f32_u64(ptr: *mut f32, index: u64, value: f32) { + unsafe { + *ptr.add(index as usize) = value; + } +} + +#[inline(always)] +fn store_i32(ptr: *mut i32, index: u64, value: i32) { + unsafe { + *ptr.add(index as usize) = value; + } +} + +#[inline(always)] +fn store_u8(ptr: *mut u8, index: u64, value: u8) { + unsafe { + *ptr.add(index as usize) = value; + } +} + +#[inline(always)] +fn store_u32(ptr: *mut u32, index: u64, value: u32) { + unsafe { + *ptr.add(index as usize) = value; + } +} + +#[inline(always)] +fn load_job(ptr: *const T, index: u32) -> T { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn j2k_packet_status(status: *mut J2kHtPacketStatus, code: u32, detail: u32, output_len: u32) { + unsafe { + (*status).code = code; + (*status).detail = detail; + (*status).output_len = output_len; + (*status).reserved0 = 0; + } +} + +#[inline(always)] +fn j2k_packet_writer_init(out: *mut u8, capacity: u32) -> J2kPacketBitWriter { + J2kPacketBitWriter { + out, + pos: 0, + capacity, + buffer: 0, + bits_in_buffer: 0, + last_byte_was_ff: 0, + failed: 0, + } +} + +#[inline(always)] +fn j2k_packet_push_byte(writer: &mut J2kPacketBitWriter, byte: u8) { + if writer.pos >= writer.capacity { + writer.failed = 1; + return; + } + store_u8(writer.out, writer.pos as u64, byte); + writer.pos += 1; + writer.last_byte_was_ff = if byte == 0xff { 1 } else { 0 }; +} + +#[inline(always)] +fn j2k_packet_flush_byte(writer: &mut J2kPacketBitWriter) { + let limit = if writer.last_byte_was_ff != 0 { 7 } else { 8 }; + let byte = ((writer.buffer >> (writer.bits_in_buffer - limit)) & 0xff) as u8; + j2k_packet_push_byte(writer, byte); + writer.bits_in_buffer -= limit; + writer.buffer = if writer.bits_in_buffer == 0 { + 0 + } else { + writer.buffer & ((1_u32 << writer.bits_in_buffer) - 1) + }; +} + +#[inline(always)] +fn j2k_packet_write_bit(writer: &mut J2kPacketBitWriter, bit: u32) { + writer.buffer = (writer.buffer << 1) | (bit & 1); + writer.bits_in_buffer += 1; + let limit = if writer.last_byte_was_ff != 0 { 7 } else { 8 }; + if writer.bits_in_buffer >= limit { + j2k_packet_flush_byte(writer); + } +} + +#[inline(always)] +fn j2k_packet_write_bits(writer: &mut J2kPacketBitWriter, value: u32, mut count: u32) { + while count > 0 { + count -= 1; + j2k_packet_write_bit(writer, (value >> count) & 1); + } +} + +#[inline(always)] +fn j2k_packet_finish(writer: &mut J2kPacketBitWriter) { + if writer.bits_in_buffer == 0 { + return; + } + let limit = if writer.last_byte_was_ff != 0 { 7 } else { 8 }; + let shift = limit - writer.bits_in_buffer; + let byte = ((writer.buffer << shift) & 0xff) as u8; + j2k_packet_push_byte(writer, byte); + writer.buffer = 0; + writer.bits_in_buffer = 0; +} + +#[inline(always)] +fn j2k_packet_encode_num_ht_passes(writer: &mut J2kPacketBitWriter, num_passes: u32) { + if num_passes == 1 { + j2k_packet_write_bit(writer, 0); + } else if num_passes == 2 { + j2k_packet_write_bits(writer, 0b10, 2); + } else if num_passes <= 5 { + j2k_packet_write_bits(writer, 0b11, 2); + j2k_packet_write_bits(writer, num_passes - 3, 2); + } else if num_passes <= 36 { + j2k_packet_write_bits(writer, 0b11, 2); + j2k_packet_write_bits(writer, 0b11, 2); + j2k_packet_write_bits(writer, num_passes - 6, 5); + } else { + j2k_packet_write_bits(writer, 0b11, 2); + j2k_packet_write_bits(writer, 0b11, 2); + j2k_packet_write_bits(writer, 31, 5); + j2k_packet_write_bits(writer, num_passes - 37, 7); + } +} + +#[inline(always)] +fn j2k_packet_value_fits(value: u32, bits: u32) -> bool { + bits >= 32 || value < (1_u32 << bits) +} + +#[inline(always)] +fn j2k_packet_ht_length_bits(l_block: u32, num_passes: u32) -> u32 { + let placeholder_groups = (if num_passes > 0 { num_passes - 1 } else { 0 }) / 3; + let placeholder_passes = placeholder_groups * 3; + let mut value = placeholder_passes + 1; + let mut log2_value = 0; + while value > 1 { + value >>= 1; + log2_value += 1; + } + l_block + log2_value +} + +#[inline(always)] +fn j2k_packet_encode_ht_segment_lengths(writer: &mut J2kPacketBitWriter, block: J2kHtPacketBlock) { + let cleanup_length = if block.num_coding_passes == 1 && block.cleanup_length == 0 { + block.data_len + } else { + block.cleanup_length + }; + let mut l_block = block.l_block; + let mut cleanup_bits = j2k_packet_ht_length_bits(l_block, block.num_coding_passes); + let refinement_extra_bits = if block.num_coding_passes > 2 { 1 } else { 0 }; + while !j2k_packet_value_fits(cleanup_length, cleanup_bits) + || (block.num_coding_passes > 1 + && !j2k_packet_value_fits(block.refinement_length, l_block + refinement_extra_bits)) + { + j2k_packet_write_bit(writer, 1); + l_block += 1; + cleanup_bits += 1; + } + j2k_packet_write_bit(writer, 0); + j2k_packet_write_bits(writer, cleanup_length, cleanup_bits); + + if block.num_coding_passes > 1 { + j2k_packet_write_bits( + writer, + block.refinement_length, + l_block + refinement_extra_bits, + ); + } +} + +#[inline(always)] +fn j2k_packet_tag_tree_new() -> J2kPacketTagTree { + J2kPacketTagTree { + values: [0; J2K_PACKET_MAX_TAG_NODES], + current: [0; J2K_PACKET_MAX_TAG_NODES], + known: [0; J2K_PACKET_MAX_TAG_NODES], + widths: [0; J2K_PACKET_MAX_TAG_LEVELS], + heights: [0; J2K_PACKET_MAX_TAG_LEVELS], + offsets: [0; J2K_PACKET_MAX_TAG_LEVELS], + levels: 0, + total_nodes: 0, + failed: 0, + } +} + +#[inline(always)] +fn j2k_packet_tag_tree_fail(tree: &mut J2kPacketTagTree) -> bool { + tree.failed = 1; + false +} + +#[inline(always)] +fn j2k_packet_tag_tree_init(tree: &mut J2kPacketTagTree, width: u32, height: u32) -> bool { + if width == 0 || height == 0 { + return j2k_packet_tag_tree_fail(tree); + } + + let mut w = width; + let mut h = height; + let mut total = 0_u32; + let mut levels = 0_u32; + loop { + if levels as usize >= J2K_PACKET_MAX_TAG_LEVELS { + return j2k_packet_tag_tree_fail(tree); + } + let Some(nodes) = w.checked_mul(h) else { + return j2k_packet_tag_tree_fail(tree); + }; + let Some(next_total) = total.checked_add(nodes) else { + return j2k_packet_tag_tree_fail(tree); + }; + if next_total as usize > J2K_PACKET_MAX_TAG_NODES { + return j2k_packet_tag_tree_fail(tree); + } + + let level = levels as usize; + tree.widths[level] = w; + tree.heights[level] = h; + tree.offsets[level] = total; + total = next_total; + levels += 1; + if w <= 1 && h <= 1 { + break; + } + w = (w + 1) >> 1; + h = (h + 1) >> 1; + } + + tree.levels = levels; + tree.total_nodes = total; + tree.failed = 0; + let mut idx = 0_u32; + while idx < total { + let slot = idx as usize; + tree.values[slot] = 0; + tree.current[slot] = 0; + tree.known[slot] = 0; + idx += 1; + } + true +} + +#[inline(always)] +fn j2k_packet_tag_tree_propagate(tree: &mut J2kPacketTagTree) { + let mut level = 1_u32; + while level < tree.levels { + let prev_w = tree.widths[(level - 1) as usize]; + let prev_h = tree.heights[(level - 1) as usize]; + let curr_w = tree.widths[level as usize]; + let curr_h = tree.heights[level as usize]; + let mut cy = 0_u32; + while cy < curr_h { + let mut cx = 0_u32; + while cx < curr_w { + let mut min_value = u32::MAX; + let child_x0 = cx << 1; + let child_y0 = cy << 1; + let child_x1 = (child_x0 + 2).min(prev_w); + let child_y1 = (child_y0 + 2).min(prev_h); + let mut y = child_y0; + while y < child_y1 { + let mut x = child_x0; + while x < child_x1 { + let child_idx = tree.offsets[(level - 1) as usize] + y * prev_w + x; + min_value = min_value.min(tree.values[child_idx as usize]); + x += 1; + } + y += 1; + } + let node_idx = tree.offsets[level as usize] + cy * curr_w + cx; + tree.values[node_idx as usize] = min_value; + cx += 1; + } + cy += 1; + } + level += 1; + } +} + +#[allow(clippy::too_many_arguments)] +#[inline(always)] +fn j2k_packet_build_tag_trees( + inclusion_tree: &mut J2kPacketTagTree, + zbp_tree: &mut J2kPacketTagTree, + subband: J2kHtPacketSubband, + blocks: *const J2kHtPacketBlock, + tag_states: *const J2kHtPacketSubbandTagState, + tag_nodes: *const J2kHtPacketTagNodeState, + tag_state_count: u64, + tag_node_count: u64, + subband_meta_idx: u32, +) { + if !j2k_packet_tag_tree_init(inclusion_tree, subband.num_cbs_x, subband.num_cbs_y) + || !j2k_packet_tag_tree_init(zbp_tree, subband.num_cbs_x, subband.num_cbs_y) + { + inclusion_tree.failed = 1; + zbp_tree.failed = 1; + return; + } + + let mut idx = 0_u32; + while idx < subband.block_count { + let block = load_job(blocks, subband.block_start + idx); + let x = idx % subband.num_cbs_x; + let y = idx / subband.num_cbs_x; + let leaf_idx = y * subband.num_cbs_x + x; + inclusion_tree.values[leaf_idx as usize] = if block.previously_included == 0 { + block.inclusion_layer + } else { + J2K_PACKET_TAG_INF + }; + zbp_tree.values[leaf_idx as usize] = block.num_zero_bitplanes; + idx += 1; + } + j2k_packet_tag_tree_propagate(inclusion_tree); + j2k_packet_tag_tree_propagate(zbp_tree); + + if tag_state_count == 0 { + return; + } + if subband_meta_idx as u64 >= tag_state_count { + inclusion_tree.failed = 1; + zbp_tree.failed = 1; + return; + } + let state = load_job(tag_states, subband_meta_idx); + if state.node_count != inclusion_tree.total_nodes { + inclusion_tree.failed = 1; + zbp_tree.failed = 1; + return; + } + let Some(inclusion_end) = + (state.inclusion_node_start as u64).checked_add(state.node_count as u64) + else { + inclusion_tree.failed = 1; + zbp_tree.failed = 1; + return; + }; + let Some(zbp_end) = + (state.zero_bitplane_node_start as u64).checked_add(state.node_count as u64) + else { + inclusion_tree.failed = 1; + zbp_tree.failed = 1; + return; + }; + if inclusion_end > tag_node_count || zbp_end > tag_node_count { + inclusion_tree.failed = 1; + zbp_tree.failed = 1; + return; + } + + let mut node_idx = 0_u32; + while node_idx < state.node_count { + let inclusion_node = load_job(tag_nodes, state.inclusion_node_start + node_idx); + let zbp_node = load_job(tag_nodes, state.zero_bitplane_node_start + node_idx); + let slot = node_idx as usize; + inclusion_tree.current[slot] = inclusion_node.current; + inclusion_tree.known[slot] = inclusion_node.known; + zbp_tree.current[slot] = zbp_node.current; + zbp_tree.known[slot] = zbp_node.known; + node_idx += 1; + } +} + +#[inline(always)] +fn j2k_packet_tag_tree_encode( + tree: &mut J2kPacketTagTree, + x: u32, + y: u32, + max_value: u32, + writer: &mut J2kPacketBitWriter, +) { + let mut path = [0_u32; J2K_PACKET_MAX_TAG_LEVELS]; + let mut cx = x; + let mut cy = y; + let mut level = 0_u32; + while level < tree.levels { + path[level as usize] = tree.offsets[level as usize] + cy * tree.widths[level as usize] + cx; + cx >>= 1; + cy >>= 1; + level += 1; + } + + let mut parent_value = 0_u32; + let mut reverse = tree.levels; + while reverse > 0 { + let node_idx = path[(reverse - 1) as usize]; + let slot = node_idx as usize; + let mut start = tree.current[slot].max(parent_value); + if tree.known[slot] == 0 { + let target = tree.values[slot].min(max_value); + while start < target { + j2k_packet_write_bit(writer, 0); + start += 1; + } + if tree.values[slot] < max_value { + j2k_packet_write_bit(writer, 1); + tree.known[slot] = 1; + } + tree.current[slot] = target; + } + parent_value = tree.current[slot]; + reverse -= 1; + } +} + +#[inline(always)] +fn j2k_packet_header_result( + code: u32, + detail: u32, + header_len: u32, + body_len: u32, + output_len: u32, +) -> J2kPacketHeaderResult { + J2kPacketHeaderResult { + code, + detail, + header_len, + body_len, + output_len, + } +} + +#[allow(clippy::too_many_arguments)] +#[inline(always)] +fn j2k_packet_build_header_serial( + payload_len: u64, + packet: J2kHtPacketJob, + subbands: *const J2kHtPacketSubband, + blocks: *const J2kHtPacketBlock, + tag_states: *const J2kHtPacketSubbandTagState, + tag_nodes: *const J2kHtPacketTagNodeState, + tag_state_count: u64, + tag_node_count: u64, + packet_out: *mut u8, +) -> J2kPacketHeaderResult { + let mut writer = j2k_packet_writer_init(packet_out, packet.output_capacity); + + let mut any_data = 0_u32; + let mut subband_idx = 0_u32; + while subband_idx < packet.subband_count { + let subband = load_job(subbands, packet.subband_start + subband_idx); + if subband.num_cbs_x == 0 + || subband.num_cbs_y == 0 + || subband.num_cbs_x.checked_mul(subband.num_cbs_y) != Some(subband.block_count) + { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 7, 0, 0, 0); + } + let mut idx = 0_u32; + while idx < subband.block_count { + let block = load_job(blocks, subband.block_start + idx); + if block.num_coding_passes > 0 { + any_data = 1; + } + if block.num_coding_passes > 164 { + return j2k_packet_header_result(J2K_ENCODE_STATUS_UNSUPPORTED, 1, 0, 0, 0); + } + let Some(data_end) = (block.data_offset as u64).checked_add(block.data_len as u64) + else { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 2, 0, 0, 0); + }; + if data_end > payload_len { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 2, 0, 0, 0); + } + if block.num_coding_passes == 0 { + if block.data_len != 0 || block.cleanup_length != 0 || block.refinement_length != 0 + { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 10, 0, 0, 0); + } + } else if block.num_coding_passes == 1 { + let cleanup_length = if block.cleanup_length == 0 { + block.data_len + } else { + block.cleanup_length + }; + if cleanup_length != block.data_len || block.refinement_length != 0 { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 11, 0, 0, 0); + } + } else { + let segment_len = block.cleanup_length as u64 + block.refinement_length as u64; + if block.cleanup_length == 0 + || block.refinement_length == 0 + || segment_len != block.data_len as u64 + || block.cleanup_length < 2 + || block.cleanup_length >= 65_535 + || block.refinement_length >= 2047 + { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 12, 0, 0, 0); + } + } + idx += 1; + } + subband_idx += 1; + } + + if any_data == 0 { + j2k_packet_write_bit(&mut writer, 0); + j2k_packet_finish(&mut writer); + if writer.failed != 0 { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 3, 0, 0, 0); + } + return j2k_packet_header_result(J2K_ENCODE_STATUS_OK, 0, writer.pos, 0, writer.pos); + } + + j2k_packet_write_bit(&mut writer, 1); + subband_idx = 0; + while subband_idx < packet.subband_count { + let subband_meta_idx = packet.subband_start + subband_idx; + let subband = load_job(subbands, subband_meta_idx); + let mut inclusion_tree = j2k_packet_tag_tree_new(); + let mut zbp_tree = j2k_packet_tag_tree_new(); + j2k_packet_build_tag_trees( + &mut inclusion_tree, + &mut zbp_tree, + subband, + blocks, + tag_states, + tag_nodes, + tag_state_count, + tag_node_count, + subband_meta_idx, + ); + if inclusion_tree.failed != 0 || zbp_tree.failed != 0 { + return j2k_packet_header_result(J2K_ENCODE_STATUS_UNSUPPORTED, 8, 0, 0, 0); + } + + let mut idx = 0_u32; + while idx < subband.block_count { + let block = load_job(blocks, subband.block_start + idx); + let x = idx % subband.num_cbs_x; + let y = idx / subband.num_cbs_x; + if block.previously_included == 0 { + if block.num_coding_passes > 0 && block.inclusion_layer != packet.layer { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 9, 0, 0, 0); + } + j2k_packet_tag_tree_encode( + &mut inclusion_tree, + x, + y, + packet.layer + 1, + &mut writer, + ); + if block.num_coding_passes == 0 { + idx += 1; + continue; + } + j2k_packet_tag_tree_encode( + &mut zbp_tree, + x, + y, + block.num_zero_bitplanes + 1, + &mut writer, + ); + } else if block.num_coding_passes > 0 { + j2k_packet_write_bit(&mut writer, 1); + } else { + j2k_packet_write_bit(&mut writer, 0); + idx += 1; + continue; + } + j2k_packet_encode_num_ht_passes(&mut writer, block.num_coding_passes); + j2k_packet_encode_ht_segment_lengths(&mut writer, block); + idx += 1; + } + subband_idx += 1; + } + + j2k_packet_finish(&mut writer); + if writer.failed != 0 { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 4, 0, 0, 0); + } + if writer.pos > 0 && load_u8(packet_out.cast_const(), (writer.pos - 1) as u64) == 0xff { + if writer.pos >= packet.output_capacity { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 5, 0, 0, 0); + } + store_u8(packet_out, writer.pos as u64, 0); + writer.pos += 1; + } + + let header_len = writer.pos; + let mut body_len = 0_u32; + subband_idx = 0; + while subband_idx < packet.subband_count { + let subband = load_job(subbands, packet.subband_start + subband_idx); + let mut idx = 0_u32; + while idx < subband.block_count { + let block = load_job(blocks, subband.block_start + idx); + if block.num_coding_passes != 0 && block.data_len != 0 { + let Some(next_body_len) = body_len.checked_add(block.data_len) else { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 6, 0, 0, 0); + }; + let Some(total_len) = header_len.checked_add(next_body_len) else { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 6, 0, 0, 0); + }; + if total_len > packet.output_capacity { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 6, 0, 0, 0); + } + body_len = next_body_len; + } + idx += 1; + } + subband_idx += 1; + } + + j2k_packet_header_result( + J2K_ENCODE_STATUS_OK, + 0, + header_len, + body_len, + header_len + body_len, + ) +} + +#[inline(always)] +fn j2k_packet_copy_body_cooperative( + payload: *const u8, + packet: J2kHtPacketJob, + subbands: *const J2kHtPacketSubband, + blocks: *const J2kHtPacketBlock, + packet_out: *mut u8, + header_len: u32, + body_len: u32, +) { + let mut body_byte = thread::threadIdx_x(); + let step = thread::blockDim_x(); + while body_byte < body_len { + let mut remaining = body_byte; + let mut copied = false; + let mut subband_idx = 0_u32; + while subband_idx < packet.subband_count && !copied { + let subband = load_job(subbands, packet.subband_start + subband_idx); + let mut idx = 0_u32; + while idx < subband.block_count { + let block = load_job(blocks, subband.block_start + idx); + if block.num_coding_passes != 0 && block.data_len != 0 { + if remaining < block.data_len { + store_u8( + packet_out, + (header_len + body_byte) as u64, + load_u8(payload, (block.data_offset + remaining) as u64), + ); + copied = true; + break; + } + remaining -= block.data_len; + } + idx += 1; + } + subband_idx += 1; + } + body_byte += step; + } +} + +#[inline(always)] +fn floor_f32(value: f32) -> f32 { + // f32::floor routes through libdevice in cuda-oxide, which emits NVVM IR + // instead of the PTX loaded by this runtime path. + let truncated = value as i32 as f32; + if truncated > value { + truncated - 1.0 + } else { + truncated + } +} + +#[inline(always)] +fn abs_f32(value: f32) -> f32 { + if value < 0.0 { -value } else { value } +} + +#[inline(always)] +fn j2k_fdwt53_predict_row(src: *const f32, row_base: u32, width: u32, high_index: u32) -> f32 { + let odd = high_index * 2 + 1; + let last_even = if width % 2 == 0 { width - 2 } else { width - 1 }; + let left = load_f32(src, row_base + odd - 1); + let right = if odd + 1 < width { + load_f32(src, row_base + odd + 1) + } else { + load_f32(src, row_base + last_even) + }; + load_f32(src, row_base + odd) - floor_f32((left + right) * 0.5) +} + +#[inline(always)] +fn j2k_fdwt53_predict_col( + src: *const f32, + x: u32, + full_width: u32, + height: u32, + high_index: u32, +) -> f32 { + let odd = high_index * 2 + 1; + let last_even = if height % 2 == 0 { + height - 2 + } else { + height - 1 + }; + let top = load_f32(src, (odd - 1) * full_width + x); + let bottom = if odd + 1 < height { + load_f32(src, (odd + 1) * full_width + x) + } else { + load_f32(src, last_even * full_width + x) + }; + load_f32(src, odd * full_width + x) - floor_f32((top + bottom) * 0.5) +} + +#[inline(always)] +fn j2k_fdwt97_high1_row(src: *const f32, row_base: u32, width: u32, high_index: u32) -> f32 { + let odd = high_index * 2 + 1; + let last_even = if width % 2 == 0 { width - 2 } else { width - 1 }; + let left = load_f32(src, row_base + odd - 1); + let right = if odd + 1 < width { + load_f32(src, row_base + odd + 1) + } else { + load_f32(src, row_base + last_even) + }; + load_f32(src, row_base + odd) + J2K_FDWT97_ALPHA * (left + right) +} + +#[inline(always)] +fn j2k_fdwt97_low1_row(src: *const f32, row_base: u32, width: u32, low_index: u32) -> f32 { + let even = low_index * 2; + let left = if low_index > 0 { + j2k_fdwt97_high1_row(src, row_base, width, low_index - 1) + } else { + j2k_fdwt97_high1_row(src, row_base, width, 0) + }; + let right = if even + 1 < width { + j2k_fdwt97_high1_row(src, row_base, width, low_index) + } else { + left + }; + load_f32(src, row_base + even) + J2K_FDWT97_BETA * (left + right) +} + +#[inline(always)] +fn j2k_fdwt97_high2_row(src: *const f32, row_base: u32, width: u32, high_index: u32) -> f32 { + let odd = high_index * 2 + 1; + let last_even = if width % 2 == 0 { width - 2 } else { width - 1 }; + let last_low = last_even / 2; + let left = j2k_fdwt97_low1_row(src, row_base, width, high_index); + let right = if odd + 1 < width { + j2k_fdwt97_low1_row(src, row_base, width, high_index + 1) + } else { + j2k_fdwt97_low1_row(src, row_base, width, last_low) + }; + j2k_fdwt97_high1_row(src, row_base, width, high_index) + J2K_FDWT97_GAMMA * (left + right) +} + +#[inline(always)] +fn j2k_fdwt97_low2_row(src: *const f32, row_base: u32, width: u32, low_index: u32) -> f32 { + let even = low_index * 2; + let left = if low_index > 0 { + j2k_fdwt97_high2_row(src, row_base, width, low_index - 1) + } else { + j2k_fdwt97_high2_row(src, row_base, width, 0) + }; + let right = if even + 1 < width { + j2k_fdwt97_high2_row(src, row_base, width, low_index) + } else { + left + }; + j2k_fdwt97_low1_row(src, row_base, width, low_index) + J2K_FDWT97_DELTA * (left + right) +} + +#[inline(always)] +fn j2k_fdwt97_high1_col( + src: *const f32, + x: u32, + full_width: u32, + height: u32, + high_index: u32, +) -> f32 { + let odd = high_index * 2 + 1; + let last_even = if height % 2 == 0 { + height - 2 + } else { + height - 1 + }; + let top = load_f32(src, (odd - 1) * full_width + x); + let bottom = if odd + 1 < height { + load_f32(src, (odd + 1) * full_width + x) + } else { + load_f32(src, last_even * full_width + x) + }; + load_f32(src, odd * full_width + x) + J2K_FDWT97_ALPHA * (top + bottom) +} + +#[inline(always)] +fn j2k_fdwt97_low1_col( + src: *const f32, + x: u32, + full_width: u32, + height: u32, + low_index: u32, +) -> f32 { + let even = low_index * 2; + let top = if low_index > 0 { + j2k_fdwt97_high1_col(src, x, full_width, height, low_index - 1) + } else { + j2k_fdwt97_high1_col(src, x, full_width, height, 0) + }; + let bottom = if even + 1 < height { + j2k_fdwt97_high1_col(src, x, full_width, height, low_index) + } else { + top + }; + load_f32(src, even * full_width + x) + J2K_FDWT97_BETA * (top + bottom) +} + +#[inline(always)] +fn j2k_fdwt97_high2_col( + src: *const f32, + x: u32, + full_width: u32, + height: u32, + high_index: u32, +) -> f32 { + let odd = high_index * 2 + 1; + let last_even = if height % 2 == 0 { + height - 2 + } else { + height - 1 + }; + let last_low = last_even / 2; + let top = j2k_fdwt97_low1_col(src, x, full_width, height, high_index); + let bottom = if odd + 1 < height { + j2k_fdwt97_low1_col(src, x, full_width, height, high_index + 1) + } else { + j2k_fdwt97_low1_col(src, x, full_width, height, last_low) + }; + j2k_fdwt97_high1_col(src, x, full_width, height, high_index) + J2K_FDWT97_GAMMA * (top + bottom) +} + +#[inline(always)] +fn j2k_fdwt97_low2_col( + src: *const f32, + x: u32, + full_width: u32, + height: u32, + low_index: u32, +) -> f32 { + let even = low_index * 2; + let top = if low_index > 0 { + j2k_fdwt97_high2_col(src, x, full_width, height, low_index - 1) + } else { + j2k_fdwt97_high2_col(src, x, full_width, height, 0) + }; + let bottom = if even + 1 < height { + j2k_fdwt97_high2_col(src, x, full_width, height, low_index) + } else { + top + }; + j2k_fdwt97_low1_col(src, x, full_width, height, low_index) + J2K_FDWT97_DELTA * (top + bottom) +} + +#[inline(always)] +fn ldexp_one_f32(exponent: i32) -> f32 { + if exponent < -149 { + 0.0 + } else if exponent < -126 { + f32::from_bits(1_u32 << ((exponent + 149) as u32)) + } else if exponent <= 127 { + f32::from_bits(((exponent + 127) as u32) << 23) + } else { + f32::INFINITY + } +} + +#[inline(always)] +fn j2k_quantize_sample( + sample: f32, + step_exponent: u32, + step_mantissa: u32, + range_bits: u32, + reversible: u32, +) -> i32 { + if reversible != 0 { + let rounded = if sample >= 0.0 { + floor_f32(sample + 0.5) + } else { + -floor_f32(-sample + 0.5) + }; + return rounded as i32; + } + + let exponent = range_bits as i32 - step_exponent as i32; + let base = ldexp_one_f32(exponent); + let delta = base * (1.0 + step_mantissa as f32 / 2048.0); + if delta <= 0.0 { + return 0; + } + + let sign = if sample < 0.0 { -1 } else { 1 }; + let magnitude = floor_f32(abs_f32(sample) / delta) as i32; + sign * magnitude +} + +#[cuda_module] +mod kernels { + use super::*; + + #[kernel] + pub unsafe fn j2k_deinterleave_to_f32( + pixels: *const u8, + components: *mut f32, + num_pixels: u64, + num_components: u32, + bit_depth: u32, + is_signed: u32, + ) { + let idx = thread::index_1d().get() as u64; + if idx >= num_pixels || num_components == 0 || num_components > 4 { + return; + } + + let bytes_per_sample = if bit_depth <= 8 { 1_u32 } else { 2_u32 }; + let unsigned_offset = if is_signed != 0 { + 0.0 + } else { + (1_u32 << (bit_depth - 1)) as f32 + }; + let pixel_base = idx * num_components as u64 * bytes_per_sample as u64; + let mut component = 0_u32; + while component < num_components { + let sample_base = pixel_base + component as u64 * bytes_per_sample as u64; + let sample = if bit_depth <= 8 { + let raw = load_u8(pixels, sample_base); + if is_signed != 0 { + (raw as i8) as f32 + } else { + raw as f32 - unsigned_offset + } + } else { + let raw = load_u8(pixels, sample_base) as u16 + | ((load_u8(pixels, sample_base + 1) as u16) << 8); + if is_signed != 0 { + (raw as i16) as f32 + } else { + raw as f32 - unsigned_offset + } + }; + store_f32_u64(components, component as u64 * num_pixels + idx, sample); + component += 1; + } + } + + #[kernel] + pub unsafe fn j2k_deinterleave_strided_to_f32( + pixels: *const u8, + components: *mut f32, + width: u64, + height: u64, + byte_offset: u64, + pitch_bytes: u64, + num_components: u32, + bit_depth: u32, + is_signed: u32, + ) { + let idx = thread::index_1d().get() as u64; + let num_pixels = width * height; + if idx >= num_pixels || num_components == 0 || num_components > 4 { + return; + } + + let bytes_per_sample = if bit_depth <= 8 { 1_u32 } else { 2_u32 }; + let unsigned_offset = if is_signed != 0 { + 0.0 + } else { + (1_u32 << (bit_depth - 1)) as f32 + }; + let y = idx / width; + let x = idx - y * width; + let pixel_base = + byte_offset + y * pitch_bytes + x * num_components as u64 * bytes_per_sample as u64; + let mut component = 0_u32; + while component < num_components { + let sample_base = pixel_base + component as u64 * bytes_per_sample as u64; + let sample = if bit_depth <= 8 { + let raw = load_u8(pixels, sample_base); + if is_signed != 0 { + (raw as i8) as f32 + } else { + raw as f32 - unsigned_offset + } + } else { + let raw = load_u8(pixels, sample_base) as u16 + | ((load_u8(pixels, sample_base + 1) as u16) << 8); + if is_signed != 0 { + (raw as i16) as f32 + } else { + raw as f32 - unsigned_offset + } + }; + store_f32_u64(components, component as u64 * num_pixels + idx, sample); + component += 1; + } + } + + #[kernel] + pub unsafe fn j2k_forward_rct(plane0: *mut f32, plane1: *mut f32, plane2: *mut f32, len: u64) { + let idx = thread::index_1d().get() as u64; + if idx >= len { + return; + } + + let r = load_f32_u64(plane0.cast_const(), idx); + let g = load_f32_u64(plane1.cast_const(), idx); + let b = load_f32_u64(plane2.cast_const(), idx); + store_f32_u64(plane0, idx, floor_f32((r + 2.0 * g + b) * 0.25)); + store_f32_u64(plane1, idx, b - g); + store_f32_u64(plane2, idx, r - g); + } + + #[kernel] + pub unsafe fn j2k_forward_ict(plane0: *mut f32, plane1: *mut f32, plane2: *mut f32, len: u64) { + let idx = thread::index_1d().get() as u64; + if idx >= len { + return; + } + + let r = load_f32_u64(plane0.cast_const(), idx); + let g = load_f32_u64(plane1.cast_const(), idx); + let b = load_f32_u64(plane2.cast_const(), idx); + store_f32_u64(plane0, idx, 0.299 * r + 0.587 * g + 0.114 * b); + store_f32_u64(plane1, idx, -0.16875 * r - 0.33126 * g + 0.5 * b); + store_f32_u64(plane2, idx, 0.5 * r - 0.41869 * g - 0.08131 * b); + } + + #[kernel] + pub unsafe fn j2k_forward_dwt53_horizontal( + src: *const f32, + dst: *mut f32, + full_width: u32, + current_width: u32, + current_height: u32, + low_width: u32, + ) { + let x = thread::index_2d_col() as u32; + let y = thread::index_2d_row() as u32; + if x >= current_width || y >= current_height { + return; + } + + let row_base = y * full_width; + if x < low_width { + let even = x * 2; + let left = if x > 0 { + j2k_fdwt53_predict_row(src, row_base, current_width, x - 1) + } else { + j2k_fdwt53_predict_row(src, row_base, current_width, 0) + }; + let right = if even + 1 < current_width { + j2k_fdwt53_predict_row(src, row_base, current_width, x) + } else { + left + }; + let value = load_f32(src, row_base + even) + floor_f32((left + right) * 0.25 + 0.5); + store_f32(dst, row_base + x, value); + return; + } + + let value = j2k_fdwt53_predict_row(src, row_base, current_width, x - low_width); + store_f32(dst, row_base + x, value); + } + + #[kernel] + pub unsafe fn j2k_forward_dwt53_vertical( + src: *const f32, + dst: *mut f32, + full_width: u32, + current_width: u32, + current_height: u32, + low_height: u32, + ) { + let x = thread::index_2d_col() as u32; + let y = thread::index_2d_row() as u32; + if x >= current_width || y >= current_height { + return; + } + + if y < low_height { + let even = y * 2; + let top = if y > 0 { + j2k_fdwt53_predict_col(src, x, full_width, current_height, y - 1) + } else { + j2k_fdwt53_predict_col(src, x, full_width, current_height, 0) + }; + let bottom = if even + 1 < current_height { + j2k_fdwt53_predict_col(src, x, full_width, current_height, y) + } else { + top + }; + let value = + load_f32(src, even * full_width + x) + floor_f32((top + bottom) * 0.25 + 0.5); + store_f32(dst, y * full_width + x, value); + return; + } + + let value = j2k_fdwt53_predict_col(src, x, full_width, current_height, y - low_height); + store_f32(dst, y * full_width + x, value); + } + + #[kernel] + pub unsafe fn j2k_forward_dwt97_horizontal( + src: *const f32, + dst: *mut f32, + full_width: u32, + current_width: u32, + current_height: u32, + low_width: u32, + ) { + let x = thread::index_2d_col() as u32; + let y = thread::index_2d_row() as u32; + if x >= current_width || y >= current_height { + return; + } + + let row_base = y * full_width; + let value = if x < low_width { + j2k_fdwt97_low2_row(src, row_base, current_width, x) * J2K_FDWT97_INV_KAPPA + } else { + j2k_fdwt97_high2_row(src, row_base, current_width, x - low_width) * J2K_FDWT97_KAPPA + }; + store_f32(dst, row_base + x, value); + } + + #[kernel] + pub unsafe fn j2k_forward_dwt97_vertical( + src: *const f32, + dst: *mut f32, + full_width: u32, + current_width: u32, + current_height: u32, + low_height: u32, + ) { + let x = thread::index_2d_col() as u32; + let y = thread::index_2d_row() as u32; + if x >= current_width || y >= current_height { + return; + } + + let value = if y < low_height { + j2k_fdwt97_low2_col(src, x, full_width, current_height, y) * J2K_FDWT97_INV_KAPPA + } else { + j2k_fdwt97_high2_col(src, x, full_width, current_height, y - low_height) + * J2K_FDWT97_KAPPA + }; + store_f32(dst, y * full_width + x, value); + } + + #[kernel] + pub unsafe fn j2k_quantize_subband( + samples: *const f32, + coefficients: *mut i32, + len: u64, + step_exponent: u32, + step_mantissa: u32, + range_bits: u32, + reversible: u32, + ) { + let idx = thread::index_1d().get() as u64; + if idx >= len { + return; + } + + let coefficient = j2k_quantize_sample( + load_f32_u64(samples, idx), + step_exponent, + step_mantissa, + range_bits, + reversible, + ); + store_i32(coefficients, idx, coefficient); + } + + #[kernel] + pub unsafe fn j2k_quantize_subband_strided( + samples: *const f32, + coefficients: *mut i32, + x0: u32, + y0: u32, + width: u32, + height: u32, + stride: u32, + step_exponent: u32, + step_mantissa: u32, + range_bits: u32, + reversible: u32, + ) { + let x = thread::index_2d_col() as u32; + let y = thread::index_2d_row() as u32; + if x >= width || y >= height { + return; + } + + let source_index = (y0 + y) as u64 * stride as u64 + (x0 + x) as u64; + let output_index = y as u64 * width as u64 + x as u64; + let coefficient = j2k_quantize_sample( + load_f32_u64(samples, source_index), + step_exponent, + step_mantissa, + range_bits, + reversible, + ); + store_i32(coefficients, output_index, coefficient); + } + + #[kernel] + pub unsafe fn j2k_htj2k_compact_codeblocks( + scratch: *const u8, + compact: *mut u8, + jobs: *const J2kHtEncodeCompactJob, + job_count: u64, + ) { + let job_idx = thread::blockIdx_x(); + if job_idx as u64 >= job_count { + return; + } + + let job = load_job(jobs, job_idx); + let mut idx = thread::threadIdx_x(); + let step = thread::blockDim_x(); + if (job.reserved & J2K_HT_COMPACT_ASSEMBLE_FLAG) != 0 { + let mel_len = job.reserved & J2K_HT_COMPACT_LENGTH_MASK; + let vlc_len = (job.reserved >> 15) & J2K_HT_COMPACT_LENGTH_MASK; + let locator_bytes = mel_len + vlc_len; + if locator_bytes > job.data_len { + return; + } + let ms_len = job.data_len - locator_bytes; + let vlc_start = J2K_HT_VLC_SIZE - vlc_len; + while idx < job.data_len { + let mut value = if idx < ms_len { + load_u8(scratch, (job.source_offset + idx) as u64) + } else if idx < ms_len + mel_len { + load_u8( + scratch, + (job.source_offset + J2K_HT_MEL_OFFSET + idx - ms_len) as u64, + ) + } else { + load_u8( + scratch, + (job.source_offset + J2K_HT_VLC_OFFSET + vlc_start + idx - ms_len - mel_len) + as u64, + ) + }; + if job.data_len >= 2 { + if idx == job.data_len - 1 { + value = (locator_bytes >> 4) as u8; + } else if idx == job.data_len - 2 { + value = ((u32::from(value) & 0xf0) | (locator_bytes & 0x0f)) as u8; + } + } + store_u8(compact, (job.compact_offset + idx) as u64, value); + idx += step; + } + return; + } + + while idx < job.data_len { + store_u8( + compact, + (job.compact_offset + idx) as u64, + load_u8(scratch, (job.source_offset + idx) as u64), + ); + idx += step; + } + } + + #[kernel] + pub unsafe fn j2k_htj2k_packetize_cleanup( + payload: *const u8, + payload_len: u64, + packets: *const J2kHtPacketJob, + subbands: *const J2kHtPacketSubband, + blocks: *const J2kHtPacketBlock, + tag_states: *const J2kHtPacketSubbandTagState, + tag_nodes: *const J2kHtPacketTagNodeState, + tag_state_count: u64, + tag_node_count: u64, + out: *mut u8, + statuses: *mut J2kHtPacketStatus, + packet_count: u64, + ) { + static mut HEADER_RESULT: SharedArray = SharedArray::UNINIT; + + let packet_idx = thread::blockIdx_x() as u64; + if packet_idx >= packet_count { + return; + } + + let packet = load_job(packets, packet_idx as u32); + let status = unsafe { statuses.add(packet_idx as usize) }; + let packet_out = unsafe { out.add(packet.output_offset as usize) }; + let header_result = unsafe { HEADER_RESULT.as_mut_ptr() }; + + if thread::threadIdx_x() == 0 { + let result = j2k_packet_build_header_serial( + payload_len, + packet, + subbands, + blocks, + tag_states, + tag_nodes, + tag_state_count, + tag_node_count, + packet_out, + ); + store_u32(header_result, 0, result.code); + store_u32(header_result, 1, result.header_len); + store_u32(header_result, 2, result.body_len); + j2k_packet_status(status, result.code, result.detail, result.output_len); + } + thread::sync_threads(); + + let shared_code = load_u32(header_result.cast_const(), 0); + let shared_body_len = load_u32(header_result.cast_const(), 2); + if shared_code != J2K_ENCODE_STATUS_OK || shared_body_len == 0 { + return; + } + j2k_packet_copy_body_cooperative( + payload, + packet, + subbands, + blocks, + packet_out, + load_u32(header_result.cast_const(), 1), + shared_body_len, + ); + } +} + +fn main() {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/src/main.rs new file mode 100644 index 00000000..f328e4d9 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/src/main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/Cargo.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/Cargo.toml new file mode 100644 index 00000000..b68af6da --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "j2k_cuda_oxide_j2k_idwt_host" +version = "0.1.0" +edition = "2024" + +[package.metadata.cuda-oxide.interop] +kind = "j2k-idwt" + +[[package.metadata.cuda-oxide.device-crates]] +manifest-path = "simt/Cargo.toml" +ptx-dir = "ptx" +artifact-name = "j2k_cuda_oxide_j2k_idwt" + +[workspace] +resolver = "2" diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/rust-toolchain.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/rust-toolchain.toml new file mode 100644 index 00000000..9f964011 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly-2026-04-03" +components = ["rust-src", "rustc-dev", "rust-analyzer", "clippy", "llvm-tools"] diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/simt/Cargo.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/simt/Cargo.toml new file mode 100644 index 00000000..538717ac --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/simt/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "j2k_cuda_oxide_j2k_idwt" +version = "0.1.0" +edition = "2024" + +[workspace] +resolver = "2" + +[dependencies] +cuda-core = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } +cuda-device = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } +cuda-host = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/simt/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/simt/src/main.rs new file mode 100644 index 00000000..fbbfad80 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/simt/src/main.rs @@ -0,0 +1,1014 @@ +#![allow(static_mut_refs)] + +use cuda_device::{SharedArray, kernel, thread}; +use cuda_host::cuda_module; + +const IDWT_COOP_SAMPLES: usize = 512; +const IDWT_COLS4_COLUMNS: u32 = 4; +const IDWT_COLS4_SAMPLES: usize = 256 * IDWT_COLS4_COLUMNS as usize; +const IDWT_NEG_ALPHA: f32 = 1.5861343; +const IDWT_NEG_BETA: f32 = 0.052980117; +const IDWT_NEG_GAMMA: f32 = -0.8829111; +const IDWT_NEG_DELTA: f32 = -0.44350687; +const IDWT_KAPPA: f32 = 1.2301741; +const IDWT_INV_KAPPA: f32 = 1.0 / IDWT_KAPPA; + +#[repr(C)] +#[derive(Clone, Copy)] +struct CudaJ2kRect { + x0: u32, + y0: u32, + x1: u32, + y1: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct CudaJ2kIdwtJob { + rect: CudaJ2kRect, + ll_rect: CudaJ2kRect, + hl_rect: CudaJ2kRect, + lh_rect: CudaJ2kRect, + hh_rect: CudaJ2kRect, + irreversible97: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct CudaJ2kIdwtMultiJob { + ll_ptr: u64, + hl_ptr: u64, + lh_ptr: u64, + hh_ptr: u64, + output_ptr: u64, + job: CudaJ2kIdwtJob, +} + +#[derive(Clone, Copy)] +struct IdwtPointers { + ll: *const f32, + hl: *const f32, + lh: *const f32, + hh: *const f32, + output: *mut f32, +} + +#[derive(Clone, Copy)] +struct SharedLine { + samples: *mut f32, + lane: u32, + stride: u32, + active: bool, +} + +impl SharedLine { + #[inline(always)] + fn offset(self, index: u32) -> u32 { + index * self.stride + self.lane + } + + #[inline(always)] + fn load(self, index: u32) -> f32 { + load_f32(self.samples.cast_const(), self.offset(index)) + } + + #[inline(always)] + fn store(self, index: u32, value: f32) { + store_f32(self.samples, self.offset(index), value); + } +} + +#[inline(always)] +fn load_f32(ptr: *const f32, index: u32) -> f32 { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn store_f32(ptr: *mut f32, index: u32, value: f32) { + unsafe { + *ptr.add(index as usize) = value; + } +} + +#[inline(always)] +fn load_job(ptr: *const T, index: u32) -> T { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn floor_f32(value: f32) -> f32 { + let truncated = value as i32 as f32; + if truncated > value { + truncated - 1.0 + } else { + truncated + } +} + +#[inline(always)] +fn rect_width(rect: CudaJ2kRect) -> u32 { + rect.x1 - rect.x0 +} + +#[inline(always)] +fn rect_height(rect: CudaJ2kRect) -> u32 { + rect.y1 - rect.y0 +} + +#[inline(always)] +fn div_ceil_2(value: u32) -> u32 { + (value + 1) >> 1 +} + +#[inline(always)] +fn idwt_band_coord(output_origin: u32, output_coord: u32, band_origin: u32, low: bool) -> u32 { + let index = if low { + div_ceil_2(output_coord) - div_ceil_2(output_origin) + } else { + (output_coord >> 1) - (output_origin >> 1) + }; + band_origin + index +} + +#[inline(always)] +fn source_get(source: *const f32, rect: CudaJ2kRect, x: u32, y: u32) -> f32 { + if x < rect.x0 || x >= rect.x1 || y < rect.y0 || y >= rect.y1 { + return 0.0; + } + let local_x = x - rect.x0; + let local_y = y - rect.y0; + load_f32(source, local_y * rect_width(rect) + local_x) +} + +#[inline(always)] +fn pse_left(idx: u32, offset: u32) -> u32 { + idx.abs_diff(offset) +} + +#[inline(always)] +fn pse_right(idx: u32, offset: u32, length: u32) -> u32 { + let new_idx = idx + offset; + if new_idx >= length { + let overshoot = new_idx - length; + length - 2 - overshoot + } else { + new_idx + } +} + +#[inline(always)] +fn lift_53_sample(sample: f32, left: f32, right: f32, update_even: bool) -> f32 { + if update_even { + sample - floor_f32((left + right) * 0.25 + 0.5) + } else { + sample + floor_f32((left + right) * 0.5) + } +} + +#[inline(always)] +fn filter_step_horizontal_53(scanline: *mut f32, width: u32, first: u32, update_even: bool) { + if first == 0 { + let left = pse_left(0, 1); + let right = pse_right(0, 1, width); + let sample = load_f32(scanline.cast_const(), 0); + let left_sample = load_f32(scanline.cast_const(), left); + let right_sample = load_f32(scanline.cast_const(), right); + store_f32( + scanline, + 0, + lift_53_sample(sample, left_sample, right_sample, update_even), + ); + } + + let mut i = if first == 0 { 2 } else { 1 }; + while i + 1 < width { + let sample = load_f32(scanline.cast_const(), i); + let left = load_f32(scanline.cast_const(), i - 1); + let right = load_f32(scanline.cast_const(), i + 1); + store_f32( + scanline, + i, + lift_53_sample(sample, left, right, update_even), + ); + i += 2; + } + + if width > 1 && ((width - 1) & 1) == first { + let last = width - 1; + let left = pse_left(last, 1); + let right = pse_right(last, 1, width); + let sample = load_f32(scanline.cast_const(), last); + let left_sample = load_f32(scanline.cast_const(), left); + let right_sample = load_f32(scanline.cast_const(), right); + store_f32( + scanline, + last, + lift_53_sample(sample, left_sample, right_sample, update_even), + ); + } +} + +#[inline(always)] +fn filter_step_horizontal_97(scanline: *mut f32, width: u32, first: u32, coefficient: f32) { + if first == 0 { + let left = pse_left(0, 1); + let right = pse_right(0, 1, width); + let sample = load_f32(scanline.cast_const(), 0); + let left_sample = load_f32(scanline.cast_const(), left); + let right_sample = load_f32(scanline.cast_const(), right); + store_f32( + scanline, + 0, + sample + (left_sample + right_sample) * coefficient, + ); + } + + let mut i = if first == 0 { 2 } else { 1 }; + while i + 1 < width { + let sample = load_f32(scanline.cast_const(), i); + let left = load_f32(scanline.cast_const(), i - 1); + let right = load_f32(scanline.cast_const(), i + 1); + store_f32(scanline, i, sample + (left + right) * coefficient); + i += 2; + } + + if width > 1 && ((width - 1) & 1) == first { + let last = width - 1; + let left = pse_left(last, 1); + let right = pse_right(last, 1, width); + let sample = load_f32(scanline.cast_const(), last); + let left_sample = load_f32(scanline.cast_const(), left); + let right_sample = load_f32(scanline.cast_const(), right); + store_f32( + scanline, + last, + sample + (left_sample + right_sample) * coefficient, + ); + } +} + +#[inline(always)] +fn filter_horizontal_scanline(scanline: *mut f32, width: u32, rect_x0: u32, irreversible97: bool) { + if width == 1 { + if (rect_x0 & 1) != 0 { + store_f32(scanline, 0, load_f32(scanline.cast_const(), 0) * 0.5); + } + return; + } + + let first_even = rect_x0 & 1; + let first_odd = 1 - first_even; + if !irreversible97 { + filter_step_horizontal_53(scanline, width, first_even, true); + filter_step_horizontal_53(scanline, width, first_odd, false); + } else { + let k0 = if first_even == 0 { + IDWT_KAPPA + } else { + IDWT_INV_KAPPA + }; + let k1 = if first_even == 0 { + IDWT_INV_KAPPA + } else { + IDWT_KAPPA + }; + let mut i = 0; + while i + 1 < width { + store_f32(scanline, i, load_f32(scanline.cast_const(), i) * k0); + store_f32(scanline, i + 1, load_f32(scanline.cast_const(), i + 1) * k1); + i += 2; + } + if (width & 1) != 0 { + let last = width - 1; + store_f32(scanline, last, load_f32(scanline.cast_const(), last) * k0); + } + filter_step_horizontal_97(scanline, width, first_even, IDWT_NEG_DELTA); + filter_step_horizontal_97(scanline, width, first_odd, IDWT_NEG_GAMMA); + filter_step_horizontal_97(scanline, width, first_even, IDWT_NEG_BETA); + filter_step_horizontal_97(scanline, width, first_odd, IDWT_NEG_ALPHA); + } +} + +#[inline(always)] +fn filter_horizontal(output: *mut f32, rect: CudaJ2kRect, irreversible97: bool) { + let width = rect_width(rect); + let height = rect_height(rect); + let mut y = 0; + while y < height { + filter_horizontal_scanline( + unsafe { output.add((y * width) as usize) }, + width, + rect.x0, + irreversible97, + ); + y += 1; + } +} + +#[inline(always)] +fn filter_step_vertical_53_column( + output: *mut f32, + width: u32, + height: u32, + col: u32, + first: u32, + update_even: bool, +) { + let mut row = first; + while row < height { + let row_above = pse_left(row, 1); + let row_below = pse_right(row, 1, height); + let idx = row * width + col; + let sample = load_f32(output.cast_const(), idx); + let above = load_f32(output.cast_const(), row_above * width + col); + let below = load_f32(output.cast_const(), row_below * width + col); + store_f32( + output, + idx, + lift_53_sample(sample, above, below, update_even), + ); + row += 2; + } +} + +#[inline(always)] +fn filter_step_vertical_97_column( + output: *mut f32, + width: u32, + height: u32, + col: u32, + first: u32, + coefficient: f32, +) { + let mut row = first; + while row < height { + let row_above = pse_left(row, 1); + let row_below = pse_right(row, 1, height); + let idx = row * width + col; + let sample = load_f32(output.cast_const(), idx); + let above = load_f32(output.cast_const(), row_above * width + col); + let below = load_f32(output.cast_const(), row_below * width + col); + store_f32(output, idx, sample + (above + below) * coefficient); + row += 2; + } +} + +#[inline(always)] +fn filter_vertical_column( + output: *mut f32, + width: u32, + height: u32, + rect_y0: u32, + col: u32, + irreversible97: bool, +) { + if height == 1 { + if (rect_y0 & 1) != 0 { + store_f32(output, col, load_f32(output.cast_const(), col) * 0.5); + } + return; + } + + let first_even = rect_y0 & 1; + let first_odd = 1 - first_even; + if !irreversible97 { + filter_step_vertical_53_column(output, width, height, col, first_even, true); + filter_step_vertical_53_column(output, width, height, col, first_odd, false); + } else { + let k0 = if first_even == 0 { + IDWT_KAPPA + } else { + IDWT_INV_KAPPA + }; + let k1 = if first_even == 0 { + IDWT_INV_KAPPA + } else { + IDWT_KAPPA + }; + let mut row = 0; + while row + 1 < height { + let idx0 = row * width + col; + let idx1 = (row + 1) * width + col; + store_f32(output, idx0, load_f32(output.cast_const(), idx0) * k0); + store_f32(output, idx1, load_f32(output.cast_const(), idx1) * k1); + row += 2; + } + if (height & 1) != 0 { + let idx = (height - 1) * width + col; + store_f32(output, idx, load_f32(output.cast_const(), idx) * k0); + } + filter_step_vertical_97_column(output, width, height, col, first_even, IDWT_NEG_DELTA); + filter_step_vertical_97_column(output, width, height, col, first_odd, IDWT_NEG_GAMMA); + filter_step_vertical_97_column(output, width, height, col, first_even, IDWT_NEG_BETA); + filter_step_vertical_97_column(output, width, height, col, first_odd, IDWT_NEG_ALPHA); + } +} + +#[inline(always)] +fn filter_vertical(output: *mut f32, rect: CudaJ2kRect, irreversible97: bool) { + let width = rect_width(rect); + let height = rect_height(rect); + let mut col = 0; + while col < width { + filter_vertical_column(output, width, height, rect.y0, col, irreversible97); + col += 1; + } +} + +#[inline(always)] +fn idwt_interleave_sample( + ll: *const f32, + hl: *const f32, + lh: *const f32, + hh: *const f32, + job: CudaJ2kIdwtJob, + local_x: u32, + local_y: u32, +) -> f32 { + let x = job.rect.x0 + local_x; + let y = job.rect.y0 + local_y; + let low_x = (x & 1) == 0; + let low_y = (y & 1) == 0; + let (source, source_rect, band_x, band_y) = if low_x && low_y { + ( + ll, + job.ll_rect, + idwt_band_coord(job.rect.x0, x, job.ll_rect.x0, true), + idwt_band_coord(job.rect.y0, y, job.ll_rect.y0, true), + ) + } else if !low_x && low_y { + ( + hl, + job.hl_rect, + idwt_band_coord(job.rect.x0, x, job.hl_rect.x0, false), + idwt_band_coord(job.rect.y0, y, job.hl_rect.y0, true), + ) + } else if low_x && !low_y { + ( + lh, + job.lh_rect, + idwt_band_coord(job.rect.x0, x, job.lh_rect.x0, true), + idwt_band_coord(job.rect.y0, y, job.lh_rect.y0, false), + ) + } else { + ( + hh, + job.hh_rect, + idwt_band_coord(job.rect.x0, x, job.hh_rect.x0, false), + idwt_band_coord(job.rect.y0, y, job.hh_rect.y0, false), + ) + }; + source_get(source, source_rect, band_x, band_y) +} + +#[inline(always)] +fn run_interleave(pointers: IdwtPointers, job: CudaJ2kIdwtJob, local_x: u32, local_y: u32) { + let width = rect_width(job.rect); + let height = rect_height(job.rect); + if local_x >= width || local_y >= height { + return; + } + store_f32( + pointers.output, + local_y * width + local_x, + idwt_interleave_sample( + pointers.ll, + pointers.hl, + pointers.lh, + pointers.hh, + job, + local_x, + local_y, + ), + ); +} + +#[inline(always)] +fn multi_job_pointers(item: CudaJ2kIdwtMultiJob) -> IdwtPointers { + IdwtPointers { + ll: item.ll_ptr as usize as *const f32, + hl: item.hl_ptr as usize as *const f32, + lh: item.lh_ptr as usize as *const f32, + hh: item.hh_ptr as usize as *const f32, + output: item.output_ptr as usize as *mut f32, + } +} + +#[inline(always)] +fn filter_shared_single_sample(line: SharedLine, index: u32, len: u32, origin: u32) -> bool { + if len != 1 { + return false; + } + if line.active && index == 0 && (origin & 1) != 0 { + line.store(0, line.load(0) * 0.5); + } + thread::sync_threads(); + true +} + +#[inline(always)] +fn filter_shared_53_step(line: SharedLine, index: u32, len: u32, first: u32, update_even: bool) { + if !line.active || index >= len || (index & 1) != first { + return; + } + + let left = pse_left(index, 1); + let right = pse_right(index, 1, len); + line.store( + index, + lift_53_sample( + line.load(index), + line.load(left), + line.load(right), + update_even, + ), + ); +} + +#[inline(always)] +fn filter_shared_53(line: SharedLine, index: u32, len: u32, origin: u32) { + if filter_shared_single_sample(line, index, len, origin) { + return; + } + + let first_even = origin & 1; + let first_odd = 1 - first_even; + filter_shared_53_step(line, index, len, first_even, true); + thread::sync_threads(); + filter_shared_53_step(line, index, len, first_odd, false); + thread::sync_threads(); +} + +#[inline(always)] +fn scale_shared_97(line: SharedLine, index: u32, len: u32, first_even: u32) { + if !line.active || index >= len { + return; + } + let k0 = if first_even == 0 { + IDWT_KAPPA + } else { + IDWT_INV_KAPPA + }; + let k1 = if first_even == 0 { + IDWT_INV_KAPPA + } else { + IDWT_KAPPA + }; + let scale = if (index & 1) == 0 { k0 } else { k1 }; + line.store(index, line.load(index) * scale); +} + +#[inline(always)] +fn filter_shared_97_step(line: SharedLine, index: u32, len: u32, first: u32, coefficient: f32) { + if !line.active || index >= len || (index & 1) != first { + return; + } + + let left = pse_left(index, 1); + let right = pse_right(index, 1, len); + line.store( + index, + line.load(index) + (line.load(left) + line.load(right)) * coefficient, + ); +} + +#[inline(always)] +fn filter_shared_97(line: SharedLine, index: u32, len: u32, origin: u32) { + if filter_shared_single_sample(line, index, len, origin) { + return; + } + + let first_even = origin & 1; + let first_odd = 1 - first_even; + scale_shared_97(line, index, len, first_even); + thread::sync_threads(); + filter_shared_97_step(line, index, len, first_even, IDWT_NEG_DELTA); + thread::sync_threads(); + filter_shared_97_step(line, index, len, first_odd, IDWT_NEG_GAMMA); + thread::sync_threads(); + filter_shared_97_step(line, index, len, first_even, IDWT_NEG_BETA); + thread::sync_threads(); + filter_shared_97_step(line, index, len, first_odd, IDWT_NEG_ALPHA); + thread::sync_threads(); +} + +#[cuda_module] +mod kernels { + use super::*; + + #[kernel] + pub unsafe fn j2k_inverse_dwt_single( + ll: *const f32, + hl: *const f32, + lh: *const f32, + hh: *const f32, + output: *mut f32, + job_buffer: *const CudaJ2kIdwtJob, + ) { + if thread::blockIdx_x() != 0 || thread::threadIdx_x() != 0 { + return; + } + + let job = load_job(job_buffer, 0); + let width = rect_width(job.rect); + let height = rect_height(job.rect); + let mut local_y = 0; + while local_y < height { + let mut local_x = 0; + while local_x < width { + store_f32( + output, + local_y * width + local_x, + idwt_interleave_sample(ll, hl, lh, hh, job, local_x, local_y), + ); + local_x += 1; + } + local_y += 1; + } + + if width > 0 && height > 0 { + let irreversible97 = job.irreversible97 != 0; + filter_horizontal(output, job.rect, irreversible97); + filter_vertical(output, job.rect, irreversible97); + } + } + + #[kernel] + pub unsafe fn j2k_idwt_interleave( + ll: *const f32, + hl: *const f32, + lh: *const f32, + hh: *const f32, + output: *mut f32, + job_buffer: *const CudaJ2kIdwtJob, + ) { + let job = load_job(job_buffer, 0); + let local_x = thread::blockIdx_x() * thread::blockDim_x() + thread::threadIdx_x(); + let local_y = thread::blockIdx_y() * thread::blockDim_y() + thread::threadIdx_y(); + run_interleave( + IdwtPointers { + ll, + hl, + lh, + hh, + output, + }, + job, + local_x, + local_y, + ); + } + + #[kernel] + pub unsafe fn j2k_idwt_interleave_horizontal_multi(jobs: *const CudaJ2kIdwtMultiJob) { + let job_idx = thread::blockIdx_y(); + let item = load_job(jobs, job_idx); + let job = item.job; + let pointers = multi_job_pointers(item); + let width = rect_width(job.rect); + let height = rect_height(job.rect); + let local_y = thread::blockIdx_x() * thread::blockDim_x() + thread::threadIdx_x(); + if local_y >= height { + return; + } + + let mut local_x = 0; + while local_x < width { + store_f32( + pointers.output, + local_y * width + local_x, + idwt_interleave_sample( + pointers.ll, + pointers.hl, + pointers.lh, + pointers.hh, + job, + local_x, + local_y, + ), + ); + local_x += 1; + } + filter_horizontal_scanline( + unsafe { pointers.output.add((local_y * width) as usize) }, + width, + job.rect.x0, + job.irreversible97 != 0, + ); + } + + #[kernel] + pub unsafe fn j2k_idwt_interleave_horizontal_53_multi(jobs: *const CudaJ2kIdwtMultiJob) { + static mut ROW_SAMPLES: SharedArray = SharedArray::UNINIT; + + let row_samples = unsafe { ROW_SAMPLES.as_mut_ptr() }; + let shared = SharedLine { + samples: row_samples, + lane: 0, + stride: 1, + active: true, + }; + let local_x = thread::threadIdx_x(); + let local_y = thread::blockIdx_x(); + let item = load_job(jobs, thread::blockIdx_y()); + let job = item.job; + let pointers = multi_job_pointers(item); + let width = rect_width(job.rect); + let height = rect_height(job.rect); + if local_y >= height { + return; + } + + if local_x < width { + shared.store( + local_x, + idwt_interleave_sample( + pointers.ll, + pointers.hl, + pointers.lh, + pointers.hh, + job, + local_x, + local_y, + ), + ); + } + thread::sync_threads(); + + filter_shared_53(shared, local_x, width, job.rect.x0); + if local_x < width { + store_f32( + pointers.output, + local_y * width + local_x, + shared.load(local_x), + ); + } + } + + #[kernel] + pub unsafe fn j2k_idwt_interleave_horizontal_97_multi(jobs: *const CudaJ2kIdwtMultiJob) { + static mut ROW_SAMPLES: SharedArray = SharedArray::UNINIT; + + let row_samples = unsafe { ROW_SAMPLES.as_mut_ptr() }; + let shared = SharedLine { + samples: row_samples, + lane: 0, + stride: 1, + active: true, + }; + let local_x = thread::threadIdx_x(); + let local_y = thread::blockIdx_x(); + let item = load_job(jobs, thread::blockIdx_y()); + let job = item.job; + let pointers = multi_job_pointers(item); + let width = rect_width(job.rect); + let height = rect_height(job.rect); + if local_y >= height { + return; + } + + if local_x < width { + shared.store( + local_x, + idwt_interleave_sample( + pointers.ll, + pointers.hl, + pointers.lh, + pointers.hh, + job, + local_x, + local_y, + ), + ); + } + thread::sync_threads(); + + filter_shared_97(shared, local_x, width, job.rect.x0); + if local_x < width { + store_f32( + pointers.output, + local_y * width + local_x, + shared.load(local_x), + ); + } + } + + #[kernel] + pub unsafe fn j2k_idwt_horizontal(output: *mut f32, job_buffer: *const CudaJ2kIdwtJob) { + let job = load_job(job_buffer, 0); + let width = rect_width(job.rect); + let height = rect_height(job.rect); + let row = thread::blockIdx_x() * thread::blockDim_x() + thread::threadIdx_x(); + if row >= height { + return; + } + filter_horizontal_scanline( + unsafe { output.add((row * width) as usize) }, + width, + job.rect.x0, + job.irreversible97 != 0, + ); + } + + #[kernel] + pub unsafe fn j2k_idwt_horizontal_53(output: *mut f32, job_buffer: *const CudaJ2kIdwtJob) { + let job = load_job(job_buffer, 0); + let width = rect_width(job.rect); + let height = rect_height(job.rect); + let row = thread::blockIdx_x() * thread::blockDim_x() + thread::threadIdx_x(); + if row >= height { + return; + } + filter_horizontal_scanline( + unsafe { output.add((row * width) as usize) }, + width, + job.rect.x0, + false, + ); + } + + #[kernel] + pub unsafe fn j2k_idwt_horizontal_97(output: *mut f32, job_buffer: *const CudaJ2kIdwtJob) { + let job = load_job(job_buffer, 0); + let width = rect_width(job.rect); + let height = rect_height(job.rect); + let row = thread::blockIdx_x() * thread::blockDim_x() + thread::threadIdx_x(); + if row >= height { + return; + } + filter_horizontal_scanline( + unsafe { output.add((row * width) as usize) }, + width, + job.rect.x0, + true, + ); + } + + #[kernel] + pub unsafe fn j2k_idwt_vertical(output: *mut f32, job_buffer: *const CudaJ2kIdwtJob) { + let job = load_job(job_buffer, 0); + let width = rect_width(job.rect); + let height = rect_height(job.rect); + let col = thread::blockIdx_x() * thread::blockDim_x() + thread::threadIdx_x(); + if col >= width { + return; + } + filter_vertical_column( + output, + width, + height, + job.rect.y0, + col, + job.irreversible97 != 0, + ); + } + + #[kernel] + pub unsafe fn j2k_idwt_vertical_53(output: *mut f32, job_buffer: *const CudaJ2kIdwtJob) { + let job = load_job(job_buffer, 0); + let width = rect_width(job.rect); + let height = rect_height(job.rect); + let col = thread::blockIdx_x() * thread::blockDim_x() + thread::threadIdx_x(); + if col >= width { + return; + } + filter_vertical_column(output, width, height, job.rect.y0, col, false); + } + + #[kernel] + pub unsafe fn j2k_idwt_vertical_97(output: *mut f32, job_buffer: *const CudaJ2kIdwtJob) { + let job = load_job(job_buffer, 0); + let width = rect_width(job.rect); + let height = rect_height(job.rect); + let col = thread::blockIdx_x() * thread::blockDim_x() + thread::threadIdx_x(); + if col >= width { + return; + } + filter_vertical_column(output, width, height, job.rect.y0, col, true); + } + + #[kernel] + pub unsafe fn j2k_idwt_vertical_multi(jobs: *const CudaJ2kIdwtMultiJob) { + let job_idx = thread::blockIdx_y(); + let item = load_job(jobs, job_idx); + let job = item.job; + let output = item.output_ptr as usize as *mut f32; + let width = rect_width(job.rect); + let height = rect_height(job.rect); + let col = thread::blockIdx_x() * thread::blockDim_x() + thread::threadIdx_x(); + if col >= width { + return; + } + filter_vertical_column( + output, + width, + height, + job.rect.y0, + col, + job.irreversible97 != 0, + ); + } + + #[kernel] + pub unsafe fn j2k_idwt_vertical_53_multi(jobs: *const CudaJ2kIdwtMultiJob) { + static mut COLUMN_SAMPLES: SharedArray = SharedArray::UNINIT; + + let column_samples = unsafe { COLUMN_SAMPLES.as_mut_ptr() }; + let shared = SharedLine { + samples: column_samples, + lane: 0, + stride: 1, + active: true, + }; + let row = thread::threadIdx_x(); + let col = thread::blockIdx_x(); + let item = load_job(jobs, thread::blockIdx_y()); + let job = item.job; + let output = item.output_ptr as usize as *mut f32; + let width = rect_width(job.rect); + let height = rect_height(job.rect); + if col >= width { + return; + } + + if row < height { + shared.store(row, load_f32(output.cast_const(), row * width + col)); + } + thread::sync_threads(); + + filter_shared_53(shared, row, height, job.rect.y0); + if row < height { + store_f32(output, row * width + col, shared.load(row)); + } + } + + #[kernel] + pub unsafe fn j2k_idwt_vertical_97_multi(jobs: *const CudaJ2kIdwtMultiJob) { + static mut COLUMN_SAMPLES: SharedArray = SharedArray::UNINIT; + + let column_samples = unsafe { COLUMN_SAMPLES.as_mut_ptr() }; + let shared = SharedLine { + samples: column_samples, + lane: 0, + stride: 1, + active: true, + }; + let row = thread::threadIdx_x(); + let col = thread::blockIdx_x(); + let item = load_job(jobs, thread::blockIdx_y()); + let job = item.job; + let output = item.output_ptr as usize as *mut f32; + let width = rect_width(job.rect); + let height = rect_height(job.rect); + if col >= width { + return; + } + + if row < height { + shared.store(row, load_f32(output.cast_const(), row * width + col)); + } + thread::sync_threads(); + + filter_shared_97(shared, row, height, job.rect.y0); + if row < height { + store_f32(output, row * width + col, shared.load(row)); + } + } + + #[kernel] + pub unsafe fn j2k_idwt_vertical_97_multi_cols4(jobs: *const CudaJ2kIdwtMultiJob) { + static mut COLUMN_SAMPLES: SharedArray = SharedArray::UNINIT; + + let column_samples = unsafe { COLUMN_SAMPLES.as_mut_ptr() }; + let local_col = thread::threadIdx_x(); + let row = thread::threadIdx_y(); + let col = thread::blockIdx_x() * IDWT_COLS4_COLUMNS + local_col; + let item = load_job(jobs, thread::blockIdx_y()); + let job = item.job; + let output = item.output_ptr as usize as *mut f32; + let width = rect_width(job.rect); + let height = rect_height(job.rect); + if height > 256 { + return; + } + + let shared = SharedLine { + samples: column_samples, + lane: local_col, + stride: IDWT_COLS4_COLUMNS, + active: col < width, + }; + let valid = shared.active && row < height; + if valid { + shared.store(row, load_f32(output.cast_const(), row * width + col)); + } + thread::sync_threads(); + + filter_shared_97(shared, row, height, job.rect.y0); + if valid { + store_f32(output, row * width + col, shared.load(row)); + } + } +} + +fn main() {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/src/main.rs new file mode 100644 index 00000000..f328e4d9 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/src/main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/Cargo.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/Cargo.toml new file mode 100644 index 00000000..7820b845 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "j2k_cuda_oxide_transcode_host" +version = "0.1.0" +edition = "2024" + +[package.metadata.cuda-oxide.interop] +kind = "transcode" + +[[package.metadata.cuda-oxide.device-crates]] +manifest-path = "simt/Cargo.toml" +ptx-dir = "ptx" +artifact-name = "j2k_cuda_oxide_transcode" + +[workspace] +resolver = "2" diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/rust-toolchain.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/rust-toolchain.toml new file mode 100644 index 00000000..9f964011 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "nightly-2026-04-03" +components = ["rust-src", "rustc-dev", "rust-analyzer", "clippy", "llvm-tools"] diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/simt/Cargo.toml b/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/simt/Cargo.toml new file mode 100644 index 00000000..995667e5 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/simt/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "j2k_cuda_oxide_transcode" +version = "0.1.0" +edition = "2024" + +[workspace] +resolver = "2" + +[dependencies] +cuda-core = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } +cuda-device = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } +cuda-host = { git = "https://github.com/NVlabs/cuda-oxide.git", rev = "a9f964a956f397dd0b3c8db88a3ca5824186c261" } diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/simt/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/simt/src/main.rs new file mode 100644 index 00000000..2f648268 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/simt/src/main.rs @@ -0,0 +1,1215 @@ +#![allow(static_mut_refs)] + +use cuda_device::{SharedArray, kernel, thread}; +use cuda_host::cuda_module; + +const CONST_BITS: i32 = 13; +const PASS1_BITS: i32 = 2; +const FIX_0_298631336: i32 = 2446; +const FIX_0_390180644: i32 = 3196; +const FIX_0_541196100: i32 = 4433; +const FIX_0_765366865: i32 = 6270; +const FIX_0_899976223: i32 = 7373; +const FIX_1_175875602: i32 = 9633; +const FIX_1_501321110: i32 = 12299; +const FIX_1_847759065: i32 = 15137; +const FIX_1_961570560: i32 = 16069; +const FIX_2_053119869: i32 = 16819; +const FIX_2_562915447: i32 = 20995; +const FIX_3_072711026: i32 = 25172; + +#[inline(always)] +fn load_i16(ptr: *const i16, index: u64) -> i16 { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn load_i32(ptr: *const i32, index: u64) -> i32 { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn load_f32(ptr: *const f32, index: u64) -> f32 { + unsafe { *ptr.add(index as usize) } +} + +#[inline(always)] +fn store_i32(ptr: *mut i32, index: u64, value: i32) { + unsafe { + *ptr.add(index as usize) = value; + } +} + +#[inline(always)] +fn store_f32(ptr: *mut f32, index: u64, value: f32) { + unsafe { + *ptr.add(index as usize) = value; + } +} + +#[inline(always)] +fn offset_i32_mut(ptr: *mut i32, index: u64) -> *mut i32 { + unsafe { ptr.add(index as usize) } +} + +#[inline(always)] +fn offset_f32_mut(ptr: *mut f32, index: u64) -> *mut f32 { + unsafe { ptr.add(index as usize) } +} + +#[inline(always)] +fn floor_div_pos(a: i32, d: i32) -> i32 { + let mut q = a / d; + let r = a - q * d; + if r < 0 { + q -= 1; + } + q +} + +#[inline(always)] +fn descale(value: i32, shift: i32) -> i32 { + (value + (1_i32 << (shift - 1))) >> shift +} + +#[inline(always)] +fn clamp_level_shift(value: i32) -> i32 { + let shifted = value + 128; + if shifted < 0 { + -128 + } else if shifted > 255 { + 127 + } else { + shifted - 128 + } +} + +#[allow(clippy::too_many_lines)] +#[inline(always)] +fn idct_islow_signed(input: *const i16, output: *mut i32) { + let mut work = [0_i32; 64]; + let mut col = 0_i32; + while col < 8 { + let p0 = load_i16(input, col as u64) as i32; + let p1 = load_i16(input, (col + 8) as u64) as i32; + let p2 = load_i16(input, (col + 16) as u64) as i32; + let p3 = load_i16(input, (col + 24) as u64) as i32; + let p4 = load_i16(input, (col + 32) as u64) as i32; + let p5 = load_i16(input, (col + 40) as u64) as i32; + let p6 = load_i16(input, (col + 48) as u64) as i32; + let p7 = load_i16(input, (col + 56) as u64) as i32; + + let mut z2 = p2; + let mut z3 = p6; + let mut z1 = (z2 + z3) * FIX_0_541196100; + let tmp2 = z1 + z3 * -FIX_1_847759065; + let tmp3 = z1 + z2 * FIX_0_765366865; + + z2 = p0; + z3 = p4; + let tmp0 = (z2 + z3) << CONST_BITS; + let tmp1 = (z2 - z3) << CONST_BITS; + + let tmp10 = tmp0 + tmp3; + let tmp13 = tmp0 - tmp3; + let tmp11 = tmp1 + tmp2; + let tmp12 = tmp1 - tmp2; + + let tmp0 = p7; + let tmp1 = p5; + let tmp2 = p3; + let tmp3 = p1; + + z1 = tmp0 + tmp3; + z2 = tmp1 + tmp2; + z3 = tmp0 + tmp2; + let mut z4 = tmp1 + tmp3; + let z5 = (z3 + z4) * FIX_1_175875602; + + let mut tmp0 = tmp0 * FIX_0_298631336; + let mut tmp1 = tmp1 * FIX_2_053119869; + let mut tmp2 = tmp2 * FIX_3_072711026; + let mut tmp3 = tmp3 * FIX_1_501321110; + z1 *= -FIX_0_899976223; + z2 *= -FIX_2_562915447; + z3 *= -FIX_1_961570560; + z4 *= -FIX_0_390180644; + + z3 += z5; + z4 += z5; + + tmp0 += z1 + z3; + tmp1 += z2 + z4; + tmp2 += z2 + z3; + tmp3 += z1 + z4; + + let shift = CONST_BITS - PASS1_BITS; + work[col as usize] = descale(tmp10 + tmp3, shift); + work[(col + 56) as usize] = descale(tmp10 - tmp3, shift); + work[(col + 8) as usize] = descale(tmp11 + tmp2, shift); + work[(col + 48) as usize] = descale(tmp11 - tmp2, shift); + work[(col + 16) as usize] = descale(tmp12 + tmp1, shift); + work[(col + 40) as usize] = descale(tmp12 - tmp1, shift); + work[(col + 24) as usize] = descale(tmp13 + tmp0, shift); + work[(col + 32) as usize] = descale(tmp13 - tmp0, shift); + col += 1; + } + + let mut row = 0_i32; + while row < 8 { + let base = row * 8; + let p0 = work[base as usize]; + let p1 = work[(base + 1) as usize]; + let p2 = work[(base + 2) as usize]; + let p3 = work[(base + 3) as usize]; + let p4 = work[(base + 4) as usize]; + let p5 = work[(base + 5) as usize]; + let p6 = work[(base + 6) as usize]; + let p7 = work[(base + 7) as usize]; + + let shift = CONST_BITS + PASS1_BITS + 3; + let mut z2 = p2; + let mut z3 = p6; + let mut z1 = (z2 + z3) * FIX_0_541196100; + let tmp2 = z1 + z3 * -FIX_1_847759065; + let tmp3 = z1 + z2 * FIX_0_765366865; + + let tmp0 = (p0 + p4) << CONST_BITS; + let tmp1 = (p0 - p4) << CONST_BITS; + + let tmp10 = tmp0 + tmp3; + let tmp13 = tmp0 - tmp3; + let tmp11 = tmp1 + tmp2; + let tmp12 = tmp1 - tmp2; + + let tmp0 = p7; + let tmp1 = p5; + let tmp2 = p3; + let tmp3 = p1; + + z1 = tmp0 + tmp3; + z2 = tmp1 + tmp2; + z3 = tmp0 + tmp2; + let mut z4 = tmp1 + tmp3; + let z5 = (z3 + z4) * FIX_1_175875602; + + let mut tmp0 = tmp0 * FIX_0_298631336; + let mut tmp1 = tmp1 * FIX_2_053119869; + let mut tmp2 = tmp2 * FIX_3_072711026; + let mut tmp3 = tmp3 * FIX_1_501321110; + z1 *= -FIX_0_899976223; + z2 *= -FIX_2_562915447; + z3 *= -FIX_1_961570560; + z4 *= -FIX_0_390180644; + + z3 += z5; + z4 += z5; + + tmp0 += z1 + z3; + tmp1 += z2 + z4; + tmp2 += z2 + z3; + tmp3 += z1 + z4; + + let values = [ + descale(tmp10 + tmp3, shift), + descale(tmp11 + tmp2, shift), + descale(tmp12 + tmp1, shift), + descale(tmp13 + tmp0, shift), + descale(tmp13 - tmp0, shift), + descale(tmp12 - tmp1, shift), + descale(tmp11 - tmp2, shift), + descale(tmp10 - tmp3, shift), + ]; + let mut k = 0_usize; + while k < 8 { + store_i32( + output, + (base as u64) + k as u64, + clamp_level_shift(values[k]), + ); + k += 1; + } + row += 1; + } +} + +#[inline(always)] +fn sample_at(samples: *const i32, block_cols: i32, x: i32, y: i32) -> i32 { + let block_idx = (y >> 3) * block_cols + (x >> 3); + let local_idx = (y & 7) * 8 + (x & 7); + load_i32(samples, block_idx as u64 * 64 + local_idx as u64) +} + +#[inline(always)] +fn vertical_high(samples: *const i32, block_cols: i32, height: i32, x: i32, high_idx: i32) -> i32 { + let odd_idx = high_idx * 2 + 1; + let current = sample_at(samples, block_cols, x, odd_idx); + let left = sample_at(samples, block_cols, x, odd_idx - 1); + if height % 2 == 0 && odd_idx + 1 == height { + return current - left; + } + let right_idx = if odd_idx + 1 < height { + odd_idx + 1 + } else { + height - 1 + }; + let right = sample_at(samples, block_cols, x, right_idx); + current - floor_div_pos(left + right, 2) +} + +#[inline(always)] +fn vertical_low(samples: *const i32, block_cols: i32, height: i32, x: i32, low_idx: i32) -> i32 { + let even_idx = low_idx * 2; + let current = sample_at(samples, block_cols, x, even_idx); + if height < 2 { + return current; + } + if height % 2 == 0 { + let right = vertical_high(samples, block_cols, height, x, low_idx); + if low_idx == 0 { + return current + floor_div_pos(right + 1, 2); + } + let left = vertical_high(samples, block_cols, height, x, low_idx - 1); + return current + floor_div_pos(left + right + 2, 4); + } + let high_len = height / 2; + if high_len == 0 { + return current; + } + let left = vertical_high( + samples, + block_cols, + height, + x, + if low_idx > 0 { low_idx - 1 } else { 0 }, + ); + let right = if low_idx < high_len { + vertical_high(samples, block_cols, height, x, low_idx) + } else { + left + }; + current + floor_div_pos(left + right + 2, 4) +} + +#[inline(always)] +fn reversible_lift_row(row: *mut i32, n: i32) { + if n < 2 { + return; + } + if n % 2 == 0 { + let mut i = 1_i32; + while i < n - 1 { + let value = load_i32(row.cast_const(), i as u64) + - floor_div_pos( + load_i32(row.cast_const(), (i - 1) as u64) + + load_i32(row.cast_const(), (i + 1) as u64), + 2, + ); + store_i32(row, i as u64, value); + i += 2; + } + let last = + load_i32(row.cast_const(), (n - 1) as u64) - load_i32(row.cast_const(), (n - 2) as u64); + store_i32(row, (n - 1) as u64, last); + store_i32( + row, + 0, + load_i32(row.cast_const(), 0) + floor_div_pos(load_i32(row.cast_const(), 1) + 1, 2), + ); + let mut i = 2_i32; + while i < n { + let value = load_i32(row.cast_const(), i as u64) + + floor_div_pos( + load_i32(row.cast_const(), (i - 1) as u64) + + load_i32(row.cast_const(), (i + 1) as u64) + + 2, + 4, + ); + store_i32(row, i as u64, value); + i += 2; + } + return; + } + + let last_even = n - 1; + let mut i = 1_i32; + while i < n { + let right = if i + 1 < n { + load_i32(row.cast_const(), (i + 1) as u64) + } else { + load_i32(row.cast_const(), last_even as u64) + }; + let value = load_i32(row.cast_const(), i as u64) + - floor_div_pos(load_i32(row.cast_const(), (i - 1) as u64) + right, 2); + store_i32(row, i as u64, value); + i += 2; + } + let mut i = 0_i32; + while i < n { + let left = if i > 0 { + load_i32(row.cast_const(), (i - 1) as u64) + } else { + load_i32(row.cast_const(), 1) + }; + let right = if i + 1 < n { + load_i32(row.cast_const(), (i + 1) as u64) + } else { + left + }; + let value = load_i32(row.cast_const(), i as u64) + floor_div_pos(left + right + 2, 4); + store_i32(row, i as u64, value); + i += 2; + } +} + +const DWT97_ALPHA: f32 = -1.586_134_3; +const DWT97_BETA: f32 = -0.052_980_117; +const DWT97_GAMMA: f32 = 0.882_911_1; +const DWT97_DELTA: f32 = 0.443_506_87; +const DWT97_KAPPA: f32 = 1.230_174_1; +const DWT97_INV_KAPPA: f32 = 1.0 / DWT97_KAPPA; +const DWT97_ROW_LIFT_MAX_WIDTH: usize = 1024; +const DWT97_ROW_LIFT_ROWS_PER_BLOCK: usize = 4; +const DWT97_ROW_LIFT_SHARED_SAMPLES: usize = + DWT97_ROW_LIFT_MAX_WIDTH * DWT97_ROW_LIFT_ROWS_PER_BLOCK; + +const DWT97_IDCT8_BASIS: [f32; 64] = [ + 0.353_553_38, + 0.490_392_65, + 0.461_939_75, + 0.415_734_8, + 0.353_553_38, + 0.277_785_12, + 0.191_341_71, + 0.097_545_16, + 0.353_553_38, + 0.415_734_8, + 0.191_341_71, + -0.097_545_16, + -0.353_553_38, + -0.490_392_65, + -0.461_939_75, + -0.277_785_12, + 0.353_553_38, + 0.277_785_12, + -0.191_341_71, + -0.490_392_65, + -0.353_553_38, + 0.097_545_16, + 0.461_939_75, + 0.415_734_8, + 0.353_553_38, + 0.097_545_16, + -0.461_939_75, + -0.277_785_12, + 0.353_553_38, + 0.415_734_8, + -0.191_341_71, + -0.490_392_65, + 0.353_553_38, + -0.097_545_16, + -0.461_939_75, + 0.277_785_12, + 0.353_553_38, + -0.415_734_8, + -0.191_341_71, + 0.490_392_65, + 0.353_553_38, + -0.277_785_12, + -0.191_341_71, + 0.490_392_65, + -0.353_553_38, + -0.097_545_16, + 0.461_939_75, + -0.415_734_8, + 0.353_553_38, + -0.415_734_8, + 0.191_341_71, + 0.097_545_16, + -0.353_553_38, + 0.490_392_65, + -0.461_939_75, + 0.277_785_12, + 0.353_553_38, + -0.490_392_65, + 0.461_939_75, + -0.415_734_8, + 0.353_553_38, + -0.277_785_12, + 0.191_341_71, + -0.097_545_16, +]; + +#[inline(always)] +fn idct8_basis(sample_idx: i32, freq: i32) -> f32 { + DWT97_IDCT8_BASIS[(sample_idx * 8 + freq) as usize] +} + +#[inline(always)] +fn idct8x8_sample(block: *const f32, local_x: i32, local_y: i32) -> f32 { + let mut sample = 0.0_f32; + let mut freq_y = 0_i32; + while freq_y < 8 { + let y_basis = idct8_basis(local_y, freq_y); + let mut freq_x = 0_i32; + while freq_x < 8 { + sample += load_f32(block, (freq_y * 8 + freq_x) as u64) + * y_basis + * idct8_basis(local_x, freq_x); + freq_x += 1; + } + freq_y += 1; + } + sample +} + +#[inline(always)] +fn idct8x8_sample_i16(block: *const i16, local_x: i32, local_y: i32) -> f32 { + let mut sample = 0.0_f32; + let mut freq_y = 0_i32; + while freq_y < 8 { + let y_basis = idct8_basis(local_y, freq_y); + let mut freq_x = 0_i32; + while freq_x < 8 { + sample += load_i16(block, (freq_y * 8 + freq_x) as u64) as f32 + * y_basis + * idct8_basis(local_x, freq_x); + freq_x += 1; + } + freq_y += 1; + } + sample +} + +#[inline(always)] +fn forward_lift_97(data: *mut f32, n: i32, stride: i32) { + if n < 2 { + return; + } + let last_even = if n % 2 == 0 { n - 2 } else { n - 1 }; + + let mut i = 1_i32; + while i < n { + let left = load_f32(data.cast_const(), ((i - 1) * stride) as u64); + let right = if i + 1 < n { + load_f32(data.cast_const(), ((i + 1) * stride) as u64) + } else { + load_f32(data.cast_const(), (last_even * stride) as u64) + }; + let value = load_f32(data.cast_const(), (i * stride) as u64) + DWT97_ALPHA * (left + right); + store_f32(data, (i * stride) as u64, value); + i += 2; + } + + let mut i = 0_i32; + while i < n { + let left = if i > 0 { + load_f32(data.cast_const(), ((i - 1) * stride) as u64) + } else { + load_f32(data.cast_const(), stride as u64) + }; + let right = if i + 1 < n { + load_f32(data.cast_const(), ((i + 1) * stride) as u64) + } else { + left + }; + let value = load_f32(data.cast_const(), (i * stride) as u64) + DWT97_BETA * (left + right); + store_f32(data, (i * stride) as u64, value); + i += 2; + } + + let mut i = 1_i32; + while i < n { + let left = load_f32(data.cast_const(), ((i - 1) * stride) as u64); + let right = if i + 1 < n { + load_f32(data.cast_const(), ((i + 1) * stride) as u64) + } else { + load_f32(data.cast_const(), (last_even * stride) as u64) + }; + let value = load_f32(data.cast_const(), (i * stride) as u64) + DWT97_GAMMA * (left + right); + store_f32(data, (i * stride) as u64, value); + i += 2; + } + + let mut i = 0_i32; + while i < n { + let left = if i > 0 { + load_f32(data.cast_const(), ((i - 1) * stride) as u64) + } else { + load_f32(data.cast_const(), stride as u64) + }; + let right = if i + 1 < n { + load_f32(data.cast_const(), ((i + 1) * stride) as u64) + } else { + left + }; + let value = load_f32(data.cast_const(), (i * stride) as u64) + DWT97_DELTA * (left + right); + store_f32(data, (i * stride) as u64, value); + i += 2; + } + + let mut i = 0_i32; + while i < n { + let value = load_f32(data.cast_const(), (i * stride) as u64) * DWT97_INV_KAPPA; + store_f32(data, (i * stride) as u64, value); + i += 2; + } + + let mut i = 1_i32; + while i < n { + let value = load_f32(data.cast_const(), (i * stride) as u64) * DWT97_KAPPA; + store_f32(data, (i * stride) as u64, value); + i += 2; + } +} + +#[inline(always)] +fn floor_f32(value: f32) -> f32 { + let truncated = value as i32 as f32; + if truncated > value { + truncated - 1.0 + } else { + truncated + } +} + +#[inline(always)] +fn abs_f32(value: f32) -> f32 { + if value < 0.0 { -value } else { value } +} + +#[inline(always)] +fn min_i32(a: i32, b: i32) -> i32 { + if a < b { a } else { b } +} + +#[inline(always)] +fn quantize_dwt97_deadzone(value: f32, inv_delta: f32) -> i32 { + let sign = if value < 0.0 { -1 } else { 1 }; + sign * floor_f32(abs_f32(value) * inv_delta) as i32 +} + +#[inline(always)] +fn dwt97_codeblock_major_offset( + x: i32, + y: i32, + width: i32, + height: i32, + cb_width: i32, + cb_height: i32, +) -> u64 { + if cb_width == 64 && cb_height == 64 { + let cbx = x >> 6; + let cby = y >> 6; + let local_x = x & 63; + let local_y = y & 63; + let block_width = min_i32(64, width - (cbx << 6)); + let block_height = min_i32(64, height - (cby << 6)); + return (cby as u64) * 64 * width as u64 + + (cbx as u64) * 64 * block_height as u64 + + (local_y as u64) * block_width as u64 + + local_x as u64; + } + let cbx = x / cb_width; + let cby = y / cb_height; + let local_x = x - cbx * cb_width; + let local_y = y - cby * cb_height; + let block_width = min_i32(cb_width, width - cbx * cb_width); + let block_height = min_i32(cb_height, height - cby * cb_height); + (cby as u64) * cb_height as u64 * width as u64 + + (cbx as u64) * cb_width as u64 * block_height as u64 + + (local_y as u64) * block_width as u64 + + local_x as u64 +} + +#[inline(always)] +fn shared_row_index(row_lane: i32, x: i32) -> u64 { + row_lane as u64 * DWT97_ROW_LIFT_MAX_WIDTH as u64 + x as u64 +} + +#[cuda_module] +mod kernels { + use super::*; + + #[kernel] + pub unsafe fn transcode_reversible53_idct( + blocks: *const i16, + samples: *mut i32, + block_count: u32, + ) { + let idx = thread::index_1d().get() as u32; + if idx >= block_count { + return; + } + idct_islow_signed(unsafe { blocks.add(idx as usize * 64) }, unsafe { + samples.add(idx as usize * 64) + }); + } + + #[kernel] + pub unsafe fn transcode_reversible53_vertical_low( + samples: *const i32, + block_cols: i32, + width: i32, + height: i32, + v_low: *mut i32, + low_height: i32, + ) { + let x = thread::index_2d_col() as i32; + let yl = thread::index_2d_row() as i32; + if x >= width || yl >= low_height { + return; + } + store_i32( + v_low, + (yl * width + x) as u64, + vertical_low(samples, block_cols, height, x, yl), + ); + } + + #[kernel] + pub unsafe fn transcode_reversible53_vertical_high( + samples: *const i32, + block_cols: i32, + width: i32, + height: i32, + v_high: *mut i32, + high_height: i32, + ) { + let x = thread::index_2d_col() as i32; + let yh = thread::index_2d_row() as i32; + if x >= width || yh >= high_height { + return; + } + store_i32( + v_high, + (yh * width + x) as u64, + vertical_high(samples, block_cols, height, x, yh), + ); + } + + #[kernel] + pub unsafe fn transcode_reversible53_horizontal_low( + v_low: *mut i32, + width: i32, + low_height: i32, + low_width: i32, + high_width: i32, + ll: *mut i32, + hl: *mut i32, + ) { + let yl = thread::index_1d().get() as i32; + if yl >= low_height { + return; + } + let row = offset_i32_mut(v_low, (yl * width) as u64); + reversible_lift_row(row, width); + let mut i = 0_i32; + while i < low_width { + store_i32( + ll, + (yl * low_width + i) as u64, + load_i32(row.cast_const(), (i * 2) as u64), + ); + i += 1; + } + let mut i = 0_i32; + while i < high_width { + store_i32( + hl, + (yl * high_width + i) as u64, + load_i32(row.cast_const(), (i * 2 + 1) as u64), + ); + i += 1; + } + } + + #[kernel] + pub unsafe fn transcode_reversible53_horizontal_high( + v_high: *mut i32, + width: i32, + high_height: i32, + low_width: i32, + high_width: i32, + lh: *mut i32, + hh: *mut i32, + ) { + let yh = thread::index_1d().get() as i32; + if yh >= high_height { + return; + } + let row = offset_i32_mut(v_high, (yh * width) as u64); + reversible_lift_row(row, width); + let mut i = 0_i32; + while i < low_width { + store_i32( + lh, + (yh * low_width + i) as u64, + load_i32(row.cast_const(), (i * 2) as u64), + ); + i += 1; + } + let mut i = 0_i32; + while i < high_width { + store_i32( + hh, + (yh * high_width + i) as u64, + load_i32(row.cast_const(), (i * 2 + 1) as u64), + ); + i += 1; + } + } + + #[kernel] + pub unsafe fn transcode_dwt97_idct( + blocks: *const f32, + block_cols: i32, + width: i32, + height: i32, + spatial: *mut f32, + ) { + let x = thread::index_2d_col() as i32; + let y = thread::index_2d_row() as i32; + if x >= width || y >= height { + return; + } + let block_idx = (y >> 3) * block_cols + (x >> 3); + let block = unsafe { blocks.add(block_idx as usize * 64) }; + store_f32( + spatial, + (y * width + x) as u64, + idct8x8_sample(block, x & 7, y & 7), + ); + } + + #[kernel] + pub unsafe fn transcode_dwt97_row_lift( + spatial: *mut f32, + width: i32, + height: i32, + low_width: i32, + high_width: i32, + row_low: *mut f32, + row_high: *mut f32, + ) { + let y = thread::index_1d().get() as i32; + if y >= height { + return; + } + let row = offset_f32_mut(spatial, (y * width) as u64); + forward_lift_97(row, width, 1); + let mut i = 0_i32; + while i < low_width { + store_f32( + row_low, + (y * low_width + i) as u64, + load_f32(row.cast_const(), (i * 2) as u64), + ); + i += 1; + } + let mut i = 0_i32; + while i < high_width { + store_f32( + row_high, + (y * high_width + i) as u64, + load_f32(row.cast_const(), (i * 2 + 1) as u64), + ); + i += 1; + } + } + + #[kernel] + pub unsafe fn transcode_dwt97_column_lift( + rows: *mut f32, + band_width: i32, + height: i32, + low_out: *mut f32, + high_out: *mut f32, + ) { + let x = thread::index_1d().get() as i32; + if x >= band_width { + return; + } + forward_lift_97(offset_f32_mut(rows, x as u64), height, band_width); + let mut i = 0_i32; + while i < height { + let value = load_f32(rows.cast_const(), (i * band_width + x) as u64); + if i & 1 == 0 { + store_f32(low_out, ((i / 2) * band_width + x) as u64, value); + } else { + store_f32(high_out, ((i / 2) * band_width + x) as u64, value); + } + i += 1; + } + } + + #[kernel] + pub unsafe fn transcode_dwt97_idct_batch( + blocks: *const f32, + block_cols: i32, + width: i32, + height: i32, + blocks_per_item: i32, + spatial: *mut f32, + ) { + let x = thread::index_2d_col() as i32; + let y = thread::index_2d_row() as i32; + let item = thread::blockIdx_z() as u64; + if x >= width || y >= height { + return; + } + let item_blocks = unsafe { blocks.add((item * blocks_per_item as u64 * 64) as usize) }; + let block_idx = (y >> 3) * block_cols + (x >> 3); + let block = unsafe { item_blocks.add(block_idx as usize * 64) }; + store_f32( + spatial, + (item * height as u64 + y as u64) * width as u64 + x as u64, + idct8x8_sample(block, x & 7, y & 7), + ); + } + + #[kernel] + pub unsafe fn transcode_dwt97_idct_i16_batch( + blocks: *const i16, + block_cols: i32, + width: i32, + height: i32, + blocks_per_item: i32, + spatial: *mut f32, + ) { + let x = thread::index_2d_col() as i32; + let y = thread::index_2d_row() as i32; + let item = thread::blockIdx_z() as u64; + if x >= width || y >= height { + return; + } + let item_blocks = unsafe { blocks.add((item * blocks_per_item as u64 * 64) as usize) }; + let block_idx = (y >> 3) * block_cols + (x >> 3); + let block = unsafe { item_blocks.add(block_idx as usize * 64) }; + store_f32( + spatial, + (item * height as u64 + y as u64) * width as u64 + x as u64, + idct8x8_sample_i16(block, x & 7, y & 7), + ); + } + + #[kernel] + pub unsafe fn transcode_dwt97_row_lift_batch( + spatial: *mut f32, + width: i32, + height: i32, + low_width: i32, + high_width: i32, + row_low: *mut f32, + row_high: *mut f32, + ) { + let y = thread::blockIdx_x() as i32 * thread::blockDim_x() as i32 + + thread::threadIdx_x() as i32; + let item = thread::blockIdx_y() as u64; + if y >= height { + return; + } + let item_spatial = offset_f32_mut(spatial, item * width as u64 * height as u64); + let item_row_low = offset_f32_mut(row_low, item * height as u64 * low_width as u64); + let item_row_high = offset_f32_mut(row_high, item * height as u64 * high_width as u64); + let row = offset_f32_mut(item_spatial, y as u64 * width as u64); + forward_lift_97(row, width, 1); + let mut i = 0_i32; + while i < low_width { + store_f32( + item_row_low, + (y * low_width + i) as u64, + load_f32(row.cast_const(), (i * 2) as u64), + ); + i += 1; + } + let mut i = 0_i32; + while i < high_width { + store_f32( + item_row_high, + (y * high_width + i) as u64, + load_f32(row.cast_const(), (i * 2 + 1) as u64), + ); + i += 1; + } + } + + #[kernel] + pub unsafe fn transcode_dwt97_row_lift_batch_coop( + spatial: *const f32, + width: i32, + height: i32, + low_width: i32, + high_width: i32, + row_low: *mut f32, + row_high: *mut f32, + ) { + static mut ROWS: SharedArray = SharedArray::UNINIT; + + let rows = unsafe { ROWS.as_mut_ptr() }; + let row_lane = thread::threadIdx_y() as i32; + let tid = thread::threadIdx_x() as i32; + let block_step = thread::blockDim_x() as i32; + let y = thread::blockIdx_x() as i32 * DWT97_ROW_LIFT_ROWS_PER_BLOCK as i32 + row_lane; + let item = thread::blockIdx_y() as u64; + let valid = y < height && width <= DWT97_ROW_LIFT_MAX_WIDTH as i32; + + if valid { + let item_spatial = + unsafe { spatial.add((item * width as u64 * height as u64) as usize) }; + let source = unsafe { item_spatial.add((y as u64 * width as u64) as usize) }; + let mut i = tid; + while i < width { + store_f32( + rows, + shared_row_index(row_lane, i), + load_f32(source, i as u64), + ); + i += block_step; + } + } + thread::sync_threads(); + + if width >= 2 && width <= DWT97_ROW_LIFT_MAX_WIDTH as i32 { + if valid { + let last_even = if width % 2 == 0 { width - 2 } else { width - 1 }; + let mut i = tid * 2 + 1; + while i < width { + let left = load_f32(rows.cast_const(), shared_row_index(row_lane, i - 1)); + let right = if i + 1 < width { + load_f32(rows.cast_const(), shared_row_index(row_lane, i + 1)) + } else { + load_f32(rows.cast_const(), shared_row_index(row_lane, last_even)) + }; + let value = load_f32(rows.cast_const(), shared_row_index(row_lane, i)) + + DWT97_ALPHA * (left + right); + store_f32(rows, shared_row_index(row_lane, i), value); + i += block_step * 2; + } + } + thread::sync_threads(); + + if valid { + let mut i = tid * 2; + while i < width { + let left = if i > 0 { + load_f32(rows.cast_const(), shared_row_index(row_lane, i - 1)) + } else { + load_f32(rows.cast_const(), shared_row_index(row_lane, 1)) + }; + let right = if i + 1 < width { + load_f32(rows.cast_const(), shared_row_index(row_lane, i + 1)) + } else { + left + }; + let value = load_f32(rows.cast_const(), shared_row_index(row_lane, i)) + + DWT97_BETA * (left + right); + store_f32(rows, shared_row_index(row_lane, i), value); + i += block_step * 2; + } + } + thread::sync_threads(); + + if valid { + let last_even = if width % 2 == 0 { width - 2 } else { width - 1 }; + let mut i = tid * 2 + 1; + while i < width { + let left = load_f32(rows.cast_const(), shared_row_index(row_lane, i - 1)); + let right = if i + 1 < width { + load_f32(rows.cast_const(), shared_row_index(row_lane, i + 1)) + } else { + load_f32(rows.cast_const(), shared_row_index(row_lane, last_even)) + }; + let value = load_f32(rows.cast_const(), shared_row_index(row_lane, i)) + + DWT97_GAMMA * (left + right); + store_f32(rows, shared_row_index(row_lane, i), value); + i += block_step * 2; + } + } + thread::sync_threads(); + + if valid { + let mut i = tid * 2; + while i < width { + let left = if i > 0 { + load_f32(rows.cast_const(), shared_row_index(row_lane, i - 1)) + } else { + load_f32(rows.cast_const(), shared_row_index(row_lane, 1)) + }; + let right = if i + 1 < width { + load_f32(rows.cast_const(), shared_row_index(row_lane, i + 1)) + } else { + left + }; + let value = load_f32(rows.cast_const(), shared_row_index(row_lane, i)) + + DWT97_DELTA * (left + right); + store_f32(rows, shared_row_index(row_lane, i), value); + i += block_step * 2; + } + } + thread::sync_threads(); + + if valid { + let mut i = tid * 2; + while i < width { + let value = load_f32(rows.cast_const(), shared_row_index(row_lane, i)) + * DWT97_INV_KAPPA; + store_f32(rows, shared_row_index(row_lane, i), value); + i += block_step * 2; + } + let mut i = tid * 2 + 1; + while i < width { + let value = + load_f32(rows.cast_const(), shared_row_index(row_lane, i)) * DWT97_KAPPA; + store_f32(rows, shared_row_index(row_lane, i), value); + i += block_step * 2; + } + } + thread::sync_threads(); + } + + if valid { + let item_row_low = offset_f32_mut(row_low, item * height as u64 * low_width as u64); + let item_row_high = offset_f32_mut(row_high, item * height as u64 * high_width as u64); + let mut i = tid; + while i < low_width { + store_f32( + item_row_low, + (y * low_width + i) as u64, + load_f32(rows.cast_const(), shared_row_index(row_lane, i * 2)), + ); + i += block_step; + } + let mut i = tid; + while i < high_width { + store_f32( + item_row_high, + (y * high_width + i) as u64, + load_f32(rows.cast_const(), shared_row_index(row_lane, i * 2 + 1)), + ); + i += block_step; + } + } + } + + #[kernel] + pub unsafe fn transcode_dwt97_column_lift_batch( + rows: *mut f32, + band_width: i32, + height: i32, + low_height: i32, + high_height: i32, + low_out: *mut f32, + high_out: *mut f32, + ) { + let x = thread::blockIdx_x() as i32 * thread::blockDim_x() as i32 + + thread::threadIdx_x() as i32; + let item = thread::blockIdx_y() as u64; + if x >= band_width { + return; + } + let item_rows = offset_f32_mut(rows, item * height as u64 * band_width as u64); + let item_low = offset_f32_mut(low_out, item * low_height as u64 * band_width as u64); + let item_high = offset_f32_mut(high_out, item * high_height as u64 * band_width as u64); + forward_lift_97(offset_f32_mut(item_rows, x as u64), height, band_width); + let mut i = 0_i32; + while i < height { + let value = load_f32(item_rows.cast_const(), (i * band_width + x) as u64); + if i & 1 == 0 { + store_f32(item_low, ((i / 2) * band_width + x) as u64, value); + } else { + store_f32(item_high, ((i / 2) * band_width + x) as u64, value); + } + i += 1; + } + } + + #[kernel] + pub unsafe fn transcode_dwt97_quantize_codeblocks( + band: *const f32, + output: *mut i32, + width: i32, + height: i32, + cb_width: i32, + cb_height: i32, + inv_delta: f32, + ) { + let x = thread::index_2d_col() as i32; + let y = thread::index_2d_row() as i32; + let item = thread::blockIdx_z() as u64; + if x >= width || y >= height { + return; + } + let item_stride = width as u64 * height as u64; + let value = load_f32( + band, + item * item_stride + y as u64 * width as u64 + x as u64, + ); + let offset = dwt97_codeblock_major_offset(x, y, width, height, cb_width, cb_height); + store_i32( + output, + item * item_stride + offset, + quantize_dwt97_deadzone(value, inv_delta), + ); + } + + #[kernel] + #[allow(clippy::too_many_arguments)] + pub unsafe fn transcode_dwt97_column_lift_quantize_codeblocks_batch( + rows: *mut f32, + band_width: i32, + height: i32, + low_height: i32, + high_height: i32, + low_out: *mut i32, + high_out: *mut i32, + cb_width: i32, + cb_height: i32, + inv_delta_low: f32, + inv_delta_high: f32, + ) { + let x = thread::blockIdx_x() as i32 * thread::blockDim_x() as i32 + + thread::threadIdx_x() as i32; + let item = thread::blockIdx_y() as u64; + if x >= band_width { + return; + } + let item_rows = offset_f32_mut(rows, item * height as u64 * band_width as u64); + let item_low = offset_i32_mut(low_out, item * low_height as u64 * band_width as u64); + let item_high = offset_i32_mut(high_out, item * high_height as u64 * band_width as u64); + + forward_lift_97(offset_f32_mut(item_rows, x as u64), height, band_width); + let mut i = 0_i32; + while i < height { + let value = load_f32(item_rows.cast_const(), (i * band_width + x) as u64); + if i & 1 == 0 { + let y = i / 2; + let offset = + dwt97_codeblock_major_offset(x, y, band_width, low_height, cb_width, cb_height); + store_i32( + item_low, + offset, + quantize_dwt97_deadzone(value, inv_delta_low), + ); + } else { + let y = i / 2; + let offset = dwt97_codeblock_major_offset( + x, + y, + band_width, + high_height, + cb_width, + cb_height, + ); + store_i32( + item_high, + offset, + quantize_dwt97_deadzone(value, inv_delta_high), + ); + } + i += 1; + } + } +} + +fn main() {} diff --git a/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/src/main.rs b/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/src/main.rs new file mode 100644 index 00000000..f328e4d9 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/cuda_oxide_transcode/src/main.rs @@ -0,0 +1 @@ +fn main() {} diff --git a/crates/j2k-cuda-runtime/src/driver.rs b/crates/j2k-cuda-runtime/src/driver.rs new file mode 100644 index 00000000..1f0bd0c9 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/driver.rs @@ -0,0 +1,322 @@ +use crate::{build_flags::CUDA_SUCCESS, error::CudaError}; +use libloading::Library; +#[cfg(feature = "cuda-profiling")] +use std::sync::OnceLock; +use std::{ + ffi::{c_void, CStr}, + os::raw::{c_char, c_int, c_uint}, +}; + +pub(crate) type CuResult = c_int; + +pub(crate) type CuDevice = c_int; + +pub(crate) type CuContext = *mut c_void; + +pub(crate) type CuDevicePtr = u64; + +pub(crate) type CuModule = *mut c_void; + +pub(crate) type CuFunction = *mut c_void; + +pub(crate) type CuStream = *mut c_void; + +pub(crate) type CuEvent = *mut c_void; + +pub(crate) type CuInit = unsafe extern "C" fn(c_uint) -> CuResult; + +pub(crate) type CuDeviceGetCount = unsafe extern "C" fn(*mut c_int) -> CuResult; + +pub(crate) type CuDeviceGet = unsafe extern "C" fn(*mut CuDevice, c_int) -> CuResult; + +pub(crate) type CuCtxCreate = unsafe extern "C" fn(*mut CuContext, c_uint, CuDevice) -> CuResult; + +pub(crate) type CuCtxDestroy = unsafe extern "C" fn(CuContext) -> CuResult; + +pub(crate) type CuCtxSetCurrent = unsafe extern "C" fn(CuContext) -> CuResult; + +pub(crate) type CuMemAlloc = unsafe extern "C" fn(*mut CuDevicePtr, usize) -> CuResult; + +pub(crate) type CuMemFree = unsafe extern "C" fn(CuDevicePtr) -> CuResult; + +pub(crate) type CuMemHostAlloc = unsafe extern "C" fn(*mut *mut c_void, usize, c_uint) -> CuResult; + +pub(crate) type CuMemFreeHost = unsafe extern "C" fn(*mut c_void) -> CuResult; + +pub(crate) type CuMemcpyHtoD = unsafe extern "C" fn(CuDevicePtr, *const c_void, usize) -> CuResult; + +pub(crate) type CuMemcpyDtoH = unsafe extern "C" fn(*mut c_void, CuDevicePtr, usize) -> CuResult; + +pub(crate) type CuMemsetD32 = unsafe extern "C" fn(CuDevicePtr, c_uint, usize) -> CuResult; + +pub(crate) type CuGetErrorName = unsafe extern "C" fn(CuResult, *mut *const c_char) -> CuResult; + +pub(crate) type CuModuleLoadData = unsafe extern "C" fn(*mut CuModule, *const c_void) -> CuResult; + +pub(crate) type CuModuleUnload = unsafe extern "C" fn(CuModule) -> CuResult; + +pub(crate) type CuModuleGetFunction = + unsafe extern "C" fn(*mut CuFunction, CuModule, *const c_char) -> CuResult; + +pub(crate) type CuLaunchKernel = unsafe extern "C" fn( + CuFunction, + c_uint, + c_uint, + c_uint, + c_uint, + c_uint, + c_uint, + c_uint, + *mut c_void, + *mut *mut c_void, + *mut *mut c_void, +) -> CuResult; + +pub(crate) type CuCtxSynchronize = unsafe extern "C" fn() -> CuResult; + +pub(crate) type CuStreamCreate = unsafe extern "C" fn(*mut CuStream, c_uint) -> CuResult; + +pub(crate) type CuStreamDestroy = unsafe extern "C" fn(CuStream) -> CuResult; + +pub(crate) type CuStreamSynchronize = unsafe extern "C" fn(CuStream) -> CuResult; + +pub(crate) type CuEventCreate = unsafe extern "C" fn(*mut CuEvent, c_uint) -> CuResult; + +pub(crate) type CuEventDestroy = unsafe extern "C" fn(CuEvent) -> CuResult; + +pub(crate) type CuEventRecord = unsafe extern "C" fn(CuEvent, CuStream) -> CuResult; + +pub(crate) type CuEventSynchronize = unsafe extern "C" fn(CuEvent) -> CuResult; + +pub(crate) type CuEventElapsedTime = unsafe extern "C" fn(*mut f32, CuEvent, CuEvent) -> CuResult; + +#[cfg(feature = "cuda-profiling")] +pub(crate) type NvtxRangePushA = unsafe extern "C" fn(*const c_char) -> c_int; + +#[cfg(feature = "cuda-profiling")] +pub(crate) type NvtxRangePop = unsafe extern "C" fn() -> c_int; + +pub(crate) struct Driver { + pub(crate) _library: Library, + pub(crate) cu_init: CuInit, + pub(crate) cu_device_get_count: CuDeviceGetCount, + pub(crate) cu_device_get: CuDeviceGet, + pub(crate) cu_ctx_create: CuCtxCreate, + pub(crate) cu_ctx_destroy: CuCtxDestroy, + pub(crate) cu_ctx_set_current: CuCtxSetCurrent, + pub(crate) cu_mem_alloc: CuMemAlloc, + pub(crate) cu_mem_free: CuMemFree, + pub(crate) cu_mem_host_alloc: CuMemHostAlloc, + pub(crate) cu_mem_free_host: CuMemFreeHost, + pub(crate) cu_memcpy_htod: CuMemcpyHtoD, + pub(crate) cu_memcpy_dtoh: CuMemcpyDtoH, + pub(crate) cu_memset_d32: CuMemsetD32, + pub(crate) cu_get_error_name: CuGetErrorName, + pub(crate) cu_module_load_data: CuModuleLoadData, + pub(crate) cu_module_unload: CuModuleUnload, + pub(crate) cu_module_get_function: CuModuleGetFunction, + pub(crate) cu_launch_kernel: CuLaunchKernel, + pub(crate) cu_ctx_synchronize: CuCtxSynchronize, + pub(crate) cu_stream_create: CuStreamCreate, + pub(crate) cu_stream_destroy: CuStreamDestroy, + pub(crate) cu_stream_synchronize: CuStreamSynchronize, + pub(crate) cu_event_create: CuEventCreate, + pub(crate) cu_event_destroy: CuEventDestroy, + pub(crate) cu_event_record: CuEventRecord, + pub(crate) cu_event_synchronize: CuEventSynchronize, + pub(crate) cu_event_elapsed_time: CuEventElapsedTime, +} + +impl Driver { + pub(crate) fn load() -> Result { + #[cfg(target_os = "linux")] + const LIBRARY_CANDIDATES: &[&str] = &["libcuda.so.1", "libcuda.so"]; + #[cfg(target_os = "windows")] + const LIBRARY_CANDIDATES: &[&str] = &["nvcuda.dll"]; + #[cfg(not(any(target_os = "linux", target_os = "windows")))] + const LIBRARY_CANDIDATES: &[&str] = &[]; + + let mut last_error = None; + for candidate in LIBRARY_CANDIDATES { + // SAFETY: Loading the CUDA driver library is required before symbol + // lookup. The resulting Library is owned by Driver and outlives all + // copied function pointers. + match unsafe { Library::new(*candidate) } { + Ok(library) => return Self::from_library(library), + Err(error) => last_error = Some(error.to_string()), + } + } + + Err(CudaError::Unavailable { + message: last_error.unwrap_or_else(|| "unsupported CUDA host platform".to_string()), + }) + } + + pub(crate) fn from_library(library: Library) -> Result { + Ok(Self { + cu_init: load_symbol(&library, b"cuInit\0")?, + cu_device_get_count: load_symbol(&library, b"cuDeviceGetCount\0")?, + cu_device_get: load_symbol(&library, b"cuDeviceGet\0")?, + cu_ctx_create: load_symbol(&library, b"cuCtxCreate_v2\0")?, + cu_ctx_destroy: load_symbol(&library, b"cuCtxDestroy_v2\0")?, + cu_ctx_set_current: load_symbol(&library, b"cuCtxSetCurrent\0")?, + cu_mem_alloc: load_symbol(&library, b"cuMemAlloc_v2\0")?, + cu_mem_free: load_symbol(&library, b"cuMemFree_v2\0")?, + cu_mem_host_alloc: load_symbol(&library, b"cuMemHostAlloc\0")?, + cu_mem_free_host: load_symbol(&library, b"cuMemFreeHost\0")?, + cu_memcpy_htod: load_symbol(&library, b"cuMemcpyHtoD_v2\0")?, + cu_memcpy_dtoh: load_symbol(&library, b"cuMemcpyDtoH_v2\0")?, + cu_memset_d32: load_symbol(&library, b"cuMemsetD32_v2\0")?, + cu_get_error_name: load_symbol(&library, b"cuGetErrorName\0")?, + cu_module_load_data: load_symbol(&library, b"cuModuleLoadData\0")?, + cu_module_unload: load_symbol(&library, b"cuModuleUnload\0")?, + cu_module_get_function: load_symbol(&library, b"cuModuleGetFunction\0")?, + cu_launch_kernel: load_symbol(&library, b"cuLaunchKernel\0")?, + cu_ctx_synchronize: load_symbol(&library, b"cuCtxSynchronize\0")?, + cu_stream_create: load_symbol(&library, b"cuStreamCreate\0")?, + cu_stream_destroy: load_symbol(&library, b"cuStreamDestroy_v2\0")?, + cu_stream_synchronize: load_symbol(&library, b"cuStreamSynchronize\0")?, + cu_event_create: load_symbol(&library, b"cuEventCreate\0")?, + cu_event_destroy: load_symbol(&library, b"cuEventDestroy_v2\0")?, + cu_event_record: load_symbol(&library, b"cuEventRecord\0")?, + cu_event_synchronize: load_symbol(&library, b"cuEventSynchronize\0")?, + cu_event_elapsed_time: load_symbol(&library, b"cuEventElapsedTime\0")?, + _library: library, + }) + } + + pub(crate) fn check(&self, operation: &'static str, result: CuResult) -> Result<(), CudaError> { + if result == CUDA_SUCCESS { + Ok(()) + } else { + Err(CudaError::Driver { + operation, + code: result, + name: self.error_name(result), + }) + } + } + + pub(crate) fn error_name(&self, result: CuResult) -> String { + let mut name = std::ptr::null(); + // SAFETY: cuGetErrorName writes a borrowed static C string pointer for + // a CUDA result code. A failure here is non-critical for diagnostics. + let status = unsafe { (self.cu_get_error_name)(result, &raw mut name) }; + if status == CUDA_SUCCESS && !name.is_null() { + // SAFETY: CUDA returns a NUL-terminated static string on success. + let cstr = unsafe { CStr::from_ptr(name) }; + format!(" ({})", cstr.to_string_lossy()) + } else { + String::new() + } + } +} + +pub(crate) fn load_symbol(library: &Library, name: &'static [u8]) -> Result { + // SAFETY: Symbol names are NUL-terminated CUDA Driver API entry points. The + // symbol value is copied, and Driver keeps the Library alive. + unsafe { library.get::(name) } + .map(|symbol| *symbol) + .map_err(|error| CudaError::Unavailable { + message: format!( + "missing CUDA driver symbol {}: {error}", + String::from_utf8_lossy(name) + ), + }) +} + +pub(crate) struct CudaNvtxRange { + #[cfg(feature = "cuda-profiling")] + pub(crate) active: bool, +} + +impl CudaNvtxRange { + pub(crate) fn push(name: &str) -> Self { + #[cfg(feature = "cuda-profiling")] + { + let Some(api) = nvtx_api() else { + return Self { active: false }; + }; + let Ok(name) = std::ffi::CString::new(name) else { + return Self { active: false }; + }; + // SAFETY: `name` is a NUL-terminated C string that lives for the + // duration of the call. The NVTX function pointer is loaded from a + // live library stored in NvtxApi. + let depth = unsafe { (api.range_push_a)(name.as_ptr()) }; + Self { active: depth >= 0 } + } + #[cfg(not(feature = "cuda-profiling"))] + { + let _ = name; + Self {} + } + } +} + +impl Drop for CudaNvtxRange { + fn drop(&mut self) { + #[cfg(feature = "cuda-profiling")] + if self.active { + if let Some(api) = nvtx_api() { + // SAFETY: Matching pop for a successful nvtxRangePushA in this + // thread. NVTX returns a depth value that is not needed here. + let _ = unsafe { (api.range_pop)() }; + } + } + } +} + +#[cfg(feature = "cuda-profiling")] +pub(crate) struct NvtxApi { + pub(crate) _library: Library, + pub(crate) range_push_a: NvtxRangePushA, + pub(crate) range_pop: NvtxRangePop, +} + +#[cfg(feature = "cuda-profiling")] +pub(crate) fn nvtx_api() -> Option<&'static NvtxApi> { + static API: OnceLock> = OnceLock::new(); + API.get_or_init(load_optional_nvtx).as_ref() +} + +#[cfg(feature = "cuda-profiling")] +pub(crate) fn load_optional_nvtx() -> Option { + #[cfg(target_os = "linux")] + const LIBRARY_CANDIDATES: &[&str] = &["libnvToolsExt.so.1", "libnvToolsExt.so"]; + #[cfg(target_os = "windows")] + const LIBRARY_CANDIDATES: &[&str] = &["nvToolsExt64_1.dll", "nvToolsExt64_64_1.dll"]; + #[cfg(target_os = "macos")] + const LIBRARY_CANDIDATES: &[&str] = &["libnvToolsExt.dylib"]; + #[cfg(not(any(target_os = "linux", target_os = "windows", target_os = "macos")))] + const LIBRARY_CANDIDATES: &[&str] = &[]; + + for candidate in LIBRARY_CANDIDATES { + // SAFETY: This optional profiling path only copies immutable NVTX + // function pointers and stores the Library in NvtxApi for their + // lifetime. Failure to load simply disables NVTX ranges. + let Ok(library) = (unsafe { Library::new(*candidate) }) else { + continue; + }; + let Ok(range_push_a) = load_symbol(&library, b"nvtxRangePushA\0") else { + continue; + }; + let Ok(range_pop) = load_symbol(&library, b"nvtxRangePop\0") else { + continue; + }; + return Some(NvtxApi { + _library: library, + range_push_a, + range_pop, + }); + } + None +} + +// SAFETY: CUDA Driver API handles are process resources guarded by the driver. +// The struct stores copied function pointers and owns the loaded library. +unsafe impl Send for Driver {} + +// SAFETY: Driver entry points are immutable function pointers, and mutable CUDA +// state is always addressed through explicit CUDA context calls. +unsafe impl Sync for Driver {} diff --git a/crates/j2k-cuda-runtime/src/error.rs b/crates/j2k-cuda-runtime/src/error.rs new file mode 100644 index 00000000..52b9b17a --- /dev/null +++ b/crates/j2k-cuda-runtime/src/error.rs @@ -0,0 +1,84 @@ +use crate::driver::CuResult; + +/// Error returned by CUDA driver and J2K CUDA kernel helpers. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum CudaError { + /// CUDA driver library or device is unavailable. + #[error("CUDA driver is unavailable: {message}")] + Unavailable { + /// Human-readable availability failure. + message: String, + }, + /// CUDA Driver API call failed. + #[error("CUDA driver call {operation} failed with CUresult {code}{name}")] + Driver { + /// Driver operation name. + operation: &'static str, + /// Raw CUDA result code. + code: CuResult, + /// CUDA error name, when available. + name: String, + }, + /// Host output buffer is too small for a device download. + #[error("CUDA copy output buffer too small: required {required}, have {have}")] + OutputTooSmall { + /// Required byte count. + required: usize, + /// Provided byte count. + have: usize, + }, + /// Byte length cannot be represented by the kernel ABI. + #[error("CUDA byte length is too large for kernel launch: {len}")] + LengthTooLarge { + /// Byte length. + len: usize, + }, + /// Device byte length is not aligned to the requested typed view element. + #[error("CUDA buffer length {bytes} is not a multiple of typed element size {element_size}")] + LengthNotElementAligned { + /// Byte length. + bytes: usize, + /// Requested element size. + element_size: usize, + }, + /// Image dimensions overflowed allocation or launch geometry. + #[error("CUDA image allocation size overflow for {width}x{height}x{channels}")] + ImageTooLarge { + /// Image width. + width: u32, + /// Image height. + height: u32, + /// Channel count. + channels: usize, + }, + /// Internal runtime state lock was poisoned. + #[error("CUDA runtime state lock is poisoned: {message}")] + StatePoisoned { + /// Poison error message. + message: String, + }, + /// A J2K CUDA kernel reported a validated runtime failure. + #[error("CUDA kernel {kernel} reported status {code} detail {detail}")] + KernelStatus { + /// Kernel entry point or logical stage name. + kernel: &'static str, + /// Kernel-defined status code. + code: u32, + /// Kernel-defined detail code. + detail: u32, + }, + /// Caller supplied arguments that cannot be represented by this runtime API. + #[error("CUDA invalid argument: {message}")] + InvalidArgument { + /// Human-readable validation failure. + message: String, + }, +} + +impl CudaError { + /// True when the error means the CUDA driver or device is unavailable. + pub fn is_unavailable(&self) -> bool { + matches!(self, Self::Unavailable { .. }) + } +} diff --git a/crates/j2k-cuda-runtime/src/execution.rs b/crates/j2k-cuda-runtime/src/execution.rs new file mode 100644 index 00000000..658c2e56 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/execution.rs @@ -0,0 +1,616 @@ +use crate::{ + build_flags::cuda_stage_timings_disabled, + context::{CudaContext, CudaKernelModule, CudaKernelName}, + driver::{CuDevicePtr, CuEvent, CuFunction, CuStream, CudaNvtxRange}, + error::CudaError, + kernels::{self, copy_u8_launch_geometry, CudaKernel}, + memory::{CudaDeviceBuffer, CudaDeviceBufferRange, CudaPooledDeviceBuffer}, +}; +use std::{ffi::c_void, os::raw::c_uint}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CudaLaunchMode { + Sync, + Async, +} + +impl CudaLaunchMode { + pub(crate) fn from_synchronize(synchronize: bool) -> Self { + if synchronize { + Self::Sync + } else { + Self::Async + } + } +} + +/// Marker for values that can be passed by address to a CUDA kernel launch. +/// +/// # Safety +/// +/// Implementors must have a stable, CUDA-compatible by-value representation for +/// the duration of `cuLaunchKernel`. +pub(crate) unsafe trait CudaKernelParam {} + +// SAFETY: `CuDevicePtr` is the raw CUDA device pointer value expected by kernels. +unsafe impl CudaKernelParam for CuDevicePtr {} +// SAFETY: CUDA kernels receive these scalar ABI types by value via parameter pointers. +unsafe impl CudaKernelParam for u32 {} +// SAFETY: CUDA kernels receive these scalar ABI types by value via parameter pointers. +unsafe impl CudaKernelParam for i32 {} +// SAFETY: CUDA kernels receive these scalar ABI types by value via parameter pointers. +unsafe impl CudaKernelParam for f32 {} + +pub(crate) fn cuda_kernel_param(value: &mut T) -> *mut c_void +where + T: CudaKernelParam, +{ + std::ptr::from_mut(value).cast::() +} + +impl CudaContext { + /// Copy host bytes through a CUDA copy kernel and return device output. + pub fn copy_with_kernel(&self, bytes: &[u8]) -> Result { + let staging = self.upload(bytes)?; + let output = self.copy_device_to_device_with_kernel(&staging)?; + let copy_dispatches = usize::from(!bytes.is_empty()); + Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats { + kernel_dispatches: copy_dispatches, + copy_kernel_dispatches: copy_dispatches, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }) + } + + /// Copy host bytes through the opt-in cuda-oxide `CopyU8` kernel. + #[cfg(feature = "cuda-oxide-copy-u8")] + pub fn copy_with_cuda_oxide_kernel(&self, bytes: &[u8]) -> Result { + let staging = self.upload(bytes)?; + let output = self.copy_device_to_device_with_cuda_oxide_kernel(&staging)?; + let copy_dispatches = usize::from(!bytes.is_empty()); + Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats { + kernel_dispatches: copy_dispatches, + copy_kernel_dispatches: copy_dispatches, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }) + } + + pub(crate) fn launch_kernel( + &self, + function: CuFunction, + geometry: kernels::CudaLaunchGeometry, + params: &mut [*mut c_void], + ) -> Result<(), CudaError> { + self.launch_kernel_async(function, geometry, params)?; + // SAFETY: `function` was loaded from a live module in this context, and + // the kernel was launched on the current context; synchronize waits for + // completion before callers inspect outputs. + self.synchronize() + } + + pub(crate) fn launch_named_kernel( + &self, + kernel: CudaKernel, + geometry: kernels::CudaLaunchGeometry, + params: &mut [*mut c_void; N], + mode: CudaLaunchMode, + ) -> Result<(), CudaError> { + let function = self.inner.kernel_function(kernel)?; + match mode { + CudaLaunchMode::Sync => self.launch_kernel(function, geometry, params), + CudaLaunchMode::Async => self.launch_kernel_async(function, geometry, params), + } + } + + pub(crate) fn launch_kernel_async( + &self, + function: CuFunction, + geometry: kernels::CudaLaunchGeometry, + params: &mut [*mut c_void], + ) -> Result<(), CudaError> { + // SAFETY: `function` was loaded from a live module in this context, and + // `params` contains kernel argument pointers valid for the launch call. + let launch_status = unsafe { + (self.inner.driver.cu_launch_kernel)( + function, + geometry.grid.0, + geometry.grid.1, + geometry.grid.2, + geometry.block.0, + geometry.block.1, + geometry.block.2, + 0, + std::ptr::null_mut(), + params.as_mut_ptr(), + std::ptr::null_mut(), + ) + }; + self.inner.driver.check("cuLaunchKernel", launch_status) + } + + /// Copy one device buffer to another through a CUDA kernel. + pub fn copy_device_to_device_with_kernel( + &self, + src: &CudaDeviceBuffer, + ) -> Result { + self.copy_device_ptr_to_device_with_kernel(src.device_ptr(), src.byte_len()) + } + + #[cfg(feature = "cuda-oxide-copy-u8")] + pub(crate) fn copy_device_to_device_with_cuda_oxide_kernel( + &self, + src: &CudaDeviceBuffer, + ) -> Result { + self.copy_device_ptr_to_device_with_cuda_oxide_kernel(src.device_ptr(), src.byte_len()) + } + + pub(crate) fn copy_device_ptr_to_device_with_kernel( + &self, + src_ptr: CuDevicePtr, + byte_len: usize, + ) -> Result { + self.copy_device_ptr_to_device_with_copy_u8_loader(src_ptr, byte_len, |context| { + context.inner.kernel_function(CudaKernel::CopyU8) + }) + } + + #[cfg(feature = "cuda-oxide-copy-u8")] + pub(crate) fn copy_device_ptr_to_device_with_cuda_oxide_kernel( + &self, + src_ptr: CuDevicePtr, + byte_len: usize, + ) -> Result { + self.copy_device_ptr_to_device_with_copy_u8_loader(src_ptr, byte_len, |context| { + context.inner.cuda_oxide_copy_u8_kernel_function() + }) + } + + fn copy_device_ptr_to_device_with_copy_u8_loader( + &self, + src_ptr: CuDevicePtr, + byte_len: usize, + load_function: impl FnOnce(&Self) -> Result, + ) -> Result { + self.inner.set_current()?; + let dst = self.allocate(byte_len)?; + if byte_len == 0 { + return Ok(dst); + } + + let function = load_function(self)?; + let mut dst_ptr = dst.device_ptr(); + let mut src_ptr = src_ptr; + let mut len = + u64::try_from(byte_len).map_err(|_| CudaError::LengthTooLarge { len: byte_len })?; + let mut params = cuda_kernel_params!(dst_ptr, src_ptr, len); + let geometry = + copy_u8_launch_geometry(byte_len).ok_or(CudaError::LengthTooLarge { len: byte_len })?; + + self.launch_kernel(function, geometry, &mut params)?; + + Ok(dst) + } + + pub(crate) fn memset_d32( + &self, + dst: &CudaDeviceBuffer, + value: c_uint, + words: usize, + ) -> Result<(), CudaError> { + self.inner.set_current()?; + let required = words + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: words })?; + if required > dst.byte_len() { + return Err(CudaError::OutputTooSmall { + required, + have: dst.byte_len(), + }); + } + if words == 0 { + return Ok(()); + } + // SAFETY: `dst` is a live CUDA allocation in this context and `words` + // was bounds-checked against the allocation byte length above. + self.inner.driver.check("cuMemsetD32_v2", unsafe { + (self.inner.driver.cu_memset_d32)(dst.device_ptr(), value, words) + }) + } + + /// Create a CUDA stream owned by this context. + pub fn create_stream(&self) -> Result { + self.inner.set_current()?; + let mut stream = std::ptr::null_mut(); + // SAFETY: CUDA writes a new stream handle, destroyed by CudaStream. + self.inner.driver.check("cuStreamCreate", unsafe { + (self.inner.driver.cu_stream_create)(&raw mut stream, 0) + })?; + Ok(CudaStream { + context: self.clone(), + stream, + }) + } + + /// Create a CUDA timing event owned by this context. + pub fn create_event(&self) -> Result { + self.inner.set_current()?; + let mut event = std::ptr::null_mut(); + // SAFETY: CUDA writes a new event handle, destroyed by CudaEvent. + self.inner.driver.check("cuEventCreate", unsafe { + (self.inner.driver.cu_event_create)(&raw mut event, 0) + })?; + Ok(CudaEvent { + context: self.clone(), + event, + }) + } + + /// Time work submitted to the default CUDA stream and return elapsed microseconds. + pub fn time_default_stream_us( + &self, + work: impl FnOnce() -> Result, + ) -> Result<(T, u128), CudaError> { + self.inner.set_current()?; + if cuda_stage_timings_disabled() { + return work().map(|output| (output, 0)); + } + let start = self.create_event()?; + let end = self.create_event()?; + start.record_default_stream()?; + let output = match work() { + Ok(output) => output, + Err(error) => { + // Timed closures may submit asynchronous default-stream work. + // On a later host-side error, wait before dropping any device + // buffers captured by the closure. + self.synchronize()?; + return Err(error); + } + }; + end.record_default_stream()?; + end.synchronize()?; + Ok((output, elapsed_event_us_ceil(&start, &end)?)) + } + + /// Run work inside an optional NVTX profiling range. + /// + /// The range is a no-op unless the crate is built with `cuda-profiling` + /// and an NVTX runtime library can be loaded dynamically. + pub fn with_nvtx_range( + &self, + name: &str, + work: impl FnOnce() -> Result, + ) -> Result { + let _range = CudaNvtxRange::push(name); + work() + } + + /// Time work submitted to the default CUDA stream inside an optional NVTX range. + /// + /// The NVTX range is a no-op unless the crate is built with + /// `cuda-profiling` and an NVTX runtime library can be loaded dynamically. + pub fn time_default_stream_named_us( + &self, + name: &str, + work: impl FnOnce() -> Result, + ) -> Result<(T, u128), CudaError> { + self.with_nvtx_range(name, || self.time_default_stream_us(work)) + } + + /// Optionally time work submitted to the default CUDA stream inside an NVTX range. + pub fn time_default_stream_named_us_if( + &self, + collect_stage_timings: bool, + name: &str, + work: impl FnOnce() -> Result, + ) -> Result<(T, u128), CudaError> { + if collect_stage_timings { + self.time_default_stream_named_us(name, work) + } else { + self.with_nvtx_range(name, || work().map(|output| (output, 0))) + } + } + + /// Synchronize all work submitted to this CUDA context. + pub fn synchronize(&self) -> Result<(), CudaError> { + self.inner.set_current()?; + // SAFETY: a CUDA context is current for this `CudaContext`. + let status = unsafe { (self.inner.driver.cu_ctx_synchronize)() }; + self.inner.driver.check("cuCtxSynchronize", status) + } + + /// Preload a bundled CUDA kernel module and return its metadata handle. + pub fn preload_kernel_module( + &self, + kernel: CudaKernelName, + ) -> Result { + let _ = self.inner.kernel_function(kernel.kernel())?; + Ok(CudaKernelModule { + kernel, + entrypoint: kernel.entrypoint(), + }) + } +} + +/// CUDA stream RAII handle. +#[derive(Debug)] +pub struct CudaStream { + pub(crate) context: CudaContext, + pub(crate) stream: CuStream, +} + +impl CudaStream { + /// Synchronize all work submitted to this stream. + pub fn synchronize(&self) -> Result<(), CudaError> { + self.context.inner.set_current()?; + // SAFETY: stream is a live CUDA stream owned by this handle. + self.context + .inner + .driver + .check("cuStreamSynchronize", unsafe { + (self.context.inner.driver.cu_stream_synchronize)(self.stream) + }) + } +} + +impl Drop for CudaStream { + fn drop(&mut self) { + if !self.stream.is_null() { + let _ = self.context.inner.set_current(); + // SAFETY: stream was created by this context. Drop cannot surface + // errors, so cleanup failures are ignored. + let _ = unsafe { (self.context.inner.driver.cu_stream_destroy)(self.stream) }; + } + } +} + +// SAFETY: CUDA stream handles are driver-owned resources. The Rust handle owns +// destruction and does not expose mutable aliasing of Rust memory. +unsafe impl Send for CudaStream {} + +/// CUDA event RAII handle for timing and synchronization. +#[derive(Debug)] +pub struct CudaEvent { + pub(crate) context: CudaContext, + pub(crate) event: CuEvent, +} + +impl CudaEvent { + /// Record this event on a CUDA stream. + pub fn record(&self, stream: &CudaStream) -> Result<(), CudaError> { + self.context.inner.set_current()?; + // SAFETY: event and stream are live CUDA handles. + self.context.inner.driver.check("cuEventRecord", unsafe { + (self.context.inner.driver.cu_event_record)(self.event, stream.stream) + }) + } + + pub(crate) fn record_default_stream(&self) -> Result<(), CudaError> { + self.context.inner.set_current()?; + // SAFETY: a null stream is CUDA's default stream for the current context. + self.context.inner.driver.check("cuEventRecord", unsafe { + (self.context.inner.driver.cu_event_record)(self.event, std::ptr::null_mut()) + }) + } + + /// Wait for this event to complete. + pub fn synchronize(&self) -> Result<(), CudaError> { + self.context.inner.set_current()?; + // SAFETY: event is a live CUDA event owned by this handle. + self.context + .inner + .driver + .check("cuEventSynchronize", unsafe { + (self.context.inner.driver.cu_event_synchronize)(self.event) + }) + } + + /// Elapsed time in microseconds from `start` to `end`. + pub fn elapsed_time_us(start: &Self, end: &Self) -> Result { + end.context.inner.set_current()?; + let mut millis = 0.0f32; + // SAFETY: start and end are live CUDA events that have been recorded. + let status = unsafe { + (end.context.inner.driver.cu_event_elapsed_time)( + &raw mut millis, + start.event, + end.event, + ) + }; + end.context + .inner + .driver + .check("cuEventElapsedTime", status)?; + Ok(millis * 1000.0) + } +} + +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +pub(crate) fn elapsed_event_us_ceil(start: &CudaEvent, end: &CudaEvent) -> Result { + let elapsed = CudaEvent::elapsed_time_us(start, end)?; + if elapsed <= 0.0 { + return Ok(1); + } + Ok(elapsed.ceil() as u128) +} + +impl Drop for CudaEvent { + fn drop(&mut self) { + if !self.event.is_null() { + let _ = self.context.inner.set_current(); + // SAFETY: event was created by this context. Drop cannot surface + // errors, so cleanup failures are ignored. + let _ = unsafe { (self.context.inner.driver.cu_event_destroy)(self.event) }; + } + } +} + +// SAFETY: CUDA event handles are driver-owned resources. The Rust handle owns +// destruction and does not expose mutable aliasing of Rust memory. +unsafe impl Send for CudaEvent {} + +/// Device buffer plus execution metadata. +#[derive(Debug)] +pub struct CudaKernelOutput { + pub(crate) buffer: CudaDeviceBuffer, + pub(crate) execution: CudaExecutionStats, +} + +/// Multiple device buffers plus shared execution metadata from one batched kernel. +#[derive(Debug)] +pub struct CudaKernelBatchOutput { + pub(crate) outputs: Vec, + pub(crate) execution: CudaExecutionStats, +} + +/// One contiguous device buffer plus per-item ranges from one batched kernel. +#[derive(Debug)] +pub struct CudaKernelContiguousBatchOutput { + pub(crate) output: CudaDeviceBuffer, + pub(crate) ranges: Vec, + pub(crate) execution: CudaExecutionStats, +} + +/// Pooled device buffer plus execution metadata. +#[derive(Debug)] +pub struct CudaPooledKernelOutput { + pub(crate) buffer: CudaPooledDeviceBuffer, + pub(crate) execution: CudaExecutionStats, +} + +/// Enqueued CUDA work plus pooled resources that must stay live until the +/// default stream is synchronized. +#[derive(Debug)] +pub struct CudaQueuedExecution { + pub(crate) resources: Vec, + pub(crate) execution: CudaExecutionStats, +} + +impl CudaQueuedExecution { + /// CUDA execution counters for the enqueued work. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// Number of pooled resource buffers held live for the queued work. + pub fn resource_count(&self) -> usize { + self.resources.len() + } +} + +impl CudaKernelOutput { + /// Device buffer produced by the kernel. + pub fn buffer(&self) -> &CudaDeviceBuffer { + &self.buffer + } + + /// CUDA execution counters for the kernel. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// Split output into device buffer and execution metadata. + pub fn into_parts(self) -> (CudaDeviceBuffer, CudaExecutionStats) { + (self.buffer, self.execution) + } +} + +impl CudaKernelBatchOutput { + /// Device buffers produced by the batched kernel. + pub fn outputs(&self) -> &[CudaDeviceBuffer] { + &self.outputs + } + + /// CUDA execution counters for the batched kernel. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// Split output into device buffers and execution metadata. + pub fn into_parts(self) -> (Vec, CudaExecutionStats) { + (self.outputs, self.execution) + } +} + +impl CudaKernelContiguousBatchOutput { + /// Contiguous device buffer produced by the batched kernel. + pub fn output(&self) -> &CudaDeviceBuffer { + &self.output + } + + /// Per-item byte ranges inside the contiguous output buffer. + pub fn ranges(&self) -> &[CudaDeviceBufferRange] { + &self.ranges + } + + /// CUDA execution counters for the batched kernel. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// Split output into the contiguous buffer, per-item ranges, and execution metadata. + pub fn into_parts( + self, + ) -> ( + CudaDeviceBuffer, + Vec, + CudaExecutionStats, + ) { + (self.output, self.ranges, self.execution) + } +} + +impl CudaPooledKernelOutput { + /// Device buffer produced by the kernel. + pub fn buffer(&self) -> Option<&CudaDeviceBuffer> { + self.buffer.as_device_buffer() + } + + /// CUDA execution counters for the kernel. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// Split output into pooled device buffer and execution metadata. + pub fn into_parts(self) -> (CudaPooledDeviceBuffer, CudaExecutionStats) { + (self.buffer, self.execution) + } +} + +/// CUDA execution counters exposed for dispatch observability. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaExecutionStats { + pub(crate) kernel_dispatches: usize, + pub(crate) copy_kernel_dispatches: usize, + pub(crate) decode_kernel_dispatches: usize, + pub(crate) hardware_decode: bool, +} + +impl CudaExecutionStats { + /// Total kernel dispatch count. + pub fn kernel_dispatches(self) -> usize { + self.kernel_dispatches + } + + /// Copy-kernel dispatch count. + pub fn copy_kernel_dispatches(self) -> usize { + self.copy_kernel_dispatches + } + + /// Hardware decode dispatch count. + pub fn decode_kernel_dispatches(self) -> usize { + self.decode_kernel_dispatches + } + + /// True when a hardware decode path was used. + pub fn used_hardware_decode(self) -> bool { + self.hardware_decode + } +} diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode.rs b/crates/j2k-cuda-runtime/src/htj2k_decode.rs new file mode 100644 index 00000000..3e02d75d --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_decode.rs @@ -0,0 +1,1486 @@ +use crate::{ + bytes::{ + htj2k_cleanup_multi_jobs_as_bytes, htj2k_dequantize_jobs_as_bytes, htj2k_jobs_as_bytes, + htj2k_statuses_as_bytes_mut, htj2k_statuses_byte_len, u16_slice_as_bytes, + }, + context::CudaContext, + error::CudaError, + execution::{cuda_kernel_param, CudaExecutionStats, CudaLaunchMode}, + kernels::{ + htj2k_codeblock_launch_geometry, htj2k_codeblock_sample_launch_geometry, CudaKernel, + CudaLaunchGeometry, + }, + memory::{pooled_device_buffer, CudaBufferPool, CudaDeviceBuffer, CudaPooledDeviceBuffer}, +}; +use std::{os::raw::c_uint, sync::Arc, time::Instant}; + +/// HTJ2K code-block decode job consumed by the CUDA entropy kernel launcher. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CudaHtj2kCodeBlockJob { + /// Byte offset into the contiguous compressed payload buffer. + pub payload_offset: u64, + /// Code-block width in coefficients. + pub width: u32, + /// Code-block height in coefficients. + pub height: u32, + /// Combined cleanup/refinement byte length. + pub payload_len: u32, + /// Cleanup segment length in bytes. + pub cleanup_length: u32, + /// Refinement segment length in bytes. + pub refinement_length: u32, + /// Missing most-significant bit planes. + pub missing_bit_planes: u8, + /// Total coded bitplanes for this code block's sub-band. + pub num_bitplanes: u8, + /// Number of HT coding passes present. + pub number_of_coding_passes: u8, + /// Output row stride, in coefficients. + pub output_stride: u32, + /// Output offset, in coefficients, into the destination plane. + pub output_offset: u32, + /// Dequantization multiplier for decoded coefficient values. + pub dequantization_step: f32, + /// Vertically causal context mode flag. + pub stripe_causal: bool, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub(crate) struct CudaHtj2kCodeBlockKernelJob { + pub(crate) coded_offset: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) coded_len: u32, + pub(crate) cleanup_length: u32, + pub(crate) refinement_length: u32, + pub(crate) missing_msbs: u32, + pub(crate) num_bitplanes: u32, + pub(crate) number_of_coding_passes: u32, + pub(crate) output_stride: u32, + pub(crate) output_offset: u32, + pub(crate) dequantization_step: f32, + pub(crate) stripe_causal: u32, +} + +/// One output buffer and its code-block jobs for batched HTJ2K cleanup decode. +#[derive(Clone, Copy, Debug)] +pub struct CudaHtj2kCleanupTarget<'a> { + /// Device buffer receiving decoded integer coefficient bits. + pub coefficients: &'a CudaDeviceBuffer, + /// Code-block jobs that write into `coefficients`. + pub jobs: &'a [CudaHtj2kCodeBlockJob], + /// Number of coefficient words available in `coefficients`. + pub output_words: usize, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub(crate) struct CudaHtj2kCleanupMultiKernelJob { + pub(crate) output_ptr: u64, + pub(crate) coded_offset: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) coded_len: u32, + pub(crate) cleanup_length: u32, + pub(crate) refinement_length: u32, + pub(crate) missing_msbs: u32, + pub(crate) num_bitplanes: u32, + pub(crate) number_of_coding_passes: u32, + pub(crate) output_stride: u32, + pub(crate) output_offset: u32, + pub(crate) dequantization_step: f32, + pub(crate) stripe_causal: u32, +} + +/// One output buffer and its code-block jobs for batched HTJ2K dequantization. +#[derive(Clone, Copy, Debug)] +pub struct CudaHtj2kDequantizeTarget<'a> { + /// Device buffer containing decoded integer coefficient bits. + pub coefficients: &'a CudaDeviceBuffer, + /// Code-block jobs that write into `coefficients`. + pub jobs: &'a [CudaHtj2kCodeBlockJob], + /// Number of coefficient words available in `coefficients`. + pub output_words: usize, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub(crate) struct CudaHtj2kDequantizeKernelJob { + pub(crate) output_ptr: u64, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) output_stride: u32, + pub(crate) output_offset: u32, + pub(crate) num_bitplanes: u32, + pub(crate) reserved: u32, + pub(crate) dequantization_step: f32, +} + +/// Static HTJ2K entropy lookup tables uploaded for CUDA code-block decode. +#[derive(Clone, Copy, Debug)] +pub struct CudaHtj2kDecodeTables<'a> { + /// HT cleanup VLC table for first quad row contexts. + pub vlc_table0: &'a [u16; 1024], + /// HT cleanup VLC table for subsequent quad row contexts. + pub vlc_table1: &'a [u16; 1024], + /// HT cleanup UVLC table for first quad row contexts. + pub uvlc_table0: &'a [u16; 320], + /// HT cleanup UVLC table for subsequent quad row contexts. + pub uvlc_table1: &'a [u16; 256], +} + +/// Status written by the CUDA HTJ2K entropy decoder for one code-block job. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaHtj2kStatus { + /// Zero on success; nonzero values are kernel-defined failures. + pub code: u32, + /// Kernel-defined failure detail. + pub detail: u32, + /// Reserved for ABI stability. + pub reserved0: u32, + /// Reserved for ABI stability. + pub reserved1: u32, +} + +impl CudaHtj2kStatus { + /// Return true when the CUDA kernel reported success. + pub fn is_ok(self) -> bool { + self.code == HTJ2K_STATUS_OK + } +} + +/// CUDA event timings for resident HTJ2K decode stages. +#[allow(clippy::struct_field_names)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaHtj2kDecodeStageTimings { + /// HT cleanup entropy decode dispatch time, in microseconds. + pub ht_cleanup_us: u128, + /// HT refinement work time, in microseconds. + /// + /// The current CUDA entropy kernel executes cleanup and refinement for a + /// code-block in one dispatch. When a batch contains refinement segments, + /// this records that fused dispatch time so higher-level profiles expose + /// refinement-bearing work instead of silently reporting zero. + pub ht_refine_us: u128, + /// Sign/magnitude dequantization time, in microseconds. + pub dequant_us: u128, + /// Host-observed status download time, in microseconds. + pub status_d2h_us: u128, +} + +/// Device-resident HTJ2K entropy decode result. +#[derive(Debug)] +pub struct CudaHtj2kDecodeOutput { + pub(crate) coefficients: CudaDeviceBuffer, + pub(crate) execution: CudaExecutionStats, + pub(crate) statuses: Vec, + pub(crate) stage_timings: CudaHtj2kDecodeStageTimings, +} + +impl CudaHtj2kDecodeOutput { + /// Device buffer containing decoded f32 coefficients. + pub fn coefficients(&self) -> &CudaDeviceBuffer { + &self.coefficients + } + + /// CUDA execution counters for the decode dispatch. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// Per-code-block kernel status rows downloaded after dispatch. + pub fn statuses(&self) -> &[CudaHtj2kStatus] { + &self.statuses + } + + /// CUDA event timings for the decode stages inside this output. + pub fn stage_timings(&self) -> CudaHtj2kDecodeStageTimings { + self.stage_timings + } + + /// Split output into device coefficients, execution counters, and statuses. + pub fn into_parts(self) -> (CudaDeviceBuffer, CudaExecutionStats, Vec) { + (self.coefficients, self.execution, self.statuses) + } +} + +/// Device-resident HTJ2K entropy decode result borrowed from a CUDA buffer pool. +#[derive(Debug)] +pub struct CudaPooledHtj2kDecodeOutput { + pub(crate) coefficients: CudaPooledDeviceBuffer, + pub(crate) execution: CudaExecutionStats, + pub(crate) statuses: Vec, + pub(crate) stage_timings: CudaHtj2kDecodeStageTimings, +} + +impl CudaPooledHtj2kDecodeOutput { + /// Device buffer containing decoded f32 coefficients. + pub fn coefficients(&self) -> Option<&CudaDeviceBuffer> { + self.coefficients.as_device_buffer() + } + + /// CUDA execution counters for the decode dispatch. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// Per-code-block kernel status rows downloaded after dispatch. + pub fn statuses(&self) -> &[CudaHtj2kStatus] { + &self.statuses + } + + /// CUDA event timings for the decode stages inside this output. + pub fn stage_timings(&self) -> CudaHtj2kDecodeStageTimings { + self.stage_timings + } + + /// Split output into pooled device coefficients, execution counters, and statuses. + pub fn into_parts( + self, + ) -> ( + CudaPooledDeviceBuffer, + CudaExecutionStats, + Vec, + ) { + (self.coefficients, self.execution, self.statuses) + } +} + +/// Device-resident static HTJ2K cleanup decode lookup tables. +#[derive(Clone, Debug)] +pub struct CudaHtj2kDecodeTableResources { + pub(crate) inner: Arc, +} + +#[derive(Debug)] +pub(crate) struct CudaHtj2kDecodeTableResourceInner { + pub(crate) vlc_table0: CudaDeviceBuffer, + pub(crate) vlc_table1: CudaDeviceBuffer, + pub(crate) uvlc_table0: CudaDeviceBuffer, + pub(crate) uvlc_table1: CudaDeviceBuffer, +} + +/// Device-resident HTJ2K decode payload plus shared lookup tables reused across sub-band dispatches. +#[derive(Debug)] +pub struct CudaHtj2kDecodeResources { + pub(crate) payload: CudaHtj2kDecodePayload, + pub(crate) payload_len: usize, + pub(crate) tables: CudaHtj2kDecodeTableResources, +} + +#[derive(Debug)] +pub(crate) enum CudaHtj2kDecodePayload { + Owned(CudaDeviceBuffer), + Pooled(CudaPooledDeviceBuffer), +} + +impl CudaHtj2kDecodePayload { + fn buffer(&self) -> Result<&CudaDeviceBuffer, CudaError> { + match self { + Self::Owned(buffer) => Ok(buffer), + Self::Pooled(buffer) => pooled_device_buffer(buffer), + } + } +} + +pub(crate) const HTJ2K_STATUS_OK: u32 = 0; + +pub(crate) const HTJ2K_STATUS_UNSUPPORTED: u32 = 2; + +impl CudaContext { + /// Decode HTJ2K code blocks into a device-resident f32 coefficient plane. + #[allow(clippy::similar_names)] + pub fn decode_htj2k_codeblocks( + &self, + payload: &[u8], + jobs: &[CudaHtj2kCodeBlockJob], + tables: CudaHtj2kDecodeTables<'_>, + output_words: usize, + ) -> Result { + if jobs.is_empty() { + return self.decode_empty_htj2k_codeblocks(jobs, output_words); + } + let resources = self.upload_htj2k_decode_resources(payload, tables)?; + self.decode_htj2k_codeblocks_with_resources(&resources, jobs, output_words) + } + + /// Decode HTJ2K code blocks without collecting CUDA event timings. + #[allow(clippy::similar_names)] + pub fn decode_htj2k_codeblocks_untimed( + &self, + payload: &[u8], + jobs: &[CudaHtj2kCodeBlockJob], + tables: CudaHtj2kDecodeTables<'_>, + output_words: usize, + ) -> Result { + if jobs.is_empty() { + return self.decode_empty_htj2k_codeblocks(jobs, output_words); + } + let resources = self.upload_htj2k_decode_resources(payload, tables)?; + self.decode_htj2k_codeblocks_with_resources_untimed(&resources, jobs, output_words) + } + + /// Upload HTJ2K decode payload and lookup tables once for reuse by sub-band dispatches. + pub fn upload_htj2k_decode_resources( + &self, + payload: &[u8], + tables: CudaHtj2kDecodeTables<'_>, + ) -> Result { + let tables = self.upload_htj2k_decode_table_resources(tables)?; + self.upload_htj2k_decode_resources_with_tables(payload, &tables) + } + + /// Upload static HTJ2K cleanup decode lookup tables once for reuse. + pub fn upload_htj2k_decode_table_resources( + &self, + tables: CudaHtj2kDecodeTables<'_>, + ) -> Result { + self.inner.set_current()?; + Ok(CudaHtj2kDecodeTableResources { + inner: Arc::new(CudaHtj2kDecodeTableResourceInner { + vlc_table0: self.upload(u16_slice_as_bytes(tables.vlc_table0))?, + vlc_table1: self.upload(u16_slice_as_bytes(tables.vlc_table1))?, + uvlc_table0: self.upload(u16_slice_as_bytes(tables.uvlc_table0))?, + uvlc_table1: self.upload(u16_slice_as_bytes(tables.uvlc_table1))?, + }), + }) + } + + /// Upload an HTJ2K decode payload while reusing already resident cleanup tables. + pub fn upload_htj2k_decode_resources_with_tables( + &self, + payload: &[u8], + tables: &CudaHtj2kDecodeTableResources, + ) -> Result { + self.inner.set_current()?; + Ok(CudaHtj2kDecodeResources { + payload: CudaHtj2kDecodePayload::Owned(self.upload_pinned(payload)?), + payload_len: payload.len(), + tables: tables.clone(), + }) + } + + /// Upload an HTJ2K decode payload into a pooled buffer while reusing already resident cleanup tables. + pub fn upload_htj2k_decode_resources_with_tables_and_pool( + &self, + payload: &[u8], + tables: &CudaHtj2kDecodeTableResources, + pool: &CudaBufferPool, + ) -> Result { + self.inner.set_current()?; + Ok(CudaHtj2kDecodeResources { + payload: CudaHtj2kDecodePayload::Pooled(pool.upload_pinned(payload)?), + payload_len: payload.len(), + tables: tables.clone(), + }) + } + + /// Decode HTJ2K code blocks using already resident payload and lookup tables. + pub fn decode_htj2k_codeblocks_with_resources( + &self, + resources: &CudaHtj2kDecodeResources, + jobs: &[CudaHtj2kCodeBlockJob], + output_words: usize, + ) -> Result { + self.decode_htj2k_codeblocks_with_resources_impl(resources, jobs, output_words, true) + } + + /// Decode HTJ2K code blocks using resident resources without CUDA event timings. + pub fn decode_htj2k_codeblocks_with_resources_untimed( + &self, + resources: &CudaHtj2kDecodeResources, + jobs: &[CudaHtj2kCodeBlockJob], + output_words: usize, + ) -> Result { + self.decode_htj2k_codeblocks_with_resources_impl(resources, jobs, output_words, false) + } + + /// Decode HTJ2K code blocks using resident resources and caller-owned + /// transient buffer reuse. + pub fn decode_htj2k_codeblocks_with_resources_and_pool( + &self, + resources: &CudaHtj2kDecodeResources, + jobs: &[CudaHtj2kCodeBlockJob], + output_words: usize, + pool: &CudaBufferPool, + ) -> Result { + self.decode_htj2k_codeblocks_with_resources_and_pool_impl( + resources, + jobs, + output_words, + pool, + true, + true, + ) + } + + /// Decode HTJ2K code blocks using resident resources and caller-owned + /// transient buffer reuse, without CUDA event timings. + pub fn decode_htj2k_codeblocks_with_resources_untimed_and_pool( + &self, + resources: &CudaHtj2kDecodeResources, + jobs: &[CudaHtj2kCodeBlockJob], + output_words: usize, + pool: &CudaBufferPool, + ) -> Result { + self.decode_htj2k_codeblocks_with_resources_and_pool_impl( + resources, + jobs, + output_words, + pool, + false, + true, + ) + } + + /// Decode HTJ2K cleanup passes into resident coefficient buffers using + /// caller-owned transient buffer reuse. Dequantization is left to a later + /// dispatch. + pub fn decode_htj2k_codeblocks_cleanup_with_resources_and_pool( + &self, + resources: &CudaHtj2kDecodeResources, + jobs: &[CudaHtj2kCodeBlockJob], + output_words: usize, + pool: &CudaBufferPool, + ) -> Result { + self.decode_htj2k_codeblocks_with_resources_and_pool_impl( + resources, + jobs, + output_words, + pool, + true, + false, + ) + } + + /// Decode HTJ2K cleanup passes into resident coefficient buffers using + /// caller-owned transient buffer reuse, without CUDA event timings. + pub fn decode_htj2k_codeblocks_cleanup_with_resources_untimed_and_pool( + &self, + resources: &CudaHtj2kDecodeResources, + jobs: &[CudaHtj2kCodeBlockJob], + output_words: usize, + pool: &CudaBufferPool, + ) -> Result { + self.decode_htj2k_codeblocks_with_resources_and_pool_impl( + resources, + jobs, + output_words, + pool, + false, + false, + ) + } + + /// Allocate and initialize an HTJ2K coefficient output buffer without + /// launching entropy cleanup decode. This is used when cleanup work is + /// batched across multiple output buffers. + pub fn allocate_htj2k_codeblock_coefficients_with_pool( + &self, + jobs: &[CudaHtj2kCodeBlockJob], + output_words: usize, + pool: &CudaBufferPool, + ) -> Result { + self.inner.set_current()?; + let output_bytes = output_words + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: output_words })?; + let coefficients = pool.take(output_bytes)?; + let coefficient_buffer = pooled_device_buffer(&coefficients)?; + if htj2k_decode_needs_zero_fill(jobs, output_words)? { + self.memset_d32(coefficient_buffer, 0, output_words)?; + } + Ok(CudaPooledHtj2kDecodeOutput { + coefficients, + execution: CudaExecutionStats::default(), + statuses: Vec::new(), + stage_timings: CudaHtj2kDecodeStageTimings::default(), + }) + } + + /// Decode HTJ2K cleanup passes for multiple output buffers with one CUDA + /// dispatch. Dequantization is left to a later dispatch. + pub fn decode_htj2k_codeblocks_cleanup_multi_with_resources_and_pool( + &self, + resources: &CudaHtj2kDecodeResources, + targets: &[CudaHtj2kCleanupTarget<'_>], + pool: &CudaBufferPool, + ) -> Result { + self.decode_htj2k_codeblocks_cleanup_multi_with_resources_and_pool_timed( + resources, targets, pool, false, + ) + .map(|(execution, _timings)| execution) + } + + /// Enqueue HTJ2K cleanup passes for multiple output buffers with one CUDA + /// dispatch. The returned value must be kept live until `finish` validates + /// the kernel statuses after the default stream has completed. + pub fn decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool( + &self, + resources: &CudaHtj2kDecodeResources, + targets: &[CudaHtj2kCleanupTarget<'_>], + pool: &CudaBufferPool, + ) -> Result { + self.inner.set_current()?; + let kernel_jobs = htj2k_cleanup_multi_kernel_jobs(targets, resources.payload_len)?; + if kernel_jobs.is_empty() { + return Ok(CudaQueuedHtj2kCleanup { + resources: Vec::new(), + status_buffer: None, + status_count: 0, + kernel_name: "j2k_htj2k_decode_codeblocks_multi", + execution: CudaExecutionStats::default(), + }); + } + let (decode_kernel, decode_kernel_name) = htj2k_decode_multi_kernel_for_jobs(&kernel_jobs); + + let jobs_buffer = pool.upload(htj2k_cleanup_multi_jobs_as_bytes(&kernel_jobs))?; + let status_buffer = pool.take(htj2k_statuses_byte_len(kernel_jobs.len())?)?; + let launch_result = self.launch_htj2k_decode_codeblocks_multi( + decode_kernel, + resources.payload.buffer()?, + pooled_device_buffer(&jobs_buffer)?, + &resources.tables.inner.vlc_table0, + &resources.tables.inner.vlc_table1, + &resources.tables.inner.uvlc_table0, + &resources.tables.inner.uvlc_table1, + pooled_device_buffer(&status_buffer)?, + kernel_jobs.len(), + CudaLaunchMode::Async, + ); + if let Err(error) = launch_result { + let _ = self.synchronize(); + return Err(error); + } + + Ok(CudaQueuedHtj2kCleanup { + resources: vec![jobs_buffer], + status_buffer: Some(status_buffer), + status_count: kernel_jobs.len(), + kernel_name: decode_kernel_name, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } + + /// Decode HTJ2K cleanup passes for multiple output buffers with one CUDA + /// dispatch and return optional host-side timing splits. + /// + /// Dequantization is left to a later dispatch. When `collect_stage_timings` + /// is false, the cleanup kernel launch is left asynchronous and the + /// mandatory status readback remains the completion point. + pub fn decode_htj2k_codeblocks_cleanup_multi_with_resources_and_pool_timed( + &self, + resources: &CudaHtj2kDecodeResources, + targets: &[CudaHtj2kCleanupTarget<'_>], + pool: &CudaBufferPool, + collect_stage_timings: bool, + ) -> Result<(CudaExecutionStats, CudaHtj2kDecodeStageTimings), CudaError> { + self.inner.set_current()?; + let kernel_jobs = htj2k_cleanup_multi_kernel_jobs(targets, resources.payload_len)?; + if kernel_jobs.is_empty() { + return Ok(( + CudaExecutionStats::default(), + CudaHtj2kDecodeStageTimings::default(), + )); + } + + let jobs_buffer = pool.upload(htj2k_cleanup_multi_jobs_as_bytes(&kernel_jobs))?; + let status_buffer = pool.take(htj2k_statuses_byte_len(kernel_jobs.len())?)?; + let (decode_kernel, decode_kernel_name) = htj2k_decode_multi_kernel_for_jobs(&kernel_jobs); + if collect_stage_timings { + self.launch_htj2k_decode_codeblocks_multi( + decode_kernel, + resources.payload.buffer()?, + pooled_device_buffer(&jobs_buffer)?, + &resources.tables.inner.vlc_table0, + &resources.tables.inner.vlc_table1, + &resources.tables.inner.uvlc_table0, + &resources.tables.inner.uvlc_table1, + pooled_device_buffer(&status_buffer)?, + kernel_jobs.len(), + CudaLaunchMode::Sync, + )?; + } else { + self.launch_htj2k_decode_codeblocks_multi( + decode_kernel, + resources.payload.buffer()?, + pooled_device_buffer(&jobs_buffer)?, + &resources.tables.inner.vlc_table0, + &resources.tables.inner.vlc_table1, + &resources.tables.inner.uvlc_table0, + &resources.tables.inner.uvlc_table1, + pooled_device_buffer(&status_buffer)?, + kernel_jobs.len(), + CudaLaunchMode::Async, + )?; + } + + let mut statuses = vec![CudaHtj2kStatus::default(); kernel_jobs.len()]; + let status_d2h_start = collect_stage_timings.then(Instant::now); + status_buffer.copy_to_host(htj2k_statuses_as_bytes_mut(&mut statuses))?; + let status_d2h_us = status_d2h_start.map_or(0, |start| start.elapsed().as_micros()); + if let Some(status) = statuses.iter().copied().find(|status| !status.is_ok()) { + return Err(CudaError::KernelStatus { + kernel: decode_kernel_name, + code: status.code, + detail: status.detail, + }); + } + + Ok(( + CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + CudaHtj2kDecodeStageTimings { + status_d2h_us, + ..CudaHtj2kDecodeStageTimings::default() + }, + )) + } + + /// Decode HTJ2K cleanup-only passes and dequantize their coefficients in + /// one CUDA dispatch. Targets containing refinement passes are rejected so + /// callers can fall back to cleanup followed by dequantization. + pub fn decode_htj2k_codeblocks_cleanup_dequantize_multi_with_resources_and_pool_timed( + &self, + resources: &CudaHtj2kDecodeResources, + targets: &[CudaHtj2kCleanupTarget<'_>], + pool: &CudaBufferPool, + collect_stage_timings: bool, + ) -> Result<(CudaExecutionStats, CudaHtj2kDecodeStageTimings), CudaError> { + self.inner.set_current()?; + let kernel_jobs = htj2k_cleanup_multi_kernel_jobs(targets, resources.payload_len)?; + if kernel_jobs.is_empty() { + return Ok(( + CudaExecutionStats::default(), + CudaHtj2kDecodeStageTimings::default(), + )); + } + let Some((decode_kernel, decode_kernel_name)) = + htj2k_decode_multi_cleanup_dequant_kernel_for_jobs(&kernel_jobs) + else { + return Err(CudaError::InvalidArgument { + message: "fused HTJ2K cleanup/dequantize requires cleanup-only jobs".to_string(), + }); + }; + + let jobs_buffer = pool.upload(htj2k_cleanup_multi_jobs_as_bytes(&kernel_jobs))?; + let status_buffer = pool.take(htj2k_statuses_byte_len(kernel_jobs.len())?)?; + if collect_stage_timings { + self.launch_htj2k_decode_codeblocks_multi( + decode_kernel, + resources.payload.buffer()?, + pooled_device_buffer(&jobs_buffer)?, + &resources.tables.inner.vlc_table0, + &resources.tables.inner.vlc_table1, + &resources.tables.inner.uvlc_table0, + &resources.tables.inner.uvlc_table1, + pooled_device_buffer(&status_buffer)?, + kernel_jobs.len(), + CudaLaunchMode::Sync, + )?; + } else { + self.launch_htj2k_decode_codeblocks_multi( + decode_kernel, + resources.payload.buffer()?, + pooled_device_buffer(&jobs_buffer)?, + &resources.tables.inner.vlc_table0, + &resources.tables.inner.vlc_table1, + &resources.tables.inner.uvlc_table0, + &resources.tables.inner.uvlc_table1, + pooled_device_buffer(&status_buffer)?, + kernel_jobs.len(), + CudaLaunchMode::Async, + )?; + } + + let mut statuses = vec![CudaHtj2kStatus::default(); kernel_jobs.len()]; + let status_d2h_start = collect_stage_timings.then(Instant::now); + status_buffer.copy_to_host(htj2k_statuses_as_bytes_mut(&mut statuses))?; + let status_d2h_us = status_d2h_start.map_or(0, |start| start.elapsed().as_micros()); + if let Some(status) = statuses.iter().copied().find(|status| !status.is_ok()) { + return Err(CudaError::KernelStatus { + kernel: decode_kernel_name, + code: status.code, + detail: status.detail, + }); + } + + Ok(( + CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + CudaHtj2kDecodeStageTimings { + status_d2h_us, + ..CudaHtj2kDecodeStageTimings::default() + }, + )) + } + + fn decode_htj2k_codeblocks_with_resources_impl( + &self, + resources: &CudaHtj2kDecodeResources, + jobs: &[CudaHtj2kCodeBlockJob], + output_words: usize, + collect_stage_timings: bool, + ) -> Result { + self.inner.set_current()?; + let output_bytes = output_words + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: output_words })?; + let coefficients = self.allocate(output_bytes)?; + if htj2k_decode_needs_zero_fill(jobs, output_words)? { + self.memset_d32(&coefficients, 0, output_words)?; + } + if jobs.is_empty() { + return Ok(CudaHtj2kDecodeOutput { + coefficients, + execution: CudaExecutionStats::default(), + statuses: Vec::new(), + stage_timings: CudaHtj2kDecodeStageTimings::default(), + }); + } + + let kernel_jobs = htj2k_kernel_jobs(jobs, resources.payload_len, output_words)?; + let jobs_buffer = self.upload(htj2k_jobs_as_bytes(&kernel_jobs))?; + let status_buffer = self.allocate(htj2k_statuses_byte_len(jobs.len())?)?; + + let has_refinement = jobs + .iter() + .any(|job| job.refinement_length > 0 || job.number_of_coding_passes > 1); + let (ht_cleanup_us, dequant_us) = self.submit_htj2k_decode_and_dequantize( + resources, + &coefficients, + &jobs_buffer, + &status_buffer, + jobs.len(), + collect_stage_timings, + )?; + + let mut statuses = vec![CudaHtj2kStatus::default(); jobs.len()]; + if let Err(error) = status_buffer.copy_to_host(htj2k_statuses_as_bytes_mut(&mut statuses)) { + if !collect_stage_timings { + let _ = self.synchronize(); + } + return Err(error); + } + if let Some(status) = statuses.iter().copied().find(|status| !status.is_ok()) { + return Err(CudaError::KernelStatus { + kernel: "j2k_htj2k_decode_codeblocks", + code: status.code, + detail: status.detail, + }); + } + + Ok(CudaHtj2kDecodeOutput { + coefficients, + execution: CudaExecutionStats { + kernel_dispatches: 2, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 2, + hardware_decode: false, + }, + statuses, + stage_timings: CudaHtj2kDecodeStageTimings { + ht_cleanup_us, + ht_refine_us: if has_refinement { ht_cleanup_us } else { 0 }, + dequant_us, + ..CudaHtj2kDecodeStageTimings::default() + }, + }) + } + + fn decode_htj2k_codeblocks_with_resources_and_pool_impl( + &self, + resources: &CudaHtj2kDecodeResources, + jobs: &[CudaHtj2kCodeBlockJob], + output_words: usize, + pool: &CudaBufferPool, + collect_stage_timings: bool, + dequantize: bool, + ) -> Result { + self.inner.set_current()?; + let output_bytes = output_words + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: output_words })?; + let coefficients = pool.take(output_bytes)?; + let coefficient_buffer = pooled_device_buffer(&coefficients)?; + if htj2k_decode_needs_zero_fill(jobs, output_words)? { + self.memset_d32(coefficient_buffer, 0, output_words)?; + } + if jobs.is_empty() { + return Ok(CudaPooledHtj2kDecodeOutput { + coefficients, + execution: CudaExecutionStats::default(), + statuses: Vec::new(), + stage_timings: CudaHtj2kDecodeStageTimings::default(), + }); + } + + let kernel_jobs = htj2k_kernel_jobs(jobs, resources.payload_len, output_words)?; + let jobs_buffer = pool.upload(htj2k_jobs_as_bytes(&kernel_jobs))?; + let status_buffer = pool.take(htj2k_statuses_byte_len(jobs.len())?)?; + + let has_refinement = jobs + .iter() + .any(|job| job.refinement_length > 0 || job.number_of_coding_passes > 1); + let jobs_device = pooled_device_buffer(&jobs_buffer)?; + let status_device = pooled_device_buffer(&status_buffer)?; + let (ht_cleanup_us, dequant_us, kernel_dispatches) = if dequantize { + let (ht_cleanup_us, dequant_us) = self.submit_htj2k_decode_and_dequantize( + resources, + coefficient_buffer, + jobs_device, + status_device, + jobs.len(), + collect_stage_timings, + )?; + (ht_cleanup_us, dequant_us, 2) + } else { + let ht_cleanup_us = self.submit_htj2k_decode_cleanup( + resources, + coefficient_buffer, + jobs_device, + status_device, + jobs.len(), + collect_stage_timings, + )?; + (ht_cleanup_us, 0, 1) + }; + + let mut statuses = vec![CudaHtj2kStatus::default(); jobs.len()]; + if let Err(error) = status_buffer.copy_to_host(htj2k_statuses_as_bytes_mut(&mut statuses)) { + if !collect_stage_timings { + let _ = self.synchronize(); + } + return Err(error); + } + if let Some(status) = statuses.iter().copied().find(|status| !status.is_ok()) { + return Err(CudaError::KernelStatus { + kernel: "j2k_htj2k_decode_codeblocks", + code: status.code, + detail: status.detail, + }); + } + + Ok(CudaPooledHtj2kDecodeOutput { + coefficients, + execution: CudaExecutionStats { + kernel_dispatches, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: kernel_dispatches, + hardware_decode: false, + }, + statuses, + stage_timings: CudaHtj2kDecodeStageTimings { + ht_cleanup_us, + ht_refine_us: if has_refinement { ht_cleanup_us } else { 0 }, + dequant_us, + ..CudaHtj2kDecodeStageTimings::default() + }, + }) + } + + fn submit_htj2k_decode_and_dequantize( + &self, + resources: &CudaHtj2kDecodeResources, + coefficients: &CudaDeviceBuffer, + jobs_buffer: &CudaDeviceBuffer, + status_buffer: &CudaDeviceBuffer, + job_count: usize, + collect_stage_timings: bool, + ) -> Result<(u128, u128), CudaError> { + let ht_cleanup_us = self.submit_htj2k_decode_cleanup( + resources, + coefficients, + jobs_buffer, + status_buffer, + job_count, + collect_stage_timings, + )?; + let dequant_us = self.submit_htj2k_dequantize_htj2k_codeblocks( + coefficients, + jobs_buffer, + job_count, + collect_stage_timings, + )?; + Ok((ht_cleanup_us, dequant_us)) + } + + fn submit_htj2k_decode_cleanup( + &self, + resources: &CudaHtj2kDecodeResources, + coefficients: &CudaDeviceBuffer, + jobs_buffer: &CudaDeviceBuffer, + status_buffer: &CudaDeviceBuffer, + job_count: usize, + collect_stage_timings: bool, + ) -> Result { + let ((), ht_cleanup_us) = self.time_default_stream_named_us_if( + collect_stage_timings, + "j2k.htj2k.decode.cleanup", + || { + if !collect_stage_timings { + return self.launch_htj2k_decode_codeblocks( + resources.payload.buffer()?, + coefficients, + jobs_buffer, + &resources.tables.inner.vlc_table0, + &resources.tables.inner.vlc_table1, + &resources.tables.inner.uvlc_table0, + &resources.tables.inner.uvlc_table1, + status_buffer, + job_count, + CudaLaunchMode::Async, + ); + } + self.launch_htj2k_decode_codeblocks( + resources.payload.buffer()?, + coefficients, + jobs_buffer, + &resources.tables.inner.vlc_table0, + &resources.tables.inner.vlc_table1, + &resources.tables.inner.uvlc_table0, + &resources.tables.inner.uvlc_table1, + status_buffer, + job_count, + CudaLaunchMode::Sync, + ) + }, + )?; + Ok(ht_cleanup_us) + } + + fn submit_htj2k_dequantize_htj2k_codeblocks( + &self, + coefficients: &CudaDeviceBuffer, + jobs_buffer: &CudaDeviceBuffer, + job_count: usize, + collect_stage_timings: bool, + ) -> Result { + let ((), dequant_us) = match self.time_default_stream_named_us_if( + collect_stage_timings, + "j2k.htj2k.decode.dequantize", + || { + if collect_stage_timings { + self.launch_j2k_dequantize_htj2k_codeblocks( + coefficients, + jobs_buffer, + job_count, + CudaLaunchMode::Sync, + ) + } else { + self.launch_j2k_dequantize_htj2k_codeblocks( + coefficients, + jobs_buffer, + job_count, + CudaLaunchMode::Async, + ) + } + }, + ) { + Ok(result) => result, + Err(error) => { + if !collect_stage_timings { + let _ = self.synchronize(); + } + return Err(error); + } + }; + Ok(dequant_us) + } + + /// Dequantize HTJ2K code-block outputs that live in multiple device buffers + /// with one CUDA dispatch. + pub fn j2k_dequantize_htj2k_codeblocks_multi_device( + &self, + targets: &[CudaHtj2kDequantizeTarget<'_>], + ) -> Result { + let pool = self.buffer_pool(); + self.j2k_dequantize_htj2k_codeblocks_multi_device_with_pool(targets, &pool) + } + + /// Dequantize HTJ2K code-block outputs that live in multiple device buffers + /// with one CUDA dispatch, reusing caller-owned transient storage. + pub fn j2k_dequantize_htj2k_codeblocks_multi_device_with_pool( + &self, + targets: &[CudaHtj2kDequantizeTarget<'_>], + pool: &CudaBufferPool, + ) -> Result { + self.j2k_dequantize_htj2k_codeblocks_multi_device_with_pool_impl(targets, pool, true) + } + + /// Dequantize HTJ2K code-block outputs in multiple device buffers without + /// CUDA event timings. The launch is still synchronized before returning + /// so the pooled job upload cannot be reused while the kernel reads it. + pub fn j2k_dequantize_htj2k_codeblocks_multi_device_untimed_with_pool( + &self, + targets: &[CudaHtj2kDequantizeTarget<'_>], + pool: &CudaBufferPool, + ) -> Result { + self.j2k_dequantize_htj2k_codeblocks_multi_device_with_pool_impl(targets, pool, true) + } + + fn j2k_dequantize_htj2k_codeblocks_multi_device_with_pool_impl( + &self, + targets: &[CudaHtj2kDequantizeTarget<'_>], + pool: &CudaBufferPool, + synchronize_each_launch: bool, + ) -> Result { + self.inner.set_current()?; + let kernel_jobs = htj2k_dequantize_kernel_jobs(targets)?; + if kernel_jobs.is_empty() { + return Ok(CudaExecutionStats::default()); + } + let jobs_buffer = pool.upload(htj2k_dequantize_jobs_as_bytes(&kernel_jobs))?; + self.launch_j2k_dequantize_htj2k_codeblocks_multi( + pooled_device_buffer(&jobs_buffer)?, + kernel_jobs.len(), + CudaLaunchMode::from_synchronize(synchronize_each_launch), + )?; + Ok(CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }) + } + + #[allow(clippy::similar_names, clippy::too_many_arguments)] + fn launch_htj2k_decode_codeblocks( + &self, + payload: &CudaDeviceBuffer, + coefficients: &CudaDeviceBuffer, + jobs: &CudaDeviceBuffer, + vlc_table0: &CudaDeviceBuffer, + vlc_table1: &CudaDeviceBuffer, + uvlc_table0: &CudaDeviceBuffer, + uvlc_table1: &CudaDeviceBuffer, + statuses: &CudaDeviceBuffer, + job_count: usize, + mode: CudaLaunchMode, + ) -> Result<(), CudaError> { + let mut payload_ptr = payload.device_ptr(); + let mut coefficients_ptr = coefficients.device_ptr(); + let mut jobs_ptr = jobs.device_ptr(); + let mut vlc_table0_ptr = vlc_table0.device_ptr(); + let mut vlc_table1_ptr = vlc_table1.device_ptr(); + let mut uvlc_table0_ptr = uvlc_table0.device_ptr(); + let mut uvlc_table1_ptr = uvlc_table1.device_ptr(); + let mut statuses_ptr = statuses.device_ptr(); + let mut job_count = c_uint::try_from(job_count) + .map_err(|_| CudaError::LengthTooLarge { len: job_count })?; + let mut params = cuda_kernel_params!( + payload_ptr, + coefficients_ptr, + jobs_ptr, + vlc_table0_ptr, + vlc_table1_ptr, + uvlc_table0_ptr, + uvlc_table1_ptr, + statuses_ptr, + job_count + ); + let geometry = htj2k_codeblock_launch_geometry(job_count as usize).ok_or( + CudaError::LengthTooLarge { + len: job_count as usize, + }, + )?; + + self.launch_named_kernel( + CudaKernel::Htj2kDecodeCodeblocks, + geometry, + &mut params, + mode, + ) + } + + #[allow(clippy::similar_names, clippy::too_many_arguments)] + fn launch_htj2k_decode_codeblocks_multi( + &self, + kernel: CudaKernel, + payload: &CudaDeviceBuffer, + jobs: &CudaDeviceBuffer, + vlc_table0: &CudaDeviceBuffer, + vlc_table1: &CudaDeviceBuffer, + uvlc_table0: &CudaDeviceBuffer, + uvlc_table1: &CudaDeviceBuffer, + statuses: &CudaDeviceBuffer, + job_count: usize, + mode: CudaLaunchMode, + ) -> Result<(), CudaError> { + let mut payload_ptr = payload.device_ptr(); + let mut jobs_ptr = jobs.device_ptr(); + let mut vlc_table0_ptr = vlc_table0.device_ptr(); + let mut vlc_table1_ptr = vlc_table1.device_ptr(); + let mut uvlc_table0_ptr = uvlc_table0.device_ptr(); + let mut uvlc_table1_ptr = uvlc_table1.device_ptr(); + let mut statuses_ptr = statuses.device_ptr(); + let mut job_count = c_uint::try_from(job_count) + .map_err(|_| CudaError::LengthTooLarge { len: job_count })?; + let mut params = cuda_kernel_params!( + payload_ptr, + jobs_ptr, + vlc_table0_ptr, + vlc_table1_ptr, + uvlc_table0_ptr, + uvlc_table1_ptr, + statuses_ptr, + job_count + ); + let geometry = htj2k_codeblock_launch_geometry(job_count as usize).ok_or( + CudaError::LengthTooLarge { + len: job_count as usize, + }, + )?; + + self.launch_named_kernel(kernel, geometry, &mut params, mode) + } + + fn launch_j2k_dequantize_htj2k_codeblocks( + &self, + coefficients: &CudaDeviceBuffer, + jobs: &CudaDeviceBuffer, + job_count: usize, + mode: CudaLaunchMode, + ) -> Result<(), CudaError> { + let mut coefficients_ptr = coefficients.device_ptr(); + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(coefficients_ptr, jobs_ptr); + let geometry = htj2k_codeblock_sample_launch_geometry(job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + + self.launch_j2k_dequantize_kernel( + CudaKernel::J2kDequantizeHtj2kCodeblocks, + geometry, + &mut params, + mode, + ) + } + + fn launch_j2k_dequantize_htj2k_codeblocks_multi( + &self, + jobs: &CudaDeviceBuffer, + job_count: usize, + mode: CudaLaunchMode, + ) -> Result<(), CudaError> { + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = htj2k_codeblock_sample_launch_geometry(job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + + self.launch_j2k_dequantize_kernel( + CudaKernel::J2kDequantizeHtj2kCodeblocksMulti, + geometry, + &mut params, + mode, + ) + } + + pub(crate) fn launch_j2k_dequantize_htj2k_cleanup_jobs_multi( + &self, + jobs: &CudaDeviceBuffer, + job_count: usize, + mode: CudaLaunchMode, + ) -> Result<(), CudaError> { + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = htj2k_codeblock_sample_launch_geometry(job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + + self.launch_j2k_dequantize_kernel( + CudaKernel::J2kDequantizeHtj2kCleanupJobsMulti, + geometry, + &mut params, + mode, + ) + } + + fn launch_j2k_dequantize_kernel( + &self, + kernel: CudaKernel, + geometry: CudaLaunchGeometry, + params: &mut [*mut std::ffi::c_void; N], + mode: CudaLaunchMode, + ) -> Result<(), CudaError> { + let function = self.j2k_dequantize_kernel_function(kernel)?; + match mode { + CudaLaunchMode::Sync => self.launch_kernel(function, geometry, params), + CudaLaunchMode::Async => self.launch_kernel_async(function, geometry, params), + } + } + + fn j2k_dequantize_kernel_function( + &self, + kernel: CudaKernel, + ) -> Result { + #[cfg(feature = "cuda-oxide-j2k-dequantize")] + { + if crate::build_flags::cuda_oxide_j2k_dequantize_enabled() { + return self.inner.cuda_oxide_j2k_dequantize_kernel_function(kernel); + } + } + self.inner.kernel_function(kernel) + } +} + +/// Enqueued HTJ2K cleanup work plus pooled resources/statuses that must stay +/// live until `finish` validates kernel completion. +#[derive(Debug)] +pub struct CudaQueuedHtj2kCleanup { + pub(crate) resources: Vec, + pub(crate) status_buffer: Option, + pub(crate) status_count: usize, + pub(crate) kernel_name: &'static str, + pub(crate) execution: CudaExecutionStats, +} + +impl CudaQueuedHtj2kCleanup { + /// CUDA execution counters for the enqueued cleanup work. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// Number of pooled resource buffers held live for the queued cleanup work. + pub fn resource_count(&self) -> usize { + self.resources.len() + usize::from(self.status_buffer.is_some()) + } + + /// Synchronize through status download and validate kernel statuses. + pub fn finish(self) -> Result { + let Some(status_buffer) = self.status_buffer else { + return Ok(self.execution); + }; + + let mut statuses = vec![CudaHtj2kStatus::default(); self.status_count]; + status_buffer.copy_to_host(htj2k_statuses_as_bytes_mut(&mut statuses))?; + if let Some(status) = statuses.iter().copied().find(|status| !status.is_ok()) { + return Err(CudaError::KernelStatus { + kernel: self.kernel_name, + code: status.code, + detail: status.detail, + }); + } + + Ok(self.execution) + } +} + +pub(crate) fn htj2k_kernel_jobs( + jobs: &[CudaHtj2kCodeBlockJob], + payload_len: usize, + output_words: usize, +) -> Result, CudaError> { + jobs.iter() + .map(|job| { + let payload_offset = usize::try_from(job.payload_offset) + .map_err(|_| CudaError::LengthTooLarge { len: usize::MAX })?; + let payload_end = payload_offset + .checked_add(job.payload_len as usize) + .ok_or(CudaError::LengthTooLarge { len: payload_len })?; + let expected_payload_len = job + .cleanup_length + .checked_add(job.refinement_length) + .ok_or(CudaError::LengthTooLarge { + len: job.payload_len as usize, + })?; + let output_stride = job.output_stride as usize; + let output_offset = job.output_offset as usize; + let output_end = if job.height == 0 { + output_offset + } else { + output_offset + .checked_add( + output_stride + .checked_mul(job.height as usize - 1) + .ok_or(CudaError::LengthTooLarge { len: output_words })?, + ) + .and_then(|last_row| last_row.checked_add(job.width as usize)) + .ok_or(CudaError::LengthTooLarge { len: output_words })? + }; + if payload_end > payload_len + || expected_payload_len != job.payload_len + || output_end > output_words + { + return Err(CudaError::LengthTooLarge { + len: payload_len.max(output_words), + }); + } + Ok(CudaHtj2kCodeBlockKernelJob { + coded_offset: u32::try_from(payload_offset) + .map_err(|_| CudaError::LengthTooLarge { len: payload_len })?, + width: job.width, + height: job.height, + coded_len: job.payload_len, + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + missing_msbs: u32::from(job.missing_bit_planes), + num_bitplanes: u32::from(job.num_bitplanes), + number_of_coding_passes: u32::from(job.number_of_coding_passes), + output_stride: job.output_stride, + output_offset: job.output_offset, + dequantization_step: job.dequantization_step, + stripe_causal: u32::from(job.stripe_causal), + }) + }) + .collect() +} + +pub(crate) fn htj2k_dequantize_kernel_jobs( + targets: &[CudaHtj2kDequantizeTarget<'_>], +) -> Result, CudaError> { + let total_jobs = targets + .iter() + .try_fold(0usize, |count, target| count.checked_add(target.jobs.len())) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + let mut kernel_jobs = Vec::with_capacity(total_jobs); + for target in targets { + let output_bytes = target + .output_words + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { + len: target.output_words, + })?; + if output_bytes > target.coefficients.byte_len() { + return Err(CudaError::LengthTooLarge { len: output_bytes }); + } + for job in target.jobs { + let output_stride = job.output_stride as usize; + let output_offset = job.output_offset as usize; + let output_end = if job.height == 0 { + output_offset + } else { + output_offset + .checked_add(output_stride.checked_mul(job.height as usize - 1).ok_or( + CudaError::LengthTooLarge { + len: target.output_words, + }, + )?) + .and_then(|last_row| last_row.checked_add(job.width as usize)) + .ok_or(CudaError::LengthTooLarge { + len: target.output_words, + })? + }; + if output_end > target.output_words { + return Err(CudaError::LengthTooLarge { + len: target.output_words, + }); + } + kernel_jobs.push(CudaHtj2kDequantizeKernelJob { + output_ptr: target.coefficients.device_ptr(), + width: job.width, + height: job.height, + output_stride: job.output_stride, + output_offset: job.output_offset, + num_bitplanes: u32::from(job.num_bitplanes), + reserved: 0, + dequantization_step: job.dequantization_step, + }); + } + } + Ok(kernel_jobs) +} + +pub(crate) fn htj2k_cleanup_multi_kernel_jobs( + targets: &[CudaHtj2kCleanupTarget<'_>], + payload_len: usize, +) -> Result, CudaError> { + let total_jobs = targets + .iter() + .try_fold(0usize, |count, target| count.checked_add(target.jobs.len())) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + let mut kernel_jobs = Vec::with_capacity(total_jobs); + for target in targets { + let output_bytes = target + .output_words + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { + len: target.output_words, + })?; + if output_bytes > target.coefficients.byte_len() { + return Err(CudaError::LengthTooLarge { len: output_bytes }); + } + for job in htj2k_kernel_jobs(target.jobs, payload_len, target.output_words)? { + kernel_jobs.push(CudaHtj2kCleanupMultiKernelJob { + output_ptr: target.coefficients.device_ptr(), + coded_offset: job.coded_offset, + width: job.width, + height: job.height, + coded_len: job.coded_len, + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + missing_msbs: job.missing_msbs, + num_bitplanes: job.num_bitplanes, + number_of_coding_passes: job.number_of_coding_passes, + output_stride: job.output_stride, + output_offset: job.output_offset, + dequantization_step: job.dequantization_step, + stripe_causal: job.stripe_causal, + }); + } + } + Ok(kernel_jobs) +} + +pub(crate) fn htj2k_decode_multi_kernel_for_jobs( + jobs: &[CudaHtj2kCleanupMultiKernelJob], +) -> (CudaKernel, &'static str) { + let cleanup_only = jobs + .iter() + .all(|job| job.refinement_length == 0 && job.number_of_coding_passes <= 1); + if cleanup_only { + ( + CudaKernel::Htj2kDecodeCodeblocksMultiCleanupOnly, + "j2k_htj2k_decode_codeblocks_multi_cleanup_only", + ) + } else { + ( + CudaKernel::Htj2kDecodeCodeblocksMulti, + "j2k_htj2k_decode_codeblocks_multi", + ) + } +} + +pub(crate) fn htj2k_decode_multi_cleanup_dequant_kernel_for_jobs( + jobs: &[CudaHtj2kCleanupMultiKernelJob], +) -> Option<(CudaKernel, &'static str)> { + let cleanup_only = jobs + .iter() + .all(|job| job.refinement_length == 0 && job.number_of_coding_passes <= 1); + cleanup_only.then_some(( + CudaKernel::Htj2kDecodeCodeblocksMultiCleanupDequantize, + "j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize", + )) +} + +pub(crate) fn htj2k_decode_needs_zero_fill( + jobs: &[CudaHtj2kCodeBlockJob], + output_words: usize, +) -> Result { + let mut covered_words = 0usize; + for job in jobs { + let area = (job.width as usize) + .checked_mul(job.height as usize) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + covered_words = covered_words + .checked_add(area) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + } + if covered_words > output_words { + return Err(CudaError::LengthTooLarge { len: covered_words }); + } + Ok(covered_words != output_words) +} diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode_kernels.cu b/crates/j2k-cuda-runtime/src/htj2k_decode_kernels.cu new file mode 100644 index 00000000..374ce572 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_decode_kernels.cu @@ -0,0 +1,2795 @@ +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +typedef unsigned char uchar; +typedef unsigned short ushort; +typedef unsigned int uint; +typedef unsigned long long j2k_ulong; + +__device__ inline uint j2k_min_u32(uint a, uint b) { return a < b ? a : b; } +__device__ inline int j2k_max_i32(int a, int b) { return a > b ? a : b; } + +struct J2kHtCleanupParams { + uint width; + uint height; + uint coded_len; + uint cleanup_length; + uint refinement_length; + uint missing_msbs; + uint num_bitplanes; + uint number_of_coding_passes; + uint output_stride; + uint output_offset; + float dequantization_step; + uint stripe_causal; +}; + +struct J2kHtCleanupBatchJob { + uint coded_offset; + uint width; + uint height; + uint coded_len; + uint cleanup_length; + uint refinement_length; + uint missing_msbs; + uint num_bitplanes; + uint number_of_coding_passes; + uint output_stride; + uint output_offset; + float dequantization_step; + uint stripe_causal; +}; + +struct J2kHtCleanupMultiBatchJob { + j2k_ulong output_ptr; + uint coded_offset; + uint width; + uint height; + uint coded_len; + uint cleanup_length; + uint refinement_length; + uint missing_msbs; + uint num_bitplanes; + uint number_of_coding_passes; + uint output_stride; + uint output_offset; + float dequantization_step; + uint stripe_causal; +}; + +struct J2kHtRepeatedBatchParams { + uint job_count; + uint output_plane_len; + uint batch_count; +}; + +struct J2kHtStatus { + uint code; + uint detail; + uint reserved0; + uint reserved1; +}; + +struct J2kHtDequantizeJob { + j2k_ulong output_ptr; + uint width; + uint height; + uint output_stride; + uint output_offset; + uint num_bitplanes; + uint reserved; + float dequantization_step; +}; + +static constexpr uint J2K_HT_STATUS_OK = 0u; +static constexpr uint J2K_HT_STATUS_FAIL = 1u; +static constexpr uint J2K_HT_STATUS_UNSUPPORTED = 2u; + +static constexpr uint J2K_HT_MAX_WIDTH = 256u; +static constexpr uint J2K_HT_MAX_HEIGHT = 256u; +static constexpr uint J2K_HT_MAX_COEFFICIENTS = 4096u; +static constexpr uint J2K_HT_MAX_SSTR = 264u; +static constexpr uint J2K_HT_MAX_SCRATCH = 3096u; +static constexpr uint J2K_HT_MAX_VN = 130u; +static constexpr uint J2K_HT_MAX_MSTR = 72u; +static constexpr uint J2K_HT_MAX_SIGMA = 528u; +static constexpr uint J2K_HT_MAX_PREV_ROW_SIG = 72u; + +__device__ inline void set_ht_status(J2kHtStatus *status, uint code, uint detail) { + status->code = code; + status->detail = detail; + status->reserved0 = 0u; + status->reserved1 = 0u; +} + +struct MelDecoder { + const uchar *data; + uint pos; + uint remaining; + bool unstuff; + uchar current_byte; + uchar bits_left; + uint k; + uint num_runs; + j2k_ulong runs; +}; + +__device__ inline MelDecoder mel_decoder_new(const uchar *data, uint lcup, uint scup) { + MelDecoder decoder; + decoder.data = data; + decoder.pos = lcup - scup; + decoder.remaining = scup - 1u; + decoder.unstuff = false; + decoder.current_byte = 0; + decoder.bits_left = 0; + decoder.k = 0u; + decoder.num_runs = 0u; + decoder.runs = 0u; + return decoder; +} + +__device__ inline bool mel_read_bit(MelDecoder &decoder, uint &bit) { + if (decoder.bits_left == 0u) { + uchar byte = decoder.remaining > 0u ? decoder.data[decoder.pos] : uchar(0xFF); + if (decoder.remaining > 0u) { + decoder.pos += 1u; + decoder.remaining -= 1u; + } + if (decoder.remaining == 0u) { + byte |= uchar(0x0F); + } + decoder.current_byte = byte; + decoder.bits_left = uchar(8u - uint(decoder.unstuff)); + decoder.unstuff = byte == uchar(0xFF); + } + + decoder.bits_left -= 1u; + bit = uint((decoder.current_byte >> decoder.bits_left) & uchar(1)); + return true; +} + +__device__ inline bool mel_read_bits(MelDecoder &decoder, uint count, uint &value) { + value = 0u; + for (uint idx = 0u; idx < count; ++idx) { + uint bit = 0u; + if (!mel_read_bit(decoder, bit)) { + return false; + } + value = (value << 1u) | bit; + } + return true; +} + +__device__ inline bool mel_decode_more_runs(MelDecoder &decoder) { + static constexpr uint MEL_EXP[13] = {0u, 0u, 0u, 1u, 1u, 1u, 2u, 2u, 2u, 3u, 3u, 4u, 5u}; + + while (decoder.num_runs < 8u) { + const uint eval = MEL_EXP[decoder.k]; + uint first = 0u; + if (!mel_read_bit(decoder, first)) { + return false; + } + + uint run = 0u; + if (first == 1u) { + decoder.k = j2k_min_u32(decoder.k + 1u, 12u); + run = ((1u << eval) - 1u) << 1u; + } else { + decoder.k = decoder.k == 0u ? 0u : decoder.k - 1u; + uint bits = 0u; + if (!mel_read_bits(decoder, eval, bits)) { + return false; + } + run = (bits << 1u) | 1u; + } + + decoder.runs |= (j2k_ulong(run) << (decoder.num_runs * 7u)); + decoder.num_runs += 1u; + + if (eval == 5u && first == 0u && decoder.num_runs >= 8u) { + break; + } + } + + return true; +} + +__device__ inline bool mel_get_run(MelDecoder &decoder, int &run) { + if (decoder.num_runs == 0u && !mel_decode_more_runs(decoder)) { + return false; + } + + run = int(decoder.runs & 0x7Ful); + decoder.runs >>= 7u; + decoder.num_runs -= 1u; + return true; +} + +struct ForwardBitReader { + const uchar *data; + uint data_len; + uint pos; + j2k_ulong tmp; + uint bits; + bool unstuff; + uchar pad; +}; + +__device__ inline ForwardBitReader forward_reader_new(const uchar *data, uint data_len, uchar pad) { + ForwardBitReader reader; + reader.data = data; + reader.data_len = data_len; + reader.pos = 0u; + reader.tmp = 0ul; + reader.bits = 0u; + reader.unstuff = false; + reader.pad = pad; + return reader; +} + +__device__ inline void forward_reader_fill(ForwardBitReader &reader) { + while (reader.bits <= 32u) { + const uchar byte = reader.pos < reader.data_len ? reader.data[reader.pos++] : reader.pad; + reader.tmp |= (j2k_ulong(byte) << reader.bits); + reader.bits += 8u - uint(reader.unstuff); + reader.unstuff = byte == uchar(0xFF); + } +} + +__device__ inline uint forward_reader_fetch(ForwardBitReader &reader) { + if (reader.bits < 32u) { + forward_reader_fill(reader); + } + return uint(reader.tmp); +} + +__device__ inline void forward_reader_advance(ForwardBitReader &reader, uint count) { + reader.tmp >>= count; + reader.bits -= count; +} + +struct ReverseBitReader { + const uchar *data; + int pos; + uint remaining; + j2k_ulong tmp; + uint bits; + bool unstuff; +}; + +__device__ inline ReverseBitReader reverse_reader_new_vlc( + const uchar *data, + uint lcup, + uint scup +) { + const uchar d = data[lcup - 2u]; + const j2k_ulong tmp = j2k_ulong(d >> 4); + + ReverseBitReader reader; + reader.data = data; + reader.pos = int(lcup) - 3; + reader.remaining = scup - 2u; + reader.tmp = tmp; + reader.bits = 4u - uint((tmp & 0x7ul) == 0x7ul); + reader.unstuff = (d | uchar(0x0F)) > uchar(0x8F); + return reader; +} + +__device__ inline ReverseBitReader reverse_reader_new_mrp( + const uchar *data, + uint lcup, + uint len2 +) { + ReverseBitReader reader; + reader.data = data; + reader.pos = int(lcup + len2) - 1; + reader.remaining = len2; + reader.tmp = 0ul; + reader.bits = 0u; + reader.unstuff = true; + return reader; +} + +__device__ inline void reverse_reader_fill(ReverseBitReader &reader) { + while (reader.bits <= 32u) { + const uchar byte = reader.remaining > 0u ? reader.data[reader.pos] : uchar(0u); + if (reader.remaining > 0u) { + reader.pos -= 1; + reader.remaining -= 1u; + } + const uint d_bits = 8u - uint(reader.unstuff && (byte & uchar(0x7F)) == uchar(0x7F)); + reader.tmp |= (j2k_ulong(byte) << reader.bits); + reader.bits += d_bits; + reader.unstuff = byte > uchar(0x8F); + } +} + +__device__ inline uint reverse_reader_fetch(ReverseBitReader &reader) { + if (reader.bits < 32u) { + reverse_reader_fill(reader); + } + return uint(reader.tmp); +} + +__device__ inline uint reverse_reader_advance(ReverseBitReader &reader, uint count) { + reader.tmp >>= count; + reader.bits -= count; + return uint(reader.tmp); +} + +__device__ inline uint read_u32_pair(const ushort *values, uint index) { + return uint(values[index]) | (uint(values[index + 1u]) << 16u); +} + +__device__ inline uint sample_mask(uint bit) { + return 1u << (4u + bit); +} + +__device__ inline int coefficient_to_i32(uint value, uint k_max) { + const uint shift = 31u - k_max; + const int magnitude = int((value & 0x7FFF'FFFFu) >> shift); + return (value & 0x8000'0000u) != 0u ? -magnitude : magnitude; +} + +__device__ inline float coefficient_to_float(uint value, uint k_max, float scale) { + return float(coefficient_to_i32(value, k_max)) * scale; +} + +__device__ inline uint coefficient_to_float_bits(uint value, uint k_max, float scale) { + return __float_as_uint(coefficient_to_float(value, k_max, scale)); +} + +template +__device__ inline uint decoded_cleanup_sample_bits(uint value, const J2kHtCleanupParams ¶ms) { + if (DEQUANTIZE) { + return coefficient_to_float_bits(value, params.num_bitplanes, params.dequantization_step); + } + return value; +} + +template +__device__ inline void store_decoded_cleanup_sample( + uint *decoded_data, + uint index, + uint value, + const J2kHtCleanupParams ¶ms +) { + decoded_data[index] = decoded_cleanup_sample_bits(value, params); +} + +__device__ inline void decode_mag_sgn_sample_with_vn( + ForwardBitReader &magsgn, + uint inf, + uint bit, + uint uq, + uint p, + uint &value, + uint &v_n +) { + if ((inf & sample_mask(bit)) == 0u) { + value = 0u; + v_n = 0u; + return; + } + + const uint ms_val = forward_reader_fetch(magsgn); + const uint m_n = uq - ((inf >> (12u + bit)) & 1u); + forward_reader_advance(magsgn, m_n); + + value = ms_val << 31u; + const uint mask = m_n == 0u ? 0u : (1u << m_n) - 1u; + v_n = ms_val & mask; + v_n |= ((inf >> (8u + bit)) & 1u) << m_n; + v_n |= 1u; + value |= (v_n + 2u) << (p - 1u); +} + +template +__device__ inline void decode_ht_cleanup_impl( + const uchar *coded_data, + uint *decoded_data, + J2kHtCleanupParams params, + const ushort *vlc_table0, + const ushort *vlc_table1, + const ushort *uvlc_table0, + const ushort *uvlc_table1, + J2kHtStatus *status +) { + set_ht_status(status, J2K_HT_STATUS_OK, 0u); + + uint num_passes = params.number_of_coding_passes; + if (num_passes > 1u && params.refinement_length == 0u) { + num_passes = 1u; + } + if (CLEANUP_ONLY && params.refinement_length != 0u) { + set_ht_status(status, J2K_HT_STATUS_UNSUPPORTED, 17u); + return; + } + if (DEQUANTIZE && (!CLEANUP_ONLY || params.number_of_coding_passes > 1u || + params.refinement_length != 0u)) { + set_ht_status(status, J2K_HT_STATUS_UNSUPPORTED, 18u); + return; + } + + if (params.width == 0u || params.height == 0u) { + return; + } + if (params.width > J2K_HT_MAX_WIDTH || params.height > J2K_HT_MAX_HEIGHT || + params.width * params.height > J2K_HT_MAX_COEFFICIENTS) { + set_ht_status(status, J2K_HT_STATUS_UNSUPPORTED, 1u); + return; + } + if (params.num_bitplanes == 0u || params.num_bitplanes > 31u) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 2u); + return; + } + if (num_passes > 3u || params.missing_msbs > 30u || params.missing_msbs == 30u) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 3u); + return; + } + if (params.missing_msbs == 29u && num_passes > 1u) { + num_passes = 1u; + } + + const uint lcup = params.cleanup_length; + if (lcup < 2u || params.coded_len < lcup + params.refinement_length) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 4u); + return; + } + + const uint scup = (uint(coded_data[lcup - 1u]) << 4u) + uint(coded_data[lcup - 2u] & uchar(0x0F)); + if (scup < 2u || scup > lcup || scup > 4079u) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 5u); + return; + } + + const uint width = params.width; + const uint height = params.height; + const uint stride = params.output_stride; + const uint quad_rows = (height + 1u) / 2u; + const uint sstr = (width + 9u) & ~7u; + if (sstr > J2K_HT_MAX_SSTR || sstr * (quad_rows + 1u) > J2K_HT_MAX_SCRATCH) { + set_ht_status(status, J2K_HT_STATUS_UNSUPPORTED, 6u); + return; + } + + ushort scratch[J2K_HT_MAX_SCRATCH]; + uint v_n_scratch[J2K_HT_MAX_VN]; + + { + MelDecoder mel = mel_decoder_new(coded_data, lcup, scup); + ReverseBitReader vlc = reverse_reader_new_vlc(coded_data, lcup, scup); + int run = 0; + if (!mel_get_run(mel, run)) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 6u); + return; + } + + uint c_q = 0u; + uint row_offset = 0u; + uint x = 0u; + + while (x < width) { + uint vlc_val = reverse_reader_fetch(vlc); + uint t0 = uint(vlc_table0[c_q + (vlc_val & 0x7Fu)]); + if (c_q == 0u) { + run -= 2; + t0 = run == -1 ? t0 : 0u; + if (run < 0 && !mel_get_run(mel, run)) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 7u); + return; + } + } + scratch[row_offset] = ushort(t0); + x += 2u; + c_q = ((t0 & 0x10u) << 3u) | ((t0 & 0xE0u) << 2u); + vlc_val = reverse_reader_advance(vlc, t0 & 0x7u); + + uint t1 = uint(vlc_table0[c_q + (vlc_val & 0x7Fu)]); + if (c_q == 0u && x < width) { + run -= 2; + t1 = run == -1 ? t1 : 0u; + if (run < 0 && !mel_get_run(mel, run)) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 8u); + return; + } + } + if (x >= width) { + t1 = 0u; + } + scratch[row_offset + 2u] = ushort(t1); + x += 2u; + c_q = ((t1 & 0x10u) << 3u) | ((t1 & 0xE0u) << 2u); + vlc_val = reverse_reader_advance(vlc, t1 & 0x7u); + + uint uvlc_mode = ((t0 & 0x8u) << 3u) | ((t1 & 0x8u) << 4u); + if (uvlc_mode == 0xC0u) { + run -= 2; + if (run == -1) { + uvlc_mode += 0x40u; + } + if (run < 0 && !mel_get_run(mel, run)) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 9u); + return; + } + } + + uint uvlc_entry = uint(uvlc_table0[uvlc_mode + (vlc_val & 0x3Fu)]); + vlc_val = reverse_reader_advance(vlc, uvlc_entry & 0x7u); + uvlc_entry >>= 3u; + uint len = uvlc_entry & 0xFu; + const uint tmp = vlc_val & ((1u << len) - 1u); + vlc_val = reverse_reader_advance(vlc, len); + uvlc_entry >>= 4u; + len = uvlc_entry & 0x7u; + uvlc_entry >>= 3u; + scratch[row_offset + 1u] = ushort(1u + (uvlc_entry & 0x7u) + (tmp & ~(0xFFu << len))); + scratch[row_offset + 3u] = ushort(1u + (uvlc_entry >> 3u) + (tmp >> len)); + + row_offset += 4u; + } + scratch[row_offset] = 0u; + scratch[row_offset + 1u] = 0u; + + for (uint y = 2u; y < height; y += 2u) { + const uint row_base = (y >> 1u) * sstr; + const uint prev_base = row_base - sstr; + uint local_x = 0u; + uint local_c_q = 0u; + uint local_row_offset = row_base; + + while (local_x < width) { + local_c_q |= (uint(scratch[prev_base + (local_row_offset - row_base)]) & 0xA0u) << 2u; + local_c_q |= (uint(scratch[prev_base + (local_row_offset - row_base) + 2u]) & 0x20u) << 4u; + + uint vlc_val = reverse_reader_fetch(vlc); + uint t0 = uint(vlc_table1[local_c_q + (vlc_val & 0x7Fu)]); + if (local_c_q == 0u) { + run -= 2; + t0 = run == -1 ? t0 : 0u; + if (run < 0 && !mel_get_run(mel, run)) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 10u); + return; + } + } + scratch[local_row_offset] = ushort(t0); + local_x += 2u; + + local_c_q = ((t0 & 0x40u) << 2u) | ((t0 & 0x80u) << 1u); + local_c_q |= uint(scratch[prev_base + (local_row_offset - row_base)]) & 0x80u; + local_c_q |= (uint(scratch[prev_base + (local_row_offset - row_base) + 2u]) & 0xA0u) << 2u; + local_c_q |= (uint(scratch[prev_base + (local_row_offset - row_base) + 4u]) & 0x20u) << 4u; + vlc_val = reverse_reader_advance(vlc, t0 & 0x7u); + + uint t1 = uint(vlc_table1[local_c_q + (vlc_val & 0x7Fu)]); + if (local_c_q == 0u && local_x < width) { + run -= 2; + t1 = run == -1 ? t1 : 0u; + if (run < 0 && !mel_get_run(mel, run)) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 11u); + return; + } + } + if (local_x >= width) { + t1 = 0u; + } + scratch[local_row_offset + 2u] = ushort(t1); + local_x += 2u; + + local_c_q = ((t1 & 0x40u) << 2u) | ((t1 & 0x80u) << 1u); + local_c_q |= uint(scratch[prev_base + (local_row_offset - row_base) + 2u]) & 0x80u; + vlc_val = reverse_reader_advance(vlc, t1 & 0x7u); + + const uint uvlc_mode = ((t0 & 0x8u) << 3u) | ((t1 & 0x8u) << 4u); + uint uvlc_entry = uint(uvlc_table1[uvlc_mode + (vlc_val & 0x3Fu)]); + vlc_val = reverse_reader_advance(vlc, uvlc_entry & 0x7u); + uvlc_entry >>= 3u; + uint len = uvlc_entry & 0xFu; + const uint tmp = vlc_val & ((1u << len) - 1u); + vlc_val = reverse_reader_advance(vlc, len); + uvlc_entry >>= 4u; + len = uvlc_entry & 0x7u; + uvlc_entry >>= 3u; + scratch[local_row_offset + 1u] = + ushort((uvlc_entry & 0x7u) + (tmp & ~(0xFFu << len))); + scratch[local_row_offset + 3u] = ushort((uvlc_entry >> 3u) + (tmp >> len)); + + local_row_offset += 4u; + } + + scratch[local_row_offset] = 0u; + scratch[local_row_offset + 1u] = 0u; + } + } + + const uint p = 30u - params.missing_msbs; + + { + ForwardBitReader magsgn = forward_reader_new(coded_data, lcup - scup, uchar(0xFF)); + const uint v_n_width = ((width + 1u) / 2u) + 2u; + if (v_n_width > J2K_HT_MAX_VN) { + set_ht_status(status, J2K_HT_STATUS_UNSUPPORTED, 12u); + return; + } + + uint prev_v_n = 0u; + uint x = 0u; + uint sp = 0u; + uint vp = 0u; + uint dp = params.output_offset; + const bool second_row_present = height > 1u; + + while (x < width) { + const uint inf = uint(scratch[sp]); + const uint uq = uint(scratch[sp + 1u]); + if (uq > params.missing_msbs + 2u) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 13u); + return; + } + + uint value0 = 0u; + uint ignored_vn = 0u; + decode_mag_sgn_sample_with_vn(magsgn, inf, 0u, uq, p, value0, ignored_vn); + store_decoded_cleanup_sample(decoded_data, dp, value0, params); + + uint value1 = 0u; + uint v_n1 = 0u; + decode_mag_sgn_sample_with_vn(magsgn, inf, 1u, uq, p, value1, v_n1); + if (second_row_present) { + store_decoded_cleanup_sample( + decoded_data, + dp + stride, + value1, + params + ); + } + v_n_scratch[vp] = prev_v_n | v_n1; + prev_v_n = 0u; + dp += 1u; + x += 1u; + + if (x >= width) { + vp += 1u; + break; + } + + uint value2 = 0u; + decode_mag_sgn_sample_with_vn(magsgn, inf, 2u, uq, p, value2, ignored_vn); + store_decoded_cleanup_sample(decoded_data, dp, value2, params); + + uint value3 = 0u; + uint v_n3 = 0u; + decode_mag_sgn_sample_with_vn(magsgn, inf, 3u, uq, p, value3, v_n3); + if (second_row_present) { + store_decoded_cleanup_sample( + decoded_data, + dp + stride, + value3, + params + ); + } + prev_v_n = v_n3; + dp += 1u; + x += 1u; + sp += 2u; + vp += 1u; + } + v_n_scratch[vp] = prev_v_n; + + for (uint y = 2u; y < height; y += 2u) { + const uint row_base = (y >> 1u) * sstr; + uint local_sp = row_base; + uint local_vp = 0u; + uint local_dp = params.output_offset + y * stride; + uint local_prev_v_n = 0u; + uint local_x = 0u; + const bool local_second_row_present = y + 1u < height; + + while (local_x < width) { + const uint inf = uint(scratch[local_sp]); + const uint u_q = uint(scratch[local_sp + 1u]); + uint gamma = inf & 0xF0u; + gamma &= gamma - 0x10u; + uint emax = v_n_scratch[local_vp] | v_n_scratch[local_vp + 1u]; + emax = 31u - __clz(emax | 2u); + const uint kappa = gamma != 0u ? emax : 1u; + const uint uq = u_q + kappa; + if (uq > params.missing_msbs + 2u) { + set_ht_status(status, J2K_HT_STATUS_FAIL, 14u); + return; + } + + uint value0 = 0u; + uint ignored_vn = 0u; + decode_mag_sgn_sample_with_vn(magsgn, inf, 0u, uq, p, value0, ignored_vn); + store_decoded_cleanup_sample( + decoded_data, + local_dp, + value0, + params + ); + + uint value1 = 0u; + uint v_n1 = 0u; + decode_mag_sgn_sample_with_vn(magsgn, inf, 1u, uq, p, value1, v_n1); + if (local_second_row_present) { + store_decoded_cleanup_sample( + decoded_data, + local_dp + stride, + value1, + params + ); + } + v_n_scratch[local_vp] = local_prev_v_n | v_n1; + local_prev_v_n = 0u; + local_dp += 1u; + local_x += 1u; + + if (local_x >= width) { + local_vp += 1u; + break; + } + + uint value2 = 0u; + decode_mag_sgn_sample_with_vn(magsgn, inf, 2u, uq, p, value2, ignored_vn); + store_decoded_cleanup_sample( + decoded_data, + local_dp, + value2, + params + ); + + uint value3 = 0u; + uint v_n3 = 0u; + decode_mag_sgn_sample_with_vn(magsgn, inf, 3u, uq, p, value3, v_n3); + if (local_second_row_present) { + store_decoded_cleanup_sample( + decoded_data, + local_dp + stride, + value3, + params + ); + } + local_prev_v_n = v_n3; + local_dp += 1u; + local_x += 1u; + local_sp += 2u; + local_vp += 1u; + } + + v_n_scratch[local_vp] = local_prev_v_n; + } + } + + if (!CLEANUP_ONLY && num_passes > 1u) { + const uint sigma_rows = ((height + 3u) / 4u) + 1u; + const uint mstr = ((((width + 3u) / 4u) + 2u + 7u) & ~7u); + const uint prev_row_len = ((width + 3u) / 4u) + 8u; + if (mstr > J2K_HT_MAX_MSTR || sigma_rows * mstr > J2K_HT_MAX_SIGMA) { + set_ht_status(status, J2K_HT_STATUS_UNSUPPORTED, 15u); + return; + } + if (prev_row_len > J2K_HT_MAX_PREV_ROW_SIG) { + set_ht_status(status, J2K_HT_STATUS_UNSUPPORTED, 16u); + return; + } + + ushort sigma[J2K_HT_MAX_SIGMA]; + ushort prev_row_sig[J2K_HT_MAX_PREV_ROW_SIG]; + + uint y = 0u; + while (y < height) { + uint sp_base = (y >> 1u) * sstr; + uint dp_base = (y >> 2u) * mstr; + uint local_x = 0u; + uint sigma_sp = sp_base; + uint sigma_dp = dp_base; + while (local_x < width) { + uint t0 = ((uint(scratch[sigma_sp]) & 0x30u) >> 4u) + | ((uint(scratch[sigma_sp]) & 0xC0u) >> 2u); + t0 |= ((uint(scratch[sigma_sp + 2u]) & 0x30u) << 4u) + | ((uint(scratch[sigma_sp + 2u]) & 0xC0u) << 6u); + uint t1 = ((uint(scratch[sigma_sp + sstr]) & 0x30u) >> 2u) + | (uint(scratch[sigma_sp + sstr]) & 0xC0u); + t1 |= ((uint(scratch[sigma_sp + sstr + 2u]) & 0x30u) << 6u) + | ((uint(scratch[sigma_sp + sstr + 2u]) & 0xC0u) << 8u); + sigma[sigma_dp] = ushort(t0 | t1); + local_x += 4u; + sigma_sp += 4u; + sigma_dp += 1u; + } + sigma[sigma_dp] = 0u; + y += 4u; + } + + const uint sigma_tail = ((height + 3u) / 4u) * mstr; + for (uint i = 0u; i <= (width + 3u) / 4u; ++i) { + sigma[sigma_tail + i] = 0u; + } + + for (uint i = 0u; i < prev_row_len; ++i) { + prev_row_sig[i] = 0u; + } + + ForwardBitReader sigprop = + forward_reader_new(coded_data + lcup, params.refinement_length, uchar(0x00)); + + for (y = 0u; y < height; y += 4u) { + uint pattern = 0xFFFFu; + if (height - y < 4u) { + pattern = 0x7777u; + if (height - y < 3u) { + pattern = 0x3333u; + if (height - y < 2u) { + pattern = 0x1111u; + } + } + } + + uint prev = 0u; + const uint cur_row = (y >> 2u) * mstr; + const uint next_row = cur_row + mstr; + const uint dpp = params.output_offset + y * stride; + + for (uint x4 = 0u; x4 < width; x4 += 4u) { + uint col_pattern = pattern; + int s = int(x4) + 4 - int(width); + s = j2k_max_i32(s, 0); + col_pattern >>= uint(s * 4); + + const uint idx = x4 >> 2u; + const uint ps = + uint(prev_row_sig[idx]) | (uint(prev_row_sig[idx + 1u]) << 16u); + const uint ns = read_u32_pair(sigma, next_row + idx); + uint u = (ps & 0x8888'8888u) >> 3u; + if (params.stripe_causal == 0u) { + u |= (ns & 0x1111'1111u) << 3u; + } + + const uint cs = read_u32_pair(sigma, cur_row + idx); + uint mbr = cs; + mbr |= (cs & 0x7777'7777u) << 1u; + mbr |= (cs & 0xEEEE'EEEEu) >> 1u; + mbr |= u; + const uint t = mbr; + mbr |= t << 4u; + mbr |= t >> 4u; + mbr |= prev >> 12u; + mbr &= col_pattern; + mbr &= ~cs; + + uint new_sig = mbr; + if (new_sig != 0u) { + uint cwd = forward_reader_fetch(sigprop); + uint cnt = 0u; + uint col_mask = 0xFu; + const uint inv_sig = ~cs & col_pattern; + + for (uint i = 0u; i < 16u; i += 4u) { + if ((col_mask & new_sig) == 0u) { + col_mask <<= 4u; + continue; + } + + uint sample_mask = 0x1111u & col_mask; + if ((new_sig & sample_mask) != 0u) { + new_sig &= ~sample_mask; + if ((cwd & 1u) != 0u) { + const uint t_bits = 0x33u << i; + new_sig |= t_bits & inv_sig; + } + cwd >>= 1u; + cnt += 1u; + } + + sample_mask <<= 1u; + if ((new_sig & sample_mask) != 0u) { + new_sig &= ~sample_mask; + if ((cwd & 1u) != 0u) { + const uint t_bits = 0x76u << i; + new_sig |= t_bits & inv_sig; + } + cwd >>= 1u; + cnt += 1u; + } + + sample_mask <<= 1u; + if ((new_sig & sample_mask) != 0u) { + new_sig &= ~sample_mask; + if ((cwd & 1u) != 0u) { + const uint t_bits = 0xECu << i; + new_sig |= t_bits & inv_sig; + } + cwd >>= 1u; + cnt += 1u; + } + + sample_mask <<= 1u; + if ((new_sig & sample_mask) != 0u) { + new_sig &= ~sample_mask; + if ((cwd & 1u) != 0u) { + const uint t_bits = 0xC8u << i; + new_sig |= t_bits & inv_sig; + } + cwd >>= 1u; + cnt += 1u; + } + + col_mask <<= 4u; + } + + if (new_sig != 0u) { + uint sig_dp = dpp + x4; + const uint value = 3u << (p - 2u); + col_mask = 0xFu; + + for (uint column = 0u; column < 4u; ++column) { + if ((col_mask & new_sig) == 0u) { + col_mask <<= 4u; + sig_dp += 1u; + continue; + } + + uint sample_mask = 0x1111u & col_mask; + if ((new_sig & sample_mask) != 0u) { + decoded_data[sig_dp] = (cwd << 31u) | value; + cwd >>= 1u; + cnt += 1u; + } + + sample_mask <<= 1u; + if ((new_sig & sample_mask) != 0u) { + decoded_data[sig_dp + stride] = (cwd << 31u) | value; + cwd >>= 1u; + cnt += 1u; + } + + sample_mask <<= 1u; + if ((new_sig & sample_mask) != 0u) { + decoded_data[sig_dp + 2u * stride] = (cwd << 31u) | value; + cwd >>= 1u; + cnt += 1u; + } + + sample_mask <<= 1u; + if ((new_sig & sample_mask) != 0u) { + decoded_data[sig_dp + 3u * stride] = (cwd << 31u) | value; + cwd >>= 1u; + cnt += 1u; + } + + col_mask <<= 4u; + sig_dp += 1u; + } + } + + forward_reader_advance(sigprop, cnt); + } + + const uint combined_sig = new_sig | cs; + prev_row_sig[idx] = ushort(combined_sig); + if (idx + 1u < prev_row_len) { + prev_row_sig[idx + 1u] = ushort(combined_sig >> 16u); + } + + const uint combined = combined_sig; + uint next_prev = combined_sig; + next_prev |= (combined & 0x7777u) << 1u; + next_prev |= (combined & 0xEEEEu) >> 1u; + prev = (next_prev | u) & 0xF000u; + } + } + + if (num_passes > 2u) { + y = 0u; + while (y < height) { + uint sp_base = (y >> 1u) * sstr; + uint dp_base = (y >> 2u) * mstr; + uint local_x = 0u; + uint sigma_sp = sp_base; + uint sigma_dp = dp_base; + while (local_x < width) { + uint t0 = ((uint(scratch[sigma_sp]) & 0x30u) >> 4u) + | ((uint(scratch[sigma_sp]) & 0xC0u) >> 2u); + t0 |= ((uint(scratch[sigma_sp + 2u]) & 0x30u) << 4u) + | ((uint(scratch[sigma_sp + 2u]) & 0xC0u) << 6u); + uint t1 = ((uint(scratch[sigma_sp + sstr]) & 0x30u) >> 2u) + | (uint(scratch[sigma_sp + sstr]) & 0xC0u); + t1 |= ((uint(scratch[sigma_sp + sstr + 2u]) & 0x30u) << 6u) + | ((uint(scratch[sigma_sp + sstr + 2u]) & 0xC0u) << 8u); + sigma[sigma_dp] = ushort(t0 | t1); + local_x += 4u; + sigma_sp += 4u; + sigma_dp += 1u; + } + sigma[sigma_dp] = 0u; + y += 4u; + } + + for (uint i = 0u; i <= (width + 3u) / 4u; ++i) { + sigma[sigma_tail + i] = 0u; + } + + ReverseBitReader magref = + reverse_reader_new_mrp(coded_data, lcup, params.refinement_length); + const uint half_value = 1u << (p - 2u); + + for (y = 0u; y < height; y += 4u) { + uint cur_sig_idx = (y >> 2u) * mstr; + const uint dpp = params.output_offset + y * stride; + + for (uint x8 = 0u; x8 < width; x8 += 8u) { + const uint cwd = reverse_reader_fetch(magref); + const uint sig = read_u32_pair(sigma, cur_sig_idx); + cur_sig_idx += 2u; + uint col_mask = 0xFu; + uint cwd_mut = cwd; + + if (sig != 0u) { + for (uint column = 0u; column < 8u; ++column) { + if ((sig & col_mask) != 0u) { + uint mag_dp = dpp + x8 + column; + uint sample_mask = 0x1111'1111u & col_mask; + + for (uint row = 0u; row < 4u; ++row) { + if ((sig & sample_mask) != 0u) { + uint sym = cwd_mut & 1u; + sym = (1u - sym) << (p - 1u); + sym |= half_value; + decoded_data[mag_dp] ^= sym; + cwd_mut >>= 1u; + } + sample_mask <<= 1u; + mag_dp += stride; + } + } + col_mask <<= 4u; + } + } + + reverse_reader_advance(magref, __popc(sig)); + } + } + } + } + +} + + +struct CudaJ2kRect { + uint x0; + uint y0; + uint x1; + uint y1; +}; + +struct CudaJ2kIdwtJob { + CudaJ2kRect rect; + CudaJ2kRect ll_rect; + CudaJ2kRect hl_rect; + CudaJ2kRect lh_rect; + CudaJ2kRect hh_rect; + uint irreversible97; +}; + +struct CudaJ2kIdwtMultiJob { + j2k_ulong ll_ptr; + j2k_ulong hl_ptr; + j2k_ulong lh_ptr; + j2k_ulong hh_ptr; + j2k_ulong output_ptr; + CudaJ2kIdwtJob job; +}; + +struct CudaJ2kStoreGray8Job { + uint input_width; + uint source_x; + uint source_y; + uint copy_width; + uint copy_height; + uint output_width; + uint output_height; + uint output_x; + uint output_y; + float addend; + uint bit_depth; +}; + +struct CudaJ2kStoreGray16Job { + uint input_width; + uint source_x; + uint source_y; + uint copy_width; + uint copy_height; + uint output_width; + uint output_height; + uint output_x; + uint output_y; + float addend; + uint bit_depth; +}; + +struct CudaJ2kInverseMctJob { + uint len; + uint irreversible97; + float addend0; + float addend1; + float addend2; +}; + +struct CudaJ2kStoreRgb8Job { + uint input_width0; + uint input_width1; + uint input_width2; + uint source_x0; + uint source_y0; + uint source_x1; + uint source_y1; + uint source_x2; + uint source_y2; + uint copy_width; + uint copy_height; + uint output_width; + uint output_height; + uint output_x; + uint output_y; + float addend0; + float addend1; + float addend2; + uint bit_depth0; + uint bit_depth1; + uint bit_depth2; + uint rgba; +}; + +struct CudaJ2kStoreRgb16Job { + uint input_width0; + uint input_width1; + uint input_width2; + uint source_x0; + uint source_y0; + uint source_x1; + uint source_y1; + uint source_x2; + uint source_y2; + uint copy_width; + uint copy_height; + uint output_width; + uint output_height; + uint output_x; + uint output_y; + float addend0; + float addend1; + float addend2; + uint bit_depth0; + uint bit_depth1; + uint bit_depth2; + uint rgba; +}; + +struct CudaJ2kStoreRgb8MctJob { + CudaJ2kStoreRgb8Job store; + uint irreversible97; +}; + +struct CudaJ2kStoreRgb8MctBatchJob { + j2k_ulong plane0_ptr; + j2k_ulong plane1_ptr; + j2k_ulong plane2_ptr; + j2k_ulong output_ptr; + CudaJ2kStoreRgb8MctJob job; +}; + +struct CudaJ2kStoreRgb16MctJob { + CudaJ2kStoreRgb16Job store; + uint irreversible97; +}; + +__device__ inline uint rect_width(CudaJ2kRect rect) { + return rect.x1 - rect.x0; +} + +__device__ inline uint rect_height(CudaJ2kRect rect) { + return rect.y1 - rect.y0; +} + +__device__ inline uint div_ceil_2(uint value) { + return (value + 1u) >> 1u; +} + +__device__ inline uint idwt_band_coord(uint output_origin, uint output_coord, uint band_origin, bool low) { + const uint global = output_coord; + uint index; + if (low) { + index = div_ceil_2(global) - div_ceil_2(output_origin); + } else { + index = (global >> 1u) - (output_origin >> 1u); + } + return band_origin + index; +} + +__device__ inline float source_get(const float *source, CudaJ2kRect rect, uint x, uint y) { + if (x < rect.x0 || x >= rect.x1 || y < rect.y0 || y >= rect.y1) { + return 0.0f; + } + const uint local_x = x - rect.x0; + const uint local_y = y - rect.y0; + return source[local_y * rect_width(rect) + local_x]; +} + +__device__ inline uint pse_left(uint idx, uint offset) { + return idx > offset ? idx - offset : offset - idx; +} + +__device__ inline uint pse_right(uint idx, uint offset, uint length) { + const uint new_idx = idx + offset; + if (new_idx >= length) { + const uint overshoot = new_idx - length; + return length - 2u - overshoot; + } + return new_idx; +} + +__device__ inline float lift_53_sample(float sample, float left, float right, bool update_even) { + if (update_even) { + return sample - floorf(fmaf(left + right, 0.25f, 0.5f)); + } + return sample + floorf((left + right) * 0.5f); +} + +__device__ inline void filter_step_horizontal_53(float *scanline, uint width, uint first, bool update_even) { + if (first == 0u) { + const uint left = pse_left(0u, 1u); + const uint right = pse_right(0u, 1u, width); + if (update_even) { + scanline[0] = scanline[0] - floorf(fmaf(scanline[left] + scanline[right], 0.25f, 0.5f)); + } else { + scanline[0] = scanline[0] + floorf((scanline[left] + scanline[right]) * 0.5f); + } + } + + const uint middle_start = first == 0u ? 2u : 1u; + for (uint i = middle_start; i + 1u < width; i += 2u) { + if (update_even) { + scanline[i] = scanline[i] - floorf(fmaf(scanline[i - 1u] + scanline[i + 1u], 0.25f, 0.5f)); + } else { + scanline[i] = scanline[i] + floorf((scanline[i - 1u] + scanline[i + 1u]) * 0.5f); + } + } + + if (width > 1u && ((width - 1u) & 1u) == first) { + const uint i = width - 1u; + const uint left = pse_left(i, 1u); + const uint right = pse_right(i, 1u, width); + if (update_even) { + scanline[i] = scanline[i] - floorf(fmaf(scanline[left] + scanline[right], 0.25f, 0.5f)); + } else { + scanline[i] = scanline[i] + floorf((scanline[left] + scanline[right]) * 0.5f); + } + } +} + +__device__ inline void filter_step_horizontal_97(float *scanline, uint width, uint first, float coefficient) { + if (first == 0u) { + const uint left = pse_left(0u, 1u); + const uint right = pse_right(0u, 1u, width); + scanline[0] = fmaf(scanline[left] + scanline[right], coefficient, scanline[0]); + } + + const uint middle_start = first == 0u ? 2u : 1u; + for (uint i = middle_start; i + 1u < width; i += 2u) { + scanline[i] = fmaf(scanline[i - 1u] + scanline[i + 1u], coefficient, scanline[i]); + } + + if (width > 1u && ((width - 1u) & 1u) == first) { + const uint i = width - 1u; + const uint left = pse_left(i, 1u); + const uint right = pse_right(i, 1u, width); + scanline[i] = fmaf(scanline[left] + scanline[right], coefficient, scanline[i]); + } +} + +__device__ inline void filter_horizontal_scanline(float *scanline, uint width, uint rect_x0, bool irreversible97) { + if (width == 1u) { + if ((rect_x0 & 1u) != 0u) { + scanline[0] *= 0.5f; + } + return; + } + + const uint first_even = rect_x0 & 1u; + const uint first_odd = 1u - first_even; + if (!irreversible97) { + filter_step_horizontal_53(scanline, width, first_even, true); + filter_step_horizontal_53(scanline, width, first_odd, false); + } else { + const float neg_alpha = 1.5861343f; + const float neg_beta = 0.052980117f; + const float neg_gamma = -0.8829111f; + const float neg_delta = -0.44350687f; + const float kappa = 1.2301741f; + const float inv_kappa = 1.0f / kappa; + const float k0 = first_even == 0u ? kappa : inv_kappa; + const float k1 = first_even == 0u ? inv_kappa : kappa; + for (uint i = 0u; i + 1u < width; i += 2u) { + scanline[i] *= k0; + scanline[i + 1u] *= k1; + } + if ((width & 1u) != 0u) { + scanline[width - 1u] *= k0; + } + filter_step_horizontal_97(scanline, width, first_even, neg_delta); + filter_step_horizontal_97(scanline, width, first_odd, neg_gamma); + filter_step_horizontal_97(scanline, width, first_even, neg_beta); + filter_step_horizontal_97(scanline, width, first_odd, neg_alpha); + } +} + +__device__ inline void filter_horizontal(float *output, CudaJ2kRect rect, bool irreversible97) { + const uint width = rect_width(rect); + const uint height = rect_height(rect); + for (uint y = 0u; y < height; ++y) { + filter_horizontal_scanline(output + y * width, width, rect.x0, irreversible97); + } +} + +__device__ inline void filter_step_vertical_53(float *output, uint width, uint height, uint first, bool update_even) { + for (uint row = first; row < height; row += 2u) { + const uint row_above = pse_left(row, 1u); + const uint row_below = pse_right(row, 1u, height); + for (uint col = 0u; col < width; ++col) { + const uint idx = row * width + col; + const float above = output[row_above * width + col]; + const float below = output[row_below * width + col]; + if (update_even) { + output[idx] = output[idx] - floorf(fmaf(above + below, 0.25f, 0.5f)); + } else { + output[idx] = output[idx] + floorf((above + below) * 0.5f); + } + } + } +} + +__device__ inline void filter_step_vertical_97(float *output, uint width, uint height, uint first, float coefficient) { + for (uint row = first; row < height; row += 2u) { + const uint row_above = pse_left(row, 1u); + const uint row_below = pse_right(row, 1u, height); + for (uint col = 0u; col < width; ++col) { + const uint idx = row * width + col; + output[idx] = fmaf( + output[row_above * width + col] + output[row_below * width + col], + coefficient, + output[idx] + ); + } + } +} + +__device__ inline void filter_step_vertical_53_column( + float *output, + uint width, + uint height, + uint col, + uint first, + bool update_even +) { + for (uint row = first; row < height; row += 2u) { + const uint row_above = pse_left(row, 1u); + const uint row_below = pse_right(row, 1u, height); + const uint idx = row * width + col; + const float above = output[row_above * width + col]; + const float below = output[row_below * width + col]; + if (update_even) { + output[idx] = output[idx] - floorf(fmaf(above + below, 0.25f, 0.5f)); + } else { + output[idx] = output[idx] + floorf((above + below) * 0.5f); + } + } +} + +__device__ inline void filter_step_vertical_97_column( + float *output, + uint width, + uint height, + uint col, + uint first, + float coefficient +) { + for (uint row = first; row < height; row += 2u) { + const uint row_above = pse_left(row, 1u); + const uint row_below = pse_right(row, 1u, height); + const uint idx = row * width + col; + output[idx] = fmaf( + output[row_above * width + col] + output[row_below * width + col], + coefficient, + output[idx] + ); + } +} + +__device__ inline void filter_vertical_column( + float *output, + uint width, + uint height, + uint rect_y0, + uint col, + bool irreversible97 +) { + if (height == 1u) { + if ((rect_y0 & 1u) != 0u) { + output[col] *= 0.5f; + } + return; + } + + const uint first_even = rect_y0 & 1u; + const uint first_odd = 1u - first_even; + if (!irreversible97) { + filter_step_vertical_53_column(output, width, height, col, first_even, true); + filter_step_vertical_53_column(output, width, height, col, first_odd, false); + } else { + const float neg_alpha = 1.5861343f; + const float neg_beta = 0.052980117f; + const float neg_gamma = -0.8829111f; + const float neg_delta = -0.44350687f; + const float kappa = 1.2301741f; + const float inv_kappa = 1.0f / kappa; + const float k0 = first_even == 0u ? kappa : inv_kappa; + const float k1 = first_even == 0u ? inv_kappa : kappa; + for (uint row = 0u; row + 1u < height; row += 2u) { + output[row * width + col] *= k0; + output[(row + 1u) * width + col] *= k1; + } + if ((height & 1u) != 0u) { + const uint row = height - 1u; + output[row * width + col] *= k0; + } + filter_step_vertical_97_column(output, width, height, col, first_even, neg_delta); + filter_step_vertical_97_column(output, width, height, col, first_odd, neg_gamma); + filter_step_vertical_97_column(output, width, height, col, first_even, neg_beta); + filter_step_vertical_97_column(output, width, height, col, first_odd, neg_alpha); + } +} + +__device__ inline void filter_vertical(float *output, CudaJ2kRect rect, bool irreversible97) { + const uint width = rect_width(rect); + const uint height = rect_height(rect); + for (uint col = 0u; col < width; ++col) { + filter_vertical_column(output, width, height, rect.y0, col, irreversible97); + } +} + +extern "C" __global__ void j2k_inverse_dwt_single( + const float *ll, + const float *hl, + const float *lh, + const float *hh, + float *output, + const CudaJ2kIdwtJob *job_buffer +) { + if (blockIdx.x != 0u || threadIdx.x != 0u) { + return; + } + + const CudaJ2kIdwtJob job = job_buffer[0]; + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + for (uint y = job.rect.y0; y < job.rect.y1; ++y) { + const bool low_y = (y & 1u) == 0u; + for (uint x = job.rect.x0; x < job.rect.x1; ++x) { + const bool low_x = (x & 1u) == 0u; + const float *source = ll; + CudaJ2kRect source_rect = job.ll_rect; + uint band_x; + uint band_y; + if (low_x && low_y) { + source = ll; + source_rect = job.ll_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.ll_rect.x0, true); + band_y = idwt_band_coord(job.rect.y0, y, job.ll_rect.y0, true); + } else if (!low_x && low_y) { + source = hl; + source_rect = job.hl_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.hl_rect.x0, false); + band_y = idwt_band_coord(job.rect.y0, y, job.hl_rect.y0, true); + } else if (low_x && !low_y) { + source = lh; + source_rect = job.lh_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.lh_rect.x0, true); + band_y = idwt_band_coord(job.rect.y0, y, job.lh_rect.y0, false); + } else { + source = hh; + source_rect = job.hh_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.hh_rect.x0, false); + band_y = idwt_band_coord(job.rect.y0, y, job.hh_rect.y0, false); + } + const uint dst = (y - job.rect.y0) * width + (x - job.rect.x0); + output[dst] = source_get(source, source_rect, band_x, band_y); + } + } + + if (width > 0u && height > 0u) { + const bool irreversible97 = job.irreversible97 != 0u; + filter_horizontal(output, job.rect, irreversible97); + filter_vertical(output, job.rect, irreversible97); + } +} + +extern "C" __global__ void j2k_idwt_interleave( + const float *ll, + const float *hl, + const float *lh, + const float *hh, + float *output, + const CudaJ2kIdwtJob *job_buffer +) { + const CudaJ2kIdwtJob job = job_buffer[0]; + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + const uint local_x = blockIdx.x * blockDim.x + threadIdx.x; + const uint local_y = blockIdx.y * blockDim.y + threadIdx.y; + if (local_x >= width || local_y >= height) { + return; + } + + const uint x = job.rect.x0 + local_x; + const uint y = job.rect.y0 + local_y; + const bool low_x = (x & 1u) == 0u; + const bool low_y = (y & 1u) == 0u; + const float *source = ll; + CudaJ2kRect source_rect = job.ll_rect; + uint band_x; + uint band_y; + if (low_x && low_y) { + source = ll; + source_rect = job.ll_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.ll_rect.x0, true); + band_y = idwt_band_coord(job.rect.y0, y, job.ll_rect.y0, true); + } else if (!low_x && low_y) { + source = hl; + source_rect = job.hl_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.hl_rect.x0, false); + band_y = idwt_band_coord(job.rect.y0, y, job.hl_rect.y0, true); + } else if (low_x && !low_y) { + source = lh; + source_rect = job.lh_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.lh_rect.x0, true); + band_y = idwt_band_coord(job.rect.y0, y, job.lh_rect.y0, false); + } else { + source = hh; + source_rect = job.hh_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.hh_rect.x0, false); + band_y = idwt_band_coord(job.rect.y0, y, job.hh_rect.y0, false); + } + output[local_y * width + local_x] = source_get(source, source_rect, band_x, band_y); +} + +extern "C" __global__ void j2k_idwt_interleave_multi( + const CudaJ2kIdwtMultiJob *jobs +) { + const uint job_idx = blockIdx.z; + const CudaJ2kIdwtMultiJob item = jobs[job_idx]; + const CudaJ2kIdwtJob job = item.job; + const float *ll = reinterpret_cast(static_cast(item.ll_ptr)); + const float *hl = reinterpret_cast(static_cast(item.hl_ptr)); + const float *lh = reinterpret_cast(static_cast(item.lh_ptr)); + const float *hh = reinterpret_cast(static_cast(item.hh_ptr)); + float *output = reinterpret_cast(static_cast(item.output_ptr)); + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + const uint local_x = blockIdx.x * blockDim.x + threadIdx.x; + const uint local_y = blockIdx.y * blockDim.y + threadIdx.y; + if (local_x >= width || local_y >= height) { + return; + } + + const uint x = job.rect.x0 + local_x; + const uint y = job.rect.y0 + local_y; + const bool low_x = (x & 1u) == 0u; + const bool low_y = (y & 1u) == 0u; + const float *source = ll; + CudaJ2kRect source_rect = job.ll_rect; + uint band_x; + uint band_y; + if (low_x && low_y) { + source = ll; + source_rect = job.ll_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.ll_rect.x0, true); + band_y = idwt_band_coord(job.rect.y0, y, job.ll_rect.y0, true); + } else if (!low_x && low_y) { + source = hl; + source_rect = job.hl_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.hl_rect.x0, false); + band_y = idwt_band_coord(job.rect.y0, y, job.hl_rect.y0, true); + } else if (low_x && !low_y) { + source = lh; + source_rect = job.lh_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.lh_rect.x0, true); + band_y = idwt_band_coord(job.rect.y0, y, job.lh_rect.y0, false); + } else { + source = hh; + source_rect = job.hh_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.hh_rect.x0, false); + band_y = idwt_band_coord(job.rect.y0, y, job.hh_rect.y0, false); + } + output[local_y * width + local_x] = source_get(source, source_rect, band_x, band_y); +} + +__device__ inline float idwt_interleave_sample( + const float *ll, + const float *hl, + const float *lh, + const float *hh, + CudaJ2kIdwtJob job, + uint local_x, + uint local_y +) { + const uint x = job.rect.x0 + local_x; + const uint y = job.rect.y0 + local_y; + const bool low_x = (x & 1u) == 0u; + const bool low_y = (y & 1u) == 0u; + const float *source = ll; + CudaJ2kRect source_rect = job.ll_rect; + uint band_x; + uint band_y; + if (low_x && low_y) { + source = ll; + source_rect = job.ll_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.ll_rect.x0, true); + band_y = idwt_band_coord(job.rect.y0, y, job.ll_rect.y0, true); + } else if (!low_x && low_y) { + source = hl; + source_rect = job.hl_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.hl_rect.x0, false); + band_y = idwt_band_coord(job.rect.y0, y, job.hl_rect.y0, true); + } else if (low_x && !low_y) { + source = lh; + source_rect = job.lh_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.lh_rect.x0, true); + band_y = idwt_band_coord(job.rect.y0, y, job.lh_rect.y0, false); + } else { + source = hh; + source_rect = job.hh_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.hh_rect.x0, false); + band_y = idwt_band_coord(job.rect.y0, y, job.hh_rect.y0, false); + } + return source_get(source, source_rect, band_x, band_y); +} + +extern "C" __global__ void j2k_idwt_interleave_horizontal_multi( + const CudaJ2kIdwtMultiJob *jobs +) { + const uint job_idx = blockIdx.y; + const CudaJ2kIdwtMultiJob item = jobs[job_idx]; + const CudaJ2kIdwtJob job = item.job; + const float *ll = reinterpret_cast(static_cast(item.ll_ptr)); + const float *hl = reinterpret_cast(static_cast(item.hl_ptr)); + const float *lh = reinterpret_cast(static_cast(item.lh_ptr)); + const float *hh = reinterpret_cast(static_cast(item.hh_ptr)); + float *output = reinterpret_cast(static_cast(item.output_ptr)); + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + const uint local_y = blockIdx.x * blockDim.x + threadIdx.x; + if (local_y >= height) { + return; + } + + const uint y = job.rect.y0 + local_y; + const bool low_y = (y & 1u) == 0u; + float *row_output = output + local_y * width; + for (uint local_x = 0u; local_x < width; ++local_x) { + const uint x = job.rect.x0 + local_x; + const bool low_x = (x & 1u) == 0u; + const float *source = ll; + CudaJ2kRect source_rect = job.ll_rect; + uint band_x; + uint band_y; + if (low_x && low_y) { + source = ll; + source_rect = job.ll_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.ll_rect.x0, true); + band_y = idwt_band_coord(job.rect.y0, y, job.ll_rect.y0, true); + } else if (!low_x && low_y) { + source = hl; + source_rect = job.hl_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.hl_rect.x0, false); + band_y = idwt_band_coord(job.rect.y0, y, job.hl_rect.y0, true); + } else if (low_x && !low_y) { + source = lh; + source_rect = job.lh_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.lh_rect.x0, true); + band_y = idwt_band_coord(job.rect.y0, y, job.lh_rect.y0, false); + } else { + source = hh; + source_rect = job.hh_rect; + band_x = idwt_band_coord(job.rect.x0, x, job.hh_rect.x0, false); + band_y = idwt_band_coord(job.rect.y0, y, job.hh_rect.y0, false); + } + row_output[local_x] = source_get(source, source_rect, band_x, band_y); + } + filter_horizontal_scanline( + row_output, + width, + job.rect.x0, + job.irreversible97 != 0u + ); +} + +extern "C" __global__ void j2k_idwt_interleave_horizontal_53_multi( + const CudaJ2kIdwtMultiJob *jobs +) { + __shared__ float row_samples[512]; + + const uint local_x = threadIdx.x; + const uint local_y = blockIdx.x; + const uint job_idx = blockIdx.y; + const CudaJ2kIdwtMultiJob item = jobs[job_idx]; + const CudaJ2kIdwtJob job = item.job; + const float *ll = reinterpret_cast(static_cast(item.ll_ptr)); + const float *hl = reinterpret_cast(static_cast(item.hl_ptr)); + const float *lh = reinterpret_cast(static_cast(item.lh_ptr)); + const float *hh = reinterpret_cast(static_cast(item.hh_ptr)); + float *output = reinterpret_cast(static_cast(item.output_ptr)); + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + if (local_y >= height) { + return; + } + + if (local_x < width) { + row_samples[local_x] = idwt_interleave_sample(ll, hl, lh, hh, job, local_x, local_y); + } + __syncthreads(); + + if (width == 1u) { + if (local_x == 0u && (job.rect.x0 & 1u) != 0u) { + row_samples[0] *= 0.5f; + } + __syncthreads(); + if (local_x == 0u) { + output[local_y * width] = row_samples[0]; + } + return; + } + + const uint first_even = job.rect.x0 & 1u; + const uint first_odd = 1u - first_even; + if (local_x < width && ((local_x & 1u) == first_even)) { + const uint left = pse_left(local_x, 1u); + const uint right = pse_right(local_x, 1u, width); + row_samples[local_x] = lift_53_sample( + row_samples[local_x], + row_samples[left], + row_samples[right], + true + ); + } + __syncthreads(); + + if (local_x < width && ((local_x & 1u) == first_odd)) { + const uint left = pse_left(local_x, 1u); + const uint right = pse_right(local_x, 1u, width); + row_samples[local_x] = lift_53_sample( + row_samples[local_x], + row_samples[left], + row_samples[right], + false + ); + } + __syncthreads(); + + if (local_x < width) { + output[local_y * width + local_x] = row_samples[local_x]; + } +} + +extern "C" __global__ void j2k_idwt_interleave_horizontal_97_multi( + const CudaJ2kIdwtMultiJob *jobs +) { + __shared__ float row_samples[512]; + + const uint local_x = threadIdx.x; + const uint local_y = blockIdx.x; + const uint job_idx = blockIdx.y; + const CudaJ2kIdwtMultiJob item = jobs[job_idx]; + const CudaJ2kIdwtJob job = item.job; + const float *ll = reinterpret_cast(static_cast(item.ll_ptr)); + const float *hl = reinterpret_cast(static_cast(item.hl_ptr)); + const float *lh = reinterpret_cast(static_cast(item.lh_ptr)); + const float *hh = reinterpret_cast(static_cast(item.hh_ptr)); + float *output = reinterpret_cast(static_cast(item.output_ptr)); + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + if (local_y >= height) { + return; + } + + if (local_x < width) { + row_samples[local_x] = idwt_interleave_sample(ll, hl, lh, hh, job, local_x, local_y); + } + __syncthreads(); + + if (width == 1u) { + if (local_x == 0u && (job.rect.x0 & 1u) != 0u) { + row_samples[0] *= 0.5f; + } + __syncthreads(); + if (local_x == 0u) { + output[local_y * width] = row_samples[0]; + } + return; + } + + const uint first_even = job.rect.x0 & 1u; + const uint first_odd = 1u - first_even; + const float neg_alpha = 1.5861343f; + const float neg_beta = 0.052980117f; + const float neg_gamma = -0.8829111f; + const float neg_delta = -0.44350687f; + const float kappa = 1.2301741f; + const float inv_kappa = 1.0f / kappa; + const float k0 = first_even == 0u ? kappa : inv_kappa; + const float k1 = first_even == 0u ? inv_kappa : kappa; + + if (local_x < width) { + row_samples[local_x] *= ((local_x & 1u) == 0u) ? k0 : k1; + } + __syncthreads(); + + if (local_x < width && ((local_x & 1u) == first_even)) { + const uint left = pse_left(local_x, 1u); + const uint right = pse_right(local_x, 1u, width); + row_samples[local_x] = fmaf(row_samples[left] + row_samples[right], neg_delta, row_samples[local_x]); + } + __syncthreads(); + + if (local_x < width && ((local_x & 1u) == first_odd)) { + const uint left = pse_left(local_x, 1u); + const uint right = pse_right(local_x, 1u, width); + row_samples[local_x] = fmaf(row_samples[left] + row_samples[right], neg_gamma, row_samples[local_x]); + } + __syncthreads(); + + if (local_x < width && ((local_x & 1u) == first_even)) { + const uint left = pse_left(local_x, 1u); + const uint right = pse_right(local_x, 1u, width); + row_samples[local_x] = fmaf(row_samples[left] + row_samples[right], neg_beta, row_samples[local_x]); + } + __syncthreads(); + + if (local_x < width && ((local_x & 1u) == first_odd)) { + const uint left = pse_left(local_x, 1u); + const uint right = pse_right(local_x, 1u, width); + row_samples[local_x] = fmaf(row_samples[left] + row_samples[right], neg_alpha, row_samples[local_x]); + } + __syncthreads(); + + if (local_x < width) { + output[local_y * width + local_x] = row_samples[local_x]; + } +} + +extern "C" __global__ void j2k_idwt_horizontal( + float *output, + const CudaJ2kIdwtJob *job_buffer +) { + const CudaJ2kIdwtJob job = job_buffer[0]; + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + const uint row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= height) { + return; + } + filter_horizontal_scanline( + output + row * width, + width, + job.rect.x0, + job.irreversible97 != 0u + ); +} + +extern "C" __global__ void j2k_idwt_horizontal_53( + float *output, + const CudaJ2kIdwtJob *job_buffer +) { + const CudaJ2kIdwtJob job = job_buffer[0]; + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + const uint row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= height) { + return; + } + filter_horizontal_scanline(output + row * width, width, job.rect.x0, false); +} + +extern "C" __global__ void j2k_idwt_horizontal_97( + float *output, + const CudaJ2kIdwtJob *job_buffer +) { + const CudaJ2kIdwtJob job = job_buffer[0]; + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + const uint row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= height) { + return; + } + filter_horizontal_scanline(output + row * width, width, job.rect.x0, true); +} + +extern "C" __global__ void j2k_idwt_horizontal_multi( + const CudaJ2kIdwtMultiJob *jobs +) { + const uint job_idx = blockIdx.y; + const CudaJ2kIdwtMultiJob item = jobs[job_idx]; + const CudaJ2kIdwtJob job = item.job; + float *output = reinterpret_cast(static_cast(item.output_ptr)); + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + const uint row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= height) { + return; + } + filter_horizontal_scanline( + output + row * width, + width, + job.rect.x0, + job.irreversible97 != 0u + ); +} + +extern "C" __global__ void j2k_idwt_vertical( + float *output, + const CudaJ2kIdwtJob *job_buffer +) { + const CudaJ2kIdwtJob job = job_buffer[0]; + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + const uint col = blockIdx.x * blockDim.x + threadIdx.x; + if (col >= width) { + return; + } + filter_vertical_column( + output, + width, + height, + job.rect.y0, + col, + job.irreversible97 != 0u + ); +} + +extern "C" __global__ void j2k_idwt_vertical_53( + float *output, + const CudaJ2kIdwtJob *job_buffer +) { + const CudaJ2kIdwtJob job = job_buffer[0]; + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + const uint col = blockIdx.x * blockDim.x + threadIdx.x; + if (col >= width) { + return; + } + filter_vertical_column(output, width, height, job.rect.y0, col, false); +} + +extern "C" __global__ void j2k_idwt_vertical_97( + float *output, + const CudaJ2kIdwtJob *job_buffer +) { + const CudaJ2kIdwtJob job = job_buffer[0]; + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + const uint col = blockIdx.x * blockDim.x + threadIdx.x; + if (col >= width) { + return; + } + filter_vertical_column(output, width, height, job.rect.y0, col, true); +} + +extern "C" __global__ void j2k_idwt_vertical_multi( + const CudaJ2kIdwtMultiJob *jobs +) { + const uint job_idx = blockIdx.y; + const CudaJ2kIdwtMultiJob item = jobs[job_idx]; + const CudaJ2kIdwtJob job = item.job; + float *output = reinterpret_cast(static_cast(item.output_ptr)); + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + const uint col = blockIdx.x * blockDim.x + threadIdx.x; + if (col >= width) { + return; + } + filter_vertical_column( + output, + width, + height, + job.rect.y0, + col, + job.irreversible97 != 0u + ); +} + +extern "C" __global__ void j2k_idwt_vertical_53_multi( + const CudaJ2kIdwtMultiJob *jobs +) { + __shared__ float column_samples[512]; + + const uint row = threadIdx.x; + const uint col = blockIdx.x; + const uint job_idx = blockIdx.y; + const CudaJ2kIdwtMultiJob item = jobs[job_idx]; + const CudaJ2kIdwtJob job = item.job; + float *output = reinterpret_cast(static_cast(item.output_ptr)); + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + if (col >= width) { + return; + } + + if (row < height) { + column_samples[row] = output[row * width + col]; + } + __syncthreads(); + + if (height == 1u) { + if (row == 0u && (job.rect.y0 & 1u) != 0u) { + column_samples[0] *= 0.5f; + } + __syncthreads(); + if (row == 0u) { + output[col] = column_samples[0]; + } + return; + } + + const uint first_even = job.rect.y0 & 1u; + const uint first_odd = 1u - first_even; + if (row < height && ((row & 1u) == first_even)) { + const uint above = pse_left(row, 1u); + const uint below = pse_right(row, 1u, height); + column_samples[row] = lift_53_sample( + column_samples[row], + column_samples[above], + column_samples[below], + true + ); + } + __syncthreads(); + + if (row < height && ((row & 1u) == first_odd)) { + const uint above = pse_left(row, 1u); + const uint below = pse_right(row, 1u, height); + column_samples[row] = lift_53_sample( + column_samples[row], + column_samples[above], + column_samples[below], + false + ); + } + __syncthreads(); + + if (row < height) { + output[row * width + col] = column_samples[row]; + } +} + +extern "C" __global__ void j2k_idwt_vertical_97_multi( + const CudaJ2kIdwtMultiJob *jobs +) { + __shared__ float column_samples[512]; + + const uint row = threadIdx.x; + const uint col = blockIdx.x; + const uint job_idx = blockIdx.y; + const CudaJ2kIdwtMultiJob item = jobs[job_idx]; + const CudaJ2kIdwtJob job = item.job; + float *output = reinterpret_cast(static_cast(item.output_ptr)); + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + if (col >= width) { + return; + } + + if (row < height) { + column_samples[row] = output[row * width + col]; + } + __syncthreads(); + + if (height == 1u) { + if (row == 0u && (job.rect.y0 & 1u) != 0u) { + column_samples[0] *= 0.5f; + } + __syncthreads(); + if (row == 0u) { + output[col] = column_samples[0]; + } + return; + } + + const uint first_even = job.rect.y0 & 1u; + const uint first_odd = 1u - first_even; + const float neg_alpha = 1.5861343f; + const float neg_beta = 0.052980117f; + const float neg_gamma = -0.8829111f; + const float neg_delta = -0.44350687f; + const float kappa = 1.2301741f; + const float inv_kappa = 1.0f / kappa; + const float k0 = first_even == 0u ? kappa : inv_kappa; + const float k1 = first_even == 0u ? inv_kappa : kappa; + + if (row < height) { + column_samples[row] *= ((row & 1u) == 0u) ? k0 : k1; + } + __syncthreads(); + + if (row < height && ((row & 1u) == first_even)) { + const uint above = pse_left(row, 1u); + const uint below = pse_right(row, 1u, height); + column_samples[row] = fmaf( + column_samples[above] + column_samples[below], + neg_delta, + column_samples[row] + ); + } + __syncthreads(); + + if (row < height && ((row & 1u) == first_odd)) { + const uint above = pse_left(row, 1u); + const uint below = pse_right(row, 1u, height); + column_samples[row] = fmaf( + column_samples[above] + column_samples[below], + neg_gamma, + column_samples[row] + ); + } + __syncthreads(); + + if (row < height && ((row & 1u) == first_even)) { + const uint above = pse_left(row, 1u); + const uint below = pse_right(row, 1u, height); + column_samples[row] = fmaf( + column_samples[above] + column_samples[below], + neg_beta, + column_samples[row] + ); + } + __syncthreads(); + + if (row < height && ((row & 1u) == first_odd)) { + const uint above = pse_left(row, 1u); + const uint below = pse_right(row, 1u, height); + column_samples[row] = fmaf( + column_samples[above] + column_samples[below], + neg_alpha, + column_samples[row] + ); + } + __syncthreads(); + + if (row < height) { + output[row * width + col] = column_samples[row]; + } +} + +extern "C" __global__ void j2k_idwt_vertical_97_multi_cols4( + const CudaJ2kIdwtMultiJob *jobs +) { + __shared__ float column_samples[256][4]; + + const uint local_col = threadIdx.x; + const uint row = threadIdx.y; + const uint col = blockIdx.x * 4u + local_col; + const uint job_idx = blockIdx.y; + const CudaJ2kIdwtMultiJob item = jobs[job_idx]; + const CudaJ2kIdwtJob job = item.job; + float *output = reinterpret_cast(static_cast(item.output_ptr)); + const uint width = rect_width(job.rect); + const uint height = rect_height(job.rect); + if (height > 256u) { + return; + } + + const bool valid = col < width && row < height; + if (valid) { + column_samples[row][local_col] = output[row * width + col]; + } + __syncthreads(); + + if (height == 1u) { + if (valid && (job.rect.y0 & 1u) != 0u) { + column_samples[0][local_col] *= 0.5f; + } + __syncthreads(); + if (valid) { + output[col] = column_samples[0][local_col]; + } + return; + } + + const uint first_even = job.rect.y0 & 1u; + const uint first_odd = 1u - first_even; + const float neg_alpha = 1.5861343f; + const float neg_beta = 0.052980117f; + const float neg_gamma = -0.8829111f; + const float neg_delta = -0.44350687f; + const float kappa = 1.2301741f; + const float inv_kappa = 1.0f / kappa; + const float k0 = first_even == 0u ? kappa : inv_kappa; + const float k1 = first_even == 0u ? inv_kappa : kappa; + + if (valid) { + column_samples[row][local_col] *= ((row & 1u) == 0u) ? k0 : k1; + } + __syncthreads(); + + if (valid && ((row & 1u) == first_even)) { + const uint above = pse_left(row, 1u); + const uint below = pse_right(row, 1u, height); + column_samples[row][local_col] = fmaf( + column_samples[above][local_col] + column_samples[below][local_col], + neg_delta, + column_samples[row][local_col] + ); + } + __syncthreads(); + + if (valid && ((row & 1u) == first_odd)) { + const uint above = pse_left(row, 1u); + const uint below = pse_right(row, 1u, height); + column_samples[row][local_col] = fmaf( + column_samples[above][local_col] + column_samples[below][local_col], + neg_gamma, + column_samples[row][local_col] + ); + } + __syncthreads(); + + if (valid && ((row & 1u) == first_even)) { + const uint above = pse_left(row, 1u); + const uint below = pse_right(row, 1u, height); + column_samples[row][local_col] = fmaf( + column_samples[above][local_col] + column_samples[below][local_col], + neg_beta, + column_samples[row][local_col] + ); + } + __syncthreads(); + + if (valid && ((row & 1u) == first_odd)) { + const uint above = pse_left(row, 1u); + const uint below = pse_right(row, 1u, height); + column_samples[row][local_col] = fmaf( + column_samples[above][local_col] + column_samples[below][local_col], + neg_alpha, + column_samples[row][local_col] + ); + } + __syncthreads(); + + if (valid) { + output[row * width + col] = column_samples[row][local_col]; + } +} + +__device__ inline uchar sample_as_u8(float sample, uint bit_depth) { + const float rounded = roundf(sample); + if (bit_depth == 8u) { + return (uchar)fminf(fmaxf(rounded, 0.0f), 255.0f); + } + const float max_value = bit_depth >= 16u ? 65535.0f : (float)(((1u << bit_depth) - 1u) > 1u ? ((1u << bit_depth) - 1u) : 1u); + return (uchar)roundf((fminf(fmaxf(rounded, 0.0f), max_value) / max_value) * 255.0f); +} + +__device__ inline ushort sample_as_u16(float sample, uint bit_depth) { + const float rounded = roundf(sample); + if (bit_depth >= 16u) { + return (ushort)fminf(fmaxf(rounded, 0.0f), 65535.0f); + } + const uint max_int = bit_depth == 0u ? 1u : ((1u << bit_depth) - 1u); + const float max_value = (float)(max_int > 1u ? max_int : 1u); + return (ushort)roundf((fminf(fmaxf(rounded, 0.0f), max_value) / max_value) * 65535.0f); +} + +extern "C" __global__ void j2k_store_gray8( + const float *input, + uchar *output, + const CudaJ2kStoreGray8Job *job_buffer +) { + const CudaJ2kStoreGray8Job job = job_buffer[0]; + const uint pixels = job.copy_width * job.copy_height; + const uint gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= pixels) { + return; + } + + const uint row = gid / job.copy_width; + const uint col = gid - row * job.copy_width; + const uint src = (job.source_y + row) * job.input_width + job.source_x + col; + const uint dst = (job.output_y + row) * job.output_width + job.output_x + col; + output[dst] = sample_as_u8(input[src] + job.addend, job.bit_depth); +} + +extern "C" __global__ void j2k_store_gray16( + const float *input, + ushort *output, + const CudaJ2kStoreGray16Job *job_buffer +) { + const CudaJ2kStoreGray16Job job = job_buffer[0]; + const uint pixels = job.copy_width * job.copy_height; + const uint gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= pixels) { + return; + } + + const uint row = gid / job.copy_width; + const uint col = gid - row * job.copy_width; + const uint src = (job.source_y + row) * job.input_width + job.source_x + col; + const uint dst = (job.output_y + row) * job.output_width + job.output_x + col; + output[dst] = sample_as_u16(input[src] + job.addend, job.bit_depth); +} + +__device__ inline void inverse_mct_sample( + float src0, + float src1, + float src2, + uint irreversible97, + float *out0, + float *out1, + float *out2 +) { + if (irreversible97 != 0u) { + *out0 = src0 + 1.402f * src2; + *out1 = src0 - 0.34413f * src1 - 0.71414f * src2; + *out2 = src0 + 1.772f * src1; + } else { + const float green = src0 - floorf((src2 + src1) * 0.25f); + *out0 = src2 + green; + *out1 = green; + *out2 = src1 + green; + } +} + +extern "C" __global__ void j2k_inverse_mct( + float *plane0, + float *plane1, + float *plane2, + const CudaJ2kInverseMctJob *job_buffer +) { + const CudaJ2kInverseMctJob job = job_buffer[0]; + const uint gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= job.len) { + return; + } + + const float src0 = plane0[gid]; + const float src1 = plane1[gid]; + const float src2 = plane2[gid]; + float out0; + float out1; + float out2; + inverse_mct_sample(src0, src1, src2, job.irreversible97, &out0, &out1, &out2); + plane0[gid] = out0 + job.addend0; + plane1[gid] = out1 + job.addend1; + plane2[gid] = out2 + job.addend2; +} + +extern "C" __global__ void j2k_store_rgb8( + const float *plane0, + const float *plane1, + const float *plane2, + uchar *output, + const CudaJ2kStoreRgb8Job *job_buffer +) { + const CudaJ2kStoreRgb8Job job = job_buffer[0]; + const uint pixels = job.copy_width * job.copy_height; + const uint gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= pixels) { + return; + } + + const uint row = gid / job.copy_width; + const uint col = gid - row * job.copy_width; + const uint src0 = (job.source_y0 + row) * job.input_width0 + job.source_x0 + col; + const uint src1 = (job.source_y1 + row) * job.input_width1 + job.source_x1 + col; + const uint src2 = (job.source_y2 + row) * job.input_width2 + job.source_x2 + col; + const uint channels = job.rgba != 0u ? 4u : 3u; + const uint dst = ((job.output_y + row) * job.output_width + job.output_x + col) * channels; + + output[dst] = sample_as_u8(plane0[src0] + job.addend0, job.bit_depth0); + output[dst + 1u] = sample_as_u8(plane1[src1] + job.addend1, job.bit_depth1); + output[dst + 2u] = sample_as_u8(plane2[src2] + job.addend2, job.bit_depth2); + if (job.rgba != 0u) { + output[dst + 3u] = 255u; + } +} + +extern "C" __global__ void j2k_store_rgb16( + const float *plane0, + const float *plane1, + const float *plane2, + ushort *output, + const CudaJ2kStoreRgb16Job *job_buffer +) { + const CudaJ2kStoreRgb16Job job = job_buffer[0]; + const uint pixels = job.copy_width * job.copy_height; + const uint gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= pixels) { + return; + } + + const uint row = gid / job.copy_width; + const uint col = gid - row * job.copy_width; + const uint src0 = (job.source_y0 + row) * job.input_width0 + job.source_x0 + col; + const uint src1 = (job.source_y1 + row) * job.input_width1 + job.source_x1 + col; + const uint src2 = (job.source_y2 + row) * job.input_width2 + job.source_x2 + col; + const uint channels = job.rgba != 0u ? 4u : 3u; + const uint dst = ((job.output_y + row) * job.output_width + job.output_x + col) * channels; + + output[dst] = sample_as_u16(plane0[src0] + job.addend0, job.bit_depth0); + output[dst + 1u] = sample_as_u16(plane1[src1] + job.addend1, job.bit_depth1); + output[dst + 2u] = sample_as_u16(plane2[src2] + job.addend2, job.bit_depth2); + if (job.rgba != 0u) { + output[dst + 3u] = 65535u; + } +} + +extern "C" __global__ void j2k_store_rgb8_mct( + const float *plane0, + const float *plane1, + const float *plane2, + uchar *output, + const CudaJ2kStoreRgb8MctJob *job_buffer +) { + const CudaJ2kStoreRgb8MctJob mct_job = job_buffer[0]; + const CudaJ2kStoreRgb8Job job = mct_job.store; + const uint pixels = job.copy_width * job.copy_height; + const uint gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= pixels) { + return; + } + + const uint row = gid / job.copy_width; + const uint col = gid - row * job.copy_width; + const uint src0 = (job.source_y0 + row) * job.input_width0 + job.source_x0 + col; + const uint src1 = (job.source_y1 + row) * job.input_width1 + job.source_x1 + col; + const uint src2 = (job.source_y2 + row) * job.input_width2 + job.source_x2 + col; + const uint channels = job.rgba != 0u ? 4u : 3u; + const uint dst = ((job.output_y + row) * job.output_width + job.output_x + col) * channels; + + float out0; + float out1; + float out2; + inverse_mct_sample( + plane0[src0], + plane1[src1], + plane2[src2], + mct_job.irreversible97, + &out0, + &out1, + &out2 + ); + output[dst] = sample_as_u8(out0 + job.addend0, job.bit_depth0); + output[dst + 1u] = sample_as_u8(out1 + job.addend1, job.bit_depth1); + output[dst + 2u] = sample_as_u8(out2 + job.addend2, job.bit_depth2); + if (job.rgba != 0u) { + output[dst + 3u] = 255u; + } +} + +extern "C" __global__ void j2k_store_rgb8_mct_batch( + const CudaJ2kStoreRgb8MctBatchJob *jobs +) { + const CudaJ2kStoreRgb8MctBatchJob item = jobs[blockIdx.y]; + const float *plane0 = reinterpret_cast(static_cast(item.plane0_ptr)); + const float *plane1 = reinterpret_cast(static_cast(item.plane1_ptr)); + const float *plane2 = reinterpret_cast(static_cast(item.plane2_ptr)); + uchar *output = reinterpret_cast(static_cast(item.output_ptr)); + const CudaJ2kStoreRgb8MctJob mct_job = item.job; + const CudaJ2kStoreRgb8Job job = mct_job.store; + const uint pixels = job.copy_width * job.copy_height; + const uint gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= pixels) { + return; + } + + const uint row = gid / job.copy_width; + const uint col = gid - row * job.copy_width; + const uint src0 = (job.source_y0 + row) * job.input_width0 + job.source_x0 + col; + const uint src1 = (job.source_y1 + row) * job.input_width1 + job.source_x1 + col; + const uint src2 = (job.source_y2 + row) * job.input_width2 + job.source_x2 + col; + const uint channels = job.rgba != 0u ? 4u : 3u; + const uint dst = ((job.output_y + row) * job.output_width + job.output_x + col) * channels; + + float out0; + float out1; + float out2; + inverse_mct_sample( + plane0[src0], + plane1[src1], + plane2[src2], + mct_job.irreversible97, + &out0, + &out1, + &out2 + ); + output[dst] = sample_as_u8(out0 + job.addend0, job.bit_depth0); + output[dst + 1u] = sample_as_u8(out1 + job.addend1, job.bit_depth1); + output[dst + 2u] = sample_as_u8(out2 + job.addend2, job.bit_depth2); + if (job.rgba != 0u) { + output[dst + 3u] = 255u; + } +} + +extern "C" __global__ void j2k_store_rgb16_mct( + const float *plane0, + const float *plane1, + const float *plane2, + ushort *output, + const CudaJ2kStoreRgb16MctJob *job_buffer +) { + const CudaJ2kStoreRgb16MctJob mct_job = job_buffer[0]; + const CudaJ2kStoreRgb16Job job = mct_job.store; + const uint pixels = job.copy_width * job.copy_height; + const uint gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= pixels) { + return; + } + + const uint row = gid / job.copy_width; + const uint col = gid - row * job.copy_width; + const uint src0 = (job.source_y0 + row) * job.input_width0 + job.source_x0 + col; + const uint src1 = (job.source_y1 + row) * job.input_width1 + job.source_x1 + col; + const uint src2 = (job.source_y2 + row) * job.input_width2 + job.source_x2 + col; + const uint channels = job.rgba != 0u ? 4u : 3u; + const uint dst = ((job.output_y + row) * job.output_width + job.output_x + col) * channels; + + float out0; + float out1; + float out2; + inverse_mct_sample( + plane0[src0], + plane1[src1], + plane2[src2], + mct_job.irreversible97, + &out0, + &out1, + &out2 + ); + output[dst] = sample_as_u16(out0 + job.addend0, job.bit_depth0); + output[dst + 1u] = sample_as_u16(out1 + job.addend1, job.bit_depth1); + output[dst + 2u] = sample_as_u16(out2 + job.addend2, job.bit_depth2); + if (job.rgba != 0u) { + output[dst + 3u] = 65535u; + } +} + +extern "C" __global__ void j2k_htj2k_decode_codeblocks( + const uchar *coded_data, + uint *decoded_data, + const J2kHtCleanupBatchJob *jobs, + const ushort *vlc_table0, + const ushort *vlc_table1, + const ushort *uvlc_table0, + const ushort *uvlc_table1, + J2kHtStatus *status, + uint job_count +) { + const uint gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= job_count) { + return; + } + const J2kHtCleanupBatchJob job = jobs[gid]; + + J2kHtCleanupParams params; + params.width = job.width; + params.height = job.height; + params.coded_len = job.coded_len; + params.cleanup_length = job.cleanup_length; + params.refinement_length = job.refinement_length; + params.missing_msbs = job.missing_msbs; + params.num_bitplanes = job.num_bitplanes; + params.number_of_coding_passes = job.number_of_coding_passes; + params.output_stride = job.output_stride; + params.output_offset = job.output_offset; + params.dequantization_step = job.dequantization_step; + params.stripe_causal = job.stripe_causal; + + decode_ht_cleanup_impl( + coded_data + job.coded_offset, + decoded_data, + params, + vlc_table0, + vlc_table1, + uvlc_table0, + uvlc_table1, + status + gid + ); +} + +extern "C" __global__ void j2k_htj2k_decode_codeblocks_multi( + const uchar *coded_data, + const J2kHtCleanupMultiBatchJob *jobs, + const ushort *vlc_table0, + const ushort *vlc_table1, + const ushort *uvlc_table0, + const ushort *uvlc_table1, + J2kHtStatus *status, + uint job_count +) { + const uint gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= job_count) { + return; + } + const J2kHtCleanupMultiBatchJob job = jobs[gid]; + uint *decoded_data = reinterpret_cast(static_cast(job.output_ptr)); + + J2kHtCleanupParams params; + params.width = job.width; + params.height = job.height; + params.coded_len = job.coded_len; + params.cleanup_length = job.cleanup_length; + params.refinement_length = job.refinement_length; + params.missing_msbs = job.missing_msbs; + params.num_bitplanes = job.num_bitplanes; + params.number_of_coding_passes = job.number_of_coding_passes; + params.output_stride = job.output_stride; + params.output_offset = job.output_offset; + params.dequantization_step = job.dequantization_step; + params.stripe_causal = job.stripe_causal; + + decode_ht_cleanup_impl( + coded_data + job.coded_offset, + decoded_data, + params, + vlc_table0, + vlc_table1, + uvlc_table0, + uvlc_table1, + status + gid + ); +} + +extern "C" __global__ void j2k_htj2k_decode_codeblocks_multi_cleanup_only( + const uchar *coded_data, + const J2kHtCleanupMultiBatchJob *jobs, + const ushort *vlc_table0, + const ushort *vlc_table1, + const ushort *uvlc_table0, + const ushort *uvlc_table1, + J2kHtStatus *status, + uint job_count +) { + const uint gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= job_count) { + return; + } + const J2kHtCleanupMultiBatchJob job = jobs[gid]; + uint *decoded_data = reinterpret_cast(static_cast(job.output_ptr)); + + J2kHtCleanupParams params; + params.width = job.width; + params.height = job.height; + params.coded_len = job.coded_len; + params.cleanup_length = job.cleanup_length; + params.refinement_length = job.refinement_length; + params.missing_msbs = job.missing_msbs; + params.num_bitplanes = job.num_bitplanes; + params.number_of_coding_passes = job.number_of_coding_passes; + params.output_stride = job.output_stride; + params.output_offset = job.output_offset; + params.dequantization_step = job.dequantization_step; + params.stripe_causal = job.stripe_causal; + + decode_ht_cleanup_impl( + coded_data + job.coded_offset, + decoded_data, + params, + vlc_table0, + vlc_table1, + uvlc_table0, + uvlc_table1, + status + gid + ); +} + +extern "C" __global__ void j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize( + const uchar *coded_data, + const J2kHtCleanupMultiBatchJob *jobs, + const ushort *vlc_table0, + const ushort *vlc_table1, + const ushort *uvlc_table0, + const ushort *uvlc_table1, + J2kHtStatus *status, + uint job_count +) { + const uint gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= job_count) { + return; + } + const J2kHtCleanupMultiBatchJob job = jobs[gid]; + uint *decoded_data = reinterpret_cast(static_cast(job.output_ptr)); + + J2kHtCleanupParams params; + params.width = job.width; + params.height = job.height; + params.coded_len = job.coded_len; + params.cleanup_length = job.cleanup_length; + params.refinement_length = job.refinement_length; + params.missing_msbs = job.missing_msbs; + params.num_bitplanes = job.num_bitplanes; + params.number_of_coding_passes = job.number_of_coding_passes; + params.output_stride = job.output_stride; + params.output_offset = job.output_offset; + params.dequantization_step = job.dequantization_step; + params.stripe_causal = job.stripe_causal; + + decode_ht_cleanup_impl( + coded_data + job.coded_offset, + decoded_data, + params, + vlc_table0, + vlc_table1, + uvlc_table0, + uvlc_table1, + status + gid + ); +} + +extern "C" __global__ void j2k_dequantize_htj2k_codeblocks( + uint *decoded_data, + const J2kHtCleanupBatchJob *jobs +) { + const uint job_idx = blockIdx.x; + const J2kHtCleanupBatchJob job = jobs[job_idx]; + const uint sample_count = job.width * job.height; + for (uint sample = threadIdx.x; sample < sample_count; sample += blockDim.x) { + const uint y = sample / job.width; + const uint x = sample - y * job.width; + const uint idx = job.output_offset + y * job.output_stride + x; + decoded_data[idx] = coefficient_to_float_bits( + decoded_data[idx], + job.num_bitplanes, + job.dequantization_step + ); + } +} + +extern "C" __global__ void j2k_dequantize_htj2k_codeblocks_multi( + const J2kHtDequantizeJob *jobs +) { + const uint job_idx = blockIdx.x; + const J2kHtDequantizeJob job = jobs[job_idx]; + uint *decoded_data = reinterpret_cast(static_cast(job.output_ptr)); + const uint sample_count = job.width * job.height; + for (uint sample = threadIdx.x; sample < sample_count; sample += blockDim.x) { + const uint y = sample / job.width; + const uint x = sample - y * job.width; + const uint idx = job.output_offset + y * job.output_stride + x; + decoded_data[idx] = coefficient_to_float_bits( + decoded_data[idx], + job.num_bitplanes, + job.dequantization_step + ); + } +} + +extern "C" __global__ void j2k_dequantize_htj2k_cleanup_jobs_multi( + const J2kHtCleanupMultiBatchJob *jobs +) { + const uint job_idx = blockIdx.x; + const J2kHtCleanupMultiBatchJob job = jobs[job_idx]; + uint *decoded_data = reinterpret_cast(static_cast(job.output_ptr)); + const uint sample_count = job.width * job.height; + for (uint sample = threadIdx.x; sample < sample_count; sample += blockDim.x) { + const uint y = sample / job.width; + const uint x = sample - y * job.width; + const uint idx = job.output_offset + y * job.output_stride + x; + decoded_data[idx] = coefficient_to_float_bits( + decoded_data[idx], + job.num_bitplanes, + job.dequantization_step + ); + } +} diff --git a/crates/j2k-cuda-runtime/src/htj2k_decode_kernels.ptx b/crates/j2k-cuda-runtime/src/htj2k_decode_kernels.ptx new file mode 100644 index 00000000..aa68ad8b --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_decode_kernels.ptx @@ -0,0 +1,24573 @@ +// +// Generated by NVIDIA NVVM Compiler +// +// Compiler Build ID: UNKNOWN +// Cuda compilation tools, release 13.2, V13.2.78 +// Based on NVVM 7.0.1 +// + +.version 9.2 +.target sm_75 +.address_size 64 + + // .globl j2k_inverse_dwt_single +.global .align 4 .b8 _ZZ20mel_decode_more_runsR10MelDecoderE7MEL_EXP[52] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5}; +// _ZZ48j2k_idwt_interleave_horizontal_53_multiE11row_samples has been demoted +// _ZZ48j2k_idwt_interleave_horizontal_97_multiE11row_samples has been demoted +// _ZZ35j2k_idwt_vertical_53_multiE14column_samples has been demoted +// _ZZ35j2k_idwt_vertical_97_multiE14column_samples has been demoted +// _ZZ41j2k_idwt_vertical_97_multi_cols4E14column_samples has been demoted + +.visible .entry j2k_inverse_dwt_single( + .param .u64 j2k_inverse_dwt_single_param_0, + .param .u64 j2k_inverse_dwt_single_param_1, + .param .u64 j2k_inverse_dwt_single_param_2, + .param .u64 j2k_inverse_dwt_single_param_3, + .param .u64 j2k_inverse_dwt_single_param_4, + .param .u64 j2k_inverse_dwt_single_param_5 +) +{ + .reg .pred %p<202>; + .reg .f32 %f<209>; + .reg .b32 %r<479>; + .reg .b64 %rd<157>; + + + ld.param.u64 %rd22, [j2k_inverse_dwt_single_param_0]; + ld.param.u64 %rd23, [j2k_inverse_dwt_single_param_1]; + ld.param.u64 %rd24, [j2k_inverse_dwt_single_param_2]; + ld.param.u64 %rd25, [j2k_inverse_dwt_single_param_3]; + ld.param.u64 %rd27, [j2k_inverse_dwt_single_param_4]; + ld.param.u64 %rd26, [j2k_inverse_dwt_single_param_5]; + cvta.to.global.u64 %rd1, %rd27; + mov.u32 %r226, %tid.x; + mov.u32 %r227, %ctaid.x; + or.b32 %r228, %r226, %r227; + setp.ne.s32 %p13, %r228, 0; + @%p13 bra $L__BB0_146; + + cvta.to.global.u64 %rd2, %rd26; + ld.global.u32 %r1, [%rd2+16]; + ld.global.u32 %r2, [%rd2+32]; + ld.global.u32 %r3, [%rd2+48]; + ld.global.u32 %r4, [%rd2+64]; + ld.global.u32 %r5, [%rd2+80]; + ld.global.u32 %r6, [%rd2+8]; + ld.global.u32 %r7, [%rd2]; + sub.s32 %r8, %r6, %r7; + ld.global.u32 %r9, [%rd2+12]; + ld.global.u32 %r10, [%rd2+4]; + sub.s32 %r11, %r9, %r10; + setp.le.u32 %p14, %r9, %r10; + @%p14 bra $L__BB0_63; + + ld.global.u32 %r12, [%rd2+20]; + ld.global.u32 %r13, [%rd2+36]; + ld.global.u32 %r14, [%rd2+52]; + ld.global.u32 %r15, [%rd2+68]; + ld.global.u32 %r19, [%rd2+28]; + ld.global.u32 %r18, [%rd2+24]; + add.s32 %r20, %r7, 1; + shr.u32 %r21, %r20, 1; + sub.s32 %r22, %r1, %r21; + ld.global.u32 %r26, [%rd2+44]; + ld.global.u32 %r25, [%rd2+40]; + shr.u32 %r229, %r7, 1; + sub.s32 %r27, %r2, %r229; + ld.global.u32 %r31, [%rd2+60]; + ld.global.u32 %r30, [%rd2+56]; + sub.s32 %r32, %r3, %r21; + ld.global.u32 %r36, [%rd2+76]; + ld.global.u32 %r35, [%rd2+72]; + sub.s32 %r37, %r4, %r229; + setp.le.u32 %p15, %r6, %r7; + @%p15 bra $L__BB0_63; + + add.s32 %r230, %r10, 1; + shr.u32 %r231, %r230, 1; + shr.u32 %r232, %r10, 1; + not.b32 %r233, %r7; + add.s32 %r38, %r6, %r233; + and.b32 %r39, %r8, 3; + and.b32 %r40, %r7, 1; + setp.eq.b32 %p1, %r40, 1; + and.b32 %r41, %r20, 1; + add.s32 %r42, %r7, 2; + shr.u32 %r234, %r42, 1; + add.s32 %r43, %r22, %r234; + setp.eq.b32 %p2, %r41, 1; + add.s32 %r44, %r27, %r21; + add.s32 %r45, %r32, %r234; + add.s32 %r46, %r37, %r21; + add.s32 %r47, %r7, 3; + shr.u32 %r235, %r47, 1; + add.s32 %r48, %r22, %r235; + add.s32 %r49, %r27, %r234; + add.s32 %r50, %r32, %r235; + add.s32 %r51, %r37, %r234; + sub.s32 %r52, %r15, %r232; + sub.s32 %r53, %r14, %r232; + sub.s32 %r54, %r13, %r231; + sub.s32 %r55, %r12, %r231; + mov.u32 %r411, %r10; + +$L__BB0_4: + and.b32 %r57, %r411, 1; + add.s32 %r236, %r411, 1; + shr.u32 %r237, %r236, 1; + add.s32 %r58, %r55, %r237; + sub.s32 %r238, %r411, %r10; + mul.lo.s32 %r59, %r238, %r8; + sub.s32 %r60, %r59, %r7; + add.s32 %r61, %r54, %r237; + setp.eq.b32 %p4, %r57, 1; + shr.u32 %r239, %r411, 1; + add.s32 %r62, %r53, %r239; + add.s32 %r63, %r52, %r239; + setp.eq.s32 %p16, %r39, 0; + mov.u32 %r430, %r7; + @%p16 bra $L__BB0_23; + + or.pred %p19, %p1, %p4; + mov.pred %p20, 0; + xor.pred %p21, %p19, %p20; + not.pred %p22, %p21; + mov.u64 %rd150, %rd22; + mov.u32 %r412, %r1; + mov.u32 %r413, %r12; + mov.u32 %r414, %r18; + mov.u32 %r415, %r19; + mov.u32 %r416, %r1; + mov.u32 %r417, %r58; + @%p22 bra $L__BB0_8; + + setp.eq.s32 %p23, %r57, 0; + and.pred %p24, %p23, %p1; + mov.u64 %rd150, %rd23; + mov.u32 %r412, %r2; + mov.u32 %r413, %r13; + mov.u32 %r414, %r25; + mov.u32 %r415, %r26; + mov.u32 %r416, %r2; + mov.u32 %r417, %r61; + @%p24 bra $L__BB0_8; + + setp.eq.s32 %p25, %r40, 0; + and.pred %p26, %p25, %p4; + selp.b64 %rd150, %rd24, %rd25, %p26; + selp.b32 %r414, %r30, %r35, %p26; + selp.b32 %r413, %r14, %r15, %p26; + selp.b32 %r412, %r3, %r4, %p26; + selp.b32 %r415, %r31, %r36, %p26; + selp.b32 %r416, %r3, %r4, %p26; + selp.b32 %r417, %r62, %r63, %p26; + +$L__BB0_8: + setp.lt.u32 %p27, %r416, %r412; + setp.le.u32 %p28, %r414, %r416; + or.pred %p29, %p27, %p28; + setp.lt.u32 %p30, %r417, %r413; + or.pred %p31, %p29, %p30; + setp.le.u32 %p32, %r415, %r417; + or.pred %p33, %p32, %p31; + mov.f32 %f202, 0f00000000; + @%p33 bra $L__BB0_10; + + cvta.to.global.u64 %rd28, %rd150; + sub.s32 %r242, %r414, %r412; + sub.s32 %r243, %r417, %r413; + sub.s32 %r244, %r416, %r412; + mad.lo.s32 %r245, %r243, %r242, %r244; + mul.wide.u32 %rd29, %r245, 4; + add.s64 %rd30, %rd28, %rd29; + ld.global.f32 %f202, [%rd30]; + +$L__BB0_10: + mul.wide.u32 %rd31, %r59, 4; + add.s64 %rd32, %rd1, %rd31; + st.global.f32 [%rd32], %f202; + setp.eq.s32 %p34, %r39, 1; + mov.u32 %r430, %r20; + @%p34 bra $L__BB0_23; + + or.pred %p37, %p2, %p4; + mov.pred %p38, 0; + xor.pred %p39, %p37, %p38; + not.pred %p40, %p39; + mov.u64 %rd151, %rd22; + mov.u32 %r418, %r1; + mov.u32 %r419, %r12; + mov.u32 %r420, %r18; + mov.u32 %r421, %r19; + mov.u32 %r422, %r43; + mov.u32 %r423, %r58; + @%p40 bra $L__BB0_14; + + setp.eq.s32 %p41, %r57, 0; + and.pred %p42, %p41, %p2; + mov.u64 %rd151, %rd23; + mov.u32 %r418, %r2; + mov.u32 %r419, %r13; + mov.u32 %r420, %r25; + mov.u32 %r421, %r26; + mov.u32 %r422, %r44; + mov.u32 %r423, %r61; + @%p42 bra $L__BB0_14; + + setp.eq.s32 %p43, %r41, 0; + and.pred %p44, %p43, %p4; + selp.b64 %rd151, %rd24, %rd25, %p44; + selp.b32 %r420, %r30, %r35, %p44; + selp.b32 %r419, %r14, %r15, %p44; + selp.b32 %r418, %r3, %r4, %p44; + selp.b32 %r421, %r31, %r36, %p44; + selp.b32 %r422, %r45, %r46, %p44; + selp.b32 %r423, %r62, %r63, %p44; + +$L__BB0_14: + setp.lt.u32 %p45, %r422, %r418; + setp.le.u32 %p46, %r420, %r422; + or.pred %p47, %p45, %p46; + setp.lt.u32 %p48, %r423, %r419; + or.pred %p49, %p47, %p48; + setp.le.u32 %p50, %r421, %r423; + or.pred %p51, %p50, %p49; + mov.f32 %f203, 0f00000000; + @%p51 bra $L__BB0_16; + + cvta.to.global.u64 %rd33, %rd151; + sub.s32 %r248, %r420, %r418; + sub.s32 %r249, %r423, %r419; + sub.s32 %r250, %r422, %r418; + mad.lo.s32 %r251, %r249, %r248, %r250; + mul.wide.u32 %rd34, %r251, 4; + add.s64 %rd35, %rd33, %rd34; + ld.global.f32 %f203, [%rd35]; + +$L__BB0_16: + add.s32 %r252, %r59, 1; + mul.wide.u32 %rd36, %r252, 4; + add.s64 %rd37, %rd1, %rd36; + st.global.f32 [%rd37], %f203; + setp.eq.s32 %p52, %r39, 2; + mov.u32 %r430, %r42; + @%p52 bra $L__BB0_23; + + and.b32 %r254, %r42, 1; + setp.eq.b32 %p54, %r254, 1; + or.pred %p55, %p54, %p4; + mov.pred %p56, 0; + xor.pred %p57, %p55, %p56; + not.pred %p58, %p57; + mov.u64 %rd152, %rd22; + mov.u32 %r424, %r1; + mov.u32 %r425, %r12; + mov.u32 %r426, %r18; + mov.u32 %r427, %r19; + mov.u32 %r428, %r48; + mov.u32 %r429, %r58; + @%p58 bra $L__BB0_20; + + setp.eq.s32 %p59, %r57, 0; + and.pred %p60, %p59, %p1; + mov.u64 %rd152, %rd23; + mov.u32 %r424, %r2; + mov.u32 %r425, %r13; + mov.u32 %r426, %r25; + mov.u32 %r427, %r26; + mov.u32 %r428, %r49; + mov.u32 %r429, %r61; + @%p60 bra $L__BB0_20; + + setp.eq.s32 %p61, %r40, 0; + and.pred %p62, %p61, %p4; + selp.b64 %rd152, %rd24, %rd25, %p62; + selp.b32 %r426, %r30, %r35, %p62; + selp.b32 %r425, %r14, %r15, %p62; + selp.b32 %r424, %r3, %r4, %p62; + selp.b32 %r427, %r31, %r36, %p62; + selp.b32 %r428, %r50, %r51, %p62; + selp.b32 %r429, %r62, %r63, %p62; + +$L__BB0_20: + setp.lt.u32 %p63, %r428, %r424; + setp.le.u32 %p64, %r426, %r428; + or.pred %p65, %p63, %p64; + setp.lt.u32 %p66, %r429, %r425; + or.pred %p67, %p65, %p66; + setp.le.u32 %p68, %r427, %r429; + or.pred %p69, %p68, %p67; + mov.f32 %f204, 0f00000000; + @%p69 bra $L__BB0_22; + + cvta.to.global.u64 %rd38, %rd152; + sub.s32 %r255, %r426, %r424; + sub.s32 %r256, %r429, %r425; + sub.s32 %r257, %r428, %r424; + mad.lo.s32 %r258, %r256, %r255, %r257; + mul.wide.u32 %rd39, %r258, 4; + add.s64 %rd40, %rd38, %rd39; + ld.global.f32 %f204, [%rd40]; + +$L__BB0_22: + add.s32 %r259, %r59, 2; + mul.wide.u32 %rd41, %r259, 4; + add.s64 %rd42, %rd1, %rd41; + st.global.f32 [%rd42], %f204; + mov.u32 %r430, %r47; + +$L__BB0_23: + setp.lt.u32 %p70, %r38, 3; + @%p70 bra $L__BB0_62; + + and.b32 %r260, %r430, 1; + or.b32 %r261, %r57, %r260; + setp.eq.s32 %p71, %r57, 0; + setp.eq.b32 %p72, %r260, 1; + setp.eq.s32 %p73, %r260, 0; + and.pred %p5, %p71, %p72; + and.pred %p6, %p73, %p4; + setp.eq.s32 %p7, %r261, 0; + and.pred %p8, %p72, %p4; + +$L__BB0_25: + and.b32 %r263, %r430, 1; + setp.eq.b32 %p75, %r263, 1; + or.pred %p76, %p75, %p4; + mov.pred %p77, 0; + xor.pred %p78, %p76, %p77; + not.pred %p79, %p78; + @%p79 bra $L__BB0_31; + bra.uni $L__BB0_26; + +$L__BB0_31: + add.s32 %r268, %r430, 1; + shr.u32 %r269, %r268, 1; + add.s32 %r436, %r22, %r269; + mov.u64 %rd153, %rd22; + mov.u32 %r432, %r1; + mov.u32 %r433, %r12; + mov.u32 %r434, %r18; + mov.u32 %r435, %r19; + mov.u32 %r437, %r58; + bra.uni $L__BB0_32; + +$L__BB0_26: + @%p5 bra $L__BB0_30; + bra.uni $L__BB0_27; + +$L__BB0_30: + shr.u32 %r267, %r430, 1; + add.s32 %r436, %r27, %r267; + mov.u64 %rd153, %rd23; + mov.u32 %r432, %r2; + mov.u32 %r433, %r13; + mov.u32 %r434, %r25; + mov.u32 %r435, %r26; + mov.u32 %r437, %r61; + bra.uni $L__BB0_32; + +$L__BB0_27: + @%p6 bra $L__BB0_29; + bra.uni $L__BB0_28; + +$L__BB0_29: + add.s32 %r265, %r430, 1; + shr.u32 %r266, %r265, 1; + add.s32 %r436, %r32, %r266; + mov.u64 %rd153, %rd24; + mov.u32 %r432, %r3; + mov.u32 %r433, %r14; + mov.u32 %r434, %r30; + mov.u32 %r435, %r31; + mov.u32 %r437, %r62; + bra.uni $L__BB0_32; + +$L__BB0_28: + shr.u32 %r264, %r430, 1; + add.s32 %r436, %r37, %r264; + mov.u64 %rd153, %rd25; + mov.u32 %r432, %r4; + mov.u32 %r433, %r15; + mov.u32 %r434, %r35; + mov.u32 %r435, %r36; + mov.u32 %r437, %r63; + +$L__BB0_32: + setp.lt.u32 %p80, %r436, %r432; + setp.le.u32 %p81, %r434, %r436; + or.pred %p82, %p80, %p81; + setp.lt.u32 %p83, %r437, %r433; + or.pred %p84, %p82, %p83; + setp.le.u32 %p85, %r435, %r437; + or.pred %p86, %p85, %p84; + mov.f32 %f205, 0f00000000; + @%p86 bra $L__BB0_34; + + cvta.to.global.u64 %rd43, %rd153; + sub.s32 %r270, %r434, %r432; + sub.s32 %r271, %r437, %r433; + sub.s32 %r272, %r436, %r432; + mad.lo.s32 %r273, %r271, %r270, %r272; + mul.wide.u32 %rd44, %r273, 4; + add.s64 %rd45, %rd43, %rd44; + ld.global.f32 %f205, [%rd45]; + +$L__BB0_34: + add.s32 %r274, %r60, %r430; + mul.wide.u32 %rd46, %r274, 4; + add.s64 %rd47, %rd1, %rd46; + st.global.f32 [%rd47], %f205; + add.s32 %r124, %r430, 1; + and.b32 %r276, %r124, 1; + setp.eq.b32 %p88, %r276, 1; + or.pred %p89, %p88, %p4; + mov.pred %p90, 0; + xor.pred %p91, %p89, %p90; + not.pred %p92, %p91; + @%p92 bra $L__BB0_40; + bra.uni $L__BB0_35; + +$L__BB0_40: + add.s32 %r281, %r430, 2; + shr.u32 %r282, %r281, 1; + add.s32 %r442, %r22, %r282; + mov.u64 %rd154, %rd22; + mov.u32 %r438, %r1; + mov.u32 %r439, %r12; + mov.u32 %r440, %r18; + mov.u32 %r441, %r19; + mov.u32 %r443, %r58; + bra.uni $L__BB0_41; + +$L__BB0_35: + @%p7 bra $L__BB0_39; + bra.uni $L__BB0_36; + +$L__BB0_39: + add.s32 %r406, %r430, 1; + shr.u32 %r280, %r406, 1; + add.s32 %r442, %r27, %r280; + mov.u64 %rd154, %rd23; + mov.u32 %r438, %r2; + mov.u32 %r439, %r13; + mov.u32 %r440, %r25; + mov.u32 %r441, %r26; + mov.u32 %r443, %r61; + bra.uni $L__BB0_41; + +$L__BB0_36: + @%p8 bra $L__BB0_38; + bra.uni $L__BB0_37; + +$L__BB0_38: + add.s32 %r278, %r430, 2; + shr.u32 %r279, %r278, 1; + add.s32 %r442, %r32, %r279; + mov.u64 %rd154, %rd24; + mov.u32 %r438, %r3; + mov.u32 %r439, %r14; + mov.u32 %r440, %r30; + mov.u32 %r441, %r31; + mov.u32 %r443, %r62; + bra.uni $L__BB0_41; + +$L__BB0_37: + add.s32 %r405, %r430, 1; + shr.u32 %r277, %r405, 1; + add.s32 %r442, %r37, %r277; + mov.u64 %rd154, %rd25; + mov.u32 %r438, %r4; + mov.u32 %r439, %r15; + mov.u32 %r440, %r35; + mov.u32 %r441, %r36; + mov.u32 %r443, %r63; + +$L__BB0_41: + setp.lt.u32 %p93, %r442, %r438; + setp.le.u32 %p94, %r440, %r442; + or.pred %p95, %p93, %p94; + setp.lt.u32 %p96, %r443, %r439; + or.pred %p97, %p95, %p96; + setp.le.u32 %p98, %r441, %r443; + or.pred %p99, %p98, %p97; + mov.f32 %f206, 0f00000000; + @%p99 bra $L__BB0_43; + + cvta.to.global.u64 %rd48, %rd154; + sub.s32 %r283, %r440, %r438; + sub.s32 %r284, %r443, %r439; + sub.s32 %r285, %r442, %r438; + mad.lo.s32 %r286, %r284, %r283, %r285; + mul.wide.u32 %rd49, %r286, 4; + add.s64 %rd50, %rd48, %rd49; + ld.global.f32 %f206, [%rd50]; + +$L__BB0_43: + add.s32 %r402, %r430, 1; + add.s32 %r287, %r60, %r402; + mul.wide.u32 %rd51, %r287, 4; + add.s64 %rd52, %rd1, %rd51; + st.global.f32 [%rd52], %f206; + add.s32 %r138, %r430, 2; + and.b32 %r289, %r138, 1; + setp.eq.b32 %p101, %r289, 1; + or.pred %p102, %p101, %p4; + mov.pred %p103, 0; + xor.pred %p104, %p102, %p103; + not.pred %p105, %p104; + @%p105 bra $L__BB0_49; + bra.uni $L__BB0_44; + +$L__BB0_49: + add.s32 %r294, %r430, 3; + shr.u32 %r295, %r294, 1; + add.s32 %r448, %r22, %r295; + mov.u64 %rd155, %rd22; + mov.u32 %r444, %r1; + mov.u32 %r445, %r12; + mov.u32 %r446, %r18; + mov.u32 %r447, %r19; + mov.u32 %r449, %r58; + bra.uni $L__BB0_50; + +$L__BB0_44: + @%p5 bra $L__BB0_48; + bra.uni $L__BB0_45; + +$L__BB0_48: + add.s32 %r408, %r430, 2; + shr.u32 %r293, %r408, 1; + add.s32 %r448, %r27, %r293; + mov.u64 %rd155, %rd23; + mov.u32 %r444, %r2; + mov.u32 %r445, %r13; + mov.u32 %r446, %r25; + mov.u32 %r447, %r26; + mov.u32 %r449, %r61; + bra.uni $L__BB0_50; + +$L__BB0_45: + @%p6 bra $L__BB0_47; + bra.uni $L__BB0_46; + +$L__BB0_47: + add.s32 %r291, %r430, 3; + shr.u32 %r292, %r291, 1; + add.s32 %r448, %r32, %r292; + mov.u64 %rd155, %rd24; + mov.u32 %r444, %r3; + mov.u32 %r445, %r14; + mov.u32 %r446, %r30; + mov.u32 %r447, %r31; + mov.u32 %r449, %r62; + bra.uni $L__BB0_50; + +$L__BB0_46: + add.s32 %r407, %r430, 2; + shr.u32 %r290, %r407, 1; + add.s32 %r448, %r37, %r290; + mov.u64 %rd155, %rd25; + mov.u32 %r444, %r4; + mov.u32 %r445, %r15; + mov.u32 %r446, %r35; + mov.u32 %r447, %r36; + mov.u32 %r449, %r63; + +$L__BB0_50: + setp.lt.u32 %p106, %r448, %r444; + setp.le.u32 %p107, %r446, %r448; + or.pred %p108, %p106, %p107; + setp.lt.u32 %p109, %r449, %r445; + or.pred %p110, %p108, %p109; + setp.le.u32 %p111, %r447, %r449; + or.pred %p112, %p111, %p110; + mov.f32 %f207, 0f00000000; + @%p112 bra $L__BB0_52; + + cvta.to.global.u64 %rd53, %rd155; + sub.s32 %r296, %r446, %r444; + sub.s32 %r297, %r449, %r445; + sub.s32 %r298, %r448, %r444; + mad.lo.s32 %r299, %r297, %r296, %r298; + mul.wide.u32 %rd54, %r299, 4; + add.s64 %rd55, %rd53, %rd54; + ld.global.f32 %f207, [%rd55]; + +$L__BB0_52: + add.s32 %r403, %r430, 2; + add.s32 %r300, %r60, %r403; + mul.wide.u32 %rd56, %r300, 4; + add.s64 %rd57, %rd1, %rd56; + st.global.f32 [%rd57], %f207; + add.s32 %r152, %r430, 3; + and.b32 %r302, %r152, 1; + setp.eq.b32 %p114, %r302, 1; + or.pred %p115, %p114, %p4; + mov.pred %p116, 0; + xor.pred %p117, %p115, %p116; + not.pred %p118, %p117; + @%p118 bra $L__BB0_58; + bra.uni $L__BB0_53; + +$L__BB0_58: + add.s32 %r307, %r430, 4; + shr.u32 %r308, %r307, 1; + add.s32 %r454, %r22, %r308; + mov.u64 %rd156, %rd22; + mov.u32 %r450, %r1; + mov.u32 %r451, %r12; + mov.u32 %r452, %r18; + mov.u32 %r453, %r19; + mov.u32 %r455, %r58; + bra.uni $L__BB0_59; + +$L__BB0_53: + @%p7 bra $L__BB0_57; + bra.uni $L__BB0_54; + +$L__BB0_57: + add.s32 %r410, %r430, 3; + shr.u32 %r306, %r410, 1; + add.s32 %r454, %r27, %r306; + mov.u64 %rd156, %rd23; + mov.u32 %r450, %r2; + mov.u32 %r451, %r13; + mov.u32 %r452, %r25; + mov.u32 %r453, %r26; + mov.u32 %r455, %r61; + bra.uni $L__BB0_59; + +$L__BB0_54: + @%p8 bra $L__BB0_56; + bra.uni $L__BB0_55; + +$L__BB0_56: + add.s32 %r304, %r430, 4; + shr.u32 %r305, %r304, 1; + add.s32 %r454, %r32, %r305; + mov.u64 %rd156, %rd24; + mov.u32 %r450, %r3; + mov.u32 %r451, %r14; + mov.u32 %r452, %r30; + mov.u32 %r453, %r31; + mov.u32 %r455, %r62; + bra.uni $L__BB0_59; + +$L__BB0_55: + add.s32 %r409, %r430, 3; + shr.u32 %r303, %r409, 1; + add.s32 %r454, %r37, %r303; + mov.u64 %rd156, %rd25; + mov.u32 %r450, %r4; + mov.u32 %r451, %r15; + mov.u32 %r452, %r35; + mov.u32 %r453, %r36; + mov.u32 %r455, %r63; + +$L__BB0_59: + setp.lt.u32 %p119, %r454, %r450; + setp.le.u32 %p120, %r452, %r454; + or.pred %p121, %p119, %p120; + setp.lt.u32 %p122, %r455, %r451; + or.pred %p123, %p121, %p122; + setp.le.u32 %p124, %r453, %r455; + or.pred %p125, %p124, %p123; + mov.f32 %f208, 0f00000000; + @%p125 bra $L__BB0_61; + + cvta.to.global.u64 %rd58, %rd156; + sub.s32 %r309, %r452, %r450; + sub.s32 %r310, %r455, %r451; + sub.s32 %r311, %r454, %r450; + mad.lo.s32 %r312, %r310, %r309, %r311; + mul.wide.u32 %rd59, %r312, 4; + add.s64 %rd60, %rd58, %rd59; + ld.global.f32 %f208, [%rd60]; + +$L__BB0_61: + add.s32 %r404, %r430, 3; + add.s32 %r313, %r60, %r404; + mul.wide.u32 %rd61, %r313, 4; + add.s64 %rd62, %rd1, %rd61; + st.global.f32 [%rd62], %f208; + add.s32 %r430, %r430, 4; + setp.lt.u32 %p126, %r430, %r6; + @%p126 bra $L__BB0_25; + +$L__BB0_62: + setp.lt.u32 %p127, %r236, %r9; + mov.u32 %r411, %r236; + @%p127 bra $L__BB0_4; + +$L__BB0_63: + setp.eq.s32 %p128, %r8, 0; + setp.eq.s32 %p129, %r11, 0; + or.pred %p130, %p128, %p129; + @%p130 bra $L__BB0_146; + + and.b32 %r168, %r7, 1; + setp.eq.s32 %p131, %r168, 0; + mov.u32 %r456, 0; + xor.b32 %r169, %r168, 1; + selp.f32 %f15, 0f3F9D7658, 0f3F5019C3, %p131; + selp.f32 %f16, 0f3F5019C3, 0f3F9D7658, %p131; + add.s32 %r315, %r8, %r8; + add.s32 %r316, %r315, -3; + setp.gt.u32 %p132, %r8, 1; + selp.b32 %r317, 1, %r316, %p132; + cvt.u64.u32 %rd13, %r317; + selp.b32 %r170, 2, 1, %p131; + mov.u32 %r318, 2; + add.s32 %r171, %r170, 1; + add.s32 %r319, %r8, -1; + cvt.u64.u32 %rd14, %r319; + and.b32 %r172, %r8, 1; + xor.b32 %r320, %r172, 1; + setp.eq.s32 %p133, %r320, %r168; + and.pred %p9, %p132, %p133; + setp.gt.u32 %p134, %r319, 1; + sub.s32 %r321, %r318, %r8; + add.s32 %r322, %r8, -2; + selp.b32 %r323, %r322, %r321, %p134; + cvt.u64.u32 %rd15, %r323; + cvt.u64.u32 %rd16, %r322; + setp.eq.b32 %p135, %r168, 1; + and.b32 %r324, %r319, 1; + setp.eq.s32 %p136, %r324, %r168; + and.pred %p10, %p132, %p136; + selp.b32 %r173, 2, 1, %p135; + add.s32 %r174, %r173, 1; + setp.eq.s32 %p137, %r172, %r168; + and.pred %p11, %p132, %p137; + setp.eq.s32 %p138, %r324, %r169; + and.pred %p12, %p132, %p138; + shl.b64 %rd64, %rd13, 2; + shl.b64 %rd65, %rd14, 2; + shl.b64 %rd66, %rd15, 2; + shl.b64 %rd67, %rd16, 2; + not.pred %p144, %p9; + not.pred %p148, %p11; + +$L__BB0_65: + mul.lo.s32 %r325, %r456, %r8; + mul.wide.u32 %rd63, %r325, 4; + add.s64 %rd17, %rd1, %rd63; + setp.eq.s32 %p139, %r8, 1; + @%p139 bra $L__BB0_114; + bra.uni $L__BB0_66; + +$L__BB0_114: + @%p131 bra $L__BB0_116; + + ld.global.f32 %f152, [%rd17]; + mul.rn.f32 %f153, %f152, 0f3F000000; + st.global.f32 [%rd17], %f153; + bra.uni $L__BB0_116; + +$L__BB0_66: + setp.ne.s32 %p140, %r5, 0; + add.s64 %rd18, %rd17, %rd64; + add.s64 %rd19, %rd17, %rd65; + add.s64 %rd20, %rd17, %rd66; + add.s64 %rd21, %rd17, %rd67; + @%p140 bra $L__BB0_81; + bra.uni $L__BB0_67; + +$L__BB0_81: + setp.lt.u32 %p149, %r8, 2; + @%p149 bra $L__BB0_84; + + mov.u32 %r461, 0; + +$L__BB0_83: + mul.wide.u32 %rd80, %r461, 4; + add.s64 %rd81, %rd17, %rd80; + ld.global.f32 %f74, [%rd81]; + mul.rn.f32 %f75, %f15, %f74; + st.global.f32 [%rd81], %f75; + ld.global.f32 %f76, [%rd81+4]; + mul.rn.f32 %f77, %f16, %f76; + st.global.f32 [%rd81+4], %f77; + add.s32 %r185, %r461, 2; + add.s32 %r329, %r461, 3; + setp.lt.u32 %p150, %r329, %r8; + mov.u32 %r461, %r185; + @%p150 bra $L__BB0_83; + +$L__BB0_84: + setp.eq.s32 %p151, %r172, 0; + @%p151 bra $L__BB0_86; + + ld.global.f32 %f78, [%rd19]; + mul.rn.f32 %f79, %f15, %f78; + st.global.f32 [%rd19], %f79; + +$L__BB0_86: + setp.ne.s32 %p152, %r168, 0; + @%p152 bra $L__BB0_88; + + ld.global.f32 %f80, [%rd18]; + ld.global.f32 %f81, [%rd17+4]; + add.rn.f32 %f82, %f81, %f80; + ld.global.f32 %f83, [%rd17]; + mov.f32 %f84, 0fBEE31355; + fma.rn.f32 %f85, %f82, %f84, %f83; + st.global.f32 [%rd17], %f85; + +$L__BB0_88: + setp.ge.u32 %p153, %r171, %r8; + @%p153 bra $L__BB0_91; + + mov.u32 %r462, %r171; + mov.u32 %r463, %r170; + +$L__BB0_90: + add.s32 %r330, %r463, -1; + mul.wide.u32 %rd82, %r330, 4; + add.s64 %rd83, %rd17, %rd82; + mul.wide.u32 %rd84, %r462, 4; + add.s64 %rd85, %rd17, %rd84; + ld.global.f32 %f86, [%rd85]; + ld.global.f32 %f87, [%rd83]; + add.rn.f32 %f88, %f87, %f86; + mul.wide.u32 %rd86, %r463, 4; + add.s64 %rd87, %rd17, %rd86; + ld.global.f32 %f89, [%rd87]; + mov.f32 %f90, 0fBEE31355; + fma.rn.f32 %f91, %f88, %f90, %f89; + st.global.f32 [%rd87], %f91; + add.s32 %r188, %r463, 2; + add.s32 %r462, %r463, 3; + setp.lt.u32 %p154, %r462, %r8; + mov.u32 %r463, %r188; + @%p154 bra $L__BB0_90; + +$L__BB0_91: + not.pred %p155, %p10; + @%p155 bra $L__BB0_93; + + ld.global.f32 %f92, [%rd20]; + ld.global.f32 %f93, [%rd21]; + add.rn.f32 %f94, %f92, %f93; + ld.global.f32 %f95, [%rd19]; + mov.f32 %f96, 0fBEE31355; + fma.rn.f32 %f97, %f94, %f96, %f95; + st.global.f32 [%rd19], %f97; + +$L__BB0_93: + setp.ne.s32 %p156, %r169, 0; + @%p156 bra $L__BB0_95; + + ld.global.f32 %f98, [%rd18]; + ld.global.f32 %f99, [%rd17+4]; + add.rn.f32 %f100, %f99, %f98; + ld.global.f32 %f101, [%rd17]; + mov.f32 %f102, 0fBF620676; + fma.rn.f32 %f103, %f100, %f102, %f101; + st.global.f32 [%rd17], %f103; + +$L__BB0_95: + setp.ge.u32 %p157, %r174, %r8; + @%p157 bra $L__BB0_98; + + mov.u32 %r464, %r174; + mov.u32 %r465, %r173; + +$L__BB0_97: + add.s32 %r331, %r465, -1; + mul.wide.u32 %rd88, %r331, 4; + add.s64 %rd89, %rd17, %rd88; + mul.wide.u32 %rd90, %r464, 4; + add.s64 %rd91, %rd17, %rd90; + ld.global.f32 %f104, [%rd91]; + ld.global.f32 %f105, [%rd89]; + add.rn.f32 %f106, %f105, %f104; + mul.wide.u32 %rd92, %r465, 4; + add.s64 %rd93, %rd17, %rd92; + ld.global.f32 %f107, [%rd93]; + mov.f32 %f108, 0fBF620676; + fma.rn.f32 %f109, %f106, %f108, %f107; + st.global.f32 [%rd93], %f109; + add.s32 %r192, %r465, 2; + add.s32 %r464, %r465, 3; + setp.lt.u32 %p158, %r464, %r8; + mov.u32 %r465, %r192; + @%p158 bra $L__BB0_97; + +$L__BB0_98: + not.pred %p159, %p12; + @%p159 bra $L__BB0_100; + + ld.global.f32 %f110, [%rd20]; + ld.global.f32 %f111, [%rd21]; + add.rn.f32 %f112, %f110, %f111; + ld.global.f32 %f113, [%rd19]; + mov.f32 %f114, 0fBF620676; + fma.rn.f32 %f115, %f112, %f114, %f113; + st.global.f32 [%rd19], %f115; + +$L__BB0_100: + @%p152 bra $L__BB0_102; + + ld.global.f32 %f116, [%rd18]; + ld.global.f32 %f117, [%rd17+4]; + add.rn.f32 %f118, %f117, %f116; + ld.global.f32 %f119, [%rd17]; + mov.f32 %f120, 0f3D5901AE; + fma.rn.f32 %f121, %f118, %f120, %f119; + st.global.f32 [%rd17], %f121; + +$L__BB0_102: + @%p153 bra $L__BB0_105; + + mov.u32 %r466, %r171; + mov.u32 %r467, %r170; + +$L__BB0_104: + add.s32 %r332, %r467, -1; + mul.wide.u32 %rd94, %r332, 4; + add.s64 %rd95, %rd17, %rd94; + mul.wide.u32 %rd96, %r466, 4; + add.s64 %rd97, %rd17, %rd96; + ld.global.f32 %f122, [%rd97]; + ld.global.f32 %f123, [%rd95]; + add.rn.f32 %f124, %f123, %f122; + mul.wide.u32 %rd98, %r467, 4; + add.s64 %rd99, %rd17, %rd98; + ld.global.f32 %f125, [%rd99]; + mov.f32 %f126, 0f3D5901AE; + fma.rn.f32 %f127, %f124, %f126, %f125; + st.global.f32 [%rd99], %f127; + add.s32 %r196, %r467, 2; + add.s32 %r466, %r467, 3; + setp.lt.u32 %p162, %r466, %r8; + mov.u32 %r467, %r196; + @%p162 bra $L__BB0_104; + +$L__BB0_105: + @%p155 bra $L__BB0_107; + + ld.global.f32 %f128, [%rd20]; + ld.global.f32 %f129, [%rd21]; + add.rn.f32 %f130, %f128, %f129; + ld.global.f32 %f131, [%rd19]; + mov.f32 %f132, 0f3D5901AE; + fma.rn.f32 %f133, %f130, %f132, %f131; + st.global.f32 [%rd19], %f133; + +$L__BB0_107: + @%p156 bra $L__BB0_109; + + ld.global.f32 %f134, [%rd18]; + ld.global.f32 %f135, [%rd17+4]; + add.rn.f32 %f136, %f135, %f134; + ld.global.f32 %f137, [%rd17]; + mov.f32 %f138, 0f3FCB0673; + fma.rn.f32 %f139, %f136, %f138, %f137; + st.global.f32 [%rd17], %f139; + +$L__BB0_109: + @%p157 bra $L__BB0_112; + + mov.u32 %r468, %r174; + mov.u32 %r469, %r173; + +$L__BB0_111: + add.s32 %r333, %r469, -1; + mul.wide.u32 %rd100, %r333, 4; + add.s64 %rd101, %rd17, %rd100; + mul.wide.u32 %rd102, %r468, 4; + add.s64 %rd103, %rd17, %rd102; + ld.global.f32 %f140, [%rd103]; + ld.global.f32 %f141, [%rd101]; + add.rn.f32 %f142, %f141, %f140; + mul.wide.u32 %rd104, %r469, 4; + add.s64 %rd105, %rd17, %rd104; + ld.global.f32 %f143, [%rd105]; + mov.f32 %f144, 0f3FCB0673; + fma.rn.f32 %f145, %f142, %f144, %f143; + st.global.f32 [%rd105], %f145; + add.s32 %r200, %r469, 2; + add.s32 %r468, %r469, 3; + setp.lt.u32 %p166, %r468, %r8; + mov.u32 %r469, %r200; + @%p166 bra $L__BB0_111; + +$L__BB0_112: + @%p159 bra $L__BB0_116; + + ld.global.f32 %f146, [%rd20]; + ld.global.f32 %f147, [%rd21]; + add.rn.f32 %f148, %f146, %f147; + ld.global.f32 %f149, [%rd19]; + mov.f32 %f150, 0f3FCB0673; + fma.rn.f32 %f151, %f148, %f150, %f149; + st.global.f32 [%rd19], %f151; + bra.uni $L__BB0_116; + +$L__BB0_67: + setp.ne.s32 %p141, %r168, 0; + @%p141 bra $L__BB0_69; + + ld.global.f32 %f26, [%rd17]; + ld.global.f32 %f27, [%rd18]; + ld.global.f32 %f28, [%rd17+4]; + add.rn.f32 %f29, %f28, %f27; + mov.f32 %f30, 0f3F000000; + mov.f32 %f31, 0f3E800000; + fma.rn.f32 %f32, %f29, %f31, %f30; + cvt.rmi.f32.f32 %f33, %f32; + sub.rn.f32 %f34, %f26, %f33; + st.global.f32 [%rd17], %f34; + +$L__BB0_69: + setp.ge.u32 %p142, %r171, %r8; + @%p142 bra $L__BB0_72; + + mov.u32 %r457, %r171; + mov.u32 %r458, %r170; + +$L__BB0_71: + mul.wide.u32 %rd68, %r458, 4; + add.s64 %rd69, %rd17, %rd68; + add.s32 %r326, %r458, -1; + mul.wide.u32 %rd70, %r326, 4; + add.s64 %rd71, %rd17, %rd70; + mul.wide.u32 %rd72, %r457, 4; + add.s64 %rd73, %rd17, %rd72; + ld.global.f32 %f35, [%rd73]; + ld.global.f32 %f36, [%rd71]; + add.rn.f32 %f37, %f36, %f35; + mov.f32 %f38, 0f3F000000; + mov.f32 %f39, 0f3E800000; + fma.rn.f32 %f40, %f37, %f39, %f38; + cvt.rmi.f32.f32 %f41, %f40; + ld.global.f32 %f42, [%rd69]; + sub.rn.f32 %f43, %f42, %f41; + st.global.f32 [%rd69], %f43; + add.s32 %r178, %r458, 2; + add.s32 %r457, %r458, 3; + setp.lt.u32 %p143, %r457, %r8; + mov.u32 %r458, %r178; + @%p143 bra $L__BB0_71; + +$L__BB0_72: + @%p144 bra $L__BB0_74; + + ld.global.f32 %f44, [%rd19]; + ld.global.f32 %f45, [%rd21]; + ld.global.f32 %f46, [%rd20]; + add.rn.f32 %f47, %f46, %f45; + mov.f32 %f48, 0f3F000000; + mov.f32 %f49, 0f3E800000; + fma.rn.f32 %f50, %f47, %f49, %f48; + cvt.rmi.f32.f32 %f51, %f50; + sub.rn.f32 %f52, %f44, %f51; + st.global.f32 [%rd19], %f52; + +$L__BB0_74: + setp.ne.s32 %p145, %r169, 0; + @%p145 bra $L__BB0_76; + + ld.global.f32 %f53, [%rd17]; + ld.global.f32 %f54, [%rd18]; + ld.global.f32 %f55, [%rd17+4]; + add.rn.f32 %f56, %f55, %f54; + mul.rn.f32 %f57, %f56, 0f3F000000; + cvt.rmi.f32.f32 %f58, %f57; + add.rn.f32 %f59, %f53, %f58; + st.global.f32 [%rd17], %f59; + +$L__BB0_76: + setp.ge.u32 %p146, %r174, %r8; + @%p146 bra $L__BB0_79; + + mov.u32 %r459, %r174; + mov.u32 %r460, %r173; + +$L__BB0_78: + mul.wide.u32 %rd74, %r460, 4; + add.s64 %rd75, %rd17, %rd74; + add.s32 %r327, %r460, -1; + mul.wide.u32 %rd76, %r327, 4; + add.s64 %rd77, %rd17, %rd76; + mul.wide.u32 %rd78, %r459, 4; + add.s64 %rd79, %rd17, %rd78; + ld.global.f32 %f60, [%rd79]; + ld.global.f32 %f61, [%rd77]; + add.rn.f32 %f62, %f61, %f60; + mul.rn.f32 %f63, %f62, 0f3F000000; + cvt.rmi.f32.f32 %f64, %f63; + ld.global.f32 %f65, [%rd75]; + add.rn.f32 %f66, %f65, %f64; + st.global.f32 [%rd75], %f66; + add.s32 %r182, %r460, 2; + add.s32 %r459, %r460, 3; + setp.lt.u32 %p147, %r459, %r8; + mov.u32 %r460, %r182; + @%p147 bra $L__BB0_78; + +$L__BB0_79: + @%p148 bra $L__BB0_116; + + ld.global.f32 %f67, [%rd19]; + ld.global.f32 %f68, [%rd21]; + ld.global.f32 %f69, [%rd20]; + add.rn.f32 %f70, %f69, %f68; + mul.rn.f32 %f71, %f70, 0f3F000000; + cvt.rmi.f32.f32 %f72, %f71; + add.rn.f32 %f73, %f67, %f72; + st.global.f32 [%rd19], %f73; + +$L__BB0_116: + add.s32 %r456, %r456, 1; + setp.lt.u32 %p169, %r456, %r11; + @%p169 bra $L__BB0_65; + + and.b32 %r203, %r10, 1; + setp.eq.s32 %p170, %r203, 0; + mov.u32 %r470, 0; + xor.b32 %r204, %r203, 1; + selp.f32 %f17, 0f3F9D7658, 0f3F5019C3, %p170; + selp.f32 %f18, 0f3F5019C3, 0f3F9D7658, %p170; + and.b32 %r205, %r11, 1; + add.s32 %r335, %r11, %r11; + add.s32 %r206, %r335, -3; + add.s32 %r336, %r11, -1; + mul.lo.s32 %r207, %r336, %r8; + +$L__BB0_118: + setp.eq.s32 %p171, %r11, 1; + @%p171 bra $L__BB0_143; + bra.uni $L__BB0_119; + +$L__BB0_143: + @%p170 bra $L__BB0_145; + + mul.wide.u32 %rd148, %r470, 4; + add.s64 %rd149, %rd1, %rd148; + ld.global.f32 %f200, [%rd149]; + mul.rn.f32 %f201, %f200, 0f3F000000; + st.global.f32 [%rd149], %f201; + bra.uni $L__BB0_145; + +$L__BB0_119: + setp.ne.s32 %p172, %r5, 0; + @%p172 bra $L__BB0_126; + bra.uni $L__BB0_120; + +$L__BB0_126: + setp.lt.u32 %p181, %r11, 2; + @%p181 bra $L__BB0_129; + + mov.u32 %r474, 0; + mov.u32 %r473, 1; + +$L__BB0_128: + mad.lo.s32 %r359, %r474, %r8, %r470; + mul.wide.u32 %rd118, %r359, 4; + add.s64 %rd119, %rd1, %rd118; + ld.global.f32 %f170, [%rd119]; + mul.rn.f32 %f171, %f17, %f170; + st.global.f32 [%rd119], %f171; + mad.lo.s32 %r360, %r473, %r8, %r470; + mul.wide.u32 %rd120, %r360, 4; + add.s64 %rd121, %rd1, %rd120; + ld.global.f32 %f172, [%rd121]; + mul.rn.f32 %f173, %f18, %f172; + st.global.f32 [%rd121], %f173; + add.s32 %r215, %r474, 2; + add.s32 %r473, %r474, 3; + setp.lt.u32 %p182, %r473, %r11; + mov.u32 %r474, %r215; + @%p182 bra $L__BB0_128; + +$L__BB0_129: + setp.eq.s32 %p183, %r205, 0; + @%p183 bra $L__BB0_131; + + add.s32 %r361, %r470, %r207; + mul.wide.u32 %rd122, %r361, 4; + add.s64 %rd123, %rd1, %rd122; + ld.global.f32 %f174, [%rd123]; + mul.rn.f32 %f175, %f17, %f174; + st.global.f32 [%rd123], %f175; + +$L__BB0_131: + setp.ge.u32 %p184, %r203, %r11; + @%p184 bra $L__BB0_134; + + mov.u32 %r475, %r203; + +$L__BB0_133: + mov.u32 %r362, 1; + sub.s32 %r363, %r362, %r475; + add.s32 %r364, %r475, -1; + setp.gt.u32 %p185, %r475, 1; + selp.b32 %r365, %r364, %r363, %p185; + add.s32 %r366, %r475, 1; + setp.lt.u32 %p186, %r366, %r11; + sub.s32 %r367, %r206, %r475; + selp.b32 %r368, %r366, %r367, %p186; + mad.lo.s32 %r369, %r475, %r8, %r470; + mad.lo.s32 %r370, %r365, %r8, %r470; + mul.wide.u32 %rd124, %r370, 4; + add.s64 %rd125, %rd1, %rd124; + mad.lo.s32 %r371, %r368, %r8, %r470; + mul.wide.u32 %rd126, %r371, 4; + add.s64 %rd127, %rd1, %rd126; + ld.global.f32 %f176, [%rd127]; + ld.global.f32 %f177, [%rd125]; + add.rn.f32 %f178, %f177, %f176; + mul.wide.u32 %rd128, %r369, 4; + add.s64 %rd129, %rd1, %rd128; + ld.global.f32 %f179, [%rd129]; + mov.f32 %f180, 0fBEE31355; + fma.rn.f32 %f181, %f178, %f180, %f179; + st.global.f32 [%rd129], %f181; + add.s32 %r475, %r475, 2; + setp.lt.u32 %p187, %r475, %r11; + @%p187 bra $L__BB0_133; + +$L__BB0_134: + setp.ge.u32 %p188, %r204, %r11; + @%p188 bra $L__BB0_137; + + mov.u32 %r476, %r204; + +$L__BB0_136: + mov.u32 %r372, 1; + sub.s32 %r373, %r372, %r476; + add.s32 %r374, %r476, -1; + setp.gt.u32 %p189, %r476, 1; + selp.b32 %r375, %r374, %r373, %p189; + add.s32 %r376, %r476, 1; + setp.lt.u32 %p190, %r376, %r11; + sub.s32 %r377, %r206, %r476; + selp.b32 %r378, %r376, %r377, %p190; + mad.lo.s32 %r379, %r476, %r8, %r470; + mad.lo.s32 %r380, %r375, %r8, %r470; + mul.wide.u32 %rd130, %r380, 4; + add.s64 %rd131, %rd1, %rd130; + mad.lo.s32 %r381, %r378, %r8, %r470; + mul.wide.u32 %rd132, %r381, 4; + add.s64 %rd133, %rd1, %rd132; + ld.global.f32 %f182, [%rd133]; + ld.global.f32 %f183, [%rd131]; + add.rn.f32 %f184, %f183, %f182; + mul.wide.u32 %rd134, %r379, 4; + add.s64 %rd135, %rd1, %rd134; + ld.global.f32 %f185, [%rd135]; + mov.f32 %f186, 0fBF620676; + fma.rn.f32 %f187, %f184, %f186, %f185; + st.global.f32 [%rd135], %f187; + add.s32 %r476, %r476, 2; + setp.lt.u32 %p191, %r476, %r11; + @%p191 bra $L__BB0_136; + +$L__BB0_137: + @%p184 bra $L__BB0_140; + + mov.u32 %r477, %r203; + +$L__BB0_139: + mov.u32 %r382, 1; + sub.s32 %r383, %r382, %r477; + add.s32 %r384, %r477, -1; + setp.gt.u32 %p193, %r477, 1; + selp.b32 %r385, %r384, %r383, %p193; + add.s32 %r386, %r477, 1; + setp.lt.u32 %p194, %r386, %r11; + sub.s32 %r387, %r206, %r477; + selp.b32 %r388, %r386, %r387, %p194; + mad.lo.s32 %r389, %r477, %r8, %r470; + mad.lo.s32 %r390, %r385, %r8, %r470; + mul.wide.u32 %rd136, %r390, 4; + add.s64 %rd137, %rd1, %rd136; + mad.lo.s32 %r391, %r388, %r8, %r470; + mul.wide.u32 %rd138, %r391, 4; + add.s64 %rd139, %rd1, %rd138; + ld.global.f32 %f188, [%rd139]; + ld.global.f32 %f189, [%rd137]; + add.rn.f32 %f190, %f189, %f188; + mul.wide.u32 %rd140, %r389, 4; + add.s64 %rd141, %rd1, %rd140; + ld.global.f32 %f191, [%rd141]; + mov.f32 %f192, 0f3D5901AE; + fma.rn.f32 %f193, %f190, %f192, %f191; + st.global.f32 [%rd141], %f193; + add.s32 %r477, %r477, 2; + setp.lt.u32 %p195, %r477, %r11; + @%p195 bra $L__BB0_139; + +$L__BB0_140: + @%p188 bra $L__BB0_145; + + mov.u32 %r478, %r204; + +$L__BB0_142: + mov.u32 %r392, 1; + sub.s32 %r393, %r392, %r478; + add.s32 %r394, %r478, -1; + setp.gt.u32 %p197, %r478, 1; + selp.b32 %r395, %r394, %r393, %p197; + add.s32 %r396, %r478, 1; + setp.lt.u32 %p198, %r396, %r11; + sub.s32 %r397, %r206, %r478; + selp.b32 %r398, %r396, %r397, %p198; + mad.lo.s32 %r399, %r478, %r8, %r470; + mad.lo.s32 %r400, %r395, %r8, %r470; + mul.wide.u32 %rd142, %r400, 4; + add.s64 %rd143, %rd1, %rd142; + mad.lo.s32 %r401, %r398, %r8, %r470; + mul.wide.u32 %rd144, %r401, 4; + add.s64 %rd145, %rd1, %rd144; + ld.global.f32 %f194, [%rd145]; + ld.global.f32 %f195, [%rd143]; + add.rn.f32 %f196, %f195, %f194; + mul.wide.u32 %rd146, %r399, 4; + add.s64 %rd147, %rd1, %rd146; + ld.global.f32 %f197, [%rd147]; + mov.f32 %f198, 0f3FCB0673; + fma.rn.f32 %f199, %f196, %f198, %f197; + st.global.f32 [%rd147], %f199; + add.s32 %r478, %r478, 2; + setp.lt.u32 %p199, %r478, %r11; + @%p199 bra $L__BB0_142; + bra.uni $L__BB0_145; + +$L__BB0_120: + setp.ge.u32 %p173, %r203, %r11; + @%p173 bra $L__BB0_123; + + mov.u32 %r471, %r203; + +$L__BB0_122: + mov.u32 %r337, 1; + sub.s32 %r338, %r337, %r471; + add.s32 %r339, %r471, -1; + setp.gt.u32 %p174, %r471, 1; + selp.b32 %r340, %r339, %r338, %p174; + add.s32 %r341, %r471, 1; + setp.lt.u32 %p175, %r341, %r11; + sub.s32 %r342, %r206, %r471; + selp.b32 %r343, %r341, %r342, %p175; + mad.lo.s32 %r344, %r471, %r8, %r470; + mad.lo.s32 %r345, %r340, %r8, %r470; + mul.wide.u32 %rd106, %r345, 4; + add.s64 %rd107, %rd1, %rd106; + mad.lo.s32 %r346, %r343, %r8, %r470; + mul.wide.u32 %rd108, %r346, 4; + add.s64 %rd109, %rd1, %rd108; + mul.wide.u32 %rd110, %r344, 4; + add.s64 %rd111, %rd1, %rd110; + ld.global.f32 %f154, [%rd109]; + ld.global.f32 %f155, [%rd107]; + add.rn.f32 %f156, %f155, %f154; + mov.f32 %f157, 0f3F000000; + mov.f32 %f158, 0f3E800000; + fma.rn.f32 %f159, %f156, %f158, %f157; + cvt.rmi.f32.f32 %f160, %f159; + ld.global.f32 %f161, [%rd111]; + sub.rn.f32 %f162, %f161, %f160; + st.global.f32 [%rd111], %f162; + add.s32 %r471, %r471, 2; + setp.lt.u32 %p176, %r471, %r11; + @%p176 bra $L__BB0_122; + +$L__BB0_123: + setp.ge.u32 %p177, %r204, %r11; + @%p177 bra $L__BB0_145; + + mov.u32 %r472, %r204; + +$L__BB0_125: + mov.u32 %r347, 1; + sub.s32 %r348, %r347, %r472; + add.s32 %r349, %r472, -1; + setp.gt.u32 %p178, %r472, 1; + selp.b32 %r350, %r349, %r348, %p178; + add.s32 %r351, %r472, 1; + setp.lt.u32 %p179, %r351, %r11; + sub.s32 %r352, %r206, %r472; + selp.b32 %r353, %r351, %r352, %p179; + mad.lo.s32 %r354, %r472, %r8, %r470; + mad.lo.s32 %r355, %r350, %r8, %r470; + mul.wide.u32 %rd112, %r355, 4; + add.s64 %rd113, %rd1, %rd112; + mad.lo.s32 %r356, %r353, %r8, %r470; + mul.wide.u32 %rd114, %r356, 4; + add.s64 %rd115, %rd1, %rd114; + mul.wide.u32 %rd116, %r354, 4; + add.s64 %rd117, %rd1, %rd116; + ld.global.f32 %f163, [%rd115]; + ld.global.f32 %f164, [%rd113]; + add.rn.f32 %f165, %f164, %f163; + mul.rn.f32 %f166, %f165, 0f3F000000; + cvt.rmi.f32.f32 %f167, %f166; + ld.global.f32 %f168, [%rd117]; + add.rn.f32 %f169, %f168, %f167; + st.global.f32 [%rd117], %f169; + add.s32 %r472, %r472, 2; + setp.lt.u32 %p180, %r472, %r11; + @%p180 bra $L__BB0_125; + +$L__BB0_145: + add.s32 %r470, %r470, 1; + setp.lt.u32 %p201, %r470, %r8; + @%p201 bra $L__BB0_118; + +$L__BB0_146: + ret; + +} + // .globl j2k_idwt_interleave +.visible .entry j2k_idwt_interleave( + .param .u64 j2k_idwt_interleave_param_0, + .param .u64 j2k_idwt_interleave_param_1, + .param .u64 j2k_idwt_interleave_param_2, + .param .u64 j2k_idwt_interleave_param_3, + .param .u64 j2k_idwt_interleave_param_4, + .param .u64 j2k_idwt_interleave_param_5 +) +{ + .reg .pred %p<23>; + .reg .f32 %f<5>; + .reg .b32 %r<90>; + .reg .b64 %rd<16>; + + + ld.param.u64 %rd3, [j2k_idwt_interleave_param_0]; + ld.param.u64 %rd4, [j2k_idwt_interleave_param_1]; + ld.param.u64 %rd5, [j2k_idwt_interleave_param_2]; + ld.param.u64 %rd15, [j2k_idwt_interleave_param_3]; + ld.param.u64 %rd7, [j2k_idwt_interleave_param_4]; + ld.param.u64 %rd8, [j2k_idwt_interleave_param_5]; + cvta.to.global.u64 %rd1, %rd8; + ld.global.u32 %r1, [%rd1+20]; + ld.global.u32 %r2, [%rd1+36]; + ld.global.u32 %r3, [%rd1+52]; + ld.global.u32 %r84, [%rd1+68]; + ld.global.u32 %r50, [%rd1+8]; + ld.global.u32 %r5, [%rd1]; + sub.s32 %r6, %r50, %r5; + ld.global.u32 %r51, [%rd1+12]; + ld.global.u32 %r88, [%rd1+4]; + sub.s32 %r52, %r51, %r88; + mov.u32 %r53, %ntid.x; + mov.u32 %r54, %ctaid.x; + mov.u32 %r55, %tid.x; + mad.lo.s32 %r8, %r54, %r53, %r55; + mov.u32 %r56, %ntid.y; + mov.u32 %r57, %ctaid.y; + mov.u32 %r58, %tid.y; + mad.lo.s32 %r9, %r57, %r56, %r58; + setp.ge.u32 %p1, %r8, %r6; + setp.ge.u32 %p2, %r9, %r52; + or.pred %p3, %p1, %p2; + @%p3 bra $L__BB1_11; + + add.s32 %r10, %r5, %r8; + and.b32 %r11, %r10, 1; + add.s32 %r89, %r88, %r9; + and.b32 %r13, %r89, 1; + setp.eq.b32 %p4, %r11, 1; + setp.eq.b32 %p5, %r13, 1; + or.pred %p6, %p5, %p4; + mov.pred %p7, 0; + xor.pred %p8, %p6, %p7; + not.pred %p9, %p8; + @%p9 bra $L__BB1_7; + bra.uni $L__BB1_2; + +$L__BB1_7: + ld.global.u32 %r86, [%rd1+28]; + ld.global.u32 %r85, [%rd1+24]; + ld.global.u32 %r83, [%rd1+16]; + add.s32 %r70, %r10, 1; + shr.u32 %r71, %r70, 1; + add.s32 %r72, %r5, 1; + shr.u32 %r73, %r72, 1; + sub.s32 %r74, %r71, %r73; + add.s32 %r87, %r74, %r83; + add.s32 %r88, %r88, 1; + add.s32 %r89, %r89, 1; + mov.u64 %rd15, %rd3; + mov.u32 %r84, %r1; + bra.uni $L__BB1_8; + +$L__BB1_2: + setp.eq.s32 %p10, %r13, 0; + setp.ne.s32 %p11, %r11, 0; + and.pred %p12, %p10, %p11; + @%p12 bra $L__BB1_6; + bra.uni $L__BB1_3; + +$L__BB1_6: + ld.global.u32 %r86, [%rd1+44]; + ld.global.u32 %r85, [%rd1+40]; + ld.global.u32 %r83, [%rd1+32]; + shr.u32 %r67, %r5, 1; + shr.u32 %r68, %r10, 1; + sub.s32 %r69, %r68, %r67; + add.s32 %r87, %r69, %r83; + add.s32 %r88, %r88, 1; + add.s32 %r89, %r89, 1; + mov.u64 %rd15, %rd4; + mov.u32 %r84, %r2; + bra.uni $L__BB1_8; + +$L__BB1_3: + setp.ne.s32 %p13, %r13, 0; + setp.eq.s32 %p14, %r11, 0; + and.pred %p15, %p14, %p13; + @%p15 bra $L__BB1_5; + bra.uni $L__BB1_4; + +$L__BB1_5: + ld.global.u32 %r86, [%rd1+60]; + ld.global.u32 %r85, [%rd1+56]; + ld.global.u32 %r83, [%rd1+48]; + add.s32 %r62, %r10, 1; + shr.u32 %r63, %r62, 1; + add.s32 %r64, %r5, 1; + shr.u32 %r65, %r64, 1; + sub.s32 %r66, %r63, %r65; + add.s32 %r87, %r66, %r83; + mov.u64 %rd15, %rd5; + mov.u32 %r84, %r3; + bra.uni $L__BB1_8; + +$L__BB1_4: + ld.global.u32 %r86, [%rd1+76]; + ld.global.u32 %r85, [%rd1+72]; + ld.global.u32 %r83, [%rd1+64]; + shr.u32 %r59, %r5, 1; + shr.u32 %r60, %r10, 1; + sub.s32 %r61, %r60, %r59; + add.s32 %r87, %r61, %r83; + +$L__BB1_8: + shr.u32 %r75, %r88, 1; + sub.s32 %r76, %r84, %r75; + shr.u32 %r77, %r89, 1; + add.s32 %r46, %r76, %r77; + setp.lt.u32 %p16, %r87, %r83; + setp.le.u32 %p17, %r85, %r87; + or.pred %p18, %p16, %p17; + setp.lt.u32 %p19, %r46, %r84; + or.pred %p20, %p18, %p19; + setp.le.u32 %p21, %r86, %r46; + or.pred %p22, %p21, %p20; + mov.f32 %f4, 0f00000000; + @%p22 bra $L__BB1_10; + + cvta.to.global.u64 %rd9, %rd15; + sub.s32 %r78, %r85, %r83; + sub.s32 %r79, %r46, %r84; + sub.s32 %r80, %r87, %r83; + mad.lo.s32 %r81, %r79, %r78, %r80; + mul.wide.u32 %rd10, %r81, 4; + add.s64 %rd11, %rd9, %rd10; + ld.global.f32 %f4, [%rd11]; + +$L__BB1_10: + mad.lo.s32 %r82, %r6, %r9, %r8; + cvta.to.global.u64 %rd12, %rd7; + mul.wide.u32 %rd13, %r82, 4; + add.s64 %rd14, %rd12, %rd13; + st.global.f32 [%rd14], %f4; + +$L__BB1_11: + ret; + +} + // .globl j2k_idwt_interleave_multi +.visible .entry j2k_idwt_interleave_multi( + .param .u64 j2k_idwt_interleave_multi_param_0 +) +{ + .reg .pred %p<23>; + .reg .f32 %f<5>; + .reg .b32 %r<119>; + .reg .b64 %rd<17>; + + + ld.param.u64 %rd8, [j2k_idwt_interleave_multi_param_0]; + cvta.to.global.u64 %rd9, %rd8; + mov.u32 %r58, %ctaid.z; + mul.wide.u32 %rd10, %r58, 128; + add.s64 %rd11, %rd9, %rd10; + add.s64 %rd1, %rd11, 112; + ld.global.v2.u32 {%r112, %r113}, [%rd11+104]; + ld.global.v2.u32 {%r61, %r62}, [%rd11+88]; + ld.global.v2.u32 {%r63, %r64}, [%rd11+72]; + ld.global.v2.u32 {%r65, %r66}, [%rd11+56]; + ld.global.v2.u32 {%r67, %r68}, [%rd11+48]; + ld.global.v2.u32 {%r71, %r117}, [%rd11+40]; + sub.s32 %r15, %r67, %r71; + sub.s32 %r73, %r68, %r117; + mov.u32 %r74, %ntid.x; + mov.u32 %r75, %ctaid.x; + mov.u32 %r76, %tid.x; + mad.lo.s32 %r16, %r75, %r74, %r76; + mov.u32 %r77, %ntid.y; + mov.u32 %r78, %ctaid.y; + mov.u32 %r79, %tid.y; + mad.lo.s32 %r17, %r78, %r77, %r79; + setp.ge.u32 %p1, %r16, %r15; + setp.ge.u32 %p2, %r17, %r73; + or.pred %p3, %p2, %p1; + @%p3 bra $L__BB2_11; + + ld.global.u64 %rd2, [%rd1+-80]; + add.s32 %r18, %r71, %r16; + and.b32 %r19, %r18, 1; + add.s32 %r118, %r117, %r17; + and.b32 %r21, %r118, 1; + setp.eq.b32 %p4, %r21, 1; + setp.eq.b32 %p5, %r19, 1; + or.pred %p6, %p5, %p4; + mov.pred %p7, 0; + xor.pred %p8, %p6, %p7; + not.pred %p9, %p8; + @%p9 bra $L__BB2_7; + bra.uni $L__BB2_2; + +$L__BB2_7: + ld.global.v2.u32 {%r114, %r115}, [%rd1+-48]; + ld.global.u64 %rd16, [%rd1+-112]; + add.s32 %r99, %r18, 1; + shr.u32 %r100, %r99, 1; + add.s32 %r101, %r71, 1; + shr.u32 %r102, %r101, 1; + sub.s32 %r103, %r65, %r102; + add.s32 %r116, %r103, %r100; + add.s32 %r117, %r117, 1; + add.s32 %r118, %r118, 1; + mov.u32 %r112, %r65; + mov.u32 %r113, %r66; + bra.uni $L__BB2_8; + +$L__BB2_2: + setp.eq.s32 %p10, %r21, 0; + setp.ne.s32 %p11, %r19, 0; + and.pred %p12, %p10, %p11; + @%p12 bra $L__BB2_6; + bra.uni $L__BB2_3; + +$L__BB2_6: + ld.global.v2.u32 {%r114, %r115}, [%rd1+-32]; + ld.global.u64 %rd16, [%rd1+-104]; + shr.u32 %r94, %r71, 1; + sub.s32 %r95, %r63, %r94; + shr.u32 %r96, %r18, 1; + add.s32 %r116, %r95, %r96; + add.s32 %r117, %r117, 1; + add.s32 %r118, %r118, 1; + mov.u32 %r112, %r63; + mov.u32 %r113, %r64; + bra.uni $L__BB2_8; + +$L__BB2_3: + setp.ne.s32 %p13, %r21, 0; + setp.eq.s32 %p14, %r19, 0; + and.pred %p15, %p14, %p13; + @%p15 bra $L__BB2_5; + bra.uni $L__BB2_4; + +$L__BB2_5: + ld.global.v2.u32 {%r114, %r115}, [%rd1+-16]; + ld.global.u64 %rd16, [%rd1+-96]; + add.s32 %r87, %r18, 1; + shr.u32 %r88, %r87, 1; + add.s32 %r89, %r71, 1; + shr.u32 %r90, %r89, 1; + sub.s32 %r91, %r61, %r90; + add.s32 %r116, %r91, %r88; + mov.u32 %r112, %r61; + mov.u32 %r113, %r62; + bra.uni $L__BB2_8; + +$L__BB2_4: + ld.global.v2.u32 {%r114, %r115}, [%rd1]; + ld.global.u64 %rd16, [%rd1+-88]; + shr.u32 %r82, %r71, 1; + sub.s32 %r83, %r112, %r82; + shr.u32 %r84, %r18, 1; + add.s32 %r116, %r83, %r84; + +$L__BB2_8: + shr.u32 %r104, %r117, 1; + sub.s32 %r105, %r113, %r104; + shr.u32 %r106, %r118, 1; + add.s32 %r54, %r105, %r106; + setp.lt.u32 %p16, %r116, %r112; + setp.le.u32 %p17, %r114, %r116; + or.pred %p18, %p16, %p17; + setp.lt.u32 %p19, %r54, %r113; + or.pred %p20, %p18, %p19; + setp.le.u32 %p21, %r115, %r54; + or.pred %p22, %p21, %p20; + mov.f32 %f4, 0f00000000; + @%p22 bra $L__BB2_10; + + sub.s32 %r107, %r114, %r112; + sub.s32 %r108, %r54, %r113; + sub.s32 %r109, %r116, %r112; + mad.lo.s32 %r110, %r108, %r107, %r109; + mul.wide.u32 %rd12, %r110, 4; + add.s64 %rd13, %rd16, %rd12; + ld.f32 %f4, [%rd13]; + +$L__BB2_10: + mad.lo.s32 %r111, %r15, %r17, %r16; + mul.wide.u32 %rd14, %r111, 4; + add.s64 %rd15, %rd2, %rd14; + st.f32 [%rd15], %f4; + +$L__BB2_11: + ret; + +} + // .globl j2k_idwt_interleave_horizontal_multi +.visible .entry j2k_idwt_interleave_horizontal_multi( + .param .u64 j2k_idwt_interleave_horizontal_multi_param_0 +) +{ + .reg .pred %p<118>; + .reg .f32 %f<143>; + .reg .b32 %r<311>; + .reg .b64 %rd<95>; + + + ld.param.u64 %rd13, [j2k_idwt_interleave_horizontal_multi_param_0]; + cvta.to.global.u64 %rd14, %rd13; + mov.u32 %r143, %ctaid.y; + mul.wide.u32 %rd15, %r143, 128; + add.s64 %rd16, %rd14, %rd15; + add.s64 %rd1, %rd16, 120; + ld.global.v2.u32 {%r291, %r292}, [%rd16+104]; + ld.global.v2.u32 {%r146, %r147}, [%rd16+88]; + ld.global.v2.u32 {%r148, %r149}, [%rd16+72]; + ld.global.v2.u32 {%r150, %r151}, [%rd16+56]; + ld.global.v2.u32 {%r152, %r153}, [%rd16+48]; + ld.global.v2.u32 {%r155, %r296}, [%rd16+40]; + ld.global.u64 %rd94, [%rd16+24]; + ld.global.u64 %rd3, [%rd16+16]; + ld.global.u64 %rd4, [%rd16+8]; + ld.global.u64 %rd5, [%rd16]; + sub.s32 %r16, %r152, %r155; + sub.s32 %r157, %r153, %r296; + mov.u32 %r158, %ntid.x; + mov.u32 %r159, %ctaid.x; + mov.u32 %r160, %tid.x; + mad.lo.s32 %r17, %r159, %r158, %r160; + setp.ge.u32 %p9, %r17, %r157; + @%p9 bra $L__BB3_81; + + ld.global.u32 %r18, [%rd1]; + add.s32 %r297, %r296, %r17; + and.b32 %r20, %r297, 1; + mul.lo.s32 %r161, %r16, %r17; + ld.global.u64 %rd17, [%rd1+-88]; + mul.wide.u32 %rd18, %r161, 4; + add.s64 %rd6, %rd17, %rd18; + setp.eq.s32 %p10, %r16, 0; + @%p10 bra $L__BB3_34; + + ld.global.v2.u32 {%r293, %r294}, [%rd1+-8]; + mov.u32 %r290, 0; + ld.global.v2.u32 {%r165, %r166}, [%rd1+-24]; + ld.global.v2.u32 {%r167, %r168}, [%rd1+-40]; + ld.global.v2.u32 {%r169, %r170}, [%rd1+-56]; + add.s32 %r171, %r155, 1; + shr.u32 %r172, %r171, 1; + sub.s32 %r25, %r150, %r172; + add.s32 %r26, %r296, 1; + add.s32 %r27, %r297, 1; + shr.u32 %r173, %r155, 1; + sub.s32 %r32, %r148, %r173; + setp.ne.s32 %p1, %r20, 0; + sub.s32 %r37, %r146, %r172; + sub.s32 %r42, %r291, %r173; + and.b32 %r43, %r16, 1; + setp.eq.s32 %p11, %r152, %r171; + @%p11 bra $L__BB3_23; + + and.b32 %r175, %r155, 1; + or.b32 %r176, %r20, %r175; + setp.eq.s32 %p12, %r20, 0; + mov.u32 %r290, 0; + sub.s32 %r275, %r16, %r43; + setp.eq.b32 %p13, %r175, 1; + setp.eq.s32 %p14, %r175, 0; + and.pred %p2, %p12, %p13; + and.pred %p3, %p14, %p1; + setp.eq.s32 %p4, %r176, 0; + and.pred %p5, %p13, %p1; + +$L__BB3_4: + add.s32 %r47, %r290, %r155; + and.b32 %r177, %r47, 1; + setp.eq.b32 %p15, %r177, 1; + setp.eq.b32 %p16, %r20, 1; + or.pred %p17, %p15, %p16; + mov.pred %p18, 0; + xor.pred %p19, %p17, %p18; + not.pred %p20, %p19; + @%p20 bra $L__BB3_10; + bra.uni $L__BB3_5; + +$L__BB3_10: + add.s32 %r183, %r47, 1; + shr.u32 %r184, %r183, 1; + add.s32 %r280, %r25, %r184; + mov.u64 %rd92, %rd5; + mov.u32 %r276, %r150; + mov.u32 %r277, %r151; + mov.u32 %r278, %r169; + mov.u32 %r279, %r170; + mov.u32 %r281, %r26; + mov.u32 %r282, %r27; + bra.uni $L__BB3_11; + +$L__BB3_5: + @%p2 bra $L__BB3_9; + bra.uni $L__BB3_6; + +$L__BB3_9: + shr.u32 %r182, %r47, 1; + add.s32 %r280, %r32, %r182; + mov.u64 %rd92, %rd4; + mov.u32 %r276, %r148; + mov.u32 %r277, %r149; + mov.u32 %r278, %r167; + mov.u32 %r279, %r168; + mov.u32 %r281, %r26; + mov.u32 %r282, %r27; + bra.uni $L__BB3_11; + +$L__BB3_6: + @%p3 bra $L__BB3_8; + bra.uni $L__BB3_7; + +$L__BB3_8: + add.s32 %r180, %r47, 1; + shr.u32 %r181, %r180, 1; + add.s32 %r280, %r37, %r181; + mov.u64 %rd92, %rd3; + mov.u32 %r276, %r146; + mov.u32 %r277, %r147; + mov.u32 %r278, %r165; + mov.u32 %r279, %r166; + mov.u32 %r281, %r296; + mov.u32 %r282, %r297; + bra.uni $L__BB3_11; + +$L__BB3_7: + shr.u32 %r179, %r47, 1; + add.s32 %r280, %r42, %r179; + mov.u64 %rd92, %rd94; + mov.u32 %r276, %r291; + mov.u32 %r277, %r292; + mov.u32 %r278, %r293; + mov.u32 %r279, %r294; + mov.u32 %r281, %r296; + mov.u32 %r282, %r297; + +$L__BB3_11: + shr.u32 %r185, %r281, 1; + sub.s32 %r186, %r277, %r185; + shr.u32 %r187, %r282, 1; + add.s32 %r60, %r186, %r187; + setp.lt.u32 %p21, %r280, %r276; + setp.le.u32 %p22, %r278, %r280; + or.pred %p23, %p21, %p22; + setp.lt.u32 %p24, %r60, %r277; + or.pred %p25, %p23, %p24; + setp.le.u32 %p26, %r279, %r60; + or.pred %p27, %p26, %p25; + mov.f32 %f140, 0f00000000; + @%p27 bra $L__BB3_13; + + sub.s32 %r188, %r278, %r276; + sub.s32 %r189, %r60, %r277; + sub.s32 %r190, %r280, %r276; + mad.lo.s32 %r191, %r189, %r188, %r190; + mul.wide.u32 %rd19, %r191, 4; + add.s64 %rd20, %rd92, %rd19; + ld.f32 %f140, [%rd20]; + +$L__BB3_13: + mul.wide.u32 %rd21, %r290, 4; + add.s64 %rd8, %rd6, %rd21; + st.f32 [%rd8], %f140; + add.s32 %r64, %r47, 1; + and.b32 %r193, %r64, 1; + setp.eq.b32 %p28, %r193, 1; + or.pred %p30, %p28, %p16; + mov.pred %p31, 0; + xor.pred %p32, %p30, %p31; + not.pred %p33, %p32; + @%p33 bra $L__BB3_19; + bra.uni $L__BB3_14; + +$L__BB3_19: + add.s32 %r199, %r64, 1; + shr.u32 %r200, %r199, 1; + add.s32 %r287, %r25, %r200; + mov.u64 %rd93, %rd5; + mov.u32 %r283, %r150; + mov.u32 %r284, %r151; + mov.u32 %r285, %r169; + mov.u32 %r286, %r170; + mov.u32 %r288, %r26; + mov.u32 %r289, %r27; + bra.uni $L__BB3_20; + +$L__BB3_14: + @%p4 bra $L__BB3_18; + bra.uni $L__BB3_15; + +$L__BB3_18: + shr.u32 %r198, %r64, 1; + add.s32 %r287, %r32, %r198; + mov.u64 %rd93, %rd4; + mov.u32 %r283, %r148; + mov.u32 %r284, %r149; + mov.u32 %r285, %r167; + mov.u32 %r286, %r168; + mov.u32 %r288, %r26; + mov.u32 %r289, %r27; + bra.uni $L__BB3_20; + +$L__BB3_15: + @%p5 bra $L__BB3_17; + bra.uni $L__BB3_16; + +$L__BB3_17: + add.s32 %r196, %r64, 1; + shr.u32 %r197, %r196, 1; + add.s32 %r287, %r37, %r197; + mov.u64 %rd93, %rd3; + mov.u32 %r283, %r146; + mov.u32 %r284, %r147; + mov.u32 %r285, %r165; + mov.u32 %r286, %r166; + mov.u32 %r288, %r296; + mov.u32 %r289, %r297; + bra.uni $L__BB3_20; + +$L__BB3_16: + shr.u32 %r195, %r64, 1; + add.s32 %r287, %r42, %r195; + mov.u64 %rd93, %rd94; + mov.u32 %r283, %r291; + mov.u32 %r284, %r292; + mov.u32 %r285, %r293; + mov.u32 %r286, %r294; + mov.u32 %r288, %r296; + mov.u32 %r289, %r297; + +$L__BB3_20: + shr.u32 %r201, %r288, 1; + sub.s32 %r202, %r284, %r201; + shr.u32 %r203, %r289, 1; + add.s32 %r77, %r202, %r203; + setp.lt.u32 %p34, %r287, %r283; + setp.le.u32 %p35, %r285, %r287; + or.pred %p36, %p34, %p35; + setp.lt.u32 %p37, %r77, %r284; + or.pred %p38, %p36, %p37; + setp.le.u32 %p39, %r286, %r77; + or.pred %p40, %p39, %p38; + mov.f32 %f141, 0f00000000; + @%p40 bra $L__BB3_22; + + sub.s32 %r204, %r285, %r283; + sub.s32 %r205, %r77, %r284; + sub.s32 %r206, %r287, %r283; + mad.lo.s32 %r207, %r205, %r204, %r206; + mul.wide.u32 %rd22, %r207, 4; + add.s64 %rd23, %rd93, %rd22; + ld.f32 %f141, [%rd23]; + +$L__BB3_22: + st.f32 [%rd8+4], %f141; + add.s32 %r290, %r290, 2; + add.s32 %r275, %r275, -2; + setp.ne.s32 %p41, %r275, 0; + @%p41 bra $L__BB3_4; + +$L__BB3_23: + setp.eq.s32 %p42, %r43, 0; + @%p42 bra $L__BB3_34; + + add.s32 %r84, %r290, %r155; + and.b32 %r85, %r84, 1; + setp.eq.b32 %p43, %r85, 1; + setp.eq.b32 %p44, %r20, 1; + or.pred %p45, %p43, %p44; + mov.pred %p46, 0; + xor.pred %p47, %p45, %p46; + not.pred %p48, %p47; + @%p48 bra $L__BB3_30; + bra.uni $L__BB3_25; + +$L__BB3_30: + add.s32 %r213, %r84, 1; + shr.u32 %r214, %r213, 1; + add.s32 %r295, %r25, %r214; + mov.u64 %rd94, %rd5; + mov.u32 %r291, %r150; + mov.u32 %r292, %r151; + mov.u32 %r293, %r169; + mov.u32 %r294, %r170; + mov.u32 %r296, %r26; + mov.u32 %r297, %r27; + bra.uni $L__BB3_31; + +$L__BB3_25: + setp.ne.s32 %p49, %r85, 0; + setp.eq.s32 %p50, %r20, 0; + and.pred %p51, %p50, %p49; + @%p51 bra $L__BB3_29; + bra.uni $L__BB3_26; + +$L__BB3_29: + shr.u32 %r212, %r84, 1; + add.s32 %r295, %r32, %r212; + mov.u64 %rd94, %rd4; + mov.u32 %r291, %r148; + mov.u32 %r292, %r149; + mov.u32 %r293, %r167; + mov.u32 %r294, %r168; + mov.u32 %r296, %r26; + mov.u32 %r297, %r27; + bra.uni $L__BB3_31; + +$L__BB3_26: + setp.eq.s32 %p52, %r85, 0; + and.pred %p53, %p52, %p1; + @%p53 bra $L__BB3_28; + bra.uni $L__BB3_27; + +$L__BB3_28: + add.s32 %r210, %r84, 1; + shr.u32 %r211, %r210, 1; + add.s32 %r295, %r37, %r211; + mov.u64 %rd94, %rd3; + mov.u32 %r291, %r146; + mov.u32 %r292, %r147; + mov.u32 %r293, %r165; + mov.u32 %r294, %r166; + bra.uni $L__BB3_31; + +$L__BB3_27: + shr.u32 %r209, %r84, 1; + add.s32 %r295, %r42, %r209; + +$L__BB3_31: + shr.u32 %r215, %r296, 1; + sub.s32 %r216, %r292, %r215; + shr.u32 %r217, %r297, 1; + add.s32 %r98, %r216, %r217; + setp.lt.u32 %p54, %r295, %r291; + setp.le.u32 %p55, %r293, %r295; + or.pred %p56, %p54, %p55; + setp.lt.u32 %p57, %r98, %r292; + or.pred %p58, %p56, %p57; + setp.le.u32 %p59, %r294, %r98; + or.pred %p60, %p59, %p58; + mov.f32 %f142, 0f00000000; + @%p60 bra $L__BB3_33; + + sub.s32 %r218, %r293, %r291; + sub.s32 %r219, %r98, %r292; + sub.s32 %r220, %r295, %r291; + mad.lo.s32 %r221, %r219, %r218, %r220; + mul.wide.u32 %rd24, %r221, 4; + add.s64 %rd25, %rd94, %rd24; + ld.f32 %f142, [%rd25]; + +$L__BB3_33: + mul.wide.u32 %rd26, %r290, 4; + add.s64 %rd27, %rd6, %rd26; + st.f32 [%rd27], %f142; + +$L__BB3_34: + setp.eq.s32 %p61, %r16, 1; + and.b32 %r102, %r155, 1; + @%p61 bra $L__BB3_79; + bra.uni $L__BB3_35; + +$L__BB3_79: + setp.eq.s32 %p117, %r102, 0; + @%p117 bra $L__BB3_81; + + ld.f32 %f138, [%rd6]; + mul.rn.f32 %f139, %f138, 0f3F000000; + st.f32 [%rd6], %f139; + bra.uni $L__BB3_81; + +$L__BB3_35: + setp.eq.s32 %p62, %r18, 0; + xor.b32 %r103, %r102, 1; + add.s32 %r104, %r16, -1; + mul.wide.u32 %rd28, %r104, 4; + add.s64 %rd11, %rd6, %rd28; + add.s32 %r105, %r16, -2; + mul.wide.u32 %rd29, %r105, 4; + add.s64 %rd12, %rd6, %rd29; + @%p62 bra $L__BB3_67; + + setp.eq.s32 %p6, %r102, 0; + selp.f32 %f7, 0f3F9D7658, 0f3F5019C3, %p6; + setp.lt.u32 %p63, %r16, 2; + @%p63 bra $L__BB3_39; + + selp.f32 %f8, 0f3F5019C3, 0f3F9D7658, %p6; + mov.u32 %r298, 0; + +$L__BB3_38: + mul.wide.u32 %rd30, %r298, 4; + add.s64 %rd31, %rd6, %rd30; + ld.f32 %f12, [%rd31]; + mul.rn.f32 %f13, %f7, %f12; + st.f32 [%rd31], %f13; + ld.f32 %f14, [%rd31+4]; + mul.rn.f32 %f15, %f8, %f14; + st.f32 [%rd31+4], %f15; + add.s32 %r107, %r298, 2; + add.s32 %r223, %r298, 3; + setp.lt.u32 %p64, %r223, %r16; + mov.u32 %r298, %r107; + @%p64 bra $L__BB3_38; + +$L__BB3_39: + and.b32 %r224, %r16, 1; + setp.eq.b32 %p65, %r224, 1; + mov.pred %p66, 0; + xor.pred %p67, %p65, %p66; + not.pred %p68, %p67; + @%p68 bra $L__BB3_41; + + ld.f32 %f16, [%rd11]; + mul.rn.f32 %f17, %f7, %f16; + st.f32 [%rd11], %f17; + +$L__BB3_41: + setp.ne.s32 %p69, %r102, 0; + @%p69 bra $L__BB3_43; + + setp.gt.u32 %p70, %r16, 1; + add.s32 %r225, %r16, %r16; + add.s32 %r226, %r225, -3; + selp.b32 %r227, 1, %r226, %p70; + mul.wide.u32 %rd32, %r227, 4; + add.s64 %rd33, %rd6, %rd32; + ld.f32 %f18, [%rd33]; + ld.f32 %f19, [%rd6+4]; + add.rn.f32 %f20, %f19, %f18; + ld.f32 %f21, [%rd6]; + mov.f32 %f22, 0fBEE31355; + fma.rn.f32 %f23, %f20, %f22, %f21; + st.f32 [%rd6], %f23; + +$L__BB3_43: + selp.b32 %r304, 2, 1, %p6; + add.s32 %r303, %r304, 1; + setp.ge.u32 %p72, %r303, %r16; + @%p72 bra $L__BB3_46; + + mov.u32 %r299, %r303; + mov.u32 %r300, %r304; + +$L__BB3_45: + add.s32 %r228, %r300, -1; + mul.wide.u32 %rd34, %r228, 4; + add.s64 %rd35, %rd6, %rd34; + mul.wide.u32 %rd36, %r299, 4; + add.s64 %rd37, %rd6, %rd36; + ld.f32 %f24, [%rd37]; + ld.f32 %f25, [%rd35]; + add.rn.f32 %f26, %f25, %f24; + mul.wide.u32 %rd38, %r300, 4; + add.s64 %rd39, %rd6, %rd38; + ld.f32 %f27, [%rd39]; + mov.f32 %f28, 0fBEE31355; + fma.rn.f32 %f29, %f26, %f28, %f27; + st.f32 [%rd39], %f29; + add.s32 %r112, %r300, 2; + add.s32 %r299, %r300, 3; + setp.lt.u32 %p73, %r299, %r16; + mov.u32 %r300, %r112; + @%p73 bra $L__BB3_45; + +$L__BB3_46: + setp.gt.u32 %p74, %r16, 1; + and.b32 %r115, %r104, 1; + setp.eq.s32 %p75, %r115, %r102; + and.pred %p7, %p74, %p75; + not.pred %p76, %p7; + @%p76 bra $L__BB3_48; + + setp.gt.u32 %p77, %r104, 1; + mov.u32 %r229, 2; + sub.s32 %r230, %r229, %r16; + selp.b32 %r232, %r105, %r230, %p77; + mul.wide.u32 %rd40, %r232, 4; + add.s64 %rd41, %rd6, %rd40; + ld.f32 %f30, [%rd12]; + ld.f32 %f31, [%rd41]; + add.rn.f32 %f32, %f31, %f30; + ld.f32 %f33, [%rd11]; + mov.f32 %f34, 0fBEE31355; + fma.rn.f32 %f35, %f32, %f34, %f33; + st.f32 [%rd11], %f35; + +$L__BB3_48: + setp.ne.s32 %p78, %r103, 0; + @%p78 bra $L__BB3_50; + + add.s32 %r233, %r16, %r16; + add.s32 %r234, %r233, -3; + selp.b32 %r235, 1, %r234, %p74; + mul.wide.u32 %rd42, %r235, 4; + add.s64 %rd43, %rd6, %rd42; + ld.f32 %f36, [%rd43]; + ld.f32 %f37, [%rd6+4]; + add.rn.f32 %f38, %f37, %f36; + ld.f32 %f39, [%rd6]; + mov.f32 %f40, 0fBF620676; + fma.rn.f32 %f41, %f38, %f40, %f39; + st.f32 [%rd6], %f41; + +$L__BB3_50: + setp.eq.s32 %p80, %r103, 0; + selp.b32 %r306, 2, 1, %p80; + add.s32 %r305, %r306, 1; + setp.ge.u32 %p81, %r305, %r16; + @%p81 bra $L__BB3_53; + + mov.u32 %r301, %r305; + mov.u32 %r302, %r306; + +$L__BB3_52: + add.s32 %r236, %r302, -1; + mul.wide.u32 %rd44, %r236, 4; + add.s64 %rd45, %rd6, %rd44; + mul.wide.u32 %rd46, %r301, 4; + add.s64 %rd47, %rd6, %rd46; + ld.f32 %f42, [%rd47]; + ld.f32 %f43, [%rd45]; + add.rn.f32 %f44, %f43, %f42; + mul.wide.u32 %rd48, %r302, 4; + add.s64 %rd49, %rd6, %rd48; + ld.f32 %f45, [%rd49]; + mov.f32 %f46, 0fBF620676; + fma.rn.f32 %f47, %f44, %f46, %f45; + st.f32 [%rd49], %f47; + add.s32 %r120, %r302, 2; + add.s32 %r301, %r302, 3; + setp.lt.u32 %p82, %r301, %r16; + mov.u32 %r302, %r120; + @%p82 bra $L__BB3_52; + +$L__BB3_53: + setp.eq.s32 %p84, %r115, %r103; + and.pred %p8, %p74, %p84; + not.pred %p85, %p8; + @%p85 bra $L__BB3_55; + + setp.gt.u32 %p86, %r104, 1; + mov.u32 %r237, 2; + sub.s32 %r238, %r237, %r16; + selp.b32 %r240, %r105, %r238, %p86; + mul.wide.u32 %rd50, %r240, 4; + add.s64 %rd51, %rd6, %rd50; + ld.f32 %f48, [%rd12]; + ld.f32 %f49, [%rd51]; + add.rn.f32 %f50, %f49, %f48; + ld.f32 %f51, [%rd11]; + mov.f32 %f52, 0fBF620676; + fma.rn.f32 %f53, %f50, %f52, %f51; + st.f32 [%rd11], %f53; + +$L__BB3_55: + @%p69 bra $L__BB3_57; + + add.s32 %r241, %r16, %r16; + add.s32 %r242, %r241, -3; + selp.b32 %r243, 1, %r242, %p74; + mul.wide.u32 %rd52, %r243, 4; + add.s64 %rd53, %rd6, %rd52; + ld.f32 %f54, [%rd53]; + ld.f32 %f55, [%rd6+4]; + add.rn.f32 %f56, %f55, %f54; + ld.f32 %f57, [%rd6]; + mov.f32 %f58, 0f3D5901AE; + fma.rn.f32 %f59, %f56, %f58, %f57; + st.f32 [%rd6], %f59; + +$L__BB3_57: + @%p72 bra $L__BB3_59; + +$L__BB3_58: + add.s32 %r244, %r304, -1; + mul.wide.u32 %rd54, %r244, 4; + add.s64 %rd55, %rd6, %rd54; + mul.wide.u32 %rd56, %r303, 4; + add.s64 %rd57, %rd6, %rd56; + ld.f32 %f60, [%rd57]; + ld.f32 %f61, [%rd55]; + add.rn.f32 %f62, %f61, %f60; + mul.wide.u32 %rd58, %r304, 4; + add.s64 %rd59, %rd6, %rd58; + ld.f32 %f63, [%rd59]; + mov.f32 %f64, 0f3D5901AE; + fma.rn.f32 %f65, %f62, %f64, %f63; + st.f32 [%rd59], %f65; + add.s32 %r124, %r304, 2; + add.s32 %r303, %r304, 3; + setp.lt.u32 %p90, %r303, %r16; + mov.u32 %r304, %r124; + @%p90 bra $L__BB3_58; + +$L__BB3_59: + @%p76 bra $L__BB3_61; + + setp.gt.u32 %p92, %r104, 1; + mov.u32 %r245, 2; + sub.s32 %r246, %r245, %r16; + selp.b32 %r248, %r105, %r246, %p92; + mul.wide.u32 %rd60, %r248, 4; + add.s64 %rd61, %rd6, %rd60; + ld.f32 %f66, [%rd12]; + ld.f32 %f67, [%rd61]; + add.rn.f32 %f68, %f67, %f66; + ld.f32 %f69, [%rd11]; + mov.f32 %f70, 0f3D5901AE; + fma.rn.f32 %f71, %f68, %f70, %f69; + st.f32 [%rd11], %f71; + +$L__BB3_61: + @%p78 bra $L__BB3_63; + + add.s32 %r249, %r16, %r16; + add.s32 %r250, %r249, -3; + selp.b32 %r251, 1, %r250, %p74; + mul.wide.u32 %rd62, %r251, 4; + add.s64 %rd63, %rd6, %rd62; + ld.f32 %f72, [%rd63]; + ld.f32 %f73, [%rd6+4]; + add.rn.f32 %f74, %f73, %f72; + ld.f32 %f75, [%rd6]; + mov.f32 %f76, 0f3FCB0673; + fma.rn.f32 %f77, %f74, %f76, %f75; + st.f32 [%rd6], %f77; + +$L__BB3_63: + @%p81 bra $L__BB3_65; + +$L__BB3_64: + add.s32 %r252, %r306, -1; + mul.wide.u32 %rd64, %r252, 4; + add.s64 %rd65, %rd6, %rd64; + mul.wide.u32 %rd66, %r305, 4; + add.s64 %rd67, %rd6, %rd66; + ld.f32 %f78, [%rd67]; + ld.f32 %f79, [%rd65]; + add.rn.f32 %f80, %f79, %f78; + mul.wide.u32 %rd68, %r306, 4; + add.s64 %rd69, %rd6, %rd68; + ld.f32 %f81, [%rd69]; + mov.f32 %f82, 0f3FCB0673; + fma.rn.f32 %f83, %f80, %f82, %f81; + st.f32 [%rd69], %f83; + add.s32 %r128, %r306, 2; + add.s32 %r305, %r306, 3; + setp.lt.u32 %p96, %r305, %r16; + mov.u32 %r306, %r128; + @%p96 bra $L__BB3_64; + +$L__BB3_65: + @%p85 bra $L__BB3_81; + + setp.gt.u32 %p98, %r104, 1; + mov.u32 %r253, 2; + sub.s32 %r254, %r253, %r16; + selp.b32 %r256, %r105, %r254, %p98; + mul.wide.u32 %rd70, %r256, 4; + add.s64 %rd71, %rd6, %rd70; + ld.f32 %f84, [%rd12]; + ld.f32 %f85, [%rd71]; + add.rn.f32 %f86, %f85, %f84; + ld.f32 %f87, [%rd11]; + mov.f32 %f88, 0f3FCB0673; + fma.rn.f32 %f89, %f86, %f88, %f87; + st.f32 [%rd11], %f89; + bra.uni $L__BB3_81; + +$L__BB3_67: + setp.ne.s32 %p99, %r102, 0; + @%p99 bra $L__BB3_69; + + setp.gt.u32 %p100, %r16, 1; + add.s32 %r257, %r16, %r16; + add.s32 %r258, %r257, -3; + selp.b32 %r259, 1, %r258, %p100; + mul.wide.u32 %rd72, %r259, 4; + add.s64 %rd73, %rd6, %rd72; + ld.f32 %f90, [%rd73]; + ld.f32 %f91, [%rd6+4]; + add.rn.f32 %f92, %f91, %f90; + mov.f32 %f93, 0f3F000000; + mov.f32 %f94, 0f3E800000; + fma.rn.f32 %f95, %f92, %f94, %f93; + cvt.rmi.f32.f32 %f96, %f95; + ld.f32 %f97, [%rd6]; + sub.rn.f32 %f98, %f97, %f96; + st.f32 [%rd6], %f98; + +$L__BB3_69: + setp.eq.s32 %p101, %r102, 0; + selp.b32 %r308, 2, 1, %p101; + add.s32 %r307, %r308, 1; + setp.ge.u32 %p102, %r307, %r16; + @%p102 bra $L__BB3_71; + +$L__BB3_70: + mul.wide.u32 %rd74, %r308, 4; + add.s64 %rd75, %rd6, %rd74; + add.s32 %r260, %r308, -1; + mul.wide.u32 %rd76, %r260, 4; + add.s64 %rd77, %rd6, %rd76; + mul.wide.u32 %rd78, %r307, 4; + add.s64 %rd79, %rd6, %rd78; + ld.f32 %f99, [%rd79]; + ld.f32 %f100, [%rd77]; + add.rn.f32 %f101, %f100, %f99; + mov.f32 %f102, 0f3F000000; + mov.f32 %f103, 0f3E800000; + fma.rn.f32 %f104, %f101, %f103, %f102; + cvt.rmi.f32.f32 %f105, %f104; + ld.f32 %f106, [%rd75]; + sub.rn.f32 %f107, %f106, %f105; + st.f32 [%rd75], %f107; + add.s32 %r134, %r308, 2; + add.s32 %r307, %r308, 3; + setp.lt.u32 %p103, %r307, %r16; + mov.u32 %r308, %r134; + @%p103 bra $L__BB3_70; + +$L__BB3_71: + setp.lt.u32 %p104, %r16, 2; + and.b32 %r136, %r16, 1; + xor.b32 %r261, %r136, 1; + setp.ne.s32 %p105, %r261, %r102; + or.pred %p106, %p104, %p105; + @%p106 bra $L__BB3_73; + + setp.gt.u32 %p107, %r104, 1; + mov.u32 %r263, 2; + sub.s32 %r264, %r263, %r16; + selp.b32 %r266, %r105, %r264, %p107; + mul.wide.u32 %rd80, %r266, 4; + add.s64 %rd81, %rd6, %rd80; + ld.f32 %f108, [%rd12]; + ld.f32 %f109, [%rd81]; + add.rn.f32 %f110, %f109, %f108; + mov.f32 %f111, 0f3F000000; + mov.f32 %f112, 0f3E800000; + fma.rn.f32 %f113, %f110, %f112, %f111; + cvt.rmi.f32.f32 %f114, %f113; + ld.f32 %f115, [%rd11]; + sub.rn.f32 %f116, %f115, %f114; + st.f32 [%rd11], %f116; + +$L__BB3_73: + setp.ne.s32 %p108, %r103, 0; + @%p108 bra $L__BB3_75; + + setp.gt.u32 %p109, %r16, 1; + add.s32 %r267, %r16, %r16; + add.s32 %r268, %r267, -3; + selp.b32 %r269, 1, %r268, %p109; + mul.wide.u32 %rd82, %r269, 4; + add.s64 %rd83, %rd6, %rd82; + ld.f32 %f117, [%rd83]; + ld.f32 %f118, [%rd6+4]; + add.rn.f32 %f119, %f118, %f117; + mul.rn.f32 %f120, %f119, 0f3F000000; + cvt.rmi.f32.f32 %f121, %f120; + ld.f32 %f122, [%rd6]; + add.rn.f32 %f123, %f122, %f121; + st.f32 [%rd6], %f123; + +$L__BB3_75: + setp.eq.s32 %p110, %r103, 0; + selp.b32 %r310, 2, 1, %p110; + add.s32 %r309, %r310, 1; + setp.ge.u32 %p111, %r309, %r16; + @%p111 bra $L__BB3_77; + +$L__BB3_76: + mul.wide.u32 %rd84, %r310, 4; + add.s64 %rd85, %rd6, %rd84; + add.s32 %r270, %r310, -1; + mul.wide.u32 %rd86, %r270, 4; + add.s64 %rd87, %rd6, %rd86; + mul.wide.u32 %rd88, %r309, 4; + add.s64 %rd89, %rd6, %rd88; + ld.f32 %f124, [%rd89]; + ld.f32 %f125, [%rd87]; + add.rn.f32 %f126, %f125, %f124; + mul.rn.f32 %f127, %f126, 0f3F000000; + cvt.rmi.f32.f32 %f128, %f127; + ld.f32 %f129, [%rd85]; + add.rn.f32 %f130, %f129, %f128; + st.f32 [%rd85], %f130; + add.s32 %r141, %r310, 2; + add.s32 %r309, %r310, 3; + setp.lt.u32 %p112, %r309, %r16; + mov.u32 %r310, %r141; + @%p112 bra $L__BB3_76; + +$L__BB3_77: + setp.ne.s32 %p114, %r136, %r102; + or.pred %p115, %p104, %p114; + @%p115 bra $L__BB3_81; + + setp.gt.u32 %p116, %r104, 1; + mov.u32 %r271, 2; + sub.s32 %r272, %r271, %r16; + selp.b32 %r273, %r105, %r272, %p116; + mul.wide.u32 %rd90, %r273, 4; + add.s64 %rd91, %rd6, %rd90; + ld.f32 %f131, [%rd12]; + ld.f32 %f132, [%rd91]; + add.rn.f32 %f133, %f132, %f131; + mul.rn.f32 %f134, %f133, 0f3F000000; + cvt.rmi.f32.f32 %f135, %f134; + ld.f32 %f136, [%rd11]; + add.rn.f32 %f137, %f136, %f135; + st.f32 [%rd11], %f137; + +$L__BB3_81: + ret; + +} + // .globl j2k_idwt_interleave_horizontal_53_multi +.visible .entry j2k_idwt_interleave_horizontal_53_multi( + .param .u64 j2k_idwt_interleave_horizontal_53_multi_param_0 +) +{ + .reg .pred %p<39>; + .reg .f32 %f<25>; + .reg .b32 %r<150>; + .reg .b64 %rd<19>; + // demoted variable + .shared .align 4 .b8 _ZZ48j2k_idwt_interleave_horizontal_53_multiE11row_samples[2048]; + + ld.param.u64 %rd8, [j2k_idwt_interleave_horizontal_53_multi_param_0]; + cvta.to.global.u64 %rd9, %rd8; + mov.u32 %r1, %tid.x; + mov.u32 %r61, %ctaid.y; + mul.wide.u32 %rd10, %r61, 128; + add.s64 %rd11, %rd9, %rd10; + add.s64 %rd1, %rd11, 112; + ld.global.v2.u32 {%r143, %r144}, [%rd11+104]; + ld.global.v2.u32 {%r64, %r65}, [%rd11+88]; + ld.global.v2.u32 {%r66, %r67}, [%rd11+72]; + ld.global.v2.u32 {%r68, %r69}, [%rd11+56]; + ld.global.v2.u32 {%r70, %r71}, [%rd11+48]; + ld.global.v2.u32 {%r74, %r148}, [%rd11+40]; + ld.global.u64 %rd2, [%rd11+32]; + sub.s32 %r16, %r70, %r74; + sub.s32 %r76, %r71, %r148; + mov.u32 %r17, %ctaid.x; + setp.ge.u32 %p1, %r17, %r76; + @%p1 bra $L__BB4_23; + + setp.ge.u32 %p2, %r1, %r16; + shl.b32 %r77, %r1, 2; + mov.u32 %r78, _ZZ48j2k_idwt_interleave_horizontal_53_multiE11row_samples; + add.s32 %r18, %r78, %r77; + @%p2 bra $L__BB4_12; + + add.s32 %r19, %r74, %r1; + and.b32 %r20, %r19, 1; + add.s32 %r149, %r148, %r17; + and.b32 %r22, %r149, 1; + setp.eq.b32 %p3, %r22, 1; + setp.eq.b32 %p4, %r20, 1; + or.pred %p5, %p4, %p3; + mov.pred %p6, 0; + xor.pred %p7, %p5, %p6; + not.pred %p8, %p7; + @%p8 bra $L__BB4_8; + bra.uni $L__BB4_3; + +$L__BB4_8: + ld.global.v2.u32 {%r145, %r146}, [%rd1+-48]; + ld.global.u64 %rd18, [%rd1+-112]; + add.s32 %r98, %r19, 1; + shr.u32 %r99, %r98, 1; + add.s32 %r100, %r74, 1; + shr.u32 %r101, %r100, 1; + sub.s32 %r102, %r68, %r101; + add.s32 %r147, %r102, %r99; + add.s32 %r148, %r148, 1; + add.s32 %r149, %r149, 1; + mov.u32 %r143, %r68; + mov.u32 %r144, %r69; + bra.uni $L__BB4_9; + +$L__BB4_3: + setp.eq.s32 %p9, %r22, 0; + setp.ne.s32 %p10, %r20, 0; + and.pred %p11, %p9, %p10; + @%p11 bra $L__BB4_7; + bra.uni $L__BB4_4; + +$L__BB4_7: + ld.global.v2.u32 {%r145, %r146}, [%rd1+-32]; + ld.global.u64 %rd18, [%rd1+-104]; + shr.u32 %r93, %r74, 1; + sub.s32 %r94, %r66, %r93; + shr.u32 %r95, %r19, 1; + add.s32 %r147, %r94, %r95; + add.s32 %r148, %r148, 1; + add.s32 %r149, %r149, 1; + mov.u32 %r143, %r66; + mov.u32 %r144, %r67; + bra.uni $L__BB4_9; + +$L__BB4_4: + setp.ne.s32 %p12, %r22, 0; + setp.eq.s32 %p13, %r20, 0; + and.pred %p14, %p13, %p12; + @%p14 bra $L__BB4_6; + bra.uni $L__BB4_5; + +$L__BB4_6: + ld.global.v2.u32 {%r145, %r146}, [%rd1+-16]; + ld.global.u64 %rd18, [%rd1+-96]; + add.s32 %r86, %r19, 1; + shr.u32 %r87, %r86, 1; + add.s32 %r88, %r74, 1; + shr.u32 %r89, %r88, 1; + sub.s32 %r90, %r64, %r89; + add.s32 %r147, %r90, %r87; + mov.u32 %r143, %r64; + mov.u32 %r144, %r65; + bra.uni $L__BB4_9; + +$L__BB4_5: + ld.global.v2.u32 {%r145, %r146}, [%rd1]; + ld.global.u64 %rd18, [%rd1+-88]; + shr.u32 %r81, %r74, 1; + sub.s32 %r82, %r143, %r81; + shr.u32 %r83, %r19, 1; + add.s32 %r147, %r82, %r83; + +$L__BB4_9: + shr.u32 %r103, %r148, 1; + sub.s32 %r104, %r144, %r103; + shr.u32 %r105, %r149, 1; + add.s32 %r55, %r104, %r105; + setp.lt.u32 %p15, %r147, %r143; + setp.le.u32 %p16, %r145, %r147; + or.pred %p17, %p15, %p16; + setp.lt.u32 %p18, %r55, %r144; + or.pred %p19, %p17, %p18; + setp.le.u32 %p20, %r146, %r55; + or.pred %p21, %p20, %p19; + mov.f32 %f24, 0f00000000; + @%p21 bra $L__BB4_11; + + sub.s32 %r106, %r145, %r143; + sub.s32 %r107, %r55, %r144; + sub.s32 %r108, %r147, %r143; + mad.lo.s32 %r109, %r107, %r106, %r108; + mul.wide.u32 %rd12, %r109, 4; + add.s64 %rd13, %rd18, %rd12; + ld.f32 %f24, [%rd13]; + +$L__BB4_11: + st.shared.f32 [%r18], %f24; + +$L__BB4_12: + bar.sync 0; + setp.eq.s32 %p22, %r16, 1; + @%p22 bra $L__BB4_19; + bra.uni $L__BB4_13; + +$L__BB4_19: + setp.ne.s32 %p34, %r1, 0; + and.b32 %r142, %r74, 1; + setp.eq.b32 %p35, %r142, 1; + not.pred %p36, %p35; + or.pred %p37, %p34, %p36; + @%p37 bra $L__BB4_21; + + ld.shared.f32 %f21, [_ZZ48j2k_idwt_interleave_horizontal_53_multiE11row_samples]; + mul.rn.f32 %f22, %f21, 0f3F000000; + st.shared.f32 [_ZZ48j2k_idwt_interleave_horizontal_53_multiE11row_samples], %f22; + +$L__BB4_21: + bar.sync 0; + @%p34 bra $L__BB4_23; + + ld.shared.f32 %f23, [_ZZ48j2k_idwt_interleave_horizontal_53_multiE11row_samples]; + mul.wide.u32 %rd16, %r17, 4; + add.s64 %rd17, %rd2, %rd16; + st.f32 [%rd17], %f23; + bra.uni $L__BB4_23; + +$L__BB4_13: + and.b32 %r59, %r74, 1; + and.b32 %r60, %r1, 1; + setp.ne.s32 %p24, %r60, %r59; + or.pred %p25, %p2, %p24; + @%p25 bra $L__BB4_15; + + setp.gt.u32 %p26, %r1, 1; + mov.u32 %r110, 1; + sub.s32 %r111, %r110, %r1; + add.s32 %r112, %r1, -1; + selp.b32 %r113, %r112, %r111, %p26; + add.s32 %r114, %r1, 1; + setp.lt.u32 %p27, %r114, %r16; + mov.u32 %r115, -3; + sub.s32 %r116, %r115, %r1; + add.s32 %r117, %r116, %r16; + add.s32 %r118, %r117, %r16; + selp.b32 %r119, %r114, %r118, %p27; + shl.b32 %r120, %r113, 2; + add.s32 %r122, %r78, %r120; + shl.b32 %r123, %r119, 2; + add.s32 %r124, %r78, %r123; + ld.shared.f32 %f4, [%r124]; + ld.shared.f32 %f5, [%r122]; + add.rn.f32 %f6, %f5, %f4; + mov.f32 %f7, 0f3F000000; + mov.f32 %f8, 0f3E800000; + fma.rn.f32 %f9, %f6, %f8, %f7; + cvt.rmi.f32.f32 %f10, %f9; + ld.shared.f32 %f11, [%r18]; + sub.rn.f32 %f12, %f11, %f10; + st.shared.f32 [%r18], %f12; + +$L__BB4_15: + bar.sync 0; + xor.b32 %r125, %r59, 1; + setp.ne.s32 %p29, %r60, %r125; + or.pred %p30, %p2, %p29; + @%p30 bra $L__BB4_17; + + setp.gt.u32 %p31, %r1, 1; + mov.u32 %r126, 1; + sub.s32 %r127, %r126, %r1; + add.s32 %r128, %r1, -1; + selp.b32 %r129, %r128, %r127, %p31; + add.s32 %r130, %r1, 1; + setp.lt.u32 %p32, %r130, %r16; + mov.u32 %r131, -3; + sub.s32 %r132, %r131, %r1; + add.s32 %r133, %r132, %r16; + add.s32 %r134, %r133, %r16; + selp.b32 %r135, %r130, %r134, %p32; + shl.b32 %r136, %r129, 2; + add.s32 %r138, %r78, %r136; + shl.b32 %r139, %r135, 2; + add.s32 %r140, %r78, %r139; + ld.shared.f32 %f13, [%r140]; + ld.shared.f32 %f14, [%r138]; + add.rn.f32 %f15, %f14, %f13; + mul.rn.f32 %f16, %f15, 0f3F000000; + cvt.rmi.f32.f32 %f17, %f16; + ld.shared.f32 %f18, [%r18]; + add.rn.f32 %f19, %f18, %f17; + st.shared.f32 [%r18], %f19; + +$L__BB4_17: + bar.sync 0; + @%p2 bra $L__BB4_23; + + ld.shared.f32 %f20, [%r18]; + mad.lo.s32 %r141, %r16, %r17, %r1; + mul.wide.u32 %rd14, %r141, 4; + add.s64 %rd15, %rd2, %rd14; + st.f32 [%rd15], %f20; + +$L__BB4_23: + ret; + +} + // .globl j2k_idwt_interleave_horizontal_97_multi +.visible .entry j2k_idwt_interleave_horizontal_97_multi( + .param .u64 j2k_idwt_interleave_horizontal_97_multi_param_0 +) +{ + .reg .pred %p<50>; + .reg .f32 %f<38>; + .reg .b32 %r<181>; + .reg .b64 %rd<19>; + // demoted variable + .shared .align 4 .b8 _ZZ48j2k_idwt_interleave_horizontal_97_multiE11row_samples[2048]; + + ld.param.u64 %rd8, [j2k_idwt_interleave_horizontal_97_multi_param_0]; + cvta.to.global.u64 %rd9, %rd8; + mov.u32 %r1, %tid.x; + mov.u32 %r61, %ctaid.y; + mul.wide.u32 %rd10, %r61, 128; + add.s64 %rd11, %rd9, %rd10; + add.s64 %rd1, %rd11, 112; + ld.global.v2.u32 {%r174, %r175}, [%rd11+104]; + ld.global.v2.u32 {%r64, %r65}, [%rd11+88]; + ld.global.v2.u32 {%r66, %r67}, [%rd11+72]; + ld.global.v2.u32 {%r68, %r69}, [%rd11+56]; + ld.global.v2.u32 {%r70, %r71}, [%rd11+48]; + ld.global.v2.u32 {%r74, %r179}, [%rd11+40]; + ld.global.u64 %rd2, [%rd11+32]; + sub.s32 %r16, %r70, %r74; + sub.s32 %r76, %r71, %r179; + mov.u32 %r17, %ctaid.x; + setp.ge.u32 %p3, %r17, %r76; + @%p3 bra $L__BB5_29; + + setp.ge.u32 %p4, %r1, %r16; + shl.b32 %r77, %r1, 2; + mov.u32 %r78, _ZZ48j2k_idwt_interleave_horizontal_97_multiE11row_samples; + add.s32 %r18, %r78, %r77; + @%p4 bra $L__BB5_12; + + add.s32 %r19, %r74, %r1; + and.b32 %r20, %r19, 1; + add.s32 %r180, %r179, %r17; + and.b32 %r22, %r180, 1; + setp.eq.b32 %p5, %r22, 1; + setp.eq.b32 %p6, %r20, 1; + or.pred %p7, %p6, %p5; + mov.pred %p8, 0; + xor.pred %p9, %p7, %p8; + not.pred %p10, %p9; + @%p10 bra $L__BB5_8; + bra.uni $L__BB5_3; + +$L__BB5_8: + ld.global.v2.u32 {%r176, %r177}, [%rd1+-48]; + ld.global.u64 %rd18, [%rd1+-112]; + add.s32 %r98, %r19, 1; + shr.u32 %r99, %r98, 1; + add.s32 %r100, %r74, 1; + shr.u32 %r101, %r100, 1; + sub.s32 %r102, %r68, %r101; + add.s32 %r178, %r102, %r99; + add.s32 %r179, %r179, 1; + add.s32 %r180, %r180, 1; + mov.u32 %r174, %r68; + mov.u32 %r175, %r69; + bra.uni $L__BB5_9; + +$L__BB5_3: + setp.eq.s32 %p11, %r22, 0; + setp.ne.s32 %p12, %r20, 0; + and.pred %p13, %p11, %p12; + @%p13 bra $L__BB5_7; + bra.uni $L__BB5_4; + +$L__BB5_7: + ld.global.v2.u32 {%r176, %r177}, [%rd1+-32]; + ld.global.u64 %rd18, [%rd1+-104]; + shr.u32 %r93, %r74, 1; + sub.s32 %r94, %r66, %r93; + shr.u32 %r95, %r19, 1; + add.s32 %r178, %r94, %r95; + add.s32 %r179, %r179, 1; + add.s32 %r180, %r180, 1; + mov.u32 %r174, %r66; + mov.u32 %r175, %r67; + bra.uni $L__BB5_9; + +$L__BB5_4: + setp.ne.s32 %p14, %r22, 0; + setp.eq.s32 %p15, %r20, 0; + and.pred %p16, %p15, %p14; + @%p16 bra $L__BB5_6; + bra.uni $L__BB5_5; + +$L__BB5_6: + ld.global.v2.u32 {%r176, %r177}, [%rd1+-16]; + ld.global.u64 %rd18, [%rd1+-96]; + add.s32 %r86, %r19, 1; + shr.u32 %r87, %r86, 1; + add.s32 %r88, %r74, 1; + shr.u32 %r89, %r88, 1; + sub.s32 %r90, %r64, %r89; + add.s32 %r178, %r90, %r87; + mov.u32 %r174, %r64; + mov.u32 %r175, %r65; + bra.uni $L__BB5_9; + +$L__BB5_5: + ld.global.v2.u32 {%r176, %r177}, [%rd1]; + ld.global.u64 %rd18, [%rd1+-88]; + shr.u32 %r81, %r74, 1; + sub.s32 %r82, %r174, %r81; + shr.u32 %r83, %r19, 1; + add.s32 %r178, %r82, %r83; + +$L__BB5_9: + shr.u32 %r103, %r179, 1; + sub.s32 %r104, %r175, %r103; + shr.u32 %r105, %r180, 1; + add.s32 %r55, %r104, %r105; + setp.lt.u32 %p17, %r178, %r174; + setp.le.u32 %p18, %r176, %r178; + or.pred %p19, %p17, %p18; + setp.lt.u32 %p20, %r55, %r175; + or.pred %p21, %p19, %p20; + setp.le.u32 %p22, %r177, %r55; + or.pred %p23, %p22, %p21; + mov.f32 %f37, 0f00000000; + @%p23 bra $L__BB5_11; + + sub.s32 %r106, %r176, %r174; + sub.s32 %r107, %r55, %r175; + sub.s32 %r108, %r178, %r174; + mad.lo.s32 %r109, %r107, %r106, %r108; + mul.wide.u32 %rd12, %r109, 4; + add.s64 %rd13, %rd18, %rd12; + ld.f32 %f37, [%rd13]; + +$L__BB5_11: + st.shared.f32 [%r18], %f37; + +$L__BB5_12: + bar.sync 0; + setp.eq.s32 %p24, %r16, 1; + @%p24 bra $L__BB5_25; + bra.uni $L__BB5_13; + +$L__BB5_25: + setp.ne.s32 %p45, %r1, 0; + and.b32 %r173, %r74, 1; + setp.eq.b32 %p46, %r173, 1; + not.pred %p47, %p46; + or.pred %p48, %p45, %p47; + @%p48 bra $L__BB5_27; + + ld.shared.f32 %f34, [_ZZ48j2k_idwt_interleave_horizontal_97_multiE11row_samples]; + mul.rn.f32 %f35, %f34, 0f3F000000; + st.shared.f32 [_ZZ48j2k_idwt_interleave_horizontal_97_multiE11row_samples], %f35; + +$L__BB5_27: + bar.sync 0; + @%p45 bra $L__BB5_29; + + ld.shared.f32 %f36, [_ZZ48j2k_idwt_interleave_horizontal_97_multiE11row_samples]; + mul.wide.u32 %rd16, %r17, 4; + add.s64 %rd17, %rd2, %rd16; + st.f32 [%rd17], %f36; + bra.uni $L__BB5_29; + +$L__BB5_13: + setp.lt.u32 %p25, %r1, %r16; + and.b32 %r59, %r74, 1; + @%p25 bra $L__BB5_14; + bra.uni $L__BB5_15; + +$L__BB5_14: + setp.eq.s32 %p26, %r59, 0; + selp.f32 %f4, 0f3F5019C3, 0f3F9D7658, %p26; + selp.f32 %f5, 0f3F9D7658, 0f3F5019C3, %p26; + and.b32 %r110, %r1, 1; + setp.eq.b32 %p27, %r110, 1; + selp.f32 %f6, %f4, %f5, %p27; + ld.shared.f32 %f7, [%r18]; + mul.rn.f32 %f8, %f6, %f7; + st.shared.f32 [%r18], %f8; + +$L__BB5_15: + bar.sync 0; + and.b32 %r60, %r1, 1; + setp.eq.s32 %p29, %r60, %r59; + and.pred %p1, %p25, %p29; + not.pred %p30, %p1; + @%p30 bra $L__BB5_17; + + setp.gt.u32 %p31, %r1, 1; + mov.u32 %r111, 1; + sub.s32 %r112, %r111, %r1; + add.s32 %r113, %r1, -1; + selp.b32 %r114, %r113, %r112, %p31; + add.s32 %r115, %r1, 1; + setp.lt.u32 %p32, %r115, %r16; + mov.u32 %r116, -3; + sub.s32 %r117, %r116, %r1; + add.s32 %r118, %r117, %r16; + add.s32 %r119, %r118, %r16; + selp.b32 %r120, %r115, %r119, %p32; + shl.b32 %r121, %r114, 2; + add.s32 %r123, %r78, %r121; + shl.b32 %r124, %r120, 2; + add.s32 %r125, %r78, %r124; + ld.shared.f32 %f9, [%r125]; + ld.shared.f32 %f10, [%r123]; + add.rn.f32 %f11, %f10, %f9; + ld.shared.f32 %f12, [%r18]; + mov.f32 %f13, 0fBEE31355; + fma.rn.f32 %f14, %f11, %f13, %f12; + st.shared.f32 [%r18], %f14; + +$L__BB5_17: + bar.sync 0; + xor.b32 %r126, %r59, 1; + setp.eq.s32 %p34, %r60, %r126; + and.pred %p2, %p25, %p34; + not.pred %p35, %p2; + @%p35 bra $L__BB5_19; + + setp.gt.u32 %p36, %r1, 1; + mov.u32 %r127, 1; + sub.s32 %r128, %r127, %r1; + add.s32 %r129, %r1, -1; + selp.b32 %r130, %r129, %r128, %p36; + add.s32 %r131, %r1, 1; + setp.lt.u32 %p37, %r131, %r16; + mov.u32 %r132, -3; + sub.s32 %r133, %r132, %r1; + add.s32 %r134, %r133, %r16; + add.s32 %r135, %r134, %r16; + selp.b32 %r136, %r131, %r135, %p37; + shl.b32 %r137, %r130, 2; + add.s32 %r139, %r78, %r137; + shl.b32 %r140, %r136, 2; + add.s32 %r141, %r78, %r140; + ld.shared.f32 %f15, [%r141]; + ld.shared.f32 %f16, [%r139]; + add.rn.f32 %f17, %f16, %f15; + ld.shared.f32 %f18, [%r18]; + mov.f32 %f19, 0fBF620676; + fma.rn.f32 %f20, %f17, %f19, %f18; + st.shared.f32 [%r18], %f20; + +$L__BB5_19: + bar.sync 0; + @%p30 bra $L__BB5_21; + + setp.gt.u32 %p39, %r1, 1; + mov.u32 %r142, 1; + sub.s32 %r143, %r142, %r1; + add.s32 %r144, %r1, -1; + selp.b32 %r145, %r144, %r143, %p39; + add.s32 %r146, %r1, 1; + setp.lt.u32 %p40, %r146, %r16; + mov.u32 %r147, -3; + sub.s32 %r148, %r147, %r1; + add.s32 %r149, %r148, %r16; + add.s32 %r150, %r149, %r16; + selp.b32 %r151, %r146, %r150, %p40; + shl.b32 %r152, %r145, 2; + add.s32 %r154, %r78, %r152; + shl.b32 %r155, %r151, 2; + add.s32 %r156, %r78, %r155; + ld.shared.f32 %f21, [%r156]; + ld.shared.f32 %f22, [%r154]; + add.rn.f32 %f23, %f22, %f21; + ld.shared.f32 %f24, [%r18]; + mov.f32 %f25, 0f3D5901AE; + fma.rn.f32 %f26, %f23, %f25, %f24; + st.shared.f32 [%r18], %f26; + +$L__BB5_21: + bar.sync 0; + @%p35 bra $L__BB5_23; + + setp.gt.u32 %p42, %r1, 1; + mov.u32 %r157, 1; + sub.s32 %r158, %r157, %r1; + add.s32 %r159, %r1, -1; + selp.b32 %r160, %r159, %r158, %p42; + add.s32 %r161, %r1, 1; + setp.lt.u32 %p43, %r161, %r16; + mov.u32 %r162, -3; + sub.s32 %r163, %r162, %r1; + add.s32 %r164, %r163, %r16; + add.s32 %r165, %r164, %r16; + selp.b32 %r166, %r161, %r165, %p43; + shl.b32 %r167, %r160, 2; + add.s32 %r169, %r78, %r167; + shl.b32 %r170, %r166, 2; + add.s32 %r171, %r78, %r170; + ld.shared.f32 %f27, [%r171]; + ld.shared.f32 %f28, [%r169]; + add.rn.f32 %f29, %f28, %f27; + ld.shared.f32 %f30, [%r18]; + mov.f32 %f31, 0f3FCB0673; + fma.rn.f32 %f32, %f29, %f31, %f30; + st.shared.f32 [%r18], %f32; + +$L__BB5_23: + bar.sync 0; + @%p4 bra $L__BB5_29; + + ld.shared.f32 %f33, [%r18]; + mad.lo.s32 %r172, %r16, %r17, %r1; + mul.wide.u32 %rd14, %r172, 4; + add.s64 %rd15, %rd2, %rd14; + st.f32 [%rd15], %f33; + +$L__BB5_29: + ret; + +} + // .globl j2k_idwt_horizontal +.visible .entry j2k_idwt_horizontal( + .param .u64 j2k_idwt_horizontal_param_0, + .param .u64 j2k_idwt_horizontal_param_1 +) +{ + .reg .pred %p<62>; + .reg .f32 %f<131>; + .reg .b32 %r<103>; + .reg .b64 %rd<73>; + + + ld.param.u64 %rd5, [j2k_idwt_horizontal_param_0]; + ld.param.u64 %rd6, [j2k_idwt_horizontal_param_1]; + cvta.to.global.u64 %rd1, %rd6; + ld.global.u32 %r29, [%rd1+8]; + ld.global.u32 %r1, [%rd1]; + sub.s32 %r2, %r29, %r1; + ld.global.u32 %r30, [%rd1+12]; + ld.global.u32 %r31, [%rd1+4]; + sub.s32 %r32, %r30, %r31; + mov.u32 %r33, %ntid.x; + mov.u32 %r34, %ctaid.x; + mov.u32 %r35, %tid.x; + mad.lo.s32 %r3, %r34, %r33, %r35; + setp.ge.u32 %p4, %r3, %r32; + @%p4 bra $L__BB6_48; + + cvta.to.global.u64 %rd7, %rd5; + mul.lo.s32 %r36, %r2, %r3; + mul.wide.u32 %rd8, %r36, 4; + add.s64 %rd2, %rd7, %rd8; + and.b32 %r4, %r1, 1; + setp.eq.s32 %p5, %r2, 1; + @%p5 bra $L__BB6_46; + bra.uni $L__BB6_2; + +$L__BB6_46: + setp.eq.s32 %p61, %r4, 0; + @%p61 bra $L__BB6_48; + + ld.global.f32 %f129, [%rd2]; + mul.rn.f32 %f130, %f129, 0f3F000000; + st.global.f32 [%rd2], %f130; + bra.uni $L__BB6_48; + +$L__BB6_2: + ld.global.u32 %r37, [%rd1+80]; + setp.eq.s32 %p6, %r37, 0; + xor.b32 %r5, %r4, 1; + add.s32 %r6, %r2, -1; + mul.wide.u32 %rd9, %r6, 4; + add.s64 %rd3, %rd2, %rd9; + add.s32 %r7, %r2, -2; + mul.wide.u32 %rd10, %r7, 4; + add.s64 %rd4, %rd2, %rd10; + @%p6 bra $L__BB6_34; + + setp.eq.s32 %p1, %r4, 0; + selp.f32 %f1, 0f3F9D7658, 0f3F5019C3, %p1; + setp.lt.u32 %p7, %r2, 2; + @%p7 bra $L__BB6_6; + + selp.f32 %f2, 0f3F5019C3, 0f3F9D7658, %p1; + mov.u32 %r96, 1; + +$L__BB6_5: + add.s32 %r39, %r96, -1; + mul.wide.u32 %rd11, %r39, 4; + add.s64 %rd12, %rd2, %rd11; + ld.global.f32 %f3, [%rd12]; + mul.rn.f32 %f4, %f1, %f3; + st.global.f32 [%rd12], %f4; + ld.global.f32 %f5, [%rd12+4]; + mul.rn.f32 %f6, %f2, %f5; + st.global.f32 [%rd12+4], %f6; + add.s32 %r96, %r96, 2; + setp.lt.u32 %p8, %r96, %r2; + @%p8 bra $L__BB6_5; + +$L__BB6_6: + and.b32 %r40, %r2, 1; + setp.eq.b32 %p9, %r40, 1; + mov.pred %p10, 0; + xor.pred %p11, %p9, %p10; + not.pred %p12, %p11; + @%p12 bra $L__BB6_8; + + ld.global.f32 %f7, [%rd3]; + mul.rn.f32 %f8, %f1, %f7; + st.global.f32 [%rd3], %f8; + +$L__BB6_8: + setp.ne.s32 %p13, %r4, 0; + @%p13 bra $L__BB6_10; + + setp.gt.u32 %p14, %r2, 1; + add.s32 %r41, %r2, %r2; + add.s32 %r42, %r41, -3; + selp.b32 %r43, 1, %r42, %p14; + mul.wide.u32 %rd13, %r43, 4; + add.s64 %rd14, %rd2, %rd13; + ld.global.f32 %f9, [%rd14]; + ld.global.f32 %f10, [%rd2+4]; + add.rn.f32 %f11, %f10, %f9; + ld.global.f32 %f12, [%rd2]; + mov.f32 %f13, 0fBEE31355; + fma.rn.f32 %f14, %f11, %f13, %f12; + st.global.f32 [%rd2], %f14; + +$L__BB6_10: + selp.b32 %r99, 3, 2, %p1; + setp.ge.u32 %p16, %r99, %r2; + @%p16 bra $L__BB6_13; + + mov.u32 %r97, %r99; + +$L__BB6_12: + add.s32 %r44, %r97, -2; + mul.wide.u32 %rd15, %r44, 4; + add.s64 %rd16, %rd2, %rd15; + mul.wide.u32 %rd17, %r97, 4; + add.s64 %rd18, %rd2, %rd17; + ld.global.f32 %f15, [%rd18]; + ld.global.f32 %f16, [%rd16]; + add.rn.f32 %f17, %f16, %f15; + add.s32 %r45, %r97, -1; + mul.wide.u32 %rd19, %r45, 4; + add.s64 %rd20, %rd2, %rd19; + ld.global.f32 %f18, [%rd20]; + mov.f32 %f19, 0fBEE31355; + fma.rn.f32 %f20, %f17, %f19, %f18; + st.global.f32 [%rd20], %f20; + add.s32 %r97, %r97, 2; + setp.lt.u32 %p17, %r97, %r2; + @%p17 bra $L__BB6_12; + +$L__BB6_13: + setp.gt.u32 %p18, %r2, 1; + and.b32 %r14, %r6, 1; + setp.eq.s32 %p19, %r14, %r4; + and.pred %p2, %p18, %p19; + not.pred %p20, %p2; + @%p20 bra $L__BB6_15; + + setp.gt.u32 %p21, %r6, 1; + mov.u32 %r46, 2; + sub.s32 %r47, %r46, %r2; + selp.b32 %r49, %r7, %r47, %p21; + mul.wide.u32 %rd21, %r49, 4; + add.s64 %rd22, %rd2, %rd21; + ld.global.f32 %f21, [%rd4]; + ld.global.f32 %f22, [%rd22]; + add.rn.f32 %f23, %f22, %f21; + ld.global.f32 %f24, [%rd3]; + mov.f32 %f25, 0fBEE31355; + fma.rn.f32 %f26, %f23, %f25, %f24; + st.global.f32 [%rd3], %f26; + +$L__BB6_15: + setp.ne.s32 %p22, %r5, 0; + @%p22 bra $L__BB6_17; + + add.s32 %r50, %r2, %r2; + add.s32 %r51, %r50, -3; + selp.b32 %r52, 1, %r51, %p18; + mul.wide.u32 %rd23, %r52, 4; + add.s64 %rd24, %rd2, %rd23; + ld.global.f32 %f27, [%rd24]; + ld.global.f32 %f28, [%rd2+4]; + add.rn.f32 %f29, %f28, %f27; + ld.global.f32 %f30, [%rd2]; + mov.f32 %f31, 0fBF620676; + fma.rn.f32 %f32, %f29, %f31, %f30; + st.global.f32 [%rd2], %f32; + +$L__BB6_17: + setp.eq.s32 %p24, %r5, 0; + selp.b32 %r100, 3, 2, %p24; + setp.ge.u32 %p25, %r100, %r2; + @%p25 bra $L__BB6_20; + + mov.u32 %r98, %r100; + +$L__BB6_19: + add.s32 %r53, %r98, -2; + mul.wide.u32 %rd25, %r53, 4; + add.s64 %rd26, %rd2, %rd25; + mul.wide.u32 %rd27, %r98, 4; + add.s64 %rd28, %rd2, %rd27; + ld.global.f32 %f33, [%rd28]; + ld.global.f32 %f34, [%rd26]; + add.rn.f32 %f35, %f34, %f33; + add.s32 %r54, %r98, -1; + mul.wide.u32 %rd29, %r54, 4; + add.s64 %rd30, %rd2, %rd29; + ld.global.f32 %f36, [%rd30]; + mov.f32 %f37, 0fBF620676; + fma.rn.f32 %f38, %f35, %f37, %f36; + st.global.f32 [%rd30], %f38; + add.s32 %r98, %r98, 2; + setp.lt.u32 %p26, %r98, %r2; + @%p26 bra $L__BB6_19; + +$L__BB6_20: + setp.eq.s32 %p28, %r14, %r5; + and.pred %p3, %p18, %p28; + not.pred %p29, %p3; + @%p29 bra $L__BB6_22; + + setp.gt.u32 %p30, %r6, 1; + mov.u32 %r55, 2; + sub.s32 %r56, %r55, %r2; + selp.b32 %r58, %r7, %r56, %p30; + mul.wide.u32 %rd31, %r58, 4; + add.s64 %rd32, %rd2, %rd31; + ld.global.f32 %f39, [%rd4]; + ld.global.f32 %f40, [%rd32]; + add.rn.f32 %f41, %f40, %f39; + ld.global.f32 %f42, [%rd3]; + mov.f32 %f43, 0fBF620676; + fma.rn.f32 %f44, %f41, %f43, %f42; + st.global.f32 [%rd3], %f44; + +$L__BB6_22: + @%p13 bra $L__BB6_24; + + add.s32 %r59, %r2, %r2; + add.s32 %r60, %r59, -3; + selp.b32 %r61, 1, %r60, %p18; + mul.wide.u32 %rd33, %r61, 4; + add.s64 %rd34, %rd2, %rd33; + ld.global.f32 %f45, [%rd34]; + ld.global.f32 %f46, [%rd2+4]; + add.rn.f32 %f47, %f46, %f45; + ld.global.f32 %f48, [%rd2]; + mov.f32 %f49, 0f3D5901AE; + fma.rn.f32 %f50, %f47, %f49, %f48; + st.global.f32 [%rd2], %f50; + +$L__BB6_24: + @%p16 bra $L__BB6_26; + +$L__BB6_25: + add.s32 %r62, %r99, -2; + mul.wide.u32 %rd35, %r62, 4; + add.s64 %rd36, %rd2, %rd35; + mul.wide.u32 %rd37, %r99, 4; + add.s64 %rd38, %rd2, %rd37; + ld.global.f32 %f51, [%rd38]; + ld.global.f32 %f52, [%rd36]; + add.rn.f32 %f53, %f52, %f51; + add.s32 %r63, %r99, -1; + mul.wide.u32 %rd39, %r63, 4; + add.s64 %rd40, %rd2, %rd39; + ld.global.f32 %f54, [%rd40]; + mov.f32 %f55, 0f3D5901AE; + fma.rn.f32 %f56, %f53, %f55, %f54; + st.global.f32 [%rd40], %f56; + add.s32 %r99, %r99, 2; + setp.lt.u32 %p34, %r99, %r2; + @%p34 bra $L__BB6_25; + +$L__BB6_26: + @%p20 bra $L__BB6_28; + + setp.gt.u32 %p36, %r6, 1; + mov.u32 %r64, 2; + sub.s32 %r65, %r64, %r2; + selp.b32 %r67, %r7, %r65, %p36; + mul.wide.u32 %rd41, %r67, 4; + add.s64 %rd42, %rd2, %rd41; + ld.global.f32 %f57, [%rd4]; + ld.global.f32 %f58, [%rd42]; + add.rn.f32 %f59, %f58, %f57; + ld.global.f32 %f60, [%rd3]; + mov.f32 %f61, 0f3D5901AE; + fma.rn.f32 %f62, %f59, %f61, %f60; + st.global.f32 [%rd3], %f62; + +$L__BB6_28: + @%p22 bra $L__BB6_30; + + add.s32 %r68, %r2, %r2; + add.s32 %r69, %r68, -3; + selp.b32 %r70, 1, %r69, %p18; + mul.wide.u32 %rd43, %r70, 4; + add.s64 %rd44, %rd2, %rd43; + ld.global.f32 %f63, [%rd44]; + ld.global.f32 %f64, [%rd2+4]; + add.rn.f32 %f65, %f64, %f63; + ld.global.f32 %f66, [%rd2]; + mov.f32 %f67, 0f3FCB0673; + fma.rn.f32 %f68, %f65, %f67, %f66; + st.global.f32 [%rd2], %f68; + +$L__BB6_30: + @%p25 bra $L__BB6_32; + +$L__BB6_31: + add.s32 %r71, %r100, -2; + mul.wide.u32 %rd45, %r71, 4; + add.s64 %rd46, %rd2, %rd45; + mul.wide.u32 %rd47, %r100, 4; + add.s64 %rd48, %rd2, %rd47; + ld.global.f32 %f69, [%rd48]; + ld.global.f32 %f70, [%rd46]; + add.rn.f32 %f71, %f70, %f69; + add.s32 %r72, %r100, -1; + mul.wide.u32 %rd49, %r72, 4; + add.s64 %rd50, %rd2, %rd49; + ld.global.f32 %f72, [%rd50]; + mov.f32 %f73, 0f3FCB0673; + fma.rn.f32 %f74, %f71, %f73, %f72; + st.global.f32 [%rd50], %f74; + add.s32 %r100, %r100, 2; + setp.lt.u32 %p40, %r100, %r2; + @%p40 bra $L__BB6_31; + +$L__BB6_32: + @%p29 bra $L__BB6_48; + + setp.gt.u32 %p42, %r6, 1; + mov.u32 %r73, 2; + sub.s32 %r74, %r73, %r2; + selp.b32 %r76, %r7, %r74, %p42; + mul.wide.u32 %rd51, %r76, 4; + add.s64 %rd52, %rd2, %rd51; + ld.global.f32 %f75, [%rd4]; + ld.global.f32 %f76, [%rd52]; + add.rn.f32 %f77, %f76, %f75; + ld.global.f32 %f78, [%rd3]; + mov.f32 %f79, 0f3FCB0673; + fma.rn.f32 %f80, %f77, %f79, %f78; + st.global.f32 [%rd3], %f80; + bra.uni $L__BB6_48; + +$L__BB6_34: + setp.ne.s32 %p43, %r4, 0; + @%p43 bra $L__BB6_36; + + setp.gt.u32 %p44, %r2, 1; + add.s32 %r77, %r2, %r2; + add.s32 %r78, %r77, -3; + selp.b32 %r79, 1, %r78, %p44; + mul.wide.u32 %rd53, %r79, 4; + add.s64 %rd54, %rd2, %rd53; + ld.global.f32 %f81, [%rd54]; + ld.global.f32 %f82, [%rd2+4]; + add.rn.f32 %f83, %f82, %f81; + mov.f32 %f84, 0f3F000000; + mov.f32 %f85, 0f3E800000; + fma.rn.f32 %f86, %f83, %f85, %f84; + cvt.rmi.f32.f32 %f87, %f86; + ld.global.f32 %f88, [%rd2]; + sub.rn.f32 %f89, %f88, %f87; + st.global.f32 [%rd2], %f89; + +$L__BB6_36: + setp.eq.s32 %p45, %r4, 0; + selp.b32 %r101, 3, 2, %p45; + setp.ge.u32 %p46, %r101, %r2; + @%p46 bra $L__BB6_38; + +$L__BB6_37: + add.s32 %r80, %r101, -1; + mul.wide.u32 %rd55, %r80, 4; + add.s64 %rd56, %rd2, %rd55; + add.s32 %r81, %r101, -2; + mul.wide.u32 %rd57, %r81, 4; + add.s64 %rd58, %rd2, %rd57; + mul.wide.u32 %rd59, %r101, 4; + add.s64 %rd60, %rd2, %rd59; + ld.global.f32 %f90, [%rd60]; + ld.global.f32 %f91, [%rd58]; + add.rn.f32 %f92, %f91, %f90; + mov.f32 %f93, 0f3F000000; + mov.f32 %f94, 0f3E800000; + fma.rn.f32 %f95, %f92, %f94, %f93; + cvt.rmi.f32.f32 %f96, %f95; + ld.global.f32 %f97, [%rd56]; + sub.rn.f32 %f98, %f97, %f96; + st.global.f32 [%rd56], %f98; + add.s32 %r101, %r101, 2; + setp.lt.u32 %p47, %r101, %r2; + @%p47 bra $L__BB6_37; + +$L__BB6_38: + setp.lt.u32 %p48, %r2, 2; + and.b32 %r25, %r2, 1; + xor.b32 %r82, %r25, 1; + setp.ne.s32 %p49, %r82, %r4; + or.pred %p50, %p48, %p49; + @%p50 bra $L__BB6_40; + + setp.gt.u32 %p51, %r6, 1; + mov.u32 %r84, 2; + sub.s32 %r85, %r84, %r2; + selp.b32 %r87, %r7, %r85, %p51; + mul.wide.u32 %rd61, %r87, 4; + add.s64 %rd62, %rd2, %rd61; + ld.global.f32 %f99, [%rd4]; + ld.global.f32 %f100, [%rd62]; + add.rn.f32 %f101, %f100, %f99; + mov.f32 %f102, 0f3F000000; + mov.f32 %f103, 0f3E800000; + fma.rn.f32 %f104, %f101, %f103, %f102; + cvt.rmi.f32.f32 %f105, %f104; + ld.global.f32 %f106, [%rd3]; + sub.rn.f32 %f107, %f106, %f105; + st.global.f32 [%rd3], %f107; + +$L__BB6_40: + setp.ne.s32 %p52, %r5, 0; + @%p52 bra $L__BB6_42; + + setp.gt.u32 %p53, %r2, 1; + add.s32 %r88, %r2, %r2; + add.s32 %r89, %r88, -3; + selp.b32 %r90, 1, %r89, %p53; + mul.wide.u32 %rd63, %r90, 4; + add.s64 %rd64, %rd2, %rd63; + ld.global.f32 %f108, [%rd64]; + ld.global.f32 %f109, [%rd2+4]; + add.rn.f32 %f110, %f109, %f108; + mul.rn.f32 %f111, %f110, 0f3F000000; + cvt.rmi.f32.f32 %f112, %f111; + ld.global.f32 %f113, [%rd2]; + add.rn.f32 %f114, %f113, %f112; + st.global.f32 [%rd2], %f114; + +$L__BB6_42: + setp.eq.s32 %p54, %r5, 0; + selp.b32 %r102, 3, 2, %p54; + setp.ge.u32 %p55, %r102, %r2; + @%p55 bra $L__BB6_44; + +$L__BB6_43: + add.s32 %r91, %r102, -1; + mul.wide.u32 %rd65, %r91, 4; + add.s64 %rd66, %rd2, %rd65; + add.s32 %r92, %r102, -2; + mul.wide.u32 %rd67, %r92, 4; + add.s64 %rd68, %rd2, %rd67; + mul.wide.u32 %rd69, %r102, 4; + add.s64 %rd70, %rd2, %rd69; + ld.global.f32 %f115, [%rd70]; + ld.global.f32 %f116, [%rd68]; + add.rn.f32 %f117, %f116, %f115; + mul.rn.f32 %f118, %f117, 0f3F000000; + cvt.rmi.f32.f32 %f119, %f118; + ld.global.f32 %f120, [%rd66]; + add.rn.f32 %f121, %f120, %f119; + st.global.f32 [%rd66], %f121; + add.s32 %r102, %r102, 2; + setp.lt.u32 %p56, %r102, %r2; + @%p56 bra $L__BB6_43; + +$L__BB6_44: + setp.ne.s32 %p58, %r25, %r4; + or.pred %p59, %p48, %p58; + @%p59 bra $L__BB6_48; + + setp.gt.u32 %p60, %r6, 1; + mov.u32 %r93, 2; + sub.s32 %r94, %r93, %r2; + selp.b32 %r95, %r7, %r94, %p60; + mul.wide.u32 %rd71, %r95, 4; + add.s64 %rd72, %rd2, %rd71; + ld.global.f32 %f122, [%rd4]; + ld.global.f32 %f123, [%rd72]; + add.rn.f32 %f124, %f123, %f122; + mul.rn.f32 %f125, %f124, 0f3F000000; + cvt.rmi.f32.f32 %f126, %f125; + ld.global.f32 %f127, [%rd3]; + add.rn.f32 %f128, %f127, %f126; + st.global.f32 [%rd3], %f128; + +$L__BB6_48: + ret; + +} + // .globl j2k_idwt_horizontal_53 +.visible .entry j2k_idwt_horizontal_53( + .param .u64 j2k_idwt_horizontal_53_param_0, + .param .u64 j2k_idwt_horizontal_53_param_1 +) +{ + .reg .pred %p<22>; + .reg .f32 %f<51>; + .reg .b32 %r<43>; + .reg .b64 %rd<31>; + + + ld.param.u64 %rd4, [j2k_idwt_horizontal_53_param_0]; + ld.param.u64 %rd5, [j2k_idwt_horizontal_53_param_1]; + cvta.to.global.u64 %rd6, %rd5; + ld.global.u32 %r14, [%rd6+8]; + ld.global.u32 %r1, [%rd6]; + sub.s32 %r2, %r14, %r1; + ld.global.u32 %r15, [%rd6+12]; + ld.global.u32 %r16, [%rd6+4]; + sub.s32 %r17, %r15, %r16; + mov.u32 %r18, %ntid.x; + mov.u32 %r19, %ctaid.x; + mov.u32 %r20, %tid.x; + mad.lo.s32 %r3, %r19, %r18, %r20; + setp.ge.u32 %p1, %r3, %r17; + @%p1 bra $L__BB7_16; + + cvta.to.global.u64 %rd7, %rd4; + mul.lo.s32 %r21, %r2, %r3; + mul.wide.u32 %rd8, %r21, 4; + add.s64 %rd1, %rd7, %rd8; + and.b32 %r4, %r1, 1; + setp.eq.s32 %p2, %r2, 1; + @%p2 bra $L__BB7_14; + bra.uni $L__BB7_2; + +$L__BB7_14: + setp.eq.s32 %p21, %r4, 0; + @%p21 bra $L__BB7_16; + + ld.global.f32 %f49, [%rd1]; + mul.rn.f32 %f50, %f49, 0f3F000000; + st.global.f32 [%rd1], %f50; + bra.uni $L__BB7_16; + +$L__BB7_2: + setp.ne.s32 %p3, %r4, 0; + @%p3 bra $L__BB7_4; + + setp.gt.u32 %p4, %r2, 1; + add.s32 %r22, %r2, %r2; + add.s32 %r23, %r22, -3; + selp.b32 %r24, 1, %r23, %p4; + mul.wide.u32 %rd9, %r24, 4; + add.s64 %rd10, %rd1, %rd9; + ld.global.f32 %f1, [%rd10]; + ld.global.f32 %f2, [%rd1+4]; + add.rn.f32 %f3, %f2, %f1; + mov.f32 %f4, 0f3F000000; + mov.f32 %f5, 0f3E800000; + fma.rn.f32 %f6, %f3, %f5, %f4; + cvt.rmi.f32.f32 %f7, %f6; + ld.global.f32 %f8, [%rd1]; + sub.rn.f32 %f9, %f8, %f7; + st.global.f32 [%rd1], %f9; + +$L__BB7_4: + setp.eq.s32 %p5, %r4, 0; + selp.b32 %r41, 3, 2, %p5; + setp.ge.u32 %p6, %r41, %r2; + @%p6 bra $L__BB7_6; + +$L__BB7_5: + add.s32 %r25, %r41, -1; + mul.wide.u32 %rd11, %r25, 4; + add.s64 %rd12, %rd1, %rd11; + add.s32 %r26, %r41, -2; + mul.wide.u32 %rd13, %r26, 4; + add.s64 %rd14, %rd1, %rd13; + mul.wide.u32 %rd15, %r41, 4; + add.s64 %rd16, %rd1, %rd15; + ld.global.f32 %f10, [%rd16]; + ld.global.f32 %f11, [%rd14]; + add.rn.f32 %f12, %f11, %f10; + mov.f32 %f13, 0f3F000000; + mov.f32 %f14, 0f3E800000; + fma.rn.f32 %f15, %f12, %f14, %f13; + cvt.rmi.f32.f32 %f16, %f15; + ld.global.f32 %f17, [%rd12]; + sub.rn.f32 %f18, %f17, %f16; + st.global.f32 [%rd12], %f18; + add.s32 %r41, %r41, 2; + setp.lt.u32 %p7, %r41, %r2; + @%p7 bra $L__BB7_5; + +$L__BB7_6: + setp.lt.u32 %p8, %r2, 2; + and.b32 %r8, %r2, 1; + xor.b32 %r27, %r8, 1; + setp.ne.s32 %p9, %r27, %r4; + add.s32 %r9, %r2, -1; + mul.wide.u32 %rd17, %r9, 4; + add.s64 %rd2, %rd1, %rd17; + add.s32 %r10, %r2, -2; + mul.wide.u32 %rd18, %r10, 4; + add.s64 %rd3, %rd1, %rd18; + or.pred %p10, %p8, %p9; + @%p10 bra $L__BB7_8; + + setp.gt.u32 %p11, %r9, 1; + mov.u32 %r29, 2; + sub.s32 %r30, %r29, %r2; + selp.b32 %r32, %r10, %r30, %p11; + mul.wide.u32 %rd19, %r32, 4; + add.s64 %rd20, %rd1, %rd19; + ld.global.f32 %f19, [%rd3]; + ld.global.f32 %f20, [%rd20]; + add.rn.f32 %f21, %f20, %f19; + mov.f32 %f22, 0f3F000000; + mov.f32 %f23, 0f3E800000; + fma.rn.f32 %f24, %f21, %f23, %f22; + cvt.rmi.f32.f32 %f25, %f24; + ld.global.f32 %f26, [%rd2]; + sub.rn.f32 %f27, %f26, %f25; + st.global.f32 [%rd2], %f27; + +$L__BB7_8: + @%p5 bra $L__BB7_10; + + setp.gt.u32 %p13, %r2, 1; + add.s32 %r33, %r2, %r2; + add.s32 %r34, %r33, -3; + selp.b32 %r35, 1, %r34, %p13; + mul.wide.u32 %rd21, %r35, 4; + add.s64 %rd22, %rd1, %rd21; + ld.global.f32 %f28, [%rd22]; + ld.global.f32 %f29, [%rd1+4]; + add.rn.f32 %f30, %f29, %f28; + mul.rn.f32 %f31, %f30, 0f3F000000; + cvt.rmi.f32.f32 %f32, %f31; + ld.global.f32 %f33, [%rd1]; + add.rn.f32 %f34, %f33, %f32; + st.global.f32 [%rd1], %f34; + +$L__BB7_10: + selp.b32 %r42, 3, 2, %p3; + setp.ge.u32 %p15, %r42, %r2; + @%p15 bra $L__BB7_12; + +$L__BB7_11: + add.s32 %r36, %r42, -1; + mul.wide.u32 %rd23, %r36, 4; + add.s64 %rd24, %rd1, %rd23; + add.s32 %r37, %r42, -2; + mul.wide.u32 %rd25, %r37, 4; + add.s64 %rd26, %rd1, %rd25; + mul.wide.u32 %rd27, %r42, 4; + add.s64 %rd28, %rd1, %rd27; + ld.global.f32 %f35, [%rd28]; + ld.global.f32 %f36, [%rd26]; + add.rn.f32 %f37, %f36, %f35; + mul.rn.f32 %f38, %f37, 0f3F000000; + cvt.rmi.f32.f32 %f39, %f38; + ld.global.f32 %f40, [%rd24]; + add.rn.f32 %f41, %f40, %f39; + st.global.f32 [%rd24], %f41; + add.s32 %r42, %r42, 2; + setp.lt.u32 %p16, %r42, %r2; + @%p16 bra $L__BB7_11; + +$L__BB7_12: + setp.ne.s32 %p18, %r8, %r4; + or.pred %p19, %p8, %p18; + @%p19 bra $L__BB7_16; + + setp.gt.u32 %p20, %r9, 1; + mov.u32 %r38, 2; + sub.s32 %r39, %r38, %r2; + selp.b32 %r40, %r10, %r39, %p20; + mul.wide.u32 %rd29, %r40, 4; + add.s64 %rd30, %rd1, %rd29; + ld.global.f32 %f42, [%rd3]; + ld.global.f32 %f43, [%rd30]; + add.rn.f32 %f44, %f43, %f42; + mul.rn.f32 %f45, %f44, 0f3F000000; + cvt.rmi.f32.f32 %f46, %f45; + ld.global.f32 %f47, [%rd2]; + add.rn.f32 %f48, %f47, %f46; + st.global.f32 [%rd2], %f48; + +$L__BB7_16: + ret; + +} + // .globl j2k_idwt_horizontal_97 +.visible .entry j2k_idwt_horizontal_97( + .param .u64 j2k_idwt_horizontal_97_param_0, + .param .u64 j2k_idwt_horizontal_97_param_1 +) +{ + .reg .pred %p<43>; + .reg .f32 %f<83>; + .reg .b32 %r<72>; + .reg .b64 %rd<53>; + + + ld.param.u64 %rd4, [j2k_idwt_horizontal_97_param_0]; + ld.param.u64 %rd5, [j2k_idwt_horizontal_97_param_1]; + cvta.to.global.u64 %rd6, %rd5; + ld.global.u32 %r21, [%rd6+8]; + ld.global.u32 %r1, [%rd6]; + sub.s32 %r2, %r21, %r1; + ld.global.u32 %r22, [%rd6+12]; + ld.global.u32 %r23, [%rd6+4]; + sub.s32 %r24, %r22, %r23; + mov.u32 %r25, %ntid.x; + mov.u32 %r26, %ctaid.x; + mov.u32 %r27, %tid.x; + mad.lo.s32 %r3, %r26, %r25, %r27; + setp.ge.u32 %p4, %r3, %r24; + @%p4 bra $L__BB8_35; + + cvta.to.global.u64 %rd7, %rd4; + mul.lo.s32 %r28, %r2, %r3; + mul.wide.u32 %rd8, %r28, 4; + add.s64 %rd1, %rd7, %rd8; + and.b32 %r4, %r1, 1; + setp.eq.s32 %p5, %r2, 1; + @%p5 bra $L__BB8_33; + bra.uni $L__BB8_2; + +$L__BB8_33: + setp.eq.s32 %p42, %r4, 0; + @%p42 bra $L__BB8_35; + + ld.global.f32 %f81, [%rd1]; + mul.rn.f32 %f82, %f81, 0f3F000000; + st.global.f32 [%rd1], %f82; + bra.uni $L__BB8_35; + +$L__BB8_2: + xor.b32 %r5, %r4, 1; + setp.eq.s32 %p1, %r4, 0; + selp.f32 %f1, 0f3F9D7658, 0f3F5019C3, %p1; + setp.eq.s32 %p6, %r2, 0; + @%p6 bra $L__BB8_5; + + selp.f32 %f2, 0f3F5019C3, 0f3F9D7658, %p1; + mov.u32 %r67, 1; + +$L__BB8_4: + add.s32 %r30, %r67, -1; + mul.wide.u32 %rd9, %r30, 4; + add.s64 %rd10, %rd1, %rd9; + ld.global.f32 %f3, [%rd10]; + mul.rn.f32 %f4, %f1, %f3; + st.global.f32 [%rd10], %f4; + ld.global.f32 %f5, [%rd10+4]; + mul.rn.f32 %f6, %f2, %f5; + st.global.f32 [%rd10+4], %f6; + add.s32 %r67, %r67, 2; + setp.lt.u32 %p7, %r67, %r2; + @%p7 bra $L__BB8_4; + +$L__BB8_5: + and.b32 %r31, %r2, 1; + setp.eq.b32 %p8, %r31, 1; + mov.pred %p9, 0; + xor.pred %p10, %p8, %p9; + not.pred %p11, %p10; + add.s32 %r8, %r2, -1; + mul.wide.u32 %rd11, %r8, 4; + add.s64 %rd2, %rd1, %rd11; + @%p11 bra $L__BB8_7; + + ld.global.f32 %f7, [%rd2]; + mul.rn.f32 %f8, %f1, %f7; + st.global.f32 [%rd2], %f8; + +$L__BB8_7: + setp.ne.s32 %p12, %r4, 0; + @%p12 bra $L__BB8_9; + + setp.gt.u32 %p13, %r2, 1; + add.s32 %r32, %r2, %r2; + add.s32 %r33, %r32, -3; + selp.b32 %r34, 1, %r33, %p13; + mul.wide.u32 %rd12, %r34, 4; + add.s64 %rd13, %rd1, %rd12; + ld.global.f32 %f9, [%rd13]; + ld.global.f32 %f10, [%rd1+4]; + add.rn.f32 %f11, %f10, %f9; + ld.global.f32 %f12, [%rd1]; + mov.f32 %f13, 0fBEE31355; + fma.rn.f32 %f14, %f11, %f13, %f12; + st.global.f32 [%rd1], %f14; + +$L__BB8_9: + selp.b32 %r70, 3, 2, %p1; + setp.ge.u32 %p15, %r70, %r2; + @%p15 bra $L__BB8_12; + + mov.u32 %r68, %r70; + +$L__BB8_11: + add.s32 %r35, %r68, -2; + mul.wide.u32 %rd14, %r35, 4; + add.s64 %rd15, %rd1, %rd14; + mul.wide.u32 %rd16, %r68, 4; + add.s64 %rd17, %rd1, %rd16; + ld.global.f32 %f15, [%rd17]; + ld.global.f32 %f16, [%rd15]; + add.rn.f32 %f17, %f16, %f15; + add.s32 %r36, %r68, -1; + mul.wide.u32 %rd18, %r36, 4; + add.s64 %rd19, %rd1, %rd18; + ld.global.f32 %f18, [%rd19]; + mov.f32 %f19, 0fBEE31355; + fma.rn.f32 %f20, %f17, %f19, %f18; + st.global.f32 [%rd19], %f20; + add.s32 %r68, %r68, 2; + setp.lt.u32 %p16, %r68, %r2; + @%p16 bra $L__BB8_11; + +$L__BB8_12: + setp.gt.u32 %p17, %r2, 1; + and.b32 %r12, %r8, 1; + setp.eq.s32 %p18, %r12, %r4; + and.pred %p2, %p17, %p18; + add.s32 %r13, %r2, -2; + mul.wide.u32 %rd20, %r13, 4; + add.s64 %rd3, %rd1, %rd20; + not.pred %p19, %p2; + @%p19 bra $L__BB8_14; + + setp.gt.u32 %p20, %r8, 1; + mov.u32 %r37, 2; + sub.s32 %r38, %r37, %r2; + selp.b32 %r40, %r13, %r38, %p20; + mul.wide.u32 %rd21, %r40, 4; + add.s64 %rd22, %rd1, %rd21; + ld.global.f32 %f21, [%rd3]; + ld.global.f32 %f22, [%rd22]; + add.rn.f32 %f23, %f22, %f21; + ld.global.f32 %f24, [%rd2]; + mov.f32 %f25, 0fBEE31355; + fma.rn.f32 %f26, %f23, %f25, %f24; + st.global.f32 [%rd2], %f26; + +$L__BB8_14: + setp.ne.s32 %p21, %r5, 0; + @%p21 bra $L__BB8_16; + + add.s32 %r41, %r2, %r2; + add.s32 %r42, %r41, -3; + selp.b32 %r43, 1, %r42, %p17; + mul.wide.u32 %rd23, %r43, 4; + add.s64 %rd24, %rd1, %rd23; + ld.global.f32 %f27, [%rd24]; + ld.global.f32 %f28, [%rd1+4]; + add.rn.f32 %f29, %f28, %f27; + ld.global.f32 %f30, [%rd1]; + mov.f32 %f31, 0fBF620676; + fma.rn.f32 %f32, %f29, %f31, %f30; + st.global.f32 [%rd1], %f32; + +$L__BB8_16: + setp.eq.s32 %p23, %r5, 0; + selp.b32 %r71, 3, 2, %p23; + setp.ge.u32 %p24, %r71, %r2; + @%p24 bra $L__BB8_19; + + mov.u32 %r69, %r71; + +$L__BB8_18: + add.s32 %r44, %r69, -2; + mul.wide.u32 %rd25, %r44, 4; + add.s64 %rd26, %rd1, %rd25; + mul.wide.u32 %rd27, %r69, 4; + add.s64 %rd28, %rd1, %rd27; + ld.global.f32 %f33, [%rd28]; + ld.global.f32 %f34, [%rd26]; + add.rn.f32 %f35, %f34, %f33; + add.s32 %r45, %r69, -1; + mul.wide.u32 %rd29, %r45, 4; + add.s64 %rd30, %rd1, %rd29; + ld.global.f32 %f36, [%rd30]; + mov.f32 %f37, 0fBF620676; + fma.rn.f32 %f38, %f35, %f37, %f36; + st.global.f32 [%rd30], %f38; + add.s32 %r69, %r69, 2; + setp.lt.u32 %p25, %r69, %r2; + @%p25 bra $L__BB8_18; + +$L__BB8_19: + setp.eq.s32 %p27, %r12, %r5; + and.pred %p3, %p17, %p27; + not.pred %p28, %p3; + @%p28 bra $L__BB8_21; + + setp.gt.u32 %p29, %r8, 1; + mov.u32 %r46, 2; + sub.s32 %r47, %r46, %r2; + selp.b32 %r49, %r13, %r47, %p29; + mul.wide.u32 %rd31, %r49, 4; + add.s64 %rd32, %rd1, %rd31; + ld.global.f32 %f39, [%rd3]; + ld.global.f32 %f40, [%rd32]; + add.rn.f32 %f41, %f40, %f39; + ld.global.f32 %f42, [%rd2]; + mov.f32 %f43, 0fBF620676; + fma.rn.f32 %f44, %f41, %f43, %f42; + st.global.f32 [%rd2], %f44; + +$L__BB8_21: + @%p12 bra $L__BB8_23; + + add.s32 %r50, %r2, %r2; + add.s32 %r51, %r50, -3; + selp.b32 %r52, 1, %r51, %p17; + mul.wide.u32 %rd33, %r52, 4; + add.s64 %rd34, %rd1, %rd33; + ld.global.f32 %f45, [%rd34]; + ld.global.f32 %f46, [%rd1+4]; + add.rn.f32 %f47, %f46, %f45; + ld.global.f32 %f48, [%rd1]; + mov.f32 %f49, 0f3D5901AE; + fma.rn.f32 %f50, %f47, %f49, %f48; + st.global.f32 [%rd1], %f50; + +$L__BB8_23: + @%p15 bra $L__BB8_25; + +$L__BB8_24: + add.s32 %r53, %r70, -2; + mul.wide.u32 %rd35, %r53, 4; + add.s64 %rd36, %rd1, %rd35; + mul.wide.u32 %rd37, %r70, 4; + add.s64 %rd38, %rd1, %rd37; + ld.global.f32 %f51, [%rd38]; + ld.global.f32 %f52, [%rd36]; + add.rn.f32 %f53, %f52, %f51; + add.s32 %r54, %r70, -1; + mul.wide.u32 %rd39, %r54, 4; + add.s64 %rd40, %rd1, %rd39; + ld.global.f32 %f54, [%rd40]; + mov.f32 %f55, 0f3D5901AE; + fma.rn.f32 %f56, %f53, %f55, %f54; + st.global.f32 [%rd40], %f56; + add.s32 %r70, %r70, 2; + setp.lt.u32 %p33, %r70, %r2; + @%p33 bra $L__BB8_24; + +$L__BB8_25: + @%p19 bra $L__BB8_27; + + setp.gt.u32 %p35, %r8, 1; + mov.u32 %r55, 2; + sub.s32 %r56, %r55, %r2; + selp.b32 %r58, %r13, %r56, %p35; + mul.wide.u32 %rd41, %r58, 4; + add.s64 %rd42, %rd1, %rd41; + ld.global.f32 %f57, [%rd3]; + ld.global.f32 %f58, [%rd42]; + add.rn.f32 %f59, %f58, %f57; + ld.global.f32 %f60, [%rd2]; + mov.f32 %f61, 0f3D5901AE; + fma.rn.f32 %f62, %f59, %f61, %f60; + st.global.f32 [%rd2], %f62; + +$L__BB8_27: + @%p21 bra $L__BB8_29; + + add.s32 %r59, %r2, %r2; + add.s32 %r60, %r59, -3; + selp.b32 %r61, 1, %r60, %p17; + mul.wide.u32 %rd43, %r61, 4; + add.s64 %rd44, %rd1, %rd43; + ld.global.f32 %f63, [%rd44]; + ld.global.f32 %f64, [%rd1+4]; + add.rn.f32 %f65, %f64, %f63; + ld.global.f32 %f66, [%rd1]; + mov.f32 %f67, 0f3FCB0673; + fma.rn.f32 %f68, %f65, %f67, %f66; + st.global.f32 [%rd1], %f68; + +$L__BB8_29: + @%p24 bra $L__BB8_31; + +$L__BB8_30: + add.s32 %r62, %r71, -2; + mul.wide.u32 %rd45, %r62, 4; + add.s64 %rd46, %rd1, %rd45; + mul.wide.u32 %rd47, %r71, 4; + add.s64 %rd48, %rd1, %rd47; + ld.global.f32 %f69, [%rd48]; + ld.global.f32 %f70, [%rd46]; + add.rn.f32 %f71, %f70, %f69; + add.s32 %r63, %r71, -1; + mul.wide.u32 %rd49, %r63, 4; + add.s64 %rd50, %rd1, %rd49; + ld.global.f32 %f72, [%rd50]; + mov.f32 %f73, 0f3FCB0673; + fma.rn.f32 %f74, %f71, %f73, %f72; + st.global.f32 [%rd50], %f74; + add.s32 %r71, %r71, 2; + setp.lt.u32 %p39, %r71, %r2; + @%p39 bra $L__BB8_30; + +$L__BB8_31: + @%p28 bra $L__BB8_35; + + setp.gt.u32 %p41, %r8, 1; + mov.u32 %r64, 2; + sub.s32 %r65, %r64, %r2; + selp.b32 %r66, %r13, %r65, %p41; + mul.wide.u32 %rd51, %r66, 4; + add.s64 %rd52, %rd1, %rd51; + ld.global.f32 %f75, [%rd3]; + ld.global.f32 %f76, [%rd52]; + add.rn.f32 %f77, %f76, %f75; + ld.global.f32 %f78, [%rd2]; + mov.f32 %f79, 0f3FCB0673; + fma.rn.f32 %f80, %f77, %f79, %f78; + st.global.f32 [%rd2], %f80; + +$L__BB8_35: + ret; + +} + // .globl j2k_idwt_horizontal_multi +.visible .entry j2k_idwt_horizontal_multi( + .param .u64 j2k_idwt_horizontal_multi_param_0 +) +{ + .reg .pred %p<62>; + .reg .f32 %f<131>; + .reg .b32 %r<108>; + .reg .b64 %rd<75>; + + + ld.param.u64 %rd5, [j2k_idwt_horizontal_multi_param_0]; + cvta.to.global.u64 %rd6, %rd5; + mov.u32 %r29, %ctaid.y; + mul.wide.u32 %rd7, %r29, 128; + add.s64 %rd8, %rd6, %rd7; + add.s64 %rd1, %rd8, 120; + ld.global.v2.u32 {%r30, %r31}, [%rd8+48]; + ld.global.v2.u32 {%r34, %r35}, [%rd8+40]; + sub.s32 %r2, %r30, %r34; + sub.s32 %r37, %r31, %r35; + mov.u32 %r38, %ntid.x; + mov.u32 %r39, %ctaid.x; + mov.u32 %r40, %tid.x; + mad.lo.s32 %r3, %r39, %r38, %r40; + setp.ge.u32 %p4, %r3, %r37; + @%p4 bra $L__BB9_48; + + mul.lo.s32 %r41, %r2, %r3; + ld.global.u64 %rd9, [%rd1+-88]; + mul.wide.u32 %rd10, %r41, 4; + add.s64 %rd2, %rd9, %rd10; + and.b32 %r4, %r34, 1; + setp.eq.s32 %p5, %r2, 1; + @%p5 bra $L__BB9_46; + bra.uni $L__BB9_2; + +$L__BB9_46: + setp.eq.s32 %p61, %r4, 0; + @%p61 bra $L__BB9_48; + + ld.f32 %f129, [%rd2]; + mul.rn.f32 %f130, %f129, 0f3F000000; + st.f32 [%rd2], %f130; + bra.uni $L__BB9_48; + +$L__BB9_2: + ld.global.u32 %r42, [%rd1]; + setp.eq.s32 %p6, %r42, 0; + xor.b32 %r5, %r4, 1; + add.s32 %r6, %r2, -1; + mul.wide.u32 %rd11, %r6, 4; + add.s64 %rd3, %rd2, %rd11; + add.s32 %r7, %r2, -2; + mul.wide.u32 %rd12, %r7, 4; + add.s64 %rd4, %rd2, %rd12; + @%p6 bra $L__BB9_34; + + setp.eq.s32 %p1, %r4, 0; + selp.f32 %f1, 0f3F9D7658, 0f3F5019C3, %p1; + setp.lt.u32 %p7, %r2, 2; + @%p7 bra $L__BB9_6; + + selp.f32 %f2, 0f3F5019C3, 0f3F9D7658, %p1; + mov.u32 %r101, 1; + +$L__BB9_5: + add.s32 %r44, %r101, -1; + mul.wide.u32 %rd13, %r44, 4; + add.s64 %rd14, %rd2, %rd13; + ld.f32 %f3, [%rd14]; + mul.rn.f32 %f4, %f1, %f3; + st.f32 [%rd14], %f4; + ld.f32 %f5, [%rd14+4]; + mul.rn.f32 %f6, %f2, %f5; + st.f32 [%rd14+4], %f6; + add.s32 %r101, %r101, 2; + setp.lt.u32 %p8, %r101, %r2; + @%p8 bra $L__BB9_5; + +$L__BB9_6: + and.b32 %r45, %r2, 1; + setp.eq.b32 %p9, %r45, 1; + mov.pred %p10, 0; + xor.pred %p11, %p9, %p10; + not.pred %p12, %p11; + @%p12 bra $L__BB9_8; + + ld.f32 %f7, [%rd3]; + mul.rn.f32 %f8, %f1, %f7; + st.f32 [%rd3], %f8; + +$L__BB9_8: + setp.ne.s32 %p13, %r4, 0; + @%p13 bra $L__BB9_10; + + setp.gt.u32 %p14, %r2, 1; + add.s32 %r46, %r2, %r2; + add.s32 %r47, %r46, -3; + selp.b32 %r48, 1, %r47, %p14; + mul.wide.u32 %rd15, %r48, 4; + add.s64 %rd16, %rd2, %rd15; + ld.f32 %f9, [%rd16]; + ld.f32 %f10, [%rd2+4]; + add.rn.f32 %f11, %f10, %f9; + ld.f32 %f12, [%rd2]; + mov.f32 %f13, 0fBEE31355; + fma.rn.f32 %f14, %f11, %f13, %f12; + st.f32 [%rd2], %f14; + +$L__BB9_10: + selp.b32 %r104, 3, 2, %p1; + setp.ge.u32 %p16, %r104, %r2; + @%p16 bra $L__BB9_13; + + mov.u32 %r102, %r104; + +$L__BB9_12: + add.s32 %r49, %r102, -2; + mul.wide.u32 %rd17, %r49, 4; + add.s64 %rd18, %rd2, %rd17; + mul.wide.u32 %rd19, %r102, 4; + add.s64 %rd20, %rd2, %rd19; + ld.f32 %f15, [%rd20]; + ld.f32 %f16, [%rd18]; + add.rn.f32 %f17, %f16, %f15; + add.s32 %r50, %r102, -1; + mul.wide.u32 %rd21, %r50, 4; + add.s64 %rd22, %rd2, %rd21; + ld.f32 %f18, [%rd22]; + mov.f32 %f19, 0fBEE31355; + fma.rn.f32 %f20, %f17, %f19, %f18; + st.f32 [%rd22], %f20; + add.s32 %r102, %r102, 2; + setp.lt.u32 %p17, %r102, %r2; + @%p17 bra $L__BB9_12; + +$L__BB9_13: + setp.gt.u32 %p18, %r2, 1; + and.b32 %r14, %r6, 1; + setp.eq.s32 %p19, %r14, %r4; + and.pred %p2, %p18, %p19; + not.pred %p20, %p2; + @%p20 bra $L__BB9_15; + + setp.gt.u32 %p21, %r6, 1; + mov.u32 %r51, 2; + sub.s32 %r52, %r51, %r2; + selp.b32 %r54, %r7, %r52, %p21; + mul.wide.u32 %rd23, %r54, 4; + add.s64 %rd24, %rd2, %rd23; + ld.f32 %f21, [%rd4]; + ld.f32 %f22, [%rd24]; + add.rn.f32 %f23, %f22, %f21; + ld.f32 %f24, [%rd3]; + mov.f32 %f25, 0fBEE31355; + fma.rn.f32 %f26, %f23, %f25, %f24; + st.f32 [%rd3], %f26; + +$L__BB9_15: + setp.ne.s32 %p22, %r5, 0; + @%p22 bra $L__BB9_17; + + add.s32 %r55, %r2, %r2; + add.s32 %r56, %r55, -3; + selp.b32 %r57, 1, %r56, %p18; + mul.wide.u32 %rd25, %r57, 4; + add.s64 %rd26, %rd2, %rd25; + ld.f32 %f27, [%rd26]; + ld.f32 %f28, [%rd2+4]; + add.rn.f32 %f29, %f28, %f27; + ld.f32 %f30, [%rd2]; + mov.f32 %f31, 0fBF620676; + fma.rn.f32 %f32, %f29, %f31, %f30; + st.f32 [%rd2], %f32; + +$L__BB9_17: + setp.eq.s32 %p24, %r5, 0; + selp.b32 %r105, 3, 2, %p24; + setp.ge.u32 %p25, %r105, %r2; + @%p25 bra $L__BB9_20; + + mov.u32 %r103, %r105; + +$L__BB9_19: + add.s32 %r58, %r103, -2; + mul.wide.u32 %rd27, %r58, 4; + add.s64 %rd28, %rd2, %rd27; + mul.wide.u32 %rd29, %r103, 4; + add.s64 %rd30, %rd2, %rd29; + ld.f32 %f33, [%rd30]; + ld.f32 %f34, [%rd28]; + add.rn.f32 %f35, %f34, %f33; + add.s32 %r59, %r103, -1; + mul.wide.u32 %rd31, %r59, 4; + add.s64 %rd32, %rd2, %rd31; + ld.f32 %f36, [%rd32]; + mov.f32 %f37, 0fBF620676; + fma.rn.f32 %f38, %f35, %f37, %f36; + st.f32 [%rd32], %f38; + add.s32 %r103, %r103, 2; + setp.lt.u32 %p26, %r103, %r2; + @%p26 bra $L__BB9_19; + +$L__BB9_20: + setp.eq.s32 %p28, %r14, %r5; + and.pred %p3, %p18, %p28; + not.pred %p29, %p3; + @%p29 bra $L__BB9_22; + + setp.gt.u32 %p30, %r6, 1; + mov.u32 %r60, 2; + sub.s32 %r61, %r60, %r2; + selp.b32 %r63, %r7, %r61, %p30; + mul.wide.u32 %rd33, %r63, 4; + add.s64 %rd34, %rd2, %rd33; + ld.f32 %f39, [%rd4]; + ld.f32 %f40, [%rd34]; + add.rn.f32 %f41, %f40, %f39; + ld.f32 %f42, [%rd3]; + mov.f32 %f43, 0fBF620676; + fma.rn.f32 %f44, %f41, %f43, %f42; + st.f32 [%rd3], %f44; + +$L__BB9_22: + @%p13 bra $L__BB9_24; + + add.s32 %r64, %r2, %r2; + add.s32 %r65, %r64, -3; + selp.b32 %r66, 1, %r65, %p18; + mul.wide.u32 %rd35, %r66, 4; + add.s64 %rd36, %rd2, %rd35; + ld.f32 %f45, [%rd36]; + ld.f32 %f46, [%rd2+4]; + add.rn.f32 %f47, %f46, %f45; + ld.f32 %f48, [%rd2]; + mov.f32 %f49, 0f3D5901AE; + fma.rn.f32 %f50, %f47, %f49, %f48; + st.f32 [%rd2], %f50; + +$L__BB9_24: + @%p16 bra $L__BB9_26; + +$L__BB9_25: + add.s32 %r67, %r104, -2; + mul.wide.u32 %rd37, %r67, 4; + add.s64 %rd38, %rd2, %rd37; + mul.wide.u32 %rd39, %r104, 4; + add.s64 %rd40, %rd2, %rd39; + ld.f32 %f51, [%rd40]; + ld.f32 %f52, [%rd38]; + add.rn.f32 %f53, %f52, %f51; + add.s32 %r68, %r104, -1; + mul.wide.u32 %rd41, %r68, 4; + add.s64 %rd42, %rd2, %rd41; + ld.f32 %f54, [%rd42]; + mov.f32 %f55, 0f3D5901AE; + fma.rn.f32 %f56, %f53, %f55, %f54; + st.f32 [%rd42], %f56; + add.s32 %r104, %r104, 2; + setp.lt.u32 %p34, %r104, %r2; + @%p34 bra $L__BB9_25; + +$L__BB9_26: + @%p20 bra $L__BB9_28; + + setp.gt.u32 %p36, %r6, 1; + mov.u32 %r69, 2; + sub.s32 %r70, %r69, %r2; + selp.b32 %r72, %r7, %r70, %p36; + mul.wide.u32 %rd43, %r72, 4; + add.s64 %rd44, %rd2, %rd43; + ld.f32 %f57, [%rd4]; + ld.f32 %f58, [%rd44]; + add.rn.f32 %f59, %f58, %f57; + ld.f32 %f60, [%rd3]; + mov.f32 %f61, 0f3D5901AE; + fma.rn.f32 %f62, %f59, %f61, %f60; + st.f32 [%rd3], %f62; + +$L__BB9_28: + @%p22 bra $L__BB9_30; + + add.s32 %r73, %r2, %r2; + add.s32 %r74, %r73, -3; + selp.b32 %r75, 1, %r74, %p18; + mul.wide.u32 %rd45, %r75, 4; + add.s64 %rd46, %rd2, %rd45; + ld.f32 %f63, [%rd46]; + ld.f32 %f64, [%rd2+4]; + add.rn.f32 %f65, %f64, %f63; + ld.f32 %f66, [%rd2]; + mov.f32 %f67, 0f3FCB0673; + fma.rn.f32 %f68, %f65, %f67, %f66; + st.f32 [%rd2], %f68; + +$L__BB9_30: + @%p25 bra $L__BB9_32; + +$L__BB9_31: + add.s32 %r76, %r105, -2; + mul.wide.u32 %rd47, %r76, 4; + add.s64 %rd48, %rd2, %rd47; + mul.wide.u32 %rd49, %r105, 4; + add.s64 %rd50, %rd2, %rd49; + ld.f32 %f69, [%rd50]; + ld.f32 %f70, [%rd48]; + add.rn.f32 %f71, %f70, %f69; + add.s32 %r77, %r105, -1; + mul.wide.u32 %rd51, %r77, 4; + add.s64 %rd52, %rd2, %rd51; + ld.f32 %f72, [%rd52]; + mov.f32 %f73, 0f3FCB0673; + fma.rn.f32 %f74, %f71, %f73, %f72; + st.f32 [%rd52], %f74; + add.s32 %r105, %r105, 2; + setp.lt.u32 %p40, %r105, %r2; + @%p40 bra $L__BB9_31; + +$L__BB9_32: + @%p29 bra $L__BB9_48; + + setp.gt.u32 %p42, %r6, 1; + mov.u32 %r78, 2; + sub.s32 %r79, %r78, %r2; + selp.b32 %r81, %r7, %r79, %p42; + mul.wide.u32 %rd53, %r81, 4; + add.s64 %rd54, %rd2, %rd53; + ld.f32 %f75, [%rd4]; + ld.f32 %f76, [%rd54]; + add.rn.f32 %f77, %f76, %f75; + ld.f32 %f78, [%rd3]; + mov.f32 %f79, 0f3FCB0673; + fma.rn.f32 %f80, %f77, %f79, %f78; + st.f32 [%rd3], %f80; + bra.uni $L__BB9_48; + +$L__BB9_34: + setp.ne.s32 %p43, %r4, 0; + @%p43 bra $L__BB9_36; + + setp.gt.u32 %p44, %r2, 1; + add.s32 %r82, %r2, %r2; + add.s32 %r83, %r82, -3; + selp.b32 %r84, 1, %r83, %p44; + mul.wide.u32 %rd55, %r84, 4; + add.s64 %rd56, %rd2, %rd55; + ld.f32 %f81, [%rd56]; + ld.f32 %f82, [%rd2+4]; + add.rn.f32 %f83, %f82, %f81; + mov.f32 %f84, 0f3F000000; + mov.f32 %f85, 0f3E800000; + fma.rn.f32 %f86, %f83, %f85, %f84; + cvt.rmi.f32.f32 %f87, %f86; + ld.f32 %f88, [%rd2]; + sub.rn.f32 %f89, %f88, %f87; + st.f32 [%rd2], %f89; + +$L__BB9_36: + setp.eq.s32 %p45, %r4, 0; + selp.b32 %r106, 3, 2, %p45; + setp.ge.u32 %p46, %r106, %r2; + @%p46 bra $L__BB9_38; + +$L__BB9_37: + add.s32 %r85, %r106, -1; + mul.wide.u32 %rd57, %r85, 4; + add.s64 %rd58, %rd2, %rd57; + add.s32 %r86, %r106, -2; + mul.wide.u32 %rd59, %r86, 4; + add.s64 %rd60, %rd2, %rd59; + mul.wide.u32 %rd61, %r106, 4; + add.s64 %rd62, %rd2, %rd61; + ld.f32 %f90, [%rd62]; + ld.f32 %f91, [%rd60]; + add.rn.f32 %f92, %f91, %f90; + mov.f32 %f93, 0f3F000000; + mov.f32 %f94, 0f3E800000; + fma.rn.f32 %f95, %f92, %f94, %f93; + cvt.rmi.f32.f32 %f96, %f95; + ld.f32 %f97, [%rd58]; + sub.rn.f32 %f98, %f97, %f96; + st.f32 [%rd58], %f98; + add.s32 %r106, %r106, 2; + setp.lt.u32 %p47, %r106, %r2; + @%p47 bra $L__BB9_37; + +$L__BB9_38: + setp.lt.u32 %p48, %r2, 2; + and.b32 %r25, %r2, 1; + xor.b32 %r87, %r25, 1; + setp.ne.s32 %p49, %r87, %r4; + or.pred %p50, %p48, %p49; + @%p50 bra $L__BB9_40; + + setp.gt.u32 %p51, %r6, 1; + mov.u32 %r89, 2; + sub.s32 %r90, %r89, %r2; + selp.b32 %r92, %r7, %r90, %p51; + mul.wide.u32 %rd63, %r92, 4; + add.s64 %rd64, %rd2, %rd63; + ld.f32 %f99, [%rd4]; + ld.f32 %f100, [%rd64]; + add.rn.f32 %f101, %f100, %f99; + mov.f32 %f102, 0f3F000000; + mov.f32 %f103, 0f3E800000; + fma.rn.f32 %f104, %f101, %f103, %f102; + cvt.rmi.f32.f32 %f105, %f104; + ld.f32 %f106, [%rd3]; + sub.rn.f32 %f107, %f106, %f105; + st.f32 [%rd3], %f107; + +$L__BB9_40: + setp.ne.s32 %p52, %r5, 0; + @%p52 bra $L__BB9_42; + + setp.gt.u32 %p53, %r2, 1; + add.s32 %r93, %r2, %r2; + add.s32 %r94, %r93, -3; + selp.b32 %r95, 1, %r94, %p53; + mul.wide.u32 %rd65, %r95, 4; + add.s64 %rd66, %rd2, %rd65; + ld.f32 %f108, [%rd66]; + ld.f32 %f109, [%rd2+4]; + add.rn.f32 %f110, %f109, %f108; + mul.rn.f32 %f111, %f110, 0f3F000000; + cvt.rmi.f32.f32 %f112, %f111; + ld.f32 %f113, [%rd2]; + add.rn.f32 %f114, %f113, %f112; + st.f32 [%rd2], %f114; + +$L__BB9_42: + setp.eq.s32 %p54, %r5, 0; + selp.b32 %r107, 3, 2, %p54; + setp.ge.u32 %p55, %r107, %r2; + @%p55 bra $L__BB9_44; + +$L__BB9_43: + add.s32 %r96, %r107, -1; + mul.wide.u32 %rd67, %r96, 4; + add.s64 %rd68, %rd2, %rd67; + add.s32 %r97, %r107, -2; + mul.wide.u32 %rd69, %r97, 4; + add.s64 %rd70, %rd2, %rd69; + mul.wide.u32 %rd71, %r107, 4; + add.s64 %rd72, %rd2, %rd71; + ld.f32 %f115, [%rd72]; + ld.f32 %f116, [%rd70]; + add.rn.f32 %f117, %f116, %f115; + mul.rn.f32 %f118, %f117, 0f3F000000; + cvt.rmi.f32.f32 %f119, %f118; + ld.f32 %f120, [%rd68]; + add.rn.f32 %f121, %f120, %f119; + st.f32 [%rd68], %f121; + add.s32 %r107, %r107, 2; + setp.lt.u32 %p56, %r107, %r2; + @%p56 bra $L__BB9_43; + +$L__BB9_44: + setp.ne.s32 %p58, %r25, %r4; + or.pred %p59, %p48, %p58; + @%p59 bra $L__BB9_48; + + setp.gt.u32 %p60, %r6, 1; + mov.u32 %r98, 2; + sub.s32 %r99, %r98, %r2; + selp.b32 %r100, %r7, %r99, %p60; + mul.wide.u32 %rd73, %r100, 4; + add.s64 %rd74, %rd2, %rd73; + ld.f32 %f122, [%rd4]; + ld.f32 %f123, [%rd74]; + add.rn.f32 %f124, %f123, %f122; + mul.rn.f32 %f125, %f124, 0f3F000000; + cvt.rmi.f32.f32 %f126, %f125; + ld.f32 %f127, [%rd3]; + add.rn.f32 %f128, %f127, %f126; + st.f32 [%rd3], %f128; + +$L__BB9_48: + ret; + +} + // .globl j2k_idwt_vertical +.visible .entry j2k_idwt_vertical( + .param .u64 j2k_idwt_vertical_param_0, + .param .u64 j2k_idwt_vertical_param_1 +) +{ + .reg .pred %p<36>; + .reg .f32 %f<51>; + .reg .b32 %r<208>; + .reg .b64 %rd<49>; + + + ld.param.u64 %rd3, [j2k_idwt_vertical_param_0]; + ld.param.u64 %rd4, [j2k_idwt_vertical_param_1]; + cvta.to.global.u64 %rd1, %rd3; + cvta.to.global.u64 %rd2, %rd4; + ld.global.u32 %r1, [%rd2+4]; + ld.global.u32 %r2, [%rd2+8]; + ld.global.u32 %r3, [%rd2]; + sub.s32 %r4, %r2, %r3; + mov.u32 %r92, %ntid.x; + mov.u32 %r93, %ctaid.x; + mul.lo.s32 %r5, %r93, %r92; + mov.u32 %r6, %tid.x; + add.s32 %r7, %r5, %r6; + setp.ge.u32 %p2, %r7, %r4; + @%p2 bra $L__BB10_28; + + ld.global.u32 %r8, [%rd2+12]; + sub.s32 %r9, %r8, %r1; + setp.eq.s32 %p3, %r9, 1; + and.b32 %r195, %r1, 1; + @%p3 bra $L__BB10_26; + bra.uni $L__BB10_2; + +$L__BB10_26: + setp.eq.s32 %p35, %r195, 0; + @%p35 bra $L__BB10_28; + + mul.wide.u32 %rd47, %r7, 4; + add.s64 %rd48, %rd1, %rd47; + ld.global.f32 %f49, [%rd48]; + mul.rn.f32 %f50, %f49, 0f3F000000; + st.global.f32 [%rd48], %f50; + bra.uni $L__BB10_28; + +$L__BB10_2: + ld.global.u32 %r94, [%rd2+80]; + setp.eq.s32 %p4, %r94, 0; + xor.b32 %r199, %r195, 1; + @%p4 bra $L__BB10_20; + + setp.eq.s32 %p1, %r195, 0; + selp.f32 %f1, 0f3F9D7658, 0f3F5019C3, %p1; + setp.lt.u32 %p5, %r9, 2; + @%p5 bra $L__BB10_6; + + add.s32 %r96, %r6, %r2; + add.s32 %r97, %r96, %r5; + sub.s32 %r183, %r97, %r3; + shl.b32 %r98, %r3, 1; + mov.u32 %r182, 1; + shl.b32 %r99, %r2, 1; + sub.s32 %r13, %r99, %r98; + selp.f32 %f2, 0f3F5019C3, 0f3F9D7658, %p1; + mov.u32 %r181, %r7; + +$L__BB10_5: + mul.wide.u32 %rd5, %r181, 4; + add.s64 %rd6, %rd1, %rd5; + ld.global.f32 %f3, [%rd6]; + mul.rn.f32 %f4, %f1, %f3; + st.global.f32 [%rd6], %f4; + mul.wide.u32 %rd7, %r183, 4; + add.s64 %rd8, %rd1, %rd7; + ld.global.f32 %f5, [%rd8]; + mul.rn.f32 %f6, %f2, %f5; + st.global.f32 [%rd8], %f6; + add.s32 %r183, %r183, %r13; + add.s32 %r181, %r181, %r13; + add.s32 %r182, %r182, 2; + setp.lt.u32 %p6, %r182, %r9; + @%p6 bra $L__BB10_5; + +$L__BB10_6: + and.b32 %r100, %r9, 1; + setp.eq.b32 %p7, %r100, 1; + mov.pred %p8, 0; + xor.pred %p9, %p7, %p8; + not.pred %p10, %p9; + @%p10 bra $L__BB10_8; + + add.s32 %r101, %r9, -1; + mad.lo.s32 %r102, %r101, %r4, %r7; + mul.wide.u32 %rd9, %r102, 4; + add.s64 %rd10, %rd1, %rd9; + ld.global.f32 %f7, [%rd10]; + mul.rn.f32 %f8, %f1, %f7; + st.global.f32 [%rd10], %f8; + +$L__BB10_8: + setp.ge.u32 %p11, %r195, %r9; + @%p11 bra $L__BB10_11; + + shl.b32 %r103, %r8, 1; + add.s32 %r104, %r103, -3; + sub.s32 %r105, %r104, %r195; + shl.b32 %r106, %r1, 1; + sub.s32 %r186, %r105, %r106; + neg.s32 %r185, %r195; + mad.lo.s32 %r184, %r4, %r195, %r7; + shl.b32 %r107, %r3, 1; + shl.b32 %r108, %r2, 1; + sub.s32 %r23, %r108, %r107; + mov.u32 %r187, %r195; + +$L__BB10_10: + add.s32 %r109, %r185, 1; + add.s32 %r110, %r187, -1; + setp.gt.u32 %p12, %r187, 1; + selp.b32 %r111, %r110, %r109, %p12; + add.s32 %r112, %r187, 1; + setp.lt.u32 %p13, %r112, %r9; + selp.b32 %r113, %r112, %r186, %p13; + mad.lo.s32 %r114, %r111, %r4, %r7; + mul.wide.u32 %rd11, %r114, 4; + add.s64 %rd12, %rd1, %rd11; + mad.lo.s32 %r115, %r113, %r4, %r7; + mul.wide.u32 %rd13, %r115, 4; + add.s64 %rd14, %rd1, %rd13; + ld.global.f32 %f9, [%rd14]; + ld.global.f32 %f10, [%rd12]; + add.rn.f32 %f11, %f10, %f9; + mul.wide.u32 %rd15, %r184, 4; + add.s64 %rd16, %rd1, %rd15; + ld.global.f32 %f12, [%rd16]; + mov.f32 %f13, 0fBEE31355; + fma.rn.f32 %f14, %f11, %f13, %f12; + st.global.f32 [%rd16], %f14; + add.s32 %r186, %r186, -2; + add.s32 %r185, %r185, -2; + add.s32 %r184, %r184, %r23; + add.s32 %r187, %r187, 2; + setp.lt.u32 %p14, %r187, %r9; + @%p14 bra $L__BB10_10; + +$L__BB10_11: + setp.ge.u32 %p15, %r199, %r9; + @%p15 bra $L__BB10_14; + + shl.b32 %r116, %r8, 1; + add.s32 %r117, %r116, -3; + sub.s32 %r118, %r117, %r199; + shl.b32 %r119, %r1, 1; + sub.s32 %r190, %r118, %r119; + neg.s32 %r189, %r199; + mad.lo.s32 %r188, %r4, %r199, %r7; + shl.b32 %r120, %r3, 1; + shl.b32 %r121, %r2, 1; + sub.s32 %r35, %r121, %r120; + mov.u32 %r191, %r199; + +$L__BB10_13: + add.s32 %r122, %r189, 1; + add.s32 %r123, %r191, -1; + setp.gt.u32 %p16, %r191, 1; + selp.b32 %r124, %r123, %r122, %p16; + add.s32 %r125, %r191, 1; + setp.lt.u32 %p17, %r125, %r9; + selp.b32 %r126, %r125, %r190, %p17; + mad.lo.s32 %r127, %r124, %r4, %r7; + mul.wide.u32 %rd17, %r127, 4; + add.s64 %rd18, %rd1, %rd17; + mad.lo.s32 %r128, %r126, %r4, %r7; + mul.wide.u32 %rd19, %r128, 4; + add.s64 %rd20, %rd1, %rd19; + ld.global.f32 %f15, [%rd20]; + ld.global.f32 %f16, [%rd18]; + add.rn.f32 %f17, %f16, %f15; + mul.wide.u32 %rd21, %r188, 4; + add.s64 %rd22, %rd1, %rd21; + ld.global.f32 %f18, [%rd22]; + mov.f32 %f19, 0fBF620676; + fma.rn.f32 %f20, %f17, %f19, %f18; + st.global.f32 [%rd22], %f20; + add.s32 %r190, %r190, -2; + add.s32 %r189, %r189, -2; + add.s32 %r188, %r188, %r35; + add.s32 %r191, %r191, 2; + setp.lt.u32 %p18, %r191, %r9; + @%p18 bra $L__BB10_13; + +$L__BB10_14: + @%p11 bra $L__BB10_17; + + shl.b32 %r129, %r8, 1; + add.s32 %r130, %r129, -3; + sub.s32 %r131, %r130, %r195; + shl.b32 %r132, %r1, 1; + sub.s32 %r194, %r131, %r132; + neg.s32 %r193, %r195; + mad.lo.s32 %r192, %r4, %r195, %r7; + shl.b32 %r133, %r3, 1; + shl.b32 %r134, %r2, 1; + sub.s32 %r47, %r134, %r133; + +$L__BB10_16: + add.s32 %r135, %r193, 1; + add.s32 %r136, %r195, -1; + setp.gt.u32 %p20, %r195, 1; + selp.b32 %r137, %r136, %r135, %p20; + add.s32 %r138, %r195, 1; + setp.lt.u32 %p21, %r138, %r9; + selp.b32 %r139, %r138, %r194, %p21; + mad.lo.s32 %r140, %r137, %r4, %r7; + mul.wide.u32 %rd23, %r140, 4; + add.s64 %rd24, %rd1, %rd23; + mad.lo.s32 %r141, %r139, %r4, %r7; + mul.wide.u32 %rd25, %r141, 4; + add.s64 %rd26, %rd1, %rd25; + ld.global.f32 %f21, [%rd26]; + ld.global.f32 %f22, [%rd24]; + add.rn.f32 %f23, %f22, %f21; + mul.wide.u32 %rd27, %r192, 4; + add.s64 %rd28, %rd1, %rd27; + ld.global.f32 %f24, [%rd28]; + mov.f32 %f25, 0f3D5901AE; + fma.rn.f32 %f26, %f23, %f25, %f24; + st.global.f32 [%rd28], %f26; + add.s32 %r194, %r194, -2; + add.s32 %r193, %r193, -2; + add.s32 %r192, %r192, %r47; + add.s32 %r195, %r195, 2; + setp.lt.u32 %p22, %r195, %r9; + @%p22 bra $L__BB10_16; + +$L__BB10_17: + @%p15 bra $L__BB10_28; + + shl.b32 %r142, %r8, 1; + add.s32 %r143, %r142, -3; + sub.s32 %r144, %r143, %r199; + shl.b32 %r145, %r1, 1; + sub.s32 %r198, %r144, %r145; + neg.s32 %r197, %r199; + mad.lo.s32 %r196, %r4, %r199, %r7; + shl.b32 %r146, %r3, 1; + shl.b32 %r147, %r2, 1; + sub.s32 %r59, %r147, %r146; + +$L__BB10_19: + add.s32 %r148, %r197, 1; + add.s32 %r149, %r199, -1; + setp.gt.u32 %p24, %r199, 1; + selp.b32 %r150, %r149, %r148, %p24; + add.s32 %r151, %r199, 1; + setp.lt.u32 %p25, %r151, %r9; + selp.b32 %r152, %r151, %r198, %p25; + mad.lo.s32 %r153, %r150, %r4, %r7; + mul.wide.u32 %rd29, %r153, 4; + add.s64 %rd30, %rd1, %rd29; + mad.lo.s32 %r154, %r152, %r4, %r7; + mul.wide.u32 %rd31, %r154, 4; + add.s64 %rd32, %rd1, %rd31; + ld.global.f32 %f27, [%rd32]; + ld.global.f32 %f28, [%rd30]; + add.rn.f32 %f29, %f28, %f27; + mul.wide.u32 %rd33, %r196, 4; + add.s64 %rd34, %rd1, %rd33; + ld.global.f32 %f30, [%rd34]; + mov.f32 %f31, 0f3FCB0673; + fma.rn.f32 %f32, %f29, %f31, %f30; + st.global.f32 [%rd34], %f32; + add.s32 %r198, %r198, -2; + add.s32 %r197, %r197, -2; + add.s32 %r196, %r196, %r59; + add.s32 %r199, %r199, 2; + setp.lt.u32 %p26, %r199, %r9; + @%p26 bra $L__BB10_19; + bra.uni $L__BB10_28; + +$L__BB10_20: + setp.ge.u32 %p27, %r195, %r9; + @%p27 bra $L__BB10_23; + + shl.b32 %r155, %r8, 1; + add.s32 %r156, %r155, -3; + sub.s32 %r157, %r156, %r195; + shl.b32 %r158, %r1, 1; + sub.s32 %r202, %r157, %r158; + neg.s32 %r201, %r195; + mad.lo.s32 %r200, %r4, %r195, %r7; + shl.b32 %r159, %r3, 1; + shl.b32 %r160, %r2, 1; + sub.s32 %r71, %r160, %r159; + +$L__BB10_22: + add.s32 %r161, %r201, 1; + add.s32 %r162, %r195, -1; + setp.gt.u32 %p28, %r195, 1; + selp.b32 %r163, %r162, %r161, %p28; + add.s32 %r164, %r195, 1; + setp.lt.u32 %p29, %r164, %r9; + selp.b32 %r165, %r164, %r202, %p29; + mad.lo.s32 %r166, %r163, %r4, %r7; + mul.wide.u32 %rd35, %r166, 4; + add.s64 %rd36, %rd1, %rd35; + mad.lo.s32 %r167, %r165, %r4, %r7; + mul.wide.u32 %rd37, %r167, 4; + add.s64 %rd38, %rd1, %rd37; + mul.wide.u32 %rd39, %r200, 4; + add.s64 %rd40, %rd1, %rd39; + ld.global.f32 %f33, [%rd38]; + ld.global.f32 %f34, [%rd36]; + add.rn.f32 %f35, %f34, %f33; + mov.f32 %f36, 0f3F000000; + mov.f32 %f37, 0f3E800000; + fma.rn.f32 %f38, %f35, %f37, %f36; + cvt.rmi.f32.f32 %f39, %f38; + ld.global.f32 %f40, [%rd40]; + sub.rn.f32 %f41, %f40, %f39; + st.global.f32 [%rd40], %f41; + add.s32 %r202, %r202, -2; + add.s32 %r201, %r201, -2; + add.s32 %r200, %r200, %r71; + add.s32 %r195, %r195, 2; + setp.lt.u32 %p30, %r195, %r9; + @%p30 bra $L__BB10_22; + +$L__BB10_23: + setp.ge.u32 %p31, %r199, %r9; + @%p31 bra $L__BB10_28; + + shl.b32 %r168, %r8, 1; + add.s32 %r169, %r168, -3; + sub.s32 %r170, %r169, %r199; + shl.b32 %r171, %r1, 1; + sub.s32 %r206, %r170, %r171; + neg.s32 %r205, %r199; + mad.lo.s32 %r204, %r4, %r199, %r7; + shl.b32 %r172, %r3, 1; + shl.b32 %r173, %r2, 1; + sub.s32 %r83, %r173, %r172; + +$L__BB10_25: + add.s32 %r174, %r205, 1; + add.s32 %r175, %r199, -1; + setp.gt.u32 %p32, %r199, 1; + selp.b32 %r176, %r175, %r174, %p32; + add.s32 %r177, %r199, 1; + setp.lt.u32 %p33, %r177, %r9; + selp.b32 %r178, %r177, %r206, %p33; + mad.lo.s32 %r179, %r176, %r4, %r7; + mul.wide.u32 %rd41, %r179, 4; + add.s64 %rd42, %rd1, %rd41; + mad.lo.s32 %r180, %r178, %r4, %r7; + mul.wide.u32 %rd43, %r180, 4; + add.s64 %rd44, %rd1, %rd43; + mul.wide.u32 %rd45, %r204, 4; + add.s64 %rd46, %rd1, %rd45; + ld.global.f32 %f42, [%rd44]; + ld.global.f32 %f43, [%rd42]; + add.rn.f32 %f44, %f43, %f42; + mul.rn.f32 %f45, %f44, 0f3F000000; + cvt.rmi.f32.f32 %f46, %f45; + ld.global.f32 %f47, [%rd46]; + add.rn.f32 %f48, %f47, %f46; + st.global.f32 [%rd46], %f48; + add.s32 %r206, %r206, -2; + add.s32 %r205, %r205, -2; + add.s32 %r204, %r204, %r83; + add.s32 %r199, %r199, 2; + setp.lt.u32 %p34, %r199, %r9; + @%p34 bra $L__BB10_25; + +$L__BB10_28: + ret; + +} + // .globl j2k_idwt_vertical_53 +.visible .entry j2k_idwt_vertical_53( + .param .u64 j2k_idwt_vertical_53_param_0, + .param .u64 j2k_idwt_vertical_53_param_1 +) +{ + .reg .pred %p<12>; + .reg .f32 %f<19>; + .reg .b32 %r<71>; + .reg .b64 %rd<19>; + + + ld.param.u64 %rd3, [j2k_idwt_vertical_53_param_0]; + ld.param.u64 %rd4, [j2k_idwt_vertical_53_param_1]; + cvta.to.global.u64 %rd1, %rd3; + cvta.to.global.u64 %rd2, %rd4; + ld.global.u32 %r1, [%rd2+4]; + ld.global.u32 %r2, [%rd2+8]; + ld.global.u32 %r3, [%rd2]; + sub.s32 %r4, %r2, %r3; + mov.u32 %r34, %ntid.x; + mov.u32 %r35, %ctaid.x; + mov.u32 %r36, %tid.x; + mad.lo.s32 %r5, %r35, %r34, %r36; + setp.ge.u32 %p1, %r5, %r4; + @%p1 bra $L__BB11_10; + + ld.global.u32 %r6, [%rd2+12]; + sub.s32 %r7, %r6, %r1; + setp.eq.s32 %p2, %r7, 1; + and.b32 %r66, %r1, 1; + @%p2 bra $L__BB11_8; + bra.uni $L__BB11_2; + +$L__BB11_8: + setp.eq.s32 %p11, %r66, 0; + @%p11 bra $L__BB11_10; + + mul.wide.u32 %rd17, %r5, 4; + add.s64 %rd18, %rd1, %rd17; + ld.global.f32 %f17, [%rd18]; + mul.rn.f32 %f18, %f17, 0f3F000000; + st.global.f32 [%rd18], %f18; + bra.uni $L__BB11_10; + +$L__BB11_2: + xor.b32 %r70, %r66, 1; + setp.ge.u32 %p3, %r66, %r7; + @%p3 bra $L__BB11_5; + + shl.b32 %r37, %r6, 1; + add.s32 %r38, %r37, -3; + sub.s32 %r39, %r38, %r66; + shl.b32 %r40, %r1, 1; + sub.s32 %r65, %r39, %r40; + neg.s32 %r64, %r66; + mad.lo.s32 %r63, %r4, %r66, %r5; + shl.b32 %r41, %r3, 1; + shl.b32 %r42, %r2, 1; + sub.s32 %r13, %r42, %r41; + +$L__BB11_4: + add.s32 %r43, %r64, 1; + add.s32 %r44, %r66, -1; + setp.gt.u32 %p4, %r66, 1; + selp.b32 %r45, %r44, %r43, %p4; + add.s32 %r46, %r66, 1; + setp.lt.u32 %p5, %r46, %r7; + selp.b32 %r47, %r46, %r65, %p5; + mad.lo.s32 %r48, %r45, %r4, %r5; + mul.wide.u32 %rd5, %r48, 4; + add.s64 %rd6, %rd1, %rd5; + mad.lo.s32 %r49, %r47, %r4, %r5; + mul.wide.u32 %rd7, %r49, 4; + add.s64 %rd8, %rd1, %rd7; + mul.wide.u32 %rd9, %r63, 4; + add.s64 %rd10, %rd1, %rd9; + ld.global.f32 %f1, [%rd8]; + ld.global.f32 %f2, [%rd6]; + add.rn.f32 %f3, %f2, %f1; + mov.f32 %f4, 0f3F000000; + mov.f32 %f5, 0f3E800000; + fma.rn.f32 %f6, %f3, %f5, %f4; + cvt.rmi.f32.f32 %f7, %f6; + ld.global.f32 %f8, [%rd10]; + sub.rn.f32 %f9, %f8, %f7; + st.global.f32 [%rd10], %f9; + add.s32 %r65, %r65, -2; + add.s32 %r64, %r64, -2; + add.s32 %r63, %r63, %r13; + add.s32 %r66, %r66, 2; + setp.lt.u32 %p6, %r66, %r7; + @%p6 bra $L__BB11_4; + +$L__BB11_5: + setp.ge.u32 %p7, %r70, %r7; + @%p7 bra $L__BB11_10; + + shl.b32 %r50, %r6, 1; + add.s32 %r51, %r50, -3; + sub.s32 %r52, %r51, %r70; + shl.b32 %r53, %r1, 1; + sub.s32 %r69, %r52, %r53; + neg.s32 %r68, %r70; + mad.lo.s32 %r67, %r4, %r70, %r5; + shl.b32 %r54, %r3, 1; + shl.b32 %r55, %r2, 1; + sub.s32 %r25, %r55, %r54; + +$L__BB11_7: + add.s32 %r56, %r68, 1; + add.s32 %r57, %r70, -1; + setp.gt.u32 %p8, %r70, 1; + selp.b32 %r58, %r57, %r56, %p8; + add.s32 %r59, %r70, 1; + setp.lt.u32 %p9, %r59, %r7; + selp.b32 %r60, %r59, %r69, %p9; + mad.lo.s32 %r61, %r58, %r4, %r5; + mul.wide.u32 %rd11, %r61, 4; + add.s64 %rd12, %rd1, %rd11; + mad.lo.s32 %r62, %r60, %r4, %r5; + mul.wide.u32 %rd13, %r62, 4; + add.s64 %rd14, %rd1, %rd13; + mul.wide.u32 %rd15, %r67, 4; + add.s64 %rd16, %rd1, %rd15; + ld.global.f32 %f10, [%rd14]; + ld.global.f32 %f11, [%rd12]; + add.rn.f32 %f12, %f11, %f10; + mul.rn.f32 %f13, %f12, 0f3F000000; + cvt.rmi.f32.f32 %f14, %f13; + ld.global.f32 %f15, [%rd16]; + add.rn.f32 %f16, %f15, %f14; + st.global.f32 [%rd16], %f16; + add.s32 %r69, %r69, -2; + add.s32 %r68, %r68, -2; + add.s32 %r67, %r67, %r25; + add.s32 %r70, %r70, 2; + setp.lt.u32 %p10, %r70, %r7; + @%p10 bra $L__BB11_7; + +$L__BB11_10: + ret; + +} + // .globl j2k_idwt_vertical_97 +.visible .entry j2k_idwt_vertical_97( + .param .u64 j2k_idwt_vertical_97_param_0, + .param .u64 j2k_idwt_vertical_97_param_1 +) +{ + .reg .pred %p<27>; + .reg .f32 %f<35>; + .reg .b32 %r<149>; + .reg .b64 %rd<37>; + + + ld.param.u64 %rd3, [j2k_idwt_vertical_97_param_0]; + ld.param.u64 %rd4, [j2k_idwt_vertical_97_param_1]; + cvta.to.global.u64 %rd1, %rd3; + cvta.to.global.u64 %rd2, %rd4; + ld.global.u32 %r1, [%rd2+4]; + ld.global.u32 %r2, [%rd2+8]; + ld.global.u32 %r3, [%rd2]; + sub.s32 %r4, %r2, %r3; + mov.u32 %r68, %ntid.x; + mov.u32 %r69, %ctaid.x; + mul.lo.s32 %r5, %r69, %r68; + mov.u32 %r6, %tid.x; + add.s32 %r7, %r5, %r6; + setp.ge.u32 %p2, %r7, %r4; + @%p2 bra $L__BB12_21; + + ld.global.u32 %r8, [%rd2+12]; + sub.s32 %r9, %r8, %r1; + setp.eq.s32 %p3, %r9, 1; + and.b32 %r144, %r1, 1; + @%p3 bra $L__BB12_19; + bra.uni $L__BB12_2; + +$L__BB12_19: + setp.eq.s32 %p26, %r144, 0; + @%p26 bra $L__BB12_21; + + mul.wide.u32 %rd35, %r7, 4; + add.s64 %rd36, %rd1, %rd35; + ld.global.f32 %f33, [%rd36]; + mul.rn.f32 %f34, %f33, 0f3F000000; + st.global.f32 [%rd36], %f34; + bra.uni $L__BB12_21; + +$L__BB12_2: + xor.b32 %r148, %r144, 1; + setp.eq.s32 %p1, %r144, 0; + selp.f32 %f1, 0f3F9D7658, 0f3F5019C3, %p1; + setp.eq.s32 %p4, %r9, 0; + @%p4 bra $L__BB12_5; + + add.s32 %r71, %r6, %r2; + add.s32 %r72, %r71, %r5; + sub.s32 %r132, %r72, %r3; + shl.b32 %r73, %r3, 1; + mov.u32 %r131, 1; + shl.b32 %r74, %r2, 1; + sub.s32 %r13, %r74, %r73; + selp.f32 %f2, 0f3F5019C3, 0f3F9D7658, %p1; + mov.u32 %r130, %r7; + +$L__BB12_4: + mul.wide.u32 %rd5, %r130, 4; + add.s64 %rd6, %rd1, %rd5; + ld.global.f32 %f3, [%rd6]; + mul.rn.f32 %f4, %f1, %f3; + st.global.f32 [%rd6], %f4; + mul.wide.u32 %rd7, %r132, 4; + add.s64 %rd8, %rd1, %rd7; + ld.global.f32 %f5, [%rd8]; + mul.rn.f32 %f6, %f2, %f5; + st.global.f32 [%rd8], %f6; + add.s32 %r132, %r132, %r13; + add.s32 %r130, %r130, %r13; + add.s32 %r131, %r131, 2; + setp.lt.u32 %p5, %r131, %r9; + @%p5 bra $L__BB12_4; + +$L__BB12_5: + and.b32 %r75, %r9, 1; + setp.eq.b32 %p6, %r75, 1; + mov.pred %p7, 0; + xor.pred %p8, %p6, %p7; + not.pred %p9, %p8; + @%p9 bra $L__BB12_7; + + add.s32 %r76, %r9, -1; + mad.lo.s32 %r77, %r76, %r4, %r7; + mul.wide.u32 %rd9, %r77, 4; + add.s64 %rd10, %rd1, %rd9; + ld.global.f32 %f7, [%rd10]; + mul.rn.f32 %f8, %f1, %f7; + st.global.f32 [%rd10], %f8; + +$L__BB12_7: + setp.ge.u32 %p10, %r144, %r9; + @%p10 bra $L__BB12_10; + + shl.b32 %r78, %r8, 1; + add.s32 %r79, %r78, -3; + sub.s32 %r80, %r79, %r144; + shl.b32 %r81, %r1, 1; + sub.s32 %r135, %r80, %r81; + neg.s32 %r134, %r144; + mad.lo.s32 %r133, %r4, %r144, %r7; + shl.b32 %r82, %r3, 1; + shl.b32 %r83, %r2, 1; + sub.s32 %r23, %r83, %r82; + mov.u32 %r136, %r144; + +$L__BB12_9: + add.s32 %r84, %r134, 1; + add.s32 %r85, %r136, -1; + setp.gt.u32 %p11, %r136, 1; + selp.b32 %r86, %r85, %r84, %p11; + add.s32 %r87, %r136, 1; + setp.lt.u32 %p12, %r87, %r9; + selp.b32 %r88, %r87, %r135, %p12; + mad.lo.s32 %r89, %r86, %r4, %r7; + mul.wide.u32 %rd11, %r89, 4; + add.s64 %rd12, %rd1, %rd11; + mad.lo.s32 %r90, %r88, %r4, %r7; + mul.wide.u32 %rd13, %r90, 4; + add.s64 %rd14, %rd1, %rd13; + ld.global.f32 %f9, [%rd14]; + ld.global.f32 %f10, [%rd12]; + add.rn.f32 %f11, %f10, %f9; + mul.wide.u32 %rd15, %r133, 4; + add.s64 %rd16, %rd1, %rd15; + ld.global.f32 %f12, [%rd16]; + mov.f32 %f13, 0fBEE31355; + fma.rn.f32 %f14, %f11, %f13, %f12; + st.global.f32 [%rd16], %f14; + add.s32 %r135, %r135, -2; + add.s32 %r134, %r134, -2; + add.s32 %r133, %r133, %r23; + add.s32 %r136, %r136, 2; + setp.lt.u32 %p13, %r136, %r9; + @%p13 bra $L__BB12_9; + +$L__BB12_10: + setp.ge.u32 %p14, %r148, %r9; + @%p14 bra $L__BB12_13; + + shl.b32 %r91, %r8, 1; + add.s32 %r92, %r91, -3; + sub.s32 %r93, %r92, %r148; + shl.b32 %r94, %r1, 1; + sub.s32 %r139, %r93, %r94; + neg.s32 %r138, %r148; + mad.lo.s32 %r137, %r4, %r148, %r7; + shl.b32 %r95, %r3, 1; + shl.b32 %r96, %r2, 1; + sub.s32 %r35, %r96, %r95; + mov.u32 %r140, %r148; + +$L__BB12_12: + add.s32 %r97, %r138, 1; + add.s32 %r98, %r140, -1; + setp.gt.u32 %p15, %r140, 1; + selp.b32 %r99, %r98, %r97, %p15; + add.s32 %r100, %r140, 1; + setp.lt.u32 %p16, %r100, %r9; + selp.b32 %r101, %r100, %r139, %p16; + mad.lo.s32 %r102, %r99, %r4, %r7; + mul.wide.u32 %rd17, %r102, 4; + add.s64 %rd18, %rd1, %rd17; + mad.lo.s32 %r103, %r101, %r4, %r7; + mul.wide.u32 %rd19, %r103, 4; + add.s64 %rd20, %rd1, %rd19; + ld.global.f32 %f15, [%rd20]; + ld.global.f32 %f16, [%rd18]; + add.rn.f32 %f17, %f16, %f15; + mul.wide.u32 %rd21, %r137, 4; + add.s64 %rd22, %rd1, %rd21; + ld.global.f32 %f18, [%rd22]; + mov.f32 %f19, 0fBF620676; + fma.rn.f32 %f20, %f17, %f19, %f18; + st.global.f32 [%rd22], %f20; + add.s32 %r139, %r139, -2; + add.s32 %r138, %r138, -2; + add.s32 %r137, %r137, %r35; + add.s32 %r140, %r140, 2; + setp.lt.u32 %p17, %r140, %r9; + @%p17 bra $L__BB12_12; + +$L__BB12_13: + @%p10 bra $L__BB12_16; + + shl.b32 %r104, %r8, 1; + add.s32 %r105, %r104, -3; + sub.s32 %r106, %r105, %r144; + shl.b32 %r107, %r1, 1; + sub.s32 %r143, %r106, %r107; + neg.s32 %r142, %r144; + mad.lo.s32 %r141, %r4, %r144, %r7; + shl.b32 %r108, %r3, 1; + shl.b32 %r109, %r2, 1; + sub.s32 %r47, %r109, %r108; + +$L__BB12_15: + add.s32 %r110, %r142, 1; + add.s32 %r111, %r144, -1; + setp.gt.u32 %p19, %r144, 1; + selp.b32 %r112, %r111, %r110, %p19; + add.s32 %r113, %r144, 1; + setp.lt.u32 %p20, %r113, %r9; + selp.b32 %r114, %r113, %r143, %p20; + mad.lo.s32 %r115, %r112, %r4, %r7; + mul.wide.u32 %rd23, %r115, 4; + add.s64 %rd24, %rd1, %rd23; + mad.lo.s32 %r116, %r114, %r4, %r7; + mul.wide.u32 %rd25, %r116, 4; + add.s64 %rd26, %rd1, %rd25; + ld.global.f32 %f21, [%rd26]; + ld.global.f32 %f22, [%rd24]; + add.rn.f32 %f23, %f22, %f21; + mul.wide.u32 %rd27, %r141, 4; + add.s64 %rd28, %rd1, %rd27; + ld.global.f32 %f24, [%rd28]; + mov.f32 %f25, 0f3D5901AE; + fma.rn.f32 %f26, %f23, %f25, %f24; + st.global.f32 [%rd28], %f26; + add.s32 %r143, %r143, -2; + add.s32 %r142, %r142, -2; + add.s32 %r141, %r141, %r47; + add.s32 %r144, %r144, 2; + setp.lt.u32 %p21, %r144, %r9; + @%p21 bra $L__BB12_15; + +$L__BB12_16: + @%p14 bra $L__BB12_21; + + shl.b32 %r117, %r8, 1; + add.s32 %r118, %r117, -3; + sub.s32 %r119, %r118, %r148; + shl.b32 %r120, %r1, 1; + sub.s32 %r147, %r119, %r120; + neg.s32 %r146, %r148; + mad.lo.s32 %r145, %r4, %r148, %r7; + shl.b32 %r121, %r3, 1; + shl.b32 %r122, %r2, 1; + sub.s32 %r59, %r122, %r121; + +$L__BB12_18: + add.s32 %r123, %r146, 1; + add.s32 %r124, %r148, -1; + setp.gt.u32 %p23, %r148, 1; + selp.b32 %r125, %r124, %r123, %p23; + add.s32 %r126, %r148, 1; + setp.lt.u32 %p24, %r126, %r9; + selp.b32 %r127, %r126, %r147, %p24; + mad.lo.s32 %r128, %r125, %r4, %r7; + mul.wide.u32 %rd29, %r128, 4; + add.s64 %rd30, %rd1, %rd29; + mad.lo.s32 %r129, %r127, %r4, %r7; + mul.wide.u32 %rd31, %r129, 4; + add.s64 %rd32, %rd1, %rd31; + ld.global.f32 %f27, [%rd32]; + ld.global.f32 %f28, [%rd30]; + add.rn.f32 %f29, %f28, %f27; + mul.wide.u32 %rd33, %r145, 4; + add.s64 %rd34, %rd1, %rd33; + ld.global.f32 %f30, [%rd34]; + mov.f32 %f31, 0f3FCB0673; + fma.rn.f32 %f32, %f29, %f31, %f30; + st.global.f32 [%rd34], %f32; + add.s32 %r147, %r147, -2; + add.s32 %r146, %r146, -2; + add.s32 %r145, %r145, %r59; + add.s32 %r148, %r148, 2; + setp.lt.u32 %p25, %r148, %r9; + @%p25 bra $L__BB12_18; + +$L__BB12_21: + ret; + +} + // .globl j2k_idwt_vertical_multi +.visible .entry j2k_idwt_vertical_multi( + .param .u64 j2k_idwt_vertical_multi_param_0 +) +{ + .reg .pred %p<36>; + .reg .f32 %f<51>; + .reg .b32 %r<211>; + .reg .b64 %rd<51>; + + + ld.param.u64 %rd3, [j2k_idwt_vertical_multi_param_0]; + cvta.to.global.u64 %rd4, %rd3; + mov.u32 %r92, %ctaid.y; + mul.wide.u32 %rd5, %r92, 128; + add.s64 %rd6, %rd4, %rd5; + add.s64 %rd1, %rd6, 120; + ld.global.v2.u32 {%r93, %r94}, [%rd6+40]; + ld.global.u32 %r3, [%rd6+48]; + sub.s32 %r4, %r3, %r93; + mov.u32 %r95, %ntid.x; + mov.u32 %r96, %ctaid.x; + mul.lo.s32 %r5, %r96, %r95; + mov.u32 %r6, %tid.x; + add.s32 %r7, %r5, %r6; + setp.ge.u32 %p2, %r7, %r4; + @%p2 bra $L__BB13_28; + + ld.global.u64 %rd2, [%rd1+-88]; + ld.global.u32 %r8, [%rd1+-68]; + sub.s32 %r9, %r8, %r94; + setp.eq.s32 %p3, %r9, 1; + and.b32 %r198, %r94, 1; + @%p3 bra $L__BB13_26; + bra.uni $L__BB13_2; + +$L__BB13_26: + setp.eq.s32 %p35, %r198, 0; + @%p35 bra $L__BB13_28; + + mul.wide.u32 %rd49, %r7, 4; + add.s64 %rd50, %rd2, %rd49; + ld.f32 %f49, [%rd50]; + mul.rn.f32 %f50, %f49, 0f3F000000; + st.f32 [%rd50], %f50; + bra.uni $L__BB13_28; + +$L__BB13_2: + ld.global.u32 %r97, [%rd1]; + setp.eq.s32 %p4, %r97, 0; + xor.b32 %r202, %r198, 1; + @%p4 bra $L__BB13_20; + + setp.eq.s32 %p1, %r198, 0; + selp.f32 %f1, 0f3F9D7658, 0f3F5019C3, %p1; + setp.lt.u32 %p5, %r9, 2; + @%p5 bra $L__BB13_6; + + add.s32 %r99, %r6, %r3; + add.s32 %r100, %r99, %r5; + sub.s32 %r186, %r100, %r93; + shl.b32 %r101, %r93, 1; + mov.u32 %r185, 1; + shl.b32 %r102, %r3, 1; + sub.s32 %r13, %r102, %r101; + selp.f32 %f2, 0f3F5019C3, 0f3F9D7658, %p1; + mov.u32 %r184, %r7; + +$L__BB13_5: + mul.wide.u32 %rd7, %r184, 4; + add.s64 %rd8, %rd2, %rd7; + ld.f32 %f3, [%rd8]; + mul.rn.f32 %f4, %f1, %f3; + st.f32 [%rd8], %f4; + mul.wide.u32 %rd9, %r186, 4; + add.s64 %rd10, %rd2, %rd9; + ld.f32 %f5, [%rd10]; + mul.rn.f32 %f6, %f2, %f5; + st.f32 [%rd10], %f6; + add.s32 %r186, %r186, %r13; + add.s32 %r184, %r184, %r13; + add.s32 %r185, %r185, 2; + setp.lt.u32 %p6, %r185, %r9; + @%p6 bra $L__BB13_5; + +$L__BB13_6: + and.b32 %r103, %r9, 1; + setp.eq.b32 %p7, %r103, 1; + mov.pred %p8, 0; + xor.pred %p9, %p7, %p8; + not.pred %p10, %p9; + @%p10 bra $L__BB13_8; + + add.s32 %r104, %r9, -1; + mad.lo.s32 %r105, %r104, %r4, %r7; + mul.wide.u32 %rd11, %r105, 4; + add.s64 %rd12, %rd2, %rd11; + ld.f32 %f7, [%rd12]; + mul.rn.f32 %f8, %f1, %f7; + st.f32 [%rd12], %f8; + +$L__BB13_8: + setp.ge.u32 %p11, %r198, %r9; + @%p11 bra $L__BB13_11; + + shl.b32 %r106, %r8, 1; + add.s32 %r107, %r106, -3; + sub.s32 %r108, %r107, %r198; + shl.b32 %r109, %r94, 1; + sub.s32 %r189, %r108, %r109; + neg.s32 %r188, %r198; + mad.lo.s32 %r187, %r4, %r198, %r7; + shl.b32 %r110, %r93, 1; + shl.b32 %r111, %r3, 1; + sub.s32 %r23, %r111, %r110; + mov.u32 %r190, %r198; + +$L__BB13_10: + add.s32 %r112, %r188, 1; + add.s32 %r113, %r190, -1; + setp.gt.u32 %p12, %r190, 1; + selp.b32 %r114, %r113, %r112, %p12; + add.s32 %r115, %r190, 1; + setp.lt.u32 %p13, %r115, %r9; + selp.b32 %r116, %r115, %r189, %p13; + mad.lo.s32 %r117, %r114, %r4, %r7; + mul.wide.u32 %rd13, %r117, 4; + add.s64 %rd14, %rd2, %rd13; + mad.lo.s32 %r118, %r116, %r4, %r7; + mul.wide.u32 %rd15, %r118, 4; + add.s64 %rd16, %rd2, %rd15; + ld.f32 %f9, [%rd16]; + ld.f32 %f10, [%rd14]; + add.rn.f32 %f11, %f10, %f9; + mul.wide.u32 %rd17, %r187, 4; + add.s64 %rd18, %rd2, %rd17; + ld.f32 %f12, [%rd18]; + mov.f32 %f13, 0fBEE31355; + fma.rn.f32 %f14, %f11, %f13, %f12; + st.f32 [%rd18], %f14; + add.s32 %r189, %r189, -2; + add.s32 %r188, %r188, -2; + add.s32 %r187, %r187, %r23; + add.s32 %r190, %r190, 2; + setp.lt.u32 %p14, %r190, %r9; + @%p14 bra $L__BB13_10; + +$L__BB13_11: + setp.ge.u32 %p15, %r202, %r9; + @%p15 bra $L__BB13_14; + + shl.b32 %r119, %r8, 1; + add.s32 %r120, %r119, -3; + sub.s32 %r121, %r120, %r202; + shl.b32 %r122, %r94, 1; + sub.s32 %r193, %r121, %r122; + neg.s32 %r192, %r202; + mad.lo.s32 %r191, %r4, %r202, %r7; + shl.b32 %r123, %r93, 1; + shl.b32 %r124, %r3, 1; + sub.s32 %r35, %r124, %r123; + mov.u32 %r194, %r202; + +$L__BB13_13: + add.s32 %r125, %r192, 1; + add.s32 %r126, %r194, -1; + setp.gt.u32 %p16, %r194, 1; + selp.b32 %r127, %r126, %r125, %p16; + add.s32 %r128, %r194, 1; + setp.lt.u32 %p17, %r128, %r9; + selp.b32 %r129, %r128, %r193, %p17; + mad.lo.s32 %r130, %r127, %r4, %r7; + mul.wide.u32 %rd19, %r130, 4; + add.s64 %rd20, %rd2, %rd19; + mad.lo.s32 %r131, %r129, %r4, %r7; + mul.wide.u32 %rd21, %r131, 4; + add.s64 %rd22, %rd2, %rd21; + ld.f32 %f15, [%rd22]; + ld.f32 %f16, [%rd20]; + add.rn.f32 %f17, %f16, %f15; + mul.wide.u32 %rd23, %r191, 4; + add.s64 %rd24, %rd2, %rd23; + ld.f32 %f18, [%rd24]; + mov.f32 %f19, 0fBF620676; + fma.rn.f32 %f20, %f17, %f19, %f18; + st.f32 [%rd24], %f20; + add.s32 %r193, %r193, -2; + add.s32 %r192, %r192, -2; + add.s32 %r191, %r191, %r35; + add.s32 %r194, %r194, 2; + setp.lt.u32 %p18, %r194, %r9; + @%p18 bra $L__BB13_13; + +$L__BB13_14: + @%p11 bra $L__BB13_17; + + shl.b32 %r132, %r8, 1; + add.s32 %r133, %r132, -3; + sub.s32 %r134, %r133, %r198; + shl.b32 %r135, %r94, 1; + sub.s32 %r197, %r134, %r135; + neg.s32 %r196, %r198; + mad.lo.s32 %r195, %r4, %r198, %r7; + shl.b32 %r136, %r93, 1; + shl.b32 %r137, %r3, 1; + sub.s32 %r47, %r137, %r136; + +$L__BB13_16: + add.s32 %r138, %r196, 1; + add.s32 %r139, %r198, -1; + setp.gt.u32 %p20, %r198, 1; + selp.b32 %r140, %r139, %r138, %p20; + add.s32 %r141, %r198, 1; + setp.lt.u32 %p21, %r141, %r9; + selp.b32 %r142, %r141, %r197, %p21; + mad.lo.s32 %r143, %r140, %r4, %r7; + mul.wide.u32 %rd25, %r143, 4; + add.s64 %rd26, %rd2, %rd25; + mad.lo.s32 %r144, %r142, %r4, %r7; + mul.wide.u32 %rd27, %r144, 4; + add.s64 %rd28, %rd2, %rd27; + ld.f32 %f21, [%rd28]; + ld.f32 %f22, [%rd26]; + add.rn.f32 %f23, %f22, %f21; + mul.wide.u32 %rd29, %r195, 4; + add.s64 %rd30, %rd2, %rd29; + ld.f32 %f24, [%rd30]; + mov.f32 %f25, 0f3D5901AE; + fma.rn.f32 %f26, %f23, %f25, %f24; + st.f32 [%rd30], %f26; + add.s32 %r197, %r197, -2; + add.s32 %r196, %r196, -2; + add.s32 %r195, %r195, %r47; + add.s32 %r198, %r198, 2; + setp.lt.u32 %p22, %r198, %r9; + @%p22 bra $L__BB13_16; + +$L__BB13_17: + @%p15 bra $L__BB13_28; + + shl.b32 %r145, %r8, 1; + add.s32 %r146, %r145, -3; + sub.s32 %r147, %r146, %r202; + shl.b32 %r148, %r94, 1; + sub.s32 %r201, %r147, %r148; + neg.s32 %r200, %r202; + mad.lo.s32 %r199, %r4, %r202, %r7; + shl.b32 %r149, %r93, 1; + shl.b32 %r150, %r3, 1; + sub.s32 %r59, %r150, %r149; + +$L__BB13_19: + add.s32 %r151, %r200, 1; + add.s32 %r152, %r202, -1; + setp.gt.u32 %p24, %r202, 1; + selp.b32 %r153, %r152, %r151, %p24; + add.s32 %r154, %r202, 1; + setp.lt.u32 %p25, %r154, %r9; + selp.b32 %r155, %r154, %r201, %p25; + mad.lo.s32 %r156, %r153, %r4, %r7; + mul.wide.u32 %rd31, %r156, 4; + add.s64 %rd32, %rd2, %rd31; + mad.lo.s32 %r157, %r155, %r4, %r7; + mul.wide.u32 %rd33, %r157, 4; + add.s64 %rd34, %rd2, %rd33; + ld.f32 %f27, [%rd34]; + ld.f32 %f28, [%rd32]; + add.rn.f32 %f29, %f28, %f27; + mul.wide.u32 %rd35, %r199, 4; + add.s64 %rd36, %rd2, %rd35; + ld.f32 %f30, [%rd36]; + mov.f32 %f31, 0f3FCB0673; + fma.rn.f32 %f32, %f29, %f31, %f30; + st.f32 [%rd36], %f32; + add.s32 %r201, %r201, -2; + add.s32 %r200, %r200, -2; + add.s32 %r199, %r199, %r59; + add.s32 %r202, %r202, 2; + setp.lt.u32 %p26, %r202, %r9; + @%p26 bra $L__BB13_19; + bra.uni $L__BB13_28; + +$L__BB13_20: + setp.ge.u32 %p27, %r198, %r9; + @%p27 bra $L__BB13_23; + + shl.b32 %r158, %r8, 1; + add.s32 %r159, %r158, -3; + sub.s32 %r160, %r159, %r198; + shl.b32 %r161, %r94, 1; + sub.s32 %r205, %r160, %r161; + neg.s32 %r204, %r198; + mad.lo.s32 %r203, %r4, %r198, %r7; + shl.b32 %r162, %r93, 1; + shl.b32 %r163, %r3, 1; + sub.s32 %r71, %r163, %r162; + +$L__BB13_22: + add.s32 %r164, %r204, 1; + add.s32 %r165, %r198, -1; + setp.gt.u32 %p28, %r198, 1; + selp.b32 %r166, %r165, %r164, %p28; + add.s32 %r167, %r198, 1; + setp.lt.u32 %p29, %r167, %r9; + selp.b32 %r168, %r167, %r205, %p29; + mad.lo.s32 %r169, %r166, %r4, %r7; + mul.wide.u32 %rd37, %r169, 4; + add.s64 %rd38, %rd2, %rd37; + mad.lo.s32 %r170, %r168, %r4, %r7; + mul.wide.u32 %rd39, %r170, 4; + add.s64 %rd40, %rd2, %rd39; + mul.wide.u32 %rd41, %r203, 4; + add.s64 %rd42, %rd2, %rd41; + ld.f32 %f33, [%rd40]; + ld.f32 %f34, [%rd38]; + add.rn.f32 %f35, %f34, %f33; + mov.f32 %f36, 0f3F000000; + mov.f32 %f37, 0f3E800000; + fma.rn.f32 %f38, %f35, %f37, %f36; + cvt.rmi.f32.f32 %f39, %f38; + ld.f32 %f40, [%rd42]; + sub.rn.f32 %f41, %f40, %f39; + st.f32 [%rd42], %f41; + add.s32 %r205, %r205, -2; + add.s32 %r204, %r204, -2; + add.s32 %r203, %r203, %r71; + add.s32 %r198, %r198, 2; + setp.lt.u32 %p30, %r198, %r9; + @%p30 bra $L__BB13_22; + +$L__BB13_23: + setp.ge.u32 %p31, %r202, %r9; + @%p31 bra $L__BB13_28; + + shl.b32 %r171, %r8, 1; + add.s32 %r172, %r171, -3; + sub.s32 %r173, %r172, %r202; + shl.b32 %r174, %r94, 1; + sub.s32 %r209, %r173, %r174; + neg.s32 %r208, %r202; + mad.lo.s32 %r207, %r4, %r202, %r7; + shl.b32 %r175, %r93, 1; + shl.b32 %r176, %r3, 1; + sub.s32 %r83, %r176, %r175; + +$L__BB13_25: + add.s32 %r177, %r208, 1; + add.s32 %r178, %r202, -1; + setp.gt.u32 %p32, %r202, 1; + selp.b32 %r179, %r178, %r177, %p32; + add.s32 %r180, %r202, 1; + setp.lt.u32 %p33, %r180, %r9; + selp.b32 %r181, %r180, %r209, %p33; + mad.lo.s32 %r182, %r179, %r4, %r7; + mul.wide.u32 %rd43, %r182, 4; + add.s64 %rd44, %rd2, %rd43; + mad.lo.s32 %r183, %r181, %r4, %r7; + mul.wide.u32 %rd45, %r183, 4; + add.s64 %rd46, %rd2, %rd45; + mul.wide.u32 %rd47, %r207, 4; + add.s64 %rd48, %rd2, %rd47; + ld.f32 %f42, [%rd46]; + ld.f32 %f43, [%rd44]; + add.rn.f32 %f44, %f43, %f42; + mul.rn.f32 %f45, %f44, 0f3F000000; + cvt.rmi.f32.f32 %f46, %f45; + ld.f32 %f47, [%rd48]; + add.rn.f32 %f48, %f47, %f46; + st.f32 [%rd48], %f48; + add.s32 %r209, %r209, -2; + add.s32 %r208, %r208, -2; + add.s32 %r207, %r207, %r83; + add.s32 %r202, %r202, 2; + setp.lt.u32 %p34, %r202, %r9; + @%p34 bra $L__BB13_25; + +$L__BB13_28: + ret; + +} + // .globl j2k_idwt_vertical_53_multi +.visible .entry j2k_idwt_vertical_53_multi( + .param .u64 j2k_idwt_vertical_53_multi_param_0 +) +{ + .reg .pred %p<20>; + .reg .f32 %f<22>; + .reg .b32 %r<52>; + .reg .b64 %rd<10>; + // demoted variable + .shared .align 4 .b8 _ZZ35j2k_idwt_vertical_53_multiE14column_samples[2048]; + + ld.param.u64 %rd3, [j2k_idwt_vertical_53_multi_param_0]; + cvta.to.global.u64 %rd4, %rd3; + mov.u32 %r1, %tid.x; + mov.u32 %r9, %ctaid.y; + mul.wide.u32 %rd5, %r9, 128; + add.s64 %rd6, %rd4, %rd5; + ld.global.v2.u32 {%r10, %r11}, [%rd6+48]; + ld.global.v2.u32 {%r14, %r15}, [%rd6+40]; + ld.global.u64 %rd1, [%rd6+32]; + sub.s32 %r3, %r10, %r14; + sub.s32 %r4, %r11, %r15; + mov.u32 %r5, %ctaid.x; + setp.ge.u32 %p1, %r5, %r3; + @%p1 bra $L__BB14_14; + + setp.ge.u32 %p2, %r1, %r4; + mad.lo.s32 %r17, %r3, %r1, %r5; + mul.wide.u32 %rd7, %r17, 4; + add.s64 %rd2, %rd1, %rd7; + shl.b32 %r18, %r1, 2; + mov.u32 %r19, _ZZ35j2k_idwt_vertical_53_multiE14column_samples; + add.s32 %r6, %r19, %r18; + @%p2 bra $L__BB14_3; + + ld.f32 %f1, [%rd2]; + st.shared.f32 [%r6], %f1; + +$L__BB14_3: + bar.sync 0; + setp.eq.s32 %p3, %r4, 1; + @%p3 bra $L__BB14_10; + bra.uni $L__BB14_4; + +$L__BB14_10: + setp.ne.s32 %p15, %r1, 0; + and.b32 %r51, %r15, 1; + setp.eq.b32 %p16, %r51, 1; + not.pred %p17, %p16; + or.pred %p18, %p15, %p17; + @%p18 bra $L__BB14_12; + + ld.shared.f32 %f19, [_ZZ35j2k_idwt_vertical_53_multiE14column_samples]; + mul.rn.f32 %f20, %f19, 0f3F000000; + st.shared.f32 [_ZZ35j2k_idwt_vertical_53_multiE14column_samples], %f20; + +$L__BB14_12: + bar.sync 0; + @%p15 bra $L__BB14_14; + + ld.shared.f32 %f21, [_ZZ35j2k_idwt_vertical_53_multiE14column_samples]; + mul.wide.u32 %rd8, %r5, 4; + add.s64 %rd9, %rd1, %rd8; + st.f32 [%rd9], %f21; + bra.uni $L__BB14_14; + +$L__BB14_4: + and.b32 %r7, %r15, 1; + and.b32 %r8, %r1, 1; + setp.ne.s32 %p5, %r8, %r7; + or.pred %p6, %p2, %p5; + @%p6 bra $L__BB14_6; + + setp.gt.u32 %p7, %r1, 1; + mov.u32 %r20, 1; + sub.s32 %r21, %r20, %r1; + add.s32 %r22, %r1, -1; + selp.b32 %r23, %r22, %r21, %p7; + add.s32 %r24, %r1, 1; + setp.lt.u32 %p8, %r24, %r4; + mov.u32 %r25, -3; + sub.s32 %r26, %r25, %r1; + add.s32 %r27, %r26, %r4; + add.s32 %r28, %r27, %r4; + selp.b32 %r29, %r24, %r28, %p8; + shl.b32 %r30, %r23, 2; + add.s32 %r32, %r19, %r30; + shl.b32 %r33, %r29, 2; + add.s32 %r34, %r19, %r33; + ld.shared.f32 %f2, [%r34]; + ld.shared.f32 %f3, [%r32]; + add.rn.f32 %f4, %f3, %f2; + mov.f32 %f5, 0f3F000000; + mov.f32 %f6, 0f3E800000; + fma.rn.f32 %f7, %f4, %f6, %f5; + cvt.rmi.f32.f32 %f8, %f7; + ld.shared.f32 %f9, [%r6]; + sub.rn.f32 %f10, %f9, %f8; + st.shared.f32 [%r6], %f10; + +$L__BB14_6: + bar.sync 0; + xor.b32 %r35, %r7, 1; + setp.ne.s32 %p10, %r8, %r35; + or.pred %p11, %p2, %p10; + @%p11 bra $L__BB14_8; + + setp.gt.u32 %p12, %r1, 1; + mov.u32 %r36, 1; + sub.s32 %r37, %r36, %r1; + add.s32 %r38, %r1, -1; + selp.b32 %r39, %r38, %r37, %p12; + add.s32 %r40, %r1, 1; + setp.lt.u32 %p13, %r40, %r4; + mov.u32 %r41, -3; + sub.s32 %r42, %r41, %r1; + add.s32 %r43, %r42, %r4; + add.s32 %r44, %r43, %r4; + selp.b32 %r45, %r40, %r44, %p13; + shl.b32 %r46, %r39, 2; + add.s32 %r48, %r19, %r46; + shl.b32 %r49, %r45, 2; + add.s32 %r50, %r19, %r49; + ld.shared.f32 %f11, [%r50]; + ld.shared.f32 %f12, [%r48]; + add.rn.f32 %f13, %f12, %f11; + mul.rn.f32 %f14, %f13, 0f3F000000; + cvt.rmi.f32.f32 %f15, %f14; + ld.shared.f32 %f16, [%r6]; + add.rn.f32 %f17, %f16, %f15; + st.shared.f32 [%r6], %f17; + +$L__BB14_8: + bar.sync 0; + @%p2 bra $L__BB14_14; + + ld.shared.f32 %f18, [%r6]; + st.f32 [%rd2], %f18; + +$L__BB14_14: + ret; + +} + // .globl j2k_idwt_vertical_97_multi +.visible .entry j2k_idwt_vertical_97_multi( + .param .u64 j2k_idwt_vertical_97_multi_param_0 +) +{ + .reg .pred %p<31>; + .reg .f32 %f<35>; + .reg .b32 %r<83>; + .reg .b64 %rd<10>; + // demoted variable + .shared .align 4 .b8 _ZZ35j2k_idwt_vertical_97_multiE14column_samples[2048]; + + ld.param.u64 %rd3, [j2k_idwt_vertical_97_multi_param_0]; + cvta.to.global.u64 %rd4, %rd3; + mov.u32 %r1, %tid.x; + mov.u32 %r9, %ctaid.y; + mul.wide.u32 %rd5, %r9, 128; + add.s64 %rd6, %rd4, %rd5; + ld.global.v2.u32 {%r10, %r11}, [%rd6+48]; + ld.global.v2.u32 {%r14, %r15}, [%rd6+40]; + ld.global.u64 %rd1, [%rd6+32]; + sub.s32 %r3, %r10, %r14; + sub.s32 %r4, %r11, %r15; + mov.u32 %r5, %ctaid.x; + setp.ge.u32 %p3, %r5, %r3; + @%p3 bra $L__BB15_20; + + setp.ge.u32 %p4, %r1, %r4; + mad.lo.s32 %r17, %r3, %r1, %r5; + mul.wide.u32 %rd7, %r17, 4; + add.s64 %rd2, %rd1, %rd7; + shl.b32 %r18, %r1, 2; + mov.u32 %r19, _ZZ35j2k_idwt_vertical_97_multiE14column_samples; + add.s32 %r6, %r19, %r18; + @%p4 bra $L__BB15_3; + + ld.f32 %f1, [%rd2]; + st.shared.f32 [%r6], %f1; + +$L__BB15_3: + bar.sync 0; + setp.eq.s32 %p5, %r4, 1; + @%p5 bra $L__BB15_16; + bra.uni $L__BB15_4; + +$L__BB15_16: + setp.ne.s32 %p26, %r1, 0; + and.b32 %r82, %r15, 1; + setp.eq.b32 %p27, %r82, 1; + not.pred %p28, %p27; + or.pred %p29, %p26, %p28; + @%p29 bra $L__BB15_18; + + ld.shared.f32 %f32, [_ZZ35j2k_idwt_vertical_97_multiE14column_samples]; + mul.rn.f32 %f33, %f32, 0f3F000000; + st.shared.f32 [_ZZ35j2k_idwt_vertical_97_multiE14column_samples], %f33; + +$L__BB15_18: + bar.sync 0; + @%p26 bra $L__BB15_20; + + ld.shared.f32 %f34, [_ZZ35j2k_idwt_vertical_97_multiE14column_samples]; + mul.wide.u32 %rd8, %r5, 4; + add.s64 %rd9, %rd1, %rd8; + st.f32 [%rd9], %f34; + bra.uni $L__BB15_20; + +$L__BB15_4: + setp.lt.u32 %p6, %r1, %r4; + and.b32 %r7, %r15, 1; + @%p6 bra $L__BB15_5; + bra.uni $L__BB15_6; + +$L__BB15_5: + setp.eq.s32 %p7, %r7, 0; + selp.f32 %f2, 0f3F5019C3, 0f3F9D7658, %p7; + selp.f32 %f3, 0f3F9D7658, 0f3F5019C3, %p7; + and.b32 %r20, %r1, 1; + setp.eq.b32 %p8, %r20, 1; + selp.f32 %f4, %f2, %f3, %p8; + ld.shared.f32 %f5, [%r6]; + mul.rn.f32 %f6, %f4, %f5; + st.shared.f32 [%r6], %f6; + +$L__BB15_6: + bar.sync 0; + and.b32 %r8, %r1, 1; + setp.eq.s32 %p10, %r8, %r7; + and.pred %p1, %p6, %p10; + not.pred %p11, %p1; + @%p11 bra $L__BB15_8; + + setp.gt.u32 %p12, %r1, 1; + mov.u32 %r21, 1; + sub.s32 %r22, %r21, %r1; + add.s32 %r23, %r1, -1; + selp.b32 %r24, %r23, %r22, %p12; + add.s32 %r25, %r1, 1; + setp.lt.u32 %p13, %r25, %r4; + mov.u32 %r26, -3; + sub.s32 %r27, %r26, %r1; + add.s32 %r28, %r27, %r4; + add.s32 %r29, %r28, %r4; + selp.b32 %r30, %r25, %r29, %p13; + shl.b32 %r31, %r24, 2; + add.s32 %r33, %r19, %r31; + shl.b32 %r34, %r30, 2; + add.s32 %r35, %r19, %r34; + ld.shared.f32 %f7, [%r35]; + ld.shared.f32 %f8, [%r33]; + add.rn.f32 %f9, %f8, %f7; + ld.shared.f32 %f10, [%r6]; + mov.f32 %f11, 0fBEE31355; + fma.rn.f32 %f12, %f9, %f11, %f10; + st.shared.f32 [%r6], %f12; + +$L__BB15_8: + bar.sync 0; + xor.b32 %r36, %r7, 1; + setp.eq.s32 %p15, %r8, %r36; + and.pred %p2, %p6, %p15; + not.pred %p16, %p2; + @%p16 bra $L__BB15_10; + + setp.gt.u32 %p17, %r1, 1; + mov.u32 %r37, 1; + sub.s32 %r38, %r37, %r1; + add.s32 %r39, %r1, -1; + selp.b32 %r40, %r39, %r38, %p17; + add.s32 %r41, %r1, 1; + setp.lt.u32 %p18, %r41, %r4; + mov.u32 %r42, -3; + sub.s32 %r43, %r42, %r1; + add.s32 %r44, %r43, %r4; + add.s32 %r45, %r44, %r4; + selp.b32 %r46, %r41, %r45, %p18; + shl.b32 %r47, %r40, 2; + add.s32 %r49, %r19, %r47; + shl.b32 %r50, %r46, 2; + add.s32 %r51, %r19, %r50; + ld.shared.f32 %f13, [%r51]; + ld.shared.f32 %f14, [%r49]; + add.rn.f32 %f15, %f14, %f13; + ld.shared.f32 %f16, [%r6]; + mov.f32 %f17, 0fBF620676; + fma.rn.f32 %f18, %f15, %f17, %f16; + st.shared.f32 [%r6], %f18; + +$L__BB15_10: + bar.sync 0; + @%p11 bra $L__BB15_12; + + setp.gt.u32 %p20, %r1, 1; + mov.u32 %r52, 1; + sub.s32 %r53, %r52, %r1; + add.s32 %r54, %r1, -1; + selp.b32 %r55, %r54, %r53, %p20; + add.s32 %r56, %r1, 1; + setp.lt.u32 %p21, %r56, %r4; + mov.u32 %r57, -3; + sub.s32 %r58, %r57, %r1; + add.s32 %r59, %r58, %r4; + add.s32 %r60, %r59, %r4; + selp.b32 %r61, %r56, %r60, %p21; + shl.b32 %r62, %r55, 2; + add.s32 %r64, %r19, %r62; + shl.b32 %r65, %r61, 2; + add.s32 %r66, %r19, %r65; + ld.shared.f32 %f19, [%r66]; + ld.shared.f32 %f20, [%r64]; + add.rn.f32 %f21, %f20, %f19; + ld.shared.f32 %f22, [%r6]; + mov.f32 %f23, 0f3D5901AE; + fma.rn.f32 %f24, %f21, %f23, %f22; + st.shared.f32 [%r6], %f24; + +$L__BB15_12: + bar.sync 0; + @%p16 bra $L__BB15_14; + + setp.gt.u32 %p23, %r1, 1; + mov.u32 %r67, 1; + sub.s32 %r68, %r67, %r1; + add.s32 %r69, %r1, -1; + selp.b32 %r70, %r69, %r68, %p23; + add.s32 %r71, %r1, 1; + setp.lt.u32 %p24, %r71, %r4; + mov.u32 %r72, -3; + sub.s32 %r73, %r72, %r1; + add.s32 %r74, %r73, %r4; + add.s32 %r75, %r74, %r4; + selp.b32 %r76, %r71, %r75, %p24; + shl.b32 %r77, %r70, 2; + add.s32 %r79, %r19, %r77; + shl.b32 %r80, %r76, 2; + add.s32 %r81, %r19, %r80; + ld.shared.f32 %f25, [%r81]; + ld.shared.f32 %f26, [%r79]; + add.rn.f32 %f27, %f26, %f25; + ld.shared.f32 %f28, [%r6]; + mov.f32 %f29, 0f3FCB0673; + fma.rn.f32 %f30, %f27, %f29, %f28; + st.shared.f32 [%r6], %f30; + +$L__BB15_14: + bar.sync 0; + @%p4 bra $L__BB15_20; + + ld.shared.f32 %f31, [%r6]; + st.f32 [%rd2], %f31; + +$L__BB15_20: + ret; + +} + // .globl j2k_idwt_vertical_97_multi_cols4 +.visible .entry j2k_idwt_vertical_97_multi_cols4( + .param .u64 j2k_idwt_vertical_97_multi_cols4_param_0 +) +{ + .reg .pred %p<30>; + .reg .f32 %f<35>; + .reg .b32 %r<102>; + .reg .b64 %rd<10>; + // demoted variable + .shared .align 4 .b8 _ZZ41j2k_idwt_vertical_97_multi_cols4E14column_samples[4096]; + + ld.param.u64 %rd3, [j2k_idwt_vertical_97_multi_cols4_param_0]; + cvta.to.global.u64 %rd4, %rd3; + mov.u32 %r1, %tid.y; + mov.u32 %r11, %ctaid.x; + shl.b32 %r12, %r11, 2; + mov.u32 %r2, %tid.x; + add.s32 %r3, %r12, %r2; + mov.u32 %r13, %ctaid.y; + mul.wide.u32 %rd5, %r13, 128; + add.s64 %rd6, %rd4, %rd5; + ld.global.v2.u32 {%r14, %r15}, [%rd6+48]; + ld.global.v2.u32 {%r18, %r19}, [%rd6+40]; + ld.global.u64 %rd1, [%rd6+32]; + sub.s32 %r5, %r14, %r18; + sub.s32 %r6, %r15, %r19; + setp.gt.u32 %p4, %r6, 256; + @%p4 bra $L__BB16_20; + + setp.lt.u32 %p5, %r3, %r5; + setp.lt.u32 %p6, %r1, %r6; + and.pred %p1, %p5, %p6; + mad.lo.s32 %r21, %r5, %r1, %r3; + mul.wide.u32 %rd7, %r21, 4; + add.s64 %rd2, %rd1, %rd7; + shl.b32 %r22, %r1, 4; + mov.u32 %r23, _ZZ41j2k_idwt_vertical_97_multi_cols4E14column_samples; + add.s32 %r24, %r23, %r22; + shl.b32 %r25, %r2, 2; + add.s32 %r7, %r24, %r25; + not.pred %p7, %p1; + @%p7 bra $L__BB16_3; + + ld.f32 %f1, [%rd2]; + st.shared.f32 [%r7], %f1; + +$L__BB16_3: + bar.sync 0; + and.b32 %r8, %r19, 1; + setp.eq.s32 %p8, %r6, 1; + @%p8 bra $L__BB16_16; + bra.uni $L__BB16_4; + +$L__BB16_16: + setp.eq.s32 %p26, %r8, 0; + add.s32 %r10, %r23, %r25; + or.pred %p28, %p26, %p7; + @%p28 bra $L__BB16_18; + + ld.shared.f32 %f32, [%r10]; + mul.rn.f32 %f33, %f32, 0f3F000000; + st.shared.f32 [%r10], %f33; + +$L__BB16_18: + bar.sync 0; + @%p7 bra $L__BB16_20; + + ld.shared.f32 %f34, [%r10]; + mul.wide.u32 %rd8, %r3, 4; + add.s64 %rd9, %rd1, %rd8; + st.f32 [%rd9], %f34; + bra.uni $L__BB16_20; + +$L__BB16_4: + @%p1 bra $L__BB16_5; + bra.uni $L__BB16_6; + +$L__BB16_5: + setp.eq.s32 %p9, %r8, 0; + selp.f32 %f2, 0f3F5019C3, 0f3F9D7658, %p9; + selp.f32 %f3, 0f3F9D7658, 0f3F5019C3, %p9; + and.b32 %r26, %r1, 1; + setp.eq.b32 %p10, %r26, 1; + selp.f32 %f4, %f2, %f3, %p10; + ld.shared.f32 %f5, [%r7]; + mul.rn.f32 %f6, %f4, %f5; + st.shared.f32 [%r7], %f6; + +$L__BB16_6: + and.b32 %r9, %r1, 1; + bar.sync 0; + setp.eq.s32 %p11, %r9, %r8; + and.pred %p2, %p11, %p1; + not.pred %p12, %p2; + @%p12 bra $L__BB16_8; + + setp.gt.u32 %p13, %r1, 1; + mov.u32 %r27, 1; + sub.s32 %r28, %r27, %r1; + add.s32 %r29, %r1, -1; + selp.b32 %r30, %r29, %r28, %p13; + add.s32 %r31, %r1, 1; + setp.lt.u32 %p14, %r31, %r6; + mov.u32 %r32, -3; + sub.s32 %r33, %r32, %r1; + add.s32 %r34, %r33, %r6; + add.s32 %r35, %r34, %r6; + selp.b32 %r36, %r31, %r35, %p14; + shl.b32 %r37, %r30, 4; + add.s32 %r39, %r23, %r37; + add.s32 %r41, %r39, %r25; + shl.b32 %r42, %r36, 4; + add.s32 %r43, %r23, %r42; + add.s32 %r44, %r43, %r25; + ld.shared.f32 %f7, [%r44]; + ld.shared.f32 %f8, [%r41]; + add.rn.f32 %f9, %f8, %f7; + ld.shared.f32 %f10, [%r7]; + mov.f32 %f11, 0fBEE31355; + fma.rn.f32 %f12, %f9, %f11, %f10; + st.shared.f32 [%r7], %f12; + +$L__BB16_8: + bar.sync 0; + xor.b32 %r45, %r8, 1; + setp.eq.s32 %p15, %r9, %r45; + and.pred %p3, %p1, %p15; + not.pred %p16, %p3; + @%p16 bra $L__BB16_10; + + setp.gt.u32 %p17, %r1, 1; + mov.u32 %r46, 1; + sub.s32 %r47, %r46, %r1; + add.s32 %r48, %r1, -1; + selp.b32 %r49, %r48, %r47, %p17; + add.s32 %r50, %r1, 1; + setp.lt.u32 %p18, %r50, %r6; + mov.u32 %r51, -3; + sub.s32 %r52, %r51, %r1; + add.s32 %r53, %r52, %r6; + add.s32 %r54, %r53, %r6; + selp.b32 %r55, %r50, %r54, %p18; + shl.b32 %r56, %r49, 4; + add.s32 %r58, %r23, %r56; + add.s32 %r60, %r58, %r25; + shl.b32 %r61, %r55, 4; + add.s32 %r62, %r23, %r61; + add.s32 %r63, %r62, %r25; + ld.shared.f32 %f13, [%r63]; + ld.shared.f32 %f14, [%r60]; + add.rn.f32 %f15, %f14, %f13; + ld.shared.f32 %f16, [%r7]; + mov.f32 %f17, 0fBF620676; + fma.rn.f32 %f18, %f15, %f17, %f16; + st.shared.f32 [%r7], %f18; + +$L__BB16_10: + bar.sync 0; + @%p12 bra $L__BB16_12; + + setp.gt.u32 %p20, %r1, 1; + mov.u32 %r64, 1; + sub.s32 %r65, %r64, %r1; + add.s32 %r66, %r1, -1; + selp.b32 %r67, %r66, %r65, %p20; + add.s32 %r68, %r1, 1; + setp.lt.u32 %p21, %r68, %r6; + mov.u32 %r69, -3; + sub.s32 %r70, %r69, %r1; + add.s32 %r71, %r70, %r6; + add.s32 %r72, %r71, %r6; + selp.b32 %r73, %r68, %r72, %p21; + shl.b32 %r74, %r67, 4; + add.s32 %r76, %r23, %r74; + add.s32 %r78, %r76, %r25; + shl.b32 %r79, %r73, 4; + add.s32 %r80, %r23, %r79; + add.s32 %r81, %r80, %r25; + ld.shared.f32 %f19, [%r81]; + ld.shared.f32 %f20, [%r78]; + add.rn.f32 %f21, %f20, %f19; + ld.shared.f32 %f22, [%r7]; + mov.f32 %f23, 0f3D5901AE; + fma.rn.f32 %f24, %f21, %f23, %f22; + st.shared.f32 [%r7], %f24; + +$L__BB16_12: + bar.sync 0; + @%p16 bra $L__BB16_14; + + setp.gt.u32 %p23, %r1, 1; + mov.u32 %r82, 1; + sub.s32 %r83, %r82, %r1; + add.s32 %r84, %r1, -1; + selp.b32 %r85, %r84, %r83, %p23; + add.s32 %r86, %r1, 1; + setp.lt.u32 %p24, %r86, %r6; + mov.u32 %r87, -3; + sub.s32 %r88, %r87, %r1; + add.s32 %r89, %r88, %r6; + add.s32 %r90, %r89, %r6; + selp.b32 %r91, %r86, %r90, %p24; + shl.b32 %r92, %r85, 4; + add.s32 %r94, %r23, %r92; + add.s32 %r96, %r94, %r25; + shl.b32 %r97, %r91, 4; + add.s32 %r98, %r23, %r97; + add.s32 %r99, %r98, %r25; + ld.shared.f32 %f25, [%r99]; + ld.shared.f32 %f26, [%r96]; + add.rn.f32 %f27, %f26, %f25; + ld.shared.f32 %f28, [%r7]; + mov.f32 %f29, 0f3FCB0673; + fma.rn.f32 %f30, %f27, %f29, %f28; + st.shared.f32 [%r7], %f30; + +$L__BB16_14: + bar.sync 0; + @%p7 bra $L__BB16_20; + + ld.shared.f32 %f31, [%r7]; + st.f32 [%rd2], %f31; + +$L__BB16_20: + ret; + +} + // .globl j2k_store_gray8 +.visible .entry j2k_store_gray8( + .param .u64 j2k_store_gray8_param_0, + .param .u64 j2k_store_gray8_param_1, + .param .u64 j2k_store_gray8_param_2 +) +{ + .reg .pred %p<5>; + .reg .f32 %f<23>; + .reg .b32 %r<32>; + .reg .b64 %rd<12>; + + + ld.param.u64 %rd3, [j2k_store_gray8_param_0]; + ld.param.u64 %rd4, [j2k_store_gray8_param_1]; + ld.param.u64 %rd5, [j2k_store_gray8_param_2]; + cvta.to.global.u64 %rd6, %rd5; + add.s64 %rd1, %rd6, 12; + ld.global.u32 %r8, [%rd6+16]; + ld.global.u32 %r1, [%rd6+12]; + mul.lo.s32 %r9, %r8, %r1; + mov.u32 %r10, %ntid.x; + mov.u32 %r11, %ctaid.x; + mov.u32 %r12, %tid.x; + mad.lo.s32 %r2, %r11, %r10, %r12; + setp.ge.u32 %p1, %r2, %r9; + @%p1 bra $L__BB17_5; + + cvta.to.global.u64 %rd2, %rd4; + div.u32 %r13, %r2, %r1; + mul.lo.s32 %r14, %r13, %r1; + sub.s32 %r15, %r2, %r14; + ld.global.u32 %r16, [%rd1+-4]; + add.s32 %r17, %r13, %r16; + ld.global.u32 %r18, [%rd1+-12]; + ld.global.u32 %r19, [%rd1+-8]; + mad.lo.s32 %r20, %r17, %r18, %r19; + add.s32 %r21, %r20, %r15; + ld.global.u32 %r22, [%rd1+20]; + add.s32 %r23, %r13, %r22; + ld.global.u32 %r24, [%rd1+8]; + ld.global.u32 %r25, [%rd1+16]; + mad.lo.s32 %r26, %r23, %r24, %r25; + add.s32 %r3, %r26, %r15; + cvta.to.global.u64 %rd7, %rd3; + mul.wide.u32 %rd8, %r21, 4; + add.s64 %rd9, %rd7, %rd8; + ld.global.f32 %f2, [%rd9]; + ld.global.f32 %f3, [%rd1+24]; + add.rn.f32 %f4, %f3, %f2; + mov.f32 %f5, 0f3F000000; + copysign.f32 %f6, %f4, %f5; + add.rz.f32 %f7, %f4, %f6; + cvt.rzi.f32.f32 %f1, %f7; + ld.global.u32 %r4, [%rd1+28]; + setp.eq.s32 %p2, %r4, 8; + @%p2 bra $L__BB17_3; + bra.uni $L__BB17_2; + +$L__BB17_3: + mov.f32 %f19, 0f00000000; + max.f32 %f20, %f1, %f19; + mov.f32 %f21, 0f437F0000; + min.f32 %f22, %f20, %f21; + cvt.rzi.u32.f32 %r31, %f22; + bra.uni $L__BB17_4; + +$L__BB17_2: + setp.gt.u32 %p3, %r4, 15; + mov.u32 %r27, -1; + shl.b32 %r28, %r27, %r4; + setp.lt.u32 %p4, %r28, -2; + not.b32 %r29, %r28; + selp.b32 %r30, %r29, 1, %p4; + cvt.rn.f32.u32 %f8, %r30; + selp.f32 %f9, 0f477FFF00, %f8, %p3; + mov.f32 %f10, 0f00000000; + max.f32 %f11, %f1, %f10; + min.f32 %f12, %f11, %f9; + div.rn.f32 %f13, %f12, %f9; + mul.rn.f32 %f14, %f13, 0f437F0000; + copysign.f32 %f16, %f14, %f5; + add.rz.f32 %f17, %f14, %f16; + cvt.rzi.f32.f32 %f18, %f17; + cvt.rzi.u32.f32 %r31, %f18; + +$L__BB17_4: + cvt.u64.u32 %rd10, %r3; + add.s64 %rd11, %rd2, %rd10; + st.global.u8 [%rd11], %r31; + +$L__BB17_5: + ret; + +} + // .globl j2k_store_gray16 +.visible .entry j2k_store_gray16( + .param .u64 j2k_store_gray16_param_0, + .param .u64 j2k_store_gray16_param_1, + .param .u64 j2k_store_gray16_param_2 +) +{ + .reg .pred %p<4>; + .reg .f32 %f<22>; + .reg .b32 %r<33>; + .reg .b64 %rd<12>; + + + ld.param.u64 %rd3, [j2k_store_gray16_param_0]; + ld.param.u64 %rd4, [j2k_store_gray16_param_1]; + ld.param.u64 %rd5, [j2k_store_gray16_param_2]; + cvta.to.global.u64 %rd6, %rd5; + add.s64 %rd1, %rd6, 12; + ld.global.u32 %r8, [%rd6+16]; + ld.global.u32 %r1, [%rd6+12]; + mul.lo.s32 %r9, %r8, %r1; + mov.u32 %r10, %ntid.x; + mov.u32 %r11, %ctaid.x; + mov.u32 %r12, %tid.x; + mad.lo.s32 %r2, %r11, %r10, %r12; + setp.ge.u32 %p1, %r2, %r9; + @%p1 bra $L__BB18_5; + + cvta.to.global.u64 %rd2, %rd4; + div.u32 %r13, %r2, %r1; + mul.lo.s32 %r14, %r13, %r1; + sub.s32 %r15, %r2, %r14; + ld.global.u32 %r16, [%rd1+-4]; + add.s32 %r17, %r13, %r16; + ld.global.u32 %r18, [%rd1+-12]; + ld.global.u32 %r19, [%rd1+-8]; + mad.lo.s32 %r20, %r17, %r18, %r19; + add.s32 %r21, %r20, %r15; + ld.global.u32 %r22, [%rd1+20]; + add.s32 %r23, %r13, %r22; + ld.global.u32 %r24, [%rd1+8]; + ld.global.u32 %r25, [%rd1+16]; + mad.lo.s32 %r26, %r23, %r24, %r25; + add.s32 %r3, %r26, %r15; + cvta.to.global.u64 %rd7, %rd3; + mul.wide.u32 %rd8, %r21, 4; + add.s64 %rd9, %rd7, %rd8; + ld.global.f32 %f2, [%rd9]; + ld.global.f32 %f3, [%rd1+24]; + add.rn.f32 %f4, %f3, %f2; + mov.f32 %f5, 0f3F000000; + copysign.f32 %f6, %f4, %f5; + add.rz.f32 %f7, %f4, %f6; + cvt.rzi.f32.f32 %f1, %f7; + ld.global.u32 %r4, [%rd1+28]; + setp.gt.u32 %p2, %r4, 15; + @%p2 bra $L__BB18_3; + bra.uni $L__BB18_2; + +$L__BB18_3: + mov.f32 %f18, 0f00000000; + max.f32 %f19, %f1, %f18; + mov.f32 %f20, 0f477FFF00; + min.f32 %f21, %f19, %f20; + cvt.rzi.u32.f32 %r32, %f21; + bra.uni $L__BB18_4; + +$L__BB18_2: + setp.eq.s32 %p3, %r4, 0; + mov.u32 %r27, -1; + shl.b32 %r28, %r27, %r4; + not.b32 %r29, %r28; + selp.b32 %r30, 1, %r29, %p3; + max.u32 %r31, %r30, 1; + cvt.rn.f32.u32 %f8, %r31; + mov.f32 %f9, 0f00000000; + max.f32 %f10, %f1, %f9; + min.f32 %f11, %f10, %f8; + div.rn.f32 %f12, %f11, %f8; + mul.rn.f32 %f13, %f12, 0f477FFF00; + copysign.f32 %f15, %f13, %f5; + add.rz.f32 %f16, %f13, %f15; + cvt.rzi.f32.f32 %f17, %f16; + cvt.rzi.u32.f32 %r32, %f17; + +$L__BB18_4: + mul.wide.u32 %rd10, %r3, 2; + add.s64 %rd11, %rd2, %rd10; + st.global.u16 [%rd11], %r32; + +$L__BB18_5: + ret; + +} + // .globl j2k_inverse_mct +.visible .entry j2k_inverse_mct( + .param .u64 j2k_inverse_mct_param_0, + .param .u64 j2k_inverse_mct_param_1, + .param .u64 j2k_inverse_mct_param_2, + .param .u64 j2k_inverse_mct_param_3 +) +{ + .reg .pred %p<3>; + .reg .f32 %f<30>; + .reg .b32 %r<7>; + .reg .b64 %rd<13>; + + + ld.param.u64 %rd5, [j2k_inverse_mct_param_0]; + ld.param.u64 %rd6, [j2k_inverse_mct_param_1]; + ld.param.u64 %rd7, [j2k_inverse_mct_param_2]; + ld.param.u64 %rd8, [j2k_inverse_mct_param_3]; + cvta.to.global.u64 %rd1, %rd8; + mov.u32 %r2, %ntid.x; + mov.u32 %r3, %ctaid.x; + mov.u32 %r4, %tid.x; + mad.lo.s32 %r1, %r3, %r2, %r4; + ld.global.u32 %r5, [%rd1]; + setp.ge.u32 %p1, %r1, %r5; + @%p1 bra $L__BB19_5; + + ld.global.f32 %f1, [%rd1+8]; + ld.global.f32 %f2, [%rd1+12]; + ld.global.f32 %f3, [%rd1+16]; + cvta.to.global.u64 %rd9, %rd5; + mul.wide.u32 %rd10, %r1, 4; + add.s64 %rd2, %rd9, %rd10; + ld.global.f32 %f4, [%rd2]; + cvta.to.global.u64 %rd11, %rd6; + add.s64 %rd3, %rd11, %rd10; + ld.global.f32 %f5, [%rd3]; + cvta.to.global.u64 %rd12, %rd7; + add.s64 %rd4, %rd12, %rd10; + ld.global.f32 %f6, [%rd4]; + ld.global.u32 %r6, [%rd1+4]; + setp.eq.s32 %p2, %r6, 0; + @%p2 bra $L__BB19_3; + + mul.rn.f32 %f16, %f6, 0f3FB374BC; + add.rn.f32 %f28, %f4, %f16; + mul.rn.f32 %f17, %f5, 0fBEB031CF; + add.rn.f32 %f18, %f4, %f17; + mul.rn.f32 %f19, %f6, 0fBF36D1E1; + add.rn.f32 %f27, %f18, %f19; + mul.rn.f32 %f20, %f5, 0f3FE2D0E5; + add.rn.f32 %f29, %f4, %f20; + bra.uni $L__BB19_4; + +$L__BB19_3: + add.rn.f32 %f21, %f5, %f6; + mul.rn.f32 %f22, %f21, 0f3E800000; + cvt.rmi.f32.f32 %f23, %f22; + sub.rn.f32 %f27, %f4, %f23; + add.rn.f32 %f28, %f6, %f27; + add.rn.f32 %f29, %f5, %f27; + +$L__BB19_4: + add.rn.f32 %f24, %f1, %f28; + st.global.f32 [%rd2], %f24; + add.rn.f32 %f25, %f2, %f27; + st.global.f32 [%rd3], %f25; + add.rn.f32 %f26, %f3, %f29; + st.global.f32 [%rd4], %f26; + +$L__BB19_5: + ret; + +} + // .globl j2k_store_rgb8 +.visible .entry j2k_store_rgb8( + .param .u64 j2k_store_rgb8_param_0, + .param .u64 j2k_store_rgb8_param_1, + .param .u64 j2k_store_rgb8_param_2, + .param .u64 j2k_store_rgb8_param_3, + .param .u64 j2k_store_rgb8_param_4 +) +{ + .reg .pred %p<13>; + .reg .b16 %rs<2>; + .reg .f32 %f<67>; + .reg .b32 %r<68>; + .reg .b64 %rd<29>; + + + ld.param.u64 %rd2, [j2k_store_rgb8_param_0]; + ld.param.u64 %rd3, [j2k_store_rgb8_param_1]; + ld.param.u64 %rd4, [j2k_store_rgb8_param_2]; + ld.param.u64 %rd5, [j2k_store_rgb8_param_3]; + ld.param.u64 %rd6, [j2k_store_rgb8_param_4]; + cvta.to.global.u64 %rd7, %rd6; + add.s64 %rd1, %rd7, 36; + ld.global.u32 %r19, [%rd7+40]; + ld.global.u32 %r1, [%rd7+36]; + mul.lo.s32 %r20, %r19, %r1; + mov.u32 %r21, %ntid.x; + mov.u32 %r22, %ctaid.x; + mov.u32 %r23, %tid.x; + mad.lo.s32 %r2, %r22, %r21, %r23; + setp.ge.u32 %p1, %r2, %r20; + @%p1 bra $L__BB20_12; + + ld.global.u32 %r3, [%rd1+44]; + ld.global.u32 %r4, [%rd1+40]; + ld.global.f32 %f1, [%rd1+32]; + ld.global.f32 %f2, [%rd1+28]; + div.u32 %r24, %r2, %r1; + mul.lo.s32 %r25, %r24, %r1; + sub.s32 %r26, %r2, %r25; + ld.global.u32 %r27, [%rd1+-20]; + add.s32 %r28, %r24, %r27; + ld.global.u32 %r29, [%rd1+-36]; + ld.global.u32 %r30, [%rd1+-24]; + mad.lo.s32 %r31, %r28, %r29, %r30; + add.s32 %r32, %r31, %r26; + ld.global.u32 %r33, [%rd1+-12]; + add.s32 %r34, %r24, %r33; + ld.global.u32 %r35, [%rd1+-32]; + ld.global.u32 %r36, [%rd1+-16]; + mad.lo.s32 %r37, %r34, %r35, %r36; + add.s32 %r5, %r37, %r26; + ld.global.u32 %r38, [%rd1+-4]; + add.s32 %r39, %r24, %r38; + ld.global.u32 %r40, [%rd1+-28]; + ld.global.u32 %r41, [%rd1+-8]; + mad.lo.s32 %r42, %r39, %r40, %r41; + add.s32 %r6, %r42, %r26; + ld.global.u32 %r7, [%rd1+48]; + setp.eq.s32 %p2, %r7, 0; + selp.b32 %r43, 3, 4, %p2; + ld.global.u32 %r44, [%rd1+20]; + add.s32 %r45, %r24, %r44; + ld.global.u32 %r46, [%rd1+8]; + ld.global.u32 %r47, [%rd1+16]; + mad.lo.s32 %r48, %r45, %r46, %r47; + add.s32 %r49, %r48, %r26; + mul.lo.s32 %r8, %r49, %r43; + cvta.to.global.u64 %rd8, %rd2; + mul.wide.u32 %rd9, %r32, 4; + add.s64 %rd10, %rd8, %rd9; + ld.global.f32 %f6, [%rd10]; + ld.global.f32 %f7, [%rd1+24]; + add.rn.f32 %f8, %f7, %f6; + mov.f32 %f9, 0f3F000000; + copysign.f32 %f10, %f8, %f9; + add.rz.f32 %f11, %f8, %f10; + cvt.rzi.f32.f32 %f3, %f11; + ld.global.u32 %r9, [%rd1+36]; + setp.eq.s32 %p3, %r9, 8; + @%p3 bra $L__BB20_3; + bra.uni $L__BB20_2; + +$L__BB20_3: + mov.f32 %f23, 0f00000000; + max.f32 %f24, %f3, %f23; + mov.f32 %f25, 0f437F0000; + min.f32 %f26, %f24, %f25; + cvt.rzi.u32.f32 %r65, %f26; + bra.uni $L__BB20_4; + +$L__BB20_2: + setp.gt.u32 %p4, %r9, 15; + mov.u32 %r50, -1; + shl.b32 %r51, %r50, %r9; + setp.lt.u32 %p5, %r51, -2; + not.b32 %r52, %r51; + selp.b32 %r53, %r52, 1, %p5; + cvt.rn.f32.u32 %f12, %r53; + selp.f32 %f13, 0f477FFF00, %f12, %p4; + mov.f32 %f14, 0f00000000; + max.f32 %f15, %f3, %f14; + min.f32 %f16, %f15, %f13; + div.rn.f32 %f17, %f16, %f13; + mul.rn.f32 %f18, %f17, 0f437F0000; + copysign.f32 %f20, %f18, %f9; + add.rz.f32 %f21, %f18, %f20; + cvt.rzi.f32.f32 %f22, %f21; + cvt.rzi.u32.f32 %r65, %f22; + +$L__BB20_4: + cvta.to.global.u64 %rd11, %rd5; + cvt.u64.u32 %rd12, %r8; + add.s64 %rd13, %rd11, %rd12; + st.global.u8 [%rd13], %r65; + cvta.to.global.u64 %rd14, %rd3; + mul.wide.u32 %rd15, %r5, 4; + add.s64 %rd16, %rd14, %rd15; + ld.global.f32 %f27, [%rd16]; + add.rn.f32 %f28, %f2, %f27; + mov.f32 %f29, 0f3F000000; + copysign.f32 %f30, %f28, %f29; + add.rz.f32 %f31, %f28, %f30; + cvt.rzi.f32.f32 %f4, %f31; + setp.eq.s32 %p6, %r4, 8; + @%p6 bra $L__BB20_6; + bra.uni $L__BB20_5; + +$L__BB20_6: + mov.f32 %f43, 0f00000000; + max.f32 %f44, %f4, %f43; + mov.f32 %f45, 0f437F0000; + min.f32 %f46, %f44, %f45; + cvt.rzi.u32.f32 %r66, %f46; + bra.uni $L__BB20_7; + +$L__BB20_5: + setp.gt.u32 %p7, %r4, 15; + mov.u32 %r54, -1; + shl.b32 %r55, %r54, %r4; + setp.lt.u32 %p8, %r55, -2; + not.b32 %r56, %r55; + selp.b32 %r57, %r56, 1, %p8; + cvt.rn.f32.u32 %f32, %r57; + selp.f32 %f33, 0f477FFF00, %f32, %p7; + mov.f32 %f34, 0f00000000; + max.f32 %f35, %f4, %f34; + min.f32 %f36, %f35, %f33; + div.rn.f32 %f37, %f36, %f33; + mul.rn.f32 %f38, %f37, 0f437F0000; + copysign.f32 %f40, %f38, %f29; + add.rz.f32 %f41, %f38, %f40; + cvt.rzi.f32.f32 %f42, %f41; + cvt.rzi.u32.f32 %r66, %f42; + +$L__BB20_7: + add.s32 %r58, %r8, 1; + cvt.u64.u32 %rd17, %r58; + add.s64 %rd19, %rd11, %rd17; + st.global.u8 [%rd19], %r66; + cvta.to.global.u64 %rd20, %rd4; + mul.wide.u32 %rd21, %r6, 4; + add.s64 %rd22, %rd20, %rd21; + ld.global.f32 %f47, [%rd22]; + add.rn.f32 %f48, %f1, %f47; + mov.f32 %f49, 0f3F000000; + copysign.f32 %f50, %f48, %f49; + add.rz.f32 %f51, %f48, %f50; + cvt.rzi.f32.f32 %f5, %f51; + setp.eq.s32 %p9, %r3, 8; + @%p9 bra $L__BB20_9; + bra.uni $L__BB20_8; + +$L__BB20_9: + mov.f32 %f63, 0f00000000; + max.f32 %f64, %f5, %f63; + mov.f32 %f65, 0f437F0000; + min.f32 %f66, %f64, %f65; + cvt.rzi.u32.f32 %r67, %f66; + bra.uni $L__BB20_10; + +$L__BB20_8: + setp.gt.u32 %p10, %r3, 15; + mov.u32 %r59, -1; + shl.b32 %r60, %r59, %r3; + setp.lt.u32 %p11, %r60, -2; + not.b32 %r61, %r60; + selp.b32 %r62, %r61, 1, %p11; + cvt.rn.f32.u32 %f52, %r62; + selp.f32 %f53, 0f477FFF00, %f52, %p10; + mov.f32 %f54, 0f00000000; + max.f32 %f55, %f5, %f54; + min.f32 %f56, %f55, %f53; + div.rn.f32 %f57, %f56, %f53; + mul.rn.f32 %f58, %f57, 0f437F0000; + copysign.f32 %f60, %f58, %f49; + add.rz.f32 %f61, %f58, %f60; + cvt.rzi.f32.f32 %f62, %f61; + cvt.rzi.u32.f32 %r67, %f62; + +$L__BB20_10: + add.s32 %r63, %r8, 2; + cvt.u64.u32 %rd23, %r63; + add.s64 %rd25, %rd11, %rd23; + st.global.u8 [%rd25], %r67; + @%p2 bra $L__BB20_12; + + add.s32 %r64, %r8, 3; + cvt.u64.u32 %rd26, %r64; + add.s64 %rd28, %rd11, %rd26; + mov.u16 %rs1, 255; + st.global.u8 [%rd28], %rs1; + +$L__BB20_12: + ret; + +} + // .globl j2k_store_rgb16 +.visible .entry j2k_store_rgb16( + .param .u64 j2k_store_rgb16_param_0, + .param .u64 j2k_store_rgb16_param_1, + .param .u64 j2k_store_rgb16_param_2, + .param .u64 j2k_store_rgb16_param_3, + .param .u64 j2k_store_rgb16_param_4 +) +{ + .reg .pred %p<10>; + .reg .b16 %rs<2>; + .reg .f32 %f<64>; + .reg .b32 %r<71>; + .reg .b64 %rd<29>; + + + ld.param.u64 %rd2, [j2k_store_rgb16_param_0]; + ld.param.u64 %rd3, [j2k_store_rgb16_param_1]; + ld.param.u64 %rd4, [j2k_store_rgb16_param_2]; + ld.param.u64 %rd5, [j2k_store_rgb16_param_3]; + ld.param.u64 %rd6, [j2k_store_rgb16_param_4]; + cvta.to.global.u64 %rd7, %rd6; + add.s64 %rd1, %rd7, 36; + ld.global.u32 %r19, [%rd7+40]; + ld.global.u32 %r1, [%rd7+36]; + mul.lo.s32 %r20, %r19, %r1; + mov.u32 %r21, %ntid.x; + mov.u32 %r22, %ctaid.x; + mov.u32 %r23, %tid.x; + mad.lo.s32 %r2, %r22, %r21, %r23; + setp.ge.u32 %p1, %r2, %r20; + @%p1 bra $L__BB21_12; + + ld.global.u32 %r3, [%rd1+44]; + ld.global.u32 %r4, [%rd1+40]; + ld.global.f32 %f1, [%rd1+32]; + ld.global.f32 %f2, [%rd1+28]; + div.u32 %r24, %r2, %r1; + mul.lo.s32 %r25, %r24, %r1; + sub.s32 %r26, %r2, %r25; + ld.global.u32 %r27, [%rd1+-20]; + add.s32 %r28, %r24, %r27; + ld.global.u32 %r29, [%rd1+-36]; + ld.global.u32 %r30, [%rd1+-24]; + mad.lo.s32 %r31, %r28, %r29, %r30; + add.s32 %r32, %r31, %r26; + ld.global.u32 %r33, [%rd1+-12]; + add.s32 %r34, %r24, %r33; + ld.global.u32 %r35, [%rd1+-32]; + ld.global.u32 %r36, [%rd1+-16]; + mad.lo.s32 %r37, %r34, %r35, %r36; + add.s32 %r5, %r37, %r26; + ld.global.u32 %r38, [%rd1+-4]; + add.s32 %r39, %r24, %r38; + ld.global.u32 %r40, [%rd1+-28]; + ld.global.u32 %r41, [%rd1+-8]; + mad.lo.s32 %r42, %r39, %r40, %r41; + add.s32 %r6, %r42, %r26; + ld.global.u32 %r7, [%rd1+48]; + setp.eq.s32 %p2, %r7, 0; + selp.b32 %r43, 3, 4, %p2; + ld.global.u32 %r44, [%rd1+20]; + add.s32 %r45, %r24, %r44; + ld.global.u32 %r46, [%rd1+8]; + ld.global.u32 %r47, [%rd1+16]; + mad.lo.s32 %r48, %r45, %r46, %r47; + add.s32 %r49, %r48, %r26; + mul.lo.s32 %r8, %r49, %r43; + cvta.to.global.u64 %rd8, %rd2; + mul.wide.u32 %rd9, %r32, 4; + add.s64 %rd10, %rd8, %rd9; + ld.global.f32 %f6, [%rd10]; + ld.global.f32 %f7, [%rd1+24]; + add.rn.f32 %f8, %f7, %f6; + mov.f32 %f9, 0f3F000000; + copysign.f32 %f10, %f8, %f9; + add.rz.f32 %f11, %f8, %f10; + cvt.rzi.f32.f32 %f3, %f11; + ld.global.u32 %r9, [%rd1+36]; + setp.gt.u32 %p3, %r9, 15; + @%p3 bra $L__BB21_3; + bra.uni $L__BB21_2; + +$L__BB21_3: + mov.f32 %f22, 0f00000000; + max.f32 %f23, %f3, %f22; + mov.f32 %f24, 0f477FFF00; + min.f32 %f25, %f23, %f24; + cvt.rzi.u32.f32 %r68, %f25; + bra.uni $L__BB21_4; + +$L__BB21_2: + setp.eq.s32 %p4, %r9, 0; + mov.u32 %r50, -1; + shl.b32 %r51, %r50, %r9; + not.b32 %r52, %r51; + selp.b32 %r53, 1, %r52, %p4; + max.u32 %r54, %r53, 1; + cvt.rn.f32.u32 %f12, %r54; + mov.f32 %f13, 0f00000000; + max.f32 %f14, %f3, %f13; + min.f32 %f15, %f14, %f12; + div.rn.f32 %f16, %f15, %f12; + mul.rn.f32 %f17, %f16, 0f477FFF00; + copysign.f32 %f19, %f17, %f9; + add.rz.f32 %f20, %f17, %f19; + cvt.rzi.f32.f32 %f21, %f20; + cvt.rzi.u32.f32 %r68, %f21; + +$L__BB21_4: + cvta.to.global.u64 %rd11, %rd5; + mul.wide.u32 %rd12, %r8, 2; + add.s64 %rd13, %rd11, %rd12; + st.global.u16 [%rd13], %r68; + cvta.to.global.u64 %rd14, %rd3; + mul.wide.u32 %rd15, %r5, 4; + add.s64 %rd16, %rd14, %rd15; + ld.global.f32 %f26, [%rd16]; + add.rn.f32 %f27, %f2, %f26; + mov.f32 %f28, 0f3F000000; + copysign.f32 %f29, %f27, %f28; + add.rz.f32 %f30, %f27, %f29; + cvt.rzi.f32.f32 %f4, %f30; + setp.gt.u32 %p5, %r4, 15; + @%p5 bra $L__BB21_6; + bra.uni $L__BB21_5; + +$L__BB21_6: + mov.f32 %f41, 0f00000000; + max.f32 %f42, %f4, %f41; + mov.f32 %f43, 0f477FFF00; + min.f32 %f44, %f42, %f43; + cvt.rzi.u32.f32 %r69, %f44; + bra.uni $L__BB21_7; + +$L__BB21_5: + setp.eq.s32 %p6, %r4, 0; + mov.u32 %r55, -1; + shl.b32 %r56, %r55, %r4; + not.b32 %r57, %r56; + selp.b32 %r58, 1, %r57, %p6; + max.u32 %r59, %r58, 1; + cvt.rn.f32.u32 %f31, %r59; + mov.f32 %f32, 0f00000000; + max.f32 %f33, %f4, %f32; + min.f32 %f34, %f33, %f31; + div.rn.f32 %f35, %f34, %f31; + mul.rn.f32 %f36, %f35, 0f477FFF00; + copysign.f32 %f38, %f36, %f28; + add.rz.f32 %f39, %f36, %f38; + cvt.rzi.f32.f32 %f40, %f39; + cvt.rzi.u32.f32 %r69, %f40; + +$L__BB21_7: + add.s32 %r60, %r8, 1; + mul.wide.u32 %rd18, %r60, 2; + add.s64 %rd19, %rd11, %rd18; + st.global.u16 [%rd19], %r69; + cvta.to.global.u64 %rd20, %rd4; + mul.wide.u32 %rd21, %r6, 4; + add.s64 %rd22, %rd20, %rd21; + ld.global.f32 %f45, [%rd22]; + add.rn.f32 %f46, %f1, %f45; + mov.f32 %f47, 0f3F000000; + copysign.f32 %f48, %f46, %f47; + add.rz.f32 %f49, %f46, %f48; + cvt.rzi.f32.f32 %f5, %f49; + setp.gt.u32 %p7, %r3, 15; + @%p7 bra $L__BB21_9; + bra.uni $L__BB21_8; + +$L__BB21_9: + mov.f32 %f60, 0f00000000; + max.f32 %f61, %f5, %f60; + mov.f32 %f62, 0f477FFF00; + min.f32 %f63, %f61, %f62; + cvt.rzi.u32.f32 %r70, %f63; + bra.uni $L__BB21_10; + +$L__BB21_8: + setp.eq.s32 %p8, %r3, 0; + mov.u32 %r61, -1; + shl.b32 %r62, %r61, %r3; + not.b32 %r63, %r62; + selp.b32 %r64, 1, %r63, %p8; + max.u32 %r65, %r64, 1; + cvt.rn.f32.u32 %f50, %r65; + mov.f32 %f51, 0f00000000; + max.f32 %f52, %f5, %f51; + min.f32 %f53, %f52, %f50; + div.rn.f32 %f54, %f53, %f50; + mul.rn.f32 %f55, %f54, 0f477FFF00; + copysign.f32 %f57, %f55, %f47; + add.rz.f32 %f58, %f55, %f57; + cvt.rzi.f32.f32 %f59, %f58; + cvt.rzi.u32.f32 %r70, %f59; + +$L__BB21_10: + add.s32 %r66, %r8, 2; + mul.wide.u32 %rd24, %r66, 2; + add.s64 %rd25, %rd11, %rd24; + st.global.u16 [%rd25], %r70; + @%p2 bra $L__BB21_12; + + add.s32 %r67, %r8, 3; + mul.wide.u32 %rd27, %r67, 2; + add.s64 %rd28, %rd11, %rd27; + mov.u16 %rs1, -1; + st.global.u16 [%rd28], %rs1; + +$L__BB21_12: + ret; + +} + // .globl j2k_store_rgb8_mct +.visible .entry j2k_store_rgb8_mct( + .param .u64 j2k_store_rgb8_mct_param_0, + .param .u64 j2k_store_rgb8_mct_param_1, + .param .u64 j2k_store_rgb8_mct_param_2, + .param .u64 j2k_store_rgb8_mct_param_3, + .param .u64 j2k_store_rgb8_mct_param_4 +) +{ + .reg .pred %p<14>; + .reg .b16 %rs<2>; + .reg .f32 %f<87>; + .reg .b32 %r<69>; + .reg .b64 %rd<26>; + + + ld.param.u64 %rd3, [j2k_store_rgb8_mct_param_0]; + ld.param.u64 %rd4, [j2k_store_rgb8_mct_param_1]; + ld.param.u64 %rd5, [j2k_store_rgb8_mct_param_2]; + ld.param.u64 %rd6, [j2k_store_rgb8_mct_param_3]; + ld.param.u64 %rd7, [j2k_store_rgb8_mct_param_4]; + cvta.to.global.u64 %rd1, %rd6; + cvta.to.global.u64 %rd8, %rd7; + add.s64 %rd2, %rd8, 36; + ld.global.u32 %r1, [%rd8+72]; + ld.global.u32 %r2, [%rd8+76]; + ld.global.u32 %r3, [%rd8+80]; + ld.global.u32 %r17, [%rd8+40]; + ld.global.u32 %r4, [%rd8+36]; + mul.lo.s32 %r18, %r17, %r4; + mov.u32 %r19, %ntid.x; + mov.u32 %r20, %ctaid.x; + mov.u32 %r21, %tid.x; + mad.lo.s32 %r5, %r20, %r19, %r21; + setp.ge.u32 %p1, %r5, %r18; + @%p1 bra $L__BB22_15; + + ld.global.f32 %f1, [%rd2+24]; + ld.global.f32 %f2, [%rd2+28]; + ld.global.f32 %f3, [%rd2+32]; + div.u32 %r22, %r5, %r4; + mul.lo.s32 %r23, %r22, %r4; + sub.s32 %r24, %r5, %r23; + ld.global.u32 %r25, [%rd2+-20]; + add.s32 %r26, %r22, %r25; + ld.global.u32 %r27, [%rd2+-36]; + ld.global.u32 %r28, [%rd2+-24]; + mad.lo.s32 %r29, %r26, %r27, %r28; + add.s32 %r30, %r29, %r24; + ld.global.u32 %r31, [%rd2+-12]; + add.s32 %r32, %r22, %r31; + ld.global.u32 %r33, [%rd2+-32]; + ld.global.u32 %r34, [%rd2+-16]; + mad.lo.s32 %r35, %r32, %r33, %r34; + add.s32 %r36, %r35, %r24; + ld.global.u32 %r37, [%rd2+-4]; + add.s32 %r38, %r22, %r37; + ld.global.u32 %r39, [%rd2+-28]; + ld.global.u32 %r40, [%rd2+-8]; + mad.lo.s32 %r41, %r38, %r39, %r40; + add.s32 %r42, %r41, %r24; + ld.global.u32 %r6, [%rd2+48]; + setp.eq.s32 %p2, %r6, 0; + selp.b32 %r43, 3, 4, %p2; + ld.global.u32 %r44, [%rd2+20]; + add.s32 %r45, %r22, %r44; + ld.global.u32 %r46, [%rd2+8]; + ld.global.u32 %r47, [%rd2+16]; + mad.lo.s32 %r48, %r45, %r46, %r47; + add.s32 %r49, %r48, %r24; + mul.lo.s32 %r7, %r49, %r43; + cvta.to.global.u64 %rd9, %rd3; + mul.wide.u32 %rd10, %r30, 4; + add.s64 %rd11, %rd9, %rd10; + ld.global.f32 %f4, [%rd11]; + cvta.to.global.u64 %rd12, %rd4; + mul.wide.u32 %rd13, %r36, 4; + add.s64 %rd14, %rd12, %rd13; + ld.global.f32 %f5, [%rd14]; + cvta.to.global.u64 %rd15, %rd5; + mul.wide.u32 %rd16, %r42, 4; + add.s64 %rd17, %rd15, %rd16; + ld.global.f32 %f6, [%rd17]; + ld.global.u32 %r50, [%rd2+52]; + setp.eq.s32 %p3, %r50, 0; + @%p3 bra $L__BB22_3; + + mul.rn.f32 %f19, %f6, 0f3FB374BC; + add.rn.f32 %f85, %f4, %f19; + mul.rn.f32 %f20, %f5, 0fBEB031CF; + add.rn.f32 %f21, %f4, %f20; + mul.rn.f32 %f22, %f6, 0fBF36D1E1; + add.rn.f32 %f84, %f21, %f22; + mul.rn.f32 %f23, %f5, 0f3FE2D0E5; + add.rn.f32 %f86, %f4, %f23; + bra.uni $L__BB22_4; + +$L__BB22_3: + add.rn.f32 %f24, %f5, %f6; + mul.rn.f32 %f25, %f24, 0f3E800000; + cvt.rmi.f32.f32 %f26, %f25; + sub.rn.f32 %f84, %f4, %f26; + add.rn.f32 %f85, %f6, %f84; + add.rn.f32 %f86, %f5, %f84; + +$L__BB22_4: + add.rn.f32 %f27, %f1, %f85; + mov.f32 %f28, 0f3F000000; + copysign.f32 %f29, %f27, %f28; + add.rz.f32 %f30, %f27, %f29; + cvt.rzi.f32.f32 %f16, %f30; + setp.eq.s32 %p4, %r1, 8; + @%p4 bra $L__BB22_6; + bra.uni $L__BB22_5; + +$L__BB22_6: + mov.f32 %f42, 0f00000000; + max.f32 %f43, %f16, %f42; + mov.f32 %f44, 0f437F0000; + min.f32 %f45, %f43, %f44; + cvt.rzi.u32.f32 %r66, %f45; + bra.uni $L__BB22_7; + +$L__BB22_5: + setp.gt.u32 %p5, %r1, 15; + mov.u32 %r51, -1; + shl.b32 %r52, %r51, %r1; + setp.lt.u32 %p6, %r52, -2; + not.b32 %r53, %r52; + selp.b32 %r54, %r53, 1, %p6; + cvt.rn.f32.u32 %f31, %r54; + selp.f32 %f32, 0f477FFF00, %f31, %p5; + mov.f32 %f33, 0f00000000; + max.f32 %f34, %f16, %f33; + min.f32 %f35, %f34, %f32; + div.rn.f32 %f36, %f35, %f32; + mul.rn.f32 %f37, %f36, 0f437F0000; + copysign.f32 %f39, %f37, %f28; + add.rz.f32 %f40, %f37, %f39; + cvt.rzi.f32.f32 %f41, %f40; + cvt.rzi.u32.f32 %r66, %f41; + +$L__BB22_7: + cvt.u64.u32 %rd18, %r7; + add.s64 %rd19, %rd1, %rd18; + st.global.u8 [%rd19], %r66; + add.rn.f32 %f46, %f2, %f84; + mov.f32 %f47, 0f3F000000; + copysign.f32 %f48, %f46, %f47; + add.rz.f32 %f49, %f46, %f48; + cvt.rzi.f32.f32 %f17, %f49; + setp.eq.s32 %p7, %r2, 8; + @%p7 bra $L__BB22_9; + bra.uni $L__BB22_8; + +$L__BB22_9: + mov.f32 %f61, 0f00000000; + max.f32 %f62, %f17, %f61; + mov.f32 %f63, 0f437F0000; + min.f32 %f64, %f62, %f63; + cvt.rzi.u32.f32 %r67, %f64; + bra.uni $L__BB22_10; + +$L__BB22_8: + setp.gt.u32 %p8, %r2, 15; + mov.u32 %r55, -1; + shl.b32 %r56, %r55, %r2; + setp.lt.u32 %p9, %r56, -2; + not.b32 %r57, %r56; + selp.b32 %r58, %r57, 1, %p9; + cvt.rn.f32.u32 %f50, %r58; + selp.f32 %f51, 0f477FFF00, %f50, %p8; + mov.f32 %f52, 0f00000000; + max.f32 %f53, %f17, %f52; + min.f32 %f54, %f53, %f51; + div.rn.f32 %f55, %f54, %f51; + mul.rn.f32 %f56, %f55, 0f437F0000; + copysign.f32 %f58, %f56, %f47; + add.rz.f32 %f59, %f56, %f58; + cvt.rzi.f32.f32 %f60, %f59; + cvt.rzi.u32.f32 %r67, %f60; + +$L__BB22_10: + add.s32 %r59, %r7, 1; + cvt.u64.u32 %rd20, %r59; + add.s64 %rd21, %rd1, %rd20; + st.global.u8 [%rd21], %r67; + add.rn.f32 %f65, %f3, %f86; + mov.f32 %f66, 0f3F000000; + copysign.f32 %f67, %f65, %f66; + add.rz.f32 %f68, %f65, %f67; + cvt.rzi.f32.f32 %f18, %f68; + setp.eq.s32 %p10, %r3, 8; + @%p10 bra $L__BB22_12; + bra.uni $L__BB22_11; + +$L__BB22_12: + mov.f32 %f80, 0f00000000; + max.f32 %f81, %f18, %f80; + mov.f32 %f82, 0f437F0000; + min.f32 %f83, %f81, %f82; + cvt.rzi.u32.f32 %r68, %f83; + bra.uni $L__BB22_13; + +$L__BB22_11: + setp.gt.u32 %p11, %r3, 15; + mov.u32 %r60, -1; + shl.b32 %r61, %r60, %r3; + setp.lt.u32 %p12, %r61, -2; + not.b32 %r62, %r61; + selp.b32 %r63, %r62, 1, %p12; + cvt.rn.f32.u32 %f69, %r63; + selp.f32 %f70, 0f477FFF00, %f69, %p11; + mov.f32 %f71, 0f00000000; + max.f32 %f72, %f18, %f71; + min.f32 %f73, %f72, %f70; + div.rn.f32 %f74, %f73, %f70; + mul.rn.f32 %f75, %f74, 0f437F0000; + copysign.f32 %f77, %f75, %f66; + add.rz.f32 %f78, %f75, %f77; + cvt.rzi.f32.f32 %f79, %f78; + cvt.rzi.u32.f32 %r68, %f79; + +$L__BB22_13: + add.s32 %r64, %r7, 2; + cvt.u64.u32 %rd22, %r64; + add.s64 %rd23, %rd1, %rd22; + st.global.u8 [%rd23], %r68; + @%p2 bra $L__BB22_15; + + add.s32 %r65, %r7, 3; + cvt.u64.u32 %rd24, %r65; + add.s64 %rd25, %rd1, %rd24; + mov.u16 %rs1, 255; + st.global.u8 [%rd25], %rs1; + +$L__BB22_15: + ret; + +} + // .globl j2k_store_rgb8_mct_batch +.visible .entry j2k_store_rgb8_mct_batch( + .param .u64 j2k_store_rgb8_mct_batch_param_0 +) +{ + .reg .pred %p<14>; + .reg .b16 %rs<2>; + .reg .f32 %f<89>; + .reg .b32 %r<84>; + .reg .b64 %rd<24>; + + + ld.param.u64 %rd3, [j2k_store_rgb8_mct_batch_param_0]; + cvta.to.global.u64 %rd4, %rd3; + mov.u32 %r17, %ctaid.y; + mul.wide.u32 %rd5, %r17, 128; + add.s64 %rd6, %rd4, %rd5; + add.s64 %rd1, %rd6, 112; + ld.global.u32 %r1, [%rd6+112]; + ld.global.v2.u32 {%r18, %r19}, [%rd6+104]; + ld.global.u64 %rd2, [%rd6+24]; + ld.global.u32 %r4, [%rd6+68]; + ld.global.u32 %r20, [%rd6+72]; + mul.lo.s32 %r21, %r4, %r20; + mov.u32 %r22, %ntid.x; + mov.u32 %r23, %ctaid.x; + mov.u32 %r24, %tid.x; + mad.lo.s32 %r5, %r23, %r22, %r24; + setp.ge.u32 %p1, %r5, %r21; + @%p1 bra $L__BB23_15; + + ld.global.v2.f32 {%f19, %f20}, [%rd1+-16]; + ld.global.f32 %f3, [%rd1+-20]; + ld.global.v2.u32 {%r25, %r26}, [%rd1+-80]; + ld.global.v2.u32 {%r29, %r30}, [%rd1+-72]; + ld.global.v2.u32 {%r33, %r34}, [%rd1+-64]; + ld.global.v2.u32 {%r37, %r38}, [%rd1+-56]; + .pragma "used_bytes_mask 15"; + ld.global.v2.u32 {%r41, %r42}, [%rd1+-48]; + .pragma "used_bytes_mask 240"; + ld.global.v2.u32 {%r44, %r45}, [%rd1+-40]; + div.u32 %r47, %r5, %r4; + mul.lo.s32 %r48, %r47, %r4; + sub.s32 %r49, %r5, %r48; + add.s32 %r50, %r47, %r33; + mad.lo.s32 %r51, %r50, %r25, %r30; + add.s32 %r52, %r51, %r49; + add.s32 %r53, %r47, %r37; + mad.lo.s32 %r54, %r53, %r26, %r34; + add.s32 %r55, %r54, %r49; + add.s32 %r56, %r47, %r41; + mad.lo.s32 %r57, %r56, %r29, %r38; + add.s32 %r58, %r57, %r49; + ld.global.u32 %r6, [%rd1+4]; + setp.eq.s32 %p2, %r6, 0; + selp.b32 %r59, 3, 4, %p2; + ld.global.u32 %r60, [%rd1+-24]; + add.s32 %r61, %r47, %r60; + ld.global.u32 %r62, [%rd1+-28]; + mad.lo.s32 %r63, %r61, %r45, %r62; + add.s32 %r64, %r63, %r49; + mul.lo.s32 %r7, %r64, %r59; + ld.global.u64 %rd7, [%rd1+-112]; + mul.wide.u32 %rd8, %r52, 4; + add.s64 %rd9, %rd7, %rd8; + ld.f32 %f4, [%rd9]; + ld.global.u64 %rd10, [%rd1+-104]; + mul.wide.u32 %rd11, %r55, 4; + add.s64 %rd12, %rd10, %rd11; + ld.f32 %f5, [%rd12]; + ld.global.u64 %rd13, [%rd1+-96]; + mul.wide.u32 %rd14, %r58, 4; + add.s64 %rd15, %rd13, %rd14; + ld.f32 %f6, [%rd15]; + ld.global.u32 %r65, [%rd1+8]; + setp.eq.s32 %p3, %r65, 0; + @%p3 bra $L__BB23_3; + + mul.rn.f32 %f21, %f6, 0f3FB374BC; + add.rn.f32 %f87, %f4, %f21; + mul.rn.f32 %f22, %f5, 0fBEB031CF; + add.rn.f32 %f23, %f4, %f22; + mul.rn.f32 %f24, %f6, 0fBF36D1E1; + add.rn.f32 %f86, %f23, %f24; + mul.rn.f32 %f25, %f5, 0f3FE2D0E5; + add.rn.f32 %f88, %f4, %f25; + bra.uni $L__BB23_4; + +$L__BB23_3: + add.rn.f32 %f26, %f5, %f6; + mul.rn.f32 %f27, %f26, 0f3E800000; + cvt.rmi.f32.f32 %f28, %f27; + sub.rn.f32 %f86, %f4, %f28; + add.rn.f32 %f87, %f6, %f86; + add.rn.f32 %f88, %f5, %f86; + +$L__BB23_4: + add.rn.f32 %f29, %f3, %f87; + mov.f32 %f30, 0f3F000000; + copysign.f32 %f31, %f29, %f30; + add.rz.f32 %f32, %f29, %f31; + cvt.rzi.f32.f32 %f16, %f32; + setp.eq.s32 %p4, %r18, 8; + @%p4 bra $L__BB23_6; + bra.uni $L__BB23_5; + +$L__BB23_6: + mov.f32 %f44, 0f00000000; + max.f32 %f45, %f16, %f44; + mov.f32 %f46, 0f437F0000; + min.f32 %f47, %f45, %f46; + cvt.rzi.u32.f32 %r81, %f47; + bra.uni $L__BB23_7; + +$L__BB23_5: + setp.gt.u32 %p5, %r18, 15; + mov.u32 %r66, -1; + shl.b32 %r67, %r66, %r18; + setp.lt.u32 %p6, %r67, -2; + not.b32 %r68, %r67; + selp.b32 %r69, %r68, 1, %p6; + cvt.rn.f32.u32 %f33, %r69; + selp.f32 %f34, 0f477FFF00, %f33, %p5; + mov.f32 %f35, 0f00000000; + max.f32 %f36, %f16, %f35; + min.f32 %f37, %f36, %f34; + div.rn.f32 %f38, %f37, %f34; + mul.rn.f32 %f39, %f38, 0f437F0000; + copysign.f32 %f41, %f39, %f30; + add.rz.f32 %f42, %f39, %f41; + cvt.rzi.f32.f32 %f43, %f42; + cvt.rzi.u32.f32 %r81, %f43; + +$L__BB23_7: + cvt.u64.u32 %rd16, %r7; + add.s64 %rd17, %rd2, %rd16; + st.u8 [%rd17], %r81; + add.rn.f32 %f48, %f19, %f86; + mov.f32 %f49, 0f3F000000; + copysign.f32 %f50, %f48, %f49; + add.rz.f32 %f51, %f48, %f50; + cvt.rzi.f32.f32 %f17, %f51; + setp.eq.s32 %p7, %r19, 8; + @%p7 bra $L__BB23_9; + bra.uni $L__BB23_8; + +$L__BB23_9: + mov.f32 %f63, 0f00000000; + max.f32 %f64, %f17, %f63; + mov.f32 %f65, 0f437F0000; + min.f32 %f66, %f64, %f65; + cvt.rzi.u32.f32 %r82, %f66; + bra.uni $L__BB23_10; + +$L__BB23_8: + setp.gt.u32 %p8, %r19, 15; + mov.u32 %r70, -1; + shl.b32 %r71, %r70, %r19; + setp.lt.u32 %p9, %r71, -2; + not.b32 %r72, %r71; + selp.b32 %r73, %r72, 1, %p9; + cvt.rn.f32.u32 %f52, %r73; + selp.f32 %f53, 0f477FFF00, %f52, %p8; + mov.f32 %f54, 0f00000000; + max.f32 %f55, %f17, %f54; + min.f32 %f56, %f55, %f53; + div.rn.f32 %f57, %f56, %f53; + mul.rn.f32 %f58, %f57, 0f437F0000; + copysign.f32 %f60, %f58, %f49; + add.rz.f32 %f61, %f58, %f60; + cvt.rzi.f32.f32 %f62, %f61; + cvt.rzi.u32.f32 %r82, %f62; + +$L__BB23_10: + add.s32 %r74, %r7, 1; + cvt.u64.u32 %rd18, %r74; + add.s64 %rd19, %rd2, %rd18; + st.u8 [%rd19], %r82; + add.rn.f32 %f67, %f20, %f88; + mov.f32 %f68, 0f3F000000; + copysign.f32 %f69, %f67, %f68; + add.rz.f32 %f70, %f67, %f69; + cvt.rzi.f32.f32 %f18, %f70; + setp.eq.s32 %p10, %r1, 8; + @%p10 bra $L__BB23_12; + bra.uni $L__BB23_11; + +$L__BB23_12: + mov.f32 %f82, 0f00000000; + max.f32 %f83, %f18, %f82; + mov.f32 %f84, 0f437F0000; + min.f32 %f85, %f83, %f84; + cvt.rzi.u32.f32 %r83, %f85; + bra.uni $L__BB23_13; + +$L__BB23_11: + setp.gt.u32 %p11, %r1, 15; + mov.u32 %r75, -1; + shl.b32 %r76, %r75, %r1; + setp.lt.u32 %p12, %r76, -2; + not.b32 %r77, %r76; + selp.b32 %r78, %r77, 1, %p12; + cvt.rn.f32.u32 %f71, %r78; + selp.f32 %f72, 0f477FFF00, %f71, %p11; + mov.f32 %f73, 0f00000000; + max.f32 %f74, %f18, %f73; + min.f32 %f75, %f74, %f72; + div.rn.f32 %f76, %f75, %f72; + mul.rn.f32 %f77, %f76, 0f437F0000; + copysign.f32 %f79, %f77, %f68; + add.rz.f32 %f80, %f77, %f79; + cvt.rzi.f32.f32 %f81, %f80; + cvt.rzi.u32.f32 %r83, %f81; + +$L__BB23_13: + add.s32 %r79, %r7, 2; + cvt.u64.u32 %rd20, %r79; + add.s64 %rd21, %rd2, %rd20; + st.u8 [%rd21], %r83; + @%p2 bra $L__BB23_15; + + add.s32 %r80, %r7, 3; + cvt.u64.u32 %rd22, %r80; + add.s64 %rd23, %rd2, %rd22; + mov.u16 %rs1, 255; + st.u8 [%rd23], %rs1; + +$L__BB23_15: + ret; + +} + // .globl j2k_store_rgb16_mct +.visible .entry j2k_store_rgb16_mct( + .param .u64 j2k_store_rgb16_mct_param_0, + .param .u64 j2k_store_rgb16_mct_param_1, + .param .u64 j2k_store_rgb16_mct_param_2, + .param .u64 j2k_store_rgb16_mct_param_3, + .param .u64 j2k_store_rgb16_mct_param_4 +) +{ + .reg .pred %p<11>; + .reg .b16 %rs<2>; + .reg .f32 %f<84>; + .reg .b32 %r<72>; + .reg .b64 %rd<26>; + + + ld.param.u64 %rd3, [j2k_store_rgb16_mct_param_0]; + ld.param.u64 %rd4, [j2k_store_rgb16_mct_param_1]; + ld.param.u64 %rd5, [j2k_store_rgb16_mct_param_2]; + ld.param.u64 %rd6, [j2k_store_rgb16_mct_param_3]; + ld.param.u64 %rd7, [j2k_store_rgb16_mct_param_4]; + cvta.to.global.u64 %rd1, %rd6; + cvta.to.global.u64 %rd8, %rd7; + add.s64 %rd2, %rd8, 36; + ld.global.u32 %r1, [%rd8+72]; + ld.global.u32 %r2, [%rd8+76]; + ld.global.u32 %r3, [%rd8+80]; + ld.global.u32 %r17, [%rd8+40]; + ld.global.u32 %r4, [%rd8+36]; + mul.lo.s32 %r18, %r17, %r4; + mov.u32 %r19, %ntid.x; + mov.u32 %r20, %ctaid.x; + mov.u32 %r21, %tid.x; + mad.lo.s32 %r5, %r20, %r19, %r21; + setp.ge.u32 %p1, %r5, %r18; + @%p1 bra $L__BB24_15; + + ld.global.f32 %f1, [%rd2+24]; + ld.global.f32 %f2, [%rd2+28]; + ld.global.f32 %f3, [%rd2+32]; + div.u32 %r22, %r5, %r4; + mul.lo.s32 %r23, %r22, %r4; + sub.s32 %r24, %r5, %r23; + ld.global.u32 %r25, [%rd2+-20]; + add.s32 %r26, %r22, %r25; + ld.global.u32 %r27, [%rd2+-36]; + ld.global.u32 %r28, [%rd2+-24]; + mad.lo.s32 %r29, %r26, %r27, %r28; + add.s32 %r30, %r29, %r24; + ld.global.u32 %r31, [%rd2+-12]; + add.s32 %r32, %r22, %r31; + ld.global.u32 %r33, [%rd2+-32]; + ld.global.u32 %r34, [%rd2+-16]; + mad.lo.s32 %r35, %r32, %r33, %r34; + add.s32 %r36, %r35, %r24; + ld.global.u32 %r37, [%rd2+-4]; + add.s32 %r38, %r22, %r37; + ld.global.u32 %r39, [%rd2+-28]; + ld.global.u32 %r40, [%rd2+-8]; + mad.lo.s32 %r41, %r38, %r39, %r40; + add.s32 %r42, %r41, %r24; + ld.global.u32 %r6, [%rd2+48]; + setp.eq.s32 %p2, %r6, 0; + selp.b32 %r43, 3, 4, %p2; + ld.global.u32 %r44, [%rd2+20]; + add.s32 %r45, %r22, %r44; + ld.global.u32 %r46, [%rd2+8]; + ld.global.u32 %r47, [%rd2+16]; + mad.lo.s32 %r48, %r45, %r46, %r47; + add.s32 %r49, %r48, %r24; + mul.lo.s32 %r7, %r49, %r43; + cvta.to.global.u64 %rd9, %rd3; + mul.wide.u32 %rd10, %r30, 4; + add.s64 %rd11, %rd9, %rd10; + ld.global.f32 %f4, [%rd11]; + cvta.to.global.u64 %rd12, %rd4; + mul.wide.u32 %rd13, %r36, 4; + add.s64 %rd14, %rd12, %rd13; + ld.global.f32 %f5, [%rd14]; + cvta.to.global.u64 %rd15, %rd5; + mul.wide.u32 %rd16, %r42, 4; + add.s64 %rd17, %rd15, %rd16; + ld.global.f32 %f6, [%rd17]; + ld.global.u32 %r50, [%rd2+52]; + setp.eq.s32 %p3, %r50, 0; + @%p3 bra $L__BB24_3; + + mul.rn.f32 %f19, %f6, 0f3FB374BC; + add.rn.f32 %f82, %f4, %f19; + mul.rn.f32 %f20, %f5, 0fBEB031CF; + add.rn.f32 %f21, %f4, %f20; + mul.rn.f32 %f22, %f6, 0fBF36D1E1; + add.rn.f32 %f81, %f21, %f22; + mul.rn.f32 %f23, %f5, 0f3FE2D0E5; + add.rn.f32 %f83, %f4, %f23; + bra.uni $L__BB24_4; + +$L__BB24_3: + add.rn.f32 %f24, %f5, %f6; + mul.rn.f32 %f25, %f24, 0f3E800000; + cvt.rmi.f32.f32 %f26, %f25; + sub.rn.f32 %f81, %f4, %f26; + add.rn.f32 %f82, %f6, %f81; + add.rn.f32 %f83, %f5, %f81; + +$L__BB24_4: + add.rn.f32 %f27, %f1, %f82; + mov.f32 %f28, 0f3F000000; + copysign.f32 %f29, %f27, %f28; + add.rz.f32 %f30, %f27, %f29; + cvt.rzi.f32.f32 %f16, %f30; + setp.gt.u32 %p4, %r1, 15; + @%p4 bra $L__BB24_6; + bra.uni $L__BB24_5; + +$L__BB24_6: + mov.f32 %f41, 0f00000000; + max.f32 %f42, %f16, %f41; + mov.f32 %f43, 0f477FFF00; + min.f32 %f44, %f42, %f43; + cvt.rzi.u32.f32 %r69, %f44; + bra.uni $L__BB24_7; + +$L__BB24_5: + setp.eq.s32 %p5, %r1, 0; + mov.u32 %r51, -1; + shl.b32 %r52, %r51, %r1; + not.b32 %r53, %r52; + selp.b32 %r54, 1, %r53, %p5; + max.u32 %r55, %r54, 1; + cvt.rn.f32.u32 %f31, %r55; + mov.f32 %f32, 0f00000000; + max.f32 %f33, %f16, %f32; + min.f32 %f34, %f33, %f31; + div.rn.f32 %f35, %f34, %f31; + mul.rn.f32 %f36, %f35, 0f477FFF00; + copysign.f32 %f38, %f36, %f28; + add.rz.f32 %f39, %f36, %f38; + cvt.rzi.f32.f32 %f40, %f39; + cvt.rzi.u32.f32 %r69, %f40; + +$L__BB24_7: + mul.wide.u32 %rd18, %r7, 2; + add.s64 %rd19, %rd1, %rd18; + st.global.u16 [%rd19], %r69; + add.rn.f32 %f45, %f2, %f81; + mov.f32 %f46, 0f3F000000; + copysign.f32 %f47, %f45, %f46; + add.rz.f32 %f48, %f45, %f47; + cvt.rzi.f32.f32 %f17, %f48; + setp.gt.u32 %p6, %r2, 15; + @%p6 bra $L__BB24_9; + bra.uni $L__BB24_8; + +$L__BB24_9: + mov.f32 %f59, 0f00000000; + max.f32 %f60, %f17, %f59; + mov.f32 %f61, 0f477FFF00; + min.f32 %f62, %f60, %f61; + cvt.rzi.u32.f32 %r70, %f62; + bra.uni $L__BB24_10; + +$L__BB24_8: + setp.eq.s32 %p7, %r2, 0; + mov.u32 %r56, -1; + shl.b32 %r57, %r56, %r2; + not.b32 %r58, %r57; + selp.b32 %r59, 1, %r58, %p7; + max.u32 %r60, %r59, 1; + cvt.rn.f32.u32 %f49, %r60; + mov.f32 %f50, 0f00000000; + max.f32 %f51, %f17, %f50; + min.f32 %f52, %f51, %f49; + div.rn.f32 %f53, %f52, %f49; + mul.rn.f32 %f54, %f53, 0f477FFF00; + copysign.f32 %f56, %f54, %f46; + add.rz.f32 %f57, %f54, %f56; + cvt.rzi.f32.f32 %f58, %f57; + cvt.rzi.u32.f32 %r70, %f58; + +$L__BB24_10: + add.s32 %r61, %r7, 1; + mul.wide.u32 %rd20, %r61, 2; + add.s64 %rd21, %rd1, %rd20; + st.global.u16 [%rd21], %r70; + add.rn.f32 %f63, %f3, %f83; + mov.f32 %f64, 0f3F000000; + copysign.f32 %f65, %f63, %f64; + add.rz.f32 %f66, %f63, %f65; + cvt.rzi.f32.f32 %f18, %f66; + setp.gt.u32 %p8, %r3, 15; + @%p8 bra $L__BB24_12; + bra.uni $L__BB24_11; + +$L__BB24_12: + mov.f32 %f77, 0f00000000; + max.f32 %f78, %f18, %f77; + mov.f32 %f79, 0f477FFF00; + min.f32 %f80, %f78, %f79; + cvt.rzi.u32.f32 %r71, %f80; + bra.uni $L__BB24_13; + +$L__BB24_11: + setp.eq.s32 %p9, %r3, 0; + mov.u32 %r62, -1; + shl.b32 %r63, %r62, %r3; + not.b32 %r64, %r63; + selp.b32 %r65, 1, %r64, %p9; + max.u32 %r66, %r65, 1; + cvt.rn.f32.u32 %f67, %r66; + mov.f32 %f68, 0f00000000; + max.f32 %f69, %f18, %f68; + min.f32 %f70, %f69, %f67; + div.rn.f32 %f71, %f70, %f67; + mul.rn.f32 %f72, %f71, 0f477FFF00; + copysign.f32 %f74, %f72, %f64; + add.rz.f32 %f75, %f72, %f74; + cvt.rzi.f32.f32 %f76, %f75; + cvt.rzi.u32.f32 %r71, %f76; + +$L__BB24_13: + add.s32 %r67, %r7, 2; + mul.wide.u32 %rd22, %r67, 2; + add.s64 %rd23, %rd1, %rd22; + st.global.u16 [%rd23], %r71; + @%p2 bra $L__BB24_15; + + add.s32 %r68, %r7, 3; + mul.wide.u32 %rd24, %r68, 2; + add.s64 %rd25, %rd1, %rd24; + mov.u16 %rs1, -1; + st.global.u16 [%rd25], %rs1; + +$L__BB24_15: + ret; + +} + // .globl j2k_htj2k_decode_codeblocks +.visible .entry j2k_htj2k_decode_codeblocks( + .param .u64 j2k_htj2k_decode_codeblocks_param_0, + .param .u64 j2k_htj2k_decode_codeblocks_param_1, + .param .u64 j2k_htj2k_decode_codeblocks_param_2, + .param .u64 j2k_htj2k_decode_codeblocks_param_3, + .param .u64 j2k_htj2k_decode_codeblocks_param_4, + .param .u64 j2k_htj2k_decode_codeblocks_param_5, + .param .u64 j2k_htj2k_decode_codeblocks_param_6, + .param .u64 j2k_htj2k_decode_codeblocks_param_7, + .param .u32 j2k_htj2k_decode_codeblocks_param_8 +) +{ + .local .align 16 .b8 __local_depot25[7920]; + .reg .b64 %SP; + .reg .b64 %SPL; + .reg .pred %p<617>; + .reg .b16 %rs<1432>; + .reg .b32 %r<2898>; + .reg .b64 %rd<660>; + + + mov.u64 %SPL, __local_depot25; + ld.param.u64 %rd136, [ j2k_htj2k_decode_codeblocks_param_0]; + ld.param.u64 %rd137, [ j2k_htj2k_decode_codeblocks_param_1]; + ld.param.u64 %rd130, [ j2k_htj2k_decode_codeblocks_param_2]; + ld.param.u64 %rd135, [ j2k_htj2k_decode_codeblocks_param_7]; + ld.param.u32 %r1130, [ j2k_htj2k_decode_codeblocks_param_8]; + cvta.to.global.u64 %rd1, %rd136; + cvta.to.global.u64 %rd2, %rd137; + mov.u32 %r1131, %ntid.x; + mov.u32 %r1132, %ctaid.x; + mov.u32 %r1133, %tid.x; + mad.lo.s32 %r1, %r1132, %r1131, %r1133; + setp.ge.u32 %p1, %r1, %r1130; + @%p1 bra $L__BB25_541; + + cvta.to.global.u64 %rd138, %rd135; + cvta.to.global.u64 %rd139, %rd130; + mul.wide.u32 %rd140, %r1, 52; + add.s64 %rd141, %rd139, %rd140; + ld.global.u32 %r2, [%rd141+4]; + ld.global.u32 %r3, [%rd141+8]; + ld.global.u32 %r4, [%rd141+12]; + ld.global.u32 %r5, [%rd141+16]; + ld.global.u32 %r2885, [%rd141+20]; + ld.global.u32 %r7, [%rd141+24]; + ld.global.u32 %r8, [%rd141+28]; + ld.global.u32 %r1134, [%rd141+32]; + ld.global.u32 %r9, [%rd141+36]; + ld.global.u32 %r10, [%rd141+40]; + ld.global.u32 %r11, [%rd141+48]; + ld.global.u32 %rd4, [%rd141]; + mul.wide.u32 %rd142, %r1, 16; + add.s64 %rd5, %rd138, %rd142; + mov.u32 %r1135, 0; + st.global.u32 [%rd5], %r1135; + st.global.u32 [%rd5+4], %r1135; + st.global.u32 [%rd5+8], %r1135; + st.global.u32 [%rd5+12], %r1135; + setp.gt.u32 %p2, %r1134, 1; + setp.eq.s32 %p3, %r2885, 0; + and.pred %p4, %p3, %p2; + selp.b32 %r12, 1, %r1134, %p4; + setp.eq.s32 %p5, %r2, 0; + setp.eq.s32 %p6, %r3, 0; + or.pred %p7, %p5, %p6; + @%p7 bra $L__BB25_541; + + setp.gt.u32 %p8, %r2, 256; + setp.gt.u32 %p9, %r3, 256; + or.pred %p10, %p8, %p9; + mul.lo.s32 %r1136, %r3, %r2; + setp.gt.u32 %p11, %r1136, 4096; + or.pred %p12, %p10, %p11; + @%p12 bra $L__BB25_540; + bra.uni $L__BB25_3; + +$L__BB25_540: + mov.u32 %r2303, 2; + st.global.u32 [%rd5], %r2303; + mov.u32 %r2304, 1; + st.global.u32 [%rd5+4], %r2304; + mov.u32 %r2305, 0; + st.global.u32 [%rd5+8], %r2305; + st.global.u32 [%rd5+12], %r2305; + bra.uni $L__BB25_541; + +$L__BB25_3: + add.s32 %r1137, %r8, -1; + setp.gt.u32 %p13, %r1137, 30; + @%p13 bra $L__BB25_539; + bra.uni $L__BB25_4; + +$L__BB25_539: + mov.u32 %r2300, 1; + st.global.u32 [%rd5], %r2300; + mov.u32 %r2301, 2; + st.global.u32 [%rd5+4], %r2301; + mov.u32 %r2302, 0; + st.global.u32 [%rd5+8], %r2302; + st.global.u32 [%rd5+12], %r2302; + bra.uni $L__BB25_541; + +$L__BB25_4: + setp.gt.u32 %p14, %r12, 3; + setp.gt.u32 %p15, %r7, 29; + or.pred %p16, %p15, %p14; + @%p16 bra $L__BB25_538; + bra.uni $L__BB25_5; + +$L__BB25_538: + mov.u32 %r2297, 1; + st.global.u32 [%rd5], %r2297; + mov.u32 %r2298, 3; + st.global.u32 [%rd5+4], %r2298; + mov.u32 %r2299, 0; + st.global.u32 [%rd5+8], %r2299; + st.global.u32 [%rd5+12], %r2299; + bra.uni $L__BB25_541; + +$L__BB25_5: + cvt.u64.u32 %rd594, %r5; + cvt.u32.u64 %r1138, %rd594; + setp.gt.u32 %p17, %r12, 1; + setp.eq.s32 %p18, %r7, 29; + and.pred %p19, %p18, %p17; + selp.b32 %r13, 1, %r12, %p19; + setp.lt.u32 %p20, %r1138, 2; + add.s32 %r1139, %r2885, %r1138; + setp.lt.u32 %p21, %r4, %r1139; + or.pred %p22, %p20, %p21; + @%p22 bra $L__BB25_537; + bra.uni $L__BB25_6; + +$L__BB25_537: + mov.u32 %r2294, 1; + st.global.u32 [%rd5], %r2294; + mov.u32 %r2295, 4; + st.global.u32 [%rd5+4], %r2295; + mov.u32 %r2296, 0; + st.global.u32 [%rd5+8], %r2296; + st.global.u32 [%rd5+12], %r2296; + bra.uni $L__BB25_541; + +$L__BB25_6: + add.s32 %r1141, %r1138, -1; + cvt.u64.u32 %rd143, %r1141; + add.s64 %rd144, %rd143, %rd4; + add.s64 %rd145, %rd1, %rd144; + ld.global.u8 %rs667, [%rd145]; + mul.wide.u16 %r1142, %rs667, 16; + add.s32 %r1143, %r1138, -2; + cvt.u64.u32 %rd146, %r1143; + add.s64 %rd147, %rd146, %rd4; + add.s64 %rd148, %rd1, %rd147; + ld.global.u8 %rs1, [%rd148]; + and.b16 %rs668, %rs1, 15; + cvt.u32.u16 %r1144, %rs668; + or.b32 %r14, %r1142, %r1144; + setp.lt.u32 %p23, %r1138, %r14; + add.s32 %r15, %r14, -2; + setp.gt.u32 %p24, %r15, 4077; + or.pred %p25, %p23, %p24; + @%p25 bra $L__BB25_536; + bra.uni $L__BB25_7; + +$L__BB25_536: + mov.u32 %r2291, 1; + st.global.u32 [%rd5], %r2291; + mov.u32 %r2292, 5; + st.global.u32 [%rd5+4], %r2292; + mov.u32 %r2293, 0; + st.global.u32 [%rd5+8], %r2293; + st.global.u32 [%rd5+12], %r2293; + bra.uni $L__BB25_541; + +$L__BB25_7: + add.s32 %r1145, %r3, 1; + shr.u32 %r1146, %r1145, 1; + add.s32 %r1147, %r2, 9; + and.b32 %r1148, %r1147, -8; + setp.gt.u32 %p26, %r1148, 264; + add.s32 %r1149, %r1146, 1; + mul.lo.s32 %r1150, %r1149, %r1148; + setp.gt.u32 %p27, %r1150, 3096; + or.pred %p28, %p26, %p27; + @%p28 bra $L__BB25_535; + bra.uni $L__BB25_8; + +$L__BB25_535: + mov.u32 %r2288, 2; + st.global.u32 [%rd5], %r2288; + mov.u32 %r2289, 6; + st.global.u32 [%rd5+4], %r2289; + mov.u32 %r2290, 0; + st.global.u32 [%rd5+8], %r2290; + st.global.u32 [%rd5+12], %r2290; + bra.uni $L__BB25_541; + +$L__BB25_8: + and.b16 %rs1031, %rs1, 15; + cvt.u32.u16 %r2312, %rs1031; + mul.wide.u16 %r2311, %rs667, 16; + or.b32 %r2310, %r2311, %r2312; + sub.s32 %r16, %r5, %r2310; + add.s32 %r2390, %r2310, -1; + mov.u64 %rd596, 0; + mov.u32 %r2480, 0; + mov.u16 %rs1104, 0; + mov.u64 %rd151, _ZZ20mel_decode_more_runsR10MelDecoderE7MEL_EXP; + mov.u32 %r2389, %r16; + mov.u16 %rs1105, %rs1104; + mov.u16 %rs1139, %rs1104; + mov.u32 %r2362, %r2480; + +$L__BB25_9: + setp.gt.u32 %p30, %r2362, 7; + @%p30 bra $L__BB25_54; + + mul.wide.u32 %rd150, %r2480, 4; + add.s64 %rd152, %rd151, %rd150; + ld.global.nc.u32 %r22, [%rd152]; + and.b16 %rs672, %rs1139, 255; + setp.ne.s16 %p31, %rs672, 0; + mov.u16 %rs1039, %rs1139; + @%p31 bra $L__BB25_14; + + setp.eq.s32 %p32, %r2390, 0; + mov.u16 %rs1036, 255; + @%p32 bra $L__BB25_13; + + cvt.u64.u32 %rd153, %r2389; + add.s64 %rd154, %rd153, %rd4; + add.s64 %rd155, %rd1, %rd154; + ld.global.u8 %rs1036, [%rd155]; + +$L__BB25_13: + setp.ne.s32 %p34, %r2390, 0; + selp.u32 %r1153, 1, 0, %p34; + add.s32 %r2389, %r2389, %r1153; + add.s32 %r1154, %r2390, -1; + selp.b32 %r2390, 0, %r1154, %p32; + setp.eq.s32 %p35, %r2390, 0; + or.b16 %rs674, %rs1036, 15; + selp.b16 %rs1105, %rs674, %rs1036, %p35; + and.b16 %rs675, %rs1105, 255; + mov.u16 %rs676, 8; + sub.s16 %rs1039, %rs676, %rs1104; + setp.eq.s16 %p36, %rs675, 255; + selp.u16 %rs1104, 1, 0, %p36; + +$L__BB25_14: + add.s16 %rs15, %rs1039, -1; + cvt.u32.u16 %r1155, %rs15; + and.b32 %r1156, %r1155, 255; + mov.u32 %r1157, 1; + shl.b32 %r1158, %r1157, %r1156; + cvt.u32.u16 %r1159, %rs1105; + and.b32 %r1160, %r1158, %r1159; + and.b32 %r27, %r1160, 255; + add.s16 %rs1139, %rs1039, -1; + setp.eq.s32 %p37, %r27, 0; + @%p37 bra $L__BB25_16; + + add.s32 %r1161, %r2480, 1; + min.u32 %r2357, %r1161, 12; + mov.u32 %r1162, -1; + shl.b32 %r1163, %r1162, %r22; + shl.b32 %r1164, %r1163, 1; + xor.b32 %r2358, %r1164, -2; + bra.uni $L__BB25_53; + +$L__BB25_16: + cvt.u64.u32 %rd593, %r2480; + add.s64 %rd156, %rd593, -3; + setp.gt.u64 %p38, %rd156, 9; + mov.u32 %r2354, 0; + @%p38 bra $L__BB25_52; + + add.s16 %rs1139, %rs1039, -1; + max.u32 %r30, %r22, 1; + add.s32 %r1168, %r30, -1; + setp.lt.u32 %p39, %r1168, 3; + mov.u32 %r2354, 0; + @%p39 bra $L__BB25_36; + + and.b32 %r2306, %r30, 3; + add.s16 %rs1139, %rs1039, -1; + sub.s32 %r2326, %r30, %r2306; + mov.u32 %r2354, 0; + +$L__BB25_19: + and.b16 %rs678, %rs1139, 255; + setp.ne.s16 %p40, %rs678, 0; + @%p40 bra $L__BB25_23; + + setp.eq.s32 %p41, %r2390, 0; + mov.u16 %rs1043, 255; + @%p41 bra $L__BB25_22; + + cvt.u64.u32 %rd157, %r2389; + add.s64 %rd158, %rd157, %rd4; + add.s64 %rd159, %rd1, %rd158; + ld.global.u8 %rs1043, [%rd159]; + +$L__BB25_22: + setp.ne.s32 %p43, %r2390, 0; + selp.u32 %r1170, 1, 0, %p43; + add.s32 %r2389, %r2389, %r1170; + add.s32 %r1171, %r2390, -1; + selp.b32 %r2390, 0, %r1171, %p41; + setp.eq.s32 %p44, %r2390, 0; + or.b16 %rs680, %rs1043, 15; + selp.b16 %rs1105, %rs680, %rs1043, %p44; + and.b16 %rs681, %rs1105, 255; + mov.u16 %rs682, 8; + sub.s16 %rs1139, %rs682, %rs1104; + setp.eq.s16 %p45, %rs681, 255; + selp.u16 %rs1104, 1, 0, %p45; + +$L__BB25_23: + add.s16 %rs1050, %rs1139, -1; + and.b16 %rs683, %rs1050, 255; + cvt.u32.u16 %r1172, %rs1050; + and.b32 %r1173, %r1172, 255; + cvt.u32.u16 %r1174, %rs1105; + and.b32 %r2332, %r1174, 255; + shr.u32 %r1175, %r2332, %r1173; + and.b32 %r1176, %r1175, 1; + bfi.b32 %r42, %r2354, %r1176, 1, 31; + setp.ne.s16 %p46, %rs683, 0; + @%p46 bra $L__BB25_27; + + setp.eq.s32 %p47, %r2390, 0; + mov.u16 %rs1047, 255; + @%p47 bra $L__BB25_26; + + cvt.u64.u32 %rd160, %r2389; + add.s64 %rd161, %rd160, %rd4; + add.s64 %rd162, %rd1, %rd161; + ld.global.u8 %rs1047, [%rd162]; + +$L__BB25_26: + setp.ne.s32 %p49, %r2390, 0; + selp.u32 %r1177, 1, 0, %p49; + add.s32 %r2389, %r2389, %r1177; + add.s32 %r1178, %r2390, -1; + selp.b32 %r2390, 0, %r1178, %p47; + setp.eq.s32 %p50, %r2390, 0; + or.b16 %rs685, %rs1047, 15; + selp.b16 %rs1105, %rs685, %rs1047, %p50; + and.b16 %rs686, %rs1105, 255; + mov.u16 %rs687, 8; + sub.s16 %rs1050, %rs687, %rs1104; + setp.eq.s16 %p51, %rs686, 255; + selp.u16 %rs1104, 1, 0, %p51; + cvt.u32.u16 %r1179, %rs1105; + and.b32 %r2332, %r1179, 255; + +$L__BB25_27: + add.s16 %rs1054, %rs1050, -1; + and.b16 %rs688, %rs1054, 255; + cvt.u32.u16 %r1180, %rs1054; + and.b32 %r1181, %r1180, 255; + shr.u32 %r1182, %r2332, %r1181; + and.b32 %r1183, %r1182, 1; + bfi.b32 %r49, %r42, %r1183, 1, 31; + setp.ne.s16 %p52, %rs688, 0; + @%p52 bra $L__BB25_31; + + setp.eq.s32 %p53, %r2390, 0; + mov.u16 %rs1051, 255; + @%p53 bra $L__BB25_30; + + cvt.u64.u32 %rd163, %r2389; + add.s64 %rd164, %rd163, %rd4; + add.s64 %rd165, %rd1, %rd164; + ld.global.u8 %rs1051, [%rd165]; + +$L__BB25_30: + setp.ne.s32 %p55, %r2390, 0; + selp.u32 %r1184, 1, 0, %p55; + add.s32 %r2389, %r2389, %r1184; + add.s32 %r1185, %r2390, -1; + selp.b32 %r2390, 0, %r1185, %p53; + setp.eq.s32 %p56, %r2390, 0; + or.b16 %rs690, %rs1051, 15; + selp.b16 %rs1105, %rs690, %rs1051, %p56; + and.b16 %rs691, %rs1105, 255; + mov.u16 %rs692, 8; + sub.s16 %rs1054, %rs692, %rs1104; + setp.eq.s16 %p57, %rs691, 255; + selp.u16 %rs1104, 1, 0, %p57; + cvt.u32.u16 %r1186, %rs1105; + and.b32 %r2332, %r1186, 255; + +$L__BB25_31: + add.s16 %rs1058, %rs1054, -1; + and.b16 %rs693, %rs1058, 255; + cvt.u32.u16 %r1187, %rs1058; + and.b32 %r1188, %r1187, 255; + shr.u32 %r1189, %r2332, %r1188; + and.b32 %r1190, %r1189, 1; + bfi.b32 %r56, %r49, %r1190, 1, 31; + setp.ne.s16 %p58, %rs693, 0; + @%p58 bra $L__BB25_35; + + setp.eq.s32 %p59, %r2390, 0; + mov.u16 %rs1055, 255; + @%p59 bra $L__BB25_34; + + cvt.u64.u32 %rd166, %r2389; + add.s64 %rd167, %rd166, %rd4; + add.s64 %rd168, %rd1, %rd167; + ld.global.u8 %rs1055, [%rd168]; + +$L__BB25_34: + setp.ne.s32 %p61, %r2390, 0; + selp.u32 %r1191, 1, 0, %p61; + add.s32 %r2389, %r2389, %r1191; + add.s32 %r1192, %r2390, -1; + selp.b32 %r2390, 0, %r1192, %p59; + setp.eq.s32 %p62, %r2390, 0; + or.b16 %rs695, %rs1055, 15; + selp.b16 %rs1105, %rs695, %rs1055, %p62; + and.b16 %rs696, %rs1105, 255; + mov.u16 %rs697, 8; + sub.s16 %rs1058, %rs697, %rs1104; + setp.eq.s16 %p63, %rs696, 255; + selp.u16 %rs1104, 1, 0, %p63; + cvt.u32.u16 %r1193, %rs1105; + and.b32 %r2332, %r1193, 255; + +$L__BB25_35: + add.s16 %rs1139, %rs1058, -1; + cvt.u32.u16 %r1194, %rs1139; + and.b32 %r1195, %r1194, 255; + shr.u32 %r1196, %r2332, %r1195; + and.b32 %r1197, %r1196, 1; + bfi.b32 %r2354, %r56, %r1197, 1, 31; + add.s32 %r2326, %r2326, -4; + setp.ne.s32 %p64, %r2326, 0; + @%p64 bra $L__BB25_19; + +$L__BB25_36: + and.b32 %r2307, %r30, 3; + setp.eq.s32 %p65, %r2307, 0; + @%p65 bra $L__BB25_52; + + and.b16 %rs698, %rs1139, 255; + setp.ne.s16 %p66, %rs698, 0; + @%p66 bra $L__BB25_41; + + setp.eq.s32 %p67, %r2390, 0; + mov.u16 %rs1065, 255; + @%p67 bra $L__BB25_40; + + cvt.u64.u32 %rd169, %r2389; + add.s64 %rd170, %rd169, %rd4; + add.s64 %rd171, %rd1, %rd170; + ld.global.u8 %rs1065, [%rd171]; + +$L__BB25_40: + setp.ne.s32 %p69, %r2390, 0; + selp.u32 %r1198, 1, 0, %p69; + add.s32 %r2389, %r2389, %r1198; + add.s32 %r1199, %r2390, -1; + selp.b32 %r2390, 0, %r1199, %p67; + setp.eq.s32 %p70, %r2390, 0; + or.b16 %rs700, %rs1065, 15; + selp.b16 %rs1105, %rs700, %rs1065, %p70; + and.b16 %rs701, %rs1105, 255; + mov.u16 %rs702, 8; + sub.s16 %rs1139, %rs702, %rs1104; + setp.eq.s16 %p71, %rs701, 255; + selp.u16 %rs1104, 1, 0, %p71; + +$L__BB25_41: + and.b32 %r2308, %r30, 3; + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1200, %rs1139; + and.b32 %r1201, %r1200, 255; + cvt.u32.u16 %r1202, %rs1105; + and.b32 %r2349, %r1202, 255; + shr.u32 %r1203, %r2349, %r1201; + and.b32 %r1204, %r1203, 1; + bfi.b32 %r2354, %r2354, %r1204, 1, 31; + setp.eq.s32 %p72, %r2308, 1; + @%p72 bra $L__BB25_52; + + and.b16 %rs703, %rs1139, 255; + setp.ne.s16 %p73, %rs703, 0; + @%p73 bra $L__BB25_46; + + setp.eq.s32 %p74, %r2390, 0; + mov.u16 %rs1069, 255; + @%p74 bra $L__BB25_45; + + cvt.u64.u32 %rd172, %r2389; + add.s64 %rd173, %rd172, %rd4; + add.s64 %rd174, %rd1, %rd173; + ld.global.u8 %rs1069, [%rd174]; + +$L__BB25_45: + setp.ne.s32 %p76, %r2390, 0; + selp.u32 %r1205, 1, 0, %p76; + add.s32 %r2389, %r2389, %r1205; + add.s32 %r1206, %r2390, -1; + selp.b32 %r2390, 0, %r1206, %p74; + setp.eq.s32 %p77, %r2390, 0; + or.b16 %rs705, %rs1069, 15; + selp.b16 %rs1105, %rs705, %rs1069, %p77; + and.b16 %rs706, %rs1105, 255; + mov.u16 %rs707, 8; + sub.s16 %rs1139, %rs707, %rs1104; + setp.eq.s16 %p78, %rs706, 255; + selp.u16 %rs1104, 1, 0, %p78; + cvt.u32.u16 %r1207, %rs1105; + and.b32 %r2349, %r1207, 255; + +$L__BB25_46: + and.b32 %r2309, %r30, 3; + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1208, %rs1139; + and.b32 %r1209, %r1208, 255; + shr.u32 %r1210, %r2349, %r1209; + and.b32 %r1211, %r1210, 1; + bfi.b32 %r2354, %r2354, %r1211, 1, 31; + setp.eq.s32 %p79, %r2309, 2; + @%p79 bra $L__BB25_52; + + and.b16 %rs708, %rs1139, 255; + setp.ne.s16 %p80, %rs708, 0; + @%p80 bra $L__BB25_51; + + setp.eq.s32 %p81, %r2390, 0; + mov.u16 %rs1073, 255; + @%p81 bra $L__BB25_50; + + cvt.u64.u32 %rd175, %r2389; + add.s64 %rd176, %rd175, %rd4; + add.s64 %rd177, %rd1, %rd176; + ld.global.u8 %rs1073, [%rd177]; + +$L__BB25_50: + setp.ne.s32 %p83, %r2390, 0; + selp.u32 %r1212, 1, 0, %p83; + add.s32 %r2389, %r2389, %r1212; + add.s32 %r1213, %r2390, -1; + selp.b32 %r2390, 0, %r1213, %p81; + setp.eq.s32 %p84, %r2390, 0; + or.b16 %rs710, %rs1073, 15; + selp.b16 %rs1105, %rs710, %rs1073, %p84; + and.b16 %rs711, %rs1105, 255; + mov.u16 %rs712, 8; + sub.s16 %rs1139, %rs712, %rs1104; + setp.eq.s16 %p85, %rs711, 255; + selp.u16 %rs1104, 1, 0, %p85; + cvt.u32.u16 %r1214, %rs1105; + and.b32 %r2349, %r1214, 255; + +$L__BB25_51: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1215, %rs1139; + and.b32 %r1216, %r1215, 255; + shr.u32 %r1217, %r2349, %r1216; + and.b32 %r1218, %r1217, 1; + bfi.b32 %r2354, %r2354, %r1218, 1, 31; + +$L__BB25_52: + shl.b32 %r1219, %r2354, 1; + or.b32 %r2358, %r1219, 1; + add.s32 %r1220, %r2480, -1; + setp.eq.s32 %p86, %r2480, 0; + selp.b32 %r2357, 0, %r1220, %p86; + +$L__BB25_53: + mul.lo.s32 %r1221, %r2362, 7; + cvt.u64.u32 %rd178, %r2358; + shl.b64 %rd179, %rd178, %r1221; + or.b64 %rd596, %rd179, %rd596; + setp.ne.s32 %p87, %r2480, 12; + setp.ne.s32 %p88, %r27, 0; + or.pred %p89, %p87, %p88; + add.s32 %r2362, %r2362, 1; + setp.lt.u32 %p90, %r2362, 8; + or.pred %p91, %p90, %p89; + mov.u32 %r2480, %r2357; + @%p91 bra $L__BB25_9; + +$L__BB25_54: + and.b16 %rs1032, %rs1, 15; + cvt.u32.u16 %r2316, %rs1032; + mul.wide.u16 %r2315, %rs667, 16; + or.b32 %r2314, %r2315, %r2316; + add.s32 %r2559, %r2314, -2; + setp.gt.u16 %p616, %rs1, 143; + selp.u16 %rs1271, 1, 0, %p616; + shr.u16 %rs1025, %rs1, 4; + ld.param.u64 %rd589, [ j2k_htj2k_decode_codeblocks_param_3]; + ld.param.u64 %rd588, [ j2k_htj2k_decode_codeblocks_param_5]; + add.s32 %r2481, %r2362, -1; + shr.u64 %rd606, %rd596, 7; + cvt.u32.u64 %r1224, %rd596; + and.b32 %r2477, %r1224, 127; + cvt.u64.u16 %rd615, %rs1025; + and.b64 %rd180, %rd615, 7; + setp.eq.s64 %p92, %rd180, 7; + selp.b32 %r2560, 3, 4, %p92; + add.s32 %r2558, %r5, -3; + cvta.to.global.u64 %rd12, %rd588; + cvta.to.global.u64 %rd13, %rd589; + add.u64 %rd14, %SPL, 0; + mov.u32 %r2363, 0; + mov.u32 %r2364, %r2363; + +$L__BB25_55: + setp.gt.u32 %p93, %r2560, 31; + @%p93 bra $L__BB25_59; + +$L__BB25_56: + setp.eq.s32 %p94, %r2559, 0; + mov.u16 %rs1091, 0; + @%p94 bra $L__BB25_58; + + cvt.s64.s32 %rd182, %r2558; + add.s64 %rd183, %rd182, %rd4; + add.s64 %rd184, %rd1, %rd183; + ld.global.u8 %rs1091, [%rd184]; + +$L__BB25_58: + setp.ne.s32 %p96, %r2559, 0; + selp.b32 %r1225, -1, 0, %p96; + add.s32 %r2558, %r2558, %r1225; + add.s32 %r1226, %r2559, -1; + selp.b32 %r2559, 0, %r1226, %p94; + and.b16 %rs714, %rs1091, 255; + and.b16 %rs715, %rs1091, 127; + setp.eq.s16 %p97, %rs715, 127; + and.b16 %rs716, %rs1271, 255; + setp.ne.s16 %p98, %rs716, 0; + and.pred %p99, %p98, %p97; + selp.b32 %r1227, 7, 8, %p99; + cvt.u64.u16 %rd185, %rs1091; + and.b64 %rd186, %rd185, 255; + shl.b64 %rd187, %rd186, %r2560; + or.b64 %rd615, %rd187, %rd615; + add.s32 %r2560, %r1227, %r2560; + setp.gt.u16 %p100, %rs714, 143; + selp.u16 %rs1271, 1, 0, %p100; + setp.lt.u32 %p101, %r2560, 33; + @%p101 bra $L__BB25_56; + +$L__BB25_59: + cvt.u32.u64 %r1228, %rd615; + and.b32 %r1229, %r1228, 127; + add.s32 %r1230, %r1229, %r2363; + mul.wide.u32 %rd188, %r1230, 2; + add.s64 %rd189, %rd13, %rd188; + ld.global.u16 %r2430, [%rd189]; + setp.ne.s32 %p102, %r2363, 0; + @%p102 bra $L__BB25_109; + + add.s32 %r129, %r2477, -2; + setp.eq.s32 %p103, %r129, -1; + selp.b32 %r2430, %r2430, 0, %p103; + setp.gt.s32 %p104, %r2477, 1; + mov.u32 %r2477, %r129; + @%p104 bra $L__BB25_109; + + setp.ne.s32 %p105, %r2481, 0; + @%p105 bra $L__BB25_108; + + mov.u32 %r2481, 0; + +$L__BB25_63: + setp.gt.u32 %p106, %r2481, 7; + @%p106 bra $L__BB25_108; + + cvt.u64.u32 %rd21, %r2480; + mul.wide.u32 %rd190, %r2480, 4; + add.s64 %rd192, %rd151, %rd190; + ld.global.nc.u32 %r135, [%rd192]; + and.b16 %rs717, %rs1139, 255; + setp.ne.s16 %p107, %rs717, 0; + @%p107 bra $L__BB25_68; + + setp.eq.s32 %p108, %r2390, 0; + mov.u16 %rs1096, 255; + @%p108 bra $L__BB25_67; + + cvt.u64.u32 %rd193, %r2389; + add.s64 %rd194, %rd193, %rd4; + add.s64 %rd195, %rd1, %rd194; + ld.global.u8 %rs1096, [%rd195]; + +$L__BB25_67: + setp.ne.s32 %p110, %r2390, 0; + selp.u32 %r1232, 1, 0, %p110; + add.s32 %r2389, %r2389, %r1232; + add.s32 %r1233, %r2390, -1; + selp.b32 %r2390, 0, %r1233, %p108; + setp.eq.s32 %p111, %r2390, 0; + or.b16 %rs719, %rs1096, 15; + selp.b16 %rs1105, %rs719, %rs1096, %p111; + and.b16 %rs720, %rs1105, 255; + mov.u16 %rs721, 8; + sub.s16 %rs1139, %rs721, %rs1104; + setp.eq.s16 %p112, %rs720, 255; + selp.u16 %rs1104, 1, 0, %p112; + +$L__BB25_68: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1234, %rs1139; + and.b32 %r1235, %r1234, 255; + mov.u32 %r1236, 1; + shl.b32 %r1237, %r1236, %r1235; + cvt.u32.u16 %r1238, %rs1105; + and.b32 %r1239, %r1237, %r1238; + and.b32 %r140, %r1239, 255; + setp.eq.s32 %p113, %r140, 0; + @%p113 bra $L__BB25_70; + + add.s32 %r1240, %r2480, 1; + min.u32 %r2419, %r1240, 12; + mov.u32 %r1241, -1; + shl.b32 %r1242, %r1241, %r135; + shl.b32 %r1243, %r1242, 1; + xor.b32 %r2420, %r1243, -2; + bra.uni $L__BB25_107; + +$L__BB25_70: + add.s64 %rd196, %rd21, -3; + setp.gt.u64 %p114, %rd196, 9; + mov.u32 %r2416, 0; + @%p114 bra $L__BB25_106; + + max.u32 %r143, %r135, 1; + add.s32 %r1247, %r143, -1; + and.b32 %r144, %r143, 3; + setp.lt.u32 %p115, %r1247, 3; + mov.u32 %r2416, 0; + @%p115 bra $L__BB25_90; + + sub.s32 %r2388, %r143, %r144; + mov.u32 %r2416, 0; + +$L__BB25_73: + and.b16 %rs723, %rs1139, 255; + setp.ne.s16 %p116, %rs723, 0; + @%p116 bra $L__BB25_77; + + setp.eq.s32 %p117, %r2390, 0; + mov.u16 %rs1103, 255; + @%p117 bra $L__BB25_76; + + cvt.u64.u32 %rd197, %r2389; + add.s64 %rd198, %rd197, %rd4; + add.s64 %rd199, %rd1, %rd198; + ld.global.u8 %rs1103, [%rd199]; + +$L__BB25_76: + setp.ne.s32 %p119, %r2390, 0; + selp.u32 %r1249, 1, 0, %p119; + add.s32 %r2389, %r2389, %r1249; + add.s32 %r1250, %r2390, -1; + selp.b32 %r2390, 0, %r1250, %p117; + setp.eq.s32 %p120, %r2390, 0; + or.b16 %rs725, %rs1103, 15; + selp.b16 %rs1105, %rs725, %rs1103, %p120; + and.b16 %rs726, %rs1105, 255; + mov.u16 %rs727, 8; + sub.s16 %rs1139, %rs727, %rs1104; + setp.eq.s16 %p121, %rs726, 255; + selp.u16 %rs1104, 1, 0, %p121; + +$L__BB25_77: + add.s16 %rs1110, %rs1139, -1; + and.b16 %rs728, %rs1110, 255; + cvt.u32.u16 %r1251, %rs1110; + and.b32 %r1252, %r1251, 255; + cvt.u32.u16 %r1253, %rs1105; + and.b32 %r2394, %r1253, 255; + shr.u32 %r1254, %r2394, %r1252; + and.b32 %r1255, %r1254, 1; + bfi.b32 %r155, %r2416, %r1255, 1, 31; + setp.ne.s16 %p122, %rs728, 0; + @%p122 bra $L__BB25_81; + + setp.eq.s32 %p123, %r2390, 0; + mov.u16 %rs1107, 255; + @%p123 bra $L__BB25_80; + + cvt.u64.u32 %rd200, %r2389; + add.s64 %rd201, %rd200, %rd4; + add.s64 %rd202, %rd1, %rd201; + ld.global.u8 %rs1107, [%rd202]; + +$L__BB25_80: + setp.ne.s32 %p125, %r2390, 0; + selp.u32 %r1256, 1, 0, %p125; + add.s32 %r2389, %r2389, %r1256; + add.s32 %r1257, %r2390, -1; + selp.b32 %r2390, 0, %r1257, %p123; + setp.eq.s32 %p126, %r2390, 0; + or.b16 %rs730, %rs1107, 15; + selp.b16 %rs1105, %rs730, %rs1107, %p126; + and.b16 %rs731, %rs1105, 255; + mov.u16 %rs732, 8; + sub.s16 %rs1110, %rs732, %rs1104; + setp.eq.s16 %p127, %rs731, 255; + selp.u16 %rs1104, 1, 0, %p127; + cvt.u32.u16 %r1258, %rs1105; + and.b32 %r2394, %r1258, 255; + +$L__BB25_81: + add.s16 %rs1114, %rs1110, -1; + and.b16 %rs733, %rs1114, 255; + cvt.u32.u16 %r1259, %rs1114; + and.b32 %r1260, %r1259, 255; + shr.u32 %r1261, %r2394, %r1260; + and.b32 %r1262, %r1261, 1; + bfi.b32 %r162, %r155, %r1262, 1, 31; + setp.ne.s16 %p128, %rs733, 0; + @%p128 bra $L__BB25_85; + + setp.eq.s32 %p129, %r2390, 0; + mov.u16 %rs1111, 255; + @%p129 bra $L__BB25_84; + + cvt.u64.u32 %rd203, %r2389; + add.s64 %rd204, %rd203, %rd4; + add.s64 %rd205, %rd1, %rd204; + ld.global.u8 %rs1111, [%rd205]; + +$L__BB25_84: + setp.ne.s32 %p131, %r2390, 0; + selp.u32 %r1263, 1, 0, %p131; + add.s32 %r2389, %r2389, %r1263; + add.s32 %r1264, %r2390, -1; + selp.b32 %r2390, 0, %r1264, %p129; + setp.eq.s32 %p132, %r2390, 0; + or.b16 %rs735, %rs1111, 15; + selp.b16 %rs1105, %rs735, %rs1111, %p132; + and.b16 %rs736, %rs1105, 255; + mov.u16 %rs737, 8; + sub.s16 %rs1114, %rs737, %rs1104; + setp.eq.s16 %p133, %rs736, 255; + selp.u16 %rs1104, 1, 0, %p133; + cvt.u32.u16 %r1265, %rs1105; + and.b32 %r2394, %r1265, 255; + +$L__BB25_85: + add.s16 %rs1118, %rs1114, -1; + and.b16 %rs738, %rs1118, 255; + cvt.u32.u16 %r1266, %rs1118; + and.b32 %r1267, %r1266, 255; + shr.u32 %r1268, %r2394, %r1267; + and.b32 %r1269, %r1268, 1; + bfi.b32 %r169, %r162, %r1269, 1, 31; + setp.ne.s16 %p134, %rs738, 0; + @%p134 bra $L__BB25_89; + + setp.eq.s32 %p135, %r2390, 0; + mov.u16 %rs1115, 255; + @%p135 bra $L__BB25_88; + + cvt.u64.u32 %rd206, %r2389; + add.s64 %rd207, %rd206, %rd4; + add.s64 %rd208, %rd1, %rd207; + ld.global.u8 %rs1115, [%rd208]; + +$L__BB25_88: + setp.ne.s32 %p137, %r2390, 0; + selp.u32 %r1270, 1, 0, %p137; + add.s32 %r2389, %r2389, %r1270; + add.s32 %r1271, %r2390, -1; + selp.b32 %r2390, 0, %r1271, %p135; + setp.eq.s32 %p138, %r2390, 0; + or.b16 %rs740, %rs1115, 15; + selp.b16 %rs1105, %rs740, %rs1115, %p138; + and.b16 %rs741, %rs1105, 255; + mov.u16 %rs742, 8; + sub.s16 %rs1118, %rs742, %rs1104; + setp.eq.s16 %p139, %rs741, 255; + selp.u16 %rs1104, 1, 0, %p139; + cvt.u32.u16 %r1272, %rs1105; + and.b32 %r2394, %r1272, 255; + +$L__BB25_89: + add.s16 %rs1139, %rs1118, -1; + cvt.u32.u16 %r1273, %rs1139; + and.b32 %r1274, %r1273, 255; + shr.u32 %r1275, %r2394, %r1274; + and.b32 %r1276, %r1275, 1; + bfi.b32 %r2416, %r169, %r1276, 1, 31; + add.s32 %r2388, %r2388, -4; + setp.ne.s32 %p140, %r2388, 0; + @%p140 bra $L__BB25_73; + +$L__BB25_90: + setp.eq.s32 %p141, %r144, 0; + @%p141 bra $L__BB25_106; + + and.b16 %rs743, %rs1139, 255; + setp.ne.s16 %p142, %rs743, 0; + @%p142 bra $L__BB25_95; + + setp.eq.s32 %p143, %r2390, 0; + mov.u16 %rs1125, 255; + @%p143 bra $L__BB25_94; + + cvt.u64.u32 %rd209, %r2389; + add.s64 %rd210, %rd209, %rd4; + add.s64 %rd211, %rd1, %rd210; + ld.global.u8 %rs1125, [%rd211]; + +$L__BB25_94: + setp.ne.s32 %p145, %r2390, 0; + selp.u32 %r1277, 1, 0, %p145; + add.s32 %r2389, %r2389, %r1277; + add.s32 %r1278, %r2390, -1; + selp.b32 %r2390, 0, %r1278, %p143; + setp.eq.s32 %p146, %r2390, 0; + or.b16 %rs745, %rs1125, 15; + selp.b16 %rs1105, %rs745, %rs1125, %p146; + and.b16 %rs746, %rs1105, 255; + mov.u16 %rs747, 8; + sub.s16 %rs1139, %rs747, %rs1104; + setp.eq.s16 %p147, %rs746, 255; + selp.u16 %rs1104, 1, 0, %p147; + +$L__BB25_95: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1279, %rs1139; + and.b32 %r1280, %r1279, 255; + cvt.u32.u16 %r1281, %rs1105; + and.b32 %r2411, %r1281, 255; + shr.u32 %r1282, %r2411, %r1280; + and.b32 %r1283, %r1282, 1; + bfi.b32 %r2416, %r2416, %r1283, 1, 31; + setp.eq.s32 %p148, %r144, 1; + @%p148 bra $L__BB25_106; + + and.b16 %rs748, %rs1139, 255; + setp.ne.s16 %p149, %rs748, 0; + @%p149 bra $L__BB25_100; + + setp.eq.s32 %p150, %r2390, 0; + mov.u16 %rs1129, 255; + @%p150 bra $L__BB25_99; + + cvt.u64.u32 %rd212, %r2389; + add.s64 %rd213, %rd212, %rd4; + add.s64 %rd214, %rd1, %rd213; + ld.global.u8 %rs1129, [%rd214]; + +$L__BB25_99: + setp.ne.s32 %p152, %r2390, 0; + selp.u32 %r1284, 1, 0, %p152; + add.s32 %r2389, %r2389, %r1284; + add.s32 %r1285, %r2390, -1; + selp.b32 %r2390, 0, %r1285, %p150; + setp.eq.s32 %p153, %r2390, 0; + or.b16 %rs750, %rs1129, 15; + selp.b16 %rs1105, %rs750, %rs1129, %p153; + and.b16 %rs751, %rs1105, 255; + mov.u16 %rs752, 8; + sub.s16 %rs1139, %rs752, %rs1104; + setp.eq.s16 %p154, %rs751, 255; + selp.u16 %rs1104, 1, 0, %p154; + cvt.u32.u16 %r1286, %rs1105; + and.b32 %r2411, %r1286, 255; + +$L__BB25_100: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1287, %rs1139; + and.b32 %r1288, %r1287, 255; + shr.u32 %r1289, %r2411, %r1288; + and.b32 %r1290, %r1289, 1; + bfi.b32 %r2416, %r2416, %r1290, 1, 31; + setp.eq.s32 %p155, %r144, 2; + @%p155 bra $L__BB25_106; + + and.b16 %rs753, %rs1139, 255; + setp.ne.s16 %p156, %rs753, 0; + @%p156 bra $L__BB25_105; + + setp.eq.s32 %p157, %r2390, 0; + mov.u16 %rs1133, 255; + @%p157 bra $L__BB25_104; + + cvt.u64.u32 %rd215, %r2389; + add.s64 %rd216, %rd215, %rd4; + add.s64 %rd217, %rd1, %rd216; + ld.global.u8 %rs1133, [%rd217]; + +$L__BB25_104: + setp.ne.s32 %p159, %r2390, 0; + selp.u32 %r1291, 1, 0, %p159; + add.s32 %r2389, %r2389, %r1291; + add.s32 %r1292, %r2390, -1; + selp.b32 %r2390, 0, %r1292, %p157; + setp.eq.s32 %p160, %r2390, 0; + or.b16 %rs755, %rs1133, 15; + selp.b16 %rs1105, %rs755, %rs1133, %p160; + and.b16 %rs756, %rs1105, 255; + mov.u16 %rs757, 8; + sub.s16 %rs1139, %rs757, %rs1104; + setp.eq.s16 %p161, %rs756, 255; + selp.u16 %rs1104, 1, 0, %p161; + cvt.u32.u16 %r1293, %rs1105; + and.b32 %r2411, %r1293, 255; + +$L__BB25_105: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1294, %rs1139; + and.b32 %r1295, %r1294, 255; + shr.u32 %r1296, %r2411, %r1295; + and.b32 %r1297, %r1296, 1; + bfi.b32 %r2416, %r2416, %r1297, 1, 31; + +$L__BB25_106: + shl.b32 %r1298, %r2416, 1; + or.b32 %r2420, %r1298, 1; + add.s32 %r1299, %r2480, -1; + setp.eq.s32 %p162, %r2480, 0; + selp.b32 %r2419, 0, %r1299, %p162; + +$L__BB25_107: + mul.lo.s32 %r1300, %r2481, 7; + cvt.u64.u32 %rd218, %r2420; + shl.b64 %rd219, %rd218, %r1300; + or.b64 %rd606, %rd219, %rd606; + setp.ne.s32 %p163, %r2480, 12; + setp.ne.s32 %p164, %r140, 0; + or.pred %p165, %p163, %p164; + add.s32 %r2481, %r2481, 1; + setp.lt.u32 %p166, %r2481, 8; + or.pred %p167, %p166, %p165; + mov.u32 %r2480, %r2419; + @%p167 bra $L__BB25_63; + +$L__BB25_108: + cvt.u32.u64 %r1301, %rd606; + and.b32 %r2477, %r1301, 127; + shr.u64 %rd606, %rd606, 7; + add.s32 %r2481, %r2481, -1; + +$L__BB25_109: + mul.wide.u32 %rd220, %r2364, 2; + add.s64 %rd26, %rd14, %rd220; + st.local.u16 [%rd26], %r2430; + shl.b32 %r1302, %r2430, 3; + and.b32 %r1303, %r1302, 128; + shl.b32 %r1304, %r2430, 2; + and.b32 %r1305, %r1304, 896; + or.b32 %r1306, %r1303, %r1305; + and.b32 %r1307, %r2430, 7; + shr.u64 %rd27, %rd615, %r1307; + sub.s32 %r226, %r2560, %r1307; + cvt.u32.u64 %r1308, %rd27; + and.b32 %r1309, %r1308, 127; + or.b32 %r1310, %r1309, %r1306; + mul.wide.u32 %rd221, %r1310, 2; + add.s64 %rd222, %rd13, %rd221; + ld.global.u16 %r2482, [%rd222]; + setp.ne.s32 %p168, %r1306, 0; + add.s32 %r228, %r2364, 2; + setp.ge.u32 %p169, %r228, %r2; + or.pred %p170, %p169, %p168; + @%p170 bra $L__BB25_159; + + add.s32 %r229, %r2477, -2; + setp.eq.s32 %p171, %r229, -1; + selp.b32 %r2482, %r2482, 0, %p171; + setp.gt.s32 %p172, %r2477, 1; + mov.u32 %r2477, %r229; + @%p172 bra $L__BB25_159; + + setp.ne.s32 %p173, %r2481, 0; + @%p173 bra $L__BB25_158; + + mov.u32 %r2481, 0; + +$L__BB25_113: + setp.gt.u32 %p174, %r2481, 7; + @%p174 bra $L__BB25_158; + + cvt.u64.u32 %rd29, %r2480; + mul.wide.u32 %rd223, %r2480, 4; + add.s64 %rd225, %rd151, %rd223; + ld.global.nc.u32 %r235, [%rd225]; + and.b16 %rs758, %rs1139, 255; + setp.ne.s16 %p175, %rs758, 0; + @%p175 bra $L__BB25_118; + + setp.eq.s32 %p176, %r2390, 0; + mov.u16 %rs1152, 255; + @%p176 bra $L__BB25_117; + + cvt.u64.u32 %rd226, %r2389; + add.s64 %rd227, %rd226, %rd4; + add.s64 %rd228, %rd1, %rd227; + ld.global.u8 %rs1152, [%rd228]; + +$L__BB25_117: + setp.ne.s32 %p178, %r2390, 0; + selp.u32 %r1312, 1, 0, %p178; + add.s32 %r2389, %r2389, %r1312; + add.s32 %r1313, %r2390, -1; + selp.b32 %r2390, 0, %r1313, %p176; + setp.eq.s32 %p179, %r2390, 0; + or.b16 %rs760, %rs1152, 15; + selp.b16 %rs1105, %rs760, %rs1152, %p179; + and.b16 %rs761, %rs1105, 255; + mov.u16 %rs762, 8; + sub.s16 %rs1139, %rs762, %rs1104; + setp.eq.s16 %p180, %rs761, 255; + selp.u16 %rs1104, 1, 0, %p180; + +$L__BB25_118: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1314, %rs1139; + and.b32 %r1315, %r1314, 255; + mov.u32 %r1316, 1; + shl.b32 %r1317, %r1316, %r1315; + cvt.u32.u16 %r1318, %rs1105; + and.b32 %r1319, %r1317, %r1318; + and.b32 %r240, %r1319, 255; + setp.eq.s32 %p181, %r240, 0; + @%p181 bra $L__BB25_120; + + add.s32 %r1320, %r2480, 1; + min.u32 %r2471, %r1320, 12; + mov.u32 %r1321, -1; + shl.b32 %r1322, %r1321, %r235; + shl.b32 %r1323, %r1322, 1; + xor.b32 %r2472, %r1323, -2; + bra.uni $L__BB25_157; + +$L__BB25_120: + add.s64 %rd229, %rd29, -3; + setp.gt.u64 %p182, %rd229, 9; + mov.u32 %r2468, 0; + @%p182 bra $L__BB25_156; + + max.u32 %r243, %r235, 1; + add.s32 %r1327, %r243, -1; + and.b32 %r244, %r243, 3; + setp.lt.u32 %p183, %r1327, 3; + mov.u32 %r2468, 0; + @%p183 bra $L__BB25_140; + + sub.s32 %r2440, %r243, %r244; + mov.u32 %r2468, 0; + +$L__BB25_123: + and.b16 %rs764, %rs1139, 255; + setp.ne.s16 %p184, %rs764, 0; + @%p184 bra $L__BB25_127; + + setp.eq.s32 %p185, %r2390, 0; + mov.u16 %rs1159, 255; + @%p185 bra $L__BB25_126; + + cvt.u64.u32 %rd230, %r2389; + add.s64 %rd231, %rd230, %rd4; + add.s64 %rd232, %rd1, %rd231; + ld.global.u8 %rs1159, [%rd232]; + +$L__BB25_126: + setp.ne.s32 %p187, %r2390, 0; + selp.u32 %r1329, 1, 0, %p187; + add.s32 %r2389, %r2389, %r1329; + add.s32 %r1330, %r2390, -1; + selp.b32 %r2390, 0, %r1330, %p185; + setp.eq.s32 %p188, %r2390, 0; + or.b16 %rs766, %rs1159, 15; + selp.b16 %rs1105, %rs766, %rs1159, %p188; + and.b16 %rs767, %rs1105, 255; + mov.u16 %rs768, 8; + sub.s16 %rs1139, %rs768, %rs1104; + setp.eq.s16 %p189, %rs767, 255; + selp.u16 %rs1104, 1, 0, %p189; + +$L__BB25_127: + add.s16 %rs1166, %rs1139, -1; + and.b16 %rs769, %rs1166, 255; + cvt.u32.u16 %r1331, %rs1166; + and.b32 %r1332, %r1331, 255; + cvt.u32.u16 %r1333, %rs1105; + and.b32 %r2446, %r1333, 255; + shr.u32 %r1334, %r2446, %r1332; + and.b32 %r1335, %r1334, 1; + bfi.b32 %r255, %r2468, %r1335, 1, 31; + setp.ne.s16 %p190, %rs769, 0; + @%p190 bra $L__BB25_131; + + setp.eq.s32 %p191, %r2390, 0; + mov.u16 %rs1163, 255; + @%p191 bra $L__BB25_130; + + cvt.u64.u32 %rd233, %r2389; + add.s64 %rd234, %rd233, %rd4; + add.s64 %rd235, %rd1, %rd234; + ld.global.u8 %rs1163, [%rd235]; + +$L__BB25_130: + setp.ne.s32 %p193, %r2390, 0; + selp.u32 %r1336, 1, 0, %p193; + add.s32 %r2389, %r2389, %r1336; + add.s32 %r1337, %r2390, -1; + selp.b32 %r2390, 0, %r1337, %p191; + setp.eq.s32 %p194, %r2390, 0; + or.b16 %rs771, %rs1163, 15; + selp.b16 %rs1105, %rs771, %rs1163, %p194; + and.b16 %rs772, %rs1105, 255; + mov.u16 %rs773, 8; + sub.s16 %rs1166, %rs773, %rs1104; + setp.eq.s16 %p195, %rs772, 255; + selp.u16 %rs1104, 1, 0, %p195; + cvt.u32.u16 %r1338, %rs1105; + and.b32 %r2446, %r1338, 255; + +$L__BB25_131: + add.s16 %rs1170, %rs1166, -1; + and.b16 %rs774, %rs1170, 255; + cvt.u32.u16 %r1339, %rs1170; + and.b32 %r1340, %r1339, 255; + shr.u32 %r1341, %r2446, %r1340; + and.b32 %r1342, %r1341, 1; + bfi.b32 %r262, %r255, %r1342, 1, 31; + setp.ne.s16 %p196, %rs774, 0; + @%p196 bra $L__BB25_135; + + setp.eq.s32 %p197, %r2390, 0; + mov.u16 %rs1167, 255; + @%p197 bra $L__BB25_134; + + cvt.u64.u32 %rd236, %r2389; + add.s64 %rd237, %rd236, %rd4; + add.s64 %rd238, %rd1, %rd237; + ld.global.u8 %rs1167, [%rd238]; + +$L__BB25_134: + setp.ne.s32 %p199, %r2390, 0; + selp.u32 %r1343, 1, 0, %p199; + add.s32 %r2389, %r2389, %r1343; + add.s32 %r1344, %r2390, -1; + selp.b32 %r2390, 0, %r1344, %p197; + setp.eq.s32 %p200, %r2390, 0; + or.b16 %rs776, %rs1167, 15; + selp.b16 %rs1105, %rs776, %rs1167, %p200; + and.b16 %rs777, %rs1105, 255; + mov.u16 %rs778, 8; + sub.s16 %rs1170, %rs778, %rs1104; + setp.eq.s16 %p201, %rs777, 255; + selp.u16 %rs1104, 1, 0, %p201; + cvt.u32.u16 %r1345, %rs1105; + and.b32 %r2446, %r1345, 255; + +$L__BB25_135: + add.s16 %rs1174, %rs1170, -1; + and.b16 %rs779, %rs1174, 255; + cvt.u32.u16 %r1346, %rs1174; + and.b32 %r1347, %r1346, 255; + shr.u32 %r1348, %r2446, %r1347; + and.b32 %r1349, %r1348, 1; + bfi.b32 %r269, %r262, %r1349, 1, 31; + setp.ne.s16 %p202, %rs779, 0; + @%p202 bra $L__BB25_139; + + setp.eq.s32 %p203, %r2390, 0; + mov.u16 %rs1171, 255; + @%p203 bra $L__BB25_138; + + cvt.u64.u32 %rd239, %r2389; + add.s64 %rd240, %rd239, %rd4; + add.s64 %rd241, %rd1, %rd240; + ld.global.u8 %rs1171, [%rd241]; + +$L__BB25_138: + setp.ne.s32 %p205, %r2390, 0; + selp.u32 %r1350, 1, 0, %p205; + add.s32 %r2389, %r2389, %r1350; + add.s32 %r1351, %r2390, -1; + selp.b32 %r2390, 0, %r1351, %p203; + setp.eq.s32 %p206, %r2390, 0; + or.b16 %rs781, %rs1171, 15; + selp.b16 %rs1105, %rs781, %rs1171, %p206; + and.b16 %rs782, %rs1105, 255; + mov.u16 %rs783, 8; + sub.s16 %rs1174, %rs783, %rs1104; + setp.eq.s16 %p207, %rs782, 255; + selp.u16 %rs1104, 1, 0, %p207; + cvt.u32.u16 %r1352, %rs1105; + and.b32 %r2446, %r1352, 255; + +$L__BB25_139: + add.s16 %rs1139, %rs1174, -1; + cvt.u32.u16 %r1353, %rs1139; + and.b32 %r1354, %r1353, 255; + shr.u32 %r1355, %r2446, %r1354; + and.b32 %r1356, %r1355, 1; + bfi.b32 %r2468, %r269, %r1356, 1, 31; + add.s32 %r2440, %r2440, -4; + setp.ne.s32 %p208, %r2440, 0; + @%p208 bra $L__BB25_123; + +$L__BB25_140: + setp.eq.s32 %p209, %r244, 0; + @%p209 bra $L__BB25_156; + + and.b16 %rs784, %rs1139, 255; + setp.ne.s16 %p210, %rs784, 0; + @%p210 bra $L__BB25_145; + + setp.eq.s32 %p211, %r2390, 0; + mov.u16 %rs1181, 255; + @%p211 bra $L__BB25_144; + + cvt.u64.u32 %rd242, %r2389; + add.s64 %rd243, %rd242, %rd4; + add.s64 %rd244, %rd1, %rd243; + ld.global.u8 %rs1181, [%rd244]; + +$L__BB25_144: + setp.ne.s32 %p213, %r2390, 0; + selp.u32 %r1357, 1, 0, %p213; + add.s32 %r2389, %r2389, %r1357; + add.s32 %r1358, %r2390, -1; + selp.b32 %r2390, 0, %r1358, %p211; + setp.eq.s32 %p214, %r2390, 0; + or.b16 %rs786, %rs1181, 15; + selp.b16 %rs1105, %rs786, %rs1181, %p214; + and.b16 %rs787, %rs1105, 255; + mov.u16 %rs788, 8; + sub.s16 %rs1139, %rs788, %rs1104; + setp.eq.s16 %p215, %rs787, 255; + selp.u16 %rs1104, 1, 0, %p215; + +$L__BB25_145: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1359, %rs1139; + and.b32 %r1360, %r1359, 255; + cvt.u32.u16 %r1361, %rs1105; + and.b32 %r2463, %r1361, 255; + shr.u32 %r1362, %r2463, %r1360; + and.b32 %r1363, %r1362, 1; + bfi.b32 %r2468, %r2468, %r1363, 1, 31; + setp.eq.s32 %p216, %r244, 1; + @%p216 bra $L__BB25_156; + + and.b16 %rs789, %rs1139, 255; + setp.ne.s16 %p217, %rs789, 0; + @%p217 bra $L__BB25_150; + + setp.eq.s32 %p218, %r2390, 0; + mov.u16 %rs1185, 255; + @%p218 bra $L__BB25_149; + + cvt.u64.u32 %rd245, %r2389; + add.s64 %rd246, %rd245, %rd4; + add.s64 %rd247, %rd1, %rd246; + ld.global.u8 %rs1185, [%rd247]; + +$L__BB25_149: + setp.ne.s32 %p220, %r2390, 0; + selp.u32 %r1364, 1, 0, %p220; + add.s32 %r2389, %r2389, %r1364; + add.s32 %r1365, %r2390, -1; + selp.b32 %r2390, 0, %r1365, %p218; + setp.eq.s32 %p221, %r2390, 0; + or.b16 %rs791, %rs1185, 15; + selp.b16 %rs1105, %rs791, %rs1185, %p221; + and.b16 %rs792, %rs1105, 255; + mov.u16 %rs793, 8; + sub.s16 %rs1139, %rs793, %rs1104; + setp.eq.s16 %p222, %rs792, 255; + selp.u16 %rs1104, 1, 0, %p222; + cvt.u32.u16 %r1366, %rs1105; + and.b32 %r2463, %r1366, 255; + +$L__BB25_150: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1367, %rs1139; + and.b32 %r1368, %r1367, 255; + shr.u32 %r1369, %r2463, %r1368; + and.b32 %r1370, %r1369, 1; + bfi.b32 %r2468, %r2468, %r1370, 1, 31; + setp.eq.s32 %p223, %r244, 2; + @%p223 bra $L__BB25_156; + + and.b16 %rs794, %rs1139, 255; + setp.ne.s16 %p224, %rs794, 0; + @%p224 bra $L__BB25_155; + + setp.eq.s32 %p225, %r2390, 0; + mov.u16 %rs1189, 255; + @%p225 bra $L__BB25_154; + + cvt.u64.u32 %rd248, %r2389; + add.s64 %rd249, %rd248, %rd4; + add.s64 %rd250, %rd1, %rd249; + ld.global.u8 %rs1189, [%rd250]; + +$L__BB25_154: + setp.ne.s32 %p227, %r2390, 0; + selp.u32 %r1371, 1, 0, %p227; + add.s32 %r2389, %r2389, %r1371; + add.s32 %r1372, %r2390, -1; + selp.b32 %r2390, 0, %r1372, %p225; + setp.eq.s32 %p228, %r2390, 0; + or.b16 %rs796, %rs1189, 15; + selp.b16 %rs1105, %rs796, %rs1189, %p228; + and.b16 %rs797, %rs1105, 255; + mov.u16 %rs798, 8; + sub.s16 %rs1139, %rs798, %rs1104; + setp.eq.s16 %p229, %rs797, 255; + selp.u16 %rs1104, 1, 0, %p229; + cvt.u32.u16 %r1373, %rs1105; + and.b32 %r2463, %r1373, 255; + +$L__BB25_155: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1374, %rs1139; + and.b32 %r1375, %r1374, 255; + shr.u32 %r1376, %r2463, %r1375; + and.b32 %r1377, %r1376, 1; + bfi.b32 %r2468, %r2468, %r1377, 1, 31; + +$L__BB25_156: + shl.b32 %r1378, %r2468, 1; + or.b32 %r2472, %r1378, 1; + add.s32 %r1379, %r2480, -1; + setp.eq.s32 %p230, %r2480, 0; + selp.b32 %r2471, 0, %r1379, %p230; + +$L__BB25_157: + mul.lo.s32 %r1380, %r2481, 7; + cvt.u64.u32 %rd251, %r2472; + shl.b64 %rd252, %rd251, %r1380; + or.b64 %rd606, %rd252, %rd606; + setp.ne.s32 %p231, %r2480, 12; + setp.ne.s32 %p232, %r240, 0; + or.pred %p233, %p231, %p232; + add.s32 %r2481, %r2481, 1; + setp.lt.u32 %p234, %r2481, 8; + or.pred %p235, %p234, %p233; + mov.u32 %r2480, %r2471; + @%p235 bra $L__BB25_113; + +$L__BB25_158: + cvt.u32.u64 %r1381, %rd606; + and.b32 %r2477, %r1381, 127; + shr.u64 %rd606, %rd606, 7; + add.s32 %r2481, %r2481, -1; + +$L__BB25_159: + setp.lt.u32 %p236, %r228, %r2; + selp.b32 %r326, %r2482, 0, %p236; + st.local.u16 [%rd26+4], %r326; + and.b32 %r1383, %r1302, 64; + shl.b32 %r1384, %r326, 4; + and.b32 %r1385, %r1384, 128; + or.b32 %r2534, %r1385, %r1383; + setp.ne.s32 %p237, %r2534, 192; + @%p237 bra $L__BB25_209; + + add.s32 %r328, %r2477, -2; + setp.eq.s32 %p238, %r328, -1; + selp.b32 %r2534, 256, 192, %p238; + setp.gt.s32 %p239, %r2477, 1; + mov.u32 %r2477, %r328; + @%p239 bra $L__BB25_209; + + setp.ne.s32 %p240, %r2481, 0; + @%p240 bra $L__BB25_208; + + mov.u32 %r2481, 0; + +$L__BB25_163: + setp.gt.u32 %p241, %r2481, 7; + @%p241 bra $L__BB25_208; + + cvt.u64.u32 %rd35, %r2480; + mul.wide.u32 %rd253, %r2480, 4; + add.s64 %rd255, %rd151, %rd253; + ld.global.nc.u32 %r334, [%rd255]; + and.b16 %rs799, %rs1139, 255; + setp.ne.s16 %p242, %rs799, 0; + @%p242 bra $L__BB25_168; + + setp.eq.s32 %p243, %r2390, 0; + mov.u16 %rs1208, 255; + @%p243 bra $L__BB25_167; + + cvt.u64.u32 %rd256, %r2389; + add.s64 %rd257, %rd256, %rd4; + add.s64 %rd258, %rd1, %rd257; + ld.global.u8 %rs1208, [%rd258]; + +$L__BB25_167: + setp.ne.s32 %p245, %r2390, 0; + selp.u32 %r1387, 1, 0, %p245; + add.s32 %r2389, %r2389, %r1387; + add.s32 %r1388, %r2390, -1; + selp.b32 %r2390, 0, %r1388, %p243; + setp.eq.s32 %p246, %r2390, 0; + or.b16 %rs801, %rs1208, 15; + selp.b16 %rs1105, %rs801, %rs1208, %p246; + and.b16 %rs802, %rs1105, 255; + mov.u16 %rs803, 8; + sub.s16 %rs1139, %rs803, %rs1104; + setp.eq.s16 %p247, %rs802, 255; + selp.u16 %rs1104, 1, 0, %p247; + +$L__BB25_168: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1389, %rs1139; + and.b32 %r1390, %r1389, 255; + mov.u32 %r1391, 1; + shl.b32 %r1392, %r1391, %r1390; + cvt.u32.u16 %r1393, %rs1105; + and.b32 %r1394, %r1392, %r1393; + and.b32 %r339, %r1394, 255; + setp.eq.s32 %p248, %r339, 0; + @%p248 bra $L__BB25_170; + + add.s32 %r1395, %r2480, 1; + min.u32 %r2523, %r1395, 12; + mov.u32 %r1396, -1; + shl.b32 %r1397, %r1396, %r334; + shl.b32 %r1398, %r1397, 1; + xor.b32 %r2524, %r1398, -2; + bra.uni $L__BB25_207; + +$L__BB25_170: + add.s64 %rd259, %rd35, -3; + setp.gt.u64 %p249, %rd259, 9; + mov.u32 %r2520, 0; + @%p249 bra $L__BB25_206; + + max.u32 %r342, %r334, 1; + add.s32 %r1402, %r342, -1; + and.b32 %r343, %r342, 3; + setp.lt.u32 %p250, %r1402, 3; + mov.u32 %r2520, 0; + @%p250 bra $L__BB25_190; + + sub.s32 %r2492, %r342, %r343; + mov.u32 %r2520, 0; + +$L__BB25_173: + and.b16 %rs805, %rs1139, 255; + setp.ne.s16 %p251, %rs805, 0; + @%p251 bra $L__BB25_177; + + setp.eq.s32 %p252, %r2390, 0; + mov.u16 %rs1215, 255; + @%p252 bra $L__BB25_176; + + cvt.u64.u32 %rd260, %r2389; + add.s64 %rd261, %rd260, %rd4; + add.s64 %rd262, %rd1, %rd261; + ld.global.u8 %rs1215, [%rd262]; + +$L__BB25_176: + setp.ne.s32 %p254, %r2390, 0; + selp.u32 %r1404, 1, 0, %p254; + add.s32 %r2389, %r2389, %r1404; + add.s32 %r1405, %r2390, -1; + selp.b32 %r2390, 0, %r1405, %p252; + setp.eq.s32 %p255, %r2390, 0; + or.b16 %rs807, %rs1215, 15; + selp.b16 %rs1105, %rs807, %rs1215, %p255; + and.b16 %rs808, %rs1105, 255; + mov.u16 %rs809, 8; + sub.s16 %rs1139, %rs809, %rs1104; + setp.eq.s16 %p256, %rs808, 255; + selp.u16 %rs1104, 1, 0, %p256; + +$L__BB25_177: + add.s16 %rs1222, %rs1139, -1; + and.b16 %rs810, %rs1222, 255; + cvt.u32.u16 %r1406, %rs1222; + and.b32 %r1407, %r1406, 255; + cvt.u32.u16 %r1408, %rs1105; + and.b32 %r2498, %r1408, 255; + shr.u32 %r1409, %r2498, %r1407; + and.b32 %r1410, %r1409, 1; + bfi.b32 %r354, %r2520, %r1410, 1, 31; + setp.ne.s16 %p257, %rs810, 0; + @%p257 bra $L__BB25_181; + + setp.eq.s32 %p258, %r2390, 0; + mov.u16 %rs1219, 255; + @%p258 bra $L__BB25_180; + + cvt.u64.u32 %rd263, %r2389; + add.s64 %rd264, %rd263, %rd4; + add.s64 %rd265, %rd1, %rd264; + ld.global.u8 %rs1219, [%rd265]; + +$L__BB25_180: + setp.ne.s32 %p260, %r2390, 0; + selp.u32 %r1411, 1, 0, %p260; + add.s32 %r2389, %r2389, %r1411; + add.s32 %r1412, %r2390, -1; + selp.b32 %r2390, 0, %r1412, %p258; + setp.eq.s32 %p261, %r2390, 0; + or.b16 %rs812, %rs1219, 15; + selp.b16 %rs1105, %rs812, %rs1219, %p261; + and.b16 %rs813, %rs1105, 255; + mov.u16 %rs814, 8; + sub.s16 %rs1222, %rs814, %rs1104; + setp.eq.s16 %p262, %rs813, 255; + selp.u16 %rs1104, 1, 0, %p262; + cvt.u32.u16 %r1413, %rs1105; + and.b32 %r2498, %r1413, 255; + +$L__BB25_181: + add.s16 %rs1226, %rs1222, -1; + and.b16 %rs815, %rs1226, 255; + cvt.u32.u16 %r1414, %rs1226; + and.b32 %r1415, %r1414, 255; + shr.u32 %r1416, %r2498, %r1415; + and.b32 %r1417, %r1416, 1; + bfi.b32 %r361, %r354, %r1417, 1, 31; + setp.ne.s16 %p263, %rs815, 0; + @%p263 bra $L__BB25_185; + + setp.eq.s32 %p264, %r2390, 0; + mov.u16 %rs1223, 255; + @%p264 bra $L__BB25_184; + + cvt.u64.u32 %rd266, %r2389; + add.s64 %rd267, %rd266, %rd4; + add.s64 %rd268, %rd1, %rd267; + ld.global.u8 %rs1223, [%rd268]; + +$L__BB25_184: + setp.ne.s32 %p266, %r2390, 0; + selp.u32 %r1418, 1, 0, %p266; + add.s32 %r2389, %r2389, %r1418; + add.s32 %r1419, %r2390, -1; + selp.b32 %r2390, 0, %r1419, %p264; + setp.eq.s32 %p267, %r2390, 0; + or.b16 %rs817, %rs1223, 15; + selp.b16 %rs1105, %rs817, %rs1223, %p267; + and.b16 %rs818, %rs1105, 255; + mov.u16 %rs819, 8; + sub.s16 %rs1226, %rs819, %rs1104; + setp.eq.s16 %p268, %rs818, 255; + selp.u16 %rs1104, 1, 0, %p268; + cvt.u32.u16 %r1420, %rs1105; + and.b32 %r2498, %r1420, 255; + +$L__BB25_185: + add.s16 %rs1230, %rs1226, -1; + and.b16 %rs820, %rs1230, 255; + cvt.u32.u16 %r1421, %rs1230; + and.b32 %r1422, %r1421, 255; + shr.u32 %r1423, %r2498, %r1422; + and.b32 %r1424, %r1423, 1; + bfi.b32 %r368, %r361, %r1424, 1, 31; + setp.ne.s16 %p269, %rs820, 0; + @%p269 bra $L__BB25_189; + + setp.eq.s32 %p270, %r2390, 0; + mov.u16 %rs1227, 255; + @%p270 bra $L__BB25_188; + + cvt.u64.u32 %rd269, %r2389; + add.s64 %rd270, %rd269, %rd4; + add.s64 %rd271, %rd1, %rd270; + ld.global.u8 %rs1227, [%rd271]; + +$L__BB25_188: + setp.ne.s32 %p272, %r2390, 0; + selp.u32 %r1425, 1, 0, %p272; + add.s32 %r2389, %r2389, %r1425; + add.s32 %r1426, %r2390, -1; + selp.b32 %r2390, 0, %r1426, %p270; + setp.eq.s32 %p273, %r2390, 0; + or.b16 %rs822, %rs1227, 15; + selp.b16 %rs1105, %rs822, %rs1227, %p273; + and.b16 %rs823, %rs1105, 255; + mov.u16 %rs824, 8; + sub.s16 %rs1230, %rs824, %rs1104; + setp.eq.s16 %p274, %rs823, 255; + selp.u16 %rs1104, 1, 0, %p274; + cvt.u32.u16 %r1427, %rs1105; + and.b32 %r2498, %r1427, 255; + +$L__BB25_189: + add.s16 %rs1139, %rs1230, -1; + cvt.u32.u16 %r1428, %rs1139; + and.b32 %r1429, %r1428, 255; + shr.u32 %r1430, %r2498, %r1429; + and.b32 %r1431, %r1430, 1; + bfi.b32 %r2520, %r368, %r1431, 1, 31; + add.s32 %r2492, %r2492, -4; + setp.ne.s32 %p275, %r2492, 0; + @%p275 bra $L__BB25_173; + +$L__BB25_190: + setp.eq.s32 %p276, %r343, 0; + @%p276 bra $L__BB25_206; + + and.b16 %rs825, %rs1139, 255; + setp.ne.s16 %p277, %rs825, 0; + @%p277 bra $L__BB25_195; + + setp.eq.s32 %p278, %r2390, 0; + mov.u16 %rs1237, 255; + @%p278 bra $L__BB25_194; + + cvt.u64.u32 %rd272, %r2389; + add.s64 %rd273, %rd272, %rd4; + add.s64 %rd274, %rd1, %rd273; + ld.global.u8 %rs1237, [%rd274]; + +$L__BB25_194: + setp.ne.s32 %p280, %r2390, 0; + selp.u32 %r1432, 1, 0, %p280; + add.s32 %r2389, %r2389, %r1432; + add.s32 %r1433, %r2390, -1; + selp.b32 %r2390, 0, %r1433, %p278; + setp.eq.s32 %p281, %r2390, 0; + or.b16 %rs827, %rs1237, 15; + selp.b16 %rs1105, %rs827, %rs1237, %p281; + and.b16 %rs828, %rs1105, 255; + mov.u16 %rs829, 8; + sub.s16 %rs1139, %rs829, %rs1104; + setp.eq.s16 %p282, %rs828, 255; + selp.u16 %rs1104, 1, 0, %p282; + +$L__BB25_195: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1434, %rs1139; + and.b32 %r1435, %r1434, 255; + cvt.u32.u16 %r1436, %rs1105; + and.b32 %r2515, %r1436, 255; + shr.u32 %r1437, %r2515, %r1435; + and.b32 %r1438, %r1437, 1; + bfi.b32 %r2520, %r2520, %r1438, 1, 31; + setp.eq.s32 %p283, %r343, 1; + @%p283 bra $L__BB25_206; + + and.b16 %rs830, %rs1139, 255; + setp.ne.s16 %p284, %rs830, 0; + @%p284 bra $L__BB25_200; + + setp.eq.s32 %p285, %r2390, 0; + mov.u16 %rs1241, 255; + @%p285 bra $L__BB25_199; + + cvt.u64.u32 %rd275, %r2389; + add.s64 %rd276, %rd275, %rd4; + add.s64 %rd277, %rd1, %rd276; + ld.global.u8 %rs1241, [%rd277]; + +$L__BB25_199: + setp.ne.s32 %p287, %r2390, 0; + selp.u32 %r1439, 1, 0, %p287; + add.s32 %r2389, %r2389, %r1439; + add.s32 %r1440, %r2390, -1; + selp.b32 %r2390, 0, %r1440, %p285; + setp.eq.s32 %p288, %r2390, 0; + or.b16 %rs832, %rs1241, 15; + selp.b16 %rs1105, %rs832, %rs1241, %p288; + and.b16 %rs833, %rs1105, 255; + mov.u16 %rs834, 8; + sub.s16 %rs1139, %rs834, %rs1104; + setp.eq.s16 %p289, %rs833, 255; + selp.u16 %rs1104, 1, 0, %p289; + cvt.u32.u16 %r1441, %rs1105; + and.b32 %r2515, %r1441, 255; + +$L__BB25_200: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1442, %rs1139; + and.b32 %r1443, %r1442, 255; + shr.u32 %r1444, %r2515, %r1443; + and.b32 %r1445, %r1444, 1; + bfi.b32 %r2520, %r2520, %r1445, 1, 31; + setp.eq.s32 %p290, %r343, 2; + @%p290 bra $L__BB25_206; + + and.b16 %rs835, %rs1139, 255; + setp.ne.s16 %p291, %rs835, 0; + @%p291 bra $L__BB25_205; + + setp.eq.s32 %p292, %r2390, 0; + mov.u16 %rs1245, 255; + @%p292 bra $L__BB25_204; + + cvt.u64.u32 %rd278, %r2389; + add.s64 %rd279, %rd278, %rd4; + add.s64 %rd280, %rd1, %rd279; + ld.global.u8 %rs1245, [%rd280]; + +$L__BB25_204: + setp.ne.s32 %p294, %r2390, 0; + selp.u32 %r1446, 1, 0, %p294; + add.s32 %r2389, %r2389, %r1446; + add.s32 %r1447, %r2390, -1; + selp.b32 %r2390, 0, %r1447, %p292; + setp.eq.s32 %p295, %r2390, 0; + or.b16 %rs837, %rs1245, 15; + selp.b16 %rs1105, %rs837, %rs1245, %p295; + and.b16 %rs838, %rs1105, 255; + mov.u16 %rs839, 8; + sub.s16 %rs1139, %rs839, %rs1104; + setp.eq.s16 %p296, %rs838, 255; + selp.u16 %rs1104, 1, 0, %p296; + cvt.u32.u16 %r1448, %rs1105; + and.b32 %r2515, %r1448, 255; + +$L__BB25_205: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1449, %rs1139; + and.b32 %r1450, %r1449, 255; + shr.u32 %r1451, %r2515, %r1450; + and.b32 %r1452, %r1451, 1; + bfi.b32 %r2520, %r2520, %r1452, 1, 31; + +$L__BB25_206: + shl.b32 %r1453, %r2520, 1; + or.b32 %r2524, %r1453, 1; + add.s32 %r1454, %r2480, -1; + setp.eq.s32 %p297, %r2480, 0; + selp.b32 %r2523, 0, %r1454, %p297; + +$L__BB25_207: + mul.lo.s32 %r1455, %r2481, 7; + cvt.u64.u32 %rd281, %r2524; + shl.b64 %rd282, %rd281, %r1455; + or.b64 %rd606, %rd282, %rd606; + setp.ne.s32 %p298, %r2480, 12; + setp.ne.s32 %p299, %r339, 0; + or.pred %p300, %p298, %p299; + add.s32 %r2481, %r2481, 1; + setp.lt.u32 %p301, %r2481, 8; + or.pred %p302, %p301, %p300; + mov.u32 %r2480, %r2523; + @%p302 bra $L__BB25_163; + +$L__BB25_208: + cvt.u32.u64 %r1456, %rd606; + and.b32 %r2477, %r1456, 127; + shr.u64 %rd606, %rd606, 7; + add.s32 %r2481, %r2481, -1; + +$L__BB25_209: + and.b32 %r1457, %r326, 7; + shr.u64 %rd283, %rd27, %r1457; + cvt.u32.u64 %r1458, %rd283; + and.b32 %r1459, %r1458, 63; + add.s32 %r1460, %r2534, %r1459; + mul.wide.u32 %rd284, %r1460, 2; + add.s64 %rd285, %rd12, %rd284; + ld.global.u16 %r1461, [%rd285]; + and.b32 %r1462, %r1461, 7; + shr.u64 %rd286, %rd283, %r1462; + sub.s32 %r1463, %r226, %r1457; + sub.s32 %r1464, %r1463, %r1462; + cvt.u32.u64 %r1465, %rd286; + shr.u32 %r1466, %r1461, 3; + and.b32 %r1467, %r1466, 15; + mov.u32 %r1468, -1; + shl.b32 %r1469, %r1468, %r1467; + not.b32 %r1470, %r1469; + and.b32 %r1471, %r1465, %r1470; + shr.u64 %rd615, %rd286, %r1467; + sub.s32 %r2560, %r1464, %r1467; + shr.u32 %r1472, %r1461, 7; + and.b32 %r1473, %r1472, 7; + shr.u32 %r1474, %r1461, 10; + and.b32 %r1475, %r1474, 7; + mov.u32 %r1476, 255; + shl.b32 %r1477, %r1476, %r1473; + not.b32 %r1478, %r1477; + and.b32 %r1479, %r1471, %r1478; + add.s32 %r1480, %r1475, %r1479; + add.s32 %r1481, %r1480, 1; + st.local.u16 [%rd26+2], %r1481; + shr.u32 %r1482, %r1461, 13; + shr.u32 %r1483, %r1471, %r1473; + add.s32 %r1484, %r1482, %r1483; + add.s32 %r1485, %r1484, 1; + st.local.u16 [%rd26+6], %r1485; + add.s32 %r2364, %r2364, 4; + setp.lt.u32 %p303, %r2364, %r2; + shl.b32 %r1486, %r326, 2; + and.b32 %r1487, %r1486, 896; + shl.b32 %r1488, %r326, 3; + and.b32 %r1489, %r1488, 128; + or.b32 %r2363, %r1489, %r1487; + @%p303 bra $L__BB25_55; + + mul.wide.u32 %rd289, %r2364, 2; + add.s64 %rd290, %rd14, %rd289; + mov.u16 %rs840, 0; + st.local.v2.u16 [%rd290], {%rs840, %rs840}; + setp.lt.u32 %p304, %r3, 3; + @%p304 bra $L__BB25_319; + + ld.param.u64 %rd592, [ j2k_htj2k_decode_codeblocks_param_4]; + ld.param.u64 %rd590, [ j2k_htj2k_decode_codeblocks_param_6]; + cvta.to.global.u64 %rd41, %rd590; + cvta.to.global.u64 %rd42, %rd592; + mov.u32 %r2535, 2; + +$L__BB25_212: + shr.u32 %r1494, %r2535, 1; + mul.lo.s32 %r438, %r1494, %r1148; + sub.s32 %r439, %r438, %r1148; + mov.u32 %r2544, 0; + mov.u32 %r2545, %r2544; + mov.u32 %r2546, %r438; + +$L__BB25_213: + sub.s32 %r1495, %r2546, %r438; + add.s32 %r451, %r1495, %r439; + mul.wide.u32 %rd291, %r451, 2; + add.s64 %rd47, %rd14, %rd291; + ld.local.u16 %r1496, [%rd47]; + shl.b32 %r1497, %r1496, 2; + and.b32 %r1498, %r1497, 640; + or.b32 %r1499, %r2545, %r1498; + add.s32 %r1500, %r451, 2; + mul.wide.u32 %rd292, %r1500, 2; + add.s64 %rd48, %rd14, %rd292; + ld.local.u16 %r1501, [%rd48]; + shl.b32 %r1502, %r1501, 4; + and.b32 %r1503, %r1502, 512; + or.b32 %r452, %r1499, %r1503; + setp.gt.u32 %p305, %r2560, 31; + @%p305 bra $L__BB25_217; + +$L__BB25_214: + setp.eq.s32 %p306, %r2559, 0; + mov.u16 %rs1270, 0; + @%p306 bra $L__BB25_216; + + cvt.s64.s32 %rd293, %r2558; + add.s64 %rd294, %rd293, %rd4; + add.s64 %rd295, %rd1, %rd294; + ld.global.u8 %rs1270, [%rd295]; + +$L__BB25_216: + setp.ne.s32 %p308, %r2559, 0; + selp.b32 %r1504, -1, 0, %p308; + add.s32 %r2558, %r2558, %r1504; + add.s32 %r1505, %r2559, -1; + selp.b32 %r2559, 0, %r1505, %p306; + and.b16 %rs842, %rs1270, 255; + and.b16 %rs843, %rs1270, 127; + setp.eq.s16 %p309, %rs843, 127; + and.b16 %rs844, %rs1271, 255; + setp.ne.s16 %p310, %rs844, 0; + and.pred %p311, %p310, %p309; + selp.b32 %r1506, 7, 8, %p311; + cvt.u64.u16 %rd296, %rs1270; + and.b64 %rd297, %rd296, 255; + shl.b64 %rd298, %rd297, %r2560; + or.b64 %rd615, %rd298, %rd615; + add.s32 %r2560, %r1506, %r2560; + setp.gt.u16 %p312, %rs842, 143; + selp.u16 %rs1271, 1, 0, %p312; + setp.lt.u32 %p313, %r2560, 33; + @%p313 bra $L__BB25_214; + +$L__BB25_217: + cvt.u32.u64 %r1507, %rd615; + and.b32 %r1508, %r1507, 127; + add.s32 %r1509, %r1508, %r452; + mul.wide.u32 %rd299, %r1509, 2; + add.s64 %rd300, %rd42, %rd299; + ld.global.u16 %r2612, [%rd300]; + setp.ne.s32 %p314, %r452, 0; + @%p314 bra $L__BB25_267; + + add.s32 %r463, %r2477, -2; + setp.eq.s32 %p315, %r463, -1; + selp.b32 %r2612, %r2612, 0, %p315; + setp.gt.s32 %p316, %r2477, 1; + mov.u32 %r2477, %r463; + @%p316 bra $L__BB25_267; + + setp.ne.s32 %p317, %r2481, 0; + @%p317 bra $L__BB25_266; + + mov.u32 %r2481, 0; + +$L__BB25_221: + setp.gt.u32 %p318, %r2481, 7; + @%p318 bra $L__BB25_266; + + cvt.u64.u32 %rd53, %r2480; + mul.wide.u32 %rd301, %r2480, 4; + add.s64 %rd303, %rd151, %rd301; + ld.global.nc.u32 %r469, [%rd303]; + and.b16 %rs845, %rs1139, 255; + setp.ne.s16 %p319, %rs845, 0; + @%p319 bra $L__BB25_226; + + setp.eq.s32 %p320, %r2390, 0; + mov.u16 %rs1275, 255; + @%p320 bra $L__BB25_225; + + cvt.u64.u32 %rd304, %r2389; + add.s64 %rd305, %rd304, %rd4; + add.s64 %rd306, %rd1, %rd305; + ld.global.u8 %rs1275, [%rd306]; + +$L__BB25_225: + setp.ne.s32 %p322, %r2390, 0; + selp.u32 %r1511, 1, 0, %p322; + add.s32 %r2389, %r2389, %r1511; + add.s32 %r1512, %r2390, -1; + selp.b32 %r2390, 0, %r1512, %p320; + setp.eq.s32 %p323, %r2390, 0; + or.b16 %rs847, %rs1275, 15; + selp.b16 %rs1105, %rs847, %rs1275, %p323; + and.b16 %rs848, %rs1105, 255; + mov.u16 %rs849, 8; + sub.s16 %rs1139, %rs849, %rs1104; + setp.eq.s16 %p324, %rs848, 255; + selp.u16 %rs1104, 1, 0, %p324; + +$L__BB25_226: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1513, %rs1139; + and.b32 %r1514, %r1513, 255; + mov.u32 %r1515, 1; + shl.b32 %r1516, %r1515, %r1514; + cvt.u32.u16 %r1517, %rs1105; + and.b32 %r1518, %r1516, %r1517; + and.b32 %r474, %r1518, 255; + setp.eq.s32 %p325, %r474, 0; + @%p325 bra $L__BB25_228; + + add.s32 %r1519, %r2480, 1; + min.u32 %r2601, %r1519, 12; + mov.u32 %r1520, -1; + shl.b32 %r1521, %r1520, %r469; + shl.b32 %r1522, %r1521, 1; + xor.b32 %r2602, %r1522, -2; + bra.uni $L__BB25_265; + +$L__BB25_228: + add.s64 %rd307, %rd53, -3; + setp.gt.u64 %p326, %rd307, 9; + mov.u32 %r2598, 0; + @%p326 bra $L__BB25_264; + + max.u32 %r477, %r469, 1; + add.s32 %r1526, %r477, -1; + and.b32 %r478, %r477, 3; + setp.lt.u32 %p327, %r1526, 3; + mov.u32 %r2598, 0; + @%p327 bra $L__BB25_248; + + sub.s32 %r2570, %r477, %r478; + mov.u32 %r2598, 0; + +$L__BB25_231: + and.b16 %rs851, %rs1139, 255; + setp.ne.s16 %p328, %rs851, 0; + @%p328 bra $L__BB25_235; + + setp.eq.s32 %p329, %r2390, 0; + mov.u16 %rs1282, 255; + @%p329 bra $L__BB25_234; + + cvt.u64.u32 %rd308, %r2389; + add.s64 %rd309, %rd308, %rd4; + add.s64 %rd310, %rd1, %rd309; + ld.global.u8 %rs1282, [%rd310]; + +$L__BB25_234: + setp.ne.s32 %p331, %r2390, 0; + selp.u32 %r1528, 1, 0, %p331; + add.s32 %r2389, %r2389, %r1528; + add.s32 %r1529, %r2390, -1; + selp.b32 %r2390, 0, %r1529, %p329; + setp.eq.s32 %p332, %r2390, 0; + or.b16 %rs853, %rs1282, 15; + selp.b16 %rs1105, %rs853, %rs1282, %p332; + and.b16 %rs854, %rs1105, 255; + mov.u16 %rs855, 8; + sub.s16 %rs1139, %rs855, %rs1104; + setp.eq.s16 %p333, %rs854, 255; + selp.u16 %rs1104, 1, 0, %p333; + +$L__BB25_235: + add.s16 %rs1289, %rs1139, -1; + and.b16 %rs856, %rs1289, 255; + cvt.u32.u16 %r1530, %rs1289; + and.b32 %r1531, %r1530, 255; + cvt.u32.u16 %r1532, %rs1105; + and.b32 %r2576, %r1532, 255; + shr.u32 %r1533, %r2576, %r1531; + and.b32 %r1534, %r1533, 1; + bfi.b32 %r489, %r2598, %r1534, 1, 31; + setp.ne.s16 %p334, %rs856, 0; + @%p334 bra $L__BB25_239; + + setp.eq.s32 %p335, %r2390, 0; + mov.u16 %rs1286, 255; + @%p335 bra $L__BB25_238; + + cvt.u64.u32 %rd311, %r2389; + add.s64 %rd312, %rd311, %rd4; + add.s64 %rd313, %rd1, %rd312; + ld.global.u8 %rs1286, [%rd313]; + +$L__BB25_238: + setp.ne.s32 %p337, %r2390, 0; + selp.u32 %r1535, 1, 0, %p337; + add.s32 %r2389, %r2389, %r1535; + add.s32 %r1536, %r2390, -1; + selp.b32 %r2390, 0, %r1536, %p335; + setp.eq.s32 %p338, %r2390, 0; + or.b16 %rs858, %rs1286, 15; + selp.b16 %rs1105, %rs858, %rs1286, %p338; + and.b16 %rs859, %rs1105, 255; + mov.u16 %rs860, 8; + sub.s16 %rs1289, %rs860, %rs1104; + setp.eq.s16 %p339, %rs859, 255; + selp.u16 %rs1104, 1, 0, %p339; + cvt.u32.u16 %r1537, %rs1105; + and.b32 %r2576, %r1537, 255; + +$L__BB25_239: + add.s16 %rs1293, %rs1289, -1; + and.b16 %rs861, %rs1293, 255; + cvt.u32.u16 %r1538, %rs1293; + and.b32 %r1539, %r1538, 255; + shr.u32 %r1540, %r2576, %r1539; + and.b32 %r1541, %r1540, 1; + bfi.b32 %r496, %r489, %r1541, 1, 31; + setp.ne.s16 %p340, %rs861, 0; + @%p340 bra $L__BB25_243; + + setp.eq.s32 %p341, %r2390, 0; + mov.u16 %rs1290, 255; + @%p341 bra $L__BB25_242; + + cvt.u64.u32 %rd314, %r2389; + add.s64 %rd315, %rd314, %rd4; + add.s64 %rd316, %rd1, %rd315; + ld.global.u8 %rs1290, [%rd316]; + +$L__BB25_242: + setp.ne.s32 %p343, %r2390, 0; + selp.u32 %r1542, 1, 0, %p343; + add.s32 %r2389, %r2389, %r1542; + add.s32 %r1543, %r2390, -1; + selp.b32 %r2390, 0, %r1543, %p341; + setp.eq.s32 %p344, %r2390, 0; + or.b16 %rs863, %rs1290, 15; + selp.b16 %rs1105, %rs863, %rs1290, %p344; + and.b16 %rs864, %rs1105, 255; + mov.u16 %rs865, 8; + sub.s16 %rs1293, %rs865, %rs1104; + setp.eq.s16 %p345, %rs864, 255; + selp.u16 %rs1104, 1, 0, %p345; + cvt.u32.u16 %r1544, %rs1105; + and.b32 %r2576, %r1544, 255; + +$L__BB25_243: + add.s16 %rs1297, %rs1293, -1; + and.b16 %rs866, %rs1297, 255; + cvt.u32.u16 %r1545, %rs1297; + and.b32 %r1546, %r1545, 255; + shr.u32 %r1547, %r2576, %r1546; + and.b32 %r1548, %r1547, 1; + bfi.b32 %r503, %r496, %r1548, 1, 31; + setp.ne.s16 %p346, %rs866, 0; + @%p346 bra $L__BB25_247; + + setp.eq.s32 %p347, %r2390, 0; + mov.u16 %rs1294, 255; + @%p347 bra $L__BB25_246; + + cvt.u64.u32 %rd317, %r2389; + add.s64 %rd318, %rd317, %rd4; + add.s64 %rd319, %rd1, %rd318; + ld.global.u8 %rs1294, [%rd319]; + +$L__BB25_246: + setp.ne.s32 %p349, %r2390, 0; + selp.u32 %r1549, 1, 0, %p349; + add.s32 %r2389, %r2389, %r1549; + add.s32 %r1550, %r2390, -1; + selp.b32 %r2390, 0, %r1550, %p347; + setp.eq.s32 %p350, %r2390, 0; + or.b16 %rs868, %rs1294, 15; + selp.b16 %rs1105, %rs868, %rs1294, %p350; + and.b16 %rs869, %rs1105, 255; + mov.u16 %rs870, 8; + sub.s16 %rs1297, %rs870, %rs1104; + setp.eq.s16 %p351, %rs869, 255; + selp.u16 %rs1104, 1, 0, %p351; + cvt.u32.u16 %r1551, %rs1105; + and.b32 %r2576, %r1551, 255; + +$L__BB25_247: + add.s16 %rs1139, %rs1297, -1; + cvt.u32.u16 %r1552, %rs1139; + and.b32 %r1553, %r1552, 255; + shr.u32 %r1554, %r2576, %r1553; + and.b32 %r1555, %r1554, 1; + bfi.b32 %r2598, %r503, %r1555, 1, 31; + add.s32 %r2570, %r2570, -4; + setp.ne.s32 %p352, %r2570, 0; + @%p352 bra $L__BB25_231; + +$L__BB25_248: + setp.eq.s32 %p353, %r478, 0; + @%p353 bra $L__BB25_264; + + and.b16 %rs871, %rs1139, 255; + setp.ne.s16 %p354, %rs871, 0; + @%p354 bra $L__BB25_253; + + setp.eq.s32 %p355, %r2390, 0; + mov.u16 %rs1304, 255; + @%p355 bra $L__BB25_252; + + cvt.u64.u32 %rd320, %r2389; + add.s64 %rd321, %rd320, %rd4; + add.s64 %rd322, %rd1, %rd321; + ld.global.u8 %rs1304, [%rd322]; + +$L__BB25_252: + setp.ne.s32 %p357, %r2390, 0; + selp.u32 %r1556, 1, 0, %p357; + add.s32 %r2389, %r2389, %r1556; + add.s32 %r1557, %r2390, -1; + selp.b32 %r2390, 0, %r1557, %p355; + setp.eq.s32 %p358, %r2390, 0; + or.b16 %rs873, %rs1304, 15; + selp.b16 %rs1105, %rs873, %rs1304, %p358; + and.b16 %rs874, %rs1105, 255; + mov.u16 %rs875, 8; + sub.s16 %rs1139, %rs875, %rs1104; + setp.eq.s16 %p359, %rs874, 255; + selp.u16 %rs1104, 1, 0, %p359; + +$L__BB25_253: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1558, %rs1139; + and.b32 %r1559, %r1558, 255; + cvt.u32.u16 %r1560, %rs1105; + and.b32 %r2593, %r1560, 255; + shr.u32 %r1561, %r2593, %r1559; + and.b32 %r1562, %r1561, 1; + bfi.b32 %r2598, %r2598, %r1562, 1, 31; + setp.eq.s32 %p360, %r478, 1; + @%p360 bra $L__BB25_264; + + and.b16 %rs876, %rs1139, 255; + setp.ne.s16 %p361, %rs876, 0; + @%p361 bra $L__BB25_258; + + setp.eq.s32 %p362, %r2390, 0; + mov.u16 %rs1308, 255; + @%p362 bra $L__BB25_257; + + cvt.u64.u32 %rd323, %r2389; + add.s64 %rd324, %rd323, %rd4; + add.s64 %rd325, %rd1, %rd324; + ld.global.u8 %rs1308, [%rd325]; + +$L__BB25_257: + setp.ne.s32 %p364, %r2390, 0; + selp.u32 %r1563, 1, 0, %p364; + add.s32 %r2389, %r2389, %r1563; + add.s32 %r1564, %r2390, -1; + selp.b32 %r2390, 0, %r1564, %p362; + setp.eq.s32 %p365, %r2390, 0; + or.b16 %rs878, %rs1308, 15; + selp.b16 %rs1105, %rs878, %rs1308, %p365; + and.b16 %rs879, %rs1105, 255; + mov.u16 %rs880, 8; + sub.s16 %rs1139, %rs880, %rs1104; + setp.eq.s16 %p366, %rs879, 255; + selp.u16 %rs1104, 1, 0, %p366; + cvt.u32.u16 %r1565, %rs1105; + and.b32 %r2593, %r1565, 255; + +$L__BB25_258: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1566, %rs1139; + and.b32 %r1567, %r1566, 255; + shr.u32 %r1568, %r2593, %r1567; + and.b32 %r1569, %r1568, 1; + bfi.b32 %r2598, %r2598, %r1569, 1, 31; + setp.eq.s32 %p367, %r478, 2; + @%p367 bra $L__BB25_264; + + and.b16 %rs881, %rs1139, 255; + setp.ne.s16 %p368, %rs881, 0; + @%p368 bra $L__BB25_263; + + setp.eq.s32 %p369, %r2390, 0; + mov.u16 %rs1312, 255; + @%p369 bra $L__BB25_262; + + cvt.u64.u32 %rd326, %r2389; + add.s64 %rd327, %rd326, %rd4; + add.s64 %rd328, %rd1, %rd327; + ld.global.u8 %rs1312, [%rd328]; + +$L__BB25_262: + setp.ne.s32 %p371, %r2390, 0; + selp.u32 %r1570, 1, 0, %p371; + add.s32 %r2389, %r2389, %r1570; + add.s32 %r1571, %r2390, -1; + selp.b32 %r2390, 0, %r1571, %p369; + setp.eq.s32 %p372, %r2390, 0; + or.b16 %rs883, %rs1312, 15; + selp.b16 %rs1105, %rs883, %rs1312, %p372; + and.b16 %rs884, %rs1105, 255; + mov.u16 %rs885, 8; + sub.s16 %rs1139, %rs885, %rs1104; + setp.eq.s16 %p373, %rs884, 255; + selp.u16 %rs1104, 1, 0, %p373; + cvt.u32.u16 %r1572, %rs1105; + and.b32 %r2593, %r1572, 255; + +$L__BB25_263: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1573, %rs1139; + and.b32 %r1574, %r1573, 255; + shr.u32 %r1575, %r2593, %r1574; + and.b32 %r1576, %r1575, 1; + bfi.b32 %r2598, %r2598, %r1576, 1, 31; + +$L__BB25_264: + shl.b32 %r1577, %r2598, 1; + or.b32 %r2602, %r1577, 1; + add.s32 %r1578, %r2480, -1; + setp.eq.s32 %p374, %r2480, 0; + selp.b32 %r2601, 0, %r1578, %p374; + +$L__BB25_265: + mul.lo.s32 %r1579, %r2481, 7; + cvt.u64.u32 %rd329, %r2602; + shl.b64 %rd330, %rd329, %r1579; + or.b64 %rd606, %rd330, %rd606; + setp.ne.s32 %p375, %r2480, 12; + setp.ne.s32 %p376, %r474, 0; + or.pred %p377, %p375, %p376; + add.s32 %r2481, %r2481, 1; + setp.lt.u32 %p378, %r2481, 8; + or.pred %p379, %p378, %p377; + mov.u32 %r2480, %r2601; + @%p379 bra $L__BB25_221; + +$L__BB25_266: + cvt.u32.u64 %r1580, %rd606; + and.b32 %r2477, %r1580, 127; + shr.u64 %rd606, %rd606, 7; + add.s32 %r2481, %r2481, -1; + +$L__BB25_267: + mul.wide.u32 %rd331, %r2546, 2; + add.s64 %rd332, %rd14, %rd331; + st.local.u16 [%rd332], %r2612; + shl.b32 %r1581, %r2612, 2; + shl.b32 %r1582, %r2612, 1; + or.b32 %r1583, %r1581, %r1582; + and.b32 %r1584, %r1583, 256; + ld.local.u16 %r1585, [%rd47]; + and.b32 %r1586, %r1585, 128; + or.b32 %r1587, %r1584, %r1586; + ld.local.u16 %r1588, [%rd48]; + shl.b32 %r1589, %r1588, 2; + and.b32 %r1590, %r1589, 640; + or.b32 %r1591, %r1587, %r1590; + add.s32 %r1592, %r451, 4; + mul.wide.u32 %rd333, %r1592, 2; + add.s64 %rd334, %rd14, %rd333; + ld.local.u16 %r1593, [%rd334]; + shl.b32 %r1594, %r1593, 4; + and.b32 %r1595, %r1594, 512; + or.b32 %r1596, %r1591, %r1595; + and.b32 %r1597, %r2612, 7; + shr.u64 %rd58, %rd615, %r1597; + sub.s32 %r560, %r2560, %r1597; + cvt.u32.u64 %r1598, %rd58; + and.b32 %r1599, %r1598, 127; + or.b32 %r1600, %r1596, %r1599; + mul.wide.u32 %rd335, %r1600, 2; + add.s64 %rd336, %rd42, %rd335; + ld.global.u16 %r2664, [%rd336]; + setp.ne.s32 %p380, %r1596, 0; + add.s32 %r562, %r2544, 2; + setp.ge.u32 %p381, %r562, %r2; + or.pred %p382, %p381, %p380; + @%p382 bra $L__BB25_317; + + add.s32 %r563, %r2477, -2; + setp.eq.s32 %p383, %r563, -1; + selp.b32 %r2664, %r2664, 0, %p383; + setp.gt.s32 %p384, %r2477, 1; + mov.u32 %r2477, %r563; + @%p384 bra $L__BB25_317; + + setp.ne.s32 %p385, %r2481, 0; + @%p385 bra $L__BB25_316; + + mov.u32 %r2481, 0; + +$L__BB25_271: + setp.gt.u32 %p386, %r2481, 7; + @%p386 bra $L__BB25_316; + + cvt.u64.u32 %rd60, %r2480; + mul.wide.u32 %rd337, %r2480, 4; + add.s64 %rd339, %rd151, %rd337; + ld.global.nc.u32 %r569, [%rd339]; + and.b16 %rs886, %rs1139, 255; + setp.ne.s16 %p387, %rs886, 0; + @%p387 bra $L__BB25_276; + + setp.eq.s32 %p388, %r2390, 0; + mov.u16 %rs1331, 255; + @%p388 bra $L__BB25_275; + + cvt.u64.u32 %rd340, %r2389; + add.s64 %rd341, %rd340, %rd4; + add.s64 %rd342, %rd1, %rd341; + ld.global.u8 %rs1331, [%rd342]; + +$L__BB25_275: + setp.ne.s32 %p390, %r2390, 0; + selp.u32 %r1602, 1, 0, %p390; + add.s32 %r2389, %r2389, %r1602; + add.s32 %r1603, %r2390, -1; + selp.b32 %r2390, 0, %r1603, %p388; + setp.eq.s32 %p391, %r2390, 0; + or.b16 %rs888, %rs1331, 15; + selp.b16 %rs1105, %rs888, %rs1331, %p391; + and.b16 %rs889, %rs1105, 255; + mov.u16 %rs890, 8; + sub.s16 %rs1139, %rs890, %rs1104; + setp.eq.s16 %p392, %rs889, 255; + selp.u16 %rs1104, 1, 0, %p392; + +$L__BB25_276: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1604, %rs1139; + and.b32 %r1605, %r1604, 255; + mov.u32 %r1606, 1; + shl.b32 %r1607, %r1606, %r1605; + cvt.u32.u16 %r1608, %rs1105; + and.b32 %r1609, %r1607, %r1608; + and.b32 %r574, %r1609, 255; + setp.eq.s32 %p393, %r574, 0; + @%p393 bra $L__BB25_278; + + add.s32 %r1610, %r2480, 1; + min.u32 %r2653, %r1610, 12; + mov.u32 %r1611, -1; + shl.b32 %r1612, %r1611, %r569; + shl.b32 %r1613, %r1612, 1; + xor.b32 %r2654, %r1613, -2; + bra.uni $L__BB25_315; + +$L__BB25_278: + add.s64 %rd343, %rd60, -3; + setp.gt.u64 %p394, %rd343, 9; + mov.u32 %r2650, 0; + @%p394 bra $L__BB25_314; + + max.u32 %r577, %r569, 1; + add.s32 %r1617, %r577, -1; + and.b32 %r578, %r577, 3; + setp.lt.u32 %p395, %r1617, 3; + mov.u32 %r2650, 0; + @%p395 bra $L__BB25_298; + + sub.s32 %r2622, %r577, %r578; + mov.u32 %r2650, 0; + +$L__BB25_281: + and.b16 %rs892, %rs1139, 255; + setp.ne.s16 %p396, %rs892, 0; + @%p396 bra $L__BB25_285; + + setp.eq.s32 %p397, %r2390, 0; + mov.u16 %rs1338, 255; + @%p397 bra $L__BB25_284; + + cvt.u64.u32 %rd344, %r2389; + add.s64 %rd345, %rd344, %rd4; + add.s64 %rd346, %rd1, %rd345; + ld.global.u8 %rs1338, [%rd346]; + +$L__BB25_284: + setp.ne.s32 %p399, %r2390, 0; + selp.u32 %r1619, 1, 0, %p399; + add.s32 %r2389, %r2389, %r1619; + add.s32 %r1620, %r2390, -1; + selp.b32 %r2390, 0, %r1620, %p397; + setp.eq.s32 %p400, %r2390, 0; + or.b16 %rs894, %rs1338, 15; + selp.b16 %rs1105, %rs894, %rs1338, %p400; + and.b16 %rs895, %rs1105, 255; + mov.u16 %rs896, 8; + sub.s16 %rs1139, %rs896, %rs1104; + setp.eq.s16 %p401, %rs895, 255; + selp.u16 %rs1104, 1, 0, %p401; + +$L__BB25_285: + add.s16 %rs1345, %rs1139, -1; + and.b16 %rs897, %rs1345, 255; + cvt.u32.u16 %r1621, %rs1345; + and.b32 %r1622, %r1621, 255; + cvt.u32.u16 %r1623, %rs1105; + and.b32 %r2628, %r1623, 255; + shr.u32 %r1624, %r2628, %r1622; + and.b32 %r1625, %r1624, 1; + bfi.b32 %r589, %r2650, %r1625, 1, 31; + setp.ne.s16 %p402, %rs897, 0; + @%p402 bra $L__BB25_289; + + setp.eq.s32 %p403, %r2390, 0; + mov.u16 %rs1342, 255; + @%p403 bra $L__BB25_288; + + cvt.u64.u32 %rd347, %r2389; + add.s64 %rd348, %rd347, %rd4; + add.s64 %rd349, %rd1, %rd348; + ld.global.u8 %rs1342, [%rd349]; + +$L__BB25_288: + setp.ne.s32 %p405, %r2390, 0; + selp.u32 %r1626, 1, 0, %p405; + add.s32 %r2389, %r2389, %r1626; + add.s32 %r1627, %r2390, -1; + selp.b32 %r2390, 0, %r1627, %p403; + setp.eq.s32 %p406, %r2390, 0; + or.b16 %rs899, %rs1342, 15; + selp.b16 %rs1105, %rs899, %rs1342, %p406; + and.b16 %rs900, %rs1105, 255; + mov.u16 %rs901, 8; + sub.s16 %rs1345, %rs901, %rs1104; + setp.eq.s16 %p407, %rs900, 255; + selp.u16 %rs1104, 1, 0, %p407; + cvt.u32.u16 %r1628, %rs1105; + and.b32 %r2628, %r1628, 255; + +$L__BB25_289: + add.s16 %rs1349, %rs1345, -1; + and.b16 %rs902, %rs1349, 255; + cvt.u32.u16 %r1629, %rs1349; + and.b32 %r1630, %r1629, 255; + shr.u32 %r1631, %r2628, %r1630; + and.b32 %r1632, %r1631, 1; + bfi.b32 %r596, %r589, %r1632, 1, 31; + setp.ne.s16 %p408, %rs902, 0; + @%p408 bra $L__BB25_293; + + setp.eq.s32 %p409, %r2390, 0; + mov.u16 %rs1346, 255; + @%p409 bra $L__BB25_292; + + cvt.u64.u32 %rd350, %r2389; + add.s64 %rd351, %rd350, %rd4; + add.s64 %rd352, %rd1, %rd351; + ld.global.u8 %rs1346, [%rd352]; + +$L__BB25_292: + setp.ne.s32 %p411, %r2390, 0; + selp.u32 %r1633, 1, 0, %p411; + add.s32 %r2389, %r2389, %r1633; + add.s32 %r1634, %r2390, -1; + selp.b32 %r2390, 0, %r1634, %p409; + setp.eq.s32 %p412, %r2390, 0; + or.b16 %rs904, %rs1346, 15; + selp.b16 %rs1105, %rs904, %rs1346, %p412; + and.b16 %rs905, %rs1105, 255; + mov.u16 %rs906, 8; + sub.s16 %rs1349, %rs906, %rs1104; + setp.eq.s16 %p413, %rs905, 255; + selp.u16 %rs1104, 1, 0, %p413; + cvt.u32.u16 %r1635, %rs1105; + and.b32 %r2628, %r1635, 255; + +$L__BB25_293: + add.s16 %rs1353, %rs1349, -1; + and.b16 %rs907, %rs1353, 255; + cvt.u32.u16 %r1636, %rs1353; + and.b32 %r1637, %r1636, 255; + shr.u32 %r1638, %r2628, %r1637; + and.b32 %r1639, %r1638, 1; + bfi.b32 %r603, %r596, %r1639, 1, 31; + setp.ne.s16 %p414, %rs907, 0; + @%p414 bra $L__BB25_297; + + setp.eq.s32 %p415, %r2390, 0; + mov.u16 %rs1350, 255; + @%p415 bra $L__BB25_296; + + cvt.u64.u32 %rd353, %r2389; + add.s64 %rd354, %rd353, %rd4; + add.s64 %rd355, %rd1, %rd354; + ld.global.u8 %rs1350, [%rd355]; + +$L__BB25_296: + setp.ne.s32 %p417, %r2390, 0; + selp.u32 %r1640, 1, 0, %p417; + add.s32 %r2389, %r2389, %r1640; + add.s32 %r1641, %r2390, -1; + selp.b32 %r2390, 0, %r1641, %p415; + setp.eq.s32 %p418, %r2390, 0; + or.b16 %rs909, %rs1350, 15; + selp.b16 %rs1105, %rs909, %rs1350, %p418; + and.b16 %rs910, %rs1105, 255; + mov.u16 %rs911, 8; + sub.s16 %rs1353, %rs911, %rs1104; + setp.eq.s16 %p419, %rs910, 255; + selp.u16 %rs1104, 1, 0, %p419; + cvt.u32.u16 %r1642, %rs1105; + and.b32 %r2628, %r1642, 255; + +$L__BB25_297: + add.s16 %rs1139, %rs1353, -1; + cvt.u32.u16 %r1643, %rs1139; + and.b32 %r1644, %r1643, 255; + shr.u32 %r1645, %r2628, %r1644; + and.b32 %r1646, %r1645, 1; + bfi.b32 %r2650, %r603, %r1646, 1, 31; + add.s32 %r2622, %r2622, -4; + setp.ne.s32 %p420, %r2622, 0; + @%p420 bra $L__BB25_281; + +$L__BB25_298: + setp.eq.s32 %p421, %r578, 0; + @%p421 bra $L__BB25_314; + + and.b16 %rs912, %rs1139, 255; + setp.ne.s16 %p422, %rs912, 0; + @%p422 bra $L__BB25_303; + + setp.eq.s32 %p423, %r2390, 0; + mov.u16 %rs1360, 255; + @%p423 bra $L__BB25_302; + + cvt.u64.u32 %rd356, %r2389; + add.s64 %rd357, %rd356, %rd4; + add.s64 %rd358, %rd1, %rd357; + ld.global.u8 %rs1360, [%rd358]; + +$L__BB25_302: + setp.ne.s32 %p425, %r2390, 0; + selp.u32 %r1647, 1, 0, %p425; + add.s32 %r2389, %r2389, %r1647; + add.s32 %r1648, %r2390, -1; + selp.b32 %r2390, 0, %r1648, %p423; + setp.eq.s32 %p426, %r2390, 0; + or.b16 %rs914, %rs1360, 15; + selp.b16 %rs1105, %rs914, %rs1360, %p426; + and.b16 %rs915, %rs1105, 255; + mov.u16 %rs916, 8; + sub.s16 %rs1139, %rs916, %rs1104; + setp.eq.s16 %p427, %rs915, 255; + selp.u16 %rs1104, 1, 0, %p427; + +$L__BB25_303: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1649, %rs1139; + and.b32 %r1650, %r1649, 255; + cvt.u32.u16 %r1651, %rs1105; + and.b32 %r2645, %r1651, 255; + shr.u32 %r1652, %r2645, %r1650; + and.b32 %r1653, %r1652, 1; + bfi.b32 %r2650, %r2650, %r1653, 1, 31; + setp.eq.s32 %p428, %r578, 1; + @%p428 bra $L__BB25_314; + + and.b16 %rs917, %rs1139, 255; + setp.ne.s16 %p429, %rs917, 0; + @%p429 bra $L__BB25_308; + + setp.eq.s32 %p430, %r2390, 0; + mov.u16 %rs1364, 255; + @%p430 bra $L__BB25_307; + + cvt.u64.u32 %rd359, %r2389; + add.s64 %rd360, %rd359, %rd4; + add.s64 %rd361, %rd1, %rd360; + ld.global.u8 %rs1364, [%rd361]; + +$L__BB25_307: + setp.ne.s32 %p432, %r2390, 0; + selp.u32 %r1654, 1, 0, %p432; + add.s32 %r2389, %r2389, %r1654; + add.s32 %r1655, %r2390, -1; + selp.b32 %r2390, 0, %r1655, %p430; + setp.eq.s32 %p433, %r2390, 0; + or.b16 %rs919, %rs1364, 15; + selp.b16 %rs1105, %rs919, %rs1364, %p433; + and.b16 %rs920, %rs1105, 255; + mov.u16 %rs921, 8; + sub.s16 %rs1139, %rs921, %rs1104; + setp.eq.s16 %p434, %rs920, 255; + selp.u16 %rs1104, 1, 0, %p434; + cvt.u32.u16 %r1656, %rs1105; + and.b32 %r2645, %r1656, 255; + +$L__BB25_308: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1657, %rs1139; + and.b32 %r1658, %r1657, 255; + shr.u32 %r1659, %r2645, %r1658; + and.b32 %r1660, %r1659, 1; + bfi.b32 %r2650, %r2650, %r1660, 1, 31; + setp.eq.s32 %p435, %r578, 2; + @%p435 bra $L__BB25_314; + + and.b16 %rs922, %rs1139, 255; + setp.ne.s16 %p436, %rs922, 0; + @%p436 bra $L__BB25_313; + + setp.eq.s32 %p437, %r2390, 0; + mov.u16 %rs1368, 255; + @%p437 bra $L__BB25_312; + + cvt.u64.u32 %rd362, %r2389; + add.s64 %rd363, %rd362, %rd4; + add.s64 %rd364, %rd1, %rd363; + ld.global.u8 %rs1368, [%rd364]; + +$L__BB25_312: + setp.ne.s32 %p439, %r2390, 0; + selp.u32 %r1661, 1, 0, %p439; + add.s32 %r2389, %r2389, %r1661; + add.s32 %r1662, %r2390, -1; + selp.b32 %r2390, 0, %r1662, %p437; + setp.eq.s32 %p440, %r2390, 0; + or.b16 %rs924, %rs1368, 15; + selp.b16 %rs1105, %rs924, %rs1368, %p440; + and.b16 %rs925, %rs1105, 255; + mov.u16 %rs926, 8; + sub.s16 %rs1139, %rs926, %rs1104; + setp.eq.s16 %p441, %rs925, 255; + selp.u16 %rs1104, 1, 0, %p441; + cvt.u32.u16 %r1663, %rs1105; + and.b32 %r2645, %r1663, 255; + +$L__BB25_313: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1664, %rs1139; + and.b32 %r1665, %r1664, 255; + shr.u32 %r1666, %r2645, %r1665; + and.b32 %r1667, %r1666, 1; + bfi.b32 %r2650, %r2650, %r1667, 1, 31; + +$L__BB25_314: + shl.b32 %r1668, %r2650, 1; + or.b32 %r2654, %r1668, 1; + add.s32 %r1669, %r2480, -1; + setp.eq.s32 %p442, %r2480, 0; + selp.b32 %r2653, 0, %r1669, %p442; + +$L__BB25_315: + mul.lo.s32 %r1670, %r2481, 7; + cvt.u64.u32 %rd365, %r2654; + shl.b64 %rd366, %rd365, %r1670; + or.b64 %rd606, %rd366, %rd606; + setp.ne.s32 %p443, %r2480, 12; + setp.ne.s32 %p444, %r574, 0; + or.pred %p445, %p443, %p444; + add.s32 %r2481, %r2481, 1; + setp.lt.u32 %p446, %r2481, 8; + or.pred %p447, %p446, %p445; + mov.u32 %r2480, %r2653; + @%p447 bra $L__BB25_271; + +$L__BB25_316: + cvt.u32.u64 %r1671, %rd606; + and.b32 %r2477, %r1671, 127; + shr.u64 %rd606, %rd606, 7; + add.s32 %r2481, %r2481, -1; + +$L__BB25_317: + setp.lt.u32 %p448, %r562, %r2; + selp.b32 %r1672, %r2664, 0, %p448; + add.s32 %r1673, %r2546, 2; + mul.wide.u32 %rd367, %r1673, 2; + add.s64 %rd368, %rd14, %rd367; + st.local.u16 [%rd368], %r1672; + shl.b32 %r1674, %r1672, 2; + shl.b32 %r1675, %r1672, 1; + or.b32 %r1676, %r1674, %r1675; + and.b32 %r1677, %r1676, 256; + ld.local.u16 %r1678, [%rd48]; + and.b32 %r1679, %r1678, 128; + or.b32 %r2545, %r1677, %r1679; + and.b32 %r1680, %r1672, 7; + shr.u64 %rd369, %rd58, %r1680; + sub.s32 %r1681, %r560, %r1680; + cvt.u32.u64 %r1682, %rd369; + shl.b32 %r1683, %r2612, 3; + and.b32 %r1684, %r1683, 64; + shl.b32 %r1685, %r1672, 4; + and.b32 %r1686, %r1685, 128; + or.b32 %r1687, %r1686, %r1684; + and.b32 %r1688, %r1682, 63; + or.b32 %r1689, %r1688, %r1687; + mul.wide.u32 %rd370, %r1689, 2; + add.s64 %rd371, %rd41, %rd370; + ld.global.u16 %r1690, [%rd371]; + and.b32 %r1691, %r1690, 7; + shr.u64 %rd372, %rd369, %r1691; + sub.s32 %r1692, %r1681, %r1691; + cvt.u32.u64 %r1693, %rd372; + shr.u32 %r1694, %r1690, 3; + and.b32 %r1695, %r1694, 15; + mov.u32 %r1696, -1; + shl.b32 %r1697, %r1696, %r1695; + not.b32 %r1698, %r1697; + and.b32 %r1699, %r1693, %r1698; + shr.u64 %rd615, %rd372, %r1695; + sub.s32 %r2560, %r1692, %r1695; + shr.u32 %r1700, %r1690, 7; + and.b32 %r1701, %r1700, 7; + shr.u32 %r1702, %r1690, 10; + and.b32 %r1703, %r1702, 7; + mov.u32 %r1704, 255; + shl.b32 %r1705, %r1704, %r1701; + not.b32 %r1706, %r1705; + and.b32 %r1707, %r1699, %r1706; + add.s32 %r1708, %r1707, %r1703; + add.s32 %r1709, %r2546, 1; + mul.wide.u32 %rd373, %r1709, 2; + add.s64 %rd374, %rd14, %rd373; + st.local.u16 [%rd374], %r1708; + shr.u32 %r1710, %r1690, 13; + shr.u32 %r1711, %r1699, %r1701; + add.s32 %r1712, %r1711, %r1710; + add.s32 %r1713, %r2546, 3; + mul.wide.u32 %rd375, %r1713, 2; + add.s64 %rd376, %rd14, %rd375; + st.local.u16 [%rd376], %r1712; + add.s32 %r2546, %r2546, 4; + add.s32 %r2544, %r2544, 4; + setp.lt.u32 %p449, %r2544, %r2; + @%p449 bra $L__BB25_213; + + mul.wide.u32 %rd377, %r2546, 2; + add.s64 %rd378, %rd14, %rd377; + mov.u16 %rs927, 0; + st.local.v2.u16 [%rd378], {%rs927, %rs927}; + add.s32 %r2535, %r2535, 2; + setp.lt.u32 %p450, %r2535, %r3; + @%p450 bra $L__BB25_212; + +$L__BB25_319: + mov.u32 %r1714, 30; + sub.s32 %r665, %r1714, %r7; + add.s32 %r1715, %r2, 1; + shr.u32 %r1716, %r1715, 1; + add.s32 %r1717, %r1716, 2; + setp.gt.u32 %p451, %r1717, 130; + @%p451 bra $L__BB25_534; + bra.uni $L__BB25_320; + +$L__BB25_534: + mov.u32 %r2281, 2; + st.global.u32 [%rd5], %r2281; + mov.u32 %r2282, 12; + st.global.u32 [%rd5+4], %r2282; + mov.u32 %r2283, 0; + st.global.u32 [%rd5+8], %r2283; + st.global.u32 [%rd5+12], %r2283; + bra.uni $L__BB25_541; + +$L__BB25_320: + add.s32 %r666, %r7, 2; + add.s32 %r667, %r665, -1; + mov.u32 %r2665, 0; + mov.u64 %rd643, 0; + mov.u16 %rs1411, 0; + mov.u32 %r2666, %r2665; + mov.u32 %r2667, %r2665; + mov.u32 %r2668, %r10; + mov.u32 %r2735, %r2665; + mov.u32 %r2732, %r2665; + +$L__BB25_321: + mov.u32 %r670, %r2667; + mul.wide.u32 %rd380, %r2666, 2; + add.s64 %rd381, %rd14, %rd380; + ld.local.u16 %r674, [%rd381]; + ld.local.u16 %r675, [%rd381+2]; + setp.lt.u32 %p452, %r666, %r675; + @%p452 bra $L__BB25_533; + + and.b32 %r1724, %r674, 16; + setp.eq.s32 %p453, %r1724, 0; + mov.u32 %r2685, 0; + mov.u32 %r2677, %r2685; + @%p453 bra $L__BB25_328; + + setp.gt.u32 %p454, %r2735, 31; + @%p454 bra $L__BB25_327; + +$L__BB25_324: + setp.ge.u32 %p455, %r2732, %r16; + mov.u16 %rs1386, 255; + @%p455 bra $L__BB25_326; + + add.s32 %r678, %r2732, 1; + cvt.u64.u32 %rd382, %r2732; + add.s64 %rd383, %rd382, %rd4; + add.s64 %rd384, %rd1, %rd383; + ld.global.u8 %rs1386, [%rd384]; + mov.u32 %r2732, %r678; + +$L__BB25_326: + and.b16 %rs930, %rs1386, 255; + cvt.u64.u16 %rd385, %rs1386; + and.b64 %rd386, %rd385, 255; + shl.b64 %rd387, %rd386, %r2735; + or.b64 %rd643, %rd387, %rd643; + add.s32 %r1725, %r2735, 8; + cvt.u32.u16 %r1726, %rs1411; + cvt.s32.s8 %r1727, %r1726; + sub.s32 %r2735, %r1725, %r1727; + setp.eq.s16 %p456, %rs930, 255; + selp.u16 %rs1411, 1, 0, %p456; + setp.lt.u32 %p457, %r2735, 33; + @%p457 bra $L__BB25_324; + +$L__BB25_327: + shr.u32 %r1728, %r674, 12; + and.b32 %r1729, %r1728, 1; + sub.s32 %r1730, %r675, %r1729; + shr.u64 %rd70, %rd643, %r1730; + sub.s32 %r2735, %r2735, %r1730; + cvt.u32.u64 %r1731, %rd643; + shl.b32 %r1732, %r1731, 31; + setp.eq.s32 %p458, %r1730, 0; + mov.u32 %r1733, -1; + shl.b32 %r1734, %r1733, %r1730; + not.b32 %r1735, %r1734; + selp.b32 %r1736, 0, %r1735, %p458; + and.b32 %r1737, %r1736, %r1731; + shr.u32 %r1738, %r674, 8; + and.b32 %r1739, %r1738, 1; + shl.b32 %r1740, %r1739, %r1730; + or.b32 %r1741, %r1740, %r1737; + or.b32 %r1742, %r1741, 1; + add.s32 %r1743, %r1742, 2; + shl.b32 %r1744, %r1743, %r667; + or.b32 %r2677, %r1744, %r1732; + mov.u64 %rd643, %rd70; + +$L__BB25_328: + mul.wide.u32 %rd388, %r2668, 4; + add.s64 %rd389, %rd2, %rd388; + st.global.u32 [%rd389], %r2677; + and.b32 %r1747, %r674, 32; + setp.eq.s32 %p459, %r1747, 0; + mov.u32 %r2686, %r2685; + @%p459 bra $L__BB25_334; + + setp.gt.u32 %p460, %r2735, 31; + @%p460 bra $L__BB25_333; + +$L__BB25_330: + setp.ge.u32 %p461, %r2732, %r16; + mov.u16 %rs1390, 255; + @%p461 bra $L__BB25_332; + + add.s32 %r690, %r2732, 1; + cvt.u64.u32 %rd390, %r2732; + add.s64 %rd391, %rd390, %rd4; + add.s64 %rd392, %rd1, %rd391; + ld.global.u8 %rs1390, [%rd392]; + mov.u32 %r2732, %r690; + +$L__BB25_332: + and.b16 %rs932, %rs1390, 255; + cvt.u64.u16 %rd393, %rs1390; + and.b64 %rd394, %rd393, 255; + shl.b64 %rd395, %rd394, %r2735; + or.b64 %rd643, %rd395, %rd643; + add.s32 %r1748, %r2735, 8; + cvt.u32.u16 %r1749, %rs1411; + cvt.s32.s8 %r1750, %r1749; + sub.s32 %r2735, %r1748, %r1750; + setp.eq.s16 %p462, %rs932, 255; + selp.u16 %rs1411, 1, 0, %p462; + setp.lt.u32 %p463, %r2735, 33; + @%p463 bra $L__BB25_330; + +$L__BB25_333: + shr.u32 %r1751, %r674, 13; + and.b32 %r1752, %r1751, 1; + sub.s32 %r1753, %r675, %r1752; + shr.u64 %rd75, %rd643, %r1753; + sub.s32 %r2735, %r2735, %r1753; + cvt.u32.u64 %r1754, %rd643; + shl.b32 %r1755, %r1754, 31; + setp.eq.s32 %p464, %r1753, 0; + mov.u32 %r1756, -1; + shl.b32 %r1757, %r1756, %r1753; + not.b32 %r1758, %r1757; + selp.b32 %r1759, 0, %r1758, %p464; + and.b32 %r1760, %r1759, %r1754; + shr.u32 %r1761, %r674, 9; + and.b32 %r1762, %r1761, 1; + shl.b32 %r1763, %r1762, %r1753; + or.b32 %r1764, %r1763, %r1760; + or.b32 %r2686, %r1764, 1; + add.s32 %r1765, %r2686, 2; + shl.b32 %r1766, %r1765, %r667; + or.b32 %r2685, %r1766, %r1755; + mov.u64 %rd643, %rd75; + +$L__BB25_334: + setp.lt.u32 %p465, %r3, 2; + @%p465 bra $L__BB25_336; + + add.s32 %r1767, %r2668, %r9; + mul.wide.u32 %rd396, %r1767, 4; + add.s64 %rd397, %rd2, %rd396; + st.global.u32 [%rd397], %r2685; + +$L__BB25_336: + or.b32 %r1768, %r2686, %r2665; + add.u64 %rd77, %SPL, 6192; + mul.wide.u32 %rd399, %r670, 4; + add.s64 %rd400, %rd77, %rd399; + st.local.u32 [%rd400], %r1768; + add.s32 %r702, %r2668, 1; + add.s32 %r1769, %r2666, 1; + setp.lt.u32 %p466, %r1769, %r2; + @%p466 bra $L__BB25_338; + bra.uni $L__BB25_337; + +$L__BB25_338: + and.b32 %r1772, %r674, 64; + setp.eq.s32 %p467, %r1772, 0; + mov.u32 %r2702, 0; + mov.u32 %r2694, %r2702; + @%p467 bra $L__BB25_344; + + setp.gt.u32 %p468, %r2735, 31; + @%p468 bra $L__BB25_343; + +$L__BB25_340: + setp.ge.u32 %p469, %r2732, %r16; + mov.u16 %rs1394, 255; + @%p469 bra $L__BB25_342; + + add.s32 %r705, %r2732, 1; + cvt.u64.u32 %rd401, %r2732; + add.s64 %rd402, %rd401, %rd4; + add.s64 %rd403, %rd1, %rd402; + ld.global.u8 %rs1394, [%rd403]; + mov.u32 %r2732, %r705; + +$L__BB25_342: + and.b16 %rs934, %rs1394, 255; + cvt.u64.u16 %rd404, %rs1394; + and.b64 %rd405, %rd404, 255; + shl.b64 %rd406, %rd405, %r2735; + or.b64 %rd643, %rd406, %rd643; + add.s32 %r1773, %r2735, 8; + cvt.u32.u16 %r1774, %rs1411; + cvt.s32.s8 %r1775, %r1774; + sub.s32 %r2735, %r1773, %r1775; + setp.eq.s16 %p470, %rs934, 255; + selp.u16 %rs1411, 1, 0, %p470; + setp.lt.u32 %p471, %r2735, 33; + @%p471 bra $L__BB25_340; + +$L__BB25_343: + shr.u32 %r1776, %r674, 14; + and.b32 %r1777, %r1776, 1; + sub.s32 %r1778, %r675, %r1777; + shr.u64 %rd81, %rd643, %r1778; + sub.s32 %r2735, %r2735, %r1778; + cvt.u32.u64 %r1779, %rd643; + shl.b32 %r1780, %r1779, 31; + setp.eq.s32 %p472, %r1778, 0; + mov.u32 %r1781, -1; + shl.b32 %r1782, %r1781, %r1778; + not.b32 %r1783, %r1782; + selp.b32 %r1784, 0, %r1783, %p472; + and.b32 %r1785, %r1784, %r1779; + shr.u32 %r1786, %r674, 10; + and.b32 %r1787, %r1786, 1; + shl.b32 %r1788, %r1787, %r1778; + or.b32 %r1789, %r1788, %r1785; + or.b32 %r1790, %r1789, 1; + add.s32 %r1791, %r1790, 2; + shl.b32 %r1792, %r1791, %r667; + or.b32 %r2694, %r1792, %r1780; + mov.u64 %rd643, %rd81; + +$L__BB25_344: + mul.wide.u32 %rd407, %r702, 4; + add.s64 %rd408, %rd2, %rd407; + st.global.u32 [%rd408], %r2694; + and.b32 %r1795, %r674, 128; + setp.eq.s32 %p473, %r1795, 0; + mov.u32 %r2665, %r2702; + @%p473 bra $L__BB25_350; + + setp.gt.u32 %p474, %r2735, 31; + @%p474 bra $L__BB25_349; + +$L__BB25_346: + setp.ge.u32 %p475, %r2732, %r16; + mov.u16 %rs1398, 255; + @%p475 bra $L__BB25_348; + + add.s32 %r717, %r2732, 1; + cvt.u64.u32 %rd409, %r2732; + add.s64 %rd410, %rd409, %rd4; + add.s64 %rd411, %rd1, %rd410; + ld.global.u8 %rs1398, [%rd411]; + mov.u32 %r2732, %r717; + +$L__BB25_348: + and.b16 %rs936, %rs1398, 255; + cvt.u64.u16 %rd412, %rs1398; + and.b64 %rd413, %rd412, 255; + shl.b64 %rd414, %rd413, %r2735; + or.b64 %rd643, %rd414, %rd643; + add.s32 %r1796, %r2735, 8; + cvt.u32.u16 %r1797, %rs1411; + cvt.s32.s8 %r1798, %r1797; + sub.s32 %r2735, %r1796, %r1798; + setp.eq.s16 %p476, %rs936, 255; + selp.u16 %rs1411, 1, 0, %p476; + setp.lt.u32 %p477, %r2735, 33; + @%p477 bra $L__BB25_346; + +$L__BB25_349: + shr.u32 %r1799, %r674, 15; + sub.s32 %r1800, %r675, %r1799; + shr.u64 %rd86, %rd643, %r1800; + sub.s32 %r2735, %r2735, %r1800; + cvt.u32.u64 %r1801, %rd643; + shl.b32 %r1802, %r1801, 31; + setp.eq.s32 %p478, %r1800, 0; + mov.u32 %r1803, -1; + shl.b32 %r1804, %r1803, %r1800; + not.b32 %r1805, %r1804; + selp.b32 %r1806, 0, %r1805, %p478; + and.b32 %r1807, %r1806, %r1801; + shr.u32 %r1808, %r674, 11; + and.b32 %r1809, %r1808, 1; + shl.b32 %r1810, %r1809, %r1800; + or.b32 %r1811, %r1810, %r1807; + or.b32 %r2665, %r1811, 1; + add.s32 %r1812, %r2665, 2; + shl.b32 %r1813, %r1812, %r667; + or.b32 %r2702, %r1813, %r1802; + mov.u64 %rd643, %rd86; + +$L__BB25_350: + @%p465 bra $L__BB25_352; + + add.s32 %r1814, %r702, %r9; + mul.wide.u32 %rd415, %r1814, 4; + add.s64 %rd416, %rd2, %rd415; + st.global.u32 [%rd416], %r2702; + +$L__BB25_352: + add.s32 %r2668, %r2668, 2; + add.s32 %r2667, %r670, 1; + add.s32 %r2666, %r2666, 2; + setp.lt.u32 %p480, %r2666, %r2; + @%p480 bra $L__BB25_321; + bra.uni $L__BB25_353; + +$L__BB25_533: + mov.u32 %r2274, 1; + st.global.u32 [%rd5], %r2274; + mov.u32 %r2275, 13; + st.global.u32 [%rd5+4], %r2275; + mov.u32 %r2276, 0; + st.global.u32 [%rd5+8], %r2276; + st.global.u32 [%rd5+12], %r2276; + bra.uni $L__BB25_541; + +$L__BB25_337: + mov.u32 %r2665, 0; + +$L__BB25_353: + add.s32 %r1815, %r670, 1; + mul.wide.u32 %rd419, %r1815, 4; + add.s64 %rd420, %rd77, %rd419; + st.local.u32 [%rd420], %r2665; + @%p304 bra $L__BB25_388; + + mov.u32 %r2708, 2; + +$L__BB25_355: + shr.u32 %r1821, %r2708, 1; + mul.lo.s32 %r2712, %r1821, %r1148; + mad.lo.s32 %r2714, %r2708, %r9, %r10; + add.s32 %r741, %r2708, 1; + ld.local.u32 %r2711, [%rd77]; + mov.u32 %r2713, 0; + mov.u32 %r2715, %r2713; + mov.u32 %r2716, %r2713; + +$L__BB25_356: + mul.wide.u32 %rd421, %r2712, 2; + add.s64 %rd422, %rd14, %rd421; + ld.local.v2.u16 {%rs937, %rs938}, [%rd422]; + cvt.u32.u16 %r751, %rs937; + cvt.u32.u16 %r1822, %rs938; + and.b32 %r1823, %r751, 240; + add.s32 %r1824, %r1823, 240; + and.b32 %r1825, %r1824, %r1823; + add.s32 %r752, %r2713, 1; + mul.wide.u32 %rd423, %r752, 4; + add.s64 %rd91, %rd77, %rd423; + ld.local.u32 %r753, [%rd91]; + or.b32 %r1826, %r2711, %r753; + or.b32 %r1827, %r1826, 2; + clz.b32 %r1828, %r1827; + xor.b32 %r1829, %r1828, 31; + setp.eq.s32 %p482, %r1825, 0; + selp.b32 %r1830, 1, %r1829, %p482; + add.s32 %r754, %r1830, %r1822; + setp.gt.u32 %p483, %r754, %r666; + @%p483 bra $L__BB25_532; + + and.b32 %r1832, %r751, 16; + setp.eq.s32 %p484, %r1832, 0; + mov.u32 %r2733, 0; + mov.u32 %r2725, %r2733; + @%p484 bra $L__BB25_363; + + setp.gt.u32 %p485, %r2735, 31; + @%p485 bra $L__BB25_362; + +$L__BB25_359: + setp.ge.u32 %p486, %r2732, %r16; + mov.u16 %rs1405, 255; + @%p486 bra $L__BB25_361; + + add.s32 %r757, %r2732, 1; + cvt.u64.u32 %rd424, %r2732; + add.s64 %rd425, %rd424, %rd4; + add.s64 %rd426, %rd1, %rd425; + ld.global.u8 %rs1405, [%rd426]; + mov.u32 %r2732, %r757; + +$L__BB25_361: + and.b16 %rs942, %rs1405, 255; + cvt.u64.u16 %rd427, %rs1405; + and.b64 %rd428, %rd427, 255; + shl.b64 %rd429, %rd428, %r2735; + or.b64 %rd643, %rd429, %rd643; + add.s32 %r1833, %r2735, 8; + cvt.u32.u16 %r1834, %rs1411; + cvt.s32.s8 %r1835, %r1834; + sub.s32 %r2735, %r1833, %r1835; + setp.eq.s16 %p487, %rs942, 255; + selp.u16 %rs1411, 1, 0, %p487; + setp.lt.u32 %p488, %r2735, 33; + @%p488 bra $L__BB25_359; + +$L__BB25_362: + shr.u32 %r1836, %r751, 12; + and.b32 %r1837, %r1836, 1; + sub.s32 %r1838, %r754, %r1837; + shr.u64 %rd95, %rd643, %r1838; + sub.s32 %r2735, %r2735, %r1838; + cvt.u32.u64 %r1839, %rd643; + shl.b32 %r1840, %r1839, 31; + setp.eq.s32 %p489, %r1838, 0; + mov.u32 %r1841, -1; + shl.b32 %r1842, %r1841, %r1838; + not.b32 %r1843, %r1842; + selp.b32 %r1844, 0, %r1843, %p489; + and.b32 %r1845, %r1844, %r1839; + shr.u32 %r1846, %r751, 8; + and.b32 %r1847, %r1846, 1; + shl.b32 %r1848, %r1847, %r1838; + or.b32 %r1849, %r1848, %r1845; + or.b32 %r1850, %r1849, 1; + add.s32 %r1851, %r1850, 2; + shl.b32 %r1852, %r1851, %r667; + or.b32 %r2725, %r1852, %r1840; + mov.u64 %rd643, %rd95; + +$L__BB25_363: + mul.wide.u32 %rd430, %r2714, 4; + add.s64 %rd431, %rd2, %rd430; + st.global.u32 [%rd431], %r2725; + and.b32 %r1855, %r751, 32; + setp.eq.s32 %p490, %r1855, 0; + mov.u32 %r2734, %r2733; + @%p490 bra $L__BB25_369; + + setp.gt.u32 %p491, %r2735, 31; + @%p491 bra $L__BB25_368; + +$L__BB25_365: + setp.ge.u32 %p492, %r2732, %r16; + mov.u16 %rs1409, 255; + @%p492 bra $L__BB25_367; + + add.s32 %r769, %r2732, 1; + cvt.u64.u32 %rd432, %r2732; + add.s64 %rd433, %rd432, %rd4; + add.s64 %rd434, %rd1, %rd433; + ld.global.u8 %rs1409, [%rd434]; + mov.u32 %r2732, %r769; + +$L__BB25_367: + and.b16 %rs944, %rs1409, 255; + cvt.u64.u16 %rd435, %rs1409; + and.b64 %rd436, %rd435, 255; + shl.b64 %rd437, %rd436, %r2735; + or.b64 %rd643, %rd437, %rd643; + add.s32 %r1856, %r2735, 8; + cvt.u32.u16 %r1857, %rs1411; + cvt.s32.s8 %r1858, %r1857; + sub.s32 %r2735, %r1856, %r1858; + setp.eq.s16 %p493, %rs944, 255; + selp.u16 %rs1411, 1, 0, %p493; + setp.lt.u32 %p494, %r2735, 33; + @%p494 bra $L__BB25_365; + +$L__BB25_368: + shr.u32 %r1859, %r751, 13; + and.b32 %r1860, %r1859, 1; + sub.s32 %r1861, %r754, %r1860; + shr.u64 %rd100, %rd643, %r1861; + sub.s32 %r2735, %r2735, %r1861; + cvt.u32.u64 %r1862, %rd643; + shl.b32 %r1863, %r1862, 31; + setp.eq.s32 %p495, %r1861, 0; + mov.u32 %r1864, -1; + shl.b32 %r1865, %r1864, %r1861; + not.b32 %r1866, %r1865; + selp.b32 %r1867, 0, %r1866, %p495; + and.b32 %r1868, %r1867, %r1862; + shr.u32 %r1869, %r751, 9; + and.b32 %r1870, %r1869, 1; + shl.b32 %r1871, %r1870, %r1861; + or.b32 %r1872, %r1871, %r1868; + or.b32 %r2734, %r1872, 1; + add.s32 %r1873, %r2734, 2; + shl.b32 %r1874, %r1873, %r667; + or.b32 %r2733, %r1874, %r1863; + mov.u64 %rd643, %rd100; + +$L__BB25_369: + setp.ge.u32 %p496, %r741, %r3; + @%p496 bra $L__BB25_371; + + add.s32 %r1875, %r2714, %r9; + mul.wide.u32 %rd438, %r1875, 4; + add.s64 %rd439, %rd2, %rd438; + st.global.u32 [%rd439], %r2733; + +$L__BB25_371: + or.b32 %r1877, %r2734, %r2715; + mul.wide.u32 %rd440, %r2713, 4; + add.s64 %rd441, %rd77, %rd440; + st.local.u32 [%rd441], %r1877; + add.s32 %r781, %r2714, 1; + add.s32 %r1878, %r2716, 1; + setp.ge.u32 %p497, %r1878, %r2; + mov.u32 %r2715, 0; + @%p497 bra $L__BB25_387; + + and.b32 %r1880, %r751, 64; + setp.eq.s32 %p498, %r1880, 0; + mov.u32 %r2750, 0; + mov.u32 %r2742, %r2750; + @%p498 bra $L__BB25_378; + + setp.gt.u32 %p499, %r2735, 31; + @%p499 bra $L__BB25_377; + +$L__BB25_374: + setp.ge.u32 %p500, %r2732, %r16; + mov.u16 %rs1413, 255; + @%p500 bra $L__BB25_376; + + add.s32 %r784, %r2732, 1; + cvt.u64.u32 %rd442, %r2732; + add.s64 %rd443, %rd442, %rd4; + add.s64 %rd444, %rd1, %rd443; + ld.global.u8 %rs1413, [%rd444]; + mov.u32 %r2732, %r784; + +$L__BB25_376: + and.b16 %rs946, %rs1413, 255; + cvt.u64.u16 %rd445, %rs1413; + and.b64 %rd446, %rd445, 255; + shl.b64 %rd447, %rd446, %r2735; + or.b64 %rd643, %rd447, %rd643; + add.s32 %r1881, %r2735, 8; + cvt.u32.u16 %r1882, %rs1411; + cvt.s32.s8 %r1883, %r1882; + sub.s32 %r2735, %r1881, %r1883; + setp.eq.s16 %p501, %rs946, 255; + selp.u16 %rs1411, 1, 0, %p501; + setp.lt.u32 %p502, %r2735, 33; + @%p502 bra $L__BB25_374; + +$L__BB25_377: + shr.u32 %r1884, %r751, 14; + and.b32 %r1885, %r1884, 1; + sub.s32 %r1886, %r754, %r1885; + shr.u64 %rd105, %rd643, %r1886; + sub.s32 %r2735, %r2735, %r1886; + cvt.u32.u64 %r1887, %rd643; + shl.b32 %r1888, %r1887, 31; + setp.eq.s32 %p503, %r1886, 0; + mov.u32 %r1889, -1; + shl.b32 %r1890, %r1889, %r1886; + not.b32 %r1891, %r1890; + selp.b32 %r1892, 0, %r1891, %p503; + and.b32 %r1893, %r1892, %r1887; + shr.u32 %r1894, %r751, 10; + and.b32 %r1895, %r1894, 1; + shl.b32 %r1896, %r1895, %r1886; + or.b32 %r1897, %r1896, %r1893; + or.b32 %r1898, %r1897, 1; + add.s32 %r1899, %r1898, 2; + shl.b32 %r1900, %r1899, %r667; + or.b32 %r2742, %r1900, %r1888; + mov.u64 %rd643, %rd105; + +$L__BB25_378: + mul.wide.u32 %rd448, %r781, 4; + add.s64 %rd449, %rd2, %rd448; + st.global.u32 [%rd449], %r2742; + and.b32 %r1903, %r751, 128; + setp.eq.s32 %p504, %r1903, 0; + mov.u32 %r2715, %r2750; + @%p504 bra $L__BB25_384; + + setp.gt.u32 %p505, %r2735, 31; + @%p505 bra $L__BB25_383; + +$L__BB25_380: + setp.ge.u32 %p506, %r2732, %r16; + mov.u16 %rs1417, 255; + @%p506 bra $L__BB25_382; + + add.s32 %r796, %r2732, 1; + cvt.u64.u32 %rd450, %r2732; + add.s64 %rd451, %rd450, %rd4; + add.s64 %rd452, %rd1, %rd451; + ld.global.u8 %rs1417, [%rd452]; + mov.u32 %r2732, %r796; + +$L__BB25_382: + and.b16 %rs948, %rs1417, 255; + cvt.u64.u16 %rd453, %rs1417; + and.b64 %rd454, %rd453, 255; + shl.b64 %rd455, %rd454, %r2735; + or.b64 %rd643, %rd455, %rd643; + add.s32 %r1904, %r2735, 8; + cvt.u32.u16 %r1905, %rs1411; + cvt.s32.s8 %r1906, %r1905; + sub.s32 %r2735, %r1904, %r1906; + setp.eq.s16 %p507, %rs948, 255; + selp.u16 %rs1411, 1, 0, %p507; + setp.lt.u32 %p508, %r2735, 33; + @%p508 bra $L__BB25_380; + +$L__BB25_383: + shr.u32 %r1907, %r751, 15; + sub.s32 %r1908, %r754, %r1907; + shr.u64 %rd110, %rd643, %r1908; + sub.s32 %r2735, %r2735, %r1908; + cvt.u32.u64 %r1909, %rd643; + shl.b32 %r1910, %r1909, 31; + setp.eq.s32 %p509, %r1908, 0; + mov.u32 %r1911, -1; + shl.b32 %r1912, %r1911, %r1908; + not.b32 %r1913, %r1912; + selp.b32 %r1914, 0, %r1913, %p509; + and.b32 %r1915, %r1914, %r1909; + shr.u32 %r1916, %r751, 11; + and.b32 %r1917, %r1916, 1; + shl.b32 %r1918, %r1917, %r1908; + or.b32 %r1919, %r1918, %r1915; + or.b32 %r2715, %r1919, 1; + add.s32 %r1920, %r2715, 2; + shl.b32 %r1921, %r1920, %r667; + or.b32 %r2750, %r1921, %r1910; + mov.u64 %rd643, %rd110; + +$L__BB25_384: + @%p496 bra $L__BB25_386; + + add.s32 %r1922, %r781, %r9; + mul.wide.u32 %rd456, %r1922, 4; + add.s64 %rd457, %rd2, %rd456; + st.global.u32 [%rd457], %r2750; + +$L__BB25_386: + add.s32 %r2714, %r2714, 2; + add.s32 %r2712, %r2712, 2; + add.s32 %r2716, %r2716, 2; + setp.lt.u32 %p511, %r2716, %r2; + mov.u32 %r2711, %r753; + mov.u32 %r2713, %r752; + @%p511 bra $L__BB25_356; + +$L__BB25_387: + st.local.u32 [%rd91], %r2715; + add.s32 %r2708, %r2708, 2; + setp.lt.u32 %p512, %r2708, %r3; + @%p512 bra $L__BB25_355; + +$L__BB25_388: + setp.lt.u32 %p513, %r13, 2; + @%p513 bra $L__BB25_541; + + add.s32 %r1923, %r3, 3; + shr.u32 %r815, %r1923, 2; + add.s32 %r1924, %r815, 1; + add.s32 %r816, %r2, 3; + shr.u32 %r817, %r816, 2; + add.s32 %r1925, %r817, 9; + and.b32 %r818, %r1925, 2147483640; + add.s32 %r819, %r817, 8; + setp.gt.u32 %p514, %r818, 72; + mul.lo.s32 %r1926, %r1924, %r818; + setp.gt.u32 %p515, %r1926, 528; + or.pred %p516, %p514, %p515; + @%p516 bra $L__BB25_531; + bra.uni $L__BB25_390; + +$L__BB25_531: + mov.u32 %r2260, 2; + st.global.u32 [%rd5], %r2260; + mov.u32 %r2261, 15; + st.global.u32 [%rd5+4], %r2261; + mov.u32 %r2262, 0; + st.global.u32 [%rd5+8], %r2262; + st.global.u32 [%rd5+12], %r2262; + bra.uni $L__BB25_541; + +$L__BB25_532: + mov.u32 %r2267, 1; + st.global.u32 [%rd5], %r2267; + mov.u32 %r2268, 14; + st.global.u32 [%rd5+4], %r2268; + mov.u32 %r2269, 0; + st.global.u32 [%rd5+8], %r2269; + st.global.u32 [%rd5+12], %r2269; + +$L__BB25_541: + ret; + +$L__BB25_390: + setp.gt.u32 %p517, %r819, 72; + @%p517 bra $L__BB25_530; + bra.uni $L__BB25_391; + +$L__BB25_530: + mov.u32 %r2253, 2; + st.global.u32 [%rd5], %r2253; + mov.u32 %r2254, 16; + st.global.u32 [%rd5+4], %r2254; + mov.u32 %r2255, 0; + st.global.u32 [%rd5+8], %r2255; + st.global.u32 [%rd5+12], %r2255; + bra.uni $L__BB25_541; + +$L__BB25_391: + add.u64 %rd113, %SPL, 6720; + mov.u32 %r1927, 0; + mov.u32 %r2756, %r1927; + +$L__BB25_392: + shr.u32 %r1930, %r2756, 1; + mul.lo.s32 %r2758, %r1930, %r1148; + shr.u32 %r1931, %r2756, 2; + mul.lo.s32 %r2757, %r1931, %r818; + mov.u32 %r2759, %r1927; + +$L__BB25_393: + mul.wide.u32 %rd459, %r2758, 2; + add.s64 %rd460, %rd14, %rd459; + ld.local.u16 %rs949, [%rd460]; + and.b16 %rs950, %rs949, 48; + shr.u16 %rs951, %rs950, 4; + and.b16 %rs952, %rs949, 192; + shr.u16 %rs953, %rs952, 2; + add.s32 %r1932, %r2758, 2; + mul.wide.u32 %rd461, %r1932, 2; + add.s64 %rd462, %rd14, %rd461; + ld.local.u16 %rs954, [%rd462]; + shl.b16 %rs955, %rs954, 4; + and.b16 %rs956, %rs955, 768; + shl.b16 %rs957, %rs954, 6; + and.b16 %rs958, %rs957, 12288; + add.s32 %r1933, %r2758, %r1148; + mul.wide.u32 %rd463, %r1933, 2; + add.s64 %rd464, %rd14, %rd463; + ld.local.u16 %rs959, [%rd464]; + and.b16 %rs960, %rs959, 48; + shr.u16 %rs961, %rs960, 2; + and.b16 %rs962, %rs959, 192; + add.s32 %r1934, %r1933, 2; + mul.wide.u32 %rd465, %r1934, 2; + add.s64 %rd466, %rd14, %rd465; + ld.local.u16 %rs963, [%rd466]; + shl.b16 %rs964, %rs963, 6; + and.b16 %rs965, %rs964, 3072; + shl.b16 %rs966, %rs963, 8; + and.b16 %rs967, %rs966, -16384; + or.b16 %rs968, %rs951, %rs953; + or.b16 %rs969, %rs968, %rs958; + or.b16 %rs970, %rs969, %rs956; + or.b16 %rs971, %rs970, %rs962; + or.b16 %rs972, %rs971, %rs961; + or.b16 %rs973, %rs972, %rs967; + or.b16 %rs974, %rs973, %rs965; + mul.wide.u32 %rd467, %r2757, 2; + add.s64 %rd468, %rd113, %rd467; + st.local.u16 [%rd468], %rs974; + add.s32 %r2758, %r2758, 4; + add.s32 %r2757, %r2757, 1; + add.s32 %r2759, %r2759, 4; + setp.lt.u32 %p518, %r2759, %r2; + @%p518 bra $L__BB25_393; + + mul.wide.u32 %rd469, %r2757, 2; + add.s64 %rd470, %rd113, %rd469; + mov.u16 %rs975, 0; + st.local.u16 [%rd470], %rs975; + add.s32 %r2756, %r2756, 4; + setp.lt.u32 %p519, %r2756, %r3; + @%p519 bra $L__BB25_392; + + mul.lo.s32 %r831, %r818, %r815; + add.s32 %r832, %r817, 1; + and.b32 %r2871, %r832, 3; + setp.lt.u32 %p520, %r816, 12; + mov.u32 %r2762, 0; + @%p520 bra $L__BB25_398; + + sub.s32 %r2761, %r832, %r2871; + mov.u32 %r2762, 0; + +$L__BB25_397: + add.s32 %r1937, %r2762, %r831; + mul.wide.u32 %rd471, %r1937, 2; + add.s64 %rd472, %rd113, %rd471; + mov.u16 %rs976, 0; + st.local.v4.u16 [%rd472], {%rs976, %rs976, %rs976, %rs976}; + add.s32 %r2762, %r2762, 4; + add.s32 %r2761, %r2761, -4; + setp.ne.s32 %p521, %r2761, 0; + @%p521 bra $L__BB25_397; + +$L__BB25_398: + setp.eq.s32 %p522, %r2871, 0; + @%p522 bra $L__BB25_401; + + mov.u32 %r2764, %r2871; + +$L__BB25_400: + .pragma "nounroll"; + add.s32 %r1938, %r2762, %r831; + mul.wide.u32 %rd473, %r1938, 2; + add.s64 %rd474, %rd113, %rd473; + mov.u16 %rs977, 0; + st.local.u16 [%rd474], %rs977; + add.s32 %r2762, %r2762, 1; + add.s32 %r2764, %r2764, -1; + setp.ne.s32 %p523, %r2764, 0; + @%p523 bra $L__BB25_400; + +$L__BB25_401: + and.b32 %r2768, %r819, 3; + sub.s32 %r2766, %r819, %r2768; + add.u64 %rd114, %SPL, 7776; + mov.u32 %r2767, 0; + +$L__BB25_402: + mul.wide.u32 %rd476, %r2767, 2; + add.s64 %rd477, %rd114, %rd476; + mov.u16 %rs978, 0; + st.local.u16 [%rd477], %rs978; + st.local.u16 [%rd477+2], %rs978; + st.local.u16 [%rd477+4], %rs978; + st.local.u16 [%rd477+6], %rs978; + add.s32 %r2767, %r2767, 4; + add.s32 %r2766, %r2766, -4; + setp.ne.s32 %p524, %r2766, 0; + @%p524 bra $L__BB25_402; + + setp.eq.s32 %p525, %r2768, 0; + @%p525 bra $L__BB25_405; + +$L__BB25_404: + .pragma "nounroll"; + mul.wide.u32 %rd478, %r2767, 2; + add.s64 %rd479, %rd114, %rd478; + mov.u16 %rs979, 0; + st.local.u16 [%rd479], %rs979; + add.s32 %r2767, %r2767, 1; + add.s32 %r2768, %r2768, -1; + setp.ne.s32 %p526, %r2768, 0; + @%p526 bra $L__BB25_404; + +$L__BB25_405: + cvt.u64.u32 %rd591, %r5; + add.s64 %rd115, %rd591, %rd4; + add.s32 %r854, %r665, -2; + mov.u32 %r1943, 3; + shl.b32 %r855, %r1943, %r854; + shl.b32 %r856, %r9, 1; + mul.lo.s32 %r857, %r9, 3; + mov.u16 %rs1426, 0; + mov.u32 %r1942, 0; + mov.u64 %rd654, 0; + mov.u32 %r2769, %r1942; + mov.u32 %r2778, %r1942; + mov.u32 %r2779, %r1942; + +$L__BB25_406: + sub.s32 %r1946, %r3, %r2769; + setp.lt.u32 %p527, %r1946, 4; + setp.lt.u32 %p528, %r1946, 3; + setp.lt.u32 %p529, %r1946, 2; + selp.b32 %r1947, 4369, 13107, %p529; + selp.b32 %r1948, %r1947, 30583, %p528; + selp.b32 %r861, %r1948, 65535, %p527; + shr.u32 %r1949, %r2769, 2; + mul.lo.s32 %r862, %r1949, %r818; + add.s32 %r863, %r862, %r818; + mad.lo.s32 %r864, %r2769, %r9, %r10; + mov.u32 %r2772, %r1942; + mov.u32 %r2773, %r1942; + +$L__BB25_407: + add.s32 %r1951, %r2772, 4; + sub.s32 %r1952, %r1951, %r2; + mov.u32 %r2802, 0; + max.s32 %r1953, %r1952, 0; + shl.b32 %r1954, %r1953, 2; + shr.u32 %r1955, %r861, %r1954; + shr.u32 %r869, %r2772, 2; + mul.wide.u32 %rd482, %r869, 2; + add.s64 %rd118, %rd114, %rd482; + ld.local.u16 %rs981, [%rd118]; + ld.local.u16 %rs982, [%rd118+2]; + mov.b32 %r1956, {%rs981, %rs982}; + add.s32 %r1957, %r863, %r869; + mul.wide.u32 %rd483, %r1957, 2; + add.s64 %rd484, %rd113, %rd483; + ld.local.u16 %rs983, [%rd484]; + add.s32 %r1958, %r1957, 1; + mul.wide.u32 %rd485, %r1958, 2; + add.s64 %rd486, %rd113, %rd485; + ld.local.u16 %rs984, [%rd486]; + mov.b32 %r1959, {%rs983, %rs984}; + and.b32 %r1960, %r1956, -2004318072; + shr.u32 %r1961, %r1960, 3; + shl.b32 %r1962, %r1959, 3; + and.b32 %r1963, %r1962, -2004318072; + setp.eq.s32 %p530, %r11, 0; + selp.b32 %r1964, %r1963, 0, %p530; + or.b32 %r870, %r1964, %r1961; + add.s32 %r1965, %r869, %r862; + mul.wide.u32 %rd487, %r1965, 2; + add.s64 %rd488, %rd113, %rd487; + ld.local.u16 %rs985, [%rd488]; + add.s32 %r1966, %r1965, 1; + mul.wide.u32 %rd489, %r1966, 2; + add.s64 %rd490, %rd113, %rd489; + ld.local.u16 %rs986, [%rd490]; + mov.b32 %r871, {%rs985, %rs986}; + shl.b32 %r1967, %r871, 1; + and.b32 %r1968, %r1967, -286331154; + or.b32 %r1969, %r1968, %r871; + and.b32 %r1970, %r871, -286331154; + shr.u32 %r1971, %r1970, 1; + or.b32 %r1972, %r1969, %r1971; + or.b32 %r1973, %r1972, %r870; + shl.b32 %r1974, %r1973, 4; + shr.u32 %r1975, %r1973, 4; + shr.u32 %r1976, %r2773, 12; + or.b32 %r1977, %r1973, %r1976; + or.b32 %r1978, %r1977, %r1974; + or.b32 %r1979, %r1978, %r1975; + not.b32 %r1980, %r871; + and.b32 %r872, %r1955, %r1980; + and.b32 %r873, %r872, %r1979; + setp.eq.s32 %p531, %r873, 0; + @%p531 bra $L__BB25_486; + + setp.gt.u32 %p532, %r2779, 31; + @%p532 bra $L__BB25_412; + +$L__BB25_409: + setp.ge.u32 %p533, %r2778, %r2885; + mov.u16 %rs1424, 0; + @%p533 bra $L__BB25_411; + + add.s32 %r876, %r2778, 1; + cvt.u64.u32 %rd491, %r2778; + add.s64 %rd492, %rd115, %rd491; + add.s64 %rd493, %rd1, %rd492; + ld.global.u8 %rs1424, [%rd493]; + mov.u32 %r2778, %r876; + +$L__BB25_411: + and.b16 %rs988, %rs1424, 255; + cvt.u64.u16 %rd494, %rs1424; + and.b64 %rd495, %rd494, 255; + shl.b64 %rd496, %rd495, %r2779; + or.b64 %rd654, %rd496, %rd654; + cvt.u32.u16 %r1981, %rs1426; + cvt.s32.s8 %r1982, %r1981; + mov.u32 %r1983, 8; + sub.s32 %r1984, %r1983, %r1982; + add.s32 %r2779, %r1984, %r2779; + setp.eq.s16 %p534, %rs988, 255; + selp.u16 %rs1426, 1, 0, %p534; + setp.lt.u32 %p535, %r2779, 33; + @%p535 bra $L__BB25_409; + +$L__BB25_412: + cvt.u32.u64 %r2803, %rd654; + and.b32 %r1986, %r873, 15; + setp.eq.s32 %p536, %r1986, 0; + mov.u32 %r2804, 0; + mov.u32 %r2802, %r873; + @%p536 bra $L__BB25_421; + + and.b32 %r1988, %r873, 1; + setp.eq.b32 %p537, %r1988, 1; + mov.pred %p538, 0; + xor.pred %p539, %p537, %p538; + not.pred %p540, %p539; + mov.u32 %r2804, 0; + mov.u32 %r2802, %r873; + @%p540 bra $L__BB25_415; + + and.b32 %r1990, %r873, -2; + and.b32 %r1991, %r2803, 1; + neg.s32 %r1992, %r1991; + mov.u32 %r2804, 1; + and.b32 %r1993, %r1992, %r872; + and.b32 %r1994, %r1993, 51; + or.b32 %r2802, %r1994, %r1990; + shr.u32 %r2803, %r2803, 1; + +$L__BB25_415: + and.b32 %r1995, %r2802, 2; + setp.eq.s32 %p541, %r1995, 0; + @%p541 bra $L__BB25_417; + + and.b32 %r1996, %r2802, -3; + and.b32 %r1997, %r2803, 1; + neg.s32 %r1998, %r1997; + and.b32 %r1999, %r1998, %r872; + and.b32 %r2000, %r1999, 118; + or.b32 %r2802, %r2000, %r1996; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_417: + and.b32 %r2001, %r2802, 4; + setp.eq.s32 %p542, %r2001, 0; + @%p542 bra $L__BB25_419; + + and.b32 %r2002, %r2802, -5; + and.b32 %r2003, %r2803, 1; + neg.s32 %r2004, %r2003; + and.b32 %r2005, %r2004, %r872; + and.b32 %r2006, %r2005, 236; + or.b32 %r2802, %r2006, %r2002; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_419: + and.b32 %r2007, %r2802, 8; + setp.eq.s32 %p543, %r2007, 0; + @%p543 bra $L__BB25_421; + + and.b32 %r2008, %r2802, -9; + and.b32 %r2009, %r2803, 1; + neg.s32 %r2010, %r2009; + and.b32 %r2011, %r2010, %r872; + and.b32 %r2012, %r2011, 200; + or.b32 %r2802, %r2012, %r2008; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_421: + and.b32 %r2013, %r2802, 240; + setp.eq.s32 %p544, %r2013, 0; + @%p544 bra $L__BB25_430; + + and.b32 %r2014, %r2802, 16; + setp.eq.s32 %p545, %r2014, 0; + @%p545 bra $L__BB25_424; + + and.b32 %r2015, %r2802, -17; + and.b32 %r2016, %r2803, 1; + neg.s32 %r2017, %r2016; + and.b32 %r2018, %r2017, %r872; + and.b32 %r2019, %r2018, 816; + or.b32 %r2802, %r2019, %r2015; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_424: + and.b32 %r2020, %r2802, 32; + setp.eq.s32 %p546, %r2020, 0; + @%p546 bra $L__BB25_426; + + and.b32 %r2021, %r2802, -33; + and.b32 %r2022, %r2803, 1; + neg.s32 %r2023, %r2022; + and.b32 %r2024, %r2023, %r872; + and.b32 %r2025, %r2024, 1888; + or.b32 %r2802, %r2025, %r2021; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_426: + and.b32 %r2026, %r2802, 64; + setp.eq.s32 %p547, %r2026, 0; + @%p547 bra $L__BB25_428; + + and.b32 %r2027, %r2802, -65; + and.b32 %r2028, %r2803, 1; + neg.s32 %r2029, %r2028; + and.b32 %r2030, %r2029, %r872; + and.b32 %r2031, %r2030, 3776; + or.b32 %r2802, %r2031, %r2027; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_428: + and.b32 %r2032, %r2802, 128; + setp.eq.s32 %p548, %r2032, 0; + @%p548 bra $L__BB25_430; + + and.b32 %r2033, %r2802, -129; + and.b32 %r2034, %r2803, 1; + neg.s32 %r2035, %r2034; + and.b32 %r2036, %r2035, %r872; + and.b32 %r2037, %r2036, 3200; + or.b32 %r2802, %r2037, %r2033; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_430: + and.b32 %r2038, %r2802, 3840; + setp.eq.s32 %p549, %r2038, 0; + @%p549 bra $L__BB25_439; + + and.b32 %r2039, %r2802, 256; + setp.eq.s32 %p550, %r2039, 0; + @%p550 bra $L__BB25_433; + + and.b32 %r2040, %r2802, -257; + and.b32 %r2041, %r2803, 1; + neg.s32 %r2042, %r2041; + and.b32 %r2043, %r2042, %r872; + and.b32 %r2044, %r2043, 13056; + or.b32 %r2802, %r2044, %r2040; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_433: + and.b32 %r2045, %r2802, 512; + setp.eq.s32 %p551, %r2045, 0; + @%p551 bra $L__BB25_435; + + and.b32 %r2046, %r2802, -513; + and.b32 %r2047, %r2803, 1; + neg.s32 %r2048, %r2047; + and.b32 %r2049, %r2048, %r872; + and.b32 %r2050, %r2049, 30208; + or.b32 %r2802, %r2050, %r2046; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_435: + and.b32 %r2051, %r2802, 1024; + setp.eq.s32 %p552, %r2051, 0; + @%p552 bra $L__BB25_437; + + and.b32 %r2052, %r2802, -1025; + and.b32 %r2053, %r2803, 1; + neg.s32 %r2054, %r2053; + and.b32 %r2055, %r2054, %r872; + and.b32 %r2056, %r2055, 60416; + or.b32 %r2802, %r2056, %r2052; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_437: + and.b32 %r2057, %r2802, 2048; + setp.eq.s32 %p553, %r2057, 0; + @%p553 bra $L__BB25_439; + + and.b32 %r2058, %r2802, -2049; + and.b32 %r2059, %r2803, 1; + neg.s32 %r2060, %r2059; + and.b32 %r2061, %r2060, %r872; + and.b32 %r2062, %r2061, 51200; + or.b32 %r2802, %r2062, %r2058; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_439: + and.b32 %r2063, %r2802, 61440; + setp.eq.s32 %p554, %r2063, 0; + @%p554 bra $L__BB25_448; + + and.b32 %r2064, %r2802, 4096; + setp.eq.s32 %p555, %r2064, 0; + @%p555 bra $L__BB25_442; + + and.b32 %r2065, %r2802, -4097; + and.b32 %r2066, %r2803, 1; + neg.s32 %r2067, %r2066; + and.b32 %r2068, %r2067, %r872; + and.b32 %r2069, %r2068, 208896; + or.b32 %r2802, %r2069, %r2065; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_442: + and.b32 %r2070, %r2802, 8192; + setp.eq.s32 %p556, %r2070, 0; + @%p556 bra $L__BB25_444; + + and.b32 %r2071, %r2802, -8193; + and.b32 %r2072, %r2803, 1; + neg.s32 %r2073, %r2072; + and.b32 %r2074, %r2073, %r872; + and.b32 %r2075, %r2074, 483328; + or.b32 %r2802, %r2075, %r2071; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_444: + and.b32 %r2076, %r2802, 16384; + setp.eq.s32 %p557, %r2076, 0; + @%p557 bra $L__BB25_446; + + and.b32 %r2077, %r2802, -16385; + and.b32 %r2078, %r2803, 1; + neg.s32 %r2079, %r2078; + and.b32 %r2080, %r2079, %r872; + and.b32 %r2081, %r2080, 966656; + or.b32 %r2802, %r2081, %r2077; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_446: + cvt.u16.u32 %rs989, %r2802; + setp.gt.s16 %p558, %rs989, -1; + @%p558 bra $L__BB25_448; + + and.b32 %r2082, %r2802, -32769; + and.b32 %r2083, %r2803, 1; + neg.s32 %r2084, %r2083; + and.b32 %r2085, %r2084, %r872; + and.b32 %r2086, %r2085, 819200; + or.b32 %r2802, %r2086, %r2082; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_448: + setp.eq.s32 %p559, %r2802, 0; + @%p559 bra $L__BB25_485; + + add.s32 %r977, %r864, %r2772; + and.b32 %r2087, %r2802, 15; + setp.eq.s32 %p560, %r2087, 0; + @%p560 bra $L__BB25_458; + + and.b32 %r2088, %r2802, 1; + setp.eq.b32 %p561, %r2088, 1; + mov.pred %p562, 0; + xor.pred %p563, %p561, %p562; + not.pred %p564, %p563; + @%p564 bra $L__BB25_452; + + shl.b32 %r2089, %r2803, 31; + or.b32 %r2090, %r2089, %r855; + mul.wide.u32 %rd497, %r977, 4; + add.s64 %rd498, %rd2, %rd497; + st.global.u32 [%rd498], %r2090; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_452: + and.b32 %r2091, %r2802, 2; + setp.eq.s32 %p565, %r2091, 0; + @%p565 bra $L__BB25_454; + + shl.b32 %r2092, %r2803, 31; + or.b32 %r2093, %r2092, %r855; + add.s32 %r2094, %r977, %r9; + mul.wide.u32 %rd499, %r2094, 4; + add.s64 %rd500, %rd2, %rd499; + st.global.u32 [%rd500], %r2093; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_454: + and.b32 %r2095, %r2802, 4; + setp.eq.s32 %p566, %r2095, 0; + @%p566 bra $L__BB25_456; + + shl.b32 %r2096, %r2803, 31; + or.b32 %r2097, %r2096, %r855; + add.s32 %r2098, %r977, %r856; + mul.wide.u32 %rd501, %r2098, 4; + add.s64 %rd502, %rd2, %rd501; + st.global.u32 [%rd502], %r2097; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_456: + and.b32 %r2099, %r2802, 8; + setp.eq.s32 %p567, %r2099, 0; + @%p567 bra $L__BB25_458; + + shl.b32 %r2100, %r2803, 31; + or.b32 %r2101, %r2100, %r855; + add.s32 %r2102, %r977, %r857; + mul.wide.u32 %rd503, %r2102, 4; + add.s64 %rd504, %rd2, %rd503; + st.global.u32 [%rd504], %r2101; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_458: + add.s32 %r994, %r977, 1; + and.b32 %r2103, %r2802, 240; + setp.eq.s32 %p568, %r2103, 0; + @%p568 bra $L__BB25_467; + + and.b32 %r2104, %r2802, 16; + setp.eq.s32 %p569, %r2104, 0; + @%p569 bra $L__BB25_461; + + shl.b32 %r2105, %r2803, 31; + or.b32 %r2106, %r2105, %r855; + mul.wide.u32 %rd505, %r994, 4; + add.s64 %rd506, %rd2, %rd505; + st.global.u32 [%rd506], %r2106; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_461: + and.b32 %r2107, %r2802, 32; + setp.eq.s32 %p570, %r2107, 0; + @%p570 bra $L__BB25_463; + + shl.b32 %r2108, %r2803, 31; + or.b32 %r2109, %r2108, %r855; + add.s32 %r2110, %r994, %r9; + mul.wide.u32 %rd507, %r2110, 4; + add.s64 %rd508, %rd2, %rd507; + st.global.u32 [%rd508], %r2109; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_463: + and.b32 %r2111, %r2802, 64; + setp.eq.s32 %p571, %r2111, 0; + @%p571 bra $L__BB25_465; + + shl.b32 %r2112, %r2803, 31; + or.b32 %r2113, %r2112, %r855; + add.s32 %r2114, %r994, %r856; + mul.wide.u32 %rd509, %r2114, 4; + add.s64 %rd510, %rd2, %rd509; + st.global.u32 [%rd510], %r2113; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_465: + and.b32 %r2115, %r2802, 128; + setp.eq.s32 %p572, %r2115, 0; + @%p572 bra $L__BB25_467; + + shl.b32 %r2116, %r2803, 31; + or.b32 %r2117, %r2116, %r855; + add.s32 %r2118, %r994, %r857; + mul.wide.u32 %rd511, %r2118, 4; + add.s64 %rd512, %rd2, %rd511; + st.global.u32 [%rd512], %r2117; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_467: + add.s32 %r1011, %r977, 2; + and.b32 %r2119, %r2802, 3840; + setp.eq.s32 %p573, %r2119, 0; + @%p573 bra $L__BB25_476; + + and.b32 %r2120, %r2802, 256; + setp.eq.s32 %p574, %r2120, 0; + @%p574 bra $L__BB25_470; + + shl.b32 %r2121, %r2803, 31; + or.b32 %r2122, %r2121, %r855; + mul.wide.u32 %rd513, %r1011, 4; + add.s64 %rd514, %rd2, %rd513; + st.global.u32 [%rd514], %r2122; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_470: + and.b32 %r2123, %r2802, 512; + setp.eq.s32 %p575, %r2123, 0; + @%p575 bra $L__BB25_472; + + shl.b32 %r2124, %r2803, 31; + or.b32 %r2125, %r2124, %r855; + add.s32 %r2126, %r1011, %r9; + mul.wide.u32 %rd515, %r2126, 4; + add.s64 %rd516, %rd2, %rd515; + st.global.u32 [%rd516], %r2125; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_472: + and.b32 %r2127, %r2802, 1024; + setp.eq.s32 %p576, %r2127, 0; + @%p576 bra $L__BB25_474; + + shl.b32 %r2128, %r2803, 31; + or.b32 %r2129, %r2128, %r855; + add.s32 %r2130, %r1011, %r856; + mul.wide.u32 %rd517, %r2130, 4; + add.s64 %rd518, %rd2, %rd517; + st.global.u32 [%rd518], %r2129; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_474: + and.b32 %r2131, %r2802, 2048; + setp.eq.s32 %p577, %r2131, 0; + @%p577 bra $L__BB25_476; + + shl.b32 %r2132, %r2803, 31; + or.b32 %r2133, %r2132, %r855; + add.s32 %r2134, %r1011, %r857; + mul.wide.u32 %rd519, %r2134, 4; + add.s64 %rd520, %rd2, %rd519; + st.global.u32 [%rd520], %r2133; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_476: + add.s32 %r1028, %r977, 3; + and.b32 %r2135, %r2802, 61440; + setp.eq.s32 %p578, %r2135, 0; + @%p578 bra $L__BB25_485; + + and.b32 %r2136, %r2802, 4096; + setp.eq.s32 %p579, %r2136, 0; + @%p579 bra $L__BB25_479; + + shl.b32 %r2137, %r2803, 31; + or.b32 %r2138, %r2137, %r855; + mul.wide.u32 %rd521, %r1028, 4; + add.s64 %rd522, %rd2, %rd521; + st.global.u32 [%rd522], %r2138; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_479: + and.b32 %r2139, %r2802, 8192; + setp.eq.s32 %p580, %r2139, 0; + @%p580 bra $L__BB25_481; + + shl.b32 %r2140, %r2803, 31; + or.b32 %r2141, %r2140, %r855; + add.s32 %r2142, %r1028, %r9; + mul.wide.u32 %rd523, %r2142, 4; + add.s64 %rd524, %rd2, %rd523; + st.global.u32 [%rd524], %r2141; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_481: + and.b32 %r2143, %r2802, 16384; + setp.eq.s32 %p581, %r2143, 0; + @%p581 bra $L__BB25_483; + + shl.b32 %r2144, %r2803, 31; + or.b32 %r2145, %r2144, %r855; + add.s32 %r2146, %r1028, %r856; + mul.wide.u32 %rd525, %r2146, 4; + add.s64 %rd526, %rd2, %rd525; + st.global.u32 [%rd526], %r2145; + shr.u32 %r2803, %r2803, 1; + add.s32 %r2804, %r2804, 1; + +$L__BB25_483: + cvt.u16.u32 %rs990, %r2802; + setp.gt.s16 %p582, %rs990, -1; + @%p582 bra $L__BB25_485; + + shl.b32 %r2147, %r2803, 31; + or.b32 %r2148, %r2147, %r855; + add.s32 %r2149, %r1028, %r857; + mul.wide.u32 %rd527, %r2149, 4; + add.s64 %rd528, %rd2, %rd527; + st.global.u32 [%rd528], %r2148; + add.s32 %r2804, %r2804, 1; + +$L__BB25_485: + shr.u64 %rd654, %rd654, %r2804; + sub.s32 %r2779, %r2779, %r2804; + +$L__BB25_486: + or.b32 %r1047, %r2802, %r871; + st.local.u16 [%rd118], %r1047; + add.s32 %r2150, %r869, 1; + setp.ge.u32 %p583, %r2150, %r819; + @%p583 bra $L__BB25_488; + + shr.u32 %r2151, %r1047, 16; + st.local.u16 [%rd118+2], %r2151; + +$L__BB25_488: + shl.b32 %r2152, %r1047, 1; + and.b32 %r2153, %r2152, 57344; + and.b32 %r2154, %r1047, 57344; + shr.u32 %r2155, %r2154, 1; + or.b32 %r2156, %r1047, %r870; + and.b32 %r2157, %r2156, 61440; + or.b32 %r2158, %r2157, %r2153; + or.b32 %r2773, %r2158, %r2155; + setp.lt.u32 %p584, %r1951, %r2; + mov.u32 %r2772, %r1951; + @%p584 bra $L__BB25_407; + + add.s32 %r2769, %r2769, 4; + setp.gt.u32 %p585, %r3, %r2769; + @%p585 bra $L__BB25_406; + + setp.lt.u32 %p586, %r13, 3; + @%p586 bra $L__BB25_541; + + mov.u32 %r2159, 0; + mov.u32 %r2863, %r2159; + +$L__BB25_492: + shr.u32 %r2161, %r2863, 1; + mul.lo.s32 %r2865, %r2161, %r1148; + shr.u32 %r2162, %r2863, 2; + mul.lo.s32 %r2864, %r2162, %r818; + mov.u32 %r2866, %r2159; + +$L__BB25_493: + mul.wide.u32 %rd529, %r2865, 2; + add.s64 %rd530, %rd14, %rd529; + ld.local.u16 %rs991, [%rd530]; + and.b16 %rs992, %rs991, 48; + shr.u16 %rs993, %rs992, 4; + and.b16 %rs994, %rs991, 192; + shr.u16 %rs995, %rs994, 2; + add.s32 %r2163, %r2865, 2; + mul.wide.u32 %rd531, %r2163, 2; + add.s64 %rd532, %rd14, %rd531; + ld.local.u16 %rs996, [%rd532]; + shl.b16 %rs997, %rs996, 4; + and.b16 %rs998, %rs997, 768; + shl.b16 %rs999, %rs996, 6; + and.b16 %rs1000, %rs999, 12288; + add.s32 %r2164, %r2865, %r1148; + mul.wide.u32 %rd533, %r2164, 2; + add.s64 %rd534, %rd14, %rd533; + ld.local.u16 %rs1001, [%rd534]; + and.b16 %rs1002, %rs1001, 48; + shr.u16 %rs1003, %rs1002, 2; + and.b16 %rs1004, %rs1001, 192; + add.s32 %r2165, %r2164, 2; + mul.wide.u32 %rd535, %r2165, 2; + add.s64 %rd536, %rd14, %rd535; + ld.local.u16 %rs1005, [%rd536]; + shl.b16 %rs1006, %rs1005, 6; + and.b16 %rs1007, %rs1006, 3072; + shl.b16 %rs1008, %rs1005, 8; + and.b16 %rs1009, %rs1008, -16384; + or.b16 %rs1010, %rs993, %rs995; + or.b16 %rs1011, %rs1010, %rs1000; + or.b16 %rs1012, %rs1011, %rs998; + or.b16 %rs1013, %rs1012, %rs1004; + or.b16 %rs1014, %rs1013, %rs1003; + or.b16 %rs1015, %rs1014, %rs1009; + or.b16 %rs1016, %rs1015, %rs1007; + mul.wide.u32 %rd537, %r2864, 2; + add.s64 %rd538, %rd113, %rd537; + st.local.u16 [%rd538], %rs1016; + add.s32 %r2865, %r2865, 4; + add.s32 %r2864, %r2864, 1; + add.s32 %r2866, %r2866, 4; + setp.lt.u32 %p587, %r2866, %r2; + @%p587 bra $L__BB25_493; + + mul.wide.u32 %rd539, %r2864, 2; + add.s64 %rd540, %rd113, %rd539; + mov.u16 %rs1017, 0; + st.local.u16 [%rd540], %rs1017; + add.s32 %r2863, %r2863, 4; + setp.lt.u32 %p588, %r2863, %r3; + @%p588 bra $L__BB25_492; + + mov.u32 %r2869, 0; + @%p520 bra $L__BB25_498; + + sub.s32 %r2868, %r832, %r2871; + mov.u32 %r2869, 0; + +$L__BB25_497: + add.s32 %r2168, %r2869, %r831; + mul.wide.u32 %rd541, %r2168, 2; + add.s64 %rd542, %rd113, %rd541; + mov.u16 %rs1018, 0; + st.local.v4.u16 [%rd542], {%rs1018, %rs1018, %rs1018, %rs1018}; + add.s32 %r2869, %r2869, 4; + add.s32 %r2868, %r2868, -4; + setp.ne.s32 %p590, %r2868, 0; + @%p590 bra $L__BB25_497; + +$L__BB25_498: + @%p522 bra $L__BB25_500; + +$L__BB25_499: + .pragma "nounroll"; + add.s32 %r2169, %r2869, %r831; + mul.wide.u32 %rd543, %r2169, 2; + add.s64 %rd544, %rd113, %rd543; + mov.u16 %rs1019, 0; + st.local.u16 [%rd544], %rs1019; + add.s32 %r2869, %r2869, 1; + add.s32 %r2871, %r2871, -1; + setp.ne.s32 %p592, %r2871, 0; + @%p592 bra $L__BB25_499; + +$L__BB25_500: + add.s32 %r2172, %r5, %r2885; + add.s32 %r2886, %r2172, -1; + mov.u32 %r2173, 1; + shl.b32 %r1072, %r2173, %r854; + mov.u16 %rs1431, 1; + mov.u32 %r2171, 0; + mov.u64 %rd659, 0; + mov.u32 %r2872, %r2171; + mov.u32 %r2884, %r2171; + +$L__BB25_501: + shr.u32 %r2175, %r2872, 2; + mul.lo.s32 %r2877, %r2175, %r818; + mad.lo.s32 %r1078, %r2872, %r9, %r10; + mov.u32 %r2876, %r2171; + +$L__BB25_502: + setp.gt.u32 %p593, %r2884, 31; + @%p593 bra $L__BB25_507; + + mov.u32 %r2882, %r2885; + +$L__BB25_504: + setp.eq.s32 %p594, %r2882, 0; + mov.u16 %rs1430, 0; + @%p594 bra $L__BB25_506; + + cvt.s64.s32 %rd546, %r2886; + add.s64 %rd547, %rd546, %rd4; + add.s64 %rd548, %rd1, %rd547; + ld.global.u8 %rs1430, [%rd548]; + +$L__BB25_506: + add.s32 %r2176, %r2882, -1; + selp.b32 %r2885, 0, %r2176, %p594; + setp.ne.s32 %p596, %r2882, 0; + selp.b32 %r2177, -1, 0, %p596; + add.s32 %r2886, %r2886, %r2177; + and.b16 %rs1022, %rs1430, 255; + and.b16 %rs1023, %rs1430, 127; + setp.eq.s16 %p597, %rs1023, 127; + and.b16 %rs1024, %rs1431, 255; + setp.ne.s16 %p598, %rs1024, 0; + and.pred %p599, %p598, %p597; + selp.b32 %r2178, 7, 8, %p599; + cvt.u64.u16 %rd549, %rs1430; + and.b64 %rd550, %rd549, 255; + shl.b64 %rd551, %rd550, %r2884; + or.b64 %rd659, %rd551, %rd659; + add.s32 %r2884, %r2178, %r2884; + setp.gt.u16 %p600, %rs1022, 143; + selp.u16 %rs1431, 1, 0, %p600; + setp.lt.u32 %p601, %r2884, 33; + mov.u32 %r2882, %r2885; + @%p601 bra $L__BB25_504; + +$L__BB25_507: + mul.wide.u32 %rd552, %r2877, 2; + add.s64 %rd553, %rd113, %rd552; + ld.local.u32 %r1093, [%rd553]; + setp.eq.s32 %p602, %r1093, 0; + @%p602 bra $L__BB25_528; + + cvt.u32.u64 %r2897, %rd659; + add.s32 %r1095, %r1078, %r2876; + mov.u32 %r2889, 15; + mov.u32 %r2887, 0; + +$L__BB25_509: + and.b32 %r2181, %r2889, %r1093; + setp.eq.s32 %p603, %r2181, 0; + @%p603 bra $L__BB25_518; + + add.s32 %r1099, %r1095, %r2887; + and.b32 %r1100, %r2889, 286331137; + and.b32 %r2182, %r1100, %r1093; + setp.eq.s32 %p604, %r2182, 0; + @%p604 bra $L__BB25_512; + + not.b32 %r2183, %r2897; + and.b32 %r2184, %r2183, 1; + shl.b32 %r2185, %r2184, %r667; + or.b32 %r2186, %r2185, %r1072; + mul.wide.u32 %rd554, %r1099, 4; + add.s64 %rd555, %rd2, %rd554; + ld.global.u32 %r2187, [%rd555]; + xor.b32 %r2188, %r2187, %r2186; + st.global.u32 [%rd555], %r2188; + shr.u32 %r2897, %r2897, 1; + +$L__BB25_512: + add.s32 %r1103, %r1099, %r9; + shl.b32 %r2189, %r1100, 1; + and.b32 %r2190, %r2189, %r1093; + setp.eq.s32 %p605, %r2190, 0; + @%p605 bra $L__BB25_514; + + not.b32 %r2191, %r2897; + and.b32 %r2192, %r2191, 1; + shl.b32 %r2193, %r2192, %r667; + or.b32 %r2194, %r2193, %r1072; + mul.wide.u32 %rd556, %r1103, 4; + add.s64 %rd557, %rd2, %rd556; + ld.global.u32 %r2195, [%rd557]; + xor.b32 %r2196, %r2195, %r2194; + st.global.u32 [%rd557], %r2196; + shr.u32 %r2897, %r2897, 1; + +$L__BB25_514: + add.s32 %r1106, %r1103, %r9; + shl.b32 %r2197, %r1100, 2; + and.b32 %r2198, %r2197, %r1093; + setp.eq.s32 %p606, %r2198, 0; + @%p606 bra $L__BB25_516; + + not.b32 %r2199, %r2897; + and.b32 %r2200, %r2199, 1; + shl.b32 %r2201, %r2200, %r667; + or.b32 %r2202, %r2201, %r1072; + mul.wide.u32 %rd558, %r1106, 4; + add.s64 %rd559, %rd2, %rd558; + ld.global.u32 %r2203, [%rd559]; + xor.b32 %r2204, %r2203, %r2202; + st.global.u32 [%rd559], %r2204; + shr.u32 %r2897, %r2897, 1; + +$L__BB25_516: + shl.b32 %r2205, %r1100, 3; + and.b32 %r2206, %r2205, %r1093; + setp.eq.s32 %p607, %r2206, 0; + @%p607 bra $L__BB25_518; + + not.b32 %r2207, %r2897; + and.b32 %r2208, %r2207, 1; + shl.b32 %r2209, %r2208, %r667; + or.b32 %r2210, %r2209, %r1072; + add.s32 %r2211, %r1106, %r9; + mul.wide.u32 %rd560, %r2211, 4; + add.s64 %rd561, %rd2, %rd560; + ld.global.u32 %r2212, [%rd561]; + xor.b32 %r2213, %r2212, %r2210; + st.global.u32 [%rd561], %r2213; + shr.u32 %r2897, %r2897, 1; + +$L__BB25_518: + shl.b32 %r1111, %r2889, 4; + and.b32 %r2214, %r1111, %r1093; + setp.eq.s32 %p608, %r2214, 0; + @%p608 bra $L__BB25_527; + + add.s32 %r2215, %r1095, %r2887; + add.s32 %r1112, %r2215, 1; + and.b32 %r1113, %r1111, 286330896; + and.b32 %r2216, %r1113, %r1093; + setp.eq.s32 %p609, %r2216, 0; + @%p609 bra $L__BB25_521; + + not.b32 %r2217, %r2897; + and.b32 %r2218, %r2217, 1; + shl.b32 %r2219, %r2218, %r667; + or.b32 %r2220, %r2219, %r1072; + mul.wide.u32 %rd562, %r1112, 4; + add.s64 %rd563, %rd2, %rd562; + ld.global.u32 %r2221, [%rd563]; + xor.b32 %r2222, %r2221, %r2220; + st.global.u32 [%rd563], %r2222; + shr.u32 %r2897, %r2897, 1; + +$L__BB25_521: + add.s32 %r1116, %r1112, %r9; + shl.b32 %r2223, %r1113, 1; + and.b32 %r2224, %r2223, %r1093; + setp.eq.s32 %p610, %r2224, 0; + @%p610 bra $L__BB25_523; + + not.b32 %r2225, %r2897; + and.b32 %r2226, %r2225, 1; + shl.b32 %r2227, %r2226, %r667; + or.b32 %r2228, %r2227, %r1072; + mul.wide.u32 %rd564, %r1116, 4; + add.s64 %rd565, %rd2, %rd564; + ld.global.u32 %r2229, [%rd565]; + xor.b32 %r2230, %r2229, %r2228; + st.global.u32 [%rd565], %r2230; + shr.u32 %r2897, %r2897, 1; + +$L__BB25_523: + add.s32 %r1119, %r1116, %r9; + shl.b32 %r2231, %r1113, 2; + and.b32 %r2232, %r2231, %r1093; + setp.eq.s32 %p611, %r2232, 0; + @%p611 bra $L__BB25_525; + + not.b32 %r2233, %r2897; + and.b32 %r2234, %r2233, 1; + shl.b32 %r2235, %r2234, %r667; + or.b32 %r2236, %r2235, %r1072; + mul.wide.u32 %rd566, %r1119, 4; + add.s64 %rd567, %rd2, %rd566; + ld.global.u32 %r2237, [%rd567]; + xor.b32 %r2238, %r2237, %r2236; + st.global.u32 [%rd567], %r2238; + shr.u32 %r2897, %r2897, 1; + +$L__BB25_525: + shl.b32 %r2239, %r1113, 3; + and.b32 %r2240, %r2239, %r1093; + setp.eq.s32 %p612, %r2240, 0; + @%p612 bra $L__BB25_527; + + not.b32 %r2241, %r2897; + and.b32 %r2242, %r2241, 1; + shl.b32 %r2243, %r2242, %r667; + or.b32 %r2244, %r2243, %r1072; + add.s32 %r2245, %r1119, %r9; + mul.wide.u32 %rd568, %r2245, 4; + add.s64 %rd569, %rd2, %rd568; + ld.global.u32 %r2246, [%rd569]; + xor.b32 %r2247, %r2246, %r2244; + st.global.u32 [%rd569], %r2247; + shr.u32 %r2897, %r2897, 1; + +$L__BB25_527: + shl.b32 %r2889, %r2889, 8; + add.s32 %r2887, %r2887, 2; + setp.ne.s32 %p613, %r2887, 8; + @%p613 bra $L__BB25_509; + +$L__BB25_528: + popc.b32 %r2248, %r1093; + shr.u64 %rd659, %rd659, %r2248; + sub.s32 %r2884, %r2884, %r2248; + add.s32 %r2876, %r2876, 8; + setp.lt.u32 %p614, %r2876, %r2; + add.s32 %r2877, %r2877, 2; + @%p614 bra $L__BB25_502; + + add.s32 %r2872, %r2872, 4; + setp.lt.u32 %p615, %r2872, %r3; + @%p615 bra $L__BB25_501; + bra.uni $L__BB25_541; + +} + // .globl j2k_htj2k_decode_codeblocks_multi +.visible .entry j2k_htj2k_decode_codeblocks_multi( + .param .u64 j2k_htj2k_decode_codeblocks_multi_param_0, + .param .u64 j2k_htj2k_decode_codeblocks_multi_param_1, + .param .u64 j2k_htj2k_decode_codeblocks_multi_param_2, + .param .u64 j2k_htj2k_decode_codeblocks_multi_param_3, + .param .u64 j2k_htj2k_decode_codeblocks_multi_param_4, + .param .u64 j2k_htj2k_decode_codeblocks_multi_param_5, + .param .u64 j2k_htj2k_decode_codeblocks_multi_param_6, + .param .u32 j2k_htj2k_decode_codeblocks_multi_param_7 +) +{ + .local .align 16 .b8 __local_depot26[7920]; + .reg .b64 %SP; + .reg .b64 %SPL; + .reg .pred %p<617>; + .reg .b16 %rs<1432>; + .reg .b32 %r<2913>; + .reg .b64 %rd<657>; + + + mov.u64 %SPL, __local_depot26; + ld.param.u64 %rd135, [ j2k_htj2k_decode_codeblocks_multi_param_0]; + ld.param.u64 %rd129, [ j2k_htj2k_decode_codeblocks_multi_param_1]; + ld.param.u64 %rd134, [ j2k_htj2k_decode_codeblocks_multi_param_6]; + ld.param.u32 %r1136, [ j2k_htj2k_decode_codeblocks_multi_param_7]; + cvta.to.global.u64 %rd1, %rd135; + mov.u32 %r1137, %ntid.x; + mov.u32 %r1138, %ctaid.x; + mov.u32 %r1139, %tid.x; + mad.lo.s32 %r1, %r1138, %r1137, %r1139; + setp.ge.u32 %p1, %r1, %r1136; + @%p1 bra $L__BB26_541; + + cvta.to.global.u64 %rd136, %rd134; + cvta.to.global.u64 %rd137, %rd129; + mul.wide.u32 %rd138, %r1, 64; + add.s64 %rd139, %rd137, %rd138; + ld.global.u64 %rd140, [%rd139]; + cvta.to.global.u64 %rd2, %rd140; + ld.global.v2.u32 {%r1140, %r1141}, [%rd139+8]; + mov.u32 %r1143, 0; + ld.global.v2.u32 {%r1144, %r1145}, [%rd139+16]; + ld.global.v2.u32 {%r1146, %r2900}, [%rd139+24]; + ld.global.v2.u32 {%r1148, %r1149}, [%rd139+32]; + ld.global.v2.u32 {%r1150, %r1151}, [%rd139+40]; + ld.global.u32 %r14, [%rd139+48]; + ld.global.u32 %r15, [%rd139+56]; + cvt.u64.u32 %rd3, %r1140; + mul.wide.u32 %rd141, %r1, 16; + add.s64 %rd4, %rd136, %rd141; + st.global.u32 [%rd4], %r1143; + st.global.u32 [%rd4+4], %r1143; + st.global.u32 [%rd4+8], %r1143; + st.global.u32 [%rd4+12], %r1143; + setp.gt.u32 %p2, %r1150, 1; + setp.eq.s32 %p3, %r2900, 0; + and.pred %p4, %p3, %p2; + selp.b32 %r16, 1, %r1150, %p4; + setp.eq.s32 %p5, %r1141, 0; + setp.eq.s32 %p6, %r1144, 0; + or.pred %p7, %p5, %p6; + @%p7 bra $L__BB26_541; + + setp.gt.u32 %p8, %r1141, 256; + setp.gt.u32 %p9, %r1144, 256; + or.pred %p10, %p8, %p9; + mul.lo.s32 %r1153, %r1144, %r1141; + setp.gt.u32 %p11, %r1153, 4096; + or.pred %p12, %p10, %p11; + @%p12 bra $L__BB26_540; + bra.uni $L__BB26_3; + +$L__BB26_540: + mov.u32 %r2318, 2; + st.global.u32 [%rd4], %r2318; + mov.u32 %r2319, 1; + st.global.u32 [%rd4+4], %r2319; + mov.u32 %r2320, 0; + st.global.u32 [%rd4+8], %r2320; + st.global.u32 [%rd4+12], %r2320; + bra.uni $L__BB26_541; + +$L__BB26_3: + add.s32 %r1154, %r1149, -1; + setp.gt.u32 %p13, %r1154, 30; + @%p13 bra $L__BB26_539; + bra.uni $L__BB26_4; + +$L__BB26_539: + mov.u32 %r2315, 1; + st.global.u32 [%rd4], %r2315; + mov.u32 %r2316, 2; + st.global.u32 [%rd4+4], %r2316; + mov.u32 %r2317, 0; + st.global.u32 [%rd4+8], %r2317; + st.global.u32 [%rd4+12], %r2317; + bra.uni $L__BB26_541; + +$L__BB26_4: + setp.gt.u32 %p14, %r16, 3; + setp.gt.u32 %p15, %r1148, 29; + or.pred %p16, %p15, %p14; + @%p16 bra $L__BB26_538; + bra.uni $L__BB26_5; + +$L__BB26_538: + mov.u32 %r2312, 1; + st.global.u32 [%rd4], %r2312; + mov.u32 %r2313, 3; + st.global.u32 [%rd4+4], %r2313; + mov.u32 %r2314, 0; + st.global.u32 [%rd4+8], %r2314; + st.global.u32 [%rd4+12], %r2314; + bra.uni $L__BB26_541; + +$L__BB26_5: + setp.eq.s32 %p17, %r1148, 29; + setp.gt.u32 %p18, %r16, 1; + and.pred %p19, %p17, %p18; + selp.b32 %r17, 1, %r16, %p19; + add.s32 %r1155, %r2900, %r1146; + setp.lt.u32 %p20, %r1145, %r1155; + setp.lt.u32 %p21, %r1146, 2; + or.pred %p22, %p21, %p20; + @%p22 bra $L__BB26_537; + bra.uni $L__BB26_6; + +$L__BB26_537: + mov.u32 %r2309, 1; + st.global.u32 [%rd4], %r2309; + mov.u32 %r2310, 4; + st.global.u32 [%rd4+4], %r2310; + mov.u32 %r2311, 0; + st.global.u32 [%rd4+8], %r2311; + st.global.u32 [%rd4+12], %r2311; + bra.uni $L__BB26_541; + +$L__BB26_6: + add.s32 %r1156, %r1146, -1; + cvt.u64.u32 %rd142, %r1156; + add.s64 %rd143, %rd142, %rd3; + add.s64 %rd144, %rd1, %rd143; + ld.global.u8 %rs667, [%rd144]; + mul.wide.u16 %r1157, %rs667, 16; + add.s32 %r1158, %r1146, -2; + cvt.u64.u32 %rd145, %r1158; + add.s64 %rd146, %rd145, %rd3; + add.s64 %rd147, %rd1, %rd146; + ld.global.u8 %rs1, [%rd147]; + and.b16 %rs668, %rs1, 15; + cvt.u32.u16 %r1159, %rs668; + or.b32 %r18, %r1157, %r1159; + setp.lt.u32 %p23, %r1146, %r18; + add.s32 %r19, %r18, -2; + setp.gt.u32 %p24, %r19, 4077; + or.pred %p25, %p23, %p24; + @%p25 bra $L__BB26_536; + bra.uni $L__BB26_7; + +$L__BB26_536: + mov.u32 %r2306, 1; + st.global.u32 [%rd4], %r2306; + mov.u32 %r2307, 5; + st.global.u32 [%rd4+4], %r2307; + mov.u32 %r2308, 0; + st.global.u32 [%rd4+8], %r2308; + st.global.u32 [%rd4+12], %r2308; + bra.uni $L__BB26_541; + +$L__BB26_7: + add.s32 %r1160, %r1144, 1; + shr.u32 %r1161, %r1160, 1; + add.s32 %r1162, %r1141, 9; + and.b32 %r1163, %r1162, -8; + setp.gt.u32 %p26, %r1163, 264; + add.s32 %r1164, %r1161, 1; + mul.lo.s32 %r1165, %r1164, %r1163; + setp.gt.u32 %p27, %r1165, 3096; + or.pred %p28, %p26, %p27; + @%p28 bra $L__BB26_535; + bra.uni $L__BB26_8; + +$L__BB26_535: + mov.u32 %r2303, 2; + st.global.u32 [%rd4], %r2303; + mov.u32 %r2304, 6; + st.global.u32 [%rd4+4], %r2304; + mov.u32 %r2305, 0; + st.global.u32 [%rd4+8], %r2305; + st.global.u32 [%rd4+12], %r2305; + bra.uni $L__BB26_541; + +$L__BB26_8: + and.b16 %rs1031, %rs1, 15; + cvt.u32.u16 %r2327, %rs1031; + mul.wide.u16 %r2326, %rs667, 16; + or.b32 %r2325, %r2326, %r2327; + mov.u64 %rd593, 0; + mov.u32 %r2495, 0; + sub.s32 %r20, %r1146, %r2325; + add.s32 %r2405, %r2325, -1; + mov.u16 %rs1104, 0; + mov.u64 %rd150, _ZZ20mel_decode_more_runsR10MelDecoderE7MEL_EXP; + mov.u32 %r2404, %r20; + mov.u16 %rs1105, %rs1104; + mov.u16 %rs1139, %rs1104; + mov.u32 %r2377, %r2495; + +$L__BB26_9: + setp.gt.u32 %p30, %r2377, 7; + @%p30 bra $L__BB26_54; + + mul.wide.u32 %rd149, %r2495, 4; + add.s64 %rd151, %rd150, %rd149; + ld.global.nc.u32 %r26, [%rd151]; + and.b16 %rs672, %rs1139, 255; + setp.ne.s16 %p31, %rs672, 0; + mov.u16 %rs1039, %rs1139; + @%p31 bra $L__BB26_14; + + setp.eq.s32 %p32, %r2405, 0; + mov.u16 %rs1036, 255; + @%p32 bra $L__BB26_13; + + cvt.u64.u32 %rd152, %r2404; + add.s64 %rd153, %rd152, %rd3; + add.s64 %rd154, %rd1, %rd153; + ld.global.u8 %rs1036, [%rd154]; + +$L__BB26_13: + setp.ne.s32 %p34, %r2405, 0; + selp.u32 %r1168, 1, 0, %p34; + add.s32 %r2404, %r2404, %r1168; + add.s32 %r1169, %r2405, -1; + selp.b32 %r2405, 0, %r1169, %p32; + setp.eq.s32 %p35, %r2405, 0; + or.b16 %rs674, %rs1036, 15; + selp.b16 %rs1105, %rs674, %rs1036, %p35; + and.b16 %rs675, %rs1105, 255; + mov.u16 %rs676, 8; + sub.s16 %rs1039, %rs676, %rs1104; + setp.eq.s16 %p36, %rs675, 255; + selp.u16 %rs1104, 1, 0, %p36; + +$L__BB26_14: + add.s16 %rs15, %rs1039, -1; + cvt.u32.u16 %r1170, %rs15; + and.b32 %r1171, %r1170, 255; + mov.u32 %r1172, 1; + shl.b32 %r1173, %r1172, %r1171; + cvt.u32.u16 %r1174, %rs1105; + and.b32 %r1175, %r1173, %r1174; + and.b32 %r31, %r1175, 255; + add.s16 %rs1139, %rs1039, -1; + setp.eq.s32 %p37, %r31, 0; + @%p37 bra $L__BB26_16; + + add.s32 %r1176, %r2495, 1; + min.u32 %r2372, %r1176, 12; + mov.u32 %r1177, -1; + shl.b32 %r1178, %r1177, %r26; + shl.b32 %r1179, %r1178, 1; + xor.b32 %r2373, %r1179, -2; + bra.uni $L__BB26_53; + +$L__BB26_16: + cvt.u64.u32 %rd591, %r2495; + add.s64 %rd155, %rd591, -3; + setp.gt.u64 %p38, %rd155, 9; + mov.u32 %r2369, 0; + @%p38 bra $L__BB26_52; + + add.s16 %rs1139, %rs1039, -1; + max.u32 %r34, %r26, 1; + add.s32 %r1183, %r34, -1; + setp.lt.u32 %p39, %r1183, 3; + mov.u32 %r2369, 0; + @%p39 bra $L__BB26_36; + + and.b32 %r2321, %r34, 3; + add.s16 %rs1139, %rs1039, -1; + sub.s32 %r2341, %r34, %r2321; + mov.u32 %r2369, 0; + +$L__BB26_19: + and.b16 %rs678, %rs1139, 255; + setp.ne.s16 %p40, %rs678, 0; + @%p40 bra $L__BB26_23; + + setp.eq.s32 %p41, %r2405, 0; + mov.u16 %rs1043, 255; + @%p41 bra $L__BB26_22; + + cvt.u64.u32 %rd156, %r2404; + add.s64 %rd157, %rd156, %rd3; + add.s64 %rd158, %rd1, %rd157; + ld.global.u8 %rs1043, [%rd158]; + +$L__BB26_22: + setp.ne.s32 %p43, %r2405, 0; + selp.u32 %r1185, 1, 0, %p43; + add.s32 %r2404, %r2404, %r1185; + add.s32 %r1186, %r2405, -1; + selp.b32 %r2405, 0, %r1186, %p41; + setp.eq.s32 %p44, %r2405, 0; + or.b16 %rs680, %rs1043, 15; + selp.b16 %rs1105, %rs680, %rs1043, %p44; + and.b16 %rs681, %rs1105, 255; + mov.u16 %rs682, 8; + sub.s16 %rs1139, %rs682, %rs1104; + setp.eq.s16 %p45, %rs681, 255; + selp.u16 %rs1104, 1, 0, %p45; + +$L__BB26_23: + add.s16 %rs1050, %rs1139, -1; + and.b16 %rs683, %rs1050, 255; + cvt.u32.u16 %r1187, %rs1050; + and.b32 %r1188, %r1187, 255; + cvt.u32.u16 %r1189, %rs1105; + and.b32 %r2347, %r1189, 255; + shr.u32 %r1190, %r2347, %r1188; + and.b32 %r1191, %r1190, 1; + bfi.b32 %r46, %r2369, %r1191, 1, 31; + setp.ne.s16 %p46, %rs683, 0; + @%p46 bra $L__BB26_27; + + setp.eq.s32 %p47, %r2405, 0; + mov.u16 %rs1047, 255; + @%p47 bra $L__BB26_26; + + cvt.u64.u32 %rd159, %r2404; + add.s64 %rd160, %rd159, %rd3; + add.s64 %rd161, %rd1, %rd160; + ld.global.u8 %rs1047, [%rd161]; + +$L__BB26_26: + setp.ne.s32 %p49, %r2405, 0; + selp.u32 %r1192, 1, 0, %p49; + add.s32 %r2404, %r2404, %r1192; + add.s32 %r1193, %r2405, -1; + selp.b32 %r2405, 0, %r1193, %p47; + setp.eq.s32 %p50, %r2405, 0; + or.b16 %rs685, %rs1047, 15; + selp.b16 %rs1105, %rs685, %rs1047, %p50; + and.b16 %rs686, %rs1105, 255; + mov.u16 %rs687, 8; + sub.s16 %rs1050, %rs687, %rs1104; + setp.eq.s16 %p51, %rs686, 255; + selp.u16 %rs1104, 1, 0, %p51; + cvt.u32.u16 %r1194, %rs1105; + and.b32 %r2347, %r1194, 255; + +$L__BB26_27: + add.s16 %rs1054, %rs1050, -1; + and.b16 %rs688, %rs1054, 255; + cvt.u32.u16 %r1195, %rs1054; + and.b32 %r1196, %r1195, 255; + shr.u32 %r1197, %r2347, %r1196; + and.b32 %r1198, %r1197, 1; + bfi.b32 %r53, %r46, %r1198, 1, 31; + setp.ne.s16 %p52, %rs688, 0; + @%p52 bra $L__BB26_31; + + setp.eq.s32 %p53, %r2405, 0; + mov.u16 %rs1051, 255; + @%p53 bra $L__BB26_30; + + cvt.u64.u32 %rd162, %r2404; + add.s64 %rd163, %rd162, %rd3; + add.s64 %rd164, %rd1, %rd163; + ld.global.u8 %rs1051, [%rd164]; + +$L__BB26_30: + setp.ne.s32 %p55, %r2405, 0; + selp.u32 %r1199, 1, 0, %p55; + add.s32 %r2404, %r2404, %r1199; + add.s32 %r1200, %r2405, -1; + selp.b32 %r2405, 0, %r1200, %p53; + setp.eq.s32 %p56, %r2405, 0; + or.b16 %rs690, %rs1051, 15; + selp.b16 %rs1105, %rs690, %rs1051, %p56; + and.b16 %rs691, %rs1105, 255; + mov.u16 %rs692, 8; + sub.s16 %rs1054, %rs692, %rs1104; + setp.eq.s16 %p57, %rs691, 255; + selp.u16 %rs1104, 1, 0, %p57; + cvt.u32.u16 %r1201, %rs1105; + and.b32 %r2347, %r1201, 255; + +$L__BB26_31: + add.s16 %rs1058, %rs1054, -1; + and.b16 %rs693, %rs1058, 255; + cvt.u32.u16 %r1202, %rs1058; + and.b32 %r1203, %r1202, 255; + shr.u32 %r1204, %r2347, %r1203; + and.b32 %r1205, %r1204, 1; + bfi.b32 %r60, %r53, %r1205, 1, 31; + setp.ne.s16 %p58, %rs693, 0; + @%p58 bra $L__BB26_35; + + setp.eq.s32 %p59, %r2405, 0; + mov.u16 %rs1055, 255; + @%p59 bra $L__BB26_34; + + cvt.u64.u32 %rd165, %r2404; + add.s64 %rd166, %rd165, %rd3; + add.s64 %rd167, %rd1, %rd166; + ld.global.u8 %rs1055, [%rd167]; + +$L__BB26_34: + setp.ne.s32 %p61, %r2405, 0; + selp.u32 %r1206, 1, 0, %p61; + add.s32 %r2404, %r2404, %r1206; + add.s32 %r1207, %r2405, -1; + selp.b32 %r2405, 0, %r1207, %p59; + setp.eq.s32 %p62, %r2405, 0; + or.b16 %rs695, %rs1055, 15; + selp.b16 %rs1105, %rs695, %rs1055, %p62; + and.b16 %rs696, %rs1105, 255; + mov.u16 %rs697, 8; + sub.s16 %rs1058, %rs697, %rs1104; + setp.eq.s16 %p63, %rs696, 255; + selp.u16 %rs1104, 1, 0, %p63; + cvt.u32.u16 %r1208, %rs1105; + and.b32 %r2347, %r1208, 255; + +$L__BB26_35: + add.s16 %rs1139, %rs1058, -1; + cvt.u32.u16 %r1209, %rs1139; + and.b32 %r1210, %r1209, 255; + shr.u32 %r1211, %r2347, %r1210; + and.b32 %r1212, %r1211, 1; + bfi.b32 %r2369, %r60, %r1212, 1, 31; + add.s32 %r2341, %r2341, -4; + setp.ne.s32 %p64, %r2341, 0; + @%p64 bra $L__BB26_19; + +$L__BB26_36: + and.b32 %r2322, %r34, 3; + setp.eq.s32 %p65, %r2322, 0; + @%p65 bra $L__BB26_52; + + and.b16 %rs698, %rs1139, 255; + setp.ne.s16 %p66, %rs698, 0; + @%p66 bra $L__BB26_41; + + setp.eq.s32 %p67, %r2405, 0; + mov.u16 %rs1065, 255; + @%p67 bra $L__BB26_40; + + cvt.u64.u32 %rd168, %r2404; + add.s64 %rd169, %rd168, %rd3; + add.s64 %rd170, %rd1, %rd169; + ld.global.u8 %rs1065, [%rd170]; + +$L__BB26_40: + setp.ne.s32 %p69, %r2405, 0; + selp.u32 %r1213, 1, 0, %p69; + add.s32 %r2404, %r2404, %r1213; + add.s32 %r1214, %r2405, -1; + selp.b32 %r2405, 0, %r1214, %p67; + setp.eq.s32 %p70, %r2405, 0; + or.b16 %rs700, %rs1065, 15; + selp.b16 %rs1105, %rs700, %rs1065, %p70; + and.b16 %rs701, %rs1105, 255; + mov.u16 %rs702, 8; + sub.s16 %rs1139, %rs702, %rs1104; + setp.eq.s16 %p71, %rs701, 255; + selp.u16 %rs1104, 1, 0, %p71; + +$L__BB26_41: + and.b32 %r2323, %r34, 3; + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1215, %rs1139; + and.b32 %r1216, %r1215, 255; + cvt.u32.u16 %r1217, %rs1105; + and.b32 %r2364, %r1217, 255; + shr.u32 %r1218, %r2364, %r1216; + and.b32 %r1219, %r1218, 1; + bfi.b32 %r2369, %r2369, %r1219, 1, 31; + setp.eq.s32 %p72, %r2323, 1; + @%p72 bra $L__BB26_52; + + and.b16 %rs703, %rs1139, 255; + setp.ne.s16 %p73, %rs703, 0; + @%p73 bra $L__BB26_46; + + setp.eq.s32 %p74, %r2405, 0; + mov.u16 %rs1069, 255; + @%p74 bra $L__BB26_45; + + cvt.u64.u32 %rd171, %r2404; + add.s64 %rd172, %rd171, %rd3; + add.s64 %rd173, %rd1, %rd172; + ld.global.u8 %rs1069, [%rd173]; + +$L__BB26_45: + setp.ne.s32 %p76, %r2405, 0; + selp.u32 %r1220, 1, 0, %p76; + add.s32 %r2404, %r2404, %r1220; + add.s32 %r1221, %r2405, -1; + selp.b32 %r2405, 0, %r1221, %p74; + setp.eq.s32 %p77, %r2405, 0; + or.b16 %rs705, %rs1069, 15; + selp.b16 %rs1105, %rs705, %rs1069, %p77; + and.b16 %rs706, %rs1105, 255; + mov.u16 %rs707, 8; + sub.s16 %rs1139, %rs707, %rs1104; + setp.eq.s16 %p78, %rs706, 255; + selp.u16 %rs1104, 1, 0, %p78; + cvt.u32.u16 %r1222, %rs1105; + and.b32 %r2364, %r1222, 255; + +$L__BB26_46: + and.b32 %r2324, %r34, 3; + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1223, %rs1139; + and.b32 %r1224, %r1223, 255; + shr.u32 %r1225, %r2364, %r1224; + and.b32 %r1226, %r1225, 1; + bfi.b32 %r2369, %r2369, %r1226, 1, 31; + setp.eq.s32 %p79, %r2324, 2; + @%p79 bra $L__BB26_52; + + and.b16 %rs708, %rs1139, 255; + setp.ne.s16 %p80, %rs708, 0; + @%p80 bra $L__BB26_51; + + setp.eq.s32 %p81, %r2405, 0; + mov.u16 %rs1073, 255; + @%p81 bra $L__BB26_50; + + cvt.u64.u32 %rd174, %r2404; + add.s64 %rd175, %rd174, %rd3; + add.s64 %rd176, %rd1, %rd175; + ld.global.u8 %rs1073, [%rd176]; + +$L__BB26_50: + setp.ne.s32 %p83, %r2405, 0; + selp.u32 %r1227, 1, 0, %p83; + add.s32 %r2404, %r2404, %r1227; + add.s32 %r1228, %r2405, -1; + selp.b32 %r2405, 0, %r1228, %p81; + setp.eq.s32 %p84, %r2405, 0; + or.b16 %rs710, %rs1073, 15; + selp.b16 %rs1105, %rs710, %rs1073, %p84; + and.b16 %rs711, %rs1105, 255; + mov.u16 %rs712, 8; + sub.s16 %rs1139, %rs712, %rs1104; + setp.eq.s16 %p85, %rs711, 255; + selp.u16 %rs1104, 1, 0, %p85; + cvt.u32.u16 %r1229, %rs1105; + and.b32 %r2364, %r1229, 255; + +$L__BB26_51: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1230, %rs1139; + and.b32 %r1231, %r1230, 255; + shr.u32 %r1232, %r2364, %r1231; + and.b32 %r1233, %r1232, 1; + bfi.b32 %r2369, %r2369, %r1233, 1, 31; + +$L__BB26_52: + shl.b32 %r1234, %r2369, 1; + or.b32 %r2373, %r1234, 1; + add.s32 %r1235, %r2495, -1; + setp.eq.s32 %p86, %r2495, 0; + selp.b32 %r2372, 0, %r1235, %p86; + +$L__BB26_53: + mul.lo.s32 %r1236, %r2377, 7; + cvt.u64.u32 %rd177, %r2373; + shl.b64 %rd178, %rd177, %r1236; + or.b64 %rd593, %rd178, %rd593; + setp.ne.s32 %p87, %r2495, 12; + setp.ne.s32 %p88, %r31, 0; + or.pred %p89, %p87, %p88; + add.s32 %r2377, %r2377, 1; + setp.lt.u32 %p90, %r2377, 8; + or.pred %p91, %p90, %p89; + mov.u32 %r2495, %r2372; + @%p91 bra $L__BB26_9; + +$L__BB26_54: + and.b16 %rs1032, %rs1, 15; + cvt.u32.u16 %r2331, %rs1032; + mul.wide.u16 %r2330, %rs667, 16; + or.b32 %r2329, %r2330, %r2331; + add.s32 %r2574, %r2329, -2; + setp.gt.u16 %p616, %rs1, 143; + selp.u16 %rs1271, 1, 0, %p616; + ld.param.u64 %rd590, [ j2k_htj2k_decode_codeblocks_multi_param_4]; + shr.u16 %rs1025, %rs1, 4; + ld.param.u64 %rd587, [ j2k_htj2k_decode_codeblocks_multi_param_2]; + add.s32 %r2496, %r2377, -1; + shr.u64 %rd603, %rd593, 7; + cvt.u32.u64 %r1239, %rd593; + and.b32 %r2492, %r1239, 127; + cvt.u64.u16 %rd612, %rs1025; + and.b64 %rd179, %rd612, 7; + setp.eq.s64 %p92, %rd179, 7; + selp.b32 %r2575, 3, 4, %p92; + mov.u32 %r2378, 0; + add.s32 %r2573, %r1146, -3; + cvta.to.global.u64 %rd11, %rd590; + cvta.to.global.u64 %rd12, %rd587; + add.u64 %rd13, %SPL, 0; + mov.u32 %r2379, %r2378; + +$L__BB26_55: + setp.gt.u32 %p93, %r2575, 31; + @%p93 bra $L__BB26_59; + +$L__BB26_56: + setp.eq.s32 %p94, %r2574, 0; + mov.u16 %rs1091, 0; + @%p94 bra $L__BB26_58; + + cvt.s64.s32 %rd181, %r2573; + add.s64 %rd182, %rd181, %rd3; + add.s64 %rd183, %rd1, %rd182; + ld.global.u8 %rs1091, [%rd183]; + +$L__BB26_58: + setp.ne.s32 %p96, %r2574, 0; + selp.b32 %r1240, -1, 0, %p96; + add.s32 %r2573, %r2573, %r1240; + add.s32 %r1241, %r2574, -1; + selp.b32 %r2574, 0, %r1241, %p94; + and.b16 %rs714, %rs1091, 255; + and.b16 %rs715, %rs1091, 127; + setp.eq.s16 %p97, %rs715, 127; + and.b16 %rs716, %rs1271, 255; + setp.ne.s16 %p98, %rs716, 0; + and.pred %p99, %p98, %p97; + selp.b32 %r1242, 7, 8, %p99; + cvt.u64.u16 %rd184, %rs1091; + and.b64 %rd185, %rd184, 255; + shl.b64 %rd186, %rd185, %r2575; + or.b64 %rd612, %rd186, %rd612; + add.s32 %r2575, %r1242, %r2575; + setp.gt.u16 %p100, %rs714, 143; + selp.u16 %rs1271, 1, 0, %p100; + setp.lt.u32 %p101, %r2575, 33; + @%p101 bra $L__BB26_56; + +$L__BB26_59: + cvt.u32.u64 %r1243, %rd612; + and.b32 %r1244, %r1243, 127; + add.s32 %r1245, %r1244, %r2378; + mul.wide.u32 %rd187, %r1245, 2; + add.s64 %rd188, %rd12, %rd187; + ld.global.u16 %r2445, [%rd188]; + setp.ne.s32 %p102, %r2378, 0; + @%p102 bra $L__BB26_109; + + add.s32 %r133, %r2492, -2; + setp.eq.s32 %p103, %r133, -1; + selp.b32 %r2445, %r2445, 0, %p103; + setp.gt.s32 %p104, %r2492, 1; + mov.u32 %r2492, %r133; + @%p104 bra $L__BB26_109; + + setp.ne.s32 %p105, %r2496, 0; + @%p105 bra $L__BB26_108; + + mov.u32 %r2496, 0; + +$L__BB26_63: + setp.gt.u32 %p106, %r2496, 7; + @%p106 bra $L__BB26_108; + + cvt.u64.u32 %rd20, %r2495; + mul.wide.u32 %rd189, %r2495, 4; + add.s64 %rd191, %rd150, %rd189; + ld.global.nc.u32 %r139, [%rd191]; + and.b16 %rs717, %rs1139, 255; + setp.ne.s16 %p107, %rs717, 0; + @%p107 bra $L__BB26_68; + + setp.eq.s32 %p108, %r2405, 0; + mov.u16 %rs1096, 255; + @%p108 bra $L__BB26_67; + + cvt.u64.u32 %rd192, %r2404; + add.s64 %rd193, %rd192, %rd3; + add.s64 %rd194, %rd1, %rd193; + ld.global.u8 %rs1096, [%rd194]; + +$L__BB26_67: + setp.ne.s32 %p110, %r2405, 0; + selp.u32 %r1247, 1, 0, %p110; + add.s32 %r2404, %r2404, %r1247; + add.s32 %r1248, %r2405, -1; + selp.b32 %r2405, 0, %r1248, %p108; + setp.eq.s32 %p111, %r2405, 0; + or.b16 %rs719, %rs1096, 15; + selp.b16 %rs1105, %rs719, %rs1096, %p111; + and.b16 %rs720, %rs1105, 255; + mov.u16 %rs721, 8; + sub.s16 %rs1139, %rs721, %rs1104; + setp.eq.s16 %p112, %rs720, 255; + selp.u16 %rs1104, 1, 0, %p112; + +$L__BB26_68: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1249, %rs1139; + and.b32 %r1250, %r1249, 255; + mov.u32 %r1251, 1; + shl.b32 %r1252, %r1251, %r1250; + cvt.u32.u16 %r1253, %rs1105; + and.b32 %r1254, %r1252, %r1253; + and.b32 %r144, %r1254, 255; + setp.eq.s32 %p113, %r144, 0; + @%p113 bra $L__BB26_70; + + add.s32 %r1255, %r2495, 1; + min.u32 %r2434, %r1255, 12; + mov.u32 %r1256, -1; + shl.b32 %r1257, %r1256, %r139; + shl.b32 %r1258, %r1257, 1; + xor.b32 %r2435, %r1258, -2; + bra.uni $L__BB26_107; + +$L__BB26_70: + add.s64 %rd195, %rd20, -3; + setp.gt.u64 %p114, %rd195, 9; + mov.u32 %r2431, 0; + @%p114 bra $L__BB26_106; + + max.u32 %r147, %r139, 1; + add.s32 %r1262, %r147, -1; + and.b32 %r148, %r147, 3; + setp.lt.u32 %p115, %r1262, 3; + mov.u32 %r2431, 0; + @%p115 bra $L__BB26_90; + + sub.s32 %r2403, %r147, %r148; + mov.u32 %r2431, 0; + +$L__BB26_73: + and.b16 %rs723, %rs1139, 255; + setp.ne.s16 %p116, %rs723, 0; + @%p116 bra $L__BB26_77; + + setp.eq.s32 %p117, %r2405, 0; + mov.u16 %rs1103, 255; + @%p117 bra $L__BB26_76; + + cvt.u64.u32 %rd196, %r2404; + add.s64 %rd197, %rd196, %rd3; + add.s64 %rd198, %rd1, %rd197; + ld.global.u8 %rs1103, [%rd198]; + +$L__BB26_76: + setp.ne.s32 %p119, %r2405, 0; + selp.u32 %r1264, 1, 0, %p119; + add.s32 %r2404, %r2404, %r1264; + add.s32 %r1265, %r2405, -1; + selp.b32 %r2405, 0, %r1265, %p117; + setp.eq.s32 %p120, %r2405, 0; + or.b16 %rs725, %rs1103, 15; + selp.b16 %rs1105, %rs725, %rs1103, %p120; + and.b16 %rs726, %rs1105, 255; + mov.u16 %rs727, 8; + sub.s16 %rs1139, %rs727, %rs1104; + setp.eq.s16 %p121, %rs726, 255; + selp.u16 %rs1104, 1, 0, %p121; + +$L__BB26_77: + add.s16 %rs1110, %rs1139, -1; + and.b16 %rs728, %rs1110, 255; + cvt.u32.u16 %r1266, %rs1110; + and.b32 %r1267, %r1266, 255; + cvt.u32.u16 %r1268, %rs1105; + and.b32 %r2409, %r1268, 255; + shr.u32 %r1269, %r2409, %r1267; + and.b32 %r1270, %r1269, 1; + bfi.b32 %r159, %r2431, %r1270, 1, 31; + setp.ne.s16 %p122, %rs728, 0; + @%p122 bra $L__BB26_81; + + setp.eq.s32 %p123, %r2405, 0; + mov.u16 %rs1107, 255; + @%p123 bra $L__BB26_80; + + cvt.u64.u32 %rd199, %r2404; + add.s64 %rd200, %rd199, %rd3; + add.s64 %rd201, %rd1, %rd200; + ld.global.u8 %rs1107, [%rd201]; + +$L__BB26_80: + setp.ne.s32 %p125, %r2405, 0; + selp.u32 %r1271, 1, 0, %p125; + add.s32 %r2404, %r2404, %r1271; + add.s32 %r1272, %r2405, -1; + selp.b32 %r2405, 0, %r1272, %p123; + setp.eq.s32 %p126, %r2405, 0; + or.b16 %rs730, %rs1107, 15; + selp.b16 %rs1105, %rs730, %rs1107, %p126; + and.b16 %rs731, %rs1105, 255; + mov.u16 %rs732, 8; + sub.s16 %rs1110, %rs732, %rs1104; + setp.eq.s16 %p127, %rs731, 255; + selp.u16 %rs1104, 1, 0, %p127; + cvt.u32.u16 %r1273, %rs1105; + and.b32 %r2409, %r1273, 255; + +$L__BB26_81: + add.s16 %rs1114, %rs1110, -1; + and.b16 %rs733, %rs1114, 255; + cvt.u32.u16 %r1274, %rs1114; + and.b32 %r1275, %r1274, 255; + shr.u32 %r1276, %r2409, %r1275; + and.b32 %r1277, %r1276, 1; + bfi.b32 %r166, %r159, %r1277, 1, 31; + setp.ne.s16 %p128, %rs733, 0; + @%p128 bra $L__BB26_85; + + setp.eq.s32 %p129, %r2405, 0; + mov.u16 %rs1111, 255; + @%p129 bra $L__BB26_84; + + cvt.u64.u32 %rd202, %r2404; + add.s64 %rd203, %rd202, %rd3; + add.s64 %rd204, %rd1, %rd203; + ld.global.u8 %rs1111, [%rd204]; + +$L__BB26_84: + setp.ne.s32 %p131, %r2405, 0; + selp.u32 %r1278, 1, 0, %p131; + add.s32 %r2404, %r2404, %r1278; + add.s32 %r1279, %r2405, -1; + selp.b32 %r2405, 0, %r1279, %p129; + setp.eq.s32 %p132, %r2405, 0; + or.b16 %rs735, %rs1111, 15; + selp.b16 %rs1105, %rs735, %rs1111, %p132; + and.b16 %rs736, %rs1105, 255; + mov.u16 %rs737, 8; + sub.s16 %rs1114, %rs737, %rs1104; + setp.eq.s16 %p133, %rs736, 255; + selp.u16 %rs1104, 1, 0, %p133; + cvt.u32.u16 %r1280, %rs1105; + and.b32 %r2409, %r1280, 255; + +$L__BB26_85: + add.s16 %rs1118, %rs1114, -1; + and.b16 %rs738, %rs1118, 255; + cvt.u32.u16 %r1281, %rs1118; + and.b32 %r1282, %r1281, 255; + shr.u32 %r1283, %r2409, %r1282; + and.b32 %r1284, %r1283, 1; + bfi.b32 %r173, %r166, %r1284, 1, 31; + setp.ne.s16 %p134, %rs738, 0; + @%p134 bra $L__BB26_89; + + setp.eq.s32 %p135, %r2405, 0; + mov.u16 %rs1115, 255; + @%p135 bra $L__BB26_88; + + cvt.u64.u32 %rd205, %r2404; + add.s64 %rd206, %rd205, %rd3; + add.s64 %rd207, %rd1, %rd206; + ld.global.u8 %rs1115, [%rd207]; + +$L__BB26_88: + setp.ne.s32 %p137, %r2405, 0; + selp.u32 %r1285, 1, 0, %p137; + add.s32 %r2404, %r2404, %r1285; + add.s32 %r1286, %r2405, -1; + selp.b32 %r2405, 0, %r1286, %p135; + setp.eq.s32 %p138, %r2405, 0; + or.b16 %rs740, %rs1115, 15; + selp.b16 %rs1105, %rs740, %rs1115, %p138; + and.b16 %rs741, %rs1105, 255; + mov.u16 %rs742, 8; + sub.s16 %rs1118, %rs742, %rs1104; + setp.eq.s16 %p139, %rs741, 255; + selp.u16 %rs1104, 1, 0, %p139; + cvt.u32.u16 %r1287, %rs1105; + and.b32 %r2409, %r1287, 255; + +$L__BB26_89: + add.s16 %rs1139, %rs1118, -1; + cvt.u32.u16 %r1288, %rs1139; + and.b32 %r1289, %r1288, 255; + shr.u32 %r1290, %r2409, %r1289; + and.b32 %r1291, %r1290, 1; + bfi.b32 %r2431, %r173, %r1291, 1, 31; + add.s32 %r2403, %r2403, -4; + setp.ne.s32 %p140, %r2403, 0; + @%p140 bra $L__BB26_73; + +$L__BB26_90: + setp.eq.s32 %p141, %r148, 0; + @%p141 bra $L__BB26_106; + + and.b16 %rs743, %rs1139, 255; + setp.ne.s16 %p142, %rs743, 0; + @%p142 bra $L__BB26_95; + + setp.eq.s32 %p143, %r2405, 0; + mov.u16 %rs1125, 255; + @%p143 bra $L__BB26_94; + + cvt.u64.u32 %rd208, %r2404; + add.s64 %rd209, %rd208, %rd3; + add.s64 %rd210, %rd1, %rd209; + ld.global.u8 %rs1125, [%rd210]; + +$L__BB26_94: + setp.ne.s32 %p145, %r2405, 0; + selp.u32 %r1292, 1, 0, %p145; + add.s32 %r2404, %r2404, %r1292; + add.s32 %r1293, %r2405, -1; + selp.b32 %r2405, 0, %r1293, %p143; + setp.eq.s32 %p146, %r2405, 0; + or.b16 %rs745, %rs1125, 15; + selp.b16 %rs1105, %rs745, %rs1125, %p146; + and.b16 %rs746, %rs1105, 255; + mov.u16 %rs747, 8; + sub.s16 %rs1139, %rs747, %rs1104; + setp.eq.s16 %p147, %rs746, 255; + selp.u16 %rs1104, 1, 0, %p147; + +$L__BB26_95: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1294, %rs1139; + and.b32 %r1295, %r1294, 255; + cvt.u32.u16 %r1296, %rs1105; + and.b32 %r2426, %r1296, 255; + shr.u32 %r1297, %r2426, %r1295; + and.b32 %r1298, %r1297, 1; + bfi.b32 %r2431, %r2431, %r1298, 1, 31; + setp.eq.s32 %p148, %r148, 1; + @%p148 bra $L__BB26_106; + + and.b16 %rs748, %rs1139, 255; + setp.ne.s16 %p149, %rs748, 0; + @%p149 bra $L__BB26_100; + + setp.eq.s32 %p150, %r2405, 0; + mov.u16 %rs1129, 255; + @%p150 bra $L__BB26_99; + + cvt.u64.u32 %rd211, %r2404; + add.s64 %rd212, %rd211, %rd3; + add.s64 %rd213, %rd1, %rd212; + ld.global.u8 %rs1129, [%rd213]; + +$L__BB26_99: + setp.ne.s32 %p152, %r2405, 0; + selp.u32 %r1299, 1, 0, %p152; + add.s32 %r2404, %r2404, %r1299; + add.s32 %r1300, %r2405, -1; + selp.b32 %r2405, 0, %r1300, %p150; + setp.eq.s32 %p153, %r2405, 0; + or.b16 %rs750, %rs1129, 15; + selp.b16 %rs1105, %rs750, %rs1129, %p153; + and.b16 %rs751, %rs1105, 255; + mov.u16 %rs752, 8; + sub.s16 %rs1139, %rs752, %rs1104; + setp.eq.s16 %p154, %rs751, 255; + selp.u16 %rs1104, 1, 0, %p154; + cvt.u32.u16 %r1301, %rs1105; + and.b32 %r2426, %r1301, 255; + +$L__BB26_100: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1302, %rs1139; + and.b32 %r1303, %r1302, 255; + shr.u32 %r1304, %r2426, %r1303; + and.b32 %r1305, %r1304, 1; + bfi.b32 %r2431, %r2431, %r1305, 1, 31; + setp.eq.s32 %p155, %r148, 2; + @%p155 bra $L__BB26_106; + + and.b16 %rs753, %rs1139, 255; + setp.ne.s16 %p156, %rs753, 0; + @%p156 bra $L__BB26_105; + + setp.eq.s32 %p157, %r2405, 0; + mov.u16 %rs1133, 255; + @%p157 bra $L__BB26_104; + + cvt.u64.u32 %rd214, %r2404; + add.s64 %rd215, %rd214, %rd3; + add.s64 %rd216, %rd1, %rd215; + ld.global.u8 %rs1133, [%rd216]; + +$L__BB26_104: + setp.ne.s32 %p159, %r2405, 0; + selp.u32 %r1306, 1, 0, %p159; + add.s32 %r2404, %r2404, %r1306; + add.s32 %r1307, %r2405, -1; + selp.b32 %r2405, 0, %r1307, %p157; + setp.eq.s32 %p160, %r2405, 0; + or.b16 %rs755, %rs1133, 15; + selp.b16 %rs1105, %rs755, %rs1133, %p160; + and.b16 %rs756, %rs1105, 255; + mov.u16 %rs757, 8; + sub.s16 %rs1139, %rs757, %rs1104; + setp.eq.s16 %p161, %rs756, 255; + selp.u16 %rs1104, 1, 0, %p161; + cvt.u32.u16 %r1308, %rs1105; + and.b32 %r2426, %r1308, 255; + +$L__BB26_105: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1309, %rs1139; + and.b32 %r1310, %r1309, 255; + shr.u32 %r1311, %r2426, %r1310; + and.b32 %r1312, %r1311, 1; + bfi.b32 %r2431, %r2431, %r1312, 1, 31; + +$L__BB26_106: + shl.b32 %r1313, %r2431, 1; + or.b32 %r2435, %r1313, 1; + add.s32 %r1314, %r2495, -1; + setp.eq.s32 %p162, %r2495, 0; + selp.b32 %r2434, 0, %r1314, %p162; + +$L__BB26_107: + mul.lo.s32 %r1315, %r2496, 7; + cvt.u64.u32 %rd217, %r2435; + shl.b64 %rd218, %rd217, %r1315; + or.b64 %rd603, %rd218, %rd603; + setp.ne.s32 %p163, %r2495, 12; + setp.ne.s32 %p164, %r144, 0; + or.pred %p165, %p163, %p164; + add.s32 %r2496, %r2496, 1; + setp.lt.u32 %p166, %r2496, 8; + or.pred %p167, %p166, %p165; + mov.u32 %r2495, %r2434; + @%p167 bra $L__BB26_63; + +$L__BB26_108: + cvt.u32.u64 %r1316, %rd603; + and.b32 %r2492, %r1316, 127; + shr.u64 %rd603, %rd603, 7; + add.s32 %r2496, %r2496, -1; + +$L__BB26_109: + mul.wide.u32 %rd219, %r2379, 2; + add.s64 %rd25, %rd13, %rd219; + st.local.u16 [%rd25], %r2445; + shl.b32 %r1317, %r2445, 3; + and.b32 %r1318, %r1317, 128; + shl.b32 %r1319, %r2445, 2; + and.b32 %r1320, %r1319, 896; + or.b32 %r1321, %r1318, %r1320; + and.b32 %r1322, %r2445, 7; + shr.u64 %rd26, %rd612, %r1322; + sub.s32 %r230, %r2575, %r1322; + cvt.u32.u64 %r1323, %rd26; + and.b32 %r1324, %r1323, 127; + or.b32 %r1325, %r1324, %r1321; + mul.wide.u32 %rd220, %r1325, 2; + add.s64 %rd221, %rd12, %rd220; + ld.global.u16 %r2497, [%rd221]; + setp.ne.s32 %p168, %r1321, 0; + add.s32 %r232, %r2379, 2; + setp.ge.u32 %p169, %r232, %r1141; + or.pred %p170, %p169, %p168; + @%p170 bra $L__BB26_159; + + add.s32 %r233, %r2492, -2; + setp.eq.s32 %p171, %r233, -1; + selp.b32 %r2497, %r2497, 0, %p171; + setp.gt.s32 %p172, %r2492, 1; + mov.u32 %r2492, %r233; + @%p172 bra $L__BB26_159; + + setp.ne.s32 %p173, %r2496, 0; + @%p173 bra $L__BB26_158; + + mov.u32 %r2496, 0; + +$L__BB26_113: + setp.gt.u32 %p174, %r2496, 7; + @%p174 bra $L__BB26_158; + + cvt.u64.u32 %rd28, %r2495; + mul.wide.u32 %rd222, %r2495, 4; + add.s64 %rd224, %rd150, %rd222; + ld.global.nc.u32 %r239, [%rd224]; + and.b16 %rs758, %rs1139, 255; + setp.ne.s16 %p175, %rs758, 0; + @%p175 bra $L__BB26_118; + + setp.eq.s32 %p176, %r2405, 0; + mov.u16 %rs1152, 255; + @%p176 bra $L__BB26_117; + + cvt.u64.u32 %rd225, %r2404; + add.s64 %rd226, %rd225, %rd3; + add.s64 %rd227, %rd1, %rd226; + ld.global.u8 %rs1152, [%rd227]; + +$L__BB26_117: + setp.ne.s32 %p178, %r2405, 0; + selp.u32 %r1327, 1, 0, %p178; + add.s32 %r2404, %r2404, %r1327; + add.s32 %r1328, %r2405, -1; + selp.b32 %r2405, 0, %r1328, %p176; + setp.eq.s32 %p179, %r2405, 0; + or.b16 %rs760, %rs1152, 15; + selp.b16 %rs1105, %rs760, %rs1152, %p179; + and.b16 %rs761, %rs1105, 255; + mov.u16 %rs762, 8; + sub.s16 %rs1139, %rs762, %rs1104; + setp.eq.s16 %p180, %rs761, 255; + selp.u16 %rs1104, 1, 0, %p180; + +$L__BB26_118: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1329, %rs1139; + and.b32 %r1330, %r1329, 255; + mov.u32 %r1331, 1; + shl.b32 %r1332, %r1331, %r1330; + cvt.u32.u16 %r1333, %rs1105; + and.b32 %r1334, %r1332, %r1333; + and.b32 %r244, %r1334, 255; + setp.eq.s32 %p181, %r244, 0; + @%p181 bra $L__BB26_120; + + add.s32 %r1335, %r2495, 1; + min.u32 %r2486, %r1335, 12; + mov.u32 %r1336, -1; + shl.b32 %r1337, %r1336, %r239; + shl.b32 %r1338, %r1337, 1; + xor.b32 %r2487, %r1338, -2; + bra.uni $L__BB26_157; + +$L__BB26_120: + add.s64 %rd228, %rd28, -3; + setp.gt.u64 %p182, %rd228, 9; + mov.u32 %r2483, 0; + @%p182 bra $L__BB26_156; + + max.u32 %r247, %r239, 1; + add.s32 %r1342, %r247, -1; + and.b32 %r248, %r247, 3; + setp.lt.u32 %p183, %r1342, 3; + mov.u32 %r2483, 0; + @%p183 bra $L__BB26_140; + + sub.s32 %r2455, %r247, %r248; + mov.u32 %r2483, 0; + +$L__BB26_123: + and.b16 %rs764, %rs1139, 255; + setp.ne.s16 %p184, %rs764, 0; + @%p184 bra $L__BB26_127; + + setp.eq.s32 %p185, %r2405, 0; + mov.u16 %rs1159, 255; + @%p185 bra $L__BB26_126; + + cvt.u64.u32 %rd229, %r2404; + add.s64 %rd230, %rd229, %rd3; + add.s64 %rd231, %rd1, %rd230; + ld.global.u8 %rs1159, [%rd231]; + +$L__BB26_126: + setp.ne.s32 %p187, %r2405, 0; + selp.u32 %r1344, 1, 0, %p187; + add.s32 %r2404, %r2404, %r1344; + add.s32 %r1345, %r2405, -1; + selp.b32 %r2405, 0, %r1345, %p185; + setp.eq.s32 %p188, %r2405, 0; + or.b16 %rs766, %rs1159, 15; + selp.b16 %rs1105, %rs766, %rs1159, %p188; + and.b16 %rs767, %rs1105, 255; + mov.u16 %rs768, 8; + sub.s16 %rs1139, %rs768, %rs1104; + setp.eq.s16 %p189, %rs767, 255; + selp.u16 %rs1104, 1, 0, %p189; + +$L__BB26_127: + add.s16 %rs1166, %rs1139, -1; + and.b16 %rs769, %rs1166, 255; + cvt.u32.u16 %r1346, %rs1166; + and.b32 %r1347, %r1346, 255; + cvt.u32.u16 %r1348, %rs1105; + and.b32 %r2461, %r1348, 255; + shr.u32 %r1349, %r2461, %r1347; + and.b32 %r1350, %r1349, 1; + bfi.b32 %r259, %r2483, %r1350, 1, 31; + setp.ne.s16 %p190, %rs769, 0; + @%p190 bra $L__BB26_131; + + setp.eq.s32 %p191, %r2405, 0; + mov.u16 %rs1163, 255; + @%p191 bra $L__BB26_130; + + cvt.u64.u32 %rd232, %r2404; + add.s64 %rd233, %rd232, %rd3; + add.s64 %rd234, %rd1, %rd233; + ld.global.u8 %rs1163, [%rd234]; + +$L__BB26_130: + setp.ne.s32 %p193, %r2405, 0; + selp.u32 %r1351, 1, 0, %p193; + add.s32 %r2404, %r2404, %r1351; + add.s32 %r1352, %r2405, -1; + selp.b32 %r2405, 0, %r1352, %p191; + setp.eq.s32 %p194, %r2405, 0; + or.b16 %rs771, %rs1163, 15; + selp.b16 %rs1105, %rs771, %rs1163, %p194; + and.b16 %rs772, %rs1105, 255; + mov.u16 %rs773, 8; + sub.s16 %rs1166, %rs773, %rs1104; + setp.eq.s16 %p195, %rs772, 255; + selp.u16 %rs1104, 1, 0, %p195; + cvt.u32.u16 %r1353, %rs1105; + and.b32 %r2461, %r1353, 255; + +$L__BB26_131: + add.s16 %rs1170, %rs1166, -1; + and.b16 %rs774, %rs1170, 255; + cvt.u32.u16 %r1354, %rs1170; + and.b32 %r1355, %r1354, 255; + shr.u32 %r1356, %r2461, %r1355; + and.b32 %r1357, %r1356, 1; + bfi.b32 %r266, %r259, %r1357, 1, 31; + setp.ne.s16 %p196, %rs774, 0; + @%p196 bra $L__BB26_135; + + setp.eq.s32 %p197, %r2405, 0; + mov.u16 %rs1167, 255; + @%p197 bra $L__BB26_134; + + cvt.u64.u32 %rd235, %r2404; + add.s64 %rd236, %rd235, %rd3; + add.s64 %rd237, %rd1, %rd236; + ld.global.u8 %rs1167, [%rd237]; + +$L__BB26_134: + setp.ne.s32 %p199, %r2405, 0; + selp.u32 %r1358, 1, 0, %p199; + add.s32 %r2404, %r2404, %r1358; + add.s32 %r1359, %r2405, -1; + selp.b32 %r2405, 0, %r1359, %p197; + setp.eq.s32 %p200, %r2405, 0; + or.b16 %rs776, %rs1167, 15; + selp.b16 %rs1105, %rs776, %rs1167, %p200; + and.b16 %rs777, %rs1105, 255; + mov.u16 %rs778, 8; + sub.s16 %rs1170, %rs778, %rs1104; + setp.eq.s16 %p201, %rs777, 255; + selp.u16 %rs1104, 1, 0, %p201; + cvt.u32.u16 %r1360, %rs1105; + and.b32 %r2461, %r1360, 255; + +$L__BB26_135: + add.s16 %rs1174, %rs1170, -1; + and.b16 %rs779, %rs1174, 255; + cvt.u32.u16 %r1361, %rs1174; + and.b32 %r1362, %r1361, 255; + shr.u32 %r1363, %r2461, %r1362; + and.b32 %r1364, %r1363, 1; + bfi.b32 %r273, %r266, %r1364, 1, 31; + setp.ne.s16 %p202, %rs779, 0; + @%p202 bra $L__BB26_139; + + setp.eq.s32 %p203, %r2405, 0; + mov.u16 %rs1171, 255; + @%p203 bra $L__BB26_138; + + cvt.u64.u32 %rd238, %r2404; + add.s64 %rd239, %rd238, %rd3; + add.s64 %rd240, %rd1, %rd239; + ld.global.u8 %rs1171, [%rd240]; + +$L__BB26_138: + setp.ne.s32 %p205, %r2405, 0; + selp.u32 %r1365, 1, 0, %p205; + add.s32 %r2404, %r2404, %r1365; + add.s32 %r1366, %r2405, -1; + selp.b32 %r2405, 0, %r1366, %p203; + setp.eq.s32 %p206, %r2405, 0; + or.b16 %rs781, %rs1171, 15; + selp.b16 %rs1105, %rs781, %rs1171, %p206; + and.b16 %rs782, %rs1105, 255; + mov.u16 %rs783, 8; + sub.s16 %rs1174, %rs783, %rs1104; + setp.eq.s16 %p207, %rs782, 255; + selp.u16 %rs1104, 1, 0, %p207; + cvt.u32.u16 %r1367, %rs1105; + and.b32 %r2461, %r1367, 255; + +$L__BB26_139: + add.s16 %rs1139, %rs1174, -1; + cvt.u32.u16 %r1368, %rs1139; + and.b32 %r1369, %r1368, 255; + shr.u32 %r1370, %r2461, %r1369; + and.b32 %r1371, %r1370, 1; + bfi.b32 %r2483, %r273, %r1371, 1, 31; + add.s32 %r2455, %r2455, -4; + setp.ne.s32 %p208, %r2455, 0; + @%p208 bra $L__BB26_123; + +$L__BB26_140: + setp.eq.s32 %p209, %r248, 0; + @%p209 bra $L__BB26_156; + + and.b16 %rs784, %rs1139, 255; + setp.ne.s16 %p210, %rs784, 0; + @%p210 bra $L__BB26_145; + + setp.eq.s32 %p211, %r2405, 0; + mov.u16 %rs1181, 255; + @%p211 bra $L__BB26_144; + + cvt.u64.u32 %rd241, %r2404; + add.s64 %rd242, %rd241, %rd3; + add.s64 %rd243, %rd1, %rd242; + ld.global.u8 %rs1181, [%rd243]; + +$L__BB26_144: + setp.ne.s32 %p213, %r2405, 0; + selp.u32 %r1372, 1, 0, %p213; + add.s32 %r2404, %r2404, %r1372; + add.s32 %r1373, %r2405, -1; + selp.b32 %r2405, 0, %r1373, %p211; + setp.eq.s32 %p214, %r2405, 0; + or.b16 %rs786, %rs1181, 15; + selp.b16 %rs1105, %rs786, %rs1181, %p214; + and.b16 %rs787, %rs1105, 255; + mov.u16 %rs788, 8; + sub.s16 %rs1139, %rs788, %rs1104; + setp.eq.s16 %p215, %rs787, 255; + selp.u16 %rs1104, 1, 0, %p215; + +$L__BB26_145: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1374, %rs1139; + and.b32 %r1375, %r1374, 255; + cvt.u32.u16 %r1376, %rs1105; + and.b32 %r2478, %r1376, 255; + shr.u32 %r1377, %r2478, %r1375; + and.b32 %r1378, %r1377, 1; + bfi.b32 %r2483, %r2483, %r1378, 1, 31; + setp.eq.s32 %p216, %r248, 1; + @%p216 bra $L__BB26_156; + + and.b16 %rs789, %rs1139, 255; + setp.ne.s16 %p217, %rs789, 0; + @%p217 bra $L__BB26_150; + + setp.eq.s32 %p218, %r2405, 0; + mov.u16 %rs1185, 255; + @%p218 bra $L__BB26_149; + + cvt.u64.u32 %rd244, %r2404; + add.s64 %rd245, %rd244, %rd3; + add.s64 %rd246, %rd1, %rd245; + ld.global.u8 %rs1185, [%rd246]; + +$L__BB26_149: + setp.ne.s32 %p220, %r2405, 0; + selp.u32 %r1379, 1, 0, %p220; + add.s32 %r2404, %r2404, %r1379; + add.s32 %r1380, %r2405, -1; + selp.b32 %r2405, 0, %r1380, %p218; + setp.eq.s32 %p221, %r2405, 0; + or.b16 %rs791, %rs1185, 15; + selp.b16 %rs1105, %rs791, %rs1185, %p221; + and.b16 %rs792, %rs1105, 255; + mov.u16 %rs793, 8; + sub.s16 %rs1139, %rs793, %rs1104; + setp.eq.s16 %p222, %rs792, 255; + selp.u16 %rs1104, 1, 0, %p222; + cvt.u32.u16 %r1381, %rs1105; + and.b32 %r2478, %r1381, 255; + +$L__BB26_150: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1382, %rs1139; + and.b32 %r1383, %r1382, 255; + shr.u32 %r1384, %r2478, %r1383; + and.b32 %r1385, %r1384, 1; + bfi.b32 %r2483, %r2483, %r1385, 1, 31; + setp.eq.s32 %p223, %r248, 2; + @%p223 bra $L__BB26_156; + + and.b16 %rs794, %rs1139, 255; + setp.ne.s16 %p224, %rs794, 0; + @%p224 bra $L__BB26_155; + + setp.eq.s32 %p225, %r2405, 0; + mov.u16 %rs1189, 255; + @%p225 bra $L__BB26_154; + + cvt.u64.u32 %rd247, %r2404; + add.s64 %rd248, %rd247, %rd3; + add.s64 %rd249, %rd1, %rd248; + ld.global.u8 %rs1189, [%rd249]; + +$L__BB26_154: + setp.ne.s32 %p227, %r2405, 0; + selp.u32 %r1386, 1, 0, %p227; + add.s32 %r2404, %r2404, %r1386; + add.s32 %r1387, %r2405, -1; + selp.b32 %r2405, 0, %r1387, %p225; + setp.eq.s32 %p228, %r2405, 0; + or.b16 %rs796, %rs1189, 15; + selp.b16 %rs1105, %rs796, %rs1189, %p228; + and.b16 %rs797, %rs1105, 255; + mov.u16 %rs798, 8; + sub.s16 %rs1139, %rs798, %rs1104; + setp.eq.s16 %p229, %rs797, 255; + selp.u16 %rs1104, 1, 0, %p229; + cvt.u32.u16 %r1388, %rs1105; + and.b32 %r2478, %r1388, 255; + +$L__BB26_155: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1389, %rs1139; + and.b32 %r1390, %r1389, 255; + shr.u32 %r1391, %r2478, %r1390; + and.b32 %r1392, %r1391, 1; + bfi.b32 %r2483, %r2483, %r1392, 1, 31; + +$L__BB26_156: + shl.b32 %r1393, %r2483, 1; + or.b32 %r2487, %r1393, 1; + add.s32 %r1394, %r2495, -1; + setp.eq.s32 %p230, %r2495, 0; + selp.b32 %r2486, 0, %r1394, %p230; + +$L__BB26_157: + mul.lo.s32 %r1395, %r2496, 7; + cvt.u64.u32 %rd250, %r2487; + shl.b64 %rd251, %rd250, %r1395; + or.b64 %rd603, %rd251, %rd603; + setp.ne.s32 %p231, %r2495, 12; + setp.ne.s32 %p232, %r244, 0; + or.pred %p233, %p231, %p232; + add.s32 %r2496, %r2496, 1; + setp.lt.u32 %p234, %r2496, 8; + or.pred %p235, %p234, %p233; + mov.u32 %r2495, %r2486; + @%p235 bra $L__BB26_113; + +$L__BB26_158: + cvt.u32.u64 %r1396, %rd603; + and.b32 %r2492, %r1396, 127; + shr.u64 %rd603, %rd603, 7; + add.s32 %r2496, %r2496, -1; + +$L__BB26_159: + setp.lt.u32 %p236, %r232, %r1141; + selp.b32 %r330, %r2497, 0, %p236; + st.local.u16 [%rd25+4], %r330; + and.b32 %r1398, %r1317, 64; + shl.b32 %r1399, %r330, 4; + and.b32 %r1400, %r1399, 128; + or.b32 %r2549, %r1400, %r1398; + setp.ne.s32 %p237, %r2549, 192; + @%p237 bra $L__BB26_209; + + add.s32 %r332, %r2492, -2; + setp.eq.s32 %p238, %r332, -1; + selp.b32 %r2549, 256, 192, %p238; + setp.gt.s32 %p239, %r2492, 1; + mov.u32 %r2492, %r332; + @%p239 bra $L__BB26_209; + + setp.ne.s32 %p240, %r2496, 0; + @%p240 bra $L__BB26_208; + + mov.u32 %r2496, 0; + +$L__BB26_163: + setp.gt.u32 %p241, %r2496, 7; + @%p241 bra $L__BB26_208; + + cvt.u64.u32 %rd34, %r2495; + mul.wide.u32 %rd252, %r2495, 4; + add.s64 %rd254, %rd150, %rd252; + ld.global.nc.u32 %r338, [%rd254]; + and.b16 %rs799, %rs1139, 255; + setp.ne.s16 %p242, %rs799, 0; + @%p242 bra $L__BB26_168; + + setp.eq.s32 %p243, %r2405, 0; + mov.u16 %rs1208, 255; + @%p243 bra $L__BB26_167; + + cvt.u64.u32 %rd255, %r2404; + add.s64 %rd256, %rd255, %rd3; + add.s64 %rd257, %rd1, %rd256; + ld.global.u8 %rs1208, [%rd257]; + +$L__BB26_167: + setp.ne.s32 %p245, %r2405, 0; + selp.u32 %r1402, 1, 0, %p245; + add.s32 %r2404, %r2404, %r1402; + add.s32 %r1403, %r2405, -1; + selp.b32 %r2405, 0, %r1403, %p243; + setp.eq.s32 %p246, %r2405, 0; + or.b16 %rs801, %rs1208, 15; + selp.b16 %rs1105, %rs801, %rs1208, %p246; + and.b16 %rs802, %rs1105, 255; + mov.u16 %rs803, 8; + sub.s16 %rs1139, %rs803, %rs1104; + setp.eq.s16 %p247, %rs802, 255; + selp.u16 %rs1104, 1, 0, %p247; + +$L__BB26_168: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1404, %rs1139; + and.b32 %r1405, %r1404, 255; + mov.u32 %r1406, 1; + shl.b32 %r1407, %r1406, %r1405; + cvt.u32.u16 %r1408, %rs1105; + and.b32 %r1409, %r1407, %r1408; + and.b32 %r343, %r1409, 255; + setp.eq.s32 %p248, %r343, 0; + @%p248 bra $L__BB26_170; + + add.s32 %r1410, %r2495, 1; + min.u32 %r2538, %r1410, 12; + mov.u32 %r1411, -1; + shl.b32 %r1412, %r1411, %r338; + shl.b32 %r1413, %r1412, 1; + xor.b32 %r2539, %r1413, -2; + bra.uni $L__BB26_207; + +$L__BB26_170: + add.s64 %rd258, %rd34, -3; + setp.gt.u64 %p249, %rd258, 9; + mov.u32 %r2535, 0; + @%p249 bra $L__BB26_206; + + max.u32 %r346, %r338, 1; + add.s32 %r1417, %r346, -1; + and.b32 %r347, %r346, 3; + setp.lt.u32 %p250, %r1417, 3; + mov.u32 %r2535, 0; + @%p250 bra $L__BB26_190; + + sub.s32 %r2507, %r346, %r347; + mov.u32 %r2535, 0; + +$L__BB26_173: + and.b16 %rs805, %rs1139, 255; + setp.ne.s16 %p251, %rs805, 0; + @%p251 bra $L__BB26_177; + + setp.eq.s32 %p252, %r2405, 0; + mov.u16 %rs1215, 255; + @%p252 bra $L__BB26_176; + + cvt.u64.u32 %rd259, %r2404; + add.s64 %rd260, %rd259, %rd3; + add.s64 %rd261, %rd1, %rd260; + ld.global.u8 %rs1215, [%rd261]; + +$L__BB26_176: + setp.ne.s32 %p254, %r2405, 0; + selp.u32 %r1419, 1, 0, %p254; + add.s32 %r2404, %r2404, %r1419; + add.s32 %r1420, %r2405, -1; + selp.b32 %r2405, 0, %r1420, %p252; + setp.eq.s32 %p255, %r2405, 0; + or.b16 %rs807, %rs1215, 15; + selp.b16 %rs1105, %rs807, %rs1215, %p255; + and.b16 %rs808, %rs1105, 255; + mov.u16 %rs809, 8; + sub.s16 %rs1139, %rs809, %rs1104; + setp.eq.s16 %p256, %rs808, 255; + selp.u16 %rs1104, 1, 0, %p256; + +$L__BB26_177: + add.s16 %rs1222, %rs1139, -1; + and.b16 %rs810, %rs1222, 255; + cvt.u32.u16 %r1421, %rs1222; + and.b32 %r1422, %r1421, 255; + cvt.u32.u16 %r1423, %rs1105; + and.b32 %r2513, %r1423, 255; + shr.u32 %r1424, %r2513, %r1422; + and.b32 %r1425, %r1424, 1; + bfi.b32 %r358, %r2535, %r1425, 1, 31; + setp.ne.s16 %p257, %rs810, 0; + @%p257 bra $L__BB26_181; + + setp.eq.s32 %p258, %r2405, 0; + mov.u16 %rs1219, 255; + @%p258 bra $L__BB26_180; + + cvt.u64.u32 %rd262, %r2404; + add.s64 %rd263, %rd262, %rd3; + add.s64 %rd264, %rd1, %rd263; + ld.global.u8 %rs1219, [%rd264]; + +$L__BB26_180: + setp.ne.s32 %p260, %r2405, 0; + selp.u32 %r1426, 1, 0, %p260; + add.s32 %r2404, %r2404, %r1426; + add.s32 %r1427, %r2405, -1; + selp.b32 %r2405, 0, %r1427, %p258; + setp.eq.s32 %p261, %r2405, 0; + or.b16 %rs812, %rs1219, 15; + selp.b16 %rs1105, %rs812, %rs1219, %p261; + and.b16 %rs813, %rs1105, 255; + mov.u16 %rs814, 8; + sub.s16 %rs1222, %rs814, %rs1104; + setp.eq.s16 %p262, %rs813, 255; + selp.u16 %rs1104, 1, 0, %p262; + cvt.u32.u16 %r1428, %rs1105; + and.b32 %r2513, %r1428, 255; + +$L__BB26_181: + add.s16 %rs1226, %rs1222, -1; + and.b16 %rs815, %rs1226, 255; + cvt.u32.u16 %r1429, %rs1226; + and.b32 %r1430, %r1429, 255; + shr.u32 %r1431, %r2513, %r1430; + and.b32 %r1432, %r1431, 1; + bfi.b32 %r365, %r358, %r1432, 1, 31; + setp.ne.s16 %p263, %rs815, 0; + @%p263 bra $L__BB26_185; + + setp.eq.s32 %p264, %r2405, 0; + mov.u16 %rs1223, 255; + @%p264 bra $L__BB26_184; + + cvt.u64.u32 %rd265, %r2404; + add.s64 %rd266, %rd265, %rd3; + add.s64 %rd267, %rd1, %rd266; + ld.global.u8 %rs1223, [%rd267]; + +$L__BB26_184: + setp.ne.s32 %p266, %r2405, 0; + selp.u32 %r1433, 1, 0, %p266; + add.s32 %r2404, %r2404, %r1433; + add.s32 %r1434, %r2405, -1; + selp.b32 %r2405, 0, %r1434, %p264; + setp.eq.s32 %p267, %r2405, 0; + or.b16 %rs817, %rs1223, 15; + selp.b16 %rs1105, %rs817, %rs1223, %p267; + and.b16 %rs818, %rs1105, 255; + mov.u16 %rs819, 8; + sub.s16 %rs1226, %rs819, %rs1104; + setp.eq.s16 %p268, %rs818, 255; + selp.u16 %rs1104, 1, 0, %p268; + cvt.u32.u16 %r1435, %rs1105; + and.b32 %r2513, %r1435, 255; + +$L__BB26_185: + add.s16 %rs1230, %rs1226, -1; + and.b16 %rs820, %rs1230, 255; + cvt.u32.u16 %r1436, %rs1230; + and.b32 %r1437, %r1436, 255; + shr.u32 %r1438, %r2513, %r1437; + and.b32 %r1439, %r1438, 1; + bfi.b32 %r372, %r365, %r1439, 1, 31; + setp.ne.s16 %p269, %rs820, 0; + @%p269 bra $L__BB26_189; + + setp.eq.s32 %p270, %r2405, 0; + mov.u16 %rs1227, 255; + @%p270 bra $L__BB26_188; + + cvt.u64.u32 %rd268, %r2404; + add.s64 %rd269, %rd268, %rd3; + add.s64 %rd270, %rd1, %rd269; + ld.global.u8 %rs1227, [%rd270]; + +$L__BB26_188: + setp.ne.s32 %p272, %r2405, 0; + selp.u32 %r1440, 1, 0, %p272; + add.s32 %r2404, %r2404, %r1440; + add.s32 %r1441, %r2405, -1; + selp.b32 %r2405, 0, %r1441, %p270; + setp.eq.s32 %p273, %r2405, 0; + or.b16 %rs822, %rs1227, 15; + selp.b16 %rs1105, %rs822, %rs1227, %p273; + and.b16 %rs823, %rs1105, 255; + mov.u16 %rs824, 8; + sub.s16 %rs1230, %rs824, %rs1104; + setp.eq.s16 %p274, %rs823, 255; + selp.u16 %rs1104, 1, 0, %p274; + cvt.u32.u16 %r1442, %rs1105; + and.b32 %r2513, %r1442, 255; + +$L__BB26_189: + add.s16 %rs1139, %rs1230, -1; + cvt.u32.u16 %r1443, %rs1139; + and.b32 %r1444, %r1443, 255; + shr.u32 %r1445, %r2513, %r1444; + and.b32 %r1446, %r1445, 1; + bfi.b32 %r2535, %r372, %r1446, 1, 31; + add.s32 %r2507, %r2507, -4; + setp.ne.s32 %p275, %r2507, 0; + @%p275 bra $L__BB26_173; + +$L__BB26_190: + setp.eq.s32 %p276, %r347, 0; + @%p276 bra $L__BB26_206; + + and.b16 %rs825, %rs1139, 255; + setp.ne.s16 %p277, %rs825, 0; + @%p277 bra $L__BB26_195; + + setp.eq.s32 %p278, %r2405, 0; + mov.u16 %rs1237, 255; + @%p278 bra $L__BB26_194; + + cvt.u64.u32 %rd271, %r2404; + add.s64 %rd272, %rd271, %rd3; + add.s64 %rd273, %rd1, %rd272; + ld.global.u8 %rs1237, [%rd273]; + +$L__BB26_194: + setp.ne.s32 %p280, %r2405, 0; + selp.u32 %r1447, 1, 0, %p280; + add.s32 %r2404, %r2404, %r1447; + add.s32 %r1448, %r2405, -1; + selp.b32 %r2405, 0, %r1448, %p278; + setp.eq.s32 %p281, %r2405, 0; + or.b16 %rs827, %rs1237, 15; + selp.b16 %rs1105, %rs827, %rs1237, %p281; + and.b16 %rs828, %rs1105, 255; + mov.u16 %rs829, 8; + sub.s16 %rs1139, %rs829, %rs1104; + setp.eq.s16 %p282, %rs828, 255; + selp.u16 %rs1104, 1, 0, %p282; + +$L__BB26_195: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1449, %rs1139; + and.b32 %r1450, %r1449, 255; + cvt.u32.u16 %r1451, %rs1105; + and.b32 %r2530, %r1451, 255; + shr.u32 %r1452, %r2530, %r1450; + and.b32 %r1453, %r1452, 1; + bfi.b32 %r2535, %r2535, %r1453, 1, 31; + setp.eq.s32 %p283, %r347, 1; + @%p283 bra $L__BB26_206; + + and.b16 %rs830, %rs1139, 255; + setp.ne.s16 %p284, %rs830, 0; + @%p284 bra $L__BB26_200; + + setp.eq.s32 %p285, %r2405, 0; + mov.u16 %rs1241, 255; + @%p285 bra $L__BB26_199; + + cvt.u64.u32 %rd274, %r2404; + add.s64 %rd275, %rd274, %rd3; + add.s64 %rd276, %rd1, %rd275; + ld.global.u8 %rs1241, [%rd276]; + +$L__BB26_199: + setp.ne.s32 %p287, %r2405, 0; + selp.u32 %r1454, 1, 0, %p287; + add.s32 %r2404, %r2404, %r1454; + add.s32 %r1455, %r2405, -1; + selp.b32 %r2405, 0, %r1455, %p285; + setp.eq.s32 %p288, %r2405, 0; + or.b16 %rs832, %rs1241, 15; + selp.b16 %rs1105, %rs832, %rs1241, %p288; + and.b16 %rs833, %rs1105, 255; + mov.u16 %rs834, 8; + sub.s16 %rs1139, %rs834, %rs1104; + setp.eq.s16 %p289, %rs833, 255; + selp.u16 %rs1104, 1, 0, %p289; + cvt.u32.u16 %r1456, %rs1105; + and.b32 %r2530, %r1456, 255; + +$L__BB26_200: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1457, %rs1139; + and.b32 %r1458, %r1457, 255; + shr.u32 %r1459, %r2530, %r1458; + and.b32 %r1460, %r1459, 1; + bfi.b32 %r2535, %r2535, %r1460, 1, 31; + setp.eq.s32 %p290, %r347, 2; + @%p290 bra $L__BB26_206; + + and.b16 %rs835, %rs1139, 255; + setp.ne.s16 %p291, %rs835, 0; + @%p291 bra $L__BB26_205; + + setp.eq.s32 %p292, %r2405, 0; + mov.u16 %rs1245, 255; + @%p292 bra $L__BB26_204; + + cvt.u64.u32 %rd277, %r2404; + add.s64 %rd278, %rd277, %rd3; + add.s64 %rd279, %rd1, %rd278; + ld.global.u8 %rs1245, [%rd279]; + +$L__BB26_204: + setp.ne.s32 %p294, %r2405, 0; + selp.u32 %r1461, 1, 0, %p294; + add.s32 %r2404, %r2404, %r1461; + add.s32 %r1462, %r2405, -1; + selp.b32 %r2405, 0, %r1462, %p292; + setp.eq.s32 %p295, %r2405, 0; + or.b16 %rs837, %rs1245, 15; + selp.b16 %rs1105, %rs837, %rs1245, %p295; + and.b16 %rs838, %rs1105, 255; + mov.u16 %rs839, 8; + sub.s16 %rs1139, %rs839, %rs1104; + setp.eq.s16 %p296, %rs838, 255; + selp.u16 %rs1104, 1, 0, %p296; + cvt.u32.u16 %r1463, %rs1105; + and.b32 %r2530, %r1463, 255; + +$L__BB26_205: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1464, %rs1139; + and.b32 %r1465, %r1464, 255; + shr.u32 %r1466, %r2530, %r1465; + and.b32 %r1467, %r1466, 1; + bfi.b32 %r2535, %r2535, %r1467, 1, 31; + +$L__BB26_206: + shl.b32 %r1468, %r2535, 1; + or.b32 %r2539, %r1468, 1; + add.s32 %r1469, %r2495, -1; + setp.eq.s32 %p297, %r2495, 0; + selp.b32 %r2538, 0, %r1469, %p297; + +$L__BB26_207: + mul.lo.s32 %r1470, %r2496, 7; + cvt.u64.u32 %rd280, %r2539; + shl.b64 %rd281, %rd280, %r1470; + or.b64 %rd603, %rd281, %rd603; + setp.ne.s32 %p298, %r2495, 12; + setp.ne.s32 %p299, %r343, 0; + or.pred %p300, %p298, %p299; + add.s32 %r2496, %r2496, 1; + setp.lt.u32 %p301, %r2496, 8; + or.pred %p302, %p301, %p300; + mov.u32 %r2495, %r2538; + @%p302 bra $L__BB26_163; + +$L__BB26_208: + cvt.u32.u64 %r1471, %rd603; + and.b32 %r2492, %r1471, 127; + shr.u64 %rd603, %rd603, 7; + add.s32 %r2496, %r2496, -1; + +$L__BB26_209: + and.b32 %r1472, %r330, 7; + shr.u64 %rd282, %rd26, %r1472; + cvt.u32.u64 %r1473, %rd282; + and.b32 %r1474, %r1473, 63; + add.s32 %r1475, %r2549, %r1474; + mul.wide.u32 %rd283, %r1475, 2; + add.s64 %rd284, %rd11, %rd283; + ld.global.u16 %r1476, [%rd284]; + and.b32 %r1477, %r1476, 7; + shr.u64 %rd285, %rd282, %r1477; + sub.s32 %r1478, %r230, %r1472; + sub.s32 %r1479, %r1478, %r1477; + cvt.u32.u64 %r1480, %rd285; + shr.u32 %r1481, %r1476, 3; + and.b32 %r1482, %r1481, 15; + mov.u32 %r1483, -1; + shl.b32 %r1484, %r1483, %r1482; + not.b32 %r1485, %r1484; + and.b32 %r1486, %r1480, %r1485; + shr.u64 %rd612, %rd285, %r1482; + sub.s32 %r2575, %r1479, %r1482; + shr.u32 %r1487, %r1476, 7; + and.b32 %r1488, %r1487, 7; + shr.u32 %r1489, %r1476, 10; + and.b32 %r1490, %r1489, 7; + mov.u32 %r1491, 255; + shl.b32 %r1492, %r1491, %r1488; + not.b32 %r1493, %r1492; + and.b32 %r1494, %r1486, %r1493; + add.s32 %r1495, %r1490, %r1494; + add.s32 %r1496, %r1495, 1; + st.local.u16 [%rd25+2], %r1496; + shr.u32 %r1497, %r1476, 13; + shr.u32 %r1498, %r1486, %r1488; + add.s32 %r1499, %r1497, %r1498; + add.s32 %r1500, %r1499, 1; + st.local.u16 [%rd25+6], %r1500; + add.s32 %r2379, %r2379, 4; + setp.lt.u32 %p303, %r2379, %r1141; + shl.b32 %r1501, %r330, 2; + and.b32 %r1502, %r1501, 896; + shl.b32 %r1503, %r330, 3; + and.b32 %r1504, %r1503, 128; + or.b32 %r2378, %r1504, %r1502; + @%p303 bra $L__BB26_55; + + mul.wide.u32 %rd288, %r2379, 2; + add.s64 %rd289, %rd13, %rd288; + mov.u16 %rs840, 0; + st.local.v2.u16 [%rd289], {%rs840, %rs840}; + setp.lt.u32 %p304, %r1144, 3; + @%p304 bra $L__BB26_319; + + ld.param.u64 %rd589, [ j2k_htj2k_decode_codeblocks_multi_param_3]; + ld.param.u64 %rd588, [ j2k_htj2k_decode_codeblocks_multi_param_5]; + cvta.to.global.u64 %rd40, %rd588; + cvta.to.global.u64 %rd41, %rd589; + mov.u32 %r2550, 2; + +$L__BB26_212: + shr.u32 %r1509, %r2550, 1; + mul.lo.s32 %r442, %r1509, %r1163; + sub.s32 %r443, %r442, %r1163; + mov.u32 %r2559, 0; + mov.u32 %r2560, %r2559; + mov.u32 %r2561, %r442; + +$L__BB26_213: + sub.s32 %r1510, %r2561, %r442; + add.s32 %r455, %r1510, %r443; + mul.wide.u32 %rd290, %r455, 2; + add.s64 %rd46, %rd13, %rd290; + ld.local.u16 %r1511, [%rd46]; + shl.b32 %r1512, %r1511, 2; + and.b32 %r1513, %r1512, 640; + or.b32 %r1514, %r2560, %r1513; + add.s32 %r1515, %r455, 2; + mul.wide.u32 %rd291, %r1515, 2; + add.s64 %rd47, %rd13, %rd291; + ld.local.u16 %r1516, [%rd47]; + shl.b32 %r1517, %r1516, 4; + and.b32 %r1518, %r1517, 512; + or.b32 %r456, %r1514, %r1518; + setp.gt.u32 %p305, %r2575, 31; + @%p305 bra $L__BB26_217; + +$L__BB26_214: + setp.eq.s32 %p306, %r2574, 0; + mov.u16 %rs1270, 0; + @%p306 bra $L__BB26_216; + + cvt.s64.s32 %rd292, %r2573; + add.s64 %rd293, %rd292, %rd3; + add.s64 %rd294, %rd1, %rd293; + ld.global.u8 %rs1270, [%rd294]; + +$L__BB26_216: + setp.ne.s32 %p308, %r2574, 0; + selp.b32 %r1519, -1, 0, %p308; + add.s32 %r2573, %r2573, %r1519; + add.s32 %r1520, %r2574, -1; + selp.b32 %r2574, 0, %r1520, %p306; + and.b16 %rs842, %rs1270, 255; + and.b16 %rs843, %rs1270, 127; + setp.eq.s16 %p309, %rs843, 127; + and.b16 %rs844, %rs1271, 255; + setp.ne.s16 %p310, %rs844, 0; + and.pred %p311, %p310, %p309; + selp.b32 %r1521, 7, 8, %p311; + cvt.u64.u16 %rd295, %rs1270; + and.b64 %rd296, %rd295, 255; + shl.b64 %rd297, %rd296, %r2575; + or.b64 %rd612, %rd297, %rd612; + add.s32 %r2575, %r1521, %r2575; + setp.gt.u16 %p312, %rs842, 143; + selp.u16 %rs1271, 1, 0, %p312; + setp.lt.u32 %p313, %r2575, 33; + @%p313 bra $L__BB26_214; + +$L__BB26_217: + cvt.u32.u64 %r1522, %rd612; + and.b32 %r1523, %r1522, 127; + add.s32 %r1524, %r1523, %r456; + mul.wide.u32 %rd298, %r1524, 2; + add.s64 %rd299, %rd41, %rd298; + ld.global.u16 %r2627, [%rd299]; + setp.ne.s32 %p314, %r456, 0; + @%p314 bra $L__BB26_267; + + add.s32 %r467, %r2492, -2; + setp.eq.s32 %p315, %r467, -1; + selp.b32 %r2627, %r2627, 0, %p315; + setp.gt.s32 %p316, %r2492, 1; + mov.u32 %r2492, %r467; + @%p316 bra $L__BB26_267; + + setp.ne.s32 %p317, %r2496, 0; + @%p317 bra $L__BB26_266; + + mov.u32 %r2496, 0; + +$L__BB26_221: + setp.gt.u32 %p318, %r2496, 7; + @%p318 bra $L__BB26_266; + + cvt.u64.u32 %rd52, %r2495; + mul.wide.u32 %rd300, %r2495, 4; + add.s64 %rd302, %rd150, %rd300; + ld.global.nc.u32 %r473, [%rd302]; + and.b16 %rs845, %rs1139, 255; + setp.ne.s16 %p319, %rs845, 0; + @%p319 bra $L__BB26_226; + + setp.eq.s32 %p320, %r2405, 0; + mov.u16 %rs1275, 255; + @%p320 bra $L__BB26_225; + + cvt.u64.u32 %rd303, %r2404; + add.s64 %rd304, %rd303, %rd3; + add.s64 %rd305, %rd1, %rd304; + ld.global.u8 %rs1275, [%rd305]; + +$L__BB26_225: + setp.ne.s32 %p322, %r2405, 0; + selp.u32 %r1526, 1, 0, %p322; + add.s32 %r2404, %r2404, %r1526; + add.s32 %r1527, %r2405, -1; + selp.b32 %r2405, 0, %r1527, %p320; + setp.eq.s32 %p323, %r2405, 0; + or.b16 %rs847, %rs1275, 15; + selp.b16 %rs1105, %rs847, %rs1275, %p323; + and.b16 %rs848, %rs1105, 255; + mov.u16 %rs849, 8; + sub.s16 %rs1139, %rs849, %rs1104; + setp.eq.s16 %p324, %rs848, 255; + selp.u16 %rs1104, 1, 0, %p324; + +$L__BB26_226: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1528, %rs1139; + and.b32 %r1529, %r1528, 255; + mov.u32 %r1530, 1; + shl.b32 %r1531, %r1530, %r1529; + cvt.u32.u16 %r1532, %rs1105; + and.b32 %r1533, %r1531, %r1532; + and.b32 %r478, %r1533, 255; + setp.eq.s32 %p325, %r478, 0; + @%p325 bra $L__BB26_228; + + add.s32 %r1534, %r2495, 1; + min.u32 %r2616, %r1534, 12; + mov.u32 %r1535, -1; + shl.b32 %r1536, %r1535, %r473; + shl.b32 %r1537, %r1536, 1; + xor.b32 %r2617, %r1537, -2; + bra.uni $L__BB26_265; + +$L__BB26_228: + add.s64 %rd306, %rd52, -3; + setp.gt.u64 %p326, %rd306, 9; + mov.u32 %r2613, 0; + @%p326 bra $L__BB26_264; + + max.u32 %r481, %r473, 1; + add.s32 %r1541, %r481, -1; + and.b32 %r482, %r481, 3; + setp.lt.u32 %p327, %r1541, 3; + mov.u32 %r2613, 0; + @%p327 bra $L__BB26_248; + + sub.s32 %r2585, %r481, %r482; + mov.u32 %r2613, 0; + +$L__BB26_231: + and.b16 %rs851, %rs1139, 255; + setp.ne.s16 %p328, %rs851, 0; + @%p328 bra $L__BB26_235; + + setp.eq.s32 %p329, %r2405, 0; + mov.u16 %rs1282, 255; + @%p329 bra $L__BB26_234; + + cvt.u64.u32 %rd307, %r2404; + add.s64 %rd308, %rd307, %rd3; + add.s64 %rd309, %rd1, %rd308; + ld.global.u8 %rs1282, [%rd309]; + +$L__BB26_234: + setp.ne.s32 %p331, %r2405, 0; + selp.u32 %r1543, 1, 0, %p331; + add.s32 %r2404, %r2404, %r1543; + add.s32 %r1544, %r2405, -1; + selp.b32 %r2405, 0, %r1544, %p329; + setp.eq.s32 %p332, %r2405, 0; + or.b16 %rs853, %rs1282, 15; + selp.b16 %rs1105, %rs853, %rs1282, %p332; + and.b16 %rs854, %rs1105, 255; + mov.u16 %rs855, 8; + sub.s16 %rs1139, %rs855, %rs1104; + setp.eq.s16 %p333, %rs854, 255; + selp.u16 %rs1104, 1, 0, %p333; + +$L__BB26_235: + add.s16 %rs1289, %rs1139, -1; + and.b16 %rs856, %rs1289, 255; + cvt.u32.u16 %r1545, %rs1289; + and.b32 %r1546, %r1545, 255; + cvt.u32.u16 %r1547, %rs1105; + and.b32 %r2591, %r1547, 255; + shr.u32 %r1548, %r2591, %r1546; + and.b32 %r1549, %r1548, 1; + bfi.b32 %r493, %r2613, %r1549, 1, 31; + setp.ne.s16 %p334, %rs856, 0; + @%p334 bra $L__BB26_239; + + setp.eq.s32 %p335, %r2405, 0; + mov.u16 %rs1286, 255; + @%p335 bra $L__BB26_238; + + cvt.u64.u32 %rd310, %r2404; + add.s64 %rd311, %rd310, %rd3; + add.s64 %rd312, %rd1, %rd311; + ld.global.u8 %rs1286, [%rd312]; + +$L__BB26_238: + setp.ne.s32 %p337, %r2405, 0; + selp.u32 %r1550, 1, 0, %p337; + add.s32 %r2404, %r2404, %r1550; + add.s32 %r1551, %r2405, -1; + selp.b32 %r2405, 0, %r1551, %p335; + setp.eq.s32 %p338, %r2405, 0; + or.b16 %rs858, %rs1286, 15; + selp.b16 %rs1105, %rs858, %rs1286, %p338; + and.b16 %rs859, %rs1105, 255; + mov.u16 %rs860, 8; + sub.s16 %rs1289, %rs860, %rs1104; + setp.eq.s16 %p339, %rs859, 255; + selp.u16 %rs1104, 1, 0, %p339; + cvt.u32.u16 %r1552, %rs1105; + and.b32 %r2591, %r1552, 255; + +$L__BB26_239: + add.s16 %rs1293, %rs1289, -1; + and.b16 %rs861, %rs1293, 255; + cvt.u32.u16 %r1553, %rs1293; + and.b32 %r1554, %r1553, 255; + shr.u32 %r1555, %r2591, %r1554; + and.b32 %r1556, %r1555, 1; + bfi.b32 %r500, %r493, %r1556, 1, 31; + setp.ne.s16 %p340, %rs861, 0; + @%p340 bra $L__BB26_243; + + setp.eq.s32 %p341, %r2405, 0; + mov.u16 %rs1290, 255; + @%p341 bra $L__BB26_242; + + cvt.u64.u32 %rd313, %r2404; + add.s64 %rd314, %rd313, %rd3; + add.s64 %rd315, %rd1, %rd314; + ld.global.u8 %rs1290, [%rd315]; + +$L__BB26_242: + setp.ne.s32 %p343, %r2405, 0; + selp.u32 %r1557, 1, 0, %p343; + add.s32 %r2404, %r2404, %r1557; + add.s32 %r1558, %r2405, -1; + selp.b32 %r2405, 0, %r1558, %p341; + setp.eq.s32 %p344, %r2405, 0; + or.b16 %rs863, %rs1290, 15; + selp.b16 %rs1105, %rs863, %rs1290, %p344; + and.b16 %rs864, %rs1105, 255; + mov.u16 %rs865, 8; + sub.s16 %rs1293, %rs865, %rs1104; + setp.eq.s16 %p345, %rs864, 255; + selp.u16 %rs1104, 1, 0, %p345; + cvt.u32.u16 %r1559, %rs1105; + and.b32 %r2591, %r1559, 255; + +$L__BB26_243: + add.s16 %rs1297, %rs1293, -1; + and.b16 %rs866, %rs1297, 255; + cvt.u32.u16 %r1560, %rs1297; + and.b32 %r1561, %r1560, 255; + shr.u32 %r1562, %r2591, %r1561; + and.b32 %r1563, %r1562, 1; + bfi.b32 %r507, %r500, %r1563, 1, 31; + setp.ne.s16 %p346, %rs866, 0; + @%p346 bra $L__BB26_247; + + setp.eq.s32 %p347, %r2405, 0; + mov.u16 %rs1294, 255; + @%p347 bra $L__BB26_246; + + cvt.u64.u32 %rd316, %r2404; + add.s64 %rd317, %rd316, %rd3; + add.s64 %rd318, %rd1, %rd317; + ld.global.u8 %rs1294, [%rd318]; + +$L__BB26_246: + setp.ne.s32 %p349, %r2405, 0; + selp.u32 %r1564, 1, 0, %p349; + add.s32 %r2404, %r2404, %r1564; + add.s32 %r1565, %r2405, -1; + selp.b32 %r2405, 0, %r1565, %p347; + setp.eq.s32 %p350, %r2405, 0; + or.b16 %rs868, %rs1294, 15; + selp.b16 %rs1105, %rs868, %rs1294, %p350; + and.b16 %rs869, %rs1105, 255; + mov.u16 %rs870, 8; + sub.s16 %rs1297, %rs870, %rs1104; + setp.eq.s16 %p351, %rs869, 255; + selp.u16 %rs1104, 1, 0, %p351; + cvt.u32.u16 %r1566, %rs1105; + and.b32 %r2591, %r1566, 255; + +$L__BB26_247: + add.s16 %rs1139, %rs1297, -1; + cvt.u32.u16 %r1567, %rs1139; + and.b32 %r1568, %r1567, 255; + shr.u32 %r1569, %r2591, %r1568; + and.b32 %r1570, %r1569, 1; + bfi.b32 %r2613, %r507, %r1570, 1, 31; + add.s32 %r2585, %r2585, -4; + setp.ne.s32 %p352, %r2585, 0; + @%p352 bra $L__BB26_231; + +$L__BB26_248: + setp.eq.s32 %p353, %r482, 0; + @%p353 bra $L__BB26_264; + + and.b16 %rs871, %rs1139, 255; + setp.ne.s16 %p354, %rs871, 0; + @%p354 bra $L__BB26_253; + + setp.eq.s32 %p355, %r2405, 0; + mov.u16 %rs1304, 255; + @%p355 bra $L__BB26_252; + + cvt.u64.u32 %rd319, %r2404; + add.s64 %rd320, %rd319, %rd3; + add.s64 %rd321, %rd1, %rd320; + ld.global.u8 %rs1304, [%rd321]; + +$L__BB26_252: + setp.ne.s32 %p357, %r2405, 0; + selp.u32 %r1571, 1, 0, %p357; + add.s32 %r2404, %r2404, %r1571; + add.s32 %r1572, %r2405, -1; + selp.b32 %r2405, 0, %r1572, %p355; + setp.eq.s32 %p358, %r2405, 0; + or.b16 %rs873, %rs1304, 15; + selp.b16 %rs1105, %rs873, %rs1304, %p358; + and.b16 %rs874, %rs1105, 255; + mov.u16 %rs875, 8; + sub.s16 %rs1139, %rs875, %rs1104; + setp.eq.s16 %p359, %rs874, 255; + selp.u16 %rs1104, 1, 0, %p359; + +$L__BB26_253: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1573, %rs1139; + and.b32 %r1574, %r1573, 255; + cvt.u32.u16 %r1575, %rs1105; + and.b32 %r2608, %r1575, 255; + shr.u32 %r1576, %r2608, %r1574; + and.b32 %r1577, %r1576, 1; + bfi.b32 %r2613, %r2613, %r1577, 1, 31; + setp.eq.s32 %p360, %r482, 1; + @%p360 bra $L__BB26_264; + + and.b16 %rs876, %rs1139, 255; + setp.ne.s16 %p361, %rs876, 0; + @%p361 bra $L__BB26_258; + + setp.eq.s32 %p362, %r2405, 0; + mov.u16 %rs1308, 255; + @%p362 bra $L__BB26_257; + + cvt.u64.u32 %rd322, %r2404; + add.s64 %rd323, %rd322, %rd3; + add.s64 %rd324, %rd1, %rd323; + ld.global.u8 %rs1308, [%rd324]; + +$L__BB26_257: + setp.ne.s32 %p364, %r2405, 0; + selp.u32 %r1578, 1, 0, %p364; + add.s32 %r2404, %r2404, %r1578; + add.s32 %r1579, %r2405, -1; + selp.b32 %r2405, 0, %r1579, %p362; + setp.eq.s32 %p365, %r2405, 0; + or.b16 %rs878, %rs1308, 15; + selp.b16 %rs1105, %rs878, %rs1308, %p365; + and.b16 %rs879, %rs1105, 255; + mov.u16 %rs880, 8; + sub.s16 %rs1139, %rs880, %rs1104; + setp.eq.s16 %p366, %rs879, 255; + selp.u16 %rs1104, 1, 0, %p366; + cvt.u32.u16 %r1580, %rs1105; + and.b32 %r2608, %r1580, 255; + +$L__BB26_258: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1581, %rs1139; + and.b32 %r1582, %r1581, 255; + shr.u32 %r1583, %r2608, %r1582; + and.b32 %r1584, %r1583, 1; + bfi.b32 %r2613, %r2613, %r1584, 1, 31; + setp.eq.s32 %p367, %r482, 2; + @%p367 bra $L__BB26_264; + + and.b16 %rs881, %rs1139, 255; + setp.ne.s16 %p368, %rs881, 0; + @%p368 bra $L__BB26_263; + + setp.eq.s32 %p369, %r2405, 0; + mov.u16 %rs1312, 255; + @%p369 bra $L__BB26_262; + + cvt.u64.u32 %rd325, %r2404; + add.s64 %rd326, %rd325, %rd3; + add.s64 %rd327, %rd1, %rd326; + ld.global.u8 %rs1312, [%rd327]; + +$L__BB26_262: + setp.ne.s32 %p371, %r2405, 0; + selp.u32 %r1585, 1, 0, %p371; + add.s32 %r2404, %r2404, %r1585; + add.s32 %r1586, %r2405, -1; + selp.b32 %r2405, 0, %r1586, %p369; + setp.eq.s32 %p372, %r2405, 0; + or.b16 %rs883, %rs1312, 15; + selp.b16 %rs1105, %rs883, %rs1312, %p372; + and.b16 %rs884, %rs1105, 255; + mov.u16 %rs885, 8; + sub.s16 %rs1139, %rs885, %rs1104; + setp.eq.s16 %p373, %rs884, 255; + selp.u16 %rs1104, 1, 0, %p373; + cvt.u32.u16 %r1587, %rs1105; + and.b32 %r2608, %r1587, 255; + +$L__BB26_263: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1588, %rs1139; + and.b32 %r1589, %r1588, 255; + shr.u32 %r1590, %r2608, %r1589; + and.b32 %r1591, %r1590, 1; + bfi.b32 %r2613, %r2613, %r1591, 1, 31; + +$L__BB26_264: + shl.b32 %r1592, %r2613, 1; + or.b32 %r2617, %r1592, 1; + add.s32 %r1593, %r2495, -1; + setp.eq.s32 %p374, %r2495, 0; + selp.b32 %r2616, 0, %r1593, %p374; + +$L__BB26_265: + mul.lo.s32 %r1594, %r2496, 7; + cvt.u64.u32 %rd328, %r2617; + shl.b64 %rd329, %rd328, %r1594; + or.b64 %rd603, %rd329, %rd603; + setp.ne.s32 %p375, %r2495, 12; + setp.ne.s32 %p376, %r478, 0; + or.pred %p377, %p375, %p376; + add.s32 %r2496, %r2496, 1; + setp.lt.u32 %p378, %r2496, 8; + or.pred %p379, %p378, %p377; + mov.u32 %r2495, %r2616; + @%p379 bra $L__BB26_221; + +$L__BB26_266: + cvt.u32.u64 %r1595, %rd603; + and.b32 %r2492, %r1595, 127; + shr.u64 %rd603, %rd603, 7; + add.s32 %r2496, %r2496, -1; + +$L__BB26_267: + mul.wide.u32 %rd330, %r2561, 2; + add.s64 %rd331, %rd13, %rd330; + st.local.u16 [%rd331], %r2627; + shl.b32 %r1596, %r2627, 2; + shl.b32 %r1597, %r2627, 1; + or.b32 %r1598, %r1596, %r1597; + and.b32 %r1599, %r1598, 256; + ld.local.u16 %r1600, [%rd46]; + and.b32 %r1601, %r1600, 128; + or.b32 %r1602, %r1599, %r1601; + ld.local.u16 %r1603, [%rd47]; + shl.b32 %r1604, %r1603, 2; + and.b32 %r1605, %r1604, 640; + or.b32 %r1606, %r1602, %r1605; + add.s32 %r1607, %r455, 4; + mul.wide.u32 %rd332, %r1607, 2; + add.s64 %rd333, %rd13, %rd332; + ld.local.u16 %r1608, [%rd333]; + shl.b32 %r1609, %r1608, 4; + and.b32 %r1610, %r1609, 512; + or.b32 %r1611, %r1606, %r1610; + and.b32 %r1612, %r2627, 7; + shr.u64 %rd57, %rd612, %r1612; + sub.s32 %r564, %r2575, %r1612; + cvt.u32.u64 %r1613, %rd57; + and.b32 %r1614, %r1613, 127; + or.b32 %r1615, %r1611, %r1614; + mul.wide.u32 %rd334, %r1615, 2; + add.s64 %rd335, %rd41, %rd334; + ld.global.u16 %r2679, [%rd335]; + setp.ne.s32 %p380, %r1611, 0; + add.s32 %r566, %r2559, 2; + setp.ge.u32 %p381, %r566, %r1141; + or.pred %p382, %p381, %p380; + @%p382 bra $L__BB26_317; + + add.s32 %r567, %r2492, -2; + setp.eq.s32 %p383, %r567, -1; + selp.b32 %r2679, %r2679, 0, %p383; + setp.gt.s32 %p384, %r2492, 1; + mov.u32 %r2492, %r567; + @%p384 bra $L__BB26_317; + + setp.ne.s32 %p385, %r2496, 0; + @%p385 bra $L__BB26_316; + + mov.u32 %r2496, 0; + +$L__BB26_271: + setp.gt.u32 %p386, %r2496, 7; + @%p386 bra $L__BB26_316; + + cvt.u64.u32 %rd59, %r2495; + mul.wide.u32 %rd336, %r2495, 4; + add.s64 %rd338, %rd150, %rd336; + ld.global.nc.u32 %r573, [%rd338]; + and.b16 %rs886, %rs1139, 255; + setp.ne.s16 %p387, %rs886, 0; + @%p387 bra $L__BB26_276; + + setp.eq.s32 %p388, %r2405, 0; + mov.u16 %rs1331, 255; + @%p388 bra $L__BB26_275; + + cvt.u64.u32 %rd339, %r2404; + add.s64 %rd340, %rd339, %rd3; + add.s64 %rd341, %rd1, %rd340; + ld.global.u8 %rs1331, [%rd341]; + +$L__BB26_275: + setp.ne.s32 %p390, %r2405, 0; + selp.u32 %r1617, 1, 0, %p390; + add.s32 %r2404, %r2404, %r1617; + add.s32 %r1618, %r2405, -1; + selp.b32 %r2405, 0, %r1618, %p388; + setp.eq.s32 %p391, %r2405, 0; + or.b16 %rs888, %rs1331, 15; + selp.b16 %rs1105, %rs888, %rs1331, %p391; + and.b16 %rs889, %rs1105, 255; + mov.u16 %rs890, 8; + sub.s16 %rs1139, %rs890, %rs1104; + setp.eq.s16 %p392, %rs889, 255; + selp.u16 %rs1104, 1, 0, %p392; + +$L__BB26_276: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1619, %rs1139; + and.b32 %r1620, %r1619, 255; + mov.u32 %r1621, 1; + shl.b32 %r1622, %r1621, %r1620; + cvt.u32.u16 %r1623, %rs1105; + and.b32 %r1624, %r1622, %r1623; + and.b32 %r578, %r1624, 255; + setp.eq.s32 %p393, %r578, 0; + @%p393 bra $L__BB26_278; + + add.s32 %r1625, %r2495, 1; + min.u32 %r2668, %r1625, 12; + mov.u32 %r1626, -1; + shl.b32 %r1627, %r1626, %r573; + shl.b32 %r1628, %r1627, 1; + xor.b32 %r2669, %r1628, -2; + bra.uni $L__BB26_315; + +$L__BB26_278: + add.s64 %rd342, %rd59, -3; + setp.gt.u64 %p394, %rd342, 9; + mov.u32 %r2665, 0; + @%p394 bra $L__BB26_314; + + max.u32 %r581, %r573, 1; + add.s32 %r1632, %r581, -1; + and.b32 %r582, %r581, 3; + setp.lt.u32 %p395, %r1632, 3; + mov.u32 %r2665, 0; + @%p395 bra $L__BB26_298; + + sub.s32 %r2637, %r581, %r582; + mov.u32 %r2665, 0; + +$L__BB26_281: + and.b16 %rs892, %rs1139, 255; + setp.ne.s16 %p396, %rs892, 0; + @%p396 bra $L__BB26_285; + + setp.eq.s32 %p397, %r2405, 0; + mov.u16 %rs1338, 255; + @%p397 bra $L__BB26_284; + + cvt.u64.u32 %rd343, %r2404; + add.s64 %rd344, %rd343, %rd3; + add.s64 %rd345, %rd1, %rd344; + ld.global.u8 %rs1338, [%rd345]; + +$L__BB26_284: + setp.ne.s32 %p399, %r2405, 0; + selp.u32 %r1634, 1, 0, %p399; + add.s32 %r2404, %r2404, %r1634; + add.s32 %r1635, %r2405, -1; + selp.b32 %r2405, 0, %r1635, %p397; + setp.eq.s32 %p400, %r2405, 0; + or.b16 %rs894, %rs1338, 15; + selp.b16 %rs1105, %rs894, %rs1338, %p400; + and.b16 %rs895, %rs1105, 255; + mov.u16 %rs896, 8; + sub.s16 %rs1139, %rs896, %rs1104; + setp.eq.s16 %p401, %rs895, 255; + selp.u16 %rs1104, 1, 0, %p401; + +$L__BB26_285: + add.s16 %rs1345, %rs1139, -1; + and.b16 %rs897, %rs1345, 255; + cvt.u32.u16 %r1636, %rs1345; + and.b32 %r1637, %r1636, 255; + cvt.u32.u16 %r1638, %rs1105; + and.b32 %r2643, %r1638, 255; + shr.u32 %r1639, %r2643, %r1637; + and.b32 %r1640, %r1639, 1; + bfi.b32 %r593, %r2665, %r1640, 1, 31; + setp.ne.s16 %p402, %rs897, 0; + @%p402 bra $L__BB26_289; + + setp.eq.s32 %p403, %r2405, 0; + mov.u16 %rs1342, 255; + @%p403 bra $L__BB26_288; + + cvt.u64.u32 %rd346, %r2404; + add.s64 %rd347, %rd346, %rd3; + add.s64 %rd348, %rd1, %rd347; + ld.global.u8 %rs1342, [%rd348]; + +$L__BB26_288: + setp.ne.s32 %p405, %r2405, 0; + selp.u32 %r1641, 1, 0, %p405; + add.s32 %r2404, %r2404, %r1641; + add.s32 %r1642, %r2405, -1; + selp.b32 %r2405, 0, %r1642, %p403; + setp.eq.s32 %p406, %r2405, 0; + or.b16 %rs899, %rs1342, 15; + selp.b16 %rs1105, %rs899, %rs1342, %p406; + and.b16 %rs900, %rs1105, 255; + mov.u16 %rs901, 8; + sub.s16 %rs1345, %rs901, %rs1104; + setp.eq.s16 %p407, %rs900, 255; + selp.u16 %rs1104, 1, 0, %p407; + cvt.u32.u16 %r1643, %rs1105; + and.b32 %r2643, %r1643, 255; + +$L__BB26_289: + add.s16 %rs1349, %rs1345, -1; + and.b16 %rs902, %rs1349, 255; + cvt.u32.u16 %r1644, %rs1349; + and.b32 %r1645, %r1644, 255; + shr.u32 %r1646, %r2643, %r1645; + and.b32 %r1647, %r1646, 1; + bfi.b32 %r600, %r593, %r1647, 1, 31; + setp.ne.s16 %p408, %rs902, 0; + @%p408 bra $L__BB26_293; + + setp.eq.s32 %p409, %r2405, 0; + mov.u16 %rs1346, 255; + @%p409 bra $L__BB26_292; + + cvt.u64.u32 %rd349, %r2404; + add.s64 %rd350, %rd349, %rd3; + add.s64 %rd351, %rd1, %rd350; + ld.global.u8 %rs1346, [%rd351]; + +$L__BB26_292: + setp.ne.s32 %p411, %r2405, 0; + selp.u32 %r1648, 1, 0, %p411; + add.s32 %r2404, %r2404, %r1648; + add.s32 %r1649, %r2405, -1; + selp.b32 %r2405, 0, %r1649, %p409; + setp.eq.s32 %p412, %r2405, 0; + or.b16 %rs904, %rs1346, 15; + selp.b16 %rs1105, %rs904, %rs1346, %p412; + and.b16 %rs905, %rs1105, 255; + mov.u16 %rs906, 8; + sub.s16 %rs1349, %rs906, %rs1104; + setp.eq.s16 %p413, %rs905, 255; + selp.u16 %rs1104, 1, 0, %p413; + cvt.u32.u16 %r1650, %rs1105; + and.b32 %r2643, %r1650, 255; + +$L__BB26_293: + add.s16 %rs1353, %rs1349, -1; + and.b16 %rs907, %rs1353, 255; + cvt.u32.u16 %r1651, %rs1353; + and.b32 %r1652, %r1651, 255; + shr.u32 %r1653, %r2643, %r1652; + and.b32 %r1654, %r1653, 1; + bfi.b32 %r607, %r600, %r1654, 1, 31; + setp.ne.s16 %p414, %rs907, 0; + @%p414 bra $L__BB26_297; + + setp.eq.s32 %p415, %r2405, 0; + mov.u16 %rs1350, 255; + @%p415 bra $L__BB26_296; + + cvt.u64.u32 %rd352, %r2404; + add.s64 %rd353, %rd352, %rd3; + add.s64 %rd354, %rd1, %rd353; + ld.global.u8 %rs1350, [%rd354]; + +$L__BB26_296: + setp.ne.s32 %p417, %r2405, 0; + selp.u32 %r1655, 1, 0, %p417; + add.s32 %r2404, %r2404, %r1655; + add.s32 %r1656, %r2405, -1; + selp.b32 %r2405, 0, %r1656, %p415; + setp.eq.s32 %p418, %r2405, 0; + or.b16 %rs909, %rs1350, 15; + selp.b16 %rs1105, %rs909, %rs1350, %p418; + and.b16 %rs910, %rs1105, 255; + mov.u16 %rs911, 8; + sub.s16 %rs1353, %rs911, %rs1104; + setp.eq.s16 %p419, %rs910, 255; + selp.u16 %rs1104, 1, 0, %p419; + cvt.u32.u16 %r1657, %rs1105; + and.b32 %r2643, %r1657, 255; + +$L__BB26_297: + add.s16 %rs1139, %rs1353, -1; + cvt.u32.u16 %r1658, %rs1139; + and.b32 %r1659, %r1658, 255; + shr.u32 %r1660, %r2643, %r1659; + and.b32 %r1661, %r1660, 1; + bfi.b32 %r2665, %r607, %r1661, 1, 31; + add.s32 %r2637, %r2637, -4; + setp.ne.s32 %p420, %r2637, 0; + @%p420 bra $L__BB26_281; + +$L__BB26_298: + setp.eq.s32 %p421, %r582, 0; + @%p421 bra $L__BB26_314; + + and.b16 %rs912, %rs1139, 255; + setp.ne.s16 %p422, %rs912, 0; + @%p422 bra $L__BB26_303; + + setp.eq.s32 %p423, %r2405, 0; + mov.u16 %rs1360, 255; + @%p423 bra $L__BB26_302; + + cvt.u64.u32 %rd355, %r2404; + add.s64 %rd356, %rd355, %rd3; + add.s64 %rd357, %rd1, %rd356; + ld.global.u8 %rs1360, [%rd357]; + +$L__BB26_302: + setp.ne.s32 %p425, %r2405, 0; + selp.u32 %r1662, 1, 0, %p425; + add.s32 %r2404, %r2404, %r1662; + add.s32 %r1663, %r2405, -1; + selp.b32 %r2405, 0, %r1663, %p423; + setp.eq.s32 %p426, %r2405, 0; + or.b16 %rs914, %rs1360, 15; + selp.b16 %rs1105, %rs914, %rs1360, %p426; + and.b16 %rs915, %rs1105, 255; + mov.u16 %rs916, 8; + sub.s16 %rs1139, %rs916, %rs1104; + setp.eq.s16 %p427, %rs915, 255; + selp.u16 %rs1104, 1, 0, %p427; + +$L__BB26_303: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1664, %rs1139; + and.b32 %r1665, %r1664, 255; + cvt.u32.u16 %r1666, %rs1105; + and.b32 %r2660, %r1666, 255; + shr.u32 %r1667, %r2660, %r1665; + and.b32 %r1668, %r1667, 1; + bfi.b32 %r2665, %r2665, %r1668, 1, 31; + setp.eq.s32 %p428, %r582, 1; + @%p428 bra $L__BB26_314; + + and.b16 %rs917, %rs1139, 255; + setp.ne.s16 %p429, %rs917, 0; + @%p429 bra $L__BB26_308; + + setp.eq.s32 %p430, %r2405, 0; + mov.u16 %rs1364, 255; + @%p430 bra $L__BB26_307; + + cvt.u64.u32 %rd358, %r2404; + add.s64 %rd359, %rd358, %rd3; + add.s64 %rd360, %rd1, %rd359; + ld.global.u8 %rs1364, [%rd360]; + +$L__BB26_307: + setp.ne.s32 %p432, %r2405, 0; + selp.u32 %r1669, 1, 0, %p432; + add.s32 %r2404, %r2404, %r1669; + add.s32 %r1670, %r2405, -1; + selp.b32 %r2405, 0, %r1670, %p430; + setp.eq.s32 %p433, %r2405, 0; + or.b16 %rs919, %rs1364, 15; + selp.b16 %rs1105, %rs919, %rs1364, %p433; + and.b16 %rs920, %rs1105, 255; + mov.u16 %rs921, 8; + sub.s16 %rs1139, %rs921, %rs1104; + setp.eq.s16 %p434, %rs920, 255; + selp.u16 %rs1104, 1, 0, %p434; + cvt.u32.u16 %r1671, %rs1105; + and.b32 %r2660, %r1671, 255; + +$L__BB26_308: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1672, %rs1139; + and.b32 %r1673, %r1672, 255; + shr.u32 %r1674, %r2660, %r1673; + and.b32 %r1675, %r1674, 1; + bfi.b32 %r2665, %r2665, %r1675, 1, 31; + setp.eq.s32 %p435, %r582, 2; + @%p435 bra $L__BB26_314; + + and.b16 %rs922, %rs1139, 255; + setp.ne.s16 %p436, %rs922, 0; + @%p436 bra $L__BB26_313; + + setp.eq.s32 %p437, %r2405, 0; + mov.u16 %rs1368, 255; + @%p437 bra $L__BB26_312; + + cvt.u64.u32 %rd361, %r2404; + add.s64 %rd362, %rd361, %rd3; + add.s64 %rd363, %rd1, %rd362; + ld.global.u8 %rs1368, [%rd363]; + +$L__BB26_312: + setp.ne.s32 %p439, %r2405, 0; + selp.u32 %r1676, 1, 0, %p439; + add.s32 %r2404, %r2404, %r1676; + add.s32 %r1677, %r2405, -1; + selp.b32 %r2405, 0, %r1677, %p437; + setp.eq.s32 %p440, %r2405, 0; + or.b16 %rs924, %rs1368, 15; + selp.b16 %rs1105, %rs924, %rs1368, %p440; + and.b16 %rs925, %rs1105, 255; + mov.u16 %rs926, 8; + sub.s16 %rs1139, %rs926, %rs1104; + setp.eq.s16 %p441, %rs925, 255; + selp.u16 %rs1104, 1, 0, %p441; + cvt.u32.u16 %r1678, %rs1105; + and.b32 %r2660, %r1678, 255; + +$L__BB26_313: + add.s16 %rs1139, %rs1139, -1; + cvt.u32.u16 %r1679, %rs1139; + and.b32 %r1680, %r1679, 255; + shr.u32 %r1681, %r2660, %r1680; + and.b32 %r1682, %r1681, 1; + bfi.b32 %r2665, %r2665, %r1682, 1, 31; + +$L__BB26_314: + shl.b32 %r1683, %r2665, 1; + or.b32 %r2669, %r1683, 1; + add.s32 %r1684, %r2495, -1; + setp.eq.s32 %p442, %r2495, 0; + selp.b32 %r2668, 0, %r1684, %p442; + +$L__BB26_315: + mul.lo.s32 %r1685, %r2496, 7; + cvt.u64.u32 %rd364, %r2669; + shl.b64 %rd365, %rd364, %r1685; + or.b64 %rd603, %rd365, %rd603; + setp.ne.s32 %p443, %r2495, 12; + setp.ne.s32 %p444, %r578, 0; + or.pred %p445, %p443, %p444; + add.s32 %r2496, %r2496, 1; + setp.lt.u32 %p446, %r2496, 8; + or.pred %p447, %p446, %p445; + mov.u32 %r2495, %r2668; + @%p447 bra $L__BB26_271; + +$L__BB26_316: + cvt.u32.u64 %r1686, %rd603; + and.b32 %r2492, %r1686, 127; + shr.u64 %rd603, %rd603, 7; + add.s32 %r2496, %r2496, -1; + +$L__BB26_317: + setp.lt.u32 %p448, %r566, %r1141; + selp.b32 %r1687, %r2679, 0, %p448; + add.s32 %r1688, %r2561, 2; + mul.wide.u32 %rd366, %r1688, 2; + add.s64 %rd367, %rd13, %rd366; + st.local.u16 [%rd367], %r1687; + shl.b32 %r1689, %r1687, 2; + shl.b32 %r1690, %r1687, 1; + or.b32 %r1691, %r1689, %r1690; + and.b32 %r1692, %r1691, 256; + ld.local.u16 %r1693, [%rd47]; + and.b32 %r1694, %r1693, 128; + or.b32 %r2560, %r1692, %r1694; + and.b32 %r1695, %r1687, 7; + shr.u64 %rd368, %rd57, %r1695; + sub.s32 %r1696, %r564, %r1695; + cvt.u32.u64 %r1697, %rd368; + shl.b32 %r1698, %r2627, 3; + and.b32 %r1699, %r1698, 64; + shl.b32 %r1700, %r1687, 4; + and.b32 %r1701, %r1700, 128; + or.b32 %r1702, %r1701, %r1699; + and.b32 %r1703, %r1697, 63; + or.b32 %r1704, %r1703, %r1702; + mul.wide.u32 %rd369, %r1704, 2; + add.s64 %rd370, %rd40, %rd369; + ld.global.u16 %r1705, [%rd370]; + and.b32 %r1706, %r1705, 7; + shr.u64 %rd371, %rd368, %r1706; + sub.s32 %r1707, %r1696, %r1706; + cvt.u32.u64 %r1708, %rd371; + shr.u32 %r1709, %r1705, 3; + and.b32 %r1710, %r1709, 15; + mov.u32 %r1711, -1; + shl.b32 %r1712, %r1711, %r1710; + not.b32 %r1713, %r1712; + and.b32 %r1714, %r1708, %r1713; + shr.u64 %rd612, %rd371, %r1710; + sub.s32 %r2575, %r1707, %r1710; + shr.u32 %r1715, %r1705, 7; + and.b32 %r1716, %r1715, 7; + shr.u32 %r1717, %r1705, 10; + and.b32 %r1718, %r1717, 7; + mov.u32 %r1719, 255; + shl.b32 %r1720, %r1719, %r1716; + not.b32 %r1721, %r1720; + and.b32 %r1722, %r1714, %r1721; + add.s32 %r1723, %r1722, %r1718; + add.s32 %r1724, %r2561, 1; + mul.wide.u32 %rd372, %r1724, 2; + add.s64 %rd373, %rd13, %rd372; + st.local.u16 [%rd373], %r1723; + shr.u32 %r1725, %r1705, 13; + shr.u32 %r1726, %r1714, %r1716; + add.s32 %r1727, %r1726, %r1725; + add.s32 %r1728, %r2561, 3; + mul.wide.u32 %rd374, %r1728, 2; + add.s64 %rd375, %rd13, %rd374; + st.local.u16 [%rd375], %r1727; + add.s32 %r2561, %r2561, 4; + add.s32 %r2559, %r2559, 4; + setp.lt.u32 %p449, %r2559, %r1141; + @%p449 bra $L__BB26_213; + + mul.wide.u32 %rd376, %r2561, 2; + add.s64 %rd377, %rd13, %rd376; + mov.u16 %rs927, 0; + st.local.v2.u16 [%rd377], {%rs927, %rs927}; + add.s32 %r2550, %r2550, 2; + setp.lt.u32 %p450, %r2550, %r1144; + @%p450 bra $L__BB26_212; + +$L__BB26_319: + mov.u32 %r1729, 30; + sub.s32 %r669, %r1729, %r1148; + add.s32 %r1730, %r1141, 1; + shr.u32 %r1731, %r1730, 1; + add.s32 %r1732, %r1731, 2; + setp.gt.u32 %p451, %r1732, 130; + @%p451 bra $L__BB26_534; + bra.uni $L__BB26_320; + +$L__BB26_534: + mov.u32 %r2296, 2; + st.global.u32 [%rd4], %r2296; + mov.u32 %r2297, 12; + st.global.u32 [%rd4+4], %r2297; + mov.u32 %r2298, 0; + st.global.u32 [%rd4+8], %r2298; + st.global.u32 [%rd4+12], %r2298; + bra.uni $L__BB26_541; + +$L__BB26_320: + add.s32 %r670, %r1148, 2; + add.s32 %r671, %r669, -1; + mov.u32 %r2680, 0; + mov.u64 %rd640, 0; + mov.u16 %rs1411, 0; + mov.u32 %r2681, %r2680; + mov.u32 %r2682, %r2680; + mov.u32 %r2683, %r14; + mov.u32 %r2750, %r2680; + mov.u32 %r2749, %r2680; + +$L__BB26_321: + mov.u32 %r674, %r2682; + mul.wide.u32 %rd379, %r2681, 2; + add.s64 %rd380, %rd13, %rd379; + ld.local.u16 %r678, [%rd380]; + ld.local.u16 %r679, [%rd380+2]; + setp.lt.u32 %p452, %r670, %r679; + @%p452 bra $L__BB26_533; + + and.b32 %r1739, %r678, 16; + setp.eq.s32 %p453, %r1739, 0; + mov.u32 %r2699, 0; + mov.u32 %r2691, %r2699; + @%p453 bra $L__BB26_328; + + setp.gt.u32 %p454, %r2750, 31; + @%p454 bra $L__BB26_327; + +$L__BB26_324: + setp.ge.u32 %p455, %r2749, %r20; + mov.u16 %rs1386, 255; + @%p455 bra $L__BB26_326; + + add.s32 %r682, %r2749, 1; + cvt.u64.u32 %rd381, %r2749; + add.s64 %rd382, %rd381, %rd3; + add.s64 %rd383, %rd1, %rd382; + ld.global.u8 %rs1386, [%rd383]; + mov.u32 %r2749, %r682; + +$L__BB26_326: + and.b16 %rs930, %rs1386, 255; + cvt.u64.u16 %rd384, %rs1386; + and.b64 %rd385, %rd384, 255; + shl.b64 %rd386, %rd385, %r2750; + or.b64 %rd640, %rd386, %rd640; + add.s32 %r1740, %r2750, 8; + cvt.u32.u16 %r1741, %rs1411; + cvt.s32.s8 %r1742, %r1741; + sub.s32 %r2750, %r1740, %r1742; + setp.eq.s16 %p456, %rs930, 255; + selp.u16 %rs1411, 1, 0, %p456; + setp.lt.u32 %p457, %r2750, 33; + @%p457 bra $L__BB26_324; + +$L__BB26_327: + shr.u32 %r1743, %r678, 12; + and.b32 %r1744, %r1743, 1; + sub.s32 %r1745, %r679, %r1744; + shr.u64 %rd69, %rd640, %r1745; + sub.s32 %r2750, %r2750, %r1745; + cvt.u32.u64 %r1746, %rd640; + shl.b32 %r1747, %r1746, 31; + setp.eq.s32 %p458, %r1745, 0; + mov.u32 %r1748, -1; + shl.b32 %r1749, %r1748, %r1745; + not.b32 %r1750, %r1749; + selp.b32 %r1751, 0, %r1750, %p458; + and.b32 %r1752, %r1751, %r1746; + shr.u32 %r1753, %r678, 8; + and.b32 %r1754, %r1753, 1; + shl.b32 %r1755, %r1754, %r1745; + or.b32 %r1756, %r1755, %r1752; + or.b32 %r1757, %r1756, 1; + add.s32 %r1758, %r1757, 2; + shl.b32 %r1759, %r1758, %r671; + or.b32 %r2691, %r1759, %r1747; + mov.u64 %rd640, %rd69; + +$L__BB26_328: + mul.wide.u32 %rd387, %r2683, 4; + add.s64 %rd388, %rd2, %rd387; + st.global.u32 [%rd388], %r2691; + and.b32 %r1762, %r678, 32; + setp.eq.s32 %p459, %r1762, 0; + mov.u32 %r2700, %r2699; + @%p459 bra $L__BB26_334; + + setp.gt.u32 %p460, %r2750, 31; + @%p460 bra $L__BB26_333; + +$L__BB26_330: + setp.ge.u32 %p461, %r2749, %r20; + mov.u16 %rs1390, 255; + @%p461 bra $L__BB26_332; + + add.s32 %r694, %r2749, 1; + cvt.u64.u32 %rd389, %r2749; + add.s64 %rd390, %rd389, %rd3; + add.s64 %rd391, %rd1, %rd390; + ld.global.u8 %rs1390, [%rd391]; + mov.u32 %r2749, %r694; + +$L__BB26_332: + and.b16 %rs932, %rs1390, 255; + cvt.u64.u16 %rd392, %rs1390; + and.b64 %rd393, %rd392, 255; + shl.b64 %rd394, %rd393, %r2750; + or.b64 %rd640, %rd394, %rd640; + add.s32 %r1763, %r2750, 8; + cvt.u32.u16 %r1764, %rs1411; + cvt.s32.s8 %r1765, %r1764; + sub.s32 %r2750, %r1763, %r1765; + setp.eq.s16 %p462, %rs932, 255; + selp.u16 %rs1411, 1, 0, %p462; + setp.lt.u32 %p463, %r2750, 33; + @%p463 bra $L__BB26_330; + +$L__BB26_333: + shr.u32 %r1766, %r678, 13; + and.b32 %r1767, %r1766, 1; + sub.s32 %r1768, %r679, %r1767; + shr.u64 %rd74, %rd640, %r1768; + sub.s32 %r2750, %r2750, %r1768; + cvt.u32.u64 %r1769, %rd640; + shl.b32 %r1770, %r1769, 31; + setp.eq.s32 %p464, %r1768, 0; + mov.u32 %r1771, -1; + shl.b32 %r1772, %r1771, %r1768; + not.b32 %r1773, %r1772; + selp.b32 %r1774, 0, %r1773, %p464; + and.b32 %r1775, %r1774, %r1769; + shr.u32 %r1776, %r678, 9; + and.b32 %r1777, %r1776, 1; + shl.b32 %r1778, %r1777, %r1768; + or.b32 %r1779, %r1778, %r1775; + or.b32 %r2699, %r1779, 1; + add.s32 %r1780, %r2699, 2; + shl.b32 %r1781, %r1780, %r671; + or.b32 %r2700, %r1781, %r1770; + mov.u64 %rd640, %rd74; + +$L__BB26_334: + setp.lt.u32 %p465, %r1144, 2; + @%p465 bra $L__BB26_336; + + add.s32 %r1782, %r2683, %r1151; + mul.wide.u32 %rd395, %r1782, 4; + add.s64 %rd396, %rd2, %rd395; + st.global.u32 [%rd396], %r2700; + +$L__BB26_336: + or.b32 %r1783, %r2699, %r2680; + add.u64 %rd76, %SPL, 6192; + mul.wide.u32 %rd398, %r674, 4; + add.s64 %rd399, %rd76, %rd398; + st.local.u32 [%rd399], %r1783; + add.s32 %r706, %r2683, 1; + add.s32 %r1784, %r2681, 1; + setp.lt.u32 %p466, %r1784, %r1141; + @%p466 bra $L__BB26_338; + bra.uni $L__BB26_337; + +$L__BB26_338: + and.b32 %r1787, %r678, 64; + setp.eq.s32 %p467, %r1787, 0; + mov.u32 %r2716, 0; + mov.u32 %r2708, %r2716; + @%p467 bra $L__BB26_344; + + setp.gt.u32 %p468, %r2750, 31; + @%p468 bra $L__BB26_343; + +$L__BB26_340: + setp.ge.u32 %p469, %r2749, %r20; + mov.u16 %rs1394, 255; + @%p469 bra $L__BB26_342; + + add.s32 %r709, %r2749, 1; + cvt.u64.u32 %rd400, %r2749; + add.s64 %rd401, %rd400, %rd3; + add.s64 %rd402, %rd1, %rd401; + ld.global.u8 %rs1394, [%rd402]; + mov.u32 %r2749, %r709; + +$L__BB26_342: + and.b16 %rs934, %rs1394, 255; + cvt.u64.u16 %rd403, %rs1394; + and.b64 %rd404, %rd403, 255; + shl.b64 %rd405, %rd404, %r2750; + or.b64 %rd640, %rd405, %rd640; + add.s32 %r1788, %r2750, 8; + cvt.u32.u16 %r1789, %rs1411; + cvt.s32.s8 %r1790, %r1789; + sub.s32 %r2750, %r1788, %r1790; + setp.eq.s16 %p470, %rs934, 255; + selp.u16 %rs1411, 1, 0, %p470; + setp.lt.u32 %p471, %r2750, 33; + @%p471 bra $L__BB26_340; + +$L__BB26_343: + shr.u32 %r1791, %r678, 14; + and.b32 %r1792, %r1791, 1; + sub.s32 %r1793, %r679, %r1792; + shr.u64 %rd80, %rd640, %r1793; + sub.s32 %r2750, %r2750, %r1793; + cvt.u32.u64 %r1794, %rd640; + shl.b32 %r1795, %r1794, 31; + setp.eq.s32 %p472, %r1793, 0; + mov.u32 %r1796, -1; + shl.b32 %r1797, %r1796, %r1793; + not.b32 %r1798, %r1797; + selp.b32 %r1799, 0, %r1798, %p472; + and.b32 %r1800, %r1799, %r1794; + shr.u32 %r1801, %r678, 10; + and.b32 %r1802, %r1801, 1; + shl.b32 %r1803, %r1802, %r1793; + or.b32 %r1804, %r1803, %r1800; + or.b32 %r1805, %r1804, 1; + add.s32 %r1806, %r1805, 2; + shl.b32 %r1807, %r1806, %r671; + or.b32 %r2708, %r1807, %r1795; + mov.u64 %rd640, %rd80; + +$L__BB26_344: + mul.wide.u32 %rd406, %r706, 4; + add.s64 %rd407, %rd2, %rd406; + st.global.u32 [%rd407], %r2708; + and.b32 %r1810, %r678, 128; + setp.eq.s32 %p473, %r1810, 0; + mov.u32 %r2680, %r2716; + @%p473 bra $L__BB26_350; + + setp.gt.u32 %p474, %r2750, 31; + @%p474 bra $L__BB26_349; + +$L__BB26_346: + setp.ge.u32 %p475, %r2749, %r20; + mov.u16 %rs1398, 255; + @%p475 bra $L__BB26_348; + + add.s32 %r721, %r2749, 1; + cvt.u64.u32 %rd408, %r2749; + add.s64 %rd409, %rd408, %rd3; + add.s64 %rd410, %rd1, %rd409; + ld.global.u8 %rs1398, [%rd410]; + mov.u32 %r2749, %r721; + +$L__BB26_348: + and.b16 %rs936, %rs1398, 255; + cvt.u64.u16 %rd411, %rs1398; + and.b64 %rd412, %rd411, 255; + shl.b64 %rd413, %rd412, %r2750; + or.b64 %rd640, %rd413, %rd640; + add.s32 %r1811, %r2750, 8; + cvt.u32.u16 %r1812, %rs1411; + cvt.s32.s8 %r1813, %r1812; + sub.s32 %r2750, %r1811, %r1813; + setp.eq.s16 %p476, %rs936, 255; + selp.u16 %rs1411, 1, 0, %p476; + setp.lt.u32 %p477, %r2750, 33; + @%p477 bra $L__BB26_346; + +$L__BB26_349: + shr.u32 %r1814, %r678, 15; + sub.s32 %r1815, %r679, %r1814; + shr.u64 %rd85, %rd640, %r1815; + sub.s32 %r2750, %r2750, %r1815; + cvt.u32.u64 %r1816, %rd640; + shl.b32 %r1817, %r1816, 31; + setp.eq.s32 %p478, %r1815, 0; + mov.u32 %r1818, -1; + shl.b32 %r1819, %r1818, %r1815; + not.b32 %r1820, %r1819; + selp.b32 %r1821, 0, %r1820, %p478; + and.b32 %r1822, %r1821, %r1816; + shr.u32 %r1823, %r678, 11; + and.b32 %r1824, %r1823, 1; + shl.b32 %r1825, %r1824, %r1815; + or.b32 %r1826, %r1825, %r1822; + or.b32 %r2680, %r1826, 1; + add.s32 %r1827, %r2680, 2; + shl.b32 %r1828, %r1827, %r671; + or.b32 %r2716, %r1828, %r1817; + mov.u64 %rd640, %rd85; + +$L__BB26_350: + @%p465 bra $L__BB26_352; + + add.s32 %r1829, %r706, %r1151; + mul.wide.u32 %rd414, %r1829, 4; + add.s64 %rd415, %rd2, %rd414; + st.global.u32 [%rd415], %r2716; + +$L__BB26_352: + add.s32 %r2683, %r2683, 2; + add.s32 %r2682, %r674, 1; + add.s32 %r2681, %r2681, 2; + setp.lt.u32 %p480, %r2681, %r1141; + @%p480 bra $L__BB26_321; + bra.uni $L__BB26_353; + +$L__BB26_533: + mov.u32 %r2289, 1; + st.global.u32 [%rd4], %r2289; + mov.u32 %r2290, 13; + st.global.u32 [%rd4+4], %r2290; + mov.u32 %r2291, 0; + st.global.u32 [%rd4+8], %r2291; + st.global.u32 [%rd4+12], %r2291; + bra.uni $L__BB26_541; + +$L__BB26_337: + mov.u32 %r2680, 0; + +$L__BB26_353: + add.s32 %r1830, %r674, 1; + mul.wide.u32 %rd418, %r1830, 4; + add.s64 %rd419, %rd76, %rd418; + st.local.u32 [%rd419], %r2680; + @%p304 bra $L__BB26_388; + + mov.u32 %r2723, 2; + +$L__BB26_355: + shr.u32 %r1836, %r2723, 1; + mul.lo.s32 %r2727, %r1836, %r1163; + mad.lo.s32 %r2729, %r2723, %r1151, %r14; + add.s32 %r745, %r2723, 1; + ld.local.u32 %r2726, [%rd76]; + mov.u32 %r2728, 0; + mov.u32 %r2730, %r2728; + mov.u32 %r2731, %r2728; + +$L__BB26_356: + mul.wide.u32 %rd420, %r2727, 2; + add.s64 %rd421, %rd13, %rd420; + ld.local.v2.u16 {%rs937, %rs938}, [%rd421]; + cvt.u32.u16 %r755, %rs937; + cvt.u32.u16 %r1837, %rs938; + and.b32 %r1838, %r755, 240; + add.s32 %r1839, %r1838, 240; + and.b32 %r1840, %r1839, %r1838; + add.s32 %r756, %r2728, 1; + mul.wide.u32 %rd422, %r756, 4; + add.s64 %rd90, %rd76, %rd422; + ld.local.u32 %r757, [%rd90]; + or.b32 %r1841, %r2726, %r757; + or.b32 %r1842, %r1841, 2; + clz.b32 %r1843, %r1842; + xor.b32 %r1844, %r1843, 31; + setp.eq.s32 %p482, %r1840, 0; + selp.b32 %r1845, 1, %r1844, %p482; + add.s32 %r758, %r1845, %r1837; + setp.gt.u32 %p483, %r758, %r670; + @%p483 bra $L__BB26_532; + + and.b32 %r1847, %r755, 16; + setp.eq.s32 %p484, %r1847, 0; + mov.u32 %r2747, 0; + mov.u32 %r2739, %r2747; + @%p484 bra $L__BB26_363; + + setp.gt.u32 %p485, %r2750, 31; + @%p485 bra $L__BB26_362; + +$L__BB26_359: + setp.ge.u32 %p486, %r2749, %r20; + mov.u16 %rs1405, 255; + @%p486 bra $L__BB26_361; + + add.s32 %r761, %r2749, 1; + cvt.u64.u32 %rd423, %r2749; + add.s64 %rd424, %rd423, %rd3; + add.s64 %rd425, %rd1, %rd424; + ld.global.u8 %rs1405, [%rd425]; + mov.u32 %r2749, %r761; + +$L__BB26_361: + and.b16 %rs942, %rs1405, 255; + cvt.u64.u16 %rd426, %rs1405; + and.b64 %rd427, %rd426, 255; + shl.b64 %rd428, %rd427, %r2750; + or.b64 %rd640, %rd428, %rd640; + add.s32 %r1848, %r2750, 8; + cvt.u32.u16 %r1849, %rs1411; + cvt.s32.s8 %r1850, %r1849; + sub.s32 %r2750, %r1848, %r1850; + setp.eq.s16 %p487, %rs942, 255; + selp.u16 %rs1411, 1, 0, %p487; + setp.lt.u32 %p488, %r2750, 33; + @%p488 bra $L__BB26_359; + +$L__BB26_362: + shr.u32 %r1851, %r755, 12; + and.b32 %r1852, %r1851, 1; + sub.s32 %r1853, %r758, %r1852; + shr.u64 %rd94, %rd640, %r1853; + sub.s32 %r2750, %r2750, %r1853; + cvt.u32.u64 %r1854, %rd640; + shl.b32 %r1855, %r1854, 31; + setp.eq.s32 %p489, %r1853, 0; + mov.u32 %r1856, -1; + shl.b32 %r1857, %r1856, %r1853; + not.b32 %r1858, %r1857; + selp.b32 %r1859, 0, %r1858, %p489; + and.b32 %r1860, %r1859, %r1854; + shr.u32 %r1861, %r755, 8; + and.b32 %r1862, %r1861, 1; + shl.b32 %r1863, %r1862, %r1853; + or.b32 %r1864, %r1863, %r1860; + or.b32 %r1865, %r1864, 1; + add.s32 %r1866, %r1865, 2; + shl.b32 %r1867, %r1866, %r671; + or.b32 %r2739, %r1867, %r1855; + mov.u64 %rd640, %rd94; + +$L__BB26_363: + mul.wide.u32 %rd429, %r2729, 4; + add.s64 %rd430, %rd2, %rd429; + st.global.u32 [%rd430], %r2739; + and.b32 %r1870, %r755, 32; + setp.eq.s32 %p490, %r1870, 0; + mov.u32 %r2748, %r2747; + @%p490 bra $L__BB26_369; + + setp.gt.u32 %p491, %r2750, 31; + @%p491 bra $L__BB26_368; + +$L__BB26_365: + setp.ge.u32 %p492, %r2749, %r20; + mov.u16 %rs1409, 255; + @%p492 bra $L__BB26_367; + + add.s32 %r773, %r2749, 1; + cvt.u64.u32 %rd431, %r2749; + add.s64 %rd432, %rd431, %rd3; + add.s64 %rd433, %rd1, %rd432; + ld.global.u8 %rs1409, [%rd433]; + mov.u32 %r2749, %r773; + +$L__BB26_367: + and.b16 %rs944, %rs1409, 255; + cvt.u64.u16 %rd434, %rs1409; + and.b64 %rd435, %rd434, 255; + shl.b64 %rd436, %rd435, %r2750; + or.b64 %rd640, %rd436, %rd640; + add.s32 %r1871, %r2750, 8; + cvt.u32.u16 %r1872, %rs1411; + cvt.s32.s8 %r1873, %r1872; + sub.s32 %r2750, %r1871, %r1873; + setp.eq.s16 %p493, %rs944, 255; + selp.u16 %rs1411, 1, 0, %p493; + setp.lt.u32 %p494, %r2750, 33; + @%p494 bra $L__BB26_365; + +$L__BB26_368: + shr.u32 %r1874, %r755, 13; + and.b32 %r1875, %r1874, 1; + sub.s32 %r1876, %r758, %r1875; + shr.u64 %rd99, %rd640, %r1876; + sub.s32 %r2750, %r2750, %r1876; + cvt.u32.u64 %r1877, %rd640; + shl.b32 %r1878, %r1877, 31; + setp.eq.s32 %p495, %r1876, 0; + mov.u32 %r1879, -1; + shl.b32 %r1880, %r1879, %r1876; + not.b32 %r1881, %r1880; + selp.b32 %r1882, 0, %r1881, %p495; + and.b32 %r1883, %r1882, %r1877; + shr.u32 %r1884, %r755, 9; + and.b32 %r1885, %r1884, 1; + shl.b32 %r1886, %r1885, %r1876; + or.b32 %r1887, %r1886, %r1883; + or.b32 %r2748, %r1887, 1; + add.s32 %r1888, %r2748, 2; + shl.b32 %r1889, %r1888, %r671; + or.b32 %r2747, %r1889, %r1878; + mov.u64 %rd640, %rd99; + +$L__BB26_369: + setp.ge.u32 %p496, %r745, %r1144; + @%p496 bra $L__BB26_371; + + add.s32 %r1890, %r2729, %r1151; + mul.wide.u32 %rd437, %r1890, 4; + add.s64 %rd438, %rd2, %rd437; + st.global.u32 [%rd438], %r2747; + +$L__BB26_371: + or.b32 %r1892, %r2748, %r2730; + mul.wide.u32 %rd439, %r2728, 4; + add.s64 %rd440, %rd76, %rd439; + st.local.u32 [%rd440], %r1892; + add.s32 %r785, %r2729, 1; + add.s32 %r1893, %r2731, 1; + setp.ge.u32 %p497, %r1893, %r1141; + mov.u32 %r2730, 0; + @%p497 bra $L__BB26_387; + + and.b32 %r1895, %r755, 64; + setp.eq.s32 %p498, %r1895, 0; + mov.u32 %r2764, 0; + mov.u32 %r2756, %r2764; + @%p498 bra $L__BB26_378; + + setp.gt.u32 %p499, %r2750, 31; + @%p499 bra $L__BB26_377; + +$L__BB26_374: + setp.ge.u32 %p500, %r2749, %r20; + mov.u16 %rs1413, 255; + @%p500 bra $L__BB26_376; + + add.s32 %r788, %r2749, 1; + cvt.u64.u32 %rd441, %r2749; + add.s64 %rd442, %rd441, %rd3; + add.s64 %rd443, %rd1, %rd442; + ld.global.u8 %rs1413, [%rd443]; + mov.u32 %r2749, %r788; + +$L__BB26_376: + and.b16 %rs946, %rs1413, 255; + cvt.u64.u16 %rd444, %rs1413; + and.b64 %rd445, %rd444, 255; + shl.b64 %rd446, %rd445, %r2750; + or.b64 %rd640, %rd446, %rd640; + add.s32 %r1896, %r2750, 8; + cvt.u32.u16 %r1897, %rs1411; + cvt.s32.s8 %r1898, %r1897; + sub.s32 %r2750, %r1896, %r1898; + setp.eq.s16 %p501, %rs946, 255; + selp.u16 %rs1411, 1, 0, %p501; + setp.lt.u32 %p502, %r2750, 33; + @%p502 bra $L__BB26_374; + +$L__BB26_377: + shr.u32 %r1899, %r755, 14; + and.b32 %r1900, %r1899, 1; + sub.s32 %r1901, %r758, %r1900; + shr.u64 %rd104, %rd640, %r1901; + sub.s32 %r2750, %r2750, %r1901; + cvt.u32.u64 %r1902, %rd640; + shl.b32 %r1903, %r1902, 31; + setp.eq.s32 %p503, %r1901, 0; + mov.u32 %r1904, -1; + shl.b32 %r1905, %r1904, %r1901; + not.b32 %r1906, %r1905; + selp.b32 %r1907, 0, %r1906, %p503; + and.b32 %r1908, %r1907, %r1902; + shr.u32 %r1909, %r755, 10; + and.b32 %r1910, %r1909, 1; + shl.b32 %r1911, %r1910, %r1901; + or.b32 %r1912, %r1911, %r1908; + or.b32 %r1913, %r1912, 1; + add.s32 %r1914, %r1913, 2; + shl.b32 %r1915, %r1914, %r671; + or.b32 %r2756, %r1915, %r1903; + mov.u64 %rd640, %rd104; + +$L__BB26_378: + mul.wide.u32 %rd447, %r785, 4; + add.s64 %rd448, %rd2, %rd447; + st.global.u32 [%rd448], %r2756; + and.b32 %r1918, %r755, 128; + setp.eq.s32 %p504, %r1918, 0; + mov.u32 %r2730, %r2764; + @%p504 bra $L__BB26_384; + + setp.gt.u32 %p505, %r2750, 31; + @%p505 bra $L__BB26_383; + +$L__BB26_380: + setp.ge.u32 %p506, %r2749, %r20; + mov.u16 %rs1417, 255; + @%p506 bra $L__BB26_382; + + add.s32 %r800, %r2749, 1; + cvt.u64.u32 %rd449, %r2749; + add.s64 %rd450, %rd449, %rd3; + add.s64 %rd451, %rd1, %rd450; + ld.global.u8 %rs1417, [%rd451]; + mov.u32 %r2749, %r800; + +$L__BB26_382: + and.b16 %rs948, %rs1417, 255; + cvt.u64.u16 %rd452, %rs1417; + and.b64 %rd453, %rd452, 255; + shl.b64 %rd454, %rd453, %r2750; + or.b64 %rd640, %rd454, %rd640; + add.s32 %r1919, %r2750, 8; + cvt.u32.u16 %r1920, %rs1411; + cvt.s32.s8 %r1921, %r1920; + sub.s32 %r2750, %r1919, %r1921; + setp.eq.s16 %p507, %rs948, 255; + selp.u16 %rs1411, 1, 0, %p507; + setp.lt.u32 %p508, %r2750, 33; + @%p508 bra $L__BB26_380; + +$L__BB26_383: + shr.u32 %r1922, %r755, 15; + sub.s32 %r1923, %r758, %r1922; + shr.u64 %rd109, %rd640, %r1923; + sub.s32 %r2750, %r2750, %r1923; + cvt.u32.u64 %r1924, %rd640; + shl.b32 %r1925, %r1924, 31; + setp.eq.s32 %p509, %r1923, 0; + mov.u32 %r1926, -1; + shl.b32 %r1927, %r1926, %r1923; + not.b32 %r1928, %r1927; + selp.b32 %r1929, 0, %r1928, %p509; + and.b32 %r1930, %r1929, %r1924; + shr.u32 %r1931, %r755, 11; + and.b32 %r1932, %r1931, 1; + shl.b32 %r1933, %r1932, %r1923; + or.b32 %r1934, %r1933, %r1930; + or.b32 %r2730, %r1934, 1; + add.s32 %r1935, %r2730, 2; + shl.b32 %r1936, %r1935, %r671; + or.b32 %r2764, %r1936, %r1925; + mov.u64 %rd640, %rd109; + +$L__BB26_384: + @%p496 bra $L__BB26_386; + + add.s32 %r1937, %r785, %r1151; + mul.wide.u32 %rd455, %r1937, 4; + add.s64 %rd456, %rd2, %rd455; + st.global.u32 [%rd456], %r2764; + +$L__BB26_386: + add.s32 %r2729, %r2729, 2; + add.s32 %r2727, %r2727, 2; + add.s32 %r2731, %r2731, 2; + setp.lt.u32 %p511, %r2731, %r1141; + mov.u32 %r2726, %r757; + mov.u32 %r2728, %r756; + @%p511 bra $L__BB26_356; + +$L__BB26_387: + st.local.u32 [%rd90], %r2730; + add.s32 %r2723, %r2723, 2; + setp.lt.u32 %p512, %r2723, %r1144; + @%p512 bra $L__BB26_355; + +$L__BB26_388: + setp.lt.u32 %p513, %r17, 2; + @%p513 bra $L__BB26_541; + + add.s32 %r1938, %r1144, 3; + shr.u32 %r819, %r1938, 2; + add.s32 %r1939, %r819, 1; + add.s32 %r820, %r1141, 3; + shr.u32 %r821, %r820, 2; + add.s32 %r1940, %r821, 9; + and.b32 %r822, %r1940, 2147483640; + add.s32 %r823, %r821, 8; + setp.gt.u32 %p514, %r822, 72; + mul.lo.s32 %r1941, %r1939, %r822; + setp.gt.u32 %p515, %r1941, 528; + or.pred %p516, %p514, %p515; + @%p516 bra $L__BB26_531; + bra.uni $L__BB26_390; + +$L__BB26_531: + mov.u32 %r2275, 2; + st.global.u32 [%rd4], %r2275; + mov.u32 %r2276, 15; + st.global.u32 [%rd4+4], %r2276; + mov.u32 %r2277, 0; + st.global.u32 [%rd4+8], %r2277; + st.global.u32 [%rd4+12], %r2277; + bra.uni $L__BB26_541; + +$L__BB26_532: + mov.u32 %r2282, 1; + st.global.u32 [%rd4], %r2282; + mov.u32 %r2283, 14; + st.global.u32 [%rd4+4], %r2283; + mov.u32 %r2284, 0; + st.global.u32 [%rd4+8], %r2284; + st.global.u32 [%rd4+12], %r2284; + +$L__BB26_541: + ret; + +$L__BB26_390: + setp.gt.u32 %p517, %r823, 72; + @%p517 bra $L__BB26_530; + bra.uni $L__BB26_391; + +$L__BB26_530: + mov.u32 %r2268, 2; + st.global.u32 [%rd4], %r2268; + mov.u32 %r2269, 16; + st.global.u32 [%rd4+4], %r2269; + mov.u32 %r2270, 0; + st.global.u32 [%rd4+8], %r2270; + st.global.u32 [%rd4+12], %r2270; + bra.uni $L__BB26_541; + +$L__BB26_391: + add.u64 %rd112, %SPL, 6720; + mov.u32 %r1942, 0; + mov.u32 %r2771, %r1942; + +$L__BB26_392: + shr.u32 %r1945, %r2771, 1; + mul.lo.s32 %r2773, %r1945, %r1163; + shr.u32 %r1946, %r2771, 2; + mul.lo.s32 %r2772, %r1946, %r822; + mov.u32 %r2774, %r1942; + +$L__BB26_393: + mul.wide.u32 %rd458, %r2773, 2; + add.s64 %rd459, %rd13, %rd458; + ld.local.u16 %rs949, [%rd459]; + and.b16 %rs950, %rs949, 48; + shr.u16 %rs951, %rs950, 4; + and.b16 %rs952, %rs949, 192; + shr.u16 %rs953, %rs952, 2; + add.s32 %r1947, %r2773, 2; + mul.wide.u32 %rd460, %r1947, 2; + add.s64 %rd461, %rd13, %rd460; + ld.local.u16 %rs954, [%rd461]; + shl.b16 %rs955, %rs954, 4; + and.b16 %rs956, %rs955, 768; + shl.b16 %rs957, %rs954, 6; + and.b16 %rs958, %rs957, 12288; + add.s32 %r1948, %r2773, %r1163; + mul.wide.u32 %rd462, %r1948, 2; + add.s64 %rd463, %rd13, %rd462; + ld.local.u16 %rs959, [%rd463]; + and.b16 %rs960, %rs959, 48; + shr.u16 %rs961, %rs960, 2; + and.b16 %rs962, %rs959, 192; + add.s32 %r1949, %r1948, 2; + mul.wide.u32 %rd464, %r1949, 2; + add.s64 %rd465, %rd13, %rd464; + ld.local.u16 %rs963, [%rd465]; + shl.b16 %rs964, %rs963, 6; + and.b16 %rs965, %rs964, 3072; + shl.b16 %rs966, %rs963, 8; + and.b16 %rs967, %rs966, -16384; + or.b16 %rs968, %rs951, %rs953; + or.b16 %rs969, %rs968, %rs958; + or.b16 %rs970, %rs969, %rs956; + or.b16 %rs971, %rs970, %rs962; + or.b16 %rs972, %rs971, %rs961; + or.b16 %rs973, %rs972, %rs967; + or.b16 %rs974, %rs973, %rs965; + mul.wide.u32 %rd466, %r2772, 2; + add.s64 %rd467, %rd112, %rd466; + st.local.u16 [%rd467], %rs974; + add.s32 %r2773, %r2773, 4; + add.s32 %r2772, %r2772, 1; + add.s32 %r2774, %r2774, 4; + setp.lt.u32 %p518, %r2774, %r1141; + @%p518 bra $L__BB26_393; + + mul.wide.u32 %rd468, %r2772, 2; + add.s64 %rd469, %rd112, %rd468; + mov.u16 %rs975, 0; + st.local.u16 [%rd469], %rs975; + add.s32 %r2771, %r2771, 4; + setp.lt.u32 %p519, %r2771, %r1144; + @%p519 bra $L__BB26_392; + + mul.lo.s32 %r835, %r822, %r819; + add.s32 %r836, %r821, 1; + and.b32 %r2886, %r836, 3; + setp.lt.u32 %p520, %r820, 12; + mov.u32 %r2777, 0; + @%p520 bra $L__BB26_398; + + sub.s32 %r2776, %r836, %r2886; + mov.u32 %r2777, 0; + +$L__BB26_397: + add.s32 %r1952, %r2777, %r835; + mul.wide.u32 %rd470, %r1952, 2; + add.s64 %rd471, %rd112, %rd470; + mov.u16 %rs976, 0; + st.local.v4.u16 [%rd471], {%rs976, %rs976, %rs976, %rs976}; + add.s32 %r2777, %r2777, 4; + add.s32 %r2776, %r2776, -4; + setp.ne.s32 %p521, %r2776, 0; + @%p521 bra $L__BB26_397; + +$L__BB26_398: + setp.eq.s32 %p522, %r2886, 0; + @%p522 bra $L__BB26_401; + + mov.u32 %r2779, %r2886; + +$L__BB26_400: + .pragma "nounroll"; + add.s32 %r1953, %r2777, %r835; + mul.wide.u32 %rd472, %r1953, 2; + add.s64 %rd473, %rd112, %rd472; + mov.u16 %rs977, 0; + st.local.u16 [%rd473], %rs977; + add.s32 %r2777, %r2777, 1; + add.s32 %r2779, %r2779, -1; + setp.ne.s32 %p523, %r2779, 0; + @%p523 bra $L__BB26_400; + +$L__BB26_401: + and.b32 %r2783, %r823, 3; + sub.s32 %r2781, %r823, %r2783; + add.u64 %rd113, %SPL, 7776; + mov.u32 %r2782, 0; + +$L__BB26_402: + mul.wide.u32 %rd475, %r2782, 2; + add.s64 %rd476, %rd113, %rd475; + mov.u16 %rs978, 0; + st.local.u16 [%rd476], %rs978; + st.local.u16 [%rd476+2], %rs978; + st.local.u16 [%rd476+4], %rs978; + st.local.u16 [%rd476+6], %rs978; + add.s32 %r2782, %r2782, 4; + add.s32 %r2781, %r2781, -4; + setp.ne.s32 %p524, %r2781, 0; + @%p524 bra $L__BB26_402; + + setp.eq.s32 %p525, %r2783, 0; + @%p525 bra $L__BB26_405; + +$L__BB26_404: + .pragma "nounroll"; + mul.wide.u32 %rd477, %r2782, 2; + add.s64 %rd478, %rd113, %rd477; + mov.u16 %rs979, 0; + st.local.u16 [%rd478], %rs979; + add.s32 %r2782, %r2782, 1; + add.s32 %r2783, %r2783, -1; + setp.ne.s32 %p526, %r2783, 0; + @%p526 bra $L__BB26_404; + +$L__BB26_405: + mov.u32 %r1957, 0; + mov.u64 %rd651, 0; + cvt.u64.u32 %rd480, %r1146; + add.s64 %rd114, %rd480, %rd3; + add.s32 %r858, %r669, -2; + mov.u32 %r1958, 3; + shl.b32 %r859, %r1958, %r858; + shl.b32 %r860, %r1151, 1; + mul.lo.s32 %r861, %r1151, 3; + mov.u16 %rs1426, 0; + mov.u32 %r2784, %r1957; + mov.u32 %r2793, %r1957; + mov.u32 %r2794, %r1957; + +$L__BB26_406: + sub.s32 %r1961, %r1144, %r2784; + setp.lt.u32 %p527, %r1961, 4; + setp.lt.u32 %p528, %r1961, 3; + setp.lt.u32 %p529, %r1961, 2; + selp.b32 %r1962, 4369, 13107, %p529; + selp.b32 %r1963, %r1962, 30583, %p528; + selp.b32 %r865, %r1963, 65535, %p527; + shr.u32 %r1964, %r2784, 2; + mul.lo.s32 %r866, %r1964, %r822; + add.s32 %r867, %r866, %r822; + mad.lo.s32 %r868, %r2784, %r1151, %r14; + mov.u32 %r2787, %r1957; + mov.u32 %r2788, %r1957; + +$L__BB26_407: + add.s32 %r1966, %r2787, 4; + sub.s32 %r1967, %r1966, %r1141; + mov.u32 %r2817, 0; + max.s32 %r1968, %r1967, 0; + shl.b32 %r1969, %r1968, 2; + shr.u32 %r1970, %r865, %r1969; + shr.u32 %r873, %r2787, 2; + mul.wide.u32 %rd481, %r873, 2; + add.s64 %rd117, %rd113, %rd481; + ld.local.u16 %rs981, [%rd117]; + ld.local.u16 %rs982, [%rd117+2]; + mov.b32 %r1971, {%rs981, %rs982}; + add.s32 %r1972, %r867, %r873; + mul.wide.u32 %rd482, %r1972, 2; + add.s64 %rd483, %rd112, %rd482; + ld.local.u16 %rs983, [%rd483]; + add.s32 %r1973, %r1972, 1; + mul.wide.u32 %rd484, %r1973, 2; + add.s64 %rd485, %rd112, %rd484; + ld.local.u16 %rs984, [%rd485]; + mov.b32 %r1974, {%rs983, %rs984}; + and.b32 %r1975, %r1971, -2004318072; + shr.u32 %r1976, %r1975, 3; + shl.b32 %r1977, %r1974, 3; + and.b32 %r1978, %r1977, -2004318072; + setp.eq.s32 %p530, %r15, 0; + selp.b32 %r1979, %r1978, 0, %p530; + or.b32 %r874, %r1979, %r1976; + add.s32 %r1980, %r873, %r866; + mul.wide.u32 %rd486, %r1980, 2; + add.s64 %rd487, %rd112, %rd486; + ld.local.u16 %rs985, [%rd487]; + add.s32 %r1981, %r1980, 1; + mul.wide.u32 %rd488, %r1981, 2; + add.s64 %rd489, %rd112, %rd488; + ld.local.u16 %rs986, [%rd489]; + mov.b32 %r875, {%rs985, %rs986}; + shl.b32 %r1982, %r875, 1; + and.b32 %r1983, %r1982, -286331154; + or.b32 %r1984, %r1983, %r875; + and.b32 %r1985, %r875, -286331154; + shr.u32 %r1986, %r1985, 1; + or.b32 %r1987, %r1984, %r1986; + or.b32 %r1988, %r1987, %r874; + shl.b32 %r1989, %r1988, 4; + shr.u32 %r1990, %r1988, 4; + shr.u32 %r1991, %r2788, 12; + or.b32 %r1992, %r1988, %r1991; + or.b32 %r1993, %r1992, %r1989; + or.b32 %r1994, %r1993, %r1990; + not.b32 %r1995, %r875; + and.b32 %r876, %r1970, %r1995; + and.b32 %r877, %r876, %r1994; + setp.eq.s32 %p531, %r877, 0; + @%p531 bra $L__BB26_486; + + setp.gt.u32 %p532, %r2794, 31; + @%p532 bra $L__BB26_412; + +$L__BB26_409: + setp.ge.u32 %p533, %r2793, %r2900; + mov.u16 %rs1424, 0; + @%p533 bra $L__BB26_411; + + add.s32 %r881, %r2793, 1; + cvt.u64.u32 %rd490, %r2793; + add.s64 %rd491, %rd114, %rd490; + add.s64 %rd492, %rd1, %rd491; + ld.global.u8 %rs1424, [%rd492]; + mov.u32 %r2793, %r881; + +$L__BB26_411: + and.b16 %rs988, %rs1424, 255; + cvt.u64.u16 %rd493, %rs1424; + and.b64 %rd494, %rd493, 255; + shl.b64 %rd495, %rd494, %r2794; + or.b64 %rd651, %rd495, %rd651; + cvt.u32.u16 %r1996, %rs1426; + cvt.s32.s8 %r1997, %r1996; + mov.u32 %r1998, 8; + sub.s32 %r1999, %r1998, %r1997; + add.s32 %r2794, %r1999, %r2794; + setp.eq.s16 %p534, %rs988, 255; + selp.u16 %rs1426, 1, 0, %p534; + setp.lt.u32 %p535, %r2794, 33; + @%p535 bra $L__BB26_409; + +$L__BB26_412: + cvt.u32.u64 %r2818, %rd651; + and.b32 %r2001, %r877, 15; + setp.eq.s32 %p536, %r2001, 0; + mov.u32 %r2819, 0; + mov.u32 %r2817, %r877; + @%p536 bra $L__BB26_421; + + and.b32 %r2003, %r877, 1; + setp.eq.b32 %p537, %r2003, 1; + mov.pred %p538, 0; + xor.pred %p539, %p537, %p538; + not.pred %p540, %p539; + mov.u32 %r2819, 0; + mov.u32 %r2817, %r877; + @%p540 bra $L__BB26_415; + + and.b32 %r2005, %r877, -2; + and.b32 %r2006, %r2818, 1; + neg.s32 %r2007, %r2006; + mov.u32 %r2819, 1; + and.b32 %r2008, %r2007, %r876; + and.b32 %r2009, %r2008, 51; + or.b32 %r2817, %r2009, %r2005; + shr.u32 %r2818, %r2818, 1; + +$L__BB26_415: + and.b32 %r2010, %r2817, 2; + setp.eq.s32 %p541, %r2010, 0; + @%p541 bra $L__BB26_417; + + and.b32 %r2011, %r2817, -3; + and.b32 %r2012, %r2818, 1; + neg.s32 %r2013, %r2012; + and.b32 %r2014, %r2013, %r876; + and.b32 %r2015, %r2014, 118; + or.b32 %r2817, %r2015, %r2011; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_417: + and.b32 %r2016, %r2817, 4; + setp.eq.s32 %p542, %r2016, 0; + @%p542 bra $L__BB26_419; + + and.b32 %r2017, %r2817, -5; + and.b32 %r2018, %r2818, 1; + neg.s32 %r2019, %r2018; + and.b32 %r2020, %r2019, %r876; + and.b32 %r2021, %r2020, 236; + or.b32 %r2817, %r2021, %r2017; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_419: + and.b32 %r2022, %r2817, 8; + setp.eq.s32 %p543, %r2022, 0; + @%p543 bra $L__BB26_421; + + and.b32 %r2023, %r2817, -9; + and.b32 %r2024, %r2818, 1; + neg.s32 %r2025, %r2024; + and.b32 %r2026, %r2025, %r876; + and.b32 %r2027, %r2026, 200; + or.b32 %r2817, %r2027, %r2023; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_421: + and.b32 %r2028, %r2817, 240; + setp.eq.s32 %p544, %r2028, 0; + @%p544 bra $L__BB26_430; + + and.b32 %r2029, %r2817, 16; + setp.eq.s32 %p545, %r2029, 0; + @%p545 bra $L__BB26_424; + + and.b32 %r2030, %r2817, -17; + and.b32 %r2031, %r2818, 1; + neg.s32 %r2032, %r2031; + and.b32 %r2033, %r2032, %r876; + and.b32 %r2034, %r2033, 816; + or.b32 %r2817, %r2034, %r2030; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_424: + and.b32 %r2035, %r2817, 32; + setp.eq.s32 %p546, %r2035, 0; + @%p546 bra $L__BB26_426; + + and.b32 %r2036, %r2817, -33; + and.b32 %r2037, %r2818, 1; + neg.s32 %r2038, %r2037; + and.b32 %r2039, %r2038, %r876; + and.b32 %r2040, %r2039, 1888; + or.b32 %r2817, %r2040, %r2036; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_426: + and.b32 %r2041, %r2817, 64; + setp.eq.s32 %p547, %r2041, 0; + @%p547 bra $L__BB26_428; + + and.b32 %r2042, %r2817, -65; + and.b32 %r2043, %r2818, 1; + neg.s32 %r2044, %r2043; + and.b32 %r2045, %r2044, %r876; + and.b32 %r2046, %r2045, 3776; + or.b32 %r2817, %r2046, %r2042; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_428: + and.b32 %r2047, %r2817, 128; + setp.eq.s32 %p548, %r2047, 0; + @%p548 bra $L__BB26_430; + + and.b32 %r2048, %r2817, -129; + and.b32 %r2049, %r2818, 1; + neg.s32 %r2050, %r2049; + and.b32 %r2051, %r2050, %r876; + and.b32 %r2052, %r2051, 3200; + or.b32 %r2817, %r2052, %r2048; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_430: + and.b32 %r2053, %r2817, 3840; + setp.eq.s32 %p549, %r2053, 0; + @%p549 bra $L__BB26_439; + + and.b32 %r2054, %r2817, 256; + setp.eq.s32 %p550, %r2054, 0; + @%p550 bra $L__BB26_433; + + and.b32 %r2055, %r2817, -257; + and.b32 %r2056, %r2818, 1; + neg.s32 %r2057, %r2056; + and.b32 %r2058, %r2057, %r876; + and.b32 %r2059, %r2058, 13056; + or.b32 %r2817, %r2059, %r2055; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_433: + and.b32 %r2060, %r2817, 512; + setp.eq.s32 %p551, %r2060, 0; + @%p551 bra $L__BB26_435; + + and.b32 %r2061, %r2817, -513; + and.b32 %r2062, %r2818, 1; + neg.s32 %r2063, %r2062; + and.b32 %r2064, %r2063, %r876; + and.b32 %r2065, %r2064, 30208; + or.b32 %r2817, %r2065, %r2061; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_435: + and.b32 %r2066, %r2817, 1024; + setp.eq.s32 %p552, %r2066, 0; + @%p552 bra $L__BB26_437; + + and.b32 %r2067, %r2817, -1025; + and.b32 %r2068, %r2818, 1; + neg.s32 %r2069, %r2068; + and.b32 %r2070, %r2069, %r876; + and.b32 %r2071, %r2070, 60416; + or.b32 %r2817, %r2071, %r2067; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_437: + and.b32 %r2072, %r2817, 2048; + setp.eq.s32 %p553, %r2072, 0; + @%p553 bra $L__BB26_439; + + and.b32 %r2073, %r2817, -2049; + and.b32 %r2074, %r2818, 1; + neg.s32 %r2075, %r2074; + and.b32 %r2076, %r2075, %r876; + and.b32 %r2077, %r2076, 51200; + or.b32 %r2817, %r2077, %r2073; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_439: + and.b32 %r2078, %r2817, 61440; + setp.eq.s32 %p554, %r2078, 0; + @%p554 bra $L__BB26_448; + + and.b32 %r2079, %r2817, 4096; + setp.eq.s32 %p555, %r2079, 0; + @%p555 bra $L__BB26_442; + + and.b32 %r2080, %r2817, -4097; + and.b32 %r2081, %r2818, 1; + neg.s32 %r2082, %r2081; + and.b32 %r2083, %r2082, %r876; + and.b32 %r2084, %r2083, 208896; + or.b32 %r2817, %r2084, %r2080; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_442: + and.b32 %r2085, %r2817, 8192; + setp.eq.s32 %p556, %r2085, 0; + @%p556 bra $L__BB26_444; + + and.b32 %r2086, %r2817, -8193; + and.b32 %r2087, %r2818, 1; + neg.s32 %r2088, %r2087; + and.b32 %r2089, %r2088, %r876; + and.b32 %r2090, %r2089, 483328; + or.b32 %r2817, %r2090, %r2086; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_444: + and.b32 %r2091, %r2817, 16384; + setp.eq.s32 %p557, %r2091, 0; + @%p557 bra $L__BB26_446; + + and.b32 %r2092, %r2817, -16385; + and.b32 %r2093, %r2818, 1; + neg.s32 %r2094, %r2093; + and.b32 %r2095, %r2094, %r876; + and.b32 %r2096, %r2095, 966656; + or.b32 %r2817, %r2096, %r2092; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_446: + cvt.u16.u32 %rs989, %r2817; + setp.gt.s16 %p558, %rs989, -1; + @%p558 bra $L__BB26_448; + + and.b32 %r2097, %r2817, -32769; + and.b32 %r2098, %r2818, 1; + neg.s32 %r2099, %r2098; + and.b32 %r2100, %r2099, %r876; + and.b32 %r2101, %r2100, 819200; + or.b32 %r2817, %r2101, %r2097; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_448: + setp.eq.s32 %p559, %r2817, 0; + @%p559 bra $L__BB26_485; + + add.s32 %r982, %r868, %r2787; + and.b32 %r2102, %r2817, 15; + setp.eq.s32 %p560, %r2102, 0; + @%p560 bra $L__BB26_458; + + and.b32 %r2103, %r2817, 1; + setp.eq.b32 %p561, %r2103, 1; + mov.pred %p562, 0; + xor.pred %p563, %p561, %p562; + not.pred %p564, %p563; + @%p564 bra $L__BB26_452; + + shl.b32 %r2104, %r2818, 31; + or.b32 %r2105, %r2104, %r859; + mul.wide.u32 %rd496, %r982, 4; + add.s64 %rd497, %rd2, %rd496; + st.global.u32 [%rd497], %r2105; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_452: + and.b32 %r2106, %r2817, 2; + setp.eq.s32 %p565, %r2106, 0; + @%p565 bra $L__BB26_454; + + shl.b32 %r2107, %r2818, 31; + or.b32 %r2108, %r2107, %r859; + add.s32 %r2109, %r982, %r1151; + mul.wide.u32 %rd498, %r2109, 4; + add.s64 %rd499, %rd2, %rd498; + st.global.u32 [%rd499], %r2108; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_454: + and.b32 %r2110, %r2817, 4; + setp.eq.s32 %p566, %r2110, 0; + @%p566 bra $L__BB26_456; + + shl.b32 %r2111, %r2818, 31; + or.b32 %r2112, %r2111, %r859; + add.s32 %r2113, %r982, %r860; + mul.wide.u32 %rd500, %r2113, 4; + add.s64 %rd501, %rd2, %rd500; + st.global.u32 [%rd501], %r2112; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_456: + and.b32 %r2114, %r2817, 8; + setp.eq.s32 %p567, %r2114, 0; + @%p567 bra $L__BB26_458; + + shl.b32 %r2115, %r2818, 31; + or.b32 %r2116, %r2115, %r859; + add.s32 %r2117, %r982, %r861; + mul.wide.u32 %rd502, %r2117, 4; + add.s64 %rd503, %rd2, %rd502; + st.global.u32 [%rd503], %r2116; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_458: + add.s32 %r999, %r982, 1; + and.b32 %r2118, %r2817, 240; + setp.eq.s32 %p568, %r2118, 0; + @%p568 bra $L__BB26_467; + + and.b32 %r2119, %r2817, 16; + setp.eq.s32 %p569, %r2119, 0; + @%p569 bra $L__BB26_461; + + shl.b32 %r2120, %r2818, 31; + or.b32 %r2121, %r2120, %r859; + mul.wide.u32 %rd504, %r999, 4; + add.s64 %rd505, %rd2, %rd504; + st.global.u32 [%rd505], %r2121; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_461: + and.b32 %r2122, %r2817, 32; + setp.eq.s32 %p570, %r2122, 0; + @%p570 bra $L__BB26_463; + + shl.b32 %r2123, %r2818, 31; + or.b32 %r2124, %r2123, %r859; + add.s32 %r2125, %r999, %r1151; + mul.wide.u32 %rd506, %r2125, 4; + add.s64 %rd507, %rd2, %rd506; + st.global.u32 [%rd507], %r2124; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_463: + and.b32 %r2126, %r2817, 64; + setp.eq.s32 %p571, %r2126, 0; + @%p571 bra $L__BB26_465; + + shl.b32 %r2127, %r2818, 31; + or.b32 %r2128, %r2127, %r859; + add.s32 %r2129, %r999, %r860; + mul.wide.u32 %rd508, %r2129, 4; + add.s64 %rd509, %rd2, %rd508; + st.global.u32 [%rd509], %r2128; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_465: + and.b32 %r2130, %r2817, 128; + setp.eq.s32 %p572, %r2130, 0; + @%p572 bra $L__BB26_467; + + shl.b32 %r2131, %r2818, 31; + or.b32 %r2132, %r2131, %r859; + add.s32 %r2133, %r999, %r861; + mul.wide.u32 %rd510, %r2133, 4; + add.s64 %rd511, %rd2, %rd510; + st.global.u32 [%rd511], %r2132; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_467: + add.s32 %r1016, %r982, 2; + and.b32 %r2134, %r2817, 3840; + setp.eq.s32 %p573, %r2134, 0; + @%p573 bra $L__BB26_476; + + and.b32 %r2135, %r2817, 256; + setp.eq.s32 %p574, %r2135, 0; + @%p574 bra $L__BB26_470; + + shl.b32 %r2136, %r2818, 31; + or.b32 %r2137, %r2136, %r859; + mul.wide.u32 %rd512, %r1016, 4; + add.s64 %rd513, %rd2, %rd512; + st.global.u32 [%rd513], %r2137; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_470: + and.b32 %r2138, %r2817, 512; + setp.eq.s32 %p575, %r2138, 0; + @%p575 bra $L__BB26_472; + + shl.b32 %r2139, %r2818, 31; + or.b32 %r2140, %r2139, %r859; + add.s32 %r2141, %r1016, %r1151; + mul.wide.u32 %rd514, %r2141, 4; + add.s64 %rd515, %rd2, %rd514; + st.global.u32 [%rd515], %r2140; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_472: + and.b32 %r2142, %r2817, 1024; + setp.eq.s32 %p576, %r2142, 0; + @%p576 bra $L__BB26_474; + + shl.b32 %r2143, %r2818, 31; + or.b32 %r2144, %r2143, %r859; + add.s32 %r2145, %r1016, %r860; + mul.wide.u32 %rd516, %r2145, 4; + add.s64 %rd517, %rd2, %rd516; + st.global.u32 [%rd517], %r2144; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_474: + and.b32 %r2146, %r2817, 2048; + setp.eq.s32 %p577, %r2146, 0; + @%p577 bra $L__BB26_476; + + shl.b32 %r2147, %r2818, 31; + or.b32 %r2148, %r2147, %r859; + add.s32 %r2149, %r1016, %r861; + mul.wide.u32 %rd518, %r2149, 4; + add.s64 %rd519, %rd2, %rd518; + st.global.u32 [%rd519], %r2148; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_476: + add.s32 %r1033, %r982, 3; + and.b32 %r2150, %r2817, 61440; + setp.eq.s32 %p578, %r2150, 0; + @%p578 bra $L__BB26_485; + + and.b32 %r2151, %r2817, 4096; + setp.eq.s32 %p579, %r2151, 0; + @%p579 bra $L__BB26_479; + + shl.b32 %r2152, %r2818, 31; + or.b32 %r2153, %r2152, %r859; + mul.wide.u32 %rd520, %r1033, 4; + add.s64 %rd521, %rd2, %rd520; + st.global.u32 [%rd521], %r2153; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_479: + and.b32 %r2154, %r2817, 8192; + setp.eq.s32 %p580, %r2154, 0; + @%p580 bra $L__BB26_481; + + shl.b32 %r2155, %r2818, 31; + or.b32 %r2156, %r2155, %r859; + add.s32 %r2157, %r1033, %r1151; + mul.wide.u32 %rd522, %r2157, 4; + add.s64 %rd523, %rd2, %rd522; + st.global.u32 [%rd523], %r2156; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_481: + and.b32 %r2158, %r2817, 16384; + setp.eq.s32 %p581, %r2158, 0; + @%p581 bra $L__BB26_483; + + shl.b32 %r2159, %r2818, 31; + or.b32 %r2160, %r2159, %r859; + add.s32 %r2161, %r1033, %r860; + mul.wide.u32 %rd524, %r2161, 4; + add.s64 %rd525, %rd2, %rd524; + st.global.u32 [%rd525], %r2160; + shr.u32 %r2818, %r2818, 1; + add.s32 %r2819, %r2819, 1; + +$L__BB26_483: + cvt.u16.u32 %rs990, %r2817; + setp.gt.s16 %p582, %rs990, -1; + @%p582 bra $L__BB26_485; + + shl.b32 %r2162, %r2818, 31; + or.b32 %r2163, %r2162, %r859; + add.s32 %r2164, %r1033, %r861; + mul.wide.u32 %rd526, %r2164, 4; + add.s64 %rd527, %rd2, %rd526; + st.global.u32 [%rd527], %r2163; + add.s32 %r2819, %r2819, 1; + +$L__BB26_485: + shr.u64 %rd651, %rd651, %r2819; + sub.s32 %r2794, %r2794, %r2819; + +$L__BB26_486: + or.b32 %r1052, %r2817, %r875; + st.local.u16 [%rd117], %r1052; + add.s32 %r2165, %r873, 1; + setp.ge.u32 %p583, %r2165, %r823; + @%p583 bra $L__BB26_488; + + shr.u32 %r2166, %r1052, 16; + st.local.u16 [%rd117+2], %r2166; + +$L__BB26_488: + shl.b32 %r2167, %r1052, 1; + and.b32 %r2168, %r2167, 57344; + and.b32 %r2169, %r1052, 57344; + shr.u32 %r2170, %r2169, 1; + or.b32 %r2171, %r1052, %r874; + and.b32 %r2172, %r2171, 61440; + or.b32 %r2173, %r2172, %r2168; + or.b32 %r2788, %r2173, %r2170; + setp.lt.u32 %p584, %r1966, %r1141; + mov.u32 %r2787, %r1966; + @%p584 bra $L__BB26_407; + + add.s32 %r2784, %r2784, 4; + setp.gt.u32 %p585, %r1144, %r2784; + @%p585 bra $L__BB26_406; + + setp.lt.u32 %p586, %r17, 3; + @%p586 bra $L__BB26_541; + + mov.u32 %r2174, 0; + mov.u32 %r2878, %r2174; + +$L__BB26_492: + shr.u32 %r2176, %r2878, 1; + mul.lo.s32 %r2880, %r2176, %r1163; + shr.u32 %r2177, %r2878, 2; + mul.lo.s32 %r2879, %r2177, %r822; + mov.u32 %r2881, %r2174; + +$L__BB26_493: + mul.wide.u32 %rd528, %r2880, 2; + add.s64 %rd529, %rd13, %rd528; + ld.local.u16 %rs991, [%rd529]; + and.b16 %rs992, %rs991, 48; + shr.u16 %rs993, %rs992, 4; + and.b16 %rs994, %rs991, 192; + shr.u16 %rs995, %rs994, 2; + add.s32 %r2178, %r2880, 2; + mul.wide.u32 %rd530, %r2178, 2; + add.s64 %rd531, %rd13, %rd530; + ld.local.u16 %rs996, [%rd531]; + shl.b16 %rs997, %rs996, 4; + and.b16 %rs998, %rs997, 768; + shl.b16 %rs999, %rs996, 6; + and.b16 %rs1000, %rs999, 12288; + add.s32 %r2179, %r2880, %r1163; + mul.wide.u32 %rd532, %r2179, 2; + add.s64 %rd533, %rd13, %rd532; + ld.local.u16 %rs1001, [%rd533]; + and.b16 %rs1002, %rs1001, 48; + shr.u16 %rs1003, %rs1002, 2; + and.b16 %rs1004, %rs1001, 192; + add.s32 %r2180, %r2179, 2; + mul.wide.u32 %rd534, %r2180, 2; + add.s64 %rd535, %rd13, %rd534; + ld.local.u16 %rs1005, [%rd535]; + shl.b16 %rs1006, %rs1005, 6; + and.b16 %rs1007, %rs1006, 3072; + shl.b16 %rs1008, %rs1005, 8; + and.b16 %rs1009, %rs1008, -16384; + or.b16 %rs1010, %rs993, %rs995; + or.b16 %rs1011, %rs1010, %rs1000; + or.b16 %rs1012, %rs1011, %rs998; + or.b16 %rs1013, %rs1012, %rs1004; + or.b16 %rs1014, %rs1013, %rs1003; + or.b16 %rs1015, %rs1014, %rs1009; + or.b16 %rs1016, %rs1015, %rs1007; + mul.wide.u32 %rd536, %r2879, 2; + add.s64 %rd537, %rd112, %rd536; + st.local.u16 [%rd537], %rs1016; + add.s32 %r2880, %r2880, 4; + add.s32 %r2879, %r2879, 1; + add.s32 %r2881, %r2881, 4; + setp.lt.u32 %p587, %r2881, %r1141; + @%p587 bra $L__BB26_493; + + mul.wide.u32 %rd538, %r2879, 2; + add.s64 %rd539, %rd112, %rd538; + mov.u16 %rs1017, 0; + st.local.u16 [%rd539], %rs1017; + add.s32 %r2878, %r2878, 4; + setp.lt.u32 %p588, %r2878, %r1144; + @%p588 bra $L__BB26_492; + + mov.u32 %r2884, 0; + @%p520 bra $L__BB26_498; + + sub.s32 %r2883, %r836, %r2886; + mov.u32 %r2884, 0; + +$L__BB26_497: + add.s32 %r2183, %r2884, %r835; + mul.wide.u32 %rd540, %r2183, 2; + add.s64 %rd541, %rd112, %rd540; + mov.u16 %rs1018, 0; + st.local.v4.u16 [%rd541], {%rs1018, %rs1018, %rs1018, %rs1018}; + add.s32 %r2884, %r2884, 4; + add.s32 %r2883, %r2883, -4; + setp.ne.s32 %p590, %r2883, 0; + @%p590 bra $L__BB26_497; + +$L__BB26_498: + @%p522 bra $L__BB26_500; + +$L__BB26_499: + .pragma "nounroll"; + add.s32 %r2184, %r2884, %r835; + mul.wide.u32 %rd542, %r2184, 2; + add.s64 %rd543, %rd112, %rd542; + mov.u16 %rs1019, 0; + st.local.u16 [%rd543], %rs1019; + add.s32 %r2884, %r2884, 1; + add.s32 %r2886, %r2886, -1; + setp.ne.s32 %p592, %r2886, 0; + @%p592 bra $L__BB26_499; + +$L__BB26_500: + mov.u32 %r2187, 1; + mov.u32 %r2186, 0; + mov.u64 %rd656, 0; + add.s32 %r2188, %r1146, %r2900; + add.s32 %r2901, %r2188, -1; + shl.b32 %r1078, %r2187, %r858; + mov.u16 %rs1431, 1; + mov.u32 %r2887, %r2186; + mov.u32 %r2899, %r2186; + +$L__BB26_501: + shr.u32 %r2190, %r2887, 2; + mul.lo.s32 %r2892, %r2190, %r822; + mad.lo.s32 %r1084, %r2887, %r1151, %r14; + mov.u32 %r2891, %r2186; + +$L__BB26_502: + setp.gt.u32 %p593, %r2899, 31; + @%p593 bra $L__BB26_507; + + mov.u32 %r2897, %r2900; + +$L__BB26_504: + setp.eq.s32 %p594, %r2897, 0; + mov.u16 %rs1430, 0; + @%p594 bra $L__BB26_506; + + cvt.s64.s32 %rd545, %r2901; + add.s64 %rd546, %rd545, %rd3; + add.s64 %rd547, %rd1, %rd546; + ld.global.u8 %rs1430, [%rd547]; + +$L__BB26_506: + add.s32 %r2191, %r2897, -1; + selp.b32 %r2900, 0, %r2191, %p594; + setp.ne.s32 %p596, %r2897, 0; + selp.b32 %r2192, -1, 0, %p596; + add.s32 %r2901, %r2901, %r2192; + and.b16 %rs1022, %rs1430, 255; + and.b16 %rs1023, %rs1430, 127; + setp.eq.s16 %p597, %rs1023, 127; + and.b16 %rs1024, %rs1431, 255; + setp.ne.s16 %p598, %rs1024, 0; + and.pred %p599, %p598, %p597; + selp.b32 %r2193, 7, 8, %p599; + cvt.u64.u16 %rd548, %rs1430; + and.b64 %rd549, %rd548, 255; + shl.b64 %rd550, %rd549, %r2899; + or.b64 %rd656, %rd550, %rd656; + add.s32 %r2899, %r2193, %r2899; + setp.gt.u16 %p600, %rs1022, 143; + selp.u16 %rs1431, 1, 0, %p600; + setp.lt.u32 %p601, %r2899, 33; + mov.u32 %r2897, %r2900; + @%p601 bra $L__BB26_504; + +$L__BB26_507: + mul.wide.u32 %rd551, %r2892, 2; + add.s64 %rd552, %rd112, %rd551; + ld.local.u32 %r1099, [%rd552]; + setp.eq.s32 %p602, %r1099, 0; + @%p602 bra $L__BB26_528; + + cvt.u32.u64 %r2912, %rd656; + add.s32 %r1101, %r1084, %r2891; + mov.u32 %r2904, 15; + mov.u32 %r2902, 0; + +$L__BB26_509: + and.b32 %r2196, %r2904, %r1099; + setp.eq.s32 %p603, %r2196, 0; + @%p603 bra $L__BB26_518; + + add.s32 %r1105, %r1101, %r2902; + and.b32 %r1106, %r2904, 286331137; + and.b32 %r2197, %r1106, %r1099; + setp.eq.s32 %p604, %r2197, 0; + @%p604 bra $L__BB26_512; + + not.b32 %r2198, %r2912; + and.b32 %r2199, %r2198, 1; + shl.b32 %r2200, %r2199, %r671; + or.b32 %r2201, %r2200, %r1078; + mul.wide.u32 %rd553, %r1105, 4; + add.s64 %rd554, %rd2, %rd553; + ld.global.u32 %r2202, [%rd554]; + xor.b32 %r2203, %r2202, %r2201; + st.global.u32 [%rd554], %r2203; + shr.u32 %r2912, %r2912, 1; + +$L__BB26_512: + add.s32 %r1109, %r1105, %r1151; + shl.b32 %r2204, %r1106, 1; + and.b32 %r2205, %r2204, %r1099; + setp.eq.s32 %p605, %r2205, 0; + @%p605 bra $L__BB26_514; + + not.b32 %r2206, %r2912; + and.b32 %r2207, %r2206, 1; + shl.b32 %r2208, %r2207, %r671; + or.b32 %r2209, %r2208, %r1078; + mul.wide.u32 %rd555, %r1109, 4; + add.s64 %rd556, %rd2, %rd555; + ld.global.u32 %r2210, [%rd556]; + xor.b32 %r2211, %r2210, %r2209; + st.global.u32 [%rd556], %r2211; + shr.u32 %r2912, %r2912, 1; + +$L__BB26_514: + add.s32 %r1112, %r1109, %r1151; + shl.b32 %r2212, %r1106, 2; + and.b32 %r2213, %r2212, %r1099; + setp.eq.s32 %p606, %r2213, 0; + @%p606 bra $L__BB26_516; + + not.b32 %r2214, %r2912; + and.b32 %r2215, %r2214, 1; + shl.b32 %r2216, %r2215, %r671; + or.b32 %r2217, %r2216, %r1078; + mul.wide.u32 %rd557, %r1112, 4; + add.s64 %rd558, %rd2, %rd557; + ld.global.u32 %r2218, [%rd558]; + xor.b32 %r2219, %r2218, %r2217; + st.global.u32 [%rd558], %r2219; + shr.u32 %r2912, %r2912, 1; + +$L__BB26_516: + shl.b32 %r2220, %r1106, 3; + and.b32 %r2221, %r2220, %r1099; + setp.eq.s32 %p607, %r2221, 0; + @%p607 bra $L__BB26_518; + + not.b32 %r2222, %r2912; + and.b32 %r2223, %r2222, 1; + shl.b32 %r2224, %r2223, %r671; + or.b32 %r2225, %r2224, %r1078; + add.s32 %r2226, %r1112, %r1151; + mul.wide.u32 %rd559, %r2226, 4; + add.s64 %rd560, %rd2, %rd559; + ld.global.u32 %r2227, [%rd560]; + xor.b32 %r2228, %r2227, %r2225; + st.global.u32 [%rd560], %r2228; + shr.u32 %r2912, %r2912, 1; + +$L__BB26_518: + shl.b32 %r1117, %r2904, 4; + and.b32 %r2229, %r1117, %r1099; + setp.eq.s32 %p608, %r2229, 0; + @%p608 bra $L__BB26_527; + + add.s32 %r2230, %r1101, %r2902; + add.s32 %r1118, %r2230, 1; + and.b32 %r1119, %r1117, 286330896; + and.b32 %r2231, %r1119, %r1099; + setp.eq.s32 %p609, %r2231, 0; + @%p609 bra $L__BB26_521; + + not.b32 %r2232, %r2912; + and.b32 %r2233, %r2232, 1; + shl.b32 %r2234, %r2233, %r671; + or.b32 %r2235, %r2234, %r1078; + mul.wide.u32 %rd561, %r1118, 4; + add.s64 %rd562, %rd2, %rd561; + ld.global.u32 %r2236, [%rd562]; + xor.b32 %r2237, %r2236, %r2235; + st.global.u32 [%rd562], %r2237; + shr.u32 %r2912, %r2912, 1; + +$L__BB26_521: + add.s32 %r1122, %r1118, %r1151; + shl.b32 %r2238, %r1119, 1; + and.b32 %r2239, %r2238, %r1099; + setp.eq.s32 %p610, %r2239, 0; + @%p610 bra $L__BB26_523; + + not.b32 %r2240, %r2912; + and.b32 %r2241, %r2240, 1; + shl.b32 %r2242, %r2241, %r671; + or.b32 %r2243, %r2242, %r1078; + mul.wide.u32 %rd563, %r1122, 4; + add.s64 %rd564, %rd2, %rd563; + ld.global.u32 %r2244, [%rd564]; + xor.b32 %r2245, %r2244, %r2243; + st.global.u32 [%rd564], %r2245; + shr.u32 %r2912, %r2912, 1; + +$L__BB26_523: + add.s32 %r1125, %r1122, %r1151; + shl.b32 %r2246, %r1119, 2; + and.b32 %r2247, %r2246, %r1099; + setp.eq.s32 %p611, %r2247, 0; + @%p611 bra $L__BB26_525; + + not.b32 %r2248, %r2912; + and.b32 %r2249, %r2248, 1; + shl.b32 %r2250, %r2249, %r671; + or.b32 %r2251, %r2250, %r1078; + mul.wide.u32 %rd565, %r1125, 4; + add.s64 %rd566, %rd2, %rd565; + ld.global.u32 %r2252, [%rd566]; + xor.b32 %r2253, %r2252, %r2251; + st.global.u32 [%rd566], %r2253; + shr.u32 %r2912, %r2912, 1; + +$L__BB26_525: + shl.b32 %r2254, %r1119, 3; + and.b32 %r2255, %r2254, %r1099; + setp.eq.s32 %p612, %r2255, 0; + @%p612 bra $L__BB26_527; + + not.b32 %r2256, %r2912; + and.b32 %r2257, %r2256, 1; + shl.b32 %r2258, %r2257, %r671; + or.b32 %r2259, %r2258, %r1078; + add.s32 %r2260, %r1125, %r1151; + mul.wide.u32 %rd567, %r2260, 4; + add.s64 %rd568, %rd2, %rd567; + ld.global.u32 %r2261, [%rd568]; + xor.b32 %r2262, %r2261, %r2259; + st.global.u32 [%rd568], %r2262; + shr.u32 %r2912, %r2912, 1; + +$L__BB26_527: + shl.b32 %r2904, %r2904, 8; + add.s32 %r2902, %r2902, 2; + setp.ne.s32 %p613, %r2902, 8; + @%p613 bra $L__BB26_509; + +$L__BB26_528: + popc.b32 %r2263, %r1099; + shr.u64 %rd656, %rd656, %r2263; + sub.s32 %r2899, %r2899, %r2263; + add.s32 %r2891, %r2891, 8; + setp.lt.u32 %p614, %r2891, %r1141; + add.s32 %r2892, %r2892, 2; + @%p614 bra $L__BB26_502; + + add.s32 %r2887, %r2887, 4; + setp.lt.u32 %p615, %r2887, %r1144; + @%p615 bra $L__BB26_501; + bra.uni $L__BB26_541; + +} + // .globl j2k_htj2k_decode_codeblocks_multi_cleanup_only +.visible .entry j2k_htj2k_decode_codeblocks_multi_cleanup_only( + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_0, + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_1, + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_2, + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_3, + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_4, + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_5, + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_6, + .param .u32 j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_7 +) +{ + .local .align 16 .b8 __local_depot27[6720]; + .reg .b64 %SP; + .reg .b64 %SPL; + .reg .pred %p<507>; + .reg .b16 %rs<1330>; + .reg .b32 %r<2113>; + .reg .b64 %rd<510>; + + + mov.u64 %SPL, __local_depot27; + ld.param.u64 %rd118, [ j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_0]; + ld.param.u64 %rd112, [ j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_1]; + ld.param.u64 %rd117, [ j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_6]; + ld.param.u32 %r815, [ j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_7]; + cvta.to.global.u64 %rd1, %rd118; + mov.u32 %r816, %ntid.x; + mov.u32 %r817, %ctaid.x; + mov.u32 %r818, %tid.x; + mad.lo.s32 %r1, %r817, %r816, %r818; + setp.ge.u32 %p1, %r1, %r815; + @%p1 bra $L__BB27_399; + + cvta.to.global.u64 %rd119, %rd117; + cvta.to.global.u64 %rd120, %rd112; + mul.wide.u32 %rd121, %r1, 64; + add.s64 %rd122, %rd120, %rd121; + ld.global.u64 %rd123, [%rd122]; + cvta.to.global.u64 %rd2, %rd123; + ld.global.v2.u32 {%r819, %r820}, [%rd122+8]; + mov.u32 %r821, 0; + ld.global.v2.u32 {%r822, %r823}, [%rd122+16]; + ld.global.v2.u32 {%r824, %r825}, [%rd122+24]; + ld.global.v2.u32 {%r827, %r828}, [%rd122+32]; + ld.global.u32 %r12, [%rd122+44]; + ld.global.u32 %r13, [%rd122+48]; + cvt.u64.u32 %rd3, %r819; + mul.wide.u32 %rd124, %r1, 16; + add.s64 %rd4, %rd119, %rd124; + st.global.u32 [%rd4], %r821; + st.global.u32 [%rd4+4], %r821; + st.global.u32 [%rd4+8], %r821; + st.global.u32 [%rd4+12], %r821; + setp.eq.s32 %p2, %r825, 0; + @%p2 bra $L__BB27_3; + + mov.u32 %r829, 2; + st.global.u32 [%rd4], %r829; + mov.u32 %r830, 17; + st.global.u32 [%rd4+4], %r830; + st.global.u32 [%rd4+8], %r821; + st.global.u32 [%rd4+12], %r821; + bra.uni $L__BB27_399; + +$L__BB27_3: + setp.eq.s32 %p3, %r820, 0; + setp.eq.s32 %p4, %r822, 0; + or.pred %p5, %p3, %p4; + @%p5 bra $L__BB27_399; + + setp.gt.u32 %p6, %r820, 256; + setp.gt.u32 %p7, %r822, 256; + or.pred %p8, %p6, %p7; + mul.lo.s32 %r832, %r822, %r820; + setp.gt.u32 %p9, %r832, 4096; + or.pred %p10, %p8, %p9; + @%p10 bra $L__BB27_398; + bra.uni $L__BB27_5; + +$L__BB27_398: + mov.u32 %r1660, 2; + st.global.u32 [%rd4], %r1660; + mov.u32 %r1661, 1; + st.global.u32 [%rd4+4], %r1661; + mov.u32 %r1662, 0; + st.global.u32 [%rd4+8], %r1662; + st.global.u32 [%rd4+12], %r1662; + bra.uni $L__BB27_399; + +$L__BB27_5: + add.s32 %r833, %r828, -1; + setp.gt.u32 %p11, %r833, 30; + @%p11 bra $L__BB27_397; + bra.uni $L__BB27_6; + +$L__BB27_397: + mov.u32 %r1657, 1; + st.global.u32 [%rd4], %r1657; + mov.u32 %r1658, 2; + st.global.u32 [%rd4+4], %r1658; + mov.u32 %r1659, 0; + st.global.u32 [%rd4+8], %r1659; + st.global.u32 [%rd4+12], %r1659; + bra.uni $L__BB27_399; + +$L__BB27_6: + setp.gt.u32 %p12, %r827, 29; + @%p12 bra $L__BB27_396; + bra.uni $L__BB27_7; + +$L__BB27_396: + mov.u32 %r1654, 1; + st.global.u32 [%rd4], %r1654; + mov.u32 %r1655, 3; + st.global.u32 [%rd4+4], %r1655; + mov.u32 %r1656, 0; + st.global.u32 [%rd4+8], %r1656; + st.global.u32 [%rd4+12], %r1656; + bra.uni $L__BB27_399; + +$L__BB27_7: + setp.lt.u32 %p13, %r824, 2; + setp.lt.u32 %p14, %r823, %r824; + or.pred %p15, %p13, %p14; + @%p15 bra $L__BB27_395; + bra.uni $L__BB27_8; + +$L__BB27_395: + mov.u32 %r1651, 1; + st.global.u32 [%rd4], %r1651; + mov.u32 %r1652, 4; + st.global.u32 [%rd4+4], %r1652; + mov.u32 %r1653, 0; + st.global.u32 [%rd4+8], %r1653; + st.global.u32 [%rd4+12], %r1653; + bra.uni $L__BB27_399; + +$L__BB27_8: + add.s32 %r834, %r824, -1; + cvt.u64.u32 %rd125, %r834; + add.s64 %rd126, %rd125, %rd3; + add.s64 %rd127, %rd1, %rd126; + ld.global.u8 %rs652, [%rd127]; + mul.wide.u16 %r835, %rs652, 16; + add.s32 %r836, %r824, -2; + cvt.u64.u32 %rd128, %r836; + add.s64 %rd129, %rd128, %rd3; + add.s64 %rd130, %rd1, %rd129; + ld.global.u8 %rs1, [%rd130]; + and.b16 %rs653, %rs1, 15; + cvt.u32.u16 %r837, %rs653; + or.b32 %r14, %r835, %r837; + setp.lt.u32 %p16, %r824, %r14; + add.s32 %r15, %r14, -2; + setp.gt.u32 %p17, %r15, 4077; + or.pred %p18, %p16, %p17; + @%p18 bra $L__BB27_394; + bra.uni $L__BB27_9; + +$L__BB27_394: + mov.u32 %r1648, 1; + st.global.u32 [%rd4], %r1648; + mov.u32 %r1649, 5; + st.global.u32 [%rd4+4], %r1649; + mov.u32 %r1650, 0; + st.global.u32 [%rd4+8], %r1650; + st.global.u32 [%rd4+12], %r1650; + bra.uni $L__BB27_399; + +$L__BB27_9: + add.s32 %r838, %r822, 1; + shr.u32 %r839, %r838, 1; + add.s32 %r840, %r820, 9; + and.b32 %r841, %r840, -8; + setp.gt.u32 %p19, %r841, 264; + add.s32 %r842, %r839, 1; + mul.lo.s32 %r843, %r842, %r841; + setp.gt.u32 %p20, %r843, 3096; + or.pred %p21, %p19, %p20; + @%p21 bra $L__BB27_393; + bra.uni $L__BB27_10; + +$L__BB27_393: + mov.u32 %r1645, 2; + st.global.u32 [%rd4], %r1645; + mov.u32 %r1646, 6; + st.global.u32 [%rd4+4], %r1646; + mov.u32 %r1647, 0; + st.global.u32 [%rd4+8], %r1647; + st.global.u32 [%rd4+12], %r1647; + bra.uni $L__BB27_399; + +$L__BB27_10: + and.b16 %rs940, %rs1, 15; + cvt.u32.u16 %r1669, %rs940; + mul.wide.u16 %r1668, %rs652, 16; + or.b32 %r1667, %r1668, %r1669; + sub.s32 %r16, %r824, %r1667; + add.s32 %r1747, %r1667, -1; + add.s32 %r1915, %r824, -3; + mov.u64 %rd455, 0; + mov.u32 %r1837, 0; + mov.u16 %rs1013, 0; + mov.u64 %rd133, _ZZ20mel_decode_more_runsR10MelDecoderE7MEL_EXP; + mov.u32 %r1746, %r16; + mov.u16 %rs1014, %rs1013; + mov.u16 %rs1048, %rs1013; + mov.u32 %r1719, %r1837; + +$L__BB27_11: + setp.gt.u32 %p23, %r1719, 7; + @%p23 bra $L__BB27_56; + + mul.wide.u32 %rd132, %r1837, 4; + add.s64 %rd134, %rd133, %rd132; + ld.global.nc.u32 %r23, [%rd134]; + and.b16 %rs657, %rs1048, 255; + setp.ne.s16 %p24, %rs657, 0; + mov.u16 %rs948, %rs1048; + @%p24 bra $L__BB27_16; + + setp.eq.s32 %p25, %r1747, 0; + mov.u16 %rs945, 255; + @%p25 bra $L__BB27_15; + + cvt.u64.u32 %rd135, %r1746; + add.s64 %rd136, %rd135, %rd3; + add.s64 %rd137, %rd1, %rd136; + ld.global.u8 %rs945, [%rd137]; + +$L__BB27_15: + setp.ne.s32 %p27, %r1747, 0; + selp.u32 %r846, 1, 0, %p27; + add.s32 %r1746, %r1746, %r846; + add.s32 %r847, %r1747, -1; + selp.b32 %r1747, 0, %r847, %p25; + setp.eq.s32 %p28, %r1747, 0; + or.b16 %rs659, %rs945, 15; + selp.b16 %rs1014, %rs659, %rs945, %p28; + and.b16 %rs660, %rs1014, 255; + mov.u16 %rs661, 8; + sub.s16 %rs948, %rs661, %rs1013; + setp.eq.s16 %p29, %rs660, 255; + selp.u16 %rs1013, 1, 0, %p29; + +$L__BB27_16: + add.s16 %rs15, %rs948, -1; + cvt.u32.u16 %r848, %rs15; + and.b32 %r849, %r848, 255; + mov.u32 %r850, 1; + shl.b32 %r851, %r850, %r849; + cvt.u32.u16 %r852, %rs1014; + and.b32 %r853, %r851, %r852; + and.b32 %r28, %r853, 255; + add.s16 %rs1048, %rs948, -1; + setp.eq.s32 %p30, %r28, 0; + @%p30 bra $L__BB27_18; + + add.s32 %r854, %r1837, 1; + min.u32 %r1714, %r854, 12; + mov.u32 %r855, -1; + shl.b32 %r856, %r855, %r23; + shl.b32 %r857, %r856, 1; + xor.b32 %r1715, %r857, -2; + bra.uni $L__BB27_55; + +$L__BB27_18: + cvt.u64.u32 %rd453, %r1837; + add.s64 %rd138, %rd453, -3; + setp.gt.u64 %p31, %rd138, 9; + mov.u32 %r1711, 0; + @%p31 bra $L__BB27_54; + + add.s16 %rs1048, %rs948, -1; + max.u32 %r31, %r23, 1; + add.s32 %r861, %r31, -1; + setp.lt.u32 %p32, %r861, 3; + mov.u32 %r1711, 0; + @%p32 bra $L__BB27_38; + + and.b32 %r1663, %r31, 3; + add.s16 %rs1048, %rs948, -1; + sub.s32 %r1683, %r31, %r1663; + mov.u32 %r1711, 0; + +$L__BB27_21: + and.b16 %rs663, %rs1048, 255; + setp.ne.s16 %p33, %rs663, 0; + @%p33 bra $L__BB27_25; + + setp.eq.s32 %p34, %r1747, 0; + mov.u16 %rs952, 255; + @%p34 bra $L__BB27_24; + + cvt.u64.u32 %rd139, %r1746; + add.s64 %rd140, %rd139, %rd3; + add.s64 %rd141, %rd1, %rd140; + ld.global.u8 %rs952, [%rd141]; + +$L__BB27_24: + setp.ne.s32 %p36, %r1747, 0; + selp.u32 %r863, 1, 0, %p36; + add.s32 %r1746, %r1746, %r863; + add.s32 %r864, %r1747, -1; + selp.b32 %r1747, 0, %r864, %p34; + setp.eq.s32 %p37, %r1747, 0; + or.b16 %rs665, %rs952, 15; + selp.b16 %rs1014, %rs665, %rs952, %p37; + and.b16 %rs666, %rs1014, 255; + mov.u16 %rs667, 8; + sub.s16 %rs1048, %rs667, %rs1013; + setp.eq.s16 %p38, %rs666, 255; + selp.u16 %rs1013, 1, 0, %p38; + +$L__BB27_25: + add.s16 %rs959, %rs1048, -1; + and.b16 %rs668, %rs959, 255; + cvt.u32.u16 %r865, %rs959; + and.b32 %r866, %r865, 255; + cvt.u32.u16 %r867, %rs1014; + and.b32 %r1689, %r867, 255; + shr.u32 %r868, %r1689, %r866; + and.b32 %r869, %r868, 1; + bfi.b32 %r43, %r1711, %r869, 1, 31; + setp.ne.s16 %p39, %rs668, 0; + @%p39 bra $L__BB27_29; + + setp.eq.s32 %p40, %r1747, 0; + mov.u16 %rs956, 255; + @%p40 bra $L__BB27_28; + + cvt.u64.u32 %rd142, %r1746; + add.s64 %rd143, %rd142, %rd3; + add.s64 %rd144, %rd1, %rd143; + ld.global.u8 %rs956, [%rd144]; + +$L__BB27_28: + setp.ne.s32 %p42, %r1747, 0; + selp.u32 %r870, 1, 0, %p42; + add.s32 %r1746, %r1746, %r870; + add.s32 %r871, %r1747, -1; + selp.b32 %r1747, 0, %r871, %p40; + setp.eq.s32 %p43, %r1747, 0; + or.b16 %rs670, %rs956, 15; + selp.b16 %rs1014, %rs670, %rs956, %p43; + and.b16 %rs671, %rs1014, 255; + mov.u16 %rs672, 8; + sub.s16 %rs959, %rs672, %rs1013; + setp.eq.s16 %p44, %rs671, 255; + selp.u16 %rs1013, 1, 0, %p44; + cvt.u32.u16 %r872, %rs1014; + and.b32 %r1689, %r872, 255; + +$L__BB27_29: + add.s16 %rs963, %rs959, -1; + and.b16 %rs673, %rs963, 255; + cvt.u32.u16 %r873, %rs963; + and.b32 %r874, %r873, 255; + shr.u32 %r875, %r1689, %r874; + and.b32 %r876, %r875, 1; + bfi.b32 %r50, %r43, %r876, 1, 31; + setp.ne.s16 %p45, %rs673, 0; + @%p45 bra $L__BB27_33; + + setp.eq.s32 %p46, %r1747, 0; + mov.u16 %rs960, 255; + @%p46 bra $L__BB27_32; + + cvt.u64.u32 %rd145, %r1746; + add.s64 %rd146, %rd145, %rd3; + add.s64 %rd147, %rd1, %rd146; + ld.global.u8 %rs960, [%rd147]; + +$L__BB27_32: + setp.ne.s32 %p48, %r1747, 0; + selp.u32 %r877, 1, 0, %p48; + add.s32 %r1746, %r1746, %r877; + add.s32 %r878, %r1747, -1; + selp.b32 %r1747, 0, %r878, %p46; + setp.eq.s32 %p49, %r1747, 0; + or.b16 %rs675, %rs960, 15; + selp.b16 %rs1014, %rs675, %rs960, %p49; + and.b16 %rs676, %rs1014, 255; + mov.u16 %rs677, 8; + sub.s16 %rs963, %rs677, %rs1013; + setp.eq.s16 %p50, %rs676, 255; + selp.u16 %rs1013, 1, 0, %p50; + cvt.u32.u16 %r879, %rs1014; + and.b32 %r1689, %r879, 255; + +$L__BB27_33: + add.s16 %rs967, %rs963, -1; + and.b16 %rs678, %rs967, 255; + cvt.u32.u16 %r880, %rs967; + and.b32 %r881, %r880, 255; + shr.u32 %r882, %r1689, %r881; + and.b32 %r883, %r882, 1; + bfi.b32 %r57, %r50, %r883, 1, 31; + setp.ne.s16 %p51, %rs678, 0; + @%p51 bra $L__BB27_37; + + setp.eq.s32 %p52, %r1747, 0; + mov.u16 %rs964, 255; + @%p52 bra $L__BB27_36; + + cvt.u64.u32 %rd148, %r1746; + add.s64 %rd149, %rd148, %rd3; + add.s64 %rd150, %rd1, %rd149; + ld.global.u8 %rs964, [%rd150]; + +$L__BB27_36: + setp.ne.s32 %p54, %r1747, 0; + selp.u32 %r884, 1, 0, %p54; + add.s32 %r1746, %r1746, %r884; + add.s32 %r885, %r1747, -1; + selp.b32 %r1747, 0, %r885, %p52; + setp.eq.s32 %p55, %r1747, 0; + or.b16 %rs680, %rs964, 15; + selp.b16 %rs1014, %rs680, %rs964, %p55; + and.b16 %rs681, %rs1014, 255; + mov.u16 %rs682, 8; + sub.s16 %rs967, %rs682, %rs1013; + setp.eq.s16 %p56, %rs681, 255; + selp.u16 %rs1013, 1, 0, %p56; + cvt.u32.u16 %r886, %rs1014; + and.b32 %r1689, %r886, 255; + +$L__BB27_37: + add.s16 %rs1048, %rs967, -1; + cvt.u32.u16 %r887, %rs1048; + and.b32 %r888, %r887, 255; + shr.u32 %r889, %r1689, %r888; + and.b32 %r890, %r889, 1; + bfi.b32 %r1711, %r57, %r890, 1, 31; + add.s32 %r1683, %r1683, -4; + setp.ne.s32 %p57, %r1683, 0; + @%p57 bra $L__BB27_21; + +$L__BB27_38: + and.b32 %r1664, %r31, 3; + setp.eq.s32 %p58, %r1664, 0; + @%p58 bra $L__BB27_54; + + and.b16 %rs683, %rs1048, 255; + setp.ne.s16 %p59, %rs683, 0; + @%p59 bra $L__BB27_43; + + setp.eq.s32 %p60, %r1747, 0; + mov.u16 %rs974, 255; + @%p60 bra $L__BB27_42; + + cvt.u64.u32 %rd151, %r1746; + add.s64 %rd152, %rd151, %rd3; + add.s64 %rd153, %rd1, %rd152; + ld.global.u8 %rs974, [%rd153]; + +$L__BB27_42: + setp.ne.s32 %p62, %r1747, 0; + selp.u32 %r891, 1, 0, %p62; + add.s32 %r1746, %r1746, %r891; + add.s32 %r892, %r1747, -1; + selp.b32 %r1747, 0, %r892, %p60; + setp.eq.s32 %p63, %r1747, 0; + or.b16 %rs685, %rs974, 15; + selp.b16 %rs1014, %rs685, %rs974, %p63; + and.b16 %rs686, %rs1014, 255; + mov.u16 %rs687, 8; + sub.s16 %rs1048, %rs687, %rs1013; + setp.eq.s16 %p64, %rs686, 255; + selp.u16 %rs1013, 1, 0, %p64; + +$L__BB27_43: + and.b32 %r1665, %r31, 3; + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r893, %rs1048; + and.b32 %r894, %r893, 255; + cvt.u32.u16 %r895, %rs1014; + and.b32 %r1706, %r895, 255; + shr.u32 %r896, %r1706, %r894; + and.b32 %r897, %r896, 1; + bfi.b32 %r1711, %r1711, %r897, 1, 31; + setp.eq.s32 %p65, %r1665, 1; + @%p65 bra $L__BB27_54; + + and.b16 %rs688, %rs1048, 255; + setp.ne.s16 %p66, %rs688, 0; + @%p66 bra $L__BB27_48; + + setp.eq.s32 %p67, %r1747, 0; + mov.u16 %rs978, 255; + @%p67 bra $L__BB27_47; + + cvt.u64.u32 %rd154, %r1746; + add.s64 %rd155, %rd154, %rd3; + add.s64 %rd156, %rd1, %rd155; + ld.global.u8 %rs978, [%rd156]; + +$L__BB27_47: + setp.ne.s32 %p69, %r1747, 0; + selp.u32 %r898, 1, 0, %p69; + add.s32 %r1746, %r1746, %r898; + add.s32 %r899, %r1747, -1; + selp.b32 %r1747, 0, %r899, %p67; + setp.eq.s32 %p70, %r1747, 0; + or.b16 %rs690, %rs978, 15; + selp.b16 %rs1014, %rs690, %rs978, %p70; + and.b16 %rs691, %rs1014, 255; + mov.u16 %rs692, 8; + sub.s16 %rs1048, %rs692, %rs1013; + setp.eq.s16 %p71, %rs691, 255; + selp.u16 %rs1013, 1, 0, %p71; + cvt.u32.u16 %r900, %rs1014; + and.b32 %r1706, %r900, 255; + +$L__BB27_48: + and.b32 %r1666, %r31, 3; + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r901, %rs1048; + and.b32 %r902, %r901, 255; + shr.u32 %r903, %r1706, %r902; + and.b32 %r904, %r903, 1; + bfi.b32 %r1711, %r1711, %r904, 1, 31; + setp.eq.s32 %p72, %r1666, 2; + @%p72 bra $L__BB27_54; + + and.b16 %rs693, %rs1048, 255; + setp.ne.s16 %p73, %rs693, 0; + @%p73 bra $L__BB27_53; + + setp.eq.s32 %p74, %r1747, 0; + mov.u16 %rs982, 255; + @%p74 bra $L__BB27_52; + + cvt.u64.u32 %rd157, %r1746; + add.s64 %rd158, %rd157, %rd3; + add.s64 %rd159, %rd1, %rd158; + ld.global.u8 %rs982, [%rd159]; + +$L__BB27_52: + setp.ne.s32 %p76, %r1747, 0; + selp.u32 %r905, 1, 0, %p76; + add.s32 %r1746, %r1746, %r905; + add.s32 %r906, %r1747, -1; + selp.b32 %r1747, 0, %r906, %p74; + setp.eq.s32 %p77, %r1747, 0; + or.b16 %rs695, %rs982, 15; + selp.b16 %rs1014, %rs695, %rs982, %p77; + and.b16 %rs696, %rs1014, 255; + mov.u16 %rs697, 8; + sub.s16 %rs1048, %rs697, %rs1013; + setp.eq.s16 %p78, %rs696, 255; + selp.u16 %rs1013, 1, 0, %p78; + cvt.u32.u16 %r907, %rs1014; + and.b32 %r1706, %r907, 255; + +$L__BB27_53: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r908, %rs1048; + and.b32 %r909, %r908, 255; + shr.u32 %r910, %r1706, %r909; + and.b32 %r911, %r910, 1; + bfi.b32 %r1711, %r1711, %r911, 1, 31; + +$L__BB27_54: + shl.b32 %r912, %r1711, 1; + or.b32 %r1715, %r912, 1; + add.s32 %r913, %r1837, -1; + setp.eq.s32 %p79, %r1837, 0; + selp.b32 %r1714, 0, %r913, %p79; + +$L__BB27_55: + mul.lo.s32 %r914, %r1719, 7; + cvt.u64.u32 %rd160, %r1715; + shl.b64 %rd161, %rd160, %r914; + or.b64 %rd455, %rd161, %rd455; + setp.ne.s32 %p80, %r1837, 12; + setp.ne.s32 %p81, %r28, 0; + or.pred %p82, %p80, %p81; + add.s32 %r1719, %r1719, 1; + setp.lt.u32 %p83, %r1719, 8; + or.pred %p84, %p83, %p82; + mov.u32 %r1837, %r1714; + @%p84 bra $L__BB27_11; + +$L__BB27_56: + and.b16 %rs941, %rs1, 15; + cvt.u32.u16 %r1673, %rs941; + mul.wide.u16 %r1672, %rs652, 16; + or.b32 %r1671, %r1672, %r1673; + add.s32 %r1916, %r1671, -2; + setp.gt.u16 %p506, %rs1, 143; + selp.u16 %rs1180, 1, 0, %p506; + ld.param.u64 %rd452, [ j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_4]; + shr.u16 %rs934, %rs1, 4; + ld.param.u64 %rd449, [ j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_2]; + add.s32 %r1838, %r1719, -1; + shr.u64 %rd465, %rd455, 7; + cvt.u32.u64 %r917, %rd455; + and.b32 %r1834, %r917, 127; + cvt.u64.u16 %rd474, %rs934; + and.b64 %rd162, %rd474, 7; + setp.eq.s64 %p85, %rd162, 7; + selp.b32 %r1917, 3, 4, %p85; + cvta.to.global.u64 %rd11, %rd452; + cvta.to.global.u64 %rd12, %rd449; + add.u64 %rd13, %SPL, 0; + mov.u32 %r1720, 0; + mov.u32 %r1721, %r1720; + +$L__BB27_57: + setp.gt.u32 %p86, %r1917, 31; + @%p86 bra $L__BB27_61; + +$L__BB27_58: + setp.eq.s32 %p87, %r1916, 0; + mov.u16 %rs1000, 0; + @%p87 bra $L__BB27_60; + + cvt.s64.s32 %rd164, %r1915; + add.s64 %rd165, %rd164, %rd3; + add.s64 %rd166, %rd1, %rd165; + ld.global.u8 %rs1000, [%rd166]; + +$L__BB27_60: + setp.ne.s32 %p89, %r1916, 0; + selp.b32 %r918, -1, 0, %p89; + add.s32 %r1915, %r1915, %r918; + add.s32 %r919, %r1916, -1; + selp.b32 %r1916, 0, %r919, %p87; + and.b16 %rs699, %rs1000, 255; + and.b16 %rs700, %rs1000, 127; + setp.eq.s16 %p90, %rs700, 127; + and.b16 %rs701, %rs1180, 255; + setp.ne.s16 %p91, %rs701, 0; + and.pred %p92, %p91, %p90; + selp.b32 %r920, 7, 8, %p92; + cvt.u64.u16 %rd167, %rs1000; + and.b64 %rd168, %rd167, 255; + shl.b64 %rd169, %rd168, %r1917; + or.b64 %rd474, %rd169, %rd474; + add.s32 %r1917, %r920, %r1917; + setp.gt.u16 %p93, %rs699, 143; + selp.u16 %rs1180, 1, 0, %p93; + setp.lt.u32 %p94, %r1917, 33; + @%p94 bra $L__BB27_58; + +$L__BB27_61: + cvt.u32.u64 %r921, %rd474; + and.b32 %r922, %r921, 127; + add.s32 %r923, %r922, %r1720; + mul.wide.u32 %rd170, %r923, 2; + add.s64 %rd171, %rd12, %rd170; + ld.global.u16 %r1787, [%rd171]; + setp.ne.s32 %p95, %r1720, 0; + @%p95 bra $L__BB27_111; + + add.s32 %r130, %r1834, -2; + setp.eq.s32 %p96, %r130, -1; + selp.b32 %r1787, %r1787, 0, %p96; + setp.gt.s32 %p97, %r1834, 1; + mov.u32 %r1834, %r130; + @%p97 bra $L__BB27_111; + + setp.ne.s32 %p98, %r1838, 0; + @%p98 bra $L__BB27_110; + + mov.u32 %r1838, 0; + +$L__BB27_65: + setp.gt.u32 %p99, %r1838, 7; + @%p99 bra $L__BB27_110; + + cvt.u64.u32 %rd20, %r1837; + mul.wide.u32 %rd172, %r1837, 4; + add.s64 %rd174, %rd133, %rd172; + ld.global.nc.u32 %r136, [%rd174]; + and.b16 %rs702, %rs1048, 255; + setp.ne.s16 %p100, %rs702, 0; + @%p100 bra $L__BB27_70; + + setp.eq.s32 %p101, %r1747, 0; + mov.u16 %rs1005, 255; + @%p101 bra $L__BB27_69; + + cvt.u64.u32 %rd175, %r1746; + add.s64 %rd176, %rd175, %rd3; + add.s64 %rd177, %rd1, %rd176; + ld.global.u8 %rs1005, [%rd177]; + +$L__BB27_69: + setp.ne.s32 %p103, %r1747, 0; + selp.u32 %r925, 1, 0, %p103; + add.s32 %r1746, %r1746, %r925; + add.s32 %r926, %r1747, -1; + selp.b32 %r1747, 0, %r926, %p101; + setp.eq.s32 %p104, %r1747, 0; + or.b16 %rs704, %rs1005, 15; + selp.b16 %rs1014, %rs704, %rs1005, %p104; + and.b16 %rs705, %rs1014, 255; + mov.u16 %rs706, 8; + sub.s16 %rs1048, %rs706, %rs1013; + setp.eq.s16 %p105, %rs705, 255; + selp.u16 %rs1013, 1, 0, %p105; + +$L__BB27_70: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r927, %rs1048; + and.b32 %r928, %r927, 255; + mov.u32 %r929, 1; + shl.b32 %r930, %r929, %r928; + cvt.u32.u16 %r931, %rs1014; + and.b32 %r932, %r930, %r931; + and.b32 %r141, %r932, 255; + setp.eq.s32 %p106, %r141, 0; + @%p106 bra $L__BB27_72; + + add.s32 %r933, %r1837, 1; + min.u32 %r1776, %r933, 12; + mov.u32 %r934, -1; + shl.b32 %r935, %r934, %r136; + shl.b32 %r936, %r935, 1; + xor.b32 %r1777, %r936, -2; + bra.uni $L__BB27_109; + +$L__BB27_72: + add.s64 %rd178, %rd20, -3; + setp.gt.u64 %p107, %rd178, 9; + mov.u32 %r1773, 0; + @%p107 bra $L__BB27_108; + + max.u32 %r144, %r136, 1; + add.s32 %r940, %r144, -1; + and.b32 %r145, %r144, 3; + setp.lt.u32 %p108, %r940, 3; + mov.u32 %r1773, 0; + @%p108 bra $L__BB27_92; + + sub.s32 %r1745, %r144, %r145; + mov.u32 %r1773, 0; + +$L__BB27_75: + and.b16 %rs708, %rs1048, 255; + setp.ne.s16 %p109, %rs708, 0; + @%p109 bra $L__BB27_79; + + setp.eq.s32 %p110, %r1747, 0; + mov.u16 %rs1012, 255; + @%p110 bra $L__BB27_78; + + cvt.u64.u32 %rd179, %r1746; + add.s64 %rd180, %rd179, %rd3; + add.s64 %rd181, %rd1, %rd180; + ld.global.u8 %rs1012, [%rd181]; + +$L__BB27_78: + setp.ne.s32 %p112, %r1747, 0; + selp.u32 %r942, 1, 0, %p112; + add.s32 %r1746, %r1746, %r942; + add.s32 %r943, %r1747, -1; + selp.b32 %r1747, 0, %r943, %p110; + setp.eq.s32 %p113, %r1747, 0; + or.b16 %rs710, %rs1012, 15; + selp.b16 %rs1014, %rs710, %rs1012, %p113; + and.b16 %rs711, %rs1014, 255; + mov.u16 %rs712, 8; + sub.s16 %rs1048, %rs712, %rs1013; + setp.eq.s16 %p114, %rs711, 255; + selp.u16 %rs1013, 1, 0, %p114; + +$L__BB27_79: + add.s16 %rs1019, %rs1048, -1; + and.b16 %rs713, %rs1019, 255; + cvt.u32.u16 %r944, %rs1019; + and.b32 %r945, %r944, 255; + cvt.u32.u16 %r946, %rs1014; + and.b32 %r1751, %r946, 255; + shr.u32 %r947, %r1751, %r945; + and.b32 %r948, %r947, 1; + bfi.b32 %r156, %r1773, %r948, 1, 31; + setp.ne.s16 %p115, %rs713, 0; + @%p115 bra $L__BB27_83; + + setp.eq.s32 %p116, %r1747, 0; + mov.u16 %rs1016, 255; + @%p116 bra $L__BB27_82; + + cvt.u64.u32 %rd182, %r1746; + add.s64 %rd183, %rd182, %rd3; + add.s64 %rd184, %rd1, %rd183; + ld.global.u8 %rs1016, [%rd184]; + +$L__BB27_82: + setp.ne.s32 %p118, %r1747, 0; + selp.u32 %r949, 1, 0, %p118; + add.s32 %r1746, %r1746, %r949; + add.s32 %r950, %r1747, -1; + selp.b32 %r1747, 0, %r950, %p116; + setp.eq.s32 %p119, %r1747, 0; + or.b16 %rs715, %rs1016, 15; + selp.b16 %rs1014, %rs715, %rs1016, %p119; + and.b16 %rs716, %rs1014, 255; + mov.u16 %rs717, 8; + sub.s16 %rs1019, %rs717, %rs1013; + setp.eq.s16 %p120, %rs716, 255; + selp.u16 %rs1013, 1, 0, %p120; + cvt.u32.u16 %r951, %rs1014; + and.b32 %r1751, %r951, 255; + +$L__BB27_83: + add.s16 %rs1023, %rs1019, -1; + and.b16 %rs718, %rs1023, 255; + cvt.u32.u16 %r952, %rs1023; + and.b32 %r953, %r952, 255; + shr.u32 %r954, %r1751, %r953; + and.b32 %r955, %r954, 1; + bfi.b32 %r163, %r156, %r955, 1, 31; + setp.ne.s16 %p121, %rs718, 0; + @%p121 bra $L__BB27_87; + + setp.eq.s32 %p122, %r1747, 0; + mov.u16 %rs1020, 255; + @%p122 bra $L__BB27_86; + + cvt.u64.u32 %rd185, %r1746; + add.s64 %rd186, %rd185, %rd3; + add.s64 %rd187, %rd1, %rd186; + ld.global.u8 %rs1020, [%rd187]; + +$L__BB27_86: + setp.ne.s32 %p124, %r1747, 0; + selp.u32 %r956, 1, 0, %p124; + add.s32 %r1746, %r1746, %r956; + add.s32 %r957, %r1747, -1; + selp.b32 %r1747, 0, %r957, %p122; + setp.eq.s32 %p125, %r1747, 0; + or.b16 %rs720, %rs1020, 15; + selp.b16 %rs1014, %rs720, %rs1020, %p125; + and.b16 %rs721, %rs1014, 255; + mov.u16 %rs722, 8; + sub.s16 %rs1023, %rs722, %rs1013; + setp.eq.s16 %p126, %rs721, 255; + selp.u16 %rs1013, 1, 0, %p126; + cvt.u32.u16 %r958, %rs1014; + and.b32 %r1751, %r958, 255; + +$L__BB27_87: + add.s16 %rs1027, %rs1023, -1; + and.b16 %rs723, %rs1027, 255; + cvt.u32.u16 %r959, %rs1027; + and.b32 %r960, %r959, 255; + shr.u32 %r961, %r1751, %r960; + and.b32 %r962, %r961, 1; + bfi.b32 %r170, %r163, %r962, 1, 31; + setp.ne.s16 %p127, %rs723, 0; + @%p127 bra $L__BB27_91; + + setp.eq.s32 %p128, %r1747, 0; + mov.u16 %rs1024, 255; + @%p128 bra $L__BB27_90; + + cvt.u64.u32 %rd188, %r1746; + add.s64 %rd189, %rd188, %rd3; + add.s64 %rd190, %rd1, %rd189; + ld.global.u8 %rs1024, [%rd190]; + +$L__BB27_90: + setp.ne.s32 %p130, %r1747, 0; + selp.u32 %r963, 1, 0, %p130; + add.s32 %r1746, %r1746, %r963; + add.s32 %r964, %r1747, -1; + selp.b32 %r1747, 0, %r964, %p128; + setp.eq.s32 %p131, %r1747, 0; + or.b16 %rs725, %rs1024, 15; + selp.b16 %rs1014, %rs725, %rs1024, %p131; + and.b16 %rs726, %rs1014, 255; + mov.u16 %rs727, 8; + sub.s16 %rs1027, %rs727, %rs1013; + setp.eq.s16 %p132, %rs726, 255; + selp.u16 %rs1013, 1, 0, %p132; + cvt.u32.u16 %r965, %rs1014; + and.b32 %r1751, %r965, 255; + +$L__BB27_91: + add.s16 %rs1048, %rs1027, -1; + cvt.u32.u16 %r966, %rs1048; + and.b32 %r967, %r966, 255; + shr.u32 %r968, %r1751, %r967; + and.b32 %r969, %r968, 1; + bfi.b32 %r1773, %r170, %r969, 1, 31; + add.s32 %r1745, %r1745, -4; + setp.ne.s32 %p133, %r1745, 0; + @%p133 bra $L__BB27_75; + +$L__BB27_92: + setp.eq.s32 %p134, %r145, 0; + @%p134 bra $L__BB27_108; + + and.b16 %rs728, %rs1048, 255; + setp.ne.s16 %p135, %rs728, 0; + @%p135 bra $L__BB27_97; + + setp.eq.s32 %p136, %r1747, 0; + mov.u16 %rs1034, 255; + @%p136 bra $L__BB27_96; + + cvt.u64.u32 %rd191, %r1746; + add.s64 %rd192, %rd191, %rd3; + add.s64 %rd193, %rd1, %rd192; + ld.global.u8 %rs1034, [%rd193]; + +$L__BB27_96: + setp.ne.s32 %p138, %r1747, 0; + selp.u32 %r970, 1, 0, %p138; + add.s32 %r1746, %r1746, %r970; + add.s32 %r971, %r1747, -1; + selp.b32 %r1747, 0, %r971, %p136; + setp.eq.s32 %p139, %r1747, 0; + or.b16 %rs730, %rs1034, 15; + selp.b16 %rs1014, %rs730, %rs1034, %p139; + and.b16 %rs731, %rs1014, 255; + mov.u16 %rs732, 8; + sub.s16 %rs1048, %rs732, %rs1013; + setp.eq.s16 %p140, %rs731, 255; + selp.u16 %rs1013, 1, 0, %p140; + +$L__BB27_97: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r972, %rs1048; + and.b32 %r973, %r972, 255; + cvt.u32.u16 %r974, %rs1014; + and.b32 %r1768, %r974, 255; + shr.u32 %r975, %r1768, %r973; + and.b32 %r976, %r975, 1; + bfi.b32 %r1773, %r1773, %r976, 1, 31; + setp.eq.s32 %p141, %r145, 1; + @%p141 bra $L__BB27_108; + + and.b16 %rs733, %rs1048, 255; + setp.ne.s16 %p142, %rs733, 0; + @%p142 bra $L__BB27_102; + + setp.eq.s32 %p143, %r1747, 0; + mov.u16 %rs1038, 255; + @%p143 bra $L__BB27_101; + + cvt.u64.u32 %rd194, %r1746; + add.s64 %rd195, %rd194, %rd3; + add.s64 %rd196, %rd1, %rd195; + ld.global.u8 %rs1038, [%rd196]; + +$L__BB27_101: + setp.ne.s32 %p145, %r1747, 0; + selp.u32 %r977, 1, 0, %p145; + add.s32 %r1746, %r1746, %r977; + add.s32 %r978, %r1747, -1; + selp.b32 %r1747, 0, %r978, %p143; + setp.eq.s32 %p146, %r1747, 0; + or.b16 %rs735, %rs1038, 15; + selp.b16 %rs1014, %rs735, %rs1038, %p146; + and.b16 %rs736, %rs1014, 255; + mov.u16 %rs737, 8; + sub.s16 %rs1048, %rs737, %rs1013; + setp.eq.s16 %p147, %rs736, 255; + selp.u16 %rs1013, 1, 0, %p147; + cvt.u32.u16 %r979, %rs1014; + and.b32 %r1768, %r979, 255; + +$L__BB27_102: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r980, %rs1048; + and.b32 %r981, %r980, 255; + shr.u32 %r982, %r1768, %r981; + and.b32 %r983, %r982, 1; + bfi.b32 %r1773, %r1773, %r983, 1, 31; + setp.eq.s32 %p148, %r145, 2; + @%p148 bra $L__BB27_108; + + and.b16 %rs738, %rs1048, 255; + setp.ne.s16 %p149, %rs738, 0; + @%p149 bra $L__BB27_107; + + setp.eq.s32 %p150, %r1747, 0; + mov.u16 %rs1042, 255; + @%p150 bra $L__BB27_106; + + cvt.u64.u32 %rd197, %r1746; + add.s64 %rd198, %rd197, %rd3; + add.s64 %rd199, %rd1, %rd198; + ld.global.u8 %rs1042, [%rd199]; + +$L__BB27_106: + setp.ne.s32 %p152, %r1747, 0; + selp.u32 %r984, 1, 0, %p152; + add.s32 %r1746, %r1746, %r984; + add.s32 %r985, %r1747, -1; + selp.b32 %r1747, 0, %r985, %p150; + setp.eq.s32 %p153, %r1747, 0; + or.b16 %rs740, %rs1042, 15; + selp.b16 %rs1014, %rs740, %rs1042, %p153; + and.b16 %rs741, %rs1014, 255; + mov.u16 %rs742, 8; + sub.s16 %rs1048, %rs742, %rs1013; + setp.eq.s16 %p154, %rs741, 255; + selp.u16 %rs1013, 1, 0, %p154; + cvt.u32.u16 %r986, %rs1014; + and.b32 %r1768, %r986, 255; + +$L__BB27_107: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r987, %rs1048; + and.b32 %r988, %r987, 255; + shr.u32 %r989, %r1768, %r988; + and.b32 %r990, %r989, 1; + bfi.b32 %r1773, %r1773, %r990, 1, 31; + +$L__BB27_108: + shl.b32 %r991, %r1773, 1; + or.b32 %r1777, %r991, 1; + add.s32 %r992, %r1837, -1; + setp.eq.s32 %p155, %r1837, 0; + selp.b32 %r1776, 0, %r992, %p155; + +$L__BB27_109: + mul.lo.s32 %r993, %r1838, 7; + cvt.u64.u32 %rd200, %r1777; + shl.b64 %rd201, %rd200, %r993; + or.b64 %rd465, %rd201, %rd465; + setp.ne.s32 %p156, %r1837, 12; + setp.ne.s32 %p157, %r141, 0; + or.pred %p158, %p156, %p157; + add.s32 %r1838, %r1838, 1; + setp.lt.u32 %p159, %r1838, 8; + or.pred %p160, %p159, %p158; + mov.u32 %r1837, %r1776; + @%p160 bra $L__BB27_65; + +$L__BB27_110: + cvt.u32.u64 %r994, %rd465; + and.b32 %r1834, %r994, 127; + shr.u64 %rd465, %rd465, 7; + add.s32 %r1838, %r1838, -1; + +$L__BB27_111: + mul.wide.u32 %rd202, %r1721, 2; + add.s64 %rd25, %rd13, %rd202; + st.local.u16 [%rd25], %r1787; + shl.b32 %r995, %r1787, 3; + and.b32 %r996, %r995, 128; + shl.b32 %r997, %r1787, 2; + and.b32 %r998, %r997, 896; + or.b32 %r999, %r996, %r998; + and.b32 %r1000, %r1787, 7; + shr.u64 %rd26, %rd474, %r1000; + sub.s32 %r227, %r1917, %r1000; + cvt.u32.u64 %r1001, %rd26; + and.b32 %r1002, %r1001, 127; + or.b32 %r1003, %r1002, %r999; + mul.wide.u32 %rd203, %r1003, 2; + add.s64 %rd204, %rd12, %rd203; + ld.global.u16 %r1839, [%rd204]; + setp.ne.s32 %p161, %r999, 0; + add.s32 %r229, %r1721, 2; + setp.ge.u32 %p162, %r229, %r820; + or.pred %p163, %p162, %p161; + @%p163 bra $L__BB27_161; + + add.s32 %r230, %r1834, -2; + setp.eq.s32 %p164, %r230, -1; + selp.b32 %r1839, %r1839, 0, %p164; + setp.gt.s32 %p165, %r1834, 1; + mov.u32 %r1834, %r230; + @%p165 bra $L__BB27_161; + + setp.ne.s32 %p166, %r1838, 0; + @%p166 bra $L__BB27_160; + + mov.u32 %r1838, 0; + +$L__BB27_115: + setp.gt.u32 %p167, %r1838, 7; + @%p167 bra $L__BB27_160; + + cvt.u64.u32 %rd28, %r1837; + mul.wide.u32 %rd205, %r1837, 4; + add.s64 %rd207, %rd133, %rd205; + ld.global.nc.u32 %r236, [%rd207]; + and.b16 %rs743, %rs1048, 255; + setp.ne.s16 %p168, %rs743, 0; + @%p168 bra $L__BB27_120; + + setp.eq.s32 %p169, %r1747, 0; + mov.u16 %rs1061, 255; + @%p169 bra $L__BB27_119; + + cvt.u64.u32 %rd208, %r1746; + add.s64 %rd209, %rd208, %rd3; + add.s64 %rd210, %rd1, %rd209; + ld.global.u8 %rs1061, [%rd210]; + +$L__BB27_119: + setp.ne.s32 %p171, %r1747, 0; + selp.u32 %r1005, 1, 0, %p171; + add.s32 %r1746, %r1746, %r1005; + add.s32 %r1006, %r1747, -1; + selp.b32 %r1747, 0, %r1006, %p169; + setp.eq.s32 %p172, %r1747, 0; + or.b16 %rs745, %rs1061, 15; + selp.b16 %rs1014, %rs745, %rs1061, %p172; + and.b16 %rs746, %rs1014, 255; + mov.u16 %rs747, 8; + sub.s16 %rs1048, %rs747, %rs1013; + setp.eq.s16 %p173, %rs746, 255; + selp.u16 %rs1013, 1, 0, %p173; + +$L__BB27_120: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1007, %rs1048; + and.b32 %r1008, %r1007, 255; + mov.u32 %r1009, 1; + shl.b32 %r1010, %r1009, %r1008; + cvt.u32.u16 %r1011, %rs1014; + and.b32 %r1012, %r1010, %r1011; + and.b32 %r241, %r1012, 255; + setp.eq.s32 %p174, %r241, 0; + @%p174 bra $L__BB27_122; + + add.s32 %r1013, %r1837, 1; + min.u32 %r1828, %r1013, 12; + mov.u32 %r1014, -1; + shl.b32 %r1015, %r1014, %r236; + shl.b32 %r1016, %r1015, 1; + xor.b32 %r1829, %r1016, -2; + bra.uni $L__BB27_159; + +$L__BB27_122: + add.s64 %rd211, %rd28, -3; + setp.gt.u64 %p175, %rd211, 9; + mov.u32 %r1825, 0; + @%p175 bra $L__BB27_158; + + max.u32 %r244, %r236, 1; + add.s32 %r1020, %r244, -1; + and.b32 %r245, %r244, 3; + setp.lt.u32 %p176, %r1020, 3; + mov.u32 %r1825, 0; + @%p176 bra $L__BB27_142; + + sub.s32 %r1797, %r244, %r245; + mov.u32 %r1825, 0; + +$L__BB27_125: + and.b16 %rs749, %rs1048, 255; + setp.ne.s16 %p177, %rs749, 0; + @%p177 bra $L__BB27_129; + + setp.eq.s32 %p178, %r1747, 0; + mov.u16 %rs1068, 255; + @%p178 bra $L__BB27_128; + + cvt.u64.u32 %rd212, %r1746; + add.s64 %rd213, %rd212, %rd3; + add.s64 %rd214, %rd1, %rd213; + ld.global.u8 %rs1068, [%rd214]; + +$L__BB27_128: + setp.ne.s32 %p180, %r1747, 0; + selp.u32 %r1022, 1, 0, %p180; + add.s32 %r1746, %r1746, %r1022; + add.s32 %r1023, %r1747, -1; + selp.b32 %r1747, 0, %r1023, %p178; + setp.eq.s32 %p181, %r1747, 0; + or.b16 %rs751, %rs1068, 15; + selp.b16 %rs1014, %rs751, %rs1068, %p181; + and.b16 %rs752, %rs1014, 255; + mov.u16 %rs753, 8; + sub.s16 %rs1048, %rs753, %rs1013; + setp.eq.s16 %p182, %rs752, 255; + selp.u16 %rs1013, 1, 0, %p182; + +$L__BB27_129: + add.s16 %rs1075, %rs1048, -1; + and.b16 %rs754, %rs1075, 255; + cvt.u32.u16 %r1024, %rs1075; + and.b32 %r1025, %r1024, 255; + cvt.u32.u16 %r1026, %rs1014; + and.b32 %r1803, %r1026, 255; + shr.u32 %r1027, %r1803, %r1025; + and.b32 %r1028, %r1027, 1; + bfi.b32 %r256, %r1825, %r1028, 1, 31; + setp.ne.s16 %p183, %rs754, 0; + @%p183 bra $L__BB27_133; + + setp.eq.s32 %p184, %r1747, 0; + mov.u16 %rs1072, 255; + @%p184 bra $L__BB27_132; + + cvt.u64.u32 %rd215, %r1746; + add.s64 %rd216, %rd215, %rd3; + add.s64 %rd217, %rd1, %rd216; + ld.global.u8 %rs1072, [%rd217]; + +$L__BB27_132: + setp.ne.s32 %p186, %r1747, 0; + selp.u32 %r1029, 1, 0, %p186; + add.s32 %r1746, %r1746, %r1029; + add.s32 %r1030, %r1747, -1; + selp.b32 %r1747, 0, %r1030, %p184; + setp.eq.s32 %p187, %r1747, 0; + or.b16 %rs756, %rs1072, 15; + selp.b16 %rs1014, %rs756, %rs1072, %p187; + and.b16 %rs757, %rs1014, 255; + mov.u16 %rs758, 8; + sub.s16 %rs1075, %rs758, %rs1013; + setp.eq.s16 %p188, %rs757, 255; + selp.u16 %rs1013, 1, 0, %p188; + cvt.u32.u16 %r1031, %rs1014; + and.b32 %r1803, %r1031, 255; + +$L__BB27_133: + add.s16 %rs1079, %rs1075, -1; + and.b16 %rs759, %rs1079, 255; + cvt.u32.u16 %r1032, %rs1079; + and.b32 %r1033, %r1032, 255; + shr.u32 %r1034, %r1803, %r1033; + and.b32 %r1035, %r1034, 1; + bfi.b32 %r263, %r256, %r1035, 1, 31; + setp.ne.s16 %p189, %rs759, 0; + @%p189 bra $L__BB27_137; + + setp.eq.s32 %p190, %r1747, 0; + mov.u16 %rs1076, 255; + @%p190 bra $L__BB27_136; + + cvt.u64.u32 %rd218, %r1746; + add.s64 %rd219, %rd218, %rd3; + add.s64 %rd220, %rd1, %rd219; + ld.global.u8 %rs1076, [%rd220]; + +$L__BB27_136: + setp.ne.s32 %p192, %r1747, 0; + selp.u32 %r1036, 1, 0, %p192; + add.s32 %r1746, %r1746, %r1036; + add.s32 %r1037, %r1747, -1; + selp.b32 %r1747, 0, %r1037, %p190; + setp.eq.s32 %p193, %r1747, 0; + or.b16 %rs761, %rs1076, 15; + selp.b16 %rs1014, %rs761, %rs1076, %p193; + and.b16 %rs762, %rs1014, 255; + mov.u16 %rs763, 8; + sub.s16 %rs1079, %rs763, %rs1013; + setp.eq.s16 %p194, %rs762, 255; + selp.u16 %rs1013, 1, 0, %p194; + cvt.u32.u16 %r1038, %rs1014; + and.b32 %r1803, %r1038, 255; + +$L__BB27_137: + add.s16 %rs1083, %rs1079, -1; + and.b16 %rs764, %rs1083, 255; + cvt.u32.u16 %r1039, %rs1083; + and.b32 %r1040, %r1039, 255; + shr.u32 %r1041, %r1803, %r1040; + and.b32 %r1042, %r1041, 1; + bfi.b32 %r270, %r263, %r1042, 1, 31; + setp.ne.s16 %p195, %rs764, 0; + @%p195 bra $L__BB27_141; + + setp.eq.s32 %p196, %r1747, 0; + mov.u16 %rs1080, 255; + @%p196 bra $L__BB27_140; + + cvt.u64.u32 %rd221, %r1746; + add.s64 %rd222, %rd221, %rd3; + add.s64 %rd223, %rd1, %rd222; + ld.global.u8 %rs1080, [%rd223]; + +$L__BB27_140: + setp.ne.s32 %p198, %r1747, 0; + selp.u32 %r1043, 1, 0, %p198; + add.s32 %r1746, %r1746, %r1043; + add.s32 %r1044, %r1747, -1; + selp.b32 %r1747, 0, %r1044, %p196; + setp.eq.s32 %p199, %r1747, 0; + or.b16 %rs766, %rs1080, 15; + selp.b16 %rs1014, %rs766, %rs1080, %p199; + and.b16 %rs767, %rs1014, 255; + mov.u16 %rs768, 8; + sub.s16 %rs1083, %rs768, %rs1013; + setp.eq.s16 %p200, %rs767, 255; + selp.u16 %rs1013, 1, 0, %p200; + cvt.u32.u16 %r1045, %rs1014; + and.b32 %r1803, %r1045, 255; + +$L__BB27_141: + add.s16 %rs1048, %rs1083, -1; + cvt.u32.u16 %r1046, %rs1048; + and.b32 %r1047, %r1046, 255; + shr.u32 %r1048, %r1803, %r1047; + and.b32 %r1049, %r1048, 1; + bfi.b32 %r1825, %r270, %r1049, 1, 31; + add.s32 %r1797, %r1797, -4; + setp.ne.s32 %p201, %r1797, 0; + @%p201 bra $L__BB27_125; + +$L__BB27_142: + setp.eq.s32 %p202, %r245, 0; + @%p202 bra $L__BB27_158; + + and.b16 %rs769, %rs1048, 255; + setp.ne.s16 %p203, %rs769, 0; + @%p203 bra $L__BB27_147; + + setp.eq.s32 %p204, %r1747, 0; + mov.u16 %rs1090, 255; + @%p204 bra $L__BB27_146; + + cvt.u64.u32 %rd224, %r1746; + add.s64 %rd225, %rd224, %rd3; + add.s64 %rd226, %rd1, %rd225; + ld.global.u8 %rs1090, [%rd226]; + +$L__BB27_146: + setp.ne.s32 %p206, %r1747, 0; + selp.u32 %r1050, 1, 0, %p206; + add.s32 %r1746, %r1746, %r1050; + add.s32 %r1051, %r1747, -1; + selp.b32 %r1747, 0, %r1051, %p204; + setp.eq.s32 %p207, %r1747, 0; + or.b16 %rs771, %rs1090, 15; + selp.b16 %rs1014, %rs771, %rs1090, %p207; + and.b16 %rs772, %rs1014, 255; + mov.u16 %rs773, 8; + sub.s16 %rs1048, %rs773, %rs1013; + setp.eq.s16 %p208, %rs772, 255; + selp.u16 %rs1013, 1, 0, %p208; + +$L__BB27_147: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1052, %rs1048; + and.b32 %r1053, %r1052, 255; + cvt.u32.u16 %r1054, %rs1014; + and.b32 %r1820, %r1054, 255; + shr.u32 %r1055, %r1820, %r1053; + and.b32 %r1056, %r1055, 1; + bfi.b32 %r1825, %r1825, %r1056, 1, 31; + setp.eq.s32 %p209, %r245, 1; + @%p209 bra $L__BB27_158; + + and.b16 %rs774, %rs1048, 255; + setp.ne.s16 %p210, %rs774, 0; + @%p210 bra $L__BB27_152; + + setp.eq.s32 %p211, %r1747, 0; + mov.u16 %rs1094, 255; + @%p211 bra $L__BB27_151; + + cvt.u64.u32 %rd227, %r1746; + add.s64 %rd228, %rd227, %rd3; + add.s64 %rd229, %rd1, %rd228; + ld.global.u8 %rs1094, [%rd229]; + +$L__BB27_151: + setp.ne.s32 %p213, %r1747, 0; + selp.u32 %r1057, 1, 0, %p213; + add.s32 %r1746, %r1746, %r1057; + add.s32 %r1058, %r1747, -1; + selp.b32 %r1747, 0, %r1058, %p211; + setp.eq.s32 %p214, %r1747, 0; + or.b16 %rs776, %rs1094, 15; + selp.b16 %rs1014, %rs776, %rs1094, %p214; + and.b16 %rs777, %rs1014, 255; + mov.u16 %rs778, 8; + sub.s16 %rs1048, %rs778, %rs1013; + setp.eq.s16 %p215, %rs777, 255; + selp.u16 %rs1013, 1, 0, %p215; + cvt.u32.u16 %r1059, %rs1014; + and.b32 %r1820, %r1059, 255; + +$L__BB27_152: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1060, %rs1048; + and.b32 %r1061, %r1060, 255; + shr.u32 %r1062, %r1820, %r1061; + and.b32 %r1063, %r1062, 1; + bfi.b32 %r1825, %r1825, %r1063, 1, 31; + setp.eq.s32 %p216, %r245, 2; + @%p216 bra $L__BB27_158; + + and.b16 %rs779, %rs1048, 255; + setp.ne.s16 %p217, %rs779, 0; + @%p217 bra $L__BB27_157; + + setp.eq.s32 %p218, %r1747, 0; + mov.u16 %rs1098, 255; + @%p218 bra $L__BB27_156; + + cvt.u64.u32 %rd230, %r1746; + add.s64 %rd231, %rd230, %rd3; + add.s64 %rd232, %rd1, %rd231; + ld.global.u8 %rs1098, [%rd232]; + +$L__BB27_156: + setp.ne.s32 %p220, %r1747, 0; + selp.u32 %r1064, 1, 0, %p220; + add.s32 %r1746, %r1746, %r1064; + add.s32 %r1065, %r1747, -1; + selp.b32 %r1747, 0, %r1065, %p218; + setp.eq.s32 %p221, %r1747, 0; + or.b16 %rs781, %rs1098, 15; + selp.b16 %rs1014, %rs781, %rs1098, %p221; + and.b16 %rs782, %rs1014, 255; + mov.u16 %rs783, 8; + sub.s16 %rs1048, %rs783, %rs1013; + setp.eq.s16 %p222, %rs782, 255; + selp.u16 %rs1013, 1, 0, %p222; + cvt.u32.u16 %r1066, %rs1014; + and.b32 %r1820, %r1066, 255; + +$L__BB27_157: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1067, %rs1048; + and.b32 %r1068, %r1067, 255; + shr.u32 %r1069, %r1820, %r1068; + and.b32 %r1070, %r1069, 1; + bfi.b32 %r1825, %r1825, %r1070, 1, 31; + +$L__BB27_158: + shl.b32 %r1071, %r1825, 1; + or.b32 %r1829, %r1071, 1; + add.s32 %r1072, %r1837, -1; + setp.eq.s32 %p223, %r1837, 0; + selp.b32 %r1828, 0, %r1072, %p223; + +$L__BB27_159: + mul.lo.s32 %r1073, %r1838, 7; + cvt.u64.u32 %rd233, %r1829; + shl.b64 %rd234, %rd233, %r1073; + or.b64 %rd465, %rd234, %rd465; + setp.ne.s32 %p224, %r1837, 12; + setp.ne.s32 %p225, %r241, 0; + or.pred %p226, %p224, %p225; + add.s32 %r1838, %r1838, 1; + setp.lt.u32 %p227, %r1838, 8; + or.pred %p228, %p227, %p226; + mov.u32 %r1837, %r1828; + @%p228 bra $L__BB27_115; + +$L__BB27_160: + cvt.u32.u64 %r1074, %rd465; + and.b32 %r1834, %r1074, 127; + shr.u64 %rd465, %rd465, 7; + add.s32 %r1838, %r1838, -1; + +$L__BB27_161: + setp.lt.u32 %p229, %r229, %r820; + selp.b32 %r327, %r1839, 0, %p229; + st.local.u16 [%rd25+4], %r327; + and.b32 %r1076, %r995, 64; + shl.b32 %r1077, %r327, 4; + and.b32 %r1078, %r1077, 128; + or.b32 %r1891, %r1078, %r1076; + setp.ne.s32 %p230, %r1891, 192; + @%p230 bra $L__BB27_211; + + add.s32 %r329, %r1834, -2; + setp.eq.s32 %p231, %r329, -1; + selp.b32 %r1891, 256, 192, %p231; + setp.gt.s32 %p232, %r1834, 1; + mov.u32 %r1834, %r329; + @%p232 bra $L__BB27_211; + + setp.ne.s32 %p233, %r1838, 0; + @%p233 bra $L__BB27_210; + + mov.u32 %r1838, 0; + +$L__BB27_165: + setp.gt.u32 %p234, %r1838, 7; + @%p234 bra $L__BB27_210; + + cvt.u64.u32 %rd34, %r1837; + mul.wide.u32 %rd235, %r1837, 4; + add.s64 %rd237, %rd133, %rd235; + ld.global.nc.u32 %r335, [%rd237]; + and.b16 %rs784, %rs1048, 255; + setp.ne.s16 %p235, %rs784, 0; + @%p235 bra $L__BB27_170; + + setp.eq.s32 %p236, %r1747, 0; + mov.u16 %rs1117, 255; + @%p236 bra $L__BB27_169; + + cvt.u64.u32 %rd238, %r1746; + add.s64 %rd239, %rd238, %rd3; + add.s64 %rd240, %rd1, %rd239; + ld.global.u8 %rs1117, [%rd240]; + +$L__BB27_169: + setp.ne.s32 %p238, %r1747, 0; + selp.u32 %r1080, 1, 0, %p238; + add.s32 %r1746, %r1746, %r1080; + add.s32 %r1081, %r1747, -1; + selp.b32 %r1747, 0, %r1081, %p236; + setp.eq.s32 %p239, %r1747, 0; + or.b16 %rs786, %rs1117, 15; + selp.b16 %rs1014, %rs786, %rs1117, %p239; + and.b16 %rs787, %rs1014, 255; + mov.u16 %rs788, 8; + sub.s16 %rs1048, %rs788, %rs1013; + setp.eq.s16 %p240, %rs787, 255; + selp.u16 %rs1013, 1, 0, %p240; + +$L__BB27_170: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1082, %rs1048; + and.b32 %r1083, %r1082, 255; + mov.u32 %r1084, 1; + shl.b32 %r1085, %r1084, %r1083; + cvt.u32.u16 %r1086, %rs1014; + and.b32 %r1087, %r1085, %r1086; + and.b32 %r340, %r1087, 255; + setp.eq.s32 %p241, %r340, 0; + @%p241 bra $L__BB27_172; + + add.s32 %r1088, %r1837, 1; + min.u32 %r1880, %r1088, 12; + mov.u32 %r1089, -1; + shl.b32 %r1090, %r1089, %r335; + shl.b32 %r1091, %r1090, 1; + xor.b32 %r1881, %r1091, -2; + bra.uni $L__BB27_209; + +$L__BB27_172: + add.s64 %rd241, %rd34, -3; + setp.gt.u64 %p242, %rd241, 9; + mov.u32 %r1877, 0; + @%p242 bra $L__BB27_208; + + max.u32 %r343, %r335, 1; + add.s32 %r1095, %r343, -1; + and.b32 %r344, %r343, 3; + setp.lt.u32 %p243, %r1095, 3; + mov.u32 %r1877, 0; + @%p243 bra $L__BB27_192; + + sub.s32 %r1849, %r343, %r344; + mov.u32 %r1877, 0; + +$L__BB27_175: + and.b16 %rs790, %rs1048, 255; + setp.ne.s16 %p244, %rs790, 0; + @%p244 bra $L__BB27_179; + + setp.eq.s32 %p245, %r1747, 0; + mov.u16 %rs1124, 255; + @%p245 bra $L__BB27_178; + + cvt.u64.u32 %rd242, %r1746; + add.s64 %rd243, %rd242, %rd3; + add.s64 %rd244, %rd1, %rd243; + ld.global.u8 %rs1124, [%rd244]; + +$L__BB27_178: + setp.ne.s32 %p247, %r1747, 0; + selp.u32 %r1097, 1, 0, %p247; + add.s32 %r1746, %r1746, %r1097; + add.s32 %r1098, %r1747, -1; + selp.b32 %r1747, 0, %r1098, %p245; + setp.eq.s32 %p248, %r1747, 0; + or.b16 %rs792, %rs1124, 15; + selp.b16 %rs1014, %rs792, %rs1124, %p248; + and.b16 %rs793, %rs1014, 255; + mov.u16 %rs794, 8; + sub.s16 %rs1048, %rs794, %rs1013; + setp.eq.s16 %p249, %rs793, 255; + selp.u16 %rs1013, 1, 0, %p249; + +$L__BB27_179: + add.s16 %rs1131, %rs1048, -1; + and.b16 %rs795, %rs1131, 255; + cvt.u32.u16 %r1099, %rs1131; + and.b32 %r1100, %r1099, 255; + cvt.u32.u16 %r1101, %rs1014; + and.b32 %r1855, %r1101, 255; + shr.u32 %r1102, %r1855, %r1100; + and.b32 %r1103, %r1102, 1; + bfi.b32 %r355, %r1877, %r1103, 1, 31; + setp.ne.s16 %p250, %rs795, 0; + @%p250 bra $L__BB27_183; + + setp.eq.s32 %p251, %r1747, 0; + mov.u16 %rs1128, 255; + @%p251 bra $L__BB27_182; + + cvt.u64.u32 %rd245, %r1746; + add.s64 %rd246, %rd245, %rd3; + add.s64 %rd247, %rd1, %rd246; + ld.global.u8 %rs1128, [%rd247]; + +$L__BB27_182: + setp.ne.s32 %p253, %r1747, 0; + selp.u32 %r1104, 1, 0, %p253; + add.s32 %r1746, %r1746, %r1104; + add.s32 %r1105, %r1747, -1; + selp.b32 %r1747, 0, %r1105, %p251; + setp.eq.s32 %p254, %r1747, 0; + or.b16 %rs797, %rs1128, 15; + selp.b16 %rs1014, %rs797, %rs1128, %p254; + and.b16 %rs798, %rs1014, 255; + mov.u16 %rs799, 8; + sub.s16 %rs1131, %rs799, %rs1013; + setp.eq.s16 %p255, %rs798, 255; + selp.u16 %rs1013, 1, 0, %p255; + cvt.u32.u16 %r1106, %rs1014; + and.b32 %r1855, %r1106, 255; + +$L__BB27_183: + add.s16 %rs1135, %rs1131, -1; + and.b16 %rs800, %rs1135, 255; + cvt.u32.u16 %r1107, %rs1135; + and.b32 %r1108, %r1107, 255; + shr.u32 %r1109, %r1855, %r1108; + and.b32 %r1110, %r1109, 1; + bfi.b32 %r362, %r355, %r1110, 1, 31; + setp.ne.s16 %p256, %rs800, 0; + @%p256 bra $L__BB27_187; + + setp.eq.s32 %p257, %r1747, 0; + mov.u16 %rs1132, 255; + @%p257 bra $L__BB27_186; + + cvt.u64.u32 %rd248, %r1746; + add.s64 %rd249, %rd248, %rd3; + add.s64 %rd250, %rd1, %rd249; + ld.global.u8 %rs1132, [%rd250]; + +$L__BB27_186: + setp.ne.s32 %p259, %r1747, 0; + selp.u32 %r1111, 1, 0, %p259; + add.s32 %r1746, %r1746, %r1111; + add.s32 %r1112, %r1747, -1; + selp.b32 %r1747, 0, %r1112, %p257; + setp.eq.s32 %p260, %r1747, 0; + or.b16 %rs802, %rs1132, 15; + selp.b16 %rs1014, %rs802, %rs1132, %p260; + and.b16 %rs803, %rs1014, 255; + mov.u16 %rs804, 8; + sub.s16 %rs1135, %rs804, %rs1013; + setp.eq.s16 %p261, %rs803, 255; + selp.u16 %rs1013, 1, 0, %p261; + cvt.u32.u16 %r1113, %rs1014; + and.b32 %r1855, %r1113, 255; + +$L__BB27_187: + add.s16 %rs1139, %rs1135, -1; + and.b16 %rs805, %rs1139, 255; + cvt.u32.u16 %r1114, %rs1139; + and.b32 %r1115, %r1114, 255; + shr.u32 %r1116, %r1855, %r1115; + and.b32 %r1117, %r1116, 1; + bfi.b32 %r369, %r362, %r1117, 1, 31; + setp.ne.s16 %p262, %rs805, 0; + @%p262 bra $L__BB27_191; + + setp.eq.s32 %p263, %r1747, 0; + mov.u16 %rs1136, 255; + @%p263 bra $L__BB27_190; + + cvt.u64.u32 %rd251, %r1746; + add.s64 %rd252, %rd251, %rd3; + add.s64 %rd253, %rd1, %rd252; + ld.global.u8 %rs1136, [%rd253]; + +$L__BB27_190: + setp.ne.s32 %p265, %r1747, 0; + selp.u32 %r1118, 1, 0, %p265; + add.s32 %r1746, %r1746, %r1118; + add.s32 %r1119, %r1747, -1; + selp.b32 %r1747, 0, %r1119, %p263; + setp.eq.s32 %p266, %r1747, 0; + or.b16 %rs807, %rs1136, 15; + selp.b16 %rs1014, %rs807, %rs1136, %p266; + and.b16 %rs808, %rs1014, 255; + mov.u16 %rs809, 8; + sub.s16 %rs1139, %rs809, %rs1013; + setp.eq.s16 %p267, %rs808, 255; + selp.u16 %rs1013, 1, 0, %p267; + cvt.u32.u16 %r1120, %rs1014; + and.b32 %r1855, %r1120, 255; + +$L__BB27_191: + add.s16 %rs1048, %rs1139, -1; + cvt.u32.u16 %r1121, %rs1048; + and.b32 %r1122, %r1121, 255; + shr.u32 %r1123, %r1855, %r1122; + and.b32 %r1124, %r1123, 1; + bfi.b32 %r1877, %r369, %r1124, 1, 31; + add.s32 %r1849, %r1849, -4; + setp.ne.s32 %p268, %r1849, 0; + @%p268 bra $L__BB27_175; + +$L__BB27_192: + setp.eq.s32 %p269, %r344, 0; + @%p269 bra $L__BB27_208; + + and.b16 %rs810, %rs1048, 255; + setp.ne.s16 %p270, %rs810, 0; + @%p270 bra $L__BB27_197; + + setp.eq.s32 %p271, %r1747, 0; + mov.u16 %rs1146, 255; + @%p271 bra $L__BB27_196; + + cvt.u64.u32 %rd254, %r1746; + add.s64 %rd255, %rd254, %rd3; + add.s64 %rd256, %rd1, %rd255; + ld.global.u8 %rs1146, [%rd256]; + +$L__BB27_196: + setp.ne.s32 %p273, %r1747, 0; + selp.u32 %r1125, 1, 0, %p273; + add.s32 %r1746, %r1746, %r1125; + add.s32 %r1126, %r1747, -1; + selp.b32 %r1747, 0, %r1126, %p271; + setp.eq.s32 %p274, %r1747, 0; + or.b16 %rs812, %rs1146, 15; + selp.b16 %rs1014, %rs812, %rs1146, %p274; + and.b16 %rs813, %rs1014, 255; + mov.u16 %rs814, 8; + sub.s16 %rs1048, %rs814, %rs1013; + setp.eq.s16 %p275, %rs813, 255; + selp.u16 %rs1013, 1, 0, %p275; + +$L__BB27_197: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1127, %rs1048; + and.b32 %r1128, %r1127, 255; + cvt.u32.u16 %r1129, %rs1014; + and.b32 %r1872, %r1129, 255; + shr.u32 %r1130, %r1872, %r1128; + and.b32 %r1131, %r1130, 1; + bfi.b32 %r1877, %r1877, %r1131, 1, 31; + setp.eq.s32 %p276, %r344, 1; + @%p276 bra $L__BB27_208; + + and.b16 %rs815, %rs1048, 255; + setp.ne.s16 %p277, %rs815, 0; + @%p277 bra $L__BB27_202; + + setp.eq.s32 %p278, %r1747, 0; + mov.u16 %rs1150, 255; + @%p278 bra $L__BB27_201; + + cvt.u64.u32 %rd257, %r1746; + add.s64 %rd258, %rd257, %rd3; + add.s64 %rd259, %rd1, %rd258; + ld.global.u8 %rs1150, [%rd259]; + +$L__BB27_201: + setp.ne.s32 %p280, %r1747, 0; + selp.u32 %r1132, 1, 0, %p280; + add.s32 %r1746, %r1746, %r1132; + add.s32 %r1133, %r1747, -1; + selp.b32 %r1747, 0, %r1133, %p278; + setp.eq.s32 %p281, %r1747, 0; + or.b16 %rs817, %rs1150, 15; + selp.b16 %rs1014, %rs817, %rs1150, %p281; + and.b16 %rs818, %rs1014, 255; + mov.u16 %rs819, 8; + sub.s16 %rs1048, %rs819, %rs1013; + setp.eq.s16 %p282, %rs818, 255; + selp.u16 %rs1013, 1, 0, %p282; + cvt.u32.u16 %r1134, %rs1014; + and.b32 %r1872, %r1134, 255; + +$L__BB27_202: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1135, %rs1048; + and.b32 %r1136, %r1135, 255; + shr.u32 %r1137, %r1872, %r1136; + and.b32 %r1138, %r1137, 1; + bfi.b32 %r1877, %r1877, %r1138, 1, 31; + setp.eq.s32 %p283, %r344, 2; + @%p283 bra $L__BB27_208; + + and.b16 %rs820, %rs1048, 255; + setp.ne.s16 %p284, %rs820, 0; + @%p284 bra $L__BB27_207; + + setp.eq.s32 %p285, %r1747, 0; + mov.u16 %rs1154, 255; + @%p285 bra $L__BB27_206; + + cvt.u64.u32 %rd260, %r1746; + add.s64 %rd261, %rd260, %rd3; + add.s64 %rd262, %rd1, %rd261; + ld.global.u8 %rs1154, [%rd262]; + +$L__BB27_206: + setp.ne.s32 %p287, %r1747, 0; + selp.u32 %r1139, 1, 0, %p287; + add.s32 %r1746, %r1746, %r1139; + add.s32 %r1140, %r1747, -1; + selp.b32 %r1747, 0, %r1140, %p285; + setp.eq.s32 %p288, %r1747, 0; + or.b16 %rs822, %rs1154, 15; + selp.b16 %rs1014, %rs822, %rs1154, %p288; + and.b16 %rs823, %rs1014, 255; + mov.u16 %rs824, 8; + sub.s16 %rs1048, %rs824, %rs1013; + setp.eq.s16 %p289, %rs823, 255; + selp.u16 %rs1013, 1, 0, %p289; + cvt.u32.u16 %r1141, %rs1014; + and.b32 %r1872, %r1141, 255; + +$L__BB27_207: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1142, %rs1048; + and.b32 %r1143, %r1142, 255; + shr.u32 %r1144, %r1872, %r1143; + and.b32 %r1145, %r1144, 1; + bfi.b32 %r1877, %r1877, %r1145, 1, 31; + +$L__BB27_208: + shl.b32 %r1146, %r1877, 1; + or.b32 %r1881, %r1146, 1; + add.s32 %r1147, %r1837, -1; + setp.eq.s32 %p290, %r1837, 0; + selp.b32 %r1880, 0, %r1147, %p290; + +$L__BB27_209: + mul.lo.s32 %r1148, %r1838, 7; + cvt.u64.u32 %rd263, %r1881; + shl.b64 %rd264, %rd263, %r1148; + or.b64 %rd465, %rd264, %rd465; + setp.ne.s32 %p291, %r1837, 12; + setp.ne.s32 %p292, %r340, 0; + or.pred %p293, %p291, %p292; + add.s32 %r1838, %r1838, 1; + setp.lt.u32 %p294, %r1838, 8; + or.pred %p295, %p294, %p293; + mov.u32 %r1837, %r1880; + @%p295 bra $L__BB27_165; + +$L__BB27_210: + cvt.u32.u64 %r1149, %rd465; + and.b32 %r1834, %r1149, 127; + shr.u64 %rd465, %rd465, 7; + add.s32 %r1838, %r1838, -1; + +$L__BB27_211: + and.b32 %r1150, %r327, 7; + shr.u64 %rd265, %rd26, %r1150; + cvt.u32.u64 %r1151, %rd265; + and.b32 %r1152, %r1151, 63; + add.s32 %r1153, %r1891, %r1152; + mul.wide.u32 %rd266, %r1153, 2; + add.s64 %rd267, %rd11, %rd266; + ld.global.u16 %r1154, [%rd267]; + and.b32 %r1155, %r1154, 7; + shr.u64 %rd268, %rd265, %r1155; + sub.s32 %r1156, %r227, %r1150; + sub.s32 %r1157, %r1156, %r1155; + cvt.u32.u64 %r1158, %rd268; + shr.u32 %r1159, %r1154, 3; + and.b32 %r1160, %r1159, 15; + mov.u32 %r1161, -1; + shl.b32 %r1162, %r1161, %r1160; + not.b32 %r1163, %r1162; + and.b32 %r1164, %r1158, %r1163; + shr.u64 %rd474, %rd268, %r1160; + sub.s32 %r1917, %r1157, %r1160; + shr.u32 %r1165, %r1154, 7; + and.b32 %r1166, %r1165, 7; + shr.u32 %r1167, %r1154, 10; + and.b32 %r1168, %r1167, 7; + mov.u32 %r1169, 255; + shl.b32 %r1170, %r1169, %r1166; + not.b32 %r1171, %r1170; + and.b32 %r1172, %r1164, %r1171; + add.s32 %r1173, %r1168, %r1172; + add.s32 %r1174, %r1173, 1; + st.local.u16 [%rd25+2], %r1174; + shr.u32 %r1175, %r1154, 13; + shr.u32 %r1176, %r1164, %r1166; + add.s32 %r1177, %r1175, %r1176; + add.s32 %r1178, %r1177, 1; + st.local.u16 [%rd25+6], %r1178; + add.s32 %r1721, %r1721, 4; + setp.lt.u32 %p296, %r1721, %r820; + shl.b32 %r1179, %r327, 2; + and.b32 %r1180, %r1179, 896; + shl.b32 %r1181, %r327, 3; + and.b32 %r1182, %r1181, 128; + or.b32 %r1720, %r1182, %r1180; + @%p296 bra $L__BB27_57; + + mul.wide.u32 %rd271, %r1721, 2; + add.s64 %rd272, %rd13, %rd271; + mov.u16 %rs825, 0; + st.local.v2.u16 [%rd272], {%rs825, %rs825}; + setp.lt.u32 %p297, %r822, 3; + @%p297 bra $L__BB27_321; + + ld.param.u64 %rd451, [ j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_3]; + ld.param.u64 %rd450, [ j2k_htj2k_decode_codeblocks_multi_cleanup_only_param_5]; + cvta.to.global.u64 %rd40, %rd450; + cvta.to.global.u64 %rd41, %rd451; + mov.u32 %r1892, 2; + +$L__BB27_214: + shr.u32 %r1187, %r1892, 1; + mul.lo.s32 %r439, %r1187, %r841; + sub.s32 %r440, %r439, %r841; + mov.u32 %r1901, 0; + mov.u32 %r1902, %r1901; + mov.u32 %r1903, %r439; + +$L__BB27_215: + sub.s32 %r1188, %r1903, %r439; + add.s32 %r452, %r1188, %r440; + mul.wide.u32 %rd273, %r452, 2; + add.s64 %rd46, %rd13, %rd273; + ld.local.u16 %r1189, [%rd46]; + shl.b32 %r1190, %r1189, 2; + and.b32 %r1191, %r1190, 640; + or.b32 %r1192, %r1902, %r1191; + add.s32 %r1193, %r452, 2; + mul.wide.u32 %rd274, %r1193, 2; + add.s64 %rd47, %rd13, %rd274; + ld.local.u16 %r1194, [%rd47]; + shl.b32 %r1195, %r1194, 4; + and.b32 %r1196, %r1195, 512; + or.b32 %r453, %r1192, %r1196; + setp.gt.u32 %p298, %r1917, 31; + @%p298 bra $L__BB27_219; + +$L__BB27_216: + setp.eq.s32 %p299, %r1916, 0; + mov.u16 %rs1179, 0; + @%p299 bra $L__BB27_218; + + cvt.s64.s32 %rd275, %r1915; + add.s64 %rd276, %rd275, %rd3; + add.s64 %rd277, %rd1, %rd276; + ld.global.u8 %rs1179, [%rd277]; + +$L__BB27_218: + setp.ne.s32 %p301, %r1916, 0; + selp.b32 %r1197, -1, 0, %p301; + add.s32 %r1915, %r1915, %r1197; + add.s32 %r1198, %r1916, -1; + selp.b32 %r1916, 0, %r1198, %p299; + and.b16 %rs827, %rs1179, 255; + and.b16 %rs828, %rs1179, 127; + setp.eq.s16 %p302, %rs828, 127; + and.b16 %rs829, %rs1180, 255; + setp.ne.s16 %p303, %rs829, 0; + and.pred %p304, %p303, %p302; + selp.b32 %r1199, 7, 8, %p304; + cvt.u64.u16 %rd278, %rs1179; + and.b64 %rd279, %rd278, 255; + shl.b64 %rd280, %rd279, %r1917; + or.b64 %rd474, %rd280, %rd474; + add.s32 %r1917, %r1199, %r1917; + setp.gt.u16 %p305, %rs827, 143; + selp.u16 %rs1180, 1, 0, %p305; + setp.lt.u32 %p306, %r1917, 33; + @%p306 bra $L__BB27_216; + +$L__BB27_219: + cvt.u32.u64 %r1200, %rd474; + and.b32 %r1201, %r1200, 127; + add.s32 %r1202, %r1201, %r453; + mul.wide.u32 %rd281, %r1202, 2; + add.s64 %rd282, %rd41, %rd281; + ld.global.u16 %r1969, [%rd282]; + setp.ne.s32 %p307, %r453, 0; + @%p307 bra $L__BB27_269; + + add.s32 %r464, %r1834, -2; + setp.eq.s32 %p308, %r464, -1; + selp.b32 %r1969, %r1969, 0, %p308; + setp.gt.s32 %p309, %r1834, 1; + mov.u32 %r1834, %r464; + @%p309 bra $L__BB27_269; + + setp.ne.s32 %p310, %r1838, 0; + @%p310 bra $L__BB27_268; + + mov.u32 %r1838, 0; + +$L__BB27_223: + setp.gt.u32 %p311, %r1838, 7; + @%p311 bra $L__BB27_268; + + cvt.u64.u32 %rd52, %r1837; + mul.wide.u32 %rd283, %r1837, 4; + add.s64 %rd285, %rd133, %rd283; + ld.global.nc.u32 %r470, [%rd285]; + and.b16 %rs830, %rs1048, 255; + setp.ne.s16 %p312, %rs830, 0; + @%p312 bra $L__BB27_228; + + setp.eq.s32 %p313, %r1747, 0; + mov.u16 %rs1184, 255; + @%p313 bra $L__BB27_227; + + cvt.u64.u32 %rd286, %r1746; + add.s64 %rd287, %rd286, %rd3; + add.s64 %rd288, %rd1, %rd287; + ld.global.u8 %rs1184, [%rd288]; + +$L__BB27_227: + setp.ne.s32 %p315, %r1747, 0; + selp.u32 %r1204, 1, 0, %p315; + add.s32 %r1746, %r1746, %r1204; + add.s32 %r1205, %r1747, -1; + selp.b32 %r1747, 0, %r1205, %p313; + setp.eq.s32 %p316, %r1747, 0; + or.b16 %rs832, %rs1184, 15; + selp.b16 %rs1014, %rs832, %rs1184, %p316; + and.b16 %rs833, %rs1014, 255; + mov.u16 %rs834, 8; + sub.s16 %rs1048, %rs834, %rs1013; + setp.eq.s16 %p317, %rs833, 255; + selp.u16 %rs1013, 1, 0, %p317; + +$L__BB27_228: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1206, %rs1048; + and.b32 %r1207, %r1206, 255; + mov.u32 %r1208, 1; + shl.b32 %r1209, %r1208, %r1207; + cvt.u32.u16 %r1210, %rs1014; + and.b32 %r1211, %r1209, %r1210; + and.b32 %r475, %r1211, 255; + setp.eq.s32 %p318, %r475, 0; + @%p318 bra $L__BB27_230; + + add.s32 %r1212, %r1837, 1; + min.u32 %r1958, %r1212, 12; + mov.u32 %r1213, -1; + shl.b32 %r1214, %r1213, %r470; + shl.b32 %r1215, %r1214, 1; + xor.b32 %r1959, %r1215, -2; + bra.uni $L__BB27_267; + +$L__BB27_230: + add.s64 %rd289, %rd52, -3; + setp.gt.u64 %p319, %rd289, 9; + mov.u32 %r1955, 0; + @%p319 bra $L__BB27_266; + + max.u32 %r478, %r470, 1; + add.s32 %r1219, %r478, -1; + and.b32 %r479, %r478, 3; + setp.lt.u32 %p320, %r1219, 3; + mov.u32 %r1955, 0; + @%p320 bra $L__BB27_250; + + sub.s32 %r1927, %r478, %r479; + mov.u32 %r1955, 0; + +$L__BB27_233: + and.b16 %rs836, %rs1048, 255; + setp.ne.s16 %p321, %rs836, 0; + @%p321 bra $L__BB27_237; + + setp.eq.s32 %p322, %r1747, 0; + mov.u16 %rs1191, 255; + @%p322 bra $L__BB27_236; + + cvt.u64.u32 %rd290, %r1746; + add.s64 %rd291, %rd290, %rd3; + add.s64 %rd292, %rd1, %rd291; + ld.global.u8 %rs1191, [%rd292]; + +$L__BB27_236: + setp.ne.s32 %p324, %r1747, 0; + selp.u32 %r1221, 1, 0, %p324; + add.s32 %r1746, %r1746, %r1221; + add.s32 %r1222, %r1747, -1; + selp.b32 %r1747, 0, %r1222, %p322; + setp.eq.s32 %p325, %r1747, 0; + or.b16 %rs838, %rs1191, 15; + selp.b16 %rs1014, %rs838, %rs1191, %p325; + and.b16 %rs839, %rs1014, 255; + mov.u16 %rs840, 8; + sub.s16 %rs1048, %rs840, %rs1013; + setp.eq.s16 %p326, %rs839, 255; + selp.u16 %rs1013, 1, 0, %p326; + +$L__BB27_237: + add.s16 %rs1198, %rs1048, -1; + and.b16 %rs841, %rs1198, 255; + cvt.u32.u16 %r1223, %rs1198; + and.b32 %r1224, %r1223, 255; + cvt.u32.u16 %r1225, %rs1014; + and.b32 %r1933, %r1225, 255; + shr.u32 %r1226, %r1933, %r1224; + and.b32 %r1227, %r1226, 1; + bfi.b32 %r490, %r1955, %r1227, 1, 31; + setp.ne.s16 %p327, %rs841, 0; + @%p327 bra $L__BB27_241; + + setp.eq.s32 %p328, %r1747, 0; + mov.u16 %rs1195, 255; + @%p328 bra $L__BB27_240; + + cvt.u64.u32 %rd293, %r1746; + add.s64 %rd294, %rd293, %rd3; + add.s64 %rd295, %rd1, %rd294; + ld.global.u8 %rs1195, [%rd295]; + +$L__BB27_240: + setp.ne.s32 %p330, %r1747, 0; + selp.u32 %r1228, 1, 0, %p330; + add.s32 %r1746, %r1746, %r1228; + add.s32 %r1229, %r1747, -1; + selp.b32 %r1747, 0, %r1229, %p328; + setp.eq.s32 %p331, %r1747, 0; + or.b16 %rs843, %rs1195, 15; + selp.b16 %rs1014, %rs843, %rs1195, %p331; + and.b16 %rs844, %rs1014, 255; + mov.u16 %rs845, 8; + sub.s16 %rs1198, %rs845, %rs1013; + setp.eq.s16 %p332, %rs844, 255; + selp.u16 %rs1013, 1, 0, %p332; + cvt.u32.u16 %r1230, %rs1014; + and.b32 %r1933, %r1230, 255; + +$L__BB27_241: + add.s16 %rs1202, %rs1198, -1; + and.b16 %rs846, %rs1202, 255; + cvt.u32.u16 %r1231, %rs1202; + and.b32 %r1232, %r1231, 255; + shr.u32 %r1233, %r1933, %r1232; + and.b32 %r1234, %r1233, 1; + bfi.b32 %r497, %r490, %r1234, 1, 31; + setp.ne.s16 %p333, %rs846, 0; + @%p333 bra $L__BB27_245; + + setp.eq.s32 %p334, %r1747, 0; + mov.u16 %rs1199, 255; + @%p334 bra $L__BB27_244; + + cvt.u64.u32 %rd296, %r1746; + add.s64 %rd297, %rd296, %rd3; + add.s64 %rd298, %rd1, %rd297; + ld.global.u8 %rs1199, [%rd298]; + +$L__BB27_244: + setp.ne.s32 %p336, %r1747, 0; + selp.u32 %r1235, 1, 0, %p336; + add.s32 %r1746, %r1746, %r1235; + add.s32 %r1236, %r1747, -1; + selp.b32 %r1747, 0, %r1236, %p334; + setp.eq.s32 %p337, %r1747, 0; + or.b16 %rs848, %rs1199, 15; + selp.b16 %rs1014, %rs848, %rs1199, %p337; + and.b16 %rs849, %rs1014, 255; + mov.u16 %rs850, 8; + sub.s16 %rs1202, %rs850, %rs1013; + setp.eq.s16 %p338, %rs849, 255; + selp.u16 %rs1013, 1, 0, %p338; + cvt.u32.u16 %r1237, %rs1014; + and.b32 %r1933, %r1237, 255; + +$L__BB27_245: + add.s16 %rs1206, %rs1202, -1; + and.b16 %rs851, %rs1206, 255; + cvt.u32.u16 %r1238, %rs1206; + and.b32 %r1239, %r1238, 255; + shr.u32 %r1240, %r1933, %r1239; + and.b32 %r1241, %r1240, 1; + bfi.b32 %r504, %r497, %r1241, 1, 31; + setp.ne.s16 %p339, %rs851, 0; + @%p339 bra $L__BB27_249; + + setp.eq.s32 %p340, %r1747, 0; + mov.u16 %rs1203, 255; + @%p340 bra $L__BB27_248; + + cvt.u64.u32 %rd299, %r1746; + add.s64 %rd300, %rd299, %rd3; + add.s64 %rd301, %rd1, %rd300; + ld.global.u8 %rs1203, [%rd301]; + +$L__BB27_248: + setp.ne.s32 %p342, %r1747, 0; + selp.u32 %r1242, 1, 0, %p342; + add.s32 %r1746, %r1746, %r1242; + add.s32 %r1243, %r1747, -1; + selp.b32 %r1747, 0, %r1243, %p340; + setp.eq.s32 %p343, %r1747, 0; + or.b16 %rs853, %rs1203, 15; + selp.b16 %rs1014, %rs853, %rs1203, %p343; + and.b16 %rs854, %rs1014, 255; + mov.u16 %rs855, 8; + sub.s16 %rs1206, %rs855, %rs1013; + setp.eq.s16 %p344, %rs854, 255; + selp.u16 %rs1013, 1, 0, %p344; + cvt.u32.u16 %r1244, %rs1014; + and.b32 %r1933, %r1244, 255; + +$L__BB27_249: + add.s16 %rs1048, %rs1206, -1; + cvt.u32.u16 %r1245, %rs1048; + and.b32 %r1246, %r1245, 255; + shr.u32 %r1247, %r1933, %r1246; + and.b32 %r1248, %r1247, 1; + bfi.b32 %r1955, %r504, %r1248, 1, 31; + add.s32 %r1927, %r1927, -4; + setp.ne.s32 %p345, %r1927, 0; + @%p345 bra $L__BB27_233; + +$L__BB27_250: + setp.eq.s32 %p346, %r479, 0; + @%p346 bra $L__BB27_266; + + and.b16 %rs856, %rs1048, 255; + setp.ne.s16 %p347, %rs856, 0; + @%p347 bra $L__BB27_255; + + setp.eq.s32 %p348, %r1747, 0; + mov.u16 %rs1213, 255; + @%p348 bra $L__BB27_254; + + cvt.u64.u32 %rd302, %r1746; + add.s64 %rd303, %rd302, %rd3; + add.s64 %rd304, %rd1, %rd303; + ld.global.u8 %rs1213, [%rd304]; + +$L__BB27_254: + setp.ne.s32 %p350, %r1747, 0; + selp.u32 %r1249, 1, 0, %p350; + add.s32 %r1746, %r1746, %r1249; + add.s32 %r1250, %r1747, -1; + selp.b32 %r1747, 0, %r1250, %p348; + setp.eq.s32 %p351, %r1747, 0; + or.b16 %rs858, %rs1213, 15; + selp.b16 %rs1014, %rs858, %rs1213, %p351; + and.b16 %rs859, %rs1014, 255; + mov.u16 %rs860, 8; + sub.s16 %rs1048, %rs860, %rs1013; + setp.eq.s16 %p352, %rs859, 255; + selp.u16 %rs1013, 1, 0, %p352; + +$L__BB27_255: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1251, %rs1048; + and.b32 %r1252, %r1251, 255; + cvt.u32.u16 %r1253, %rs1014; + and.b32 %r1950, %r1253, 255; + shr.u32 %r1254, %r1950, %r1252; + and.b32 %r1255, %r1254, 1; + bfi.b32 %r1955, %r1955, %r1255, 1, 31; + setp.eq.s32 %p353, %r479, 1; + @%p353 bra $L__BB27_266; + + and.b16 %rs861, %rs1048, 255; + setp.ne.s16 %p354, %rs861, 0; + @%p354 bra $L__BB27_260; + + setp.eq.s32 %p355, %r1747, 0; + mov.u16 %rs1217, 255; + @%p355 bra $L__BB27_259; + + cvt.u64.u32 %rd305, %r1746; + add.s64 %rd306, %rd305, %rd3; + add.s64 %rd307, %rd1, %rd306; + ld.global.u8 %rs1217, [%rd307]; + +$L__BB27_259: + setp.ne.s32 %p357, %r1747, 0; + selp.u32 %r1256, 1, 0, %p357; + add.s32 %r1746, %r1746, %r1256; + add.s32 %r1257, %r1747, -1; + selp.b32 %r1747, 0, %r1257, %p355; + setp.eq.s32 %p358, %r1747, 0; + or.b16 %rs863, %rs1217, 15; + selp.b16 %rs1014, %rs863, %rs1217, %p358; + and.b16 %rs864, %rs1014, 255; + mov.u16 %rs865, 8; + sub.s16 %rs1048, %rs865, %rs1013; + setp.eq.s16 %p359, %rs864, 255; + selp.u16 %rs1013, 1, 0, %p359; + cvt.u32.u16 %r1258, %rs1014; + and.b32 %r1950, %r1258, 255; + +$L__BB27_260: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1259, %rs1048; + and.b32 %r1260, %r1259, 255; + shr.u32 %r1261, %r1950, %r1260; + and.b32 %r1262, %r1261, 1; + bfi.b32 %r1955, %r1955, %r1262, 1, 31; + setp.eq.s32 %p360, %r479, 2; + @%p360 bra $L__BB27_266; + + and.b16 %rs866, %rs1048, 255; + setp.ne.s16 %p361, %rs866, 0; + @%p361 bra $L__BB27_265; + + setp.eq.s32 %p362, %r1747, 0; + mov.u16 %rs1221, 255; + @%p362 bra $L__BB27_264; + + cvt.u64.u32 %rd308, %r1746; + add.s64 %rd309, %rd308, %rd3; + add.s64 %rd310, %rd1, %rd309; + ld.global.u8 %rs1221, [%rd310]; + +$L__BB27_264: + setp.ne.s32 %p364, %r1747, 0; + selp.u32 %r1263, 1, 0, %p364; + add.s32 %r1746, %r1746, %r1263; + add.s32 %r1264, %r1747, -1; + selp.b32 %r1747, 0, %r1264, %p362; + setp.eq.s32 %p365, %r1747, 0; + or.b16 %rs868, %rs1221, 15; + selp.b16 %rs1014, %rs868, %rs1221, %p365; + and.b16 %rs869, %rs1014, 255; + mov.u16 %rs870, 8; + sub.s16 %rs1048, %rs870, %rs1013; + setp.eq.s16 %p366, %rs869, 255; + selp.u16 %rs1013, 1, 0, %p366; + cvt.u32.u16 %r1265, %rs1014; + and.b32 %r1950, %r1265, 255; + +$L__BB27_265: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1266, %rs1048; + and.b32 %r1267, %r1266, 255; + shr.u32 %r1268, %r1950, %r1267; + and.b32 %r1269, %r1268, 1; + bfi.b32 %r1955, %r1955, %r1269, 1, 31; + +$L__BB27_266: + shl.b32 %r1270, %r1955, 1; + or.b32 %r1959, %r1270, 1; + add.s32 %r1271, %r1837, -1; + setp.eq.s32 %p367, %r1837, 0; + selp.b32 %r1958, 0, %r1271, %p367; + +$L__BB27_267: + mul.lo.s32 %r1272, %r1838, 7; + cvt.u64.u32 %rd311, %r1959; + shl.b64 %rd312, %rd311, %r1272; + or.b64 %rd465, %rd312, %rd465; + setp.ne.s32 %p368, %r1837, 12; + setp.ne.s32 %p369, %r475, 0; + or.pred %p370, %p368, %p369; + add.s32 %r1838, %r1838, 1; + setp.lt.u32 %p371, %r1838, 8; + or.pred %p372, %p371, %p370; + mov.u32 %r1837, %r1958; + @%p372 bra $L__BB27_223; + +$L__BB27_268: + cvt.u32.u64 %r1273, %rd465; + and.b32 %r1834, %r1273, 127; + shr.u64 %rd465, %rd465, 7; + add.s32 %r1838, %r1838, -1; + +$L__BB27_269: + mul.wide.u32 %rd313, %r1903, 2; + add.s64 %rd314, %rd13, %rd313; + st.local.u16 [%rd314], %r1969; + shl.b32 %r1274, %r1969, 2; + shl.b32 %r1275, %r1969, 1; + or.b32 %r1276, %r1274, %r1275; + and.b32 %r1277, %r1276, 256; + ld.local.u16 %r1278, [%rd46]; + and.b32 %r1279, %r1278, 128; + or.b32 %r1280, %r1277, %r1279; + ld.local.u16 %r1281, [%rd47]; + shl.b32 %r1282, %r1281, 2; + and.b32 %r1283, %r1282, 640; + or.b32 %r1284, %r1280, %r1283; + add.s32 %r1285, %r452, 4; + mul.wide.u32 %rd315, %r1285, 2; + add.s64 %rd316, %rd13, %rd315; + ld.local.u16 %r1286, [%rd316]; + shl.b32 %r1287, %r1286, 4; + and.b32 %r1288, %r1287, 512; + or.b32 %r1289, %r1284, %r1288; + and.b32 %r1290, %r1969, 7; + shr.u64 %rd57, %rd474, %r1290; + sub.s32 %r561, %r1917, %r1290; + cvt.u32.u64 %r1291, %rd57; + and.b32 %r1292, %r1291, 127; + or.b32 %r1293, %r1289, %r1292; + mul.wide.u32 %rd317, %r1293, 2; + add.s64 %rd318, %rd41, %rd317; + ld.global.u16 %r2021, [%rd318]; + setp.ne.s32 %p373, %r1289, 0; + add.s32 %r563, %r1901, 2; + setp.ge.u32 %p374, %r563, %r820; + or.pred %p375, %p374, %p373; + @%p375 bra $L__BB27_319; + + add.s32 %r564, %r1834, -2; + setp.eq.s32 %p376, %r564, -1; + selp.b32 %r2021, %r2021, 0, %p376; + setp.gt.s32 %p377, %r1834, 1; + mov.u32 %r1834, %r564; + @%p377 bra $L__BB27_319; + + setp.ne.s32 %p378, %r1838, 0; + @%p378 bra $L__BB27_318; + + mov.u32 %r1838, 0; + +$L__BB27_273: + setp.gt.u32 %p379, %r1838, 7; + @%p379 bra $L__BB27_318; + + cvt.u64.u32 %rd59, %r1837; + mul.wide.u32 %rd319, %r1837, 4; + add.s64 %rd321, %rd133, %rd319; + ld.global.nc.u32 %r570, [%rd321]; + and.b16 %rs871, %rs1048, 255; + setp.ne.s16 %p380, %rs871, 0; + @%p380 bra $L__BB27_278; + + setp.eq.s32 %p381, %r1747, 0; + mov.u16 %rs1240, 255; + @%p381 bra $L__BB27_277; + + cvt.u64.u32 %rd322, %r1746; + add.s64 %rd323, %rd322, %rd3; + add.s64 %rd324, %rd1, %rd323; + ld.global.u8 %rs1240, [%rd324]; + +$L__BB27_277: + setp.ne.s32 %p383, %r1747, 0; + selp.u32 %r1295, 1, 0, %p383; + add.s32 %r1746, %r1746, %r1295; + add.s32 %r1296, %r1747, -1; + selp.b32 %r1747, 0, %r1296, %p381; + setp.eq.s32 %p384, %r1747, 0; + or.b16 %rs873, %rs1240, 15; + selp.b16 %rs1014, %rs873, %rs1240, %p384; + and.b16 %rs874, %rs1014, 255; + mov.u16 %rs875, 8; + sub.s16 %rs1048, %rs875, %rs1013; + setp.eq.s16 %p385, %rs874, 255; + selp.u16 %rs1013, 1, 0, %p385; + +$L__BB27_278: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1297, %rs1048; + and.b32 %r1298, %r1297, 255; + mov.u32 %r1299, 1; + shl.b32 %r1300, %r1299, %r1298; + cvt.u32.u16 %r1301, %rs1014; + and.b32 %r1302, %r1300, %r1301; + and.b32 %r575, %r1302, 255; + setp.eq.s32 %p386, %r575, 0; + @%p386 bra $L__BB27_280; + + add.s32 %r1303, %r1837, 1; + min.u32 %r2010, %r1303, 12; + mov.u32 %r1304, -1; + shl.b32 %r1305, %r1304, %r570; + shl.b32 %r1306, %r1305, 1; + xor.b32 %r2011, %r1306, -2; + bra.uni $L__BB27_317; + +$L__BB27_280: + add.s64 %rd325, %rd59, -3; + setp.gt.u64 %p387, %rd325, 9; + mov.u32 %r2007, 0; + @%p387 bra $L__BB27_316; + + max.u32 %r578, %r570, 1; + add.s32 %r1310, %r578, -1; + and.b32 %r579, %r578, 3; + setp.lt.u32 %p388, %r1310, 3; + mov.u32 %r2007, 0; + @%p388 bra $L__BB27_300; + + sub.s32 %r1979, %r578, %r579; + mov.u32 %r2007, 0; + +$L__BB27_283: + and.b16 %rs877, %rs1048, 255; + setp.ne.s16 %p389, %rs877, 0; + @%p389 bra $L__BB27_287; + + setp.eq.s32 %p390, %r1747, 0; + mov.u16 %rs1247, 255; + @%p390 bra $L__BB27_286; + + cvt.u64.u32 %rd326, %r1746; + add.s64 %rd327, %rd326, %rd3; + add.s64 %rd328, %rd1, %rd327; + ld.global.u8 %rs1247, [%rd328]; + +$L__BB27_286: + setp.ne.s32 %p392, %r1747, 0; + selp.u32 %r1312, 1, 0, %p392; + add.s32 %r1746, %r1746, %r1312; + add.s32 %r1313, %r1747, -1; + selp.b32 %r1747, 0, %r1313, %p390; + setp.eq.s32 %p393, %r1747, 0; + or.b16 %rs879, %rs1247, 15; + selp.b16 %rs1014, %rs879, %rs1247, %p393; + and.b16 %rs880, %rs1014, 255; + mov.u16 %rs881, 8; + sub.s16 %rs1048, %rs881, %rs1013; + setp.eq.s16 %p394, %rs880, 255; + selp.u16 %rs1013, 1, 0, %p394; + +$L__BB27_287: + add.s16 %rs1254, %rs1048, -1; + and.b16 %rs882, %rs1254, 255; + cvt.u32.u16 %r1314, %rs1254; + and.b32 %r1315, %r1314, 255; + cvt.u32.u16 %r1316, %rs1014; + and.b32 %r1985, %r1316, 255; + shr.u32 %r1317, %r1985, %r1315; + and.b32 %r1318, %r1317, 1; + bfi.b32 %r590, %r2007, %r1318, 1, 31; + setp.ne.s16 %p395, %rs882, 0; + @%p395 bra $L__BB27_291; + + setp.eq.s32 %p396, %r1747, 0; + mov.u16 %rs1251, 255; + @%p396 bra $L__BB27_290; + + cvt.u64.u32 %rd329, %r1746; + add.s64 %rd330, %rd329, %rd3; + add.s64 %rd331, %rd1, %rd330; + ld.global.u8 %rs1251, [%rd331]; + +$L__BB27_290: + setp.ne.s32 %p398, %r1747, 0; + selp.u32 %r1319, 1, 0, %p398; + add.s32 %r1746, %r1746, %r1319; + add.s32 %r1320, %r1747, -1; + selp.b32 %r1747, 0, %r1320, %p396; + setp.eq.s32 %p399, %r1747, 0; + or.b16 %rs884, %rs1251, 15; + selp.b16 %rs1014, %rs884, %rs1251, %p399; + and.b16 %rs885, %rs1014, 255; + mov.u16 %rs886, 8; + sub.s16 %rs1254, %rs886, %rs1013; + setp.eq.s16 %p400, %rs885, 255; + selp.u16 %rs1013, 1, 0, %p400; + cvt.u32.u16 %r1321, %rs1014; + and.b32 %r1985, %r1321, 255; + +$L__BB27_291: + add.s16 %rs1258, %rs1254, -1; + and.b16 %rs887, %rs1258, 255; + cvt.u32.u16 %r1322, %rs1258; + and.b32 %r1323, %r1322, 255; + shr.u32 %r1324, %r1985, %r1323; + and.b32 %r1325, %r1324, 1; + bfi.b32 %r597, %r590, %r1325, 1, 31; + setp.ne.s16 %p401, %rs887, 0; + @%p401 bra $L__BB27_295; + + setp.eq.s32 %p402, %r1747, 0; + mov.u16 %rs1255, 255; + @%p402 bra $L__BB27_294; + + cvt.u64.u32 %rd332, %r1746; + add.s64 %rd333, %rd332, %rd3; + add.s64 %rd334, %rd1, %rd333; + ld.global.u8 %rs1255, [%rd334]; + +$L__BB27_294: + setp.ne.s32 %p404, %r1747, 0; + selp.u32 %r1326, 1, 0, %p404; + add.s32 %r1746, %r1746, %r1326; + add.s32 %r1327, %r1747, -1; + selp.b32 %r1747, 0, %r1327, %p402; + setp.eq.s32 %p405, %r1747, 0; + or.b16 %rs889, %rs1255, 15; + selp.b16 %rs1014, %rs889, %rs1255, %p405; + and.b16 %rs890, %rs1014, 255; + mov.u16 %rs891, 8; + sub.s16 %rs1258, %rs891, %rs1013; + setp.eq.s16 %p406, %rs890, 255; + selp.u16 %rs1013, 1, 0, %p406; + cvt.u32.u16 %r1328, %rs1014; + and.b32 %r1985, %r1328, 255; + +$L__BB27_295: + add.s16 %rs1262, %rs1258, -1; + and.b16 %rs892, %rs1262, 255; + cvt.u32.u16 %r1329, %rs1262; + and.b32 %r1330, %r1329, 255; + shr.u32 %r1331, %r1985, %r1330; + and.b32 %r1332, %r1331, 1; + bfi.b32 %r604, %r597, %r1332, 1, 31; + setp.ne.s16 %p407, %rs892, 0; + @%p407 bra $L__BB27_299; + + setp.eq.s32 %p408, %r1747, 0; + mov.u16 %rs1259, 255; + @%p408 bra $L__BB27_298; + + cvt.u64.u32 %rd335, %r1746; + add.s64 %rd336, %rd335, %rd3; + add.s64 %rd337, %rd1, %rd336; + ld.global.u8 %rs1259, [%rd337]; + +$L__BB27_298: + setp.ne.s32 %p410, %r1747, 0; + selp.u32 %r1333, 1, 0, %p410; + add.s32 %r1746, %r1746, %r1333; + add.s32 %r1334, %r1747, -1; + selp.b32 %r1747, 0, %r1334, %p408; + setp.eq.s32 %p411, %r1747, 0; + or.b16 %rs894, %rs1259, 15; + selp.b16 %rs1014, %rs894, %rs1259, %p411; + and.b16 %rs895, %rs1014, 255; + mov.u16 %rs896, 8; + sub.s16 %rs1262, %rs896, %rs1013; + setp.eq.s16 %p412, %rs895, 255; + selp.u16 %rs1013, 1, 0, %p412; + cvt.u32.u16 %r1335, %rs1014; + and.b32 %r1985, %r1335, 255; + +$L__BB27_299: + add.s16 %rs1048, %rs1262, -1; + cvt.u32.u16 %r1336, %rs1048; + and.b32 %r1337, %r1336, 255; + shr.u32 %r1338, %r1985, %r1337; + and.b32 %r1339, %r1338, 1; + bfi.b32 %r2007, %r604, %r1339, 1, 31; + add.s32 %r1979, %r1979, -4; + setp.ne.s32 %p413, %r1979, 0; + @%p413 bra $L__BB27_283; + +$L__BB27_300: + setp.eq.s32 %p414, %r579, 0; + @%p414 bra $L__BB27_316; + + and.b16 %rs897, %rs1048, 255; + setp.ne.s16 %p415, %rs897, 0; + @%p415 bra $L__BB27_305; + + setp.eq.s32 %p416, %r1747, 0; + mov.u16 %rs1269, 255; + @%p416 bra $L__BB27_304; + + cvt.u64.u32 %rd338, %r1746; + add.s64 %rd339, %rd338, %rd3; + add.s64 %rd340, %rd1, %rd339; + ld.global.u8 %rs1269, [%rd340]; + +$L__BB27_304: + setp.ne.s32 %p418, %r1747, 0; + selp.u32 %r1340, 1, 0, %p418; + add.s32 %r1746, %r1746, %r1340; + add.s32 %r1341, %r1747, -1; + selp.b32 %r1747, 0, %r1341, %p416; + setp.eq.s32 %p419, %r1747, 0; + or.b16 %rs899, %rs1269, 15; + selp.b16 %rs1014, %rs899, %rs1269, %p419; + and.b16 %rs900, %rs1014, 255; + mov.u16 %rs901, 8; + sub.s16 %rs1048, %rs901, %rs1013; + setp.eq.s16 %p420, %rs900, 255; + selp.u16 %rs1013, 1, 0, %p420; + +$L__BB27_305: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1342, %rs1048; + and.b32 %r1343, %r1342, 255; + cvt.u32.u16 %r1344, %rs1014; + and.b32 %r2002, %r1344, 255; + shr.u32 %r1345, %r2002, %r1343; + and.b32 %r1346, %r1345, 1; + bfi.b32 %r2007, %r2007, %r1346, 1, 31; + setp.eq.s32 %p421, %r579, 1; + @%p421 bra $L__BB27_316; + + and.b16 %rs902, %rs1048, 255; + setp.ne.s16 %p422, %rs902, 0; + @%p422 bra $L__BB27_310; + + setp.eq.s32 %p423, %r1747, 0; + mov.u16 %rs1273, 255; + @%p423 bra $L__BB27_309; + + cvt.u64.u32 %rd341, %r1746; + add.s64 %rd342, %rd341, %rd3; + add.s64 %rd343, %rd1, %rd342; + ld.global.u8 %rs1273, [%rd343]; + +$L__BB27_309: + setp.ne.s32 %p425, %r1747, 0; + selp.u32 %r1347, 1, 0, %p425; + add.s32 %r1746, %r1746, %r1347; + add.s32 %r1348, %r1747, -1; + selp.b32 %r1747, 0, %r1348, %p423; + setp.eq.s32 %p426, %r1747, 0; + or.b16 %rs904, %rs1273, 15; + selp.b16 %rs1014, %rs904, %rs1273, %p426; + and.b16 %rs905, %rs1014, 255; + mov.u16 %rs906, 8; + sub.s16 %rs1048, %rs906, %rs1013; + setp.eq.s16 %p427, %rs905, 255; + selp.u16 %rs1013, 1, 0, %p427; + cvt.u32.u16 %r1349, %rs1014; + and.b32 %r2002, %r1349, 255; + +$L__BB27_310: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1350, %rs1048; + and.b32 %r1351, %r1350, 255; + shr.u32 %r1352, %r2002, %r1351; + and.b32 %r1353, %r1352, 1; + bfi.b32 %r2007, %r2007, %r1353, 1, 31; + setp.eq.s32 %p428, %r579, 2; + @%p428 bra $L__BB27_316; + + and.b16 %rs907, %rs1048, 255; + setp.ne.s16 %p429, %rs907, 0; + @%p429 bra $L__BB27_315; + + setp.eq.s32 %p430, %r1747, 0; + mov.u16 %rs1277, 255; + @%p430 bra $L__BB27_314; + + cvt.u64.u32 %rd344, %r1746; + add.s64 %rd345, %rd344, %rd3; + add.s64 %rd346, %rd1, %rd345; + ld.global.u8 %rs1277, [%rd346]; + +$L__BB27_314: + setp.ne.s32 %p432, %r1747, 0; + selp.u32 %r1354, 1, 0, %p432; + add.s32 %r1746, %r1746, %r1354; + add.s32 %r1355, %r1747, -1; + selp.b32 %r1747, 0, %r1355, %p430; + setp.eq.s32 %p433, %r1747, 0; + or.b16 %rs909, %rs1277, 15; + selp.b16 %rs1014, %rs909, %rs1277, %p433; + and.b16 %rs910, %rs1014, 255; + mov.u16 %rs911, 8; + sub.s16 %rs1048, %rs911, %rs1013; + setp.eq.s16 %p434, %rs910, 255; + selp.u16 %rs1013, 1, 0, %p434; + cvt.u32.u16 %r1356, %rs1014; + and.b32 %r2002, %r1356, 255; + +$L__BB27_315: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1357, %rs1048; + and.b32 %r1358, %r1357, 255; + shr.u32 %r1359, %r2002, %r1358; + and.b32 %r1360, %r1359, 1; + bfi.b32 %r2007, %r2007, %r1360, 1, 31; + +$L__BB27_316: + shl.b32 %r1361, %r2007, 1; + or.b32 %r2011, %r1361, 1; + add.s32 %r1362, %r1837, -1; + setp.eq.s32 %p435, %r1837, 0; + selp.b32 %r2010, 0, %r1362, %p435; + +$L__BB27_317: + mul.lo.s32 %r1363, %r1838, 7; + cvt.u64.u32 %rd347, %r2011; + shl.b64 %rd348, %rd347, %r1363; + or.b64 %rd465, %rd348, %rd465; + setp.ne.s32 %p436, %r1837, 12; + setp.ne.s32 %p437, %r575, 0; + or.pred %p438, %p436, %p437; + add.s32 %r1838, %r1838, 1; + setp.lt.u32 %p439, %r1838, 8; + or.pred %p440, %p439, %p438; + mov.u32 %r1837, %r2010; + @%p440 bra $L__BB27_273; + +$L__BB27_318: + cvt.u32.u64 %r1364, %rd465; + and.b32 %r1834, %r1364, 127; + shr.u64 %rd465, %rd465, 7; + add.s32 %r1838, %r1838, -1; + +$L__BB27_319: + setp.lt.u32 %p441, %r563, %r820; + selp.b32 %r1365, %r2021, 0, %p441; + add.s32 %r1366, %r1903, 2; + mul.wide.u32 %rd349, %r1366, 2; + add.s64 %rd350, %rd13, %rd349; + st.local.u16 [%rd350], %r1365; + shl.b32 %r1367, %r1365, 2; + shl.b32 %r1368, %r1365, 1; + or.b32 %r1369, %r1367, %r1368; + and.b32 %r1370, %r1369, 256; + ld.local.u16 %r1371, [%rd47]; + and.b32 %r1372, %r1371, 128; + or.b32 %r1902, %r1370, %r1372; + and.b32 %r1373, %r1365, 7; + shr.u64 %rd351, %rd57, %r1373; + sub.s32 %r1374, %r561, %r1373; + cvt.u32.u64 %r1375, %rd351; + shl.b32 %r1376, %r1969, 3; + and.b32 %r1377, %r1376, 64; + shl.b32 %r1378, %r1365, 4; + and.b32 %r1379, %r1378, 128; + or.b32 %r1380, %r1379, %r1377; + and.b32 %r1381, %r1375, 63; + or.b32 %r1382, %r1381, %r1380; + mul.wide.u32 %rd352, %r1382, 2; + add.s64 %rd353, %rd40, %rd352; + ld.global.u16 %r1383, [%rd353]; + and.b32 %r1384, %r1383, 7; + shr.u64 %rd354, %rd351, %r1384; + sub.s32 %r1385, %r1374, %r1384; + cvt.u32.u64 %r1386, %rd354; + shr.u32 %r1387, %r1383, 3; + and.b32 %r1388, %r1387, 15; + mov.u32 %r1389, -1; + shl.b32 %r1390, %r1389, %r1388; + not.b32 %r1391, %r1390; + and.b32 %r1392, %r1386, %r1391; + shr.u64 %rd474, %rd354, %r1388; + sub.s32 %r1917, %r1385, %r1388; + shr.u32 %r1393, %r1383, 7; + and.b32 %r1394, %r1393, 7; + shr.u32 %r1395, %r1383, 10; + and.b32 %r1396, %r1395, 7; + mov.u32 %r1397, 255; + shl.b32 %r1398, %r1397, %r1394; + not.b32 %r1399, %r1398; + and.b32 %r1400, %r1392, %r1399; + add.s32 %r1401, %r1400, %r1396; + add.s32 %r1402, %r1903, 1; + mul.wide.u32 %rd355, %r1402, 2; + add.s64 %rd356, %rd13, %rd355; + st.local.u16 [%rd356], %r1401; + shr.u32 %r1403, %r1383, 13; + shr.u32 %r1404, %r1392, %r1394; + add.s32 %r1405, %r1404, %r1403; + add.s32 %r1406, %r1903, 3; + mul.wide.u32 %rd357, %r1406, 2; + add.s64 %rd358, %rd13, %rd357; + st.local.u16 [%rd358], %r1405; + add.s32 %r1903, %r1903, 4; + add.s32 %r1901, %r1901, 4; + setp.lt.u32 %p442, %r1901, %r820; + @%p442 bra $L__BB27_215; + + mul.wide.u32 %rd359, %r1903, 2; + add.s64 %rd360, %rd13, %rd359; + mov.u16 %rs912, 0; + st.local.v2.u16 [%rd360], {%rs912, %rs912}; + add.s32 %r1892, %r1892, 2; + setp.lt.u32 %p443, %r1892, %r822; + @%p443 bra $L__BB27_214; + +$L__BB27_321: + add.s32 %r1407, %r820, 1; + shr.u32 %r1408, %r1407, 1; + add.s32 %r1409, %r1408, 2; + setp.gt.u32 %p444, %r1409, 130; + @%p444 bra $L__BB27_392; + bra.uni $L__BB27_322; + +$L__BB27_392: + mov.u32 %r1642, 2; + st.global.u32 [%rd4], %r1642; + mov.u32 %r1643, 12; + st.global.u32 [%rd4+4], %r1643; + mov.u32 %r1644, 0; + st.global.u32 [%rd4+8], %r1644; + st.global.u32 [%rd4+12], %r1644; + bra.uni $L__BB27_399; + +$L__BB27_322: + add.s32 %r666, %r827, 2; + mov.u32 %r1415, 29; + sub.s32 %r667, %r1415, %r827; + mov.u16 %rs1320, 0; + mov.u32 %r2022, 0; + mov.u64 %rd502, 0; + mov.u32 %r2023, %r2022; + mov.u32 %r2024, %r2022; + mov.u32 %r2025, %r13; + mov.u32 %r2090, %r2022; + mov.u32 %r2089, %r2022; + +$L__BB27_323: + mov.u32 %r670, %r2024; + mul.wide.u32 %rd362, %r2023, 2; + add.s64 %rd363, %rd13, %rd362; + ld.local.u16 %r674, [%rd363]; + ld.local.u16 %r675, [%rd363+2]; + setp.lt.u32 %p445, %r666, %r675; + @%p445 bra $L__BB27_391; + + and.b32 %r1417, %r674, 16; + setp.eq.s32 %p446, %r1417, 0; + mov.u32 %r2043, 0; + mov.u32 %r2035, %r2043; + @%p446 bra $L__BB27_330; + + setp.gt.u32 %p447, %r2089, 31; + @%p447 bra $L__BB27_329; + +$L__BB27_326: + setp.ge.u32 %p448, %r2090, %r16; + mov.u16 %rs1295, 255; + @%p448 bra $L__BB27_328; + + add.s32 %r678, %r2090, 1; + cvt.u64.u32 %rd364, %r2090; + add.s64 %rd365, %rd364, %rd3; + add.s64 %rd366, %rd1, %rd365; + ld.global.u8 %rs1295, [%rd366]; + mov.u32 %r2090, %r678; + +$L__BB27_328: + and.b16 %rs915, %rs1295, 255; + cvt.u64.u16 %rd367, %rs1295; + and.b64 %rd368, %rd367, 255; + shl.b64 %rd369, %rd368, %r2089; + or.b64 %rd502, %rd369, %rd502; + cvt.u32.u16 %r1418, %rs1320; + cvt.s32.s8 %r1419, %r1418; + mov.u32 %r1420, 8; + sub.s32 %r1421, %r1420, %r1419; + add.s32 %r2089, %r1421, %r2089; + setp.eq.s16 %p449, %rs915, 255; + selp.u16 %rs1320, 1, 0, %p449; + setp.lt.u32 %p450, %r2089, 33; + @%p450 bra $L__BB27_326; + +$L__BB27_329: + shr.u32 %r1422, %r674, 12; + and.b32 %r1423, %r1422, 1; + sub.s32 %r1424, %r675, %r1423; + shr.u64 %rd69, %rd502, %r1424; + sub.s32 %r2089, %r2089, %r1424; + cvt.u32.u64 %r1425, %rd502; + shl.b32 %r1426, %r1425, 31; + setp.eq.s32 %p451, %r1424, 0; + mov.u32 %r1427, -1; + shl.b32 %r1428, %r1427, %r1424; + not.b32 %r1429, %r1428; + selp.b32 %r1430, 0, %r1429, %p451; + and.b32 %r1431, %r1430, %r1425; + shr.u32 %r1432, %r674, 8; + and.b32 %r1433, %r1432, 1; + shl.b32 %r1434, %r1433, %r1424; + or.b32 %r1435, %r1434, %r1431; + or.b32 %r1436, %r1435, 1; + add.s32 %r1437, %r1436, 2; + shl.b32 %r1438, %r1437, %r667; + or.b32 %r2035, %r1438, %r1426; + mov.u64 %rd502, %rd69; + +$L__BB27_330: + mul.wide.u32 %rd370, %r2025, 4; + add.s64 %rd371, %rd2, %rd370; + st.global.u32 [%rd371], %r2035; + and.b32 %r1441, %r674, 32; + setp.eq.s32 %p452, %r1441, 0; + mov.u32 %r2044, %r2043; + @%p452 bra $L__BB27_336; + + setp.gt.u32 %p453, %r2089, 31; + @%p453 bra $L__BB27_335; + +$L__BB27_332: + setp.ge.u32 %p454, %r2090, %r16; + mov.u16 %rs1299, 255; + @%p454 bra $L__BB27_334; + + add.s32 %r690, %r2090, 1; + cvt.u64.u32 %rd372, %r2090; + add.s64 %rd373, %rd372, %rd3; + add.s64 %rd374, %rd1, %rd373; + ld.global.u8 %rs1299, [%rd374]; + mov.u32 %r2090, %r690; + +$L__BB27_334: + and.b16 %rs917, %rs1299, 255; + cvt.u64.u16 %rd375, %rs1299; + and.b64 %rd376, %rd375, 255; + shl.b64 %rd377, %rd376, %r2089; + or.b64 %rd502, %rd377, %rd502; + cvt.u32.u16 %r1442, %rs1320; + cvt.s32.s8 %r1443, %r1442; + mov.u32 %r1444, 8; + sub.s32 %r1445, %r1444, %r1443; + add.s32 %r2089, %r1445, %r2089; + setp.eq.s16 %p455, %rs917, 255; + selp.u16 %rs1320, 1, 0, %p455; + setp.lt.u32 %p456, %r2089, 33; + @%p456 bra $L__BB27_332; + +$L__BB27_335: + shr.u32 %r1446, %r674, 13; + and.b32 %r1447, %r1446, 1; + sub.s32 %r1448, %r675, %r1447; + shr.u64 %rd74, %rd502, %r1448; + sub.s32 %r2089, %r2089, %r1448; + cvt.u32.u64 %r1449, %rd502; + shl.b32 %r1450, %r1449, 31; + setp.eq.s32 %p457, %r1448, 0; + mov.u32 %r1451, -1; + shl.b32 %r1452, %r1451, %r1448; + not.b32 %r1453, %r1452; + selp.b32 %r1454, 0, %r1453, %p457; + and.b32 %r1455, %r1454, %r1449; + shr.u32 %r1456, %r674, 9; + and.b32 %r1457, %r1456, 1; + shl.b32 %r1458, %r1457, %r1448; + or.b32 %r1459, %r1458, %r1455; + or.b32 %r2044, %r1459, 1; + add.s32 %r1460, %r2044, 2; + shl.b32 %r1461, %r1460, %r667; + or.b32 %r2043, %r1461, %r1450; + mov.u64 %rd502, %rd74; + +$L__BB27_336: + setp.lt.u32 %p458, %r822, 2; + @%p458 bra $L__BB27_338; + + add.s32 %r1462, %r2025, %r12; + mul.wide.u32 %rd378, %r1462, 4; + add.s64 %rd379, %rd2, %rd378; + st.global.u32 [%rd379], %r2043; + +$L__BB27_338: + or.b32 %r1463, %r2044, %r2022; + add.u64 %rd76, %SPL, 6192; + mul.wide.u32 %rd381, %r670, 4; + add.s64 %rd382, %rd76, %rd381; + st.local.u32 [%rd382], %r1463; + add.s32 %r702, %r2025, 1; + add.s32 %r1464, %r2023, 1; + setp.lt.u32 %p459, %r1464, %r820; + @%p459 bra $L__BB27_340; + bra.uni $L__BB27_339; + +$L__BB27_340: + and.b32 %r1467, %r674, 64; + setp.eq.s32 %p460, %r1467, 0; + mov.u32 %r2060, 0; + mov.u32 %r2052, %r2060; + @%p460 bra $L__BB27_346; + + setp.gt.u32 %p461, %r2089, 31; + @%p461 bra $L__BB27_345; + +$L__BB27_342: + setp.ge.u32 %p462, %r2090, %r16; + mov.u16 %rs1303, 255; + @%p462 bra $L__BB27_344; + + add.s32 %r705, %r2090, 1; + cvt.u64.u32 %rd383, %r2090; + add.s64 %rd384, %rd383, %rd3; + add.s64 %rd385, %rd1, %rd384; + ld.global.u8 %rs1303, [%rd385]; + mov.u32 %r2090, %r705; + +$L__BB27_344: + and.b16 %rs919, %rs1303, 255; + cvt.u64.u16 %rd386, %rs1303; + and.b64 %rd387, %rd386, 255; + shl.b64 %rd388, %rd387, %r2089; + or.b64 %rd502, %rd388, %rd502; + cvt.u32.u16 %r1468, %rs1320; + cvt.s32.s8 %r1469, %r1468; + mov.u32 %r1470, 8; + sub.s32 %r1471, %r1470, %r1469; + add.s32 %r2089, %r1471, %r2089; + setp.eq.s16 %p463, %rs919, 255; + selp.u16 %rs1320, 1, 0, %p463; + setp.lt.u32 %p464, %r2089, 33; + @%p464 bra $L__BB27_342; + +$L__BB27_345: + shr.u32 %r1472, %r674, 14; + and.b32 %r1473, %r1472, 1; + sub.s32 %r1474, %r675, %r1473; + shr.u64 %rd80, %rd502, %r1474; + sub.s32 %r2089, %r2089, %r1474; + cvt.u32.u64 %r1475, %rd502; + shl.b32 %r1476, %r1475, 31; + setp.eq.s32 %p465, %r1474, 0; + mov.u32 %r1477, -1; + shl.b32 %r1478, %r1477, %r1474; + not.b32 %r1479, %r1478; + selp.b32 %r1480, 0, %r1479, %p465; + and.b32 %r1481, %r1480, %r1475; + shr.u32 %r1482, %r674, 10; + and.b32 %r1483, %r1482, 1; + shl.b32 %r1484, %r1483, %r1474; + or.b32 %r1485, %r1484, %r1481; + or.b32 %r1486, %r1485, 1; + add.s32 %r1487, %r1486, 2; + shl.b32 %r1488, %r1487, %r667; + or.b32 %r2052, %r1488, %r1476; + mov.u64 %rd502, %rd80; + +$L__BB27_346: + mul.wide.u32 %rd389, %r702, 4; + add.s64 %rd390, %rd2, %rd389; + st.global.u32 [%rd390], %r2052; + and.b32 %r1491, %r674, 128; + setp.eq.s32 %p466, %r1491, 0; + mov.u32 %r2022, %r2060; + @%p466 bra $L__BB27_352; + + setp.gt.u32 %p467, %r2089, 31; + @%p467 bra $L__BB27_351; + +$L__BB27_348: + setp.ge.u32 %p468, %r2090, %r16; + mov.u16 %rs1307, 255; + @%p468 bra $L__BB27_350; + + add.s32 %r717, %r2090, 1; + cvt.u64.u32 %rd391, %r2090; + add.s64 %rd392, %rd391, %rd3; + add.s64 %rd393, %rd1, %rd392; + ld.global.u8 %rs1307, [%rd393]; + mov.u32 %r2090, %r717; + +$L__BB27_350: + and.b16 %rs921, %rs1307, 255; + cvt.u64.u16 %rd394, %rs1307; + and.b64 %rd395, %rd394, 255; + shl.b64 %rd396, %rd395, %r2089; + or.b64 %rd502, %rd396, %rd502; + cvt.u32.u16 %r1492, %rs1320; + cvt.s32.s8 %r1493, %r1492; + mov.u32 %r1494, 8; + sub.s32 %r1495, %r1494, %r1493; + add.s32 %r2089, %r1495, %r2089; + setp.eq.s16 %p469, %rs921, 255; + selp.u16 %rs1320, 1, 0, %p469; + setp.lt.u32 %p470, %r2089, 33; + @%p470 bra $L__BB27_348; + +$L__BB27_351: + shr.u32 %r1496, %r674, 15; + sub.s32 %r1497, %r675, %r1496; + shr.u64 %rd85, %rd502, %r1497; + sub.s32 %r2089, %r2089, %r1497; + cvt.u32.u64 %r1498, %rd502; + shl.b32 %r1499, %r1498, 31; + setp.eq.s32 %p471, %r1497, 0; + mov.u32 %r1500, -1; + shl.b32 %r1501, %r1500, %r1497; + not.b32 %r1502, %r1501; + selp.b32 %r1503, 0, %r1502, %p471; + and.b32 %r1504, %r1503, %r1498; + shr.u32 %r1505, %r674, 11; + and.b32 %r1506, %r1505, 1; + shl.b32 %r1507, %r1506, %r1497; + or.b32 %r1508, %r1507, %r1504; + or.b32 %r2022, %r1508, 1; + add.s32 %r1509, %r2022, 2; + shl.b32 %r1510, %r1509, %r667; + or.b32 %r2060, %r1510, %r1499; + mov.u64 %rd502, %rd85; + +$L__BB27_352: + @%p458 bra $L__BB27_354; + + add.s32 %r1511, %r702, %r12; + mul.wide.u32 %rd397, %r1511, 4; + add.s64 %rd398, %rd2, %rd397; + st.global.u32 [%rd398], %r2060; + +$L__BB27_354: + add.s32 %r2025, %r2025, 2; + add.s32 %r2024, %r670, 1; + add.s32 %r2023, %r2023, 2; + setp.lt.u32 %p473, %r2023, %r820; + @%p473 bra $L__BB27_323; + bra.uni $L__BB27_355; + +$L__BB27_391: + mov.u32 %r1635, 1; + st.global.u32 [%rd4], %r1635; + mov.u32 %r1636, 13; + st.global.u32 [%rd4+4], %r1636; + mov.u32 %r1637, 0; + st.global.u32 [%rd4+8], %r1637; + st.global.u32 [%rd4+12], %r1637; + bra.uni $L__BB27_399; + +$L__BB27_339: + mov.u32 %r2022, 0; + +$L__BB27_355: + add.s32 %r1512, %r670, 1; + mul.wide.u32 %rd401, %r1512, 4; + add.s64 %rd402, %rd76, %rd401; + st.local.u32 [%rd402], %r2022; + @%p297 bra $L__BB27_399; + + mov.u32 %r2065, 2; + +$L__BB27_357: + shr.u32 %r1518, %r2065, 1; + mul.lo.s32 %r2069, %r1518, %r841; + mad.lo.s32 %r2071, %r2065, %r12, %r13; + add.s32 %r741, %r2065, 1; + ld.local.u32 %r2068, [%rd76]; + mov.u32 %r2070, 0; + mov.u32 %r2072, %r2070; + mov.u32 %r2073, %r2070; + +$L__BB27_358: + mul.wide.u32 %rd403, %r2069, 2; + add.s64 %rd404, %rd13, %rd403; + ld.local.v2.u16 {%rs922, %rs923}, [%rd404]; + cvt.u32.u16 %r751, %rs922; + cvt.u32.u16 %r1519, %rs923; + and.b32 %r1520, %r751, 240; + add.s32 %r1521, %r1520, 240; + and.b32 %r1522, %r1521, %r1520; + add.s32 %r752, %r2070, 1; + mul.wide.u32 %rd405, %r752, 4; + add.s64 %rd90, %rd76, %rd405; + ld.local.u32 %r753, [%rd90]; + or.b32 %r1523, %r2068, %r753; + or.b32 %r1524, %r1523, 2; + clz.b32 %r1525, %r1524; + xor.b32 %r1526, %r1525, 31; + setp.eq.s32 %p475, %r1522, 0; + selp.b32 %r1527, 1, %r1526, %p475; + add.s32 %r754, %r1527, %r1519; + setp.gt.u32 %p476, %r754, %r666; + @%p476 bra $L__BB27_390; + + and.b32 %r1529, %r751, 16; + setp.eq.s32 %p477, %r1529, 0; + mov.u32 %r2091, 0; + mov.u32 %r2083, %r2091; + @%p477 bra $L__BB27_365; + + setp.gt.u32 %p478, %r2089, 31; + @%p478 bra $L__BB27_364; + +$L__BB27_361: + setp.ge.u32 %p479, %r2090, %r16; + mov.u16 %rs1314, 255; + @%p479 bra $L__BB27_363; + + add.s32 %r757, %r2090, 1; + cvt.u64.u32 %rd406, %r2090; + add.s64 %rd407, %rd406, %rd3; + add.s64 %rd408, %rd1, %rd407; + ld.global.u8 %rs1314, [%rd408]; + mov.u32 %r2090, %r757; + +$L__BB27_363: + and.b16 %rs927, %rs1314, 255; + cvt.u64.u16 %rd409, %rs1314; + and.b64 %rd410, %rd409, 255; + shl.b64 %rd411, %rd410, %r2089; + or.b64 %rd502, %rd411, %rd502; + cvt.u32.u16 %r1530, %rs1320; + cvt.s32.s8 %r1531, %r1530; + mov.u32 %r1532, 8; + sub.s32 %r1533, %r1532, %r1531; + add.s32 %r2089, %r1533, %r2089; + setp.eq.s16 %p480, %rs927, 255; + selp.u16 %rs1320, 1, 0, %p480; + setp.lt.u32 %p481, %r2089, 33; + @%p481 bra $L__BB27_361; + +$L__BB27_364: + shr.u32 %r1534, %r751, 12; + and.b32 %r1535, %r1534, 1; + sub.s32 %r1536, %r754, %r1535; + shr.u64 %rd94, %rd502, %r1536; + sub.s32 %r2089, %r2089, %r1536; + cvt.u32.u64 %r1537, %rd502; + shl.b32 %r1538, %r1537, 31; + setp.eq.s32 %p482, %r1536, 0; + mov.u32 %r1539, -1; + shl.b32 %r1540, %r1539, %r1536; + not.b32 %r1541, %r1540; + selp.b32 %r1542, 0, %r1541, %p482; + and.b32 %r1543, %r1542, %r1537; + shr.u32 %r1544, %r751, 8; + and.b32 %r1545, %r1544, 1; + shl.b32 %r1546, %r1545, %r1536; + or.b32 %r1547, %r1546, %r1543; + or.b32 %r1548, %r1547, 1; + add.s32 %r1549, %r1548, 2; + shl.b32 %r1550, %r1549, %r667; + or.b32 %r2083, %r1550, %r1538; + mov.u64 %rd502, %rd94; + +$L__BB27_365: + mul.wide.u32 %rd412, %r2071, 4; + add.s64 %rd413, %rd2, %rd412; + st.global.u32 [%rd413], %r2083; + and.b32 %r1553, %r751, 32; + setp.eq.s32 %p483, %r1553, 0; + mov.u32 %r2092, %r2091; + @%p483 bra $L__BB27_371; + + setp.gt.u32 %p484, %r2089, 31; + @%p484 bra $L__BB27_370; + +$L__BB27_367: + setp.ge.u32 %p485, %r2090, %r16; + mov.u16 %rs1318, 255; + @%p485 bra $L__BB27_369; + + add.s32 %r769, %r2090, 1; + cvt.u64.u32 %rd414, %r2090; + add.s64 %rd415, %rd414, %rd3; + add.s64 %rd416, %rd1, %rd415; + ld.global.u8 %rs1318, [%rd416]; + mov.u32 %r2090, %r769; + +$L__BB27_369: + and.b16 %rs929, %rs1318, 255; + cvt.u64.u16 %rd417, %rs1318; + and.b64 %rd418, %rd417, 255; + shl.b64 %rd419, %rd418, %r2089; + or.b64 %rd502, %rd419, %rd502; + cvt.u32.u16 %r1554, %rs1320; + cvt.s32.s8 %r1555, %r1554; + mov.u32 %r1556, 8; + sub.s32 %r1557, %r1556, %r1555; + add.s32 %r2089, %r1557, %r2089; + setp.eq.s16 %p486, %rs929, 255; + selp.u16 %rs1320, 1, 0, %p486; + setp.lt.u32 %p487, %r2089, 33; + @%p487 bra $L__BB27_367; + +$L__BB27_370: + shr.u32 %r1558, %r751, 13; + and.b32 %r1559, %r1558, 1; + sub.s32 %r1560, %r754, %r1559; + shr.u64 %rd99, %rd502, %r1560; + sub.s32 %r2089, %r2089, %r1560; + cvt.u32.u64 %r1561, %rd502; + shl.b32 %r1562, %r1561, 31; + setp.eq.s32 %p488, %r1560, 0; + mov.u32 %r1563, -1; + shl.b32 %r1564, %r1563, %r1560; + not.b32 %r1565, %r1564; + selp.b32 %r1566, 0, %r1565, %p488; + and.b32 %r1567, %r1566, %r1561; + shr.u32 %r1568, %r751, 9; + and.b32 %r1569, %r1568, 1; + shl.b32 %r1570, %r1569, %r1560; + or.b32 %r1571, %r1570, %r1567; + or.b32 %r2092, %r1571, 1; + add.s32 %r1572, %r2092, 2; + shl.b32 %r1573, %r1572, %r667; + or.b32 %r2091, %r1573, %r1562; + mov.u64 %rd502, %rd99; + +$L__BB27_371: + setp.ge.u32 %p489, %r741, %r822; + @%p489 bra $L__BB27_373; + + add.s32 %r1574, %r2071, %r12; + mul.wide.u32 %rd420, %r1574, 4; + add.s64 %rd421, %rd2, %rd420; + st.global.u32 [%rd421], %r2091; + +$L__BB27_373: + or.b32 %r1576, %r2092, %r2072; + mul.wide.u32 %rd422, %r2070, 4; + add.s64 %rd423, %rd76, %rd422; + st.local.u32 [%rd423], %r1576; + add.s32 %r781, %r2071, 1; + add.s32 %r1577, %r2073, 1; + setp.ge.u32 %p490, %r1577, %r820; + mov.u32 %r2072, 0; + @%p490 bra $L__BB27_389; + + and.b32 %r1579, %r751, 64; + setp.eq.s32 %p491, %r1579, 0; + mov.u32 %r2108, 0; + mov.u32 %r2100, %r2108; + @%p491 bra $L__BB27_380; + + setp.gt.u32 %p492, %r2089, 31; + @%p492 bra $L__BB27_379; + +$L__BB27_376: + setp.ge.u32 %p493, %r2090, %r16; + mov.u16 %rs1322, 255; + @%p493 bra $L__BB27_378; + + add.s32 %r784, %r2090, 1; + cvt.u64.u32 %rd424, %r2090; + add.s64 %rd425, %rd424, %rd3; + add.s64 %rd426, %rd1, %rd425; + ld.global.u8 %rs1322, [%rd426]; + mov.u32 %r2090, %r784; + +$L__BB27_378: + and.b16 %rs931, %rs1322, 255; + cvt.u64.u16 %rd427, %rs1322; + and.b64 %rd428, %rd427, 255; + shl.b64 %rd429, %rd428, %r2089; + or.b64 %rd502, %rd429, %rd502; + cvt.u32.u16 %r1580, %rs1320; + cvt.s32.s8 %r1581, %r1580; + mov.u32 %r1582, 8; + sub.s32 %r1583, %r1582, %r1581; + add.s32 %r2089, %r1583, %r2089; + setp.eq.s16 %p494, %rs931, 255; + selp.u16 %rs1320, 1, 0, %p494; + setp.lt.u32 %p495, %r2089, 33; + @%p495 bra $L__BB27_376; + +$L__BB27_379: + shr.u32 %r1584, %r751, 14; + and.b32 %r1585, %r1584, 1; + sub.s32 %r1586, %r754, %r1585; + shr.u64 %rd104, %rd502, %r1586; + sub.s32 %r2089, %r2089, %r1586; + cvt.u32.u64 %r1587, %rd502; + shl.b32 %r1588, %r1587, 31; + setp.eq.s32 %p496, %r1586, 0; + mov.u32 %r1589, -1; + shl.b32 %r1590, %r1589, %r1586; + not.b32 %r1591, %r1590; + selp.b32 %r1592, 0, %r1591, %p496; + and.b32 %r1593, %r1592, %r1587; + shr.u32 %r1594, %r751, 10; + and.b32 %r1595, %r1594, 1; + shl.b32 %r1596, %r1595, %r1586; + or.b32 %r1597, %r1596, %r1593; + or.b32 %r1598, %r1597, 1; + add.s32 %r1599, %r1598, 2; + shl.b32 %r1600, %r1599, %r667; + or.b32 %r2100, %r1600, %r1588; + mov.u64 %rd502, %rd104; + +$L__BB27_380: + mul.wide.u32 %rd430, %r781, 4; + add.s64 %rd431, %rd2, %rd430; + st.global.u32 [%rd431], %r2100; + and.b32 %r1603, %r751, 128; + setp.eq.s32 %p497, %r1603, 0; + mov.u32 %r2072, %r2108; + @%p497 bra $L__BB27_386; + + setp.gt.u32 %p498, %r2089, 31; + @%p498 bra $L__BB27_385; + +$L__BB27_382: + setp.ge.u32 %p499, %r2090, %r16; + mov.u16 %rs1326, 255; + @%p499 bra $L__BB27_384; + + add.s32 %r796, %r2090, 1; + cvt.u64.u32 %rd432, %r2090; + add.s64 %rd433, %rd432, %rd3; + add.s64 %rd434, %rd1, %rd433; + ld.global.u8 %rs1326, [%rd434]; + mov.u32 %r2090, %r796; + +$L__BB27_384: + and.b16 %rs933, %rs1326, 255; + cvt.u64.u16 %rd435, %rs1326; + and.b64 %rd436, %rd435, 255; + shl.b64 %rd437, %rd436, %r2089; + or.b64 %rd502, %rd437, %rd502; + cvt.u32.u16 %r1604, %rs1320; + cvt.s32.s8 %r1605, %r1604; + mov.u32 %r1606, 8; + sub.s32 %r1607, %r1606, %r1605; + add.s32 %r2089, %r1607, %r2089; + setp.eq.s16 %p500, %rs933, 255; + selp.u16 %rs1320, 1, 0, %p500; + setp.lt.u32 %p501, %r2089, 33; + @%p501 bra $L__BB27_382; + +$L__BB27_385: + shr.u32 %r1608, %r751, 15; + sub.s32 %r1609, %r754, %r1608; + shr.u64 %rd109, %rd502, %r1609; + sub.s32 %r2089, %r2089, %r1609; + cvt.u32.u64 %r1610, %rd502; + shl.b32 %r1611, %r1610, 31; + setp.eq.s32 %p502, %r1609, 0; + mov.u32 %r1612, -1; + shl.b32 %r1613, %r1612, %r1609; + not.b32 %r1614, %r1613; + selp.b32 %r1615, 0, %r1614, %p502; + and.b32 %r1616, %r1615, %r1610; + shr.u32 %r1617, %r751, 11; + and.b32 %r1618, %r1617, 1; + shl.b32 %r1619, %r1618, %r1609; + or.b32 %r1620, %r1619, %r1616; + or.b32 %r2072, %r1620, 1; + add.s32 %r1621, %r2072, 2; + shl.b32 %r1622, %r1621, %r667; + or.b32 %r2108, %r1622, %r1611; + mov.u64 %rd502, %rd109; + +$L__BB27_386: + @%p489 bra $L__BB27_388; + + add.s32 %r1623, %r781, %r12; + mul.wide.u32 %rd438, %r1623, 4; + add.s64 %rd439, %rd2, %rd438; + st.global.u32 [%rd439], %r2108; + +$L__BB27_388: + add.s32 %r2071, %r2071, 2; + add.s32 %r2069, %r2069, 2; + add.s32 %r2073, %r2073, 2; + setp.lt.u32 %p504, %r2073, %r820; + mov.u32 %r2068, %r753; + mov.u32 %r2070, %r752; + @%p504 bra $L__BB27_358; + +$L__BB27_389: + st.local.u32 [%rd90], %r2072; + add.s32 %r2065, %r2065, 2; + setp.lt.u32 %p505, %r2065, %r822; + @%p505 bra $L__BB27_357; + bra.uni $L__BB27_399; + +$L__BB27_390: + mov.u32 %r1628, 1; + st.global.u32 [%rd4], %r1628; + mov.u32 %r1629, 14; + st.global.u32 [%rd4+4], %r1629; + mov.u32 %r1630, 0; + st.global.u32 [%rd4+8], %r1630; + st.global.u32 [%rd4+12], %r1630; + +$L__BB27_399: + ret; + +} + // .globl j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize +.visible .entry j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize( + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_0, + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_1, + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_2, + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_3, + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_4, + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_5, + .param .u64 j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_6, + .param .u32 j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_7 +) +{ + .local .align 16 .b8 __local_depot28[6720]; + .reg .b64 %SP; + .reg .b64 %SPL; + .reg .pred %p<516>; + .reg .b16 %rs<1330>; + .reg .f32 %f<25>; + .reg .b32 %r<2157>; + .reg .b64 %rd<510>; + + + mov.u64 %SPL, __local_depot28; + ld.param.u64 %rd118, [ j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_0]; + ld.param.u64 %rd112, [ j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_1]; + ld.param.u64 %rd117, [ j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_6]; + ld.param.u32 %r819, [ j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_7]; + cvta.to.global.u64 %rd1, %rd118; + mov.u32 %r820, %ntid.x; + mov.u32 %r821, %ctaid.x; + mov.u32 %r822, %tid.x; + mad.lo.s32 %r1, %r821, %r820, %r822; + setp.ge.u32 %p1, %r1, %r819; + @%p1 bra $L__BB28_401; + + cvta.to.global.u64 %rd119, %rd117; + cvta.to.global.u64 %rd120, %rd112; + mul.wide.u32 %rd121, %r1, 64; + add.s64 %rd122, %rd120, %rd121; + ld.global.u64 %rd123, [%rd122]; + cvta.to.global.u64 %rd2, %rd123; + ld.global.v2.u32 {%r823, %r824}, [%rd122+8]; + mov.u32 %r825, 0; + ld.global.v2.u32 {%r826, %r827}, [%rd122+16]; + ld.global.v2.u32 {%r828, %r829}, [%rd122+24]; + ld.global.v2.u32 {%r831, %r832}, [%rd122+32]; + ld.global.v2.u32 {%r833, %r834}, [%rd122+40]; + ld.global.v2.u32 {%r835, %r836}, [%rd122+48]; + cvt.u64.u32 %rd3, %r823; + mul.wide.u32 %rd124, %r1, 16; + add.s64 %rd4, %rd119, %rd124; + st.global.u32 [%rd4], %r825; + st.global.u32 [%rd4+4], %r825; + st.global.u32 [%rd4+8], %r825; + st.global.u32 [%rd4+12], %r825; + setp.eq.s32 %p2, %r829, 0; + @%p2 bra $L__BB28_3; + + mov.u32 %r837, 2; + st.global.u32 [%rd4], %r837; + mov.u32 %r838, 17; + st.global.u32 [%rd4+4], %r838; + st.global.u32 [%rd4+8], %r825; + st.global.u32 [%rd4+12], %r825; + bra.uni $L__BB28_401; + +$L__BB28_3: + setp.gt.u32 %p3, %r833, 1; + @%p3 bra $L__BB28_400; + bra.uni $L__BB28_4; + +$L__BB28_400: + mov.u32 %r1704, 2; + st.global.u32 [%rd4], %r1704; + mov.u32 %r1705, 18; + st.global.u32 [%rd4+4], %r1705; + mov.u32 %r1706, 0; + st.global.u32 [%rd4+8], %r1706; + st.global.u32 [%rd4+12], %r1706; + bra.uni $L__BB28_401; + +$L__BB28_4: + setp.eq.s32 %p4, %r824, 0; + setp.eq.s32 %p5, %r826, 0; + or.pred %p6, %p4, %p5; + @%p6 bra $L__BB28_401; + + setp.gt.u32 %p7, %r824, 256; + setp.gt.u32 %p8, %r826, 256; + or.pred %p9, %p7, %p8; + mul.lo.s32 %r840, %r826, %r824; + setp.gt.u32 %p10, %r840, 4096; + or.pred %p11, %p9, %p10; + @%p11 bra $L__BB28_399; + bra.uni $L__BB28_6; + +$L__BB28_399: + mov.u32 %r1701, 2; + st.global.u32 [%rd4], %r1701; + mov.u32 %r1702, 1; + st.global.u32 [%rd4+4], %r1702; + mov.u32 %r1703, 0; + st.global.u32 [%rd4+8], %r1703; + st.global.u32 [%rd4+12], %r1703; + bra.uni $L__BB28_401; + +$L__BB28_6: + add.s32 %r841, %r832, -1; + setp.gt.u32 %p12, %r841, 30; + @%p12 bra $L__BB28_398; + bra.uni $L__BB28_7; + +$L__BB28_398: + mov.u32 %r1698, 1; + st.global.u32 [%rd4], %r1698; + mov.u32 %r1699, 2; + st.global.u32 [%rd4+4], %r1699; + mov.u32 %r1700, 0; + st.global.u32 [%rd4+8], %r1700; + st.global.u32 [%rd4+12], %r1700; + bra.uni $L__BB28_401; + +$L__BB28_7: + setp.gt.u32 %p13, %r831, 29; + @%p13 bra $L__BB28_397; + bra.uni $L__BB28_8; + +$L__BB28_397: + mov.u32 %r1695, 1; + st.global.u32 [%rd4], %r1695; + mov.u32 %r1696, 3; + st.global.u32 [%rd4+4], %r1696; + mov.u32 %r1697, 0; + st.global.u32 [%rd4+8], %r1697; + st.global.u32 [%rd4+12], %r1697; + bra.uni $L__BB28_401; + +$L__BB28_8: + setp.lt.u32 %p14, %r828, 2; + setp.lt.u32 %p15, %r827, %r828; + or.pred %p16, %p14, %p15; + @%p16 bra $L__BB28_396; + bra.uni $L__BB28_9; + +$L__BB28_396: + mov.u32 %r1692, 1; + st.global.u32 [%rd4], %r1692; + mov.u32 %r1693, 4; + st.global.u32 [%rd4+4], %r1693; + mov.u32 %r1694, 0; + st.global.u32 [%rd4+8], %r1694; + st.global.u32 [%rd4+12], %r1694; + bra.uni $L__BB28_401; + +$L__BB28_9: + add.s32 %r842, %r828, -1; + cvt.u64.u32 %rd125, %r842; + add.s64 %rd126, %rd125, %rd3; + add.s64 %rd127, %rd1, %rd126; + ld.global.u8 %rs652, [%rd127]; + mul.wide.u16 %r843, %rs652, 16; + add.s32 %r844, %r828, -2; + cvt.u64.u32 %rd128, %r844; + add.s64 %rd129, %rd128, %rd3; + add.s64 %rd130, %rd1, %rd129; + ld.global.u8 %rs1, [%rd130]; + and.b16 %rs653, %rs1, 15; + cvt.u32.u16 %r845, %rs653; + or.b32 %r16, %r843, %r845; + setp.lt.u32 %p17, %r828, %r16; + add.s32 %r17, %r16, -2; + setp.gt.u32 %p18, %r17, 4077; + or.pred %p19, %p17, %p18; + @%p19 bra $L__BB28_395; + bra.uni $L__BB28_10; + +$L__BB28_395: + mov.u32 %r1689, 1; + st.global.u32 [%rd4], %r1689; + mov.u32 %r1690, 5; + st.global.u32 [%rd4+4], %r1690; + mov.u32 %r1691, 0; + st.global.u32 [%rd4+8], %r1691; + st.global.u32 [%rd4+12], %r1691; + bra.uni $L__BB28_401; + +$L__BB28_10: + add.s32 %r846, %r826, 1; + shr.u32 %r847, %r846, 1; + add.s32 %r848, %r824, 9; + and.b32 %r849, %r848, -8; + setp.gt.u32 %p20, %r849, 264; + add.s32 %r850, %r847, 1; + mul.lo.s32 %r851, %r850, %r849; + setp.gt.u32 %p21, %r851, 3096; + or.pred %p22, %p20, %p21; + @%p22 bra $L__BB28_394; + bra.uni $L__BB28_11; + +$L__BB28_394: + mov.u32 %r1686, 2; + st.global.u32 [%rd4], %r1686; + mov.u32 %r1687, 6; + st.global.u32 [%rd4+4], %r1687; + mov.u32 %r1688, 0; + st.global.u32 [%rd4+8], %r1688; + st.global.u32 [%rd4+12], %r1688; + bra.uni $L__BB28_401; + +$L__BB28_11: + and.b16 %rs940, %rs1, 15; + cvt.u32.u16 %r1713, %rs940; + mul.wide.u16 %r1712, %rs652, 16; + or.b32 %r1711, %r1712, %r1713; + sub.s32 %r18, %r828, %r1711; + add.s32 %r1791, %r1711, -1; + add.s32 %r1959, %r828, -3; + mov.u64 %rd455, 0; + mov.u32 %r1881, 0; + mov.u16 %rs1013, 0; + mov.u64 %rd133, _ZZ20mel_decode_more_runsR10MelDecoderE7MEL_EXP; + mov.u32 %r1790, %r18; + mov.u16 %rs1014, %rs1013; + mov.u16 %rs1048, %rs1013; + mov.u32 %r1763, %r1881; + +$L__BB28_12: + setp.gt.u32 %p24, %r1763, 7; + @%p24 bra $L__BB28_57; + + mul.wide.u32 %rd132, %r1881, 4; + add.s64 %rd134, %rd133, %rd132; + ld.global.nc.u32 %r25, [%rd134]; + and.b16 %rs657, %rs1048, 255; + setp.ne.s16 %p25, %rs657, 0; + mov.u16 %rs948, %rs1048; + @%p25 bra $L__BB28_17; + + setp.eq.s32 %p26, %r1791, 0; + mov.u16 %rs945, 255; + @%p26 bra $L__BB28_16; + + cvt.u64.u32 %rd135, %r1790; + add.s64 %rd136, %rd135, %rd3; + add.s64 %rd137, %rd1, %rd136; + ld.global.u8 %rs945, [%rd137]; + +$L__BB28_16: + setp.ne.s32 %p28, %r1791, 0; + selp.u32 %r854, 1, 0, %p28; + add.s32 %r1790, %r1790, %r854; + add.s32 %r855, %r1791, -1; + selp.b32 %r1791, 0, %r855, %p26; + setp.eq.s32 %p29, %r1791, 0; + or.b16 %rs659, %rs945, 15; + selp.b16 %rs1014, %rs659, %rs945, %p29; + and.b16 %rs660, %rs1014, 255; + mov.u16 %rs661, 8; + sub.s16 %rs948, %rs661, %rs1013; + setp.eq.s16 %p30, %rs660, 255; + selp.u16 %rs1013, 1, 0, %p30; + +$L__BB28_17: + add.s16 %rs15, %rs948, -1; + cvt.u32.u16 %r856, %rs15; + and.b32 %r857, %r856, 255; + mov.u32 %r858, 1; + shl.b32 %r859, %r858, %r857; + cvt.u32.u16 %r860, %rs1014; + and.b32 %r861, %r859, %r860; + and.b32 %r30, %r861, 255; + add.s16 %rs1048, %rs948, -1; + setp.eq.s32 %p31, %r30, 0; + @%p31 bra $L__BB28_19; + + add.s32 %r862, %r1881, 1; + min.u32 %r1758, %r862, 12; + mov.u32 %r863, -1; + shl.b32 %r864, %r863, %r25; + shl.b32 %r865, %r864, 1; + xor.b32 %r1759, %r865, -2; + bra.uni $L__BB28_56; + +$L__BB28_19: + cvt.u64.u32 %rd453, %r1881; + add.s64 %rd138, %rd453, -3; + setp.gt.u64 %p32, %rd138, 9; + mov.u32 %r1755, 0; + @%p32 bra $L__BB28_55; + + add.s16 %rs1048, %rs948, -1; + max.u32 %r33, %r25, 1; + add.s32 %r869, %r33, -1; + setp.lt.u32 %p33, %r869, 3; + mov.u32 %r1755, 0; + @%p33 bra $L__BB28_39; + + and.b32 %r1707, %r33, 3; + add.s16 %rs1048, %rs948, -1; + sub.s32 %r1727, %r33, %r1707; + mov.u32 %r1755, 0; + +$L__BB28_22: + and.b16 %rs663, %rs1048, 255; + setp.ne.s16 %p34, %rs663, 0; + @%p34 bra $L__BB28_26; + + setp.eq.s32 %p35, %r1791, 0; + mov.u16 %rs952, 255; + @%p35 bra $L__BB28_25; + + cvt.u64.u32 %rd139, %r1790; + add.s64 %rd140, %rd139, %rd3; + add.s64 %rd141, %rd1, %rd140; + ld.global.u8 %rs952, [%rd141]; + +$L__BB28_25: + setp.ne.s32 %p37, %r1791, 0; + selp.u32 %r871, 1, 0, %p37; + add.s32 %r1790, %r1790, %r871; + add.s32 %r872, %r1791, -1; + selp.b32 %r1791, 0, %r872, %p35; + setp.eq.s32 %p38, %r1791, 0; + or.b16 %rs665, %rs952, 15; + selp.b16 %rs1014, %rs665, %rs952, %p38; + and.b16 %rs666, %rs1014, 255; + mov.u16 %rs667, 8; + sub.s16 %rs1048, %rs667, %rs1013; + setp.eq.s16 %p39, %rs666, 255; + selp.u16 %rs1013, 1, 0, %p39; + +$L__BB28_26: + add.s16 %rs959, %rs1048, -1; + and.b16 %rs668, %rs959, 255; + cvt.u32.u16 %r873, %rs959; + and.b32 %r874, %r873, 255; + cvt.u32.u16 %r875, %rs1014; + and.b32 %r1733, %r875, 255; + shr.u32 %r876, %r1733, %r874; + and.b32 %r877, %r876, 1; + bfi.b32 %r45, %r1755, %r877, 1, 31; + setp.ne.s16 %p40, %rs668, 0; + @%p40 bra $L__BB28_30; + + setp.eq.s32 %p41, %r1791, 0; + mov.u16 %rs956, 255; + @%p41 bra $L__BB28_29; + + cvt.u64.u32 %rd142, %r1790; + add.s64 %rd143, %rd142, %rd3; + add.s64 %rd144, %rd1, %rd143; + ld.global.u8 %rs956, [%rd144]; + +$L__BB28_29: + setp.ne.s32 %p43, %r1791, 0; + selp.u32 %r878, 1, 0, %p43; + add.s32 %r1790, %r1790, %r878; + add.s32 %r879, %r1791, -1; + selp.b32 %r1791, 0, %r879, %p41; + setp.eq.s32 %p44, %r1791, 0; + or.b16 %rs670, %rs956, 15; + selp.b16 %rs1014, %rs670, %rs956, %p44; + and.b16 %rs671, %rs1014, 255; + mov.u16 %rs672, 8; + sub.s16 %rs959, %rs672, %rs1013; + setp.eq.s16 %p45, %rs671, 255; + selp.u16 %rs1013, 1, 0, %p45; + cvt.u32.u16 %r880, %rs1014; + and.b32 %r1733, %r880, 255; + +$L__BB28_30: + add.s16 %rs963, %rs959, -1; + and.b16 %rs673, %rs963, 255; + cvt.u32.u16 %r881, %rs963; + and.b32 %r882, %r881, 255; + shr.u32 %r883, %r1733, %r882; + and.b32 %r884, %r883, 1; + bfi.b32 %r52, %r45, %r884, 1, 31; + setp.ne.s16 %p46, %rs673, 0; + @%p46 bra $L__BB28_34; + + setp.eq.s32 %p47, %r1791, 0; + mov.u16 %rs960, 255; + @%p47 bra $L__BB28_33; + + cvt.u64.u32 %rd145, %r1790; + add.s64 %rd146, %rd145, %rd3; + add.s64 %rd147, %rd1, %rd146; + ld.global.u8 %rs960, [%rd147]; + +$L__BB28_33: + setp.ne.s32 %p49, %r1791, 0; + selp.u32 %r885, 1, 0, %p49; + add.s32 %r1790, %r1790, %r885; + add.s32 %r886, %r1791, -1; + selp.b32 %r1791, 0, %r886, %p47; + setp.eq.s32 %p50, %r1791, 0; + or.b16 %rs675, %rs960, 15; + selp.b16 %rs1014, %rs675, %rs960, %p50; + and.b16 %rs676, %rs1014, 255; + mov.u16 %rs677, 8; + sub.s16 %rs963, %rs677, %rs1013; + setp.eq.s16 %p51, %rs676, 255; + selp.u16 %rs1013, 1, 0, %p51; + cvt.u32.u16 %r887, %rs1014; + and.b32 %r1733, %r887, 255; + +$L__BB28_34: + add.s16 %rs967, %rs963, -1; + and.b16 %rs678, %rs967, 255; + cvt.u32.u16 %r888, %rs967; + and.b32 %r889, %r888, 255; + shr.u32 %r890, %r1733, %r889; + and.b32 %r891, %r890, 1; + bfi.b32 %r59, %r52, %r891, 1, 31; + setp.ne.s16 %p52, %rs678, 0; + @%p52 bra $L__BB28_38; + + setp.eq.s32 %p53, %r1791, 0; + mov.u16 %rs964, 255; + @%p53 bra $L__BB28_37; + + cvt.u64.u32 %rd148, %r1790; + add.s64 %rd149, %rd148, %rd3; + add.s64 %rd150, %rd1, %rd149; + ld.global.u8 %rs964, [%rd150]; + +$L__BB28_37: + setp.ne.s32 %p55, %r1791, 0; + selp.u32 %r892, 1, 0, %p55; + add.s32 %r1790, %r1790, %r892; + add.s32 %r893, %r1791, -1; + selp.b32 %r1791, 0, %r893, %p53; + setp.eq.s32 %p56, %r1791, 0; + or.b16 %rs680, %rs964, 15; + selp.b16 %rs1014, %rs680, %rs964, %p56; + and.b16 %rs681, %rs1014, 255; + mov.u16 %rs682, 8; + sub.s16 %rs967, %rs682, %rs1013; + setp.eq.s16 %p57, %rs681, 255; + selp.u16 %rs1013, 1, 0, %p57; + cvt.u32.u16 %r894, %rs1014; + and.b32 %r1733, %r894, 255; + +$L__BB28_38: + add.s16 %rs1048, %rs967, -1; + cvt.u32.u16 %r895, %rs1048; + and.b32 %r896, %r895, 255; + shr.u32 %r897, %r1733, %r896; + and.b32 %r898, %r897, 1; + bfi.b32 %r1755, %r59, %r898, 1, 31; + add.s32 %r1727, %r1727, -4; + setp.ne.s32 %p58, %r1727, 0; + @%p58 bra $L__BB28_22; + +$L__BB28_39: + and.b32 %r1708, %r33, 3; + setp.eq.s32 %p59, %r1708, 0; + @%p59 bra $L__BB28_55; + + and.b16 %rs683, %rs1048, 255; + setp.ne.s16 %p60, %rs683, 0; + @%p60 bra $L__BB28_44; + + setp.eq.s32 %p61, %r1791, 0; + mov.u16 %rs974, 255; + @%p61 bra $L__BB28_43; + + cvt.u64.u32 %rd151, %r1790; + add.s64 %rd152, %rd151, %rd3; + add.s64 %rd153, %rd1, %rd152; + ld.global.u8 %rs974, [%rd153]; + +$L__BB28_43: + setp.ne.s32 %p63, %r1791, 0; + selp.u32 %r899, 1, 0, %p63; + add.s32 %r1790, %r1790, %r899; + add.s32 %r900, %r1791, -1; + selp.b32 %r1791, 0, %r900, %p61; + setp.eq.s32 %p64, %r1791, 0; + or.b16 %rs685, %rs974, 15; + selp.b16 %rs1014, %rs685, %rs974, %p64; + and.b16 %rs686, %rs1014, 255; + mov.u16 %rs687, 8; + sub.s16 %rs1048, %rs687, %rs1013; + setp.eq.s16 %p65, %rs686, 255; + selp.u16 %rs1013, 1, 0, %p65; + +$L__BB28_44: + and.b32 %r1709, %r33, 3; + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r901, %rs1048; + and.b32 %r902, %r901, 255; + cvt.u32.u16 %r903, %rs1014; + and.b32 %r1750, %r903, 255; + shr.u32 %r904, %r1750, %r902; + and.b32 %r905, %r904, 1; + bfi.b32 %r1755, %r1755, %r905, 1, 31; + setp.eq.s32 %p66, %r1709, 1; + @%p66 bra $L__BB28_55; + + and.b16 %rs688, %rs1048, 255; + setp.ne.s16 %p67, %rs688, 0; + @%p67 bra $L__BB28_49; + + setp.eq.s32 %p68, %r1791, 0; + mov.u16 %rs978, 255; + @%p68 bra $L__BB28_48; + + cvt.u64.u32 %rd154, %r1790; + add.s64 %rd155, %rd154, %rd3; + add.s64 %rd156, %rd1, %rd155; + ld.global.u8 %rs978, [%rd156]; + +$L__BB28_48: + setp.ne.s32 %p70, %r1791, 0; + selp.u32 %r906, 1, 0, %p70; + add.s32 %r1790, %r1790, %r906; + add.s32 %r907, %r1791, -1; + selp.b32 %r1791, 0, %r907, %p68; + setp.eq.s32 %p71, %r1791, 0; + or.b16 %rs690, %rs978, 15; + selp.b16 %rs1014, %rs690, %rs978, %p71; + and.b16 %rs691, %rs1014, 255; + mov.u16 %rs692, 8; + sub.s16 %rs1048, %rs692, %rs1013; + setp.eq.s16 %p72, %rs691, 255; + selp.u16 %rs1013, 1, 0, %p72; + cvt.u32.u16 %r908, %rs1014; + and.b32 %r1750, %r908, 255; + +$L__BB28_49: + and.b32 %r1710, %r33, 3; + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r909, %rs1048; + and.b32 %r910, %r909, 255; + shr.u32 %r911, %r1750, %r910; + and.b32 %r912, %r911, 1; + bfi.b32 %r1755, %r1755, %r912, 1, 31; + setp.eq.s32 %p73, %r1710, 2; + @%p73 bra $L__BB28_55; + + and.b16 %rs693, %rs1048, 255; + setp.ne.s16 %p74, %rs693, 0; + @%p74 bra $L__BB28_54; + + setp.eq.s32 %p75, %r1791, 0; + mov.u16 %rs982, 255; + @%p75 bra $L__BB28_53; + + cvt.u64.u32 %rd157, %r1790; + add.s64 %rd158, %rd157, %rd3; + add.s64 %rd159, %rd1, %rd158; + ld.global.u8 %rs982, [%rd159]; + +$L__BB28_53: + setp.ne.s32 %p77, %r1791, 0; + selp.u32 %r913, 1, 0, %p77; + add.s32 %r1790, %r1790, %r913; + add.s32 %r914, %r1791, -1; + selp.b32 %r1791, 0, %r914, %p75; + setp.eq.s32 %p78, %r1791, 0; + or.b16 %rs695, %rs982, 15; + selp.b16 %rs1014, %rs695, %rs982, %p78; + and.b16 %rs696, %rs1014, 255; + mov.u16 %rs697, 8; + sub.s16 %rs1048, %rs697, %rs1013; + setp.eq.s16 %p79, %rs696, 255; + selp.u16 %rs1013, 1, 0, %p79; + cvt.u32.u16 %r915, %rs1014; + and.b32 %r1750, %r915, 255; + +$L__BB28_54: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r916, %rs1048; + and.b32 %r917, %r916, 255; + shr.u32 %r918, %r1750, %r917; + and.b32 %r919, %r918, 1; + bfi.b32 %r1755, %r1755, %r919, 1, 31; + +$L__BB28_55: + shl.b32 %r920, %r1755, 1; + or.b32 %r1759, %r920, 1; + add.s32 %r921, %r1881, -1; + setp.eq.s32 %p80, %r1881, 0; + selp.b32 %r1758, 0, %r921, %p80; + +$L__BB28_56: + mul.lo.s32 %r922, %r1763, 7; + cvt.u64.u32 %rd160, %r1759; + shl.b64 %rd161, %rd160, %r922; + or.b64 %rd455, %rd161, %rd455; + setp.ne.s32 %p81, %r1881, 12; + setp.ne.s32 %p82, %r30, 0; + or.pred %p83, %p81, %p82; + add.s32 %r1763, %r1763, 1; + setp.lt.u32 %p84, %r1763, 8; + or.pred %p85, %p84, %p83; + mov.u32 %r1881, %r1758; + @%p85 bra $L__BB28_12; + +$L__BB28_57: + and.b16 %rs941, %rs1, 15; + cvt.u32.u16 %r1717, %rs941; + mul.wide.u16 %r1716, %rs652, 16; + or.b32 %r1715, %r1716, %r1717; + add.s32 %r1960, %r1715, -2; + setp.gt.u16 %p515, %rs1, 143; + selp.u16 %rs1180, 1, 0, %p515; + ld.param.u64 %rd452, [ j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_4]; + shr.u16 %rs934, %rs1, 4; + ld.param.u64 %rd449, [ j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_2]; + add.s32 %r1882, %r1763, -1; + shr.u64 %rd465, %rd455, 7; + cvt.u32.u64 %r925, %rd455; + and.b32 %r1878, %r925, 127; + cvt.u64.u16 %rd474, %rs934; + and.b64 %rd162, %rd474, 7; + setp.eq.s64 %p86, %rd162, 7; + selp.b32 %r1961, 3, 4, %p86; + cvta.to.global.u64 %rd11, %rd452; + cvta.to.global.u64 %rd12, %rd449; + add.u64 %rd13, %SPL, 0; + mov.u32 %r1764, 0; + mov.u32 %r1765, %r1764; + +$L__BB28_58: + setp.gt.u32 %p87, %r1961, 31; + @%p87 bra $L__BB28_62; + +$L__BB28_59: + setp.eq.s32 %p88, %r1960, 0; + mov.u16 %rs1000, 0; + @%p88 bra $L__BB28_61; + + cvt.s64.s32 %rd164, %r1959; + add.s64 %rd165, %rd164, %rd3; + add.s64 %rd166, %rd1, %rd165; + ld.global.u8 %rs1000, [%rd166]; + +$L__BB28_61: + setp.ne.s32 %p90, %r1960, 0; + selp.b32 %r926, -1, 0, %p90; + add.s32 %r1959, %r1959, %r926; + add.s32 %r927, %r1960, -1; + selp.b32 %r1960, 0, %r927, %p88; + and.b16 %rs699, %rs1000, 255; + and.b16 %rs700, %rs1000, 127; + setp.eq.s16 %p91, %rs700, 127; + and.b16 %rs701, %rs1180, 255; + setp.ne.s16 %p92, %rs701, 0; + and.pred %p93, %p92, %p91; + selp.b32 %r928, 7, 8, %p93; + cvt.u64.u16 %rd167, %rs1000; + and.b64 %rd168, %rd167, 255; + shl.b64 %rd169, %rd168, %r1961; + or.b64 %rd474, %rd169, %rd474; + add.s32 %r1961, %r928, %r1961; + setp.gt.u16 %p94, %rs699, 143; + selp.u16 %rs1180, 1, 0, %p94; + setp.lt.u32 %p95, %r1961, 33; + @%p95 bra $L__BB28_59; + +$L__BB28_62: + cvt.u32.u64 %r929, %rd474; + and.b32 %r930, %r929, 127; + add.s32 %r931, %r930, %r1764; + mul.wide.u32 %rd170, %r931, 2; + add.s64 %rd171, %rd12, %rd170; + ld.global.u16 %r1831, [%rd171]; + setp.ne.s32 %p96, %r1764, 0; + @%p96 bra $L__BB28_112; + + add.s32 %r132, %r1878, -2; + setp.eq.s32 %p97, %r132, -1; + selp.b32 %r1831, %r1831, 0, %p97; + setp.gt.s32 %p98, %r1878, 1; + mov.u32 %r1878, %r132; + @%p98 bra $L__BB28_112; + + setp.ne.s32 %p99, %r1882, 0; + @%p99 bra $L__BB28_111; + + mov.u32 %r1882, 0; + +$L__BB28_66: + setp.gt.u32 %p100, %r1882, 7; + @%p100 bra $L__BB28_111; + + cvt.u64.u32 %rd20, %r1881; + mul.wide.u32 %rd172, %r1881, 4; + add.s64 %rd174, %rd133, %rd172; + ld.global.nc.u32 %r138, [%rd174]; + and.b16 %rs702, %rs1048, 255; + setp.ne.s16 %p101, %rs702, 0; + @%p101 bra $L__BB28_71; + + setp.eq.s32 %p102, %r1791, 0; + mov.u16 %rs1005, 255; + @%p102 bra $L__BB28_70; + + cvt.u64.u32 %rd175, %r1790; + add.s64 %rd176, %rd175, %rd3; + add.s64 %rd177, %rd1, %rd176; + ld.global.u8 %rs1005, [%rd177]; + +$L__BB28_70: + setp.ne.s32 %p104, %r1791, 0; + selp.u32 %r933, 1, 0, %p104; + add.s32 %r1790, %r1790, %r933; + add.s32 %r934, %r1791, -1; + selp.b32 %r1791, 0, %r934, %p102; + setp.eq.s32 %p105, %r1791, 0; + or.b16 %rs704, %rs1005, 15; + selp.b16 %rs1014, %rs704, %rs1005, %p105; + and.b16 %rs705, %rs1014, 255; + mov.u16 %rs706, 8; + sub.s16 %rs1048, %rs706, %rs1013; + setp.eq.s16 %p106, %rs705, 255; + selp.u16 %rs1013, 1, 0, %p106; + +$L__BB28_71: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r935, %rs1048; + and.b32 %r936, %r935, 255; + mov.u32 %r937, 1; + shl.b32 %r938, %r937, %r936; + cvt.u32.u16 %r939, %rs1014; + and.b32 %r940, %r938, %r939; + and.b32 %r143, %r940, 255; + setp.eq.s32 %p107, %r143, 0; + @%p107 bra $L__BB28_73; + + add.s32 %r941, %r1881, 1; + min.u32 %r1820, %r941, 12; + mov.u32 %r942, -1; + shl.b32 %r943, %r942, %r138; + shl.b32 %r944, %r943, 1; + xor.b32 %r1821, %r944, -2; + bra.uni $L__BB28_110; + +$L__BB28_73: + add.s64 %rd178, %rd20, -3; + setp.gt.u64 %p108, %rd178, 9; + mov.u32 %r1817, 0; + @%p108 bra $L__BB28_109; + + max.u32 %r146, %r138, 1; + add.s32 %r948, %r146, -1; + and.b32 %r147, %r146, 3; + setp.lt.u32 %p109, %r948, 3; + mov.u32 %r1817, 0; + @%p109 bra $L__BB28_93; + + sub.s32 %r1789, %r146, %r147; + mov.u32 %r1817, 0; + +$L__BB28_76: + and.b16 %rs708, %rs1048, 255; + setp.ne.s16 %p110, %rs708, 0; + @%p110 bra $L__BB28_80; + + setp.eq.s32 %p111, %r1791, 0; + mov.u16 %rs1012, 255; + @%p111 bra $L__BB28_79; + + cvt.u64.u32 %rd179, %r1790; + add.s64 %rd180, %rd179, %rd3; + add.s64 %rd181, %rd1, %rd180; + ld.global.u8 %rs1012, [%rd181]; + +$L__BB28_79: + setp.ne.s32 %p113, %r1791, 0; + selp.u32 %r950, 1, 0, %p113; + add.s32 %r1790, %r1790, %r950; + add.s32 %r951, %r1791, -1; + selp.b32 %r1791, 0, %r951, %p111; + setp.eq.s32 %p114, %r1791, 0; + or.b16 %rs710, %rs1012, 15; + selp.b16 %rs1014, %rs710, %rs1012, %p114; + and.b16 %rs711, %rs1014, 255; + mov.u16 %rs712, 8; + sub.s16 %rs1048, %rs712, %rs1013; + setp.eq.s16 %p115, %rs711, 255; + selp.u16 %rs1013, 1, 0, %p115; + +$L__BB28_80: + add.s16 %rs1019, %rs1048, -1; + and.b16 %rs713, %rs1019, 255; + cvt.u32.u16 %r952, %rs1019; + and.b32 %r953, %r952, 255; + cvt.u32.u16 %r954, %rs1014; + and.b32 %r1795, %r954, 255; + shr.u32 %r955, %r1795, %r953; + and.b32 %r956, %r955, 1; + bfi.b32 %r158, %r1817, %r956, 1, 31; + setp.ne.s16 %p116, %rs713, 0; + @%p116 bra $L__BB28_84; + + setp.eq.s32 %p117, %r1791, 0; + mov.u16 %rs1016, 255; + @%p117 bra $L__BB28_83; + + cvt.u64.u32 %rd182, %r1790; + add.s64 %rd183, %rd182, %rd3; + add.s64 %rd184, %rd1, %rd183; + ld.global.u8 %rs1016, [%rd184]; + +$L__BB28_83: + setp.ne.s32 %p119, %r1791, 0; + selp.u32 %r957, 1, 0, %p119; + add.s32 %r1790, %r1790, %r957; + add.s32 %r958, %r1791, -1; + selp.b32 %r1791, 0, %r958, %p117; + setp.eq.s32 %p120, %r1791, 0; + or.b16 %rs715, %rs1016, 15; + selp.b16 %rs1014, %rs715, %rs1016, %p120; + and.b16 %rs716, %rs1014, 255; + mov.u16 %rs717, 8; + sub.s16 %rs1019, %rs717, %rs1013; + setp.eq.s16 %p121, %rs716, 255; + selp.u16 %rs1013, 1, 0, %p121; + cvt.u32.u16 %r959, %rs1014; + and.b32 %r1795, %r959, 255; + +$L__BB28_84: + add.s16 %rs1023, %rs1019, -1; + and.b16 %rs718, %rs1023, 255; + cvt.u32.u16 %r960, %rs1023; + and.b32 %r961, %r960, 255; + shr.u32 %r962, %r1795, %r961; + and.b32 %r963, %r962, 1; + bfi.b32 %r165, %r158, %r963, 1, 31; + setp.ne.s16 %p122, %rs718, 0; + @%p122 bra $L__BB28_88; + + setp.eq.s32 %p123, %r1791, 0; + mov.u16 %rs1020, 255; + @%p123 bra $L__BB28_87; + + cvt.u64.u32 %rd185, %r1790; + add.s64 %rd186, %rd185, %rd3; + add.s64 %rd187, %rd1, %rd186; + ld.global.u8 %rs1020, [%rd187]; + +$L__BB28_87: + setp.ne.s32 %p125, %r1791, 0; + selp.u32 %r964, 1, 0, %p125; + add.s32 %r1790, %r1790, %r964; + add.s32 %r965, %r1791, -1; + selp.b32 %r1791, 0, %r965, %p123; + setp.eq.s32 %p126, %r1791, 0; + or.b16 %rs720, %rs1020, 15; + selp.b16 %rs1014, %rs720, %rs1020, %p126; + and.b16 %rs721, %rs1014, 255; + mov.u16 %rs722, 8; + sub.s16 %rs1023, %rs722, %rs1013; + setp.eq.s16 %p127, %rs721, 255; + selp.u16 %rs1013, 1, 0, %p127; + cvt.u32.u16 %r966, %rs1014; + and.b32 %r1795, %r966, 255; + +$L__BB28_88: + add.s16 %rs1027, %rs1023, -1; + and.b16 %rs723, %rs1027, 255; + cvt.u32.u16 %r967, %rs1027; + and.b32 %r968, %r967, 255; + shr.u32 %r969, %r1795, %r968; + and.b32 %r970, %r969, 1; + bfi.b32 %r172, %r165, %r970, 1, 31; + setp.ne.s16 %p128, %rs723, 0; + @%p128 bra $L__BB28_92; + + setp.eq.s32 %p129, %r1791, 0; + mov.u16 %rs1024, 255; + @%p129 bra $L__BB28_91; + + cvt.u64.u32 %rd188, %r1790; + add.s64 %rd189, %rd188, %rd3; + add.s64 %rd190, %rd1, %rd189; + ld.global.u8 %rs1024, [%rd190]; + +$L__BB28_91: + setp.ne.s32 %p131, %r1791, 0; + selp.u32 %r971, 1, 0, %p131; + add.s32 %r1790, %r1790, %r971; + add.s32 %r972, %r1791, -1; + selp.b32 %r1791, 0, %r972, %p129; + setp.eq.s32 %p132, %r1791, 0; + or.b16 %rs725, %rs1024, 15; + selp.b16 %rs1014, %rs725, %rs1024, %p132; + and.b16 %rs726, %rs1014, 255; + mov.u16 %rs727, 8; + sub.s16 %rs1027, %rs727, %rs1013; + setp.eq.s16 %p133, %rs726, 255; + selp.u16 %rs1013, 1, 0, %p133; + cvt.u32.u16 %r973, %rs1014; + and.b32 %r1795, %r973, 255; + +$L__BB28_92: + add.s16 %rs1048, %rs1027, -1; + cvt.u32.u16 %r974, %rs1048; + and.b32 %r975, %r974, 255; + shr.u32 %r976, %r1795, %r975; + and.b32 %r977, %r976, 1; + bfi.b32 %r1817, %r172, %r977, 1, 31; + add.s32 %r1789, %r1789, -4; + setp.ne.s32 %p134, %r1789, 0; + @%p134 bra $L__BB28_76; + +$L__BB28_93: + setp.eq.s32 %p135, %r147, 0; + @%p135 bra $L__BB28_109; + + and.b16 %rs728, %rs1048, 255; + setp.ne.s16 %p136, %rs728, 0; + @%p136 bra $L__BB28_98; + + setp.eq.s32 %p137, %r1791, 0; + mov.u16 %rs1034, 255; + @%p137 bra $L__BB28_97; + + cvt.u64.u32 %rd191, %r1790; + add.s64 %rd192, %rd191, %rd3; + add.s64 %rd193, %rd1, %rd192; + ld.global.u8 %rs1034, [%rd193]; + +$L__BB28_97: + setp.ne.s32 %p139, %r1791, 0; + selp.u32 %r978, 1, 0, %p139; + add.s32 %r1790, %r1790, %r978; + add.s32 %r979, %r1791, -1; + selp.b32 %r1791, 0, %r979, %p137; + setp.eq.s32 %p140, %r1791, 0; + or.b16 %rs730, %rs1034, 15; + selp.b16 %rs1014, %rs730, %rs1034, %p140; + and.b16 %rs731, %rs1014, 255; + mov.u16 %rs732, 8; + sub.s16 %rs1048, %rs732, %rs1013; + setp.eq.s16 %p141, %rs731, 255; + selp.u16 %rs1013, 1, 0, %p141; + +$L__BB28_98: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r980, %rs1048; + and.b32 %r981, %r980, 255; + cvt.u32.u16 %r982, %rs1014; + and.b32 %r1812, %r982, 255; + shr.u32 %r983, %r1812, %r981; + and.b32 %r984, %r983, 1; + bfi.b32 %r1817, %r1817, %r984, 1, 31; + setp.eq.s32 %p142, %r147, 1; + @%p142 bra $L__BB28_109; + + and.b16 %rs733, %rs1048, 255; + setp.ne.s16 %p143, %rs733, 0; + @%p143 bra $L__BB28_103; + + setp.eq.s32 %p144, %r1791, 0; + mov.u16 %rs1038, 255; + @%p144 bra $L__BB28_102; + + cvt.u64.u32 %rd194, %r1790; + add.s64 %rd195, %rd194, %rd3; + add.s64 %rd196, %rd1, %rd195; + ld.global.u8 %rs1038, [%rd196]; + +$L__BB28_102: + setp.ne.s32 %p146, %r1791, 0; + selp.u32 %r985, 1, 0, %p146; + add.s32 %r1790, %r1790, %r985; + add.s32 %r986, %r1791, -1; + selp.b32 %r1791, 0, %r986, %p144; + setp.eq.s32 %p147, %r1791, 0; + or.b16 %rs735, %rs1038, 15; + selp.b16 %rs1014, %rs735, %rs1038, %p147; + and.b16 %rs736, %rs1014, 255; + mov.u16 %rs737, 8; + sub.s16 %rs1048, %rs737, %rs1013; + setp.eq.s16 %p148, %rs736, 255; + selp.u16 %rs1013, 1, 0, %p148; + cvt.u32.u16 %r987, %rs1014; + and.b32 %r1812, %r987, 255; + +$L__BB28_103: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r988, %rs1048; + and.b32 %r989, %r988, 255; + shr.u32 %r990, %r1812, %r989; + and.b32 %r991, %r990, 1; + bfi.b32 %r1817, %r1817, %r991, 1, 31; + setp.eq.s32 %p149, %r147, 2; + @%p149 bra $L__BB28_109; + + and.b16 %rs738, %rs1048, 255; + setp.ne.s16 %p150, %rs738, 0; + @%p150 bra $L__BB28_108; + + setp.eq.s32 %p151, %r1791, 0; + mov.u16 %rs1042, 255; + @%p151 bra $L__BB28_107; + + cvt.u64.u32 %rd197, %r1790; + add.s64 %rd198, %rd197, %rd3; + add.s64 %rd199, %rd1, %rd198; + ld.global.u8 %rs1042, [%rd199]; + +$L__BB28_107: + setp.ne.s32 %p153, %r1791, 0; + selp.u32 %r992, 1, 0, %p153; + add.s32 %r1790, %r1790, %r992; + add.s32 %r993, %r1791, -1; + selp.b32 %r1791, 0, %r993, %p151; + setp.eq.s32 %p154, %r1791, 0; + or.b16 %rs740, %rs1042, 15; + selp.b16 %rs1014, %rs740, %rs1042, %p154; + and.b16 %rs741, %rs1014, 255; + mov.u16 %rs742, 8; + sub.s16 %rs1048, %rs742, %rs1013; + setp.eq.s16 %p155, %rs741, 255; + selp.u16 %rs1013, 1, 0, %p155; + cvt.u32.u16 %r994, %rs1014; + and.b32 %r1812, %r994, 255; + +$L__BB28_108: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r995, %rs1048; + and.b32 %r996, %r995, 255; + shr.u32 %r997, %r1812, %r996; + and.b32 %r998, %r997, 1; + bfi.b32 %r1817, %r1817, %r998, 1, 31; + +$L__BB28_109: + shl.b32 %r999, %r1817, 1; + or.b32 %r1821, %r999, 1; + add.s32 %r1000, %r1881, -1; + setp.eq.s32 %p156, %r1881, 0; + selp.b32 %r1820, 0, %r1000, %p156; + +$L__BB28_110: + mul.lo.s32 %r1001, %r1882, 7; + cvt.u64.u32 %rd200, %r1821; + shl.b64 %rd201, %rd200, %r1001; + or.b64 %rd465, %rd201, %rd465; + setp.ne.s32 %p157, %r1881, 12; + setp.ne.s32 %p158, %r143, 0; + or.pred %p159, %p157, %p158; + add.s32 %r1882, %r1882, 1; + setp.lt.u32 %p160, %r1882, 8; + or.pred %p161, %p160, %p159; + mov.u32 %r1881, %r1820; + @%p161 bra $L__BB28_66; + +$L__BB28_111: + cvt.u32.u64 %r1002, %rd465; + and.b32 %r1878, %r1002, 127; + shr.u64 %rd465, %rd465, 7; + add.s32 %r1882, %r1882, -1; + +$L__BB28_112: + mul.wide.u32 %rd202, %r1765, 2; + add.s64 %rd25, %rd13, %rd202; + st.local.u16 [%rd25], %r1831; + shl.b32 %r1003, %r1831, 3; + and.b32 %r1004, %r1003, 128; + shl.b32 %r1005, %r1831, 2; + and.b32 %r1006, %r1005, 896; + or.b32 %r1007, %r1004, %r1006; + and.b32 %r1008, %r1831, 7; + shr.u64 %rd26, %rd474, %r1008; + sub.s32 %r229, %r1961, %r1008; + cvt.u32.u64 %r1009, %rd26; + and.b32 %r1010, %r1009, 127; + or.b32 %r1011, %r1010, %r1007; + mul.wide.u32 %rd203, %r1011, 2; + add.s64 %rd204, %rd12, %rd203; + ld.global.u16 %r1883, [%rd204]; + setp.ne.s32 %p162, %r1007, 0; + add.s32 %r231, %r1765, 2; + setp.ge.u32 %p163, %r231, %r824; + or.pred %p164, %p163, %p162; + @%p164 bra $L__BB28_162; + + add.s32 %r232, %r1878, -2; + setp.eq.s32 %p165, %r232, -1; + selp.b32 %r1883, %r1883, 0, %p165; + setp.gt.s32 %p166, %r1878, 1; + mov.u32 %r1878, %r232; + @%p166 bra $L__BB28_162; + + setp.ne.s32 %p167, %r1882, 0; + @%p167 bra $L__BB28_161; + + mov.u32 %r1882, 0; + +$L__BB28_116: + setp.gt.u32 %p168, %r1882, 7; + @%p168 bra $L__BB28_161; + + cvt.u64.u32 %rd28, %r1881; + mul.wide.u32 %rd205, %r1881, 4; + add.s64 %rd207, %rd133, %rd205; + ld.global.nc.u32 %r238, [%rd207]; + and.b16 %rs743, %rs1048, 255; + setp.ne.s16 %p169, %rs743, 0; + @%p169 bra $L__BB28_121; + + setp.eq.s32 %p170, %r1791, 0; + mov.u16 %rs1061, 255; + @%p170 bra $L__BB28_120; + + cvt.u64.u32 %rd208, %r1790; + add.s64 %rd209, %rd208, %rd3; + add.s64 %rd210, %rd1, %rd209; + ld.global.u8 %rs1061, [%rd210]; + +$L__BB28_120: + setp.ne.s32 %p172, %r1791, 0; + selp.u32 %r1013, 1, 0, %p172; + add.s32 %r1790, %r1790, %r1013; + add.s32 %r1014, %r1791, -1; + selp.b32 %r1791, 0, %r1014, %p170; + setp.eq.s32 %p173, %r1791, 0; + or.b16 %rs745, %rs1061, 15; + selp.b16 %rs1014, %rs745, %rs1061, %p173; + and.b16 %rs746, %rs1014, 255; + mov.u16 %rs747, 8; + sub.s16 %rs1048, %rs747, %rs1013; + setp.eq.s16 %p174, %rs746, 255; + selp.u16 %rs1013, 1, 0, %p174; + +$L__BB28_121: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1015, %rs1048; + and.b32 %r1016, %r1015, 255; + mov.u32 %r1017, 1; + shl.b32 %r1018, %r1017, %r1016; + cvt.u32.u16 %r1019, %rs1014; + and.b32 %r1020, %r1018, %r1019; + and.b32 %r243, %r1020, 255; + setp.eq.s32 %p175, %r243, 0; + @%p175 bra $L__BB28_123; + + add.s32 %r1021, %r1881, 1; + min.u32 %r1872, %r1021, 12; + mov.u32 %r1022, -1; + shl.b32 %r1023, %r1022, %r238; + shl.b32 %r1024, %r1023, 1; + xor.b32 %r1873, %r1024, -2; + bra.uni $L__BB28_160; + +$L__BB28_123: + add.s64 %rd211, %rd28, -3; + setp.gt.u64 %p176, %rd211, 9; + mov.u32 %r1869, 0; + @%p176 bra $L__BB28_159; + + max.u32 %r246, %r238, 1; + add.s32 %r1028, %r246, -1; + and.b32 %r247, %r246, 3; + setp.lt.u32 %p177, %r1028, 3; + mov.u32 %r1869, 0; + @%p177 bra $L__BB28_143; + + sub.s32 %r1841, %r246, %r247; + mov.u32 %r1869, 0; + +$L__BB28_126: + and.b16 %rs749, %rs1048, 255; + setp.ne.s16 %p178, %rs749, 0; + @%p178 bra $L__BB28_130; + + setp.eq.s32 %p179, %r1791, 0; + mov.u16 %rs1068, 255; + @%p179 bra $L__BB28_129; + + cvt.u64.u32 %rd212, %r1790; + add.s64 %rd213, %rd212, %rd3; + add.s64 %rd214, %rd1, %rd213; + ld.global.u8 %rs1068, [%rd214]; + +$L__BB28_129: + setp.ne.s32 %p181, %r1791, 0; + selp.u32 %r1030, 1, 0, %p181; + add.s32 %r1790, %r1790, %r1030; + add.s32 %r1031, %r1791, -1; + selp.b32 %r1791, 0, %r1031, %p179; + setp.eq.s32 %p182, %r1791, 0; + or.b16 %rs751, %rs1068, 15; + selp.b16 %rs1014, %rs751, %rs1068, %p182; + and.b16 %rs752, %rs1014, 255; + mov.u16 %rs753, 8; + sub.s16 %rs1048, %rs753, %rs1013; + setp.eq.s16 %p183, %rs752, 255; + selp.u16 %rs1013, 1, 0, %p183; + +$L__BB28_130: + add.s16 %rs1075, %rs1048, -1; + and.b16 %rs754, %rs1075, 255; + cvt.u32.u16 %r1032, %rs1075; + and.b32 %r1033, %r1032, 255; + cvt.u32.u16 %r1034, %rs1014; + and.b32 %r1847, %r1034, 255; + shr.u32 %r1035, %r1847, %r1033; + and.b32 %r1036, %r1035, 1; + bfi.b32 %r258, %r1869, %r1036, 1, 31; + setp.ne.s16 %p184, %rs754, 0; + @%p184 bra $L__BB28_134; + + setp.eq.s32 %p185, %r1791, 0; + mov.u16 %rs1072, 255; + @%p185 bra $L__BB28_133; + + cvt.u64.u32 %rd215, %r1790; + add.s64 %rd216, %rd215, %rd3; + add.s64 %rd217, %rd1, %rd216; + ld.global.u8 %rs1072, [%rd217]; + +$L__BB28_133: + setp.ne.s32 %p187, %r1791, 0; + selp.u32 %r1037, 1, 0, %p187; + add.s32 %r1790, %r1790, %r1037; + add.s32 %r1038, %r1791, -1; + selp.b32 %r1791, 0, %r1038, %p185; + setp.eq.s32 %p188, %r1791, 0; + or.b16 %rs756, %rs1072, 15; + selp.b16 %rs1014, %rs756, %rs1072, %p188; + and.b16 %rs757, %rs1014, 255; + mov.u16 %rs758, 8; + sub.s16 %rs1075, %rs758, %rs1013; + setp.eq.s16 %p189, %rs757, 255; + selp.u16 %rs1013, 1, 0, %p189; + cvt.u32.u16 %r1039, %rs1014; + and.b32 %r1847, %r1039, 255; + +$L__BB28_134: + add.s16 %rs1079, %rs1075, -1; + and.b16 %rs759, %rs1079, 255; + cvt.u32.u16 %r1040, %rs1079; + and.b32 %r1041, %r1040, 255; + shr.u32 %r1042, %r1847, %r1041; + and.b32 %r1043, %r1042, 1; + bfi.b32 %r265, %r258, %r1043, 1, 31; + setp.ne.s16 %p190, %rs759, 0; + @%p190 bra $L__BB28_138; + + setp.eq.s32 %p191, %r1791, 0; + mov.u16 %rs1076, 255; + @%p191 bra $L__BB28_137; + + cvt.u64.u32 %rd218, %r1790; + add.s64 %rd219, %rd218, %rd3; + add.s64 %rd220, %rd1, %rd219; + ld.global.u8 %rs1076, [%rd220]; + +$L__BB28_137: + setp.ne.s32 %p193, %r1791, 0; + selp.u32 %r1044, 1, 0, %p193; + add.s32 %r1790, %r1790, %r1044; + add.s32 %r1045, %r1791, -1; + selp.b32 %r1791, 0, %r1045, %p191; + setp.eq.s32 %p194, %r1791, 0; + or.b16 %rs761, %rs1076, 15; + selp.b16 %rs1014, %rs761, %rs1076, %p194; + and.b16 %rs762, %rs1014, 255; + mov.u16 %rs763, 8; + sub.s16 %rs1079, %rs763, %rs1013; + setp.eq.s16 %p195, %rs762, 255; + selp.u16 %rs1013, 1, 0, %p195; + cvt.u32.u16 %r1046, %rs1014; + and.b32 %r1847, %r1046, 255; + +$L__BB28_138: + add.s16 %rs1083, %rs1079, -1; + and.b16 %rs764, %rs1083, 255; + cvt.u32.u16 %r1047, %rs1083; + and.b32 %r1048, %r1047, 255; + shr.u32 %r1049, %r1847, %r1048; + and.b32 %r1050, %r1049, 1; + bfi.b32 %r272, %r265, %r1050, 1, 31; + setp.ne.s16 %p196, %rs764, 0; + @%p196 bra $L__BB28_142; + + setp.eq.s32 %p197, %r1791, 0; + mov.u16 %rs1080, 255; + @%p197 bra $L__BB28_141; + + cvt.u64.u32 %rd221, %r1790; + add.s64 %rd222, %rd221, %rd3; + add.s64 %rd223, %rd1, %rd222; + ld.global.u8 %rs1080, [%rd223]; + +$L__BB28_141: + setp.ne.s32 %p199, %r1791, 0; + selp.u32 %r1051, 1, 0, %p199; + add.s32 %r1790, %r1790, %r1051; + add.s32 %r1052, %r1791, -1; + selp.b32 %r1791, 0, %r1052, %p197; + setp.eq.s32 %p200, %r1791, 0; + or.b16 %rs766, %rs1080, 15; + selp.b16 %rs1014, %rs766, %rs1080, %p200; + and.b16 %rs767, %rs1014, 255; + mov.u16 %rs768, 8; + sub.s16 %rs1083, %rs768, %rs1013; + setp.eq.s16 %p201, %rs767, 255; + selp.u16 %rs1013, 1, 0, %p201; + cvt.u32.u16 %r1053, %rs1014; + and.b32 %r1847, %r1053, 255; + +$L__BB28_142: + add.s16 %rs1048, %rs1083, -1; + cvt.u32.u16 %r1054, %rs1048; + and.b32 %r1055, %r1054, 255; + shr.u32 %r1056, %r1847, %r1055; + and.b32 %r1057, %r1056, 1; + bfi.b32 %r1869, %r272, %r1057, 1, 31; + add.s32 %r1841, %r1841, -4; + setp.ne.s32 %p202, %r1841, 0; + @%p202 bra $L__BB28_126; + +$L__BB28_143: + setp.eq.s32 %p203, %r247, 0; + @%p203 bra $L__BB28_159; + + and.b16 %rs769, %rs1048, 255; + setp.ne.s16 %p204, %rs769, 0; + @%p204 bra $L__BB28_148; + + setp.eq.s32 %p205, %r1791, 0; + mov.u16 %rs1090, 255; + @%p205 bra $L__BB28_147; + + cvt.u64.u32 %rd224, %r1790; + add.s64 %rd225, %rd224, %rd3; + add.s64 %rd226, %rd1, %rd225; + ld.global.u8 %rs1090, [%rd226]; + +$L__BB28_147: + setp.ne.s32 %p207, %r1791, 0; + selp.u32 %r1058, 1, 0, %p207; + add.s32 %r1790, %r1790, %r1058; + add.s32 %r1059, %r1791, -1; + selp.b32 %r1791, 0, %r1059, %p205; + setp.eq.s32 %p208, %r1791, 0; + or.b16 %rs771, %rs1090, 15; + selp.b16 %rs1014, %rs771, %rs1090, %p208; + and.b16 %rs772, %rs1014, 255; + mov.u16 %rs773, 8; + sub.s16 %rs1048, %rs773, %rs1013; + setp.eq.s16 %p209, %rs772, 255; + selp.u16 %rs1013, 1, 0, %p209; + +$L__BB28_148: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1060, %rs1048; + and.b32 %r1061, %r1060, 255; + cvt.u32.u16 %r1062, %rs1014; + and.b32 %r1864, %r1062, 255; + shr.u32 %r1063, %r1864, %r1061; + and.b32 %r1064, %r1063, 1; + bfi.b32 %r1869, %r1869, %r1064, 1, 31; + setp.eq.s32 %p210, %r247, 1; + @%p210 bra $L__BB28_159; + + and.b16 %rs774, %rs1048, 255; + setp.ne.s16 %p211, %rs774, 0; + @%p211 bra $L__BB28_153; + + setp.eq.s32 %p212, %r1791, 0; + mov.u16 %rs1094, 255; + @%p212 bra $L__BB28_152; + + cvt.u64.u32 %rd227, %r1790; + add.s64 %rd228, %rd227, %rd3; + add.s64 %rd229, %rd1, %rd228; + ld.global.u8 %rs1094, [%rd229]; + +$L__BB28_152: + setp.ne.s32 %p214, %r1791, 0; + selp.u32 %r1065, 1, 0, %p214; + add.s32 %r1790, %r1790, %r1065; + add.s32 %r1066, %r1791, -1; + selp.b32 %r1791, 0, %r1066, %p212; + setp.eq.s32 %p215, %r1791, 0; + or.b16 %rs776, %rs1094, 15; + selp.b16 %rs1014, %rs776, %rs1094, %p215; + and.b16 %rs777, %rs1014, 255; + mov.u16 %rs778, 8; + sub.s16 %rs1048, %rs778, %rs1013; + setp.eq.s16 %p216, %rs777, 255; + selp.u16 %rs1013, 1, 0, %p216; + cvt.u32.u16 %r1067, %rs1014; + and.b32 %r1864, %r1067, 255; + +$L__BB28_153: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1068, %rs1048; + and.b32 %r1069, %r1068, 255; + shr.u32 %r1070, %r1864, %r1069; + and.b32 %r1071, %r1070, 1; + bfi.b32 %r1869, %r1869, %r1071, 1, 31; + setp.eq.s32 %p217, %r247, 2; + @%p217 bra $L__BB28_159; + + and.b16 %rs779, %rs1048, 255; + setp.ne.s16 %p218, %rs779, 0; + @%p218 bra $L__BB28_158; + + setp.eq.s32 %p219, %r1791, 0; + mov.u16 %rs1098, 255; + @%p219 bra $L__BB28_157; + + cvt.u64.u32 %rd230, %r1790; + add.s64 %rd231, %rd230, %rd3; + add.s64 %rd232, %rd1, %rd231; + ld.global.u8 %rs1098, [%rd232]; + +$L__BB28_157: + setp.ne.s32 %p221, %r1791, 0; + selp.u32 %r1072, 1, 0, %p221; + add.s32 %r1790, %r1790, %r1072; + add.s32 %r1073, %r1791, -1; + selp.b32 %r1791, 0, %r1073, %p219; + setp.eq.s32 %p222, %r1791, 0; + or.b16 %rs781, %rs1098, 15; + selp.b16 %rs1014, %rs781, %rs1098, %p222; + and.b16 %rs782, %rs1014, 255; + mov.u16 %rs783, 8; + sub.s16 %rs1048, %rs783, %rs1013; + setp.eq.s16 %p223, %rs782, 255; + selp.u16 %rs1013, 1, 0, %p223; + cvt.u32.u16 %r1074, %rs1014; + and.b32 %r1864, %r1074, 255; + +$L__BB28_158: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1075, %rs1048; + and.b32 %r1076, %r1075, 255; + shr.u32 %r1077, %r1864, %r1076; + and.b32 %r1078, %r1077, 1; + bfi.b32 %r1869, %r1869, %r1078, 1, 31; + +$L__BB28_159: + shl.b32 %r1079, %r1869, 1; + or.b32 %r1873, %r1079, 1; + add.s32 %r1080, %r1881, -1; + setp.eq.s32 %p224, %r1881, 0; + selp.b32 %r1872, 0, %r1080, %p224; + +$L__BB28_160: + mul.lo.s32 %r1081, %r1882, 7; + cvt.u64.u32 %rd233, %r1873; + shl.b64 %rd234, %rd233, %r1081; + or.b64 %rd465, %rd234, %rd465; + setp.ne.s32 %p225, %r1881, 12; + setp.ne.s32 %p226, %r243, 0; + or.pred %p227, %p225, %p226; + add.s32 %r1882, %r1882, 1; + setp.lt.u32 %p228, %r1882, 8; + or.pred %p229, %p228, %p227; + mov.u32 %r1881, %r1872; + @%p229 bra $L__BB28_116; + +$L__BB28_161: + cvt.u32.u64 %r1082, %rd465; + and.b32 %r1878, %r1082, 127; + shr.u64 %rd465, %rd465, 7; + add.s32 %r1882, %r1882, -1; + +$L__BB28_162: + setp.lt.u32 %p230, %r231, %r824; + selp.b32 %r329, %r1883, 0, %p230; + st.local.u16 [%rd25+4], %r329; + and.b32 %r1084, %r1003, 64; + shl.b32 %r1085, %r329, 4; + and.b32 %r1086, %r1085, 128; + or.b32 %r1935, %r1086, %r1084; + setp.ne.s32 %p231, %r1935, 192; + @%p231 bra $L__BB28_212; + + add.s32 %r331, %r1878, -2; + setp.eq.s32 %p232, %r331, -1; + selp.b32 %r1935, 256, 192, %p232; + setp.gt.s32 %p233, %r1878, 1; + mov.u32 %r1878, %r331; + @%p233 bra $L__BB28_212; + + setp.ne.s32 %p234, %r1882, 0; + @%p234 bra $L__BB28_211; + + mov.u32 %r1882, 0; + +$L__BB28_166: + setp.gt.u32 %p235, %r1882, 7; + @%p235 bra $L__BB28_211; + + cvt.u64.u32 %rd34, %r1881; + mul.wide.u32 %rd235, %r1881, 4; + add.s64 %rd237, %rd133, %rd235; + ld.global.nc.u32 %r337, [%rd237]; + and.b16 %rs784, %rs1048, 255; + setp.ne.s16 %p236, %rs784, 0; + @%p236 bra $L__BB28_171; + + setp.eq.s32 %p237, %r1791, 0; + mov.u16 %rs1117, 255; + @%p237 bra $L__BB28_170; + + cvt.u64.u32 %rd238, %r1790; + add.s64 %rd239, %rd238, %rd3; + add.s64 %rd240, %rd1, %rd239; + ld.global.u8 %rs1117, [%rd240]; + +$L__BB28_170: + setp.ne.s32 %p239, %r1791, 0; + selp.u32 %r1088, 1, 0, %p239; + add.s32 %r1790, %r1790, %r1088; + add.s32 %r1089, %r1791, -1; + selp.b32 %r1791, 0, %r1089, %p237; + setp.eq.s32 %p240, %r1791, 0; + or.b16 %rs786, %rs1117, 15; + selp.b16 %rs1014, %rs786, %rs1117, %p240; + and.b16 %rs787, %rs1014, 255; + mov.u16 %rs788, 8; + sub.s16 %rs1048, %rs788, %rs1013; + setp.eq.s16 %p241, %rs787, 255; + selp.u16 %rs1013, 1, 0, %p241; + +$L__BB28_171: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1090, %rs1048; + and.b32 %r1091, %r1090, 255; + mov.u32 %r1092, 1; + shl.b32 %r1093, %r1092, %r1091; + cvt.u32.u16 %r1094, %rs1014; + and.b32 %r1095, %r1093, %r1094; + and.b32 %r342, %r1095, 255; + setp.eq.s32 %p242, %r342, 0; + @%p242 bra $L__BB28_173; + + add.s32 %r1096, %r1881, 1; + min.u32 %r1924, %r1096, 12; + mov.u32 %r1097, -1; + shl.b32 %r1098, %r1097, %r337; + shl.b32 %r1099, %r1098, 1; + xor.b32 %r1925, %r1099, -2; + bra.uni $L__BB28_210; + +$L__BB28_173: + add.s64 %rd241, %rd34, -3; + setp.gt.u64 %p243, %rd241, 9; + mov.u32 %r1921, 0; + @%p243 bra $L__BB28_209; + + max.u32 %r345, %r337, 1; + add.s32 %r1103, %r345, -1; + and.b32 %r346, %r345, 3; + setp.lt.u32 %p244, %r1103, 3; + mov.u32 %r1921, 0; + @%p244 bra $L__BB28_193; + + sub.s32 %r1893, %r345, %r346; + mov.u32 %r1921, 0; + +$L__BB28_176: + and.b16 %rs790, %rs1048, 255; + setp.ne.s16 %p245, %rs790, 0; + @%p245 bra $L__BB28_180; + + setp.eq.s32 %p246, %r1791, 0; + mov.u16 %rs1124, 255; + @%p246 bra $L__BB28_179; + + cvt.u64.u32 %rd242, %r1790; + add.s64 %rd243, %rd242, %rd3; + add.s64 %rd244, %rd1, %rd243; + ld.global.u8 %rs1124, [%rd244]; + +$L__BB28_179: + setp.ne.s32 %p248, %r1791, 0; + selp.u32 %r1105, 1, 0, %p248; + add.s32 %r1790, %r1790, %r1105; + add.s32 %r1106, %r1791, -1; + selp.b32 %r1791, 0, %r1106, %p246; + setp.eq.s32 %p249, %r1791, 0; + or.b16 %rs792, %rs1124, 15; + selp.b16 %rs1014, %rs792, %rs1124, %p249; + and.b16 %rs793, %rs1014, 255; + mov.u16 %rs794, 8; + sub.s16 %rs1048, %rs794, %rs1013; + setp.eq.s16 %p250, %rs793, 255; + selp.u16 %rs1013, 1, 0, %p250; + +$L__BB28_180: + add.s16 %rs1131, %rs1048, -1; + and.b16 %rs795, %rs1131, 255; + cvt.u32.u16 %r1107, %rs1131; + and.b32 %r1108, %r1107, 255; + cvt.u32.u16 %r1109, %rs1014; + and.b32 %r1899, %r1109, 255; + shr.u32 %r1110, %r1899, %r1108; + and.b32 %r1111, %r1110, 1; + bfi.b32 %r357, %r1921, %r1111, 1, 31; + setp.ne.s16 %p251, %rs795, 0; + @%p251 bra $L__BB28_184; + + setp.eq.s32 %p252, %r1791, 0; + mov.u16 %rs1128, 255; + @%p252 bra $L__BB28_183; + + cvt.u64.u32 %rd245, %r1790; + add.s64 %rd246, %rd245, %rd3; + add.s64 %rd247, %rd1, %rd246; + ld.global.u8 %rs1128, [%rd247]; + +$L__BB28_183: + setp.ne.s32 %p254, %r1791, 0; + selp.u32 %r1112, 1, 0, %p254; + add.s32 %r1790, %r1790, %r1112; + add.s32 %r1113, %r1791, -1; + selp.b32 %r1791, 0, %r1113, %p252; + setp.eq.s32 %p255, %r1791, 0; + or.b16 %rs797, %rs1128, 15; + selp.b16 %rs1014, %rs797, %rs1128, %p255; + and.b16 %rs798, %rs1014, 255; + mov.u16 %rs799, 8; + sub.s16 %rs1131, %rs799, %rs1013; + setp.eq.s16 %p256, %rs798, 255; + selp.u16 %rs1013, 1, 0, %p256; + cvt.u32.u16 %r1114, %rs1014; + and.b32 %r1899, %r1114, 255; + +$L__BB28_184: + add.s16 %rs1135, %rs1131, -1; + and.b16 %rs800, %rs1135, 255; + cvt.u32.u16 %r1115, %rs1135; + and.b32 %r1116, %r1115, 255; + shr.u32 %r1117, %r1899, %r1116; + and.b32 %r1118, %r1117, 1; + bfi.b32 %r364, %r357, %r1118, 1, 31; + setp.ne.s16 %p257, %rs800, 0; + @%p257 bra $L__BB28_188; + + setp.eq.s32 %p258, %r1791, 0; + mov.u16 %rs1132, 255; + @%p258 bra $L__BB28_187; + + cvt.u64.u32 %rd248, %r1790; + add.s64 %rd249, %rd248, %rd3; + add.s64 %rd250, %rd1, %rd249; + ld.global.u8 %rs1132, [%rd250]; + +$L__BB28_187: + setp.ne.s32 %p260, %r1791, 0; + selp.u32 %r1119, 1, 0, %p260; + add.s32 %r1790, %r1790, %r1119; + add.s32 %r1120, %r1791, -1; + selp.b32 %r1791, 0, %r1120, %p258; + setp.eq.s32 %p261, %r1791, 0; + or.b16 %rs802, %rs1132, 15; + selp.b16 %rs1014, %rs802, %rs1132, %p261; + and.b16 %rs803, %rs1014, 255; + mov.u16 %rs804, 8; + sub.s16 %rs1135, %rs804, %rs1013; + setp.eq.s16 %p262, %rs803, 255; + selp.u16 %rs1013, 1, 0, %p262; + cvt.u32.u16 %r1121, %rs1014; + and.b32 %r1899, %r1121, 255; + +$L__BB28_188: + add.s16 %rs1139, %rs1135, -1; + and.b16 %rs805, %rs1139, 255; + cvt.u32.u16 %r1122, %rs1139; + and.b32 %r1123, %r1122, 255; + shr.u32 %r1124, %r1899, %r1123; + and.b32 %r1125, %r1124, 1; + bfi.b32 %r371, %r364, %r1125, 1, 31; + setp.ne.s16 %p263, %rs805, 0; + @%p263 bra $L__BB28_192; + + setp.eq.s32 %p264, %r1791, 0; + mov.u16 %rs1136, 255; + @%p264 bra $L__BB28_191; + + cvt.u64.u32 %rd251, %r1790; + add.s64 %rd252, %rd251, %rd3; + add.s64 %rd253, %rd1, %rd252; + ld.global.u8 %rs1136, [%rd253]; + +$L__BB28_191: + setp.ne.s32 %p266, %r1791, 0; + selp.u32 %r1126, 1, 0, %p266; + add.s32 %r1790, %r1790, %r1126; + add.s32 %r1127, %r1791, -1; + selp.b32 %r1791, 0, %r1127, %p264; + setp.eq.s32 %p267, %r1791, 0; + or.b16 %rs807, %rs1136, 15; + selp.b16 %rs1014, %rs807, %rs1136, %p267; + and.b16 %rs808, %rs1014, 255; + mov.u16 %rs809, 8; + sub.s16 %rs1139, %rs809, %rs1013; + setp.eq.s16 %p268, %rs808, 255; + selp.u16 %rs1013, 1, 0, %p268; + cvt.u32.u16 %r1128, %rs1014; + and.b32 %r1899, %r1128, 255; + +$L__BB28_192: + add.s16 %rs1048, %rs1139, -1; + cvt.u32.u16 %r1129, %rs1048; + and.b32 %r1130, %r1129, 255; + shr.u32 %r1131, %r1899, %r1130; + and.b32 %r1132, %r1131, 1; + bfi.b32 %r1921, %r371, %r1132, 1, 31; + add.s32 %r1893, %r1893, -4; + setp.ne.s32 %p269, %r1893, 0; + @%p269 bra $L__BB28_176; + +$L__BB28_193: + setp.eq.s32 %p270, %r346, 0; + @%p270 bra $L__BB28_209; + + and.b16 %rs810, %rs1048, 255; + setp.ne.s16 %p271, %rs810, 0; + @%p271 bra $L__BB28_198; + + setp.eq.s32 %p272, %r1791, 0; + mov.u16 %rs1146, 255; + @%p272 bra $L__BB28_197; + + cvt.u64.u32 %rd254, %r1790; + add.s64 %rd255, %rd254, %rd3; + add.s64 %rd256, %rd1, %rd255; + ld.global.u8 %rs1146, [%rd256]; + +$L__BB28_197: + setp.ne.s32 %p274, %r1791, 0; + selp.u32 %r1133, 1, 0, %p274; + add.s32 %r1790, %r1790, %r1133; + add.s32 %r1134, %r1791, -1; + selp.b32 %r1791, 0, %r1134, %p272; + setp.eq.s32 %p275, %r1791, 0; + or.b16 %rs812, %rs1146, 15; + selp.b16 %rs1014, %rs812, %rs1146, %p275; + and.b16 %rs813, %rs1014, 255; + mov.u16 %rs814, 8; + sub.s16 %rs1048, %rs814, %rs1013; + setp.eq.s16 %p276, %rs813, 255; + selp.u16 %rs1013, 1, 0, %p276; + +$L__BB28_198: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1135, %rs1048; + and.b32 %r1136, %r1135, 255; + cvt.u32.u16 %r1137, %rs1014; + and.b32 %r1916, %r1137, 255; + shr.u32 %r1138, %r1916, %r1136; + and.b32 %r1139, %r1138, 1; + bfi.b32 %r1921, %r1921, %r1139, 1, 31; + setp.eq.s32 %p277, %r346, 1; + @%p277 bra $L__BB28_209; + + and.b16 %rs815, %rs1048, 255; + setp.ne.s16 %p278, %rs815, 0; + @%p278 bra $L__BB28_203; + + setp.eq.s32 %p279, %r1791, 0; + mov.u16 %rs1150, 255; + @%p279 bra $L__BB28_202; + + cvt.u64.u32 %rd257, %r1790; + add.s64 %rd258, %rd257, %rd3; + add.s64 %rd259, %rd1, %rd258; + ld.global.u8 %rs1150, [%rd259]; + +$L__BB28_202: + setp.ne.s32 %p281, %r1791, 0; + selp.u32 %r1140, 1, 0, %p281; + add.s32 %r1790, %r1790, %r1140; + add.s32 %r1141, %r1791, -1; + selp.b32 %r1791, 0, %r1141, %p279; + setp.eq.s32 %p282, %r1791, 0; + or.b16 %rs817, %rs1150, 15; + selp.b16 %rs1014, %rs817, %rs1150, %p282; + and.b16 %rs818, %rs1014, 255; + mov.u16 %rs819, 8; + sub.s16 %rs1048, %rs819, %rs1013; + setp.eq.s16 %p283, %rs818, 255; + selp.u16 %rs1013, 1, 0, %p283; + cvt.u32.u16 %r1142, %rs1014; + and.b32 %r1916, %r1142, 255; + +$L__BB28_203: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1143, %rs1048; + and.b32 %r1144, %r1143, 255; + shr.u32 %r1145, %r1916, %r1144; + and.b32 %r1146, %r1145, 1; + bfi.b32 %r1921, %r1921, %r1146, 1, 31; + setp.eq.s32 %p284, %r346, 2; + @%p284 bra $L__BB28_209; + + and.b16 %rs820, %rs1048, 255; + setp.ne.s16 %p285, %rs820, 0; + @%p285 bra $L__BB28_208; + + setp.eq.s32 %p286, %r1791, 0; + mov.u16 %rs1154, 255; + @%p286 bra $L__BB28_207; + + cvt.u64.u32 %rd260, %r1790; + add.s64 %rd261, %rd260, %rd3; + add.s64 %rd262, %rd1, %rd261; + ld.global.u8 %rs1154, [%rd262]; + +$L__BB28_207: + setp.ne.s32 %p288, %r1791, 0; + selp.u32 %r1147, 1, 0, %p288; + add.s32 %r1790, %r1790, %r1147; + add.s32 %r1148, %r1791, -1; + selp.b32 %r1791, 0, %r1148, %p286; + setp.eq.s32 %p289, %r1791, 0; + or.b16 %rs822, %rs1154, 15; + selp.b16 %rs1014, %rs822, %rs1154, %p289; + and.b16 %rs823, %rs1014, 255; + mov.u16 %rs824, 8; + sub.s16 %rs1048, %rs824, %rs1013; + setp.eq.s16 %p290, %rs823, 255; + selp.u16 %rs1013, 1, 0, %p290; + cvt.u32.u16 %r1149, %rs1014; + and.b32 %r1916, %r1149, 255; + +$L__BB28_208: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1150, %rs1048; + and.b32 %r1151, %r1150, 255; + shr.u32 %r1152, %r1916, %r1151; + and.b32 %r1153, %r1152, 1; + bfi.b32 %r1921, %r1921, %r1153, 1, 31; + +$L__BB28_209: + shl.b32 %r1154, %r1921, 1; + or.b32 %r1925, %r1154, 1; + add.s32 %r1155, %r1881, -1; + setp.eq.s32 %p291, %r1881, 0; + selp.b32 %r1924, 0, %r1155, %p291; + +$L__BB28_210: + mul.lo.s32 %r1156, %r1882, 7; + cvt.u64.u32 %rd263, %r1925; + shl.b64 %rd264, %rd263, %r1156; + or.b64 %rd465, %rd264, %rd465; + setp.ne.s32 %p292, %r1881, 12; + setp.ne.s32 %p293, %r342, 0; + or.pred %p294, %p292, %p293; + add.s32 %r1882, %r1882, 1; + setp.lt.u32 %p295, %r1882, 8; + or.pred %p296, %p295, %p294; + mov.u32 %r1881, %r1924; + @%p296 bra $L__BB28_166; + +$L__BB28_211: + cvt.u32.u64 %r1157, %rd465; + and.b32 %r1878, %r1157, 127; + shr.u64 %rd465, %rd465, 7; + add.s32 %r1882, %r1882, -1; + +$L__BB28_212: + and.b32 %r1158, %r329, 7; + shr.u64 %rd265, %rd26, %r1158; + cvt.u32.u64 %r1159, %rd265; + and.b32 %r1160, %r1159, 63; + add.s32 %r1161, %r1935, %r1160; + mul.wide.u32 %rd266, %r1161, 2; + add.s64 %rd267, %rd11, %rd266; + ld.global.u16 %r1162, [%rd267]; + and.b32 %r1163, %r1162, 7; + shr.u64 %rd268, %rd265, %r1163; + sub.s32 %r1164, %r229, %r1158; + sub.s32 %r1165, %r1164, %r1163; + cvt.u32.u64 %r1166, %rd268; + shr.u32 %r1167, %r1162, 3; + and.b32 %r1168, %r1167, 15; + mov.u32 %r1169, -1; + shl.b32 %r1170, %r1169, %r1168; + not.b32 %r1171, %r1170; + and.b32 %r1172, %r1166, %r1171; + shr.u64 %rd474, %rd268, %r1168; + sub.s32 %r1961, %r1165, %r1168; + shr.u32 %r1173, %r1162, 7; + and.b32 %r1174, %r1173, 7; + shr.u32 %r1175, %r1162, 10; + and.b32 %r1176, %r1175, 7; + mov.u32 %r1177, 255; + shl.b32 %r1178, %r1177, %r1174; + not.b32 %r1179, %r1178; + and.b32 %r1180, %r1172, %r1179; + add.s32 %r1181, %r1176, %r1180; + add.s32 %r1182, %r1181, 1; + st.local.u16 [%rd25+2], %r1182; + shr.u32 %r1183, %r1162, 13; + shr.u32 %r1184, %r1172, %r1174; + add.s32 %r1185, %r1183, %r1184; + add.s32 %r1186, %r1185, 1; + st.local.u16 [%rd25+6], %r1186; + add.s32 %r1765, %r1765, 4; + setp.lt.u32 %p297, %r1765, %r824; + shl.b32 %r1187, %r329, 2; + and.b32 %r1188, %r1187, 896; + shl.b32 %r1189, %r329, 3; + and.b32 %r1190, %r1189, 128; + or.b32 %r1764, %r1190, %r1188; + @%p297 bra $L__BB28_58; + + mul.wide.u32 %rd271, %r1765, 2; + add.s64 %rd272, %rd13, %rd271; + mov.u16 %rs825, 0; + st.local.v2.u16 [%rd272], {%rs825, %rs825}; + setp.lt.u32 %p298, %r826, 3; + @%p298 bra $L__BB28_322; + + ld.param.u64 %rd451, [ j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_3]; + ld.param.u64 %rd450, [ j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize_param_5]; + cvta.to.global.u64 %rd40, %rd450; + cvta.to.global.u64 %rd41, %rd451; + mov.u32 %r1936, 2; + +$L__BB28_215: + shr.u32 %r1195, %r1936, 1; + mul.lo.s32 %r441, %r1195, %r849; + sub.s32 %r442, %r441, %r849; + mov.u32 %r1945, 0; + mov.u32 %r1946, %r1945; + mov.u32 %r1947, %r441; + +$L__BB28_216: + sub.s32 %r1196, %r1947, %r441; + add.s32 %r454, %r1196, %r442; + mul.wide.u32 %rd273, %r454, 2; + add.s64 %rd46, %rd13, %rd273; + ld.local.u16 %r1197, [%rd46]; + shl.b32 %r1198, %r1197, 2; + and.b32 %r1199, %r1198, 640; + or.b32 %r1200, %r1946, %r1199; + add.s32 %r1201, %r454, 2; + mul.wide.u32 %rd274, %r1201, 2; + add.s64 %rd47, %rd13, %rd274; + ld.local.u16 %r1202, [%rd47]; + shl.b32 %r1203, %r1202, 4; + and.b32 %r1204, %r1203, 512; + or.b32 %r455, %r1200, %r1204; + setp.gt.u32 %p299, %r1961, 31; + @%p299 bra $L__BB28_220; + +$L__BB28_217: + setp.eq.s32 %p300, %r1960, 0; + mov.u16 %rs1179, 0; + @%p300 bra $L__BB28_219; + + cvt.s64.s32 %rd275, %r1959; + add.s64 %rd276, %rd275, %rd3; + add.s64 %rd277, %rd1, %rd276; + ld.global.u8 %rs1179, [%rd277]; + +$L__BB28_219: + setp.ne.s32 %p302, %r1960, 0; + selp.b32 %r1205, -1, 0, %p302; + add.s32 %r1959, %r1959, %r1205; + add.s32 %r1206, %r1960, -1; + selp.b32 %r1960, 0, %r1206, %p300; + and.b16 %rs827, %rs1179, 255; + and.b16 %rs828, %rs1179, 127; + setp.eq.s16 %p303, %rs828, 127; + and.b16 %rs829, %rs1180, 255; + setp.ne.s16 %p304, %rs829, 0; + and.pred %p305, %p304, %p303; + selp.b32 %r1207, 7, 8, %p305; + cvt.u64.u16 %rd278, %rs1179; + and.b64 %rd279, %rd278, 255; + shl.b64 %rd280, %rd279, %r1961; + or.b64 %rd474, %rd280, %rd474; + add.s32 %r1961, %r1207, %r1961; + setp.gt.u16 %p306, %rs827, 143; + selp.u16 %rs1180, 1, 0, %p306; + setp.lt.u32 %p307, %r1961, 33; + @%p307 bra $L__BB28_217; + +$L__BB28_220: + cvt.u32.u64 %r1208, %rd474; + and.b32 %r1209, %r1208, 127; + add.s32 %r1210, %r1209, %r455; + mul.wide.u32 %rd281, %r1210, 2; + add.s64 %rd282, %rd41, %rd281; + ld.global.u16 %r2013, [%rd282]; + setp.ne.s32 %p308, %r455, 0; + @%p308 bra $L__BB28_270; + + add.s32 %r466, %r1878, -2; + setp.eq.s32 %p309, %r466, -1; + selp.b32 %r2013, %r2013, 0, %p309; + setp.gt.s32 %p310, %r1878, 1; + mov.u32 %r1878, %r466; + @%p310 bra $L__BB28_270; + + setp.ne.s32 %p311, %r1882, 0; + @%p311 bra $L__BB28_269; + + mov.u32 %r1882, 0; + +$L__BB28_224: + setp.gt.u32 %p312, %r1882, 7; + @%p312 bra $L__BB28_269; + + cvt.u64.u32 %rd52, %r1881; + mul.wide.u32 %rd283, %r1881, 4; + add.s64 %rd285, %rd133, %rd283; + ld.global.nc.u32 %r472, [%rd285]; + and.b16 %rs830, %rs1048, 255; + setp.ne.s16 %p313, %rs830, 0; + @%p313 bra $L__BB28_229; + + setp.eq.s32 %p314, %r1791, 0; + mov.u16 %rs1184, 255; + @%p314 bra $L__BB28_228; + + cvt.u64.u32 %rd286, %r1790; + add.s64 %rd287, %rd286, %rd3; + add.s64 %rd288, %rd1, %rd287; + ld.global.u8 %rs1184, [%rd288]; + +$L__BB28_228: + setp.ne.s32 %p316, %r1791, 0; + selp.u32 %r1212, 1, 0, %p316; + add.s32 %r1790, %r1790, %r1212; + add.s32 %r1213, %r1791, -1; + selp.b32 %r1791, 0, %r1213, %p314; + setp.eq.s32 %p317, %r1791, 0; + or.b16 %rs832, %rs1184, 15; + selp.b16 %rs1014, %rs832, %rs1184, %p317; + and.b16 %rs833, %rs1014, 255; + mov.u16 %rs834, 8; + sub.s16 %rs1048, %rs834, %rs1013; + setp.eq.s16 %p318, %rs833, 255; + selp.u16 %rs1013, 1, 0, %p318; + +$L__BB28_229: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1214, %rs1048; + and.b32 %r1215, %r1214, 255; + mov.u32 %r1216, 1; + shl.b32 %r1217, %r1216, %r1215; + cvt.u32.u16 %r1218, %rs1014; + and.b32 %r1219, %r1217, %r1218; + and.b32 %r477, %r1219, 255; + setp.eq.s32 %p319, %r477, 0; + @%p319 bra $L__BB28_231; + + add.s32 %r1220, %r1881, 1; + min.u32 %r2002, %r1220, 12; + mov.u32 %r1221, -1; + shl.b32 %r1222, %r1221, %r472; + shl.b32 %r1223, %r1222, 1; + xor.b32 %r2003, %r1223, -2; + bra.uni $L__BB28_268; + +$L__BB28_231: + add.s64 %rd289, %rd52, -3; + setp.gt.u64 %p320, %rd289, 9; + mov.u32 %r1999, 0; + @%p320 bra $L__BB28_267; + + max.u32 %r480, %r472, 1; + add.s32 %r1227, %r480, -1; + and.b32 %r481, %r480, 3; + setp.lt.u32 %p321, %r1227, 3; + mov.u32 %r1999, 0; + @%p321 bra $L__BB28_251; + + sub.s32 %r1971, %r480, %r481; + mov.u32 %r1999, 0; + +$L__BB28_234: + and.b16 %rs836, %rs1048, 255; + setp.ne.s16 %p322, %rs836, 0; + @%p322 bra $L__BB28_238; + + setp.eq.s32 %p323, %r1791, 0; + mov.u16 %rs1191, 255; + @%p323 bra $L__BB28_237; + + cvt.u64.u32 %rd290, %r1790; + add.s64 %rd291, %rd290, %rd3; + add.s64 %rd292, %rd1, %rd291; + ld.global.u8 %rs1191, [%rd292]; + +$L__BB28_237: + setp.ne.s32 %p325, %r1791, 0; + selp.u32 %r1229, 1, 0, %p325; + add.s32 %r1790, %r1790, %r1229; + add.s32 %r1230, %r1791, -1; + selp.b32 %r1791, 0, %r1230, %p323; + setp.eq.s32 %p326, %r1791, 0; + or.b16 %rs838, %rs1191, 15; + selp.b16 %rs1014, %rs838, %rs1191, %p326; + and.b16 %rs839, %rs1014, 255; + mov.u16 %rs840, 8; + sub.s16 %rs1048, %rs840, %rs1013; + setp.eq.s16 %p327, %rs839, 255; + selp.u16 %rs1013, 1, 0, %p327; + +$L__BB28_238: + add.s16 %rs1198, %rs1048, -1; + and.b16 %rs841, %rs1198, 255; + cvt.u32.u16 %r1231, %rs1198; + and.b32 %r1232, %r1231, 255; + cvt.u32.u16 %r1233, %rs1014; + and.b32 %r1977, %r1233, 255; + shr.u32 %r1234, %r1977, %r1232; + and.b32 %r1235, %r1234, 1; + bfi.b32 %r492, %r1999, %r1235, 1, 31; + setp.ne.s16 %p328, %rs841, 0; + @%p328 bra $L__BB28_242; + + setp.eq.s32 %p329, %r1791, 0; + mov.u16 %rs1195, 255; + @%p329 bra $L__BB28_241; + + cvt.u64.u32 %rd293, %r1790; + add.s64 %rd294, %rd293, %rd3; + add.s64 %rd295, %rd1, %rd294; + ld.global.u8 %rs1195, [%rd295]; + +$L__BB28_241: + setp.ne.s32 %p331, %r1791, 0; + selp.u32 %r1236, 1, 0, %p331; + add.s32 %r1790, %r1790, %r1236; + add.s32 %r1237, %r1791, -1; + selp.b32 %r1791, 0, %r1237, %p329; + setp.eq.s32 %p332, %r1791, 0; + or.b16 %rs843, %rs1195, 15; + selp.b16 %rs1014, %rs843, %rs1195, %p332; + and.b16 %rs844, %rs1014, 255; + mov.u16 %rs845, 8; + sub.s16 %rs1198, %rs845, %rs1013; + setp.eq.s16 %p333, %rs844, 255; + selp.u16 %rs1013, 1, 0, %p333; + cvt.u32.u16 %r1238, %rs1014; + and.b32 %r1977, %r1238, 255; + +$L__BB28_242: + add.s16 %rs1202, %rs1198, -1; + and.b16 %rs846, %rs1202, 255; + cvt.u32.u16 %r1239, %rs1202; + and.b32 %r1240, %r1239, 255; + shr.u32 %r1241, %r1977, %r1240; + and.b32 %r1242, %r1241, 1; + bfi.b32 %r499, %r492, %r1242, 1, 31; + setp.ne.s16 %p334, %rs846, 0; + @%p334 bra $L__BB28_246; + + setp.eq.s32 %p335, %r1791, 0; + mov.u16 %rs1199, 255; + @%p335 bra $L__BB28_245; + + cvt.u64.u32 %rd296, %r1790; + add.s64 %rd297, %rd296, %rd3; + add.s64 %rd298, %rd1, %rd297; + ld.global.u8 %rs1199, [%rd298]; + +$L__BB28_245: + setp.ne.s32 %p337, %r1791, 0; + selp.u32 %r1243, 1, 0, %p337; + add.s32 %r1790, %r1790, %r1243; + add.s32 %r1244, %r1791, -1; + selp.b32 %r1791, 0, %r1244, %p335; + setp.eq.s32 %p338, %r1791, 0; + or.b16 %rs848, %rs1199, 15; + selp.b16 %rs1014, %rs848, %rs1199, %p338; + and.b16 %rs849, %rs1014, 255; + mov.u16 %rs850, 8; + sub.s16 %rs1202, %rs850, %rs1013; + setp.eq.s16 %p339, %rs849, 255; + selp.u16 %rs1013, 1, 0, %p339; + cvt.u32.u16 %r1245, %rs1014; + and.b32 %r1977, %r1245, 255; + +$L__BB28_246: + add.s16 %rs1206, %rs1202, -1; + and.b16 %rs851, %rs1206, 255; + cvt.u32.u16 %r1246, %rs1206; + and.b32 %r1247, %r1246, 255; + shr.u32 %r1248, %r1977, %r1247; + and.b32 %r1249, %r1248, 1; + bfi.b32 %r506, %r499, %r1249, 1, 31; + setp.ne.s16 %p340, %rs851, 0; + @%p340 bra $L__BB28_250; + + setp.eq.s32 %p341, %r1791, 0; + mov.u16 %rs1203, 255; + @%p341 bra $L__BB28_249; + + cvt.u64.u32 %rd299, %r1790; + add.s64 %rd300, %rd299, %rd3; + add.s64 %rd301, %rd1, %rd300; + ld.global.u8 %rs1203, [%rd301]; + +$L__BB28_249: + setp.ne.s32 %p343, %r1791, 0; + selp.u32 %r1250, 1, 0, %p343; + add.s32 %r1790, %r1790, %r1250; + add.s32 %r1251, %r1791, -1; + selp.b32 %r1791, 0, %r1251, %p341; + setp.eq.s32 %p344, %r1791, 0; + or.b16 %rs853, %rs1203, 15; + selp.b16 %rs1014, %rs853, %rs1203, %p344; + and.b16 %rs854, %rs1014, 255; + mov.u16 %rs855, 8; + sub.s16 %rs1206, %rs855, %rs1013; + setp.eq.s16 %p345, %rs854, 255; + selp.u16 %rs1013, 1, 0, %p345; + cvt.u32.u16 %r1252, %rs1014; + and.b32 %r1977, %r1252, 255; + +$L__BB28_250: + add.s16 %rs1048, %rs1206, -1; + cvt.u32.u16 %r1253, %rs1048; + and.b32 %r1254, %r1253, 255; + shr.u32 %r1255, %r1977, %r1254; + and.b32 %r1256, %r1255, 1; + bfi.b32 %r1999, %r506, %r1256, 1, 31; + add.s32 %r1971, %r1971, -4; + setp.ne.s32 %p346, %r1971, 0; + @%p346 bra $L__BB28_234; + +$L__BB28_251: + setp.eq.s32 %p347, %r481, 0; + @%p347 bra $L__BB28_267; + + and.b16 %rs856, %rs1048, 255; + setp.ne.s16 %p348, %rs856, 0; + @%p348 bra $L__BB28_256; + + setp.eq.s32 %p349, %r1791, 0; + mov.u16 %rs1213, 255; + @%p349 bra $L__BB28_255; + + cvt.u64.u32 %rd302, %r1790; + add.s64 %rd303, %rd302, %rd3; + add.s64 %rd304, %rd1, %rd303; + ld.global.u8 %rs1213, [%rd304]; + +$L__BB28_255: + setp.ne.s32 %p351, %r1791, 0; + selp.u32 %r1257, 1, 0, %p351; + add.s32 %r1790, %r1790, %r1257; + add.s32 %r1258, %r1791, -1; + selp.b32 %r1791, 0, %r1258, %p349; + setp.eq.s32 %p352, %r1791, 0; + or.b16 %rs858, %rs1213, 15; + selp.b16 %rs1014, %rs858, %rs1213, %p352; + and.b16 %rs859, %rs1014, 255; + mov.u16 %rs860, 8; + sub.s16 %rs1048, %rs860, %rs1013; + setp.eq.s16 %p353, %rs859, 255; + selp.u16 %rs1013, 1, 0, %p353; + +$L__BB28_256: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1259, %rs1048; + and.b32 %r1260, %r1259, 255; + cvt.u32.u16 %r1261, %rs1014; + and.b32 %r1994, %r1261, 255; + shr.u32 %r1262, %r1994, %r1260; + and.b32 %r1263, %r1262, 1; + bfi.b32 %r1999, %r1999, %r1263, 1, 31; + setp.eq.s32 %p354, %r481, 1; + @%p354 bra $L__BB28_267; + + and.b16 %rs861, %rs1048, 255; + setp.ne.s16 %p355, %rs861, 0; + @%p355 bra $L__BB28_261; + + setp.eq.s32 %p356, %r1791, 0; + mov.u16 %rs1217, 255; + @%p356 bra $L__BB28_260; + + cvt.u64.u32 %rd305, %r1790; + add.s64 %rd306, %rd305, %rd3; + add.s64 %rd307, %rd1, %rd306; + ld.global.u8 %rs1217, [%rd307]; + +$L__BB28_260: + setp.ne.s32 %p358, %r1791, 0; + selp.u32 %r1264, 1, 0, %p358; + add.s32 %r1790, %r1790, %r1264; + add.s32 %r1265, %r1791, -1; + selp.b32 %r1791, 0, %r1265, %p356; + setp.eq.s32 %p359, %r1791, 0; + or.b16 %rs863, %rs1217, 15; + selp.b16 %rs1014, %rs863, %rs1217, %p359; + and.b16 %rs864, %rs1014, 255; + mov.u16 %rs865, 8; + sub.s16 %rs1048, %rs865, %rs1013; + setp.eq.s16 %p360, %rs864, 255; + selp.u16 %rs1013, 1, 0, %p360; + cvt.u32.u16 %r1266, %rs1014; + and.b32 %r1994, %r1266, 255; + +$L__BB28_261: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1267, %rs1048; + and.b32 %r1268, %r1267, 255; + shr.u32 %r1269, %r1994, %r1268; + and.b32 %r1270, %r1269, 1; + bfi.b32 %r1999, %r1999, %r1270, 1, 31; + setp.eq.s32 %p361, %r481, 2; + @%p361 bra $L__BB28_267; + + and.b16 %rs866, %rs1048, 255; + setp.ne.s16 %p362, %rs866, 0; + @%p362 bra $L__BB28_266; + + setp.eq.s32 %p363, %r1791, 0; + mov.u16 %rs1221, 255; + @%p363 bra $L__BB28_265; + + cvt.u64.u32 %rd308, %r1790; + add.s64 %rd309, %rd308, %rd3; + add.s64 %rd310, %rd1, %rd309; + ld.global.u8 %rs1221, [%rd310]; + +$L__BB28_265: + setp.ne.s32 %p365, %r1791, 0; + selp.u32 %r1271, 1, 0, %p365; + add.s32 %r1790, %r1790, %r1271; + add.s32 %r1272, %r1791, -1; + selp.b32 %r1791, 0, %r1272, %p363; + setp.eq.s32 %p366, %r1791, 0; + or.b16 %rs868, %rs1221, 15; + selp.b16 %rs1014, %rs868, %rs1221, %p366; + and.b16 %rs869, %rs1014, 255; + mov.u16 %rs870, 8; + sub.s16 %rs1048, %rs870, %rs1013; + setp.eq.s16 %p367, %rs869, 255; + selp.u16 %rs1013, 1, 0, %p367; + cvt.u32.u16 %r1273, %rs1014; + and.b32 %r1994, %r1273, 255; + +$L__BB28_266: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1274, %rs1048; + and.b32 %r1275, %r1274, 255; + shr.u32 %r1276, %r1994, %r1275; + and.b32 %r1277, %r1276, 1; + bfi.b32 %r1999, %r1999, %r1277, 1, 31; + +$L__BB28_267: + shl.b32 %r1278, %r1999, 1; + or.b32 %r2003, %r1278, 1; + add.s32 %r1279, %r1881, -1; + setp.eq.s32 %p368, %r1881, 0; + selp.b32 %r2002, 0, %r1279, %p368; + +$L__BB28_268: + mul.lo.s32 %r1280, %r1882, 7; + cvt.u64.u32 %rd311, %r2003; + shl.b64 %rd312, %rd311, %r1280; + or.b64 %rd465, %rd312, %rd465; + setp.ne.s32 %p369, %r1881, 12; + setp.ne.s32 %p370, %r477, 0; + or.pred %p371, %p369, %p370; + add.s32 %r1882, %r1882, 1; + setp.lt.u32 %p372, %r1882, 8; + or.pred %p373, %p372, %p371; + mov.u32 %r1881, %r2002; + @%p373 bra $L__BB28_224; + +$L__BB28_269: + cvt.u32.u64 %r1281, %rd465; + and.b32 %r1878, %r1281, 127; + shr.u64 %rd465, %rd465, 7; + add.s32 %r1882, %r1882, -1; + +$L__BB28_270: + mul.wide.u32 %rd313, %r1947, 2; + add.s64 %rd314, %rd13, %rd313; + st.local.u16 [%rd314], %r2013; + shl.b32 %r1282, %r2013, 2; + shl.b32 %r1283, %r2013, 1; + or.b32 %r1284, %r1282, %r1283; + and.b32 %r1285, %r1284, 256; + ld.local.u16 %r1286, [%rd46]; + and.b32 %r1287, %r1286, 128; + or.b32 %r1288, %r1285, %r1287; + ld.local.u16 %r1289, [%rd47]; + shl.b32 %r1290, %r1289, 2; + and.b32 %r1291, %r1290, 640; + or.b32 %r1292, %r1288, %r1291; + add.s32 %r1293, %r454, 4; + mul.wide.u32 %rd315, %r1293, 2; + add.s64 %rd316, %rd13, %rd315; + ld.local.u16 %r1294, [%rd316]; + shl.b32 %r1295, %r1294, 4; + and.b32 %r1296, %r1295, 512; + or.b32 %r1297, %r1292, %r1296; + and.b32 %r1298, %r2013, 7; + shr.u64 %rd57, %rd474, %r1298; + sub.s32 %r563, %r1961, %r1298; + cvt.u32.u64 %r1299, %rd57; + and.b32 %r1300, %r1299, 127; + or.b32 %r1301, %r1297, %r1300; + mul.wide.u32 %rd317, %r1301, 2; + add.s64 %rd318, %rd41, %rd317; + ld.global.u16 %r2065, [%rd318]; + setp.ne.s32 %p374, %r1297, 0; + add.s32 %r565, %r1945, 2; + setp.ge.u32 %p375, %r565, %r824; + or.pred %p376, %p375, %p374; + @%p376 bra $L__BB28_320; + + add.s32 %r566, %r1878, -2; + setp.eq.s32 %p377, %r566, -1; + selp.b32 %r2065, %r2065, 0, %p377; + setp.gt.s32 %p378, %r1878, 1; + mov.u32 %r1878, %r566; + @%p378 bra $L__BB28_320; + + setp.ne.s32 %p379, %r1882, 0; + @%p379 bra $L__BB28_319; + + mov.u32 %r1882, 0; + +$L__BB28_274: + setp.gt.u32 %p380, %r1882, 7; + @%p380 bra $L__BB28_319; + + cvt.u64.u32 %rd59, %r1881; + mul.wide.u32 %rd319, %r1881, 4; + add.s64 %rd321, %rd133, %rd319; + ld.global.nc.u32 %r572, [%rd321]; + and.b16 %rs871, %rs1048, 255; + setp.ne.s16 %p381, %rs871, 0; + @%p381 bra $L__BB28_279; + + setp.eq.s32 %p382, %r1791, 0; + mov.u16 %rs1240, 255; + @%p382 bra $L__BB28_278; + + cvt.u64.u32 %rd322, %r1790; + add.s64 %rd323, %rd322, %rd3; + add.s64 %rd324, %rd1, %rd323; + ld.global.u8 %rs1240, [%rd324]; + +$L__BB28_278: + setp.ne.s32 %p384, %r1791, 0; + selp.u32 %r1303, 1, 0, %p384; + add.s32 %r1790, %r1790, %r1303; + add.s32 %r1304, %r1791, -1; + selp.b32 %r1791, 0, %r1304, %p382; + setp.eq.s32 %p385, %r1791, 0; + or.b16 %rs873, %rs1240, 15; + selp.b16 %rs1014, %rs873, %rs1240, %p385; + and.b16 %rs874, %rs1014, 255; + mov.u16 %rs875, 8; + sub.s16 %rs1048, %rs875, %rs1013; + setp.eq.s16 %p386, %rs874, 255; + selp.u16 %rs1013, 1, 0, %p386; + +$L__BB28_279: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1305, %rs1048; + and.b32 %r1306, %r1305, 255; + mov.u32 %r1307, 1; + shl.b32 %r1308, %r1307, %r1306; + cvt.u32.u16 %r1309, %rs1014; + and.b32 %r1310, %r1308, %r1309; + and.b32 %r577, %r1310, 255; + setp.eq.s32 %p387, %r577, 0; + @%p387 bra $L__BB28_281; + + add.s32 %r1311, %r1881, 1; + min.u32 %r2054, %r1311, 12; + mov.u32 %r1312, -1; + shl.b32 %r1313, %r1312, %r572; + shl.b32 %r1314, %r1313, 1; + xor.b32 %r2055, %r1314, -2; + bra.uni $L__BB28_318; + +$L__BB28_281: + add.s64 %rd325, %rd59, -3; + setp.gt.u64 %p388, %rd325, 9; + mov.u32 %r2051, 0; + @%p388 bra $L__BB28_317; + + max.u32 %r580, %r572, 1; + add.s32 %r1318, %r580, -1; + and.b32 %r581, %r580, 3; + setp.lt.u32 %p389, %r1318, 3; + mov.u32 %r2051, 0; + @%p389 bra $L__BB28_301; + + sub.s32 %r2023, %r580, %r581; + mov.u32 %r2051, 0; + +$L__BB28_284: + and.b16 %rs877, %rs1048, 255; + setp.ne.s16 %p390, %rs877, 0; + @%p390 bra $L__BB28_288; + + setp.eq.s32 %p391, %r1791, 0; + mov.u16 %rs1247, 255; + @%p391 bra $L__BB28_287; + + cvt.u64.u32 %rd326, %r1790; + add.s64 %rd327, %rd326, %rd3; + add.s64 %rd328, %rd1, %rd327; + ld.global.u8 %rs1247, [%rd328]; + +$L__BB28_287: + setp.ne.s32 %p393, %r1791, 0; + selp.u32 %r1320, 1, 0, %p393; + add.s32 %r1790, %r1790, %r1320; + add.s32 %r1321, %r1791, -1; + selp.b32 %r1791, 0, %r1321, %p391; + setp.eq.s32 %p394, %r1791, 0; + or.b16 %rs879, %rs1247, 15; + selp.b16 %rs1014, %rs879, %rs1247, %p394; + and.b16 %rs880, %rs1014, 255; + mov.u16 %rs881, 8; + sub.s16 %rs1048, %rs881, %rs1013; + setp.eq.s16 %p395, %rs880, 255; + selp.u16 %rs1013, 1, 0, %p395; + +$L__BB28_288: + add.s16 %rs1254, %rs1048, -1; + and.b16 %rs882, %rs1254, 255; + cvt.u32.u16 %r1322, %rs1254; + and.b32 %r1323, %r1322, 255; + cvt.u32.u16 %r1324, %rs1014; + and.b32 %r2029, %r1324, 255; + shr.u32 %r1325, %r2029, %r1323; + and.b32 %r1326, %r1325, 1; + bfi.b32 %r592, %r2051, %r1326, 1, 31; + setp.ne.s16 %p396, %rs882, 0; + @%p396 bra $L__BB28_292; + + setp.eq.s32 %p397, %r1791, 0; + mov.u16 %rs1251, 255; + @%p397 bra $L__BB28_291; + + cvt.u64.u32 %rd329, %r1790; + add.s64 %rd330, %rd329, %rd3; + add.s64 %rd331, %rd1, %rd330; + ld.global.u8 %rs1251, [%rd331]; + +$L__BB28_291: + setp.ne.s32 %p399, %r1791, 0; + selp.u32 %r1327, 1, 0, %p399; + add.s32 %r1790, %r1790, %r1327; + add.s32 %r1328, %r1791, -1; + selp.b32 %r1791, 0, %r1328, %p397; + setp.eq.s32 %p400, %r1791, 0; + or.b16 %rs884, %rs1251, 15; + selp.b16 %rs1014, %rs884, %rs1251, %p400; + and.b16 %rs885, %rs1014, 255; + mov.u16 %rs886, 8; + sub.s16 %rs1254, %rs886, %rs1013; + setp.eq.s16 %p401, %rs885, 255; + selp.u16 %rs1013, 1, 0, %p401; + cvt.u32.u16 %r1329, %rs1014; + and.b32 %r2029, %r1329, 255; + +$L__BB28_292: + add.s16 %rs1258, %rs1254, -1; + and.b16 %rs887, %rs1258, 255; + cvt.u32.u16 %r1330, %rs1258; + and.b32 %r1331, %r1330, 255; + shr.u32 %r1332, %r2029, %r1331; + and.b32 %r1333, %r1332, 1; + bfi.b32 %r599, %r592, %r1333, 1, 31; + setp.ne.s16 %p402, %rs887, 0; + @%p402 bra $L__BB28_296; + + setp.eq.s32 %p403, %r1791, 0; + mov.u16 %rs1255, 255; + @%p403 bra $L__BB28_295; + + cvt.u64.u32 %rd332, %r1790; + add.s64 %rd333, %rd332, %rd3; + add.s64 %rd334, %rd1, %rd333; + ld.global.u8 %rs1255, [%rd334]; + +$L__BB28_295: + setp.ne.s32 %p405, %r1791, 0; + selp.u32 %r1334, 1, 0, %p405; + add.s32 %r1790, %r1790, %r1334; + add.s32 %r1335, %r1791, -1; + selp.b32 %r1791, 0, %r1335, %p403; + setp.eq.s32 %p406, %r1791, 0; + or.b16 %rs889, %rs1255, 15; + selp.b16 %rs1014, %rs889, %rs1255, %p406; + and.b16 %rs890, %rs1014, 255; + mov.u16 %rs891, 8; + sub.s16 %rs1258, %rs891, %rs1013; + setp.eq.s16 %p407, %rs890, 255; + selp.u16 %rs1013, 1, 0, %p407; + cvt.u32.u16 %r1336, %rs1014; + and.b32 %r2029, %r1336, 255; + +$L__BB28_296: + add.s16 %rs1262, %rs1258, -1; + and.b16 %rs892, %rs1262, 255; + cvt.u32.u16 %r1337, %rs1262; + and.b32 %r1338, %r1337, 255; + shr.u32 %r1339, %r2029, %r1338; + and.b32 %r1340, %r1339, 1; + bfi.b32 %r606, %r599, %r1340, 1, 31; + setp.ne.s16 %p408, %rs892, 0; + @%p408 bra $L__BB28_300; + + setp.eq.s32 %p409, %r1791, 0; + mov.u16 %rs1259, 255; + @%p409 bra $L__BB28_299; + + cvt.u64.u32 %rd335, %r1790; + add.s64 %rd336, %rd335, %rd3; + add.s64 %rd337, %rd1, %rd336; + ld.global.u8 %rs1259, [%rd337]; + +$L__BB28_299: + setp.ne.s32 %p411, %r1791, 0; + selp.u32 %r1341, 1, 0, %p411; + add.s32 %r1790, %r1790, %r1341; + add.s32 %r1342, %r1791, -1; + selp.b32 %r1791, 0, %r1342, %p409; + setp.eq.s32 %p412, %r1791, 0; + or.b16 %rs894, %rs1259, 15; + selp.b16 %rs1014, %rs894, %rs1259, %p412; + and.b16 %rs895, %rs1014, 255; + mov.u16 %rs896, 8; + sub.s16 %rs1262, %rs896, %rs1013; + setp.eq.s16 %p413, %rs895, 255; + selp.u16 %rs1013, 1, 0, %p413; + cvt.u32.u16 %r1343, %rs1014; + and.b32 %r2029, %r1343, 255; + +$L__BB28_300: + add.s16 %rs1048, %rs1262, -1; + cvt.u32.u16 %r1344, %rs1048; + and.b32 %r1345, %r1344, 255; + shr.u32 %r1346, %r2029, %r1345; + and.b32 %r1347, %r1346, 1; + bfi.b32 %r2051, %r606, %r1347, 1, 31; + add.s32 %r2023, %r2023, -4; + setp.ne.s32 %p414, %r2023, 0; + @%p414 bra $L__BB28_284; + +$L__BB28_301: + setp.eq.s32 %p415, %r581, 0; + @%p415 bra $L__BB28_317; + + and.b16 %rs897, %rs1048, 255; + setp.ne.s16 %p416, %rs897, 0; + @%p416 bra $L__BB28_306; + + setp.eq.s32 %p417, %r1791, 0; + mov.u16 %rs1269, 255; + @%p417 bra $L__BB28_305; + + cvt.u64.u32 %rd338, %r1790; + add.s64 %rd339, %rd338, %rd3; + add.s64 %rd340, %rd1, %rd339; + ld.global.u8 %rs1269, [%rd340]; + +$L__BB28_305: + setp.ne.s32 %p419, %r1791, 0; + selp.u32 %r1348, 1, 0, %p419; + add.s32 %r1790, %r1790, %r1348; + add.s32 %r1349, %r1791, -1; + selp.b32 %r1791, 0, %r1349, %p417; + setp.eq.s32 %p420, %r1791, 0; + or.b16 %rs899, %rs1269, 15; + selp.b16 %rs1014, %rs899, %rs1269, %p420; + and.b16 %rs900, %rs1014, 255; + mov.u16 %rs901, 8; + sub.s16 %rs1048, %rs901, %rs1013; + setp.eq.s16 %p421, %rs900, 255; + selp.u16 %rs1013, 1, 0, %p421; + +$L__BB28_306: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1350, %rs1048; + and.b32 %r1351, %r1350, 255; + cvt.u32.u16 %r1352, %rs1014; + and.b32 %r2046, %r1352, 255; + shr.u32 %r1353, %r2046, %r1351; + and.b32 %r1354, %r1353, 1; + bfi.b32 %r2051, %r2051, %r1354, 1, 31; + setp.eq.s32 %p422, %r581, 1; + @%p422 bra $L__BB28_317; + + and.b16 %rs902, %rs1048, 255; + setp.ne.s16 %p423, %rs902, 0; + @%p423 bra $L__BB28_311; + + setp.eq.s32 %p424, %r1791, 0; + mov.u16 %rs1273, 255; + @%p424 bra $L__BB28_310; + + cvt.u64.u32 %rd341, %r1790; + add.s64 %rd342, %rd341, %rd3; + add.s64 %rd343, %rd1, %rd342; + ld.global.u8 %rs1273, [%rd343]; + +$L__BB28_310: + setp.ne.s32 %p426, %r1791, 0; + selp.u32 %r1355, 1, 0, %p426; + add.s32 %r1790, %r1790, %r1355; + add.s32 %r1356, %r1791, -1; + selp.b32 %r1791, 0, %r1356, %p424; + setp.eq.s32 %p427, %r1791, 0; + or.b16 %rs904, %rs1273, 15; + selp.b16 %rs1014, %rs904, %rs1273, %p427; + and.b16 %rs905, %rs1014, 255; + mov.u16 %rs906, 8; + sub.s16 %rs1048, %rs906, %rs1013; + setp.eq.s16 %p428, %rs905, 255; + selp.u16 %rs1013, 1, 0, %p428; + cvt.u32.u16 %r1357, %rs1014; + and.b32 %r2046, %r1357, 255; + +$L__BB28_311: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1358, %rs1048; + and.b32 %r1359, %r1358, 255; + shr.u32 %r1360, %r2046, %r1359; + and.b32 %r1361, %r1360, 1; + bfi.b32 %r2051, %r2051, %r1361, 1, 31; + setp.eq.s32 %p429, %r581, 2; + @%p429 bra $L__BB28_317; + + and.b16 %rs907, %rs1048, 255; + setp.ne.s16 %p430, %rs907, 0; + @%p430 bra $L__BB28_316; + + setp.eq.s32 %p431, %r1791, 0; + mov.u16 %rs1277, 255; + @%p431 bra $L__BB28_315; + + cvt.u64.u32 %rd344, %r1790; + add.s64 %rd345, %rd344, %rd3; + add.s64 %rd346, %rd1, %rd345; + ld.global.u8 %rs1277, [%rd346]; + +$L__BB28_315: + setp.ne.s32 %p433, %r1791, 0; + selp.u32 %r1362, 1, 0, %p433; + add.s32 %r1790, %r1790, %r1362; + add.s32 %r1363, %r1791, -1; + selp.b32 %r1791, 0, %r1363, %p431; + setp.eq.s32 %p434, %r1791, 0; + or.b16 %rs909, %rs1277, 15; + selp.b16 %rs1014, %rs909, %rs1277, %p434; + and.b16 %rs910, %rs1014, 255; + mov.u16 %rs911, 8; + sub.s16 %rs1048, %rs911, %rs1013; + setp.eq.s16 %p435, %rs910, 255; + selp.u16 %rs1013, 1, 0, %p435; + cvt.u32.u16 %r1364, %rs1014; + and.b32 %r2046, %r1364, 255; + +$L__BB28_316: + add.s16 %rs1048, %rs1048, -1; + cvt.u32.u16 %r1365, %rs1048; + and.b32 %r1366, %r1365, 255; + shr.u32 %r1367, %r2046, %r1366; + and.b32 %r1368, %r1367, 1; + bfi.b32 %r2051, %r2051, %r1368, 1, 31; + +$L__BB28_317: + shl.b32 %r1369, %r2051, 1; + or.b32 %r2055, %r1369, 1; + add.s32 %r1370, %r1881, -1; + setp.eq.s32 %p436, %r1881, 0; + selp.b32 %r2054, 0, %r1370, %p436; + +$L__BB28_318: + mul.lo.s32 %r1371, %r1882, 7; + cvt.u64.u32 %rd347, %r2055; + shl.b64 %rd348, %rd347, %r1371; + or.b64 %rd465, %rd348, %rd465; + setp.ne.s32 %p437, %r1881, 12; + setp.ne.s32 %p438, %r577, 0; + or.pred %p439, %p437, %p438; + add.s32 %r1882, %r1882, 1; + setp.lt.u32 %p440, %r1882, 8; + or.pred %p441, %p440, %p439; + mov.u32 %r1881, %r2054; + @%p441 bra $L__BB28_274; + +$L__BB28_319: + cvt.u32.u64 %r1372, %rd465; + and.b32 %r1878, %r1372, 127; + shr.u64 %rd465, %rd465, 7; + add.s32 %r1882, %r1882, -1; + +$L__BB28_320: + setp.lt.u32 %p442, %r565, %r824; + selp.b32 %r1373, %r2065, 0, %p442; + add.s32 %r1374, %r1947, 2; + mul.wide.u32 %rd349, %r1374, 2; + add.s64 %rd350, %rd13, %rd349; + st.local.u16 [%rd350], %r1373; + shl.b32 %r1375, %r1373, 2; + shl.b32 %r1376, %r1373, 1; + or.b32 %r1377, %r1375, %r1376; + and.b32 %r1378, %r1377, 256; + ld.local.u16 %r1379, [%rd47]; + and.b32 %r1380, %r1379, 128; + or.b32 %r1946, %r1378, %r1380; + and.b32 %r1381, %r1373, 7; + shr.u64 %rd351, %rd57, %r1381; + sub.s32 %r1382, %r563, %r1381; + cvt.u32.u64 %r1383, %rd351; + shl.b32 %r1384, %r2013, 3; + and.b32 %r1385, %r1384, 64; + shl.b32 %r1386, %r1373, 4; + and.b32 %r1387, %r1386, 128; + or.b32 %r1388, %r1387, %r1385; + and.b32 %r1389, %r1383, 63; + or.b32 %r1390, %r1389, %r1388; + mul.wide.u32 %rd352, %r1390, 2; + add.s64 %rd353, %rd40, %rd352; + ld.global.u16 %r1391, [%rd353]; + and.b32 %r1392, %r1391, 7; + shr.u64 %rd354, %rd351, %r1392; + sub.s32 %r1393, %r1382, %r1392; + cvt.u32.u64 %r1394, %rd354; + shr.u32 %r1395, %r1391, 3; + and.b32 %r1396, %r1395, 15; + mov.u32 %r1397, -1; + shl.b32 %r1398, %r1397, %r1396; + not.b32 %r1399, %r1398; + and.b32 %r1400, %r1394, %r1399; + shr.u64 %rd474, %rd354, %r1396; + sub.s32 %r1961, %r1393, %r1396; + shr.u32 %r1401, %r1391, 7; + and.b32 %r1402, %r1401, 7; + shr.u32 %r1403, %r1391, 10; + and.b32 %r1404, %r1403, 7; + mov.u32 %r1405, 255; + shl.b32 %r1406, %r1405, %r1402; + not.b32 %r1407, %r1406; + and.b32 %r1408, %r1400, %r1407; + add.s32 %r1409, %r1408, %r1404; + add.s32 %r1410, %r1947, 1; + mul.wide.u32 %rd355, %r1410, 2; + add.s64 %rd356, %rd13, %rd355; + st.local.u16 [%rd356], %r1409; + shr.u32 %r1411, %r1391, 13; + shr.u32 %r1412, %r1400, %r1402; + add.s32 %r1413, %r1412, %r1411; + add.s32 %r1414, %r1947, 3; + mul.wide.u32 %rd357, %r1414, 2; + add.s64 %rd358, %rd13, %rd357; + st.local.u16 [%rd358], %r1413; + add.s32 %r1947, %r1947, 4; + add.s32 %r1945, %r1945, 4; + setp.lt.u32 %p443, %r1945, %r824; + @%p443 bra $L__BB28_216; + + mul.wide.u32 %rd359, %r1947, 2; + add.s64 %rd360, %rd13, %rd359; + mov.u16 %rs912, 0; + st.local.v2.u16 [%rd360], {%rs912, %rs912}; + add.s32 %r1936, %r1936, 2; + setp.lt.u32 %p444, %r1936, %r826; + @%p444 bra $L__BB28_215; + +$L__BB28_322: + add.s32 %r1415, %r824, 1; + shr.u32 %r1416, %r1415, 1; + add.s32 %r1417, %r1416, 2; + setp.gt.u32 %p445, %r1417, 130; + @%p445 bra $L__BB28_393; + bra.uni $L__BB28_323; + +$L__BB28_393: + mov.u32 %r1683, 2; + st.global.u32 [%rd4], %r1683; + mov.u32 %r1684, 12; + st.global.u32 [%rd4+4], %r1684; + mov.u32 %r1685, 0; + st.global.u32 [%rd4+8], %r1685; + st.global.u32 [%rd4+12], %r1685; + bra.uni $L__BB28_401; + +$L__BB28_323: + mov.u32 %r2066, 0; + mov.u64 %rd502, 0; + add.s32 %r668, %r831, 2; + mov.u32 %r1423, 31; + sub.s32 %r669, %r1423, %r832; + mov.u32 %r1424, 29; + sub.s32 %r670, %r1424, %r831; + mov.u16 %rs1320, 0; + mov.b32 %f2, %r836; + mov.u32 %r2067, %r2066; + mov.u32 %r2068, %r2066; + mov.u32 %r2069, %r835; + mov.u32 %r2134, %r2066; + mov.u32 %r2133, %r2066; + +$L__BB28_324: + mov.u32 %r674, %r2068; + mul.wide.u32 %rd362, %r2067, 2; + add.s64 %rd363, %rd13, %rd362; + ld.local.u16 %r678, [%rd363]; + ld.local.u16 %r679, [%rd363+2]; + setp.lt.u32 %p446, %r668, %r679; + @%p446 bra $L__BB28_392; + + and.b32 %r1426, %r678, 16; + setp.eq.s32 %p447, %r1426, 0; + mov.u32 %r2087, 0; + mov.u32 %r2079, %r2087; + @%p447 bra $L__BB28_331; + + setp.gt.u32 %p448, %r2133, 31; + @%p448 bra $L__BB28_330; + +$L__BB28_327: + setp.ge.u32 %p449, %r2134, %r18; + mov.u16 %rs1295, 255; + @%p449 bra $L__BB28_329; + + add.s32 %r682, %r2134, 1; + cvt.u64.u32 %rd364, %r2134; + add.s64 %rd365, %rd364, %rd3; + add.s64 %rd366, %rd1, %rd365; + ld.global.u8 %rs1295, [%rd366]; + mov.u32 %r2134, %r682; + +$L__BB28_329: + and.b16 %rs915, %rs1295, 255; + cvt.u64.u16 %rd367, %rs1295; + and.b64 %rd368, %rd367, 255; + shl.b64 %rd369, %rd368, %r2133; + or.b64 %rd502, %rd369, %rd502; + cvt.u32.u16 %r1427, %rs1320; + cvt.s32.s8 %r1428, %r1427; + mov.u32 %r1429, 8; + sub.s32 %r1430, %r1429, %r1428; + add.s32 %r2133, %r1430, %r2133; + setp.eq.s16 %p450, %rs915, 255; + selp.u16 %rs1320, 1, 0, %p450; + setp.lt.u32 %p451, %r2133, 33; + @%p451 bra $L__BB28_327; + +$L__BB28_330: + shr.u32 %r1431, %r678, 12; + and.b32 %r1432, %r1431, 1; + sub.s32 %r1433, %r679, %r1432; + shr.u64 %rd69, %rd502, %r1433; + sub.s32 %r2133, %r2133, %r1433; + cvt.u32.u64 %r1434, %rd502; + shl.b32 %r1435, %r1434, 31; + setp.eq.s32 %p452, %r1433, 0; + mov.u32 %r1436, -1; + shl.b32 %r1437, %r1436, %r1433; + not.b32 %r1438, %r1437; + selp.b32 %r1439, 0, %r1438, %p452; + and.b32 %r1440, %r1439, %r1434; + shr.u32 %r1441, %r678, 8; + and.b32 %r1442, %r1441, 1; + shl.b32 %r1443, %r1442, %r1433; + or.b32 %r1444, %r1443, %r1440; + or.b32 %r1445, %r1444, 1; + add.s32 %r1446, %r1445, 2; + shl.b32 %r1447, %r1446, %r670; + or.b32 %r2079, %r1447, %r1435; + mov.u64 %rd502, %rd69; + +$L__BB28_331: + and.b32 %r1450, %r2079, 2147483647; + shr.u32 %r1451, %r1450, %r669; + neg.s32 %r1452, %r1451; + setp.lt.s32 %p453, %r2079, 0; + selp.b32 %r1453, %r1452, %r1451, %p453; + cvt.rn.f32.s32 %f1, %r1453; + mul.rn.f32 %f3, %f2, %f1; + mul.wide.u32 %rd370, %r2069, 4; + add.s64 %rd371, %rd2, %rd370; + st.global.f32 [%rd371], %f3; + and.b32 %r1454, %r678, 32; + setp.eq.s32 %p454, %r1454, 0; + mov.u32 %r2088, %r2087; + @%p454 bra $L__BB28_337; + + setp.gt.u32 %p455, %r2133, 31; + @%p455 bra $L__BB28_336; + +$L__BB28_333: + setp.ge.u32 %p456, %r2134, %r18; + mov.u16 %rs1299, 255; + @%p456 bra $L__BB28_335; + + add.s32 %r694, %r2134, 1; + cvt.u64.u32 %rd372, %r2134; + add.s64 %rd373, %rd372, %rd3; + add.s64 %rd374, %rd1, %rd373; + ld.global.u8 %rs1299, [%rd374]; + mov.u32 %r2134, %r694; + +$L__BB28_335: + and.b16 %rs917, %rs1299, 255; + cvt.u64.u16 %rd375, %rs1299; + and.b64 %rd376, %rd375, 255; + shl.b64 %rd377, %rd376, %r2133; + or.b64 %rd502, %rd377, %rd502; + cvt.u32.u16 %r1455, %rs1320; + cvt.s32.s8 %r1456, %r1455; + mov.u32 %r1457, 8; + sub.s32 %r1458, %r1457, %r1456; + add.s32 %r2133, %r1458, %r2133; + setp.eq.s16 %p457, %rs917, 255; + selp.u16 %rs1320, 1, 0, %p457; + setp.lt.u32 %p458, %r2133, 33; + @%p458 bra $L__BB28_333; + +$L__BB28_336: + shr.u32 %r1459, %r678, 13; + and.b32 %r1460, %r1459, 1; + sub.s32 %r1461, %r679, %r1460; + shr.u64 %rd74, %rd502, %r1461; + sub.s32 %r2133, %r2133, %r1461; + cvt.u32.u64 %r1462, %rd502; + shl.b32 %r1463, %r1462, 31; + setp.eq.s32 %p459, %r1461, 0; + mov.u32 %r1464, -1; + shl.b32 %r1465, %r1464, %r1461; + not.b32 %r1466, %r1465; + selp.b32 %r1467, 0, %r1466, %p459; + and.b32 %r1468, %r1467, %r1462; + shr.u32 %r1469, %r678, 9; + and.b32 %r1470, %r1469, 1; + shl.b32 %r1471, %r1470, %r1461; + or.b32 %r1472, %r1471, %r1468; + or.b32 %r2088, %r1472, 1; + add.s32 %r1473, %r2088, 2; + shl.b32 %r1474, %r1473, %r670; + or.b32 %r2087, %r1474, %r1463; + mov.u64 %rd502, %rd74; + +$L__BB28_337: + setp.lt.u32 %p460, %r826, 2; + @%p460 bra $L__BB28_339; + + add.s32 %r1475, %r2069, %r834; + and.b32 %r1476, %r2087, 2147483647; + shr.u32 %r1477, %r1476, %r669; + neg.s32 %r1478, %r1477; + setp.lt.s32 %p461, %r2087, 0; + selp.b32 %r1479, %r1478, %r1477, %p461; + cvt.rn.f32.s32 %f4, %r1479; + mul.rn.f32 %f6, %f2, %f4; + mul.wide.u32 %rd378, %r1475, 4; + add.s64 %rd379, %rd2, %rd378; + st.global.f32 [%rd379], %f6; + +$L__BB28_339: + or.b32 %r1480, %r2088, %r2066; + add.u64 %rd76, %SPL, 6192; + mul.wide.u32 %rd381, %r674, 4; + add.s64 %rd382, %rd76, %rd381; + st.local.u32 [%rd382], %r1480; + add.s32 %r706, %r2069, 1; + add.s32 %r1481, %r2067, 1; + setp.lt.u32 %p462, %r1481, %r824; + @%p462 bra $L__BB28_341; + bra.uni $L__BB28_340; + +$L__BB28_341: + and.b32 %r1484, %r678, 64; + setp.eq.s32 %p463, %r1484, 0; + mov.u32 %r2104, 0; + mov.u32 %r2096, %r2104; + @%p463 bra $L__BB28_347; + + setp.gt.u32 %p464, %r2133, 31; + @%p464 bra $L__BB28_346; + +$L__BB28_343: + setp.ge.u32 %p465, %r2134, %r18; + mov.u16 %rs1303, 255; + @%p465 bra $L__BB28_345; + + add.s32 %r709, %r2134, 1; + cvt.u64.u32 %rd383, %r2134; + add.s64 %rd384, %rd383, %rd3; + add.s64 %rd385, %rd1, %rd384; + ld.global.u8 %rs1303, [%rd385]; + mov.u32 %r2134, %r709; + +$L__BB28_345: + and.b16 %rs919, %rs1303, 255; + cvt.u64.u16 %rd386, %rs1303; + and.b64 %rd387, %rd386, 255; + shl.b64 %rd388, %rd387, %r2133; + or.b64 %rd502, %rd388, %rd502; + cvt.u32.u16 %r1485, %rs1320; + cvt.s32.s8 %r1486, %r1485; + mov.u32 %r1487, 8; + sub.s32 %r1488, %r1487, %r1486; + add.s32 %r2133, %r1488, %r2133; + setp.eq.s16 %p466, %rs919, 255; + selp.u16 %rs1320, 1, 0, %p466; + setp.lt.u32 %p467, %r2133, 33; + @%p467 bra $L__BB28_343; + +$L__BB28_346: + shr.u32 %r1489, %r678, 14; + and.b32 %r1490, %r1489, 1; + sub.s32 %r1491, %r679, %r1490; + shr.u64 %rd80, %rd502, %r1491; + sub.s32 %r2133, %r2133, %r1491; + cvt.u32.u64 %r1492, %rd502; + shl.b32 %r1493, %r1492, 31; + setp.eq.s32 %p468, %r1491, 0; + mov.u32 %r1494, -1; + shl.b32 %r1495, %r1494, %r1491; + not.b32 %r1496, %r1495; + selp.b32 %r1497, 0, %r1496, %p468; + and.b32 %r1498, %r1497, %r1492; + shr.u32 %r1499, %r678, 10; + and.b32 %r1500, %r1499, 1; + shl.b32 %r1501, %r1500, %r1491; + or.b32 %r1502, %r1501, %r1498; + or.b32 %r1503, %r1502, 1; + add.s32 %r1504, %r1503, 2; + shl.b32 %r1505, %r1504, %r670; + or.b32 %r2096, %r1505, %r1493; + mov.u64 %rd502, %rd80; + +$L__BB28_347: + and.b32 %r1508, %r2096, 2147483647; + shr.u32 %r1509, %r1508, %r669; + neg.s32 %r1510, %r1509; + setp.lt.s32 %p469, %r2096, 0; + selp.b32 %r1511, %r1510, %r1509, %p469; + cvt.rn.f32.s32 %f7, %r1511; + mul.rn.f32 %f9, %f2, %f7; + mul.wide.u32 %rd389, %r706, 4; + add.s64 %rd390, %rd2, %rd389; + st.global.f32 [%rd390], %f9; + and.b32 %r1512, %r678, 128; + setp.eq.s32 %p470, %r1512, 0; + mov.u32 %r2066, %r2104; + @%p470 bra $L__BB28_353; + + setp.gt.u32 %p471, %r2133, 31; + @%p471 bra $L__BB28_352; + +$L__BB28_349: + setp.ge.u32 %p472, %r2134, %r18; + mov.u16 %rs1307, 255; + @%p472 bra $L__BB28_351; + + add.s32 %r721, %r2134, 1; + cvt.u64.u32 %rd391, %r2134; + add.s64 %rd392, %rd391, %rd3; + add.s64 %rd393, %rd1, %rd392; + ld.global.u8 %rs1307, [%rd393]; + mov.u32 %r2134, %r721; + +$L__BB28_351: + and.b16 %rs921, %rs1307, 255; + cvt.u64.u16 %rd394, %rs1307; + and.b64 %rd395, %rd394, 255; + shl.b64 %rd396, %rd395, %r2133; + or.b64 %rd502, %rd396, %rd502; + cvt.u32.u16 %r1513, %rs1320; + cvt.s32.s8 %r1514, %r1513; + mov.u32 %r1515, 8; + sub.s32 %r1516, %r1515, %r1514; + add.s32 %r2133, %r1516, %r2133; + setp.eq.s16 %p473, %rs921, 255; + selp.u16 %rs1320, 1, 0, %p473; + setp.lt.u32 %p474, %r2133, 33; + @%p474 bra $L__BB28_349; + +$L__BB28_352: + shr.u32 %r1517, %r678, 15; + sub.s32 %r1518, %r679, %r1517; + shr.u64 %rd85, %rd502, %r1518; + sub.s32 %r2133, %r2133, %r1518; + cvt.u32.u64 %r1519, %rd502; + shl.b32 %r1520, %r1519, 31; + setp.eq.s32 %p475, %r1518, 0; + mov.u32 %r1521, -1; + shl.b32 %r1522, %r1521, %r1518; + not.b32 %r1523, %r1522; + selp.b32 %r1524, 0, %r1523, %p475; + and.b32 %r1525, %r1524, %r1519; + shr.u32 %r1526, %r678, 11; + and.b32 %r1527, %r1526, 1; + shl.b32 %r1528, %r1527, %r1518; + or.b32 %r1529, %r1528, %r1525; + or.b32 %r2066, %r1529, 1; + add.s32 %r1530, %r2066, 2; + shl.b32 %r1531, %r1530, %r670; + or.b32 %r2104, %r1531, %r1520; + mov.u64 %rd502, %rd85; + +$L__BB28_353: + @%p460 bra $L__BB28_355; + + add.s32 %r1532, %r706, %r834; + and.b32 %r1533, %r2104, 2147483647; + shr.u32 %r1534, %r1533, %r669; + neg.s32 %r1535, %r1534; + setp.lt.s32 %p477, %r2104, 0; + selp.b32 %r1536, %r1535, %r1534, %p477; + cvt.rn.f32.s32 %f10, %r1536; + mul.rn.f32 %f12, %f2, %f10; + mul.wide.u32 %rd397, %r1532, 4; + add.s64 %rd398, %rd2, %rd397; + st.global.f32 [%rd398], %f12; + +$L__BB28_355: + add.s32 %r2069, %r2069, 2; + add.s32 %r2068, %r674, 1; + add.s32 %r2067, %r2067, 2; + setp.lt.u32 %p478, %r2067, %r824; + @%p478 bra $L__BB28_324; + bra.uni $L__BB28_356; + +$L__BB28_392: + mov.u32 %r1676, 1; + st.global.u32 [%rd4], %r1676; + mov.u32 %r1677, 13; + st.global.u32 [%rd4+4], %r1677; + mov.u32 %r1678, 0; + st.global.u32 [%rd4+8], %r1678; + st.global.u32 [%rd4+12], %r1678; + bra.uni $L__BB28_401; + +$L__BB28_340: + mov.u32 %r2066, 0; + +$L__BB28_356: + add.s32 %r1537, %r674, 1; + mul.wide.u32 %rd401, %r1537, 4; + add.s64 %rd402, %rd76, %rd401; + st.local.u32 [%rd402], %r2066; + @%p298 bra $L__BB28_401; + + mov.u32 %r2109, 2; + +$L__BB28_358: + shr.u32 %r1543, %r2109, 1; + mul.lo.s32 %r2113, %r1543, %r849; + mad.lo.s32 %r2115, %r2109, %r834, %r835; + add.s32 %r745, %r2109, 1; + ld.local.u32 %r2112, [%rd76]; + mov.u32 %r2114, 0; + mov.u32 %r2116, %r2114; + mov.u32 %r2117, %r2114; + +$L__BB28_359: + mul.wide.u32 %rd403, %r2113, 2; + add.s64 %rd404, %rd13, %rd403; + ld.local.v2.u16 {%rs922, %rs923}, [%rd404]; + cvt.u32.u16 %r755, %rs922; + cvt.u32.u16 %r1544, %rs923; + and.b32 %r1545, %r755, 240; + add.s32 %r1546, %r1545, 240; + and.b32 %r1547, %r1546, %r1545; + add.s32 %r756, %r2114, 1; + mul.wide.u32 %rd405, %r756, 4; + add.s64 %rd90, %rd76, %rd405; + ld.local.u32 %r757, [%rd90]; + or.b32 %r1548, %r2112, %r757; + or.b32 %r1549, %r1548, 2; + clz.b32 %r1550, %r1549; + xor.b32 %r1551, %r1550, 31; + setp.eq.s32 %p480, %r1547, 0; + selp.b32 %r1552, 1, %r1551, %p480; + add.s32 %r758, %r1552, %r1544; + setp.gt.u32 %p481, %r758, %r668; + @%p481 bra $L__BB28_391; + + and.b32 %r1554, %r755, 16; + setp.eq.s32 %p482, %r1554, 0; + mov.u32 %r2135, 0; + mov.u32 %r2127, %r2135; + @%p482 bra $L__BB28_366; + + setp.gt.u32 %p483, %r2133, 31; + @%p483 bra $L__BB28_365; + +$L__BB28_362: + setp.ge.u32 %p484, %r2134, %r18; + mov.u16 %rs1314, 255; + @%p484 bra $L__BB28_364; + + add.s32 %r761, %r2134, 1; + cvt.u64.u32 %rd406, %r2134; + add.s64 %rd407, %rd406, %rd3; + add.s64 %rd408, %rd1, %rd407; + ld.global.u8 %rs1314, [%rd408]; + mov.u32 %r2134, %r761; + +$L__BB28_364: + and.b16 %rs927, %rs1314, 255; + cvt.u64.u16 %rd409, %rs1314; + and.b64 %rd410, %rd409, 255; + shl.b64 %rd411, %rd410, %r2133; + or.b64 %rd502, %rd411, %rd502; + cvt.u32.u16 %r1555, %rs1320; + cvt.s32.s8 %r1556, %r1555; + mov.u32 %r1557, 8; + sub.s32 %r1558, %r1557, %r1556; + add.s32 %r2133, %r1558, %r2133; + setp.eq.s16 %p485, %rs927, 255; + selp.u16 %rs1320, 1, 0, %p485; + setp.lt.u32 %p486, %r2133, 33; + @%p486 bra $L__BB28_362; + +$L__BB28_365: + shr.u32 %r1559, %r755, 12; + and.b32 %r1560, %r1559, 1; + sub.s32 %r1561, %r758, %r1560; + shr.u64 %rd94, %rd502, %r1561; + sub.s32 %r2133, %r2133, %r1561; + cvt.u32.u64 %r1562, %rd502; + shl.b32 %r1563, %r1562, 31; + setp.eq.s32 %p487, %r1561, 0; + mov.u32 %r1564, -1; + shl.b32 %r1565, %r1564, %r1561; + not.b32 %r1566, %r1565; + selp.b32 %r1567, 0, %r1566, %p487; + and.b32 %r1568, %r1567, %r1562; + shr.u32 %r1569, %r755, 8; + and.b32 %r1570, %r1569, 1; + shl.b32 %r1571, %r1570, %r1561; + or.b32 %r1572, %r1571, %r1568; + or.b32 %r1573, %r1572, 1; + add.s32 %r1574, %r1573, 2; + shl.b32 %r1575, %r1574, %r670; + or.b32 %r2127, %r1575, %r1563; + mov.u64 %rd502, %rd94; + +$L__BB28_366: + and.b32 %r1578, %r2127, 2147483647; + shr.u32 %r1579, %r1578, %r669; + neg.s32 %r1580, %r1579; + setp.lt.s32 %p488, %r2127, 0; + selp.b32 %r1581, %r1580, %r1579, %p488; + cvt.rn.f32.s32 %f13, %r1581; + mul.rn.f32 %f15, %f2, %f13; + mul.wide.u32 %rd412, %r2115, 4; + add.s64 %rd413, %rd2, %rd412; + st.global.f32 [%rd413], %f15; + and.b32 %r1582, %r755, 32; + setp.eq.s32 %p489, %r1582, 0; + mov.u32 %r2136, %r2135; + @%p489 bra $L__BB28_372; + + setp.gt.u32 %p490, %r2133, 31; + @%p490 bra $L__BB28_371; + +$L__BB28_368: + setp.ge.u32 %p491, %r2134, %r18; + mov.u16 %rs1318, 255; + @%p491 bra $L__BB28_370; + + add.s32 %r773, %r2134, 1; + cvt.u64.u32 %rd414, %r2134; + add.s64 %rd415, %rd414, %rd3; + add.s64 %rd416, %rd1, %rd415; + ld.global.u8 %rs1318, [%rd416]; + mov.u32 %r2134, %r773; + +$L__BB28_370: + and.b16 %rs929, %rs1318, 255; + cvt.u64.u16 %rd417, %rs1318; + and.b64 %rd418, %rd417, 255; + shl.b64 %rd419, %rd418, %r2133; + or.b64 %rd502, %rd419, %rd502; + cvt.u32.u16 %r1583, %rs1320; + cvt.s32.s8 %r1584, %r1583; + mov.u32 %r1585, 8; + sub.s32 %r1586, %r1585, %r1584; + add.s32 %r2133, %r1586, %r2133; + setp.eq.s16 %p492, %rs929, 255; + selp.u16 %rs1320, 1, 0, %p492; + setp.lt.u32 %p493, %r2133, 33; + @%p493 bra $L__BB28_368; + +$L__BB28_371: + shr.u32 %r1587, %r755, 13; + and.b32 %r1588, %r1587, 1; + sub.s32 %r1589, %r758, %r1588; + shr.u64 %rd99, %rd502, %r1589; + sub.s32 %r2133, %r2133, %r1589; + cvt.u32.u64 %r1590, %rd502; + shl.b32 %r1591, %r1590, 31; + setp.eq.s32 %p494, %r1589, 0; + mov.u32 %r1592, -1; + shl.b32 %r1593, %r1592, %r1589; + not.b32 %r1594, %r1593; + selp.b32 %r1595, 0, %r1594, %p494; + and.b32 %r1596, %r1595, %r1590; + shr.u32 %r1597, %r755, 9; + and.b32 %r1598, %r1597, 1; + shl.b32 %r1599, %r1598, %r1589; + or.b32 %r1600, %r1599, %r1596; + or.b32 %r2136, %r1600, 1; + add.s32 %r1601, %r2136, 2; + shl.b32 %r1602, %r1601, %r670; + or.b32 %r2135, %r1602, %r1591; + mov.u64 %rd502, %rd99; + +$L__BB28_372: + setp.ge.u32 %p495, %r745, %r826; + @%p495 bra $L__BB28_374; + + add.s32 %r1603, %r2115, %r834; + and.b32 %r1604, %r2135, 2147483647; + shr.u32 %r1605, %r1604, %r669; + neg.s32 %r1606, %r1605; + setp.lt.s32 %p496, %r2135, 0; + selp.b32 %r1607, %r1606, %r1605, %p496; + cvt.rn.f32.s32 %f16, %r1607; + mul.rn.f32 %f18, %f2, %f16; + mul.wide.u32 %rd420, %r1603, 4; + add.s64 %rd421, %rd2, %rd420; + st.global.f32 [%rd421], %f18; + +$L__BB28_374: + or.b32 %r1609, %r2136, %r2116; + mul.wide.u32 %rd422, %r2114, 4; + add.s64 %rd423, %rd76, %rd422; + st.local.u32 [%rd423], %r1609; + add.s32 %r785, %r2115, 1; + add.s32 %r1610, %r2117, 1; + setp.ge.u32 %p497, %r1610, %r824; + mov.u32 %r2116, 0; + @%p497 bra $L__BB28_390; + + and.b32 %r1612, %r755, 64; + setp.eq.s32 %p498, %r1612, 0; + mov.u32 %r2152, 0; + mov.u32 %r2144, %r2152; + @%p498 bra $L__BB28_381; + + setp.gt.u32 %p499, %r2133, 31; + @%p499 bra $L__BB28_380; + +$L__BB28_377: + setp.ge.u32 %p500, %r2134, %r18; + mov.u16 %rs1322, 255; + @%p500 bra $L__BB28_379; + + add.s32 %r788, %r2134, 1; + cvt.u64.u32 %rd424, %r2134; + add.s64 %rd425, %rd424, %rd3; + add.s64 %rd426, %rd1, %rd425; + ld.global.u8 %rs1322, [%rd426]; + mov.u32 %r2134, %r788; + +$L__BB28_379: + and.b16 %rs931, %rs1322, 255; + cvt.u64.u16 %rd427, %rs1322; + and.b64 %rd428, %rd427, 255; + shl.b64 %rd429, %rd428, %r2133; + or.b64 %rd502, %rd429, %rd502; + cvt.u32.u16 %r1613, %rs1320; + cvt.s32.s8 %r1614, %r1613; + mov.u32 %r1615, 8; + sub.s32 %r1616, %r1615, %r1614; + add.s32 %r2133, %r1616, %r2133; + setp.eq.s16 %p501, %rs931, 255; + selp.u16 %rs1320, 1, 0, %p501; + setp.lt.u32 %p502, %r2133, 33; + @%p502 bra $L__BB28_377; + +$L__BB28_380: + shr.u32 %r1617, %r755, 14; + and.b32 %r1618, %r1617, 1; + sub.s32 %r1619, %r758, %r1618; + shr.u64 %rd104, %rd502, %r1619; + sub.s32 %r2133, %r2133, %r1619; + cvt.u32.u64 %r1620, %rd502; + shl.b32 %r1621, %r1620, 31; + setp.eq.s32 %p503, %r1619, 0; + mov.u32 %r1622, -1; + shl.b32 %r1623, %r1622, %r1619; + not.b32 %r1624, %r1623; + selp.b32 %r1625, 0, %r1624, %p503; + and.b32 %r1626, %r1625, %r1620; + shr.u32 %r1627, %r755, 10; + and.b32 %r1628, %r1627, 1; + shl.b32 %r1629, %r1628, %r1619; + or.b32 %r1630, %r1629, %r1626; + or.b32 %r1631, %r1630, 1; + add.s32 %r1632, %r1631, 2; + shl.b32 %r1633, %r1632, %r670; + or.b32 %r2144, %r1633, %r1621; + mov.u64 %rd502, %rd104; + +$L__BB28_381: + and.b32 %r1636, %r2144, 2147483647; + shr.u32 %r1637, %r1636, %r669; + neg.s32 %r1638, %r1637; + setp.lt.s32 %p504, %r2144, 0; + selp.b32 %r1639, %r1638, %r1637, %p504; + cvt.rn.f32.s32 %f19, %r1639; + mul.rn.f32 %f21, %f2, %f19; + mul.wide.u32 %rd430, %r785, 4; + add.s64 %rd431, %rd2, %rd430; + st.global.f32 [%rd431], %f21; + and.b32 %r1640, %r755, 128; + setp.eq.s32 %p505, %r1640, 0; + mov.u32 %r2116, %r2152; + @%p505 bra $L__BB28_387; + + setp.gt.u32 %p506, %r2133, 31; + @%p506 bra $L__BB28_386; + +$L__BB28_383: + setp.ge.u32 %p507, %r2134, %r18; + mov.u16 %rs1326, 255; + @%p507 bra $L__BB28_385; + + add.s32 %r800, %r2134, 1; + cvt.u64.u32 %rd432, %r2134; + add.s64 %rd433, %rd432, %rd3; + add.s64 %rd434, %rd1, %rd433; + ld.global.u8 %rs1326, [%rd434]; + mov.u32 %r2134, %r800; + +$L__BB28_385: + and.b16 %rs933, %rs1326, 255; + cvt.u64.u16 %rd435, %rs1326; + and.b64 %rd436, %rd435, 255; + shl.b64 %rd437, %rd436, %r2133; + or.b64 %rd502, %rd437, %rd502; + cvt.u32.u16 %r1641, %rs1320; + cvt.s32.s8 %r1642, %r1641; + mov.u32 %r1643, 8; + sub.s32 %r1644, %r1643, %r1642; + add.s32 %r2133, %r1644, %r2133; + setp.eq.s16 %p508, %rs933, 255; + selp.u16 %rs1320, 1, 0, %p508; + setp.lt.u32 %p509, %r2133, 33; + @%p509 bra $L__BB28_383; + +$L__BB28_386: + shr.u32 %r1645, %r755, 15; + sub.s32 %r1646, %r758, %r1645; + shr.u64 %rd109, %rd502, %r1646; + sub.s32 %r2133, %r2133, %r1646; + cvt.u32.u64 %r1647, %rd502; + shl.b32 %r1648, %r1647, 31; + setp.eq.s32 %p510, %r1646, 0; + mov.u32 %r1649, -1; + shl.b32 %r1650, %r1649, %r1646; + not.b32 %r1651, %r1650; + selp.b32 %r1652, 0, %r1651, %p510; + and.b32 %r1653, %r1652, %r1647; + shr.u32 %r1654, %r755, 11; + and.b32 %r1655, %r1654, 1; + shl.b32 %r1656, %r1655, %r1646; + or.b32 %r1657, %r1656, %r1653; + or.b32 %r2116, %r1657, 1; + add.s32 %r1658, %r2116, 2; + shl.b32 %r1659, %r1658, %r670; + or.b32 %r2152, %r1659, %r1648; + mov.u64 %rd502, %rd109; + +$L__BB28_387: + @%p495 bra $L__BB28_389; + + add.s32 %r1660, %r785, %r834; + and.b32 %r1661, %r2152, 2147483647; + shr.u32 %r1662, %r1661, %r669; + neg.s32 %r1663, %r1662; + setp.lt.s32 %p512, %r2152, 0; + selp.b32 %r1664, %r1663, %r1662, %p512; + cvt.rn.f32.s32 %f22, %r1664; + mul.rn.f32 %f24, %f2, %f22; + mul.wide.u32 %rd438, %r1660, 4; + add.s64 %rd439, %rd2, %rd438; + st.global.f32 [%rd439], %f24; + +$L__BB28_389: + add.s32 %r2115, %r2115, 2; + add.s32 %r2113, %r2113, 2; + add.s32 %r2117, %r2117, 2; + setp.lt.u32 %p513, %r2117, %r824; + mov.u32 %r2112, %r757; + mov.u32 %r2114, %r756; + @%p513 bra $L__BB28_359; + +$L__BB28_390: + st.local.u32 [%rd90], %r2116; + add.s32 %r2109, %r2109, 2; + setp.lt.u32 %p514, %r2109, %r826; + @%p514 bra $L__BB28_358; + bra.uni $L__BB28_401; + +$L__BB28_391: + mov.u32 %r1669, 1; + st.global.u32 [%rd4], %r1669; + mov.u32 %r1670, 14; + st.global.u32 [%rd4+4], %r1670; + mov.u32 %r1671, 0; + st.global.u32 [%rd4+8], %r1671; + st.global.u32 [%rd4+12], %r1671; + +$L__BB28_401: + ret; + +} + // .globl j2k_dequantize_htj2k_codeblocks +.visible .entry j2k_dequantize_htj2k_codeblocks( + .param .u64 j2k_dequantize_htj2k_codeblocks_param_0, + .param .u64 j2k_dequantize_htj2k_codeblocks_param_1 +) +{ + .reg .pred %p<4>; + .reg .f32 %f<4>; + .reg .b32 %r<24>; + .reg .b64 %rd<10>; + + + ld.param.u64 %rd3, [j2k_dequantize_htj2k_codeblocks_param_0]; + ld.param.u64 %rd4, [j2k_dequantize_htj2k_codeblocks_param_1]; + mov.u32 %r10, %ctaid.x; + cvta.to.global.u64 %rd5, %rd4; + mul.wide.u32 %rd6, %r10, 52; + add.s64 %rd7, %rd5, %rd6; + add.s64 %rd1, %rd7, 4; + ld.global.u32 %r11, [%rd7+8]; + ld.global.u32 %r1, [%rd7+4]; + mul.lo.s32 %r2, %r11, %r1; + mov.u32 %r23, %tid.x; + setp.ge.u32 %p1, %r23, %r2; + @%p1 bra $L__BB29_3; + + ld.global.u32 %r4, [%rd1+36]; + ld.global.f32 %f1, [%rd1+40]; + ld.global.u32 %r12, [%rd1+32]; + sub.s32 %r5, %r12, %r1; + ld.global.u32 %r13, [%rd1+24]; + mov.u32 %r14, 31; + sub.s32 %r6, %r14, %r13; + mov.u32 %r7, %ntid.x; + cvta.to.global.u64 %rd2, %rd3; + +$L__BB29_2: + div.u32 %r15, %r23, %r1; + add.s32 %r16, %r23, %r4; + mad.lo.s32 %r17, %r5, %r15, %r16; + mul.wide.u32 %rd8, %r17, 4; + add.s64 %rd9, %rd2, %rd8; + ld.global.u32 %r18, [%rd9]; + and.b32 %r19, %r18, 2147483647; + shr.u32 %r20, %r19, %r6; + setp.lt.s32 %p2, %r18, 0; + neg.s32 %r21, %r20; + selp.b32 %r22, %r21, %r20, %p2; + cvt.rn.f32.s32 %f2, %r22; + mul.rn.f32 %f3, %f1, %f2; + st.global.f32 [%rd9], %f3; + add.s32 %r23, %r23, %r7; + setp.lt.u32 %p3, %r23, %r2; + @%p3 bra $L__BB29_2; + +$L__BB29_3: + ret; + +} + // .globl j2k_dequantize_htj2k_codeblocks_multi +.visible .entry j2k_dequantize_htj2k_codeblocks_multi( + .param .u64 j2k_dequantize_htj2k_codeblocks_multi_param_0 +) +{ + .reg .pred %p<4>; + .reg .f32 %f<4>; + .reg .b32 %r<28>; + .reg .b64 %rd<9>; + + + ld.param.u64 %rd3, [j2k_dequantize_htj2k_codeblocks_multi_param_0]; + cvta.to.global.u64 %rd4, %rd3; + mov.u32 %r10, %ctaid.x; + mul.wide.u32 %rd5, %r10, 40; + add.s64 %rd1, %rd4, %rd5; + ld.global.v2.u32 {%r11, %r12}, [%rd1+8]; + mul.lo.s32 %r2, %r12, %r11; + mov.u32 %r27, %tid.x; + setp.ge.u32 %p1, %r27, %r2; + @%p1 bra $L__BB30_3; + + ld.global.u64 %rd6, [%rd1]; + ld.global.u32 %r4, [%rd1+20]; + ld.global.f32 %f1, [%rd1+32]; + .pragma "used_bytes_mask 15"; + ld.global.v2.u32 {%r14, %r15}, [%rd1+16]; + sub.s32 %r5, %r14, %r11; + ld.global.u32 %r17, [%rd1+24]; + mov.u32 %r18, 31; + sub.s32 %r6, %r18, %r17; + mov.u32 %r7, %ntid.x; + cvta.to.global.u64 %rd2, %rd6; + +$L__BB30_2: + div.u32 %r19, %r27, %r11; + add.s32 %r20, %r27, %r4; + mad.lo.s32 %r21, %r5, %r19, %r20; + mul.wide.u32 %rd7, %r21, 4; + add.s64 %rd8, %rd2, %rd7; + ld.global.u32 %r22, [%rd8]; + and.b32 %r23, %r22, 2147483647; + shr.u32 %r24, %r23, %r6; + setp.lt.s32 %p2, %r22, 0; + neg.s32 %r25, %r24; + selp.b32 %r26, %r25, %r24, %p2; + cvt.rn.f32.s32 %f2, %r26; + mul.rn.f32 %f3, %f1, %f2; + st.global.f32 [%rd8], %f3; + add.s32 %r27, %r27, %r7; + setp.lt.u32 %p3, %r27, %r2; + @%p3 bra $L__BB30_2; + +$L__BB30_3: + ret; + +} + // .globl j2k_dequantize_htj2k_cleanup_jobs_multi +.visible .entry j2k_dequantize_htj2k_cleanup_jobs_multi( + .param .u64 j2k_dequantize_htj2k_cleanup_jobs_multi_param_0 +) +{ + .reg .pred %p<4>; + .reg .f32 %f<4>; + .reg .b32 %r<27>; + .reg .b64 %rd<9>; + + + ld.param.u64 %rd3, [j2k_dequantize_htj2k_cleanup_jobs_multi_param_0]; + cvta.to.global.u64 %rd4, %rd3; + mov.u32 %r10, %ctaid.x; + mul.wide.u32 %rd5, %r10, 64; + add.s64 %rd1, %rd4, %rd5; + ld.global.u32 %r11, [%rd1+16]; + ld.global.u32 %r1, [%rd1+12]; + mul.lo.s32 %r2, %r11, %r1; + mov.u32 %r26, %tid.x; + setp.ge.u32 %p1, %r26, %r2; + @%p1 bra $L__BB31_3; + + ld.global.u64 %rd6, [%rd1]; + ld.global.v2.u32 {%r12, %r13}, [%rd1+48]; + ld.global.u32 %r15, [%rd1+44]; + sub.s32 %r4, %r15, %r1; + ld.global.u32 %r16, [%rd1+36]; + mov.u32 %r17, 31; + sub.s32 %r5, %r17, %r16; + mov.u32 %r6, %ntid.x; + mov.b32 %f1, %r13; + cvta.to.global.u64 %rd2, %rd6; + +$L__BB31_2: + div.u32 %r18, %r26, %r1; + add.s32 %r19, %r26, %r12; + mad.lo.s32 %r20, %r4, %r18, %r19; + mul.wide.u32 %rd7, %r20, 4; + add.s64 %rd8, %rd2, %rd7; + ld.global.u32 %r21, [%rd8]; + and.b32 %r22, %r21, 2147483647; + shr.u32 %r23, %r22, %r5; + setp.lt.s32 %p2, %r21, 0; + neg.s32 %r24, %r23; + selp.b32 %r25, %r24, %r23, %p2; + cvt.rn.f32.s32 %f2, %r25; + mul.rn.f32 %f3, %f1, %f2; + st.global.f32 [%rd8], %f3; + add.s32 %r26, %r26, %r6; + setp.lt.u32 %p3, %r26, %r2; + @%p3 bra $L__BB31_2; + +$L__BB31_3: + ret; + +} + + \ No newline at end of file diff --git a/crates/j2k-cuda-runtime/src/htj2k_encode.rs b/crates/j2k-cuda-runtime/src/htj2k_encode.rs new file mode 100644 index 00000000..96ac0296 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_encode.rs @@ -0,0 +1,1520 @@ +use crate::{ + bytes::{ + htj2k_encode_compact_jobs_as_bytes, htj2k_encode_jobs_as_bytes, + htj2k_encode_multi_input_jobs_as_bytes, htj2k_encode_params_as_bytes, + htj2k_encode_status_as_bytes, htj2k_encode_status_as_bytes_mut, + htj2k_encode_statuses_as_bytes_mut, htj2k_encode_statuses_byte_len, u16_slice_as_bytes, + }, + context::{ + CudaContext, CudaHtj2kCompactEncodedCodeBlock, CudaHtj2kCompactEncodedCodeBlocks, + HTJ2K_UVLC_ENCODE_TABLE_BYTES, + }, + driver::CuFunction, + error::CudaError, + execution::{cuda_kernel_param, CudaExecutionStats}, + htj2k_decode::{HTJ2K_STATUS_OK, HTJ2K_STATUS_UNSUPPORTED}, + kernels::{ + htj2k_codeblock_sample_launch_geometry, htj2k_encode_codeblock_launch_geometry, CudaKernel, + }, + memory::{ + checked_image_words, copy_pooled_bytes_to_vec_uninit, pooled_device_buffer, CudaBufferPool, + CudaDeviceBuffer, + }, +}; + +/// Static HTJ2K cleanup encoder lookup tables uploaded for CUDA code-block encode. +#[derive(Clone, Copy, Debug)] +pub struct CudaHtj2kEncodeTables<'a> { + /// HT cleanup encoder VLC table for first quad row contexts. + pub vlc_table0: &'a [u16; 2048], + /// HT cleanup encoder VLC table for subsequent quad row contexts. + pub vlc_table1: &'a [u16; 2048], + /// Packed HT cleanup encoder UVLC table rows, six bytes per row. + pub uvlc_table: &'a [u8], +} + +/// Device-resident HTJ2K cleanup encode lookup tables reused across sub-band dispatches. +#[derive(Debug)] +pub struct CudaHtj2kEncodeResources { + pub(crate) vlc_table0: CudaDeviceBuffer, + pub(crate) vlc_table1: CudaDeviceBuffer, + pub(crate) uvlc_table: CudaDeviceBuffer, +} + +pub(crate) const HTJ2K_ENCODE_MAX_CODEBLOCK_WIDTH: u32 = 1024; + +pub(crate) const HTJ2K_ENCODE_MAX_CODEBLOCK_SAMPLES: usize = 4096; + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub(crate) struct CudaHtj2kEncodeParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) coefficient_stride: u32, + pub(crate) total_bitplanes: u32, + pub(crate) output_capacity: u32, + pub(crate) target_coding_passes: u32, +} + +/// One HTJ2K code-block encode job consumed by the CUDA batch encoder. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaHtj2kEncodeCodeBlockJob { + /// Offset, in i32 coefficients, into the contiguous coefficient buffer. + pub coefficient_offset: u32, + /// Code-block width in coefficients. + pub width: u32, + /// Code-block height in coefficients. + pub height: u32, + /// Total coded bitplanes for this code block's sub-band. + pub total_bitplanes: u8, + /// Requested HT coding passes. `1` emits cleanup-only output; `2` emits a + /// zero `SigProp` segment for exactly representable blocks; `3` emits + /// `SigProp` bits for newly significant magnitude-3 samples plus `MagRef` + /// bits for cleanup-significant samples. + pub target_coding_passes: u8, +} + +/// One HTJ2K code-block region consumed from a strided resident coefficient buffer. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaHtj2kEncodeCodeBlockRegionJob { + /// Offset, in i32 coefficients, to the top-left coefficient of this code block. + pub coefficient_offset: u32, + /// Source row stride in i32 coefficients. + pub coefficient_stride: u32, + /// Code-block width in coefficients. + pub width: u32, + /// Code-block height in coefficients. + pub height: u32, + /// Total coded bitplanes for this code block's sub-band. + pub total_bitplanes: u8, + /// Requested HT coding passes. `1` emits cleanup-only output; `2` emits a + /// zero `SigProp` segment for exactly representable blocks; `3` emits + /// `SigProp` bits for newly significant magnitude-3 samples plus `MagRef` + /// bits for cleanup-significant samples. + pub target_coding_passes: u8, +} + +/// Resident coefficient buffer and jobs for a multi-input HTJ2K encode batch. +#[derive(Clone, Copy, Debug)] +pub struct CudaHtj2kEncodeResidentTarget<'a> { + /// Device buffer containing quantized i32 coefficients. + pub coefficients: &'a CudaDeviceBuffer, + /// Number of i32 coefficients available in `coefficients`. + pub coefficient_count: usize, + /// Code-block jobs that read from `coefficients`. + pub jobs: &'a [CudaHtj2kEncodeCodeBlockJob], +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub(crate) struct CudaHtj2kEncodeKernelJob { + pub(crate) coefficient_offset: u32, + pub(crate) coefficient_stride: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) total_bitplanes: u32, + pub(crate) output_offset: u32, + pub(crate) output_capacity: u32, + pub(crate) target_coding_passes: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub(crate) struct CudaHtj2kEncodeMultiInputKernelJob { + pub(crate) coefficient_ptr: u64, + pub(crate) coefficient_offset: u32, + pub(crate) coefficient_stride: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) total_bitplanes: u32, + pub(crate) output_offset: u32, + pub(crate) output_capacity: u32, + pub(crate) target_coding_passes: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct CudaHtj2kEncodeCompactJob { + pub(crate) source_offset: u32, + pub(crate) compact_offset: u32, + pub(crate) data_len: u32, + pub(crate) reserved: u32, +} + +/// Status written by the CUDA HTJ2K code-block cleanup-pass encoder. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaHtj2kEncodeStatus { + /// Zero on success; nonzero values are kernel-defined failures. + pub code: u32, + /// Kernel-defined failure detail. + pub detail: u32, + /// Encoded payload byte length. + pub data_len: u32, + /// Number of coding passes in the encoded payload. + pub number_of_coding_passes: u32, + /// Number of missing most-significant bitplanes. + pub missing_bit_planes: u32, + /// Reserved for ABI stability. + pub reserved0: u32, + /// Reserved for ABI stability. + pub reserved1: u32, + /// Reserved for ABI stability. + pub reserved2: u32, +} + +impl CudaHtj2kEncodeStatus { + /// Return true when the CUDA kernel reported success. + pub fn is_ok(self) -> bool { + self.code == HTJ2K_STATUS_OK + } +} + +/// CUDA event timings for HTJ2K cleanup-pass encode stages. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaHtj2kEncodeStageTimings { + /// Total HT cleanup-pass encode, compaction, and required result readback time, in microseconds. + pub ht_encode_us: u128, + /// HT cleanup-pass encode kernel time, in microseconds. + pub ht_kernel_us: u128, + /// Status-buffer device-to-host readback time, in microseconds. + pub ht_status_readback_us: u128, + /// Encoded-byte compaction kernel time, in microseconds. + pub ht_compact_us: u128, + /// Compacted encoded-byte device-to-host readback time, in microseconds. + pub ht_output_readback_us: u128, +} + +impl CudaHtj2kEncodeStageTimings { + fn from_parts( + ht_kernel_us: u128, + ht_status_readback_us: u128, + ht_compact_us: u128, + ht_output_readback_us: u128, + ) -> Self { + Self { + ht_encode_us: ht_kernel_us + .saturating_add(ht_status_readback_us) + .saturating_add(ht_compact_us) + .saturating_add(ht_output_readback_us), + ht_kernel_us, + ht_status_readback_us, + ht_compact_us, + ht_output_readback_us, + } + } +} + +/// Host-visible HTJ2K cleanup-pass encode result produced by a CUDA kernel. +#[derive(Debug)] +pub struct CudaHtj2kEncodedCodeBlock { + pub(crate) data: Vec, + pub(crate) status: CudaHtj2kEncodeStatus, + pub(crate) execution: CudaExecutionStats, + pub(crate) stage_timings: CudaHtj2kEncodeStageTimings, +} + +impl CudaHtj2kEncodedCodeBlock { + /// Encoded cleanup-pass payload bytes. + pub fn data(&self) -> &[u8] { + &self.data + } + + /// HTJ2K cleanup segment length in bytes. + pub fn cleanup_length(&self) -> u32 { + if self.status.number_of_coding_passes <= 1 { + self.status.data_len + } else { + self.status.reserved0 + } + } + + /// HTJ2K refinement segment length in bytes. + pub fn refinement_length(&self) -> u32 { + if self.status.number_of_coding_passes <= 1 { + 0 + } else { + self.status.reserved1 + } + } + + /// Number of coding passes in the encoded payload. + pub fn num_coding_passes(&self) -> u8 { + u8::try_from(self.status.number_of_coding_passes).unwrap_or(u8::MAX) + } + + /// Number of missing most-significant bitplanes. + pub fn num_zero_bitplanes(&self) -> u8 { + u8::try_from(self.status.missing_bit_planes).unwrap_or(u8::MAX) + } + + /// Consume this code block and return its encoded payload plus segment + /// metadata. + pub fn into_parts(self) -> (Vec, u32, u32, u8, u8) { + let cleanup_length = if self.status.number_of_coding_passes <= 1 { + self.status.data_len + } else { + self.status.reserved0 + }; + let refinement_length = if self.status.number_of_coding_passes <= 1 { + 0 + } else { + self.status.reserved1 + }; + ( + self.data, + cleanup_length, + refinement_length, + u8::try_from(self.status.number_of_coding_passes).unwrap_or(u8::MAX), + u8::try_from(self.status.missing_bit_planes).unwrap_or(u8::MAX), + ) + } + + /// Kernel status row downloaded after dispatch. + pub fn status(&self) -> CudaHtj2kEncodeStatus { + self.status + } + + /// CUDA execution counters for the encode dispatch. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// CUDA event timings for the encode dispatch. + pub fn stage_timings(&self) -> CudaHtj2kEncodeStageTimings { + self.stage_timings + } +} + +/// Host-visible HTJ2K cleanup-pass encode batch produced by one CUDA kernel dispatch. +#[derive(Debug)] +pub struct CudaHtj2kEncodedCodeBlocks { + pub(crate) code_blocks: Vec, + pub(crate) execution: CudaExecutionStats, + pub(crate) stage_timings: CudaHtj2kEncodeStageTimings, +} + +impl CudaHtj2kEncodedCodeBlocks { + /// Encoded cleanup code-block payloads, in the same order as the submitted jobs. + pub fn code_blocks(&self) -> &[CudaHtj2kEncodedCodeBlock] { + &self.code_blocks + } + + /// Consume the batch and return its per-code-block outputs. + pub fn into_code_blocks(self) -> Vec { + self.code_blocks + } + + /// CUDA execution counters for the batch encode dispatch. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// CUDA event timings for the batch encode dispatch. + pub fn stage_timings(&self) -> CudaHtj2kEncodeStageTimings { + self.stage_timings + } +} + +pub(crate) const HTJ2K_ENCODE_OUTPUT_CAPACITY: usize = 24 * 1024; + +impl CudaContext { + /// Upload static HTJ2K cleanup encoder lookup tables once for reuse. + pub fn upload_htj2k_encode_resources( + &self, + tables: CudaHtj2kEncodeTables<'_>, + ) -> Result { + if tables.uvlc_table.len() != HTJ2K_UVLC_ENCODE_TABLE_BYTES { + return Err(CudaError::LengthTooLarge { + len: tables.uvlc_table.len(), + }); + } + self.inner.set_current()?; + Ok(CudaHtj2kEncodeResources { + vlc_table0: self.upload(u16_slice_as_bytes(tables.vlc_table0))?, + vlc_table1: self.upload(u16_slice_as_bytes(tables.vlc_table1))?, + uvlc_table: self.upload(tables.uvlc_table)?, + }) + } + + /// Encode one HTJ2K cleanup-pass code block with CUDA. + pub fn encode_htj2k_codeblock( + &self, + coefficients: &[i32], + width: u32, + height: u32, + total_bitplanes: u8, + tables: CudaHtj2kEncodeTables<'_>, + ) -> Result { + let resources = self.upload_htj2k_encode_resources(tables)?; + self.encode_htj2k_codeblock_with_resources( + coefficients, + width, + height, + total_bitplanes, + &resources, + ) + } + + /// Encode one HTJ2K cleanup-pass code block with pre-uploaded lookup tables. + pub fn encode_htj2k_codeblock_with_resources( + &self, + coefficients: &[i32], + width: u32, + height: u32, + total_bitplanes: u8, + resources: &CudaHtj2kEncodeResources, + ) -> Result { + let expected_len = checked_image_words(width, height, 1)?; + if coefficients.len() != expected_len { + return Err(CudaError::LengthTooLarge { + len: coefficients.len(), + }); + } + + self.inner.set_current()?; + let coefficient_buffer = self.upload_i32_pinned(coefficients)?; + let output_buffer = self.allocate(HTJ2K_ENCODE_OUTPUT_CAPACITY)?; + let params = CudaHtj2kEncodeParams { + width, + height, + coefficient_stride: width, + total_bitplanes: u32::from(total_bitplanes), + output_capacity: u32::try_from(HTJ2K_ENCODE_OUTPUT_CAPACITY).map_err(|_| { + CudaError::LengthTooLarge { + len: HTJ2K_ENCODE_OUTPUT_CAPACITY, + } + })?, + target_coding_passes: 1, + }; + let params_buffer = self.upload(htj2k_encode_params_as_bytes(¶ms))?; + let initial_status = CudaHtj2kEncodeStatus { + code: HTJ2K_STATUS_UNSUPPORTED, + ..CudaHtj2kEncodeStatus::default() + }; + let status_buffer = self.upload(htj2k_encode_status_as_bytes(&initial_status))?; + + let ((), ht_encode_us) = + self.time_default_stream_named_us("j2k.htj2k.encode.codeblock", || { + self.launch_htj2k_encode_codeblock( + &coefficient_buffer, + &output_buffer, + ¶ms_buffer, + &resources.vlc_table0, + &resources.vlc_table1, + &resources.uvlc_table, + &status_buffer, + ) + })?; + let (status, status_readback_us) = self.time_default_stream_named_us( + "j2k.htj2k.encode.codeblock.status_readback", + || { + let mut status = CudaHtj2kEncodeStatus::default(); + status_buffer.copy_to_host(htj2k_encode_status_as_bytes_mut(&mut status))?; + if !status.is_ok() { + return Err(CudaError::KernelStatus { + kernel: "j2k_htj2k_encode_codeblock", + code: status.code, + detail: status.detail, + }); + } + Ok(status) + }, + )?; + let data_len = usize::try_from(status.data_len) + .map_err(|_| CudaError::LengthTooLarge { len: usize::MAX })?; + if data_len > HTJ2K_ENCODE_OUTPUT_CAPACITY { + return Err(CudaError::LengthTooLarge { len: data_len }); + } + let (data, output_readback_us) = if data_len == 0 { + (Vec::new(), 0) + } else { + self.time_default_stream_named_us("j2k.htj2k.encode.codeblock.output_readback", || { + let mut data = vec![0u8; data_len]; + output_buffer.copy_range_to_host(0, &mut data)?; + Ok(data) + })? + }; + let stage_timings = CudaHtj2kEncodeStageTimings::from_parts( + ht_encode_us, + status_readback_us, + 0, + output_readback_us, + ); + + Ok(CudaHtj2kEncodedCodeBlock { + data, + status, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + stage_timings, + }) + } + + /// Encode multiple HTJ2K cleanup-pass code blocks with one CUDA dispatch. + pub fn encode_htj2k_codeblocks( + &self, + coefficients: &[i32], + jobs: &[CudaHtj2kEncodeCodeBlockJob], + tables: CudaHtj2kEncodeTables<'_>, + ) -> Result { + let resources = self.upload_htj2k_encode_resources(tables)?; + self.encode_htj2k_codeblocks_with_resources(coefficients, jobs, &resources) + } + + /// Encode multiple HTJ2K cleanup-pass code blocks with pre-uploaded lookup tables. + pub fn encode_htj2k_codeblocks_with_resources( + &self, + coefficients: &[i32], + jobs: &[CudaHtj2kEncodeCodeBlockJob], + resources: &CudaHtj2kEncodeResources, + ) -> Result { + if jobs.is_empty() { + return Ok(CudaHtj2kEncodedCodeBlocks { + code_blocks: Vec::new(), + execution: CudaExecutionStats::default(), + stage_timings: CudaHtj2kEncodeStageTimings::default(), + }); + } + + self.inner.set_current()?; + let coefficient_buffer = self.upload_i32_pinned(coefficients)?; + self.encode_htj2k_codeblocks_device_with_resources( + &coefficient_buffer, + coefficients.len(), + jobs, + resources, + ) + } + + /// Encode multiple HTJ2K cleanup-pass code blocks from resident quantized coefficients. + pub fn encode_htj2k_codeblocks_resident( + &self, + coefficients: &CudaDeviceBuffer, + coefficient_count: usize, + jobs: &[CudaHtj2kEncodeCodeBlockJob], + tables: CudaHtj2kEncodeTables<'_>, + ) -> Result { + let resources = self.upload_htj2k_encode_resources(tables)?; + self.encode_htj2k_codeblocks_resident_with_resources( + coefficients, + coefficient_count, + jobs, + &resources, + ) + } + + /// Encode multiple cleanup-pass code blocks from resident coefficients with lookup table reuse. + pub fn encode_htj2k_codeblocks_resident_with_resources( + &self, + coefficients: &CudaDeviceBuffer, + coefficient_count: usize, + jobs: &[CudaHtj2kEncodeCodeBlockJob], + resources: &CudaHtj2kEncodeResources, + ) -> Result { + let pool = self.buffer_pool(); + self.encode_htj2k_codeblocks_resident_with_resources_and_pool( + coefficients, + coefficient_count, + jobs, + resources, + &pool, + ) + } + + /// Encode multiple cleanup-pass code blocks from resident coefficients with + /// lookup table reuse and caller-owned transient buffer reuse. + pub fn encode_htj2k_codeblocks_resident_with_resources_and_pool( + &self, + coefficients: &CudaDeviceBuffer, + coefficient_count: usize, + jobs: &[CudaHtj2kEncodeCodeBlockJob], + resources: &CudaHtj2kEncodeResources, + pool: &CudaBufferPool, + ) -> Result { + if jobs.is_empty() { + return Ok(CudaHtj2kEncodedCodeBlocks { + code_blocks: Vec::new(), + execution: CudaExecutionStats::default(), + stage_timings: CudaHtj2kEncodeStageTimings::default(), + }); + } + let available_coefficients = coefficients.typed_view::()?.len(); + if available_coefficients < coefficient_count { + return Err(CudaError::OutputTooSmall { + required: coefficient_count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { + len: coefficient_count, + })?, + have: coefficients.byte_len(), + }); + } + + let kernel_jobs = htj2k_encode_kernel_jobs(jobs, coefficient_count)?; + self.inner.set_current()?; + self.encode_htj2k_kernel_jobs_device_with_resources_and_pool( + coefficients, + &kernel_jobs, + resources, + pool, + ) + } + + /// Encode multiple cleanup-pass code-block batches from independent + /// resident coefficient buffers with one CUDA dispatch. + pub fn encode_htj2k_codeblocks_multi_resident_with_resources_and_pool( + &self, + targets: &[CudaHtj2kEncodeResidentTarget<'_>], + resources: &CudaHtj2kEncodeResources, + pool: &CudaBufferPool, + ) -> Result { + self.encode_htj2k_codeblocks_multi_resident_compact_with_resources_and_pool( + targets, resources, pool, + )? + .into_owned_code_blocks() + } + + /// Encode multiple cleanup-pass code-block batches from independent resident + /// coefficient buffers with one CUDA dispatch, returning one compact payload + /// plus per-block ranges. + pub fn encode_htj2k_codeblocks_multi_resident_compact_with_resources_and_pool( + &self, + targets: &[CudaHtj2kEncodeResidentTarget<'_>], + resources: &CudaHtj2kEncodeResources, + pool: &CudaBufferPool, + ) -> Result { + let kernel_jobs = htj2k_encode_multi_input_kernel_jobs(targets)?; + if kernel_jobs.is_empty() { + return Ok(CudaHtj2kCompactEncodedCodeBlocks { + payload: Vec::new(), + code_blocks: Vec::new(), + execution: CudaExecutionStats::default(), + stage_timings: CudaHtj2kEncodeStageTimings::default(), + }); + } + self.inner.set_current()?; + self.encode_htj2k_multi_input_kernel_jobs_device_compact_with_resources_and_pool( + &kernel_jobs, + resources, + pool, + ) + } + + /// Encode cleanup-pass code blocks from strided resident coefficient regions. + pub fn encode_htj2k_codeblock_regions_resident( + &self, + coefficients: &CudaDeviceBuffer, + coefficient_count: usize, + jobs: &[CudaHtj2kEncodeCodeBlockRegionJob], + tables: CudaHtj2kEncodeTables<'_>, + ) -> Result { + let resources = self.upload_htj2k_encode_resources(tables)?; + self.encode_htj2k_codeblock_regions_resident_with_resources( + coefficients, + coefficient_count, + jobs, + &resources, + ) + } + + /// Encode strided resident code-block regions with pre-uploaded lookup tables. + pub fn encode_htj2k_codeblock_regions_resident_with_resources( + &self, + coefficients: &CudaDeviceBuffer, + coefficient_count: usize, + jobs: &[CudaHtj2kEncodeCodeBlockRegionJob], + resources: &CudaHtj2kEncodeResources, + ) -> Result { + let pool = self.buffer_pool(); + self.encode_htj2k_codeblock_regions_resident_with_resources_and_pool( + coefficients, + coefficient_count, + jobs, + resources, + &pool, + ) + } + + /// Encode strided resident code-block regions with pre-uploaded lookup + /// tables and caller-owned transient buffer reuse. + pub fn encode_htj2k_codeblock_regions_resident_with_resources_and_pool( + &self, + coefficients: &CudaDeviceBuffer, + coefficient_count: usize, + jobs: &[CudaHtj2kEncodeCodeBlockRegionJob], + resources: &CudaHtj2kEncodeResources, + pool: &CudaBufferPool, + ) -> Result { + if jobs.is_empty() { + return Ok(CudaHtj2kEncodedCodeBlocks { + code_blocks: Vec::new(), + execution: CudaExecutionStats::default(), + stage_timings: CudaHtj2kEncodeStageTimings::default(), + }); + } + let available_coefficients = coefficients.typed_view::()?.len(); + if available_coefficients < coefficient_count { + return Err(CudaError::OutputTooSmall { + required: coefficient_count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { + len: coefficient_count, + })?, + have: coefficients.byte_len(), + }); + } + + let kernel_jobs = htj2k_encode_region_kernel_jobs(jobs, coefficient_count)?; + self.inner.set_current()?; + self.encode_htj2k_kernel_jobs_device_with_resources_and_pool( + coefficients, + &kernel_jobs, + resources, + pool, + ) + } + + fn encode_htj2k_codeblocks_device_with_resources( + &self, + coefficient_buffer: &CudaDeviceBuffer, + coefficient_count: usize, + jobs: &[CudaHtj2kEncodeCodeBlockJob], + resources: &CudaHtj2kEncodeResources, + ) -> Result { + let kernel_jobs = htj2k_encode_kernel_jobs(jobs, coefficient_count)?; + self.encode_htj2k_kernel_jobs_device_with_resources( + coefficient_buffer, + &kernel_jobs, + resources, + ) + } + + #[allow(clippy::too_many_lines)] + fn encode_htj2k_kernel_jobs_device_with_resources( + &self, + coefficient_buffer: &CudaDeviceBuffer, + kernel_jobs: &[CudaHtj2kEncodeKernelJob], + resources: &CudaHtj2kEncodeResources, + ) -> Result { + let pool = self.buffer_pool(); + self.encode_htj2k_kernel_jobs_device_with_resources_and_pool( + coefficient_buffer, + kernel_jobs, + resources, + &pool, + ) + } + + #[allow(clippy::too_many_lines)] + fn encode_htj2k_kernel_jobs_device_with_resources_and_pool( + &self, + coefficient_buffer: &CudaDeviceBuffer, + kernel_jobs: &[CudaHtj2kEncodeKernelJob], + resources: &CudaHtj2kEncodeResources, + pool: &CudaBufferPool, + ) -> Result { + let output_bytes = kernel_jobs + .last() + .map(|job| { + (job.output_offset as usize) + .checked_add(job.output_capacity as usize) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX }) + }) + .transpose()? + .unwrap_or(0); + + let jobs_buffer = pool.upload(htj2k_encode_jobs_as_bytes(kernel_jobs))?; + let output_buffer = pool.take(output_bytes)?; + let status_buffer = pool.take(htj2k_encode_statuses_byte_len(kernel_jobs.len())?)?; + + let ((), ht_encode_us) = + self.time_default_stream_named_us("j2k.htj2k.encode.codeblocks", || { + self.launch_htj2k_encode_codeblocks( + coefficient_buffer, + pooled_device_buffer(&output_buffer)?, + pooled_device_buffer(&jobs_buffer)?, + &resources.vlc_table0, + &resources.vlc_table1, + &resources.uvlc_table, + pooled_device_buffer(&status_buffer)?, + kernel_jobs.len(), + ) + })?; + let (statuses, status_readback_us) = self.time_default_stream_named_us( + "j2k.htj2k.encode.codeblocks.status_readback", + || { + let mut statuses = vec![CudaHtj2kEncodeStatus::default(); kernel_jobs.len()]; + status_buffer.copy_to_host(htj2k_encode_statuses_as_bytes_mut(&mut statuses))?; + if let Some(status) = statuses.iter().copied().find(|status| !status.is_ok()) { + return Err(CudaError::KernelStatus { + kernel: "j2k_htj2k_encode_codeblocks", + code: status.code, + detail: status.detail, + }); + } + Ok(statuses) + }, + )?; + + let (compact_jobs, compact_output_bytes) = + htj2k_encode_compact_jobs(&statuses, kernel_jobs)?; + let compact_output_buffer = pool.take(compact_output_bytes)?; + let compact_dispatched = compact_output_bytes != 0; + let compact_us = if compact_dispatched { + let compact_jobs_buffer = + pool.upload(htj2k_encode_compact_jobs_as_bytes(&compact_jobs))?; + let ((), compact_us) = + self.time_default_stream_named_us("j2k.htj2k.encode.codeblocks.compact", || { + self.launch_htj2k_compact_codeblocks( + pooled_device_buffer(&output_buffer)?, + pooled_device_buffer(&compact_output_buffer)?, + pooled_device_buffer(&compact_jobs_buffer)?, + compact_jobs.len(), + ) + })?; + compact_us + } else { + 0 + }; + let (output, output_readback_us) = if compact_output_bytes == 0 { + (Vec::new(), 0) + } else { + self.time_default_stream_named_us( + "j2k.htj2k.encode.codeblocks.output_readback", + || copy_pooled_bytes_to_vec_uninit(&compact_output_buffer, compact_output_bytes), + )? + }; + + let mut code_blocks = statuses + .into_iter() + .zip(kernel_jobs.iter()) + .zip(compact_jobs.iter()) + .map(|((status, job), compact_job)| { + let data_len = usize::try_from(status.data_len) + .map_err(|_| CudaError::LengthTooLarge { len: usize::MAX })?; + if data_len > job.output_capacity as usize { + return Err(CudaError::LengthTooLarge { len: data_len }); + } + let start = compact_job.compact_offset as usize; + let end = start + .checked_add(data_len) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if end > compact_output_bytes { + return Err(CudaError::LengthTooLarge { len: end }); + } + let data = output[start..end].to_vec(); + Ok(CudaHtj2kEncodedCodeBlock { + data, + status, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: usize::from(compact_dispatched), + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + stage_timings: CudaHtj2kEncodeStageTimings::default(), + }) + }) + .collect::, CudaError>>()?; + let stage_timings = CudaHtj2kEncodeStageTimings::from_parts( + ht_encode_us, + status_readback_us, + compact_us, + output_readback_us, + ); + for block in &mut code_blocks { + block.stage_timings = stage_timings; + } + let copy_kernel_dispatches = + usize::from(code_blocks.iter().any(|block| !block.data().is_empty())); + + Ok(CudaHtj2kEncodedCodeBlocks { + code_blocks, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + stage_timings, + }) + } + + #[allow(clippy::too_many_lines)] + fn encode_htj2k_multi_input_kernel_jobs_device_compact_with_resources_and_pool( + &self, + kernel_jobs: &[CudaHtj2kEncodeMultiInputKernelJob], + resources: &CudaHtj2kEncodeResources, + pool: &CudaBufferPool, + ) -> Result { + let output_bytes = kernel_jobs + .last() + .map(|job| { + (job.output_offset as usize) + .checked_add(job.output_capacity as usize) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX }) + }) + .transpose()? + .unwrap_or(0); + + let jobs_buffer = pool.upload(htj2k_encode_multi_input_jobs_as_bytes(kernel_jobs))?; + let output_buffer = pool.take(output_bytes)?; + let status_buffer = pool.take(htj2k_encode_statuses_byte_len(kernel_jobs.len())?)?; + let cleanup_only = kernel_jobs.iter().all(|job| job.target_coding_passes == 1); + let cleanup_only_64 = cleanup_only + && kernel_jobs + .iter() + .all(|job| job.width == 64 && job.height == 64 && job.coefficient_stride == 64); + let status_kernel = if cleanup_only_64 { + "j2k_htj2k_encode_codeblocks_multi_input_cleanup_64" + } else if cleanup_only { + "j2k_htj2k_encode_codeblocks_multi_input_cleanup" + } else { + "j2k_htj2k_encode_codeblocks_multi_input" + }; + + let ((), ht_encode_us) = + self.time_default_stream_named_us("j2k.htj2k.encode.codeblocks.multi_input", || { + if cleanup_only_64 { + self.launch_htj2k_encode_codeblocks_multi_input_cleanup_64( + pooled_device_buffer(&output_buffer)?, + pooled_device_buffer(&jobs_buffer)?, + &resources.vlc_table0, + &resources.vlc_table1, + &resources.uvlc_table, + pooled_device_buffer(&status_buffer)?, + kernel_jobs.len(), + ) + } else if cleanup_only { + self.launch_htj2k_encode_codeblocks_multi_input_cleanup( + pooled_device_buffer(&output_buffer)?, + pooled_device_buffer(&jobs_buffer)?, + &resources.vlc_table0, + &resources.vlc_table1, + &resources.uvlc_table, + pooled_device_buffer(&status_buffer)?, + kernel_jobs.len(), + ) + } else { + self.launch_htj2k_encode_codeblocks_multi_input( + pooled_device_buffer(&output_buffer)?, + pooled_device_buffer(&jobs_buffer)?, + &resources.vlc_table0, + &resources.vlc_table1, + &resources.uvlc_table, + pooled_device_buffer(&status_buffer)?, + kernel_jobs.len(), + ) + } + })?; + let (statuses, status_readback_us) = self.time_default_stream_named_us( + "j2k.htj2k.encode.codeblocks.multi_input.status_readback", + || { + let mut statuses = vec![CudaHtj2kEncodeStatus::default(); kernel_jobs.len()]; + status_buffer.copy_to_host(htj2k_encode_statuses_as_bytes_mut(&mut statuses))?; + if let Some(status) = statuses.iter().copied().find(|status| !status.is_ok()) { + return Err(CudaError::KernelStatus { + kernel: status_kernel, + code: status.code, + detail: status.detail, + }); + } + Ok(statuses) + }, + )?; + + let (compact_jobs, compact_output_bytes) = + htj2k_encode_compact_jobs_multi_input(&statuses, kernel_jobs)?; + let compact_output_buffer = pool.take(compact_output_bytes)?; + let compact_dispatched = compact_output_bytes != 0; + let compact_us = if compact_dispatched { + let compact_jobs_buffer = + pool.upload(htj2k_encode_compact_jobs_as_bytes(&compact_jobs))?; + let ((), compact_us) = self.time_default_stream_named_us( + "j2k.htj2k.encode.codeblocks.multi_input.compact", + || { + self.launch_htj2k_compact_codeblocks( + pooled_device_buffer(&output_buffer)?, + pooled_device_buffer(&compact_output_buffer)?, + pooled_device_buffer(&compact_jobs_buffer)?, + compact_jobs.len(), + ) + }, + )?; + compact_us + } else { + 0 + }; + let (output, output_readback_us) = if compact_output_bytes == 0 { + (Vec::new(), 0) + } else { + self.time_default_stream_named_us( + "j2k.htj2k.encode.codeblocks.multi_input.output_readback", + || copy_pooled_bytes_to_vec_uninit(&compact_output_buffer, compact_output_bytes), + )? + }; + + let mut code_blocks = statuses + .into_iter() + .zip(kernel_jobs.iter()) + .zip(compact_jobs.iter()) + .map(|((status, job), compact_job)| { + let data_len = usize::try_from(status.data_len) + .map_err(|_| CudaError::LengthTooLarge { len: usize::MAX })?; + if data_len > job.output_capacity as usize { + return Err(CudaError::LengthTooLarge { len: data_len }); + } + let start = compact_job.compact_offset as usize; + let end = start + .checked_add(data_len) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if end > compact_output_bytes { + return Err(CudaError::LengthTooLarge { len: end }); + } + Ok(CudaHtj2kCompactEncodedCodeBlock { + payload_range: start..end, + status, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: usize::from(compact_dispatched), + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + stage_timings: CudaHtj2kEncodeStageTimings::default(), + }) + }) + .collect::, CudaError>>()?; + let stage_timings = CudaHtj2kEncodeStageTimings::from_parts( + ht_encode_us, + status_readback_us, + compact_us, + output_readback_us, + ); + for block in &mut code_blocks { + block.stage_timings = stage_timings; + } + let copy_kernel_dispatches = usize::from(!output.is_empty()); + + Ok(CudaHtj2kCompactEncodedCodeBlocks { + payload: output, + code_blocks, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + stage_timings, + }) + } + + #[allow(clippy::too_many_arguments)] + fn launch_htj2k_encode_codeblock( + &self, + coefficients: &CudaDeviceBuffer, + output: &CudaDeviceBuffer, + params_buffer: &CudaDeviceBuffer, + vlc_table0: &CudaDeviceBuffer, + vlc_table1: &CudaDeviceBuffer, + uvlc_table: &CudaDeviceBuffer, + status: &CudaDeviceBuffer, + ) -> Result<(), CudaError> { + let function = self + .inner + .kernel_function(CudaKernel::Htj2kEncodeCodeblock)?; + let mut coefficients_ptr = coefficients.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut params_ptr = params_buffer.device_ptr(); + let mut vlc_table0_ptr = vlc_table0.device_ptr(); + let mut vlc_table1_ptr = vlc_table1.device_ptr(); + let mut uvlc_table_ptr = uvlc_table.device_ptr(); + let mut status_ptr = status.device_ptr(); + let mut params = cuda_kernel_params!( + coefficients_ptr, + output_ptr, + params_ptr, + vlc_table0_ptr, + vlc_table1_ptr, + uvlc_table_ptr, + status_ptr + ); + let geometry = htj2k_encode_codeblock_launch_geometry(1) + .ok_or(CudaError::LengthTooLarge { len: 1 })?; + self.launch_kernel_async(function, geometry, &mut params) + } + + #[allow(clippy::too_many_arguments)] + fn launch_htj2k_encode_codeblocks( + &self, + coefficients: &CudaDeviceBuffer, + output: &CudaDeviceBuffer, + jobs: &CudaDeviceBuffer, + vlc_table0: &CudaDeviceBuffer, + vlc_table1: &CudaDeviceBuffer, + uvlc_table: &CudaDeviceBuffer, + statuses: &CudaDeviceBuffer, + job_count: usize, + ) -> Result<(), CudaError> { + let function = self + .inner + .kernel_function(CudaKernel::Htj2kEncodeCodeblocks)?; + let mut coefficients_ptr = coefficients.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut jobs_ptr = jobs.device_ptr(); + let mut vlc_table0_ptr = vlc_table0.device_ptr(); + let mut vlc_table1_ptr = vlc_table1.device_ptr(); + let mut uvlc_table_ptr = uvlc_table.device_ptr(); + let mut statuses_ptr = statuses.device_ptr(); + let mut job_count_u64 = + u64::try_from(job_count).map_err(|_| CudaError::LengthTooLarge { len: job_count })?; + let mut params = cuda_kernel_params!( + coefficients_ptr, + output_ptr, + jobs_ptr, + vlc_table0_ptr, + vlc_table1_ptr, + uvlc_table_ptr, + statuses_ptr, + job_count_u64 + ); + let geometry = htj2k_encode_codeblock_launch_geometry(job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + self.launch_kernel_async(function, geometry, &mut params) + } + + #[allow(clippy::too_many_arguments)] + fn launch_htj2k_encode_codeblocks_multi_input( + &self, + output: &CudaDeviceBuffer, + jobs: &CudaDeviceBuffer, + vlc_table0: &CudaDeviceBuffer, + vlc_table1: &CudaDeviceBuffer, + uvlc_table: &CudaDeviceBuffer, + statuses: &CudaDeviceBuffer, + job_count: usize, + ) -> Result<(), CudaError> { + let function = self + .inner + .kernel_function(CudaKernel::Htj2kEncodeCodeblocksMultiInput)?; + let mut output_ptr = output.device_ptr(); + let mut jobs_ptr = jobs.device_ptr(); + let mut vlc_table0_ptr = vlc_table0.device_ptr(); + let mut vlc_table1_ptr = vlc_table1.device_ptr(); + let mut uvlc_table_ptr = uvlc_table.device_ptr(); + let mut statuses_ptr = statuses.device_ptr(); + let mut job_count_u64 = + u64::try_from(job_count).map_err(|_| CudaError::LengthTooLarge { len: job_count })?; + let mut params = cuda_kernel_params!( + output_ptr, + jobs_ptr, + vlc_table0_ptr, + vlc_table1_ptr, + uvlc_table_ptr, + statuses_ptr, + job_count_u64 + ); + let geometry = htj2k_encode_codeblock_launch_geometry(job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + self.launch_kernel_async(function, geometry, &mut params) + } + + #[allow(clippy::too_many_arguments)] + fn launch_htj2k_encode_codeblocks_multi_input_cleanup( + &self, + output: &CudaDeviceBuffer, + jobs: &CudaDeviceBuffer, + vlc_table0: &CudaDeviceBuffer, + vlc_table1: &CudaDeviceBuffer, + uvlc_table: &CudaDeviceBuffer, + statuses: &CudaDeviceBuffer, + job_count: usize, + ) -> Result<(), CudaError> { + let function = self + .inner + .kernel_function(CudaKernel::Htj2kEncodeCodeblocksMultiInputCleanup)?; + let mut output_ptr = output.device_ptr(); + let mut jobs_ptr = jobs.device_ptr(); + let mut vlc_table0_ptr = vlc_table0.device_ptr(); + let mut vlc_table1_ptr = vlc_table1.device_ptr(); + let mut uvlc_table_ptr = uvlc_table.device_ptr(); + let mut statuses_ptr = statuses.device_ptr(); + let mut job_count_u64 = + u64::try_from(job_count).map_err(|_| CudaError::LengthTooLarge { len: job_count })?; + let mut params = cuda_kernel_params!( + output_ptr, + jobs_ptr, + vlc_table0_ptr, + vlc_table1_ptr, + uvlc_table_ptr, + statuses_ptr, + job_count_u64 + ); + let geometry = htj2k_encode_codeblock_launch_geometry(job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + self.launch_kernel_async(function, geometry, &mut params) + } + + #[allow(clippy::too_many_arguments)] + fn launch_htj2k_encode_codeblocks_multi_input_cleanup_64( + &self, + output: &CudaDeviceBuffer, + jobs: &CudaDeviceBuffer, + vlc_table0: &CudaDeviceBuffer, + vlc_table1: &CudaDeviceBuffer, + uvlc_table: &CudaDeviceBuffer, + statuses: &CudaDeviceBuffer, + job_count: usize, + ) -> Result<(), CudaError> { + let function = self + .inner + .kernel_function(CudaKernel::Htj2kEncodeCodeblocksMultiInputCleanup64)?; + let mut output_ptr = output.device_ptr(); + let mut jobs_ptr = jobs.device_ptr(); + let mut vlc_table0_ptr = vlc_table0.device_ptr(); + let mut vlc_table1_ptr = vlc_table1.device_ptr(); + let mut uvlc_table_ptr = uvlc_table.device_ptr(); + let mut statuses_ptr = statuses.device_ptr(); + let mut job_count_u64 = + u64::try_from(job_count).map_err(|_| CudaError::LengthTooLarge { len: job_count })?; + let mut params = cuda_kernel_params!( + output_ptr, + jobs_ptr, + vlc_table0_ptr, + vlc_table1_ptr, + uvlc_table_ptr, + statuses_ptr, + job_count_u64 + ); + let geometry = htj2k_encode_codeblock_launch_geometry(job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + self.launch_kernel_async(function, geometry, &mut params) + } + + pub(crate) fn launch_htj2k_compact_codeblocks( + &self, + scratch: &CudaDeviceBuffer, + compact: &CudaDeviceBuffer, + jobs: &CudaDeviceBuffer, + job_count: usize, + ) -> Result<(), CudaError> { + let function = self.htj2k_encode_kernel_function(CudaKernel::Htj2kCompactCodeblocks)?; + let mut scratch_ptr = scratch.device_ptr(); + let mut compact_ptr = compact.device_ptr(); + let mut jobs_ptr = jobs.device_ptr(); + let mut job_count_u64 = + u64::try_from(job_count).map_err(|_| CudaError::LengthTooLarge { len: job_count })?; + let mut params = cuda_kernel_params!(scratch_ptr, compact_ptr, jobs_ptr, job_count_u64); + let geometry = htj2k_codeblock_sample_launch_geometry(job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + self.launch_kernel_async(function, geometry, &mut params) + } + + fn htj2k_encode_kernel_function(&self, kernel: CudaKernel) -> Result { + #[cfg(feature = "cuda-oxide-j2k-encode")] + { + if crate::build_flags::cuda_oxide_j2k_encode_enabled() + && kernel.is_cuda_oxide_j2k_encode_stage() + { + return self.inner.cuda_oxide_j2k_encode_kernel_function(kernel); + } + } + self.inner.kernel_function(kernel) + } +} + +pub(crate) fn htj2k_encode_kernel_jobs( + jobs: &[CudaHtj2kEncodeCodeBlockJob], + coefficient_words: usize, +) -> Result, CudaError> { + let mut output_offset = 0usize; + let mut kernel_jobs = Vec::with_capacity(jobs.len()); + for job in jobs { + validate_htj2k_encode_codeblock_shape(job.width, job.height)?; + let coefficient_offset = job.coefficient_offset as usize; + let coefficient_len = checked_image_words(job.width, job.height, 1)?; + let coefficient_end = + coefficient_offset + .checked_add(coefficient_len) + .ok_or(CudaError::LengthTooLarge { + len: coefficient_words, + })?; + if coefficient_end > coefficient_words { + return Err(CudaError::LengthTooLarge { + len: coefficient_end, + }); + } + + let output_end = output_offset + .checked_add(HTJ2K_ENCODE_OUTPUT_CAPACITY) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if output_end > u32::MAX as usize { + return Err(CudaError::LengthTooLarge { len: output_end }); + } + kernel_jobs.push(CudaHtj2kEncodeKernelJob { + coefficient_offset: job.coefficient_offset, + coefficient_stride: job.width, + width: job.width, + height: job.height, + total_bitplanes: u32::from(job.total_bitplanes), + output_offset: u32::try_from(output_offset) + .map_err(|_| CudaError::LengthTooLarge { len: output_offset })?, + output_capacity: u32::try_from(HTJ2K_ENCODE_OUTPUT_CAPACITY).map_err(|_| { + CudaError::LengthTooLarge { + len: HTJ2K_ENCODE_OUTPUT_CAPACITY, + } + })?, + target_coding_passes: u32::from(job.target_coding_passes), + }); + output_offset = output_end; + } + Ok(kernel_jobs) +} + +pub(crate) fn htj2k_encode_multi_input_kernel_jobs( + targets: &[CudaHtj2kEncodeResidentTarget<'_>], +) -> Result, CudaError> { + let job_count = targets + .iter() + .try_fold(0usize, |sum, target| sum.checked_add(target.jobs.len())) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + let mut output_offset = 0usize; + let mut kernel_jobs = Vec::with_capacity(job_count); + for target in targets { + let available_coefficients = target.coefficients.typed_view::()?.len(); + if available_coefficients < target.coefficient_count { + return Err(CudaError::OutputTooSmall { + required: target + .coefficient_count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { + len: target.coefficient_count, + })?, + have: target.coefficients.byte_len(), + }); + } + for job in target.jobs { + validate_htj2k_encode_codeblock_shape(job.width, job.height)?; + let coefficient_offset = job.coefficient_offset as usize; + let coefficient_len = checked_image_words(job.width, job.height, 1)?; + let coefficient_end = coefficient_offset.checked_add(coefficient_len).ok_or( + CudaError::LengthTooLarge { + len: target.coefficient_count, + }, + )?; + if coefficient_end > target.coefficient_count { + return Err(CudaError::LengthTooLarge { + len: coefficient_end, + }); + } + + let output_end = output_offset + .checked_add(HTJ2K_ENCODE_OUTPUT_CAPACITY) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if output_end > u32::MAX as usize { + return Err(CudaError::LengthTooLarge { len: output_end }); + } + kernel_jobs.push(CudaHtj2kEncodeMultiInputKernelJob { + coefficient_ptr: target.coefficients.device_ptr(), + coefficient_offset: job.coefficient_offset, + coefficient_stride: job.width, + width: job.width, + height: job.height, + total_bitplanes: u32::from(job.total_bitplanes), + output_offset: u32::try_from(output_offset) + .map_err(|_| CudaError::LengthTooLarge { len: output_offset })?, + output_capacity: u32::try_from(HTJ2K_ENCODE_OUTPUT_CAPACITY).map_err(|_| { + CudaError::LengthTooLarge { + len: HTJ2K_ENCODE_OUTPUT_CAPACITY, + } + })?, + target_coding_passes: u32::from(job.target_coding_passes), + }); + output_offset = output_end; + } + } + Ok(kernel_jobs) +} + +pub(crate) fn htj2k_encode_region_kernel_jobs( + jobs: &[CudaHtj2kEncodeCodeBlockRegionJob], + coefficient_words: usize, +) -> Result, CudaError> { + let mut output_offset = 0usize; + let mut kernel_jobs = Vec::with_capacity(jobs.len()); + for job in jobs { + validate_htj2k_encode_codeblock_shape(job.width, job.height)?; + if job.width == 0 || job.height == 0 || job.coefficient_stride < job.width { + return Err(CudaError::LengthTooLarge { + len: coefficient_words, + }); + } + let row_offset = (job.height as usize - 1) + .checked_mul(job.coefficient_stride as usize) + .ok_or(CudaError::LengthTooLarge { + len: coefficient_words, + })?; + let coefficient_end = job + .coefficient_offset + .try_into() + .ok() + .and_then(|offset: usize| offset.checked_add(row_offset)) + .and_then(|offset| offset.checked_add(job.width as usize)) + .ok_or(CudaError::LengthTooLarge { + len: coefficient_words, + })?; + if coefficient_end > coefficient_words { + return Err(CudaError::LengthTooLarge { + len: coefficient_end, + }); + } + + let output_end = output_offset + .checked_add(HTJ2K_ENCODE_OUTPUT_CAPACITY) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if output_end > u32::MAX as usize { + return Err(CudaError::LengthTooLarge { len: output_end }); + } + kernel_jobs.push(CudaHtj2kEncodeKernelJob { + coefficient_offset: job.coefficient_offset, + coefficient_stride: job.coefficient_stride, + width: job.width, + height: job.height, + total_bitplanes: u32::from(job.total_bitplanes), + output_offset: u32::try_from(output_offset) + .map_err(|_| CudaError::LengthTooLarge { len: output_offset })?, + output_capacity: u32::try_from(HTJ2K_ENCODE_OUTPUT_CAPACITY).map_err(|_| { + CudaError::LengthTooLarge { + len: HTJ2K_ENCODE_OUTPUT_CAPACITY, + } + })?, + target_coding_passes: u32::from(job.target_coding_passes), + }); + output_offset = output_end; + } + Ok(kernel_jobs) +} + +pub(crate) fn htj2k_encode_compact_jobs( + statuses: &[CudaHtj2kEncodeStatus], + kernel_jobs: &[CudaHtj2kEncodeKernelJob], +) -> Result<(Vec, usize), CudaError> { + if statuses.len() != kernel_jobs.len() { + return Err(CudaError::InvalidArgument { + message: "HTJ2K encode status count does not match job count".to_string(), + }); + } + + let mut compact_offset = 0usize; + let mut compact_jobs = Vec::with_capacity(kernel_jobs.len()); + for (status, job) in statuses.iter().zip(kernel_jobs) { + let data_len = usize::try_from(status.data_len) + .map_err(|_| CudaError::LengthTooLarge { len: usize::MAX })?; + if data_len > job.output_capacity as usize { + return Err(CudaError::LengthTooLarge { len: data_len }); + } + let source_end = (job.output_offset as usize) + .checked_add(data_len) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + let job_output_end = (job.output_offset as usize) + .checked_add(job.output_capacity as usize) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if source_end > job_output_end { + return Err(CudaError::LengthTooLarge { len: source_end }); + } + compact_jobs.push(CudaHtj2kEncodeCompactJob { + source_offset: job.output_offset, + compact_offset: u32::try_from(compact_offset).map_err(|_| { + CudaError::LengthTooLarge { + len: compact_offset, + } + })?, + data_len: status.data_len, + reserved: status.reserved2, + }); + compact_offset = compact_offset + .checked_add(data_len) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if compact_offset > u32::MAX as usize { + return Err(CudaError::LengthTooLarge { + len: compact_offset, + }); + } + } + + Ok((compact_jobs, compact_offset)) +} + +pub(crate) fn htj2k_encode_compact_jobs_multi_input( + statuses: &[CudaHtj2kEncodeStatus], + kernel_jobs: &[CudaHtj2kEncodeMultiInputKernelJob], +) -> Result<(Vec, usize), CudaError> { + if statuses.len() != kernel_jobs.len() { + return Err(CudaError::InvalidArgument { + message: "HTJ2K encode status count does not match job count".to_string(), + }); + } + + let mut compact_offset = 0usize; + let mut compact_jobs = Vec::with_capacity(kernel_jobs.len()); + for (status, job) in statuses.iter().zip(kernel_jobs) { + let data_len = usize::try_from(status.data_len) + .map_err(|_| CudaError::LengthTooLarge { len: usize::MAX })?; + if data_len > job.output_capacity as usize { + return Err(CudaError::LengthTooLarge { len: data_len }); + } + let source_end = (job.output_offset as usize) + .checked_add(data_len) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + let job_output_end = (job.output_offset as usize) + .checked_add(job.output_capacity as usize) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if source_end > job_output_end { + return Err(CudaError::LengthTooLarge { len: source_end }); + } + compact_jobs.push(CudaHtj2kEncodeCompactJob { + source_offset: job.output_offset, + compact_offset: u32::try_from(compact_offset).map_err(|_| { + CudaError::LengthTooLarge { + len: compact_offset, + } + })?, + data_len: status.data_len, + reserved: status.reserved2, + }); + compact_offset = compact_offset + .checked_add(data_len) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if compact_offset > u32::MAX as usize { + return Err(CudaError::LengthTooLarge { + len: compact_offset, + }); + } + } + + Ok((compact_jobs, compact_offset)) +} + +pub(crate) fn validate_htj2k_encode_codeblock_shape( + width: u32, + height: u32, +) -> Result<(), CudaError> { + let samples = usize::try_from(width) + .ok() + .and_then(|w| usize::try_from(height).ok().and_then(|h| w.checked_mul(h))) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if width == 0 + || height == 0 + || width > HTJ2K_ENCODE_MAX_CODEBLOCK_WIDTH + || samples > HTJ2K_ENCODE_MAX_CODEBLOCK_SAMPLES + { + return Err(CudaError::InvalidArgument { + message: "HTJ2K encode code-block dimensions exceed CUDA kernel limits".to_string(), + }); + } + Ok(()) +} diff --git a/crates/j2k-cuda-runtime/src/htj2k_encode_kernels.cu b/crates/j2k-cuda-runtime/src/htj2k_encode_kernels.cu new file mode 100644 index 00000000..5bc48e76 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_encode_kernels.cu @@ -0,0 +1,2630 @@ +// SPDX-License-Identifier: Apache-2.0 + +#include + +typedef unsigned char uchar; +typedef unsigned short ushort; +typedef unsigned int uint; +typedef unsigned long long j2k_ulong; + +#ifndef min +#define min(a, b) ((a) < (b) ? (a) : (b)) +#endif +#ifndef max +#define max(a, b) ((a) > (b) ? (a) : (b)) +#endif + +__device__ inline uint j2k_unsigned_magnitude(int value) { + const uint bits = uint(value); + return value < 0 ? ~bits + 1u : bits; +} + +__device__ inline uint j2k_classic_magnitude(int value) { + return j2k_unsigned_magnitude(value); +} + +static constexpr uint J2K_ENCODE_STATUS_OK = 0u; +static constexpr uint J2K_ENCODE_STATUS_FAIL = 1u; +static constexpr uint J2K_ENCODE_STATUS_UNSUPPORTED = 2u; + +static constexpr uint J2K_HT_MAX_BITPLANES = 30u; +static constexpr uint J2K_HT_MEL_SIZE = 192u; +static constexpr uint J2K_HT_VLC_SIZE = 3072u - J2K_HT_MEL_SIZE; +static constexpr uint J2K_HT_MS_SIZE = ((16384u * 16u) + 14u) / 15u; +static constexpr uint J2K_HT_MEL_OFFSET = J2K_HT_MS_SIZE; +static constexpr uint J2K_HT_VLC_OFFSET = J2K_HT_MS_SIZE + J2K_HT_MEL_SIZE; +static constexpr uint J2K_HT_SIGPROP_SCRATCH = 513u; +static constexpr uint J2K_HT_MAX_CODEBLOCK_WIDTH = 1024u; +static constexpr uint J2K_HT_MAX_CODEBLOCK_SAMPLES = 4096u; +static constexpr uint J2K_HT_ENCODE_THREADS = 128u; +static constexpr uint J2K_HT_COMPACT_ASSEMBLE_FLAG = 0x80000000u; +static constexpr uint J2K_HT_COMPACT_LENGTH_MASK = 0x7FFFu; + +struct J2kHtEncodeParams { + uint width; + uint height; + uint coefficient_stride; + uint total_bitplanes; + uint output_capacity; + uint target_coding_passes; +}; + +struct J2kHtEncodeStatus { + uint code; + uint detail; + uint data_len; + uint num_coding_passes; + uint num_zero_bitplanes; + uint reserved0; + uint reserved1; + uint reserved2; +}; + +struct J2kHtMelEncoder { + uint pos; + uint remaining_bits; + uchar tmp; + uint run; + uint k; + uint threshold; + uint failed; +}; + +struct J2kHtVlcEncoder { + uint pos; + uint used_bits; + uchar tmp; + uint last_greater_than_8f; + uint failed; +}; + +struct J2kHtMagSgnEncoder { + uint pos; + uint max_bits; + uint used_bits; + uint tmp; + uint failed; +}; + +struct J2kHtSigPropWriter { + uint pos; + uint used_bits; + uint previous_was_ff; + uint capacity; + uchar tmp; + uint failed; +}; + +__device__ inline uint j2k_ht_mel_exp(uint k) { + return k < 3u ? 0u + : k < 6u ? 1u + : k < 9u ? 2u + : k < 11u ? 3u + : k == 11u ? 4u + : 5u; +} + +__device__ inline void j2k_set_ht_encode_status( + J2kHtEncodeStatus *status, + uint code, + uint detail, + uint data_len, + uint passes, + uint zbp +) { + status->code = code; + status->detail = detail; + status->data_len = data_len; + status->num_coding_passes = passes; + status->num_zero_bitplanes = zbp; + status->reserved0 = 0u; + status->reserved1 = 0u; + status->reserved2 = 0u; +} + +__device__ inline void j2k_set_ht_encode_status_with_segments( + J2kHtEncodeStatus *status, + uint code, + uint detail, + uint data_len, + uint passes, + uint zbp, + uint cleanup_len, + uint refinement_len, + uint reserved +) { + status->code = code; + status->detail = detail; + status->data_len = data_len; + status->num_coding_passes = passes; + status->num_zero_bitplanes = zbp; + status->reserved0 = cleanup_len; + status->reserved1 = refinement_len; + status->reserved2 = reserved; +} + +__device__ inline uint j2k_ht_pack_compact_assembly_lengths(uint mel_len, uint vlc_len) { + return J2K_HT_COMPACT_ASSEMBLE_FLAG + | (mel_len & J2K_HT_COMPACT_LENGTH_MASK) + | ((vlc_len & J2K_HT_COMPACT_LENGTH_MASK) << 15u); +} + +__device__ inline uint j2k_ht_aligned_sign_magnitude(int coefficient, uint total_bitplanes) { + if (coefficient == 0) { + return 0u; + } + const uint sign = coefficient < 0 ? 0x80000000u : 0u; + const uint magnitude = j2k_unsigned_magnitude(coefficient) << (31u - total_bitplanes); + return sign | magnitude; +} + +__device__ inline uint j2k_ht_sigprop_spread_mask(uint bit) { + switch (bit) { + case 0u: return 0x33u; + case 1u: return 0x76u; + case 2u: return 0xECu; + case 3u: return 0xC8u; + case 4u: return 0x330u; + case 5u: return 0x760u; + case 6u: return 0xEC0u; + case 7u: return 0xC80u; + case 8u: return 0x3300u; + case 9u: return 0x7600u; + case 10u: return 0xEC00u; + case 11u: return 0xC800u; + case 12u: return 0x33000u; + case 13u: return 0x76000u; + case 14u: return 0xEC000u; + case 15u: return 0xC8000u; + default: return 0u; + } +} + +__device__ inline void j2k_ht_sigprop_writer_init(J2kHtSigPropWriter &writer, uint capacity) { + writer.pos = 0u; + writer.used_bits = 0u; + writer.previous_was_ff = 0u; + writer.capacity = capacity; + writer.tmp = uchar(0u); + writer.failed = 0u; +} + +__device__ inline void j2k_ht_sigprop_write_bit( + J2kHtSigPropWriter &writer, + uchar *out, + uint bit +) { + const uint max_bits = writer.previous_was_ff != 0u ? 7u : 8u; + writer.tmp = uchar(uint(writer.tmp) | ((bit & 1u) << writer.used_bits)); + writer.used_bits += 1u; + if (writer.used_bits < max_bits) { + return; + } + + if (writer.pos >= writer.capacity) { + writer.failed = 1u; + return; + } + if (out != 0) { + out[writer.pos] = writer.tmp; + } + writer.previous_was_ff = writer.tmp == uchar(0xFFu) ? 1u : 0u; + writer.tmp = uchar(0u); + writer.used_bits = 0u; + writer.pos += 1u; +} + +__device__ inline void j2k_ht_sigprop_finish(J2kHtSigPropWriter &writer, uchar *out) { + if (writer.used_bits == 0u) { + return; + } + if (writer.pos >= writer.capacity) { + writer.failed = 1u; + return; + } + if (out != 0) { + out[writer.pos] = writer.tmp; + } + writer.pos += 1u; + writer.tmp = uchar(0u); + writer.used_bits = 0u; +} + +__device__ inline uint j2k_ht_sigprop_cleanup_sig16( + const int *coefficients, + uint coefficient_stride, + uint width, + uint height, + uint x_base, + uint y_base +) { + uint mask = 0u; + for (uint col = 0u; col < 4u; ++col) { + const uint x = x_base + col; + if (x >= width) { + continue; + } + for (uint row = 0u; row < 4u; ++row) { + const uint y = y_base + row; + if (y >= height) { + continue; + } + const uint magnitude = + j2k_classic_magnitude(coefficients[y * coefficient_stride + x]); + if (magnitude >= 5u && (magnitude & 1u) != 0u) { + mask |= 1u << (col * 4u + row); + } + } + } + return mask; +} + +__device__ inline uint j2k_ht_sigprop_target_sig16( + const int *coefficients, + uint coefficient_stride, + uint width, + uint height, + uint x_base, + uint y_base +) { + uint mask = 0u; + for (uint col = 0u; col < 4u; ++col) { + const uint x = x_base + col; + if (x >= width) { + continue; + } + for (uint row = 0u; row < 4u; ++row) { + const uint y = y_base + row; + if (y >= height) { + continue; + } + const uint magnitude = + j2k_classic_magnitude(coefficients[y * coefficient_stride + x]); + if (magnitude == 3u) { + mask |= 1u << (col * 4u + row); + } + } + } + return mask; +} + +__device__ inline uint j2k_ht_sigprop_coefficient_sign( + const int *coefficients, + uint coefficient_stride, + uint x_base, + uint y_base, + uint bit +) { + const uint col = bit >> 2u; + const uint row = bit & 3u; + return coefficients[(y_base + row) * coefficient_stride + x_base + col] < 0 ? 1u : 0u; +} + +__device__ inline uint j2k_ht_write_sigprop_segment( + const int *coefficients, + uint coefficient_stride, + uint width, + uint height, + uchar *out, + uint capacity, + uint *bytes_written +) { + const uint group_count = (width + 3u) >> 2u; + if (group_count + 8u > J2K_HT_SIGPROP_SCRATCH) { + return 0u; + } + + ushort prev_row_sig[J2K_HT_SIGPROP_SCRATCH]; + for (uint idx = 0u; idx < J2K_HT_SIGPROP_SCRATCH; ++idx) { + prev_row_sig[idx] = ushort(0u); + } + + J2kHtSigPropWriter writer; + j2k_ht_sigprop_writer_init(writer, capacity); + + for (uint y = 0u; y < height; y += 4u) { + uint pattern = 0xFFFFu; + if (height - y < 4u) { + pattern = 0x7777u; + if (height - y < 3u) { + pattern = 0x3333u; + if (height - y < 2u) { + pattern = 0x1111u; + } + } + } + + uint prev = 0u; + for (uint x = 0u; x < width; x += 4u) { + uint col_pattern = pattern; + const int overhang = int(x + 4u) - int(width); + if (overhang > 0) { + col_pattern >>= uint(overhang * 4); + } + + const uint idx = x >> 2u; + const uint ps = uint(prev_row_sig[idx]) | (uint(prev_row_sig[idx + 1u]) << 16u); + const uint ns = + j2k_ht_sigprop_cleanup_sig16(coefficients, coefficient_stride, width, height, x, y + 4u) + | (j2k_ht_sigprop_cleanup_sig16(coefficients, coefficient_stride, width, height, x + 4u, y + 4u) << 16u); + uint u = (ps & 0x88888888u) >> 3u; + u |= (ns & 0x11111111u) << 3u; + + const uint cs = + j2k_ht_sigprop_cleanup_sig16(coefficients, coefficient_stride, width, height, x, y) + | (j2k_ht_sigprop_cleanup_sig16(coefficients, coefficient_stride, width, height, x + 4u, y) << 16u); + uint mbr = cs; + mbr |= (cs & 0x77777777u) << 1u; + mbr |= (cs & 0xEEEEEEEEu) >> 1u; + mbr |= u; + const uint t_mbr = mbr; + mbr |= t_mbr << 4u; + mbr |= t_mbr >> 4u; + mbr |= prev >> 12u; + mbr &= col_pattern; + mbr &= ~cs; + + uint new_sig = 0u; + const uint target_sig = + j2k_ht_sigprop_target_sig16(coefficients, coefficient_stride, width, height, x, y) + & col_pattern; + if (mbr != 0u) { + uint candidates = mbr; + uint processed = 0u; + const uint inv_sig = ~cs & col_pattern; + + while (candidates != 0u) { + const uint bit = uint(__ffs(int(candidates)) - 1); + const uint sample_mask = 1u << bit; + candidates &= ~sample_mask; + processed |= sample_mask; + + const uint desired = (target_sig & sample_mask) != 0u ? 1u : 0u; + j2k_ht_sigprop_write_bit(writer, out, desired); + if (writer.failed != 0u) { + return 0u; + } + if (desired != 0u) { + new_sig |= sample_mask; + candidates |= j2k_ht_sigprop_spread_mask(bit) & inv_sig & ~processed; + } + } + + if (new_sig != 0u) { + uint sign_bits = new_sig; + while (sign_bits != 0u) { + const uint bit = uint(__ffs(int(sign_bits)) - 1); + const uint sample_mask = 1u << bit; + sign_bits &= ~sample_mask; + j2k_ht_sigprop_write_bit( + writer, + out, + j2k_ht_sigprop_coefficient_sign(coefficients, coefficient_stride, x, y, bit) + ); + if (writer.failed != 0u) { + return 0u; + } + } + } + } + + if ((target_sig & ~new_sig) != 0u) { + return 0u; + } + + const uint combined_sig = new_sig | cs; + prev_row_sig[idx] = ushort(combined_sig & 0xFFFFu); + prev_row_sig[idx + 1u] = ushort((combined_sig >> 16u) & 0xFFFFu); + + const uint t = combined_sig; + uint next_prev = combined_sig; + next_prev |= (t & 0x7777u) << 1u; + next_prev |= (t & 0xEEEEu) >> 1u; + prev = (next_prev | u) & 0xF000u; + } + } + + j2k_ht_sigprop_finish(writer, out); + if (writer.failed != 0u) { + return 0u; + } + bytes_written[0] = writer.pos; + return 1u; +} + +__device__ inline uint j2k_ht_write_magref_segment( + const int *coefficients, + uint coefficient_stride, + uint width, + uint height, + uchar *out, + uint magref_len, + uint expected_bits +) { + if (magref_len == 0u) { + return expected_bits == 0u; + } + + for (uint idx = 0u; idx < magref_len; ++idx) { + out[idx] = uchar(0u); + } + + uint bit_idx = 0u; + uint byte_from_end = 0u; + uint used_bits = 0u; + uint unstuff = 1u; + uchar current = uchar(0u); + for (uint y = 0u; y < height; y += 4u) { + for (uint x_base = 0u; x_base < width; x_base += 8u) { + for (uint col = 0u; col < 8u; ++col) { + const uint x = x_base + col; + if (x >= width) { + continue; + } + for (uint row = 0u; row < 4u; ++row) { + const uint yy = y + row; + if (yy >= height) { + continue; + } + + const uint magnitude = + j2k_classic_magnitude(coefficients[yy * coefficient_stride + x]); + if (magnitude < 5u || (magnitude & 1u) == 0u) { + continue; + } + + current = uchar(uint(current) | (((magnitude >> 1u) & 1u) << used_bits)); + used_bits += 1u; + bit_idx += 1u; + + const bool stuffed = + unstuff != 0u && used_bits == 7u && (current & uchar(0x7Fu)) == uchar(0x7Fu); + if (stuffed || used_bits == 8u) { + if (byte_from_end >= magref_len) { + return 0u; + } + out[magref_len - 1u - byte_from_end] = current; + byte_from_end += 1u; + unstuff = current > uchar(0x8Fu) ? 1u : 0u; + current = uchar(0u); + used_bits = 0u; + } + } + } + } + } + + if (used_bits != 0u) { + if (byte_from_end >= magref_len) { + return 0u; + } + out[magref_len - 1u - byte_from_end] = current; + byte_from_end += 1u; + } + + if (bit_idx != expected_bits || byte_from_end > magref_len) { + return 0u; + } + + return 1u; +} + +__device__ inline void j2k_ht_mel_init(J2kHtMelEncoder &mel) { + mel.pos = 0u; + mel.remaining_bits = 8u; + mel.tmp = uchar(0u); + mel.run = 0u; + mel.k = 0u; + mel.threshold = 1u; + mel.failed = 0u; +} + +__device__ inline void j2k_ht_vlc_init(J2kHtVlcEncoder &vlc, uchar *out) { + vlc.pos = 1u; + vlc.used_bits = 4u; + vlc.tmp = uchar(0x0Fu); + vlc.last_greater_than_8f = 1u; + vlc.failed = 0u; + out[J2K_HT_VLC_OFFSET + J2K_HT_VLC_SIZE - 1u] = uchar(0xFFu); +} + +__device__ inline void j2k_ht_ms_init(J2kHtMagSgnEncoder &ms) { + ms.pos = 0u; + ms.max_bits = 8u; + ms.used_bits = 0u; + ms.tmp = 0u; + ms.failed = 0u; +} + +__device__ inline uint j2k_ht_cleanup_scratch_entries(uint width) { + return min(((width + 1u) >> 1u) + 2u, J2K_HT_SIGPROP_SCRATCH); +} + +__device__ inline void j2k_ht_mel_emit_bit(J2kHtMelEncoder &mel, uchar *out, bool bit) { + mel.tmp = uchar((uint(mel.tmp) << 1u) | (bit ? 1u : 0u)); + mel.remaining_bits -= 1u; + if (mel.remaining_bits == 0u) { + if (mel.pos >= J2K_HT_MEL_SIZE) { + mel.failed = 1u; + return; + } + out[J2K_HT_MEL_OFFSET + mel.pos] = mel.tmp; + mel.pos += 1u; + mel.remaining_bits = mel.tmp == uchar(0xFFu) ? 7u : 8u; + mel.tmp = uchar(0u); + } +} + +__device__ inline void j2k_ht_mel_encode(J2kHtMelEncoder &mel, uchar *out, bool bit) { + if (!bit) { + mel.run += 1u; + if (mel.run >= mel.threshold) { + j2k_ht_mel_emit_bit(mel, out, true); + mel.run = 0u; + mel.k = min(mel.k + 1u, 12u); + mel.threshold = 1u << j2k_ht_mel_exp(mel.k); + } + } else { + j2k_ht_mel_emit_bit(mel, out, false); + uint t = j2k_ht_mel_exp(mel.k); + while (t > 0u) { + t -= 1u; + j2k_ht_mel_emit_bit(mel, out, ((mel.run >> t) & 1u) != 0u); + } + mel.run = 0u; + mel.k = mel.k == 0u ? 0u : mel.k - 1u; + mel.threshold = 1u << j2k_ht_mel_exp(mel.k); + } +} + +__device__ inline void j2k_ht_vlc_encode( + J2kHtVlcEncoder &vlc, + uchar *out, + uint codeword, + uint codeword_len +) { + while (codeword_len > 0u) { + if (vlc.pos >= J2K_HT_VLC_SIZE) { + vlc.failed = 1u; + return; + } + + uint available_bits = 8u - vlc.last_greater_than_8f - vlc.used_bits; + const uint take = min(available_bits, codeword_len); + const uint mask = take == 32u ? 0xFFFFFFFFu : ((1u << take) - 1u); + vlc.tmp = uchar(uint(vlc.tmp) | ((codeword & mask) << vlc.used_bits)); + vlc.used_bits += take; + available_bits -= take; + codeword_len -= take; + codeword >>= take; + + if (available_bits == 0u) { + if (vlc.last_greater_than_8f != 0u && vlc.tmp != uchar(0x7Fu)) { + vlc.last_greater_than_8f = 0u; + continue; + } + + const uint write_index = J2K_HT_VLC_SIZE - 1u - vlc.pos; + out[J2K_HT_VLC_OFFSET + write_index] = vlc.tmp; + vlc.pos += 1u; + vlc.last_greater_than_8f = vlc.tmp > uchar(0x8Fu) ? 1u : 0u; + vlc.tmp = uchar(0u); + vlc.used_bits = 0u; + } + } +} + +__device__ inline void j2k_ht_ms_encode( + J2kHtMagSgnEncoder &ms, + uchar *out, + uint codeword, + uint codeword_len +) { + while (codeword_len > 0u) { + if (ms.pos >= J2K_HT_MS_SIZE) { + ms.failed = 1u; + return; + } + + const uint take = min(ms.max_bits - ms.used_bits, codeword_len); + const uint mask = take == 32u ? 0xFFFFFFFFu : ((1u << take) - 1u); + ms.tmp |= (codeword & mask) << ms.used_bits; + ms.used_bits += take; + codeword >>= take; + codeword_len -= take; + + if (ms.used_bits >= ms.max_bits) { + out[ms.pos] = uchar(ms.tmp); + ms.pos += 1u; + ms.max_bits = ms.tmp == 0xFFu ? 7u : 8u; + ms.tmp = 0u; + ms.used_bits = 0u; + } + } +} + +__device__ inline void j2k_ht_ms_terminate(J2kHtMagSgnEncoder &ms, uchar *out) { + if (ms.used_bits > 0u) { + const uint unused = ms.max_bits - ms.used_bits; + ms.tmp |= (0xFFu & ((1u << unused) - 1u)) << ms.used_bits; + ms.used_bits += unused; + if (ms.tmp != 0xFFu) { + if (ms.pos >= J2K_HT_MS_SIZE) { + ms.failed = 1u; + return; + } + out[ms.pos] = uchar(ms.tmp); + ms.pos += 1u; + } + } else if (ms.max_bits == 7u) { + ms.pos = ms.pos == 0u ? 0u : ms.pos - 1u; + } +} + +__device__ inline void j2k_ht_process_sample( + uint slot, + uint value, + uint p, + int *rho_acc, + int *e_q, + int &e_qmax, + uint *s +) { + uint val = value + value; + val >>= p; + val &= ~1u; + if (val != 0u) { + rho_acc[0] |= int(1u << (slot & 0x3u)); + val -= 1u; + e_q[slot] = int(32u - __clz(val)); + e_qmax = max(e_qmax, e_q[slot]); + val -= 1u; + s[slot] = val + (value >> 31u); + } +} + +__device__ inline uchar j2k_ht_uvlc_byte(const uchar *table, uint index, uint field) { + return table[index * 6u + field]; +} + +__device__ inline void j2k_ht_encode_uvlc_pair( + J2kHtVlcEncoder &vlc, + uchar *out, + const uchar *uvlc_table, + uint first_index, + uint second_index +) { + const uchar first_pre = j2k_ht_uvlc_byte(uvlc_table, first_index, 0u); + const uchar first_pre_len = j2k_ht_uvlc_byte(uvlc_table, first_index, 1u); + const uchar first_suf = j2k_ht_uvlc_byte(uvlc_table, first_index, 2u); + const uchar first_suf_len = j2k_ht_uvlc_byte(uvlc_table, first_index, 3u); + const uchar second_pre = j2k_ht_uvlc_byte(uvlc_table, second_index, 0u); + const uchar second_pre_len = j2k_ht_uvlc_byte(uvlc_table, second_index, 1u); + const uchar second_suf = j2k_ht_uvlc_byte(uvlc_table, second_index, 2u); + const uchar second_suf_len = j2k_ht_uvlc_byte(uvlc_table, second_index, 3u); + j2k_ht_vlc_encode(vlc, out, uint(first_pre), uint(first_pre_len)); + j2k_ht_vlc_encode(vlc, out, uint(second_pre), uint(second_pre_len)); + j2k_ht_vlc_encode(vlc, out, uint(first_suf), uint(first_suf_len)); + j2k_ht_vlc_encode(vlc, out, uint(second_suf), uint(second_suf_len)); +} + +__device__ inline void j2k_ht_encode_uvlc( + int u_q0, + int u_q1, + J2kHtVlcEncoder &vlc, + uchar *out, + const uchar *uvlc_table +) { + if (u_q0 > 2 && u_q1 > 2) { + j2k_ht_encode_uvlc_pair(vlc, out, uvlc_table, uint(u_q0 - 2), uint(u_q1 - 2)); + } else if (u_q0 > 2 && u_q1 > 0) { + const uint first_index = uint(u_q0); + const uchar first_pre = j2k_ht_uvlc_byte(uvlc_table, first_index, 0u); + const uchar first_pre_len = j2k_ht_uvlc_byte(uvlc_table, first_index, 1u); + const uchar first_suf = j2k_ht_uvlc_byte(uvlc_table, first_index, 2u); + const uchar first_suf_len = j2k_ht_uvlc_byte(uvlc_table, first_index, 3u); + j2k_ht_vlc_encode(vlc, out, uint(first_pre), uint(first_pre_len)); + j2k_ht_vlc_encode(vlc, out, uint(u_q1 - 1), 1u); + j2k_ht_vlc_encode(vlc, out, uint(first_suf), uint(first_suf_len)); + } else { + j2k_ht_encode_uvlc_pair( + vlc, + out, + uvlc_table, + uint(max(u_q0, 0)), + uint(max(u_q1, 0)) + ); + } +} + +__device__ inline void j2k_ht_encode_uvlc_non_initial( + int u_q0, + int u_q1, + J2kHtVlcEncoder &vlc, + uchar *out, + const uchar *uvlc_table +) { + j2k_ht_encode_uvlc_pair( + vlc, + out, + uvlc_table, + uint(max(u_q0, 0)), + uint(max(u_q1, 0)) + ); +} + +__device__ inline void j2k_ht_encode_mag_signs( + int rho, + int u_q, + ushort tuple, + const uint *s, + uint offset, + J2kHtMagSgnEncoder &ms, + uchar *out +) { + const uint e_k = uint(tuple & ushort(0xFu)); + for (uint bit = 0u; bit < 4u; ++bit) { + const int sample_mask = int(1u << bit); + if ((rho & sample_mask) == 0) { + continue; + } + const int reduction = int((e_k >> bit) & 1u); + const uint magnitude_bits = uint(u_q - reduction); + const uint payload = magnitude_bits == 0u + ? 0u + : (s[offset + bit] & ((1u << magnitude_bits) - 1u)); + j2k_ht_ms_encode(ms, out, payload, magnitude_bits); + } +} + +__device__ inline int j2k_ht_encode_quad_initial_row( + uint offset, + uint c_q, + int rho, + int e_qmax, + const int *e_q, + const uint *s, + uint lep, + uint lcxp, + uchar *e_val, + uchar *cx_val, + J2kHtMelEncoder &mel, + J2kHtVlcEncoder &vlc, + J2kHtMagSgnEncoder &ms, + uchar *out, + const ushort *vlc_table0 +) { + const int u_q = max(e_qmax, 1) - 1; + uint eps = 0u; + if (u_q > 0) { + eps |= uint(e_q[offset] == e_qmax); + eps |= uint(e_q[offset + 1u] == e_qmax) << 1u; + eps |= uint(e_q[offset + 2u] == e_qmax) << 2u; + eps |= uint(e_q[offset + 3u] == e_qmax) << 3u; + } + + e_val[lep] = max(e_val[lep], uchar(e_q[offset + 1u])); + e_val[lep + 1u] = uchar(e_q[offset + 3u]); + cx_val[lcxp] = uchar(uint(cx_val[lcxp]) | uint((rho & 2) >> 1)); + cx_val[lcxp + 1u] = uchar((rho & 8) >> 3); + + const ushort tuple = vlc_table0[(c_q << 8u) | (uint(rho) << 4u) | eps]; + j2k_ht_vlc_encode(vlc, out, uint(tuple >> 8u), uint((tuple >> 4u) & ushort(0x7u))); + if (c_q == 0u) { + j2k_ht_mel_encode(mel, out, rho != 0); + } + j2k_ht_encode_mag_signs(rho, max(e_qmax, 1), tuple, s, offset, ms, out); + return u_q; +} + +__device__ inline int j2k_ht_encode_quad_non_initial_row( + uint offset, + uint c_q, + int rho, + int e_qmax, + int max_e, + const int *e_q, + const uint *s, + J2kHtMelEncoder &mel, + J2kHtVlcEncoder &vlc, + J2kHtMagSgnEncoder &ms, + uchar *out, + const ushort *vlc_table1 +) { + const int kappa = (rho & (rho - 1)) != 0 ? max(max_e, 1) : 1; + const int u_q = max(e_qmax, kappa) - kappa; + uint eps = 0u; + if (u_q > 0) { + eps |= uint(e_q[offset] == e_qmax); + eps |= uint(e_q[offset + 1u] == e_qmax) << 1u; + eps |= uint(e_q[offset + 2u] == e_qmax) << 2u; + eps |= uint(e_q[offset + 3u] == e_qmax) << 3u; + } + + const ushort tuple = vlc_table1[(c_q << 8u) | (uint(rho) << 4u) | eps]; + j2k_ht_vlc_encode(vlc, out, uint(tuple >> 8u), uint((tuple >> 4u) & ushort(0x7u))); + if (c_q == 0u) { + j2k_ht_mel_encode(mel, out, rho != 0); + } + j2k_ht_encode_mag_signs(rho, max(e_qmax, kappa), tuple, s, offset, ms, out); + return u_q; +} + +__device__ inline void j2k_ht_clear_quad_state(int *rho, int *e_q, int *e_qmax, uint *s) { + rho[0] = 0; + rho[1] = 0; + for (uint idx = 0u; idx < 8u; ++idx) { + e_q[idx] = 0; + s[idx] = 0u; + } + e_qmax[0] = 0; + e_qmax[1] = 0; +} + +template +__device__ inline int j2k_ht_encode_first_quad_pair( + const int *coefficients, + uint source_stride, + uint width, + uint height, + uint total_bitplanes, + uint p, + uint &sp, + uint x, + uchar *e_val, + uchar *cx_val, + uint &c_q0, + int *rho, + int *e_q, + int *e_qmax, + uint *s, + J2kHtMelEncoder &mel, + J2kHtVlcEncoder &vlc, + J2kHtMagSgnEncoder &ms, + uchar *out, + const ushort *vlc_table0, + const uchar *uvlc_table +) { + const uint lep = x / 2u; + const uint lcxp = x / 2u; + + j2k_ht_process_sample(0u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[0], e_q, e_qmax[0], s); + j2k_ht_process_sample( + 1u, + (FIXED_64 || height > 1u) + ? j2k_ht_aligned_sign_magnitude(coefficients[sp + source_stride], total_bitplanes) + : 0u, + p, + &rho[0], + e_q, + e_qmax[0], + s + ); + sp += 1u; + + if (FIXED_64 || x + 1u < width) { + j2k_ht_process_sample(2u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[0], e_q, e_qmax[0], s); + j2k_ht_process_sample( + 3u, + (FIXED_64 || height > 1u) + ? j2k_ht_aligned_sign_magnitude(coefficients[sp + source_stride], total_bitplanes) + : 0u, + p, + &rho[0], + e_q, + e_qmax[0], + s + ); + sp += 1u; + } + + const int u_q0 = j2k_ht_encode_quad_initial_row( + 0u, c_q0, rho[0], e_qmax[0], e_q, s, lep, lcxp, e_val, cx_val, mel, vlc, ms, out, vlc_table0 + ); + + if (FIXED_64 || x + 2u < width) { + j2k_ht_process_sample(4u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[1], e_q, e_qmax[1], s); + j2k_ht_process_sample( + 5u, + (FIXED_64 || height > 1u) + ? j2k_ht_aligned_sign_magnitude(coefficients[sp + source_stride], total_bitplanes) + : 0u, + p, + &rho[1], + e_q, + e_qmax[1], + s + ); + sp += 1u; + + if (FIXED_64 || x + 3u < width) { + j2k_ht_process_sample(6u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[1], e_q, e_qmax[1], s); + j2k_ht_process_sample( + 7u, + (FIXED_64 || height > 1u) + ? j2k_ht_aligned_sign_magnitude(coefficients[sp + source_stride], total_bitplanes) + : 0u, + p, + &rho[1], + e_q, + e_qmax[1], + s + ); + sp += 1u; + } + + const uint c_q1 = uint((rho[0] >> 1) | (rho[0] & 1)); + const int u_q1 = j2k_ht_encode_quad_initial_row( + 4u, c_q1, rho[1], e_qmax[1], e_q, s, lep + 1u, lcxp + 1u, e_val, cx_val, mel, vlc, ms, out, vlc_table0 + ); + + if (u_q0 > 0 && u_q1 > 0) { + j2k_ht_mel_encode(mel, out, min(u_q0, u_q1) > 2); + } + j2k_ht_encode_uvlc(u_q0, u_q1, vlc, out, uvlc_table); + c_q0 = uint((rho[1] >> 1) | (rho[1] & 1)); + } else { + j2k_ht_encode_uvlc(u_q0, 0, vlc, out, uvlc_table); + c_q0 = 0u; + } + + j2k_ht_clear_quad_state(rho, e_q, e_qmax, s); + return 0; +} + +template +__device__ inline int j2k_ht_encode_non_initial_quad_pair( + const int *coefficients, + uint stride, + uint width, + uint height, + uint y, + uint total_bitplanes, + uint p, + uint &sp, + uint x, + uchar *e_val, + uchar *cx_val, + uint &lep, + uint &lcxp, + int &max_e, + uint &c_q0, + int *rho, + int *e_q, + int *e_qmax, + uint *s, + J2kHtMelEncoder &mel, + J2kHtVlcEncoder &vlc, + J2kHtMagSgnEncoder &ms, + uchar *out, + const ushort *vlc_table1, + const uchar *uvlc_table +) { + j2k_ht_process_sample(0u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[0], e_q, e_qmax[0], s); + j2k_ht_process_sample( + 1u, + (FIXED_64 || y + 1u < height) + ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) + : 0u, + p, + &rho[0], + e_q, + e_qmax[0], + s + ); + sp += 1u; + + if (FIXED_64 || x + 1u < width) { + j2k_ht_process_sample(2u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[0], e_q, e_qmax[0], s); + j2k_ht_process_sample( + 3u, + (FIXED_64 || y + 1u < height) + ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) + : 0u, + p, + &rho[0], + e_q, + e_qmax[0], + s + ); + sp += 1u; + } + + const int prev_max = max_e; + const int u_q0 = j2k_ht_encode_quad_non_initial_row( + 0u, c_q0, rho[0], e_qmax[0], prev_max, e_q, s, mel, vlc, ms, out, vlc_table1 + ); + + e_val[lep] = max(e_val[lep], uchar(e_q[1])); + lep += 1u; + max_e = int(max(e_val[lep], e_val[lep + 1u])) - 1; + e_val[lep] = uchar(e_q[3]); + cx_val[lcxp] = uchar(uint(cx_val[lcxp]) | uint((rho[0] & 2) >> 1)); + lcxp += 1u; + uint c_q1 = uint(cx_val[lcxp]) + (uint(cx_val[lcxp + 1u]) << 2u); + cx_val[lcxp] = uchar((rho[0] & 8) >> 3); + + int u_q1 = 0; + if (FIXED_64 || x + 2u < width) { + j2k_ht_process_sample(4u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[1], e_q, e_qmax[1], s); + j2k_ht_process_sample( + 5u, + (FIXED_64 || y + 1u < height) + ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) + : 0u, + p, + &rho[1], + e_q, + e_qmax[1], + s + ); + sp += 1u; + + if (FIXED_64 || x + 3u < width) { + j2k_ht_process_sample(6u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[1], e_q, e_qmax[1], s); + j2k_ht_process_sample( + 7u, + (FIXED_64 || y + 1u < height) + ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) + : 0u, + p, + &rho[1], + e_q, + e_qmax[1], + s + ); + sp += 1u; + } + + c_q1 |= uint((rho[0] & 4) >> 1); + c_q1 |= uint((rho[0] & 8) >> 2); + u_q1 = j2k_ht_encode_quad_non_initial_row( + 4u, c_q1, rho[1], e_qmax[1], max_e, e_q, s, mel, vlc, ms, out, vlc_table1 + ); + + e_val[lep] = max(e_val[lep], uchar(e_q[5])); + lep += 1u; + max_e = int(max(e_val[lep], e_val[lep + 1u])) - 1; + e_val[lep] = uchar(e_q[7]); + cx_val[lcxp] = uchar(uint(cx_val[lcxp]) | uint((rho[1] & 2) >> 1)); + lcxp += 1u; + c_q0 = uint(cx_val[lcxp]) + (uint(cx_val[lcxp + 1u]) << 2u); + cx_val[lcxp] = uchar((rho[1] & 8) >> 3); + c_q0 |= uint((rho[1] & 4) >> 1); + c_q0 |= uint((rho[1] & 8) >> 2); + } else { + c_q0 = 0u; + } + + j2k_ht_encode_uvlc_non_initial(u_q0, u_q1, vlc, out, uvlc_table); + j2k_ht_clear_quad_state(rho, e_q, e_qmax, s); + return 0; +} + +__device__ inline void j2k_ht_terminate_mel_vlc( + J2kHtMelEncoder &mel, + J2kHtVlcEncoder &vlc, + uchar *out +) { + if (mel.run > 0u) { + j2k_ht_mel_emit_bit(mel, out, true); + } + + mel.tmp = uchar(uint(mel.tmp) << mel.remaining_bits); + const uchar mel_mask = uchar((0xFFu << mel.remaining_bits) & 0xFFu); + const uchar vlc_mask = vlc.used_bits == 0u + ? uchar(0u) + : uchar((1u << vlc.used_bits) - 1u); + + if ((mel_mask | vlc_mask) == uchar(0u)) { + return; + } + + const uchar fused = mel.tmp | vlc.tmp; + const bool fused_ok = + ((((fused ^ mel.tmp) & mel_mask) | ((fused ^ vlc.tmp) & vlc_mask)) == uchar(0u)) && + fused != uchar(0xFFu); + + if (fused_ok && vlc.pos > 1u) { + if (mel.pos >= J2K_HT_MEL_SIZE) { + mel.failed = 1u; + return; + } + out[J2K_HT_MEL_OFFSET + mel.pos] = fused; + mel.pos += 1u; + } else { + if (mel.pos >= J2K_HT_MEL_SIZE || vlc.pos >= J2K_HT_VLC_SIZE) { + mel.failed = 1u; + vlc.failed = 1u; + return; + } + out[J2K_HT_MEL_OFFSET + mel.pos] = mel.tmp; + mel.pos += 1u; + const uint write_index = J2K_HT_VLC_SIZE - 1u - vlc.pos; + out[J2K_HT_VLC_OFFSET + write_index] = vlc.tmp; + vlc.pos += 1u; + } +} + +template +__device__ inline void j2k_encode_ht_code_block_impl_with_max_and_assembly_t( + const int *coefficients, + uchar *out, + J2kHtEncodeParams params, + const ushort *vlc_table0, + const ushort *vlc_table1, + const uchar *uvlc_table, + J2kHtEncodeStatus *status, + uchar *e_val, + uchar *cx_val, + uint max_magnitude +) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u, 0u, 0u); + + if (params.width == 0u || params.height == 0u || params.coefficient_stride < params.width || + params.width > J2K_HT_MAX_CODEBLOCK_WIDTH || + params.height > J2K_HT_MAX_CODEBLOCK_SAMPLES / params.width || + params.total_bitplanes == 0u || params.total_bitplanes > J2K_HT_MAX_BITPLANES || + params.output_capacity < J2K_HT_MS_SIZE + J2K_HT_MEL_SIZE + J2K_HT_VLC_SIZE) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 1u, 0u, 0u, 0u); + return; + } + if (FIXED_64 && (params.width != 64u || params.height != 64u || params.coefficient_stride != 64u)) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 1u, 0u, 0u, 0u); + return; + } + if (CLEANUP_ONLY) { + if (params.target_coding_passes != 1u) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 5u, 0u, 0u, 0u); + return; + } + } else { + if (params.target_coding_passes == 0u || params.target_coding_passes > 164u) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 5u, 0u, 0u, 0u); + return; + } + if (params.target_coding_passes > 3u) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 5u, 0u, 0u, 0u); + return; + } + } + + if (max_magnitude == 0u) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_OK, 0u, 0u, 0u, params.total_bitplanes); + return; + } + + const uint block_bitplanes = 32u - __clz(max_magnitude); + if (block_bitplanes > params.total_bitplanes) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_FAIL, 2u, 0u, 0u, 0u); + return; + } + + uint significant_count = 0u; + if (!CLEANUP_ONLY && params.target_coding_passes > 1u) { + if (params.total_bitplanes < params.target_coding_passes) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 5u, 0u, 0u, 0u); + return; + } + } + if (!CLEANUP_ONLY && params.target_coding_passes == 2u) { + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint magnitude = + j2k_classic_magnitude(coefficients[y * params.coefficient_stride + x]); + if (magnitude == 0u) { + continue; + } + if (magnitude < 3u || (magnitude & 1u) == 0u) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 6u, 0u, 0u, 0u); + return; + } + } + } + } else if (!CLEANUP_ONLY && params.target_coding_passes == 3u) { + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint magnitude = + j2k_classic_magnitude(coefficients[y * params.coefficient_stride + x]); + if (magnitude == 0u || magnitude == 3u) { + continue; + } + significant_count += 1u; + if (magnitude < 5u || (magnitude & 1u) == 0u) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 6u, 0u, 0u, 0u); + return; + } + } + } + } + + const uint width = FIXED_64 ? 64u : params.width; + const uint height = FIXED_64 ? 64u : params.height; + const uint coefficient_stride = FIXED_64 ? 64u : params.coefficient_stride; + const uint pass_span = CLEANUP_ONLY ? 1u : params.target_coding_passes; + const uint missing_msbs = params.total_bitplanes - pass_span; + const uint p = 30u - missing_msbs; + + J2kHtMelEncoder mel; + J2kHtVlcEncoder vlc; + J2kHtMagSgnEncoder ms; + j2k_ht_mel_init(mel); + j2k_ht_vlc_init(vlc, out); + j2k_ht_ms_init(ms); + + const uint cleanup_scratch_entries = FIXED_64 ? 34u : j2k_ht_cleanup_scratch_entries(width); + for (uint idx = 0u; idx < cleanup_scratch_entries; ++idx) { + e_val[idx] = uchar(0u); + cx_val[idx] = uchar(0u); + } + + int e_qmax[2]; + int e_q[8]; + int rho[2]; + uint s[8]; + j2k_ht_clear_quad_state(rho, e_q, e_qmax, s); + + uint c_q0 = 0u; + uint sp = 0u; + uint x = 0u; + while (x < width) { + j2k_ht_encode_first_quad_pair( + coefficients, + coefficient_stride, + width, + height, + params.total_bitplanes, + p, + sp, + x, + e_val, + cx_val, + c_q0, + rho, + e_q, + e_qmax, + s, + mel, + vlc, + ms, + out, + vlc_table0, + uvlc_table + ); + x += 4u; + } + + const uint e_val_sentinel = FIXED_64 ? 33u : (width + 1u) / 2u + 1u; + if (e_val_sentinel < 513u) { + e_val[e_val_sentinel] = uchar(0u); + } + + uint y = 2u; + while (y < height) { + uint lep = 0u; + int max_e = int(max(e_val[lep], e_val[lep + 1u])) - 1; + e_val[lep] = uchar(0u); + + uint lcxp = 0u; + c_q0 = uint(cx_val[lcxp]) + (uint(cx_val[lcxp + 1u]) << 2u); + cx_val[lcxp] = uchar(0u); + + sp = y * coefficient_stride; + x = 0u; + while (x < width) { + j2k_ht_encode_non_initial_quad_pair( + coefficients, + coefficient_stride, + width, + height, + y, + params.total_bitplanes, + p, + sp, + x, + e_val, + cx_val, + lep, + lcxp, + max_e, + c_q0, + rho, + e_q, + e_qmax, + s, + mel, + vlc, + ms, + out, + vlc_table1, + uvlc_table + ); + x += 4u; + } + + y += 2u; + } + + j2k_ht_terminate_mel_vlc(mel, vlc, out); + j2k_ht_ms_terminate(ms, out); + + if (mel.failed != 0u || vlc.failed != 0u || ms.failed != 0u) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_FAIL, 3u, 0u, 0u, 0u); + return; + } + + const uint ms_len = ms.pos; + const uint mel_len = mel.pos; + const uint vlc_len = vlc.pos; + const uint cleanup_len = ms_len + mel_len + vlc_len; + uint sigprop_len = 0u; + uint magref_len = 0u; + uint refinement_len = 0u; + if (!CLEANUP_ONLY && params.target_coding_passes == 2u) { + refinement_len = 1u; + } else if (!CLEANUP_ONLY && params.target_coding_passes == 3u) { + const uint sample_count = width * height; + uint actual_sigprop_len = 0u; + if (j2k_ht_write_sigprop_segment( + coefficients, + coefficient_stride, + width, + height, + 0, + 0xFFFFFFFFu, + &actual_sigprop_len + ) == 0u) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 6u, 0u, 0u, 0u); + return; + } + sigprop_len = max((sample_count + 7u) >> 3u, actual_sigprop_len); + magref_len = (significant_count + 6u) / 7u; + refinement_len = sigprop_len + magref_len; + } + const uint total_len = cleanup_len + refinement_len; + if (cleanup_len < 2u || total_len > params.output_capacity) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_FAIL, 4u, 0u, 0u, 0u); + return; + } + + if (ASSEMBLE_FINAL) { + for (uint idx = 0u; idx < mel_len; ++idx) { + out[ms_len + idx] = out[J2K_HT_MEL_OFFSET + idx]; + } + const uint vlc_start = J2K_HT_VLC_SIZE - vlc_len; + for (uint idx = 0u; idx < vlc_len; ++idx) { + out[ms_len + mel_len + idx] = out[J2K_HT_VLC_OFFSET + vlc_start + idx]; + } + + const uint locator_bytes = mel_len + vlc_len; + const uint cleanup_last = cleanup_len - 1u; + const uint cleanup_prev = cleanup_len - 2u; + out[cleanup_last] = uchar(locator_bytes >> 4u); + out[cleanup_prev] = uchar((out[cleanup_prev] & uchar(0xF0u)) | uchar(locator_bytes & 0x0Fu)); + if (!CLEANUP_ONLY && refinement_len != 0u) { + for (uint idx = 0u; idx < refinement_len; ++idx) { + out[cleanup_len + idx] = uchar(0u); + } + } + if (!CLEANUP_ONLY && params.target_coding_passes == 3u) { + uint actual_sigprop_len = 0u; + if (j2k_ht_write_sigprop_segment( + coefficients, + coefficient_stride, + width, + height, + out + cleanup_len, + sigprop_len, + &actual_sigprop_len + ) == 0u) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 6u, 0u, 0u, 0u); + return; + } + } + if (!CLEANUP_ONLY + && params.target_coding_passes == 3u + && j2k_ht_write_magref_segment( + coefficients, + coefficient_stride, + width, + height, + out + cleanup_len + sigprop_len, + magref_len, + significant_count + ) == 0u) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 7u, 0u, 0u, 0u); + return; + } + } + + j2k_set_ht_encode_status_with_segments( + status, + J2K_ENCODE_STATUS_OK, + 0u, + total_len, + pass_span, + missing_msbs, + cleanup_len, + refinement_len, + ASSEMBLE_FINAL ? 0u : j2k_ht_pack_compact_assembly_lengths(mel_len, vlc_len) + ); +} + +__device__ inline void j2k_encode_ht_code_block_impl_with_max_and_assembly( + const int *coefficients, + uchar *out, + J2kHtEncodeParams params, + const ushort *vlc_table0, + const ushort *vlc_table1, + const uchar *uvlc_table, + J2kHtEncodeStatus *status, + uchar *e_val, + uchar *cx_val, + uint max_magnitude, + bool assemble_final +) { + if (assemble_final) { + j2k_encode_ht_code_block_impl_with_max_and_assembly_t( + coefficients, + out, + params, + vlc_table0, + vlc_table1, + uvlc_table, + status, + e_val, + cx_val, + max_magnitude + ); + } else { + j2k_encode_ht_code_block_impl_with_max_and_assembly_t( + coefficients, + out, + params, + vlc_table0, + vlc_table1, + uvlc_table, + status, + e_val, + cx_val, + max_magnitude + ); + } +} + +__device__ inline void j2k_encode_ht_code_block_impl_with_max( + const int *coefficients, + uchar *out, + J2kHtEncodeParams params, + const ushort *vlc_table0, + const ushort *vlc_table1, + const uchar *uvlc_table, + J2kHtEncodeStatus *status, + uint max_magnitude +) { + uchar e_val[J2K_HT_SIGPROP_SCRATCH]; + uchar cx_val[J2K_HT_SIGPROP_SCRATCH]; + j2k_encode_ht_code_block_impl_with_max_and_assembly( + coefficients, + out, + params, + vlc_table0, + vlc_table1, + uvlc_table, + status, + e_val, + cx_val, + max_magnitude, + true + ); +} + +__device__ inline void j2k_encode_ht_code_block_impl( + const int *coefficients, + uchar *out, + J2kHtEncodeParams params, + const ushort *vlc_table0, + const ushort *vlc_table1, + const uchar *uvlc_table, + J2kHtEncodeStatus *status +) { + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + max_magnitude = max(max_magnitude, j2k_classic_magnitude(coefficients[y * params.coefficient_stride + x])); + } + } + j2k_encode_ht_code_block_impl_with_max( + coefficients, + out, + params, + vlc_table0, + vlc_table1, + uvlc_table, + status, + max_magnitude + ); +} + +__device__ inline uint j2k_ht_reduce_max_magnitude_cooperative( + const int *coefficients, + uint width, + uint height, + uint coefficient_stride, + uint *block_max +) { + if (width == 0u || height == 0u) { + return 0u; + } + + const uint tid = threadIdx.x; + uint local_max = 0u; + const uint sample_count = width * height; + if (coefficient_stride == width) { + for (uint sample = tid; sample < sample_count; sample += blockDim.x) { + local_max = max(local_max, j2k_classic_magnitude(coefficients[sample])); + } + } else { + for (uint sample = tid; sample < sample_count; sample += blockDim.x) { + const uint y = sample / width; + const uint x = sample - y * width; + local_max = max( + local_max, + j2k_classic_magnitude(coefficients[y * coefficient_stride + x]) + ); + } + } + + block_max[tid] = local_max; + __syncthreads(); + + for (uint stride = blockDim.x >> 1u; stride > 0u; stride >>= 1u) { + if (tid < stride) { + block_max[tid] = max(block_max[tid], block_max[tid + stride]); + } + __syncthreads(); + } + + return block_max[0]; +} + +extern "C" __global__ void j2k_htj2k_encode_codeblock( + const int *coefficients, + uchar *out, + const J2kHtEncodeParams *params, + const ushort *vlc_table0, + const ushort *vlc_table1, + const uchar *uvlc_table, + J2kHtEncodeStatus *status +) { + if (blockIdx.x != 0u) { + return; + } + __shared__ uint block_max[J2K_HT_ENCODE_THREADS]; + __shared__ uchar cleanup_e_val[J2K_HT_SIGPROP_SCRATCH]; + __shared__ uchar cleanup_cx_val[J2K_HT_SIGPROP_SCRATCH]; + const J2kHtEncodeParams params_value = params[0]; + const uint max_magnitude = j2k_ht_reduce_max_magnitude_cooperative( + coefficients, + params_value.width, + params_value.height, + params_value.coefficient_stride, + block_max + ); + if (threadIdx.x != 0u) { + return; + } + j2k_encode_ht_code_block_impl_with_max_and_assembly( + coefficients, + out, + params_value, + vlc_table0, + vlc_table1, + uvlc_table, + status, + cleanup_e_val, + cleanup_cx_val, + max_magnitude, + true + ); +} + +struct J2kHtEncodeJob { + uint coefficient_offset; + uint coefficient_stride; + uint width; + uint height; + uint total_bitplanes; + uint output_offset; + uint output_capacity; + uint target_coding_passes; +}; + +struct J2kHtEncodeMultiInputJob { + j2k_ulong coefficient_ptr; + uint coefficient_offset; + uint coefficient_stride; + uint width; + uint height; + uint total_bitplanes; + uint output_offset; + uint output_capacity; + uint target_coding_passes; +}; + +struct J2kHtEncodeCompactJob { + uint source_offset; + uint compact_offset; + uint data_len; + uint reserved; +}; + +extern "C" __global__ void j2k_htj2k_encode_codeblocks( + const int *coefficients, + uchar *out, + const J2kHtEncodeJob *jobs, + const ushort *vlc_table0, + const ushort *vlc_table1, + const uchar *uvlc_table, + J2kHtEncodeStatus *statuses, + unsigned long long job_count +) { + const uint job_idx = blockIdx.x; + if (j2k_ulong(job_idx) >= job_count) { + return; + } + + const J2kHtEncodeJob job = jobs[job_idx]; + J2kHtEncodeParams params; + params.width = job.width; + params.height = job.height; + params.coefficient_stride = job.coefficient_stride; + params.total_bitplanes = job.total_bitplanes; + params.output_capacity = job.output_capacity; + params.target_coding_passes = job.target_coding_passes; + + __shared__ uint block_max[J2K_HT_ENCODE_THREADS]; + __shared__ uchar cleanup_e_val[J2K_HT_SIGPROP_SCRATCH]; + __shared__ uchar cleanup_cx_val[J2K_HT_SIGPROP_SCRATCH]; + const int *codeblock_coefficients = coefficients + job.coefficient_offset; + const uint max_magnitude = j2k_ht_reduce_max_magnitude_cooperative( + codeblock_coefficients, + params.width, + params.height, + params.coefficient_stride, + block_max + ); + if (threadIdx.x != 0u) { + return; + } + + j2k_encode_ht_code_block_impl_with_max_and_assembly( + codeblock_coefficients, + out + job.output_offset, + params, + vlc_table0, + vlc_table1, + uvlc_table, + &statuses[job_idx], + cleanup_e_val, + cleanup_cx_val, + max_magnitude, + params.target_coding_passes != 1u + ); +} + +extern "C" __global__ void __launch_bounds__(J2K_HT_ENCODE_THREADS) j2k_htj2k_encode_codeblocks_multi_input( + uchar *out, + const J2kHtEncodeMultiInputJob *jobs, + const ushort *vlc_table0, + const ushort *vlc_table1, + const uchar *uvlc_table, + J2kHtEncodeStatus *statuses, + unsigned long long job_count +) { + const uint job_idx = blockIdx.x; + if (j2k_ulong(job_idx) >= job_count) { + return; + } + + const J2kHtEncodeMultiInputJob job = jobs[job_idx]; + J2kHtEncodeParams params; + params.width = job.width; + params.height = job.height; + params.coefficient_stride = job.coefficient_stride; + params.total_bitplanes = job.total_bitplanes; + params.output_capacity = job.output_capacity; + params.target_coding_passes = job.target_coding_passes; + + __shared__ uint block_max[J2K_HT_ENCODE_THREADS]; + __shared__ uchar cleanup_e_val[J2K_HT_SIGPROP_SCRATCH]; + __shared__ uchar cleanup_cx_val[J2K_HT_SIGPROP_SCRATCH]; + const int *coefficients = reinterpret_cast(job.coefficient_ptr); + const int *codeblock_coefficients = coefficients + job.coefficient_offset; + const uint max_magnitude = j2k_ht_reduce_max_magnitude_cooperative( + codeblock_coefficients, + params.width, + params.height, + params.coefficient_stride, + block_max + ); + if (threadIdx.x != 0u) { + return; + } + + j2k_encode_ht_code_block_impl_with_max_and_assembly( + codeblock_coefficients, + out + job.output_offset, + params, + vlc_table0, + vlc_table1, + uvlc_table, + &statuses[job_idx], + cleanup_e_val, + cleanup_cx_val, + max_magnitude, + params.target_coding_passes != 1u + ); +} + +extern "C" __global__ void __launch_bounds__(J2K_HT_ENCODE_THREADS) j2k_htj2k_encode_codeblocks_multi_input_cleanup( + uchar *out, + const J2kHtEncodeMultiInputJob *jobs, + const ushort *vlc_table0, + const ushort *vlc_table1, + const uchar *uvlc_table, + J2kHtEncodeStatus *statuses, + unsigned long long job_count +) { + const uint job_idx = blockIdx.x; + if (j2k_ulong(job_idx) >= job_count) { + return; + } + + const J2kHtEncodeMultiInputJob job = jobs[job_idx]; + J2kHtEncodeParams params; + params.width = job.width; + params.height = job.height; + params.coefficient_stride = job.coefficient_stride; + params.total_bitplanes = job.total_bitplanes; + params.output_capacity = job.output_capacity; + params.target_coding_passes = job.target_coding_passes; + + __shared__ uint block_max[J2K_HT_ENCODE_THREADS]; + __shared__ uchar cleanup_e_val[J2K_HT_SIGPROP_SCRATCH]; + __shared__ uchar cleanup_cx_val[J2K_HT_SIGPROP_SCRATCH]; + const int *coefficients = reinterpret_cast(job.coefficient_ptr); + const int *codeblock_coefficients = coefficients + job.coefficient_offset; + const uint max_magnitude = j2k_ht_reduce_max_magnitude_cooperative( + codeblock_coefficients, + params.width, + params.height, + params.coefficient_stride, + block_max + ); + if (threadIdx.x != 0u) { + return; + } + + if (params.width == 64u && params.height == 64u && params.coefficient_stride == 64u) { + j2k_encode_ht_code_block_impl_with_max_and_assembly_t( + codeblock_coefficients, + out + job.output_offset, + params, + vlc_table0, + vlc_table1, + uvlc_table, + &statuses[job_idx], + cleanup_e_val, + cleanup_cx_val, + max_magnitude + ); + } else { + j2k_encode_ht_code_block_impl_with_max_and_assembly_t( + codeblock_coefficients, + out + job.output_offset, + params, + vlc_table0, + vlc_table1, + uvlc_table, + &statuses[job_idx], + cleanup_e_val, + cleanup_cx_val, + max_magnitude + ); + } +} + +extern "C" __global__ void __launch_bounds__(J2K_HT_ENCODE_THREADS) j2k_htj2k_encode_codeblocks_multi_input_cleanup_64( + uchar *out, + const J2kHtEncodeMultiInputJob *jobs, + const ushort *vlc_table0, + const ushort *vlc_table1, + const uchar *uvlc_table, + J2kHtEncodeStatus *statuses, + unsigned long long job_count +) { + const uint job_idx = blockIdx.x; + if (j2k_ulong(job_idx) >= job_count) { + return; + } + + const J2kHtEncodeMultiInputJob job = jobs[job_idx]; + J2kHtEncodeParams params; + params.width = job.width; + params.height = job.height; + params.coefficient_stride = job.coefficient_stride; + params.total_bitplanes = job.total_bitplanes; + params.output_capacity = job.output_capacity; + params.target_coding_passes = job.target_coding_passes; + + __shared__ uint block_max[J2K_HT_ENCODE_THREADS]; + __shared__ uchar cleanup_e_val[J2K_HT_SIGPROP_SCRATCH]; + __shared__ uchar cleanup_cx_val[J2K_HT_SIGPROP_SCRATCH]; + const int *coefficients = reinterpret_cast(job.coefficient_ptr); + const int *codeblock_coefficients = coefficients + job.coefficient_offset; + const uint max_magnitude = j2k_ht_reduce_max_magnitude_cooperative( + codeblock_coefficients, + 64u, + 64u, + 64u, + block_max + ); + if (threadIdx.x != 0u) { + return; + } + + j2k_encode_ht_code_block_impl_with_max_and_assembly_t( + codeblock_coefficients, + out + job.output_offset, + params, + vlc_table0, + vlc_table1, + uvlc_table, + &statuses[job_idx], + cleanup_e_val, + cleanup_cx_val, + max_magnitude + ); +} + +extern "C" __global__ void j2k_htj2k_compact_codeblocks( + const uchar *scratch, + uchar *compact, + const J2kHtEncodeCompactJob *jobs, + unsigned long long job_count +) { + const uint job_idx = blockIdx.x; + if (j2k_ulong(job_idx) >= job_count) { + return; + } + + const J2kHtEncodeCompactJob job = jobs[job_idx]; + if ((job.reserved & J2K_HT_COMPACT_ASSEMBLE_FLAG) != 0u) { + const uint mel_len = job.reserved & J2K_HT_COMPACT_LENGTH_MASK; + const uint vlc_len = (job.reserved >> 15u) & J2K_HT_COMPACT_LENGTH_MASK; + const uint locator_bytes = mel_len + vlc_len; + if (locator_bytes > job.data_len) { + return; + } + const uint ms_len = job.data_len - locator_bytes; + const uint vlc_start = J2K_HT_VLC_SIZE - vlc_len; + for (uint idx = threadIdx.x; idx < job.data_len; idx += blockDim.x) { + uchar value; + if (idx < ms_len) { + value = scratch[job.source_offset + idx]; + } else if (idx < ms_len + mel_len) { + value = scratch[job.source_offset + J2K_HT_MEL_OFFSET + idx - ms_len]; + } else { + value = scratch[ + job.source_offset + + J2K_HT_VLC_OFFSET + + vlc_start + + idx + - ms_len + - mel_len + ]; + } + if (job.data_len >= 2u) { + if (idx == job.data_len - 1u) { + value = uchar(locator_bytes >> 4u); + } else if (idx == job.data_len - 2u) { + value = uchar((uint(value) & 0xF0u) | (locator_bytes & 0x0Fu)); + } + } + compact[job.compact_offset + idx] = value; + } + return; + } + for (uint idx = threadIdx.x; idx < job.data_len; idx += blockDim.x) { + compact[job.compact_offset + idx] = scratch[job.source_offset + idx]; + } +} + +struct J2kHtPacketJob { + uint block_start; + uint block_count; + uint subband_start; + uint subband_count; + uint output_offset; + uint output_capacity; + uint layer; +}; + +struct J2kHtPacketSubband { + uint block_start; + uint block_count; + uint num_cbs_x; + uint num_cbs_y; +}; + +struct J2kHtPacketBlock { + uint data_offset; + uint data_len; + uint cleanup_length; + uint refinement_length; + uint num_coding_passes; + uint num_zero_bitplanes; + uint l_block; + uint previously_included; + uint inclusion_layer; +}; + +struct J2kHtPacketSubbandTagState { + uint inclusion_node_start; + uint zero_bitplane_node_start; + uint node_count; + uint reserved0; +}; + +struct J2kHtPacketTagNodeState { + uint current; + uint known; +}; + +struct J2kHtPacketStatus { + uint code; + uint detail; + uint output_len; + uint reserved0; +}; + +struct J2kPacketBitWriter { + uchar *out; + uint pos; + uint capacity; + uint buffer; + uint bits_in_buffer; + uint last_byte_was_ff; + uint failed; +}; + +static constexpr uint J2K_PACKET_TAG_INF = 0x7FFFFFFFu; +static constexpr uint J2K_PACKET_MAX_TAG_NODES = 2048u; +static constexpr uint J2K_PACKET_MAX_TAG_LEVELS = 16u; + +struct J2kPacketTagTree { + uint values[J2K_PACKET_MAX_TAG_NODES]; + uint current[J2K_PACKET_MAX_TAG_NODES]; + uint known[J2K_PACKET_MAX_TAG_NODES]; + uint widths[J2K_PACKET_MAX_TAG_LEVELS]; + uint heights[J2K_PACKET_MAX_TAG_LEVELS]; + uint offsets[J2K_PACKET_MAX_TAG_LEVELS]; + uint levels; + uint total_nodes; + uint failed; +}; + +__device__ inline void j2k_packet_status( + J2kHtPacketStatus *status, + uint code, + uint detail, + uint output_len +) { + status->code = code; + status->detail = detail; + status->output_len = output_len; + status->reserved0 = 0u; +} + +__device__ inline void j2k_packet_writer_init(J2kPacketBitWriter &writer, uchar *out, uint capacity) { + writer.out = out; + writer.pos = 0u; + writer.capacity = capacity; + writer.buffer = 0u; + writer.bits_in_buffer = 0u; + writer.last_byte_was_ff = 0u; + writer.failed = 0u; +} + +__device__ inline void j2k_packet_push_byte(J2kPacketBitWriter &writer, uchar byte) { + if (writer.pos >= writer.capacity) { + writer.failed = 1u; + return; + } + writer.out[writer.pos] = byte; + writer.pos += 1u; + writer.last_byte_was_ff = byte == uchar(0xFFu) ? 1u : 0u; +} + +__device__ inline void j2k_packet_flush_byte(J2kPacketBitWriter &writer) { + const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; + const uchar byte = uchar((writer.buffer >> (writer.bits_in_buffer - limit)) & 0xFFu); + j2k_packet_push_byte(writer, byte); + writer.bits_in_buffer -= limit; + writer.buffer &= writer.bits_in_buffer == 0u ? 0u : ((1u << writer.bits_in_buffer) - 1u); +} + +__device__ inline void j2k_packet_write_bit(J2kPacketBitWriter &writer, uint bit) { + writer.buffer = (writer.buffer << 1u) | (bit & 1u); + writer.bits_in_buffer += 1u; + const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; + if (writer.bits_in_buffer >= limit) { + j2k_packet_flush_byte(writer); + } +} + +__device__ inline void j2k_packet_write_bits( + J2kPacketBitWriter &writer, + uint value, + uint count +) { + while (count > 0u) { + count -= 1u; + j2k_packet_write_bit(writer, (value >> count) & 1u); + } +} + +__device__ inline void j2k_packet_finish(J2kPacketBitWriter &writer) { + if (writer.bits_in_buffer == 0u) { + return; + } + const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; + const uint shift = limit - writer.bits_in_buffer; + const uchar byte = uchar((writer.buffer << shift) & 0xFFu); + j2k_packet_push_byte(writer, byte); + writer.buffer = 0u; + writer.bits_in_buffer = 0u; +} + +__device__ inline void j2k_packet_encode_num_ht_passes( + J2kPacketBitWriter &writer, + uint num_passes +) { + if (num_passes == 1u) { + j2k_packet_write_bit(writer, 0u); + } else if (num_passes == 2u) { + j2k_packet_write_bits(writer, 0b10u, 2u); + } else if (num_passes <= 5u) { + j2k_packet_write_bits(writer, 0b11u, 2u); + j2k_packet_write_bits(writer, num_passes - 3u, 2u); + } else if (num_passes <= 36u) { + j2k_packet_write_bits(writer, 0b11u, 2u); + j2k_packet_write_bits(writer, 0b11u, 2u); + j2k_packet_write_bits(writer, num_passes - 6u, 5u); + } else { + j2k_packet_write_bits(writer, 0b11u, 2u); + j2k_packet_write_bits(writer, 0b11u, 2u); + j2k_packet_write_bits(writer, 31u, 5u); + j2k_packet_write_bits(writer, num_passes - 37u, 7u); + } +} + +__device__ inline uint j2k_packet_value_fits(uint value, uint bits) { + return bits >= 32u || value < (1u << bits); +} + +__device__ inline uint j2k_packet_ht_length_bits(uint l_block, uint num_passes) { + const uint placeholder_groups = (num_passes > 0u ? num_passes - 1u : 0u) / 3u; + const uint placeholder_passes = placeholder_groups * 3u; + uint value = placeholder_passes + 1u; + uint log2_value = 0u; + while (value > 1u) { + value >>= 1u; + log2_value += 1u; + } + return l_block + log2_value; +} + +__device__ inline void j2k_packet_encode_length( + J2kPacketBitWriter &writer, + uint length, + uint l_block, + uint num_bits +) { + while (!j2k_packet_value_fits(length, num_bits)) { + j2k_packet_write_bit(writer, 1u); + l_block += 1u; + num_bits += 1u; + } + j2k_packet_write_bit(writer, 0u); + j2k_packet_write_bits(writer, length, num_bits); +} + +__device__ inline void j2k_packet_encode_ht_segment_lengths( + J2kPacketBitWriter &writer, + J2kHtPacketBlock block +) { + const uint cleanup_length = + (block.num_coding_passes == 1u && block.cleanup_length == 0u) + ? block.data_len + : block.cleanup_length; + uint l_block = block.l_block; + uint cleanup_bits = j2k_packet_ht_length_bits(l_block, block.num_coding_passes); + const uint refinement_extra_bits = block.num_coding_passes > 2u ? 1u : 0u; + while (!j2k_packet_value_fits(cleanup_length, cleanup_bits) + || (block.num_coding_passes > 1u + && !j2k_packet_value_fits(block.refinement_length, l_block + refinement_extra_bits))) { + j2k_packet_write_bit(writer, 1u); + l_block += 1u; + cleanup_bits += 1u; + } + j2k_packet_write_bit(writer, 0u); + j2k_packet_write_bits(writer, cleanup_length, cleanup_bits); + + if (block.num_coding_passes > 1u) { + j2k_packet_write_bits(writer, block.refinement_length, l_block + refinement_extra_bits); + } +} + +__device__ inline uint j2k_packet_tag_tree_init( + J2kPacketTagTree &tree, + uint width, + uint height +) { + if (width == 0u || height == 0u) { + tree.failed = 1u; + return 0u; + } + + uint w = width; + uint h = height; + uint total = 0u; + uint levels = 0u; + while (true) { + if (levels >= J2K_PACKET_MAX_TAG_LEVELS) { + tree.failed = 1u; + return 0u; + } + const uint nodes = w * h; + if (w != 0u && nodes / w != h) { + tree.failed = 1u; + return 0u; + } + if (total + nodes < total || total + nodes > J2K_PACKET_MAX_TAG_NODES) { + tree.failed = 1u; + return 0u; + } + tree.widths[levels] = w; + tree.heights[levels] = h; + tree.offsets[levels] = total; + total += nodes; + levels += 1u; + if (w <= 1u && h <= 1u) { + break; + } + w = (w + 1u) >> 1u; + h = (h + 1u) >> 1u; + } + + tree.levels = levels; + tree.total_nodes = total; + tree.failed = 0u; + for (uint idx = 0u; idx < total; ++idx) { + tree.values[idx] = 0u; + tree.current[idx] = 0u; + tree.known[idx] = 0u; + } + return 1u; +} + +__device__ inline void j2k_packet_tag_tree_propagate(J2kPacketTagTree &tree) { + for (uint level = 1u; level < tree.levels; ++level) { + const uint prev_w = tree.widths[level - 1u]; + const uint prev_h = tree.heights[level - 1u]; + const uint curr_w = tree.widths[level]; + const uint curr_h = tree.heights[level]; + for (uint cy = 0u; cy < curr_h; ++cy) { + for (uint cx = 0u; cx < curr_w; ++cx) { + uint min_value = 0xFFFFFFFFu; + const uint child_x0 = cx << 1u; + const uint child_y0 = cy << 1u; + const uint child_x1 = min(child_x0 + 2u, prev_w); + const uint child_y1 = min(child_y0 + 2u, prev_h); + for (uint y = child_y0; y < child_y1; ++y) { + for (uint x = child_x0; x < child_x1; ++x) { + const uint child_idx = tree.offsets[level - 1u] + y * prev_w + x; + min_value = min(min_value, tree.values[child_idx]); + } + } + tree.values[tree.offsets[level] + cy * curr_w + cx] = min_value; + } + } + } +} + +__device__ inline void j2k_packet_build_tag_trees( + J2kPacketTagTree &inclusion_tree, + J2kPacketTagTree &zbp_tree, + const J2kHtPacketSubband &subband, + const J2kHtPacketBlock *blocks, + const J2kHtPacketSubbandTagState *tag_states, + const J2kHtPacketTagNodeState *tag_nodes, + j2k_ulong tag_state_count, + j2k_ulong tag_node_count, + uint subband_meta_idx +) { + if (j2k_packet_tag_tree_init(inclusion_tree, subband.num_cbs_x, subband.num_cbs_y) == 0u + || j2k_packet_tag_tree_init(zbp_tree, subband.num_cbs_x, subband.num_cbs_y) == 0u) { + inclusion_tree.failed = 1u; + zbp_tree.failed = 1u; + return; + } + + for (uint idx = 0u; idx < subband.block_count; ++idx) { + const J2kHtPacketBlock block = blocks[subband.block_start + idx]; + const uint x = idx % subband.num_cbs_x; + const uint y = idx / subband.num_cbs_x; + const uint leaf_idx = y * subband.num_cbs_x + x; + inclusion_tree.values[leaf_idx] = + block.previously_included == 0u + ? block.inclusion_layer + : J2K_PACKET_TAG_INF; + zbp_tree.values[leaf_idx] = block.num_zero_bitplanes; + } + j2k_packet_tag_tree_propagate(inclusion_tree); + j2k_packet_tag_tree_propagate(zbp_tree); + + if (tag_state_count == 0u) { + return; + } + if (j2k_ulong(subband_meta_idx) >= tag_state_count) { + inclusion_tree.failed = 1u; + zbp_tree.failed = 1u; + return; + } + const J2kHtPacketSubbandTagState state = tag_states[subband_meta_idx]; + if (state.node_count != inclusion_tree.total_nodes) { + inclusion_tree.failed = 1u; + zbp_tree.failed = 1u; + return; + } + const j2k_ulong inclusion_end = j2k_ulong(state.inclusion_node_start) + j2k_ulong(state.node_count); + const j2k_ulong zbp_end = j2k_ulong(state.zero_bitplane_node_start) + j2k_ulong(state.node_count); + if (inclusion_end < j2k_ulong(state.inclusion_node_start) + || zbp_end < j2k_ulong(state.zero_bitplane_node_start) + || inclusion_end > tag_node_count + || zbp_end > tag_node_count) { + inclusion_tree.failed = 1u; + zbp_tree.failed = 1u; + return; + } + for (uint idx = 0u; idx < state.node_count; ++idx) { + const J2kHtPacketTagNodeState inclusion_node = + tag_nodes[state.inclusion_node_start + idx]; + const J2kHtPacketTagNodeState zbp_node = + tag_nodes[state.zero_bitplane_node_start + idx]; + inclusion_tree.current[idx] = inclusion_node.current; + inclusion_tree.known[idx] = inclusion_node.known; + zbp_tree.current[idx] = zbp_node.current; + zbp_tree.known[idx] = zbp_node.known; + } +} + +__device__ inline void j2k_packet_tag_tree_encode( + J2kPacketTagTree &tree, + uint x, + uint y, + uint max_value, + J2kPacketBitWriter &writer +) { + uint path[J2K_PACKET_MAX_TAG_LEVELS]; + uint cx = x; + uint cy = y; + for (uint level = 0u; level < tree.levels; ++level) { + path[level] = tree.offsets[level] + cy * tree.widths[level] + cx; + cx >>= 1u; + cy >>= 1u; + } + + uint parent_value = 0u; + for (uint reverse = tree.levels; reverse > 0u; --reverse) { + const uint node_idx = path[reverse - 1u]; + uint start = max(tree.current[node_idx], parent_value); + if (tree.known[node_idx] == 0u) { + const uint target = min(tree.values[node_idx], max_value); + while (start < target) { + j2k_packet_write_bit(writer, 0u); + start += 1u; + } + if (tree.values[node_idx] < max_value) { + j2k_packet_write_bit(writer, 1u); + tree.known[node_idx] = 1u; + } + tree.current[node_idx] = target; + } + parent_value = tree.current[node_idx]; + } +} + +struct J2kPacketHeaderResult { + uint code; + uint detail; + uint header_len; + uint body_len; + uint output_len; +}; + +__device__ inline J2kPacketHeaderResult j2k_packet_header_result( + uint code, + uint detail, + uint header_len, + uint body_len, + uint output_len +) { + J2kPacketHeaderResult result; + result.code = code; + result.detail = detail; + result.header_len = header_len; + result.body_len = body_len; + result.output_len = output_len; + return result; +} + +__device__ inline J2kPacketHeaderResult j2k_packet_build_header_serial( + unsigned long long payload_len, + J2kHtPacketJob packet, + const J2kHtPacketSubband *subbands, + const J2kHtPacketBlock *blocks, + const J2kHtPacketSubbandTagState *tag_states, + const J2kHtPacketTagNodeState *tag_nodes, + unsigned long long tag_state_count, + unsigned long long tag_node_count, + uchar *packet_out +) { + J2kPacketBitWriter writer; + j2k_packet_writer_init(writer, packet_out, packet.output_capacity); + + uint any_data = 0u; + for (uint subband_idx = 0u; subband_idx < packet.subband_count; ++subband_idx) { + const J2kHtPacketSubband subband = subbands[packet.subband_start + subband_idx]; + if (subband.num_cbs_x == 0u + || subband.num_cbs_y == 0u + || subband.num_cbs_x * subband.num_cbs_y != subband.block_count) { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 7u, 0u, 0u, 0u); + } + for (uint idx = 0u; idx < subband.block_count; ++idx) { + const J2kHtPacketBlock block = blocks[subband.block_start + idx]; + if (block.num_coding_passes > 0u) { + any_data = 1u; + } + if (block.num_coding_passes > 164u) { + return j2k_packet_header_result( + J2K_ENCODE_STATUS_UNSUPPORTED, + 1u, + 0u, + 0u, + 0u + ); + } + if (block.data_offset + block.data_len < block.data_offset + || j2k_ulong(block.data_offset) + j2k_ulong(block.data_len) > payload_len) { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 2u, 0u, 0u, 0u); + } + if (block.num_coding_passes == 0u) { + if (block.data_len != 0u || block.cleanup_length != 0u || block.refinement_length != 0u) { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 10u, 0u, 0u, 0u); + } + } else if (block.num_coding_passes == 1u) { + const uint cleanup_length = + block.cleanup_length == 0u ? block.data_len : block.cleanup_length; + if (cleanup_length != block.data_len || block.refinement_length != 0u) { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 11u, 0u, 0u, 0u); + } + } else { + const j2k_ulong segment_len = + j2k_ulong(block.cleanup_length) + j2k_ulong(block.refinement_length); + if (block.cleanup_length == 0u + || block.refinement_length == 0u + || segment_len != j2k_ulong(block.data_len) + || block.cleanup_length < 2u + || block.cleanup_length >= 65535u + || block.refinement_length >= 2047u) { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 12u, 0u, 0u, 0u); + } + } + } + } + + if (any_data == 0u) { + j2k_packet_write_bit(writer, 0u); + j2k_packet_finish(writer); + if (writer.failed != 0u) { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 3u, 0u, 0u, 0u); + } + return j2k_packet_header_result( + J2K_ENCODE_STATUS_OK, + 0u, + writer.pos, + 0u, + writer.pos + ); + } + + j2k_packet_write_bit(writer, 1u); + for (uint subband_idx = 0u; subband_idx < packet.subband_count; ++subband_idx) { + const uint subband_meta_idx = packet.subband_start + subband_idx; + const J2kHtPacketSubband subband = subbands[subband_meta_idx]; + J2kPacketTagTree inclusion_tree; + J2kPacketTagTree zbp_tree; + j2k_packet_build_tag_trees( + inclusion_tree, + zbp_tree, + subband, + blocks, + tag_states, + tag_nodes, + tag_state_count, + tag_node_count, + subband_meta_idx + ); + if (inclusion_tree.failed != 0u || zbp_tree.failed != 0u) { + return j2k_packet_header_result( + J2K_ENCODE_STATUS_UNSUPPORTED, + 8u, + 0u, + 0u, + 0u + ); + } + + for (uint idx = 0u; idx < subband.block_count; ++idx) { + const J2kHtPacketBlock block = blocks[subband.block_start + idx]; + const uint x = idx % subband.num_cbs_x; + const uint y = idx / subband.num_cbs_x; + if (block.previously_included == 0u) { + if (block.num_coding_passes > 0u && block.inclusion_layer != packet.layer) { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 9u, 0u, 0u, 0u); + } + j2k_packet_tag_tree_encode(inclusion_tree, x, y, packet.layer + 1u, writer); + if (block.num_coding_passes == 0u) { + continue; + } + j2k_packet_tag_tree_encode(zbp_tree, x, y, block.num_zero_bitplanes + 1u, writer); + } else if (block.num_coding_passes > 0u) { + j2k_packet_write_bit(writer, 1u); + } else { + j2k_packet_write_bit(writer, 0u); + continue; + } + j2k_packet_encode_num_ht_passes(writer, block.num_coding_passes); + j2k_packet_encode_ht_segment_lengths(writer, block); + } + } + + j2k_packet_finish(writer); + if (writer.failed != 0u) { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 4u, 0u, 0u, 0u); + } + if (writer.pos > 0u && packet_out[writer.pos - 1u] == uchar(0xFFu)) { + if (writer.pos >= packet.output_capacity) { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 5u, 0u, 0u, 0u); + } + packet_out[writer.pos] = uchar(0u); + writer.pos += 1u; + } + + const uint header_len = writer.pos; + uint body_len = 0u; + for (uint subband_idx = 0u; subband_idx < packet.subband_count; ++subband_idx) { + const J2kHtPacketSubband subband = subbands[packet.subband_start + subband_idx]; + for (uint idx = 0u; idx < subband.block_count; ++idx) { + const J2kHtPacketBlock block = blocks[subband.block_start + idx]; + if (block.num_coding_passes == 0u || block.data_len == 0u) { + continue; + } + if (body_len + block.data_len < body_len + || header_len + body_len + block.data_len < header_len + || header_len + body_len + block.data_len > packet.output_capacity) { + return j2k_packet_header_result(J2K_ENCODE_STATUS_FAIL, 6u, 0u, 0u, 0u); + } + body_len += block.data_len; + } + } + + return j2k_packet_header_result( + J2K_ENCODE_STATUS_OK, + 0u, + header_len, + body_len, + header_len + body_len + ); +} + +__device__ inline void j2k_packet_copy_body_cooperative( + const uchar *payload, + J2kHtPacketJob packet, + const J2kHtPacketSubband *subbands, + const J2kHtPacketBlock *blocks, + uchar *packet_out, + uint header_len, + uint body_len +) { + for (uint body_byte = threadIdx.x; body_byte < body_len; body_byte += blockDim.x) { + uint remaining = body_byte; + for (uint subband_idx = 0u; subband_idx < packet.subband_count; ++subband_idx) { + const J2kHtPacketSubband subband = subbands[packet.subband_start + subband_idx]; + for (uint idx = 0u; idx < subband.block_count; ++idx) { + const J2kHtPacketBlock block = blocks[subband.block_start + idx]; + if (block.num_coding_passes == 0u || block.data_len == 0u) { + continue; + } + if (remaining < block.data_len) { + packet_out[header_len + body_byte] = payload[block.data_offset + remaining]; + subband_idx = packet.subband_count; + break; + } + remaining -= block.data_len; + } + } + } +} + +extern "C" __global__ void j2k_htj2k_packetize_cleanup( + const uchar *payload, + unsigned long long payload_len, + const J2kHtPacketJob *packets, + const J2kHtPacketSubband *subbands, + const J2kHtPacketBlock *blocks, + const J2kHtPacketSubbandTagState *tag_states, + const J2kHtPacketTagNodeState *tag_nodes, + unsigned long long tag_state_count, + unsigned long long tag_node_count, + uchar *out, + J2kHtPacketStatus *statuses, + unsigned long long packet_count +) { + const unsigned long long packet_idx = blockIdx.x; + if (packet_idx >= packet_count) { + return; + } + + const J2kHtPacketJob packet = packets[packet_idx]; + J2kHtPacketStatus *status = &statuses[packet_idx]; + uchar *packet_out = out + packet.output_offset; + __shared__ uint shared_code; + __shared__ uint shared_header_len; + __shared__ uint shared_body_len; + + if (threadIdx.x == 0u) { + const J2kPacketHeaderResult result = j2k_packet_build_header_serial( + payload_len, + packet, + subbands, + blocks, + tag_states, + tag_nodes, + tag_state_count, + tag_node_count, + packet_out + ); + shared_code = result.code; + shared_header_len = result.header_len; + shared_body_len = result.body_len; + j2k_packet_status(status, result.code, result.detail, result.output_len); + } + __syncthreads(); + + if (shared_code != J2K_ENCODE_STATUS_OK || shared_body_len == 0u) { + return; + } + j2k_packet_copy_body_cooperative( + payload, + packet, + subbands, + blocks, + packet_out, + shared_header_len, + shared_body_len + ); +} diff --git a/crates/j2k-cuda-runtime/src/htj2k_encode_kernels.ptx b/crates/j2k-cuda-runtime/src/htj2k_encode_kernels.ptx new file mode 100644 index 00000000..1ffa20b5 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_encode_kernels.ptx @@ -0,0 +1,65201 @@ +// +// Generated by NVIDIA NVVM Compiler +// +// Compiler Build ID: UNKNOWN +// Cuda compilation tools, release 13.2, V13.2.78 +// Based on NVVM 7.0.1 +// + +.version 9.2 +.target sm_75 +.address_size 64 + + // .globl j2k_htj2k_encode_codeblock +// _ZZ31 j2k_htj2k_encode_codeblockE9block_max has been demoted +// _ZZ31 j2k_htj2k_encode_codeblockE13cleanup_e_val has been demoted +// _ZZ31 j2k_htj2k_encode_codeblockE14cleanup_cx_val has been demoted +// _ZZ32 j2k_htj2k_encode_codeblocksE9block_max has been demoted +// _ZZ32 j2k_htj2k_encode_codeblocksE13cleanup_e_val has been demoted +// _ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val has been demoted +// _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE9block_max has been demoted +// _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE13cleanup_e_val has been demoted +// _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val has been demoted +// _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE9block_max has been demoted +// _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val has been demoted +// _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val has been demoted +// _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E9block_max has been demoted +// _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val has been demoted +// _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val has been demoted +// _ZZ32 j2k_htj2k_packetize_cleanupE11shared_code has been demoted +// _ZZ32 j2k_htj2k_packetize_cleanupE17shared_header_len has been demoted +// _ZZ32 j2k_htj2k_packetize_cleanupE15shared_body_len has been demoted + +.visible .entry j2k_htj2k_encode_codeblock( + .param .u64 j2k_htj2k_encode_codeblock_param_0, + .param .u64 j2k_htj2k_encode_codeblock_param_1, + .param .u64 j2k_htj2k_encode_codeblock_param_2, + .param .u64 j2k_htj2k_encode_codeblock_param_3, + .param .u64 j2k_htj2k_encode_codeblock_param_4, + .param .u64 j2k_htj2k_encode_codeblock_param_5, + .param .u64 j2k_htj2k_encode_codeblock_param_6 +) +{ + .local .align 2 .b8 __local_depot0[1026]; + .reg .b64 %SP; + .reg .b64 %SPL; + .reg .pred %p<1636>; + .reg .b16 %rs<766>; + .reg .b32 %r<6486>; + .reg .b64 %rd<640>; + // demoted variable + .shared .align 4 .b8 _ZZ31 j2k_htj2k_encode_codeblockE9block_max[512]; + // demoted variable + .shared .align 1 .b8 _ZZ31 j2k_htj2k_encode_codeblockE13cleanup_e_val[513]; + // demoted variable + .shared .align 1 .b8 _ZZ31 j2k_htj2k_encode_codeblockE14cleanup_cx_val[513]; + + mov.u64 %SPL, __local_depot0; + ld.param.u64 %rd61, [ j2k_htj2k_encode_codeblock_param_0]; + ld.param.u64 %rd55, [ j2k_htj2k_encode_codeblock_param_1]; + ld.param.u64 %rd56, [ j2k_htj2k_encode_codeblock_param_2]; + ld.param.u64 %rd57, [ j2k_htj2k_encode_codeblock_param_3]; + ld.param.u64 %rd59, [ j2k_htj2k_encode_codeblock_param_5]; + ld.param.u64 %rd60, [ j2k_htj2k_encode_codeblock_param_6]; + cvta.to.global.u64 %rd1, %rd55; + cvta.to.global.u64 %rd2, %rd61; + mov.u32 %r2352, %ctaid.x; + setp.ne.s32 %p8, %r2352, 0; + @%p8 bra $L__BB0_1254; + + cvta.to.global.u64 %rd62, %rd56; + ld.global.u32 %r1, [%rd62+8]; + ld.global.u32 %r2, [%rd62+12]; + ld.global.u32 %r3, [%rd62+16]; + ld.global.u32 %r4, [%rd62+20]; + ld.global.u32 %r5, [%rd62]; + setp.eq.s32 %p9, %r5, 0; + ld.global.u32 %r6, [%rd62+4]; + setp.eq.s32 %p10, %r6, 0; + or.pred %p11, %p9, %p10; + @%p11 bra $L__BB0_14; + bra.uni $L__BB0_2; + +$L__BB0_14: + mov.u32 %r5172, 0; + bra.uni $L__BB0_15; + +$L__BB0_2: + mov.u32 %r7, %tid.x; + mul.lo.s32 %r8, %r6, %r5; + setp.eq.s32 %p12, %r1, %r5; + @%p12 bra $L__BB0_6; + bra.uni $L__BB0_3; + +$L__BB0_6: + setp.ge.u32 %p15, %r7, %r8; + mov.u32 %r5170, 0; + @%p15 bra $L__BB0_9; + + mov.u32 %r5170, 0; + mov.u32 %r15, %ntid.x; + mov.u32 %r5168, %r7; + +$L__BB0_8: + mul.wide.u32 %rd65, %r5168, 4; + add.s64 %rd66, %rd2, %rd65; + ld.global.u32 %r2361, [%rd66]; + abs.s32 %r2362, %r2361; + max.u32 %r5170, %r5170, %r2362; + add.s32 %r5168, %r5168, %r15; + setp.lt.u32 %p16, %r5168, %r8; + @%p16 bra $L__BB0_8; + bra.uni $L__BB0_9; + +$L__BB0_3: + setp.ge.u32 %p13, %r7, %r8; + mov.u32 %r5170, 0; + @%p13 bra $L__BB0_9; + + sub.s32 %r9, %r1, %r5; + mov.u32 %r5170, 0; + mov.u32 %r10, %ntid.x; + mov.u32 %r5166, %r7; + +$L__BB0_5: + div.u32 %r2355, %r5166, %r5; + mad.lo.s32 %r2356, %r9, %r2355, %r5166; + mul.wide.u32 %rd63, %r2356, 4; + add.s64 %rd64, %rd2, %rd63; + ld.global.u32 %r2357, [%rd64]; + abs.s32 %r2358, %r2357; + max.u32 %r5170, %r5170, %r2358; + add.s32 %r5166, %r5166, %r10; + setp.lt.u32 %p14, %r5166, %r8; + @%p14 bra $L__BB0_5; + +$L__BB0_9: + shl.b32 %r2363, %r7, 2; + mov.u32 %r2364, _ZZ31 j2k_htj2k_encode_codeblockE9block_max; + add.s32 %r21, %r2364, %r2363; + st.shared.u32 [%r21], %r5170; + bar.sync 0; + mov.u32 %r2365, %ntid.x; + shr.u32 %r5171, %r2365, 1; + setp.eq.s32 %p17, %r5171, 0; + @%p17 bra $L__BB0_13; + +$L__BB0_10: + setp.ge.u32 %p18, %r7, %r5171; + @%p18 bra $L__BB0_12; + + ld.shared.u32 %r2366, [%r21]; + shl.b32 %r2367, %r5171, 2; + add.s32 %r2368, %r21, %r2367; + ld.shared.u32 %r2369, [%r2368]; + setp.gt.u32 %p19, %r2366, %r2369; + add.s32 %r2370, %r5171, %r7; + selp.b32 %r2371, %r7, %r2370, %p19; + shl.b32 %r2372, %r2371, 2; + add.s32 %r2374, %r2364, %r2372; + ld.shared.u32 %r2375, [%r2374]; + st.shared.u32 [%r21], %r2375; + +$L__BB0_12: + bar.sync 0; + shr.u32 %r5171, %r5171, 1; + setp.ne.s32 %p20, %r5171, 0; + @%p20 bra $L__BB0_10; + +$L__BB0_13: + ld.shared.u32 %r5172, [_ZZ31 j2k_htj2k_encode_codeblockE9block_max]; + +$L__BB0_15: + mov.u32 %r2377, %tid.x; + setp.ne.s32 %p21, %r2377, 0; + @%p21 bra $L__BB0_1254; + + mov.u32 %r2378, 0; + cvta.to.global.u64 %rd3, %rd60; + mov.u32 %r2379, 1; + st.global.u32 [%rd3], %r2379; + st.global.u32 [%rd3+4], %r2378; + st.global.u32 [%rd3+8], %r2378; + st.global.u32 [%rd3+12], %r2378; + st.global.u32 [%rd3+16], %r2378; + st.global.u32 [%rd3+20], %r2378; + st.global.u32 [%rd3+24], %r2378; + st.global.u32 [%rd3+28], %r2378; + add.s32 %r2380, %r5, -1; + setp.ge.u32 %p23, %r2380, %r1; + or.pred %p24, %p10, %p23; + setp.gt.u32 %p25, %r5, 1024; + or.pred %p26, %p25, %p24; + @%p26 bra $L__BB0_1253; + + cvt.u16.u32 %rs271, %r5; + mov.u16 %rs272, 4096; + div.u16 %rs273, %rs272, %rs271; + cvt.u32.u16 %r2381, %rs273; + setp.gt.u32 %p27, %r6, %r2381; + add.s32 %r2382, %r2, -1; + setp.gt.u32 %p28, %r2382, 29; + or.pred %p29, %p28, %p27; + setp.lt.u32 %p30, %r3, 20549; + or.pred %p31, %p30, %p29; + @%p31 bra $L__BB0_1253; + bra.uni $L__BB0_18; + +$L__BB0_1253: + mov.u32 %r5102, 2; + st.global.u32 [%rd3], %r5102; + st.global.u32 [%rd3+4], %r2379; + st.global.u32 [%rd3+8], %r2378; + st.global.u32 [%rd3+12], %r2378; + st.global.u32 [%rd3+16], %r2378; + st.global.u32 [%rd3+20], %r2378; + st.global.u32 [%rd3+24], %r2378; + st.global.u32 [%rd3+28], %r2378; + +$L__BB0_1254: + ret; + +$L__BB0_18: + add.s32 %r2383, %r4, -1; + setp.gt.u32 %p32, %r2383, 163; + @%p32 bra $L__BB0_1252; + bra.uni $L__BB0_19; + +$L__BB0_1252: + mov.u32 %r5099, 2; + st.global.u32 [%rd3], %r5099; + mov.u32 %r5100, 5; + st.global.u32 [%rd3+4], %r5100; + mov.u32 %r5101, 0; + st.global.u32 [%rd3+8], %r5101; + st.global.u32 [%rd3+12], %r5101; + st.global.u32 [%rd3+16], %r5101; + st.global.u32 [%rd3+20], %r5101; + st.global.u32 [%rd3+24], %r5101; + st.global.u32 [%rd3+28], %r5101; + bra.uni $L__BB0_1254; + +$L__BB0_19: + setp.gt.u32 %p33, %r4, 3; + @%p33 bra $L__BB0_1251; + bra.uni $L__BB0_20; + +$L__BB0_1251: + mov.u32 %r5096, 2; + st.global.u32 [%rd3], %r5096; + mov.u32 %r5097, 5; + st.global.u32 [%rd3+4], %r5097; + mov.u32 %r5098, 0; + st.global.u32 [%rd3+8], %r5098; + st.global.u32 [%rd3+12], %r5098; + st.global.u32 [%rd3+16], %r5098; + st.global.u32 [%rd3+20], %r5098; + st.global.u32 [%rd3+24], %r5098; + st.global.u32 [%rd3+28], %r5098; + bra.uni $L__BB0_1254; + +$L__BB0_20: + setp.eq.s32 %p34, %r5172, 0; + @%p34 bra $L__BB0_1250; + + clz.b32 %r2384, %r5172; + mov.u32 %r2385, 32; + sub.s32 %r2386, %r2385, %r2384; + setp.gt.u32 %p35, %r2386, %r2; + @%p35 bra $L__BB0_1249; + bra.uni $L__BB0_22; + +$L__BB0_1249: + mov.u32 %r5092, 1; + st.global.u32 [%rd3], %r5092; + mov.u32 %r5093, 2; + st.global.u32 [%rd3+4], %r5093; + mov.u32 %r5094, 0; + st.global.u32 [%rd3+8], %r5094; + st.global.u32 [%rd3+12], %r5094; + st.global.u32 [%rd3+16], %r5094; + st.global.u32 [%rd3+20], %r5094; + st.global.u32 [%rd3+24], %r5094; + st.global.u32 [%rd3+28], %r5094; + bra.uni $L__BB0_1254; + +$L__BB0_1250: + mov.u32 %r5095, 0; + st.global.u32 [%rd3], %r5095; + st.global.u32 [%rd3+4], %r5095; + st.global.u32 [%rd3+8], %r5095; + st.global.u32 [%rd3+12], %r5095; + st.global.u32 [%rd3+16], %r2; + st.global.u32 [%rd3+20], %r5095; + st.global.u32 [%rd3+24], %r5095; + st.global.u32 [%rd3+28], %r5095; + bra.uni $L__BB0_1254; + +$L__BB0_22: + setp.gt.u32 %p36, %r4, 1; + setp.lt.u32 %p37, %r2, %r4; + and.pred %p38, %p36, %p37; + @%p38 bra $L__BB0_1248; + bra.uni $L__BB0_23; + +$L__BB0_1248: + mov.u32 %r5089, 2; + st.global.u32 [%rd3], %r5089; + mov.u32 %r5090, 5; + st.global.u32 [%rd3+4], %r5090; + mov.u32 %r5091, 0; + st.global.u32 [%rd3+8], %r5091; + st.global.u32 [%rd3+12], %r5091; + st.global.u32 [%rd3+16], %r5091; + st.global.u32 [%rd3+20], %r5091; + st.global.u32 [%rd3+24], %r5091; + st.global.u32 [%rd3+28], %r5091; + bra.uni $L__BB0_1254; + +$L__BB0_23: + mov.u32 %r5180, 0; + setp.eq.s32 %p39, %r4, 2; + @%p39 bra $L__BB0_34; + + setp.ne.s32 %p40, %r4, 3; + @%p40 bra $L__BB0_42; + + @%p9 bra $L__BB0_42; + + mov.u32 %r2390, 0; + mov.u32 %r5173, %r2390; + mov.u32 %r5180, %r2390; + +$L__BB0_27: + mul.lo.s32 %r29, %r5173, %r1; + mov.u32 %r5175, %r2390; + +$L__BB0_28: + add.s32 %r2392, %r5175, %r29; + mul.wide.u32 %rd67, %r2392, 4; + add.s64 %rd68, %rd2, %rd67; + ld.global.u32 %r2393, [%rd68]; + abs.s32 %r32, %r2393; + setp.eq.s32 %p42, %r32, 0; + @%p42 bra $L__BB0_31; + + setp.eq.s32 %p43, %r32, 3; + @%p43 bra $L__BB0_31; + + add.s32 %r5180, %r5180, 1; + and.b32 %r2394, %r32, 1; + setp.eq.b32 %p44, %r2394, 1; + not.pred %p45, %p44; + setp.lt.u32 %p46, %r32, 5; + or.pred %p47, %p46, %p45; + @%p47 bra $L__BB0_33; + +$L__BB0_31: + add.s32 %r5175, %r5175, 1; + setp.lt.u32 %p48, %r5175, %r5; + @%p48 bra $L__BB0_28; + + add.s32 %r5173, %r5173, 1; + setp.lt.u32 %p49, %r5173, %r6; + @%p49 bra $L__BB0_27; + bra.uni $L__BB0_42; + +$L__BB0_34: + @%p9 bra $L__BB0_42; + + mov.u32 %r2399, 0; + mov.u32 %r5178, %r2399; + +$L__BB0_36: + mul.lo.s32 %r38, %r5178, %r1; + mov.u32 %r5179, %r2399; + +$L__BB0_37: + add.s32 %r2401, %r5179, %r38; + mul.wide.u32 %rd69, %r2401, 4; + add.s64 %rd70, %rd2, %rd69; + ld.global.u32 %r2402, [%rd70]; + abs.s32 %r40, %r2402; + setp.eq.s32 %p51, %r40, 0; + @%p51 bra $L__BB0_40; + + setp.gt.u32 %p52, %r40, 2; + and.b32 %r2403, %r40, 1; + setp.eq.b32 %p53, %r2403, 1; + and.pred %p54, %p52, %p53; + @%p54 bra $L__BB0_40; + bra.uni $L__BB0_39; + +$L__BB0_40: + add.s32 %r5179, %r5179, 1; + setp.lt.u32 %p55, %r5179, %r5; + @%p55 bra $L__BB0_37; + + add.s32 %r5178, %r5178, 1; + setp.lt.u32 %p56, %r5178, %r6; + mov.u32 %r5180, 0; + @%p56 bra $L__BB0_36; + +$L__BB0_42: + sub.s32 %r44, %r2, %r4; + mov.u32 %r2409, 30; + sub.s32 %r45, %r2409, %r44; + mov.u16 %rs274, 255; + st.global.u8 [%rd1+20548], %rs274; + add.s32 %r2410, %r5, 1; + shr.u32 %r2411, %r2410, 1; + add.s32 %r2412, %r2411, 2; + min.u32 %r46, %r2412, 513; + mov.u32 %r2413, -3; + sub.s32 %r2414, %r2413, %r2411; + max.u32 %r2415, %r2414, -514; + mov.u32 %r2416, -2; + sub.s32 %r2417, %r2416, %r2415; + and.b32 %r5185, %r46, 3; + setp.lt.u32 %p57, %r2417, 3; + mov.u32 %r5183, 0; + @%p57 bra $L__BB0_45; + + sub.s32 %r5182, %r46, %r5185; + mov.u32 %r5183, 0; + +$L__BB0_44: + mov.u32 %r2419, _ZZ31 j2k_htj2k_encode_codeblockE13cleanup_e_val; + add.s32 %r2420, %r2419, %r5183; + mov.u16 %rs275, 0; + st.shared.u8 [%r2420], %rs275; + mov.u32 %r2421, _ZZ31 j2k_htj2k_encode_codeblockE14cleanup_cx_val; + add.s32 %r2422, %r2421, %r5183; + st.shared.u8 [%r2422], %rs275; + st.shared.u8 [%r2420+1], %rs275; + st.shared.u8 [%r2422+1], %rs275; + st.shared.u8 [%r2420+2], %rs275; + st.shared.u8 [%r2422+2], %rs275; + st.shared.u8 [%r2420+3], %rs275; + st.shared.u8 [%r2422+3], %rs275; + add.s32 %r5183, %r5183, 4; + add.s32 %r5182, %r5182, -4; + setp.ne.s32 %p58, %r5182, 0; + @%p58 bra $L__BB0_44; + +$L__BB0_45: + setp.eq.s32 %p59, %r5185, 0; + @%p59 bra $L__BB0_48; + + mov.u32 %r2423, _ZZ31 j2k_htj2k_encode_codeblockE13cleanup_e_val; + mov.u32 %r2425, _ZZ31 j2k_htj2k_encode_codeblockE14cleanup_cx_val; + +$L__BB0_47: + .pragma "nounroll"; + add.s32 %r2424, %r2423, %r5183; + mov.u16 %rs276, 0; + st.shared.u8 [%r2424], %rs276; + add.s32 %r2426, %r2425, %r5183; + st.shared.u8 [%r2426], %rs276; + add.s32 %r5183, %r5183, 1; + add.s32 %r5185, %r5185, -1; + setp.ne.s32 %p60, %r5185, 0; + @%p60 bra $L__BB0_47; + +$L__BB0_48: + mov.u32 %r5480, 0; + mov.u32 %r5277, 8; + mov.u32 %r5481, 1; + mov.u32 %r5718, 4; + mov.u16 %rs688, 15; + mov.u16 %rs705, 0; + mov.u32 %r5482, %r5480; + mov.u32 %r5483, %r5480; + mov.u32 %r5271, %r5480; + mov.u32 %r5716, %r5480; + mov.u32 %r5717, %r5481; + mov.u32 %r5719, %r5481; + mov.u32 %r5933, %r5480; + mov.u32 %r5904, %r5480; + mov.u32 %r5905, %r5480; + mov.u32 %r5906, %r5277; + mov.u32 %r5907, %r5480; + @%p9 bra $L__BB0_417; + + mov.u32 %r2459, 0; + mov.u32 %r5719, 1; + mov.u16 %rs705, 0; + mov.u32 %r5906, 8; + mov.u16 %rs688, 15; + mov.u32 %r5718, 4; + cvta.to.global.u64 %rd79, %rd57; + cvta.to.global.u64 %rd108, %rd59; + mov.u32 %r5186, %r2459; + mov.u32 %r5187, %r2459; + mov.u32 %r5736, %r2459; + mov.u32 %r5907, %r2459; + mov.u32 %r5905, %r2459; + mov.u32 %r5904, %r2459; + mov.u32 %r5933, %r2459; + mov.u32 %r5717, %r5719; + mov.u32 %r5716, %r2459; + mov.u32 %r5271, %r2459; + mov.u32 %r5277, %r5906; + mov.u32 %r5483, %r2459; + mov.u32 %r5482, %r2459; + mov.u32 %r5481, %r5719; + mov.u32 %r5480, %r2459; + bra.uni $L__BB0_50; + +$L__BB0_39: + mov.u32 %r2404, 2; + st.global.u32 [%rd3], %r2404; + mov.u32 %r2405, 6; + st.global.u32 [%rd3+4], %r2405; + mov.u32 %r2406, 0; + st.global.u32 [%rd3+8], %r2406; + st.global.u32 [%rd3+12], %r2406; + st.global.u32 [%rd3+16], %r2406; + st.global.u32 [%rd3+20], %r2406; + st.global.u32 [%rd3+24], %r2406; + st.global.u32 [%rd3+28], %r2406; + bra.uni $L__BB0_1254; + +$L__BB0_33: + mov.u32 %r2395, 2; + st.global.u32 [%rd3], %r2395; + mov.u32 %r2396, 6; + st.global.u32 [%rd3+4], %r2396; + mov.u32 %r2397, 0; + st.global.u32 [%rd3+8], %r2397; + st.global.u32 [%rd3+12], %r2397; + st.global.u32 [%rd3+16], %r2397; + st.global.u32 [%rd3+20], %r2397; + st.global.u32 [%rd3+24], %r2397; + st.global.u32 [%rd3+28], %r2397; + bra.uni $L__BB0_1254; + +$L__BB0_255: + shl.b16 %rs600, %rs705, 1; + or.b16 %rs646, %rs600, 1; + setp.gt.u32 %p291, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5476, 1; + @%p291 bra $L__BB0_257; + + shl.b16 %rs602, %rs705, 1; + or.b16 %rs601, %rs602, 1; + and.b16 %rs364, %rs601, 255; + st.global.u8 [%rd6], %rs601; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p292, %rs364, 255; + selp.b32 %r5277, 7, 8, %p292; + mov.u16 %rs646, 0; + mov.u32 %r5476, %r5480; + bra.uni $L__BB0_257; + +$L__BB0_50: + mul.wide.u32 %rd71, %r5187, 4; + add.s64 %rd72, %rd2, %rd71; + ld.global.u32 %r76, [%rd72]; + setp.eq.s32 %p62, %r76, 0; + mov.u32 %r5204, %r2459; + @%p62 bra $L__BB0_52; + + and.b32 %r2461, %r76, -2147483648; + abs.s32 %r2462, %r76; + mov.u32 %r2463, 31; + sub.s32 %r2464, %r2463, %r2; + shl.b32 %r2465, %r2462, %r2464; + or.b32 %r5204, %r2465, %r2461; + +$L__BB0_52: + shl.b32 %r2469, %r5204, 1; + shr.u32 %r2470, %r2469, %r45; + and.b32 %r79, %r2470, -2; + setp.eq.s32 %p63, %r79, 0; + mov.u32 %r5205, 0; + mov.u32 %r5206, %r5205; + mov.u32 %r5212, %r5205; + @%p63 bra $L__BB0_54; + + add.s32 %r2472, %r79, -1; + clz.b32 %r2473, %r2472; + mov.u32 %r2474, 32; + sub.s32 %r5205, %r2474, %r2473; + shr.u32 %r2475, %r5204, 31; + add.s32 %r2476, %r2475, %r79; + add.s32 %r5206, %r2476, -2; + mov.u32 %r5212, 1; + +$L__BB0_54: + mov.u32 %r5208, 0; + setp.lt.u32 %p64, %r6, 2; + @%p64 bra $L__BB0_57; + + mov.u32 %r5208, 0; + add.s32 %r2479, %r5187, %r1; + mul.wide.u32 %rd73, %r2479, 4; + add.s64 %rd74, %rd2, %rd73; + ld.global.u32 %r85, [%rd74]; + setp.eq.s32 %p65, %r85, 0; + @%p65 bra $L__BB0_57; + + and.b32 %r2480, %r85, -2147483648; + abs.s32 %r2481, %r85; + mov.u32 %r2482, 31; + sub.s32 %r2483, %r2482, %r2; + shl.b32 %r2484, %r2481, %r2483; + or.b32 %r5208, %r2484, %r2480; + +$L__BB0_57: + shl.b32 %r2487, %r5208, 1; + shr.u32 %r2488, %r2487, %r45; + and.b32 %r88, %r2488, -2; + setp.eq.s32 %p66, %r88, 0; + mov.u32 %r5223, 0; + mov.u32 %r5209, %r5223; + mov.u32 %r5210, %r5223; + mov.u32 %r5227, %r5205; + @%p66 bra $L__BB0_59; + + or.b32 %r5212, %r5212, 2; + add.s32 %r2489, %r88, -1; + clz.b32 %r2490, %r2489; + mov.u32 %r2491, 32; + sub.s32 %r5209, %r2491, %r2490; + max.s32 %r5227, %r5205, %r5209; + shr.u32 %r2492, %r5208, 31; + add.s32 %r2493, %r2492, %r88; + add.s32 %r5210, %r2493, -2; + +$L__BB0_59: + add.s32 %r5229, %r5187, 1; + add.s32 %r2498, %r5186, 1; + setp.ge.u32 %p67, %r2498, %r5; + mov.u32 %r5224, %r5223; + mov.u32 %r5225, %r5223; + mov.u32 %r5226, %r5223; + @%p67 bra $L__BB0_70; + + mul.wide.u32 %rd75, %r5229, 4; + add.s64 %rd76, %rd2, %rd75; + ld.global.u32 %r98, [%rd76]; + setp.eq.s32 %p68, %r98, 0; + mov.u32 %r5224, 0; + mov.u32 %r5213, %r5224; + @%p68 bra $L__BB0_62; + + and.b32 %r2500, %r98, -2147483648; + abs.s32 %r2501, %r98; + mov.u32 %r2502, 31; + sub.s32 %r2503, %r2502, %r2; + shl.b32 %r2504, %r2501, %r2503; + or.b32 %r5213, %r2504, %r2500; + +$L__BB0_62: + shl.b32 %r2507, %r5213, 1; + shr.u32 %r2508, %r2507, %r45; + and.b32 %r101, %r2508, -2; + setp.eq.s32 %p69, %r101, 0; + mov.u32 %r5226, %r5224; + @%p69 bra $L__BB0_64; + + or.b32 %r5212, %r5212, 4; + add.s32 %r2509, %r101, -1; + clz.b32 %r2510, %r2509; + mov.u32 %r2511, 32; + sub.s32 %r5224, %r2511, %r2510; + max.s32 %r5227, %r5227, %r5224; + shr.u32 %r2512, %r5213, 31; + add.s32 %r2513, %r2512, %r101; + add.s32 %r5226, %r2513, -2; + +$L__BB0_64: + mov.u32 %r5223, 0; + mov.u32 %r5218, %r5223; + @%p64 bra $L__BB0_67; + + add.s32 %r5118, %r5187, 1; + add.s32 %r2516, %r5118, %r1; + mul.wide.u32 %rd77, %r2516, 4; + add.s64 %rd78, %rd2, %rd77; + ld.global.u32 %r110, [%rd78]; + setp.eq.s32 %p71, %r110, 0; + @%p71 bra $L__BB0_67; + + and.b32 %r2517, %r110, -2147483648; + abs.s32 %r2518, %r110; + mov.u32 %r2519, 31; + sub.s32 %r2520, %r2519, %r2; + shl.b32 %r2521, %r2518, %r2520; + or.b32 %r5218, %r2521, %r2517; + +$L__BB0_67: + shl.b32 %r2524, %r5218, 1; + shr.u32 %r2525, %r2524, %r45; + and.b32 %r113, %r2525, -2; + setp.eq.s32 %p72, %r113, 0; + mov.u32 %r5225, %r5223; + @%p72 bra $L__BB0_69; + + or.b32 %r5212, %r5212, 8; + add.s32 %r2526, %r113, -1; + clz.b32 %r2527, %r2526; + mov.u32 %r2528, 32; + sub.s32 %r5223, %r2528, %r2527; + max.s32 %r5227, %r5227, %r5223; + shr.u32 %r2529, %r5218, 31; + add.s32 %r2530, %r2529, %r113; + add.s32 %r5225, %r2530, -2; + +$L__BB0_69: + add.s32 %r5229, %r5187, 2; + +$L__BB0_70: + mov.u32 %r5187, %r5229; + add.s32 %r2532, %r5227, -1; + setp.lt.s32 %p73, %r5227, 2; + setp.gt.s32 %p74, %r5227, 1; + selp.b32 %r130, %r2532, 0, %p74; + mov.u32 %r5230, 0; + @%p73 bra $L__BB0_72; + + setp.eq.s32 %p75, %r5205, %r5227; + selp.u32 %r2533, 1, 0, %p75; + setp.eq.s32 %p76, %r5209, %r5227; + selp.u32 %r2534, -1, 0, %p76; + bfi.b32 %r2535, %r2534, %r2533, 1, 1; + setp.eq.s32 %p77, %r5224, %r5227; + selp.u16 %rs281, 1, 0, %p77; + mul.wide.u16 %r2536, %rs281, 4; + or.b32 %r2537, %r2535, %r2536; + setp.eq.s32 %p78, %r5223, %r5227; + selp.u16 %rs282, 1, 0, %p78; + mul.wide.u16 %r2538, %rs282, 8; + or.b32 %r5230, %r2537, %r2538; + +$L__BB0_72: + shr.u32 %r2539, %r5186, 1; + mov.u32 %r2540, _ZZ31 j2k_htj2k_encode_codeblockE13cleanup_e_val; + add.s32 %r2541, %r2540, %r2539; + ld.shared.u8 %rs283, [%r2541]; + cvt.u32.u16 %r2542, %rs283; + and.b32 %r2543, %r2542, 255; + and.b32 %r2544, %r5209, 255; + setp.lt.u32 %p79, %r2544, %r2543; + cvt.u16.u32 %rs284, %r5209; + selp.b16 %rs285, %rs283, %rs284, %p79; + st.shared.u8 [%r2541], %rs285; + st.shared.u8 [%r2541+1], %r5223; + mov.u32 %r2545, _ZZ31 j2k_htj2k_encode_codeblockE14cleanup_cx_val; + add.s32 %r2546, %r2545, %r2539; + and.b32 %r133, %r5212, 2; + cvt.u16.u32 %rs286, %r133; + shr.u16 %rs287, %rs286, 1; + ld.shared.u8 %rs288, [%r2546]; + or.b16 %rs289, %rs288, %rs287; + st.shared.u8 [%r2546], %rs289; + and.b32 %r134, %r5212, 8; + shr.u32 %r135, %r134, 3; + st.shared.u8 [%r2546+1], %r135; + shl.b32 %r2547, %r5212, 4; + shl.b32 %r2548, %r5736, 8; + or.b32 %r2549, %r2547, %r2548; + or.b32 %r2550, %r2549, %r5230; + mul.wide.u32 %rd80, %r2550, 2; + add.s64 %rd81, %rd79, %rd80; + ld.global.u16 %rs3, [%rd81]; + shr.u16 %rs290, %rs3, 4; + and.b16 %rs4, %rs290, 7; + setp.eq.s16 %p80, %rs4, 0; + mov.u32 %r5242, %r5716; + @%p80 bra $L__BB0_79; + + cvt.u32.u16 %r5231, %rs4; + shr.u16 %rs291, %rs3, 8; + cvt.u32.u16 %r5232, %rs291; + +$L__BB0_74: + mov.u16 %rs5, %rs688; + mov.u32 %r138, %r5231; + setp.gt.u32 %p81, %r5719, 2879; + mov.u32 %r5242, 1; + @%p81 bra $L__BB0_79; + + mov.u32 %r2552, 8; + sub.s32 %r2553, %r2552, %r5717; + sub.s32 %r2554, %r2553, %r5718; + min.u32 %r2555, %r2554, %r138; + setp.eq.s32 %p82, %r2555, 32; + mov.u32 %r2556, -1; + shl.b32 %r2557, %r2556, %r2555; + not.b32 %r2558, %r2557; + selp.b32 %r2559, -1, %r2558, %p82; + and.b32 %r2560, %r2559, %r5232; + shl.b32 %r2561, %r2560, %r5718; + cvt.u16.u32 %rs292, %r2561; + or.b16 %rs688, %rs5, %rs292; + add.s32 %r5718, %r2555, %r5718; + sub.s32 %r5231, %r138, %r2555; + shr.u32 %r5232, %r5232, %r2555; + setp.gt.u32 %p83, %r2554, %r138; + @%p83 bra $L__BB0_78; + + setp.ne.s32 %p84, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs293, %rs688, 255; + setp.ne.s16 %p85, %rs293, 127; + and.pred %p86, %p84, %p85; + @%p86 bra $L__BB0_78; + + cvt.u16.u32 %rs588, %r2561; + or.b16 %rs587, %rs5, %rs588; + mov.u32 %r2564, 20548; + sub.s32 %r2565, %r2564, %r5719; + cvt.u64.u32 %rd82, %r2565; + add.s64 %rd83, %rd1, %rd82; + st.global.u8 [%rd83], %rs587; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p87, %rs293, 143; + selp.u32 %r5717, 1, 0, %p87; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_78: + setp.ne.s32 %p88, %r5231, 0; + mov.u32 %r5242, %r5716; + @%p88 bra $L__BB0_74; + +$L__BB0_79: + setp.ne.s32 %p89, %r5736, 0; + @%p89 bra $L__BB0_127; + + setp.eq.s32 %p90, %r5212, 0; + add.s32 %r2566, %r5271, 17477; + cvt.u64.u32 %rd84, %r2566; + add.s64 %rd4, %rd1, %rd84; + @%p90 bra $L__BB0_119; + + shl.b16 %rs619, %rs705, 1; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p91, %r5277, 0; + mov.u32 %r5276, %r5480; + @%p91 bra $L__BB0_84; + bra.uni $L__BB0_82; + +$L__BB0_84: + setp.lt.u32 %p93, %r5482, 3; + mov.u32 %r5246, 0; + @%p93 bra $L__BB0_87; + + setp.lt.u32 %p94, %r5482, 6; + mov.u32 %r5246, 1; + @%p94 bra $L__BB0_87; + + setp.lt.u32 %p95, %r5482, 9; + setp.eq.s32 %p96, %r5482, 11; + selp.b32 %r2572, 4, 5, %p96; + setp.lt.u32 %p97, %r5482, 11; + selp.b32 %r2573, 3, %r2572, %p97; + selp.b32 %r5246, 2, %r2573, %p95; + +$L__BB0_87: + setp.eq.s32 %p98, %r5246, 0; + @%p98 bra $L__BB0_115; + + and.b32 %r163, %r5246, 3; + setp.eq.s32 %p99, %r163, 0; + mov.u32 %r5256, %r5246; + mov.u32 %r5259, %r5276; + @%p99 bra $L__BB0_100; + + add.s32 %r5119, %r5246, -1; + mov.u32 %r2575, 1; + shl.b32 %r2576, %r2575, %r5119; + and.b32 %r2577, %r2576, %r5483; + setp.ne.s32 %p100, %r2577, 0; + selp.u32 %r2578, 1, 0, %p100; + cvt.u32.u16 %r2579, %rs619; + bfi.b32 %r2580, %r2579, %r2578, 1, 8; + cvt.u16.u32 %rs619, %r2580; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p101, %r5277, 0; + mov.u32 %r5259, %r5276; + @%p101 bra $L__BB0_92; + + setp.gt.u32 %p102, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5259, %r2575; + @%p102 bra $L__BB0_92; + + add.s32 %r2584, %r5271, 17477; + cvt.u64.u32 %rd85, %r2584; + add.s64 %rd86, %rd1, %rd85; + st.global.u8 [%rd86], %rs619; + add.s32 %r5271, %r5271, 1; + mov.u32 %r5277, 8; + mov.u16 %rs619, 0; + mov.u32 %r5259, %r5276; + +$L__BB0_92: + and.b32 %r5121, %r5246, 3; + add.s32 %r5256, %r5246, -1; + setp.eq.s32 %p103, %r5121, 1; + mov.u32 %r5276, %r5259; + @%p103 bra $L__BB0_100; + + add.s32 %r5256, %r5246, -2; + mov.u32 %r2585, 1; + shl.b32 %r2586, %r2585, %r5256; + and.b32 %r2587, %r2586, %r5483; + setp.ne.s32 %p104, %r2587, 0; + selp.u32 %r2588, 1, 0, %p104; + cvt.u32.u16 %r2589, %rs619; + bfi.b32 %r2590, %r2589, %r2588, 1, 8; + cvt.u16.u32 %rs619, %r2590; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p105, %r5277, 0; + mov.u32 %r5250, %r5259; + @%p105 bra $L__BB0_96; + + setp.gt.u32 %p106, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5250, %r2585; + @%p106 bra $L__BB0_96; + + add.s32 %r2593, %r5271, 17477; + cvt.u64.u32 %rd87, %r2593; + add.s64 %rd88, %rd1, %rd87; + and.b16 %rs300, %rs619, 255; + st.global.u8 [%rd88], %rs619; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p107, %rs300, 255; + selp.b32 %r5277, 7, 8, %p107; + mov.u16 %rs619, 0; + mov.u32 %r5250, %r5259; + +$L__BB0_96: + and.b32 %r5122, %r5246, 3; + setp.eq.s32 %p108, %r5122, 2; + mov.u32 %r5276, %r5250; + mov.u32 %r5259, %r5250; + @%p108 bra $L__BB0_100; + + add.s32 %r5256, %r5246, -3; + mov.u32 %r2594, 1; + shl.b32 %r2595, %r2594, %r5256; + and.b32 %r2596, %r2595, %r5483; + setp.ne.s32 %p109, %r2596, 0; + selp.u32 %r2597, 1, 0, %p109; + cvt.u32.u16 %r2598, %rs619; + bfi.b32 %r2599, %r2598, %r2597, 1, 8; + cvt.u16.u32 %rs619, %r2599; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p110, %r5277, 0; + mov.u32 %r5276, %r5250; + mov.u32 %r5259, %r5250; + @%p110 bra $L__BB0_100; + + add.s32 %r5256, %r5246, -3; + setp.gt.u32 %p111, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5276, %r2594; + mov.u32 %r5259, %r2594; + @%p111 bra $L__BB0_100; + + add.s32 %r5256, %r5246, -3; + add.s32 %r2604, %r5271, 17477; + cvt.u64.u32 %rd89, %r2604; + add.s64 %rd90, %rd1, %rd89; + and.b16 %rs303, %rs619, 255; + st.global.u8 [%rd90], %rs619; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p112, %rs303, 255; + selp.b32 %r5277, 7, 8, %p112; + mov.u16 %rs619, 0; + mov.u32 %r5276, %r5250; + mov.u32 %r5259, %r5250; + +$L__BB0_100: + add.s32 %r5123, %r5246, -1; + setp.lt.u32 %p113, %r5123, 3; + @%p113 bra $L__BB0_115; + + mov.u32 %r5276, %r5259; + +$L__BB0_102: + add.s32 %r2605, %r5256, -1; + mov.u32 %r2606, 1; + shl.b32 %r2607, %r2606, %r2605; + and.b32 %r2608, %r2607, %r5483; + setp.ne.s32 %p114, %r2608, 0; + selp.u32 %r2609, 1, 0, %p114; + cvt.u32.u16 %r2610, %rs619; + bfi.b32 %r5265, %r2610, %r2609, 1, 8; + add.s32 %r5266, %r5277, -1; + setp.ne.s32 %p115, %r5266, 0; + mov.u32 %r5264, %r5276; + @%p115 bra $L__BB0_105; + + setp.gt.u32 %p116, %r5271, 191; + mov.u32 %r5266, 0; + mov.u32 %r5264, %r2606; + @%p116 bra $L__BB0_105; + + cvt.u16.u32 %rs304, %r5265; + and.b16 %rs305, %rs304, 255; + add.s32 %r2614, %r5271, 17477; + cvt.u64.u32 %rd91, %r2614; + add.s64 %rd92, %rd1, %rd91; + st.global.u8 [%rd92], %rs304; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p117, %rs305, 255; + selp.b32 %r5266, 7, 8, %p117; + mov.u32 %r5265, 0; + mov.u32 %r5264, %r5276; + +$L__BB0_105: + add.s32 %r2615, %r5256, -2; + shl.b32 %r2617, %r2606, %r2615; + and.b32 %r2618, %r2617, %r5483; + setp.ne.s32 %p118, %r2618, 0; + and.b32 %r2619, %r5265, 127; + selp.u32 %r2620, 1, 0, %p118; + bfi.b32 %r5269, %r2619, %r2620, 1, 7; + add.s32 %r5270, %r5266, -1; + setp.ne.s32 %p119, %r5270, 0; + mov.u32 %r5268, %r5264; + @%p119 bra $L__BB0_108; + + setp.gt.u32 %p120, %r5271, 191; + mov.u32 %r5270, 0; + mov.u32 %r5268, 1; + @%p120 bra $L__BB0_108; + + cvt.u16.u32 %rs306, %r5269; + and.b16 %rs307, %rs306, 255; + add.s32 %r2624, %r5271, 17477; + cvt.u64.u32 %rd93, %r2624; + add.s64 %rd94, %rd1, %rd93; + st.global.u8 [%rd94], %rs306; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p121, %rs307, 255; + selp.b32 %r5270, 7, 8, %p121; + mov.u32 %r5269, 0; + mov.u32 %r5268, %r5264; + +$L__BB0_108: + add.s32 %r2625, %r5256, -3; + mov.u32 %r2626, 1; + shl.b32 %r2627, %r2626, %r2625; + and.b32 %r2628, %r2627, %r5483; + setp.ne.s32 %p122, %r2628, 0; + and.b32 %r2629, %r5269, 127; + selp.u32 %r2630, 1, 0, %p122; + bfi.b32 %r5273, %r2629, %r2630, 1, 7; + add.s32 %r5274, %r5270, -1; + setp.ne.s32 %p123, %r5274, 0; + mov.u32 %r5272, %r5268; + @%p123 bra $L__BB0_111; + + setp.gt.u32 %p124, %r5271, 191; + mov.u32 %r5274, 0; + mov.u32 %r5272, %r2626; + @%p124 bra $L__BB0_111; + + cvt.u16.u32 %rs308, %r5273; + and.b16 %rs309, %rs308, 255; + add.s32 %r2634, %r5271, 17477; + cvt.u64.u32 %rd95, %r2634; + add.s64 %rd96, %rd1, %rd95; + st.global.u8 [%rd96], %rs308; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p125, %rs309, 255; + selp.b32 %r5274, 7, 8, %p125; + mov.u32 %r5273, 0; + mov.u32 %r5272, %r5268; + +$L__BB0_111: + add.s32 %r5256, %r5256, -4; + shl.b32 %r2636, %r2626, %r5256; + and.b32 %r2637, %r2636, %r5483; + setp.ne.s32 %p126, %r2637, 0; + and.b32 %r2638, %r5273, 127; + selp.u32 %r2639, 1, 0, %p126; + bfi.b32 %r2640, %r2638, %r2639, 1, 15; + cvt.u16.u32 %rs619, %r2640; + add.s32 %r5277, %r5274, -1; + setp.ne.s32 %p127, %r5277, 0; + mov.u32 %r5276, %r5272; + @%p127 bra $L__BB0_114; + + setp.gt.u32 %p128, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5276, 1; + @%p128 bra $L__BB0_114; + + add.s32 %r2643, %r5271, 17477; + cvt.u64.u32 %rd97, %r2643; + add.s64 %rd98, %rd1, %rd97; + and.b16 %rs311, %rs619, 255; + st.global.u8 [%rd98], %rs619; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p129, %rs311, 255; + selp.b32 %r5277, 7, 8, %p129; + mov.u16 %rs619, 0; + mov.u32 %r5276, %r5272; + +$L__BB0_114: + setp.ne.s32 %p130, %r5256, 0; + @%p130 bra $L__BB0_102; + +$L__BB0_115: + add.s32 %r2645, %r5482, -1; + setp.eq.s32 %p131, %r5482, 0; + mov.u32 %r5483, 0; + selp.b32 %r5482, 0, %r2645, %p131; + setp.lt.u32 %p132, %r5482, 3; + mov.u32 %r5282, %r5483; + @%p132 bra $L__BB0_118; + + setp.lt.u32 %p133, %r5482, 6; + mov.u32 %r5282, 1; + @%p133 bra $L__BB0_118; + + setp.lt.u32 %p134, %r5482, 9; + setp.eq.s32 %p135, %r5482, 11; + selp.b32 %r2647, 4, 5, %p135; + setp.lt.u32 %p136, %r5482, 11; + selp.b32 %r2648, 3, %r2647, %p136; + selp.b32 %r5282, 2, %r2648, %p134; + +$L__BB0_118: + mov.u32 %r2650, 1; + shl.b32 %r5481, %r2650, %r5282; + mov.u32 %r5480, %r5276; + mov.u16 %rs705, %rs619; + bra.uni $L__BB0_127; + +$L__BB0_119: + add.s32 %r5483, %r5483, 1; + setp.lt.u32 %p137, %r5483, %r5481; + @%p137 bra $L__BB0_127; + + shl.b16 %rs312, %rs705, 1; + or.b16 %rs622, %rs312, 1; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p138, %r5277, 0; + mov.u32 %r5283, %r5480; + @%p138 bra $L__BB0_123; + + shl.b16 %rs592, %rs705, 1; + or.b16 %rs622, %rs592, 1; + setp.gt.u32 %p139, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5283, 1; + @%p139 bra $L__BB0_123; + + shl.b16 %rs594, %rs705, 1; + or.b16 %rs593, %rs594, 1; + and.b16 %rs314, %rs593, 255; + st.global.u8 [%rd4], %rs593; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p140, %rs314, 255; + selp.b32 %r5277, 7, 8, %p140; + mov.u16 %rs622, 0; + mov.u32 %r5283, %r5480; + +$L__BB0_123: + add.s32 %r2654, %r5482, 1; + min.u32 %r236, %r2654, 12; + setp.lt.u32 %p141, %r236, 3; + mov.u32 %r5483, 0; + mov.u32 %r5286, %r5483; + @%p141 bra $L__BB0_126; + + add.s32 %r5127, %r5482, 1; + min.u32 %r5126, %r5127, 12; + setp.lt.u32 %p142, %r5126, 6; + mov.u32 %r5286, 1; + @%p142 bra $L__BB0_126; + + add.s32 %r5129, %r5482, 1; + min.u32 %r5128, %r5129, 12; + setp.lt.u32 %p143, %r5128, 9; + setp.eq.s32 %p144, %r5128, 11; + selp.b32 %r2656, 4, 5, %p144; + setp.lt.u32 %p145, %r5128, 11; + selp.b32 %r2657, 3, %r2656, %p145; + selp.b32 %r5286, 2, %r2657, %p143; + +$L__BB0_126: + add.s32 %r5131, %r5482, 1; + min.u32 %r5482, %r5131, 12; + mov.u32 %r2659, 1; + shl.b32 %r5481, %r2659, %r5286; + mov.u32 %r5480, %r5283; + mov.u16 %rs705, %rs622; + +$L__BB0_127: + max.s32 %r246, %r5227, 1; + and.b16 %rs315, %rs3, 15; + cvt.u32.u16 %r247, %rs315; + and.b32 %r2660, %r5212, 1; + setp.eq.b32 %p146, %r2660, 1; + mov.pred %p147, 0; + xor.pred %p148, %p146, %p147; + not.pred %p149, %p148; + mov.u32 %r5303, %r5933; + @%p149 bra $L__BB0_134; + + and.b32 %r2661, %r247, 1; + sub.s32 %r5293, %r246, %r2661; + setp.eq.s32 %p150, %r5293, 0; + mov.u32 %r5303, %r5933; + @%p150 bra $L__BB0_134; + + mov.u32 %r2662, -1; + shl.b32 %r2663, %r2662, %r5293; + not.b32 %r2664, %r2663; + and.b32 %r5294, %r5206, %r2664; + +$L__BB0_130: + setp.gt.u32 %p151, %r5907, 17476; + mov.u32 %r5303, 1; + @%p151 bra $L__BB0_134; + + sub.s32 %r2666, %r5906, %r5905; + min.u32 %r2667, %r2666, %r5293; + setp.eq.s32 %p152, %r2667, 32; + mov.u32 %r2668, -1; + shl.b32 %r2669, %r2668, %r2667; + not.b32 %r2670, %r2669; + selp.b32 %r2671, -1, %r2670, %p152; + and.b32 %r2672, %r2671, %r5294; + shl.b32 %r2673, %r2672, %r5905; + or.b32 %r5904, %r2673, %r5904; + add.s32 %r5905, %r2667, %r5905; + shr.u32 %r5294, %r5294, %r2667; + sub.s32 %r5293, %r5293, %r2667; + setp.lt.u32 %p153, %r5905, %r5906; + @%p153 bra $L__BB0_133; + + cvt.u64.u32 %rd99, %r5907; + add.s64 %rd100, %rd1, %rd99; + st.global.u8 [%rd100], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p154, %r5904, 255; + selp.b32 %r5906, 7, 8, %p154; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_133: + setp.ne.s32 %p155, %r5293, 0; + mov.u32 %r5303, %r5933; + @%p155 bra $L__BB0_130; + +$L__BB0_134: + and.b32 %r5105, %r5212, 2; + setp.eq.s32 %p156, %r5105, 0; + mov.u32 %r5318, %r5303; + @%p156 bra $L__BB0_141; + + shr.u32 %r2676, %r247, 1; + and.b32 %r2677, %r2676, 1; + sub.s32 %r5308, %r246, %r2677; + setp.eq.s32 %p157, %r5308, 0; + mov.u32 %r5318, %r5303; + @%p157 bra $L__BB0_141; + + mov.u32 %r2678, -1; + shl.b32 %r2679, %r2678, %r5308; + not.b32 %r2680, %r2679; + and.b32 %r5309, %r5210, %r2680; + +$L__BB0_137: + setp.gt.u32 %p158, %r5907, 17476; + mov.u32 %r5318, 1; + @%p158 bra $L__BB0_141; + + sub.s32 %r2682, %r5906, %r5905; + min.u32 %r2683, %r2682, %r5308; + setp.eq.s32 %p159, %r2683, 32; + mov.u32 %r2684, -1; + shl.b32 %r2685, %r2684, %r2683; + not.b32 %r2686, %r2685; + selp.b32 %r2687, -1, %r2686, %p159; + and.b32 %r2688, %r2687, %r5309; + shl.b32 %r2689, %r2688, %r5905; + or.b32 %r5904, %r2689, %r5904; + add.s32 %r5905, %r2683, %r5905; + shr.u32 %r5309, %r5309, %r2683; + sub.s32 %r5308, %r5308, %r2683; + setp.lt.u32 %p160, %r5905, %r5906; + @%p160 bra $L__BB0_140; + + cvt.u64.u32 %rd101, %r5907; + add.s64 %rd102, %rd1, %rd101; + st.global.u8 [%rd102], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p161, %r5904, 255; + selp.b32 %r5906, 7, 8, %p161; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_140: + setp.ne.s32 %p162, %r5308, 0; + mov.u32 %r5318, %r5303; + @%p162 bra $L__BB0_137; + +$L__BB0_141: + and.b32 %r2692, %r5212, 4; + setp.eq.s32 %p163, %r2692, 0; + mov.u32 %r5333, %r5318; + @%p163 bra $L__BB0_148; + + shr.u32 %r2693, %r247, 2; + and.b32 %r2694, %r2693, 1; + sub.s32 %r5323, %r246, %r2694; + setp.eq.s32 %p164, %r5323, 0; + mov.u32 %r5333, %r5318; + @%p164 bra $L__BB0_148; + + mov.u32 %r2695, -1; + shl.b32 %r2696, %r2695, %r5323; + not.b32 %r2697, %r2696; + and.b32 %r5324, %r5226, %r2697; + +$L__BB0_144: + setp.gt.u32 %p165, %r5907, 17476; + mov.u32 %r5333, 1; + @%p165 bra $L__BB0_148; + + sub.s32 %r2699, %r5906, %r5905; + min.u32 %r2700, %r2699, %r5323; + setp.eq.s32 %p166, %r2700, 32; + mov.u32 %r2701, -1; + shl.b32 %r2702, %r2701, %r2700; + not.b32 %r2703, %r2702; + selp.b32 %r2704, -1, %r2703, %p166; + and.b32 %r2705, %r2704, %r5324; + shl.b32 %r2706, %r2705, %r5905; + or.b32 %r5904, %r2706, %r5904; + add.s32 %r5905, %r2700, %r5905; + shr.u32 %r5324, %r5324, %r2700; + sub.s32 %r5323, %r5323, %r2700; + setp.lt.u32 %p167, %r5905, %r5906; + @%p167 bra $L__BB0_147; + + cvt.u64.u32 %rd103, %r5907; + add.s64 %rd104, %rd1, %rd103; + st.global.u8 [%rd104], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p168, %r5904, 255; + selp.b32 %r5906, 7, 8, %p168; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_147: + setp.ne.s32 %p169, %r5323, 0; + mov.u32 %r5333, %r5318; + @%p169 bra $L__BB0_144; + +$L__BB0_148: + and.b32 %r5106, %r5212, 8; + setp.eq.s32 %p170, %r5106, 0; + mov.u32 %r5933, %r5333; + @%p170 bra $L__BB0_155; + + shr.u32 %r2709, %r247, 3; + sub.s32 %r5338, %r246, %r2709; + setp.eq.s32 %p171, %r5338, 0; + mov.u32 %r5933, %r5333; + @%p171 bra $L__BB0_155; + + mov.u32 %r2710, -1; + shl.b32 %r2711, %r2710, %r5338; + not.b32 %r2712, %r2711; + and.b32 %r5339, %r5225, %r2712; + +$L__BB0_151: + setp.gt.u32 %p172, %r5907, 17476; + mov.u32 %r5933, 1; + @%p172 bra $L__BB0_155; + + sub.s32 %r2714, %r5906, %r5905; + min.u32 %r2715, %r2714, %r5338; + setp.eq.s32 %p173, %r2715, 32; + mov.u32 %r2716, -1; + shl.b32 %r2717, %r2716, %r2715; + not.b32 %r2718, %r2717; + selp.b32 %r2719, -1, %r2718, %p173; + and.b32 %r2720, %r2719, %r5339; + shl.b32 %r2721, %r2720, %r5905; + or.b32 %r5904, %r2721, %r5904; + add.s32 %r5905, %r2715, %r5905; + shr.u32 %r5339, %r5339, %r2715; + sub.s32 %r5338, %r5338, %r2715; + setp.lt.u32 %p174, %r5905, %r5906; + @%p174 bra $L__BB0_154; + + cvt.u64.u32 %rd105, %r5907; + add.s64 %rd106, %rd1, %rd105; + st.global.u8 [%rd106], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p175, %r5904, 255; + selp.b32 %r5906, 7, 8, %p175; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_154: + setp.ne.s32 %p176, %r5338, 0; + mov.u32 %r5933, %r5333; + @%p176 bra $L__BB0_151; + +$L__BB0_155: + add.s32 %r2724, %r5186, 2; + setp.lt.u32 %p177, %r2724, %r5; + mul.lo.s32 %r340, %r130, 6; + @%p177 bra $L__BB0_184; + bra.uni $L__BB0_156; + +$L__BB0_184: + mul.wide.u32 %rd119, %r5187, 4; + add.s64 %rd120, %rd2, %rd119; + ld.global.u32 %r413, [%rd120]; + setp.eq.s32 %p214, %r413, 0; + mov.u32 %r5398, 0; + mov.u32 %r5397, %r5398; + @%p214 bra $L__BB0_186; + + and.b32 %r2796, %r413, -2147483648; + abs.s32 %r2797, %r413; + mov.u32 %r2798, 31; + sub.s32 %r2799, %r2798, %r2; + shl.b32 %r2800, %r2797, %r2799; + or.b32 %r5397, %r2800, %r2796; + +$L__BB0_186: + shl.b32 %r2804, %r5397, 1; + shr.u32 %r2805, %r2804, %r45; + and.b32 %r416, %r2805, -2; + setp.eq.s32 %p215, %r416, 0; + mov.u32 %r5399, %r5398; + mov.u32 %r5405, %r5398; + @%p215 bra $L__BB0_188; + + add.s32 %r2807, %r416, -1; + clz.b32 %r2808, %r2807; + mov.u32 %r2809, 32; + sub.s32 %r5398, %r2809, %r2808; + shr.u32 %r2810, %r5397, 31; + add.s32 %r2811, %r2810, %r416; + add.s32 %r5399, %r2811, -2; + mov.u32 %r5405, 1; + +$L__BB0_188: + mov.u32 %r5402, 0; + mov.u32 %r5401, %r5402; + @%p64 bra $L__BB0_191; + + add.s32 %r2814, %r5187, %r1; + mul.wide.u32 %rd121, %r2814, 4; + add.s64 %rd122, %rd2, %rd121; + ld.global.u32 %r422, [%rd122]; + setp.eq.s32 %p217, %r422, 0; + @%p217 bra $L__BB0_191; + + and.b32 %r2815, %r422, -2147483648; + abs.s32 %r2816, %r422; + mov.u32 %r2817, 31; + sub.s32 %r2818, %r2817, %r2; + shl.b32 %r2819, %r2816, %r2818; + or.b32 %r5401, %r2819, %r2815; + +$L__BB0_191: + shl.b32 %r2822, %r5401, 1; + shr.u32 %r2823, %r2822, %r45; + and.b32 %r425, %r2823, -2; + setp.eq.s32 %p218, %r425, 0; + mov.u32 %r5403, %r5402; + mov.u32 %r5420, %r5398; + @%p218 bra $L__BB0_193; + + or.b32 %r5405, %r5405, 2; + add.s32 %r2824, %r425, -1; + clz.b32 %r2825, %r2824; + mov.u32 %r2826, 32; + sub.s32 %r5402, %r2826, %r2825; + max.s32 %r5420, %r5398, %r5402; + shr.u32 %r2827, %r5401, 31; + add.s32 %r2828, %r2827, %r425; + add.s32 %r5403, %r2828, -2; + +$L__BB0_193: + add.s32 %r5422, %r5187, 1; + add.s32 %r2833, %r5186, 3; + setp.ge.u32 %p219, %r2833, %r5; + mov.u32 %r5423, 0; + mov.u32 %r5416, %r5423; + mov.u32 %r5417, %r5423; + mov.u32 %r5418, %r5423; + mov.u32 %r5419, %r5423; + @%p219 bra $L__BB0_204; + + add.s32 %r5132, %r5187, 1; + mul.wide.u32 %rd123, %r5132, 4; + add.s64 %rd124, %rd2, %rd123; + ld.global.u32 %r435, [%rd124]; + setp.eq.s32 %p220, %r435, 0; + mov.u32 %r5417, 0; + mov.u32 %r5406, %r5417; + @%p220 bra $L__BB0_196; + + and.b32 %r2835, %r435, -2147483648; + abs.s32 %r2836, %r435; + mov.u32 %r2837, 31; + sub.s32 %r2838, %r2837, %r2; + shl.b32 %r2839, %r2836, %r2838; + or.b32 %r5406, %r2839, %r2835; + +$L__BB0_196: + shl.b32 %r2842, %r5406, 1; + shr.u32 %r2843, %r2842, %r45; + and.b32 %r438, %r2843, -2; + setp.eq.s32 %p221, %r438, 0; + mov.u32 %r5419, %r5417; + @%p221 bra $L__BB0_198; + + or.b32 %r5405, %r5405, 4; + add.s32 %r2844, %r438, -1; + clz.b32 %r2845, %r2844; + mov.u32 %r2846, 32; + sub.s32 %r5417, %r2846, %r2845; + max.s32 %r5420, %r5420, %r5417; + shr.u32 %r2847, %r5406, 31; + add.s32 %r2848, %r2847, %r438; + add.s32 %r5419, %r2848, -2; + +$L__BB0_198: + mov.u32 %r5416, 0; + mov.u32 %r5411, %r5416; + @%p64 bra $L__BB0_201; + + add.s32 %r5133, %r5187, 1; + add.s32 %r2851, %r5133, %r1; + mul.wide.u32 %rd125, %r2851, 4; + add.s64 %rd126, %rd2, %rd125; + ld.global.u32 %r447, [%rd126]; + setp.eq.s32 %p223, %r447, 0; + @%p223 bra $L__BB0_201; + + and.b32 %r2852, %r447, -2147483648; + abs.s32 %r2853, %r447; + mov.u32 %r2854, 31; + sub.s32 %r2855, %r2854, %r2; + shl.b32 %r2856, %r2853, %r2855; + or.b32 %r5411, %r2856, %r2852; + +$L__BB0_201: + shl.b32 %r2859, %r5411, 1; + shr.u32 %r2860, %r2859, %r45; + and.b32 %r450, %r2860, -2; + setp.eq.s32 %p224, %r450, 0; + mov.u32 %r5418, %r5416; + @%p224 bra $L__BB0_203; + + or.b32 %r5405, %r5405, 8; + add.s32 %r2861, %r450, -1; + clz.b32 %r2862, %r2861; + mov.u32 %r2863, 32; + sub.s32 %r5416, %r2863, %r2862; + max.s32 %r5420, %r5420, %r5416; + shr.u32 %r2864, %r5411, 31; + add.s32 %r2865, %r2864, %r450; + add.s32 %r5418, %r2865, -2; + +$L__BB0_203: + add.s32 %r5422, %r5187, 2; + +$L__BB0_204: + mov.u32 %r5187, %r5422; + and.b32 %r5117, %r5212, 1; + shr.u32 %r2868, %r5212, 1; + or.b32 %r467, %r2868, %r5117; + add.s32 %r2869, %r5420, -1; + setp.lt.s32 %p225, %r5420, 2; + setp.gt.s32 %p226, %r5420, 1; + selp.b32 %r468, %r2869, 0, %p226; + @%p225 bra $L__BB0_206; + + setp.eq.s32 %p227, %r5398, %r5420; + selp.u32 %r2870, 1, 0, %p227; + setp.eq.s32 %p228, %r5402, %r5420; + selp.u32 %r2871, -1, 0, %p228; + bfi.b32 %r2872, %r2871, %r2870, 1, 1; + setp.eq.s32 %p229, %r5417, %r5420; + selp.u16 %rs335, 1, 0, %p229; + mul.wide.u16 %r2873, %rs335, 4; + or.b32 %r2874, %r2872, %r2873; + setp.eq.s32 %p230, %r5416, %r5420; + selp.u16 %rs336, 1, 0, %p230; + mul.wide.u16 %r2875, %rs336, 8; + or.b32 %r5423, %r2874, %r2875; + +$L__BB0_206: + shr.u32 %r5116, %r5186, 1; + mov.u32 %r5115, _ZZ31 j2k_htj2k_encode_codeblockE14cleanup_cx_val; + add.s32 %r5114, %r5115, %r5116; + and.b32 %r5113, %r5212, 8; + shr.u32 %r5112, %r5113, 3; + mov.u32 %r5111, _ZZ31 j2k_htj2k_encode_codeblockE13cleanup_e_val; + add.s32 %r5110, %r5111, %r5116; + and.b32 %r2876, %r5402, 255; + and.b32 %r2877, %r5223, 255; + setp.lt.u32 %p231, %r2876, %r2877; + cvt.u16.u32 %rs337, %r5223; + cvt.u16.u32 %rs338, %r5402; + selp.b16 %rs339, %rs337, %rs338, %p231; + st.shared.u8 [%r5110+1], %rs339; + st.shared.u8 [%r5110+2], %r5416; + and.b32 %r471, %r5405, 2; + shr.u32 %r2881, %r471, 1; + or.b32 %r2882, %r5112, %r2881; + st.shared.u8 [%r5114+1], %r2882; + and.b32 %r472, %r5405, 8; + shr.u32 %r2885, %r472, 3; + st.shared.u8 [%r5114+2], %r2885; + shl.b32 %r2886, %r5405, 4; + shl.b32 %r2887, %r467, 8; + or.b32 %r2888, %r2886, %r2887; + or.b32 %r2889, %r2888, %r5423; + mul.wide.u32 %rd128, %r2889, 2; + add.s64 %rd129, %rd79, %rd128; + ld.global.u16 %rs47, [%rd129]; + shr.u16 %rs340, %rs47, 4; + and.b16 %rs48, %rs340, 7; + setp.eq.s16 %p232, %rs48, 0; + mov.u32 %r5435, %r5242; + @%p232 bra $L__BB0_213; + + cvt.u32.u16 %r5424, %rs48; + shr.u16 %rs341, %rs47, 8; + cvt.u32.u16 %r5425, %rs341; + +$L__BB0_208: + mov.u16 %rs49, %rs688; + mov.u32 %r475, %r5424; + setp.gt.u32 %p233, %r5719, 2879; + mov.u32 %r5435, 1; + @%p233 bra $L__BB0_213; + + mov.u32 %r2891, 8; + sub.s32 %r2892, %r2891, %r5717; + sub.s32 %r2893, %r2892, %r5718; + min.u32 %r2894, %r2893, %r475; + setp.eq.s32 %p234, %r2894, 32; + mov.u32 %r2895, -1; + shl.b32 %r2896, %r2895, %r2894; + not.b32 %r2897, %r2896; + selp.b32 %r2898, -1, %r2897, %p234; + and.b32 %r2899, %r2898, %r5425; + shl.b32 %r2900, %r2899, %r5718; + cvt.u16.u32 %rs342, %r2900; + or.b16 %rs688, %rs49, %rs342; + add.s32 %r5718, %r2894, %r5718; + sub.s32 %r5424, %r475, %r2894; + shr.u32 %r5425, %r5425, %r2894; + setp.gt.u32 %p235, %r2893, %r475; + @%p235 bra $L__BB0_212; + + setp.ne.s32 %p236, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs343, %rs688, 255; + setp.ne.s16 %p237, %rs343, 127; + and.pred %p238, %p236, %p237; + @%p238 bra $L__BB0_212; + + cvt.u16.u32 %rs596, %r2900; + or.b16 %rs595, %rs49, %rs596; + mov.u32 %r2903, 20548; + sub.s32 %r2904, %r2903, %r5719; + cvt.u64.u32 %rd130, %r2904; + add.s64 %rd131, %rd1, %rd130; + st.global.u8 [%rd131], %rs595; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p239, %rs343, 143; + selp.u32 %r5717, 1, 0, %p239; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_212: + setp.ne.s32 %p240, %r5424, 0; + mov.u32 %r5435, %r5242; + @%p240 bra $L__BB0_208; + +$L__BB0_213: + setp.ne.s32 %p241, %r467, 0; + @%p241 bra $L__BB0_261; + + setp.eq.s32 %p242, %r5405, 0; + add.s32 %r2905, %r5271, 17477; + cvt.u64.u32 %rd132, %r2905; + add.s64 %rd6, %rd1, %rd132; + @%p242 bra $L__BB0_253; + + shl.b16 %rs643, %rs705, 1; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p243, %r5277, 0; + mov.u32 %r5469, %r5480; + @%p243 bra $L__BB0_218; + + shl.b16 %rs643, %rs705, 1; + setp.gt.u32 %p244, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5469, 1; + @%p244 bra $L__BB0_218; + + shl.b16 %rs598, %rs705, 1; + st.global.u8 [%rd6], %rs598; + add.s32 %r5271, %r5271, 1; + mov.u32 %r5277, 8; + mov.u16 %rs643, 0; + mov.u32 %r5469, %r5480; + +$L__BB0_218: + setp.lt.u32 %p245, %r5482, 3; + mov.u32 %r5439, 0; + @%p245 bra $L__BB0_221; + + setp.lt.u32 %p246, %r5482, 6; + mov.u32 %r5439, 1; + @%p246 bra $L__BB0_221; + + setp.lt.u32 %p247, %r5482, 9; + setp.eq.s32 %p248, %r5482, 11; + selp.b32 %r2911, 4, 5, %p248; + setp.lt.u32 %p249, %r5482, 11; + selp.b32 %r2912, 3, %r2911, %p249; + selp.b32 %r5439, 2, %r2912, %p247; + +$L__BB0_221: + setp.eq.s32 %p250, %r5439, 0; + @%p250 bra $L__BB0_249; + + and.b32 %r500, %r5439, 3; + setp.eq.s32 %p251, %r500, 0; + mov.u32 %r5449, %r5439; + mov.u32 %r5452, %r5469; + @%p251 bra $L__BB0_234; + + add.s32 %r5136, %r5439, -1; + mov.u32 %r2914, 1; + shl.b32 %r2915, %r2914, %r5136; + and.b32 %r2916, %r2915, %r5483; + setp.ne.s32 %p252, %r2916, 0; + selp.u32 %r2917, 1, 0, %p252; + cvt.u32.u16 %r2918, %rs643; + bfi.b32 %r2919, %r2918, %r2917, 1, 8; + cvt.u16.u32 %rs643, %r2919; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p253, %r5277, 0; + mov.u32 %r5452, %r5469; + @%p253 bra $L__BB0_226; + + setp.gt.u32 %p254, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5452, %r2914; + @%p254 bra $L__BB0_226; + + add.s32 %r2923, %r5271, 17477; + cvt.u64.u32 %rd133, %r2923; + add.s64 %rd134, %rd1, %rd133; + st.global.u8 [%rd134], %rs643; + add.s32 %r5271, %r5271, 1; + mov.u32 %r5277, 8; + mov.u16 %rs643, 0; + mov.u32 %r5452, %r5469; + +$L__BB0_226: + and.b32 %r5138, %r5439, 3; + add.s32 %r5449, %r5439, -1; + setp.eq.s32 %p255, %r5138, 1; + mov.u32 %r5469, %r5452; + @%p255 bra $L__BB0_234; + + add.s32 %r5449, %r5439, -2; + mov.u32 %r2924, 1; + shl.b32 %r2925, %r2924, %r5449; + and.b32 %r2926, %r2925, %r5483; + setp.ne.s32 %p256, %r2926, 0; + selp.u32 %r2927, 1, 0, %p256; + cvt.u32.u16 %r2928, %rs643; + bfi.b32 %r2929, %r2928, %r2927, 1, 8; + cvt.u16.u32 %rs643, %r2929; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p257, %r5277, 0; + mov.u32 %r5443, %r5452; + @%p257 bra $L__BB0_230; + + setp.gt.u32 %p258, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5443, %r2924; + @%p258 bra $L__BB0_230; + + add.s32 %r2932, %r5271, 17477; + cvt.u64.u32 %rd135, %r2932; + add.s64 %rd136, %rd1, %rd135; + and.b16 %rs350, %rs643, 255; + st.global.u8 [%rd136], %rs643; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p259, %rs350, 255; + selp.b32 %r5277, 7, 8, %p259; + mov.u16 %rs643, 0; + mov.u32 %r5443, %r5452; + +$L__BB0_230: + and.b32 %r5139, %r5439, 3; + setp.eq.s32 %p260, %r5139, 2; + mov.u32 %r5469, %r5443; + mov.u32 %r5452, %r5443; + @%p260 bra $L__BB0_234; + + add.s32 %r5449, %r5439, -3; + mov.u32 %r2933, 1; + shl.b32 %r2934, %r2933, %r5449; + and.b32 %r2935, %r2934, %r5483; + setp.ne.s32 %p261, %r2935, 0; + selp.u32 %r2936, 1, 0, %p261; + cvt.u32.u16 %r2937, %rs643; + bfi.b32 %r2938, %r2937, %r2936, 1, 8; + cvt.u16.u32 %rs643, %r2938; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p262, %r5277, 0; + mov.u32 %r5469, %r5443; + mov.u32 %r5452, %r5443; + @%p262 bra $L__BB0_234; + + add.s32 %r5449, %r5439, -3; + setp.gt.u32 %p263, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5469, %r2933; + mov.u32 %r5452, %r2933; + @%p263 bra $L__BB0_234; + + add.s32 %r5449, %r5439, -3; + add.s32 %r2943, %r5271, 17477; + cvt.u64.u32 %rd137, %r2943; + add.s64 %rd138, %rd1, %rd137; + and.b16 %rs353, %rs643, 255; + st.global.u8 [%rd138], %rs643; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p264, %rs353, 255; + selp.b32 %r5277, 7, 8, %p264; + mov.u16 %rs643, 0; + mov.u32 %r5469, %r5443; + mov.u32 %r5452, %r5443; + +$L__BB0_234: + add.s32 %r5140, %r5439, -1; + setp.lt.u32 %p265, %r5140, 3; + @%p265 bra $L__BB0_249; + + mov.u32 %r5469, %r5452; + +$L__BB0_236: + add.s32 %r2944, %r5449, -1; + mov.u32 %r2945, 1; + shl.b32 %r2946, %r2945, %r2944; + and.b32 %r2947, %r2946, %r5483; + setp.ne.s32 %p266, %r2947, 0; + selp.u32 %r2948, 1, 0, %p266; + cvt.u32.u16 %r2949, %rs643; + bfi.b32 %r5458, %r2949, %r2948, 1, 8; + add.s32 %r5459, %r5277, -1; + setp.ne.s32 %p267, %r5459, 0; + mov.u32 %r5457, %r5469; + @%p267 bra $L__BB0_239; + + setp.gt.u32 %p268, %r5271, 191; + mov.u32 %r5459, 0; + mov.u32 %r5457, %r2945; + @%p268 bra $L__BB0_239; + + cvt.u16.u32 %rs354, %r5458; + and.b16 %rs355, %rs354, 255; + add.s32 %r2953, %r5271, 17477; + cvt.u64.u32 %rd139, %r2953; + add.s64 %rd140, %rd1, %rd139; + st.global.u8 [%rd140], %rs354; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p269, %rs355, 255; + selp.b32 %r5459, 7, 8, %p269; + mov.u32 %r5458, 0; + mov.u32 %r5457, %r5469; + +$L__BB0_239: + add.s32 %r2954, %r5449, -2; + shl.b32 %r2956, %r2945, %r2954; + and.b32 %r2957, %r2956, %r5483; + setp.ne.s32 %p270, %r2957, 0; + and.b32 %r2958, %r5458, 127; + selp.u32 %r2959, 1, 0, %p270; + bfi.b32 %r5462, %r2958, %r2959, 1, 7; + add.s32 %r5463, %r5459, -1; + setp.ne.s32 %p271, %r5463, 0; + mov.u32 %r5461, %r5457; + @%p271 bra $L__BB0_242; + + setp.gt.u32 %p272, %r5271, 191; + mov.u32 %r5463, 0; + mov.u32 %r5461, 1; + @%p272 bra $L__BB0_242; + + cvt.u16.u32 %rs356, %r5462; + and.b16 %rs357, %rs356, 255; + add.s32 %r2963, %r5271, 17477; + cvt.u64.u32 %rd141, %r2963; + add.s64 %rd142, %rd1, %rd141; + st.global.u8 [%rd142], %rs356; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p273, %rs357, 255; + selp.b32 %r5463, 7, 8, %p273; + mov.u32 %r5462, 0; + mov.u32 %r5461, %r5457; + +$L__BB0_242: + add.s32 %r2964, %r5449, -3; + mov.u32 %r2965, 1; + shl.b32 %r2966, %r2965, %r2964; + and.b32 %r2967, %r2966, %r5483; + setp.ne.s32 %p274, %r2967, 0; + and.b32 %r2968, %r5462, 127; + selp.u32 %r2969, 1, 0, %p274; + bfi.b32 %r5466, %r2968, %r2969, 1, 7; + add.s32 %r5467, %r5463, -1; + setp.ne.s32 %p275, %r5467, 0; + mov.u32 %r5465, %r5461; + @%p275 bra $L__BB0_245; + + setp.gt.u32 %p276, %r5271, 191; + mov.u32 %r5467, 0; + mov.u32 %r5465, %r2965; + @%p276 bra $L__BB0_245; + + cvt.u16.u32 %rs358, %r5466; + and.b16 %rs359, %rs358, 255; + add.s32 %r2973, %r5271, 17477; + cvt.u64.u32 %rd143, %r2973; + add.s64 %rd144, %rd1, %rd143; + st.global.u8 [%rd144], %rs358; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p277, %rs359, 255; + selp.b32 %r5467, 7, 8, %p277; + mov.u32 %r5466, 0; + mov.u32 %r5465, %r5461; + +$L__BB0_245: + add.s32 %r5449, %r5449, -4; + shl.b32 %r2975, %r2965, %r5449; + and.b32 %r2976, %r2975, %r5483; + setp.ne.s32 %p278, %r2976, 0; + and.b32 %r2977, %r5466, 127; + selp.u32 %r2978, 1, 0, %p278; + bfi.b32 %r2979, %r2977, %r2978, 1, 15; + cvt.u16.u32 %rs643, %r2979; + add.s32 %r5277, %r5467, -1; + setp.ne.s32 %p279, %r5277, 0; + mov.u32 %r5469, %r5465; + @%p279 bra $L__BB0_248; + + setp.gt.u32 %p280, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5469, 1; + @%p280 bra $L__BB0_248; + + add.s32 %r2982, %r5271, 17477; + cvt.u64.u32 %rd145, %r2982; + add.s64 %rd146, %rd1, %rd145; + and.b16 %rs361, %rs643, 255; + st.global.u8 [%rd146], %rs643; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p281, %rs361, 255; + selp.b32 %r5277, 7, 8, %p281; + mov.u16 %rs643, 0; + mov.u32 %r5469, %r5465; + +$L__BB0_248: + setp.ne.s32 %p282, %r5449, 0; + @%p282 bra $L__BB0_236; + +$L__BB0_249: + add.s32 %r2984, %r5482, -1; + setp.eq.s32 %p283, %r5482, 0; + mov.u32 %r5483, 0; + selp.b32 %r5482, 0, %r2984, %p283; + setp.lt.u32 %p284, %r5482, 3; + mov.u32 %r5475, %r5483; + @%p284 bra $L__BB0_252; + + setp.lt.u32 %p285, %r5482, 6; + mov.u32 %r5475, 1; + @%p285 bra $L__BB0_252; + + setp.lt.u32 %p286, %r5482, 9; + setp.eq.s32 %p287, %r5482, 11; + selp.b32 %r2986, 4, 5, %p287; + setp.lt.u32 %p288, %r5482, 11; + selp.b32 %r2987, 3, %r2986, %p288; + selp.b32 %r5475, 2, %r2987, %p286; + +$L__BB0_252: + mov.u32 %r2989, 1; + shl.b32 %r5481, %r2989, %r5475; + mov.u32 %r5480, %r5469; + mov.u16 %rs705, %rs643; + bra.uni $L__BB0_261; + +$L__BB0_156: + cvt.u64.u32 %rd107, %r340; + add.s64 %rd5, %rd108, %rd107; + add.s32 %r2725, %r340, 2; + cvt.u64.u32 %rd109, %r2725; + add.s64 %rd110, %rd108, %rd109; + ld.global.u8 %rs25, [%rd5+1]; + ld.global.u8 %rs26, [%rd110]; + ld.global.u8 %rs27, [%rd110+1]; + ld.global.u8 %rs28, [%rd108]; + ld.global.u8 %rs29, [%rd108+1]; + ld.global.u8 %rs30, [%rd108+2]; + ld.global.u8 %rs31, [%rd108+3]; + setp.eq.s16 %p178, %rs25, 0; + mov.u32 %r5364, %r5242; + @%p178 bra $L__BB0_163; + + ld.global.u8 %r5354, [%rd5]; + cvt.u32.u16 %r5353, %rs25; + +$L__BB0_158: + mov.u16 %rs32, %rs688; + mov.u32 %r343, %r5353; + setp.gt.u32 %p179, %r5719, 2879; + mov.u32 %r5364, 1; + @%p179 bra $L__BB0_163; + + mov.u32 %r2727, 8; + sub.s32 %r2728, %r2727, %r5717; + sub.s32 %r2729, %r2728, %r5718; + min.u32 %r2730, %r2729, %r343; + setp.eq.s32 %p180, %r2730, 32; + mov.u32 %r2731, -1; + shl.b32 %r2732, %r2731, %r2730; + not.b32 %r2733, %r2732; + selp.b32 %r2734, -1, %r2733, %p180; + and.b32 %r2735, %r2734, %r5354; + shl.b32 %r2736, %r2735, %r5718; + cvt.u16.u32 %rs316, %r2736; + or.b16 %rs688, %rs32, %rs316; + add.s32 %r5718, %r2730, %r5718; + sub.s32 %r5353, %r343, %r2730; + shr.u32 %r5354, %r5354, %r2730; + setp.gt.u32 %p181, %r2729, %r343; + @%p181 bra $L__BB0_162; + + setp.ne.s32 %p182, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs317, %rs688, 255; + setp.ne.s16 %p183, %rs317, 127; + and.pred %p184, %p182, %p183; + @%p184 bra $L__BB0_162; + + cvt.u16.u32 %rs604, %r2736; + or.b16 %rs603, %rs32, %rs604; + mov.u32 %r2739, 20548; + sub.s32 %r2740, %r2739, %r5719; + cvt.u64.u32 %rd111, %r2740; + add.s64 %rd112, %rd1, %rd111; + st.global.u8 [%rd112], %rs603; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p185, %rs317, 143; + selp.u32 %r5717, 1, 0, %p185; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_162: + setp.ne.s32 %p186, %r5353, 0; + mov.u32 %r5364, %r5242; + @%p186 bra $L__BB0_158; + +$L__BB0_163: + setp.eq.s16 %p187, %rs29, 0; + mov.u32 %r5376, %r5364; + @%p187 bra $L__BB0_170; + + cvt.u32.u16 %r2741, %rs28; + and.b32 %r5366, %r2741, 255; + cvt.u32.u16 %r2742, %rs29; + and.b32 %r5365, %r2742, 255; + +$L__BB0_165: + mov.u32 %r362, %r5365; + setp.gt.u32 %p188, %r5719, 2879; + mov.u32 %r5376, 1; + @%p188 bra $L__BB0_170; + + mov.u32 %r2744, 8; + sub.s32 %r2745, %r2744, %r5717; + sub.s32 %r2746, %r2745, %r5718; + min.u32 %r2747, %r2746, %r362; + setp.eq.s32 %p189, %r2747, 32; + mov.u32 %r2748, -1; + shl.b32 %r2749, %r2748, %r2747; + not.b32 %r2750, %r2749; + selp.b32 %r2751, -1, %r2750, %p189; + and.b32 %r2752, %r2751, %r5366; + shl.b32 %r2753, %r2752, %r5718; + cvt.u16.u32 %rs321, %r2753; + or.b16 %rs688, %rs688, %rs321; + add.s32 %r5718, %r2747, %r5718; + sub.s32 %r5365, %r362, %r2747; + shr.u32 %r5366, %r5366, %r2747; + setp.gt.u32 %p190, %r2746, %r362; + @%p190 bra $L__BB0_169; + + setp.ne.s32 %p191, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs322, %rs688, 255; + setp.ne.s16 %p192, %rs322, 127; + and.pred %p193, %p191, %p192; + @%p193 bra $L__BB0_169; + + mov.u32 %r2756, 20548; + sub.s32 %r2757, %r2756, %r5719; + cvt.u64.u32 %rd113, %r2757; + add.s64 %rd114, %rd1, %rd113; + st.global.u8 [%rd114], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p194, %rs322, 143; + selp.u32 %r5717, 1, 0, %p194; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_169: + setp.ne.s32 %p195, %r5365, 0; + mov.u32 %r5376, %r5364; + @%p195 bra $L__BB0_165; + +$L__BB0_170: + setp.eq.s16 %p196, %rs27, 0; + mov.u32 %r5388, %r5376; + @%p196 bra $L__BB0_177; + + cvt.u32.u16 %r2758, %rs27; + and.b32 %r5377, %r2758, 255; + cvt.u32.u16 %r2759, %rs26; + and.b32 %r5378, %r2759, 255; + +$L__BB0_172: + mov.u32 %r381, %r5377; + setp.gt.u32 %p197, %r5719, 2879; + mov.u32 %r5388, 1; + @%p197 bra $L__BB0_177; + + mov.u32 %r2761, 8; + sub.s32 %r2762, %r2761, %r5717; + sub.s32 %r2763, %r2762, %r5718; + min.u32 %r2764, %r2763, %r381; + setp.eq.s32 %p198, %r2764, 32; + mov.u32 %r2765, -1; + shl.b32 %r2766, %r2765, %r2764; + not.b32 %r2767, %r2766; + selp.b32 %r2768, -1, %r2767, %p198; + and.b32 %r2769, %r2768, %r5378; + shl.b32 %r2770, %r2769, %r5718; + cvt.u16.u32 %rs326, %r2770; + or.b16 %rs688, %rs688, %rs326; + add.s32 %r5718, %r2764, %r5718; + sub.s32 %r5377, %r381, %r2764; + shr.u32 %r5378, %r5378, %r2764; + setp.gt.u32 %p199, %r2763, %r381; + @%p199 bra $L__BB0_176; + + setp.ne.s32 %p200, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs327, %rs688, 255; + setp.ne.s16 %p201, %rs327, 127; + and.pred %p202, %p200, %p201; + @%p202 bra $L__BB0_176; + + mov.u32 %r2773, 20548; + sub.s32 %r2774, %r2773, %r5719; + cvt.u64.u32 %rd115, %r2774; + add.s64 %rd116, %rd1, %rd115; + st.global.u8 [%rd116], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p203, %rs327, 143; + selp.u32 %r5717, 1, 0, %p203; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_176: + setp.ne.s32 %p204, %r5377, 0; + mov.u32 %r5388, %r5376; + @%p204 bra $L__BB0_172; + +$L__BB0_177: + setp.eq.s16 %p205, %rs31, 0; + mov.u32 %r5736, 0; + mov.u32 %r5716, %r5388; + @%p205 bra $L__BB0_416; + + cvt.u32.u16 %r2776, %rs30; + and.b32 %r5390, %r2776, 255; + cvt.u32.u16 %r2777, %rs31; + and.b32 %r5389, %r2777, 255; + +$L__BB0_179: + mov.u32 %r400, %r5389; + mov.u32 %r5736, 0; + setp.gt.u32 %p206, %r5719, 2879; + mov.u32 %r5716, 1; + @%p206 bra $L__BB0_416; + + mov.u32 %r2780, 8; + sub.s32 %r2781, %r2780, %r5717; + sub.s32 %r2782, %r2781, %r5718; + min.u32 %r2783, %r2782, %r400; + setp.eq.s32 %p207, %r2783, 32; + mov.u32 %r2784, -1; + shl.b32 %r2785, %r2784, %r2783; + not.b32 %r2786, %r2785; + selp.b32 %r2787, -1, %r2786, %p207; + and.b32 %r2788, %r2787, %r5390; + shl.b32 %r2789, %r2788, %r5718; + cvt.u16.u32 %rs331, %r2789; + or.b16 %rs688, %rs688, %rs331; + add.s32 %r5718, %r2783, %r5718; + sub.s32 %r5389, %r400, %r2783; + shr.u32 %r5390, %r5390, %r2783; + setp.gt.u32 %p208, %r2782, %r400; + @%p208 bra $L__BB0_183; + + setp.ne.s32 %p209, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs332, %rs688, 255; + setp.ne.s16 %p210, %rs332, 127; + and.pred %p211, %p209, %p210; + @%p211 bra $L__BB0_183; + + mov.u32 %r2792, 20548; + sub.s32 %r2793, %r2792, %r5719; + cvt.u64.u32 %rd117, %r2793; + add.s64 %rd118, %rd1, %rd117; + st.global.u8 [%rd118], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p212, %rs332, 143; + selp.u32 %r5717, 1, 0, %p212; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_183: + mov.u32 %r5736, 0; + setp.eq.s32 %p213, %r5389, 0; + mov.u32 %r5716, %r5388; + @%p213 bra $L__BB0_416; + bra.uni $L__BB0_179; + +$L__BB0_82: + shl.b16 %rs619, %rs705, 1; + setp.gt.u32 %p92, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5276, 1; + @%p92 bra $L__BB0_84; + + shl.b16 %rs590, %rs705, 1; + st.global.u8 [%rd4], %rs590; + add.s32 %r5271, %r5271, 1; + mov.u32 %r5277, 8; + mov.u16 %rs619, 0; + mov.u32 %r5276, %r5480; + bra.uni $L__BB0_84; + +$L__BB0_253: + add.s32 %r5483, %r5483, 1; + setp.lt.u32 %p289, %r5483, %r5481; + @%p289 bra $L__BB0_261; + + shl.b16 %rs362, %rs705, 1; + or.b16 %rs646, %rs362, 1; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p290, %r5277, 0; + mov.u32 %r5476, %r5480; + @%p290 bra $L__BB0_257; + bra.uni $L__BB0_255; + +$L__BB0_257: + add.s32 %r2993, %r5482, 1; + min.u32 %r573, %r2993, 12; + setp.lt.u32 %p293, %r573, 3; + mov.u32 %r5483, 0; + mov.u32 %r5479, %r5483; + @%p293 bra $L__BB0_260; + + add.s32 %r5144, %r5482, 1; + min.u32 %r5143, %r5144, 12; + setp.lt.u32 %p294, %r5143, 6; + mov.u32 %r5479, 1; + @%p294 bra $L__BB0_260; + + add.s32 %r5146, %r5482, 1; + min.u32 %r5145, %r5146, 12; + setp.lt.u32 %p295, %r5145, 9; + setp.eq.s32 %p296, %r5145, 11; + selp.b32 %r2995, 4, 5, %p296; + setp.lt.u32 %p297, %r5145, 11; + selp.b32 %r2996, 3, %r2995, %p297; + selp.b32 %r5479, 2, %r2996, %p295; + +$L__BB0_260: + add.s32 %r5148, %r5482, 1; + min.u32 %r5482, %r5148, 12; + mov.u32 %r2998, 1; + shl.b32 %r5481, %r2998, %r5479; + mov.u32 %r5480, %r5476; + mov.u16 %rs705, %rs646; + +$L__BB0_261: + max.s32 %r583, %r5420, 1; + and.b16 %rs365, %rs47, 15; + cvt.u32.u16 %r584, %rs365; + and.b32 %r585, %r5405, 1; + setp.eq.s32 %p298, %r585, 0; + mov.u32 %r5496, %r5933; + @%p298 bra $L__BB0_268; + + and.b32 %r2999, %r584, 1; + sub.s32 %r5486, %r583, %r2999; + setp.eq.s32 %p299, %r5486, 0; + mov.u32 %r5496, %r5933; + @%p299 bra $L__BB0_268; + + mov.u32 %r3000, -1; + shl.b32 %r3001, %r3000, %r5486; + not.b32 %r3002, %r3001; + and.b32 %r5487, %r5399, %r3002; + +$L__BB0_264: + setp.gt.u32 %p300, %r5907, 17476; + mov.u32 %r5496, 1; + @%p300 bra $L__BB0_268; + + sub.s32 %r3004, %r5906, %r5905; + min.u32 %r3005, %r3004, %r5486; + setp.eq.s32 %p301, %r3005, 32; + mov.u32 %r3006, -1; + shl.b32 %r3007, %r3006, %r3005; + not.b32 %r3008, %r3007; + selp.b32 %r3009, -1, %r3008, %p301; + and.b32 %r3010, %r3009, %r5487; + shl.b32 %r3011, %r3010, %r5905; + or.b32 %r5904, %r3011, %r5904; + add.s32 %r5905, %r3005, %r5905; + shr.u32 %r5487, %r5487, %r3005; + sub.s32 %r5486, %r5486, %r3005; + setp.lt.u32 %p302, %r5905, %r5906; + @%p302 bra $L__BB0_267; + + cvt.u64.u32 %rd147, %r5907; + add.s64 %rd148, %rd1, %rd147; + st.global.u8 [%rd148], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p303, %r5904, 255; + selp.b32 %r5906, 7, 8, %p303; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_267: + setp.ne.s32 %p304, %r5486, 0; + mov.u32 %r5496, %r5933; + @%p304 bra $L__BB0_264; + +$L__BB0_268: + and.b32 %r5134, %r5405, 2; + setp.eq.s32 %p305, %r5134, 0; + mov.u32 %r5511, %r5496; + @%p305 bra $L__BB0_275; + + shr.u32 %r3014, %r584, 1; + and.b32 %r3015, %r3014, 1; + sub.s32 %r5501, %r583, %r3015; + setp.eq.s32 %p306, %r5501, 0; + mov.u32 %r5511, %r5496; + @%p306 bra $L__BB0_275; + + mov.u32 %r3016, -1; + shl.b32 %r3017, %r3016, %r5501; + not.b32 %r3018, %r3017; + and.b32 %r5502, %r5403, %r3018; + +$L__BB0_271: + setp.gt.u32 %p307, %r5907, 17476; + mov.u32 %r5511, 1; + @%p307 bra $L__BB0_275; + + sub.s32 %r3020, %r5906, %r5905; + min.u32 %r3021, %r3020, %r5501; + setp.eq.s32 %p308, %r3021, 32; + mov.u32 %r3022, -1; + shl.b32 %r3023, %r3022, %r3021; + not.b32 %r3024, %r3023; + selp.b32 %r3025, -1, %r3024, %p308; + and.b32 %r3026, %r3025, %r5502; + shl.b32 %r3027, %r3026, %r5905; + or.b32 %r5904, %r3027, %r5904; + add.s32 %r5905, %r3021, %r5905; + shr.u32 %r5502, %r5502, %r3021; + sub.s32 %r5501, %r5501, %r3021; + setp.lt.u32 %p309, %r5905, %r5906; + @%p309 bra $L__BB0_274; + + cvt.u64.u32 %rd149, %r5907; + add.s64 %rd150, %rd1, %rd149; + st.global.u8 [%rd150], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p310, %r5904, 255; + selp.b32 %r5906, 7, 8, %p310; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_274: + setp.ne.s32 %p311, %r5501, 0; + mov.u32 %r5511, %r5496; + @%p311 bra $L__BB0_271; + +$L__BB0_275: + and.b32 %r3030, %r5405, 4; + setp.eq.s32 %p312, %r3030, 0; + mov.u32 %r5526, %r5511; + @%p312 bra $L__BB0_282; + + shr.u32 %r3031, %r584, 2; + and.b32 %r3032, %r3031, 1; + sub.s32 %r5516, %r583, %r3032; + setp.eq.s32 %p313, %r5516, 0; + mov.u32 %r5526, %r5511; + @%p313 bra $L__BB0_282; + + mov.u32 %r3033, -1; + shl.b32 %r3034, %r3033, %r5516; + not.b32 %r3035, %r3034; + and.b32 %r5517, %r5419, %r3035; + +$L__BB0_278: + setp.gt.u32 %p314, %r5907, 17476; + mov.u32 %r5526, 1; + @%p314 bra $L__BB0_282; + + sub.s32 %r3037, %r5906, %r5905; + min.u32 %r3038, %r3037, %r5516; + setp.eq.s32 %p315, %r3038, 32; + mov.u32 %r3039, -1; + shl.b32 %r3040, %r3039, %r3038; + not.b32 %r3041, %r3040; + selp.b32 %r3042, -1, %r3041, %p315; + and.b32 %r3043, %r3042, %r5517; + shl.b32 %r3044, %r3043, %r5905; + or.b32 %r5904, %r3044, %r5904; + add.s32 %r5905, %r3038, %r5905; + shr.u32 %r5517, %r5517, %r3038; + sub.s32 %r5516, %r5516, %r3038; + setp.lt.u32 %p316, %r5905, %r5906; + @%p316 bra $L__BB0_281; + + cvt.u64.u32 %rd151, %r5907; + add.s64 %rd152, %rd1, %rd151; + st.global.u8 [%rd152], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p317, %r5904, 255; + selp.b32 %r5906, 7, 8, %p317; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_281: + setp.ne.s32 %p318, %r5516, 0; + mov.u32 %r5526, %r5511; + @%p318 bra $L__BB0_278; + +$L__BB0_282: + and.b32 %r5135, %r5405, 8; + setp.eq.s32 %p319, %r5135, 0; + mov.u32 %r5933, %r5526; + @%p319 bra $L__BB0_289; + + shr.u32 %r3047, %r584, 3; + sub.s32 %r5531, %r583, %r3047; + setp.eq.s32 %p320, %r5531, 0; + mov.u32 %r5933, %r5526; + @%p320 bra $L__BB0_289; + + mov.u32 %r3048, -1; + shl.b32 %r3049, %r3048, %r5531; + not.b32 %r3050, %r3049; + and.b32 %r5532, %r5418, %r3050; + +$L__BB0_285: + setp.gt.u32 %p321, %r5907, 17476; + mov.u32 %r5933, 1; + @%p321 bra $L__BB0_289; + + sub.s32 %r3052, %r5906, %r5905; + min.u32 %r3053, %r3052, %r5531; + setp.eq.s32 %p322, %r3053, 32; + mov.u32 %r3054, -1; + shl.b32 %r3055, %r3054, %r3053; + not.b32 %r3056, %r3055; + selp.b32 %r3057, -1, %r3056, %p322; + and.b32 %r3058, %r3057, %r5532; + shl.b32 %r3059, %r3058, %r5905; + or.b32 %r5904, %r3059, %r5904; + add.s32 %r5905, %r3053, %r5905; + shr.u32 %r5532, %r5532, %r3053; + sub.s32 %r5531, %r5531, %r3053; + setp.lt.u32 %p323, %r5905, %r5906; + @%p323 bra $L__BB0_288; + + cvt.u64.u32 %rd153, %r5907; + add.s64 %rd154, %rd1, %rd153; + st.global.u8 [%rd154], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p324, %r5904, 255; + selp.b32 %r5906, 7, 8, %p324; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_288: + setp.ne.s32 %p325, %r5531, 0; + mov.u32 %r5933, %r5526; + @%p325 bra $L__BB0_285; + +$L__BB0_289: + setp.lt.s32 %p326, %r468, 1; + setp.lt.s32 %p327, %r130, 1; + or.pred %p328, %p327, %p326; + @%p328 bra $L__BB0_337; + + min.s32 %r3062, %r130, %r468; + setp.lt.s32 %p329, %r3062, 3; + add.s32 %r3063, %r5271, 17477; + cvt.u64.u32 %rd155, %r3063; + add.s64 %rd7, %rd1, %rd155; + @%p329 bra $L__BB0_329; + bra.uni $L__BB0_291; + +$L__BB0_329: + add.s32 %r5483, %r5483, 1; + setp.lt.u32 %p376, %r5483, %r5481; + @%p376 bra $L__BB0_337; + + shl.b16 %rs382, %rs705, 1; + or.b16 %rs705, %rs382, 1; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p377, %r5277, 0; + mov.u32 %r5586, %r5480; + @%p377 bra $L__BB0_333; + + setp.gt.u32 %p378, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5586, 1; + @%p378 bra $L__BB0_333; + + and.b16 %rs384, %rs705, 255; + st.global.u8 [%rd7], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p379, %rs384, 255; + selp.b32 %r5277, 7, 8, %p379; + mov.u16 %rs705, 0; + mov.u32 %r5586, %r5480; + +$L__BB0_333: + add.s32 %r3151, %r5482, 1; + min.u32 %r5482, %r3151, 12; + setp.lt.u32 %p380, %r5482, 3; + mov.u32 %r5483, 0; + mov.u32 %r5589, %r5483; + @%p380 bra $L__BB0_336; + + setp.lt.u32 %p381, %r5482, 6; + mov.u32 %r5589, 1; + @%p381 bra $L__BB0_336; + + setp.lt.u32 %p382, %r5482, 9; + setp.eq.s32 %p383, %r5482, 11; + selp.b32 %r3153, 4, 5, %p383; + setp.lt.u32 %p384, %r5482, 11; + selp.b32 %r3154, 3, %r3153, %p384; + selp.b32 %r5589, 2, %r3154, %p382; + +$L__BB0_336: + mov.u32 %r3156, 1; + shl.b32 %r5481, %r3156, %r5589; + mov.u32 %r5480, %r5586; + bra.uni $L__BB0_337; + +$L__BB0_291: + shl.b16 %rs705, %rs705, 1; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p330, %r5277, 0; + mov.u32 %r5579, %r5480; + @%p330 bra $L__BB0_294; + + setp.gt.u32 %p331, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5579, 1; + @%p331 bra $L__BB0_294; + + st.global.u8 [%rd7], %rs705; + add.s32 %r5271, %r5271, 1; + mov.u32 %r5277, 8; + mov.u16 %rs705, 0; + mov.u32 %r5579, %r5480; + +$L__BB0_294: + setp.lt.u32 %p332, %r5482, 3; + mov.u32 %r5549, 0; + @%p332 bra $L__BB0_297; + + setp.lt.u32 %p333, %r5482, 6; + mov.u32 %r5549, 1; + @%p333 bra $L__BB0_297; + + setp.lt.u32 %p334, %r5482, 9; + setp.eq.s32 %p335, %r5482, 11; + selp.b32 %r3069, 4, 5, %p335; + setp.lt.u32 %p336, %r5482, 11; + selp.b32 %r3070, 3, %r3069, %p336; + selp.b32 %r5549, 2, %r3070, %p334; + +$L__BB0_297: + setp.eq.s32 %p337, %r5549, 0; + @%p337 bra $L__BB0_325; + + and.b32 %r686, %r5549, 3; + setp.eq.s32 %p338, %r686, 0; + mov.u32 %r5559, %r5549; + mov.u32 %r5562, %r5579; + @%p338 bra $L__BB0_310; + + add.s32 %r5157, %r5549, -1; + mov.u32 %r3072, 1; + shl.b32 %r3073, %r3072, %r5157; + and.b32 %r3074, %r3073, %r5483; + setp.ne.s32 %p339, %r3074, 0; + selp.u32 %r3075, 1, 0, %p339; + cvt.u32.u16 %r3076, %rs705; + bfi.b32 %r3077, %r3076, %r3075, 1, 8; + cvt.u16.u32 %rs705, %r3077; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p340, %r5277, 0; + mov.u32 %r5562, %r5579; + @%p340 bra $L__BB0_302; + + setp.gt.u32 %p341, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5562, %r3072; + @%p341 bra $L__BB0_302; + + add.s32 %r3081, %r5271, 17477; + cvt.u64.u32 %rd156, %r3081; + add.s64 %rd157, %rd1, %rd156; + st.global.u8 [%rd157], %rs705; + add.s32 %r5271, %r5271, 1; + mov.u32 %r5277, 8; + mov.u16 %rs705, 0; + mov.u32 %r5562, %r5579; + +$L__BB0_302: + and.b32 %r5159, %r5549, 3; + add.s32 %r5559, %r5549, -1; + setp.eq.s32 %p342, %r5159, 1; + mov.u32 %r5579, %r5562; + @%p342 bra $L__BB0_310; + + add.s32 %r5559, %r5549, -2; + mov.u32 %r3082, 1; + shl.b32 %r3083, %r3082, %r5559; + and.b32 %r3084, %r3083, %r5483; + setp.ne.s32 %p343, %r3084, 0; + selp.u32 %r3085, 1, 0, %p343; + cvt.u32.u16 %r3086, %rs705; + bfi.b32 %r3087, %r3086, %r3085, 1, 8; + cvt.u16.u32 %rs705, %r3087; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p344, %r5277, 0; + mov.u32 %r5553, %r5562; + @%p344 bra $L__BB0_306; + + setp.gt.u32 %p345, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5553, %r3082; + @%p345 bra $L__BB0_306; + + add.s32 %r3090, %r5271, 17477; + cvt.u64.u32 %rd158, %r3090; + add.s64 %rd159, %rd1, %rd158; + and.b16 %rs370, %rs705, 255; + st.global.u8 [%rd159], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p346, %rs370, 255; + selp.b32 %r5277, 7, 8, %p346; + mov.u16 %rs705, 0; + mov.u32 %r5553, %r5562; + +$L__BB0_306: + and.b32 %r5160, %r5549, 3; + setp.eq.s32 %p347, %r5160, 2; + mov.u32 %r5579, %r5553; + mov.u32 %r5562, %r5553; + @%p347 bra $L__BB0_310; + + add.s32 %r5559, %r5549, -3; + mov.u32 %r3091, 1; + shl.b32 %r3092, %r3091, %r5559; + and.b32 %r3093, %r3092, %r5483; + setp.ne.s32 %p348, %r3093, 0; + selp.u32 %r3094, 1, 0, %p348; + cvt.u32.u16 %r3095, %rs705; + bfi.b32 %r3096, %r3095, %r3094, 1, 8; + cvt.u16.u32 %rs705, %r3096; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p349, %r5277, 0; + mov.u32 %r5579, %r5553; + mov.u32 %r5562, %r5553; + @%p349 bra $L__BB0_310; + + add.s32 %r5559, %r5549, -3; + setp.gt.u32 %p350, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5579, %r3091; + mov.u32 %r5562, %r3091; + @%p350 bra $L__BB0_310; + + add.s32 %r5559, %r5549, -3; + add.s32 %r3101, %r5271, 17477; + cvt.u64.u32 %rd160, %r3101; + add.s64 %rd161, %rd1, %rd160; + and.b16 %rs373, %rs705, 255; + st.global.u8 [%rd161], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p351, %rs373, 255; + selp.b32 %r5277, 7, 8, %p351; + mov.u16 %rs705, 0; + mov.u32 %r5579, %r5553; + mov.u32 %r5562, %r5553; + +$L__BB0_310: + add.s32 %r5161, %r5549, -1; + setp.lt.u32 %p352, %r5161, 3; + @%p352 bra $L__BB0_325; + + mov.u32 %r5579, %r5562; + +$L__BB0_312: + add.s32 %r3102, %r5559, -1; + mov.u32 %r3103, 1; + shl.b32 %r3104, %r3103, %r3102; + and.b32 %r3105, %r3104, %r5483; + setp.ne.s32 %p353, %r3105, 0; + selp.u32 %r3106, 1, 0, %p353; + cvt.u32.u16 %r3107, %rs705; + bfi.b32 %r5568, %r3107, %r3106, 1, 8; + add.s32 %r5569, %r5277, -1; + setp.ne.s32 %p354, %r5569, 0; + mov.u32 %r5567, %r5579; + @%p354 bra $L__BB0_315; + + setp.gt.u32 %p355, %r5271, 191; + mov.u32 %r5569, 0; + mov.u32 %r5567, %r3103; + @%p355 bra $L__BB0_315; + + cvt.u16.u32 %rs374, %r5568; + and.b16 %rs375, %rs374, 255; + add.s32 %r3111, %r5271, 17477; + cvt.u64.u32 %rd162, %r3111; + add.s64 %rd163, %rd1, %rd162; + st.global.u8 [%rd163], %rs374; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p356, %rs375, 255; + selp.b32 %r5569, 7, 8, %p356; + mov.u32 %r5568, 0; + mov.u32 %r5567, %r5579; + +$L__BB0_315: + add.s32 %r3112, %r5559, -2; + shl.b32 %r3114, %r3103, %r3112; + and.b32 %r3115, %r3114, %r5483; + setp.ne.s32 %p357, %r3115, 0; + and.b32 %r3116, %r5568, 127; + selp.u32 %r3117, 1, 0, %p357; + bfi.b32 %r5572, %r3116, %r3117, 1, 7; + add.s32 %r5573, %r5569, -1; + setp.ne.s32 %p358, %r5573, 0; + mov.u32 %r5571, %r5567; + @%p358 bra $L__BB0_318; + + setp.gt.u32 %p359, %r5271, 191; + mov.u32 %r5573, 0; + mov.u32 %r5571, 1; + @%p359 bra $L__BB0_318; + + cvt.u16.u32 %rs376, %r5572; + and.b16 %rs377, %rs376, 255; + add.s32 %r3121, %r5271, 17477; + cvt.u64.u32 %rd164, %r3121; + add.s64 %rd165, %rd1, %rd164; + st.global.u8 [%rd165], %rs376; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p360, %rs377, 255; + selp.b32 %r5573, 7, 8, %p360; + mov.u32 %r5572, 0; + mov.u32 %r5571, %r5567; + +$L__BB0_318: + add.s32 %r3122, %r5559, -3; + mov.u32 %r3123, 1; + shl.b32 %r3124, %r3123, %r3122; + and.b32 %r3125, %r3124, %r5483; + setp.ne.s32 %p361, %r3125, 0; + and.b32 %r3126, %r5572, 127; + selp.u32 %r3127, 1, 0, %p361; + bfi.b32 %r5576, %r3126, %r3127, 1, 7; + add.s32 %r5577, %r5573, -1; + setp.ne.s32 %p362, %r5577, 0; + mov.u32 %r5575, %r5571; + @%p362 bra $L__BB0_321; + + setp.gt.u32 %p363, %r5271, 191; + mov.u32 %r5577, 0; + mov.u32 %r5575, %r3123; + @%p363 bra $L__BB0_321; + + cvt.u16.u32 %rs378, %r5576; + and.b16 %rs379, %rs378, 255; + add.s32 %r3131, %r5271, 17477; + cvt.u64.u32 %rd166, %r3131; + add.s64 %rd167, %rd1, %rd166; + st.global.u8 [%rd167], %rs378; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p364, %rs379, 255; + selp.b32 %r5577, 7, 8, %p364; + mov.u32 %r5576, 0; + mov.u32 %r5575, %r5571; + +$L__BB0_321: + add.s32 %r5559, %r5559, -4; + shl.b32 %r3133, %r3123, %r5559; + and.b32 %r3134, %r3133, %r5483; + setp.ne.s32 %p365, %r3134, 0; + and.b32 %r3135, %r5576, 127; + selp.u32 %r3136, 1, 0, %p365; + bfi.b32 %r3137, %r3135, %r3136, 1, 15; + cvt.u16.u32 %rs705, %r3137; + add.s32 %r5277, %r5577, -1; + setp.ne.s32 %p366, %r5277, 0; + mov.u32 %r5579, %r5575; + @%p366 bra $L__BB0_324; + + setp.gt.u32 %p367, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5579, 1; + @%p367 bra $L__BB0_324; + + add.s32 %r3140, %r5271, 17477; + cvt.u64.u32 %rd168, %r3140; + add.s64 %rd169, %rd1, %rd168; + and.b16 %rs381, %rs705, 255; + st.global.u8 [%rd169], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p368, %rs381, 255; + selp.b32 %r5277, 7, 8, %p368; + mov.u16 %rs705, 0; + mov.u32 %r5579, %r5575; + +$L__BB0_324: + setp.ne.s32 %p369, %r5559, 0; + @%p369 bra $L__BB0_312; + +$L__BB0_325: + add.s32 %r3142, %r5482, -1; + setp.eq.s32 %p370, %r5482, 0; + mov.u32 %r5483, 0; + selp.b32 %r5482, 0, %r3142, %p370; + setp.lt.u32 %p371, %r5482, 3; + mov.u32 %r5585, %r5483; + @%p371 bra $L__BB0_328; + + setp.lt.u32 %p372, %r5482, 6; + mov.u32 %r5585, 1; + @%p372 bra $L__BB0_328; + + setp.lt.u32 %p373, %r5482, 9; + setp.eq.s32 %p374, %r5482, 11; + selp.b32 %r3144, 4, 5, %p374; + setp.lt.u32 %p375, %r5482, 11; + selp.b32 %r3145, 3, %r3144, %p375; + selp.b32 %r5585, 2, %r3145, %p373; + +$L__BB0_328: + mov.u32 %r3147, 1; + shl.b32 %r5481, %r3147, %r5585; + mov.u32 %r5480, %r5579; + +$L__BB0_337: + setp.gt.s32 %p385, %r468, 2; + setp.gt.s32 %p386, %r130, 2; + and.pred %p387, %p386, %p385; + @%p387 bra $L__BB0_387; + bra.uni $L__BB0_338; + +$L__BB0_387: + add.s32 %r3279, %r340, -11; + cvt.u64.u32 %rd200, %r3279; + add.s64 %rd9, %rd108, %rd200; + ld.global.u8 %rs121, [%rd9]; + add.s32 %r3280, %r340, -10; + cvt.u64.u32 %rd202, %r3280; + add.s64 %rd203, %rd108, %rd202; + ld.global.u8 %rs122, [%rd203]; + ld.global.u8 %rs123, [%rd203+1]; + mul.lo.s32 %r3281, %r468, 6; + add.s32 %r3282, %r3281, -12; + cvt.u64.u32 %rd204, %r3282; + add.s64 %rd205, %rd108, %rd204; + ld.global.u8 %rs124, [%rd205]; + ld.global.u8 %rs125, [%rd205+1]; + add.s32 %r3283, %r3281, -10; + cvt.u64.u32 %rd206, %r3283; + add.s64 %rd207, %rd108, %rd206; + ld.global.u8 %rs126, [%rd207]; + ld.global.u8 %rs127, [%rd207+1]; + setp.eq.s16 %p455, %rs121, 0; + mov.u32 %r5683, %r5435; + @%p455 bra $L__BB0_394; + + ld.global.u8 %r5673, [%rd9+-1]; + cvt.u32.u16 %r5672, %rs121; + +$L__BB0_389: + mov.u16 %rs128, %rs688; + mov.u32 %r895, %r5672; + setp.gt.u32 %p456, %r5719, 2879; + mov.u32 %r5683, 1; + @%p456 bra $L__BB0_394; + + mov.u32 %r3285, 8; + sub.s32 %r3286, %r3285, %r5717; + sub.s32 %r3287, %r3286, %r5718; + min.u32 %r3288, %r3287, %r895; + setp.eq.s32 %p457, %r3288, 32; + mov.u32 %r3289, -1; + shl.b32 %r3290, %r3289, %r3288; + not.b32 %r3291, %r3290; + selp.b32 %r3292, -1, %r3291, %p457; + and.b32 %r3293, %r3292, %r5673; + shl.b32 %r3294, %r3293, %r5718; + cvt.u16.u32 %rs417, %r3294; + or.b16 %rs688, %rs128, %rs417; + add.s32 %r5718, %r3288, %r5718; + sub.s32 %r5672, %r895, %r3288; + shr.u32 %r5673, %r5673, %r3288; + setp.gt.u32 %p458, %r3287, %r895; + @%p458 bra $L__BB0_393; + + setp.ne.s32 %p459, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs418, %rs688, 255; + setp.ne.s16 %p460, %rs418, 127; + and.pred %p461, %p459, %p460; + @%p461 bra $L__BB0_393; + + cvt.u16.u32 %rs608, %r3294; + or.b16 %rs607, %rs128, %rs608; + mov.u32 %r3297, 20548; + sub.s32 %r3298, %r3297, %r5719; + cvt.u64.u32 %rd208, %r3298; + add.s64 %rd209, %rd1, %rd208; + st.global.u8 [%rd209], %rs607; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p462, %rs418, 143; + selp.u32 %r5717, 1, 0, %p462; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_393: + setp.ne.s32 %p463, %r5672, 0; + mov.u32 %r5683, %r5435; + @%p463 bra $L__BB0_389; + +$L__BB0_394: + setp.eq.s16 %p464, %rs125, 0; + mov.u32 %r5695, %r5683; + @%p464 bra $L__BB0_401; + + cvt.u32.u16 %r3299, %rs124; + and.b32 %r5685, %r3299, 255; + cvt.u32.u16 %r3300, %rs125; + and.b32 %r5684, %r3300, 255; + +$L__BB0_396: + mov.u32 %r914, %r5684; + setp.gt.u32 %p465, %r5719, 2879; + mov.u32 %r5695, 1; + @%p465 bra $L__BB0_401; + + mov.u32 %r3302, 8; + sub.s32 %r3303, %r3302, %r5717; + sub.s32 %r3304, %r3303, %r5718; + min.u32 %r3305, %r3304, %r914; + setp.eq.s32 %p466, %r3305, 32; + mov.u32 %r3306, -1; + shl.b32 %r3307, %r3306, %r3305; + not.b32 %r3308, %r3307; + selp.b32 %r3309, -1, %r3308, %p466; + and.b32 %r3310, %r3309, %r5685; + shl.b32 %r3311, %r3310, %r5718; + cvt.u16.u32 %rs422, %r3311; + or.b16 %rs688, %rs688, %rs422; + add.s32 %r5718, %r3305, %r5718; + sub.s32 %r5684, %r914, %r3305; + shr.u32 %r5685, %r5685, %r3305; + setp.gt.u32 %p467, %r3304, %r914; + @%p467 bra $L__BB0_400; + + setp.ne.s32 %p468, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs423, %rs688, 255; + setp.ne.s16 %p469, %rs423, 127; + and.pred %p470, %p468, %p469; + @%p470 bra $L__BB0_400; + + mov.u32 %r3314, 20548; + sub.s32 %r3315, %r3314, %r5719; + cvt.u64.u32 %rd210, %r3315; + add.s64 %rd211, %rd1, %rd210; + st.global.u8 [%rd211], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p471, %rs423, 143; + selp.u32 %r5717, 1, 0, %p471; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_400: + setp.ne.s32 %p472, %r5684, 0; + mov.u32 %r5695, %r5683; + @%p472 bra $L__BB0_396; + +$L__BB0_401: + setp.eq.s16 %p473, %rs123, 0; + mov.u32 %r5707, %r5695; + @%p473 bra $L__BB0_408; + + cvt.u32.u16 %r3316, %rs123; + and.b32 %r5696, %r3316, 255; + cvt.u32.u16 %r3317, %rs122; + and.b32 %r5697, %r3317, 255; + +$L__BB0_403: + mov.u32 %r933, %r5696; + setp.gt.u32 %p474, %r5719, 2879; + mov.u32 %r5707, 1; + @%p474 bra $L__BB0_408; + + mov.u32 %r3319, 8; + sub.s32 %r3320, %r3319, %r5717; + sub.s32 %r3321, %r3320, %r5718; + min.u32 %r3322, %r3321, %r933; + setp.eq.s32 %p475, %r3322, 32; + mov.u32 %r3323, -1; + shl.b32 %r3324, %r3323, %r3322; + not.b32 %r3325, %r3324; + selp.b32 %r3326, -1, %r3325, %p475; + and.b32 %r3327, %r3326, %r5697; + shl.b32 %r3328, %r3327, %r5718; + cvt.u16.u32 %rs427, %r3328; + or.b16 %rs688, %rs688, %rs427; + add.s32 %r5718, %r3322, %r5718; + sub.s32 %r5696, %r933, %r3322; + shr.u32 %r5697, %r5697, %r3322; + setp.gt.u32 %p476, %r3321, %r933; + @%p476 bra $L__BB0_407; + + setp.ne.s32 %p477, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs428, %rs688, 255; + setp.ne.s16 %p478, %rs428, 127; + and.pred %p479, %p477, %p478; + @%p479 bra $L__BB0_407; + + mov.u32 %r3331, 20548; + sub.s32 %r3332, %r3331, %r5719; + cvt.u64.u32 %rd212, %r3332; + add.s64 %rd213, %rd1, %rd212; + st.global.u8 [%rd213], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p480, %rs428, 143; + selp.u32 %r5717, 1, 0, %p480; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_407: + setp.ne.s32 %p481, %r5696, 0; + mov.u32 %r5707, %r5695; + @%p481 bra $L__BB0_403; + +$L__BB0_408: + setp.eq.s16 %p482, %rs127, 0; + mov.u32 %r5716, %r5707; + @%p482 bra $L__BB0_415; + + cvt.u32.u16 %r3333, %rs126; + and.b32 %r5709, %r3333, 255; + cvt.u32.u16 %r3334, %rs127; + and.b32 %r5708, %r3334, 255; + +$L__BB0_410: + mov.u32 %r952, %r5708; + setp.gt.u32 %p483, %r5719, 2879; + mov.u32 %r5716, 1; + @%p483 bra $L__BB0_415; + + mov.u32 %r3336, 8; + sub.s32 %r3337, %r3336, %r5717; + sub.s32 %r3338, %r3337, %r5718; + min.u32 %r3339, %r3338, %r952; + setp.eq.s32 %p484, %r3339, 32; + mov.u32 %r3340, -1; + shl.b32 %r3341, %r3340, %r3339; + not.b32 %r3342, %r3341; + selp.b32 %r3343, -1, %r3342, %p484; + and.b32 %r3344, %r3343, %r5709; + shl.b32 %r3345, %r3344, %r5718; + cvt.u16.u32 %rs432, %r3345; + or.b16 %rs688, %rs688, %rs432; + add.s32 %r5718, %r3339, %r5718; + sub.s32 %r5708, %r952, %r3339; + shr.u32 %r5709, %r5709, %r3339; + setp.gt.u32 %p485, %r3338, %r952; + @%p485 bra $L__BB0_414; + + setp.ne.s32 %p486, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs433, %rs688, 255; + setp.ne.s16 %p487, %rs433, 127; + and.pred %p488, %p486, %p487; + @%p488 bra $L__BB0_414; + + mov.u32 %r3348, 20548; + sub.s32 %r3349, %r3348, %r5719; + cvt.u64.u32 %rd214, %r3349; + add.s64 %rd215, %rd1, %rd214; + st.global.u8 [%rd215], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p489, %rs433, 143; + selp.u32 %r5717, 1, 0, %p489; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_414: + setp.ne.s32 %p490, %r5708, 0; + mov.u32 %r5716, %r5707; + @%p490 bra $L__BB0_410; + bra.uni $L__BB0_415; + +$L__BB0_338: + setp.gt.s32 %p388, %r468, 0; + and.pred %p390, %p386, %p388; + @%p390 bra $L__BB0_367; + bra.uni $L__BB0_339; + +$L__BB0_367: + cvt.u64.u32 %rd186, %r340; + add.s64 %rd188, %rd108, %rd186; + ld.global.u8 %rs107, [%rd188+1]; + add.s32 %r3229, %r340, 2; + cvt.u64.u32 %rd189, %r3229; + add.s64 %rd190, %rd108, %rd189; + ld.global.u8 %rs108, [%rd190]; + ld.global.u8 %rs109, [%rd190+1]; + setp.eq.s16 %p429, %rs107, 0; + mov.u32 %r5651, %r5435; + @%p429 bra $L__BB0_374; + + ld.global.u8 %r5641, [%rd188]; + cvt.u32.u16 %r5640, %rs107; + +$L__BB0_369: + mov.u32 %r843, %r5640; + setp.gt.u32 %p430, %r5719, 2879; + mov.u32 %r5651, 1; + @%p430 bra $L__BB0_374; + + mov.u32 %r3231, 8; + sub.s32 %r3232, %r3231, %r5717; + sub.s32 %r3233, %r3232, %r5718; + min.u32 %r3234, %r3233, %r843; + setp.eq.s32 %p431, %r3234, 32; + mov.u32 %r3235, -1; + shl.b32 %r3236, %r3235, %r3234; + not.b32 %r3237, %r3236; + selp.b32 %r3238, -1, %r3237, %p431; + and.b32 %r3239, %r3238, %r5641; + shl.b32 %r3240, %r3239, %r5718; + cvt.u16.u32 %rs404, %r3240; + or.b16 %rs688, %rs688, %rs404; + add.s32 %r5718, %r3234, %r5718; + sub.s32 %r5640, %r843, %r3234; + shr.u32 %r5641, %r5641, %r3234; + setp.gt.u32 %p432, %r3233, %r843; + @%p432 bra $L__BB0_373; + + setp.ne.s32 %p433, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs405, %rs688, 255; + setp.ne.s16 %p434, %rs405, 127; + and.pred %p435, %p433, %p434; + @%p435 bra $L__BB0_373; + + mov.u32 %r3243, 20548; + sub.s32 %r3244, %r3243, %r5719; + cvt.u64.u32 %rd194, %r3244; + add.s64 %rd195, %rd1, %rd194; + st.global.u8 [%rd195], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p436, %rs405, 143; + selp.u32 %r5717, 1, 0, %p436; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_373: + setp.ne.s32 %p437, %r5640, 0; + mov.u32 %r5651, %r5435; + @%p437 bra $L__BB0_369; + +$L__BB0_374: + add.s32 %r5653, %r468, -1; + mov.u32 %r5652, 1; + +$L__BB0_375: + mov.u32 %r863, %r5652; + mov.u32 %r5663, 1; + setp.gt.u32 %p438, %r5719, 2879; + @%p438 bra $L__BB0_380; + + mov.u32 %r3249, 8; + sub.s32 %r3250, %r3249, %r5717; + sub.s32 %r3251, %r3250, %r5718; + min.u32 %r3252, %r3251, %r863; + setp.eq.s32 %p439, %r3252, 32; + mov.u32 %r3253, -1; + shl.b32 %r3254, %r3253, %r3252; + not.b32 %r3255, %r3254; + selp.b32 %r3256, -1, %r3255, %p439; + and.b32 %r3257, %r3256, %r5653; + shl.b32 %r3258, %r3257, %r5718; + cvt.u16.u32 %rs408, %r3258; + or.b16 %rs688, %rs688, %rs408; + add.s32 %r5718, %r3252, %r5718; + sub.s32 %r5652, %r863, %r3252; + shr.u32 %r5653, %r5653, %r3252; + setp.gt.u32 %p440, %r3251, %r863; + @%p440 bra $L__BB0_379; + + setp.ne.s32 %p441, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs409, %rs688, 255; + setp.ne.s16 %p442, %rs409, 127; + and.pred %p443, %p441, %p442; + @%p443 bra $L__BB0_379; + + mov.u32 %r3261, 20548; + sub.s32 %r3262, %r3261, %r5719; + cvt.u64.u32 %rd196, %r3262; + add.s64 %rd197, %rd1, %rd196; + st.global.u8 [%rd197], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p444, %rs409, 143; + selp.u32 %r5717, 1, 0, %p444; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_379: + setp.ne.s32 %p445, %r5652, 0; + mov.u32 %r5663, %r5651; + @%p445 bra $L__BB0_375; + +$L__BB0_380: + setp.eq.s16 %p446, %rs109, 0; + mov.u32 %r5716, %r5663; + @%p446 bra $L__BB0_415; + + cvt.u32.u16 %r5165, %rs108; + and.b32 %r5665, %r5165, 255; + cvt.u32.u16 %r5152, %rs109; + and.b32 %r5664, %r5152, 255; + +$L__BB0_382: + mov.u32 %r880, %r5664; + setp.gt.u32 %p447, %r5719, 2879; + mov.u32 %r5716, 1; + @%p447 bra $L__BB0_415; + + mov.u32 %r3264, 8; + sub.s32 %r3265, %r3264, %r5717; + sub.s32 %r3266, %r3265, %r5718; + min.u32 %r3267, %r3266, %r880; + setp.eq.s32 %p448, %r3267, 32; + mov.u32 %r3268, -1; + shl.b32 %r3269, %r3268, %r3267; + not.b32 %r3270, %r3269; + selp.b32 %r3271, -1, %r3270, %p448; + and.b32 %r3272, %r3271, %r5665; + shl.b32 %r3273, %r3272, %r5718; + cvt.u16.u32 %rs413, %r3273; + or.b16 %rs688, %rs688, %rs413; + add.s32 %r5718, %r3267, %r5718; + sub.s32 %r5664, %r880, %r3267; + shr.u32 %r5665, %r5665, %r3267; + setp.gt.u32 %p449, %r3266, %r880; + @%p449 bra $L__BB0_386; + + setp.ne.s32 %p450, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs414, %rs688, 255; + setp.ne.s16 %p451, %rs414, 127; + and.pred %p452, %p450, %p451; + @%p452 bra $L__BB0_386; + + mov.u32 %r3276, 20548; + sub.s32 %r3277, %r3276, %r5719; + cvt.u64.u32 %rd198, %r3277; + add.s64 %rd199, %rd1, %rd198; + st.global.u8 [%rd199], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p453, %rs414, 143; + selp.u32 %r5717, 1, 0, %p453; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_386: + setp.eq.s32 %p454, %r5664, 0; + mov.u32 %r5716, %r5663; + @%p454 bra $L__BB0_415; + bra.uni $L__BB0_382; + +$L__BB0_339: + setp.gt.s32 %p392, %r130, 0; + selp.b32 %r3158, %r340, 0, %p392; + cvt.u64.u32 %rd170, %r3158; + add.s64 %rd8, %rd108, %rd170; + ld.global.u8 %rs85, [%rd8+1]; + add.s32 %r3159, %r3158, 2; + cvt.u64.u32 %rd172, %r3159; + add.s64 %rd173, %rd108, %rd172; + ld.global.u8 %rs86, [%rd173]; + ld.global.u8 %rs87, [%rd173+1]; + mul.lo.s32 %r3160, %r468, 6; + selp.b32 %r3161, %r3160, 0, %p388; + cvt.u64.u32 %rd174, %r3161; + add.s64 %rd175, %rd108, %rd174; + ld.global.u8 %rs88, [%rd175]; + ld.global.u8 %rs89, [%rd175+1]; + add.s32 %r3162, %r3161, 2; + cvt.u64.u32 %rd176, %r3162; + add.s64 %rd177, %rd108, %rd176; + ld.global.u8 %rs90, [%rd177]; + ld.global.u8 %rs91, [%rd177+1]; + setp.eq.s16 %p393, %rs85, 0; + mov.u32 %r5607, %r5435; + @%p393 bra $L__BB0_346; + + ld.global.u8 %r5597, [%rd8]; + cvt.u32.u16 %r5596, %rs85; + +$L__BB0_341: + mov.u16 %rs92, %rs688; + mov.u32 %r771, %r5596; + setp.gt.u32 %p394, %r5719, 2879; + mov.u32 %r5607, 1; + @%p394 bra $L__BB0_346; + + mov.u32 %r3164, 8; + sub.s32 %r3165, %r3164, %r5717; + sub.s32 %r3166, %r3165, %r5718; + min.u32 %r3167, %r3166, %r771; + setp.eq.s32 %p395, %r3167, 32; + mov.u32 %r3168, -1; + shl.b32 %r3169, %r3168, %r3167; + not.b32 %r3170, %r3169; + selp.b32 %r3171, -1, %r3170, %p395; + and.b32 %r3172, %r3171, %r5597; + shl.b32 %r3173, %r3172, %r5718; + cvt.u16.u32 %rs385, %r3173; + or.b16 %rs688, %rs92, %rs385; + add.s32 %r5718, %r3167, %r5718; + sub.s32 %r5596, %r771, %r3167; + shr.u32 %r5597, %r5597, %r3167; + setp.gt.u32 %p396, %r3166, %r771; + @%p396 bra $L__BB0_345; + + setp.ne.s32 %p397, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs386, %rs688, 255; + setp.ne.s16 %p398, %rs386, 127; + and.pred %p399, %p397, %p398; + @%p399 bra $L__BB0_345; + + cvt.u16.u32 %rs606, %r3173; + or.b16 %rs605, %rs92, %rs606; + mov.u32 %r3176, 20548; + sub.s32 %r3177, %r3176, %r5719; + cvt.u64.u32 %rd178, %r3177; + add.s64 %rd179, %rd1, %rd178; + st.global.u8 [%rd179], %rs605; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p400, %rs386, 143; + selp.u32 %r5717, 1, 0, %p400; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_345: + setp.ne.s32 %p401, %r5596, 0; + mov.u32 %r5607, %r5435; + @%p401 bra $L__BB0_341; + +$L__BB0_346: + setp.eq.s16 %p402, %rs89, 0; + mov.u32 %r5619, %r5607; + @%p402 bra $L__BB0_353; + + cvt.u32.u16 %r3178, %rs88; + and.b32 %r5609, %r3178, 255; + cvt.u32.u16 %r3179, %rs89; + and.b32 %r5608, %r3179, 255; + +$L__BB0_348: + mov.u32 %r790, %r5608; + setp.gt.u32 %p403, %r5719, 2879; + mov.u32 %r5619, 1; + @%p403 bra $L__BB0_353; + + mov.u32 %r3181, 8; + sub.s32 %r3182, %r3181, %r5717; + sub.s32 %r3183, %r3182, %r5718; + min.u32 %r3184, %r3183, %r790; + setp.eq.s32 %p404, %r3184, 32; + mov.u32 %r3185, -1; + shl.b32 %r3186, %r3185, %r3184; + not.b32 %r3187, %r3186; + selp.b32 %r3188, -1, %r3187, %p404; + and.b32 %r3189, %r3188, %r5609; + shl.b32 %r3190, %r3189, %r5718; + cvt.u16.u32 %rs390, %r3190; + or.b16 %rs688, %rs688, %rs390; + add.s32 %r5718, %r3184, %r5718; + sub.s32 %r5608, %r790, %r3184; + shr.u32 %r5609, %r5609, %r3184; + setp.gt.u32 %p405, %r3183, %r790; + @%p405 bra $L__BB0_352; + + setp.ne.s32 %p406, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs391, %rs688, 255; + setp.ne.s16 %p407, %rs391, 127; + and.pred %p408, %p406, %p407; + @%p408 bra $L__BB0_352; + + mov.u32 %r3193, 20548; + sub.s32 %r3194, %r3193, %r5719; + cvt.u64.u32 %rd180, %r3194; + add.s64 %rd181, %rd1, %rd180; + st.global.u8 [%rd181], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p409, %rs391, 143; + selp.u32 %r5717, 1, 0, %p409; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_352: + setp.ne.s32 %p410, %r5608, 0; + mov.u32 %r5619, %r5607; + @%p410 bra $L__BB0_348; + +$L__BB0_353: + setp.eq.s16 %p411, %rs87, 0; + mov.u32 %r5631, %r5619; + @%p411 bra $L__BB0_360; + + cvt.u32.u16 %r3195, %rs87; + and.b32 %r5620, %r3195, 255; + cvt.u32.u16 %r3196, %rs86; + and.b32 %r5621, %r3196, 255; + +$L__BB0_355: + mov.u32 %r809, %r5620; + setp.gt.u32 %p412, %r5719, 2879; + mov.u32 %r5631, 1; + @%p412 bra $L__BB0_360; + + mov.u32 %r3198, 8; + sub.s32 %r3199, %r3198, %r5717; + sub.s32 %r3200, %r3199, %r5718; + min.u32 %r3201, %r3200, %r809; + setp.eq.s32 %p413, %r3201, 32; + mov.u32 %r3202, -1; + shl.b32 %r3203, %r3202, %r3201; + not.b32 %r3204, %r3203; + selp.b32 %r3205, -1, %r3204, %p413; + and.b32 %r3206, %r3205, %r5621; + shl.b32 %r3207, %r3206, %r5718; + cvt.u16.u32 %rs395, %r3207; + or.b16 %rs688, %rs688, %rs395; + add.s32 %r5718, %r3201, %r5718; + sub.s32 %r5620, %r809, %r3201; + shr.u32 %r5621, %r5621, %r3201; + setp.gt.u32 %p414, %r3200, %r809; + @%p414 bra $L__BB0_359; + + setp.ne.s32 %p415, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs396, %rs688, 255; + setp.ne.s16 %p416, %rs396, 127; + and.pred %p417, %p415, %p416; + @%p417 bra $L__BB0_359; + + mov.u32 %r3210, 20548; + sub.s32 %r3211, %r3210, %r5719; + cvt.u64.u32 %rd182, %r3211; + add.s64 %rd183, %rd1, %rd182; + st.global.u8 [%rd183], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p418, %rs396, 143; + selp.u32 %r5717, 1, 0, %p418; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_359: + setp.ne.s32 %p419, %r5620, 0; + mov.u32 %r5631, %r5619; + @%p419 bra $L__BB0_355; + +$L__BB0_360: + setp.eq.s16 %p420, %rs91, 0; + mov.u32 %r5716, %r5631; + @%p420 bra $L__BB0_415; + + cvt.u32.u16 %r3212, %rs90; + and.b32 %r5633, %r3212, 255; + cvt.u32.u16 %r3213, %rs91; + and.b32 %r5632, %r3213, 255; + +$L__BB0_362: + mov.u32 %r828, %r5632; + setp.gt.u32 %p421, %r5719, 2879; + mov.u32 %r5716, 1; + @%p421 bra $L__BB0_415; + + mov.u32 %r3215, 8; + sub.s32 %r3216, %r3215, %r5717; + sub.s32 %r3217, %r3216, %r5718; + min.u32 %r3218, %r3217, %r828; + setp.eq.s32 %p422, %r3218, 32; + mov.u32 %r3219, -1; + shl.b32 %r3220, %r3219, %r3218; + not.b32 %r3221, %r3220; + selp.b32 %r3222, -1, %r3221, %p422; + and.b32 %r3223, %r3222, %r5633; + shl.b32 %r3224, %r3223, %r5718; + cvt.u16.u32 %rs400, %r3224; + or.b16 %rs688, %rs688, %rs400; + add.s32 %r5718, %r3218, %r5718; + sub.s32 %r5632, %r828, %r3218; + shr.u32 %r5633, %r5633, %r3218; + setp.gt.u32 %p423, %r3217, %r828; + @%p423 bra $L__BB0_366; + + setp.ne.s32 %p424, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs401, %rs688, 255; + setp.ne.s16 %p425, %rs401, 127; + and.pred %p426, %p424, %p425; + @%p426 bra $L__BB0_366; + + mov.u32 %r3227, 20548; + sub.s32 %r3228, %r3227, %r5719; + cvt.u64.u32 %rd184, %r3228; + add.s64 %rd185, %rd1, %rd184; + st.global.u8 [%rd185], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p427, %rs401, 143; + selp.u32 %r5717, 1, 0, %p427; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_366: + setp.eq.s32 %p428, %r5632, 0; + mov.u32 %r5716, %r5631; + @%p428 bra $L__BB0_415; + bra.uni $L__BB0_362; + +$L__BB0_415: + and.b32 %r5149, %r5405, 1; + shr.u32 %r3350, %r5405, 1; + or.b32 %r5736, %r3350, %r5149; + +$L__BB0_416: + add.s32 %r5186, %r5186, 4; + setp.lt.u32 %p491, %r5186, %r5; + @%p491 bra $L__BB0_50; + +$L__BB0_417: + add.s32 %r5108, %r5, 1; + shr.u32 %r5107, %r5108, 1; + add.s32 %r1003, %r5107, 1; + setp.gt.u32 %p492, %r1003, 512; + @%p492 bra $L__BB0_419; + + mov.u32 %r3353, _ZZ31 j2k_htj2k_encode_codeblockE13cleanup_e_val; + add.s32 %r3354, %r3353, %r1003; + mov.u16 %rs436, 0; + st.shared.u8 [%r3354], %rs436; + +$L__BB0_419: + setp.lt.u32 %p493, %r6, 3; + @%p493 bra $L__BB0_665; + + ld.param.u64 %rd639, [ j2k_htj2k_encode_codeblock_param_4]; + ld.param.u64 %rd638, [ j2k_htj2k_encode_codeblock_param_5]; + mov.u32 %r3356, 31; + sub.s32 %r1004, %r3356, %r2; + mov.u32 %r5752, 2; + cvta.to.global.u64 %rd10, %rd638; + cvta.to.global.u64 %rd11, %rd639; + +$L__BB0_421: + ld.shared.u8 %rs150, [_ZZ31 j2k_htj2k_encode_codeblockE13cleanup_e_val]; + mov.u16 %rs437, 0; + st.shared.u8 [_ZZ31 j2k_htj2k_encode_codeblockE13cleanup_e_val], %rs437; + ld.shared.u8 %rs151, [_ZZ31 j2k_htj2k_encode_codeblockE14cleanup_cx_val]; + st.shared.u8 [_ZZ31 j2k_htj2k_encode_codeblockE14cleanup_cx_val], %rs437; + @%p9 bra $L__BB0_664; + + mov.u32 %r3359, 0; + ld.shared.u8 %rs438, [_ZZ31 j2k_htj2k_encode_codeblockE13cleanup_e_val+1]; + ld.shared.u8 %rs439, [_ZZ31 j2k_htj2k_encode_codeblockE14cleanup_cx_val+1]; + max.u16 %rs441, %rs150, %rs438; + cvt.u32.u16 %r3360, %rs441; + add.s32 %r5770, %r3360, -1; + add.s32 %r1022, %r5752, 1; + mul.lo.s32 %r5772, %r5752, %r1; + mul.wide.u16 %r3361, %rs439, 4; + cvt.u32.u16 %r3362, %rs151; + and.b32 %r3363, %r3362, 255; + add.s32 %r5773, %r3361, %r3363; + mov.u32 %r5768, %r3359; + mov.u32 %r5769, %r3359; + mov.u32 %r5771, %r3359; + +$L__BB0_423: + mul.wide.u32 %rd216, %r5772, 4; + add.s64 %rd217, %rd2, %rd216; + ld.global.u32 %r1046, [%rd217]; + setp.eq.s32 %p495, %r1046, 0; + mov.u32 %r5789, %r3359; + @%p495 bra $L__BB0_425; + + and.b32 %r3365, %r1046, -2147483648; + abs.s32 %r3366, %r1046; + shl.b32 %r3367, %r3366, %r1004; + or.b32 %r5789, %r3367, %r3365; + +$L__BB0_425: + shl.b32 %r3371, %r5789, 1; + shr.u32 %r3372, %r3371, %r45; + and.b32 %r1049, %r3372, -2; + setp.eq.s32 %p496, %r1049, 0; + mov.u32 %r5793, 0; + mov.u32 %r5790, %r5793; + mov.u32 %r5791, %r5793; + mov.u32 %r5797, %r5793; + @%p496 bra $L__BB0_427; + + add.s32 %r3374, %r1049, -1; + clz.b32 %r3375, %r3374; + mov.u32 %r3376, 32; + sub.s32 %r5790, %r3376, %r3375; + shr.u32 %r3377, %r5789, 31; + add.s32 %r3378, %r3377, %r1049; + add.s32 %r5791, %r3378, -2; + mov.u32 %r5797, 1; + +$L__BB0_427: + setp.ge.u32 %p497, %r1022, %r6; + @%p497 bra $L__BB0_430; + + add.s32 %r3381, %r5772, %r1; + mul.wide.u32 %rd218, %r3381, 4; + add.s64 %rd219, %rd2, %rd218; + ld.global.u32 %r1055, [%rd219]; + setp.eq.s32 %p498, %r1055, 0; + @%p498 bra $L__BB0_430; + + and.b32 %r3382, %r1055, -2147483648; + abs.s32 %r3383, %r1055; + shl.b32 %r3384, %r3383, %r1004; + or.b32 %r5793, %r3384, %r3382; + +$L__BB0_430: + shl.b32 %r3387, %r5793, 1; + shr.u32 %r3388, %r3387, %r45; + and.b32 %r1058, %r3388, -2; + setp.eq.s32 %p499, %r1058, 0; + mov.u32 %r5808, 0; + mov.u32 %r5794, %r5808; + mov.u32 %r5795, %r5808; + mov.u32 %r5812, %r5790; + @%p499 bra $L__BB0_432; + + or.b32 %r5797, %r5797, 2; + add.s32 %r3389, %r1058, -1; + clz.b32 %r3390, %r3389; + mov.u32 %r3391, 32; + sub.s32 %r5794, %r3391, %r3390; + max.s32 %r5812, %r5790, %r5794; + shr.u32 %r3392, %r5793, 31; + add.s32 %r3393, %r3392, %r1058; + add.s32 %r5795, %r3393, -2; + +$L__BB0_432: + add.s32 %r6102, %r5772, 1; + add.s32 %r3398, %r5768, 1; + setp.ge.u32 %p500, %r3398, %r5; + mov.u32 %r5809, %r5808; + mov.u32 %r5810, %r5808; + mov.u32 %r5811, %r5808; + @%p500 bra $L__BB0_443; + + mul.wide.u32 %rd220, %r6102, 4; + add.s64 %rd221, %rd2, %rd220; + ld.global.u32 %r1068, [%rd221]; + setp.eq.s32 %p501, %r1068, 0; + mov.u32 %r5809, 0; + mov.u32 %r5798, %r5809; + @%p501 bra $L__BB0_435; + + and.b32 %r3400, %r1068, -2147483648; + abs.s32 %r3401, %r1068; + shl.b32 %r3402, %r3401, %r1004; + or.b32 %r5798, %r3402, %r3400; + +$L__BB0_435: + shl.b32 %r3405, %r5798, 1; + shr.u32 %r3406, %r3405, %r45; + and.b32 %r1071, %r3406, -2; + setp.eq.s32 %p502, %r1071, 0; + mov.u32 %r5811, %r5809; + @%p502 bra $L__BB0_437; + + or.b32 %r5797, %r5797, 4; + add.s32 %r3407, %r1071, -1; + clz.b32 %r3408, %r3407; + mov.u32 %r3409, 32; + sub.s32 %r5809, %r3409, %r3408; + max.s32 %r5812, %r5812, %r5809; + shr.u32 %r3410, %r5798, 31; + add.s32 %r3411, %r3410, %r1071; + add.s32 %r5811, %r3411, -2; + +$L__BB0_437: + mov.u32 %r5808, 0; + mov.u32 %r5803, %r5808; + @%p497 bra $L__BB0_440; + + add.s32 %r3414, %r6102, %r1; + mul.wide.u32 %rd222, %r3414, 4; + add.s64 %rd223, %rd2, %rd222; + ld.global.u32 %r1080, [%rd223]; + setp.eq.s32 %p504, %r1080, 0; + @%p504 bra $L__BB0_440; + + and.b32 %r3415, %r1080, -2147483648; + abs.s32 %r3416, %r1080; + shl.b32 %r3417, %r3416, %r1004; + or.b32 %r5803, %r3417, %r3415; + +$L__BB0_440: + shl.b32 %r3420, %r5803, 1; + shr.u32 %r3421, %r3420, %r45; + and.b32 %r1083, %r3421, -2; + setp.eq.s32 %p505, %r1083, 0; + mov.u32 %r5810, %r5808; + @%p505 bra $L__BB0_442; + + or.b32 %r5797, %r5797, 8; + add.s32 %r3422, %r1083, -1; + clz.b32 %r3423, %r3422; + mov.u32 %r3424, 32; + sub.s32 %r5808, %r3424, %r3423; + max.s32 %r5812, %r5812, %r5808; + shr.u32 %r3425, %r5803, 31; + add.s32 %r3426, %r3425, %r1083; + add.s32 %r5810, %r3426, -2; + +$L__BB0_442: + add.s32 %r6102, %r5772, 2; + +$L__BB0_443: + add.s32 %r3428, %r5797, -1; + and.b32 %r3429, %r3428, %r5797; + setp.ne.s32 %p506, %r3429, 0; + mov.u32 %r5815, 0; + setp.gt.s32 %p507, %r5770, 1; + and.pred %p508, %p507, %p506; + selp.b32 %r3430, %r5770, 1, %p508; + max.s32 %r1100, %r3430, %r5812; + sub.s32 %r1101, %r1100, %r3430; + setp.lt.s32 %p509, %r1101, 1; + @%p509 bra $L__BB0_445; + + setp.eq.s32 %p510, %r5790, %r5812; + selp.u32 %r3431, 1, 0, %p510; + setp.eq.s32 %p511, %r5794, %r5812; + selp.u32 %r3432, -1, 0, %p511; + bfi.b32 %r3433, %r3432, %r3431, 1, 1; + setp.eq.s32 %p512, %r5809, %r5812; + selp.u16 %rs442, 1, 0, %p512; + mul.wide.u16 %r3434, %rs442, 4; + or.b32 %r3435, %r3433, %r3434; + setp.eq.s32 %p513, %r5808, %r5812; + selp.u16 %rs443, 1, 0, %p513; + mul.wide.u16 %r3436, %rs443, 8; + or.b32 %r5815, %r3435, %r3436; + +$L__BB0_445: + shl.b32 %r3437, %r5797, 4; + shl.b32 %r3438, %r5773, 8; + or.b32 %r3439, %r3437, %r3438; + or.b32 %r3440, %r3439, %r5815; + mul.wide.u32 %rd224, %r3440, 2; + add.s64 %rd225, %rd11, %rd224; + ld.global.u16 %rs154, [%rd225]; + shr.u16 %rs444, %rs154, 4; + and.b16 %rs155, %rs444, 7; + setp.eq.s16 %p514, %rs155, 0; + mov.u32 %r5827, %r5716; + @%p514 bra $L__BB0_452; + + cvt.u32.u16 %r5816, %rs155; + shr.u16 %rs445, %rs154, 8; + cvt.u32.u16 %r5817, %rs445; + +$L__BB0_447: + mov.u32 %r1106, %r5816; + setp.gt.u32 %p515, %r5719, 2879; + mov.u32 %r5827, 1; + @%p515 bra $L__BB0_452; + + mov.u32 %r3442, 8; + sub.s32 %r3443, %r3442, %r5717; + sub.s32 %r3444, %r3443, %r5718; + min.u32 %r3445, %r3444, %r1106; + setp.eq.s32 %p516, %r3445, 32; + mov.u32 %r3446, -1; + shl.b32 %r3447, %r3446, %r3445; + not.b32 %r3448, %r3447; + selp.b32 %r3449, -1, %r3448, %p516; + and.b32 %r3450, %r3449, %r5817; + shl.b32 %r3451, %r3450, %r5718; + cvt.u16.u32 %rs446, %r3451; + or.b16 %rs688, %rs688, %rs446; + add.s32 %r5718, %r3445, %r5718; + sub.s32 %r5816, %r1106, %r3445; + shr.u32 %r5817, %r5817, %r3445; + setp.gt.u32 %p517, %r3444, %r1106; + @%p517 bra $L__BB0_451; + + setp.ne.s32 %p518, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs447, %rs688, 255; + setp.ne.s16 %p519, %rs447, 127; + and.pred %p520, %p518, %p519; + @%p520 bra $L__BB0_451; + + mov.u32 %r3454, 20548; + sub.s32 %r3455, %r3454, %r5719; + cvt.u64.u32 %rd226, %r3455; + add.s64 %rd227, %rd1, %rd226; + st.global.u8 [%rd227], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p521, %rs447, 143; + selp.u32 %r5717, 1, 0, %p521; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_451: + setp.ne.s32 %p522, %r5816, 0; + mov.u32 %r5827, %r5716; + @%p522 bra $L__BB0_447; + +$L__BB0_452: + setp.ne.s32 %p523, %r5773, 0; + @%p523 bra $L__BB0_500; + + setp.eq.s32 %p524, %r5797, 0; + add.s32 %r3456, %r5271, 17477; + cvt.u64.u32 %rd228, %r3456; + add.s64 %rd12, %rd1, %rd228; + @%p524 bra $L__BB0_492; + + shl.b16 %rs705, %rs705, 1; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p525, %r5277, 0; + mov.u32 %r5861, %r5480; + @%p525 bra $L__BB0_457; + + setp.gt.u32 %p526, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5861, 1; + @%p526 bra $L__BB0_457; + + st.global.u8 [%rd12], %rs705; + add.s32 %r5271, %r5271, 1; + mov.u32 %r5277, 8; + mov.u16 %rs705, 0; + mov.u32 %r5861, %r5480; + +$L__BB0_457: + setp.lt.u32 %p527, %r5482, 3; + mov.u32 %r5831, 0; + @%p527 bra $L__BB0_460; + + setp.lt.u32 %p528, %r5482, 6; + mov.u32 %r5831, 1; + @%p528 bra $L__BB0_460; + + setp.lt.u32 %p529, %r5482, 9; + setp.eq.s32 %p530, %r5482, 11; + selp.b32 %r3462, 4, 5, %p530; + setp.lt.u32 %p531, %r5482, 11; + selp.b32 %r3463, 3, %r3462, %p531; + selp.b32 %r5831, 2, %r3463, %p529; + +$L__BB0_460: + setp.eq.s32 %p532, %r5831, 0; + @%p532 bra $L__BB0_488; + + add.s32 %r1130, %r5831, -1; + and.b32 %r1131, %r5831, 3; + setp.eq.s32 %p533, %r1131, 0; + mov.u32 %r5841, %r5831; + mov.u32 %r5844, %r5861; + @%p533 bra $L__BB0_473; + + mov.u32 %r3465, 1; + shl.b32 %r3466, %r3465, %r1130; + and.b32 %r3467, %r3466, %r5483; + setp.ne.s32 %p534, %r3467, 0; + selp.u32 %r3468, 1, 0, %p534; + cvt.u32.u16 %r3469, %rs705; + bfi.b32 %r3470, %r3469, %r3468, 1, 8; + cvt.u16.u32 %rs705, %r3470; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p535, %r5277, 0; + mov.u32 %r5844, %r5861; + @%p535 bra $L__BB0_465; + + setp.gt.u32 %p536, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5844, %r3465; + @%p536 bra $L__BB0_465; + + add.s32 %r3474, %r5271, 17477; + cvt.u64.u32 %rd229, %r3474; + add.s64 %rd230, %rd1, %rd229; + st.global.u8 [%rd230], %rs705; + add.s32 %r5271, %r5271, 1; + mov.u32 %r5277, 8; + mov.u16 %rs705, 0; + mov.u32 %r5844, %r5861; + +$L__BB0_465: + setp.eq.s32 %p537, %r1131, 1; + mov.u32 %r5861, %r5844; + mov.u32 %r5841, %r1130; + @%p537 bra $L__BB0_473; + + add.s32 %r5841, %r5831, -2; + mov.u32 %r3475, 1; + shl.b32 %r3476, %r3475, %r5841; + and.b32 %r3477, %r3476, %r5483; + setp.ne.s32 %p538, %r3477, 0; + selp.u32 %r3478, 1, 0, %p538; + cvt.u32.u16 %r3479, %rs705; + bfi.b32 %r3480, %r3479, %r3478, 1, 8; + cvt.u16.u32 %rs705, %r3480; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p539, %r5277, 0; + mov.u32 %r5835, %r5844; + @%p539 bra $L__BB0_469; + + setp.gt.u32 %p540, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5835, %r3475; + @%p540 bra $L__BB0_469; + + add.s32 %r3483, %r5271, 17477; + cvt.u64.u32 %rd231, %r3483; + add.s64 %rd232, %rd1, %rd231; + and.b16 %rs454, %rs705, 255; + st.global.u8 [%rd232], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p541, %rs454, 255; + selp.b32 %r5277, 7, 8, %p541; + mov.u16 %rs705, 0; + mov.u32 %r5835, %r5844; + +$L__BB0_469: + setp.eq.s32 %p542, %r1131, 2; + mov.u32 %r5861, %r5835; + mov.u32 %r5844, %r5835; + @%p542 bra $L__BB0_473; + + add.s32 %r5841, %r5831, -3; + mov.u32 %r3484, 1; + shl.b32 %r3485, %r3484, %r5841; + and.b32 %r3486, %r3485, %r5483; + setp.ne.s32 %p543, %r3486, 0; + selp.u32 %r3487, 1, 0, %p543; + cvt.u32.u16 %r3488, %rs705; + bfi.b32 %r3489, %r3488, %r3487, 1, 8; + cvt.u16.u32 %rs705, %r3489; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p544, %r5277, 0; + mov.u32 %r5861, %r5835; + mov.u32 %r5844, %r5835; + @%p544 bra $L__BB0_473; + + setp.gt.u32 %p545, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5861, %r3484; + mov.u32 %r5844, %r3484; + @%p545 bra $L__BB0_473; + + add.s32 %r3494, %r5271, 17477; + cvt.u64.u32 %rd233, %r3494; + add.s64 %rd234, %rd1, %rd233; + and.b16 %rs457, %rs705, 255; + st.global.u8 [%rd234], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p546, %rs457, 255; + selp.b32 %r5277, 7, 8, %p546; + mov.u16 %rs705, 0; + mov.u32 %r5861, %r5835; + mov.u32 %r5844, %r5835; + +$L__BB0_473: + setp.lt.u32 %p547, %r1130, 3; + @%p547 bra $L__BB0_488; + + mov.u32 %r5861, %r5844; + +$L__BB0_475: + add.s32 %r3495, %r5841, -1; + mov.u32 %r3496, 1; + shl.b32 %r3497, %r3496, %r3495; + and.b32 %r3498, %r3497, %r5483; + setp.ne.s32 %p548, %r3498, 0; + selp.u32 %r3499, 1, 0, %p548; + cvt.u32.u16 %r3500, %rs705; + bfi.b32 %r5850, %r3500, %r3499, 1, 8; + add.s32 %r5851, %r5277, -1; + setp.ne.s32 %p549, %r5851, 0; + mov.u32 %r5849, %r5861; + @%p549 bra $L__BB0_478; + + setp.gt.u32 %p550, %r5271, 191; + mov.u32 %r5851, 0; + mov.u32 %r5849, %r3496; + @%p550 bra $L__BB0_478; + + cvt.u16.u32 %rs458, %r5850; + and.b16 %rs459, %rs458, 255; + add.s32 %r3504, %r5271, 17477; + cvt.u64.u32 %rd235, %r3504; + add.s64 %rd236, %rd1, %rd235; + st.global.u8 [%rd236], %rs458; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p551, %rs459, 255; + selp.b32 %r5851, 7, 8, %p551; + mov.u32 %r5850, 0; + mov.u32 %r5849, %r5861; + +$L__BB0_478: + add.s32 %r3505, %r5841, -2; + shl.b32 %r3507, %r3496, %r3505; + and.b32 %r3508, %r3507, %r5483; + setp.ne.s32 %p552, %r3508, 0; + and.b32 %r3509, %r5850, 127; + selp.u32 %r3510, 1, 0, %p552; + bfi.b32 %r5854, %r3509, %r3510, 1, 7; + add.s32 %r5855, %r5851, -1; + setp.ne.s32 %p553, %r5855, 0; + mov.u32 %r5853, %r5849; + @%p553 bra $L__BB0_481; + + setp.gt.u32 %p554, %r5271, 191; + mov.u32 %r5855, 0; + mov.u32 %r5853, 1; + @%p554 bra $L__BB0_481; + + cvt.u16.u32 %rs460, %r5854; + and.b16 %rs461, %rs460, 255; + add.s32 %r3514, %r5271, 17477; + cvt.u64.u32 %rd237, %r3514; + add.s64 %rd238, %rd1, %rd237; + st.global.u8 [%rd238], %rs460; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p555, %rs461, 255; + selp.b32 %r5855, 7, 8, %p555; + mov.u32 %r5854, 0; + mov.u32 %r5853, %r5849; + +$L__BB0_481: + add.s32 %r3515, %r5841, -3; + mov.u32 %r3516, 1; + shl.b32 %r3517, %r3516, %r3515; + and.b32 %r3518, %r3517, %r5483; + setp.ne.s32 %p556, %r3518, 0; + and.b32 %r3519, %r5854, 127; + selp.u32 %r3520, 1, 0, %p556; + bfi.b32 %r5858, %r3519, %r3520, 1, 7; + add.s32 %r5859, %r5855, -1; + setp.ne.s32 %p557, %r5859, 0; + mov.u32 %r5857, %r5853; + @%p557 bra $L__BB0_484; + + setp.gt.u32 %p558, %r5271, 191; + mov.u32 %r5859, 0; + mov.u32 %r5857, %r3516; + @%p558 bra $L__BB0_484; + + cvt.u16.u32 %rs462, %r5858; + and.b16 %rs463, %rs462, 255; + add.s32 %r3524, %r5271, 17477; + cvt.u64.u32 %rd239, %r3524; + add.s64 %rd240, %rd1, %rd239; + st.global.u8 [%rd240], %rs462; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p559, %rs463, 255; + selp.b32 %r5859, 7, 8, %p559; + mov.u32 %r5858, 0; + mov.u32 %r5857, %r5853; + +$L__BB0_484: + add.s32 %r5841, %r5841, -4; + shl.b32 %r3526, %r3516, %r5841; + and.b32 %r3527, %r3526, %r5483; + setp.ne.s32 %p560, %r3527, 0; + and.b32 %r3528, %r5858, 127; + selp.u32 %r3529, 1, 0, %p560; + bfi.b32 %r3530, %r3528, %r3529, 1, 15; + cvt.u16.u32 %rs705, %r3530; + add.s32 %r5277, %r5859, -1; + setp.ne.s32 %p561, %r5277, 0; + mov.u32 %r5861, %r5857; + @%p561 bra $L__BB0_487; + + setp.gt.u32 %p562, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5861, 1; + @%p562 bra $L__BB0_487; + + add.s32 %r3533, %r5271, 17477; + cvt.u64.u32 %rd241, %r3533; + add.s64 %rd242, %rd1, %rd241; + and.b16 %rs465, %rs705, 255; + st.global.u8 [%rd242], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p563, %rs465, 255; + selp.b32 %r5277, 7, 8, %p563; + mov.u16 %rs705, 0; + mov.u32 %r5861, %r5857; + +$L__BB0_487: + setp.ne.s32 %p564, %r5841, 0; + @%p564 bra $L__BB0_475; + +$L__BB0_488: + add.s32 %r3535, %r5482, -1; + setp.eq.s32 %p565, %r5482, 0; + mov.u32 %r5483, 0; + selp.b32 %r5482, 0, %r3535, %p565; + setp.lt.u32 %p566, %r5482, 3; + mov.u32 %r5867, %r5483; + @%p566 bra $L__BB0_491; + + setp.lt.u32 %p567, %r5482, 6; + mov.u32 %r5867, 1; + @%p567 bra $L__BB0_491; + + setp.lt.u32 %p568, %r5482, 9; + setp.eq.s32 %p569, %r5482, 11; + selp.b32 %r3537, 4, 5, %p569; + setp.lt.u32 %p570, %r5482, 11; + selp.b32 %r3538, 3, %r3537, %p570; + selp.b32 %r5867, 2, %r3538, %p568; + +$L__BB0_491: + mov.u32 %r3540, 1; + shl.b32 %r5481, %r3540, %r5867; + mov.u32 %r5480, %r5861; + bra.uni $L__BB0_500; + +$L__BB0_492: + add.s32 %r5483, %r5483, 1; + setp.lt.u32 %p571, %r5483, %r5481; + @%p571 bra $L__BB0_500; + + shl.b16 %rs466, %rs705, 1; + or.b16 %rs705, %rs466, 1; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p572, %r5277, 0; + mov.u32 %r5868, %r5480; + @%p572 bra $L__BB0_496; + + setp.gt.u32 %p573, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5868, 1; + @%p573 bra $L__BB0_496; + + and.b16 %rs468, %rs705, 255; + st.global.u8 [%rd12], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p574, %rs468, 255; + selp.b32 %r5277, 7, 8, %p574; + mov.u16 %rs705, 0; + mov.u32 %r5868, %r5480; + +$L__BB0_496: + add.s32 %r3544, %r5482, 1; + min.u32 %r5482, %r3544, 12; + setp.lt.u32 %p575, %r5482, 3; + mov.u32 %r5483, 0; + mov.u32 %r5871, %r5483; + @%p575 bra $L__BB0_499; + + setp.lt.u32 %p576, %r5482, 6; + mov.u32 %r5871, 1; + @%p576 bra $L__BB0_499; + + setp.lt.u32 %p577, %r5482, 9; + setp.eq.s32 %p578, %r5482, 11; + selp.b32 %r3546, 4, 5, %p578; + setp.lt.u32 %p579, %r5482, 11; + selp.b32 %r3547, 3, %r3546, %p579; + selp.b32 %r5871, 2, %r3547, %p577; + +$L__BB0_499: + mov.u32 %r3549, 1; + shl.b32 %r5481, %r3549, %r5871; + mov.u32 %r5480, %r5868; + +$L__BB0_500: + and.b16 %rs469, %rs154, 15; + cvt.u32.u16 %r1214, %rs469; + and.b32 %r3550, %r5797, 1; + setp.eq.b32 %p580, %r3550, 1; + mov.pred %p581, 0; + xor.pred %p582, %p580, %p581; + not.pred %p583, %p582; + mov.u32 %r5888, %r5933; + @%p583 bra $L__BB0_507; + + and.b32 %r3551, %r1214, 1; + sub.s32 %r5878, %r1100, %r3551; + setp.eq.s32 %p584, %r5878, 0; + mov.u32 %r5888, %r5933; + @%p584 bra $L__BB0_507; + + mov.u32 %r3552, -1; + shl.b32 %r3553, %r3552, %r5878; + not.b32 %r3554, %r3553; + and.b32 %r5879, %r5791, %r3554; + +$L__BB0_503: + setp.gt.u32 %p585, %r5907, 17476; + mov.u32 %r5888, 1; + @%p585 bra $L__BB0_507; + + sub.s32 %r3556, %r5906, %r5905; + min.u32 %r3557, %r3556, %r5878; + setp.eq.s32 %p586, %r3557, 32; + mov.u32 %r3558, -1; + shl.b32 %r3559, %r3558, %r3557; + not.b32 %r3560, %r3559; + selp.b32 %r3561, -1, %r3560, %p586; + and.b32 %r3562, %r3561, %r5879; + shl.b32 %r3563, %r3562, %r5905; + or.b32 %r5904, %r3563, %r5904; + add.s32 %r5905, %r3557, %r5905; + shr.u32 %r5879, %r5879, %r3557; + sub.s32 %r5878, %r5878, %r3557; + setp.lt.u32 %p587, %r5905, %r5906; + @%p587 bra $L__BB0_506; + + cvt.u64.u32 %rd243, %r5907; + add.s64 %rd244, %rd1, %rd243; + st.global.u8 [%rd244], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p588, %r5904, 255; + selp.b32 %r5906, 7, 8, %p588; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_506: + setp.ne.s32 %p589, %r5878, 0; + mov.u32 %r5888, %r5933; + @%p589 bra $L__BB0_503; + +$L__BB0_507: + and.b32 %r1238, %r5797, 2; + setp.eq.s32 %p590, %r1238, 0; + mov.u32 %r5903, %r5888; + @%p590 bra $L__BB0_514; + + shr.u32 %r3566, %r1214, 1; + and.b32 %r3567, %r3566, 1; + sub.s32 %r5893, %r1100, %r3567; + setp.eq.s32 %p591, %r5893, 0; + mov.u32 %r5903, %r5888; + @%p591 bra $L__BB0_514; + + mov.u32 %r3568, -1; + shl.b32 %r3569, %r3568, %r5893; + not.b32 %r3570, %r3569; + and.b32 %r5894, %r5795, %r3570; + +$L__BB0_510: + setp.gt.u32 %p592, %r5907, 17476; + mov.u32 %r5903, 1; + @%p592 bra $L__BB0_514; + + sub.s32 %r3572, %r5906, %r5905; + min.u32 %r3573, %r3572, %r5893; + setp.eq.s32 %p593, %r3573, 32; + mov.u32 %r3574, -1; + shl.b32 %r3575, %r3574, %r3573; + not.b32 %r3576, %r3575; + selp.b32 %r3577, -1, %r3576, %p593; + and.b32 %r3578, %r3577, %r5894; + shl.b32 %r3579, %r3578, %r5905; + or.b32 %r5904, %r3579, %r5904; + add.s32 %r5905, %r3573, %r5905; + shr.u32 %r5894, %r5894, %r3573; + sub.s32 %r5893, %r5893, %r3573; + setp.lt.u32 %p594, %r5905, %r5906; + @%p594 bra $L__BB0_513; + + cvt.u64.u32 %rd245, %r5907; + add.s64 %rd246, %rd1, %rd245; + st.global.u8 [%rd246], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p595, %r5904, 255; + selp.b32 %r5906, 7, 8, %p595; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_513: + setp.ne.s32 %p596, %r5893, 0; + mov.u32 %r5903, %r5888; + @%p596 bra $L__BB0_510; + +$L__BB0_514: + and.b32 %r1262, %r5797, 4; + setp.eq.s32 %p597, %r1262, 0; + mov.u32 %r5918, %r5903; + @%p597 bra $L__BB0_521; + + shr.u32 %r3582, %r1214, 2; + and.b32 %r3583, %r3582, 1; + sub.s32 %r5908, %r1100, %r3583; + setp.eq.s32 %p598, %r5908, 0; + mov.u32 %r5918, %r5903; + @%p598 bra $L__BB0_521; + + mov.u32 %r3584, -1; + shl.b32 %r3585, %r3584, %r5908; + not.b32 %r3586, %r3585; + and.b32 %r5909, %r5811, %r3586; + +$L__BB0_517: + setp.gt.u32 %p599, %r5907, 17476; + mov.u32 %r5918, 1; + @%p599 bra $L__BB0_521; + + sub.s32 %r3588, %r5906, %r5905; + min.u32 %r3589, %r3588, %r5908; + setp.eq.s32 %p600, %r3589, 32; + mov.u32 %r3590, -1; + shl.b32 %r3591, %r3590, %r3589; + not.b32 %r3592, %r3591; + selp.b32 %r3593, -1, %r3592, %p600; + and.b32 %r3594, %r3593, %r5909; + shl.b32 %r3595, %r3594, %r5905; + or.b32 %r5904, %r3595, %r5904; + add.s32 %r5905, %r3589, %r5905; + shr.u32 %r5909, %r5909, %r3589; + sub.s32 %r5908, %r5908, %r3589; + setp.lt.u32 %p601, %r5905, %r5906; + @%p601 bra $L__BB0_520; + + cvt.u64.u32 %rd247, %r5907; + add.s64 %rd248, %rd1, %rd247; + st.global.u8 [%rd248], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p602, %r5904, 255; + selp.b32 %r5906, 7, 8, %p602; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_520: + setp.ne.s32 %p603, %r5908, 0; + mov.u32 %r5918, %r5903; + @%p603 bra $L__BB0_517; + +$L__BB0_521: + and.b32 %r1286, %r5797, 8; + setp.eq.s32 %p604, %r1286, 0; + mov.u32 %r5933, %r5918; + @%p604 bra $L__BB0_528; + + shr.u32 %r3598, %r1214, 3; + sub.s32 %r5923, %r1100, %r3598; + setp.eq.s32 %p605, %r5923, 0; + mov.u32 %r5933, %r5918; + @%p605 bra $L__BB0_528; + + mov.u32 %r3599, -1; + shl.b32 %r3600, %r3599, %r5923; + not.b32 %r3601, %r3600; + and.b32 %r5924, %r5810, %r3601; + +$L__BB0_524: + setp.gt.u32 %p606, %r5907, 17476; + mov.u32 %r5933, 1; + @%p606 bra $L__BB0_528; + + sub.s32 %r3603, %r5906, %r5905; + min.u32 %r3604, %r3603, %r5923; + setp.eq.s32 %p607, %r3604, 32; + mov.u32 %r3605, -1; + shl.b32 %r3606, %r3605, %r3604; + not.b32 %r3607, %r3606; + selp.b32 %r3608, -1, %r3607, %p607; + and.b32 %r3609, %r3608, %r5924; + shl.b32 %r3610, %r3609, %r5905; + or.b32 %r5904, %r3610, %r5904; + add.s32 %r5905, %r3604, %r5905; + shr.u32 %r5924, %r5924, %r3604; + sub.s32 %r5923, %r5923, %r3604; + setp.lt.u32 %p608, %r5905, %r5906; + @%p608 bra $L__BB0_527; + + cvt.u64.u32 %rd249, %r5907; + add.s64 %rd250, %rd1, %rd249; + st.global.u8 [%rd250], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p609, %r5904, 255; + selp.b32 %r5906, 7, 8, %p609; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_527: + setp.ne.s32 %p610, %r5923, 0; + mov.u32 %r5933, %r5918; + @%p610 bra $L__BB0_524; + +$L__BB0_528: + mov.u32 %r3615, _ZZ31 j2k_htj2k_encode_codeblockE13cleanup_e_val; + add.s32 %r1310, %r3615, %r5771; + ld.shared.u8 %rs470, [%r1310]; + mov.u32 %r5773, 0; + cvt.u32.u16 %r3616, %rs470; + and.b32 %r3617, %r3616, 255; + and.b32 %r3618, %r5794, 255; + setp.lt.u32 %p611, %r3618, %r3617; + cvt.u16.u32 %rs471, %r5794; + selp.b16 %rs472, %rs470, %rs471, %p611; + st.shared.u8 [%r1310], %rs472; + ld.shared.u8 %rs176, [%r1310+2]; + ld.shared.u8 %rs473, [%r1310+1]; + setp.gt.u16 %p612, %rs473, %rs176; + add.s32 %r6103, %r5771, 1; + add.s32 %r3619, %r5771, 2; + selp.b32 %r3620, %r6103, %r3619, %p612; + add.s32 %r3621, %r3615, %r3620; + ld.shared.u8 %rs177, [%r3621]; + cvt.u32.u16 %r3622, %rs177; + and.b32 %r3623, %r3622, 255; + add.s32 %r5770, %r3623, -1; + cvt.u16.u32 %rs178, %r5808; + cvt.u16.u32 %rs474, %r1238; + shr.u16 %rs475, %rs474, 1; + mov.u32 %r3624, _ZZ31 j2k_htj2k_encode_codeblockE14cleanup_cx_val; + add.s32 %r1313, %r3624, %r5769; + st.shared.u8 [%r1310+1], %r5808; + ld.shared.u8 %rs476, [%r1313]; + or.b16 %rs477, %rs476, %rs475; + st.shared.u8 [%r1313], %rs477; + add.s32 %r5769, %r5769, 1; + ld.shared.u8 %rs179, [%r1313+1]; + ld.shared.u8 %r1315, [%r1313+2]; + shr.u32 %r1316, %r1286, 3; + st.shared.u8 [%r1313+1], %r1316; + add.s32 %r3625, %r5768, 2; + setp.ge.u32 %p613, %r3625, %r5; + mov.u32 %r6107, %r5773; + @%p613 bra $L__BB0_635; + + mul.wide.u32 %rd251, %r6102, 4; + add.s64 %rd252, %rd2, %rd251; + ld.global.u32 %r1317, [%rd252]; + setp.eq.s32 %p614, %r1317, 0; + mov.u32 %r5939, 0; + mov.u32 %r5938, %r5939; + @%p614 bra $L__BB0_531; + + and.b32 %r3627, %r1317, -2147483648; + abs.s32 %r3628, %r1317; + shl.b32 %r3629, %r3628, %r1004; + or.b32 %r5938, %r3629, %r3627; + +$L__BB0_531: + shl.b32 %r3633, %r5938, 1; + shr.u32 %r3634, %r3633, %r45; + and.b32 %r1320, %r3634, -2; + setp.eq.s32 %p615, %r1320, 0; + mov.u32 %r5940, %r5939; + mov.u32 %r5946, %r5939; + @%p615 bra $L__BB0_533; + + add.s32 %r3636, %r1320, -1; + clz.b32 %r3637, %r3636; + mov.u32 %r3638, 32; + sub.s32 %r5939, %r3638, %r3637; + shr.u32 %r3639, %r5938, 31; + add.s32 %r3640, %r3639, %r1320; + add.s32 %r5940, %r3640, -2; + mov.u32 %r5946, 1; + +$L__BB0_533: + mov.u32 %r5943, 0; + mov.u32 %r5942, %r5943; + @%p497 bra $L__BB0_536; + + add.s32 %r3643, %r6102, %r1; + mul.wide.u32 %rd253, %r3643, 4; + add.s64 %rd254, %rd2, %rd253; + ld.global.u32 %r1326, [%rd254]; + setp.eq.s32 %p617, %r1326, 0; + @%p617 bra $L__BB0_536; + + and.b32 %r3644, %r1326, -2147483648; + abs.s32 %r3645, %r1326; + shl.b32 %r3646, %r3645, %r1004; + or.b32 %r5942, %r3646, %r3644; + +$L__BB0_536: + shl.b32 %r3649, %r5942, 1; + shr.u32 %r3650, %r3649, %r45; + and.b32 %r1329, %r3650, -2; + setp.eq.s32 %p618, %r1329, 0; + mov.u32 %r5944, %r5943; + mov.u32 %r5961, %r5939; + @%p618 bra $L__BB0_538; + + or.b32 %r5946, %r5946, 2; + add.s32 %r3651, %r1329, -1; + clz.b32 %r3652, %r3651; + mov.u32 %r3653, 32; + sub.s32 %r5943, %r3653, %r3652; + max.s32 %r5961, %r5939, %r5943; + shr.u32 %r3654, %r5942, 31; + add.s32 %r3655, %r3654, %r1329; + add.s32 %r5944, %r3655, -2; + +$L__BB0_538: + add.s32 %r5963, %r6102, 1; + add.s32 %r3660, %r5768, 3; + setp.ge.u32 %p619, %r3660, %r5; + mov.u32 %r5964, 0; + mov.u32 %r5957, %r5964; + mov.u32 %r5958, %r5964; + mov.u32 %r5959, %r5964; + mov.u32 %r5960, %r5964; + @%p619 bra $L__BB0_549; + + mul.wide.u32 %rd255, %r5963, 4; + add.s64 %rd256, %rd2, %rd255; + ld.global.u32 %r1339, [%rd256]; + setp.eq.s32 %p620, %r1339, 0; + mov.u32 %r5958, 0; + mov.u32 %r5947, %r5958; + @%p620 bra $L__BB0_541; + + and.b32 %r3662, %r1339, -2147483648; + abs.s32 %r3663, %r1339; + shl.b32 %r3664, %r3663, %r1004; + or.b32 %r5947, %r3664, %r3662; + +$L__BB0_541: + shl.b32 %r3667, %r5947, 1; + shr.u32 %r3668, %r3667, %r45; + and.b32 %r1342, %r3668, -2; + setp.eq.s32 %p621, %r1342, 0; + mov.u32 %r5960, %r5958; + @%p621 bra $L__BB0_543; + + or.b32 %r5946, %r5946, 4; + add.s32 %r3669, %r1342, -1; + clz.b32 %r3670, %r3669; + mov.u32 %r3671, 32; + sub.s32 %r5958, %r3671, %r3670; + max.s32 %r5961, %r5961, %r5958; + shr.u32 %r3672, %r5947, 31; + add.s32 %r3673, %r3672, %r1342; + add.s32 %r5960, %r3673, -2; + +$L__BB0_543: + mov.u32 %r5957, 0; + mov.u32 %r5952, %r5957; + @%p497 bra $L__BB0_546; + + add.s32 %r3676, %r5963, %r1; + mul.wide.u32 %rd257, %r3676, 4; + add.s64 %rd258, %rd2, %rd257; + ld.global.u32 %r1351, [%rd258]; + setp.eq.s32 %p623, %r1351, 0; + @%p623 bra $L__BB0_546; + + and.b32 %r3677, %r1351, -2147483648; + abs.s32 %r3678, %r1351; + shl.b32 %r3679, %r3678, %r1004; + or.b32 %r5952, %r3679, %r3677; + +$L__BB0_546: + shl.b32 %r3682, %r5952, 1; + shr.u32 %r3683, %r3682, %r45; + and.b32 %r1354, %r3683, -2; + setp.eq.s32 %p624, %r1354, 0; + mov.u32 %r5959, %r5957; + @%p624 bra $L__BB0_548; + + or.b32 %r5946, %r5946, 8; + add.s32 %r3684, %r1354, -1; + clz.b32 %r3685, %r3684; + mov.u32 %r3686, 32; + sub.s32 %r5957, %r3686, %r3685; + max.s32 %r5961, %r5961, %r5957; + shr.u32 %r3687, %r5952, 31; + add.s32 %r3688, %r3687, %r1354; + add.s32 %r5959, %r3688, -2; + +$L__BB0_548: + add.s32 %r5963, %r6102, 2; + +$L__BB0_549: + mov.u32 %r6102, %r5963; + shr.u32 %r3690, %r1286, 2; + shr.u32 %r3691, %r1262, 1; + or.b32 %r3692, %r3690, %r3691; + cvt.u32.u16 %r3693, %rs179; + and.b32 %r3694, %r3693, 255; + shl.b32 %r3695, %r1315, 2; + add.s32 %r3696, %r3695, %r3694; + or.b32 %r1371, %r3692, %r3696; + add.s32 %r3697, %r5946, -1; + and.b32 %r3698, %r3697, %r5946; + setp.ne.s32 %p625, %r3698, 0; + setp.gt.u16 %p626, %rs177, 2; + and.pred %p627, %p626, %p625; + selp.b32 %r3699, %r5770, 1, %p627; + max.s32 %r1372, %r3699, %r5961; + sub.s32 %r6107, %r1372, %r3699; + setp.lt.s32 %p628, %r6107, 1; + @%p628 bra $L__BB0_551; + + setp.eq.s32 %p629, %r5939, %r5961; + selp.u32 %r3700, 1, 0, %p629; + setp.eq.s32 %p630, %r5943, %r5961; + selp.u32 %r3701, -1, 0, %p630; + bfi.b32 %r3702, %r3701, %r3700, 1, 1; + setp.eq.s32 %p631, %r5958, %r5961; + selp.u16 %rs479, 1, 0, %p631; + mul.wide.u16 %r3703, %rs479, 4; + or.b32 %r3704, %r3702, %r3703; + setp.eq.s32 %p632, %r5957, %r5961; + selp.u16 %rs480, 1, 0, %p632; + mul.wide.u16 %r3705, %rs480, 8; + or.b32 %r5964, %r3704, %r3705; + +$L__BB0_551: + shl.b32 %r3706, %r5946, 4; + shl.b32 %r3707, %r1371, 8; + or.b32 %r3708, %r3706, %r3707; + or.b32 %r3709, %r3708, %r5964; + mul.wide.u32 %rd260, %r3709, 2; + add.s64 %rd261, %rd11, %rd260; + ld.global.u16 %rs180, [%rd261]; + shr.u16 %rs481, %rs180, 4; + and.b16 %rs181, %rs481, 7; + setp.eq.s16 %p633, %rs181, 0; + mov.u32 %r5976, %r5827; + @%p633 bra $L__BB0_558; + + cvt.u32.u16 %r5965, %rs181; + shr.u16 %rs482, %rs180, 8; + cvt.u32.u16 %r5966, %rs482; + +$L__BB0_553: + mov.u32 %r1378, %r5965; + setp.gt.u32 %p634, %r5719, 2879; + mov.u32 %r5976, 1; + @%p634 bra $L__BB0_558; + + mov.u32 %r3711, 8; + sub.s32 %r3712, %r3711, %r5717; + sub.s32 %r3713, %r3712, %r5718; + min.u32 %r3714, %r3713, %r1378; + setp.eq.s32 %p635, %r3714, 32; + mov.u32 %r3715, -1; + shl.b32 %r3716, %r3715, %r3714; + not.b32 %r3717, %r3716; + selp.b32 %r3718, -1, %r3717, %p635; + and.b32 %r3719, %r3718, %r5966; + shl.b32 %r3720, %r3719, %r5718; + cvt.u16.u32 %rs483, %r3720; + or.b16 %rs688, %rs688, %rs483; + add.s32 %r5718, %r3714, %r5718; + sub.s32 %r5965, %r1378, %r3714; + shr.u32 %r5966, %r5966, %r3714; + setp.gt.u32 %p636, %r3713, %r1378; + @%p636 bra $L__BB0_557; + + setp.ne.s32 %p637, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs484, %rs688, 255; + setp.ne.s16 %p638, %rs484, 127; + and.pred %p639, %p637, %p638; + @%p639 bra $L__BB0_557; + + mov.u32 %r3723, 20548; + sub.s32 %r3724, %r3723, %r5719; + cvt.u64.u32 %rd262, %r3724; + add.s64 %rd263, %rd1, %rd262; + st.global.u8 [%rd263], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p640, %rs484, 143; + selp.u32 %r5717, 1, 0, %p640; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_557: + setp.ne.s32 %p641, %r5965, 0; + mov.u32 %r5976, %r5827; + @%p641 bra $L__BB0_553; + +$L__BB0_558: + setp.ne.s32 %p642, %r1371, 0; + @%p642 bra $L__BB0_606; + + setp.eq.s32 %p643, %r5946, 0; + add.s32 %r3725, %r5271, 17477; + cvt.u64.u32 %rd264, %r3725; + add.s64 %rd13, %rd1, %rd264; + @%p643 bra $L__BB0_598; + + shl.b16 %rs705, %rs705, 1; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p644, %r5277, 0; + mov.u32 %r6010, %r5480; + @%p644 bra $L__BB0_563; + + setp.gt.u32 %p645, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r6010, 1; + @%p645 bra $L__BB0_563; + + st.global.u8 [%rd13], %rs705; + add.s32 %r5271, %r5271, 1; + mov.u32 %r5277, 8; + mov.u16 %rs705, 0; + mov.u32 %r6010, %r5480; + +$L__BB0_563: + setp.lt.u32 %p646, %r5482, 3; + mov.u32 %r5980, 0; + @%p646 bra $L__BB0_566; + + setp.lt.u32 %p647, %r5482, 6; + mov.u32 %r5980, 1; + @%p647 bra $L__BB0_566; + + setp.lt.u32 %p648, %r5482, 9; + setp.eq.s32 %p649, %r5482, 11; + selp.b32 %r3731, 4, 5, %p649; + setp.lt.u32 %p650, %r5482, 11; + selp.b32 %r3732, 3, %r3731, %p650; + selp.b32 %r5980, 2, %r3732, %p648; + +$L__BB0_566: + setp.eq.s32 %p651, %r5980, 0; + @%p651 bra $L__BB0_594; + + add.s32 %r1402, %r5980, -1; + and.b32 %r1403, %r5980, 3; + setp.eq.s32 %p652, %r1403, 0; + mov.u32 %r5990, %r5980; + mov.u32 %r5993, %r6010; + @%p652 bra $L__BB0_579; + + mov.u32 %r3734, 1; + shl.b32 %r3735, %r3734, %r1402; + and.b32 %r3736, %r3735, %r5483; + setp.ne.s32 %p653, %r3736, 0; + selp.u32 %r3737, 1, 0, %p653; + cvt.u32.u16 %r3738, %rs705; + bfi.b32 %r3739, %r3738, %r3737, 1, 8; + cvt.u16.u32 %rs705, %r3739; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p654, %r5277, 0; + mov.u32 %r5993, %r6010; + @%p654 bra $L__BB0_571; + + setp.gt.u32 %p655, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5993, %r3734; + @%p655 bra $L__BB0_571; + + add.s32 %r3743, %r5271, 17477; + cvt.u64.u32 %rd265, %r3743; + add.s64 %rd266, %rd1, %rd265; + st.global.u8 [%rd266], %rs705; + add.s32 %r5271, %r5271, 1; + mov.u32 %r5277, 8; + mov.u16 %rs705, 0; + mov.u32 %r5993, %r6010; + +$L__BB0_571: + setp.eq.s32 %p656, %r1403, 1; + mov.u32 %r6010, %r5993; + mov.u32 %r5990, %r1402; + @%p656 bra $L__BB0_579; + + add.s32 %r5990, %r5980, -2; + mov.u32 %r3744, 1; + shl.b32 %r3745, %r3744, %r5990; + and.b32 %r3746, %r3745, %r5483; + setp.ne.s32 %p657, %r3746, 0; + selp.u32 %r3747, 1, 0, %p657; + cvt.u32.u16 %r3748, %rs705; + bfi.b32 %r3749, %r3748, %r3747, 1, 8; + cvt.u16.u32 %rs705, %r3749; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p658, %r5277, 0; + mov.u32 %r5984, %r5993; + @%p658 bra $L__BB0_575; + + setp.gt.u32 %p659, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r5984, %r3744; + @%p659 bra $L__BB0_575; + + add.s32 %r3752, %r5271, 17477; + cvt.u64.u32 %rd267, %r3752; + add.s64 %rd268, %rd1, %rd267; + and.b16 %rs491, %rs705, 255; + st.global.u8 [%rd268], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p660, %rs491, 255; + selp.b32 %r5277, 7, 8, %p660; + mov.u16 %rs705, 0; + mov.u32 %r5984, %r5993; + +$L__BB0_575: + setp.eq.s32 %p661, %r1403, 2; + mov.u32 %r6010, %r5984; + mov.u32 %r5993, %r5984; + @%p661 bra $L__BB0_579; + + add.s32 %r5990, %r5980, -3; + mov.u32 %r3753, 1; + shl.b32 %r3754, %r3753, %r5990; + and.b32 %r3755, %r3754, %r5483; + setp.ne.s32 %p662, %r3755, 0; + selp.u32 %r3756, 1, 0, %p662; + cvt.u32.u16 %r3757, %rs705; + bfi.b32 %r3758, %r3757, %r3756, 1, 8; + cvt.u16.u32 %rs705, %r3758; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p663, %r5277, 0; + mov.u32 %r6010, %r5984; + mov.u32 %r5993, %r5984; + @%p663 bra $L__BB0_579; + + setp.gt.u32 %p664, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r6010, %r3753; + mov.u32 %r5993, %r3753; + @%p664 bra $L__BB0_579; + + add.s32 %r3763, %r5271, 17477; + cvt.u64.u32 %rd269, %r3763; + add.s64 %rd270, %rd1, %rd269; + and.b16 %rs494, %rs705, 255; + st.global.u8 [%rd270], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p665, %rs494, 255; + selp.b32 %r5277, 7, 8, %p665; + mov.u16 %rs705, 0; + mov.u32 %r6010, %r5984; + mov.u32 %r5993, %r5984; + +$L__BB0_579: + setp.lt.u32 %p666, %r1402, 3; + @%p666 bra $L__BB0_594; + + mov.u32 %r6010, %r5993; + +$L__BB0_581: + add.s32 %r3764, %r5990, -1; + mov.u32 %r3765, 1; + shl.b32 %r3766, %r3765, %r3764; + and.b32 %r3767, %r3766, %r5483; + setp.ne.s32 %p667, %r3767, 0; + selp.u32 %r3768, 1, 0, %p667; + cvt.u32.u16 %r3769, %rs705; + bfi.b32 %r5999, %r3769, %r3768, 1, 8; + add.s32 %r6000, %r5277, -1; + setp.ne.s32 %p668, %r6000, 0; + mov.u32 %r5998, %r6010; + @%p668 bra $L__BB0_584; + + setp.gt.u32 %p669, %r5271, 191; + mov.u32 %r6000, 0; + mov.u32 %r5998, %r3765; + @%p669 bra $L__BB0_584; + + cvt.u16.u32 %rs495, %r5999; + and.b16 %rs496, %rs495, 255; + add.s32 %r3773, %r5271, 17477; + cvt.u64.u32 %rd271, %r3773; + add.s64 %rd272, %rd1, %rd271; + st.global.u8 [%rd272], %rs495; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p670, %rs496, 255; + selp.b32 %r6000, 7, 8, %p670; + mov.u32 %r5999, 0; + mov.u32 %r5998, %r6010; + +$L__BB0_584: + add.s32 %r3774, %r5990, -2; + shl.b32 %r3776, %r3765, %r3774; + and.b32 %r3777, %r3776, %r5483; + setp.ne.s32 %p671, %r3777, 0; + and.b32 %r3778, %r5999, 127; + selp.u32 %r3779, 1, 0, %p671; + bfi.b32 %r6003, %r3778, %r3779, 1, 7; + add.s32 %r6004, %r6000, -1; + setp.ne.s32 %p672, %r6004, 0; + mov.u32 %r6002, %r5998; + @%p672 bra $L__BB0_587; + + setp.gt.u32 %p673, %r5271, 191; + mov.u32 %r6004, 0; + mov.u32 %r6002, 1; + @%p673 bra $L__BB0_587; + + cvt.u16.u32 %rs497, %r6003; + and.b16 %rs498, %rs497, 255; + add.s32 %r3783, %r5271, 17477; + cvt.u64.u32 %rd273, %r3783; + add.s64 %rd274, %rd1, %rd273; + st.global.u8 [%rd274], %rs497; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p674, %rs498, 255; + selp.b32 %r6004, 7, 8, %p674; + mov.u32 %r6003, 0; + mov.u32 %r6002, %r5998; + +$L__BB0_587: + add.s32 %r3784, %r5990, -3; + mov.u32 %r3785, 1; + shl.b32 %r3786, %r3785, %r3784; + and.b32 %r3787, %r3786, %r5483; + setp.ne.s32 %p675, %r3787, 0; + and.b32 %r3788, %r6003, 127; + selp.u32 %r3789, 1, 0, %p675; + bfi.b32 %r6007, %r3788, %r3789, 1, 7; + add.s32 %r6008, %r6004, -1; + setp.ne.s32 %p676, %r6008, 0; + mov.u32 %r6006, %r6002; + @%p676 bra $L__BB0_590; + + setp.gt.u32 %p677, %r5271, 191; + mov.u32 %r6008, 0; + mov.u32 %r6006, %r3785; + @%p677 bra $L__BB0_590; + + cvt.u16.u32 %rs499, %r6007; + and.b16 %rs500, %rs499, 255; + add.s32 %r3793, %r5271, 17477; + cvt.u64.u32 %rd275, %r3793; + add.s64 %rd276, %rd1, %rd275; + st.global.u8 [%rd276], %rs499; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p678, %rs500, 255; + selp.b32 %r6008, 7, 8, %p678; + mov.u32 %r6007, 0; + mov.u32 %r6006, %r6002; + +$L__BB0_590: + add.s32 %r5990, %r5990, -4; + shl.b32 %r3795, %r3785, %r5990; + and.b32 %r3796, %r3795, %r5483; + setp.ne.s32 %p679, %r3796, 0; + and.b32 %r3797, %r6007, 127; + selp.u32 %r3798, 1, 0, %p679; + bfi.b32 %r3799, %r3797, %r3798, 1, 15; + cvt.u16.u32 %rs705, %r3799; + add.s32 %r5277, %r6008, -1; + setp.ne.s32 %p680, %r5277, 0; + mov.u32 %r6010, %r6006; + @%p680 bra $L__BB0_593; + + setp.gt.u32 %p681, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r6010, 1; + @%p681 bra $L__BB0_593; + + add.s32 %r3802, %r5271, 17477; + cvt.u64.u32 %rd277, %r3802; + add.s64 %rd278, %rd1, %rd277; + and.b16 %rs502, %rs705, 255; + st.global.u8 [%rd278], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p682, %rs502, 255; + selp.b32 %r5277, 7, 8, %p682; + mov.u16 %rs705, 0; + mov.u32 %r6010, %r6006; + +$L__BB0_593: + setp.ne.s32 %p683, %r5990, 0; + @%p683 bra $L__BB0_581; + +$L__BB0_594: + add.s32 %r3804, %r5482, -1; + setp.eq.s32 %p684, %r5482, 0; + mov.u32 %r5483, 0; + selp.b32 %r5482, 0, %r3804, %p684; + setp.lt.u32 %p685, %r5482, 3; + mov.u32 %r6016, %r5483; + @%p685 bra $L__BB0_597; + + setp.lt.u32 %p686, %r5482, 6; + mov.u32 %r6016, 1; + @%p686 bra $L__BB0_597; + + setp.lt.u32 %p687, %r5482, 9; + setp.eq.s32 %p688, %r5482, 11; + selp.b32 %r3806, 4, 5, %p688; + setp.lt.u32 %p689, %r5482, 11; + selp.b32 %r3807, 3, %r3806, %p689; + selp.b32 %r6016, 2, %r3807, %p687; + +$L__BB0_597: + mov.u32 %r3809, 1; + shl.b32 %r5481, %r3809, %r6016; + mov.u32 %r5480, %r6010; + bra.uni $L__BB0_606; + +$L__BB0_598: + add.s32 %r5483, %r5483, 1; + setp.lt.u32 %p690, %r5483, %r5481; + @%p690 bra $L__BB0_606; + + shl.b16 %rs503, %rs705, 1; + or.b16 %rs705, %rs503, 1; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p691, %r5277, 0; + mov.u32 %r6017, %r5480; + @%p691 bra $L__BB0_602; + + setp.gt.u32 %p692, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r6017, 1; + @%p692 bra $L__BB0_602; + + and.b16 %rs505, %rs705, 255; + st.global.u8 [%rd13], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p693, %rs505, 255; + selp.b32 %r5277, 7, 8, %p693; + mov.u16 %rs705, 0; + mov.u32 %r6017, %r5480; + +$L__BB0_602: + add.s32 %r3813, %r5482, 1; + min.u32 %r5482, %r3813, 12; + setp.lt.u32 %p694, %r5482, 3; + mov.u32 %r5483, 0; + mov.u32 %r6020, %r5483; + @%p694 bra $L__BB0_605; + + setp.lt.u32 %p695, %r5482, 6; + mov.u32 %r6020, 1; + @%p695 bra $L__BB0_605; + + setp.lt.u32 %p696, %r5482, 9; + setp.eq.s32 %p697, %r5482, 11; + selp.b32 %r3815, 4, 5, %p697; + setp.lt.u32 %p698, %r5482, 11; + selp.b32 %r3816, 3, %r3815, %p698; + selp.b32 %r6020, 2, %r3816, %p696; + +$L__BB0_605: + mov.u32 %r3818, 1; + shl.b32 %r5481, %r3818, %r6020; + mov.u32 %r5480, %r6017; + +$L__BB0_606: + and.b16 %rs506, %rs180, 15; + cvt.u32.u16 %r1486, %rs506; + and.b32 %r3819, %r5946, 1; + setp.eq.b32 %p699, %r3819, 1; + mov.pred %p700, 0; + xor.pred %p701, %p699, %p700; + not.pred %p702, %p701; + mov.u32 %r6037, %r5933; + @%p702 bra $L__BB0_613; + + and.b32 %r3820, %r1486, 1; + sub.s32 %r6027, %r1372, %r3820; + setp.eq.s32 %p703, %r6027, 0; + mov.u32 %r6037, %r5933; + @%p703 bra $L__BB0_613; + + mov.u32 %r3821, -1; + shl.b32 %r3822, %r3821, %r6027; + not.b32 %r3823, %r3822; + and.b32 %r6028, %r5940, %r3823; + +$L__BB0_609: + setp.gt.u32 %p704, %r5907, 17476; + mov.u32 %r6037, 1; + @%p704 bra $L__BB0_613; + + sub.s32 %r3825, %r5906, %r5905; + min.u32 %r3826, %r3825, %r6027; + setp.eq.s32 %p705, %r3826, 32; + mov.u32 %r3827, -1; + shl.b32 %r3828, %r3827, %r3826; + not.b32 %r3829, %r3828; + selp.b32 %r3830, -1, %r3829, %p705; + and.b32 %r3831, %r3830, %r6028; + shl.b32 %r3832, %r3831, %r5905; + or.b32 %r5904, %r3832, %r5904; + add.s32 %r5905, %r3826, %r5905; + shr.u32 %r6028, %r6028, %r3826; + sub.s32 %r6027, %r6027, %r3826; + setp.lt.u32 %p706, %r5905, %r5906; + @%p706 bra $L__BB0_612; + + cvt.u64.u32 %rd279, %r5907; + add.s64 %rd280, %rd1, %rd279; + st.global.u8 [%rd280], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p707, %r5904, 255; + selp.b32 %r5906, 7, 8, %p707; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_612: + setp.ne.s32 %p708, %r6027, 0; + mov.u32 %r6037, %r5933; + @%p708 bra $L__BB0_609; + +$L__BB0_613: + and.b32 %r1510, %r5946, 2; + setp.eq.s32 %p709, %r1510, 0; + mov.u32 %r6052, %r6037; + @%p709 bra $L__BB0_620; + + shr.u32 %r3835, %r1486, 1; + and.b32 %r3836, %r3835, 1; + sub.s32 %r6042, %r1372, %r3836; + setp.eq.s32 %p710, %r6042, 0; + mov.u32 %r6052, %r6037; + @%p710 bra $L__BB0_620; + + mov.u32 %r3837, -1; + shl.b32 %r3838, %r3837, %r6042; + not.b32 %r3839, %r3838; + and.b32 %r6043, %r5944, %r3839; + +$L__BB0_616: + setp.gt.u32 %p711, %r5907, 17476; + mov.u32 %r6052, 1; + @%p711 bra $L__BB0_620; + + sub.s32 %r3841, %r5906, %r5905; + min.u32 %r3842, %r3841, %r6042; + setp.eq.s32 %p712, %r3842, 32; + mov.u32 %r3843, -1; + shl.b32 %r3844, %r3843, %r3842; + not.b32 %r3845, %r3844; + selp.b32 %r3846, -1, %r3845, %p712; + and.b32 %r3847, %r3846, %r6043; + shl.b32 %r3848, %r3847, %r5905; + or.b32 %r5904, %r3848, %r5904; + add.s32 %r5905, %r3842, %r5905; + shr.u32 %r6043, %r6043, %r3842; + sub.s32 %r6042, %r6042, %r3842; + setp.lt.u32 %p713, %r5905, %r5906; + @%p713 bra $L__BB0_619; + + cvt.u64.u32 %rd281, %r5907; + add.s64 %rd282, %rd1, %rd281; + st.global.u8 [%rd282], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p714, %r5904, 255; + selp.b32 %r5906, 7, 8, %p714; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_619: + setp.ne.s32 %p715, %r6042, 0; + mov.u32 %r6052, %r6037; + @%p715 bra $L__BB0_616; + +$L__BB0_620: + and.b32 %r1534, %r5946, 4; + setp.eq.s32 %p716, %r1534, 0; + mov.u32 %r6067, %r6052; + @%p716 bra $L__BB0_627; + + shr.u32 %r3851, %r1486, 2; + and.b32 %r3852, %r3851, 1; + sub.s32 %r6057, %r1372, %r3852; + setp.eq.s32 %p717, %r6057, 0; + mov.u32 %r6067, %r6052; + @%p717 bra $L__BB0_627; + + mov.u32 %r3853, -1; + shl.b32 %r3854, %r3853, %r6057; + not.b32 %r3855, %r3854; + and.b32 %r6058, %r5960, %r3855; + +$L__BB0_623: + setp.gt.u32 %p718, %r5907, 17476; + mov.u32 %r6067, 1; + @%p718 bra $L__BB0_627; + + sub.s32 %r3857, %r5906, %r5905; + min.u32 %r3858, %r3857, %r6057; + setp.eq.s32 %p719, %r3858, 32; + mov.u32 %r3859, -1; + shl.b32 %r3860, %r3859, %r3858; + not.b32 %r3861, %r3860; + selp.b32 %r3862, -1, %r3861, %p719; + and.b32 %r3863, %r3862, %r6058; + shl.b32 %r3864, %r3863, %r5905; + or.b32 %r5904, %r3864, %r5904; + add.s32 %r5905, %r3858, %r5905; + shr.u32 %r6058, %r6058, %r3858; + sub.s32 %r6057, %r6057, %r3858; + setp.lt.u32 %p720, %r5905, %r5906; + @%p720 bra $L__BB0_626; + + cvt.u64.u32 %rd283, %r5907; + add.s64 %rd284, %rd1, %rd283; + st.global.u8 [%rd284], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p721, %r5904, 255; + selp.b32 %r5906, 7, 8, %p721; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_626: + setp.ne.s32 %p722, %r6057, 0; + mov.u32 %r6067, %r6052; + @%p722 bra $L__BB0_623; + +$L__BB0_627: + and.b32 %r1558, %r5946, 8; + setp.eq.s32 %p723, %r1558, 0; + mov.u32 %r5933, %r6067; + @%p723 bra $L__BB0_634; + + shr.u32 %r3867, %r1486, 3; + sub.s32 %r6072, %r1372, %r3867; + setp.eq.s32 %p724, %r6072, 0; + mov.u32 %r5933, %r6067; + @%p724 bra $L__BB0_634; + + mov.u32 %r3868, -1; + shl.b32 %r3869, %r3868, %r6072; + not.b32 %r3870, %r3869; + and.b32 %r6073, %r5959, %r3870; + +$L__BB0_630: + setp.gt.u32 %p725, %r5907, 17476; + mov.u32 %r5933, 1; + @%p725 bra $L__BB0_634; + + sub.s32 %r3872, %r5906, %r5905; + min.u32 %r3873, %r3872, %r6072; + setp.eq.s32 %p726, %r3873, 32; + mov.u32 %r3874, -1; + shl.b32 %r3875, %r3874, %r3873; + not.b32 %r3876, %r3875; + selp.b32 %r3877, -1, %r3876, %p726; + and.b32 %r3878, %r3877, %r6073; + shl.b32 %r3879, %r3878, %r5905; + or.b32 %r5904, %r3879, %r5904; + add.s32 %r5905, %r3873, %r5905; + shr.u32 %r6073, %r6073, %r3873; + sub.s32 %r6072, %r6072, %r3873; + setp.lt.u32 %p727, %r5905, %r5906; + @%p727 bra $L__BB0_633; + + cvt.u64.u32 %rd285, %r5907; + add.s64 %rd286, %rd1, %rd285; + st.global.u8 [%rd286], %r5904; + add.s32 %r5907, %r5907, 1; + setp.eq.s32 %p728, %r5904, 255; + selp.b32 %r5906, 7, 8, %p728; + mov.u32 %r5904, 0; + mov.u32 %r5905, %r5904; + +$L__BB0_633: + setp.ne.s32 %p729, %r6072, 0; + mov.u32 %r5933, %r6067; + @%p729 bra $L__BB0_630; + +$L__BB0_634: + and.b32 %r3882, %r5943, 255; + and.b32 %r3883, %r5808, 255; + setp.lt.u32 %p730, %r3882, %r3883; + cvt.u16.u32 %rs507, %r5943; + selp.b16 %rs508, %rs178, %rs507, %p730; + st.shared.u8 [%r1310+1], %rs508; + ld.shared.u8 %rs509, [%r1310+3]; + setp.gt.u16 %p731, %rs176, %rs509; + add.s32 %r6103, %r6103, 1; + add.s32 %r3884, %r5771, 3; + selp.b32 %r3885, %r6103, %r3884, %p731; + add.s32 %r3887, %r3615, %r3885; + ld.shared.u8 %r3888, [%r3887]; + add.s32 %r5770, %r3888, -1; + shr.u32 %r3889, %r1510, 1; + or.b32 %r3890, %r1316, %r3889; + st.shared.u8 [%r1310+2], %r5957; + st.shared.u8 [%r1313+1], %r3890; + ld.shared.u8 %rs510, [%r1313+3]; + mul.wide.u16 %r3891, %rs510, 4; + add.s32 %r3892, %r3891, %r1315; + shr.u32 %r3893, %r1558, 3; + st.shared.u8 [%r1313+2], %r3893; + shr.u32 %r3894, %r1558, 2; + shr.u32 %r3895, %r1534, 1; + or.b32 %r3896, %r3894, %r3895; + or.b32 %r5773, %r3896, %r3892; + add.s32 %r5769, %r5769, 1; + mov.u32 %r5827, %r5976; + +$L__BB0_635: + mov.u32 %r5771, %r6103; + mov.u32 %r5772, %r6102; + max.s32 %r3897, %r6107, 0; + mul.lo.s32 %r3898, %r1101, 6; + setp.gt.s32 %p732, %r1101, 0; + selp.b32 %r3899, %r3898, 0, %p732; + cvt.u64.u32 %rd287, %r3899; + add.s64 %rd14, %rd10, %rd287; + ld.global.u8 %rs204, [%rd14+1]; + add.s32 %r3900, %r3899, 2; + cvt.u64.u32 %rd288, %r3900; + add.s64 %rd289, %rd10, %rd288; + ld.global.u8 %rs205, [%rd289]; + ld.global.u8 %rs206, [%rd289+1]; + mul.lo.s32 %r3901, %r3897, 6; + cvt.u64.u32 %rd290, %r3901; + add.s64 %rd291, %rd10, %rd290; + ld.global.u8 %rs207, [%rd291]; + ld.global.u8 %rs208, [%rd291+1]; + add.s32 %r3902, %r3901, 2; + cvt.u64.u32 %rd292, %r3902; + add.s64 %rd293, %rd10, %rd292; + ld.global.u8 %rs209, [%rd293]; + ld.global.u8 %rs210, [%rd293+1]; + setp.eq.s16 %p733, %rs204, 0; + mov.u32 %r6119, %r5827; + @%p733 bra $L__BB0_642; + + ld.global.u8 %r6109, [%rd14]; + cvt.u32.u16 %r6108, %rs204; + +$L__BB0_637: + mov.u32 %r1609, %r6108; + setp.gt.u32 %p734, %r5719, 2879; + mov.u32 %r6119, 1; + @%p734 bra $L__BB0_642; + + mov.u32 %r3904, 8; + sub.s32 %r3905, %r3904, %r5717; + sub.s32 %r3906, %r3905, %r5718; + min.u32 %r3907, %r3906, %r1609; + setp.eq.s32 %p735, %r3907, 32; + mov.u32 %r3908, -1; + shl.b32 %r3909, %r3908, %r3907; + not.b32 %r3910, %r3909; + selp.b32 %r3911, -1, %r3910, %p735; + and.b32 %r3912, %r3911, %r6109; + shl.b32 %r3913, %r3912, %r5718; + cvt.u16.u32 %rs511, %r3913; + or.b16 %rs688, %rs688, %rs511; + add.s32 %r5718, %r3907, %r5718; + sub.s32 %r6108, %r1609, %r3907; + shr.u32 %r6109, %r6109, %r3907; + setp.gt.u32 %p736, %r3906, %r1609; + @%p736 bra $L__BB0_641; + + setp.ne.s32 %p737, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs512, %rs688, 255; + setp.ne.s16 %p738, %rs512, 127; + and.pred %p739, %p737, %p738; + @%p739 bra $L__BB0_641; + + mov.u32 %r3916, 20548; + sub.s32 %r3917, %r3916, %r5719; + cvt.u64.u32 %rd294, %r3917; + add.s64 %rd295, %rd1, %rd294; + st.global.u8 [%rd295], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p740, %rs512, 143; + selp.u32 %r5717, 1, 0, %p740; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_641: + setp.ne.s32 %p741, %r6108, 0; + mov.u32 %r6119, %r5827; + @%p741 bra $L__BB0_637; + +$L__BB0_642: + setp.eq.s16 %p742, %rs208, 0; + mov.u32 %r6131, %r6119; + @%p742 bra $L__BB0_649; + + cvt.u32.u16 %r3918, %rs207; + and.b32 %r6121, %r3918, 255; + cvt.u32.u16 %r3919, %rs208; + and.b32 %r6120, %r3919, 255; + +$L__BB0_644: + mov.u32 %r1628, %r6120; + setp.gt.u32 %p743, %r5719, 2879; + mov.u32 %r6131, 1; + @%p743 bra $L__BB0_649; + + mov.u32 %r3921, 8; + sub.s32 %r3922, %r3921, %r5717; + sub.s32 %r3923, %r3922, %r5718; + min.u32 %r3924, %r3923, %r1628; + setp.eq.s32 %p744, %r3924, 32; + mov.u32 %r3925, -1; + shl.b32 %r3926, %r3925, %r3924; + not.b32 %r3927, %r3926; + selp.b32 %r3928, -1, %r3927, %p744; + and.b32 %r3929, %r3928, %r6121; + shl.b32 %r3930, %r3929, %r5718; + cvt.u16.u32 %rs516, %r3930; + or.b16 %rs688, %rs688, %rs516; + add.s32 %r5718, %r3924, %r5718; + sub.s32 %r6120, %r1628, %r3924; + shr.u32 %r6121, %r6121, %r3924; + setp.gt.u32 %p745, %r3923, %r1628; + @%p745 bra $L__BB0_648; + + setp.ne.s32 %p746, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs517, %rs688, 255; + setp.ne.s16 %p747, %rs517, 127; + and.pred %p748, %p746, %p747; + @%p748 bra $L__BB0_648; + + mov.u32 %r3933, 20548; + sub.s32 %r3934, %r3933, %r5719; + cvt.u64.u32 %rd296, %r3934; + add.s64 %rd297, %rd1, %rd296; + st.global.u8 [%rd297], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p749, %rs517, 143; + selp.u32 %r5717, 1, 0, %p749; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_648: + setp.ne.s32 %p750, %r6120, 0; + mov.u32 %r6131, %r6119; + @%p750 bra $L__BB0_644; + +$L__BB0_649: + setp.eq.s16 %p751, %rs206, 0; + mov.u32 %r6143, %r6131; + @%p751 bra $L__BB0_656; + + cvt.u32.u16 %r3935, %rs206; + and.b32 %r6132, %r3935, 255; + cvt.u32.u16 %r3936, %rs205; + and.b32 %r6133, %r3936, 255; + +$L__BB0_651: + mov.u32 %r1647, %r6132; + setp.gt.u32 %p752, %r5719, 2879; + mov.u32 %r6143, 1; + @%p752 bra $L__BB0_656; + + mov.u32 %r3938, 8; + sub.s32 %r3939, %r3938, %r5717; + sub.s32 %r3940, %r3939, %r5718; + min.u32 %r3941, %r3940, %r1647; + setp.eq.s32 %p753, %r3941, 32; + mov.u32 %r3942, -1; + shl.b32 %r3943, %r3942, %r3941; + not.b32 %r3944, %r3943; + selp.b32 %r3945, -1, %r3944, %p753; + and.b32 %r3946, %r3945, %r6133; + shl.b32 %r3947, %r3946, %r5718; + cvt.u16.u32 %rs521, %r3947; + or.b16 %rs688, %rs688, %rs521; + add.s32 %r5718, %r3941, %r5718; + sub.s32 %r6132, %r1647, %r3941; + shr.u32 %r6133, %r6133, %r3941; + setp.gt.u32 %p754, %r3940, %r1647; + @%p754 bra $L__BB0_655; + + setp.ne.s32 %p755, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs522, %rs688, 255; + setp.ne.s16 %p756, %rs522, 127; + and.pred %p757, %p755, %p756; + @%p757 bra $L__BB0_655; + + mov.u32 %r3950, 20548; + sub.s32 %r3951, %r3950, %r5719; + cvt.u64.u32 %rd298, %r3951; + add.s64 %rd299, %rd1, %rd298; + st.global.u8 [%rd299], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p758, %rs522, 143; + selp.u32 %r5717, 1, 0, %p758; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_655: + setp.ne.s32 %p759, %r6132, 0; + mov.u32 %r6143, %r6131; + @%p759 bra $L__BB0_651; + +$L__BB0_656: + setp.eq.s16 %p760, %rs210, 0; + mov.u32 %r5716, %r6143; + @%p760 bra $L__BB0_663; + + cvt.u32.u16 %r3952, %rs209; + and.b32 %r6145, %r3952, 255; + cvt.u32.u16 %r3953, %rs210; + and.b32 %r6144, %r3953, 255; + +$L__BB0_658: + mov.u32 %r1666, %r6144; + setp.gt.u32 %p761, %r5719, 2879; + mov.u32 %r5716, 1; + @%p761 bra $L__BB0_663; + + mov.u32 %r3955, 8; + sub.s32 %r3956, %r3955, %r5717; + sub.s32 %r3957, %r3956, %r5718; + min.u32 %r3958, %r3957, %r1666; + setp.eq.s32 %p762, %r3958, 32; + mov.u32 %r3959, -1; + shl.b32 %r3960, %r3959, %r3958; + not.b32 %r3961, %r3960; + selp.b32 %r3962, -1, %r3961, %p762; + and.b32 %r3963, %r3962, %r6145; + shl.b32 %r3964, %r3963, %r5718; + cvt.u16.u32 %rs526, %r3964; + or.b16 %rs688, %rs688, %rs526; + add.s32 %r5718, %r3958, %r5718; + sub.s32 %r6144, %r1666, %r3958; + shr.u32 %r6145, %r6145, %r3958; + setp.gt.u32 %p763, %r3957, %r1666; + @%p763 bra $L__BB0_662; + + setp.ne.s32 %p764, %r5717, 0; + mov.u32 %r5717, 0; + and.b16 %rs527, %rs688, 255; + setp.ne.s16 %p765, %rs527, 127; + and.pred %p766, %p764, %p765; + @%p766 bra $L__BB0_662; + + mov.u32 %r3967, 20548; + sub.s32 %r3968, %r3967, %r5719; + cvt.u64.u32 %rd300, %r3968; + add.s64 %rd301, %rd1, %rd300; + st.global.u8 [%rd301], %rs688; + add.s32 %r5719, %r5719, 1; + setp.gt.u16 %p767, %rs527, 143; + selp.u32 %r5717, 1, 0, %p767; + mov.u32 %r5718, 0; + mov.u16 %rs688, 0; + +$L__BB0_662: + setp.ne.s32 %p768, %r6144, 0; + mov.u32 %r5716, %r6143; + @%p768 bra $L__BB0_658; + +$L__BB0_663: + add.s32 %r5768, %r5768, 4; + setp.lt.u32 %p769, %r5768, %r5; + @%p769 bra $L__BB0_423; + +$L__BB0_664: + add.s32 %r5752, %r5752, 2; + setp.lt.u32 %p770, %r5752, %r6; + @%p770 bra $L__BB0_421; + +$L__BB0_665: + setp.eq.s32 %p771, %r5483, 0; + mov.u32 %r6183, %r5480; + @%p771 bra $L__BB0_669; + + shl.b16 %rs530, %rs705, 1; + or.b16 %rs705, %rs530, 1; + add.s32 %r5277, %r5277, -1; + setp.ne.s32 %p772, %r5277, 0; + mov.u32 %r6183, %r5480; + @%p772 bra $L__BB0_669; + + setp.gt.u32 %p773, %r5271, 191; + mov.u32 %r5277, 0; + mov.u32 %r6183, 1; + @%p773 bra $L__BB0_669; + + add.s32 %r3971, %r5271, 17477; + cvt.u64.u32 %rd302, %r3971; + add.s64 %rd303, %rd1, %rd302; + and.b16 %rs532, %rs705, 255; + st.global.u8 [%rd303], %rs705; + add.s32 %r5271, %r5271, 1; + setp.eq.s16 %p774, %rs532, 255; + selp.b32 %r5277, 7, 8, %p774; + mov.u16 %rs705, 0; + mov.u32 %r6183, %r5480; + +$L__BB0_669: + cvt.u32.u16 %r3972, %rs705; + and.b32 %r3973, %r3972, 255; + shl.b32 %r3974, %r3973, %r5277; + cvt.u16.u32 %rs233, %r3974; + mov.u32 %r3975, -1; + shl.b32 %r3976, %r3975, %r5718; + not.b32 %r3977, %r3976; + mov.u32 %r3978, 255; + and.b32 %r3979, %r3977, 255; + setp.eq.s32 %p775, %r5718, 0; + selp.b32 %r1718, 0, %r3979, %p775; + shl.b32 %r1719, %r3978, %r5277; + and.b32 %r3980, %r1719, 255; + or.b32 %r3981, %r3980, %r1718; + setp.eq.s32 %p776, %r3981, 0; + mov.u32 %r6186, %r6183; + mov.u32 %r6188, %r5716; + @%p776 bra $L__BB0_675; + + or.b16 %rs234, %rs688, %rs233; + and.b16 %rs533, %rs234, 255; + xor.b16 %rs534, %rs234, %rs233; + cvt.u32.u16 %r3982, %rs534; + and.b32 %r3983, %r1719, %r3982; + and.b32 %r3984, %r3983, 255; + xor.b16 %rs535, %rs234, %rs688; + cvt.u32.u16 %r3985, %rs535; + and.b32 %r3986, %r1718, %r3985; + or.b32 %r3987, %r3984, %r3986; + setp.eq.s32 %p777, %r3987, 0; + setp.ne.s16 %p778, %rs533, 255; + and.pred %p779, %p778, %p777; + setp.gt.u32 %p780, %r5719, 1; + and.pred %p781, %p780, %p779; + add.s32 %r3988, %r5271, 17477; + cvt.u64.u32 %rd304, %r3988; + add.s64 %rd15, %rd1, %rd304; + @%p781 bra $L__BB0_673; + bra.uni $L__BB0_671; + +$L__BB0_673: + setp.gt.u32 %p785, %r5271, 191; + mov.u32 %r6186, 1; + mov.u32 %r6188, %r5716; + @%p785 bra $L__BB0_675; + + st.global.u8 [%rd15], %rs234; + add.s32 %r5271, %r5271, 1; + mov.u32 %r6186, %r6183; + mov.u32 %r6188, %r5716; + bra.uni $L__BB0_675; + +$L__BB0_671: + setp.gt.u32 %p782, %r5271, 191; + setp.gt.u32 %p783, %r5719, 2879; + or.pred %p784, %p783, %p782; + mov.u32 %r6186, 1; + mov.u32 %r6188, %r6186; + @%p784 bra $L__BB0_675; + + st.global.u8 [%rd15], %rs233; + add.s32 %r5271, %r5271, 1; + mov.u32 %r3991, 20548; + sub.s32 %r3992, %r3991, %r5719; + cvt.u64.u32 %rd305, %r3992; + add.s64 %rd306, %rd1, %rd305; + st.global.u8 [%rd306], %rs688; + add.s32 %r5719, %r5719, 1; + mov.u32 %r6186, %r6183; + mov.u32 %r6188, %r5716; + +$L__BB0_675: + setp.eq.s32 %p786, %r5905, 0; + @%p786 bra $L__BB0_679; + + sub.s32 %r3994, %r5906, %r5905; + mov.u32 %r3995, -1; + shl.b32 %r3996, %r3995, %r3994; + not.b32 %r3997, %r3996; + and.b32 %r3998, %r3997, 255; + shl.b32 %r3999, %r3998, %r5905; + or.b32 %r1727, %r3999, %r5904; + setp.eq.s32 %p787, %r1727, 255; + mov.u32 %r6190, %r5933; + @%p787 bra $L__BB0_681; + + setp.gt.u32 %p788, %r5907, 17476; + mov.u32 %r6190, 1; + @%p788 bra $L__BB0_681; + + cvt.u64.u32 %rd307, %r5907; + add.s64 %rd308, %rd1, %rd307; + st.global.u8 [%rd308], %r1727; + add.s32 %r5907, %r5907, 1; + mov.u32 %r6190, %r5933; + bra.uni $L__BB0_681; + +$L__BB0_679: + setp.ne.s32 %p789, %r5906, 7; + mov.u32 %r6190, %r5933; + @%p789 bra $L__BB0_681; + + setp.eq.s32 %p790, %r5907, 0; + add.s32 %r4001, %r5907, -1; + selp.b32 %r5907, 0, %r4001, %p790; + mov.u32 %r6190, %r5933; + +$L__BB0_681: + or.b32 %r4002, %r6188, %r6186; + or.b32 %r4003, %r4002, %r6190; + setp.eq.s32 %p791, %r4003, 0; + @%p791 bra $L__BB0_683; + + mov.u32 %r4004, 1; + st.global.u32 [%rd3], %r4004; + mov.u32 %r4005, 3; + st.global.u32 [%rd3+4], %r4005; + mov.u32 %r4006, 0; + st.global.u32 [%rd3+8], %r4006; + st.global.u32 [%rd3+12], %r4006; + st.global.u32 [%rd3+16], %r4006; + st.global.u32 [%rd3+20], %r4006; + st.global.u32 [%rd3+24], %r4006; + st.global.u32 [%rd3+28], %r4006; + bra.uni $L__BB0_1254; + +$L__BB0_683: + setp.eq.s32 %p1629, %r4, 2; + add.s32 %r1732, %r5907, %r5271; + add.s32 %r1733, %r1732, %r5719; + add.u64 %rd16, %SPL, 0; + mov.u32 %r6310, 1; + mov.u32 %r6308, 0; + mov.u32 %r6309, %r6308; + @%p1629 bra $L__BB0_932; + + setp.ne.s32 %p793, %r4, 3; + @%p793 bra $L__BB0_931; + + add.s32 %r4011, %r5, 3; + shr.u32 %r4012, %r4011, 2; + add.s32 %r4013, %r4012, 8; + setp.gt.u32 %p795, %r4013, 513; + mov.pred %p794, -1; + mov.u32 %r6307, 0; + mov.pred %p1632, %p794; + @%p795 bra $L__BB0_928; + + mov.u16 %rs747, 0; + st.local.u16 [%rd16], %rs747; + st.local.u16 [%rd16+2], %rs747; + st.local.u16 [%rd16+4], %rs747; + st.local.u16 [%rd16+6], %rs747; + st.local.u16 [%rd16+8], %rs747; + st.local.u16 [%rd16+10], %rs747; + st.local.u16 [%rd16+12], %rs747; + st.local.u16 [%rd16+14], %rs747; + st.local.u16 [%rd16+16], %rs747; + st.local.u16 [%rd16+18], %rs747; + st.local.u16 [%rd16+20], %rs747; + st.local.u16 [%rd16+22], %rs747; + st.local.u16 [%rd16+24], %rs747; + st.local.u16 [%rd16+26], %rs747; + st.local.u16 [%rd16+28], %rs747; + st.local.u16 [%rd16+30], %rs747; + st.local.u16 [%rd16+32], %rs747; + st.local.u16 [%rd16+34], %rs747; + st.local.u16 [%rd16+36], %rs747; + st.local.u16 [%rd16+38], %rs747; + st.local.u16 [%rd16+40], %rs747; + st.local.u16 [%rd16+42], %rs747; + st.local.u16 [%rd16+44], %rs747; + st.local.u16 [%rd16+46], %rs747; + st.local.u16 [%rd16+48], %rs747; + st.local.u16 [%rd16+50], %rs747; + st.local.u16 [%rd16+52], %rs747; + st.local.u16 [%rd16+54], %rs747; + st.local.u16 [%rd16+56], %rs747; + st.local.u16 [%rd16+58], %rs747; + st.local.u16 [%rd16+60], %rs747; + st.local.u16 [%rd16+62], %rs747; + st.local.u16 [%rd16+64], %rs747; + st.local.u16 [%rd16+66], %rs747; + st.local.u16 [%rd16+68], %rs747; + st.local.u16 [%rd16+70], %rs747; + st.local.u16 [%rd16+72], %rs747; + st.local.u16 [%rd16+74], %rs747; + st.local.u16 [%rd16+76], %rs747; + st.local.u16 [%rd16+78], %rs747; + st.local.u16 [%rd16+80], %rs747; + st.local.u16 [%rd16+82], %rs747; + st.local.u16 [%rd16+84], %rs747; + st.local.u16 [%rd16+86], %rs747; + st.local.u16 [%rd16+88], %rs747; + st.local.u16 [%rd16+90], %rs747; + st.local.u16 [%rd16+92], %rs747; + st.local.u16 [%rd16+94], %rs747; + st.local.u16 [%rd16+96], %rs747; + st.local.u16 [%rd16+98], %rs747; + st.local.u16 [%rd16+100], %rs747; + st.local.u16 [%rd16+102], %rs747; + st.local.u16 [%rd16+104], %rs747; + st.local.u16 [%rd16+106], %rs747; + st.local.u16 [%rd16+108], %rs747; + st.local.u16 [%rd16+110], %rs747; + st.local.u16 [%rd16+112], %rs747; + st.local.u16 [%rd16+114], %rs747; + st.local.u16 [%rd16+116], %rs747; + st.local.u16 [%rd16+118], %rs747; + st.local.u16 [%rd16+120], %rs747; + st.local.u16 [%rd16+122], %rs747; + st.local.u16 [%rd16+124], %rs747; + st.local.u16 [%rd16+126], %rs747; + st.local.u16 [%rd16+128], %rs747; + st.local.u16 [%rd16+130], %rs747; + st.local.u16 [%rd16+132], %rs747; + st.local.u16 [%rd16+134], %rs747; + st.local.u16 [%rd16+136], %rs747; + st.local.u16 [%rd16+138], %rs747; + st.local.u16 [%rd16+140], %rs747; + st.local.u16 [%rd16+142], %rs747; + st.local.u16 [%rd16+144], %rs747; + st.local.u16 [%rd16+146], %rs747; + st.local.u16 [%rd16+148], %rs747; + st.local.u16 [%rd16+150], %rs747; + st.local.u16 [%rd16+152], %rs747; + st.local.u16 [%rd16+154], %rs747; + st.local.u16 [%rd16+156], %rs747; + st.local.u16 [%rd16+158], %rs747; + st.local.u16 [%rd16+160], %rs747; + st.local.u16 [%rd16+162], %rs747; + st.local.u16 [%rd16+164], %rs747; + st.local.u16 [%rd16+166], %rs747; + st.local.u16 [%rd16+168], %rs747; + st.local.u16 [%rd16+170], %rs747; + st.local.u16 [%rd16+172], %rs747; + st.local.u16 [%rd16+174], %rs747; + st.local.u16 [%rd16+176], %rs747; + st.local.u16 [%rd16+178], %rs747; + st.local.u16 [%rd16+180], %rs747; + st.local.u16 [%rd16+182], %rs747; + st.local.u16 [%rd16+184], %rs747; + st.local.u16 [%rd16+186], %rs747; + st.local.u16 [%rd16+188], %rs747; + st.local.u16 [%rd16+190], %rs747; + st.local.u16 [%rd16+192], %rs747; + st.local.u16 [%rd16+194], %rs747; + st.local.u16 [%rd16+196], %rs747; + st.local.u16 [%rd16+198], %rs747; + st.local.u16 [%rd16+200], %rs747; + st.local.u16 [%rd16+202], %rs747; + st.local.u16 [%rd16+204], %rs747; + st.local.u16 [%rd16+206], %rs747; + st.local.u16 [%rd16+208], %rs747; + st.local.u16 [%rd16+210], %rs747; + st.local.u16 [%rd16+212], %rs747; + st.local.u16 [%rd16+214], %rs747; + st.local.u16 [%rd16+216], %rs747; + st.local.u16 [%rd16+218], %rs747; + st.local.u16 [%rd16+220], %rs747; + st.local.u16 [%rd16+222], %rs747; + st.local.u16 [%rd16+224], %rs747; + st.local.u16 [%rd16+226], %rs747; + st.local.u16 [%rd16+228], %rs747; + st.local.u16 [%rd16+230], %rs747; + st.local.u16 [%rd16+232], %rs747; + st.local.u16 [%rd16+234], %rs747; + st.local.u16 [%rd16+236], %rs747; + st.local.u16 [%rd16+238], %rs747; + st.local.u16 [%rd16+240], %rs747; + st.local.u16 [%rd16+242], %rs747; + st.local.u16 [%rd16+244], %rs747; + st.local.u16 [%rd16+246], %rs747; + st.local.u16 [%rd16+248], %rs747; + st.local.u16 [%rd16+250], %rs747; + st.local.u16 [%rd16+252], %rs747; + st.local.u16 [%rd16+254], %rs747; + st.local.u16 [%rd16+256], %rs747; + st.local.u16 [%rd16+258], %rs747; + st.local.u16 [%rd16+260], %rs747; + st.local.u16 [%rd16+262], %rs747; + st.local.u16 [%rd16+264], %rs747; + st.local.u16 [%rd16+266], %rs747; + st.local.u16 [%rd16+268], %rs747; + st.local.u16 [%rd16+270], %rs747; + st.local.u16 [%rd16+272], %rs747; + st.local.u16 [%rd16+274], %rs747; + st.local.u16 [%rd16+276], %rs747; + st.local.u16 [%rd16+278], %rs747; + st.local.u16 [%rd16+280], %rs747; + st.local.u16 [%rd16+282], %rs747; + st.local.u16 [%rd16+284], %rs747; + st.local.u16 [%rd16+286], %rs747; + st.local.u16 [%rd16+288], %rs747; + st.local.u16 [%rd16+290], %rs747; + st.local.u16 [%rd16+292], %rs747; + st.local.u16 [%rd16+294], %rs747; + st.local.u16 [%rd16+296], %rs747; + st.local.u16 [%rd16+298], %rs747; + st.local.u16 [%rd16+300], %rs747; + st.local.u16 [%rd16+302], %rs747; + st.local.u16 [%rd16+304], %rs747; + st.local.u16 [%rd16+306], %rs747; + st.local.u16 [%rd16+308], %rs747; + st.local.u16 [%rd16+310], %rs747; + st.local.u16 [%rd16+312], %rs747; + st.local.u16 [%rd16+314], %rs747; + st.local.u16 [%rd16+316], %rs747; + st.local.u16 [%rd16+318], %rs747; + st.local.u16 [%rd16+320], %rs747; + st.local.u16 [%rd16+322], %rs747; + st.local.u16 [%rd16+324], %rs747; + st.local.u16 [%rd16+326], %rs747; + st.local.u16 [%rd16+328], %rs747; + st.local.u16 [%rd16+330], %rs747; + st.local.u16 [%rd16+332], %rs747; + st.local.u16 [%rd16+334], %rs747; + st.local.u16 [%rd16+336], %rs747; + st.local.u16 [%rd16+338], %rs747; + st.local.u16 [%rd16+340], %rs747; + st.local.u16 [%rd16+342], %rs747; + st.local.u16 [%rd16+344], %rs747; + st.local.u16 [%rd16+346], %rs747; + st.local.u16 [%rd16+348], %rs747; + st.local.u16 [%rd16+350], %rs747; + st.local.u16 [%rd16+352], %rs747; + st.local.u16 [%rd16+354], %rs747; + st.local.u16 [%rd16+356], %rs747; + st.local.u16 [%rd16+358], %rs747; + st.local.u16 [%rd16+360], %rs747; + st.local.u16 [%rd16+362], %rs747; + st.local.u16 [%rd16+364], %rs747; + st.local.u16 [%rd16+366], %rs747; + st.local.u16 [%rd16+368], %rs747; + st.local.u16 [%rd16+370], %rs747; + st.local.u16 [%rd16+372], %rs747; + st.local.u16 [%rd16+374], %rs747; + st.local.u16 [%rd16+376], %rs747; + st.local.u16 [%rd16+378], %rs747; + st.local.u16 [%rd16+380], %rs747; + st.local.u16 [%rd16+382], %rs747; + st.local.u16 [%rd16+384], %rs747; + st.local.u16 [%rd16+386], %rs747; + st.local.u16 [%rd16+388], %rs747; + st.local.u16 [%rd16+390], %rs747; + st.local.u16 [%rd16+392], %rs747; + st.local.u16 [%rd16+394], %rs747; + st.local.u16 [%rd16+396], %rs747; + st.local.u16 [%rd16+398], %rs747; + st.local.u16 [%rd16+400], %rs747; + st.local.u16 [%rd16+402], %rs747; + st.local.u16 [%rd16+404], %rs747; + st.local.u16 [%rd16+406], %rs747; + st.local.u16 [%rd16+408], %rs747; + st.local.u16 [%rd16+410], %rs747; + st.local.u16 [%rd16+412], %rs747; + st.local.u16 [%rd16+414], %rs747; + st.local.u16 [%rd16+416], %rs747; + st.local.u16 [%rd16+418], %rs747; + st.local.u16 [%rd16+420], %rs747; + st.local.u16 [%rd16+422], %rs747; + st.local.u16 [%rd16+424], %rs747; + st.local.u16 [%rd16+426], %rs747; + st.local.u16 [%rd16+428], %rs747; + st.local.u16 [%rd16+430], %rs747; + st.local.u16 [%rd16+432], %rs747; + st.local.u16 [%rd16+434], %rs747; + st.local.u16 [%rd16+436], %rs747; + st.local.u16 [%rd16+438], %rs747; + st.local.u16 [%rd16+440], %rs747; + st.local.u16 [%rd16+442], %rs747; + st.local.u16 [%rd16+444], %rs747; + st.local.u16 [%rd16+446], %rs747; + st.local.u16 [%rd16+448], %rs747; + st.local.u16 [%rd16+450], %rs747; + st.local.u16 [%rd16+452], %rs747; + st.local.u16 [%rd16+454], %rs747; + st.local.u16 [%rd16+456], %rs747; + st.local.u16 [%rd16+458], %rs747; + st.local.u16 [%rd16+460], %rs747; + st.local.u16 [%rd16+462], %rs747; + st.local.u16 [%rd16+464], %rs747; + st.local.u16 [%rd16+466], %rs747; + st.local.u16 [%rd16+468], %rs747; + st.local.u16 [%rd16+470], %rs747; + st.local.u16 [%rd16+472], %rs747; + st.local.u16 [%rd16+474], %rs747; + st.local.u16 [%rd16+476], %rs747; + st.local.u16 [%rd16+478], %rs747; + st.local.u16 [%rd16+480], %rs747; + st.local.u16 [%rd16+482], %rs747; + st.local.u16 [%rd16+484], %rs747; + st.local.u16 [%rd16+486], %rs747; + st.local.u16 [%rd16+488], %rs747; + st.local.u16 [%rd16+490], %rs747; + st.local.u16 [%rd16+492], %rs747; + st.local.u16 [%rd16+494], %rs747; + st.local.u16 [%rd16+496], %rs747; + st.local.u16 [%rd16+498], %rs747; + st.local.u16 [%rd16+500], %rs747; + st.local.u16 [%rd16+502], %rs747; + st.local.u16 [%rd16+504], %rs747; + st.local.u16 [%rd16+506], %rs747; + st.local.u16 [%rd16+508], %rs747; + st.local.u16 [%rd16+510], %rs747; + st.local.u16 [%rd16+512], %rs747; + st.local.u16 [%rd16+514], %rs747; + st.local.u16 [%rd16+516], %rs747; + st.local.u16 [%rd16+518], %rs747; + st.local.u16 [%rd16+520], %rs747; + st.local.u16 [%rd16+522], %rs747; + st.local.u16 [%rd16+524], %rs747; + st.local.u16 [%rd16+526], %rs747; + st.local.u16 [%rd16+528], %rs747; + st.local.u16 [%rd16+530], %rs747; + st.local.u16 [%rd16+532], %rs747; + st.local.u16 [%rd16+534], %rs747; + st.local.u16 [%rd16+536], %rs747; + st.local.u16 [%rd16+538], %rs747; + st.local.u16 [%rd16+540], %rs747; + st.local.u16 [%rd16+542], %rs747; + st.local.u16 [%rd16+544], %rs747; + st.local.u16 [%rd16+546], %rs747; + st.local.u16 [%rd16+548], %rs747; + st.local.u16 [%rd16+550], %rs747; + st.local.u16 [%rd16+552], %rs747; + st.local.u16 [%rd16+554], %rs747; + st.local.u16 [%rd16+556], %rs747; + st.local.u16 [%rd16+558], %rs747; + st.local.u16 [%rd16+560], %rs747; + st.local.u16 [%rd16+562], %rs747; + st.local.u16 [%rd16+564], %rs747; + st.local.u16 [%rd16+566], %rs747; + st.local.u16 [%rd16+568], %rs747; + st.local.u16 [%rd16+570], %rs747; + st.local.u16 [%rd16+572], %rs747; + st.local.u16 [%rd16+574], %rs747; + st.local.u16 [%rd16+576], %rs747; + st.local.u16 [%rd16+578], %rs747; + st.local.u16 [%rd16+580], %rs747; + st.local.u16 [%rd16+582], %rs747; + st.local.u16 [%rd16+584], %rs747; + st.local.u16 [%rd16+586], %rs747; + st.local.u16 [%rd16+588], %rs747; + st.local.u16 [%rd16+590], %rs747; + st.local.u16 [%rd16+592], %rs747; + st.local.u16 [%rd16+594], %rs747; + st.local.u16 [%rd16+596], %rs747; + st.local.u16 [%rd16+598], %rs747; + st.local.u16 [%rd16+600], %rs747; + st.local.u16 [%rd16+602], %rs747; + st.local.u16 [%rd16+604], %rs747; + st.local.u16 [%rd16+606], %rs747; + st.local.u16 [%rd16+608], %rs747; + st.local.u16 [%rd16+610], %rs747; + st.local.u16 [%rd16+612], %rs747; + st.local.u16 [%rd16+614], %rs747; + st.local.u16 [%rd16+616], %rs747; + st.local.u16 [%rd16+618], %rs747; + st.local.u16 [%rd16+620], %rs747; + st.local.u16 [%rd16+622], %rs747; + st.local.u16 [%rd16+624], %rs747; + st.local.u16 [%rd16+626], %rs747; + st.local.u16 [%rd16+628], %rs747; + st.local.u16 [%rd16+630], %rs747; + st.local.u16 [%rd16+632], %rs747; + st.local.u16 [%rd16+634], %rs747; + st.local.u16 [%rd16+636], %rs747; + st.local.u16 [%rd16+638], %rs747; + st.local.u16 [%rd16+640], %rs747; + st.local.u16 [%rd16+642], %rs747; + st.local.u16 [%rd16+644], %rs747; + st.local.u16 [%rd16+646], %rs747; + st.local.u16 [%rd16+648], %rs747; + st.local.u16 [%rd16+650], %rs747; + st.local.u16 [%rd16+652], %rs747; + st.local.u16 [%rd16+654], %rs747; + st.local.u16 [%rd16+656], %rs747; + st.local.u16 [%rd16+658], %rs747; + st.local.u16 [%rd16+660], %rs747; + st.local.u16 [%rd16+662], %rs747; + st.local.u16 [%rd16+664], %rs747; + st.local.u16 [%rd16+666], %rs747; + st.local.u16 [%rd16+668], %rs747; + st.local.u16 [%rd16+670], %rs747; + st.local.u16 [%rd16+672], %rs747; + st.local.u16 [%rd16+674], %rs747; + st.local.u16 [%rd16+676], %rs747; + st.local.u16 [%rd16+678], %rs747; + st.local.u16 [%rd16+680], %rs747; + st.local.u16 [%rd16+682], %rs747; + st.local.u16 [%rd16+684], %rs747; + st.local.u16 [%rd16+686], %rs747; + st.local.u16 [%rd16+688], %rs747; + st.local.u16 [%rd16+690], %rs747; + st.local.u16 [%rd16+692], %rs747; + st.local.u16 [%rd16+694], %rs747; + st.local.u16 [%rd16+696], %rs747; + st.local.u16 [%rd16+698], %rs747; + st.local.u16 [%rd16+700], %rs747; + st.local.u16 [%rd16+702], %rs747; + st.local.u16 [%rd16+704], %rs747; + st.local.u16 [%rd16+706], %rs747; + st.local.u16 [%rd16+708], %rs747; + st.local.u16 [%rd16+710], %rs747; + st.local.u16 [%rd16+712], %rs747; + st.local.u16 [%rd16+714], %rs747; + st.local.u16 [%rd16+716], %rs747; + st.local.u16 [%rd16+718], %rs747; + st.local.u16 [%rd16+720], %rs747; + st.local.u16 [%rd16+722], %rs747; + st.local.u16 [%rd16+724], %rs747; + st.local.u16 [%rd16+726], %rs747; + st.local.u16 [%rd16+728], %rs747; + st.local.u16 [%rd16+730], %rs747; + st.local.u16 [%rd16+732], %rs747; + st.local.u16 [%rd16+734], %rs747; + st.local.u16 [%rd16+736], %rs747; + st.local.u16 [%rd16+738], %rs747; + st.local.u16 [%rd16+740], %rs747; + st.local.u16 [%rd16+742], %rs747; + st.local.u16 [%rd16+744], %rs747; + st.local.u16 [%rd16+746], %rs747; + st.local.u16 [%rd16+748], %rs747; + st.local.u16 [%rd16+750], %rs747; + st.local.u16 [%rd16+752], %rs747; + st.local.u16 [%rd16+754], %rs747; + st.local.u16 [%rd16+756], %rs747; + st.local.u16 [%rd16+758], %rs747; + st.local.u16 [%rd16+760], %rs747; + st.local.u16 [%rd16+762], %rs747; + st.local.u16 [%rd16+764], %rs747; + st.local.u16 [%rd16+766], %rs747; + st.local.u16 [%rd16+768], %rs747; + st.local.u16 [%rd16+770], %rs747; + st.local.u16 [%rd16+772], %rs747; + st.local.u16 [%rd16+774], %rs747; + st.local.u16 [%rd16+776], %rs747; + st.local.u16 [%rd16+778], %rs747; + st.local.u16 [%rd16+780], %rs747; + st.local.u16 [%rd16+782], %rs747; + st.local.u16 [%rd16+784], %rs747; + st.local.u16 [%rd16+786], %rs747; + st.local.u16 [%rd16+788], %rs747; + st.local.u16 [%rd16+790], %rs747; + st.local.u16 [%rd16+792], %rs747; + st.local.u16 [%rd16+794], %rs747; + st.local.u16 [%rd16+796], %rs747; + st.local.u16 [%rd16+798], %rs747; + st.local.u16 [%rd16+800], %rs747; + st.local.u16 [%rd16+802], %rs747; + st.local.u16 [%rd16+804], %rs747; + st.local.u16 [%rd16+806], %rs747; + st.local.u16 [%rd16+808], %rs747; + st.local.u16 [%rd16+810], %rs747; + st.local.u16 [%rd16+812], %rs747; + st.local.u16 [%rd16+814], %rs747; + st.local.u16 [%rd16+816], %rs747; + st.local.u16 [%rd16+818], %rs747; + st.local.u16 [%rd16+820], %rs747; + st.local.u16 [%rd16+822], %rs747; + st.local.u16 [%rd16+824], %rs747; + st.local.u16 [%rd16+826], %rs747; + st.local.u16 [%rd16+828], %rs747; + st.local.u16 [%rd16+830], %rs747; + st.local.u16 [%rd16+832], %rs747; + st.local.u16 [%rd16+834], %rs747; + st.local.u16 [%rd16+836], %rs747; + st.local.u16 [%rd16+838], %rs747; + st.local.u16 [%rd16+840], %rs747; + st.local.u16 [%rd16+842], %rs747; + st.local.u16 [%rd16+844], %rs747; + st.local.u16 [%rd16+846], %rs747; + st.local.u16 [%rd16+848], %rs747; + st.local.u16 [%rd16+850], %rs747; + st.local.u16 [%rd16+852], %rs747; + st.local.u16 [%rd16+854], %rs747; + st.local.u16 [%rd16+856], %rs747; + st.local.u16 [%rd16+858], %rs747; + st.local.u16 [%rd16+860], %rs747; + st.local.u16 [%rd16+862], %rs747; + st.local.u16 [%rd16+864], %rs747; + st.local.u16 [%rd16+866], %rs747; + st.local.u16 [%rd16+868], %rs747; + st.local.u16 [%rd16+870], %rs747; + st.local.u16 [%rd16+872], %rs747; + st.local.u16 [%rd16+874], %rs747; + st.local.u16 [%rd16+876], %rs747; + st.local.u16 [%rd16+878], %rs747; + st.local.u16 [%rd16+880], %rs747; + st.local.u16 [%rd16+882], %rs747; + st.local.u16 [%rd16+884], %rs747; + st.local.u16 [%rd16+886], %rs747; + st.local.u16 [%rd16+888], %rs747; + st.local.u16 [%rd16+890], %rs747; + st.local.u16 [%rd16+892], %rs747; + st.local.u16 [%rd16+894], %rs747; + st.local.u16 [%rd16+896], %rs747; + st.local.u16 [%rd16+898], %rs747; + st.local.u16 [%rd16+900], %rs747; + st.local.u16 [%rd16+902], %rs747; + st.local.u16 [%rd16+904], %rs747; + st.local.u16 [%rd16+906], %rs747; + st.local.u16 [%rd16+908], %rs747; + st.local.u16 [%rd16+910], %rs747; + st.local.u16 [%rd16+912], %rs747; + st.local.u16 [%rd16+914], %rs747; + st.local.u16 [%rd16+916], %rs747; + st.local.u16 [%rd16+918], %rs747; + st.local.u16 [%rd16+920], %rs747; + st.local.u16 [%rd16+922], %rs747; + st.local.u16 [%rd16+924], %rs747; + st.local.u16 [%rd16+926], %rs747; + st.local.u16 [%rd16+928], %rs747; + st.local.u16 [%rd16+930], %rs747; + st.local.u16 [%rd16+932], %rs747; + st.local.u16 [%rd16+934], %rs747; + st.local.u16 [%rd16+936], %rs747; + st.local.u16 [%rd16+938], %rs747; + st.local.u16 [%rd16+940], %rs747; + st.local.u16 [%rd16+942], %rs747; + st.local.u16 [%rd16+944], %rs747; + st.local.u16 [%rd16+946], %rs747; + st.local.u16 [%rd16+948], %rs747; + st.local.u16 [%rd16+950], %rs747; + st.local.u16 [%rd16+952], %rs747; + st.local.u16 [%rd16+954], %rs747; + st.local.u16 [%rd16+956], %rs747; + st.local.u16 [%rd16+958], %rs747; + st.local.u16 [%rd16+960], %rs747; + st.local.u16 [%rd16+962], %rs747; + st.local.u16 [%rd16+964], %rs747; + st.local.u16 [%rd16+966], %rs747; + st.local.u16 [%rd16+968], %rs747; + st.local.u16 [%rd16+970], %rs747; + st.local.u16 [%rd16+972], %rs747; + st.local.u16 [%rd16+974], %rs747; + st.local.u16 [%rd16+976], %rs747; + st.local.u16 [%rd16+978], %rs747; + st.local.u16 [%rd16+980], %rs747; + st.local.u16 [%rd16+982], %rs747; + st.local.u16 [%rd16+984], %rs747; + st.local.u16 [%rd16+986], %rs747; + st.local.u16 [%rd16+988], %rs747; + st.local.u16 [%rd16+990], %rs747; + st.local.u16 [%rd16+992], %rs747; + st.local.u16 [%rd16+994], %rs747; + st.local.u16 [%rd16+996], %rs747; + st.local.u16 [%rd16+998], %rs747; + st.local.u16 [%rd16+1000], %rs747; + st.local.u16 [%rd16+1002], %rs747; + st.local.u16 [%rd16+1004], %rs747; + st.local.u16 [%rd16+1006], %rs747; + st.local.u16 [%rd16+1008], %rs747; + st.local.u16 [%rd16+1010], %rs747; + st.local.u16 [%rd16+1012], %rs747; + st.local.u16 [%rd16+1014], %rs747; + st.local.u16 [%rd16+1016], %rs747; + st.local.u16 [%rd16+1018], %rs747; + st.local.u16 [%rd16+1020], %rs747; + st.local.u16 [%rd16+1022], %rs747; + st.local.u16 [%rd16+1024], %rs747; + mov.u32 %r6192, 0; + mov.u32 %r6302, %r6192; + mov.u32 %r6298, %r6192; + mov.u32 %r6300, %r6192; + +$L__BB0_687: + @%p9 bra $L__BB0_926; + + sub.s32 %r4020, %r6, %r6192; + add.s32 %r1738, %r6192, 4; + mul.lo.s32 %r1739, %r1738, %r1; + add.s32 %r1740, %r6192, 5; + add.s32 %r1741, %r1739, %r1; + add.s32 %r1742, %r6192, 6; + shl.b32 %r4021, %r1, 1; + add.s32 %r1743, %r1739, %r4021; + add.s32 %r1744, %r6192, 7; + mul.lo.s32 %r4022, %r1, 3; + add.s32 %r1745, %r1739, %r4022; + add.s32 %r1746, %r6192, 1; + add.s32 %r1747, %r6192, 2; + add.s32 %r1748, %r6192, 3; + mul.lo.s32 %r1749, %r6192, %r1; + add.s32 %r1750, %r1749, %r4022; + sub.s32 %r1751, %r1750, %r1; + sub.s32 %r1752, %r1751, %r1; + setp.lt.u32 %p797, %r4020, 2; + selp.b32 %r4023, 4369, 13107, %p797; + setp.lt.u32 %p798, %r4020, 3; + selp.b32 %r4024, %r4023, 30583, %p798; + setp.lt.u32 %p799, %r4020, 4; + selp.b32 %r1753, %r4024, 65535, %p799; + mov.u32 %r4019, 0; + mov.u32 %r6196, %r4019; + mov.u32 %r6197, %r4019; + +$L__BB0_689: + shr.u32 %r4026, %r6196, 2; + mul.wide.u32 %rd311, %r4026, 2; + add.s64 %rd18, %rd16, %rd311; + ld.local.u16 %rs237, [%rd18]; + ld.local.u16 %rs238, [%rd18+2]; + setp.ge.u32 %p800, %r6196, %r5; + mov.u32 %r6208, %r4019; + @%p800 bra $L__BB0_698; + + setp.ge.u32 %p801, %r1738, %r6; + mov.u32 %r6208, 0; + @%p801 bra $L__BB0_692; + + add.s32 %r4028, %r1739, %r6196; + mul.wide.u32 %rd312, %r4028, 4; + add.s64 %rd313, %rd2, %rd312; + ld.global.u32 %r4029, [%rd313]; + abs.s32 %r4030, %r4029; + setp.gt.u32 %p802, %r4030, 4; + and.b32 %r4031, %r4030, 1; + setp.eq.b32 %p803, %r4031, 1; + and.pred %p804, %p802, %p803; + selp.u32 %r6208, 1, 0, %p804; + +$L__BB0_692: + setp.ge.u32 %p805, %r1740, %r6; + @%p805 bra $L__BB0_694; + + add.s32 %r4032, %r1741, %r6196; + mul.wide.u32 %rd314, %r4032, 4; + add.s64 %rd315, %rd2, %rd314; + ld.global.u32 %r4033, [%rd315]; + abs.s32 %r4034, %r4033; + setp.gt.u32 %p806, %r4034, 4; + and.b32 %r4035, %r4034, 1; + setp.eq.b32 %p807, %r4035, 1; + and.pred %p808, %p806, %p807; + selp.b32 %r4036, 2, 0, %p808; + or.b32 %r6208, %r4036, %r6208; + +$L__BB0_694: + setp.ge.u32 %p809, %r1742, %r6; + @%p809 bra $L__BB0_696; + + add.s32 %r4037, %r1743, %r6196; + mul.wide.u32 %rd316, %r4037, 4; + add.s64 %rd317, %rd2, %rd316; + ld.global.u32 %r4038, [%rd317]; + abs.s32 %r4039, %r4038; + setp.gt.u32 %p810, %r4039, 4; + and.b32 %r4040, %r4039, 1; + setp.eq.b32 %p811, %r4040, 1; + and.pred %p812, %p810, %p811; + selp.b32 %r4041, 4, 0, %p812; + or.b32 %r6208, %r4041, %r6208; + +$L__BB0_696: + setp.ge.u32 %p813, %r1744, %r6; + @%p813 bra $L__BB0_698; + + add.s32 %r4042, %r1745, %r6196; + mul.wide.u32 %rd318, %r4042, 4; + add.s64 %rd319, %rd2, %rd318; + ld.global.u32 %r4043, [%rd319]; + abs.s32 %r4044, %r4043; + setp.gt.u32 %p814, %r4044, 4; + and.b32 %r4045, %r4044, 1; + setp.eq.b32 %p815, %r4045, 1; + and.pred %p816, %p814, %p815; + selp.b32 %r4046, 8, 0, %p816; + or.b32 %r6208, %r4046, %r6208; + +$L__BB0_698: + add.s32 %r1767, %r6196, 1; + setp.ge.u32 %p817, %r1767, %r5; + @%p817 bra $L__BB0_707; + + setp.ge.u32 %p818, %r1738, %r6; + @%p818 bra $L__BB0_701; + + add.s32 %r4047, %r1739, %r1767; + mul.wide.u32 %rd320, %r4047, 4; + add.s64 %rd321, %rd2, %rd320; + ld.global.u32 %r4048, [%rd321]; + abs.s32 %r4049, %r4048; + setp.gt.u32 %p819, %r4049, 4; + and.b32 %r4050, %r4049, 1; + setp.eq.b32 %p820, %r4050, 1; + and.pred %p821, %p819, %p820; + selp.b32 %r4051, 16, 0, %p821; + or.b32 %r6208, %r4051, %r6208; + +$L__BB0_701: + setp.ge.u32 %p822, %r1740, %r6; + @%p822 bra $L__BB0_703; + + add.s32 %r4052, %r1741, %r1767; + mul.wide.u32 %rd322, %r4052, 4; + add.s64 %rd323, %rd2, %rd322; + ld.global.u32 %r4053, [%rd323]; + abs.s32 %r4054, %r4053; + setp.gt.u32 %p823, %r4054, 4; + and.b32 %r4055, %r4054, 1; + setp.eq.b32 %p824, %r4055, 1; + and.pred %p825, %p823, %p824; + selp.b32 %r4056, 32, 0, %p825; + or.b32 %r6208, %r4056, %r6208; + +$L__BB0_703: + setp.ge.u32 %p826, %r1742, %r6; + @%p826 bra $L__BB0_705; + + add.s32 %r4057, %r1743, %r1767; + mul.wide.u32 %rd324, %r4057, 4; + add.s64 %rd325, %rd2, %rd324; + ld.global.u32 %r4058, [%rd325]; + abs.s32 %r4059, %r4058; + setp.gt.u32 %p827, %r4059, 4; + and.b32 %r4060, %r4059, 1; + setp.eq.b32 %p828, %r4060, 1; + and.pred %p829, %p827, %p828; + selp.b32 %r4061, 64, 0, %p829; + or.b32 %r6208, %r4061, %r6208; + +$L__BB0_705: + setp.ge.u32 %p830, %r1744, %r6; + @%p830 bra $L__BB0_707; + + add.s32 %r4062, %r1745, %r1767; + mul.wide.u32 %rd326, %r4062, 4; + add.s64 %rd327, %rd2, %rd326; + ld.global.u32 %r4063, [%rd327]; + abs.s32 %r4064, %r4063; + setp.gt.u32 %p831, %r4064, 4; + and.b32 %r4065, %r4064, 1; + setp.eq.b32 %p832, %r4065, 1; + and.pred %p833, %p831, %p832; + selp.b32 %r4066, 128, 0, %p833; + or.b32 %r6208, %r4066, %r6208; + +$L__BB0_707: + add.s32 %r1776, %r6196, 2; + setp.ge.u32 %p834, %r1776, %r5; + @%p834 bra $L__BB0_716; + + setp.ge.u32 %p835, %r1738, %r6; + @%p835 bra $L__BB0_710; + + add.s32 %r4067, %r1739, %r1776; + mul.wide.u32 %rd328, %r4067, 4; + add.s64 %rd329, %rd2, %rd328; + ld.global.u32 %r4068, [%rd329]; + abs.s32 %r4069, %r4068; + setp.gt.u32 %p836, %r4069, 4; + and.b32 %r4070, %r4069, 1; + setp.eq.b32 %p837, %r4070, 1; + and.pred %p838, %p836, %p837; + selp.b32 %r4071, 256, 0, %p838; + or.b32 %r6208, %r4071, %r6208; + +$L__BB0_710: + setp.ge.u32 %p839, %r1740, %r6; + @%p839 bra $L__BB0_712; + + add.s32 %r4072, %r1741, %r1776; + mul.wide.u32 %rd330, %r4072, 4; + add.s64 %rd331, %rd2, %rd330; + ld.global.u32 %r4073, [%rd331]; + abs.s32 %r4074, %r4073; + setp.gt.u32 %p840, %r4074, 4; + and.b32 %r4075, %r4074, 1; + setp.eq.b32 %p841, %r4075, 1; + and.pred %p842, %p840, %p841; + selp.b32 %r4076, 512, 0, %p842; + or.b32 %r6208, %r4076, %r6208; + +$L__BB0_712: + setp.ge.u32 %p843, %r1742, %r6; + @%p843 bra $L__BB0_714; + + add.s32 %r4077, %r1743, %r1776; + mul.wide.u32 %rd332, %r4077, 4; + add.s64 %rd333, %rd2, %rd332; + ld.global.u32 %r4078, [%rd333]; + abs.s32 %r4079, %r4078; + setp.gt.u32 %p844, %r4079, 4; + and.b32 %r4080, %r4079, 1; + setp.eq.b32 %p845, %r4080, 1; + and.pred %p846, %p844, %p845; + selp.b32 %r4081, 1024, 0, %p846; + or.b32 %r6208, %r4081, %r6208; + +$L__BB0_714: + setp.ge.u32 %p847, %r1744, %r6; + @%p847 bra $L__BB0_716; + + add.s32 %r4082, %r1745, %r1776; + mul.wide.u32 %rd334, %r4082, 4; + add.s64 %rd335, %rd2, %rd334; + ld.global.u32 %r4083, [%rd335]; + abs.s32 %r4084, %r4083; + setp.gt.u32 %p848, %r4084, 4; + and.b32 %r4085, %r4084, 1; + setp.eq.b32 %p849, %r4085, 1; + and.pred %p850, %p848, %p849; + selp.b32 %r4086, 2048, 0, %p850; + or.b32 %r6208, %r4086, %r6208; + +$L__BB0_716: + add.s32 %r1785, %r6196, 3; + setp.ge.u32 %p851, %r1785, %r5; + @%p851 bra $L__BB0_725; + + setp.ge.u32 %p852, %r1738, %r6; + @%p852 bra $L__BB0_719; + + add.s32 %r4087, %r1739, %r1785; + mul.wide.u32 %rd336, %r4087, 4; + add.s64 %rd337, %rd2, %rd336; + ld.global.u32 %r4088, [%rd337]; + abs.s32 %r4089, %r4088; + setp.gt.u32 %p853, %r4089, 4; + and.b32 %r4090, %r4089, 1; + setp.eq.b32 %p854, %r4090, 1; + and.pred %p855, %p853, %p854; + selp.b32 %r4091, 4096, 0, %p855; + or.b32 %r6208, %r4091, %r6208; + +$L__BB0_719: + setp.ge.u32 %p856, %r1740, %r6; + @%p856 bra $L__BB0_721; + + add.s32 %r4092, %r1741, %r1785; + mul.wide.u32 %rd338, %r4092, 4; + add.s64 %rd339, %rd2, %rd338; + ld.global.u32 %r4093, [%rd339]; + abs.s32 %r4094, %r4093; + setp.gt.u32 %p857, %r4094, 4; + and.b32 %r4095, %r4094, 1; + setp.eq.b32 %p858, %r4095, 1; + and.pred %p859, %p857, %p858; + selp.b32 %r4096, 8192, 0, %p859; + or.b32 %r6208, %r4096, %r6208; + +$L__BB0_721: + setp.ge.u32 %p860, %r1742, %r6; + @%p860 bra $L__BB0_723; + + add.s32 %r4097, %r1743, %r1785; + mul.wide.u32 %rd340, %r4097, 4; + add.s64 %rd341, %rd2, %rd340; + ld.global.u32 %r4098, [%rd341]; + abs.s32 %r4099, %r4098; + setp.gt.u32 %p861, %r4099, 4; + and.b32 %r4100, %r4099, 1; + setp.eq.b32 %p862, %r4100, 1; + and.pred %p863, %p861, %p862; + selp.b32 %r4101, 16384, 0, %p863; + or.b32 %r6208, %r4101, %r6208; + +$L__BB0_723: + setp.ge.u32 %p864, %r1744, %r6; + @%p864 bra $L__BB0_725; + + add.s32 %r4102, %r1745, %r1785; + mul.wide.u32 %rd342, %r4102, 4; + add.s64 %rd343, %rd2, %rd342; + ld.global.u32 %r4103, [%rd343]; + abs.s32 %r4104, %r4103; + setp.gt.u32 %p865, %r4104, 4; + and.b32 %r4105, %r4104, 1; + setp.eq.b32 %p866, %r4105, 1; + and.pred %p867, %p865, %p866; + selp.b32 %r4106, 32768, 0, %p867; + or.b32 %r6208, %r4106, %r6208; + +$L__BB0_725: + add.s32 %r4108, %r6196, 4; + setp.ge.u32 %p868, %r4108, %r5; + mov.u32 %r6224, 0; + @%p868 bra $L__BB0_734; + + setp.ge.u32 %p869, %r1738, %r6; + mov.u32 %r6224, 0; + @%p869 bra $L__BB0_728; + + add.s32 %r4110, %r1739, %r6196; + add.s32 %r4111, %r4110, 4; + mul.wide.u32 %rd344, %r4111, 4; + add.s64 %rd345, %rd2, %rd344; + ld.global.u32 %r4112, [%rd345]; + abs.s32 %r4113, %r4112; + setp.gt.u32 %p870, %r4113, 4; + and.b32 %r4114, %r4113, 1; + setp.eq.b32 %p871, %r4114, 1; + and.pred %p872, %p870, %p871; + selp.u32 %r6224, 1, 0, %p872; + +$L__BB0_728: + setp.ge.u32 %p873, %r1740, %r6; + @%p873 bra $L__BB0_730; + + add.s32 %r4115, %r1741, %r6196; + add.s32 %r4116, %r4115, 4; + mul.wide.u32 %rd346, %r4116, 4; + add.s64 %rd347, %rd2, %rd346; + ld.global.u32 %r4117, [%rd347]; + abs.s32 %r4118, %r4117; + setp.gt.u32 %p874, %r4118, 4; + and.b32 %r4119, %r4118, 1; + setp.eq.b32 %p875, %r4119, 1; + and.pred %p876, %p874, %p875; + selp.b32 %r4120, 2, 0, %p876; + or.b32 %r6224, %r4120, %r6224; + +$L__BB0_730: + setp.ge.u32 %p877, %r1742, %r6; + @%p877 bra $L__BB0_732; + + add.s32 %r4121, %r1743, %r6196; + add.s32 %r4122, %r4121, 4; + mul.wide.u32 %rd348, %r4122, 4; + add.s64 %rd349, %rd2, %rd348; + ld.global.u32 %r4123, [%rd349]; + abs.s32 %r4124, %r4123; + setp.gt.u32 %p878, %r4124, 4; + and.b32 %r4125, %r4124, 1; + setp.eq.b32 %p879, %r4125, 1; + and.pred %p880, %p878, %p879; + selp.b32 %r4126, 4, 0, %p880; + or.b32 %r6224, %r4126, %r6224; + +$L__BB0_732: + setp.ge.u32 %p881, %r1744, %r6; + @%p881 bra $L__BB0_734; + + add.s32 %r4127, %r1745, %r6196; + add.s32 %r4128, %r4127, 4; + mul.wide.u32 %rd350, %r4128, 4; + add.s64 %rd351, %rd2, %rd350; + ld.global.u32 %r4129, [%rd351]; + abs.s32 %r4130, %r4129; + setp.gt.u32 %p882, %r4130, 4; + and.b32 %r4131, %r4130, 1; + setp.eq.b32 %p883, %r4131, 1; + and.pred %p884, %p882, %p883; + selp.b32 %r4132, 8, 0, %p884; + or.b32 %r6224, %r4132, %r6224; + +$L__BB0_734: + add.s32 %r1802, %r6196, 5; + setp.ge.u32 %p885, %r1802, %r5; + @%p885 bra $L__BB0_743; + + setp.ge.u32 %p886, %r1738, %r6; + @%p886 bra $L__BB0_737; + + add.s32 %r4133, %r1739, %r1802; + mul.wide.u32 %rd352, %r4133, 4; + add.s64 %rd353, %rd2, %rd352; + ld.global.u32 %r4134, [%rd353]; + abs.s32 %r4135, %r4134; + setp.gt.u32 %p887, %r4135, 4; + and.b32 %r4136, %r4135, 1; + setp.eq.b32 %p888, %r4136, 1; + and.pred %p889, %p887, %p888; + selp.b32 %r4137, 16, 0, %p889; + or.b32 %r6224, %r4137, %r6224; + +$L__BB0_737: + setp.ge.u32 %p890, %r1740, %r6; + @%p890 bra $L__BB0_739; + + add.s32 %r4138, %r1741, %r1802; + mul.wide.u32 %rd354, %r4138, 4; + add.s64 %rd355, %rd2, %rd354; + ld.global.u32 %r4139, [%rd355]; + abs.s32 %r4140, %r4139; + setp.gt.u32 %p891, %r4140, 4; + and.b32 %r4141, %r4140, 1; + setp.eq.b32 %p892, %r4141, 1; + and.pred %p893, %p891, %p892; + selp.b32 %r4142, 32, 0, %p893; + or.b32 %r6224, %r4142, %r6224; + +$L__BB0_739: + setp.ge.u32 %p894, %r1742, %r6; + @%p894 bra $L__BB0_741; + + add.s32 %r4143, %r1743, %r1802; + mul.wide.u32 %rd356, %r4143, 4; + add.s64 %rd357, %rd2, %rd356; + ld.global.u32 %r4144, [%rd357]; + abs.s32 %r4145, %r4144; + setp.gt.u32 %p895, %r4145, 4; + and.b32 %r4146, %r4145, 1; + setp.eq.b32 %p896, %r4146, 1; + and.pred %p897, %p895, %p896; + selp.b32 %r4147, 64, 0, %p897; + or.b32 %r6224, %r4147, %r6224; + +$L__BB0_741: + setp.ge.u32 %p898, %r1744, %r6; + @%p898 bra $L__BB0_743; + + add.s32 %r4148, %r1745, %r1802; + mul.wide.u32 %rd358, %r4148, 4; + add.s64 %rd359, %rd2, %rd358; + ld.global.u32 %r4149, [%rd359]; + abs.s32 %r4150, %r4149; + setp.gt.u32 %p899, %r4150, 4; + and.b32 %r4151, %r4150, 1; + setp.eq.b32 %p900, %r4151, 1; + and.pred %p901, %p899, %p900; + selp.b32 %r4152, 128, 0, %p901; + or.b32 %r6224, %r4152, %r6224; + +$L__BB0_743: + add.s32 %r1811, %r6196, 6; + setp.ge.u32 %p902, %r1811, %r5; + @%p902 bra $L__BB0_752; + + setp.ge.u32 %p903, %r1738, %r6; + @%p903 bra $L__BB0_746; + + add.s32 %r4153, %r1739, %r1811; + mul.wide.u32 %rd360, %r4153, 4; + add.s64 %rd361, %rd2, %rd360; + ld.global.u32 %r4154, [%rd361]; + abs.s32 %r4155, %r4154; + setp.gt.u32 %p904, %r4155, 4; + and.b32 %r4156, %r4155, 1; + setp.eq.b32 %p905, %r4156, 1; + and.pred %p906, %p904, %p905; + selp.b32 %r4157, 256, 0, %p906; + or.b32 %r6224, %r4157, %r6224; + +$L__BB0_746: + setp.ge.u32 %p907, %r1740, %r6; + @%p907 bra $L__BB0_748; + + add.s32 %r4158, %r1741, %r1811; + mul.wide.u32 %rd362, %r4158, 4; + add.s64 %rd363, %rd2, %rd362; + ld.global.u32 %r4159, [%rd363]; + abs.s32 %r4160, %r4159; + setp.gt.u32 %p908, %r4160, 4; + and.b32 %r4161, %r4160, 1; + setp.eq.b32 %p909, %r4161, 1; + and.pred %p910, %p908, %p909; + selp.b32 %r4162, 512, 0, %p910; + or.b32 %r6224, %r4162, %r6224; + +$L__BB0_748: + setp.ge.u32 %p911, %r1742, %r6; + @%p911 bra $L__BB0_750; + + add.s32 %r4163, %r1743, %r1811; + mul.wide.u32 %rd364, %r4163, 4; + add.s64 %rd365, %rd2, %rd364; + ld.global.u32 %r4164, [%rd365]; + abs.s32 %r4165, %r4164; + setp.gt.u32 %p912, %r4165, 4; + and.b32 %r4166, %r4165, 1; + setp.eq.b32 %p913, %r4166, 1; + and.pred %p914, %p912, %p913; + selp.b32 %r4167, 1024, 0, %p914; + or.b32 %r6224, %r4167, %r6224; + +$L__BB0_750: + setp.ge.u32 %p915, %r1744, %r6; + @%p915 bra $L__BB0_752; + + add.s32 %r4168, %r1745, %r1811; + mul.wide.u32 %rd366, %r4168, 4; + add.s64 %rd367, %rd2, %rd366; + ld.global.u32 %r4169, [%rd367]; + abs.s32 %r4170, %r4169; + setp.gt.u32 %p916, %r4170, 4; + and.b32 %r4171, %r4170, 1; + setp.eq.b32 %p917, %r4171, 1; + and.pred %p918, %p916, %p917; + selp.b32 %r4172, 2048, 0, %p918; + or.b32 %r6224, %r4172, %r6224; + +$L__BB0_752: + add.s32 %r1820, %r6196, 7; + setp.ge.u32 %p919, %r1820, %r5; + @%p919 bra $L__BB0_761; + + setp.ge.u32 %p920, %r1738, %r6; + @%p920 bra $L__BB0_755; + + add.s32 %r4173, %r1739, %r1820; + mul.wide.u32 %rd368, %r4173, 4; + add.s64 %rd369, %rd2, %rd368; + ld.global.u32 %r4174, [%rd369]; + abs.s32 %r4175, %r4174; + setp.gt.u32 %p921, %r4175, 4; + and.b32 %r4176, %r4175, 1; + setp.eq.b32 %p922, %r4176, 1; + and.pred %p923, %p921, %p922; + selp.b32 %r4177, 4096, 0, %p923; + or.b32 %r6224, %r4177, %r6224; + +$L__BB0_755: + setp.ge.u32 %p924, %r1740, %r6; + @%p924 bra $L__BB0_757; + + add.s32 %r4178, %r1741, %r1820; + mul.wide.u32 %rd370, %r4178, 4; + add.s64 %rd371, %rd2, %rd370; + ld.global.u32 %r4179, [%rd371]; + abs.s32 %r4180, %r4179; + setp.gt.u32 %p925, %r4180, 4; + and.b32 %r4181, %r4180, 1; + setp.eq.b32 %p926, %r4181, 1; + and.pred %p927, %p925, %p926; + selp.b32 %r4182, 8192, 0, %p927; + or.b32 %r6224, %r4182, %r6224; + +$L__BB0_757: + setp.ge.u32 %p928, %r1742, %r6; + @%p928 bra $L__BB0_759; + + add.s32 %r4183, %r1743, %r1820; + mul.wide.u32 %rd372, %r4183, 4; + add.s64 %rd373, %rd2, %rd372; + ld.global.u32 %r4184, [%rd373]; + abs.s32 %r4185, %r4184; + setp.gt.u32 %p929, %r4185, 4; + and.b32 %r4186, %r4185, 1; + setp.eq.b32 %p930, %r4186, 1; + and.pred %p931, %p929, %p930; + selp.b32 %r4187, 16384, 0, %p931; + or.b32 %r6224, %r4187, %r6224; + +$L__BB0_759: + setp.ge.u32 %p932, %r1744, %r6; + @%p932 bra $L__BB0_761; + + add.s32 %r4188, %r1745, %r1820; + mul.wide.u32 %rd374, %r4188, 4; + add.s64 %rd375, %rd2, %rd374; + ld.global.u32 %r4189, [%rd375]; + abs.s32 %r4190, %r4189; + setp.gt.u32 %p933, %r4190, 4; + and.b32 %r4191, %r4190, 1; + setp.eq.b32 %p934, %r4191, 1; + and.pred %p935, %p933, %p934; + selp.b32 %r4192, 32768, 0, %p935; + or.b32 %r6224, %r4192, %r6224; + +$L__BB0_761: + mov.b32 %r1829, {%rs237, %rs238}; + add.s32 %r4194, %r1749, %r6196; + mul.wide.u32 %rd376, %r4194, 4; + add.s64 %rd19, %rd2, %rd376; + add.s32 %r4195, %r1752, %r6196; + mul.wide.u32 %rd377, %r4195, 4; + add.s64 %rd20, %rd2, %rd377; + add.s32 %r4196, %r1751, %r6196; + mul.wide.u32 %rd378, %r4196, 4; + add.s64 %rd21, %rd2, %rd378; + add.s32 %r4197, %r1750, %r6196; + mul.wide.u32 %rd379, %r4197, 4; + add.s64 %rd22, %rd2, %rd379; + mov.u32 %r6240, 0; + @%p800 bra $L__BB0_770; + + setp.le.u32 %p937, %r6, %r6192; + mov.u32 %r6240, 0; + @%p937 bra $L__BB0_764; + + ld.global.u32 %r4199, [%rd19]; + abs.s32 %r4200, %r4199; + setp.gt.u32 %p938, %r4200, 4; + and.b32 %r4201, %r4200, 1; + setp.eq.b32 %p939, %r4201, 1; + and.pred %p940, %p938, %p939; + selp.u32 %r6240, 1, 0, %p940; + +$L__BB0_764: + setp.ge.u32 %p941, %r1746, %r6; + @%p941 bra $L__BB0_766; + + ld.global.u32 %r4202, [%rd20]; + abs.s32 %r4203, %r4202; + setp.gt.u32 %p942, %r4203, 4; + and.b32 %r4204, %r4203, 1; + setp.eq.b32 %p943, %r4204, 1; + and.pred %p944, %p942, %p943; + selp.b32 %r4205, 2, 0, %p944; + or.b32 %r6240, %r4205, %r6240; + +$L__BB0_766: + setp.ge.u32 %p945, %r1747, %r6; + @%p945 bra $L__BB0_768; + + ld.global.u32 %r4206, [%rd21]; + abs.s32 %r4207, %r4206; + setp.gt.u32 %p946, %r4207, 4; + and.b32 %r4208, %r4207, 1; + setp.eq.b32 %p947, %r4208, 1; + and.pred %p948, %p946, %p947; + selp.b32 %r4209, 4, 0, %p948; + or.b32 %r6240, %r4209, %r6240; + +$L__BB0_768: + setp.ge.u32 %p949, %r1748, %r6; + @%p949 bra $L__BB0_770; + + ld.global.u32 %r4210, [%rd22]; + abs.s32 %r4211, %r4210; + setp.gt.u32 %p950, %r4211, 4; + and.b32 %r4212, %r4211, 1; + setp.eq.b32 %p951, %r4212, 1; + and.pred %p952, %p950, %p951; + selp.b32 %r4213, 8, 0, %p952; + or.b32 %r6240, %r4213, %r6240; + +$L__BB0_770: + add.s32 %r4214, %r1749, %r1767; + mul.wide.u32 %rd380, %r4214, 4; + add.s64 %rd23, %rd2, %rd380; + add.s32 %r4215, %r1752, %r1767; + mul.wide.u32 %rd381, %r4215, 4; + add.s64 %rd24, %rd2, %rd381; + add.s32 %r4216, %r1751, %r1767; + mul.wide.u32 %rd382, %r4216, 4; + add.s64 %rd25, %rd2, %rd382; + add.s32 %r4217, %r1750, %r1767; + mul.wide.u32 %rd383, %r4217, 4; + add.s64 %rd26, %rd2, %rd383; + shl.b32 %r4218, %r6224, 16; + or.b32 %r1838, %r4218, %r6208; + @%p817 bra $L__BB0_779; + + setp.le.u32 %p954, %r6, %r6192; + @%p954 bra $L__BB0_773; + + ld.global.u32 %r4219, [%rd23]; + abs.s32 %r4220, %r4219; + setp.gt.u32 %p955, %r4220, 4; + and.b32 %r4221, %r4220, 1; + setp.eq.b32 %p956, %r4221, 1; + and.pred %p957, %p955, %p956; + selp.b32 %r4222, 16, 0, %p957; + or.b32 %r6240, %r4222, %r6240; + +$L__BB0_773: + setp.ge.u32 %p958, %r1746, %r6; + @%p958 bra $L__BB0_775; + + ld.global.u32 %r4223, [%rd24]; + abs.s32 %r4224, %r4223; + setp.gt.u32 %p959, %r4224, 4; + and.b32 %r4225, %r4224, 1; + setp.eq.b32 %p960, %r4225, 1; + and.pred %p961, %p959, %p960; + selp.b32 %r4226, 32, 0, %p961; + or.b32 %r6240, %r4226, %r6240; + +$L__BB0_775: + setp.ge.u32 %p962, %r1747, %r6; + @%p962 bra $L__BB0_777; + + ld.global.u32 %r4227, [%rd25]; + abs.s32 %r4228, %r4227; + setp.gt.u32 %p963, %r4228, 4; + and.b32 %r4229, %r4228, 1; + setp.eq.b32 %p964, %r4229, 1; + and.pred %p965, %p963, %p964; + selp.b32 %r4230, 64, 0, %p965; + or.b32 %r6240, %r4230, %r6240; + +$L__BB0_777: + setp.ge.u32 %p966, %r1748, %r6; + @%p966 bra $L__BB0_779; + + ld.global.u32 %r4231, [%rd26]; + abs.s32 %r4232, %r4231; + setp.gt.u32 %p967, %r4232, 4; + and.b32 %r4233, %r4232, 1; + setp.eq.b32 %p968, %r4233, 1; + and.pred %p969, %p967, %p968; + selp.b32 %r4234, 128, 0, %p969; + or.b32 %r6240, %r4234, %r6240; + +$L__BB0_779: + add.s32 %r4235, %r1749, %r1776; + mul.wide.u32 %rd384, %r4235, 4; + add.s64 %rd27, %rd2, %rd384; + add.s32 %r4236, %r1752, %r1776; + mul.wide.u32 %rd385, %r4236, 4; + add.s64 %rd28, %rd2, %rd385; + add.s32 %r4237, %r1751, %r1776; + mul.wide.u32 %rd386, %r4237, 4; + add.s64 %rd29, %rd2, %rd386; + add.s32 %r4238, %r1750, %r1776; + mul.wide.u32 %rd387, %r4238, 4; + add.s64 %rd30, %rd2, %rd387; + @%p834 bra $L__BB0_788; + + setp.le.u32 %p971, %r6, %r6192; + @%p971 bra $L__BB0_782; + + ld.global.u32 %r4239, [%rd27]; + abs.s32 %r4240, %r4239; + setp.gt.u32 %p972, %r4240, 4; + and.b32 %r4241, %r4240, 1; + setp.eq.b32 %p973, %r4241, 1; + and.pred %p974, %p972, %p973; + selp.b32 %r4242, 256, 0, %p974; + or.b32 %r6240, %r4242, %r6240; + +$L__BB0_782: + setp.ge.u32 %p975, %r1746, %r6; + @%p975 bra $L__BB0_784; + + ld.global.u32 %r4243, [%rd28]; + abs.s32 %r4244, %r4243; + setp.gt.u32 %p976, %r4244, 4; + and.b32 %r4245, %r4244, 1; + setp.eq.b32 %p977, %r4245, 1; + and.pred %p978, %p976, %p977; + selp.b32 %r4246, 512, 0, %p978; + or.b32 %r6240, %r4246, %r6240; + +$L__BB0_784: + setp.ge.u32 %p979, %r1747, %r6; + @%p979 bra $L__BB0_786; + + ld.global.u32 %r4247, [%rd29]; + abs.s32 %r4248, %r4247; + setp.gt.u32 %p980, %r4248, 4; + and.b32 %r4249, %r4248, 1; + setp.eq.b32 %p981, %r4249, 1; + and.pred %p982, %p980, %p981; + selp.b32 %r4250, 1024, 0, %p982; + or.b32 %r6240, %r4250, %r6240; + +$L__BB0_786: + setp.ge.u32 %p983, %r1748, %r6; + @%p983 bra $L__BB0_788; + + ld.global.u32 %r4251, [%rd30]; + abs.s32 %r4252, %r4251; + setp.gt.u32 %p984, %r4252, 4; + and.b32 %r4253, %r4252, 1; + setp.eq.b32 %p985, %r4253, 1; + and.pred %p986, %p984, %p985; + selp.b32 %r4254, 2048, 0, %p986; + or.b32 %r6240, %r4254, %r6240; + +$L__BB0_788: + add.s32 %r4255, %r1749, %r1785; + mul.wide.u32 %rd388, %r4255, 4; + add.s64 %rd31, %rd2, %rd388; + add.s32 %r4256, %r1752, %r1785; + mul.wide.u32 %rd389, %r4256, 4; + add.s64 %rd32, %rd2, %rd389; + add.s32 %r4257, %r1751, %r1785; + mul.wide.u32 %rd390, %r4257, 4; + add.s64 %rd33, %rd2, %rd390; + add.s32 %r4258, %r1750, %r1785; + mul.wide.u32 %rd391, %r4258, 4; + add.s64 %rd34, %rd2, %rd391; + @%p851 bra $L__BB0_797; + + setp.le.u32 %p988, %r6, %r6192; + @%p988 bra $L__BB0_791; + + ld.global.u32 %r4259, [%rd31]; + abs.s32 %r4260, %r4259; + setp.gt.u32 %p989, %r4260, 4; + and.b32 %r4261, %r4260, 1; + setp.eq.b32 %p990, %r4261, 1; + and.pred %p991, %p989, %p990; + selp.b32 %r4262, 4096, 0, %p991; + or.b32 %r6240, %r4262, %r6240; + +$L__BB0_791: + setp.ge.u32 %p992, %r1746, %r6; + @%p992 bra $L__BB0_793; + + ld.global.u32 %r4263, [%rd32]; + abs.s32 %r4264, %r4263; + setp.gt.u32 %p993, %r4264, 4; + and.b32 %r4265, %r4264, 1; + setp.eq.b32 %p994, %r4265, 1; + and.pred %p995, %p993, %p994; + selp.b32 %r4266, 8192, 0, %p995; + or.b32 %r6240, %r4266, %r6240; + +$L__BB0_793: + setp.ge.u32 %p996, %r1747, %r6; + @%p996 bra $L__BB0_795; + + ld.global.u32 %r4267, [%rd33]; + abs.s32 %r4268, %r4267; + setp.gt.u32 %p997, %r4268, 4; + and.b32 %r4269, %r4268, 1; + setp.eq.b32 %p998, %r4269, 1; + and.pred %p999, %p997, %p998; + selp.b32 %r4270, 16384, 0, %p999; + or.b32 %r6240, %r4270, %r6240; + +$L__BB0_795: + setp.ge.u32 %p1000, %r1748, %r6; + @%p1000 bra $L__BB0_797; + + ld.global.u32 %r4271, [%rd34]; + abs.s32 %r4272, %r4271; + setp.gt.u32 %p1001, %r4272, 4; + and.b32 %r4273, %r4272, 1; + setp.eq.b32 %p1002, %r4273, 1; + and.pred %p1003, %p1001, %p1002; + selp.b32 %r4274, 32768, 0, %p1003; + or.b32 %r6240, %r4274, %r6240; + +$L__BB0_797: + mov.u32 %r6256, 0; + @%p868 bra $L__BB0_806; + + setp.le.u32 %p1005, %r6, %r6192; + mov.u32 %r6256, 0; + @%p1005 bra $L__BB0_800; + + add.s32 %r4279, %r4194, 4; + mul.wide.u32 %rd392, %r4279, 4; + add.s64 %rd393, %rd2, %rd392; + ld.global.u32 %r4280, [%rd393]; + abs.s32 %r4281, %r4280; + setp.gt.u32 %p1006, %r4281, 4; + and.b32 %r4282, %r4281, 1; + setp.eq.b32 %p1007, %r4282, 1; + and.pred %p1008, %p1006, %p1007; + selp.u32 %r6256, 1, 0, %p1008; + +$L__BB0_800: + setp.ge.u32 %p1009, %r1746, %r6; + @%p1009 bra $L__BB0_802; + + add.s32 %r4284, %r4195, 4; + mul.wide.u32 %rd394, %r4284, 4; + add.s64 %rd395, %rd2, %rd394; + ld.global.u32 %r4285, [%rd395]; + abs.s32 %r4286, %r4285; + setp.gt.u32 %p1010, %r4286, 4; + and.b32 %r4287, %r4286, 1; + setp.eq.b32 %p1011, %r4287, 1; + and.pred %p1012, %p1010, %p1011; + selp.b32 %r4288, 2, 0, %p1012; + or.b32 %r6256, %r4288, %r6256; + +$L__BB0_802: + setp.ge.u32 %p1013, %r1747, %r6; + @%p1013 bra $L__BB0_804; + + add.s32 %r4290, %r4196, 4; + mul.wide.u32 %rd396, %r4290, 4; + add.s64 %rd397, %rd2, %rd396; + ld.global.u32 %r4291, [%rd397]; + abs.s32 %r4292, %r4291; + setp.gt.u32 %p1014, %r4292, 4; + and.b32 %r4293, %r4292, 1; + setp.eq.b32 %p1015, %r4293, 1; + and.pred %p1016, %p1014, %p1015; + selp.b32 %r4294, 4, 0, %p1016; + or.b32 %r6256, %r4294, %r6256; + +$L__BB0_804: + setp.ge.u32 %p1017, %r1748, %r6; + @%p1017 bra $L__BB0_806; + + add.s32 %r4296, %r4197, 4; + mul.wide.u32 %rd398, %r4296, 4; + add.s64 %rd399, %rd2, %rd398; + ld.global.u32 %r4297, [%rd399]; + abs.s32 %r4298, %r4297; + setp.gt.u32 %p1018, %r4298, 4; + and.b32 %r4299, %r4298, 1; + setp.eq.b32 %p1019, %r4299, 1; + and.pred %p1020, %p1018, %p1019; + selp.b32 %r4300, 8, 0, %p1020; + or.b32 %r6256, %r4300, %r6256; + +$L__BB0_806: + @%p885 bra $L__BB0_815; + + setp.le.u32 %p1022, %r6, %r6192; + @%p1022 bra $L__BB0_809; + + add.s32 %r4301, %r1749, %r1802; + mul.wide.u32 %rd400, %r4301, 4; + add.s64 %rd401, %rd2, %rd400; + ld.global.u32 %r4302, [%rd401]; + abs.s32 %r4303, %r4302; + setp.gt.u32 %p1023, %r4303, 4; + and.b32 %r4304, %r4303, 1; + setp.eq.b32 %p1024, %r4304, 1; + and.pred %p1025, %p1023, %p1024; + selp.b32 %r4305, 16, 0, %p1025; + or.b32 %r6256, %r4305, %r6256; + +$L__BB0_809: + setp.ge.u32 %p1026, %r1746, %r6; + @%p1026 bra $L__BB0_811; + + add.s32 %r4306, %r1752, %r1802; + mul.wide.u32 %rd402, %r4306, 4; + add.s64 %rd403, %rd2, %rd402; + ld.global.u32 %r4307, [%rd403]; + abs.s32 %r4308, %r4307; + setp.gt.u32 %p1027, %r4308, 4; + and.b32 %r4309, %r4308, 1; + setp.eq.b32 %p1028, %r4309, 1; + and.pred %p1029, %p1027, %p1028; + selp.b32 %r4310, 32, 0, %p1029; + or.b32 %r6256, %r4310, %r6256; + +$L__BB0_811: + setp.ge.u32 %p1030, %r1747, %r6; + @%p1030 bra $L__BB0_813; + + add.s32 %r4311, %r1751, %r1802; + mul.wide.u32 %rd404, %r4311, 4; + add.s64 %rd405, %rd2, %rd404; + ld.global.u32 %r4312, [%rd405]; + abs.s32 %r4313, %r4312; + setp.gt.u32 %p1031, %r4313, 4; + and.b32 %r4314, %r4313, 1; + setp.eq.b32 %p1032, %r4314, 1; + and.pred %p1033, %p1031, %p1032; + selp.b32 %r4315, 64, 0, %p1033; + or.b32 %r6256, %r4315, %r6256; + +$L__BB0_813: + setp.ge.u32 %p1034, %r1748, %r6; + @%p1034 bra $L__BB0_815; + + add.s32 %r4316, %r1750, %r1802; + mul.wide.u32 %rd406, %r4316, 4; + add.s64 %rd407, %rd2, %rd406; + ld.global.u32 %r4317, [%rd407]; + abs.s32 %r4318, %r4317; + setp.gt.u32 %p1035, %r4318, 4; + and.b32 %r4319, %r4318, 1; + setp.eq.b32 %p1036, %r4319, 1; + and.pred %p1037, %p1035, %p1036; + selp.b32 %r4320, 128, 0, %p1037; + or.b32 %r6256, %r4320, %r6256; + +$L__BB0_815: + @%p902 bra $L__BB0_824; + + setp.le.u32 %p1039, %r6, %r6192; + @%p1039 bra $L__BB0_818; + + add.s32 %r4321, %r1749, %r1811; + mul.wide.u32 %rd408, %r4321, 4; + add.s64 %rd409, %rd2, %rd408; + ld.global.u32 %r4322, [%rd409]; + abs.s32 %r4323, %r4322; + setp.gt.u32 %p1040, %r4323, 4; + and.b32 %r4324, %r4323, 1; + setp.eq.b32 %p1041, %r4324, 1; + and.pred %p1042, %p1040, %p1041; + selp.b32 %r4325, 256, 0, %p1042; + or.b32 %r6256, %r4325, %r6256; + +$L__BB0_818: + setp.ge.u32 %p1043, %r1746, %r6; + @%p1043 bra $L__BB0_820; + + add.s32 %r4326, %r1752, %r1811; + mul.wide.u32 %rd410, %r4326, 4; + add.s64 %rd411, %rd2, %rd410; + ld.global.u32 %r4327, [%rd411]; + abs.s32 %r4328, %r4327; + setp.gt.u32 %p1044, %r4328, 4; + and.b32 %r4329, %r4328, 1; + setp.eq.b32 %p1045, %r4329, 1; + and.pred %p1046, %p1044, %p1045; + selp.b32 %r4330, 512, 0, %p1046; + or.b32 %r6256, %r4330, %r6256; + +$L__BB0_820: + setp.ge.u32 %p1047, %r1747, %r6; + @%p1047 bra $L__BB0_822; + + add.s32 %r4331, %r1751, %r1811; + mul.wide.u32 %rd412, %r4331, 4; + add.s64 %rd413, %rd2, %rd412; + ld.global.u32 %r4332, [%rd413]; + abs.s32 %r4333, %r4332; + setp.gt.u32 %p1048, %r4333, 4; + and.b32 %r4334, %r4333, 1; + setp.eq.b32 %p1049, %r4334, 1; + and.pred %p1050, %p1048, %p1049; + selp.b32 %r4335, 1024, 0, %p1050; + or.b32 %r6256, %r4335, %r6256; + +$L__BB0_822: + setp.ge.u32 %p1051, %r1748, %r6; + @%p1051 bra $L__BB0_824; + + add.s32 %r4336, %r1750, %r1811; + mul.wide.u32 %rd414, %r4336, 4; + add.s64 %rd415, %rd2, %rd414; + ld.global.u32 %r4337, [%rd415]; + abs.s32 %r4338, %r4337; + setp.gt.u32 %p1052, %r4338, 4; + and.b32 %r4339, %r4338, 1; + setp.eq.b32 %p1053, %r4339, 1; + and.pred %p1054, %p1052, %p1053; + selp.b32 %r4340, 2048, 0, %p1054; + or.b32 %r6256, %r4340, %r6256; + +$L__BB0_824: + @%p919 bra $L__BB0_833; + + setp.le.u32 %p1056, %r6, %r6192; + @%p1056 bra $L__BB0_827; + + add.s32 %r4341, %r1749, %r1820; + mul.wide.u32 %rd416, %r4341, 4; + add.s64 %rd417, %rd2, %rd416; + ld.global.u32 %r4342, [%rd417]; + abs.s32 %r4343, %r4342; + setp.gt.u32 %p1057, %r4343, 4; + and.b32 %r4344, %r4343, 1; + setp.eq.b32 %p1058, %r4344, 1; + and.pred %p1059, %p1057, %p1058; + selp.b32 %r4345, 4096, 0, %p1059; + or.b32 %r6256, %r4345, %r6256; + +$L__BB0_827: + setp.ge.u32 %p1060, %r1746, %r6; + @%p1060 bra $L__BB0_829; + + add.s32 %r4346, %r1752, %r1820; + mul.wide.u32 %rd418, %r4346, 4; + add.s64 %rd419, %rd2, %rd418; + ld.global.u32 %r4347, [%rd419]; + abs.s32 %r4348, %r4347; + setp.gt.u32 %p1061, %r4348, 4; + and.b32 %r4349, %r4348, 1; + setp.eq.b32 %p1062, %r4349, 1; + and.pred %p1063, %p1061, %p1062; + selp.b32 %r4350, 8192, 0, %p1063; + or.b32 %r6256, %r4350, %r6256; + +$L__BB0_829: + setp.ge.u32 %p1064, %r1747, %r6; + @%p1064 bra $L__BB0_831; + + add.s32 %r4351, %r1751, %r1820; + mul.wide.u32 %rd420, %r4351, 4; + add.s64 %rd421, %rd2, %rd420; + ld.global.u32 %r4352, [%rd421]; + abs.s32 %r4353, %r4352; + setp.gt.u32 %p1065, %r4353, 4; + and.b32 %r4354, %r4353, 1; + setp.eq.b32 %p1066, %r4354, 1; + and.pred %p1067, %p1065, %p1066; + selp.b32 %r4355, 16384, 0, %p1067; + or.b32 %r6256, %r4355, %r6256; + +$L__BB0_831: + setp.ge.u32 %p1068, %r1748, %r6; + @%p1068 bra $L__BB0_833; + + add.s32 %r4356, %r1750, %r1820; + mul.wide.u32 %rd422, %r4356, 4; + add.s64 %rd423, %rd2, %rd422; + ld.global.u32 %r4357, [%rd423]; + abs.s32 %r4358, %r4357; + setp.gt.u32 %p1069, %r4358, 4; + and.b32 %r4359, %r4358, 1; + setp.eq.b32 %p1070, %r4359, 1; + and.pred %p1071, %p1069, %p1070; + selp.b32 %r4360, 32768, 0, %p1071; + or.b32 %r6256, %r4360, %r6256; + +$L__BB0_833: + sub.s32 %r4363, %r4108, %r5; + shl.b32 %r4364, %r6256, 16; + or.b32 %r1895, %r4364, %r6240; + and.b32 %r4365, %r1829, -2004318072; + shr.u32 %r4366, %r4365, 3; + shl.b32 %r4367, %r1838, 3; + and.b32 %r4368, %r4367, -2004318072; + or.b32 %r1896, %r4368, %r4366; + not.b32 %r4369, %r1895; + setp.gt.s32 %p1072, %r4363, 0; + mov.u32 %r6272, 0; + shl.b32 %r4370, %r4363, 2; + selp.b32 %r4371, %r4370, 0, %p1072; + shr.u32 %r1897, %r1753, %r4371; + and.b32 %r1898, %r1897, %r4369; + @%p800 bra $L__BB0_842; + + setp.le.u32 %p1074, %r6, %r6192; + mov.u32 %r6272, 0; + @%p1074 bra $L__BB0_836; + + ld.global.u32 %r4373, [%rd19]; + abs.s32 %r4374, %r4373; + setp.eq.s32 %p1075, %r4374, 3; + selp.u32 %r6272, 1, 0, %p1075; + +$L__BB0_836: + setp.ge.u32 %p1076, %r1746, %r6; + @%p1076 bra $L__BB0_838; + + ld.global.u32 %r4375, [%rd20]; + abs.s32 %r4376, %r4375; + setp.eq.s32 %p1077, %r4376, 3; + selp.b32 %r4377, 2, 0, %p1077; + or.b32 %r6272, %r4377, %r6272; + +$L__BB0_838: + setp.ge.u32 %p1078, %r1747, %r6; + @%p1078 bra $L__BB0_840; + + ld.global.u32 %r4378, [%rd21]; + abs.s32 %r4379, %r4378; + setp.eq.s32 %p1079, %r4379, 3; + selp.b32 %r4380, 4, 0, %p1079; + or.b32 %r6272, %r4380, %r6272; + +$L__BB0_840: + setp.ge.u32 %p1080, %r1748, %r6; + @%p1080 bra $L__BB0_842; + + ld.global.u32 %r4381, [%rd22]; + abs.s32 %r4382, %r4381; + setp.eq.s32 %p1081, %r4382, 3; + selp.b32 %r4383, 8, 0, %p1081; + or.b32 %r6272, %r4383, %r6272; + +$L__BB0_842: + @%p817 bra $L__BB0_851; + + setp.le.u32 %p1083, %r6, %r6192; + @%p1083 bra $L__BB0_845; + + ld.global.u32 %r4384, [%rd23]; + abs.s32 %r4385, %r4384; + setp.eq.s32 %p1084, %r4385, 3; + selp.b32 %r4386, 16, 0, %p1084; + or.b32 %r6272, %r4386, %r6272; + +$L__BB0_845: + setp.ge.u32 %p1085, %r1746, %r6; + @%p1085 bra $L__BB0_847; + + ld.global.u32 %r4387, [%rd24]; + abs.s32 %r4388, %r4387; + setp.eq.s32 %p1086, %r4388, 3; + selp.b32 %r4389, 32, 0, %p1086; + or.b32 %r6272, %r4389, %r6272; + +$L__BB0_847: + setp.ge.u32 %p1087, %r1747, %r6; + @%p1087 bra $L__BB0_849; + + ld.global.u32 %r4390, [%rd25]; + abs.s32 %r4391, %r4390; + setp.eq.s32 %p1088, %r4391, 3; + selp.b32 %r4392, 64, 0, %p1088; + or.b32 %r6272, %r4392, %r6272; + +$L__BB0_849: + setp.ge.u32 %p1089, %r1748, %r6; + @%p1089 bra $L__BB0_851; + + ld.global.u32 %r4393, [%rd26]; + abs.s32 %r4394, %r4393; + setp.eq.s32 %p1090, %r4394, 3; + selp.b32 %r4395, 128, 0, %p1090; + or.b32 %r6272, %r4395, %r6272; + +$L__BB0_851: + @%p834 bra $L__BB0_860; + + setp.le.u32 %p1092, %r6, %r6192; + @%p1092 bra $L__BB0_854; + + ld.global.u32 %r4396, [%rd27]; + abs.s32 %r4397, %r4396; + setp.eq.s32 %p1093, %r4397, 3; + selp.b32 %r4398, 256, 0, %p1093; + or.b32 %r6272, %r4398, %r6272; + +$L__BB0_854: + setp.ge.u32 %p1094, %r1746, %r6; + @%p1094 bra $L__BB0_856; + + ld.global.u32 %r4399, [%rd28]; + abs.s32 %r4400, %r4399; + setp.eq.s32 %p1095, %r4400, 3; + selp.b32 %r4401, 512, 0, %p1095; + or.b32 %r6272, %r4401, %r6272; + +$L__BB0_856: + setp.ge.u32 %p1096, %r1747, %r6; + @%p1096 bra $L__BB0_858; + + ld.global.u32 %r4402, [%rd29]; + abs.s32 %r4403, %r4402; + setp.eq.s32 %p1097, %r4403, 3; + selp.b32 %r4404, 1024, 0, %p1097; + or.b32 %r6272, %r4404, %r6272; + +$L__BB0_858: + setp.ge.u32 %p1098, %r1748, %r6; + @%p1098 bra $L__BB0_860; + + ld.global.u32 %r4405, [%rd30]; + abs.s32 %r4406, %r4405; + setp.eq.s32 %p1099, %r4406, 3; + selp.b32 %r4407, 2048, 0, %p1099; + or.b32 %r6272, %r4407, %r6272; + +$L__BB0_860: + @%p851 bra $L__BB0_869; + + setp.le.u32 %p1101, %r6, %r6192; + @%p1101 bra $L__BB0_863; + + ld.global.u32 %r4408, [%rd31]; + abs.s32 %r4409, %r4408; + setp.eq.s32 %p1102, %r4409, 3; + selp.b32 %r4410, 4096, 0, %p1102; + or.b32 %r6272, %r4410, %r6272; + +$L__BB0_863: + setp.ge.u32 %p1103, %r1746, %r6; + @%p1103 bra $L__BB0_865; + + ld.global.u32 %r4411, [%rd32]; + abs.s32 %r4412, %r4411; + setp.eq.s32 %p1104, %r4412, 3; + selp.b32 %r4413, 8192, 0, %p1104; + or.b32 %r6272, %r4413, %r6272; + +$L__BB0_865: + setp.ge.u32 %p1105, %r1747, %r6; + @%p1105 bra $L__BB0_867; + + ld.global.u32 %r4414, [%rd33]; + abs.s32 %r4415, %r4414; + setp.eq.s32 %p1106, %r4415, 3; + selp.b32 %r4416, 16384, 0, %p1106; + or.b32 %r6272, %r4416, %r6272; + +$L__BB0_867: + setp.ge.u32 %p1107, %r1748, %r6; + @%p1107 bra $L__BB0_869; + + ld.global.u32 %r4417, [%rd34]; + abs.s32 %r4418, %r4417; + setp.eq.s32 %p1108, %r4418, 3; + selp.b32 %r4419, 32768, 0, %p1108; + or.b32 %r6272, %r4419, %r6272; + +$L__BB0_869: + and.b32 %r4421, %r1895, -286331154; + shr.u32 %r4422, %r4421, 1; + shl.b32 %r4423, %r1895, 1; + and.b32 %r4424, %r4423, -286331154; + or.b32 %r4425, %r1895, %r1896; + or.b32 %r4426, %r4425, %r4424; + or.b32 %r4427, %r4426, %r4422; + and.b32 %r1931, %r6272, %r1897; + shr.u32 %r4428, %r4427, 4; + shl.b32 %r4429, %r4427, 4; + shr.u32 %r4430, %r6197, 12; + or.b32 %r4431, %r4427, %r4430; + or.b32 %r4432, %r4431, %r4429; + or.b32 %r4433, %r4432, %r4428; + and.b32 %r6282, %r1898, %r4433; + setp.eq.s32 %p1109, %r6282, 0; + mov.u32 %r4420, 0; + mov.u32 %r6303, %r4420; + @%p1109 bra $L__BB0_924; + + mov.u32 %r6281, 0; + mov.u32 %r6283, %r6281; + mov.u32 %r6286, %r6300; + +$L__BB0_871: + brev.b32 %r4436, %r6282; + bfind.shiftamt.u32 %r1939, %r4436; + mov.pred %p1632, -1; + mov.u32 %r4437, 1; + shl.b32 %r1940, %r4437, %r1939; + mov.u32 %r4438, -2; + shf.l.wrap.b32 %r4439, %r4438, %r4438, %r1939; + and.b32 %r6282, %r6282, %r4439; + or.b32 %r6281, %r1940, %r6281; + and.b32 %r1943, %r1940, %r1931; + setp.ne.s32 %p1111, %r1943, 0; + selp.u32 %r4440, 1, 0, %p1111; + setp.eq.s32 %p1112, %r6302, 0; + selp.b32 %r4441, 8, 7, %p1112; + shl.b32 %r4442, %r4440, %r6298; + cvt.u16.u32 %rs537, %r4442; + or.b16 %rs747, %rs747, %rs537; + add.s32 %r6298, %r6298, 1; + setp.lt.u32 %p1113, %r6298, %r4441; + mov.pred %p1630, %p1632; + @%p1113 bra $L__BB0_874; + + setp.eq.s32 %p1115, %r6286, -1; + mov.pred %p1630, 0; + mov.u32 %r6300, -1; + @%p1115 bra $L__BB0_874; + + and.b16 %rs539, %rs747, 255; + setp.eq.s16 %p1117, %rs539, 255; + selp.u32 %r6302, 1, 0, %p1117; + add.s32 %r6300, %r6286, 1; + mov.u16 %rs747, 0; + mov.u32 %r6298, 0; + mov.pred %p1630, %p1632; + +$L__BB0_874: + mov.u32 %r6307, 0; + not.pred %p1119, %p1630; + @%p1119 bra $L__BB0_928; + + setp.eq.s32 %p1120, %r1943, 0; + @%p1120 bra $L__BB0_916; + + or.b32 %r6283, %r1940, %r6283; + mov.u32 %r6290, 51; + setp.gt.s32 %p1121, %r1939, 7; + @%p1121 bra $L__BB0_892; + + setp.gt.s32 %p1133, %r1939, 3; + @%p1133 bra $L__BB0_885; + + setp.gt.s32 %p1139, %r1939, 1; + @%p1139 bra $L__BB0_882; + + setp.eq.s32 %p1142, %r1939, 0; + @%p1142 bra $L__BB0_915; + + setp.eq.s32 %p1143, %r1939, 1; + @%p1143 bra $L__BB0_881; + bra.uni $L__BB0_914; + +$L__BB0_881: + mov.u32 %r6290, 118; + bra.uni $L__BB0_915; + +$L__BB0_892: + setp.gt.s32 %p1122, %r1939, 11; + @%p1122 bra $L__BB0_900; + + setp.gt.s32 %p1128, %r1939, 9; + @%p1128 bra $L__BB0_897; + + setp.eq.s32 %p1131, %r1939, 8; + @%p1131 bra $L__BB0_910; + + setp.eq.s32 %p1132, %r1939, 9; + @%p1132 bra $L__BB0_896; + bra.uni $L__BB0_914; + +$L__BB0_896: + mov.u32 %r6290, 30208; + bra.uni $L__BB0_915; + +$L__BB0_885: + setp.gt.s32 %p1134, %r1939, 5; + @%p1134 bra $L__BB0_889; + + setp.eq.s32 %p1137, %r1939, 4; + @%p1137 bra $L__BB0_912; + + setp.eq.s32 %p1138, %r1939, 5; + @%p1138 bra $L__BB0_888; + bra.uni $L__BB0_914; + +$L__BB0_888: + mov.u32 %r6290, 1888; + bra.uni $L__BB0_915; + +$L__BB0_900: + setp.gt.s32 %p1123, %r1939, 13; + @%p1123 bra $L__BB0_904; + + setp.eq.s32 %p1126, %r1939, 12; + @%p1126 bra $L__BB0_908; + + setp.eq.s32 %p1127, %r1939, 13; + @%p1127 bra $L__BB0_903; + bra.uni $L__BB0_914; + +$L__BB0_903: + mov.u32 %r6290, 483328; + bra.uni $L__BB0_915; + +$L__BB0_882: + setp.eq.s32 %p1140, %r1939, 2; + @%p1140 bra $L__BB0_913; + + setp.eq.s32 %p1141, %r1939, 3; + @%p1141 bra $L__BB0_884; + bra.uni $L__BB0_914; + +$L__BB0_884: + mov.u32 %r6290, 200; + bra.uni $L__BB0_915; + +$L__BB0_897: + setp.eq.s32 %p1129, %r1939, 10; + @%p1129 bra $L__BB0_909; + + setp.eq.s32 %p1130, %r1939, 11; + @%p1130 bra $L__BB0_899; + bra.uni $L__BB0_914; + +$L__BB0_899: + mov.u32 %r6290, 51200; + bra.uni $L__BB0_915; + +$L__BB0_889: + setp.eq.s32 %p1135, %r1939, 6; + @%p1135 bra $L__BB0_911; + + setp.eq.s32 %p1136, %r1939, 7; + @%p1136 bra $L__BB0_891; + bra.uni $L__BB0_914; + +$L__BB0_891: + mov.u32 %r6290, 3200; + bra.uni $L__BB0_915; + +$L__BB0_904: + setp.eq.s32 %p1124, %r1939, 14; + @%p1124 bra $L__BB0_907; + + setp.ne.s32 %p1125, %r1939, 15; + @%p1125 bra $L__BB0_914; + + mov.u32 %r6290, 819200; + bra.uni $L__BB0_915; + +$L__BB0_910: + mov.u32 %r6290, 13056; + bra.uni $L__BB0_915; + +$L__BB0_912: + mov.u32 %r6290, 816; + bra.uni $L__BB0_915; + +$L__BB0_908: + mov.u32 %r6290, 208896; + bra.uni $L__BB0_915; + +$L__BB0_913: + mov.u32 %r6290, 236; + bra.uni $L__BB0_915; + +$L__BB0_909: + mov.u32 %r6290, 60416; + bra.uni $L__BB0_915; + +$L__BB0_911: + mov.u32 %r6290, 3776; + bra.uni $L__BB0_915; + +$L__BB0_907: + mov.u32 %r6290, 966656; + bra.uni $L__BB0_915; + +$L__BB0_914: + mov.u32 %r6290, 0; + +$L__BB0_915: + not.b32 %r4463, %r6281; + and.b32 %r4464, %r1898, %r4463; + and.b32 %r4465, %r4464, %r6290; + or.b32 %r6282, %r4465, %r6282; + +$L__BB0_916: + setp.ne.s32 %p1144, %r6282, 0; + mov.u32 %r6286, %r6300; + @%p1144 bra $L__BB0_871; + + setp.eq.s32 %p1145, %r6283, 0; + mov.u32 %r6303, 0; + @%p1145 bra $L__BB0_924; + + mov.u32 %r6297, %r6300; + mov.u32 %r6296, %r6283; + +$L__BB0_919: + mov.u32 %r6300, %r6297; + setp.eq.s32 %p1146, %r6296, 0; + mov.u32 %r6303, %r6283; + @%p1146 bra $L__BB0_924; + + brev.b32 %r4467, %r6296; + bfind.shiftamt.u32 %r4468, %r4467; + mov.pred %p1632, -1; + mov.u32 %r4469, -2; + shf.l.wrap.b32 %r4470, %r4469, %r4469, %r4468; + and.b32 %r6296, %r6296, %r4470; + shr.u32 %r4471, %r4468, 2; + and.b32 %r4472, %r4468, 3; + add.s32 %r4473, %r4472, %r6192; + add.s32 %r4474, %r4471, %r6196; + mad.lo.s32 %r4475, %r4473, %r1, %r4474; + mul.wide.u32 %rd424, %r4475, 4; + add.s64 %rd425, %rd2, %rd424; + ld.global.u32 %r4476, [%rd425]; + shr.u32 %r4477, %r4476, 31; + setp.eq.s32 %p1148, %r6302, 0; + selp.b32 %r4478, 8, 7, %p1148; + shl.b32 %r4479, %r4477, %r6298; + cvt.u16.u32 %rs540, %r4479; + or.b16 %rs747, %rs747, %rs540; + add.s32 %r6298, %r6298, 1; + setp.lt.u32 %p1149, %r6298, %r4478; + mov.u32 %r6297, %r6300; + mov.pred %p1631, %p1632; + @%p1149 bra $L__BB0_923; + + setp.eq.s32 %p1151, %r6300, -1; + mov.pred %p1631, 0; + mov.u32 %r6297, -1; + @%p1151 bra $L__BB0_923; + + and.b16 %rs542, %rs747, 255; + setp.eq.s16 %p1153, %rs542, 255; + selp.u32 %r6302, 1, 0, %p1153; + add.s32 %r6297, %r6300, 1; + mov.u16 %rs747, 0; + mov.u32 %r6298, 0; + mov.pred %p1631, %p1632; + +$L__BB0_923: + mov.u32 %r6307, 0; + @%p1631 bra $L__BB0_919; + bra.uni $L__BB0_928; + +$L__BB0_924: + not.b32 %r4484, %r6303; + and.b32 %r4485, %r1931, %r4484; + setp.ne.s32 %p1156, %r4485, 0; + mov.u32 %r6307, %r4420; + mov.pred %p1632, %p794; + @%p1156 bra $L__BB0_928; + + setp.lt.u32 %p1157, %r4108, %r5; + or.b32 %r4486, %r6303, %r1895; + st.local.u16 [%rd18], %r4486; + shr.u32 %r4487, %r4486, 16; + st.local.u16 [%rd18+2], %r4487; + shl.b32 %r4488, %r4486, 1; + and.b32 %r4489, %r4488, 57344; + and.b32 %r4490, %r4486, 57344; + shr.u32 %r4491, %r4490, 1; + or.b32 %r4492, %r4486, %r1896; + and.b32 %r4493, %r4492, 61440; + or.b32 %r4494, %r4493, %r4489; + or.b32 %r6197, %r4494, %r4491; + mov.u32 %r6196, %r4108; + @%p1157 bra $L__BB0_689; + +$L__BB0_926: + add.s32 %r6192, %r6192, 4; + setp.gt.u32 %p1158, %r6, %r6192; + @%p1158 bra $L__BB0_687; + + setp.eq.s32 %p1159, %r6298, 0; + add.s32 %r4495, %r6300, 1; + setp.eq.s32 %p1160, %r6300, -1; + selp.b32 %r4496, -1, %r4495, %p1160; + selp.b32 %r4497, %r6300, %r4496, %p1159; + setp.ne.s32 %p1161, %r6300, -1; + or.pred %p1162, %p1159, %p1161; + selp.b32 %r6307, %r4497, 0, %p1162; + not.pred %p1632, %p1162; + +$L__BB0_928: + @%p1632 bra $L__BB0_930; + bra.uni $L__BB0_929; + +$L__BB0_930: + mov.u32 %r4505, 2; + st.global.u32 [%rd3], %r4505; + mov.u32 %r4506, 6; + st.global.u32 [%rd3+4], %r4506; + mov.u32 %r4507, 0; + st.global.u32 [%rd3+8], %r4507; + st.global.u32 [%rd3+12], %r4507; + st.global.u32 [%rd3+16], %r4507; + st.global.u32 [%rd3+20], %r4507; + st.global.u32 [%rd3+24], %r4507; + st.global.u32 [%rd3+28], %r4507; + bra.uni $L__BB0_1254; + +$L__BB0_931: + mov.u32 %r6308, 0; + mov.u32 %r6309, %r6308; + mov.u32 %r6310, %r6308; + bra.uni $L__BB0_932; + +$L__BB0_929: + mad.lo.s32 %r4498, %r6, %r5, 7; + shr.u32 %r4499, %r4498, 3; + max.u32 %r6308, %r4499, %r6307; + add.s32 %r4500, %r5180, 6; + mul.wide.u32 %rd426, %r4500, 613566757; + shr.u64 %rd427, %rd426, 32; + cvt.u32.u64 %r4501, %rd427; + sub.s32 %r4502, %r4500, %r4501; + shr.u32 %r4503, %r4502, 1; + add.s32 %r4504, %r4503, %r4501; + shr.u32 %r6309, %r4504, 2; + add.s32 %r6310, %r6308, %r6309; + +$L__BB0_932: + add.s32 %r1984, %r6310, %r1733; + setp.gt.u32 %p1163, %r1984, %r3; + setp.lt.u32 %p1164, %r1733, 2; + or.pred %p1165, %p1164, %p1163; + @%p1165 bra $L__BB0_1247; + bra.uni $L__BB0_933; + +$L__BB0_1247: + mov.u32 %r5086, 1; + st.global.u32 [%rd3], %r5086; + mov.u32 %r5087, 4; + st.global.u32 [%rd3+4], %r5087; + mov.u32 %r5088, 0; + st.global.u32 [%rd3+8], %r5088; + st.global.u32 [%rd3+12], %r5088; + st.global.u32 [%rd3+16], %r5088; + st.global.u32 [%rd3+20], %r5088; + st.global.u32 [%rd3+24], %r5088; + st.global.u32 [%rd3+28], %r5088; + bra.uni $L__BB0_1254; + +$L__BB0_933: + setp.eq.s32 %p1166, %r5271, 0; + @%p1166 bra $L__BB0_939; + + add.s32 %r4512, %r5271, -1; + and.b32 %r6315, %r5271, 3; + setp.lt.u32 %p1167, %r4512, 3; + mov.u32 %r6313, 0; + @%p1167 bra $L__BB0_937; + + sub.s32 %r6312, %r5271, %r6315; + mov.u32 %r6313, 0; + +$L__BB0_936: + add.s32 %r4514, %r6313, 17477; + cvt.u64.u32 %rd429, %r4514; + add.s64 %rd430, %rd1, %rd429; + ld.global.u8 %rs543, [%rd430]; + add.s32 %r4515, %r6313, %r5907; + cvt.u64.u32 %rd431, %r4515; + add.s64 %rd432, %rd1, %rd431; + st.global.u8 [%rd432], %rs543; + ld.global.u8 %rs544, [%rd430+1]; + add.s32 %r4516, %r4515, 1; + cvt.u64.u32 %rd433, %r4516; + add.s64 %rd434, %rd1, %rd433; + st.global.u8 [%rd434], %rs544; + ld.global.u8 %rs545, [%rd430+2]; + add.s32 %r4517, %r4515, 2; + cvt.u64.u32 %rd435, %r4517; + add.s64 %rd436, %rd1, %rd435; + st.global.u8 [%rd436], %rs545; + add.s32 %r4518, %r6313, 17480; + cvt.u64.u32 %rd437, %r4518; + add.s64 %rd438, %rd1, %rd437; + ld.global.u8 %rs546, [%rd438]; + add.s32 %r4519, %r4515, 3; + cvt.u64.u32 %rd439, %r4519; + add.s64 %rd440, %rd1, %rd439; + st.global.u8 [%rd440], %rs546; + add.s32 %r6313, %r6313, 4; + add.s32 %r6312, %r6312, -4; + setp.ne.s32 %p1168, %r6312, 0; + @%p1168 bra $L__BB0_936; + +$L__BB0_937: + setp.eq.s32 %p1169, %r6315, 0; + @%p1169 bra $L__BB0_939; + +$L__BB0_938: + .pragma "nounroll"; + add.s32 %r4520, %r6313, 17477; + cvt.u64.u32 %rd441, %r4520; + add.s64 %rd442, %rd1, %rd441; + ld.global.u8 %rs547, [%rd442]; + add.s32 %r4521, %r6313, %r5907; + cvt.u64.u32 %rd443, %r4521; + add.s64 %rd444, %rd1, %rd443; + st.global.u8 [%rd444], %rs547; + add.s32 %r6313, %r6313, 1; + add.s32 %r6315, %r6315, -1; + setp.ne.s32 %p1170, %r6315, 0; + @%p1170 bra $L__BB0_938; + +$L__BB0_939: + setp.eq.s32 %p1171, %r5719, 0; + @%p1171 bra $L__BB0_945; + + mov.u32 %r4523, 20549; + sub.s32 %r1996, %r4523, %r5719; + and.b32 %r6320, %r5719, 3; + add.s32 %r4524, %r5719, -1; + setp.lt.u32 %p1172, %r4524, 3; + mov.u32 %r6318, 0; + @%p1172 bra $L__BB0_943; + + sub.s32 %r6317, %r5719, %r6320; + mov.u32 %r6318, 0; + +$L__BB0_942: + add.s32 %r4526, %r1996, %r6318; + cvt.u64.u32 %rd445, %r4526; + add.s64 %rd446, %rd1, %rd445; + ld.global.u8 %rs548, [%rd446]; + add.s32 %r4527, %r6318, %r1732; + cvt.u64.u32 %rd447, %r4527; + add.s64 %rd448, %rd1, %rd447; + st.global.u8 [%rd448], %rs548; + add.s32 %r4528, %r6318, 1; + add.s32 %r4529, %r1996, %r4528; + cvt.u64.u32 %rd449, %r4529; + add.s64 %rd450, %rd1, %rd449; + ld.global.u8 %rs549, [%rd450]; + add.s32 %r4530, %r4528, %r1732; + cvt.u64.u32 %rd451, %r4530; + add.s64 %rd452, %rd1, %rd451; + st.global.u8 [%rd452], %rs549; + add.s32 %r4531, %r6318, 2; + add.s32 %r4532, %r1996, %r4531; + cvt.u64.u32 %rd453, %r4532; + add.s64 %rd454, %rd1, %rd453; + ld.global.u8 %rs550, [%rd454]; + add.s32 %r4533, %r4531, %r1732; + cvt.u64.u32 %rd455, %r4533; + add.s64 %rd456, %rd1, %rd455; + st.global.u8 [%rd456], %rs550; + add.s32 %r4534, %r6318, 3; + add.s32 %r4535, %r1996, %r4534; + cvt.u64.u32 %rd457, %r4535; + add.s64 %rd458, %rd1, %rd457; + ld.global.u8 %rs551, [%rd458]; + add.s32 %r4536, %r4534, %r1732; + cvt.u64.u32 %rd459, %r4536; + add.s64 %rd460, %rd1, %rd459; + st.global.u8 [%rd460], %rs551; + add.s32 %r6318, %r6318, 4; + add.s32 %r6317, %r6317, -4; + setp.ne.s32 %p1173, %r6317, 0; + @%p1173 bra $L__BB0_942; + +$L__BB0_943: + setp.eq.s32 %p1174, %r6320, 0; + @%p1174 bra $L__BB0_945; + +$L__BB0_944: + .pragma "nounroll"; + add.s32 %r4537, %r1996, %r6318; + cvt.u64.u32 %rd461, %r4537; + add.s64 %rd462, %rd1, %rd461; + ld.global.u8 %rs552, [%rd462]; + add.s32 %r4538, %r6318, %r1732; + cvt.u64.u32 %rd463, %r4538; + add.s64 %rd464, %rd1, %rd463; + st.global.u8 [%rd464], %rs552; + add.s32 %r6318, %r6318, 1; + add.s32 %r6320, %r6320, -1; + setp.ne.s32 %p1175, %r6320, 0; + @%p1175 bra $L__BB0_944; + +$L__BB0_945: + add.s32 %r4539, %r5719, %r5271; + shr.u32 %r4540, %r4539, 4; + add.s32 %r4541, %r1733, -1; + cvt.u64.u32 %rd465, %r4541; + add.s64 %rd466, %rd1, %rd465; + st.global.u8 [%rd466], %r4540; + add.s32 %r4542, %r1733, -2; + cvt.u64.u32 %rd467, %r4542; + add.s64 %rd468, %rd1, %rd467; + ld.global.u8 %rs553, [%rd468]; + and.b16 %rs554, %rs553, 240; + cvt.u16.u32 %rs555, %r4539; + and.b16 %rs556, %rs555, 15; + or.b16 %rs557, %rs554, %rs556; + st.global.u8 [%rd468], %rs557; + setp.eq.s32 %p1176, %r6310, 0; + @%p1176 bra $L__BB0_951; + + add.s32 %r4544, %r6310, -1; + and.b32 %r6325, %r6310, 3; + setp.lt.u32 %p1177, %r4544, 3; + mov.u32 %r6323, 0; + @%p1177 bra $L__BB0_949; + + sub.s32 %r6322, %r6310, %r6325; + mov.u32 %r6323, 0; + +$L__BB0_948: + add.s32 %r4546, %r6323, %r1733; + cvt.u64.u32 %rd469, %r4546; + add.s64 %rd470, %rd1, %rd469; + mov.u16 %rs558, 0; + st.global.u8 [%rd470], %rs558; + add.s32 %r4547, %r4546, 1; + cvt.u64.u32 %rd471, %r4547; + add.s64 %rd472, %rd1, %rd471; + st.global.u8 [%rd472], %rs558; + add.s32 %r4548, %r4546, 2; + cvt.u64.u32 %rd473, %r4548; + add.s64 %rd474, %rd1, %rd473; + st.global.u8 [%rd474], %rs558; + add.s32 %r4549, %r4546, 3; + cvt.u64.u32 %rd475, %r4549; + add.s64 %rd476, %rd1, %rd475; + st.global.u8 [%rd476], %rs558; + add.s32 %r6323, %r6323, 4; + add.s32 %r6322, %r6322, -4; + setp.ne.s32 %p1178, %r6322, 0; + @%p1178 bra $L__BB0_948; + +$L__BB0_949: + setp.eq.s32 %p1179, %r6325, 0; + @%p1179 bra $L__BB0_951; + +$L__BB0_950: + .pragma "nounroll"; + add.s32 %r4550, %r6323, %r1733; + cvt.u64.u32 %rd477, %r4550; + add.s64 %rd478, %rd1, %rd477; + mov.u16 %rs559, 0; + st.global.u8 [%rd478], %rs559; + add.s32 %r6323, %r6323, 1; + add.s32 %r6325, %r6325, -1; + setp.ne.s32 %p1180, %r6325, 0; + @%p1180 bra $L__BB0_950; + +$L__BB0_951: + setp.ne.s32 %p1181, %r4, 3; + @%p1181 bra $L__BB0_1243; + + ld.param.u64 %rd637, [ j2k_htj2k_encode_codeblock_param_1]; + cvt.u64.u32 %rd35, %r1733; + add.s64 %rd36, %rd637, %rd35; + add.s32 %r4551, %r5, 3; + shr.u32 %r4552, %r4551, 2; + add.s32 %r4553, %r4552, 8; + setp.gt.u32 %p1183, %r4553, 513; + mov.pred %p1182, -1; + mov.pred %p1635, %p1182; + @%p1183 bra $L__BB0_1202; + + mov.u16 %rs755, 0; + st.local.u16 [%rd16], %rs755; + st.local.u16 [%rd16+2], %rs755; + st.local.u16 [%rd16+4], %rs755; + st.local.u16 [%rd16+6], %rs755; + st.local.u16 [%rd16+8], %rs755; + st.local.u16 [%rd16+10], %rs755; + st.local.u16 [%rd16+12], %rs755; + st.local.u16 [%rd16+14], %rs755; + st.local.u16 [%rd16+16], %rs755; + st.local.u16 [%rd16+18], %rs755; + st.local.u16 [%rd16+20], %rs755; + st.local.u16 [%rd16+22], %rs755; + st.local.u16 [%rd16+24], %rs755; + st.local.u16 [%rd16+26], %rs755; + st.local.u16 [%rd16+28], %rs755; + st.local.u16 [%rd16+30], %rs755; + st.local.u16 [%rd16+32], %rs755; + st.local.u16 [%rd16+34], %rs755; + st.local.u16 [%rd16+36], %rs755; + st.local.u16 [%rd16+38], %rs755; + st.local.u16 [%rd16+40], %rs755; + st.local.u16 [%rd16+42], %rs755; + st.local.u16 [%rd16+44], %rs755; + st.local.u16 [%rd16+46], %rs755; + st.local.u16 [%rd16+48], %rs755; + st.local.u16 [%rd16+50], %rs755; + st.local.u16 [%rd16+52], %rs755; + st.local.u16 [%rd16+54], %rs755; + st.local.u16 [%rd16+56], %rs755; + st.local.u16 [%rd16+58], %rs755; + st.local.u16 [%rd16+60], %rs755; + st.local.u16 [%rd16+62], %rs755; + st.local.u16 [%rd16+64], %rs755; + st.local.u16 [%rd16+66], %rs755; + st.local.u16 [%rd16+68], %rs755; + st.local.u16 [%rd16+70], %rs755; + st.local.u16 [%rd16+72], %rs755; + st.local.u16 [%rd16+74], %rs755; + st.local.u16 [%rd16+76], %rs755; + st.local.u16 [%rd16+78], %rs755; + st.local.u16 [%rd16+80], %rs755; + st.local.u16 [%rd16+82], %rs755; + st.local.u16 [%rd16+84], %rs755; + st.local.u16 [%rd16+86], %rs755; + st.local.u16 [%rd16+88], %rs755; + st.local.u16 [%rd16+90], %rs755; + st.local.u16 [%rd16+92], %rs755; + st.local.u16 [%rd16+94], %rs755; + st.local.u16 [%rd16+96], %rs755; + st.local.u16 [%rd16+98], %rs755; + st.local.u16 [%rd16+100], %rs755; + st.local.u16 [%rd16+102], %rs755; + st.local.u16 [%rd16+104], %rs755; + st.local.u16 [%rd16+106], %rs755; + st.local.u16 [%rd16+108], %rs755; + st.local.u16 [%rd16+110], %rs755; + st.local.u16 [%rd16+112], %rs755; + st.local.u16 [%rd16+114], %rs755; + st.local.u16 [%rd16+116], %rs755; + st.local.u16 [%rd16+118], %rs755; + st.local.u16 [%rd16+120], %rs755; + st.local.u16 [%rd16+122], %rs755; + st.local.u16 [%rd16+124], %rs755; + st.local.u16 [%rd16+126], %rs755; + st.local.u16 [%rd16+128], %rs755; + st.local.u16 [%rd16+130], %rs755; + st.local.u16 [%rd16+132], %rs755; + st.local.u16 [%rd16+134], %rs755; + st.local.u16 [%rd16+136], %rs755; + st.local.u16 [%rd16+138], %rs755; + st.local.u16 [%rd16+140], %rs755; + st.local.u16 [%rd16+142], %rs755; + st.local.u16 [%rd16+144], %rs755; + st.local.u16 [%rd16+146], %rs755; + st.local.u16 [%rd16+148], %rs755; + st.local.u16 [%rd16+150], %rs755; + st.local.u16 [%rd16+152], %rs755; + st.local.u16 [%rd16+154], %rs755; + st.local.u16 [%rd16+156], %rs755; + st.local.u16 [%rd16+158], %rs755; + st.local.u16 [%rd16+160], %rs755; + st.local.u16 [%rd16+162], %rs755; + st.local.u16 [%rd16+164], %rs755; + st.local.u16 [%rd16+166], %rs755; + st.local.u16 [%rd16+168], %rs755; + st.local.u16 [%rd16+170], %rs755; + st.local.u16 [%rd16+172], %rs755; + st.local.u16 [%rd16+174], %rs755; + st.local.u16 [%rd16+176], %rs755; + st.local.u16 [%rd16+178], %rs755; + st.local.u16 [%rd16+180], %rs755; + st.local.u16 [%rd16+182], %rs755; + st.local.u16 [%rd16+184], %rs755; + st.local.u16 [%rd16+186], %rs755; + st.local.u16 [%rd16+188], %rs755; + st.local.u16 [%rd16+190], %rs755; + st.local.u16 [%rd16+192], %rs755; + st.local.u16 [%rd16+194], %rs755; + st.local.u16 [%rd16+196], %rs755; + st.local.u16 [%rd16+198], %rs755; + st.local.u16 [%rd16+200], %rs755; + st.local.u16 [%rd16+202], %rs755; + st.local.u16 [%rd16+204], %rs755; + st.local.u16 [%rd16+206], %rs755; + st.local.u16 [%rd16+208], %rs755; + st.local.u16 [%rd16+210], %rs755; + st.local.u16 [%rd16+212], %rs755; + st.local.u16 [%rd16+214], %rs755; + st.local.u16 [%rd16+216], %rs755; + st.local.u16 [%rd16+218], %rs755; + st.local.u16 [%rd16+220], %rs755; + st.local.u16 [%rd16+222], %rs755; + st.local.u16 [%rd16+224], %rs755; + st.local.u16 [%rd16+226], %rs755; + st.local.u16 [%rd16+228], %rs755; + st.local.u16 [%rd16+230], %rs755; + st.local.u16 [%rd16+232], %rs755; + st.local.u16 [%rd16+234], %rs755; + st.local.u16 [%rd16+236], %rs755; + st.local.u16 [%rd16+238], %rs755; + st.local.u16 [%rd16+240], %rs755; + st.local.u16 [%rd16+242], %rs755; + st.local.u16 [%rd16+244], %rs755; + st.local.u16 [%rd16+246], %rs755; + st.local.u16 [%rd16+248], %rs755; + st.local.u16 [%rd16+250], %rs755; + st.local.u16 [%rd16+252], %rs755; + st.local.u16 [%rd16+254], %rs755; + st.local.u16 [%rd16+256], %rs755; + st.local.u16 [%rd16+258], %rs755; + st.local.u16 [%rd16+260], %rs755; + st.local.u16 [%rd16+262], %rs755; + st.local.u16 [%rd16+264], %rs755; + st.local.u16 [%rd16+266], %rs755; + st.local.u16 [%rd16+268], %rs755; + st.local.u16 [%rd16+270], %rs755; + st.local.u16 [%rd16+272], %rs755; + st.local.u16 [%rd16+274], %rs755; + st.local.u16 [%rd16+276], %rs755; + st.local.u16 [%rd16+278], %rs755; + st.local.u16 [%rd16+280], %rs755; + st.local.u16 [%rd16+282], %rs755; + st.local.u16 [%rd16+284], %rs755; + st.local.u16 [%rd16+286], %rs755; + st.local.u16 [%rd16+288], %rs755; + st.local.u16 [%rd16+290], %rs755; + st.local.u16 [%rd16+292], %rs755; + st.local.u16 [%rd16+294], %rs755; + st.local.u16 [%rd16+296], %rs755; + st.local.u16 [%rd16+298], %rs755; + st.local.u16 [%rd16+300], %rs755; + st.local.u16 [%rd16+302], %rs755; + st.local.u16 [%rd16+304], %rs755; + st.local.u16 [%rd16+306], %rs755; + st.local.u16 [%rd16+308], %rs755; + st.local.u16 [%rd16+310], %rs755; + st.local.u16 [%rd16+312], %rs755; + st.local.u16 [%rd16+314], %rs755; + st.local.u16 [%rd16+316], %rs755; + st.local.u16 [%rd16+318], %rs755; + st.local.u16 [%rd16+320], %rs755; + st.local.u16 [%rd16+322], %rs755; + st.local.u16 [%rd16+324], %rs755; + st.local.u16 [%rd16+326], %rs755; + st.local.u16 [%rd16+328], %rs755; + st.local.u16 [%rd16+330], %rs755; + st.local.u16 [%rd16+332], %rs755; + st.local.u16 [%rd16+334], %rs755; + st.local.u16 [%rd16+336], %rs755; + st.local.u16 [%rd16+338], %rs755; + st.local.u16 [%rd16+340], %rs755; + st.local.u16 [%rd16+342], %rs755; + st.local.u16 [%rd16+344], %rs755; + st.local.u16 [%rd16+346], %rs755; + st.local.u16 [%rd16+348], %rs755; + st.local.u16 [%rd16+350], %rs755; + st.local.u16 [%rd16+352], %rs755; + st.local.u16 [%rd16+354], %rs755; + st.local.u16 [%rd16+356], %rs755; + st.local.u16 [%rd16+358], %rs755; + st.local.u16 [%rd16+360], %rs755; + st.local.u16 [%rd16+362], %rs755; + st.local.u16 [%rd16+364], %rs755; + st.local.u16 [%rd16+366], %rs755; + st.local.u16 [%rd16+368], %rs755; + st.local.u16 [%rd16+370], %rs755; + st.local.u16 [%rd16+372], %rs755; + st.local.u16 [%rd16+374], %rs755; + st.local.u16 [%rd16+376], %rs755; + st.local.u16 [%rd16+378], %rs755; + st.local.u16 [%rd16+380], %rs755; + st.local.u16 [%rd16+382], %rs755; + st.local.u16 [%rd16+384], %rs755; + st.local.u16 [%rd16+386], %rs755; + st.local.u16 [%rd16+388], %rs755; + st.local.u16 [%rd16+390], %rs755; + st.local.u16 [%rd16+392], %rs755; + st.local.u16 [%rd16+394], %rs755; + st.local.u16 [%rd16+396], %rs755; + st.local.u16 [%rd16+398], %rs755; + st.local.u16 [%rd16+400], %rs755; + st.local.u16 [%rd16+402], %rs755; + st.local.u16 [%rd16+404], %rs755; + st.local.u16 [%rd16+406], %rs755; + st.local.u16 [%rd16+408], %rs755; + st.local.u16 [%rd16+410], %rs755; + st.local.u16 [%rd16+412], %rs755; + st.local.u16 [%rd16+414], %rs755; + st.local.u16 [%rd16+416], %rs755; + st.local.u16 [%rd16+418], %rs755; + st.local.u16 [%rd16+420], %rs755; + st.local.u16 [%rd16+422], %rs755; + st.local.u16 [%rd16+424], %rs755; + st.local.u16 [%rd16+426], %rs755; + st.local.u16 [%rd16+428], %rs755; + st.local.u16 [%rd16+430], %rs755; + st.local.u16 [%rd16+432], %rs755; + st.local.u16 [%rd16+434], %rs755; + st.local.u16 [%rd16+436], %rs755; + st.local.u16 [%rd16+438], %rs755; + st.local.u16 [%rd16+440], %rs755; + st.local.u16 [%rd16+442], %rs755; + st.local.u16 [%rd16+444], %rs755; + st.local.u16 [%rd16+446], %rs755; + st.local.u16 [%rd16+448], %rs755; + st.local.u16 [%rd16+450], %rs755; + st.local.u16 [%rd16+452], %rs755; + st.local.u16 [%rd16+454], %rs755; + st.local.u16 [%rd16+456], %rs755; + st.local.u16 [%rd16+458], %rs755; + st.local.u16 [%rd16+460], %rs755; + st.local.u16 [%rd16+462], %rs755; + st.local.u16 [%rd16+464], %rs755; + st.local.u16 [%rd16+466], %rs755; + st.local.u16 [%rd16+468], %rs755; + st.local.u16 [%rd16+470], %rs755; + st.local.u16 [%rd16+472], %rs755; + st.local.u16 [%rd16+474], %rs755; + st.local.u16 [%rd16+476], %rs755; + st.local.u16 [%rd16+478], %rs755; + st.local.u16 [%rd16+480], %rs755; + st.local.u16 [%rd16+482], %rs755; + st.local.u16 [%rd16+484], %rs755; + st.local.u16 [%rd16+486], %rs755; + st.local.u16 [%rd16+488], %rs755; + st.local.u16 [%rd16+490], %rs755; + st.local.u16 [%rd16+492], %rs755; + st.local.u16 [%rd16+494], %rs755; + st.local.u16 [%rd16+496], %rs755; + st.local.u16 [%rd16+498], %rs755; + st.local.u16 [%rd16+500], %rs755; + st.local.u16 [%rd16+502], %rs755; + st.local.u16 [%rd16+504], %rs755; + st.local.u16 [%rd16+506], %rs755; + st.local.u16 [%rd16+508], %rs755; + st.local.u16 [%rd16+510], %rs755; + st.local.u16 [%rd16+512], %rs755; + st.local.u16 [%rd16+514], %rs755; + st.local.u16 [%rd16+516], %rs755; + st.local.u16 [%rd16+518], %rs755; + st.local.u16 [%rd16+520], %rs755; + st.local.u16 [%rd16+522], %rs755; + st.local.u16 [%rd16+524], %rs755; + st.local.u16 [%rd16+526], %rs755; + st.local.u16 [%rd16+528], %rs755; + st.local.u16 [%rd16+530], %rs755; + st.local.u16 [%rd16+532], %rs755; + st.local.u16 [%rd16+534], %rs755; + st.local.u16 [%rd16+536], %rs755; + st.local.u16 [%rd16+538], %rs755; + st.local.u16 [%rd16+540], %rs755; + st.local.u16 [%rd16+542], %rs755; + st.local.u16 [%rd16+544], %rs755; + st.local.u16 [%rd16+546], %rs755; + st.local.u16 [%rd16+548], %rs755; + st.local.u16 [%rd16+550], %rs755; + st.local.u16 [%rd16+552], %rs755; + st.local.u16 [%rd16+554], %rs755; + st.local.u16 [%rd16+556], %rs755; + st.local.u16 [%rd16+558], %rs755; + st.local.u16 [%rd16+560], %rs755; + st.local.u16 [%rd16+562], %rs755; + st.local.u16 [%rd16+564], %rs755; + st.local.u16 [%rd16+566], %rs755; + st.local.u16 [%rd16+568], %rs755; + st.local.u16 [%rd16+570], %rs755; + st.local.u16 [%rd16+572], %rs755; + st.local.u16 [%rd16+574], %rs755; + st.local.u16 [%rd16+576], %rs755; + st.local.u16 [%rd16+578], %rs755; + st.local.u16 [%rd16+580], %rs755; + st.local.u16 [%rd16+582], %rs755; + st.local.u16 [%rd16+584], %rs755; + st.local.u16 [%rd16+586], %rs755; + st.local.u16 [%rd16+588], %rs755; + st.local.u16 [%rd16+590], %rs755; + st.local.u16 [%rd16+592], %rs755; + st.local.u16 [%rd16+594], %rs755; + st.local.u16 [%rd16+596], %rs755; + st.local.u16 [%rd16+598], %rs755; + st.local.u16 [%rd16+600], %rs755; + st.local.u16 [%rd16+602], %rs755; + st.local.u16 [%rd16+604], %rs755; + st.local.u16 [%rd16+606], %rs755; + st.local.u16 [%rd16+608], %rs755; + st.local.u16 [%rd16+610], %rs755; + st.local.u16 [%rd16+612], %rs755; + st.local.u16 [%rd16+614], %rs755; + st.local.u16 [%rd16+616], %rs755; + st.local.u16 [%rd16+618], %rs755; + st.local.u16 [%rd16+620], %rs755; + st.local.u16 [%rd16+622], %rs755; + st.local.u16 [%rd16+624], %rs755; + st.local.u16 [%rd16+626], %rs755; + st.local.u16 [%rd16+628], %rs755; + st.local.u16 [%rd16+630], %rs755; + st.local.u16 [%rd16+632], %rs755; + st.local.u16 [%rd16+634], %rs755; + st.local.u16 [%rd16+636], %rs755; + st.local.u16 [%rd16+638], %rs755; + st.local.u16 [%rd16+640], %rs755; + st.local.u16 [%rd16+642], %rs755; + st.local.u16 [%rd16+644], %rs755; + st.local.u16 [%rd16+646], %rs755; + st.local.u16 [%rd16+648], %rs755; + st.local.u16 [%rd16+650], %rs755; + st.local.u16 [%rd16+652], %rs755; + st.local.u16 [%rd16+654], %rs755; + st.local.u16 [%rd16+656], %rs755; + st.local.u16 [%rd16+658], %rs755; + st.local.u16 [%rd16+660], %rs755; + st.local.u16 [%rd16+662], %rs755; + st.local.u16 [%rd16+664], %rs755; + st.local.u16 [%rd16+666], %rs755; + st.local.u16 [%rd16+668], %rs755; + st.local.u16 [%rd16+670], %rs755; + st.local.u16 [%rd16+672], %rs755; + st.local.u16 [%rd16+674], %rs755; + st.local.u16 [%rd16+676], %rs755; + st.local.u16 [%rd16+678], %rs755; + st.local.u16 [%rd16+680], %rs755; + st.local.u16 [%rd16+682], %rs755; + st.local.u16 [%rd16+684], %rs755; + st.local.u16 [%rd16+686], %rs755; + st.local.u16 [%rd16+688], %rs755; + st.local.u16 [%rd16+690], %rs755; + st.local.u16 [%rd16+692], %rs755; + st.local.u16 [%rd16+694], %rs755; + st.local.u16 [%rd16+696], %rs755; + st.local.u16 [%rd16+698], %rs755; + st.local.u16 [%rd16+700], %rs755; + st.local.u16 [%rd16+702], %rs755; + st.local.u16 [%rd16+704], %rs755; + st.local.u16 [%rd16+706], %rs755; + st.local.u16 [%rd16+708], %rs755; + st.local.u16 [%rd16+710], %rs755; + st.local.u16 [%rd16+712], %rs755; + st.local.u16 [%rd16+714], %rs755; + st.local.u16 [%rd16+716], %rs755; + st.local.u16 [%rd16+718], %rs755; + st.local.u16 [%rd16+720], %rs755; + st.local.u16 [%rd16+722], %rs755; + st.local.u16 [%rd16+724], %rs755; + st.local.u16 [%rd16+726], %rs755; + st.local.u16 [%rd16+728], %rs755; + st.local.u16 [%rd16+730], %rs755; + st.local.u16 [%rd16+732], %rs755; + st.local.u16 [%rd16+734], %rs755; + st.local.u16 [%rd16+736], %rs755; + st.local.u16 [%rd16+738], %rs755; + st.local.u16 [%rd16+740], %rs755; + st.local.u16 [%rd16+742], %rs755; + st.local.u16 [%rd16+744], %rs755; + st.local.u16 [%rd16+746], %rs755; + st.local.u16 [%rd16+748], %rs755; + st.local.u16 [%rd16+750], %rs755; + st.local.u16 [%rd16+752], %rs755; + st.local.u16 [%rd16+754], %rs755; + st.local.u16 [%rd16+756], %rs755; + st.local.u16 [%rd16+758], %rs755; + st.local.u16 [%rd16+760], %rs755; + st.local.u16 [%rd16+762], %rs755; + st.local.u16 [%rd16+764], %rs755; + st.local.u16 [%rd16+766], %rs755; + st.local.u16 [%rd16+768], %rs755; + st.local.u16 [%rd16+770], %rs755; + st.local.u16 [%rd16+772], %rs755; + st.local.u16 [%rd16+774], %rs755; + st.local.u16 [%rd16+776], %rs755; + st.local.u16 [%rd16+778], %rs755; + st.local.u16 [%rd16+780], %rs755; + st.local.u16 [%rd16+782], %rs755; + st.local.u16 [%rd16+784], %rs755; + st.local.u16 [%rd16+786], %rs755; + st.local.u16 [%rd16+788], %rs755; + st.local.u16 [%rd16+790], %rs755; + st.local.u16 [%rd16+792], %rs755; + st.local.u16 [%rd16+794], %rs755; + st.local.u16 [%rd16+796], %rs755; + st.local.u16 [%rd16+798], %rs755; + st.local.u16 [%rd16+800], %rs755; + st.local.u16 [%rd16+802], %rs755; + st.local.u16 [%rd16+804], %rs755; + st.local.u16 [%rd16+806], %rs755; + st.local.u16 [%rd16+808], %rs755; + st.local.u16 [%rd16+810], %rs755; + st.local.u16 [%rd16+812], %rs755; + st.local.u16 [%rd16+814], %rs755; + st.local.u16 [%rd16+816], %rs755; + st.local.u16 [%rd16+818], %rs755; + st.local.u16 [%rd16+820], %rs755; + st.local.u16 [%rd16+822], %rs755; + st.local.u16 [%rd16+824], %rs755; + st.local.u16 [%rd16+826], %rs755; + st.local.u16 [%rd16+828], %rs755; + st.local.u16 [%rd16+830], %rs755; + st.local.u16 [%rd16+832], %rs755; + st.local.u16 [%rd16+834], %rs755; + st.local.u16 [%rd16+836], %rs755; + st.local.u16 [%rd16+838], %rs755; + st.local.u16 [%rd16+840], %rs755; + st.local.u16 [%rd16+842], %rs755; + st.local.u16 [%rd16+844], %rs755; + st.local.u16 [%rd16+846], %rs755; + st.local.u16 [%rd16+848], %rs755; + st.local.u16 [%rd16+850], %rs755; + st.local.u16 [%rd16+852], %rs755; + st.local.u16 [%rd16+854], %rs755; + st.local.u16 [%rd16+856], %rs755; + st.local.u16 [%rd16+858], %rs755; + st.local.u16 [%rd16+860], %rs755; + st.local.u16 [%rd16+862], %rs755; + st.local.u16 [%rd16+864], %rs755; + st.local.u16 [%rd16+866], %rs755; + st.local.u16 [%rd16+868], %rs755; + st.local.u16 [%rd16+870], %rs755; + st.local.u16 [%rd16+872], %rs755; + st.local.u16 [%rd16+874], %rs755; + st.local.u16 [%rd16+876], %rs755; + st.local.u16 [%rd16+878], %rs755; + st.local.u16 [%rd16+880], %rs755; + st.local.u16 [%rd16+882], %rs755; + st.local.u16 [%rd16+884], %rs755; + st.local.u16 [%rd16+886], %rs755; + st.local.u16 [%rd16+888], %rs755; + st.local.u16 [%rd16+890], %rs755; + st.local.u16 [%rd16+892], %rs755; + st.local.u16 [%rd16+894], %rs755; + st.local.u16 [%rd16+896], %rs755; + st.local.u16 [%rd16+898], %rs755; + st.local.u16 [%rd16+900], %rs755; + st.local.u16 [%rd16+902], %rs755; + st.local.u16 [%rd16+904], %rs755; + st.local.u16 [%rd16+906], %rs755; + st.local.u16 [%rd16+908], %rs755; + st.local.u16 [%rd16+910], %rs755; + st.local.u16 [%rd16+912], %rs755; + st.local.u16 [%rd16+914], %rs755; + st.local.u16 [%rd16+916], %rs755; + st.local.u16 [%rd16+918], %rs755; + st.local.u16 [%rd16+920], %rs755; + st.local.u16 [%rd16+922], %rs755; + st.local.u16 [%rd16+924], %rs755; + st.local.u16 [%rd16+926], %rs755; + st.local.u16 [%rd16+928], %rs755; + st.local.u16 [%rd16+930], %rs755; + st.local.u16 [%rd16+932], %rs755; + st.local.u16 [%rd16+934], %rs755; + st.local.u16 [%rd16+936], %rs755; + st.local.u16 [%rd16+938], %rs755; + st.local.u16 [%rd16+940], %rs755; + st.local.u16 [%rd16+942], %rs755; + st.local.u16 [%rd16+944], %rs755; + st.local.u16 [%rd16+946], %rs755; + st.local.u16 [%rd16+948], %rs755; + st.local.u16 [%rd16+950], %rs755; + st.local.u16 [%rd16+952], %rs755; + st.local.u16 [%rd16+954], %rs755; + st.local.u16 [%rd16+956], %rs755; + st.local.u16 [%rd16+958], %rs755; + st.local.u16 [%rd16+960], %rs755; + st.local.u16 [%rd16+962], %rs755; + st.local.u16 [%rd16+964], %rs755; + st.local.u16 [%rd16+966], %rs755; + st.local.u16 [%rd16+968], %rs755; + st.local.u16 [%rd16+970], %rs755; + st.local.u16 [%rd16+972], %rs755; + st.local.u16 [%rd16+974], %rs755; + st.local.u16 [%rd16+976], %rs755; + st.local.u16 [%rd16+978], %rs755; + st.local.u16 [%rd16+980], %rs755; + st.local.u16 [%rd16+982], %rs755; + st.local.u16 [%rd16+984], %rs755; + st.local.u16 [%rd16+986], %rs755; + st.local.u16 [%rd16+988], %rs755; + st.local.u16 [%rd16+990], %rs755; + st.local.u16 [%rd16+992], %rs755; + st.local.u16 [%rd16+994], %rs755; + st.local.u16 [%rd16+996], %rs755; + st.local.u16 [%rd16+998], %rs755; + st.local.u16 [%rd16+1000], %rs755; + st.local.u16 [%rd16+1002], %rs755; + st.local.u16 [%rd16+1004], %rs755; + st.local.u16 [%rd16+1006], %rs755; + st.local.u16 [%rd16+1008], %rs755; + st.local.u16 [%rd16+1010], %rs755; + st.local.u16 [%rd16+1012], %rs755; + st.local.u16 [%rd16+1014], %rs755; + st.local.u16 [%rd16+1016], %rs755; + st.local.u16 [%rd16+1018], %rs755; + st.local.u16 [%rd16+1020], %rs755; + st.local.u16 [%rd16+1022], %rs755; + st.local.u16 [%rd16+1024], %rs755; + mov.u32 %r6326, 0; + mov.u32 %r6436, %r6326; + mov.u32 %r6432, %r6326; + mov.u32 %r6434, %r6326; + +$L__BB0_954: + @%p9 bra $L__BB0_1197; + + sub.s32 %r4560, %r6, %r6326; + add.s32 %r2023, %r6326, 4; + mul.lo.s32 %r2024, %r2023, %r1; + add.s32 %r2025, %r6326, 5; + add.s32 %r2026, %r2024, %r1; + add.s32 %r2027, %r6326, 6; + shl.b32 %r4561, %r1, 1; + add.s32 %r2028, %r2024, %r4561; + add.s32 %r2029, %r6326, 7; + mul.lo.s32 %r4562, %r1, 3; + add.s32 %r2030, %r2024, %r4562; + add.s32 %r2031, %r6326, 1; + add.s32 %r2032, %r6326, 2; + add.s32 %r2033, %r6326, 3; + mul.lo.s32 %r2034, %r6326, %r1; + add.s32 %r2035, %r2034, %r4562; + sub.s32 %r2036, %r2035, %r1; + sub.s32 %r2037, %r2036, %r1; + setp.lt.u32 %p1185, %r4560, 2; + selp.b32 %r4563, 4369, 13107, %p1185; + setp.lt.u32 %p1186, %r4560, 3; + selp.b32 %r4564, %r4563, 30583, %p1186; + setp.lt.u32 %p1187, %r4560, 4; + selp.b32 %r2038, %r4564, 65535, %p1187; + mov.u32 %r4559, 0; + mov.u32 %r6330, %r4559; + mov.u32 %r6331, %r4559; + +$L__BB0_956: + shr.u32 %r4566, %r6330, 2; + mul.wide.u32 %rd479, %r4566, 2; + add.s64 %rd37, %rd16, %rd479; + ld.local.u16 %rs249, [%rd37]; + ld.local.u16 %rs250, [%rd37+2]; + setp.ge.u32 %p1188, %r6330, %r5; + mov.u32 %r6342, %r4559; + @%p1188 bra $L__BB0_965; + + setp.ge.u32 %p1189, %r2023, %r6; + mov.u32 %r6342, 0; + @%p1189 bra $L__BB0_959; + + add.s32 %r4568, %r2024, %r6330; + mul.wide.u32 %rd480, %r4568, 4; + add.s64 %rd481, %rd2, %rd480; + ld.global.u32 %r4569, [%rd481]; + abs.s32 %r4570, %r4569; + setp.gt.u32 %p1190, %r4570, 4; + and.b32 %r4571, %r4570, 1; + setp.eq.b32 %p1191, %r4571, 1; + and.pred %p1192, %p1190, %p1191; + selp.u32 %r6342, 1, 0, %p1192; + +$L__BB0_959: + setp.ge.u32 %p1193, %r2025, %r6; + @%p1193 bra $L__BB0_961; + + add.s32 %r4572, %r2026, %r6330; + mul.wide.u32 %rd482, %r4572, 4; + add.s64 %rd483, %rd2, %rd482; + ld.global.u32 %r4573, [%rd483]; + abs.s32 %r4574, %r4573; + setp.gt.u32 %p1194, %r4574, 4; + and.b32 %r4575, %r4574, 1; + setp.eq.b32 %p1195, %r4575, 1; + and.pred %p1196, %p1194, %p1195; + selp.b32 %r4576, 2, 0, %p1196; + or.b32 %r6342, %r4576, %r6342; + +$L__BB0_961: + setp.ge.u32 %p1197, %r2027, %r6; + @%p1197 bra $L__BB0_963; + + add.s32 %r4577, %r2028, %r6330; + mul.wide.u32 %rd484, %r4577, 4; + add.s64 %rd485, %rd2, %rd484; + ld.global.u32 %r4578, [%rd485]; + abs.s32 %r4579, %r4578; + setp.gt.u32 %p1198, %r4579, 4; + and.b32 %r4580, %r4579, 1; + setp.eq.b32 %p1199, %r4580, 1; + and.pred %p1200, %p1198, %p1199; + selp.b32 %r4581, 4, 0, %p1200; + or.b32 %r6342, %r4581, %r6342; + +$L__BB0_963: + setp.ge.u32 %p1201, %r2029, %r6; + @%p1201 bra $L__BB0_965; + + add.s32 %r4582, %r2030, %r6330; + mul.wide.u32 %rd486, %r4582, 4; + add.s64 %rd487, %rd2, %rd486; + ld.global.u32 %r4583, [%rd487]; + abs.s32 %r4584, %r4583; + setp.gt.u32 %p1202, %r4584, 4; + and.b32 %r4585, %r4584, 1; + setp.eq.b32 %p1203, %r4585, 1; + and.pred %p1204, %p1202, %p1203; + selp.b32 %r4586, 8, 0, %p1204; + or.b32 %r6342, %r4586, %r6342; + +$L__BB0_965: + add.s32 %r2052, %r6330, 1; + setp.ge.u32 %p1205, %r2052, %r5; + @%p1205 bra $L__BB0_974; + + setp.ge.u32 %p1206, %r2023, %r6; + @%p1206 bra $L__BB0_968; + + add.s32 %r4587, %r2024, %r2052; + mul.wide.u32 %rd488, %r4587, 4; + add.s64 %rd489, %rd2, %rd488; + ld.global.u32 %r4588, [%rd489]; + abs.s32 %r4589, %r4588; + setp.gt.u32 %p1207, %r4589, 4; + and.b32 %r4590, %r4589, 1; + setp.eq.b32 %p1208, %r4590, 1; + and.pred %p1209, %p1207, %p1208; + selp.b32 %r4591, 16, 0, %p1209; + or.b32 %r6342, %r4591, %r6342; + +$L__BB0_968: + setp.ge.u32 %p1210, %r2025, %r6; + @%p1210 bra $L__BB0_970; + + add.s32 %r4592, %r2026, %r2052; + mul.wide.u32 %rd490, %r4592, 4; + add.s64 %rd491, %rd2, %rd490; + ld.global.u32 %r4593, [%rd491]; + abs.s32 %r4594, %r4593; + setp.gt.u32 %p1211, %r4594, 4; + and.b32 %r4595, %r4594, 1; + setp.eq.b32 %p1212, %r4595, 1; + and.pred %p1213, %p1211, %p1212; + selp.b32 %r4596, 32, 0, %p1213; + or.b32 %r6342, %r4596, %r6342; + +$L__BB0_970: + setp.ge.u32 %p1214, %r2027, %r6; + @%p1214 bra $L__BB0_972; + + add.s32 %r4597, %r2028, %r2052; + mul.wide.u32 %rd492, %r4597, 4; + add.s64 %rd493, %rd2, %rd492; + ld.global.u32 %r4598, [%rd493]; + abs.s32 %r4599, %r4598; + setp.gt.u32 %p1215, %r4599, 4; + and.b32 %r4600, %r4599, 1; + setp.eq.b32 %p1216, %r4600, 1; + and.pred %p1217, %p1215, %p1216; + selp.b32 %r4601, 64, 0, %p1217; + or.b32 %r6342, %r4601, %r6342; + +$L__BB0_972: + setp.ge.u32 %p1218, %r2029, %r6; + @%p1218 bra $L__BB0_974; + + add.s32 %r4602, %r2030, %r2052; + mul.wide.u32 %rd494, %r4602, 4; + add.s64 %rd495, %rd2, %rd494; + ld.global.u32 %r4603, [%rd495]; + abs.s32 %r4604, %r4603; + setp.gt.u32 %p1219, %r4604, 4; + and.b32 %r4605, %r4604, 1; + setp.eq.b32 %p1220, %r4605, 1; + and.pred %p1221, %p1219, %p1220; + selp.b32 %r4606, 128, 0, %p1221; + or.b32 %r6342, %r4606, %r6342; + +$L__BB0_974: + add.s32 %r2061, %r6330, 2; + setp.ge.u32 %p1222, %r2061, %r5; + @%p1222 bra $L__BB0_983; + + setp.ge.u32 %p1223, %r2023, %r6; + @%p1223 bra $L__BB0_977; + + add.s32 %r4607, %r2024, %r2061; + mul.wide.u32 %rd496, %r4607, 4; + add.s64 %rd497, %rd2, %rd496; + ld.global.u32 %r4608, [%rd497]; + abs.s32 %r4609, %r4608; + setp.gt.u32 %p1224, %r4609, 4; + and.b32 %r4610, %r4609, 1; + setp.eq.b32 %p1225, %r4610, 1; + and.pred %p1226, %p1224, %p1225; + selp.b32 %r4611, 256, 0, %p1226; + or.b32 %r6342, %r4611, %r6342; + +$L__BB0_977: + setp.ge.u32 %p1227, %r2025, %r6; + @%p1227 bra $L__BB0_979; + + add.s32 %r4612, %r2026, %r2061; + mul.wide.u32 %rd498, %r4612, 4; + add.s64 %rd499, %rd2, %rd498; + ld.global.u32 %r4613, [%rd499]; + abs.s32 %r4614, %r4613; + setp.gt.u32 %p1228, %r4614, 4; + and.b32 %r4615, %r4614, 1; + setp.eq.b32 %p1229, %r4615, 1; + and.pred %p1230, %p1228, %p1229; + selp.b32 %r4616, 512, 0, %p1230; + or.b32 %r6342, %r4616, %r6342; + +$L__BB0_979: + setp.ge.u32 %p1231, %r2027, %r6; + @%p1231 bra $L__BB0_981; + + add.s32 %r4617, %r2028, %r2061; + mul.wide.u32 %rd500, %r4617, 4; + add.s64 %rd501, %rd2, %rd500; + ld.global.u32 %r4618, [%rd501]; + abs.s32 %r4619, %r4618; + setp.gt.u32 %p1232, %r4619, 4; + and.b32 %r4620, %r4619, 1; + setp.eq.b32 %p1233, %r4620, 1; + and.pred %p1234, %p1232, %p1233; + selp.b32 %r4621, 1024, 0, %p1234; + or.b32 %r6342, %r4621, %r6342; + +$L__BB0_981: + setp.ge.u32 %p1235, %r2029, %r6; + @%p1235 bra $L__BB0_983; + + add.s32 %r4622, %r2030, %r2061; + mul.wide.u32 %rd502, %r4622, 4; + add.s64 %rd503, %rd2, %rd502; + ld.global.u32 %r4623, [%rd503]; + abs.s32 %r4624, %r4623; + setp.gt.u32 %p1236, %r4624, 4; + and.b32 %r4625, %r4624, 1; + setp.eq.b32 %p1237, %r4625, 1; + and.pred %p1238, %p1236, %p1237; + selp.b32 %r4626, 2048, 0, %p1238; + or.b32 %r6342, %r4626, %r6342; + +$L__BB0_983: + add.s32 %r2070, %r6330, 3; + setp.ge.u32 %p1239, %r2070, %r5; + @%p1239 bra $L__BB0_992; + + setp.ge.u32 %p1240, %r2023, %r6; + @%p1240 bra $L__BB0_986; + + add.s32 %r4627, %r2024, %r2070; + mul.wide.u32 %rd504, %r4627, 4; + add.s64 %rd505, %rd2, %rd504; + ld.global.u32 %r4628, [%rd505]; + abs.s32 %r4629, %r4628; + setp.gt.u32 %p1241, %r4629, 4; + and.b32 %r4630, %r4629, 1; + setp.eq.b32 %p1242, %r4630, 1; + and.pred %p1243, %p1241, %p1242; + selp.b32 %r4631, 4096, 0, %p1243; + or.b32 %r6342, %r4631, %r6342; + +$L__BB0_986: + setp.ge.u32 %p1244, %r2025, %r6; + @%p1244 bra $L__BB0_988; + + add.s32 %r4632, %r2026, %r2070; + mul.wide.u32 %rd506, %r4632, 4; + add.s64 %rd507, %rd2, %rd506; + ld.global.u32 %r4633, [%rd507]; + abs.s32 %r4634, %r4633; + setp.gt.u32 %p1245, %r4634, 4; + and.b32 %r4635, %r4634, 1; + setp.eq.b32 %p1246, %r4635, 1; + and.pred %p1247, %p1245, %p1246; + selp.b32 %r4636, 8192, 0, %p1247; + or.b32 %r6342, %r4636, %r6342; + +$L__BB0_988: + setp.ge.u32 %p1248, %r2027, %r6; + @%p1248 bra $L__BB0_990; + + add.s32 %r4637, %r2028, %r2070; + mul.wide.u32 %rd508, %r4637, 4; + add.s64 %rd509, %rd2, %rd508; + ld.global.u32 %r4638, [%rd509]; + abs.s32 %r4639, %r4638; + setp.gt.u32 %p1249, %r4639, 4; + and.b32 %r4640, %r4639, 1; + setp.eq.b32 %p1250, %r4640, 1; + and.pred %p1251, %p1249, %p1250; + selp.b32 %r4641, 16384, 0, %p1251; + or.b32 %r6342, %r4641, %r6342; + +$L__BB0_990: + setp.ge.u32 %p1252, %r2029, %r6; + @%p1252 bra $L__BB0_992; + + add.s32 %r4642, %r2030, %r2070; + mul.wide.u32 %rd510, %r4642, 4; + add.s64 %rd511, %rd2, %rd510; + ld.global.u32 %r4643, [%rd511]; + abs.s32 %r4644, %r4643; + setp.gt.u32 %p1253, %r4644, 4; + and.b32 %r4645, %r4644, 1; + setp.eq.b32 %p1254, %r4645, 1; + and.pred %p1255, %p1253, %p1254; + selp.b32 %r4646, 32768, 0, %p1255; + or.b32 %r6342, %r4646, %r6342; + +$L__BB0_992: + add.s32 %r4648, %r6330, 4; + setp.ge.u32 %p1256, %r4648, %r5; + mov.u32 %r6358, 0; + @%p1256 bra $L__BB0_1001; + + setp.ge.u32 %p1257, %r2023, %r6; + mov.u32 %r6358, 0; + @%p1257 bra $L__BB0_995; + + add.s32 %r4650, %r2024, %r6330; + add.s32 %r4651, %r4650, 4; + mul.wide.u32 %rd512, %r4651, 4; + add.s64 %rd513, %rd2, %rd512; + ld.global.u32 %r4652, [%rd513]; + abs.s32 %r4653, %r4652; + setp.gt.u32 %p1258, %r4653, 4; + and.b32 %r4654, %r4653, 1; + setp.eq.b32 %p1259, %r4654, 1; + and.pred %p1260, %p1258, %p1259; + selp.u32 %r6358, 1, 0, %p1260; + +$L__BB0_995: + setp.ge.u32 %p1261, %r2025, %r6; + @%p1261 bra $L__BB0_997; + + add.s32 %r4655, %r2026, %r6330; + add.s32 %r4656, %r4655, 4; + mul.wide.u32 %rd514, %r4656, 4; + add.s64 %rd515, %rd2, %rd514; + ld.global.u32 %r4657, [%rd515]; + abs.s32 %r4658, %r4657; + setp.gt.u32 %p1262, %r4658, 4; + and.b32 %r4659, %r4658, 1; + setp.eq.b32 %p1263, %r4659, 1; + and.pred %p1264, %p1262, %p1263; + selp.b32 %r4660, 2, 0, %p1264; + or.b32 %r6358, %r4660, %r6358; + +$L__BB0_997: + setp.ge.u32 %p1265, %r2027, %r6; + @%p1265 bra $L__BB0_999; + + add.s32 %r4661, %r2028, %r6330; + add.s32 %r4662, %r4661, 4; + mul.wide.u32 %rd516, %r4662, 4; + add.s64 %rd517, %rd2, %rd516; + ld.global.u32 %r4663, [%rd517]; + abs.s32 %r4664, %r4663; + setp.gt.u32 %p1266, %r4664, 4; + and.b32 %r4665, %r4664, 1; + setp.eq.b32 %p1267, %r4665, 1; + and.pred %p1268, %p1266, %p1267; + selp.b32 %r4666, 4, 0, %p1268; + or.b32 %r6358, %r4666, %r6358; + +$L__BB0_999: + setp.ge.u32 %p1269, %r2029, %r6; + @%p1269 bra $L__BB0_1001; + + add.s32 %r4667, %r2030, %r6330; + add.s32 %r4668, %r4667, 4; + mul.wide.u32 %rd518, %r4668, 4; + add.s64 %rd519, %rd2, %rd518; + ld.global.u32 %r4669, [%rd519]; + abs.s32 %r4670, %r4669; + setp.gt.u32 %p1270, %r4670, 4; + and.b32 %r4671, %r4670, 1; + setp.eq.b32 %p1271, %r4671, 1; + and.pred %p1272, %p1270, %p1271; + selp.b32 %r4672, 8, 0, %p1272; + or.b32 %r6358, %r4672, %r6358; + +$L__BB0_1001: + add.s32 %r2087, %r6330, 5; + setp.ge.u32 %p1273, %r2087, %r5; + @%p1273 bra $L__BB0_1010; + + setp.ge.u32 %p1274, %r2023, %r6; + @%p1274 bra $L__BB0_1004; + + add.s32 %r4673, %r2024, %r2087; + mul.wide.u32 %rd520, %r4673, 4; + add.s64 %rd521, %rd2, %rd520; + ld.global.u32 %r4674, [%rd521]; + abs.s32 %r4675, %r4674; + setp.gt.u32 %p1275, %r4675, 4; + and.b32 %r4676, %r4675, 1; + setp.eq.b32 %p1276, %r4676, 1; + and.pred %p1277, %p1275, %p1276; + selp.b32 %r4677, 16, 0, %p1277; + or.b32 %r6358, %r4677, %r6358; + +$L__BB0_1004: + setp.ge.u32 %p1278, %r2025, %r6; + @%p1278 bra $L__BB0_1006; + + add.s32 %r4678, %r2026, %r2087; + mul.wide.u32 %rd522, %r4678, 4; + add.s64 %rd523, %rd2, %rd522; + ld.global.u32 %r4679, [%rd523]; + abs.s32 %r4680, %r4679; + setp.gt.u32 %p1279, %r4680, 4; + and.b32 %r4681, %r4680, 1; + setp.eq.b32 %p1280, %r4681, 1; + and.pred %p1281, %p1279, %p1280; + selp.b32 %r4682, 32, 0, %p1281; + or.b32 %r6358, %r4682, %r6358; + +$L__BB0_1006: + setp.ge.u32 %p1282, %r2027, %r6; + @%p1282 bra $L__BB0_1008; + + add.s32 %r4683, %r2028, %r2087; + mul.wide.u32 %rd524, %r4683, 4; + add.s64 %rd525, %rd2, %rd524; + ld.global.u32 %r4684, [%rd525]; + abs.s32 %r4685, %r4684; + setp.gt.u32 %p1283, %r4685, 4; + and.b32 %r4686, %r4685, 1; + setp.eq.b32 %p1284, %r4686, 1; + and.pred %p1285, %p1283, %p1284; + selp.b32 %r4687, 64, 0, %p1285; + or.b32 %r6358, %r4687, %r6358; + +$L__BB0_1008: + setp.ge.u32 %p1286, %r2029, %r6; + @%p1286 bra $L__BB0_1010; + + add.s32 %r4688, %r2030, %r2087; + mul.wide.u32 %rd526, %r4688, 4; + add.s64 %rd527, %rd2, %rd526; + ld.global.u32 %r4689, [%rd527]; + abs.s32 %r4690, %r4689; + setp.gt.u32 %p1287, %r4690, 4; + and.b32 %r4691, %r4690, 1; + setp.eq.b32 %p1288, %r4691, 1; + and.pred %p1289, %p1287, %p1288; + selp.b32 %r4692, 128, 0, %p1289; + or.b32 %r6358, %r4692, %r6358; + +$L__BB0_1010: + add.s32 %r2096, %r6330, 6; + setp.ge.u32 %p1290, %r2096, %r5; + @%p1290 bra $L__BB0_1019; + + setp.ge.u32 %p1291, %r2023, %r6; + @%p1291 bra $L__BB0_1013; + + add.s32 %r4693, %r2024, %r2096; + mul.wide.u32 %rd528, %r4693, 4; + add.s64 %rd529, %rd2, %rd528; + ld.global.u32 %r4694, [%rd529]; + abs.s32 %r4695, %r4694; + setp.gt.u32 %p1292, %r4695, 4; + and.b32 %r4696, %r4695, 1; + setp.eq.b32 %p1293, %r4696, 1; + and.pred %p1294, %p1292, %p1293; + selp.b32 %r4697, 256, 0, %p1294; + or.b32 %r6358, %r4697, %r6358; + +$L__BB0_1013: + setp.ge.u32 %p1295, %r2025, %r6; + @%p1295 bra $L__BB0_1015; + + add.s32 %r4698, %r2026, %r2096; + mul.wide.u32 %rd530, %r4698, 4; + add.s64 %rd531, %rd2, %rd530; + ld.global.u32 %r4699, [%rd531]; + abs.s32 %r4700, %r4699; + setp.gt.u32 %p1296, %r4700, 4; + and.b32 %r4701, %r4700, 1; + setp.eq.b32 %p1297, %r4701, 1; + and.pred %p1298, %p1296, %p1297; + selp.b32 %r4702, 512, 0, %p1298; + or.b32 %r6358, %r4702, %r6358; + +$L__BB0_1015: + setp.ge.u32 %p1299, %r2027, %r6; + @%p1299 bra $L__BB0_1017; + + add.s32 %r4703, %r2028, %r2096; + mul.wide.u32 %rd532, %r4703, 4; + add.s64 %rd533, %rd2, %rd532; + ld.global.u32 %r4704, [%rd533]; + abs.s32 %r4705, %r4704; + setp.gt.u32 %p1300, %r4705, 4; + and.b32 %r4706, %r4705, 1; + setp.eq.b32 %p1301, %r4706, 1; + and.pred %p1302, %p1300, %p1301; + selp.b32 %r4707, 1024, 0, %p1302; + or.b32 %r6358, %r4707, %r6358; + +$L__BB0_1017: + setp.ge.u32 %p1303, %r2029, %r6; + @%p1303 bra $L__BB0_1019; + + add.s32 %r4708, %r2030, %r2096; + mul.wide.u32 %rd534, %r4708, 4; + add.s64 %rd535, %rd2, %rd534; + ld.global.u32 %r4709, [%rd535]; + abs.s32 %r4710, %r4709; + setp.gt.u32 %p1304, %r4710, 4; + and.b32 %r4711, %r4710, 1; + setp.eq.b32 %p1305, %r4711, 1; + and.pred %p1306, %p1304, %p1305; + selp.b32 %r4712, 2048, 0, %p1306; + or.b32 %r6358, %r4712, %r6358; + +$L__BB0_1019: + add.s32 %r2105, %r6330, 7; + setp.ge.u32 %p1307, %r2105, %r5; + @%p1307 bra $L__BB0_1028; + + setp.ge.u32 %p1308, %r2023, %r6; + @%p1308 bra $L__BB0_1022; + + add.s32 %r4713, %r2024, %r2105; + mul.wide.u32 %rd536, %r4713, 4; + add.s64 %rd537, %rd2, %rd536; + ld.global.u32 %r4714, [%rd537]; + abs.s32 %r4715, %r4714; + setp.gt.u32 %p1309, %r4715, 4; + and.b32 %r4716, %r4715, 1; + setp.eq.b32 %p1310, %r4716, 1; + and.pred %p1311, %p1309, %p1310; + selp.b32 %r4717, 4096, 0, %p1311; + or.b32 %r6358, %r4717, %r6358; + +$L__BB0_1022: + setp.ge.u32 %p1312, %r2025, %r6; + @%p1312 bra $L__BB0_1024; + + add.s32 %r4718, %r2026, %r2105; + mul.wide.u32 %rd538, %r4718, 4; + add.s64 %rd539, %rd2, %rd538; + ld.global.u32 %r4719, [%rd539]; + abs.s32 %r4720, %r4719; + setp.gt.u32 %p1313, %r4720, 4; + and.b32 %r4721, %r4720, 1; + setp.eq.b32 %p1314, %r4721, 1; + and.pred %p1315, %p1313, %p1314; + selp.b32 %r4722, 8192, 0, %p1315; + or.b32 %r6358, %r4722, %r6358; + +$L__BB0_1024: + setp.ge.u32 %p1316, %r2027, %r6; + @%p1316 bra $L__BB0_1026; + + add.s32 %r4723, %r2028, %r2105; + mul.wide.u32 %rd540, %r4723, 4; + add.s64 %rd541, %rd2, %rd540; + ld.global.u32 %r4724, [%rd541]; + abs.s32 %r4725, %r4724; + setp.gt.u32 %p1317, %r4725, 4; + and.b32 %r4726, %r4725, 1; + setp.eq.b32 %p1318, %r4726, 1; + and.pred %p1319, %p1317, %p1318; + selp.b32 %r4727, 16384, 0, %p1319; + or.b32 %r6358, %r4727, %r6358; + +$L__BB0_1026: + setp.ge.u32 %p1320, %r2029, %r6; + @%p1320 bra $L__BB0_1028; + + add.s32 %r4728, %r2030, %r2105; + mul.wide.u32 %rd542, %r4728, 4; + add.s64 %rd543, %rd2, %rd542; + ld.global.u32 %r4729, [%rd543]; + abs.s32 %r4730, %r4729; + setp.gt.u32 %p1321, %r4730, 4; + and.b32 %r4731, %r4730, 1; + setp.eq.b32 %p1322, %r4731, 1; + and.pred %p1323, %p1321, %p1322; + selp.b32 %r4732, 32768, 0, %p1323; + or.b32 %r6358, %r4732, %r6358; + +$L__BB0_1028: + mov.b32 %r2114, {%rs249, %rs250}; + add.s32 %r4734, %r2034, %r6330; + mul.wide.u32 %rd544, %r4734, 4; + add.s64 %rd38, %rd2, %rd544; + add.s32 %r4735, %r2037, %r6330; + mul.wide.u32 %rd545, %r4735, 4; + add.s64 %rd39, %rd2, %rd545; + add.s32 %r4736, %r2036, %r6330; + mul.wide.u32 %rd546, %r4736, 4; + add.s64 %rd40, %rd2, %rd546; + add.s32 %r4737, %r2035, %r6330; + mul.wide.u32 %rd547, %r4737, 4; + add.s64 %rd41, %rd2, %rd547; + mov.u32 %r6374, 0; + @%p1188 bra $L__BB0_1037; + + setp.le.u32 %p1325, %r6, %r6326; + mov.u32 %r6374, 0; + @%p1325 bra $L__BB0_1031; + + ld.global.u32 %r4739, [%rd38]; + abs.s32 %r4740, %r4739; + setp.gt.u32 %p1326, %r4740, 4; + and.b32 %r4741, %r4740, 1; + setp.eq.b32 %p1327, %r4741, 1; + and.pred %p1328, %p1326, %p1327; + selp.u32 %r6374, 1, 0, %p1328; + +$L__BB0_1031: + setp.ge.u32 %p1329, %r2031, %r6; + @%p1329 bra $L__BB0_1033; + + ld.global.u32 %r4742, [%rd39]; + abs.s32 %r4743, %r4742; + setp.gt.u32 %p1330, %r4743, 4; + and.b32 %r4744, %r4743, 1; + setp.eq.b32 %p1331, %r4744, 1; + and.pred %p1332, %p1330, %p1331; + selp.b32 %r4745, 2, 0, %p1332; + or.b32 %r6374, %r4745, %r6374; + +$L__BB0_1033: + setp.ge.u32 %p1333, %r2032, %r6; + @%p1333 bra $L__BB0_1035; + + ld.global.u32 %r4746, [%rd40]; + abs.s32 %r4747, %r4746; + setp.gt.u32 %p1334, %r4747, 4; + and.b32 %r4748, %r4747, 1; + setp.eq.b32 %p1335, %r4748, 1; + and.pred %p1336, %p1334, %p1335; + selp.b32 %r4749, 4, 0, %p1336; + or.b32 %r6374, %r4749, %r6374; + +$L__BB0_1035: + setp.ge.u32 %p1337, %r2033, %r6; + @%p1337 bra $L__BB0_1037; + + ld.global.u32 %r4750, [%rd41]; + abs.s32 %r4751, %r4750; + setp.gt.u32 %p1338, %r4751, 4; + and.b32 %r4752, %r4751, 1; + setp.eq.b32 %p1339, %r4752, 1; + and.pred %p1340, %p1338, %p1339; + selp.b32 %r4753, 8, 0, %p1340; + or.b32 %r6374, %r4753, %r6374; + +$L__BB0_1037: + add.s32 %r4754, %r2034, %r2052; + mul.wide.u32 %rd548, %r4754, 4; + add.s64 %rd42, %rd2, %rd548; + add.s32 %r4755, %r2037, %r2052; + mul.wide.u32 %rd549, %r4755, 4; + add.s64 %rd43, %rd2, %rd549; + add.s32 %r4756, %r2036, %r2052; + mul.wide.u32 %rd550, %r4756, 4; + add.s64 %rd44, %rd2, %rd550; + add.s32 %r4757, %r2035, %r2052; + mul.wide.u32 %rd551, %r4757, 4; + add.s64 %rd45, %rd2, %rd551; + shl.b32 %r4758, %r6358, 16; + or.b32 %r2123, %r4758, %r6342; + @%p1205 bra $L__BB0_1046; + + setp.le.u32 %p1342, %r6, %r6326; + @%p1342 bra $L__BB0_1040; + + ld.global.u32 %r4759, [%rd42]; + abs.s32 %r4760, %r4759; + setp.gt.u32 %p1343, %r4760, 4; + and.b32 %r4761, %r4760, 1; + setp.eq.b32 %p1344, %r4761, 1; + and.pred %p1345, %p1343, %p1344; + selp.b32 %r4762, 16, 0, %p1345; + or.b32 %r6374, %r4762, %r6374; + +$L__BB0_1040: + setp.ge.u32 %p1346, %r2031, %r6; + @%p1346 bra $L__BB0_1042; + + ld.global.u32 %r4763, [%rd43]; + abs.s32 %r4764, %r4763; + setp.gt.u32 %p1347, %r4764, 4; + and.b32 %r4765, %r4764, 1; + setp.eq.b32 %p1348, %r4765, 1; + and.pred %p1349, %p1347, %p1348; + selp.b32 %r4766, 32, 0, %p1349; + or.b32 %r6374, %r4766, %r6374; + +$L__BB0_1042: + setp.ge.u32 %p1350, %r2032, %r6; + @%p1350 bra $L__BB0_1044; + + ld.global.u32 %r4767, [%rd44]; + abs.s32 %r4768, %r4767; + setp.gt.u32 %p1351, %r4768, 4; + and.b32 %r4769, %r4768, 1; + setp.eq.b32 %p1352, %r4769, 1; + and.pred %p1353, %p1351, %p1352; + selp.b32 %r4770, 64, 0, %p1353; + or.b32 %r6374, %r4770, %r6374; + +$L__BB0_1044: + setp.ge.u32 %p1354, %r2033, %r6; + @%p1354 bra $L__BB0_1046; + + ld.global.u32 %r4771, [%rd45]; + abs.s32 %r4772, %r4771; + setp.gt.u32 %p1355, %r4772, 4; + and.b32 %r4773, %r4772, 1; + setp.eq.b32 %p1356, %r4773, 1; + and.pred %p1357, %p1355, %p1356; + selp.b32 %r4774, 128, 0, %p1357; + or.b32 %r6374, %r4774, %r6374; + +$L__BB0_1046: + add.s32 %r4775, %r2034, %r2061; + mul.wide.u32 %rd552, %r4775, 4; + add.s64 %rd46, %rd2, %rd552; + add.s32 %r4776, %r2037, %r2061; + mul.wide.u32 %rd553, %r4776, 4; + add.s64 %rd47, %rd2, %rd553; + add.s32 %r4777, %r2036, %r2061; + mul.wide.u32 %rd554, %r4777, 4; + add.s64 %rd48, %rd2, %rd554; + add.s32 %r4778, %r2035, %r2061; + mul.wide.u32 %rd555, %r4778, 4; + add.s64 %rd49, %rd2, %rd555; + @%p1222 bra $L__BB0_1055; + + setp.le.u32 %p1359, %r6, %r6326; + @%p1359 bra $L__BB0_1049; + + ld.global.u32 %r4779, [%rd46]; + abs.s32 %r4780, %r4779; + setp.gt.u32 %p1360, %r4780, 4; + and.b32 %r4781, %r4780, 1; + setp.eq.b32 %p1361, %r4781, 1; + and.pred %p1362, %p1360, %p1361; + selp.b32 %r4782, 256, 0, %p1362; + or.b32 %r6374, %r4782, %r6374; + +$L__BB0_1049: + setp.ge.u32 %p1363, %r2031, %r6; + @%p1363 bra $L__BB0_1051; + + ld.global.u32 %r4783, [%rd47]; + abs.s32 %r4784, %r4783; + setp.gt.u32 %p1364, %r4784, 4; + and.b32 %r4785, %r4784, 1; + setp.eq.b32 %p1365, %r4785, 1; + and.pred %p1366, %p1364, %p1365; + selp.b32 %r4786, 512, 0, %p1366; + or.b32 %r6374, %r4786, %r6374; + +$L__BB0_1051: + setp.ge.u32 %p1367, %r2032, %r6; + @%p1367 bra $L__BB0_1053; + + ld.global.u32 %r4787, [%rd48]; + abs.s32 %r4788, %r4787; + setp.gt.u32 %p1368, %r4788, 4; + and.b32 %r4789, %r4788, 1; + setp.eq.b32 %p1369, %r4789, 1; + and.pred %p1370, %p1368, %p1369; + selp.b32 %r4790, 1024, 0, %p1370; + or.b32 %r6374, %r4790, %r6374; + +$L__BB0_1053: + setp.ge.u32 %p1371, %r2033, %r6; + @%p1371 bra $L__BB0_1055; + + ld.global.u32 %r4791, [%rd49]; + abs.s32 %r4792, %r4791; + setp.gt.u32 %p1372, %r4792, 4; + and.b32 %r4793, %r4792, 1; + setp.eq.b32 %p1373, %r4793, 1; + and.pred %p1374, %p1372, %p1373; + selp.b32 %r4794, 2048, 0, %p1374; + or.b32 %r6374, %r4794, %r6374; + +$L__BB0_1055: + add.s32 %r4795, %r2034, %r2070; + mul.wide.u32 %rd556, %r4795, 4; + add.s64 %rd50, %rd2, %rd556; + add.s32 %r4796, %r2037, %r2070; + mul.wide.u32 %rd557, %r4796, 4; + add.s64 %rd51, %rd2, %rd557; + add.s32 %r4797, %r2036, %r2070; + mul.wide.u32 %rd558, %r4797, 4; + add.s64 %rd52, %rd2, %rd558; + add.s32 %r4798, %r2035, %r2070; + mul.wide.u32 %rd559, %r4798, 4; + add.s64 %rd53, %rd2, %rd559; + @%p1239 bra $L__BB0_1064; + + setp.le.u32 %p1376, %r6, %r6326; + @%p1376 bra $L__BB0_1058; + + ld.global.u32 %r4799, [%rd50]; + abs.s32 %r4800, %r4799; + setp.gt.u32 %p1377, %r4800, 4; + and.b32 %r4801, %r4800, 1; + setp.eq.b32 %p1378, %r4801, 1; + and.pred %p1379, %p1377, %p1378; + selp.b32 %r4802, 4096, 0, %p1379; + or.b32 %r6374, %r4802, %r6374; + +$L__BB0_1058: + setp.ge.u32 %p1380, %r2031, %r6; + @%p1380 bra $L__BB0_1060; + + ld.global.u32 %r4803, [%rd51]; + abs.s32 %r4804, %r4803; + setp.gt.u32 %p1381, %r4804, 4; + and.b32 %r4805, %r4804, 1; + setp.eq.b32 %p1382, %r4805, 1; + and.pred %p1383, %p1381, %p1382; + selp.b32 %r4806, 8192, 0, %p1383; + or.b32 %r6374, %r4806, %r6374; + +$L__BB0_1060: + setp.ge.u32 %p1384, %r2032, %r6; + @%p1384 bra $L__BB0_1062; + + ld.global.u32 %r4807, [%rd52]; + abs.s32 %r4808, %r4807; + setp.gt.u32 %p1385, %r4808, 4; + and.b32 %r4809, %r4808, 1; + setp.eq.b32 %p1386, %r4809, 1; + and.pred %p1387, %p1385, %p1386; + selp.b32 %r4810, 16384, 0, %p1387; + or.b32 %r6374, %r4810, %r6374; + +$L__BB0_1062: + setp.ge.u32 %p1388, %r2033, %r6; + @%p1388 bra $L__BB0_1064; + + ld.global.u32 %r4811, [%rd53]; + abs.s32 %r4812, %r4811; + setp.gt.u32 %p1389, %r4812, 4; + and.b32 %r4813, %r4812, 1; + setp.eq.b32 %p1390, %r4813, 1; + and.pred %p1391, %p1389, %p1390; + selp.b32 %r4814, 32768, 0, %p1391; + or.b32 %r6374, %r4814, %r6374; + +$L__BB0_1064: + mov.u32 %r6390, 0; + @%p1256 bra $L__BB0_1073; + + setp.le.u32 %p1393, %r6, %r6326; + mov.u32 %r6390, 0; + @%p1393 bra $L__BB0_1067; + + add.s32 %r4819, %r4734, 4; + mul.wide.u32 %rd560, %r4819, 4; + add.s64 %rd561, %rd2, %rd560; + ld.global.u32 %r4820, [%rd561]; + abs.s32 %r4821, %r4820; + setp.gt.u32 %p1394, %r4821, 4; + and.b32 %r4822, %r4821, 1; + setp.eq.b32 %p1395, %r4822, 1; + and.pred %p1396, %p1394, %p1395; + selp.u32 %r6390, 1, 0, %p1396; + +$L__BB0_1067: + setp.ge.u32 %p1397, %r2031, %r6; + @%p1397 bra $L__BB0_1069; + + add.s32 %r4824, %r4735, 4; + mul.wide.u32 %rd562, %r4824, 4; + add.s64 %rd563, %rd2, %rd562; + ld.global.u32 %r4825, [%rd563]; + abs.s32 %r4826, %r4825; + setp.gt.u32 %p1398, %r4826, 4; + and.b32 %r4827, %r4826, 1; + setp.eq.b32 %p1399, %r4827, 1; + and.pred %p1400, %p1398, %p1399; + selp.b32 %r4828, 2, 0, %p1400; + or.b32 %r6390, %r4828, %r6390; + +$L__BB0_1069: + setp.ge.u32 %p1401, %r2032, %r6; + @%p1401 bra $L__BB0_1071; + + add.s32 %r4830, %r4736, 4; + mul.wide.u32 %rd564, %r4830, 4; + add.s64 %rd565, %rd2, %rd564; + ld.global.u32 %r4831, [%rd565]; + abs.s32 %r4832, %r4831; + setp.gt.u32 %p1402, %r4832, 4; + and.b32 %r4833, %r4832, 1; + setp.eq.b32 %p1403, %r4833, 1; + and.pred %p1404, %p1402, %p1403; + selp.b32 %r4834, 4, 0, %p1404; + or.b32 %r6390, %r4834, %r6390; + +$L__BB0_1071: + setp.ge.u32 %p1405, %r2033, %r6; + @%p1405 bra $L__BB0_1073; + + add.s32 %r4836, %r4737, 4; + mul.wide.u32 %rd566, %r4836, 4; + add.s64 %rd567, %rd2, %rd566; + ld.global.u32 %r4837, [%rd567]; + abs.s32 %r4838, %r4837; + setp.gt.u32 %p1406, %r4838, 4; + and.b32 %r4839, %r4838, 1; + setp.eq.b32 %p1407, %r4839, 1; + and.pred %p1408, %p1406, %p1407; + selp.b32 %r4840, 8, 0, %p1408; + or.b32 %r6390, %r4840, %r6390; + +$L__BB0_1073: + @%p1273 bra $L__BB0_1082; + + setp.le.u32 %p1410, %r6, %r6326; + @%p1410 bra $L__BB0_1076; + + add.s32 %r4841, %r2034, %r2087; + mul.wide.u32 %rd568, %r4841, 4; + add.s64 %rd569, %rd2, %rd568; + ld.global.u32 %r4842, [%rd569]; + abs.s32 %r4843, %r4842; + setp.gt.u32 %p1411, %r4843, 4; + and.b32 %r4844, %r4843, 1; + setp.eq.b32 %p1412, %r4844, 1; + and.pred %p1413, %p1411, %p1412; + selp.b32 %r4845, 16, 0, %p1413; + or.b32 %r6390, %r4845, %r6390; + +$L__BB0_1076: + setp.ge.u32 %p1414, %r2031, %r6; + @%p1414 bra $L__BB0_1078; + + add.s32 %r4846, %r2037, %r2087; + mul.wide.u32 %rd570, %r4846, 4; + add.s64 %rd571, %rd2, %rd570; + ld.global.u32 %r4847, [%rd571]; + abs.s32 %r4848, %r4847; + setp.gt.u32 %p1415, %r4848, 4; + and.b32 %r4849, %r4848, 1; + setp.eq.b32 %p1416, %r4849, 1; + and.pred %p1417, %p1415, %p1416; + selp.b32 %r4850, 32, 0, %p1417; + or.b32 %r6390, %r4850, %r6390; + +$L__BB0_1078: + setp.ge.u32 %p1418, %r2032, %r6; + @%p1418 bra $L__BB0_1080; + + add.s32 %r4851, %r2036, %r2087; + mul.wide.u32 %rd572, %r4851, 4; + add.s64 %rd573, %rd2, %rd572; + ld.global.u32 %r4852, [%rd573]; + abs.s32 %r4853, %r4852; + setp.gt.u32 %p1419, %r4853, 4; + and.b32 %r4854, %r4853, 1; + setp.eq.b32 %p1420, %r4854, 1; + and.pred %p1421, %p1419, %p1420; + selp.b32 %r4855, 64, 0, %p1421; + or.b32 %r6390, %r4855, %r6390; + +$L__BB0_1080: + setp.ge.u32 %p1422, %r2033, %r6; + @%p1422 bra $L__BB0_1082; + + add.s32 %r4856, %r2035, %r2087; + mul.wide.u32 %rd574, %r4856, 4; + add.s64 %rd575, %rd2, %rd574; + ld.global.u32 %r4857, [%rd575]; + abs.s32 %r4858, %r4857; + setp.gt.u32 %p1423, %r4858, 4; + and.b32 %r4859, %r4858, 1; + setp.eq.b32 %p1424, %r4859, 1; + and.pred %p1425, %p1423, %p1424; + selp.b32 %r4860, 128, 0, %p1425; + or.b32 %r6390, %r4860, %r6390; + +$L__BB0_1082: + @%p1290 bra $L__BB0_1091; + + setp.le.u32 %p1427, %r6, %r6326; + @%p1427 bra $L__BB0_1085; + + add.s32 %r4861, %r2034, %r2096; + mul.wide.u32 %rd576, %r4861, 4; + add.s64 %rd577, %rd2, %rd576; + ld.global.u32 %r4862, [%rd577]; + abs.s32 %r4863, %r4862; + setp.gt.u32 %p1428, %r4863, 4; + and.b32 %r4864, %r4863, 1; + setp.eq.b32 %p1429, %r4864, 1; + and.pred %p1430, %p1428, %p1429; + selp.b32 %r4865, 256, 0, %p1430; + or.b32 %r6390, %r4865, %r6390; + +$L__BB0_1085: + setp.ge.u32 %p1431, %r2031, %r6; + @%p1431 bra $L__BB0_1087; + + add.s32 %r4866, %r2037, %r2096; + mul.wide.u32 %rd578, %r4866, 4; + add.s64 %rd579, %rd2, %rd578; + ld.global.u32 %r4867, [%rd579]; + abs.s32 %r4868, %r4867; + setp.gt.u32 %p1432, %r4868, 4; + and.b32 %r4869, %r4868, 1; + setp.eq.b32 %p1433, %r4869, 1; + and.pred %p1434, %p1432, %p1433; + selp.b32 %r4870, 512, 0, %p1434; + or.b32 %r6390, %r4870, %r6390; + +$L__BB0_1087: + setp.ge.u32 %p1435, %r2032, %r6; + @%p1435 bra $L__BB0_1089; + + add.s32 %r4871, %r2036, %r2096; + mul.wide.u32 %rd580, %r4871, 4; + add.s64 %rd581, %rd2, %rd580; + ld.global.u32 %r4872, [%rd581]; + abs.s32 %r4873, %r4872; + setp.gt.u32 %p1436, %r4873, 4; + and.b32 %r4874, %r4873, 1; + setp.eq.b32 %p1437, %r4874, 1; + and.pred %p1438, %p1436, %p1437; + selp.b32 %r4875, 1024, 0, %p1438; + or.b32 %r6390, %r4875, %r6390; + +$L__BB0_1089: + setp.ge.u32 %p1439, %r2033, %r6; + @%p1439 bra $L__BB0_1091; + + add.s32 %r4876, %r2035, %r2096; + mul.wide.u32 %rd582, %r4876, 4; + add.s64 %rd583, %rd2, %rd582; + ld.global.u32 %r4877, [%rd583]; + abs.s32 %r4878, %r4877; + setp.gt.u32 %p1440, %r4878, 4; + and.b32 %r4879, %r4878, 1; + setp.eq.b32 %p1441, %r4879, 1; + and.pred %p1442, %p1440, %p1441; + selp.b32 %r4880, 2048, 0, %p1442; + or.b32 %r6390, %r4880, %r6390; + +$L__BB0_1091: + @%p1307 bra $L__BB0_1100; + + setp.le.u32 %p1444, %r6, %r6326; + @%p1444 bra $L__BB0_1094; + + add.s32 %r4881, %r2034, %r2105; + mul.wide.u32 %rd584, %r4881, 4; + add.s64 %rd585, %rd2, %rd584; + ld.global.u32 %r4882, [%rd585]; + abs.s32 %r4883, %r4882; + setp.gt.u32 %p1445, %r4883, 4; + and.b32 %r4884, %r4883, 1; + setp.eq.b32 %p1446, %r4884, 1; + and.pred %p1447, %p1445, %p1446; + selp.b32 %r4885, 4096, 0, %p1447; + or.b32 %r6390, %r4885, %r6390; + +$L__BB0_1094: + setp.ge.u32 %p1448, %r2031, %r6; + @%p1448 bra $L__BB0_1096; + + add.s32 %r4886, %r2037, %r2105; + mul.wide.u32 %rd586, %r4886, 4; + add.s64 %rd587, %rd2, %rd586; + ld.global.u32 %r4887, [%rd587]; + abs.s32 %r4888, %r4887; + setp.gt.u32 %p1449, %r4888, 4; + and.b32 %r4889, %r4888, 1; + setp.eq.b32 %p1450, %r4889, 1; + and.pred %p1451, %p1449, %p1450; + selp.b32 %r4890, 8192, 0, %p1451; + or.b32 %r6390, %r4890, %r6390; + +$L__BB0_1096: + setp.ge.u32 %p1452, %r2032, %r6; + @%p1452 bra $L__BB0_1098; + + add.s32 %r4891, %r2036, %r2105; + mul.wide.u32 %rd588, %r4891, 4; + add.s64 %rd589, %rd2, %rd588; + ld.global.u32 %r4892, [%rd589]; + abs.s32 %r4893, %r4892; + setp.gt.u32 %p1453, %r4893, 4; + and.b32 %r4894, %r4893, 1; + setp.eq.b32 %p1454, %r4894, 1; + and.pred %p1455, %p1453, %p1454; + selp.b32 %r4895, 16384, 0, %p1455; + or.b32 %r6390, %r4895, %r6390; + +$L__BB0_1098: + setp.ge.u32 %p1456, %r2033, %r6; + @%p1456 bra $L__BB0_1100; + + add.s32 %r4896, %r2035, %r2105; + mul.wide.u32 %rd590, %r4896, 4; + add.s64 %rd591, %rd2, %rd590; + ld.global.u32 %r4897, [%rd591]; + abs.s32 %r4898, %r4897; + setp.gt.u32 %p1457, %r4898, 4; + and.b32 %r4899, %r4898, 1; + setp.eq.b32 %p1458, %r4899, 1; + and.pred %p1459, %p1457, %p1458; + selp.b32 %r4900, 32768, 0, %p1459; + or.b32 %r6390, %r4900, %r6390; + +$L__BB0_1100: + sub.s32 %r4903, %r4648, %r5; + shl.b32 %r4904, %r6390, 16; + or.b32 %r2180, %r4904, %r6374; + and.b32 %r4905, %r2114, -2004318072; + shr.u32 %r4906, %r4905, 3; + shl.b32 %r4907, %r2123, 3; + and.b32 %r4908, %r4907, -2004318072; + or.b32 %r2181, %r4908, %r4906; + not.b32 %r4909, %r2180; + setp.gt.s32 %p1460, %r4903, 0; + mov.u32 %r6406, 0; + shl.b32 %r4910, %r4903, 2; + selp.b32 %r4911, %r4910, 0, %p1460; + shr.u32 %r2182, %r2038, %r4911; + and.b32 %r2183, %r2182, %r4909; + @%p1188 bra $L__BB0_1109; + + setp.le.u32 %p1462, %r6, %r6326; + mov.u32 %r6406, 0; + @%p1462 bra $L__BB0_1103; + + ld.global.u32 %r4913, [%rd38]; + abs.s32 %r4914, %r4913; + setp.eq.s32 %p1463, %r4914, 3; + selp.u32 %r6406, 1, 0, %p1463; + +$L__BB0_1103: + setp.ge.u32 %p1464, %r2031, %r6; + @%p1464 bra $L__BB0_1105; + + ld.global.u32 %r4915, [%rd39]; + abs.s32 %r4916, %r4915; + setp.eq.s32 %p1465, %r4916, 3; + selp.b32 %r4917, 2, 0, %p1465; + or.b32 %r6406, %r4917, %r6406; + +$L__BB0_1105: + setp.ge.u32 %p1466, %r2032, %r6; + @%p1466 bra $L__BB0_1107; + + ld.global.u32 %r4918, [%rd40]; + abs.s32 %r4919, %r4918; + setp.eq.s32 %p1467, %r4919, 3; + selp.b32 %r4920, 4, 0, %p1467; + or.b32 %r6406, %r4920, %r6406; + +$L__BB0_1107: + setp.ge.u32 %p1468, %r2033, %r6; + @%p1468 bra $L__BB0_1109; + + ld.global.u32 %r4921, [%rd41]; + abs.s32 %r4922, %r4921; + setp.eq.s32 %p1469, %r4922, 3; + selp.b32 %r4923, 8, 0, %p1469; + or.b32 %r6406, %r4923, %r6406; + +$L__BB0_1109: + @%p1205 bra $L__BB0_1118; + + setp.le.u32 %p1471, %r6, %r6326; + @%p1471 bra $L__BB0_1112; + + ld.global.u32 %r4924, [%rd42]; + abs.s32 %r4925, %r4924; + setp.eq.s32 %p1472, %r4925, 3; + selp.b32 %r4926, 16, 0, %p1472; + or.b32 %r6406, %r4926, %r6406; + +$L__BB0_1112: + setp.ge.u32 %p1473, %r2031, %r6; + @%p1473 bra $L__BB0_1114; + + ld.global.u32 %r4927, [%rd43]; + abs.s32 %r4928, %r4927; + setp.eq.s32 %p1474, %r4928, 3; + selp.b32 %r4929, 32, 0, %p1474; + or.b32 %r6406, %r4929, %r6406; + +$L__BB0_1114: + setp.ge.u32 %p1475, %r2032, %r6; + @%p1475 bra $L__BB0_1116; + + ld.global.u32 %r4930, [%rd44]; + abs.s32 %r4931, %r4930; + setp.eq.s32 %p1476, %r4931, 3; + selp.b32 %r4932, 64, 0, %p1476; + or.b32 %r6406, %r4932, %r6406; + +$L__BB0_1116: + setp.ge.u32 %p1477, %r2033, %r6; + @%p1477 bra $L__BB0_1118; + + ld.global.u32 %r4933, [%rd45]; + abs.s32 %r4934, %r4933; + setp.eq.s32 %p1478, %r4934, 3; + selp.b32 %r4935, 128, 0, %p1478; + or.b32 %r6406, %r4935, %r6406; + +$L__BB0_1118: + @%p1222 bra $L__BB0_1127; + + setp.le.u32 %p1480, %r6, %r6326; + @%p1480 bra $L__BB0_1121; + + ld.global.u32 %r4936, [%rd46]; + abs.s32 %r4937, %r4936; + setp.eq.s32 %p1481, %r4937, 3; + selp.b32 %r4938, 256, 0, %p1481; + or.b32 %r6406, %r4938, %r6406; + +$L__BB0_1121: + setp.ge.u32 %p1482, %r2031, %r6; + @%p1482 bra $L__BB0_1123; + + ld.global.u32 %r4939, [%rd47]; + abs.s32 %r4940, %r4939; + setp.eq.s32 %p1483, %r4940, 3; + selp.b32 %r4941, 512, 0, %p1483; + or.b32 %r6406, %r4941, %r6406; + +$L__BB0_1123: + setp.ge.u32 %p1484, %r2032, %r6; + @%p1484 bra $L__BB0_1125; + + ld.global.u32 %r4942, [%rd48]; + abs.s32 %r4943, %r4942; + setp.eq.s32 %p1485, %r4943, 3; + selp.b32 %r4944, 1024, 0, %p1485; + or.b32 %r6406, %r4944, %r6406; + +$L__BB0_1125: + setp.ge.u32 %p1486, %r2033, %r6; + @%p1486 bra $L__BB0_1127; + + ld.global.u32 %r4945, [%rd49]; + abs.s32 %r4946, %r4945; + setp.eq.s32 %p1487, %r4946, 3; + selp.b32 %r4947, 2048, 0, %p1487; + or.b32 %r6406, %r4947, %r6406; + +$L__BB0_1127: + @%p1239 bra $L__BB0_1136; + + setp.le.u32 %p1489, %r6, %r6326; + @%p1489 bra $L__BB0_1130; + + ld.global.u32 %r4948, [%rd50]; + abs.s32 %r4949, %r4948; + setp.eq.s32 %p1490, %r4949, 3; + selp.b32 %r4950, 4096, 0, %p1490; + or.b32 %r6406, %r4950, %r6406; + +$L__BB0_1130: + setp.ge.u32 %p1491, %r2031, %r6; + @%p1491 bra $L__BB0_1132; + + ld.global.u32 %r4951, [%rd51]; + abs.s32 %r4952, %r4951; + setp.eq.s32 %p1492, %r4952, 3; + selp.b32 %r4953, 8192, 0, %p1492; + or.b32 %r6406, %r4953, %r6406; + +$L__BB0_1132: + setp.ge.u32 %p1493, %r2032, %r6; + @%p1493 bra $L__BB0_1134; + + ld.global.u32 %r4954, [%rd52]; + abs.s32 %r4955, %r4954; + setp.eq.s32 %p1494, %r4955, 3; + selp.b32 %r4956, 16384, 0, %p1494; + or.b32 %r6406, %r4956, %r6406; + +$L__BB0_1134: + setp.ge.u32 %p1495, %r2033, %r6; + @%p1495 bra $L__BB0_1136; + + ld.global.u32 %r4957, [%rd53]; + abs.s32 %r4958, %r4957; + setp.eq.s32 %p1496, %r4958, 3; + selp.b32 %r4959, 32768, 0, %p1496; + or.b32 %r6406, %r4959, %r6406; + +$L__BB0_1136: + and.b32 %r4961, %r2180, -286331154; + shr.u32 %r4962, %r4961, 1; + shl.b32 %r4963, %r2180, 1; + and.b32 %r4964, %r4963, -286331154; + or.b32 %r4965, %r2180, %r2181; + or.b32 %r4966, %r4965, %r4964; + or.b32 %r4967, %r4966, %r4962; + and.b32 %r2216, %r6406, %r2182; + shr.u32 %r4968, %r4967, 4; + shl.b32 %r4969, %r4967, 4; + shr.u32 %r4970, %r6331, 12; + or.b32 %r4971, %r4967, %r4970; + or.b32 %r4972, %r4971, %r4969; + or.b32 %r4973, %r4972, %r4968; + and.b32 %r6416, %r2183, %r4973; + setp.eq.s32 %p1497, %r6416, 0; + mov.u32 %r6437, 0; + @%p1497 bra $L__BB0_1195; + + mov.u32 %r6415, 0; + mov.u32 %r6417, %r6415; + +$L__BB0_1138: + brev.b32 %r4976, %r6416; + bfind.shiftamt.u32 %r2224, %r4976; + mov.pred %p1635, -1; + mov.u32 %r4977, 1; + shl.b32 %r2225, %r4977, %r2224; + mov.u32 %r4978, -2; + shf.l.wrap.b32 %r4979, %r4978, %r4978, %r2224; + and.b32 %r6416, %r6416, %r4979; + or.b32 %r6415, %r2225, %r6415; + and.b32 %r2228, %r2225, %r2216; + setp.ne.s32 %p1499, %r2228, 0; + selp.u32 %r4980, 1, 0, %p1499; + setp.eq.s32 %p1500, %r6436, 0; + selp.b32 %r4981, 8, 7, %p1500; + shl.b32 %r4982, %r4980, %r6432; + cvt.u16.u32 %rs561, %r4982; + or.b16 %rs755, %rs755, %rs561; + add.s32 %r6432, %r6432, 1; + setp.lt.u32 %p1501, %r6432, %r4981; + mov.pred %p1633, %p1635; + @%p1501 bra $L__BB0_1143; + + setp.ge.u32 %p1503, %r6434, %r6308; + mov.pred %p1633, 0; + @%p1503 bra $L__BB0_1143; + + setp.eq.s64 %p1504, %rd36, 0; + @%p1504 bra $L__BB0_1142; + + cvt.u64.u32 %rd592, %r6434; + add.s64 %rd593, %rd592, %rd35; + add.s64 %rd594, %rd1, %rd593; + st.global.u8 [%rd594], %rs755; + +$L__BB0_1142: + and.b16 %rs563, %rs755, 255; + setp.eq.s16 %p1506, %rs563, 255; + selp.u32 %r6436, 1, 0, %p1506; + add.s32 %r6434, %r6434, 1; + mov.u16 %rs755, 0; + mov.u32 %r6432, 0; + mov.pred %p1633, %p1635; + +$L__BB0_1143: + not.pred %p1508, %p1633; + @%p1508 bra $L__BB0_1202; + + setp.eq.s32 %p1509, %r2228, 0; + @%p1509 bra $L__BB0_1185; + + or.b32 %r6417, %r2225, %r6417; + mov.u32 %r6424, 51; + setp.gt.s32 %p1510, %r2224, 7; + @%p1510 bra $L__BB0_1161; + + setp.gt.s32 %p1522, %r2224, 3; + @%p1522 bra $L__BB0_1154; + + setp.gt.s32 %p1528, %r2224, 1; + @%p1528 bra $L__BB0_1151; + + setp.eq.s32 %p1531, %r2224, 0; + @%p1531 bra $L__BB0_1184; + + setp.eq.s32 %p1532, %r2224, 1; + @%p1532 bra $L__BB0_1150; + bra.uni $L__BB0_1183; + +$L__BB0_1150: + mov.u32 %r6424, 118; + bra.uni $L__BB0_1184; + +$L__BB0_1161: + setp.gt.s32 %p1511, %r2224, 11; + @%p1511 bra $L__BB0_1169; + + setp.gt.s32 %p1517, %r2224, 9; + @%p1517 bra $L__BB0_1166; + + setp.eq.s32 %p1520, %r2224, 8; + @%p1520 bra $L__BB0_1179; + + setp.eq.s32 %p1521, %r2224, 9; + @%p1521 bra $L__BB0_1165; + bra.uni $L__BB0_1183; + +$L__BB0_1165: + mov.u32 %r6424, 30208; + bra.uni $L__BB0_1184; + +$L__BB0_1154: + setp.gt.s32 %p1523, %r2224, 5; + @%p1523 bra $L__BB0_1158; + + setp.eq.s32 %p1526, %r2224, 4; + @%p1526 bra $L__BB0_1181; + + setp.eq.s32 %p1527, %r2224, 5; + @%p1527 bra $L__BB0_1157; + bra.uni $L__BB0_1183; + +$L__BB0_1157: + mov.u32 %r6424, 1888; + bra.uni $L__BB0_1184; + +$L__BB0_1169: + setp.gt.s32 %p1512, %r2224, 13; + @%p1512 bra $L__BB0_1173; + + setp.eq.s32 %p1515, %r2224, 12; + @%p1515 bra $L__BB0_1177; + + setp.eq.s32 %p1516, %r2224, 13; + @%p1516 bra $L__BB0_1172; + bra.uni $L__BB0_1183; + +$L__BB0_1172: + mov.u32 %r6424, 483328; + bra.uni $L__BB0_1184; + +$L__BB0_1151: + setp.eq.s32 %p1529, %r2224, 2; + @%p1529 bra $L__BB0_1182; + + setp.eq.s32 %p1530, %r2224, 3; + @%p1530 bra $L__BB0_1153; + bra.uni $L__BB0_1183; + +$L__BB0_1153: + mov.u32 %r6424, 200; + bra.uni $L__BB0_1184; + +$L__BB0_1166: + setp.eq.s32 %p1518, %r2224, 10; + @%p1518 bra $L__BB0_1178; + + setp.eq.s32 %p1519, %r2224, 11; + @%p1519 bra $L__BB0_1168; + bra.uni $L__BB0_1183; + +$L__BB0_1168: + mov.u32 %r6424, 51200; + bra.uni $L__BB0_1184; + +$L__BB0_1158: + setp.eq.s32 %p1524, %r2224, 6; + @%p1524 bra $L__BB0_1180; + + setp.eq.s32 %p1525, %r2224, 7; + @%p1525 bra $L__BB0_1160; + bra.uni $L__BB0_1183; + +$L__BB0_1160: + mov.u32 %r6424, 3200; + bra.uni $L__BB0_1184; + +$L__BB0_1173: + setp.eq.s32 %p1513, %r2224, 14; + @%p1513 bra $L__BB0_1176; + + setp.ne.s32 %p1514, %r2224, 15; + @%p1514 bra $L__BB0_1183; + + mov.u32 %r6424, 819200; + bra.uni $L__BB0_1184; + +$L__BB0_1179: + mov.u32 %r6424, 13056; + bra.uni $L__BB0_1184; + +$L__BB0_1181: + mov.u32 %r6424, 816; + bra.uni $L__BB0_1184; + +$L__BB0_1177: + mov.u32 %r6424, 208896; + bra.uni $L__BB0_1184; + +$L__BB0_1182: + mov.u32 %r6424, 236; + bra.uni $L__BB0_1184; + +$L__BB0_1178: + mov.u32 %r6424, 60416; + bra.uni $L__BB0_1184; + +$L__BB0_1180: + mov.u32 %r6424, 3776; + bra.uni $L__BB0_1184; + +$L__BB0_1176: + mov.u32 %r6424, 966656; + bra.uni $L__BB0_1184; + +$L__BB0_1183: + mov.u32 %r6424, 0; + +$L__BB0_1184: + not.b32 %r5001, %r6415; + and.b32 %r5002, %r2183, %r5001; + and.b32 %r5003, %r5002, %r6424; + or.b32 %r6416, %r5003, %r6416; + +$L__BB0_1185: + setp.ne.s32 %p1533, %r6416, 0; + @%p1533 bra $L__BB0_1138; + + setp.eq.s32 %p1534, %r6417, 0; + mov.u32 %r6437, 0; + @%p1534 bra $L__BB0_1195; + + mov.u32 %r6430, %r6417; + +$L__BB0_1188: + setp.eq.s32 %p1535, %r6430, 0; + mov.u32 %r6437, %r6417; + @%p1535 bra $L__BB0_1195; + + brev.b32 %r5005, %r6430; + bfind.shiftamt.u32 %r5006, %r5005; + mov.pred %p1635, -1; + mov.u32 %r5007, -2; + shf.l.wrap.b32 %r5008, %r5007, %r5007, %r5006; + and.b32 %r6430, %r6430, %r5008; + shr.u32 %r5009, %r5006, 2; + and.b32 %r5010, %r5006, 3; + add.s32 %r5011, %r5010, %r6326; + add.s32 %r5012, %r5009, %r6330; + mad.lo.s32 %r5013, %r5011, %r1, %r5012; + mul.wide.u32 %rd595, %r5013, 4; + add.s64 %rd596, %rd2, %rd595; + ld.global.u32 %r5014, [%rd596]; + shr.u32 %r5015, %r5014, 31; + setp.eq.s32 %p1537, %r6436, 0; + selp.b32 %r5016, 8, 7, %p1537; + shl.b32 %r5017, %r5015, %r6432; + cvt.u16.u32 %rs564, %r5017; + or.b16 %rs755, %rs755, %rs564; + add.s32 %r6432, %r6432, 1; + setp.lt.u32 %p1538, %r6432, %r5016; + mov.pred %p1634, %p1635; + @%p1538 bra $L__BB0_1194; + + setp.ge.u32 %p1540, %r6434, %r6308; + mov.pred %p1634, 0; + @%p1540 bra $L__BB0_1194; + + setp.eq.s64 %p1541, %rd36, 0; + @%p1541 bra $L__BB0_1193; + + cvt.u64.u32 %rd597, %r6434; + add.s64 %rd598, %rd597, %rd35; + add.s64 %rd599, %rd1, %rd598; + st.global.u8 [%rd599], %rs755; + +$L__BB0_1193: + and.b16 %rs566, %rs755, 255; + setp.eq.s16 %p1543, %rs566, 255; + selp.u32 %r6436, 1, 0, %p1543; + add.s32 %r6434, %r6434, 1; + mov.u16 %rs755, 0; + mov.u32 %r6432, 0; + mov.pred %p1634, %p1635; + +$L__BB0_1194: + @%p1634 bra $L__BB0_1188; + bra.uni $L__BB0_1202; + +$L__BB0_1195: + not.b32 %r5019, %r6437; + and.b32 %r5020, %r2216, %r5019; + setp.ne.s32 %p1546, %r5020, 0; + mov.pred %p1635, %p1182; + @%p1546 bra $L__BB0_1202; + + setp.lt.u32 %p1547, %r4648, %r5; + or.b32 %r5021, %r6437, %r2180; + st.local.u16 [%rd37], %r5021; + shr.u32 %r5022, %r5021, 16; + st.local.u16 [%rd37+2], %r5022; + shl.b32 %r5023, %r5021, 1; + and.b32 %r5024, %r5023, 57344; + and.b32 %r5025, %r5021, 57344; + shr.u32 %r5026, %r5025, 1; + or.b32 %r5027, %r5021, %r2181; + and.b32 %r5028, %r5027, 61440; + or.b32 %r5029, %r5028, %r5024; + or.b32 %r6331, %r5029, %r5026; + mov.u32 %r6330, %r4648; + @%p1547 bra $L__BB0_956; + +$L__BB0_1197: + add.s32 %r6326, %r6326, 4; + setp.gt.u32 %p1548, %r6, %r6326; + @%p1548 bra $L__BB0_954; + + setp.eq.s32 %p1550, %r6432, 0; + mov.pred %p1549, 0; + mov.pred %p1635, %p1549; + @%p1550 bra $L__BB0_1202; + + setp.ge.u32 %p1552, %r6434, %r6308; + mov.pred %p1635, %p1182; + @%p1552 bra $L__BB0_1202; + + setp.eq.s64 %p1554, %rd36, 0; + mov.pred %p1635, %p1549; + @%p1554 bra $L__BB0_1202; + + cvt.u64.u32 %rd600, %r6434; + add.s64 %rd601, %rd600, %rd35; + add.s64 %rd602, %rd1, %rd601; + st.global.u8 [%rd602], %rs755; + mov.pred %p1635, %p1549; + +$L__BB0_1202: + @%p1635 bra $L__BB0_1246; + bra.uni $L__BB0_1203; + +$L__BB0_1246: + mov.u32 %r5083, 2; + st.global.u32 [%rd3], %r5083; + mov.u32 %r5084, 6; + st.global.u32 [%rd3+4], %r5084; + mov.u32 %r5085, 0; + st.global.u32 [%rd3+8], %r5085; + st.global.u32 [%rd3+12], %r5085; + st.global.u32 [%rd3+16], %r5085; + st.global.u32 [%rd3+20], %r5085; + st.global.u32 [%rd3+24], %r5085; + st.global.u32 [%rd3+28], %r5085; + bra.uni $L__BB0_1254; + +$L__BB0_1203: + cvt.u64.u32 %rd603, %r6308; + add.s64 %rd54, %rd603, %rd35; + setp.eq.s32 %p1556, %r6309, 0; + @%p1556 bra $L__BB0_1244; + + add.s32 %r5031, %r6309, -1; + and.b32 %r6445, %r6309, 3; + setp.lt.u32 %p1557, %r5031, 3; + mov.u32 %r6443, 0; + @%p1557 bra $L__BB0_1207; + + sub.s32 %r6442, %r6309, %r6445; + mov.u32 %r6443, 0; + +$L__BB0_1206: + cvt.u64.u32 %rd604, %r6443; + add.s64 %rd605, %rd54, %rd604; + add.s64 %rd606, %rd1, %rd605; + mov.u16 %rs567, 0; + st.global.u8 [%rd606], %rs567; + st.global.u8 [%rd606+1], %rs567; + st.global.u8 [%rd606+2], %rs567; + st.global.u8 [%rd606+3], %rs567; + add.s32 %r6443, %r6443, 4; + add.s32 %r6442, %r6442, -4; + setp.ne.s32 %p1558, %r6442, 0; + @%p1558 bra $L__BB0_1206; + +$L__BB0_1207: + setp.eq.s32 %p1559, %r6445, 0; + @%p1559 bra $L__BB0_1209; + +$L__BB0_1208: + .pragma "nounroll"; + cvt.u64.u32 %rd607, %r6443; + add.s64 %rd608, %rd54, %rd607; + add.s64 %rd609, %rd1, %rd608; + mov.u16 %rs568, 0; + st.global.u8 [%rd609], %rs568; + add.s32 %r6443, %r6443, 1; + add.s32 %r6445, %r6445, -1; + setp.ne.s32 %p1560, %r6445, 0; + @%p1560 bra $L__BB0_1208; + +$L__BB0_1209: + @%p9 bra $L__BB0_1237; + + mov.u32 %r5037, 0; + mov.u32 %r6472, 1; + mov.u16 %rs762, 0; + mov.u32 %r6446, %r5037; + mov.u32 %r6471, %r5037; + mov.u32 %r6470, %r5037; + mov.u32 %r6469, %r5037; + +$L__BB0_1211: + mul.lo.s32 %r2277, %r6446, %r1; + add.s32 %r2278, %r6446, 3; + mul.lo.s32 %r2279, %r1, %r2278; + add.s32 %r2280, %r6446, 2; + mul.lo.s32 %r2281, %r1, %r2280; + add.s32 %r2282, %r6446, 1; + mul.lo.s32 %r2283, %r1, %r2282; + mov.u32 %r6451, %r5037; + +$L__BB0_1212: + add.s32 %r6459, %r2279, %r6451; + add.s32 %r6458, %r2281, %r6451; + add.s32 %r6457, %r2283, %r6451; + add.s32 %r6456, %r2277, %r6451; + mov.u32 %r6460, 0; + +$L__BB0_1213: + add.s32 %r5040, %r6451, %r6460; + setp.ge.u32 %p1562, %r5040, %r5; + @%p1562 bra $L__BB0_1234; + + setp.ge.u32 %p1563, %r6446, %r6; + @%p1563 bra $L__BB0_1219; + + mul.wide.u32 %rd610, %r6456, 4; + add.s64 %rd611, %rd2, %rd610; + ld.global.u32 %r5041, [%rd611]; + abs.s32 %r2302, %r5041; + setp.lt.u32 %p1564, %r2302, 5; + and.b32 %r5042, %r2302, 1; + setp.eq.b32 %p1565, %r5042, 1; + not.pred %p1566, %p1565; + or.pred %p1567, %p1564, %p1566; + @%p1567 bra $L__BB0_1219; + + shr.u32 %r5043, %r2302, 1; + and.b32 %r5044, %r5043, 1; + shl.b32 %r5045, %r5044, %r6469; + cvt.u16.u32 %rs570, %r5045; + or.b16 %rs762, %rs762, %rs570; + add.s32 %r6471, %r6471, 1; + add.s32 %r6469, %r6469, 1; + setp.ne.s32 %p1568, %r6469, 7; + setp.eq.s32 %p1569, %r6472, 0; + or.pred %p1570, %p1568, %p1569; + and.b16 %rs571, %rs762, 127; + setp.ne.s16 %p1571, %rs571, 127; + or.pred %p1572, %p1570, %p1571; + setp.ne.s32 %p1573, %r6469, 8; + and.pred %p1574, %p1573, %p1572; + @%p1574 bra $L__BB0_1219; + + setp.ge.u32 %p1575, %r6470, %r6309; + @%p1575 bra $L__BB0_1245; + + not.b32 %r5047, %r6470; + add.s32 %r5048, %r6309, %r5047; + cvt.u64.u32 %rd612, %r5048; + add.s64 %rd613, %rd54, %rd612; + add.s64 %rd614, %rd1, %rd613; + and.b16 %rs573, %rs762, 255; + st.global.u8 [%rd614], %rs762; + add.s32 %r6470, %r6470, 1; + setp.gt.u16 %p1576, %rs573, 143; + selp.u32 %r6472, 1, 0, %p1576; + mov.u16 %rs762, 0; + mov.u32 %r6469, 0; + +$L__BB0_1219: + setp.ge.u32 %p1577, %r2282, %r6; + @%p1577 bra $L__BB0_1224; + + mul.wide.u32 %rd615, %r6457, 4; + add.s64 %rd616, %rd2, %rd615; + ld.global.u32 %r5049, [%rd616]; + abs.s32 %r2311, %r5049; + setp.lt.u32 %p1578, %r2311, 5; + and.b32 %r5050, %r2311, 1; + setp.eq.b32 %p1579, %r5050, 1; + not.pred %p1580, %p1579; + or.pred %p1581, %p1578, %p1580; + @%p1581 bra $L__BB0_1224; + + shr.u32 %r5051, %r2311, 1; + and.b32 %r5052, %r5051, 1; + shl.b32 %r5053, %r5052, %r6469; + cvt.u16.u32 %rs574, %r5053; + or.b16 %rs762, %rs762, %rs574; + add.s32 %r6471, %r6471, 1; + add.s32 %r6469, %r6469, 1; + setp.ne.s32 %p1582, %r6469, 7; + setp.eq.s32 %p1583, %r6472, 0; + or.pred %p1584, %p1582, %p1583; + and.b16 %rs575, %rs762, 127; + setp.ne.s16 %p1585, %rs575, 127; + or.pred %p1586, %p1584, %p1585; + setp.ne.s32 %p1587, %r6469, 8; + and.pred %p1588, %p1587, %p1586; + @%p1588 bra $L__BB0_1224; + + setp.ge.u32 %p1589, %r6470, %r6309; + @%p1589 bra $L__BB0_1245; + + not.b32 %r5055, %r6470; + add.s32 %r5056, %r6309, %r5055; + cvt.u64.u32 %rd617, %r5056; + add.s64 %rd618, %rd54, %rd617; + add.s64 %rd619, %rd1, %rd618; + and.b16 %rs577, %rs762, 255; + st.global.u8 [%rd619], %rs762; + add.s32 %r6470, %r6470, 1; + setp.gt.u16 %p1590, %rs577, 143; + selp.u32 %r6472, 1, 0, %p1590; + mov.u16 %rs762, 0; + mov.u32 %r6469, 0; + +$L__BB0_1224: + setp.ge.u32 %p1591, %r2280, %r6; + @%p1591 bra $L__BB0_1229; + + mul.wide.u32 %rd620, %r6458, 4; + add.s64 %rd621, %rd2, %rd620; + ld.global.u32 %r5057, [%rd621]; + abs.s32 %r2320, %r5057; + setp.lt.u32 %p1592, %r2320, 5; + and.b32 %r5058, %r2320, 1; + setp.eq.b32 %p1593, %r5058, 1; + not.pred %p1594, %p1593; + or.pred %p1595, %p1592, %p1594; + @%p1595 bra $L__BB0_1229; + + shr.u32 %r5059, %r2320, 1; + and.b32 %r5060, %r5059, 1; + shl.b32 %r5061, %r5060, %r6469; + cvt.u16.u32 %rs578, %r5061; + or.b16 %rs762, %rs762, %rs578; + add.s32 %r6471, %r6471, 1; + add.s32 %r6469, %r6469, 1; + setp.ne.s32 %p1596, %r6469, 7; + setp.eq.s32 %p1597, %r6472, 0; + or.pred %p1598, %p1596, %p1597; + and.b16 %rs579, %rs762, 127; + setp.ne.s16 %p1599, %rs579, 127; + or.pred %p1600, %p1598, %p1599; + setp.ne.s32 %p1601, %r6469, 8; + and.pred %p1602, %p1601, %p1600; + @%p1602 bra $L__BB0_1229; + + setp.ge.u32 %p1603, %r6470, %r6309; + @%p1603 bra $L__BB0_1245; + + not.b32 %r5063, %r6470; + add.s32 %r5064, %r6309, %r5063; + cvt.u64.u32 %rd622, %r5064; + add.s64 %rd623, %rd54, %rd622; + add.s64 %rd624, %rd1, %rd623; + and.b16 %rs581, %rs762, 255; + st.global.u8 [%rd624], %rs762; + add.s32 %r6470, %r6470, 1; + setp.gt.u16 %p1604, %rs581, 143; + selp.u32 %r6472, 1, 0, %p1604; + mov.u16 %rs762, 0; + mov.u32 %r6469, 0; + +$L__BB0_1229: + setp.ge.u32 %p1605, %r2278, %r6; + @%p1605 bra $L__BB0_1234; + + mul.wide.u32 %rd625, %r6459, 4; + add.s64 %rd626, %rd2, %rd625; + ld.global.u32 %r5065, [%rd626]; + abs.s32 %r2329, %r5065; + setp.lt.u32 %p1606, %r2329, 5; + and.b32 %r5066, %r2329, 1; + setp.eq.b32 %p1607, %r5066, 1; + not.pred %p1608, %p1607; + or.pred %p1609, %p1606, %p1608; + @%p1609 bra $L__BB0_1234; + + shr.u32 %r5067, %r2329, 1; + and.b32 %r5068, %r5067, 1; + shl.b32 %r5069, %r5068, %r6469; + cvt.u16.u32 %rs582, %r5069; + or.b16 %rs762, %rs762, %rs582; + add.s32 %r6471, %r6471, 1; + add.s32 %r6469, %r6469, 1; + setp.ne.s32 %p1610, %r6469, 7; + setp.eq.s32 %p1611, %r6472, 0; + or.pred %p1612, %p1610, %p1611; + and.b16 %rs583, %rs762, 127; + setp.ne.s16 %p1613, %rs583, 127; + or.pred %p1614, %p1612, %p1613; + setp.ne.s32 %p1615, %r6469, 8; + and.pred %p1616, %p1615, %p1614; + @%p1616 bra $L__BB0_1234; + + setp.ge.u32 %p1617, %r6470, %r6309; + @%p1617 bra $L__BB0_1245; + + not.b32 %r5071, %r6470; + add.s32 %r5072, %r6309, %r5071; + cvt.u64.u32 %rd627, %r5072; + add.s64 %rd628, %rd54, %rd627; + add.s64 %rd629, %rd1, %rd628; + and.b16 %rs585, %rs762, 255; + st.global.u8 [%rd629], %rs762; + add.s32 %r6470, %r6470, 1; + setp.gt.u16 %p1618, %rs585, 143; + selp.u32 %r6472, 1, 0, %p1618; + mov.u16 %rs762, 0; + mov.u32 %r6469, 0; + +$L__BB0_1234: + add.s32 %r6459, %r6459, 1; + add.s32 %r6458, %r6458, 1; + add.s32 %r6457, %r6457, 1; + add.s32 %r6456, %r6456, 1; + add.s32 %r6460, %r6460, 1; + setp.lt.u32 %p1619, %r6460, 8; + @%p1619 bra $L__BB0_1213; + + add.s32 %r6451, %r6451, 8; + setp.lt.u32 %p1620, %r6451, %r5; + @%p1620 bra $L__BB0_1212; + + add.s32 %r6446, %r6446, 4; + setp.lt.u32 %p1621, %r6446, %r6; + @%p1621 bra $L__BB0_1211; + bra.uni $L__BB0_1239; + +$L__BB0_1244: + setp.eq.s32 %p1628, %r5180, 0; + @%p1628 bra $L__BB0_1243; + bra.uni $L__BB0_1245; + +$L__BB0_1237: + mov.u32 %r6469, 0; + mov.u32 %r6481, %r6469; + +$L__BB0_1238: + add.s32 %r6481, %r6481, 4; + setp.lt.u32 %p1622, %r6481, %r6; + mov.u16 %rs762, 0; + mov.u32 %r6470, %r6469; + mov.u32 %r6471, %r6469; + @%p1622 bra $L__BB0_1238; + +$L__BB0_1239: + setp.eq.s32 %p1623, %r6469, 0; + @%p1623 bra $L__BB0_1242; + + setp.ge.u32 %p1624, %r6470, %r6309; + @%p1624 bra $L__BB0_1245; + + not.b32 %r5077, %r6470; + add.s32 %r5078, %r6309, %r5077; + cvt.u64.u32 %rd630, %r5078; + add.s64 %rd631, %rd54, %rd630; + add.s64 %rd632, %rd1, %rd631; + st.global.u8 [%rd632], %rs762; + add.s32 %r6470, %r6470, 1; + +$L__BB0_1242: + setp.le.u32 %p1625, %r6470, %r6309; + setp.eq.s32 %p1626, %r6471, %r5180; + and.pred %p1627, %p1626, %p1625; + @%p1627 bra $L__BB0_1243; + bra.uni $L__BB0_1245; + +$L__BB0_1243: + sub.s32 %r5109, %r2, %r4; + mov.u32 %r5082, 0; + st.global.u32 [%rd3], %r5082; + st.global.u32 [%rd3+4], %r5082; + st.global.u32 [%rd3+8], %r1984; + st.global.u32 [%rd3+12], %r4; + st.global.u32 [%rd3+16], %r5109; + st.global.u32 [%rd3+20], %r1733; + st.global.u32 [%rd3+24], %r6310; + st.global.u32 [%rd3+28], %r5082; + bra.uni $L__BB0_1254; + +$L__BB0_1245: + mov.u32 %r5079, 2; + st.global.u32 [%rd3], %r5079; + mov.u32 %r5080, 7; + st.global.u32 [%rd3+4], %r5080; + mov.u32 %r5081, 0; + st.global.u32 [%rd3+8], %r5081; + st.global.u32 [%rd3+12], %r5081; + st.global.u32 [%rd3+16], %r5081; + st.global.u32 [%rd3+20], %r5081; + st.global.u32 [%rd3+24], %r5081; + st.global.u32 [%rd3+28], %r5081; + bra.uni $L__BB0_1254; + +} + // .globl j2k_htj2k_encode_codeblocks +.visible .entry j2k_htj2k_encode_codeblocks( + .param .u64 j2k_htj2k_encode_codeblocks_param_0, + .param .u64 j2k_htj2k_encode_codeblocks_param_1, + .param .u64 j2k_htj2k_encode_codeblocks_param_2, + .param .u64 j2k_htj2k_encode_codeblocks_param_3, + .param .u64 j2k_htj2k_encode_codeblocks_param_4, + .param .u64 j2k_htj2k_encode_codeblocks_param_5, + .param .u64 j2k_htj2k_encode_codeblocks_param_6, + .param .u64 j2k_htj2k_encode_codeblocks_param_7 +) +{ + .local .align 2 .b8 __local_depot1[1026]; + .reg .b64 %SP; + .reg .b64 %SPL; + .reg .pred %p<2374>; + .reg .b16 %rs<1376>; + .reg .b32 %r<10744>; + .reg .b64 %rd<1425>; + // demoted variable + .shared .align 4 .b8 _ZZ32 j2k_htj2k_encode_codeblocksE9block_max[512]; + // demoted variable + .shared .align 1 .b8 _ZZ32 j2k_htj2k_encode_codeblocksE13cleanup_e_val[513]; + // demoted variable + .shared .align 1 .b8 _ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val[513]; + + mov.u64 %SPL, __local_depot1; + ld.param.u64 %rd78, [ j2k_htj2k_encode_codeblocks_param_1]; + ld.param.u64 %rd84, [ j2k_htj2k_encode_codeblocks_param_7]; + cvta.to.global.u64 %rd1, %rd78; + mov.u32 %r4049, %ctaid.x; + cvt.u64.u32 %rd2, %r4049; + setp.ge.u64 %p9, %rd2, %rd84; + @%p9 bra $L__BB1_1905; + + ld.param.u64 %rd1424, [ j2k_htj2k_encode_codeblocks_param_2]; + ld.param.u64 %rd1423, [ j2k_htj2k_encode_codeblocks_param_0]; + cvta.to.global.u64 %rd3, %rd1423; + cvta.to.global.u64 %rd85, %rd1424; + shl.b64 %rd86, %rd2, 5; + add.s64 %rd87, %rd85, %rd86; + ld.global.u32 %r1, [%rd87+4]; + ld.global.u32 %r2, [%rd87+16]; + ld.global.u32 %rd4, [%rd87+20]; + ld.global.u32 %r3, [%rd87+24]; + ld.global.u32 %r4, [%rd87+28]; + ld.global.u32 %rd5, [%rd87]; + ld.global.u32 %r5, [%rd87+8]; + setp.eq.s32 %p10, %r5, 0; + ld.global.u32 %r6, [%rd87+12]; + setp.eq.s32 %p11, %r6, 0; + or.pred %p12, %p10, %p11; + @%p12 bra $L__BB1_14; + bra.uni $L__BB1_2; + +$L__BB1_14: + mov.u32 %r8415, 0; + bra.uni $L__BB1_15; + +$L__BB1_2: + setp.eq.s32 %p13, %r1, %r5; + @%p13 bra $L__BB1_6; + bra.uni $L__BB1_3; + +$L__BB1_6: + mov.u32 %r8411, %tid.x; + mul.lo.s32 %r4063, %r6, %r5; + setp.ge.u32 %p16, %r8411, %r4063; + mov.u32 %r8413, 0; + @%p16 bra $L__BB1_9; + + mov.u32 %r8413, 0; + +$L__BB1_8: + cvt.u64.u32 %rd92, %r8411; + add.s64 %rd93, %rd92, %rd5; + shl.b64 %rd94, %rd93, 2; + add.s64 %rd95, %rd3, %rd94; + ld.global.u32 %r4065, [%rd95]; + abs.s32 %r4066, %r4065; + max.u32 %r8413, %r8413, %r4066; + mov.u32 %r4067, %ntid.x; + add.s32 %r8411, %r8411, %r4067; + setp.lt.u32 %p17, %r8411, %r4063; + @%p17 bra $L__BB1_8; + bra.uni $L__BB1_9; + +$L__BB1_3: + mov.u32 %r8409, %tid.x; + mul.lo.s32 %r4052, %r6, %r5; + setp.ge.u32 %p14, %r8409, %r4052; + mov.u32 %r8413, 0; + @%p14 bra $L__BB1_9; + + mov.u32 %r8413, 0; + +$L__BB1_5: + sub.s32 %r4054, %r1, %r5; + div.u32 %r4055, %r8409, %r5; + mad.lo.s32 %r4056, %r4054, %r4055, %r8409; + cvt.u64.u32 %rd88, %r4056; + add.s64 %rd89, %rd88, %rd5; + shl.b64 %rd90, %rd89, 2; + add.s64 %rd91, %rd3, %rd90; + ld.global.u32 %r4057, [%rd91]; + abs.s32 %r4058, %r4057; + max.u32 %r8413, %r8413, %r4058; + mov.u32 %r4059, %ntid.x; + add.s32 %r8409, %r8409, %r4059; + setp.lt.u32 %p15, %r8409, %r4052; + @%p15 bra $L__BB1_5; + +$L__BB1_9: + mov.u32 %r4069, %tid.x; + shl.b32 %r4070, %r4069, 2; + mov.u32 %r4071, _ZZ32 j2k_htj2k_encode_codeblocksE9block_max; + add.s32 %r4072, %r4071, %r4070; + st.shared.u32 [%r4072], %r8413; + bar.sync 0; + mov.u32 %r4073, %ntid.x; + shr.u32 %r8414, %r4073, 1; + setp.eq.s32 %p18, %r8414, 0; + @%p18 bra $L__BB1_13; + +$L__BB1_10: + setp.ge.u32 %p19, %r4069, %r8414; + @%p19 bra $L__BB1_12; + + add.s32 %r4079, %r8414, %r4069; + shl.b32 %r4080, %r8414, 2; + add.s32 %r4081, %r4072, %r4080; + ld.shared.u32 %r4082, [%r4081]; + ld.shared.u32 %r4083, [%r4072]; + setp.gt.u32 %p20, %r4083, %r4082; + selp.b32 %r4084, %r4069, %r4079, %p20; + shl.b32 %r4085, %r4084, 2; + add.s32 %r4086, %r4071, %r4085; + ld.shared.u32 %r4087, [%r4086]; + st.shared.u32 [%r4072], %r4087; + +$L__BB1_12: + bar.sync 0; + shr.u32 %r8414, %r8414, 1; + setp.ne.s32 %p21, %r8414, 0; + @%p21 bra $L__BB1_10; + +$L__BB1_13: + ld.shared.u32 %r8415, [_ZZ32 j2k_htj2k_encode_codeblocksE9block_max]; + +$L__BB1_15: + mov.u32 %r4089, %tid.x; + setp.ne.s32 %p22, %r4089, 0; + @%p22 bra $L__BB1_1905; + + setp.eq.s32 %p2367, %r6, 0; + mov.u32 %r8394, %ctaid.x; + ld.param.u64 %rd1418, [ j2k_htj2k_encode_codeblocks_param_6]; + mov.u32 %r4090, 0; + cvta.to.global.u64 %rd96, %rd1418; + mul.wide.u32 %rd97, %r8394, 32; + add.s64 %rd6, %rd96, %rd97; + mov.u32 %r4092, 1; + st.global.u32 [%rd6], %r4092; + st.global.u32 [%rd6+4], %r4090; + st.global.u32 [%rd6+8], %r4090; + st.global.u32 [%rd6+12], %r4090; + st.global.u32 [%rd6+16], %r4090; + st.global.u32 [%rd6+20], %r4090; + st.global.u32 [%rd6+24], %r4090; + st.global.u32 [%rd6+28], %r4090; + add.s32 %r4093, %r5, -1; + setp.ge.u32 %p24, %r4093, %r1; + or.pred %p25, %p24, %p2367; + setp.gt.u32 %p26, %r5, 1024; + or.pred %p1, %p26, %p25; + mov.u32 %r4095, _ZZ32 j2k_htj2k_encode_codeblocksE13cleanup_e_val; + setp.eq.s32 %p27, %r4, 1; + @%p27 bra $L__BB1_1254; + bra.uni $L__BB1_17; + +$L__BB1_1254: + @%p1 bra $L__BB1_1903; + + cvt.u16.u32 %rs822, %r5; + mov.u16 %rs823, 4096; + div.u16 %rs824, %rs823, %rs822; + cvt.u32.u16 %r6791, %rs824; + setp.gt.u32 %p1625, %r6, %r6791; + add.s32 %r10742, %r2, -1; + setp.gt.u32 %p1626, %r10742, 29; + or.pred %p1627, %p1626, %p1625; + setp.lt.u32 %p1628, %r3, 20549; + or.pred %p1629, %p1628, %p1627; + @%p1629 bra $L__BB1_1903; + bra.uni $L__BB1_1256; + +$L__BB1_1903: + mov.u32 %r8389, 2; + st.global.u32 [%rd6], %r8389; + mov.u32 %r8390, 1; + st.global.u32 [%rd6+4], %r8390; + mov.u32 %r10740, 0; + mov.u32 %r10741, %r10740; + mov.u32 %r10742, %r10740; + mov.u32 %r10743, %r10740; + +$L__BB1_1904: + st.global.u32 [%rd6+8], %r10741; + st.global.u32 [%rd6+12], %r10743; + st.global.u32 [%rd6+16], %r10742; + st.global.u32 [%rd6+20], %r10741; + mov.u32 %r8391, 0; + st.global.u32 [%rd6+24], %r8391; + st.global.u32 [%rd6+28], %r10740; + bra.uni $L__BB1_1905; + +$L__BB1_17: + @%p1 bra $L__BB1_1253; + + cvt.u16.u32 %rs507, %r5; + mov.u16 %rs508, 4096; + div.u16 %rs509, %rs508, %rs507; + cvt.u32.u16 %r4097, %rs509; + setp.gt.u32 %p28, %r6, %r4097; + add.s32 %r4098, %r2, -1; + setp.gt.u32 %p29, %r4098, 29; + or.pred %p30, %p29, %p28; + setp.lt.u32 %p31, %r3, 20549; + or.pred %p32, %p31, %p30; + @%p32 bra $L__BB1_1253; + bra.uni $L__BB1_19; + +$L__BB1_1253: + mov.u32 %r6788, 2; + st.global.u32 [%rd6], %r6788; + mov.u32 %r6789, 1; + st.global.u32 [%rd6+4], %r6789; + mov.u32 %r6790, 0; + st.global.u32 [%rd6+8], %r6790; + st.global.u32 [%rd6+12], %r6790; + st.global.u32 [%rd6+16], %r6790; + st.global.u32 [%rd6+20], %r6790; + st.global.u32 [%rd6+24], %r6790; + st.global.u32 [%rd6+28], %r6790; + +$L__BB1_1905: + ret; + +$L__BB1_1256: + setp.eq.s32 %p1630, %r8415, 0; + @%p1630 bra $L__BB1_1902; + + clz.b32 %r6792, %r8415; + mov.u32 %r6793, 32; + sub.s32 %r6794, %r6793, %r6792; + setp.gt.u32 %p1631, %r6794, %r2; + @%p1631 bra $L__BB1_1901; + bra.uni $L__BB1_1258; + +$L__BB1_1901: + mov.u32 %r8380, 1; + st.global.u32 [%rd6], %r8380; + mov.u32 %r8381, 2; + st.global.u32 [%rd6+4], %r8381; + mov.u32 %r10740, 0; + mov.u32 %r10741, %r10740; + mov.u32 %r10742, %r10740; + mov.u32 %r10743, %r10740; + bra.uni $L__BB1_1904; + +$L__BB1_19: + add.s32 %r4099, %r4, -1; + setp.gt.u32 %p33, %r4099, 163; + @%p33 bra $L__BB1_1252; + bra.uni $L__BB1_20; + +$L__BB1_1252: + mov.u32 %r6785, 2; + st.global.u32 [%rd6], %r6785; + mov.u32 %r6786, 5; + st.global.u32 [%rd6+4], %r6786; + mov.u32 %r6787, 0; + st.global.u32 [%rd6+8], %r6787; + st.global.u32 [%rd6+12], %r6787; + st.global.u32 [%rd6+16], %r6787; + st.global.u32 [%rd6+20], %r6787; + st.global.u32 [%rd6+24], %r6787; + st.global.u32 [%rd6+28], %r6787; + bra.uni $L__BB1_1905; + +$L__BB1_20: + setp.gt.u32 %p34, %r4, 3; + @%p34 bra $L__BB1_1251; + bra.uni $L__BB1_21; + +$L__BB1_1251: + mov.u32 %r6782, 2; + st.global.u32 [%rd6], %r6782; + mov.u32 %r6783, 5; + st.global.u32 [%rd6+4], %r6783; + mov.u32 %r6784, 0; + st.global.u32 [%rd6+8], %r6784; + st.global.u32 [%rd6+12], %r6784; + st.global.u32 [%rd6+16], %r6784; + st.global.u32 [%rd6+20], %r6784; + st.global.u32 [%rd6+24], %r6784; + st.global.u32 [%rd6+28], %r6784; + bra.uni $L__BB1_1905; + +$L__BB1_1902: + mov.u32 %r10740, 0; + st.global.u32 [%rd6], %r10740; + st.global.u32 [%rd6+4], %r10740; + mov.u32 %r10741, %r10740; + mov.u32 %r10742, %r2; + mov.u32 %r10743, %r10740; + bra.uni $L__BB1_1904; + +$L__BB1_1258: + add.s32 %r8403, %r5, 1; + shr.u32 %r8402, %r8403, 1; + add.s64 %rd1415, %rd1, %rd4; + add.s64 %rd1414, %rd1415, 20548; + mov.u32 %r6796, 31; + sub.s32 %r2355, %r6796, %r2; + mov.u16 %rs825, 255; + st.global.u8 [%rd1414], %rs825; + add.s32 %r6797, %r8402, 2; + min.u32 %r2356, %r6797, 513; + mov.u32 %r6798, -3; + sub.s32 %r6799, %r6798, %r8402; + max.u32 %r6800, %r6799, -514; + mov.u32 %r6801, -2; + sub.s32 %r6802, %r6801, %r6800; + and.b32 %r9733, %r2356, 3; + setp.lt.u32 %p1632, %r6802, 3; + mov.u32 %r9731, 0; + @%p1632 bra $L__BB1_1261; + + sub.s32 %r9730, %r2356, %r9733; + mov.u32 %r9731, 0; + +$L__BB1_1260: + add.s32 %r6805, %r4095, %r9731; + mov.u16 %rs826, 0; + st.shared.u8 [%r6805], %rs826; + mov.u32 %r6806, _ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val; + add.s32 %r6807, %r6806, %r9731; + st.shared.u8 [%r6807], %rs826; + st.shared.u8 [%r6805+1], %rs826; + st.shared.u8 [%r6807+1], %rs826; + st.shared.u8 [%r6805+2], %rs826; + st.shared.u8 [%r6807+2], %rs826; + st.shared.u8 [%r6805+3], %rs826; + st.shared.u8 [%r6807+3], %rs826; + add.s32 %r9731, %r9731, 4; + add.s32 %r9730, %r9730, -4; + setp.ne.s32 %p1633, %r9730, 0; + @%p1633 bra $L__BB1_1260; + +$L__BB1_1261: + setp.eq.s32 %p1634, %r9733, 0; + @%p1634 bra $L__BB1_1264; + + mov.u32 %r6810, _ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val; + +$L__BB1_1263: + .pragma "nounroll"; + add.s32 %r6809, %r4095, %r9731; + mov.u16 %rs827, 0; + st.shared.u8 [%r6809], %rs827; + add.s32 %r6811, %r6810, %r9731; + st.shared.u8 [%r6811], %rs827; + add.s32 %r9731, %r9731, 1; + add.s32 %r9733, %r9733, -1; + setp.ne.s32 %p1635, %r9733, 0; + @%p1635 bra $L__BB1_1263; + +$L__BB1_1264: + mov.u32 %r10451, 0; + mov.u32 %r10264, 1; + mov.u16 %rs1253, 0; + mov.u32 %r10452, 8; + mov.u16 %rs1322, 15; + mov.u32 %r10265, 4; + mov.u32 %r10453, %r10451; + mov.u32 %r10454, %r10451; + mov.u32 %r10485, %r10451; + mov.u32 %r10266, %r10264; + mov.u32 %r10267, %r10451; + mov.u32 %r9816, %r10451; + mov.u32 %r9825, %r10452; + mov.u32 %r10030, %r10451; + mov.u32 %r10031, %r10451; + mov.u32 %r10032, %r10264; + mov.u32 %r10033, %r10451; + @%p10 bra $L__BB1_1632; + + ld.param.u64 %rd1421, [ j2k_htj2k_encode_codeblocks_param_5]; + ld.param.u64 %rd1416, [ j2k_htj2k_encode_codeblocks_param_3]; + cvta.to.global.u64 %rd62, %rd1416; + cvta.to.global.u64 %rd63, %rd1421; + mov.u32 %r6844, 0; + mov.u32 %r9825, 8; + mov.u32 %r10032, 1; + mov.u32 %r10265, 4; + mov.u16 %rs1322, 15; + mov.u16 %rs1253, 0; + mov.u32 %r9734, %r6844; + mov.u32 %r10033, %r6844; + mov.u32 %r10031, %r6844; + mov.u32 %r10030, %r6844; + mov.u32 %r9816, %r6844; + mov.u32 %r10267, %r6844; + mov.u32 %r10266, %r10032; + mov.u32 %r10264, %r10032; + mov.u32 %r10485, %r6844; + mov.u32 %r10454, %r6844; + mov.u32 %r10453, %r6844; + mov.u32 %r10452, %r9825; + mov.u32 %r10451, %r6844; + mov.u32 %r9750, %r6844; + mov.u32 %r9751, %r6844; + bra.uni $L__BB1_1266; + +$L__BB1_21: + setp.eq.s32 %p35, %r8415, 0; + @%p35 bra $L__BB1_1250; + + clz.b32 %r4100, %r8415; + mov.u32 %r4101, 32; + sub.s32 %r4102, %r4101, %r4100; + setp.gt.u32 %p36, %r4102, %r2; + @%p36 bra $L__BB1_1249; + bra.uni $L__BB1_23; + +$L__BB1_1249: + mov.u32 %r6778, 1; + st.global.u32 [%rd6], %r6778; + mov.u32 %r6779, 2; + st.global.u32 [%rd6+4], %r6779; + mov.u32 %r6780, 0; + st.global.u32 [%rd6+8], %r6780; + st.global.u32 [%rd6+12], %r6780; + st.global.u32 [%rd6+16], %r6780; + st.global.u32 [%rd6+20], %r6780; + st.global.u32 [%rd6+24], %r6780; + st.global.u32 [%rd6+28], %r6780; + bra.uni $L__BB1_1905; + +$L__BB1_1432: + setp.gt.u32 %p1816, %r9816, 191; + mov.u32 %r10019, 1; + mov.u32 %r9825, 0; + @%p1816 bra $L__BB1_1434; + + st.global.u8 [%rd67], %rs1253; + add.s32 %r9816, %r9816, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9825, 8; + mov.u32 %r10019, %r10033; + bra.uni $L__BB1_1434; + +$L__BB1_1337: + setp.gt.u32 %p1714, %r9816, 191; + mov.u32 %r9833, 1; + mov.u32 %r9825, 0; + @%p1714 bra $L__BB1_1339; + + and.b16 %rs865, %rs1253, 255; + st.global.u8 [%rd64], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1715, %rs865, 255; + selp.b32 %r9825, 7, 8, %p1715; + mov.u16 %rs1253, 0; + mov.u32 %r9833, %r10033; + bra.uni $L__BB1_1339; + +$L__BB1_1471: + setp.gt.u32 %p1863, %r9816, 191; + mov.u32 %r10026, 1; + mov.u32 %r9825, 0; + @%p1863 bra $L__BB1_1473; + + and.b16 %rs914, %rs1253, 255; + st.global.u8 [%rd67], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1864, %rs914, 255; + selp.b32 %r9825, 7, 8, %p1864; + mov.u16 %rs1253, 0; + mov.u32 %r10026, %r10033; + bra.uni $L__BB1_1473; + +$L__BB1_1266: + cvt.u64.u32 %rd1066, %r9751; + add.s64 %rd1067, %rd1066, %rd5; + shl.b64 %rd1068, %rd1067, 2; + add.s64 %rd1069, %rd3, %rd1068; + ld.global.u32 %r2386, [%rd1069]; + setp.eq.s32 %p1637, %r2386, 0; + mov.u32 %r9752, %r6844; + @%p1637 bra $L__BB1_1268; + + and.b32 %r6846, %r2386, -2147483648; + abs.s32 %r6847, %r2386; + shl.b32 %r6848, %r6847, %r2355; + or.b32 %r9752, %r6848, %r6846; + +$L__BB1_1268: + shl.b32 %r6852, %r9752, 1; + shr.u32 %r6853, %r6852, %r2355; + and.b32 %r2389, %r6853, -2; + setp.eq.s32 %p1638, %r2389, 0; + mov.u32 %r9756, 0; + mov.u32 %r9753, %r9756; + mov.u32 %r9754, %r9756; + mov.u32 %r9760, %r9756; + @%p1638 bra $L__BB1_1270; + + add.s32 %r6855, %r2389, -1; + clz.b32 %r6856, %r6855; + mov.u32 %r6857, 32; + sub.s32 %r9753, %r6857, %r6856; + shr.u32 %r6858, %r9752, 31; + add.s32 %r6859, %r6858, %r2389; + add.s32 %r9754, %r6859, -2; + mov.u32 %r9760, 1; + +$L__BB1_1270: + setp.lt.u32 %p1639, %r6, 2; + @%p1639 bra $L__BB1_1273; + + add.s32 %r6862, %r9751, %r1; + cvt.u64.u32 %rd1070, %r6862; + add.s64 %rd1071, %rd1070, %rd5; + shl.b64 %rd1072, %rd1071, 2; + add.s64 %rd1073, %rd3, %rd1072; + ld.global.u32 %r2395, [%rd1073]; + setp.eq.s32 %p1640, %r2395, 0; + @%p1640 bra $L__BB1_1273; + + and.b32 %r6863, %r2395, -2147483648; + abs.s32 %r6864, %r2395; + shl.b32 %r6865, %r6864, %r2355; + or.b32 %r9756, %r6865, %r6863; + +$L__BB1_1273: + shl.b32 %r6868, %r9756, 1; + shr.u32 %r6869, %r6868, %r2355; + and.b32 %r2398, %r6869, -2; + setp.eq.s32 %p1641, %r2398, 0; + mov.u32 %r9771, 0; + mov.u32 %r9757, %r9771; + mov.u32 %r9758, %r9771; + mov.u32 %r9776, %r9753; + @%p1641 bra $L__BB1_1275; + + or.b32 %r9760, %r9760, 2; + add.s32 %r6870, %r2398, -1; + clz.b32 %r6871, %r6870; + mov.u32 %r6872, 32; + sub.s32 %r9757, %r6872, %r6871; + max.s32 %r9776, %r9753, %r9757; + shr.u32 %r6873, %r9756, 31; + add.s32 %r6874, %r6873, %r2398; + add.s32 %r9758, %r6874, -2; + +$L__BB1_1275: + add.s32 %r9775, %r9751, 1; + add.s32 %r6879, %r9734, 1; + setp.ge.u32 %p1642, %r6879, %r5; + mov.u32 %r9772, %r9771; + mov.u32 %r9773, %r9771; + mov.u32 %r9774, %r9771; + @%p1642 bra $L__BB1_1286; + + cvt.u64.u32 %rd1074, %r9775; + add.s64 %rd1075, %rd1074, %rd5; + shl.b64 %rd1076, %rd1075, 2; + add.s64 %rd1077, %rd3, %rd1076; + ld.global.u32 %r2408, [%rd1077]; + setp.eq.s32 %p1643, %r2408, 0; + mov.u32 %r9772, 0; + mov.u32 %r9761, %r9772; + @%p1643 bra $L__BB1_1278; + + and.b32 %r6881, %r2408, -2147483648; + abs.s32 %r6882, %r2408; + shl.b32 %r6883, %r6882, %r2355; + or.b32 %r9761, %r6883, %r6881; + +$L__BB1_1278: + shl.b32 %r6886, %r9761, 1; + shr.u32 %r6887, %r6886, %r2355; + and.b32 %r2411, %r6887, -2; + setp.eq.s32 %p1644, %r2411, 0; + mov.u32 %r9774, %r9772; + @%p1644 bra $L__BB1_1280; + + or.b32 %r9760, %r9760, 4; + add.s32 %r6888, %r2411, -1; + clz.b32 %r6889, %r6888; + mov.u32 %r6890, 32; + sub.s32 %r9772, %r6890, %r6889; + max.s32 %r9776, %r9776, %r9772; + shr.u32 %r6891, %r9761, 31; + add.s32 %r6892, %r6891, %r2411; + add.s32 %r9774, %r6892, -2; + +$L__BB1_1280: + mov.u32 %r9771, 0; + mov.u32 %r9766, %r9771; + @%p1639 bra $L__BB1_1283; + + add.s32 %r6895, %r9775, %r1; + cvt.u64.u32 %rd1078, %r6895; + add.s64 %rd1079, %rd1078, %rd5; + shl.b64 %rd1080, %rd1079, 2; + add.s64 %rd1081, %rd3, %rd1080; + ld.global.u32 %r2420, [%rd1081]; + setp.eq.s32 %p1646, %r2420, 0; + @%p1646 bra $L__BB1_1283; + + and.b32 %r6896, %r2420, -2147483648; + abs.s32 %r6897, %r2420; + shl.b32 %r6898, %r6897, %r2355; + or.b32 %r9766, %r6898, %r6896; + +$L__BB1_1283: + shl.b32 %r6901, %r9766, 1; + shr.u32 %r6902, %r6901, %r2355; + and.b32 %r2423, %r6902, -2; + setp.eq.s32 %p1647, %r2423, 0; + mov.u32 %r9773, %r9771; + @%p1647 bra $L__BB1_1285; + + or.b32 %r9760, %r9760, 8; + add.s32 %r6903, %r2423, -1; + clz.b32 %r6904, %r6903; + mov.u32 %r6905, 32; + sub.s32 %r9771, %r6905, %r6904; + max.s32 %r9776, %r9776, %r9771; + shr.u32 %r6906, %r9766, 31; + add.s32 %r6907, %r6906, %r2423; + add.s32 %r9773, %r6907, -2; + +$L__BB1_1285: + add.s32 %r9775, %r9751, 2; + +$L__BB1_1286: + mov.u32 %r9751, %r9775; + add.s32 %r6909, %r9776, -1; + setp.lt.s32 %p1648, %r9776, 2; + setp.gt.s32 %p1649, %r9776, 1; + selp.b32 %r2440, %r6909, 0, %p1649; + mov.u32 %r9778, 0; + @%p1648 bra $L__BB1_1288; + + setp.eq.s32 %p1650, %r9753, %r9776; + selp.u32 %r6910, 1, 0, %p1650; + setp.eq.s32 %p1651, %r9757, %r9776; + selp.u32 %r6911, -1, 0, %p1651; + bfi.b32 %r6912, %r6911, %r6910, 1, 1; + setp.eq.s32 %p1652, %r9772, %r9776; + selp.u16 %rs832, 1, 0, %p1652; + mul.wide.u16 %r6913, %rs832, 4; + or.b32 %r6914, %r6912, %r6913; + setp.eq.s32 %p1653, %r9771, %r9776; + selp.u16 %rs833, 1, 0, %p1653; + mul.wide.u16 %r6915, %rs833, 8; + or.b32 %r9778, %r6914, %r6915; + +$L__BB1_1288: + shr.u32 %r6916, %r9734, 1; + add.s32 %r2443, %r4095, %r6916; + ld.shared.u8 %rs834, [%r2443]; + cvt.u32.u16 %r6918, %rs834; + and.b32 %r6919, %r6918, 255; + and.b32 %r6920, %r9757, 255; + setp.lt.u32 %p1654, %r6920, %r6919; + cvt.u16.u32 %rs835, %r9757; + selp.b16 %rs836, %rs834, %rs835, %p1654; + st.shared.u8 [%r2443], %rs836; + cvt.u16.u32 %rs274, %r9771; + st.shared.u8 [%r2443+1], %rs274; + and.b32 %r2444, %r9760, 2; + cvt.u16.u32 %rs837, %r2444; + shr.u16 %rs838, %rs837, 1; + mov.u32 %r6921, _ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val; + add.s32 %r2445, %r6921, %r6916; + ld.shared.u8 %rs839, [%r2445]; + or.b16 %rs840, %rs839, %rs838; + st.shared.u8 [%r2445], %rs840; + and.b32 %r2446, %r9760, 8; + shr.u32 %r2447, %r2446, 3; + st.shared.u8 [%r2445+1], %r2447; + shl.b32 %r6922, %r9760, 4; + shl.b32 %r6923, %r9750, 8; + or.b32 %r6924, %r6922, %r6923; + or.b32 %r6925, %r6924, %r9778; + mul.wide.u32 %rd1082, %r6925, 2; + add.s64 %rd1083, %rd62, %rd1082; + ld.global.u16 %rs275, [%rd1083]; + shr.u16 %rs841, %rs275, 4; + and.b16 %rs276, %rs841, 7; + setp.eq.s16 %p1655, %rs276, 0; + mov.u32 %r9790, %r10267; + @%p1655 bra $L__BB1_1295; + + cvt.u32.u16 %r9779, %rs276; + shr.u16 %rs842, %rs275, 8; + cvt.u32.u16 %r9780, %rs842; + +$L__BB1_1290: + mov.u32 %r2450, %r9779; + setp.gt.u32 %p1656, %r10264, 2879; + mov.u32 %r9790, 1; + @%p1656 bra $L__BB1_1295; + + mov.u32 %r6927, 8; + sub.s32 %r6928, %r6927, %r10266; + sub.s32 %r6929, %r6928, %r10265; + min.u32 %r6930, %r6929, %r2450; + setp.eq.s32 %p1657, %r6930, 32; + mov.u32 %r6931, -1; + shl.b32 %r6932, %r6931, %r6930; + not.b32 %r6933, %r6932; + selp.b32 %r6934, -1, %r6933, %p1657; + and.b32 %r6935, %r6934, %r9780; + shl.b32 %r6936, %r6935, %r10265; + cvt.u16.u32 %rs843, %r6936; + or.b16 %rs1322, %rs1322, %rs843; + add.s32 %r10265, %r6930, %r10265; + sub.s32 %r9779, %r2450, %r6930; + shr.u32 %r9780, %r9780, %r6930; + setp.gt.u32 %p1658, %r6929, %r2450; + @%p1658 bra $L__BB1_1294; + + setp.ne.s32 %p1659, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs844, %rs1322, 255; + setp.ne.s16 %p1660, %rs844, 127; + and.pred %p1661, %p1659, %p1660; + @%p1661 bra $L__BB1_1294; + + mov.u32 %r6939, 20548; + sub.s32 %r6940, %r6939, %r10264; + cvt.u64.u32 %rd1084, %r6940; + add.s64 %rd1085, %rd1084, %rd4; + add.s64 %rd1086, %rd1, %rd1085; + st.global.u8 [%rd1086], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p1662, %rs844, 143; + selp.u32 %r10266, 1, 0, %p1662; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1294: + setp.ne.s32 %p1663, %r9779, 0; + mov.u32 %r9790, %r10267; + @%p1663 bra $L__BB1_1290; + +$L__BB1_1295: + setp.ne.s32 %p1664, %r9750, 0; + @%p1664 bra $L__BB1_1343; + + setp.eq.s32 %p1665, %r9760, 0; + add.s32 %r6941, %r9816, 17477; + cvt.u64.u32 %rd1087, %r6941; + add.s64 %rd1088, %rd1087, %rd4; + add.s64 %rd64, %rd1, %rd1088; + @%p1665 bra $L__BB1_1335; + + shl.b16 %rs1253, %rs1253, 1; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1666, %r9825, 0; + mov.u32 %r9826, %r10033; + @%p1666 bra $L__BB1_1300; + + setp.gt.u32 %p1667, %r9816, 191; + mov.u32 %r9826, 1; + mov.u32 %r9825, 0; + @%p1667 bra $L__BB1_1300; + + st.global.u8 [%rd64], %rs1253; + add.s32 %r9816, %r9816, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9825, 8; + mov.u32 %r9826, %r10033; + +$L__BB1_1300: + setp.lt.u32 %p1668, %r10031, 3; + mov.u32 %r9794, 0; + @%p1668 bra $L__BB1_1303; + + setp.lt.u32 %p1669, %r10031, 6; + mov.u32 %r9794, 1; + @%p1669 bra $L__BB1_1303; + + setp.lt.u32 %p1670, %r10031, 9; + setp.eq.s32 %p1671, %r10031, 11; + selp.b32 %r6947, 4, 5, %p1671; + setp.lt.u32 %p1672, %r10031, 11; + selp.b32 %r6948, 3, %r6947, %p1672; + selp.b32 %r9794, 2, %r6948, %p1670; + +$L__BB1_1303: + setp.eq.s32 %p1673, %r9794, 0; + @%p1673 bra $L__BB1_1331; + + add.s32 %r2474, %r9794, -1; + and.b32 %r2475, %r9794, 3; + setp.eq.s32 %p1674, %r2475, 0; + mov.u32 %r9804, %r9794; + mov.u32 %r9805, %r9826; + @%p1674 bra $L__BB1_1316; + + mov.u32 %r6950, 1; + shl.b32 %r6951, %r6950, %r2474; + and.b32 %r6952, %r6951, %r10030; + setp.ne.s32 %p1675, %r6952, 0; + selp.u32 %r6953, 1, 0, %p1675; + cvt.u32.u16 %r6954, %rs1253; + bfi.b32 %r6955, %r6954, %r6953, 1, 8; + cvt.u16.u32 %rs1253, %r6955; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1676, %r9825, 0; + mov.u32 %r9805, %r9826; + @%p1676 bra $L__BB1_1308; + + setp.gt.u32 %p1677, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r9805, %r6950; + @%p1677 bra $L__BB1_1308; + + add.s32 %r6959, %r9816, 17477; + cvt.u64.u32 %rd1089, %r6959; + add.s64 %rd1090, %rd1089, %rd4; + add.s64 %rd1091, %rd1, %rd1090; + st.global.u8 [%rd1091], %rs1253; + add.s32 %r9816, %r9816, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9825, 8; + mov.u32 %r9805, %r9826; + +$L__BB1_1308: + setp.eq.s32 %p1678, %r2475, 1; + mov.u32 %r9826, %r9805; + mov.u32 %r9804, %r2474; + @%p1678 bra $L__BB1_1316; + + add.s32 %r9804, %r9794, -2; + mov.u32 %r6960, 1; + shl.b32 %r6961, %r6960, %r9804; + and.b32 %r6962, %r6961, %r10030; + setp.ne.s32 %p1679, %r6962, 0; + selp.u32 %r6963, 1, 0, %p1679; + cvt.u32.u16 %r6964, %rs1253; + bfi.b32 %r6965, %r6964, %r6963, 1, 8; + cvt.u16.u32 %rs1253, %r6965; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1680, %r9825, 0; + mov.u32 %r9800, %r9805; + @%p1680 bra $L__BB1_1312; + + setp.gt.u32 %p1681, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r9800, %r6960; + @%p1681 bra $L__BB1_1312; + + add.s32 %r6968, %r9816, 17477; + cvt.u64.u32 %rd1092, %r6968; + add.s64 %rd1093, %rd1092, %rd4; + add.s64 %rd1094, %rd1, %rd1093; + and.b16 %rs851, %rs1253, 255; + st.global.u8 [%rd1094], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1682, %rs851, 255; + selp.b32 %r9825, 7, 8, %p1682; + mov.u16 %rs1253, 0; + mov.u32 %r9800, %r9805; + +$L__BB1_1312: + setp.eq.s32 %p1683, %r2475, 2; + mov.u32 %r9826, %r9800; + mov.u32 %r9805, %r9800; + @%p1683 bra $L__BB1_1316; + + add.s32 %r9804, %r9794, -3; + mov.u32 %r6969, 1; + shl.b32 %r6970, %r6969, %r9804; + and.b32 %r6971, %r6970, %r10030; + setp.ne.s32 %p1684, %r6971, 0; + selp.u32 %r6972, 1, 0, %p1684; + cvt.u32.u16 %r6973, %rs1253; + bfi.b32 %r6974, %r6973, %r6972, 1, 8; + cvt.u16.u32 %rs1253, %r6974; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1685, %r9825, 0; + mov.u32 %r9826, %r9800; + mov.u32 %r9805, %r9800; + @%p1685 bra $L__BB1_1316; + + setp.gt.u32 %p1686, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r9826, %r6969; + mov.u32 %r9805, %r6969; + @%p1686 bra $L__BB1_1316; + + add.s32 %r6979, %r9816, 17477; + cvt.u64.u32 %rd1095, %r6979; + add.s64 %rd1096, %rd1095, %rd4; + add.s64 %rd1097, %rd1, %rd1096; + and.b16 %rs854, %rs1253, 255; + st.global.u8 [%rd1097], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1687, %rs854, 255; + selp.b32 %r9825, 7, 8, %p1687; + mov.u16 %rs1253, 0; + mov.u32 %r9826, %r9800; + mov.u32 %r9805, %r9800; + +$L__BB1_1316: + setp.lt.u32 %p1688, %r2474, 3; + @%p1688 bra $L__BB1_1331; + + mov.u32 %r9826, %r9805; + +$L__BB1_1318: + add.s32 %r6980, %r9804, -1; + mov.u32 %r6981, 1; + shl.b32 %r6982, %r6981, %r6980; + and.b32 %r6983, %r6982, %r10030; + setp.ne.s32 %p1689, %r6983, 0; + selp.u32 %r6984, 1, 0, %p1689; + cvt.u32.u16 %r6985, %rs1253; + bfi.b32 %r9814, %r6985, %r6984, 1, 8; + add.s32 %r9813, %r9825, -1; + setp.ne.s32 %p1690, %r9813, 0; + mov.u32 %r9815, %r9826; + @%p1690 bra $L__BB1_1321; + + setp.gt.u32 %p1691, %r9816, 191; + mov.u32 %r9813, 0; + mov.u32 %r9815, %r6981; + @%p1691 bra $L__BB1_1321; + + cvt.u16.u32 %rs855, %r9814; + and.b16 %rs856, %rs855, 255; + add.s32 %r6989, %r9816, 17477; + cvt.u64.u32 %rd1098, %r6989; + add.s64 %rd1099, %rd1098, %rd4; + add.s64 %rd1100, %rd1, %rd1099; + st.global.u8 [%rd1100], %rs855; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1692, %rs856, 255; + selp.b32 %r9813, 7, 8, %p1692; + mov.u32 %r9814, 0; + mov.u32 %r9815, %r9826; + +$L__BB1_1321: + add.s32 %r6990, %r9804, -2; + shl.b32 %r6992, %r6981, %r6990; + and.b32 %r6993, %r6992, %r10030; + setp.ne.s32 %p1693, %r6993, 0; + and.b32 %r6994, %r9814, 127; + selp.u32 %r6995, 1, 0, %p1693; + bfi.b32 %r9818, %r6994, %r6995, 1, 7; + add.s32 %r9817, %r9813, -1; + setp.ne.s32 %p1694, %r9817, 0; + mov.u32 %r9819, %r9815; + @%p1694 bra $L__BB1_1324; + + setp.gt.u32 %p1695, %r9816, 191; + mov.u32 %r9819, 1; + mov.u32 %r9817, 0; + @%p1695 bra $L__BB1_1324; + + cvt.u16.u32 %rs857, %r9818; + and.b16 %rs858, %rs857, 255; + add.s32 %r6999, %r9816, 17477; + cvt.u64.u32 %rd1101, %r6999; + add.s64 %rd1102, %rd1101, %rd4; + add.s64 %rd1103, %rd1, %rd1102; + st.global.u8 [%rd1103], %rs857; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1696, %rs858, 255; + selp.b32 %r9817, 7, 8, %p1696; + mov.u32 %r9818, 0; + mov.u32 %r9819, %r9815; + +$L__BB1_1324: + add.s32 %r7000, %r9804, -3; + mov.u32 %r7001, 1; + shl.b32 %r7002, %r7001, %r7000; + and.b32 %r7003, %r7002, %r10030; + setp.ne.s32 %p1697, %r7003, 0; + and.b32 %r7004, %r9818, 127; + selp.u32 %r7005, 1, 0, %p1697; + bfi.b32 %r9822, %r7004, %r7005, 1, 7; + add.s32 %r9821, %r9817, -1; + setp.ne.s32 %p1698, %r9821, 0; + mov.u32 %r9823, %r9819; + @%p1698 bra $L__BB1_1327; + + setp.gt.u32 %p1699, %r9816, 191; + mov.u32 %r9821, 0; + mov.u32 %r9823, %r7001; + @%p1699 bra $L__BB1_1327; + + cvt.u16.u32 %rs859, %r9822; + and.b16 %rs860, %rs859, 255; + add.s32 %r7009, %r9816, 17477; + cvt.u64.u32 %rd1104, %r7009; + add.s64 %rd1105, %rd1104, %rd4; + add.s64 %rd1106, %rd1, %rd1105; + st.global.u8 [%rd1106], %rs859; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1700, %rs860, 255; + selp.b32 %r9821, 7, 8, %p1700; + mov.u32 %r9822, 0; + mov.u32 %r9823, %r9819; + +$L__BB1_1327: + add.s32 %r9804, %r9804, -4; + shl.b32 %r7011, %r7001, %r9804; + and.b32 %r7012, %r7011, %r10030; + setp.ne.s32 %p1701, %r7012, 0; + and.b32 %r7013, %r9822, 127; + selp.u32 %r7014, 1, 0, %p1701; + bfi.b32 %r7015, %r7013, %r7014, 1, 15; + cvt.u16.u32 %rs1253, %r7015; + add.s32 %r9825, %r9821, -1; + setp.ne.s32 %p1702, %r9825, 0; + mov.u32 %r9826, %r9823; + @%p1702 bra $L__BB1_1330; + + setp.gt.u32 %p1703, %r9816, 191; + mov.u32 %r9826, 1; + mov.u32 %r9825, 0; + @%p1703 bra $L__BB1_1330; + + add.s32 %r7018, %r9816, 17477; + cvt.u64.u32 %rd1107, %r7018; + add.s64 %rd1108, %rd1107, %rd4; + add.s64 %rd1109, %rd1, %rd1108; + and.b16 %rs862, %rs1253, 255; + st.global.u8 [%rd1109], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1704, %rs862, 255; + selp.b32 %r9825, 7, 8, %p1704; + mov.u16 %rs1253, 0; + mov.u32 %r9826, %r9823; + +$L__BB1_1330: + setp.ne.s32 %p1705, %r9804, 0; + @%p1705 bra $L__BB1_1318; + +$L__BB1_1331: + add.s32 %r7020, %r10031, -1; + setp.eq.s32 %p1706, %r10031, 0; + mov.u32 %r10030, 0; + selp.b32 %r10031, 0, %r7020, %p1706; + setp.lt.u32 %p1707, %r10031, 3; + mov.u32 %r9830, %r10030; + @%p1707 bra $L__BB1_1334; + + setp.lt.u32 %p1708, %r10031, 6; + mov.u32 %r9830, 1; + @%p1708 bra $L__BB1_1334; + + setp.lt.u32 %p1709, %r10031, 9; + setp.eq.s32 %p1710, %r10031, 11; + selp.b32 %r7022, 4, 5, %p1710; + setp.lt.u32 %p1711, %r10031, 11; + selp.b32 %r7023, 3, %r7022, %p1711; + selp.b32 %r9830, 2, %r7023, %p1709; + +$L__BB1_1334: + mov.u32 %r7025, 1; + shl.b32 %r10032, %r7025, %r9830; + mov.u32 %r10033, %r9826; + bra.uni $L__BB1_1343; + +$L__BB1_1335: + add.s32 %r10030, %r10030, 1; + setp.lt.u32 %p1712, %r10030, %r10032; + @%p1712 bra $L__BB1_1343; + + shl.b16 %rs863, %rs1253, 1; + or.b16 %rs1253, %rs863, 1; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1713, %r9825, 0; + mov.u32 %r9833, %r10033; + @%p1713 bra $L__BB1_1339; + bra.uni $L__BB1_1337; + +$L__BB1_1339: + add.s32 %r7029, %r10031, 1; + min.u32 %r10031, %r7029, 12; + setp.lt.u32 %p1716, %r10031, 3; + mov.u32 %r10030, 0; + mov.u32 %r9834, %r10030; + @%p1716 bra $L__BB1_1342; + + setp.lt.u32 %p1717, %r10031, 6; + mov.u32 %r9834, 1; + @%p1717 bra $L__BB1_1342; + + setp.lt.u32 %p1718, %r10031, 9; + setp.eq.s32 %p1719, %r10031, 11; + selp.b32 %r7031, 4, 5, %p1719; + setp.lt.u32 %p1720, %r10031, 11; + selp.b32 %r7032, 3, %r7031, %p1720; + selp.b32 %r9834, 2, %r7032, %p1718; + +$L__BB1_1342: + mov.u32 %r7034, 1; + shl.b32 %r10032, %r7034, %r9834; + mov.u32 %r10033, %r9833; + +$L__BB1_1343: + max.s32 %r2558, %r9776, 1; + and.b16 %rs866, %rs275, 15; + cvt.u32.u16 %r2559, %rs866; + and.b32 %r2560, %r9760, 1; + setp.eq.s32 %p1721, %r2560, 0; + mov.u32 %r9855, %r10485; + @%p1721 bra $L__BB1_1350; + + and.b32 %r7035, %r2559, 1; + sub.s32 %r9841, %r2558, %r7035; + setp.eq.s32 %p1722, %r9841, 0; + mov.u32 %r9855, %r10485; + @%p1722 bra $L__BB1_1350; + + mov.u32 %r7036, -1; + shl.b32 %r7037, %r7036, %r9841; + not.b32 %r7038, %r7037; + and.b32 %r9842, %r9754, %r7038; + +$L__BB1_1346: + setp.gt.u32 %p1723, %r10451, 17476; + mov.u32 %r9855, 1; + @%p1723 bra $L__BB1_1350; + + sub.s32 %r7040, %r10452, %r10453; + min.u32 %r7041, %r7040, %r9841; + setp.eq.s32 %p1724, %r7041, 32; + mov.u32 %r7042, -1; + shl.b32 %r7043, %r7042, %r7041; + not.b32 %r7044, %r7043; + selp.b32 %r7045, -1, %r7044, %p1724; + and.b32 %r7046, %r7045, %r9842; + shl.b32 %r7047, %r7046, %r10453; + or.b32 %r10454, %r7047, %r10454; + add.s32 %r10453, %r7041, %r10453; + shr.u32 %r9842, %r9842, %r7041; + sub.s32 %r9841, %r9841, %r7041; + setp.lt.u32 %p1725, %r10453, %r10452; + @%p1725 bra $L__BB1_1349; + + cvt.u64.u32 %rd1110, %r10451; + add.s64 %rd1111, %rd1110, %rd4; + add.s64 %rd1112, %rd1, %rd1111; + st.global.u8 [%rd1112], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p1726, %r10454, 255; + selp.b32 %r10452, 7, 8, %p1726; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1349: + setp.ne.s32 %p1727, %r9841, 0; + mov.u32 %r9855, %r10485; + @%p1727 bra $L__BB1_1346; + +$L__BB1_1350: + setp.eq.s32 %p1728, %r2444, 0; + mov.u32 %r9870, %r9855; + @%p1728 bra $L__BB1_1357; + + shr.u32 %r7050, %r2559, 1; + and.b32 %r7051, %r7050, 1; + sub.s32 %r9856, %r2558, %r7051; + setp.eq.s32 %p1729, %r9856, 0; + mov.u32 %r9870, %r9855; + @%p1729 bra $L__BB1_1357; + + mov.u32 %r7052, -1; + shl.b32 %r7053, %r7052, %r9856; + not.b32 %r7054, %r7053; + and.b32 %r9857, %r9758, %r7054; + +$L__BB1_1353: + setp.gt.u32 %p1730, %r10451, 17476; + mov.u32 %r9870, 1; + @%p1730 bra $L__BB1_1357; + + sub.s32 %r7056, %r10452, %r10453; + min.u32 %r7057, %r7056, %r9856; + setp.eq.s32 %p1731, %r7057, 32; + mov.u32 %r7058, -1; + shl.b32 %r7059, %r7058, %r7057; + not.b32 %r7060, %r7059; + selp.b32 %r7061, -1, %r7060, %p1731; + and.b32 %r7062, %r7061, %r9857; + shl.b32 %r7063, %r7062, %r10453; + or.b32 %r10454, %r7063, %r10454; + add.s32 %r10453, %r7057, %r10453; + shr.u32 %r9857, %r9857, %r7057; + sub.s32 %r9856, %r9856, %r7057; + setp.lt.u32 %p1732, %r10453, %r10452; + @%p1732 bra $L__BB1_1356; + + cvt.u64.u32 %rd1113, %r10451; + add.s64 %rd1114, %rd1113, %rd4; + add.s64 %rd1115, %rd1, %rd1114; + st.global.u8 [%rd1115], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p1733, %r10454, 255; + selp.b32 %r10452, 7, 8, %p1733; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1356: + setp.ne.s32 %p1734, %r9856, 0; + mov.u32 %r9870, %r9855; + @%p1734 bra $L__BB1_1353; + +$L__BB1_1357: + and.b32 %r7066, %r9760, 4; + setp.eq.s32 %p1735, %r7066, 0; + mov.u32 %r9885, %r9870; + @%p1735 bra $L__BB1_1364; + + shr.u32 %r7067, %r2559, 2; + and.b32 %r7068, %r7067, 1; + sub.s32 %r9871, %r2558, %r7068; + setp.eq.s32 %p1736, %r9871, 0; + mov.u32 %r9885, %r9870; + @%p1736 bra $L__BB1_1364; + + mov.u32 %r7069, -1; + shl.b32 %r7070, %r7069, %r9871; + not.b32 %r7071, %r7070; + and.b32 %r9872, %r9774, %r7071; + +$L__BB1_1360: + setp.gt.u32 %p1737, %r10451, 17476; + mov.u32 %r9885, 1; + @%p1737 bra $L__BB1_1364; + + sub.s32 %r7073, %r10452, %r10453; + min.u32 %r7074, %r7073, %r9871; + setp.eq.s32 %p1738, %r7074, 32; + mov.u32 %r7075, -1; + shl.b32 %r7076, %r7075, %r7074; + not.b32 %r7077, %r7076; + selp.b32 %r7078, -1, %r7077, %p1738; + and.b32 %r7079, %r7078, %r9872; + shl.b32 %r7080, %r7079, %r10453; + or.b32 %r10454, %r7080, %r10454; + add.s32 %r10453, %r7074, %r10453; + shr.u32 %r9872, %r9872, %r7074; + sub.s32 %r9871, %r9871, %r7074; + setp.lt.u32 %p1739, %r10453, %r10452; + @%p1739 bra $L__BB1_1363; + + cvt.u64.u32 %rd1116, %r10451; + add.s64 %rd1117, %rd1116, %rd4; + add.s64 %rd1118, %rd1, %rd1117; + st.global.u8 [%rd1118], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p1740, %r10454, 255; + selp.b32 %r10452, 7, 8, %p1740; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1363: + setp.ne.s32 %p1741, %r9871, 0; + mov.u32 %r9885, %r9870; + @%p1741 bra $L__BB1_1360; + +$L__BB1_1364: + setp.eq.s32 %p1742, %r2446, 0; + mov.u32 %r10485, %r9885; + @%p1742 bra $L__BB1_1371; + + shr.u32 %r7083, %r2559, 3; + sub.s32 %r9886, %r2558, %r7083; + setp.eq.s32 %p1743, %r9886, 0; + mov.u32 %r10485, %r9885; + @%p1743 bra $L__BB1_1371; + + mov.u32 %r7084, -1; + shl.b32 %r7085, %r7084, %r9886; + not.b32 %r7086, %r7085; + and.b32 %r9887, %r9773, %r7086; + +$L__BB1_1367: + setp.gt.u32 %p1744, %r10451, 17476; + mov.u32 %r10485, 1; + @%p1744 bra $L__BB1_1371; + + sub.s32 %r7088, %r10452, %r10453; + min.u32 %r7089, %r7088, %r9886; + setp.eq.s32 %p1745, %r7089, 32; + mov.u32 %r7090, -1; + shl.b32 %r7091, %r7090, %r7089; + not.b32 %r7092, %r7091; + selp.b32 %r7093, -1, %r7092, %p1745; + and.b32 %r7094, %r7093, %r9887; + shl.b32 %r7095, %r7094, %r10453; + or.b32 %r10454, %r7095, %r10454; + add.s32 %r10453, %r7089, %r10453; + shr.u32 %r9887, %r9887, %r7089; + sub.s32 %r9886, %r9886, %r7089; + setp.lt.u32 %p1746, %r10453, %r10452; + @%p1746 bra $L__BB1_1370; + + cvt.u64.u32 %rd1119, %r10451; + add.s64 %rd1120, %rd1119, %rd4; + add.s64 %rd1121, %rd1, %rd1120; + st.global.u8 [%rd1121], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p1747, %r10454, 255; + selp.b32 %r10452, 7, 8, %p1747; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1370: + setp.ne.s32 %p1748, %r9886, 0; + mov.u32 %r10485, %r9885; + @%p1748 bra $L__BB1_1367; + +$L__BB1_1371: + add.s32 %r7098, %r9734, 2; + setp.lt.u32 %p1749, %r7098, %r5; + mul.lo.s32 %r2653, %r2440, 6; + cvt.u64.u32 %rd1122, %r2653; + add.s64 %rd65, %rd63, %rd1122; + add.s32 %r7099, %r2653, 2; + cvt.u64.u32 %rd1123, %r7099; + add.s64 %rd66, %rd63, %rd1123; + @%p1749 bra $L__BB1_1400; + bra.uni $L__BB1_1372; + +$L__BB1_1400: + cvt.u64.u32 %rd1137, %r9751; + add.s64 %rd1138, %rd1137, %rd5; + shl.b64 %rd1139, %rd1138, 2; + add.s64 %rd1140, %rd3, %rd1139; + ld.global.u32 %r2726, [%rd1140]; + setp.eq.s32 %p1786, %r2726, 0; + mov.u32 %r9946, 0; + mov.u32 %r9945, %r9946; + @%p1786 bra $L__BB1_1402; + + and.b32 %r7170, %r2726, -2147483648; + abs.s32 %r7171, %r2726; + shl.b32 %r7172, %r7171, %r2355; + or.b32 %r9945, %r7172, %r7170; + +$L__BB1_1402: + shl.b32 %r7176, %r9945, 1; + shr.u32 %r7177, %r7176, %r2355; + and.b32 %r2729, %r7177, -2; + setp.eq.s32 %p1787, %r2729, 0; + mov.u32 %r9947, %r9946; + mov.u32 %r9953, %r9946; + @%p1787 bra $L__BB1_1404; + + add.s32 %r7179, %r2729, -1; + clz.b32 %r7180, %r7179; + mov.u32 %r7181, 32; + sub.s32 %r9946, %r7181, %r7180; + shr.u32 %r7182, %r9945, 31; + add.s32 %r7183, %r7182, %r2729; + add.s32 %r9947, %r7183, -2; + mov.u32 %r9953, 1; + +$L__BB1_1404: + mov.u32 %r9950, 0; + mov.u32 %r9949, %r9950; + @%p1639 bra $L__BB1_1407; + + add.s32 %r7186, %r9751, %r1; + cvt.u64.u32 %rd1141, %r7186; + add.s64 %rd1142, %rd1141, %rd5; + shl.b64 %rd1143, %rd1142, 2; + add.s64 %rd1144, %rd3, %rd1143; + ld.global.u32 %r2735, [%rd1144]; + setp.eq.s32 %p1789, %r2735, 0; + @%p1789 bra $L__BB1_1407; + + and.b32 %r7187, %r2735, -2147483648; + abs.s32 %r7188, %r2735; + shl.b32 %r7189, %r7188, %r2355; + or.b32 %r9949, %r7189, %r7187; + +$L__BB1_1407: + shl.b32 %r7192, %r9949, 1; + shr.u32 %r7193, %r7192, %r2355; + and.b32 %r2738, %r7193, -2; + setp.eq.s32 %p1790, %r2738, 0; + mov.u32 %r9951, %r9950; + mov.u32 %r9969, %r9946; + @%p1790 bra $L__BB1_1409; + + or.b32 %r9953, %r9953, 2; + add.s32 %r7194, %r2738, -1; + clz.b32 %r7195, %r7194; + mov.u32 %r7196, 32; + sub.s32 %r9950, %r7196, %r7195; + max.s32 %r9969, %r9946, %r9950; + shr.u32 %r7197, %r9949, 31; + add.s32 %r7198, %r7197, %r2738; + add.s32 %r9951, %r7198, -2; + +$L__BB1_1409: + add.s32 %r9968, %r9751, 1; + add.s32 %r7203, %r9734, 3; + setp.ge.u32 %p1791, %r7203, %r5; + mov.u32 %r9971, 0; + mov.u32 %r9964, %r9971; + mov.u32 %r9965, %r9971; + mov.u32 %r9966, %r9971; + mov.u32 %r9967, %r9971; + @%p1791 bra $L__BB1_1420; + + cvt.u64.u32 %rd1145, %r9968; + add.s64 %rd1146, %rd1145, %rd5; + shl.b64 %rd1147, %rd1146, 2; + add.s64 %rd1148, %rd3, %rd1147; + ld.global.u32 %r2748, [%rd1148]; + setp.eq.s32 %p1792, %r2748, 0; + mov.u32 %r9965, 0; + mov.u32 %r9954, %r9965; + @%p1792 bra $L__BB1_1412; + + and.b32 %r7205, %r2748, -2147483648; + abs.s32 %r7206, %r2748; + shl.b32 %r7207, %r7206, %r2355; + or.b32 %r9954, %r7207, %r7205; + +$L__BB1_1412: + shl.b32 %r7210, %r9954, 1; + shr.u32 %r7211, %r7210, %r2355; + and.b32 %r2751, %r7211, -2; + setp.eq.s32 %p1793, %r2751, 0; + mov.u32 %r9967, %r9965; + @%p1793 bra $L__BB1_1414; + + or.b32 %r9953, %r9953, 4; + add.s32 %r7212, %r2751, -1; + clz.b32 %r7213, %r7212; + mov.u32 %r7214, 32; + sub.s32 %r9965, %r7214, %r7213; + max.s32 %r9969, %r9969, %r9965; + shr.u32 %r7215, %r9954, 31; + add.s32 %r7216, %r7215, %r2751; + add.s32 %r9967, %r7216, -2; + +$L__BB1_1414: + mov.u32 %r9964, 0; + mov.u32 %r9959, %r9964; + @%p1639 bra $L__BB1_1417; + + add.s32 %r7219, %r9968, %r1; + cvt.u64.u32 %rd1149, %r7219; + add.s64 %rd1150, %rd1149, %rd5; + shl.b64 %rd1151, %rd1150, 2; + add.s64 %rd1152, %rd3, %rd1151; + ld.global.u32 %r2760, [%rd1152]; + setp.eq.s32 %p1795, %r2760, 0; + @%p1795 bra $L__BB1_1417; + + and.b32 %r7220, %r2760, -2147483648; + abs.s32 %r7221, %r2760; + shl.b32 %r7222, %r7221, %r2355; + or.b32 %r9959, %r7222, %r7220; + +$L__BB1_1417: + shl.b32 %r7225, %r9959, 1; + shr.u32 %r7226, %r7225, %r2355; + and.b32 %r2763, %r7226, -2; + setp.eq.s32 %p1796, %r2763, 0; + mov.u32 %r9966, %r9964; + @%p1796 bra $L__BB1_1419; + + or.b32 %r9953, %r9953, 8; + add.s32 %r7227, %r2763, -1; + clz.b32 %r7228, %r7227; + mov.u32 %r7229, 32; + sub.s32 %r9964, %r7229, %r7228; + max.s32 %r9969, %r9969, %r9964; + shr.u32 %r7230, %r9959, 31; + add.s32 %r7231, %r7230, %r2763; + add.s32 %r9966, %r7231, -2; + +$L__BB1_1419: + add.s32 %r9968, %r9751, 2; + +$L__BB1_1420: + mov.u32 %r9751, %r9968; + shr.u32 %r7233, %r9760, 1; + or.b32 %r2780, %r7233, %r2560; + add.s32 %r7234, %r9969, -1; + setp.lt.s32 %p1797, %r9969, 2; + setp.gt.s32 %p1798, %r9969, 1; + selp.b32 %r2781, %r7234, 0, %p1798; + @%p1797 bra $L__BB1_1422; + + setp.eq.s32 %p1799, %r9946, %r9969; + selp.u32 %r7235, 1, 0, %p1799; + setp.eq.s32 %p1800, %r9950, %r9969; + selp.u32 %r7236, -1, 0, %p1800; + bfi.b32 %r7237, %r7236, %r7235, 1, 1; + setp.eq.s32 %p1801, %r9965, %r9969; + selp.u16 %rs886, 1, 0, %p1801; + mul.wide.u16 %r7238, %rs886, 4; + or.b32 %r7239, %r7237, %r7238; + setp.eq.s32 %p1802, %r9964, %r9969; + selp.u16 %rs887, 1, 0, %p1802; + mul.wide.u16 %r7240, %rs887, 8; + or.b32 %r9971, %r7239, %r7240; + +$L__BB1_1422: + and.b32 %r7241, %r9950, 255; + and.b32 %r7242, %r9771, 255; + setp.lt.u32 %p1803, %r7241, %r7242; + cvt.u16.u32 %rs888, %r9950; + selp.b16 %rs889, %rs274, %rs888, %p1803; + st.shared.u8 [%r2443+1], %rs889; + st.shared.u8 [%r2443+2], %r9964; + and.b32 %r2784, %r9953, 2; + shr.u32 %r7243, %r2784, 1; + or.b32 %r7244, %r2447, %r7243; + st.shared.u8 [%r2445+1], %r7244; + and.b32 %r2785, %r9953, 8; + shr.u32 %r7245, %r2785, 3; + st.shared.u8 [%r2445+2], %r7245; + shl.b32 %r7246, %r9953, 4; + shl.b32 %r7247, %r2780, 8; + or.b32 %r7248, %r7246, %r7247; + or.b32 %r7249, %r7248, %r9971; + mul.wide.u32 %rd1154, %r7249, 2; + add.s64 %rd1155, %rd62, %rd1154; + ld.global.u16 %rs319, [%rd1155]; + shr.u16 %rs890, %rs319, 4; + and.b16 %rs320, %rs890, 7; + setp.eq.s16 %p1804, %rs320, 0; + mov.u32 %r9983, %r9790; + @%p1804 bra $L__BB1_1429; + + cvt.u32.u16 %r9972, %rs320; + shr.u16 %rs891, %rs319, 8; + cvt.u32.u16 %r9973, %rs891; + +$L__BB1_1424: + mov.u32 %r2788, %r9972; + setp.gt.u32 %p1805, %r10264, 2879; + mov.u32 %r9983, 1; + @%p1805 bra $L__BB1_1429; + + mov.u32 %r7251, 8; + sub.s32 %r7252, %r7251, %r10266; + sub.s32 %r7253, %r7252, %r10265; + min.u32 %r7254, %r7253, %r2788; + setp.eq.s32 %p1806, %r7254, 32; + mov.u32 %r7255, -1; + shl.b32 %r7256, %r7255, %r7254; + not.b32 %r7257, %r7256; + selp.b32 %r7258, -1, %r7257, %p1806; + and.b32 %r7259, %r7258, %r9973; + shl.b32 %r7260, %r7259, %r10265; + cvt.u16.u32 %rs892, %r7260; + or.b16 %rs1322, %rs1322, %rs892; + add.s32 %r10265, %r7254, %r10265; + sub.s32 %r9972, %r2788, %r7254; + shr.u32 %r9973, %r9973, %r7254; + setp.gt.u32 %p1807, %r7253, %r2788; + @%p1807 bra $L__BB1_1428; + + setp.ne.s32 %p1808, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs893, %rs1322, 255; + setp.ne.s16 %p1809, %rs893, 127; + and.pred %p1810, %p1808, %p1809; + @%p1810 bra $L__BB1_1428; + + mov.u32 %r7263, 20548; + sub.s32 %r7264, %r7263, %r10264; + cvt.u64.u32 %rd1156, %r7264; + add.s64 %rd1157, %rd1156, %rd4; + add.s64 %rd1158, %rd1, %rd1157; + st.global.u8 [%rd1158], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p1811, %rs893, 143; + selp.u32 %r10266, 1, 0, %p1811; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1428: + setp.ne.s32 %p1812, %r9972, 0; + mov.u32 %r9983, %r9790; + @%p1812 bra $L__BB1_1424; + +$L__BB1_1429: + setp.ne.s32 %p1813, %r2780, 0; + @%p1813 bra $L__BB1_1477; + + setp.eq.s32 %p1814, %r9953, 0; + add.s32 %r7265, %r9816, 17477; + cvt.u64.u32 %rd1159, %r7265; + add.s64 %rd1160, %rd1159, %rd4; + add.s64 %rd67, %rd1, %rd1160; + @%p1814 bra $L__BB1_1469; + + shl.b16 %rs1253, %rs1253, 1; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1815, %r9825, 0; + mov.u32 %r10019, %r10033; + @%p1815 bra $L__BB1_1434; + bra.uni $L__BB1_1432; + +$L__BB1_1434: + setp.lt.u32 %p1817, %r10031, 3; + mov.u32 %r9987, 0; + @%p1817 bra $L__BB1_1437; + + setp.lt.u32 %p1818, %r10031, 6; + mov.u32 %r9987, 1; + @%p1818 bra $L__BB1_1437; + + setp.lt.u32 %p1819, %r10031, 9; + setp.eq.s32 %p1820, %r10031, 11; + selp.b32 %r7271, 4, 5, %p1820; + setp.lt.u32 %p1821, %r10031, 11; + selp.b32 %r7272, 3, %r7271, %p1821; + selp.b32 %r9987, 2, %r7272, %p1819; + +$L__BB1_1437: + setp.eq.s32 %p1822, %r9987, 0; + @%p1822 bra $L__BB1_1465; + + add.s32 %r2812, %r9987, -1; + and.b32 %r2813, %r9987, 3; + setp.eq.s32 %p1823, %r2813, 0; + mov.u32 %r9997, %r9987; + mov.u32 %r9998, %r10019; + @%p1823 bra $L__BB1_1450; + + mov.u32 %r7274, 1; + shl.b32 %r7275, %r7274, %r2812; + and.b32 %r7276, %r7275, %r10030; + setp.ne.s32 %p1824, %r7276, 0; + selp.u32 %r7277, 1, 0, %p1824; + cvt.u32.u16 %r7278, %rs1253; + bfi.b32 %r7279, %r7278, %r7277, 1, 8; + cvt.u16.u32 %rs1253, %r7279; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1825, %r9825, 0; + mov.u32 %r9998, %r10019; + @%p1825 bra $L__BB1_1442; + + setp.gt.u32 %p1826, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r9998, %r7274; + @%p1826 bra $L__BB1_1442; + + add.s32 %r7283, %r9816, 17477; + cvt.u64.u32 %rd1161, %r7283; + add.s64 %rd1162, %rd1161, %rd4; + add.s64 %rd1163, %rd1, %rd1162; + st.global.u8 [%rd1163], %rs1253; + add.s32 %r9816, %r9816, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9825, 8; + mov.u32 %r9998, %r10019; + +$L__BB1_1442: + setp.eq.s32 %p1827, %r2813, 1; + mov.u32 %r10019, %r9998; + mov.u32 %r9997, %r2812; + @%p1827 bra $L__BB1_1450; + + add.s32 %r9997, %r9987, -2; + mov.u32 %r7284, 1; + shl.b32 %r7285, %r7284, %r9997; + and.b32 %r7286, %r7285, %r10030; + setp.ne.s32 %p1828, %r7286, 0; + selp.u32 %r7287, 1, 0, %p1828; + cvt.u32.u16 %r7288, %rs1253; + bfi.b32 %r7289, %r7288, %r7287, 1, 8; + cvt.u16.u32 %rs1253, %r7289; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1829, %r9825, 0; + mov.u32 %r9993, %r9998; + @%p1829 bra $L__BB1_1446; + + setp.gt.u32 %p1830, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r9993, %r7284; + @%p1830 bra $L__BB1_1446; + + add.s32 %r7292, %r9816, 17477; + cvt.u64.u32 %rd1164, %r7292; + add.s64 %rd1165, %rd1164, %rd4; + add.s64 %rd1166, %rd1, %rd1165; + and.b16 %rs900, %rs1253, 255; + st.global.u8 [%rd1166], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1831, %rs900, 255; + selp.b32 %r9825, 7, 8, %p1831; + mov.u16 %rs1253, 0; + mov.u32 %r9993, %r9998; + +$L__BB1_1446: + setp.eq.s32 %p1832, %r2813, 2; + mov.u32 %r10019, %r9993; + mov.u32 %r9998, %r9993; + @%p1832 bra $L__BB1_1450; + + add.s32 %r9997, %r9987, -3; + mov.u32 %r7293, 1; + shl.b32 %r7294, %r7293, %r9997; + and.b32 %r7295, %r7294, %r10030; + setp.ne.s32 %p1833, %r7295, 0; + selp.u32 %r7296, 1, 0, %p1833; + cvt.u32.u16 %r7297, %rs1253; + bfi.b32 %r7298, %r7297, %r7296, 1, 8; + cvt.u16.u32 %rs1253, %r7298; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1834, %r9825, 0; + mov.u32 %r10019, %r9993; + mov.u32 %r9998, %r9993; + @%p1834 bra $L__BB1_1450; + + setp.gt.u32 %p1835, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r10019, %r7293; + mov.u32 %r9998, %r7293; + @%p1835 bra $L__BB1_1450; + + add.s32 %r7303, %r9816, 17477; + cvt.u64.u32 %rd1167, %r7303; + add.s64 %rd1168, %rd1167, %rd4; + add.s64 %rd1169, %rd1, %rd1168; + and.b16 %rs903, %rs1253, 255; + st.global.u8 [%rd1169], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1836, %rs903, 255; + selp.b32 %r9825, 7, 8, %p1836; + mov.u16 %rs1253, 0; + mov.u32 %r10019, %r9993; + mov.u32 %r9998, %r9993; + +$L__BB1_1450: + setp.lt.u32 %p1837, %r2812, 3; + @%p1837 bra $L__BB1_1465; + + mov.u32 %r10019, %r9998; + +$L__BB1_1452: + add.s32 %r7304, %r9997, -1; + mov.u32 %r7305, 1; + shl.b32 %r7306, %r7305, %r7304; + and.b32 %r7307, %r7306, %r10030; + setp.ne.s32 %p1838, %r7307, 0; + selp.u32 %r7308, 1, 0, %p1838; + cvt.u32.u16 %r7309, %rs1253; + bfi.b32 %r10007, %r7309, %r7308, 1, 8; + add.s32 %r10006, %r9825, -1; + setp.ne.s32 %p1839, %r10006, 0; + mov.u32 %r10008, %r10019; + @%p1839 bra $L__BB1_1455; + + setp.gt.u32 %p1840, %r9816, 191; + mov.u32 %r10006, 0; + mov.u32 %r10008, %r7305; + @%p1840 bra $L__BB1_1455; + + cvt.u16.u32 %rs904, %r10007; + and.b16 %rs905, %rs904, 255; + add.s32 %r7313, %r9816, 17477; + cvt.u64.u32 %rd1170, %r7313; + add.s64 %rd1171, %rd1170, %rd4; + add.s64 %rd1172, %rd1, %rd1171; + st.global.u8 [%rd1172], %rs904; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1841, %rs905, 255; + selp.b32 %r10006, 7, 8, %p1841; + mov.u32 %r10007, 0; + mov.u32 %r10008, %r10019; + +$L__BB1_1455: + add.s32 %r7314, %r9997, -2; + shl.b32 %r7316, %r7305, %r7314; + and.b32 %r7317, %r7316, %r10030; + setp.ne.s32 %p1842, %r7317, 0; + and.b32 %r7318, %r10007, 127; + selp.u32 %r7319, 1, 0, %p1842; + bfi.b32 %r10011, %r7318, %r7319, 1, 7; + add.s32 %r10010, %r10006, -1; + setp.ne.s32 %p1843, %r10010, 0; + mov.u32 %r10012, %r10008; + @%p1843 bra $L__BB1_1458; + + setp.gt.u32 %p1844, %r9816, 191; + mov.u32 %r10012, 1; + mov.u32 %r10010, 0; + @%p1844 bra $L__BB1_1458; + + cvt.u16.u32 %rs906, %r10011; + and.b16 %rs907, %rs906, 255; + add.s32 %r7323, %r9816, 17477; + cvt.u64.u32 %rd1173, %r7323; + add.s64 %rd1174, %rd1173, %rd4; + add.s64 %rd1175, %rd1, %rd1174; + st.global.u8 [%rd1175], %rs906; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1845, %rs907, 255; + selp.b32 %r10010, 7, 8, %p1845; + mov.u32 %r10011, 0; + mov.u32 %r10012, %r10008; + +$L__BB1_1458: + add.s32 %r7324, %r9997, -3; + mov.u32 %r7325, 1; + shl.b32 %r7326, %r7325, %r7324; + and.b32 %r7327, %r7326, %r10030; + setp.ne.s32 %p1846, %r7327, 0; + and.b32 %r7328, %r10011, 127; + selp.u32 %r7329, 1, 0, %p1846; + bfi.b32 %r10015, %r7328, %r7329, 1, 7; + add.s32 %r10014, %r10010, -1; + setp.ne.s32 %p1847, %r10014, 0; + mov.u32 %r10016, %r10012; + @%p1847 bra $L__BB1_1461; + + setp.gt.u32 %p1848, %r9816, 191; + mov.u32 %r10014, 0; + mov.u32 %r10016, %r7325; + @%p1848 bra $L__BB1_1461; + + cvt.u16.u32 %rs908, %r10015; + and.b16 %rs909, %rs908, 255; + add.s32 %r7333, %r9816, 17477; + cvt.u64.u32 %rd1176, %r7333; + add.s64 %rd1177, %rd1176, %rd4; + add.s64 %rd1178, %rd1, %rd1177; + st.global.u8 [%rd1178], %rs908; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1849, %rs909, 255; + selp.b32 %r10014, 7, 8, %p1849; + mov.u32 %r10015, 0; + mov.u32 %r10016, %r10012; + +$L__BB1_1461: + add.s32 %r9997, %r9997, -4; + shl.b32 %r7335, %r7325, %r9997; + and.b32 %r7336, %r7335, %r10030; + setp.ne.s32 %p1850, %r7336, 0; + and.b32 %r7337, %r10015, 127; + selp.u32 %r7338, 1, 0, %p1850; + bfi.b32 %r7339, %r7337, %r7338, 1, 15; + cvt.u16.u32 %rs1253, %r7339; + add.s32 %r9825, %r10014, -1; + setp.ne.s32 %p1851, %r9825, 0; + mov.u32 %r10019, %r10016; + @%p1851 bra $L__BB1_1464; + + setp.gt.u32 %p1852, %r9816, 191; + mov.u32 %r10019, 1; + mov.u32 %r9825, 0; + @%p1852 bra $L__BB1_1464; + + add.s32 %r7342, %r9816, 17477; + cvt.u64.u32 %rd1179, %r7342; + add.s64 %rd1180, %rd1179, %rd4; + add.s64 %rd1181, %rd1, %rd1180; + and.b16 %rs911, %rs1253, 255; + st.global.u8 [%rd1181], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1853, %rs911, 255; + selp.b32 %r9825, 7, 8, %p1853; + mov.u16 %rs1253, 0; + mov.u32 %r10019, %r10016; + +$L__BB1_1464: + setp.ne.s32 %p1854, %r9997, 0; + @%p1854 bra $L__BB1_1452; + +$L__BB1_1465: + add.s32 %r7344, %r10031, -1; + setp.eq.s32 %p1855, %r10031, 0; + mov.u32 %r10030, 0; + selp.b32 %r10031, 0, %r7344, %p1855; + setp.lt.u32 %p1856, %r10031, 3; + mov.u32 %r10023, %r10030; + @%p1856 bra $L__BB1_1468; + + setp.lt.u32 %p1857, %r10031, 6; + mov.u32 %r10023, 1; + @%p1857 bra $L__BB1_1468; + + setp.lt.u32 %p1858, %r10031, 9; + setp.eq.s32 %p1859, %r10031, 11; + selp.b32 %r7346, 4, 5, %p1859; + setp.lt.u32 %p1860, %r10031, 11; + selp.b32 %r7347, 3, %r7346, %p1860; + selp.b32 %r10023, 2, %r7347, %p1858; + +$L__BB1_1468: + mov.u32 %r7349, 1; + shl.b32 %r10032, %r7349, %r10023; + mov.u32 %r10033, %r10019; + bra.uni $L__BB1_1477; + +$L__BB1_1372: + ld.global.u8 %rs297, [%rd65+1]; + ld.global.u8 %rs298, [%rd66]; + ld.global.u8 %rs299, [%rd66+1]; + ld.global.u8 %rs300, [%rd63]; + ld.global.u8 %rs301, [%rd63+1]; + ld.global.u8 %rs302, [%rd63+2]; + ld.global.u8 %rs303, [%rd63+3]; + setp.eq.s16 %p1750, %rs297, 0; + mov.u32 %r9912, %r9790; + @%p1750 bra $L__BB1_1379; + + ld.global.u8 %r9902, [%rd65]; + cvt.u32.u16 %r9901, %rs297; + +$L__BB1_1374: + mov.u32 %r2656, %r9901; + setp.gt.u32 %p1751, %r10264, 2879; + mov.u32 %r9912, 1; + @%p1751 bra $L__BB1_1379; + + mov.u32 %r7101, 8; + sub.s32 %r7102, %r7101, %r10266; + sub.s32 %r7103, %r7102, %r10265; + min.u32 %r7104, %r7103, %r2656; + setp.eq.s32 %p1752, %r7104, 32; + mov.u32 %r7105, -1; + shl.b32 %r7106, %r7105, %r7104; + not.b32 %r7107, %r7106; + selp.b32 %r7108, -1, %r7107, %p1752; + and.b32 %r7109, %r7108, %r9902; + shl.b32 %r7110, %r7109, %r10265; + cvt.u16.u32 %rs867, %r7110; + or.b16 %rs1322, %rs1322, %rs867; + add.s32 %r10265, %r7104, %r10265; + sub.s32 %r9901, %r2656, %r7104; + shr.u32 %r9902, %r9902, %r7104; + setp.gt.u32 %p1753, %r7103, %r2656; + @%p1753 bra $L__BB1_1378; + + setp.ne.s32 %p1754, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs868, %rs1322, 255; + setp.ne.s16 %p1755, %rs868, 127; + and.pred %p1756, %p1754, %p1755; + @%p1756 bra $L__BB1_1378; + + mov.u32 %r7113, 20548; + sub.s32 %r7114, %r7113, %r10264; + cvt.u64.u32 %rd1125, %r7114; + add.s64 %rd1126, %rd1125, %rd4; + add.s64 %rd1127, %rd1, %rd1126; + st.global.u8 [%rd1127], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p1757, %rs868, 143; + selp.u32 %r10266, 1, 0, %p1757; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1378: + setp.ne.s32 %p1758, %r9901, 0; + mov.u32 %r9912, %r9790; + @%p1758 bra $L__BB1_1374; + +$L__BB1_1379: + setp.eq.s16 %p1759, %rs301, 0; + mov.u32 %r9924, %r9912; + @%p1759 bra $L__BB1_1386; + + cvt.u32.u16 %r7115, %rs300; + and.b32 %r9914, %r7115, 255; + cvt.u32.u16 %r7116, %rs301; + and.b32 %r9913, %r7116, 255; + +$L__BB1_1381: + mov.u32 %r2675, %r9913; + setp.gt.u32 %p1760, %r10264, 2879; + mov.u32 %r9924, 1; + @%p1760 bra $L__BB1_1386; + + mov.u32 %r7118, 8; + sub.s32 %r7119, %r7118, %r10266; + sub.s32 %r7120, %r7119, %r10265; + min.u32 %r7121, %r7120, %r2675; + setp.eq.s32 %p1761, %r7121, 32; + mov.u32 %r7122, -1; + shl.b32 %r7123, %r7122, %r7121; + not.b32 %r7124, %r7123; + selp.b32 %r7125, -1, %r7124, %p1761; + and.b32 %r7126, %r7125, %r9914; + shl.b32 %r7127, %r7126, %r10265; + cvt.u16.u32 %rs872, %r7127; + or.b16 %rs1322, %rs1322, %rs872; + add.s32 %r10265, %r7121, %r10265; + sub.s32 %r9913, %r2675, %r7121; + shr.u32 %r9914, %r9914, %r7121; + setp.gt.u32 %p1762, %r7120, %r2675; + @%p1762 bra $L__BB1_1385; + + setp.ne.s32 %p1763, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs873, %rs1322, 255; + setp.ne.s16 %p1764, %rs873, 127; + and.pred %p1765, %p1763, %p1764; + @%p1765 bra $L__BB1_1385; + + mov.u32 %r7130, 20548; + sub.s32 %r7131, %r7130, %r10264; + cvt.u64.u32 %rd1128, %r7131; + add.s64 %rd1129, %rd1128, %rd4; + add.s64 %rd1130, %rd1, %rd1129; + st.global.u8 [%rd1130], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p1766, %rs873, 143; + selp.u32 %r10266, 1, 0, %p1766; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1385: + setp.ne.s32 %p1767, %r9913, 0; + mov.u32 %r9924, %r9912; + @%p1767 bra $L__BB1_1381; + +$L__BB1_1386: + setp.eq.s16 %p1768, %rs299, 0; + mov.u32 %r9936, %r9924; + @%p1768 bra $L__BB1_1393; + + cvt.u32.u16 %r7132, %rs299; + and.b32 %r9925, %r7132, 255; + cvt.u32.u16 %r7133, %rs298; + and.b32 %r9926, %r7133, 255; + +$L__BB1_1388: + mov.u32 %r2694, %r9925; + setp.gt.u32 %p1769, %r10264, 2879; + mov.u32 %r9936, 1; + @%p1769 bra $L__BB1_1393; + + mov.u32 %r7135, 8; + sub.s32 %r7136, %r7135, %r10266; + sub.s32 %r7137, %r7136, %r10265; + min.u32 %r7138, %r7137, %r2694; + setp.eq.s32 %p1770, %r7138, 32; + mov.u32 %r7139, -1; + shl.b32 %r7140, %r7139, %r7138; + not.b32 %r7141, %r7140; + selp.b32 %r7142, -1, %r7141, %p1770; + and.b32 %r7143, %r7142, %r9926; + shl.b32 %r7144, %r7143, %r10265; + cvt.u16.u32 %rs877, %r7144; + or.b16 %rs1322, %rs1322, %rs877; + add.s32 %r10265, %r7138, %r10265; + sub.s32 %r9925, %r2694, %r7138; + shr.u32 %r9926, %r9926, %r7138; + setp.gt.u32 %p1771, %r7137, %r2694; + @%p1771 bra $L__BB1_1392; + + setp.ne.s32 %p1772, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs878, %rs1322, 255; + setp.ne.s16 %p1773, %rs878, 127; + and.pred %p1774, %p1772, %p1773; + @%p1774 bra $L__BB1_1392; + + mov.u32 %r7147, 20548; + sub.s32 %r7148, %r7147, %r10264; + cvt.u64.u32 %rd1131, %r7148; + add.s64 %rd1132, %rd1131, %rd4; + add.s64 %rd1133, %rd1, %rd1132; + st.global.u8 [%rd1133], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p1775, %rs878, 143; + selp.u32 %r10266, 1, 0, %p1775; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1392: + setp.ne.s32 %p1776, %r9925, 0; + mov.u32 %r9936, %r9924; + @%p1776 bra $L__BB1_1388; + +$L__BB1_1393: + setp.eq.s16 %p1777, %rs303, 0; + mov.u32 %r9750, 0; + mov.u32 %r10267, %r9936; + @%p1777 bra $L__BB1_1631; + + cvt.u32.u16 %r7150, %rs302; + and.b32 %r9938, %r7150, 255; + cvt.u32.u16 %r7151, %rs303; + and.b32 %r9937, %r7151, 255; + +$L__BB1_1395: + mov.u32 %r2713, %r9937; + setp.gt.u32 %p1778, %r10264, 2879; + mov.u32 %r10267, 1; + @%p1778 bra $L__BB1_1631; + + mov.u32 %r7154, 8; + sub.s32 %r7155, %r7154, %r10266; + sub.s32 %r7156, %r7155, %r10265; + min.u32 %r7157, %r7156, %r2713; + setp.eq.s32 %p1779, %r7157, 32; + mov.u32 %r7158, -1; + shl.b32 %r7159, %r7158, %r7157; + not.b32 %r7160, %r7159; + selp.b32 %r7161, -1, %r7160, %p1779; + and.b32 %r7162, %r7161, %r9938; + shl.b32 %r7163, %r7162, %r10265; + cvt.u16.u32 %rs882, %r7163; + or.b16 %rs1322, %rs1322, %rs882; + add.s32 %r10265, %r7157, %r10265; + sub.s32 %r9937, %r2713, %r7157; + shr.u32 %r9938, %r9938, %r7157; + setp.gt.u32 %p1780, %r7156, %r2713; + @%p1780 bra $L__BB1_1399; + + setp.ne.s32 %p1781, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs883, %rs1322, 255; + setp.ne.s16 %p1782, %rs883, 127; + and.pred %p1783, %p1781, %p1782; + @%p1783 bra $L__BB1_1399; + + mov.u32 %r7166, 20548; + sub.s32 %r7167, %r7166, %r10264; + cvt.u64.u32 %rd1134, %r7167; + add.s64 %rd1135, %rd1134, %rd4; + add.s64 %rd1136, %rd1, %rd1135; + st.global.u8 [%rd1136], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p1784, %rs883, 143; + selp.u32 %r10266, 1, 0, %p1784; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1399: + setp.eq.s32 %p1785, %r9937, 0; + mov.u32 %r10267, %r9936; + @%p1785 bra $L__BB1_1631; + bra.uni $L__BB1_1395; + +$L__BB1_1469: + add.s32 %r10030, %r10030, 1; + setp.lt.u32 %p1861, %r10030, %r10032; + @%p1861 bra $L__BB1_1477; + + shl.b16 %rs912, %rs1253, 1; + or.b16 %rs1253, %rs912, 1; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1862, %r9825, 0; + mov.u32 %r10026, %r10033; + @%p1862 bra $L__BB1_1473; + bra.uni $L__BB1_1471; + +$L__BB1_1473: + add.s32 %r7353, %r10031, 1; + min.u32 %r10031, %r7353, 12; + setp.lt.u32 %p1865, %r10031, 3; + mov.u32 %r10030, 0; + mov.u32 %r10027, %r10030; + @%p1865 bra $L__BB1_1476; + + setp.lt.u32 %p1866, %r10031, 6; + mov.u32 %r10027, 1; + @%p1866 bra $L__BB1_1476; + + setp.lt.u32 %p1867, %r10031, 9; + setp.eq.s32 %p1868, %r10031, 11; + selp.b32 %r7355, 4, 5, %p1868; + setp.lt.u32 %p1869, %r10031, 11; + selp.b32 %r7356, 3, %r7355, %p1869; + selp.b32 %r10027, 2, %r7356, %p1867; + +$L__BB1_1476: + mov.u32 %r7358, 1; + shl.b32 %r10032, %r7358, %r10027; + mov.u32 %r10033, %r10026; + +$L__BB1_1477: + max.s32 %r2896, %r9969, 1; + and.b16 %rs915, %rs319, 15; + cvt.u32.u16 %r2897, %rs915; + and.b32 %r2898, %r9953, 1; + setp.eq.s32 %p1870, %r2898, 0; + mov.u32 %r10048, %r10485; + @%p1870 bra $L__BB1_1484; + + and.b32 %r7359, %r2897, 1; + sub.s32 %r10034, %r2896, %r7359; + setp.eq.s32 %p1871, %r10034, 0; + mov.u32 %r10048, %r10485; + @%p1871 bra $L__BB1_1484; + + mov.u32 %r7360, -1; + shl.b32 %r7361, %r7360, %r10034; + not.b32 %r7362, %r7361; + and.b32 %r10035, %r9947, %r7362; + +$L__BB1_1480: + setp.gt.u32 %p1872, %r10451, 17476; + mov.u32 %r10048, 1; + @%p1872 bra $L__BB1_1484; + + sub.s32 %r7364, %r10452, %r10453; + min.u32 %r7365, %r7364, %r10034; + setp.eq.s32 %p1873, %r7365, 32; + mov.u32 %r7366, -1; + shl.b32 %r7367, %r7366, %r7365; + not.b32 %r7368, %r7367; + selp.b32 %r7369, -1, %r7368, %p1873; + and.b32 %r7370, %r7369, %r10035; + shl.b32 %r7371, %r7370, %r10453; + or.b32 %r10454, %r7371, %r10454; + add.s32 %r10453, %r7365, %r10453; + shr.u32 %r10035, %r10035, %r7365; + sub.s32 %r10034, %r10034, %r7365; + setp.lt.u32 %p1874, %r10453, %r10452; + @%p1874 bra $L__BB1_1483; + + cvt.u64.u32 %rd1182, %r10451; + add.s64 %rd1183, %rd1182, %rd4; + add.s64 %rd1184, %rd1, %rd1183; + st.global.u8 [%rd1184], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p1875, %r10454, 255; + selp.b32 %r10452, 7, 8, %p1875; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1483: + setp.ne.s32 %p1876, %r10034, 0; + mov.u32 %r10048, %r10485; + @%p1876 bra $L__BB1_1480; + +$L__BB1_1484: + setp.eq.s32 %p1877, %r2784, 0; + mov.u32 %r10063, %r10048; + @%p1877 bra $L__BB1_1491; + + shr.u32 %r7374, %r2897, 1; + and.b32 %r7375, %r7374, 1; + sub.s32 %r10049, %r2896, %r7375; + setp.eq.s32 %p1878, %r10049, 0; + mov.u32 %r10063, %r10048; + @%p1878 bra $L__BB1_1491; + + mov.u32 %r7376, -1; + shl.b32 %r7377, %r7376, %r10049; + not.b32 %r7378, %r7377; + and.b32 %r10050, %r9951, %r7378; + +$L__BB1_1487: + setp.gt.u32 %p1879, %r10451, 17476; + mov.u32 %r10063, 1; + @%p1879 bra $L__BB1_1491; + + sub.s32 %r7380, %r10452, %r10453; + min.u32 %r7381, %r7380, %r10049; + setp.eq.s32 %p1880, %r7381, 32; + mov.u32 %r7382, -1; + shl.b32 %r7383, %r7382, %r7381; + not.b32 %r7384, %r7383; + selp.b32 %r7385, -1, %r7384, %p1880; + and.b32 %r7386, %r7385, %r10050; + shl.b32 %r7387, %r7386, %r10453; + or.b32 %r10454, %r7387, %r10454; + add.s32 %r10453, %r7381, %r10453; + shr.u32 %r10050, %r10050, %r7381; + sub.s32 %r10049, %r10049, %r7381; + setp.lt.u32 %p1881, %r10453, %r10452; + @%p1881 bra $L__BB1_1490; + + cvt.u64.u32 %rd1185, %r10451; + add.s64 %rd1186, %rd1185, %rd4; + add.s64 %rd1187, %rd1, %rd1186; + st.global.u8 [%rd1187], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p1882, %r10454, 255; + selp.b32 %r10452, 7, 8, %p1882; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1490: + setp.ne.s32 %p1883, %r10049, 0; + mov.u32 %r10063, %r10048; + @%p1883 bra $L__BB1_1487; + +$L__BB1_1491: + and.b32 %r7390, %r9953, 4; + setp.eq.s32 %p1884, %r7390, 0; + mov.u32 %r10078, %r10063; + @%p1884 bra $L__BB1_1498; + + shr.u32 %r7391, %r2897, 2; + and.b32 %r7392, %r7391, 1; + sub.s32 %r10064, %r2896, %r7392; + setp.eq.s32 %p1885, %r10064, 0; + mov.u32 %r10078, %r10063; + @%p1885 bra $L__BB1_1498; + + mov.u32 %r7393, -1; + shl.b32 %r7394, %r7393, %r10064; + not.b32 %r7395, %r7394; + and.b32 %r10065, %r9967, %r7395; + +$L__BB1_1494: + setp.gt.u32 %p1886, %r10451, 17476; + mov.u32 %r10078, 1; + @%p1886 bra $L__BB1_1498; + + sub.s32 %r7397, %r10452, %r10453; + min.u32 %r7398, %r7397, %r10064; + setp.eq.s32 %p1887, %r7398, 32; + mov.u32 %r7399, -1; + shl.b32 %r7400, %r7399, %r7398; + not.b32 %r7401, %r7400; + selp.b32 %r7402, -1, %r7401, %p1887; + and.b32 %r7403, %r7402, %r10065; + shl.b32 %r7404, %r7403, %r10453; + or.b32 %r10454, %r7404, %r10454; + add.s32 %r10453, %r7398, %r10453; + shr.u32 %r10065, %r10065, %r7398; + sub.s32 %r10064, %r10064, %r7398; + setp.lt.u32 %p1888, %r10453, %r10452; + @%p1888 bra $L__BB1_1497; + + cvt.u64.u32 %rd1188, %r10451; + add.s64 %rd1189, %rd1188, %rd4; + add.s64 %rd1190, %rd1, %rd1189; + st.global.u8 [%rd1190], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p1889, %r10454, 255; + selp.b32 %r10452, 7, 8, %p1889; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1497: + setp.ne.s32 %p1890, %r10064, 0; + mov.u32 %r10078, %r10063; + @%p1890 bra $L__BB1_1494; + +$L__BB1_1498: + setp.eq.s32 %p1891, %r2785, 0; + mov.u32 %r10485, %r10078; + @%p1891 bra $L__BB1_1505; + + shr.u32 %r7407, %r2897, 3; + sub.s32 %r10079, %r2896, %r7407; + setp.eq.s32 %p1892, %r10079, 0; + mov.u32 %r10485, %r10078; + @%p1892 bra $L__BB1_1505; + + mov.u32 %r7408, -1; + shl.b32 %r7409, %r7408, %r10079; + not.b32 %r7410, %r7409; + and.b32 %r10080, %r9966, %r7410; + +$L__BB1_1501: + setp.gt.u32 %p1893, %r10451, 17476; + mov.u32 %r10485, 1; + @%p1893 bra $L__BB1_1505; + + sub.s32 %r7412, %r10452, %r10453; + min.u32 %r7413, %r7412, %r10079; + setp.eq.s32 %p1894, %r7413, 32; + mov.u32 %r7414, -1; + shl.b32 %r7415, %r7414, %r7413; + not.b32 %r7416, %r7415; + selp.b32 %r7417, -1, %r7416, %p1894; + and.b32 %r7418, %r7417, %r10080; + shl.b32 %r7419, %r7418, %r10453; + or.b32 %r10454, %r7419, %r10454; + add.s32 %r10453, %r7413, %r10453; + shr.u32 %r10080, %r10080, %r7413; + sub.s32 %r10079, %r10079, %r7413; + setp.lt.u32 %p1895, %r10453, %r10452; + @%p1895 bra $L__BB1_1504; + + cvt.u64.u32 %rd1191, %r10451; + add.s64 %rd1192, %rd1191, %rd4; + add.s64 %rd1193, %rd1, %rd1192; + st.global.u8 [%rd1193], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p1896, %r10454, 255; + selp.b32 %r10452, 7, 8, %p1896; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1504: + setp.ne.s32 %p1897, %r10079, 0; + mov.u32 %r10485, %r10078; + @%p1897 bra $L__BB1_1501; + +$L__BB1_1505: + setp.lt.s32 %p1898, %r2781, 1; + setp.lt.s32 %p1899, %r2440, 1; + or.pred %p1900, %p1899, %p1898; + @%p1900 bra $L__BB1_1553; + + min.s32 %r7422, %r2440, %r2781; + setp.lt.s32 %p1901, %r7422, 3; + add.s32 %r7423, %r9816, 17477; + cvt.u64.u32 %rd1194, %r7423; + add.s64 %rd1195, %rd1194, %rd4; + add.s64 %rd68, %rd1, %rd1195; + @%p1901 bra $L__BB1_1545; + bra.uni $L__BB1_1507; + +$L__BB1_1545: + add.s32 %r10030, %r10030, 1; + setp.lt.u32 %p1948, %r10030, %r10032; + @%p1948 bra $L__BB1_1553; + + shl.b16 %rs932, %rs1253, 1; + or.b16 %rs1253, %rs932, 1; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1949, %r9825, 0; + mov.u32 %r10136, %r10033; + @%p1949 bra $L__BB1_1549; + + setp.gt.u32 %p1950, %r9816, 191; + mov.u32 %r10136, 1; + mov.u32 %r9825, 0; + @%p1950 bra $L__BB1_1549; + + and.b16 %rs934, %rs1253, 255; + st.global.u8 [%rd68], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1951, %rs934, 255; + selp.b32 %r9825, 7, 8, %p1951; + mov.u16 %rs1253, 0; + mov.u32 %r10136, %r10033; + +$L__BB1_1549: + add.s32 %r7511, %r10031, 1; + min.u32 %r10031, %r7511, 12; + setp.lt.u32 %p1952, %r10031, 3; + mov.u32 %r10030, 0; + mov.u32 %r10137, %r10030; + @%p1952 bra $L__BB1_1552; + + setp.lt.u32 %p1953, %r10031, 6; + mov.u32 %r10137, 1; + @%p1953 bra $L__BB1_1552; + + setp.lt.u32 %p1954, %r10031, 9; + setp.eq.s32 %p1955, %r10031, 11; + selp.b32 %r7513, 4, 5, %p1955; + setp.lt.u32 %p1956, %r10031, 11; + selp.b32 %r7514, 3, %r7513, %p1956; + selp.b32 %r10137, 2, %r7514, %p1954; + +$L__BB1_1552: + mov.u32 %r7516, 1; + shl.b32 %r10032, %r7516, %r10137; + mov.u32 %r10033, %r10136; + bra.uni $L__BB1_1553; + +$L__BB1_1507: + shl.b16 %rs1253, %rs1253, 1; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1902, %r9825, 0; + mov.u32 %r10129, %r10033; + @%p1902 bra $L__BB1_1510; + + setp.gt.u32 %p1903, %r9816, 191; + mov.u32 %r10129, 1; + mov.u32 %r9825, 0; + @%p1903 bra $L__BB1_1510; + + st.global.u8 [%rd68], %rs1253; + add.s32 %r9816, %r9816, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9825, 8; + mov.u32 %r10129, %r10033; + +$L__BB1_1510: + setp.lt.u32 %p1904, %r10031, 3; + mov.u32 %r10097, 0; + @%p1904 bra $L__BB1_1513; + + setp.lt.u32 %p1905, %r10031, 6; + mov.u32 %r10097, 1; + @%p1905 bra $L__BB1_1513; + + setp.lt.u32 %p1906, %r10031, 9; + setp.eq.s32 %p1907, %r10031, 11; + selp.b32 %r7429, 4, 5, %p1907; + setp.lt.u32 %p1908, %r10031, 11; + selp.b32 %r7430, 3, %r7429, %p1908; + selp.b32 %r10097, 2, %r7430, %p1906; + +$L__BB1_1513: + setp.eq.s32 %p1909, %r10097, 0; + @%p1909 bra $L__BB1_1541; + + add.s32 %r2998, %r10097, -1; + and.b32 %r2999, %r10097, 3; + setp.eq.s32 %p1910, %r2999, 0; + mov.u32 %r10107, %r10097; + mov.u32 %r10108, %r10129; + @%p1910 bra $L__BB1_1526; + + mov.u32 %r7432, 1; + shl.b32 %r7433, %r7432, %r2998; + and.b32 %r7434, %r7433, %r10030; + setp.ne.s32 %p1911, %r7434, 0; + selp.u32 %r7435, 1, 0, %p1911; + cvt.u32.u16 %r7436, %rs1253; + bfi.b32 %r7437, %r7436, %r7435, 1, 8; + cvt.u16.u32 %rs1253, %r7437; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1912, %r9825, 0; + mov.u32 %r10108, %r10129; + @%p1912 bra $L__BB1_1518; + + setp.gt.u32 %p1913, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r10108, %r7432; + @%p1913 bra $L__BB1_1518; + + add.s32 %r7441, %r9816, 17477; + cvt.u64.u32 %rd1196, %r7441; + add.s64 %rd1197, %rd1196, %rd4; + add.s64 %rd1198, %rd1, %rd1197; + st.global.u8 [%rd1198], %rs1253; + add.s32 %r9816, %r9816, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9825, 8; + mov.u32 %r10108, %r10129; + +$L__BB1_1518: + setp.eq.s32 %p1914, %r2999, 1; + mov.u32 %r10129, %r10108; + mov.u32 %r10107, %r2998; + @%p1914 bra $L__BB1_1526; + + add.s32 %r10107, %r10097, -2; + mov.u32 %r7442, 1; + shl.b32 %r7443, %r7442, %r10107; + and.b32 %r7444, %r7443, %r10030; + setp.ne.s32 %p1915, %r7444, 0; + selp.u32 %r7445, 1, 0, %p1915; + cvt.u32.u16 %r7446, %rs1253; + bfi.b32 %r7447, %r7446, %r7445, 1, 8; + cvt.u16.u32 %rs1253, %r7447; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1916, %r9825, 0; + mov.u32 %r10103, %r10108; + @%p1916 bra $L__BB1_1522; + + setp.gt.u32 %p1917, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r10103, %r7442; + @%p1917 bra $L__BB1_1522; + + add.s32 %r7450, %r9816, 17477; + cvt.u64.u32 %rd1199, %r7450; + add.s64 %rd1200, %rd1199, %rd4; + add.s64 %rd1201, %rd1, %rd1200; + and.b16 %rs920, %rs1253, 255; + st.global.u8 [%rd1201], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1918, %rs920, 255; + selp.b32 %r9825, 7, 8, %p1918; + mov.u16 %rs1253, 0; + mov.u32 %r10103, %r10108; + +$L__BB1_1522: + setp.eq.s32 %p1919, %r2999, 2; + mov.u32 %r10129, %r10103; + mov.u32 %r10108, %r10103; + @%p1919 bra $L__BB1_1526; + + add.s32 %r10107, %r10097, -3; + mov.u32 %r7451, 1; + shl.b32 %r7452, %r7451, %r10107; + and.b32 %r7453, %r7452, %r10030; + setp.ne.s32 %p1920, %r7453, 0; + selp.u32 %r7454, 1, 0, %p1920; + cvt.u32.u16 %r7455, %rs1253; + bfi.b32 %r7456, %r7455, %r7454, 1, 8; + cvt.u16.u32 %rs1253, %r7456; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p1921, %r9825, 0; + mov.u32 %r10129, %r10103; + mov.u32 %r10108, %r10103; + @%p1921 bra $L__BB1_1526; + + setp.gt.u32 %p1922, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r10129, %r7451; + mov.u32 %r10108, %r7451; + @%p1922 bra $L__BB1_1526; + + add.s32 %r7461, %r9816, 17477; + cvt.u64.u32 %rd1202, %r7461; + add.s64 %rd1203, %rd1202, %rd4; + add.s64 %rd1204, %rd1, %rd1203; + and.b16 %rs923, %rs1253, 255; + st.global.u8 [%rd1204], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1923, %rs923, 255; + selp.b32 %r9825, 7, 8, %p1923; + mov.u16 %rs1253, 0; + mov.u32 %r10129, %r10103; + mov.u32 %r10108, %r10103; + +$L__BB1_1526: + setp.lt.u32 %p1924, %r2998, 3; + @%p1924 bra $L__BB1_1541; + + mov.u32 %r10129, %r10108; + +$L__BB1_1528: + add.s32 %r7462, %r10107, -1; + mov.u32 %r7463, 1; + shl.b32 %r7464, %r7463, %r7462; + and.b32 %r7465, %r7464, %r10030; + setp.ne.s32 %p1925, %r7465, 0; + selp.u32 %r7466, 1, 0, %p1925; + cvt.u32.u16 %r7467, %rs1253; + bfi.b32 %r10117, %r7467, %r7466, 1, 8; + add.s32 %r10116, %r9825, -1; + setp.ne.s32 %p1926, %r10116, 0; + mov.u32 %r10118, %r10129; + @%p1926 bra $L__BB1_1531; + + setp.gt.u32 %p1927, %r9816, 191; + mov.u32 %r10116, 0; + mov.u32 %r10118, %r7463; + @%p1927 bra $L__BB1_1531; + + cvt.u16.u32 %rs924, %r10117; + and.b16 %rs925, %rs924, 255; + add.s32 %r7471, %r9816, 17477; + cvt.u64.u32 %rd1205, %r7471; + add.s64 %rd1206, %rd1205, %rd4; + add.s64 %rd1207, %rd1, %rd1206; + st.global.u8 [%rd1207], %rs924; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1928, %rs925, 255; + selp.b32 %r10116, 7, 8, %p1928; + mov.u32 %r10117, 0; + mov.u32 %r10118, %r10129; + +$L__BB1_1531: + add.s32 %r7472, %r10107, -2; + shl.b32 %r7474, %r7463, %r7472; + and.b32 %r7475, %r7474, %r10030; + setp.ne.s32 %p1929, %r7475, 0; + and.b32 %r7476, %r10117, 127; + selp.u32 %r7477, 1, 0, %p1929; + bfi.b32 %r10121, %r7476, %r7477, 1, 7; + add.s32 %r10120, %r10116, -1; + setp.ne.s32 %p1930, %r10120, 0; + mov.u32 %r10122, %r10118; + @%p1930 bra $L__BB1_1534; + + setp.gt.u32 %p1931, %r9816, 191; + mov.u32 %r10122, 1; + mov.u32 %r10120, 0; + @%p1931 bra $L__BB1_1534; + + cvt.u16.u32 %rs926, %r10121; + and.b16 %rs927, %rs926, 255; + add.s32 %r7481, %r9816, 17477; + cvt.u64.u32 %rd1208, %r7481; + add.s64 %rd1209, %rd1208, %rd4; + add.s64 %rd1210, %rd1, %rd1209; + st.global.u8 [%rd1210], %rs926; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1932, %rs927, 255; + selp.b32 %r10120, 7, 8, %p1932; + mov.u32 %r10121, 0; + mov.u32 %r10122, %r10118; + +$L__BB1_1534: + add.s32 %r7482, %r10107, -3; + mov.u32 %r7483, 1; + shl.b32 %r7484, %r7483, %r7482; + and.b32 %r7485, %r7484, %r10030; + setp.ne.s32 %p1933, %r7485, 0; + and.b32 %r7486, %r10121, 127; + selp.u32 %r7487, 1, 0, %p1933; + bfi.b32 %r10125, %r7486, %r7487, 1, 7; + add.s32 %r10124, %r10120, -1; + setp.ne.s32 %p1934, %r10124, 0; + mov.u32 %r10126, %r10122; + @%p1934 bra $L__BB1_1537; + + setp.gt.u32 %p1935, %r9816, 191; + mov.u32 %r10124, 0; + mov.u32 %r10126, %r7483; + @%p1935 bra $L__BB1_1537; + + cvt.u16.u32 %rs928, %r10125; + and.b16 %rs929, %rs928, 255; + add.s32 %r7491, %r9816, 17477; + cvt.u64.u32 %rd1211, %r7491; + add.s64 %rd1212, %rd1211, %rd4; + add.s64 %rd1213, %rd1, %rd1212; + st.global.u8 [%rd1213], %rs928; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1936, %rs929, 255; + selp.b32 %r10124, 7, 8, %p1936; + mov.u32 %r10125, 0; + mov.u32 %r10126, %r10122; + +$L__BB1_1537: + add.s32 %r10107, %r10107, -4; + shl.b32 %r7493, %r7483, %r10107; + and.b32 %r7494, %r7493, %r10030; + setp.ne.s32 %p1937, %r7494, 0; + and.b32 %r7495, %r10125, 127; + selp.u32 %r7496, 1, 0, %p1937; + bfi.b32 %r7497, %r7495, %r7496, 1, 15; + cvt.u16.u32 %rs1253, %r7497; + add.s32 %r9825, %r10124, -1; + setp.ne.s32 %p1938, %r9825, 0; + mov.u32 %r10129, %r10126; + @%p1938 bra $L__BB1_1540; + + setp.gt.u32 %p1939, %r9816, 191; + mov.u32 %r10129, 1; + mov.u32 %r9825, 0; + @%p1939 bra $L__BB1_1540; + + add.s32 %r7500, %r9816, 17477; + cvt.u64.u32 %rd1214, %r7500; + add.s64 %rd1215, %rd1214, %rd4; + add.s64 %rd1216, %rd1, %rd1215; + and.b16 %rs931, %rs1253, 255; + st.global.u8 [%rd1216], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p1940, %rs931, 255; + selp.b32 %r9825, 7, 8, %p1940; + mov.u16 %rs1253, 0; + mov.u32 %r10129, %r10126; + +$L__BB1_1540: + setp.ne.s32 %p1941, %r10107, 0; + @%p1941 bra $L__BB1_1528; + +$L__BB1_1541: + add.s32 %r7502, %r10031, -1; + setp.eq.s32 %p1942, %r10031, 0; + mov.u32 %r10030, 0; + selp.b32 %r10031, 0, %r7502, %p1942; + setp.lt.u32 %p1943, %r10031, 3; + mov.u32 %r10133, %r10030; + @%p1943 bra $L__BB1_1544; + + setp.lt.u32 %p1944, %r10031, 6; + mov.u32 %r10133, 1; + @%p1944 bra $L__BB1_1544; + + setp.lt.u32 %p1945, %r10031, 9; + setp.eq.s32 %p1946, %r10031, 11; + selp.b32 %r7504, 4, 5, %p1946; + setp.lt.u32 %p1947, %r10031, 11; + selp.b32 %r7505, 3, %r7504, %p1947; + selp.b32 %r10133, 2, %r7505, %p1945; + +$L__BB1_1544: + mov.u32 %r7507, 1; + shl.b32 %r10032, %r7507, %r10133; + mov.u32 %r10033, %r10129; + +$L__BB1_1553: + setp.gt.s32 %p1957, %r2781, 2; + setp.gt.s32 %p1958, %r2440, 2; + and.pred %p1959, %p1958, %p1957; + @%p1959 bra $L__BB1_1602; + bra.uni $L__BB1_1554; + +$L__BB1_1602: + add.s32 %r7637, %r2653, -11; + cvt.u64.u32 %rd1246, %r7637; + add.s64 %rd70, %rd63, %rd1246; + ld.global.u8 %rs393, [%rd70]; + add.s32 %r7638, %r2653, -10; + cvt.u64.u32 %rd1248, %r7638; + add.s64 %rd1249, %rd63, %rd1248; + ld.global.u8 %rs394, [%rd1249]; + ld.global.u8 %rs395, [%rd1249+1]; + mul.lo.s32 %r7639, %r2781, 6; + add.s32 %r7640, %r7639, -12; + cvt.u64.u32 %rd1250, %r7640; + add.s64 %rd1251, %rd63, %rd1250; + ld.global.u8 %rs396, [%rd1251]; + ld.global.u8 %rs397, [%rd1251+1]; + add.s32 %r7641, %r7639, -10; + cvt.u64.u32 %rd1252, %r7641; + add.s64 %rd1253, %rd63, %rd1252; + ld.global.u8 %rs398, [%rd1253]; + ld.global.u8 %rs399, [%rd1253+1]; + setp.eq.s16 %p2027, %rs393, 0; + mov.u32 %r10231, %r9983; + @%p2027 bra $L__BB1_1609; + + ld.global.u8 %r10221, [%rd70+-1]; + cvt.u32.u16 %r10220, %rs393; + +$L__BB1_1604: + mov.u32 %r3208, %r10220; + setp.gt.u32 %p2028, %r10264, 2879; + mov.u32 %r10231, 1; + @%p2028 bra $L__BB1_1609; + + mov.u32 %r7643, 8; + sub.s32 %r7644, %r7643, %r10266; + sub.s32 %r7645, %r7644, %r10265; + min.u32 %r7646, %r7645, %r3208; + setp.eq.s32 %p2029, %r7646, 32; + mov.u32 %r7647, -1; + shl.b32 %r7648, %r7647, %r7646; + not.b32 %r7649, %r7648; + selp.b32 %r7650, -1, %r7649, %p2029; + and.b32 %r7651, %r7650, %r10221; + shl.b32 %r7652, %r7651, %r10265; + cvt.u16.u32 %rs967, %r7652; + or.b16 %rs1322, %rs1322, %rs967; + add.s32 %r10265, %r7646, %r10265; + sub.s32 %r10220, %r3208, %r7646; + shr.u32 %r10221, %r10221, %r7646; + setp.gt.u32 %p2030, %r7645, %r3208; + @%p2030 bra $L__BB1_1608; + + setp.ne.s32 %p2031, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs968, %rs1322, 255; + setp.ne.s16 %p2032, %rs968, 127; + and.pred %p2033, %p2031, %p2032; + @%p2033 bra $L__BB1_1608; + + mov.u32 %r7655, 20548; + sub.s32 %r7656, %r7655, %r10264; + cvt.u64.u32 %rd1254, %r7656; + add.s64 %rd1255, %rd1254, %rd4; + add.s64 %rd1256, %rd1, %rd1255; + st.global.u8 [%rd1256], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2034, %rs968, 143; + selp.u32 %r10266, 1, 0, %p2034; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1608: + setp.ne.s32 %p2035, %r10220, 0; + mov.u32 %r10231, %r9983; + @%p2035 bra $L__BB1_1604; + +$L__BB1_1609: + setp.eq.s16 %p2036, %rs397, 0; + mov.u32 %r10243, %r10231; + @%p2036 bra $L__BB1_1616; + + cvt.u32.u16 %r7657, %rs396; + and.b32 %r10233, %r7657, 255; + cvt.u32.u16 %r7658, %rs397; + and.b32 %r10232, %r7658, 255; + +$L__BB1_1611: + mov.u32 %r3227, %r10232; + setp.gt.u32 %p2037, %r10264, 2879; + mov.u32 %r10243, 1; + @%p2037 bra $L__BB1_1616; + + mov.u32 %r7660, 8; + sub.s32 %r7661, %r7660, %r10266; + sub.s32 %r7662, %r7661, %r10265; + min.u32 %r7663, %r7662, %r3227; + setp.eq.s32 %p2038, %r7663, 32; + mov.u32 %r7664, -1; + shl.b32 %r7665, %r7664, %r7663; + not.b32 %r7666, %r7665; + selp.b32 %r7667, -1, %r7666, %p2038; + and.b32 %r7668, %r7667, %r10233; + shl.b32 %r7669, %r7668, %r10265; + cvt.u16.u32 %rs972, %r7669; + or.b16 %rs1322, %rs1322, %rs972; + add.s32 %r10265, %r7663, %r10265; + sub.s32 %r10232, %r3227, %r7663; + shr.u32 %r10233, %r10233, %r7663; + setp.gt.u32 %p2039, %r7662, %r3227; + @%p2039 bra $L__BB1_1615; + + setp.ne.s32 %p2040, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs973, %rs1322, 255; + setp.ne.s16 %p2041, %rs973, 127; + and.pred %p2042, %p2040, %p2041; + @%p2042 bra $L__BB1_1615; + + mov.u32 %r7672, 20548; + sub.s32 %r7673, %r7672, %r10264; + cvt.u64.u32 %rd1257, %r7673; + add.s64 %rd1258, %rd1257, %rd4; + add.s64 %rd1259, %rd1, %rd1258; + st.global.u8 [%rd1259], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2043, %rs973, 143; + selp.u32 %r10266, 1, 0, %p2043; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1615: + setp.ne.s32 %p2044, %r10232, 0; + mov.u32 %r10243, %r10231; + @%p2044 bra $L__BB1_1611; + +$L__BB1_1616: + setp.eq.s16 %p2045, %rs395, 0; + mov.u32 %r10255, %r10243; + @%p2045 bra $L__BB1_1623; + + cvt.u32.u16 %r7674, %rs395; + and.b32 %r10244, %r7674, 255; + cvt.u32.u16 %r7675, %rs394; + and.b32 %r10245, %r7675, 255; + +$L__BB1_1618: + mov.u32 %r3246, %r10244; + setp.gt.u32 %p2046, %r10264, 2879; + mov.u32 %r10255, 1; + @%p2046 bra $L__BB1_1623; + + mov.u32 %r7677, 8; + sub.s32 %r7678, %r7677, %r10266; + sub.s32 %r7679, %r7678, %r10265; + min.u32 %r7680, %r7679, %r3246; + setp.eq.s32 %p2047, %r7680, 32; + mov.u32 %r7681, -1; + shl.b32 %r7682, %r7681, %r7680; + not.b32 %r7683, %r7682; + selp.b32 %r7684, -1, %r7683, %p2047; + and.b32 %r7685, %r7684, %r10245; + shl.b32 %r7686, %r7685, %r10265; + cvt.u16.u32 %rs977, %r7686; + or.b16 %rs1322, %rs1322, %rs977; + add.s32 %r10265, %r7680, %r10265; + sub.s32 %r10244, %r3246, %r7680; + shr.u32 %r10245, %r10245, %r7680; + setp.gt.u32 %p2048, %r7679, %r3246; + @%p2048 bra $L__BB1_1622; + + setp.ne.s32 %p2049, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs978, %rs1322, 255; + setp.ne.s16 %p2050, %rs978, 127; + and.pred %p2051, %p2049, %p2050; + @%p2051 bra $L__BB1_1622; + + mov.u32 %r7689, 20548; + sub.s32 %r7690, %r7689, %r10264; + cvt.u64.u32 %rd1260, %r7690; + add.s64 %rd1261, %rd1260, %rd4; + add.s64 %rd1262, %rd1, %rd1261; + st.global.u8 [%rd1262], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2052, %rs978, 143; + selp.u32 %r10266, 1, 0, %p2052; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1622: + setp.ne.s32 %p2053, %r10244, 0; + mov.u32 %r10255, %r10243; + @%p2053 bra $L__BB1_1618; + +$L__BB1_1623: + setp.eq.s16 %p2054, %rs399, 0; + mov.u32 %r10267, %r10255; + @%p2054 bra $L__BB1_1630; + + cvt.u32.u16 %r7691, %rs398; + and.b32 %r10257, %r7691, 255; + cvt.u32.u16 %r7692, %rs399; + and.b32 %r10256, %r7692, 255; + +$L__BB1_1625: + mov.u32 %r3265, %r10256; + setp.gt.u32 %p2055, %r10264, 2879; + mov.u32 %r10267, 1; + @%p2055 bra $L__BB1_1630; + + mov.u32 %r7694, 8; + sub.s32 %r7695, %r7694, %r10266; + sub.s32 %r7696, %r7695, %r10265; + min.u32 %r7697, %r7696, %r3265; + setp.eq.s32 %p2056, %r7697, 32; + mov.u32 %r7698, -1; + shl.b32 %r7699, %r7698, %r7697; + not.b32 %r7700, %r7699; + selp.b32 %r7701, -1, %r7700, %p2056; + and.b32 %r7702, %r7701, %r10257; + shl.b32 %r7703, %r7702, %r10265; + cvt.u16.u32 %rs982, %r7703; + or.b16 %rs1322, %rs1322, %rs982; + add.s32 %r10265, %r7697, %r10265; + sub.s32 %r10256, %r3265, %r7697; + shr.u32 %r10257, %r10257, %r7697; + setp.gt.u32 %p2057, %r7696, %r3265; + @%p2057 bra $L__BB1_1629; + + setp.ne.s32 %p2058, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs983, %rs1322, 255; + setp.ne.s16 %p2059, %rs983, 127; + and.pred %p2060, %p2058, %p2059; + @%p2060 bra $L__BB1_1629; + + mov.u32 %r7706, 20548; + sub.s32 %r7707, %r7706, %r10264; + cvt.u64.u32 %rd1263, %r7707; + add.s64 %rd1264, %rd1263, %rd4; + add.s64 %rd1265, %rd1, %rd1264; + st.global.u8 [%rd1265], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2061, %rs983, 143; + selp.u32 %r10266, 1, 0, %p2061; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1629: + setp.ne.s32 %p2062, %r10256, 0; + mov.u32 %r10267, %r10255; + @%p2062 bra $L__BB1_1625; + bra.uni $L__BB1_1630; + +$L__BB1_1554: + setp.gt.s32 %p1960, %r2781, 0; + and.pred %p1962, %p1958, %p1960; + @%p1962 bra $L__BB1_1583; + bra.uni $L__BB1_1555; + +$L__BB1_1583: + ld.global.u8 %rs379, [%rd65+1]; + ld.global.u8 %rs380, [%rd66]; + ld.global.u8 %rs381, [%rd66+1]; + setp.eq.s16 %p2001, %rs379, 0; + mov.u32 %r10199, %r9983; + @%p2001 bra $L__BB1_1590; + + ld.global.u8 %r10189, [%rd65]; + cvt.u32.u16 %r10188, %rs379; + +$L__BB1_1585: + mov.u32 %r3156, %r10188; + setp.gt.u32 %p2002, %r10264, 2879; + mov.u32 %r10199, 1; + @%p2002 bra $L__BB1_1590; + + mov.u32 %r7589, 8; + sub.s32 %r7590, %r7589, %r10266; + sub.s32 %r7591, %r7590, %r10265; + min.u32 %r7592, %r7591, %r3156; + setp.eq.s32 %p2003, %r7592, 32; + mov.u32 %r7593, -1; + shl.b32 %r7594, %r7593, %r7592; + not.b32 %r7595, %r7594; + selp.b32 %r7596, -1, %r7595, %p2003; + and.b32 %r7597, %r7596, %r10189; + shl.b32 %r7598, %r7597, %r10265; + cvt.u16.u32 %rs954, %r7598; + or.b16 %rs1322, %rs1322, %rs954; + add.s32 %r10265, %r7592, %r10265; + sub.s32 %r10188, %r3156, %r7592; + shr.u32 %r10189, %r10189, %r7592; + setp.gt.u32 %p2004, %r7591, %r3156; + @%p2004 bra $L__BB1_1589; + + setp.ne.s32 %p2005, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs955, %rs1322, 255; + setp.ne.s16 %p2006, %rs955, 127; + and.pred %p2007, %p2005, %p2006; + @%p2007 bra $L__BB1_1589; + + mov.u32 %r7601, 20548; + sub.s32 %r7602, %r7601, %r10264; + cvt.u64.u32 %rd1237, %r7602; + add.s64 %rd1238, %rd1237, %rd4; + add.s64 %rd1239, %rd1, %rd1238; + st.global.u8 [%rd1239], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2008, %rs955, 143; + selp.u32 %r10266, 1, 0, %p2008; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1589: + setp.ne.s32 %p2009, %r10188, 0; + mov.u32 %r10199, %r9983; + @%p2009 bra $L__BB1_1585; + +$L__BB1_1590: + add.s32 %r10201, %r2781, -1; + cvt.u32.u16 %r7604, %rs381; + and.b32 %r10212, %r7604, 255; + cvt.u32.u16 %r7605, %rs380; + and.b32 %r10213, %r7605, 255; + mov.u32 %r7603, 1; + mov.u32 %r10200, %r7603; + +$L__BB1_1591: + mov.u32 %r3176, %r10200; + setp.gt.u32 %p2010, %r10264, 2879; + mov.u32 %r10211, %r7603; + @%p2010 bra $L__BB1_1596; + + mov.u32 %r7607, 8; + sub.s32 %r7608, %r7607, %r10266; + sub.s32 %r7609, %r7608, %r10265; + min.u32 %r7610, %r7609, %r3176; + setp.eq.s32 %p2011, %r7610, 32; + mov.u32 %r7611, -1; + shl.b32 %r7612, %r7611, %r7610; + not.b32 %r7613, %r7612; + selp.b32 %r7614, -1, %r7613, %p2011; + and.b32 %r7615, %r7614, %r10201; + shl.b32 %r7616, %r7615, %r10265; + cvt.u16.u32 %rs958, %r7616; + or.b16 %rs1322, %rs1322, %rs958; + add.s32 %r10265, %r7610, %r10265; + sub.s32 %r10200, %r3176, %r7610; + shr.u32 %r10201, %r10201, %r7610; + setp.gt.u32 %p2012, %r7609, %r3176; + @%p2012 bra $L__BB1_1595; + + setp.ne.s32 %p2013, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs959, %rs1322, 255; + setp.ne.s16 %p2014, %rs959, 127; + and.pred %p2015, %p2013, %p2014; + @%p2015 bra $L__BB1_1595; + + mov.u32 %r7619, 20548; + sub.s32 %r7620, %r7619, %r10264; + cvt.u64.u32 %rd1240, %r7620; + add.s64 %rd1241, %rd1240, %rd4; + add.s64 %rd1242, %rd1, %rd1241; + st.global.u8 [%rd1242], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2016, %rs959, 143; + selp.u32 %r10266, 1, 0, %p2016; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1595: + setp.ne.s32 %p2017, %r10200, 0; + mov.u32 %r10211, %r10199; + @%p2017 bra $L__BB1_1591; + +$L__BB1_1596: + setp.eq.s16 %p2018, %rs381, 0; + mov.u32 %r10267, %r10211; + @%p2018 bra $L__BB1_1630; + +$L__BB1_1597: + mov.u32 %r3193, %r10212; + setp.gt.u32 %p2019, %r10264, 2879; + mov.u32 %r10267, 1; + @%p2019 bra $L__BB1_1630; + + mov.u32 %r7622, 8; + sub.s32 %r7623, %r7622, %r10266; + sub.s32 %r7624, %r7623, %r10265; + min.u32 %r7625, %r7624, %r3193; + setp.eq.s32 %p2020, %r7625, 32; + mov.u32 %r7626, -1; + shl.b32 %r7627, %r7626, %r7625; + not.b32 %r7628, %r7627; + selp.b32 %r7629, -1, %r7628, %p2020; + and.b32 %r7630, %r7629, %r10213; + shl.b32 %r7631, %r7630, %r10265; + cvt.u16.u32 %rs963, %r7631; + or.b16 %rs1322, %rs1322, %rs963; + add.s32 %r10265, %r7625, %r10265; + sub.s32 %r10212, %r3193, %r7625; + shr.u32 %r10213, %r10213, %r7625; + setp.gt.u32 %p2021, %r7624, %r3193; + @%p2021 bra $L__BB1_1601; + + setp.ne.s32 %p2022, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs964, %rs1322, 255; + setp.ne.s16 %p2023, %rs964, 127; + and.pred %p2024, %p2022, %p2023; + @%p2024 bra $L__BB1_1601; + + mov.u32 %r7634, 20548; + sub.s32 %r7635, %r7634, %r10264; + cvt.u64.u32 %rd1243, %r7635; + add.s64 %rd1244, %rd1243, %rd4; + add.s64 %rd1245, %rd1, %rd1244; + st.global.u8 [%rd1245], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2025, %rs964, 143; + selp.u32 %r10266, 1, 0, %p2025; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1601: + setp.eq.s32 %p2026, %r10212, 0; + mov.u32 %r10267, %r10211; + @%p2026 bra $L__BB1_1630; + bra.uni $L__BB1_1597; + +$L__BB1_1555: + setp.gt.s32 %p1964, %r2440, 0; + selp.b32 %r7517, %r2653, 0, %p1964; + cvt.u64.u32 %rd1217, %r7517; + add.s64 %rd69, %rd63, %rd1217; + ld.global.u8 %rs357, [%rd69+1]; + add.s32 %r7518, %r7517, 2; + cvt.u64.u32 %rd1219, %r7518; + add.s64 %rd1220, %rd63, %rd1219; + ld.global.u8 %rs358, [%rd1220]; + ld.global.u8 %rs359, [%rd1220+1]; + mul.lo.s32 %r7519, %r2781, 6; + selp.b32 %r7520, %r7519, 0, %p1960; + cvt.u64.u32 %rd1221, %r7520; + add.s64 %rd1222, %rd63, %rd1221; + ld.global.u8 %rs360, [%rd1222]; + ld.global.u8 %rs361, [%rd1222+1]; + add.s32 %r7521, %r7520, 2; + cvt.u64.u32 %rd1223, %r7521; + add.s64 %rd1224, %rd63, %rd1223; + ld.global.u8 %rs362, [%rd1224]; + ld.global.u8 %rs363, [%rd1224+1]; + setp.eq.s16 %p1965, %rs357, 0; + mov.u32 %r10155, %r9983; + @%p1965 bra $L__BB1_1562; + + ld.global.u8 %r10145, [%rd69]; + cvt.u32.u16 %r10144, %rs357; + +$L__BB1_1557: + mov.u32 %r3084, %r10144; + setp.gt.u32 %p1966, %r10264, 2879; + mov.u32 %r10155, 1; + @%p1966 bra $L__BB1_1562; + + mov.u32 %r7523, 8; + sub.s32 %r7524, %r7523, %r10266; + sub.s32 %r7525, %r7524, %r10265; + min.u32 %r7526, %r7525, %r3084; + setp.eq.s32 %p1967, %r7526, 32; + mov.u32 %r7527, -1; + shl.b32 %r7528, %r7527, %r7526; + not.b32 %r7529, %r7528; + selp.b32 %r7530, -1, %r7529, %p1967; + and.b32 %r7531, %r7530, %r10145; + shl.b32 %r7532, %r7531, %r10265; + cvt.u16.u32 %rs935, %r7532; + or.b16 %rs1322, %rs1322, %rs935; + add.s32 %r10265, %r7526, %r10265; + sub.s32 %r10144, %r3084, %r7526; + shr.u32 %r10145, %r10145, %r7526; + setp.gt.u32 %p1968, %r7525, %r3084; + @%p1968 bra $L__BB1_1561; + + setp.ne.s32 %p1969, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs936, %rs1322, 255; + setp.ne.s16 %p1970, %rs936, 127; + and.pred %p1971, %p1969, %p1970; + @%p1971 bra $L__BB1_1561; + + mov.u32 %r7535, 20548; + sub.s32 %r7536, %r7535, %r10264; + cvt.u64.u32 %rd1225, %r7536; + add.s64 %rd1226, %rd1225, %rd4; + add.s64 %rd1227, %rd1, %rd1226; + st.global.u8 [%rd1227], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p1972, %rs936, 143; + selp.u32 %r10266, 1, 0, %p1972; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1561: + setp.ne.s32 %p1973, %r10144, 0; + mov.u32 %r10155, %r9983; + @%p1973 bra $L__BB1_1557; + +$L__BB1_1562: + setp.eq.s16 %p1974, %rs361, 0; + mov.u32 %r10167, %r10155; + @%p1974 bra $L__BB1_1569; + + cvt.u32.u16 %r7537, %rs360; + and.b32 %r10157, %r7537, 255; + cvt.u32.u16 %r7538, %rs361; + and.b32 %r10156, %r7538, 255; + +$L__BB1_1564: + mov.u32 %r3103, %r10156; + setp.gt.u32 %p1975, %r10264, 2879; + mov.u32 %r10167, 1; + @%p1975 bra $L__BB1_1569; + + mov.u32 %r7540, 8; + sub.s32 %r7541, %r7540, %r10266; + sub.s32 %r7542, %r7541, %r10265; + min.u32 %r7543, %r7542, %r3103; + setp.eq.s32 %p1976, %r7543, 32; + mov.u32 %r7544, -1; + shl.b32 %r7545, %r7544, %r7543; + not.b32 %r7546, %r7545; + selp.b32 %r7547, -1, %r7546, %p1976; + and.b32 %r7548, %r7547, %r10157; + shl.b32 %r7549, %r7548, %r10265; + cvt.u16.u32 %rs940, %r7549; + or.b16 %rs1322, %rs1322, %rs940; + add.s32 %r10265, %r7543, %r10265; + sub.s32 %r10156, %r3103, %r7543; + shr.u32 %r10157, %r10157, %r7543; + setp.gt.u32 %p1977, %r7542, %r3103; + @%p1977 bra $L__BB1_1568; + + setp.ne.s32 %p1978, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs941, %rs1322, 255; + setp.ne.s16 %p1979, %rs941, 127; + and.pred %p1980, %p1978, %p1979; + @%p1980 bra $L__BB1_1568; + + mov.u32 %r7552, 20548; + sub.s32 %r7553, %r7552, %r10264; + cvt.u64.u32 %rd1228, %r7553; + add.s64 %rd1229, %rd1228, %rd4; + add.s64 %rd1230, %rd1, %rd1229; + st.global.u8 [%rd1230], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p1981, %rs941, 143; + selp.u32 %r10266, 1, 0, %p1981; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1568: + setp.ne.s32 %p1982, %r10156, 0; + mov.u32 %r10167, %r10155; + @%p1982 bra $L__BB1_1564; + +$L__BB1_1569: + setp.eq.s16 %p1983, %rs359, 0; + mov.u32 %r10179, %r10167; + @%p1983 bra $L__BB1_1576; + + cvt.u32.u16 %r7554, %rs359; + and.b32 %r10168, %r7554, 255; + cvt.u32.u16 %r7555, %rs358; + and.b32 %r10169, %r7555, 255; + +$L__BB1_1571: + mov.u32 %r3122, %r10168; + setp.gt.u32 %p1984, %r10264, 2879; + mov.u32 %r10179, 1; + @%p1984 bra $L__BB1_1576; + + mov.u32 %r7557, 8; + sub.s32 %r7558, %r7557, %r10266; + sub.s32 %r7559, %r7558, %r10265; + min.u32 %r7560, %r7559, %r3122; + setp.eq.s32 %p1985, %r7560, 32; + mov.u32 %r7561, -1; + shl.b32 %r7562, %r7561, %r7560; + not.b32 %r7563, %r7562; + selp.b32 %r7564, -1, %r7563, %p1985; + and.b32 %r7565, %r7564, %r10169; + shl.b32 %r7566, %r7565, %r10265; + cvt.u16.u32 %rs945, %r7566; + or.b16 %rs1322, %rs1322, %rs945; + add.s32 %r10265, %r7560, %r10265; + sub.s32 %r10168, %r3122, %r7560; + shr.u32 %r10169, %r10169, %r7560; + setp.gt.u32 %p1986, %r7559, %r3122; + @%p1986 bra $L__BB1_1575; + + setp.ne.s32 %p1987, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs946, %rs1322, 255; + setp.ne.s16 %p1988, %rs946, 127; + and.pred %p1989, %p1987, %p1988; + @%p1989 bra $L__BB1_1575; + + mov.u32 %r7569, 20548; + sub.s32 %r7570, %r7569, %r10264; + cvt.u64.u32 %rd1231, %r7570; + add.s64 %rd1232, %rd1231, %rd4; + add.s64 %rd1233, %rd1, %rd1232; + st.global.u8 [%rd1233], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p1990, %rs946, 143; + selp.u32 %r10266, 1, 0, %p1990; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1575: + setp.ne.s32 %p1991, %r10168, 0; + mov.u32 %r10179, %r10167; + @%p1991 bra $L__BB1_1571; + +$L__BB1_1576: + setp.eq.s16 %p1992, %rs363, 0; + mov.u32 %r10267, %r10179; + @%p1992 bra $L__BB1_1630; + + cvt.u32.u16 %r7571, %rs362; + and.b32 %r10181, %r7571, 255; + cvt.u32.u16 %r7572, %rs363; + and.b32 %r10180, %r7572, 255; + +$L__BB1_1578: + mov.u32 %r3141, %r10180; + setp.gt.u32 %p1993, %r10264, 2879; + mov.u32 %r10267, 1; + @%p1993 bra $L__BB1_1630; + + mov.u32 %r7574, 8; + sub.s32 %r7575, %r7574, %r10266; + sub.s32 %r7576, %r7575, %r10265; + min.u32 %r7577, %r7576, %r3141; + setp.eq.s32 %p1994, %r7577, 32; + mov.u32 %r7578, -1; + shl.b32 %r7579, %r7578, %r7577; + not.b32 %r7580, %r7579; + selp.b32 %r7581, -1, %r7580, %p1994; + and.b32 %r7582, %r7581, %r10181; + shl.b32 %r7583, %r7582, %r10265; + cvt.u16.u32 %rs950, %r7583; + or.b16 %rs1322, %rs1322, %rs950; + add.s32 %r10265, %r7577, %r10265; + sub.s32 %r10180, %r3141, %r7577; + shr.u32 %r10181, %r10181, %r7577; + setp.gt.u32 %p1995, %r7576, %r3141; + @%p1995 bra $L__BB1_1582; + + setp.ne.s32 %p1996, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs951, %rs1322, 255; + setp.ne.s16 %p1997, %rs951, 127; + and.pred %p1998, %p1996, %p1997; + @%p1998 bra $L__BB1_1582; + + mov.u32 %r7586, 20548; + sub.s32 %r7587, %r7586, %r10264; + cvt.u64.u32 %rd1234, %r7587; + add.s64 %rd1235, %rd1234, %rd4; + add.s64 %rd1236, %rd1, %rd1235; + st.global.u8 [%rd1236], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p1999, %rs951, 143; + selp.u32 %r10266, 1, 0, %p1999; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1582: + setp.eq.s32 %p2000, %r10180, 0; + mov.u32 %r10267, %r10179; + @%p2000 bra $L__BB1_1630; + bra.uni $L__BB1_1578; + +$L__BB1_1630: + shr.u32 %r7708, %r9953, 1; + or.b32 %r9750, %r7708, %r2898; + +$L__BB1_1631: + add.s32 %r9734, %r9734, 4; + setp.lt.u32 %p2063, %r9734, %r5; + @%p2063 bra $L__BB1_1266; + +$L__BB1_1632: + add.s32 %r8405, %r5, 1; + shr.u32 %r8404, %r8405, 1; + add.s32 %r7709, %r8404, 1; + setp.gt.u32 %p2064, %r7709, 512; + @%p2064 bra $L__BB1_1634; + + add.s32 %r8408, %r5, 1; + shr.u32 %r8407, %r8408, 1; + add.s32 %r8406, %r4095, %r8407; + mov.u16 %rs986, 0; + add.s32 %r8393, %r8406, 1; + st.shared.u8 [%r8393], %rs986; + +$L__BB1_1634: + setp.lt.u32 %p2065, %r6, 3; + @%p2065 bra $L__BB1_1880; + + ld.param.u64 %rd1422, [ j2k_htj2k_encode_codeblocks_param_5]; + ld.param.u64 %rd1417, [ j2k_htj2k_encode_codeblocks_param_4]; + mov.u32 %r10300, 2; + cvta.to.global.u64 %rd71, %rd1422; + cvta.to.global.u64 %rd72, %rd1417; + +$L__BB1_1636: + ld.shared.u8 %rs422, [_ZZ32 j2k_htj2k_encode_codeblocksE13cleanup_e_val]; + mov.u16 %rs987, 0; + st.shared.u8 [_ZZ32 j2k_htj2k_encode_codeblocksE13cleanup_e_val], %rs987; + ld.shared.u8 %rs423, [_ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val]; + st.shared.u8 [_ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val], %rs987; + @%p10 bra $L__BB1_1879; + + mov.u32 %r7713, 0; + ld.shared.u8 %rs988, [_ZZ32 j2k_htj2k_encode_codeblocksE13cleanup_e_val+1]; + ld.shared.u8 %rs989, [_ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val+1]; + max.u16 %rs991, %rs422, %rs988; + cvt.u32.u16 %r7714, %rs991; + add.s32 %r10335, %r7714, -1; + add.s32 %r3333, %r10300, 1; + mul.lo.s32 %r10333, %r10300, %r1; + mul.wide.u16 %r7715, %rs989, 4; + cvt.u32.u16 %r7716, %rs423; + and.b32 %r7717, %r7716, 255; + add.s32 %r10332, %r7715, %r7717; + mov.u32 %r10316, %r7713; + mov.u32 %r10334, %r7713; + mov.u32 %r10336, %r7713; + bra.uni $L__BB1_1638; + +$L__BB1_1709: + setp.gt.u32 %p2145, %r9816, 191; + mov.u32 %r10418, 1; + mov.u32 %r9825, 0; + @%p2145 bra $L__BB1_1711; + + and.b16 %rs1018, %rs1253, 255; + st.global.u8 [%rd73], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2146, %rs1018, 255; + selp.b32 %r9825, 7, 8, %p2146; + mov.u16 %rs1253, 0; + mov.u32 %r10418, %r10033; + bra.uni $L__BB1_1711; + +$L__BB1_1815: + setp.gt.u32 %p2264, %r9816, 191; + mov.u32 %r10567, 1; + mov.u32 %r9825, 0; + @%p2264 bra $L__BB1_1817; + + and.b16 %rs1055, %rs1253, 255; + st.global.u8 [%rd74], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2265, %rs1055, 255; + selp.b32 %r9825, 7, 8, %p2265; + mov.u16 %rs1253, 0; + mov.u32 %r10567, %r10033; + bra.uni $L__BB1_1817; + +$L__BB1_1638: + cvt.u64.u32 %rd1266, %r10333; + add.s64 %rd1267, %rd1266, %rd5; + shl.b64 %rd1268, %rd1267, 2; + add.s64 %rd1269, %rd3, %rd1268; + ld.global.u32 %r3357, [%rd1269]; + setp.eq.s32 %p2067, %r3357, 0; + mov.u32 %r10337, %r7713; + @%p2067 bra $L__BB1_1640; + + and.b32 %r7719, %r3357, -2147483648; + abs.s32 %r7720, %r3357; + shl.b32 %r7721, %r7720, %r2355; + or.b32 %r10337, %r7721, %r7719; + +$L__BB1_1640: + shl.b32 %r7725, %r10337, 1; + shr.u32 %r7726, %r7725, %r2355; + and.b32 %r3360, %r7726, -2; + setp.eq.s32 %p2068, %r3360, 0; + mov.u32 %r10341, 0; + mov.u32 %r10338, %r10341; + mov.u32 %r10339, %r10341; + mov.u32 %r10345, %r10341; + @%p2068 bra $L__BB1_1642; + + add.s32 %r7728, %r3360, -1; + clz.b32 %r7729, %r7728; + mov.u32 %r7730, 32; + sub.s32 %r10338, %r7730, %r7729; + shr.u32 %r7731, %r10337, 31; + add.s32 %r7732, %r7731, %r3360; + add.s32 %r10339, %r7732, -2; + mov.u32 %r10345, 1; + +$L__BB1_1642: + setp.ge.u32 %p2069, %r3333, %r6; + @%p2069 bra $L__BB1_1645; + + add.s32 %r7735, %r10333, %r1; + cvt.u64.u32 %rd1270, %r7735; + add.s64 %rd1271, %rd1270, %rd5; + shl.b64 %rd1272, %rd1271, 2; + add.s64 %rd1273, %rd3, %rd1272; + ld.global.u32 %r3366, [%rd1273]; + setp.eq.s32 %p2070, %r3366, 0; + @%p2070 bra $L__BB1_1645; + + and.b32 %r7736, %r3366, -2147483648; + abs.s32 %r7737, %r3366; + shl.b32 %r7738, %r7737, %r2355; + or.b32 %r10341, %r7738, %r7736; + +$L__BB1_1645: + shl.b32 %r7741, %r10341, 1; + shr.u32 %r7742, %r7741, %r2355; + and.b32 %r3369, %r7742, -2; + setp.eq.s32 %p2071, %r3369, 0; + mov.u32 %r10356, 0; + mov.u32 %r10342, %r10356; + mov.u32 %r10343, %r10356; + mov.u32 %r10361, %r10338; + @%p2071 bra $L__BB1_1647; + + or.b32 %r10345, %r10345, 2; + add.s32 %r7743, %r3369, -1; + clz.b32 %r7744, %r7743; + mov.u32 %r7745, 32; + sub.s32 %r10342, %r7745, %r7744; + max.s32 %r10361, %r10338, %r10342; + shr.u32 %r7746, %r10341, 31; + add.s32 %r7747, %r7746, %r3369; + add.s32 %r10343, %r7747, -2; + +$L__BB1_1647: + add.s32 %r10638, %r10333, 1; + add.s32 %r7752, %r10316, 1; + setp.ge.u32 %p2072, %r7752, %r5; + mov.u32 %r10357, %r10356; + mov.u32 %r10358, %r10356; + mov.u32 %r10359, %r10356; + @%p2072 bra $L__BB1_1658; + + cvt.u64.u32 %rd1274, %r10638; + add.s64 %rd1275, %rd1274, %rd5; + shl.b64 %rd1276, %rd1275, 2; + add.s64 %rd1277, %rd3, %rd1276; + ld.global.u32 %r3379, [%rd1277]; + setp.eq.s32 %p2073, %r3379, 0; + mov.u32 %r10357, 0; + mov.u32 %r10346, %r10357; + @%p2073 bra $L__BB1_1650; + + and.b32 %r7754, %r3379, -2147483648; + abs.s32 %r7755, %r3379; + shl.b32 %r7756, %r7755, %r2355; + or.b32 %r10346, %r7756, %r7754; + +$L__BB1_1650: + shl.b32 %r7759, %r10346, 1; + shr.u32 %r7760, %r7759, %r2355; + and.b32 %r3382, %r7760, -2; + setp.eq.s32 %p2074, %r3382, 0; + mov.u32 %r10359, %r10357; + @%p2074 bra $L__BB1_1652; + + or.b32 %r10345, %r10345, 4; + add.s32 %r7761, %r3382, -1; + clz.b32 %r7762, %r7761; + mov.u32 %r7763, 32; + sub.s32 %r10357, %r7763, %r7762; + max.s32 %r10361, %r10361, %r10357; + shr.u32 %r7764, %r10346, 31; + add.s32 %r7765, %r7764, %r3382; + add.s32 %r10359, %r7765, -2; + +$L__BB1_1652: + mov.u32 %r10356, 0; + mov.u32 %r10351, %r10356; + @%p2069 bra $L__BB1_1655; + + add.s32 %r7768, %r10638, %r1; + cvt.u64.u32 %rd1278, %r7768; + add.s64 %rd1279, %rd1278, %rd5; + shl.b64 %rd1280, %rd1279, 2; + add.s64 %rd1281, %rd3, %rd1280; + ld.global.u32 %r3391, [%rd1281]; + setp.eq.s32 %p2076, %r3391, 0; + @%p2076 bra $L__BB1_1655; + + and.b32 %r7769, %r3391, -2147483648; + abs.s32 %r7770, %r3391; + shl.b32 %r7771, %r7770, %r2355; + or.b32 %r10351, %r7771, %r7769; + +$L__BB1_1655: + shl.b32 %r7774, %r10351, 1; + shr.u32 %r7775, %r7774, %r2355; + and.b32 %r3394, %r7775, -2; + setp.eq.s32 %p2077, %r3394, 0; + mov.u32 %r10358, %r10356; + @%p2077 bra $L__BB1_1657; + + or.b32 %r10345, %r10345, 8; + add.s32 %r7776, %r3394, -1; + clz.b32 %r7777, %r7776; + mov.u32 %r7778, 32; + sub.s32 %r10356, %r7778, %r7777; + max.s32 %r10361, %r10361, %r10356; + shr.u32 %r7779, %r10351, 31; + add.s32 %r7780, %r7779, %r3394; + add.s32 %r10358, %r7780, -2; + +$L__BB1_1657: + add.s32 %r10638, %r10333, 2; + +$L__BB1_1658: + add.s32 %r7782, %r10345, -1; + and.b32 %r7783, %r7782, %r10345; + setp.ne.s32 %p2078, %r7783, 0; + mov.u32 %r10363, 0; + setp.gt.s32 %p2079, %r10335, 1; + and.pred %p2080, %p2079, %p2078; + selp.b32 %r7784, %r10335, 1, %p2080; + max.s32 %r3411, %r7784, %r10361; + sub.s32 %r3412, %r3411, %r7784; + setp.lt.s32 %p2081, %r3412, 1; + @%p2081 bra $L__BB1_1660; + + setp.eq.s32 %p2082, %r10338, %r10361; + selp.u32 %r7785, 1, 0, %p2082; + setp.eq.s32 %p2083, %r10342, %r10361; + selp.u32 %r7786, -1, 0, %p2083; + bfi.b32 %r7787, %r7786, %r7785, 1, 1; + setp.eq.s32 %p2084, %r10357, %r10361; + selp.u16 %rs992, 1, 0, %p2084; + mul.wide.u16 %r7788, %rs992, 4; + or.b32 %r7789, %r7787, %r7788; + setp.eq.s32 %p2085, %r10356, %r10361; + selp.u16 %rs993, 1, 0, %p2085; + mul.wide.u16 %r7790, %rs993, 8; + or.b32 %r10363, %r7789, %r7790; + +$L__BB1_1660: + shl.b32 %r7791, %r10345, 4; + shl.b32 %r7792, %r10332, 8; + or.b32 %r7793, %r7791, %r7792; + or.b32 %r7794, %r7793, %r10363; + mul.wide.u32 %rd1282, %r7794, 2; + add.s64 %rd1283, %rd72, %rd1282; + ld.global.u16 %rs426, [%rd1283]; + shr.u16 %rs994, %rs426, 4; + and.b16 %rs427, %rs994, 7; + setp.eq.s16 %p2086, %rs427, 0; + mov.u32 %r10375, %r10267; + @%p2086 bra $L__BB1_1667; + + cvt.u32.u16 %r10364, %rs427; + shr.u16 %rs995, %rs426, 8; + cvt.u32.u16 %r10365, %rs995; + +$L__BB1_1662: + mov.u32 %r3417, %r10364; + setp.gt.u32 %p2087, %r10264, 2879; + mov.u32 %r10375, 1; + @%p2087 bra $L__BB1_1667; + + mov.u32 %r7796, 8; + sub.s32 %r7797, %r7796, %r10266; + sub.s32 %r7798, %r7797, %r10265; + min.u32 %r7799, %r7798, %r3417; + setp.eq.s32 %p2088, %r7799, 32; + mov.u32 %r7800, -1; + shl.b32 %r7801, %r7800, %r7799; + not.b32 %r7802, %r7801; + selp.b32 %r7803, -1, %r7802, %p2088; + and.b32 %r7804, %r7803, %r10365; + shl.b32 %r7805, %r7804, %r10265; + cvt.u16.u32 %rs996, %r7805; + or.b16 %rs1322, %rs1322, %rs996; + add.s32 %r10265, %r7799, %r10265; + sub.s32 %r10364, %r3417, %r7799; + shr.u32 %r10365, %r10365, %r7799; + setp.gt.u32 %p2089, %r7798, %r3417; + @%p2089 bra $L__BB1_1666; + + setp.ne.s32 %p2090, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs997, %rs1322, 255; + setp.ne.s16 %p2091, %rs997, 127; + and.pred %p2092, %p2090, %p2091; + @%p2092 bra $L__BB1_1666; + + mov.u32 %r7808, 20548; + sub.s32 %r7809, %r7808, %r10264; + cvt.u64.u32 %rd1284, %r7809; + add.s64 %rd1285, %rd1284, %rd4; + add.s64 %rd1286, %rd1, %rd1285; + st.global.u8 [%rd1286], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2093, %rs997, 143; + selp.u32 %r10266, 1, 0, %p2093; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1666: + setp.ne.s32 %p2094, %r10364, 0; + mov.u32 %r10375, %r10267; + @%p2094 bra $L__BB1_1662; + +$L__BB1_1667: + setp.ne.s32 %p2095, %r10332, 0; + @%p2095 bra $L__BB1_1715; + + setp.eq.s32 %p2096, %r10345, 0; + add.s32 %r7810, %r9816, 17477; + cvt.u64.u32 %rd1287, %r7810; + add.s64 %rd1288, %rd1287, %rd4; + add.s64 %rd73, %rd1, %rd1288; + @%p2096 bra $L__BB1_1707; + + shl.b16 %rs1253, %rs1253, 1; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p2097, %r9825, 0; + mov.u32 %r10411, %r10033; + @%p2097 bra $L__BB1_1672; + + setp.gt.u32 %p2098, %r9816, 191; + mov.u32 %r10411, 1; + mov.u32 %r9825, 0; + @%p2098 bra $L__BB1_1672; + + st.global.u8 [%rd73], %rs1253; + add.s32 %r9816, %r9816, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9825, 8; + mov.u32 %r10411, %r10033; + +$L__BB1_1672: + setp.lt.u32 %p2099, %r10031, 3; + mov.u32 %r10379, 0; + @%p2099 bra $L__BB1_1675; + + setp.lt.u32 %p2100, %r10031, 6; + mov.u32 %r10379, 1; + @%p2100 bra $L__BB1_1675; + + setp.lt.u32 %p2101, %r10031, 9; + setp.eq.s32 %p2102, %r10031, 11; + selp.b32 %r7816, 4, 5, %p2102; + setp.lt.u32 %p2103, %r10031, 11; + selp.b32 %r7817, 3, %r7816, %p2103; + selp.b32 %r10379, 2, %r7817, %p2101; + +$L__BB1_1675: + setp.eq.s32 %p2104, %r10379, 0; + @%p2104 bra $L__BB1_1703; + + add.s32 %r3441, %r10379, -1; + and.b32 %r3442, %r10379, 3; + setp.eq.s32 %p2105, %r3442, 0; + mov.u32 %r10389, %r10379; + mov.u32 %r10390, %r10411; + @%p2105 bra $L__BB1_1688; + + mov.u32 %r7819, 1; + shl.b32 %r7820, %r7819, %r3441; + and.b32 %r7821, %r7820, %r10030; + setp.ne.s32 %p2106, %r7821, 0; + selp.u32 %r7822, 1, 0, %p2106; + cvt.u32.u16 %r7823, %rs1253; + bfi.b32 %r7824, %r7823, %r7822, 1, 8; + cvt.u16.u32 %rs1253, %r7824; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p2107, %r9825, 0; + mov.u32 %r10390, %r10411; + @%p2107 bra $L__BB1_1680; + + setp.gt.u32 %p2108, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r10390, %r7819; + @%p2108 bra $L__BB1_1680; + + add.s32 %r7828, %r9816, 17477; + cvt.u64.u32 %rd1289, %r7828; + add.s64 %rd1290, %rd1289, %rd4; + add.s64 %rd1291, %rd1, %rd1290; + st.global.u8 [%rd1291], %rs1253; + add.s32 %r9816, %r9816, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9825, 8; + mov.u32 %r10390, %r10411; + +$L__BB1_1680: + setp.eq.s32 %p2109, %r3442, 1; + mov.u32 %r10411, %r10390; + mov.u32 %r10389, %r3441; + @%p2109 bra $L__BB1_1688; + + add.s32 %r10389, %r10379, -2; + mov.u32 %r7829, 1; + shl.b32 %r7830, %r7829, %r10389; + and.b32 %r7831, %r7830, %r10030; + setp.ne.s32 %p2110, %r7831, 0; + selp.u32 %r7832, 1, 0, %p2110; + cvt.u32.u16 %r7833, %rs1253; + bfi.b32 %r7834, %r7833, %r7832, 1, 8; + cvt.u16.u32 %rs1253, %r7834; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p2111, %r9825, 0; + mov.u32 %r10385, %r10390; + @%p2111 bra $L__BB1_1684; + + setp.gt.u32 %p2112, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r10385, %r7829; + @%p2112 bra $L__BB1_1684; + + add.s32 %r7837, %r9816, 17477; + cvt.u64.u32 %rd1292, %r7837; + add.s64 %rd1293, %rd1292, %rd4; + add.s64 %rd1294, %rd1, %rd1293; + and.b16 %rs1004, %rs1253, 255; + st.global.u8 [%rd1294], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2113, %rs1004, 255; + selp.b32 %r9825, 7, 8, %p2113; + mov.u16 %rs1253, 0; + mov.u32 %r10385, %r10390; + +$L__BB1_1684: + setp.eq.s32 %p2114, %r3442, 2; + mov.u32 %r10411, %r10385; + mov.u32 %r10390, %r10385; + @%p2114 bra $L__BB1_1688; + + add.s32 %r10389, %r10379, -3; + mov.u32 %r7838, 1; + shl.b32 %r7839, %r7838, %r10389; + and.b32 %r7840, %r7839, %r10030; + setp.ne.s32 %p2115, %r7840, 0; + selp.u32 %r7841, 1, 0, %p2115; + cvt.u32.u16 %r7842, %rs1253; + bfi.b32 %r7843, %r7842, %r7841, 1, 8; + cvt.u16.u32 %rs1253, %r7843; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p2116, %r9825, 0; + mov.u32 %r10411, %r10385; + mov.u32 %r10390, %r10385; + @%p2116 bra $L__BB1_1688; + + setp.gt.u32 %p2117, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r10411, %r7838; + mov.u32 %r10390, %r7838; + @%p2117 bra $L__BB1_1688; + + add.s32 %r7848, %r9816, 17477; + cvt.u64.u32 %rd1295, %r7848; + add.s64 %rd1296, %rd1295, %rd4; + add.s64 %rd1297, %rd1, %rd1296; + and.b16 %rs1007, %rs1253, 255; + st.global.u8 [%rd1297], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2118, %rs1007, 255; + selp.b32 %r9825, 7, 8, %p2118; + mov.u16 %rs1253, 0; + mov.u32 %r10411, %r10385; + mov.u32 %r10390, %r10385; + +$L__BB1_1688: + setp.lt.u32 %p2119, %r3441, 3; + @%p2119 bra $L__BB1_1703; + + mov.u32 %r10411, %r10390; + +$L__BB1_1690: + add.s32 %r7849, %r10389, -1; + mov.u32 %r7850, 1; + shl.b32 %r7851, %r7850, %r7849; + and.b32 %r7852, %r7851, %r10030; + setp.ne.s32 %p2120, %r7852, 0; + selp.u32 %r7853, 1, 0, %p2120; + cvt.u32.u16 %r7854, %rs1253; + bfi.b32 %r10399, %r7854, %r7853, 1, 8; + add.s32 %r10398, %r9825, -1; + setp.ne.s32 %p2121, %r10398, 0; + mov.u32 %r10400, %r10411; + @%p2121 bra $L__BB1_1693; + + setp.gt.u32 %p2122, %r9816, 191; + mov.u32 %r10398, 0; + mov.u32 %r10400, %r7850; + @%p2122 bra $L__BB1_1693; + + cvt.u16.u32 %rs1008, %r10399; + and.b16 %rs1009, %rs1008, 255; + add.s32 %r7858, %r9816, 17477; + cvt.u64.u32 %rd1298, %r7858; + add.s64 %rd1299, %rd1298, %rd4; + add.s64 %rd1300, %rd1, %rd1299; + st.global.u8 [%rd1300], %rs1008; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2123, %rs1009, 255; + selp.b32 %r10398, 7, 8, %p2123; + mov.u32 %r10399, 0; + mov.u32 %r10400, %r10411; + +$L__BB1_1693: + add.s32 %r7859, %r10389, -2; + shl.b32 %r7861, %r7850, %r7859; + and.b32 %r7862, %r7861, %r10030; + setp.ne.s32 %p2124, %r7862, 0; + and.b32 %r7863, %r10399, 127; + selp.u32 %r7864, 1, 0, %p2124; + bfi.b32 %r10403, %r7863, %r7864, 1, 7; + add.s32 %r10402, %r10398, -1; + setp.ne.s32 %p2125, %r10402, 0; + mov.u32 %r10404, %r10400; + @%p2125 bra $L__BB1_1696; + + setp.gt.u32 %p2126, %r9816, 191; + mov.u32 %r10404, 1; + mov.u32 %r10402, 0; + @%p2126 bra $L__BB1_1696; + + cvt.u16.u32 %rs1010, %r10403; + and.b16 %rs1011, %rs1010, 255; + add.s32 %r7868, %r9816, 17477; + cvt.u64.u32 %rd1301, %r7868; + add.s64 %rd1302, %rd1301, %rd4; + add.s64 %rd1303, %rd1, %rd1302; + st.global.u8 [%rd1303], %rs1010; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2127, %rs1011, 255; + selp.b32 %r10402, 7, 8, %p2127; + mov.u32 %r10403, 0; + mov.u32 %r10404, %r10400; + +$L__BB1_1696: + add.s32 %r7869, %r10389, -3; + mov.u32 %r7870, 1; + shl.b32 %r7871, %r7870, %r7869; + and.b32 %r7872, %r7871, %r10030; + setp.ne.s32 %p2128, %r7872, 0; + and.b32 %r7873, %r10403, 127; + selp.u32 %r7874, 1, 0, %p2128; + bfi.b32 %r10407, %r7873, %r7874, 1, 7; + add.s32 %r10406, %r10402, -1; + setp.ne.s32 %p2129, %r10406, 0; + mov.u32 %r10408, %r10404; + @%p2129 bra $L__BB1_1699; + + setp.gt.u32 %p2130, %r9816, 191; + mov.u32 %r10406, 0; + mov.u32 %r10408, %r7870; + @%p2130 bra $L__BB1_1699; + + cvt.u16.u32 %rs1012, %r10407; + and.b16 %rs1013, %rs1012, 255; + add.s32 %r7878, %r9816, 17477; + cvt.u64.u32 %rd1304, %r7878; + add.s64 %rd1305, %rd1304, %rd4; + add.s64 %rd1306, %rd1, %rd1305; + st.global.u8 [%rd1306], %rs1012; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2131, %rs1013, 255; + selp.b32 %r10406, 7, 8, %p2131; + mov.u32 %r10407, 0; + mov.u32 %r10408, %r10404; + +$L__BB1_1699: + add.s32 %r10389, %r10389, -4; + shl.b32 %r7880, %r7870, %r10389; + and.b32 %r7881, %r7880, %r10030; + setp.ne.s32 %p2132, %r7881, 0; + and.b32 %r7882, %r10407, 127; + selp.u32 %r7883, 1, 0, %p2132; + bfi.b32 %r7884, %r7882, %r7883, 1, 15; + cvt.u16.u32 %rs1253, %r7884; + add.s32 %r9825, %r10406, -1; + setp.ne.s32 %p2133, %r9825, 0; + mov.u32 %r10411, %r10408; + @%p2133 bra $L__BB1_1702; + + setp.gt.u32 %p2134, %r9816, 191; + mov.u32 %r10411, 1; + mov.u32 %r9825, 0; + @%p2134 bra $L__BB1_1702; + + add.s32 %r7887, %r9816, 17477; + cvt.u64.u32 %rd1307, %r7887; + add.s64 %rd1308, %rd1307, %rd4; + add.s64 %rd1309, %rd1, %rd1308; + and.b16 %rs1015, %rs1253, 255; + st.global.u8 [%rd1309], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2135, %rs1015, 255; + selp.b32 %r9825, 7, 8, %p2135; + mov.u16 %rs1253, 0; + mov.u32 %r10411, %r10408; + +$L__BB1_1702: + setp.ne.s32 %p2136, %r10389, 0; + @%p2136 bra $L__BB1_1690; + +$L__BB1_1703: + add.s32 %r7889, %r10031, -1; + setp.eq.s32 %p2137, %r10031, 0; + mov.u32 %r10030, 0; + selp.b32 %r10031, 0, %r7889, %p2137; + setp.lt.u32 %p2138, %r10031, 3; + mov.u32 %r10415, %r10030; + @%p2138 bra $L__BB1_1706; + + setp.lt.u32 %p2139, %r10031, 6; + mov.u32 %r10415, 1; + @%p2139 bra $L__BB1_1706; + + setp.lt.u32 %p2140, %r10031, 9; + setp.eq.s32 %p2141, %r10031, 11; + selp.b32 %r7891, 4, 5, %p2141; + setp.lt.u32 %p2142, %r10031, 11; + selp.b32 %r7892, 3, %r7891, %p2142; + selp.b32 %r10415, 2, %r7892, %p2140; + +$L__BB1_1706: + mov.u32 %r7894, 1; + shl.b32 %r10032, %r7894, %r10415; + mov.u32 %r10033, %r10411; + bra.uni $L__BB1_1715; + +$L__BB1_1707: + add.s32 %r10030, %r10030, 1; + setp.lt.u32 %p2143, %r10030, %r10032; + @%p2143 bra $L__BB1_1715; + + shl.b16 %rs1016, %rs1253, 1; + or.b16 %rs1253, %rs1016, 1; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p2144, %r9825, 0; + mov.u32 %r10418, %r10033; + @%p2144 bra $L__BB1_1711; + bra.uni $L__BB1_1709; + +$L__BB1_1711: + add.s32 %r7898, %r10031, 1; + min.u32 %r10031, %r7898, 12; + setp.lt.u32 %p2147, %r10031, 3; + mov.u32 %r10030, 0; + mov.u32 %r10419, %r10030; + @%p2147 bra $L__BB1_1714; + + setp.lt.u32 %p2148, %r10031, 6; + mov.u32 %r10419, 1; + @%p2148 bra $L__BB1_1714; + + setp.lt.u32 %p2149, %r10031, 9; + setp.eq.s32 %p2150, %r10031, 11; + selp.b32 %r7900, 4, 5, %p2150; + setp.lt.u32 %p2151, %r10031, 11; + selp.b32 %r7901, 3, %r7900, %p2151; + selp.b32 %r10419, 2, %r7901, %p2149; + +$L__BB1_1714: + mov.u32 %r7903, 1; + shl.b32 %r10032, %r7903, %r10419; + mov.u32 %r10033, %r10418; + +$L__BB1_1715: + and.b16 %rs1019, %rs426, 15; + cvt.u32.u16 %r3525, %rs1019; + and.b32 %r7904, %r10345, 1; + setp.eq.b32 %p2152, %r7904, 1; + mov.pred %p2153, 0; + xor.pred %p2154, %p2152, %p2153; + not.pred %p2155, %p2154; + mov.u32 %r10440, %r10485; + @%p2155 bra $L__BB1_1722; + + and.b32 %r7905, %r3525, 1; + sub.s32 %r10426, %r3411, %r7905; + setp.eq.s32 %p2156, %r10426, 0; + mov.u32 %r10440, %r10485; + @%p2156 bra $L__BB1_1722; + + mov.u32 %r7906, -1; + shl.b32 %r7907, %r7906, %r10426; + not.b32 %r7908, %r7907; + and.b32 %r10427, %r10339, %r7908; + +$L__BB1_1718: + setp.gt.u32 %p2157, %r10451, 17476; + mov.u32 %r10440, 1; + @%p2157 bra $L__BB1_1722; + + sub.s32 %r7910, %r10452, %r10453; + min.u32 %r7911, %r7910, %r10426; + setp.eq.s32 %p2158, %r7911, 32; + mov.u32 %r7912, -1; + shl.b32 %r7913, %r7912, %r7911; + not.b32 %r7914, %r7913; + selp.b32 %r7915, -1, %r7914, %p2158; + and.b32 %r7916, %r7915, %r10427; + shl.b32 %r7917, %r7916, %r10453; + or.b32 %r10454, %r7917, %r10454; + add.s32 %r10453, %r7911, %r10453; + shr.u32 %r10427, %r10427, %r7911; + sub.s32 %r10426, %r10426, %r7911; + setp.lt.u32 %p2159, %r10453, %r10452; + @%p2159 bra $L__BB1_1721; + + cvt.u64.u32 %rd1310, %r10451; + add.s64 %rd1311, %rd1310, %rd4; + add.s64 %rd1312, %rd1, %rd1311; + st.global.u8 [%rd1312], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p2160, %r10454, 255; + selp.b32 %r10452, 7, 8, %p2160; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1721: + setp.ne.s32 %p2161, %r10426, 0; + mov.u32 %r10440, %r10485; + @%p2161 bra $L__BB1_1718; + +$L__BB1_1722: + and.b32 %r3549, %r10345, 2; + setp.eq.s32 %p2162, %r3549, 0; + mov.u32 %r10455, %r10440; + @%p2162 bra $L__BB1_1729; + + shr.u32 %r7920, %r3525, 1; + and.b32 %r7921, %r7920, 1; + sub.s32 %r10441, %r3411, %r7921; + setp.eq.s32 %p2163, %r10441, 0; + mov.u32 %r10455, %r10440; + @%p2163 bra $L__BB1_1729; + + mov.u32 %r7922, -1; + shl.b32 %r7923, %r7922, %r10441; + not.b32 %r7924, %r7923; + and.b32 %r10442, %r10343, %r7924; + +$L__BB1_1725: + setp.gt.u32 %p2164, %r10451, 17476; + mov.u32 %r10455, 1; + @%p2164 bra $L__BB1_1729; + + sub.s32 %r7926, %r10452, %r10453; + min.u32 %r7927, %r7926, %r10441; + setp.eq.s32 %p2165, %r7927, 32; + mov.u32 %r7928, -1; + shl.b32 %r7929, %r7928, %r7927; + not.b32 %r7930, %r7929; + selp.b32 %r7931, -1, %r7930, %p2165; + and.b32 %r7932, %r7931, %r10442; + shl.b32 %r7933, %r7932, %r10453; + or.b32 %r10454, %r7933, %r10454; + add.s32 %r10453, %r7927, %r10453; + shr.u32 %r10442, %r10442, %r7927; + sub.s32 %r10441, %r10441, %r7927; + setp.lt.u32 %p2166, %r10453, %r10452; + @%p2166 bra $L__BB1_1728; + + cvt.u64.u32 %rd1313, %r10451; + add.s64 %rd1314, %rd1313, %rd4; + add.s64 %rd1315, %rd1, %rd1314; + st.global.u8 [%rd1315], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p2167, %r10454, 255; + selp.b32 %r10452, 7, 8, %p2167; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1728: + setp.ne.s32 %p2168, %r10441, 0; + mov.u32 %r10455, %r10440; + @%p2168 bra $L__BB1_1725; + +$L__BB1_1729: + and.b32 %r3573, %r10345, 4; + setp.eq.s32 %p2169, %r3573, 0; + mov.u32 %r10470, %r10455; + @%p2169 bra $L__BB1_1736; + + shr.u32 %r7936, %r3525, 2; + and.b32 %r7937, %r7936, 1; + sub.s32 %r10456, %r3411, %r7937; + setp.eq.s32 %p2170, %r10456, 0; + mov.u32 %r10470, %r10455; + @%p2170 bra $L__BB1_1736; + + mov.u32 %r7938, -1; + shl.b32 %r7939, %r7938, %r10456; + not.b32 %r7940, %r7939; + and.b32 %r10457, %r10359, %r7940; + +$L__BB1_1732: + setp.gt.u32 %p2171, %r10451, 17476; + mov.u32 %r10470, 1; + @%p2171 bra $L__BB1_1736; + + sub.s32 %r7942, %r10452, %r10453; + min.u32 %r7943, %r7942, %r10456; + setp.eq.s32 %p2172, %r7943, 32; + mov.u32 %r7944, -1; + shl.b32 %r7945, %r7944, %r7943; + not.b32 %r7946, %r7945; + selp.b32 %r7947, -1, %r7946, %p2172; + and.b32 %r7948, %r7947, %r10457; + shl.b32 %r7949, %r7948, %r10453; + or.b32 %r10454, %r7949, %r10454; + add.s32 %r10453, %r7943, %r10453; + shr.u32 %r10457, %r10457, %r7943; + sub.s32 %r10456, %r10456, %r7943; + setp.lt.u32 %p2173, %r10453, %r10452; + @%p2173 bra $L__BB1_1735; + + cvt.u64.u32 %rd1316, %r10451; + add.s64 %rd1317, %rd1316, %rd4; + add.s64 %rd1318, %rd1, %rd1317; + st.global.u8 [%rd1318], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p2174, %r10454, 255; + selp.b32 %r10452, 7, 8, %p2174; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1735: + setp.ne.s32 %p2175, %r10456, 0; + mov.u32 %r10470, %r10455; + @%p2175 bra $L__BB1_1732; + +$L__BB1_1736: + and.b32 %r3597, %r10345, 8; + setp.eq.s32 %p2176, %r3597, 0; + mov.u32 %r10485, %r10470; + @%p2176 bra $L__BB1_1743; + + shr.u32 %r7952, %r3525, 3; + sub.s32 %r10471, %r3411, %r7952; + setp.eq.s32 %p2177, %r10471, 0; + mov.u32 %r10485, %r10470; + @%p2177 bra $L__BB1_1743; + + mov.u32 %r7953, -1; + shl.b32 %r7954, %r7953, %r10471; + not.b32 %r7955, %r7954; + and.b32 %r10472, %r10358, %r7955; + +$L__BB1_1739: + setp.gt.u32 %p2178, %r10451, 17476; + mov.u32 %r10485, 1; + @%p2178 bra $L__BB1_1743; + + sub.s32 %r7957, %r10452, %r10453; + min.u32 %r7958, %r7957, %r10471; + setp.eq.s32 %p2179, %r7958, 32; + mov.u32 %r7959, -1; + shl.b32 %r7960, %r7959, %r7958; + not.b32 %r7961, %r7960; + selp.b32 %r7962, -1, %r7961, %p2179; + and.b32 %r7963, %r7962, %r10472; + shl.b32 %r7964, %r7963, %r10453; + or.b32 %r10454, %r7964, %r10454; + add.s32 %r10453, %r7958, %r10453; + shr.u32 %r10472, %r10472, %r7958; + sub.s32 %r10471, %r10471, %r7958; + setp.lt.u32 %p2180, %r10453, %r10452; + @%p2180 bra $L__BB1_1742; + + cvt.u64.u32 %rd1319, %r10451; + add.s64 %rd1320, %rd1319, %rd4; + add.s64 %rd1321, %rd1, %rd1320; + st.global.u8 [%rd1321], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p2181, %r10454, 255; + selp.b32 %r10452, 7, 8, %p2181; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1742: + setp.ne.s32 %p2182, %r10471, 0; + mov.u32 %r10485, %r10470; + @%p2182 bra $L__BB1_1739; + +$L__BB1_1743: + add.s32 %r3621, %r4095, %r10334; + ld.shared.u8 %rs1020, [%r3621]; + mov.u32 %r10332, 0; + cvt.u32.u16 %r7970, %rs1020; + and.b32 %r7971, %r7970, 255; + and.b32 %r7972, %r10342, 255; + setp.lt.u32 %p2183, %r7972, %r7971; + cvt.u16.u32 %rs1021, %r10342; + selp.b16 %rs1022, %rs1020, %rs1021, %p2183; + st.shared.u8 [%r3621], %rs1022; + ld.shared.u8 %rs448, [%r3621+2]; + ld.shared.u8 %rs1023, [%r3621+1]; + setp.gt.u16 %p2184, %rs1023, %rs448; + add.s32 %r10637, %r10334, 1; + add.s32 %r7973, %r10334, 2; + selp.b32 %r7974, %r10637, %r7973, %p2184; + add.s32 %r7975, %r4095, %r7974; + ld.shared.u8 %rs449, [%r7975]; + cvt.u32.u16 %r7976, %rs449; + and.b32 %r7977, %r7976, 255; + add.s32 %r10335, %r7977, -1; + cvt.u16.u32 %rs450, %r10356; + cvt.u16.u32 %rs1024, %r3549; + shr.u16 %rs1025, %rs1024, 1; + mov.u32 %r7978, _ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val; + add.s32 %r3624, %r7978, %r10336; + st.shared.u8 [%r3621+1], %r10356; + ld.shared.u8 %rs1026, [%r3624]; + or.b16 %rs1027, %rs1026, %rs1025; + st.shared.u8 [%r3624], %rs1027; + add.s32 %r10336, %r10336, 1; + ld.shared.u8 %rs451, [%r3624+1]; + ld.shared.u8 %r3626, [%r3624+2]; + shr.u32 %r3627, %r3597, 3; + st.shared.u8 [%r3624+1], %r3627; + add.s32 %r7979, %r10316, 2; + setp.ge.u32 %p2185, %r7979, %r5; + mov.u32 %r10655, %r10332; + @%p2185 bra $L__BB1_1850; + + cvt.u64.u32 %rd1322, %r10638; + add.s64 %rd1323, %rd1322, %rd5; + shl.b64 %rd1324, %rd1323, 2; + add.s64 %rd1325, %rd3, %rd1324; + ld.global.u32 %r3628, [%rd1325]; + setp.eq.s32 %p2186, %r3628, 0; + mov.u32 %r10487, 0; + mov.u32 %r10486, %r10487; + @%p2186 bra $L__BB1_1746; + + and.b32 %r7981, %r3628, -2147483648; + abs.s32 %r7982, %r3628; + shl.b32 %r7983, %r7982, %r2355; + or.b32 %r10486, %r7983, %r7981; + +$L__BB1_1746: + shl.b32 %r7987, %r10486, 1; + shr.u32 %r7988, %r7987, %r2355; + and.b32 %r3631, %r7988, -2; + setp.eq.s32 %p2187, %r3631, 0; + mov.u32 %r10488, %r10487; + mov.u32 %r10494, %r10487; + @%p2187 bra $L__BB1_1748; + + add.s32 %r7990, %r3631, -1; + clz.b32 %r7991, %r7990; + mov.u32 %r7992, 32; + sub.s32 %r10487, %r7992, %r7991; + shr.u32 %r7993, %r10486, 31; + add.s32 %r7994, %r7993, %r3631; + add.s32 %r10488, %r7994, -2; + mov.u32 %r10494, 1; + +$L__BB1_1748: + mov.u32 %r10491, 0; + mov.u32 %r10490, %r10491; + @%p2069 bra $L__BB1_1751; + + add.s32 %r7997, %r10638, %r1; + cvt.u64.u32 %rd1326, %r7997; + add.s64 %rd1327, %rd1326, %rd5; + shl.b64 %rd1328, %rd1327, 2; + add.s64 %rd1329, %rd3, %rd1328; + ld.global.u32 %r3637, [%rd1329]; + setp.eq.s32 %p2189, %r3637, 0; + @%p2189 bra $L__BB1_1751; + + and.b32 %r7998, %r3637, -2147483648; + abs.s32 %r7999, %r3637; + shl.b32 %r8000, %r7999, %r2355; + or.b32 %r10490, %r8000, %r7998; + +$L__BB1_1751: + shl.b32 %r8003, %r10490, 1; + shr.u32 %r8004, %r8003, %r2355; + and.b32 %r3640, %r8004, -2; + setp.eq.s32 %p2190, %r3640, 0; + mov.u32 %r10492, %r10491; + mov.u32 %r10510, %r10487; + @%p2190 bra $L__BB1_1753; + + or.b32 %r10494, %r10494, 2; + add.s32 %r8005, %r3640, -1; + clz.b32 %r8006, %r8005; + mov.u32 %r8007, 32; + sub.s32 %r10491, %r8007, %r8006; + max.s32 %r10510, %r10487, %r10491; + shr.u32 %r8008, %r10490, 31; + add.s32 %r8009, %r8008, %r3640; + add.s32 %r10492, %r8009, -2; + +$L__BB1_1753: + add.s32 %r10509, %r10638, 1; + add.s32 %r8014, %r10316, 3; + setp.ge.u32 %p2191, %r8014, %r5; + mov.u32 %r10512, 0; + mov.u32 %r10505, %r10512; + mov.u32 %r10506, %r10512; + mov.u32 %r10507, %r10512; + mov.u32 %r10508, %r10512; + @%p2191 bra $L__BB1_1764; + + cvt.u64.u32 %rd1330, %r10509; + add.s64 %rd1331, %rd1330, %rd5; + shl.b64 %rd1332, %rd1331, 2; + add.s64 %rd1333, %rd3, %rd1332; + ld.global.u32 %r3650, [%rd1333]; + setp.eq.s32 %p2192, %r3650, 0; + mov.u32 %r10506, 0; + mov.u32 %r10495, %r10506; + @%p2192 bra $L__BB1_1756; + + and.b32 %r8016, %r3650, -2147483648; + abs.s32 %r8017, %r3650; + shl.b32 %r8018, %r8017, %r2355; + or.b32 %r10495, %r8018, %r8016; + +$L__BB1_1756: + shl.b32 %r8021, %r10495, 1; + shr.u32 %r8022, %r8021, %r2355; + and.b32 %r3653, %r8022, -2; + setp.eq.s32 %p2193, %r3653, 0; + mov.u32 %r10508, %r10506; + @%p2193 bra $L__BB1_1758; + + or.b32 %r10494, %r10494, 4; + add.s32 %r8023, %r3653, -1; + clz.b32 %r8024, %r8023; + mov.u32 %r8025, 32; + sub.s32 %r10506, %r8025, %r8024; + max.s32 %r10510, %r10510, %r10506; + shr.u32 %r8026, %r10495, 31; + add.s32 %r8027, %r8026, %r3653; + add.s32 %r10508, %r8027, -2; + +$L__BB1_1758: + mov.u32 %r10505, 0; + mov.u32 %r10500, %r10505; + @%p2069 bra $L__BB1_1761; + + add.s32 %r8030, %r10509, %r1; + cvt.u64.u32 %rd1334, %r8030; + add.s64 %rd1335, %rd1334, %rd5; + shl.b64 %rd1336, %rd1335, 2; + add.s64 %rd1337, %rd3, %rd1336; + ld.global.u32 %r3662, [%rd1337]; + setp.eq.s32 %p2195, %r3662, 0; + @%p2195 bra $L__BB1_1761; + + and.b32 %r8031, %r3662, -2147483648; + abs.s32 %r8032, %r3662; + shl.b32 %r8033, %r8032, %r2355; + or.b32 %r10500, %r8033, %r8031; + +$L__BB1_1761: + shl.b32 %r8036, %r10500, 1; + shr.u32 %r8037, %r8036, %r2355; + and.b32 %r3665, %r8037, -2; + setp.eq.s32 %p2196, %r3665, 0; + mov.u32 %r10507, %r10505; + @%p2196 bra $L__BB1_1763; + + or.b32 %r10494, %r10494, 8; + add.s32 %r8038, %r3665, -1; + clz.b32 %r8039, %r8038; + mov.u32 %r8040, 32; + sub.s32 %r10505, %r8040, %r8039; + max.s32 %r10510, %r10510, %r10505; + shr.u32 %r8041, %r10500, 31; + add.s32 %r8042, %r8041, %r3665; + add.s32 %r10507, %r8042, -2; + +$L__BB1_1763: + add.s32 %r10509, %r10638, 2; + +$L__BB1_1764: + mov.u32 %r10638, %r10509; + shr.u32 %r8044, %r3597, 2; + shr.u32 %r8045, %r3573, 1; + or.b32 %r8046, %r8044, %r8045; + cvt.u32.u16 %r8047, %rs451; + and.b32 %r8048, %r8047, 255; + shl.b32 %r8049, %r3626, 2; + add.s32 %r8050, %r8049, %r8048; + or.b32 %r3682, %r8046, %r8050; + add.s32 %r8051, %r10494, -1; + and.b32 %r8052, %r8051, %r10494; + setp.ne.s32 %p2197, %r8052, 0; + setp.gt.u16 %p2198, %rs449, 2; + and.pred %p2199, %p2198, %p2197; + selp.b32 %r8053, %r10335, 1, %p2199; + max.s32 %r3683, %r8053, %r10510; + sub.s32 %r10655, %r3683, %r8053; + setp.lt.s32 %p2200, %r10655, 1; + @%p2200 bra $L__BB1_1766; + + setp.eq.s32 %p2201, %r10487, %r10510; + selp.u32 %r8054, 1, 0, %p2201; + setp.eq.s32 %p2202, %r10491, %r10510; + selp.u32 %r8055, -1, 0, %p2202; + bfi.b32 %r8056, %r8055, %r8054, 1, 1; + setp.eq.s32 %p2203, %r10506, %r10510; + selp.u16 %rs1029, 1, 0, %p2203; + mul.wide.u16 %r8057, %rs1029, 4; + or.b32 %r8058, %r8056, %r8057; + setp.eq.s32 %p2204, %r10505, %r10510; + selp.u16 %rs1030, 1, 0, %p2204; + mul.wide.u16 %r8059, %rs1030, 8; + or.b32 %r10512, %r8058, %r8059; + +$L__BB1_1766: + shl.b32 %r8060, %r10494, 4; + shl.b32 %r8061, %r3682, 8; + or.b32 %r8062, %r8060, %r8061; + or.b32 %r8063, %r8062, %r10512; + mul.wide.u32 %rd1339, %r8063, 2; + add.s64 %rd1340, %rd72, %rd1339; + ld.global.u16 %rs452, [%rd1340]; + shr.u16 %rs1031, %rs452, 4; + and.b16 %rs453, %rs1031, 7; + setp.eq.s16 %p2205, %rs453, 0; + mov.u32 %r10524, %r10375; + @%p2205 bra $L__BB1_1773; + + cvt.u32.u16 %r10513, %rs453; + shr.u16 %rs1032, %rs452, 8; + cvt.u32.u16 %r10514, %rs1032; + +$L__BB1_1768: + mov.u32 %r3689, %r10513; + setp.gt.u32 %p2206, %r10264, 2879; + mov.u32 %r10524, 1; + @%p2206 bra $L__BB1_1773; + + mov.u32 %r8065, 8; + sub.s32 %r8066, %r8065, %r10266; + sub.s32 %r8067, %r8066, %r10265; + min.u32 %r8068, %r8067, %r3689; + setp.eq.s32 %p2207, %r8068, 32; + mov.u32 %r8069, -1; + shl.b32 %r8070, %r8069, %r8068; + not.b32 %r8071, %r8070; + selp.b32 %r8072, -1, %r8071, %p2207; + and.b32 %r8073, %r8072, %r10514; + shl.b32 %r8074, %r8073, %r10265; + cvt.u16.u32 %rs1033, %r8074; + or.b16 %rs1322, %rs1322, %rs1033; + add.s32 %r10265, %r8068, %r10265; + sub.s32 %r10513, %r3689, %r8068; + shr.u32 %r10514, %r10514, %r8068; + setp.gt.u32 %p2208, %r8067, %r3689; + @%p2208 bra $L__BB1_1772; + + setp.ne.s32 %p2209, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs1034, %rs1322, 255; + setp.ne.s16 %p2210, %rs1034, 127; + and.pred %p2211, %p2209, %p2210; + @%p2211 bra $L__BB1_1772; + + mov.u32 %r8077, 20548; + sub.s32 %r8078, %r8077, %r10264; + cvt.u64.u32 %rd1341, %r8078; + add.s64 %rd1342, %rd1341, %rd4; + add.s64 %rd1343, %rd1, %rd1342; + st.global.u8 [%rd1343], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2212, %rs1034, 143; + selp.u32 %r10266, 1, 0, %p2212; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1772: + setp.ne.s32 %p2213, %r10513, 0; + mov.u32 %r10524, %r10375; + @%p2213 bra $L__BB1_1768; + +$L__BB1_1773: + setp.ne.s32 %p2214, %r3682, 0; + @%p2214 bra $L__BB1_1821; + + setp.eq.s32 %p2215, %r10494, 0; + add.s32 %r8079, %r9816, 17477; + cvt.u64.u32 %rd1344, %r8079; + add.s64 %rd1345, %rd1344, %rd4; + add.s64 %rd74, %rd1, %rd1345; + @%p2215 bra $L__BB1_1813; + + shl.b16 %rs1253, %rs1253, 1; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p2216, %r9825, 0; + mov.u32 %r10560, %r10033; + @%p2216 bra $L__BB1_1778; + + setp.gt.u32 %p2217, %r9816, 191; + mov.u32 %r10560, 1; + mov.u32 %r9825, 0; + @%p2217 bra $L__BB1_1778; + + st.global.u8 [%rd74], %rs1253; + add.s32 %r9816, %r9816, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9825, 8; + mov.u32 %r10560, %r10033; + +$L__BB1_1778: + setp.lt.u32 %p2218, %r10031, 3; + mov.u32 %r10528, 0; + @%p2218 bra $L__BB1_1781; + + setp.lt.u32 %p2219, %r10031, 6; + mov.u32 %r10528, 1; + @%p2219 bra $L__BB1_1781; + + setp.lt.u32 %p2220, %r10031, 9; + setp.eq.s32 %p2221, %r10031, 11; + selp.b32 %r8085, 4, 5, %p2221; + setp.lt.u32 %p2222, %r10031, 11; + selp.b32 %r8086, 3, %r8085, %p2222; + selp.b32 %r10528, 2, %r8086, %p2220; + +$L__BB1_1781: + setp.eq.s32 %p2223, %r10528, 0; + @%p2223 bra $L__BB1_1809; + + add.s32 %r3713, %r10528, -1; + and.b32 %r3714, %r10528, 3; + setp.eq.s32 %p2224, %r3714, 0; + mov.u32 %r10538, %r10528; + mov.u32 %r10539, %r10560; + @%p2224 bra $L__BB1_1794; + + mov.u32 %r8088, 1; + shl.b32 %r8089, %r8088, %r3713; + and.b32 %r8090, %r8089, %r10030; + setp.ne.s32 %p2225, %r8090, 0; + selp.u32 %r8091, 1, 0, %p2225; + cvt.u32.u16 %r8092, %rs1253; + bfi.b32 %r8093, %r8092, %r8091, 1, 8; + cvt.u16.u32 %rs1253, %r8093; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p2226, %r9825, 0; + mov.u32 %r10539, %r10560; + @%p2226 bra $L__BB1_1786; + + setp.gt.u32 %p2227, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r10539, %r8088; + @%p2227 bra $L__BB1_1786; + + add.s32 %r8097, %r9816, 17477; + cvt.u64.u32 %rd1346, %r8097; + add.s64 %rd1347, %rd1346, %rd4; + add.s64 %rd1348, %rd1, %rd1347; + st.global.u8 [%rd1348], %rs1253; + add.s32 %r9816, %r9816, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9825, 8; + mov.u32 %r10539, %r10560; + +$L__BB1_1786: + setp.eq.s32 %p2228, %r3714, 1; + mov.u32 %r10560, %r10539; + mov.u32 %r10538, %r3713; + @%p2228 bra $L__BB1_1794; + + add.s32 %r10538, %r10528, -2; + mov.u32 %r8098, 1; + shl.b32 %r8099, %r8098, %r10538; + and.b32 %r8100, %r8099, %r10030; + setp.ne.s32 %p2229, %r8100, 0; + selp.u32 %r8101, 1, 0, %p2229; + cvt.u32.u16 %r8102, %rs1253; + bfi.b32 %r8103, %r8102, %r8101, 1, 8; + cvt.u16.u32 %rs1253, %r8103; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p2230, %r9825, 0; + mov.u32 %r10534, %r10539; + @%p2230 bra $L__BB1_1790; + + setp.gt.u32 %p2231, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r10534, %r8098; + @%p2231 bra $L__BB1_1790; + + add.s32 %r8106, %r9816, 17477; + cvt.u64.u32 %rd1349, %r8106; + add.s64 %rd1350, %rd1349, %rd4; + add.s64 %rd1351, %rd1, %rd1350; + and.b16 %rs1041, %rs1253, 255; + st.global.u8 [%rd1351], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2232, %rs1041, 255; + selp.b32 %r9825, 7, 8, %p2232; + mov.u16 %rs1253, 0; + mov.u32 %r10534, %r10539; + +$L__BB1_1790: + setp.eq.s32 %p2233, %r3714, 2; + mov.u32 %r10560, %r10534; + mov.u32 %r10539, %r10534; + @%p2233 bra $L__BB1_1794; + + add.s32 %r10538, %r10528, -3; + mov.u32 %r8107, 1; + shl.b32 %r8108, %r8107, %r10538; + and.b32 %r8109, %r8108, %r10030; + setp.ne.s32 %p2234, %r8109, 0; + selp.u32 %r8110, 1, 0, %p2234; + cvt.u32.u16 %r8111, %rs1253; + bfi.b32 %r8112, %r8111, %r8110, 1, 8; + cvt.u16.u32 %rs1253, %r8112; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p2235, %r9825, 0; + mov.u32 %r10560, %r10534; + mov.u32 %r10539, %r10534; + @%p2235 bra $L__BB1_1794; + + setp.gt.u32 %p2236, %r9816, 191; + mov.u32 %r9825, 0; + mov.u32 %r10560, %r8107; + mov.u32 %r10539, %r8107; + @%p2236 bra $L__BB1_1794; + + add.s32 %r8117, %r9816, 17477; + cvt.u64.u32 %rd1352, %r8117; + add.s64 %rd1353, %rd1352, %rd4; + add.s64 %rd1354, %rd1, %rd1353; + and.b16 %rs1044, %rs1253, 255; + st.global.u8 [%rd1354], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2237, %rs1044, 255; + selp.b32 %r9825, 7, 8, %p2237; + mov.u16 %rs1253, 0; + mov.u32 %r10560, %r10534; + mov.u32 %r10539, %r10534; + +$L__BB1_1794: + setp.lt.u32 %p2238, %r3713, 3; + @%p2238 bra $L__BB1_1809; + + mov.u32 %r10560, %r10539; + +$L__BB1_1796: + add.s32 %r8118, %r10538, -1; + mov.u32 %r8119, 1; + shl.b32 %r8120, %r8119, %r8118; + and.b32 %r8121, %r8120, %r10030; + setp.ne.s32 %p2239, %r8121, 0; + selp.u32 %r8122, 1, 0, %p2239; + cvt.u32.u16 %r8123, %rs1253; + bfi.b32 %r10548, %r8123, %r8122, 1, 8; + add.s32 %r10547, %r9825, -1; + setp.ne.s32 %p2240, %r10547, 0; + mov.u32 %r10549, %r10560; + @%p2240 bra $L__BB1_1799; + + setp.gt.u32 %p2241, %r9816, 191; + mov.u32 %r10547, 0; + mov.u32 %r10549, %r8119; + @%p2241 bra $L__BB1_1799; + + cvt.u16.u32 %rs1045, %r10548; + and.b16 %rs1046, %rs1045, 255; + add.s32 %r8127, %r9816, 17477; + cvt.u64.u32 %rd1355, %r8127; + add.s64 %rd1356, %rd1355, %rd4; + add.s64 %rd1357, %rd1, %rd1356; + st.global.u8 [%rd1357], %rs1045; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2242, %rs1046, 255; + selp.b32 %r10547, 7, 8, %p2242; + mov.u32 %r10548, 0; + mov.u32 %r10549, %r10560; + +$L__BB1_1799: + add.s32 %r8128, %r10538, -2; + shl.b32 %r8130, %r8119, %r8128; + and.b32 %r8131, %r8130, %r10030; + setp.ne.s32 %p2243, %r8131, 0; + and.b32 %r8132, %r10548, 127; + selp.u32 %r8133, 1, 0, %p2243; + bfi.b32 %r10552, %r8132, %r8133, 1, 7; + add.s32 %r10551, %r10547, -1; + setp.ne.s32 %p2244, %r10551, 0; + mov.u32 %r10553, %r10549; + @%p2244 bra $L__BB1_1802; + + setp.gt.u32 %p2245, %r9816, 191; + mov.u32 %r10553, 1; + mov.u32 %r10551, 0; + @%p2245 bra $L__BB1_1802; + + cvt.u16.u32 %rs1047, %r10552; + and.b16 %rs1048, %rs1047, 255; + add.s32 %r8137, %r9816, 17477; + cvt.u64.u32 %rd1358, %r8137; + add.s64 %rd1359, %rd1358, %rd4; + add.s64 %rd1360, %rd1, %rd1359; + st.global.u8 [%rd1360], %rs1047; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2246, %rs1048, 255; + selp.b32 %r10551, 7, 8, %p2246; + mov.u32 %r10552, 0; + mov.u32 %r10553, %r10549; + +$L__BB1_1802: + add.s32 %r8138, %r10538, -3; + mov.u32 %r8139, 1; + shl.b32 %r8140, %r8139, %r8138; + and.b32 %r8141, %r8140, %r10030; + setp.ne.s32 %p2247, %r8141, 0; + and.b32 %r8142, %r10552, 127; + selp.u32 %r8143, 1, 0, %p2247; + bfi.b32 %r10556, %r8142, %r8143, 1, 7; + add.s32 %r10555, %r10551, -1; + setp.ne.s32 %p2248, %r10555, 0; + mov.u32 %r10557, %r10553; + @%p2248 bra $L__BB1_1805; + + setp.gt.u32 %p2249, %r9816, 191; + mov.u32 %r10555, 0; + mov.u32 %r10557, %r8139; + @%p2249 bra $L__BB1_1805; + + cvt.u16.u32 %rs1049, %r10556; + and.b16 %rs1050, %rs1049, 255; + add.s32 %r8147, %r9816, 17477; + cvt.u64.u32 %rd1361, %r8147; + add.s64 %rd1362, %rd1361, %rd4; + add.s64 %rd1363, %rd1, %rd1362; + st.global.u8 [%rd1363], %rs1049; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2250, %rs1050, 255; + selp.b32 %r10555, 7, 8, %p2250; + mov.u32 %r10556, 0; + mov.u32 %r10557, %r10553; + +$L__BB1_1805: + add.s32 %r10538, %r10538, -4; + shl.b32 %r8149, %r8139, %r10538; + and.b32 %r8150, %r8149, %r10030; + setp.ne.s32 %p2251, %r8150, 0; + and.b32 %r8151, %r10556, 127; + selp.u32 %r8152, 1, 0, %p2251; + bfi.b32 %r8153, %r8151, %r8152, 1, 15; + cvt.u16.u32 %rs1253, %r8153; + add.s32 %r9825, %r10555, -1; + setp.ne.s32 %p2252, %r9825, 0; + mov.u32 %r10560, %r10557; + @%p2252 bra $L__BB1_1808; + + setp.gt.u32 %p2253, %r9816, 191; + mov.u32 %r10560, 1; + mov.u32 %r9825, 0; + @%p2253 bra $L__BB1_1808; + + add.s32 %r8156, %r9816, 17477; + cvt.u64.u32 %rd1364, %r8156; + add.s64 %rd1365, %rd1364, %rd4; + add.s64 %rd1366, %rd1, %rd1365; + and.b16 %rs1052, %rs1253, 255; + st.global.u8 [%rd1366], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2254, %rs1052, 255; + selp.b32 %r9825, 7, 8, %p2254; + mov.u16 %rs1253, 0; + mov.u32 %r10560, %r10557; + +$L__BB1_1808: + setp.ne.s32 %p2255, %r10538, 0; + @%p2255 bra $L__BB1_1796; + +$L__BB1_1809: + add.s32 %r8158, %r10031, -1; + setp.eq.s32 %p2256, %r10031, 0; + mov.u32 %r10030, 0; + selp.b32 %r10031, 0, %r8158, %p2256; + setp.lt.u32 %p2257, %r10031, 3; + mov.u32 %r10564, %r10030; + @%p2257 bra $L__BB1_1812; + + setp.lt.u32 %p2258, %r10031, 6; + mov.u32 %r10564, 1; + @%p2258 bra $L__BB1_1812; + + setp.lt.u32 %p2259, %r10031, 9; + setp.eq.s32 %p2260, %r10031, 11; + selp.b32 %r8160, 4, 5, %p2260; + setp.lt.u32 %p2261, %r10031, 11; + selp.b32 %r8161, 3, %r8160, %p2261; + selp.b32 %r10564, 2, %r8161, %p2259; + +$L__BB1_1812: + mov.u32 %r8163, 1; + shl.b32 %r10032, %r8163, %r10564; + mov.u32 %r10033, %r10560; + bra.uni $L__BB1_1821; + +$L__BB1_1813: + add.s32 %r10030, %r10030, 1; + setp.lt.u32 %p2262, %r10030, %r10032; + @%p2262 bra $L__BB1_1821; + + shl.b16 %rs1053, %rs1253, 1; + or.b16 %rs1253, %rs1053, 1; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p2263, %r9825, 0; + mov.u32 %r10567, %r10033; + @%p2263 bra $L__BB1_1817; + bra.uni $L__BB1_1815; + +$L__BB1_1817: + add.s32 %r8167, %r10031, 1; + min.u32 %r10031, %r8167, 12; + setp.lt.u32 %p2266, %r10031, 3; + mov.u32 %r10030, 0; + mov.u32 %r10568, %r10030; + @%p2266 bra $L__BB1_1820; + + setp.lt.u32 %p2267, %r10031, 6; + mov.u32 %r10568, 1; + @%p2267 bra $L__BB1_1820; + + setp.lt.u32 %p2268, %r10031, 9; + setp.eq.s32 %p2269, %r10031, 11; + selp.b32 %r8169, 4, 5, %p2269; + setp.lt.u32 %p2270, %r10031, 11; + selp.b32 %r8170, 3, %r8169, %p2270; + selp.b32 %r10568, 2, %r8170, %p2268; + +$L__BB1_1820: + mov.u32 %r8172, 1; + shl.b32 %r10032, %r8172, %r10568; + mov.u32 %r10033, %r10567; + +$L__BB1_1821: + and.b16 %rs1056, %rs452, 15; + cvt.u32.u16 %r3797, %rs1056; + and.b32 %r8173, %r10494, 1; + setp.eq.b32 %p2271, %r8173, 1; + mov.pred %p2272, 0; + xor.pred %p2273, %p2271, %p2272; + not.pred %p2274, %p2273; + mov.u32 %r10589, %r10485; + @%p2274 bra $L__BB1_1828; + + and.b32 %r8174, %r3797, 1; + sub.s32 %r10575, %r3683, %r8174; + setp.eq.s32 %p2275, %r10575, 0; + mov.u32 %r10589, %r10485; + @%p2275 bra $L__BB1_1828; + + mov.u32 %r8175, -1; + shl.b32 %r8176, %r8175, %r10575; + not.b32 %r8177, %r8176; + and.b32 %r10576, %r10488, %r8177; + +$L__BB1_1824: + setp.gt.u32 %p2276, %r10451, 17476; + mov.u32 %r10589, 1; + @%p2276 bra $L__BB1_1828; + + sub.s32 %r8179, %r10452, %r10453; + min.u32 %r8180, %r8179, %r10575; + setp.eq.s32 %p2277, %r8180, 32; + mov.u32 %r8181, -1; + shl.b32 %r8182, %r8181, %r8180; + not.b32 %r8183, %r8182; + selp.b32 %r8184, -1, %r8183, %p2277; + and.b32 %r8185, %r8184, %r10576; + shl.b32 %r8186, %r8185, %r10453; + or.b32 %r10454, %r8186, %r10454; + add.s32 %r10453, %r8180, %r10453; + shr.u32 %r10576, %r10576, %r8180; + sub.s32 %r10575, %r10575, %r8180; + setp.lt.u32 %p2278, %r10453, %r10452; + @%p2278 bra $L__BB1_1827; + + cvt.u64.u32 %rd1367, %r10451; + add.s64 %rd1368, %rd1367, %rd4; + add.s64 %rd1369, %rd1, %rd1368; + st.global.u8 [%rd1369], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p2279, %r10454, 255; + selp.b32 %r10452, 7, 8, %p2279; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1827: + setp.ne.s32 %p2280, %r10575, 0; + mov.u32 %r10589, %r10485; + @%p2280 bra $L__BB1_1824; + +$L__BB1_1828: + and.b32 %r3821, %r10494, 2; + setp.eq.s32 %p2281, %r3821, 0; + mov.u32 %r10604, %r10589; + @%p2281 bra $L__BB1_1835; + + shr.u32 %r8189, %r3797, 1; + and.b32 %r8190, %r8189, 1; + sub.s32 %r10590, %r3683, %r8190; + setp.eq.s32 %p2282, %r10590, 0; + mov.u32 %r10604, %r10589; + @%p2282 bra $L__BB1_1835; + + mov.u32 %r8191, -1; + shl.b32 %r8192, %r8191, %r10590; + not.b32 %r8193, %r8192; + and.b32 %r10591, %r10492, %r8193; + +$L__BB1_1831: + setp.gt.u32 %p2283, %r10451, 17476; + mov.u32 %r10604, 1; + @%p2283 bra $L__BB1_1835; + + sub.s32 %r8195, %r10452, %r10453; + min.u32 %r8196, %r8195, %r10590; + setp.eq.s32 %p2284, %r8196, 32; + mov.u32 %r8197, -1; + shl.b32 %r8198, %r8197, %r8196; + not.b32 %r8199, %r8198; + selp.b32 %r8200, -1, %r8199, %p2284; + and.b32 %r8201, %r8200, %r10591; + shl.b32 %r8202, %r8201, %r10453; + or.b32 %r10454, %r8202, %r10454; + add.s32 %r10453, %r8196, %r10453; + shr.u32 %r10591, %r10591, %r8196; + sub.s32 %r10590, %r10590, %r8196; + setp.lt.u32 %p2285, %r10453, %r10452; + @%p2285 bra $L__BB1_1834; + + cvt.u64.u32 %rd1370, %r10451; + add.s64 %rd1371, %rd1370, %rd4; + add.s64 %rd1372, %rd1, %rd1371; + st.global.u8 [%rd1372], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p2286, %r10454, 255; + selp.b32 %r10452, 7, 8, %p2286; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1834: + setp.ne.s32 %p2287, %r10590, 0; + mov.u32 %r10604, %r10589; + @%p2287 bra $L__BB1_1831; + +$L__BB1_1835: + and.b32 %r3845, %r10494, 4; + setp.eq.s32 %p2288, %r3845, 0; + mov.u32 %r10619, %r10604; + @%p2288 bra $L__BB1_1842; + + shr.u32 %r8205, %r3797, 2; + and.b32 %r8206, %r8205, 1; + sub.s32 %r10605, %r3683, %r8206; + setp.eq.s32 %p2289, %r10605, 0; + mov.u32 %r10619, %r10604; + @%p2289 bra $L__BB1_1842; + + mov.u32 %r8207, -1; + shl.b32 %r8208, %r8207, %r10605; + not.b32 %r8209, %r8208; + and.b32 %r10606, %r10508, %r8209; + +$L__BB1_1838: + setp.gt.u32 %p2290, %r10451, 17476; + mov.u32 %r10619, 1; + @%p2290 bra $L__BB1_1842; + + sub.s32 %r8211, %r10452, %r10453; + min.u32 %r8212, %r8211, %r10605; + setp.eq.s32 %p2291, %r8212, 32; + mov.u32 %r8213, -1; + shl.b32 %r8214, %r8213, %r8212; + not.b32 %r8215, %r8214; + selp.b32 %r8216, -1, %r8215, %p2291; + and.b32 %r8217, %r8216, %r10606; + shl.b32 %r8218, %r8217, %r10453; + or.b32 %r10454, %r8218, %r10454; + add.s32 %r10453, %r8212, %r10453; + shr.u32 %r10606, %r10606, %r8212; + sub.s32 %r10605, %r10605, %r8212; + setp.lt.u32 %p2292, %r10453, %r10452; + @%p2292 bra $L__BB1_1841; + + cvt.u64.u32 %rd1373, %r10451; + add.s64 %rd1374, %rd1373, %rd4; + add.s64 %rd1375, %rd1, %rd1374; + st.global.u8 [%rd1375], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p2293, %r10454, 255; + selp.b32 %r10452, 7, 8, %p2293; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1841: + setp.ne.s32 %p2294, %r10605, 0; + mov.u32 %r10619, %r10604; + @%p2294 bra $L__BB1_1838; + +$L__BB1_1842: + and.b32 %r3869, %r10494, 8; + setp.eq.s32 %p2295, %r3869, 0; + mov.u32 %r10485, %r10619; + @%p2295 bra $L__BB1_1849; + + shr.u32 %r8221, %r3797, 3; + sub.s32 %r10620, %r3683, %r8221; + setp.eq.s32 %p2296, %r10620, 0; + mov.u32 %r10485, %r10619; + @%p2296 bra $L__BB1_1849; + + mov.u32 %r8222, -1; + shl.b32 %r8223, %r8222, %r10620; + not.b32 %r8224, %r8223; + and.b32 %r10621, %r10507, %r8224; + +$L__BB1_1845: + setp.gt.u32 %p2297, %r10451, 17476; + mov.u32 %r10485, 1; + @%p2297 bra $L__BB1_1849; + + sub.s32 %r8226, %r10452, %r10453; + min.u32 %r8227, %r8226, %r10620; + setp.eq.s32 %p2298, %r8227, 32; + mov.u32 %r8228, -1; + shl.b32 %r8229, %r8228, %r8227; + not.b32 %r8230, %r8229; + selp.b32 %r8231, -1, %r8230, %p2298; + and.b32 %r8232, %r8231, %r10621; + shl.b32 %r8233, %r8232, %r10453; + or.b32 %r10454, %r8233, %r10454; + add.s32 %r10453, %r8227, %r10453; + shr.u32 %r10621, %r10621, %r8227; + sub.s32 %r10620, %r10620, %r8227; + setp.lt.u32 %p2299, %r10453, %r10452; + @%p2299 bra $L__BB1_1848; + + cvt.u64.u32 %rd1376, %r10451; + add.s64 %rd1377, %rd1376, %rd4; + add.s64 %rd1378, %rd1, %rd1377; + st.global.u8 [%rd1378], %r10454; + add.s32 %r10451, %r10451, 1; + setp.eq.s32 %p2300, %r10454, 255; + selp.b32 %r10452, 7, 8, %p2300; + mov.u32 %r10453, 0; + mov.u32 %r10454, %r10453; + +$L__BB1_1848: + setp.ne.s32 %p2301, %r10620, 0; + mov.u32 %r10485, %r10619; + @%p2301 bra $L__BB1_1845; + +$L__BB1_1849: + and.b32 %r8236, %r10491, 255; + and.b32 %r8237, %r10356, 255; + setp.lt.u32 %p2302, %r8236, %r8237; + cvt.u16.u32 %rs1057, %r10491; + selp.b16 %rs1058, %rs450, %rs1057, %p2302; + st.shared.u8 [%r3621+1], %rs1058; + ld.shared.u8 %rs1059, [%r3621+3]; + setp.gt.u16 %p2303, %rs448, %rs1059; + add.s32 %r10637, %r10637, 1; + add.s32 %r8238, %r10334, 3; + selp.b32 %r8239, %r10637, %r8238, %p2303; + add.s32 %r8241, %r4095, %r8239; + ld.shared.u8 %r8242, [%r8241]; + add.s32 %r10335, %r8242, -1; + shr.u32 %r8243, %r3821, 1; + or.b32 %r8244, %r3627, %r8243; + st.shared.u8 [%r3621+2], %r10505; + st.shared.u8 [%r3624+1], %r8244; + ld.shared.u8 %rs1060, [%r3624+3]; + mul.wide.u16 %r8245, %rs1060, 4; + add.s32 %r8246, %r8245, %r3626; + shr.u32 %r8247, %r3869, 3; + st.shared.u8 [%r3624+2], %r8247; + shr.u32 %r8248, %r3869, 2; + shr.u32 %r8249, %r3845, 1; + or.b32 %r8250, %r8248, %r8249; + or.b32 %r10332, %r8250, %r8246; + add.s32 %r10336, %r10336, 1; + mov.u32 %r10375, %r10524; + +$L__BB1_1850: + mov.u32 %r10333, %r10638; + mov.u32 %r10334, %r10637; + max.s32 %r8251, %r10655, 0; + mul.lo.s32 %r8252, %r3412, 6; + setp.gt.s32 %p2304, %r3412, 0; + selp.b32 %r8253, %r8252, 0, %p2304; + cvt.u64.u32 %rd1379, %r8253; + add.s64 %rd75, %rd71, %rd1379; + ld.global.u8 %rs476, [%rd75+1]; + add.s32 %r8254, %r8253, 2; + cvt.u64.u32 %rd1380, %r8254; + add.s64 %rd1381, %rd71, %rd1380; + ld.global.u8 %rs477, [%rd1381]; + ld.global.u8 %rs478, [%rd1381+1]; + mul.lo.s32 %r8255, %r8251, 6; + cvt.u64.u32 %rd1382, %r8255; + add.s64 %rd1383, %rd71, %rd1382; + ld.global.u8 %rs479, [%rd1383]; + ld.global.u8 %rs480, [%rd1383+1]; + add.s32 %r8256, %r8255, 2; + cvt.u64.u32 %rd1384, %r8256; + add.s64 %rd1385, %rd71, %rd1384; + ld.global.u8 %rs481, [%rd1385]; + ld.global.u8 %rs482, [%rd1385+1]; + setp.eq.s16 %p2305, %rs476, 0; + mov.u32 %r10667, %r10375; + @%p2305 bra $L__BB1_1857; + + ld.global.u8 %r10657, [%rd75]; + cvt.u32.u16 %r10656, %rs476; + +$L__BB1_1852: + mov.u32 %r3920, %r10656; + setp.gt.u32 %p2306, %r10264, 2879; + mov.u32 %r10667, 1; + @%p2306 bra $L__BB1_1857; + + mov.u32 %r8258, 8; + sub.s32 %r8259, %r8258, %r10266; + sub.s32 %r8260, %r8259, %r10265; + min.u32 %r8261, %r8260, %r3920; + setp.eq.s32 %p2307, %r8261, 32; + mov.u32 %r8262, -1; + shl.b32 %r8263, %r8262, %r8261; + not.b32 %r8264, %r8263; + selp.b32 %r8265, -1, %r8264, %p2307; + and.b32 %r8266, %r8265, %r10657; + shl.b32 %r8267, %r8266, %r10265; + cvt.u16.u32 %rs1061, %r8267; + or.b16 %rs1322, %rs1322, %rs1061; + add.s32 %r10265, %r8261, %r10265; + sub.s32 %r10656, %r3920, %r8261; + shr.u32 %r10657, %r10657, %r8261; + setp.gt.u32 %p2308, %r8260, %r3920; + @%p2308 bra $L__BB1_1856; + + setp.ne.s32 %p2309, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs1062, %rs1322, 255; + setp.ne.s16 %p2310, %rs1062, 127; + and.pred %p2311, %p2309, %p2310; + @%p2311 bra $L__BB1_1856; + + mov.u32 %r8270, 20548; + sub.s32 %r8271, %r8270, %r10264; + cvt.u64.u32 %rd1386, %r8271; + add.s64 %rd1387, %rd1386, %rd4; + add.s64 %rd1388, %rd1, %rd1387; + st.global.u8 [%rd1388], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2312, %rs1062, 143; + selp.u32 %r10266, 1, 0, %p2312; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1856: + setp.ne.s32 %p2313, %r10656, 0; + mov.u32 %r10667, %r10375; + @%p2313 bra $L__BB1_1852; + +$L__BB1_1857: + setp.eq.s16 %p2314, %rs480, 0; + mov.u32 %r10679, %r10667; + @%p2314 bra $L__BB1_1864; + + cvt.u32.u16 %r8272, %rs479; + and.b32 %r10669, %r8272, 255; + cvt.u32.u16 %r8273, %rs480; + and.b32 %r10668, %r8273, 255; + +$L__BB1_1859: + mov.u32 %r3939, %r10668; + setp.gt.u32 %p2315, %r10264, 2879; + mov.u32 %r10679, 1; + @%p2315 bra $L__BB1_1864; + + mov.u32 %r8275, 8; + sub.s32 %r8276, %r8275, %r10266; + sub.s32 %r8277, %r8276, %r10265; + min.u32 %r8278, %r8277, %r3939; + setp.eq.s32 %p2316, %r8278, 32; + mov.u32 %r8279, -1; + shl.b32 %r8280, %r8279, %r8278; + not.b32 %r8281, %r8280; + selp.b32 %r8282, -1, %r8281, %p2316; + and.b32 %r8283, %r8282, %r10669; + shl.b32 %r8284, %r8283, %r10265; + cvt.u16.u32 %rs1066, %r8284; + or.b16 %rs1322, %rs1322, %rs1066; + add.s32 %r10265, %r8278, %r10265; + sub.s32 %r10668, %r3939, %r8278; + shr.u32 %r10669, %r10669, %r8278; + setp.gt.u32 %p2317, %r8277, %r3939; + @%p2317 bra $L__BB1_1863; + + setp.ne.s32 %p2318, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs1067, %rs1322, 255; + setp.ne.s16 %p2319, %rs1067, 127; + and.pred %p2320, %p2318, %p2319; + @%p2320 bra $L__BB1_1863; + + mov.u32 %r8287, 20548; + sub.s32 %r8288, %r8287, %r10264; + cvt.u64.u32 %rd1389, %r8288; + add.s64 %rd1390, %rd1389, %rd4; + add.s64 %rd1391, %rd1, %rd1390; + st.global.u8 [%rd1391], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2321, %rs1067, 143; + selp.u32 %r10266, 1, 0, %p2321; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1863: + setp.ne.s32 %p2322, %r10668, 0; + mov.u32 %r10679, %r10667; + @%p2322 bra $L__BB1_1859; + +$L__BB1_1864: + setp.eq.s16 %p2323, %rs478, 0; + mov.u32 %r10691, %r10679; + @%p2323 bra $L__BB1_1871; + + cvt.u32.u16 %r8289, %rs478; + and.b32 %r10680, %r8289, 255; + cvt.u32.u16 %r8290, %rs477; + and.b32 %r10681, %r8290, 255; + +$L__BB1_1866: + mov.u32 %r3958, %r10680; + setp.gt.u32 %p2324, %r10264, 2879; + mov.u32 %r10691, 1; + @%p2324 bra $L__BB1_1871; + + mov.u32 %r8292, 8; + sub.s32 %r8293, %r8292, %r10266; + sub.s32 %r8294, %r8293, %r10265; + min.u32 %r8295, %r8294, %r3958; + setp.eq.s32 %p2325, %r8295, 32; + mov.u32 %r8296, -1; + shl.b32 %r8297, %r8296, %r8295; + not.b32 %r8298, %r8297; + selp.b32 %r8299, -1, %r8298, %p2325; + and.b32 %r8300, %r8299, %r10681; + shl.b32 %r8301, %r8300, %r10265; + cvt.u16.u32 %rs1071, %r8301; + or.b16 %rs1322, %rs1322, %rs1071; + add.s32 %r10265, %r8295, %r10265; + sub.s32 %r10680, %r3958, %r8295; + shr.u32 %r10681, %r10681, %r8295; + setp.gt.u32 %p2326, %r8294, %r3958; + @%p2326 bra $L__BB1_1870; + + setp.ne.s32 %p2327, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs1072, %rs1322, 255; + setp.ne.s16 %p2328, %rs1072, 127; + and.pred %p2329, %p2327, %p2328; + @%p2329 bra $L__BB1_1870; + + mov.u32 %r8304, 20548; + sub.s32 %r8305, %r8304, %r10264; + cvt.u64.u32 %rd1392, %r8305; + add.s64 %rd1393, %rd1392, %rd4; + add.s64 %rd1394, %rd1, %rd1393; + st.global.u8 [%rd1394], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2330, %rs1072, 143; + selp.u32 %r10266, 1, 0, %p2330; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1870: + setp.ne.s32 %p2331, %r10680, 0; + mov.u32 %r10691, %r10679; + @%p2331 bra $L__BB1_1866; + +$L__BB1_1871: + setp.eq.s16 %p2332, %rs482, 0; + mov.u32 %r10267, %r10691; + @%p2332 bra $L__BB1_1878; + + cvt.u32.u16 %r8306, %rs481; + and.b32 %r10693, %r8306, 255; + cvt.u32.u16 %r8307, %rs482; + and.b32 %r10692, %r8307, 255; + +$L__BB1_1873: + mov.u32 %r3977, %r10692; + setp.gt.u32 %p2333, %r10264, 2879; + mov.u32 %r10267, 1; + @%p2333 bra $L__BB1_1878; + + mov.u32 %r8309, 8; + sub.s32 %r8310, %r8309, %r10266; + sub.s32 %r8311, %r8310, %r10265; + min.u32 %r8312, %r8311, %r3977; + setp.eq.s32 %p2334, %r8312, 32; + mov.u32 %r8313, -1; + shl.b32 %r8314, %r8313, %r8312; + not.b32 %r8315, %r8314; + selp.b32 %r8316, -1, %r8315, %p2334; + and.b32 %r8317, %r8316, %r10693; + shl.b32 %r8318, %r8317, %r10265; + cvt.u16.u32 %rs1076, %r8318; + or.b16 %rs1322, %rs1322, %rs1076; + add.s32 %r10265, %r8312, %r10265; + sub.s32 %r10692, %r3977, %r8312; + shr.u32 %r10693, %r10693, %r8312; + setp.gt.u32 %p2335, %r8311, %r3977; + @%p2335 bra $L__BB1_1877; + + setp.ne.s32 %p2336, %r10266, 0; + mov.u32 %r10266, 0; + and.b16 %rs1077, %rs1322, 255; + setp.ne.s16 %p2337, %rs1077, 127; + and.pred %p2338, %p2336, %p2337; + @%p2338 bra $L__BB1_1877; + + mov.u32 %r8321, 20548; + sub.s32 %r8322, %r8321, %r10264; + cvt.u64.u32 %rd1395, %r8322; + add.s64 %rd1396, %rd1395, %rd4; + add.s64 %rd1397, %rd1, %rd1396; + st.global.u8 [%rd1397], %rs1322; + add.s32 %r10264, %r10264, 1; + setp.gt.u16 %p2339, %rs1077, 143; + selp.u32 %r10266, 1, 0, %p2339; + mov.u16 %rs1322, 0; + mov.u32 %r10265, 0; + +$L__BB1_1877: + setp.ne.s32 %p2340, %r10692, 0; + mov.u32 %r10267, %r10691; + @%p2340 bra $L__BB1_1873; + +$L__BB1_1878: + add.s32 %r10316, %r10316, 4; + setp.lt.u32 %p2341, %r10316, %r5; + @%p2341 bra $L__BB1_1638; + +$L__BB1_1879: + add.s32 %r10300, %r10300, 2; + setp.lt.u32 %p2342, %r10300, %r6; + @%p2342 bra $L__BB1_1636; + +$L__BB1_1880: + setp.eq.s32 %p2343, %r10030, 0; + mov.u32 %r10733, %r10033; + @%p2343 bra $L__BB1_1884; + + shl.b16 %rs1080, %rs1253, 1; + or.b16 %rs1253, %rs1080, 1; + add.s32 %r9825, %r9825, -1; + setp.ne.s32 %p2344, %r9825, 0; + mov.u32 %r10733, %r10033; + @%p2344 bra $L__BB1_1884; + + setp.gt.u32 %p2345, %r9816, 191; + mov.u32 %r10733, 1; + mov.u32 %r9825, 0; + @%p2345 bra $L__BB1_1884; + + add.s32 %r8325, %r9816, 17477; + cvt.u64.u32 %rd1398, %r8325; + add.s64 %rd1399, %rd1398, %rd4; + add.s64 %rd1400, %rd1, %rd1399; + and.b16 %rs1082, %rs1253, 255; + st.global.u8 [%rd1400], %rs1253; + add.s32 %r9816, %r9816, 1; + setp.eq.s16 %p2346, %rs1082, 255; + selp.b32 %r9825, 7, 8, %p2346; + mov.u16 %rs1253, 0; + mov.u32 %r10733, %r10033; + +$L__BB1_1884: + cvt.u32.u16 %r8326, %rs1253; + and.b32 %r8327, %r8326, 255; + shl.b32 %r8328, %r8327, %r9825; + cvt.u16.u32 %rs505, %r8328; + mov.u32 %r8329, -1; + shl.b32 %r8330, %r8329, %r10265; + not.b32 %r8331, %r8330; + mov.u32 %r8332, 255; + and.b32 %r8333, %r8331, 255; + setp.eq.s32 %p2347, %r10265, 0; + selp.b32 %r4029, 0, %r8333, %p2347; + shl.b32 %r4030, %r8332, %r9825; + and.b32 %r8334, %r4030, 255; + or.b32 %r8335, %r8334, %r4029; + setp.eq.s32 %p2348, %r8335, 0; + mov.u32 %r10735, %r10267; + mov.u32 %r10737, %r10733; + @%p2348 bra $L__BB1_1890; + + or.b16 %rs506, %rs1322, %rs505; + and.b16 %rs1083, %rs506, 255; + xor.b16 %rs1084, %rs506, %rs505; + cvt.u32.u16 %r8336, %rs1084; + and.b32 %r8337, %r4030, %r8336; + and.b32 %r8338, %r8337, 255; + xor.b16 %rs1085, %rs506, %rs1322; + cvt.u32.u16 %r8339, %rs1085; + and.b32 %r8340, %r4029, %r8339; + or.b32 %r8341, %r8338, %r8340; + setp.eq.s32 %p2349, %r8341, 0; + setp.ne.s16 %p2350, %rs1083, 255; + and.pred %p2351, %p2350, %p2349; + setp.gt.u32 %p2352, %r10264, 1; + and.pred %p2353, %p2352, %p2351; + add.s32 %r8342, %r9816, 17477; + cvt.u64.u32 %rd1401, %r8342; + add.s64 %rd1402, %rd1401, %rd4; + add.s64 %rd76, %rd1, %rd1402; + @%p2353 bra $L__BB1_1888; + bra.uni $L__BB1_1886; + +$L__BB1_1888: + setp.gt.u32 %p2357, %r9816, 191; + mov.u32 %r10737, 1; + mov.u32 %r10735, %r10267; + @%p2357 bra $L__BB1_1890; + + st.global.u8 [%rd76], %rs506; + add.s32 %r9816, %r9816, 1; + mov.u32 %r10735, %r10267; + mov.u32 %r10737, %r10733; + bra.uni $L__BB1_1890; + +$L__BB1_1886: + setp.gt.u32 %p2354, %r9816, 191; + setp.gt.u32 %p2355, %r10264, 2879; + or.pred %p2356, %p2355, %p2354; + mov.u32 %r10735, 1; + mov.u32 %r10737, %r10735; + @%p2356 bra $L__BB1_1890; + + st.global.u8 [%rd76], %rs505; + add.s32 %r9816, %r9816, 1; + mov.u32 %r8345, 20548; + sub.s32 %r8346, %r8345, %r10264; + cvt.u64.u32 %rd1403, %r8346; + add.s64 %rd1404, %rd1403, %rd4; + add.s64 %rd1405, %rd1, %rd1404; + st.global.u8 [%rd1405], %rs1322; + add.s32 %r10264, %r10264, 1; + mov.u32 %r10735, %r10267; + mov.u32 %r10737, %r10733; + +$L__BB1_1890: + setp.eq.s32 %p2358, %r10453, 0; + @%p2358 bra $L__BB1_1894; + + sub.s32 %r8348, %r10452, %r10453; + mov.u32 %r8349, -1; + shl.b32 %r8350, %r8349, %r8348; + not.b32 %r8351, %r8350; + and.b32 %r8352, %r8351, 255; + shl.b32 %r8353, %r8352, %r10453; + or.b32 %r4038, %r8353, %r10454; + setp.eq.s32 %p2359, %r4038, 255; + mov.u32 %r10739, %r10485; + @%p2359 bra $L__BB1_1896; + + setp.gt.u32 %p2360, %r10451, 17476; + mov.u32 %r10739, 1; + @%p2360 bra $L__BB1_1896; + + cvt.u64.u32 %rd1406, %r10451; + add.s64 %rd1407, %rd1406, %rd4; + add.s64 %rd1408, %rd1, %rd1407; + st.global.u8 [%rd1408], %r4038; + add.s32 %r10451, %r10451, 1; + mov.u32 %r10739, %r10485; + bra.uni $L__BB1_1896; + +$L__BB1_1894: + setp.ne.s32 %p2361, %r10452, 7; + mov.u32 %r10739, %r10485; + @%p2361 bra $L__BB1_1896; + + setp.eq.s32 %p2362, %r10451, 0; + add.s32 %r8355, %r10451, -1; + selp.b32 %r10451, 0, %r8355, %p2362; + mov.u32 %r10739, %r10485; + +$L__BB1_1896: + or.b32 %r8356, %r10737, %r10735; + or.b32 %r8357, %r8356, %r10739; + setp.eq.s32 %p2363, %r8357, 0; + @%p2363 bra $L__BB1_1898; + + mov.u32 %r8362, 1; + st.global.u32 [%rd6], %r8362; + mov.u32 %r8363, 3; + st.global.u32 [%rd6+4], %r8363; + mov.u32 %r10740, 0; + mov.u32 %r10741, %r10740; + mov.u32 %r10742, %r10740; + mov.u32 %r10743, %r10740; + bra.uni $L__BB1_1904; + +$L__BB1_1898: + add.s32 %r8364, %r9816, %r10264; + add.s32 %r10741, %r8364, %r10451; + setp.lt.u32 %p2364, %r10741, 2; + setp.gt.u32 %p2365, %r10741, %r3; + or.pred %p2366, %p2364, %p2365; + @%p2366 bra $L__BB1_1900; + bra.uni $L__BB1_1899; + +$L__BB1_1900: + mov.u32 %r8374, 1; + st.global.u32 [%rd6], %r8374; + mov.u32 %r8375, 4; + st.global.u32 [%rd6+4], %r8375; + mov.u32 %r10740, 0; + mov.u32 %r10741, %r10740; + mov.u32 %r10742, %r10740; + mov.u32 %r10743, %r10740; + bra.uni $L__BB1_1904; + +$L__BB1_1250: + mov.u32 %r6781, 0; + st.global.u32 [%rd6], %r6781; + st.global.u32 [%rd6+4], %r6781; + st.global.u32 [%rd6+8], %r6781; + st.global.u32 [%rd6+12], %r6781; + st.global.u32 [%rd6+16], %r2; + st.global.u32 [%rd6+20], %r6781; + st.global.u32 [%rd6+24], %r6781; + st.global.u32 [%rd6+28], %r6781; + bra.uni $L__BB1_1905; + +$L__BB1_23: + setp.lt.u32 %p37, %r2, %r4; + @%p37 bra $L__BB1_1248; + bra.uni $L__BB1_24; + +$L__BB1_1248: + mov.u32 %r6775, 2; + st.global.u32 [%rd6], %r6775; + mov.u32 %r6776, 5; + st.global.u32 [%rd6+4], %r6776; + mov.u32 %r6777, 0; + st.global.u32 [%rd6+8], %r6777; + st.global.u32 [%rd6+12], %r6777; + st.global.u32 [%rd6+16], %r6777; + st.global.u32 [%rd6+20], %r6777; + st.global.u32 [%rd6+24], %r6777; + st.global.u32 [%rd6+28], %r6777; + bra.uni $L__BB1_1905; + +$L__BB1_1899: + and.b32 %r8366, %r9816, 32767; + and.b32 %r8367, %r10264, 32767; + bfi.b32 %r8368, %r8367, %r8366, 15, 15; + or.b32 %r10740, %r8368, -2147483648; + mov.u32 %r8369, 0; + st.global.u32 [%rd6], %r8369; + st.global.u32 [%rd6+4], %r8369; + mov.u32 %r10743, 1; + bra.uni $L__BB1_1904; + +$L__BB1_24: + mov.u32 %r8423, 0; + setp.eq.s32 %p38, %r4, 2; + @%p38 bra $L__BB1_35; + + setp.ne.s32 %p39, %r4, 3; + @%p39 bra $L__BB1_43; + + @%p10 bra $L__BB1_43; + + mov.u32 %r4106, 0; + mov.u32 %r8416, %r4106; + mov.u32 %r8423, %r4106; + +$L__BB1_28: + mul.lo.s32 %r27, %r8416, %r1; + mov.u32 %r8418, %r4106; + +$L__BB1_29: + add.s32 %r4108, %r8418, %r27; + cvt.u64.u32 %rd99, %r4108; + add.s64 %rd100, %rd99, %rd5; + shl.b64 %rd101, %rd100, 2; + add.s64 %rd102, %rd3, %rd101; + ld.global.u32 %r4109, [%rd102]; + abs.s32 %r30, %r4109; + setp.eq.s32 %p41, %r30, 0; + @%p41 bra $L__BB1_32; + + setp.eq.s32 %p42, %r30, 3; + @%p42 bra $L__BB1_32; + + add.s32 %r8423, %r8423, 1; + and.b32 %r4110, %r30, 1; + setp.eq.b32 %p43, %r4110, 1; + not.pred %p44, %p43; + setp.lt.u32 %p45, %r30, 5; + or.pred %p46, %p45, %p44; + @%p46 bra $L__BB1_34; + +$L__BB1_32: + add.s32 %r8418, %r8418, 1; + setp.lt.u32 %p47, %r8418, %r5; + @%p47 bra $L__BB1_29; + + add.s32 %r8416, %r8416, 1; + setp.lt.u32 %p48, %r8416, %r6; + @%p48 bra $L__BB1_28; + bra.uni $L__BB1_43; + +$L__BB1_35: + @%p10 bra $L__BB1_43; + + mov.u32 %r4115, 0; + mov.u32 %r8421, %r4115; + +$L__BB1_37: + mul.lo.s32 %r36, %r8421, %r1; + mov.u32 %r8422, %r4115; + +$L__BB1_38: + add.s32 %r4117, %r8422, %r36; + cvt.u64.u32 %rd103, %r4117; + add.s64 %rd104, %rd103, %rd5; + shl.b64 %rd105, %rd104, 2; + add.s64 %rd106, %rd3, %rd105; + ld.global.u32 %r4118, [%rd106]; + abs.s32 %r38, %r4118; + setp.eq.s32 %p50, %r38, 0; + @%p50 bra $L__BB1_41; + + setp.gt.u32 %p51, %r38, 2; + and.b32 %r4119, %r38, 1; + setp.eq.b32 %p52, %r4119, 1; + and.pred %p53, %p51, %p52; + @%p53 bra $L__BB1_41; + bra.uni $L__BB1_40; + +$L__BB1_41: + add.s32 %r8422, %r8422, 1; + setp.lt.u32 %p54, %r8422, %r5; + @%p54 bra $L__BB1_38; + + add.s32 %r8421, %r8421, 1; + setp.lt.u32 %p55, %r8421, %r6; + mov.u32 %r8423, 0; + @%p55 bra $L__BB1_37; + +$L__BB1_43: + add.s32 %r8396, %r5, 1; + shr.u32 %r8395, %r8396, 1; + add.s64 %rd1410, %rd1, %rd4; + add.s64 %rd1409, %rd1410, 20548; + sub.s32 %r42, %r2, %r4; + mov.u32 %r4125, 30; + sub.s32 %r43, %r4125, %r42; + mov.u16 %rs510, 255; + st.global.u8 [%rd1409], %rs510; + add.s32 %r4127, %r8395, 2; + min.u32 %r45, %r4127, 513; + mov.u32 %r4128, -3; + sub.s32 %r4129, %r4128, %r8395; + max.u32 %r4130, %r4129, -514; + mov.u32 %r4131, -2; + sub.s32 %r4132, %r4131, %r4130; + and.b32 %r8428, %r45, 3; + setp.lt.u32 %p56, %r4132, 3; + mov.u32 %r8426, 0; + @%p56 bra $L__BB1_46; + + sub.s32 %r8425, %r45, %r8428; + mov.u32 %r8426, 0; + +$L__BB1_45: + add.s32 %r4135, %r4095, %r8426; + mov.u16 %rs511, 0; + st.shared.u8 [%r4135], %rs511; + mov.u32 %r4136, _ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val; + add.s32 %r4137, %r4136, %r8426; + st.shared.u8 [%r4137], %rs511; + st.shared.u8 [%r4135+1], %rs511; + st.shared.u8 [%r4137+1], %rs511; + st.shared.u8 [%r4135+2], %rs511; + st.shared.u8 [%r4137+2], %rs511; + st.shared.u8 [%r4135+3], %rs511; + st.shared.u8 [%r4137+3], %rs511; + add.s32 %r8426, %r8426, 4; + add.s32 %r8425, %r8425, -4; + setp.ne.s32 %p57, %r8425, 0; + @%p57 bra $L__BB1_45; + +$L__BB1_46: + setp.eq.s32 %p58, %r8428, 0; + @%p58 bra $L__BB1_49; + + mov.u32 %r4140, _ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val; + +$L__BB1_48: + .pragma "nounroll"; + add.s32 %r4139, %r4095, %r8426; + mov.u16 %rs512, 0; + st.shared.u8 [%r4139], %rs512; + add.s32 %r4141, %r4140, %r8426; + st.shared.u8 [%r4141], %rs512; + add.s32 %r8426, %r8426, 1; + add.s32 %r8428, %r8428, -1; + setp.ne.s32 %p59, %r8428, 0; + @%p59 bra $L__BB1_48; + +$L__BB1_49: + mov.u32 %r8723, 0; + mov.u32 %r8520, 8; + mov.u32 %r8724, 1; + mov.u32 %r8961, 4; + mov.u16 %rs1165, 15; + mov.u16 %rs1096, 0; + mov.u32 %r8725, %r8723; + mov.u32 %r8726, %r8723; + mov.u32 %r8514, %r8723; + mov.u32 %r8959, %r8723; + mov.u32 %r8960, %r8724; + mov.u32 %r8962, %r8724; + mov.u32 %r9176, %r8723; + mov.u32 %r9147, %r8723; + mov.u32 %r9148, %r8723; + mov.u32 %r9149, %r8520; + mov.u32 %r9150, %r8723; + @%p10 bra $L__BB1_417; + + ld.param.u64 %rd1419, [ j2k_htj2k_encode_codeblocks_param_5]; + ld.param.u64 %rd1411, [ j2k_htj2k_encode_codeblocks_param_3]; + mov.u32 %r4175, 31; + sub.s32 %r57, %r4175, %r2; + cvta.to.global.u64 %rd8, %rd1411; + cvta.to.global.u64 %rd9, %rd1419; + mov.u32 %r4174, 0; + mov.u32 %r8962, 1; + mov.u16 %rs1096, 0; + mov.u32 %r9149, 8; + mov.u16 %rs1165, 15; + mov.u32 %r8961, 4; + mov.u32 %r8429, %r4174; + mov.u32 %r8430, %r4174; + mov.u32 %r8431, %r4174; + mov.u32 %r9150, %r4174; + mov.u32 %r9148, %r4174; + mov.u32 %r9147, %r4174; + mov.u32 %r9176, %r4174; + mov.u32 %r8960, %r8962; + mov.u32 %r8959, %r4174; + mov.u32 %r8514, %r4174; + mov.u32 %r8520, %r9149; + mov.u32 %r8726, %r4174; + mov.u32 %r8725, %r4174; + mov.u32 %r8724, %r8962; + mov.u32 %r8723, %r4174; + bra.uni $L__BB1_51; + +$L__BB1_40: + mov.u32 %r4120, 2; + st.global.u32 [%rd6], %r4120; + mov.u32 %r4121, 6; + st.global.u32 [%rd6+4], %r4121; + mov.u32 %r4122, 0; + st.global.u32 [%rd6+8], %r4122; + st.global.u32 [%rd6+12], %r4122; + st.global.u32 [%rd6+16], %r4122; + st.global.u32 [%rd6+20], %r4122; + st.global.u32 [%rd6+24], %r4122; + st.global.u32 [%rd6+28], %r4122; + bra.uni $L__BB1_1905; + +$L__BB1_34: + mov.u32 %r4111, 2; + st.global.u32 [%rd6], %r4111; + mov.u32 %r4112, 6; + st.global.u32 [%rd6+4], %r4112; + mov.u32 %r4113, 0; + st.global.u32 [%rd6+8], %r4113; + st.global.u32 [%rd6+12], %r4113; + st.global.u32 [%rd6+16], %r4113; + st.global.u32 [%rd6+20], %r4113; + st.global.u32 [%rd6+24], %r4113; + st.global.u32 [%rd6+28], %r4113; + bra.uni $L__BB1_1905; + +$L__BB1_256: + setp.gt.u32 %p287, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8719, 1; + @%p287 bra $L__BB1_258; + + and.b16 %rs599, %rs1096, 255; + st.global.u8 [%rd13], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p288, %rs599, 255; + selp.b32 %r8520, 7, 8, %p288; + mov.u16 %rs1096, 0; + mov.u32 %r8719, %r8723; + bra.uni $L__BB1_258; + +$L__BB1_51: + cvt.u64.u32 %rd107, %r8430; + add.s64 %rd108, %rd107, %rd5; + shl.b64 %rd109, %rd108, 2; + add.s64 %rd110, %rd3, %rd109; + ld.global.u32 %r76, [%rd110]; + setp.eq.s32 %p61, %r76, 0; + mov.u32 %r8447, %r4174; + @%p61 bra $L__BB1_53; + + and.b32 %r4177, %r76, -2147483648; + abs.s32 %r4178, %r76; + shl.b32 %r4179, %r4178, %r57; + or.b32 %r8447, %r4179, %r4177; + +$L__BB1_53: + shl.b32 %r4183, %r8447, 1; + shr.u32 %r4184, %r4183, %r43; + and.b32 %r79, %r4184, -2; + setp.eq.s32 %p62, %r79, 0; + mov.u32 %r8451, 0; + mov.u32 %r8448, %r8451; + mov.u32 %r8449, %r8451; + mov.u32 %r8455, %r8451; + @%p62 bra $L__BB1_55; + + add.s32 %r4186, %r79, -1; + clz.b32 %r4187, %r4186; + mov.u32 %r4188, 32; + sub.s32 %r8448, %r4188, %r4187; + shr.u32 %r4189, %r8447, 31; + add.s32 %r4190, %r4189, %r79; + add.s32 %r8449, %r4190, -2; + mov.u32 %r8455, 1; + +$L__BB1_55: + setp.lt.u32 %p63, %r6, 2; + @%p63 bra $L__BB1_58; + + add.s32 %r4193, %r8430, %r1; + cvt.u64.u32 %rd111, %r4193; + add.s64 %rd112, %rd111, %rd5; + shl.b64 %rd113, %rd112, 2; + add.s64 %rd114, %rd3, %rd113; + ld.global.u32 %r85, [%rd114]; + setp.eq.s32 %p64, %r85, 0; + @%p64 bra $L__BB1_58; + + and.b32 %r4194, %r85, -2147483648; + abs.s32 %r4195, %r85; + shl.b32 %r4196, %r4195, %r57; + or.b32 %r8451, %r4196, %r4194; + +$L__BB1_58: + shl.b32 %r4199, %r8451, 1; + shr.u32 %r4200, %r4199, %r43; + and.b32 %r88, %r4200, -2; + setp.eq.s32 %p65, %r88, 0; + mov.u32 %r8466, 0; + mov.u32 %r8452, %r8466; + mov.u32 %r8453, %r8466; + mov.u32 %r8470, %r8448; + @%p65 bra $L__BB1_60; + + or.b32 %r8455, %r8455, 2; + add.s32 %r4201, %r88, -1; + clz.b32 %r4202, %r4201; + mov.u32 %r4203, 32; + sub.s32 %r8452, %r4203, %r4202; + max.s32 %r8470, %r8448, %r8452; + shr.u32 %r4204, %r8451, 31; + add.s32 %r4205, %r4204, %r88; + add.s32 %r8453, %r4205, -2; + +$L__BB1_60: + add.s32 %r8472, %r8430, 1; + add.s32 %r4210, %r8429, 1; + setp.ge.u32 %p66, %r4210, %r5; + mov.u32 %r8467, %r8466; + mov.u32 %r8468, %r8466; + mov.u32 %r8469, %r8466; + @%p66 bra $L__BB1_71; + + cvt.u64.u32 %rd115, %r8472; + add.s64 %rd116, %rd115, %rd5; + shl.b64 %rd117, %rd116, 2; + add.s64 %rd118, %rd3, %rd117; + ld.global.u32 %r98, [%rd118]; + setp.eq.s32 %p67, %r98, 0; + mov.u32 %r8467, 0; + mov.u32 %r8456, %r8467; + @%p67 bra $L__BB1_63; + + and.b32 %r4212, %r98, -2147483648; + abs.s32 %r4213, %r98; + shl.b32 %r4214, %r4213, %r57; + or.b32 %r8456, %r4214, %r4212; + +$L__BB1_63: + shl.b32 %r4217, %r8456, 1; + shr.u32 %r4218, %r4217, %r43; + and.b32 %r101, %r4218, -2; + setp.eq.s32 %p68, %r101, 0; + mov.u32 %r8469, %r8467; + @%p68 bra $L__BB1_65; + + or.b32 %r8455, %r8455, 4; + add.s32 %r4219, %r101, -1; + clz.b32 %r4220, %r4219; + mov.u32 %r4221, 32; + sub.s32 %r8467, %r4221, %r4220; + max.s32 %r8470, %r8470, %r8467; + shr.u32 %r4222, %r8456, 31; + add.s32 %r4223, %r4222, %r101; + add.s32 %r8469, %r4223, -2; + +$L__BB1_65: + mov.u32 %r8466, 0; + mov.u32 %r8461, %r8466; + @%p63 bra $L__BB1_68; + + add.s32 %r4226, %r8472, %r1; + cvt.u64.u32 %rd119, %r4226; + add.s64 %rd120, %rd119, %rd5; + shl.b64 %rd121, %rd120, 2; + add.s64 %rd122, %rd3, %rd121; + ld.global.u32 %r110, [%rd122]; + setp.eq.s32 %p70, %r110, 0; + @%p70 bra $L__BB1_68; + + and.b32 %r4227, %r110, -2147483648; + abs.s32 %r4228, %r110; + shl.b32 %r4229, %r4228, %r57; + or.b32 %r8461, %r4229, %r4227; + +$L__BB1_68: + shl.b32 %r4232, %r8461, 1; + shr.u32 %r4233, %r4232, %r43; + and.b32 %r113, %r4233, -2; + setp.eq.s32 %p71, %r113, 0; + mov.u32 %r8468, %r8466; + @%p71 bra $L__BB1_70; + + or.b32 %r8455, %r8455, 8; + add.s32 %r4234, %r113, -1; + clz.b32 %r4235, %r4234; + mov.u32 %r4236, 32; + sub.s32 %r8466, %r4236, %r4235; + max.s32 %r8470, %r8470, %r8466; + shr.u32 %r4237, %r8461, 31; + add.s32 %r4238, %r4237, %r113; + add.s32 %r8468, %r4238, -2; + +$L__BB1_70: + add.s32 %r8472, %r8430, 2; + +$L__BB1_71: + mov.u32 %r8430, %r8472; + add.s32 %r4240, %r8470, -1; + setp.lt.s32 %p72, %r8470, 2; + setp.gt.s32 %p73, %r8470, 1; + selp.b32 %r130, %r4240, 0, %p73; + mov.u32 %r8473, 0; + @%p72 bra $L__BB1_73; + + setp.eq.s32 %p74, %r8448, %r8470; + selp.u32 %r4241, 1, 0, %p74; + setp.eq.s32 %p75, %r8452, %r8470; + selp.u32 %r4242, -1, 0, %p75; + bfi.b32 %r4243, %r4242, %r4241, 1, 1; + setp.eq.s32 %p76, %r8467, %r8470; + selp.u16 %rs517, 1, 0, %p76; + mul.wide.u16 %r4244, %rs517, 4; + or.b32 %r4245, %r4243, %r4244; + setp.eq.s32 %p77, %r8466, %r8470; + selp.u16 %rs518, 1, 0, %p77; + mul.wide.u16 %r4246, %rs518, 8; + or.b32 %r8473, %r4245, %r4246; + +$L__BB1_73: + shr.u32 %r4247, %r8429, 1; + add.s32 %r133, %r4095, %r4247; + ld.shared.u8 %rs519, [%r133]; + cvt.u32.u16 %r4249, %rs519; + and.b32 %r4250, %r4249, 255; + and.b32 %r4251, %r8452, 255; + setp.lt.u32 %p78, %r4251, %r4250; + cvt.u16.u32 %rs520, %r8452; + selp.b16 %rs521, %rs519, %rs520, %p78; + st.shared.u8 [%r133], %rs521; + cvt.u16.u32 %rs3, %r8466; + st.shared.u8 [%r133+1], %rs3; + and.b32 %r134, %r8455, 2; + cvt.u16.u32 %rs522, %r134; + shr.u16 %rs523, %rs522, 1; + mov.u32 %r4252, _ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val; + add.s32 %r135, %r4252, %r4247; + ld.shared.u8 %rs524, [%r135]; + or.b16 %rs525, %rs524, %rs523; + st.shared.u8 [%r135], %rs525; + and.b32 %r136, %r8455, 8; + shr.u32 %r137, %r136, 3; + st.shared.u8 [%r135+1], %r137; + shl.b32 %r4253, %r8455, 4; + shl.b32 %r4254, %r8431, 8; + or.b32 %r4255, %r4253, %r4254; + or.b32 %r4256, %r4255, %r8473; + mul.wide.u32 %rd123, %r4256, 2; + add.s64 %rd124, %rd8, %rd123; + ld.global.u16 %rs4, [%rd124]; + shr.u16 %rs526, %rs4, 4; + and.b16 %rs5, %rs526, 7; + setp.eq.s16 %p79, %rs5, 0; + mov.u32 %r8485, %r8959; + @%p79 bra $L__BB1_80; + + cvt.u32.u16 %r8474, %rs5; + shr.u16 %rs527, %rs4, 8; + cvt.u32.u16 %r8475, %rs527; + +$L__BB1_75: + mov.u32 %r140, %r8474; + setp.gt.u32 %p80, %r8962, 2879; + mov.u32 %r8485, 1; + @%p80 bra $L__BB1_80; + + mov.u32 %r4258, 8; + sub.s32 %r4259, %r4258, %r8960; + sub.s32 %r4260, %r4259, %r8961; + min.u32 %r4261, %r4260, %r140; + setp.eq.s32 %p81, %r4261, 32; + mov.u32 %r4262, -1; + shl.b32 %r4263, %r4262, %r4261; + not.b32 %r4264, %r4263; + selp.b32 %r4265, -1, %r4264, %p81; + and.b32 %r4266, %r4265, %r8475; + shl.b32 %r4267, %r4266, %r8961; + cvt.u16.u32 %rs528, %r4267; + or.b16 %rs1165, %rs1165, %rs528; + add.s32 %r8961, %r4261, %r8961; + sub.s32 %r8474, %r140, %r4261; + shr.u32 %r8475, %r8475, %r4261; + setp.gt.u32 %p82, %r4260, %r140; + @%p82 bra $L__BB1_79; + + setp.ne.s32 %p83, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs529, %rs1165, 255; + setp.ne.s16 %p84, %rs529, 127; + and.pred %p85, %p83, %p84; + @%p85 bra $L__BB1_79; + + mov.u32 %r4270, 20548; + sub.s32 %r4271, %r4270, %r8962; + cvt.u64.u32 %rd125, %r4271; + add.s64 %rd126, %rd125, %rd4; + add.s64 %rd127, %rd1, %rd126; + st.global.u8 [%rd127], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p86, %rs529, 143; + selp.u32 %r8960, 1, 0, %p86; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_79: + setp.ne.s32 %p87, %r8474, 0; + mov.u32 %r8485, %r8959; + @%p87 bra $L__BB1_75; + +$L__BB1_80: + setp.ne.s32 %p88, %r8431, 0; + @%p88 bra $L__BB1_128; + + setp.eq.s32 %p89, %r8455, 0; + add.s32 %r4272, %r8514, 17477; + cvt.u64.u32 %rd128, %r4272; + add.s64 %rd129, %rd128, %rd4; + add.s64 %rd10, %rd1, %rd129; + @%p89 bra $L__BB1_120; + + shl.b16 %rs1096, %rs1096, 1; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p90, %r8520, 0; + mov.u32 %r8519, %r8723; + @%p90 bra $L__BB1_85; + bra.uni $L__BB1_83; + +$L__BB1_85: + setp.lt.u32 %p92, %r8725, 3; + mov.u32 %r8489, 0; + @%p92 bra $L__BB1_88; + + setp.lt.u32 %p93, %r8725, 6; + mov.u32 %r8489, 1; + @%p93 bra $L__BB1_88; + + setp.lt.u32 %p94, %r8725, 9; + setp.eq.s32 %p95, %r8725, 11; + selp.b32 %r4278, 4, 5, %p95; + setp.lt.u32 %p96, %r8725, 11; + selp.b32 %r4279, 3, %r4278, %p96; + selp.b32 %r8489, 2, %r4279, %p94; + +$L__BB1_88: + setp.eq.s32 %p97, %r8489, 0; + @%p97 bra $L__BB1_116; + + add.s32 %r164, %r8489, -1; + and.b32 %r165, %r8489, 3; + setp.eq.s32 %p98, %r165, 0; + mov.u32 %r8499, %r8489; + mov.u32 %r8502, %r8519; + @%p98 bra $L__BB1_101; + + mov.u32 %r4281, 1; + shl.b32 %r4282, %r4281, %r164; + and.b32 %r4283, %r4282, %r8726; + setp.ne.s32 %p99, %r4283, 0; + selp.u32 %r4284, 1, 0, %p99; + cvt.u32.u16 %r4285, %rs1096; + bfi.b32 %r4286, %r4285, %r4284, 1, 8; + cvt.u16.u32 %rs1096, %r4286; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p100, %r8520, 0; + mov.u32 %r8502, %r8519; + @%p100 bra $L__BB1_93; + + setp.gt.u32 %p101, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8502, %r4281; + @%p101 bra $L__BB1_93; + + add.s32 %r4290, %r8514, 17477; + cvt.u64.u32 %rd130, %r4290; + add.s64 %rd131, %rd130, %rd4; + add.s64 %rd132, %rd1, %rd131; + st.global.u8 [%rd132], %rs1096; + add.s32 %r8514, %r8514, 1; + mov.u32 %r8520, 8; + mov.u16 %rs1096, 0; + mov.u32 %r8502, %r8519; + +$L__BB1_93: + setp.eq.s32 %p102, %r165, 1; + mov.u32 %r8519, %r8502; + mov.u32 %r8499, %r164; + @%p102 bra $L__BB1_101; + + add.s32 %r8499, %r8489, -2; + mov.u32 %r4291, 1; + shl.b32 %r4292, %r4291, %r8499; + and.b32 %r4293, %r4292, %r8726; + setp.ne.s32 %p103, %r4293, 0; + selp.u32 %r4294, 1, 0, %p103; + cvt.u32.u16 %r4295, %rs1096; + bfi.b32 %r4296, %r4295, %r4294, 1, 8; + cvt.u16.u32 %rs1096, %r4296; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p104, %r8520, 0; + mov.u32 %r8493, %r8502; + @%p104 bra $L__BB1_97; + + setp.gt.u32 %p105, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8493, %r4291; + @%p105 bra $L__BB1_97; + + add.s32 %r4299, %r8514, 17477; + cvt.u64.u32 %rd133, %r4299; + add.s64 %rd134, %rd133, %rd4; + add.s64 %rd135, %rd1, %rd134; + and.b16 %rs536, %rs1096, 255; + st.global.u8 [%rd135], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p106, %rs536, 255; + selp.b32 %r8520, 7, 8, %p106; + mov.u16 %rs1096, 0; + mov.u32 %r8493, %r8502; + +$L__BB1_97: + setp.eq.s32 %p107, %r165, 2; + mov.u32 %r8519, %r8493; + mov.u32 %r8502, %r8493; + @%p107 bra $L__BB1_101; + + add.s32 %r8499, %r8489, -3; + mov.u32 %r4300, 1; + shl.b32 %r4301, %r4300, %r8499; + and.b32 %r4302, %r4301, %r8726; + setp.ne.s32 %p108, %r4302, 0; + selp.u32 %r4303, 1, 0, %p108; + cvt.u32.u16 %r4304, %rs1096; + bfi.b32 %r4305, %r4304, %r4303, 1, 8; + cvt.u16.u32 %rs1096, %r4305; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p109, %r8520, 0; + mov.u32 %r8519, %r8493; + mov.u32 %r8502, %r8493; + @%p109 bra $L__BB1_101; + + setp.gt.u32 %p110, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8519, %r4300; + mov.u32 %r8502, %r4300; + @%p110 bra $L__BB1_101; + + add.s32 %r4310, %r8514, 17477; + cvt.u64.u32 %rd136, %r4310; + add.s64 %rd137, %rd136, %rd4; + add.s64 %rd138, %rd1, %rd137; + and.b16 %rs539, %rs1096, 255; + st.global.u8 [%rd138], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p111, %rs539, 255; + selp.b32 %r8520, 7, 8, %p111; + mov.u16 %rs1096, 0; + mov.u32 %r8519, %r8493; + mov.u32 %r8502, %r8493; + +$L__BB1_101: + setp.lt.u32 %p112, %r164, 3; + @%p112 bra $L__BB1_116; + + mov.u32 %r8519, %r8502; + +$L__BB1_103: + add.s32 %r4311, %r8499, -1; + mov.u32 %r4312, 1; + shl.b32 %r4313, %r4312, %r4311; + and.b32 %r4314, %r4313, %r8726; + setp.ne.s32 %p113, %r4314, 0; + selp.u32 %r4315, 1, 0, %p113; + cvt.u32.u16 %r4316, %rs1096; + bfi.b32 %r8508, %r4316, %r4315, 1, 8; + add.s32 %r8509, %r8520, -1; + setp.ne.s32 %p114, %r8509, 0; + mov.u32 %r8507, %r8519; + @%p114 bra $L__BB1_106; + + setp.gt.u32 %p115, %r8514, 191; + mov.u32 %r8509, 0; + mov.u32 %r8507, %r4312; + @%p115 bra $L__BB1_106; + + cvt.u16.u32 %rs540, %r8508; + and.b16 %rs541, %rs540, 255; + add.s32 %r4320, %r8514, 17477; + cvt.u64.u32 %rd139, %r4320; + add.s64 %rd140, %rd139, %rd4; + add.s64 %rd141, %rd1, %rd140; + st.global.u8 [%rd141], %rs540; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p116, %rs541, 255; + selp.b32 %r8509, 7, 8, %p116; + mov.u32 %r8508, 0; + mov.u32 %r8507, %r8519; + +$L__BB1_106: + add.s32 %r4321, %r8499, -2; + shl.b32 %r4323, %r4312, %r4321; + and.b32 %r4324, %r4323, %r8726; + setp.ne.s32 %p117, %r4324, 0; + and.b32 %r4325, %r8508, 127; + selp.u32 %r4326, 1, 0, %p117; + bfi.b32 %r8512, %r4325, %r4326, 1, 7; + add.s32 %r8513, %r8509, -1; + setp.ne.s32 %p118, %r8513, 0; + mov.u32 %r8511, %r8507; + @%p118 bra $L__BB1_109; + + setp.gt.u32 %p119, %r8514, 191; + mov.u32 %r8513, 0; + mov.u32 %r8511, 1; + @%p119 bra $L__BB1_109; + + cvt.u16.u32 %rs542, %r8512; + and.b16 %rs543, %rs542, 255; + add.s32 %r4330, %r8514, 17477; + cvt.u64.u32 %rd142, %r4330; + add.s64 %rd143, %rd142, %rd4; + add.s64 %rd144, %rd1, %rd143; + st.global.u8 [%rd144], %rs542; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p120, %rs543, 255; + selp.b32 %r8513, 7, 8, %p120; + mov.u32 %r8512, 0; + mov.u32 %r8511, %r8507; + +$L__BB1_109: + add.s32 %r4331, %r8499, -3; + mov.u32 %r4332, 1; + shl.b32 %r4333, %r4332, %r4331; + and.b32 %r4334, %r4333, %r8726; + setp.ne.s32 %p121, %r4334, 0; + and.b32 %r4335, %r8512, 127; + selp.u32 %r4336, 1, 0, %p121; + bfi.b32 %r8516, %r4335, %r4336, 1, 7; + add.s32 %r8517, %r8513, -1; + setp.ne.s32 %p122, %r8517, 0; + mov.u32 %r8515, %r8511; + @%p122 bra $L__BB1_112; + + setp.gt.u32 %p123, %r8514, 191; + mov.u32 %r8517, 0; + mov.u32 %r8515, %r4332; + @%p123 bra $L__BB1_112; + + cvt.u16.u32 %rs544, %r8516; + and.b16 %rs545, %rs544, 255; + add.s32 %r4340, %r8514, 17477; + cvt.u64.u32 %rd145, %r4340; + add.s64 %rd146, %rd145, %rd4; + add.s64 %rd147, %rd1, %rd146; + st.global.u8 [%rd147], %rs544; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p124, %rs545, 255; + selp.b32 %r8517, 7, 8, %p124; + mov.u32 %r8516, 0; + mov.u32 %r8515, %r8511; + +$L__BB1_112: + add.s32 %r8499, %r8499, -4; + shl.b32 %r4342, %r4332, %r8499; + and.b32 %r4343, %r4342, %r8726; + setp.ne.s32 %p125, %r4343, 0; + and.b32 %r4344, %r8516, 127; + selp.u32 %r4345, 1, 0, %p125; + bfi.b32 %r4346, %r4344, %r4345, 1, 15; + cvt.u16.u32 %rs1096, %r4346; + add.s32 %r8520, %r8517, -1; + setp.ne.s32 %p126, %r8520, 0; + mov.u32 %r8519, %r8515; + @%p126 bra $L__BB1_115; + + setp.gt.u32 %p127, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8519, 1; + @%p127 bra $L__BB1_115; + + add.s32 %r4349, %r8514, 17477; + cvt.u64.u32 %rd148, %r4349; + add.s64 %rd149, %rd148, %rd4; + add.s64 %rd150, %rd1, %rd149; + and.b16 %rs547, %rs1096, 255; + st.global.u8 [%rd150], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p128, %rs547, 255; + selp.b32 %r8520, 7, 8, %p128; + mov.u16 %rs1096, 0; + mov.u32 %r8519, %r8515; + +$L__BB1_115: + setp.ne.s32 %p129, %r8499, 0; + @%p129 bra $L__BB1_103; + +$L__BB1_116: + add.s32 %r4351, %r8725, -1; + setp.eq.s32 %p130, %r8725, 0; + mov.u32 %r8726, 0; + selp.b32 %r8725, 0, %r4351, %p130; + setp.lt.u32 %p131, %r8725, 3; + mov.u32 %r8525, %r8726; + @%p131 bra $L__BB1_119; + + setp.lt.u32 %p132, %r8725, 6; + mov.u32 %r8525, 1; + @%p132 bra $L__BB1_119; + + setp.lt.u32 %p133, %r8725, 9; + setp.eq.s32 %p134, %r8725, 11; + selp.b32 %r4353, 4, 5, %p134; + setp.lt.u32 %p135, %r8725, 11; + selp.b32 %r4354, 3, %r4353, %p135; + selp.b32 %r8525, 2, %r4354, %p133; + +$L__BB1_119: + mov.u32 %r4356, 1; + shl.b32 %r8724, %r4356, %r8525; + mov.u32 %r8723, %r8519; + bra.uni $L__BB1_128; + +$L__BB1_120: + add.s32 %r8726, %r8726, 1; + setp.lt.u32 %p136, %r8726, %r8724; + @%p136 bra $L__BB1_128; + + shl.b16 %rs548, %rs1096, 1; + or.b16 %rs1096, %rs548, 1; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p137, %r8520, 0; + mov.u32 %r8526, %r8723; + @%p137 bra $L__BB1_124; + + setp.gt.u32 %p138, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8526, 1; + @%p138 bra $L__BB1_124; + + and.b16 %rs550, %rs1096, 255; + st.global.u8 [%rd10], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p139, %rs550, 255; + selp.b32 %r8520, 7, 8, %p139; + mov.u16 %rs1096, 0; + mov.u32 %r8526, %r8723; + +$L__BB1_124: + add.s32 %r4360, %r8725, 1; + min.u32 %r8725, %r4360, 12; + setp.lt.u32 %p140, %r8725, 3; + mov.u32 %r8726, 0; + mov.u32 %r8529, %r8726; + @%p140 bra $L__BB1_127; + + setp.lt.u32 %p141, %r8725, 6; + mov.u32 %r8529, 1; + @%p141 bra $L__BB1_127; + + setp.lt.u32 %p142, %r8725, 9; + setp.eq.s32 %p143, %r8725, 11; + selp.b32 %r4362, 4, 5, %p143; + setp.lt.u32 %p144, %r8725, 11; + selp.b32 %r4363, 3, %r4362, %p144; + selp.b32 %r8529, 2, %r4363, %p142; + +$L__BB1_127: + mov.u32 %r4365, 1; + shl.b32 %r8724, %r4365, %r8529; + mov.u32 %r8723, %r8526; + +$L__BB1_128: + max.s32 %r248, %r8470, 1; + and.b16 %rs551, %rs4, 15; + cvt.u32.u16 %r249, %rs551; + and.b32 %r250, %r8455, 1; + setp.eq.s32 %p145, %r250, 0; + mov.u32 %r8546, %r9176; + @%p145 bra $L__BB1_135; + + and.b32 %r4366, %r249, 1; + sub.s32 %r8536, %r248, %r4366; + setp.eq.s32 %p146, %r8536, 0; + mov.u32 %r8546, %r9176; + @%p146 bra $L__BB1_135; + + mov.u32 %r4367, -1; + shl.b32 %r4368, %r4367, %r8536; + not.b32 %r4369, %r4368; + and.b32 %r8537, %r8449, %r4369; + +$L__BB1_131: + setp.gt.u32 %p147, %r9150, 17476; + mov.u32 %r8546, 1; + @%p147 bra $L__BB1_135; + + sub.s32 %r4371, %r9149, %r9148; + min.u32 %r4372, %r4371, %r8536; + setp.eq.s32 %p148, %r4372, 32; + mov.u32 %r4373, -1; + shl.b32 %r4374, %r4373, %r4372; + not.b32 %r4375, %r4374; + selp.b32 %r4376, -1, %r4375, %p148; + and.b32 %r4377, %r4376, %r8537; + shl.b32 %r4378, %r4377, %r9148; + or.b32 %r9147, %r4378, %r9147; + add.s32 %r9148, %r4372, %r9148; + shr.u32 %r8537, %r8537, %r4372; + sub.s32 %r8536, %r8536, %r4372; + setp.lt.u32 %p149, %r9148, %r9149; + @%p149 bra $L__BB1_134; + + cvt.u64.u32 %rd151, %r9150; + add.s64 %rd152, %rd151, %rd4; + add.s64 %rd153, %rd1, %rd152; + st.global.u8 [%rd153], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p150, %r9147, 255; + selp.b32 %r9149, 7, 8, %p150; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_134: + setp.ne.s32 %p151, %r8536, 0; + mov.u32 %r8546, %r9176; + @%p151 bra $L__BB1_131; + +$L__BB1_135: + setp.eq.s32 %p152, %r134, 0; + mov.u32 %r8561, %r8546; + @%p152 bra $L__BB1_142; + + shr.u32 %r4381, %r249, 1; + and.b32 %r4382, %r4381, 1; + sub.s32 %r8551, %r248, %r4382; + setp.eq.s32 %p153, %r8551, 0; + mov.u32 %r8561, %r8546; + @%p153 bra $L__BB1_142; + + mov.u32 %r4383, -1; + shl.b32 %r4384, %r4383, %r8551; + not.b32 %r4385, %r4384; + and.b32 %r8552, %r8453, %r4385; + +$L__BB1_138: + setp.gt.u32 %p154, %r9150, 17476; + mov.u32 %r8561, 1; + @%p154 bra $L__BB1_142; + + sub.s32 %r4387, %r9149, %r9148; + min.u32 %r4388, %r4387, %r8551; + setp.eq.s32 %p155, %r4388, 32; + mov.u32 %r4389, -1; + shl.b32 %r4390, %r4389, %r4388; + not.b32 %r4391, %r4390; + selp.b32 %r4392, -1, %r4391, %p155; + and.b32 %r4393, %r4392, %r8552; + shl.b32 %r4394, %r4393, %r9148; + or.b32 %r9147, %r4394, %r9147; + add.s32 %r9148, %r4388, %r9148; + shr.u32 %r8552, %r8552, %r4388; + sub.s32 %r8551, %r8551, %r4388; + setp.lt.u32 %p156, %r9148, %r9149; + @%p156 bra $L__BB1_141; + + cvt.u64.u32 %rd154, %r9150; + add.s64 %rd155, %rd154, %rd4; + add.s64 %rd156, %rd1, %rd155; + st.global.u8 [%rd156], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p157, %r9147, 255; + selp.b32 %r9149, 7, 8, %p157; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_141: + setp.ne.s32 %p158, %r8551, 0; + mov.u32 %r8561, %r8546; + @%p158 bra $L__BB1_138; + +$L__BB1_142: + and.b32 %r4397, %r8455, 4; + setp.eq.s32 %p159, %r4397, 0; + mov.u32 %r8576, %r8561; + @%p159 bra $L__BB1_149; + + shr.u32 %r4398, %r249, 2; + and.b32 %r4399, %r4398, 1; + sub.s32 %r8566, %r248, %r4399; + setp.eq.s32 %p160, %r8566, 0; + mov.u32 %r8576, %r8561; + @%p160 bra $L__BB1_149; + + mov.u32 %r4400, -1; + shl.b32 %r4401, %r4400, %r8566; + not.b32 %r4402, %r4401; + and.b32 %r8567, %r8469, %r4402; + +$L__BB1_145: + setp.gt.u32 %p161, %r9150, 17476; + mov.u32 %r8576, 1; + @%p161 bra $L__BB1_149; + + sub.s32 %r4404, %r9149, %r9148; + min.u32 %r4405, %r4404, %r8566; + setp.eq.s32 %p162, %r4405, 32; + mov.u32 %r4406, -1; + shl.b32 %r4407, %r4406, %r4405; + not.b32 %r4408, %r4407; + selp.b32 %r4409, -1, %r4408, %p162; + and.b32 %r4410, %r4409, %r8567; + shl.b32 %r4411, %r4410, %r9148; + or.b32 %r9147, %r4411, %r9147; + add.s32 %r9148, %r4405, %r9148; + shr.u32 %r8567, %r8567, %r4405; + sub.s32 %r8566, %r8566, %r4405; + setp.lt.u32 %p163, %r9148, %r9149; + @%p163 bra $L__BB1_148; + + cvt.u64.u32 %rd157, %r9150; + add.s64 %rd158, %rd157, %rd4; + add.s64 %rd159, %rd1, %rd158; + st.global.u8 [%rd159], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p164, %r9147, 255; + selp.b32 %r9149, 7, 8, %p164; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_148: + setp.ne.s32 %p165, %r8566, 0; + mov.u32 %r8576, %r8561; + @%p165 bra $L__BB1_145; + +$L__BB1_149: + setp.eq.s32 %p166, %r136, 0; + mov.u32 %r9176, %r8576; + @%p166 bra $L__BB1_156; + + shr.u32 %r4414, %r249, 3; + sub.s32 %r8581, %r248, %r4414; + setp.eq.s32 %p167, %r8581, 0; + mov.u32 %r9176, %r8576; + @%p167 bra $L__BB1_156; + + mov.u32 %r4415, -1; + shl.b32 %r4416, %r4415, %r8581; + not.b32 %r4417, %r4416; + and.b32 %r8582, %r8468, %r4417; + +$L__BB1_152: + setp.gt.u32 %p168, %r9150, 17476; + mov.u32 %r9176, 1; + @%p168 bra $L__BB1_156; + + sub.s32 %r4419, %r9149, %r9148; + min.u32 %r4420, %r4419, %r8581; + setp.eq.s32 %p169, %r4420, 32; + mov.u32 %r4421, -1; + shl.b32 %r4422, %r4421, %r4420; + not.b32 %r4423, %r4422; + selp.b32 %r4424, -1, %r4423, %p169; + and.b32 %r4425, %r4424, %r8582; + shl.b32 %r4426, %r4425, %r9148; + or.b32 %r9147, %r4426, %r9147; + add.s32 %r9148, %r4420, %r9148; + shr.u32 %r8582, %r8582, %r4420; + sub.s32 %r8581, %r8581, %r4420; + setp.lt.u32 %p170, %r9148, %r9149; + @%p170 bra $L__BB1_155; + + cvt.u64.u32 %rd160, %r9150; + add.s64 %rd161, %rd160, %rd4; + add.s64 %rd162, %rd1, %rd161; + st.global.u8 [%rd162], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p171, %r9147, 255; + selp.b32 %r9149, 7, 8, %p171; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_155: + setp.ne.s32 %p172, %r8581, 0; + mov.u32 %r9176, %r8576; + @%p172 bra $L__BB1_152; + +$L__BB1_156: + add.s32 %r4429, %r8429, 2; + setp.lt.u32 %p173, %r4429, %r5; + mul.lo.s32 %r343, %r130, 6; + cvt.u64.u32 %rd163, %r343; + add.s64 %rd11, %rd9, %rd163; + add.s32 %r4430, %r343, 2; + cvt.u64.u32 %rd164, %r4430; + add.s64 %rd12, %rd9, %rd164; + @%p173 bra $L__BB1_185; + bra.uni $L__BB1_157; + +$L__BB1_185: + cvt.u64.u32 %rd178, %r8430; + add.s64 %rd179, %rd178, %rd5; + shl.b64 %rd180, %rd179, 2; + add.s64 %rd181, %rd3, %rd180; + ld.global.u32 %r416, [%rd181]; + setp.eq.s32 %p210, %r416, 0; + mov.u32 %r8641, 0; + mov.u32 %r8640, %r8641; + @%p210 bra $L__BB1_187; + + and.b32 %r4501, %r416, -2147483648; + abs.s32 %r4502, %r416; + shl.b32 %r4503, %r4502, %r57; + or.b32 %r8640, %r4503, %r4501; + +$L__BB1_187: + shl.b32 %r4507, %r8640, 1; + shr.u32 %r4508, %r4507, %r43; + and.b32 %r419, %r4508, -2; + setp.eq.s32 %p211, %r419, 0; + mov.u32 %r8642, %r8641; + mov.u32 %r8648, %r8641; + @%p211 bra $L__BB1_189; + + add.s32 %r4510, %r419, -1; + clz.b32 %r4511, %r4510; + mov.u32 %r4512, 32; + sub.s32 %r8641, %r4512, %r4511; + shr.u32 %r4513, %r8640, 31; + add.s32 %r4514, %r4513, %r419; + add.s32 %r8642, %r4514, -2; + mov.u32 %r8648, 1; + +$L__BB1_189: + mov.u32 %r8645, 0; + mov.u32 %r8644, %r8645; + @%p63 bra $L__BB1_192; + + add.s32 %r4517, %r8430, %r1; + cvt.u64.u32 %rd182, %r4517; + add.s64 %rd183, %rd182, %rd5; + shl.b64 %rd184, %rd183, 2; + add.s64 %rd185, %rd3, %rd184; + ld.global.u32 %r425, [%rd185]; + setp.eq.s32 %p213, %r425, 0; + @%p213 bra $L__BB1_192; + + and.b32 %r4518, %r425, -2147483648; + abs.s32 %r4519, %r425; + shl.b32 %r4520, %r4519, %r57; + or.b32 %r8644, %r4520, %r4518; + +$L__BB1_192: + shl.b32 %r4523, %r8644, 1; + shr.u32 %r4524, %r4523, %r43; + and.b32 %r428, %r4524, -2; + setp.eq.s32 %p214, %r428, 0; + mov.u32 %r8646, %r8645; + mov.u32 %r8663, %r8641; + @%p214 bra $L__BB1_194; + + or.b32 %r8648, %r8648, 2; + add.s32 %r4525, %r428, -1; + clz.b32 %r4526, %r4525; + mov.u32 %r4527, 32; + sub.s32 %r8645, %r4527, %r4526; + max.s32 %r8663, %r8641, %r8645; + shr.u32 %r4528, %r8644, 31; + add.s32 %r4529, %r4528, %r428; + add.s32 %r8646, %r4529, -2; + +$L__BB1_194: + add.s32 %r8665, %r8430, 1; + add.s32 %r4534, %r8429, 3; + setp.ge.u32 %p215, %r4534, %r5; + mov.u32 %r8666, 0; + mov.u32 %r8659, %r8666; + mov.u32 %r8660, %r8666; + mov.u32 %r8661, %r8666; + mov.u32 %r8662, %r8666; + @%p215 bra $L__BB1_205; + + cvt.u64.u32 %rd186, %r8665; + add.s64 %rd187, %rd186, %rd5; + shl.b64 %rd188, %rd187, 2; + add.s64 %rd189, %rd3, %rd188; + ld.global.u32 %r438, [%rd189]; + setp.eq.s32 %p216, %r438, 0; + mov.u32 %r8660, 0; + mov.u32 %r8649, %r8660; + @%p216 bra $L__BB1_197; + + and.b32 %r4536, %r438, -2147483648; + abs.s32 %r4537, %r438; + shl.b32 %r4538, %r4537, %r57; + or.b32 %r8649, %r4538, %r4536; + +$L__BB1_197: + shl.b32 %r4541, %r8649, 1; + shr.u32 %r4542, %r4541, %r43; + and.b32 %r441, %r4542, -2; + setp.eq.s32 %p217, %r441, 0; + mov.u32 %r8662, %r8660; + @%p217 bra $L__BB1_199; + + or.b32 %r8648, %r8648, 4; + add.s32 %r4543, %r441, -1; + clz.b32 %r4544, %r4543; + mov.u32 %r4545, 32; + sub.s32 %r8660, %r4545, %r4544; + max.s32 %r8663, %r8663, %r8660; + shr.u32 %r4546, %r8649, 31; + add.s32 %r4547, %r4546, %r441; + add.s32 %r8662, %r4547, -2; + +$L__BB1_199: + mov.u32 %r8659, 0; + mov.u32 %r8654, %r8659; + @%p63 bra $L__BB1_202; + + add.s32 %r4550, %r8665, %r1; + cvt.u64.u32 %rd190, %r4550; + add.s64 %rd191, %rd190, %rd5; + shl.b64 %rd192, %rd191, 2; + add.s64 %rd193, %rd3, %rd192; + ld.global.u32 %r450, [%rd193]; + setp.eq.s32 %p219, %r450, 0; + @%p219 bra $L__BB1_202; + + and.b32 %r4551, %r450, -2147483648; + abs.s32 %r4552, %r450; + shl.b32 %r4553, %r4552, %r57; + or.b32 %r8654, %r4553, %r4551; + +$L__BB1_202: + shl.b32 %r4556, %r8654, 1; + shr.u32 %r4557, %r4556, %r43; + and.b32 %r453, %r4557, -2; + setp.eq.s32 %p220, %r453, 0; + mov.u32 %r8661, %r8659; + @%p220 bra $L__BB1_204; + + or.b32 %r8648, %r8648, 8; + add.s32 %r4558, %r453, -1; + clz.b32 %r4559, %r4558; + mov.u32 %r4560, 32; + sub.s32 %r8659, %r4560, %r4559; + max.s32 %r8663, %r8663, %r8659; + shr.u32 %r4561, %r8654, 31; + add.s32 %r4562, %r4561, %r453; + add.s32 %r8661, %r4562, -2; + +$L__BB1_204: + add.s32 %r8665, %r8430, 2; + +$L__BB1_205: + mov.u32 %r8430, %r8665; + shr.u32 %r4564, %r8455, 1; + or.b32 %r470, %r4564, %r250; + add.s32 %r4565, %r8663, -1; + setp.lt.s32 %p221, %r8663, 2; + setp.gt.s32 %p222, %r8663, 1; + selp.b32 %r471, %r4565, 0, %p222; + @%p221 bra $L__BB1_207; + + setp.eq.s32 %p223, %r8641, %r8663; + selp.u32 %r4566, 1, 0, %p223; + setp.eq.s32 %p224, %r8645, %r8663; + selp.u32 %r4567, -1, 0, %p224; + bfi.b32 %r4568, %r4567, %r4566, 1, 1; + setp.eq.s32 %p225, %r8660, %r8663; + selp.u16 %rs571, 1, 0, %p225; + mul.wide.u16 %r4569, %rs571, 4; + or.b32 %r4570, %r4568, %r4569; + setp.eq.s32 %p226, %r8659, %r8663; + selp.u16 %rs572, 1, 0, %p226; + mul.wide.u16 %r4571, %rs572, 8; + or.b32 %r8666, %r4570, %r4571; + +$L__BB1_207: + and.b32 %r4572, %r8645, 255; + and.b32 %r4573, %r8466, 255; + setp.lt.u32 %p227, %r4572, %r4573; + cvt.u16.u32 %rs573, %r8645; + selp.b16 %rs574, %rs3, %rs573, %p227; + st.shared.u8 [%r133+1], %rs574; + st.shared.u8 [%r133+2], %r8659; + and.b32 %r474, %r8648, 2; + shr.u32 %r4574, %r474, 1; + or.b32 %r4575, %r137, %r4574; + st.shared.u8 [%r135+1], %r4575; + and.b32 %r475, %r8648, 8; + shr.u32 %r4576, %r475, 3; + st.shared.u8 [%r135+2], %r4576; + shl.b32 %r4577, %r8648, 4; + shl.b32 %r4578, %r470, 8; + or.b32 %r4579, %r4577, %r4578; + or.b32 %r4580, %r4579, %r8666; + mul.wide.u32 %rd195, %r4580, 2; + add.s64 %rd196, %rd8, %rd195; + ld.global.u16 %rs48, [%rd196]; + shr.u16 %rs575, %rs48, 4; + and.b16 %rs49, %rs575, 7; + setp.eq.s16 %p228, %rs49, 0; + mov.u32 %r8678, %r8485; + @%p228 bra $L__BB1_214; + + cvt.u32.u16 %r8667, %rs49; + shr.u16 %rs576, %rs48, 8; + cvt.u32.u16 %r8668, %rs576; + +$L__BB1_209: + mov.u32 %r478, %r8667; + setp.gt.u32 %p229, %r8962, 2879; + mov.u32 %r8678, 1; + @%p229 bra $L__BB1_214; + + mov.u32 %r4582, 8; + sub.s32 %r4583, %r4582, %r8960; + sub.s32 %r4584, %r4583, %r8961; + min.u32 %r4585, %r4584, %r478; + setp.eq.s32 %p230, %r4585, 32; + mov.u32 %r4586, -1; + shl.b32 %r4587, %r4586, %r4585; + not.b32 %r4588, %r4587; + selp.b32 %r4589, -1, %r4588, %p230; + and.b32 %r4590, %r4589, %r8668; + shl.b32 %r4591, %r4590, %r8961; + cvt.u16.u32 %rs577, %r4591; + or.b16 %rs1165, %rs1165, %rs577; + add.s32 %r8961, %r4585, %r8961; + sub.s32 %r8667, %r478, %r4585; + shr.u32 %r8668, %r8668, %r4585; + setp.gt.u32 %p231, %r4584, %r478; + @%p231 bra $L__BB1_213; + + setp.ne.s32 %p232, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs578, %rs1165, 255; + setp.ne.s16 %p233, %rs578, 127; + and.pred %p234, %p232, %p233; + @%p234 bra $L__BB1_213; + + mov.u32 %r4594, 20548; + sub.s32 %r4595, %r4594, %r8962; + cvt.u64.u32 %rd197, %r4595; + add.s64 %rd198, %rd197, %rd4; + add.s64 %rd199, %rd1, %rd198; + st.global.u8 [%rd199], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p235, %rs578, 143; + selp.u32 %r8960, 1, 0, %p235; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_213: + setp.ne.s32 %p236, %r8667, 0; + mov.u32 %r8678, %r8485; + @%p236 bra $L__BB1_209; + +$L__BB1_214: + setp.ne.s32 %p237, %r470, 0; + @%p237 bra $L__BB1_262; + + setp.eq.s32 %p238, %r8648, 0; + add.s32 %r4596, %r8514, 17477; + cvt.u64.u32 %rd200, %r4596; + add.s64 %rd201, %rd200, %rd4; + add.s64 %rd13, %rd1, %rd201; + @%p238 bra $L__BB1_254; + + shl.b16 %rs1096, %rs1096, 1; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p239, %r8520, 0; + mov.u32 %r8712, %r8723; + @%p239 bra $L__BB1_219; + + setp.gt.u32 %p240, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8712, 1; + @%p240 bra $L__BB1_219; + + st.global.u8 [%rd13], %rs1096; + add.s32 %r8514, %r8514, 1; + mov.u32 %r8520, 8; + mov.u16 %rs1096, 0; + mov.u32 %r8712, %r8723; + +$L__BB1_219: + setp.lt.u32 %p241, %r8725, 3; + mov.u32 %r8682, 0; + @%p241 bra $L__BB1_222; + + setp.lt.u32 %p242, %r8725, 6; + mov.u32 %r8682, 1; + @%p242 bra $L__BB1_222; + + setp.lt.u32 %p243, %r8725, 9; + setp.eq.s32 %p244, %r8725, 11; + selp.b32 %r4602, 4, 5, %p244; + setp.lt.u32 %p245, %r8725, 11; + selp.b32 %r4603, 3, %r4602, %p245; + selp.b32 %r8682, 2, %r4603, %p243; + +$L__BB1_222: + setp.eq.s32 %p246, %r8682, 0; + @%p246 bra $L__BB1_250; + + add.s32 %r502, %r8682, -1; + and.b32 %r503, %r8682, 3; + setp.eq.s32 %p247, %r503, 0; + mov.u32 %r8692, %r8682; + mov.u32 %r8695, %r8712; + @%p247 bra $L__BB1_235; + + mov.u32 %r4605, 1; + shl.b32 %r4606, %r4605, %r502; + and.b32 %r4607, %r4606, %r8726; + setp.ne.s32 %p248, %r4607, 0; + selp.u32 %r4608, 1, 0, %p248; + cvt.u32.u16 %r4609, %rs1096; + bfi.b32 %r4610, %r4609, %r4608, 1, 8; + cvt.u16.u32 %rs1096, %r4610; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p249, %r8520, 0; + mov.u32 %r8695, %r8712; + @%p249 bra $L__BB1_227; + + setp.gt.u32 %p250, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8695, %r4605; + @%p250 bra $L__BB1_227; + + add.s32 %r4614, %r8514, 17477; + cvt.u64.u32 %rd202, %r4614; + add.s64 %rd203, %rd202, %rd4; + add.s64 %rd204, %rd1, %rd203; + st.global.u8 [%rd204], %rs1096; + add.s32 %r8514, %r8514, 1; + mov.u32 %r8520, 8; + mov.u16 %rs1096, 0; + mov.u32 %r8695, %r8712; + +$L__BB1_227: + setp.eq.s32 %p251, %r503, 1; + mov.u32 %r8712, %r8695; + mov.u32 %r8692, %r502; + @%p251 bra $L__BB1_235; + + add.s32 %r8692, %r8682, -2; + mov.u32 %r4615, 1; + shl.b32 %r4616, %r4615, %r8692; + and.b32 %r4617, %r4616, %r8726; + setp.ne.s32 %p252, %r4617, 0; + selp.u32 %r4618, 1, 0, %p252; + cvt.u32.u16 %r4619, %rs1096; + bfi.b32 %r4620, %r4619, %r4618, 1, 8; + cvt.u16.u32 %rs1096, %r4620; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p253, %r8520, 0; + mov.u32 %r8686, %r8695; + @%p253 bra $L__BB1_231; + + setp.gt.u32 %p254, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8686, %r4615; + @%p254 bra $L__BB1_231; + + add.s32 %r4623, %r8514, 17477; + cvt.u64.u32 %rd205, %r4623; + add.s64 %rd206, %rd205, %rd4; + add.s64 %rd207, %rd1, %rd206; + and.b16 %rs585, %rs1096, 255; + st.global.u8 [%rd207], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p255, %rs585, 255; + selp.b32 %r8520, 7, 8, %p255; + mov.u16 %rs1096, 0; + mov.u32 %r8686, %r8695; + +$L__BB1_231: + setp.eq.s32 %p256, %r503, 2; + mov.u32 %r8712, %r8686; + mov.u32 %r8695, %r8686; + @%p256 bra $L__BB1_235; + + add.s32 %r8692, %r8682, -3; + mov.u32 %r4624, 1; + shl.b32 %r4625, %r4624, %r8692; + and.b32 %r4626, %r4625, %r8726; + setp.ne.s32 %p257, %r4626, 0; + selp.u32 %r4627, 1, 0, %p257; + cvt.u32.u16 %r4628, %rs1096; + bfi.b32 %r4629, %r4628, %r4627, 1, 8; + cvt.u16.u32 %rs1096, %r4629; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p258, %r8520, 0; + mov.u32 %r8712, %r8686; + mov.u32 %r8695, %r8686; + @%p258 bra $L__BB1_235; + + setp.gt.u32 %p259, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8712, %r4624; + mov.u32 %r8695, %r4624; + @%p259 bra $L__BB1_235; + + add.s32 %r4634, %r8514, 17477; + cvt.u64.u32 %rd208, %r4634; + add.s64 %rd209, %rd208, %rd4; + add.s64 %rd210, %rd1, %rd209; + and.b16 %rs588, %rs1096, 255; + st.global.u8 [%rd210], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p260, %rs588, 255; + selp.b32 %r8520, 7, 8, %p260; + mov.u16 %rs1096, 0; + mov.u32 %r8712, %r8686; + mov.u32 %r8695, %r8686; + +$L__BB1_235: + setp.lt.u32 %p261, %r502, 3; + @%p261 bra $L__BB1_250; + + mov.u32 %r8712, %r8695; + +$L__BB1_237: + add.s32 %r4635, %r8692, -1; + mov.u32 %r4636, 1; + shl.b32 %r4637, %r4636, %r4635; + and.b32 %r4638, %r4637, %r8726; + setp.ne.s32 %p262, %r4638, 0; + selp.u32 %r4639, 1, 0, %p262; + cvt.u32.u16 %r4640, %rs1096; + bfi.b32 %r8701, %r4640, %r4639, 1, 8; + add.s32 %r8702, %r8520, -1; + setp.ne.s32 %p263, %r8702, 0; + mov.u32 %r8700, %r8712; + @%p263 bra $L__BB1_240; + + setp.gt.u32 %p264, %r8514, 191; + mov.u32 %r8702, 0; + mov.u32 %r8700, %r4636; + @%p264 bra $L__BB1_240; + + cvt.u16.u32 %rs589, %r8701; + and.b16 %rs590, %rs589, 255; + add.s32 %r4644, %r8514, 17477; + cvt.u64.u32 %rd211, %r4644; + add.s64 %rd212, %rd211, %rd4; + add.s64 %rd213, %rd1, %rd212; + st.global.u8 [%rd213], %rs589; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p265, %rs590, 255; + selp.b32 %r8702, 7, 8, %p265; + mov.u32 %r8701, 0; + mov.u32 %r8700, %r8712; + +$L__BB1_240: + add.s32 %r4645, %r8692, -2; + shl.b32 %r4647, %r4636, %r4645; + and.b32 %r4648, %r4647, %r8726; + setp.ne.s32 %p266, %r4648, 0; + and.b32 %r4649, %r8701, 127; + selp.u32 %r4650, 1, 0, %p266; + bfi.b32 %r8705, %r4649, %r4650, 1, 7; + add.s32 %r8706, %r8702, -1; + setp.ne.s32 %p267, %r8706, 0; + mov.u32 %r8704, %r8700; + @%p267 bra $L__BB1_243; + + setp.gt.u32 %p268, %r8514, 191; + mov.u32 %r8706, 0; + mov.u32 %r8704, 1; + @%p268 bra $L__BB1_243; + + cvt.u16.u32 %rs591, %r8705; + and.b16 %rs592, %rs591, 255; + add.s32 %r4654, %r8514, 17477; + cvt.u64.u32 %rd214, %r4654; + add.s64 %rd215, %rd214, %rd4; + add.s64 %rd216, %rd1, %rd215; + st.global.u8 [%rd216], %rs591; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p269, %rs592, 255; + selp.b32 %r8706, 7, 8, %p269; + mov.u32 %r8705, 0; + mov.u32 %r8704, %r8700; + +$L__BB1_243: + add.s32 %r4655, %r8692, -3; + mov.u32 %r4656, 1; + shl.b32 %r4657, %r4656, %r4655; + and.b32 %r4658, %r4657, %r8726; + setp.ne.s32 %p270, %r4658, 0; + and.b32 %r4659, %r8705, 127; + selp.u32 %r4660, 1, 0, %p270; + bfi.b32 %r8709, %r4659, %r4660, 1, 7; + add.s32 %r8710, %r8706, -1; + setp.ne.s32 %p271, %r8710, 0; + mov.u32 %r8708, %r8704; + @%p271 bra $L__BB1_246; + + setp.gt.u32 %p272, %r8514, 191; + mov.u32 %r8710, 0; + mov.u32 %r8708, %r4656; + @%p272 bra $L__BB1_246; + + cvt.u16.u32 %rs593, %r8709; + and.b16 %rs594, %rs593, 255; + add.s32 %r4664, %r8514, 17477; + cvt.u64.u32 %rd217, %r4664; + add.s64 %rd218, %rd217, %rd4; + add.s64 %rd219, %rd1, %rd218; + st.global.u8 [%rd219], %rs593; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p273, %rs594, 255; + selp.b32 %r8710, 7, 8, %p273; + mov.u32 %r8709, 0; + mov.u32 %r8708, %r8704; + +$L__BB1_246: + add.s32 %r8692, %r8692, -4; + shl.b32 %r4666, %r4656, %r8692; + and.b32 %r4667, %r4666, %r8726; + setp.ne.s32 %p274, %r4667, 0; + and.b32 %r4668, %r8709, 127; + selp.u32 %r4669, 1, 0, %p274; + bfi.b32 %r4670, %r4668, %r4669, 1, 15; + cvt.u16.u32 %rs1096, %r4670; + add.s32 %r8520, %r8710, -1; + setp.ne.s32 %p275, %r8520, 0; + mov.u32 %r8712, %r8708; + @%p275 bra $L__BB1_249; + + setp.gt.u32 %p276, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8712, 1; + @%p276 bra $L__BB1_249; + + add.s32 %r4673, %r8514, 17477; + cvt.u64.u32 %rd220, %r4673; + add.s64 %rd221, %rd220, %rd4; + add.s64 %rd222, %rd1, %rd221; + and.b16 %rs596, %rs1096, 255; + st.global.u8 [%rd222], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p277, %rs596, 255; + selp.b32 %r8520, 7, 8, %p277; + mov.u16 %rs1096, 0; + mov.u32 %r8712, %r8708; + +$L__BB1_249: + setp.ne.s32 %p278, %r8692, 0; + @%p278 bra $L__BB1_237; + +$L__BB1_250: + add.s32 %r4675, %r8725, -1; + setp.eq.s32 %p279, %r8725, 0; + mov.u32 %r8726, 0; + selp.b32 %r8725, 0, %r4675, %p279; + setp.lt.u32 %p280, %r8725, 3; + mov.u32 %r8718, %r8726; + @%p280 bra $L__BB1_253; + + setp.lt.u32 %p281, %r8725, 6; + mov.u32 %r8718, 1; + @%p281 bra $L__BB1_253; + + setp.lt.u32 %p282, %r8725, 9; + setp.eq.s32 %p283, %r8725, 11; + selp.b32 %r4677, 4, 5, %p283; + setp.lt.u32 %p284, %r8725, 11; + selp.b32 %r4678, 3, %r4677, %p284; + selp.b32 %r8718, 2, %r4678, %p282; + +$L__BB1_253: + mov.u32 %r4680, 1; + shl.b32 %r8724, %r4680, %r8718; + mov.u32 %r8723, %r8712; + bra.uni $L__BB1_262; + +$L__BB1_157: + ld.global.u8 %rs26, [%rd11+1]; + ld.global.u8 %rs27, [%rd12]; + ld.global.u8 %rs28, [%rd12+1]; + ld.global.u8 %rs29, [%rd9]; + ld.global.u8 %rs30, [%rd9+1]; + ld.global.u8 %rs31, [%rd9+2]; + ld.global.u8 %rs32, [%rd9+3]; + setp.eq.s16 %p174, %rs26, 0; + mov.u32 %r8607, %r8485; + @%p174 bra $L__BB1_164; + + ld.global.u8 %r8597, [%rd11]; + cvt.u32.u16 %r8596, %rs26; + +$L__BB1_159: + mov.u32 %r346, %r8596; + setp.gt.u32 %p175, %r8962, 2879; + mov.u32 %r8607, 1; + @%p175 bra $L__BB1_164; + + mov.u32 %r4432, 8; + sub.s32 %r4433, %r4432, %r8960; + sub.s32 %r4434, %r4433, %r8961; + min.u32 %r4435, %r4434, %r346; + setp.eq.s32 %p176, %r4435, 32; + mov.u32 %r4436, -1; + shl.b32 %r4437, %r4436, %r4435; + not.b32 %r4438, %r4437; + selp.b32 %r4439, -1, %r4438, %p176; + and.b32 %r4440, %r4439, %r8597; + shl.b32 %r4441, %r4440, %r8961; + cvt.u16.u32 %rs552, %r4441; + or.b16 %rs1165, %rs1165, %rs552; + add.s32 %r8961, %r4435, %r8961; + sub.s32 %r8596, %r346, %r4435; + shr.u32 %r8597, %r8597, %r4435; + setp.gt.u32 %p177, %r4434, %r346; + @%p177 bra $L__BB1_163; + + setp.ne.s32 %p178, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs553, %rs1165, 255; + setp.ne.s16 %p179, %rs553, 127; + and.pred %p180, %p178, %p179; + @%p180 bra $L__BB1_163; + + mov.u32 %r4444, 20548; + sub.s32 %r4445, %r4444, %r8962; + cvt.u64.u32 %rd166, %r4445; + add.s64 %rd167, %rd166, %rd4; + add.s64 %rd168, %rd1, %rd167; + st.global.u8 [%rd168], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p181, %rs553, 143; + selp.u32 %r8960, 1, 0, %p181; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_163: + setp.ne.s32 %p182, %r8596, 0; + mov.u32 %r8607, %r8485; + @%p182 bra $L__BB1_159; + +$L__BB1_164: + setp.eq.s16 %p183, %rs30, 0; + mov.u32 %r8619, %r8607; + @%p183 bra $L__BB1_171; + + cvt.u32.u16 %r4446, %rs29; + and.b32 %r8609, %r4446, 255; + cvt.u32.u16 %r4447, %rs30; + and.b32 %r8608, %r4447, 255; + +$L__BB1_166: + mov.u32 %r365, %r8608; + setp.gt.u32 %p184, %r8962, 2879; + mov.u32 %r8619, 1; + @%p184 bra $L__BB1_171; + + mov.u32 %r4449, 8; + sub.s32 %r4450, %r4449, %r8960; + sub.s32 %r4451, %r4450, %r8961; + min.u32 %r4452, %r4451, %r365; + setp.eq.s32 %p185, %r4452, 32; + mov.u32 %r4453, -1; + shl.b32 %r4454, %r4453, %r4452; + not.b32 %r4455, %r4454; + selp.b32 %r4456, -1, %r4455, %p185; + and.b32 %r4457, %r4456, %r8609; + shl.b32 %r4458, %r4457, %r8961; + cvt.u16.u32 %rs557, %r4458; + or.b16 %rs1165, %rs1165, %rs557; + add.s32 %r8961, %r4452, %r8961; + sub.s32 %r8608, %r365, %r4452; + shr.u32 %r8609, %r8609, %r4452; + setp.gt.u32 %p186, %r4451, %r365; + @%p186 bra $L__BB1_170; + + setp.ne.s32 %p187, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs558, %rs1165, 255; + setp.ne.s16 %p188, %rs558, 127; + and.pred %p189, %p187, %p188; + @%p189 bra $L__BB1_170; + + mov.u32 %r4461, 20548; + sub.s32 %r4462, %r4461, %r8962; + cvt.u64.u32 %rd169, %r4462; + add.s64 %rd170, %rd169, %rd4; + add.s64 %rd171, %rd1, %rd170; + st.global.u8 [%rd171], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p190, %rs558, 143; + selp.u32 %r8960, 1, 0, %p190; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_170: + setp.ne.s32 %p191, %r8608, 0; + mov.u32 %r8619, %r8607; + @%p191 bra $L__BB1_166; + +$L__BB1_171: + setp.eq.s16 %p192, %rs28, 0; + mov.u32 %r8631, %r8619; + @%p192 bra $L__BB1_178; + + cvt.u32.u16 %r4463, %rs28; + and.b32 %r8620, %r4463, 255; + cvt.u32.u16 %r4464, %rs27; + and.b32 %r8621, %r4464, 255; + +$L__BB1_173: + mov.u32 %r384, %r8620; + setp.gt.u32 %p193, %r8962, 2879; + mov.u32 %r8631, 1; + @%p193 bra $L__BB1_178; + + mov.u32 %r4466, 8; + sub.s32 %r4467, %r4466, %r8960; + sub.s32 %r4468, %r4467, %r8961; + min.u32 %r4469, %r4468, %r384; + setp.eq.s32 %p194, %r4469, 32; + mov.u32 %r4470, -1; + shl.b32 %r4471, %r4470, %r4469; + not.b32 %r4472, %r4471; + selp.b32 %r4473, -1, %r4472, %p194; + and.b32 %r4474, %r4473, %r8621; + shl.b32 %r4475, %r4474, %r8961; + cvt.u16.u32 %rs562, %r4475; + or.b16 %rs1165, %rs1165, %rs562; + add.s32 %r8961, %r4469, %r8961; + sub.s32 %r8620, %r384, %r4469; + shr.u32 %r8621, %r8621, %r4469; + setp.gt.u32 %p195, %r4468, %r384; + @%p195 bra $L__BB1_177; + + setp.ne.s32 %p196, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs563, %rs1165, 255; + setp.ne.s16 %p197, %rs563, 127; + and.pred %p198, %p196, %p197; + @%p198 bra $L__BB1_177; + + mov.u32 %r4478, 20548; + sub.s32 %r4479, %r4478, %r8962; + cvt.u64.u32 %rd172, %r4479; + add.s64 %rd173, %rd172, %rd4; + add.s64 %rd174, %rd1, %rd173; + st.global.u8 [%rd174], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p199, %rs563, 143; + selp.u32 %r8960, 1, 0, %p199; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_177: + setp.ne.s32 %p200, %r8620, 0; + mov.u32 %r8631, %r8619; + @%p200 bra $L__BB1_173; + +$L__BB1_178: + setp.eq.s16 %p201, %rs32, 0; + mov.u32 %r8431, 0; + mov.u32 %r8959, %r8631; + @%p201 bra $L__BB1_416; + + cvt.u32.u16 %r4481, %rs31; + and.b32 %r8633, %r4481, 255; + cvt.u32.u16 %r4482, %rs32; + and.b32 %r8632, %r4482, 255; + +$L__BB1_180: + mov.u32 %r403, %r8632; + setp.gt.u32 %p202, %r8962, 2879; + mov.u32 %r8959, 1; + @%p202 bra $L__BB1_416; + + mov.u32 %r4485, 8; + sub.s32 %r4486, %r4485, %r8960; + sub.s32 %r4487, %r4486, %r8961; + min.u32 %r4488, %r4487, %r403; + setp.eq.s32 %p203, %r4488, 32; + mov.u32 %r4489, -1; + shl.b32 %r4490, %r4489, %r4488; + not.b32 %r4491, %r4490; + selp.b32 %r4492, -1, %r4491, %p203; + and.b32 %r4493, %r4492, %r8633; + shl.b32 %r4494, %r4493, %r8961; + cvt.u16.u32 %rs567, %r4494; + or.b16 %rs1165, %rs1165, %rs567; + add.s32 %r8961, %r4488, %r8961; + sub.s32 %r8632, %r403, %r4488; + shr.u32 %r8633, %r8633, %r4488; + setp.gt.u32 %p204, %r4487, %r403; + @%p204 bra $L__BB1_184; + + setp.ne.s32 %p205, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs568, %rs1165, 255; + setp.ne.s16 %p206, %rs568, 127; + and.pred %p207, %p205, %p206; + @%p207 bra $L__BB1_184; + + mov.u32 %r4497, 20548; + sub.s32 %r4498, %r4497, %r8962; + cvt.u64.u32 %rd175, %r4498; + add.s64 %rd176, %rd175, %rd4; + add.s64 %rd177, %rd1, %rd176; + st.global.u8 [%rd177], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p208, %rs568, 143; + selp.u32 %r8960, 1, 0, %p208; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_184: + setp.eq.s32 %p209, %r8632, 0; + mov.u32 %r8959, %r8631; + @%p209 bra $L__BB1_416; + bra.uni $L__BB1_180; + +$L__BB1_83: + setp.gt.u32 %p91, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8519, 1; + @%p91 bra $L__BB1_85; + + st.global.u8 [%rd10], %rs1096; + add.s32 %r8514, %r8514, 1; + mov.u32 %r8520, 8; + mov.u16 %rs1096, 0; + mov.u32 %r8519, %r8723; + bra.uni $L__BB1_85; + +$L__BB1_254: + add.s32 %r8726, %r8726, 1; + setp.lt.u32 %p285, %r8726, %r8724; + @%p285 bra $L__BB1_262; + + shl.b16 %rs597, %rs1096, 1; + or.b16 %rs1096, %rs597, 1; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p286, %r8520, 0; + mov.u32 %r8719, %r8723; + @%p286 bra $L__BB1_258; + bra.uni $L__BB1_256; + +$L__BB1_258: + add.s32 %r4684, %r8725, 1; + min.u32 %r8725, %r4684, 12; + setp.lt.u32 %p289, %r8725, 3; + mov.u32 %r8726, 0; + mov.u32 %r8722, %r8726; + @%p289 bra $L__BB1_261; + + setp.lt.u32 %p290, %r8725, 6; + mov.u32 %r8722, 1; + @%p290 bra $L__BB1_261; + + setp.lt.u32 %p291, %r8725, 9; + setp.eq.s32 %p292, %r8725, 11; + selp.b32 %r4686, 4, 5, %p292; + setp.lt.u32 %p293, %r8725, 11; + selp.b32 %r4687, 3, %r4686, %p293; + selp.b32 %r8722, 2, %r4687, %p291; + +$L__BB1_261: + mov.u32 %r4689, 1; + shl.b32 %r8724, %r4689, %r8722; + mov.u32 %r8723, %r8719; + +$L__BB1_262: + max.s32 %r586, %r8663, 1; + and.b16 %rs600, %rs48, 15; + cvt.u32.u16 %r587, %rs600; + and.b32 %r588, %r8648, 1; + setp.eq.s32 %p294, %r588, 0; + mov.u32 %r8739, %r9176; + @%p294 bra $L__BB1_269; + + and.b32 %r4690, %r587, 1; + sub.s32 %r8729, %r586, %r4690; + setp.eq.s32 %p295, %r8729, 0; + mov.u32 %r8739, %r9176; + @%p295 bra $L__BB1_269; + + mov.u32 %r4691, -1; + shl.b32 %r4692, %r4691, %r8729; + not.b32 %r4693, %r4692; + and.b32 %r8730, %r8642, %r4693; + +$L__BB1_265: + setp.gt.u32 %p296, %r9150, 17476; + mov.u32 %r8739, 1; + @%p296 bra $L__BB1_269; + + sub.s32 %r4695, %r9149, %r9148; + min.u32 %r4696, %r4695, %r8729; + setp.eq.s32 %p297, %r4696, 32; + mov.u32 %r4697, -1; + shl.b32 %r4698, %r4697, %r4696; + not.b32 %r4699, %r4698; + selp.b32 %r4700, -1, %r4699, %p297; + and.b32 %r4701, %r4700, %r8730; + shl.b32 %r4702, %r4701, %r9148; + or.b32 %r9147, %r4702, %r9147; + add.s32 %r9148, %r4696, %r9148; + shr.u32 %r8730, %r8730, %r4696; + sub.s32 %r8729, %r8729, %r4696; + setp.lt.u32 %p298, %r9148, %r9149; + @%p298 bra $L__BB1_268; + + cvt.u64.u32 %rd223, %r9150; + add.s64 %rd224, %rd223, %rd4; + add.s64 %rd225, %rd1, %rd224; + st.global.u8 [%rd225], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p299, %r9147, 255; + selp.b32 %r9149, 7, 8, %p299; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_268: + setp.ne.s32 %p300, %r8729, 0; + mov.u32 %r8739, %r9176; + @%p300 bra $L__BB1_265; + +$L__BB1_269: + setp.eq.s32 %p301, %r474, 0; + mov.u32 %r8754, %r8739; + @%p301 bra $L__BB1_276; + + shr.u32 %r4705, %r587, 1; + and.b32 %r4706, %r4705, 1; + sub.s32 %r8744, %r586, %r4706; + setp.eq.s32 %p302, %r8744, 0; + mov.u32 %r8754, %r8739; + @%p302 bra $L__BB1_276; + + mov.u32 %r4707, -1; + shl.b32 %r4708, %r4707, %r8744; + not.b32 %r4709, %r4708; + and.b32 %r8745, %r8646, %r4709; + +$L__BB1_272: + setp.gt.u32 %p303, %r9150, 17476; + mov.u32 %r8754, 1; + @%p303 bra $L__BB1_276; + + sub.s32 %r4711, %r9149, %r9148; + min.u32 %r4712, %r4711, %r8744; + setp.eq.s32 %p304, %r4712, 32; + mov.u32 %r4713, -1; + shl.b32 %r4714, %r4713, %r4712; + not.b32 %r4715, %r4714; + selp.b32 %r4716, -1, %r4715, %p304; + and.b32 %r4717, %r4716, %r8745; + shl.b32 %r4718, %r4717, %r9148; + or.b32 %r9147, %r4718, %r9147; + add.s32 %r9148, %r4712, %r9148; + shr.u32 %r8745, %r8745, %r4712; + sub.s32 %r8744, %r8744, %r4712; + setp.lt.u32 %p305, %r9148, %r9149; + @%p305 bra $L__BB1_275; + + cvt.u64.u32 %rd226, %r9150; + add.s64 %rd227, %rd226, %rd4; + add.s64 %rd228, %rd1, %rd227; + st.global.u8 [%rd228], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p306, %r9147, 255; + selp.b32 %r9149, 7, 8, %p306; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_275: + setp.ne.s32 %p307, %r8744, 0; + mov.u32 %r8754, %r8739; + @%p307 bra $L__BB1_272; + +$L__BB1_276: + and.b32 %r4721, %r8648, 4; + setp.eq.s32 %p308, %r4721, 0; + mov.u32 %r8769, %r8754; + @%p308 bra $L__BB1_283; + + shr.u32 %r4722, %r587, 2; + and.b32 %r4723, %r4722, 1; + sub.s32 %r8759, %r586, %r4723; + setp.eq.s32 %p309, %r8759, 0; + mov.u32 %r8769, %r8754; + @%p309 bra $L__BB1_283; + + mov.u32 %r4724, -1; + shl.b32 %r4725, %r4724, %r8759; + not.b32 %r4726, %r4725; + and.b32 %r8760, %r8662, %r4726; + +$L__BB1_279: + setp.gt.u32 %p310, %r9150, 17476; + mov.u32 %r8769, 1; + @%p310 bra $L__BB1_283; + + sub.s32 %r4728, %r9149, %r9148; + min.u32 %r4729, %r4728, %r8759; + setp.eq.s32 %p311, %r4729, 32; + mov.u32 %r4730, -1; + shl.b32 %r4731, %r4730, %r4729; + not.b32 %r4732, %r4731; + selp.b32 %r4733, -1, %r4732, %p311; + and.b32 %r4734, %r4733, %r8760; + shl.b32 %r4735, %r4734, %r9148; + or.b32 %r9147, %r4735, %r9147; + add.s32 %r9148, %r4729, %r9148; + shr.u32 %r8760, %r8760, %r4729; + sub.s32 %r8759, %r8759, %r4729; + setp.lt.u32 %p312, %r9148, %r9149; + @%p312 bra $L__BB1_282; + + cvt.u64.u32 %rd229, %r9150; + add.s64 %rd230, %rd229, %rd4; + add.s64 %rd231, %rd1, %rd230; + st.global.u8 [%rd231], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p313, %r9147, 255; + selp.b32 %r9149, 7, 8, %p313; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_282: + setp.ne.s32 %p314, %r8759, 0; + mov.u32 %r8769, %r8754; + @%p314 bra $L__BB1_279; + +$L__BB1_283: + setp.eq.s32 %p315, %r475, 0; + mov.u32 %r9176, %r8769; + @%p315 bra $L__BB1_290; + + shr.u32 %r4738, %r587, 3; + sub.s32 %r8774, %r586, %r4738; + setp.eq.s32 %p316, %r8774, 0; + mov.u32 %r9176, %r8769; + @%p316 bra $L__BB1_290; + + mov.u32 %r4739, -1; + shl.b32 %r4740, %r4739, %r8774; + not.b32 %r4741, %r4740; + and.b32 %r8775, %r8661, %r4741; + +$L__BB1_286: + setp.gt.u32 %p317, %r9150, 17476; + mov.u32 %r9176, 1; + @%p317 bra $L__BB1_290; + + sub.s32 %r4743, %r9149, %r9148; + min.u32 %r4744, %r4743, %r8774; + setp.eq.s32 %p318, %r4744, 32; + mov.u32 %r4745, -1; + shl.b32 %r4746, %r4745, %r4744; + not.b32 %r4747, %r4746; + selp.b32 %r4748, -1, %r4747, %p318; + and.b32 %r4749, %r4748, %r8775; + shl.b32 %r4750, %r4749, %r9148; + or.b32 %r9147, %r4750, %r9147; + add.s32 %r9148, %r4744, %r9148; + shr.u32 %r8775, %r8775, %r4744; + sub.s32 %r8774, %r8774, %r4744; + setp.lt.u32 %p319, %r9148, %r9149; + @%p319 bra $L__BB1_289; + + cvt.u64.u32 %rd232, %r9150; + add.s64 %rd233, %rd232, %rd4; + add.s64 %rd234, %rd1, %rd233; + st.global.u8 [%rd234], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p320, %r9147, 255; + selp.b32 %r9149, 7, 8, %p320; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_289: + setp.ne.s32 %p321, %r8774, 0; + mov.u32 %r9176, %r8769; + @%p321 bra $L__BB1_286; + +$L__BB1_290: + setp.lt.s32 %p322, %r471, 1; + setp.lt.s32 %p323, %r130, 1; + or.pred %p324, %p323, %p322; + @%p324 bra $L__BB1_338; + + min.s32 %r4753, %r130, %r471; + setp.lt.s32 %p325, %r4753, 3; + add.s32 %r4754, %r8514, 17477; + cvt.u64.u32 %rd235, %r4754; + add.s64 %rd236, %rd235, %rd4; + add.s64 %rd14, %rd1, %rd236; + @%p325 bra $L__BB1_330; + bra.uni $L__BB1_292; + +$L__BB1_330: + add.s32 %r8726, %r8726, 1; + setp.lt.u32 %p372, %r8726, %r8724; + @%p372 bra $L__BB1_338; + + shl.b16 %rs617, %rs1096, 1; + or.b16 %rs1096, %rs617, 1; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p373, %r8520, 0; + mov.u32 %r8829, %r8723; + @%p373 bra $L__BB1_334; + + setp.gt.u32 %p374, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8829, 1; + @%p374 bra $L__BB1_334; + + and.b16 %rs619, %rs1096, 255; + st.global.u8 [%rd14], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p375, %rs619, 255; + selp.b32 %r8520, 7, 8, %p375; + mov.u16 %rs1096, 0; + mov.u32 %r8829, %r8723; + +$L__BB1_334: + add.s32 %r4842, %r8725, 1; + min.u32 %r8725, %r4842, 12; + setp.lt.u32 %p376, %r8725, 3; + mov.u32 %r8726, 0; + mov.u32 %r8832, %r8726; + @%p376 bra $L__BB1_337; + + setp.lt.u32 %p377, %r8725, 6; + mov.u32 %r8832, 1; + @%p377 bra $L__BB1_337; + + setp.lt.u32 %p378, %r8725, 9; + setp.eq.s32 %p379, %r8725, 11; + selp.b32 %r4844, 4, 5, %p379; + setp.lt.u32 %p380, %r8725, 11; + selp.b32 %r4845, 3, %r4844, %p380; + selp.b32 %r8832, 2, %r4845, %p378; + +$L__BB1_337: + mov.u32 %r4847, 1; + shl.b32 %r8724, %r4847, %r8832; + mov.u32 %r8723, %r8829; + bra.uni $L__BB1_338; + +$L__BB1_292: + shl.b16 %rs1096, %rs1096, 1; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p326, %r8520, 0; + mov.u32 %r8822, %r8723; + @%p326 bra $L__BB1_295; + + setp.gt.u32 %p327, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8822, 1; + @%p327 bra $L__BB1_295; + + st.global.u8 [%rd14], %rs1096; + add.s32 %r8514, %r8514, 1; + mov.u32 %r8520, 8; + mov.u16 %rs1096, 0; + mov.u32 %r8822, %r8723; + +$L__BB1_295: + setp.lt.u32 %p328, %r8725, 3; + mov.u32 %r8792, 0; + @%p328 bra $L__BB1_298; + + setp.lt.u32 %p329, %r8725, 6; + mov.u32 %r8792, 1; + @%p329 bra $L__BB1_298; + + setp.lt.u32 %p330, %r8725, 9; + setp.eq.s32 %p331, %r8725, 11; + selp.b32 %r4760, 4, 5, %p331; + setp.lt.u32 %p332, %r8725, 11; + selp.b32 %r4761, 3, %r4760, %p332; + selp.b32 %r8792, 2, %r4761, %p330; + +$L__BB1_298: + setp.eq.s32 %p333, %r8792, 0; + @%p333 bra $L__BB1_326; + + add.s32 %r688, %r8792, -1; + and.b32 %r689, %r8792, 3; + setp.eq.s32 %p334, %r689, 0; + mov.u32 %r8802, %r8792; + mov.u32 %r8805, %r8822; + @%p334 bra $L__BB1_311; + + mov.u32 %r4763, 1; + shl.b32 %r4764, %r4763, %r688; + and.b32 %r4765, %r4764, %r8726; + setp.ne.s32 %p335, %r4765, 0; + selp.u32 %r4766, 1, 0, %p335; + cvt.u32.u16 %r4767, %rs1096; + bfi.b32 %r4768, %r4767, %r4766, 1, 8; + cvt.u16.u32 %rs1096, %r4768; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p336, %r8520, 0; + mov.u32 %r8805, %r8822; + @%p336 bra $L__BB1_303; + + setp.gt.u32 %p337, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8805, %r4763; + @%p337 bra $L__BB1_303; + + add.s32 %r4772, %r8514, 17477; + cvt.u64.u32 %rd237, %r4772; + add.s64 %rd238, %rd237, %rd4; + add.s64 %rd239, %rd1, %rd238; + st.global.u8 [%rd239], %rs1096; + add.s32 %r8514, %r8514, 1; + mov.u32 %r8520, 8; + mov.u16 %rs1096, 0; + mov.u32 %r8805, %r8822; + +$L__BB1_303: + setp.eq.s32 %p338, %r689, 1; + mov.u32 %r8822, %r8805; + mov.u32 %r8802, %r688; + @%p338 bra $L__BB1_311; + + add.s32 %r8802, %r8792, -2; + mov.u32 %r4773, 1; + shl.b32 %r4774, %r4773, %r8802; + and.b32 %r4775, %r4774, %r8726; + setp.ne.s32 %p339, %r4775, 0; + selp.u32 %r4776, 1, 0, %p339; + cvt.u32.u16 %r4777, %rs1096; + bfi.b32 %r4778, %r4777, %r4776, 1, 8; + cvt.u16.u32 %rs1096, %r4778; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p340, %r8520, 0; + mov.u32 %r8796, %r8805; + @%p340 bra $L__BB1_307; + + setp.gt.u32 %p341, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8796, %r4773; + @%p341 bra $L__BB1_307; + + add.s32 %r4781, %r8514, 17477; + cvt.u64.u32 %rd240, %r4781; + add.s64 %rd241, %rd240, %rd4; + add.s64 %rd242, %rd1, %rd241; + and.b16 %rs605, %rs1096, 255; + st.global.u8 [%rd242], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p342, %rs605, 255; + selp.b32 %r8520, 7, 8, %p342; + mov.u16 %rs1096, 0; + mov.u32 %r8796, %r8805; + +$L__BB1_307: + setp.eq.s32 %p343, %r689, 2; + mov.u32 %r8822, %r8796; + mov.u32 %r8805, %r8796; + @%p343 bra $L__BB1_311; + + add.s32 %r8802, %r8792, -3; + mov.u32 %r4782, 1; + shl.b32 %r4783, %r4782, %r8802; + and.b32 %r4784, %r4783, %r8726; + setp.ne.s32 %p344, %r4784, 0; + selp.u32 %r4785, 1, 0, %p344; + cvt.u32.u16 %r4786, %rs1096; + bfi.b32 %r4787, %r4786, %r4785, 1, 8; + cvt.u16.u32 %rs1096, %r4787; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p345, %r8520, 0; + mov.u32 %r8822, %r8796; + mov.u32 %r8805, %r8796; + @%p345 bra $L__BB1_311; + + setp.gt.u32 %p346, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8822, %r4782; + mov.u32 %r8805, %r4782; + @%p346 bra $L__BB1_311; + + add.s32 %r4792, %r8514, 17477; + cvt.u64.u32 %rd243, %r4792; + add.s64 %rd244, %rd243, %rd4; + add.s64 %rd245, %rd1, %rd244; + and.b16 %rs608, %rs1096, 255; + st.global.u8 [%rd245], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p347, %rs608, 255; + selp.b32 %r8520, 7, 8, %p347; + mov.u16 %rs1096, 0; + mov.u32 %r8822, %r8796; + mov.u32 %r8805, %r8796; + +$L__BB1_311: + setp.lt.u32 %p348, %r688, 3; + @%p348 bra $L__BB1_326; + + mov.u32 %r8822, %r8805; + +$L__BB1_313: + add.s32 %r4793, %r8802, -1; + mov.u32 %r4794, 1; + shl.b32 %r4795, %r4794, %r4793; + and.b32 %r4796, %r4795, %r8726; + setp.ne.s32 %p349, %r4796, 0; + selp.u32 %r4797, 1, 0, %p349; + cvt.u32.u16 %r4798, %rs1096; + bfi.b32 %r8811, %r4798, %r4797, 1, 8; + add.s32 %r8812, %r8520, -1; + setp.ne.s32 %p350, %r8812, 0; + mov.u32 %r8810, %r8822; + @%p350 bra $L__BB1_316; + + setp.gt.u32 %p351, %r8514, 191; + mov.u32 %r8812, 0; + mov.u32 %r8810, %r4794; + @%p351 bra $L__BB1_316; + + cvt.u16.u32 %rs609, %r8811; + and.b16 %rs610, %rs609, 255; + add.s32 %r4802, %r8514, 17477; + cvt.u64.u32 %rd246, %r4802; + add.s64 %rd247, %rd246, %rd4; + add.s64 %rd248, %rd1, %rd247; + st.global.u8 [%rd248], %rs609; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p352, %rs610, 255; + selp.b32 %r8812, 7, 8, %p352; + mov.u32 %r8811, 0; + mov.u32 %r8810, %r8822; + +$L__BB1_316: + add.s32 %r4803, %r8802, -2; + shl.b32 %r4805, %r4794, %r4803; + and.b32 %r4806, %r4805, %r8726; + setp.ne.s32 %p353, %r4806, 0; + and.b32 %r4807, %r8811, 127; + selp.u32 %r4808, 1, 0, %p353; + bfi.b32 %r8815, %r4807, %r4808, 1, 7; + add.s32 %r8816, %r8812, -1; + setp.ne.s32 %p354, %r8816, 0; + mov.u32 %r8814, %r8810; + @%p354 bra $L__BB1_319; + + setp.gt.u32 %p355, %r8514, 191; + mov.u32 %r8816, 0; + mov.u32 %r8814, 1; + @%p355 bra $L__BB1_319; + + cvt.u16.u32 %rs611, %r8815; + and.b16 %rs612, %rs611, 255; + add.s32 %r4812, %r8514, 17477; + cvt.u64.u32 %rd249, %r4812; + add.s64 %rd250, %rd249, %rd4; + add.s64 %rd251, %rd1, %rd250; + st.global.u8 [%rd251], %rs611; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p356, %rs612, 255; + selp.b32 %r8816, 7, 8, %p356; + mov.u32 %r8815, 0; + mov.u32 %r8814, %r8810; + +$L__BB1_319: + add.s32 %r4813, %r8802, -3; + mov.u32 %r4814, 1; + shl.b32 %r4815, %r4814, %r4813; + and.b32 %r4816, %r4815, %r8726; + setp.ne.s32 %p357, %r4816, 0; + and.b32 %r4817, %r8815, 127; + selp.u32 %r4818, 1, 0, %p357; + bfi.b32 %r8819, %r4817, %r4818, 1, 7; + add.s32 %r8820, %r8816, -1; + setp.ne.s32 %p358, %r8820, 0; + mov.u32 %r8818, %r8814; + @%p358 bra $L__BB1_322; + + setp.gt.u32 %p359, %r8514, 191; + mov.u32 %r8820, 0; + mov.u32 %r8818, %r4814; + @%p359 bra $L__BB1_322; + + cvt.u16.u32 %rs613, %r8819; + and.b16 %rs614, %rs613, 255; + add.s32 %r4822, %r8514, 17477; + cvt.u64.u32 %rd252, %r4822; + add.s64 %rd253, %rd252, %rd4; + add.s64 %rd254, %rd1, %rd253; + st.global.u8 [%rd254], %rs613; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p360, %rs614, 255; + selp.b32 %r8820, 7, 8, %p360; + mov.u32 %r8819, 0; + mov.u32 %r8818, %r8814; + +$L__BB1_322: + add.s32 %r8802, %r8802, -4; + shl.b32 %r4824, %r4814, %r8802; + and.b32 %r4825, %r4824, %r8726; + setp.ne.s32 %p361, %r4825, 0; + and.b32 %r4826, %r8819, 127; + selp.u32 %r4827, 1, 0, %p361; + bfi.b32 %r4828, %r4826, %r4827, 1, 15; + cvt.u16.u32 %rs1096, %r4828; + add.s32 %r8520, %r8820, -1; + setp.ne.s32 %p362, %r8520, 0; + mov.u32 %r8822, %r8818; + @%p362 bra $L__BB1_325; + + setp.gt.u32 %p363, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r8822, 1; + @%p363 bra $L__BB1_325; + + add.s32 %r4831, %r8514, 17477; + cvt.u64.u32 %rd255, %r4831; + add.s64 %rd256, %rd255, %rd4; + add.s64 %rd257, %rd1, %rd256; + and.b16 %rs616, %rs1096, 255; + st.global.u8 [%rd257], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p364, %rs616, 255; + selp.b32 %r8520, 7, 8, %p364; + mov.u16 %rs1096, 0; + mov.u32 %r8822, %r8818; + +$L__BB1_325: + setp.ne.s32 %p365, %r8802, 0; + @%p365 bra $L__BB1_313; + +$L__BB1_326: + add.s32 %r4833, %r8725, -1; + setp.eq.s32 %p366, %r8725, 0; + mov.u32 %r8726, 0; + selp.b32 %r8725, 0, %r4833, %p366; + setp.lt.u32 %p367, %r8725, 3; + mov.u32 %r8828, %r8726; + @%p367 bra $L__BB1_329; + + setp.lt.u32 %p368, %r8725, 6; + mov.u32 %r8828, 1; + @%p368 bra $L__BB1_329; + + setp.lt.u32 %p369, %r8725, 9; + setp.eq.s32 %p370, %r8725, 11; + selp.b32 %r4835, 4, 5, %p370; + setp.lt.u32 %p371, %r8725, 11; + selp.b32 %r4836, 3, %r4835, %p371; + selp.b32 %r8828, 2, %r4836, %p369; + +$L__BB1_329: + mov.u32 %r4838, 1; + shl.b32 %r8724, %r4838, %r8828; + mov.u32 %r8723, %r8822; + +$L__BB1_338: + setp.gt.s32 %p381, %r471, 2; + setp.gt.s32 %p382, %r130, 2; + and.pred %p383, %p382, %p381; + @%p383 bra $L__BB1_387; + bra.uni $L__BB1_339; + +$L__BB1_387: + add.s32 %r4968, %r343, -11; + cvt.u64.u32 %rd287, %r4968; + add.s64 %rd16, %rd9, %rd287; + ld.global.u8 %rs122, [%rd16]; + add.s32 %r4969, %r343, -10; + cvt.u64.u32 %rd289, %r4969; + add.s64 %rd290, %rd9, %rd289; + ld.global.u8 %rs123, [%rd290]; + ld.global.u8 %rs124, [%rd290+1]; + mul.lo.s32 %r4970, %r471, 6; + add.s32 %r4971, %r4970, -12; + cvt.u64.u32 %rd291, %r4971; + add.s64 %rd292, %rd9, %rd291; + ld.global.u8 %rs125, [%rd292]; + ld.global.u8 %rs126, [%rd292+1]; + add.s32 %r4972, %r4970, -10; + cvt.u64.u32 %rd293, %r4972; + add.s64 %rd294, %rd9, %rd293; + ld.global.u8 %rs127, [%rd294]; + ld.global.u8 %rs128, [%rd294+1]; + setp.eq.s16 %p451, %rs122, 0; + mov.u32 %r8926, %r8678; + @%p451 bra $L__BB1_394; + + ld.global.u8 %r8916, [%rd16+-1]; + cvt.u32.u16 %r8915, %rs122; + +$L__BB1_389: + mov.u32 %r898, %r8915; + setp.gt.u32 %p452, %r8962, 2879; + mov.u32 %r8926, 1; + @%p452 bra $L__BB1_394; + + mov.u32 %r4974, 8; + sub.s32 %r4975, %r4974, %r8960; + sub.s32 %r4976, %r4975, %r8961; + min.u32 %r4977, %r4976, %r898; + setp.eq.s32 %p453, %r4977, 32; + mov.u32 %r4978, -1; + shl.b32 %r4979, %r4978, %r4977; + not.b32 %r4980, %r4979; + selp.b32 %r4981, -1, %r4980, %p453; + and.b32 %r4982, %r4981, %r8916; + shl.b32 %r4983, %r4982, %r8961; + cvt.u16.u32 %rs652, %r4983; + or.b16 %rs1165, %rs1165, %rs652; + add.s32 %r8961, %r4977, %r8961; + sub.s32 %r8915, %r898, %r4977; + shr.u32 %r8916, %r8916, %r4977; + setp.gt.u32 %p454, %r4976, %r898; + @%p454 bra $L__BB1_393; + + setp.ne.s32 %p455, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs653, %rs1165, 255; + setp.ne.s16 %p456, %rs653, 127; + and.pred %p457, %p455, %p456; + @%p457 bra $L__BB1_393; + + mov.u32 %r4986, 20548; + sub.s32 %r4987, %r4986, %r8962; + cvt.u64.u32 %rd295, %r4987; + add.s64 %rd296, %rd295, %rd4; + add.s64 %rd297, %rd1, %rd296; + st.global.u8 [%rd297], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p458, %rs653, 143; + selp.u32 %r8960, 1, 0, %p458; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_393: + setp.ne.s32 %p459, %r8915, 0; + mov.u32 %r8926, %r8678; + @%p459 bra $L__BB1_389; + +$L__BB1_394: + setp.eq.s16 %p460, %rs126, 0; + mov.u32 %r8938, %r8926; + @%p460 bra $L__BB1_401; + + cvt.u32.u16 %r4988, %rs125; + and.b32 %r8928, %r4988, 255; + cvt.u32.u16 %r4989, %rs126; + and.b32 %r8927, %r4989, 255; + +$L__BB1_396: + mov.u32 %r917, %r8927; + setp.gt.u32 %p461, %r8962, 2879; + mov.u32 %r8938, 1; + @%p461 bra $L__BB1_401; + + mov.u32 %r4991, 8; + sub.s32 %r4992, %r4991, %r8960; + sub.s32 %r4993, %r4992, %r8961; + min.u32 %r4994, %r4993, %r917; + setp.eq.s32 %p462, %r4994, 32; + mov.u32 %r4995, -1; + shl.b32 %r4996, %r4995, %r4994; + not.b32 %r4997, %r4996; + selp.b32 %r4998, -1, %r4997, %p462; + and.b32 %r4999, %r4998, %r8928; + shl.b32 %r5000, %r4999, %r8961; + cvt.u16.u32 %rs657, %r5000; + or.b16 %rs1165, %rs1165, %rs657; + add.s32 %r8961, %r4994, %r8961; + sub.s32 %r8927, %r917, %r4994; + shr.u32 %r8928, %r8928, %r4994; + setp.gt.u32 %p463, %r4993, %r917; + @%p463 bra $L__BB1_400; + + setp.ne.s32 %p464, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs658, %rs1165, 255; + setp.ne.s16 %p465, %rs658, 127; + and.pred %p466, %p464, %p465; + @%p466 bra $L__BB1_400; + + mov.u32 %r5003, 20548; + sub.s32 %r5004, %r5003, %r8962; + cvt.u64.u32 %rd298, %r5004; + add.s64 %rd299, %rd298, %rd4; + add.s64 %rd300, %rd1, %rd299; + st.global.u8 [%rd300], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p467, %rs658, 143; + selp.u32 %r8960, 1, 0, %p467; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_400: + setp.ne.s32 %p468, %r8927, 0; + mov.u32 %r8938, %r8926; + @%p468 bra $L__BB1_396; + +$L__BB1_401: + setp.eq.s16 %p469, %rs124, 0; + mov.u32 %r8950, %r8938; + @%p469 bra $L__BB1_408; + + cvt.u32.u16 %r5005, %rs124; + and.b32 %r8939, %r5005, 255; + cvt.u32.u16 %r5006, %rs123; + and.b32 %r8940, %r5006, 255; + +$L__BB1_403: + mov.u32 %r936, %r8939; + setp.gt.u32 %p470, %r8962, 2879; + mov.u32 %r8950, 1; + @%p470 bra $L__BB1_408; + + mov.u32 %r5008, 8; + sub.s32 %r5009, %r5008, %r8960; + sub.s32 %r5010, %r5009, %r8961; + min.u32 %r5011, %r5010, %r936; + setp.eq.s32 %p471, %r5011, 32; + mov.u32 %r5012, -1; + shl.b32 %r5013, %r5012, %r5011; + not.b32 %r5014, %r5013; + selp.b32 %r5015, -1, %r5014, %p471; + and.b32 %r5016, %r5015, %r8940; + shl.b32 %r5017, %r5016, %r8961; + cvt.u16.u32 %rs662, %r5017; + or.b16 %rs1165, %rs1165, %rs662; + add.s32 %r8961, %r5011, %r8961; + sub.s32 %r8939, %r936, %r5011; + shr.u32 %r8940, %r8940, %r5011; + setp.gt.u32 %p472, %r5010, %r936; + @%p472 bra $L__BB1_407; + + setp.ne.s32 %p473, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs663, %rs1165, 255; + setp.ne.s16 %p474, %rs663, 127; + and.pred %p475, %p473, %p474; + @%p475 bra $L__BB1_407; + + mov.u32 %r5020, 20548; + sub.s32 %r5021, %r5020, %r8962; + cvt.u64.u32 %rd301, %r5021; + add.s64 %rd302, %rd301, %rd4; + add.s64 %rd303, %rd1, %rd302; + st.global.u8 [%rd303], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p476, %rs663, 143; + selp.u32 %r8960, 1, 0, %p476; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_407: + setp.ne.s32 %p477, %r8939, 0; + mov.u32 %r8950, %r8938; + @%p477 bra $L__BB1_403; + +$L__BB1_408: + setp.eq.s16 %p478, %rs128, 0; + mov.u32 %r8959, %r8950; + @%p478 bra $L__BB1_415; + + cvt.u32.u16 %r5022, %rs127; + and.b32 %r8952, %r5022, 255; + cvt.u32.u16 %r5023, %rs128; + and.b32 %r8951, %r5023, 255; + +$L__BB1_410: + mov.u32 %r955, %r8951; + setp.gt.u32 %p479, %r8962, 2879; + mov.u32 %r8959, 1; + @%p479 bra $L__BB1_415; + + mov.u32 %r5025, 8; + sub.s32 %r5026, %r5025, %r8960; + sub.s32 %r5027, %r5026, %r8961; + min.u32 %r5028, %r5027, %r955; + setp.eq.s32 %p480, %r5028, 32; + mov.u32 %r5029, -1; + shl.b32 %r5030, %r5029, %r5028; + not.b32 %r5031, %r5030; + selp.b32 %r5032, -1, %r5031, %p480; + and.b32 %r5033, %r5032, %r8952; + shl.b32 %r5034, %r5033, %r8961; + cvt.u16.u32 %rs667, %r5034; + or.b16 %rs1165, %rs1165, %rs667; + add.s32 %r8961, %r5028, %r8961; + sub.s32 %r8951, %r955, %r5028; + shr.u32 %r8952, %r8952, %r5028; + setp.gt.u32 %p481, %r5027, %r955; + @%p481 bra $L__BB1_414; + + setp.ne.s32 %p482, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs668, %rs1165, 255; + setp.ne.s16 %p483, %rs668, 127; + and.pred %p484, %p482, %p483; + @%p484 bra $L__BB1_414; + + mov.u32 %r5037, 20548; + sub.s32 %r5038, %r5037, %r8962; + cvt.u64.u32 %rd304, %r5038; + add.s64 %rd305, %rd304, %rd4; + add.s64 %rd306, %rd1, %rd305; + st.global.u8 [%rd306], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p485, %rs668, 143; + selp.u32 %r8960, 1, 0, %p485; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_414: + setp.ne.s32 %p486, %r8951, 0; + mov.u32 %r8959, %r8950; + @%p486 bra $L__BB1_410; + bra.uni $L__BB1_415; + +$L__BB1_339: + setp.gt.s32 %p384, %r471, 0; + and.pred %p386, %p382, %p384; + @%p386 bra $L__BB1_368; + bra.uni $L__BB1_340; + +$L__BB1_368: + ld.global.u8 %rs108, [%rd11+1]; + ld.global.u8 %rs109, [%rd12]; + ld.global.u8 %rs110, [%rd12+1]; + setp.eq.s16 %p425, %rs108, 0; + mov.u32 %r8894, %r8678; + @%p425 bra $L__BB1_375; + + ld.global.u8 %r8884, [%rd11]; + cvt.u32.u16 %r8883, %rs108; + +$L__BB1_370: + mov.u32 %r846, %r8883; + setp.gt.u32 %p426, %r8962, 2879; + mov.u32 %r8894, 1; + @%p426 bra $L__BB1_375; + + mov.u32 %r4920, 8; + sub.s32 %r4921, %r4920, %r8960; + sub.s32 %r4922, %r4921, %r8961; + min.u32 %r4923, %r4922, %r846; + setp.eq.s32 %p427, %r4923, 32; + mov.u32 %r4924, -1; + shl.b32 %r4925, %r4924, %r4923; + not.b32 %r4926, %r4925; + selp.b32 %r4927, -1, %r4926, %p427; + and.b32 %r4928, %r4927, %r8884; + shl.b32 %r4929, %r4928, %r8961; + cvt.u16.u32 %rs639, %r4929; + or.b16 %rs1165, %rs1165, %rs639; + add.s32 %r8961, %r4923, %r8961; + sub.s32 %r8883, %r846, %r4923; + shr.u32 %r8884, %r8884, %r4923; + setp.gt.u32 %p428, %r4922, %r846; + @%p428 bra $L__BB1_374; + + setp.ne.s32 %p429, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs640, %rs1165, 255; + setp.ne.s16 %p430, %rs640, 127; + and.pred %p431, %p429, %p430; + @%p431 bra $L__BB1_374; + + mov.u32 %r4932, 20548; + sub.s32 %r4933, %r4932, %r8962; + cvt.u64.u32 %rd278, %r4933; + add.s64 %rd279, %rd278, %rd4; + add.s64 %rd280, %rd1, %rd279; + st.global.u8 [%rd280], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p432, %rs640, 143; + selp.u32 %r8960, 1, 0, %p432; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_374: + setp.ne.s32 %p433, %r8883, 0; + mov.u32 %r8894, %r8678; + @%p433 bra $L__BB1_370; + +$L__BB1_375: + add.s32 %r8896, %r471, -1; + cvt.u32.u16 %r4935, %rs110; + and.b32 %r8907, %r4935, 255; + cvt.u32.u16 %r4936, %rs109; + and.b32 %r8908, %r4936, 255; + mov.u32 %r4934, 1; + mov.u32 %r8895, %r4934; + +$L__BB1_376: + mov.u32 %r866, %r8895; + setp.gt.u32 %p434, %r8962, 2879; + mov.u32 %r8906, %r4934; + @%p434 bra $L__BB1_381; + + mov.u32 %r4938, 8; + sub.s32 %r4939, %r4938, %r8960; + sub.s32 %r4940, %r4939, %r8961; + min.u32 %r4941, %r4940, %r866; + setp.eq.s32 %p435, %r4941, 32; + mov.u32 %r4942, -1; + shl.b32 %r4943, %r4942, %r4941; + not.b32 %r4944, %r4943; + selp.b32 %r4945, -1, %r4944, %p435; + and.b32 %r4946, %r4945, %r8896; + shl.b32 %r4947, %r4946, %r8961; + cvt.u16.u32 %rs643, %r4947; + or.b16 %rs1165, %rs1165, %rs643; + add.s32 %r8961, %r4941, %r8961; + sub.s32 %r8895, %r866, %r4941; + shr.u32 %r8896, %r8896, %r4941; + setp.gt.u32 %p436, %r4940, %r866; + @%p436 bra $L__BB1_380; + + setp.ne.s32 %p437, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs644, %rs1165, 255; + setp.ne.s16 %p438, %rs644, 127; + and.pred %p439, %p437, %p438; + @%p439 bra $L__BB1_380; + + mov.u32 %r4950, 20548; + sub.s32 %r4951, %r4950, %r8962; + cvt.u64.u32 %rd281, %r4951; + add.s64 %rd282, %rd281, %rd4; + add.s64 %rd283, %rd1, %rd282; + st.global.u8 [%rd283], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p440, %rs644, 143; + selp.u32 %r8960, 1, 0, %p440; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_380: + setp.ne.s32 %p441, %r8895, 0; + mov.u32 %r8906, %r8894; + @%p441 bra $L__BB1_376; + +$L__BB1_381: + setp.eq.s16 %p442, %rs110, 0; + mov.u32 %r8959, %r8906; + @%p442 bra $L__BB1_415; + +$L__BB1_382: + mov.u32 %r883, %r8907; + setp.gt.u32 %p443, %r8962, 2879; + mov.u32 %r8959, 1; + @%p443 bra $L__BB1_415; + + mov.u32 %r4953, 8; + sub.s32 %r4954, %r4953, %r8960; + sub.s32 %r4955, %r4954, %r8961; + min.u32 %r4956, %r4955, %r883; + setp.eq.s32 %p444, %r4956, 32; + mov.u32 %r4957, -1; + shl.b32 %r4958, %r4957, %r4956; + not.b32 %r4959, %r4958; + selp.b32 %r4960, -1, %r4959, %p444; + and.b32 %r4961, %r4960, %r8908; + shl.b32 %r4962, %r4961, %r8961; + cvt.u16.u32 %rs648, %r4962; + or.b16 %rs1165, %rs1165, %rs648; + add.s32 %r8961, %r4956, %r8961; + sub.s32 %r8907, %r883, %r4956; + shr.u32 %r8908, %r8908, %r4956; + setp.gt.u32 %p445, %r4955, %r883; + @%p445 bra $L__BB1_386; + + setp.ne.s32 %p446, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs649, %rs1165, 255; + setp.ne.s16 %p447, %rs649, 127; + and.pred %p448, %p446, %p447; + @%p448 bra $L__BB1_386; + + mov.u32 %r4965, 20548; + sub.s32 %r4966, %r4965, %r8962; + cvt.u64.u32 %rd284, %r4966; + add.s64 %rd285, %rd284, %rd4; + add.s64 %rd286, %rd1, %rd285; + st.global.u8 [%rd286], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p449, %rs649, 143; + selp.u32 %r8960, 1, 0, %p449; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_386: + setp.eq.s32 %p450, %r8907, 0; + mov.u32 %r8959, %r8906; + @%p450 bra $L__BB1_415; + bra.uni $L__BB1_382; + +$L__BB1_340: + setp.gt.s32 %p388, %r130, 0; + selp.b32 %r4848, %r343, 0, %p388; + cvt.u64.u32 %rd258, %r4848; + add.s64 %rd15, %rd9, %rd258; + ld.global.u8 %rs86, [%rd15+1]; + add.s32 %r4849, %r4848, 2; + cvt.u64.u32 %rd260, %r4849; + add.s64 %rd261, %rd9, %rd260; + ld.global.u8 %rs87, [%rd261]; + ld.global.u8 %rs88, [%rd261+1]; + mul.lo.s32 %r4850, %r471, 6; + selp.b32 %r4851, %r4850, 0, %p384; + cvt.u64.u32 %rd262, %r4851; + add.s64 %rd263, %rd9, %rd262; + ld.global.u8 %rs89, [%rd263]; + ld.global.u8 %rs90, [%rd263+1]; + add.s32 %r4852, %r4851, 2; + cvt.u64.u32 %rd264, %r4852; + add.s64 %rd265, %rd9, %rd264; + ld.global.u8 %rs91, [%rd265]; + ld.global.u8 %rs92, [%rd265+1]; + setp.eq.s16 %p389, %rs86, 0; + mov.u32 %r8850, %r8678; + @%p389 bra $L__BB1_347; + + ld.global.u8 %r8840, [%rd15]; + cvt.u32.u16 %r8839, %rs86; + +$L__BB1_342: + mov.u32 %r774, %r8839; + setp.gt.u32 %p390, %r8962, 2879; + mov.u32 %r8850, 1; + @%p390 bra $L__BB1_347; + + mov.u32 %r4854, 8; + sub.s32 %r4855, %r4854, %r8960; + sub.s32 %r4856, %r4855, %r8961; + min.u32 %r4857, %r4856, %r774; + setp.eq.s32 %p391, %r4857, 32; + mov.u32 %r4858, -1; + shl.b32 %r4859, %r4858, %r4857; + not.b32 %r4860, %r4859; + selp.b32 %r4861, -1, %r4860, %p391; + and.b32 %r4862, %r4861, %r8840; + shl.b32 %r4863, %r4862, %r8961; + cvt.u16.u32 %rs620, %r4863; + or.b16 %rs1165, %rs1165, %rs620; + add.s32 %r8961, %r4857, %r8961; + sub.s32 %r8839, %r774, %r4857; + shr.u32 %r8840, %r8840, %r4857; + setp.gt.u32 %p392, %r4856, %r774; + @%p392 bra $L__BB1_346; + + setp.ne.s32 %p393, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs621, %rs1165, 255; + setp.ne.s16 %p394, %rs621, 127; + and.pred %p395, %p393, %p394; + @%p395 bra $L__BB1_346; + + mov.u32 %r4866, 20548; + sub.s32 %r4867, %r4866, %r8962; + cvt.u64.u32 %rd266, %r4867; + add.s64 %rd267, %rd266, %rd4; + add.s64 %rd268, %rd1, %rd267; + st.global.u8 [%rd268], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p396, %rs621, 143; + selp.u32 %r8960, 1, 0, %p396; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_346: + setp.ne.s32 %p397, %r8839, 0; + mov.u32 %r8850, %r8678; + @%p397 bra $L__BB1_342; + +$L__BB1_347: + setp.eq.s16 %p398, %rs90, 0; + mov.u32 %r8862, %r8850; + @%p398 bra $L__BB1_354; + + cvt.u32.u16 %r4868, %rs89; + and.b32 %r8852, %r4868, 255; + cvt.u32.u16 %r4869, %rs90; + and.b32 %r8851, %r4869, 255; + +$L__BB1_349: + mov.u32 %r793, %r8851; + setp.gt.u32 %p399, %r8962, 2879; + mov.u32 %r8862, 1; + @%p399 bra $L__BB1_354; + + mov.u32 %r4871, 8; + sub.s32 %r4872, %r4871, %r8960; + sub.s32 %r4873, %r4872, %r8961; + min.u32 %r4874, %r4873, %r793; + setp.eq.s32 %p400, %r4874, 32; + mov.u32 %r4875, -1; + shl.b32 %r4876, %r4875, %r4874; + not.b32 %r4877, %r4876; + selp.b32 %r4878, -1, %r4877, %p400; + and.b32 %r4879, %r4878, %r8852; + shl.b32 %r4880, %r4879, %r8961; + cvt.u16.u32 %rs625, %r4880; + or.b16 %rs1165, %rs1165, %rs625; + add.s32 %r8961, %r4874, %r8961; + sub.s32 %r8851, %r793, %r4874; + shr.u32 %r8852, %r8852, %r4874; + setp.gt.u32 %p401, %r4873, %r793; + @%p401 bra $L__BB1_353; + + setp.ne.s32 %p402, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs626, %rs1165, 255; + setp.ne.s16 %p403, %rs626, 127; + and.pred %p404, %p402, %p403; + @%p404 bra $L__BB1_353; + + mov.u32 %r4883, 20548; + sub.s32 %r4884, %r4883, %r8962; + cvt.u64.u32 %rd269, %r4884; + add.s64 %rd270, %rd269, %rd4; + add.s64 %rd271, %rd1, %rd270; + st.global.u8 [%rd271], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p405, %rs626, 143; + selp.u32 %r8960, 1, 0, %p405; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_353: + setp.ne.s32 %p406, %r8851, 0; + mov.u32 %r8862, %r8850; + @%p406 bra $L__BB1_349; + +$L__BB1_354: + setp.eq.s16 %p407, %rs88, 0; + mov.u32 %r8874, %r8862; + @%p407 bra $L__BB1_361; + + cvt.u32.u16 %r4885, %rs88; + and.b32 %r8863, %r4885, 255; + cvt.u32.u16 %r4886, %rs87; + and.b32 %r8864, %r4886, 255; + +$L__BB1_356: + mov.u32 %r812, %r8863; + setp.gt.u32 %p408, %r8962, 2879; + mov.u32 %r8874, 1; + @%p408 bra $L__BB1_361; + + mov.u32 %r4888, 8; + sub.s32 %r4889, %r4888, %r8960; + sub.s32 %r4890, %r4889, %r8961; + min.u32 %r4891, %r4890, %r812; + setp.eq.s32 %p409, %r4891, 32; + mov.u32 %r4892, -1; + shl.b32 %r4893, %r4892, %r4891; + not.b32 %r4894, %r4893; + selp.b32 %r4895, -1, %r4894, %p409; + and.b32 %r4896, %r4895, %r8864; + shl.b32 %r4897, %r4896, %r8961; + cvt.u16.u32 %rs630, %r4897; + or.b16 %rs1165, %rs1165, %rs630; + add.s32 %r8961, %r4891, %r8961; + sub.s32 %r8863, %r812, %r4891; + shr.u32 %r8864, %r8864, %r4891; + setp.gt.u32 %p410, %r4890, %r812; + @%p410 bra $L__BB1_360; + + setp.ne.s32 %p411, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs631, %rs1165, 255; + setp.ne.s16 %p412, %rs631, 127; + and.pred %p413, %p411, %p412; + @%p413 bra $L__BB1_360; + + mov.u32 %r4900, 20548; + sub.s32 %r4901, %r4900, %r8962; + cvt.u64.u32 %rd272, %r4901; + add.s64 %rd273, %rd272, %rd4; + add.s64 %rd274, %rd1, %rd273; + st.global.u8 [%rd274], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p414, %rs631, 143; + selp.u32 %r8960, 1, 0, %p414; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_360: + setp.ne.s32 %p415, %r8863, 0; + mov.u32 %r8874, %r8862; + @%p415 bra $L__BB1_356; + +$L__BB1_361: + setp.eq.s16 %p416, %rs92, 0; + mov.u32 %r8959, %r8874; + @%p416 bra $L__BB1_415; + + cvt.u32.u16 %r4902, %rs91; + and.b32 %r8876, %r4902, 255; + cvt.u32.u16 %r4903, %rs92; + and.b32 %r8875, %r4903, 255; + +$L__BB1_363: + mov.u32 %r831, %r8875; + setp.gt.u32 %p417, %r8962, 2879; + mov.u32 %r8959, 1; + @%p417 bra $L__BB1_415; + + mov.u32 %r4905, 8; + sub.s32 %r4906, %r4905, %r8960; + sub.s32 %r4907, %r4906, %r8961; + min.u32 %r4908, %r4907, %r831; + setp.eq.s32 %p418, %r4908, 32; + mov.u32 %r4909, -1; + shl.b32 %r4910, %r4909, %r4908; + not.b32 %r4911, %r4910; + selp.b32 %r4912, -1, %r4911, %p418; + and.b32 %r4913, %r4912, %r8876; + shl.b32 %r4914, %r4913, %r8961; + cvt.u16.u32 %rs635, %r4914; + or.b16 %rs1165, %rs1165, %rs635; + add.s32 %r8961, %r4908, %r8961; + sub.s32 %r8875, %r831, %r4908; + shr.u32 %r8876, %r8876, %r4908; + setp.gt.u32 %p419, %r4907, %r831; + @%p419 bra $L__BB1_367; + + setp.ne.s32 %p420, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs636, %rs1165, 255; + setp.ne.s16 %p421, %rs636, 127; + and.pred %p422, %p420, %p421; + @%p422 bra $L__BB1_367; + + mov.u32 %r4917, 20548; + sub.s32 %r4918, %r4917, %r8962; + cvt.u64.u32 %rd275, %r4918; + add.s64 %rd276, %rd275, %rd4; + add.s64 %rd277, %rd1, %rd276; + st.global.u8 [%rd277], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p423, %rs636, 143; + selp.u32 %r8960, 1, 0, %p423; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_367: + setp.eq.s32 %p424, %r8875, 0; + mov.u32 %r8959, %r8874; + @%p424 bra $L__BB1_415; + bra.uni $L__BB1_363; + +$L__BB1_415: + shr.u32 %r5039, %r8648, 1; + or.b32 %r8431, %r5039, %r588; + +$L__BB1_416: + add.s32 %r8429, %r8429, 4; + setp.lt.u32 %p487, %r8429, %r5; + @%p487 bra $L__BB1_51; + +$L__BB1_417: + add.s32 %r8398, %r5, 1; + shr.u32 %r8397, %r8398, 1; + add.s32 %r5040, %r8397, 1; + setp.gt.u32 %p488, %r5040, 512; + @%p488 bra $L__BB1_419; + + add.s32 %r8401, %r5, 1; + shr.u32 %r8400, %r8401, 1; + add.s32 %r8399, %r4095, %r8400; + mov.u16 %rs671, 0; + add.s32 %r8392, %r8399, 1; + st.shared.u8 [%r8392], %rs671; + +$L__BB1_419: + setp.lt.u32 %p489, %r6, 3; + @%p489 bra $L__BB1_665; + + ld.param.u64 %rd1420, [ j2k_htj2k_encode_codeblocks_param_5]; + ld.param.u64 %rd1413, [ j2k_htj2k_encode_codeblocks_param_4]; + mov.u32 %r5042, 31; + sub.s32 %r1006, %r5042, %r2; + mov.u32 %r8995, 2; + cvta.to.global.u64 %rd17, %rd1420; + cvta.to.global.u64 %rd18, %rd1413; + +$L__BB1_421: + ld.shared.u8 %rs151, [_ZZ32 j2k_htj2k_encode_codeblocksE13cleanup_e_val]; + mov.u16 %rs672, 0; + st.shared.u8 [_ZZ32 j2k_htj2k_encode_codeblocksE13cleanup_e_val], %rs672; + ld.shared.u8 %rs152, [_ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val]; + st.shared.u8 [_ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val], %rs672; + @%p10 bra $L__BB1_664; + + mov.u32 %r5045, 0; + ld.shared.u8 %rs673, [_ZZ32 j2k_htj2k_encode_codeblocksE13cleanup_e_val+1]; + ld.shared.u8 %rs674, [_ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val+1]; + max.u16 %rs676, %rs151, %rs673; + cvt.u32.u16 %r5046, %rs676; + add.s32 %r9013, %r5046, -1; + add.s32 %r1024, %r8995, 1; + mul.lo.s32 %r9015, %r8995, %r1; + mul.wide.u16 %r5047, %rs674, 4; + cvt.u32.u16 %r5048, %rs152; + and.b32 %r5049, %r5048, 255; + add.s32 %r9016, %r5047, %r5049; + mov.u32 %r9011, %r5045; + mov.u32 %r9012, %r5045; + mov.u32 %r9014, %r5045; + +$L__BB1_423: + cvt.u64.u32 %rd307, %r9015; + add.s64 %rd308, %rd307, %rd5; + shl.b64 %rd309, %rd308, 2; + add.s64 %rd310, %rd3, %rd309; + ld.global.u32 %r1048, [%rd310]; + setp.eq.s32 %p491, %r1048, 0; + mov.u32 %r9032, %r5045; + @%p491 bra $L__BB1_425; + + and.b32 %r5051, %r1048, -2147483648; + abs.s32 %r5052, %r1048; + shl.b32 %r5053, %r5052, %r1006; + or.b32 %r9032, %r5053, %r5051; + +$L__BB1_425: + shl.b32 %r5057, %r9032, 1; + shr.u32 %r5058, %r5057, %r43; + and.b32 %r1051, %r5058, -2; + setp.eq.s32 %p492, %r1051, 0; + mov.u32 %r9036, 0; + mov.u32 %r9033, %r9036; + mov.u32 %r9034, %r9036; + mov.u32 %r9040, %r9036; + @%p492 bra $L__BB1_427; + + add.s32 %r5060, %r1051, -1; + clz.b32 %r5061, %r5060; + mov.u32 %r5062, 32; + sub.s32 %r9033, %r5062, %r5061; + shr.u32 %r5063, %r9032, 31; + add.s32 %r5064, %r5063, %r1051; + add.s32 %r9034, %r5064, -2; + mov.u32 %r9040, 1; + +$L__BB1_427: + setp.ge.u32 %p493, %r1024, %r6; + @%p493 bra $L__BB1_430; + + add.s32 %r5067, %r9015, %r1; + cvt.u64.u32 %rd311, %r5067; + add.s64 %rd312, %rd311, %rd5; + shl.b64 %rd313, %rd312, 2; + add.s64 %rd314, %rd3, %rd313; + ld.global.u32 %r1057, [%rd314]; + setp.eq.s32 %p494, %r1057, 0; + @%p494 bra $L__BB1_430; + + and.b32 %r5068, %r1057, -2147483648; + abs.s32 %r5069, %r1057; + shl.b32 %r5070, %r5069, %r1006; + or.b32 %r9036, %r5070, %r5068; + +$L__BB1_430: + shl.b32 %r5073, %r9036, 1; + shr.u32 %r5074, %r5073, %r43; + and.b32 %r1060, %r5074, -2; + setp.eq.s32 %p495, %r1060, 0; + mov.u32 %r9051, 0; + mov.u32 %r9037, %r9051; + mov.u32 %r9038, %r9051; + mov.u32 %r9055, %r9033; + @%p495 bra $L__BB1_432; + + or.b32 %r9040, %r9040, 2; + add.s32 %r5075, %r1060, -1; + clz.b32 %r5076, %r5075; + mov.u32 %r5077, 32; + sub.s32 %r9037, %r5077, %r5076; + max.s32 %r9055, %r9033, %r9037; + shr.u32 %r5078, %r9036, 31; + add.s32 %r5079, %r5078, %r1060; + add.s32 %r9038, %r5079, -2; + +$L__BB1_432: + add.s32 %r9345, %r9015, 1; + add.s32 %r5084, %r9011, 1; + setp.ge.u32 %p496, %r5084, %r5; + mov.u32 %r9052, %r9051; + mov.u32 %r9053, %r9051; + mov.u32 %r9054, %r9051; + @%p496 bra $L__BB1_443; + + cvt.u64.u32 %rd315, %r9345; + add.s64 %rd316, %rd315, %rd5; + shl.b64 %rd317, %rd316, 2; + add.s64 %rd318, %rd3, %rd317; + ld.global.u32 %r1070, [%rd318]; + setp.eq.s32 %p497, %r1070, 0; + mov.u32 %r9052, 0; + mov.u32 %r9041, %r9052; + @%p497 bra $L__BB1_435; + + and.b32 %r5086, %r1070, -2147483648; + abs.s32 %r5087, %r1070; + shl.b32 %r5088, %r5087, %r1006; + or.b32 %r9041, %r5088, %r5086; + +$L__BB1_435: + shl.b32 %r5091, %r9041, 1; + shr.u32 %r5092, %r5091, %r43; + and.b32 %r1073, %r5092, -2; + setp.eq.s32 %p498, %r1073, 0; + mov.u32 %r9054, %r9052; + @%p498 bra $L__BB1_437; + + or.b32 %r9040, %r9040, 4; + add.s32 %r5093, %r1073, -1; + clz.b32 %r5094, %r5093; + mov.u32 %r5095, 32; + sub.s32 %r9052, %r5095, %r5094; + max.s32 %r9055, %r9055, %r9052; + shr.u32 %r5096, %r9041, 31; + add.s32 %r5097, %r5096, %r1073; + add.s32 %r9054, %r5097, -2; + +$L__BB1_437: + mov.u32 %r9051, 0; + mov.u32 %r9046, %r9051; + @%p493 bra $L__BB1_440; + + add.s32 %r5100, %r9345, %r1; + cvt.u64.u32 %rd319, %r5100; + add.s64 %rd320, %rd319, %rd5; + shl.b64 %rd321, %rd320, 2; + add.s64 %rd322, %rd3, %rd321; + ld.global.u32 %r1082, [%rd322]; + setp.eq.s32 %p500, %r1082, 0; + @%p500 bra $L__BB1_440; + + and.b32 %r5101, %r1082, -2147483648; + abs.s32 %r5102, %r1082; + shl.b32 %r5103, %r5102, %r1006; + or.b32 %r9046, %r5103, %r5101; + +$L__BB1_440: + shl.b32 %r5106, %r9046, 1; + shr.u32 %r5107, %r5106, %r43; + and.b32 %r1085, %r5107, -2; + setp.eq.s32 %p501, %r1085, 0; + mov.u32 %r9053, %r9051; + @%p501 bra $L__BB1_442; + + or.b32 %r9040, %r9040, 8; + add.s32 %r5108, %r1085, -1; + clz.b32 %r5109, %r5108; + mov.u32 %r5110, 32; + sub.s32 %r9051, %r5110, %r5109; + max.s32 %r9055, %r9055, %r9051; + shr.u32 %r5111, %r9046, 31; + add.s32 %r5112, %r5111, %r1085; + add.s32 %r9053, %r5112, -2; + +$L__BB1_442: + add.s32 %r9345, %r9015, 2; + +$L__BB1_443: + add.s32 %r5114, %r9040, -1; + and.b32 %r5115, %r5114, %r9040; + setp.ne.s32 %p502, %r5115, 0; + mov.u32 %r9058, 0; + setp.gt.s32 %p503, %r9013, 1; + and.pred %p504, %p503, %p502; + selp.b32 %r5116, %r9013, 1, %p504; + max.s32 %r1102, %r5116, %r9055; + sub.s32 %r1103, %r1102, %r5116; + setp.lt.s32 %p505, %r1103, 1; + @%p505 bra $L__BB1_445; + + setp.eq.s32 %p506, %r9033, %r9055; + selp.u32 %r5117, 1, 0, %p506; + setp.eq.s32 %p507, %r9037, %r9055; + selp.u32 %r5118, -1, 0, %p507; + bfi.b32 %r5119, %r5118, %r5117, 1, 1; + setp.eq.s32 %p508, %r9052, %r9055; + selp.u16 %rs677, 1, 0, %p508; + mul.wide.u16 %r5120, %rs677, 4; + or.b32 %r5121, %r5119, %r5120; + setp.eq.s32 %p509, %r9051, %r9055; + selp.u16 %rs678, 1, 0, %p509; + mul.wide.u16 %r5122, %rs678, 8; + or.b32 %r9058, %r5121, %r5122; + +$L__BB1_445: + shl.b32 %r5123, %r9040, 4; + shl.b32 %r5124, %r9016, 8; + or.b32 %r5125, %r5123, %r5124; + or.b32 %r5126, %r5125, %r9058; + mul.wide.u32 %rd323, %r5126, 2; + add.s64 %rd324, %rd18, %rd323; + ld.global.u16 %rs155, [%rd324]; + shr.u16 %rs679, %rs155, 4; + and.b16 %rs156, %rs679, 7; + setp.eq.s16 %p510, %rs156, 0; + mov.u32 %r9070, %r8959; + @%p510 bra $L__BB1_452; + + cvt.u32.u16 %r9059, %rs156; + shr.u16 %rs680, %rs155, 8; + cvt.u32.u16 %r9060, %rs680; + +$L__BB1_447: + mov.u32 %r1108, %r9059; + setp.gt.u32 %p511, %r8962, 2879; + mov.u32 %r9070, 1; + @%p511 bra $L__BB1_452; + + mov.u32 %r5128, 8; + sub.s32 %r5129, %r5128, %r8960; + sub.s32 %r5130, %r5129, %r8961; + min.u32 %r5131, %r5130, %r1108; + setp.eq.s32 %p512, %r5131, 32; + mov.u32 %r5132, -1; + shl.b32 %r5133, %r5132, %r5131; + not.b32 %r5134, %r5133; + selp.b32 %r5135, -1, %r5134, %p512; + and.b32 %r5136, %r5135, %r9060; + shl.b32 %r5137, %r5136, %r8961; + cvt.u16.u32 %rs681, %r5137; + or.b16 %rs1165, %rs1165, %rs681; + add.s32 %r8961, %r5131, %r8961; + sub.s32 %r9059, %r1108, %r5131; + shr.u32 %r9060, %r9060, %r5131; + setp.gt.u32 %p513, %r5130, %r1108; + @%p513 bra $L__BB1_451; + + setp.ne.s32 %p514, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs682, %rs1165, 255; + setp.ne.s16 %p515, %rs682, 127; + and.pred %p516, %p514, %p515; + @%p516 bra $L__BB1_451; + + mov.u32 %r5140, 20548; + sub.s32 %r5141, %r5140, %r8962; + cvt.u64.u32 %rd325, %r5141; + add.s64 %rd326, %rd325, %rd4; + add.s64 %rd327, %rd1, %rd326; + st.global.u8 [%rd327], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p517, %rs682, 143; + selp.u32 %r8960, 1, 0, %p517; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_451: + setp.ne.s32 %p518, %r9059, 0; + mov.u32 %r9070, %r8959; + @%p518 bra $L__BB1_447; + +$L__BB1_452: + setp.ne.s32 %p519, %r9016, 0; + @%p519 bra $L__BB1_500; + + setp.eq.s32 %p520, %r9040, 0; + add.s32 %r5142, %r8514, 17477; + cvt.u64.u32 %rd328, %r5142; + add.s64 %rd329, %rd328, %rd4; + add.s64 %rd19, %rd1, %rd329; + @%p520 bra $L__BB1_492; + + shl.b16 %rs1096, %rs1096, 1; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p521, %r8520, 0; + mov.u32 %r9104, %r8723; + @%p521 bra $L__BB1_457; + + setp.gt.u32 %p522, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9104, 1; + @%p522 bra $L__BB1_457; + + st.global.u8 [%rd19], %rs1096; + add.s32 %r8514, %r8514, 1; + mov.u32 %r8520, 8; + mov.u16 %rs1096, 0; + mov.u32 %r9104, %r8723; + +$L__BB1_457: + setp.lt.u32 %p523, %r8725, 3; + mov.u32 %r9074, 0; + @%p523 bra $L__BB1_460; + + setp.lt.u32 %p524, %r8725, 6; + mov.u32 %r9074, 1; + @%p524 bra $L__BB1_460; + + setp.lt.u32 %p525, %r8725, 9; + setp.eq.s32 %p526, %r8725, 11; + selp.b32 %r5148, 4, 5, %p526; + setp.lt.u32 %p527, %r8725, 11; + selp.b32 %r5149, 3, %r5148, %p527; + selp.b32 %r9074, 2, %r5149, %p525; + +$L__BB1_460: + setp.eq.s32 %p528, %r9074, 0; + @%p528 bra $L__BB1_488; + + add.s32 %r1132, %r9074, -1; + and.b32 %r1133, %r9074, 3; + setp.eq.s32 %p529, %r1133, 0; + mov.u32 %r9084, %r9074; + mov.u32 %r9087, %r9104; + @%p529 bra $L__BB1_473; + + mov.u32 %r5151, 1; + shl.b32 %r5152, %r5151, %r1132; + and.b32 %r5153, %r5152, %r8726; + setp.ne.s32 %p530, %r5153, 0; + selp.u32 %r5154, 1, 0, %p530; + cvt.u32.u16 %r5155, %rs1096; + bfi.b32 %r5156, %r5155, %r5154, 1, 8; + cvt.u16.u32 %rs1096, %r5156; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p531, %r8520, 0; + mov.u32 %r9087, %r9104; + @%p531 bra $L__BB1_465; + + setp.gt.u32 %p532, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9087, %r5151; + @%p532 bra $L__BB1_465; + + add.s32 %r5160, %r8514, 17477; + cvt.u64.u32 %rd330, %r5160; + add.s64 %rd331, %rd330, %rd4; + add.s64 %rd332, %rd1, %rd331; + st.global.u8 [%rd332], %rs1096; + add.s32 %r8514, %r8514, 1; + mov.u32 %r8520, 8; + mov.u16 %rs1096, 0; + mov.u32 %r9087, %r9104; + +$L__BB1_465: + setp.eq.s32 %p533, %r1133, 1; + mov.u32 %r9104, %r9087; + mov.u32 %r9084, %r1132; + @%p533 bra $L__BB1_473; + + add.s32 %r9084, %r9074, -2; + mov.u32 %r5161, 1; + shl.b32 %r5162, %r5161, %r9084; + and.b32 %r5163, %r5162, %r8726; + setp.ne.s32 %p534, %r5163, 0; + selp.u32 %r5164, 1, 0, %p534; + cvt.u32.u16 %r5165, %rs1096; + bfi.b32 %r5166, %r5165, %r5164, 1, 8; + cvt.u16.u32 %rs1096, %r5166; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p535, %r8520, 0; + mov.u32 %r9078, %r9087; + @%p535 bra $L__BB1_469; + + setp.gt.u32 %p536, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9078, %r5161; + @%p536 bra $L__BB1_469; + + add.s32 %r5169, %r8514, 17477; + cvt.u64.u32 %rd333, %r5169; + add.s64 %rd334, %rd333, %rd4; + add.s64 %rd335, %rd1, %rd334; + and.b16 %rs689, %rs1096, 255; + st.global.u8 [%rd335], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p537, %rs689, 255; + selp.b32 %r8520, 7, 8, %p537; + mov.u16 %rs1096, 0; + mov.u32 %r9078, %r9087; + +$L__BB1_469: + setp.eq.s32 %p538, %r1133, 2; + mov.u32 %r9104, %r9078; + mov.u32 %r9087, %r9078; + @%p538 bra $L__BB1_473; + + add.s32 %r9084, %r9074, -3; + mov.u32 %r5170, 1; + shl.b32 %r5171, %r5170, %r9084; + and.b32 %r5172, %r5171, %r8726; + setp.ne.s32 %p539, %r5172, 0; + selp.u32 %r5173, 1, 0, %p539; + cvt.u32.u16 %r5174, %rs1096; + bfi.b32 %r5175, %r5174, %r5173, 1, 8; + cvt.u16.u32 %rs1096, %r5175; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p540, %r8520, 0; + mov.u32 %r9104, %r9078; + mov.u32 %r9087, %r9078; + @%p540 bra $L__BB1_473; + + setp.gt.u32 %p541, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9104, %r5170; + mov.u32 %r9087, %r5170; + @%p541 bra $L__BB1_473; + + add.s32 %r5180, %r8514, 17477; + cvt.u64.u32 %rd336, %r5180; + add.s64 %rd337, %rd336, %rd4; + add.s64 %rd338, %rd1, %rd337; + and.b16 %rs692, %rs1096, 255; + st.global.u8 [%rd338], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p542, %rs692, 255; + selp.b32 %r8520, 7, 8, %p542; + mov.u16 %rs1096, 0; + mov.u32 %r9104, %r9078; + mov.u32 %r9087, %r9078; + +$L__BB1_473: + setp.lt.u32 %p543, %r1132, 3; + @%p543 bra $L__BB1_488; + + mov.u32 %r9104, %r9087; + +$L__BB1_475: + add.s32 %r5181, %r9084, -1; + mov.u32 %r5182, 1; + shl.b32 %r5183, %r5182, %r5181; + and.b32 %r5184, %r5183, %r8726; + setp.ne.s32 %p544, %r5184, 0; + selp.u32 %r5185, 1, 0, %p544; + cvt.u32.u16 %r5186, %rs1096; + bfi.b32 %r9093, %r5186, %r5185, 1, 8; + add.s32 %r9094, %r8520, -1; + setp.ne.s32 %p545, %r9094, 0; + mov.u32 %r9092, %r9104; + @%p545 bra $L__BB1_478; + + setp.gt.u32 %p546, %r8514, 191; + mov.u32 %r9094, 0; + mov.u32 %r9092, %r5182; + @%p546 bra $L__BB1_478; + + cvt.u16.u32 %rs693, %r9093; + and.b16 %rs694, %rs693, 255; + add.s32 %r5190, %r8514, 17477; + cvt.u64.u32 %rd339, %r5190; + add.s64 %rd340, %rd339, %rd4; + add.s64 %rd341, %rd1, %rd340; + st.global.u8 [%rd341], %rs693; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p547, %rs694, 255; + selp.b32 %r9094, 7, 8, %p547; + mov.u32 %r9093, 0; + mov.u32 %r9092, %r9104; + +$L__BB1_478: + add.s32 %r5191, %r9084, -2; + shl.b32 %r5193, %r5182, %r5191; + and.b32 %r5194, %r5193, %r8726; + setp.ne.s32 %p548, %r5194, 0; + and.b32 %r5195, %r9093, 127; + selp.u32 %r5196, 1, 0, %p548; + bfi.b32 %r9097, %r5195, %r5196, 1, 7; + add.s32 %r9098, %r9094, -1; + setp.ne.s32 %p549, %r9098, 0; + mov.u32 %r9096, %r9092; + @%p549 bra $L__BB1_481; + + setp.gt.u32 %p550, %r8514, 191; + mov.u32 %r9098, 0; + mov.u32 %r9096, 1; + @%p550 bra $L__BB1_481; + + cvt.u16.u32 %rs695, %r9097; + and.b16 %rs696, %rs695, 255; + add.s32 %r5200, %r8514, 17477; + cvt.u64.u32 %rd342, %r5200; + add.s64 %rd343, %rd342, %rd4; + add.s64 %rd344, %rd1, %rd343; + st.global.u8 [%rd344], %rs695; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p551, %rs696, 255; + selp.b32 %r9098, 7, 8, %p551; + mov.u32 %r9097, 0; + mov.u32 %r9096, %r9092; + +$L__BB1_481: + add.s32 %r5201, %r9084, -3; + mov.u32 %r5202, 1; + shl.b32 %r5203, %r5202, %r5201; + and.b32 %r5204, %r5203, %r8726; + setp.ne.s32 %p552, %r5204, 0; + and.b32 %r5205, %r9097, 127; + selp.u32 %r5206, 1, 0, %p552; + bfi.b32 %r9101, %r5205, %r5206, 1, 7; + add.s32 %r9102, %r9098, -1; + setp.ne.s32 %p553, %r9102, 0; + mov.u32 %r9100, %r9096; + @%p553 bra $L__BB1_484; + + setp.gt.u32 %p554, %r8514, 191; + mov.u32 %r9102, 0; + mov.u32 %r9100, %r5202; + @%p554 bra $L__BB1_484; + + cvt.u16.u32 %rs697, %r9101; + and.b16 %rs698, %rs697, 255; + add.s32 %r5210, %r8514, 17477; + cvt.u64.u32 %rd345, %r5210; + add.s64 %rd346, %rd345, %rd4; + add.s64 %rd347, %rd1, %rd346; + st.global.u8 [%rd347], %rs697; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p555, %rs698, 255; + selp.b32 %r9102, 7, 8, %p555; + mov.u32 %r9101, 0; + mov.u32 %r9100, %r9096; + +$L__BB1_484: + add.s32 %r9084, %r9084, -4; + shl.b32 %r5212, %r5202, %r9084; + and.b32 %r5213, %r5212, %r8726; + setp.ne.s32 %p556, %r5213, 0; + and.b32 %r5214, %r9101, 127; + selp.u32 %r5215, 1, 0, %p556; + bfi.b32 %r5216, %r5214, %r5215, 1, 15; + cvt.u16.u32 %rs1096, %r5216; + add.s32 %r8520, %r9102, -1; + setp.ne.s32 %p557, %r8520, 0; + mov.u32 %r9104, %r9100; + @%p557 bra $L__BB1_487; + + setp.gt.u32 %p558, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9104, 1; + @%p558 bra $L__BB1_487; + + add.s32 %r5219, %r8514, 17477; + cvt.u64.u32 %rd348, %r5219; + add.s64 %rd349, %rd348, %rd4; + add.s64 %rd350, %rd1, %rd349; + and.b16 %rs700, %rs1096, 255; + st.global.u8 [%rd350], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p559, %rs700, 255; + selp.b32 %r8520, 7, 8, %p559; + mov.u16 %rs1096, 0; + mov.u32 %r9104, %r9100; + +$L__BB1_487: + setp.ne.s32 %p560, %r9084, 0; + @%p560 bra $L__BB1_475; + +$L__BB1_488: + add.s32 %r5221, %r8725, -1; + setp.eq.s32 %p561, %r8725, 0; + mov.u32 %r8726, 0; + selp.b32 %r8725, 0, %r5221, %p561; + setp.lt.u32 %p562, %r8725, 3; + mov.u32 %r9110, %r8726; + @%p562 bra $L__BB1_491; + + setp.lt.u32 %p563, %r8725, 6; + mov.u32 %r9110, 1; + @%p563 bra $L__BB1_491; + + setp.lt.u32 %p564, %r8725, 9; + setp.eq.s32 %p565, %r8725, 11; + selp.b32 %r5223, 4, 5, %p565; + setp.lt.u32 %p566, %r8725, 11; + selp.b32 %r5224, 3, %r5223, %p566; + selp.b32 %r9110, 2, %r5224, %p564; + +$L__BB1_491: + mov.u32 %r5226, 1; + shl.b32 %r8724, %r5226, %r9110; + mov.u32 %r8723, %r9104; + bra.uni $L__BB1_500; + +$L__BB1_492: + add.s32 %r8726, %r8726, 1; + setp.lt.u32 %p567, %r8726, %r8724; + @%p567 bra $L__BB1_500; + + shl.b16 %rs701, %rs1096, 1; + or.b16 %rs1096, %rs701, 1; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p568, %r8520, 0; + mov.u32 %r9111, %r8723; + @%p568 bra $L__BB1_496; + + setp.gt.u32 %p569, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9111, 1; + @%p569 bra $L__BB1_496; + + and.b16 %rs703, %rs1096, 255; + st.global.u8 [%rd19], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p570, %rs703, 255; + selp.b32 %r8520, 7, 8, %p570; + mov.u16 %rs1096, 0; + mov.u32 %r9111, %r8723; + +$L__BB1_496: + add.s32 %r5230, %r8725, 1; + min.u32 %r8725, %r5230, 12; + setp.lt.u32 %p571, %r8725, 3; + mov.u32 %r8726, 0; + mov.u32 %r9114, %r8726; + @%p571 bra $L__BB1_499; + + setp.lt.u32 %p572, %r8725, 6; + mov.u32 %r9114, 1; + @%p572 bra $L__BB1_499; + + setp.lt.u32 %p573, %r8725, 9; + setp.eq.s32 %p574, %r8725, 11; + selp.b32 %r5232, 4, 5, %p574; + setp.lt.u32 %p575, %r8725, 11; + selp.b32 %r5233, 3, %r5232, %p575; + selp.b32 %r9114, 2, %r5233, %p573; + +$L__BB1_499: + mov.u32 %r5235, 1; + shl.b32 %r8724, %r5235, %r9114; + mov.u32 %r8723, %r9111; + +$L__BB1_500: + and.b16 %rs704, %rs155, 15; + cvt.u32.u16 %r1216, %rs704; + and.b32 %r5236, %r9040, 1; + setp.eq.b32 %p576, %r5236, 1; + mov.pred %p577, 0; + xor.pred %p578, %p576, %p577; + not.pred %p579, %p578; + mov.u32 %r9131, %r9176; + @%p579 bra $L__BB1_507; + + and.b32 %r5237, %r1216, 1; + sub.s32 %r9121, %r1102, %r5237; + setp.eq.s32 %p580, %r9121, 0; + mov.u32 %r9131, %r9176; + @%p580 bra $L__BB1_507; + + mov.u32 %r5238, -1; + shl.b32 %r5239, %r5238, %r9121; + not.b32 %r5240, %r5239; + and.b32 %r9122, %r9034, %r5240; + +$L__BB1_503: + setp.gt.u32 %p581, %r9150, 17476; + mov.u32 %r9131, 1; + @%p581 bra $L__BB1_507; + + sub.s32 %r5242, %r9149, %r9148; + min.u32 %r5243, %r5242, %r9121; + setp.eq.s32 %p582, %r5243, 32; + mov.u32 %r5244, -1; + shl.b32 %r5245, %r5244, %r5243; + not.b32 %r5246, %r5245; + selp.b32 %r5247, -1, %r5246, %p582; + and.b32 %r5248, %r5247, %r9122; + shl.b32 %r5249, %r5248, %r9148; + or.b32 %r9147, %r5249, %r9147; + add.s32 %r9148, %r5243, %r9148; + shr.u32 %r9122, %r9122, %r5243; + sub.s32 %r9121, %r9121, %r5243; + setp.lt.u32 %p583, %r9148, %r9149; + @%p583 bra $L__BB1_506; + + cvt.u64.u32 %rd351, %r9150; + add.s64 %rd352, %rd351, %rd4; + add.s64 %rd353, %rd1, %rd352; + st.global.u8 [%rd353], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p584, %r9147, 255; + selp.b32 %r9149, 7, 8, %p584; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_506: + setp.ne.s32 %p585, %r9121, 0; + mov.u32 %r9131, %r9176; + @%p585 bra $L__BB1_503; + +$L__BB1_507: + and.b32 %r1240, %r9040, 2; + setp.eq.s32 %p586, %r1240, 0; + mov.u32 %r9146, %r9131; + @%p586 bra $L__BB1_514; + + shr.u32 %r5252, %r1216, 1; + and.b32 %r5253, %r5252, 1; + sub.s32 %r9136, %r1102, %r5253; + setp.eq.s32 %p587, %r9136, 0; + mov.u32 %r9146, %r9131; + @%p587 bra $L__BB1_514; + + mov.u32 %r5254, -1; + shl.b32 %r5255, %r5254, %r9136; + not.b32 %r5256, %r5255; + and.b32 %r9137, %r9038, %r5256; + +$L__BB1_510: + setp.gt.u32 %p588, %r9150, 17476; + mov.u32 %r9146, 1; + @%p588 bra $L__BB1_514; + + sub.s32 %r5258, %r9149, %r9148; + min.u32 %r5259, %r5258, %r9136; + setp.eq.s32 %p589, %r5259, 32; + mov.u32 %r5260, -1; + shl.b32 %r5261, %r5260, %r5259; + not.b32 %r5262, %r5261; + selp.b32 %r5263, -1, %r5262, %p589; + and.b32 %r5264, %r5263, %r9137; + shl.b32 %r5265, %r5264, %r9148; + or.b32 %r9147, %r5265, %r9147; + add.s32 %r9148, %r5259, %r9148; + shr.u32 %r9137, %r9137, %r5259; + sub.s32 %r9136, %r9136, %r5259; + setp.lt.u32 %p590, %r9148, %r9149; + @%p590 bra $L__BB1_513; + + cvt.u64.u32 %rd354, %r9150; + add.s64 %rd355, %rd354, %rd4; + add.s64 %rd356, %rd1, %rd355; + st.global.u8 [%rd356], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p591, %r9147, 255; + selp.b32 %r9149, 7, 8, %p591; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_513: + setp.ne.s32 %p592, %r9136, 0; + mov.u32 %r9146, %r9131; + @%p592 bra $L__BB1_510; + +$L__BB1_514: + and.b32 %r1264, %r9040, 4; + setp.eq.s32 %p593, %r1264, 0; + mov.u32 %r9161, %r9146; + @%p593 bra $L__BB1_521; + + shr.u32 %r5268, %r1216, 2; + and.b32 %r5269, %r5268, 1; + sub.s32 %r9151, %r1102, %r5269; + setp.eq.s32 %p594, %r9151, 0; + mov.u32 %r9161, %r9146; + @%p594 bra $L__BB1_521; + + mov.u32 %r5270, -1; + shl.b32 %r5271, %r5270, %r9151; + not.b32 %r5272, %r5271; + and.b32 %r9152, %r9054, %r5272; + +$L__BB1_517: + setp.gt.u32 %p595, %r9150, 17476; + mov.u32 %r9161, 1; + @%p595 bra $L__BB1_521; + + sub.s32 %r5274, %r9149, %r9148; + min.u32 %r5275, %r5274, %r9151; + setp.eq.s32 %p596, %r5275, 32; + mov.u32 %r5276, -1; + shl.b32 %r5277, %r5276, %r5275; + not.b32 %r5278, %r5277; + selp.b32 %r5279, -1, %r5278, %p596; + and.b32 %r5280, %r5279, %r9152; + shl.b32 %r5281, %r5280, %r9148; + or.b32 %r9147, %r5281, %r9147; + add.s32 %r9148, %r5275, %r9148; + shr.u32 %r9152, %r9152, %r5275; + sub.s32 %r9151, %r9151, %r5275; + setp.lt.u32 %p597, %r9148, %r9149; + @%p597 bra $L__BB1_520; + + cvt.u64.u32 %rd357, %r9150; + add.s64 %rd358, %rd357, %rd4; + add.s64 %rd359, %rd1, %rd358; + st.global.u8 [%rd359], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p598, %r9147, 255; + selp.b32 %r9149, 7, 8, %p598; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_520: + setp.ne.s32 %p599, %r9151, 0; + mov.u32 %r9161, %r9146; + @%p599 bra $L__BB1_517; + +$L__BB1_521: + and.b32 %r1288, %r9040, 8; + setp.eq.s32 %p600, %r1288, 0; + mov.u32 %r9176, %r9161; + @%p600 bra $L__BB1_528; + + shr.u32 %r5284, %r1216, 3; + sub.s32 %r9166, %r1102, %r5284; + setp.eq.s32 %p601, %r9166, 0; + mov.u32 %r9176, %r9161; + @%p601 bra $L__BB1_528; + + mov.u32 %r5285, -1; + shl.b32 %r5286, %r5285, %r9166; + not.b32 %r5287, %r5286; + and.b32 %r9167, %r9053, %r5287; + +$L__BB1_524: + setp.gt.u32 %p602, %r9150, 17476; + mov.u32 %r9176, 1; + @%p602 bra $L__BB1_528; + + sub.s32 %r5289, %r9149, %r9148; + min.u32 %r5290, %r5289, %r9166; + setp.eq.s32 %p603, %r5290, 32; + mov.u32 %r5291, -1; + shl.b32 %r5292, %r5291, %r5290; + not.b32 %r5293, %r5292; + selp.b32 %r5294, -1, %r5293, %p603; + and.b32 %r5295, %r5294, %r9167; + shl.b32 %r5296, %r5295, %r9148; + or.b32 %r9147, %r5296, %r9147; + add.s32 %r9148, %r5290, %r9148; + shr.u32 %r9167, %r9167, %r5290; + sub.s32 %r9166, %r9166, %r5290; + setp.lt.u32 %p604, %r9148, %r9149; + @%p604 bra $L__BB1_527; + + cvt.u64.u32 %rd360, %r9150; + add.s64 %rd361, %rd360, %rd4; + add.s64 %rd362, %rd1, %rd361; + st.global.u8 [%rd362], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p605, %r9147, 255; + selp.b32 %r9149, 7, 8, %p605; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_527: + setp.ne.s32 %p606, %r9166, 0; + mov.u32 %r9176, %r9161; + @%p606 bra $L__BB1_524; + +$L__BB1_528: + add.s32 %r1312, %r4095, %r9014; + ld.shared.u8 %rs705, [%r1312]; + mov.u32 %r9016, 0; + cvt.u32.u16 %r5302, %rs705; + and.b32 %r5303, %r5302, 255; + and.b32 %r5304, %r9037, 255; + setp.lt.u32 %p607, %r5304, %r5303; + cvt.u16.u32 %rs706, %r9037; + selp.b16 %rs707, %rs705, %rs706, %p607; + st.shared.u8 [%r1312], %rs707; + ld.shared.u8 %rs177, [%r1312+2]; + ld.shared.u8 %rs708, [%r1312+1]; + setp.gt.u16 %p608, %rs708, %rs177; + add.s32 %r9346, %r9014, 1; + add.s32 %r5305, %r9014, 2; + selp.b32 %r5306, %r9346, %r5305, %p608; + add.s32 %r5307, %r4095, %r5306; + ld.shared.u8 %rs178, [%r5307]; + cvt.u32.u16 %r5308, %rs178; + and.b32 %r5309, %r5308, 255; + add.s32 %r9013, %r5309, -1; + cvt.u16.u32 %rs179, %r9051; + cvt.u16.u32 %rs709, %r1240; + shr.u16 %rs710, %rs709, 1; + mov.u32 %r5310, _ZZ32 j2k_htj2k_encode_codeblocksE14cleanup_cx_val; + add.s32 %r1315, %r5310, %r9012; + st.shared.u8 [%r1312+1], %r9051; + ld.shared.u8 %rs711, [%r1315]; + or.b16 %rs712, %rs711, %rs710; + st.shared.u8 [%r1315], %rs712; + add.s32 %r9012, %r9012, 1; + ld.shared.u8 %rs180, [%r1315+1]; + ld.shared.u8 %r1317, [%r1315+2]; + shr.u32 %r1318, %r1288, 3; + st.shared.u8 [%r1315+1], %r1318; + add.s32 %r5311, %r9011, 2; + setp.ge.u32 %p609, %r5311, %r5; + mov.u32 %r9350, %r9016; + @%p609 bra $L__BB1_635; + + cvt.u64.u32 %rd363, %r9345; + add.s64 %rd364, %rd363, %rd5; + shl.b64 %rd365, %rd364, 2; + add.s64 %rd366, %rd3, %rd365; + ld.global.u32 %r1319, [%rd366]; + setp.eq.s32 %p610, %r1319, 0; + mov.u32 %r9182, 0; + mov.u32 %r9181, %r9182; + @%p610 bra $L__BB1_531; + + and.b32 %r5313, %r1319, -2147483648; + abs.s32 %r5314, %r1319; + shl.b32 %r5315, %r5314, %r1006; + or.b32 %r9181, %r5315, %r5313; + +$L__BB1_531: + shl.b32 %r5319, %r9181, 1; + shr.u32 %r5320, %r5319, %r43; + and.b32 %r1322, %r5320, -2; + setp.eq.s32 %p611, %r1322, 0; + mov.u32 %r9183, %r9182; + mov.u32 %r9189, %r9182; + @%p611 bra $L__BB1_533; + + add.s32 %r5322, %r1322, -1; + clz.b32 %r5323, %r5322; + mov.u32 %r5324, 32; + sub.s32 %r9182, %r5324, %r5323; + shr.u32 %r5325, %r9181, 31; + add.s32 %r5326, %r5325, %r1322; + add.s32 %r9183, %r5326, -2; + mov.u32 %r9189, 1; + +$L__BB1_533: + mov.u32 %r9186, 0; + mov.u32 %r9185, %r9186; + @%p493 bra $L__BB1_536; + + add.s32 %r5329, %r9345, %r1; + cvt.u64.u32 %rd367, %r5329; + add.s64 %rd368, %rd367, %rd5; + shl.b64 %rd369, %rd368, 2; + add.s64 %rd370, %rd3, %rd369; + ld.global.u32 %r1328, [%rd370]; + setp.eq.s32 %p613, %r1328, 0; + @%p613 bra $L__BB1_536; + + and.b32 %r5330, %r1328, -2147483648; + abs.s32 %r5331, %r1328; + shl.b32 %r5332, %r5331, %r1006; + or.b32 %r9185, %r5332, %r5330; + +$L__BB1_536: + shl.b32 %r5335, %r9185, 1; + shr.u32 %r5336, %r5335, %r43; + and.b32 %r1331, %r5336, -2; + setp.eq.s32 %p614, %r1331, 0; + mov.u32 %r9187, %r9186; + mov.u32 %r9204, %r9182; + @%p614 bra $L__BB1_538; + + or.b32 %r9189, %r9189, 2; + add.s32 %r5337, %r1331, -1; + clz.b32 %r5338, %r5337; + mov.u32 %r5339, 32; + sub.s32 %r9186, %r5339, %r5338; + max.s32 %r9204, %r9182, %r9186; + shr.u32 %r5340, %r9185, 31; + add.s32 %r5341, %r5340, %r1331; + add.s32 %r9187, %r5341, -2; + +$L__BB1_538: + add.s32 %r9206, %r9345, 1; + add.s32 %r5346, %r9011, 3; + setp.ge.u32 %p615, %r5346, %r5; + mov.u32 %r9207, 0; + mov.u32 %r9200, %r9207; + mov.u32 %r9201, %r9207; + mov.u32 %r9202, %r9207; + mov.u32 %r9203, %r9207; + @%p615 bra $L__BB1_549; + + cvt.u64.u32 %rd371, %r9206; + add.s64 %rd372, %rd371, %rd5; + shl.b64 %rd373, %rd372, 2; + add.s64 %rd374, %rd3, %rd373; + ld.global.u32 %r1341, [%rd374]; + setp.eq.s32 %p616, %r1341, 0; + mov.u32 %r9201, 0; + mov.u32 %r9190, %r9201; + @%p616 bra $L__BB1_541; + + and.b32 %r5348, %r1341, -2147483648; + abs.s32 %r5349, %r1341; + shl.b32 %r5350, %r5349, %r1006; + or.b32 %r9190, %r5350, %r5348; + +$L__BB1_541: + shl.b32 %r5353, %r9190, 1; + shr.u32 %r5354, %r5353, %r43; + and.b32 %r1344, %r5354, -2; + setp.eq.s32 %p617, %r1344, 0; + mov.u32 %r9203, %r9201; + @%p617 bra $L__BB1_543; + + or.b32 %r9189, %r9189, 4; + add.s32 %r5355, %r1344, -1; + clz.b32 %r5356, %r5355; + mov.u32 %r5357, 32; + sub.s32 %r9201, %r5357, %r5356; + max.s32 %r9204, %r9204, %r9201; + shr.u32 %r5358, %r9190, 31; + add.s32 %r5359, %r5358, %r1344; + add.s32 %r9203, %r5359, -2; + +$L__BB1_543: + mov.u32 %r9200, 0; + mov.u32 %r9195, %r9200; + @%p493 bra $L__BB1_546; + + add.s32 %r5362, %r9206, %r1; + cvt.u64.u32 %rd375, %r5362; + add.s64 %rd376, %rd375, %rd5; + shl.b64 %rd377, %rd376, 2; + add.s64 %rd378, %rd3, %rd377; + ld.global.u32 %r1353, [%rd378]; + setp.eq.s32 %p619, %r1353, 0; + @%p619 bra $L__BB1_546; + + and.b32 %r5363, %r1353, -2147483648; + abs.s32 %r5364, %r1353; + shl.b32 %r5365, %r5364, %r1006; + or.b32 %r9195, %r5365, %r5363; + +$L__BB1_546: + shl.b32 %r5368, %r9195, 1; + shr.u32 %r5369, %r5368, %r43; + and.b32 %r1356, %r5369, -2; + setp.eq.s32 %p620, %r1356, 0; + mov.u32 %r9202, %r9200; + @%p620 bra $L__BB1_548; + + or.b32 %r9189, %r9189, 8; + add.s32 %r5370, %r1356, -1; + clz.b32 %r5371, %r5370; + mov.u32 %r5372, 32; + sub.s32 %r9200, %r5372, %r5371; + max.s32 %r9204, %r9204, %r9200; + shr.u32 %r5373, %r9195, 31; + add.s32 %r5374, %r5373, %r1356; + add.s32 %r9202, %r5374, -2; + +$L__BB1_548: + add.s32 %r9206, %r9345, 2; + +$L__BB1_549: + mov.u32 %r9345, %r9206; + shr.u32 %r5376, %r1288, 2; + shr.u32 %r5377, %r1264, 1; + or.b32 %r5378, %r5376, %r5377; + cvt.u32.u16 %r5379, %rs180; + and.b32 %r5380, %r5379, 255; + shl.b32 %r5381, %r1317, 2; + add.s32 %r5382, %r5381, %r5380; + or.b32 %r1373, %r5378, %r5382; + add.s32 %r5383, %r9189, -1; + and.b32 %r5384, %r5383, %r9189; + setp.ne.s32 %p621, %r5384, 0; + setp.gt.u16 %p622, %rs178, 2; + and.pred %p623, %p622, %p621; + selp.b32 %r5385, %r9013, 1, %p623; + max.s32 %r1374, %r5385, %r9204; + sub.s32 %r9350, %r1374, %r5385; + setp.lt.s32 %p624, %r9350, 1; + @%p624 bra $L__BB1_551; + + setp.eq.s32 %p625, %r9182, %r9204; + selp.u32 %r5386, 1, 0, %p625; + setp.eq.s32 %p626, %r9186, %r9204; + selp.u32 %r5387, -1, 0, %p626; + bfi.b32 %r5388, %r5387, %r5386, 1, 1; + setp.eq.s32 %p627, %r9201, %r9204; + selp.u16 %rs714, 1, 0, %p627; + mul.wide.u16 %r5389, %rs714, 4; + or.b32 %r5390, %r5388, %r5389; + setp.eq.s32 %p628, %r9200, %r9204; + selp.u16 %rs715, 1, 0, %p628; + mul.wide.u16 %r5391, %rs715, 8; + or.b32 %r9207, %r5390, %r5391; + +$L__BB1_551: + shl.b32 %r5392, %r9189, 4; + shl.b32 %r5393, %r1373, 8; + or.b32 %r5394, %r5392, %r5393; + or.b32 %r5395, %r5394, %r9207; + mul.wide.u32 %rd380, %r5395, 2; + add.s64 %rd381, %rd18, %rd380; + ld.global.u16 %rs181, [%rd381]; + shr.u16 %rs716, %rs181, 4; + and.b16 %rs182, %rs716, 7; + setp.eq.s16 %p629, %rs182, 0; + mov.u32 %r9219, %r9070; + @%p629 bra $L__BB1_558; + + cvt.u32.u16 %r9208, %rs182; + shr.u16 %rs717, %rs181, 8; + cvt.u32.u16 %r9209, %rs717; + +$L__BB1_553: + mov.u32 %r1380, %r9208; + setp.gt.u32 %p630, %r8962, 2879; + mov.u32 %r9219, 1; + @%p630 bra $L__BB1_558; + + mov.u32 %r5397, 8; + sub.s32 %r5398, %r5397, %r8960; + sub.s32 %r5399, %r5398, %r8961; + min.u32 %r5400, %r5399, %r1380; + setp.eq.s32 %p631, %r5400, 32; + mov.u32 %r5401, -1; + shl.b32 %r5402, %r5401, %r5400; + not.b32 %r5403, %r5402; + selp.b32 %r5404, -1, %r5403, %p631; + and.b32 %r5405, %r5404, %r9209; + shl.b32 %r5406, %r5405, %r8961; + cvt.u16.u32 %rs718, %r5406; + or.b16 %rs1165, %rs1165, %rs718; + add.s32 %r8961, %r5400, %r8961; + sub.s32 %r9208, %r1380, %r5400; + shr.u32 %r9209, %r9209, %r5400; + setp.gt.u32 %p632, %r5399, %r1380; + @%p632 bra $L__BB1_557; + + setp.ne.s32 %p633, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs719, %rs1165, 255; + setp.ne.s16 %p634, %rs719, 127; + and.pred %p635, %p633, %p634; + @%p635 bra $L__BB1_557; + + mov.u32 %r5409, 20548; + sub.s32 %r5410, %r5409, %r8962; + cvt.u64.u32 %rd382, %r5410; + add.s64 %rd383, %rd382, %rd4; + add.s64 %rd384, %rd1, %rd383; + st.global.u8 [%rd384], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p636, %rs719, 143; + selp.u32 %r8960, 1, 0, %p636; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_557: + setp.ne.s32 %p637, %r9208, 0; + mov.u32 %r9219, %r9070; + @%p637 bra $L__BB1_553; + +$L__BB1_558: + setp.ne.s32 %p638, %r1373, 0; + @%p638 bra $L__BB1_606; + + setp.eq.s32 %p639, %r9189, 0; + add.s32 %r5411, %r8514, 17477; + cvt.u64.u32 %rd385, %r5411; + add.s64 %rd386, %rd385, %rd4; + add.s64 %rd20, %rd1, %rd386; + @%p639 bra $L__BB1_598; + + shl.b16 %rs1096, %rs1096, 1; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p640, %r8520, 0; + mov.u32 %r9253, %r8723; + @%p640 bra $L__BB1_563; + + setp.gt.u32 %p641, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9253, 1; + @%p641 bra $L__BB1_563; + + st.global.u8 [%rd20], %rs1096; + add.s32 %r8514, %r8514, 1; + mov.u32 %r8520, 8; + mov.u16 %rs1096, 0; + mov.u32 %r9253, %r8723; + +$L__BB1_563: + setp.lt.u32 %p642, %r8725, 3; + mov.u32 %r9223, 0; + @%p642 bra $L__BB1_566; + + setp.lt.u32 %p643, %r8725, 6; + mov.u32 %r9223, 1; + @%p643 bra $L__BB1_566; + + setp.lt.u32 %p644, %r8725, 9; + setp.eq.s32 %p645, %r8725, 11; + selp.b32 %r5417, 4, 5, %p645; + setp.lt.u32 %p646, %r8725, 11; + selp.b32 %r5418, 3, %r5417, %p646; + selp.b32 %r9223, 2, %r5418, %p644; + +$L__BB1_566: + setp.eq.s32 %p647, %r9223, 0; + @%p647 bra $L__BB1_594; + + add.s32 %r1404, %r9223, -1; + and.b32 %r1405, %r9223, 3; + setp.eq.s32 %p648, %r1405, 0; + mov.u32 %r9233, %r9223; + mov.u32 %r9236, %r9253; + @%p648 bra $L__BB1_579; + + mov.u32 %r5420, 1; + shl.b32 %r5421, %r5420, %r1404; + and.b32 %r5422, %r5421, %r8726; + setp.ne.s32 %p649, %r5422, 0; + selp.u32 %r5423, 1, 0, %p649; + cvt.u32.u16 %r5424, %rs1096; + bfi.b32 %r5425, %r5424, %r5423, 1, 8; + cvt.u16.u32 %rs1096, %r5425; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p650, %r8520, 0; + mov.u32 %r9236, %r9253; + @%p650 bra $L__BB1_571; + + setp.gt.u32 %p651, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9236, %r5420; + @%p651 bra $L__BB1_571; + + add.s32 %r5429, %r8514, 17477; + cvt.u64.u32 %rd387, %r5429; + add.s64 %rd388, %rd387, %rd4; + add.s64 %rd389, %rd1, %rd388; + st.global.u8 [%rd389], %rs1096; + add.s32 %r8514, %r8514, 1; + mov.u32 %r8520, 8; + mov.u16 %rs1096, 0; + mov.u32 %r9236, %r9253; + +$L__BB1_571: + setp.eq.s32 %p652, %r1405, 1; + mov.u32 %r9253, %r9236; + mov.u32 %r9233, %r1404; + @%p652 bra $L__BB1_579; + + add.s32 %r9233, %r9223, -2; + mov.u32 %r5430, 1; + shl.b32 %r5431, %r5430, %r9233; + and.b32 %r5432, %r5431, %r8726; + setp.ne.s32 %p653, %r5432, 0; + selp.u32 %r5433, 1, 0, %p653; + cvt.u32.u16 %r5434, %rs1096; + bfi.b32 %r5435, %r5434, %r5433, 1, 8; + cvt.u16.u32 %rs1096, %r5435; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p654, %r8520, 0; + mov.u32 %r9227, %r9236; + @%p654 bra $L__BB1_575; + + setp.gt.u32 %p655, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9227, %r5430; + @%p655 bra $L__BB1_575; + + add.s32 %r5438, %r8514, 17477; + cvt.u64.u32 %rd390, %r5438; + add.s64 %rd391, %rd390, %rd4; + add.s64 %rd392, %rd1, %rd391; + and.b16 %rs726, %rs1096, 255; + st.global.u8 [%rd392], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p656, %rs726, 255; + selp.b32 %r8520, 7, 8, %p656; + mov.u16 %rs1096, 0; + mov.u32 %r9227, %r9236; + +$L__BB1_575: + setp.eq.s32 %p657, %r1405, 2; + mov.u32 %r9253, %r9227; + mov.u32 %r9236, %r9227; + @%p657 bra $L__BB1_579; + + add.s32 %r9233, %r9223, -3; + mov.u32 %r5439, 1; + shl.b32 %r5440, %r5439, %r9233; + and.b32 %r5441, %r5440, %r8726; + setp.ne.s32 %p658, %r5441, 0; + selp.u32 %r5442, 1, 0, %p658; + cvt.u32.u16 %r5443, %rs1096; + bfi.b32 %r5444, %r5443, %r5442, 1, 8; + cvt.u16.u32 %rs1096, %r5444; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p659, %r8520, 0; + mov.u32 %r9253, %r9227; + mov.u32 %r9236, %r9227; + @%p659 bra $L__BB1_579; + + setp.gt.u32 %p660, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9253, %r5439; + mov.u32 %r9236, %r5439; + @%p660 bra $L__BB1_579; + + add.s32 %r5449, %r8514, 17477; + cvt.u64.u32 %rd393, %r5449; + add.s64 %rd394, %rd393, %rd4; + add.s64 %rd395, %rd1, %rd394; + and.b16 %rs729, %rs1096, 255; + st.global.u8 [%rd395], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p661, %rs729, 255; + selp.b32 %r8520, 7, 8, %p661; + mov.u16 %rs1096, 0; + mov.u32 %r9253, %r9227; + mov.u32 %r9236, %r9227; + +$L__BB1_579: + setp.lt.u32 %p662, %r1404, 3; + @%p662 bra $L__BB1_594; + + mov.u32 %r9253, %r9236; + +$L__BB1_581: + add.s32 %r5450, %r9233, -1; + mov.u32 %r5451, 1; + shl.b32 %r5452, %r5451, %r5450; + and.b32 %r5453, %r5452, %r8726; + setp.ne.s32 %p663, %r5453, 0; + selp.u32 %r5454, 1, 0, %p663; + cvt.u32.u16 %r5455, %rs1096; + bfi.b32 %r9242, %r5455, %r5454, 1, 8; + add.s32 %r9243, %r8520, -1; + setp.ne.s32 %p664, %r9243, 0; + mov.u32 %r9241, %r9253; + @%p664 bra $L__BB1_584; + + setp.gt.u32 %p665, %r8514, 191; + mov.u32 %r9243, 0; + mov.u32 %r9241, %r5451; + @%p665 bra $L__BB1_584; + + cvt.u16.u32 %rs730, %r9242; + and.b16 %rs731, %rs730, 255; + add.s32 %r5459, %r8514, 17477; + cvt.u64.u32 %rd396, %r5459; + add.s64 %rd397, %rd396, %rd4; + add.s64 %rd398, %rd1, %rd397; + st.global.u8 [%rd398], %rs730; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p666, %rs731, 255; + selp.b32 %r9243, 7, 8, %p666; + mov.u32 %r9242, 0; + mov.u32 %r9241, %r9253; + +$L__BB1_584: + add.s32 %r5460, %r9233, -2; + shl.b32 %r5462, %r5451, %r5460; + and.b32 %r5463, %r5462, %r8726; + setp.ne.s32 %p667, %r5463, 0; + and.b32 %r5464, %r9242, 127; + selp.u32 %r5465, 1, 0, %p667; + bfi.b32 %r9246, %r5464, %r5465, 1, 7; + add.s32 %r9247, %r9243, -1; + setp.ne.s32 %p668, %r9247, 0; + mov.u32 %r9245, %r9241; + @%p668 bra $L__BB1_587; + + setp.gt.u32 %p669, %r8514, 191; + mov.u32 %r9247, 0; + mov.u32 %r9245, 1; + @%p669 bra $L__BB1_587; + + cvt.u16.u32 %rs732, %r9246; + and.b16 %rs733, %rs732, 255; + add.s32 %r5469, %r8514, 17477; + cvt.u64.u32 %rd399, %r5469; + add.s64 %rd400, %rd399, %rd4; + add.s64 %rd401, %rd1, %rd400; + st.global.u8 [%rd401], %rs732; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p670, %rs733, 255; + selp.b32 %r9247, 7, 8, %p670; + mov.u32 %r9246, 0; + mov.u32 %r9245, %r9241; + +$L__BB1_587: + add.s32 %r5470, %r9233, -3; + mov.u32 %r5471, 1; + shl.b32 %r5472, %r5471, %r5470; + and.b32 %r5473, %r5472, %r8726; + setp.ne.s32 %p671, %r5473, 0; + and.b32 %r5474, %r9246, 127; + selp.u32 %r5475, 1, 0, %p671; + bfi.b32 %r9250, %r5474, %r5475, 1, 7; + add.s32 %r9251, %r9247, -1; + setp.ne.s32 %p672, %r9251, 0; + mov.u32 %r9249, %r9245; + @%p672 bra $L__BB1_590; + + setp.gt.u32 %p673, %r8514, 191; + mov.u32 %r9251, 0; + mov.u32 %r9249, %r5471; + @%p673 bra $L__BB1_590; + + cvt.u16.u32 %rs734, %r9250; + and.b16 %rs735, %rs734, 255; + add.s32 %r5479, %r8514, 17477; + cvt.u64.u32 %rd402, %r5479; + add.s64 %rd403, %rd402, %rd4; + add.s64 %rd404, %rd1, %rd403; + st.global.u8 [%rd404], %rs734; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p674, %rs735, 255; + selp.b32 %r9251, 7, 8, %p674; + mov.u32 %r9250, 0; + mov.u32 %r9249, %r9245; + +$L__BB1_590: + add.s32 %r9233, %r9233, -4; + shl.b32 %r5481, %r5471, %r9233; + and.b32 %r5482, %r5481, %r8726; + setp.ne.s32 %p675, %r5482, 0; + and.b32 %r5483, %r9250, 127; + selp.u32 %r5484, 1, 0, %p675; + bfi.b32 %r5485, %r5483, %r5484, 1, 15; + cvt.u16.u32 %rs1096, %r5485; + add.s32 %r8520, %r9251, -1; + setp.ne.s32 %p676, %r8520, 0; + mov.u32 %r9253, %r9249; + @%p676 bra $L__BB1_593; + + setp.gt.u32 %p677, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9253, 1; + @%p677 bra $L__BB1_593; + + add.s32 %r5488, %r8514, 17477; + cvt.u64.u32 %rd405, %r5488; + add.s64 %rd406, %rd405, %rd4; + add.s64 %rd407, %rd1, %rd406; + and.b16 %rs737, %rs1096, 255; + st.global.u8 [%rd407], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p678, %rs737, 255; + selp.b32 %r8520, 7, 8, %p678; + mov.u16 %rs1096, 0; + mov.u32 %r9253, %r9249; + +$L__BB1_593: + setp.ne.s32 %p679, %r9233, 0; + @%p679 bra $L__BB1_581; + +$L__BB1_594: + add.s32 %r5490, %r8725, -1; + setp.eq.s32 %p680, %r8725, 0; + mov.u32 %r8726, 0; + selp.b32 %r8725, 0, %r5490, %p680; + setp.lt.u32 %p681, %r8725, 3; + mov.u32 %r9259, %r8726; + @%p681 bra $L__BB1_597; + + setp.lt.u32 %p682, %r8725, 6; + mov.u32 %r9259, 1; + @%p682 bra $L__BB1_597; + + setp.lt.u32 %p683, %r8725, 9; + setp.eq.s32 %p684, %r8725, 11; + selp.b32 %r5492, 4, 5, %p684; + setp.lt.u32 %p685, %r8725, 11; + selp.b32 %r5493, 3, %r5492, %p685; + selp.b32 %r9259, 2, %r5493, %p683; + +$L__BB1_597: + mov.u32 %r5495, 1; + shl.b32 %r8724, %r5495, %r9259; + mov.u32 %r8723, %r9253; + bra.uni $L__BB1_606; + +$L__BB1_598: + add.s32 %r8726, %r8726, 1; + setp.lt.u32 %p686, %r8726, %r8724; + @%p686 bra $L__BB1_606; + + shl.b16 %rs738, %rs1096, 1; + or.b16 %rs1096, %rs738, 1; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p687, %r8520, 0; + mov.u32 %r9260, %r8723; + @%p687 bra $L__BB1_602; + + setp.gt.u32 %p688, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9260, 1; + @%p688 bra $L__BB1_602; + + and.b16 %rs740, %rs1096, 255; + st.global.u8 [%rd20], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p689, %rs740, 255; + selp.b32 %r8520, 7, 8, %p689; + mov.u16 %rs1096, 0; + mov.u32 %r9260, %r8723; + +$L__BB1_602: + add.s32 %r5499, %r8725, 1; + min.u32 %r8725, %r5499, 12; + setp.lt.u32 %p690, %r8725, 3; + mov.u32 %r8726, 0; + mov.u32 %r9263, %r8726; + @%p690 bra $L__BB1_605; + + setp.lt.u32 %p691, %r8725, 6; + mov.u32 %r9263, 1; + @%p691 bra $L__BB1_605; + + setp.lt.u32 %p692, %r8725, 9; + setp.eq.s32 %p693, %r8725, 11; + selp.b32 %r5501, 4, 5, %p693; + setp.lt.u32 %p694, %r8725, 11; + selp.b32 %r5502, 3, %r5501, %p694; + selp.b32 %r9263, 2, %r5502, %p692; + +$L__BB1_605: + mov.u32 %r5504, 1; + shl.b32 %r8724, %r5504, %r9263; + mov.u32 %r8723, %r9260; + +$L__BB1_606: + and.b16 %rs741, %rs181, 15; + cvt.u32.u16 %r1488, %rs741; + and.b32 %r5505, %r9189, 1; + setp.eq.b32 %p695, %r5505, 1; + mov.pred %p696, 0; + xor.pred %p697, %p695, %p696; + not.pred %p698, %p697; + mov.u32 %r9280, %r9176; + @%p698 bra $L__BB1_613; + + and.b32 %r5506, %r1488, 1; + sub.s32 %r9270, %r1374, %r5506; + setp.eq.s32 %p699, %r9270, 0; + mov.u32 %r9280, %r9176; + @%p699 bra $L__BB1_613; + + mov.u32 %r5507, -1; + shl.b32 %r5508, %r5507, %r9270; + not.b32 %r5509, %r5508; + and.b32 %r9271, %r9183, %r5509; + +$L__BB1_609: + setp.gt.u32 %p700, %r9150, 17476; + mov.u32 %r9280, 1; + @%p700 bra $L__BB1_613; + + sub.s32 %r5511, %r9149, %r9148; + min.u32 %r5512, %r5511, %r9270; + setp.eq.s32 %p701, %r5512, 32; + mov.u32 %r5513, -1; + shl.b32 %r5514, %r5513, %r5512; + not.b32 %r5515, %r5514; + selp.b32 %r5516, -1, %r5515, %p701; + and.b32 %r5517, %r5516, %r9271; + shl.b32 %r5518, %r5517, %r9148; + or.b32 %r9147, %r5518, %r9147; + add.s32 %r9148, %r5512, %r9148; + shr.u32 %r9271, %r9271, %r5512; + sub.s32 %r9270, %r9270, %r5512; + setp.lt.u32 %p702, %r9148, %r9149; + @%p702 bra $L__BB1_612; + + cvt.u64.u32 %rd408, %r9150; + add.s64 %rd409, %rd408, %rd4; + add.s64 %rd410, %rd1, %rd409; + st.global.u8 [%rd410], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p703, %r9147, 255; + selp.b32 %r9149, 7, 8, %p703; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_612: + setp.ne.s32 %p704, %r9270, 0; + mov.u32 %r9280, %r9176; + @%p704 bra $L__BB1_609; + +$L__BB1_613: + and.b32 %r1512, %r9189, 2; + setp.eq.s32 %p705, %r1512, 0; + mov.u32 %r9295, %r9280; + @%p705 bra $L__BB1_620; + + shr.u32 %r5521, %r1488, 1; + and.b32 %r5522, %r5521, 1; + sub.s32 %r9285, %r1374, %r5522; + setp.eq.s32 %p706, %r9285, 0; + mov.u32 %r9295, %r9280; + @%p706 bra $L__BB1_620; + + mov.u32 %r5523, -1; + shl.b32 %r5524, %r5523, %r9285; + not.b32 %r5525, %r5524; + and.b32 %r9286, %r9187, %r5525; + +$L__BB1_616: + setp.gt.u32 %p707, %r9150, 17476; + mov.u32 %r9295, 1; + @%p707 bra $L__BB1_620; + + sub.s32 %r5527, %r9149, %r9148; + min.u32 %r5528, %r5527, %r9285; + setp.eq.s32 %p708, %r5528, 32; + mov.u32 %r5529, -1; + shl.b32 %r5530, %r5529, %r5528; + not.b32 %r5531, %r5530; + selp.b32 %r5532, -1, %r5531, %p708; + and.b32 %r5533, %r5532, %r9286; + shl.b32 %r5534, %r5533, %r9148; + or.b32 %r9147, %r5534, %r9147; + add.s32 %r9148, %r5528, %r9148; + shr.u32 %r9286, %r9286, %r5528; + sub.s32 %r9285, %r9285, %r5528; + setp.lt.u32 %p709, %r9148, %r9149; + @%p709 bra $L__BB1_619; + + cvt.u64.u32 %rd411, %r9150; + add.s64 %rd412, %rd411, %rd4; + add.s64 %rd413, %rd1, %rd412; + st.global.u8 [%rd413], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p710, %r9147, 255; + selp.b32 %r9149, 7, 8, %p710; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_619: + setp.ne.s32 %p711, %r9285, 0; + mov.u32 %r9295, %r9280; + @%p711 bra $L__BB1_616; + +$L__BB1_620: + and.b32 %r1536, %r9189, 4; + setp.eq.s32 %p712, %r1536, 0; + mov.u32 %r9310, %r9295; + @%p712 bra $L__BB1_627; + + shr.u32 %r5537, %r1488, 2; + and.b32 %r5538, %r5537, 1; + sub.s32 %r9300, %r1374, %r5538; + setp.eq.s32 %p713, %r9300, 0; + mov.u32 %r9310, %r9295; + @%p713 bra $L__BB1_627; + + mov.u32 %r5539, -1; + shl.b32 %r5540, %r5539, %r9300; + not.b32 %r5541, %r5540; + and.b32 %r9301, %r9203, %r5541; + +$L__BB1_623: + setp.gt.u32 %p714, %r9150, 17476; + mov.u32 %r9310, 1; + @%p714 bra $L__BB1_627; + + sub.s32 %r5543, %r9149, %r9148; + min.u32 %r5544, %r5543, %r9300; + setp.eq.s32 %p715, %r5544, 32; + mov.u32 %r5545, -1; + shl.b32 %r5546, %r5545, %r5544; + not.b32 %r5547, %r5546; + selp.b32 %r5548, -1, %r5547, %p715; + and.b32 %r5549, %r5548, %r9301; + shl.b32 %r5550, %r5549, %r9148; + or.b32 %r9147, %r5550, %r9147; + add.s32 %r9148, %r5544, %r9148; + shr.u32 %r9301, %r9301, %r5544; + sub.s32 %r9300, %r9300, %r5544; + setp.lt.u32 %p716, %r9148, %r9149; + @%p716 bra $L__BB1_626; + + cvt.u64.u32 %rd414, %r9150; + add.s64 %rd415, %rd414, %rd4; + add.s64 %rd416, %rd1, %rd415; + st.global.u8 [%rd416], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p717, %r9147, 255; + selp.b32 %r9149, 7, 8, %p717; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_626: + setp.ne.s32 %p718, %r9300, 0; + mov.u32 %r9310, %r9295; + @%p718 bra $L__BB1_623; + +$L__BB1_627: + and.b32 %r1560, %r9189, 8; + setp.eq.s32 %p719, %r1560, 0; + mov.u32 %r9176, %r9310; + @%p719 bra $L__BB1_634; + + shr.u32 %r5553, %r1488, 3; + sub.s32 %r9315, %r1374, %r5553; + setp.eq.s32 %p720, %r9315, 0; + mov.u32 %r9176, %r9310; + @%p720 bra $L__BB1_634; + + mov.u32 %r5554, -1; + shl.b32 %r5555, %r5554, %r9315; + not.b32 %r5556, %r5555; + and.b32 %r9316, %r9202, %r5556; + +$L__BB1_630: + setp.gt.u32 %p721, %r9150, 17476; + mov.u32 %r9176, 1; + @%p721 bra $L__BB1_634; + + sub.s32 %r5558, %r9149, %r9148; + min.u32 %r5559, %r5558, %r9315; + setp.eq.s32 %p722, %r5559, 32; + mov.u32 %r5560, -1; + shl.b32 %r5561, %r5560, %r5559; + not.b32 %r5562, %r5561; + selp.b32 %r5563, -1, %r5562, %p722; + and.b32 %r5564, %r5563, %r9316; + shl.b32 %r5565, %r5564, %r9148; + or.b32 %r9147, %r5565, %r9147; + add.s32 %r9148, %r5559, %r9148; + shr.u32 %r9316, %r9316, %r5559; + sub.s32 %r9315, %r9315, %r5559; + setp.lt.u32 %p723, %r9148, %r9149; + @%p723 bra $L__BB1_633; + + cvt.u64.u32 %rd417, %r9150; + add.s64 %rd418, %rd417, %rd4; + add.s64 %rd419, %rd1, %rd418; + st.global.u8 [%rd419], %r9147; + add.s32 %r9150, %r9150, 1; + setp.eq.s32 %p724, %r9147, 255; + selp.b32 %r9149, 7, 8, %p724; + mov.u32 %r9147, 0; + mov.u32 %r9148, %r9147; + +$L__BB1_633: + setp.ne.s32 %p725, %r9315, 0; + mov.u32 %r9176, %r9310; + @%p725 bra $L__BB1_630; + +$L__BB1_634: + and.b32 %r5568, %r9186, 255; + and.b32 %r5569, %r9051, 255; + setp.lt.u32 %p726, %r5568, %r5569; + cvt.u16.u32 %rs742, %r9186; + selp.b16 %rs743, %rs179, %rs742, %p726; + st.shared.u8 [%r1312+1], %rs743; + ld.shared.u8 %rs744, [%r1312+3]; + setp.gt.u16 %p727, %rs177, %rs744; + add.s32 %r9346, %r9346, 1; + add.s32 %r5570, %r9014, 3; + selp.b32 %r5571, %r9346, %r5570, %p727; + add.s32 %r5573, %r4095, %r5571; + ld.shared.u8 %r5574, [%r5573]; + add.s32 %r9013, %r5574, -1; + shr.u32 %r5575, %r1512, 1; + or.b32 %r5576, %r1318, %r5575; + st.shared.u8 [%r1312+2], %r9200; + st.shared.u8 [%r1315+1], %r5576; + ld.shared.u8 %rs745, [%r1315+3]; + mul.wide.u16 %r5577, %rs745, 4; + add.s32 %r5578, %r5577, %r1317; + shr.u32 %r5579, %r1560, 3; + st.shared.u8 [%r1315+2], %r5579; + shr.u32 %r5580, %r1560, 2; + shr.u32 %r5581, %r1536, 1; + or.b32 %r5582, %r5580, %r5581; + or.b32 %r9016, %r5582, %r5578; + add.s32 %r9012, %r9012, 1; + mov.u32 %r9070, %r9219; + +$L__BB1_635: + mov.u32 %r9014, %r9346; + mov.u32 %r9015, %r9345; + max.s32 %r5583, %r9350, 0; + mul.lo.s32 %r5584, %r1103, 6; + setp.gt.s32 %p728, %r1103, 0; + selp.b32 %r5585, %r5584, 0, %p728; + cvt.u64.u32 %rd420, %r5585; + add.s64 %rd21, %rd17, %rd420; + ld.global.u8 %rs205, [%rd21+1]; + add.s32 %r5586, %r5585, 2; + cvt.u64.u32 %rd421, %r5586; + add.s64 %rd422, %rd17, %rd421; + ld.global.u8 %rs206, [%rd422]; + ld.global.u8 %rs207, [%rd422+1]; + mul.lo.s32 %r5587, %r5583, 6; + cvt.u64.u32 %rd423, %r5587; + add.s64 %rd424, %rd17, %rd423; + ld.global.u8 %rs208, [%rd424]; + ld.global.u8 %rs209, [%rd424+1]; + add.s32 %r5588, %r5587, 2; + cvt.u64.u32 %rd425, %r5588; + add.s64 %rd426, %rd17, %rd425; + ld.global.u8 %rs210, [%rd426]; + ld.global.u8 %rs211, [%rd426+1]; + setp.eq.s16 %p729, %rs205, 0; + mov.u32 %r9362, %r9070; + @%p729 bra $L__BB1_642; + + ld.global.u8 %r9352, [%rd21]; + cvt.u32.u16 %r9351, %rs205; + +$L__BB1_637: + mov.u32 %r1611, %r9351; + setp.gt.u32 %p730, %r8962, 2879; + mov.u32 %r9362, 1; + @%p730 bra $L__BB1_642; + + mov.u32 %r5590, 8; + sub.s32 %r5591, %r5590, %r8960; + sub.s32 %r5592, %r5591, %r8961; + min.u32 %r5593, %r5592, %r1611; + setp.eq.s32 %p731, %r5593, 32; + mov.u32 %r5594, -1; + shl.b32 %r5595, %r5594, %r5593; + not.b32 %r5596, %r5595; + selp.b32 %r5597, -1, %r5596, %p731; + and.b32 %r5598, %r5597, %r9352; + shl.b32 %r5599, %r5598, %r8961; + cvt.u16.u32 %rs746, %r5599; + or.b16 %rs1165, %rs1165, %rs746; + add.s32 %r8961, %r5593, %r8961; + sub.s32 %r9351, %r1611, %r5593; + shr.u32 %r9352, %r9352, %r5593; + setp.gt.u32 %p732, %r5592, %r1611; + @%p732 bra $L__BB1_641; + + setp.ne.s32 %p733, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs747, %rs1165, 255; + setp.ne.s16 %p734, %rs747, 127; + and.pred %p735, %p733, %p734; + @%p735 bra $L__BB1_641; + + mov.u32 %r5602, 20548; + sub.s32 %r5603, %r5602, %r8962; + cvt.u64.u32 %rd427, %r5603; + add.s64 %rd428, %rd427, %rd4; + add.s64 %rd429, %rd1, %rd428; + st.global.u8 [%rd429], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p736, %rs747, 143; + selp.u32 %r8960, 1, 0, %p736; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_641: + setp.ne.s32 %p737, %r9351, 0; + mov.u32 %r9362, %r9070; + @%p737 bra $L__BB1_637; + +$L__BB1_642: + setp.eq.s16 %p738, %rs209, 0; + mov.u32 %r9374, %r9362; + @%p738 bra $L__BB1_649; + + cvt.u32.u16 %r5604, %rs208; + and.b32 %r9364, %r5604, 255; + cvt.u32.u16 %r5605, %rs209; + and.b32 %r9363, %r5605, 255; + +$L__BB1_644: + mov.u32 %r1630, %r9363; + setp.gt.u32 %p739, %r8962, 2879; + mov.u32 %r9374, 1; + @%p739 bra $L__BB1_649; + + mov.u32 %r5607, 8; + sub.s32 %r5608, %r5607, %r8960; + sub.s32 %r5609, %r5608, %r8961; + min.u32 %r5610, %r5609, %r1630; + setp.eq.s32 %p740, %r5610, 32; + mov.u32 %r5611, -1; + shl.b32 %r5612, %r5611, %r5610; + not.b32 %r5613, %r5612; + selp.b32 %r5614, -1, %r5613, %p740; + and.b32 %r5615, %r5614, %r9364; + shl.b32 %r5616, %r5615, %r8961; + cvt.u16.u32 %rs751, %r5616; + or.b16 %rs1165, %rs1165, %rs751; + add.s32 %r8961, %r5610, %r8961; + sub.s32 %r9363, %r1630, %r5610; + shr.u32 %r9364, %r9364, %r5610; + setp.gt.u32 %p741, %r5609, %r1630; + @%p741 bra $L__BB1_648; + + setp.ne.s32 %p742, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs752, %rs1165, 255; + setp.ne.s16 %p743, %rs752, 127; + and.pred %p744, %p742, %p743; + @%p744 bra $L__BB1_648; + + mov.u32 %r5619, 20548; + sub.s32 %r5620, %r5619, %r8962; + cvt.u64.u32 %rd430, %r5620; + add.s64 %rd431, %rd430, %rd4; + add.s64 %rd432, %rd1, %rd431; + st.global.u8 [%rd432], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p745, %rs752, 143; + selp.u32 %r8960, 1, 0, %p745; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_648: + setp.ne.s32 %p746, %r9363, 0; + mov.u32 %r9374, %r9362; + @%p746 bra $L__BB1_644; + +$L__BB1_649: + setp.eq.s16 %p747, %rs207, 0; + mov.u32 %r9386, %r9374; + @%p747 bra $L__BB1_656; + + cvt.u32.u16 %r5621, %rs207; + and.b32 %r9375, %r5621, 255; + cvt.u32.u16 %r5622, %rs206; + and.b32 %r9376, %r5622, 255; + +$L__BB1_651: + mov.u32 %r1649, %r9375; + setp.gt.u32 %p748, %r8962, 2879; + mov.u32 %r9386, 1; + @%p748 bra $L__BB1_656; + + mov.u32 %r5624, 8; + sub.s32 %r5625, %r5624, %r8960; + sub.s32 %r5626, %r5625, %r8961; + min.u32 %r5627, %r5626, %r1649; + setp.eq.s32 %p749, %r5627, 32; + mov.u32 %r5628, -1; + shl.b32 %r5629, %r5628, %r5627; + not.b32 %r5630, %r5629; + selp.b32 %r5631, -1, %r5630, %p749; + and.b32 %r5632, %r5631, %r9376; + shl.b32 %r5633, %r5632, %r8961; + cvt.u16.u32 %rs756, %r5633; + or.b16 %rs1165, %rs1165, %rs756; + add.s32 %r8961, %r5627, %r8961; + sub.s32 %r9375, %r1649, %r5627; + shr.u32 %r9376, %r9376, %r5627; + setp.gt.u32 %p750, %r5626, %r1649; + @%p750 bra $L__BB1_655; + + setp.ne.s32 %p751, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs757, %rs1165, 255; + setp.ne.s16 %p752, %rs757, 127; + and.pred %p753, %p751, %p752; + @%p753 bra $L__BB1_655; + + mov.u32 %r5636, 20548; + sub.s32 %r5637, %r5636, %r8962; + cvt.u64.u32 %rd433, %r5637; + add.s64 %rd434, %rd433, %rd4; + add.s64 %rd435, %rd1, %rd434; + st.global.u8 [%rd435], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p754, %rs757, 143; + selp.u32 %r8960, 1, 0, %p754; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_655: + setp.ne.s32 %p755, %r9375, 0; + mov.u32 %r9386, %r9374; + @%p755 bra $L__BB1_651; + +$L__BB1_656: + setp.eq.s16 %p756, %rs211, 0; + mov.u32 %r8959, %r9386; + @%p756 bra $L__BB1_663; + + cvt.u32.u16 %r5638, %rs210; + and.b32 %r9388, %r5638, 255; + cvt.u32.u16 %r5639, %rs211; + and.b32 %r9387, %r5639, 255; + +$L__BB1_658: + mov.u32 %r1668, %r9387; + setp.gt.u32 %p757, %r8962, 2879; + mov.u32 %r8959, 1; + @%p757 bra $L__BB1_663; + + mov.u32 %r5641, 8; + sub.s32 %r5642, %r5641, %r8960; + sub.s32 %r5643, %r5642, %r8961; + min.u32 %r5644, %r5643, %r1668; + setp.eq.s32 %p758, %r5644, 32; + mov.u32 %r5645, -1; + shl.b32 %r5646, %r5645, %r5644; + not.b32 %r5647, %r5646; + selp.b32 %r5648, -1, %r5647, %p758; + and.b32 %r5649, %r5648, %r9388; + shl.b32 %r5650, %r5649, %r8961; + cvt.u16.u32 %rs761, %r5650; + or.b16 %rs1165, %rs1165, %rs761; + add.s32 %r8961, %r5644, %r8961; + sub.s32 %r9387, %r1668, %r5644; + shr.u32 %r9388, %r9388, %r5644; + setp.gt.u32 %p759, %r5643, %r1668; + @%p759 bra $L__BB1_662; + + setp.ne.s32 %p760, %r8960, 0; + mov.u32 %r8960, 0; + and.b16 %rs762, %rs1165, 255; + setp.ne.s16 %p761, %rs762, 127; + and.pred %p762, %p760, %p761; + @%p762 bra $L__BB1_662; + + mov.u32 %r5653, 20548; + sub.s32 %r5654, %r5653, %r8962; + cvt.u64.u32 %rd436, %r5654; + add.s64 %rd437, %rd436, %rd4; + add.s64 %rd438, %rd1, %rd437; + st.global.u8 [%rd438], %rs1165; + add.s32 %r8962, %r8962, 1; + setp.gt.u16 %p763, %rs762, 143; + selp.u32 %r8960, 1, 0, %p763; + mov.u32 %r8961, 0; + mov.u16 %rs1165, 0; + +$L__BB1_662: + setp.ne.s32 %p764, %r9387, 0; + mov.u32 %r8959, %r9386; + @%p764 bra $L__BB1_658; + +$L__BB1_663: + add.s32 %r9011, %r9011, 4; + setp.lt.u32 %p765, %r9011, %r5; + @%p765 bra $L__BB1_423; + +$L__BB1_664: + add.s32 %r8995, %r8995, 2; + setp.lt.u32 %p766, %r8995, %r6; + @%p766 bra $L__BB1_421; + +$L__BB1_665: + setp.eq.s32 %p767, %r8726, 0; + mov.u32 %r9426, %r8723; + @%p767 bra $L__BB1_669; + + shl.b16 %rs765, %rs1096, 1; + or.b16 %rs1096, %rs765, 1; + add.s32 %r8520, %r8520, -1; + setp.ne.s32 %p768, %r8520, 0; + mov.u32 %r9426, %r8723; + @%p768 bra $L__BB1_669; + + setp.gt.u32 %p769, %r8514, 191; + mov.u32 %r8520, 0; + mov.u32 %r9426, 1; + @%p769 bra $L__BB1_669; + + add.s32 %r5657, %r8514, 17477; + cvt.u64.u32 %rd439, %r5657; + add.s64 %rd440, %rd439, %rd4; + add.s64 %rd441, %rd1, %rd440; + and.b16 %rs767, %rs1096, 255; + st.global.u8 [%rd441], %rs1096; + add.s32 %r8514, %r8514, 1; + setp.eq.s16 %p770, %rs767, 255; + selp.b32 %r8520, 7, 8, %p770; + mov.u16 %rs1096, 0; + mov.u32 %r9426, %r8723; + +$L__BB1_669: + cvt.u32.u16 %r5658, %rs1096; + and.b32 %r5659, %r5658, 255; + shl.b32 %r5660, %r5659, %r8520; + cvt.u16.u32 %rs234, %r5660; + mov.u32 %r5661, -1; + shl.b32 %r5662, %r5661, %r8961; + not.b32 %r5663, %r5662; + mov.u32 %r5664, 255; + and.b32 %r5665, %r5663, 255; + setp.eq.s32 %p771, %r8961, 0; + selp.b32 %r1720, 0, %r5665, %p771; + shl.b32 %r1721, %r5664, %r8520; + and.b32 %r5666, %r1721, 255; + or.b32 %r5667, %r5666, %r1720; + setp.eq.s32 %p772, %r5667, 0; + mov.u32 %r9429, %r9426; + mov.u32 %r9431, %r8959; + @%p772 bra $L__BB1_675; + + or.b16 %rs235, %rs1165, %rs234; + and.b16 %rs768, %rs235, 255; + xor.b16 %rs769, %rs235, %rs234; + cvt.u32.u16 %r5668, %rs769; + and.b32 %r5669, %r1721, %r5668; + and.b32 %r5670, %r5669, 255; + xor.b16 %rs770, %rs235, %rs1165; + cvt.u32.u16 %r5671, %rs770; + and.b32 %r5672, %r1720, %r5671; + or.b32 %r5673, %r5670, %r5672; + setp.eq.s32 %p773, %r5673, 0; + setp.ne.s16 %p774, %rs768, 255; + and.pred %p775, %p774, %p773; + setp.gt.u32 %p776, %r8962, 1; + and.pred %p777, %p776, %p775; + add.s32 %r5674, %r8514, 17477; + cvt.u64.u32 %rd442, %r5674; + add.s64 %rd443, %rd442, %rd4; + add.s64 %rd22, %rd1, %rd443; + @%p777 bra $L__BB1_673; + bra.uni $L__BB1_671; + +$L__BB1_673: + setp.gt.u32 %p781, %r8514, 191; + mov.u32 %r9429, 1; + mov.u32 %r9431, %r8959; + @%p781 bra $L__BB1_675; + + st.global.u8 [%rd22], %rs235; + add.s32 %r8514, %r8514, 1; + mov.u32 %r9429, %r9426; + mov.u32 %r9431, %r8959; + bra.uni $L__BB1_675; + +$L__BB1_671: + setp.gt.u32 %p778, %r8514, 191; + setp.gt.u32 %p779, %r8962, 2879; + or.pred %p780, %p779, %p778; + mov.u32 %r9429, 1; + mov.u32 %r9431, %r9429; + @%p780 bra $L__BB1_675; + + st.global.u8 [%rd22], %rs234; + add.s32 %r8514, %r8514, 1; + mov.u32 %r5677, 20548; + sub.s32 %r5678, %r5677, %r8962; + cvt.u64.u32 %rd444, %r5678; + add.s64 %rd445, %rd444, %rd4; + add.s64 %rd446, %rd1, %rd445; + st.global.u8 [%rd446], %rs1165; + add.s32 %r8962, %r8962, 1; + mov.u32 %r9429, %r9426; + mov.u32 %r9431, %r8959; + +$L__BB1_675: + setp.eq.s32 %p782, %r9148, 0; + @%p782 bra $L__BB1_679; + + sub.s32 %r5680, %r9149, %r9148; + mov.u32 %r5681, -1; + shl.b32 %r5682, %r5681, %r5680; + not.b32 %r5683, %r5682; + and.b32 %r5684, %r5683, 255; + shl.b32 %r5685, %r5684, %r9148; + or.b32 %r1729, %r5685, %r9147; + setp.eq.s32 %p783, %r1729, 255; + mov.u32 %r9433, %r9176; + @%p783 bra $L__BB1_681; + + setp.gt.u32 %p784, %r9150, 17476; + mov.u32 %r9433, 1; + @%p784 bra $L__BB1_681; + + cvt.u64.u32 %rd447, %r9150; + add.s64 %rd448, %rd447, %rd4; + add.s64 %rd449, %rd1, %rd448; + st.global.u8 [%rd449], %r1729; + add.s32 %r9150, %r9150, 1; + mov.u32 %r9433, %r9176; + bra.uni $L__BB1_681; + +$L__BB1_679: + setp.ne.s32 %p785, %r9149, 7; + mov.u32 %r9433, %r9176; + @%p785 bra $L__BB1_681; + + setp.eq.s32 %p786, %r9150, 0; + add.s32 %r5687, %r9150, -1; + selp.b32 %r9150, 0, %r5687, %p786; + mov.u32 %r9433, %r9176; + +$L__BB1_681: + or.b32 %r5688, %r9431, %r9429; + or.b32 %r5689, %r5688, %r9433; + setp.eq.s32 %p787, %r5689, 0; + @%p787 bra $L__BB1_683; + + mov.u32 %r5690, 1; + st.global.u32 [%rd6], %r5690; + mov.u32 %r5691, 3; + st.global.u32 [%rd6+4], %r5691; + mov.u32 %r5692, 0; + st.global.u32 [%rd6+8], %r5692; + st.global.u32 [%rd6+12], %r5692; + st.global.u32 [%rd6+16], %r5692; + st.global.u32 [%rd6+20], %r5692; + st.global.u32 [%rd6+24], %r5692; + st.global.u32 [%rd6+28], %r5692; + bra.uni $L__BB1_1905; + +$L__BB1_683: + add.s32 %r1734, %r9150, %r8514; + add.s32 %r1735, %r1734, %r8962; + add.u64 %rd23, %SPL, 0; + mov.u32 %r9553, 1; + mov.u32 %r9551, 0; + mov.u32 %r9552, %r9551; + @%p38 bra $L__BB1_932; + + setp.ne.s32 %p789, %r4, 3; + @%p789 bra $L__BB1_931; + + add.s32 %r5697, %r5, 3; + shr.u32 %r5698, %r5697, 2; + add.s32 %r5699, %r5698, 8; + setp.gt.u32 %p791, %r5699, 513; + mov.pred %p790, -1; + mov.u32 %r9550, 0; + mov.pred %p2370, %p790; + @%p791 bra $L__BB1_928; + + mov.u16 %rs1224, 0; + st.local.u16 [%rd23], %rs1224; + st.local.u16 [%rd23+2], %rs1224; + st.local.u16 [%rd23+4], %rs1224; + st.local.u16 [%rd23+6], %rs1224; + st.local.u16 [%rd23+8], %rs1224; + st.local.u16 [%rd23+10], %rs1224; + st.local.u16 [%rd23+12], %rs1224; + st.local.u16 [%rd23+14], %rs1224; + st.local.u16 [%rd23+16], %rs1224; + st.local.u16 [%rd23+18], %rs1224; + st.local.u16 [%rd23+20], %rs1224; + st.local.u16 [%rd23+22], %rs1224; + st.local.u16 [%rd23+24], %rs1224; + st.local.u16 [%rd23+26], %rs1224; + st.local.u16 [%rd23+28], %rs1224; + st.local.u16 [%rd23+30], %rs1224; + st.local.u16 [%rd23+32], %rs1224; + st.local.u16 [%rd23+34], %rs1224; + st.local.u16 [%rd23+36], %rs1224; + st.local.u16 [%rd23+38], %rs1224; + st.local.u16 [%rd23+40], %rs1224; + st.local.u16 [%rd23+42], %rs1224; + st.local.u16 [%rd23+44], %rs1224; + st.local.u16 [%rd23+46], %rs1224; + st.local.u16 [%rd23+48], %rs1224; + st.local.u16 [%rd23+50], %rs1224; + st.local.u16 [%rd23+52], %rs1224; + st.local.u16 [%rd23+54], %rs1224; + st.local.u16 [%rd23+56], %rs1224; + st.local.u16 [%rd23+58], %rs1224; + st.local.u16 [%rd23+60], %rs1224; + st.local.u16 [%rd23+62], %rs1224; + st.local.u16 [%rd23+64], %rs1224; + st.local.u16 [%rd23+66], %rs1224; + st.local.u16 [%rd23+68], %rs1224; + st.local.u16 [%rd23+70], %rs1224; + st.local.u16 [%rd23+72], %rs1224; + st.local.u16 [%rd23+74], %rs1224; + st.local.u16 [%rd23+76], %rs1224; + st.local.u16 [%rd23+78], %rs1224; + st.local.u16 [%rd23+80], %rs1224; + st.local.u16 [%rd23+82], %rs1224; + st.local.u16 [%rd23+84], %rs1224; + st.local.u16 [%rd23+86], %rs1224; + st.local.u16 [%rd23+88], %rs1224; + st.local.u16 [%rd23+90], %rs1224; + st.local.u16 [%rd23+92], %rs1224; + st.local.u16 [%rd23+94], %rs1224; + st.local.u16 [%rd23+96], %rs1224; + st.local.u16 [%rd23+98], %rs1224; + st.local.u16 [%rd23+100], %rs1224; + st.local.u16 [%rd23+102], %rs1224; + st.local.u16 [%rd23+104], %rs1224; + st.local.u16 [%rd23+106], %rs1224; + st.local.u16 [%rd23+108], %rs1224; + st.local.u16 [%rd23+110], %rs1224; + st.local.u16 [%rd23+112], %rs1224; + st.local.u16 [%rd23+114], %rs1224; + st.local.u16 [%rd23+116], %rs1224; + st.local.u16 [%rd23+118], %rs1224; + st.local.u16 [%rd23+120], %rs1224; + st.local.u16 [%rd23+122], %rs1224; + st.local.u16 [%rd23+124], %rs1224; + st.local.u16 [%rd23+126], %rs1224; + st.local.u16 [%rd23+128], %rs1224; + st.local.u16 [%rd23+130], %rs1224; + st.local.u16 [%rd23+132], %rs1224; + st.local.u16 [%rd23+134], %rs1224; + st.local.u16 [%rd23+136], %rs1224; + st.local.u16 [%rd23+138], %rs1224; + st.local.u16 [%rd23+140], %rs1224; + st.local.u16 [%rd23+142], %rs1224; + st.local.u16 [%rd23+144], %rs1224; + st.local.u16 [%rd23+146], %rs1224; + st.local.u16 [%rd23+148], %rs1224; + st.local.u16 [%rd23+150], %rs1224; + st.local.u16 [%rd23+152], %rs1224; + st.local.u16 [%rd23+154], %rs1224; + st.local.u16 [%rd23+156], %rs1224; + st.local.u16 [%rd23+158], %rs1224; + st.local.u16 [%rd23+160], %rs1224; + st.local.u16 [%rd23+162], %rs1224; + st.local.u16 [%rd23+164], %rs1224; + st.local.u16 [%rd23+166], %rs1224; + st.local.u16 [%rd23+168], %rs1224; + st.local.u16 [%rd23+170], %rs1224; + st.local.u16 [%rd23+172], %rs1224; + st.local.u16 [%rd23+174], %rs1224; + st.local.u16 [%rd23+176], %rs1224; + st.local.u16 [%rd23+178], %rs1224; + st.local.u16 [%rd23+180], %rs1224; + st.local.u16 [%rd23+182], %rs1224; + st.local.u16 [%rd23+184], %rs1224; + st.local.u16 [%rd23+186], %rs1224; + st.local.u16 [%rd23+188], %rs1224; + st.local.u16 [%rd23+190], %rs1224; + st.local.u16 [%rd23+192], %rs1224; + st.local.u16 [%rd23+194], %rs1224; + st.local.u16 [%rd23+196], %rs1224; + st.local.u16 [%rd23+198], %rs1224; + st.local.u16 [%rd23+200], %rs1224; + st.local.u16 [%rd23+202], %rs1224; + st.local.u16 [%rd23+204], %rs1224; + st.local.u16 [%rd23+206], %rs1224; + st.local.u16 [%rd23+208], %rs1224; + st.local.u16 [%rd23+210], %rs1224; + st.local.u16 [%rd23+212], %rs1224; + st.local.u16 [%rd23+214], %rs1224; + st.local.u16 [%rd23+216], %rs1224; + st.local.u16 [%rd23+218], %rs1224; + st.local.u16 [%rd23+220], %rs1224; + st.local.u16 [%rd23+222], %rs1224; + st.local.u16 [%rd23+224], %rs1224; + st.local.u16 [%rd23+226], %rs1224; + st.local.u16 [%rd23+228], %rs1224; + st.local.u16 [%rd23+230], %rs1224; + st.local.u16 [%rd23+232], %rs1224; + st.local.u16 [%rd23+234], %rs1224; + st.local.u16 [%rd23+236], %rs1224; + st.local.u16 [%rd23+238], %rs1224; + st.local.u16 [%rd23+240], %rs1224; + st.local.u16 [%rd23+242], %rs1224; + st.local.u16 [%rd23+244], %rs1224; + st.local.u16 [%rd23+246], %rs1224; + st.local.u16 [%rd23+248], %rs1224; + st.local.u16 [%rd23+250], %rs1224; + st.local.u16 [%rd23+252], %rs1224; + st.local.u16 [%rd23+254], %rs1224; + st.local.u16 [%rd23+256], %rs1224; + st.local.u16 [%rd23+258], %rs1224; + st.local.u16 [%rd23+260], %rs1224; + st.local.u16 [%rd23+262], %rs1224; + st.local.u16 [%rd23+264], %rs1224; + st.local.u16 [%rd23+266], %rs1224; + st.local.u16 [%rd23+268], %rs1224; + st.local.u16 [%rd23+270], %rs1224; + st.local.u16 [%rd23+272], %rs1224; + st.local.u16 [%rd23+274], %rs1224; + st.local.u16 [%rd23+276], %rs1224; + st.local.u16 [%rd23+278], %rs1224; + st.local.u16 [%rd23+280], %rs1224; + st.local.u16 [%rd23+282], %rs1224; + st.local.u16 [%rd23+284], %rs1224; + st.local.u16 [%rd23+286], %rs1224; + st.local.u16 [%rd23+288], %rs1224; + st.local.u16 [%rd23+290], %rs1224; + st.local.u16 [%rd23+292], %rs1224; + st.local.u16 [%rd23+294], %rs1224; + st.local.u16 [%rd23+296], %rs1224; + st.local.u16 [%rd23+298], %rs1224; + st.local.u16 [%rd23+300], %rs1224; + st.local.u16 [%rd23+302], %rs1224; + st.local.u16 [%rd23+304], %rs1224; + st.local.u16 [%rd23+306], %rs1224; + st.local.u16 [%rd23+308], %rs1224; + st.local.u16 [%rd23+310], %rs1224; + st.local.u16 [%rd23+312], %rs1224; + st.local.u16 [%rd23+314], %rs1224; + st.local.u16 [%rd23+316], %rs1224; + st.local.u16 [%rd23+318], %rs1224; + st.local.u16 [%rd23+320], %rs1224; + st.local.u16 [%rd23+322], %rs1224; + st.local.u16 [%rd23+324], %rs1224; + st.local.u16 [%rd23+326], %rs1224; + st.local.u16 [%rd23+328], %rs1224; + st.local.u16 [%rd23+330], %rs1224; + st.local.u16 [%rd23+332], %rs1224; + st.local.u16 [%rd23+334], %rs1224; + st.local.u16 [%rd23+336], %rs1224; + st.local.u16 [%rd23+338], %rs1224; + st.local.u16 [%rd23+340], %rs1224; + st.local.u16 [%rd23+342], %rs1224; + st.local.u16 [%rd23+344], %rs1224; + st.local.u16 [%rd23+346], %rs1224; + st.local.u16 [%rd23+348], %rs1224; + st.local.u16 [%rd23+350], %rs1224; + st.local.u16 [%rd23+352], %rs1224; + st.local.u16 [%rd23+354], %rs1224; + st.local.u16 [%rd23+356], %rs1224; + st.local.u16 [%rd23+358], %rs1224; + st.local.u16 [%rd23+360], %rs1224; + st.local.u16 [%rd23+362], %rs1224; + st.local.u16 [%rd23+364], %rs1224; + st.local.u16 [%rd23+366], %rs1224; + st.local.u16 [%rd23+368], %rs1224; + st.local.u16 [%rd23+370], %rs1224; + st.local.u16 [%rd23+372], %rs1224; + st.local.u16 [%rd23+374], %rs1224; + st.local.u16 [%rd23+376], %rs1224; + st.local.u16 [%rd23+378], %rs1224; + st.local.u16 [%rd23+380], %rs1224; + st.local.u16 [%rd23+382], %rs1224; + st.local.u16 [%rd23+384], %rs1224; + st.local.u16 [%rd23+386], %rs1224; + st.local.u16 [%rd23+388], %rs1224; + st.local.u16 [%rd23+390], %rs1224; + st.local.u16 [%rd23+392], %rs1224; + st.local.u16 [%rd23+394], %rs1224; + st.local.u16 [%rd23+396], %rs1224; + st.local.u16 [%rd23+398], %rs1224; + st.local.u16 [%rd23+400], %rs1224; + st.local.u16 [%rd23+402], %rs1224; + st.local.u16 [%rd23+404], %rs1224; + st.local.u16 [%rd23+406], %rs1224; + st.local.u16 [%rd23+408], %rs1224; + st.local.u16 [%rd23+410], %rs1224; + st.local.u16 [%rd23+412], %rs1224; + st.local.u16 [%rd23+414], %rs1224; + st.local.u16 [%rd23+416], %rs1224; + st.local.u16 [%rd23+418], %rs1224; + st.local.u16 [%rd23+420], %rs1224; + st.local.u16 [%rd23+422], %rs1224; + st.local.u16 [%rd23+424], %rs1224; + st.local.u16 [%rd23+426], %rs1224; + st.local.u16 [%rd23+428], %rs1224; + st.local.u16 [%rd23+430], %rs1224; + st.local.u16 [%rd23+432], %rs1224; + st.local.u16 [%rd23+434], %rs1224; + st.local.u16 [%rd23+436], %rs1224; + st.local.u16 [%rd23+438], %rs1224; + st.local.u16 [%rd23+440], %rs1224; + st.local.u16 [%rd23+442], %rs1224; + st.local.u16 [%rd23+444], %rs1224; + st.local.u16 [%rd23+446], %rs1224; + st.local.u16 [%rd23+448], %rs1224; + st.local.u16 [%rd23+450], %rs1224; + st.local.u16 [%rd23+452], %rs1224; + st.local.u16 [%rd23+454], %rs1224; + st.local.u16 [%rd23+456], %rs1224; + st.local.u16 [%rd23+458], %rs1224; + st.local.u16 [%rd23+460], %rs1224; + st.local.u16 [%rd23+462], %rs1224; + st.local.u16 [%rd23+464], %rs1224; + st.local.u16 [%rd23+466], %rs1224; + st.local.u16 [%rd23+468], %rs1224; + st.local.u16 [%rd23+470], %rs1224; + st.local.u16 [%rd23+472], %rs1224; + st.local.u16 [%rd23+474], %rs1224; + st.local.u16 [%rd23+476], %rs1224; + st.local.u16 [%rd23+478], %rs1224; + st.local.u16 [%rd23+480], %rs1224; + st.local.u16 [%rd23+482], %rs1224; + st.local.u16 [%rd23+484], %rs1224; + st.local.u16 [%rd23+486], %rs1224; + st.local.u16 [%rd23+488], %rs1224; + st.local.u16 [%rd23+490], %rs1224; + st.local.u16 [%rd23+492], %rs1224; + st.local.u16 [%rd23+494], %rs1224; + st.local.u16 [%rd23+496], %rs1224; + st.local.u16 [%rd23+498], %rs1224; + st.local.u16 [%rd23+500], %rs1224; + st.local.u16 [%rd23+502], %rs1224; + st.local.u16 [%rd23+504], %rs1224; + st.local.u16 [%rd23+506], %rs1224; + st.local.u16 [%rd23+508], %rs1224; + st.local.u16 [%rd23+510], %rs1224; + st.local.u16 [%rd23+512], %rs1224; + st.local.u16 [%rd23+514], %rs1224; + st.local.u16 [%rd23+516], %rs1224; + st.local.u16 [%rd23+518], %rs1224; + st.local.u16 [%rd23+520], %rs1224; + st.local.u16 [%rd23+522], %rs1224; + st.local.u16 [%rd23+524], %rs1224; + st.local.u16 [%rd23+526], %rs1224; + st.local.u16 [%rd23+528], %rs1224; + st.local.u16 [%rd23+530], %rs1224; + st.local.u16 [%rd23+532], %rs1224; + st.local.u16 [%rd23+534], %rs1224; + st.local.u16 [%rd23+536], %rs1224; + st.local.u16 [%rd23+538], %rs1224; + st.local.u16 [%rd23+540], %rs1224; + st.local.u16 [%rd23+542], %rs1224; + st.local.u16 [%rd23+544], %rs1224; + st.local.u16 [%rd23+546], %rs1224; + st.local.u16 [%rd23+548], %rs1224; + st.local.u16 [%rd23+550], %rs1224; + st.local.u16 [%rd23+552], %rs1224; + st.local.u16 [%rd23+554], %rs1224; + st.local.u16 [%rd23+556], %rs1224; + st.local.u16 [%rd23+558], %rs1224; + st.local.u16 [%rd23+560], %rs1224; + st.local.u16 [%rd23+562], %rs1224; + st.local.u16 [%rd23+564], %rs1224; + st.local.u16 [%rd23+566], %rs1224; + st.local.u16 [%rd23+568], %rs1224; + st.local.u16 [%rd23+570], %rs1224; + st.local.u16 [%rd23+572], %rs1224; + st.local.u16 [%rd23+574], %rs1224; + st.local.u16 [%rd23+576], %rs1224; + st.local.u16 [%rd23+578], %rs1224; + st.local.u16 [%rd23+580], %rs1224; + st.local.u16 [%rd23+582], %rs1224; + st.local.u16 [%rd23+584], %rs1224; + st.local.u16 [%rd23+586], %rs1224; + st.local.u16 [%rd23+588], %rs1224; + st.local.u16 [%rd23+590], %rs1224; + st.local.u16 [%rd23+592], %rs1224; + st.local.u16 [%rd23+594], %rs1224; + st.local.u16 [%rd23+596], %rs1224; + st.local.u16 [%rd23+598], %rs1224; + st.local.u16 [%rd23+600], %rs1224; + st.local.u16 [%rd23+602], %rs1224; + st.local.u16 [%rd23+604], %rs1224; + st.local.u16 [%rd23+606], %rs1224; + st.local.u16 [%rd23+608], %rs1224; + st.local.u16 [%rd23+610], %rs1224; + st.local.u16 [%rd23+612], %rs1224; + st.local.u16 [%rd23+614], %rs1224; + st.local.u16 [%rd23+616], %rs1224; + st.local.u16 [%rd23+618], %rs1224; + st.local.u16 [%rd23+620], %rs1224; + st.local.u16 [%rd23+622], %rs1224; + st.local.u16 [%rd23+624], %rs1224; + st.local.u16 [%rd23+626], %rs1224; + st.local.u16 [%rd23+628], %rs1224; + st.local.u16 [%rd23+630], %rs1224; + st.local.u16 [%rd23+632], %rs1224; + st.local.u16 [%rd23+634], %rs1224; + st.local.u16 [%rd23+636], %rs1224; + st.local.u16 [%rd23+638], %rs1224; + st.local.u16 [%rd23+640], %rs1224; + st.local.u16 [%rd23+642], %rs1224; + st.local.u16 [%rd23+644], %rs1224; + st.local.u16 [%rd23+646], %rs1224; + st.local.u16 [%rd23+648], %rs1224; + st.local.u16 [%rd23+650], %rs1224; + st.local.u16 [%rd23+652], %rs1224; + st.local.u16 [%rd23+654], %rs1224; + st.local.u16 [%rd23+656], %rs1224; + st.local.u16 [%rd23+658], %rs1224; + st.local.u16 [%rd23+660], %rs1224; + st.local.u16 [%rd23+662], %rs1224; + st.local.u16 [%rd23+664], %rs1224; + st.local.u16 [%rd23+666], %rs1224; + st.local.u16 [%rd23+668], %rs1224; + st.local.u16 [%rd23+670], %rs1224; + st.local.u16 [%rd23+672], %rs1224; + st.local.u16 [%rd23+674], %rs1224; + st.local.u16 [%rd23+676], %rs1224; + st.local.u16 [%rd23+678], %rs1224; + st.local.u16 [%rd23+680], %rs1224; + st.local.u16 [%rd23+682], %rs1224; + st.local.u16 [%rd23+684], %rs1224; + st.local.u16 [%rd23+686], %rs1224; + st.local.u16 [%rd23+688], %rs1224; + st.local.u16 [%rd23+690], %rs1224; + st.local.u16 [%rd23+692], %rs1224; + st.local.u16 [%rd23+694], %rs1224; + st.local.u16 [%rd23+696], %rs1224; + st.local.u16 [%rd23+698], %rs1224; + st.local.u16 [%rd23+700], %rs1224; + st.local.u16 [%rd23+702], %rs1224; + st.local.u16 [%rd23+704], %rs1224; + st.local.u16 [%rd23+706], %rs1224; + st.local.u16 [%rd23+708], %rs1224; + st.local.u16 [%rd23+710], %rs1224; + st.local.u16 [%rd23+712], %rs1224; + st.local.u16 [%rd23+714], %rs1224; + st.local.u16 [%rd23+716], %rs1224; + st.local.u16 [%rd23+718], %rs1224; + st.local.u16 [%rd23+720], %rs1224; + st.local.u16 [%rd23+722], %rs1224; + st.local.u16 [%rd23+724], %rs1224; + st.local.u16 [%rd23+726], %rs1224; + st.local.u16 [%rd23+728], %rs1224; + st.local.u16 [%rd23+730], %rs1224; + st.local.u16 [%rd23+732], %rs1224; + st.local.u16 [%rd23+734], %rs1224; + st.local.u16 [%rd23+736], %rs1224; + st.local.u16 [%rd23+738], %rs1224; + st.local.u16 [%rd23+740], %rs1224; + st.local.u16 [%rd23+742], %rs1224; + st.local.u16 [%rd23+744], %rs1224; + st.local.u16 [%rd23+746], %rs1224; + st.local.u16 [%rd23+748], %rs1224; + st.local.u16 [%rd23+750], %rs1224; + st.local.u16 [%rd23+752], %rs1224; + st.local.u16 [%rd23+754], %rs1224; + st.local.u16 [%rd23+756], %rs1224; + st.local.u16 [%rd23+758], %rs1224; + st.local.u16 [%rd23+760], %rs1224; + st.local.u16 [%rd23+762], %rs1224; + st.local.u16 [%rd23+764], %rs1224; + st.local.u16 [%rd23+766], %rs1224; + st.local.u16 [%rd23+768], %rs1224; + st.local.u16 [%rd23+770], %rs1224; + st.local.u16 [%rd23+772], %rs1224; + st.local.u16 [%rd23+774], %rs1224; + st.local.u16 [%rd23+776], %rs1224; + st.local.u16 [%rd23+778], %rs1224; + st.local.u16 [%rd23+780], %rs1224; + st.local.u16 [%rd23+782], %rs1224; + st.local.u16 [%rd23+784], %rs1224; + st.local.u16 [%rd23+786], %rs1224; + st.local.u16 [%rd23+788], %rs1224; + st.local.u16 [%rd23+790], %rs1224; + st.local.u16 [%rd23+792], %rs1224; + st.local.u16 [%rd23+794], %rs1224; + st.local.u16 [%rd23+796], %rs1224; + st.local.u16 [%rd23+798], %rs1224; + st.local.u16 [%rd23+800], %rs1224; + st.local.u16 [%rd23+802], %rs1224; + st.local.u16 [%rd23+804], %rs1224; + st.local.u16 [%rd23+806], %rs1224; + st.local.u16 [%rd23+808], %rs1224; + st.local.u16 [%rd23+810], %rs1224; + st.local.u16 [%rd23+812], %rs1224; + st.local.u16 [%rd23+814], %rs1224; + st.local.u16 [%rd23+816], %rs1224; + st.local.u16 [%rd23+818], %rs1224; + st.local.u16 [%rd23+820], %rs1224; + st.local.u16 [%rd23+822], %rs1224; + st.local.u16 [%rd23+824], %rs1224; + st.local.u16 [%rd23+826], %rs1224; + st.local.u16 [%rd23+828], %rs1224; + st.local.u16 [%rd23+830], %rs1224; + st.local.u16 [%rd23+832], %rs1224; + st.local.u16 [%rd23+834], %rs1224; + st.local.u16 [%rd23+836], %rs1224; + st.local.u16 [%rd23+838], %rs1224; + st.local.u16 [%rd23+840], %rs1224; + st.local.u16 [%rd23+842], %rs1224; + st.local.u16 [%rd23+844], %rs1224; + st.local.u16 [%rd23+846], %rs1224; + st.local.u16 [%rd23+848], %rs1224; + st.local.u16 [%rd23+850], %rs1224; + st.local.u16 [%rd23+852], %rs1224; + st.local.u16 [%rd23+854], %rs1224; + st.local.u16 [%rd23+856], %rs1224; + st.local.u16 [%rd23+858], %rs1224; + st.local.u16 [%rd23+860], %rs1224; + st.local.u16 [%rd23+862], %rs1224; + st.local.u16 [%rd23+864], %rs1224; + st.local.u16 [%rd23+866], %rs1224; + st.local.u16 [%rd23+868], %rs1224; + st.local.u16 [%rd23+870], %rs1224; + st.local.u16 [%rd23+872], %rs1224; + st.local.u16 [%rd23+874], %rs1224; + st.local.u16 [%rd23+876], %rs1224; + st.local.u16 [%rd23+878], %rs1224; + st.local.u16 [%rd23+880], %rs1224; + st.local.u16 [%rd23+882], %rs1224; + st.local.u16 [%rd23+884], %rs1224; + st.local.u16 [%rd23+886], %rs1224; + st.local.u16 [%rd23+888], %rs1224; + st.local.u16 [%rd23+890], %rs1224; + st.local.u16 [%rd23+892], %rs1224; + st.local.u16 [%rd23+894], %rs1224; + st.local.u16 [%rd23+896], %rs1224; + st.local.u16 [%rd23+898], %rs1224; + st.local.u16 [%rd23+900], %rs1224; + st.local.u16 [%rd23+902], %rs1224; + st.local.u16 [%rd23+904], %rs1224; + st.local.u16 [%rd23+906], %rs1224; + st.local.u16 [%rd23+908], %rs1224; + st.local.u16 [%rd23+910], %rs1224; + st.local.u16 [%rd23+912], %rs1224; + st.local.u16 [%rd23+914], %rs1224; + st.local.u16 [%rd23+916], %rs1224; + st.local.u16 [%rd23+918], %rs1224; + st.local.u16 [%rd23+920], %rs1224; + st.local.u16 [%rd23+922], %rs1224; + st.local.u16 [%rd23+924], %rs1224; + st.local.u16 [%rd23+926], %rs1224; + st.local.u16 [%rd23+928], %rs1224; + st.local.u16 [%rd23+930], %rs1224; + st.local.u16 [%rd23+932], %rs1224; + st.local.u16 [%rd23+934], %rs1224; + st.local.u16 [%rd23+936], %rs1224; + st.local.u16 [%rd23+938], %rs1224; + st.local.u16 [%rd23+940], %rs1224; + st.local.u16 [%rd23+942], %rs1224; + st.local.u16 [%rd23+944], %rs1224; + st.local.u16 [%rd23+946], %rs1224; + st.local.u16 [%rd23+948], %rs1224; + st.local.u16 [%rd23+950], %rs1224; + st.local.u16 [%rd23+952], %rs1224; + st.local.u16 [%rd23+954], %rs1224; + st.local.u16 [%rd23+956], %rs1224; + st.local.u16 [%rd23+958], %rs1224; + st.local.u16 [%rd23+960], %rs1224; + st.local.u16 [%rd23+962], %rs1224; + st.local.u16 [%rd23+964], %rs1224; + st.local.u16 [%rd23+966], %rs1224; + st.local.u16 [%rd23+968], %rs1224; + st.local.u16 [%rd23+970], %rs1224; + st.local.u16 [%rd23+972], %rs1224; + st.local.u16 [%rd23+974], %rs1224; + st.local.u16 [%rd23+976], %rs1224; + st.local.u16 [%rd23+978], %rs1224; + st.local.u16 [%rd23+980], %rs1224; + st.local.u16 [%rd23+982], %rs1224; + st.local.u16 [%rd23+984], %rs1224; + st.local.u16 [%rd23+986], %rs1224; + st.local.u16 [%rd23+988], %rs1224; + st.local.u16 [%rd23+990], %rs1224; + st.local.u16 [%rd23+992], %rs1224; + st.local.u16 [%rd23+994], %rs1224; + st.local.u16 [%rd23+996], %rs1224; + st.local.u16 [%rd23+998], %rs1224; + st.local.u16 [%rd23+1000], %rs1224; + st.local.u16 [%rd23+1002], %rs1224; + st.local.u16 [%rd23+1004], %rs1224; + st.local.u16 [%rd23+1006], %rs1224; + st.local.u16 [%rd23+1008], %rs1224; + st.local.u16 [%rd23+1010], %rs1224; + st.local.u16 [%rd23+1012], %rs1224; + st.local.u16 [%rd23+1014], %rs1224; + st.local.u16 [%rd23+1016], %rs1224; + st.local.u16 [%rd23+1018], %rs1224; + st.local.u16 [%rd23+1020], %rs1224; + st.local.u16 [%rd23+1022], %rs1224; + st.local.u16 [%rd23+1024], %rs1224; + mov.u32 %r9435, 0; + mov.u32 %r9545, %r9435; + mov.u32 %r9541, %r9435; + mov.u32 %r9543, %r9435; + +$L__BB1_687: + @%p10 bra $L__BB1_926; + + sub.s32 %r5706, %r6, %r9435; + add.s32 %r1740, %r9435, 4; + mul.lo.s32 %r1741, %r1740, %r1; + add.s32 %r1742, %r9435, 5; + add.s32 %r1743, %r1741, %r1; + add.s32 %r1744, %r9435, 6; + shl.b32 %r5707, %r1, 1; + add.s32 %r1745, %r1741, %r5707; + add.s32 %r1746, %r9435, 7; + mul.lo.s32 %r5708, %r1, 3; + add.s32 %r1747, %r1741, %r5708; + add.s32 %r1748, %r9435, 1; + add.s32 %r1749, %r9435, 2; + add.s32 %r1750, %r9435, 3; + mul.lo.s32 %r1751, %r9435, %r1; + add.s32 %r1752, %r1751, %r5708; + sub.s32 %r1753, %r1752, %r1; + sub.s32 %r1754, %r1753, %r1; + setp.lt.u32 %p793, %r5706, 2; + selp.b32 %r5709, 4369, 13107, %p793; + setp.lt.u32 %p794, %r5706, 3; + selp.b32 %r5710, %r5709, 30583, %p794; + setp.lt.u32 %p795, %r5706, 4; + selp.b32 %r1755, %r5710, 65535, %p795; + mov.u32 %r5705, 0; + mov.u32 %r9439, %r5705; + mov.u32 %r9440, %r5705; + +$L__BB1_689: + shr.u32 %r5712, %r9439, 2; + mul.wide.u32 %rd451, %r5712, 2; + add.s64 %rd25, %rd23, %rd451; + ld.local.u16 %rs238, [%rd25]; + ld.local.u16 %rs239, [%rd25+2]; + setp.ge.u32 %p796, %r9439, %r5; + mov.u32 %r9451, %r5705; + @%p796 bra $L__BB1_698; + + setp.ge.u32 %p797, %r1740, %r6; + mov.u32 %r9451, 0; + @%p797 bra $L__BB1_692; + + add.s32 %r5714, %r1741, %r9439; + cvt.u64.u32 %rd452, %r5714; + add.s64 %rd453, %rd452, %rd5; + shl.b64 %rd454, %rd453, 2; + add.s64 %rd455, %rd3, %rd454; + ld.global.u32 %r5715, [%rd455]; + abs.s32 %r5716, %r5715; + setp.gt.u32 %p798, %r5716, 4; + and.b32 %r5717, %r5716, 1; + setp.eq.b32 %p799, %r5717, 1; + and.pred %p800, %p798, %p799; + selp.u32 %r9451, 1, 0, %p800; + +$L__BB1_692: + setp.ge.u32 %p801, %r1742, %r6; + @%p801 bra $L__BB1_694; + + add.s32 %r5718, %r1743, %r9439; + cvt.u64.u32 %rd456, %r5718; + add.s64 %rd457, %rd456, %rd5; + shl.b64 %rd458, %rd457, 2; + add.s64 %rd459, %rd3, %rd458; + ld.global.u32 %r5719, [%rd459]; + abs.s32 %r5720, %r5719; + setp.gt.u32 %p802, %r5720, 4; + and.b32 %r5721, %r5720, 1; + setp.eq.b32 %p803, %r5721, 1; + and.pred %p804, %p802, %p803; + selp.b32 %r5722, 2, 0, %p804; + or.b32 %r9451, %r5722, %r9451; + +$L__BB1_694: + setp.ge.u32 %p805, %r1744, %r6; + @%p805 bra $L__BB1_696; + + add.s32 %r5723, %r1745, %r9439; + cvt.u64.u32 %rd460, %r5723; + add.s64 %rd461, %rd460, %rd5; + shl.b64 %rd462, %rd461, 2; + add.s64 %rd463, %rd3, %rd462; + ld.global.u32 %r5724, [%rd463]; + abs.s32 %r5725, %r5724; + setp.gt.u32 %p806, %r5725, 4; + and.b32 %r5726, %r5725, 1; + setp.eq.b32 %p807, %r5726, 1; + and.pred %p808, %p806, %p807; + selp.b32 %r5727, 4, 0, %p808; + or.b32 %r9451, %r5727, %r9451; + +$L__BB1_696: + setp.ge.u32 %p809, %r1746, %r6; + @%p809 bra $L__BB1_698; + + add.s32 %r5728, %r1747, %r9439; + cvt.u64.u32 %rd464, %r5728; + add.s64 %rd465, %rd464, %rd5; + shl.b64 %rd466, %rd465, 2; + add.s64 %rd467, %rd3, %rd466; + ld.global.u32 %r5729, [%rd467]; + abs.s32 %r5730, %r5729; + setp.gt.u32 %p810, %r5730, 4; + and.b32 %r5731, %r5730, 1; + setp.eq.b32 %p811, %r5731, 1; + and.pred %p812, %p810, %p811; + selp.b32 %r5732, 8, 0, %p812; + or.b32 %r9451, %r5732, %r9451; + +$L__BB1_698: + add.s32 %r1769, %r9439, 1; + setp.ge.u32 %p813, %r1769, %r5; + @%p813 bra $L__BB1_707; + + setp.ge.u32 %p814, %r1740, %r6; + @%p814 bra $L__BB1_701; + + add.s32 %r5733, %r1741, %r1769; + cvt.u64.u32 %rd468, %r5733; + add.s64 %rd469, %rd468, %rd5; + shl.b64 %rd470, %rd469, 2; + add.s64 %rd471, %rd3, %rd470; + ld.global.u32 %r5734, [%rd471]; + abs.s32 %r5735, %r5734; + setp.gt.u32 %p815, %r5735, 4; + and.b32 %r5736, %r5735, 1; + setp.eq.b32 %p816, %r5736, 1; + and.pred %p817, %p815, %p816; + selp.b32 %r5737, 16, 0, %p817; + or.b32 %r9451, %r5737, %r9451; + +$L__BB1_701: + setp.ge.u32 %p818, %r1742, %r6; + @%p818 bra $L__BB1_703; + + add.s32 %r5738, %r1743, %r1769; + cvt.u64.u32 %rd472, %r5738; + add.s64 %rd473, %rd472, %rd5; + shl.b64 %rd474, %rd473, 2; + add.s64 %rd475, %rd3, %rd474; + ld.global.u32 %r5739, [%rd475]; + abs.s32 %r5740, %r5739; + setp.gt.u32 %p819, %r5740, 4; + and.b32 %r5741, %r5740, 1; + setp.eq.b32 %p820, %r5741, 1; + and.pred %p821, %p819, %p820; + selp.b32 %r5742, 32, 0, %p821; + or.b32 %r9451, %r5742, %r9451; + +$L__BB1_703: + setp.ge.u32 %p822, %r1744, %r6; + @%p822 bra $L__BB1_705; + + add.s32 %r5743, %r1745, %r1769; + cvt.u64.u32 %rd476, %r5743; + add.s64 %rd477, %rd476, %rd5; + shl.b64 %rd478, %rd477, 2; + add.s64 %rd479, %rd3, %rd478; + ld.global.u32 %r5744, [%rd479]; + abs.s32 %r5745, %r5744; + setp.gt.u32 %p823, %r5745, 4; + and.b32 %r5746, %r5745, 1; + setp.eq.b32 %p824, %r5746, 1; + and.pred %p825, %p823, %p824; + selp.b32 %r5747, 64, 0, %p825; + or.b32 %r9451, %r5747, %r9451; + +$L__BB1_705: + setp.ge.u32 %p826, %r1746, %r6; + @%p826 bra $L__BB1_707; + + add.s32 %r5748, %r1747, %r1769; + cvt.u64.u32 %rd480, %r5748; + add.s64 %rd481, %rd480, %rd5; + shl.b64 %rd482, %rd481, 2; + add.s64 %rd483, %rd3, %rd482; + ld.global.u32 %r5749, [%rd483]; + abs.s32 %r5750, %r5749; + setp.gt.u32 %p827, %r5750, 4; + and.b32 %r5751, %r5750, 1; + setp.eq.b32 %p828, %r5751, 1; + and.pred %p829, %p827, %p828; + selp.b32 %r5752, 128, 0, %p829; + or.b32 %r9451, %r5752, %r9451; + +$L__BB1_707: + add.s32 %r1778, %r9439, 2; + setp.ge.u32 %p830, %r1778, %r5; + @%p830 bra $L__BB1_716; + + setp.ge.u32 %p831, %r1740, %r6; + @%p831 bra $L__BB1_710; + + add.s32 %r5753, %r1741, %r1778; + cvt.u64.u32 %rd484, %r5753; + add.s64 %rd485, %rd484, %rd5; + shl.b64 %rd486, %rd485, 2; + add.s64 %rd487, %rd3, %rd486; + ld.global.u32 %r5754, [%rd487]; + abs.s32 %r5755, %r5754; + setp.gt.u32 %p832, %r5755, 4; + and.b32 %r5756, %r5755, 1; + setp.eq.b32 %p833, %r5756, 1; + and.pred %p834, %p832, %p833; + selp.b32 %r5757, 256, 0, %p834; + or.b32 %r9451, %r5757, %r9451; + +$L__BB1_710: + setp.ge.u32 %p835, %r1742, %r6; + @%p835 bra $L__BB1_712; + + add.s32 %r5758, %r1743, %r1778; + cvt.u64.u32 %rd488, %r5758; + add.s64 %rd489, %rd488, %rd5; + shl.b64 %rd490, %rd489, 2; + add.s64 %rd491, %rd3, %rd490; + ld.global.u32 %r5759, [%rd491]; + abs.s32 %r5760, %r5759; + setp.gt.u32 %p836, %r5760, 4; + and.b32 %r5761, %r5760, 1; + setp.eq.b32 %p837, %r5761, 1; + and.pred %p838, %p836, %p837; + selp.b32 %r5762, 512, 0, %p838; + or.b32 %r9451, %r5762, %r9451; + +$L__BB1_712: + setp.ge.u32 %p839, %r1744, %r6; + @%p839 bra $L__BB1_714; + + add.s32 %r5763, %r1745, %r1778; + cvt.u64.u32 %rd492, %r5763; + add.s64 %rd493, %rd492, %rd5; + shl.b64 %rd494, %rd493, 2; + add.s64 %rd495, %rd3, %rd494; + ld.global.u32 %r5764, [%rd495]; + abs.s32 %r5765, %r5764; + setp.gt.u32 %p840, %r5765, 4; + and.b32 %r5766, %r5765, 1; + setp.eq.b32 %p841, %r5766, 1; + and.pred %p842, %p840, %p841; + selp.b32 %r5767, 1024, 0, %p842; + or.b32 %r9451, %r5767, %r9451; + +$L__BB1_714: + setp.ge.u32 %p843, %r1746, %r6; + @%p843 bra $L__BB1_716; + + add.s32 %r5768, %r1747, %r1778; + cvt.u64.u32 %rd496, %r5768; + add.s64 %rd497, %rd496, %rd5; + shl.b64 %rd498, %rd497, 2; + add.s64 %rd499, %rd3, %rd498; + ld.global.u32 %r5769, [%rd499]; + abs.s32 %r5770, %r5769; + setp.gt.u32 %p844, %r5770, 4; + and.b32 %r5771, %r5770, 1; + setp.eq.b32 %p845, %r5771, 1; + and.pred %p846, %p844, %p845; + selp.b32 %r5772, 2048, 0, %p846; + or.b32 %r9451, %r5772, %r9451; + +$L__BB1_716: + add.s32 %r1787, %r9439, 3; + setp.ge.u32 %p847, %r1787, %r5; + @%p847 bra $L__BB1_725; + + setp.ge.u32 %p848, %r1740, %r6; + @%p848 bra $L__BB1_719; + + add.s32 %r5773, %r1741, %r1787; + cvt.u64.u32 %rd500, %r5773; + add.s64 %rd501, %rd500, %rd5; + shl.b64 %rd502, %rd501, 2; + add.s64 %rd503, %rd3, %rd502; + ld.global.u32 %r5774, [%rd503]; + abs.s32 %r5775, %r5774; + setp.gt.u32 %p849, %r5775, 4; + and.b32 %r5776, %r5775, 1; + setp.eq.b32 %p850, %r5776, 1; + and.pred %p851, %p849, %p850; + selp.b32 %r5777, 4096, 0, %p851; + or.b32 %r9451, %r5777, %r9451; + +$L__BB1_719: + setp.ge.u32 %p852, %r1742, %r6; + @%p852 bra $L__BB1_721; + + add.s32 %r5778, %r1743, %r1787; + cvt.u64.u32 %rd504, %r5778; + add.s64 %rd505, %rd504, %rd5; + shl.b64 %rd506, %rd505, 2; + add.s64 %rd507, %rd3, %rd506; + ld.global.u32 %r5779, [%rd507]; + abs.s32 %r5780, %r5779; + setp.gt.u32 %p853, %r5780, 4; + and.b32 %r5781, %r5780, 1; + setp.eq.b32 %p854, %r5781, 1; + and.pred %p855, %p853, %p854; + selp.b32 %r5782, 8192, 0, %p855; + or.b32 %r9451, %r5782, %r9451; + +$L__BB1_721: + setp.ge.u32 %p856, %r1744, %r6; + @%p856 bra $L__BB1_723; + + add.s32 %r5783, %r1745, %r1787; + cvt.u64.u32 %rd508, %r5783; + add.s64 %rd509, %rd508, %rd5; + shl.b64 %rd510, %rd509, 2; + add.s64 %rd511, %rd3, %rd510; + ld.global.u32 %r5784, [%rd511]; + abs.s32 %r5785, %r5784; + setp.gt.u32 %p857, %r5785, 4; + and.b32 %r5786, %r5785, 1; + setp.eq.b32 %p858, %r5786, 1; + and.pred %p859, %p857, %p858; + selp.b32 %r5787, 16384, 0, %p859; + or.b32 %r9451, %r5787, %r9451; + +$L__BB1_723: + setp.ge.u32 %p860, %r1746, %r6; + @%p860 bra $L__BB1_725; + + add.s32 %r5788, %r1747, %r1787; + cvt.u64.u32 %rd512, %r5788; + add.s64 %rd513, %rd512, %rd5; + shl.b64 %rd514, %rd513, 2; + add.s64 %rd515, %rd3, %rd514; + ld.global.u32 %r5789, [%rd515]; + abs.s32 %r5790, %r5789; + setp.gt.u32 %p861, %r5790, 4; + and.b32 %r5791, %r5790, 1; + setp.eq.b32 %p862, %r5791, 1; + and.pred %p863, %p861, %p862; + selp.b32 %r5792, 32768, 0, %p863; + or.b32 %r9451, %r5792, %r9451; + +$L__BB1_725: + add.s32 %r5794, %r9439, 4; + setp.ge.u32 %p864, %r5794, %r5; + mov.u32 %r9467, 0; + @%p864 bra $L__BB1_734; + + setp.ge.u32 %p865, %r1740, %r6; + mov.u32 %r9467, 0; + @%p865 bra $L__BB1_728; + + add.s32 %r5796, %r1741, %r9439; + add.s32 %r5797, %r5796, 4; + cvt.u64.u32 %rd516, %r5797; + add.s64 %rd517, %rd516, %rd5; + shl.b64 %rd518, %rd517, 2; + add.s64 %rd519, %rd3, %rd518; + ld.global.u32 %r5798, [%rd519]; + abs.s32 %r5799, %r5798; + setp.gt.u32 %p866, %r5799, 4; + and.b32 %r5800, %r5799, 1; + setp.eq.b32 %p867, %r5800, 1; + and.pred %p868, %p866, %p867; + selp.u32 %r9467, 1, 0, %p868; + +$L__BB1_728: + setp.ge.u32 %p869, %r1742, %r6; + @%p869 bra $L__BB1_730; + + add.s32 %r5801, %r1743, %r9439; + add.s32 %r5802, %r5801, 4; + cvt.u64.u32 %rd520, %r5802; + add.s64 %rd521, %rd520, %rd5; + shl.b64 %rd522, %rd521, 2; + add.s64 %rd523, %rd3, %rd522; + ld.global.u32 %r5803, [%rd523]; + abs.s32 %r5804, %r5803; + setp.gt.u32 %p870, %r5804, 4; + and.b32 %r5805, %r5804, 1; + setp.eq.b32 %p871, %r5805, 1; + and.pred %p872, %p870, %p871; + selp.b32 %r5806, 2, 0, %p872; + or.b32 %r9467, %r5806, %r9467; + +$L__BB1_730: + setp.ge.u32 %p873, %r1744, %r6; + @%p873 bra $L__BB1_732; + + add.s32 %r5807, %r1745, %r9439; + add.s32 %r5808, %r5807, 4; + cvt.u64.u32 %rd524, %r5808; + add.s64 %rd525, %rd524, %rd5; + shl.b64 %rd526, %rd525, 2; + add.s64 %rd527, %rd3, %rd526; + ld.global.u32 %r5809, [%rd527]; + abs.s32 %r5810, %r5809; + setp.gt.u32 %p874, %r5810, 4; + and.b32 %r5811, %r5810, 1; + setp.eq.b32 %p875, %r5811, 1; + and.pred %p876, %p874, %p875; + selp.b32 %r5812, 4, 0, %p876; + or.b32 %r9467, %r5812, %r9467; + +$L__BB1_732: + setp.ge.u32 %p877, %r1746, %r6; + @%p877 bra $L__BB1_734; + + add.s32 %r5813, %r1747, %r9439; + add.s32 %r5814, %r5813, 4; + cvt.u64.u32 %rd528, %r5814; + add.s64 %rd529, %rd528, %rd5; + shl.b64 %rd530, %rd529, 2; + add.s64 %rd531, %rd3, %rd530; + ld.global.u32 %r5815, [%rd531]; + abs.s32 %r5816, %r5815; + setp.gt.u32 %p878, %r5816, 4; + and.b32 %r5817, %r5816, 1; + setp.eq.b32 %p879, %r5817, 1; + and.pred %p880, %p878, %p879; + selp.b32 %r5818, 8, 0, %p880; + or.b32 %r9467, %r5818, %r9467; + +$L__BB1_734: + add.s32 %r1804, %r9439, 5; + setp.ge.u32 %p881, %r1804, %r5; + @%p881 bra $L__BB1_743; + + setp.ge.u32 %p882, %r1740, %r6; + @%p882 bra $L__BB1_737; + + add.s32 %r5819, %r1741, %r1804; + cvt.u64.u32 %rd532, %r5819; + add.s64 %rd533, %rd532, %rd5; + shl.b64 %rd534, %rd533, 2; + add.s64 %rd535, %rd3, %rd534; + ld.global.u32 %r5820, [%rd535]; + abs.s32 %r5821, %r5820; + setp.gt.u32 %p883, %r5821, 4; + and.b32 %r5822, %r5821, 1; + setp.eq.b32 %p884, %r5822, 1; + and.pred %p885, %p883, %p884; + selp.b32 %r5823, 16, 0, %p885; + or.b32 %r9467, %r5823, %r9467; + +$L__BB1_737: + setp.ge.u32 %p886, %r1742, %r6; + @%p886 bra $L__BB1_739; + + add.s32 %r5824, %r1743, %r1804; + cvt.u64.u32 %rd536, %r5824; + add.s64 %rd537, %rd536, %rd5; + shl.b64 %rd538, %rd537, 2; + add.s64 %rd539, %rd3, %rd538; + ld.global.u32 %r5825, [%rd539]; + abs.s32 %r5826, %r5825; + setp.gt.u32 %p887, %r5826, 4; + and.b32 %r5827, %r5826, 1; + setp.eq.b32 %p888, %r5827, 1; + and.pred %p889, %p887, %p888; + selp.b32 %r5828, 32, 0, %p889; + or.b32 %r9467, %r5828, %r9467; + +$L__BB1_739: + setp.ge.u32 %p890, %r1744, %r6; + @%p890 bra $L__BB1_741; + + add.s32 %r5829, %r1745, %r1804; + cvt.u64.u32 %rd540, %r5829; + add.s64 %rd541, %rd540, %rd5; + shl.b64 %rd542, %rd541, 2; + add.s64 %rd543, %rd3, %rd542; + ld.global.u32 %r5830, [%rd543]; + abs.s32 %r5831, %r5830; + setp.gt.u32 %p891, %r5831, 4; + and.b32 %r5832, %r5831, 1; + setp.eq.b32 %p892, %r5832, 1; + and.pred %p893, %p891, %p892; + selp.b32 %r5833, 64, 0, %p893; + or.b32 %r9467, %r5833, %r9467; + +$L__BB1_741: + setp.ge.u32 %p894, %r1746, %r6; + @%p894 bra $L__BB1_743; + + add.s32 %r5834, %r1747, %r1804; + cvt.u64.u32 %rd544, %r5834; + add.s64 %rd545, %rd544, %rd5; + shl.b64 %rd546, %rd545, 2; + add.s64 %rd547, %rd3, %rd546; + ld.global.u32 %r5835, [%rd547]; + abs.s32 %r5836, %r5835; + setp.gt.u32 %p895, %r5836, 4; + and.b32 %r5837, %r5836, 1; + setp.eq.b32 %p896, %r5837, 1; + and.pred %p897, %p895, %p896; + selp.b32 %r5838, 128, 0, %p897; + or.b32 %r9467, %r5838, %r9467; + +$L__BB1_743: + add.s32 %r1813, %r9439, 6; + setp.ge.u32 %p898, %r1813, %r5; + @%p898 bra $L__BB1_752; + + setp.ge.u32 %p899, %r1740, %r6; + @%p899 bra $L__BB1_746; + + add.s32 %r5839, %r1741, %r1813; + cvt.u64.u32 %rd548, %r5839; + add.s64 %rd549, %rd548, %rd5; + shl.b64 %rd550, %rd549, 2; + add.s64 %rd551, %rd3, %rd550; + ld.global.u32 %r5840, [%rd551]; + abs.s32 %r5841, %r5840; + setp.gt.u32 %p900, %r5841, 4; + and.b32 %r5842, %r5841, 1; + setp.eq.b32 %p901, %r5842, 1; + and.pred %p902, %p900, %p901; + selp.b32 %r5843, 256, 0, %p902; + or.b32 %r9467, %r5843, %r9467; + +$L__BB1_746: + setp.ge.u32 %p903, %r1742, %r6; + @%p903 bra $L__BB1_748; + + add.s32 %r5844, %r1743, %r1813; + cvt.u64.u32 %rd552, %r5844; + add.s64 %rd553, %rd552, %rd5; + shl.b64 %rd554, %rd553, 2; + add.s64 %rd555, %rd3, %rd554; + ld.global.u32 %r5845, [%rd555]; + abs.s32 %r5846, %r5845; + setp.gt.u32 %p904, %r5846, 4; + and.b32 %r5847, %r5846, 1; + setp.eq.b32 %p905, %r5847, 1; + and.pred %p906, %p904, %p905; + selp.b32 %r5848, 512, 0, %p906; + or.b32 %r9467, %r5848, %r9467; + +$L__BB1_748: + setp.ge.u32 %p907, %r1744, %r6; + @%p907 bra $L__BB1_750; + + add.s32 %r5849, %r1745, %r1813; + cvt.u64.u32 %rd556, %r5849; + add.s64 %rd557, %rd556, %rd5; + shl.b64 %rd558, %rd557, 2; + add.s64 %rd559, %rd3, %rd558; + ld.global.u32 %r5850, [%rd559]; + abs.s32 %r5851, %r5850; + setp.gt.u32 %p908, %r5851, 4; + and.b32 %r5852, %r5851, 1; + setp.eq.b32 %p909, %r5852, 1; + and.pred %p910, %p908, %p909; + selp.b32 %r5853, 1024, 0, %p910; + or.b32 %r9467, %r5853, %r9467; + +$L__BB1_750: + setp.ge.u32 %p911, %r1746, %r6; + @%p911 bra $L__BB1_752; + + add.s32 %r5854, %r1747, %r1813; + cvt.u64.u32 %rd560, %r5854; + add.s64 %rd561, %rd560, %rd5; + shl.b64 %rd562, %rd561, 2; + add.s64 %rd563, %rd3, %rd562; + ld.global.u32 %r5855, [%rd563]; + abs.s32 %r5856, %r5855; + setp.gt.u32 %p912, %r5856, 4; + and.b32 %r5857, %r5856, 1; + setp.eq.b32 %p913, %r5857, 1; + and.pred %p914, %p912, %p913; + selp.b32 %r5858, 2048, 0, %p914; + or.b32 %r9467, %r5858, %r9467; + +$L__BB1_752: + add.s32 %r1822, %r9439, 7; + setp.ge.u32 %p915, %r1822, %r5; + @%p915 bra $L__BB1_761; + + setp.ge.u32 %p916, %r1740, %r6; + @%p916 bra $L__BB1_755; + + add.s32 %r5859, %r1741, %r1822; + cvt.u64.u32 %rd564, %r5859; + add.s64 %rd565, %rd564, %rd5; + shl.b64 %rd566, %rd565, 2; + add.s64 %rd567, %rd3, %rd566; + ld.global.u32 %r5860, [%rd567]; + abs.s32 %r5861, %r5860; + setp.gt.u32 %p917, %r5861, 4; + and.b32 %r5862, %r5861, 1; + setp.eq.b32 %p918, %r5862, 1; + and.pred %p919, %p917, %p918; + selp.b32 %r5863, 4096, 0, %p919; + or.b32 %r9467, %r5863, %r9467; + +$L__BB1_755: + setp.ge.u32 %p920, %r1742, %r6; + @%p920 bra $L__BB1_757; + + add.s32 %r5864, %r1743, %r1822; + cvt.u64.u32 %rd568, %r5864; + add.s64 %rd569, %rd568, %rd5; + shl.b64 %rd570, %rd569, 2; + add.s64 %rd571, %rd3, %rd570; + ld.global.u32 %r5865, [%rd571]; + abs.s32 %r5866, %r5865; + setp.gt.u32 %p921, %r5866, 4; + and.b32 %r5867, %r5866, 1; + setp.eq.b32 %p922, %r5867, 1; + and.pred %p923, %p921, %p922; + selp.b32 %r5868, 8192, 0, %p923; + or.b32 %r9467, %r5868, %r9467; + +$L__BB1_757: + setp.ge.u32 %p924, %r1744, %r6; + @%p924 bra $L__BB1_759; + + add.s32 %r5869, %r1745, %r1822; + cvt.u64.u32 %rd572, %r5869; + add.s64 %rd573, %rd572, %rd5; + shl.b64 %rd574, %rd573, 2; + add.s64 %rd575, %rd3, %rd574; + ld.global.u32 %r5870, [%rd575]; + abs.s32 %r5871, %r5870; + setp.gt.u32 %p925, %r5871, 4; + and.b32 %r5872, %r5871, 1; + setp.eq.b32 %p926, %r5872, 1; + and.pred %p927, %p925, %p926; + selp.b32 %r5873, 16384, 0, %p927; + or.b32 %r9467, %r5873, %r9467; + +$L__BB1_759: + setp.ge.u32 %p928, %r1746, %r6; + @%p928 bra $L__BB1_761; + + add.s32 %r5874, %r1747, %r1822; + cvt.u64.u32 %rd576, %r5874; + add.s64 %rd577, %rd576, %rd5; + shl.b64 %rd578, %rd577, 2; + add.s64 %rd579, %rd3, %rd578; + ld.global.u32 %r5875, [%rd579]; + abs.s32 %r5876, %r5875; + setp.gt.u32 %p929, %r5876, 4; + and.b32 %r5877, %r5876, 1; + setp.eq.b32 %p930, %r5877, 1; + and.pred %p931, %p929, %p930; + selp.b32 %r5878, 32768, 0, %p931; + or.b32 %r9467, %r5878, %r9467; + +$L__BB1_761: + mov.b32 %r1831, {%rs238, %rs239}; + add.s32 %r5880, %r1751, %r9439; + cvt.u64.u32 %rd580, %r5880; + add.s64 %rd581, %rd580, %rd5; + shl.b64 %rd582, %rd581, 2; + add.s64 %rd26, %rd3, %rd582; + add.s32 %r5881, %r1754, %r9439; + cvt.u64.u32 %rd583, %r5881; + add.s64 %rd584, %rd583, %rd5; + shl.b64 %rd585, %rd584, 2; + add.s64 %rd27, %rd3, %rd585; + add.s32 %r5882, %r1753, %r9439; + cvt.u64.u32 %rd586, %r5882; + add.s64 %rd587, %rd586, %rd5; + shl.b64 %rd588, %rd587, 2; + add.s64 %rd28, %rd3, %rd588; + add.s32 %r5883, %r1752, %r9439; + cvt.u64.u32 %rd589, %r5883; + add.s64 %rd590, %rd589, %rd5; + shl.b64 %rd591, %rd590, 2; + add.s64 %rd29, %rd3, %rd591; + mov.u32 %r9483, 0; + @%p796 bra $L__BB1_770; + + setp.le.u32 %p933, %r6, %r9435; + mov.u32 %r9483, 0; + @%p933 bra $L__BB1_764; + + ld.global.u32 %r5885, [%rd26]; + abs.s32 %r5886, %r5885; + setp.gt.u32 %p934, %r5886, 4; + and.b32 %r5887, %r5886, 1; + setp.eq.b32 %p935, %r5887, 1; + and.pred %p936, %p934, %p935; + selp.u32 %r9483, 1, 0, %p936; + +$L__BB1_764: + setp.ge.u32 %p937, %r1748, %r6; + @%p937 bra $L__BB1_766; + + ld.global.u32 %r5888, [%rd27]; + abs.s32 %r5889, %r5888; + setp.gt.u32 %p938, %r5889, 4; + and.b32 %r5890, %r5889, 1; + setp.eq.b32 %p939, %r5890, 1; + and.pred %p940, %p938, %p939; + selp.b32 %r5891, 2, 0, %p940; + or.b32 %r9483, %r5891, %r9483; + +$L__BB1_766: + setp.ge.u32 %p941, %r1749, %r6; + @%p941 bra $L__BB1_768; + + ld.global.u32 %r5892, [%rd28]; + abs.s32 %r5893, %r5892; + setp.gt.u32 %p942, %r5893, 4; + and.b32 %r5894, %r5893, 1; + setp.eq.b32 %p943, %r5894, 1; + and.pred %p944, %p942, %p943; + selp.b32 %r5895, 4, 0, %p944; + or.b32 %r9483, %r5895, %r9483; + +$L__BB1_768: + setp.ge.u32 %p945, %r1750, %r6; + @%p945 bra $L__BB1_770; + + ld.global.u32 %r5896, [%rd29]; + abs.s32 %r5897, %r5896; + setp.gt.u32 %p946, %r5897, 4; + and.b32 %r5898, %r5897, 1; + setp.eq.b32 %p947, %r5898, 1; + and.pred %p948, %p946, %p947; + selp.b32 %r5899, 8, 0, %p948; + or.b32 %r9483, %r5899, %r9483; + +$L__BB1_770: + add.s32 %r5900, %r1751, %r1769; + cvt.u64.u32 %rd592, %r5900; + add.s64 %rd593, %rd592, %rd5; + shl.b64 %rd594, %rd593, 2; + add.s64 %rd30, %rd3, %rd594; + add.s32 %r5901, %r1754, %r1769; + cvt.u64.u32 %rd595, %r5901; + add.s64 %rd596, %rd595, %rd5; + shl.b64 %rd597, %rd596, 2; + add.s64 %rd31, %rd3, %rd597; + add.s32 %r5902, %r1753, %r1769; + cvt.u64.u32 %rd598, %r5902; + add.s64 %rd599, %rd598, %rd5; + shl.b64 %rd600, %rd599, 2; + add.s64 %rd32, %rd3, %rd600; + add.s32 %r5903, %r1752, %r1769; + cvt.u64.u32 %rd601, %r5903; + add.s64 %rd602, %rd601, %rd5; + shl.b64 %rd603, %rd602, 2; + add.s64 %rd33, %rd3, %rd603; + shl.b32 %r5904, %r9467, 16; + or.b32 %r1840, %r5904, %r9451; + @%p813 bra $L__BB1_779; + + setp.le.u32 %p950, %r6, %r9435; + @%p950 bra $L__BB1_773; + + ld.global.u32 %r5905, [%rd30]; + abs.s32 %r5906, %r5905; + setp.gt.u32 %p951, %r5906, 4; + and.b32 %r5907, %r5906, 1; + setp.eq.b32 %p952, %r5907, 1; + and.pred %p953, %p951, %p952; + selp.b32 %r5908, 16, 0, %p953; + or.b32 %r9483, %r5908, %r9483; + +$L__BB1_773: + setp.ge.u32 %p954, %r1748, %r6; + @%p954 bra $L__BB1_775; + + ld.global.u32 %r5909, [%rd31]; + abs.s32 %r5910, %r5909; + setp.gt.u32 %p955, %r5910, 4; + and.b32 %r5911, %r5910, 1; + setp.eq.b32 %p956, %r5911, 1; + and.pred %p957, %p955, %p956; + selp.b32 %r5912, 32, 0, %p957; + or.b32 %r9483, %r5912, %r9483; + +$L__BB1_775: + setp.ge.u32 %p958, %r1749, %r6; + @%p958 bra $L__BB1_777; + + ld.global.u32 %r5913, [%rd32]; + abs.s32 %r5914, %r5913; + setp.gt.u32 %p959, %r5914, 4; + and.b32 %r5915, %r5914, 1; + setp.eq.b32 %p960, %r5915, 1; + and.pred %p961, %p959, %p960; + selp.b32 %r5916, 64, 0, %p961; + or.b32 %r9483, %r5916, %r9483; + +$L__BB1_777: + setp.ge.u32 %p962, %r1750, %r6; + @%p962 bra $L__BB1_779; + + ld.global.u32 %r5917, [%rd33]; + abs.s32 %r5918, %r5917; + setp.gt.u32 %p963, %r5918, 4; + and.b32 %r5919, %r5918, 1; + setp.eq.b32 %p964, %r5919, 1; + and.pred %p965, %p963, %p964; + selp.b32 %r5920, 128, 0, %p965; + or.b32 %r9483, %r5920, %r9483; + +$L__BB1_779: + add.s32 %r5921, %r1751, %r1778; + cvt.u64.u32 %rd604, %r5921; + add.s64 %rd605, %rd604, %rd5; + shl.b64 %rd606, %rd605, 2; + add.s64 %rd34, %rd3, %rd606; + add.s32 %r5922, %r1754, %r1778; + cvt.u64.u32 %rd607, %r5922; + add.s64 %rd608, %rd607, %rd5; + shl.b64 %rd609, %rd608, 2; + add.s64 %rd35, %rd3, %rd609; + add.s32 %r5923, %r1753, %r1778; + cvt.u64.u32 %rd610, %r5923; + add.s64 %rd611, %rd610, %rd5; + shl.b64 %rd612, %rd611, 2; + add.s64 %rd36, %rd3, %rd612; + add.s32 %r5924, %r1752, %r1778; + cvt.u64.u32 %rd613, %r5924; + add.s64 %rd614, %rd613, %rd5; + shl.b64 %rd615, %rd614, 2; + add.s64 %rd37, %rd3, %rd615; + @%p830 bra $L__BB1_788; + + setp.le.u32 %p967, %r6, %r9435; + @%p967 bra $L__BB1_782; + + ld.global.u32 %r5925, [%rd34]; + abs.s32 %r5926, %r5925; + setp.gt.u32 %p968, %r5926, 4; + and.b32 %r5927, %r5926, 1; + setp.eq.b32 %p969, %r5927, 1; + and.pred %p970, %p968, %p969; + selp.b32 %r5928, 256, 0, %p970; + or.b32 %r9483, %r5928, %r9483; + +$L__BB1_782: + setp.ge.u32 %p971, %r1748, %r6; + @%p971 bra $L__BB1_784; + + ld.global.u32 %r5929, [%rd35]; + abs.s32 %r5930, %r5929; + setp.gt.u32 %p972, %r5930, 4; + and.b32 %r5931, %r5930, 1; + setp.eq.b32 %p973, %r5931, 1; + and.pred %p974, %p972, %p973; + selp.b32 %r5932, 512, 0, %p974; + or.b32 %r9483, %r5932, %r9483; + +$L__BB1_784: + setp.ge.u32 %p975, %r1749, %r6; + @%p975 bra $L__BB1_786; + + ld.global.u32 %r5933, [%rd36]; + abs.s32 %r5934, %r5933; + setp.gt.u32 %p976, %r5934, 4; + and.b32 %r5935, %r5934, 1; + setp.eq.b32 %p977, %r5935, 1; + and.pred %p978, %p976, %p977; + selp.b32 %r5936, 1024, 0, %p978; + or.b32 %r9483, %r5936, %r9483; + +$L__BB1_786: + setp.ge.u32 %p979, %r1750, %r6; + @%p979 bra $L__BB1_788; + + ld.global.u32 %r5937, [%rd37]; + abs.s32 %r5938, %r5937; + setp.gt.u32 %p980, %r5938, 4; + and.b32 %r5939, %r5938, 1; + setp.eq.b32 %p981, %r5939, 1; + and.pred %p982, %p980, %p981; + selp.b32 %r5940, 2048, 0, %p982; + or.b32 %r9483, %r5940, %r9483; + +$L__BB1_788: + add.s32 %r5941, %r1751, %r1787; + cvt.u64.u32 %rd616, %r5941; + add.s64 %rd617, %rd616, %rd5; + shl.b64 %rd618, %rd617, 2; + add.s64 %rd38, %rd3, %rd618; + add.s32 %r5942, %r1754, %r1787; + cvt.u64.u32 %rd619, %r5942; + add.s64 %rd620, %rd619, %rd5; + shl.b64 %rd621, %rd620, 2; + add.s64 %rd39, %rd3, %rd621; + add.s32 %r5943, %r1753, %r1787; + cvt.u64.u32 %rd622, %r5943; + add.s64 %rd623, %rd622, %rd5; + shl.b64 %rd624, %rd623, 2; + add.s64 %rd40, %rd3, %rd624; + add.s32 %r5944, %r1752, %r1787; + cvt.u64.u32 %rd625, %r5944; + add.s64 %rd626, %rd625, %rd5; + shl.b64 %rd627, %rd626, 2; + add.s64 %rd41, %rd3, %rd627; + @%p847 bra $L__BB1_797; + + setp.le.u32 %p984, %r6, %r9435; + @%p984 bra $L__BB1_791; + + ld.global.u32 %r5945, [%rd38]; + abs.s32 %r5946, %r5945; + setp.gt.u32 %p985, %r5946, 4; + and.b32 %r5947, %r5946, 1; + setp.eq.b32 %p986, %r5947, 1; + and.pred %p987, %p985, %p986; + selp.b32 %r5948, 4096, 0, %p987; + or.b32 %r9483, %r5948, %r9483; + +$L__BB1_791: + setp.ge.u32 %p988, %r1748, %r6; + @%p988 bra $L__BB1_793; + + ld.global.u32 %r5949, [%rd39]; + abs.s32 %r5950, %r5949; + setp.gt.u32 %p989, %r5950, 4; + and.b32 %r5951, %r5950, 1; + setp.eq.b32 %p990, %r5951, 1; + and.pred %p991, %p989, %p990; + selp.b32 %r5952, 8192, 0, %p991; + or.b32 %r9483, %r5952, %r9483; + +$L__BB1_793: + setp.ge.u32 %p992, %r1749, %r6; + @%p992 bra $L__BB1_795; + + ld.global.u32 %r5953, [%rd40]; + abs.s32 %r5954, %r5953; + setp.gt.u32 %p993, %r5954, 4; + and.b32 %r5955, %r5954, 1; + setp.eq.b32 %p994, %r5955, 1; + and.pred %p995, %p993, %p994; + selp.b32 %r5956, 16384, 0, %p995; + or.b32 %r9483, %r5956, %r9483; + +$L__BB1_795: + setp.ge.u32 %p996, %r1750, %r6; + @%p996 bra $L__BB1_797; + + ld.global.u32 %r5957, [%rd41]; + abs.s32 %r5958, %r5957; + setp.gt.u32 %p997, %r5958, 4; + and.b32 %r5959, %r5958, 1; + setp.eq.b32 %p998, %r5959, 1; + and.pred %p999, %p997, %p998; + selp.b32 %r5960, 32768, 0, %p999; + or.b32 %r9483, %r5960, %r9483; + +$L__BB1_797: + mov.u32 %r9499, 0; + @%p864 bra $L__BB1_806; + + setp.le.u32 %p1001, %r6, %r9435; + mov.u32 %r9499, 0; + @%p1001 bra $L__BB1_800; + + add.s32 %r5965, %r5880, 4; + cvt.u64.u32 %rd628, %r5965; + add.s64 %rd629, %rd628, %rd5; + shl.b64 %rd630, %rd629, 2; + add.s64 %rd631, %rd3, %rd630; + ld.global.u32 %r5966, [%rd631]; + abs.s32 %r5967, %r5966; + setp.gt.u32 %p1002, %r5967, 4; + and.b32 %r5968, %r5967, 1; + setp.eq.b32 %p1003, %r5968, 1; + and.pred %p1004, %p1002, %p1003; + selp.u32 %r9499, 1, 0, %p1004; + +$L__BB1_800: + setp.ge.u32 %p1005, %r1748, %r6; + @%p1005 bra $L__BB1_802; + + add.s32 %r5970, %r5881, 4; + cvt.u64.u32 %rd632, %r5970; + add.s64 %rd633, %rd632, %rd5; + shl.b64 %rd634, %rd633, 2; + add.s64 %rd635, %rd3, %rd634; + ld.global.u32 %r5971, [%rd635]; + abs.s32 %r5972, %r5971; + setp.gt.u32 %p1006, %r5972, 4; + and.b32 %r5973, %r5972, 1; + setp.eq.b32 %p1007, %r5973, 1; + and.pred %p1008, %p1006, %p1007; + selp.b32 %r5974, 2, 0, %p1008; + or.b32 %r9499, %r5974, %r9499; + +$L__BB1_802: + setp.ge.u32 %p1009, %r1749, %r6; + @%p1009 bra $L__BB1_804; + + add.s32 %r5976, %r5882, 4; + cvt.u64.u32 %rd636, %r5976; + add.s64 %rd637, %rd636, %rd5; + shl.b64 %rd638, %rd637, 2; + add.s64 %rd639, %rd3, %rd638; + ld.global.u32 %r5977, [%rd639]; + abs.s32 %r5978, %r5977; + setp.gt.u32 %p1010, %r5978, 4; + and.b32 %r5979, %r5978, 1; + setp.eq.b32 %p1011, %r5979, 1; + and.pred %p1012, %p1010, %p1011; + selp.b32 %r5980, 4, 0, %p1012; + or.b32 %r9499, %r5980, %r9499; + +$L__BB1_804: + setp.ge.u32 %p1013, %r1750, %r6; + @%p1013 bra $L__BB1_806; + + add.s32 %r5982, %r5883, 4; + cvt.u64.u32 %rd640, %r5982; + add.s64 %rd641, %rd640, %rd5; + shl.b64 %rd642, %rd641, 2; + add.s64 %rd643, %rd3, %rd642; + ld.global.u32 %r5983, [%rd643]; + abs.s32 %r5984, %r5983; + setp.gt.u32 %p1014, %r5984, 4; + and.b32 %r5985, %r5984, 1; + setp.eq.b32 %p1015, %r5985, 1; + and.pred %p1016, %p1014, %p1015; + selp.b32 %r5986, 8, 0, %p1016; + or.b32 %r9499, %r5986, %r9499; + +$L__BB1_806: + @%p881 bra $L__BB1_815; + + setp.le.u32 %p1018, %r6, %r9435; + @%p1018 bra $L__BB1_809; + + add.s32 %r5987, %r1751, %r1804; + cvt.u64.u32 %rd644, %r5987; + add.s64 %rd645, %rd644, %rd5; + shl.b64 %rd646, %rd645, 2; + add.s64 %rd647, %rd3, %rd646; + ld.global.u32 %r5988, [%rd647]; + abs.s32 %r5989, %r5988; + setp.gt.u32 %p1019, %r5989, 4; + and.b32 %r5990, %r5989, 1; + setp.eq.b32 %p1020, %r5990, 1; + and.pred %p1021, %p1019, %p1020; + selp.b32 %r5991, 16, 0, %p1021; + or.b32 %r9499, %r5991, %r9499; + +$L__BB1_809: + setp.ge.u32 %p1022, %r1748, %r6; + @%p1022 bra $L__BB1_811; + + add.s32 %r5992, %r1754, %r1804; + cvt.u64.u32 %rd648, %r5992; + add.s64 %rd649, %rd648, %rd5; + shl.b64 %rd650, %rd649, 2; + add.s64 %rd651, %rd3, %rd650; + ld.global.u32 %r5993, [%rd651]; + abs.s32 %r5994, %r5993; + setp.gt.u32 %p1023, %r5994, 4; + and.b32 %r5995, %r5994, 1; + setp.eq.b32 %p1024, %r5995, 1; + and.pred %p1025, %p1023, %p1024; + selp.b32 %r5996, 32, 0, %p1025; + or.b32 %r9499, %r5996, %r9499; + +$L__BB1_811: + setp.ge.u32 %p1026, %r1749, %r6; + @%p1026 bra $L__BB1_813; + + add.s32 %r5997, %r1753, %r1804; + cvt.u64.u32 %rd652, %r5997; + add.s64 %rd653, %rd652, %rd5; + shl.b64 %rd654, %rd653, 2; + add.s64 %rd655, %rd3, %rd654; + ld.global.u32 %r5998, [%rd655]; + abs.s32 %r5999, %r5998; + setp.gt.u32 %p1027, %r5999, 4; + and.b32 %r6000, %r5999, 1; + setp.eq.b32 %p1028, %r6000, 1; + and.pred %p1029, %p1027, %p1028; + selp.b32 %r6001, 64, 0, %p1029; + or.b32 %r9499, %r6001, %r9499; + +$L__BB1_813: + setp.ge.u32 %p1030, %r1750, %r6; + @%p1030 bra $L__BB1_815; + + add.s32 %r6002, %r1752, %r1804; + cvt.u64.u32 %rd656, %r6002; + add.s64 %rd657, %rd656, %rd5; + shl.b64 %rd658, %rd657, 2; + add.s64 %rd659, %rd3, %rd658; + ld.global.u32 %r6003, [%rd659]; + abs.s32 %r6004, %r6003; + setp.gt.u32 %p1031, %r6004, 4; + and.b32 %r6005, %r6004, 1; + setp.eq.b32 %p1032, %r6005, 1; + and.pred %p1033, %p1031, %p1032; + selp.b32 %r6006, 128, 0, %p1033; + or.b32 %r9499, %r6006, %r9499; + +$L__BB1_815: + @%p898 bra $L__BB1_824; + + setp.le.u32 %p1035, %r6, %r9435; + @%p1035 bra $L__BB1_818; + + add.s32 %r6007, %r1751, %r1813; + cvt.u64.u32 %rd660, %r6007; + add.s64 %rd661, %rd660, %rd5; + shl.b64 %rd662, %rd661, 2; + add.s64 %rd663, %rd3, %rd662; + ld.global.u32 %r6008, [%rd663]; + abs.s32 %r6009, %r6008; + setp.gt.u32 %p1036, %r6009, 4; + and.b32 %r6010, %r6009, 1; + setp.eq.b32 %p1037, %r6010, 1; + and.pred %p1038, %p1036, %p1037; + selp.b32 %r6011, 256, 0, %p1038; + or.b32 %r9499, %r6011, %r9499; + +$L__BB1_818: + setp.ge.u32 %p1039, %r1748, %r6; + @%p1039 bra $L__BB1_820; + + add.s32 %r6012, %r1754, %r1813; + cvt.u64.u32 %rd664, %r6012; + add.s64 %rd665, %rd664, %rd5; + shl.b64 %rd666, %rd665, 2; + add.s64 %rd667, %rd3, %rd666; + ld.global.u32 %r6013, [%rd667]; + abs.s32 %r6014, %r6013; + setp.gt.u32 %p1040, %r6014, 4; + and.b32 %r6015, %r6014, 1; + setp.eq.b32 %p1041, %r6015, 1; + and.pred %p1042, %p1040, %p1041; + selp.b32 %r6016, 512, 0, %p1042; + or.b32 %r9499, %r6016, %r9499; + +$L__BB1_820: + setp.ge.u32 %p1043, %r1749, %r6; + @%p1043 bra $L__BB1_822; + + add.s32 %r6017, %r1753, %r1813; + cvt.u64.u32 %rd668, %r6017; + add.s64 %rd669, %rd668, %rd5; + shl.b64 %rd670, %rd669, 2; + add.s64 %rd671, %rd3, %rd670; + ld.global.u32 %r6018, [%rd671]; + abs.s32 %r6019, %r6018; + setp.gt.u32 %p1044, %r6019, 4; + and.b32 %r6020, %r6019, 1; + setp.eq.b32 %p1045, %r6020, 1; + and.pred %p1046, %p1044, %p1045; + selp.b32 %r6021, 1024, 0, %p1046; + or.b32 %r9499, %r6021, %r9499; + +$L__BB1_822: + setp.ge.u32 %p1047, %r1750, %r6; + @%p1047 bra $L__BB1_824; + + add.s32 %r6022, %r1752, %r1813; + cvt.u64.u32 %rd672, %r6022; + add.s64 %rd673, %rd672, %rd5; + shl.b64 %rd674, %rd673, 2; + add.s64 %rd675, %rd3, %rd674; + ld.global.u32 %r6023, [%rd675]; + abs.s32 %r6024, %r6023; + setp.gt.u32 %p1048, %r6024, 4; + and.b32 %r6025, %r6024, 1; + setp.eq.b32 %p1049, %r6025, 1; + and.pred %p1050, %p1048, %p1049; + selp.b32 %r6026, 2048, 0, %p1050; + or.b32 %r9499, %r6026, %r9499; + +$L__BB1_824: + @%p915 bra $L__BB1_833; + + setp.le.u32 %p1052, %r6, %r9435; + @%p1052 bra $L__BB1_827; + + add.s32 %r6027, %r1751, %r1822; + cvt.u64.u32 %rd676, %r6027; + add.s64 %rd677, %rd676, %rd5; + shl.b64 %rd678, %rd677, 2; + add.s64 %rd679, %rd3, %rd678; + ld.global.u32 %r6028, [%rd679]; + abs.s32 %r6029, %r6028; + setp.gt.u32 %p1053, %r6029, 4; + and.b32 %r6030, %r6029, 1; + setp.eq.b32 %p1054, %r6030, 1; + and.pred %p1055, %p1053, %p1054; + selp.b32 %r6031, 4096, 0, %p1055; + or.b32 %r9499, %r6031, %r9499; + +$L__BB1_827: + setp.ge.u32 %p1056, %r1748, %r6; + @%p1056 bra $L__BB1_829; + + add.s32 %r6032, %r1754, %r1822; + cvt.u64.u32 %rd680, %r6032; + add.s64 %rd681, %rd680, %rd5; + shl.b64 %rd682, %rd681, 2; + add.s64 %rd683, %rd3, %rd682; + ld.global.u32 %r6033, [%rd683]; + abs.s32 %r6034, %r6033; + setp.gt.u32 %p1057, %r6034, 4; + and.b32 %r6035, %r6034, 1; + setp.eq.b32 %p1058, %r6035, 1; + and.pred %p1059, %p1057, %p1058; + selp.b32 %r6036, 8192, 0, %p1059; + or.b32 %r9499, %r6036, %r9499; + +$L__BB1_829: + setp.ge.u32 %p1060, %r1749, %r6; + @%p1060 bra $L__BB1_831; + + add.s32 %r6037, %r1753, %r1822; + cvt.u64.u32 %rd684, %r6037; + add.s64 %rd685, %rd684, %rd5; + shl.b64 %rd686, %rd685, 2; + add.s64 %rd687, %rd3, %rd686; + ld.global.u32 %r6038, [%rd687]; + abs.s32 %r6039, %r6038; + setp.gt.u32 %p1061, %r6039, 4; + and.b32 %r6040, %r6039, 1; + setp.eq.b32 %p1062, %r6040, 1; + and.pred %p1063, %p1061, %p1062; + selp.b32 %r6041, 16384, 0, %p1063; + or.b32 %r9499, %r6041, %r9499; + +$L__BB1_831: + setp.ge.u32 %p1064, %r1750, %r6; + @%p1064 bra $L__BB1_833; + + add.s32 %r6042, %r1752, %r1822; + cvt.u64.u32 %rd688, %r6042; + add.s64 %rd689, %rd688, %rd5; + shl.b64 %rd690, %rd689, 2; + add.s64 %rd691, %rd3, %rd690; + ld.global.u32 %r6043, [%rd691]; + abs.s32 %r6044, %r6043; + setp.gt.u32 %p1065, %r6044, 4; + and.b32 %r6045, %r6044, 1; + setp.eq.b32 %p1066, %r6045, 1; + and.pred %p1067, %p1065, %p1066; + selp.b32 %r6046, 32768, 0, %p1067; + or.b32 %r9499, %r6046, %r9499; + +$L__BB1_833: + sub.s32 %r6049, %r5794, %r5; + shl.b32 %r6050, %r9499, 16; + or.b32 %r1897, %r6050, %r9483; + and.b32 %r6051, %r1831, -2004318072; + shr.u32 %r6052, %r6051, 3; + shl.b32 %r6053, %r1840, 3; + and.b32 %r6054, %r6053, -2004318072; + or.b32 %r1898, %r6054, %r6052; + not.b32 %r6055, %r1897; + setp.gt.s32 %p1068, %r6049, 0; + mov.u32 %r9515, 0; + shl.b32 %r6056, %r6049, 2; + selp.b32 %r6057, %r6056, 0, %p1068; + shr.u32 %r1899, %r1755, %r6057; + and.b32 %r1900, %r1899, %r6055; + @%p796 bra $L__BB1_842; + + setp.le.u32 %p1070, %r6, %r9435; + mov.u32 %r9515, 0; + @%p1070 bra $L__BB1_836; + + ld.global.u32 %r6059, [%rd26]; + abs.s32 %r6060, %r6059; + setp.eq.s32 %p1071, %r6060, 3; + selp.u32 %r9515, 1, 0, %p1071; + +$L__BB1_836: + setp.ge.u32 %p1072, %r1748, %r6; + @%p1072 bra $L__BB1_838; + + ld.global.u32 %r6061, [%rd27]; + abs.s32 %r6062, %r6061; + setp.eq.s32 %p1073, %r6062, 3; + selp.b32 %r6063, 2, 0, %p1073; + or.b32 %r9515, %r6063, %r9515; + +$L__BB1_838: + setp.ge.u32 %p1074, %r1749, %r6; + @%p1074 bra $L__BB1_840; + + ld.global.u32 %r6064, [%rd28]; + abs.s32 %r6065, %r6064; + setp.eq.s32 %p1075, %r6065, 3; + selp.b32 %r6066, 4, 0, %p1075; + or.b32 %r9515, %r6066, %r9515; + +$L__BB1_840: + setp.ge.u32 %p1076, %r1750, %r6; + @%p1076 bra $L__BB1_842; + + ld.global.u32 %r6067, [%rd29]; + abs.s32 %r6068, %r6067; + setp.eq.s32 %p1077, %r6068, 3; + selp.b32 %r6069, 8, 0, %p1077; + or.b32 %r9515, %r6069, %r9515; + +$L__BB1_842: + @%p813 bra $L__BB1_851; + + setp.le.u32 %p1079, %r6, %r9435; + @%p1079 bra $L__BB1_845; + + ld.global.u32 %r6070, [%rd30]; + abs.s32 %r6071, %r6070; + setp.eq.s32 %p1080, %r6071, 3; + selp.b32 %r6072, 16, 0, %p1080; + or.b32 %r9515, %r6072, %r9515; + +$L__BB1_845: + setp.ge.u32 %p1081, %r1748, %r6; + @%p1081 bra $L__BB1_847; + + ld.global.u32 %r6073, [%rd31]; + abs.s32 %r6074, %r6073; + setp.eq.s32 %p1082, %r6074, 3; + selp.b32 %r6075, 32, 0, %p1082; + or.b32 %r9515, %r6075, %r9515; + +$L__BB1_847: + setp.ge.u32 %p1083, %r1749, %r6; + @%p1083 bra $L__BB1_849; + + ld.global.u32 %r6076, [%rd32]; + abs.s32 %r6077, %r6076; + setp.eq.s32 %p1084, %r6077, 3; + selp.b32 %r6078, 64, 0, %p1084; + or.b32 %r9515, %r6078, %r9515; + +$L__BB1_849: + setp.ge.u32 %p1085, %r1750, %r6; + @%p1085 bra $L__BB1_851; + + ld.global.u32 %r6079, [%rd33]; + abs.s32 %r6080, %r6079; + setp.eq.s32 %p1086, %r6080, 3; + selp.b32 %r6081, 128, 0, %p1086; + or.b32 %r9515, %r6081, %r9515; + +$L__BB1_851: + @%p830 bra $L__BB1_860; + + setp.le.u32 %p1088, %r6, %r9435; + @%p1088 bra $L__BB1_854; + + ld.global.u32 %r6082, [%rd34]; + abs.s32 %r6083, %r6082; + setp.eq.s32 %p1089, %r6083, 3; + selp.b32 %r6084, 256, 0, %p1089; + or.b32 %r9515, %r6084, %r9515; + +$L__BB1_854: + setp.ge.u32 %p1090, %r1748, %r6; + @%p1090 bra $L__BB1_856; + + ld.global.u32 %r6085, [%rd35]; + abs.s32 %r6086, %r6085; + setp.eq.s32 %p1091, %r6086, 3; + selp.b32 %r6087, 512, 0, %p1091; + or.b32 %r9515, %r6087, %r9515; + +$L__BB1_856: + setp.ge.u32 %p1092, %r1749, %r6; + @%p1092 bra $L__BB1_858; + + ld.global.u32 %r6088, [%rd36]; + abs.s32 %r6089, %r6088; + setp.eq.s32 %p1093, %r6089, 3; + selp.b32 %r6090, 1024, 0, %p1093; + or.b32 %r9515, %r6090, %r9515; + +$L__BB1_858: + setp.ge.u32 %p1094, %r1750, %r6; + @%p1094 bra $L__BB1_860; + + ld.global.u32 %r6091, [%rd37]; + abs.s32 %r6092, %r6091; + setp.eq.s32 %p1095, %r6092, 3; + selp.b32 %r6093, 2048, 0, %p1095; + or.b32 %r9515, %r6093, %r9515; + +$L__BB1_860: + @%p847 bra $L__BB1_869; + + setp.le.u32 %p1097, %r6, %r9435; + @%p1097 bra $L__BB1_863; + + ld.global.u32 %r6094, [%rd38]; + abs.s32 %r6095, %r6094; + setp.eq.s32 %p1098, %r6095, 3; + selp.b32 %r6096, 4096, 0, %p1098; + or.b32 %r9515, %r6096, %r9515; + +$L__BB1_863: + setp.ge.u32 %p1099, %r1748, %r6; + @%p1099 bra $L__BB1_865; + + ld.global.u32 %r6097, [%rd39]; + abs.s32 %r6098, %r6097; + setp.eq.s32 %p1100, %r6098, 3; + selp.b32 %r6099, 8192, 0, %p1100; + or.b32 %r9515, %r6099, %r9515; + +$L__BB1_865: + setp.ge.u32 %p1101, %r1749, %r6; + @%p1101 bra $L__BB1_867; + + ld.global.u32 %r6100, [%rd40]; + abs.s32 %r6101, %r6100; + setp.eq.s32 %p1102, %r6101, 3; + selp.b32 %r6102, 16384, 0, %p1102; + or.b32 %r9515, %r6102, %r9515; + +$L__BB1_867: + setp.ge.u32 %p1103, %r1750, %r6; + @%p1103 bra $L__BB1_869; + + ld.global.u32 %r6103, [%rd41]; + abs.s32 %r6104, %r6103; + setp.eq.s32 %p1104, %r6104, 3; + selp.b32 %r6105, 32768, 0, %p1104; + or.b32 %r9515, %r6105, %r9515; + +$L__BB1_869: + and.b32 %r6107, %r1897, -286331154; + shr.u32 %r6108, %r6107, 1; + shl.b32 %r6109, %r1897, 1; + and.b32 %r6110, %r6109, -286331154; + or.b32 %r6111, %r1897, %r1898; + or.b32 %r6112, %r6111, %r6110; + or.b32 %r6113, %r6112, %r6108; + and.b32 %r1933, %r9515, %r1899; + shr.u32 %r6114, %r6113, 4; + shl.b32 %r6115, %r6113, 4; + shr.u32 %r6116, %r9440, 12; + or.b32 %r6117, %r6113, %r6116; + or.b32 %r6118, %r6117, %r6115; + or.b32 %r6119, %r6118, %r6114; + and.b32 %r9525, %r1900, %r6119; + setp.eq.s32 %p1105, %r9525, 0; + mov.u32 %r6106, 0; + mov.u32 %r9546, %r6106; + @%p1105 bra $L__BB1_924; + + mov.u32 %r9524, 0; + mov.u32 %r9526, %r9524; + mov.u32 %r9527, %r9545; + +$L__BB1_871: + brev.b32 %r6122, %r9525; + bfind.shiftamt.u32 %r1941, %r6122; + mov.pred %p2370, -1; + mov.u32 %r6123, 1; + shl.b32 %r1942, %r6123, %r1941; + mov.u32 %r6124, -2; + shf.l.wrap.b32 %r6125, %r6124, %r6124, %r1941; + and.b32 %r9525, %r9525, %r6125; + or.b32 %r9524, %r1942, %r9524; + and.b32 %r1945, %r1942, %r1933; + setp.ne.s32 %p1107, %r1945, 0; + selp.u32 %r6126, 1, 0, %p1107; + setp.eq.s32 %p1108, %r9543, 0; + selp.b32 %r6127, 8, 7, %p1108; + shl.b32 %r6128, %r6126, %r9541; + cvt.u16.u32 %rs772, %r6128; + or.b16 %rs1224, %rs1224, %rs772; + add.s32 %r9541, %r9541, 1; + setp.lt.u32 %p1109, %r9541, %r6127; + mov.pred %p2368, %p2370; + @%p1109 bra $L__BB1_874; + + setp.eq.s32 %p1111, %r9527, -1; + mov.u32 %r9545, -1; + mov.pred %p2368, 0; + @%p1111 bra $L__BB1_874; + + and.b16 %rs774, %rs1224, 255; + setp.eq.s16 %p1113, %rs774, 255; + selp.u32 %r9543, 1, 0, %p1113; + add.s32 %r9545, %r9527, 1; + mov.u32 %r9541, 0; + mov.u16 %rs1224, 0; + mov.pred %p2368, %p2370; + +$L__BB1_874: + mov.u32 %r9550, 0; + not.pred %p1115, %p2368; + @%p1115 bra $L__BB1_928; + + setp.eq.s32 %p1116, %r1945, 0; + @%p1116 bra $L__BB1_916; + + or.b32 %r9526, %r1942, %r9526; + mov.u32 %r9533, 51; + setp.gt.s32 %p1117, %r1941, 7; + @%p1117 bra $L__BB1_892; + + setp.gt.s32 %p1129, %r1941, 3; + @%p1129 bra $L__BB1_885; + + setp.gt.s32 %p1135, %r1941, 1; + @%p1135 bra $L__BB1_882; + + setp.eq.s32 %p1138, %r1941, 0; + @%p1138 bra $L__BB1_915; + + setp.eq.s32 %p1139, %r1941, 1; + @%p1139 bra $L__BB1_881; + bra.uni $L__BB1_914; + +$L__BB1_881: + mov.u32 %r9533, 118; + bra.uni $L__BB1_915; + +$L__BB1_892: + setp.gt.s32 %p1118, %r1941, 11; + @%p1118 bra $L__BB1_900; + + setp.gt.s32 %p1124, %r1941, 9; + @%p1124 bra $L__BB1_897; + + setp.eq.s32 %p1127, %r1941, 8; + @%p1127 bra $L__BB1_910; + + setp.eq.s32 %p1128, %r1941, 9; + @%p1128 bra $L__BB1_896; + bra.uni $L__BB1_914; + +$L__BB1_896: + mov.u32 %r9533, 30208; + bra.uni $L__BB1_915; + +$L__BB1_885: + setp.gt.s32 %p1130, %r1941, 5; + @%p1130 bra $L__BB1_889; + + setp.eq.s32 %p1133, %r1941, 4; + @%p1133 bra $L__BB1_912; + + setp.eq.s32 %p1134, %r1941, 5; + @%p1134 bra $L__BB1_888; + bra.uni $L__BB1_914; + +$L__BB1_888: + mov.u32 %r9533, 1888; + bra.uni $L__BB1_915; + +$L__BB1_900: + setp.gt.s32 %p1119, %r1941, 13; + @%p1119 bra $L__BB1_904; + + setp.eq.s32 %p1122, %r1941, 12; + @%p1122 bra $L__BB1_908; + + setp.eq.s32 %p1123, %r1941, 13; + @%p1123 bra $L__BB1_903; + bra.uni $L__BB1_914; + +$L__BB1_903: + mov.u32 %r9533, 483328; + bra.uni $L__BB1_915; + +$L__BB1_882: + setp.eq.s32 %p1136, %r1941, 2; + @%p1136 bra $L__BB1_913; + + setp.eq.s32 %p1137, %r1941, 3; + @%p1137 bra $L__BB1_884; + bra.uni $L__BB1_914; + +$L__BB1_884: + mov.u32 %r9533, 200; + bra.uni $L__BB1_915; + +$L__BB1_897: + setp.eq.s32 %p1125, %r1941, 10; + @%p1125 bra $L__BB1_909; + + setp.eq.s32 %p1126, %r1941, 11; + @%p1126 bra $L__BB1_899; + bra.uni $L__BB1_914; + +$L__BB1_899: + mov.u32 %r9533, 51200; + bra.uni $L__BB1_915; + +$L__BB1_889: + setp.eq.s32 %p1131, %r1941, 6; + @%p1131 bra $L__BB1_911; + + setp.eq.s32 %p1132, %r1941, 7; + @%p1132 bra $L__BB1_891; + bra.uni $L__BB1_914; + +$L__BB1_891: + mov.u32 %r9533, 3200; + bra.uni $L__BB1_915; + +$L__BB1_904: + setp.eq.s32 %p1120, %r1941, 14; + @%p1120 bra $L__BB1_907; + + setp.ne.s32 %p1121, %r1941, 15; + @%p1121 bra $L__BB1_914; + + mov.u32 %r9533, 819200; + bra.uni $L__BB1_915; + +$L__BB1_910: + mov.u32 %r9533, 13056; + bra.uni $L__BB1_915; + +$L__BB1_912: + mov.u32 %r9533, 816; + bra.uni $L__BB1_915; + +$L__BB1_908: + mov.u32 %r9533, 208896; + bra.uni $L__BB1_915; + +$L__BB1_913: + mov.u32 %r9533, 236; + bra.uni $L__BB1_915; + +$L__BB1_909: + mov.u32 %r9533, 60416; + bra.uni $L__BB1_915; + +$L__BB1_911: + mov.u32 %r9533, 3776; + bra.uni $L__BB1_915; + +$L__BB1_907: + mov.u32 %r9533, 966656; + bra.uni $L__BB1_915; + +$L__BB1_914: + mov.u32 %r9533, 0; + +$L__BB1_915: + not.b32 %r6149, %r9524; + and.b32 %r6150, %r1900, %r6149; + and.b32 %r6151, %r6150, %r9533; + or.b32 %r9525, %r6151, %r9525; + +$L__BB1_916: + setp.ne.s32 %p1140, %r9525, 0; + mov.u32 %r9527, %r9545; + @%p1140 bra $L__BB1_871; + + setp.eq.s32 %p1141, %r9526, 0; + mov.u32 %r9546, 0; + @%p1141 bra $L__BB1_924; + + mov.u32 %r9542, %r9545; + mov.u32 %r9539, %r9526; + +$L__BB1_919: + mov.u32 %r9545, %r9542; + setp.eq.s32 %p1142, %r9539, 0; + mov.u32 %r9546, %r9526; + @%p1142 bra $L__BB1_924; + + brev.b32 %r6153, %r9539; + bfind.shiftamt.u32 %r6154, %r6153; + mov.pred %p2370, -1; + mov.u32 %r6155, -2; + shf.l.wrap.b32 %r6156, %r6155, %r6155, %r6154; + and.b32 %r9539, %r9539, %r6156; + shr.u32 %r6157, %r6154, 2; + and.b32 %r6158, %r6154, 3; + add.s32 %r6159, %r6158, %r9435; + add.s32 %r6160, %r6157, %r9439; + mad.lo.s32 %r6161, %r6159, %r1, %r6160; + cvt.u64.u32 %rd692, %r6161; + add.s64 %rd693, %rd692, %rd5; + shl.b64 %rd694, %rd693, 2; + add.s64 %rd695, %rd3, %rd694; + ld.global.u32 %r6162, [%rd695]; + shr.u32 %r6163, %r6162, 31; + setp.eq.s32 %p1144, %r9543, 0; + selp.b32 %r6164, 8, 7, %p1144; + shl.b32 %r6165, %r6163, %r9541; + cvt.u16.u32 %rs775, %r6165; + or.b16 %rs1224, %rs1224, %rs775; + add.s32 %r9541, %r9541, 1; + setp.lt.u32 %p1145, %r9541, %r6164; + mov.pred %p2369, %p2370; + mov.u32 %r9542, %r9545; + @%p1145 bra $L__BB1_923; + + setp.eq.s32 %p1147, %r9545, -1; + mov.u32 %r9542, -1; + mov.pred %p2369, 0; + @%p1147 bra $L__BB1_923; + + and.b16 %rs777, %rs1224, 255; + setp.eq.s16 %p1149, %rs777, 255; + selp.u32 %r9543, 1, 0, %p1149; + add.s32 %r9542, %r9545, 1; + mov.u32 %r9541, 0; + mov.u16 %rs1224, 0; + mov.pred %p2369, %p2370; + +$L__BB1_923: + mov.u32 %r9550, 0; + @%p2369 bra $L__BB1_919; + bra.uni $L__BB1_928; + +$L__BB1_924: + not.b32 %r6170, %r9546; + and.b32 %r6171, %r1933, %r6170; + setp.ne.s32 %p1152, %r6171, 0; + mov.u32 %r9550, %r6106; + mov.pred %p2370, %p790; + @%p1152 bra $L__BB1_928; + + setp.lt.u32 %p1153, %r5794, %r5; + or.b32 %r6172, %r9546, %r1897; + st.local.u16 [%rd25], %r6172; + shr.u32 %r6173, %r6172, 16; + st.local.u16 [%rd25+2], %r6173; + shl.b32 %r6174, %r6172, 1; + and.b32 %r6175, %r6174, 57344; + and.b32 %r6176, %r6172, 57344; + shr.u32 %r6177, %r6176, 1; + or.b32 %r6178, %r6172, %r1898; + and.b32 %r6179, %r6178, 61440; + or.b32 %r6180, %r6179, %r6175; + or.b32 %r9440, %r6180, %r6177; + mov.u32 %r9439, %r5794; + @%p1153 bra $L__BB1_689; + +$L__BB1_926: + add.s32 %r9435, %r9435, 4; + setp.gt.u32 %p1154, %r6, %r9435; + @%p1154 bra $L__BB1_687; + + setp.eq.s32 %p1155, %r9541, 0; + add.s32 %r6181, %r9545, 1; + setp.eq.s32 %p1156, %r9545, -1; + selp.b32 %r6182, -1, %r6181, %p1156; + selp.b32 %r6183, %r9545, %r6182, %p1155; + setp.ne.s32 %p1157, %r9545, -1; + or.pred %p1158, %p1155, %p1157; + selp.b32 %r9550, %r6183, 0, %p1158; + not.pred %p2370, %p1158; + +$L__BB1_928: + @%p2370 bra $L__BB1_930; + bra.uni $L__BB1_929; + +$L__BB1_930: + mov.u32 %r6191, 2; + st.global.u32 [%rd6], %r6191; + mov.u32 %r6192, 6; + st.global.u32 [%rd6+4], %r6192; + mov.u32 %r6193, 0; + st.global.u32 [%rd6+8], %r6193; + st.global.u32 [%rd6+12], %r6193; + st.global.u32 [%rd6+16], %r6193; + st.global.u32 [%rd6+20], %r6193; + st.global.u32 [%rd6+24], %r6193; + st.global.u32 [%rd6+28], %r6193; + bra.uni $L__BB1_1905; + +$L__BB1_931: + mov.u32 %r9551, 0; + mov.u32 %r9552, %r9551; + mov.u32 %r9553, %r9551; + bra.uni $L__BB1_932; + +$L__BB1_929: + mad.lo.s32 %r6184, %r6, %r5, 7; + shr.u32 %r6185, %r6184, 3; + max.u32 %r9551, %r6185, %r9550; + add.s32 %r6186, %r8423, 6; + mul.wide.u32 %rd696, %r6186, 613566757; + shr.u64 %rd697, %rd696, 32; + cvt.u32.u64 %r6187, %rd697; + sub.s32 %r6188, %r6186, %r6187; + shr.u32 %r6189, %r6188, 1; + add.s32 %r6190, %r6189, %r6187; + shr.u32 %r9552, %r6190, 2; + add.s32 %r9553, %r9551, %r9552; + +$L__BB1_932: + add.s32 %r1986, %r9553, %r1735; + setp.gt.u32 %p1159, %r1986, %r3; + setp.lt.u32 %p1160, %r1735, 2; + or.pred %p1161, %p1160, %p1159; + @%p1161 bra $L__BB1_1247; + bra.uni $L__BB1_933; + +$L__BB1_1247: + mov.u32 %r6772, 1; + st.global.u32 [%rd6], %r6772; + mov.u32 %r6773, 4; + st.global.u32 [%rd6+4], %r6773; + mov.u32 %r6774, 0; + st.global.u32 [%rd6+8], %r6774; + st.global.u32 [%rd6+12], %r6774; + st.global.u32 [%rd6+16], %r6774; + st.global.u32 [%rd6+20], %r6774; + st.global.u32 [%rd6+24], %r6774; + st.global.u32 [%rd6+28], %r6774; + bra.uni $L__BB1_1905; + +$L__BB1_933: + setp.eq.s32 %p1162, %r8514, 0; + @%p1162 bra $L__BB1_939; + + add.s32 %r6198, %r8514, -1; + and.b32 %r9558, %r8514, 3; + setp.lt.u32 %p1163, %r6198, 3; + mov.u32 %r9556, 0; + @%p1163 bra $L__BB1_937; + + sub.s32 %r9555, %r8514, %r9558; + mov.u32 %r9556, 0; + +$L__BB1_936: + add.s32 %r6200, %r9556, 17477; + cvt.u64.u32 %rd698, %r6200; + add.s64 %rd699, %rd698, %rd4; + add.s64 %rd700, %rd1, %rd699; + ld.global.u8 %rs778, [%rd700]; + add.s32 %r6201, %r9556, %r9150; + cvt.u64.u32 %rd701, %r6201; + add.s64 %rd702, %rd701, %rd4; + add.s64 %rd703, %rd1, %rd702; + st.global.u8 [%rd703], %rs778; + ld.global.u8 %rs779, [%rd700+1]; + add.s32 %r6202, %r6201, 1; + cvt.u64.u32 %rd704, %r6202; + add.s64 %rd705, %rd704, %rd4; + add.s64 %rd706, %rd1, %rd705; + st.global.u8 [%rd706], %rs779; + ld.global.u8 %rs780, [%rd700+2]; + add.s32 %r6203, %r6201, 2; + cvt.u64.u32 %rd707, %r6203; + add.s64 %rd708, %rd707, %rd4; + add.s64 %rd709, %rd1, %rd708; + st.global.u8 [%rd709], %rs780; + add.s32 %r6204, %r9556, 17480; + cvt.u64.u32 %rd710, %r6204; + add.s64 %rd711, %rd710, %rd4; + add.s64 %rd712, %rd1, %rd711; + ld.global.u8 %rs781, [%rd712]; + add.s32 %r6205, %r6201, 3; + cvt.u64.u32 %rd713, %r6205; + add.s64 %rd714, %rd713, %rd4; + add.s64 %rd715, %rd1, %rd714; + st.global.u8 [%rd715], %rs781; + add.s32 %r9556, %r9556, 4; + add.s32 %r9555, %r9555, -4; + setp.ne.s32 %p1164, %r9555, 0; + @%p1164 bra $L__BB1_936; + +$L__BB1_937: + setp.eq.s32 %p1165, %r9558, 0; + @%p1165 bra $L__BB1_939; + +$L__BB1_938: + .pragma "nounroll"; + add.s32 %r6206, %r9556, 17477; + cvt.u64.u32 %rd716, %r6206; + add.s64 %rd717, %rd716, %rd4; + add.s64 %rd718, %rd1, %rd717; + ld.global.u8 %rs782, [%rd718]; + add.s32 %r6207, %r9556, %r9150; + cvt.u64.u32 %rd719, %r6207; + add.s64 %rd720, %rd719, %rd4; + add.s64 %rd721, %rd1, %rd720; + st.global.u8 [%rd721], %rs782; + add.s32 %r9556, %r9556, 1; + add.s32 %r9558, %r9558, -1; + setp.ne.s32 %p1166, %r9558, 0; + @%p1166 bra $L__BB1_938; + +$L__BB1_939: + setp.eq.s32 %p1167, %r8962, 0; + @%p1167 bra $L__BB1_945; + + mov.u32 %r6209, 20549; + sub.s32 %r1998, %r6209, %r8962; + and.b32 %r9563, %r8962, 3; + add.s32 %r6210, %r8962, -1; + setp.lt.u32 %p1168, %r6210, 3; + mov.u32 %r9561, 0; + @%p1168 bra $L__BB1_943; + + sub.s32 %r9560, %r8962, %r9563; + mov.u32 %r9561, 0; + +$L__BB1_942: + add.s32 %r6212, %r1998, %r9561; + cvt.u64.u32 %rd722, %r6212; + add.s64 %rd723, %rd722, %rd4; + add.s64 %rd724, %rd1, %rd723; + ld.global.u8 %rs783, [%rd724]; + add.s32 %r6213, %r9561, %r1734; + cvt.u64.u32 %rd725, %r6213; + add.s64 %rd726, %rd725, %rd4; + add.s64 %rd727, %rd1, %rd726; + st.global.u8 [%rd727], %rs783; + add.s32 %r6214, %r9561, 1; + add.s32 %r6215, %r1998, %r6214; + cvt.u64.u32 %rd728, %r6215; + add.s64 %rd729, %rd728, %rd4; + add.s64 %rd730, %rd1, %rd729; + ld.global.u8 %rs784, [%rd730]; + add.s32 %r6216, %r6214, %r1734; + cvt.u64.u32 %rd731, %r6216; + add.s64 %rd732, %rd731, %rd4; + add.s64 %rd733, %rd1, %rd732; + st.global.u8 [%rd733], %rs784; + add.s32 %r6217, %r9561, 2; + add.s32 %r6218, %r1998, %r6217; + cvt.u64.u32 %rd734, %r6218; + add.s64 %rd735, %rd734, %rd4; + add.s64 %rd736, %rd1, %rd735; + ld.global.u8 %rs785, [%rd736]; + add.s32 %r6219, %r6217, %r1734; + cvt.u64.u32 %rd737, %r6219; + add.s64 %rd738, %rd737, %rd4; + add.s64 %rd739, %rd1, %rd738; + st.global.u8 [%rd739], %rs785; + add.s32 %r6220, %r9561, 3; + add.s32 %r6221, %r1998, %r6220; + cvt.u64.u32 %rd740, %r6221; + add.s64 %rd741, %rd740, %rd4; + add.s64 %rd742, %rd1, %rd741; + ld.global.u8 %rs786, [%rd742]; + add.s32 %r6222, %r6220, %r1734; + cvt.u64.u32 %rd743, %r6222; + add.s64 %rd744, %rd743, %rd4; + add.s64 %rd745, %rd1, %rd744; + st.global.u8 [%rd745], %rs786; + add.s32 %r9561, %r9561, 4; + add.s32 %r9560, %r9560, -4; + setp.ne.s32 %p1169, %r9560, 0; + @%p1169 bra $L__BB1_942; + +$L__BB1_943: + setp.eq.s32 %p1170, %r9563, 0; + @%p1170 bra $L__BB1_945; + +$L__BB1_944: + .pragma "nounroll"; + add.s32 %r6223, %r1998, %r9561; + cvt.u64.u32 %rd746, %r6223; + add.s64 %rd747, %rd746, %rd4; + add.s64 %rd748, %rd1, %rd747; + ld.global.u8 %rs787, [%rd748]; + add.s32 %r6224, %r9561, %r1734; + cvt.u64.u32 %rd749, %r6224; + add.s64 %rd750, %rd749, %rd4; + add.s64 %rd751, %rd1, %rd750; + st.global.u8 [%rd751], %rs787; + add.s32 %r9561, %r9561, 1; + add.s32 %r9563, %r9563, -1; + setp.ne.s32 %p1171, %r9563, 0; + @%p1171 bra $L__BB1_944; + +$L__BB1_945: + add.s32 %r6225, %r8962, %r8514; + shr.u32 %r6226, %r6225, 4; + add.s32 %r6227, %r1735, -1; + cvt.u64.u32 %rd752, %r6227; + add.s64 %rd753, %rd752, %rd4; + add.s64 %rd754, %rd1, %rd753; + st.global.u8 [%rd754], %r6226; + add.s32 %r6228, %r1735, -2; + cvt.u64.u32 %rd755, %r6228; + add.s64 %rd756, %rd755, %rd4; + add.s64 %rd757, %rd1, %rd756; + ld.global.u8 %rs788, [%rd757]; + and.b16 %rs789, %rs788, 240; + cvt.u16.u32 %rs790, %r6225; + and.b16 %rs791, %rs790, 15; + or.b16 %rs792, %rs789, %rs791; + st.global.u8 [%rd757], %rs792; + setp.eq.s32 %p1172, %r9553, 0; + @%p1172 bra $L__BB1_951; + + add.s32 %r6230, %r9553, -1; + and.b32 %r9568, %r9553, 3; + setp.lt.u32 %p1173, %r6230, 3; + mov.u32 %r9566, 0; + @%p1173 bra $L__BB1_949; + + sub.s32 %r9565, %r9553, %r9568; + mov.u32 %r9566, 0; + +$L__BB1_948: + add.s32 %r6232, %r9566, %r1735; + cvt.u64.u32 %rd758, %r6232; + add.s64 %rd759, %rd758, %rd4; + add.s64 %rd760, %rd1, %rd759; + mov.u16 %rs793, 0; + st.global.u8 [%rd760], %rs793; + add.s32 %r6233, %r6232, 1; + cvt.u64.u32 %rd761, %r6233; + add.s64 %rd762, %rd761, %rd4; + add.s64 %rd763, %rd1, %rd762; + st.global.u8 [%rd763], %rs793; + add.s32 %r6234, %r6232, 2; + cvt.u64.u32 %rd764, %r6234; + add.s64 %rd765, %rd764, %rd4; + add.s64 %rd766, %rd1, %rd765; + st.global.u8 [%rd766], %rs793; + add.s32 %r6235, %r6232, 3; + cvt.u64.u32 %rd767, %r6235; + add.s64 %rd768, %rd767, %rd4; + add.s64 %rd769, %rd1, %rd768; + st.global.u8 [%rd769], %rs793; + add.s32 %r9566, %r9566, 4; + add.s32 %r9565, %r9565, -4; + setp.ne.s32 %p1174, %r9565, 0; + @%p1174 bra $L__BB1_948; + +$L__BB1_949: + setp.eq.s32 %p1175, %r9568, 0; + @%p1175 bra $L__BB1_951; + +$L__BB1_950: + .pragma "nounroll"; + add.s32 %r6236, %r9566, %r1735; + cvt.u64.u32 %rd770, %r6236; + add.s64 %rd771, %rd770, %rd4; + add.s64 %rd772, %rd1, %rd771; + mov.u16 %rs794, 0; + st.global.u8 [%rd772], %rs794; + add.s32 %r9566, %r9566, 1; + add.s32 %r9568, %r9568, -1; + setp.ne.s32 %p1176, %r9568, 0; + @%p1176 bra $L__BB1_950; + +$L__BB1_951: + setp.ne.s32 %p1177, %r4, 3; + @%p1177 bra $L__BB1_1243; + + ld.param.u64 %rd1412, [ j2k_htj2k_encode_codeblocks_param_1]; + cvt.u64.u32 %rd773, %r1735; + add.s64 %rd42, %rd773, %rd4; + add.s64 %rd43, %rd1412, %rd42; + add.s32 %r6237, %r5, 3; + shr.u32 %r6238, %r6237, 2; + add.s32 %r6239, %r6238, 8; + setp.gt.u32 %p1179, %r6239, 513; + mov.pred %p1178, -1; + mov.pred %p2373, %p1178; + @%p1179 bra $L__BB1_1202; + + mov.u16 %rs1232, 0; + st.local.u16 [%rd23], %rs1232; + st.local.u16 [%rd23+2], %rs1232; + st.local.u16 [%rd23+4], %rs1232; + st.local.u16 [%rd23+6], %rs1232; + st.local.u16 [%rd23+8], %rs1232; + st.local.u16 [%rd23+10], %rs1232; + st.local.u16 [%rd23+12], %rs1232; + st.local.u16 [%rd23+14], %rs1232; + st.local.u16 [%rd23+16], %rs1232; + st.local.u16 [%rd23+18], %rs1232; + st.local.u16 [%rd23+20], %rs1232; + st.local.u16 [%rd23+22], %rs1232; + st.local.u16 [%rd23+24], %rs1232; + st.local.u16 [%rd23+26], %rs1232; + st.local.u16 [%rd23+28], %rs1232; + st.local.u16 [%rd23+30], %rs1232; + st.local.u16 [%rd23+32], %rs1232; + st.local.u16 [%rd23+34], %rs1232; + st.local.u16 [%rd23+36], %rs1232; + st.local.u16 [%rd23+38], %rs1232; + st.local.u16 [%rd23+40], %rs1232; + st.local.u16 [%rd23+42], %rs1232; + st.local.u16 [%rd23+44], %rs1232; + st.local.u16 [%rd23+46], %rs1232; + st.local.u16 [%rd23+48], %rs1232; + st.local.u16 [%rd23+50], %rs1232; + st.local.u16 [%rd23+52], %rs1232; + st.local.u16 [%rd23+54], %rs1232; + st.local.u16 [%rd23+56], %rs1232; + st.local.u16 [%rd23+58], %rs1232; + st.local.u16 [%rd23+60], %rs1232; + st.local.u16 [%rd23+62], %rs1232; + st.local.u16 [%rd23+64], %rs1232; + st.local.u16 [%rd23+66], %rs1232; + st.local.u16 [%rd23+68], %rs1232; + st.local.u16 [%rd23+70], %rs1232; + st.local.u16 [%rd23+72], %rs1232; + st.local.u16 [%rd23+74], %rs1232; + st.local.u16 [%rd23+76], %rs1232; + st.local.u16 [%rd23+78], %rs1232; + st.local.u16 [%rd23+80], %rs1232; + st.local.u16 [%rd23+82], %rs1232; + st.local.u16 [%rd23+84], %rs1232; + st.local.u16 [%rd23+86], %rs1232; + st.local.u16 [%rd23+88], %rs1232; + st.local.u16 [%rd23+90], %rs1232; + st.local.u16 [%rd23+92], %rs1232; + st.local.u16 [%rd23+94], %rs1232; + st.local.u16 [%rd23+96], %rs1232; + st.local.u16 [%rd23+98], %rs1232; + st.local.u16 [%rd23+100], %rs1232; + st.local.u16 [%rd23+102], %rs1232; + st.local.u16 [%rd23+104], %rs1232; + st.local.u16 [%rd23+106], %rs1232; + st.local.u16 [%rd23+108], %rs1232; + st.local.u16 [%rd23+110], %rs1232; + st.local.u16 [%rd23+112], %rs1232; + st.local.u16 [%rd23+114], %rs1232; + st.local.u16 [%rd23+116], %rs1232; + st.local.u16 [%rd23+118], %rs1232; + st.local.u16 [%rd23+120], %rs1232; + st.local.u16 [%rd23+122], %rs1232; + st.local.u16 [%rd23+124], %rs1232; + st.local.u16 [%rd23+126], %rs1232; + st.local.u16 [%rd23+128], %rs1232; + st.local.u16 [%rd23+130], %rs1232; + st.local.u16 [%rd23+132], %rs1232; + st.local.u16 [%rd23+134], %rs1232; + st.local.u16 [%rd23+136], %rs1232; + st.local.u16 [%rd23+138], %rs1232; + st.local.u16 [%rd23+140], %rs1232; + st.local.u16 [%rd23+142], %rs1232; + st.local.u16 [%rd23+144], %rs1232; + st.local.u16 [%rd23+146], %rs1232; + st.local.u16 [%rd23+148], %rs1232; + st.local.u16 [%rd23+150], %rs1232; + st.local.u16 [%rd23+152], %rs1232; + st.local.u16 [%rd23+154], %rs1232; + st.local.u16 [%rd23+156], %rs1232; + st.local.u16 [%rd23+158], %rs1232; + st.local.u16 [%rd23+160], %rs1232; + st.local.u16 [%rd23+162], %rs1232; + st.local.u16 [%rd23+164], %rs1232; + st.local.u16 [%rd23+166], %rs1232; + st.local.u16 [%rd23+168], %rs1232; + st.local.u16 [%rd23+170], %rs1232; + st.local.u16 [%rd23+172], %rs1232; + st.local.u16 [%rd23+174], %rs1232; + st.local.u16 [%rd23+176], %rs1232; + st.local.u16 [%rd23+178], %rs1232; + st.local.u16 [%rd23+180], %rs1232; + st.local.u16 [%rd23+182], %rs1232; + st.local.u16 [%rd23+184], %rs1232; + st.local.u16 [%rd23+186], %rs1232; + st.local.u16 [%rd23+188], %rs1232; + st.local.u16 [%rd23+190], %rs1232; + st.local.u16 [%rd23+192], %rs1232; + st.local.u16 [%rd23+194], %rs1232; + st.local.u16 [%rd23+196], %rs1232; + st.local.u16 [%rd23+198], %rs1232; + st.local.u16 [%rd23+200], %rs1232; + st.local.u16 [%rd23+202], %rs1232; + st.local.u16 [%rd23+204], %rs1232; + st.local.u16 [%rd23+206], %rs1232; + st.local.u16 [%rd23+208], %rs1232; + st.local.u16 [%rd23+210], %rs1232; + st.local.u16 [%rd23+212], %rs1232; + st.local.u16 [%rd23+214], %rs1232; + st.local.u16 [%rd23+216], %rs1232; + st.local.u16 [%rd23+218], %rs1232; + st.local.u16 [%rd23+220], %rs1232; + st.local.u16 [%rd23+222], %rs1232; + st.local.u16 [%rd23+224], %rs1232; + st.local.u16 [%rd23+226], %rs1232; + st.local.u16 [%rd23+228], %rs1232; + st.local.u16 [%rd23+230], %rs1232; + st.local.u16 [%rd23+232], %rs1232; + st.local.u16 [%rd23+234], %rs1232; + st.local.u16 [%rd23+236], %rs1232; + st.local.u16 [%rd23+238], %rs1232; + st.local.u16 [%rd23+240], %rs1232; + st.local.u16 [%rd23+242], %rs1232; + st.local.u16 [%rd23+244], %rs1232; + st.local.u16 [%rd23+246], %rs1232; + st.local.u16 [%rd23+248], %rs1232; + st.local.u16 [%rd23+250], %rs1232; + st.local.u16 [%rd23+252], %rs1232; + st.local.u16 [%rd23+254], %rs1232; + st.local.u16 [%rd23+256], %rs1232; + st.local.u16 [%rd23+258], %rs1232; + st.local.u16 [%rd23+260], %rs1232; + st.local.u16 [%rd23+262], %rs1232; + st.local.u16 [%rd23+264], %rs1232; + st.local.u16 [%rd23+266], %rs1232; + st.local.u16 [%rd23+268], %rs1232; + st.local.u16 [%rd23+270], %rs1232; + st.local.u16 [%rd23+272], %rs1232; + st.local.u16 [%rd23+274], %rs1232; + st.local.u16 [%rd23+276], %rs1232; + st.local.u16 [%rd23+278], %rs1232; + st.local.u16 [%rd23+280], %rs1232; + st.local.u16 [%rd23+282], %rs1232; + st.local.u16 [%rd23+284], %rs1232; + st.local.u16 [%rd23+286], %rs1232; + st.local.u16 [%rd23+288], %rs1232; + st.local.u16 [%rd23+290], %rs1232; + st.local.u16 [%rd23+292], %rs1232; + st.local.u16 [%rd23+294], %rs1232; + st.local.u16 [%rd23+296], %rs1232; + st.local.u16 [%rd23+298], %rs1232; + st.local.u16 [%rd23+300], %rs1232; + st.local.u16 [%rd23+302], %rs1232; + st.local.u16 [%rd23+304], %rs1232; + st.local.u16 [%rd23+306], %rs1232; + st.local.u16 [%rd23+308], %rs1232; + st.local.u16 [%rd23+310], %rs1232; + st.local.u16 [%rd23+312], %rs1232; + st.local.u16 [%rd23+314], %rs1232; + st.local.u16 [%rd23+316], %rs1232; + st.local.u16 [%rd23+318], %rs1232; + st.local.u16 [%rd23+320], %rs1232; + st.local.u16 [%rd23+322], %rs1232; + st.local.u16 [%rd23+324], %rs1232; + st.local.u16 [%rd23+326], %rs1232; + st.local.u16 [%rd23+328], %rs1232; + st.local.u16 [%rd23+330], %rs1232; + st.local.u16 [%rd23+332], %rs1232; + st.local.u16 [%rd23+334], %rs1232; + st.local.u16 [%rd23+336], %rs1232; + st.local.u16 [%rd23+338], %rs1232; + st.local.u16 [%rd23+340], %rs1232; + st.local.u16 [%rd23+342], %rs1232; + st.local.u16 [%rd23+344], %rs1232; + st.local.u16 [%rd23+346], %rs1232; + st.local.u16 [%rd23+348], %rs1232; + st.local.u16 [%rd23+350], %rs1232; + st.local.u16 [%rd23+352], %rs1232; + st.local.u16 [%rd23+354], %rs1232; + st.local.u16 [%rd23+356], %rs1232; + st.local.u16 [%rd23+358], %rs1232; + st.local.u16 [%rd23+360], %rs1232; + st.local.u16 [%rd23+362], %rs1232; + st.local.u16 [%rd23+364], %rs1232; + st.local.u16 [%rd23+366], %rs1232; + st.local.u16 [%rd23+368], %rs1232; + st.local.u16 [%rd23+370], %rs1232; + st.local.u16 [%rd23+372], %rs1232; + st.local.u16 [%rd23+374], %rs1232; + st.local.u16 [%rd23+376], %rs1232; + st.local.u16 [%rd23+378], %rs1232; + st.local.u16 [%rd23+380], %rs1232; + st.local.u16 [%rd23+382], %rs1232; + st.local.u16 [%rd23+384], %rs1232; + st.local.u16 [%rd23+386], %rs1232; + st.local.u16 [%rd23+388], %rs1232; + st.local.u16 [%rd23+390], %rs1232; + st.local.u16 [%rd23+392], %rs1232; + st.local.u16 [%rd23+394], %rs1232; + st.local.u16 [%rd23+396], %rs1232; + st.local.u16 [%rd23+398], %rs1232; + st.local.u16 [%rd23+400], %rs1232; + st.local.u16 [%rd23+402], %rs1232; + st.local.u16 [%rd23+404], %rs1232; + st.local.u16 [%rd23+406], %rs1232; + st.local.u16 [%rd23+408], %rs1232; + st.local.u16 [%rd23+410], %rs1232; + st.local.u16 [%rd23+412], %rs1232; + st.local.u16 [%rd23+414], %rs1232; + st.local.u16 [%rd23+416], %rs1232; + st.local.u16 [%rd23+418], %rs1232; + st.local.u16 [%rd23+420], %rs1232; + st.local.u16 [%rd23+422], %rs1232; + st.local.u16 [%rd23+424], %rs1232; + st.local.u16 [%rd23+426], %rs1232; + st.local.u16 [%rd23+428], %rs1232; + st.local.u16 [%rd23+430], %rs1232; + st.local.u16 [%rd23+432], %rs1232; + st.local.u16 [%rd23+434], %rs1232; + st.local.u16 [%rd23+436], %rs1232; + st.local.u16 [%rd23+438], %rs1232; + st.local.u16 [%rd23+440], %rs1232; + st.local.u16 [%rd23+442], %rs1232; + st.local.u16 [%rd23+444], %rs1232; + st.local.u16 [%rd23+446], %rs1232; + st.local.u16 [%rd23+448], %rs1232; + st.local.u16 [%rd23+450], %rs1232; + st.local.u16 [%rd23+452], %rs1232; + st.local.u16 [%rd23+454], %rs1232; + st.local.u16 [%rd23+456], %rs1232; + st.local.u16 [%rd23+458], %rs1232; + st.local.u16 [%rd23+460], %rs1232; + st.local.u16 [%rd23+462], %rs1232; + st.local.u16 [%rd23+464], %rs1232; + st.local.u16 [%rd23+466], %rs1232; + st.local.u16 [%rd23+468], %rs1232; + st.local.u16 [%rd23+470], %rs1232; + st.local.u16 [%rd23+472], %rs1232; + st.local.u16 [%rd23+474], %rs1232; + st.local.u16 [%rd23+476], %rs1232; + st.local.u16 [%rd23+478], %rs1232; + st.local.u16 [%rd23+480], %rs1232; + st.local.u16 [%rd23+482], %rs1232; + st.local.u16 [%rd23+484], %rs1232; + st.local.u16 [%rd23+486], %rs1232; + st.local.u16 [%rd23+488], %rs1232; + st.local.u16 [%rd23+490], %rs1232; + st.local.u16 [%rd23+492], %rs1232; + st.local.u16 [%rd23+494], %rs1232; + st.local.u16 [%rd23+496], %rs1232; + st.local.u16 [%rd23+498], %rs1232; + st.local.u16 [%rd23+500], %rs1232; + st.local.u16 [%rd23+502], %rs1232; + st.local.u16 [%rd23+504], %rs1232; + st.local.u16 [%rd23+506], %rs1232; + st.local.u16 [%rd23+508], %rs1232; + st.local.u16 [%rd23+510], %rs1232; + st.local.u16 [%rd23+512], %rs1232; + st.local.u16 [%rd23+514], %rs1232; + st.local.u16 [%rd23+516], %rs1232; + st.local.u16 [%rd23+518], %rs1232; + st.local.u16 [%rd23+520], %rs1232; + st.local.u16 [%rd23+522], %rs1232; + st.local.u16 [%rd23+524], %rs1232; + st.local.u16 [%rd23+526], %rs1232; + st.local.u16 [%rd23+528], %rs1232; + st.local.u16 [%rd23+530], %rs1232; + st.local.u16 [%rd23+532], %rs1232; + st.local.u16 [%rd23+534], %rs1232; + st.local.u16 [%rd23+536], %rs1232; + st.local.u16 [%rd23+538], %rs1232; + st.local.u16 [%rd23+540], %rs1232; + st.local.u16 [%rd23+542], %rs1232; + st.local.u16 [%rd23+544], %rs1232; + st.local.u16 [%rd23+546], %rs1232; + st.local.u16 [%rd23+548], %rs1232; + st.local.u16 [%rd23+550], %rs1232; + st.local.u16 [%rd23+552], %rs1232; + st.local.u16 [%rd23+554], %rs1232; + st.local.u16 [%rd23+556], %rs1232; + st.local.u16 [%rd23+558], %rs1232; + st.local.u16 [%rd23+560], %rs1232; + st.local.u16 [%rd23+562], %rs1232; + st.local.u16 [%rd23+564], %rs1232; + st.local.u16 [%rd23+566], %rs1232; + st.local.u16 [%rd23+568], %rs1232; + st.local.u16 [%rd23+570], %rs1232; + st.local.u16 [%rd23+572], %rs1232; + st.local.u16 [%rd23+574], %rs1232; + st.local.u16 [%rd23+576], %rs1232; + st.local.u16 [%rd23+578], %rs1232; + st.local.u16 [%rd23+580], %rs1232; + st.local.u16 [%rd23+582], %rs1232; + st.local.u16 [%rd23+584], %rs1232; + st.local.u16 [%rd23+586], %rs1232; + st.local.u16 [%rd23+588], %rs1232; + st.local.u16 [%rd23+590], %rs1232; + st.local.u16 [%rd23+592], %rs1232; + st.local.u16 [%rd23+594], %rs1232; + st.local.u16 [%rd23+596], %rs1232; + st.local.u16 [%rd23+598], %rs1232; + st.local.u16 [%rd23+600], %rs1232; + st.local.u16 [%rd23+602], %rs1232; + st.local.u16 [%rd23+604], %rs1232; + st.local.u16 [%rd23+606], %rs1232; + st.local.u16 [%rd23+608], %rs1232; + st.local.u16 [%rd23+610], %rs1232; + st.local.u16 [%rd23+612], %rs1232; + st.local.u16 [%rd23+614], %rs1232; + st.local.u16 [%rd23+616], %rs1232; + st.local.u16 [%rd23+618], %rs1232; + st.local.u16 [%rd23+620], %rs1232; + st.local.u16 [%rd23+622], %rs1232; + st.local.u16 [%rd23+624], %rs1232; + st.local.u16 [%rd23+626], %rs1232; + st.local.u16 [%rd23+628], %rs1232; + st.local.u16 [%rd23+630], %rs1232; + st.local.u16 [%rd23+632], %rs1232; + st.local.u16 [%rd23+634], %rs1232; + st.local.u16 [%rd23+636], %rs1232; + st.local.u16 [%rd23+638], %rs1232; + st.local.u16 [%rd23+640], %rs1232; + st.local.u16 [%rd23+642], %rs1232; + st.local.u16 [%rd23+644], %rs1232; + st.local.u16 [%rd23+646], %rs1232; + st.local.u16 [%rd23+648], %rs1232; + st.local.u16 [%rd23+650], %rs1232; + st.local.u16 [%rd23+652], %rs1232; + st.local.u16 [%rd23+654], %rs1232; + st.local.u16 [%rd23+656], %rs1232; + st.local.u16 [%rd23+658], %rs1232; + st.local.u16 [%rd23+660], %rs1232; + st.local.u16 [%rd23+662], %rs1232; + st.local.u16 [%rd23+664], %rs1232; + st.local.u16 [%rd23+666], %rs1232; + st.local.u16 [%rd23+668], %rs1232; + st.local.u16 [%rd23+670], %rs1232; + st.local.u16 [%rd23+672], %rs1232; + st.local.u16 [%rd23+674], %rs1232; + st.local.u16 [%rd23+676], %rs1232; + st.local.u16 [%rd23+678], %rs1232; + st.local.u16 [%rd23+680], %rs1232; + st.local.u16 [%rd23+682], %rs1232; + st.local.u16 [%rd23+684], %rs1232; + st.local.u16 [%rd23+686], %rs1232; + st.local.u16 [%rd23+688], %rs1232; + st.local.u16 [%rd23+690], %rs1232; + st.local.u16 [%rd23+692], %rs1232; + st.local.u16 [%rd23+694], %rs1232; + st.local.u16 [%rd23+696], %rs1232; + st.local.u16 [%rd23+698], %rs1232; + st.local.u16 [%rd23+700], %rs1232; + st.local.u16 [%rd23+702], %rs1232; + st.local.u16 [%rd23+704], %rs1232; + st.local.u16 [%rd23+706], %rs1232; + st.local.u16 [%rd23+708], %rs1232; + st.local.u16 [%rd23+710], %rs1232; + st.local.u16 [%rd23+712], %rs1232; + st.local.u16 [%rd23+714], %rs1232; + st.local.u16 [%rd23+716], %rs1232; + st.local.u16 [%rd23+718], %rs1232; + st.local.u16 [%rd23+720], %rs1232; + st.local.u16 [%rd23+722], %rs1232; + st.local.u16 [%rd23+724], %rs1232; + st.local.u16 [%rd23+726], %rs1232; + st.local.u16 [%rd23+728], %rs1232; + st.local.u16 [%rd23+730], %rs1232; + st.local.u16 [%rd23+732], %rs1232; + st.local.u16 [%rd23+734], %rs1232; + st.local.u16 [%rd23+736], %rs1232; + st.local.u16 [%rd23+738], %rs1232; + st.local.u16 [%rd23+740], %rs1232; + st.local.u16 [%rd23+742], %rs1232; + st.local.u16 [%rd23+744], %rs1232; + st.local.u16 [%rd23+746], %rs1232; + st.local.u16 [%rd23+748], %rs1232; + st.local.u16 [%rd23+750], %rs1232; + st.local.u16 [%rd23+752], %rs1232; + st.local.u16 [%rd23+754], %rs1232; + st.local.u16 [%rd23+756], %rs1232; + st.local.u16 [%rd23+758], %rs1232; + st.local.u16 [%rd23+760], %rs1232; + st.local.u16 [%rd23+762], %rs1232; + st.local.u16 [%rd23+764], %rs1232; + st.local.u16 [%rd23+766], %rs1232; + st.local.u16 [%rd23+768], %rs1232; + st.local.u16 [%rd23+770], %rs1232; + st.local.u16 [%rd23+772], %rs1232; + st.local.u16 [%rd23+774], %rs1232; + st.local.u16 [%rd23+776], %rs1232; + st.local.u16 [%rd23+778], %rs1232; + st.local.u16 [%rd23+780], %rs1232; + st.local.u16 [%rd23+782], %rs1232; + st.local.u16 [%rd23+784], %rs1232; + st.local.u16 [%rd23+786], %rs1232; + st.local.u16 [%rd23+788], %rs1232; + st.local.u16 [%rd23+790], %rs1232; + st.local.u16 [%rd23+792], %rs1232; + st.local.u16 [%rd23+794], %rs1232; + st.local.u16 [%rd23+796], %rs1232; + st.local.u16 [%rd23+798], %rs1232; + st.local.u16 [%rd23+800], %rs1232; + st.local.u16 [%rd23+802], %rs1232; + st.local.u16 [%rd23+804], %rs1232; + st.local.u16 [%rd23+806], %rs1232; + st.local.u16 [%rd23+808], %rs1232; + st.local.u16 [%rd23+810], %rs1232; + st.local.u16 [%rd23+812], %rs1232; + st.local.u16 [%rd23+814], %rs1232; + st.local.u16 [%rd23+816], %rs1232; + st.local.u16 [%rd23+818], %rs1232; + st.local.u16 [%rd23+820], %rs1232; + st.local.u16 [%rd23+822], %rs1232; + st.local.u16 [%rd23+824], %rs1232; + st.local.u16 [%rd23+826], %rs1232; + st.local.u16 [%rd23+828], %rs1232; + st.local.u16 [%rd23+830], %rs1232; + st.local.u16 [%rd23+832], %rs1232; + st.local.u16 [%rd23+834], %rs1232; + st.local.u16 [%rd23+836], %rs1232; + st.local.u16 [%rd23+838], %rs1232; + st.local.u16 [%rd23+840], %rs1232; + st.local.u16 [%rd23+842], %rs1232; + st.local.u16 [%rd23+844], %rs1232; + st.local.u16 [%rd23+846], %rs1232; + st.local.u16 [%rd23+848], %rs1232; + st.local.u16 [%rd23+850], %rs1232; + st.local.u16 [%rd23+852], %rs1232; + st.local.u16 [%rd23+854], %rs1232; + st.local.u16 [%rd23+856], %rs1232; + st.local.u16 [%rd23+858], %rs1232; + st.local.u16 [%rd23+860], %rs1232; + st.local.u16 [%rd23+862], %rs1232; + st.local.u16 [%rd23+864], %rs1232; + st.local.u16 [%rd23+866], %rs1232; + st.local.u16 [%rd23+868], %rs1232; + st.local.u16 [%rd23+870], %rs1232; + st.local.u16 [%rd23+872], %rs1232; + st.local.u16 [%rd23+874], %rs1232; + st.local.u16 [%rd23+876], %rs1232; + st.local.u16 [%rd23+878], %rs1232; + st.local.u16 [%rd23+880], %rs1232; + st.local.u16 [%rd23+882], %rs1232; + st.local.u16 [%rd23+884], %rs1232; + st.local.u16 [%rd23+886], %rs1232; + st.local.u16 [%rd23+888], %rs1232; + st.local.u16 [%rd23+890], %rs1232; + st.local.u16 [%rd23+892], %rs1232; + st.local.u16 [%rd23+894], %rs1232; + st.local.u16 [%rd23+896], %rs1232; + st.local.u16 [%rd23+898], %rs1232; + st.local.u16 [%rd23+900], %rs1232; + st.local.u16 [%rd23+902], %rs1232; + st.local.u16 [%rd23+904], %rs1232; + st.local.u16 [%rd23+906], %rs1232; + st.local.u16 [%rd23+908], %rs1232; + st.local.u16 [%rd23+910], %rs1232; + st.local.u16 [%rd23+912], %rs1232; + st.local.u16 [%rd23+914], %rs1232; + st.local.u16 [%rd23+916], %rs1232; + st.local.u16 [%rd23+918], %rs1232; + st.local.u16 [%rd23+920], %rs1232; + st.local.u16 [%rd23+922], %rs1232; + st.local.u16 [%rd23+924], %rs1232; + st.local.u16 [%rd23+926], %rs1232; + st.local.u16 [%rd23+928], %rs1232; + st.local.u16 [%rd23+930], %rs1232; + st.local.u16 [%rd23+932], %rs1232; + st.local.u16 [%rd23+934], %rs1232; + st.local.u16 [%rd23+936], %rs1232; + st.local.u16 [%rd23+938], %rs1232; + st.local.u16 [%rd23+940], %rs1232; + st.local.u16 [%rd23+942], %rs1232; + st.local.u16 [%rd23+944], %rs1232; + st.local.u16 [%rd23+946], %rs1232; + st.local.u16 [%rd23+948], %rs1232; + st.local.u16 [%rd23+950], %rs1232; + st.local.u16 [%rd23+952], %rs1232; + st.local.u16 [%rd23+954], %rs1232; + st.local.u16 [%rd23+956], %rs1232; + st.local.u16 [%rd23+958], %rs1232; + st.local.u16 [%rd23+960], %rs1232; + st.local.u16 [%rd23+962], %rs1232; + st.local.u16 [%rd23+964], %rs1232; + st.local.u16 [%rd23+966], %rs1232; + st.local.u16 [%rd23+968], %rs1232; + st.local.u16 [%rd23+970], %rs1232; + st.local.u16 [%rd23+972], %rs1232; + st.local.u16 [%rd23+974], %rs1232; + st.local.u16 [%rd23+976], %rs1232; + st.local.u16 [%rd23+978], %rs1232; + st.local.u16 [%rd23+980], %rs1232; + st.local.u16 [%rd23+982], %rs1232; + st.local.u16 [%rd23+984], %rs1232; + st.local.u16 [%rd23+986], %rs1232; + st.local.u16 [%rd23+988], %rs1232; + st.local.u16 [%rd23+990], %rs1232; + st.local.u16 [%rd23+992], %rs1232; + st.local.u16 [%rd23+994], %rs1232; + st.local.u16 [%rd23+996], %rs1232; + st.local.u16 [%rd23+998], %rs1232; + st.local.u16 [%rd23+1000], %rs1232; + st.local.u16 [%rd23+1002], %rs1232; + st.local.u16 [%rd23+1004], %rs1232; + st.local.u16 [%rd23+1006], %rs1232; + st.local.u16 [%rd23+1008], %rs1232; + st.local.u16 [%rd23+1010], %rs1232; + st.local.u16 [%rd23+1012], %rs1232; + st.local.u16 [%rd23+1014], %rs1232; + st.local.u16 [%rd23+1016], %rs1232; + st.local.u16 [%rd23+1018], %rs1232; + st.local.u16 [%rd23+1020], %rs1232; + st.local.u16 [%rd23+1022], %rs1232; + st.local.u16 [%rd23+1024], %rs1232; + mov.u32 %r9569, 0; + mov.u32 %r9679, %r9569; + mov.u32 %r9675, %r9569; + mov.u32 %r9677, %r9569; + +$L__BB1_954: + @%p10 bra $L__BB1_1197; + + sub.s32 %r6246, %r6, %r9569; + add.s32 %r2025, %r9569, 4; + mul.lo.s32 %r2026, %r2025, %r1; + add.s32 %r2027, %r9569, 5; + add.s32 %r2028, %r2026, %r1; + add.s32 %r2029, %r9569, 6; + shl.b32 %r6247, %r1, 1; + add.s32 %r2030, %r2026, %r6247; + add.s32 %r2031, %r9569, 7; + mul.lo.s32 %r6248, %r1, 3; + add.s32 %r2032, %r2026, %r6248; + add.s32 %r2033, %r9569, 1; + add.s32 %r2034, %r9569, 2; + add.s32 %r2035, %r9569, 3; + mul.lo.s32 %r2036, %r9569, %r1; + add.s32 %r2037, %r2036, %r6248; + sub.s32 %r2038, %r2037, %r1; + sub.s32 %r2039, %r2038, %r1; + setp.lt.u32 %p1181, %r6246, 2; + selp.b32 %r6249, 4369, 13107, %p1181; + setp.lt.u32 %p1182, %r6246, 3; + selp.b32 %r6250, %r6249, 30583, %p1182; + setp.lt.u32 %p1183, %r6246, 4; + selp.b32 %r2040, %r6250, 65535, %p1183; + mov.u32 %r6245, 0; + mov.u32 %r9573, %r6245; + mov.u32 %r9574, %r6245; + +$L__BB1_956: + shr.u32 %r6252, %r9573, 2; + mul.wide.u32 %rd774, %r6252, 2; + add.s64 %rd44, %rd23, %rd774; + ld.local.u16 %rs250, [%rd44]; + ld.local.u16 %rs251, [%rd44+2]; + setp.ge.u32 %p1184, %r9573, %r5; + mov.u32 %r9585, %r6245; + @%p1184 bra $L__BB1_965; + + setp.ge.u32 %p1185, %r2025, %r6; + mov.u32 %r9585, 0; + @%p1185 bra $L__BB1_959; + + add.s32 %r6254, %r2026, %r9573; + cvt.u64.u32 %rd775, %r6254; + add.s64 %rd776, %rd775, %rd5; + shl.b64 %rd777, %rd776, 2; + add.s64 %rd778, %rd3, %rd777; + ld.global.u32 %r6255, [%rd778]; + abs.s32 %r6256, %r6255; + setp.gt.u32 %p1186, %r6256, 4; + and.b32 %r6257, %r6256, 1; + setp.eq.b32 %p1187, %r6257, 1; + and.pred %p1188, %p1186, %p1187; + selp.u32 %r9585, 1, 0, %p1188; + +$L__BB1_959: + setp.ge.u32 %p1189, %r2027, %r6; + @%p1189 bra $L__BB1_961; + + add.s32 %r6258, %r2028, %r9573; + cvt.u64.u32 %rd779, %r6258; + add.s64 %rd780, %rd779, %rd5; + shl.b64 %rd781, %rd780, 2; + add.s64 %rd782, %rd3, %rd781; + ld.global.u32 %r6259, [%rd782]; + abs.s32 %r6260, %r6259; + setp.gt.u32 %p1190, %r6260, 4; + and.b32 %r6261, %r6260, 1; + setp.eq.b32 %p1191, %r6261, 1; + and.pred %p1192, %p1190, %p1191; + selp.b32 %r6262, 2, 0, %p1192; + or.b32 %r9585, %r6262, %r9585; + +$L__BB1_961: + setp.ge.u32 %p1193, %r2029, %r6; + @%p1193 bra $L__BB1_963; + + add.s32 %r6263, %r2030, %r9573; + cvt.u64.u32 %rd783, %r6263; + add.s64 %rd784, %rd783, %rd5; + shl.b64 %rd785, %rd784, 2; + add.s64 %rd786, %rd3, %rd785; + ld.global.u32 %r6264, [%rd786]; + abs.s32 %r6265, %r6264; + setp.gt.u32 %p1194, %r6265, 4; + and.b32 %r6266, %r6265, 1; + setp.eq.b32 %p1195, %r6266, 1; + and.pred %p1196, %p1194, %p1195; + selp.b32 %r6267, 4, 0, %p1196; + or.b32 %r9585, %r6267, %r9585; + +$L__BB1_963: + setp.ge.u32 %p1197, %r2031, %r6; + @%p1197 bra $L__BB1_965; + + add.s32 %r6268, %r2032, %r9573; + cvt.u64.u32 %rd787, %r6268; + add.s64 %rd788, %rd787, %rd5; + shl.b64 %rd789, %rd788, 2; + add.s64 %rd790, %rd3, %rd789; + ld.global.u32 %r6269, [%rd790]; + abs.s32 %r6270, %r6269; + setp.gt.u32 %p1198, %r6270, 4; + and.b32 %r6271, %r6270, 1; + setp.eq.b32 %p1199, %r6271, 1; + and.pred %p1200, %p1198, %p1199; + selp.b32 %r6272, 8, 0, %p1200; + or.b32 %r9585, %r6272, %r9585; + +$L__BB1_965: + add.s32 %r2054, %r9573, 1; + setp.ge.u32 %p1201, %r2054, %r5; + @%p1201 bra $L__BB1_974; + + setp.ge.u32 %p1202, %r2025, %r6; + @%p1202 bra $L__BB1_968; + + add.s32 %r6273, %r2026, %r2054; + cvt.u64.u32 %rd791, %r6273; + add.s64 %rd792, %rd791, %rd5; + shl.b64 %rd793, %rd792, 2; + add.s64 %rd794, %rd3, %rd793; + ld.global.u32 %r6274, [%rd794]; + abs.s32 %r6275, %r6274; + setp.gt.u32 %p1203, %r6275, 4; + and.b32 %r6276, %r6275, 1; + setp.eq.b32 %p1204, %r6276, 1; + and.pred %p1205, %p1203, %p1204; + selp.b32 %r6277, 16, 0, %p1205; + or.b32 %r9585, %r6277, %r9585; + +$L__BB1_968: + setp.ge.u32 %p1206, %r2027, %r6; + @%p1206 bra $L__BB1_970; + + add.s32 %r6278, %r2028, %r2054; + cvt.u64.u32 %rd795, %r6278; + add.s64 %rd796, %rd795, %rd5; + shl.b64 %rd797, %rd796, 2; + add.s64 %rd798, %rd3, %rd797; + ld.global.u32 %r6279, [%rd798]; + abs.s32 %r6280, %r6279; + setp.gt.u32 %p1207, %r6280, 4; + and.b32 %r6281, %r6280, 1; + setp.eq.b32 %p1208, %r6281, 1; + and.pred %p1209, %p1207, %p1208; + selp.b32 %r6282, 32, 0, %p1209; + or.b32 %r9585, %r6282, %r9585; + +$L__BB1_970: + setp.ge.u32 %p1210, %r2029, %r6; + @%p1210 bra $L__BB1_972; + + add.s32 %r6283, %r2030, %r2054; + cvt.u64.u32 %rd799, %r6283; + add.s64 %rd800, %rd799, %rd5; + shl.b64 %rd801, %rd800, 2; + add.s64 %rd802, %rd3, %rd801; + ld.global.u32 %r6284, [%rd802]; + abs.s32 %r6285, %r6284; + setp.gt.u32 %p1211, %r6285, 4; + and.b32 %r6286, %r6285, 1; + setp.eq.b32 %p1212, %r6286, 1; + and.pred %p1213, %p1211, %p1212; + selp.b32 %r6287, 64, 0, %p1213; + or.b32 %r9585, %r6287, %r9585; + +$L__BB1_972: + setp.ge.u32 %p1214, %r2031, %r6; + @%p1214 bra $L__BB1_974; + + add.s32 %r6288, %r2032, %r2054; + cvt.u64.u32 %rd803, %r6288; + add.s64 %rd804, %rd803, %rd5; + shl.b64 %rd805, %rd804, 2; + add.s64 %rd806, %rd3, %rd805; + ld.global.u32 %r6289, [%rd806]; + abs.s32 %r6290, %r6289; + setp.gt.u32 %p1215, %r6290, 4; + and.b32 %r6291, %r6290, 1; + setp.eq.b32 %p1216, %r6291, 1; + and.pred %p1217, %p1215, %p1216; + selp.b32 %r6292, 128, 0, %p1217; + or.b32 %r9585, %r6292, %r9585; + +$L__BB1_974: + add.s32 %r2063, %r9573, 2; + setp.ge.u32 %p1218, %r2063, %r5; + @%p1218 bra $L__BB1_983; + + setp.ge.u32 %p1219, %r2025, %r6; + @%p1219 bra $L__BB1_977; + + add.s32 %r6293, %r2026, %r2063; + cvt.u64.u32 %rd807, %r6293; + add.s64 %rd808, %rd807, %rd5; + shl.b64 %rd809, %rd808, 2; + add.s64 %rd810, %rd3, %rd809; + ld.global.u32 %r6294, [%rd810]; + abs.s32 %r6295, %r6294; + setp.gt.u32 %p1220, %r6295, 4; + and.b32 %r6296, %r6295, 1; + setp.eq.b32 %p1221, %r6296, 1; + and.pred %p1222, %p1220, %p1221; + selp.b32 %r6297, 256, 0, %p1222; + or.b32 %r9585, %r6297, %r9585; + +$L__BB1_977: + setp.ge.u32 %p1223, %r2027, %r6; + @%p1223 bra $L__BB1_979; + + add.s32 %r6298, %r2028, %r2063; + cvt.u64.u32 %rd811, %r6298; + add.s64 %rd812, %rd811, %rd5; + shl.b64 %rd813, %rd812, 2; + add.s64 %rd814, %rd3, %rd813; + ld.global.u32 %r6299, [%rd814]; + abs.s32 %r6300, %r6299; + setp.gt.u32 %p1224, %r6300, 4; + and.b32 %r6301, %r6300, 1; + setp.eq.b32 %p1225, %r6301, 1; + and.pred %p1226, %p1224, %p1225; + selp.b32 %r6302, 512, 0, %p1226; + or.b32 %r9585, %r6302, %r9585; + +$L__BB1_979: + setp.ge.u32 %p1227, %r2029, %r6; + @%p1227 bra $L__BB1_981; + + add.s32 %r6303, %r2030, %r2063; + cvt.u64.u32 %rd815, %r6303; + add.s64 %rd816, %rd815, %rd5; + shl.b64 %rd817, %rd816, 2; + add.s64 %rd818, %rd3, %rd817; + ld.global.u32 %r6304, [%rd818]; + abs.s32 %r6305, %r6304; + setp.gt.u32 %p1228, %r6305, 4; + and.b32 %r6306, %r6305, 1; + setp.eq.b32 %p1229, %r6306, 1; + and.pred %p1230, %p1228, %p1229; + selp.b32 %r6307, 1024, 0, %p1230; + or.b32 %r9585, %r6307, %r9585; + +$L__BB1_981: + setp.ge.u32 %p1231, %r2031, %r6; + @%p1231 bra $L__BB1_983; + + add.s32 %r6308, %r2032, %r2063; + cvt.u64.u32 %rd819, %r6308; + add.s64 %rd820, %rd819, %rd5; + shl.b64 %rd821, %rd820, 2; + add.s64 %rd822, %rd3, %rd821; + ld.global.u32 %r6309, [%rd822]; + abs.s32 %r6310, %r6309; + setp.gt.u32 %p1232, %r6310, 4; + and.b32 %r6311, %r6310, 1; + setp.eq.b32 %p1233, %r6311, 1; + and.pred %p1234, %p1232, %p1233; + selp.b32 %r6312, 2048, 0, %p1234; + or.b32 %r9585, %r6312, %r9585; + +$L__BB1_983: + add.s32 %r2072, %r9573, 3; + setp.ge.u32 %p1235, %r2072, %r5; + @%p1235 bra $L__BB1_992; + + setp.ge.u32 %p1236, %r2025, %r6; + @%p1236 bra $L__BB1_986; + + add.s32 %r6313, %r2026, %r2072; + cvt.u64.u32 %rd823, %r6313; + add.s64 %rd824, %rd823, %rd5; + shl.b64 %rd825, %rd824, 2; + add.s64 %rd826, %rd3, %rd825; + ld.global.u32 %r6314, [%rd826]; + abs.s32 %r6315, %r6314; + setp.gt.u32 %p1237, %r6315, 4; + and.b32 %r6316, %r6315, 1; + setp.eq.b32 %p1238, %r6316, 1; + and.pred %p1239, %p1237, %p1238; + selp.b32 %r6317, 4096, 0, %p1239; + or.b32 %r9585, %r6317, %r9585; + +$L__BB1_986: + setp.ge.u32 %p1240, %r2027, %r6; + @%p1240 bra $L__BB1_988; + + add.s32 %r6318, %r2028, %r2072; + cvt.u64.u32 %rd827, %r6318; + add.s64 %rd828, %rd827, %rd5; + shl.b64 %rd829, %rd828, 2; + add.s64 %rd830, %rd3, %rd829; + ld.global.u32 %r6319, [%rd830]; + abs.s32 %r6320, %r6319; + setp.gt.u32 %p1241, %r6320, 4; + and.b32 %r6321, %r6320, 1; + setp.eq.b32 %p1242, %r6321, 1; + and.pred %p1243, %p1241, %p1242; + selp.b32 %r6322, 8192, 0, %p1243; + or.b32 %r9585, %r6322, %r9585; + +$L__BB1_988: + setp.ge.u32 %p1244, %r2029, %r6; + @%p1244 bra $L__BB1_990; + + add.s32 %r6323, %r2030, %r2072; + cvt.u64.u32 %rd831, %r6323; + add.s64 %rd832, %rd831, %rd5; + shl.b64 %rd833, %rd832, 2; + add.s64 %rd834, %rd3, %rd833; + ld.global.u32 %r6324, [%rd834]; + abs.s32 %r6325, %r6324; + setp.gt.u32 %p1245, %r6325, 4; + and.b32 %r6326, %r6325, 1; + setp.eq.b32 %p1246, %r6326, 1; + and.pred %p1247, %p1245, %p1246; + selp.b32 %r6327, 16384, 0, %p1247; + or.b32 %r9585, %r6327, %r9585; + +$L__BB1_990: + setp.ge.u32 %p1248, %r2031, %r6; + @%p1248 bra $L__BB1_992; + + add.s32 %r6328, %r2032, %r2072; + cvt.u64.u32 %rd835, %r6328; + add.s64 %rd836, %rd835, %rd5; + shl.b64 %rd837, %rd836, 2; + add.s64 %rd838, %rd3, %rd837; + ld.global.u32 %r6329, [%rd838]; + abs.s32 %r6330, %r6329; + setp.gt.u32 %p1249, %r6330, 4; + and.b32 %r6331, %r6330, 1; + setp.eq.b32 %p1250, %r6331, 1; + and.pred %p1251, %p1249, %p1250; + selp.b32 %r6332, 32768, 0, %p1251; + or.b32 %r9585, %r6332, %r9585; + +$L__BB1_992: + add.s32 %r6334, %r9573, 4; + setp.ge.u32 %p1252, %r6334, %r5; + mov.u32 %r9601, 0; + @%p1252 bra $L__BB1_1001; + + setp.ge.u32 %p1253, %r2025, %r6; + mov.u32 %r9601, 0; + @%p1253 bra $L__BB1_995; + + add.s32 %r6336, %r2026, %r9573; + add.s32 %r6337, %r6336, 4; + cvt.u64.u32 %rd839, %r6337; + add.s64 %rd840, %rd839, %rd5; + shl.b64 %rd841, %rd840, 2; + add.s64 %rd842, %rd3, %rd841; + ld.global.u32 %r6338, [%rd842]; + abs.s32 %r6339, %r6338; + setp.gt.u32 %p1254, %r6339, 4; + and.b32 %r6340, %r6339, 1; + setp.eq.b32 %p1255, %r6340, 1; + and.pred %p1256, %p1254, %p1255; + selp.u32 %r9601, 1, 0, %p1256; + +$L__BB1_995: + setp.ge.u32 %p1257, %r2027, %r6; + @%p1257 bra $L__BB1_997; + + add.s32 %r6341, %r2028, %r9573; + add.s32 %r6342, %r6341, 4; + cvt.u64.u32 %rd843, %r6342; + add.s64 %rd844, %rd843, %rd5; + shl.b64 %rd845, %rd844, 2; + add.s64 %rd846, %rd3, %rd845; + ld.global.u32 %r6343, [%rd846]; + abs.s32 %r6344, %r6343; + setp.gt.u32 %p1258, %r6344, 4; + and.b32 %r6345, %r6344, 1; + setp.eq.b32 %p1259, %r6345, 1; + and.pred %p1260, %p1258, %p1259; + selp.b32 %r6346, 2, 0, %p1260; + or.b32 %r9601, %r6346, %r9601; + +$L__BB1_997: + setp.ge.u32 %p1261, %r2029, %r6; + @%p1261 bra $L__BB1_999; + + add.s32 %r6347, %r2030, %r9573; + add.s32 %r6348, %r6347, 4; + cvt.u64.u32 %rd847, %r6348; + add.s64 %rd848, %rd847, %rd5; + shl.b64 %rd849, %rd848, 2; + add.s64 %rd850, %rd3, %rd849; + ld.global.u32 %r6349, [%rd850]; + abs.s32 %r6350, %r6349; + setp.gt.u32 %p1262, %r6350, 4; + and.b32 %r6351, %r6350, 1; + setp.eq.b32 %p1263, %r6351, 1; + and.pred %p1264, %p1262, %p1263; + selp.b32 %r6352, 4, 0, %p1264; + or.b32 %r9601, %r6352, %r9601; + +$L__BB1_999: + setp.ge.u32 %p1265, %r2031, %r6; + @%p1265 bra $L__BB1_1001; + + add.s32 %r6353, %r2032, %r9573; + add.s32 %r6354, %r6353, 4; + cvt.u64.u32 %rd851, %r6354; + add.s64 %rd852, %rd851, %rd5; + shl.b64 %rd853, %rd852, 2; + add.s64 %rd854, %rd3, %rd853; + ld.global.u32 %r6355, [%rd854]; + abs.s32 %r6356, %r6355; + setp.gt.u32 %p1266, %r6356, 4; + and.b32 %r6357, %r6356, 1; + setp.eq.b32 %p1267, %r6357, 1; + and.pred %p1268, %p1266, %p1267; + selp.b32 %r6358, 8, 0, %p1268; + or.b32 %r9601, %r6358, %r9601; + +$L__BB1_1001: + add.s32 %r2089, %r9573, 5; + setp.ge.u32 %p1269, %r2089, %r5; + @%p1269 bra $L__BB1_1010; + + setp.ge.u32 %p1270, %r2025, %r6; + @%p1270 bra $L__BB1_1004; + + add.s32 %r6359, %r2026, %r2089; + cvt.u64.u32 %rd855, %r6359; + add.s64 %rd856, %rd855, %rd5; + shl.b64 %rd857, %rd856, 2; + add.s64 %rd858, %rd3, %rd857; + ld.global.u32 %r6360, [%rd858]; + abs.s32 %r6361, %r6360; + setp.gt.u32 %p1271, %r6361, 4; + and.b32 %r6362, %r6361, 1; + setp.eq.b32 %p1272, %r6362, 1; + and.pred %p1273, %p1271, %p1272; + selp.b32 %r6363, 16, 0, %p1273; + or.b32 %r9601, %r6363, %r9601; + +$L__BB1_1004: + setp.ge.u32 %p1274, %r2027, %r6; + @%p1274 bra $L__BB1_1006; + + add.s32 %r6364, %r2028, %r2089; + cvt.u64.u32 %rd859, %r6364; + add.s64 %rd860, %rd859, %rd5; + shl.b64 %rd861, %rd860, 2; + add.s64 %rd862, %rd3, %rd861; + ld.global.u32 %r6365, [%rd862]; + abs.s32 %r6366, %r6365; + setp.gt.u32 %p1275, %r6366, 4; + and.b32 %r6367, %r6366, 1; + setp.eq.b32 %p1276, %r6367, 1; + and.pred %p1277, %p1275, %p1276; + selp.b32 %r6368, 32, 0, %p1277; + or.b32 %r9601, %r6368, %r9601; + +$L__BB1_1006: + setp.ge.u32 %p1278, %r2029, %r6; + @%p1278 bra $L__BB1_1008; + + add.s32 %r6369, %r2030, %r2089; + cvt.u64.u32 %rd863, %r6369; + add.s64 %rd864, %rd863, %rd5; + shl.b64 %rd865, %rd864, 2; + add.s64 %rd866, %rd3, %rd865; + ld.global.u32 %r6370, [%rd866]; + abs.s32 %r6371, %r6370; + setp.gt.u32 %p1279, %r6371, 4; + and.b32 %r6372, %r6371, 1; + setp.eq.b32 %p1280, %r6372, 1; + and.pred %p1281, %p1279, %p1280; + selp.b32 %r6373, 64, 0, %p1281; + or.b32 %r9601, %r6373, %r9601; + +$L__BB1_1008: + setp.ge.u32 %p1282, %r2031, %r6; + @%p1282 bra $L__BB1_1010; + + add.s32 %r6374, %r2032, %r2089; + cvt.u64.u32 %rd867, %r6374; + add.s64 %rd868, %rd867, %rd5; + shl.b64 %rd869, %rd868, 2; + add.s64 %rd870, %rd3, %rd869; + ld.global.u32 %r6375, [%rd870]; + abs.s32 %r6376, %r6375; + setp.gt.u32 %p1283, %r6376, 4; + and.b32 %r6377, %r6376, 1; + setp.eq.b32 %p1284, %r6377, 1; + and.pred %p1285, %p1283, %p1284; + selp.b32 %r6378, 128, 0, %p1285; + or.b32 %r9601, %r6378, %r9601; + +$L__BB1_1010: + add.s32 %r2098, %r9573, 6; + setp.ge.u32 %p1286, %r2098, %r5; + @%p1286 bra $L__BB1_1019; + + setp.ge.u32 %p1287, %r2025, %r6; + @%p1287 bra $L__BB1_1013; + + add.s32 %r6379, %r2026, %r2098; + cvt.u64.u32 %rd871, %r6379; + add.s64 %rd872, %rd871, %rd5; + shl.b64 %rd873, %rd872, 2; + add.s64 %rd874, %rd3, %rd873; + ld.global.u32 %r6380, [%rd874]; + abs.s32 %r6381, %r6380; + setp.gt.u32 %p1288, %r6381, 4; + and.b32 %r6382, %r6381, 1; + setp.eq.b32 %p1289, %r6382, 1; + and.pred %p1290, %p1288, %p1289; + selp.b32 %r6383, 256, 0, %p1290; + or.b32 %r9601, %r6383, %r9601; + +$L__BB1_1013: + setp.ge.u32 %p1291, %r2027, %r6; + @%p1291 bra $L__BB1_1015; + + add.s32 %r6384, %r2028, %r2098; + cvt.u64.u32 %rd875, %r6384; + add.s64 %rd876, %rd875, %rd5; + shl.b64 %rd877, %rd876, 2; + add.s64 %rd878, %rd3, %rd877; + ld.global.u32 %r6385, [%rd878]; + abs.s32 %r6386, %r6385; + setp.gt.u32 %p1292, %r6386, 4; + and.b32 %r6387, %r6386, 1; + setp.eq.b32 %p1293, %r6387, 1; + and.pred %p1294, %p1292, %p1293; + selp.b32 %r6388, 512, 0, %p1294; + or.b32 %r9601, %r6388, %r9601; + +$L__BB1_1015: + setp.ge.u32 %p1295, %r2029, %r6; + @%p1295 bra $L__BB1_1017; + + add.s32 %r6389, %r2030, %r2098; + cvt.u64.u32 %rd879, %r6389; + add.s64 %rd880, %rd879, %rd5; + shl.b64 %rd881, %rd880, 2; + add.s64 %rd882, %rd3, %rd881; + ld.global.u32 %r6390, [%rd882]; + abs.s32 %r6391, %r6390; + setp.gt.u32 %p1296, %r6391, 4; + and.b32 %r6392, %r6391, 1; + setp.eq.b32 %p1297, %r6392, 1; + and.pred %p1298, %p1296, %p1297; + selp.b32 %r6393, 1024, 0, %p1298; + or.b32 %r9601, %r6393, %r9601; + +$L__BB1_1017: + setp.ge.u32 %p1299, %r2031, %r6; + @%p1299 bra $L__BB1_1019; + + add.s32 %r6394, %r2032, %r2098; + cvt.u64.u32 %rd883, %r6394; + add.s64 %rd884, %rd883, %rd5; + shl.b64 %rd885, %rd884, 2; + add.s64 %rd886, %rd3, %rd885; + ld.global.u32 %r6395, [%rd886]; + abs.s32 %r6396, %r6395; + setp.gt.u32 %p1300, %r6396, 4; + and.b32 %r6397, %r6396, 1; + setp.eq.b32 %p1301, %r6397, 1; + and.pred %p1302, %p1300, %p1301; + selp.b32 %r6398, 2048, 0, %p1302; + or.b32 %r9601, %r6398, %r9601; + +$L__BB1_1019: + add.s32 %r2107, %r9573, 7; + setp.ge.u32 %p1303, %r2107, %r5; + @%p1303 bra $L__BB1_1028; + + setp.ge.u32 %p1304, %r2025, %r6; + @%p1304 bra $L__BB1_1022; + + add.s32 %r6399, %r2026, %r2107; + cvt.u64.u32 %rd887, %r6399; + add.s64 %rd888, %rd887, %rd5; + shl.b64 %rd889, %rd888, 2; + add.s64 %rd890, %rd3, %rd889; + ld.global.u32 %r6400, [%rd890]; + abs.s32 %r6401, %r6400; + setp.gt.u32 %p1305, %r6401, 4; + and.b32 %r6402, %r6401, 1; + setp.eq.b32 %p1306, %r6402, 1; + and.pred %p1307, %p1305, %p1306; + selp.b32 %r6403, 4096, 0, %p1307; + or.b32 %r9601, %r6403, %r9601; + +$L__BB1_1022: + setp.ge.u32 %p1308, %r2027, %r6; + @%p1308 bra $L__BB1_1024; + + add.s32 %r6404, %r2028, %r2107; + cvt.u64.u32 %rd891, %r6404; + add.s64 %rd892, %rd891, %rd5; + shl.b64 %rd893, %rd892, 2; + add.s64 %rd894, %rd3, %rd893; + ld.global.u32 %r6405, [%rd894]; + abs.s32 %r6406, %r6405; + setp.gt.u32 %p1309, %r6406, 4; + and.b32 %r6407, %r6406, 1; + setp.eq.b32 %p1310, %r6407, 1; + and.pred %p1311, %p1309, %p1310; + selp.b32 %r6408, 8192, 0, %p1311; + or.b32 %r9601, %r6408, %r9601; + +$L__BB1_1024: + setp.ge.u32 %p1312, %r2029, %r6; + @%p1312 bra $L__BB1_1026; + + add.s32 %r6409, %r2030, %r2107; + cvt.u64.u32 %rd895, %r6409; + add.s64 %rd896, %rd895, %rd5; + shl.b64 %rd897, %rd896, 2; + add.s64 %rd898, %rd3, %rd897; + ld.global.u32 %r6410, [%rd898]; + abs.s32 %r6411, %r6410; + setp.gt.u32 %p1313, %r6411, 4; + and.b32 %r6412, %r6411, 1; + setp.eq.b32 %p1314, %r6412, 1; + and.pred %p1315, %p1313, %p1314; + selp.b32 %r6413, 16384, 0, %p1315; + or.b32 %r9601, %r6413, %r9601; + +$L__BB1_1026: + setp.ge.u32 %p1316, %r2031, %r6; + @%p1316 bra $L__BB1_1028; + + add.s32 %r6414, %r2032, %r2107; + cvt.u64.u32 %rd899, %r6414; + add.s64 %rd900, %rd899, %rd5; + shl.b64 %rd901, %rd900, 2; + add.s64 %rd902, %rd3, %rd901; + ld.global.u32 %r6415, [%rd902]; + abs.s32 %r6416, %r6415; + setp.gt.u32 %p1317, %r6416, 4; + and.b32 %r6417, %r6416, 1; + setp.eq.b32 %p1318, %r6417, 1; + and.pred %p1319, %p1317, %p1318; + selp.b32 %r6418, 32768, 0, %p1319; + or.b32 %r9601, %r6418, %r9601; + +$L__BB1_1028: + mov.b32 %r2116, {%rs250, %rs251}; + add.s32 %r6420, %r2036, %r9573; + cvt.u64.u32 %rd903, %r6420; + add.s64 %rd904, %rd903, %rd5; + shl.b64 %rd905, %rd904, 2; + add.s64 %rd45, %rd3, %rd905; + add.s32 %r6421, %r2039, %r9573; + cvt.u64.u32 %rd906, %r6421; + add.s64 %rd907, %rd906, %rd5; + shl.b64 %rd908, %rd907, 2; + add.s64 %rd46, %rd3, %rd908; + add.s32 %r6422, %r2038, %r9573; + cvt.u64.u32 %rd909, %r6422; + add.s64 %rd910, %rd909, %rd5; + shl.b64 %rd911, %rd910, 2; + add.s64 %rd47, %rd3, %rd911; + add.s32 %r6423, %r2037, %r9573; + cvt.u64.u32 %rd912, %r6423; + add.s64 %rd913, %rd912, %rd5; + shl.b64 %rd914, %rd913, 2; + add.s64 %rd48, %rd3, %rd914; + mov.u32 %r9617, 0; + @%p1184 bra $L__BB1_1037; + + setp.le.u32 %p1321, %r6, %r9569; + mov.u32 %r9617, 0; + @%p1321 bra $L__BB1_1031; + + ld.global.u32 %r6425, [%rd45]; + abs.s32 %r6426, %r6425; + setp.gt.u32 %p1322, %r6426, 4; + and.b32 %r6427, %r6426, 1; + setp.eq.b32 %p1323, %r6427, 1; + and.pred %p1324, %p1322, %p1323; + selp.u32 %r9617, 1, 0, %p1324; + +$L__BB1_1031: + setp.ge.u32 %p1325, %r2033, %r6; + @%p1325 bra $L__BB1_1033; + + ld.global.u32 %r6428, [%rd46]; + abs.s32 %r6429, %r6428; + setp.gt.u32 %p1326, %r6429, 4; + and.b32 %r6430, %r6429, 1; + setp.eq.b32 %p1327, %r6430, 1; + and.pred %p1328, %p1326, %p1327; + selp.b32 %r6431, 2, 0, %p1328; + or.b32 %r9617, %r6431, %r9617; + +$L__BB1_1033: + setp.ge.u32 %p1329, %r2034, %r6; + @%p1329 bra $L__BB1_1035; + + ld.global.u32 %r6432, [%rd47]; + abs.s32 %r6433, %r6432; + setp.gt.u32 %p1330, %r6433, 4; + and.b32 %r6434, %r6433, 1; + setp.eq.b32 %p1331, %r6434, 1; + and.pred %p1332, %p1330, %p1331; + selp.b32 %r6435, 4, 0, %p1332; + or.b32 %r9617, %r6435, %r9617; + +$L__BB1_1035: + setp.ge.u32 %p1333, %r2035, %r6; + @%p1333 bra $L__BB1_1037; + + ld.global.u32 %r6436, [%rd48]; + abs.s32 %r6437, %r6436; + setp.gt.u32 %p1334, %r6437, 4; + and.b32 %r6438, %r6437, 1; + setp.eq.b32 %p1335, %r6438, 1; + and.pred %p1336, %p1334, %p1335; + selp.b32 %r6439, 8, 0, %p1336; + or.b32 %r9617, %r6439, %r9617; + +$L__BB1_1037: + add.s32 %r6440, %r2036, %r2054; + cvt.u64.u32 %rd915, %r6440; + add.s64 %rd916, %rd915, %rd5; + shl.b64 %rd917, %rd916, 2; + add.s64 %rd49, %rd3, %rd917; + add.s32 %r6441, %r2039, %r2054; + cvt.u64.u32 %rd918, %r6441; + add.s64 %rd919, %rd918, %rd5; + shl.b64 %rd920, %rd919, 2; + add.s64 %rd50, %rd3, %rd920; + add.s32 %r6442, %r2038, %r2054; + cvt.u64.u32 %rd921, %r6442; + add.s64 %rd922, %rd921, %rd5; + shl.b64 %rd923, %rd922, 2; + add.s64 %rd51, %rd3, %rd923; + add.s32 %r6443, %r2037, %r2054; + cvt.u64.u32 %rd924, %r6443; + add.s64 %rd925, %rd924, %rd5; + shl.b64 %rd926, %rd925, 2; + add.s64 %rd52, %rd3, %rd926; + shl.b32 %r6444, %r9601, 16; + or.b32 %r2125, %r6444, %r9585; + @%p1201 bra $L__BB1_1046; + + setp.le.u32 %p1338, %r6, %r9569; + @%p1338 bra $L__BB1_1040; + + ld.global.u32 %r6445, [%rd49]; + abs.s32 %r6446, %r6445; + setp.gt.u32 %p1339, %r6446, 4; + and.b32 %r6447, %r6446, 1; + setp.eq.b32 %p1340, %r6447, 1; + and.pred %p1341, %p1339, %p1340; + selp.b32 %r6448, 16, 0, %p1341; + or.b32 %r9617, %r6448, %r9617; + +$L__BB1_1040: + setp.ge.u32 %p1342, %r2033, %r6; + @%p1342 bra $L__BB1_1042; + + ld.global.u32 %r6449, [%rd50]; + abs.s32 %r6450, %r6449; + setp.gt.u32 %p1343, %r6450, 4; + and.b32 %r6451, %r6450, 1; + setp.eq.b32 %p1344, %r6451, 1; + and.pred %p1345, %p1343, %p1344; + selp.b32 %r6452, 32, 0, %p1345; + or.b32 %r9617, %r6452, %r9617; + +$L__BB1_1042: + setp.ge.u32 %p1346, %r2034, %r6; + @%p1346 bra $L__BB1_1044; + + ld.global.u32 %r6453, [%rd51]; + abs.s32 %r6454, %r6453; + setp.gt.u32 %p1347, %r6454, 4; + and.b32 %r6455, %r6454, 1; + setp.eq.b32 %p1348, %r6455, 1; + and.pred %p1349, %p1347, %p1348; + selp.b32 %r6456, 64, 0, %p1349; + or.b32 %r9617, %r6456, %r9617; + +$L__BB1_1044: + setp.ge.u32 %p1350, %r2035, %r6; + @%p1350 bra $L__BB1_1046; + + ld.global.u32 %r6457, [%rd52]; + abs.s32 %r6458, %r6457; + setp.gt.u32 %p1351, %r6458, 4; + and.b32 %r6459, %r6458, 1; + setp.eq.b32 %p1352, %r6459, 1; + and.pred %p1353, %p1351, %p1352; + selp.b32 %r6460, 128, 0, %p1353; + or.b32 %r9617, %r6460, %r9617; + +$L__BB1_1046: + add.s32 %r6461, %r2036, %r2063; + cvt.u64.u32 %rd927, %r6461; + add.s64 %rd928, %rd927, %rd5; + shl.b64 %rd929, %rd928, 2; + add.s64 %rd53, %rd3, %rd929; + add.s32 %r6462, %r2039, %r2063; + cvt.u64.u32 %rd930, %r6462; + add.s64 %rd931, %rd930, %rd5; + shl.b64 %rd932, %rd931, 2; + add.s64 %rd54, %rd3, %rd932; + add.s32 %r6463, %r2038, %r2063; + cvt.u64.u32 %rd933, %r6463; + add.s64 %rd934, %rd933, %rd5; + shl.b64 %rd935, %rd934, 2; + add.s64 %rd55, %rd3, %rd935; + add.s32 %r6464, %r2037, %r2063; + cvt.u64.u32 %rd936, %r6464; + add.s64 %rd937, %rd936, %rd5; + shl.b64 %rd938, %rd937, 2; + add.s64 %rd56, %rd3, %rd938; + @%p1218 bra $L__BB1_1055; + + setp.le.u32 %p1355, %r6, %r9569; + @%p1355 bra $L__BB1_1049; + + ld.global.u32 %r6465, [%rd53]; + abs.s32 %r6466, %r6465; + setp.gt.u32 %p1356, %r6466, 4; + and.b32 %r6467, %r6466, 1; + setp.eq.b32 %p1357, %r6467, 1; + and.pred %p1358, %p1356, %p1357; + selp.b32 %r6468, 256, 0, %p1358; + or.b32 %r9617, %r6468, %r9617; + +$L__BB1_1049: + setp.ge.u32 %p1359, %r2033, %r6; + @%p1359 bra $L__BB1_1051; + + ld.global.u32 %r6469, [%rd54]; + abs.s32 %r6470, %r6469; + setp.gt.u32 %p1360, %r6470, 4; + and.b32 %r6471, %r6470, 1; + setp.eq.b32 %p1361, %r6471, 1; + and.pred %p1362, %p1360, %p1361; + selp.b32 %r6472, 512, 0, %p1362; + or.b32 %r9617, %r6472, %r9617; + +$L__BB1_1051: + setp.ge.u32 %p1363, %r2034, %r6; + @%p1363 bra $L__BB1_1053; + + ld.global.u32 %r6473, [%rd55]; + abs.s32 %r6474, %r6473; + setp.gt.u32 %p1364, %r6474, 4; + and.b32 %r6475, %r6474, 1; + setp.eq.b32 %p1365, %r6475, 1; + and.pred %p1366, %p1364, %p1365; + selp.b32 %r6476, 1024, 0, %p1366; + or.b32 %r9617, %r6476, %r9617; + +$L__BB1_1053: + setp.ge.u32 %p1367, %r2035, %r6; + @%p1367 bra $L__BB1_1055; + + ld.global.u32 %r6477, [%rd56]; + abs.s32 %r6478, %r6477; + setp.gt.u32 %p1368, %r6478, 4; + and.b32 %r6479, %r6478, 1; + setp.eq.b32 %p1369, %r6479, 1; + and.pred %p1370, %p1368, %p1369; + selp.b32 %r6480, 2048, 0, %p1370; + or.b32 %r9617, %r6480, %r9617; + +$L__BB1_1055: + add.s32 %r6481, %r2036, %r2072; + cvt.u64.u32 %rd939, %r6481; + add.s64 %rd940, %rd939, %rd5; + shl.b64 %rd941, %rd940, 2; + add.s64 %rd57, %rd3, %rd941; + add.s32 %r6482, %r2039, %r2072; + cvt.u64.u32 %rd942, %r6482; + add.s64 %rd943, %rd942, %rd5; + shl.b64 %rd944, %rd943, 2; + add.s64 %rd58, %rd3, %rd944; + add.s32 %r6483, %r2038, %r2072; + cvt.u64.u32 %rd945, %r6483; + add.s64 %rd946, %rd945, %rd5; + shl.b64 %rd947, %rd946, 2; + add.s64 %rd59, %rd3, %rd947; + add.s32 %r6484, %r2037, %r2072; + cvt.u64.u32 %rd948, %r6484; + add.s64 %rd949, %rd948, %rd5; + shl.b64 %rd950, %rd949, 2; + add.s64 %rd60, %rd3, %rd950; + @%p1235 bra $L__BB1_1064; + + setp.le.u32 %p1372, %r6, %r9569; + @%p1372 bra $L__BB1_1058; + + ld.global.u32 %r6485, [%rd57]; + abs.s32 %r6486, %r6485; + setp.gt.u32 %p1373, %r6486, 4; + and.b32 %r6487, %r6486, 1; + setp.eq.b32 %p1374, %r6487, 1; + and.pred %p1375, %p1373, %p1374; + selp.b32 %r6488, 4096, 0, %p1375; + or.b32 %r9617, %r6488, %r9617; + +$L__BB1_1058: + setp.ge.u32 %p1376, %r2033, %r6; + @%p1376 bra $L__BB1_1060; + + ld.global.u32 %r6489, [%rd58]; + abs.s32 %r6490, %r6489; + setp.gt.u32 %p1377, %r6490, 4; + and.b32 %r6491, %r6490, 1; + setp.eq.b32 %p1378, %r6491, 1; + and.pred %p1379, %p1377, %p1378; + selp.b32 %r6492, 8192, 0, %p1379; + or.b32 %r9617, %r6492, %r9617; + +$L__BB1_1060: + setp.ge.u32 %p1380, %r2034, %r6; + @%p1380 bra $L__BB1_1062; + + ld.global.u32 %r6493, [%rd59]; + abs.s32 %r6494, %r6493; + setp.gt.u32 %p1381, %r6494, 4; + and.b32 %r6495, %r6494, 1; + setp.eq.b32 %p1382, %r6495, 1; + and.pred %p1383, %p1381, %p1382; + selp.b32 %r6496, 16384, 0, %p1383; + or.b32 %r9617, %r6496, %r9617; + +$L__BB1_1062: + setp.ge.u32 %p1384, %r2035, %r6; + @%p1384 bra $L__BB1_1064; + + ld.global.u32 %r6497, [%rd60]; + abs.s32 %r6498, %r6497; + setp.gt.u32 %p1385, %r6498, 4; + and.b32 %r6499, %r6498, 1; + setp.eq.b32 %p1386, %r6499, 1; + and.pred %p1387, %p1385, %p1386; + selp.b32 %r6500, 32768, 0, %p1387; + or.b32 %r9617, %r6500, %r9617; + +$L__BB1_1064: + mov.u32 %r9633, 0; + @%p1252 bra $L__BB1_1073; + + setp.le.u32 %p1389, %r6, %r9569; + mov.u32 %r9633, 0; + @%p1389 bra $L__BB1_1067; + + add.s32 %r6505, %r6420, 4; + cvt.u64.u32 %rd951, %r6505; + add.s64 %rd952, %rd951, %rd5; + shl.b64 %rd953, %rd952, 2; + add.s64 %rd954, %rd3, %rd953; + ld.global.u32 %r6506, [%rd954]; + abs.s32 %r6507, %r6506; + setp.gt.u32 %p1390, %r6507, 4; + and.b32 %r6508, %r6507, 1; + setp.eq.b32 %p1391, %r6508, 1; + and.pred %p1392, %p1390, %p1391; + selp.u32 %r9633, 1, 0, %p1392; + +$L__BB1_1067: + setp.ge.u32 %p1393, %r2033, %r6; + @%p1393 bra $L__BB1_1069; + + add.s32 %r6510, %r6421, 4; + cvt.u64.u32 %rd955, %r6510; + add.s64 %rd956, %rd955, %rd5; + shl.b64 %rd957, %rd956, 2; + add.s64 %rd958, %rd3, %rd957; + ld.global.u32 %r6511, [%rd958]; + abs.s32 %r6512, %r6511; + setp.gt.u32 %p1394, %r6512, 4; + and.b32 %r6513, %r6512, 1; + setp.eq.b32 %p1395, %r6513, 1; + and.pred %p1396, %p1394, %p1395; + selp.b32 %r6514, 2, 0, %p1396; + or.b32 %r9633, %r6514, %r9633; + +$L__BB1_1069: + setp.ge.u32 %p1397, %r2034, %r6; + @%p1397 bra $L__BB1_1071; + + add.s32 %r6516, %r6422, 4; + cvt.u64.u32 %rd959, %r6516; + add.s64 %rd960, %rd959, %rd5; + shl.b64 %rd961, %rd960, 2; + add.s64 %rd962, %rd3, %rd961; + ld.global.u32 %r6517, [%rd962]; + abs.s32 %r6518, %r6517; + setp.gt.u32 %p1398, %r6518, 4; + and.b32 %r6519, %r6518, 1; + setp.eq.b32 %p1399, %r6519, 1; + and.pred %p1400, %p1398, %p1399; + selp.b32 %r6520, 4, 0, %p1400; + or.b32 %r9633, %r6520, %r9633; + +$L__BB1_1071: + setp.ge.u32 %p1401, %r2035, %r6; + @%p1401 bra $L__BB1_1073; + + add.s32 %r6522, %r6423, 4; + cvt.u64.u32 %rd963, %r6522; + add.s64 %rd964, %rd963, %rd5; + shl.b64 %rd965, %rd964, 2; + add.s64 %rd966, %rd3, %rd965; + ld.global.u32 %r6523, [%rd966]; + abs.s32 %r6524, %r6523; + setp.gt.u32 %p1402, %r6524, 4; + and.b32 %r6525, %r6524, 1; + setp.eq.b32 %p1403, %r6525, 1; + and.pred %p1404, %p1402, %p1403; + selp.b32 %r6526, 8, 0, %p1404; + or.b32 %r9633, %r6526, %r9633; + +$L__BB1_1073: + @%p1269 bra $L__BB1_1082; + + setp.le.u32 %p1406, %r6, %r9569; + @%p1406 bra $L__BB1_1076; + + add.s32 %r6527, %r2036, %r2089; + cvt.u64.u32 %rd967, %r6527; + add.s64 %rd968, %rd967, %rd5; + shl.b64 %rd969, %rd968, 2; + add.s64 %rd970, %rd3, %rd969; + ld.global.u32 %r6528, [%rd970]; + abs.s32 %r6529, %r6528; + setp.gt.u32 %p1407, %r6529, 4; + and.b32 %r6530, %r6529, 1; + setp.eq.b32 %p1408, %r6530, 1; + and.pred %p1409, %p1407, %p1408; + selp.b32 %r6531, 16, 0, %p1409; + or.b32 %r9633, %r6531, %r9633; + +$L__BB1_1076: + setp.ge.u32 %p1410, %r2033, %r6; + @%p1410 bra $L__BB1_1078; + + add.s32 %r6532, %r2039, %r2089; + cvt.u64.u32 %rd971, %r6532; + add.s64 %rd972, %rd971, %rd5; + shl.b64 %rd973, %rd972, 2; + add.s64 %rd974, %rd3, %rd973; + ld.global.u32 %r6533, [%rd974]; + abs.s32 %r6534, %r6533; + setp.gt.u32 %p1411, %r6534, 4; + and.b32 %r6535, %r6534, 1; + setp.eq.b32 %p1412, %r6535, 1; + and.pred %p1413, %p1411, %p1412; + selp.b32 %r6536, 32, 0, %p1413; + or.b32 %r9633, %r6536, %r9633; + +$L__BB1_1078: + setp.ge.u32 %p1414, %r2034, %r6; + @%p1414 bra $L__BB1_1080; + + add.s32 %r6537, %r2038, %r2089; + cvt.u64.u32 %rd975, %r6537; + add.s64 %rd976, %rd975, %rd5; + shl.b64 %rd977, %rd976, 2; + add.s64 %rd978, %rd3, %rd977; + ld.global.u32 %r6538, [%rd978]; + abs.s32 %r6539, %r6538; + setp.gt.u32 %p1415, %r6539, 4; + and.b32 %r6540, %r6539, 1; + setp.eq.b32 %p1416, %r6540, 1; + and.pred %p1417, %p1415, %p1416; + selp.b32 %r6541, 64, 0, %p1417; + or.b32 %r9633, %r6541, %r9633; + +$L__BB1_1080: + setp.ge.u32 %p1418, %r2035, %r6; + @%p1418 bra $L__BB1_1082; + + add.s32 %r6542, %r2037, %r2089; + cvt.u64.u32 %rd979, %r6542; + add.s64 %rd980, %rd979, %rd5; + shl.b64 %rd981, %rd980, 2; + add.s64 %rd982, %rd3, %rd981; + ld.global.u32 %r6543, [%rd982]; + abs.s32 %r6544, %r6543; + setp.gt.u32 %p1419, %r6544, 4; + and.b32 %r6545, %r6544, 1; + setp.eq.b32 %p1420, %r6545, 1; + and.pred %p1421, %p1419, %p1420; + selp.b32 %r6546, 128, 0, %p1421; + or.b32 %r9633, %r6546, %r9633; + +$L__BB1_1082: + @%p1286 bra $L__BB1_1091; + + setp.le.u32 %p1423, %r6, %r9569; + @%p1423 bra $L__BB1_1085; + + add.s32 %r6547, %r2036, %r2098; + cvt.u64.u32 %rd983, %r6547; + add.s64 %rd984, %rd983, %rd5; + shl.b64 %rd985, %rd984, 2; + add.s64 %rd986, %rd3, %rd985; + ld.global.u32 %r6548, [%rd986]; + abs.s32 %r6549, %r6548; + setp.gt.u32 %p1424, %r6549, 4; + and.b32 %r6550, %r6549, 1; + setp.eq.b32 %p1425, %r6550, 1; + and.pred %p1426, %p1424, %p1425; + selp.b32 %r6551, 256, 0, %p1426; + or.b32 %r9633, %r6551, %r9633; + +$L__BB1_1085: + setp.ge.u32 %p1427, %r2033, %r6; + @%p1427 bra $L__BB1_1087; + + add.s32 %r6552, %r2039, %r2098; + cvt.u64.u32 %rd987, %r6552; + add.s64 %rd988, %rd987, %rd5; + shl.b64 %rd989, %rd988, 2; + add.s64 %rd990, %rd3, %rd989; + ld.global.u32 %r6553, [%rd990]; + abs.s32 %r6554, %r6553; + setp.gt.u32 %p1428, %r6554, 4; + and.b32 %r6555, %r6554, 1; + setp.eq.b32 %p1429, %r6555, 1; + and.pred %p1430, %p1428, %p1429; + selp.b32 %r6556, 512, 0, %p1430; + or.b32 %r9633, %r6556, %r9633; + +$L__BB1_1087: + setp.ge.u32 %p1431, %r2034, %r6; + @%p1431 bra $L__BB1_1089; + + add.s32 %r6557, %r2038, %r2098; + cvt.u64.u32 %rd991, %r6557; + add.s64 %rd992, %rd991, %rd5; + shl.b64 %rd993, %rd992, 2; + add.s64 %rd994, %rd3, %rd993; + ld.global.u32 %r6558, [%rd994]; + abs.s32 %r6559, %r6558; + setp.gt.u32 %p1432, %r6559, 4; + and.b32 %r6560, %r6559, 1; + setp.eq.b32 %p1433, %r6560, 1; + and.pred %p1434, %p1432, %p1433; + selp.b32 %r6561, 1024, 0, %p1434; + or.b32 %r9633, %r6561, %r9633; + +$L__BB1_1089: + setp.ge.u32 %p1435, %r2035, %r6; + @%p1435 bra $L__BB1_1091; + + add.s32 %r6562, %r2037, %r2098; + cvt.u64.u32 %rd995, %r6562; + add.s64 %rd996, %rd995, %rd5; + shl.b64 %rd997, %rd996, 2; + add.s64 %rd998, %rd3, %rd997; + ld.global.u32 %r6563, [%rd998]; + abs.s32 %r6564, %r6563; + setp.gt.u32 %p1436, %r6564, 4; + and.b32 %r6565, %r6564, 1; + setp.eq.b32 %p1437, %r6565, 1; + and.pred %p1438, %p1436, %p1437; + selp.b32 %r6566, 2048, 0, %p1438; + or.b32 %r9633, %r6566, %r9633; + +$L__BB1_1091: + @%p1303 bra $L__BB1_1100; + + setp.le.u32 %p1440, %r6, %r9569; + @%p1440 bra $L__BB1_1094; + + add.s32 %r6567, %r2036, %r2107; + cvt.u64.u32 %rd999, %r6567; + add.s64 %rd1000, %rd999, %rd5; + shl.b64 %rd1001, %rd1000, 2; + add.s64 %rd1002, %rd3, %rd1001; + ld.global.u32 %r6568, [%rd1002]; + abs.s32 %r6569, %r6568; + setp.gt.u32 %p1441, %r6569, 4; + and.b32 %r6570, %r6569, 1; + setp.eq.b32 %p1442, %r6570, 1; + and.pred %p1443, %p1441, %p1442; + selp.b32 %r6571, 4096, 0, %p1443; + or.b32 %r9633, %r6571, %r9633; + +$L__BB1_1094: + setp.ge.u32 %p1444, %r2033, %r6; + @%p1444 bra $L__BB1_1096; + + add.s32 %r6572, %r2039, %r2107; + cvt.u64.u32 %rd1003, %r6572; + add.s64 %rd1004, %rd1003, %rd5; + shl.b64 %rd1005, %rd1004, 2; + add.s64 %rd1006, %rd3, %rd1005; + ld.global.u32 %r6573, [%rd1006]; + abs.s32 %r6574, %r6573; + setp.gt.u32 %p1445, %r6574, 4; + and.b32 %r6575, %r6574, 1; + setp.eq.b32 %p1446, %r6575, 1; + and.pred %p1447, %p1445, %p1446; + selp.b32 %r6576, 8192, 0, %p1447; + or.b32 %r9633, %r6576, %r9633; + +$L__BB1_1096: + setp.ge.u32 %p1448, %r2034, %r6; + @%p1448 bra $L__BB1_1098; + + add.s32 %r6577, %r2038, %r2107; + cvt.u64.u32 %rd1007, %r6577; + add.s64 %rd1008, %rd1007, %rd5; + shl.b64 %rd1009, %rd1008, 2; + add.s64 %rd1010, %rd3, %rd1009; + ld.global.u32 %r6578, [%rd1010]; + abs.s32 %r6579, %r6578; + setp.gt.u32 %p1449, %r6579, 4; + and.b32 %r6580, %r6579, 1; + setp.eq.b32 %p1450, %r6580, 1; + and.pred %p1451, %p1449, %p1450; + selp.b32 %r6581, 16384, 0, %p1451; + or.b32 %r9633, %r6581, %r9633; + +$L__BB1_1098: + setp.ge.u32 %p1452, %r2035, %r6; + @%p1452 bra $L__BB1_1100; + + add.s32 %r6582, %r2037, %r2107; + cvt.u64.u32 %rd1011, %r6582; + add.s64 %rd1012, %rd1011, %rd5; + shl.b64 %rd1013, %rd1012, 2; + add.s64 %rd1014, %rd3, %rd1013; + ld.global.u32 %r6583, [%rd1014]; + abs.s32 %r6584, %r6583; + setp.gt.u32 %p1453, %r6584, 4; + and.b32 %r6585, %r6584, 1; + setp.eq.b32 %p1454, %r6585, 1; + and.pred %p1455, %p1453, %p1454; + selp.b32 %r6586, 32768, 0, %p1455; + or.b32 %r9633, %r6586, %r9633; + +$L__BB1_1100: + sub.s32 %r6589, %r6334, %r5; + shl.b32 %r6590, %r9633, 16; + or.b32 %r2182, %r6590, %r9617; + and.b32 %r6591, %r2116, -2004318072; + shr.u32 %r6592, %r6591, 3; + shl.b32 %r6593, %r2125, 3; + and.b32 %r6594, %r6593, -2004318072; + or.b32 %r2183, %r6594, %r6592; + not.b32 %r6595, %r2182; + setp.gt.s32 %p1456, %r6589, 0; + mov.u32 %r9649, 0; + shl.b32 %r6596, %r6589, 2; + selp.b32 %r6597, %r6596, 0, %p1456; + shr.u32 %r2184, %r2040, %r6597; + and.b32 %r2185, %r2184, %r6595; + @%p1184 bra $L__BB1_1109; + + setp.le.u32 %p1458, %r6, %r9569; + mov.u32 %r9649, 0; + @%p1458 bra $L__BB1_1103; + + ld.global.u32 %r6599, [%rd45]; + abs.s32 %r6600, %r6599; + setp.eq.s32 %p1459, %r6600, 3; + selp.u32 %r9649, 1, 0, %p1459; + +$L__BB1_1103: + setp.ge.u32 %p1460, %r2033, %r6; + @%p1460 bra $L__BB1_1105; + + ld.global.u32 %r6601, [%rd46]; + abs.s32 %r6602, %r6601; + setp.eq.s32 %p1461, %r6602, 3; + selp.b32 %r6603, 2, 0, %p1461; + or.b32 %r9649, %r6603, %r9649; + +$L__BB1_1105: + setp.ge.u32 %p1462, %r2034, %r6; + @%p1462 bra $L__BB1_1107; + + ld.global.u32 %r6604, [%rd47]; + abs.s32 %r6605, %r6604; + setp.eq.s32 %p1463, %r6605, 3; + selp.b32 %r6606, 4, 0, %p1463; + or.b32 %r9649, %r6606, %r9649; + +$L__BB1_1107: + setp.ge.u32 %p1464, %r2035, %r6; + @%p1464 bra $L__BB1_1109; + + ld.global.u32 %r6607, [%rd48]; + abs.s32 %r6608, %r6607; + setp.eq.s32 %p1465, %r6608, 3; + selp.b32 %r6609, 8, 0, %p1465; + or.b32 %r9649, %r6609, %r9649; + +$L__BB1_1109: + @%p1201 bra $L__BB1_1118; + + setp.le.u32 %p1467, %r6, %r9569; + @%p1467 bra $L__BB1_1112; + + ld.global.u32 %r6610, [%rd49]; + abs.s32 %r6611, %r6610; + setp.eq.s32 %p1468, %r6611, 3; + selp.b32 %r6612, 16, 0, %p1468; + or.b32 %r9649, %r6612, %r9649; + +$L__BB1_1112: + setp.ge.u32 %p1469, %r2033, %r6; + @%p1469 bra $L__BB1_1114; + + ld.global.u32 %r6613, [%rd50]; + abs.s32 %r6614, %r6613; + setp.eq.s32 %p1470, %r6614, 3; + selp.b32 %r6615, 32, 0, %p1470; + or.b32 %r9649, %r6615, %r9649; + +$L__BB1_1114: + setp.ge.u32 %p1471, %r2034, %r6; + @%p1471 bra $L__BB1_1116; + + ld.global.u32 %r6616, [%rd51]; + abs.s32 %r6617, %r6616; + setp.eq.s32 %p1472, %r6617, 3; + selp.b32 %r6618, 64, 0, %p1472; + or.b32 %r9649, %r6618, %r9649; + +$L__BB1_1116: + setp.ge.u32 %p1473, %r2035, %r6; + @%p1473 bra $L__BB1_1118; + + ld.global.u32 %r6619, [%rd52]; + abs.s32 %r6620, %r6619; + setp.eq.s32 %p1474, %r6620, 3; + selp.b32 %r6621, 128, 0, %p1474; + or.b32 %r9649, %r6621, %r9649; + +$L__BB1_1118: + @%p1218 bra $L__BB1_1127; + + setp.le.u32 %p1476, %r6, %r9569; + @%p1476 bra $L__BB1_1121; + + ld.global.u32 %r6622, [%rd53]; + abs.s32 %r6623, %r6622; + setp.eq.s32 %p1477, %r6623, 3; + selp.b32 %r6624, 256, 0, %p1477; + or.b32 %r9649, %r6624, %r9649; + +$L__BB1_1121: + setp.ge.u32 %p1478, %r2033, %r6; + @%p1478 bra $L__BB1_1123; + + ld.global.u32 %r6625, [%rd54]; + abs.s32 %r6626, %r6625; + setp.eq.s32 %p1479, %r6626, 3; + selp.b32 %r6627, 512, 0, %p1479; + or.b32 %r9649, %r6627, %r9649; + +$L__BB1_1123: + setp.ge.u32 %p1480, %r2034, %r6; + @%p1480 bra $L__BB1_1125; + + ld.global.u32 %r6628, [%rd55]; + abs.s32 %r6629, %r6628; + setp.eq.s32 %p1481, %r6629, 3; + selp.b32 %r6630, 1024, 0, %p1481; + or.b32 %r9649, %r6630, %r9649; + +$L__BB1_1125: + setp.ge.u32 %p1482, %r2035, %r6; + @%p1482 bra $L__BB1_1127; + + ld.global.u32 %r6631, [%rd56]; + abs.s32 %r6632, %r6631; + setp.eq.s32 %p1483, %r6632, 3; + selp.b32 %r6633, 2048, 0, %p1483; + or.b32 %r9649, %r6633, %r9649; + +$L__BB1_1127: + @%p1235 bra $L__BB1_1136; + + setp.le.u32 %p1485, %r6, %r9569; + @%p1485 bra $L__BB1_1130; + + ld.global.u32 %r6634, [%rd57]; + abs.s32 %r6635, %r6634; + setp.eq.s32 %p1486, %r6635, 3; + selp.b32 %r6636, 4096, 0, %p1486; + or.b32 %r9649, %r6636, %r9649; + +$L__BB1_1130: + setp.ge.u32 %p1487, %r2033, %r6; + @%p1487 bra $L__BB1_1132; + + ld.global.u32 %r6637, [%rd58]; + abs.s32 %r6638, %r6637; + setp.eq.s32 %p1488, %r6638, 3; + selp.b32 %r6639, 8192, 0, %p1488; + or.b32 %r9649, %r6639, %r9649; + +$L__BB1_1132: + setp.ge.u32 %p1489, %r2034, %r6; + @%p1489 bra $L__BB1_1134; + + ld.global.u32 %r6640, [%rd59]; + abs.s32 %r6641, %r6640; + setp.eq.s32 %p1490, %r6641, 3; + selp.b32 %r6642, 16384, 0, %p1490; + or.b32 %r9649, %r6642, %r9649; + +$L__BB1_1134: + setp.ge.u32 %p1491, %r2035, %r6; + @%p1491 bra $L__BB1_1136; + + ld.global.u32 %r6643, [%rd60]; + abs.s32 %r6644, %r6643; + setp.eq.s32 %p1492, %r6644, 3; + selp.b32 %r6645, 32768, 0, %p1492; + or.b32 %r9649, %r6645, %r9649; + +$L__BB1_1136: + and.b32 %r6647, %r2182, -286331154; + shr.u32 %r6648, %r6647, 1; + shl.b32 %r6649, %r2182, 1; + and.b32 %r6650, %r6649, -286331154; + or.b32 %r6651, %r2182, %r2183; + or.b32 %r6652, %r6651, %r6650; + or.b32 %r6653, %r6652, %r6648; + and.b32 %r2218, %r9649, %r2184; + shr.u32 %r6654, %r6653, 4; + shl.b32 %r6655, %r6653, 4; + shr.u32 %r6656, %r9574, 12; + or.b32 %r6657, %r6653, %r6656; + or.b32 %r6658, %r6657, %r6655; + or.b32 %r6659, %r6658, %r6654; + and.b32 %r9659, %r2185, %r6659; + setp.eq.s32 %p1493, %r9659, 0; + mov.u32 %r9680, 0; + @%p1493 bra $L__BB1_1195; + + mov.u32 %r9658, 0; + mov.u32 %r9660, %r9658; + +$L__BB1_1138: + brev.b32 %r6662, %r9659; + bfind.shiftamt.u32 %r2226, %r6662; + mov.pred %p2373, -1; + mov.u32 %r6663, 1; + shl.b32 %r2227, %r6663, %r2226; + mov.u32 %r6664, -2; + shf.l.wrap.b32 %r6665, %r6664, %r6664, %r2226; + and.b32 %r9659, %r9659, %r6665; + or.b32 %r9658, %r2227, %r9658; + and.b32 %r2230, %r2227, %r2218; + setp.ne.s32 %p1495, %r2230, 0; + selp.u32 %r6666, 1, 0, %p1495; + setp.eq.s32 %p1496, %r9677, 0; + selp.b32 %r6667, 8, 7, %p1496; + shl.b32 %r6668, %r6666, %r9675; + cvt.u16.u32 %rs796, %r6668; + or.b16 %rs1232, %rs1232, %rs796; + add.s32 %r9675, %r9675, 1; + setp.lt.u32 %p1497, %r9675, %r6667; + mov.pred %p2371, %p2373; + @%p1497 bra $L__BB1_1143; + + setp.ge.u32 %p1499, %r9679, %r9551; + mov.pred %p2371, 0; + @%p1499 bra $L__BB1_1143; + + setp.eq.s64 %p1500, %rd43, 0; + @%p1500 bra $L__BB1_1142; + + cvt.u64.u32 %rd1015, %r9679; + add.s64 %rd1016, %rd42, %rd1015; + add.s64 %rd1017, %rd1, %rd1016; + st.global.u8 [%rd1017], %rs1232; + +$L__BB1_1142: + and.b16 %rs798, %rs1232, 255; + setp.eq.s16 %p1502, %rs798, 255; + selp.u32 %r9677, 1, 0, %p1502; + add.s32 %r9679, %r9679, 1; + mov.u32 %r9675, 0; + mov.u16 %rs1232, 0; + mov.pred %p2371, %p2373; + +$L__BB1_1143: + not.pred %p1504, %p2371; + @%p1504 bra $L__BB1_1202; + + setp.eq.s32 %p1505, %r2230, 0; + @%p1505 bra $L__BB1_1185; + + or.b32 %r9660, %r2227, %r9660; + mov.u32 %r9667, 51; + setp.gt.s32 %p1506, %r2226, 7; + @%p1506 bra $L__BB1_1161; + + setp.gt.s32 %p1518, %r2226, 3; + @%p1518 bra $L__BB1_1154; + + setp.gt.s32 %p1524, %r2226, 1; + @%p1524 bra $L__BB1_1151; + + setp.eq.s32 %p1527, %r2226, 0; + @%p1527 bra $L__BB1_1184; + + setp.eq.s32 %p1528, %r2226, 1; + @%p1528 bra $L__BB1_1150; + bra.uni $L__BB1_1183; + +$L__BB1_1150: + mov.u32 %r9667, 118; + bra.uni $L__BB1_1184; + +$L__BB1_1161: + setp.gt.s32 %p1507, %r2226, 11; + @%p1507 bra $L__BB1_1169; + + setp.gt.s32 %p1513, %r2226, 9; + @%p1513 bra $L__BB1_1166; + + setp.eq.s32 %p1516, %r2226, 8; + @%p1516 bra $L__BB1_1179; + + setp.eq.s32 %p1517, %r2226, 9; + @%p1517 bra $L__BB1_1165; + bra.uni $L__BB1_1183; + +$L__BB1_1165: + mov.u32 %r9667, 30208; + bra.uni $L__BB1_1184; + +$L__BB1_1154: + setp.gt.s32 %p1519, %r2226, 5; + @%p1519 bra $L__BB1_1158; + + setp.eq.s32 %p1522, %r2226, 4; + @%p1522 bra $L__BB1_1181; + + setp.eq.s32 %p1523, %r2226, 5; + @%p1523 bra $L__BB1_1157; + bra.uni $L__BB1_1183; + +$L__BB1_1157: + mov.u32 %r9667, 1888; + bra.uni $L__BB1_1184; + +$L__BB1_1169: + setp.gt.s32 %p1508, %r2226, 13; + @%p1508 bra $L__BB1_1173; + + setp.eq.s32 %p1511, %r2226, 12; + @%p1511 bra $L__BB1_1177; + + setp.eq.s32 %p1512, %r2226, 13; + @%p1512 bra $L__BB1_1172; + bra.uni $L__BB1_1183; + +$L__BB1_1172: + mov.u32 %r9667, 483328; + bra.uni $L__BB1_1184; + +$L__BB1_1151: + setp.eq.s32 %p1525, %r2226, 2; + @%p1525 bra $L__BB1_1182; + + setp.eq.s32 %p1526, %r2226, 3; + @%p1526 bra $L__BB1_1153; + bra.uni $L__BB1_1183; + +$L__BB1_1153: + mov.u32 %r9667, 200; + bra.uni $L__BB1_1184; + +$L__BB1_1166: + setp.eq.s32 %p1514, %r2226, 10; + @%p1514 bra $L__BB1_1178; + + setp.eq.s32 %p1515, %r2226, 11; + @%p1515 bra $L__BB1_1168; + bra.uni $L__BB1_1183; + +$L__BB1_1168: + mov.u32 %r9667, 51200; + bra.uni $L__BB1_1184; + +$L__BB1_1158: + setp.eq.s32 %p1520, %r2226, 6; + @%p1520 bra $L__BB1_1180; + + setp.eq.s32 %p1521, %r2226, 7; + @%p1521 bra $L__BB1_1160; + bra.uni $L__BB1_1183; + +$L__BB1_1160: + mov.u32 %r9667, 3200; + bra.uni $L__BB1_1184; + +$L__BB1_1173: + setp.eq.s32 %p1509, %r2226, 14; + @%p1509 bra $L__BB1_1176; + + setp.ne.s32 %p1510, %r2226, 15; + @%p1510 bra $L__BB1_1183; + + mov.u32 %r9667, 819200; + bra.uni $L__BB1_1184; + +$L__BB1_1179: + mov.u32 %r9667, 13056; + bra.uni $L__BB1_1184; + +$L__BB1_1181: + mov.u32 %r9667, 816; + bra.uni $L__BB1_1184; + +$L__BB1_1177: + mov.u32 %r9667, 208896; + bra.uni $L__BB1_1184; + +$L__BB1_1182: + mov.u32 %r9667, 236; + bra.uni $L__BB1_1184; + +$L__BB1_1178: + mov.u32 %r9667, 60416; + bra.uni $L__BB1_1184; + +$L__BB1_1180: + mov.u32 %r9667, 3776; + bra.uni $L__BB1_1184; + +$L__BB1_1176: + mov.u32 %r9667, 966656; + bra.uni $L__BB1_1184; + +$L__BB1_1183: + mov.u32 %r9667, 0; + +$L__BB1_1184: + not.b32 %r6687, %r9658; + and.b32 %r6688, %r2185, %r6687; + and.b32 %r6689, %r6688, %r9667; + or.b32 %r9659, %r6689, %r9659; + +$L__BB1_1185: + setp.ne.s32 %p1529, %r9659, 0; + @%p1529 bra $L__BB1_1138; + + setp.eq.s32 %p1530, %r9660, 0; + mov.u32 %r9680, 0; + @%p1530 bra $L__BB1_1195; + + mov.u32 %r9673, %r9660; + +$L__BB1_1188: + setp.eq.s32 %p1531, %r9673, 0; + mov.u32 %r9680, %r9660; + @%p1531 bra $L__BB1_1195; + + brev.b32 %r6691, %r9673; + bfind.shiftamt.u32 %r6692, %r6691; + mov.pred %p2373, -1; + mov.u32 %r6693, -2; + shf.l.wrap.b32 %r6694, %r6693, %r6693, %r6692; + and.b32 %r9673, %r9673, %r6694; + shr.u32 %r6695, %r6692, 2; + and.b32 %r6696, %r6692, 3; + add.s32 %r6697, %r6696, %r9569; + add.s32 %r6698, %r6695, %r9573; + mad.lo.s32 %r6699, %r6697, %r1, %r6698; + cvt.u64.u32 %rd1018, %r6699; + add.s64 %rd1019, %rd1018, %rd5; + shl.b64 %rd1020, %rd1019, 2; + add.s64 %rd1021, %rd3, %rd1020; + ld.global.u32 %r6700, [%rd1021]; + shr.u32 %r6701, %r6700, 31; + setp.eq.s32 %p1533, %r9677, 0; + selp.b32 %r6702, 8, 7, %p1533; + shl.b32 %r6703, %r6701, %r9675; + cvt.u16.u32 %rs799, %r6703; + or.b16 %rs1232, %rs1232, %rs799; + add.s32 %r9675, %r9675, 1; + setp.lt.u32 %p1534, %r9675, %r6702; + mov.pred %p2372, %p2373; + @%p1534 bra $L__BB1_1194; + + setp.ge.u32 %p1536, %r9679, %r9551; + mov.pred %p2372, 0; + @%p1536 bra $L__BB1_1194; + + setp.eq.s64 %p1537, %rd43, 0; + @%p1537 bra $L__BB1_1193; + + cvt.u64.u32 %rd1022, %r9679; + add.s64 %rd1023, %rd42, %rd1022; + add.s64 %rd1024, %rd1, %rd1023; + st.global.u8 [%rd1024], %rs1232; + +$L__BB1_1193: + and.b16 %rs801, %rs1232, 255; + setp.eq.s16 %p1539, %rs801, 255; + selp.u32 %r9677, 1, 0, %p1539; + add.s32 %r9679, %r9679, 1; + mov.u32 %r9675, 0; + mov.u16 %rs1232, 0; + mov.pred %p2372, %p2373; + +$L__BB1_1194: + @%p2372 bra $L__BB1_1188; + bra.uni $L__BB1_1202; + +$L__BB1_1195: + not.b32 %r6705, %r9680; + and.b32 %r6706, %r2218, %r6705; + setp.ne.s32 %p1542, %r6706, 0; + mov.pred %p2373, %p1178; + @%p1542 bra $L__BB1_1202; + + setp.lt.u32 %p1543, %r6334, %r5; + or.b32 %r6707, %r9680, %r2182; + st.local.u16 [%rd44], %r6707; + shr.u32 %r6708, %r6707, 16; + st.local.u16 [%rd44+2], %r6708; + shl.b32 %r6709, %r6707, 1; + and.b32 %r6710, %r6709, 57344; + and.b32 %r6711, %r6707, 57344; + shr.u32 %r6712, %r6711, 1; + or.b32 %r6713, %r6707, %r2183; + and.b32 %r6714, %r6713, 61440; + or.b32 %r6715, %r6714, %r6710; + or.b32 %r9574, %r6715, %r6712; + mov.u32 %r9573, %r6334; + @%p1543 bra $L__BB1_956; + +$L__BB1_1197: + add.s32 %r9569, %r9569, 4; + setp.gt.u32 %p1544, %r6, %r9569; + @%p1544 bra $L__BB1_954; + + setp.eq.s32 %p1546, %r9675, 0; + mov.pred %p1545, 0; + mov.pred %p2373, %p1545; + @%p1546 bra $L__BB1_1202; + + setp.ge.u32 %p1548, %r9679, %r9551; + mov.pred %p2373, %p1178; + @%p1548 bra $L__BB1_1202; + + setp.eq.s64 %p1550, %rd43, 0; + mov.pred %p2373, %p1545; + @%p1550 bra $L__BB1_1202; + + cvt.u64.u32 %rd1025, %r9679; + add.s64 %rd1026, %rd42, %rd1025; + add.s64 %rd1027, %rd1, %rd1026; + st.global.u8 [%rd1027], %rs1232; + mov.pred %p2373, %p1545; + +$L__BB1_1202: + @%p2373 bra $L__BB1_1246; + bra.uni $L__BB1_1203; + +$L__BB1_1246: + mov.u32 %r6769, 2; + st.global.u32 [%rd6], %r6769; + mov.u32 %r6770, 6; + st.global.u32 [%rd6+4], %r6770; + mov.u32 %r6771, 0; + st.global.u32 [%rd6+8], %r6771; + st.global.u32 [%rd6+12], %r6771; + st.global.u32 [%rd6+16], %r6771; + st.global.u32 [%rd6+20], %r6771; + st.global.u32 [%rd6+24], %r6771; + st.global.u32 [%rd6+28], %r6771; + bra.uni $L__BB1_1905; + +$L__BB1_1203: + cvt.u64.u32 %rd1028, %r9551; + add.s64 %rd61, %rd42, %rd1028; + setp.eq.s32 %p1552, %r9552, 0; + @%p1552 bra $L__BB1_1244; + + add.s32 %r6717, %r9552, -1; + and.b32 %r9688, %r9552, 3; + setp.lt.u32 %p1553, %r6717, 3; + mov.u32 %r9686, 0; + @%p1553 bra $L__BB1_1207; + + sub.s32 %r9685, %r9552, %r9688; + mov.u32 %r9686, 0; + +$L__BB1_1206: + cvt.u64.u32 %rd1029, %r9686; + add.s64 %rd1030, %rd61, %rd1029; + add.s64 %rd1031, %rd1, %rd1030; + mov.u16 %rs802, 0; + st.global.u8 [%rd1031], %rs802; + st.global.u8 [%rd1031+1], %rs802; + st.global.u8 [%rd1031+2], %rs802; + st.global.u8 [%rd1031+3], %rs802; + add.s32 %r9686, %r9686, 4; + add.s32 %r9685, %r9685, -4; + setp.ne.s32 %p1554, %r9685, 0; + @%p1554 bra $L__BB1_1206; + +$L__BB1_1207: + setp.eq.s32 %p1555, %r9688, 0; + @%p1555 bra $L__BB1_1209; + +$L__BB1_1208: + .pragma "nounroll"; + cvt.u64.u32 %rd1032, %r9686; + add.s64 %rd1033, %rd61, %rd1032; + add.s64 %rd1034, %rd1, %rd1033; + mov.u16 %rs803, 0; + st.global.u8 [%rd1034], %rs803; + add.s32 %r9686, %r9686, 1; + add.s32 %r9688, %r9688, -1; + setp.ne.s32 %p1556, %r9688, 0; + @%p1556 bra $L__BB1_1208; + +$L__BB1_1209: + @%p10 bra $L__BB1_1237; + + mov.u32 %r6723, 0; + mov.u32 %r9715, 1; + mov.u16 %rs1239, 0; + mov.u32 %r9689, %r6723; + mov.u32 %r9714, %r6723; + mov.u32 %r9713, %r6723; + mov.u32 %r9712, %r6723; + +$L__BB1_1211: + mul.lo.s32 %r2279, %r9689, %r1; + add.s32 %r2280, %r9689, 3; + mul.lo.s32 %r2281, %r1, %r2280; + add.s32 %r2282, %r9689, 2; + mul.lo.s32 %r2283, %r1, %r2282; + add.s32 %r2284, %r9689, 1; + mul.lo.s32 %r2285, %r1, %r2284; + mov.u32 %r9694, %r6723; + +$L__BB1_1212: + add.s32 %r9702, %r2281, %r9694; + add.s32 %r9701, %r2283, %r9694; + add.s32 %r9700, %r2285, %r9694; + add.s32 %r9699, %r2279, %r9694; + mov.u32 %r9703, 0; + +$L__BB1_1213: + add.s32 %r6726, %r9694, %r9703; + setp.ge.u32 %p1558, %r6726, %r5; + @%p1558 bra $L__BB1_1234; + + setp.ge.u32 %p1559, %r9689, %r6; + @%p1559 bra $L__BB1_1219; + + cvt.u64.u32 %rd1035, %r9699; + add.s64 %rd1036, %rd1035, %rd5; + shl.b64 %rd1037, %rd1036, 2; + add.s64 %rd1038, %rd3, %rd1037; + ld.global.u32 %r6727, [%rd1038]; + abs.s32 %r2304, %r6727; + setp.lt.u32 %p1560, %r2304, 5; + and.b32 %r6728, %r2304, 1; + setp.eq.b32 %p1561, %r6728, 1; + not.pred %p1562, %p1561; + or.pred %p1563, %p1560, %p1562; + @%p1563 bra $L__BB1_1219; + + shr.u32 %r6729, %r2304, 1; + and.b32 %r6730, %r6729, 1; + shl.b32 %r6731, %r6730, %r9712; + cvt.u16.u32 %rs805, %r6731; + or.b16 %rs1239, %rs1239, %rs805; + add.s32 %r9714, %r9714, 1; + add.s32 %r9712, %r9712, 1; + setp.ne.s32 %p1564, %r9712, 7; + setp.eq.s32 %p1565, %r9715, 0; + or.pred %p1566, %p1564, %p1565; + and.b16 %rs806, %rs1239, 127; + setp.ne.s16 %p1567, %rs806, 127; + or.pred %p1568, %p1566, %p1567; + setp.ne.s32 %p1569, %r9712, 8; + and.pred %p1570, %p1569, %p1568; + @%p1570 bra $L__BB1_1219; + + setp.ge.u32 %p1571, %r9713, %r9552; + @%p1571 bra $L__BB1_1245; + + not.b32 %r6733, %r9713; + add.s32 %r6734, %r9552, %r6733; + cvt.u64.u32 %rd1039, %r6734; + add.s64 %rd1040, %rd61, %rd1039; + add.s64 %rd1041, %rd1, %rd1040; + and.b16 %rs808, %rs1239, 255; + st.global.u8 [%rd1041], %rs1239; + add.s32 %r9713, %r9713, 1; + setp.gt.u16 %p1572, %rs808, 143; + selp.u32 %r9715, 1, 0, %p1572; + mov.u16 %rs1239, 0; + mov.u32 %r9712, 0; + +$L__BB1_1219: + setp.ge.u32 %p1573, %r2284, %r6; + @%p1573 bra $L__BB1_1224; + + cvt.u64.u32 %rd1042, %r9700; + add.s64 %rd1043, %rd1042, %rd5; + shl.b64 %rd1044, %rd1043, 2; + add.s64 %rd1045, %rd3, %rd1044; + ld.global.u32 %r6735, [%rd1045]; + abs.s32 %r2313, %r6735; + setp.lt.u32 %p1574, %r2313, 5; + and.b32 %r6736, %r2313, 1; + setp.eq.b32 %p1575, %r6736, 1; + not.pred %p1576, %p1575; + or.pred %p1577, %p1574, %p1576; + @%p1577 bra $L__BB1_1224; + + shr.u32 %r6737, %r2313, 1; + and.b32 %r6738, %r6737, 1; + shl.b32 %r6739, %r6738, %r9712; + cvt.u16.u32 %rs809, %r6739; + or.b16 %rs1239, %rs1239, %rs809; + add.s32 %r9714, %r9714, 1; + add.s32 %r9712, %r9712, 1; + setp.ne.s32 %p1578, %r9712, 7; + setp.eq.s32 %p1579, %r9715, 0; + or.pred %p1580, %p1578, %p1579; + and.b16 %rs810, %rs1239, 127; + setp.ne.s16 %p1581, %rs810, 127; + or.pred %p1582, %p1580, %p1581; + setp.ne.s32 %p1583, %r9712, 8; + and.pred %p1584, %p1583, %p1582; + @%p1584 bra $L__BB1_1224; + + setp.ge.u32 %p1585, %r9713, %r9552; + @%p1585 bra $L__BB1_1245; + + not.b32 %r6741, %r9713; + add.s32 %r6742, %r9552, %r6741; + cvt.u64.u32 %rd1046, %r6742; + add.s64 %rd1047, %rd61, %rd1046; + add.s64 %rd1048, %rd1, %rd1047; + and.b16 %rs812, %rs1239, 255; + st.global.u8 [%rd1048], %rs1239; + add.s32 %r9713, %r9713, 1; + setp.gt.u16 %p1586, %rs812, 143; + selp.u32 %r9715, 1, 0, %p1586; + mov.u16 %rs1239, 0; + mov.u32 %r9712, 0; + +$L__BB1_1224: + setp.ge.u32 %p1587, %r2282, %r6; + @%p1587 bra $L__BB1_1229; + + cvt.u64.u32 %rd1049, %r9701; + add.s64 %rd1050, %rd1049, %rd5; + shl.b64 %rd1051, %rd1050, 2; + add.s64 %rd1052, %rd3, %rd1051; + ld.global.u32 %r6743, [%rd1052]; + abs.s32 %r2322, %r6743; + setp.lt.u32 %p1588, %r2322, 5; + and.b32 %r6744, %r2322, 1; + setp.eq.b32 %p1589, %r6744, 1; + not.pred %p1590, %p1589; + or.pred %p1591, %p1588, %p1590; + @%p1591 bra $L__BB1_1229; + + shr.u32 %r6745, %r2322, 1; + and.b32 %r6746, %r6745, 1; + shl.b32 %r6747, %r6746, %r9712; + cvt.u16.u32 %rs813, %r6747; + or.b16 %rs1239, %rs1239, %rs813; + add.s32 %r9714, %r9714, 1; + add.s32 %r9712, %r9712, 1; + setp.ne.s32 %p1592, %r9712, 7; + setp.eq.s32 %p1593, %r9715, 0; + or.pred %p1594, %p1592, %p1593; + and.b16 %rs814, %rs1239, 127; + setp.ne.s16 %p1595, %rs814, 127; + or.pred %p1596, %p1594, %p1595; + setp.ne.s32 %p1597, %r9712, 8; + and.pred %p1598, %p1597, %p1596; + @%p1598 bra $L__BB1_1229; + + setp.ge.u32 %p1599, %r9713, %r9552; + @%p1599 bra $L__BB1_1245; + + not.b32 %r6749, %r9713; + add.s32 %r6750, %r9552, %r6749; + cvt.u64.u32 %rd1053, %r6750; + add.s64 %rd1054, %rd61, %rd1053; + add.s64 %rd1055, %rd1, %rd1054; + and.b16 %rs816, %rs1239, 255; + st.global.u8 [%rd1055], %rs1239; + add.s32 %r9713, %r9713, 1; + setp.gt.u16 %p1600, %rs816, 143; + selp.u32 %r9715, 1, 0, %p1600; + mov.u16 %rs1239, 0; + mov.u32 %r9712, 0; + +$L__BB1_1229: + setp.ge.u32 %p1601, %r2280, %r6; + @%p1601 bra $L__BB1_1234; + + cvt.u64.u32 %rd1056, %r9702; + add.s64 %rd1057, %rd1056, %rd5; + shl.b64 %rd1058, %rd1057, 2; + add.s64 %rd1059, %rd3, %rd1058; + ld.global.u32 %r6751, [%rd1059]; + abs.s32 %r2331, %r6751; + setp.lt.u32 %p1602, %r2331, 5; + and.b32 %r6752, %r2331, 1; + setp.eq.b32 %p1603, %r6752, 1; + not.pred %p1604, %p1603; + or.pred %p1605, %p1602, %p1604; + @%p1605 bra $L__BB1_1234; + + shr.u32 %r6753, %r2331, 1; + and.b32 %r6754, %r6753, 1; + shl.b32 %r6755, %r6754, %r9712; + cvt.u16.u32 %rs817, %r6755; + or.b16 %rs1239, %rs1239, %rs817; + add.s32 %r9714, %r9714, 1; + add.s32 %r9712, %r9712, 1; + setp.ne.s32 %p1606, %r9712, 7; + setp.eq.s32 %p1607, %r9715, 0; + or.pred %p1608, %p1606, %p1607; + and.b16 %rs818, %rs1239, 127; + setp.ne.s16 %p1609, %rs818, 127; + or.pred %p1610, %p1608, %p1609; + setp.ne.s32 %p1611, %r9712, 8; + and.pred %p1612, %p1611, %p1610; + @%p1612 bra $L__BB1_1234; + + setp.ge.u32 %p1613, %r9713, %r9552; + @%p1613 bra $L__BB1_1245; + + not.b32 %r6757, %r9713; + add.s32 %r6758, %r9552, %r6757; + cvt.u64.u32 %rd1060, %r6758; + add.s64 %rd1061, %rd61, %rd1060; + add.s64 %rd1062, %rd1, %rd1061; + and.b16 %rs820, %rs1239, 255; + st.global.u8 [%rd1062], %rs1239; + add.s32 %r9713, %r9713, 1; + setp.gt.u16 %p1614, %rs820, 143; + selp.u32 %r9715, 1, 0, %p1614; + mov.u16 %rs1239, 0; + mov.u32 %r9712, 0; + +$L__BB1_1234: + add.s32 %r9702, %r9702, 1; + add.s32 %r9701, %r9701, 1; + add.s32 %r9700, %r9700, 1; + add.s32 %r9699, %r9699, 1; + add.s32 %r9703, %r9703, 1; + setp.lt.u32 %p1615, %r9703, 8; + @%p1615 bra $L__BB1_1213; + + add.s32 %r9694, %r9694, 8; + setp.lt.u32 %p1616, %r9694, %r5; + @%p1616 bra $L__BB1_1212; + + add.s32 %r9689, %r9689, 4; + setp.lt.u32 %p1617, %r9689, %r6; + @%p1617 bra $L__BB1_1211; + bra.uni $L__BB1_1239; + +$L__BB1_1244: + setp.eq.s32 %p1624, %r8423, 0; + @%p1624 bra $L__BB1_1243; + bra.uni $L__BB1_1245; + +$L__BB1_1237: + mov.u32 %r9712, 0; + mov.u32 %r9724, %r9712; + +$L__BB1_1238: + add.s32 %r9724, %r9724, 4; + setp.lt.u32 %p1618, %r9724, %r6; + mov.u16 %rs1239, 0; + mov.u32 %r9713, %r9712; + mov.u32 %r9714, %r9712; + @%p1618 bra $L__BB1_1238; + +$L__BB1_1239: + setp.eq.s32 %p1619, %r9712, 0; + @%p1619 bra $L__BB1_1242; + + setp.ge.u32 %p1620, %r9713, %r9552; + @%p1620 bra $L__BB1_1245; + + not.b32 %r6763, %r9713; + add.s32 %r6764, %r9552, %r6763; + cvt.u64.u32 %rd1063, %r6764; + add.s64 %rd1064, %rd61, %rd1063; + add.s64 %rd1065, %rd1, %rd1064; + st.global.u8 [%rd1065], %rs1239; + add.s32 %r9713, %r9713, 1; + +$L__BB1_1242: + setp.le.u32 %p1621, %r9713, %r9552; + setp.eq.s32 %p1622, %r9714, %r8423; + and.pred %p1623, %p1622, %p1621; + @%p1623 bra $L__BB1_1243; + bra.uni $L__BB1_1245; + +$L__BB1_1243: + mov.u32 %r6768, 0; + st.global.u32 [%rd6], %r6768; + st.global.u32 [%rd6+4], %r6768; + st.global.u32 [%rd6+8], %r1986; + st.global.u32 [%rd6+12], %r4; + st.global.u32 [%rd6+16], %r42; + st.global.u32 [%rd6+20], %r1735; + st.global.u32 [%rd6+24], %r9553; + st.global.u32 [%rd6+28], %r6768; + bra.uni $L__BB1_1905; + +$L__BB1_1245: + mov.u32 %r6765, 2; + st.global.u32 [%rd6], %r6765; + mov.u32 %r6766, 7; + st.global.u32 [%rd6+4], %r6766; + mov.u32 %r6767, 0; + st.global.u32 [%rd6+8], %r6767; + st.global.u32 [%rd6+12], %r6767; + st.global.u32 [%rd6+16], %r6767; + st.global.u32 [%rd6+20], %r6767; + st.global.u32 [%rd6+24], %r6767; + st.global.u32 [%rd6+28], %r6767; + bra.uni $L__BB1_1905; + +} + // .globl j2k_htj2k_encode_codeblocks_multi_input +.visible .entry j2k_htj2k_encode_codeblocks_multi_input( + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_param_0, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_param_1, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_param_2, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_param_3, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_param_4, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_param_5, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_param_6 +) +.maxntid 128, 1, 1 +{ + .local .align 2 .b8 __local_depot2[1026]; + .reg .b64 %SP; + .reg .b64 %SPL; + .reg .pred %p<2374>; + .reg .b16 %rs<1376>; + .reg .b32 %r<10754>; + .reg .b64 %rd<1420>; + // demoted variable + .shared .align 4 .b8 _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE9block_max[512]; + // demoted variable + .shared .align 1 .b8 _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE13cleanup_e_val[513]; + // demoted variable + .shared .align 1 .b8 _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val[513]; + + mov.u64 %SPL, __local_depot2; + ld.param.u64 %rd77, [ j2k_htj2k_encode_codeblocks_multi_input_param_0]; + ld.param.u64 %rd83, [ j2k_htj2k_encode_codeblocks_multi_input_param_6]; + cvta.to.global.u64 %rd1, %rd77; + mov.u32 %r4053, %ctaid.x; + cvt.u64.u32 %rd2, %r4053; + setp.ge.u64 %p9, %rd2, %rd83; + @%p9 bra $L__BB2_1905; + + ld.param.u64 %rd1419, [ j2k_htj2k_encode_codeblocks_multi_input_param_1]; + cvta.to.global.u64 %rd84, %rd1419; + mul.lo.s64 %rd85, %rd2, 40; + add.s64 %rd86, %rd84, %rd85; + ld.global.u64 %rd87, [%rd86]; + cvta.to.global.u64 %rd3, %rd87; + ld.global.v2.u32 {%r4054, %r4055}, [%rd86+8]; + ld.global.v2.u32 {%r4057, %r4058}, [%rd86+16]; + ld.global.v2.u32 {%r4059, %r4060}, [%rd86+24]; + ld.global.v2.u32 {%r4061, %r4062}, [%rd86+32]; + cvt.u64.u32 %rd4, %r4054; + setp.eq.s32 %p10, %r4057, 0; + setp.eq.s32 %p11, %r4058, 0; + or.pred %p12, %p10, %p11; + @%p12 bra $L__BB2_14; + bra.uni $L__BB2_2; + +$L__BB2_14: + mov.u32 %r8425, 0; + bra.uni $L__BB2_15; + +$L__BB2_2: + mul.lo.s32 %r8, %r4058, %r4057; + setp.eq.s32 %p13, %r4055, %r4057; + @%p13 bra $L__BB2_6; + bra.uni $L__BB2_3; + +$L__BB2_6: + mov.u32 %r8421, %tid.x; + setp.ge.u32 %p16, %r8421, %r8; + mov.u32 %r8423, 0; + @%p16 bra $L__BB2_9; + + mov.u32 %r8423, 0; + +$L__BB2_8: + cvt.u64.u32 %rd92, %r8421; + add.s64 %rd93, %rd92, %rd4; + shl.b64 %rd94, %rd93, 2; + add.s64 %rd95, %rd3, %rd94; + ld.global.u32 %r4074, [%rd95]; + abs.s32 %r4075, %r4074; + max.u32 %r8423, %r8423, %r4075; + mov.u32 %r4076, %ntid.x; + add.s32 %r8421, %r8421, %r4076; + setp.lt.u32 %p17, %r8421, %r8; + @%p17 bra $L__BB2_8; + bra.uni $L__BB2_9; + +$L__BB2_3: + mov.u32 %r8419, %tid.x; + setp.ge.u32 %p14, %r8419, %r8; + mov.u32 %r8423, 0; + @%p14 bra $L__BB2_9; + + sub.s32 %r9, %r4055, %r4057; + mov.u32 %r8423, 0; + +$L__BB2_5: + div.u32 %r4066, %r8419, %r4057; + mad.lo.s32 %r4067, %r9, %r4066, %r8419; + cvt.u64.u32 %rd88, %r4067; + add.s64 %rd89, %rd88, %rd4; + shl.b64 %rd90, %rd89, 2; + add.s64 %rd91, %rd3, %rd90; + ld.global.u32 %r4068, [%rd91]; + abs.s32 %r4069, %r4068; + max.u32 %r8423, %r8423, %r4069; + mov.u32 %r4070, %ntid.x; + add.s32 %r8419, %r8419, %r4070; + setp.lt.u32 %p15, %r8419, %r8; + @%p15 bra $L__BB2_5; + +$L__BB2_9: + mov.u32 %r4077, %tid.x; + shl.b32 %r4078, %r4077, 2; + mov.u32 %r4079, _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE9block_max; + add.s32 %r4080, %r4079, %r4078; + st.shared.u32 [%r4080], %r8423; + bar.sync 0; + mov.u32 %r4081, %ntid.x; + shr.u32 %r8424, %r4081, 1; + setp.eq.s32 %p18, %r8424, 0; + @%p18 bra $L__BB2_13; + +$L__BB2_10: + setp.ge.u32 %p19, %r4077, %r8424; + @%p19 bra $L__BB2_12; + + add.s32 %r4087, %r8424, %r4077; + shl.b32 %r4088, %r8424, 2; + add.s32 %r4089, %r4080, %r4088; + ld.shared.u32 %r4090, [%r4089]; + ld.shared.u32 %r4091, [%r4080]; + setp.gt.u32 %p20, %r4091, %r4090; + selp.b32 %r4092, %r4077, %r4087, %p20; + shl.b32 %r4093, %r4092, 2; + add.s32 %r4094, %r4079, %r4093; + ld.shared.u32 %r4095, [%r4094]; + st.shared.u32 [%r4080], %r4095; + +$L__BB2_12: + bar.sync 0; + shr.u32 %r8424, %r8424, 1; + setp.ne.s32 %p21, %r8424, 0; + @%p21 bra $L__BB2_10; + +$L__BB2_13: + ld.shared.u32 %r8425, [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE9block_max]; + +$L__BB2_15: + mov.u32 %r4097, %tid.x; + setp.ne.s32 %p22, %r4097, 0; + @%p22 bra $L__BB2_1905; + + setp.eq.s32 %p2367, %r4058, 0; + mov.u32 %r8410, %ctaid.x; + ld.param.u64 %rd1414, [ j2k_htj2k_encode_codeblocks_multi_input_param_5]; + mov.u32 %r4098, 0; + mov.u32 %r4099, 1; + cvt.u64.u32 %rd5, %r4060; + setp.eq.s32 %p24, %r4062, 1; + cvta.to.global.u64 %rd96, %rd1414; + mul.wide.u32 %rd97, %r8410, 32; + add.s64 %rd6, %rd96, %rd97; + st.global.u32 [%rd6], %r4099; + st.global.u32 [%rd6+4], %r4098; + st.global.u32 [%rd6+8], %r4098; + st.global.u32 [%rd6+12], %r4098; + st.global.u32 [%rd6+16], %r4098; + st.global.u32 [%rd6+20], %r4098; + st.global.u32 [%rd6+24], %r4098; + st.global.u32 [%rd6+28], %r4098; + add.s32 %r4101, %r4057, -1; + setp.ge.u32 %p25, %r4101, %r4055; + or.pred %p26, %p25, %p2367; + setp.gt.u32 %p27, %r4057, 1024; + or.pred %p1, %p27, %p26; + add.s64 %rd98, %rd1, %rd5; + add.s64 %rd7, %rd98, 20548; + mov.u32 %r4103, _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE13cleanup_e_val; + @%p24 bra $L__BB2_1254; + bra.uni $L__BB2_17; + +$L__BB2_1254: + @%p1 bra $L__BB2_1903; + + cvt.u16.u32 %rs822, %r4057; + mov.u16 %rs823, 4096; + div.u16 %rs824, %rs823, %rs822; + cvt.u32.u16 %r6801, %rs824; + setp.gt.u32 %p1625, %r4058, %r6801; + add.s32 %r10752, %r4059, -1; + setp.gt.u32 %p1626, %r10752, 29; + or.pred %p1627, %p1626, %p1625; + setp.lt.u32 %p1628, %r4061, 20549; + or.pred %p1629, %p1628, %p1627; + @%p1629 bra $L__BB2_1903; + bra.uni $L__BB2_1256; + +$L__BB2_1903: + mov.u32 %r8399, 2; + st.global.u32 [%rd6], %r8399; + mov.u32 %r8400, 1; + st.global.u32 [%rd6+4], %r8400; + mov.u32 %r10750, 0; + mov.u32 %r10751, %r10750; + mov.u32 %r10752, %r10750; + mov.u32 %r10753, %r10750; + +$L__BB2_1904: + st.global.u32 [%rd6+8], %r10751; + st.global.u32 [%rd6+12], %r10753; + st.global.u32 [%rd6+16], %r10752; + st.global.u32 [%rd6+20], %r10751; + mov.u32 %r8401, 0; + st.global.u32 [%rd6+24], %r8401; + st.global.u32 [%rd6+28], %r10750; + bra.uni $L__BB2_1905; + +$L__BB2_17: + @%p1 bra $L__BB2_1253; + + cvt.u16.u32 %rs507, %r4057; + mov.u16 %rs508, 4096; + div.u16 %rs509, %rs508, %rs507; + cvt.u32.u16 %r4105, %rs509; + setp.gt.u32 %p28, %r4058, %r4105; + add.s32 %r4106, %r4059, -1; + setp.gt.u32 %p29, %r4106, 29; + or.pred %p30, %p29, %p28; + setp.lt.u32 %p31, %r4061, 20549; + or.pred %p32, %p31, %p30; + @%p32 bra $L__BB2_1253; + bra.uni $L__BB2_19; + +$L__BB2_1253: + mov.u32 %r6798, 2; + st.global.u32 [%rd6], %r6798; + mov.u32 %r6799, 1; + st.global.u32 [%rd6+4], %r6799; + mov.u32 %r6800, 0; + st.global.u32 [%rd6+8], %r6800; + st.global.u32 [%rd6+12], %r6800; + st.global.u32 [%rd6+16], %r6800; + st.global.u32 [%rd6+20], %r6800; + st.global.u32 [%rd6+24], %r6800; + st.global.u32 [%rd6+28], %r6800; + +$L__BB2_1905: + ret; + +$L__BB2_1256: + setp.eq.s32 %p1630, %r8425, 0; + @%p1630 bra $L__BB2_1902; + + clz.b32 %r6802, %r8425; + mov.u32 %r6803, 32; + sub.s32 %r6804, %r6803, %r6802; + setp.gt.u32 %p1631, %r6804, %r4059; + @%p1631 bra $L__BB2_1901; + bra.uni $L__BB2_1258; + +$L__BB2_1901: + mov.u32 %r8390, 1; + st.global.u32 [%rd6], %r8390; + mov.u32 %r8391, 2; + st.global.u32 [%rd6+4], %r8391; + mov.u32 %r10750, 0; + mov.u32 %r10751, %r10750; + mov.u32 %r10752, %r10750; + mov.u32 %r10753, %r10750; + bra.uni $L__BB2_1904; + +$L__BB2_19: + add.s32 %r4107, %r4062, -1; + setp.gt.u32 %p33, %r4107, 163; + @%p33 bra $L__BB2_1252; + bra.uni $L__BB2_20; + +$L__BB2_1252: + mov.u32 %r6795, 2; + st.global.u32 [%rd6], %r6795; + mov.u32 %r6796, 5; + st.global.u32 [%rd6+4], %r6796; + mov.u32 %r6797, 0; + st.global.u32 [%rd6+8], %r6797; + st.global.u32 [%rd6+12], %r6797; + st.global.u32 [%rd6+16], %r6797; + st.global.u32 [%rd6+20], %r6797; + st.global.u32 [%rd6+24], %r6797; + st.global.u32 [%rd6+28], %r6797; + bra.uni $L__BB2_1905; + +$L__BB2_20: + setp.gt.u32 %p34, %r4062, 3; + @%p34 bra $L__BB2_1251; + bra.uni $L__BB2_21; + +$L__BB2_1251: + mov.u32 %r6792, 2; + st.global.u32 [%rd6], %r6792; + mov.u32 %r6793, 5; + st.global.u32 [%rd6+4], %r6793; + mov.u32 %r6794, 0; + st.global.u32 [%rd6+8], %r6794; + st.global.u32 [%rd6+12], %r6794; + st.global.u32 [%rd6+16], %r6794; + st.global.u32 [%rd6+20], %r6794; + st.global.u32 [%rd6+24], %r6794; + st.global.u32 [%rd6+28], %r6794; + bra.uni $L__BB2_1905; + +$L__BB2_1902: + mov.u32 %r10750, 0; + st.global.u32 [%rd6], %r10750; + st.global.u32 [%rd6+4], %r10750; + mov.u32 %r10751, %r10750; + mov.u32 %r10752, %r4059; + mov.u32 %r10753, %r10750; + bra.uni $L__BB2_1904; + +$L__BB2_1258: + add.s32 %r8416, %r4057, 1; + shr.u32 %r8415, %r8416, 1; + mov.u32 %r9741, 0; + mov.u32 %r6806, 31; + sub.s32 %r2358, %r6806, %r4059; + mov.u16 %rs825, 255; + st.global.u8 [%rd7], %rs825; + add.s32 %r6807, %r8415, 2; + min.u32 %r2359, %r6807, 513; + mov.u32 %r6808, -3; + sub.s32 %r6809, %r6808, %r8415; + max.u32 %r6810, %r6809, -514; + mov.u32 %r6811, -2; + sub.s32 %r6812, %r6811, %r6810; + and.b32 %r9743, %r2359, 3; + setp.lt.u32 %p1632, %r6812, 3; + @%p1632 bra $L__BB2_1261; + + sub.s32 %r9740, %r2359, %r9743; + mov.u32 %r9741, 0; + +$L__BB2_1260: + add.s32 %r6815, %r4103, %r9741; + mov.u16 %rs826, 0; + st.shared.u8 [%r6815], %rs826; + mov.u32 %r6816, _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val; + add.s32 %r6817, %r6816, %r9741; + st.shared.u8 [%r6817], %rs826; + st.shared.u8 [%r6815+1], %rs826; + st.shared.u8 [%r6817+1], %rs826; + st.shared.u8 [%r6815+2], %rs826; + st.shared.u8 [%r6817+2], %rs826; + st.shared.u8 [%r6815+3], %rs826; + st.shared.u8 [%r6817+3], %rs826; + add.s32 %r9741, %r9741, 4; + add.s32 %r9740, %r9740, -4; + setp.ne.s32 %p1633, %r9740, 0; + @%p1633 bra $L__BB2_1260; + +$L__BB2_1261: + setp.eq.s32 %p1634, %r9743, 0; + @%p1634 bra $L__BB2_1264; + + mov.u32 %r6820, _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val; + +$L__BB2_1263: + .pragma "nounroll"; + add.s32 %r6819, %r4103, %r9741; + mov.u16 %rs827, 0; + st.shared.u8 [%r6819], %rs827; + add.s32 %r6821, %r6820, %r9741; + st.shared.u8 [%r6821], %rs827; + add.s32 %r9741, %r9741, 1; + add.s32 %r9743, %r9743, -1; + setp.ne.s32 %p1635, %r9743, 0; + @%p1635 bra $L__BB2_1263; + +$L__BB2_1264: + mov.u32 %r10461, 0; + mov.u32 %r10274, 1; + mov.u16 %rs1253, 0; + mov.u32 %r10462, 8; + mov.u16 %rs1322, 15; + mov.u32 %r10275, 4; + mov.u32 %r10463, %r10461; + mov.u32 %r10464, %r10461; + mov.u32 %r10495, %r10461; + mov.u32 %r10276, %r10274; + mov.u32 %r10277, %r10461; + mov.u32 %r9826, %r10461; + mov.u32 %r9835, %r10462; + mov.u32 %r10040, %r10461; + mov.u32 %r10041, %r10461; + mov.u32 %r10042, %r10274; + mov.u32 %r10043, %r10461; + @%p10 bra $L__BB2_1632; + + ld.param.u64 %rd1417, [ j2k_htj2k_encode_codeblocks_multi_input_param_4]; + ld.param.u64 %rd1412, [ j2k_htj2k_encode_codeblocks_multi_input_param_2]; + cvta.to.global.u64 %rd62, %rd1412; + cvta.to.global.u64 %rd63, %rd1417; + mov.u32 %r6854, 0; + mov.u32 %r9835, 8; + mov.u32 %r10042, 1; + mov.u32 %r10275, 4; + mov.u16 %rs1322, 15; + mov.u16 %rs1253, 0; + mov.u32 %r9744, %r6854; + mov.u32 %r10043, %r6854; + mov.u32 %r10041, %r6854; + mov.u32 %r10040, %r6854; + mov.u32 %r9826, %r6854; + mov.u32 %r10277, %r6854; + mov.u32 %r10276, %r10042; + mov.u32 %r10274, %r10042; + mov.u32 %r10495, %r6854; + mov.u32 %r10464, %r6854; + mov.u32 %r10463, %r6854; + mov.u32 %r10462, %r9835; + mov.u32 %r10461, %r6854; + mov.u32 %r9760, %r6854; + mov.u32 %r9761, %r6854; + bra.uni $L__BB2_1266; + +$L__BB2_21: + setp.eq.s32 %p35, %r8425, 0; + @%p35 bra $L__BB2_1250; + + clz.b32 %r4108, %r8425; + mov.u32 %r4109, 32; + sub.s32 %r4110, %r4109, %r4108; + setp.gt.u32 %p36, %r4110, %r4059; + @%p36 bra $L__BB2_1249; + bra.uni $L__BB2_23; + +$L__BB2_1249: + mov.u32 %r6788, 1; + st.global.u32 [%rd6], %r6788; + mov.u32 %r6789, 2; + st.global.u32 [%rd6+4], %r6789; + mov.u32 %r6790, 0; + st.global.u32 [%rd6+8], %r6790; + st.global.u32 [%rd6+12], %r6790; + st.global.u32 [%rd6+16], %r6790; + st.global.u32 [%rd6+20], %r6790; + st.global.u32 [%rd6+24], %r6790; + st.global.u32 [%rd6+28], %r6790; + bra.uni $L__BB2_1905; + +$L__BB2_1432: + setp.gt.u32 %p1816, %r9826, 191; + mov.u32 %r10029, 1; + mov.u32 %r9835, 0; + @%p1816 bra $L__BB2_1434; + + st.global.u8 [%rd67], %rs1253; + add.s32 %r9826, %r9826, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9835, 8; + mov.u32 %r10029, %r10043; + bra.uni $L__BB2_1434; + +$L__BB2_1337: + setp.gt.u32 %p1714, %r9826, 191; + mov.u32 %r9843, 1; + mov.u32 %r9835, 0; + @%p1714 bra $L__BB2_1339; + + and.b16 %rs865, %rs1253, 255; + st.global.u8 [%rd64], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1715, %rs865, 255; + selp.b32 %r9835, 7, 8, %p1715; + mov.u16 %rs1253, 0; + mov.u32 %r9843, %r10043; + bra.uni $L__BB2_1339; + +$L__BB2_1471: + setp.gt.u32 %p1863, %r9826, 191; + mov.u32 %r10036, 1; + mov.u32 %r9835, 0; + @%p1863 bra $L__BB2_1473; + + and.b16 %rs914, %rs1253, 255; + st.global.u8 [%rd67], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1864, %rs914, 255; + selp.b32 %r9835, 7, 8, %p1864; + mov.u16 %rs1253, 0; + mov.u32 %r10036, %r10043; + bra.uni $L__BB2_1473; + +$L__BB2_1266: + cvt.u64.u32 %rd1066, %r9761; + add.s64 %rd1067, %rd1066, %rd4; + shl.b64 %rd1068, %rd1067, 2; + add.s64 %rd1069, %rd3, %rd1068; + ld.global.u32 %r2389, [%rd1069]; + setp.eq.s32 %p1637, %r2389, 0; + mov.u32 %r9762, %r6854; + @%p1637 bra $L__BB2_1268; + + and.b32 %r6856, %r2389, -2147483648; + abs.s32 %r6857, %r2389; + shl.b32 %r6858, %r6857, %r2358; + or.b32 %r9762, %r6858, %r6856; + +$L__BB2_1268: + shl.b32 %r6862, %r9762, 1; + shr.u32 %r6863, %r6862, %r2358; + and.b32 %r2392, %r6863, -2; + setp.eq.s32 %p1638, %r2392, 0; + mov.u32 %r9766, 0; + mov.u32 %r9763, %r9766; + mov.u32 %r9764, %r9766; + mov.u32 %r9770, %r9766; + @%p1638 bra $L__BB2_1270; + + add.s32 %r6865, %r2392, -1; + clz.b32 %r6866, %r6865; + mov.u32 %r6867, 32; + sub.s32 %r9763, %r6867, %r6866; + shr.u32 %r6868, %r9762, 31; + add.s32 %r6869, %r6868, %r2392; + add.s32 %r9764, %r6869, -2; + mov.u32 %r9770, 1; + +$L__BB2_1270: + setp.lt.u32 %p1639, %r4058, 2; + @%p1639 bra $L__BB2_1273; + + add.s32 %r6872, %r9761, %r4055; + cvt.u64.u32 %rd1070, %r6872; + add.s64 %rd1071, %rd1070, %rd4; + shl.b64 %rd1072, %rd1071, 2; + add.s64 %rd1073, %rd3, %rd1072; + ld.global.u32 %r2398, [%rd1073]; + setp.eq.s32 %p1640, %r2398, 0; + @%p1640 bra $L__BB2_1273; + + and.b32 %r6873, %r2398, -2147483648; + abs.s32 %r6874, %r2398; + shl.b32 %r6875, %r6874, %r2358; + or.b32 %r9766, %r6875, %r6873; + +$L__BB2_1273: + shl.b32 %r6878, %r9766, 1; + shr.u32 %r6879, %r6878, %r2358; + and.b32 %r2401, %r6879, -2; + setp.eq.s32 %p1641, %r2401, 0; + mov.u32 %r9781, 0; + mov.u32 %r9767, %r9781; + mov.u32 %r9768, %r9781; + mov.u32 %r9786, %r9763; + @%p1641 bra $L__BB2_1275; + + or.b32 %r9770, %r9770, 2; + add.s32 %r6880, %r2401, -1; + clz.b32 %r6881, %r6880; + mov.u32 %r6882, 32; + sub.s32 %r9767, %r6882, %r6881; + max.s32 %r9786, %r9763, %r9767; + shr.u32 %r6883, %r9766, 31; + add.s32 %r6884, %r6883, %r2401; + add.s32 %r9768, %r6884, -2; + +$L__BB2_1275: + add.s32 %r9785, %r9761, 1; + add.s32 %r6889, %r9744, 1; + setp.ge.u32 %p1642, %r6889, %r4057; + mov.u32 %r9782, %r9781; + mov.u32 %r9783, %r9781; + mov.u32 %r9784, %r9781; + @%p1642 bra $L__BB2_1286; + + cvt.u64.u32 %rd1074, %r9785; + add.s64 %rd1075, %rd1074, %rd4; + shl.b64 %rd1076, %rd1075, 2; + add.s64 %rd1077, %rd3, %rd1076; + ld.global.u32 %r2411, [%rd1077]; + setp.eq.s32 %p1643, %r2411, 0; + mov.u32 %r9782, 0; + mov.u32 %r9771, %r9782; + @%p1643 bra $L__BB2_1278; + + and.b32 %r6891, %r2411, -2147483648; + abs.s32 %r6892, %r2411; + shl.b32 %r6893, %r6892, %r2358; + or.b32 %r9771, %r6893, %r6891; + +$L__BB2_1278: + shl.b32 %r6896, %r9771, 1; + shr.u32 %r6897, %r6896, %r2358; + and.b32 %r2414, %r6897, -2; + setp.eq.s32 %p1644, %r2414, 0; + mov.u32 %r9784, %r9782; + @%p1644 bra $L__BB2_1280; + + or.b32 %r9770, %r9770, 4; + add.s32 %r6898, %r2414, -1; + clz.b32 %r6899, %r6898; + mov.u32 %r6900, 32; + sub.s32 %r9782, %r6900, %r6899; + max.s32 %r9786, %r9786, %r9782; + shr.u32 %r6901, %r9771, 31; + add.s32 %r6902, %r6901, %r2414; + add.s32 %r9784, %r6902, -2; + +$L__BB2_1280: + mov.u32 %r9781, 0; + mov.u32 %r9776, %r9781; + @%p1639 bra $L__BB2_1283; + + add.s32 %r6905, %r9785, %r4055; + cvt.u64.u32 %rd1078, %r6905; + add.s64 %rd1079, %rd1078, %rd4; + shl.b64 %rd1080, %rd1079, 2; + add.s64 %rd1081, %rd3, %rd1080; + ld.global.u32 %r2423, [%rd1081]; + setp.eq.s32 %p1646, %r2423, 0; + @%p1646 bra $L__BB2_1283; + + and.b32 %r6906, %r2423, -2147483648; + abs.s32 %r6907, %r2423; + shl.b32 %r6908, %r6907, %r2358; + or.b32 %r9776, %r6908, %r6906; + +$L__BB2_1283: + shl.b32 %r6911, %r9776, 1; + shr.u32 %r6912, %r6911, %r2358; + and.b32 %r2426, %r6912, -2; + setp.eq.s32 %p1647, %r2426, 0; + mov.u32 %r9783, %r9781; + @%p1647 bra $L__BB2_1285; + + or.b32 %r9770, %r9770, 8; + add.s32 %r6913, %r2426, -1; + clz.b32 %r6914, %r6913; + mov.u32 %r6915, 32; + sub.s32 %r9781, %r6915, %r6914; + max.s32 %r9786, %r9786, %r9781; + shr.u32 %r6916, %r9776, 31; + add.s32 %r6917, %r6916, %r2426; + add.s32 %r9783, %r6917, -2; + +$L__BB2_1285: + add.s32 %r9785, %r9761, 2; + +$L__BB2_1286: + mov.u32 %r9761, %r9785; + add.s32 %r6919, %r9786, -1; + setp.lt.s32 %p1648, %r9786, 2; + setp.gt.s32 %p1649, %r9786, 1; + selp.b32 %r2443, %r6919, 0, %p1649; + mov.u32 %r9788, 0; + @%p1648 bra $L__BB2_1288; + + setp.eq.s32 %p1650, %r9763, %r9786; + selp.u32 %r6920, 1, 0, %p1650; + setp.eq.s32 %p1651, %r9767, %r9786; + selp.u32 %r6921, -1, 0, %p1651; + bfi.b32 %r6922, %r6921, %r6920, 1, 1; + setp.eq.s32 %p1652, %r9782, %r9786; + selp.u16 %rs832, 1, 0, %p1652; + mul.wide.u16 %r6923, %rs832, 4; + or.b32 %r6924, %r6922, %r6923; + setp.eq.s32 %p1653, %r9781, %r9786; + selp.u16 %rs833, 1, 0, %p1653; + mul.wide.u16 %r6925, %rs833, 8; + or.b32 %r9788, %r6924, %r6925; + +$L__BB2_1288: + shr.u32 %r6926, %r9744, 1; + add.s32 %r2446, %r4103, %r6926; + ld.shared.u8 %rs834, [%r2446]; + cvt.u32.u16 %r6928, %rs834; + and.b32 %r6929, %r6928, 255; + and.b32 %r6930, %r9767, 255; + setp.lt.u32 %p1654, %r6930, %r6929; + cvt.u16.u32 %rs835, %r9767; + selp.b16 %rs836, %rs834, %rs835, %p1654; + st.shared.u8 [%r2446], %rs836; + cvt.u16.u32 %rs274, %r9781; + st.shared.u8 [%r2446+1], %rs274; + and.b32 %r2447, %r9770, 2; + cvt.u16.u32 %rs837, %r2447; + shr.u16 %rs838, %rs837, 1; + mov.u32 %r6931, _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val; + add.s32 %r2448, %r6931, %r6926; + ld.shared.u8 %rs839, [%r2448]; + or.b16 %rs840, %rs839, %rs838; + st.shared.u8 [%r2448], %rs840; + and.b32 %r2449, %r9770, 8; + shr.u32 %r2450, %r2449, 3; + st.shared.u8 [%r2448+1], %r2450; + shl.b32 %r6932, %r9770, 4; + shl.b32 %r6933, %r9760, 8; + or.b32 %r6934, %r6932, %r6933; + or.b32 %r6935, %r6934, %r9788; + mul.wide.u32 %rd1082, %r6935, 2; + add.s64 %rd1083, %rd62, %rd1082; + ld.global.u16 %rs275, [%rd1083]; + shr.u16 %rs841, %rs275, 4; + and.b16 %rs276, %rs841, 7; + setp.eq.s16 %p1655, %rs276, 0; + mov.u32 %r9800, %r10277; + @%p1655 bra $L__BB2_1295; + + cvt.u32.u16 %r9789, %rs276; + shr.u16 %rs842, %rs275, 8; + cvt.u32.u16 %r9790, %rs842; + +$L__BB2_1290: + mov.u32 %r2453, %r9789; + setp.gt.u32 %p1656, %r10274, 2879; + mov.u32 %r9800, 1; + @%p1656 bra $L__BB2_1295; + + mov.u32 %r6937, 8; + sub.s32 %r6938, %r6937, %r10276; + sub.s32 %r6939, %r6938, %r10275; + min.u32 %r6940, %r6939, %r2453; + setp.eq.s32 %p1657, %r6940, 32; + mov.u32 %r6941, -1; + shl.b32 %r6942, %r6941, %r6940; + not.b32 %r6943, %r6942; + selp.b32 %r6944, -1, %r6943, %p1657; + and.b32 %r6945, %r6944, %r9790; + shl.b32 %r6946, %r6945, %r10275; + cvt.u16.u32 %rs843, %r6946; + or.b16 %rs1322, %rs1322, %rs843; + add.s32 %r10275, %r6940, %r10275; + sub.s32 %r9789, %r2453, %r6940; + shr.u32 %r9790, %r9790, %r6940; + setp.gt.u32 %p1658, %r6939, %r2453; + @%p1658 bra $L__BB2_1294; + + setp.ne.s32 %p1659, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs844, %rs1322, 255; + setp.ne.s16 %p1660, %rs844, 127; + and.pred %p1661, %p1659, %p1660; + @%p1661 bra $L__BB2_1294; + + mov.u32 %r6949, 20548; + sub.s32 %r6950, %r6949, %r10274; + cvt.u64.u32 %rd1084, %r6950; + add.s64 %rd1085, %rd1084, %rd5; + add.s64 %rd1086, %rd1, %rd1085; + st.global.u8 [%rd1086], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p1662, %rs844, 143; + selp.u32 %r10276, 1, 0, %p1662; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1294: + setp.ne.s32 %p1663, %r9789, 0; + mov.u32 %r9800, %r10277; + @%p1663 bra $L__BB2_1290; + +$L__BB2_1295: + setp.ne.s32 %p1664, %r9760, 0; + @%p1664 bra $L__BB2_1343; + + setp.eq.s32 %p1665, %r9770, 0; + add.s32 %r6951, %r9826, 17477; + cvt.u64.u32 %rd1087, %r6951; + add.s64 %rd1088, %rd1087, %rd5; + add.s64 %rd64, %rd1, %rd1088; + @%p1665 bra $L__BB2_1335; + + shl.b16 %rs1253, %rs1253, 1; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1666, %r9835, 0; + mov.u32 %r9836, %r10043; + @%p1666 bra $L__BB2_1300; + + setp.gt.u32 %p1667, %r9826, 191; + mov.u32 %r9836, 1; + mov.u32 %r9835, 0; + @%p1667 bra $L__BB2_1300; + + st.global.u8 [%rd64], %rs1253; + add.s32 %r9826, %r9826, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9835, 8; + mov.u32 %r9836, %r10043; + +$L__BB2_1300: + setp.lt.u32 %p1668, %r10041, 3; + mov.u32 %r9804, 0; + @%p1668 bra $L__BB2_1303; + + setp.lt.u32 %p1669, %r10041, 6; + mov.u32 %r9804, 1; + @%p1669 bra $L__BB2_1303; + + setp.lt.u32 %p1670, %r10041, 9; + setp.eq.s32 %p1671, %r10041, 11; + selp.b32 %r6957, 4, 5, %p1671; + setp.lt.u32 %p1672, %r10041, 11; + selp.b32 %r6958, 3, %r6957, %p1672; + selp.b32 %r9804, 2, %r6958, %p1670; + +$L__BB2_1303: + setp.eq.s32 %p1673, %r9804, 0; + @%p1673 bra $L__BB2_1331; + + add.s32 %r2477, %r9804, -1; + and.b32 %r2478, %r9804, 3; + setp.eq.s32 %p1674, %r2478, 0; + mov.u32 %r9814, %r9804; + mov.u32 %r9815, %r9836; + @%p1674 bra $L__BB2_1316; + + mov.u32 %r6960, 1; + shl.b32 %r6961, %r6960, %r2477; + and.b32 %r6962, %r6961, %r10040; + setp.ne.s32 %p1675, %r6962, 0; + selp.u32 %r6963, 1, 0, %p1675; + cvt.u32.u16 %r6964, %rs1253; + bfi.b32 %r6965, %r6964, %r6963, 1, 8; + cvt.u16.u32 %rs1253, %r6965; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1676, %r9835, 0; + mov.u32 %r9815, %r9836; + @%p1676 bra $L__BB2_1308; + + setp.gt.u32 %p1677, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r9815, %r6960; + @%p1677 bra $L__BB2_1308; + + add.s32 %r6969, %r9826, 17477; + cvt.u64.u32 %rd1089, %r6969; + add.s64 %rd1090, %rd1089, %rd5; + add.s64 %rd1091, %rd1, %rd1090; + st.global.u8 [%rd1091], %rs1253; + add.s32 %r9826, %r9826, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9835, 8; + mov.u32 %r9815, %r9836; + +$L__BB2_1308: + setp.eq.s32 %p1678, %r2478, 1; + mov.u32 %r9836, %r9815; + mov.u32 %r9814, %r2477; + @%p1678 bra $L__BB2_1316; + + add.s32 %r9814, %r9804, -2; + mov.u32 %r6970, 1; + shl.b32 %r6971, %r6970, %r9814; + and.b32 %r6972, %r6971, %r10040; + setp.ne.s32 %p1679, %r6972, 0; + selp.u32 %r6973, 1, 0, %p1679; + cvt.u32.u16 %r6974, %rs1253; + bfi.b32 %r6975, %r6974, %r6973, 1, 8; + cvt.u16.u32 %rs1253, %r6975; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1680, %r9835, 0; + mov.u32 %r9810, %r9815; + @%p1680 bra $L__BB2_1312; + + setp.gt.u32 %p1681, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r9810, %r6970; + @%p1681 bra $L__BB2_1312; + + add.s32 %r6978, %r9826, 17477; + cvt.u64.u32 %rd1092, %r6978; + add.s64 %rd1093, %rd1092, %rd5; + add.s64 %rd1094, %rd1, %rd1093; + and.b16 %rs851, %rs1253, 255; + st.global.u8 [%rd1094], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1682, %rs851, 255; + selp.b32 %r9835, 7, 8, %p1682; + mov.u16 %rs1253, 0; + mov.u32 %r9810, %r9815; + +$L__BB2_1312: + setp.eq.s32 %p1683, %r2478, 2; + mov.u32 %r9836, %r9810; + mov.u32 %r9815, %r9810; + @%p1683 bra $L__BB2_1316; + + add.s32 %r9814, %r9804, -3; + mov.u32 %r6979, 1; + shl.b32 %r6980, %r6979, %r9814; + and.b32 %r6981, %r6980, %r10040; + setp.ne.s32 %p1684, %r6981, 0; + selp.u32 %r6982, 1, 0, %p1684; + cvt.u32.u16 %r6983, %rs1253; + bfi.b32 %r6984, %r6983, %r6982, 1, 8; + cvt.u16.u32 %rs1253, %r6984; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1685, %r9835, 0; + mov.u32 %r9836, %r9810; + mov.u32 %r9815, %r9810; + @%p1685 bra $L__BB2_1316; + + setp.gt.u32 %p1686, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r9836, %r6979; + mov.u32 %r9815, %r6979; + @%p1686 bra $L__BB2_1316; + + add.s32 %r6989, %r9826, 17477; + cvt.u64.u32 %rd1095, %r6989; + add.s64 %rd1096, %rd1095, %rd5; + add.s64 %rd1097, %rd1, %rd1096; + and.b16 %rs854, %rs1253, 255; + st.global.u8 [%rd1097], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1687, %rs854, 255; + selp.b32 %r9835, 7, 8, %p1687; + mov.u16 %rs1253, 0; + mov.u32 %r9836, %r9810; + mov.u32 %r9815, %r9810; + +$L__BB2_1316: + setp.lt.u32 %p1688, %r2477, 3; + @%p1688 bra $L__BB2_1331; + + mov.u32 %r9836, %r9815; + +$L__BB2_1318: + add.s32 %r6990, %r9814, -1; + mov.u32 %r6991, 1; + shl.b32 %r6992, %r6991, %r6990; + and.b32 %r6993, %r6992, %r10040; + setp.ne.s32 %p1689, %r6993, 0; + selp.u32 %r6994, 1, 0, %p1689; + cvt.u32.u16 %r6995, %rs1253; + bfi.b32 %r9824, %r6995, %r6994, 1, 8; + add.s32 %r9823, %r9835, -1; + setp.ne.s32 %p1690, %r9823, 0; + mov.u32 %r9825, %r9836; + @%p1690 bra $L__BB2_1321; + + setp.gt.u32 %p1691, %r9826, 191; + mov.u32 %r9823, 0; + mov.u32 %r9825, %r6991; + @%p1691 bra $L__BB2_1321; + + cvt.u16.u32 %rs855, %r9824; + and.b16 %rs856, %rs855, 255; + add.s32 %r6999, %r9826, 17477; + cvt.u64.u32 %rd1098, %r6999; + add.s64 %rd1099, %rd1098, %rd5; + add.s64 %rd1100, %rd1, %rd1099; + st.global.u8 [%rd1100], %rs855; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1692, %rs856, 255; + selp.b32 %r9823, 7, 8, %p1692; + mov.u32 %r9824, 0; + mov.u32 %r9825, %r9836; + +$L__BB2_1321: + add.s32 %r7000, %r9814, -2; + shl.b32 %r7002, %r6991, %r7000; + and.b32 %r7003, %r7002, %r10040; + setp.ne.s32 %p1693, %r7003, 0; + and.b32 %r7004, %r9824, 127; + selp.u32 %r7005, 1, 0, %p1693; + bfi.b32 %r9828, %r7004, %r7005, 1, 7; + add.s32 %r9827, %r9823, -1; + setp.ne.s32 %p1694, %r9827, 0; + mov.u32 %r9829, %r9825; + @%p1694 bra $L__BB2_1324; + + setp.gt.u32 %p1695, %r9826, 191; + mov.u32 %r9829, 1; + mov.u32 %r9827, 0; + @%p1695 bra $L__BB2_1324; + + cvt.u16.u32 %rs857, %r9828; + and.b16 %rs858, %rs857, 255; + add.s32 %r7009, %r9826, 17477; + cvt.u64.u32 %rd1101, %r7009; + add.s64 %rd1102, %rd1101, %rd5; + add.s64 %rd1103, %rd1, %rd1102; + st.global.u8 [%rd1103], %rs857; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1696, %rs858, 255; + selp.b32 %r9827, 7, 8, %p1696; + mov.u32 %r9828, 0; + mov.u32 %r9829, %r9825; + +$L__BB2_1324: + add.s32 %r7010, %r9814, -3; + mov.u32 %r7011, 1; + shl.b32 %r7012, %r7011, %r7010; + and.b32 %r7013, %r7012, %r10040; + setp.ne.s32 %p1697, %r7013, 0; + and.b32 %r7014, %r9828, 127; + selp.u32 %r7015, 1, 0, %p1697; + bfi.b32 %r9832, %r7014, %r7015, 1, 7; + add.s32 %r9831, %r9827, -1; + setp.ne.s32 %p1698, %r9831, 0; + mov.u32 %r9833, %r9829; + @%p1698 bra $L__BB2_1327; + + setp.gt.u32 %p1699, %r9826, 191; + mov.u32 %r9831, 0; + mov.u32 %r9833, %r7011; + @%p1699 bra $L__BB2_1327; + + cvt.u16.u32 %rs859, %r9832; + and.b16 %rs860, %rs859, 255; + add.s32 %r7019, %r9826, 17477; + cvt.u64.u32 %rd1104, %r7019; + add.s64 %rd1105, %rd1104, %rd5; + add.s64 %rd1106, %rd1, %rd1105; + st.global.u8 [%rd1106], %rs859; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1700, %rs860, 255; + selp.b32 %r9831, 7, 8, %p1700; + mov.u32 %r9832, 0; + mov.u32 %r9833, %r9829; + +$L__BB2_1327: + add.s32 %r9814, %r9814, -4; + shl.b32 %r7021, %r7011, %r9814; + and.b32 %r7022, %r7021, %r10040; + setp.ne.s32 %p1701, %r7022, 0; + and.b32 %r7023, %r9832, 127; + selp.u32 %r7024, 1, 0, %p1701; + bfi.b32 %r7025, %r7023, %r7024, 1, 15; + cvt.u16.u32 %rs1253, %r7025; + add.s32 %r9835, %r9831, -1; + setp.ne.s32 %p1702, %r9835, 0; + mov.u32 %r9836, %r9833; + @%p1702 bra $L__BB2_1330; + + setp.gt.u32 %p1703, %r9826, 191; + mov.u32 %r9836, 1; + mov.u32 %r9835, 0; + @%p1703 bra $L__BB2_1330; + + add.s32 %r7028, %r9826, 17477; + cvt.u64.u32 %rd1107, %r7028; + add.s64 %rd1108, %rd1107, %rd5; + add.s64 %rd1109, %rd1, %rd1108; + and.b16 %rs862, %rs1253, 255; + st.global.u8 [%rd1109], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1704, %rs862, 255; + selp.b32 %r9835, 7, 8, %p1704; + mov.u16 %rs1253, 0; + mov.u32 %r9836, %r9833; + +$L__BB2_1330: + setp.ne.s32 %p1705, %r9814, 0; + @%p1705 bra $L__BB2_1318; + +$L__BB2_1331: + add.s32 %r7030, %r10041, -1; + setp.eq.s32 %p1706, %r10041, 0; + mov.u32 %r10040, 0; + selp.b32 %r10041, 0, %r7030, %p1706; + setp.lt.u32 %p1707, %r10041, 3; + mov.u32 %r9840, %r10040; + @%p1707 bra $L__BB2_1334; + + setp.lt.u32 %p1708, %r10041, 6; + mov.u32 %r9840, 1; + @%p1708 bra $L__BB2_1334; + + setp.lt.u32 %p1709, %r10041, 9; + setp.eq.s32 %p1710, %r10041, 11; + selp.b32 %r7032, 4, 5, %p1710; + setp.lt.u32 %p1711, %r10041, 11; + selp.b32 %r7033, 3, %r7032, %p1711; + selp.b32 %r9840, 2, %r7033, %p1709; + +$L__BB2_1334: + mov.u32 %r7035, 1; + shl.b32 %r10042, %r7035, %r9840; + mov.u32 %r10043, %r9836; + bra.uni $L__BB2_1343; + +$L__BB2_1335: + add.s32 %r10040, %r10040, 1; + setp.lt.u32 %p1712, %r10040, %r10042; + @%p1712 bra $L__BB2_1343; + + shl.b16 %rs863, %rs1253, 1; + or.b16 %rs1253, %rs863, 1; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1713, %r9835, 0; + mov.u32 %r9843, %r10043; + @%p1713 bra $L__BB2_1339; + bra.uni $L__BB2_1337; + +$L__BB2_1339: + add.s32 %r7039, %r10041, 1; + min.u32 %r10041, %r7039, 12; + setp.lt.u32 %p1716, %r10041, 3; + mov.u32 %r10040, 0; + mov.u32 %r9844, %r10040; + @%p1716 bra $L__BB2_1342; + + setp.lt.u32 %p1717, %r10041, 6; + mov.u32 %r9844, 1; + @%p1717 bra $L__BB2_1342; + + setp.lt.u32 %p1718, %r10041, 9; + setp.eq.s32 %p1719, %r10041, 11; + selp.b32 %r7041, 4, 5, %p1719; + setp.lt.u32 %p1720, %r10041, 11; + selp.b32 %r7042, 3, %r7041, %p1720; + selp.b32 %r9844, 2, %r7042, %p1718; + +$L__BB2_1342: + mov.u32 %r7044, 1; + shl.b32 %r10042, %r7044, %r9844; + mov.u32 %r10043, %r9843; + +$L__BB2_1343: + max.s32 %r2561, %r9786, 1; + and.b16 %rs866, %rs275, 15; + cvt.u32.u16 %r2562, %rs866; + and.b32 %r2563, %r9770, 1; + setp.eq.s32 %p1721, %r2563, 0; + mov.u32 %r9865, %r10495; + @%p1721 bra $L__BB2_1350; + + and.b32 %r7045, %r2562, 1; + sub.s32 %r9851, %r2561, %r7045; + setp.eq.s32 %p1722, %r9851, 0; + mov.u32 %r9865, %r10495; + @%p1722 bra $L__BB2_1350; + + mov.u32 %r7046, -1; + shl.b32 %r7047, %r7046, %r9851; + not.b32 %r7048, %r7047; + and.b32 %r9852, %r9764, %r7048; + +$L__BB2_1346: + setp.gt.u32 %p1723, %r10461, 17476; + mov.u32 %r9865, 1; + @%p1723 bra $L__BB2_1350; + + sub.s32 %r7050, %r10462, %r10463; + min.u32 %r7051, %r7050, %r9851; + setp.eq.s32 %p1724, %r7051, 32; + mov.u32 %r7052, -1; + shl.b32 %r7053, %r7052, %r7051; + not.b32 %r7054, %r7053; + selp.b32 %r7055, -1, %r7054, %p1724; + and.b32 %r7056, %r7055, %r9852; + shl.b32 %r7057, %r7056, %r10463; + or.b32 %r10464, %r7057, %r10464; + add.s32 %r10463, %r7051, %r10463; + shr.u32 %r9852, %r9852, %r7051; + sub.s32 %r9851, %r9851, %r7051; + setp.lt.u32 %p1725, %r10463, %r10462; + @%p1725 bra $L__BB2_1349; + + cvt.u64.u32 %rd1110, %r10461; + add.s64 %rd1111, %rd1110, %rd5; + add.s64 %rd1112, %rd1, %rd1111; + st.global.u8 [%rd1112], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p1726, %r10464, 255; + selp.b32 %r10462, 7, 8, %p1726; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1349: + setp.ne.s32 %p1727, %r9851, 0; + mov.u32 %r9865, %r10495; + @%p1727 bra $L__BB2_1346; + +$L__BB2_1350: + setp.eq.s32 %p1728, %r2447, 0; + mov.u32 %r9880, %r9865; + @%p1728 bra $L__BB2_1357; + + shr.u32 %r7060, %r2562, 1; + and.b32 %r7061, %r7060, 1; + sub.s32 %r9866, %r2561, %r7061; + setp.eq.s32 %p1729, %r9866, 0; + mov.u32 %r9880, %r9865; + @%p1729 bra $L__BB2_1357; + + mov.u32 %r7062, -1; + shl.b32 %r7063, %r7062, %r9866; + not.b32 %r7064, %r7063; + and.b32 %r9867, %r9768, %r7064; + +$L__BB2_1353: + setp.gt.u32 %p1730, %r10461, 17476; + mov.u32 %r9880, 1; + @%p1730 bra $L__BB2_1357; + + sub.s32 %r7066, %r10462, %r10463; + min.u32 %r7067, %r7066, %r9866; + setp.eq.s32 %p1731, %r7067, 32; + mov.u32 %r7068, -1; + shl.b32 %r7069, %r7068, %r7067; + not.b32 %r7070, %r7069; + selp.b32 %r7071, -1, %r7070, %p1731; + and.b32 %r7072, %r7071, %r9867; + shl.b32 %r7073, %r7072, %r10463; + or.b32 %r10464, %r7073, %r10464; + add.s32 %r10463, %r7067, %r10463; + shr.u32 %r9867, %r9867, %r7067; + sub.s32 %r9866, %r9866, %r7067; + setp.lt.u32 %p1732, %r10463, %r10462; + @%p1732 bra $L__BB2_1356; + + cvt.u64.u32 %rd1113, %r10461; + add.s64 %rd1114, %rd1113, %rd5; + add.s64 %rd1115, %rd1, %rd1114; + st.global.u8 [%rd1115], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p1733, %r10464, 255; + selp.b32 %r10462, 7, 8, %p1733; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1356: + setp.ne.s32 %p1734, %r9866, 0; + mov.u32 %r9880, %r9865; + @%p1734 bra $L__BB2_1353; + +$L__BB2_1357: + and.b32 %r7076, %r9770, 4; + setp.eq.s32 %p1735, %r7076, 0; + mov.u32 %r9895, %r9880; + @%p1735 bra $L__BB2_1364; + + shr.u32 %r7077, %r2562, 2; + and.b32 %r7078, %r7077, 1; + sub.s32 %r9881, %r2561, %r7078; + setp.eq.s32 %p1736, %r9881, 0; + mov.u32 %r9895, %r9880; + @%p1736 bra $L__BB2_1364; + + mov.u32 %r7079, -1; + shl.b32 %r7080, %r7079, %r9881; + not.b32 %r7081, %r7080; + and.b32 %r9882, %r9784, %r7081; + +$L__BB2_1360: + setp.gt.u32 %p1737, %r10461, 17476; + mov.u32 %r9895, 1; + @%p1737 bra $L__BB2_1364; + + sub.s32 %r7083, %r10462, %r10463; + min.u32 %r7084, %r7083, %r9881; + setp.eq.s32 %p1738, %r7084, 32; + mov.u32 %r7085, -1; + shl.b32 %r7086, %r7085, %r7084; + not.b32 %r7087, %r7086; + selp.b32 %r7088, -1, %r7087, %p1738; + and.b32 %r7089, %r7088, %r9882; + shl.b32 %r7090, %r7089, %r10463; + or.b32 %r10464, %r7090, %r10464; + add.s32 %r10463, %r7084, %r10463; + shr.u32 %r9882, %r9882, %r7084; + sub.s32 %r9881, %r9881, %r7084; + setp.lt.u32 %p1739, %r10463, %r10462; + @%p1739 bra $L__BB2_1363; + + cvt.u64.u32 %rd1116, %r10461; + add.s64 %rd1117, %rd1116, %rd5; + add.s64 %rd1118, %rd1, %rd1117; + st.global.u8 [%rd1118], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p1740, %r10464, 255; + selp.b32 %r10462, 7, 8, %p1740; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1363: + setp.ne.s32 %p1741, %r9881, 0; + mov.u32 %r9895, %r9880; + @%p1741 bra $L__BB2_1360; + +$L__BB2_1364: + setp.eq.s32 %p1742, %r2449, 0; + mov.u32 %r10495, %r9895; + @%p1742 bra $L__BB2_1371; + + shr.u32 %r7093, %r2562, 3; + sub.s32 %r9896, %r2561, %r7093; + setp.eq.s32 %p1743, %r9896, 0; + mov.u32 %r10495, %r9895; + @%p1743 bra $L__BB2_1371; + + mov.u32 %r7094, -1; + shl.b32 %r7095, %r7094, %r9896; + not.b32 %r7096, %r7095; + and.b32 %r9897, %r9783, %r7096; + +$L__BB2_1367: + setp.gt.u32 %p1744, %r10461, 17476; + mov.u32 %r10495, 1; + @%p1744 bra $L__BB2_1371; + + sub.s32 %r7098, %r10462, %r10463; + min.u32 %r7099, %r7098, %r9896; + setp.eq.s32 %p1745, %r7099, 32; + mov.u32 %r7100, -1; + shl.b32 %r7101, %r7100, %r7099; + not.b32 %r7102, %r7101; + selp.b32 %r7103, -1, %r7102, %p1745; + and.b32 %r7104, %r7103, %r9897; + shl.b32 %r7105, %r7104, %r10463; + or.b32 %r10464, %r7105, %r10464; + add.s32 %r10463, %r7099, %r10463; + shr.u32 %r9897, %r9897, %r7099; + sub.s32 %r9896, %r9896, %r7099; + setp.lt.u32 %p1746, %r10463, %r10462; + @%p1746 bra $L__BB2_1370; + + cvt.u64.u32 %rd1119, %r10461; + add.s64 %rd1120, %rd1119, %rd5; + add.s64 %rd1121, %rd1, %rd1120; + st.global.u8 [%rd1121], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p1747, %r10464, 255; + selp.b32 %r10462, 7, 8, %p1747; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1370: + setp.ne.s32 %p1748, %r9896, 0; + mov.u32 %r10495, %r9895; + @%p1748 bra $L__BB2_1367; + +$L__BB2_1371: + add.s32 %r7108, %r9744, 2; + setp.lt.u32 %p1749, %r7108, %r4057; + mul.lo.s32 %r2656, %r2443, 6; + cvt.u64.u32 %rd1122, %r2656; + add.s64 %rd65, %rd63, %rd1122; + add.s32 %r7109, %r2656, 2; + cvt.u64.u32 %rd1123, %r7109; + add.s64 %rd66, %rd63, %rd1123; + @%p1749 bra $L__BB2_1400; + bra.uni $L__BB2_1372; + +$L__BB2_1400: + cvt.u64.u32 %rd1137, %r9761; + add.s64 %rd1138, %rd1137, %rd4; + shl.b64 %rd1139, %rd1138, 2; + add.s64 %rd1140, %rd3, %rd1139; + ld.global.u32 %r2729, [%rd1140]; + setp.eq.s32 %p1786, %r2729, 0; + mov.u32 %r9956, 0; + mov.u32 %r9955, %r9956; + @%p1786 bra $L__BB2_1402; + + and.b32 %r7180, %r2729, -2147483648; + abs.s32 %r7181, %r2729; + shl.b32 %r7182, %r7181, %r2358; + or.b32 %r9955, %r7182, %r7180; + +$L__BB2_1402: + shl.b32 %r7186, %r9955, 1; + shr.u32 %r7187, %r7186, %r2358; + and.b32 %r2732, %r7187, -2; + setp.eq.s32 %p1787, %r2732, 0; + mov.u32 %r9957, %r9956; + mov.u32 %r9963, %r9956; + @%p1787 bra $L__BB2_1404; + + add.s32 %r7189, %r2732, -1; + clz.b32 %r7190, %r7189; + mov.u32 %r7191, 32; + sub.s32 %r9956, %r7191, %r7190; + shr.u32 %r7192, %r9955, 31; + add.s32 %r7193, %r7192, %r2732; + add.s32 %r9957, %r7193, -2; + mov.u32 %r9963, 1; + +$L__BB2_1404: + mov.u32 %r9960, 0; + mov.u32 %r9959, %r9960; + @%p1639 bra $L__BB2_1407; + + add.s32 %r7196, %r9761, %r4055; + cvt.u64.u32 %rd1141, %r7196; + add.s64 %rd1142, %rd1141, %rd4; + shl.b64 %rd1143, %rd1142, 2; + add.s64 %rd1144, %rd3, %rd1143; + ld.global.u32 %r2738, [%rd1144]; + setp.eq.s32 %p1789, %r2738, 0; + @%p1789 bra $L__BB2_1407; + + and.b32 %r7197, %r2738, -2147483648; + abs.s32 %r7198, %r2738; + shl.b32 %r7199, %r7198, %r2358; + or.b32 %r9959, %r7199, %r7197; + +$L__BB2_1407: + shl.b32 %r7202, %r9959, 1; + shr.u32 %r7203, %r7202, %r2358; + and.b32 %r2741, %r7203, -2; + setp.eq.s32 %p1790, %r2741, 0; + mov.u32 %r9961, %r9960; + mov.u32 %r9979, %r9956; + @%p1790 bra $L__BB2_1409; + + or.b32 %r9963, %r9963, 2; + add.s32 %r7204, %r2741, -1; + clz.b32 %r7205, %r7204; + mov.u32 %r7206, 32; + sub.s32 %r9960, %r7206, %r7205; + max.s32 %r9979, %r9956, %r9960; + shr.u32 %r7207, %r9959, 31; + add.s32 %r7208, %r7207, %r2741; + add.s32 %r9961, %r7208, -2; + +$L__BB2_1409: + add.s32 %r9978, %r9761, 1; + add.s32 %r7213, %r9744, 3; + setp.ge.u32 %p1791, %r7213, %r4057; + mov.u32 %r9981, 0; + mov.u32 %r9974, %r9981; + mov.u32 %r9975, %r9981; + mov.u32 %r9976, %r9981; + mov.u32 %r9977, %r9981; + @%p1791 bra $L__BB2_1420; + + cvt.u64.u32 %rd1145, %r9978; + add.s64 %rd1146, %rd1145, %rd4; + shl.b64 %rd1147, %rd1146, 2; + add.s64 %rd1148, %rd3, %rd1147; + ld.global.u32 %r2751, [%rd1148]; + setp.eq.s32 %p1792, %r2751, 0; + mov.u32 %r9975, 0; + mov.u32 %r9964, %r9975; + @%p1792 bra $L__BB2_1412; + + and.b32 %r7215, %r2751, -2147483648; + abs.s32 %r7216, %r2751; + shl.b32 %r7217, %r7216, %r2358; + or.b32 %r9964, %r7217, %r7215; + +$L__BB2_1412: + shl.b32 %r7220, %r9964, 1; + shr.u32 %r7221, %r7220, %r2358; + and.b32 %r2754, %r7221, -2; + setp.eq.s32 %p1793, %r2754, 0; + mov.u32 %r9977, %r9975; + @%p1793 bra $L__BB2_1414; + + or.b32 %r9963, %r9963, 4; + add.s32 %r7222, %r2754, -1; + clz.b32 %r7223, %r7222; + mov.u32 %r7224, 32; + sub.s32 %r9975, %r7224, %r7223; + max.s32 %r9979, %r9979, %r9975; + shr.u32 %r7225, %r9964, 31; + add.s32 %r7226, %r7225, %r2754; + add.s32 %r9977, %r7226, -2; + +$L__BB2_1414: + mov.u32 %r9974, 0; + mov.u32 %r9969, %r9974; + @%p1639 bra $L__BB2_1417; + + add.s32 %r7229, %r9978, %r4055; + cvt.u64.u32 %rd1149, %r7229; + add.s64 %rd1150, %rd1149, %rd4; + shl.b64 %rd1151, %rd1150, 2; + add.s64 %rd1152, %rd3, %rd1151; + ld.global.u32 %r2763, [%rd1152]; + setp.eq.s32 %p1795, %r2763, 0; + @%p1795 bra $L__BB2_1417; + + and.b32 %r7230, %r2763, -2147483648; + abs.s32 %r7231, %r2763; + shl.b32 %r7232, %r7231, %r2358; + or.b32 %r9969, %r7232, %r7230; + +$L__BB2_1417: + shl.b32 %r7235, %r9969, 1; + shr.u32 %r7236, %r7235, %r2358; + and.b32 %r2766, %r7236, -2; + setp.eq.s32 %p1796, %r2766, 0; + mov.u32 %r9976, %r9974; + @%p1796 bra $L__BB2_1419; + + or.b32 %r9963, %r9963, 8; + add.s32 %r7237, %r2766, -1; + clz.b32 %r7238, %r7237; + mov.u32 %r7239, 32; + sub.s32 %r9974, %r7239, %r7238; + max.s32 %r9979, %r9979, %r9974; + shr.u32 %r7240, %r9969, 31; + add.s32 %r7241, %r7240, %r2766; + add.s32 %r9976, %r7241, -2; + +$L__BB2_1419: + add.s32 %r9978, %r9761, 2; + +$L__BB2_1420: + mov.u32 %r9761, %r9978; + shr.u32 %r7243, %r9770, 1; + or.b32 %r2783, %r7243, %r2563; + add.s32 %r7244, %r9979, -1; + setp.lt.s32 %p1797, %r9979, 2; + setp.gt.s32 %p1798, %r9979, 1; + selp.b32 %r2784, %r7244, 0, %p1798; + @%p1797 bra $L__BB2_1422; + + setp.eq.s32 %p1799, %r9956, %r9979; + selp.u32 %r7245, 1, 0, %p1799; + setp.eq.s32 %p1800, %r9960, %r9979; + selp.u32 %r7246, -1, 0, %p1800; + bfi.b32 %r7247, %r7246, %r7245, 1, 1; + setp.eq.s32 %p1801, %r9975, %r9979; + selp.u16 %rs886, 1, 0, %p1801; + mul.wide.u16 %r7248, %rs886, 4; + or.b32 %r7249, %r7247, %r7248; + setp.eq.s32 %p1802, %r9974, %r9979; + selp.u16 %rs887, 1, 0, %p1802; + mul.wide.u16 %r7250, %rs887, 8; + or.b32 %r9981, %r7249, %r7250; + +$L__BB2_1422: + and.b32 %r7251, %r9960, 255; + and.b32 %r7252, %r9781, 255; + setp.lt.u32 %p1803, %r7251, %r7252; + cvt.u16.u32 %rs888, %r9960; + selp.b16 %rs889, %rs274, %rs888, %p1803; + st.shared.u8 [%r2446+1], %rs889; + st.shared.u8 [%r2446+2], %r9974; + and.b32 %r2787, %r9963, 2; + shr.u32 %r7253, %r2787, 1; + or.b32 %r7254, %r2450, %r7253; + st.shared.u8 [%r2448+1], %r7254; + and.b32 %r2788, %r9963, 8; + shr.u32 %r7255, %r2788, 3; + st.shared.u8 [%r2448+2], %r7255; + shl.b32 %r7256, %r9963, 4; + shl.b32 %r7257, %r2783, 8; + or.b32 %r7258, %r7256, %r7257; + or.b32 %r7259, %r7258, %r9981; + mul.wide.u32 %rd1154, %r7259, 2; + add.s64 %rd1155, %rd62, %rd1154; + ld.global.u16 %rs319, [%rd1155]; + shr.u16 %rs890, %rs319, 4; + and.b16 %rs320, %rs890, 7; + setp.eq.s16 %p1804, %rs320, 0; + mov.u32 %r9993, %r9800; + @%p1804 bra $L__BB2_1429; + + cvt.u32.u16 %r9982, %rs320; + shr.u16 %rs891, %rs319, 8; + cvt.u32.u16 %r9983, %rs891; + +$L__BB2_1424: + mov.u32 %r2791, %r9982; + setp.gt.u32 %p1805, %r10274, 2879; + mov.u32 %r9993, 1; + @%p1805 bra $L__BB2_1429; + + mov.u32 %r7261, 8; + sub.s32 %r7262, %r7261, %r10276; + sub.s32 %r7263, %r7262, %r10275; + min.u32 %r7264, %r7263, %r2791; + setp.eq.s32 %p1806, %r7264, 32; + mov.u32 %r7265, -1; + shl.b32 %r7266, %r7265, %r7264; + not.b32 %r7267, %r7266; + selp.b32 %r7268, -1, %r7267, %p1806; + and.b32 %r7269, %r7268, %r9983; + shl.b32 %r7270, %r7269, %r10275; + cvt.u16.u32 %rs892, %r7270; + or.b16 %rs1322, %rs1322, %rs892; + add.s32 %r10275, %r7264, %r10275; + sub.s32 %r9982, %r2791, %r7264; + shr.u32 %r9983, %r9983, %r7264; + setp.gt.u32 %p1807, %r7263, %r2791; + @%p1807 bra $L__BB2_1428; + + setp.ne.s32 %p1808, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs893, %rs1322, 255; + setp.ne.s16 %p1809, %rs893, 127; + and.pred %p1810, %p1808, %p1809; + @%p1810 bra $L__BB2_1428; + + mov.u32 %r7273, 20548; + sub.s32 %r7274, %r7273, %r10274; + cvt.u64.u32 %rd1156, %r7274; + add.s64 %rd1157, %rd1156, %rd5; + add.s64 %rd1158, %rd1, %rd1157; + st.global.u8 [%rd1158], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p1811, %rs893, 143; + selp.u32 %r10276, 1, 0, %p1811; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1428: + setp.ne.s32 %p1812, %r9982, 0; + mov.u32 %r9993, %r9800; + @%p1812 bra $L__BB2_1424; + +$L__BB2_1429: + setp.ne.s32 %p1813, %r2783, 0; + @%p1813 bra $L__BB2_1477; + + setp.eq.s32 %p1814, %r9963, 0; + add.s32 %r7275, %r9826, 17477; + cvt.u64.u32 %rd1159, %r7275; + add.s64 %rd1160, %rd1159, %rd5; + add.s64 %rd67, %rd1, %rd1160; + @%p1814 bra $L__BB2_1469; + + shl.b16 %rs1253, %rs1253, 1; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1815, %r9835, 0; + mov.u32 %r10029, %r10043; + @%p1815 bra $L__BB2_1434; + bra.uni $L__BB2_1432; + +$L__BB2_1434: + setp.lt.u32 %p1817, %r10041, 3; + mov.u32 %r9997, 0; + @%p1817 bra $L__BB2_1437; + + setp.lt.u32 %p1818, %r10041, 6; + mov.u32 %r9997, 1; + @%p1818 bra $L__BB2_1437; + + setp.lt.u32 %p1819, %r10041, 9; + setp.eq.s32 %p1820, %r10041, 11; + selp.b32 %r7281, 4, 5, %p1820; + setp.lt.u32 %p1821, %r10041, 11; + selp.b32 %r7282, 3, %r7281, %p1821; + selp.b32 %r9997, 2, %r7282, %p1819; + +$L__BB2_1437: + setp.eq.s32 %p1822, %r9997, 0; + @%p1822 bra $L__BB2_1465; + + add.s32 %r2815, %r9997, -1; + and.b32 %r2816, %r9997, 3; + setp.eq.s32 %p1823, %r2816, 0; + mov.u32 %r10007, %r9997; + mov.u32 %r10008, %r10029; + @%p1823 bra $L__BB2_1450; + + mov.u32 %r7284, 1; + shl.b32 %r7285, %r7284, %r2815; + and.b32 %r7286, %r7285, %r10040; + setp.ne.s32 %p1824, %r7286, 0; + selp.u32 %r7287, 1, 0, %p1824; + cvt.u32.u16 %r7288, %rs1253; + bfi.b32 %r7289, %r7288, %r7287, 1, 8; + cvt.u16.u32 %rs1253, %r7289; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1825, %r9835, 0; + mov.u32 %r10008, %r10029; + @%p1825 bra $L__BB2_1442; + + setp.gt.u32 %p1826, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r10008, %r7284; + @%p1826 bra $L__BB2_1442; + + add.s32 %r7293, %r9826, 17477; + cvt.u64.u32 %rd1161, %r7293; + add.s64 %rd1162, %rd1161, %rd5; + add.s64 %rd1163, %rd1, %rd1162; + st.global.u8 [%rd1163], %rs1253; + add.s32 %r9826, %r9826, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9835, 8; + mov.u32 %r10008, %r10029; + +$L__BB2_1442: + setp.eq.s32 %p1827, %r2816, 1; + mov.u32 %r10029, %r10008; + mov.u32 %r10007, %r2815; + @%p1827 bra $L__BB2_1450; + + add.s32 %r10007, %r9997, -2; + mov.u32 %r7294, 1; + shl.b32 %r7295, %r7294, %r10007; + and.b32 %r7296, %r7295, %r10040; + setp.ne.s32 %p1828, %r7296, 0; + selp.u32 %r7297, 1, 0, %p1828; + cvt.u32.u16 %r7298, %rs1253; + bfi.b32 %r7299, %r7298, %r7297, 1, 8; + cvt.u16.u32 %rs1253, %r7299; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1829, %r9835, 0; + mov.u32 %r10003, %r10008; + @%p1829 bra $L__BB2_1446; + + setp.gt.u32 %p1830, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r10003, %r7294; + @%p1830 bra $L__BB2_1446; + + add.s32 %r7302, %r9826, 17477; + cvt.u64.u32 %rd1164, %r7302; + add.s64 %rd1165, %rd1164, %rd5; + add.s64 %rd1166, %rd1, %rd1165; + and.b16 %rs900, %rs1253, 255; + st.global.u8 [%rd1166], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1831, %rs900, 255; + selp.b32 %r9835, 7, 8, %p1831; + mov.u16 %rs1253, 0; + mov.u32 %r10003, %r10008; + +$L__BB2_1446: + setp.eq.s32 %p1832, %r2816, 2; + mov.u32 %r10029, %r10003; + mov.u32 %r10008, %r10003; + @%p1832 bra $L__BB2_1450; + + add.s32 %r10007, %r9997, -3; + mov.u32 %r7303, 1; + shl.b32 %r7304, %r7303, %r10007; + and.b32 %r7305, %r7304, %r10040; + setp.ne.s32 %p1833, %r7305, 0; + selp.u32 %r7306, 1, 0, %p1833; + cvt.u32.u16 %r7307, %rs1253; + bfi.b32 %r7308, %r7307, %r7306, 1, 8; + cvt.u16.u32 %rs1253, %r7308; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1834, %r9835, 0; + mov.u32 %r10029, %r10003; + mov.u32 %r10008, %r10003; + @%p1834 bra $L__BB2_1450; + + setp.gt.u32 %p1835, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r10029, %r7303; + mov.u32 %r10008, %r7303; + @%p1835 bra $L__BB2_1450; + + add.s32 %r7313, %r9826, 17477; + cvt.u64.u32 %rd1167, %r7313; + add.s64 %rd1168, %rd1167, %rd5; + add.s64 %rd1169, %rd1, %rd1168; + and.b16 %rs903, %rs1253, 255; + st.global.u8 [%rd1169], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1836, %rs903, 255; + selp.b32 %r9835, 7, 8, %p1836; + mov.u16 %rs1253, 0; + mov.u32 %r10029, %r10003; + mov.u32 %r10008, %r10003; + +$L__BB2_1450: + setp.lt.u32 %p1837, %r2815, 3; + @%p1837 bra $L__BB2_1465; + + mov.u32 %r10029, %r10008; + +$L__BB2_1452: + add.s32 %r7314, %r10007, -1; + mov.u32 %r7315, 1; + shl.b32 %r7316, %r7315, %r7314; + and.b32 %r7317, %r7316, %r10040; + setp.ne.s32 %p1838, %r7317, 0; + selp.u32 %r7318, 1, 0, %p1838; + cvt.u32.u16 %r7319, %rs1253; + bfi.b32 %r10017, %r7319, %r7318, 1, 8; + add.s32 %r10016, %r9835, -1; + setp.ne.s32 %p1839, %r10016, 0; + mov.u32 %r10018, %r10029; + @%p1839 bra $L__BB2_1455; + + setp.gt.u32 %p1840, %r9826, 191; + mov.u32 %r10016, 0; + mov.u32 %r10018, %r7315; + @%p1840 bra $L__BB2_1455; + + cvt.u16.u32 %rs904, %r10017; + and.b16 %rs905, %rs904, 255; + add.s32 %r7323, %r9826, 17477; + cvt.u64.u32 %rd1170, %r7323; + add.s64 %rd1171, %rd1170, %rd5; + add.s64 %rd1172, %rd1, %rd1171; + st.global.u8 [%rd1172], %rs904; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1841, %rs905, 255; + selp.b32 %r10016, 7, 8, %p1841; + mov.u32 %r10017, 0; + mov.u32 %r10018, %r10029; + +$L__BB2_1455: + add.s32 %r7324, %r10007, -2; + shl.b32 %r7326, %r7315, %r7324; + and.b32 %r7327, %r7326, %r10040; + setp.ne.s32 %p1842, %r7327, 0; + and.b32 %r7328, %r10017, 127; + selp.u32 %r7329, 1, 0, %p1842; + bfi.b32 %r10021, %r7328, %r7329, 1, 7; + add.s32 %r10020, %r10016, -1; + setp.ne.s32 %p1843, %r10020, 0; + mov.u32 %r10022, %r10018; + @%p1843 bra $L__BB2_1458; + + setp.gt.u32 %p1844, %r9826, 191; + mov.u32 %r10022, 1; + mov.u32 %r10020, 0; + @%p1844 bra $L__BB2_1458; + + cvt.u16.u32 %rs906, %r10021; + and.b16 %rs907, %rs906, 255; + add.s32 %r7333, %r9826, 17477; + cvt.u64.u32 %rd1173, %r7333; + add.s64 %rd1174, %rd1173, %rd5; + add.s64 %rd1175, %rd1, %rd1174; + st.global.u8 [%rd1175], %rs906; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1845, %rs907, 255; + selp.b32 %r10020, 7, 8, %p1845; + mov.u32 %r10021, 0; + mov.u32 %r10022, %r10018; + +$L__BB2_1458: + add.s32 %r7334, %r10007, -3; + mov.u32 %r7335, 1; + shl.b32 %r7336, %r7335, %r7334; + and.b32 %r7337, %r7336, %r10040; + setp.ne.s32 %p1846, %r7337, 0; + and.b32 %r7338, %r10021, 127; + selp.u32 %r7339, 1, 0, %p1846; + bfi.b32 %r10025, %r7338, %r7339, 1, 7; + add.s32 %r10024, %r10020, -1; + setp.ne.s32 %p1847, %r10024, 0; + mov.u32 %r10026, %r10022; + @%p1847 bra $L__BB2_1461; + + setp.gt.u32 %p1848, %r9826, 191; + mov.u32 %r10024, 0; + mov.u32 %r10026, %r7335; + @%p1848 bra $L__BB2_1461; + + cvt.u16.u32 %rs908, %r10025; + and.b16 %rs909, %rs908, 255; + add.s32 %r7343, %r9826, 17477; + cvt.u64.u32 %rd1176, %r7343; + add.s64 %rd1177, %rd1176, %rd5; + add.s64 %rd1178, %rd1, %rd1177; + st.global.u8 [%rd1178], %rs908; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1849, %rs909, 255; + selp.b32 %r10024, 7, 8, %p1849; + mov.u32 %r10025, 0; + mov.u32 %r10026, %r10022; + +$L__BB2_1461: + add.s32 %r10007, %r10007, -4; + shl.b32 %r7345, %r7335, %r10007; + and.b32 %r7346, %r7345, %r10040; + setp.ne.s32 %p1850, %r7346, 0; + and.b32 %r7347, %r10025, 127; + selp.u32 %r7348, 1, 0, %p1850; + bfi.b32 %r7349, %r7347, %r7348, 1, 15; + cvt.u16.u32 %rs1253, %r7349; + add.s32 %r9835, %r10024, -1; + setp.ne.s32 %p1851, %r9835, 0; + mov.u32 %r10029, %r10026; + @%p1851 bra $L__BB2_1464; + + setp.gt.u32 %p1852, %r9826, 191; + mov.u32 %r10029, 1; + mov.u32 %r9835, 0; + @%p1852 bra $L__BB2_1464; + + add.s32 %r7352, %r9826, 17477; + cvt.u64.u32 %rd1179, %r7352; + add.s64 %rd1180, %rd1179, %rd5; + add.s64 %rd1181, %rd1, %rd1180; + and.b16 %rs911, %rs1253, 255; + st.global.u8 [%rd1181], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1853, %rs911, 255; + selp.b32 %r9835, 7, 8, %p1853; + mov.u16 %rs1253, 0; + mov.u32 %r10029, %r10026; + +$L__BB2_1464: + setp.ne.s32 %p1854, %r10007, 0; + @%p1854 bra $L__BB2_1452; + +$L__BB2_1465: + add.s32 %r7354, %r10041, -1; + setp.eq.s32 %p1855, %r10041, 0; + mov.u32 %r10040, 0; + selp.b32 %r10041, 0, %r7354, %p1855; + setp.lt.u32 %p1856, %r10041, 3; + mov.u32 %r10033, %r10040; + @%p1856 bra $L__BB2_1468; + + setp.lt.u32 %p1857, %r10041, 6; + mov.u32 %r10033, 1; + @%p1857 bra $L__BB2_1468; + + setp.lt.u32 %p1858, %r10041, 9; + setp.eq.s32 %p1859, %r10041, 11; + selp.b32 %r7356, 4, 5, %p1859; + setp.lt.u32 %p1860, %r10041, 11; + selp.b32 %r7357, 3, %r7356, %p1860; + selp.b32 %r10033, 2, %r7357, %p1858; + +$L__BB2_1468: + mov.u32 %r7359, 1; + shl.b32 %r10042, %r7359, %r10033; + mov.u32 %r10043, %r10029; + bra.uni $L__BB2_1477; + +$L__BB2_1372: + ld.global.u8 %rs297, [%rd65+1]; + ld.global.u8 %rs298, [%rd66]; + ld.global.u8 %rs299, [%rd66+1]; + ld.global.u8 %rs300, [%rd63]; + ld.global.u8 %rs301, [%rd63+1]; + ld.global.u8 %rs302, [%rd63+2]; + ld.global.u8 %rs303, [%rd63+3]; + setp.eq.s16 %p1750, %rs297, 0; + mov.u32 %r9922, %r9800; + @%p1750 bra $L__BB2_1379; + + ld.global.u8 %r9912, [%rd65]; + cvt.u32.u16 %r9911, %rs297; + +$L__BB2_1374: + mov.u32 %r2659, %r9911; + setp.gt.u32 %p1751, %r10274, 2879; + mov.u32 %r9922, 1; + @%p1751 bra $L__BB2_1379; + + mov.u32 %r7111, 8; + sub.s32 %r7112, %r7111, %r10276; + sub.s32 %r7113, %r7112, %r10275; + min.u32 %r7114, %r7113, %r2659; + setp.eq.s32 %p1752, %r7114, 32; + mov.u32 %r7115, -1; + shl.b32 %r7116, %r7115, %r7114; + not.b32 %r7117, %r7116; + selp.b32 %r7118, -1, %r7117, %p1752; + and.b32 %r7119, %r7118, %r9912; + shl.b32 %r7120, %r7119, %r10275; + cvt.u16.u32 %rs867, %r7120; + or.b16 %rs1322, %rs1322, %rs867; + add.s32 %r10275, %r7114, %r10275; + sub.s32 %r9911, %r2659, %r7114; + shr.u32 %r9912, %r9912, %r7114; + setp.gt.u32 %p1753, %r7113, %r2659; + @%p1753 bra $L__BB2_1378; + + setp.ne.s32 %p1754, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs868, %rs1322, 255; + setp.ne.s16 %p1755, %rs868, 127; + and.pred %p1756, %p1754, %p1755; + @%p1756 bra $L__BB2_1378; + + mov.u32 %r7123, 20548; + sub.s32 %r7124, %r7123, %r10274; + cvt.u64.u32 %rd1125, %r7124; + add.s64 %rd1126, %rd1125, %rd5; + add.s64 %rd1127, %rd1, %rd1126; + st.global.u8 [%rd1127], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p1757, %rs868, 143; + selp.u32 %r10276, 1, 0, %p1757; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1378: + setp.ne.s32 %p1758, %r9911, 0; + mov.u32 %r9922, %r9800; + @%p1758 bra $L__BB2_1374; + +$L__BB2_1379: + setp.eq.s16 %p1759, %rs301, 0; + mov.u32 %r9934, %r9922; + @%p1759 bra $L__BB2_1386; + + cvt.u32.u16 %r7125, %rs300; + and.b32 %r9924, %r7125, 255; + cvt.u32.u16 %r7126, %rs301; + and.b32 %r9923, %r7126, 255; + +$L__BB2_1381: + mov.u32 %r2678, %r9923; + setp.gt.u32 %p1760, %r10274, 2879; + mov.u32 %r9934, 1; + @%p1760 bra $L__BB2_1386; + + mov.u32 %r7128, 8; + sub.s32 %r7129, %r7128, %r10276; + sub.s32 %r7130, %r7129, %r10275; + min.u32 %r7131, %r7130, %r2678; + setp.eq.s32 %p1761, %r7131, 32; + mov.u32 %r7132, -1; + shl.b32 %r7133, %r7132, %r7131; + not.b32 %r7134, %r7133; + selp.b32 %r7135, -1, %r7134, %p1761; + and.b32 %r7136, %r7135, %r9924; + shl.b32 %r7137, %r7136, %r10275; + cvt.u16.u32 %rs872, %r7137; + or.b16 %rs1322, %rs1322, %rs872; + add.s32 %r10275, %r7131, %r10275; + sub.s32 %r9923, %r2678, %r7131; + shr.u32 %r9924, %r9924, %r7131; + setp.gt.u32 %p1762, %r7130, %r2678; + @%p1762 bra $L__BB2_1385; + + setp.ne.s32 %p1763, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs873, %rs1322, 255; + setp.ne.s16 %p1764, %rs873, 127; + and.pred %p1765, %p1763, %p1764; + @%p1765 bra $L__BB2_1385; + + mov.u32 %r7140, 20548; + sub.s32 %r7141, %r7140, %r10274; + cvt.u64.u32 %rd1128, %r7141; + add.s64 %rd1129, %rd1128, %rd5; + add.s64 %rd1130, %rd1, %rd1129; + st.global.u8 [%rd1130], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p1766, %rs873, 143; + selp.u32 %r10276, 1, 0, %p1766; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1385: + setp.ne.s32 %p1767, %r9923, 0; + mov.u32 %r9934, %r9922; + @%p1767 bra $L__BB2_1381; + +$L__BB2_1386: + setp.eq.s16 %p1768, %rs299, 0; + mov.u32 %r9946, %r9934; + @%p1768 bra $L__BB2_1393; + + cvt.u32.u16 %r7142, %rs299; + and.b32 %r9935, %r7142, 255; + cvt.u32.u16 %r7143, %rs298; + and.b32 %r9936, %r7143, 255; + +$L__BB2_1388: + mov.u32 %r2697, %r9935; + setp.gt.u32 %p1769, %r10274, 2879; + mov.u32 %r9946, 1; + @%p1769 bra $L__BB2_1393; + + mov.u32 %r7145, 8; + sub.s32 %r7146, %r7145, %r10276; + sub.s32 %r7147, %r7146, %r10275; + min.u32 %r7148, %r7147, %r2697; + setp.eq.s32 %p1770, %r7148, 32; + mov.u32 %r7149, -1; + shl.b32 %r7150, %r7149, %r7148; + not.b32 %r7151, %r7150; + selp.b32 %r7152, -1, %r7151, %p1770; + and.b32 %r7153, %r7152, %r9936; + shl.b32 %r7154, %r7153, %r10275; + cvt.u16.u32 %rs877, %r7154; + or.b16 %rs1322, %rs1322, %rs877; + add.s32 %r10275, %r7148, %r10275; + sub.s32 %r9935, %r2697, %r7148; + shr.u32 %r9936, %r9936, %r7148; + setp.gt.u32 %p1771, %r7147, %r2697; + @%p1771 bra $L__BB2_1392; + + setp.ne.s32 %p1772, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs878, %rs1322, 255; + setp.ne.s16 %p1773, %rs878, 127; + and.pred %p1774, %p1772, %p1773; + @%p1774 bra $L__BB2_1392; + + mov.u32 %r7157, 20548; + sub.s32 %r7158, %r7157, %r10274; + cvt.u64.u32 %rd1131, %r7158; + add.s64 %rd1132, %rd1131, %rd5; + add.s64 %rd1133, %rd1, %rd1132; + st.global.u8 [%rd1133], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p1775, %rs878, 143; + selp.u32 %r10276, 1, 0, %p1775; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1392: + setp.ne.s32 %p1776, %r9935, 0; + mov.u32 %r9946, %r9934; + @%p1776 bra $L__BB2_1388; + +$L__BB2_1393: + setp.eq.s16 %p1777, %rs303, 0; + mov.u32 %r9760, 0; + mov.u32 %r10277, %r9946; + @%p1777 bra $L__BB2_1631; + + cvt.u32.u16 %r7160, %rs302; + and.b32 %r9948, %r7160, 255; + cvt.u32.u16 %r7161, %rs303; + and.b32 %r9947, %r7161, 255; + +$L__BB2_1395: + mov.u32 %r2716, %r9947; + setp.gt.u32 %p1778, %r10274, 2879; + mov.u32 %r10277, 1; + @%p1778 bra $L__BB2_1631; + + mov.u32 %r7164, 8; + sub.s32 %r7165, %r7164, %r10276; + sub.s32 %r7166, %r7165, %r10275; + min.u32 %r7167, %r7166, %r2716; + setp.eq.s32 %p1779, %r7167, 32; + mov.u32 %r7168, -1; + shl.b32 %r7169, %r7168, %r7167; + not.b32 %r7170, %r7169; + selp.b32 %r7171, -1, %r7170, %p1779; + and.b32 %r7172, %r7171, %r9948; + shl.b32 %r7173, %r7172, %r10275; + cvt.u16.u32 %rs882, %r7173; + or.b16 %rs1322, %rs1322, %rs882; + add.s32 %r10275, %r7167, %r10275; + sub.s32 %r9947, %r2716, %r7167; + shr.u32 %r9948, %r9948, %r7167; + setp.gt.u32 %p1780, %r7166, %r2716; + @%p1780 bra $L__BB2_1399; + + setp.ne.s32 %p1781, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs883, %rs1322, 255; + setp.ne.s16 %p1782, %rs883, 127; + and.pred %p1783, %p1781, %p1782; + @%p1783 bra $L__BB2_1399; + + mov.u32 %r7176, 20548; + sub.s32 %r7177, %r7176, %r10274; + cvt.u64.u32 %rd1134, %r7177; + add.s64 %rd1135, %rd1134, %rd5; + add.s64 %rd1136, %rd1, %rd1135; + st.global.u8 [%rd1136], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p1784, %rs883, 143; + selp.u32 %r10276, 1, 0, %p1784; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1399: + setp.eq.s32 %p1785, %r9947, 0; + mov.u32 %r10277, %r9946; + @%p1785 bra $L__BB2_1631; + bra.uni $L__BB2_1395; + +$L__BB2_1469: + add.s32 %r10040, %r10040, 1; + setp.lt.u32 %p1861, %r10040, %r10042; + @%p1861 bra $L__BB2_1477; + + shl.b16 %rs912, %rs1253, 1; + or.b16 %rs1253, %rs912, 1; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1862, %r9835, 0; + mov.u32 %r10036, %r10043; + @%p1862 bra $L__BB2_1473; + bra.uni $L__BB2_1471; + +$L__BB2_1473: + add.s32 %r7363, %r10041, 1; + min.u32 %r10041, %r7363, 12; + setp.lt.u32 %p1865, %r10041, 3; + mov.u32 %r10040, 0; + mov.u32 %r10037, %r10040; + @%p1865 bra $L__BB2_1476; + + setp.lt.u32 %p1866, %r10041, 6; + mov.u32 %r10037, 1; + @%p1866 bra $L__BB2_1476; + + setp.lt.u32 %p1867, %r10041, 9; + setp.eq.s32 %p1868, %r10041, 11; + selp.b32 %r7365, 4, 5, %p1868; + setp.lt.u32 %p1869, %r10041, 11; + selp.b32 %r7366, 3, %r7365, %p1869; + selp.b32 %r10037, 2, %r7366, %p1867; + +$L__BB2_1476: + mov.u32 %r7368, 1; + shl.b32 %r10042, %r7368, %r10037; + mov.u32 %r10043, %r10036; + +$L__BB2_1477: + max.s32 %r2899, %r9979, 1; + and.b16 %rs915, %rs319, 15; + cvt.u32.u16 %r2900, %rs915; + and.b32 %r2901, %r9963, 1; + setp.eq.s32 %p1870, %r2901, 0; + mov.u32 %r10058, %r10495; + @%p1870 bra $L__BB2_1484; + + and.b32 %r7369, %r2900, 1; + sub.s32 %r10044, %r2899, %r7369; + setp.eq.s32 %p1871, %r10044, 0; + mov.u32 %r10058, %r10495; + @%p1871 bra $L__BB2_1484; + + mov.u32 %r7370, -1; + shl.b32 %r7371, %r7370, %r10044; + not.b32 %r7372, %r7371; + and.b32 %r10045, %r9957, %r7372; + +$L__BB2_1480: + setp.gt.u32 %p1872, %r10461, 17476; + mov.u32 %r10058, 1; + @%p1872 bra $L__BB2_1484; + + sub.s32 %r7374, %r10462, %r10463; + min.u32 %r7375, %r7374, %r10044; + setp.eq.s32 %p1873, %r7375, 32; + mov.u32 %r7376, -1; + shl.b32 %r7377, %r7376, %r7375; + not.b32 %r7378, %r7377; + selp.b32 %r7379, -1, %r7378, %p1873; + and.b32 %r7380, %r7379, %r10045; + shl.b32 %r7381, %r7380, %r10463; + or.b32 %r10464, %r7381, %r10464; + add.s32 %r10463, %r7375, %r10463; + shr.u32 %r10045, %r10045, %r7375; + sub.s32 %r10044, %r10044, %r7375; + setp.lt.u32 %p1874, %r10463, %r10462; + @%p1874 bra $L__BB2_1483; + + cvt.u64.u32 %rd1182, %r10461; + add.s64 %rd1183, %rd1182, %rd5; + add.s64 %rd1184, %rd1, %rd1183; + st.global.u8 [%rd1184], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p1875, %r10464, 255; + selp.b32 %r10462, 7, 8, %p1875; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1483: + setp.ne.s32 %p1876, %r10044, 0; + mov.u32 %r10058, %r10495; + @%p1876 bra $L__BB2_1480; + +$L__BB2_1484: + setp.eq.s32 %p1877, %r2787, 0; + mov.u32 %r10073, %r10058; + @%p1877 bra $L__BB2_1491; + + shr.u32 %r7384, %r2900, 1; + and.b32 %r7385, %r7384, 1; + sub.s32 %r10059, %r2899, %r7385; + setp.eq.s32 %p1878, %r10059, 0; + mov.u32 %r10073, %r10058; + @%p1878 bra $L__BB2_1491; + + mov.u32 %r7386, -1; + shl.b32 %r7387, %r7386, %r10059; + not.b32 %r7388, %r7387; + and.b32 %r10060, %r9961, %r7388; + +$L__BB2_1487: + setp.gt.u32 %p1879, %r10461, 17476; + mov.u32 %r10073, 1; + @%p1879 bra $L__BB2_1491; + + sub.s32 %r7390, %r10462, %r10463; + min.u32 %r7391, %r7390, %r10059; + setp.eq.s32 %p1880, %r7391, 32; + mov.u32 %r7392, -1; + shl.b32 %r7393, %r7392, %r7391; + not.b32 %r7394, %r7393; + selp.b32 %r7395, -1, %r7394, %p1880; + and.b32 %r7396, %r7395, %r10060; + shl.b32 %r7397, %r7396, %r10463; + or.b32 %r10464, %r7397, %r10464; + add.s32 %r10463, %r7391, %r10463; + shr.u32 %r10060, %r10060, %r7391; + sub.s32 %r10059, %r10059, %r7391; + setp.lt.u32 %p1881, %r10463, %r10462; + @%p1881 bra $L__BB2_1490; + + cvt.u64.u32 %rd1185, %r10461; + add.s64 %rd1186, %rd1185, %rd5; + add.s64 %rd1187, %rd1, %rd1186; + st.global.u8 [%rd1187], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p1882, %r10464, 255; + selp.b32 %r10462, 7, 8, %p1882; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1490: + setp.ne.s32 %p1883, %r10059, 0; + mov.u32 %r10073, %r10058; + @%p1883 bra $L__BB2_1487; + +$L__BB2_1491: + and.b32 %r7400, %r9963, 4; + setp.eq.s32 %p1884, %r7400, 0; + mov.u32 %r10088, %r10073; + @%p1884 bra $L__BB2_1498; + + shr.u32 %r7401, %r2900, 2; + and.b32 %r7402, %r7401, 1; + sub.s32 %r10074, %r2899, %r7402; + setp.eq.s32 %p1885, %r10074, 0; + mov.u32 %r10088, %r10073; + @%p1885 bra $L__BB2_1498; + + mov.u32 %r7403, -1; + shl.b32 %r7404, %r7403, %r10074; + not.b32 %r7405, %r7404; + and.b32 %r10075, %r9977, %r7405; + +$L__BB2_1494: + setp.gt.u32 %p1886, %r10461, 17476; + mov.u32 %r10088, 1; + @%p1886 bra $L__BB2_1498; + + sub.s32 %r7407, %r10462, %r10463; + min.u32 %r7408, %r7407, %r10074; + setp.eq.s32 %p1887, %r7408, 32; + mov.u32 %r7409, -1; + shl.b32 %r7410, %r7409, %r7408; + not.b32 %r7411, %r7410; + selp.b32 %r7412, -1, %r7411, %p1887; + and.b32 %r7413, %r7412, %r10075; + shl.b32 %r7414, %r7413, %r10463; + or.b32 %r10464, %r7414, %r10464; + add.s32 %r10463, %r7408, %r10463; + shr.u32 %r10075, %r10075, %r7408; + sub.s32 %r10074, %r10074, %r7408; + setp.lt.u32 %p1888, %r10463, %r10462; + @%p1888 bra $L__BB2_1497; + + cvt.u64.u32 %rd1188, %r10461; + add.s64 %rd1189, %rd1188, %rd5; + add.s64 %rd1190, %rd1, %rd1189; + st.global.u8 [%rd1190], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p1889, %r10464, 255; + selp.b32 %r10462, 7, 8, %p1889; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1497: + setp.ne.s32 %p1890, %r10074, 0; + mov.u32 %r10088, %r10073; + @%p1890 bra $L__BB2_1494; + +$L__BB2_1498: + setp.eq.s32 %p1891, %r2788, 0; + mov.u32 %r10495, %r10088; + @%p1891 bra $L__BB2_1505; + + shr.u32 %r7417, %r2900, 3; + sub.s32 %r10089, %r2899, %r7417; + setp.eq.s32 %p1892, %r10089, 0; + mov.u32 %r10495, %r10088; + @%p1892 bra $L__BB2_1505; + + mov.u32 %r7418, -1; + shl.b32 %r7419, %r7418, %r10089; + not.b32 %r7420, %r7419; + and.b32 %r10090, %r9976, %r7420; + +$L__BB2_1501: + setp.gt.u32 %p1893, %r10461, 17476; + mov.u32 %r10495, 1; + @%p1893 bra $L__BB2_1505; + + sub.s32 %r7422, %r10462, %r10463; + min.u32 %r7423, %r7422, %r10089; + setp.eq.s32 %p1894, %r7423, 32; + mov.u32 %r7424, -1; + shl.b32 %r7425, %r7424, %r7423; + not.b32 %r7426, %r7425; + selp.b32 %r7427, -1, %r7426, %p1894; + and.b32 %r7428, %r7427, %r10090; + shl.b32 %r7429, %r7428, %r10463; + or.b32 %r10464, %r7429, %r10464; + add.s32 %r10463, %r7423, %r10463; + shr.u32 %r10090, %r10090, %r7423; + sub.s32 %r10089, %r10089, %r7423; + setp.lt.u32 %p1895, %r10463, %r10462; + @%p1895 bra $L__BB2_1504; + + cvt.u64.u32 %rd1191, %r10461; + add.s64 %rd1192, %rd1191, %rd5; + add.s64 %rd1193, %rd1, %rd1192; + st.global.u8 [%rd1193], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p1896, %r10464, 255; + selp.b32 %r10462, 7, 8, %p1896; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1504: + setp.ne.s32 %p1897, %r10089, 0; + mov.u32 %r10495, %r10088; + @%p1897 bra $L__BB2_1501; + +$L__BB2_1505: + setp.lt.s32 %p1898, %r2784, 1; + setp.lt.s32 %p1899, %r2443, 1; + or.pred %p1900, %p1899, %p1898; + @%p1900 bra $L__BB2_1553; + + min.s32 %r7432, %r2443, %r2784; + setp.lt.s32 %p1901, %r7432, 3; + add.s32 %r7433, %r9826, 17477; + cvt.u64.u32 %rd1194, %r7433; + add.s64 %rd1195, %rd1194, %rd5; + add.s64 %rd68, %rd1, %rd1195; + @%p1901 bra $L__BB2_1545; + bra.uni $L__BB2_1507; + +$L__BB2_1545: + add.s32 %r10040, %r10040, 1; + setp.lt.u32 %p1948, %r10040, %r10042; + @%p1948 bra $L__BB2_1553; + + shl.b16 %rs932, %rs1253, 1; + or.b16 %rs1253, %rs932, 1; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1949, %r9835, 0; + mov.u32 %r10146, %r10043; + @%p1949 bra $L__BB2_1549; + + setp.gt.u32 %p1950, %r9826, 191; + mov.u32 %r10146, 1; + mov.u32 %r9835, 0; + @%p1950 bra $L__BB2_1549; + + and.b16 %rs934, %rs1253, 255; + st.global.u8 [%rd68], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1951, %rs934, 255; + selp.b32 %r9835, 7, 8, %p1951; + mov.u16 %rs1253, 0; + mov.u32 %r10146, %r10043; + +$L__BB2_1549: + add.s32 %r7521, %r10041, 1; + min.u32 %r10041, %r7521, 12; + setp.lt.u32 %p1952, %r10041, 3; + mov.u32 %r10040, 0; + mov.u32 %r10147, %r10040; + @%p1952 bra $L__BB2_1552; + + setp.lt.u32 %p1953, %r10041, 6; + mov.u32 %r10147, 1; + @%p1953 bra $L__BB2_1552; + + setp.lt.u32 %p1954, %r10041, 9; + setp.eq.s32 %p1955, %r10041, 11; + selp.b32 %r7523, 4, 5, %p1955; + setp.lt.u32 %p1956, %r10041, 11; + selp.b32 %r7524, 3, %r7523, %p1956; + selp.b32 %r10147, 2, %r7524, %p1954; + +$L__BB2_1552: + mov.u32 %r7526, 1; + shl.b32 %r10042, %r7526, %r10147; + mov.u32 %r10043, %r10146; + bra.uni $L__BB2_1553; + +$L__BB2_1507: + shl.b16 %rs1253, %rs1253, 1; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1902, %r9835, 0; + mov.u32 %r10139, %r10043; + @%p1902 bra $L__BB2_1510; + + setp.gt.u32 %p1903, %r9826, 191; + mov.u32 %r10139, 1; + mov.u32 %r9835, 0; + @%p1903 bra $L__BB2_1510; + + st.global.u8 [%rd68], %rs1253; + add.s32 %r9826, %r9826, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9835, 8; + mov.u32 %r10139, %r10043; + +$L__BB2_1510: + setp.lt.u32 %p1904, %r10041, 3; + mov.u32 %r10107, 0; + @%p1904 bra $L__BB2_1513; + + setp.lt.u32 %p1905, %r10041, 6; + mov.u32 %r10107, 1; + @%p1905 bra $L__BB2_1513; + + setp.lt.u32 %p1906, %r10041, 9; + setp.eq.s32 %p1907, %r10041, 11; + selp.b32 %r7439, 4, 5, %p1907; + setp.lt.u32 %p1908, %r10041, 11; + selp.b32 %r7440, 3, %r7439, %p1908; + selp.b32 %r10107, 2, %r7440, %p1906; + +$L__BB2_1513: + setp.eq.s32 %p1909, %r10107, 0; + @%p1909 bra $L__BB2_1541; + + add.s32 %r3001, %r10107, -1; + and.b32 %r3002, %r10107, 3; + setp.eq.s32 %p1910, %r3002, 0; + mov.u32 %r10117, %r10107; + mov.u32 %r10118, %r10139; + @%p1910 bra $L__BB2_1526; + + mov.u32 %r7442, 1; + shl.b32 %r7443, %r7442, %r3001; + and.b32 %r7444, %r7443, %r10040; + setp.ne.s32 %p1911, %r7444, 0; + selp.u32 %r7445, 1, 0, %p1911; + cvt.u32.u16 %r7446, %rs1253; + bfi.b32 %r7447, %r7446, %r7445, 1, 8; + cvt.u16.u32 %rs1253, %r7447; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1912, %r9835, 0; + mov.u32 %r10118, %r10139; + @%p1912 bra $L__BB2_1518; + + setp.gt.u32 %p1913, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r10118, %r7442; + @%p1913 bra $L__BB2_1518; + + add.s32 %r7451, %r9826, 17477; + cvt.u64.u32 %rd1196, %r7451; + add.s64 %rd1197, %rd1196, %rd5; + add.s64 %rd1198, %rd1, %rd1197; + st.global.u8 [%rd1198], %rs1253; + add.s32 %r9826, %r9826, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9835, 8; + mov.u32 %r10118, %r10139; + +$L__BB2_1518: + setp.eq.s32 %p1914, %r3002, 1; + mov.u32 %r10139, %r10118; + mov.u32 %r10117, %r3001; + @%p1914 bra $L__BB2_1526; + + add.s32 %r10117, %r10107, -2; + mov.u32 %r7452, 1; + shl.b32 %r7453, %r7452, %r10117; + and.b32 %r7454, %r7453, %r10040; + setp.ne.s32 %p1915, %r7454, 0; + selp.u32 %r7455, 1, 0, %p1915; + cvt.u32.u16 %r7456, %rs1253; + bfi.b32 %r7457, %r7456, %r7455, 1, 8; + cvt.u16.u32 %rs1253, %r7457; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1916, %r9835, 0; + mov.u32 %r10113, %r10118; + @%p1916 bra $L__BB2_1522; + + setp.gt.u32 %p1917, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r10113, %r7452; + @%p1917 bra $L__BB2_1522; + + add.s32 %r7460, %r9826, 17477; + cvt.u64.u32 %rd1199, %r7460; + add.s64 %rd1200, %rd1199, %rd5; + add.s64 %rd1201, %rd1, %rd1200; + and.b16 %rs920, %rs1253, 255; + st.global.u8 [%rd1201], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1918, %rs920, 255; + selp.b32 %r9835, 7, 8, %p1918; + mov.u16 %rs1253, 0; + mov.u32 %r10113, %r10118; + +$L__BB2_1522: + setp.eq.s32 %p1919, %r3002, 2; + mov.u32 %r10139, %r10113; + mov.u32 %r10118, %r10113; + @%p1919 bra $L__BB2_1526; + + add.s32 %r10117, %r10107, -3; + mov.u32 %r7461, 1; + shl.b32 %r7462, %r7461, %r10117; + and.b32 %r7463, %r7462, %r10040; + setp.ne.s32 %p1920, %r7463, 0; + selp.u32 %r7464, 1, 0, %p1920; + cvt.u32.u16 %r7465, %rs1253; + bfi.b32 %r7466, %r7465, %r7464, 1, 8; + cvt.u16.u32 %rs1253, %r7466; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p1921, %r9835, 0; + mov.u32 %r10139, %r10113; + mov.u32 %r10118, %r10113; + @%p1921 bra $L__BB2_1526; + + setp.gt.u32 %p1922, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r10139, %r7461; + mov.u32 %r10118, %r7461; + @%p1922 bra $L__BB2_1526; + + add.s32 %r7471, %r9826, 17477; + cvt.u64.u32 %rd1202, %r7471; + add.s64 %rd1203, %rd1202, %rd5; + add.s64 %rd1204, %rd1, %rd1203; + and.b16 %rs923, %rs1253, 255; + st.global.u8 [%rd1204], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1923, %rs923, 255; + selp.b32 %r9835, 7, 8, %p1923; + mov.u16 %rs1253, 0; + mov.u32 %r10139, %r10113; + mov.u32 %r10118, %r10113; + +$L__BB2_1526: + setp.lt.u32 %p1924, %r3001, 3; + @%p1924 bra $L__BB2_1541; + + mov.u32 %r10139, %r10118; + +$L__BB2_1528: + add.s32 %r7472, %r10117, -1; + mov.u32 %r7473, 1; + shl.b32 %r7474, %r7473, %r7472; + and.b32 %r7475, %r7474, %r10040; + setp.ne.s32 %p1925, %r7475, 0; + selp.u32 %r7476, 1, 0, %p1925; + cvt.u32.u16 %r7477, %rs1253; + bfi.b32 %r10127, %r7477, %r7476, 1, 8; + add.s32 %r10126, %r9835, -1; + setp.ne.s32 %p1926, %r10126, 0; + mov.u32 %r10128, %r10139; + @%p1926 bra $L__BB2_1531; + + setp.gt.u32 %p1927, %r9826, 191; + mov.u32 %r10126, 0; + mov.u32 %r10128, %r7473; + @%p1927 bra $L__BB2_1531; + + cvt.u16.u32 %rs924, %r10127; + and.b16 %rs925, %rs924, 255; + add.s32 %r7481, %r9826, 17477; + cvt.u64.u32 %rd1205, %r7481; + add.s64 %rd1206, %rd1205, %rd5; + add.s64 %rd1207, %rd1, %rd1206; + st.global.u8 [%rd1207], %rs924; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1928, %rs925, 255; + selp.b32 %r10126, 7, 8, %p1928; + mov.u32 %r10127, 0; + mov.u32 %r10128, %r10139; + +$L__BB2_1531: + add.s32 %r7482, %r10117, -2; + shl.b32 %r7484, %r7473, %r7482; + and.b32 %r7485, %r7484, %r10040; + setp.ne.s32 %p1929, %r7485, 0; + and.b32 %r7486, %r10127, 127; + selp.u32 %r7487, 1, 0, %p1929; + bfi.b32 %r10131, %r7486, %r7487, 1, 7; + add.s32 %r10130, %r10126, -1; + setp.ne.s32 %p1930, %r10130, 0; + mov.u32 %r10132, %r10128; + @%p1930 bra $L__BB2_1534; + + setp.gt.u32 %p1931, %r9826, 191; + mov.u32 %r10132, 1; + mov.u32 %r10130, 0; + @%p1931 bra $L__BB2_1534; + + cvt.u16.u32 %rs926, %r10131; + and.b16 %rs927, %rs926, 255; + add.s32 %r7491, %r9826, 17477; + cvt.u64.u32 %rd1208, %r7491; + add.s64 %rd1209, %rd1208, %rd5; + add.s64 %rd1210, %rd1, %rd1209; + st.global.u8 [%rd1210], %rs926; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1932, %rs927, 255; + selp.b32 %r10130, 7, 8, %p1932; + mov.u32 %r10131, 0; + mov.u32 %r10132, %r10128; + +$L__BB2_1534: + add.s32 %r7492, %r10117, -3; + mov.u32 %r7493, 1; + shl.b32 %r7494, %r7493, %r7492; + and.b32 %r7495, %r7494, %r10040; + setp.ne.s32 %p1933, %r7495, 0; + and.b32 %r7496, %r10131, 127; + selp.u32 %r7497, 1, 0, %p1933; + bfi.b32 %r10135, %r7496, %r7497, 1, 7; + add.s32 %r10134, %r10130, -1; + setp.ne.s32 %p1934, %r10134, 0; + mov.u32 %r10136, %r10132; + @%p1934 bra $L__BB2_1537; + + setp.gt.u32 %p1935, %r9826, 191; + mov.u32 %r10134, 0; + mov.u32 %r10136, %r7493; + @%p1935 bra $L__BB2_1537; + + cvt.u16.u32 %rs928, %r10135; + and.b16 %rs929, %rs928, 255; + add.s32 %r7501, %r9826, 17477; + cvt.u64.u32 %rd1211, %r7501; + add.s64 %rd1212, %rd1211, %rd5; + add.s64 %rd1213, %rd1, %rd1212; + st.global.u8 [%rd1213], %rs928; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1936, %rs929, 255; + selp.b32 %r10134, 7, 8, %p1936; + mov.u32 %r10135, 0; + mov.u32 %r10136, %r10132; + +$L__BB2_1537: + add.s32 %r10117, %r10117, -4; + shl.b32 %r7503, %r7493, %r10117; + and.b32 %r7504, %r7503, %r10040; + setp.ne.s32 %p1937, %r7504, 0; + and.b32 %r7505, %r10135, 127; + selp.u32 %r7506, 1, 0, %p1937; + bfi.b32 %r7507, %r7505, %r7506, 1, 15; + cvt.u16.u32 %rs1253, %r7507; + add.s32 %r9835, %r10134, -1; + setp.ne.s32 %p1938, %r9835, 0; + mov.u32 %r10139, %r10136; + @%p1938 bra $L__BB2_1540; + + setp.gt.u32 %p1939, %r9826, 191; + mov.u32 %r10139, 1; + mov.u32 %r9835, 0; + @%p1939 bra $L__BB2_1540; + + add.s32 %r7510, %r9826, 17477; + cvt.u64.u32 %rd1214, %r7510; + add.s64 %rd1215, %rd1214, %rd5; + add.s64 %rd1216, %rd1, %rd1215; + and.b16 %rs931, %rs1253, 255; + st.global.u8 [%rd1216], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p1940, %rs931, 255; + selp.b32 %r9835, 7, 8, %p1940; + mov.u16 %rs1253, 0; + mov.u32 %r10139, %r10136; + +$L__BB2_1540: + setp.ne.s32 %p1941, %r10117, 0; + @%p1941 bra $L__BB2_1528; + +$L__BB2_1541: + add.s32 %r7512, %r10041, -1; + setp.eq.s32 %p1942, %r10041, 0; + mov.u32 %r10040, 0; + selp.b32 %r10041, 0, %r7512, %p1942; + setp.lt.u32 %p1943, %r10041, 3; + mov.u32 %r10143, %r10040; + @%p1943 bra $L__BB2_1544; + + setp.lt.u32 %p1944, %r10041, 6; + mov.u32 %r10143, 1; + @%p1944 bra $L__BB2_1544; + + setp.lt.u32 %p1945, %r10041, 9; + setp.eq.s32 %p1946, %r10041, 11; + selp.b32 %r7514, 4, 5, %p1946; + setp.lt.u32 %p1947, %r10041, 11; + selp.b32 %r7515, 3, %r7514, %p1947; + selp.b32 %r10143, 2, %r7515, %p1945; + +$L__BB2_1544: + mov.u32 %r7517, 1; + shl.b32 %r10042, %r7517, %r10143; + mov.u32 %r10043, %r10139; + +$L__BB2_1553: + setp.gt.s32 %p1957, %r2784, 2; + setp.gt.s32 %p1958, %r2443, 2; + and.pred %p1959, %p1958, %p1957; + @%p1959 bra $L__BB2_1602; + bra.uni $L__BB2_1554; + +$L__BB2_1602: + add.s32 %r7647, %r2656, -11; + cvt.u64.u32 %rd1246, %r7647; + add.s64 %rd70, %rd63, %rd1246; + ld.global.u8 %rs393, [%rd70]; + add.s32 %r7648, %r2656, -10; + cvt.u64.u32 %rd1248, %r7648; + add.s64 %rd1249, %rd63, %rd1248; + ld.global.u8 %rs394, [%rd1249]; + ld.global.u8 %rs395, [%rd1249+1]; + mul.lo.s32 %r7649, %r2784, 6; + add.s32 %r7650, %r7649, -12; + cvt.u64.u32 %rd1250, %r7650; + add.s64 %rd1251, %rd63, %rd1250; + ld.global.u8 %rs396, [%rd1251]; + ld.global.u8 %rs397, [%rd1251+1]; + add.s32 %r7651, %r7649, -10; + cvt.u64.u32 %rd1252, %r7651; + add.s64 %rd1253, %rd63, %rd1252; + ld.global.u8 %rs398, [%rd1253]; + ld.global.u8 %rs399, [%rd1253+1]; + setp.eq.s16 %p2027, %rs393, 0; + mov.u32 %r10241, %r9993; + @%p2027 bra $L__BB2_1609; + + ld.global.u8 %r10231, [%rd70+-1]; + cvt.u32.u16 %r10230, %rs393; + +$L__BB2_1604: + mov.u32 %r3211, %r10230; + setp.gt.u32 %p2028, %r10274, 2879; + mov.u32 %r10241, 1; + @%p2028 bra $L__BB2_1609; + + mov.u32 %r7653, 8; + sub.s32 %r7654, %r7653, %r10276; + sub.s32 %r7655, %r7654, %r10275; + min.u32 %r7656, %r7655, %r3211; + setp.eq.s32 %p2029, %r7656, 32; + mov.u32 %r7657, -1; + shl.b32 %r7658, %r7657, %r7656; + not.b32 %r7659, %r7658; + selp.b32 %r7660, -1, %r7659, %p2029; + and.b32 %r7661, %r7660, %r10231; + shl.b32 %r7662, %r7661, %r10275; + cvt.u16.u32 %rs967, %r7662; + or.b16 %rs1322, %rs1322, %rs967; + add.s32 %r10275, %r7656, %r10275; + sub.s32 %r10230, %r3211, %r7656; + shr.u32 %r10231, %r10231, %r7656; + setp.gt.u32 %p2030, %r7655, %r3211; + @%p2030 bra $L__BB2_1608; + + setp.ne.s32 %p2031, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs968, %rs1322, 255; + setp.ne.s16 %p2032, %rs968, 127; + and.pred %p2033, %p2031, %p2032; + @%p2033 bra $L__BB2_1608; + + mov.u32 %r7665, 20548; + sub.s32 %r7666, %r7665, %r10274; + cvt.u64.u32 %rd1254, %r7666; + add.s64 %rd1255, %rd1254, %rd5; + add.s64 %rd1256, %rd1, %rd1255; + st.global.u8 [%rd1256], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2034, %rs968, 143; + selp.u32 %r10276, 1, 0, %p2034; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1608: + setp.ne.s32 %p2035, %r10230, 0; + mov.u32 %r10241, %r9993; + @%p2035 bra $L__BB2_1604; + +$L__BB2_1609: + setp.eq.s16 %p2036, %rs397, 0; + mov.u32 %r10253, %r10241; + @%p2036 bra $L__BB2_1616; + + cvt.u32.u16 %r7667, %rs396; + and.b32 %r10243, %r7667, 255; + cvt.u32.u16 %r7668, %rs397; + and.b32 %r10242, %r7668, 255; + +$L__BB2_1611: + mov.u32 %r3230, %r10242; + setp.gt.u32 %p2037, %r10274, 2879; + mov.u32 %r10253, 1; + @%p2037 bra $L__BB2_1616; + + mov.u32 %r7670, 8; + sub.s32 %r7671, %r7670, %r10276; + sub.s32 %r7672, %r7671, %r10275; + min.u32 %r7673, %r7672, %r3230; + setp.eq.s32 %p2038, %r7673, 32; + mov.u32 %r7674, -1; + shl.b32 %r7675, %r7674, %r7673; + not.b32 %r7676, %r7675; + selp.b32 %r7677, -1, %r7676, %p2038; + and.b32 %r7678, %r7677, %r10243; + shl.b32 %r7679, %r7678, %r10275; + cvt.u16.u32 %rs972, %r7679; + or.b16 %rs1322, %rs1322, %rs972; + add.s32 %r10275, %r7673, %r10275; + sub.s32 %r10242, %r3230, %r7673; + shr.u32 %r10243, %r10243, %r7673; + setp.gt.u32 %p2039, %r7672, %r3230; + @%p2039 bra $L__BB2_1615; + + setp.ne.s32 %p2040, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs973, %rs1322, 255; + setp.ne.s16 %p2041, %rs973, 127; + and.pred %p2042, %p2040, %p2041; + @%p2042 bra $L__BB2_1615; + + mov.u32 %r7682, 20548; + sub.s32 %r7683, %r7682, %r10274; + cvt.u64.u32 %rd1257, %r7683; + add.s64 %rd1258, %rd1257, %rd5; + add.s64 %rd1259, %rd1, %rd1258; + st.global.u8 [%rd1259], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2043, %rs973, 143; + selp.u32 %r10276, 1, 0, %p2043; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1615: + setp.ne.s32 %p2044, %r10242, 0; + mov.u32 %r10253, %r10241; + @%p2044 bra $L__BB2_1611; + +$L__BB2_1616: + setp.eq.s16 %p2045, %rs395, 0; + mov.u32 %r10265, %r10253; + @%p2045 bra $L__BB2_1623; + + cvt.u32.u16 %r7684, %rs395; + and.b32 %r10254, %r7684, 255; + cvt.u32.u16 %r7685, %rs394; + and.b32 %r10255, %r7685, 255; + +$L__BB2_1618: + mov.u32 %r3249, %r10254; + setp.gt.u32 %p2046, %r10274, 2879; + mov.u32 %r10265, 1; + @%p2046 bra $L__BB2_1623; + + mov.u32 %r7687, 8; + sub.s32 %r7688, %r7687, %r10276; + sub.s32 %r7689, %r7688, %r10275; + min.u32 %r7690, %r7689, %r3249; + setp.eq.s32 %p2047, %r7690, 32; + mov.u32 %r7691, -1; + shl.b32 %r7692, %r7691, %r7690; + not.b32 %r7693, %r7692; + selp.b32 %r7694, -1, %r7693, %p2047; + and.b32 %r7695, %r7694, %r10255; + shl.b32 %r7696, %r7695, %r10275; + cvt.u16.u32 %rs977, %r7696; + or.b16 %rs1322, %rs1322, %rs977; + add.s32 %r10275, %r7690, %r10275; + sub.s32 %r10254, %r3249, %r7690; + shr.u32 %r10255, %r10255, %r7690; + setp.gt.u32 %p2048, %r7689, %r3249; + @%p2048 bra $L__BB2_1622; + + setp.ne.s32 %p2049, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs978, %rs1322, 255; + setp.ne.s16 %p2050, %rs978, 127; + and.pred %p2051, %p2049, %p2050; + @%p2051 bra $L__BB2_1622; + + mov.u32 %r7699, 20548; + sub.s32 %r7700, %r7699, %r10274; + cvt.u64.u32 %rd1260, %r7700; + add.s64 %rd1261, %rd1260, %rd5; + add.s64 %rd1262, %rd1, %rd1261; + st.global.u8 [%rd1262], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2052, %rs978, 143; + selp.u32 %r10276, 1, 0, %p2052; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1622: + setp.ne.s32 %p2053, %r10254, 0; + mov.u32 %r10265, %r10253; + @%p2053 bra $L__BB2_1618; + +$L__BB2_1623: + setp.eq.s16 %p2054, %rs399, 0; + mov.u32 %r10277, %r10265; + @%p2054 bra $L__BB2_1630; + + cvt.u32.u16 %r7701, %rs398; + and.b32 %r10267, %r7701, 255; + cvt.u32.u16 %r7702, %rs399; + and.b32 %r10266, %r7702, 255; + +$L__BB2_1625: + mov.u32 %r3268, %r10266; + setp.gt.u32 %p2055, %r10274, 2879; + mov.u32 %r10277, 1; + @%p2055 bra $L__BB2_1630; + + mov.u32 %r7704, 8; + sub.s32 %r7705, %r7704, %r10276; + sub.s32 %r7706, %r7705, %r10275; + min.u32 %r7707, %r7706, %r3268; + setp.eq.s32 %p2056, %r7707, 32; + mov.u32 %r7708, -1; + shl.b32 %r7709, %r7708, %r7707; + not.b32 %r7710, %r7709; + selp.b32 %r7711, -1, %r7710, %p2056; + and.b32 %r7712, %r7711, %r10267; + shl.b32 %r7713, %r7712, %r10275; + cvt.u16.u32 %rs982, %r7713; + or.b16 %rs1322, %rs1322, %rs982; + add.s32 %r10275, %r7707, %r10275; + sub.s32 %r10266, %r3268, %r7707; + shr.u32 %r10267, %r10267, %r7707; + setp.gt.u32 %p2057, %r7706, %r3268; + @%p2057 bra $L__BB2_1629; + + setp.ne.s32 %p2058, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs983, %rs1322, 255; + setp.ne.s16 %p2059, %rs983, 127; + and.pred %p2060, %p2058, %p2059; + @%p2060 bra $L__BB2_1629; + + mov.u32 %r7716, 20548; + sub.s32 %r7717, %r7716, %r10274; + cvt.u64.u32 %rd1263, %r7717; + add.s64 %rd1264, %rd1263, %rd5; + add.s64 %rd1265, %rd1, %rd1264; + st.global.u8 [%rd1265], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2061, %rs983, 143; + selp.u32 %r10276, 1, 0, %p2061; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1629: + setp.ne.s32 %p2062, %r10266, 0; + mov.u32 %r10277, %r10265; + @%p2062 bra $L__BB2_1625; + bra.uni $L__BB2_1630; + +$L__BB2_1554: + setp.gt.s32 %p1960, %r2784, 0; + and.pred %p1962, %p1958, %p1960; + @%p1962 bra $L__BB2_1583; + bra.uni $L__BB2_1555; + +$L__BB2_1583: + ld.global.u8 %rs379, [%rd65+1]; + ld.global.u8 %rs380, [%rd66]; + ld.global.u8 %rs381, [%rd66+1]; + setp.eq.s16 %p2001, %rs379, 0; + mov.u32 %r10209, %r9993; + @%p2001 bra $L__BB2_1590; + + ld.global.u8 %r10199, [%rd65]; + cvt.u32.u16 %r10198, %rs379; + +$L__BB2_1585: + mov.u32 %r3159, %r10198; + setp.gt.u32 %p2002, %r10274, 2879; + mov.u32 %r10209, 1; + @%p2002 bra $L__BB2_1590; + + mov.u32 %r7599, 8; + sub.s32 %r7600, %r7599, %r10276; + sub.s32 %r7601, %r7600, %r10275; + min.u32 %r7602, %r7601, %r3159; + setp.eq.s32 %p2003, %r7602, 32; + mov.u32 %r7603, -1; + shl.b32 %r7604, %r7603, %r7602; + not.b32 %r7605, %r7604; + selp.b32 %r7606, -1, %r7605, %p2003; + and.b32 %r7607, %r7606, %r10199; + shl.b32 %r7608, %r7607, %r10275; + cvt.u16.u32 %rs954, %r7608; + or.b16 %rs1322, %rs1322, %rs954; + add.s32 %r10275, %r7602, %r10275; + sub.s32 %r10198, %r3159, %r7602; + shr.u32 %r10199, %r10199, %r7602; + setp.gt.u32 %p2004, %r7601, %r3159; + @%p2004 bra $L__BB2_1589; + + setp.ne.s32 %p2005, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs955, %rs1322, 255; + setp.ne.s16 %p2006, %rs955, 127; + and.pred %p2007, %p2005, %p2006; + @%p2007 bra $L__BB2_1589; + + mov.u32 %r7611, 20548; + sub.s32 %r7612, %r7611, %r10274; + cvt.u64.u32 %rd1237, %r7612; + add.s64 %rd1238, %rd1237, %rd5; + add.s64 %rd1239, %rd1, %rd1238; + st.global.u8 [%rd1239], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2008, %rs955, 143; + selp.u32 %r10276, 1, 0, %p2008; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1589: + setp.ne.s32 %p2009, %r10198, 0; + mov.u32 %r10209, %r9993; + @%p2009 bra $L__BB2_1585; + +$L__BB2_1590: + add.s32 %r10211, %r2784, -1; + cvt.u32.u16 %r7614, %rs381; + and.b32 %r10222, %r7614, 255; + cvt.u32.u16 %r7615, %rs380; + and.b32 %r10223, %r7615, 255; + mov.u32 %r7613, 1; + mov.u32 %r10210, %r7613; + +$L__BB2_1591: + mov.u32 %r3179, %r10210; + setp.gt.u32 %p2010, %r10274, 2879; + mov.u32 %r10221, %r7613; + @%p2010 bra $L__BB2_1596; + + mov.u32 %r7617, 8; + sub.s32 %r7618, %r7617, %r10276; + sub.s32 %r7619, %r7618, %r10275; + min.u32 %r7620, %r7619, %r3179; + setp.eq.s32 %p2011, %r7620, 32; + mov.u32 %r7621, -1; + shl.b32 %r7622, %r7621, %r7620; + not.b32 %r7623, %r7622; + selp.b32 %r7624, -1, %r7623, %p2011; + and.b32 %r7625, %r7624, %r10211; + shl.b32 %r7626, %r7625, %r10275; + cvt.u16.u32 %rs958, %r7626; + or.b16 %rs1322, %rs1322, %rs958; + add.s32 %r10275, %r7620, %r10275; + sub.s32 %r10210, %r3179, %r7620; + shr.u32 %r10211, %r10211, %r7620; + setp.gt.u32 %p2012, %r7619, %r3179; + @%p2012 bra $L__BB2_1595; + + setp.ne.s32 %p2013, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs959, %rs1322, 255; + setp.ne.s16 %p2014, %rs959, 127; + and.pred %p2015, %p2013, %p2014; + @%p2015 bra $L__BB2_1595; + + mov.u32 %r7629, 20548; + sub.s32 %r7630, %r7629, %r10274; + cvt.u64.u32 %rd1240, %r7630; + add.s64 %rd1241, %rd1240, %rd5; + add.s64 %rd1242, %rd1, %rd1241; + st.global.u8 [%rd1242], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2016, %rs959, 143; + selp.u32 %r10276, 1, 0, %p2016; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1595: + setp.ne.s32 %p2017, %r10210, 0; + mov.u32 %r10221, %r10209; + @%p2017 bra $L__BB2_1591; + +$L__BB2_1596: + setp.eq.s16 %p2018, %rs381, 0; + mov.u32 %r10277, %r10221; + @%p2018 bra $L__BB2_1630; + +$L__BB2_1597: + mov.u32 %r3196, %r10222; + setp.gt.u32 %p2019, %r10274, 2879; + mov.u32 %r10277, 1; + @%p2019 bra $L__BB2_1630; + + mov.u32 %r7632, 8; + sub.s32 %r7633, %r7632, %r10276; + sub.s32 %r7634, %r7633, %r10275; + min.u32 %r7635, %r7634, %r3196; + setp.eq.s32 %p2020, %r7635, 32; + mov.u32 %r7636, -1; + shl.b32 %r7637, %r7636, %r7635; + not.b32 %r7638, %r7637; + selp.b32 %r7639, -1, %r7638, %p2020; + and.b32 %r7640, %r7639, %r10223; + shl.b32 %r7641, %r7640, %r10275; + cvt.u16.u32 %rs963, %r7641; + or.b16 %rs1322, %rs1322, %rs963; + add.s32 %r10275, %r7635, %r10275; + sub.s32 %r10222, %r3196, %r7635; + shr.u32 %r10223, %r10223, %r7635; + setp.gt.u32 %p2021, %r7634, %r3196; + @%p2021 bra $L__BB2_1601; + + setp.ne.s32 %p2022, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs964, %rs1322, 255; + setp.ne.s16 %p2023, %rs964, 127; + and.pred %p2024, %p2022, %p2023; + @%p2024 bra $L__BB2_1601; + + mov.u32 %r7644, 20548; + sub.s32 %r7645, %r7644, %r10274; + cvt.u64.u32 %rd1243, %r7645; + add.s64 %rd1244, %rd1243, %rd5; + add.s64 %rd1245, %rd1, %rd1244; + st.global.u8 [%rd1245], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2025, %rs964, 143; + selp.u32 %r10276, 1, 0, %p2025; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1601: + setp.eq.s32 %p2026, %r10222, 0; + mov.u32 %r10277, %r10221; + @%p2026 bra $L__BB2_1630; + bra.uni $L__BB2_1597; + +$L__BB2_1555: + setp.gt.s32 %p1964, %r2443, 0; + selp.b32 %r7527, %r2656, 0, %p1964; + cvt.u64.u32 %rd1217, %r7527; + add.s64 %rd69, %rd63, %rd1217; + ld.global.u8 %rs357, [%rd69+1]; + add.s32 %r7528, %r7527, 2; + cvt.u64.u32 %rd1219, %r7528; + add.s64 %rd1220, %rd63, %rd1219; + ld.global.u8 %rs358, [%rd1220]; + ld.global.u8 %rs359, [%rd1220+1]; + mul.lo.s32 %r7529, %r2784, 6; + selp.b32 %r7530, %r7529, 0, %p1960; + cvt.u64.u32 %rd1221, %r7530; + add.s64 %rd1222, %rd63, %rd1221; + ld.global.u8 %rs360, [%rd1222]; + ld.global.u8 %rs361, [%rd1222+1]; + add.s32 %r7531, %r7530, 2; + cvt.u64.u32 %rd1223, %r7531; + add.s64 %rd1224, %rd63, %rd1223; + ld.global.u8 %rs362, [%rd1224]; + ld.global.u8 %rs363, [%rd1224+1]; + setp.eq.s16 %p1965, %rs357, 0; + mov.u32 %r10165, %r9993; + @%p1965 bra $L__BB2_1562; + + ld.global.u8 %r10155, [%rd69]; + cvt.u32.u16 %r10154, %rs357; + +$L__BB2_1557: + mov.u32 %r3087, %r10154; + setp.gt.u32 %p1966, %r10274, 2879; + mov.u32 %r10165, 1; + @%p1966 bra $L__BB2_1562; + + mov.u32 %r7533, 8; + sub.s32 %r7534, %r7533, %r10276; + sub.s32 %r7535, %r7534, %r10275; + min.u32 %r7536, %r7535, %r3087; + setp.eq.s32 %p1967, %r7536, 32; + mov.u32 %r7537, -1; + shl.b32 %r7538, %r7537, %r7536; + not.b32 %r7539, %r7538; + selp.b32 %r7540, -1, %r7539, %p1967; + and.b32 %r7541, %r7540, %r10155; + shl.b32 %r7542, %r7541, %r10275; + cvt.u16.u32 %rs935, %r7542; + or.b16 %rs1322, %rs1322, %rs935; + add.s32 %r10275, %r7536, %r10275; + sub.s32 %r10154, %r3087, %r7536; + shr.u32 %r10155, %r10155, %r7536; + setp.gt.u32 %p1968, %r7535, %r3087; + @%p1968 bra $L__BB2_1561; + + setp.ne.s32 %p1969, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs936, %rs1322, 255; + setp.ne.s16 %p1970, %rs936, 127; + and.pred %p1971, %p1969, %p1970; + @%p1971 bra $L__BB2_1561; + + mov.u32 %r7545, 20548; + sub.s32 %r7546, %r7545, %r10274; + cvt.u64.u32 %rd1225, %r7546; + add.s64 %rd1226, %rd1225, %rd5; + add.s64 %rd1227, %rd1, %rd1226; + st.global.u8 [%rd1227], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p1972, %rs936, 143; + selp.u32 %r10276, 1, 0, %p1972; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1561: + setp.ne.s32 %p1973, %r10154, 0; + mov.u32 %r10165, %r9993; + @%p1973 bra $L__BB2_1557; + +$L__BB2_1562: + setp.eq.s16 %p1974, %rs361, 0; + mov.u32 %r10177, %r10165; + @%p1974 bra $L__BB2_1569; + + cvt.u32.u16 %r7547, %rs360; + and.b32 %r10167, %r7547, 255; + cvt.u32.u16 %r7548, %rs361; + and.b32 %r10166, %r7548, 255; + +$L__BB2_1564: + mov.u32 %r3106, %r10166; + setp.gt.u32 %p1975, %r10274, 2879; + mov.u32 %r10177, 1; + @%p1975 bra $L__BB2_1569; + + mov.u32 %r7550, 8; + sub.s32 %r7551, %r7550, %r10276; + sub.s32 %r7552, %r7551, %r10275; + min.u32 %r7553, %r7552, %r3106; + setp.eq.s32 %p1976, %r7553, 32; + mov.u32 %r7554, -1; + shl.b32 %r7555, %r7554, %r7553; + not.b32 %r7556, %r7555; + selp.b32 %r7557, -1, %r7556, %p1976; + and.b32 %r7558, %r7557, %r10167; + shl.b32 %r7559, %r7558, %r10275; + cvt.u16.u32 %rs940, %r7559; + or.b16 %rs1322, %rs1322, %rs940; + add.s32 %r10275, %r7553, %r10275; + sub.s32 %r10166, %r3106, %r7553; + shr.u32 %r10167, %r10167, %r7553; + setp.gt.u32 %p1977, %r7552, %r3106; + @%p1977 bra $L__BB2_1568; + + setp.ne.s32 %p1978, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs941, %rs1322, 255; + setp.ne.s16 %p1979, %rs941, 127; + and.pred %p1980, %p1978, %p1979; + @%p1980 bra $L__BB2_1568; + + mov.u32 %r7562, 20548; + sub.s32 %r7563, %r7562, %r10274; + cvt.u64.u32 %rd1228, %r7563; + add.s64 %rd1229, %rd1228, %rd5; + add.s64 %rd1230, %rd1, %rd1229; + st.global.u8 [%rd1230], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p1981, %rs941, 143; + selp.u32 %r10276, 1, 0, %p1981; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1568: + setp.ne.s32 %p1982, %r10166, 0; + mov.u32 %r10177, %r10165; + @%p1982 bra $L__BB2_1564; + +$L__BB2_1569: + setp.eq.s16 %p1983, %rs359, 0; + mov.u32 %r10189, %r10177; + @%p1983 bra $L__BB2_1576; + + cvt.u32.u16 %r7564, %rs359; + and.b32 %r10178, %r7564, 255; + cvt.u32.u16 %r7565, %rs358; + and.b32 %r10179, %r7565, 255; + +$L__BB2_1571: + mov.u32 %r3125, %r10178; + setp.gt.u32 %p1984, %r10274, 2879; + mov.u32 %r10189, 1; + @%p1984 bra $L__BB2_1576; + + mov.u32 %r7567, 8; + sub.s32 %r7568, %r7567, %r10276; + sub.s32 %r7569, %r7568, %r10275; + min.u32 %r7570, %r7569, %r3125; + setp.eq.s32 %p1985, %r7570, 32; + mov.u32 %r7571, -1; + shl.b32 %r7572, %r7571, %r7570; + not.b32 %r7573, %r7572; + selp.b32 %r7574, -1, %r7573, %p1985; + and.b32 %r7575, %r7574, %r10179; + shl.b32 %r7576, %r7575, %r10275; + cvt.u16.u32 %rs945, %r7576; + or.b16 %rs1322, %rs1322, %rs945; + add.s32 %r10275, %r7570, %r10275; + sub.s32 %r10178, %r3125, %r7570; + shr.u32 %r10179, %r10179, %r7570; + setp.gt.u32 %p1986, %r7569, %r3125; + @%p1986 bra $L__BB2_1575; + + setp.ne.s32 %p1987, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs946, %rs1322, 255; + setp.ne.s16 %p1988, %rs946, 127; + and.pred %p1989, %p1987, %p1988; + @%p1989 bra $L__BB2_1575; + + mov.u32 %r7579, 20548; + sub.s32 %r7580, %r7579, %r10274; + cvt.u64.u32 %rd1231, %r7580; + add.s64 %rd1232, %rd1231, %rd5; + add.s64 %rd1233, %rd1, %rd1232; + st.global.u8 [%rd1233], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p1990, %rs946, 143; + selp.u32 %r10276, 1, 0, %p1990; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1575: + setp.ne.s32 %p1991, %r10178, 0; + mov.u32 %r10189, %r10177; + @%p1991 bra $L__BB2_1571; + +$L__BB2_1576: + setp.eq.s16 %p1992, %rs363, 0; + mov.u32 %r10277, %r10189; + @%p1992 bra $L__BB2_1630; + + cvt.u32.u16 %r7581, %rs362; + and.b32 %r10191, %r7581, 255; + cvt.u32.u16 %r7582, %rs363; + and.b32 %r10190, %r7582, 255; + +$L__BB2_1578: + mov.u32 %r3144, %r10190; + setp.gt.u32 %p1993, %r10274, 2879; + mov.u32 %r10277, 1; + @%p1993 bra $L__BB2_1630; + + mov.u32 %r7584, 8; + sub.s32 %r7585, %r7584, %r10276; + sub.s32 %r7586, %r7585, %r10275; + min.u32 %r7587, %r7586, %r3144; + setp.eq.s32 %p1994, %r7587, 32; + mov.u32 %r7588, -1; + shl.b32 %r7589, %r7588, %r7587; + not.b32 %r7590, %r7589; + selp.b32 %r7591, -1, %r7590, %p1994; + and.b32 %r7592, %r7591, %r10191; + shl.b32 %r7593, %r7592, %r10275; + cvt.u16.u32 %rs950, %r7593; + or.b16 %rs1322, %rs1322, %rs950; + add.s32 %r10275, %r7587, %r10275; + sub.s32 %r10190, %r3144, %r7587; + shr.u32 %r10191, %r10191, %r7587; + setp.gt.u32 %p1995, %r7586, %r3144; + @%p1995 bra $L__BB2_1582; + + setp.ne.s32 %p1996, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs951, %rs1322, 255; + setp.ne.s16 %p1997, %rs951, 127; + and.pred %p1998, %p1996, %p1997; + @%p1998 bra $L__BB2_1582; + + mov.u32 %r7596, 20548; + sub.s32 %r7597, %r7596, %r10274; + cvt.u64.u32 %rd1234, %r7597; + add.s64 %rd1235, %rd1234, %rd5; + add.s64 %rd1236, %rd1, %rd1235; + st.global.u8 [%rd1236], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p1999, %rs951, 143; + selp.u32 %r10276, 1, 0, %p1999; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1582: + setp.eq.s32 %p2000, %r10190, 0; + mov.u32 %r10277, %r10189; + @%p2000 bra $L__BB2_1630; + bra.uni $L__BB2_1578; + +$L__BB2_1630: + shr.u32 %r7718, %r9963, 1; + or.b32 %r9760, %r7718, %r2901; + +$L__BB2_1631: + add.s32 %r9744, %r9744, 4; + setp.lt.u32 %p2063, %r9744, %r4057; + @%p2063 bra $L__BB2_1266; + +$L__BB2_1632: + add.s32 %r8418, %r4057, 1; + shr.u32 %r8417, %r8418, 1; + add.s32 %r7719, %r8417, 1; + setp.gt.u32 %p2064, %r7719, 512; + @%p2064 bra $L__BB2_1634; + + add.s32 %r8409, %r4057, 1; + shr.u32 %r8408, %r8409, 1; + add.s32 %r8407, %r4103, %r8408; + mov.u16 %rs986, 0; + add.s32 %r8403, %r8407, 1; + st.shared.u8 [%r8403], %rs986; + +$L__BB2_1634: + setp.lt.u32 %p2065, %r4058, 3; + @%p2065 bra $L__BB2_1880; + + ld.param.u64 %rd1418, [ j2k_htj2k_encode_codeblocks_multi_input_param_4]; + ld.param.u64 %rd1413, [ j2k_htj2k_encode_codeblocks_multi_input_param_3]; + mov.u32 %r10310, 2; + cvta.to.global.u64 %rd71, %rd1418; + cvta.to.global.u64 %rd72, %rd1413; + +$L__BB2_1636: + ld.shared.u8 %rs422, [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE13cleanup_e_val]; + mov.u16 %rs987, 0; + st.shared.u8 [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE13cleanup_e_val], %rs987; + ld.shared.u8 %rs423, [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val]; + st.shared.u8 [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val], %rs987; + @%p10 bra $L__BB2_1879; + + mov.u32 %r7723, 0; + ld.shared.u8 %rs988, [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE13cleanup_e_val+1]; + ld.shared.u8 %rs989, [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val+1]; + max.u16 %rs991, %rs422, %rs988; + cvt.u32.u16 %r7724, %rs991; + add.s32 %r10345, %r7724, -1; + add.s32 %r3336, %r10310, 1; + mul.lo.s32 %r10343, %r10310, %r4055; + mul.wide.u16 %r7725, %rs989, 4; + cvt.u32.u16 %r7726, %rs423; + and.b32 %r7727, %r7726, 255; + add.s32 %r10342, %r7725, %r7727; + mov.u32 %r10326, %r7723; + mov.u32 %r10344, %r7723; + mov.u32 %r10346, %r7723; + bra.uni $L__BB2_1638; + +$L__BB2_1709: + setp.gt.u32 %p2145, %r9826, 191; + mov.u32 %r10428, 1; + mov.u32 %r9835, 0; + @%p2145 bra $L__BB2_1711; + + and.b16 %rs1018, %rs1253, 255; + st.global.u8 [%rd73], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2146, %rs1018, 255; + selp.b32 %r9835, 7, 8, %p2146; + mov.u16 %rs1253, 0; + mov.u32 %r10428, %r10043; + bra.uni $L__BB2_1711; + +$L__BB2_1815: + setp.gt.u32 %p2264, %r9826, 191; + mov.u32 %r10577, 1; + mov.u32 %r9835, 0; + @%p2264 bra $L__BB2_1817; + + and.b16 %rs1055, %rs1253, 255; + st.global.u8 [%rd74], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2265, %rs1055, 255; + selp.b32 %r9835, 7, 8, %p2265; + mov.u16 %rs1253, 0; + mov.u32 %r10577, %r10043; + bra.uni $L__BB2_1817; + +$L__BB2_1638: + cvt.u64.u32 %rd1266, %r10343; + add.s64 %rd1267, %rd1266, %rd4; + shl.b64 %rd1268, %rd1267, 2; + add.s64 %rd1269, %rd3, %rd1268; + ld.global.u32 %r3360, [%rd1269]; + setp.eq.s32 %p2067, %r3360, 0; + mov.u32 %r10347, %r7723; + @%p2067 bra $L__BB2_1640; + + and.b32 %r7729, %r3360, -2147483648; + abs.s32 %r7730, %r3360; + shl.b32 %r7731, %r7730, %r2358; + or.b32 %r10347, %r7731, %r7729; + +$L__BB2_1640: + shl.b32 %r7735, %r10347, 1; + shr.u32 %r7736, %r7735, %r2358; + and.b32 %r3363, %r7736, -2; + setp.eq.s32 %p2068, %r3363, 0; + mov.u32 %r10351, 0; + mov.u32 %r10348, %r10351; + mov.u32 %r10349, %r10351; + mov.u32 %r10355, %r10351; + @%p2068 bra $L__BB2_1642; + + add.s32 %r7738, %r3363, -1; + clz.b32 %r7739, %r7738; + mov.u32 %r7740, 32; + sub.s32 %r10348, %r7740, %r7739; + shr.u32 %r7741, %r10347, 31; + add.s32 %r7742, %r7741, %r3363; + add.s32 %r10349, %r7742, -2; + mov.u32 %r10355, 1; + +$L__BB2_1642: + setp.ge.u32 %p2069, %r3336, %r4058; + @%p2069 bra $L__BB2_1645; + + add.s32 %r7745, %r10343, %r4055; + cvt.u64.u32 %rd1270, %r7745; + add.s64 %rd1271, %rd1270, %rd4; + shl.b64 %rd1272, %rd1271, 2; + add.s64 %rd1273, %rd3, %rd1272; + ld.global.u32 %r3369, [%rd1273]; + setp.eq.s32 %p2070, %r3369, 0; + @%p2070 bra $L__BB2_1645; + + and.b32 %r7746, %r3369, -2147483648; + abs.s32 %r7747, %r3369; + shl.b32 %r7748, %r7747, %r2358; + or.b32 %r10351, %r7748, %r7746; + +$L__BB2_1645: + shl.b32 %r7751, %r10351, 1; + shr.u32 %r7752, %r7751, %r2358; + and.b32 %r3372, %r7752, -2; + setp.eq.s32 %p2071, %r3372, 0; + mov.u32 %r10366, 0; + mov.u32 %r10352, %r10366; + mov.u32 %r10353, %r10366; + mov.u32 %r10371, %r10348; + @%p2071 bra $L__BB2_1647; + + or.b32 %r10355, %r10355, 2; + add.s32 %r7753, %r3372, -1; + clz.b32 %r7754, %r7753; + mov.u32 %r7755, 32; + sub.s32 %r10352, %r7755, %r7754; + max.s32 %r10371, %r10348, %r10352; + shr.u32 %r7756, %r10351, 31; + add.s32 %r7757, %r7756, %r3372; + add.s32 %r10353, %r7757, -2; + +$L__BB2_1647: + add.s32 %r10648, %r10343, 1; + add.s32 %r7762, %r10326, 1; + setp.ge.u32 %p2072, %r7762, %r4057; + mov.u32 %r10367, %r10366; + mov.u32 %r10368, %r10366; + mov.u32 %r10369, %r10366; + @%p2072 bra $L__BB2_1658; + + cvt.u64.u32 %rd1274, %r10648; + add.s64 %rd1275, %rd1274, %rd4; + shl.b64 %rd1276, %rd1275, 2; + add.s64 %rd1277, %rd3, %rd1276; + ld.global.u32 %r3382, [%rd1277]; + setp.eq.s32 %p2073, %r3382, 0; + mov.u32 %r10367, 0; + mov.u32 %r10356, %r10367; + @%p2073 bra $L__BB2_1650; + + and.b32 %r7764, %r3382, -2147483648; + abs.s32 %r7765, %r3382; + shl.b32 %r7766, %r7765, %r2358; + or.b32 %r10356, %r7766, %r7764; + +$L__BB2_1650: + shl.b32 %r7769, %r10356, 1; + shr.u32 %r7770, %r7769, %r2358; + and.b32 %r3385, %r7770, -2; + setp.eq.s32 %p2074, %r3385, 0; + mov.u32 %r10369, %r10367; + @%p2074 bra $L__BB2_1652; + + or.b32 %r10355, %r10355, 4; + add.s32 %r7771, %r3385, -1; + clz.b32 %r7772, %r7771; + mov.u32 %r7773, 32; + sub.s32 %r10367, %r7773, %r7772; + max.s32 %r10371, %r10371, %r10367; + shr.u32 %r7774, %r10356, 31; + add.s32 %r7775, %r7774, %r3385; + add.s32 %r10369, %r7775, -2; + +$L__BB2_1652: + mov.u32 %r10366, 0; + mov.u32 %r10361, %r10366; + @%p2069 bra $L__BB2_1655; + + add.s32 %r7778, %r10648, %r4055; + cvt.u64.u32 %rd1278, %r7778; + add.s64 %rd1279, %rd1278, %rd4; + shl.b64 %rd1280, %rd1279, 2; + add.s64 %rd1281, %rd3, %rd1280; + ld.global.u32 %r3394, [%rd1281]; + setp.eq.s32 %p2076, %r3394, 0; + @%p2076 bra $L__BB2_1655; + + and.b32 %r7779, %r3394, -2147483648; + abs.s32 %r7780, %r3394; + shl.b32 %r7781, %r7780, %r2358; + or.b32 %r10361, %r7781, %r7779; + +$L__BB2_1655: + shl.b32 %r7784, %r10361, 1; + shr.u32 %r7785, %r7784, %r2358; + and.b32 %r3397, %r7785, -2; + setp.eq.s32 %p2077, %r3397, 0; + mov.u32 %r10368, %r10366; + @%p2077 bra $L__BB2_1657; + + or.b32 %r10355, %r10355, 8; + add.s32 %r7786, %r3397, -1; + clz.b32 %r7787, %r7786; + mov.u32 %r7788, 32; + sub.s32 %r10366, %r7788, %r7787; + max.s32 %r10371, %r10371, %r10366; + shr.u32 %r7789, %r10361, 31; + add.s32 %r7790, %r7789, %r3397; + add.s32 %r10368, %r7790, -2; + +$L__BB2_1657: + add.s32 %r10648, %r10343, 2; + +$L__BB2_1658: + add.s32 %r7792, %r10355, -1; + and.b32 %r7793, %r7792, %r10355; + setp.ne.s32 %p2078, %r7793, 0; + mov.u32 %r10373, 0; + setp.gt.s32 %p2079, %r10345, 1; + and.pred %p2080, %p2079, %p2078; + selp.b32 %r7794, %r10345, 1, %p2080; + max.s32 %r3414, %r7794, %r10371; + sub.s32 %r3415, %r3414, %r7794; + setp.lt.s32 %p2081, %r3415, 1; + @%p2081 bra $L__BB2_1660; + + setp.eq.s32 %p2082, %r10348, %r10371; + selp.u32 %r7795, 1, 0, %p2082; + setp.eq.s32 %p2083, %r10352, %r10371; + selp.u32 %r7796, -1, 0, %p2083; + bfi.b32 %r7797, %r7796, %r7795, 1, 1; + setp.eq.s32 %p2084, %r10367, %r10371; + selp.u16 %rs992, 1, 0, %p2084; + mul.wide.u16 %r7798, %rs992, 4; + or.b32 %r7799, %r7797, %r7798; + setp.eq.s32 %p2085, %r10366, %r10371; + selp.u16 %rs993, 1, 0, %p2085; + mul.wide.u16 %r7800, %rs993, 8; + or.b32 %r10373, %r7799, %r7800; + +$L__BB2_1660: + shl.b32 %r7801, %r10355, 4; + shl.b32 %r7802, %r10342, 8; + or.b32 %r7803, %r7801, %r7802; + or.b32 %r7804, %r7803, %r10373; + mul.wide.u32 %rd1282, %r7804, 2; + add.s64 %rd1283, %rd72, %rd1282; + ld.global.u16 %rs426, [%rd1283]; + shr.u16 %rs994, %rs426, 4; + and.b16 %rs427, %rs994, 7; + setp.eq.s16 %p2086, %rs427, 0; + mov.u32 %r10385, %r10277; + @%p2086 bra $L__BB2_1667; + + cvt.u32.u16 %r10374, %rs427; + shr.u16 %rs995, %rs426, 8; + cvt.u32.u16 %r10375, %rs995; + +$L__BB2_1662: + mov.u32 %r3420, %r10374; + setp.gt.u32 %p2087, %r10274, 2879; + mov.u32 %r10385, 1; + @%p2087 bra $L__BB2_1667; + + mov.u32 %r7806, 8; + sub.s32 %r7807, %r7806, %r10276; + sub.s32 %r7808, %r7807, %r10275; + min.u32 %r7809, %r7808, %r3420; + setp.eq.s32 %p2088, %r7809, 32; + mov.u32 %r7810, -1; + shl.b32 %r7811, %r7810, %r7809; + not.b32 %r7812, %r7811; + selp.b32 %r7813, -1, %r7812, %p2088; + and.b32 %r7814, %r7813, %r10375; + shl.b32 %r7815, %r7814, %r10275; + cvt.u16.u32 %rs996, %r7815; + or.b16 %rs1322, %rs1322, %rs996; + add.s32 %r10275, %r7809, %r10275; + sub.s32 %r10374, %r3420, %r7809; + shr.u32 %r10375, %r10375, %r7809; + setp.gt.u32 %p2089, %r7808, %r3420; + @%p2089 bra $L__BB2_1666; + + setp.ne.s32 %p2090, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs997, %rs1322, 255; + setp.ne.s16 %p2091, %rs997, 127; + and.pred %p2092, %p2090, %p2091; + @%p2092 bra $L__BB2_1666; + + mov.u32 %r7818, 20548; + sub.s32 %r7819, %r7818, %r10274; + cvt.u64.u32 %rd1284, %r7819; + add.s64 %rd1285, %rd1284, %rd5; + add.s64 %rd1286, %rd1, %rd1285; + st.global.u8 [%rd1286], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2093, %rs997, 143; + selp.u32 %r10276, 1, 0, %p2093; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1666: + setp.ne.s32 %p2094, %r10374, 0; + mov.u32 %r10385, %r10277; + @%p2094 bra $L__BB2_1662; + +$L__BB2_1667: + setp.ne.s32 %p2095, %r10342, 0; + @%p2095 bra $L__BB2_1715; + + setp.eq.s32 %p2096, %r10355, 0; + add.s32 %r7820, %r9826, 17477; + cvt.u64.u32 %rd1287, %r7820; + add.s64 %rd1288, %rd1287, %rd5; + add.s64 %rd73, %rd1, %rd1288; + @%p2096 bra $L__BB2_1707; + + shl.b16 %rs1253, %rs1253, 1; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p2097, %r9835, 0; + mov.u32 %r10421, %r10043; + @%p2097 bra $L__BB2_1672; + + setp.gt.u32 %p2098, %r9826, 191; + mov.u32 %r10421, 1; + mov.u32 %r9835, 0; + @%p2098 bra $L__BB2_1672; + + st.global.u8 [%rd73], %rs1253; + add.s32 %r9826, %r9826, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9835, 8; + mov.u32 %r10421, %r10043; + +$L__BB2_1672: + setp.lt.u32 %p2099, %r10041, 3; + mov.u32 %r10389, 0; + @%p2099 bra $L__BB2_1675; + + setp.lt.u32 %p2100, %r10041, 6; + mov.u32 %r10389, 1; + @%p2100 bra $L__BB2_1675; + + setp.lt.u32 %p2101, %r10041, 9; + setp.eq.s32 %p2102, %r10041, 11; + selp.b32 %r7826, 4, 5, %p2102; + setp.lt.u32 %p2103, %r10041, 11; + selp.b32 %r7827, 3, %r7826, %p2103; + selp.b32 %r10389, 2, %r7827, %p2101; + +$L__BB2_1675: + setp.eq.s32 %p2104, %r10389, 0; + @%p2104 bra $L__BB2_1703; + + add.s32 %r3444, %r10389, -1; + and.b32 %r3445, %r10389, 3; + setp.eq.s32 %p2105, %r3445, 0; + mov.u32 %r10399, %r10389; + mov.u32 %r10400, %r10421; + @%p2105 bra $L__BB2_1688; + + mov.u32 %r7829, 1; + shl.b32 %r7830, %r7829, %r3444; + and.b32 %r7831, %r7830, %r10040; + setp.ne.s32 %p2106, %r7831, 0; + selp.u32 %r7832, 1, 0, %p2106; + cvt.u32.u16 %r7833, %rs1253; + bfi.b32 %r7834, %r7833, %r7832, 1, 8; + cvt.u16.u32 %rs1253, %r7834; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p2107, %r9835, 0; + mov.u32 %r10400, %r10421; + @%p2107 bra $L__BB2_1680; + + setp.gt.u32 %p2108, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r10400, %r7829; + @%p2108 bra $L__BB2_1680; + + add.s32 %r7838, %r9826, 17477; + cvt.u64.u32 %rd1289, %r7838; + add.s64 %rd1290, %rd1289, %rd5; + add.s64 %rd1291, %rd1, %rd1290; + st.global.u8 [%rd1291], %rs1253; + add.s32 %r9826, %r9826, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9835, 8; + mov.u32 %r10400, %r10421; + +$L__BB2_1680: + setp.eq.s32 %p2109, %r3445, 1; + mov.u32 %r10421, %r10400; + mov.u32 %r10399, %r3444; + @%p2109 bra $L__BB2_1688; + + add.s32 %r10399, %r10389, -2; + mov.u32 %r7839, 1; + shl.b32 %r7840, %r7839, %r10399; + and.b32 %r7841, %r7840, %r10040; + setp.ne.s32 %p2110, %r7841, 0; + selp.u32 %r7842, 1, 0, %p2110; + cvt.u32.u16 %r7843, %rs1253; + bfi.b32 %r7844, %r7843, %r7842, 1, 8; + cvt.u16.u32 %rs1253, %r7844; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p2111, %r9835, 0; + mov.u32 %r10395, %r10400; + @%p2111 bra $L__BB2_1684; + + setp.gt.u32 %p2112, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r10395, %r7839; + @%p2112 bra $L__BB2_1684; + + add.s32 %r7847, %r9826, 17477; + cvt.u64.u32 %rd1292, %r7847; + add.s64 %rd1293, %rd1292, %rd5; + add.s64 %rd1294, %rd1, %rd1293; + and.b16 %rs1004, %rs1253, 255; + st.global.u8 [%rd1294], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2113, %rs1004, 255; + selp.b32 %r9835, 7, 8, %p2113; + mov.u16 %rs1253, 0; + mov.u32 %r10395, %r10400; + +$L__BB2_1684: + setp.eq.s32 %p2114, %r3445, 2; + mov.u32 %r10421, %r10395; + mov.u32 %r10400, %r10395; + @%p2114 bra $L__BB2_1688; + + add.s32 %r10399, %r10389, -3; + mov.u32 %r7848, 1; + shl.b32 %r7849, %r7848, %r10399; + and.b32 %r7850, %r7849, %r10040; + setp.ne.s32 %p2115, %r7850, 0; + selp.u32 %r7851, 1, 0, %p2115; + cvt.u32.u16 %r7852, %rs1253; + bfi.b32 %r7853, %r7852, %r7851, 1, 8; + cvt.u16.u32 %rs1253, %r7853; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p2116, %r9835, 0; + mov.u32 %r10421, %r10395; + mov.u32 %r10400, %r10395; + @%p2116 bra $L__BB2_1688; + + setp.gt.u32 %p2117, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r10421, %r7848; + mov.u32 %r10400, %r7848; + @%p2117 bra $L__BB2_1688; + + add.s32 %r7858, %r9826, 17477; + cvt.u64.u32 %rd1295, %r7858; + add.s64 %rd1296, %rd1295, %rd5; + add.s64 %rd1297, %rd1, %rd1296; + and.b16 %rs1007, %rs1253, 255; + st.global.u8 [%rd1297], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2118, %rs1007, 255; + selp.b32 %r9835, 7, 8, %p2118; + mov.u16 %rs1253, 0; + mov.u32 %r10421, %r10395; + mov.u32 %r10400, %r10395; + +$L__BB2_1688: + setp.lt.u32 %p2119, %r3444, 3; + @%p2119 bra $L__BB2_1703; + + mov.u32 %r10421, %r10400; + +$L__BB2_1690: + add.s32 %r7859, %r10399, -1; + mov.u32 %r7860, 1; + shl.b32 %r7861, %r7860, %r7859; + and.b32 %r7862, %r7861, %r10040; + setp.ne.s32 %p2120, %r7862, 0; + selp.u32 %r7863, 1, 0, %p2120; + cvt.u32.u16 %r7864, %rs1253; + bfi.b32 %r10409, %r7864, %r7863, 1, 8; + add.s32 %r10408, %r9835, -1; + setp.ne.s32 %p2121, %r10408, 0; + mov.u32 %r10410, %r10421; + @%p2121 bra $L__BB2_1693; + + setp.gt.u32 %p2122, %r9826, 191; + mov.u32 %r10408, 0; + mov.u32 %r10410, %r7860; + @%p2122 bra $L__BB2_1693; + + cvt.u16.u32 %rs1008, %r10409; + and.b16 %rs1009, %rs1008, 255; + add.s32 %r7868, %r9826, 17477; + cvt.u64.u32 %rd1298, %r7868; + add.s64 %rd1299, %rd1298, %rd5; + add.s64 %rd1300, %rd1, %rd1299; + st.global.u8 [%rd1300], %rs1008; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2123, %rs1009, 255; + selp.b32 %r10408, 7, 8, %p2123; + mov.u32 %r10409, 0; + mov.u32 %r10410, %r10421; + +$L__BB2_1693: + add.s32 %r7869, %r10399, -2; + shl.b32 %r7871, %r7860, %r7869; + and.b32 %r7872, %r7871, %r10040; + setp.ne.s32 %p2124, %r7872, 0; + and.b32 %r7873, %r10409, 127; + selp.u32 %r7874, 1, 0, %p2124; + bfi.b32 %r10413, %r7873, %r7874, 1, 7; + add.s32 %r10412, %r10408, -1; + setp.ne.s32 %p2125, %r10412, 0; + mov.u32 %r10414, %r10410; + @%p2125 bra $L__BB2_1696; + + setp.gt.u32 %p2126, %r9826, 191; + mov.u32 %r10414, 1; + mov.u32 %r10412, 0; + @%p2126 bra $L__BB2_1696; + + cvt.u16.u32 %rs1010, %r10413; + and.b16 %rs1011, %rs1010, 255; + add.s32 %r7878, %r9826, 17477; + cvt.u64.u32 %rd1301, %r7878; + add.s64 %rd1302, %rd1301, %rd5; + add.s64 %rd1303, %rd1, %rd1302; + st.global.u8 [%rd1303], %rs1010; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2127, %rs1011, 255; + selp.b32 %r10412, 7, 8, %p2127; + mov.u32 %r10413, 0; + mov.u32 %r10414, %r10410; + +$L__BB2_1696: + add.s32 %r7879, %r10399, -3; + mov.u32 %r7880, 1; + shl.b32 %r7881, %r7880, %r7879; + and.b32 %r7882, %r7881, %r10040; + setp.ne.s32 %p2128, %r7882, 0; + and.b32 %r7883, %r10413, 127; + selp.u32 %r7884, 1, 0, %p2128; + bfi.b32 %r10417, %r7883, %r7884, 1, 7; + add.s32 %r10416, %r10412, -1; + setp.ne.s32 %p2129, %r10416, 0; + mov.u32 %r10418, %r10414; + @%p2129 bra $L__BB2_1699; + + setp.gt.u32 %p2130, %r9826, 191; + mov.u32 %r10416, 0; + mov.u32 %r10418, %r7880; + @%p2130 bra $L__BB2_1699; + + cvt.u16.u32 %rs1012, %r10417; + and.b16 %rs1013, %rs1012, 255; + add.s32 %r7888, %r9826, 17477; + cvt.u64.u32 %rd1304, %r7888; + add.s64 %rd1305, %rd1304, %rd5; + add.s64 %rd1306, %rd1, %rd1305; + st.global.u8 [%rd1306], %rs1012; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2131, %rs1013, 255; + selp.b32 %r10416, 7, 8, %p2131; + mov.u32 %r10417, 0; + mov.u32 %r10418, %r10414; + +$L__BB2_1699: + add.s32 %r10399, %r10399, -4; + shl.b32 %r7890, %r7880, %r10399; + and.b32 %r7891, %r7890, %r10040; + setp.ne.s32 %p2132, %r7891, 0; + and.b32 %r7892, %r10417, 127; + selp.u32 %r7893, 1, 0, %p2132; + bfi.b32 %r7894, %r7892, %r7893, 1, 15; + cvt.u16.u32 %rs1253, %r7894; + add.s32 %r9835, %r10416, -1; + setp.ne.s32 %p2133, %r9835, 0; + mov.u32 %r10421, %r10418; + @%p2133 bra $L__BB2_1702; + + setp.gt.u32 %p2134, %r9826, 191; + mov.u32 %r10421, 1; + mov.u32 %r9835, 0; + @%p2134 bra $L__BB2_1702; + + add.s32 %r7897, %r9826, 17477; + cvt.u64.u32 %rd1307, %r7897; + add.s64 %rd1308, %rd1307, %rd5; + add.s64 %rd1309, %rd1, %rd1308; + and.b16 %rs1015, %rs1253, 255; + st.global.u8 [%rd1309], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2135, %rs1015, 255; + selp.b32 %r9835, 7, 8, %p2135; + mov.u16 %rs1253, 0; + mov.u32 %r10421, %r10418; + +$L__BB2_1702: + setp.ne.s32 %p2136, %r10399, 0; + @%p2136 bra $L__BB2_1690; + +$L__BB2_1703: + add.s32 %r7899, %r10041, -1; + setp.eq.s32 %p2137, %r10041, 0; + mov.u32 %r10040, 0; + selp.b32 %r10041, 0, %r7899, %p2137; + setp.lt.u32 %p2138, %r10041, 3; + mov.u32 %r10425, %r10040; + @%p2138 bra $L__BB2_1706; + + setp.lt.u32 %p2139, %r10041, 6; + mov.u32 %r10425, 1; + @%p2139 bra $L__BB2_1706; + + setp.lt.u32 %p2140, %r10041, 9; + setp.eq.s32 %p2141, %r10041, 11; + selp.b32 %r7901, 4, 5, %p2141; + setp.lt.u32 %p2142, %r10041, 11; + selp.b32 %r7902, 3, %r7901, %p2142; + selp.b32 %r10425, 2, %r7902, %p2140; + +$L__BB2_1706: + mov.u32 %r7904, 1; + shl.b32 %r10042, %r7904, %r10425; + mov.u32 %r10043, %r10421; + bra.uni $L__BB2_1715; + +$L__BB2_1707: + add.s32 %r10040, %r10040, 1; + setp.lt.u32 %p2143, %r10040, %r10042; + @%p2143 bra $L__BB2_1715; + + shl.b16 %rs1016, %rs1253, 1; + or.b16 %rs1253, %rs1016, 1; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p2144, %r9835, 0; + mov.u32 %r10428, %r10043; + @%p2144 bra $L__BB2_1711; + bra.uni $L__BB2_1709; + +$L__BB2_1711: + add.s32 %r7908, %r10041, 1; + min.u32 %r10041, %r7908, 12; + setp.lt.u32 %p2147, %r10041, 3; + mov.u32 %r10040, 0; + mov.u32 %r10429, %r10040; + @%p2147 bra $L__BB2_1714; + + setp.lt.u32 %p2148, %r10041, 6; + mov.u32 %r10429, 1; + @%p2148 bra $L__BB2_1714; + + setp.lt.u32 %p2149, %r10041, 9; + setp.eq.s32 %p2150, %r10041, 11; + selp.b32 %r7910, 4, 5, %p2150; + setp.lt.u32 %p2151, %r10041, 11; + selp.b32 %r7911, 3, %r7910, %p2151; + selp.b32 %r10429, 2, %r7911, %p2149; + +$L__BB2_1714: + mov.u32 %r7913, 1; + shl.b32 %r10042, %r7913, %r10429; + mov.u32 %r10043, %r10428; + +$L__BB2_1715: + and.b16 %rs1019, %rs426, 15; + cvt.u32.u16 %r3528, %rs1019; + and.b32 %r7914, %r10355, 1; + setp.eq.b32 %p2152, %r7914, 1; + mov.pred %p2153, 0; + xor.pred %p2154, %p2152, %p2153; + not.pred %p2155, %p2154; + mov.u32 %r10450, %r10495; + @%p2155 bra $L__BB2_1722; + + and.b32 %r7915, %r3528, 1; + sub.s32 %r10436, %r3414, %r7915; + setp.eq.s32 %p2156, %r10436, 0; + mov.u32 %r10450, %r10495; + @%p2156 bra $L__BB2_1722; + + mov.u32 %r7916, -1; + shl.b32 %r7917, %r7916, %r10436; + not.b32 %r7918, %r7917; + and.b32 %r10437, %r10349, %r7918; + +$L__BB2_1718: + setp.gt.u32 %p2157, %r10461, 17476; + mov.u32 %r10450, 1; + @%p2157 bra $L__BB2_1722; + + sub.s32 %r7920, %r10462, %r10463; + min.u32 %r7921, %r7920, %r10436; + setp.eq.s32 %p2158, %r7921, 32; + mov.u32 %r7922, -1; + shl.b32 %r7923, %r7922, %r7921; + not.b32 %r7924, %r7923; + selp.b32 %r7925, -1, %r7924, %p2158; + and.b32 %r7926, %r7925, %r10437; + shl.b32 %r7927, %r7926, %r10463; + or.b32 %r10464, %r7927, %r10464; + add.s32 %r10463, %r7921, %r10463; + shr.u32 %r10437, %r10437, %r7921; + sub.s32 %r10436, %r10436, %r7921; + setp.lt.u32 %p2159, %r10463, %r10462; + @%p2159 bra $L__BB2_1721; + + cvt.u64.u32 %rd1310, %r10461; + add.s64 %rd1311, %rd1310, %rd5; + add.s64 %rd1312, %rd1, %rd1311; + st.global.u8 [%rd1312], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p2160, %r10464, 255; + selp.b32 %r10462, 7, 8, %p2160; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1721: + setp.ne.s32 %p2161, %r10436, 0; + mov.u32 %r10450, %r10495; + @%p2161 bra $L__BB2_1718; + +$L__BB2_1722: + and.b32 %r3552, %r10355, 2; + setp.eq.s32 %p2162, %r3552, 0; + mov.u32 %r10465, %r10450; + @%p2162 bra $L__BB2_1729; + + shr.u32 %r7930, %r3528, 1; + and.b32 %r7931, %r7930, 1; + sub.s32 %r10451, %r3414, %r7931; + setp.eq.s32 %p2163, %r10451, 0; + mov.u32 %r10465, %r10450; + @%p2163 bra $L__BB2_1729; + + mov.u32 %r7932, -1; + shl.b32 %r7933, %r7932, %r10451; + not.b32 %r7934, %r7933; + and.b32 %r10452, %r10353, %r7934; + +$L__BB2_1725: + setp.gt.u32 %p2164, %r10461, 17476; + mov.u32 %r10465, 1; + @%p2164 bra $L__BB2_1729; + + sub.s32 %r7936, %r10462, %r10463; + min.u32 %r7937, %r7936, %r10451; + setp.eq.s32 %p2165, %r7937, 32; + mov.u32 %r7938, -1; + shl.b32 %r7939, %r7938, %r7937; + not.b32 %r7940, %r7939; + selp.b32 %r7941, -1, %r7940, %p2165; + and.b32 %r7942, %r7941, %r10452; + shl.b32 %r7943, %r7942, %r10463; + or.b32 %r10464, %r7943, %r10464; + add.s32 %r10463, %r7937, %r10463; + shr.u32 %r10452, %r10452, %r7937; + sub.s32 %r10451, %r10451, %r7937; + setp.lt.u32 %p2166, %r10463, %r10462; + @%p2166 bra $L__BB2_1728; + + cvt.u64.u32 %rd1313, %r10461; + add.s64 %rd1314, %rd1313, %rd5; + add.s64 %rd1315, %rd1, %rd1314; + st.global.u8 [%rd1315], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p2167, %r10464, 255; + selp.b32 %r10462, 7, 8, %p2167; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1728: + setp.ne.s32 %p2168, %r10451, 0; + mov.u32 %r10465, %r10450; + @%p2168 bra $L__BB2_1725; + +$L__BB2_1729: + and.b32 %r3576, %r10355, 4; + setp.eq.s32 %p2169, %r3576, 0; + mov.u32 %r10480, %r10465; + @%p2169 bra $L__BB2_1736; + + shr.u32 %r7946, %r3528, 2; + and.b32 %r7947, %r7946, 1; + sub.s32 %r10466, %r3414, %r7947; + setp.eq.s32 %p2170, %r10466, 0; + mov.u32 %r10480, %r10465; + @%p2170 bra $L__BB2_1736; + + mov.u32 %r7948, -1; + shl.b32 %r7949, %r7948, %r10466; + not.b32 %r7950, %r7949; + and.b32 %r10467, %r10369, %r7950; + +$L__BB2_1732: + setp.gt.u32 %p2171, %r10461, 17476; + mov.u32 %r10480, 1; + @%p2171 bra $L__BB2_1736; + + sub.s32 %r7952, %r10462, %r10463; + min.u32 %r7953, %r7952, %r10466; + setp.eq.s32 %p2172, %r7953, 32; + mov.u32 %r7954, -1; + shl.b32 %r7955, %r7954, %r7953; + not.b32 %r7956, %r7955; + selp.b32 %r7957, -1, %r7956, %p2172; + and.b32 %r7958, %r7957, %r10467; + shl.b32 %r7959, %r7958, %r10463; + or.b32 %r10464, %r7959, %r10464; + add.s32 %r10463, %r7953, %r10463; + shr.u32 %r10467, %r10467, %r7953; + sub.s32 %r10466, %r10466, %r7953; + setp.lt.u32 %p2173, %r10463, %r10462; + @%p2173 bra $L__BB2_1735; + + cvt.u64.u32 %rd1316, %r10461; + add.s64 %rd1317, %rd1316, %rd5; + add.s64 %rd1318, %rd1, %rd1317; + st.global.u8 [%rd1318], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p2174, %r10464, 255; + selp.b32 %r10462, 7, 8, %p2174; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1735: + setp.ne.s32 %p2175, %r10466, 0; + mov.u32 %r10480, %r10465; + @%p2175 bra $L__BB2_1732; + +$L__BB2_1736: + and.b32 %r3600, %r10355, 8; + setp.eq.s32 %p2176, %r3600, 0; + mov.u32 %r10495, %r10480; + @%p2176 bra $L__BB2_1743; + + shr.u32 %r7962, %r3528, 3; + sub.s32 %r10481, %r3414, %r7962; + setp.eq.s32 %p2177, %r10481, 0; + mov.u32 %r10495, %r10480; + @%p2177 bra $L__BB2_1743; + + mov.u32 %r7963, -1; + shl.b32 %r7964, %r7963, %r10481; + not.b32 %r7965, %r7964; + and.b32 %r10482, %r10368, %r7965; + +$L__BB2_1739: + setp.gt.u32 %p2178, %r10461, 17476; + mov.u32 %r10495, 1; + @%p2178 bra $L__BB2_1743; + + sub.s32 %r7967, %r10462, %r10463; + min.u32 %r7968, %r7967, %r10481; + setp.eq.s32 %p2179, %r7968, 32; + mov.u32 %r7969, -1; + shl.b32 %r7970, %r7969, %r7968; + not.b32 %r7971, %r7970; + selp.b32 %r7972, -1, %r7971, %p2179; + and.b32 %r7973, %r7972, %r10482; + shl.b32 %r7974, %r7973, %r10463; + or.b32 %r10464, %r7974, %r10464; + add.s32 %r10463, %r7968, %r10463; + shr.u32 %r10482, %r10482, %r7968; + sub.s32 %r10481, %r10481, %r7968; + setp.lt.u32 %p2180, %r10463, %r10462; + @%p2180 bra $L__BB2_1742; + + cvt.u64.u32 %rd1319, %r10461; + add.s64 %rd1320, %rd1319, %rd5; + add.s64 %rd1321, %rd1, %rd1320; + st.global.u8 [%rd1321], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p2181, %r10464, 255; + selp.b32 %r10462, 7, 8, %p2181; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1742: + setp.ne.s32 %p2182, %r10481, 0; + mov.u32 %r10495, %r10480; + @%p2182 bra $L__BB2_1739; + +$L__BB2_1743: + add.s32 %r3624, %r4103, %r10344; + ld.shared.u8 %rs1020, [%r3624]; + mov.u32 %r10342, 0; + cvt.u32.u16 %r7980, %rs1020; + and.b32 %r7981, %r7980, 255; + and.b32 %r7982, %r10352, 255; + setp.lt.u32 %p2183, %r7982, %r7981; + cvt.u16.u32 %rs1021, %r10352; + selp.b16 %rs1022, %rs1020, %rs1021, %p2183; + st.shared.u8 [%r3624], %rs1022; + ld.shared.u8 %rs448, [%r3624+2]; + ld.shared.u8 %rs1023, [%r3624+1]; + setp.gt.u16 %p2184, %rs1023, %rs448; + add.s32 %r10647, %r10344, 1; + add.s32 %r7983, %r10344, 2; + selp.b32 %r7984, %r10647, %r7983, %p2184; + add.s32 %r7985, %r4103, %r7984; + ld.shared.u8 %rs449, [%r7985]; + cvt.u32.u16 %r7986, %rs449; + and.b32 %r7987, %r7986, 255; + add.s32 %r10345, %r7987, -1; + cvt.u16.u32 %rs450, %r10366; + cvt.u16.u32 %rs1024, %r3552; + shr.u16 %rs1025, %rs1024, 1; + mov.u32 %r7988, _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val; + add.s32 %r3627, %r7988, %r10346; + st.shared.u8 [%r3624+1], %r10366; + ld.shared.u8 %rs1026, [%r3627]; + or.b16 %rs1027, %rs1026, %rs1025; + st.shared.u8 [%r3627], %rs1027; + add.s32 %r10346, %r10346, 1; + ld.shared.u8 %rs451, [%r3627+1]; + ld.shared.u8 %r3629, [%r3627+2]; + shr.u32 %r3630, %r3600, 3; + st.shared.u8 [%r3627+1], %r3630; + add.s32 %r7989, %r10326, 2; + setp.ge.u32 %p2185, %r7989, %r4057; + mov.u32 %r10665, %r10342; + @%p2185 bra $L__BB2_1850; + + cvt.u64.u32 %rd1322, %r10648; + add.s64 %rd1323, %rd1322, %rd4; + shl.b64 %rd1324, %rd1323, 2; + add.s64 %rd1325, %rd3, %rd1324; + ld.global.u32 %r3631, [%rd1325]; + setp.eq.s32 %p2186, %r3631, 0; + mov.u32 %r10497, 0; + mov.u32 %r10496, %r10497; + @%p2186 bra $L__BB2_1746; + + and.b32 %r7991, %r3631, -2147483648; + abs.s32 %r7992, %r3631; + shl.b32 %r7993, %r7992, %r2358; + or.b32 %r10496, %r7993, %r7991; + +$L__BB2_1746: + shl.b32 %r7997, %r10496, 1; + shr.u32 %r7998, %r7997, %r2358; + and.b32 %r3634, %r7998, -2; + setp.eq.s32 %p2187, %r3634, 0; + mov.u32 %r10498, %r10497; + mov.u32 %r10504, %r10497; + @%p2187 bra $L__BB2_1748; + + add.s32 %r8000, %r3634, -1; + clz.b32 %r8001, %r8000; + mov.u32 %r8002, 32; + sub.s32 %r10497, %r8002, %r8001; + shr.u32 %r8003, %r10496, 31; + add.s32 %r8004, %r8003, %r3634; + add.s32 %r10498, %r8004, -2; + mov.u32 %r10504, 1; + +$L__BB2_1748: + mov.u32 %r10501, 0; + mov.u32 %r10500, %r10501; + @%p2069 bra $L__BB2_1751; + + add.s32 %r8007, %r10648, %r4055; + cvt.u64.u32 %rd1326, %r8007; + add.s64 %rd1327, %rd1326, %rd4; + shl.b64 %rd1328, %rd1327, 2; + add.s64 %rd1329, %rd3, %rd1328; + ld.global.u32 %r3640, [%rd1329]; + setp.eq.s32 %p2189, %r3640, 0; + @%p2189 bra $L__BB2_1751; + + and.b32 %r8008, %r3640, -2147483648; + abs.s32 %r8009, %r3640; + shl.b32 %r8010, %r8009, %r2358; + or.b32 %r10500, %r8010, %r8008; + +$L__BB2_1751: + shl.b32 %r8013, %r10500, 1; + shr.u32 %r8014, %r8013, %r2358; + and.b32 %r3643, %r8014, -2; + setp.eq.s32 %p2190, %r3643, 0; + mov.u32 %r10502, %r10501; + mov.u32 %r10520, %r10497; + @%p2190 bra $L__BB2_1753; + + or.b32 %r10504, %r10504, 2; + add.s32 %r8015, %r3643, -1; + clz.b32 %r8016, %r8015; + mov.u32 %r8017, 32; + sub.s32 %r10501, %r8017, %r8016; + max.s32 %r10520, %r10497, %r10501; + shr.u32 %r8018, %r10500, 31; + add.s32 %r8019, %r8018, %r3643; + add.s32 %r10502, %r8019, -2; + +$L__BB2_1753: + add.s32 %r10519, %r10648, 1; + add.s32 %r8024, %r10326, 3; + setp.ge.u32 %p2191, %r8024, %r4057; + mov.u32 %r10522, 0; + mov.u32 %r10515, %r10522; + mov.u32 %r10516, %r10522; + mov.u32 %r10517, %r10522; + mov.u32 %r10518, %r10522; + @%p2191 bra $L__BB2_1764; + + cvt.u64.u32 %rd1330, %r10519; + add.s64 %rd1331, %rd1330, %rd4; + shl.b64 %rd1332, %rd1331, 2; + add.s64 %rd1333, %rd3, %rd1332; + ld.global.u32 %r3653, [%rd1333]; + setp.eq.s32 %p2192, %r3653, 0; + mov.u32 %r10516, 0; + mov.u32 %r10505, %r10516; + @%p2192 bra $L__BB2_1756; + + and.b32 %r8026, %r3653, -2147483648; + abs.s32 %r8027, %r3653; + shl.b32 %r8028, %r8027, %r2358; + or.b32 %r10505, %r8028, %r8026; + +$L__BB2_1756: + shl.b32 %r8031, %r10505, 1; + shr.u32 %r8032, %r8031, %r2358; + and.b32 %r3656, %r8032, -2; + setp.eq.s32 %p2193, %r3656, 0; + mov.u32 %r10518, %r10516; + @%p2193 bra $L__BB2_1758; + + or.b32 %r10504, %r10504, 4; + add.s32 %r8033, %r3656, -1; + clz.b32 %r8034, %r8033; + mov.u32 %r8035, 32; + sub.s32 %r10516, %r8035, %r8034; + max.s32 %r10520, %r10520, %r10516; + shr.u32 %r8036, %r10505, 31; + add.s32 %r8037, %r8036, %r3656; + add.s32 %r10518, %r8037, -2; + +$L__BB2_1758: + mov.u32 %r10515, 0; + mov.u32 %r10510, %r10515; + @%p2069 bra $L__BB2_1761; + + add.s32 %r8040, %r10519, %r4055; + cvt.u64.u32 %rd1334, %r8040; + add.s64 %rd1335, %rd1334, %rd4; + shl.b64 %rd1336, %rd1335, 2; + add.s64 %rd1337, %rd3, %rd1336; + ld.global.u32 %r3665, [%rd1337]; + setp.eq.s32 %p2195, %r3665, 0; + @%p2195 bra $L__BB2_1761; + + and.b32 %r8041, %r3665, -2147483648; + abs.s32 %r8042, %r3665; + shl.b32 %r8043, %r8042, %r2358; + or.b32 %r10510, %r8043, %r8041; + +$L__BB2_1761: + shl.b32 %r8046, %r10510, 1; + shr.u32 %r8047, %r8046, %r2358; + and.b32 %r3668, %r8047, -2; + setp.eq.s32 %p2196, %r3668, 0; + mov.u32 %r10517, %r10515; + @%p2196 bra $L__BB2_1763; + + or.b32 %r10504, %r10504, 8; + add.s32 %r8048, %r3668, -1; + clz.b32 %r8049, %r8048; + mov.u32 %r8050, 32; + sub.s32 %r10515, %r8050, %r8049; + max.s32 %r10520, %r10520, %r10515; + shr.u32 %r8051, %r10510, 31; + add.s32 %r8052, %r8051, %r3668; + add.s32 %r10517, %r8052, -2; + +$L__BB2_1763: + add.s32 %r10519, %r10648, 2; + +$L__BB2_1764: + mov.u32 %r10648, %r10519; + shr.u32 %r8054, %r3600, 2; + shr.u32 %r8055, %r3576, 1; + or.b32 %r8056, %r8054, %r8055; + cvt.u32.u16 %r8057, %rs451; + and.b32 %r8058, %r8057, 255; + shl.b32 %r8059, %r3629, 2; + add.s32 %r8060, %r8059, %r8058; + or.b32 %r3685, %r8056, %r8060; + add.s32 %r8061, %r10504, -1; + and.b32 %r8062, %r8061, %r10504; + setp.ne.s32 %p2197, %r8062, 0; + setp.gt.u16 %p2198, %rs449, 2; + and.pred %p2199, %p2198, %p2197; + selp.b32 %r8063, %r10345, 1, %p2199; + max.s32 %r3686, %r8063, %r10520; + sub.s32 %r10665, %r3686, %r8063; + setp.lt.s32 %p2200, %r10665, 1; + @%p2200 bra $L__BB2_1766; + + setp.eq.s32 %p2201, %r10497, %r10520; + selp.u32 %r8064, 1, 0, %p2201; + setp.eq.s32 %p2202, %r10501, %r10520; + selp.u32 %r8065, -1, 0, %p2202; + bfi.b32 %r8066, %r8065, %r8064, 1, 1; + setp.eq.s32 %p2203, %r10516, %r10520; + selp.u16 %rs1029, 1, 0, %p2203; + mul.wide.u16 %r8067, %rs1029, 4; + or.b32 %r8068, %r8066, %r8067; + setp.eq.s32 %p2204, %r10515, %r10520; + selp.u16 %rs1030, 1, 0, %p2204; + mul.wide.u16 %r8069, %rs1030, 8; + or.b32 %r10522, %r8068, %r8069; + +$L__BB2_1766: + shl.b32 %r8070, %r10504, 4; + shl.b32 %r8071, %r3685, 8; + or.b32 %r8072, %r8070, %r8071; + or.b32 %r8073, %r8072, %r10522; + mul.wide.u32 %rd1339, %r8073, 2; + add.s64 %rd1340, %rd72, %rd1339; + ld.global.u16 %rs452, [%rd1340]; + shr.u16 %rs1031, %rs452, 4; + and.b16 %rs453, %rs1031, 7; + setp.eq.s16 %p2205, %rs453, 0; + mov.u32 %r10534, %r10385; + @%p2205 bra $L__BB2_1773; + + cvt.u32.u16 %r10523, %rs453; + shr.u16 %rs1032, %rs452, 8; + cvt.u32.u16 %r10524, %rs1032; + +$L__BB2_1768: + mov.u32 %r3692, %r10523; + setp.gt.u32 %p2206, %r10274, 2879; + mov.u32 %r10534, 1; + @%p2206 bra $L__BB2_1773; + + mov.u32 %r8075, 8; + sub.s32 %r8076, %r8075, %r10276; + sub.s32 %r8077, %r8076, %r10275; + min.u32 %r8078, %r8077, %r3692; + setp.eq.s32 %p2207, %r8078, 32; + mov.u32 %r8079, -1; + shl.b32 %r8080, %r8079, %r8078; + not.b32 %r8081, %r8080; + selp.b32 %r8082, -1, %r8081, %p2207; + and.b32 %r8083, %r8082, %r10524; + shl.b32 %r8084, %r8083, %r10275; + cvt.u16.u32 %rs1033, %r8084; + or.b16 %rs1322, %rs1322, %rs1033; + add.s32 %r10275, %r8078, %r10275; + sub.s32 %r10523, %r3692, %r8078; + shr.u32 %r10524, %r10524, %r8078; + setp.gt.u32 %p2208, %r8077, %r3692; + @%p2208 bra $L__BB2_1772; + + setp.ne.s32 %p2209, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs1034, %rs1322, 255; + setp.ne.s16 %p2210, %rs1034, 127; + and.pred %p2211, %p2209, %p2210; + @%p2211 bra $L__BB2_1772; + + mov.u32 %r8087, 20548; + sub.s32 %r8088, %r8087, %r10274; + cvt.u64.u32 %rd1341, %r8088; + add.s64 %rd1342, %rd1341, %rd5; + add.s64 %rd1343, %rd1, %rd1342; + st.global.u8 [%rd1343], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2212, %rs1034, 143; + selp.u32 %r10276, 1, 0, %p2212; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1772: + setp.ne.s32 %p2213, %r10523, 0; + mov.u32 %r10534, %r10385; + @%p2213 bra $L__BB2_1768; + +$L__BB2_1773: + setp.ne.s32 %p2214, %r3685, 0; + @%p2214 bra $L__BB2_1821; + + setp.eq.s32 %p2215, %r10504, 0; + add.s32 %r8089, %r9826, 17477; + cvt.u64.u32 %rd1344, %r8089; + add.s64 %rd1345, %rd1344, %rd5; + add.s64 %rd74, %rd1, %rd1345; + @%p2215 bra $L__BB2_1813; + + shl.b16 %rs1253, %rs1253, 1; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p2216, %r9835, 0; + mov.u32 %r10570, %r10043; + @%p2216 bra $L__BB2_1778; + + setp.gt.u32 %p2217, %r9826, 191; + mov.u32 %r10570, 1; + mov.u32 %r9835, 0; + @%p2217 bra $L__BB2_1778; + + st.global.u8 [%rd74], %rs1253; + add.s32 %r9826, %r9826, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9835, 8; + mov.u32 %r10570, %r10043; + +$L__BB2_1778: + setp.lt.u32 %p2218, %r10041, 3; + mov.u32 %r10538, 0; + @%p2218 bra $L__BB2_1781; + + setp.lt.u32 %p2219, %r10041, 6; + mov.u32 %r10538, 1; + @%p2219 bra $L__BB2_1781; + + setp.lt.u32 %p2220, %r10041, 9; + setp.eq.s32 %p2221, %r10041, 11; + selp.b32 %r8095, 4, 5, %p2221; + setp.lt.u32 %p2222, %r10041, 11; + selp.b32 %r8096, 3, %r8095, %p2222; + selp.b32 %r10538, 2, %r8096, %p2220; + +$L__BB2_1781: + setp.eq.s32 %p2223, %r10538, 0; + @%p2223 bra $L__BB2_1809; + + add.s32 %r3716, %r10538, -1; + and.b32 %r3717, %r10538, 3; + setp.eq.s32 %p2224, %r3717, 0; + mov.u32 %r10548, %r10538; + mov.u32 %r10549, %r10570; + @%p2224 bra $L__BB2_1794; + + mov.u32 %r8098, 1; + shl.b32 %r8099, %r8098, %r3716; + and.b32 %r8100, %r8099, %r10040; + setp.ne.s32 %p2225, %r8100, 0; + selp.u32 %r8101, 1, 0, %p2225; + cvt.u32.u16 %r8102, %rs1253; + bfi.b32 %r8103, %r8102, %r8101, 1, 8; + cvt.u16.u32 %rs1253, %r8103; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p2226, %r9835, 0; + mov.u32 %r10549, %r10570; + @%p2226 bra $L__BB2_1786; + + setp.gt.u32 %p2227, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r10549, %r8098; + @%p2227 bra $L__BB2_1786; + + add.s32 %r8107, %r9826, 17477; + cvt.u64.u32 %rd1346, %r8107; + add.s64 %rd1347, %rd1346, %rd5; + add.s64 %rd1348, %rd1, %rd1347; + st.global.u8 [%rd1348], %rs1253; + add.s32 %r9826, %r9826, 1; + mov.u16 %rs1253, 0; + mov.u32 %r9835, 8; + mov.u32 %r10549, %r10570; + +$L__BB2_1786: + setp.eq.s32 %p2228, %r3717, 1; + mov.u32 %r10570, %r10549; + mov.u32 %r10548, %r3716; + @%p2228 bra $L__BB2_1794; + + add.s32 %r10548, %r10538, -2; + mov.u32 %r8108, 1; + shl.b32 %r8109, %r8108, %r10548; + and.b32 %r8110, %r8109, %r10040; + setp.ne.s32 %p2229, %r8110, 0; + selp.u32 %r8111, 1, 0, %p2229; + cvt.u32.u16 %r8112, %rs1253; + bfi.b32 %r8113, %r8112, %r8111, 1, 8; + cvt.u16.u32 %rs1253, %r8113; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p2230, %r9835, 0; + mov.u32 %r10544, %r10549; + @%p2230 bra $L__BB2_1790; + + setp.gt.u32 %p2231, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r10544, %r8108; + @%p2231 bra $L__BB2_1790; + + add.s32 %r8116, %r9826, 17477; + cvt.u64.u32 %rd1349, %r8116; + add.s64 %rd1350, %rd1349, %rd5; + add.s64 %rd1351, %rd1, %rd1350; + and.b16 %rs1041, %rs1253, 255; + st.global.u8 [%rd1351], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2232, %rs1041, 255; + selp.b32 %r9835, 7, 8, %p2232; + mov.u16 %rs1253, 0; + mov.u32 %r10544, %r10549; + +$L__BB2_1790: + setp.eq.s32 %p2233, %r3717, 2; + mov.u32 %r10570, %r10544; + mov.u32 %r10549, %r10544; + @%p2233 bra $L__BB2_1794; + + add.s32 %r10548, %r10538, -3; + mov.u32 %r8117, 1; + shl.b32 %r8118, %r8117, %r10548; + and.b32 %r8119, %r8118, %r10040; + setp.ne.s32 %p2234, %r8119, 0; + selp.u32 %r8120, 1, 0, %p2234; + cvt.u32.u16 %r8121, %rs1253; + bfi.b32 %r8122, %r8121, %r8120, 1, 8; + cvt.u16.u32 %rs1253, %r8122; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p2235, %r9835, 0; + mov.u32 %r10570, %r10544; + mov.u32 %r10549, %r10544; + @%p2235 bra $L__BB2_1794; + + setp.gt.u32 %p2236, %r9826, 191; + mov.u32 %r9835, 0; + mov.u32 %r10570, %r8117; + mov.u32 %r10549, %r8117; + @%p2236 bra $L__BB2_1794; + + add.s32 %r8127, %r9826, 17477; + cvt.u64.u32 %rd1352, %r8127; + add.s64 %rd1353, %rd1352, %rd5; + add.s64 %rd1354, %rd1, %rd1353; + and.b16 %rs1044, %rs1253, 255; + st.global.u8 [%rd1354], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2237, %rs1044, 255; + selp.b32 %r9835, 7, 8, %p2237; + mov.u16 %rs1253, 0; + mov.u32 %r10570, %r10544; + mov.u32 %r10549, %r10544; + +$L__BB2_1794: + setp.lt.u32 %p2238, %r3716, 3; + @%p2238 bra $L__BB2_1809; + + mov.u32 %r10570, %r10549; + +$L__BB2_1796: + add.s32 %r8128, %r10548, -1; + mov.u32 %r8129, 1; + shl.b32 %r8130, %r8129, %r8128; + and.b32 %r8131, %r8130, %r10040; + setp.ne.s32 %p2239, %r8131, 0; + selp.u32 %r8132, 1, 0, %p2239; + cvt.u32.u16 %r8133, %rs1253; + bfi.b32 %r10558, %r8133, %r8132, 1, 8; + add.s32 %r10557, %r9835, -1; + setp.ne.s32 %p2240, %r10557, 0; + mov.u32 %r10559, %r10570; + @%p2240 bra $L__BB2_1799; + + setp.gt.u32 %p2241, %r9826, 191; + mov.u32 %r10557, 0; + mov.u32 %r10559, %r8129; + @%p2241 bra $L__BB2_1799; + + cvt.u16.u32 %rs1045, %r10558; + and.b16 %rs1046, %rs1045, 255; + add.s32 %r8137, %r9826, 17477; + cvt.u64.u32 %rd1355, %r8137; + add.s64 %rd1356, %rd1355, %rd5; + add.s64 %rd1357, %rd1, %rd1356; + st.global.u8 [%rd1357], %rs1045; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2242, %rs1046, 255; + selp.b32 %r10557, 7, 8, %p2242; + mov.u32 %r10558, 0; + mov.u32 %r10559, %r10570; + +$L__BB2_1799: + add.s32 %r8138, %r10548, -2; + shl.b32 %r8140, %r8129, %r8138; + and.b32 %r8141, %r8140, %r10040; + setp.ne.s32 %p2243, %r8141, 0; + and.b32 %r8142, %r10558, 127; + selp.u32 %r8143, 1, 0, %p2243; + bfi.b32 %r10562, %r8142, %r8143, 1, 7; + add.s32 %r10561, %r10557, -1; + setp.ne.s32 %p2244, %r10561, 0; + mov.u32 %r10563, %r10559; + @%p2244 bra $L__BB2_1802; + + setp.gt.u32 %p2245, %r9826, 191; + mov.u32 %r10563, 1; + mov.u32 %r10561, 0; + @%p2245 bra $L__BB2_1802; + + cvt.u16.u32 %rs1047, %r10562; + and.b16 %rs1048, %rs1047, 255; + add.s32 %r8147, %r9826, 17477; + cvt.u64.u32 %rd1358, %r8147; + add.s64 %rd1359, %rd1358, %rd5; + add.s64 %rd1360, %rd1, %rd1359; + st.global.u8 [%rd1360], %rs1047; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2246, %rs1048, 255; + selp.b32 %r10561, 7, 8, %p2246; + mov.u32 %r10562, 0; + mov.u32 %r10563, %r10559; + +$L__BB2_1802: + add.s32 %r8148, %r10548, -3; + mov.u32 %r8149, 1; + shl.b32 %r8150, %r8149, %r8148; + and.b32 %r8151, %r8150, %r10040; + setp.ne.s32 %p2247, %r8151, 0; + and.b32 %r8152, %r10562, 127; + selp.u32 %r8153, 1, 0, %p2247; + bfi.b32 %r10566, %r8152, %r8153, 1, 7; + add.s32 %r10565, %r10561, -1; + setp.ne.s32 %p2248, %r10565, 0; + mov.u32 %r10567, %r10563; + @%p2248 bra $L__BB2_1805; + + setp.gt.u32 %p2249, %r9826, 191; + mov.u32 %r10565, 0; + mov.u32 %r10567, %r8149; + @%p2249 bra $L__BB2_1805; + + cvt.u16.u32 %rs1049, %r10566; + and.b16 %rs1050, %rs1049, 255; + add.s32 %r8157, %r9826, 17477; + cvt.u64.u32 %rd1361, %r8157; + add.s64 %rd1362, %rd1361, %rd5; + add.s64 %rd1363, %rd1, %rd1362; + st.global.u8 [%rd1363], %rs1049; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2250, %rs1050, 255; + selp.b32 %r10565, 7, 8, %p2250; + mov.u32 %r10566, 0; + mov.u32 %r10567, %r10563; + +$L__BB2_1805: + add.s32 %r10548, %r10548, -4; + shl.b32 %r8159, %r8149, %r10548; + and.b32 %r8160, %r8159, %r10040; + setp.ne.s32 %p2251, %r8160, 0; + and.b32 %r8161, %r10566, 127; + selp.u32 %r8162, 1, 0, %p2251; + bfi.b32 %r8163, %r8161, %r8162, 1, 15; + cvt.u16.u32 %rs1253, %r8163; + add.s32 %r9835, %r10565, -1; + setp.ne.s32 %p2252, %r9835, 0; + mov.u32 %r10570, %r10567; + @%p2252 bra $L__BB2_1808; + + setp.gt.u32 %p2253, %r9826, 191; + mov.u32 %r10570, 1; + mov.u32 %r9835, 0; + @%p2253 bra $L__BB2_1808; + + add.s32 %r8166, %r9826, 17477; + cvt.u64.u32 %rd1364, %r8166; + add.s64 %rd1365, %rd1364, %rd5; + add.s64 %rd1366, %rd1, %rd1365; + and.b16 %rs1052, %rs1253, 255; + st.global.u8 [%rd1366], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2254, %rs1052, 255; + selp.b32 %r9835, 7, 8, %p2254; + mov.u16 %rs1253, 0; + mov.u32 %r10570, %r10567; + +$L__BB2_1808: + setp.ne.s32 %p2255, %r10548, 0; + @%p2255 bra $L__BB2_1796; + +$L__BB2_1809: + add.s32 %r8168, %r10041, -1; + setp.eq.s32 %p2256, %r10041, 0; + mov.u32 %r10040, 0; + selp.b32 %r10041, 0, %r8168, %p2256; + setp.lt.u32 %p2257, %r10041, 3; + mov.u32 %r10574, %r10040; + @%p2257 bra $L__BB2_1812; + + setp.lt.u32 %p2258, %r10041, 6; + mov.u32 %r10574, 1; + @%p2258 bra $L__BB2_1812; + + setp.lt.u32 %p2259, %r10041, 9; + setp.eq.s32 %p2260, %r10041, 11; + selp.b32 %r8170, 4, 5, %p2260; + setp.lt.u32 %p2261, %r10041, 11; + selp.b32 %r8171, 3, %r8170, %p2261; + selp.b32 %r10574, 2, %r8171, %p2259; + +$L__BB2_1812: + mov.u32 %r8173, 1; + shl.b32 %r10042, %r8173, %r10574; + mov.u32 %r10043, %r10570; + bra.uni $L__BB2_1821; + +$L__BB2_1813: + add.s32 %r10040, %r10040, 1; + setp.lt.u32 %p2262, %r10040, %r10042; + @%p2262 bra $L__BB2_1821; + + shl.b16 %rs1053, %rs1253, 1; + or.b16 %rs1253, %rs1053, 1; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p2263, %r9835, 0; + mov.u32 %r10577, %r10043; + @%p2263 bra $L__BB2_1817; + bra.uni $L__BB2_1815; + +$L__BB2_1817: + add.s32 %r8177, %r10041, 1; + min.u32 %r10041, %r8177, 12; + setp.lt.u32 %p2266, %r10041, 3; + mov.u32 %r10040, 0; + mov.u32 %r10578, %r10040; + @%p2266 bra $L__BB2_1820; + + setp.lt.u32 %p2267, %r10041, 6; + mov.u32 %r10578, 1; + @%p2267 bra $L__BB2_1820; + + setp.lt.u32 %p2268, %r10041, 9; + setp.eq.s32 %p2269, %r10041, 11; + selp.b32 %r8179, 4, 5, %p2269; + setp.lt.u32 %p2270, %r10041, 11; + selp.b32 %r8180, 3, %r8179, %p2270; + selp.b32 %r10578, 2, %r8180, %p2268; + +$L__BB2_1820: + mov.u32 %r8182, 1; + shl.b32 %r10042, %r8182, %r10578; + mov.u32 %r10043, %r10577; + +$L__BB2_1821: + and.b16 %rs1056, %rs452, 15; + cvt.u32.u16 %r3800, %rs1056; + and.b32 %r8183, %r10504, 1; + setp.eq.b32 %p2271, %r8183, 1; + mov.pred %p2272, 0; + xor.pred %p2273, %p2271, %p2272; + not.pred %p2274, %p2273; + mov.u32 %r10599, %r10495; + @%p2274 bra $L__BB2_1828; + + and.b32 %r8184, %r3800, 1; + sub.s32 %r10585, %r3686, %r8184; + setp.eq.s32 %p2275, %r10585, 0; + mov.u32 %r10599, %r10495; + @%p2275 bra $L__BB2_1828; + + mov.u32 %r8185, -1; + shl.b32 %r8186, %r8185, %r10585; + not.b32 %r8187, %r8186; + and.b32 %r10586, %r10498, %r8187; + +$L__BB2_1824: + setp.gt.u32 %p2276, %r10461, 17476; + mov.u32 %r10599, 1; + @%p2276 bra $L__BB2_1828; + + sub.s32 %r8189, %r10462, %r10463; + min.u32 %r8190, %r8189, %r10585; + setp.eq.s32 %p2277, %r8190, 32; + mov.u32 %r8191, -1; + shl.b32 %r8192, %r8191, %r8190; + not.b32 %r8193, %r8192; + selp.b32 %r8194, -1, %r8193, %p2277; + and.b32 %r8195, %r8194, %r10586; + shl.b32 %r8196, %r8195, %r10463; + or.b32 %r10464, %r8196, %r10464; + add.s32 %r10463, %r8190, %r10463; + shr.u32 %r10586, %r10586, %r8190; + sub.s32 %r10585, %r10585, %r8190; + setp.lt.u32 %p2278, %r10463, %r10462; + @%p2278 bra $L__BB2_1827; + + cvt.u64.u32 %rd1367, %r10461; + add.s64 %rd1368, %rd1367, %rd5; + add.s64 %rd1369, %rd1, %rd1368; + st.global.u8 [%rd1369], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p2279, %r10464, 255; + selp.b32 %r10462, 7, 8, %p2279; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1827: + setp.ne.s32 %p2280, %r10585, 0; + mov.u32 %r10599, %r10495; + @%p2280 bra $L__BB2_1824; + +$L__BB2_1828: + and.b32 %r3824, %r10504, 2; + setp.eq.s32 %p2281, %r3824, 0; + mov.u32 %r10614, %r10599; + @%p2281 bra $L__BB2_1835; + + shr.u32 %r8199, %r3800, 1; + and.b32 %r8200, %r8199, 1; + sub.s32 %r10600, %r3686, %r8200; + setp.eq.s32 %p2282, %r10600, 0; + mov.u32 %r10614, %r10599; + @%p2282 bra $L__BB2_1835; + + mov.u32 %r8201, -1; + shl.b32 %r8202, %r8201, %r10600; + not.b32 %r8203, %r8202; + and.b32 %r10601, %r10502, %r8203; + +$L__BB2_1831: + setp.gt.u32 %p2283, %r10461, 17476; + mov.u32 %r10614, 1; + @%p2283 bra $L__BB2_1835; + + sub.s32 %r8205, %r10462, %r10463; + min.u32 %r8206, %r8205, %r10600; + setp.eq.s32 %p2284, %r8206, 32; + mov.u32 %r8207, -1; + shl.b32 %r8208, %r8207, %r8206; + not.b32 %r8209, %r8208; + selp.b32 %r8210, -1, %r8209, %p2284; + and.b32 %r8211, %r8210, %r10601; + shl.b32 %r8212, %r8211, %r10463; + or.b32 %r10464, %r8212, %r10464; + add.s32 %r10463, %r8206, %r10463; + shr.u32 %r10601, %r10601, %r8206; + sub.s32 %r10600, %r10600, %r8206; + setp.lt.u32 %p2285, %r10463, %r10462; + @%p2285 bra $L__BB2_1834; + + cvt.u64.u32 %rd1370, %r10461; + add.s64 %rd1371, %rd1370, %rd5; + add.s64 %rd1372, %rd1, %rd1371; + st.global.u8 [%rd1372], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p2286, %r10464, 255; + selp.b32 %r10462, 7, 8, %p2286; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1834: + setp.ne.s32 %p2287, %r10600, 0; + mov.u32 %r10614, %r10599; + @%p2287 bra $L__BB2_1831; + +$L__BB2_1835: + and.b32 %r3848, %r10504, 4; + setp.eq.s32 %p2288, %r3848, 0; + mov.u32 %r10629, %r10614; + @%p2288 bra $L__BB2_1842; + + shr.u32 %r8215, %r3800, 2; + and.b32 %r8216, %r8215, 1; + sub.s32 %r10615, %r3686, %r8216; + setp.eq.s32 %p2289, %r10615, 0; + mov.u32 %r10629, %r10614; + @%p2289 bra $L__BB2_1842; + + mov.u32 %r8217, -1; + shl.b32 %r8218, %r8217, %r10615; + not.b32 %r8219, %r8218; + and.b32 %r10616, %r10518, %r8219; + +$L__BB2_1838: + setp.gt.u32 %p2290, %r10461, 17476; + mov.u32 %r10629, 1; + @%p2290 bra $L__BB2_1842; + + sub.s32 %r8221, %r10462, %r10463; + min.u32 %r8222, %r8221, %r10615; + setp.eq.s32 %p2291, %r8222, 32; + mov.u32 %r8223, -1; + shl.b32 %r8224, %r8223, %r8222; + not.b32 %r8225, %r8224; + selp.b32 %r8226, -1, %r8225, %p2291; + and.b32 %r8227, %r8226, %r10616; + shl.b32 %r8228, %r8227, %r10463; + or.b32 %r10464, %r8228, %r10464; + add.s32 %r10463, %r8222, %r10463; + shr.u32 %r10616, %r10616, %r8222; + sub.s32 %r10615, %r10615, %r8222; + setp.lt.u32 %p2292, %r10463, %r10462; + @%p2292 bra $L__BB2_1841; + + cvt.u64.u32 %rd1373, %r10461; + add.s64 %rd1374, %rd1373, %rd5; + add.s64 %rd1375, %rd1, %rd1374; + st.global.u8 [%rd1375], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p2293, %r10464, 255; + selp.b32 %r10462, 7, 8, %p2293; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1841: + setp.ne.s32 %p2294, %r10615, 0; + mov.u32 %r10629, %r10614; + @%p2294 bra $L__BB2_1838; + +$L__BB2_1842: + and.b32 %r3872, %r10504, 8; + setp.eq.s32 %p2295, %r3872, 0; + mov.u32 %r10495, %r10629; + @%p2295 bra $L__BB2_1849; + + shr.u32 %r8231, %r3800, 3; + sub.s32 %r10630, %r3686, %r8231; + setp.eq.s32 %p2296, %r10630, 0; + mov.u32 %r10495, %r10629; + @%p2296 bra $L__BB2_1849; + + mov.u32 %r8232, -1; + shl.b32 %r8233, %r8232, %r10630; + not.b32 %r8234, %r8233; + and.b32 %r10631, %r10517, %r8234; + +$L__BB2_1845: + setp.gt.u32 %p2297, %r10461, 17476; + mov.u32 %r10495, 1; + @%p2297 bra $L__BB2_1849; + + sub.s32 %r8236, %r10462, %r10463; + min.u32 %r8237, %r8236, %r10630; + setp.eq.s32 %p2298, %r8237, 32; + mov.u32 %r8238, -1; + shl.b32 %r8239, %r8238, %r8237; + not.b32 %r8240, %r8239; + selp.b32 %r8241, -1, %r8240, %p2298; + and.b32 %r8242, %r8241, %r10631; + shl.b32 %r8243, %r8242, %r10463; + or.b32 %r10464, %r8243, %r10464; + add.s32 %r10463, %r8237, %r10463; + shr.u32 %r10631, %r10631, %r8237; + sub.s32 %r10630, %r10630, %r8237; + setp.lt.u32 %p2299, %r10463, %r10462; + @%p2299 bra $L__BB2_1848; + + cvt.u64.u32 %rd1376, %r10461; + add.s64 %rd1377, %rd1376, %rd5; + add.s64 %rd1378, %rd1, %rd1377; + st.global.u8 [%rd1378], %r10464; + add.s32 %r10461, %r10461, 1; + setp.eq.s32 %p2300, %r10464, 255; + selp.b32 %r10462, 7, 8, %p2300; + mov.u32 %r10463, 0; + mov.u32 %r10464, %r10463; + +$L__BB2_1848: + setp.ne.s32 %p2301, %r10630, 0; + mov.u32 %r10495, %r10629; + @%p2301 bra $L__BB2_1845; + +$L__BB2_1849: + and.b32 %r8246, %r10501, 255; + and.b32 %r8247, %r10366, 255; + setp.lt.u32 %p2302, %r8246, %r8247; + cvt.u16.u32 %rs1057, %r10501; + selp.b16 %rs1058, %rs450, %rs1057, %p2302; + st.shared.u8 [%r3624+1], %rs1058; + ld.shared.u8 %rs1059, [%r3624+3]; + setp.gt.u16 %p2303, %rs448, %rs1059; + add.s32 %r10647, %r10647, 1; + add.s32 %r8248, %r10344, 3; + selp.b32 %r8249, %r10647, %r8248, %p2303; + add.s32 %r8251, %r4103, %r8249; + ld.shared.u8 %r8252, [%r8251]; + add.s32 %r10345, %r8252, -1; + shr.u32 %r8253, %r3824, 1; + or.b32 %r8254, %r3630, %r8253; + st.shared.u8 [%r3624+2], %r10515; + st.shared.u8 [%r3627+1], %r8254; + ld.shared.u8 %rs1060, [%r3627+3]; + mul.wide.u16 %r8255, %rs1060, 4; + add.s32 %r8256, %r8255, %r3629; + shr.u32 %r8257, %r3872, 3; + st.shared.u8 [%r3627+2], %r8257; + shr.u32 %r8258, %r3872, 2; + shr.u32 %r8259, %r3848, 1; + or.b32 %r8260, %r8258, %r8259; + or.b32 %r10342, %r8260, %r8256; + add.s32 %r10346, %r10346, 1; + mov.u32 %r10385, %r10534; + +$L__BB2_1850: + mov.u32 %r10343, %r10648; + mov.u32 %r10344, %r10647; + max.s32 %r8261, %r10665, 0; + mul.lo.s32 %r8262, %r3415, 6; + setp.gt.s32 %p2304, %r3415, 0; + selp.b32 %r8263, %r8262, 0, %p2304; + cvt.u64.u32 %rd1379, %r8263; + add.s64 %rd75, %rd71, %rd1379; + ld.global.u8 %rs476, [%rd75+1]; + add.s32 %r8264, %r8263, 2; + cvt.u64.u32 %rd1380, %r8264; + add.s64 %rd1381, %rd71, %rd1380; + ld.global.u8 %rs477, [%rd1381]; + ld.global.u8 %rs478, [%rd1381+1]; + mul.lo.s32 %r8265, %r8261, 6; + cvt.u64.u32 %rd1382, %r8265; + add.s64 %rd1383, %rd71, %rd1382; + ld.global.u8 %rs479, [%rd1383]; + ld.global.u8 %rs480, [%rd1383+1]; + add.s32 %r8266, %r8265, 2; + cvt.u64.u32 %rd1384, %r8266; + add.s64 %rd1385, %rd71, %rd1384; + ld.global.u8 %rs481, [%rd1385]; + ld.global.u8 %rs482, [%rd1385+1]; + setp.eq.s16 %p2305, %rs476, 0; + mov.u32 %r10677, %r10385; + @%p2305 bra $L__BB2_1857; + + ld.global.u8 %r10667, [%rd75]; + cvt.u32.u16 %r10666, %rs476; + +$L__BB2_1852: + mov.u32 %r3923, %r10666; + setp.gt.u32 %p2306, %r10274, 2879; + mov.u32 %r10677, 1; + @%p2306 bra $L__BB2_1857; + + mov.u32 %r8268, 8; + sub.s32 %r8269, %r8268, %r10276; + sub.s32 %r8270, %r8269, %r10275; + min.u32 %r8271, %r8270, %r3923; + setp.eq.s32 %p2307, %r8271, 32; + mov.u32 %r8272, -1; + shl.b32 %r8273, %r8272, %r8271; + not.b32 %r8274, %r8273; + selp.b32 %r8275, -1, %r8274, %p2307; + and.b32 %r8276, %r8275, %r10667; + shl.b32 %r8277, %r8276, %r10275; + cvt.u16.u32 %rs1061, %r8277; + or.b16 %rs1322, %rs1322, %rs1061; + add.s32 %r10275, %r8271, %r10275; + sub.s32 %r10666, %r3923, %r8271; + shr.u32 %r10667, %r10667, %r8271; + setp.gt.u32 %p2308, %r8270, %r3923; + @%p2308 bra $L__BB2_1856; + + setp.ne.s32 %p2309, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs1062, %rs1322, 255; + setp.ne.s16 %p2310, %rs1062, 127; + and.pred %p2311, %p2309, %p2310; + @%p2311 bra $L__BB2_1856; + + mov.u32 %r8280, 20548; + sub.s32 %r8281, %r8280, %r10274; + cvt.u64.u32 %rd1386, %r8281; + add.s64 %rd1387, %rd1386, %rd5; + add.s64 %rd1388, %rd1, %rd1387; + st.global.u8 [%rd1388], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2312, %rs1062, 143; + selp.u32 %r10276, 1, 0, %p2312; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1856: + setp.ne.s32 %p2313, %r10666, 0; + mov.u32 %r10677, %r10385; + @%p2313 bra $L__BB2_1852; + +$L__BB2_1857: + setp.eq.s16 %p2314, %rs480, 0; + mov.u32 %r10689, %r10677; + @%p2314 bra $L__BB2_1864; + + cvt.u32.u16 %r8282, %rs479; + and.b32 %r10679, %r8282, 255; + cvt.u32.u16 %r8283, %rs480; + and.b32 %r10678, %r8283, 255; + +$L__BB2_1859: + mov.u32 %r3942, %r10678; + setp.gt.u32 %p2315, %r10274, 2879; + mov.u32 %r10689, 1; + @%p2315 bra $L__BB2_1864; + + mov.u32 %r8285, 8; + sub.s32 %r8286, %r8285, %r10276; + sub.s32 %r8287, %r8286, %r10275; + min.u32 %r8288, %r8287, %r3942; + setp.eq.s32 %p2316, %r8288, 32; + mov.u32 %r8289, -1; + shl.b32 %r8290, %r8289, %r8288; + not.b32 %r8291, %r8290; + selp.b32 %r8292, -1, %r8291, %p2316; + and.b32 %r8293, %r8292, %r10679; + shl.b32 %r8294, %r8293, %r10275; + cvt.u16.u32 %rs1066, %r8294; + or.b16 %rs1322, %rs1322, %rs1066; + add.s32 %r10275, %r8288, %r10275; + sub.s32 %r10678, %r3942, %r8288; + shr.u32 %r10679, %r10679, %r8288; + setp.gt.u32 %p2317, %r8287, %r3942; + @%p2317 bra $L__BB2_1863; + + setp.ne.s32 %p2318, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs1067, %rs1322, 255; + setp.ne.s16 %p2319, %rs1067, 127; + and.pred %p2320, %p2318, %p2319; + @%p2320 bra $L__BB2_1863; + + mov.u32 %r8297, 20548; + sub.s32 %r8298, %r8297, %r10274; + cvt.u64.u32 %rd1389, %r8298; + add.s64 %rd1390, %rd1389, %rd5; + add.s64 %rd1391, %rd1, %rd1390; + st.global.u8 [%rd1391], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2321, %rs1067, 143; + selp.u32 %r10276, 1, 0, %p2321; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1863: + setp.ne.s32 %p2322, %r10678, 0; + mov.u32 %r10689, %r10677; + @%p2322 bra $L__BB2_1859; + +$L__BB2_1864: + setp.eq.s16 %p2323, %rs478, 0; + mov.u32 %r10701, %r10689; + @%p2323 bra $L__BB2_1871; + + cvt.u32.u16 %r8299, %rs478; + and.b32 %r10690, %r8299, 255; + cvt.u32.u16 %r8300, %rs477; + and.b32 %r10691, %r8300, 255; + +$L__BB2_1866: + mov.u32 %r3961, %r10690; + setp.gt.u32 %p2324, %r10274, 2879; + mov.u32 %r10701, 1; + @%p2324 bra $L__BB2_1871; + + mov.u32 %r8302, 8; + sub.s32 %r8303, %r8302, %r10276; + sub.s32 %r8304, %r8303, %r10275; + min.u32 %r8305, %r8304, %r3961; + setp.eq.s32 %p2325, %r8305, 32; + mov.u32 %r8306, -1; + shl.b32 %r8307, %r8306, %r8305; + not.b32 %r8308, %r8307; + selp.b32 %r8309, -1, %r8308, %p2325; + and.b32 %r8310, %r8309, %r10691; + shl.b32 %r8311, %r8310, %r10275; + cvt.u16.u32 %rs1071, %r8311; + or.b16 %rs1322, %rs1322, %rs1071; + add.s32 %r10275, %r8305, %r10275; + sub.s32 %r10690, %r3961, %r8305; + shr.u32 %r10691, %r10691, %r8305; + setp.gt.u32 %p2326, %r8304, %r3961; + @%p2326 bra $L__BB2_1870; + + setp.ne.s32 %p2327, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs1072, %rs1322, 255; + setp.ne.s16 %p2328, %rs1072, 127; + and.pred %p2329, %p2327, %p2328; + @%p2329 bra $L__BB2_1870; + + mov.u32 %r8314, 20548; + sub.s32 %r8315, %r8314, %r10274; + cvt.u64.u32 %rd1392, %r8315; + add.s64 %rd1393, %rd1392, %rd5; + add.s64 %rd1394, %rd1, %rd1393; + st.global.u8 [%rd1394], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2330, %rs1072, 143; + selp.u32 %r10276, 1, 0, %p2330; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1870: + setp.ne.s32 %p2331, %r10690, 0; + mov.u32 %r10701, %r10689; + @%p2331 bra $L__BB2_1866; + +$L__BB2_1871: + setp.eq.s16 %p2332, %rs482, 0; + mov.u32 %r10277, %r10701; + @%p2332 bra $L__BB2_1878; + + cvt.u32.u16 %r8316, %rs481; + and.b32 %r10703, %r8316, 255; + cvt.u32.u16 %r8317, %rs482; + and.b32 %r10702, %r8317, 255; + +$L__BB2_1873: + mov.u32 %r3980, %r10702; + setp.gt.u32 %p2333, %r10274, 2879; + mov.u32 %r10277, 1; + @%p2333 bra $L__BB2_1878; + + mov.u32 %r8319, 8; + sub.s32 %r8320, %r8319, %r10276; + sub.s32 %r8321, %r8320, %r10275; + min.u32 %r8322, %r8321, %r3980; + setp.eq.s32 %p2334, %r8322, 32; + mov.u32 %r8323, -1; + shl.b32 %r8324, %r8323, %r8322; + not.b32 %r8325, %r8324; + selp.b32 %r8326, -1, %r8325, %p2334; + and.b32 %r8327, %r8326, %r10703; + shl.b32 %r8328, %r8327, %r10275; + cvt.u16.u32 %rs1076, %r8328; + or.b16 %rs1322, %rs1322, %rs1076; + add.s32 %r10275, %r8322, %r10275; + sub.s32 %r10702, %r3980, %r8322; + shr.u32 %r10703, %r10703, %r8322; + setp.gt.u32 %p2335, %r8321, %r3980; + @%p2335 bra $L__BB2_1877; + + setp.ne.s32 %p2336, %r10276, 0; + mov.u32 %r10276, 0; + and.b16 %rs1077, %rs1322, 255; + setp.ne.s16 %p2337, %rs1077, 127; + and.pred %p2338, %p2336, %p2337; + @%p2338 bra $L__BB2_1877; + + mov.u32 %r8331, 20548; + sub.s32 %r8332, %r8331, %r10274; + cvt.u64.u32 %rd1395, %r8332; + add.s64 %rd1396, %rd1395, %rd5; + add.s64 %rd1397, %rd1, %rd1396; + st.global.u8 [%rd1397], %rs1322; + add.s32 %r10274, %r10274, 1; + setp.gt.u16 %p2339, %rs1077, 143; + selp.u32 %r10276, 1, 0, %p2339; + mov.u16 %rs1322, 0; + mov.u32 %r10275, 0; + +$L__BB2_1877: + setp.ne.s32 %p2340, %r10702, 0; + mov.u32 %r10277, %r10701; + @%p2340 bra $L__BB2_1873; + +$L__BB2_1878: + add.s32 %r10326, %r10326, 4; + setp.lt.u32 %p2341, %r10326, %r4057; + @%p2341 bra $L__BB2_1638; + +$L__BB2_1879: + add.s32 %r10310, %r10310, 2; + setp.lt.u32 %p2342, %r10310, %r4058; + @%p2342 bra $L__BB2_1636; + +$L__BB2_1880: + setp.eq.s32 %p2343, %r10040, 0; + mov.u32 %r10743, %r10043; + @%p2343 bra $L__BB2_1884; + + shl.b16 %rs1080, %rs1253, 1; + or.b16 %rs1253, %rs1080, 1; + add.s32 %r9835, %r9835, -1; + setp.ne.s32 %p2344, %r9835, 0; + mov.u32 %r10743, %r10043; + @%p2344 bra $L__BB2_1884; + + setp.gt.u32 %p2345, %r9826, 191; + mov.u32 %r10743, 1; + mov.u32 %r9835, 0; + @%p2345 bra $L__BB2_1884; + + add.s32 %r8335, %r9826, 17477; + cvt.u64.u32 %rd1398, %r8335; + add.s64 %rd1399, %rd1398, %rd5; + add.s64 %rd1400, %rd1, %rd1399; + and.b16 %rs1082, %rs1253, 255; + st.global.u8 [%rd1400], %rs1253; + add.s32 %r9826, %r9826, 1; + setp.eq.s16 %p2346, %rs1082, 255; + selp.b32 %r9835, 7, 8, %p2346; + mov.u16 %rs1253, 0; + mov.u32 %r10743, %r10043; + +$L__BB2_1884: + cvt.u32.u16 %r8336, %rs1253; + and.b32 %r8337, %r8336, 255; + shl.b32 %r8338, %r8337, %r9835; + cvt.u16.u32 %rs505, %r8338; + mov.u32 %r8339, -1; + shl.b32 %r8340, %r8339, %r10275; + not.b32 %r8341, %r8340; + mov.u32 %r8342, 255; + and.b32 %r8343, %r8341, 255; + setp.eq.s32 %p2347, %r10275, 0; + selp.b32 %r4032, 0, %r8343, %p2347; + shl.b32 %r4033, %r8342, %r9835; + and.b32 %r8344, %r4033, 255; + or.b32 %r8345, %r8344, %r4032; + setp.eq.s32 %p2348, %r8345, 0; + mov.u32 %r10745, %r10277; + mov.u32 %r10747, %r10743; + @%p2348 bra $L__BB2_1890; + + or.b16 %rs506, %rs1322, %rs505; + and.b16 %rs1083, %rs506, 255; + xor.b16 %rs1084, %rs506, %rs505; + cvt.u32.u16 %r8346, %rs1084; + and.b32 %r8347, %r4033, %r8346; + and.b32 %r8348, %r8347, 255; + xor.b16 %rs1085, %rs506, %rs1322; + cvt.u32.u16 %r8349, %rs1085; + and.b32 %r8350, %r4032, %r8349; + or.b32 %r8351, %r8348, %r8350; + setp.eq.s32 %p2349, %r8351, 0; + setp.ne.s16 %p2350, %rs1083, 255; + and.pred %p2351, %p2350, %p2349; + setp.gt.u32 %p2352, %r10274, 1; + and.pred %p2353, %p2352, %p2351; + add.s32 %r8352, %r9826, 17477; + cvt.u64.u32 %rd1401, %r8352; + add.s64 %rd1402, %rd1401, %rd5; + add.s64 %rd76, %rd1, %rd1402; + @%p2353 bra $L__BB2_1888; + bra.uni $L__BB2_1886; + +$L__BB2_1888: + setp.gt.u32 %p2357, %r9826, 191; + mov.u32 %r10747, 1; + mov.u32 %r10745, %r10277; + @%p2357 bra $L__BB2_1890; + + st.global.u8 [%rd76], %rs506; + add.s32 %r9826, %r9826, 1; + mov.u32 %r10745, %r10277; + mov.u32 %r10747, %r10743; + bra.uni $L__BB2_1890; + +$L__BB2_1886: + setp.gt.u32 %p2354, %r9826, 191; + setp.gt.u32 %p2355, %r10274, 2879; + or.pred %p2356, %p2355, %p2354; + mov.u32 %r10745, 1; + mov.u32 %r10747, %r10745; + @%p2356 bra $L__BB2_1890; + + st.global.u8 [%rd76], %rs505; + add.s32 %r9826, %r9826, 1; + mov.u32 %r8355, 20548; + sub.s32 %r8356, %r8355, %r10274; + cvt.u64.u32 %rd1403, %r8356; + add.s64 %rd1404, %rd1403, %rd5; + add.s64 %rd1405, %rd1, %rd1404; + st.global.u8 [%rd1405], %rs1322; + add.s32 %r10274, %r10274, 1; + mov.u32 %r10745, %r10277; + mov.u32 %r10747, %r10743; + +$L__BB2_1890: + setp.eq.s32 %p2358, %r10463, 0; + @%p2358 bra $L__BB2_1894; + + sub.s32 %r8358, %r10462, %r10463; + mov.u32 %r8359, -1; + shl.b32 %r8360, %r8359, %r8358; + not.b32 %r8361, %r8360; + and.b32 %r8362, %r8361, 255; + shl.b32 %r8363, %r8362, %r10463; + or.b32 %r4041, %r8363, %r10464; + setp.eq.s32 %p2359, %r4041, 255; + mov.u32 %r10749, %r10495; + @%p2359 bra $L__BB2_1896; + + setp.gt.u32 %p2360, %r10461, 17476; + mov.u32 %r10749, 1; + @%p2360 bra $L__BB2_1896; + + cvt.u64.u32 %rd1406, %r10461; + add.s64 %rd1407, %rd1406, %rd5; + add.s64 %rd1408, %rd1, %rd1407; + st.global.u8 [%rd1408], %r4041; + add.s32 %r10461, %r10461, 1; + mov.u32 %r10749, %r10495; + bra.uni $L__BB2_1896; + +$L__BB2_1894: + setp.ne.s32 %p2361, %r10462, 7; + mov.u32 %r10749, %r10495; + @%p2361 bra $L__BB2_1896; + + setp.eq.s32 %p2362, %r10461, 0; + add.s32 %r8365, %r10461, -1; + selp.b32 %r10461, 0, %r8365, %p2362; + mov.u32 %r10749, %r10495; + +$L__BB2_1896: + or.b32 %r8366, %r10747, %r10745; + or.b32 %r8367, %r8366, %r10749; + setp.eq.s32 %p2363, %r8367, 0; + @%p2363 bra $L__BB2_1898; + + mov.u32 %r8372, 1; + st.global.u32 [%rd6], %r8372; + mov.u32 %r8373, 3; + st.global.u32 [%rd6+4], %r8373; + mov.u32 %r10750, 0; + mov.u32 %r10751, %r10750; + mov.u32 %r10752, %r10750; + mov.u32 %r10753, %r10750; + bra.uni $L__BB2_1904; + +$L__BB2_1898: + add.s32 %r8374, %r9826, %r10274; + add.s32 %r10751, %r8374, %r10461; + setp.lt.u32 %p2364, %r10751, 2; + setp.gt.u32 %p2365, %r10751, %r4061; + or.pred %p2366, %p2364, %p2365; + @%p2366 bra $L__BB2_1900; + bra.uni $L__BB2_1899; + +$L__BB2_1900: + mov.u32 %r8384, 1; + st.global.u32 [%rd6], %r8384; + mov.u32 %r8385, 4; + st.global.u32 [%rd6+4], %r8385; + mov.u32 %r10750, 0; + mov.u32 %r10751, %r10750; + mov.u32 %r10752, %r10750; + mov.u32 %r10753, %r10750; + bra.uni $L__BB2_1904; + +$L__BB2_1250: + mov.u32 %r6791, 0; + st.global.u32 [%rd6], %r6791; + st.global.u32 [%rd6+4], %r6791; + st.global.u32 [%rd6+8], %r6791; + st.global.u32 [%rd6+12], %r6791; + st.global.u32 [%rd6+16], %r4059; + st.global.u32 [%rd6+20], %r6791; + st.global.u32 [%rd6+24], %r6791; + st.global.u32 [%rd6+28], %r6791; + bra.uni $L__BB2_1905; + +$L__BB2_23: + setp.lt.u32 %p37, %r4059, %r4062; + @%p37 bra $L__BB2_1248; + bra.uni $L__BB2_24; + +$L__BB2_1248: + mov.u32 %r6785, 2; + st.global.u32 [%rd6], %r6785; + mov.u32 %r6786, 5; + st.global.u32 [%rd6+4], %r6786; + mov.u32 %r6787, 0; + st.global.u32 [%rd6+8], %r6787; + st.global.u32 [%rd6+12], %r6787; + st.global.u32 [%rd6+16], %r6787; + st.global.u32 [%rd6+20], %r6787; + st.global.u32 [%rd6+24], %r6787; + st.global.u32 [%rd6+28], %r6787; + bra.uni $L__BB2_1905; + +$L__BB2_1899: + and.b32 %r8376, %r9826, 32767; + and.b32 %r8377, %r10274, 32767; + bfi.b32 %r8378, %r8377, %r8376, 15, 15; + or.b32 %r10750, %r8378, -2147483648; + mov.u32 %r8379, 0; + st.global.u32 [%rd6], %r8379; + st.global.u32 [%rd6+4], %r8379; + mov.u32 %r10753, 1; + bra.uni $L__BB2_1904; + +$L__BB2_24: + mov.u32 %r8433, 0; + setp.eq.s32 %p38, %r4062, 2; + @%p38 bra $L__BB2_35; + + setp.ne.s32 %p39, %r4062, 3; + @%p39 bra $L__BB2_43; + + @%p10 bra $L__BB2_43; + + mov.u32 %r4115, 0; + mov.u32 %r8426, %r4115; + mov.u32 %r8433, %r4115; + +$L__BB2_28: + mul.lo.s32 %r30, %r8426, %r4055; + mov.u32 %r8428, %r4115; + +$L__BB2_29: + add.s32 %r4117, %r8428, %r30; + cvt.u64.u32 %rd99, %r4117; + add.s64 %rd100, %rd99, %rd4; + shl.b64 %rd101, %rd100, 2; + add.s64 %rd102, %rd3, %rd101; + ld.global.u32 %r4118, [%rd102]; + abs.s32 %r33, %r4118; + setp.eq.s32 %p41, %r33, 0; + @%p41 bra $L__BB2_32; + + setp.eq.s32 %p42, %r33, 3; + @%p42 bra $L__BB2_32; + + add.s32 %r8433, %r8433, 1; + and.b32 %r4119, %r33, 1; + setp.eq.b32 %p43, %r4119, 1; + not.pred %p44, %p43; + setp.lt.u32 %p45, %r33, 5; + or.pred %p46, %p45, %p44; + @%p46 bra $L__BB2_34; + +$L__BB2_32: + add.s32 %r8428, %r8428, 1; + setp.lt.u32 %p47, %r8428, %r4057; + @%p47 bra $L__BB2_29; + + add.s32 %r8426, %r8426, 1; + setp.lt.u32 %p48, %r8426, %r4058; + @%p48 bra $L__BB2_28; + bra.uni $L__BB2_43; + +$L__BB2_35: + @%p10 bra $L__BB2_43; + + mov.u32 %r4124, 0; + mov.u32 %r8431, %r4124; + +$L__BB2_37: + mul.lo.s32 %r39, %r8431, %r4055; + mov.u32 %r8432, %r4124; + +$L__BB2_38: + add.s32 %r4126, %r8432, %r39; + cvt.u64.u32 %rd103, %r4126; + add.s64 %rd104, %rd103, %rd4; + shl.b64 %rd105, %rd104, 2; + add.s64 %rd106, %rd3, %rd105; + ld.global.u32 %r4127, [%rd106]; + abs.s32 %r41, %r4127; + setp.eq.s32 %p50, %r41, 0; + @%p50 bra $L__BB2_41; + + setp.gt.u32 %p51, %r41, 2; + and.b32 %r4128, %r41, 1; + setp.eq.b32 %p52, %r4128, 1; + and.pred %p53, %p51, %p52; + @%p53 bra $L__BB2_41; + bra.uni $L__BB2_40; + +$L__BB2_41: + add.s32 %r8432, %r8432, 1; + setp.lt.u32 %p54, %r8432, %r4057; + @%p54 bra $L__BB2_38; + + add.s32 %r8431, %r8431, 1; + setp.lt.u32 %p55, %r8431, %r4058; + mov.u32 %r8433, 0; + @%p55 bra $L__BB2_37; + +$L__BB2_43: + add.s32 %r8412, %r4057, 1; + shr.u32 %r8411, %r8412, 1; + mov.u32 %r8436, 0; + sub.s32 %r45, %r4059, %r4062; + mov.u32 %r4134, 30; + sub.s32 %r46, %r4134, %r45; + mov.u16 %rs510, 255; + st.global.u8 [%rd7], %rs510; + add.s32 %r4136, %r8411, 2; + min.u32 %r48, %r4136, 513; + mov.u32 %r4137, -3; + sub.s32 %r4138, %r4137, %r8411; + max.u32 %r4139, %r4138, -514; + mov.u32 %r4140, -2; + sub.s32 %r4141, %r4140, %r4139; + and.b32 %r8438, %r48, 3; + setp.lt.u32 %p56, %r4141, 3; + @%p56 bra $L__BB2_46; + + sub.s32 %r8435, %r48, %r8438; + mov.u32 %r8436, 0; + +$L__BB2_45: + add.s32 %r4144, %r4103, %r8436; + mov.u16 %rs511, 0; + st.shared.u8 [%r4144], %rs511; + mov.u32 %r4145, _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val; + add.s32 %r4146, %r4145, %r8436; + st.shared.u8 [%r4146], %rs511; + st.shared.u8 [%r4144+1], %rs511; + st.shared.u8 [%r4146+1], %rs511; + st.shared.u8 [%r4144+2], %rs511; + st.shared.u8 [%r4146+2], %rs511; + st.shared.u8 [%r4144+3], %rs511; + st.shared.u8 [%r4146+3], %rs511; + add.s32 %r8436, %r8436, 4; + add.s32 %r8435, %r8435, -4; + setp.ne.s32 %p57, %r8435, 0; + @%p57 bra $L__BB2_45; + +$L__BB2_46: + setp.eq.s32 %p58, %r8438, 0; + @%p58 bra $L__BB2_49; + + mov.u32 %r4149, _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val; + +$L__BB2_48: + .pragma "nounroll"; + add.s32 %r4148, %r4103, %r8436; + mov.u16 %rs512, 0; + st.shared.u8 [%r4148], %rs512; + add.s32 %r4150, %r4149, %r8436; + st.shared.u8 [%r4150], %rs512; + add.s32 %r8436, %r8436, 1; + add.s32 %r8438, %r8438, -1; + setp.ne.s32 %p59, %r8438, 0; + @%p59 bra $L__BB2_48; + +$L__BB2_49: + mov.u32 %r8733, 0; + mov.u32 %r8530, 8; + mov.u32 %r8734, 1; + mov.u32 %r8971, 4; + mov.u16 %rs1165, 15; + mov.u16 %rs1096, 0; + mov.u32 %r8735, %r8733; + mov.u32 %r8736, %r8733; + mov.u32 %r8524, %r8733; + mov.u32 %r8969, %r8733; + mov.u32 %r8970, %r8734; + mov.u32 %r8972, %r8734; + mov.u32 %r9186, %r8733; + mov.u32 %r9157, %r8733; + mov.u32 %r9158, %r8733; + mov.u32 %r9159, %r8530; + mov.u32 %r9160, %r8733; + @%p10 bra $L__BB2_417; + + ld.param.u64 %rd1415, [ j2k_htj2k_encode_codeblocks_multi_input_param_4]; + ld.param.u64 %rd1409, [ j2k_htj2k_encode_codeblocks_multi_input_param_2]; + mov.u32 %r4183, 0; + mov.u32 %r4184, 31; + sub.s32 %r60, %r4184, %r4059; + cvta.to.global.u64 %rd8, %rd1409; + cvta.to.global.u64 %rd9, %rd1415; + mov.u32 %r8972, 1; + mov.u16 %rs1096, 0; + mov.u32 %r9159, 8; + mov.u16 %rs1165, 15; + mov.u32 %r8971, 4; + mov.u32 %r8439, %r4183; + mov.u32 %r8440, %r4183; + mov.u32 %r8441, %r4183; + mov.u32 %r9160, %r4183; + mov.u32 %r9158, %r4183; + mov.u32 %r9157, %r4183; + mov.u32 %r9186, %r4183; + mov.u32 %r8970, %r8972; + mov.u32 %r8969, %r4183; + mov.u32 %r8524, %r4183; + mov.u32 %r8530, %r9159; + mov.u32 %r8736, %r4183; + mov.u32 %r8735, %r4183; + mov.u32 %r8734, %r8972; + mov.u32 %r8733, %r4183; + bra.uni $L__BB2_51; + +$L__BB2_40: + mov.u32 %r4129, 2; + st.global.u32 [%rd6], %r4129; + mov.u32 %r4130, 6; + st.global.u32 [%rd6+4], %r4130; + mov.u32 %r4131, 0; + st.global.u32 [%rd6+8], %r4131; + st.global.u32 [%rd6+12], %r4131; + st.global.u32 [%rd6+16], %r4131; + st.global.u32 [%rd6+20], %r4131; + st.global.u32 [%rd6+24], %r4131; + st.global.u32 [%rd6+28], %r4131; + bra.uni $L__BB2_1905; + +$L__BB2_34: + mov.u32 %r4120, 2; + st.global.u32 [%rd6], %r4120; + mov.u32 %r4121, 6; + st.global.u32 [%rd6+4], %r4121; + mov.u32 %r4122, 0; + st.global.u32 [%rd6+8], %r4122; + st.global.u32 [%rd6+12], %r4122; + st.global.u32 [%rd6+16], %r4122; + st.global.u32 [%rd6+20], %r4122; + st.global.u32 [%rd6+24], %r4122; + st.global.u32 [%rd6+28], %r4122; + bra.uni $L__BB2_1905; + +$L__BB2_256: + setp.gt.u32 %p287, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8729, 1; + @%p287 bra $L__BB2_258; + + and.b16 %rs599, %rs1096, 255; + st.global.u8 [%rd13], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p288, %rs599, 255; + selp.b32 %r8530, 7, 8, %p288; + mov.u16 %rs1096, 0; + mov.u32 %r8729, %r8733; + bra.uni $L__BB2_258; + +$L__BB2_51: + cvt.u64.u32 %rd107, %r8440; + add.s64 %rd108, %rd107, %rd4; + shl.b64 %rd109, %rd108, 2; + add.s64 %rd110, %rd3, %rd109; + ld.global.u32 %r79, [%rd110]; + setp.eq.s32 %p61, %r79, 0; + mov.u32 %r8457, %r4183; + @%p61 bra $L__BB2_53; + + and.b32 %r4186, %r79, -2147483648; + abs.s32 %r4187, %r79; + shl.b32 %r4188, %r4187, %r60; + or.b32 %r8457, %r4188, %r4186; + +$L__BB2_53: + shl.b32 %r4192, %r8457, 1; + shr.u32 %r4193, %r4192, %r46; + and.b32 %r82, %r4193, -2; + setp.eq.s32 %p62, %r82, 0; + mov.u32 %r8461, 0; + mov.u32 %r8458, %r8461; + mov.u32 %r8459, %r8461; + mov.u32 %r8465, %r8461; + @%p62 bra $L__BB2_55; + + add.s32 %r4195, %r82, -1; + clz.b32 %r4196, %r4195; + mov.u32 %r4197, 32; + sub.s32 %r8458, %r4197, %r4196; + shr.u32 %r4198, %r8457, 31; + add.s32 %r4199, %r4198, %r82; + add.s32 %r8459, %r4199, -2; + mov.u32 %r8465, 1; + +$L__BB2_55: + setp.lt.u32 %p63, %r4058, 2; + @%p63 bra $L__BB2_58; + + add.s32 %r4202, %r8440, %r4055; + cvt.u64.u32 %rd111, %r4202; + add.s64 %rd112, %rd111, %rd4; + shl.b64 %rd113, %rd112, 2; + add.s64 %rd114, %rd3, %rd113; + ld.global.u32 %r88, [%rd114]; + setp.eq.s32 %p64, %r88, 0; + @%p64 bra $L__BB2_58; + + and.b32 %r4203, %r88, -2147483648; + abs.s32 %r4204, %r88; + shl.b32 %r4205, %r4204, %r60; + or.b32 %r8461, %r4205, %r4203; + +$L__BB2_58: + shl.b32 %r4208, %r8461, 1; + shr.u32 %r4209, %r4208, %r46; + and.b32 %r91, %r4209, -2; + setp.eq.s32 %p65, %r91, 0; + mov.u32 %r8476, 0; + mov.u32 %r8462, %r8476; + mov.u32 %r8463, %r8476; + mov.u32 %r8480, %r8458; + @%p65 bra $L__BB2_60; + + or.b32 %r8465, %r8465, 2; + add.s32 %r4210, %r91, -1; + clz.b32 %r4211, %r4210; + mov.u32 %r4212, 32; + sub.s32 %r8462, %r4212, %r4211; + max.s32 %r8480, %r8458, %r8462; + shr.u32 %r4213, %r8461, 31; + add.s32 %r4214, %r4213, %r91; + add.s32 %r8463, %r4214, -2; + +$L__BB2_60: + add.s32 %r8482, %r8440, 1; + add.s32 %r4219, %r8439, 1; + setp.ge.u32 %p66, %r4219, %r4057; + mov.u32 %r8477, %r8476; + mov.u32 %r8478, %r8476; + mov.u32 %r8479, %r8476; + @%p66 bra $L__BB2_71; + + cvt.u64.u32 %rd115, %r8482; + add.s64 %rd116, %rd115, %rd4; + shl.b64 %rd117, %rd116, 2; + add.s64 %rd118, %rd3, %rd117; + ld.global.u32 %r101, [%rd118]; + setp.eq.s32 %p67, %r101, 0; + mov.u32 %r8477, 0; + mov.u32 %r8466, %r8477; + @%p67 bra $L__BB2_63; + + and.b32 %r4221, %r101, -2147483648; + abs.s32 %r4222, %r101; + shl.b32 %r4223, %r4222, %r60; + or.b32 %r8466, %r4223, %r4221; + +$L__BB2_63: + shl.b32 %r4226, %r8466, 1; + shr.u32 %r4227, %r4226, %r46; + and.b32 %r104, %r4227, -2; + setp.eq.s32 %p68, %r104, 0; + mov.u32 %r8479, %r8477; + @%p68 bra $L__BB2_65; + + or.b32 %r8465, %r8465, 4; + add.s32 %r4228, %r104, -1; + clz.b32 %r4229, %r4228; + mov.u32 %r4230, 32; + sub.s32 %r8477, %r4230, %r4229; + max.s32 %r8480, %r8480, %r8477; + shr.u32 %r4231, %r8466, 31; + add.s32 %r4232, %r4231, %r104; + add.s32 %r8479, %r4232, -2; + +$L__BB2_65: + mov.u32 %r8476, 0; + mov.u32 %r8471, %r8476; + @%p63 bra $L__BB2_68; + + add.s32 %r4235, %r8482, %r4055; + cvt.u64.u32 %rd119, %r4235; + add.s64 %rd120, %rd119, %rd4; + shl.b64 %rd121, %rd120, 2; + add.s64 %rd122, %rd3, %rd121; + ld.global.u32 %r113, [%rd122]; + setp.eq.s32 %p70, %r113, 0; + @%p70 bra $L__BB2_68; + + and.b32 %r4236, %r113, -2147483648; + abs.s32 %r4237, %r113; + shl.b32 %r4238, %r4237, %r60; + or.b32 %r8471, %r4238, %r4236; + +$L__BB2_68: + shl.b32 %r4241, %r8471, 1; + shr.u32 %r4242, %r4241, %r46; + and.b32 %r116, %r4242, -2; + setp.eq.s32 %p71, %r116, 0; + mov.u32 %r8478, %r8476; + @%p71 bra $L__BB2_70; + + or.b32 %r8465, %r8465, 8; + add.s32 %r4243, %r116, -1; + clz.b32 %r4244, %r4243; + mov.u32 %r4245, 32; + sub.s32 %r8476, %r4245, %r4244; + max.s32 %r8480, %r8480, %r8476; + shr.u32 %r4246, %r8471, 31; + add.s32 %r4247, %r4246, %r116; + add.s32 %r8478, %r4247, -2; + +$L__BB2_70: + add.s32 %r8482, %r8440, 2; + +$L__BB2_71: + mov.u32 %r8440, %r8482; + add.s32 %r4249, %r8480, -1; + setp.lt.s32 %p72, %r8480, 2; + setp.gt.s32 %p73, %r8480, 1; + selp.b32 %r133, %r4249, 0, %p73; + mov.u32 %r8483, 0; + @%p72 bra $L__BB2_73; + + setp.eq.s32 %p74, %r8458, %r8480; + selp.u32 %r4250, 1, 0, %p74; + setp.eq.s32 %p75, %r8462, %r8480; + selp.u32 %r4251, -1, 0, %p75; + bfi.b32 %r4252, %r4251, %r4250, 1, 1; + setp.eq.s32 %p76, %r8477, %r8480; + selp.u16 %rs517, 1, 0, %p76; + mul.wide.u16 %r4253, %rs517, 4; + or.b32 %r4254, %r4252, %r4253; + setp.eq.s32 %p77, %r8476, %r8480; + selp.u16 %rs518, 1, 0, %p77; + mul.wide.u16 %r4255, %rs518, 8; + or.b32 %r8483, %r4254, %r4255; + +$L__BB2_73: + shr.u32 %r4256, %r8439, 1; + add.s32 %r136, %r4103, %r4256; + ld.shared.u8 %rs519, [%r136]; + cvt.u32.u16 %r4258, %rs519; + and.b32 %r4259, %r4258, 255; + and.b32 %r4260, %r8462, 255; + setp.lt.u32 %p78, %r4260, %r4259; + cvt.u16.u32 %rs520, %r8462; + selp.b16 %rs521, %rs519, %rs520, %p78; + st.shared.u8 [%r136], %rs521; + cvt.u16.u32 %rs3, %r8476; + st.shared.u8 [%r136+1], %rs3; + and.b32 %r137, %r8465, 2; + cvt.u16.u32 %rs522, %r137; + shr.u16 %rs523, %rs522, 1; + mov.u32 %r4261, _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val; + add.s32 %r138, %r4261, %r4256; + ld.shared.u8 %rs524, [%r138]; + or.b16 %rs525, %rs524, %rs523; + st.shared.u8 [%r138], %rs525; + and.b32 %r139, %r8465, 8; + shr.u32 %r140, %r139, 3; + st.shared.u8 [%r138+1], %r140; + shl.b32 %r4262, %r8465, 4; + shl.b32 %r4263, %r8441, 8; + or.b32 %r4264, %r4262, %r4263; + or.b32 %r4265, %r4264, %r8483; + mul.wide.u32 %rd123, %r4265, 2; + add.s64 %rd124, %rd8, %rd123; + ld.global.u16 %rs4, [%rd124]; + shr.u16 %rs526, %rs4, 4; + and.b16 %rs5, %rs526, 7; + setp.eq.s16 %p79, %rs5, 0; + mov.u32 %r8495, %r8969; + @%p79 bra $L__BB2_80; + + cvt.u32.u16 %r8484, %rs5; + shr.u16 %rs527, %rs4, 8; + cvt.u32.u16 %r8485, %rs527; + +$L__BB2_75: + mov.u32 %r143, %r8484; + setp.gt.u32 %p80, %r8972, 2879; + mov.u32 %r8495, 1; + @%p80 bra $L__BB2_80; + + mov.u32 %r4267, 8; + sub.s32 %r4268, %r4267, %r8970; + sub.s32 %r4269, %r4268, %r8971; + min.u32 %r4270, %r4269, %r143; + setp.eq.s32 %p81, %r4270, 32; + mov.u32 %r4271, -1; + shl.b32 %r4272, %r4271, %r4270; + not.b32 %r4273, %r4272; + selp.b32 %r4274, -1, %r4273, %p81; + and.b32 %r4275, %r4274, %r8485; + shl.b32 %r4276, %r4275, %r8971; + cvt.u16.u32 %rs528, %r4276; + or.b16 %rs1165, %rs1165, %rs528; + add.s32 %r8971, %r4270, %r8971; + sub.s32 %r8484, %r143, %r4270; + shr.u32 %r8485, %r8485, %r4270; + setp.gt.u32 %p82, %r4269, %r143; + @%p82 bra $L__BB2_79; + + setp.ne.s32 %p83, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs529, %rs1165, 255; + setp.ne.s16 %p84, %rs529, 127; + and.pred %p85, %p83, %p84; + @%p85 bra $L__BB2_79; + + mov.u32 %r4279, 20548; + sub.s32 %r4280, %r4279, %r8972; + cvt.u64.u32 %rd125, %r4280; + add.s64 %rd126, %rd125, %rd5; + add.s64 %rd127, %rd1, %rd126; + st.global.u8 [%rd127], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p86, %rs529, 143; + selp.u32 %r8970, 1, 0, %p86; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_79: + setp.ne.s32 %p87, %r8484, 0; + mov.u32 %r8495, %r8969; + @%p87 bra $L__BB2_75; + +$L__BB2_80: + setp.ne.s32 %p88, %r8441, 0; + @%p88 bra $L__BB2_128; + + setp.eq.s32 %p89, %r8465, 0; + add.s32 %r4281, %r8524, 17477; + cvt.u64.u32 %rd128, %r4281; + add.s64 %rd129, %rd128, %rd5; + add.s64 %rd10, %rd1, %rd129; + @%p89 bra $L__BB2_120; + + shl.b16 %rs1096, %rs1096, 1; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p90, %r8530, 0; + mov.u32 %r8529, %r8733; + @%p90 bra $L__BB2_85; + bra.uni $L__BB2_83; + +$L__BB2_85: + setp.lt.u32 %p92, %r8735, 3; + mov.u32 %r8499, 0; + @%p92 bra $L__BB2_88; + + setp.lt.u32 %p93, %r8735, 6; + mov.u32 %r8499, 1; + @%p93 bra $L__BB2_88; + + setp.lt.u32 %p94, %r8735, 9; + setp.eq.s32 %p95, %r8735, 11; + selp.b32 %r4287, 4, 5, %p95; + setp.lt.u32 %p96, %r8735, 11; + selp.b32 %r4288, 3, %r4287, %p96; + selp.b32 %r8499, 2, %r4288, %p94; + +$L__BB2_88: + setp.eq.s32 %p97, %r8499, 0; + @%p97 bra $L__BB2_116; + + add.s32 %r167, %r8499, -1; + and.b32 %r168, %r8499, 3; + setp.eq.s32 %p98, %r168, 0; + mov.u32 %r8509, %r8499; + mov.u32 %r8512, %r8529; + @%p98 bra $L__BB2_101; + + mov.u32 %r4290, 1; + shl.b32 %r4291, %r4290, %r167; + and.b32 %r4292, %r4291, %r8736; + setp.ne.s32 %p99, %r4292, 0; + selp.u32 %r4293, 1, 0, %p99; + cvt.u32.u16 %r4294, %rs1096; + bfi.b32 %r4295, %r4294, %r4293, 1, 8; + cvt.u16.u32 %rs1096, %r4295; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p100, %r8530, 0; + mov.u32 %r8512, %r8529; + @%p100 bra $L__BB2_93; + + setp.gt.u32 %p101, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8512, %r4290; + @%p101 bra $L__BB2_93; + + add.s32 %r4299, %r8524, 17477; + cvt.u64.u32 %rd130, %r4299; + add.s64 %rd131, %rd130, %rd5; + add.s64 %rd132, %rd1, %rd131; + st.global.u8 [%rd132], %rs1096; + add.s32 %r8524, %r8524, 1; + mov.u32 %r8530, 8; + mov.u16 %rs1096, 0; + mov.u32 %r8512, %r8529; + +$L__BB2_93: + setp.eq.s32 %p102, %r168, 1; + mov.u32 %r8529, %r8512; + mov.u32 %r8509, %r167; + @%p102 bra $L__BB2_101; + + add.s32 %r8509, %r8499, -2; + mov.u32 %r4300, 1; + shl.b32 %r4301, %r4300, %r8509; + and.b32 %r4302, %r4301, %r8736; + setp.ne.s32 %p103, %r4302, 0; + selp.u32 %r4303, 1, 0, %p103; + cvt.u32.u16 %r4304, %rs1096; + bfi.b32 %r4305, %r4304, %r4303, 1, 8; + cvt.u16.u32 %rs1096, %r4305; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p104, %r8530, 0; + mov.u32 %r8503, %r8512; + @%p104 bra $L__BB2_97; + + setp.gt.u32 %p105, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8503, %r4300; + @%p105 bra $L__BB2_97; + + add.s32 %r4308, %r8524, 17477; + cvt.u64.u32 %rd133, %r4308; + add.s64 %rd134, %rd133, %rd5; + add.s64 %rd135, %rd1, %rd134; + and.b16 %rs536, %rs1096, 255; + st.global.u8 [%rd135], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p106, %rs536, 255; + selp.b32 %r8530, 7, 8, %p106; + mov.u16 %rs1096, 0; + mov.u32 %r8503, %r8512; + +$L__BB2_97: + setp.eq.s32 %p107, %r168, 2; + mov.u32 %r8529, %r8503; + mov.u32 %r8512, %r8503; + @%p107 bra $L__BB2_101; + + add.s32 %r8509, %r8499, -3; + mov.u32 %r4309, 1; + shl.b32 %r4310, %r4309, %r8509; + and.b32 %r4311, %r4310, %r8736; + setp.ne.s32 %p108, %r4311, 0; + selp.u32 %r4312, 1, 0, %p108; + cvt.u32.u16 %r4313, %rs1096; + bfi.b32 %r4314, %r4313, %r4312, 1, 8; + cvt.u16.u32 %rs1096, %r4314; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p109, %r8530, 0; + mov.u32 %r8529, %r8503; + mov.u32 %r8512, %r8503; + @%p109 bra $L__BB2_101; + + setp.gt.u32 %p110, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8529, %r4309; + mov.u32 %r8512, %r4309; + @%p110 bra $L__BB2_101; + + add.s32 %r4319, %r8524, 17477; + cvt.u64.u32 %rd136, %r4319; + add.s64 %rd137, %rd136, %rd5; + add.s64 %rd138, %rd1, %rd137; + and.b16 %rs539, %rs1096, 255; + st.global.u8 [%rd138], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p111, %rs539, 255; + selp.b32 %r8530, 7, 8, %p111; + mov.u16 %rs1096, 0; + mov.u32 %r8529, %r8503; + mov.u32 %r8512, %r8503; + +$L__BB2_101: + setp.lt.u32 %p112, %r167, 3; + @%p112 bra $L__BB2_116; + + mov.u32 %r8529, %r8512; + +$L__BB2_103: + add.s32 %r4320, %r8509, -1; + mov.u32 %r4321, 1; + shl.b32 %r4322, %r4321, %r4320; + and.b32 %r4323, %r4322, %r8736; + setp.ne.s32 %p113, %r4323, 0; + selp.u32 %r4324, 1, 0, %p113; + cvt.u32.u16 %r4325, %rs1096; + bfi.b32 %r8518, %r4325, %r4324, 1, 8; + add.s32 %r8519, %r8530, -1; + setp.ne.s32 %p114, %r8519, 0; + mov.u32 %r8517, %r8529; + @%p114 bra $L__BB2_106; + + setp.gt.u32 %p115, %r8524, 191; + mov.u32 %r8519, 0; + mov.u32 %r8517, %r4321; + @%p115 bra $L__BB2_106; + + cvt.u16.u32 %rs540, %r8518; + and.b16 %rs541, %rs540, 255; + add.s32 %r4329, %r8524, 17477; + cvt.u64.u32 %rd139, %r4329; + add.s64 %rd140, %rd139, %rd5; + add.s64 %rd141, %rd1, %rd140; + st.global.u8 [%rd141], %rs540; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p116, %rs541, 255; + selp.b32 %r8519, 7, 8, %p116; + mov.u32 %r8518, 0; + mov.u32 %r8517, %r8529; + +$L__BB2_106: + add.s32 %r4330, %r8509, -2; + shl.b32 %r4332, %r4321, %r4330; + and.b32 %r4333, %r4332, %r8736; + setp.ne.s32 %p117, %r4333, 0; + and.b32 %r4334, %r8518, 127; + selp.u32 %r4335, 1, 0, %p117; + bfi.b32 %r8522, %r4334, %r4335, 1, 7; + add.s32 %r8523, %r8519, -1; + setp.ne.s32 %p118, %r8523, 0; + mov.u32 %r8521, %r8517; + @%p118 bra $L__BB2_109; + + setp.gt.u32 %p119, %r8524, 191; + mov.u32 %r8523, 0; + mov.u32 %r8521, 1; + @%p119 bra $L__BB2_109; + + cvt.u16.u32 %rs542, %r8522; + and.b16 %rs543, %rs542, 255; + add.s32 %r4339, %r8524, 17477; + cvt.u64.u32 %rd142, %r4339; + add.s64 %rd143, %rd142, %rd5; + add.s64 %rd144, %rd1, %rd143; + st.global.u8 [%rd144], %rs542; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p120, %rs543, 255; + selp.b32 %r8523, 7, 8, %p120; + mov.u32 %r8522, 0; + mov.u32 %r8521, %r8517; + +$L__BB2_109: + add.s32 %r4340, %r8509, -3; + mov.u32 %r4341, 1; + shl.b32 %r4342, %r4341, %r4340; + and.b32 %r4343, %r4342, %r8736; + setp.ne.s32 %p121, %r4343, 0; + and.b32 %r4344, %r8522, 127; + selp.u32 %r4345, 1, 0, %p121; + bfi.b32 %r8526, %r4344, %r4345, 1, 7; + add.s32 %r8527, %r8523, -1; + setp.ne.s32 %p122, %r8527, 0; + mov.u32 %r8525, %r8521; + @%p122 bra $L__BB2_112; + + setp.gt.u32 %p123, %r8524, 191; + mov.u32 %r8527, 0; + mov.u32 %r8525, %r4341; + @%p123 bra $L__BB2_112; + + cvt.u16.u32 %rs544, %r8526; + and.b16 %rs545, %rs544, 255; + add.s32 %r4349, %r8524, 17477; + cvt.u64.u32 %rd145, %r4349; + add.s64 %rd146, %rd145, %rd5; + add.s64 %rd147, %rd1, %rd146; + st.global.u8 [%rd147], %rs544; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p124, %rs545, 255; + selp.b32 %r8527, 7, 8, %p124; + mov.u32 %r8526, 0; + mov.u32 %r8525, %r8521; + +$L__BB2_112: + add.s32 %r8509, %r8509, -4; + shl.b32 %r4351, %r4341, %r8509; + and.b32 %r4352, %r4351, %r8736; + setp.ne.s32 %p125, %r4352, 0; + and.b32 %r4353, %r8526, 127; + selp.u32 %r4354, 1, 0, %p125; + bfi.b32 %r4355, %r4353, %r4354, 1, 15; + cvt.u16.u32 %rs1096, %r4355; + add.s32 %r8530, %r8527, -1; + setp.ne.s32 %p126, %r8530, 0; + mov.u32 %r8529, %r8525; + @%p126 bra $L__BB2_115; + + setp.gt.u32 %p127, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8529, 1; + @%p127 bra $L__BB2_115; + + add.s32 %r4358, %r8524, 17477; + cvt.u64.u32 %rd148, %r4358; + add.s64 %rd149, %rd148, %rd5; + add.s64 %rd150, %rd1, %rd149; + and.b16 %rs547, %rs1096, 255; + st.global.u8 [%rd150], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p128, %rs547, 255; + selp.b32 %r8530, 7, 8, %p128; + mov.u16 %rs1096, 0; + mov.u32 %r8529, %r8525; + +$L__BB2_115: + setp.ne.s32 %p129, %r8509, 0; + @%p129 bra $L__BB2_103; + +$L__BB2_116: + add.s32 %r4360, %r8735, -1; + setp.eq.s32 %p130, %r8735, 0; + mov.u32 %r8736, 0; + selp.b32 %r8735, 0, %r4360, %p130; + setp.lt.u32 %p131, %r8735, 3; + mov.u32 %r8535, %r8736; + @%p131 bra $L__BB2_119; + + setp.lt.u32 %p132, %r8735, 6; + mov.u32 %r8535, 1; + @%p132 bra $L__BB2_119; + + setp.lt.u32 %p133, %r8735, 9; + setp.eq.s32 %p134, %r8735, 11; + selp.b32 %r4362, 4, 5, %p134; + setp.lt.u32 %p135, %r8735, 11; + selp.b32 %r4363, 3, %r4362, %p135; + selp.b32 %r8535, 2, %r4363, %p133; + +$L__BB2_119: + mov.u32 %r4365, 1; + shl.b32 %r8734, %r4365, %r8535; + mov.u32 %r8733, %r8529; + bra.uni $L__BB2_128; + +$L__BB2_120: + add.s32 %r8736, %r8736, 1; + setp.lt.u32 %p136, %r8736, %r8734; + @%p136 bra $L__BB2_128; + + shl.b16 %rs548, %rs1096, 1; + or.b16 %rs1096, %rs548, 1; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p137, %r8530, 0; + mov.u32 %r8536, %r8733; + @%p137 bra $L__BB2_124; + + setp.gt.u32 %p138, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8536, 1; + @%p138 bra $L__BB2_124; + + and.b16 %rs550, %rs1096, 255; + st.global.u8 [%rd10], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p139, %rs550, 255; + selp.b32 %r8530, 7, 8, %p139; + mov.u16 %rs1096, 0; + mov.u32 %r8536, %r8733; + +$L__BB2_124: + add.s32 %r4369, %r8735, 1; + min.u32 %r8735, %r4369, 12; + setp.lt.u32 %p140, %r8735, 3; + mov.u32 %r8736, 0; + mov.u32 %r8539, %r8736; + @%p140 bra $L__BB2_127; + + setp.lt.u32 %p141, %r8735, 6; + mov.u32 %r8539, 1; + @%p141 bra $L__BB2_127; + + setp.lt.u32 %p142, %r8735, 9; + setp.eq.s32 %p143, %r8735, 11; + selp.b32 %r4371, 4, 5, %p143; + setp.lt.u32 %p144, %r8735, 11; + selp.b32 %r4372, 3, %r4371, %p144; + selp.b32 %r8539, 2, %r4372, %p142; + +$L__BB2_127: + mov.u32 %r4374, 1; + shl.b32 %r8734, %r4374, %r8539; + mov.u32 %r8733, %r8536; + +$L__BB2_128: + max.s32 %r251, %r8480, 1; + and.b16 %rs551, %rs4, 15; + cvt.u32.u16 %r252, %rs551; + and.b32 %r253, %r8465, 1; + setp.eq.s32 %p145, %r253, 0; + mov.u32 %r8556, %r9186; + @%p145 bra $L__BB2_135; + + and.b32 %r4375, %r252, 1; + sub.s32 %r8546, %r251, %r4375; + setp.eq.s32 %p146, %r8546, 0; + mov.u32 %r8556, %r9186; + @%p146 bra $L__BB2_135; + + mov.u32 %r4376, -1; + shl.b32 %r4377, %r4376, %r8546; + not.b32 %r4378, %r4377; + and.b32 %r8547, %r8459, %r4378; + +$L__BB2_131: + setp.gt.u32 %p147, %r9160, 17476; + mov.u32 %r8556, 1; + @%p147 bra $L__BB2_135; + + sub.s32 %r4380, %r9159, %r9158; + min.u32 %r4381, %r4380, %r8546; + setp.eq.s32 %p148, %r4381, 32; + mov.u32 %r4382, -1; + shl.b32 %r4383, %r4382, %r4381; + not.b32 %r4384, %r4383; + selp.b32 %r4385, -1, %r4384, %p148; + and.b32 %r4386, %r4385, %r8547; + shl.b32 %r4387, %r4386, %r9158; + or.b32 %r9157, %r4387, %r9157; + add.s32 %r9158, %r4381, %r9158; + shr.u32 %r8547, %r8547, %r4381; + sub.s32 %r8546, %r8546, %r4381; + setp.lt.u32 %p149, %r9158, %r9159; + @%p149 bra $L__BB2_134; + + cvt.u64.u32 %rd151, %r9160; + add.s64 %rd152, %rd151, %rd5; + add.s64 %rd153, %rd1, %rd152; + st.global.u8 [%rd153], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p150, %r9157, 255; + selp.b32 %r9159, 7, 8, %p150; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_134: + setp.ne.s32 %p151, %r8546, 0; + mov.u32 %r8556, %r9186; + @%p151 bra $L__BB2_131; + +$L__BB2_135: + setp.eq.s32 %p152, %r137, 0; + mov.u32 %r8571, %r8556; + @%p152 bra $L__BB2_142; + + shr.u32 %r4390, %r252, 1; + and.b32 %r4391, %r4390, 1; + sub.s32 %r8561, %r251, %r4391; + setp.eq.s32 %p153, %r8561, 0; + mov.u32 %r8571, %r8556; + @%p153 bra $L__BB2_142; + + mov.u32 %r4392, -1; + shl.b32 %r4393, %r4392, %r8561; + not.b32 %r4394, %r4393; + and.b32 %r8562, %r8463, %r4394; + +$L__BB2_138: + setp.gt.u32 %p154, %r9160, 17476; + mov.u32 %r8571, 1; + @%p154 bra $L__BB2_142; + + sub.s32 %r4396, %r9159, %r9158; + min.u32 %r4397, %r4396, %r8561; + setp.eq.s32 %p155, %r4397, 32; + mov.u32 %r4398, -1; + shl.b32 %r4399, %r4398, %r4397; + not.b32 %r4400, %r4399; + selp.b32 %r4401, -1, %r4400, %p155; + and.b32 %r4402, %r4401, %r8562; + shl.b32 %r4403, %r4402, %r9158; + or.b32 %r9157, %r4403, %r9157; + add.s32 %r9158, %r4397, %r9158; + shr.u32 %r8562, %r8562, %r4397; + sub.s32 %r8561, %r8561, %r4397; + setp.lt.u32 %p156, %r9158, %r9159; + @%p156 bra $L__BB2_141; + + cvt.u64.u32 %rd154, %r9160; + add.s64 %rd155, %rd154, %rd5; + add.s64 %rd156, %rd1, %rd155; + st.global.u8 [%rd156], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p157, %r9157, 255; + selp.b32 %r9159, 7, 8, %p157; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_141: + setp.ne.s32 %p158, %r8561, 0; + mov.u32 %r8571, %r8556; + @%p158 bra $L__BB2_138; + +$L__BB2_142: + and.b32 %r4406, %r8465, 4; + setp.eq.s32 %p159, %r4406, 0; + mov.u32 %r8586, %r8571; + @%p159 bra $L__BB2_149; + + shr.u32 %r4407, %r252, 2; + and.b32 %r4408, %r4407, 1; + sub.s32 %r8576, %r251, %r4408; + setp.eq.s32 %p160, %r8576, 0; + mov.u32 %r8586, %r8571; + @%p160 bra $L__BB2_149; + + mov.u32 %r4409, -1; + shl.b32 %r4410, %r4409, %r8576; + not.b32 %r4411, %r4410; + and.b32 %r8577, %r8479, %r4411; + +$L__BB2_145: + setp.gt.u32 %p161, %r9160, 17476; + mov.u32 %r8586, 1; + @%p161 bra $L__BB2_149; + + sub.s32 %r4413, %r9159, %r9158; + min.u32 %r4414, %r4413, %r8576; + setp.eq.s32 %p162, %r4414, 32; + mov.u32 %r4415, -1; + shl.b32 %r4416, %r4415, %r4414; + not.b32 %r4417, %r4416; + selp.b32 %r4418, -1, %r4417, %p162; + and.b32 %r4419, %r4418, %r8577; + shl.b32 %r4420, %r4419, %r9158; + or.b32 %r9157, %r4420, %r9157; + add.s32 %r9158, %r4414, %r9158; + shr.u32 %r8577, %r8577, %r4414; + sub.s32 %r8576, %r8576, %r4414; + setp.lt.u32 %p163, %r9158, %r9159; + @%p163 bra $L__BB2_148; + + cvt.u64.u32 %rd157, %r9160; + add.s64 %rd158, %rd157, %rd5; + add.s64 %rd159, %rd1, %rd158; + st.global.u8 [%rd159], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p164, %r9157, 255; + selp.b32 %r9159, 7, 8, %p164; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_148: + setp.ne.s32 %p165, %r8576, 0; + mov.u32 %r8586, %r8571; + @%p165 bra $L__BB2_145; + +$L__BB2_149: + setp.eq.s32 %p166, %r139, 0; + mov.u32 %r9186, %r8586; + @%p166 bra $L__BB2_156; + + shr.u32 %r4423, %r252, 3; + sub.s32 %r8591, %r251, %r4423; + setp.eq.s32 %p167, %r8591, 0; + mov.u32 %r9186, %r8586; + @%p167 bra $L__BB2_156; + + mov.u32 %r4424, -1; + shl.b32 %r4425, %r4424, %r8591; + not.b32 %r4426, %r4425; + and.b32 %r8592, %r8478, %r4426; + +$L__BB2_152: + setp.gt.u32 %p168, %r9160, 17476; + mov.u32 %r9186, 1; + @%p168 bra $L__BB2_156; + + sub.s32 %r4428, %r9159, %r9158; + min.u32 %r4429, %r4428, %r8591; + setp.eq.s32 %p169, %r4429, 32; + mov.u32 %r4430, -1; + shl.b32 %r4431, %r4430, %r4429; + not.b32 %r4432, %r4431; + selp.b32 %r4433, -1, %r4432, %p169; + and.b32 %r4434, %r4433, %r8592; + shl.b32 %r4435, %r4434, %r9158; + or.b32 %r9157, %r4435, %r9157; + add.s32 %r9158, %r4429, %r9158; + shr.u32 %r8592, %r8592, %r4429; + sub.s32 %r8591, %r8591, %r4429; + setp.lt.u32 %p170, %r9158, %r9159; + @%p170 bra $L__BB2_155; + + cvt.u64.u32 %rd160, %r9160; + add.s64 %rd161, %rd160, %rd5; + add.s64 %rd162, %rd1, %rd161; + st.global.u8 [%rd162], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p171, %r9157, 255; + selp.b32 %r9159, 7, 8, %p171; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_155: + setp.ne.s32 %p172, %r8591, 0; + mov.u32 %r9186, %r8586; + @%p172 bra $L__BB2_152; + +$L__BB2_156: + add.s32 %r4438, %r8439, 2; + setp.lt.u32 %p173, %r4438, %r4057; + mul.lo.s32 %r346, %r133, 6; + cvt.u64.u32 %rd163, %r346; + add.s64 %rd11, %rd9, %rd163; + add.s32 %r4439, %r346, 2; + cvt.u64.u32 %rd164, %r4439; + add.s64 %rd12, %rd9, %rd164; + @%p173 bra $L__BB2_185; + bra.uni $L__BB2_157; + +$L__BB2_185: + cvt.u64.u32 %rd178, %r8440; + add.s64 %rd179, %rd178, %rd4; + shl.b64 %rd180, %rd179, 2; + add.s64 %rd181, %rd3, %rd180; + ld.global.u32 %r419, [%rd181]; + setp.eq.s32 %p210, %r419, 0; + mov.u32 %r8651, 0; + mov.u32 %r8650, %r8651; + @%p210 bra $L__BB2_187; + + and.b32 %r4510, %r419, -2147483648; + abs.s32 %r4511, %r419; + shl.b32 %r4512, %r4511, %r60; + or.b32 %r8650, %r4512, %r4510; + +$L__BB2_187: + shl.b32 %r4516, %r8650, 1; + shr.u32 %r4517, %r4516, %r46; + and.b32 %r422, %r4517, -2; + setp.eq.s32 %p211, %r422, 0; + mov.u32 %r8652, %r8651; + mov.u32 %r8658, %r8651; + @%p211 bra $L__BB2_189; + + add.s32 %r4519, %r422, -1; + clz.b32 %r4520, %r4519; + mov.u32 %r4521, 32; + sub.s32 %r8651, %r4521, %r4520; + shr.u32 %r4522, %r8650, 31; + add.s32 %r4523, %r4522, %r422; + add.s32 %r8652, %r4523, -2; + mov.u32 %r8658, 1; + +$L__BB2_189: + mov.u32 %r8655, 0; + mov.u32 %r8654, %r8655; + @%p63 bra $L__BB2_192; + + add.s32 %r4526, %r8440, %r4055; + cvt.u64.u32 %rd182, %r4526; + add.s64 %rd183, %rd182, %rd4; + shl.b64 %rd184, %rd183, 2; + add.s64 %rd185, %rd3, %rd184; + ld.global.u32 %r428, [%rd185]; + setp.eq.s32 %p213, %r428, 0; + @%p213 bra $L__BB2_192; + + and.b32 %r4527, %r428, -2147483648; + abs.s32 %r4528, %r428; + shl.b32 %r4529, %r4528, %r60; + or.b32 %r8654, %r4529, %r4527; + +$L__BB2_192: + shl.b32 %r4532, %r8654, 1; + shr.u32 %r4533, %r4532, %r46; + and.b32 %r431, %r4533, -2; + setp.eq.s32 %p214, %r431, 0; + mov.u32 %r8656, %r8655; + mov.u32 %r8673, %r8651; + @%p214 bra $L__BB2_194; + + or.b32 %r8658, %r8658, 2; + add.s32 %r4534, %r431, -1; + clz.b32 %r4535, %r4534; + mov.u32 %r4536, 32; + sub.s32 %r8655, %r4536, %r4535; + max.s32 %r8673, %r8651, %r8655; + shr.u32 %r4537, %r8654, 31; + add.s32 %r4538, %r4537, %r431; + add.s32 %r8656, %r4538, -2; + +$L__BB2_194: + add.s32 %r8675, %r8440, 1; + add.s32 %r4543, %r8439, 3; + setp.ge.u32 %p215, %r4543, %r4057; + mov.u32 %r8676, 0; + mov.u32 %r8669, %r8676; + mov.u32 %r8670, %r8676; + mov.u32 %r8671, %r8676; + mov.u32 %r8672, %r8676; + @%p215 bra $L__BB2_205; + + cvt.u64.u32 %rd186, %r8675; + add.s64 %rd187, %rd186, %rd4; + shl.b64 %rd188, %rd187, 2; + add.s64 %rd189, %rd3, %rd188; + ld.global.u32 %r441, [%rd189]; + setp.eq.s32 %p216, %r441, 0; + mov.u32 %r8670, 0; + mov.u32 %r8659, %r8670; + @%p216 bra $L__BB2_197; + + and.b32 %r4545, %r441, -2147483648; + abs.s32 %r4546, %r441; + shl.b32 %r4547, %r4546, %r60; + or.b32 %r8659, %r4547, %r4545; + +$L__BB2_197: + shl.b32 %r4550, %r8659, 1; + shr.u32 %r4551, %r4550, %r46; + and.b32 %r444, %r4551, -2; + setp.eq.s32 %p217, %r444, 0; + mov.u32 %r8672, %r8670; + @%p217 bra $L__BB2_199; + + or.b32 %r8658, %r8658, 4; + add.s32 %r4552, %r444, -1; + clz.b32 %r4553, %r4552; + mov.u32 %r4554, 32; + sub.s32 %r8670, %r4554, %r4553; + max.s32 %r8673, %r8673, %r8670; + shr.u32 %r4555, %r8659, 31; + add.s32 %r4556, %r4555, %r444; + add.s32 %r8672, %r4556, -2; + +$L__BB2_199: + mov.u32 %r8669, 0; + mov.u32 %r8664, %r8669; + @%p63 bra $L__BB2_202; + + add.s32 %r4559, %r8675, %r4055; + cvt.u64.u32 %rd190, %r4559; + add.s64 %rd191, %rd190, %rd4; + shl.b64 %rd192, %rd191, 2; + add.s64 %rd193, %rd3, %rd192; + ld.global.u32 %r453, [%rd193]; + setp.eq.s32 %p219, %r453, 0; + @%p219 bra $L__BB2_202; + + and.b32 %r4560, %r453, -2147483648; + abs.s32 %r4561, %r453; + shl.b32 %r4562, %r4561, %r60; + or.b32 %r8664, %r4562, %r4560; + +$L__BB2_202: + shl.b32 %r4565, %r8664, 1; + shr.u32 %r4566, %r4565, %r46; + and.b32 %r456, %r4566, -2; + setp.eq.s32 %p220, %r456, 0; + mov.u32 %r8671, %r8669; + @%p220 bra $L__BB2_204; + + or.b32 %r8658, %r8658, 8; + add.s32 %r4567, %r456, -1; + clz.b32 %r4568, %r4567; + mov.u32 %r4569, 32; + sub.s32 %r8669, %r4569, %r4568; + max.s32 %r8673, %r8673, %r8669; + shr.u32 %r4570, %r8664, 31; + add.s32 %r4571, %r4570, %r456; + add.s32 %r8671, %r4571, -2; + +$L__BB2_204: + add.s32 %r8675, %r8440, 2; + +$L__BB2_205: + mov.u32 %r8440, %r8675; + shr.u32 %r4573, %r8465, 1; + or.b32 %r473, %r4573, %r253; + add.s32 %r4574, %r8673, -1; + setp.lt.s32 %p221, %r8673, 2; + setp.gt.s32 %p222, %r8673, 1; + selp.b32 %r474, %r4574, 0, %p222; + @%p221 bra $L__BB2_207; + + setp.eq.s32 %p223, %r8651, %r8673; + selp.u32 %r4575, 1, 0, %p223; + setp.eq.s32 %p224, %r8655, %r8673; + selp.u32 %r4576, -1, 0, %p224; + bfi.b32 %r4577, %r4576, %r4575, 1, 1; + setp.eq.s32 %p225, %r8670, %r8673; + selp.u16 %rs571, 1, 0, %p225; + mul.wide.u16 %r4578, %rs571, 4; + or.b32 %r4579, %r4577, %r4578; + setp.eq.s32 %p226, %r8669, %r8673; + selp.u16 %rs572, 1, 0, %p226; + mul.wide.u16 %r4580, %rs572, 8; + or.b32 %r8676, %r4579, %r4580; + +$L__BB2_207: + and.b32 %r4581, %r8655, 255; + and.b32 %r4582, %r8476, 255; + setp.lt.u32 %p227, %r4581, %r4582; + cvt.u16.u32 %rs573, %r8655; + selp.b16 %rs574, %rs3, %rs573, %p227; + st.shared.u8 [%r136+1], %rs574; + st.shared.u8 [%r136+2], %r8669; + and.b32 %r477, %r8658, 2; + shr.u32 %r4583, %r477, 1; + or.b32 %r4584, %r140, %r4583; + st.shared.u8 [%r138+1], %r4584; + and.b32 %r478, %r8658, 8; + shr.u32 %r4585, %r478, 3; + st.shared.u8 [%r138+2], %r4585; + shl.b32 %r4586, %r8658, 4; + shl.b32 %r4587, %r473, 8; + or.b32 %r4588, %r4586, %r4587; + or.b32 %r4589, %r4588, %r8676; + mul.wide.u32 %rd195, %r4589, 2; + add.s64 %rd196, %rd8, %rd195; + ld.global.u16 %rs48, [%rd196]; + shr.u16 %rs575, %rs48, 4; + and.b16 %rs49, %rs575, 7; + setp.eq.s16 %p228, %rs49, 0; + mov.u32 %r8688, %r8495; + @%p228 bra $L__BB2_214; + + cvt.u32.u16 %r8677, %rs49; + shr.u16 %rs576, %rs48, 8; + cvt.u32.u16 %r8678, %rs576; + +$L__BB2_209: + mov.u32 %r481, %r8677; + setp.gt.u32 %p229, %r8972, 2879; + mov.u32 %r8688, 1; + @%p229 bra $L__BB2_214; + + mov.u32 %r4591, 8; + sub.s32 %r4592, %r4591, %r8970; + sub.s32 %r4593, %r4592, %r8971; + min.u32 %r4594, %r4593, %r481; + setp.eq.s32 %p230, %r4594, 32; + mov.u32 %r4595, -1; + shl.b32 %r4596, %r4595, %r4594; + not.b32 %r4597, %r4596; + selp.b32 %r4598, -1, %r4597, %p230; + and.b32 %r4599, %r4598, %r8678; + shl.b32 %r4600, %r4599, %r8971; + cvt.u16.u32 %rs577, %r4600; + or.b16 %rs1165, %rs1165, %rs577; + add.s32 %r8971, %r4594, %r8971; + sub.s32 %r8677, %r481, %r4594; + shr.u32 %r8678, %r8678, %r4594; + setp.gt.u32 %p231, %r4593, %r481; + @%p231 bra $L__BB2_213; + + setp.ne.s32 %p232, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs578, %rs1165, 255; + setp.ne.s16 %p233, %rs578, 127; + and.pred %p234, %p232, %p233; + @%p234 bra $L__BB2_213; + + mov.u32 %r4603, 20548; + sub.s32 %r4604, %r4603, %r8972; + cvt.u64.u32 %rd197, %r4604; + add.s64 %rd198, %rd197, %rd5; + add.s64 %rd199, %rd1, %rd198; + st.global.u8 [%rd199], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p235, %rs578, 143; + selp.u32 %r8970, 1, 0, %p235; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_213: + setp.ne.s32 %p236, %r8677, 0; + mov.u32 %r8688, %r8495; + @%p236 bra $L__BB2_209; + +$L__BB2_214: + setp.ne.s32 %p237, %r473, 0; + @%p237 bra $L__BB2_262; + + setp.eq.s32 %p238, %r8658, 0; + add.s32 %r4605, %r8524, 17477; + cvt.u64.u32 %rd200, %r4605; + add.s64 %rd201, %rd200, %rd5; + add.s64 %rd13, %rd1, %rd201; + @%p238 bra $L__BB2_254; + + shl.b16 %rs1096, %rs1096, 1; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p239, %r8530, 0; + mov.u32 %r8722, %r8733; + @%p239 bra $L__BB2_219; + + setp.gt.u32 %p240, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8722, 1; + @%p240 bra $L__BB2_219; + + st.global.u8 [%rd13], %rs1096; + add.s32 %r8524, %r8524, 1; + mov.u32 %r8530, 8; + mov.u16 %rs1096, 0; + mov.u32 %r8722, %r8733; + +$L__BB2_219: + setp.lt.u32 %p241, %r8735, 3; + mov.u32 %r8692, 0; + @%p241 bra $L__BB2_222; + + setp.lt.u32 %p242, %r8735, 6; + mov.u32 %r8692, 1; + @%p242 bra $L__BB2_222; + + setp.lt.u32 %p243, %r8735, 9; + setp.eq.s32 %p244, %r8735, 11; + selp.b32 %r4611, 4, 5, %p244; + setp.lt.u32 %p245, %r8735, 11; + selp.b32 %r4612, 3, %r4611, %p245; + selp.b32 %r8692, 2, %r4612, %p243; + +$L__BB2_222: + setp.eq.s32 %p246, %r8692, 0; + @%p246 bra $L__BB2_250; + + add.s32 %r505, %r8692, -1; + and.b32 %r506, %r8692, 3; + setp.eq.s32 %p247, %r506, 0; + mov.u32 %r8702, %r8692; + mov.u32 %r8705, %r8722; + @%p247 bra $L__BB2_235; + + mov.u32 %r4614, 1; + shl.b32 %r4615, %r4614, %r505; + and.b32 %r4616, %r4615, %r8736; + setp.ne.s32 %p248, %r4616, 0; + selp.u32 %r4617, 1, 0, %p248; + cvt.u32.u16 %r4618, %rs1096; + bfi.b32 %r4619, %r4618, %r4617, 1, 8; + cvt.u16.u32 %rs1096, %r4619; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p249, %r8530, 0; + mov.u32 %r8705, %r8722; + @%p249 bra $L__BB2_227; + + setp.gt.u32 %p250, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8705, %r4614; + @%p250 bra $L__BB2_227; + + add.s32 %r4623, %r8524, 17477; + cvt.u64.u32 %rd202, %r4623; + add.s64 %rd203, %rd202, %rd5; + add.s64 %rd204, %rd1, %rd203; + st.global.u8 [%rd204], %rs1096; + add.s32 %r8524, %r8524, 1; + mov.u32 %r8530, 8; + mov.u16 %rs1096, 0; + mov.u32 %r8705, %r8722; + +$L__BB2_227: + setp.eq.s32 %p251, %r506, 1; + mov.u32 %r8722, %r8705; + mov.u32 %r8702, %r505; + @%p251 bra $L__BB2_235; + + add.s32 %r8702, %r8692, -2; + mov.u32 %r4624, 1; + shl.b32 %r4625, %r4624, %r8702; + and.b32 %r4626, %r4625, %r8736; + setp.ne.s32 %p252, %r4626, 0; + selp.u32 %r4627, 1, 0, %p252; + cvt.u32.u16 %r4628, %rs1096; + bfi.b32 %r4629, %r4628, %r4627, 1, 8; + cvt.u16.u32 %rs1096, %r4629; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p253, %r8530, 0; + mov.u32 %r8696, %r8705; + @%p253 bra $L__BB2_231; + + setp.gt.u32 %p254, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8696, %r4624; + @%p254 bra $L__BB2_231; + + add.s32 %r4632, %r8524, 17477; + cvt.u64.u32 %rd205, %r4632; + add.s64 %rd206, %rd205, %rd5; + add.s64 %rd207, %rd1, %rd206; + and.b16 %rs585, %rs1096, 255; + st.global.u8 [%rd207], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p255, %rs585, 255; + selp.b32 %r8530, 7, 8, %p255; + mov.u16 %rs1096, 0; + mov.u32 %r8696, %r8705; + +$L__BB2_231: + setp.eq.s32 %p256, %r506, 2; + mov.u32 %r8722, %r8696; + mov.u32 %r8705, %r8696; + @%p256 bra $L__BB2_235; + + add.s32 %r8702, %r8692, -3; + mov.u32 %r4633, 1; + shl.b32 %r4634, %r4633, %r8702; + and.b32 %r4635, %r4634, %r8736; + setp.ne.s32 %p257, %r4635, 0; + selp.u32 %r4636, 1, 0, %p257; + cvt.u32.u16 %r4637, %rs1096; + bfi.b32 %r4638, %r4637, %r4636, 1, 8; + cvt.u16.u32 %rs1096, %r4638; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p258, %r8530, 0; + mov.u32 %r8722, %r8696; + mov.u32 %r8705, %r8696; + @%p258 bra $L__BB2_235; + + setp.gt.u32 %p259, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8722, %r4633; + mov.u32 %r8705, %r4633; + @%p259 bra $L__BB2_235; + + add.s32 %r4643, %r8524, 17477; + cvt.u64.u32 %rd208, %r4643; + add.s64 %rd209, %rd208, %rd5; + add.s64 %rd210, %rd1, %rd209; + and.b16 %rs588, %rs1096, 255; + st.global.u8 [%rd210], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p260, %rs588, 255; + selp.b32 %r8530, 7, 8, %p260; + mov.u16 %rs1096, 0; + mov.u32 %r8722, %r8696; + mov.u32 %r8705, %r8696; + +$L__BB2_235: + setp.lt.u32 %p261, %r505, 3; + @%p261 bra $L__BB2_250; + + mov.u32 %r8722, %r8705; + +$L__BB2_237: + add.s32 %r4644, %r8702, -1; + mov.u32 %r4645, 1; + shl.b32 %r4646, %r4645, %r4644; + and.b32 %r4647, %r4646, %r8736; + setp.ne.s32 %p262, %r4647, 0; + selp.u32 %r4648, 1, 0, %p262; + cvt.u32.u16 %r4649, %rs1096; + bfi.b32 %r8711, %r4649, %r4648, 1, 8; + add.s32 %r8712, %r8530, -1; + setp.ne.s32 %p263, %r8712, 0; + mov.u32 %r8710, %r8722; + @%p263 bra $L__BB2_240; + + setp.gt.u32 %p264, %r8524, 191; + mov.u32 %r8712, 0; + mov.u32 %r8710, %r4645; + @%p264 bra $L__BB2_240; + + cvt.u16.u32 %rs589, %r8711; + and.b16 %rs590, %rs589, 255; + add.s32 %r4653, %r8524, 17477; + cvt.u64.u32 %rd211, %r4653; + add.s64 %rd212, %rd211, %rd5; + add.s64 %rd213, %rd1, %rd212; + st.global.u8 [%rd213], %rs589; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p265, %rs590, 255; + selp.b32 %r8712, 7, 8, %p265; + mov.u32 %r8711, 0; + mov.u32 %r8710, %r8722; + +$L__BB2_240: + add.s32 %r4654, %r8702, -2; + shl.b32 %r4656, %r4645, %r4654; + and.b32 %r4657, %r4656, %r8736; + setp.ne.s32 %p266, %r4657, 0; + and.b32 %r4658, %r8711, 127; + selp.u32 %r4659, 1, 0, %p266; + bfi.b32 %r8715, %r4658, %r4659, 1, 7; + add.s32 %r8716, %r8712, -1; + setp.ne.s32 %p267, %r8716, 0; + mov.u32 %r8714, %r8710; + @%p267 bra $L__BB2_243; + + setp.gt.u32 %p268, %r8524, 191; + mov.u32 %r8716, 0; + mov.u32 %r8714, 1; + @%p268 bra $L__BB2_243; + + cvt.u16.u32 %rs591, %r8715; + and.b16 %rs592, %rs591, 255; + add.s32 %r4663, %r8524, 17477; + cvt.u64.u32 %rd214, %r4663; + add.s64 %rd215, %rd214, %rd5; + add.s64 %rd216, %rd1, %rd215; + st.global.u8 [%rd216], %rs591; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p269, %rs592, 255; + selp.b32 %r8716, 7, 8, %p269; + mov.u32 %r8715, 0; + mov.u32 %r8714, %r8710; + +$L__BB2_243: + add.s32 %r4664, %r8702, -3; + mov.u32 %r4665, 1; + shl.b32 %r4666, %r4665, %r4664; + and.b32 %r4667, %r4666, %r8736; + setp.ne.s32 %p270, %r4667, 0; + and.b32 %r4668, %r8715, 127; + selp.u32 %r4669, 1, 0, %p270; + bfi.b32 %r8719, %r4668, %r4669, 1, 7; + add.s32 %r8720, %r8716, -1; + setp.ne.s32 %p271, %r8720, 0; + mov.u32 %r8718, %r8714; + @%p271 bra $L__BB2_246; + + setp.gt.u32 %p272, %r8524, 191; + mov.u32 %r8720, 0; + mov.u32 %r8718, %r4665; + @%p272 bra $L__BB2_246; + + cvt.u16.u32 %rs593, %r8719; + and.b16 %rs594, %rs593, 255; + add.s32 %r4673, %r8524, 17477; + cvt.u64.u32 %rd217, %r4673; + add.s64 %rd218, %rd217, %rd5; + add.s64 %rd219, %rd1, %rd218; + st.global.u8 [%rd219], %rs593; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p273, %rs594, 255; + selp.b32 %r8720, 7, 8, %p273; + mov.u32 %r8719, 0; + mov.u32 %r8718, %r8714; + +$L__BB2_246: + add.s32 %r8702, %r8702, -4; + shl.b32 %r4675, %r4665, %r8702; + and.b32 %r4676, %r4675, %r8736; + setp.ne.s32 %p274, %r4676, 0; + and.b32 %r4677, %r8719, 127; + selp.u32 %r4678, 1, 0, %p274; + bfi.b32 %r4679, %r4677, %r4678, 1, 15; + cvt.u16.u32 %rs1096, %r4679; + add.s32 %r8530, %r8720, -1; + setp.ne.s32 %p275, %r8530, 0; + mov.u32 %r8722, %r8718; + @%p275 bra $L__BB2_249; + + setp.gt.u32 %p276, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8722, 1; + @%p276 bra $L__BB2_249; + + add.s32 %r4682, %r8524, 17477; + cvt.u64.u32 %rd220, %r4682; + add.s64 %rd221, %rd220, %rd5; + add.s64 %rd222, %rd1, %rd221; + and.b16 %rs596, %rs1096, 255; + st.global.u8 [%rd222], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p277, %rs596, 255; + selp.b32 %r8530, 7, 8, %p277; + mov.u16 %rs1096, 0; + mov.u32 %r8722, %r8718; + +$L__BB2_249: + setp.ne.s32 %p278, %r8702, 0; + @%p278 bra $L__BB2_237; + +$L__BB2_250: + add.s32 %r4684, %r8735, -1; + setp.eq.s32 %p279, %r8735, 0; + mov.u32 %r8736, 0; + selp.b32 %r8735, 0, %r4684, %p279; + setp.lt.u32 %p280, %r8735, 3; + mov.u32 %r8728, %r8736; + @%p280 bra $L__BB2_253; + + setp.lt.u32 %p281, %r8735, 6; + mov.u32 %r8728, 1; + @%p281 bra $L__BB2_253; + + setp.lt.u32 %p282, %r8735, 9; + setp.eq.s32 %p283, %r8735, 11; + selp.b32 %r4686, 4, 5, %p283; + setp.lt.u32 %p284, %r8735, 11; + selp.b32 %r4687, 3, %r4686, %p284; + selp.b32 %r8728, 2, %r4687, %p282; + +$L__BB2_253: + mov.u32 %r4689, 1; + shl.b32 %r8734, %r4689, %r8728; + mov.u32 %r8733, %r8722; + bra.uni $L__BB2_262; + +$L__BB2_157: + ld.global.u8 %rs26, [%rd11+1]; + ld.global.u8 %rs27, [%rd12]; + ld.global.u8 %rs28, [%rd12+1]; + ld.global.u8 %rs29, [%rd9]; + ld.global.u8 %rs30, [%rd9+1]; + ld.global.u8 %rs31, [%rd9+2]; + ld.global.u8 %rs32, [%rd9+3]; + setp.eq.s16 %p174, %rs26, 0; + mov.u32 %r8617, %r8495; + @%p174 bra $L__BB2_164; + + ld.global.u8 %r8607, [%rd11]; + cvt.u32.u16 %r8606, %rs26; + +$L__BB2_159: + mov.u32 %r349, %r8606; + setp.gt.u32 %p175, %r8972, 2879; + mov.u32 %r8617, 1; + @%p175 bra $L__BB2_164; + + mov.u32 %r4441, 8; + sub.s32 %r4442, %r4441, %r8970; + sub.s32 %r4443, %r4442, %r8971; + min.u32 %r4444, %r4443, %r349; + setp.eq.s32 %p176, %r4444, 32; + mov.u32 %r4445, -1; + shl.b32 %r4446, %r4445, %r4444; + not.b32 %r4447, %r4446; + selp.b32 %r4448, -1, %r4447, %p176; + and.b32 %r4449, %r4448, %r8607; + shl.b32 %r4450, %r4449, %r8971; + cvt.u16.u32 %rs552, %r4450; + or.b16 %rs1165, %rs1165, %rs552; + add.s32 %r8971, %r4444, %r8971; + sub.s32 %r8606, %r349, %r4444; + shr.u32 %r8607, %r8607, %r4444; + setp.gt.u32 %p177, %r4443, %r349; + @%p177 bra $L__BB2_163; + + setp.ne.s32 %p178, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs553, %rs1165, 255; + setp.ne.s16 %p179, %rs553, 127; + and.pred %p180, %p178, %p179; + @%p180 bra $L__BB2_163; + + mov.u32 %r4453, 20548; + sub.s32 %r4454, %r4453, %r8972; + cvt.u64.u32 %rd166, %r4454; + add.s64 %rd167, %rd166, %rd5; + add.s64 %rd168, %rd1, %rd167; + st.global.u8 [%rd168], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p181, %rs553, 143; + selp.u32 %r8970, 1, 0, %p181; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_163: + setp.ne.s32 %p182, %r8606, 0; + mov.u32 %r8617, %r8495; + @%p182 bra $L__BB2_159; + +$L__BB2_164: + setp.eq.s16 %p183, %rs30, 0; + mov.u32 %r8629, %r8617; + @%p183 bra $L__BB2_171; + + cvt.u32.u16 %r4455, %rs29; + and.b32 %r8619, %r4455, 255; + cvt.u32.u16 %r4456, %rs30; + and.b32 %r8618, %r4456, 255; + +$L__BB2_166: + mov.u32 %r368, %r8618; + setp.gt.u32 %p184, %r8972, 2879; + mov.u32 %r8629, 1; + @%p184 bra $L__BB2_171; + + mov.u32 %r4458, 8; + sub.s32 %r4459, %r4458, %r8970; + sub.s32 %r4460, %r4459, %r8971; + min.u32 %r4461, %r4460, %r368; + setp.eq.s32 %p185, %r4461, 32; + mov.u32 %r4462, -1; + shl.b32 %r4463, %r4462, %r4461; + not.b32 %r4464, %r4463; + selp.b32 %r4465, -1, %r4464, %p185; + and.b32 %r4466, %r4465, %r8619; + shl.b32 %r4467, %r4466, %r8971; + cvt.u16.u32 %rs557, %r4467; + or.b16 %rs1165, %rs1165, %rs557; + add.s32 %r8971, %r4461, %r8971; + sub.s32 %r8618, %r368, %r4461; + shr.u32 %r8619, %r8619, %r4461; + setp.gt.u32 %p186, %r4460, %r368; + @%p186 bra $L__BB2_170; + + setp.ne.s32 %p187, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs558, %rs1165, 255; + setp.ne.s16 %p188, %rs558, 127; + and.pred %p189, %p187, %p188; + @%p189 bra $L__BB2_170; + + mov.u32 %r4470, 20548; + sub.s32 %r4471, %r4470, %r8972; + cvt.u64.u32 %rd169, %r4471; + add.s64 %rd170, %rd169, %rd5; + add.s64 %rd171, %rd1, %rd170; + st.global.u8 [%rd171], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p190, %rs558, 143; + selp.u32 %r8970, 1, 0, %p190; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_170: + setp.ne.s32 %p191, %r8618, 0; + mov.u32 %r8629, %r8617; + @%p191 bra $L__BB2_166; + +$L__BB2_171: + setp.eq.s16 %p192, %rs28, 0; + mov.u32 %r8641, %r8629; + @%p192 bra $L__BB2_178; + + cvt.u32.u16 %r4472, %rs28; + and.b32 %r8630, %r4472, 255; + cvt.u32.u16 %r4473, %rs27; + and.b32 %r8631, %r4473, 255; + +$L__BB2_173: + mov.u32 %r387, %r8630; + setp.gt.u32 %p193, %r8972, 2879; + mov.u32 %r8641, 1; + @%p193 bra $L__BB2_178; + + mov.u32 %r4475, 8; + sub.s32 %r4476, %r4475, %r8970; + sub.s32 %r4477, %r4476, %r8971; + min.u32 %r4478, %r4477, %r387; + setp.eq.s32 %p194, %r4478, 32; + mov.u32 %r4479, -1; + shl.b32 %r4480, %r4479, %r4478; + not.b32 %r4481, %r4480; + selp.b32 %r4482, -1, %r4481, %p194; + and.b32 %r4483, %r4482, %r8631; + shl.b32 %r4484, %r4483, %r8971; + cvt.u16.u32 %rs562, %r4484; + or.b16 %rs1165, %rs1165, %rs562; + add.s32 %r8971, %r4478, %r8971; + sub.s32 %r8630, %r387, %r4478; + shr.u32 %r8631, %r8631, %r4478; + setp.gt.u32 %p195, %r4477, %r387; + @%p195 bra $L__BB2_177; + + setp.ne.s32 %p196, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs563, %rs1165, 255; + setp.ne.s16 %p197, %rs563, 127; + and.pred %p198, %p196, %p197; + @%p198 bra $L__BB2_177; + + mov.u32 %r4487, 20548; + sub.s32 %r4488, %r4487, %r8972; + cvt.u64.u32 %rd172, %r4488; + add.s64 %rd173, %rd172, %rd5; + add.s64 %rd174, %rd1, %rd173; + st.global.u8 [%rd174], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p199, %rs563, 143; + selp.u32 %r8970, 1, 0, %p199; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_177: + setp.ne.s32 %p200, %r8630, 0; + mov.u32 %r8641, %r8629; + @%p200 bra $L__BB2_173; + +$L__BB2_178: + setp.eq.s16 %p201, %rs32, 0; + mov.u32 %r8441, 0; + mov.u32 %r8969, %r8641; + @%p201 bra $L__BB2_416; + + cvt.u32.u16 %r4490, %rs31; + and.b32 %r8643, %r4490, 255; + cvt.u32.u16 %r4491, %rs32; + and.b32 %r8642, %r4491, 255; + +$L__BB2_180: + mov.u32 %r406, %r8642; + setp.gt.u32 %p202, %r8972, 2879; + mov.u32 %r8969, 1; + @%p202 bra $L__BB2_416; + + mov.u32 %r4494, 8; + sub.s32 %r4495, %r4494, %r8970; + sub.s32 %r4496, %r4495, %r8971; + min.u32 %r4497, %r4496, %r406; + setp.eq.s32 %p203, %r4497, 32; + mov.u32 %r4498, -1; + shl.b32 %r4499, %r4498, %r4497; + not.b32 %r4500, %r4499; + selp.b32 %r4501, -1, %r4500, %p203; + and.b32 %r4502, %r4501, %r8643; + shl.b32 %r4503, %r4502, %r8971; + cvt.u16.u32 %rs567, %r4503; + or.b16 %rs1165, %rs1165, %rs567; + add.s32 %r8971, %r4497, %r8971; + sub.s32 %r8642, %r406, %r4497; + shr.u32 %r8643, %r8643, %r4497; + setp.gt.u32 %p204, %r4496, %r406; + @%p204 bra $L__BB2_184; + + setp.ne.s32 %p205, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs568, %rs1165, 255; + setp.ne.s16 %p206, %rs568, 127; + and.pred %p207, %p205, %p206; + @%p207 bra $L__BB2_184; + + mov.u32 %r4506, 20548; + sub.s32 %r4507, %r4506, %r8972; + cvt.u64.u32 %rd175, %r4507; + add.s64 %rd176, %rd175, %rd5; + add.s64 %rd177, %rd1, %rd176; + st.global.u8 [%rd177], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p208, %rs568, 143; + selp.u32 %r8970, 1, 0, %p208; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_184: + setp.eq.s32 %p209, %r8642, 0; + mov.u32 %r8969, %r8641; + @%p209 bra $L__BB2_416; + bra.uni $L__BB2_180; + +$L__BB2_83: + setp.gt.u32 %p91, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8529, 1; + @%p91 bra $L__BB2_85; + + st.global.u8 [%rd10], %rs1096; + add.s32 %r8524, %r8524, 1; + mov.u32 %r8530, 8; + mov.u16 %rs1096, 0; + mov.u32 %r8529, %r8733; + bra.uni $L__BB2_85; + +$L__BB2_254: + add.s32 %r8736, %r8736, 1; + setp.lt.u32 %p285, %r8736, %r8734; + @%p285 bra $L__BB2_262; + + shl.b16 %rs597, %rs1096, 1; + or.b16 %rs1096, %rs597, 1; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p286, %r8530, 0; + mov.u32 %r8729, %r8733; + @%p286 bra $L__BB2_258; + bra.uni $L__BB2_256; + +$L__BB2_258: + add.s32 %r4693, %r8735, 1; + min.u32 %r8735, %r4693, 12; + setp.lt.u32 %p289, %r8735, 3; + mov.u32 %r8736, 0; + mov.u32 %r8732, %r8736; + @%p289 bra $L__BB2_261; + + setp.lt.u32 %p290, %r8735, 6; + mov.u32 %r8732, 1; + @%p290 bra $L__BB2_261; + + setp.lt.u32 %p291, %r8735, 9; + setp.eq.s32 %p292, %r8735, 11; + selp.b32 %r4695, 4, 5, %p292; + setp.lt.u32 %p293, %r8735, 11; + selp.b32 %r4696, 3, %r4695, %p293; + selp.b32 %r8732, 2, %r4696, %p291; + +$L__BB2_261: + mov.u32 %r4698, 1; + shl.b32 %r8734, %r4698, %r8732; + mov.u32 %r8733, %r8729; + +$L__BB2_262: + max.s32 %r589, %r8673, 1; + and.b16 %rs600, %rs48, 15; + cvt.u32.u16 %r590, %rs600; + and.b32 %r591, %r8658, 1; + setp.eq.s32 %p294, %r591, 0; + mov.u32 %r8749, %r9186; + @%p294 bra $L__BB2_269; + + and.b32 %r4699, %r590, 1; + sub.s32 %r8739, %r589, %r4699; + setp.eq.s32 %p295, %r8739, 0; + mov.u32 %r8749, %r9186; + @%p295 bra $L__BB2_269; + + mov.u32 %r4700, -1; + shl.b32 %r4701, %r4700, %r8739; + not.b32 %r4702, %r4701; + and.b32 %r8740, %r8652, %r4702; + +$L__BB2_265: + setp.gt.u32 %p296, %r9160, 17476; + mov.u32 %r8749, 1; + @%p296 bra $L__BB2_269; + + sub.s32 %r4704, %r9159, %r9158; + min.u32 %r4705, %r4704, %r8739; + setp.eq.s32 %p297, %r4705, 32; + mov.u32 %r4706, -1; + shl.b32 %r4707, %r4706, %r4705; + not.b32 %r4708, %r4707; + selp.b32 %r4709, -1, %r4708, %p297; + and.b32 %r4710, %r4709, %r8740; + shl.b32 %r4711, %r4710, %r9158; + or.b32 %r9157, %r4711, %r9157; + add.s32 %r9158, %r4705, %r9158; + shr.u32 %r8740, %r8740, %r4705; + sub.s32 %r8739, %r8739, %r4705; + setp.lt.u32 %p298, %r9158, %r9159; + @%p298 bra $L__BB2_268; + + cvt.u64.u32 %rd223, %r9160; + add.s64 %rd224, %rd223, %rd5; + add.s64 %rd225, %rd1, %rd224; + st.global.u8 [%rd225], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p299, %r9157, 255; + selp.b32 %r9159, 7, 8, %p299; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_268: + setp.ne.s32 %p300, %r8739, 0; + mov.u32 %r8749, %r9186; + @%p300 bra $L__BB2_265; + +$L__BB2_269: + setp.eq.s32 %p301, %r477, 0; + mov.u32 %r8764, %r8749; + @%p301 bra $L__BB2_276; + + shr.u32 %r4714, %r590, 1; + and.b32 %r4715, %r4714, 1; + sub.s32 %r8754, %r589, %r4715; + setp.eq.s32 %p302, %r8754, 0; + mov.u32 %r8764, %r8749; + @%p302 bra $L__BB2_276; + + mov.u32 %r4716, -1; + shl.b32 %r4717, %r4716, %r8754; + not.b32 %r4718, %r4717; + and.b32 %r8755, %r8656, %r4718; + +$L__BB2_272: + setp.gt.u32 %p303, %r9160, 17476; + mov.u32 %r8764, 1; + @%p303 bra $L__BB2_276; + + sub.s32 %r4720, %r9159, %r9158; + min.u32 %r4721, %r4720, %r8754; + setp.eq.s32 %p304, %r4721, 32; + mov.u32 %r4722, -1; + shl.b32 %r4723, %r4722, %r4721; + not.b32 %r4724, %r4723; + selp.b32 %r4725, -1, %r4724, %p304; + and.b32 %r4726, %r4725, %r8755; + shl.b32 %r4727, %r4726, %r9158; + or.b32 %r9157, %r4727, %r9157; + add.s32 %r9158, %r4721, %r9158; + shr.u32 %r8755, %r8755, %r4721; + sub.s32 %r8754, %r8754, %r4721; + setp.lt.u32 %p305, %r9158, %r9159; + @%p305 bra $L__BB2_275; + + cvt.u64.u32 %rd226, %r9160; + add.s64 %rd227, %rd226, %rd5; + add.s64 %rd228, %rd1, %rd227; + st.global.u8 [%rd228], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p306, %r9157, 255; + selp.b32 %r9159, 7, 8, %p306; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_275: + setp.ne.s32 %p307, %r8754, 0; + mov.u32 %r8764, %r8749; + @%p307 bra $L__BB2_272; + +$L__BB2_276: + and.b32 %r4730, %r8658, 4; + setp.eq.s32 %p308, %r4730, 0; + mov.u32 %r8779, %r8764; + @%p308 bra $L__BB2_283; + + shr.u32 %r4731, %r590, 2; + and.b32 %r4732, %r4731, 1; + sub.s32 %r8769, %r589, %r4732; + setp.eq.s32 %p309, %r8769, 0; + mov.u32 %r8779, %r8764; + @%p309 bra $L__BB2_283; + + mov.u32 %r4733, -1; + shl.b32 %r4734, %r4733, %r8769; + not.b32 %r4735, %r4734; + and.b32 %r8770, %r8672, %r4735; + +$L__BB2_279: + setp.gt.u32 %p310, %r9160, 17476; + mov.u32 %r8779, 1; + @%p310 bra $L__BB2_283; + + sub.s32 %r4737, %r9159, %r9158; + min.u32 %r4738, %r4737, %r8769; + setp.eq.s32 %p311, %r4738, 32; + mov.u32 %r4739, -1; + shl.b32 %r4740, %r4739, %r4738; + not.b32 %r4741, %r4740; + selp.b32 %r4742, -1, %r4741, %p311; + and.b32 %r4743, %r4742, %r8770; + shl.b32 %r4744, %r4743, %r9158; + or.b32 %r9157, %r4744, %r9157; + add.s32 %r9158, %r4738, %r9158; + shr.u32 %r8770, %r8770, %r4738; + sub.s32 %r8769, %r8769, %r4738; + setp.lt.u32 %p312, %r9158, %r9159; + @%p312 bra $L__BB2_282; + + cvt.u64.u32 %rd229, %r9160; + add.s64 %rd230, %rd229, %rd5; + add.s64 %rd231, %rd1, %rd230; + st.global.u8 [%rd231], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p313, %r9157, 255; + selp.b32 %r9159, 7, 8, %p313; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_282: + setp.ne.s32 %p314, %r8769, 0; + mov.u32 %r8779, %r8764; + @%p314 bra $L__BB2_279; + +$L__BB2_283: + setp.eq.s32 %p315, %r478, 0; + mov.u32 %r9186, %r8779; + @%p315 bra $L__BB2_290; + + shr.u32 %r4747, %r590, 3; + sub.s32 %r8784, %r589, %r4747; + setp.eq.s32 %p316, %r8784, 0; + mov.u32 %r9186, %r8779; + @%p316 bra $L__BB2_290; + + mov.u32 %r4748, -1; + shl.b32 %r4749, %r4748, %r8784; + not.b32 %r4750, %r4749; + and.b32 %r8785, %r8671, %r4750; + +$L__BB2_286: + setp.gt.u32 %p317, %r9160, 17476; + mov.u32 %r9186, 1; + @%p317 bra $L__BB2_290; + + sub.s32 %r4752, %r9159, %r9158; + min.u32 %r4753, %r4752, %r8784; + setp.eq.s32 %p318, %r4753, 32; + mov.u32 %r4754, -1; + shl.b32 %r4755, %r4754, %r4753; + not.b32 %r4756, %r4755; + selp.b32 %r4757, -1, %r4756, %p318; + and.b32 %r4758, %r4757, %r8785; + shl.b32 %r4759, %r4758, %r9158; + or.b32 %r9157, %r4759, %r9157; + add.s32 %r9158, %r4753, %r9158; + shr.u32 %r8785, %r8785, %r4753; + sub.s32 %r8784, %r8784, %r4753; + setp.lt.u32 %p319, %r9158, %r9159; + @%p319 bra $L__BB2_289; + + cvt.u64.u32 %rd232, %r9160; + add.s64 %rd233, %rd232, %rd5; + add.s64 %rd234, %rd1, %rd233; + st.global.u8 [%rd234], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p320, %r9157, 255; + selp.b32 %r9159, 7, 8, %p320; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_289: + setp.ne.s32 %p321, %r8784, 0; + mov.u32 %r9186, %r8779; + @%p321 bra $L__BB2_286; + +$L__BB2_290: + setp.lt.s32 %p322, %r474, 1; + setp.lt.s32 %p323, %r133, 1; + or.pred %p324, %p323, %p322; + @%p324 bra $L__BB2_338; + + min.s32 %r4762, %r133, %r474; + setp.lt.s32 %p325, %r4762, 3; + add.s32 %r4763, %r8524, 17477; + cvt.u64.u32 %rd235, %r4763; + add.s64 %rd236, %rd235, %rd5; + add.s64 %rd14, %rd1, %rd236; + @%p325 bra $L__BB2_330; + bra.uni $L__BB2_292; + +$L__BB2_330: + add.s32 %r8736, %r8736, 1; + setp.lt.u32 %p372, %r8736, %r8734; + @%p372 bra $L__BB2_338; + + shl.b16 %rs617, %rs1096, 1; + or.b16 %rs1096, %rs617, 1; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p373, %r8530, 0; + mov.u32 %r8839, %r8733; + @%p373 bra $L__BB2_334; + + setp.gt.u32 %p374, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8839, 1; + @%p374 bra $L__BB2_334; + + and.b16 %rs619, %rs1096, 255; + st.global.u8 [%rd14], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p375, %rs619, 255; + selp.b32 %r8530, 7, 8, %p375; + mov.u16 %rs1096, 0; + mov.u32 %r8839, %r8733; + +$L__BB2_334: + add.s32 %r4851, %r8735, 1; + min.u32 %r8735, %r4851, 12; + setp.lt.u32 %p376, %r8735, 3; + mov.u32 %r8736, 0; + mov.u32 %r8842, %r8736; + @%p376 bra $L__BB2_337; + + setp.lt.u32 %p377, %r8735, 6; + mov.u32 %r8842, 1; + @%p377 bra $L__BB2_337; + + setp.lt.u32 %p378, %r8735, 9; + setp.eq.s32 %p379, %r8735, 11; + selp.b32 %r4853, 4, 5, %p379; + setp.lt.u32 %p380, %r8735, 11; + selp.b32 %r4854, 3, %r4853, %p380; + selp.b32 %r8842, 2, %r4854, %p378; + +$L__BB2_337: + mov.u32 %r4856, 1; + shl.b32 %r8734, %r4856, %r8842; + mov.u32 %r8733, %r8839; + bra.uni $L__BB2_338; + +$L__BB2_292: + shl.b16 %rs1096, %rs1096, 1; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p326, %r8530, 0; + mov.u32 %r8832, %r8733; + @%p326 bra $L__BB2_295; + + setp.gt.u32 %p327, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8832, 1; + @%p327 bra $L__BB2_295; + + st.global.u8 [%rd14], %rs1096; + add.s32 %r8524, %r8524, 1; + mov.u32 %r8530, 8; + mov.u16 %rs1096, 0; + mov.u32 %r8832, %r8733; + +$L__BB2_295: + setp.lt.u32 %p328, %r8735, 3; + mov.u32 %r8802, 0; + @%p328 bra $L__BB2_298; + + setp.lt.u32 %p329, %r8735, 6; + mov.u32 %r8802, 1; + @%p329 bra $L__BB2_298; + + setp.lt.u32 %p330, %r8735, 9; + setp.eq.s32 %p331, %r8735, 11; + selp.b32 %r4769, 4, 5, %p331; + setp.lt.u32 %p332, %r8735, 11; + selp.b32 %r4770, 3, %r4769, %p332; + selp.b32 %r8802, 2, %r4770, %p330; + +$L__BB2_298: + setp.eq.s32 %p333, %r8802, 0; + @%p333 bra $L__BB2_326; + + add.s32 %r691, %r8802, -1; + and.b32 %r692, %r8802, 3; + setp.eq.s32 %p334, %r692, 0; + mov.u32 %r8812, %r8802; + mov.u32 %r8815, %r8832; + @%p334 bra $L__BB2_311; + + mov.u32 %r4772, 1; + shl.b32 %r4773, %r4772, %r691; + and.b32 %r4774, %r4773, %r8736; + setp.ne.s32 %p335, %r4774, 0; + selp.u32 %r4775, 1, 0, %p335; + cvt.u32.u16 %r4776, %rs1096; + bfi.b32 %r4777, %r4776, %r4775, 1, 8; + cvt.u16.u32 %rs1096, %r4777; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p336, %r8530, 0; + mov.u32 %r8815, %r8832; + @%p336 bra $L__BB2_303; + + setp.gt.u32 %p337, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8815, %r4772; + @%p337 bra $L__BB2_303; + + add.s32 %r4781, %r8524, 17477; + cvt.u64.u32 %rd237, %r4781; + add.s64 %rd238, %rd237, %rd5; + add.s64 %rd239, %rd1, %rd238; + st.global.u8 [%rd239], %rs1096; + add.s32 %r8524, %r8524, 1; + mov.u32 %r8530, 8; + mov.u16 %rs1096, 0; + mov.u32 %r8815, %r8832; + +$L__BB2_303: + setp.eq.s32 %p338, %r692, 1; + mov.u32 %r8832, %r8815; + mov.u32 %r8812, %r691; + @%p338 bra $L__BB2_311; + + add.s32 %r8812, %r8802, -2; + mov.u32 %r4782, 1; + shl.b32 %r4783, %r4782, %r8812; + and.b32 %r4784, %r4783, %r8736; + setp.ne.s32 %p339, %r4784, 0; + selp.u32 %r4785, 1, 0, %p339; + cvt.u32.u16 %r4786, %rs1096; + bfi.b32 %r4787, %r4786, %r4785, 1, 8; + cvt.u16.u32 %rs1096, %r4787; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p340, %r8530, 0; + mov.u32 %r8806, %r8815; + @%p340 bra $L__BB2_307; + + setp.gt.u32 %p341, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8806, %r4782; + @%p341 bra $L__BB2_307; + + add.s32 %r4790, %r8524, 17477; + cvt.u64.u32 %rd240, %r4790; + add.s64 %rd241, %rd240, %rd5; + add.s64 %rd242, %rd1, %rd241; + and.b16 %rs605, %rs1096, 255; + st.global.u8 [%rd242], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p342, %rs605, 255; + selp.b32 %r8530, 7, 8, %p342; + mov.u16 %rs1096, 0; + mov.u32 %r8806, %r8815; + +$L__BB2_307: + setp.eq.s32 %p343, %r692, 2; + mov.u32 %r8832, %r8806; + mov.u32 %r8815, %r8806; + @%p343 bra $L__BB2_311; + + add.s32 %r8812, %r8802, -3; + mov.u32 %r4791, 1; + shl.b32 %r4792, %r4791, %r8812; + and.b32 %r4793, %r4792, %r8736; + setp.ne.s32 %p344, %r4793, 0; + selp.u32 %r4794, 1, 0, %p344; + cvt.u32.u16 %r4795, %rs1096; + bfi.b32 %r4796, %r4795, %r4794, 1, 8; + cvt.u16.u32 %rs1096, %r4796; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p345, %r8530, 0; + mov.u32 %r8832, %r8806; + mov.u32 %r8815, %r8806; + @%p345 bra $L__BB2_311; + + setp.gt.u32 %p346, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8832, %r4791; + mov.u32 %r8815, %r4791; + @%p346 bra $L__BB2_311; + + add.s32 %r4801, %r8524, 17477; + cvt.u64.u32 %rd243, %r4801; + add.s64 %rd244, %rd243, %rd5; + add.s64 %rd245, %rd1, %rd244; + and.b16 %rs608, %rs1096, 255; + st.global.u8 [%rd245], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p347, %rs608, 255; + selp.b32 %r8530, 7, 8, %p347; + mov.u16 %rs1096, 0; + mov.u32 %r8832, %r8806; + mov.u32 %r8815, %r8806; + +$L__BB2_311: + setp.lt.u32 %p348, %r691, 3; + @%p348 bra $L__BB2_326; + + mov.u32 %r8832, %r8815; + +$L__BB2_313: + add.s32 %r4802, %r8812, -1; + mov.u32 %r4803, 1; + shl.b32 %r4804, %r4803, %r4802; + and.b32 %r4805, %r4804, %r8736; + setp.ne.s32 %p349, %r4805, 0; + selp.u32 %r4806, 1, 0, %p349; + cvt.u32.u16 %r4807, %rs1096; + bfi.b32 %r8821, %r4807, %r4806, 1, 8; + add.s32 %r8822, %r8530, -1; + setp.ne.s32 %p350, %r8822, 0; + mov.u32 %r8820, %r8832; + @%p350 bra $L__BB2_316; + + setp.gt.u32 %p351, %r8524, 191; + mov.u32 %r8822, 0; + mov.u32 %r8820, %r4803; + @%p351 bra $L__BB2_316; + + cvt.u16.u32 %rs609, %r8821; + and.b16 %rs610, %rs609, 255; + add.s32 %r4811, %r8524, 17477; + cvt.u64.u32 %rd246, %r4811; + add.s64 %rd247, %rd246, %rd5; + add.s64 %rd248, %rd1, %rd247; + st.global.u8 [%rd248], %rs609; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p352, %rs610, 255; + selp.b32 %r8822, 7, 8, %p352; + mov.u32 %r8821, 0; + mov.u32 %r8820, %r8832; + +$L__BB2_316: + add.s32 %r4812, %r8812, -2; + shl.b32 %r4814, %r4803, %r4812; + and.b32 %r4815, %r4814, %r8736; + setp.ne.s32 %p353, %r4815, 0; + and.b32 %r4816, %r8821, 127; + selp.u32 %r4817, 1, 0, %p353; + bfi.b32 %r8825, %r4816, %r4817, 1, 7; + add.s32 %r8826, %r8822, -1; + setp.ne.s32 %p354, %r8826, 0; + mov.u32 %r8824, %r8820; + @%p354 bra $L__BB2_319; + + setp.gt.u32 %p355, %r8524, 191; + mov.u32 %r8826, 0; + mov.u32 %r8824, 1; + @%p355 bra $L__BB2_319; + + cvt.u16.u32 %rs611, %r8825; + and.b16 %rs612, %rs611, 255; + add.s32 %r4821, %r8524, 17477; + cvt.u64.u32 %rd249, %r4821; + add.s64 %rd250, %rd249, %rd5; + add.s64 %rd251, %rd1, %rd250; + st.global.u8 [%rd251], %rs611; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p356, %rs612, 255; + selp.b32 %r8826, 7, 8, %p356; + mov.u32 %r8825, 0; + mov.u32 %r8824, %r8820; + +$L__BB2_319: + add.s32 %r4822, %r8812, -3; + mov.u32 %r4823, 1; + shl.b32 %r4824, %r4823, %r4822; + and.b32 %r4825, %r4824, %r8736; + setp.ne.s32 %p357, %r4825, 0; + and.b32 %r4826, %r8825, 127; + selp.u32 %r4827, 1, 0, %p357; + bfi.b32 %r8829, %r4826, %r4827, 1, 7; + add.s32 %r8830, %r8826, -1; + setp.ne.s32 %p358, %r8830, 0; + mov.u32 %r8828, %r8824; + @%p358 bra $L__BB2_322; + + setp.gt.u32 %p359, %r8524, 191; + mov.u32 %r8830, 0; + mov.u32 %r8828, %r4823; + @%p359 bra $L__BB2_322; + + cvt.u16.u32 %rs613, %r8829; + and.b16 %rs614, %rs613, 255; + add.s32 %r4831, %r8524, 17477; + cvt.u64.u32 %rd252, %r4831; + add.s64 %rd253, %rd252, %rd5; + add.s64 %rd254, %rd1, %rd253; + st.global.u8 [%rd254], %rs613; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p360, %rs614, 255; + selp.b32 %r8830, 7, 8, %p360; + mov.u32 %r8829, 0; + mov.u32 %r8828, %r8824; + +$L__BB2_322: + add.s32 %r8812, %r8812, -4; + shl.b32 %r4833, %r4823, %r8812; + and.b32 %r4834, %r4833, %r8736; + setp.ne.s32 %p361, %r4834, 0; + and.b32 %r4835, %r8829, 127; + selp.u32 %r4836, 1, 0, %p361; + bfi.b32 %r4837, %r4835, %r4836, 1, 15; + cvt.u16.u32 %rs1096, %r4837; + add.s32 %r8530, %r8830, -1; + setp.ne.s32 %p362, %r8530, 0; + mov.u32 %r8832, %r8828; + @%p362 bra $L__BB2_325; + + setp.gt.u32 %p363, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r8832, 1; + @%p363 bra $L__BB2_325; + + add.s32 %r4840, %r8524, 17477; + cvt.u64.u32 %rd255, %r4840; + add.s64 %rd256, %rd255, %rd5; + add.s64 %rd257, %rd1, %rd256; + and.b16 %rs616, %rs1096, 255; + st.global.u8 [%rd257], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p364, %rs616, 255; + selp.b32 %r8530, 7, 8, %p364; + mov.u16 %rs1096, 0; + mov.u32 %r8832, %r8828; + +$L__BB2_325: + setp.ne.s32 %p365, %r8812, 0; + @%p365 bra $L__BB2_313; + +$L__BB2_326: + add.s32 %r4842, %r8735, -1; + setp.eq.s32 %p366, %r8735, 0; + mov.u32 %r8736, 0; + selp.b32 %r8735, 0, %r4842, %p366; + setp.lt.u32 %p367, %r8735, 3; + mov.u32 %r8838, %r8736; + @%p367 bra $L__BB2_329; + + setp.lt.u32 %p368, %r8735, 6; + mov.u32 %r8838, 1; + @%p368 bra $L__BB2_329; + + setp.lt.u32 %p369, %r8735, 9; + setp.eq.s32 %p370, %r8735, 11; + selp.b32 %r4844, 4, 5, %p370; + setp.lt.u32 %p371, %r8735, 11; + selp.b32 %r4845, 3, %r4844, %p371; + selp.b32 %r8838, 2, %r4845, %p369; + +$L__BB2_329: + mov.u32 %r4847, 1; + shl.b32 %r8734, %r4847, %r8838; + mov.u32 %r8733, %r8832; + +$L__BB2_338: + setp.gt.s32 %p381, %r474, 2; + setp.gt.s32 %p382, %r133, 2; + and.pred %p383, %p382, %p381; + @%p383 bra $L__BB2_387; + bra.uni $L__BB2_339; + +$L__BB2_387: + add.s32 %r4977, %r346, -11; + cvt.u64.u32 %rd287, %r4977; + add.s64 %rd16, %rd9, %rd287; + ld.global.u8 %rs122, [%rd16]; + add.s32 %r4978, %r346, -10; + cvt.u64.u32 %rd289, %r4978; + add.s64 %rd290, %rd9, %rd289; + ld.global.u8 %rs123, [%rd290]; + ld.global.u8 %rs124, [%rd290+1]; + mul.lo.s32 %r4979, %r474, 6; + add.s32 %r4980, %r4979, -12; + cvt.u64.u32 %rd291, %r4980; + add.s64 %rd292, %rd9, %rd291; + ld.global.u8 %rs125, [%rd292]; + ld.global.u8 %rs126, [%rd292+1]; + add.s32 %r4981, %r4979, -10; + cvt.u64.u32 %rd293, %r4981; + add.s64 %rd294, %rd9, %rd293; + ld.global.u8 %rs127, [%rd294]; + ld.global.u8 %rs128, [%rd294+1]; + setp.eq.s16 %p451, %rs122, 0; + mov.u32 %r8936, %r8688; + @%p451 bra $L__BB2_394; + + ld.global.u8 %r8926, [%rd16+-1]; + cvt.u32.u16 %r8925, %rs122; + +$L__BB2_389: + mov.u32 %r901, %r8925; + setp.gt.u32 %p452, %r8972, 2879; + mov.u32 %r8936, 1; + @%p452 bra $L__BB2_394; + + mov.u32 %r4983, 8; + sub.s32 %r4984, %r4983, %r8970; + sub.s32 %r4985, %r4984, %r8971; + min.u32 %r4986, %r4985, %r901; + setp.eq.s32 %p453, %r4986, 32; + mov.u32 %r4987, -1; + shl.b32 %r4988, %r4987, %r4986; + not.b32 %r4989, %r4988; + selp.b32 %r4990, -1, %r4989, %p453; + and.b32 %r4991, %r4990, %r8926; + shl.b32 %r4992, %r4991, %r8971; + cvt.u16.u32 %rs652, %r4992; + or.b16 %rs1165, %rs1165, %rs652; + add.s32 %r8971, %r4986, %r8971; + sub.s32 %r8925, %r901, %r4986; + shr.u32 %r8926, %r8926, %r4986; + setp.gt.u32 %p454, %r4985, %r901; + @%p454 bra $L__BB2_393; + + setp.ne.s32 %p455, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs653, %rs1165, 255; + setp.ne.s16 %p456, %rs653, 127; + and.pred %p457, %p455, %p456; + @%p457 bra $L__BB2_393; + + mov.u32 %r4995, 20548; + sub.s32 %r4996, %r4995, %r8972; + cvt.u64.u32 %rd295, %r4996; + add.s64 %rd296, %rd295, %rd5; + add.s64 %rd297, %rd1, %rd296; + st.global.u8 [%rd297], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p458, %rs653, 143; + selp.u32 %r8970, 1, 0, %p458; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_393: + setp.ne.s32 %p459, %r8925, 0; + mov.u32 %r8936, %r8688; + @%p459 bra $L__BB2_389; + +$L__BB2_394: + setp.eq.s16 %p460, %rs126, 0; + mov.u32 %r8948, %r8936; + @%p460 bra $L__BB2_401; + + cvt.u32.u16 %r4997, %rs125; + and.b32 %r8938, %r4997, 255; + cvt.u32.u16 %r4998, %rs126; + and.b32 %r8937, %r4998, 255; + +$L__BB2_396: + mov.u32 %r920, %r8937; + setp.gt.u32 %p461, %r8972, 2879; + mov.u32 %r8948, 1; + @%p461 bra $L__BB2_401; + + mov.u32 %r5000, 8; + sub.s32 %r5001, %r5000, %r8970; + sub.s32 %r5002, %r5001, %r8971; + min.u32 %r5003, %r5002, %r920; + setp.eq.s32 %p462, %r5003, 32; + mov.u32 %r5004, -1; + shl.b32 %r5005, %r5004, %r5003; + not.b32 %r5006, %r5005; + selp.b32 %r5007, -1, %r5006, %p462; + and.b32 %r5008, %r5007, %r8938; + shl.b32 %r5009, %r5008, %r8971; + cvt.u16.u32 %rs657, %r5009; + or.b16 %rs1165, %rs1165, %rs657; + add.s32 %r8971, %r5003, %r8971; + sub.s32 %r8937, %r920, %r5003; + shr.u32 %r8938, %r8938, %r5003; + setp.gt.u32 %p463, %r5002, %r920; + @%p463 bra $L__BB2_400; + + setp.ne.s32 %p464, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs658, %rs1165, 255; + setp.ne.s16 %p465, %rs658, 127; + and.pred %p466, %p464, %p465; + @%p466 bra $L__BB2_400; + + mov.u32 %r5012, 20548; + sub.s32 %r5013, %r5012, %r8972; + cvt.u64.u32 %rd298, %r5013; + add.s64 %rd299, %rd298, %rd5; + add.s64 %rd300, %rd1, %rd299; + st.global.u8 [%rd300], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p467, %rs658, 143; + selp.u32 %r8970, 1, 0, %p467; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_400: + setp.ne.s32 %p468, %r8937, 0; + mov.u32 %r8948, %r8936; + @%p468 bra $L__BB2_396; + +$L__BB2_401: + setp.eq.s16 %p469, %rs124, 0; + mov.u32 %r8960, %r8948; + @%p469 bra $L__BB2_408; + + cvt.u32.u16 %r5014, %rs124; + and.b32 %r8949, %r5014, 255; + cvt.u32.u16 %r5015, %rs123; + and.b32 %r8950, %r5015, 255; + +$L__BB2_403: + mov.u32 %r939, %r8949; + setp.gt.u32 %p470, %r8972, 2879; + mov.u32 %r8960, 1; + @%p470 bra $L__BB2_408; + + mov.u32 %r5017, 8; + sub.s32 %r5018, %r5017, %r8970; + sub.s32 %r5019, %r5018, %r8971; + min.u32 %r5020, %r5019, %r939; + setp.eq.s32 %p471, %r5020, 32; + mov.u32 %r5021, -1; + shl.b32 %r5022, %r5021, %r5020; + not.b32 %r5023, %r5022; + selp.b32 %r5024, -1, %r5023, %p471; + and.b32 %r5025, %r5024, %r8950; + shl.b32 %r5026, %r5025, %r8971; + cvt.u16.u32 %rs662, %r5026; + or.b16 %rs1165, %rs1165, %rs662; + add.s32 %r8971, %r5020, %r8971; + sub.s32 %r8949, %r939, %r5020; + shr.u32 %r8950, %r8950, %r5020; + setp.gt.u32 %p472, %r5019, %r939; + @%p472 bra $L__BB2_407; + + setp.ne.s32 %p473, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs663, %rs1165, 255; + setp.ne.s16 %p474, %rs663, 127; + and.pred %p475, %p473, %p474; + @%p475 bra $L__BB2_407; + + mov.u32 %r5029, 20548; + sub.s32 %r5030, %r5029, %r8972; + cvt.u64.u32 %rd301, %r5030; + add.s64 %rd302, %rd301, %rd5; + add.s64 %rd303, %rd1, %rd302; + st.global.u8 [%rd303], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p476, %rs663, 143; + selp.u32 %r8970, 1, 0, %p476; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_407: + setp.ne.s32 %p477, %r8949, 0; + mov.u32 %r8960, %r8948; + @%p477 bra $L__BB2_403; + +$L__BB2_408: + setp.eq.s16 %p478, %rs128, 0; + mov.u32 %r8969, %r8960; + @%p478 bra $L__BB2_415; + + cvt.u32.u16 %r5031, %rs127; + and.b32 %r8962, %r5031, 255; + cvt.u32.u16 %r5032, %rs128; + and.b32 %r8961, %r5032, 255; + +$L__BB2_410: + mov.u32 %r958, %r8961; + setp.gt.u32 %p479, %r8972, 2879; + mov.u32 %r8969, 1; + @%p479 bra $L__BB2_415; + + mov.u32 %r5034, 8; + sub.s32 %r5035, %r5034, %r8970; + sub.s32 %r5036, %r5035, %r8971; + min.u32 %r5037, %r5036, %r958; + setp.eq.s32 %p480, %r5037, 32; + mov.u32 %r5038, -1; + shl.b32 %r5039, %r5038, %r5037; + not.b32 %r5040, %r5039; + selp.b32 %r5041, -1, %r5040, %p480; + and.b32 %r5042, %r5041, %r8962; + shl.b32 %r5043, %r5042, %r8971; + cvt.u16.u32 %rs667, %r5043; + or.b16 %rs1165, %rs1165, %rs667; + add.s32 %r8971, %r5037, %r8971; + sub.s32 %r8961, %r958, %r5037; + shr.u32 %r8962, %r8962, %r5037; + setp.gt.u32 %p481, %r5036, %r958; + @%p481 bra $L__BB2_414; + + setp.ne.s32 %p482, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs668, %rs1165, 255; + setp.ne.s16 %p483, %rs668, 127; + and.pred %p484, %p482, %p483; + @%p484 bra $L__BB2_414; + + mov.u32 %r5046, 20548; + sub.s32 %r5047, %r5046, %r8972; + cvt.u64.u32 %rd304, %r5047; + add.s64 %rd305, %rd304, %rd5; + add.s64 %rd306, %rd1, %rd305; + st.global.u8 [%rd306], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p485, %rs668, 143; + selp.u32 %r8970, 1, 0, %p485; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_414: + setp.ne.s32 %p486, %r8961, 0; + mov.u32 %r8969, %r8960; + @%p486 bra $L__BB2_410; + bra.uni $L__BB2_415; + +$L__BB2_339: + setp.gt.s32 %p384, %r474, 0; + and.pred %p386, %p382, %p384; + @%p386 bra $L__BB2_368; + bra.uni $L__BB2_340; + +$L__BB2_368: + ld.global.u8 %rs108, [%rd11+1]; + ld.global.u8 %rs109, [%rd12]; + ld.global.u8 %rs110, [%rd12+1]; + setp.eq.s16 %p425, %rs108, 0; + mov.u32 %r8904, %r8688; + @%p425 bra $L__BB2_375; + + ld.global.u8 %r8894, [%rd11]; + cvt.u32.u16 %r8893, %rs108; + +$L__BB2_370: + mov.u32 %r849, %r8893; + setp.gt.u32 %p426, %r8972, 2879; + mov.u32 %r8904, 1; + @%p426 bra $L__BB2_375; + + mov.u32 %r4929, 8; + sub.s32 %r4930, %r4929, %r8970; + sub.s32 %r4931, %r4930, %r8971; + min.u32 %r4932, %r4931, %r849; + setp.eq.s32 %p427, %r4932, 32; + mov.u32 %r4933, -1; + shl.b32 %r4934, %r4933, %r4932; + not.b32 %r4935, %r4934; + selp.b32 %r4936, -1, %r4935, %p427; + and.b32 %r4937, %r4936, %r8894; + shl.b32 %r4938, %r4937, %r8971; + cvt.u16.u32 %rs639, %r4938; + or.b16 %rs1165, %rs1165, %rs639; + add.s32 %r8971, %r4932, %r8971; + sub.s32 %r8893, %r849, %r4932; + shr.u32 %r8894, %r8894, %r4932; + setp.gt.u32 %p428, %r4931, %r849; + @%p428 bra $L__BB2_374; + + setp.ne.s32 %p429, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs640, %rs1165, 255; + setp.ne.s16 %p430, %rs640, 127; + and.pred %p431, %p429, %p430; + @%p431 bra $L__BB2_374; + + mov.u32 %r4941, 20548; + sub.s32 %r4942, %r4941, %r8972; + cvt.u64.u32 %rd278, %r4942; + add.s64 %rd279, %rd278, %rd5; + add.s64 %rd280, %rd1, %rd279; + st.global.u8 [%rd280], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p432, %rs640, 143; + selp.u32 %r8970, 1, 0, %p432; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_374: + setp.ne.s32 %p433, %r8893, 0; + mov.u32 %r8904, %r8688; + @%p433 bra $L__BB2_370; + +$L__BB2_375: + add.s32 %r8906, %r474, -1; + cvt.u32.u16 %r4944, %rs110; + and.b32 %r8917, %r4944, 255; + cvt.u32.u16 %r4945, %rs109; + and.b32 %r8918, %r4945, 255; + mov.u32 %r4943, 1; + mov.u32 %r8905, %r4943; + +$L__BB2_376: + mov.u32 %r869, %r8905; + setp.gt.u32 %p434, %r8972, 2879; + mov.u32 %r8916, %r4943; + @%p434 bra $L__BB2_381; + + mov.u32 %r4947, 8; + sub.s32 %r4948, %r4947, %r8970; + sub.s32 %r4949, %r4948, %r8971; + min.u32 %r4950, %r4949, %r869; + setp.eq.s32 %p435, %r4950, 32; + mov.u32 %r4951, -1; + shl.b32 %r4952, %r4951, %r4950; + not.b32 %r4953, %r4952; + selp.b32 %r4954, -1, %r4953, %p435; + and.b32 %r4955, %r4954, %r8906; + shl.b32 %r4956, %r4955, %r8971; + cvt.u16.u32 %rs643, %r4956; + or.b16 %rs1165, %rs1165, %rs643; + add.s32 %r8971, %r4950, %r8971; + sub.s32 %r8905, %r869, %r4950; + shr.u32 %r8906, %r8906, %r4950; + setp.gt.u32 %p436, %r4949, %r869; + @%p436 bra $L__BB2_380; + + setp.ne.s32 %p437, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs644, %rs1165, 255; + setp.ne.s16 %p438, %rs644, 127; + and.pred %p439, %p437, %p438; + @%p439 bra $L__BB2_380; + + mov.u32 %r4959, 20548; + sub.s32 %r4960, %r4959, %r8972; + cvt.u64.u32 %rd281, %r4960; + add.s64 %rd282, %rd281, %rd5; + add.s64 %rd283, %rd1, %rd282; + st.global.u8 [%rd283], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p440, %rs644, 143; + selp.u32 %r8970, 1, 0, %p440; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_380: + setp.ne.s32 %p441, %r8905, 0; + mov.u32 %r8916, %r8904; + @%p441 bra $L__BB2_376; + +$L__BB2_381: + setp.eq.s16 %p442, %rs110, 0; + mov.u32 %r8969, %r8916; + @%p442 bra $L__BB2_415; + +$L__BB2_382: + mov.u32 %r886, %r8917; + setp.gt.u32 %p443, %r8972, 2879; + mov.u32 %r8969, 1; + @%p443 bra $L__BB2_415; + + mov.u32 %r4962, 8; + sub.s32 %r4963, %r4962, %r8970; + sub.s32 %r4964, %r4963, %r8971; + min.u32 %r4965, %r4964, %r886; + setp.eq.s32 %p444, %r4965, 32; + mov.u32 %r4966, -1; + shl.b32 %r4967, %r4966, %r4965; + not.b32 %r4968, %r4967; + selp.b32 %r4969, -1, %r4968, %p444; + and.b32 %r4970, %r4969, %r8918; + shl.b32 %r4971, %r4970, %r8971; + cvt.u16.u32 %rs648, %r4971; + or.b16 %rs1165, %rs1165, %rs648; + add.s32 %r8971, %r4965, %r8971; + sub.s32 %r8917, %r886, %r4965; + shr.u32 %r8918, %r8918, %r4965; + setp.gt.u32 %p445, %r4964, %r886; + @%p445 bra $L__BB2_386; + + setp.ne.s32 %p446, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs649, %rs1165, 255; + setp.ne.s16 %p447, %rs649, 127; + and.pred %p448, %p446, %p447; + @%p448 bra $L__BB2_386; + + mov.u32 %r4974, 20548; + sub.s32 %r4975, %r4974, %r8972; + cvt.u64.u32 %rd284, %r4975; + add.s64 %rd285, %rd284, %rd5; + add.s64 %rd286, %rd1, %rd285; + st.global.u8 [%rd286], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p449, %rs649, 143; + selp.u32 %r8970, 1, 0, %p449; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_386: + setp.eq.s32 %p450, %r8917, 0; + mov.u32 %r8969, %r8916; + @%p450 bra $L__BB2_415; + bra.uni $L__BB2_382; + +$L__BB2_340: + setp.gt.s32 %p388, %r133, 0; + selp.b32 %r4857, %r346, 0, %p388; + cvt.u64.u32 %rd258, %r4857; + add.s64 %rd15, %rd9, %rd258; + ld.global.u8 %rs86, [%rd15+1]; + add.s32 %r4858, %r4857, 2; + cvt.u64.u32 %rd260, %r4858; + add.s64 %rd261, %rd9, %rd260; + ld.global.u8 %rs87, [%rd261]; + ld.global.u8 %rs88, [%rd261+1]; + mul.lo.s32 %r4859, %r474, 6; + selp.b32 %r4860, %r4859, 0, %p384; + cvt.u64.u32 %rd262, %r4860; + add.s64 %rd263, %rd9, %rd262; + ld.global.u8 %rs89, [%rd263]; + ld.global.u8 %rs90, [%rd263+1]; + add.s32 %r4861, %r4860, 2; + cvt.u64.u32 %rd264, %r4861; + add.s64 %rd265, %rd9, %rd264; + ld.global.u8 %rs91, [%rd265]; + ld.global.u8 %rs92, [%rd265+1]; + setp.eq.s16 %p389, %rs86, 0; + mov.u32 %r8860, %r8688; + @%p389 bra $L__BB2_347; + + ld.global.u8 %r8850, [%rd15]; + cvt.u32.u16 %r8849, %rs86; + +$L__BB2_342: + mov.u32 %r777, %r8849; + setp.gt.u32 %p390, %r8972, 2879; + mov.u32 %r8860, 1; + @%p390 bra $L__BB2_347; + + mov.u32 %r4863, 8; + sub.s32 %r4864, %r4863, %r8970; + sub.s32 %r4865, %r4864, %r8971; + min.u32 %r4866, %r4865, %r777; + setp.eq.s32 %p391, %r4866, 32; + mov.u32 %r4867, -1; + shl.b32 %r4868, %r4867, %r4866; + not.b32 %r4869, %r4868; + selp.b32 %r4870, -1, %r4869, %p391; + and.b32 %r4871, %r4870, %r8850; + shl.b32 %r4872, %r4871, %r8971; + cvt.u16.u32 %rs620, %r4872; + or.b16 %rs1165, %rs1165, %rs620; + add.s32 %r8971, %r4866, %r8971; + sub.s32 %r8849, %r777, %r4866; + shr.u32 %r8850, %r8850, %r4866; + setp.gt.u32 %p392, %r4865, %r777; + @%p392 bra $L__BB2_346; + + setp.ne.s32 %p393, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs621, %rs1165, 255; + setp.ne.s16 %p394, %rs621, 127; + and.pred %p395, %p393, %p394; + @%p395 bra $L__BB2_346; + + mov.u32 %r4875, 20548; + sub.s32 %r4876, %r4875, %r8972; + cvt.u64.u32 %rd266, %r4876; + add.s64 %rd267, %rd266, %rd5; + add.s64 %rd268, %rd1, %rd267; + st.global.u8 [%rd268], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p396, %rs621, 143; + selp.u32 %r8970, 1, 0, %p396; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_346: + setp.ne.s32 %p397, %r8849, 0; + mov.u32 %r8860, %r8688; + @%p397 bra $L__BB2_342; + +$L__BB2_347: + setp.eq.s16 %p398, %rs90, 0; + mov.u32 %r8872, %r8860; + @%p398 bra $L__BB2_354; + + cvt.u32.u16 %r4877, %rs89; + and.b32 %r8862, %r4877, 255; + cvt.u32.u16 %r4878, %rs90; + and.b32 %r8861, %r4878, 255; + +$L__BB2_349: + mov.u32 %r796, %r8861; + setp.gt.u32 %p399, %r8972, 2879; + mov.u32 %r8872, 1; + @%p399 bra $L__BB2_354; + + mov.u32 %r4880, 8; + sub.s32 %r4881, %r4880, %r8970; + sub.s32 %r4882, %r4881, %r8971; + min.u32 %r4883, %r4882, %r796; + setp.eq.s32 %p400, %r4883, 32; + mov.u32 %r4884, -1; + shl.b32 %r4885, %r4884, %r4883; + not.b32 %r4886, %r4885; + selp.b32 %r4887, -1, %r4886, %p400; + and.b32 %r4888, %r4887, %r8862; + shl.b32 %r4889, %r4888, %r8971; + cvt.u16.u32 %rs625, %r4889; + or.b16 %rs1165, %rs1165, %rs625; + add.s32 %r8971, %r4883, %r8971; + sub.s32 %r8861, %r796, %r4883; + shr.u32 %r8862, %r8862, %r4883; + setp.gt.u32 %p401, %r4882, %r796; + @%p401 bra $L__BB2_353; + + setp.ne.s32 %p402, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs626, %rs1165, 255; + setp.ne.s16 %p403, %rs626, 127; + and.pred %p404, %p402, %p403; + @%p404 bra $L__BB2_353; + + mov.u32 %r4892, 20548; + sub.s32 %r4893, %r4892, %r8972; + cvt.u64.u32 %rd269, %r4893; + add.s64 %rd270, %rd269, %rd5; + add.s64 %rd271, %rd1, %rd270; + st.global.u8 [%rd271], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p405, %rs626, 143; + selp.u32 %r8970, 1, 0, %p405; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_353: + setp.ne.s32 %p406, %r8861, 0; + mov.u32 %r8872, %r8860; + @%p406 bra $L__BB2_349; + +$L__BB2_354: + setp.eq.s16 %p407, %rs88, 0; + mov.u32 %r8884, %r8872; + @%p407 bra $L__BB2_361; + + cvt.u32.u16 %r4894, %rs88; + and.b32 %r8873, %r4894, 255; + cvt.u32.u16 %r4895, %rs87; + and.b32 %r8874, %r4895, 255; + +$L__BB2_356: + mov.u32 %r815, %r8873; + setp.gt.u32 %p408, %r8972, 2879; + mov.u32 %r8884, 1; + @%p408 bra $L__BB2_361; + + mov.u32 %r4897, 8; + sub.s32 %r4898, %r4897, %r8970; + sub.s32 %r4899, %r4898, %r8971; + min.u32 %r4900, %r4899, %r815; + setp.eq.s32 %p409, %r4900, 32; + mov.u32 %r4901, -1; + shl.b32 %r4902, %r4901, %r4900; + not.b32 %r4903, %r4902; + selp.b32 %r4904, -1, %r4903, %p409; + and.b32 %r4905, %r4904, %r8874; + shl.b32 %r4906, %r4905, %r8971; + cvt.u16.u32 %rs630, %r4906; + or.b16 %rs1165, %rs1165, %rs630; + add.s32 %r8971, %r4900, %r8971; + sub.s32 %r8873, %r815, %r4900; + shr.u32 %r8874, %r8874, %r4900; + setp.gt.u32 %p410, %r4899, %r815; + @%p410 bra $L__BB2_360; + + setp.ne.s32 %p411, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs631, %rs1165, 255; + setp.ne.s16 %p412, %rs631, 127; + and.pred %p413, %p411, %p412; + @%p413 bra $L__BB2_360; + + mov.u32 %r4909, 20548; + sub.s32 %r4910, %r4909, %r8972; + cvt.u64.u32 %rd272, %r4910; + add.s64 %rd273, %rd272, %rd5; + add.s64 %rd274, %rd1, %rd273; + st.global.u8 [%rd274], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p414, %rs631, 143; + selp.u32 %r8970, 1, 0, %p414; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_360: + setp.ne.s32 %p415, %r8873, 0; + mov.u32 %r8884, %r8872; + @%p415 bra $L__BB2_356; + +$L__BB2_361: + setp.eq.s16 %p416, %rs92, 0; + mov.u32 %r8969, %r8884; + @%p416 bra $L__BB2_415; + + cvt.u32.u16 %r4911, %rs91; + and.b32 %r8886, %r4911, 255; + cvt.u32.u16 %r4912, %rs92; + and.b32 %r8885, %r4912, 255; + +$L__BB2_363: + mov.u32 %r834, %r8885; + setp.gt.u32 %p417, %r8972, 2879; + mov.u32 %r8969, 1; + @%p417 bra $L__BB2_415; + + mov.u32 %r4914, 8; + sub.s32 %r4915, %r4914, %r8970; + sub.s32 %r4916, %r4915, %r8971; + min.u32 %r4917, %r4916, %r834; + setp.eq.s32 %p418, %r4917, 32; + mov.u32 %r4918, -1; + shl.b32 %r4919, %r4918, %r4917; + not.b32 %r4920, %r4919; + selp.b32 %r4921, -1, %r4920, %p418; + and.b32 %r4922, %r4921, %r8886; + shl.b32 %r4923, %r4922, %r8971; + cvt.u16.u32 %rs635, %r4923; + or.b16 %rs1165, %rs1165, %rs635; + add.s32 %r8971, %r4917, %r8971; + sub.s32 %r8885, %r834, %r4917; + shr.u32 %r8886, %r8886, %r4917; + setp.gt.u32 %p419, %r4916, %r834; + @%p419 bra $L__BB2_367; + + setp.ne.s32 %p420, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs636, %rs1165, 255; + setp.ne.s16 %p421, %rs636, 127; + and.pred %p422, %p420, %p421; + @%p422 bra $L__BB2_367; + + mov.u32 %r4926, 20548; + sub.s32 %r4927, %r4926, %r8972; + cvt.u64.u32 %rd275, %r4927; + add.s64 %rd276, %rd275, %rd5; + add.s64 %rd277, %rd1, %rd276; + st.global.u8 [%rd277], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p423, %rs636, 143; + selp.u32 %r8970, 1, 0, %p423; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_367: + setp.eq.s32 %p424, %r8885, 0; + mov.u32 %r8969, %r8884; + @%p424 bra $L__BB2_415; + bra.uni $L__BB2_363; + +$L__BB2_415: + shr.u32 %r5048, %r8658, 1; + or.b32 %r8441, %r5048, %r591; + +$L__BB2_416: + add.s32 %r8439, %r8439, 4; + setp.lt.u32 %p487, %r8439, %r4057; + @%p487 bra $L__BB2_51; + +$L__BB2_417: + add.s32 %r8414, %r4057, 1; + shr.u32 %r8413, %r8414, 1; + add.s32 %r5049, %r8413, 1; + setp.gt.u32 %p488, %r5049, 512; + @%p488 bra $L__BB2_419; + + add.s32 %r8406, %r4057, 1; + shr.u32 %r8405, %r8406, 1; + add.s32 %r8404, %r4103, %r8405; + mov.u16 %rs671, 0; + add.s32 %r8402, %r8404, 1; + st.shared.u8 [%r8402], %rs671; + +$L__BB2_419: + setp.lt.u32 %p489, %r4058, 3; + @%p489 bra $L__BB2_665; + + ld.param.u64 %rd1416, [ j2k_htj2k_encode_codeblocks_multi_input_param_4]; + ld.param.u64 %rd1411, [ j2k_htj2k_encode_codeblocks_multi_input_param_3]; + mov.u32 %r5051, 31; + sub.s32 %r1009, %r5051, %r4059; + mov.u32 %r9005, 2; + cvta.to.global.u64 %rd17, %rd1416; + cvta.to.global.u64 %rd18, %rd1411; + +$L__BB2_421: + ld.shared.u8 %rs151, [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE13cleanup_e_val]; + mov.u16 %rs672, 0; + st.shared.u8 [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE13cleanup_e_val], %rs672; + ld.shared.u8 %rs152, [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val]; + st.shared.u8 [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val], %rs672; + @%p10 bra $L__BB2_664; + + mov.u32 %r5054, 0; + ld.shared.u8 %rs673, [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE13cleanup_e_val+1]; + ld.shared.u8 %rs674, [_ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val+1]; + max.u16 %rs676, %rs151, %rs673; + cvt.u32.u16 %r5055, %rs676; + add.s32 %r9023, %r5055, -1; + add.s32 %r1027, %r9005, 1; + mul.lo.s32 %r9025, %r9005, %r4055; + mul.wide.u16 %r5056, %rs674, 4; + cvt.u32.u16 %r5057, %rs152; + and.b32 %r5058, %r5057, 255; + add.s32 %r9026, %r5056, %r5058; + mov.u32 %r9021, %r5054; + mov.u32 %r9022, %r5054; + mov.u32 %r9024, %r5054; + +$L__BB2_423: + cvt.u64.u32 %rd307, %r9025; + add.s64 %rd308, %rd307, %rd4; + shl.b64 %rd309, %rd308, 2; + add.s64 %rd310, %rd3, %rd309; + ld.global.u32 %r1051, [%rd310]; + setp.eq.s32 %p491, %r1051, 0; + mov.u32 %r9042, %r5054; + @%p491 bra $L__BB2_425; + + and.b32 %r5060, %r1051, -2147483648; + abs.s32 %r5061, %r1051; + shl.b32 %r5062, %r5061, %r1009; + or.b32 %r9042, %r5062, %r5060; + +$L__BB2_425: + shl.b32 %r5066, %r9042, 1; + shr.u32 %r5067, %r5066, %r46; + and.b32 %r1054, %r5067, -2; + setp.eq.s32 %p492, %r1054, 0; + mov.u32 %r9046, 0; + mov.u32 %r9043, %r9046; + mov.u32 %r9044, %r9046; + mov.u32 %r9050, %r9046; + @%p492 bra $L__BB2_427; + + add.s32 %r5069, %r1054, -1; + clz.b32 %r5070, %r5069; + mov.u32 %r5071, 32; + sub.s32 %r9043, %r5071, %r5070; + shr.u32 %r5072, %r9042, 31; + add.s32 %r5073, %r5072, %r1054; + add.s32 %r9044, %r5073, -2; + mov.u32 %r9050, 1; + +$L__BB2_427: + setp.ge.u32 %p493, %r1027, %r4058; + @%p493 bra $L__BB2_430; + + add.s32 %r5076, %r9025, %r4055; + cvt.u64.u32 %rd311, %r5076; + add.s64 %rd312, %rd311, %rd4; + shl.b64 %rd313, %rd312, 2; + add.s64 %rd314, %rd3, %rd313; + ld.global.u32 %r1060, [%rd314]; + setp.eq.s32 %p494, %r1060, 0; + @%p494 bra $L__BB2_430; + + and.b32 %r5077, %r1060, -2147483648; + abs.s32 %r5078, %r1060; + shl.b32 %r5079, %r5078, %r1009; + or.b32 %r9046, %r5079, %r5077; + +$L__BB2_430: + shl.b32 %r5082, %r9046, 1; + shr.u32 %r5083, %r5082, %r46; + and.b32 %r1063, %r5083, -2; + setp.eq.s32 %p495, %r1063, 0; + mov.u32 %r9061, 0; + mov.u32 %r9047, %r9061; + mov.u32 %r9048, %r9061; + mov.u32 %r9065, %r9043; + @%p495 bra $L__BB2_432; + + or.b32 %r9050, %r9050, 2; + add.s32 %r5084, %r1063, -1; + clz.b32 %r5085, %r5084; + mov.u32 %r5086, 32; + sub.s32 %r9047, %r5086, %r5085; + max.s32 %r9065, %r9043, %r9047; + shr.u32 %r5087, %r9046, 31; + add.s32 %r5088, %r5087, %r1063; + add.s32 %r9048, %r5088, -2; + +$L__BB2_432: + add.s32 %r9355, %r9025, 1; + add.s32 %r5093, %r9021, 1; + setp.ge.u32 %p496, %r5093, %r4057; + mov.u32 %r9062, %r9061; + mov.u32 %r9063, %r9061; + mov.u32 %r9064, %r9061; + @%p496 bra $L__BB2_443; + + cvt.u64.u32 %rd315, %r9355; + add.s64 %rd316, %rd315, %rd4; + shl.b64 %rd317, %rd316, 2; + add.s64 %rd318, %rd3, %rd317; + ld.global.u32 %r1073, [%rd318]; + setp.eq.s32 %p497, %r1073, 0; + mov.u32 %r9062, 0; + mov.u32 %r9051, %r9062; + @%p497 bra $L__BB2_435; + + and.b32 %r5095, %r1073, -2147483648; + abs.s32 %r5096, %r1073; + shl.b32 %r5097, %r5096, %r1009; + or.b32 %r9051, %r5097, %r5095; + +$L__BB2_435: + shl.b32 %r5100, %r9051, 1; + shr.u32 %r5101, %r5100, %r46; + and.b32 %r1076, %r5101, -2; + setp.eq.s32 %p498, %r1076, 0; + mov.u32 %r9064, %r9062; + @%p498 bra $L__BB2_437; + + or.b32 %r9050, %r9050, 4; + add.s32 %r5102, %r1076, -1; + clz.b32 %r5103, %r5102; + mov.u32 %r5104, 32; + sub.s32 %r9062, %r5104, %r5103; + max.s32 %r9065, %r9065, %r9062; + shr.u32 %r5105, %r9051, 31; + add.s32 %r5106, %r5105, %r1076; + add.s32 %r9064, %r5106, -2; + +$L__BB2_437: + mov.u32 %r9061, 0; + mov.u32 %r9056, %r9061; + @%p493 bra $L__BB2_440; + + add.s32 %r5109, %r9355, %r4055; + cvt.u64.u32 %rd319, %r5109; + add.s64 %rd320, %rd319, %rd4; + shl.b64 %rd321, %rd320, 2; + add.s64 %rd322, %rd3, %rd321; + ld.global.u32 %r1085, [%rd322]; + setp.eq.s32 %p500, %r1085, 0; + @%p500 bra $L__BB2_440; + + and.b32 %r5110, %r1085, -2147483648; + abs.s32 %r5111, %r1085; + shl.b32 %r5112, %r5111, %r1009; + or.b32 %r9056, %r5112, %r5110; + +$L__BB2_440: + shl.b32 %r5115, %r9056, 1; + shr.u32 %r5116, %r5115, %r46; + and.b32 %r1088, %r5116, -2; + setp.eq.s32 %p501, %r1088, 0; + mov.u32 %r9063, %r9061; + @%p501 bra $L__BB2_442; + + or.b32 %r9050, %r9050, 8; + add.s32 %r5117, %r1088, -1; + clz.b32 %r5118, %r5117; + mov.u32 %r5119, 32; + sub.s32 %r9061, %r5119, %r5118; + max.s32 %r9065, %r9065, %r9061; + shr.u32 %r5120, %r9056, 31; + add.s32 %r5121, %r5120, %r1088; + add.s32 %r9063, %r5121, -2; + +$L__BB2_442: + add.s32 %r9355, %r9025, 2; + +$L__BB2_443: + add.s32 %r5123, %r9050, -1; + and.b32 %r5124, %r5123, %r9050; + setp.ne.s32 %p502, %r5124, 0; + mov.u32 %r9068, 0; + setp.gt.s32 %p503, %r9023, 1; + and.pred %p504, %p503, %p502; + selp.b32 %r5125, %r9023, 1, %p504; + max.s32 %r1105, %r5125, %r9065; + sub.s32 %r1106, %r1105, %r5125; + setp.lt.s32 %p505, %r1106, 1; + @%p505 bra $L__BB2_445; + + setp.eq.s32 %p506, %r9043, %r9065; + selp.u32 %r5126, 1, 0, %p506; + setp.eq.s32 %p507, %r9047, %r9065; + selp.u32 %r5127, -1, 0, %p507; + bfi.b32 %r5128, %r5127, %r5126, 1, 1; + setp.eq.s32 %p508, %r9062, %r9065; + selp.u16 %rs677, 1, 0, %p508; + mul.wide.u16 %r5129, %rs677, 4; + or.b32 %r5130, %r5128, %r5129; + setp.eq.s32 %p509, %r9061, %r9065; + selp.u16 %rs678, 1, 0, %p509; + mul.wide.u16 %r5131, %rs678, 8; + or.b32 %r9068, %r5130, %r5131; + +$L__BB2_445: + shl.b32 %r5132, %r9050, 4; + shl.b32 %r5133, %r9026, 8; + or.b32 %r5134, %r5132, %r5133; + or.b32 %r5135, %r5134, %r9068; + mul.wide.u32 %rd323, %r5135, 2; + add.s64 %rd324, %rd18, %rd323; + ld.global.u16 %rs155, [%rd324]; + shr.u16 %rs679, %rs155, 4; + and.b16 %rs156, %rs679, 7; + setp.eq.s16 %p510, %rs156, 0; + mov.u32 %r9080, %r8969; + @%p510 bra $L__BB2_452; + + cvt.u32.u16 %r9069, %rs156; + shr.u16 %rs680, %rs155, 8; + cvt.u32.u16 %r9070, %rs680; + +$L__BB2_447: + mov.u32 %r1111, %r9069; + setp.gt.u32 %p511, %r8972, 2879; + mov.u32 %r9080, 1; + @%p511 bra $L__BB2_452; + + mov.u32 %r5137, 8; + sub.s32 %r5138, %r5137, %r8970; + sub.s32 %r5139, %r5138, %r8971; + min.u32 %r5140, %r5139, %r1111; + setp.eq.s32 %p512, %r5140, 32; + mov.u32 %r5141, -1; + shl.b32 %r5142, %r5141, %r5140; + not.b32 %r5143, %r5142; + selp.b32 %r5144, -1, %r5143, %p512; + and.b32 %r5145, %r5144, %r9070; + shl.b32 %r5146, %r5145, %r8971; + cvt.u16.u32 %rs681, %r5146; + or.b16 %rs1165, %rs1165, %rs681; + add.s32 %r8971, %r5140, %r8971; + sub.s32 %r9069, %r1111, %r5140; + shr.u32 %r9070, %r9070, %r5140; + setp.gt.u32 %p513, %r5139, %r1111; + @%p513 bra $L__BB2_451; + + setp.ne.s32 %p514, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs682, %rs1165, 255; + setp.ne.s16 %p515, %rs682, 127; + and.pred %p516, %p514, %p515; + @%p516 bra $L__BB2_451; + + mov.u32 %r5149, 20548; + sub.s32 %r5150, %r5149, %r8972; + cvt.u64.u32 %rd325, %r5150; + add.s64 %rd326, %rd325, %rd5; + add.s64 %rd327, %rd1, %rd326; + st.global.u8 [%rd327], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p517, %rs682, 143; + selp.u32 %r8970, 1, 0, %p517; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_451: + setp.ne.s32 %p518, %r9069, 0; + mov.u32 %r9080, %r8969; + @%p518 bra $L__BB2_447; + +$L__BB2_452: + setp.ne.s32 %p519, %r9026, 0; + @%p519 bra $L__BB2_500; + + setp.eq.s32 %p520, %r9050, 0; + add.s32 %r5151, %r8524, 17477; + cvt.u64.u32 %rd328, %r5151; + add.s64 %rd329, %rd328, %rd5; + add.s64 %rd19, %rd1, %rd329; + @%p520 bra $L__BB2_492; + + shl.b16 %rs1096, %rs1096, 1; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p521, %r8530, 0; + mov.u32 %r9114, %r8733; + @%p521 bra $L__BB2_457; + + setp.gt.u32 %p522, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9114, 1; + @%p522 bra $L__BB2_457; + + st.global.u8 [%rd19], %rs1096; + add.s32 %r8524, %r8524, 1; + mov.u32 %r8530, 8; + mov.u16 %rs1096, 0; + mov.u32 %r9114, %r8733; + +$L__BB2_457: + setp.lt.u32 %p523, %r8735, 3; + mov.u32 %r9084, 0; + @%p523 bra $L__BB2_460; + + setp.lt.u32 %p524, %r8735, 6; + mov.u32 %r9084, 1; + @%p524 bra $L__BB2_460; + + setp.lt.u32 %p525, %r8735, 9; + setp.eq.s32 %p526, %r8735, 11; + selp.b32 %r5157, 4, 5, %p526; + setp.lt.u32 %p527, %r8735, 11; + selp.b32 %r5158, 3, %r5157, %p527; + selp.b32 %r9084, 2, %r5158, %p525; + +$L__BB2_460: + setp.eq.s32 %p528, %r9084, 0; + @%p528 bra $L__BB2_488; + + add.s32 %r1135, %r9084, -1; + and.b32 %r1136, %r9084, 3; + setp.eq.s32 %p529, %r1136, 0; + mov.u32 %r9094, %r9084; + mov.u32 %r9097, %r9114; + @%p529 bra $L__BB2_473; + + mov.u32 %r5160, 1; + shl.b32 %r5161, %r5160, %r1135; + and.b32 %r5162, %r5161, %r8736; + setp.ne.s32 %p530, %r5162, 0; + selp.u32 %r5163, 1, 0, %p530; + cvt.u32.u16 %r5164, %rs1096; + bfi.b32 %r5165, %r5164, %r5163, 1, 8; + cvt.u16.u32 %rs1096, %r5165; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p531, %r8530, 0; + mov.u32 %r9097, %r9114; + @%p531 bra $L__BB2_465; + + setp.gt.u32 %p532, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9097, %r5160; + @%p532 bra $L__BB2_465; + + add.s32 %r5169, %r8524, 17477; + cvt.u64.u32 %rd330, %r5169; + add.s64 %rd331, %rd330, %rd5; + add.s64 %rd332, %rd1, %rd331; + st.global.u8 [%rd332], %rs1096; + add.s32 %r8524, %r8524, 1; + mov.u32 %r8530, 8; + mov.u16 %rs1096, 0; + mov.u32 %r9097, %r9114; + +$L__BB2_465: + setp.eq.s32 %p533, %r1136, 1; + mov.u32 %r9114, %r9097; + mov.u32 %r9094, %r1135; + @%p533 bra $L__BB2_473; + + add.s32 %r9094, %r9084, -2; + mov.u32 %r5170, 1; + shl.b32 %r5171, %r5170, %r9094; + and.b32 %r5172, %r5171, %r8736; + setp.ne.s32 %p534, %r5172, 0; + selp.u32 %r5173, 1, 0, %p534; + cvt.u32.u16 %r5174, %rs1096; + bfi.b32 %r5175, %r5174, %r5173, 1, 8; + cvt.u16.u32 %rs1096, %r5175; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p535, %r8530, 0; + mov.u32 %r9088, %r9097; + @%p535 bra $L__BB2_469; + + setp.gt.u32 %p536, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9088, %r5170; + @%p536 bra $L__BB2_469; + + add.s32 %r5178, %r8524, 17477; + cvt.u64.u32 %rd333, %r5178; + add.s64 %rd334, %rd333, %rd5; + add.s64 %rd335, %rd1, %rd334; + and.b16 %rs689, %rs1096, 255; + st.global.u8 [%rd335], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p537, %rs689, 255; + selp.b32 %r8530, 7, 8, %p537; + mov.u16 %rs1096, 0; + mov.u32 %r9088, %r9097; + +$L__BB2_469: + setp.eq.s32 %p538, %r1136, 2; + mov.u32 %r9114, %r9088; + mov.u32 %r9097, %r9088; + @%p538 bra $L__BB2_473; + + add.s32 %r9094, %r9084, -3; + mov.u32 %r5179, 1; + shl.b32 %r5180, %r5179, %r9094; + and.b32 %r5181, %r5180, %r8736; + setp.ne.s32 %p539, %r5181, 0; + selp.u32 %r5182, 1, 0, %p539; + cvt.u32.u16 %r5183, %rs1096; + bfi.b32 %r5184, %r5183, %r5182, 1, 8; + cvt.u16.u32 %rs1096, %r5184; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p540, %r8530, 0; + mov.u32 %r9114, %r9088; + mov.u32 %r9097, %r9088; + @%p540 bra $L__BB2_473; + + setp.gt.u32 %p541, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9114, %r5179; + mov.u32 %r9097, %r5179; + @%p541 bra $L__BB2_473; + + add.s32 %r5189, %r8524, 17477; + cvt.u64.u32 %rd336, %r5189; + add.s64 %rd337, %rd336, %rd5; + add.s64 %rd338, %rd1, %rd337; + and.b16 %rs692, %rs1096, 255; + st.global.u8 [%rd338], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p542, %rs692, 255; + selp.b32 %r8530, 7, 8, %p542; + mov.u16 %rs1096, 0; + mov.u32 %r9114, %r9088; + mov.u32 %r9097, %r9088; + +$L__BB2_473: + setp.lt.u32 %p543, %r1135, 3; + @%p543 bra $L__BB2_488; + + mov.u32 %r9114, %r9097; + +$L__BB2_475: + add.s32 %r5190, %r9094, -1; + mov.u32 %r5191, 1; + shl.b32 %r5192, %r5191, %r5190; + and.b32 %r5193, %r5192, %r8736; + setp.ne.s32 %p544, %r5193, 0; + selp.u32 %r5194, 1, 0, %p544; + cvt.u32.u16 %r5195, %rs1096; + bfi.b32 %r9103, %r5195, %r5194, 1, 8; + add.s32 %r9104, %r8530, -1; + setp.ne.s32 %p545, %r9104, 0; + mov.u32 %r9102, %r9114; + @%p545 bra $L__BB2_478; + + setp.gt.u32 %p546, %r8524, 191; + mov.u32 %r9104, 0; + mov.u32 %r9102, %r5191; + @%p546 bra $L__BB2_478; + + cvt.u16.u32 %rs693, %r9103; + and.b16 %rs694, %rs693, 255; + add.s32 %r5199, %r8524, 17477; + cvt.u64.u32 %rd339, %r5199; + add.s64 %rd340, %rd339, %rd5; + add.s64 %rd341, %rd1, %rd340; + st.global.u8 [%rd341], %rs693; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p547, %rs694, 255; + selp.b32 %r9104, 7, 8, %p547; + mov.u32 %r9103, 0; + mov.u32 %r9102, %r9114; + +$L__BB2_478: + add.s32 %r5200, %r9094, -2; + shl.b32 %r5202, %r5191, %r5200; + and.b32 %r5203, %r5202, %r8736; + setp.ne.s32 %p548, %r5203, 0; + and.b32 %r5204, %r9103, 127; + selp.u32 %r5205, 1, 0, %p548; + bfi.b32 %r9107, %r5204, %r5205, 1, 7; + add.s32 %r9108, %r9104, -1; + setp.ne.s32 %p549, %r9108, 0; + mov.u32 %r9106, %r9102; + @%p549 bra $L__BB2_481; + + setp.gt.u32 %p550, %r8524, 191; + mov.u32 %r9108, 0; + mov.u32 %r9106, 1; + @%p550 bra $L__BB2_481; + + cvt.u16.u32 %rs695, %r9107; + and.b16 %rs696, %rs695, 255; + add.s32 %r5209, %r8524, 17477; + cvt.u64.u32 %rd342, %r5209; + add.s64 %rd343, %rd342, %rd5; + add.s64 %rd344, %rd1, %rd343; + st.global.u8 [%rd344], %rs695; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p551, %rs696, 255; + selp.b32 %r9108, 7, 8, %p551; + mov.u32 %r9107, 0; + mov.u32 %r9106, %r9102; + +$L__BB2_481: + add.s32 %r5210, %r9094, -3; + mov.u32 %r5211, 1; + shl.b32 %r5212, %r5211, %r5210; + and.b32 %r5213, %r5212, %r8736; + setp.ne.s32 %p552, %r5213, 0; + and.b32 %r5214, %r9107, 127; + selp.u32 %r5215, 1, 0, %p552; + bfi.b32 %r9111, %r5214, %r5215, 1, 7; + add.s32 %r9112, %r9108, -1; + setp.ne.s32 %p553, %r9112, 0; + mov.u32 %r9110, %r9106; + @%p553 bra $L__BB2_484; + + setp.gt.u32 %p554, %r8524, 191; + mov.u32 %r9112, 0; + mov.u32 %r9110, %r5211; + @%p554 bra $L__BB2_484; + + cvt.u16.u32 %rs697, %r9111; + and.b16 %rs698, %rs697, 255; + add.s32 %r5219, %r8524, 17477; + cvt.u64.u32 %rd345, %r5219; + add.s64 %rd346, %rd345, %rd5; + add.s64 %rd347, %rd1, %rd346; + st.global.u8 [%rd347], %rs697; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p555, %rs698, 255; + selp.b32 %r9112, 7, 8, %p555; + mov.u32 %r9111, 0; + mov.u32 %r9110, %r9106; + +$L__BB2_484: + add.s32 %r9094, %r9094, -4; + shl.b32 %r5221, %r5211, %r9094; + and.b32 %r5222, %r5221, %r8736; + setp.ne.s32 %p556, %r5222, 0; + and.b32 %r5223, %r9111, 127; + selp.u32 %r5224, 1, 0, %p556; + bfi.b32 %r5225, %r5223, %r5224, 1, 15; + cvt.u16.u32 %rs1096, %r5225; + add.s32 %r8530, %r9112, -1; + setp.ne.s32 %p557, %r8530, 0; + mov.u32 %r9114, %r9110; + @%p557 bra $L__BB2_487; + + setp.gt.u32 %p558, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9114, 1; + @%p558 bra $L__BB2_487; + + add.s32 %r5228, %r8524, 17477; + cvt.u64.u32 %rd348, %r5228; + add.s64 %rd349, %rd348, %rd5; + add.s64 %rd350, %rd1, %rd349; + and.b16 %rs700, %rs1096, 255; + st.global.u8 [%rd350], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p559, %rs700, 255; + selp.b32 %r8530, 7, 8, %p559; + mov.u16 %rs1096, 0; + mov.u32 %r9114, %r9110; + +$L__BB2_487: + setp.ne.s32 %p560, %r9094, 0; + @%p560 bra $L__BB2_475; + +$L__BB2_488: + add.s32 %r5230, %r8735, -1; + setp.eq.s32 %p561, %r8735, 0; + mov.u32 %r8736, 0; + selp.b32 %r8735, 0, %r5230, %p561; + setp.lt.u32 %p562, %r8735, 3; + mov.u32 %r9120, %r8736; + @%p562 bra $L__BB2_491; + + setp.lt.u32 %p563, %r8735, 6; + mov.u32 %r9120, 1; + @%p563 bra $L__BB2_491; + + setp.lt.u32 %p564, %r8735, 9; + setp.eq.s32 %p565, %r8735, 11; + selp.b32 %r5232, 4, 5, %p565; + setp.lt.u32 %p566, %r8735, 11; + selp.b32 %r5233, 3, %r5232, %p566; + selp.b32 %r9120, 2, %r5233, %p564; + +$L__BB2_491: + mov.u32 %r5235, 1; + shl.b32 %r8734, %r5235, %r9120; + mov.u32 %r8733, %r9114; + bra.uni $L__BB2_500; + +$L__BB2_492: + add.s32 %r8736, %r8736, 1; + setp.lt.u32 %p567, %r8736, %r8734; + @%p567 bra $L__BB2_500; + + shl.b16 %rs701, %rs1096, 1; + or.b16 %rs1096, %rs701, 1; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p568, %r8530, 0; + mov.u32 %r9121, %r8733; + @%p568 bra $L__BB2_496; + + setp.gt.u32 %p569, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9121, 1; + @%p569 bra $L__BB2_496; + + and.b16 %rs703, %rs1096, 255; + st.global.u8 [%rd19], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p570, %rs703, 255; + selp.b32 %r8530, 7, 8, %p570; + mov.u16 %rs1096, 0; + mov.u32 %r9121, %r8733; + +$L__BB2_496: + add.s32 %r5239, %r8735, 1; + min.u32 %r8735, %r5239, 12; + setp.lt.u32 %p571, %r8735, 3; + mov.u32 %r8736, 0; + mov.u32 %r9124, %r8736; + @%p571 bra $L__BB2_499; + + setp.lt.u32 %p572, %r8735, 6; + mov.u32 %r9124, 1; + @%p572 bra $L__BB2_499; + + setp.lt.u32 %p573, %r8735, 9; + setp.eq.s32 %p574, %r8735, 11; + selp.b32 %r5241, 4, 5, %p574; + setp.lt.u32 %p575, %r8735, 11; + selp.b32 %r5242, 3, %r5241, %p575; + selp.b32 %r9124, 2, %r5242, %p573; + +$L__BB2_499: + mov.u32 %r5244, 1; + shl.b32 %r8734, %r5244, %r9124; + mov.u32 %r8733, %r9121; + +$L__BB2_500: + and.b16 %rs704, %rs155, 15; + cvt.u32.u16 %r1219, %rs704; + and.b32 %r5245, %r9050, 1; + setp.eq.b32 %p576, %r5245, 1; + mov.pred %p577, 0; + xor.pred %p578, %p576, %p577; + not.pred %p579, %p578; + mov.u32 %r9141, %r9186; + @%p579 bra $L__BB2_507; + + and.b32 %r5246, %r1219, 1; + sub.s32 %r9131, %r1105, %r5246; + setp.eq.s32 %p580, %r9131, 0; + mov.u32 %r9141, %r9186; + @%p580 bra $L__BB2_507; + + mov.u32 %r5247, -1; + shl.b32 %r5248, %r5247, %r9131; + not.b32 %r5249, %r5248; + and.b32 %r9132, %r9044, %r5249; + +$L__BB2_503: + setp.gt.u32 %p581, %r9160, 17476; + mov.u32 %r9141, 1; + @%p581 bra $L__BB2_507; + + sub.s32 %r5251, %r9159, %r9158; + min.u32 %r5252, %r5251, %r9131; + setp.eq.s32 %p582, %r5252, 32; + mov.u32 %r5253, -1; + shl.b32 %r5254, %r5253, %r5252; + not.b32 %r5255, %r5254; + selp.b32 %r5256, -1, %r5255, %p582; + and.b32 %r5257, %r5256, %r9132; + shl.b32 %r5258, %r5257, %r9158; + or.b32 %r9157, %r5258, %r9157; + add.s32 %r9158, %r5252, %r9158; + shr.u32 %r9132, %r9132, %r5252; + sub.s32 %r9131, %r9131, %r5252; + setp.lt.u32 %p583, %r9158, %r9159; + @%p583 bra $L__BB2_506; + + cvt.u64.u32 %rd351, %r9160; + add.s64 %rd352, %rd351, %rd5; + add.s64 %rd353, %rd1, %rd352; + st.global.u8 [%rd353], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p584, %r9157, 255; + selp.b32 %r9159, 7, 8, %p584; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_506: + setp.ne.s32 %p585, %r9131, 0; + mov.u32 %r9141, %r9186; + @%p585 bra $L__BB2_503; + +$L__BB2_507: + and.b32 %r1243, %r9050, 2; + setp.eq.s32 %p586, %r1243, 0; + mov.u32 %r9156, %r9141; + @%p586 bra $L__BB2_514; + + shr.u32 %r5261, %r1219, 1; + and.b32 %r5262, %r5261, 1; + sub.s32 %r9146, %r1105, %r5262; + setp.eq.s32 %p587, %r9146, 0; + mov.u32 %r9156, %r9141; + @%p587 bra $L__BB2_514; + + mov.u32 %r5263, -1; + shl.b32 %r5264, %r5263, %r9146; + not.b32 %r5265, %r5264; + and.b32 %r9147, %r9048, %r5265; + +$L__BB2_510: + setp.gt.u32 %p588, %r9160, 17476; + mov.u32 %r9156, 1; + @%p588 bra $L__BB2_514; + + sub.s32 %r5267, %r9159, %r9158; + min.u32 %r5268, %r5267, %r9146; + setp.eq.s32 %p589, %r5268, 32; + mov.u32 %r5269, -1; + shl.b32 %r5270, %r5269, %r5268; + not.b32 %r5271, %r5270; + selp.b32 %r5272, -1, %r5271, %p589; + and.b32 %r5273, %r5272, %r9147; + shl.b32 %r5274, %r5273, %r9158; + or.b32 %r9157, %r5274, %r9157; + add.s32 %r9158, %r5268, %r9158; + shr.u32 %r9147, %r9147, %r5268; + sub.s32 %r9146, %r9146, %r5268; + setp.lt.u32 %p590, %r9158, %r9159; + @%p590 bra $L__BB2_513; + + cvt.u64.u32 %rd354, %r9160; + add.s64 %rd355, %rd354, %rd5; + add.s64 %rd356, %rd1, %rd355; + st.global.u8 [%rd356], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p591, %r9157, 255; + selp.b32 %r9159, 7, 8, %p591; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_513: + setp.ne.s32 %p592, %r9146, 0; + mov.u32 %r9156, %r9141; + @%p592 bra $L__BB2_510; + +$L__BB2_514: + and.b32 %r1267, %r9050, 4; + setp.eq.s32 %p593, %r1267, 0; + mov.u32 %r9171, %r9156; + @%p593 bra $L__BB2_521; + + shr.u32 %r5277, %r1219, 2; + and.b32 %r5278, %r5277, 1; + sub.s32 %r9161, %r1105, %r5278; + setp.eq.s32 %p594, %r9161, 0; + mov.u32 %r9171, %r9156; + @%p594 bra $L__BB2_521; + + mov.u32 %r5279, -1; + shl.b32 %r5280, %r5279, %r9161; + not.b32 %r5281, %r5280; + and.b32 %r9162, %r9064, %r5281; + +$L__BB2_517: + setp.gt.u32 %p595, %r9160, 17476; + mov.u32 %r9171, 1; + @%p595 bra $L__BB2_521; + + sub.s32 %r5283, %r9159, %r9158; + min.u32 %r5284, %r5283, %r9161; + setp.eq.s32 %p596, %r5284, 32; + mov.u32 %r5285, -1; + shl.b32 %r5286, %r5285, %r5284; + not.b32 %r5287, %r5286; + selp.b32 %r5288, -1, %r5287, %p596; + and.b32 %r5289, %r5288, %r9162; + shl.b32 %r5290, %r5289, %r9158; + or.b32 %r9157, %r5290, %r9157; + add.s32 %r9158, %r5284, %r9158; + shr.u32 %r9162, %r9162, %r5284; + sub.s32 %r9161, %r9161, %r5284; + setp.lt.u32 %p597, %r9158, %r9159; + @%p597 bra $L__BB2_520; + + cvt.u64.u32 %rd357, %r9160; + add.s64 %rd358, %rd357, %rd5; + add.s64 %rd359, %rd1, %rd358; + st.global.u8 [%rd359], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p598, %r9157, 255; + selp.b32 %r9159, 7, 8, %p598; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_520: + setp.ne.s32 %p599, %r9161, 0; + mov.u32 %r9171, %r9156; + @%p599 bra $L__BB2_517; + +$L__BB2_521: + and.b32 %r1291, %r9050, 8; + setp.eq.s32 %p600, %r1291, 0; + mov.u32 %r9186, %r9171; + @%p600 bra $L__BB2_528; + + shr.u32 %r5293, %r1219, 3; + sub.s32 %r9176, %r1105, %r5293; + setp.eq.s32 %p601, %r9176, 0; + mov.u32 %r9186, %r9171; + @%p601 bra $L__BB2_528; + + mov.u32 %r5294, -1; + shl.b32 %r5295, %r5294, %r9176; + not.b32 %r5296, %r5295; + and.b32 %r9177, %r9063, %r5296; + +$L__BB2_524: + setp.gt.u32 %p602, %r9160, 17476; + mov.u32 %r9186, 1; + @%p602 bra $L__BB2_528; + + sub.s32 %r5298, %r9159, %r9158; + min.u32 %r5299, %r5298, %r9176; + setp.eq.s32 %p603, %r5299, 32; + mov.u32 %r5300, -1; + shl.b32 %r5301, %r5300, %r5299; + not.b32 %r5302, %r5301; + selp.b32 %r5303, -1, %r5302, %p603; + and.b32 %r5304, %r5303, %r9177; + shl.b32 %r5305, %r5304, %r9158; + or.b32 %r9157, %r5305, %r9157; + add.s32 %r9158, %r5299, %r9158; + shr.u32 %r9177, %r9177, %r5299; + sub.s32 %r9176, %r9176, %r5299; + setp.lt.u32 %p604, %r9158, %r9159; + @%p604 bra $L__BB2_527; + + cvt.u64.u32 %rd360, %r9160; + add.s64 %rd361, %rd360, %rd5; + add.s64 %rd362, %rd1, %rd361; + st.global.u8 [%rd362], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p605, %r9157, 255; + selp.b32 %r9159, 7, 8, %p605; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_527: + setp.ne.s32 %p606, %r9176, 0; + mov.u32 %r9186, %r9171; + @%p606 bra $L__BB2_524; + +$L__BB2_528: + add.s32 %r1315, %r4103, %r9024; + ld.shared.u8 %rs705, [%r1315]; + mov.u32 %r9026, 0; + cvt.u32.u16 %r5311, %rs705; + and.b32 %r5312, %r5311, 255; + and.b32 %r5313, %r9047, 255; + setp.lt.u32 %p607, %r5313, %r5312; + cvt.u16.u32 %rs706, %r9047; + selp.b16 %rs707, %rs705, %rs706, %p607; + st.shared.u8 [%r1315], %rs707; + ld.shared.u8 %rs177, [%r1315+2]; + ld.shared.u8 %rs708, [%r1315+1]; + setp.gt.u16 %p608, %rs708, %rs177; + add.s32 %r9356, %r9024, 1; + add.s32 %r5314, %r9024, 2; + selp.b32 %r5315, %r9356, %r5314, %p608; + add.s32 %r5316, %r4103, %r5315; + ld.shared.u8 %rs178, [%r5316]; + cvt.u32.u16 %r5317, %rs178; + and.b32 %r5318, %r5317, 255; + add.s32 %r9023, %r5318, -1; + cvt.u16.u32 %rs179, %r9061; + cvt.u16.u32 %rs709, %r1243; + shr.u16 %rs710, %rs709, 1; + mov.u32 %r5319, _ZZ44 j2k_htj2k_encode_codeblocks_multi_inputE14cleanup_cx_val; + add.s32 %r1318, %r5319, %r9022; + st.shared.u8 [%r1315+1], %r9061; + ld.shared.u8 %rs711, [%r1318]; + or.b16 %rs712, %rs711, %rs710; + st.shared.u8 [%r1318], %rs712; + add.s32 %r9022, %r9022, 1; + ld.shared.u8 %rs180, [%r1318+1]; + ld.shared.u8 %r1320, [%r1318+2]; + shr.u32 %r1321, %r1291, 3; + st.shared.u8 [%r1318+1], %r1321; + add.s32 %r5320, %r9021, 2; + setp.ge.u32 %p609, %r5320, %r4057; + mov.u32 %r9360, %r9026; + @%p609 bra $L__BB2_635; + + cvt.u64.u32 %rd363, %r9355; + add.s64 %rd364, %rd363, %rd4; + shl.b64 %rd365, %rd364, 2; + add.s64 %rd366, %rd3, %rd365; + ld.global.u32 %r1322, [%rd366]; + setp.eq.s32 %p610, %r1322, 0; + mov.u32 %r9192, 0; + mov.u32 %r9191, %r9192; + @%p610 bra $L__BB2_531; + + and.b32 %r5322, %r1322, -2147483648; + abs.s32 %r5323, %r1322; + shl.b32 %r5324, %r5323, %r1009; + or.b32 %r9191, %r5324, %r5322; + +$L__BB2_531: + shl.b32 %r5328, %r9191, 1; + shr.u32 %r5329, %r5328, %r46; + and.b32 %r1325, %r5329, -2; + setp.eq.s32 %p611, %r1325, 0; + mov.u32 %r9193, %r9192; + mov.u32 %r9199, %r9192; + @%p611 bra $L__BB2_533; + + add.s32 %r5331, %r1325, -1; + clz.b32 %r5332, %r5331; + mov.u32 %r5333, 32; + sub.s32 %r9192, %r5333, %r5332; + shr.u32 %r5334, %r9191, 31; + add.s32 %r5335, %r5334, %r1325; + add.s32 %r9193, %r5335, -2; + mov.u32 %r9199, 1; + +$L__BB2_533: + mov.u32 %r9196, 0; + mov.u32 %r9195, %r9196; + @%p493 bra $L__BB2_536; + + add.s32 %r5338, %r9355, %r4055; + cvt.u64.u32 %rd367, %r5338; + add.s64 %rd368, %rd367, %rd4; + shl.b64 %rd369, %rd368, 2; + add.s64 %rd370, %rd3, %rd369; + ld.global.u32 %r1331, [%rd370]; + setp.eq.s32 %p613, %r1331, 0; + @%p613 bra $L__BB2_536; + + and.b32 %r5339, %r1331, -2147483648; + abs.s32 %r5340, %r1331; + shl.b32 %r5341, %r5340, %r1009; + or.b32 %r9195, %r5341, %r5339; + +$L__BB2_536: + shl.b32 %r5344, %r9195, 1; + shr.u32 %r5345, %r5344, %r46; + and.b32 %r1334, %r5345, -2; + setp.eq.s32 %p614, %r1334, 0; + mov.u32 %r9197, %r9196; + mov.u32 %r9214, %r9192; + @%p614 bra $L__BB2_538; + + or.b32 %r9199, %r9199, 2; + add.s32 %r5346, %r1334, -1; + clz.b32 %r5347, %r5346; + mov.u32 %r5348, 32; + sub.s32 %r9196, %r5348, %r5347; + max.s32 %r9214, %r9192, %r9196; + shr.u32 %r5349, %r9195, 31; + add.s32 %r5350, %r5349, %r1334; + add.s32 %r9197, %r5350, -2; + +$L__BB2_538: + add.s32 %r9216, %r9355, 1; + add.s32 %r5355, %r9021, 3; + setp.ge.u32 %p615, %r5355, %r4057; + mov.u32 %r9217, 0; + mov.u32 %r9210, %r9217; + mov.u32 %r9211, %r9217; + mov.u32 %r9212, %r9217; + mov.u32 %r9213, %r9217; + @%p615 bra $L__BB2_549; + + cvt.u64.u32 %rd371, %r9216; + add.s64 %rd372, %rd371, %rd4; + shl.b64 %rd373, %rd372, 2; + add.s64 %rd374, %rd3, %rd373; + ld.global.u32 %r1344, [%rd374]; + setp.eq.s32 %p616, %r1344, 0; + mov.u32 %r9211, 0; + mov.u32 %r9200, %r9211; + @%p616 bra $L__BB2_541; + + and.b32 %r5357, %r1344, -2147483648; + abs.s32 %r5358, %r1344; + shl.b32 %r5359, %r5358, %r1009; + or.b32 %r9200, %r5359, %r5357; + +$L__BB2_541: + shl.b32 %r5362, %r9200, 1; + shr.u32 %r5363, %r5362, %r46; + and.b32 %r1347, %r5363, -2; + setp.eq.s32 %p617, %r1347, 0; + mov.u32 %r9213, %r9211; + @%p617 bra $L__BB2_543; + + or.b32 %r9199, %r9199, 4; + add.s32 %r5364, %r1347, -1; + clz.b32 %r5365, %r5364; + mov.u32 %r5366, 32; + sub.s32 %r9211, %r5366, %r5365; + max.s32 %r9214, %r9214, %r9211; + shr.u32 %r5367, %r9200, 31; + add.s32 %r5368, %r5367, %r1347; + add.s32 %r9213, %r5368, -2; + +$L__BB2_543: + mov.u32 %r9210, 0; + mov.u32 %r9205, %r9210; + @%p493 bra $L__BB2_546; + + add.s32 %r5371, %r9216, %r4055; + cvt.u64.u32 %rd375, %r5371; + add.s64 %rd376, %rd375, %rd4; + shl.b64 %rd377, %rd376, 2; + add.s64 %rd378, %rd3, %rd377; + ld.global.u32 %r1356, [%rd378]; + setp.eq.s32 %p619, %r1356, 0; + @%p619 bra $L__BB2_546; + + and.b32 %r5372, %r1356, -2147483648; + abs.s32 %r5373, %r1356; + shl.b32 %r5374, %r5373, %r1009; + or.b32 %r9205, %r5374, %r5372; + +$L__BB2_546: + shl.b32 %r5377, %r9205, 1; + shr.u32 %r5378, %r5377, %r46; + and.b32 %r1359, %r5378, -2; + setp.eq.s32 %p620, %r1359, 0; + mov.u32 %r9212, %r9210; + @%p620 bra $L__BB2_548; + + or.b32 %r9199, %r9199, 8; + add.s32 %r5379, %r1359, -1; + clz.b32 %r5380, %r5379; + mov.u32 %r5381, 32; + sub.s32 %r9210, %r5381, %r5380; + max.s32 %r9214, %r9214, %r9210; + shr.u32 %r5382, %r9205, 31; + add.s32 %r5383, %r5382, %r1359; + add.s32 %r9212, %r5383, -2; + +$L__BB2_548: + add.s32 %r9216, %r9355, 2; + +$L__BB2_549: + mov.u32 %r9355, %r9216; + shr.u32 %r5385, %r1291, 2; + shr.u32 %r5386, %r1267, 1; + or.b32 %r5387, %r5385, %r5386; + cvt.u32.u16 %r5388, %rs180; + and.b32 %r5389, %r5388, 255; + shl.b32 %r5390, %r1320, 2; + add.s32 %r5391, %r5390, %r5389; + or.b32 %r1376, %r5387, %r5391; + add.s32 %r5392, %r9199, -1; + and.b32 %r5393, %r5392, %r9199; + setp.ne.s32 %p621, %r5393, 0; + setp.gt.u16 %p622, %rs178, 2; + and.pred %p623, %p622, %p621; + selp.b32 %r5394, %r9023, 1, %p623; + max.s32 %r1377, %r5394, %r9214; + sub.s32 %r9360, %r1377, %r5394; + setp.lt.s32 %p624, %r9360, 1; + @%p624 bra $L__BB2_551; + + setp.eq.s32 %p625, %r9192, %r9214; + selp.u32 %r5395, 1, 0, %p625; + setp.eq.s32 %p626, %r9196, %r9214; + selp.u32 %r5396, -1, 0, %p626; + bfi.b32 %r5397, %r5396, %r5395, 1, 1; + setp.eq.s32 %p627, %r9211, %r9214; + selp.u16 %rs714, 1, 0, %p627; + mul.wide.u16 %r5398, %rs714, 4; + or.b32 %r5399, %r5397, %r5398; + setp.eq.s32 %p628, %r9210, %r9214; + selp.u16 %rs715, 1, 0, %p628; + mul.wide.u16 %r5400, %rs715, 8; + or.b32 %r9217, %r5399, %r5400; + +$L__BB2_551: + shl.b32 %r5401, %r9199, 4; + shl.b32 %r5402, %r1376, 8; + or.b32 %r5403, %r5401, %r5402; + or.b32 %r5404, %r5403, %r9217; + mul.wide.u32 %rd380, %r5404, 2; + add.s64 %rd381, %rd18, %rd380; + ld.global.u16 %rs181, [%rd381]; + shr.u16 %rs716, %rs181, 4; + and.b16 %rs182, %rs716, 7; + setp.eq.s16 %p629, %rs182, 0; + mov.u32 %r9229, %r9080; + @%p629 bra $L__BB2_558; + + cvt.u32.u16 %r9218, %rs182; + shr.u16 %rs717, %rs181, 8; + cvt.u32.u16 %r9219, %rs717; + +$L__BB2_553: + mov.u32 %r1383, %r9218; + setp.gt.u32 %p630, %r8972, 2879; + mov.u32 %r9229, 1; + @%p630 bra $L__BB2_558; + + mov.u32 %r5406, 8; + sub.s32 %r5407, %r5406, %r8970; + sub.s32 %r5408, %r5407, %r8971; + min.u32 %r5409, %r5408, %r1383; + setp.eq.s32 %p631, %r5409, 32; + mov.u32 %r5410, -1; + shl.b32 %r5411, %r5410, %r5409; + not.b32 %r5412, %r5411; + selp.b32 %r5413, -1, %r5412, %p631; + and.b32 %r5414, %r5413, %r9219; + shl.b32 %r5415, %r5414, %r8971; + cvt.u16.u32 %rs718, %r5415; + or.b16 %rs1165, %rs1165, %rs718; + add.s32 %r8971, %r5409, %r8971; + sub.s32 %r9218, %r1383, %r5409; + shr.u32 %r9219, %r9219, %r5409; + setp.gt.u32 %p632, %r5408, %r1383; + @%p632 bra $L__BB2_557; + + setp.ne.s32 %p633, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs719, %rs1165, 255; + setp.ne.s16 %p634, %rs719, 127; + and.pred %p635, %p633, %p634; + @%p635 bra $L__BB2_557; + + mov.u32 %r5418, 20548; + sub.s32 %r5419, %r5418, %r8972; + cvt.u64.u32 %rd382, %r5419; + add.s64 %rd383, %rd382, %rd5; + add.s64 %rd384, %rd1, %rd383; + st.global.u8 [%rd384], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p636, %rs719, 143; + selp.u32 %r8970, 1, 0, %p636; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_557: + setp.ne.s32 %p637, %r9218, 0; + mov.u32 %r9229, %r9080; + @%p637 bra $L__BB2_553; + +$L__BB2_558: + setp.ne.s32 %p638, %r1376, 0; + @%p638 bra $L__BB2_606; + + setp.eq.s32 %p639, %r9199, 0; + add.s32 %r5420, %r8524, 17477; + cvt.u64.u32 %rd385, %r5420; + add.s64 %rd386, %rd385, %rd5; + add.s64 %rd20, %rd1, %rd386; + @%p639 bra $L__BB2_598; + + shl.b16 %rs1096, %rs1096, 1; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p640, %r8530, 0; + mov.u32 %r9263, %r8733; + @%p640 bra $L__BB2_563; + + setp.gt.u32 %p641, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9263, 1; + @%p641 bra $L__BB2_563; + + st.global.u8 [%rd20], %rs1096; + add.s32 %r8524, %r8524, 1; + mov.u32 %r8530, 8; + mov.u16 %rs1096, 0; + mov.u32 %r9263, %r8733; + +$L__BB2_563: + setp.lt.u32 %p642, %r8735, 3; + mov.u32 %r9233, 0; + @%p642 bra $L__BB2_566; + + setp.lt.u32 %p643, %r8735, 6; + mov.u32 %r9233, 1; + @%p643 bra $L__BB2_566; + + setp.lt.u32 %p644, %r8735, 9; + setp.eq.s32 %p645, %r8735, 11; + selp.b32 %r5426, 4, 5, %p645; + setp.lt.u32 %p646, %r8735, 11; + selp.b32 %r5427, 3, %r5426, %p646; + selp.b32 %r9233, 2, %r5427, %p644; + +$L__BB2_566: + setp.eq.s32 %p647, %r9233, 0; + @%p647 bra $L__BB2_594; + + add.s32 %r1407, %r9233, -1; + and.b32 %r1408, %r9233, 3; + setp.eq.s32 %p648, %r1408, 0; + mov.u32 %r9243, %r9233; + mov.u32 %r9246, %r9263; + @%p648 bra $L__BB2_579; + + mov.u32 %r5429, 1; + shl.b32 %r5430, %r5429, %r1407; + and.b32 %r5431, %r5430, %r8736; + setp.ne.s32 %p649, %r5431, 0; + selp.u32 %r5432, 1, 0, %p649; + cvt.u32.u16 %r5433, %rs1096; + bfi.b32 %r5434, %r5433, %r5432, 1, 8; + cvt.u16.u32 %rs1096, %r5434; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p650, %r8530, 0; + mov.u32 %r9246, %r9263; + @%p650 bra $L__BB2_571; + + setp.gt.u32 %p651, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9246, %r5429; + @%p651 bra $L__BB2_571; + + add.s32 %r5438, %r8524, 17477; + cvt.u64.u32 %rd387, %r5438; + add.s64 %rd388, %rd387, %rd5; + add.s64 %rd389, %rd1, %rd388; + st.global.u8 [%rd389], %rs1096; + add.s32 %r8524, %r8524, 1; + mov.u32 %r8530, 8; + mov.u16 %rs1096, 0; + mov.u32 %r9246, %r9263; + +$L__BB2_571: + setp.eq.s32 %p652, %r1408, 1; + mov.u32 %r9263, %r9246; + mov.u32 %r9243, %r1407; + @%p652 bra $L__BB2_579; + + add.s32 %r9243, %r9233, -2; + mov.u32 %r5439, 1; + shl.b32 %r5440, %r5439, %r9243; + and.b32 %r5441, %r5440, %r8736; + setp.ne.s32 %p653, %r5441, 0; + selp.u32 %r5442, 1, 0, %p653; + cvt.u32.u16 %r5443, %rs1096; + bfi.b32 %r5444, %r5443, %r5442, 1, 8; + cvt.u16.u32 %rs1096, %r5444; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p654, %r8530, 0; + mov.u32 %r9237, %r9246; + @%p654 bra $L__BB2_575; + + setp.gt.u32 %p655, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9237, %r5439; + @%p655 bra $L__BB2_575; + + add.s32 %r5447, %r8524, 17477; + cvt.u64.u32 %rd390, %r5447; + add.s64 %rd391, %rd390, %rd5; + add.s64 %rd392, %rd1, %rd391; + and.b16 %rs726, %rs1096, 255; + st.global.u8 [%rd392], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p656, %rs726, 255; + selp.b32 %r8530, 7, 8, %p656; + mov.u16 %rs1096, 0; + mov.u32 %r9237, %r9246; + +$L__BB2_575: + setp.eq.s32 %p657, %r1408, 2; + mov.u32 %r9263, %r9237; + mov.u32 %r9246, %r9237; + @%p657 bra $L__BB2_579; + + add.s32 %r9243, %r9233, -3; + mov.u32 %r5448, 1; + shl.b32 %r5449, %r5448, %r9243; + and.b32 %r5450, %r5449, %r8736; + setp.ne.s32 %p658, %r5450, 0; + selp.u32 %r5451, 1, 0, %p658; + cvt.u32.u16 %r5452, %rs1096; + bfi.b32 %r5453, %r5452, %r5451, 1, 8; + cvt.u16.u32 %rs1096, %r5453; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p659, %r8530, 0; + mov.u32 %r9263, %r9237; + mov.u32 %r9246, %r9237; + @%p659 bra $L__BB2_579; + + setp.gt.u32 %p660, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9263, %r5448; + mov.u32 %r9246, %r5448; + @%p660 bra $L__BB2_579; + + add.s32 %r5458, %r8524, 17477; + cvt.u64.u32 %rd393, %r5458; + add.s64 %rd394, %rd393, %rd5; + add.s64 %rd395, %rd1, %rd394; + and.b16 %rs729, %rs1096, 255; + st.global.u8 [%rd395], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p661, %rs729, 255; + selp.b32 %r8530, 7, 8, %p661; + mov.u16 %rs1096, 0; + mov.u32 %r9263, %r9237; + mov.u32 %r9246, %r9237; + +$L__BB2_579: + setp.lt.u32 %p662, %r1407, 3; + @%p662 bra $L__BB2_594; + + mov.u32 %r9263, %r9246; + +$L__BB2_581: + add.s32 %r5459, %r9243, -1; + mov.u32 %r5460, 1; + shl.b32 %r5461, %r5460, %r5459; + and.b32 %r5462, %r5461, %r8736; + setp.ne.s32 %p663, %r5462, 0; + selp.u32 %r5463, 1, 0, %p663; + cvt.u32.u16 %r5464, %rs1096; + bfi.b32 %r9252, %r5464, %r5463, 1, 8; + add.s32 %r9253, %r8530, -1; + setp.ne.s32 %p664, %r9253, 0; + mov.u32 %r9251, %r9263; + @%p664 bra $L__BB2_584; + + setp.gt.u32 %p665, %r8524, 191; + mov.u32 %r9253, 0; + mov.u32 %r9251, %r5460; + @%p665 bra $L__BB2_584; + + cvt.u16.u32 %rs730, %r9252; + and.b16 %rs731, %rs730, 255; + add.s32 %r5468, %r8524, 17477; + cvt.u64.u32 %rd396, %r5468; + add.s64 %rd397, %rd396, %rd5; + add.s64 %rd398, %rd1, %rd397; + st.global.u8 [%rd398], %rs730; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p666, %rs731, 255; + selp.b32 %r9253, 7, 8, %p666; + mov.u32 %r9252, 0; + mov.u32 %r9251, %r9263; + +$L__BB2_584: + add.s32 %r5469, %r9243, -2; + shl.b32 %r5471, %r5460, %r5469; + and.b32 %r5472, %r5471, %r8736; + setp.ne.s32 %p667, %r5472, 0; + and.b32 %r5473, %r9252, 127; + selp.u32 %r5474, 1, 0, %p667; + bfi.b32 %r9256, %r5473, %r5474, 1, 7; + add.s32 %r9257, %r9253, -1; + setp.ne.s32 %p668, %r9257, 0; + mov.u32 %r9255, %r9251; + @%p668 bra $L__BB2_587; + + setp.gt.u32 %p669, %r8524, 191; + mov.u32 %r9257, 0; + mov.u32 %r9255, 1; + @%p669 bra $L__BB2_587; + + cvt.u16.u32 %rs732, %r9256; + and.b16 %rs733, %rs732, 255; + add.s32 %r5478, %r8524, 17477; + cvt.u64.u32 %rd399, %r5478; + add.s64 %rd400, %rd399, %rd5; + add.s64 %rd401, %rd1, %rd400; + st.global.u8 [%rd401], %rs732; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p670, %rs733, 255; + selp.b32 %r9257, 7, 8, %p670; + mov.u32 %r9256, 0; + mov.u32 %r9255, %r9251; + +$L__BB2_587: + add.s32 %r5479, %r9243, -3; + mov.u32 %r5480, 1; + shl.b32 %r5481, %r5480, %r5479; + and.b32 %r5482, %r5481, %r8736; + setp.ne.s32 %p671, %r5482, 0; + and.b32 %r5483, %r9256, 127; + selp.u32 %r5484, 1, 0, %p671; + bfi.b32 %r9260, %r5483, %r5484, 1, 7; + add.s32 %r9261, %r9257, -1; + setp.ne.s32 %p672, %r9261, 0; + mov.u32 %r9259, %r9255; + @%p672 bra $L__BB2_590; + + setp.gt.u32 %p673, %r8524, 191; + mov.u32 %r9261, 0; + mov.u32 %r9259, %r5480; + @%p673 bra $L__BB2_590; + + cvt.u16.u32 %rs734, %r9260; + and.b16 %rs735, %rs734, 255; + add.s32 %r5488, %r8524, 17477; + cvt.u64.u32 %rd402, %r5488; + add.s64 %rd403, %rd402, %rd5; + add.s64 %rd404, %rd1, %rd403; + st.global.u8 [%rd404], %rs734; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p674, %rs735, 255; + selp.b32 %r9261, 7, 8, %p674; + mov.u32 %r9260, 0; + mov.u32 %r9259, %r9255; + +$L__BB2_590: + add.s32 %r9243, %r9243, -4; + shl.b32 %r5490, %r5480, %r9243; + and.b32 %r5491, %r5490, %r8736; + setp.ne.s32 %p675, %r5491, 0; + and.b32 %r5492, %r9260, 127; + selp.u32 %r5493, 1, 0, %p675; + bfi.b32 %r5494, %r5492, %r5493, 1, 15; + cvt.u16.u32 %rs1096, %r5494; + add.s32 %r8530, %r9261, -1; + setp.ne.s32 %p676, %r8530, 0; + mov.u32 %r9263, %r9259; + @%p676 bra $L__BB2_593; + + setp.gt.u32 %p677, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9263, 1; + @%p677 bra $L__BB2_593; + + add.s32 %r5497, %r8524, 17477; + cvt.u64.u32 %rd405, %r5497; + add.s64 %rd406, %rd405, %rd5; + add.s64 %rd407, %rd1, %rd406; + and.b16 %rs737, %rs1096, 255; + st.global.u8 [%rd407], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p678, %rs737, 255; + selp.b32 %r8530, 7, 8, %p678; + mov.u16 %rs1096, 0; + mov.u32 %r9263, %r9259; + +$L__BB2_593: + setp.ne.s32 %p679, %r9243, 0; + @%p679 bra $L__BB2_581; + +$L__BB2_594: + add.s32 %r5499, %r8735, -1; + setp.eq.s32 %p680, %r8735, 0; + mov.u32 %r8736, 0; + selp.b32 %r8735, 0, %r5499, %p680; + setp.lt.u32 %p681, %r8735, 3; + mov.u32 %r9269, %r8736; + @%p681 bra $L__BB2_597; + + setp.lt.u32 %p682, %r8735, 6; + mov.u32 %r9269, 1; + @%p682 bra $L__BB2_597; + + setp.lt.u32 %p683, %r8735, 9; + setp.eq.s32 %p684, %r8735, 11; + selp.b32 %r5501, 4, 5, %p684; + setp.lt.u32 %p685, %r8735, 11; + selp.b32 %r5502, 3, %r5501, %p685; + selp.b32 %r9269, 2, %r5502, %p683; + +$L__BB2_597: + mov.u32 %r5504, 1; + shl.b32 %r8734, %r5504, %r9269; + mov.u32 %r8733, %r9263; + bra.uni $L__BB2_606; + +$L__BB2_598: + add.s32 %r8736, %r8736, 1; + setp.lt.u32 %p686, %r8736, %r8734; + @%p686 bra $L__BB2_606; + + shl.b16 %rs738, %rs1096, 1; + or.b16 %rs1096, %rs738, 1; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p687, %r8530, 0; + mov.u32 %r9270, %r8733; + @%p687 bra $L__BB2_602; + + setp.gt.u32 %p688, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9270, 1; + @%p688 bra $L__BB2_602; + + and.b16 %rs740, %rs1096, 255; + st.global.u8 [%rd20], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p689, %rs740, 255; + selp.b32 %r8530, 7, 8, %p689; + mov.u16 %rs1096, 0; + mov.u32 %r9270, %r8733; + +$L__BB2_602: + add.s32 %r5508, %r8735, 1; + min.u32 %r8735, %r5508, 12; + setp.lt.u32 %p690, %r8735, 3; + mov.u32 %r8736, 0; + mov.u32 %r9273, %r8736; + @%p690 bra $L__BB2_605; + + setp.lt.u32 %p691, %r8735, 6; + mov.u32 %r9273, 1; + @%p691 bra $L__BB2_605; + + setp.lt.u32 %p692, %r8735, 9; + setp.eq.s32 %p693, %r8735, 11; + selp.b32 %r5510, 4, 5, %p693; + setp.lt.u32 %p694, %r8735, 11; + selp.b32 %r5511, 3, %r5510, %p694; + selp.b32 %r9273, 2, %r5511, %p692; + +$L__BB2_605: + mov.u32 %r5513, 1; + shl.b32 %r8734, %r5513, %r9273; + mov.u32 %r8733, %r9270; + +$L__BB2_606: + and.b16 %rs741, %rs181, 15; + cvt.u32.u16 %r1491, %rs741; + and.b32 %r5514, %r9199, 1; + setp.eq.b32 %p695, %r5514, 1; + mov.pred %p696, 0; + xor.pred %p697, %p695, %p696; + not.pred %p698, %p697; + mov.u32 %r9290, %r9186; + @%p698 bra $L__BB2_613; + + and.b32 %r5515, %r1491, 1; + sub.s32 %r9280, %r1377, %r5515; + setp.eq.s32 %p699, %r9280, 0; + mov.u32 %r9290, %r9186; + @%p699 bra $L__BB2_613; + + mov.u32 %r5516, -1; + shl.b32 %r5517, %r5516, %r9280; + not.b32 %r5518, %r5517; + and.b32 %r9281, %r9193, %r5518; + +$L__BB2_609: + setp.gt.u32 %p700, %r9160, 17476; + mov.u32 %r9290, 1; + @%p700 bra $L__BB2_613; + + sub.s32 %r5520, %r9159, %r9158; + min.u32 %r5521, %r5520, %r9280; + setp.eq.s32 %p701, %r5521, 32; + mov.u32 %r5522, -1; + shl.b32 %r5523, %r5522, %r5521; + not.b32 %r5524, %r5523; + selp.b32 %r5525, -1, %r5524, %p701; + and.b32 %r5526, %r5525, %r9281; + shl.b32 %r5527, %r5526, %r9158; + or.b32 %r9157, %r5527, %r9157; + add.s32 %r9158, %r5521, %r9158; + shr.u32 %r9281, %r9281, %r5521; + sub.s32 %r9280, %r9280, %r5521; + setp.lt.u32 %p702, %r9158, %r9159; + @%p702 bra $L__BB2_612; + + cvt.u64.u32 %rd408, %r9160; + add.s64 %rd409, %rd408, %rd5; + add.s64 %rd410, %rd1, %rd409; + st.global.u8 [%rd410], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p703, %r9157, 255; + selp.b32 %r9159, 7, 8, %p703; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_612: + setp.ne.s32 %p704, %r9280, 0; + mov.u32 %r9290, %r9186; + @%p704 bra $L__BB2_609; + +$L__BB2_613: + and.b32 %r1515, %r9199, 2; + setp.eq.s32 %p705, %r1515, 0; + mov.u32 %r9305, %r9290; + @%p705 bra $L__BB2_620; + + shr.u32 %r5530, %r1491, 1; + and.b32 %r5531, %r5530, 1; + sub.s32 %r9295, %r1377, %r5531; + setp.eq.s32 %p706, %r9295, 0; + mov.u32 %r9305, %r9290; + @%p706 bra $L__BB2_620; + + mov.u32 %r5532, -1; + shl.b32 %r5533, %r5532, %r9295; + not.b32 %r5534, %r5533; + and.b32 %r9296, %r9197, %r5534; + +$L__BB2_616: + setp.gt.u32 %p707, %r9160, 17476; + mov.u32 %r9305, 1; + @%p707 bra $L__BB2_620; + + sub.s32 %r5536, %r9159, %r9158; + min.u32 %r5537, %r5536, %r9295; + setp.eq.s32 %p708, %r5537, 32; + mov.u32 %r5538, -1; + shl.b32 %r5539, %r5538, %r5537; + not.b32 %r5540, %r5539; + selp.b32 %r5541, -1, %r5540, %p708; + and.b32 %r5542, %r5541, %r9296; + shl.b32 %r5543, %r5542, %r9158; + or.b32 %r9157, %r5543, %r9157; + add.s32 %r9158, %r5537, %r9158; + shr.u32 %r9296, %r9296, %r5537; + sub.s32 %r9295, %r9295, %r5537; + setp.lt.u32 %p709, %r9158, %r9159; + @%p709 bra $L__BB2_619; + + cvt.u64.u32 %rd411, %r9160; + add.s64 %rd412, %rd411, %rd5; + add.s64 %rd413, %rd1, %rd412; + st.global.u8 [%rd413], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p710, %r9157, 255; + selp.b32 %r9159, 7, 8, %p710; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_619: + setp.ne.s32 %p711, %r9295, 0; + mov.u32 %r9305, %r9290; + @%p711 bra $L__BB2_616; + +$L__BB2_620: + and.b32 %r1539, %r9199, 4; + setp.eq.s32 %p712, %r1539, 0; + mov.u32 %r9320, %r9305; + @%p712 bra $L__BB2_627; + + shr.u32 %r5546, %r1491, 2; + and.b32 %r5547, %r5546, 1; + sub.s32 %r9310, %r1377, %r5547; + setp.eq.s32 %p713, %r9310, 0; + mov.u32 %r9320, %r9305; + @%p713 bra $L__BB2_627; + + mov.u32 %r5548, -1; + shl.b32 %r5549, %r5548, %r9310; + not.b32 %r5550, %r5549; + and.b32 %r9311, %r9213, %r5550; + +$L__BB2_623: + setp.gt.u32 %p714, %r9160, 17476; + mov.u32 %r9320, 1; + @%p714 bra $L__BB2_627; + + sub.s32 %r5552, %r9159, %r9158; + min.u32 %r5553, %r5552, %r9310; + setp.eq.s32 %p715, %r5553, 32; + mov.u32 %r5554, -1; + shl.b32 %r5555, %r5554, %r5553; + not.b32 %r5556, %r5555; + selp.b32 %r5557, -1, %r5556, %p715; + and.b32 %r5558, %r5557, %r9311; + shl.b32 %r5559, %r5558, %r9158; + or.b32 %r9157, %r5559, %r9157; + add.s32 %r9158, %r5553, %r9158; + shr.u32 %r9311, %r9311, %r5553; + sub.s32 %r9310, %r9310, %r5553; + setp.lt.u32 %p716, %r9158, %r9159; + @%p716 bra $L__BB2_626; + + cvt.u64.u32 %rd414, %r9160; + add.s64 %rd415, %rd414, %rd5; + add.s64 %rd416, %rd1, %rd415; + st.global.u8 [%rd416], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p717, %r9157, 255; + selp.b32 %r9159, 7, 8, %p717; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_626: + setp.ne.s32 %p718, %r9310, 0; + mov.u32 %r9320, %r9305; + @%p718 bra $L__BB2_623; + +$L__BB2_627: + and.b32 %r1563, %r9199, 8; + setp.eq.s32 %p719, %r1563, 0; + mov.u32 %r9186, %r9320; + @%p719 bra $L__BB2_634; + + shr.u32 %r5562, %r1491, 3; + sub.s32 %r9325, %r1377, %r5562; + setp.eq.s32 %p720, %r9325, 0; + mov.u32 %r9186, %r9320; + @%p720 bra $L__BB2_634; + + mov.u32 %r5563, -1; + shl.b32 %r5564, %r5563, %r9325; + not.b32 %r5565, %r5564; + and.b32 %r9326, %r9212, %r5565; + +$L__BB2_630: + setp.gt.u32 %p721, %r9160, 17476; + mov.u32 %r9186, 1; + @%p721 bra $L__BB2_634; + + sub.s32 %r5567, %r9159, %r9158; + min.u32 %r5568, %r5567, %r9325; + setp.eq.s32 %p722, %r5568, 32; + mov.u32 %r5569, -1; + shl.b32 %r5570, %r5569, %r5568; + not.b32 %r5571, %r5570; + selp.b32 %r5572, -1, %r5571, %p722; + and.b32 %r5573, %r5572, %r9326; + shl.b32 %r5574, %r5573, %r9158; + or.b32 %r9157, %r5574, %r9157; + add.s32 %r9158, %r5568, %r9158; + shr.u32 %r9326, %r9326, %r5568; + sub.s32 %r9325, %r9325, %r5568; + setp.lt.u32 %p723, %r9158, %r9159; + @%p723 bra $L__BB2_633; + + cvt.u64.u32 %rd417, %r9160; + add.s64 %rd418, %rd417, %rd5; + add.s64 %rd419, %rd1, %rd418; + st.global.u8 [%rd419], %r9157; + add.s32 %r9160, %r9160, 1; + setp.eq.s32 %p724, %r9157, 255; + selp.b32 %r9159, 7, 8, %p724; + mov.u32 %r9157, 0; + mov.u32 %r9158, %r9157; + +$L__BB2_633: + setp.ne.s32 %p725, %r9325, 0; + mov.u32 %r9186, %r9320; + @%p725 bra $L__BB2_630; + +$L__BB2_634: + and.b32 %r5577, %r9196, 255; + and.b32 %r5578, %r9061, 255; + setp.lt.u32 %p726, %r5577, %r5578; + cvt.u16.u32 %rs742, %r9196; + selp.b16 %rs743, %rs179, %rs742, %p726; + st.shared.u8 [%r1315+1], %rs743; + ld.shared.u8 %rs744, [%r1315+3]; + setp.gt.u16 %p727, %rs177, %rs744; + add.s32 %r9356, %r9356, 1; + add.s32 %r5579, %r9024, 3; + selp.b32 %r5580, %r9356, %r5579, %p727; + add.s32 %r5582, %r4103, %r5580; + ld.shared.u8 %r5583, [%r5582]; + add.s32 %r9023, %r5583, -1; + shr.u32 %r5584, %r1515, 1; + or.b32 %r5585, %r1321, %r5584; + st.shared.u8 [%r1315+2], %r9210; + st.shared.u8 [%r1318+1], %r5585; + ld.shared.u8 %rs745, [%r1318+3]; + mul.wide.u16 %r5586, %rs745, 4; + add.s32 %r5587, %r5586, %r1320; + shr.u32 %r5588, %r1563, 3; + st.shared.u8 [%r1318+2], %r5588; + shr.u32 %r5589, %r1563, 2; + shr.u32 %r5590, %r1539, 1; + or.b32 %r5591, %r5589, %r5590; + or.b32 %r9026, %r5591, %r5587; + add.s32 %r9022, %r9022, 1; + mov.u32 %r9080, %r9229; + +$L__BB2_635: + mov.u32 %r9024, %r9356; + mov.u32 %r9025, %r9355; + max.s32 %r5592, %r9360, 0; + mul.lo.s32 %r5593, %r1106, 6; + setp.gt.s32 %p728, %r1106, 0; + selp.b32 %r5594, %r5593, 0, %p728; + cvt.u64.u32 %rd420, %r5594; + add.s64 %rd21, %rd17, %rd420; + ld.global.u8 %rs205, [%rd21+1]; + add.s32 %r5595, %r5594, 2; + cvt.u64.u32 %rd421, %r5595; + add.s64 %rd422, %rd17, %rd421; + ld.global.u8 %rs206, [%rd422]; + ld.global.u8 %rs207, [%rd422+1]; + mul.lo.s32 %r5596, %r5592, 6; + cvt.u64.u32 %rd423, %r5596; + add.s64 %rd424, %rd17, %rd423; + ld.global.u8 %rs208, [%rd424]; + ld.global.u8 %rs209, [%rd424+1]; + add.s32 %r5597, %r5596, 2; + cvt.u64.u32 %rd425, %r5597; + add.s64 %rd426, %rd17, %rd425; + ld.global.u8 %rs210, [%rd426]; + ld.global.u8 %rs211, [%rd426+1]; + setp.eq.s16 %p729, %rs205, 0; + mov.u32 %r9372, %r9080; + @%p729 bra $L__BB2_642; + + ld.global.u8 %r9362, [%rd21]; + cvt.u32.u16 %r9361, %rs205; + +$L__BB2_637: + mov.u32 %r1614, %r9361; + setp.gt.u32 %p730, %r8972, 2879; + mov.u32 %r9372, 1; + @%p730 bra $L__BB2_642; + + mov.u32 %r5599, 8; + sub.s32 %r5600, %r5599, %r8970; + sub.s32 %r5601, %r5600, %r8971; + min.u32 %r5602, %r5601, %r1614; + setp.eq.s32 %p731, %r5602, 32; + mov.u32 %r5603, -1; + shl.b32 %r5604, %r5603, %r5602; + not.b32 %r5605, %r5604; + selp.b32 %r5606, -1, %r5605, %p731; + and.b32 %r5607, %r5606, %r9362; + shl.b32 %r5608, %r5607, %r8971; + cvt.u16.u32 %rs746, %r5608; + or.b16 %rs1165, %rs1165, %rs746; + add.s32 %r8971, %r5602, %r8971; + sub.s32 %r9361, %r1614, %r5602; + shr.u32 %r9362, %r9362, %r5602; + setp.gt.u32 %p732, %r5601, %r1614; + @%p732 bra $L__BB2_641; + + setp.ne.s32 %p733, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs747, %rs1165, 255; + setp.ne.s16 %p734, %rs747, 127; + and.pred %p735, %p733, %p734; + @%p735 bra $L__BB2_641; + + mov.u32 %r5611, 20548; + sub.s32 %r5612, %r5611, %r8972; + cvt.u64.u32 %rd427, %r5612; + add.s64 %rd428, %rd427, %rd5; + add.s64 %rd429, %rd1, %rd428; + st.global.u8 [%rd429], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p736, %rs747, 143; + selp.u32 %r8970, 1, 0, %p736; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_641: + setp.ne.s32 %p737, %r9361, 0; + mov.u32 %r9372, %r9080; + @%p737 bra $L__BB2_637; + +$L__BB2_642: + setp.eq.s16 %p738, %rs209, 0; + mov.u32 %r9384, %r9372; + @%p738 bra $L__BB2_649; + + cvt.u32.u16 %r5613, %rs208; + and.b32 %r9374, %r5613, 255; + cvt.u32.u16 %r5614, %rs209; + and.b32 %r9373, %r5614, 255; + +$L__BB2_644: + mov.u32 %r1633, %r9373; + setp.gt.u32 %p739, %r8972, 2879; + mov.u32 %r9384, 1; + @%p739 bra $L__BB2_649; + + mov.u32 %r5616, 8; + sub.s32 %r5617, %r5616, %r8970; + sub.s32 %r5618, %r5617, %r8971; + min.u32 %r5619, %r5618, %r1633; + setp.eq.s32 %p740, %r5619, 32; + mov.u32 %r5620, -1; + shl.b32 %r5621, %r5620, %r5619; + not.b32 %r5622, %r5621; + selp.b32 %r5623, -1, %r5622, %p740; + and.b32 %r5624, %r5623, %r9374; + shl.b32 %r5625, %r5624, %r8971; + cvt.u16.u32 %rs751, %r5625; + or.b16 %rs1165, %rs1165, %rs751; + add.s32 %r8971, %r5619, %r8971; + sub.s32 %r9373, %r1633, %r5619; + shr.u32 %r9374, %r9374, %r5619; + setp.gt.u32 %p741, %r5618, %r1633; + @%p741 bra $L__BB2_648; + + setp.ne.s32 %p742, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs752, %rs1165, 255; + setp.ne.s16 %p743, %rs752, 127; + and.pred %p744, %p742, %p743; + @%p744 bra $L__BB2_648; + + mov.u32 %r5628, 20548; + sub.s32 %r5629, %r5628, %r8972; + cvt.u64.u32 %rd430, %r5629; + add.s64 %rd431, %rd430, %rd5; + add.s64 %rd432, %rd1, %rd431; + st.global.u8 [%rd432], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p745, %rs752, 143; + selp.u32 %r8970, 1, 0, %p745; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_648: + setp.ne.s32 %p746, %r9373, 0; + mov.u32 %r9384, %r9372; + @%p746 bra $L__BB2_644; + +$L__BB2_649: + setp.eq.s16 %p747, %rs207, 0; + mov.u32 %r9396, %r9384; + @%p747 bra $L__BB2_656; + + cvt.u32.u16 %r5630, %rs207; + and.b32 %r9385, %r5630, 255; + cvt.u32.u16 %r5631, %rs206; + and.b32 %r9386, %r5631, 255; + +$L__BB2_651: + mov.u32 %r1652, %r9385; + setp.gt.u32 %p748, %r8972, 2879; + mov.u32 %r9396, 1; + @%p748 bra $L__BB2_656; + + mov.u32 %r5633, 8; + sub.s32 %r5634, %r5633, %r8970; + sub.s32 %r5635, %r5634, %r8971; + min.u32 %r5636, %r5635, %r1652; + setp.eq.s32 %p749, %r5636, 32; + mov.u32 %r5637, -1; + shl.b32 %r5638, %r5637, %r5636; + not.b32 %r5639, %r5638; + selp.b32 %r5640, -1, %r5639, %p749; + and.b32 %r5641, %r5640, %r9386; + shl.b32 %r5642, %r5641, %r8971; + cvt.u16.u32 %rs756, %r5642; + or.b16 %rs1165, %rs1165, %rs756; + add.s32 %r8971, %r5636, %r8971; + sub.s32 %r9385, %r1652, %r5636; + shr.u32 %r9386, %r9386, %r5636; + setp.gt.u32 %p750, %r5635, %r1652; + @%p750 bra $L__BB2_655; + + setp.ne.s32 %p751, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs757, %rs1165, 255; + setp.ne.s16 %p752, %rs757, 127; + and.pred %p753, %p751, %p752; + @%p753 bra $L__BB2_655; + + mov.u32 %r5645, 20548; + sub.s32 %r5646, %r5645, %r8972; + cvt.u64.u32 %rd433, %r5646; + add.s64 %rd434, %rd433, %rd5; + add.s64 %rd435, %rd1, %rd434; + st.global.u8 [%rd435], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p754, %rs757, 143; + selp.u32 %r8970, 1, 0, %p754; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_655: + setp.ne.s32 %p755, %r9385, 0; + mov.u32 %r9396, %r9384; + @%p755 bra $L__BB2_651; + +$L__BB2_656: + setp.eq.s16 %p756, %rs211, 0; + mov.u32 %r8969, %r9396; + @%p756 bra $L__BB2_663; + + cvt.u32.u16 %r5647, %rs210; + and.b32 %r9398, %r5647, 255; + cvt.u32.u16 %r5648, %rs211; + and.b32 %r9397, %r5648, 255; + +$L__BB2_658: + mov.u32 %r1671, %r9397; + setp.gt.u32 %p757, %r8972, 2879; + mov.u32 %r8969, 1; + @%p757 bra $L__BB2_663; + + mov.u32 %r5650, 8; + sub.s32 %r5651, %r5650, %r8970; + sub.s32 %r5652, %r5651, %r8971; + min.u32 %r5653, %r5652, %r1671; + setp.eq.s32 %p758, %r5653, 32; + mov.u32 %r5654, -1; + shl.b32 %r5655, %r5654, %r5653; + not.b32 %r5656, %r5655; + selp.b32 %r5657, -1, %r5656, %p758; + and.b32 %r5658, %r5657, %r9398; + shl.b32 %r5659, %r5658, %r8971; + cvt.u16.u32 %rs761, %r5659; + or.b16 %rs1165, %rs1165, %rs761; + add.s32 %r8971, %r5653, %r8971; + sub.s32 %r9397, %r1671, %r5653; + shr.u32 %r9398, %r9398, %r5653; + setp.gt.u32 %p759, %r5652, %r1671; + @%p759 bra $L__BB2_662; + + setp.ne.s32 %p760, %r8970, 0; + mov.u32 %r8970, 0; + and.b16 %rs762, %rs1165, 255; + setp.ne.s16 %p761, %rs762, 127; + and.pred %p762, %p760, %p761; + @%p762 bra $L__BB2_662; + + mov.u32 %r5662, 20548; + sub.s32 %r5663, %r5662, %r8972; + cvt.u64.u32 %rd436, %r5663; + add.s64 %rd437, %rd436, %rd5; + add.s64 %rd438, %rd1, %rd437; + st.global.u8 [%rd438], %rs1165; + add.s32 %r8972, %r8972, 1; + setp.gt.u16 %p763, %rs762, 143; + selp.u32 %r8970, 1, 0, %p763; + mov.u32 %r8971, 0; + mov.u16 %rs1165, 0; + +$L__BB2_662: + setp.ne.s32 %p764, %r9397, 0; + mov.u32 %r8969, %r9396; + @%p764 bra $L__BB2_658; + +$L__BB2_663: + add.s32 %r9021, %r9021, 4; + setp.lt.u32 %p765, %r9021, %r4057; + @%p765 bra $L__BB2_423; + +$L__BB2_664: + add.s32 %r9005, %r9005, 2; + setp.lt.u32 %p766, %r9005, %r4058; + @%p766 bra $L__BB2_421; + +$L__BB2_665: + setp.eq.s32 %p767, %r8736, 0; + mov.u32 %r9436, %r8733; + @%p767 bra $L__BB2_669; + + shl.b16 %rs765, %rs1096, 1; + or.b16 %rs1096, %rs765, 1; + add.s32 %r8530, %r8530, -1; + setp.ne.s32 %p768, %r8530, 0; + mov.u32 %r9436, %r8733; + @%p768 bra $L__BB2_669; + + setp.gt.u32 %p769, %r8524, 191; + mov.u32 %r8530, 0; + mov.u32 %r9436, 1; + @%p769 bra $L__BB2_669; + + add.s32 %r5666, %r8524, 17477; + cvt.u64.u32 %rd439, %r5666; + add.s64 %rd440, %rd439, %rd5; + add.s64 %rd441, %rd1, %rd440; + and.b16 %rs767, %rs1096, 255; + st.global.u8 [%rd441], %rs1096; + add.s32 %r8524, %r8524, 1; + setp.eq.s16 %p770, %rs767, 255; + selp.b32 %r8530, 7, 8, %p770; + mov.u16 %rs1096, 0; + mov.u32 %r9436, %r8733; + +$L__BB2_669: + cvt.u32.u16 %r5667, %rs1096; + and.b32 %r5668, %r5667, 255; + shl.b32 %r5669, %r5668, %r8530; + cvt.u16.u32 %rs234, %r5669; + mov.u32 %r5670, -1; + shl.b32 %r5671, %r5670, %r8971; + not.b32 %r5672, %r5671; + mov.u32 %r5673, 255; + and.b32 %r5674, %r5672, 255; + setp.eq.s32 %p771, %r8971, 0; + selp.b32 %r1723, 0, %r5674, %p771; + shl.b32 %r1724, %r5673, %r8530; + and.b32 %r5675, %r1724, 255; + or.b32 %r5676, %r5675, %r1723; + setp.eq.s32 %p772, %r5676, 0; + mov.u32 %r9439, %r9436; + mov.u32 %r9441, %r8969; + @%p772 bra $L__BB2_675; + + or.b16 %rs235, %rs1165, %rs234; + and.b16 %rs768, %rs235, 255; + xor.b16 %rs769, %rs235, %rs234; + cvt.u32.u16 %r5677, %rs769; + and.b32 %r5678, %r1724, %r5677; + and.b32 %r5679, %r5678, 255; + xor.b16 %rs770, %rs235, %rs1165; + cvt.u32.u16 %r5680, %rs770; + and.b32 %r5681, %r1723, %r5680; + or.b32 %r5682, %r5679, %r5681; + setp.eq.s32 %p773, %r5682, 0; + setp.ne.s16 %p774, %rs768, 255; + and.pred %p775, %p774, %p773; + setp.gt.u32 %p776, %r8972, 1; + and.pred %p777, %p776, %p775; + add.s32 %r5683, %r8524, 17477; + cvt.u64.u32 %rd442, %r5683; + add.s64 %rd443, %rd442, %rd5; + add.s64 %rd22, %rd1, %rd443; + @%p777 bra $L__BB2_673; + bra.uni $L__BB2_671; + +$L__BB2_673: + setp.gt.u32 %p781, %r8524, 191; + mov.u32 %r9439, 1; + mov.u32 %r9441, %r8969; + @%p781 bra $L__BB2_675; + + st.global.u8 [%rd22], %rs235; + add.s32 %r8524, %r8524, 1; + mov.u32 %r9439, %r9436; + mov.u32 %r9441, %r8969; + bra.uni $L__BB2_675; + +$L__BB2_671: + setp.gt.u32 %p778, %r8524, 191; + setp.gt.u32 %p779, %r8972, 2879; + or.pred %p780, %p779, %p778; + mov.u32 %r9439, 1; + mov.u32 %r9441, %r9439; + @%p780 bra $L__BB2_675; + + st.global.u8 [%rd22], %rs234; + add.s32 %r8524, %r8524, 1; + mov.u32 %r5686, 20548; + sub.s32 %r5687, %r5686, %r8972; + cvt.u64.u32 %rd444, %r5687; + add.s64 %rd445, %rd444, %rd5; + add.s64 %rd446, %rd1, %rd445; + st.global.u8 [%rd446], %rs1165; + add.s32 %r8972, %r8972, 1; + mov.u32 %r9439, %r9436; + mov.u32 %r9441, %r8969; + +$L__BB2_675: + setp.eq.s32 %p782, %r9158, 0; + @%p782 bra $L__BB2_679; + + sub.s32 %r5689, %r9159, %r9158; + mov.u32 %r5690, -1; + shl.b32 %r5691, %r5690, %r5689; + not.b32 %r5692, %r5691; + and.b32 %r5693, %r5692, 255; + shl.b32 %r5694, %r5693, %r9158; + or.b32 %r1732, %r5694, %r9157; + setp.eq.s32 %p783, %r1732, 255; + mov.u32 %r9443, %r9186; + @%p783 bra $L__BB2_681; + + setp.gt.u32 %p784, %r9160, 17476; + mov.u32 %r9443, 1; + @%p784 bra $L__BB2_681; + + cvt.u64.u32 %rd447, %r9160; + add.s64 %rd448, %rd447, %rd5; + add.s64 %rd449, %rd1, %rd448; + st.global.u8 [%rd449], %r1732; + add.s32 %r9160, %r9160, 1; + mov.u32 %r9443, %r9186; + bra.uni $L__BB2_681; + +$L__BB2_679: + setp.ne.s32 %p785, %r9159, 7; + mov.u32 %r9443, %r9186; + @%p785 bra $L__BB2_681; + + setp.eq.s32 %p786, %r9160, 0; + add.s32 %r5696, %r9160, -1; + selp.b32 %r9160, 0, %r5696, %p786; + mov.u32 %r9443, %r9186; + +$L__BB2_681: + or.b32 %r5697, %r9441, %r9439; + or.b32 %r5698, %r5697, %r9443; + setp.eq.s32 %p787, %r5698, 0; + @%p787 bra $L__BB2_683; + + mov.u32 %r5699, 1; + st.global.u32 [%rd6], %r5699; + mov.u32 %r5700, 3; + st.global.u32 [%rd6+4], %r5700; + mov.u32 %r5701, 0; + st.global.u32 [%rd6+8], %r5701; + st.global.u32 [%rd6+12], %r5701; + st.global.u32 [%rd6+16], %r5701; + st.global.u32 [%rd6+20], %r5701; + st.global.u32 [%rd6+24], %r5701; + st.global.u32 [%rd6+28], %r5701; + bra.uni $L__BB2_1905; + +$L__BB2_683: + add.s32 %r1737, %r9160, %r8524; + add.s32 %r1738, %r1737, %r8972; + add.u64 %rd23, %SPL, 0; + mov.u32 %r9563, 1; + mov.u32 %r9561, 0; + mov.u32 %r9562, %r9561; + @%p38 bra $L__BB2_932; + + setp.ne.s32 %p789, %r4062, 3; + @%p789 bra $L__BB2_931; + + add.s32 %r5707, %r4057, 3; + shr.u32 %r5708, %r5707, 2; + add.s32 %r5709, %r5708, 8; + setp.gt.u32 %p791, %r5709, 513; + mov.pred %p790, -1; + mov.u32 %r9560, 0; + mov.pred %p2370, %p790; + @%p791 bra $L__BB2_928; + + mov.u16 %rs1224, 0; + st.local.u16 [%rd23], %rs1224; + st.local.u16 [%rd23+2], %rs1224; + st.local.u16 [%rd23+4], %rs1224; + st.local.u16 [%rd23+6], %rs1224; + st.local.u16 [%rd23+8], %rs1224; + st.local.u16 [%rd23+10], %rs1224; + st.local.u16 [%rd23+12], %rs1224; + st.local.u16 [%rd23+14], %rs1224; + st.local.u16 [%rd23+16], %rs1224; + st.local.u16 [%rd23+18], %rs1224; + st.local.u16 [%rd23+20], %rs1224; + st.local.u16 [%rd23+22], %rs1224; + st.local.u16 [%rd23+24], %rs1224; + st.local.u16 [%rd23+26], %rs1224; + st.local.u16 [%rd23+28], %rs1224; + st.local.u16 [%rd23+30], %rs1224; + st.local.u16 [%rd23+32], %rs1224; + st.local.u16 [%rd23+34], %rs1224; + st.local.u16 [%rd23+36], %rs1224; + st.local.u16 [%rd23+38], %rs1224; + st.local.u16 [%rd23+40], %rs1224; + st.local.u16 [%rd23+42], %rs1224; + st.local.u16 [%rd23+44], %rs1224; + st.local.u16 [%rd23+46], %rs1224; + st.local.u16 [%rd23+48], %rs1224; + st.local.u16 [%rd23+50], %rs1224; + st.local.u16 [%rd23+52], %rs1224; + st.local.u16 [%rd23+54], %rs1224; + st.local.u16 [%rd23+56], %rs1224; + st.local.u16 [%rd23+58], %rs1224; + st.local.u16 [%rd23+60], %rs1224; + st.local.u16 [%rd23+62], %rs1224; + st.local.u16 [%rd23+64], %rs1224; + st.local.u16 [%rd23+66], %rs1224; + st.local.u16 [%rd23+68], %rs1224; + st.local.u16 [%rd23+70], %rs1224; + st.local.u16 [%rd23+72], %rs1224; + st.local.u16 [%rd23+74], %rs1224; + st.local.u16 [%rd23+76], %rs1224; + st.local.u16 [%rd23+78], %rs1224; + st.local.u16 [%rd23+80], %rs1224; + st.local.u16 [%rd23+82], %rs1224; + st.local.u16 [%rd23+84], %rs1224; + st.local.u16 [%rd23+86], %rs1224; + st.local.u16 [%rd23+88], %rs1224; + st.local.u16 [%rd23+90], %rs1224; + st.local.u16 [%rd23+92], %rs1224; + st.local.u16 [%rd23+94], %rs1224; + st.local.u16 [%rd23+96], %rs1224; + st.local.u16 [%rd23+98], %rs1224; + st.local.u16 [%rd23+100], %rs1224; + st.local.u16 [%rd23+102], %rs1224; + st.local.u16 [%rd23+104], %rs1224; + st.local.u16 [%rd23+106], %rs1224; + st.local.u16 [%rd23+108], %rs1224; + st.local.u16 [%rd23+110], %rs1224; + st.local.u16 [%rd23+112], %rs1224; + st.local.u16 [%rd23+114], %rs1224; + st.local.u16 [%rd23+116], %rs1224; + st.local.u16 [%rd23+118], %rs1224; + st.local.u16 [%rd23+120], %rs1224; + st.local.u16 [%rd23+122], %rs1224; + st.local.u16 [%rd23+124], %rs1224; + st.local.u16 [%rd23+126], %rs1224; + st.local.u16 [%rd23+128], %rs1224; + st.local.u16 [%rd23+130], %rs1224; + st.local.u16 [%rd23+132], %rs1224; + st.local.u16 [%rd23+134], %rs1224; + st.local.u16 [%rd23+136], %rs1224; + st.local.u16 [%rd23+138], %rs1224; + st.local.u16 [%rd23+140], %rs1224; + st.local.u16 [%rd23+142], %rs1224; + st.local.u16 [%rd23+144], %rs1224; + st.local.u16 [%rd23+146], %rs1224; + st.local.u16 [%rd23+148], %rs1224; + st.local.u16 [%rd23+150], %rs1224; + st.local.u16 [%rd23+152], %rs1224; + st.local.u16 [%rd23+154], %rs1224; + st.local.u16 [%rd23+156], %rs1224; + st.local.u16 [%rd23+158], %rs1224; + st.local.u16 [%rd23+160], %rs1224; + st.local.u16 [%rd23+162], %rs1224; + st.local.u16 [%rd23+164], %rs1224; + st.local.u16 [%rd23+166], %rs1224; + st.local.u16 [%rd23+168], %rs1224; + st.local.u16 [%rd23+170], %rs1224; + st.local.u16 [%rd23+172], %rs1224; + st.local.u16 [%rd23+174], %rs1224; + st.local.u16 [%rd23+176], %rs1224; + st.local.u16 [%rd23+178], %rs1224; + st.local.u16 [%rd23+180], %rs1224; + st.local.u16 [%rd23+182], %rs1224; + st.local.u16 [%rd23+184], %rs1224; + st.local.u16 [%rd23+186], %rs1224; + st.local.u16 [%rd23+188], %rs1224; + st.local.u16 [%rd23+190], %rs1224; + st.local.u16 [%rd23+192], %rs1224; + st.local.u16 [%rd23+194], %rs1224; + st.local.u16 [%rd23+196], %rs1224; + st.local.u16 [%rd23+198], %rs1224; + st.local.u16 [%rd23+200], %rs1224; + st.local.u16 [%rd23+202], %rs1224; + st.local.u16 [%rd23+204], %rs1224; + st.local.u16 [%rd23+206], %rs1224; + st.local.u16 [%rd23+208], %rs1224; + st.local.u16 [%rd23+210], %rs1224; + st.local.u16 [%rd23+212], %rs1224; + st.local.u16 [%rd23+214], %rs1224; + st.local.u16 [%rd23+216], %rs1224; + st.local.u16 [%rd23+218], %rs1224; + st.local.u16 [%rd23+220], %rs1224; + st.local.u16 [%rd23+222], %rs1224; + st.local.u16 [%rd23+224], %rs1224; + st.local.u16 [%rd23+226], %rs1224; + st.local.u16 [%rd23+228], %rs1224; + st.local.u16 [%rd23+230], %rs1224; + st.local.u16 [%rd23+232], %rs1224; + st.local.u16 [%rd23+234], %rs1224; + st.local.u16 [%rd23+236], %rs1224; + st.local.u16 [%rd23+238], %rs1224; + st.local.u16 [%rd23+240], %rs1224; + st.local.u16 [%rd23+242], %rs1224; + st.local.u16 [%rd23+244], %rs1224; + st.local.u16 [%rd23+246], %rs1224; + st.local.u16 [%rd23+248], %rs1224; + st.local.u16 [%rd23+250], %rs1224; + st.local.u16 [%rd23+252], %rs1224; + st.local.u16 [%rd23+254], %rs1224; + st.local.u16 [%rd23+256], %rs1224; + st.local.u16 [%rd23+258], %rs1224; + st.local.u16 [%rd23+260], %rs1224; + st.local.u16 [%rd23+262], %rs1224; + st.local.u16 [%rd23+264], %rs1224; + st.local.u16 [%rd23+266], %rs1224; + st.local.u16 [%rd23+268], %rs1224; + st.local.u16 [%rd23+270], %rs1224; + st.local.u16 [%rd23+272], %rs1224; + st.local.u16 [%rd23+274], %rs1224; + st.local.u16 [%rd23+276], %rs1224; + st.local.u16 [%rd23+278], %rs1224; + st.local.u16 [%rd23+280], %rs1224; + st.local.u16 [%rd23+282], %rs1224; + st.local.u16 [%rd23+284], %rs1224; + st.local.u16 [%rd23+286], %rs1224; + st.local.u16 [%rd23+288], %rs1224; + st.local.u16 [%rd23+290], %rs1224; + st.local.u16 [%rd23+292], %rs1224; + st.local.u16 [%rd23+294], %rs1224; + st.local.u16 [%rd23+296], %rs1224; + st.local.u16 [%rd23+298], %rs1224; + st.local.u16 [%rd23+300], %rs1224; + st.local.u16 [%rd23+302], %rs1224; + st.local.u16 [%rd23+304], %rs1224; + st.local.u16 [%rd23+306], %rs1224; + st.local.u16 [%rd23+308], %rs1224; + st.local.u16 [%rd23+310], %rs1224; + st.local.u16 [%rd23+312], %rs1224; + st.local.u16 [%rd23+314], %rs1224; + st.local.u16 [%rd23+316], %rs1224; + st.local.u16 [%rd23+318], %rs1224; + st.local.u16 [%rd23+320], %rs1224; + st.local.u16 [%rd23+322], %rs1224; + st.local.u16 [%rd23+324], %rs1224; + st.local.u16 [%rd23+326], %rs1224; + st.local.u16 [%rd23+328], %rs1224; + st.local.u16 [%rd23+330], %rs1224; + st.local.u16 [%rd23+332], %rs1224; + st.local.u16 [%rd23+334], %rs1224; + st.local.u16 [%rd23+336], %rs1224; + st.local.u16 [%rd23+338], %rs1224; + st.local.u16 [%rd23+340], %rs1224; + st.local.u16 [%rd23+342], %rs1224; + st.local.u16 [%rd23+344], %rs1224; + st.local.u16 [%rd23+346], %rs1224; + st.local.u16 [%rd23+348], %rs1224; + st.local.u16 [%rd23+350], %rs1224; + st.local.u16 [%rd23+352], %rs1224; + st.local.u16 [%rd23+354], %rs1224; + st.local.u16 [%rd23+356], %rs1224; + st.local.u16 [%rd23+358], %rs1224; + st.local.u16 [%rd23+360], %rs1224; + st.local.u16 [%rd23+362], %rs1224; + st.local.u16 [%rd23+364], %rs1224; + st.local.u16 [%rd23+366], %rs1224; + st.local.u16 [%rd23+368], %rs1224; + st.local.u16 [%rd23+370], %rs1224; + st.local.u16 [%rd23+372], %rs1224; + st.local.u16 [%rd23+374], %rs1224; + st.local.u16 [%rd23+376], %rs1224; + st.local.u16 [%rd23+378], %rs1224; + st.local.u16 [%rd23+380], %rs1224; + st.local.u16 [%rd23+382], %rs1224; + st.local.u16 [%rd23+384], %rs1224; + st.local.u16 [%rd23+386], %rs1224; + st.local.u16 [%rd23+388], %rs1224; + st.local.u16 [%rd23+390], %rs1224; + st.local.u16 [%rd23+392], %rs1224; + st.local.u16 [%rd23+394], %rs1224; + st.local.u16 [%rd23+396], %rs1224; + st.local.u16 [%rd23+398], %rs1224; + st.local.u16 [%rd23+400], %rs1224; + st.local.u16 [%rd23+402], %rs1224; + st.local.u16 [%rd23+404], %rs1224; + st.local.u16 [%rd23+406], %rs1224; + st.local.u16 [%rd23+408], %rs1224; + st.local.u16 [%rd23+410], %rs1224; + st.local.u16 [%rd23+412], %rs1224; + st.local.u16 [%rd23+414], %rs1224; + st.local.u16 [%rd23+416], %rs1224; + st.local.u16 [%rd23+418], %rs1224; + st.local.u16 [%rd23+420], %rs1224; + st.local.u16 [%rd23+422], %rs1224; + st.local.u16 [%rd23+424], %rs1224; + st.local.u16 [%rd23+426], %rs1224; + st.local.u16 [%rd23+428], %rs1224; + st.local.u16 [%rd23+430], %rs1224; + st.local.u16 [%rd23+432], %rs1224; + st.local.u16 [%rd23+434], %rs1224; + st.local.u16 [%rd23+436], %rs1224; + st.local.u16 [%rd23+438], %rs1224; + st.local.u16 [%rd23+440], %rs1224; + st.local.u16 [%rd23+442], %rs1224; + st.local.u16 [%rd23+444], %rs1224; + st.local.u16 [%rd23+446], %rs1224; + st.local.u16 [%rd23+448], %rs1224; + st.local.u16 [%rd23+450], %rs1224; + st.local.u16 [%rd23+452], %rs1224; + st.local.u16 [%rd23+454], %rs1224; + st.local.u16 [%rd23+456], %rs1224; + st.local.u16 [%rd23+458], %rs1224; + st.local.u16 [%rd23+460], %rs1224; + st.local.u16 [%rd23+462], %rs1224; + st.local.u16 [%rd23+464], %rs1224; + st.local.u16 [%rd23+466], %rs1224; + st.local.u16 [%rd23+468], %rs1224; + st.local.u16 [%rd23+470], %rs1224; + st.local.u16 [%rd23+472], %rs1224; + st.local.u16 [%rd23+474], %rs1224; + st.local.u16 [%rd23+476], %rs1224; + st.local.u16 [%rd23+478], %rs1224; + st.local.u16 [%rd23+480], %rs1224; + st.local.u16 [%rd23+482], %rs1224; + st.local.u16 [%rd23+484], %rs1224; + st.local.u16 [%rd23+486], %rs1224; + st.local.u16 [%rd23+488], %rs1224; + st.local.u16 [%rd23+490], %rs1224; + st.local.u16 [%rd23+492], %rs1224; + st.local.u16 [%rd23+494], %rs1224; + st.local.u16 [%rd23+496], %rs1224; + st.local.u16 [%rd23+498], %rs1224; + st.local.u16 [%rd23+500], %rs1224; + st.local.u16 [%rd23+502], %rs1224; + st.local.u16 [%rd23+504], %rs1224; + st.local.u16 [%rd23+506], %rs1224; + st.local.u16 [%rd23+508], %rs1224; + st.local.u16 [%rd23+510], %rs1224; + st.local.u16 [%rd23+512], %rs1224; + st.local.u16 [%rd23+514], %rs1224; + st.local.u16 [%rd23+516], %rs1224; + st.local.u16 [%rd23+518], %rs1224; + st.local.u16 [%rd23+520], %rs1224; + st.local.u16 [%rd23+522], %rs1224; + st.local.u16 [%rd23+524], %rs1224; + st.local.u16 [%rd23+526], %rs1224; + st.local.u16 [%rd23+528], %rs1224; + st.local.u16 [%rd23+530], %rs1224; + st.local.u16 [%rd23+532], %rs1224; + st.local.u16 [%rd23+534], %rs1224; + st.local.u16 [%rd23+536], %rs1224; + st.local.u16 [%rd23+538], %rs1224; + st.local.u16 [%rd23+540], %rs1224; + st.local.u16 [%rd23+542], %rs1224; + st.local.u16 [%rd23+544], %rs1224; + st.local.u16 [%rd23+546], %rs1224; + st.local.u16 [%rd23+548], %rs1224; + st.local.u16 [%rd23+550], %rs1224; + st.local.u16 [%rd23+552], %rs1224; + st.local.u16 [%rd23+554], %rs1224; + st.local.u16 [%rd23+556], %rs1224; + st.local.u16 [%rd23+558], %rs1224; + st.local.u16 [%rd23+560], %rs1224; + st.local.u16 [%rd23+562], %rs1224; + st.local.u16 [%rd23+564], %rs1224; + st.local.u16 [%rd23+566], %rs1224; + st.local.u16 [%rd23+568], %rs1224; + st.local.u16 [%rd23+570], %rs1224; + st.local.u16 [%rd23+572], %rs1224; + st.local.u16 [%rd23+574], %rs1224; + st.local.u16 [%rd23+576], %rs1224; + st.local.u16 [%rd23+578], %rs1224; + st.local.u16 [%rd23+580], %rs1224; + st.local.u16 [%rd23+582], %rs1224; + st.local.u16 [%rd23+584], %rs1224; + st.local.u16 [%rd23+586], %rs1224; + st.local.u16 [%rd23+588], %rs1224; + st.local.u16 [%rd23+590], %rs1224; + st.local.u16 [%rd23+592], %rs1224; + st.local.u16 [%rd23+594], %rs1224; + st.local.u16 [%rd23+596], %rs1224; + st.local.u16 [%rd23+598], %rs1224; + st.local.u16 [%rd23+600], %rs1224; + st.local.u16 [%rd23+602], %rs1224; + st.local.u16 [%rd23+604], %rs1224; + st.local.u16 [%rd23+606], %rs1224; + st.local.u16 [%rd23+608], %rs1224; + st.local.u16 [%rd23+610], %rs1224; + st.local.u16 [%rd23+612], %rs1224; + st.local.u16 [%rd23+614], %rs1224; + st.local.u16 [%rd23+616], %rs1224; + st.local.u16 [%rd23+618], %rs1224; + st.local.u16 [%rd23+620], %rs1224; + st.local.u16 [%rd23+622], %rs1224; + st.local.u16 [%rd23+624], %rs1224; + st.local.u16 [%rd23+626], %rs1224; + st.local.u16 [%rd23+628], %rs1224; + st.local.u16 [%rd23+630], %rs1224; + st.local.u16 [%rd23+632], %rs1224; + st.local.u16 [%rd23+634], %rs1224; + st.local.u16 [%rd23+636], %rs1224; + st.local.u16 [%rd23+638], %rs1224; + st.local.u16 [%rd23+640], %rs1224; + st.local.u16 [%rd23+642], %rs1224; + st.local.u16 [%rd23+644], %rs1224; + st.local.u16 [%rd23+646], %rs1224; + st.local.u16 [%rd23+648], %rs1224; + st.local.u16 [%rd23+650], %rs1224; + st.local.u16 [%rd23+652], %rs1224; + st.local.u16 [%rd23+654], %rs1224; + st.local.u16 [%rd23+656], %rs1224; + st.local.u16 [%rd23+658], %rs1224; + st.local.u16 [%rd23+660], %rs1224; + st.local.u16 [%rd23+662], %rs1224; + st.local.u16 [%rd23+664], %rs1224; + st.local.u16 [%rd23+666], %rs1224; + st.local.u16 [%rd23+668], %rs1224; + st.local.u16 [%rd23+670], %rs1224; + st.local.u16 [%rd23+672], %rs1224; + st.local.u16 [%rd23+674], %rs1224; + st.local.u16 [%rd23+676], %rs1224; + st.local.u16 [%rd23+678], %rs1224; + st.local.u16 [%rd23+680], %rs1224; + st.local.u16 [%rd23+682], %rs1224; + st.local.u16 [%rd23+684], %rs1224; + st.local.u16 [%rd23+686], %rs1224; + st.local.u16 [%rd23+688], %rs1224; + st.local.u16 [%rd23+690], %rs1224; + st.local.u16 [%rd23+692], %rs1224; + st.local.u16 [%rd23+694], %rs1224; + st.local.u16 [%rd23+696], %rs1224; + st.local.u16 [%rd23+698], %rs1224; + st.local.u16 [%rd23+700], %rs1224; + st.local.u16 [%rd23+702], %rs1224; + st.local.u16 [%rd23+704], %rs1224; + st.local.u16 [%rd23+706], %rs1224; + st.local.u16 [%rd23+708], %rs1224; + st.local.u16 [%rd23+710], %rs1224; + st.local.u16 [%rd23+712], %rs1224; + st.local.u16 [%rd23+714], %rs1224; + st.local.u16 [%rd23+716], %rs1224; + st.local.u16 [%rd23+718], %rs1224; + st.local.u16 [%rd23+720], %rs1224; + st.local.u16 [%rd23+722], %rs1224; + st.local.u16 [%rd23+724], %rs1224; + st.local.u16 [%rd23+726], %rs1224; + st.local.u16 [%rd23+728], %rs1224; + st.local.u16 [%rd23+730], %rs1224; + st.local.u16 [%rd23+732], %rs1224; + st.local.u16 [%rd23+734], %rs1224; + st.local.u16 [%rd23+736], %rs1224; + st.local.u16 [%rd23+738], %rs1224; + st.local.u16 [%rd23+740], %rs1224; + st.local.u16 [%rd23+742], %rs1224; + st.local.u16 [%rd23+744], %rs1224; + st.local.u16 [%rd23+746], %rs1224; + st.local.u16 [%rd23+748], %rs1224; + st.local.u16 [%rd23+750], %rs1224; + st.local.u16 [%rd23+752], %rs1224; + st.local.u16 [%rd23+754], %rs1224; + st.local.u16 [%rd23+756], %rs1224; + st.local.u16 [%rd23+758], %rs1224; + st.local.u16 [%rd23+760], %rs1224; + st.local.u16 [%rd23+762], %rs1224; + st.local.u16 [%rd23+764], %rs1224; + st.local.u16 [%rd23+766], %rs1224; + st.local.u16 [%rd23+768], %rs1224; + st.local.u16 [%rd23+770], %rs1224; + st.local.u16 [%rd23+772], %rs1224; + st.local.u16 [%rd23+774], %rs1224; + st.local.u16 [%rd23+776], %rs1224; + st.local.u16 [%rd23+778], %rs1224; + st.local.u16 [%rd23+780], %rs1224; + st.local.u16 [%rd23+782], %rs1224; + st.local.u16 [%rd23+784], %rs1224; + st.local.u16 [%rd23+786], %rs1224; + st.local.u16 [%rd23+788], %rs1224; + st.local.u16 [%rd23+790], %rs1224; + st.local.u16 [%rd23+792], %rs1224; + st.local.u16 [%rd23+794], %rs1224; + st.local.u16 [%rd23+796], %rs1224; + st.local.u16 [%rd23+798], %rs1224; + st.local.u16 [%rd23+800], %rs1224; + st.local.u16 [%rd23+802], %rs1224; + st.local.u16 [%rd23+804], %rs1224; + st.local.u16 [%rd23+806], %rs1224; + st.local.u16 [%rd23+808], %rs1224; + st.local.u16 [%rd23+810], %rs1224; + st.local.u16 [%rd23+812], %rs1224; + st.local.u16 [%rd23+814], %rs1224; + st.local.u16 [%rd23+816], %rs1224; + st.local.u16 [%rd23+818], %rs1224; + st.local.u16 [%rd23+820], %rs1224; + st.local.u16 [%rd23+822], %rs1224; + st.local.u16 [%rd23+824], %rs1224; + st.local.u16 [%rd23+826], %rs1224; + st.local.u16 [%rd23+828], %rs1224; + st.local.u16 [%rd23+830], %rs1224; + st.local.u16 [%rd23+832], %rs1224; + st.local.u16 [%rd23+834], %rs1224; + st.local.u16 [%rd23+836], %rs1224; + st.local.u16 [%rd23+838], %rs1224; + st.local.u16 [%rd23+840], %rs1224; + st.local.u16 [%rd23+842], %rs1224; + st.local.u16 [%rd23+844], %rs1224; + st.local.u16 [%rd23+846], %rs1224; + st.local.u16 [%rd23+848], %rs1224; + st.local.u16 [%rd23+850], %rs1224; + st.local.u16 [%rd23+852], %rs1224; + st.local.u16 [%rd23+854], %rs1224; + st.local.u16 [%rd23+856], %rs1224; + st.local.u16 [%rd23+858], %rs1224; + st.local.u16 [%rd23+860], %rs1224; + st.local.u16 [%rd23+862], %rs1224; + st.local.u16 [%rd23+864], %rs1224; + st.local.u16 [%rd23+866], %rs1224; + st.local.u16 [%rd23+868], %rs1224; + st.local.u16 [%rd23+870], %rs1224; + st.local.u16 [%rd23+872], %rs1224; + st.local.u16 [%rd23+874], %rs1224; + st.local.u16 [%rd23+876], %rs1224; + st.local.u16 [%rd23+878], %rs1224; + st.local.u16 [%rd23+880], %rs1224; + st.local.u16 [%rd23+882], %rs1224; + st.local.u16 [%rd23+884], %rs1224; + st.local.u16 [%rd23+886], %rs1224; + st.local.u16 [%rd23+888], %rs1224; + st.local.u16 [%rd23+890], %rs1224; + st.local.u16 [%rd23+892], %rs1224; + st.local.u16 [%rd23+894], %rs1224; + st.local.u16 [%rd23+896], %rs1224; + st.local.u16 [%rd23+898], %rs1224; + st.local.u16 [%rd23+900], %rs1224; + st.local.u16 [%rd23+902], %rs1224; + st.local.u16 [%rd23+904], %rs1224; + st.local.u16 [%rd23+906], %rs1224; + st.local.u16 [%rd23+908], %rs1224; + st.local.u16 [%rd23+910], %rs1224; + st.local.u16 [%rd23+912], %rs1224; + st.local.u16 [%rd23+914], %rs1224; + st.local.u16 [%rd23+916], %rs1224; + st.local.u16 [%rd23+918], %rs1224; + st.local.u16 [%rd23+920], %rs1224; + st.local.u16 [%rd23+922], %rs1224; + st.local.u16 [%rd23+924], %rs1224; + st.local.u16 [%rd23+926], %rs1224; + st.local.u16 [%rd23+928], %rs1224; + st.local.u16 [%rd23+930], %rs1224; + st.local.u16 [%rd23+932], %rs1224; + st.local.u16 [%rd23+934], %rs1224; + st.local.u16 [%rd23+936], %rs1224; + st.local.u16 [%rd23+938], %rs1224; + st.local.u16 [%rd23+940], %rs1224; + st.local.u16 [%rd23+942], %rs1224; + st.local.u16 [%rd23+944], %rs1224; + st.local.u16 [%rd23+946], %rs1224; + st.local.u16 [%rd23+948], %rs1224; + st.local.u16 [%rd23+950], %rs1224; + st.local.u16 [%rd23+952], %rs1224; + st.local.u16 [%rd23+954], %rs1224; + st.local.u16 [%rd23+956], %rs1224; + st.local.u16 [%rd23+958], %rs1224; + st.local.u16 [%rd23+960], %rs1224; + st.local.u16 [%rd23+962], %rs1224; + st.local.u16 [%rd23+964], %rs1224; + st.local.u16 [%rd23+966], %rs1224; + st.local.u16 [%rd23+968], %rs1224; + st.local.u16 [%rd23+970], %rs1224; + st.local.u16 [%rd23+972], %rs1224; + st.local.u16 [%rd23+974], %rs1224; + st.local.u16 [%rd23+976], %rs1224; + st.local.u16 [%rd23+978], %rs1224; + st.local.u16 [%rd23+980], %rs1224; + st.local.u16 [%rd23+982], %rs1224; + st.local.u16 [%rd23+984], %rs1224; + st.local.u16 [%rd23+986], %rs1224; + st.local.u16 [%rd23+988], %rs1224; + st.local.u16 [%rd23+990], %rs1224; + st.local.u16 [%rd23+992], %rs1224; + st.local.u16 [%rd23+994], %rs1224; + st.local.u16 [%rd23+996], %rs1224; + st.local.u16 [%rd23+998], %rs1224; + st.local.u16 [%rd23+1000], %rs1224; + st.local.u16 [%rd23+1002], %rs1224; + st.local.u16 [%rd23+1004], %rs1224; + st.local.u16 [%rd23+1006], %rs1224; + st.local.u16 [%rd23+1008], %rs1224; + st.local.u16 [%rd23+1010], %rs1224; + st.local.u16 [%rd23+1012], %rs1224; + st.local.u16 [%rd23+1014], %rs1224; + st.local.u16 [%rd23+1016], %rs1224; + st.local.u16 [%rd23+1018], %rs1224; + st.local.u16 [%rd23+1020], %rs1224; + st.local.u16 [%rd23+1022], %rs1224; + st.local.u16 [%rd23+1024], %rs1224; + mov.u32 %r9445, 0; + mov.u32 %r9555, %r9445; + mov.u32 %r9551, %r9445; + mov.u32 %r9553, %r9445; + +$L__BB2_687: + @%p10 bra $L__BB2_926; + + sub.s32 %r5716, %r4058, %r9445; + add.s32 %r1743, %r9445, 4; + mul.lo.s32 %r1744, %r1743, %r4055; + add.s32 %r1745, %r9445, 5; + add.s32 %r1746, %r1744, %r4055; + add.s32 %r1747, %r9445, 6; + shl.b32 %r5717, %r4055, 1; + add.s32 %r1748, %r1744, %r5717; + add.s32 %r1749, %r9445, 7; + mul.lo.s32 %r5718, %r4055, 3; + add.s32 %r1750, %r1744, %r5718; + add.s32 %r1751, %r9445, 1; + add.s32 %r1752, %r9445, 2; + add.s32 %r1753, %r9445, 3; + mul.lo.s32 %r1754, %r9445, %r4055; + add.s32 %r1755, %r1754, %r5718; + sub.s32 %r1756, %r1755, %r4055; + sub.s32 %r1757, %r1756, %r4055; + setp.lt.u32 %p793, %r5716, 2; + selp.b32 %r5719, 4369, 13107, %p793; + setp.lt.u32 %p794, %r5716, 3; + selp.b32 %r5720, %r5719, 30583, %p794; + setp.lt.u32 %p795, %r5716, 4; + selp.b32 %r1758, %r5720, 65535, %p795; + mov.u32 %r5715, 0; + mov.u32 %r9449, %r5715; + mov.u32 %r9450, %r5715; + +$L__BB2_689: + shr.u32 %r5722, %r9449, 2; + mul.wide.u32 %rd451, %r5722, 2; + add.s64 %rd25, %rd23, %rd451; + ld.local.u16 %rs238, [%rd25]; + ld.local.u16 %rs239, [%rd25+2]; + setp.ge.u32 %p796, %r9449, %r4057; + mov.u32 %r9461, %r5715; + @%p796 bra $L__BB2_698; + + setp.ge.u32 %p797, %r1743, %r4058; + mov.u32 %r9461, 0; + @%p797 bra $L__BB2_692; + + add.s32 %r5724, %r1744, %r9449; + cvt.u64.u32 %rd452, %r5724; + add.s64 %rd453, %rd452, %rd4; + shl.b64 %rd454, %rd453, 2; + add.s64 %rd455, %rd3, %rd454; + ld.global.u32 %r5725, [%rd455]; + abs.s32 %r5726, %r5725; + setp.gt.u32 %p798, %r5726, 4; + and.b32 %r5727, %r5726, 1; + setp.eq.b32 %p799, %r5727, 1; + and.pred %p800, %p798, %p799; + selp.u32 %r9461, 1, 0, %p800; + +$L__BB2_692: + setp.ge.u32 %p801, %r1745, %r4058; + @%p801 bra $L__BB2_694; + + add.s32 %r5728, %r1746, %r9449; + cvt.u64.u32 %rd456, %r5728; + add.s64 %rd457, %rd456, %rd4; + shl.b64 %rd458, %rd457, 2; + add.s64 %rd459, %rd3, %rd458; + ld.global.u32 %r5729, [%rd459]; + abs.s32 %r5730, %r5729; + setp.gt.u32 %p802, %r5730, 4; + and.b32 %r5731, %r5730, 1; + setp.eq.b32 %p803, %r5731, 1; + and.pred %p804, %p802, %p803; + selp.b32 %r5732, 2, 0, %p804; + or.b32 %r9461, %r5732, %r9461; + +$L__BB2_694: + setp.ge.u32 %p805, %r1747, %r4058; + @%p805 bra $L__BB2_696; + + add.s32 %r5733, %r1748, %r9449; + cvt.u64.u32 %rd460, %r5733; + add.s64 %rd461, %rd460, %rd4; + shl.b64 %rd462, %rd461, 2; + add.s64 %rd463, %rd3, %rd462; + ld.global.u32 %r5734, [%rd463]; + abs.s32 %r5735, %r5734; + setp.gt.u32 %p806, %r5735, 4; + and.b32 %r5736, %r5735, 1; + setp.eq.b32 %p807, %r5736, 1; + and.pred %p808, %p806, %p807; + selp.b32 %r5737, 4, 0, %p808; + or.b32 %r9461, %r5737, %r9461; + +$L__BB2_696: + setp.ge.u32 %p809, %r1749, %r4058; + @%p809 bra $L__BB2_698; + + add.s32 %r5738, %r1750, %r9449; + cvt.u64.u32 %rd464, %r5738; + add.s64 %rd465, %rd464, %rd4; + shl.b64 %rd466, %rd465, 2; + add.s64 %rd467, %rd3, %rd466; + ld.global.u32 %r5739, [%rd467]; + abs.s32 %r5740, %r5739; + setp.gt.u32 %p810, %r5740, 4; + and.b32 %r5741, %r5740, 1; + setp.eq.b32 %p811, %r5741, 1; + and.pred %p812, %p810, %p811; + selp.b32 %r5742, 8, 0, %p812; + or.b32 %r9461, %r5742, %r9461; + +$L__BB2_698: + add.s32 %r1772, %r9449, 1; + setp.ge.u32 %p813, %r1772, %r4057; + @%p813 bra $L__BB2_707; + + setp.ge.u32 %p814, %r1743, %r4058; + @%p814 bra $L__BB2_701; + + add.s32 %r5743, %r1744, %r1772; + cvt.u64.u32 %rd468, %r5743; + add.s64 %rd469, %rd468, %rd4; + shl.b64 %rd470, %rd469, 2; + add.s64 %rd471, %rd3, %rd470; + ld.global.u32 %r5744, [%rd471]; + abs.s32 %r5745, %r5744; + setp.gt.u32 %p815, %r5745, 4; + and.b32 %r5746, %r5745, 1; + setp.eq.b32 %p816, %r5746, 1; + and.pred %p817, %p815, %p816; + selp.b32 %r5747, 16, 0, %p817; + or.b32 %r9461, %r5747, %r9461; + +$L__BB2_701: + setp.ge.u32 %p818, %r1745, %r4058; + @%p818 bra $L__BB2_703; + + add.s32 %r5748, %r1746, %r1772; + cvt.u64.u32 %rd472, %r5748; + add.s64 %rd473, %rd472, %rd4; + shl.b64 %rd474, %rd473, 2; + add.s64 %rd475, %rd3, %rd474; + ld.global.u32 %r5749, [%rd475]; + abs.s32 %r5750, %r5749; + setp.gt.u32 %p819, %r5750, 4; + and.b32 %r5751, %r5750, 1; + setp.eq.b32 %p820, %r5751, 1; + and.pred %p821, %p819, %p820; + selp.b32 %r5752, 32, 0, %p821; + or.b32 %r9461, %r5752, %r9461; + +$L__BB2_703: + setp.ge.u32 %p822, %r1747, %r4058; + @%p822 bra $L__BB2_705; + + add.s32 %r5753, %r1748, %r1772; + cvt.u64.u32 %rd476, %r5753; + add.s64 %rd477, %rd476, %rd4; + shl.b64 %rd478, %rd477, 2; + add.s64 %rd479, %rd3, %rd478; + ld.global.u32 %r5754, [%rd479]; + abs.s32 %r5755, %r5754; + setp.gt.u32 %p823, %r5755, 4; + and.b32 %r5756, %r5755, 1; + setp.eq.b32 %p824, %r5756, 1; + and.pred %p825, %p823, %p824; + selp.b32 %r5757, 64, 0, %p825; + or.b32 %r9461, %r5757, %r9461; + +$L__BB2_705: + setp.ge.u32 %p826, %r1749, %r4058; + @%p826 bra $L__BB2_707; + + add.s32 %r5758, %r1750, %r1772; + cvt.u64.u32 %rd480, %r5758; + add.s64 %rd481, %rd480, %rd4; + shl.b64 %rd482, %rd481, 2; + add.s64 %rd483, %rd3, %rd482; + ld.global.u32 %r5759, [%rd483]; + abs.s32 %r5760, %r5759; + setp.gt.u32 %p827, %r5760, 4; + and.b32 %r5761, %r5760, 1; + setp.eq.b32 %p828, %r5761, 1; + and.pred %p829, %p827, %p828; + selp.b32 %r5762, 128, 0, %p829; + or.b32 %r9461, %r5762, %r9461; + +$L__BB2_707: + add.s32 %r1781, %r9449, 2; + setp.ge.u32 %p830, %r1781, %r4057; + @%p830 bra $L__BB2_716; + + setp.ge.u32 %p831, %r1743, %r4058; + @%p831 bra $L__BB2_710; + + add.s32 %r5763, %r1744, %r1781; + cvt.u64.u32 %rd484, %r5763; + add.s64 %rd485, %rd484, %rd4; + shl.b64 %rd486, %rd485, 2; + add.s64 %rd487, %rd3, %rd486; + ld.global.u32 %r5764, [%rd487]; + abs.s32 %r5765, %r5764; + setp.gt.u32 %p832, %r5765, 4; + and.b32 %r5766, %r5765, 1; + setp.eq.b32 %p833, %r5766, 1; + and.pred %p834, %p832, %p833; + selp.b32 %r5767, 256, 0, %p834; + or.b32 %r9461, %r5767, %r9461; + +$L__BB2_710: + setp.ge.u32 %p835, %r1745, %r4058; + @%p835 bra $L__BB2_712; + + add.s32 %r5768, %r1746, %r1781; + cvt.u64.u32 %rd488, %r5768; + add.s64 %rd489, %rd488, %rd4; + shl.b64 %rd490, %rd489, 2; + add.s64 %rd491, %rd3, %rd490; + ld.global.u32 %r5769, [%rd491]; + abs.s32 %r5770, %r5769; + setp.gt.u32 %p836, %r5770, 4; + and.b32 %r5771, %r5770, 1; + setp.eq.b32 %p837, %r5771, 1; + and.pred %p838, %p836, %p837; + selp.b32 %r5772, 512, 0, %p838; + or.b32 %r9461, %r5772, %r9461; + +$L__BB2_712: + setp.ge.u32 %p839, %r1747, %r4058; + @%p839 bra $L__BB2_714; + + add.s32 %r5773, %r1748, %r1781; + cvt.u64.u32 %rd492, %r5773; + add.s64 %rd493, %rd492, %rd4; + shl.b64 %rd494, %rd493, 2; + add.s64 %rd495, %rd3, %rd494; + ld.global.u32 %r5774, [%rd495]; + abs.s32 %r5775, %r5774; + setp.gt.u32 %p840, %r5775, 4; + and.b32 %r5776, %r5775, 1; + setp.eq.b32 %p841, %r5776, 1; + and.pred %p842, %p840, %p841; + selp.b32 %r5777, 1024, 0, %p842; + or.b32 %r9461, %r5777, %r9461; + +$L__BB2_714: + setp.ge.u32 %p843, %r1749, %r4058; + @%p843 bra $L__BB2_716; + + add.s32 %r5778, %r1750, %r1781; + cvt.u64.u32 %rd496, %r5778; + add.s64 %rd497, %rd496, %rd4; + shl.b64 %rd498, %rd497, 2; + add.s64 %rd499, %rd3, %rd498; + ld.global.u32 %r5779, [%rd499]; + abs.s32 %r5780, %r5779; + setp.gt.u32 %p844, %r5780, 4; + and.b32 %r5781, %r5780, 1; + setp.eq.b32 %p845, %r5781, 1; + and.pred %p846, %p844, %p845; + selp.b32 %r5782, 2048, 0, %p846; + or.b32 %r9461, %r5782, %r9461; + +$L__BB2_716: + add.s32 %r1790, %r9449, 3; + setp.ge.u32 %p847, %r1790, %r4057; + @%p847 bra $L__BB2_725; + + setp.ge.u32 %p848, %r1743, %r4058; + @%p848 bra $L__BB2_719; + + add.s32 %r5783, %r1744, %r1790; + cvt.u64.u32 %rd500, %r5783; + add.s64 %rd501, %rd500, %rd4; + shl.b64 %rd502, %rd501, 2; + add.s64 %rd503, %rd3, %rd502; + ld.global.u32 %r5784, [%rd503]; + abs.s32 %r5785, %r5784; + setp.gt.u32 %p849, %r5785, 4; + and.b32 %r5786, %r5785, 1; + setp.eq.b32 %p850, %r5786, 1; + and.pred %p851, %p849, %p850; + selp.b32 %r5787, 4096, 0, %p851; + or.b32 %r9461, %r5787, %r9461; + +$L__BB2_719: + setp.ge.u32 %p852, %r1745, %r4058; + @%p852 bra $L__BB2_721; + + add.s32 %r5788, %r1746, %r1790; + cvt.u64.u32 %rd504, %r5788; + add.s64 %rd505, %rd504, %rd4; + shl.b64 %rd506, %rd505, 2; + add.s64 %rd507, %rd3, %rd506; + ld.global.u32 %r5789, [%rd507]; + abs.s32 %r5790, %r5789; + setp.gt.u32 %p853, %r5790, 4; + and.b32 %r5791, %r5790, 1; + setp.eq.b32 %p854, %r5791, 1; + and.pred %p855, %p853, %p854; + selp.b32 %r5792, 8192, 0, %p855; + or.b32 %r9461, %r5792, %r9461; + +$L__BB2_721: + setp.ge.u32 %p856, %r1747, %r4058; + @%p856 bra $L__BB2_723; + + add.s32 %r5793, %r1748, %r1790; + cvt.u64.u32 %rd508, %r5793; + add.s64 %rd509, %rd508, %rd4; + shl.b64 %rd510, %rd509, 2; + add.s64 %rd511, %rd3, %rd510; + ld.global.u32 %r5794, [%rd511]; + abs.s32 %r5795, %r5794; + setp.gt.u32 %p857, %r5795, 4; + and.b32 %r5796, %r5795, 1; + setp.eq.b32 %p858, %r5796, 1; + and.pred %p859, %p857, %p858; + selp.b32 %r5797, 16384, 0, %p859; + or.b32 %r9461, %r5797, %r9461; + +$L__BB2_723: + setp.ge.u32 %p860, %r1749, %r4058; + @%p860 bra $L__BB2_725; + + add.s32 %r5798, %r1750, %r1790; + cvt.u64.u32 %rd512, %r5798; + add.s64 %rd513, %rd512, %rd4; + shl.b64 %rd514, %rd513, 2; + add.s64 %rd515, %rd3, %rd514; + ld.global.u32 %r5799, [%rd515]; + abs.s32 %r5800, %r5799; + setp.gt.u32 %p861, %r5800, 4; + and.b32 %r5801, %r5800, 1; + setp.eq.b32 %p862, %r5801, 1; + and.pred %p863, %p861, %p862; + selp.b32 %r5802, 32768, 0, %p863; + or.b32 %r9461, %r5802, %r9461; + +$L__BB2_725: + add.s32 %r5804, %r9449, 4; + setp.ge.u32 %p864, %r5804, %r4057; + mov.u32 %r9477, 0; + @%p864 bra $L__BB2_734; + + setp.ge.u32 %p865, %r1743, %r4058; + mov.u32 %r9477, 0; + @%p865 bra $L__BB2_728; + + add.s32 %r5806, %r1744, %r9449; + add.s32 %r5807, %r5806, 4; + cvt.u64.u32 %rd516, %r5807; + add.s64 %rd517, %rd516, %rd4; + shl.b64 %rd518, %rd517, 2; + add.s64 %rd519, %rd3, %rd518; + ld.global.u32 %r5808, [%rd519]; + abs.s32 %r5809, %r5808; + setp.gt.u32 %p866, %r5809, 4; + and.b32 %r5810, %r5809, 1; + setp.eq.b32 %p867, %r5810, 1; + and.pred %p868, %p866, %p867; + selp.u32 %r9477, 1, 0, %p868; + +$L__BB2_728: + setp.ge.u32 %p869, %r1745, %r4058; + @%p869 bra $L__BB2_730; + + add.s32 %r5811, %r1746, %r9449; + add.s32 %r5812, %r5811, 4; + cvt.u64.u32 %rd520, %r5812; + add.s64 %rd521, %rd520, %rd4; + shl.b64 %rd522, %rd521, 2; + add.s64 %rd523, %rd3, %rd522; + ld.global.u32 %r5813, [%rd523]; + abs.s32 %r5814, %r5813; + setp.gt.u32 %p870, %r5814, 4; + and.b32 %r5815, %r5814, 1; + setp.eq.b32 %p871, %r5815, 1; + and.pred %p872, %p870, %p871; + selp.b32 %r5816, 2, 0, %p872; + or.b32 %r9477, %r5816, %r9477; + +$L__BB2_730: + setp.ge.u32 %p873, %r1747, %r4058; + @%p873 bra $L__BB2_732; + + add.s32 %r5817, %r1748, %r9449; + add.s32 %r5818, %r5817, 4; + cvt.u64.u32 %rd524, %r5818; + add.s64 %rd525, %rd524, %rd4; + shl.b64 %rd526, %rd525, 2; + add.s64 %rd527, %rd3, %rd526; + ld.global.u32 %r5819, [%rd527]; + abs.s32 %r5820, %r5819; + setp.gt.u32 %p874, %r5820, 4; + and.b32 %r5821, %r5820, 1; + setp.eq.b32 %p875, %r5821, 1; + and.pred %p876, %p874, %p875; + selp.b32 %r5822, 4, 0, %p876; + or.b32 %r9477, %r5822, %r9477; + +$L__BB2_732: + setp.ge.u32 %p877, %r1749, %r4058; + @%p877 bra $L__BB2_734; + + add.s32 %r5823, %r1750, %r9449; + add.s32 %r5824, %r5823, 4; + cvt.u64.u32 %rd528, %r5824; + add.s64 %rd529, %rd528, %rd4; + shl.b64 %rd530, %rd529, 2; + add.s64 %rd531, %rd3, %rd530; + ld.global.u32 %r5825, [%rd531]; + abs.s32 %r5826, %r5825; + setp.gt.u32 %p878, %r5826, 4; + and.b32 %r5827, %r5826, 1; + setp.eq.b32 %p879, %r5827, 1; + and.pred %p880, %p878, %p879; + selp.b32 %r5828, 8, 0, %p880; + or.b32 %r9477, %r5828, %r9477; + +$L__BB2_734: + add.s32 %r1807, %r9449, 5; + setp.ge.u32 %p881, %r1807, %r4057; + @%p881 bra $L__BB2_743; + + setp.ge.u32 %p882, %r1743, %r4058; + @%p882 bra $L__BB2_737; + + add.s32 %r5829, %r1744, %r1807; + cvt.u64.u32 %rd532, %r5829; + add.s64 %rd533, %rd532, %rd4; + shl.b64 %rd534, %rd533, 2; + add.s64 %rd535, %rd3, %rd534; + ld.global.u32 %r5830, [%rd535]; + abs.s32 %r5831, %r5830; + setp.gt.u32 %p883, %r5831, 4; + and.b32 %r5832, %r5831, 1; + setp.eq.b32 %p884, %r5832, 1; + and.pred %p885, %p883, %p884; + selp.b32 %r5833, 16, 0, %p885; + or.b32 %r9477, %r5833, %r9477; + +$L__BB2_737: + setp.ge.u32 %p886, %r1745, %r4058; + @%p886 bra $L__BB2_739; + + add.s32 %r5834, %r1746, %r1807; + cvt.u64.u32 %rd536, %r5834; + add.s64 %rd537, %rd536, %rd4; + shl.b64 %rd538, %rd537, 2; + add.s64 %rd539, %rd3, %rd538; + ld.global.u32 %r5835, [%rd539]; + abs.s32 %r5836, %r5835; + setp.gt.u32 %p887, %r5836, 4; + and.b32 %r5837, %r5836, 1; + setp.eq.b32 %p888, %r5837, 1; + and.pred %p889, %p887, %p888; + selp.b32 %r5838, 32, 0, %p889; + or.b32 %r9477, %r5838, %r9477; + +$L__BB2_739: + setp.ge.u32 %p890, %r1747, %r4058; + @%p890 bra $L__BB2_741; + + add.s32 %r5839, %r1748, %r1807; + cvt.u64.u32 %rd540, %r5839; + add.s64 %rd541, %rd540, %rd4; + shl.b64 %rd542, %rd541, 2; + add.s64 %rd543, %rd3, %rd542; + ld.global.u32 %r5840, [%rd543]; + abs.s32 %r5841, %r5840; + setp.gt.u32 %p891, %r5841, 4; + and.b32 %r5842, %r5841, 1; + setp.eq.b32 %p892, %r5842, 1; + and.pred %p893, %p891, %p892; + selp.b32 %r5843, 64, 0, %p893; + or.b32 %r9477, %r5843, %r9477; + +$L__BB2_741: + setp.ge.u32 %p894, %r1749, %r4058; + @%p894 bra $L__BB2_743; + + add.s32 %r5844, %r1750, %r1807; + cvt.u64.u32 %rd544, %r5844; + add.s64 %rd545, %rd544, %rd4; + shl.b64 %rd546, %rd545, 2; + add.s64 %rd547, %rd3, %rd546; + ld.global.u32 %r5845, [%rd547]; + abs.s32 %r5846, %r5845; + setp.gt.u32 %p895, %r5846, 4; + and.b32 %r5847, %r5846, 1; + setp.eq.b32 %p896, %r5847, 1; + and.pred %p897, %p895, %p896; + selp.b32 %r5848, 128, 0, %p897; + or.b32 %r9477, %r5848, %r9477; + +$L__BB2_743: + add.s32 %r1816, %r9449, 6; + setp.ge.u32 %p898, %r1816, %r4057; + @%p898 bra $L__BB2_752; + + setp.ge.u32 %p899, %r1743, %r4058; + @%p899 bra $L__BB2_746; + + add.s32 %r5849, %r1744, %r1816; + cvt.u64.u32 %rd548, %r5849; + add.s64 %rd549, %rd548, %rd4; + shl.b64 %rd550, %rd549, 2; + add.s64 %rd551, %rd3, %rd550; + ld.global.u32 %r5850, [%rd551]; + abs.s32 %r5851, %r5850; + setp.gt.u32 %p900, %r5851, 4; + and.b32 %r5852, %r5851, 1; + setp.eq.b32 %p901, %r5852, 1; + and.pred %p902, %p900, %p901; + selp.b32 %r5853, 256, 0, %p902; + or.b32 %r9477, %r5853, %r9477; + +$L__BB2_746: + setp.ge.u32 %p903, %r1745, %r4058; + @%p903 bra $L__BB2_748; + + add.s32 %r5854, %r1746, %r1816; + cvt.u64.u32 %rd552, %r5854; + add.s64 %rd553, %rd552, %rd4; + shl.b64 %rd554, %rd553, 2; + add.s64 %rd555, %rd3, %rd554; + ld.global.u32 %r5855, [%rd555]; + abs.s32 %r5856, %r5855; + setp.gt.u32 %p904, %r5856, 4; + and.b32 %r5857, %r5856, 1; + setp.eq.b32 %p905, %r5857, 1; + and.pred %p906, %p904, %p905; + selp.b32 %r5858, 512, 0, %p906; + or.b32 %r9477, %r5858, %r9477; + +$L__BB2_748: + setp.ge.u32 %p907, %r1747, %r4058; + @%p907 bra $L__BB2_750; + + add.s32 %r5859, %r1748, %r1816; + cvt.u64.u32 %rd556, %r5859; + add.s64 %rd557, %rd556, %rd4; + shl.b64 %rd558, %rd557, 2; + add.s64 %rd559, %rd3, %rd558; + ld.global.u32 %r5860, [%rd559]; + abs.s32 %r5861, %r5860; + setp.gt.u32 %p908, %r5861, 4; + and.b32 %r5862, %r5861, 1; + setp.eq.b32 %p909, %r5862, 1; + and.pred %p910, %p908, %p909; + selp.b32 %r5863, 1024, 0, %p910; + or.b32 %r9477, %r5863, %r9477; + +$L__BB2_750: + setp.ge.u32 %p911, %r1749, %r4058; + @%p911 bra $L__BB2_752; + + add.s32 %r5864, %r1750, %r1816; + cvt.u64.u32 %rd560, %r5864; + add.s64 %rd561, %rd560, %rd4; + shl.b64 %rd562, %rd561, 2; + add.s64 %rd563, %rd3, %rd562; + ld.global.u32 %r5865, [%rd563]; + abs.s32 %r5866, %r5865; + setp.gt.u32 %p912, %r5866, 4; + and.b32 %r5867, %r5866, 1; + setp.eq.b32 %p913, %r5867, 1; + and.pred %p914, %p912, %p913; + selp.b32 %r5868, 2048, 0, %p914; + or.b32 %r9477, %r5868, %r9477; + +$L__BB2_752: + add.s32 %r1825, %r9449, 7; + setp.ge.u32 %p915, %r1825, %r4057; + @%p915 bra $L__BB2_761; + + setp.ge.u32 %p916, %r1743, %r4058; + @%p916 bra $L__BB2_755; + + add.s32 %r5869, %r1744, %r1825; + cvt.u64.u32 %rd564, %r5869; + add.s64 %rd565, %rd564, %rd4; + shl.b64 %rd566, %rd565, 2; + add.s64 %rd567, %rd3, %rd566; + ld.global.u32 %r5870, [%rd567]; + abs.s32 %r5871, %r5870; + setp.gt.u32 %p917, %r5871, 4; + and.b32 %r5872, %r5871, 1; + setp.eq.b32 %p918, %r5872, 1; + and.pred %p919, %p917, %p918; + selp.b32 %r5873, 4096, 0, %p919; + or.b32 %r9477, %r5873, %r9477; + +$L__BB2_755: + setp.ge.u32 %p920, %r1745, %r4058; + @%p920 bra $L__BB2_757; + + add.s32 %r5874, %r1746, %r1825; + cvt.u64.u32 %rd568, %r5874; + add.s64 %rd569, %rd568, %rd4; + shl.b64 %rd570, %rd569, 2; + add.s64 %rd571, %rd3, %rd570; + ld.global.u32 %r5875, [%rd571]; + abs.s32 %r5876, %r5875; + setp.gt.u32 %p921, %r5876, 4; + and.b32 %r5877, %r5876, 1; + setp.eq.b32 %p922, %r5877, 1; + and.pred %p923, %p921, %p922; + selp.b32 %r5878, 8192, 0, %p923; + or.b32 %r9477, %r5878, %r9477; + +$L__BB2_757: + setp.ge.u32 %p924, %r1747, %r4058; + @%p924 bra $L__BB2_759; + + add.s32 %r5879, %r1748, %r1825; + cvt.u64.u32 %rd572, %r5879; + add.s64 %rd573, %rd572, %rd4; + shl.b64 %rd574, %rd573, 2; + add.s64 %rd575, %rd3, %rd574; + ld.global.u32 %r5880, [%rd575]; + abs.s32 %r5881, %r5880; + setp.gt.u32 %p925, %r5881, 4; + and.b32 %r5882, %r5881, 1; + setp.eq.b32 %p926, %r5882, 1; + and.pred %p927, %p925, %p926; + selp.b32 %r5883, 16384, 0, %p927; + or.b32 %r9477, %r5883, %r9477; + +$L__BB2_759: + setp.ge.u32 %p928, %r1749, %r4058; + @%p928 bra $L__BB2_761; + + add.s32 %r5884, %r1750, %r1825; + cvt.u64.u32 %rd576, %r5884; + add.s64 %rd577, %rd576, %rd4; + shl.b64 %rd578, %rd577, 2; + add.s64 %rd579, %rd3, %rd578; + ld.global.u32 %r5885, [%rd579]; + abs.s32 %r5886, %r5885; + setp.gt.u32 %p929, %r5886, 4; + and.b32 %r5887, %r5886, 1; + setp.eq.b32 %p930, %r5887, 1; + and.pred %p931, %p929, %p930; + selp.b32 %r5888, 32768, 0, %p931; + or.b32 %r9477, %r5888, %r9477; + +$L__BB2_761: + mov.b32 %r1834, {%rs238, %rs239}; + add.s32 %r5890, %r1754, %r9449; + cvt.u64.u32 %rd580, %r5890; + add.s64 %rd581, %rd580, %rd4; + shl.b64 %rd582, %rd581, 2; + add.s64 %rd26, %rd3, %rd582; + add.s32 %r5891, %r1757, %r9449; + cvt.u64.u32 %rd583, %r5891; + add.s64 %rd584, %rd583, %rd4; + shl.b64 %rd585, %rd584, 2; + add.s64 %rd27, %rd3, %rd585; + add.s32 %r5892, %r1756, %r9449; + cvt.u64.u32 %rd586, %r5892; + add.s64 %rd587, %rd586, %rd4; + shl.b64 %rd588, %rd587, 2; + add.s64 %rd28, %rd3, %rd588; + add.s32 %r5893, %r1755, %r9449; + cvt.u64.u32 %rd589, %r5893; + add.s64 %rd590, %rd589, %rd4; + shl.b64 %rd591, %rd590, 2; + add.s64 %rd29, %rd3, %rd591; + mov.u32 %r9493, 0; + @%p796 bra $L__BB2_770; + + setp.le.u32 %p933, %r4058, %r9445; + mov.u32 %r9493, 0; + @%p933 bra $L__BB2_764; + + ld.global.u32 %r5895, [%rd26]; + abs.s32 %r5896, %r5895; + setp.gt.u32 %p934, %r5896, 4; + and.b32 %r5897, %r5896, 1; + setp.eq.b32 %p935, %r5897, 1; + and.pred %p936, %p934, %p935; + selp.u32 %r9493, 1, 0, %p936; + +$L__BB2_764: + setp.ge.u32 %p937, %r1751, %r4058; + @%p937 bra $L__BB2_766; + + ld.global.u32 %r5898, [%rd27]; + abs.s32 %r5899, %r5898; + setp.gt.u32 %p938, %r5899, 4; + and.b32 %r5900, %r5899, 1; + setp.eq.b32 %p939, %r5900, 1; + and.pred %p940, %p938, %p939; + selp.b32 %r5901, 2, 0, %p940; + or.b32 %r9493, %r5901, %r9493; + +$L__BB2_766: + setp.ge.u32 %p941, %r1752, %r4058; + @%p941 bra $L__BB2_768; + + ld.global.u32 %r5902, [%rd28]; + abs.s32 %r5903, %r5902; + setp.gt.u32 %p942, %r5903, 4; + and.b32 %r5904, %r5903, 1; + setp.eq.b32 %p943, %r5904, 1; + and.pred %p944, %p942, %p943; + selp.b32 %r5905, 4, 0, %p944; + or.b32 %r9493, %r5905, %r9493; + +$L__BB2_768: + setp.ge.u32 %p945, %r1753, %r4058; + @%p945 bra $L__BB2_770; + + ld.global.u32 %r5906, [%rd29]; + abs.s32 %r5907, %r5906; + setp.gt.u32 %p946, %r5907, 4; + and.b32 %r5908, %r5907, 1; + setp.eq.b32 %p947, %r5908, 1; + and.pred %p948, %p946, %p947; + selp.b32 %r5909, 8, 0, %p948; + or.b32 %r9493, %r5909, %r9493; + +$L__BB2_770: + add.s32 %r5910, %r1754, %r1772; + cvt.u64.u32 %rd592, %r5910; + add.s64 %rd593, %rd592, %rd4; + shl.b64 %rd594, %rd593, 2; + add.s64 %rd30, %rd3, %rd594; + add.s32 %r5911, %r1757, %r1772; + cvt.u64.u32 %rd595, %r5911; + add.s64 %rd596, %rd595, %rd4; + shl.b64 %rd597, %rd596, 2; + add.s64 %rd31, %rd3, %rd597; + add.s32 %r5912, %r1756, %r1772; + cvt.u64.u32 %rd598, %r5912; + add.s64 %rd599, %rd598, %rd4; + shl.b64 %rd600, %rd599, 2; + add.s64 %rd32, %rd3, %rd600; + add.s32 %r5913, %r1755, %r1772; + cvt.u64.u32 %rd601, %r5913; + add.s64 %rd602, %rd601, %rd4; + shl.b64 %rd603, %rd602, 2; + add.s64 %rd33, %rd3, %rd603; + shl.b32 %r5914, %r9477, 16; + or.b32 %r1843, %r5914, %r9461; + @%p813 bra $L__BB2_779; + + setp.le.u32 %p950, %r4058, %r9445; + @%p950 bra $L__BB2_773; + + ld.global.u32 %r5915, [%rd30]; + abs.s32 %r5916, %r5915; + setp.gt.u32 %p951, %r5916, 4; + and.b32 %r5917, %r5916, 1; + setp.eq.b32 %p952, %r5917, 1; + and.pred %p953, %p951, %p952; + selp.b32 %r5918, 16, 0, %p953; + or.b32 %r9493, %r5918, %r9493; + +$L__BB2_773: + setp.ge.u32 %p954, %r1751, %r4058; + @%p954 bra $L__BB2_775; + + ld.global.u32 %r5919, [%rd31]; + abs.s32 %r5920, %r5919; + setp.gt.u32 %p955, %r5920, 4; + and.b32 %r5921, %r5920, 1; + setp.eq.b32 %p956, %r5921, 1; + and.pred %p957, %p955, %p956; + selp.b32 %r5922, 32, 0, %p957; + or.b32 %r9493, %r5922, %r9493; + +$L__BB2_775: + setp.ge.u32 %p958, %r1752, %r4058; + @%p958 bra $L__BB2_777; + + ld.global.u32 %r5923, [%rd32]; + abs.s32 %r5924, %r5923; + setp.gt.u32 %p959, %r5924, 4; + and.b32 %r5925, %r5924, 1; + setp.eq.b32 %p960, %r5925, 1; + and.pred %p961, %p959, %p960; + selp.b32 %r5926, 64, 0, %p961; + or.b32 %r9493, %r5926, %r9493; + +$L__BB2_777: + setp.ge.u32 %p962, %r1753, %r4058; + @%p962 bra $L__BB2_779; + + ld.global.u32 %r5927, [%rd33]; + abs.s32 %r5928, %r5927; + setp.gt.u32 %p963, %r5928, 4; + and.b32 %r5929, %r5928, 1; + setp.eq.b32 %p964, %r5929, 1; + and.pred %p965, %p963, %p964; + selp.b32 %r5930, 128, 0, %p965; + or.b32 %r9493, %r5930, %r9493; + +$L__BB2_779: + add.s32 %r5931, %r1754, %r1781; + cvt.u64.u32 %rd604, %r5931; + add.s64 %rd605, %rd604, %rd4; + shl.b64 %rd606, %rd605, 2; + add.s64 %rd34, %rd3, %rd606; + add.s32 %r5932, %r1757, %r1781; + cvt.u64.u32 %rd607, %r5932; + add.s64 %rd608, %rd607, %rd4; + shl.b64 %rd609, %rd608, 2; + add.s64 %rd35, %rd3, %rd609; + add.s32 %r5933, %r1756, %r1781; + cvt.u64.u32 %rd610, %r5933; + add.s64 %rd611, %rd610, %rd4; + shl.b64 %rd612, %rd611, 2; + add.s64 %rd36, %rd3, %rd612; + add.s32 %r5934, %r1755, %r1781; + cvt.u64.u32 %rd613, %r5934; + add.s64 %rd614, %rd613, %rd4; + shl.b64 %rd615, %rd614, 2; + add.s64 %rd37, %rd3, %rd615; + @%p830 bra $L__BB2_788; + + setp.le.u32 %p967, %r4058, %r9445; + @%p967 bra $L__BB2_782; + + ld.global.u32 %r5935, [%rd34]; + abs.s32 %r5936, %r5935; + setp.gt.u32 %p968, %r5936, 4; + and.b32 %r5937, %r5936, 1; + setp.eq.b32 %p969, %r5937, 1; + and.pred %p970, %p968, %p969; + selp.b32 %r5938, 256, 0, %p970; + or.b32 %r9493, %r5938, %r9493; + +$L__BB2_782: + setp.ge.u32 %p971, %r1751, %r4058; + @%p971 bra $L__BB2_784; + + ld.global.u32 %r5939, [%rd35]; + abs.s32 %r5940, %r5939; + setp.gt.u32 %p972, %r5940, 4; + and.b32 %r5941, %r5940, 1; + setp.eq.b32 %p973, %r5941, 1; + and.pred %p974, %p972, %p973; + selp.b32 %r5942, 512, 0, %p974; + or.b32 %r9493, %r5942, %r9493; + +$L__BB2_784: + setp.ge.u32 %p975, %r1752, %r4058; + @%p975 bra $L__BB2_786; + + ld.global.u32 %r5943, [%rd36]; + abs.s32 %r5944, %r5943; + setp.gt.u32 %p976, %r5944, 4; + and.b32 %r5945, %r5944, 1; + setp.eq.b32 %p977, %r5945, 1; + and.pred %p978, %p976, %p977; + selp.b32 %r5946, 1024, 0, %p978; + or.b32 %r9493, %r5946, %r9493; + +$L__BB2_786: + setp.ge.u32 %p979, %r1753, %r4058; + @%p979 bra $L__BB2_788; + + ld.global.u32 %r5947, [%rd37]; + abs.s32 %r5948, %r5947; + setp.gt.u32 %p980, %r5948, 4; + and.b32 %r5949, %r5948, 1; + setp.eq.b32 %p981, %r5949, 1; + and.pred %p982, %p980, %p981; + selp.b32 %r5950, 2048, 0, %p982; + or.b32 %r9493, %r5950, %r9493; + +$L__BB2_788: + add.s32 %r5951, %r1754, %r1790; + cvt.u64.u32 %rd616, %r5951; + add.s64 %rd617, %rd616, %rd4; + shl.b64 %rd618, %rd617, 2; + add.s64 %rd38, %rd3, %rd618; + add.s32 %r5952, %r1757, %r1790; + cvt.u64.u32 %rd619, %r5952; + add.s64 %rd620, %rd619, %rd4; + shl.b64 %rd621, %rd620, 2; + add.s64 %rd39, %rd3, %rd621; + add.s32 %r5953, %r1756, %r1790; + cvt.u64.u32 %rd622, %r5953; + add.s64 %rd623, %rd622, %rd4; + shl.b64 %rd624, %rd623, 2; + add.s64 %rd40, %rd3, %rd624; + add.s32 %r5954, %r1755, %r1790; + cvt.u64.u32 %rd625, %r5954; + add.s64 %rd626, %rd625, %rd4; + shl.b64 %rd627, %rd626, 2; + add.s64 %rd41, %rd3, %rd627; + @%p847 bra $L__BB2_797; + + setp.le.u32 %p984, %r4058, %r9445; + @%p984 bra $L__BB2_791; + + ld.global.u32 %r5955, [%rd38]; + abs.s32 %r5956, %r5955; + setp.gt.u32 %p985, %r5956, 4; + and.b32 %r5957, %r5956, 1; + setp.eq.b32 %p986, %r5957, 1; + and.pred %p987, %p985, %p986; + selp.b32 %r5958, 4096, 0, %p987; + or.b32 %r9493, %r5958, %r9493; + +$L__BB2_791: + setp.ge.u32 %p988, %r1751, %r4058; + @%p988 bra $L__BB2_793; + + ld.global.u32 %r5959, [%rd39]; + abs.s32 %r5960, %r5959; + setp.gt.u32 %p989, %r5960, 4; + and.b32 %r5961, %r5960, 1; + setp.eq.b32 %p990, %r5961, 1; + and.pred %p991, %p989, %p990; + selp.b32 %r5962, 8192, 0, %p991; + or.b32 %r9493, %r5962, %r9493; + +$L__BB2_793: + setp.ge.u32 %p992, %r1752, %r4058; + @%p992 bra $L__BB2_795; + + ld.global.u32 %r5963, [%rd40]; + abs.s32 %r5964, %r5963; + setp.gt.u32 %p993, %r5964, 4; + and.b32 %r5965, %r5964, 1; + setp.eq.b32 %p994, %r5965, 1; + and.pred %p995, %p993, %p994; + selp.b32 %r5966, 16384, 0, %p995; + or.b32 %r9493, %r5966, %r9493; + +$L__BB2_795: + setp.ge.u32 %p996, %r1753, %r4058; + @%p996 bra $L__BB2_797; + + ld.global.u32 %r5967, [%rd41]; + abs.s32 %r5968, %r5967; + setp.gt.u32 %p997, %r5968, 4; + and.b32 %r5969, %r5968, 1; + setp.eq.b32 %p998, %r5969, 1; + and.pred %p999, %p997, %p998; + selp.b32 %r5970, 32768, 0, %p999; + or.b32 %r9493, %r5970, %r9493; + +$L__BB2_797: + mov.u32 %r9509, 0; + @%p864 bra $L__BB2_806; + + setp.le.u32 %p1001, %r4058, %r9445; + mov.u32 %r9509, 0; + @%p1001 bra $L__BB2_800; + + add.s32 %r5975, %r5890, 4; + cvt.u64.u32 %rd628, %r5975; + add.s64 %rd629, %rd628, %rd4; + shl.b64 %rd630, %rd629, 2; + add.s64 %rd631, %rd3, %rd630; + ld.global.u32 %r5976, [%rd631]; + abs.s32 %r5977, %r5976; + setp.gt.u32 %p1002, %r5977, 4; + and.b32 %r5978, %r5977, 1; + setp.eq.b32 %p1003, %r5978, 1; + and.pred %p1004, %p1002, %p1003; + selp.u32 %r9509, 1, 0, %p1004; + +$L__BB2_800: + setp.ge.u32 %p1005, %r1751, %r4058; + @%p1005 bra $L__BB2_802; + + add.s32 %r5980, %r5891, 4; + cvt.u64.u32 %rd632, %r5980; + add.s64 %rd633, %rd632, %rd4; + shl.b64 %rd634, %rd633, 2; + add.s64 %rd635, %rd3, %rd634; + ld.global.u32 %r5981, [%rd635]; + abs.s32 %r5982, %r5981; + setp.gt.u32 %p1006, %r5982, 4; + and.b32 %r5983, %r5982, 1; + setp.eq.b32 %p1007, %r5983, 1; + and.pred %p1008, %p1006, %p1007; + selp.b32 %r5984, 2, 0, %p1008; + or.b32 %r9509, %r5984, %r9509; + +$L__BB2_802: + setp.ge.u32 %p1009, %r1752, %r4058; + @%p1009 bra $L__BB2_804; + + add.s32 %r5986, %r5892, 4; + cvt.u64.u32 %rd636, %r5986; + add.s64 %rd637, %rd636, %rd4; + shl.b64 %rd638, %rd637, 2; + add.s64 %rd639, %rd3, %rd638; + ld.global.u32 %r5987, [%rd639]; + abs.s32 %r5988, %r5987; + setp.gt.u32 %p1010, %r5988, 4; + and.b32 %r5989, %r5988, 1; + setp.eq.b32 %p1011, %r5989, 1; + and.pred %p1012, %p1010, %p1011; + selp.b32 %r5990, 4, 0, %p1012; + or.b32 %r9509, %r5990, %r9509; + +$L__BB2_804: + setp.ge.u32 %p1013, %r1753, %r4058; + @%p1013 bra $L__BB2_806; + + add.s32 %r5992, %r5893, 4; + cvt.u64.u32 %rd640, %r5992; + add.s64 %rd641, %rd640, %rd4; + shl.b64 %rd642, %rd641, 2; + add.s64 %rd643, %rd3, %rd642; + ld.global.u32 %r5993, [%rd643]; + abs.s32 %r5994, %r5993; + setp.gt.u32 %p1014, %r5994, 4; + and.b32 %r5995, %r5994, 1; + setp.eq.b32 %p1015, %r5995, 1; + and.pred %p1016, %p1014, %p1015; + selp.b32 %r5996, 8, 0, %p1016; + or.b32 %r9509, %r5996, %r9509; + +$L__BB2_806: + @%p881 bra $L__BB2_815; + + setp.le.u32 %p1018, %r4058, %r9445; + @%p1018 bra $L__BB2_809; + + add.s32 %r5997, %r1754, %r1807; + cvt.u64.u32 %rd644, %r5997; + add.s64 %rd645, %rd644, %rd4; + shl.b64 %rd646, %rd645, 2; + add.s64 %rd647, %rd3, %rd646; + ld.global.u32 %r5998, [%rd647]; + abs.s32 %r5999, %r5998; + setp.gt.u32 %p1019, %r5999, 4; + and.b32 %r6000, %r5999, 1; + setp.eq.b32 %p1020, %r6000, 1; + and.pred %p1021, %p1019, %p1020; + selp.b32 %r6001, 16, 0, %p1021; + or.b32 %r9509, %r6001, %r9509; + +$L__BB2_809: + setp.ge.u32 %p1022, %r1751, %r4058; + @%p1022 bra $L__BB2_811; + + add.s32 %r6002, %r1757, %r1807; + cvt.u64.u32 %rd648, %r6002; + add.s64 %rd649, %rd648, %rd4; + shl.b64 %rd650, %rd649, 2; + add.s64 %rd651, %rd3, %rd650; + ld.global.u32 %r6003, [%rd651]; + abs.s32 %r6004, %r6003; + setp.gt.u32 %p1023, %r6004, 4; + and.b32 %r6005, %r6004, 1; + setp.eq.b32 %p1024, %r6005, 1; + and.pred %p1025, %p1023, %p1024; + selp.b32 %r6006, 32, 0, %p1025; + or.b32 %r9509, %r6006, %r9509; + +$L__BB2_811: + setp.ge.u32 %p1026, %r1752, %r4058; + @%p1026 bra $L__BB2_813; + + add.s32 %r6007, %r1756, %r1807; + cvt.u64.u32 %rd652, %r6007; + add.s64 %rd653, %rd652, %rd4; + shl.b64 %rd654, %rd653, 2; + add.s64 %rd655, %rd3, %rd654; + ld.global.u32 %r6008, [%rd655]; + abs.s32 %r6009, %r6008; + setp.gt.u32 %p1027, %r6009, 4; + and.b32 %r6010, %r6009, 1; + setp.eq.b32 %p1028, %r6010, 1; + and.pred %p1029, %p1027, %p1028; + selp.b32 %r6011, 64, 0, %p1029; + or.b32 %r9509, %r6011, %r9509; + +$L__BB2_813: + setp.ge.u32 %p1030, %r1753, %r4058; + @%p1030 bra $L__BB2_815; + + add.s32 %r6012, %r1755, %r1807; + cvt.u64.u32 %rd656, %r6012; + add.s64 %rd657, %rd656, %rd4; + shl.b64 %rd658, %rd657, 2; + add.s64 %rd659, %rd3, %rd658; + ld.global.u32 %r6013, [%rd659]; + abs.s32 %r6014, %r6013; + setp.gt.u32 %p1031, %r6014, 4; + and.b32 %r6015, %r6014, 1; + setp.eq.b32 %p1032, %r6015, 1; + and.pred %p1033, %p1031, %p1032; + selp.b32 %r6016, 128, 0, %p1033; + or.b32 %r9509, %r6016, %r9509; + +$L__BB2_815: + @%p898 bra $L__BB2_824; + + setp.le.u32 %p1035, %r4058, %r9445; + @%p1035 bra $L__BB2_818; + + add.s32 %r6017, %r1754, %r1816; + cvt.u64.u32 %rd660, %r6017; + add.s64 %rd661, %rd660, %rd4; + shl.b64 %rd662, %rd661, 2; + add.s64 %rd663, %rd3, %rd662; + ld.global.u32 %r6018, [%rd663]; + abs.s32 %r6019, %r6018; + setp.gt.u32 %p1036, %r6019, 4; + and.b32 %r6020, %r6019, 1; + setp.eq.b32 %p1037, %r6020, 1; + and.pred %p1038, %p1036, %p1037; + selp.b32 %r6021, 256, 0, %p1038; + or.b32 %r9509, %r6021, %r9509; + +$L__BB2_818: + setp.ge.u32 %p1039, %r1751, %r4058; + @%p1039 bra $L__BB2_820; + + add.s32 %r6022, %r1757, %r1816; + cvt.u64.u32 %rd664, %r6022; + add.s64 %rd665, %rd664, %rd4; + shl.b64 %rd666, %rd665, 2; + add.s64 %rd667, %rd3, %rd666; + ld.global.u32 %r6023, [%rd667]; + abs.s32 %r6024, %r6023; + setp.gt.u32 %p1040, %r6024, 4; + and.b32 %r6025, %r6024, 1; + setp.eq.b32 %p1041, %r6025, 1; + and.pred %p1042, %p1040, %p1041; + selp.b32 %r6026, 512, 0, %p1042; + or.b32 %r9509, %r6026, %r9509; + +$L__BB2_820: + setp.ge.u32 %p1043, %r1752, %r4058; + @%p1043 bra $L__BB2_822; + + add.s32 %r6027, %r1756, %r1816; + cvt.u64.u32 %rd668, %r6027; + add.s64 %rd669, %rd668, %rd4; + shl.b64 %rd670, %rd669, 2; + add.s64 %rd671, %rd3, %rd670; + ld.global.u32 %r6028, [%rd671]; + abs.s32 %r6029, %r6028; + setp.gt.u32 %p1044, %r6029, 4; + and.b32 %r6030, %r6029, 1; + setp.eq.b32 %p1045, %r6030, 1; + and.pred %p1046, %p1044, %p1045; + selp.b32 %r6031, 1024, 0, %p1046; + or.b32 %r9509, %r6031, %r9509; + +$L__BB2_822: + setp.ge.u32 %p1047, %r1753, %r4058; + @%p1047 bra $L__BB2_824; + + add.s32 %r6032, %r1755, %r1816; + cvt.u64.u32 %rd672, %r6032; + add.s64 %rd673, %rd672, %rd4; + shl.b64 %rd674, %rd673, 2; + add.s64 %rd675, %rd3, %rd674; + ld.global.u32 %r6033, [%rd675]; + abs.s32 %r6034, %r6033; + setp.gt.u32 %p1048, %r6034, 4; + and.b32 %r6035, %r6034, 1; + setp.eq.b32 %p1049, %r6035, 1; + and.pred %p1050, %p1048, %p1049; + selp.b32 %r6036, 2048, 0, %p1050; + or.b32 %r9509, %r6036, %r9509; + +$L__BB2_824: + @%p915 bra $L__BB2_833; + + setp.le.u32 %p1052, %r4058, %r9445; + @%p1052 bra $L__BB2_827; + + add.s32 %r6037, %r1754, %r1825; + cvt.u64.u32 %rd676, %r6037; + add.s64 %rd677, %rd676, %rd4; + shl.b64 %rd678, %rd677, 2; + add.s64 %rd679, %rd3, %rd678; + ld.global.u32 %r6038, [%rd679]; + abs.s32 %r6039, %r6038; + setp.gt.u32 %p1053, %r6039, 4; + and.b32 %r6040, %r6039, 1; + setp.eq.b32 %p1054, %r6040, 1; + and.pred %p1055, %p1053, %p1054; + selp.b32 %r6041, 4096, 0, %p1055; + or.b32 %r9509, %r6041, %r9509; + +$L__BB2_827: + setp.ge.u32 %p1056, %r1751, %r4058; + @%p1056 bra $L__BB2_829; + + add.s32 %r6042, %r1757, %r1825; + cvt.u64.u32 %rd680, %r6042; + add.s64 %rd681, %rd680, %rd4; + shl.b64 %rd682, %rd681, 2; + add.s64 %rd683, %rd3, %rd682; + ld.global.u32 %r6043, [%rd683]; + abs.s32 %r6044, %r6043; + setp.gt.u32 %p1057, %r6044, 4; + and.b32 %r6045, %r6044, 1; + setp.eq.b32 %p1058, %r6045, 1; + and.pred %p1059, %p1057, %p1058; + selp.b32 %r6046, 8192, 0, %p1059; + or.b32 %r9509, %r6046, %r9509; + +$L__BB2_829: + setp.ge.u32 %p1060, %r1752, %r4058; + @%p1060 bra $L__BB2_831; + + add.s32 %r6047, %r1756, %r1825; + cvt.u64.u32 %rd684, %r6047; + add.s64 %rd685, %rd684, %rd4; + shl.b64 %rd686, %rd685, 2; + add.s64 %rd687, %rd3, %rd686; + ld.global.u32 %r6048, [%rd687]; + abs.s32 %r6049, %r6048; + setp.gt.u32 %p1061, %r6049, 4; + and.b32 %r6050, %r6049, 1; + setp.eq.b32 %p1062, %r6050, 1; + and.pred %p1063, %p1061, %p1062; + selp.b32 %r6051, 16384, 0, %p1063; + or.b32 %r9509, %r6051, %r9509; + +$L__BB2_831: + setp.ge.u32 %p1064, %r1753, %r4058; + @%p1064 bra $L__BB2_833; + + add.s32 %r6052, %r1755, %r1825; + cvt.u64.u32 %rd688, %r6052; + add.s64 %rd689, %rd688, %rd4; + shl.b64 %rd690, %rd689, 2; + add.s64 %rd691, %rd3, %rd690; + ld.global.u32 %r6053, [%rd691]; + abs.s32 %r6054, %r6053; + setp.gt.u32 %p1065, %r6054, 4; + and.b32 %r6055, %r6054, 1; + setp.eq.b32 %p1066, %r6055, 1; + and.pred %p1067, %p1065, %p1066; + selp.b32 %r6056, 32768, 0, %p1067; + or.b32 %r9509, %r6056, %r9509; + +$L__BB2_833: + sub.s32 %r6059, %r5804, %r4057; + shl.b32 %r6060, %r9509, 16; + or.b32 %r1900, %r6060, %r9493; + and.b32 %r6061, %r1834, -2004318072; + shr.u32 %r6062, %r6061, 3; + shl.b32 %r6063, %r1843, 3; + and.b32 %r6064, %r6063, -2004318072; + or.b32 %r1901, %r6064, %r6062; + not.b32 %r6065, %r1900; + setp.gt.s32 %p1068, %r6059, 0; + mov.u32 %r9525, 0; + shl.b32 %r6066, %r6059, 2; + selp.b32 %r6067, %r6066, 0, %p1068; + shr.u32 %r1902, %r1758, %r6067; + and.b32 %r1903, %r1902, %r6065; + @%p796 bra $L__BB2_842; + + setp.le.u32 %p1070, %r4058, %r9445; + mov.u32 %r9525, 0; + @%p1070 bra $L__BB2_836; + + ld.global.u32 %r6069, [%rd26]; + abs.s32 %r6070, %r6069; + setp.eq.s32 %p1071, %r6070, 3; + selp.u32 %r9525, 1, 0, %p1071; + +$L__BB2_836: + setp.ge.u32 %p1072, %r1751, %r4058; + @%p1072 bra $L__BB2_838; + + ld.global.u32 %r6071, [%rd27]; + abs.s32 %r6072, %r6071; + setp.eq.s32 %p1073, %r6072, 3; + selp.b32 %r6073, 2, 0, %p1073; + or.b32 %r9525, %r6073, %r9525; + +$L__BB2_838: + setp.ge.u32 %p1074, %r1752, %r4058; + @%p1074 bra $L__BB2_840; + + ld.global.u32 %r6074, [%rd28]; + abs.s32 %r6075, %r6074; + setp.eq.s32 %p1075, %r6075, 3; + selp.b32 %r6076, 4, 0, %p1075; + or.b32 %r9525, %r6076, %r9525; + +$L__BB2_840: + setp.ge.u32 %p1076, %r1753, %r4058; + @%p1076 bra $L__BB2_842; + + ld.global.u32 %r6077, [%rd29]; + abs.s32 %r6078, %r6077; + setp.eq.s32 %p1077, %r6078, 3; + selp.b32 %r6079, 8, 0, %p1077; + or.b32 %r9525, %r6079, %r9525; + +$L__BB2_842: + @%p813 bra $L__BB2_851; + + setp.le.u32 %p1079, %r4058, %r9445; + @%p1079 bra $L__BB2_845; + + ld.global.u32 %r6080, [%rd30]; + abs.s32 %r6081, %r6080; + setp.eq.s32 %p1080, %r6081, 3; + selp.b32 %r6082, 16, 0, %p1080; + or.b32 %r9525, %r6082, %r9525; + +$L__BB2_845: + setp.ge.u32 %p1081, %r1751, %r4058; + @%p1081 bra $L__BB2_847; + + ld.global.u32 %r6083, [%rd31]; + abs.s32 %r6084, %r6083; + setp.eq.s32 %p1082, %r6084, 3; + selp.b32 %r6085, 32, 0, %p1082; + or.b32 %r9525, %r6085, %r9525; + +$L__BB2_847: + setp.ge.u32 %p1083, %r1752, %r4058; + @%p1083 bra $L__BB2_849; + + ld.global.u32 %r6086, [%rd32]; + abs.s32 %r6087, %r6086; + setp.eq.s32 %p1084, %r6087, 3; + selp.b32 %r6088, 64, 0, %p1084; + or.b32 %r9525, %r6088, %r9525; + +$L__BB2_849: + setp.ge.u32 %p1085, %r1753, %r4058; + @%p1085 bra $L__BB2_851; + + ld.global.u32 %r6089, [%rd33]; + abs.s32 %r6090, %r6089; + setp.eq.s32 %p1086, %r6090, 3; + selp.b32 %r6091, 128, 0, %p1086; + or.b32 %r9525, %r6091, %r9525; + +$L__BB2_851: + @%p830 bra $L__BB2_860; + + setp.le.u32 %p1088, %r4058, %r9445; + @%p1088 bra $L__BB2_854; + + ld.global.u32 %r6092, [%rd34]; + abs.s32 %r6093, %r6092; + setp.eq.s32 %p1089, %r6093, 3; + selp.b32 %r6094, 256, 0, %p1089; + or.b32 %r9525, %r6094, %r9525; + +$L__BB2_854: + setp.ge.u32 %p1090, %r1751, %r4058; + @%p1090 bra $L__BB2_856; + + ld.global.u32 %r6095, [%rd35]; + abs.s32 %r6096, %r6095; + setp.eq.s32 %p1091, %r6096, 3; + selp.b32 %r6097, 512, 0, %p1091; + or.b32 %r9525, %r6097, %r9525; + +$L__BB2_856: + setp.ge.u32 %p1092, %r1752, %r4058; + @%p1092 bra $L__BB2_858; + + ld.global.u32 %r6098, [%rd36]; + abs.s32 %r6099, %r6098; + setp.eq.s32 %p1093, %r6099, 3; + selp.b32 %r6100, 1024, 0, %p1093; + or.b32 %r9525, %r6100, %r9525; + +$L__BB2_858: + setp.ge.u32 %p1094, %r1753, %r4058; + @%p1094 bra $L__BB2_860; + + ld.global.u32 %r6101, [%rd37]; + abs.s32 %r6102, %r6101; + setp.eq.s32 %p1095, %r6102, 3; + selp.b32 %r6103, 2048, 0, %p1095; + or.b32 %r9525, %r6103, %r9525; + +$L__BB2_860: + @%p847 bra $L__BB2_869; + + setp.le.u32 %p1097, %r4058, %r9445; + @%p1097 bra $L__BB2_863; + + ld.global.u32 %r6104, [%rd38]; + abs.s32 %r6105, %r6104; + setp.eq.s32 %p1098, %r6105, 3; + selp.b32 %r6106, 4096, 0, %p1098; + or.b32 %r9525, %r6106, %r9525; + +$L__BB2_863: + setp.ge.u32 %p1099, %r1751, %r4058; + @%p1099 bra $L__BB2_865; + + ld.global.u32 %r6107, [%rd39]; + abs.s32 %r6108, %r6107; + setp.eq.s32 %p1100, %r6108, 3; + selp.b32 %r6109, 8192, 0, %p1100; + or.b32 %r9525, %r6109, %r9525; + +$L__BB2_865: + setp.ge.u32 %p1101, %r1752, %r4058; + @%p1101 bra $L__BB2_867; + + ld.global.u32 %r6110, [%rd40]; + abs.s32 %r6111, %r6110; + setp.eq.s32 %p1102, %r6111, 3; + selp.b32 %r6112, 16384, 0, %p1102; + or.b32 %r9525, %r6112, %r9525; + +$L__BB2_867: + setp.ge.u32 %p1103, %r1753, %r4058; + @%p1103 bra $L__BB2_869; + + ld.global.u32 %r6113, [%rd41]; + abs.s32 %r6114, %r6113; + setp.eq.s32 %p1104, %r6114, 3; + selp.b32 %r6115, 32768, 0, %p1104; + or.b32 %r9525, %r6115, %r9525; + +$L__BB2_869: + and.b32 %r6117, %r1900, -286331154; + shr.u32 %r6118, %r6117, 1; + shl.b32 %r6119, %r1900, 1; + and.b32 %r6120, %r6119, -286331154; + or.b32 %r6121, %r1900, %r1901; + or.b32 %r6122, %r6121, %r6120; + or.b32 %r6123, %r6122, %r6118; + and.b32 %r1936, %r9525, %r1902; + shr.u32 %r6124, %r6123, 4; + shl.b32 %r6125, %r6123, 4; + shr.u32 %r6126, %r9450, 12; + or.b32 %r6127, %r6123, %r6126; + or.b32 %r6128, %r6127, %r6125; + or.b32 %r6129, %r6128, %r6124; + and.b32 %r9535, %r1903, %r6129; + setp.eq.s32 %p1105, %r9535, 0; + mov.u32 %r6116, 0; + mov.u32 %r9556, %r6116; + @%p1105 bra $L__BB2_924; + + mov.u32 %r9534, 0; + mov.u32 %r9536, %r9534; + mov.u32 %r9537, %r9555; + +$L__BB2_871: + brev.b32 %r6132, %r9535; + bfind.shiftamt.u32 %r1944, %r6132; + mov.pred %p2370, -1; + mov.u32 %r6133, 1; + shl.b32 %r1945, %r6133, %r1944; + mov.u32 %r6134, -2; + shf.l.wrap.b32 %r6135, %r6134, %r6134, %r1944; + and.b32 %r9535, %r9535, %r6135; + or.b32 %r9534, %r1945, %r9534; + and.b32 %r1948, %r1945, %r1936; + setp.ne.s32 %p1107, %r1948, 0; + selp.u32 %r6136, 1, 0, %p1107; + setp.eq.s32 %p1108, %r9553, 0; + selp.b32 %r6137, 8, 7, %p1108; + shl.b32 %r6138, %r6136, %r9551; + cvt.u16.u32 %rs772, %r6138; + or.b16 %rs1224, %rs1224, %rs772; + add.s32 %r9551, %r9551, 1; + setp.lt.u32 %p1109, %r9551, %r6137; + mov.pred %p2368, %p2370; + @%p1109 bra $L__BB2_874; + + setp.eq.s32 %p1111, %r9537, -1; + mov.u32 %r9555, -1; + mov.pred %p2368, 0; + @%p1111 bra $L__BB2_874; + + and.b16 %rs774, %rs1224, 255; + setp.eq.s16 %p1113, %rs774, 255; + selp.u32 %r9553, 1, 0, %p1113; + add.s32 %r9555, %r9537, 1; + mov.u32 %r9551, 0; + mov.u16 %rs1224, 0; + mov.pred %p2368, %p2370; + +$L__BB2_874: + mov.u32 %r9560, 0; + not.pred %p1115, %p2368; + @%p1115 bra $L__BB2_928; + + setp.eq.s32 %p1116, %r1948, 0; + @%p1116 bra $L__BB2_916; + + or.b32 %r9536, %r1945, %r9536; + mov.u32 %r9543, 51; + setp.gt.s32 %p1117, %r1944, 7; + @%p1117 bra $L__BB2_892; + + setp.gt.s32 %p1129, %r1944, 3; + @%p1129 bra $L__BB2_885; + + setp.gt.s32 %p1135, %r1944, 1; + @%p1135 bra $L__BB2_882; + + setp.eq.s32 %p1138, %r1944, 0; + @%p1138 bra $L__BB2_915; + + setp.eq.s32 %p1139, %r1944, 1; + @%p1139 bra $L__BB2_881; + bra.uni $L__BB2_914; + +$L__BB2_881: + mov.u32 %r9543, 118; + bra.uni $L__BB2_915; + +$L__BB2_892: + setp.gt.s32 %p1118, %r1944, 11; + @%p1118 bra $L__BB2_900; + + setp.gt.s32 %p1124, %r1944, 9; + @%p1124 bra $L__BB2_897; + + setp.eq.s32 %p1127, %r1944, 8; + @%p1127 bra $L__BB2_910; + + setp.eq.s32 %p1128, %r1944, 9; + @%p1128 bra $L__BB2_896; + bra.uni $L__BB2_914; + +$L__BB2_896: + mov.u32 %r9543, 30208; + bra.uni $L__BB2_915; + +$L__BB2_885: + setp.gt.s32 %p1130, %r1944, 5; + @%p1130 bra $L__BB2_889; + + setp.eq.s32 %p1133, %r1944, 4; + @%p1133 bra $L__BB2_912; + + setp.eq.s32 %p1134, %r1944, 5; + @%p1134 bra $L__BB2_888; + bra.uni $L__BB2_914; + +$L__BB2_888: + mov.u32 %r9543, 1888; + bra.uni $L__BB2_915; + +$L__BB2_900: + setp.gt.s32 %p1119, %r1944, 13; + @%p1119 bra $L__BB2_904; + + setp.eq.s32 %p1122, %r1944, 12; + @%p1122 bra $L__BB2_908; + + setp.eq.s32 %p1123, %r1944, 13; + @%p1123 bra $L__BB2_903; + bra.uni $L__BB2_914; + +$L__BB2_903: + mov.u32 %r9543, 483328; + bra.uni $L__BB2_915; + +$L__BB2_882: + setp.eq.s32 %p1136, %r1944, 2; + @%p1136 bra $L__BB2_913; + + setp.eq.s32 %p1137, %r1944, 3; + @%p1137 bra $L__BB2_884; + bra.uni $L__BB2_914; + +$L__BB2_884: + mov.u32 %r9543, 200; + bra.uni $L__BB2_915; + +$L__BB2_897: + setp.eq.s32 %p1125, %r1944, 10; + @%p1125 bra $L__BB2_909; + + setp.eq.s32 %p1126, %r1944, 11; + @%p1126 bra $L__BB2_899; + bra.uni $L__BB2_914; + +$L__BB2_899: + mov.u32 %r9543, 51200; + bra.uni $L__BB2_915; + +$L__BB2_889: + setp.eq.s32 %p1131, %r1944, 6; + @%p1131 bra $L__BB2_911; + + setp.eq.s32 %p1132, %r1944, 7; + @%p1132 bra $L__BB2_891; + bra.uni $L__BB2_914; + +$L__BB2_891: + mov.u32 %r9543, 3200; + bra.uni $L__BB2_915; + +$L__BB2_904: + setp.eq.s32 %p1120, %r1944, 14; + @%p1120 bra $L__BB2_907; + + setp.ne.s32 %p1121, %r1944, 15; + @%p1121 bra $L__BB2_914; + + mov.u32 %r9543, 819200; + bra.uni $L__BB2_915; + +$L__BB2_910: + mov.u32 %r9543, 13056; + bra.uni $L__BB2_915; + +$L__BB2_912: + mov.u32 %r9543, 816; + bra.uni $L__BB2_915; + +$L__BB2_908: + mov.u32 %r9543, 208896; + bra.uni $L__BB2_915; + +$L__BB2_913: + mov.u32 %r9543, 236; + bra.uni $L__BB2_915; + +$L__BB2_909: + mov.u32 %r9543, 60416; + bra.uni $L__BB2_915; + +$L__BB2_911: + mov.u32 %r9543, 3776; + bra.uni $L__BB2_915; + +$L__BB2_907: + mov.u32 %r9543, 966656; + bra.uni $L__BB2_915; + +$L__BB2_914: + mov.u32 %r9543, 0; + +$L__BB2_915: + not.b32 %r6159, %r9534; + and.b32 %r6160, %r1903, %r6159; + and.b32 %r6161, %r6160, %r9543; + or.b32 %r9535, %r6161, %r9535; + +$L__BB2_916: + setp.ne.s32 %p1140, %r9535, 0; + mov.u32 %r9537, %r9555; + @%p1140 bra $L__BB2_871; + + setp.eq.s32 %p1141, %r9536, 0; + mov.u32 %r9556, 0; + @%p1141 bra $L__BB2_924; + + mov.u32 %r9552, %r9555; + mov.u32 %r9549, %r9536; + +$L__BB2_919: + mov.u32 %r9555, %r9552; + setp.eq.s32 %p1142, %r9549, 0; + mov.u32 %r9556, %r9536; + @%p1142 bra $L__BB2_924; + + brev.b32 %r6163, %r9549; + bfind.shiftamt.u32 %r6164, %r6163; + mov.pred %p2370, -1; + mov.u32 %r6165, -2; + shf.l.wrap.b32 %r6166, %r6165, %r6165, %r6164; + and.b32 %r9549, %r9549, %r6166; + shr.u32 %r6167, %r6164, 2; + and.b32 %r6168, %r6164, 3; + add.s32 %r6169, %r6168, %r9445; + add.s32 %r6170, %r6167, %r9449; + mad.lo.s32 %r6171, %r6169, %r4055, %r6170; + cvt.u64.u32 %rd692, %r6171; + add.s64 %rd693, %rd692, %rd4; + shl.b64 %rd694, %rd693, 2; + add.s64 %rd695, %rd3, %rd694; + ld.global.u32 %r6172, [%rd695]; + shr.u32 %r6173, %r6172, 31; + setp.eq.s32 %p1144, %r9553, 0; + selp.b32 %r6174, 8, 7, %p1144; + shl.b32 %r6175, %r6173, %r9551; + cvt.u16.u32 %rs775, %r6175; + or.b16 %rs1224, %rs1224, %rs775; + add.s32 %r9551, %r9551, 1; + setp.lt.u32 %p1145, %r9551, %r6174; + mov.pred %p2369, %p2370; + mov.u32 %r9552, %r9555; + @%p1145 bra $L__BB2_923; + + setp.eq.s32 %p1147, %r9555, -1; + mov.u32 %r9552, -1; + mov.pred %p2369, 0; + @%p1147 bra $L__BB2_923; + + and.b16 %rs777, %rs1224, 255; + setp.eq.s16 %p1149, %rs777, 255; + selp.u32 %r9553, 1, 0, %p1149; + add.s32 %r9552, %r9555, 1; + mov.u32 %r9551, 0; + mov.u16 %rs1224, 0; + mov.pred %p2369, %p2370; + +$L__BB2_923: + mov.u32 %r9560, 0; + @%p2369 bra $L__BB2_919; + bra.uni $L__BB2_928; + +$L__BB2_924: + not.b32 %r6180, %r9556; + and.b32 %r6181, %r1936, %r6180; + setp.ne.s32 %p1152, %r6181, 0; + mov.u32 %r9560, %r6116; + mov.pred %p2370, %p790; + @%p1152 bra $L__BB2_928; + + setp.lt.u32 %p1153, %r5804, %r4057; + or.b32 %r6182, %r9556, %r1900; + st.local.u16 [%rd25], %r6182; + shr.u32 %r6183, %r6182, 16; + st.local.u16 [%rd25+2], %r6183; + shl.b32 %r6184, %r6182, 1; + and.b32 %r6185, %r6184, 57344; + and.b32 %r6186, %r6182, 57344; + shr.u32 %r6187, %r6186, 1; + or.b32 %r6188, %r6182, %r1901; + and.b32 %r6189, %r6188, 61440; + or.b32 %r6190, %r6189, %r6185; + or.b32 %r9450, %r6190, %r6187; + mov.u32 %r9449, %r5804; + @%p1153 bra $L__BB2_689; + +$L__BB2_926: + add.s32 %r9445, %r9445, 4; + setp.gt.u32 %p1154, %r4058, %r9445; + @%p1154 bra $L__BB2_687; + + setp.eq.s32 %p1155, %r9551, 0; + add.s32 %r6191, %r9555, 1; + setp.eq.s32 %p1156, %r9555, -1; + selp.b32 %r6192, -1, %r6191, %p1156; + selp.b32 %r6193, %r9555, %r6192, %p1155; + setp.ne.s32 %p1157, %r9555, -1; + or.pred %p1158, %p1155, %p1157; + selp.b32 %r9560, %r6193, 0, %p1158; + not.pred %p2370, %p1158; + +$L__BB2_928: + @%p2370 bra $L__BB2_930; + bra.uni $L__BB2_929; + +$L__BB2_930: + mov.u32 %r6201, 2; + st.global.u32 [%rd6], %r6201; + mov.u32 %r6202, 6; + st.global.u32 [%rd6+4], %r6202; + mov.u32 %r6203, 0; + st.global.u32 [%rd6+8], %r6203; + st.global.u32 [%rd6+12], %r6203; + st.global.u32 [%rd6+16], %r6203; + st.global.u32 [%rd6+20], %r6203; + st.global.u32 [%rd6+24], %r6203; + st.global.u32 [%rd6+28], %r6203; + bra.uni $L__BB2_1905; + +$L__BB2_931: + mov.u32 %r9561, 0; + mov.u32 %r9562, %r9561; + mov.u32 %r9563, %r9561; + bra.uni $L__BB2_932; + +$L__BB2_929: + mad.lo.s32 %r6194, %r4058, %r4057, 7; + shr.u32 %r6195, %r6194, 3; + max.u32 %r9561, %r6195, %r9560; + add.s32 %r6196, %r8433, 6; + mul.wide.u32 %rd696, %r6196, 613566757; + shr.u64 %rd697, %rd696, 32; + cvt.u32.u64 %r6197, %rd697; + sub.s32 %r6198, %r6196, %r6197; + shr.u32 %r6199, %r6198, 1; + add.s32 %r6200, %r6199, %r6197; + shr.u32 %r9562, %r6200, 2; + add.s32 %r9563, %r9561, %r9562; + +$L__BB2_932: + add.s32 %r1989, %r9563, %r1738; + setp.gt.u32 %p1159, %r1989, %r4061; + setp.lt.u32 %p1160, %r1738, 2; + or.pred %p1161, %p1160, %p1159; + @%p1161 bra $L__BB2_1247; + bra.uni $L__BB2_933; + +$L__BB2_1247: + mov.u32 %r6782, 1; + st.global.u32 [%rd6], %r6782; + mov.u32 %r6783, 4; + st.global.u32 [%rd6+4], %r6783; + mov.u32 %r6784, 0; + st.global.u32 [%rd6+8], %r6784; + st.global.u32 [%rd6+12], %r6784; + st.global.u32 [%rd6+16], %r6784; + st.global.u32 [%rd6+20], %r6784; + st.global.u32 [%rd6+24], %r6784; + st.global.u32 [%rd6+28], %r6784; + bra.uni $L__BB2_1905; + +$L__BB2_933: + setp.eq.s32 %p1162, %r8524, 0; + @%p1162 bra $L__BB2_939; + + add.s32 %r6208, %r8524, -1; + and.b32 %r9568, %r8524, 3; + setp.lt.u32 %p1163, %r6208, 3; + mov.u32 %r9566, 0; + @%p1163 bra $L__BB2_937; + + sub.s32 %r9565, %r8524, %r9568; + mov.u32 %r9566, 0; + +$L__BB2_936: + add.s32 %r6210, %r9566, 17477; + cvt.u64.u32 %rd698, %r6210; + add.s64 %rd699, %rd698, %rd5; + add.s64 %rd700, %rd1, %rd699; + ld.global.u8 %rs778, [%rd700]; + add.s32 %r6211, %r9566, %r9160; + cvt.u64.u32 %rd701, %r6211; + add.s64 %rd702, %rd701, %rd5; + add.s64 %rd703, %rd1, %rd702; + st.global.u8 [%rd703], %rs778; + ld.global.u8 %rs779, [%rd700+1]; + add.s32 %r6212, %r6211, 1; + cvt.u64.u32 %rd704, %r6212; + add.s64 %rd705, %rd704, %rd5; + add.s64 %rd706, %rd1, %rd705; + st.global.u8 [%rd706], %rs779; + ld.global.u8 %rs780, [%rd700+2]; + add.s32 %r6213, %r6211, 2; + cvt.u64.u32 %rd707, %r6213; + add.s64 %rd708, %rd707, %rd5; + add.s64 %rd709, %rd1, %rd708; + st.global.u8 [%rd709], %rs780; + add.s32 %r6214, %r9566, 17480; + cvt.u64.u32 %rd710, %r6214; + add.s64 %rd711, %rd710, %rd5; + add.s64 %rd712, %rd1, %rd711; + ld.global.u8 %rs781, [%rd712]; + add.s32 %r6215, %r6211, 3; + cvt.u64.u32 %rd713, %r6215; + add.s64 %rd714, %rd713, %rd5; + add.s64 %rd715, %rd1, %rd714; + st.global.u8 [%rd715], %rs781; + add.s32 %r9566, %r9566, 4; + add.s32 %r9565, %r9565, -4; + setp.ne.s32 %p1164, %r9565, 0; + @%p1164 bra $L__BB2_936; + +$L__BB2_937: + setp.eq.s32 %p1165, %r9568, 0; + @%p1165 bra $L__BB2_939; + +$L__BB2_938: + .pragma "nounroll"; + add.s32 %r6216, %r9566, 17477; + cvt.u64.u32 %rd716, %r6216; + add.s64 %rd717, %rd716, %rd5; + add.s64 %rd718, %rd1, %rd717; + ld.global.u8 %rs782, [%rd718]; + add.s32 %r6217, %r9566, %r9160; + cvt.u64.u32 %rd719, %r6217; + add.s64 %rd720, %rd719, %rd5; + add.s64 %rd721, %rd1, %rd720; + st.global.u8 [%rd721], %rs782; + add.s32 %r9566, %r9566, 1; + add.s32 %r9568, %r9568, -1; + setp.ne.s32 %p1166, %r9568, 0; + @%p1166 bra $L__BB2_938; + +$L__BB2_939: + setp.eq.s32 %p1167, %r8972, 0; + @%p1167 bra $L__BB2_945; + + mov.u32 %r6219, 20549; + sub.s32 %r2001, %r6219, %r8972; + and.b32 %r9573, %r8972, 3; + add.s32 %r6220, %r8972, -1; + setp.lt.u32 %p1168, %r6220, 3; + mov.u32 %r9571, 0; + @%p1168 bra $L__BB2_943; + + sub.s32 %r9570, %r8972, %r9573; + mov.u32 %r9571, 0; + +$L__BB2_942: + add.s32 %r6222, %r2001, %r9571; + cvt.u64.u32 %rd722, %r6222; + add.s64 %rd723, %rd722, %rd5; + add.s64 %rd724, %rd1, %rd723; + ld.global.u8 %rs783, [%rd724]; + add.s32 %r6223, %r9571, %r1737; + cvt.u64.u32 %rd725, %r6223; + add.s64 %rd726, %rd725, %rd5; + add.s64 %rd727, %rd1, %rd726; + st.global.u8 [%rd727], %rs783; + add.s32 %r6224, %r9571, 1; + add.s32 %r6225, %r2001, %r6224; + cvt.u64.u32 %rd728, %r6225; + add.s64 %rd729, %rd728, %rd5; + add.s64 %rd730, %rd1, %rd729; + ld.global.u8 %rs784, [%rd730]; + add.s32 %r6226, %r6224, %r1737; + cvt.u64.u32 %rd731, %r6226; + add.s64 %rd732, %rd731, %rd5; + add.s64 %rd733, %rd1, %rd732; + st.global.u8 [%rd733], %rs784; + add.s32 %r6227, %r9571, 2; + add.s32 %r6228, %r2001, %r6227; + cvt.u64.u32 %rd734, %r6228; + add.s64 %rd735, %rd734, %rd5; + add.s64 %rd736, %rd1, %rd735; + ld.global.u8 %rs785, [%rd736]; + add.s32 %r6229, %r6227, %r1737; + cvt.u64.u32 %rd737, %r6229; + add.s64 %rd738, %rd737, %rd5; + add.s64 %rd739, %rd1, %rd738; + st.global.u8 [%rd739], %rs785; + add.s32 %r6230, %r9571, 3; + add.s32 %r6231, %r2001, %r6230; + cvt.u64.u32 %rd740, %r6231; + add.s64 %rd741, %rd740, %rd5; + add.s64 %rd742, %rd1, %rd741; + ld.global.u8 %rs786, [%rd742]; + add.s32 %r6232, %r6230, %r1737; + cvt.u64.u32 %rd743, %r6232; + add.s64 %rd744, %rd743, %rd5; + add.s64 %rd745, %rd1, %rd744; + st.global.u8 [%rd745], %rs786; + add.s32 %r9571, %r9571, 4; + add.s32 %r9570, %r9570, -4; + setp.ne.s32 %p1169, %r9570, 0; + @%p1169 bra $L__BB2_942; + +$L__BB2_943: + setp.eq.s32 %p1170, %r9573, 0; + @%p1170 bra $L__BB2_945; + +$L__BB2_944: + .pragma "nounroll"; + add.s32 %r6233, %r2001, %r9571; + cvt.u64.u32 %rd746, %r6233; + add.s64 %rd747, %rd746, %rd5; + add.s64 %rd748, %rd1, %rd747; + ld.global.u8 %rs787, [%rd748]; + add.s32 %r6234, %r9571, %r1737; + cvt.u64.u32 %rd749, %r6234; + add.s64 %rd750, %rd749, %rd5; + add.s64 %rd751, %rd1, %rd750; + st.global.u8 [%rd751], %rs787; + add.s32 %r9571, %r9571, 1; + add.s32 %r9573, %r9573, -1; + setp.ne.s32 %p1171, %r9573, 0; + @%p1171 bra $L__BB2_944; + +$L__BB2_945: + add.s32 %r6235, %r8972, %r8524; + shr.u32 %r6236, %r6235, 4; + add.s32 %r6237, %r1738, -1; + cvt.u64.u32 %rd752, %r6237; + add.s64 %rd753, %rd752, %rd5; + add.s64 %rd754, %rd1, %rd753; + st.global.u8 [%rd754], %r6236; + add.s32 %r6238, %r1738, -2; + cvt.u64.u32 %rd755, %r6238; + add.s64 %rd756, %rd755, %rd5; + add.s64 %rd757, %rd1, %rd756; + ld.global.u8 %rs788, [%rd757]; + and.b16 %rs789, %rs788, 240; + cvt.u16.u32 %rs790, %r6235; + and.b16 %rs791, %rs790, 15; + or.b16 %rs792, %rs789, %rs791; + st.global.u8 [%rd757], %rs792; + setp.eq.s32 %p1172, %r9563, 0; + @%p1172 bra $L__BB2_951; + + add.s32 %r6240, %r9563, -1; + and.b32 %r9578, %r9563, 3; + setp.lt.u32 %p1173, %r6240, 3; + mov.u32 %r9576, 0; + @%p1173 bra $L__BB2_949; + + sub.s32 %r9575, %r9563, %r9578; + mov.u32 %r9576, 0; + +$L__BB2_948: + add.s32 %r6242, %r9576, %r1738; + cvt.u64.u32 %rd758, %r6242; + add.s64 %rd759, %rd758, %rd5; + add.s64 %rd760, %rd1, %rd759; + mov.u16 %rs793, 0; + st.global.u8 [%rd760], %rs793; + add.s32 %r6243, %r6242, 1; + cvt.u64.u32 %rd761, %r6243; + add.s64 %rd762, %rd761, %rd5; + add.s64 %rd763, %rd1, %rd762; + st.global.u8 [%rd763], %rs793; + add.s32 %r6244, %r6242, 2; + cvt.u64.u32 %rd764, %r6244; + add.s64 %rd765, %rd764, %rd5; + add.s64 %rd766, %rd1, %rd765; + st.global.u8 [%rd766], %rs793; + add.s32 %r6245, %r6242, 3; + cvt.u64.u32 %rd767, %r6245; + add.s64 %rd768, %rd767, %rd5; + add.s64 %rd769, %rd1, %rd768; + st.global.u8 [%rd769], %rs793; + add.s32 %r9576, %r9576, 4; + add.s32 %r9575, %r9575, -4; + setp.ne.s32 %p1174, %r9575, 0; + @%p1174 bra $L__BB2_948; + +$L__BB2_949: + setp.eq.s32 %p1175, %r9578, 0; + @%p1175 bra $L__BB2_951; + +$L__BB2_950: + .pragma "nounroll"; + add.s32 %r6246, %r9576, %r1738; + cvt.u64.u32 %rd770, %r6246; + add.s64 %rd771, %rd770, %rd5; + add.s64 %rd772, %rd1, %rd771; + mov.u16 %rs794, 0; + st.global.u8 [%rd772], %rs794; + add.s32 %r9576, %r9576, 1; + add.s32 %r9578, %r9578, -1; + setp.ne.s32 %p1176, %r9578, 0; + @%p1176 bra $L__BB2_950; + +$L__BB2_951: + setp.ne.s32 %p1177, %r4062, 3; + @%p1177 bra $L__BB2_1243; + + ld.param.u64 %rd1410, [ j2k_htj2k_encode_codeblocks_multi_input_param_0]; + cvt.u64.u32 %rd773, %r1738; + add.s64 %rd42, %rd773, %rd5; + add.s64 %rd43, %rd1410, %rd42; + add.s32 %r6247, %r4057, 3; + shr.u32 %r6248, %r6247, 2; + add.s32 %r6249, %r6248, 8; + setp.gt.u32 %p1179, %r6249, 513; + mov.pred %p1178, -1; + mov.pred %p2373, %p1178; + @%p1179 bra $L__BB2_1202; + + mov.u16 %rs1232, 0; + st.local.u16 [%rd23], %rs1232; + st.local.u16 [%rd23+2], %rs1232; + st.local.u16 [%rd23+4], %rs1232; + st.local.u16 [%rd23+6], %rs1232; + st.local.u16 [%rd23+8], %rs1232; + st.local.u16 [%rd23+10], %rs1232; + st.local.u16 [%rd23+12], %rs1232; + st.local.u16 [%rd23+14], %rs1232; + st.local.u16 [%rd23+16], %rs1232; + st.local.u16 [%rd23+18], %rs1232; + st.local.u16 [%rd23+20], %rs1232; + st.local.u16 [%rd23+22], %rs1232; + st.local.u16 [%rd23+24], %rs1232; + st.local.u16 [%rd23+26], %rs1232; + st.local.u16 [%rd23+28], %rs1232; + st.local.u16 [%rd23+30], %rs1232; + st.local.u16 [%rd23+32], %rs1232; + st.local.u16 [%rd23+34], %rs1232; + st.local.u16 [%rd23+36], %rs1232; + st.local.u16 [%rd23+38], %rs1232; + st.local.u16 [%rd23+40], %rs1232; + st.local.u16 [%rd23+42], %rs1232; + st.local.u16 [%rd23+44], %rs1232; + st.local.u16 [%rd23+46], %rs1232; + st.local.u16 [%rd23+48], %rs1232; + st.local.u16 [%rd23+50], %rs1232; + st.local.u16 [%rd23+52], %rs1232; + st.local.u16 [%rd23+54], %rs1232; + st.local.u16 [%rd23+56], %rs1232; + st.local.u16 [%rd23+58], %rs1232; + st.local.u16 [%rd23+60], %rs1232; + st.local.u16 [%rd23+62], %rs1232; + st.local.u16 [%rd23+64], %rs1232; + st.local.u16 [%rd23+66], %rs1232; + st.local.u16 [%rd23+68], %rs1232; + st.local.u16 [%rd23+70], %rs1232; + st.local.u16 [%rd23+72], %rs1232; + st.local.u16 [%rd23+74], %rs1232; + st.local.u16 [%rd23+76], %rs1232; + st.local.u16 [%rd23+78], %rs1232; + st.local.u16 [%rd23+80], %rs1232; + st.local.u16 [%rd23+82], %rs1232; + st.local.u16 [%rd23+84], %rs1232; + st.local.u16 [%rd23+86], %rs1232; + st.local.u16 [%rd23+88], %rs1232; + st.local.u16 [%rd23+90], %rs1232; + st.local.u16 [%rd23+92], %rs1232; + st.local.u16 [%rd23+94], %rs1232; + st.local.u16 [%rd23+96], %rs1232; + st.local.u16 [%rd23+98], %rs1232; + st.local.u16 [%rd23+100], %rs1232; + st.local.u16 [%rd23+102], %rs1232; + st.local.u16 [%rd23+104], %rs1232; + st.local.u16 [%rd23+106], %rs1232; + st.local.u16 [%rd23+108], %rs1232; + st.local.u16 [%rd23+110], %rs1232; + st.local.u16 [%rd23+112], %rs1232; + st.local.u16 [%rd23+114], %rs1232; + st.local.u16 [%rd23+116], %rs1232; + st.local.u16 [%rd23+118], %rs1232; + st.local.u16 [%rd23+120], %rs1232; + st.local.u16 [%rd23+122], %rs1232; + st.local.u16 [%rd23+124], %rs1232; + st.local.u16 [%rd23+126], %rs1232; + st.local.u16 [%rd23+128], %rs1232; + st.local.u16 [%rd23+130], %rs1232; + st.local.u16 [%rd23+132], %rs1232; + st.local.u16 [%rd23+134], %rs1232; + st.local.u16 [%rd23+136], %rs1232; + st.local.u16 [%rd23+138], %rs1232; + st.local.u16 [%rd23+140], %rs1232; + st.local.u16 [%rd23+142], %rs1232; + st.local.u16 [%rd23+144], %rs1232; + st.local.u16 [%rd23+146], %rs1232; + st.local.u16 [%rd23+148], %rs1232; + st.local.u16 [%rd23+150], %rs1232; + st.local.u16 [%rd23+152], %rs1232; + st.local.u16 [%rd23+154], %rs1232; + st.local.u16 [%rd23+156], %rs1232; + st.local.u16 [%rd23+158], %rs1232; + st.local.u16 [%rd23+160], %rs1232; + st.local.u16 [%rd23+162], %rs1232; + st.local.u16 [%rd23+164], %rs1232; + st.local.u16 [%rd23+166], %rs1232; + st.local.u16 [%rd23+168], %rs1232; + st.local.u16 [%rd23+170], %rs1232; + st.local.u16 [%rd23+172], %rs1232; + st.local.u16 [%rd23+174], %rs1232; + st.local.u16 [%rd23+176], %rs1232; + st.local.u16 [%rd23+178], %rs1232; + st.local.u16 [%rd23+180], %rs1232; + st.local.u16 [%rd23+182], %rs1232; + st.local.u16 [%rd23+184], %rs1232; + st.local.u16 [%rd23+186], %rs1232; + st.local.u16 [%rd23+188], %rs1232; + st.local.u16 [%rd23+190], %rs1232; + st.local.u16 [%rd23+192], %rs1232; + st.local.u16 [%rd23+194], %rs1232; + st.local.u16 [%rd23+196], %rs1232; + st.local.u16 [%rd23+198], %rs1232; + st.local.u16 [%rd23+200], %rs1232; + st.local.u16 [%rd23+202], %rs1232; + st.local.u16 [%rd23+204], %rs1232; + st.local.u16 [%rd23+206], %rs1232; + st.local.u16 [%rd23+208], %rs1232; + st.local.u16 [%rd23+210], %rs1232; + st.local.u16 [%rd23+212], %rs1232; + st.local.u16 [%rd23+214], %rs1232; + st.local.u16 [%rd23+216], %rs1232; + st.local.u16 [%rd23+218], %rs1232; + st.local.u16 [%rd23+220], %rs1232; + st.local.u16 [%rd23+222], %rs1232; + st.local.u16 [%rd23+224], %rs1232; + st.local.u16 [%rd23+226], %rs1232; + st.local.u16 [%rd23+228], %rs1232; + st.local.u16 [%rd23+230], %rs1232; + st.local.u16 [%rd23+232], %rs1232; + st.local.u16 [%rd23+234], %rs1232; + st.local.u16 [%rd23+236], %rs1232; + st.local.u16 [%rd23+238], %rs1232; + st.local.u16 [%rd23+240], %rs1232; + st.local.u16 [%rd23+242], %rs1232; + st.local.u16 [%rd23+244], %rs1232; + st.local.u16 [%rd23+246], %rs1232; + st.local.u16 [%rd23+248], %rs1232; + st.local.u16 [%rd23+250], %rs1232; + st.local.u16 [%rd23+252], %rs1232; + st.local.u16 [%rd23+254], %rs1232; + st.local.u16 [%rd23+256], %rs1232; + st.local.u16 [%rd23+258], %rs1232; + st.local.u16 [%rd23+260], %rs1232; + st.local.u16 [%rd23+262], %rs1232; + st.local.u16 [%rd23+264], %rs1232; + st.local.u16 [%rd23+266], %rs1232; + st.local.u16 [%rd23+268], %rs1232; + st.local.u16 [%rd23+270], %rs1232; + st.local.u16 [%rd23+272], %rs1232; + st.local.u16 [%rd23+274], %rs1232; + st.local.u16 [%rd23+276], %rs1232; + st.local.u16 [%rd23+278], %rs1232; + st.local.u16 [%rd23+280], %rs1232; + st.local.u16 [%rd23+282], %rs1232; + st.local.u16 [%rd23+284], %rs1232; + st.local.u16 [%rd23+286], %rs1232; + st.local.u16 [%rd23+288], %rs1232; + st.local.u16 [%rd23+290], %rs1232; + st.local.u16 [%rd23+292], %rs1232; + st.local.u16 [%rd23+294], %rs1232; + st.local.u16 [%rd23+296], %rs1232; + st.local.u16 [%rd23+298], %rs1232; + st.local.u16 [%rd23+300], %rs1232; + st.local.u16 [%rd23+302], %rs1232; + st.local.u16 [%rd23+304], %rs1232; + st.local.u16 [%rd23+306], %rs1232; + st.local.u16 [%rd23+308], %rs1232; + st.local.u16 [%rd23+310], %rs1232; + st.local.u16 [%rd23+312], %rs1232; + st.local.u16 [%rd23+314], %rs1232; + st.local.u16 [%rd23+316], %rs1232; + st.local.u16 [%rd23+318], %rs1232; + st.local.u16 [%rd23+320], %rs1232; + st.local.u16 [%rd23+322], %rs1232; + st.local.u16 [%rd23+324], %rs1232; + st.local.u16 [%rd23+326], %rs1232; + st.local.u16 [%rd23+328], %rs1232; + st.local.u16 [%rd23+330], %rs1232; + st.local.u16 [%rd23+332], %rs1232; + st.local.u16 [%rd23+334], %rs1232; + st.local.u16 [%rd23+336], %rs1232; + st.local.u16 [%rd23+338], %rs1232; + st.local.u16 [%rd23+340], %rs1232; + st.local.u16 [%rd23+342], %rs1232; + st.local.u16 [%rd23+344], %rs1232; + st.local.u16 [%rd23+346], %rs1232; + st.local.u16 [%rd23+348], %rs1232; + st.local.u16 [%rd23+350], %rs1232; + st.local.u16 [%rd23+352], %rs1232; + st.local.u16 [%rd23+354], %rs1232; + st.local.u16 [%rd23+356], %rs1232; + st.local.u16 [%rd23+358], %rs1232; + st.local.u16 [%rd23+360], %rs1232; + st.local.u16 [%rd23+362], %rs1232; + st.local.u16 [%rd23+364], %rs1232; + st.local.u16 [%rd23+366], %rs1232; + st.local.u16 [%rd23+368], %rs1232; + st.local.u16 [%rd23+370], %rs1232; + st.local.u16 [%rd23+372], %rs1232; + st.local.u16 [%rd23+374], %rs1232; + st.local.u16 [%rd23+376], %rs1232; + st.local.u16 [%rd23+378], %rs1232; + st.local.u16 [%rd23+380], %rs1232; + st.local.u16 [%rd23+382], %rs1232; + st.local.u16 [%rd23+384], %rs1232; + st.local.u16 [%rd23+386], %rs1232; + st.local.u16 [%rd23+388], %rs1232; + st.local.u16 [%rd23+390], %rs1232; + st.local.u16 [%rd23+392], %rs1232; + st.local.u16 [%rd23+394], %rs1232; + st.local.u16 [%rd23+396], %rs1232; + st.local.u16 [%rd23+398], %rs1232; + st.local.u16 [%rd23+400], %rs1232; + st.local.u16 [%rd23+402], %rs1232; + st.local.u16 [%rd23+404], %rs1232; + st.local.u16 [%rd23+406], %rs1232; + st.local.u16 [%rd23+408], %rs1232; + st.local.u16 [%rd23+410], %rs1232; + st.local.u16 [%rd23+412], %rs1232; + st.local.u16 [%rd23+414], %rs1232; + st.local.u16 [%rd23+416], %rs1232; + st.local.u16 [%rd23+418], %rs1232; + st.local.u16 [%rd23+420], %rs1232; + st.local.u16 [%rd23+422], %rs1232; + st.local.u16 [%rd23+424], %rs1232; + st.local.u16 [%rd23+426], %rs1232; + st.local.u16 [%rd23+428], %rs1232; + st.local.u16 [%rd23+430], %rs1232; + st.local.u16 [%rd23+432], %rs1232; + st.local.u16 [%rd23+434], %rs1232; + st.local.u16 [%rd23+436], %rs1232; + st.local.u16 [%rd23+438], %rs1232; + st.local.u16 [%rd23+440], %rs1232; + st.local.u16 [%rd23+442], %rs1232; + st.local.u16 [%rd23+444], %rs1232; + st.local.u16 [%rd23+446], %rs1232; + st.local.u16 [%rd23+448], %rs1232; + st.local.u16 [%rd23+450], %rs1232; + st.local.u16 [%rd23+452], %rs1232; + st.local.u16 [%rd23+454], %rs1232; + st.local.u16 [%rd23+456], %rs1232; + st.local.u16 [%rd23+458], %rs1232; + st.local.u16 [%rd23+460], %rs1232; + st.local.u16 [%rd23+462], %rs1232; + st.local.u16 [%rd23+464], %rs1232; + st.local.u16 [%rd23+466], %rs1232; + st.local.u16 [%rd23+468], %rs1232; + st.local.u16 [%rd23+470], %rs1232; + st.local.u16 [%rd23+472], %rs1232; + st.local.u16 [%rd23+474], %rs1232; + st.local.u16 [%rd23+476], %rs1232; + st.local.u16 [%rd23+478], %rs1232; + st.local.u16 [%rd23+480], %rs1232; + st.local.u16 [%rd23+482], %rs1232; + st.local.u16 [%rd23+484], %rs1232; + st.local.u16 [%rd23+486], %rs1232; + st.local.u16 [%rd23+488], %rs1232; + st.local.u16 [%rd23+490], %rs1232; + st.local.u16 [%rd23+492], %rs1232; + st.local.u16 [%rd23+494], %rs1232; + st.local.u16 [%rd23+496], %rs1232; + st.local.u16 [%rd23+498], %rs1232; + st.local.u16 [%rd23+500], %rs1232; + st.local.u16 [%rd23+502], %rs1232; + st.local.u16 [%rd23+504], %rs1232; + st.local.u16 [%rd23+506], %rs1232; + st.local.u16 [%rd23+508], %rs1232; + st.local.u16 [%rd23+510], %rs1232; + st.local.u16 [%rd23+512], %rs1232; + st.local.u16 [%rd23+514], %rs1232; + st.local.u16 [%rd23+516], %rs1232; + st.local.u16 [%rd23+518], %rs1232; + st.local.u16 [%rd23+520], %rs1232; + st.local.u16 [%rd23+522], %rs1232; + st.local.u16 [%rd23+524], %rs1232; + st.local.u16 [%rd23+526], %rs1232; + st.local.u16 [%rd23+528], %rs1232; + st.local.u16 [%rd23+530], %rs1232; + st.local.u16 [%rd23+532], %rs1232; + st.local.u16 [%rd23+534], %rs1232; + st.local.u16 [%rd23+536], %rs1232; + st.local.u16 [%rd23+538], %rs1232; + st.local.u16 [%rd23+540], %rs1232; + st.local.u16 [%rd23+542], %rs1232; + st.local.u16 [%rd23+544], %rs1232; + st.local.u16 [%rd23+546], %rs1232; + st.local.u16 [%rd23+548], %rs1232; + st.local.u16 [%rd23+550], %rs1232; + st.local.u16 [%rd23+552], %rs1232; + st.local.u16 [%rd23+554], %rs1232; + st.local.u16 [%rd23+556], %rs1232; + st.local.u16 [%rd23+558], %rs1232; + st.local.u16 [%rd23+560], %rs1232; + st.local.u16 [%rd23+562], %rs1232; + st.local.u16 [%rd23+564], %rs1232; + st.local.u16 [%rd23+566], %rs1232; + st.local.u16 [%rd23+568], %rs1232; + st.local.u16 [%rd23+570], %rs1232; + st.local.u16 [%rd23+572], %rs1232; + st.local.u16 [%rd23+574], %rs1232; + st.local.u16 [%rd23+576], %rs1232; + st.local.u16 [%rd23+578], %rs1232; + st.local.u16 [%rd23+580], %rs1232; + st.local.u16 [%rd23+582], %rs1232; + st.local.u16 [%rd23+584], %rs1232; + st.local.u16 [%rd23+586], %rs1232; + st.local.u16 [%rd23+588], %rs1232; + st.local.u16 [%rd23+590], %rs1232; + st.local.u16 [%rd23+592], %rs1232; + st.local.u16 [%rd23+594], %rs1232; + st.local.u16 [%rd23+596], %rs1232; + st.local.u16 [%rd23+598], %rs1232; + st.local.u16 [%rd23+600], %rs1232; + st.local.u16 [%rd23+602], %rs1232; + st.local.u16 [%rd23+604], %rs1232; + st.local.u16 [%rd23+606], %rs1232; + st.local.u16 [%rd23+608], %rs1232; + st.local.u16 [%rd23+610], %rs1232; + st.local.u16 [%rd23+612], %rs1232; + st.local.u16 [%rd23+614], %rs1232; + st.local.u16 [%rd23+616], %rs1232; + st.local.u16 [%rd23+618], %rs1232; + st.local.u16 [%rd23+620], %rs1232; + st.local.u16 [%rd23+622], %rs1232; + st.local.u16 [%rd23+624], %rs1232; + st.local.u16 [%rd23+626], %rs1232; + st.local.u16 [%rd23+628], %rs1232; + st.local.u16 [%rd23+630], %rs1232; + st.local.u16 [%rd23+632], %rs1232; + st.local.u16 [%rd23+634], %rs1232; + st.local.u16 [%rd23+636], %rs1232; + st.local.u16 [%rd23+638], %rs1232; + st.local.u16 [%rd23+640], %rs1232; + st.local.u16 [%rd23+642], %rs1232; + st.local.u16 [%rd23+644], %rs1232; + st.local.u16 [%rd23+646], %rs1232; + st.local.u16 [%rd23+648], %rs1232; + st.local.u16 [%rd23+650], %rs1232; + st.local.u16 [%rd23+652], %rs1232; + st.local.u16 [%rd23+654], %rs1232; + st.local.u16 [%rd23+656], %rs1232; + st.local.u16 [%rd23+658], %rs1232; + st.local.u16 [%rd23+660], %rs1232; + st.local.u16 [%rd23+662], %rs1232; + st.local.u16 [%rd23+664], %rs1232; + st.local.u16 [%rd23+666], %rs1232; + st.local.u16 [%rd23+668], %rs1232; + st.local.u16 [%rd23+670], %rs1232; + st.local.u16 [%rd23+672], %rs1232; + st.local.u16 [%rd23+674], %rs1232; + st.local.u16 [%rd23+676], %rs1232; + st.local.u16 [%rd23+678], %rs1232; + st.local.u16 [%rd23+680], %rs1232; + st.local.u16 [%rd23+682], %rs1232; + st.local.u16 [%rd23+684], %rs1232; + st.local.u16 [%rd23+686], %rs1232; + st.local.u16 [%rd23+688], %rs1232; + st.local.u16 [%rd23+690], %rs1232; + st.local.u16 [%rd23+692], %rs1232; + st.local.u16 [%rd23+694], %rs1232; + st.local.u16 [%rd23+696], %rs1232; + st.local.u16 [%rd23+698], %rs1232; + st.local.u16 [%rd23+700], %rs1232; + st.local.u16 [%rd23+702], %rs1232; + st.local.u16 [%rd23+704], %rs1232; + st.local.u16 [%rd23+706], %rs1232; + st.local.u16 [%rd23+708], %rs1232; + st.local.u16 [%rd23+710], %rs1232; + st.local.u16 [%rd23+712], %rs1232; + st.local.u16 [%rd23+714], %rs1232; + st.local.u16 [%rd23+716], %rs1232; + st.local.u16 [%rd23+718], %rs1232; + st.local.u16 [%rd23+720], %rs1232; + st.local.u16 [%rd23+722], %rs1232; + st.local.u16 [%rd23+724], %rs1232; + st.local.u16 [%rd23+726], %rs1232; + st.local.u16 [%rd23+728], %rs1232; + st.local.u16 [%rd23+730], %rs1232; + st.local.u16 [%rd23+732], %rs1232; + st.local.u16 [%rd23+734], %rs1232; + st.local.u16 [%rd23+736], %rs1232; + st.local.u16 [%rd23+738], %rs1232; + st.local.u16 [%rd23+740], %rs1232; + st.local.u16 [%rd23+742], %rs1232; + st.local.u16 [%rd23+744], %rs1232; + st.local.u16 [%rd23+746], %rs1232; + st.local.u16 [%rd23+748], %rs1232; + st.local.u16 [%rd23+750], %rs1232; + st.local.u16 [%rd23+752], %rs1232; + st.local.u16 [%rd23+754], %rs1232; + st.local.u16 [%rd23+756], %rs1232; + st.local.u16 [%rd23+758], %rs1232; + st.local.u16 [%rd23+760], %rs1232; + st.local.u16 [%rd23+762], %rs1232; + st.local.u16 [%rd23+764], %rs1232; + st.local.u16 [%rd23+766], %rs1232; + st.local.u16 [%rd23+768], %rs1232; + st.local.u16 [%rd23+770], %rs1232; + st.local.u16 [%rd23+772], %rs1232; + st.local.u16 [%rd23+774], %rs1232; + st.local.u16 [%rd23+776], %rs1232; + st.local.u16 [%rd23+778], %rs1232; + st.local.u16 [%rd23+780], %rs1232; + st.local.u16 [%rd23+782], %rs1232; + st.local.u16 [%rd23+784], %rs1232; + st.local.u16 [%rd23+786], %rs1232; + st.local.u16 [%rd23+788], %rs1232; + st.local.u16 [%rd23+790], %rs1232; + st.local.u16 [%rd23+792], %rs1232; + st.local.u16 [%rd23+794], %rs1232; + st.local.u16 [%rd23+796], %rs1232; + st.local.u16 [%rd23+798], %rs1232; + st.local.u16 [%rd23+800], %rs1232; + st.local.u16 [%rd23+802], %rs1232; + st.local.u16 [%rd23+804], %rs1232; + st.local.u16 [%rd23+806], %rs1232; + st.local.u16 [%rd23+808], %rs1232; + st.local.u16 [%rd23+810], %rs1232; + st.local.u16 [%rd23+812], %rs1232; + st.local.u16 [%rd23+814], %rs1232; + st.local.u16 [%rd23+816], %rs1232; + st.local.u16 [%rd23+818], %rs1232; + st.local.u16 [%rd23+820], %rs1232; + st.local.u16 [%rd23+822], %rs1232; + st.local.u16 [%rd23+824], %rs1232; + st.local.u16 [%rd23+826], %rs1232; + st.local.u16 [%rd23+828], %rs1232; + st.local.u16 [%rd23+830], %rs1232; + st.local.u16 [%rd23+832], %rs1232; + st.local.u16 [%rd23+834], %rs1232; + st.local.u16 [%rd23+836], %rs1232; + st.local.u16 [%rd23+838], %rs1232; + st.local.u16 [%rd23+840], %rs1232; + st.local.u16 [%rd23+842], %rs1232; + st.local.u16 [%rd23+844], %rs1232; + st.local.u16 [%rd23+846], %rs1232; + st.local.u16 [%rd23+848], %rs1232; + st.local.u16 [%rd23+850], %rs1232; + st.local.u16 [%rd23+852], %rs1232; + st.local.u16 [%rd23+854], %rs1232; + st.local.u16 [%rd23+856], %rs1232; + st.local.u16 [%rd23+858], %rs1232; + st.local.u16 [%rd23+860], %rs1232; + st.local.u16 [%rd23+862], %rs1232; + st.local.u16 [%rd23+864], %rs1232; + st.local.u16 [%rd23+866], %rs1232; + st.local.u16 [%rd23+868], %rs1232; + st.local.u16 [%rd23+870], %rs1232; + st.local.u16 [%rd23+872], %rs1232; + st.local.u16 [%rd23+874], %rs1232; + st.local.u16 [%rd23+876], %rs1232; + st.local.u16 [%rd23+878], %rs1232; + st.local.u16 [%rd23+880], %rs1232; + st.local.u16 [%rd23+882], %rs1232; + st.local.u16 [%rd23+884], %rs1232; + st.local.u16 [%rd23+886], %rs1232; + st.local.u16 [%rd23+888], %rs1232; + st.local.u16 [%rd23+890], %rs1232; + st.local.u16 [%rd23+892], %rs1232; + st.local.u16 [%rd23+894], %rs1232; + st.local.u16 [%rd23+896], %rs1232; + st.local.u16 [%rd23+898], %rs1232; + st.local.u16 [%rd23+900], %rs1232; + st.local.u16 [%rd23+902], %rs1232; + st.local.u16 [%rd23+904], %rs1232; + st.local.u16 [%rd23+906], %rs1232; + st.local.u16 [%rd23+908], %rs1232; + st.local.u16 [%rd23+910], %rs1232; + st.local.u16 [%rd23+912], %rs1232; + st.local.u16 [%rd23+914], %rs1232; + st.local.u16 [%rd23+916], %rs1232; + st.local.u16 [%rd23+918], %rs1232; + st.local.u16 [%rd23+920], %rs1232; + st.local.u16 [%rd23+922], %rs1232; + st.local.u16 [%rd23+924], %rs1232; + st.local.u16 [%rd23+926], %rs1232; + st.local.u16 [%rd23+928], %rs1232; + st.local.u16 [%rd23+930], %rs1232; + st.local.u16 [%rd23+932], %rs1232; + st.local.u16 [%rd23+934], %rs1232; + st.local.u16 [%rd23+936], %rs1232; + st.local.u16 [%rd23+938], %rs1232; + st.local.u16 [%rd23+940], %rs1232; + st.local.u16 [%rd23+942], %rs1232; + st.local.u16 [%rd23+944], %rs1232; + st.local.u16 [%rd23+946], %rs1232; + st.local.u16 [%rd23+948], %rs1232; + st.local.u16 [%rd23+950], %rs1232; + st.local.u16 [%rd23+952], %rs1232; + st.local.u16 [%rd23+954], %rs1232; + st.local.u16 [%rd23+956], %rs1232; + st.local.u16 [%rd23+958], %rs1232; + st.local.u16 [%rd23+960], %rs1232; + st.local.u16 [%rd23+962], %rs1232; + st.local.u16 [%rd23+964], %rs1232; + st.local.u16 [%rd23+966], %rs1232; + st.local.u16 [%rd23+968], %rs1232; + st.local.u16 [%rd23+970], %rs1232; + st.local.u16 [%rd23+972], %rs1232; + st.local.u16 [%rd23+974], %rs1232; + st.local.u16 [%rd23+976], %rs1232; + st.local.u16 [%rd23+978], %rs1232; + st.local.u16 [%rd23+980], %rs1232; + st.local.u16 [%rd23+982], %rs1232; + st.local.u16 [%rd23+984], %rs1232; + st.local.u16 [%rd23+986], %rs1232; + st.local.u16 [%rd23+988], %rs1232; + st.local.u16 [%rd23+990], %rs1232; + st.local.u16 [%rd23+992], %rs1232; + st.local.u16 [%rd23+994], %rs1232; + st.local.u16 [%rd23+996], %rs1232; + st.local.u16 [%rd23+998], %rs1232; + st.local.u16 [%rd23+1000], %rs1232; + st.local.u16 [%rd23+1002], %rs1232; + st.local.u16 [%rd23+1004], %rs1232; + st.local.u16 [%rd23+1006], %rs1232; + st.local.u16 [%rd23+1008], %rs1232; + st.local.u16 [%rd23+1010], %rs1232; + st.local.u16 [%rd23+1012], %rs1232; + st.local.u16 [%rd23+1014], %rs1232; + st.local.u16 [%rd23+1016], %rs1232; + st.local.u16 [%rd23+1018], %rs1232; + st.local.u16 [%rd23+1020], %rs1232; + st.local.u16 [%rd23+1022], %rs1232; + st.local.u16 [%rd23+1024], %rs1232; + mov.u32 %r9579, 0; + mov.u32 %r9689, %r9579; + mov.u32 %r9685, %r9579; + mov.u32 %r9687, %r9579; + +$L__BB2_954: + @%p10 bra $L__BB2_1197; + + sub.s32 %r6256, %r4058, %r9579; + add.s32 %r2028, %r9579, 4; + mul.lo.s32 %r2029, %r2028, %r4055; + add.s32 %r2030, %r9579, 5; + add.s32 %r2031, %r2029, %r4055; + add.s32 %r2032, %r9579, 6; + shl.b32 %r6257, %r4055, 1; + add.s32 %r2033, %r2029, %r6257; + add.s32 %r2034, %r9579, 7; + mul.lo.s32 %r6258, %r4055, 3; + add.s32 %r2035, %r2029, %r6258; + add.s32 %r2036, %r9579, 1; + add.s32 %r2037, %r9579, 2; + add.s32 %r2038, %r9579, 3; + mul.lo.s32 %r2039, %r9579, %r4055; + add.s32 %r2040, %r2039, %r6258; + sub.s32 %r2041, %r2040, %r4055; + sub.s32 %r2042, %r2041, %r4055; + setp.lt.u32 %p1181, %r6256, 2; + selp.b32 %r6259, 4369, 13107, %p1181; + setp.lt.u32 %p1182, %r6256, 3; + selp.b32 %r6260, %r6259, 30583, %p1182; + setp.lt.u32 %p1183, %r6256, 4; + selp.b32 %r2043, %r6260, 65535, %p1183; + mov.u32 %r6255, 0; + mov.u32 %r9583, %r6255; + mov.u32 %r9584, %r6255; + +$L__BB2_956: + shr.u32 %r6262, %r9583, 2; + mul.wide.u32 %rd774, %r6262, 2; + add.s64 %rd44, %rd23, %rd774; + ld.local.u16 %rs250, [%rd44]; + ld.local.u16 %rs251, [%rd44+2]; + setp.ge.u32 %p1184, %r9583, %r4057; + mov.u32 %r9595, %r6255; + @%p1184 bra $L__BB2_965; + + setp.ge.u32 %p1185, %r2028, %r4058; + mov.u32 %r9595, 0; + @%p1185 bra $L__BB2_959; + + add.s32 %r6264, %r2029, %r9583; + cvt.u64.u32 %rd775, %r6264; + add.s64 %rd776, %rd775, %rd4; + shl.b64 %rd777, %rd776, 2; + add.s64 %rd778, %rd3, %rd777; + ld.global.u32 %r6265, [%rd778]; + abs.s32 %r6266, %r6265; + setp.gt.u32 %p1186, %r6266, 4; + and.b32 %r6267, %r6266, 1; + setp.eq.b32 %p1187, %r6267, 1; + and.pred %p1188, %p1186, %p1187; + selp.u32 %r9595, 1, 0, %p1188; + +$L__BB2_959: + setp.ge.u32 %p1189, %r2030, %r4058; + @%p1189 bra $L__BB2_961; + + add.s32 %r6268, %r2031, %r9583; + cvt.u64.u32 %rd779, %r6268; + add.s64 %rd780, %rd779, %rd4; + shl.b64 %rd781, %rd780, 2; + add.s64 %rd782, %rd3, %rd781; + ld.global.u32 %r6269, [%rd782]; + abs.s32 %r6270, %r6269; + setp.gt.u32 %p1190, %r6270, 4; + and.b32 %r6271, %r6270, 1; + setp.eq.b32 %p1191, %r6271, 1; + and.pred %p1192, %p1190, %p1191; + selp.b32 %r6272, 2, 0, %p1192; + or.b32 %r9595, %r6272, %r9595; + +$L__BB2_961: + setp.ge.u32 %p1193, %r2032, %r4058; + @%p1193 bra $L__BB2_963; + + add.s32 %r6273, %r2033, %r9583; + cvt.u64.u32 %rd783, %r6273; + add.s64 %rd784, %rd783, %rd4; + shl.b64 %rd785, %rd784, 2; + add.s64 %rd786, %rd3, %rd785; + ld.global.u32 %r6274, [%rd786]; + abs.s32 %r6275, %r6274; + setp.gt.u32 %p1194, %r6275, 4; + and.b32 %r6276, %r6275, 1; + setp.eq.b32 %p1195, %r6276, 1; + and.pred %p1196, %p1194, %p1195; + selp.b32 %r6277, 4, 0, %p1196; + or.b32 %r9595, %r6277, %r9595; + +$L__BB2_963: + setp.ge.u32 %p1197, %r2034, %r4058; + @%p1197 bra $L__BB2_965; + + add.s32 %r6278, %r2035, %r9583; + cvt.u64.u32 %rd787, %r6278; + add.s64 %rd788, %rd787, %rd4; + shl.b64 %rd789, %rd788, 2; + add.s64 %rd790, %rd3, %rd789; + ld.global.u32 %r6279, [%rd790]; + abs.s32 %r6280, %r6279; + setp.gt.u32 %p1198, %r6280, 4; + and.b32 %r6281, %r6280, 1; + setp.eq.b32 %p1199, %r6281, 1; + and.pred %p1200, %p1198, %p1199; + selp.b32 %r6282, 8, 0, %p1200; + or.b32 %r9595, %r6282, %r9595; + +$L__BB2_965: + add.s32 %r2057, %r9583, 1; + setp.ge.u32 %p1201, %r2057, %r4057; + @%p1201 bra $L__BB2_974; + + setp.ge.u32 %p1202, %r2028, %r4058; + @%p1202 bra $L__BB2_968; + + add.s32 %r6283, %r2029, %r2057; + cvt.u64.u32 %rd791, %r6283; + add.s64 %rd792, %rd791, %rd4; + shl.b64 %rd793, %rd792, 2; + add.s64 %rd794, %rd3, %rd793; + ld.global.u32 %r6284, [%rd794]; + abs.s32 %r6285, %r6284; + setp.gt.u32 %p1203, %r6285, 4; + and.b32 %r6286, %r6285, 1; + setp.eq.b32 %p1204, %r6286, 1; + and.pred %p1205, %p1203, %p1204; + selp.b32 %r6287, 16, 0, %p1205; + or.b32 %r9595, %r6287, %r9595; + +$L__BB2_968: + setp.ge.u32 %p1206, %r2030, %r4058; + @%p1206 bra $L__BB2_970; + + add.s32 %r6288, %r2031, %r2057; + cvt.u64.u32 %rd795, %r6288; + add.s64 %rd796, %rd795, %rd4; + shl.b64 %rd797, %rd796, 2; + add.s64 %rd798, %rd3, %rd797; + ld.global.u32 %r6289, [%rd798]; + abs.s32 %r6290, %r6289; + setp.gt.u32 %p1207, %r6290, 4; + and.b32 %r6291, %r6290, 1; + setp.eq.b32 %p1208, %r6291, 1; + and.pred %p1209, %p1207, %p1208; + selp.b32 %r6292, 32, 0, %p1209; + or.b32 %r9595, %r6292, %r9595; + +$L__BB2_970: + setp.ge.u32 %p1210, %r2032, %r4058; + @%p1210 bra $L__BB2_972; + + add.s32 %r6293, %r2033, %r2057; + cvt.u64.u32 %rd799, %r6293; + add.s64 %rd800, %rd799, %rd4; + shl.b64 %rd801, %rd800, 2; + add.s64 %rd802, %rd3, %rd801; + ld.global.u32 %r6294, [%rd802]; + abs.s32 %r6295, %r6294; + setp.gt.u32 %p1211, %r6295, 4; + and.b32 %r6296, %r6295, 1; + setp.eq.b32 %p1212, %r6296, 1; + and.pred %p1213, %p1211, %p1212; + selp.b32 %r6297, 64, 0, %p1213; + or.b32 %r9595, %r6297, %r9595; + +$L__BB2_972: + setp.ge.u32 %p1214, %r2034, %r4058; + @%p1214 bra $L__BB2_974; + + add.s32 %r6298, %r2035, %r2057; + cvt.u64.u32 %rd803, %r6298; + add.s64 %rd804, %rd803, %rd4; + shl.b64 %rd805, %rd804, 2; + add.s64 %rd806, %rd3, %rd805; + ld.global.u32 %r6299, [%rd806]; + abs.s32 %r6300, %r6299; + setp.gt.u32 %p1215, %r6300, 4; + and.b32 %r6301, %r6300, 1; + setp.eq.b32 %p1216, %r6301, 1; + and.pred %p1217, %p1215, %p1216; + selp.b32 %r6302, 128, 0, %p1217; + or.b32 %r9595, %r6302, %r9595; + +$L__BB2_974: + add.s32 %r2066, %r9583, 2; + setp.ge.u32 %p1218, %r2066, %r4057; + @%p1218 bra $L__BB2_983; + + setp.ge.u32 %p1219, %r2028, %r4058; + @%p1219 bra $L__BB2_977; + + add.s32 %r6303, %r2029, %r2066; + cvt.u64.u32 %rd807, %r6303; + add.s64 %rd808, %rd807, %rd4; + shl.b64 %rd809, %rd808, 2; + add.s64 %rd810, %rd3, %rd809; + ld.global.u32 %r6304, [%rd810]; + abs.s32 %r6305, %r6304; + setp.gt.u32 %p1220, %r6305, 4; + and.b32 %r6306, %r6305, 1; + setp.eq.b32 %p1221, %r6306, 1; + and.pred %p1222, %p1220, %p1221; + selp.b32 %r6307, 256, 0, %p1222; + or.b32 %r9595, %r6307, %r9595; + +$L__BB2_977: + setp.ge.u32 %p1223, %r2030, %r4058; + @%p1223 bra $L__BB2_979; + + add.s32 %r6308, %r2031, %r2066; + cvt.u64.u32 %rd811, %r6308; + add.s64 %rd812, %rd811, %rd4; + shl.b64 %rd813, %rd812, 2; + add.s64 %rd814, %rd3, %rd813; + ld.global.u32 %r6309, [%rd814]; + abs.s32 %r6310, %r6309; + setp.gt.u32 %p1224, %r6310, 4; + and.b32 %r6311, %r6310, 1; + setp.eq.b32 %p1225, %r6311, 1; + and.pred %p1226, %p1224, %p1225; + selp.b32 %r6312, 512, 0, %p1226; + or.b32 %r9595, %r6312, %r9595; + +$L__BB2_979: + setp.ge.u32 %p1227, %r2032, %r4058; + @%p1227 bra $L__BB2_981; + + add.s32 %r6313, %r2033, %r2066; + cvt.u64.u32 %rd815, %r6313; + add.s64 %rd816, %rd815, %rd4; + shl.b64 %rd817, %rd816, 2; + add.s64 %rd818, %rd3, %rd817; + ld.global.u32 %r6314, [%rd818]; + abs.s32 %r6315, %r6314; + setp.gt.u32 %p1228, %r6315, 4; + and.b32 %r6316, %r6315, 1; + setp.eq.b32 %p1229, %r6316, 1; + and.pred %p1230, %p1228, %p1229; + selp.b32 %r6317, 1024, 0, %p1230; + or.b32 %r9595, %r6317, %r9595; + +$L__BB2_981: + setp.ge.u32 %p1231, %r2034, %r4058; + @%p1231 bra $L__BB2_983; + + add.s32 %r6318, %r2035, %r2066; + cvt.u64.u32 %rd819, %r6318; + add.s64 %rd820, %rd819, %rd4; + shl.b64 %rd821, %rd820, 2; + add.s64 %rd822, %rd3, %rd821; + ld.global.u32 %r6319, [%rd822]; + abs.s32 %r6320, %r6319; + setp.gt.u32 %p1232, %r6320, 4; + and.b32 %r6321, %r6320, 1; + setp.eq.b32 %p1233, %r6321, 1; + and.pred %p1234, %p1232, %p1233; + selp.b32 %r6322, 2048, 0, %p1234; + or.b32 %r9595, %r6322, %r9595; + +$L__BB2_983: + add.s32 %r2075, %r9583, 3; + setp.ge.u32 %p1235, %r2075, %r4057; + @%p1235 bra $L__BB2_992; + + setp.ge.u32 %p1236, %r2028, %r4058; + @%p1236 bra $L__BB2_986; + + add.s32 %r6323, %r2029, %r2075; + cvt.u64.u32 %rd823, %r6323; + add.s64 %rd824, %rd823, %rd4; + shl.b64 %rd825, %rd824, 2; + add.s64 %rd826, %rd3, %rd825; + ld.global.u32 %r6324, [%rd826]; + abs.s32 %r6325, %r6324; + setp.gt.u32 %p1237, %r6325, 4; + and.b32 %r6326, %r6325, 1; + setp.eq.b32 %p1238, %r6326, 1; + and.pred %p1239, %p1237, %p1238; + selp.b32 %r6327, 4096, 0, %p1239; + or.b32 %r9595, %r6327, %r9595; + +$L__BB2_986: + setp.ge.u32 %p1240, %r2030, %r4058; + @%p1240 bra $L__BB2_988; + + add.s32 %r6328, %r2031, %r2075; + cvt.u64.u32 %rd827, %r6328; + add.s64 %rd828, %rd827, %rd4; + shl.b64 %rd829, %rd828, 2; + add.s64 %rd830, %rd3, %rd829; + ld.global.u32 %r6329, [%rd830]; + abs.s32 %r6330, %r6329; + setp.gt.u32 %p1241, %r6330, 4; + and.b32 %r6331, %r6330, 1; + setp.eq.b32 %p1242, %r6331, 1; + and.pred %p1243, %p1241, %p1242; + selp.b32 %r6332, 8192, 0, %p1243; + or.b32 %r9595, %r6332, %r9595; + +$L__BB2_988: + setp.ge.u32 %p1244, %r2032, %r4058; + @%p1244 bra $L__BB2_990; + + add.s32 %r6333, %r2033, %r2075; + cvt.u64.u32 %rd831, %r6333; + add.s64 %rd832, %rd831, %rd4; + shl.b64 %rd833, %rd832, 2; + add.s64 %rd834, %rd3, %rd833; + ld.global.u32 %r6334, [%rd834]; + abs.s32 %r6335, %r6334; + setp.gt.u32 %p1245, %r6335, 4; + and.b32 %r6336, %r6335, 1; + setp.eq.b32 %p1246, %r6336, 1; + and.pred %p1247, %p1245, %p1246; + selp.b32 %r6337, 16384, 0, %p1247; + or.b32 %r9595, %r6337, %r9595; + +$L__BB2_990: + setp.ge.u32 %p1248, %r2034, %r4058; + @%p1248 bra $L__BB2_992; + + add.s32 %r6338, %r2035, %r2075; + cvt.u64.u32 %rd835, %r6338; + add.s64 %rd836, %rd835, %rd4; + shl.b64 %rd837, %rd836, 2; + add.s64 %rd838, %rd3, %rd837; + ld.global.u32 %r6339, [%rd838]; + abs.s32 %r6340, %r6339; + setp.gt.u32 %p1249, %r6340, 4; + and.b32 %r6341, %r6340, 1; + setp.eq.b32 %p1250, %r6341, 1; + and.pred %p1251, %p1249, %p1250; + selp.b32 %r6342, 32768, 0, %p1251; + or.b32 %r9595, %r6342, %r9595; + +$L__BB2_992: + add.s32 %r6344, %r9583, 4; + setp.ge.u32 %p1252, %r6344, %r4057; + mov.u32 %r9611, 0; + @%p1252 bra $L__BB2_1001; + + setp.ge.u32 %p1253, %r2028, %r4058; + mov.u32 %r9611, 0; + @%p1253 bra $L__BB2_995; + + add.s32 %r6346, %r2029, %r9583; + add.s32 %r6347, %r6346, 4; + cvt.u64.u32 %rd839, %r6347; + add.s64 %rd840, %rd839, %rd4; + shl.b64 %rd841, %rd840, 2; + add.s64 %rd842, %rd3, %rd841; + ld.global.u32 %r6348, [%rd842]; + abs.s32 %r6349, %r6348; + setp.gt.u32 %p1254, %r6349, 4; + and.b32 %r6350, %r6349, 1; + setp.eq.b32 %p1255, %r6350, 1; + and.pred %p1256, %p1254, %p1255; + selp.u32 %r9611, 1, 0, %p1256; + +$L__BB2_995: + setp.ge.u32 %p1257, %r2030, %r4058; + @%p1257 bra $L__BB2_997; + + add.s32 %r6351, %r2031, %r9583; + add.s32 %r6352, %r6351, 4; + cvt.u64.u32 %rd843, %r6352; + add.s64 %rd844, %rd843, %rd4; + shl.b64 %rd845, %rd844, 2; + add.s64 %rd846, %rd3, %rd845; + ld.global.u32 %r6353, [%rd846]; + abs.s32 %r6354, %r6353; + setp.gt.u32 %p1258, %r6354, 4; + and.b32 %r6355, %r6354, 1; + setp.eq.b32 %p1259, %r6355, 1; + and.pred %p1260, %p1258, %p1259; + selp.b32 %r6356, 2, 0, %p1260; + or.b32 %r9611, %r6356, %r9611; + +$L__BB2_997: + setp.ge.u32 %p1261, %r2032, %r4058; + @%p1261 bra $L__BB2_999; + + add.s32 %r6357, %r2033, %r9583; + add.s32 %r6358, %r6357, 4; + cvt.u64.u32 %rd847, %r6358; + add.s64 %rd848, %rd847, %rd4; + shl.b64 %rd849, %rd848, 2; + add.s64 %rd850, %rd3, %rd849; + ld.global.u32 %r6359, [%rd850]; + abs.s32 %r6360, %r6359; + setp.gt.u32 %p1262, %r6360, 4; + and.b32 %r6361, %r6360, 1; + setp.eq.b32 %p1263, %r6361, 1; + and.pred %p1264, %p1262, %p1263; + selp.b32 %r6362, 4, 0, %p1264; + or.b32 %r9611, %r6362, %r9611; + +$L__BB2_999: + setp.ge.u32 %p1265, %r2034, %r4058; + @%p1265 bra $L__BB2_1001; + + add.s32 %r6363, %r2035, %r9583; + add.s32 %r6364, %r6363, 4; + cvt.u64.u32 %rd851, %r6364; + add.s64 %rd852, %rd851, %rd4; + shl.b64 %rd853, %rd852, 2; + add.s64 %rd854, %rd3, %rd853; + ld.global.u32 %r6365, [%rd854]; + abs.s32 %r6366, %r6365; + setp.gt.u32 %p1266, %r6366, 4; + and.b32 %r6367, %r6366, 1; + setp.eq.b32 %p1267, %r6367, 1; + and.pred %p1268, %p1266, %p1267; + selp.b32 %r6368, 8, 0, %p1268; + or.b32 %r9611, %r6368, %r9611; + +$L__BB2_1001: + add.s32 %r2092, %r9583, 5; + setp.ge.u32 %p1269, %r2092, %r4057; + @%p1269 bra $L__BB2_1010; + + setp.ge.u32 %p1270, %r2028, %r4058; + @%p1270 bra $L__BB2_1004; + + add.s32 %r6369, %r2029, %r2092; + cvt.u64.u32 %rd855, %r6369; + add.s64 %rd856, %rd855, %rd4; + shl.b64 %rd857, %rd856, 2; + add.s64 %rd858, %rd3, %rd857; + ld.global.u32 %r6370, [%rd858]; + abs.s32 %r6371, %r6370; + setp.gt.u32 %p1271, %r6371, 4; + and.b32 %r6372, %r6371, 1; + setp.eq.b32 %p1272, %r6372, 1; + and.pred %p1273, %p1271, %p1272; + selp.b32 %r6373, 16, 0, %p1273; + or.b32 %r9611, %r6373, %r9611; + +$L__BB2_1004: + setp.ge.u32 %p1274, %r2030, %r4058; + @%p1274 bra $L__BB2_1006; + + add.s32 %r6374, %r2031, %r2092; + cvt.u64.u32 %rd859, %r6374; + add.s64 %rd860, %rd859, %rd4; + shl.b64 %rd861, %rd860, 2; + add.s64 %rd862, %rd3, %rd861; + ld.global.u32 %r6375, [%rd862]; + abs.s32 %r6376, %r6375; + setp.gt.u32 %p1275, %r6376, 4; + and.b32 %r6377, %r6376, 1; + setp.eq.b32 %p1276, %r6377, 1; + and.pred %p1277, %p1275, %p1276; + selp.b32 %r6378, 32, 0, %p1277; + or.b32 %r9611, %r6378, %r9611; + +$L__BB2_1006: + setp.ge.u32 %p1278, %r2032, %r4058; + @%p1278 bra $L__BB2_1008; + + add.s32 %r6379, %r2033, %r2092; + cvt.u64.u32 %rd863, %r6379; + add.s64 %rd864, %rd863, %rd4; + shl.b64 %rd865, %rd864, 2; + add.s64 %rd866, %rd3, %rd865; + ld.global.u32 %r6380, [%rd866]; + abs.s32 %r6381, %r6380; + setp.gt.u32 %p1279, %r6381, 4; + and.b32 %r6382, %r6381, 1; + setp.eq.b32 %p1280, %r6382, 1; + and.pred %p1281, %p1279, %p1280; + selp.b32 %r6383, 64, 0, %p1281; + or.b32 %r9611, %r6383, %r9611; + +$L__BB2_1008: + setp.ge.u32 %p1282, %r2034, %r4058; + @%p1282 bra $L__BB2_1010; + + add.s32 %r6384, %r2035, %r2092; + cvt.u64.u32 %rd867, %r6384; + add.s64 %rd868, %rd867, %rd4; + shl.b64 %rd869, %rd868, 2; + add.s64 %rd870, %rd3, %rd869; + ld.global.u32 %r6385, [%rd870]; + abs.s32 %r6386, %r6385; + setp.gt.u32 %p1283, %r6386, 4; + and.b32 %r6387, %r6386, 1; + setp.eq.b32 %p1284, %r6387, 1; + and.pred %p1285, %p1283, %p1284; + selp.b32 %r6388, 128, 0, %p1285; + or.b32 %r9611, %r6388, %r9611; + +$L__BB2_1010: + add.s32 %r2101, %r9583, 6; + setp.ge.u32 %p1286, %r2101, %r4057; + @%p1286 bra $L__BB2_1019; + + setp.ge.u32 %p1287, %r2028, %r4058; + @%p1287 bra $L__BB2_1013; + + add.s32 %r6389, %r2029, %r2101; + cvt.u64.u32 %rd871, %r6389; + add.s64 %rd872, %rd871, %rd4; + shl.b64 %rd873, %rd872, 2; + add.s64 %rd874, %rd3, %rd873; + ld.global.u32 %r6390, [%rd874]; + abs.s32 %r6391, %r6390; + setp.gt.u32 %p1288, %r6391, 4; + and.b32 %r6392, %r6391, 1; + setp.eq.b32 %p1289, %r6392, 1; + and.pred %p1290, %p1288, %p1289; + selp.b32 %r6393, 256, 0, %p1290; + or.b32 %r9611, %r6393, %r9611; + +$L__BB2_1013: + setp.ge.u32 %p1291, %r2030, %r4058; + @%p1291 bra $L__BB2_1015; + + add.s32 %r6394, %r2031, %r2101; + cvt.u64.u32 %rd875, %r6394; + add.s64 %rd876, %rd875, %rd4; + shl.b64 %rd877, %rd876, 2; + add.s64 %rd878, %rd3, %rd877; + ld.global.u32 %r6395, [%rd878]; + abs.s32 %r6396, %r6395; + setp.gt.u32 %p1292, %r6396, 4; + and.b32 %r6397, %r6396, 1; + setp.eq.b32 %p1293, %r6397, 1; + and.pred %p1294, %p1292, %p1293; + selp.b32 %r6398, 512, 0, %p1294; + or.b32 %r9611, %r6398, %r9611; + +$L__BB2_1015: + setp.ge.u32 %p1295, %r2032, %r4058; + @%p1295 bra $L__BB2_1017; + + add.s32 %r6399, %r2033, %r2101; + cvt.u64.u32 %rd879, %r6399; + add.s64 %rd880, %rd879, %rd4; + shl.b64 %rd881, %rd880, 2; + add.s64 %rd882, %rd3, %rd881; + ld.global.u32 %r6400, [%rd882]; + abs.s32 %r6401, %r6400; + setp.gt.u32 %p1296, %r6401, 4; + and.b32 %r6402, %r6401, 1; + setp.eq.b32 %p1297, %r6402, 1; + and.pred %p1298, %p1296, %p1297; + selp.b32 %r6403, 1024, 0, %p1298; + or.b32 %r9611, %r6403, %r9611; + +$L__BB2_1017: + setp.ge.u32 %p1299, %r2034, %r4058; + @%p1299 bra $L__BB2_1019; + + add.s32 %r6404, %r2035, %r2101; + cvt.u64.u32 %rd883, %r6404; + add.s64 %rd884, %rd883, %rd4; + shl.b64 %rd885, %rd884, 2; + add.s64 %rd886, %rd3, %rd885; + ld.global.u32 %r6405, [%rd886]; + abs.s32 %r6406, %r6405; + setp.gt.u32 %p1300, %r6406, 4; + and.b32 %r6407, %r6406, 1; + setp.eq.b32 %p1301, %r6407, 1; + and.pred %p1302, %p1300, %p1301; + selp.b32 %r6408, 2048, 0, %p1302; + or.b32 %r9611, %r6408, %r9611; + +$L__BB2_1019: + add.s32 %r2110, %r9583, 7; + setp.ge.u32 %p1303, %r2110, %r4057; + @%p1303 bra $L__BB2_1028; + + setp.ge.u32 %p1304, %r2028, %r4058; + @%p1304 bra $L__BB2_1022; + + add.s32 %r6409, %r2029, %r2110; + cvt.u64.u32 %rd887, %r6409; + add.s64 %rd888, %rd887, %rd4; + shl.b64 %rd889, %rd888, 2; + add.s64 %rd890, %rd3, %rd889; + ld.global.u32 %r6410, [%rd890]; + abs.s32 %r6411, %r6410; + setp.gt.u32 %p1305, %r6411, 4; + and.b32 %r6412, %r6411, 1; + setp.eq.b32 %p1306, %r6412, 1; + and.pred %p1307, %p1305, %p1306; + selp.b32 %r6413, 4096, 0, %p1307; + or.b32 %r9611, %r6413, %r9611; + +$L__BB2_1022: + setp.ge.u32 %p1308, %r2030, %r4058; + @%p1308 bra $L__BB2_1024; + + add.s32 %r6414, %r2031, %r2110; + cvt.u64.u32 %rd891, %r6414; + add.s64 %rd892, %rd891, %rd4; + shl.b64 %rd893, %rd892, 2; + add.s64 %rd894, %rd3, %rd893; + ld.global.u32 %r6415, [%rd894]; + abs.s32 %r6416, %r6415; + setp.gt.u32 %p1309, %r6416, 4; + and.b32 %r6417, %r6416, 1; + setp.eq.b32 %p1310, %r6417, 1; + and.pred %p1311, %p1309, %p1310; + selp.b32 %r6418, 8192, 0, %p1311; + or.b32 %r9611, %r6418, %r9611; + +$L__BB2_1024: + setp.ge.u32 %p1312, %r2032, %r4058; + @%p1312 bra $L__BB2_1026; + + add.s32 %r6419, %r2033, %r2110; + cvt.u64.u32 %rd895, %r6419; + add.s64 %rd896, %rd895, %rd4; + shl.b64 %rd897, %rd896, 2; + add.s64 %rd898, %rd3, %rd897; + ld.global.u32 %r6420, [%rd898]; + abs.s32 %r6421, %r6420; + setp.gt.u32 %p1313, %r6421, 4; + and.b32 %r6422, %r6421, 1; + setp.eq.b32 %p1314, %r6422, 1; + and.pred %p1315, %p1313, %p1314; + selp.b32 %r6423, 16384, 0, %p1315; + or.b32 %r9611, %r6423, %r9611; + +$L__BB2_1026: + setp.ge.u32 %p1316, %r2034, %r4058; + @%p1316 bra $L__BB2_1028; + + add.s32 %r6424, %r2035, %r2110; + cvt.u64.u32 %rd899, %r6424; + add.s64 %rd900, %rd899, %rd4; + shl.b64 %rd901, %rd900, 2; + add.s64 %rd902, %rd3, %rd901; + ld.global.u32 %r6425, [%rd902]; + abs.s32 %r6426, %r6425; + setp.gt.u32 %p1317, %r6426, 4; + and.b32 %r6427, %r6426, 1; + setp.eq.b32 %p1318, %r6427, 1; + and.pred %p1319, %p1317, %p1318; + selp.b32 %r6428, 32768, 0, %p1319; + or.b32 %r9611, %r6428, %r9611; + +$L__BB2_1028: + mov.b32 %r2119, {%rs250, %rs251}; + add.s32 %r6430, %r2039, %r9583; + cvt.u64.u32 %rd903, %r6430; + add.s64 %rd904, %rd903, %rd4; + shl.b64 %rd905, %rd904, 2; + add.s64 %rd45, %rd3, %rd905; + add.s32 %r6431, %r2042, %r9583; + cvt.u64.u32 %rd906, %r6431; + add.s64 %rd907, %rd906, %rd4; + shl.b64 %rd908, %rd907, 2; + add.s64 %rd46, %rd3, %rd908; + add.s32 %r6432, %r2041, %r9583; + cvt.u64.u32 %rd909, %r6432; + add.s64 %rd910, %rd909, %rd4; + shl.b64 %rd911, %rd910, 2; + add.s64 %rd47, %rd3, %rd911; + add.s32 %r6433, %r2040, %r9583; + cvt.u64.u32 %rd912, %r6433; + add.s64 %rd913, %rd912, %rd4; + shl.b64 %rd914, %rd913, 2; + add.s64 %rd48, %rd3, %rd914; + mov.u32 %r9627, 0; + @%p1184 bra $L__BB2_1037; + + setp.le.u32 %p1321, %r4058, %r9579; + mov.u32 %r9627, 0; + @%p1321 bra $L__BB2_1031; + + ld.global.u32 %r6435, [%rd45]; + abs.s32 %r6436, %r6435; + setp.gt.u32 %p1322, %r6436, 4; + and.b32 %r6437, %r6436, 1; + setp.eq.b32 %p1323, %r6437, 1; + and.pred %p1324, %p1322, %p1323; + selp.u32 %r9627, 1, 0, %p1324; + +$L__BB2_1031: + setp.ge.u32 %p1325, %r2036, %r4058; + @%p1325 bra $L__BB2_1033; + + ld.global.u32 %r6438, [%rd46]; + abs.s32 %r6439, %r6438; + setp.gt.u32 %p1326, %r6439, 4; + and.b32 %r6440, %r6439, 1; + setp.eq.b32 %p1327, %r6440, 1; + and.pred %p1328, %p1326, %p1327; + selp.b32 %r6441, 2, 0, %p1328; + or.b32 %r9627, %r6441, %r9627; + +$L__BB2_1033: + setp.ge.u32 %p1329, %r2037, %r4058; + @%p1329 bra $L__BB2_1035; + + ld.global.u32 %r6442, [%rd47]; + abs.s32 %r6443, %r6442; + setp.gt.u32 %p1330, %r6443, 4; + and.b32 %r6444, %r6443, 1; + setp.eq.b32 %p1331, %r6444, 1; + and.pred %p1332, %p1330, %p1331; + selp.b32 %r6445, 4, 0, %p1332; + or.b32 %r9627, %r6445, %r9627; + +$L__BB2_1035: + setp.ge.u32 %p1333, %r2038, %r4058; + @%p1333 bra $L__BB2_1037; + + ld.global.u32 %r6446, [%rd48]; + abs.s32 %r6447, %r6446; + setp.gt.u32 %p1334, %r6447, 4; + and.b32 %r6448, %r6447, 1; + setp.eq.b32 %p1335, %r6448, 1; + and.pred %p1336, %p1334, %p1335; + selp.b32 %r6449, 8, 0, %p1336; + or.b32 %r9627, %r6449, %r9627; + +$L__BB2_1037: + add.s32 %r6450, %r2039, %r2057; + cvt.u64.u32 %rd915, %r6450; + add.s64 %rd916, %rd915, %rd4; + shl.b64 %rd917, %rd916, 2; + add.s64 %rd49, %rd3, %rd917; + add.s32 %r6451, %r2042, %r2057; + cvt.u64.u32 %rd918, %r6451; + add.s64 %rd919, %rd918, %rd4; + shl.b64 %rd920, %rd919, 2; + add.s64 %rd50, %rd3, %rd920; + add.s32 %r6452, %r2041, %r2057; + cvt.u64.u32 %rd921, %r6452; + add.s64 %rd922, %rd921, %rd4; + shl.b64 %rd923, %rd922, 2; + add.s64 %rd51, %rd3, %rd923; + add.s32 %r6453, %r2040, %r2057; + cvt.u64.u32 %rd924, %r6453; + add.s64 %rd925, %rd924, %rd4; + shl.b64 %rd926, %rd925, 2; + add.s64 %rd52, %rd3, %rd926; + shl.b32 %r6454, %r9611, 16; + or.b32 %r2128, %r6454, %r9595; + @%p1201 bra $L__BB2_1046; + + setp.le.u32 %p1338, %r4058, %r9579; + @%p1338 bra $L__BB2_1040; + + ld.global.u32 %r6455, [%rd49]; + abs.s32 %r6456, %r6455; + setp.gt.u32 %p1339, %r6456, 4; + and.b32 %r6457, %r6456, 1; + setp.eq.b32 %p1340, %r6457, 1; + and.pred %p1341, %p1339, %p1340; + selp.b32 %r6458, 16, 0, %p1341; + or.b32 %r9627, %r6458, %r9627; + +$L__BB2_1040: + setp.ge.u32 %p1342, %r2036, %r4058; + @%p1342 bra $L__BB2_1042; + + ld.global.u32 %r6459, [%rd50]; + abs.s32 %r6460, %r6459; + setp.gt.u32 %p1343, %r6460, 4; + and.b32 %r6461, %r6460, 1; + setp.eq.b32 %p1344, %r6461, 1; + and.pred %p1345, %p1343, %p1344; + selp.b32 %r6462, 32, 0, %p1345; + or.b32 %r9627, %r6462, %r9627; + +$L__BB2_1042: + setp.ge.u32 %p1346, %r2037, %r4058; + @%p1346 bra $L__BB2_1044; + + ld.global.u32 %r6463, [%rd51]; + abs.s32 %r6464, %r6463; + setp.gt.u32 %p1347, %r6464, 4; + and.b32 %r6465, %r6464, 1; + setp.eq.b32 %p1348, %r6465, 1; + and.pred %p1349, %p1347, %p1348; + selp.b32 %r6466, 64, 0, %p1349; + or.b32 %r9627, %r6466, %r9627; + +$L__BB2_1044: + setp.ge.u32 %p1350, %r2038, %r4058; + @%p1350 bra $L__BB2_1046; + + ld.global.u32 %r6467, [%rd52]; + abs.s32 %r6468, %r6467; + setp.gt.u32 %p1351, %r6468, 4; + and.b32 %r6469, %r6468, 1; + setp.eq.b32 %p1352, %r6469, 1; + and.pred %p1353, %p1351, %p1352; + selp.b32 %r6470, 128, 0, %p1353; + or.b32 %r9627, %r6470, %r9627; + +$L__BB2_1046: + add.s32 %r6471, %r2039, %r2066; + cvt.u64.u32 %rd927, %r6471; + add.s64 %rd928, %rd927, %rd4; + shl.b64 %rd929, %rd928, 2; + add.s64 %rd53, %rd3, %rd929; + add.s32 %r6472, %r2042, %r2066; + cvt.u64.u32 %rd930, %r6472; + add.s64 %rd931, %rd930, %rd4; + shl.b64 %rd932, %rd931, 2; + add.s64 %rd54, %rd3, %rd932; + add.s32 %r6473, %r2041, %r2066; + cvt.u64.u32 %rd933, %r6473; + add.s64 %rd934, %rd933, %rd4; + shl.b64 %rd935, %rd934, 2; + add.s64 %rd55, %rd3, %rd935; + add.s32 %r6474, %r2040, %r2066; + cvt.u64.u32 %rd936, %r6474; + add.s64 %rd937, %rd936, %rd4; + shl.b64 %rd938, %rd937, 2; + add.s64 %rd56, %rd3, %rd938; + @%p1218 bra $L__BB2_1055; + + setp.le.u32 %p1355, %r4058, %r9579; + @%p1355 bra $L__BB2_1049; + + ld.global.u32 %r6475, [%rd53]; + abs.s32 %r6476, %r6475; + setp.gt.u32 %p1356, %r6476, 4; + and.b32 %r6477, %r6476, 1; + setp.eq.b32 %p1357, %r6477, 1; + and.pred %p1358, %p1356, %p1357; + selp.b32 %r6478, 256, 0, %p1358; + or.b32 %r9627, %r6478, %r9627; + +$L__BB2_1049: + setp.ge.u32 %p1359, %r2036, %r4058; + @%p1359 bra $L__BB2_1051; + + ld.global.u32 %r6479, [%rd54]; + abs.s32 %r6480, %r6479; + setp.gt.u32 %p1360, %r6480, 4; + and.b32 %r6481, %r6480, 1; + setp.eq.b32 %p1361, %r6481, 1; + and.pred %p1362, %p1360, %p1361; + selp.b32 %r6482, 512, 0, %p1362; + or.b32 %r9627, %r6482, %r9627; + +$L__BB2_1051: + setp.ge.u32 %p1363, %r2037, %r4058; + @%p1363 bra $L__BB2_1053; + + ld.global.u32 %r6483, [%rd55]; + abs.s32 %r6484, %r6483; + setp.gt.u32 %p1364, %r6484, 4; + and.b32 %r6485, %r6484, 1; + setp.eq.b32 %p1365, %r6485, 1; + and.pred %p1366, %p1364, %p1365; + selp.b32 %r6486, 1024, 0, %p1366; + or.b32 %r9627, %r6486, %r9627; + +$L__BB2_1053: + setp.ge.u32 %p1367, %r2038, %r4058; + @%p1367 bra $L__BB2_1055; + + ld.global.u32 %r6487, [%rd56]; + abs.s32 %r6488, %r6487; + setp.gt.u32 %p1368, %r6488, 4; + and.b32 %r6489, %r6488, 1; + setp.eq.b32 %p1369, %r6489, 1; + and.pred %p1370, %p1368, %p1369; + selp.b32 %r6490, 2048, 0, %p1370; + or.b32 %r9627, %r6490, %r9627; + +$L__BB2_1055: + add.s32 %r6491, %r2039, %r2075; + cvt.u64.u32 %rd939, %r6491; + add.s64 %rd940, %rd939, %rd4; + shl.b64 %rd941, %rd940, 2; + add.s64 %rd57, %rd3, %rd941; + add.s32 %r6492, %r2042, %r2075; + cvt.u64.u32 %rd942, %r6492; + add.s64 %rd943, %rd942, %rd4; + shl.b64 %rd944, %rd943, 2; + add.s64 %rd58, %rd3, %rd944; + add.s32 %r6493, %r2041, %r2075; + cvt.u64.u32 %rd945, %r6493; + add.s64 %rd946, %rd945, %rd4; + shl.b64 %rd947, %rd946, 2; + add.s64 %rd59, %rd3, %rd947; + add.s32 %r6494, %r2040, %r2075; + cvt.u64.u32 %rd948, %r6494; + add.s64 %rd949, %rd948, %rd4; + shl.b64 %rd950, %rd949, 2; + add.s64 %rd60, %rd3, %rd950; + @%p1235 bra $L__BB2_1064; + + setp.le.u32 %p1372, %r4058, %r9579; + @%p1372 bra $L__BB2_1058; + + ld.global.u32 %r6495, [%rd57]; + abs.s32 %r6496, %r6495; + setp.gt.u32 %p1373, %r6496, 4; + and.b32 %r6497, %r6496, 1; + setp.eq.b32 %p1374, %r6497, 1; + and.pred %p1375, %p1373, %p1374; + selp.b32 %r6498, 4096, 0, %p1375; + or.b32 %r9627, %r6498, %r9627; + +$L__BB2_1058: + setp.ge.u32 %p1376, %r2036, %r4058; + @%p1376 bra $L__BB2_1060; + + ld.global.u32 %r6499, [%rd58]; + abs.s32 %r6500, %r6499; + setp.gt.u32 %p1377, %r6500, 4; + and.b32 %r6501, %r6500, 1; + setp.eq.b32 %p1378, %r6501, 1; + and.pred %p1379, %p1377, %p1378; + selp.b32 %r6502, 8192, 0, %p1379; + or.b32 %r9627, %r6502, %r9627; + +$L__BB2_1060: + setp.ge.u32 %p1380, %r2037, %r4058; + @%p1380 bra $L__BB2_1062; + + ld.global.u32 %r6503, [%rd59]; + abs.s32 %r6504, %r6503; + setp.gt.u32 %p1381, %r6504, 4; + and.b32 %r6505, %r6504, 1; + setp.eq.b32 %p1382, %r6505, 1; + and.pred %p1383, %p1381, %p1382; + selp.b32 %r6506, 16384, 0, %p1383; + or.b32 %r9627, %r6506, %r9627; + +$L__BB2_1062: + setp.ge.u32 %p1384, %r2038, %r4058; + @%p1384 bra $L__BB2_1064; + + ld.global.u32 %r6507, [%rd60]; + abs.s32 %r6508, %r6507; + setp.gt.u32 %p1385, %r6508, 4; + and.b32 %r6509, %r6508, 1; + setp.eq.b32 %p1386, %r6509, 1; + and.pred %p1387, %p1385, %p1386; + selp.b32 %r6510, 32768, 0, %p1387; + or.b32 %r9627, %r6510, %r9627; + +$L__BB2_1064: + mov.u32 %r9643, 0; + @%p1252 bra $L__BB2_1073; + + setp.le.u32 %p1389, %r4058, %r9579; + mov.u32 %r9643, 0; + @%p1389 bra $L__BB2_1067; + + add.s32 %r6515, %r6430, 4; + cvt.u64.u32 %rd951, %r6515; + add.s64 %rd952, %rd951, %rd4; + shl.b64 %rd953, %rd952, 2; + add.s64 %rd954, %rd3, %rd953; + ld.global.u32 %r6516, [%rd954]; + abs.s32 %r6517, %r6516; + setp.gt.u32 %p1390, %r6517, 4; + and.b32 %r6518, %r6517, 1; + setp.eq.b32 %p1391, %r6518, 1; + and.pred %p1392, %p1390, %p1391; + selp.u32 %r9643, 1, 0, %p1392; + +$L__BB2_1067: + setp.ge.u32 %p1393, %r2036, %r4058; + @%p1393 bra $L__BB2_1069; + + add.s32 %r6520, %r6431, 4; + cvt.u64.u32 %rd955, %r6520; + add.s64 %rd956, %rd955, %rd4; + shl.b64 %rd957, %rd956, 2; + add.s64 %rd958, %rd3, %rd957; + ld.global.u32 %r6521, [%rd958]; + abs.s32 %r6522, %r6521; + setp.gt.u32 %p1394, %r6522, 4; + and.b32 %r6523, %r6522, 1; + setp.eq.b32 %p1395, %r6523, 1; + and.pred %p1396, %p1394, %p1395; + selp.b32 %r6524, 2, 0, %p1396; + or.b32 %r9643, %r6524, %r9643; + +$L__BB2_1069: + setp.ge.u32 %p1397, %r2037, %r4058; + @%p1397 bra $L__BB2_1071; + + add.s32 %r6526, %r6432, 4; + cvt.u64.u32 %rd959, %r6526; + add.s64 %rd960, %rd959, %rd4; + shl.b64 %rd961, %rd960, 2; + add.s64 %rd962, %rd3, %rd961; + ld.global.u32 %r6527, [%rd962]; + abs.s32 %r6528, %r6527; + setp.gt.u32 %p1398, %r6528, 4; + and.b32 %r6529, %r6528, 1; + setp.eq.b32 %p1399, %r6529, 1; + and.pred %p1400, %p1398, %p1399; + selp.b32 %r6530, 4, 0, %p1400; + or.b32 %r9643, %r6530, %r9643; + +$L__BB2_1071: + setp.ge.u32 %p1401, %r2038, %r4058; + @%p1401 bra $L__BB2_1073; + + add.s32 %r6532, %r6433, 4; + cvt.u64.u32 %rd963, %r6532; + add.s64 %rd964, %rd963, %rd4; + shl.b64 %rd965, %rd964, 2; + add.s64 %rd966, %rd3, %rd965; + ld.global.u32 %r6533, [%rd966]; + abs.s32 %r6534, %r6533; + setp.gt.u32 %p1402, %r6534, 4; + and.b32 %r6535, %r6534, 1; + setp.eq.b32 %p1403, %r6535, 1; + and.pred %p1404, %p1402, %p1403; + selp.b32 %r6536, 8, 0, %p1404; + or.b32 %r9643, %r6536, %r9643; + +$L__BB2_1073: + @%p1269 bra $L__BB2_1082; + + setp.le.u32 %p1406, %r4058, %r9579; + @%p1406 bra $L__BB2_1076; + + add.s32 %r6537, %r2039, %r2092; + cvt.u64.u32 %rd967, %r6537; + add.s64 %rd968, %rd967, %rd4; + shl.b64 %rd969, %rd968, 2; + add.s64 %rd970, %rd3, %rd969; + ld.global.u32 %r6538, [%rd970]; + abs.s32 %r6539, %r6538; + setp.gt.u32 %p1407, %r6539, 4; + and.b32 %r6540, %r6539, 1; + setp.eq.b32 %p1408, %r6540, 1; + and.pred %p1409, %p1407, %p1408; + selp.b32 %r6541, 16, 0, %p1409; + or.b32 %r9643, %r6541, %r9643; + +$L__BB2_1076: + setp.ge.u32 %p1410, %r2036, %r4058; + @%p1410 bra $L__BB2_1078; + + add.s32 %r6542, %r2042, %r2092; + cvt.u64.u32 %rd971, %r6542; + add.s64 %rd972, %rd971, %rd4; + shl.b64 %rd973, %rd972, 2; + add.s64 %rd974, %rd3, %rd973; + ld.global.u32 %r6543, [%rd974]; + abs.s32 %r6544, %r6543; + setp.gt.u32 %p1411, %r6544, 4; + and.b32 %r6545, %r6544, 1; + setp.eq.b32 %p1412, %r6545, 1; + and.pred %p1413, %p1411, %p1412; + selp.b32 %r6546, 32, 0, %p1413; + or.b32 %r9643, %r6546, %r9643; + +$L__BB2_1078: + setp.ge.u32 %p1414, %r2037, %r4058; + @%p1414 bra $L__BB2_1080; + + add.s32 %r6547, %r2041, %r2092; + cvt.u64.u32 %rd975, %r6547; + add.s64 %rd976, %rd975, %rd4; + shl.b64 %rd977, %rd976, 2; + add.s64 %rd978, %rd3, %rd977; + ld.global.u32 %r6548, [%rd978]; + abs.s32 %r6549, %r6548; + setp.gt.u32 %p1415, %r6549, 4; + and.b32 %r6550, %r6549, 1; + setp.eq.b32 %p1416, %r6550, 1; + and.pred %p1417, %p1415, %p1416; + selp.b32 %r6551, 64, 0, %p1417; + or.b32 %r9643, %r6551, %r9643; + +$L__BB2_1080: + setp.ge.u32 %p1418, %r2038, %r4058; + @%p1418 bra $L__BB2_1082; + + add.s32 %r6552, %r2040, %r2092; + cvt.u64.u32 %rd979, %r6552; + add.s64 %rd980, %rd979, %rd4; + shl.b64 %rd981, %rd980, 2; + add.s64 %rd982, %rd3, %rd981; + ld.global.u32 %r6553, [%rd982]; + abs.s32 %r6554, %r6553; + setp.gt.u32 %p1419, %r6554, 4; + and.b32 %r6555, %r6554, 1; + setp.eq.b32 %p1420, %r6555, 1; + and.pred %p1421, %p1419, %p1420; + selp.b32 %r6556, 128, 0, %p1421; + or.b32 %r9643, %r6556, %r9643; + +$L__BB2_1082: + @%p1286 bra $L__BB2_1091; + + setp.le.u32 %p1423, %r4058, %r9579; + @%p1423 bra $L__BB2_1085; + + add.s32 %r6557, %r2039, %r2101; + cvt.u64.u32 %rd983, %r6557; + add.s64 %rd984, %rd983, %rd4; + shl.b64 %rd985, %rd984, 2; + add.s64 %rd986, %rd3, %rd985; + ld.global.u32 %r6558, [%rd986]; + abs.s32 %r6559, %r6558; + setp.gt.u32 %p1424, %r6559, 4; + and.b32 %r6560, %r6559, 1; + setp.eq.b32 %p1425, %r6560, 1; + and.pred %p1426, %p1424, %p1425; + selp.b32 %r6561, 256, 0, %p1426; + or.b32 %r9643, %r6561, %r9643; + +$L__BB2_1085: + setp.ge.u32 %p1427, %r2036, %r4058; + @%p1427 bra $L__BB2_1087; + + add.s32 %r6562, %r2042, %r2101; + cvt.u64.u32 %rd987, %r6562; + add.s64 %rd988, %rd987, %rd4; + shl.b64 %rd989, %rd988, 2; + add.s64 %rd990, %rd3, %rd989; + ld.global.u32 %r6563, [%rd990]; + abs.s32 %r6564, %r6563; + setp.gt.u32 %p1428, %r6564, 4; + and.b32 %r6565, %r6564, 1; + setp.eq.b32 %p1429, %r6565, 1; + and.pred %p1430, %p1428, %p1429; + selp.b32 %r6566, 512, 0, %p1430; + or.b32 %r9643, %r6566, %r9643; + +$L__BB2_1087: + setp.ge.u32 %p1431, %r2037, %r4058; + @%p1431 bra $L__BB2_1089; + + add.s32 %r6567, %r2041, %r2101; + cvt.u64.u32 %rd991, %r6567; + add.s64 %rd992, %rd991, %rd4; + shl.b64 %rd993, %rd992, 2; + add.s64 %rd994, %rd3, %rd993; + ld.global.u32 %r6568, [%rd994]; + abs.s32 %r6569, %r6568; + setp.gt.u32 %p1432, %r6569, 4; + and.b32 %r6570, %r6569, 1; + setp.eq.b32 %p1433, %r6570, 1; + and.pred %p1434, %p1432, %p1433; + selp.b32 %r6571, 1024, 0, %p1434; + or.b32 %r9643, %r6571, %r9643; + +$L__BB2_1089: + setp.ge.u32 %p1435, %r2038, %r4058; + @%p1435 bra $L__BB2_1091; + + add.s32 %r6572, %r2040, %r2101; + cvt.u64.u32 %rd995, %r6572; + add.s64 %rd996, %rd995, %rd4; + shl.b64 %rd997, %rd996, 2; + add.s64 %rd998, %rd3, %rd997; + ld.global.u32 %r6573, [%rd998]; + abs.s32 %r6574, %r6573; + setp.gt.u32 %p1436, %r6574, 4; + and.b32 %r6575, %r6574, 1; + setp.eq.b32 %p1437, %r6575, 1; + and.pred %p1438, %p1436, %p1437; + selp.b32 %r6576, 2048, 0, %p1438; + or.b32 %r9643, %r6576, %r9643; + +$L__BB2_1091: + @%p1303 bra $L__BB2_1100; + + setp.le.u32 %p1440, %r4058, %r9579; + @%p1440 bra $L__BB2_1094; + + add.s32 %r6577, %r2039, %r2110; + cvt.u64.u32 %rd999, %r6577; + add.s64 %rd1000, %rd999, %rd4; + shl.b64 %rd1001, %rd1000, 2; + add.s64 %rd1002, %rd3, %rd1001; + ld.global.u32 %r6578, [%rd1002]; + abs.s32 %r6579, %r6578; + setp.gt.u32 %p1441, %r6579, 4; + and.b32 %r6580, %r6579, 1; + setp.eq.b32 %p1442, %r6580, 1; + and.pred %p1443, %p1441, %p1442; + selp.b32 %r6581, 4096, 0, %p1443; + or.b32 %r9643, %r6581, %r9643; + +$L__BB2_1094: + setp.ge.u32 %p1444, %r2036, %r4058; + @%p1444 bra $L__BB2_1096; + + add.s32 %r6582, %r2042, %r2110; + cvt.u64.u32 %rd1003, %r6582; + add.s64 %rd1004, %rd1003, %rd4; + shl.b64 %rd1005, %rd1004, 2; + add.s64 %rd1006, %rd3, %rd1005; + ld.global.u32 %r6583, [%rd1006]; + abs.s32 %r6584, %r6583; + setp.gt.u32 %p1445, %r6584, 4; + and.b32 %r6585, %r6584, 1; + setp.eq.b32 %p1446, %r6585, 1; + and.pred %p1447, %p1445, %p1446; + selp.b32 %r6586, 8192, 0, %p1447; + or.b32 %r9643, %r6586, %r9643; + +$L__BB2_1096: + setp.ge.u32 %p1448, %r2037, %r4058; + @%p1448 bra $L__BB2_1098; + + add.s32 %r6587, %r2041, %r2110; + cvt.u64.u32 %rd1007, %r6587; + add.s64 %rd1008, %rd1007, %rd4; + shl.b64 %rd1009, %rd1008, 2; + add.s64 %rd1010, %rd3, %rd1009; + ld.global.u32 %r6588, [%rd1010]; + abs.s32 %r6589, %r6588; + setp.gt.u32 %p1449, %r6589, 4; + and.b32 %r6590, %r6589, 1; + setp.eq.b32 %p1450, %r6590, 1; + and.pred %p1451, %p1449, %p1450; + selp.b32 %r6591, 16384, 0, %p1451; + or.b32 %r9643, %r6591, %r9643; + +$L__BB2_1098: + setp.ge.u32 %p1452, %r2038, %r4058; + @%p1452 bra $L__BB2_1100; + + add.s32 %r6592, %r2040, %r2110; + cvt.u64.u32 %rd1011, %r6592; + add.s64 %rd1012, %rd1011, %rd4; + shl.b64 %rd1013, %rd1012, 2; + add.s64 %rd1014, %rd3, %rd1013; + ld.global.u32 %r6593, [%rd1014]; + abs.s32 %r6594, %r6593; + setp.gt.u32 %p1453, %r6594, 4; + and.b32 %r6595, %r6594, 1; + setp.eq.b32 %p1454, %r6595, 1; + and.pred %p1455, %p1453, %p1454; + selp.b32 %r6596, 32768, 0, %p1455; + or.b32 %r9643, %r6596, %r9643; + +$L__BB2_1100: + sub.s32 %r6599, %r6344, %r4057; + shl.b32 %r6600, %r9643, 16; + or.b32 %r2185, %r6600, %r9627; + and.b32 %r6601, %r2119, -2004318072; + shr.u32 %r6602, %r6601, 3; + shl.b32 %r6603, %r2128, 3; + and.b32 %r6604, %r6603, -2004318072; + or.b32 %r2186, %r6604, %r6602; + not.b32 %r6605, %r2185; + setp.gt.s32 %p1456, %r6599, 0; + mov.u32 %r9659, 0; + shl.b32 %r6606, %r6599, 2; + selp.b32 %r6607, %r6606, 0, %p1456; + shr.u32 %r2187, %r2043, %r6607; + and.b32 %r2188, %r2187, %r6605; + @%p1184 bra $L__BB2_1109; + + setp.le.u32 %p1458, %r4058, %r9579; + mov.u32 %r9659, 0; + @%p1458 bra $L__BB2_1103; + + ld.global.u32 %r6609, [%rd45]; + abs.s32 %r6610, %r6609; + setp.eq.s32 %p1459, %r6610, 3; + selp.u32 %r9659, 1, 0, %p1459; + +$L__BB2_1103: + setp.ge.u32 %p1460, %r2036, %r4058; + @%p1460 bra $L__BB2_1105; + + ld.global.u32 %r6611, [%rd46]; + abs.s32 %r6612, %r6611; + setp.eq.s32 %p1461, %r6612, 3; + selp.b32 %r6613, 2, 0, %p1461; + or.b32 %r9659, %r6613, %r9659; + +$L__BB2_1105: + setp.ge.u32 %p1462, %r2037, %r4058; + @%p1462 bra $L__BB2_1107; + + ld.global.u32 %r6614, [%rd47]; + abs.s32 %r6615, %r6614; + setp.eq.s32 %p1463, %r6615, 3; + selp.b32 %r6616, 4, 0, %p1463; + or.b32 %r9659, %r6616, %r9659; + +$L__BB2_1107: + setp.ge.u32 %p1464, %r2038, %r4058; + @%p1464 bra $L__BB2_1109; + + ld.global.u32 %r6617, [%rd48]; + abs.s32 %r6618, %r6617; + setp.eq.s32 %p1465, %r6618, 3; + selp.b32 %r6619, 8, 0, %p1465; + or.b32 %r9659, %r6619, %r9659; + +$L__BB2_1109: + @%p1201 bra $L__BB2_1118; + + setp.le.u32 %p1467, %r4058, %r9579; + @%p1467 bra $L__BB2_1112; + + ld.global.u32 %r6620, [%rd49]; + abs.s32 %r6621, %r6620; + setp.eq.s32 %p1468, %r6621, 3; + selp.b32 %r6622, 16, 0, %p1468; + or.b32 %r9659, %r6622, %r9659; + +$L__BB2_1112: + setp.ge.u32 %p1469, %r2036, %r4058; + @%p1469 bra $L__BB2_1114; + + ld.global.u32 %r6623, [%rd50]; + abs.s32 %r6624, %r6623; + setp.eq.s32 %p1470, %r6624, 3; + selp.b32 %r6625, 32, 0, %p1470; + or.b32 %r9659, %r6625, %r9659; + +$L__BB2_1114: + setp.ge.u32 %p1471, %r2037, %r4058; + @%p1471 bra $L__BB2_1116; + + ld.global.u32 %r6626, [%rd51]; + abs.s32 %r6627, %r6626; + setp.eq.s32 %p1472, %r6627, 3; + selp.b32 %r6628, 64, 0, %p1472; + or.b32 %r9659, %r6628, %r9659; + +$L__BB2_1116: + setp.ge.u32 %p1473, %r2038, %r4058; + @%p1473 bra $L__BB2_1118; + + ld.global.u32 %r6629, [%rd52]; + abs.s32 %r6630, %r6629; + setp.eq.s32 %p1474, %r6630, 3; + selp.b32 %r6631, 128, 0, %p1474; + or.b32 %r9659, %r6631, %r9659; + +$L__BB2_1118: + @%p1218 bra $L__BB2_1127; + + setp.le.u32 %p1476, %r4058, %r9579; + @%p1476 bra $L__BB2_1121; + + ld.global.u32 %r6632, [%rd53]; + abs.s32 %r6633, %r6632; + setp.eq.s32 %p1477, %r6633, 3; + selp.b32 %r6634, 256, 0, %p1477; + or.b32 %r9659, %r6634, %r9659; + +$L__BB2_1121: + setp.ge.u32 %p1478, %r2036, %r4058; + @%p1478 bra $L__BB2_1123; + + ld.global.u32 %r6635, [%rd54]; + abs.s32 %r6636, %r6635; + setp.eq.s32 %p1479, %r6636, 3; + selp.b32 %r6637, 512, 0, %p1479; + or.b32 %r9659, %r6637, %r9659; + +$L__BB2_1123: + setp.ge.u32 %p1480, %r2037, %r4058; + @%p1480 bra $L__BB2_1125; + + ld.global.u32 %r6638, [%rd55]; + abs.s32 %r6639, %r6638; + setp.eq.s32 %p1481, %r6639, 3; + selp.b32 %r6640, 1024, 0, %p1481; + or.b32 %r9659, %r6640, %r9659; + +$L__BB2_1125: + setp.ge.u32 %p1482, %r2038, %r4058; + @%p1482 bra $L__BB2_1127; + + ld.global.u32 %r6641, [%rd56]; + abs.s32 %r6642, %r6641; + setp.eq.s32 %p1483, %r6642, 3; + selp.b32 %r6643, 2048, 0, %p1483; + or.b32 %r9659, %r6643, %r9659; + +$L__BB2_1127: + @%p1235 bra $L__BB2_1136; + + setp.le.u32 %p1485, %r4058, %r9579; + @%p1485 bra $L__BB2_1130; + + ld.global.u32 %r6644, [%rd57]; + abs.s32 %r6645, %r6644; + setp.eq.s32 %p1486, %r6645, 3; + selp.b32 %r6646, 4096, 0, %p1486; + or.b32 %r9659, %r6646, %r9659; + +$L__BB2_1130: + setp.ge.u32 %p1487, %r2036, %r4058; + @%p1487 bra $L__BB2_1132; + + ld.global.u32 %r6647, [%rd58]; + abs.s32 %r6648, %r6647; + setp.eq.s32 %p1488, %r6648, 3; + selp.b32 %r6649, 8192, 0, %p1488; + or.b32 %r9659, %r6649, %r9659; + +$L__BB2_1132: + setp.ge.u32 %p1489, %r2037, %r4058; + @%p1489 bra $L__BB2_1134; + + ld.global.u32 %r6650, [%rd59]; + abs.s32 %r6651, %r6650; + setp.eq.s32 %p1490, %r6651, 3; + selp.b32 %r6652, 16384, 0, %p1490; + or.b32 %r9659, %r6652, %r9659; + +$L__BB2_1134: + setp.ge.u32 %p1491, %r2038, %r4058; + @%p1491 bra $L__BB2_1136; + + ld.global.u32 %r6653, [%rd60]; + abs.s32 %r6654, %r6653; + setp.eq.s32 %p1492, %r6654, 3; + selp.b32 %r6655, 32768, 0, %p1492; + or.b32 %r9659, %r6655, %r9659; + +$L__BB2_1136: + and.b32 %r6657, %r2185, -286331154; + shr.u32 %r6658, %r6657, 1; + shl.b32 %r6659, %r2185, 1; + and.b32 %r6660, %r6659, -286331154; + or.b32 %r6661, %r2185, %r2186; + or.b32 %r6662, %r6661, %r6660; + or.b32 %r6663, %r6662, %r6658; + and.b32 %r2221, %r9659, %r2187; + shr.u32 %r6664, %r6663, 4; + shl.b32 %r6665, %r6663, 4; + shr.u32 %r6666, %r9584, 12; + or.b32 %r6667, %r6663, %r6666; + or.b32 %r6668, %r6667, %r6665; + or.b32 %r6669, %r6668, %r6664; + and.b32 %r9669, %r2188, %r6669; + setp.eq.s32 %p1493, %r9669, 0; + mov.u32 %r9690, 0; + @%p1493 bra $L__BB2_1195; + + mov.u32 %r9668, 0; + mov.u32 %r9670, %r9668; + +$L__BB2_1138: + brev.b32 %r6672, %r9669; + bfind.shiftamt.u32 %r2229, %r6672; + mov.pred %p2373, -1; + mov.u32 %r6673, 1; + shl.b32 %r2230, %r6673, %r2229; + mov.u32 %r6674, -2; + shf.l.wrap.b32 %r6675, %r6674, %r6674, %r2229; + and.b32 %r9669, %r9669, %r6675; + or.b32 %r9668, %r2230, %r9668; + and.b32 %r2233, %r2230, %r2221; + setp.ne.s32 %p1495, %r2233, 0; + selp.u32 %r6676, 1, 0, %p1495; + setp.eq.s32 %p1496, %r9687, 0; + selp.b32 %r6677, 8, 7, %p1496; + shl.b32 %r6678, %r6676, %r9685; + cvt.u16.u32 %rs796, %r6678; + or.b16 %rs1232, %rs1232, %rs796; + add.s32 %r9685, %r9685, 1; + setp.lt.u32 %p1497, %r9685, %r6677; + mov.pred %p2371, %p2373; + @%p1497 bra $L__BB2_1143; + + setp.ge.u32 %p1499, %r9689, %r9561; + mov.pred %p2371, 0; + @%p1499 bra $L__BB2_1143; + + setp.eq.s64 %p1500, %rd43, 0; + @%p1500 bra $L__BB2_1142; + + cvt.u64.u32 %rd1015, %r9689; + add.s64 %rd1016, %rd42, %rd1015; + add.s64 %rd1017, %rd1, %rd1016; + st.global.u8 [%rd1017], %rs1232; + +$L__BB2_1142: + and.b16 %rs798, %rs1232, 255; + setp.eq.s16 %p1502, %rs798, 255; + selp.u32 %r9687, 1, 0, %p1502; + add.s32 %r9689, %r9689, 1; + mov.u32 %r9685, 0; + mov.u16 %rs1232, 0; + mov.pred %p2371, %p2373; + +$L__BB2_1143: + not.pred %p1504, %p2371; + @%p1504 bra $L__BB2_1202; + + setp.eq.s32 %p1505, %r2233, 0; + @%p1505 bra $L__BB2_1185; + + or.b32 %r9670, %r2230, %r9670; + mov.u32 %r9677, 51; + setp.gt.s32 %p1506, %r2229, 7; + @%p1506 bra $L__BB2_1161; + + setp.gt.s32 %p1518, %r2229, 3; + @%p1518 bra $L__BB2_1154; + + setp.gt.s32 %p1524, %r2229, 1; + @%p1524 bra $L__BB2_1151; + + setp.eq.s32 %p1527, %r2229, 0; + @%p1527 bra $L__BB2_1184; + + setp.eq.s32 %p1528, %r2229, 1; + @%p1528 bra $L__BB2_1150; + bra.uni $L__BB2_1183; + +$L__BB2_1150: + mov.u32 %r9677, 118; + bra.uni $L__BB2_1184; + +$L__BB2_1161: + setp.gt.s32 %p1507, %r2229, 11; + @%p1507 bra $L__BB2_1169; + + setp.gt.s32 %p1513, %r2229, 9; + @%p1513 bra $L__BB2_1166; + + setp.eq.s32 %p1516, %r2229, 8; + @%p1516 bra $L__BB2_1179; + + setp.eq.s32 %p1517, %r2229, 9; + @%p1517 bra $L__BB2_1165; + bra.uni $L__BB2_1183; + +$L__BB2_1165: + mov.u32 %r9677, 30208; + bra.uni $L__BB2_1184; + +$L__BB2_1154: + setp.gt.s32 %p1519, %r2229, 5; + @%p1519 bra $L__BB2_1158; + + setp.eq.s32 %p1522, %r2229, 4; + @%p1522 bra $L__BB2_1181; + + setp.eq.s32 %p1523, %r2229, 5; + @%p1523 bra $L__BB2_1157; + bra.uni $L__BB2_1183; + +$L__BB2_1157: + mov.u32 %r9677, 1888; + bra.uni $L__BB2_1184; + +$L__BB2_1169: + setp.gt.s32 %p1508, %r2229, 13; + @%p1508 bra $L__BB2_1173; + + setp.eq.s32 %p1511, %r2229, 12; + @%p1511 bra $L__BB2_1177; + + setp.eq.s32 %p1512, %r2229, 13; + @%p1512 bra $L__BB2_1172; + bra.uni $L__BB2_1183; + +$L__BB2_1172: + mov.u32 %r9677, 483328; + bra.uni $L__BB2_1184; + +$L__BB2_1151: + setp.eq.s32 %p1525, %r2229, 2; + @%p1525 bra $L__BB2_1182; + + setp.eq.s32 %p1526, %r2229, 3; + @%p1526 bra $L__BB2_1153; + bra.uni $L__BB2_1183; + +$L__BB2_1153: + mov.u32 %r9677, 200; + bra.uni $L__BB2_1184; + +$L__BB2_1166: + setp.eq.s32 %p1514, %r2229, 10; + @%p1514 bra $L__BB2_1178; + + setp.eq.s32 %p1515, %r2229, 11; + @%p1515 bra $L__BB2_1168; + bra.uni $L__BB2_1183; + +$L__BB2_1168: + mov.u32 %r9677, 51200; + bra.uni $L__BB2_1184; + +$L__BB2_1158: + setp.eq.s32 %p1520, %r2229, 6; + @%p1520 bra $L__BB2_1180; + + setp.eq.s32 %p1521, %r2229, 7; + @%p1521 bra $L__BB2_1160; + bra.uni $L__BB2_1183; + +$L__BB2_1160: + mov.u32 %r9677, 3200; + bra.uni $L__BB2_1184; + +$L__BB2_1173: + setp.eq.s32 %p1509, %r2229, 14; + @%p1509 bra $L__BB2_1176; + + setp.ne.s32 %p1510, %r2229, 15; + @%p1510 bra $L__BB2_1183; + + mov.u32 %r9677, 819200; + bra.uni $L__BB2_1184; + +$L__BB2_1179: + mov.u32 %r9677, 13056; + bra.uni $L__BB2_1184; + +$L__BB2_1181: + mov.u32 %r9677, 816; + bra.uni $L__BB2_1184; + +$L__BB2_1177: + mov.u32 %r9677, 208896; + bra.uni $L__BB2_1184; + +$L__BB2_1182: + mov.u32 %r9677, 236; + bra.uni $L__BB2_1184; + +$L__BB2_1178: + mov.u32 %r9677, 60416; + bra.uni $L__BB2_1184; + +$L__BB2_1180: + mov.u32 %r9677, 3776; + bra.uni $L__BB2_1184; + +$L__BB2_1176: + mov.u32 %r9677, 966656; + bra.uni $L__BB2_1184; + +$L__BB2_1183: + mov.u32 %r9677, 0; + +$L__BB2_1184: + not.b32 %r6697, %r9668; + and.b32 %r6698, %r2188, %r6697; + and.b32 %r6699, %r6698, %r9677; + or.b32 %r9669, %r6699, %r9669; + +$L__BB2_1185: + setp.ne.s32 %p1529, %r9669, 0; + @%p1529 bra $L__BB2_1138; + + setp.eq.s32 %p1530, %r9670, 0; + mov.u32 %r9690, 0; + @%p1530 bra $L__BB2_1195; + + mov.u32 %r9683, %r9670; + +$L__BB2_1188: + setp.eq.s32 %p1531, %r9683, 0; + mov.u32 %r9690, %r9670; + @%p1531 bra $L__BB2_1195; + + brev.b32 %r6701, %r9683; + bfind.shiftamt.u32 %r6702, %r6701; + mov.pred %p2373, -1; + mov.u32 %r6703, -2; + shf.l.wrap.b32 %r6704, %r6703, %r6703, %r6702; + and.b32 %r9683, %r9683, %r6704; + shr.u32 %r6705, %r6702, 2; + and.b32 %r6706, %r6702, 3; + add.s32 %r6707, %r6706, %r9579; + add.s32 %r6708, %r6705, %r9583; + mad.lo.s32 %r6709, %r6707, %r4055, %r6708; + cvt.u64.u32 %rd1018, %r6709; + add.s64 %rd1019, %rd1018, %rd4; + shl.b64 %rd1020, %rd1019, 2; + add.s64 %rd1021, %rd3, %rd1020; + ld.global.u32 %r6710, [%rd1021]; + shr.u32 %r6711, %r6710, 31; + setp.eq.s32 %p1533, %r9687, 0; + selp.b32 %r6712, 8, 7, %p1533; + shl.b32 %r6713, %r6711, %r9685; + cvt.u16.u32 %rs799, %r6713; + or.b16 %rs1232, %rs1232, %rs799; + add.s32 %r9685, %r9685, 1; + setp.lt.u32 %p1534, %r9685, %r6712; + mov.pred %p2372, %p2373; + @%p1534 bra $L__BB2_1194; + + setp.ge.u32 %p1536, %r9689, %r9561; + mov.pred %p2372, 0; + @%p1536 bra $L__BB2_1194; + + setp.eq.s64 %p1537, %rd43, 0; + @%p1537 bra $L__BB2_1193; + + cvt.u64.u32 %rd1022, %r9689; + add.s64 %rd1023, %rd42, %rd1022; + add.s64 %rd1024, %rd1, %rd1023; + st.global.u8 [%rd1024], %rs1232; + +$L__BB2_1193: + and.b16 %rs801, %rs1232, 255; + setp.eq.s16 %p1539, %rs801, 255; + selp.u32 %r9687, 1, 0, %p1539; + add.s32 %r9689, %r9689, 1; + mov.u32 %r9685, 0; + mov.u16 %rs1232, 0; + mov.pred %p2372, %p2373; + +$L__BB2_1194: + @%p2372 bra $L__BB2_1188; + bra.uni $L__BB2_1202; + +$L__BB2_1195: + not.b32 %r6715, %r9690; + and.b32 %r6716, %r2221, %r6715; + setp.ne.s32 %p1542, %r6716, 0; + mov.pred %p2373, %p1178; + @%p1542 bra $L__BB2_1202; + + setp.lt.u32 %p1543, %r6344, %r4057; + or.b32 %r6717, %r9690, %r2185; + st.local.u16 [%rd44], %r6717; + shr.u32 %r6718, %r6717, 16; + st.local.u16 [%rd44+2], %r6718; + shl.b32 %r6719, %r6717, 1; + and.b32 %r6720, %r6719, 57344; + and.b32 %r6721, %r6717, 57344; + shr.u32 %r6722, %r6721, 1; + or.b32 %r6723, %r6717, %r2186; + and.b32 %r6724, %r6723, 61440; + or.b32 %r6725, %r6724, %r6720; + or.b32 %r9584, %r6725, %r6722; + mov.u32 %r9583, %r6344; + @%p1543 bra $L__BB2_956; + +$L__BB2_1197: + add.s32 %r9579, %r9579, 4; + setp.gt.u32 %p1544, %r4058, %r9579; + @%p1544 bra $L__BB2_954; + + setp.eq.s32 %p1546, %r9685, 0; + mov.pred %p1545, 0; + mov.pred %p2373, %p1545; + @%p1546 bra $L__BB2_1202; + + setp.ge.u32 %p1548, %r9689, %r9561; + mov.pred %p2373, %p1178; + @%p1548 bra $L__BB2_1202; + + setp.eq.s64 %p1550, %rd43, 0; + mov.pred %p2373, %p1545; + @%p1550 bra $L__BB2_1202; + + cvt.u64.u32 %rd1025, %r9689; + add.s64 %rd1026, %rd42, %rd1025; + add.s64 %rd1027, %rd1, %rd1026; + st.global.u8 [%rd1027], %rs1232; + mov.pred %p2373, %p1545; + +$L__BB2_1202: + @%p2373 bra $L__BB2_1246; + bra.uni $L__BB2_1203; + +$L__BB2_1246: + mov.u32 %r6779, 2; + st.global.u32 [%rd6], %r6779; + mov.u32 %r6780, 6; + st.global.u32 [%rd6+4], %r6780; + mov.u32 %r6781, 0; + st.global.u32 [%rd6+8], %r6781; + st.global.u32 [%rd6+12], %r6781; + st.global.u32 [%rd6+16], %r6781; + st.global.u32 [%rd6+20], %r6781; + st.global.u32 [%rd6+24], %r6781; + st.global.u32 [%rd6+28], %r6781; + bra.uni $L__BB2_1905; + +$L__BB2_1203: + cvt.u64.u32 %rd1028, %r9561; + add.s64 %rd61, %rd42, %rd1028; + setp.eq.s32 %p1552, %r9562, 0; + @%p1552 bra $L__BB2_1244; + + add.s32 %r6727, %r9562, -1; + and.b32 %r9698, %r9562, 3; + setp.lt.u32 %p1553, %r6727, 3; + mov.u32 %r9696, 0; + @%p1553 bra $L__BB2_1207; + + sub.s32 %r9695, %r9562, %r9698; + mov.u32 %r9696, 0; + +$L__BB2_1206: + cvt.u64.u32 %rd1029, %r9696; + add.s64 %rd1030, %rd61, %rd1029; + add.s64 %rd1031, %rd1, %rd1030; + mov.u16 %rs802, 0; + st.global.u8 [%rd1031], %rs802; + st.global.u8 [%rd1031+1], %rs802; + st.global.u8 [%rd1031+2], %rs802; + st.global.u8 [%rd1031+3], %rs802; + add.s32 %r9696, %r9696, 4; + add.s32 %r9695, %r9695, -4; + setp.ne.s32 %p1554, %r9695, 0; + @%p1554 bra $L__BB2_1206; + +$L__BB2_1207: + setp.eq.s32 %p1555, %r9698, 0; + @%p1555 bra $L__BB2_1209; + +$L__BB2_1208: + .pragma "nounroll"; + cvt.u64.u32 %rd1032, %r9696; + add.s64 %rd1033, %rd61, %rd1032; + add.s64 %rd1034, %rd1, %rd1033; + mov.u16 %rs803, 0; + st.global.u8 [%rd1034], %rs803; + add.s32 %r9696, %r9696, 1; + add.s32 %r9698, %r9698, -1; + setp.ne.s32 %p1556, %r9698, 0; + @%p1556 bra $L__BB2_1208; + +$L__BB2_1209: + @%p10 bra $L__BB2_1237; + + mov.u32 %r6733, 0; + mov.u32 %r9725, 1; + mov.u16 %rs1239, 0; + mov.u32 %r9699, %r6733; + mov.u32 %r9724, %r6733; + mov.u32 %r9723, %r6733; + mov.u32 %r9722, %r6733; + +$L__BB2_1211: + mul.lo.s32 %r2282, %r9699, %r4055; + add.s32 %r2283, %r9699, 3; + mul.lo.s32 %r2284, %r4055, %r2283; + add.s32 %r2285, %r9699, 2; + mul.lo.s32 %r2286, %r4055, %r2285; + add.s32 %r2287, %r9699, 1; + mul.lo.s32 %r2288, %r4055, %r2287; + mov.u32 %r9704, %r6733; + +$L__BB2_1212: + add.s32 %r9712, %r2284, %r9704; + add.s32 %r9711, %r2286, %r9704; + add.s32 %r9710, %r2288, %r9704; + add.s32 %r9709, %r2282, %r9704; + mov.u32 %r9713, 0; + +$L__BB2_1213: + add.s32 %r6736, %r9704, %r9713; + setp.ge.u32 %p1558, %r6736, %r4057; + @%p1558 bra $L__BB2_1234; + + setp.ge.u32 %p1559, %r9699, %r4058; + @%p1559 bra $L__BB2_1219; + + cvt.u64.u32 %rd1035, %r9709; + add.s64 %rd1036, %rd1035, %rd4; + shl.b64 %rd1037, %rd1036, 2; + add.s64 %rd1038, %rd3, %rd1037; + ld.global.u32 %r6737, [%rd1038]; + abs.s32 %r2307, %r6737; + setp.lt.u32 %p1560, %r2307, 5; + and.b32 %r6738, %r2307, 1; + setp.eq.b32 %p1561, %r6738, 1; + not.pred %p1562, %p1561; + or.pred %p1563, %p1560, %p1562; + @%p1563 bra $L__BB2_1219; + + shr.u32 %r6739, %r2307, 1; + and.b32 %r6740, %r6739, 1; + shl.b32 %r6741, %r6740, %r9722; + cvt.u16.u32 %rs805, %r6741; + or.b16 %rs1239, %rs1239, %rs805; + add.s32 %r9724, %r9724, 1; + add.s32 %r9722, %r9722, 1; + setp.ne.s32 %p1564, %r9722, 7; + setp.eq.s32 %p1565, %r9725, 0; + or.pred %p1566, %p1564, %p1565; + and.b16 %rs806, %rs1239, 127; + setp.ne.s16 %p1567, %rs806, 127; + or.pred %p1568, %p1566, %p1567; + setp.ne.s32 %p1569, %r9722, 8; + and.pred %p1570, %p1569, %p1568; + @%p1570 bra $L__BB2_1219; + + setp.ge.u32 %p1571, %r9723, %r9562; + @%p1571 bra $L__BB2_1245; + + not.b32 %r6743, %r9723; + add.s32 %r6744, %r9562, %r6743; + cvt.u64.u32 %rd1039, %r6744; + add.s64 %rd1040, %rd61, %rd1039; + add.s64 %rd1041, %rd1, %rd1040; + and.b16 %rs808, %rs1239, 255; + st.global.u8 [%rd1041], %rs1239; + add.s32 %r9723, %r9723, 1; + setp.gt.u16 %p1572, %rs808, 143; + selp.u32 %r9725, 1, 0, %p1572; + mov.u16 %rs1239, 0; + mov.u32 %r9722, 0; + +$L__BB2_1219: + setp.ge.u32 %p1573, %r2287, %r4058; + @%p1573 bra $L__BB2_1224; + + cvt.u64.u32 %rd1042, %r9710; + add.s64 %rd1043, %rd1042, %rd4; + shl.b64 %rd1044, %rd1043, 2; + add.s64 %rd1045, %rd3, %rd1044; + ld.global.u32 %r6745, [%rd1045]; + abs.s32 %r2316, %r6745; + setp.lt.u32 %p1574, %r2316, 5; + and.b32 %r6746, %r2316, 1; + setp.eq.b32 %p1575, %r6746, 1; + not.pred %p1576, %p1575; + or.pred %p1577, %p1574, %p1576; + @%p1577 bra $L__BB2_1224; + + shr.u32 %r6747, %r2316, 1; + and.b32 %r6748, %r6747, 1; + shl.b32 %r6749, %r6748, %r9722; + cvt.u16.u32 %rs809, %r6749; + or.b16 %rs1239, %rs1239, %rs809; + add.s32 %r9724, %r9724, 1; + add.s32 %r9722, %r9722, 1; + setp.ne.s32 %p1578, %r9722, 7; + setp.eq.s32 %p1579, %r9725, 0; + or.pred %p1580, %p1578, %p1579; + and.b16 %rs810, %rs1239, 127; + setp.ne.s16 %p1581, %rs810, 127; + or.pred %p1582, %p1580, %p1581; + setp.ne.s32 %p1583, %r9722, 8; + and.pred %p1584, %p1583, %p1582; + @%p1584 bra $L__BB2_1224; + + setp.ge.u32 %p1585, %r9723, %r9562; + @%p1585 bra $L__BB2_1245; + + not.b32 %r6751, %r9723; + add.s32 %r6752, %r9562, %r6751; + cvt.u64.u32 %rd1046, %r6752; + add.s64 %rd1047, %rd61, %rd1046; + add.s64 %rd1048, %rd1, %rd1047; + and.b16 %rs812, %rs1239, 255; + st.global.u8 [%rd1048], %rs1239; + add.s32 %r9723, %r9723, 1; + setp.gt.u16 %p1586, %rs812, 143; + selp.u32 %r9725, 1, 0, %p1586; + mov.u16 %rs1239, 0; + mov.u32 %r9722, 0; + +$L__BB2_1224: + setp.ge.u32 %p1587, %r2285, %r4058; + @%p1587 bra $L__BB2_1229; + + cvt.u64.u32 %rd1049, %r9711; + add.s64 %rd1050, %rd1049, %rd4; + shl.b64 %rd1051, %rd1050, 2; + add.s64 %rd1052, %rd3, %rd1051; + ld.global.u32 %r6753, [%rd1052]; + abs.s32 %r2325, %r6753; + setp.lt.u32 %p1588, %r2325, 5; + and.b32 %r6754, %r2325, 1; + setp.eq.b32 %p1589, %r6754, 1; + not.pred %p1590, %p1589; + or.pred %p1591, %p1588, %p1590; + @%p1591 bra $L__BB2_1229; + + shr.u32 %r6755, %r2325, 1; + and.b32 %r6756, %r6755, 1; + shl.b32 %r6757, %r6756, %r9722; + cvt.u16.u32 %rs813, %r6757; + or.b16 %rs1239, %rs1239, %rs813; + add.s32 %r9724, %r9724, 1; + add.s32 %r9722, %r9722, 1; + setp.ne.s32 %p1592, %r9722, 7; + setp.eq.s32 %p1593, %r9725, 0; + or.pred %p1594, %p1592, %p1593; + and.b16 %rs814, %rs1239, 127; + setp.ne.s16 %p1595, %rs814, 127; + or.pred %p1596, %p1594, %p1595; + setp.ne.s32 %p1597, %r9722, 8; + and.pred %p1598, %p1597, %p1596; + @%p1598 bra $L__BB2_1229; + + setp.ge.u32 %p1599, %r9723, %r9562; + @%p1599 bra $L__BB2_1245; + + not.b32 %r6759, %r9723; + add.s32 %r6760, %r9562, %r6759; + cvt.u64.u32 %rd1053, %r6760; + add.s64 %rd1054, %rd61, %rd1053; + add.s64 %rd1055, %rd1, %rd1054; + and.b16 %rs816, %rs1239, 255; + st.global.u8 [%rd1055], %rs1239; + add.s32 %r9723, %r9723, 1; + setp.gt.u16 %p1600, %rs816, 143; + selp.u32 %r9725, 1, 0, %p1600; + mov.u16 %rs1239, 0; + mov.u32 %r9722, 0; + +$L__BB2_1229: + setp.ge.u32 %p1601, %r2283, %r4058; + @%p1601 bra $L__BB2_1234; + + cvt.u64.u32 %rd1056, %r9712; + add.s64 %rd1057, %rd1056, %rd4; + shl.b64 %rd1058, %rd1057, 2; + add.s64 %rd1059, %rd3, %rd1058; + ld.global.u32 %r6761, [%rd1059]; + abs.s32 %r2334, %r6761; + setp.lt.u32 %p1602, %r2334, 5; + and.b32 %r6762, %r2334, 1; + setp.eq.b32 %p1603, %r6762, 1; + not.pred %p1604, %p1603; + or.pred %p1605, %p1602, %p1604; + @%p1605 bra $L__BB2_1234; + + shr.u32 %r6763, %r2334, 1; + and.b32 %r6764, %r6763, 1; + shl.b32 %r6765, %r6764, %r9722; + cvt.u16.u32 %rs817, %r6765; + or.b16 %rs1239, %rs1239, %rs817; + add.s32 %r9724, %r9724, 1; + add.s32 %r9722, %r9722, 1; + setp.ne.s32 %p1606, %r9722, 7; + setp.eq.s32 %p1607, %r9725, 0; + or.pred %p1608, %p1606, %p1607; + and.b16 %rs818, %rs1239, 127; + setp.ne.s16 %p1609, %rs818, 127; + or.pred %p1610, %p1608, %p1609; + setp.ne.s32 %p1611, %r9722, 8; + and.pred %p1612, %p1611, %p1610; + @%p1612 bra $L__BB2_1234; + + setp.ge.u32 %p1613, %r9723, %r9562; + @%p1613 bra $L__BB2_1245; + + not.b32 %r6767, %r9723; + add.s32 %r6768, %r9562, %r6767; + cvt.u64.u32 %rd1060, %r6768; + add.s64 %rd1061, %rd61, %rd1060; + add.s64 %rd1062, %rd1, %rd1061; + and.b16 %rs820, %rs1239, 255; + st.global.u8 [%rd1062], %rs1239; + add.s32 %r9723, %r9723, 1; + setp.gt.u16 %p1614, %rs820, 143; + selp.u32 %r9725, 1, 0, %p1614; + mov.u16 %rs1239, 0; + mov.u32 %r9722, 0; + +$L__BB2_1234: + add.s32 %r9712, %r9712, 1; + add.s32 %r9711, %r9711, 1; + add.s32 %r9710, %r9710, 1; + add.s32 %r9709, %r9709, 1; + add.s32 %r9713, %r9713, 1; + setp.lt.u32 %p1615, %r9713, 8; + @%p1615 bra $L__BB2_1213; + + add.s32 %r9704, %r9704, 8; + setp.lt.u32 %p1616, %r9704, %r4057; + @%p1616 bra $L__BB2_1212; + + add.s32 %r9699, %r9699, 4; + setp.lt.u32 %p1617, %r9699, %r4058; + @%p1617 bra $L__BB2_1211; + bra.uni $L__BB2_1239; + +$L__BB2_1244: + setp.eq.s32 %p1624, %r8433, 0; + @%p1624 bra $L__BB2_1243; + bra.uni $L__BB2_1245; + +$L__BB2_1237: + mov.u32 %r9722, 0; + mov.u32 %r9734, %r9722; + +$L__BB2_1238: + add.s32 %r9734, %r9734, 4; + setp.lt.u32 %p1618, %r9734, %r4058; + mov.u16 %rs1239, 0; + mov.u32 %r9723, %r9722; + mov.u32 %r9724, %r9722; + @%p1618 bra $L__BB2_1238; + +$L__BB2_1239: + setp.eq.s32 %p1619, %r9722, 0; + @%p1619 bra $L__BB2_1242; + + setp.ge.u32 %p1620, %r9723, %r9562; + @%p1620 bra $L__BB2_1245; + + not.b32 %r6773, %r9723; + add.s32 %r6774, %r9562, %r6773; + cvt.u64.u32 %rd1063, %r6774; + add.s64 %rd1064, %rd61, %rd1063; + add.s64 %rd1065, %rd1, %rd1064; + st.global.u8 [%rd1065], %rs1239; + add.s32 %r9723, %r9723, 1; + +$L__BB2_1242: + setp.le.u32 %p1621, %r9723, %r9562; + setp.eq.s32 %p1622, %r9724, %r8433; + and.pred %p1623, %p1622, %p1621; + @%p1623 bra $L__BB2_1243; + bra.uni $L__BB2_1245; + +$L__BB2_1243: + mov.u32 %r6778, 0; + st.global.u32 [%rd6], %r6778; + st.global.u32 [%rd6+4], %r6778; + st.global.u32 [%rd6+8], %r1989; + st.global.u32 [%rd6+12], %r4062; + st.global.u32 [%rd6+16], %r45; + st.global.u32 [%rd6+20], %r1738; + st.global.u32 [%rd6+24], %r9563; + st.global.u32 [%rd6+28], %r6778; + bra.uni $L__BB2_1905; + +$L__BB2_1245: + mov.u32 %r6775, 2; + st.global.u32 [%rd6], %r6775; + mov.u32 %r6776, 7; + st.global.u32 [%rd6+4], %r6776; + mov.u32 %r6777, 0; + st.global.u32 [%rd6+8], %r6777; + st.global.u32 [%rd6+12], %r6777; + st.global.u32 [%rd6+16], %r6777; + st.global.u32 [%rd6+20], %r6777; + st.global.u32 [%rd6+24], %r6777; + st.global.u32 [%rd6+28], %r6777; + bra.uni $L__BB2_1905; + +} + // .globl j2k_htj2k_encode_codeblocks_multi_input_cleanup +.visible .entry j2k_htj2k_encode_codeblocks_multi_input_cleanup( + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_0, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_1, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_2, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_3, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_4, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_5, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_6 +) +.maxntid 128, 1, 1 +{ + .reg .pred %p<1453>; + .reg .b16 %rs<1196>; + .reg .b32 %r<8155>; + .reg .b64 %rd<690>; + // demoted variable + .shared .align 4 .b8 _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE9block_max[512]; + // demoted variable + .shared .align 1 .b8 _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val[513]; + // demoted variable + .shared .align 1 .b8 _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val[513]; + + ld.param.u64 %rd50, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_0]; + ld.param.u64 %rd48, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_4]; + ld.param.u64 %rd51, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_6]; + cvta.to.global.u64 %rd1, %rd50; + mov.u32 %r3196, %ctaid.x; + cvt.u64.u32 %rd2, %r3196; + setp.ge.u64 %p1, %rd2, %rd51; + @%p1 bra $L__BB3_1261; + + ld.param.u64 %rd686, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_1]; + cvta.to.global.u64 %rd52, %rd686; + mul.lo.s64 %rd53, %rd2, 40; + add.s64 %rd54, %rd52, %rd53; + ld.global.u64 %rd55, [%rd54]; + cvta.to.global.u64 %rd3, %rd55; + ld.global.v2.u32 {%r3197, %r3198}, [%rd54+8]; + ld.global.v2.u32 {%r3200, %r3201}, [%rd54+16]; + ld.global.v2.u32 {%r3202, %r3203}, [%rd54+24]; + ld.global.v2.u32 {%r3204, %r3205}, [%rd54+32]; + cvt.u64.u32 %rd4, %r3197; + setp.eq.s32 %p2, %r3200, 0; + setp.eq.s32 %p3, %r3201, 0; + or.pred %p4, %p2, %p3; + @%p4 bra $L__BB3_14; + bra.uni $L__BB3_2; + +$L__BB3_14: + mov.u32 %r25, 0; + bra.uni $L__BB3_15; + +$L__BB3_2: + mul.lo.s32 %r8, %r3201, %r3200; + setp.eq.s32 %p5, %r3198, %r3200; + @%p5 bra $L__BB3_6; + bra.uni $L__BB3_3; + +$L__BB3_6: + mov.u32 %r6289, %tid.x; + setp.ge.u32 %p8, %r6289, %r8; + mov.u32 %r6291, 0; + @%p8 bra $L__BB3_9; + + mov.u32 %r6291, 0; + +$L__BB3_8: + cvt.u64.u32 %rd60, %r6289; + add.s64 %rd61, %rd60, %rd4; + shl.b64 %rd62, %rd61, 2; + add.s64 %rd63, %rd3, %rd62; + ld.global.u32 %r3217, [%rd63]; + abs.s32 %r3218, %r3217; + max.u32 %r6291, %r6291, %r3218; + mov.u32 %r3219, %ntid.x; + add.s32 %r6289, %r6289, %r3219; + setp.lt.u32 %p9, %r6289, %r8; + @%p9 bra $L__BB3_8; + bra.uni $L__BB3_9; + +$L__BB3_3: + mov.u32 %r6287, %tid.x; + setp.ge.u32 %p6, %r6287, %r8; + mov.u32 %r6291, 0; + @%p6 bra $L__BB3_9; + + sub.s32 %r9, %r3198, %r3200; + mov.u32 %r6291, 0; + +$L__BB3_5: + div.u32 %r3209, %r6287, %r3200; + mad.lo.s32 %r3210, %r9, %r3209, %r6287; + cvt.u64.u32 %rd56, %r3210; + add.s64 %rd57, %rd56, %rd4; + shl.b64 %rd58, %rd57, 2; + add.s64 %rd59, %rd3, %rd58; + ld.global.u32 %r3211, [%rd59]; + abs.s32 %r3212, %r3211; + max.u32 %r6291, %r6291, %r3212; + mov.u32 %r3213, %ntid.x; + add.s32 %r6287, %r6287, %r3213; + setp.lt.u32 %p7, %r6287, %r8; + @%p7 bra $L__BB3_5; + +$L__BB3_9: + mov.u32 %r3220, %tid.x; + shl.b32 %r3221, %r3220, 2; + mov.u32 %r3222, _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE9block_max; + add.s32 %r3223, %r3222, %r3221; + st.shared.u32 [%r3223], %r6291; + bar.sync 0; + mov.u32 %r3224, %ntid.x; + shr.u32 %r6292, %r3224, 1; + setp.eq.s32 %p10, %r6292, 0; + @%p10 bra $L__BB3_13; + +$L__BB3_10: + setp.ge.u32 %p11, %r3220, %r6292; + @%p11 bra $L__BB3_12; + + add.s32 %r3230, %r6292, %r3220; + shl.b32 %r3231, %r6292, 2; + add.s32 %r3232, %r3223, %r3231; + ld.shared.u32 %r3233, [%r3232]; + ld.shared.u32 %r3234, [%r3223]; + setp.gt.u32 %p12, %r3234, %r3233; + selp.b32 %r3235, %r3220, %r3230, %p12; + shl.b32 %r3236, %r3235, 2; + add.s32 %r3237, %r3222, %r3236; + ld.shared.u32 %r3238, [%r3237]; + st.shared.u32 [%r3223], %r3238; + +$L__BB3_12: + bar.sync 0; + shr.u32 %r6292, %r6292, 1; + setp.ne.s32 %p13, %r6292, 0; + @%p13 bra $L__BB3_10; + +$L__BB3_13: + ld.shared.u32 %r25, [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE9block_max]; + +$L__BB3_15: + mov.u32 %r3240, %tid.x; + setp.ne.s32 %p14, %r3240, 0; + @%p14 bra $L__BB3_1261; + bra.uni $L__BB3_16; + +$L__BB3_1261: + ret; + +$L__BB3_16: + mov.u32 %r6286, %ctaid.x; + ld.param.u64 %rd681, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_5]; + setp.eq.s32 %p15, %r3200, 64; + setp.eq.s32 %p16, %r3201, 64; + and.pred %p17, %p15, %p16; + setp.eq.s32 %p18, %r3198, 64; + and.pred %p19, %p18, %p17; + mov.u32 %r3241, 1; + cvt.u64.u32 %rd5, %r3203; + cvta.to.global.u64 %rd64, %rd681; + mul.wide.u32 %rd65, %r6286, 32; + add.s64 %rd6, %rd64, %rd65; + st.global.u32 [%rd6], %r3241; + mov.u32 %r3243, 0; + st.global.u32 [%rd6+4], %r3243; + st.global.u32 [%rd6+8], %r3243; + st.global.u32 [%rd6+12], %r3243; + st.global.u32 [%rd6+16], %r3243; + st.global.u32 [%rd6+20], %r3243; + st.global.u32 [%rd6+24], %r3243; + st.global.u32 [%rd6+28], %r3243; + add.s64 %rd66, %rd1, %rd5; + add.s64 %rd7, %rd66, 20548; + @%p19 bra $L__BB3_669; + bra.uni $L__BB3_17; + +$L__BB3_669: + add.s32 %r1718, %r3202, -1; + setp.gt.u32 %p768, %r1718, 29; + setp.lt.u32 %p769, %r3204, 20549; + or.pred %p770, %p768, %p769; + @%p770 bra $L__BB3_1260; + bra.uni $L__BB3_670; + +$L__BB3_1260: + mov.u32 %r6281, 2; + st.global.u32 [%rd6], %r6281; + mov.u32 %r6282, 1; + st.global.u32 [%rd6+4], %r6282; + mov.u32 %r6283, 0; + st.global.u32 [%rd6+8], %r6283; + st.global.u32 [%rd6+12], %r6283; + st.global.u32 [%rd6+16], %r6283; + st.global.u32 [%rd6+20], %r6283; + st.global.u32 [%rd6+24], %r6283; + st.global.u32 [%rd6+28], %r6283; + bra.uni $L__BB3_1261; + +$L__BB3_17: + setp.eq.s32 %p1452, %r3201, 0; + add.s32 %r3244, %r3200, -1; + setp.ge.u32 %p21, %r3244, %r3198; + or.pred %p22, %p21, %p1452; + setp.gt.u32 %p23, %r3200, 1024; + or.pred %p24, %p23, %p22; + @%p24 bra $L__BB3_668; + + cvt.u16.u32 %rs449, %r3200; + mov.u16 %rs450, 4096; + div.u16 %rs451, %rs450, %rs449; + cvt.u32.u16 %r3245, %rs451; + setp.gt.u32 %p25, %r3201, %r3245; + add.s32 %r26, %r3202, -1; + setp.gt.u32 %p26, %r26, 29; + or.pred %p27, %p26, %p25; + setp.lt.u32 %p28, %r3204, 20549; + or.pred %p29, %p28, %p27; + @%p29 bra $L__BB3_668; + bra.uni $L__BB3_19; + +$L__BB3_668: + mov.u32 %r4834, 2; + st.global.u32 [%rd6], %r4834; + mov.u32 %r4835, 1; + st.global.u32 [%rd6+4], %r4835; + mov.u32 %r4836, 0; + st.global.u32 [%rd6+8], %r4836; + st.global.u32 [%rd6+12], %r4836; + st.global.u32 [%rd6+16], %r4836; + st.global.u32 [%rd6+20], %r4836; + st.global.u32 [%rd6+24], %r4836; + st.global.u32 [%rd6+28], %r4836; + bra.uni $L__BB3_1261; + +$L__BB3_670: + setp.eq.s32 %p771, %r3205, 1; + @%p771 bra $L__BB3_672; + bra.uni $L__BB3_671; + +$L__BB3_672: + setp.eq.s32 %p772, %r25, 0; + @%p772 bra $L__BB3_1259; + + clz.b32 %r4840, %r25; + mov.u32 %r4841, 32; + sub.s32 %r4842, %r4841, %r4840; + setp.gt.u32 %p773, %r4842, %r3202; + @%p773 bra $L__BB3_1258; + bra.uni $L__BB3_674; + +$L__BB3_1258: + mov.u32 %r6277, 1; + st.global.u32 [%rd6], %r6277; + mov.u32 %r6278, 2; + st.global.u32 [%rd6+4], %r6278; + mov.u32 %r6279, 0; + st.global.u32 [%rd6+8], %r6279; + st.global.u32 [%rd6+12], %r6279; + st.global.u32 [%rd6+16], %r6279; + st.global.u32 [%rd6+20], %r6279; + st.global.u32 [%rd6+24], %r6279; + st.global.u32 [%rd6+28], %r6279; + bra.uni $L__BB3_1261; + +$L__BB3_19: + setp.eq.s32 %p30, %r3205, 1; + @%p30 bra $L__BB3_21; + bra.uni $L__BB3_20; + +$L__BB3_21: + setp.eq.s32 %p31, %r25, 0; + @%p31 bra $L__BB3_667; + + clz.b32 %r3249, %r25; + mov.u32 %r3250, 32; + sub.s32 %r3251, %r3250, %r3249; + setp.gt.u32 %p32, %r3251, %r3202; + @%p32 bra $L__BB3_666; + bra.uni $L__BB3_23; + +$L__BB3_666: + mov.u32 %r4830, 1; + st.global.u32 [%rd6], %r4830; + mov.u32 %r4831, 2; + st.global.u32 [%rd6+4], %r4831; + mov.u32 %r4832, 0; + st.global.u32 [%rd6+8], %r4832; + st.global.u32 [%rd6+12], %r4832; + st.global.u32 [%rd6+16], %r4832; + st.global.u32 [%rd6+20], %r4832; + st.global.u32 [%rd6+24], %r4832; + st.global.u32 [%rd6+28], %r4832; + bra.uni $L__BB3_1261; + +$L__BB3_671: + mov.u32 %r4837, 2; + st.global.u32 [%rd6], %r4837; + mov.u32 %r4838, 5; + st.global.u32 [%rd6+4], %r4838; + mov.u32 %r4839, 0; + st.global.u32 [%rd6+8], %r4839; + st.global.u32 [%rd6+12], %r4839; + st.global.u32 [%rd6+16], %r4839; + st.global.u32 [%rd6+20], %r4839; + st.global.u32 [%rd6+24], %r4839; + st.global.u32 [%rd6+28], %r4839; + bra.uni $L__BB3_1261; + +$L__BB3_20: + mov.u32 %r3246, 2; + st.global.u32 [%rd6], %r3246; + mov.u32 %r3247, 5; + st.global.u32 [%rd6+4], %r3247; + mov.u32 %r3248, 0; + st.global.u32 [%rd6+8], %r3248; + st.global.u32 [%rd6+12], %r3248; + st.global.u32 [%rd6+16], %r3248; + st.global.u32 [%rd6+20], %r3248; + st.global.u32 [%rd6+24], %r3248; + st.global.u32 [%rd6+28], %r3248; + bra.uni $L__BB3_1261; + +$L__BB3_1259: + mov.u32 %r6280, 0; + st.global.u32 [%rd6], %r6280; + st.global.u32 [%rd6+4], %r6280; + st.global.u32 [%rd6+8], %r6280; + st.global.u32 [%rd6+12], %r6280; + st.global.u32 [%rd6+16], %r3202; + st.global.u32 [%rd6+20], %r6280; + st.global.u32 [%rd6+24], %r6280; + st.global.u32 [%rd6+28], %r6280; + bra.uni $L__BB3_1261; + +$L__BB3_674: + ld.param.u64 %rd683, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_2]; + mov.u16 %rs715, 255; + st.global.u8 [%rd7], %rs715; + mov.u32 %r4859, 0; + mov.u16 %rs1089, 0; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val], %rs1089; + mov.u32 %r7779, 1; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+1], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+1], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+2], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+2], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+3], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+3], %rs1089; + mov.u32 %r7778, 4; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+4], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+4], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+5], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+5], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+6], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+6], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+7], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+7], %rs1089; + mov.u32 %r7924, 8; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+8], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+8], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+9], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+9], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+10], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+10], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+11], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+11], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+12], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+12], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+13], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+13], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+14], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+14], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+15], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+15], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+16], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+16], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+17], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+17], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+18], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+18], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+19], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+19], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+20], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+20], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+21], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+21], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+22], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+22], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+23], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+23], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+24], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+24], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+25], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+25], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+26], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+26], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+27], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+27], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+28], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+28], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+29], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+29], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+30], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+30], %rs1089; + mov.u32 %r4860, 31; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+31], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+31], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+32], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+32], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+33], %rs1089; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+33], %rs1089; + sub.s32 %r1719, %r4860, %r3202; + shl.b64 %rd410, %rd4, 2; + add.s64 %rd687, %rd3, %rd410; + cvta.to.global.u64 %rd24, %rd683; + mov.u16 %rs1147, 15; + mov.u32 %r7305, %r4859; + mov.u32 %r7306, %r4859; + mov.u32 %r7925, %r4859; + mov.u32 %r7923, %r4859; + mov.u32 %r7922, %r4859; + mov.u32 %r8093, %r4859; + mov.u32 %r7777, %r7779; + mov.u32 %r7776, %r4859; + mov.u32 %r7382, %r4859; + mov.u32 %r7388, %r7924; + mov.u32 %r7543, %r4859; + mov.u32 %r7542, %r4859; + mov.u32 %r7541, %r7779; + mov.u32 %r7540, %r4859; + +$L__BB3_675: + ld.global.u32 %r1737, [%rd687]; + setp.eq.s32 %p774, %r1737, 0; + mov.u32 %r7322, %r4859; + @%p774 bra $L__BB3_677; + + and.b32 %r4862, %r1737, -2147483648; + abs.s32 %r4863, %r1737; + shl.b32 %r4864, %r4863, %r1719; + or.b32 %r7322, %r4864, %r4862; + +$L__BB3_677: + shl.b32 %r4868, %r7322, 1; + shr.u32 %r4869, %r4868, %r1719; + and.b32 %r1740, %r4869, -2; + setp.eq.s32 %p775, %r1740, 0; + mov.u32 %r7326, 0; + mov.u32 %r7323, %r7326; + mov.u32 %r7324, %r7326; + mov.u32 %r7330, %r7326; + @%p775 bra $L__BB3_679; + + add.s32 %r4871, %r1740, -1; + clz.b32 %r4872, %r4871; + mov.u32 %r4873, 32; + sub.s32 %r7323, %r4873, %r4872; + shr.u32 %r4874, %r7322, 31; + add.s32 %r4875, %r4874, %r1740; + add.s32 %r7324, %r4875, -2; + mov.u32 %r7330, 1; + +$L__BB3_679: + ld.global.u32 %r1746, [%rd687+256]; + setp.eq.s32 %p776, %r1746, 0; + @%p776 bra $L__BB3_681; + + and.b32 %r4877, %r1746, -2147483648; + abs.s32 %r4878, %r1746; + shl.b32 %r4879, %r4878, %r1719; + or.b32 %r7326, %r4879, %r4877; + +$L__BB3_681: + shl.b32 %r4882, %r7326, 1; + shr.u32 %r4883, %r4882, %r1719; + and.b32 %r1749, %r4883, -2; + setp.eq.s32 %p777, %r1749, 0; + mov.u32 %r7331, 0; + mov.u32 %r7327, %r7331; + mov.u32 %r7328, %r7331; + mov.u32 %r7334, %r7323; + @%p777 bra $L__BB3_683; + + or.b32 %r7330, %r7330, 2; + add.s32 %r4884, %r1749, -1; + clz.b32 %r4885, %r4884; + mov.u32 %r4886, 32; + sub.s32 %r7327, %r4886, %r4885; + max.s32 %r7334, %r7323, %r7327; + shr.u32 %r4887, %r7326, 31; + add.s32 %r4888, %r4887, %r1749; + add.s32 %r7328, %r4888, -2; + +$L__BB3_683: + ld.global.u32 %r1758, [%rd687+4]; + setp.eq.s32 %p778, %r1758, 0; + @%p778 bra $L__BB3_685; + + and.b32 %r4890, %r1758, -2147483648; + abs.s32 %r4891, %r1758; + shl.b32 %r4892, %r4891, %r1719; + or.b32 %r7331, %r4892, %r4890; + +$L__BB3_685: + shl.b32 %r4895, %r7331, 1; + shr.u32 %r4896, %r4895, %r1719; + and.b32 %r1761, %r4896, -2; + setp.eq.s32 %p779, %r1761, 0; + mov.u32 %r7336, 0; + mov.u32 %r7332, %r7336; + mov.u32 %r7333, %r7336; + @%p779 bra $L__BB3_687; + + or.b32 %r7330, %r7330, 4; + add.s32 %r4897, %r1761, -1; + clz.b32 %r4898, %r4897; + mov.u32 %r4899, 32; + sub.s32 %r7332, %r4899, %r4898; + max.s32 %r7334, %r7334, %r7332; + shr.u32 %r4900, %r7331, 31; + add.s32 %r4901, %r4900, %r1761; + add.s32 %r7333, %r4901, -2; + +$L__BB3_687: + ld.global.u32 %r1770, [%rd687+260]; + setp.eq.s32 %p780, %r1770, 0; + @%p780 bra $L__BB3_689; + + and.b32 %r4903, %r1770, -2147483648; + abs.s32 %r4904, %r1770; + shl.b32 %r4905, %r4904, %r1719; + or.b32 %r7336, %r4905, %r4903; + +$L__BB3_689: + shl.b32 %r4908, %r7336, 1; + shr.u32 %r4909, %r4908, %r1719; + and.b32 %r1773, %r4909, -2; + setp.eq.s32 %p781, %r1773, 0; + mov.u32 %r7341, 0; + mov.u32 %r7337, %r7341; + mov.u32 %r7338, %r7341; + @%p781 bra $L__BB3_691; + + or.b32 %r7330, %r7330, 8; + add.s32 %r4910, %r1773, -1; + clz.b32 %r4911, %r4910; + mov.u32 %r4912, 32; + sub.s32 %r7337, %r4912, %r4911; + max.s32 %r7334, %r7334, %r7337; + shr.u32 %r4913, %r7336, 31; + add.s32 %r4914, %r4913, %r1773; + add.s32 %r7338, %r4914, -2; + +$L__BB3_691: + add.s32 %r4916, %r7334, -1; + setp.lt.s32 %p782, %r7334, 2; + setp.gt.s32 %p783, %r7334, 1; + selp.b32 %r1782, %r4916, 0, %p783; + @%p782 bra $L__BB3_693; + + setp.eq.s32 %p784, %r7323, %r7334; + selp.u32 %r4917, 1, 0, %p784; + setp.eq.s32 %p785, %r7327, %r7334; + selp.u32 %r4918, -1, 0, %p785; + bfi.b32 %r4919, %r4918, %r4917, 1, 1; + setp.eq.s32 %p786, %r7332, %r7334; + selp.u16 %rs716, 1, 0, %p786; + mul.wide.u16 %r4920, %rs716, 4; + or.b32 %r4921, %r4919, %r4920; + setp.eq.s32 %p787, %r7337, %r7334; + selp.u16 %rs717, 1, 0, %p787; + mul.wide.u16 %r4922, %rs717, 8; + or.b32 %r7341, %r4921, %r4922; + +$L__BB3_693: + shr.u32 %r4923, %r7305, 1; + mov.u32 %r4924, _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val; + add.s32 %r1785, %r4924, %r4923; + ld.shared.u8 %rs718, [%r1785]; + cvt.u32.u16 %r4925, %rs718; + and.b32 %r4926, %r4925, 255; + and.b32 %r4927, %r7327, 255; + setp.lt.u32 %p788, %r4927, %r4926; + cvt.u16.u32 %rs719, %r7327; + selp.b16 %rs720, %rs718, %rs719, %p788; + st.shared.u8 [%r1785], %rs720; + cvt.u16.u32 %rs238, %r7337; + st.shared.u8 [%r1785+1], %rs238; + and.b32 %r1786, %r7330, 2; + cvt.u16.u32 %rs721, %r1786; + shr.u16 %rs722, %rs721, 1; + mov.u32 %r4928, _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val; + add.s32 %r1787, %r4928, %r4923; + ld.shared.u8 %rs723, [%r1787]; + or.b16 %rs724, %rs723, %rs722; + st.shared.u8 [%r1787], %rs724; + and.b32 %r1788, %r7330, 8; + shr.u32 %r1789, %r1788, 3; + st.shared.u8 [%r1787+1], %r1789; + shl.b32 %r4929, %r7330, 4; + shl.b32 %r4930, %r7306, 8; + or.b32 %r4931, %r4929, %r4930; + or.b32 %r4932, %r4931, %r7341; + mul.wide.u32 %rd411, %r4932, 2; + add.s64 %rd412, %rd24, %rd411; + ld.global.u16 %rs239, [%rd412]; + shr.u16 %rs725, %rs239, 4; + and.b16 %rs240, %rs725, 7; + setp.eq.s16 %p789, %rs240, 0; + mov.u32 %r7353, %r7776; + @%p789 bra $L__BB3_700; + + cvt.u32.u16 %r7342, %rs240; + shr.u16 %rs726, %rs239, 8; + cvt.u32.u16 %r7343, %rs726; + +$L__BB3_695: + mov.u32 %r1792, %r7342; + setp.gt.u32 %p790, %r7779, 2879; + mov.u32 %r7353, 1; + @%p790 bra $L__BB3_700; + + mov.u32 %r4934, 8; + sub.s32 %r4935, %r4934, %r7777; + sub.s32 %r4936, %r4935, %r7778; + min.u32 %r4937, %r4936, %r1792; + setp.eq.s32 %p791, %r4937, 32; + mov.u32 %r4938, -1; + shl.b32 %r4939, %r4938, %r4937; + not.b32 %r4940, %r4939; + selp.b32 %r4941, -1, %r4940, %p791; + and.b32 %r4942, %r4941, %r7343; + shl.b32 %r4943, %r4942, %r7778; + cvt.u16.u32 %rs727, %r4943; + or.b16 %rs1147, %rs1147, %rs727; + add.s32 %r7778, %r4937, %r7778; + sub.s32 %r7342, %r1792, %r4937; + shr.u32 %r7343, %r7343, %r4937; + setp.gt.u32 %p792, %r4936, %r1792; + @%p792 bra $L__BB3_699; + + setp.ne.s32 %p793, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs728, %rs1147, 255; + setp.ne.s16 %p794, %rs728, 127; + and.pred %p795, %p793, %p794; + @%p795 bra $L__BB3_699; + + mov.u32 %r4946, 20548; + sub.s32 %r4947, %r4946, %r7779; + cvt.u64.u32 %rd413, %r4947; + add.s64 %rd414, %rd413, %rd5; + add.s64 %rd415, %rd1, %rd414; + st.global.u8 [%rd415], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p796, %rs728, 143; + selp.u32 %r7777, 1, 0, %p796; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_699: + setp.ne.s32 %p797, %r7342, 0; + mov.u32 %r7353, %r7776; + @%p797 bra $L__BB3_695; + +$L__BB3_700: + setp.ne.s32 %p798, %r7306, 0; + @%p798 bra $L__BB3_748; + + setp.eq.s32 %p799, %r7330, 0; + add.s32 %r4948, %r7382, 17477; + cvt.u64.u32 %rd416, %r4948; + add.s64 %rd417, %rd416, %rd5; + add.s64 %rd26, %rd1, %rd417; + @%p799 bra $L__BB3_740; + + shl.b16 %rs1089, %rs1089, 1; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p800, %r7388, 0; + mov.u32 %r7387, %r7540; + @%p800 bra $L__BB3_705; + + setp.gt.u32 %p801, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7387, 1; + @%p801 bra $L__BB3_705; + + st.global.u8 [%rd26], %rs1089; + add.s32 %r7382, %r7382, 1; + mov.u32 %r7388, 8; + mov.u16 %rs1089, 0; + mov.u32 %r7387, %r7540; + +$L__BB3_705: + setp.lt.u32 %p802, %r7542, 3; + mov.u32 %r7357, 0; + @%p802 bra $L__BB3_708; + + setp.lt.u32 %p803, %r7542, 6; + mov.u32 %r7357, 1; + @%p803 bra $L__BB3_708; + + setp.lt.u32 %p804, %r7542, 9; + setp.eq.s32 %p805, %r7542, 11; + selp.b32 %r4954, 4, 5, %p805; + setp.lt.u32 %p806, %r7542, 11; + selp.b32 %r4955, 3, %r4954, %p806; + selp.b32 %r7357, 2, %r4955, %p804; + +$L__BB3_708: + setp.eq.s32 %p807, %r7357, 0; + @%p807 bra $L__BB3_736; + + add.s32 %r1816, %r7357, -1; + and.b32 %r1817, %r7357, 3; + setp.eq.s32 %p808, %r1817, 0; + mov.u32 %r7367, %r7357; + mov.u32 %r7370, %r7387; + @%p808 bra $L__BB3_721; + + mov.u32 %r4957, 1; + shl.b32 %r4958, %r4957, %r1816; + and.b32 %r4959, %r4958, %r7543; + setp.ne.s32 %p809, %r4959, 0; + selp.u32 %r4960, 1, 0, %p809; + cvt.u32.u16 %r4961, %rs1089; + bfi.b32 %r4962, %r4961, %r4960, 1, 8; + cvt.u16.u32 %rs1089, %r4962; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p810, %r7388, 0; + mov.u32 %r7370, %r7387; + @%p810 bra $L__BB3_713; + + setp.gt.u32 %p811, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7370, %r4957; + @%p811 bra $L__BB3_713; + + add.s32 %r4966, %r7382, 17477; + cvt.u64.u32 %rd418, %r4966; + add.s64 %rd419, %rd418, %rd5; + add.s64 %rd420, %rd1, %rd419; + st.global.u8 [%rd420], %rs1089; + add.s32 %r7382, %r7382, 1; + mov.u32 %r7388, 8; + mov.u16 %rs1089, 0; + mov.u32 %r7370, %r7387; + +$L__BB3_713: + setp.eq.s32 %p812, %r1817, 1; + mov.u32 %r7387, %r7370; + mov.u32 %r7367, %r1816; + @%p812 bra $L__BB3_721; + + add.s32 %r7367, %r7357, -2; + mov.u32 %r4967, 1; + shl.b32 %r4968, %r4967, %r7367; + and.b32 %r4969, %r4968, %r7543; + setp.ne.s32 %p813, %r4969, 0; + selp.u32 %r4970, 1, 0, %p813; + cvt.u32.u16 %r4971, %rs1089; + bfi.b32 %r4972, %r4971, %r4970, 1, 8; + cvt.u16.u32 %rs1089, %r4972; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p814, %r7388, 0; + mov.u32 %r7361, %r7370; + @%p814 bra $L__BB3_717; + + setp.gt.u32 %p815, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7361, %r4967; + @%p815 bra $L__BB3_717; + + add.s32 %r4975, %r7382, 17477; + cvt.u64.u32 %rd421, %r4975; + add.s64 %rd422, %rd421, %rd5; + add.s64 %rd423, %rd1, %rd422; + and.b16 %rs735, %rs1089, 255; + st.global.u8 [%rd423], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p816, %rs735, 255; + selp.b32 %r7388, 7, 8, %p816; + mov.u16 %rs1089, 0; + mov.u32 %r7361, %r7370; + +$L__BB3_717: + setp.eq.s32 %p817, %r1817, 2; + mov.u32 %r7387, %r7361; + mov.u32 %r7370, %r7361; + @%p817 bra $L__BB3_721; + + add.s32 %r7367, %r7357, -3; + mov.u32 %r4976, 1; + shl.b32 %r4977, %r4976, %r7367; + and.b32 %r4978, %r4977, %r7543; + setp.ne.s32 %p818, %r4978, 0; + selp.u32 %r4979, 1, 0, %p818; + cvt.u32.u16 %r4980, %rs1089; + bfi.b32 %r4981, %r4980, %r4979, 1, 8; + cvt.u16.u32 %rs1089, %r4981; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p819, %r7388, 0; + mov.u32 %r7387, %r7361; + mov.u32 %r7370, %r7361; + @%p819 bra $L__BB3_721; + + setp.gt.u32 %p820, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7387, %r4976; + mov.u32 %r7370, %r4976; + @%p820 bra $L__BB3_721; + + add.s32 %r4986, %r7382, 17477; + cvt.u64.u32 %rd424, %r4986; + add.s64 %rd425, %rd424, %rd5; + add.s64 %rd426, %rd1, %rd425; + and.b16 %rs738, %rs1089, 255; + st.global.u8 [%rd426], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p821, %rs738, 255; + selp.b32 %r7388, 7, 8, %p821; + mov.u16 %rs1089, 0; + mov.u32 %r7387, %r7361; + mov.u32 %r7370, %r7361; + +$L__BB3_721: + setp.lt.u32 %p822, %r1816, 3; + @%p822 bra $L__BB3_736; + + mov.u32 %r7387, %r7370; + +$L__BB3_723: + add.s32 %r4987, %r7367, -1; + mov.u32 %r4988, 1; + shl.b32 %r4989, %r4988, %r4987; + and.b32 %r4990, %r4989, %r7543; + setp.ne.s32 %p823, %r4990, 0; + selp.u32 %r4991, 1, 0, %p823; + cvt.u32.u16 %r4992, %rs1089; + bfi.b32 %r7376, %r4992, %r4991, 1, 8; + add.s32 %r7377, %r7388, -1; + setp.ne.s32 %p824, %r7377, 0; + mov.u32 %r7375, %r7387; + @%p824 bra $L__BB3_726; + + setp.gt.u32 %p825, %r7382, 191; + mov.u32 %r7377, 0; + mov.u32 %r7375, %r4988; + @%p825 bra $L__BB3_726; + + cvt.u16.u32 %rs739, %r7376; + and.b16 %rs740, %rs739, 255; + add.s32 %r4996, %r7382, 17477; + cvt.u64.u32 %rd427, %r4996; + add.s64 %rd428, %rd427, %rd5; + add.s64 %rd429, %rd1, %rd428; + st.global.u8 [%rd429], %rs739; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p826, %rs740, 255; + selp.b32 %r7377, 7, 8, %p826; + mov.u32 %r7376, 0; + mov.u32 %r7375, %r7387; + +$L__BB3_726: + add.s32 %r4997, %r7367, -2; + shl.b32 %r4999, %r4988, %r4997; + and.b32 %r5000, %r4999, %r7543; + setp.ne.s32 %p827, %r5000, 0; + and.b32 %r5001, %r7376, 127; + selp.u32 %r5002, 1, 0, %p827; + bfi.b32 %r7380, %r5001, %r5002, 1, 7; + add.s32 %r7381, %r7377, -1; + setp.ne.s32 %p828, %r7381, 0; + mov.u32 %r7379, %r7375; + @%p828 bra $L__BB3_729; + + setp.gt.u32 %p829, %r7382, 191; + mov.u32 %r7381, 0; + mov.u32 %r7379, 1; + @%p829 bra $L__BB3_729; + + cvt.u16.u32 %rs741, %r7380; + and.b16 %rs742, %rs741, 255; + add.s32 %r5006, %r7382, 17477; + cvt.u64.u32 %rd430, %r5006; + add.s64 %rd431, %rd430, %rd5; + add.s64 %rd432, %rd1, %rd431; + st.global.u8 [%rd432], %rs741; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p830, %rs742, 255; + selp.b32 %r7381, 7, 8, %p830; + mov.u32 %r7380, 0; + mov.u32 %r7379, %r7375; + +$L__BB3_729: + add.s32 %r5007, %r7367, -3; + mov.u32 %r5008, 1; + shl.b32 %r5009, %r5008, %r5007; + and.b32 %r5010, %r5009, %r7543; + setp.ne.s32 %p831, %r5010, 0; + and.b32 %r5011, %r7380, 127; + selp.u32 %r5012, 1, 0, %p831; + bfi.b32 %r7384, %r5011, %r5012, 1, 7; + add.s32 %r7385, %r7381, -1; + setp.ne.s32 %p832, %r7385, 0; + mov.u32 %r7383, %r7379; + @%p832 bra $L__BB3_732; + + setp.gt.u32 %p833, %r7382, 191; + mov.u32 %r7385, 0; + mov.u32 %r7383, %r5008; + @%p833 bra $L__BB3_732; + + cvt.u16.u32 %rs743, %r7384; + and.b16 %rs744, %rs743, 255; + add.s32 %r5016, %r7382, 17477; + cvt.u64.u32 %rd433, %r5016; + add.s64 %rd434, %rd433, %rd5; + add.s64 %rd435, %rd1, %rd434; + st.global.u8 [%rd435], %rs743; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p834, %rs744, 255; + selp.b32 %r7385, 7, 8, %p834; + mov.u32 %r7384, 0; + mov.u32 %r7383, %r7379; + +$L__BB3_732: + add.s32 %r7367, %r7367, -4; + shl.b32 %r5018, %r5008, %r7367; + and.b32 %r5019, %r5018, %r7543; + setp.ne.s32 %p835, %r5019, 0; + and.b32 %r5020, %r7384, 127; + selp.u32 %r5021, 1, 0, %p835; + bfi.b32 %r5022, %r5020, %r5021, 1, 15; + cvt.u16.u32 %rs1089, %r5022; + add.s32 %r7388, %r7385, -1; + setp.ne.s32 %p836, %r7388, 0; + mov.u32 %r7387, %r7383; + @%p836 bra $L__BB3_735; + + setp.gt.u32 %p837, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7387, 1; + @%p837 bra $L__BB3_735; + + add.s32 %r5025, %r7382, 17477; + cvt.u64.u32 %rd436, %r5025; + add.s64 %rd437, %rd436, %rd5; + add.s64 %rd438, %rd1, %rd437; + and.b16 %rs746, %rs1089, 255; + st.global.u8 [%rd438], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p838, %rs746, 255; + selp.b32 %r7388, 7, 8, %p838; + mov.u16 %rs1089, 0; + mov.u32 %r7387, %r7383; + +$L__BB3_735: + setp.ne.s32 %p839, %r7367, 0; + @%p839 bra $L__BB3_723; + +$L__BB3_736: + add.s32 %r5027, %r7542, -1; + setp.eq.s32 %p840, %r7542, 0; + mov.u32 %r7543, 0; + selp.b32 %r7542, 0, %r5027, %p840; + setp.lt.u32 %p841, %r7542, 3; + mov.u32 %r7393, %r7543; + @%p841 bra $L__BB3_739; + + setp.lt.u32 %p842, %r7542, 6; + mov.u32 %r7393, 1; + @%p842 bra $L__BB3_739; + + setp.lt.u32 %p843, %r7542, 9; + setp.eq.s32 %p844, %r7542, 11; + selp.b32 %r5029, 4, 5, %p844; + setp.lt.u32 %p845, %r7542, 11; + selp.b32 %r5030, 3, %r5029, %p845; + selp.b32 %r7393, 2, %r5030, %p843; + +$L__BB3_739: + mov.u32 %r5032, 1; + shl.b32 %r7541, %r5032, %r7393; + mov.u32 %r7540, %r7387; + bra.uni $L__BB3_748; + +$L__BB3_740: + add.s32 %r7543, %r7543, 1; + setp.lt.u32 %p846, %r7543, %r7541; + @%p846 bra $L__BB3_748; + + shl.b16 %rs747, %rs1089, 1; + or.b16 %rs1089, %rs747, 1; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p847, %r7388, 0; + mov.u32 %r7394, %r7540; + @%p847 bra $L__BB3_744; + + setp.gt.u32 %p848, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7394, 1; + @%p848 bra $L__BB3_744; + + and.b16 %rs749, %rs1089, 255; + st.global.u8 [%rd26], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p849, %rs749, 255; + selp.b32 %r7388, 7, 8, %p849; + mov.u16 %rs1089, 0; + mov.u32 %r7394, %r7540; + +$L__BB3_744: + add.s32 %r5036, %r7542, 1; + min.u32 %r7542, %r5036, 12; + setp.lt.u32 %p850, %r7542, 3; + mov.u32 %r7543, 0; + mov.u32 %r7397, %r7543; + @%p850 bra $L__BB3_747; + + setp.lt.u32 %p851, %r7542, 6; + mov.u32 %r7397, 1; + @%p851 bra $L__BB3_747; + + setp.lt.u32 %p852, %r7542, 9; + setp.eq.s32 %p853, %r7542, 11; + selp.b32 %r5038, 4, 5, %p853; + setp.lt.u32 %p854, %r7542, 11; + selp.b32 %r5039, 3, %r5038, %p854; + selp.b32 %r7397, 2, %r5039, %p852; + +$L__BB3_747: + mov.u32 %r5041, 1; + shl.b32 %r7541, %r5041, %r7397; + mov.u32 %r7540, %r7394; + +$L__BB3_748: + max.s32 %r1900, %r7334, 1; + and.b16 %rs750, %rs239, 15; + cvt.u32.u16 %r1901, %rs750; + and.b32 %r1902, %r7330, 1; + setp.eq.s32 %p855, %r1902, 0; + mov.u32 %r7414, %r8093; + @%p855 bra $L__BB3_755; + + and.b32 %r5042, %r1901, 1; + sub.s32 %r7404, %r1900, %r5042; + setp.eq.s32 %p856, %r7404, 0; + mov.u32 %r7414, %r8093; + @%p856 bra $L__BB3_755; + + mov.u32 %r5043, -1; + shl.b32 %r5044, %r5043, %r7404; + not.b32 %r5045, %r5044; + and.b32 %r7405, %r7324, %r5045; + +$L__BB3_751: + setp.gt.u32 %p857, %r7925, 17476; + mov.u32 %r7414, 1; + @%p857 bra $L__BB3_755; + + sub.s32 %r5047, %r7924, %r7923; + min.u32 %r5048, %r5047, %r7404; + setp.eq.s32 %p858, %r5048, 32; + mov.u32 %r5049, -1; + shl.b32 %r5050, %r5049, %r5048; + not.b32 %r5051, %r5050; + selp.b32 %r5052, -1, %r5051, %p858; + and.b32 %r5053, %r5052, %r7405; + shl.b32 %r5054, %r5053, %r7923; + or.b32 %r7922, %r5054, %r7922; + add.s32 %r7923, %r5048, %r7923; + shr.u32 %r7405, %r7405, %r5048; + sub.s32 %r7404, %r7404, %r5048; + setp.lt.u32 %p859, %r7923, %r7924; + @%p859 bra $L__BB3_754; + + cvt.u64.u32 %rd439, %r7925; + add.s64 %rd440, %rd439, %rd5; + add.s64 %rd441, %rd1, %rd440; + st.global.u8 [%rd441], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p860, %r7922, 255; + selp.b32 %r7924, 7, 8, %p860; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_754: + setp.ne.s32 %p861, %r7404, 0; + mov.u32 %r7414, %r8093; + @%p861 bra $L__BB3_751; + +$L__BB3_755: + setp.eq.s32 %p862, %r1786, 0; + mov.u32 %r7429, %r7414; + @%p862 bra $L__BB3_762; + + shr.u32 %r5057, %r1901, 1; + and.b32 %r5058, %r5057, 1; + sub.s32 %r7419, %r1900, %r5058; + setp.eq.s32 %p863, %r7419, 0; + mov.u32 %r7429, %r7414; + @%p863 bra $L__BB3_762; + + mov.u32 %r5059, -1; + shl.b32 %r5060, %r5059, %r7419; + not.b32 %r5061, %r5060; + and.b32 %r7420, %r7328, %r5061; + +$L__BB3_758: + setp.gt.u32 %p864, %r7925, 17476; + mov.u32 %r7429, 1; + @%p864 bra $L__BB3_762; + + sub.s32 %r5063, %r7924, %r7923; + min.u32 %r5064, %r5063, %r7419; + setp.eq.s32 %p865, %r5064, 32; + mov.u32 %r5065, -1; + shl.b32 %r5066, %r5065, %r5064; + not.b32 %r5067, %r5066; + selp.b32 %r5068, -1, %r5067, %p865; + and.b32 %r5069, %r5068, %r7420; + shl.b32 %r5070, %r5069, %r7923; + or.b32 %r7922, %r5070, %r7922; + add.s32 %r7923, %r5064, %r7923; + shr.u32 %r7420, %r7420, %r5064; + sub.s32 %r7419, %r7419, %r5064; + setp.lt.u32 %p866, %r7923, %r7924; + @%p866 bra $L__BB3_761; + + cvt.u64.u32 %rd442, %r7925; + add.s64 %rd443, %rd442, %rd5; + add.s64 %rd444, %rd1, %rd443; + st.global.u8 [%rd444], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p867, %r7922, 255; + selp.b32 %r7924, 7, 8, %p867; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_761: + setp.ne.s32 %p868, %r7419, 0; + mov.u32 %r7429, %r7414; + @%p868 bra $L__BB3_758; + +$L__BB3_762: + and.b32 %r5073, %r7330, 4; + setp.eq.s32 %p869, %r5073, 0; + mov.u32 %r7444, %r7429; + @%p869 bra $L__BB3_769; + + shr.u32 %r5074, %r1901, 2; + and.b32 %r5075, %r5074, 1; + sub.s32 %r7434, %r1900, %r5075; + setp.eq.s32 %p870, %r7434, 0; + mov.u32 %r7444, %r7429; + @%p870 bra $L__BB3_769; + + mov.u32 %r5076, -1; + shl.b32 %r5077, %r5076, %r7434; + not.b32 %r5078, %r5077; + and.b32 %r7435, %r7333, %r5078; + +$L__BB3_765: + setp.gt.u32 %p871, %r7925, 17476; + mov.u32 %r7444, 1; + @%p871 bra $L__BB3_769; + + sub.s32 %r5080, %r7924, %r7923; + min.u32 %r5081, %r5080, %r7434; + setp.eq.s32 %p872, %r5081, 32; + mov.u32 %r5082, -1; + shl.b32 %r5083, %r5082, %r5081; + not.b32 %r5084, %r5083; + selp.b32 %r5085, -1, %r5084, %p872; + and.b32 %r5086, %r5085, %r7435; + shl.b32 %r5087, %r5086, %r7923; + or.b32 %r7922, %r5087, %r7922; + add.s32 %r7923, %r5081, %r7923; + shr.u32 %r7435, %r7435, %r5081; + sub.s32 %r7434, %r7434, %r5081; + setp.lt.u32 %p873, %r7923, %r7924; + @%p873 bra $L__BB3_768; + + cvt.u64.u32 %rd445, %r7925; + add.s64 %rd446, %rd445, %rd5; + add.s64 %rd447, %rd1, %rd446; + st.global.u8 [%rd447], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p874, %r7922, 255; + selp.b32 %r7924, 7, 8, %p874; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_768: + setp.ne.s32 %p875, %r7434, 0; + mov.u32 %r7444, %r7429; + @%p875 bra $L__BB3_765; + +$L__BB3_769: + setp.eq.s32 %p876, %r1788, 0; + mov.u32 %r7459, %r7444; + @%p876 bra $L__BB3_776; + + shr.u32 %r5090, %r1901, 3; + sub.s32 %r7449, %r1900, %r5090; + setp.eq.s32 %p877, %r7449, 0; + mov.u32 %r7459, %r7444; + @%p877 bra $L__BB3_776; + + mov.u32 %r5091, -1; + shl.b32 %r5092, %r5091, %r7449; + not.b32 %r5093, %r5092; + and.b32 %r7450, %r7338, %r5093; + +$L__BB3_772: + setp.gt.u32 %p878, %r7925, 17476; + mov.u32 %r7459, 1; + @%p878 bra $L__BB3_776; + + sub.s32 %r5095, %r7924, %r7923; + min.u32 %r5096, %r5095, %r7449; + setp.eq.s32 %p879, %r5096, 32; + mov.u32 %r5097, -1; + shl.b32 %r5098, %r5097, %r5096; + not.b32 %r5099, %r5098; + selp.b32 %r5100, -1, %r5099, %p879; + and.b32 %r5101, %r5100, %r7450; + shl.b32 %r5102, %r5101, %r7923; + or.b32 %r7922, %r5102, %r7922; + add.s32 %r7923, %r5096, %r7923; + shr.u32 %r7450, %r7450, %r5096; + sub.s32 %r7449, %r7449, %r5096; + setp.lt.u32 %p880, %r7923, %r7924; + @%p880 bra $L__BB3_775; + + cvt.u64.u32 %rd448, %r7925; + add.s64 %rd449, %rd448, %rd5; + add.s64 %rd450, %rd1, %rd449; + st.global.u8 [%rd450], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p881, %r7922, 255; + selp.b32 %r7924, 7, 8, %p881; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_775: + setp.ne.s32 %p882, %r7449, 0; + mov.u32 %r7459, %r7444; + @%p882 bra $L__BB3_772; + +$L__BB3_776: + ld.global.u32 %r1995, [%rd687+8]; + setp.eq.s32 %p883, %r1995, 0; + mov.u32 %r7465, 0; + mov.u32 %r7464, %r7465; + @%p883 bra $L__BB3_778; + + and.b32 %r5106, %r1995, -2147483648; + abs.s32 %r5107, %r1995; + shl.b32 %r5108, %r5107, %r1719; + or.b32 %r7464, %r5108, %r5106; + +$L__BB3_778: + shl.b32 %r5112, %r7464, 1; + shr.u32 %r5113, %r5112, %r1719; + and.b32 %r1998, %r5113, -2; + setp.eq.s32 %p884, %r1998, 0; + mov.u32 %r7466, %r7465; + mov.u32 %r7472, %r7465; + @%p884 bra $L__BB3_780; + + add.s32 %r5115, %r1998, -1; + clz.b32 %r5116, %r5115; + mov.u32 %r5117, 32; + sub.s32 %r7465, %r5117, %r5116; + shr.u32 %r5118, %r7464, 31; + add.s32 %r5119, %r5118, %r1998; + add.s32 %r7466, %r5119, -2; + mov.u32 %r7472, 1; + +$L__BB3_780: + ld.global.u32 %r2004, [%rd687+264]; + setp.eq.s32 %p885, %r2004, 0; + mov.u32 %r7469, 0; + mov.u32 %r7468, %r7469; + @%p885 bra $L__BB3_782; + + and.b32 %r5121, %r2004, -2147483648; + abs.s32 %r5122, %r2004; + shl.b32 %r5123, %r5122, %r1719; + or.b32 %r7468, %r5123, %r5121; + +$L__BB3_782: + shl.b32 %r5126, %r7468, 1; + shr.u32 %r5127, %r5126, %r1719; + and.b32 %r2007, %r5127, -2; + setp.eq.s32 %p886, %r2007, 0; + mov.u32 %r7470, %r7469; + mov.u32 %r7476, %r7465; + @%p886 bra $L__BB3_784; + + or.b32 %r7472, %r7472, 2; + add.s32 %r5128, %r2007, -1; + clz.b32 %r5129, %r5128; + mov.u32 %r5130, 32; + sub.s32 %r7469, %r5130, %r5129; + max.s32 %r7476, %r7465, %r7469; + shr.u32 %r5131, %r7468, 31; + add.s32 %r5132, %r5131, %r2007; + add.s32 %r7470, %r5132, -2; + +$L__BB3_784: + ld.global.u32 %r2016, [%rd687+12]; + setp.eq.s32 %p887, %r2016, 0; + mov.u32 %r7474, 0; + mov.u32 %r7473, %r7474; + @%p887 bra $L__BB3_786; + + and.b32 %r5134, %r2016, -2147483648; + abs.s32 %r5135, %r2016; + shl.b32 %r5136, %r5135, %r1719; + or.b32 %r7473, %r5136, %r5134; + +$L__BB3_786: + shl.b32 %r5139, %r7473, 1; + shr.u32 %r5140, %r5139, %r1719; + and.b32 %r2019, %r5140, -2; + setp.eq.s32 %p888, %r2019, 0; + mov.u32 %r7475, %r7474; + @%p888 bra $L__BB3_788; + + or.b32 %r7472, %r7472, 4; + add.s32 %r5141, %r2019, -1; + clz.b32 %r5142, %r5141; + mov.u32 %r5143, 32; + sub.s32 %r7474, %r5143, %r5142; + max.s32 %r7476, %r7476, %r7474; + shr.u32 %r5144, %r7473, 31; + add.s32 %r5145, %r5144, %r2019; + add.s32 %r7475, %r5145, -2; + +$L__BB3_788: + ld.global.u32 %r2028, [%rd687+268]; + setp.eq.s32 %p889, %r2028, 0; + mov.u32 %r7479, 0; + mov.u32 %r7478, %r7479; + @%p889 bra $L__BB3_790; + + and.b32 %r5147, %r2028, -2147483648; + abs.s32 %r5148, %r2028; + shl.b32 %r5149, %r5148, %r1719; + or.b32 %r7478, %r5149, %r5147; + +$L__BB3_790: + shl.b32 %r5152, %r7478, 1; + shr.u32 %r5153, %r5152, %r1719; + and.b32 %r2031, %r5153, -2; + setp.eq.s32 %p890, %r2031, 0; + mov.u32 %r7480, %r7479; + @%p890 bra $L__BB3_792; + + or.b32 %r7472, %r7472, 8; + add.s32 %r5154, %r2031, -1; + clz.b32 %r5155, %r5154; + mov.u32 %r5156, 32; + sub.s32 %r7479, %r5156, %r5155; + max.s32 %r7476, %r7476, %r7479; + shr.u32 %r5157, %r7478, 31; + add.s32 %r5158, %r5157, %r2031; + add.s32 %r7480, %r5158, -2; + +$L__BB3_792: + shr.u32 %r5160, %r7330, 1; + or.b32 %r2040, %r5160, %r1902; + add.s32 %r5161, %r7476, -1; + setp.lt.s32 %p891, %r7476, 2; + setp.gt.s32 %p892, %r7476, 1; + selp.b32 %r2041, %r5161, 0, %p892; + mov.u32 %r7483, 0; + @%p891 bra $L__BB3_794; + + setp.eq.s32 %p893, %r7465, %r7476; + selp.u32 %r5162, 1, 0, %p893; + setp.eq.s32 %p894, %r7469, %r7476; + selp.u32 %r5163, -1, 0, %p894; + bfi.b32 %r5164, %r5163, %r5162, 1, 1; + setp.eq.s32 %p895, %r7474, %r7476; + selp.u16 %rs751, 1, 0, %p895; + mul.wide.u16 %r5165, %rs751, 4; + or.b32 %r5166, %r5164, %r5165; + setp.eq.s32 %p896, %r7479, %r7476; + selp.u16 %rs752, 1, 0, %p896; + mul.wide.u16 %r5167, %rs752, 8; + or.b32 %r7483, %r5166, %r5167; + +$L__BB3_794: + and.b32 %r5168, %r7469, 255; + and.b32 %r5169, %r7337, 255; + setp.lt.u32 %p897, %r5168, %r5169; + cvt.u16.u32 %rs753, %r7469; + selp.b16 %rs754, %rs238, %rs753, %p897; + st.shared.u8 [%r1785+1], %rs754; + st.shared.u8 [%r1785+2], %r7479; + and.b32 %r2044, %r7472, 2; + shr.u32 %r5170, %r2044, 1; + or.b32 %r5171, %r1789, %r5170; + st.shared.u8 [%r1787+1], %r5171; + and.b32 %r2045, %r7472, 8; + shr.u32 %r5172, %r2045, 3; + st.shared.u8 [%r1787+2], %r5172; + shl.b32 %r5173, %r7472, 4; + shl.b32 %r5174, %r2040, 8; + or.b32 %r5175, %r5173, %r5174; + or.b32 %r5176, %r5175, %r7483; + mul.wide.u32 %rd451, %r5176, 2; + add.s64 %rd452, %rd24, %rd451; + ld.global.u16 %rs261, [%rd452]; + shr.u16 %rs755, %rs261, 4; + and.b16 %rs262, %rs755, 7; + setp.eq.s16 %p898, %rs262, 0; + mov.u32 %r7495, %r7353; + @%p898 bra $L__BB3_801; + + cvt.u32.u16 %r7484, %rs262; + shr.u16 %rs756, %rs261, 8; + cvt.u32.u16 %r7485, %rs756; + +$L__BB3_796: + mov.u32 %r2048, %r7484; + setp.gt.u32 %p899, %r7779, 2879; + mov.u32 %r7495, 1; + @%p899 bra $L__BB3_801; + + mov.u32 %r5178, 8; + sub.s32 %r5179, %r5178, %r7777; + sub.s32 %r5180, %r5179, %r7778; + min.u32 %r5181, %r5180, %r2048; + setp.eq.s32 %p900, %r5181, 32; + mov.u32 %r5182, -1; + shl.b32 %r5183, %r5182, %r5181; + not.b32 %r5184, %r5183; + selp.b32 %r5185, -1, %r5184, %p900; + and.b32 %r5186, %r5185, %r7485; + shl.b32 %r5187, %r5186, %r7778; + cvt.u16.u32 %rs757, %r5187; + or.b16 %rs1147, %rs1147, %rs757; + add.s32 %r7778, %r5181, %r7778; + sub.s32 %r7484, %r2048, %r5181; + shr.u32 %r7485, %r7485, %r5181; + setp.gt.u32 %p901, %r5180, %r2048; + @%p901 bra $L__BB3_800; + + setp.ne.s32 %p902, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs758, %rs1147, 255; + setp.ne.s16 %p903, %rs758, 127; + and.pred %p904, %p902, %p903; + @%p904 bra $L__BB3_800; + + mov.u32 %r5190, 20548; + sub.s32 %r5191, %r5190, %r7779; + cvt.u64.u32 %rd453, %r5191; + add.s64 %rd454, %rd453, %rd5; + add.s64 %rd455, %rd1, %rd454; + st.global.u8 [%rd455], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p905, %rs758, 143; + selp.u32 %r7777, 1, 0, %p905; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_800: + setp.ne.s32 %p906, %r7484, 0; + mov.u32 %r7495, %r7353; + @%p906 bra $L__BB3_796; + +$L__BB3_801: + setp.ne.s32 %p907, %r2040, 0; + @%p907 bra $L__BB3_849; + + setp.eq.s32 %p908, %r7472, 0; + add.s32 %r5192, %r7382, 17477; + cvt.u64.u32 %rd456, %r5192; + add.s64 %rd457, %rd456, %rd5; + add.s64 %rd27, %rd1, %rd457; + @%p908 bra $L__BB3_841; + + shl.b16 %rs1089, %rs1089, 1; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p909, %r7388, 0; + mov.u32 %r7529, %r7540; + @%p909 bra $L__BB3_806; + + setp.gt.u32 %p910, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7529, 1; + @%p910 bra $L__BB3_806; + + st.global.u8 [%rd27], %rs1089; + add.s32 %r7382, %r7382, 1; + mov.u32 %r7388, 8; + mov.u16 %rs1089, 0; + mov.u32 %r7529, %r7540; + +$L__BB3_806: + setp.lt.u32 %p911, %r7542, 3; + mov.u32 %r7499, 0; + @%p911 bra $L__BB3_809; + + setp.lt.u32 %p912, %r7542, 6; + mov.u32 %r7499, 1; + @%p912 bra $L__BB3_809; + + setp.lt.u32 %p913, %r7542, 9; + setp.eq.s32 %p914, %r7542, 11; + selp.b32 %r5198, 4, 5, %p914; + setp.lt.u32 %p915, %r7542, 11; + selp.b32 %r5199, 3, %r5198, %p915; + selp.b32 %r7499, 2, %r5199, %p913; + +$L__BB3_809: + setp.eq.s32 %p916, %r7499, 0; + @%p916 bra $L__BB3_837; + + add.s32 %r2072, %r7499, -1; + and.b32 %r2073, %r7499, 3; + setp.eq.s32 %p917, %r2073, 0; + mov.u32 %r7509, %r7499; + mov.u32 %r7512, %r7529; + @%p917 bra $L__BB3_822; + + mov.u32 %r5201, 1; + shl.b32 %r5202, %r5201, %r2072; + and.b32 %r5203, %r5202, %r7543; + setp.ne.s32 %p918, %r5203, 0; + selp.u32 %r5204, 1, 0, %p918; + cvt.u32.u16 %r5205, %rs1089; + bfi.b32 %r5206, %r5205, %r5204, 1, 8; + cvt.u16.u32 %rs1089, %r5206; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p919, %r7388, 0; + mov.u32 %r7512, %r7529; + @%p919 bra $L__BB3_814; + + setp.gt.u32 %p920, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7512, %r5201; + @%p920 bra $L__BB3_814; + + add.s32 %r5210, %r7382, 17477; + cvt.u64.u32 %rd458, %r5210; + add.s64 %rd459, %rd458, %rd5; + add.s64 %rd460, %rd1, %rd459; + st.global.u8 [%rd460], %rs1089; + add.s32 %r7382, %r7382, 1; + mov.u32 %r7388, 8; + mov.u16 %rs1089, 0; + mov.u32 %r7512, %r7529; + +$L__BB3_814: + setp.eq.s32 %p921, %r2073, 1; + mov.u32 %r7529, %r7512; + mov.u32 %r7509, %r2072; + @%p921 bra $L__BB3_822; + + add.s32 %r7509, %r7499, -2; + mov.u32 %r5211, 1; + shl.b32 %r5212, %r5211, %r7509; + and.b32 %r5213, %r5212, %r7543; + setp.ne.s32 %p922, %r5213, 0; + selp.u32 %r5214, 1, 0, %p922; + cvt.u32.u16 %r5215, %rs1089; + bfi.b32 %r5216, %r5215, %r5214, 1, 8; + cvt.u16.u32 %rs1089, %r5216; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p923, %r7388, 0; + mov.u32 %r7503, %r7512; + @%p923 bra $L__BB3_818; + + setp.gt.u32 %p924, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7503, %r5211; + @%p924 bra $L__BB3_818; + + add.s32 %r5219, %r7382, 17477; + cvt.u64.u32 %rd461, %r5219; + add.s64 %rd462, %rd461, %rd5; + add.s64 %rd463, %rd1, %rd462; + and.b16 %rs765, %rs1089, 255; + st.global.u8 [%rd463], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p925, %rs765, 255; + selp.b32 %r7388, 7, 8, %p925; + mov.u16 %rs1089, 0; + mov.u32 %r7503, %r7512; + +$L__BB3_818: + setp.eq.s32 %p926, %r2073, 2; + mov.u32 %r7529, %r7503; + mov.u32 %r7512, %r7503; + @%p926 bra $L__BB3_822; + + add.s32 %r7509, %r7499, -3; + mov.u32 %r5220, 1; + shl.b32 %r5221, %r5220, %r7509; + and.b32 %r5222, %r5221, %r7543; + setp.ne.s32 %p927, %r5222, 0; + selp.u32 %r5223, 1, 0, %p927; + cvt.u32.u16 %r5224, %rs1089; + bfi.b32 %r5225, %r5224, %r5223, 1, 8; + cvt.u16.u32 %rs1089, %r5225; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p928, %r7388, 0; + mov.u32 %r7529, %r7503; + mov.u32 %r7512, %r7503; + @%p928 bra $L__BB3_822; + + setp.gt.u32 %p929, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7529, %r5220; + mov.u32 %r7512, %r5220; + @%p929 bra $L__BB3_822; + + add.s32 %r5230, %r7382, 17477; + cvt.u64.u32 %rd464, %r5230; + add.s64 %rd465, %rd464, %rd5; + add.s64 %rd466, %rd1, %rd465; + and.b16 %rs768, %rs1089, 255; + st.global.u8 [%rd466], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p930, %rs768, 255; + selp.b32 %r7388, 7, 8, %p930; + mov.u16 %rs1089, 0; + mov.u32 %r7529, %r7503; + mov.u32 %r7512, %r7503; + +$L__BB3_822: + setp.lt.u32 %p931, %r2072, 3; + @%p931 bra $L__BB3_837; + + mov.u32 %r7529, %r7512; + +$L__BB3_824: + add.s32 %r5231, %r7509, -1; + mov.u32 %r5232, 1; + shl.b32 %r5233, %r5232, %r5231; + and.b32 %r5234, %r5233, %r7543; + setp.ne.s32 %p932, %r5234, 0; + selp.u32 %r5235, 1, 0, %p932; + cvt.u32.u16 %r5236, %rs1089; + bfi.b32 %r7518, %r5236, %r5235, 1, 8; + add.s32 %r7519, %r7388, -1; + setp.ne.s32 %p933, %r7519, 0; + mov.u32 %r7517, %r7529; + @%p933 bra $L__BB3_827; + + setp.gt.u32 %p934, %r7382, 191; + mov.u32 %r7519, 0; + mov.u32 %r7517, %r5232; + @%p934 bra $L__BB3_827; + + cvt.u16.u32 %rs769, %r7518; + and.b16 %rs770, %rs769, 255; + add.s32 %r5240, %r7382, 17477; + cvt.u64.u32 %rd467, %r5240; + add.s64 %rd468, %rd467, %rd5; + add.s64 %rd469, %rd1, %rd468; + st.global.u8 [%rd469], %rs769; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p935, %rs770, 255; + selp.b32 %r7519, 7, 8, %p935; + mov.u32 %r7518, 0; + mov.u32 %r7517, %r7529; + +$L__BB3_827: + add.s32 %r5241, %r7509, -2; + shl.b32 %r5243, %r5232, %r5241; + and.b32 %r5244, %r5243, %r7543; + setp.ne.s32 %p936, %r5244, 0; + and.b32 %r5245, %r7518, 127; + selp.u32 %r5246, 1, 0, %p936; + bfi.b32 %r7522, %r5245, %r5246, 1, 7; + add.s32 %r7523, %r7519, -1; + setp.ne.s32 %p937, %r7523, 0; + mov.u32 %r7521, %r7517; + @%p937 bra $L__BB3_830; + + setp.gt.u32 %p938, %r7382, 191; + mov.u32 %r7523, 0; + mov.u32 %r7521, 1; + @%p938 bra $L__BB3_830; + + cvt.u16.u32 %rs771, %r7522; + and.b16 %rs772, %rs771, 255; + add.s32 %r5250, %r7382, 17477; + cvt.u64.u32 %rd470, %r5250; + add.s64 %rd471, %rd470, %rd5; + add.s64 %rd472, %rd1, %rd471; + st.global.u8 [%rd472], %rs771; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p939, %rs772, 255; + selp.b32 %r7523, 7, 8, %p939; + mov.u32 %r7522, 0; + mov.u32 %r7521, %r7517; + +$L__BB3_830: + add.s32 %r5251, %r7509, -3; + mov.u32 %r5252, 1; + shl.b32 %r5253, %r5252, %r5251; + and.b32 %r5254, %r5253, %r7543; + setp.ne.s32 %p940, %r5254, 0; + and.b32 %r5255, %r7522, 127; + selp.u32 %r5256, 1, 0, %p940; + bfi.b32 %r7526, %r5255, %r5256, 1, 7; + add.s32 %r7527, %r7523, -1; + setp.ne.s32 %p941, %r7527, 0; + mov.u32 %r7525, %r7521; + @%p941 bra $L__BB3_833; + + setp.gt.u32 %p942, %r7382, 191; + mov.u32 %r7527, 0; + mov.u32 %r7525, %r5252; + @%p942 bra $L__BB3_833; + + cvt.u16.u32 %rs773, %r7526; + and.b16 %rs774, %rs773, 255; + add.s32 %r5260, %r7382, 17477; + cvt.u64.u32 %rd473, %r5260; + add.s64 %rd474, %rd473, %rd5; + add.s64 %rd475, %rd1, %rd474; + st.global.u8 [%rd475], %rs773; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p943, %rs774, 255; + selp.b32 %r7527, 7, 8, %p943; + mov.u32 %r7526, 0; + mov.u32 %r7525, %r7521; + +$L__BB3_833: + add.s32 %r7509, %r7509, -4; + shl.b32 %r5262, %r5252, %r7509; + and.b32 %r5263, %r5262, %r7543; + setp.ne.s32 %p944, %r5263, 0; + and.b32 %r5264, %r7526, 127; + selp.u32 %r5265, 1, 0, %p944; + bfi.b32 %r5266, %r5264, %r5265, 1, 15; + cvt.u16.u32 %rs1089, %r5266; + add.s32 %r7388, %r7527, -1; + setp.ne.s32 %p945, %r7388, 0; + mov.u32 %r7529, %r7525; + @%p945 bra $L__BB3_836; + + setp.gt.u32 %p946, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7529, 1; + @%p946 bra $L__BB3_836; + + add.s32 %r5269, %r7382, 17477; + cvt.u64.u32 %rd476, %r5269; + add.s64 %rd477, %rd476, %rd5; + add.s64 %rd478, %rd1, %rd477; + and.b16 %rs776, %rs1089, 255; + st.global.u8 [%rd478], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p947, %rs776, 255; + selp.b32 %r7388, 7, 8, %p947; + mov.u16 %rs1089, 0; + mov.u32 %r7529, %r7525; + +$L__BB3_836: + setp.ne.s32 %p948, %r7509, 0; + @%p948 bra $L__BB3_824; + +$L__BB3_837: + add.s32 %r5271, %r7542, -1; + setp.eq.s32 %p949, %r7542, 0; + mov.u32 %r7543, 0; + selp.b32 %r7542, 0, %r5271, %p949; + setp.lt.u32 %p950, %r7542, 3; + mov.u32 %r7535, %r7543; + @%p950 bra $L__BB3_840; + + setp.lt.u32 %p951, %r7542, 6; + mov.u32 %r7535, 1; + @%p951 bra $L__BB3_840; + + setp.lt.u32 %p952, %r7542, 9; + setp.eq.s32 %p953, %r7542, 11; + selp.b32 %r5273, 4, 5, %p953; + setp.lt.u32 %p954, %r7542, 11; + selp.b32 %r5274, 3, %r5273, %p954; + selp.b32 %r7535, 2, %r5274, %p952; + +$L__BB3_840: + mov.u32 %r5276, 1; + shl.b32 %r7541, %r5276, %r7535; + mov.u32 %r7540, %r7529; + bra.uni $L__BB3_849; + +$L__BB3_841: + add.s32 %r7543, %r7543, 1; + setp.lt.u32 %p955, %r7543, %r7541; + @%p955 bra $L__BB3_849; + + shl.b16 %rs777, %rs1089, 1; + or.b16 %rs1089, %rs777, 1; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p956, %r7388, 0; + mov.u32 %r7536, %r7540; + @%p956 bra $L__BB3_845; + + setp.gt.u32 %p957, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7536, 1; + @%p957 bra $L__BB3_845; + + and.b16 %rs779, %rs1089, 255; + st.global.u8 [%rd27], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p958, %rs779, 255; + selp.b32 %r7388, 7, 8, %p958; + mov.u16 %rs1089, 0; + mov.u32 %r7536, %r7540; + +$L__BB3_845: + add.s32 %r5280, %r7542, 1; + min.u32 %r7542, %r5280, 12; + setp.lt.u32 %p959, %r7542, 3; + mov.u32 %r7543, 0; + mov.u32 %r7539, %r7543; + @%p959 bra $L__BB3_848; + + setp.lt.u32 %p960, %r7542, 6; + mov.u32 %r7539, 1; + @%p960 bra $L__BB3_848; + + setp.lt.u32 %p961, %r7542, 9; + setp.eq.s32 %p962, %r7542, 11; + selp.b32 %r5282, 4, 5, %p962; + setp.lt.u32 %p963, %r7542, 11; + selp.b32 %r5283, 3, %r5282, %p963; + selp.b32 %r7539, 2, %r5283, %p961; + +$L__BB3_848: + mov.u32 %r5285, 1; + shl.b32 %r7541, %r5285, %r7539; + mov.u32 %r7540, %r7536; + +$L__BB3_849: + max.s32 %r2156, %r7476, 1; + and.b16 %rs780, %rs261, 15; + cvt.u32.u16 %r2157, %rs780; + and.b32 %r2158, %r7472, 1; + setp.eq.s32 %p964, %r2158, 0; + mov.u32 %r7556, %r7459; + @%p964 bra $L__BB3_856; + + and.b32 %r5286, %r2157, 1; + sub.s32 %r7546, %r2156, %r5286; + setp.eq.s32 %p965, %r7546, 0; + mov.u32 %r7556, %r7459; + @%p965 bra $L__BB3_856; + + mov.u32 %r5287, -1; + shl.b32 %r5288, %r5287, %r7546; + not.b32 %r5289, %r5288; + and.b32 %r7547, %r7466, %r5289; + +$L__BB3_852: + setp.gt.u32 %p966, %r7925, 17476; + mov.u32 %r7556, 1; + @%p966 bra $L__BB3_856; + + sub.s32 %r5291, %r7924, %r7923; + min.u32 %r5292, %r5291, %r7546; + setp.eq.s32 %p967, %r5292, 32; + mov.u32 %r5293, -1; + shl.b32 %r5294, %r5293, %r5292; + not.b32 %r5295, %r5294; + selp.b32 %r5296, -1, %r5295, %p967; + and.b32 %r5297, %r5296, %r7547; + shl.b32 %r5298, %r5297, %r7923; + or.b32 %r7922, %r5298, %r7922; + add.s32 %r7923, %r5292, %r7923; + shr.u32 %r7547, %r7547, %r5292; + sub.s32 %r7546, %r7546, %r5292; + setp.lt.u32 %p968, %r7923, %r7924; + @%p968 bra $L__BB3_855; + + cvt.u64.u32 %rd479, %r7925; + add.s64 %rd480, %rd479, %rd5; + add.s64 %rd481, %rd1, %rd480; + st.global.u8 [%rd481], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p969, %r7922, 255; + selp.b32 %r7924, 7, 8, %p969; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_855: + setp.ne.s32 %p970, %r7546, 0; + mov.u32 %r7556, %r7459; + @%p970 bra $L__BB3_852; + +$L__BB3_856: + setp.eq.s32 %p971, %r2044, 0; + mov.u32 %r7571, %r7556; + @%p971 bra $L__BB3_863; + + shr.u32 %r5301, %r2157, 1; + and.b32 %r5302, %r5301, 1; + sub.s32 %r7561, %r2156, %r5302; + setp.eq.s32 %p972, %r7561, 0; + mov.u32 %r7571, %r7556; + @%p972 bra $L__BB3_863; + + mov.u32 %r5303, -1; + shl.b32 %r5304, %r5303, %r7561; + not.b32 %r5305, %r5304; + and.b32 %r7562, %r7470, %r5305; + +$L__BB3_859: + setp.gt.u32 %p973, %r7925, 17476; + mov.u32 %r7571, 1; + @%p973 bra $L__BB3_863; + + sub.s32 %r5307, %r7924, %r7923; + min.u32 %r5308, %r5307, %r7561; + setp.eq.s32 %p974, %r5308, 32; + mov.u32 %r5309, -1; + shl.b32 %r5310, %r5309, %r5308; + not.b32 %r5311, %r5310; + selp.b32 %r5312, -1, %r5311, %p974; + and.b32 %r5313, %r5312, %r7562; + shl.b32 %r5314, %r5313, %r7923; + or.b32 %r7922, %r5314, %r7922; + add.s32 %r7923, %r5308, %r7923; + shr.u32 %r7562, %r7562, %r5308; + sub.s32 %r7561, %r7561, %r5308; + setp.lt.u32 %p975, %r7923, %r7924; + @%p975 bra $L__BB3_862; + + cvt.u64.u32 %rd482, %r7925; + add.s64 %rd483, %rd482, %rd5; + add.s64 %rd484, %rd1, %rd483; + st.global.u8 [%rd484], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p976, %r7922, 255; + selp.b32 %r7924, 7, 8, %p976; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_862: + setp.ne.s32 %p977, %r7561, 0; + mov.u32 %r7571, %r7556; + @%p977 bra $L__BB3_859; + +$L__BB3_863: + and.b32 %r5317, %r7472, 4; + setp.eq.s32 %p978, %r5317, 0; + mov.u32 %r7586, %r7571; + @%p978 bra $L__BB3_870; + + shr.u32 %r5318, %r2157, 2; + and.b32 %r5319, %r5318, 1; + sub.s32 %r7576, %r2156, %r5319; + setp.eq.s32 %p979, %r7576, 0; + mov.u32 %r7586, %r7571; + @%p979 bra $L__BB3_870; + + mov.u32 %r5320, -1; + shl.b32 %r5321, %r5320, %r7576; + not.b32 %r5322, %r5321; + and.b32 %r7577, %r7475, %r5322; + +$L__BB3_866: + setp.gt.u32 %p980, %r7925, 17476; + mov.u32 %r7586, 1; + @%p980 bra $L__BB3_870; + + sub.s32 %r5324, %r7924, %r7923; + min.u32 %r5325, %r5324, %r7576; + setp.eq.s32 %p981, %r5325, 32; + mov.u32 %r5326, -1; + shl.b32 %r5327, %r5326, %r5325; + not.b32 %r5328, %r5327; + selp.b32 %r5329, -1, %r5328, %p981; + and.b32 %r5330, %r5329, %r7577; + shl.b32 %r5331, %r5330, %r7923; + or.b32 %r7922, %r5331, %r7922; + add.s32 %r7923, %r5325, %r7923; + shr.u32 %r7577, %r7577, %r5325; + sub.s32 %r7576, %r7576, %r5325; + setp.lt.u32 %p982, %r7923, %r7924; + @%p982 bra $L__BB3_869; + + cvt.u64.u32 %rd485, %r7925; + add.s64 %rd486, %rd485, %rd5; + add.s64 %rd487, %rd1, %rd486; + st.global.u8 [%rd487], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p983, %r7922, 255; + selp.b32 %r7924, 7, 8, %p983; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_869: + setp.ne.s32 %p984, %r7576, 0; + mov.u32 %r7586, %r7571; + @%p984 bra $L__BB3_866; + +$L__BB3_870: + setp.eq.s32 %p985, %r2045, 0; + mov.u32 %r8093, %r7586; + @%p985 bra $L__BB3_877; + + shr.u32 %r5334, %r2157, 3; + sub.s32 %r7591, %r2156, %r5334; + setp.eq.s32 %p986, %r7591, 0; + mov.u32 %r8093, %r7586; + @%p986 bra $L__BB3_877; + + mov.u32 %r5335, -1; + shl.b32 %r5336, %r5335, %r7591; + not.b32 %r5337, %r5336; + and.b32 %r7592, %r7480, %r5337; + +$L__BB3_873: + setp.gt.u32 %p987, %r7925, 17476; + mov.u32 %r8093, 1; + @%p987 bra $L__BB3_877; + + sub.s32 %r5339, %r7924, %r7923; + min.u32 %r5340, %r5339, %r7591; + setp.eq.s32 %p988, %r5340, 32; + mov.u32 %r5341, -1; + shl.b32 %r5342, %r5341, %r5340; + not.b32 %r5343, %r5342; + selp.b32 %r5344, -1, %r5343, %p988; + and.b32 %r5345, %r5344, %r7592; + shl.b32 %r5346, %r5345, %r7923; + or.b32 %r7922, %r5346, %r7922; + add.s32 %r7923, %r5340, %r7923; + shr.u32 %r7592, %r7592, %r5340; + sub.s32 %r7591, %r7591, %r5340; + setp.lt.u32 %p989, %r7923, %r7924; + @%p989 bra $L__BB3_876; + + cvt.u64.u32 %rd488, %r7925; + add.s64 %rd489, %rd488, %rd5; + add.s64 %rd490, %rd1, %rd489; + st.global.u8 [%rd490], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p990, %r7922, 255; + selp.b32 %r7924, 7, 8, %p990; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_876: + setp.ne.s32 %p991, %r7591, 0; + mov.u32 %r8093, %r7586; + @%p991 bra $L__BB3_873; + +$L__BB3_877: + setp.lt.s32 %p992, %r2041, 1; + setp.lt.s32 %p993, %r1782, 1; + or.pred %p994, %p993, %p992; + @%p994 bra $L__BB3_925; + + min.s32 %r5349, %r1782, %r2041; + setp.lt.s32 %p995, %r5349, 3; + add.s32 %r5350, %r7382, 17477; + cvt.u64.u32 %rd491, %r5350; + add.s64 %rd492, %rd491, %rd5; + add.s64 %rd28, %rd1, %rd492; + @%p995 bra $L__BB3_917; + bra.uni $L__BB3_879; + +$L__BB3_917: + add.s32 %r7543, %r7543, 1; + setp.lt.u32 %p1042, %r7543, %r7541; + @%p1042 bra $L__BB3_925; + + shl.b16 %rs797, %rs1089, 1; + or.b16 %rs1089, %rs797, 1; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1043, %r7388, 0; + mov.u32 %r7646, %r7540; + @%p1043 bra $L__BB3_921; + + setp.gt.u32 %p1044, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7646, 1; + @%p1044 bra $L__BB3_921; + + and.b16 %rs799, %rs1089, 255; + st.global.u8 [%rd28], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1045, %rs799, 255; + selp.b32 %r7388, 7, 8, %p1045; + mov.u16 %rs1089, 0; + mov.u32 %r7646, %r7540; + +$L__BB3_921: + add.s32 %r5438, %r7542, 1; + min.u32 %r7542, %r5438, 12; + setp.lt.u32 %p1046, %r7542, 3; + mov.u32 %r7543, 0; + mov.u32 %r7649, %r7543; + @%p1046 bra $L__BB3_924; + + setp.lt.u32 %p1047, %r7542, 6; + mov.u32 %r7649, 1; + @%p1047 bra $L__BB3_924; + + setp.lt.u32 %p1048, %r7542, 9; + setp.eq.s32 %p1049, %r7542, 11; + selp.b32 %r5440, 4, 5, %p1049; + setp.lt.u32 %p1050, %r7542, 11; + selp.b32 %r5441, 3, %r5440, %p1050; + selp.b32 %r7649, 2, %r5441, %p1048; + +$L__BB3_924: + mov.u32 %r5443, 1; + shl.b32 %r7541, %r5443, %r7649; + mov.u32 %r7540, %r7646; + bra.uni $L__BB3_925; + +$L__BB3_879: + shl.b16 %rs1089, %rs1089, 1; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p996, %r7388, 0; + mov.u32 %r7639, %r7540; + @%p996 bra $L__BB3_882; + + setp.gt.u32 %p997, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7639, 1; + @%p997 bra $L__BB3_882; + + st.global.u8 [%rd28], %rs1089; + add.s32 %r7382, %r7382, 1; + mov.u32 %r7388, 8; + mov.u16 %rs1089, 0; + mov.u32 %r7639, %r7540; + +$L__BB3_882: + setp.lt.u32 %p998, %r7542, 3; + mov.u32 %r7609, 0; + @%p998 bra $L__BB3_885; + + setp.lt.u32 %p999, %r7542, 6; + mov.u32 %r7609, 1; + @%p999 bra $L__BB3_885; + + setp.lt.u32 %p1000, %r7542, 9; + setp.eq.s32 %p1001, %r7542, 11; + selp.b32 %r5356, 4, 5, %p1001; + setp.lt.u32 %p1002, %r7542, 11; + selp.b32 %r5357, 3, %r5356, %p1002; + selp.b32 %r7609, 2, %r5357, %p1000; + +$L__BB3_885: + setp.eq.s32 %p1003, %r7609, 0; + @%p1003 bra $L__BB3_913; + + add.s32 %r2258, %r7609, -1; + and.b32 %r2259, %r7609, 3; + setp.eq.s32 %p1004, %r2259, 0; + mov.u32 %r7619, %r7609; + mov.u32 %r7622, %r7639; + @%p1004 bra $L__BB3_898; + + mov.u32 %r5359, 1; + shl.b32 %r5360, %r5359, %r2258; + and.b32 %r5361, %r5360, %r7543; + setp.ne.s32 %p1005, %r5361, 0; + selp.u32 %r5362, 1, 0, %p1005; + cvt.u32.u16 %r5363, %rs1089; + bfi.b32 %r5364, %r5363, %r5362, 1, 8; + cvt.u16.u32 %rs1089, %r5364; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1006, %r7388, 0; + mov.u32 %r7622, %r7639; + @%p1006 bra $L__BB3_890; + + setp.gt.u32 %p1007, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7622, %r5359; + @%p1007 bra $L__BB3_890; + + add.s32 %r5368, %r7382, 17477; + cvt.u64.u32 %rd493, %r5368; + add.s64 %rd494, %rd493, %rd5; + add.s64 %rd495, %rd1, %rd494; + st.global.u8 [%rd495], %rs1089; + add.s32 %r7382, %r7382, 1; + mov.u32 %r7388, 8; + mov.u16 %rs1089, 0; + mov.u32 %r7622, %r7639; + +$L__BB3_890: + setp.eq.s32 %p1008, %r2259, 1; + mov.u32 %r7639, %r7622; + mov.u32 %r7619, %r2258; + @%p1008 bra $L__BB3_898; + + add.s32 %r7619, %r7609, -2; + mov.u32 %r5369, 1; + shl.b32 %r5370, %r5369, %r7619; + and.b32 %r5371, %r5370, %r7543; + setp.ne.s32 %p1009, %r5371, 0; + selp.u32 %r5372, 1, 0, %p1009; + cvt.u32.u16 %r5373, %rs1089; + bfi.b32 %r5374, %r5373, %r5372, 1, 8; + cvt.u16.u32 %rs1089, %r5374; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1010, %r7388, 0; + mov.u32 %r7613, %r7622; + @%p1010 bra $L__BB3_894; + + setp.gt.u32 %p1011, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7613, %r5369; + @%p1011 bra $L__BB3_894; + + add.s32 %r5377, %r7382, 17477; + cvt.u64.u32 %rd496, %r5377; + add.s64 %rd497, %rd496, %rd5; + add.s64 %rd498, %rd1, %rd497; + and.b16 %rs785, %rs1089, 255; + st.global.u8 [%rd498], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1012, %rs785, 255; + selp.b32 %r7388, 7, 8, %p1012; + mov.u16 %rs1089, 0; + mov.u32 %r7613, %r7622; + +$L__BB3_894: + setp.eq.s32 %p1013, %r2259, 2; + mov.u32 %r7639, %r7613; + mov.u32 %r7622, %r7613; + @%p1013 bra $L__BB3_898; + + add.s32 %r7619, %r7609, -3; + mov.u32 %r5378, 1; + shl.b32 %r5379, %r5378, %r7619; + and.b32 %r5380, %r5379, %r7543; + setp.ne.s32 %p1014, %r5380, 0; + selp.u32 %r5381, 1, 0, %p1014; + cvt.u32.u16 %r5382, %rs1089; + bfi.b32 %r5383, %r5382, %r5381, 1, 8; + cvt.u16.u32 %rs1089, %r5383; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1015, %r7388, 0; + mov.u32 %r7639, %r7613; + mov.u32 %r7622, %r7613; + @%p1015 bra $L__BB3_898; + + setp.gt.u32 %p1016, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7639, %r5378; + mov.u32 %r7622, %r5378; + @%p1016 bra $L__BB3_898; + + add.s32 %r5388, %r7382, 17477; + cvt.u64.u32 %rd499, %r5388; + add.s64 %rd500, %rd499, %rd5; + add.s64 %rd501, %rd1, %rd500; + and.b16 %rs788, %rs1089, 255; + st.global.u8 [%rd501], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1017, %rs788, 255; + selp.b32 %r7388, 7, 8, %p1017; + mov.u16 %rs1089, 0; + mov.u32 %r7639, %r7613; + mov.u32 %r7622, %r7613; + +$L__BB3_898: + setp.lt.u32 %p1018, %r2258, 3; + @%p1018 bra $L__BB3_913; + + mov.u32 %r7639, %r7622; + +$L__BB3_900: + add.s32 %r5389, %r7619, -1; + mov.u32 %r5390, 1; + shl.b32 %r5391, %r5390, %r5389; + and.b32 %r5392, %r5391, %r7543; + setp.ne.s32 %p1019, %r5392, 0; + selp.u32 %r5393, 1, 0, %p1019; + cvt.u32.u16 %r5394, %rs1089; + bfi.b32 %r7628, %r5394, %r5393, 1, 8; + add.s32 %r7629, %r7388, -1; + setp.ne.s32 %p1020, %r7629, 0; + mov.u32 %r7627, %r7639; + @%p1020 bra $L__BB3_903; + + setp.gt.u32 %p1021, %r7382, 191; + mov.u32 %r7629, 0; + mov.u32 %r7627, %r5390; + @%p1021 bra $L__BB3_903; + + cvt.u16.u32 %rs789, %r7628; + and.b16 %rs790, %rs789, 255; + add.s32 %r5398, %r7382, 17477; + cvt.u64.u32 %rd502, %r5398; + add.s64 %rd503, %rd502, %rd5; + add.s64 %rd504, %rd1, %rd503; + st.global.u8 [%rd504], %rs789; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1022, %rs790, 255; + selp.b32 %r7629, 7, 8, %p1022; + mov.u32 %r7628, 0; + mov.u32 %r7627, %r7639; + +$L__BB3_903: + add.s32 %r5399, %r7619, -2; + shl.b32 %r5401, %r5390, %r5399; + and.b32 %r5402, %r5401, %r7543; + setp.ne.s32 %p1023, %r5402, 0; + and.b32 %r5403, %r7628, 127; + selp.u32 %r5404, 1, 0, %p1023; + bfi.b32 %r7632, %r5403, %r5404, 1, 7; + add.s32 %r7633, %r7629, -1; + setp.ne.s32 %p1024, %r7633, 0; + mov.u32 %r7631, %r7627; + @%p1024 bra $L__BB3_906; + + setp.gt.u32 %p1025, %r7382, 191; + mov.u32 %r7633, 0; + mov.u32 %r7631, 1; + @%p1025 bra $L__BB3_906; + + cvt.u16.u32 %rs791, %r7632; + and.b16 %rs792, %rs791, 255; + add.s32 %r5408, %r7382, 17477; + cvt.u64.u32 %rd505, %r5408; + add.s64 %rd506, %rd505, %rd5; + add.s64 %rd507, %rd1, %rd506; + st.global.u8 [%rd507], %rs791; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1026, %rs792, 255; + selp.b32 %r7633, 7, 8, %p1026; + mov.u32 %r7632, 0; + mov.u32 %r7631, %r7627; + +$L__BB3_906: + add.s32 %r5409, %r7619, -3; + mov.u32 %r5410, 1; + shl.b32 %r5411, %r5410, %r5409; + and.b32 %r5412, %r5411, %r7543; + setp.ne.s32 %p1027, %r5412, 0; + and.b32 %r5413, %r7632, 127; + selp.u32 %r5414, 1, 0, %p1027; + bfi.b32 %r7636, %r5413, %r5414, 1, 7; + add.s32 %r7637, %r7633, -1; + setp.ne.s32 %p1028, %r7637, 0; + mov.u32 %r7635, %r7631; + @%p1028 bra $L__BB3_909; + + setp.gt.u32 %p1029, %r7382, 191; + mov.u32 %r7637, 0; + mov.u32 %r7635, %r5410; + @%p1029 bra $L__BB3_909; + + cvt.u16.u32 %rs793, %r7636; + and.b16 %rs794, %rs793, 255; + add.s32 %r5418, %r7382, 17477; + cvt.u64.u32 %rd508, %r5418; + add.s64 %rd509, %rd508, %rd5; + add.s64 %rd510, %rd1, %rd509; + st.global.u8 [%rd510], %rs793; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1030, %rs794, 255; + selp.b32 %r7637, 7, 8, %p1030; + mov.u32 %r7636, 0; + mov.u32 %r7635, %r7631; + +$L__BB3_909: + add.s32 %r7619, %r7619, -4; + shl.b32 %r5420, %r5410, %r7619; + and.b32 %r5421, %r5420, %r7543; + setp.ne.s32 %p1031, %r5421, 0; + and.b32 %r5422, %r7636, 127; + selp.u32 %r5423, 1, 0, %p1031; + bfi.b32 %r5424, %r5422, %r5423, 1, 15; + cvt.u16.u32 %rs1089, %r5424; + add.s32 %r7388, %r7637, -1; + setp.ne.s32 %p1032, %r7388, 0; + mov.u32 %r7639, %r7635; + @%p1032 bra $L__BB3_912; + + setp.gt.u32 %p1033, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7639, 1; + @%p1033 bra $L__BB3_912; + + add.s32 %r5427, %r7382, 17477; + cvt.u64.u32 %rd511, %r5427; + add.s64 %rd512, %rd511, %rd5; + add.s64 %rd513, %rd1, %rd512; + and.b16 %rs796, %rs1089, 255; + st.global.u8 [%rd513], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1034, %rs796, 255; + selp.b32 %r7388, 7, 8, %p1034; + mov.u16 %rs1089, 0; + mov.u32 %r7639, %r7635; + +$L__BB3_912: + setp.ne.s32 %p1035, %r7619, 0; + @%p1035 bra $L__BB3_900; + +$L__BB3_913: + add.s32 %r5429, %r7542, -1; + setp.eq.s32 %p1036, %r7542, 0; + mov.u32 %r7543, 0; + selp.b32 %r7542, 0, %r5429, %p1036; + setp.lt.u32 %p1037, %r7542, 3; + mov.u32 %r7645, %r7543; + @%p1037 bra $L__BB3_916; + + setp.lt.u32 %p1038, %r7542, 6; + mov.u32 %r7645, 1; + @%p1038 bra $L__BB3_916; + + setp.lt.u32 %p1039, %r7542, 9; + setp.eq.s32 %p1040, %r7542, 11; + selp.b32 %r5431, 4, 5, %p1040; + setp.lt.u32 %p1041, %r7542, 11; + selp.b32 %r5432, 3, %r5431, %p1041; + selp.b32 %r7645, 2, %r5432, %p1039; + +$L__BB3_916: + mov.u32 %r5434, 1; + shl.b32 %r7541, %r5434, %r7645; + mov.u32 %r7540, %r7639; + +$L__BB3_925: + setp.gt.s32 %p1051, %r2041, 2; + setp.gt.s32 %p1052, %r1782, 2; + and.pred %p1053, %p1052, %p1051; + @%p1053 bra $L__BB3_974; + bra.uni $L__BB3_926; + +$L__BB3_974: + mul.lo.s32 %r5564, %r1782, 6; + add.s32 %r5565, %r5564, -11; + cvt.u64.u32 %rd547, %r5565; + cvta.to.global.u64 %rd548, %rd48; + add.s64 %rd31, %rd548, %rd547; + ld.global.u8 %rs335, [%rd31]; + add.s32 %r5566, %r5564, -10; + cvt.u64.u32 %rd549, %r5566; + add.s64 %rd550, %rd548, %rd549; + ld.global.u8 %rs336, [%rd550]; + ld.global.u8 %rs337, [%rd550+1]; + mul.lo.s32 %r5567, %r2041, 6; + add.s32 %r5568, %r5567, -12; + cvt.u64.u32 %rd551, %r5568; + add.s64 %rd552, %rd548, %rd551; + ld.global.u8 %rs338, [%rd552]; + ld.global.u8 %rs339, [%rd552+1]; + add.s32 %r5569, %r5567, -10; + cvt.u64.u32 %rd553, %r5569; + add.s64 %rd554, %rd548, %rd553; + ld.global.u8 %rs340, [%rd554]; + ld.global.u8 %rs341, [%rd554+1]; + setp.eq.s16 %p1121, %rs335, 0; + mov.u32 %r7743, %r7495; + @%p1121 bra $L__BB3_981; + + ld.global.u8 %r7733, [%rd31+-1]; + cvt.u32.u16 %r7732, %rs335; + +$L__BB3_976: + mov.u32 %r2469, %r7732; + setp.gt.u32 %p1122, %r7779, 2879; + mov.u32 %r7743, 1; + @%p1122 bra $L__BB3_981; + + mov.u32 %r5571, 8; + sub.s32 %r5572, %r5571, %r7777; + sub.s32 %r5573, %r5572, %r7778; + min.u32 %r5574, %r5573, %r2469; + setp.eq.s32 %p1123, %r5574, 32; + mov.u32 %r5575, -1; + shl.b32 %r5576, %r5575, %r5574; + not.b32 %r5577, %r5576; + selp.b32 %r5578, -1, %r5577, %p1123; + and.b32 %r5579, %r5578, %r7733; + shl.b32 %r5580, %r5579, %r7778; + cvt.u16.u32 %rs832, %r5580; + or.b16 %rs1147, %rs1147, %rs832; + add.s32 %r7778, %r5574, %r7778; + sub.s32 %r7732, %r2469, %r5574; + shr.u32 %r7733, %r7733, %r5574; + setp.gt.u32 %p1124, %r5573, %r2469; + @%p1124 bra $L__BB3_980; + + setp.ne.s32 %p1125, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs833, %rs1147, 255; + setp.ne.s16 %p1126, %rs833, 127; + and.pred %p1127, %p1125, %p1126; + @%p1127 bra $L__BB3_980; + + mov.u32 %r5583, 20548; + sub.s32 %r5584, %r5583, %r7779; + cvt.u64.u32 %rd555, %r5584; + add.s64 %rd556, %rd555, %rd5; + add.s64 %rd557, %rd1, %rd556; + st.global.u8 [%rd557], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1128, %rs833, 143; + selp.u32 %r7777, 1, 0, %p1128; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_980: + setp.ne.s32 %p1129, %r7732, 0; + mov.u32 %r7743, %r7495; + @%p1129 bra $L__BB3_976; + +$L__BB3_981: + setp.eq.s16 %p1130, %rs339, 0; + mov.u32 %r7755, %r7743; + @%p1130 bra $L__BB3_988; + + cvt.u32.u16 %r5585, %rs338; + and.b32 %r7745, %r5585, 255; + cvt.u32.u16 %r5586, %rs339; + and.b32 %r7744, %r5586, 255; + +$L__BB3_983: + mov.u32 %r2488, %r7744; + setp.gt.u32 %p1131, %r7779, 2879; + mov.u32 %r7755, 1; + @%p1131 bra $L__BB3_988; + + mov.u32 %r5588, 8; + sub.s32 %r5589, %r5588, %r7777; + sub.s32 %r5590, %r5589, %r7778; + min.u32 %r5591, %r5590, %r2488; + setp.eq.s32 %p1132, %r5591, 32; + mov.u32 %r5592, -1; + shl.b32 %r5593, %r5592, %r5591; + not.b32 %r5594, %r5593; + selp.b32 %r5595, -1, %r5594, %p1132; + and.b32 %r5596, %r5595, %r7745; + shl.b32 %r5597, %r5596, %r7778; + cvt.u16.u32 %rs837, %r5597; + or.b16 %rs1147, %rs1147, %rs837; + add.s32 %r7778, %r5591, %r7778; + sub.s32 %r7744, %r2488, %r5591; + shr.u32 %r7745, %r7745, %r5591; + setp.gt.u32 %p1133, %r5590, %r2488; + @%p1133 bra $L__BB3_987; + + setp.ne.s32 %p1134, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs838, %rs1147, 255; + setp.ne.s16 %p1135, %rs838, 127; + and.pred %p1136, %p1134, %p1135; + @%p1136 bra $L__BB3_987; + + mov.u32 %r5600, 20548; + sub.s32 %r5601, %r5600, %r7779; + cvt.u64.u32 %rd558, %r5601; + add.s64 %rd559, %rd558, %rd5; + add.s64 %rd560, %rd1, %rd559; + st.global.u8 [%rd560], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1137, %rs838, 143; + selp.u32 %r7777, 1, 0, %p1137; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_987: + setp.ne.s32 %p1138, %r7744, 0; + mov.u32 %r7755, %r7743; + @%p1138 bra $L__BB3_983; + +$L__BB3_988: + setp.eq.s16 %p1139, %rs337, 0; + mov.u32 %r7767, %r7755; + @%p1139 bra $L__BB3_995; + + cvt.u32.u16 %r5602, %rs337; + and.b32 %r7756, %r5602, 255; + cvt.u32.u16 %r5603, %rs336; + and.b32 %r7757, %r5603, 255; + +$L__BB3_990: + mov.u32 %r2507, %r7756; + setp.gt.u32 %p1140, %r7779, 2879; + mov.u32 %r7767, 1; + @%p1140 bra $L__BB3_995; + + mov.u32 %r5605, 8; + sub.s32 %r5606, %r5605, %r7777; + sub.s32 %r5607, %r5606, %r7778; + min.u32 %r5608, %r5607, %r2507; + setp.eq.s32 %p1141, %r5608, 32; + mov.u32 %r5609, -1; + shl.b32 %r5610, %r5609, %r5608; + not.b32 %r5611, %r5610; + selp.b32 %r5612, -1, %r5611, %p1141; + and.b32 %r5613, %r5612, %r7757; + shl.b32 %r5614, %r5613, %r7778; + cvt.u16.u32 %rs842, %r5614; + or.b16 %rs1147, %rs1147, %rs842; + add.s32 %r7778, %r5608, %r7778; + sub.s32 %r7756, %r2507, %r5608; + shr.u32 %r7757, %r7757, %r5608; + setp.gt.u32 %p1142, %r5607, %r2507; + @%p1142 bra $L__BB3_994; + + setp.ne.s32 %p1143, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs843, %rs1147, 255; + setp.ne.s16 %p1144, %rs843, 127; + and.pred %p1145, %p1143, %p1144; + @%p1145 bra $L__BB3_994; + + mov.u32 %r5617, 20548; + sub.s32 %r5618, %r5617, %r7779; + cvt.u64.u32 %rd561, %r5618; + add.s64 %rd562, %rd561, %rd5; + add.s64 %rd563, %rd1, %rd562; + st.global.u8 [%rd563], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1146, %rs843, 143; + selp.u32 %r7777, 1, 0, %p1146; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_994: + setp.ne.s32 %p1147, %r7756, 0; + mov.u32 %r7767, %r7755; + @%p1147 bra $L__BB3_990; + +$L__BB3_995: + setp.eq.s16 %p1148, %rs341, 0; + mov.u32 %r7776, %r7767; + @%p1148 bra $L__BB3_1002; + + cvt.u32.u16 %r5619, %rs340; + and.b32 %r7769, %r5619, 255; + cvt.u32.u16 %r5620, %rs341; + and.b32 %r7768, %r5620, 255; + +$L__BB3_997: + mov.u32 %r2526, %r7768; + setp.gt.u32 %p1149, %r7779, 2879; + mov.u32 %r7776, 1; + @%p1149 bra $L__BB3_1002; + + mov.u32 %r5622, 8; + sub.s32 %r5623, %r5622, %r7777; + sub.s32 %r5624, %r5623, %r7778; + min.u32 %r5625, %r5624, %r2526; + setp.eq.s32 %p1150, %r5625, 32; + mov.u32 %r5626, -1; + shl.b32 %r5627, %r5626, %r5625; + not.b32 %r5628, %r5627; + selp.b32 %r5629, -1, %r5628, %p1150; + and.b32 %r5630, %r5629, %r7769; + shl.b32 %r5631, %r5630, %r7778; + cvt.u16.u32 %rs847, %r5631; + or.b16 %rs1147, %rs1147, %rs847; + add.s32 %r7778, %r5625, %r7778; + sub.s32 %r7768, %r2526, %r5625; + shr.u32 %r7769, %r7769, %r5625; + setp.gt.u32 %p1151, %r5624, %r2526; + @%p1151 bra $L__BB3_1001; + + setp.ne.s32 %p1152, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs848, %rs1147, 255; + setp.ne.s16 %p1153, %rs848, 127; + and.pred %p1154, %p1152, %p1153; + @%p1154 bra $L__BB3_1001; + + mov.u32 %r5634, 20548; + sub.s32 %r5635, %r5634, %r7779; + cvt.u64.u32 %rd564, %r5635; + add.s64 %rd565, %rd564, %rd5; + add.s64 %rd566, %rd1, %rd565; + st.global.u8 [%rd566], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1155, %rs848, 143; + selp.u32 %r7777, 1, 0, %p1155; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_1001: + setp.ne.s32 %p1156, %r7768, 0; + mov.u32 %r7776, %r7767; + @%p1156 bra $L__BB3_997; + bra.uni $L__BB3_1002; + +$L__BB3_926: + setp.gt.s32 %p1054, %r2041, 0; + and.pred %p1056, %p1052, %p1054; + mul.lo.s32 %r2342, %r1782, 6; + @%p1056 bra $L__BB3_955; + bra.uni $L__BB3_927; + +$L__BB3_955: + cvt.u64.u32 %rd534, %r2342; + cvta.to.global.u64 %rd535, %rd48; + add.s64 %rd30, %rd535, %rd534; + ld.global.u8 %rs321, [%rd30+1]; + add.s32 %r5515, %r2342, 2; + cvt.u64.u32 %rd536, %r5515; + add.s64 %rd537, %rd535, %rd536; + ld.global.u8 %rs322, [%rd537]; + ld.global.u8 %rs323, [%rd537+1]; + setp.eq.s16 %p1095, %rs321, 0; + mov.u32 %r7711, %r7495; + @%p1095 bra $L__BB3_962; + + ld.global.u8 %r7701, [%rd30]; + cvt.u32.u16 %r7700, %rs321; + +$L__BB3_957: + mov.u32 %r2417, %r7700; + setp.gt.u32 %p1096, %r7779, 2879; + mov.u32 %r7711, 1; + @%p1096 bra $L__BB3_962; + + mov.u32 %r5517, 8; + sub.s32 %r5518, %r5517, %r7777; + sub.s32 %r5519, %r5518, %r7778; + min.u32 %r5520, %r5519, %r2417; + setp.eq.s32 %p1097, %r5520, 32; + mov.u32 %r5521, -1; + shl.b32 %r5522, %r5521, %r5520; + not.b32 %r5523, %r5522; + selp.b32 %r5524, -1, %r5523, %p1097; + and.b32 %r5525, %r5524, %r7701; + shl.b32 %r5526, %r5525, %r7778; + cvt.u16.u32 %rs819, %r5526; + or.b16 %rs1147, %rs1147, %rs819; + add.s32 %r7778, %r5520, %r7778; + sub.s32 %r7700, %r2417, %r5520; + shr.u32 %r7701, %r7701, %r5520; + setp.gt.u32 %p1098, %r5519, %r2417; + @%p1098 bra $L__BB3_961; + + setp.ne.s32 %p1099, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs820, %rs1147, 255; + setp.ne.s16 %p1100, %rs820, 127; + and.pred %p1101, %p1099, %p1100; + @%p1101 bra $L__BB3_961; + + mov.u32 %r5529, 20548; + sub.s32 %r5530, %r5529, %r7779; + cvt.u64.u32 %rd538, %r5530; + add.s64 %rd539, %rd538, %rd5; + add.s64 %rd540, %rd1, %rd539; + st.global.u8 [%rd540], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1102, %rs820, 143; + selp.u32 %r7777, 1, 0, %p1102; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_961: + setp.ne.s32 %p1103, %r7700, 0; + mov.u32 %r7711, %r7495; + @%p1103 bra $L__BB3_957; + +$L__BB3_962: + add.s32 %r7713, %r2041, -1; + cvt.u32.u16 %r5532, %rs323; + and.b32 %r7724, %r5532, 255; + cvt.u32.u16 %r5533, %rs322; + and.b32 %r7725, %r5533, 255; + mov.u32 %r5531, 1; + mov.u32 %r7712, %r5531; + +$L__BB3_963: + mov.u32 %r2437, %r7712; + setp.gt.u32 %p1104, %r7779, 2879; + mov.u32 %r7723, %r5531; + @%p1104 bra $L__BB3_968; + + mov.u32 %r5535, 8; + sub.s32 %r5536, %r5535, %r7777; + sub.s32 %r5537, %r5536, %r7778; + min.u32 %r5538, %r5537, %r2437; + setp.eq.s32 %p1105, %r5538, 32; + mov.u32 %r5539, -1; + shl.b32 %r5540, %r5539, %r5538; + not.b32 %r5541, %r5540; + selp.b32 %r5542, -1, %r5541, %p1105; + and.b32 %r5543, %r5542, %r7713; + shl.b32 %r5544, %r5543, %r7778; + cvt.u16.u32 %rs823, %r5544; + or.b16 %rs1147, %rs1147, %rs823; + add.s32 %r7778, %r5538, %r7778; + sub.s32 %r7712, %r2437, %r5538; + shr.u32 %r7713, %r7713, %r5538; + setp.gt.u32 %p1106, %r5537, %r2437; + @%p1106 bra $L__BB3_967; + + setp.ne.s32 %p1107, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs824, %rs1147, 255; + setp.ne.s16 %p1108, %rs824, 127; + and.pred %p1109, %p1107, %p1108; + @%p1109 bra $L__BB3_967; + + mov.u32 %r5547, 20548; + sub.s32 %r5548, %r5547, %r7779; + cvt.u64.u32 %rd541, %r5548; + add.s64 %rd542, %rd541, %rd5; + add.s64 %rd543, %rd1, %rd542; + st.global.u8 [%rd543], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1110, %rs824, 143; + selp.u32 %r7777, 1, 0, %p1110; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_967: + setp.ne.s32 %p1111, %r7712, 0; + mov.u32 %r7723, %r7711; + @%p1111 bra $L__BB3_963; + +$L__BB3_968: + setp.eq.s16 %p1112, %rs323, 0; + mov.u32 %r7776, %r7723; + @%p1112 bra $L__BB3_1002; + +$L__BB3_969: + mov.u32 %r2454, %r7724; + setp.gt.u32 %p1113, %r7779, 2879; + mov.u32 %r7776, 1; + @%p1113 bra $L__BB3_1002; + + mov.u32 %r5550, 8; + sub.s32 %r5551, %r5550, %r7777; + sub.s32 %r5552, %r5551, %r7778; + min.u32 %r5553, %r5552, %r2454; + setp.eq.s32 %p1114, %r5553, 32; + mov.u32 %r5554, -1; + shl.b32 %r5555, %r5554, %r5553; + not.b32 %r5556, %r5555; + selp.b32 %r5557, -1, %r5556, %p1114; + and.b32 %r5558, %r5557, %r7725; + shl.b32 %r5559, %r5558, %r7778; + cvt.u16.u32 %rs828, %r5559; + or.b16 %rs1147, %rs1147, %rs828; + add.s32 %r7778, %r5553, %r7778; + sub.s32 %r7724, %r2454, %r5553; + shr.u32 %r7725, %r7725, %r5553; + setp.gt.u32 %p1115, %r5552, %r2454; + @%p1115 bra $L__BB3_973; + + setp.ne.s32 %p1116, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs829, %rs1147, 255; + setp.ne.s16 %p1117, %rs829, 127; + and.pred %p1118, %p1116, %p1117; + @%p1118 bra $L__BB3_973; + + mov.u32 %r5562, 20548; + sub.s32 %r5563, %r5562, %r7779; + cvt.u64.u32 %rd544, %r5563; + add.s64 %rd545, %rd544, %rd5; + add.s64 %rd546, %rd1, %rd545; + st.global.u8 [%rd546], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1119, %rs829, 143; + selp.u32 %r7777, 1, 0, %p1119; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_973: + setp.eq.s32 %p1120, %r7724, 0; + mov.u32 %r7776, %r7723; + @%p1120 bra $L__BB3_1002; + bra.uni $L__BB3_969; + +$L__BB3_927: + setp.gt.s32 %p1058, %r1782, 0; + selp.b32 %r5444, %r2342, 0, %p1058; + cvt.u64.u32 %rd514, %r5444; + cvta.to.global.u64 %rd515, %rd48; + add.s64 %rd29, %rd515, %rd514; + ld.global.u8 %rs299, [%rd29+1]; + add.s32 %r5445, %r5444, 2; + cvt.u64.u32 %rd516, %r5445; + add.s64 %rd517, %rd515, %rd516; + ld.global.u8 %rs300, [%rd517]; + ld.global.u8 %rs301, [%rd517+1]; + mul.lo.s32 %r5446, %r2041, 6; + selp.b32 %r5447, %r5446, 0, %p1054; + cvt.u64.u32 %rd518, %r5447; + add.s64 %rd519, %rd515, %rd518; + ld.global.u8 %rs302, [%rd519]; + ld.global.u8 %rs303, [%rd519+1]; + add.s32 %r5448, %r5447, 2; + cvt.u64.u32 %rd520, %r5448; + add.s64 %rd521, %rd515, %rd520; + ld.global.u8 %rs304, [%rd521]; + ld.global.u8 %rs305, [%rd521+1]; + setp.eq.s16 %p1059, %rs299, 0; + mov.u32 %r7667, %r7495; + @%p1059 bra $L__BB3_934; + + ld.global.u8 %r7657, [%rd29]; + cvt.u32.u16 %r7656, %rs299; + +$L__BB3_929: + mov.u32 %r2345, %r7656; + setp.gt.u32 %p1060, %r7779, 2879; + mov.u32 %r7667, 1; + @%p1060 bra $L__BB3_934; + + mov.u32 %r5450, 8; + sub.s32 %r5451, %r5450, %r7777; + sub.s32 %r5452, %r5451, %r7778; + min.u32 %r5453, %r5452, %r2345; + setp.eq.s32 %p1061, %r5453, 32; + mov.u32 %r5454, -1; + shl.b32 %r5455, %r5454, %r5453; + not.b32 %r5456, %r5455; + selp.b32 %r5457, -1, %r5456, %p1061; + and.b32 %r5458, %r5457, %r7657; + shl.b32 %r5459, %r5458, %r7778; + cvt.u16.u32 %rs800, %r5459; + or.b16 %rs1147, %rs1147, %rs800; + add.s32 %r7778, %r5453, %r7778; + sub.s32 %r7656, %r2345, %r5453; + shr.u32 %r7657, %r7657, %r5453; + setp.gt.u32 %p1062, %r5452, %r2345; + @%p1062 bra $L__BB3_933; + + setp.ne.s32 %p1063, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs801, %rs1147, 255; + setp.ne.s16 %p1064, %rs801, 127; + and.pred %p1065, %p1063, %p1064; + @%p1065 bra $L__BB3_933; + + mov.u32 %r5462, 20548; + sub.s32 %r5463, %r5462, %r7779; + cvt.u64.u32 %rd522, %r5463; + add.s64 %rd523, %rd522, %rd5; + add.s64 %rd524, %rd1, %rd523; + st.global.u8 [%rd524], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1066, %rs801, 143; + selp.u32 %r7777, 1, 0, %p1066; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_933: + setp.ne.s32 %p1067, %r7656, 0; + mov.u32 %r7667, %r7495; + @%p1067 bra $L__BB3_929; + +$L__BB3_934: + setp.eq.s16 %p1068, %rs303, 0; + mov.u32 %r7679, %r7667; + @%p1068 bra $L__BB3_941; + + cvt.u32.u16 %r5464, %rs302; + and.b32 %r7669, %r5464, 255; + cvt.u32.u16 %r5465, %rs303; + and.b32 %r7668, %r5465, 255; + +$L__BB3_936: + mov.u32 %r2364, %r7668; + setp.gt.u32 %p1069, %r7779, 2879; + mov.u32 %r7679, 1; + @%p1069 bra $L__BB3_941; + + mov.u32 %r5467, 8; + sub.s32 %r5468, %r5467, %r7777; + sub.s32 %r5469, %r5468, %r7778; + min.u32 %r5470, %r5469, %r2364; + setp.eq.s32 %p1070, %r5470, 32; + mov.u32 %r5471, -1; + shl.b32 %r5472, %r5471, %r5470; + not.b32 %r5473, %r5472; + selp.b32 %r5474, -1, %r5473, %p1070; + and.b32 %r5475, %r5474, %r7669; + shl.b32 %r5476, %r5475, %r7778; + cvt.u16.u32 %rs805, %r5476; + or.b16 %rs1147, %rs1147, %rs805; + add.s32 %r7778, %r5470, %r7778; + sub.s32 %r7668, %r2364, %r5470; + shr.u32 %r7669, %r7669, %r5470; + setp.gt.u32 %p1071, %r5469, %r2364; + @%p1071 bra $L__BB3_940; + + setp.ne.s32 %p1072, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs806, %rs1147, 255; + setp.ne.s16 %p1073, %rs806, 127; + and.pred %p1074, %p1072, %p1073; + @%p1074 bra $L__BB3_940; + + mov.u32 %r5479, 20548; + sub.s32 %r5480, %r5479, %r7779; + cvt.u64.u32 %rd525, %r5480; + add.s64 %rd526, %rd525, %rd5; + add.s64 %rd527, %rd1, %rd526; + st.global.u8 [%rd527], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1075, %rs806, 143; + selp.u32 %r7777, 1, 0, %p1075; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_940: + setp.ne.s32 %p1076, %r7668, 0; + mov.u32 %r7679, %r7667; + @%p1076 bra $L__BB3_936; + +$L__BB3_941: + setp.eq.s16 %p1077, %rs301, 0; + mov.u32 %r7691, %r7679; + @%p1077 bra $L__BB3_948; + + cvt.u32.u16 %r5481, %rs301; + and.b32 %r7680, %r5481, 255; + cvt.u32.u16 %r5482, %rs300; + and.b32 %r7681, %r5482, 255; + +$L__BB3_943: + mov.u32 %r2383, %r7680; + setp.gt.u32 %p1078, %r7779, 2879; + mov.u32 %r7691, 1; + @%p1078 bra $L__BB3_948; + + mov.u32 %r5484, 8; + sub.s32 %r5485, %r5484, %r7777; + sub.s32 %r5486, %r5485, %r7778; + min.u32 %r5487, %r5486, %r2383; + setp.eq.s32 %p1079, %r5487, 32; + mov.u32 %r5488, -1; + shl.b32 %r5489, %r5488, %r5487; + not.b32 %r5490, %r5489; + selp.b32 %r5491, -1, %r5490, %p1079; + and.b32 %r5492, %r5491, %r7681; + shl.b32 %r5493, %r5492, %r7778; + cvt.u16.u32 %rs810, %r5493; + or.b16 %rs1147, %rs1147, %rs810; + add.s32 %r7778, %r5487, %r7778; + sub.s32 %r7680, %r2383, %r5487; + shr.u32 %r7681, %r7681, %r5487; + setp.gt.u32 %p1080, %r5486, %r2383; + @%p1080 bra $L__BB3_947; + + setp.ne.s32 %p1081, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs811, %rs1147, 255; + setp.ne.s16 %p1082, %rs811, 127; + and.pred %p1083, %p1081, %p1082; + @%p1083 bra $L__BB3_947; + + mov.u32 %r5496, 20548; + sub.s32 %r5497, %r5496, %r7779; + cvt.u64.u32 %rd528, %r5497; + add.s64 %rd529, %rd528, %rd5; + add.s64 %rd530, %rd1, %rd529; + st.global.u8 [%rd530], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1084, %rs811, 143; + selp.u32 %r7777, 1, 0, %p1084; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_947: + setp.ne.s32 %p1085, %r7680, 0; + mov.u32 %r7691, %r7679; + @%p1085 bra $L__BB3_943; + +$L__BB3_948: + setp.eq.s16 %p1086, %rs305, 0; + mov.u32 %r7776, %r7691; + @%p1086 bra $L__BB3_1002; + + cvt.u32.u16 %r5498, %rs304; + and.b32 %r7693, %r5498, 255; + cvt.u32.u16 %r5499, %rs305; + and.b32 %r7692, %r5499, 255; + +$L__BB3_950: + mov.u32 %r2402, %r7692; + setp.gt.u32 %p1087, %r7779, 2879; + mov.u32 %r7776, 1; + @%p1087 bra $L__BB3_1002; + + mov.u32 %r5501, 8; + sub.s32 %r5502, %r5501, %r7777; + sub.s32 %r5503, %r5502, %r7778; + min.u32 %r5504, %r5503, %r2402; + setp.eq.s32 %p1088, %r5504, 32; + mov.u32 %r5505, -1; + shl.b32 %r5506, %r5505, %r5504; + not.b32 %r5507, %r5506; + selp.b32 %r5508, -1, %r5507, %p1088; + and.b32 %r5509, %r5508, %r7693; + shl.b32 %r5510, %r5509, %r7778; + cvt.u16.u32 %rs815, %r5510; + or.b16 %rs1147, %rs1147, %rs815; + add.s32 %r7778, %r5504, %r7778; + sub.s32 %r7692, %r2402, %r5504; + shr.u32 %r7693, %r7693, %r5504; + setp.gt.u32 %p1089, %r5503, %r2402; + @%p1089 bra $L__BB3_954; + + setp.ne.s32 %p1090, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs816, %rs1147, 255; + setp.ne.s16 %p1091, %rs816, 127; + and.pred %p1092, %p1090, %p1091; + @%p1092 bra $L__BB3_954; + + mov.u32 %r5513, 20548; + sub.s32 %r5514, %r5513, %r7779; + cvt.u64.u32 %rd531, %r5514; + add.s64 %rd532, %rd531, %rd5; + add.s64 %rd533, %rd1, %rd532; + st.global.u8 [%rd533], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1093, %rs816, 143; + selp.u32 %r7777, 1, 0, %p1093; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_954: + setp.eq.s32 %p1094, %r7692, 0; + mov.u32 %r7776, %r7691; + @%p1094 bra $L__BB3_1002; + bra.uni $L__BB3_950; + +$L__BB3_1002: + add.s64 %rd687, %rd687, 16; + shr.u32 %r5636, %r7472, 1; + or.b32 %r7306, %r5636, %r2158; + add.s32 %r7305, %r7305, 4; + setp.lt.u32 %p1157, %r7305, 64; + @%p1157 bra $L__BB3_675; + + ld.param.u64 %rd685, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_3]; + mov.u16 %rs851, 0; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+33], %rs851; + add.s64 %rd33, %rd4, 128; + cvta.to.global.u64 %rd34, %rd48; + cvta.to.global.u64 %rd35, %rd685; + mov.u32 %r7780, 2; + mov.u64 %rd688, 0; + +$L__BB3_1004: + shl.b64 %rd568, %rd688, 7; + add.s64 %rd569, %rd33, %rd568; + shl.b64 %rd570, %rd569, 2; + add.s64 %rd689, %rd3, %rd570; + ld.shared.u8 %rs1152, [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+1]; + mov.u32 %r5639, 0; + ld.shared.u8 %rs854, [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val]; + max.u16 %rs1154, %rs854, %rs1152; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val], %rs851; + ld.shared.u8 %r5640, [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val]; + ld.shared.u8 %rs1150, [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+1]; + mul.wide.u16 %r5641, %rs1150, 4; + add.s32 %r7798, %r5641, %r5640; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val], %rs851; + mov.u16 %rs1151, %rs851; + mov.u16 %rs1153, %rs851; + mov.u32 %r7796, %r5639; + mov.u32 %r7797, %r5639; + bra.uni $L__BB3_1005; + +$L__BB3_1072: + setp.gt.u32 %p1233, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7886, 1; + @%p1233 bra $L__BB3_1074; + + and.b16 %rs882, %rs1089, 255; + st.global.u8 [%rd39], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1234, %rs882, 255; + selp.b32 %r7388, 7, 8, %p1234; + mov.u16 %rs1089, 0; + mov.u32 %r7886, %r7540; + bra.uni $L__BB3_1074; + +$L__BB3_1173: + setp.gt.u32 %p1348, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r8028, 1; + @%p1348 bra $L__BB3_1175; + + and.b16 %rs917, %rs1089, 255; + st.global.u8 [%rd40], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1349, %rs917, 255; + selp.b32 %r7388, 7, 8, %p1349; + mov.u16 %rs1089, 0; + mov.u32 %r8028, %r7540; + bra.uni $L__BB3_1175; + +$L__BB3_1005: + mov.u32 %r2563, %r7797; + ld.global.u32 %r2580, [%rd689]; + setp.eq.s32 %p1158, %r2580, 0; + mov.u32 %r7814, %r5639; + @%p1158 bra $L__BB3_1007; + + and.b32 %r5643, %r2580, -2147483648; + abs.s32 %r5644, %r2580; + shl.b32 %r5645, %r5644, %r1719; + or.b32 %r7814, %r5645, %r5643; + +$L__BB3_1007: + shl.b32 %r5649, %r7814, 1; + shr.u32 %r5650, %r5649, %r1719; + and.b32 %r2583, %r5650, -2; + setp.eq.s32 %p1159, %r2583, 0; + mov.u32 %r7818, 0; + mov.u32 %r7815, %r7818; + mov.u32 %r7816, %r7818; + mov.u32 %r7822, %r7818; + @%p1159 bra $L__BB3_1009; + + add.s32 %r5652, %r2583, -1; + clz.b32 %r5653, %r5652; + mov.u32 %r5654, 32; + sub.s32 %r7815, %r5654, %r5653; + shr.u32 %r5655, %r7814, 31; + add.s32 %r5656, %r5655, %r2583; + add.s32 %r7816, %r5656, -2; + mov.u32 %r7822, 1; + +$L__BB3_1009: + ld.global.u32 %r2589, [%rd689+256]; + setp.eq.s32 %p1160, %r2589, 0; + @%p1160 bra $L__BB3_1011; + + and.b32 %r5658, %r2589, -2147483648; + abs.s32 %r5659, %r2589; + shl.b32 %r5660, %r5659, %r1719; + or.b32 %r7818, %r5660, %r5658; + +$L__BB3_1011: + shl.b32 %r5663, %r7818, 1; + shr.u32 %r5664, %r5663, %r1719; + and.b32 %r2592, %r5664, -2; + setp.eq.s32 %p1161, %r2592, 0; + mov.u32 %r7823, 0; + mov.u32 %r7819, %r7823; + mov.u32 %r7820, %r7823; + mov.u32 %r7826, %r7815; + @%p1161 bra $L__BB3_1013; + + or.b32 %r7822, %r7822, 2; + add.s32 %r5665, %r2592, -1; + clz.b32 %r5666, %r5665; + mov.u32 %r5667, 32; + sub.s32 %r7819, %r5667, %r5666; + max.s32 %r7826, %r7815, %r7819; + shr.u32 %r5668, %r7818, 31; + add.s32 %r5669, %r5668, %r2592; + add.s32 %r7820, %r5669, -2; + +$L__BB3_1013: + ld.global.u32 %r2601, [%rd689+4]; + setp.eq.s32 %p1162, %r2601, 0; + @%p1162 bra $L__BB3_1015; + + and.b32 %r5671, %r2601, -2147483648; + abs.s32 %r5672, %r2601; + shl.b32 %r5673, %r5672, %r1719; + or.b32 %r7823, %r5673, %r5671; + +$L__BB3_1015: + shl.b32 %r5676, %r7823, 1; + shr.u32 %r5677, %r5676, %r1719; + and.b32 %r2604, %r5677, -2; + setp.eq.s32 %p1163, %r2604, 0; + mov.u32 %r7828, 0; + mov.u32 %r7824, %r7828; + mov.u32 %r7825, %r7828; + @%p1163 bra $L__BB3_1017; + + or.b32 %r7822, %r7822, 4; + add.s32 %r5678, %r2604, -1; + clz.b32 %r5679, %r5678; + mov.u32 %r5680, 32; + sub.s32 %r7824, %r5680, %r5679; + max.s32 %r7826, %r7826, %r7824; + shr.u32 %r5681, %r7823, 31; + add.s32 %r5682, %r5681, %r2604; + add.s32 %r7825, %r5682, -2; + +$L__BB3_1017: + ld.global.u32 %r2613, [%rd689+260]; + setp.eq.s32 %p1164, %r2613, 0; + @%p1164 bra $L__BB3_1019; + + and.b32 %r5684, %r2613, -2147483648; + abs.s32 %r5685, %r2613; + shl.b32 %r5686, %r5685, %r1719; + or.b32 %r7828, %r5686, %r5684; + +$L__BB3_1019: + shl.b32 %r5689, %r7828, 1; + shr.u32 %r5690, %r5689, %r1719; + and.b32 %r2616, %r5690, -2; + setp.eq.s32 %p1165, %r2616, 0; + mov.u32 %r7833, 0; + mov.u32 %r7829, %r7833; + mov.u32 %r7830, %r7833; + @%p1165 bra $L__BB3_1021; + + or.b32 %r7822, %r7822, 8; + add.s32 %r5691, %r2616, -1; + clz.b32 %r5692, %r5691; + mov.u32 %r5693, 32; + sub.s32 %r7829, %r5693, %r5692; + max.s32 %r7826, %r7826, %r7829; + shr.u32 %r5694, %r7828, 31; + add.s32 %r5695, %r5694, %r2616; + add.s32 %r7830, %r5695, -2; + +$L__BB3_1021: + add.s32 %r5697, %r7822, -1; + and.b32 %r5698, %r5697, %r7822; + setp.ne.s32 %p1166, %r5698, 0; + and.b16 %rs855, %rs1154, 255; + setp.gt.u16 %p1167, %rs855, 2; + and.pred %p1168, %p1167, %p1166; + cvt.u32.u16 %r5699, %rs1154; + and.b32 %r5700, %r5699, 255; + add.s32 %r5701, %r5700, -1; + selp.b32 %r5702, %r5701, 1, %p1168; + max.s32 %r2625, %r5702, %r7826; + sub.s32 %r2626, %r2625, %r5702; + setp.lt.s32 %p1169, %r2626, 1; + @%p1169 bra $L__BB3_1023; + + setp.eq.s32 %p1170, %r7815, %r7826; + selp.u32 %r5703, 1, 0, %p1170; + setp.eq.s32 %p1171, %r7819, %r7826; + selp.u32 %r5704, -1, 0, %p1171; + bfi.b32 %r5705, %r5704, %r5703, 1, 1; + setp.eq.s32 %p1172, %r7824, %r7826; + selp.u16 %rs856, 1, 0, %p1172; + mul.wide.u16 %r5706, %rs856, 4; + or.b32 %r5707, %r5705, %r5706; + setp.eq.s32 %p1173, %r7829, %r7826; + selp.u16 %rs857, 1, 0, %p1173; + mul.wide.u16 %r5708, %rs857, 8; + or.b32 %r7833, %r5707, %r5708; + +$L__BB3_1023: + shl.b32 %r5709, %r7822, 4; + shl.b32 %r5710, %r7798, 8; + or.b32 %r5711, %r5709, %r5710; + or.b32 %r5712, %r5711, %r7833; + mul.wide.u32 %rd571, %r5712, 2; + add.s64 %rd572, %rd35, %rd571; + ld.global.u16 %rs370, [%rd572]; + shr.u16 %rs858, %rs370, 4; + and.b16 %rs371, %rs858, 7; + setp.eq.s16 %p1174, %rs371, 0; + mov.u32 %r7845, %r7776; + @%p1174 bra $L__BB3_1030; + + cvt.u32.u16 %r7834, %rs371; + shr.u16 %rs859, %rs370, 8; + cvt.u32.u16 %r7835, %rs859; + +$L__BB3_1025: + mov.u32 %r2631, %r7834; + setp.gt.u32 %p1175, %r7779, 2879; + mov.u32 %r7845, 1; + @%p1175 bra $L__BB3_1030; + + mov.u32 %r5714, 8; + sub.s32 %r5715, %r5714, %r7777; + sub.s32 %r5716, %r5715, %r7778; + min.u32 %r5717, %r5716, %r2631; + setp.eq.s32 %p1176, %r5717, 32; + mov.u32 %r5718, -1; + shl.b32 %r5719, %r5718, %r5717; + not.b32 %r5720, %r5719; + selp.b32 %r5721, -1, %r5720, %p1176; + and.b32 %r5722, %r5721, %r7835; + shl.b32 %r5723, %r5722, %r7778; + cvt.u16.u32 %rs860, %r5723; + or.b16 %rs1147, %rs1147, %rs860; + add.s32 %r7778, %r5717, %r7778; + sub.s32 %r7834, %r2631, %r5717; + shr.u32 %r7835, %r7835, %r5717; + setp.gt.u32 %p1177, %r5716, %r2631; + @%p1177 bra $L__BB3_1029; + + setp.ne.s32 %p1178, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs861, %rs1147, 255; + setp.ne.s16 %p1179, %rs861, 127; + and.pred %p1180, %p1178, %p1179; + @%p1180 bra $L__BB3_1029; + + mov.u32 %r5726, 20548; + sub.s32 %r5727, %r5726, %r7779; + cvt.u64.u32 %rd573, %r5727; + add.s64 %rd574, %rd573, %rd5; + add.s64 %rd575, %rd1, %rd574; + st.global.u8 [%rd575], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1181, %rs861, 143; + selp.u32 %r7777, 1, 0, %p1181; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_1029: + setp.ne.s32 %p1182, %r7834, 0; + mov.u32 %r7845, %r7776; + @%p1182 bra $L__BB3_1025; + +$L__BB3_1030: + setp.ne.s32 %p1183, %r7798, 0; + @%p1183 bra $L__BB3_1078; + + setp.eq.s32 %p1184, %r7822, 0; + add.s32 %r5728, %r7382, 17477; + cvt.u64.u32 %rd576, %r5728; + add.s64 %rd577, %rd576, %rd5; + add.s64 %rd39, %rd1, %rd577; + @%p1184 bra $L__BB3_1070; + + shl.b16 %rs1089, %rs1089, 1; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1185, %r7388, 0; + mov.u32 %r7879, %r7540; + @%p1185 bra $L__BB3_1035; + + setp.gt.u32 %p1186, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7879, 1; + @%p1186 bra $L__BB3_1035; + + st.global.u8 [%rd39], %rs1089; + add.s32 %r7382, %r7382, 1; + mov.u32 %r7388, 8; + mov.u16 %rs1089, 0; + mov.u32 %r7879, %r7540; + +$L__BB3_1035: + setp.lt.u32 %p1187, %r7542, 3; + mov.u32 %r7849, 0; + @%p1187 bra $L__BB3_1038; + + setp.lt.u32 %p1188, %r7542, 6; + mov.u32 %r7849, 1; + @%p1188 bra $L__BB3_1038; + + setp.lt.u32 %p1189, %r7542, 9; + setp.eq.s32 %p1190, %r7542, 11; + selp.b32 %r5734, 4, 5, %p1190; + setp.lt.u32 %p1191, %r7542, 11; + selp.b32 %r5735, 3, %r5734, %p1191; + selp.b32 %r7849, 2, %r5735, %p1189; + +$L__BB3_1038: + setp.eq.s32 %p1192, %r7849, 0; + @%p1192 bra $L__BB3_1066; + + add.s32 %r2655, %r7849, -1; + and.b32 %r2656, %r7849, 3; + setp.eq.s32 %p1193, %r2656, 0; + mov.u32 %r7859, %r7849; + mov.u32 %r7862, %r7879; + @%p1193 bra $L__BB3_1051; + + mov.u32 %r5737, 1; + shl.b32 %r5738, %r5737, %r2655; + and.b32 %r5739, %r5738, %r7543; + setp.ne.s32 %p1194, %r5739, 0; + selp.u32 %r5740, 1, 0, %p1194; + cvt.u32.u16 %r5741, %rs1089; + bfi.b32 %r5742, %r5741, %r5740, 1, 8; + cvt.u16.u32 %rs1089, %r5742; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1195, %r7388, 0; + mov.u32 %r7862, %r7879; + @%p1195 bra $L__BB3_1043; + + setp.gt.u32 %p1196, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7862, %r5737; + @%p1196 bra $L__BB3_1043; + + add.s32 %r5746, %r7382, 17477; + cvt.u64.u32 %rd578, %r5746; + add.s64 %rd579, %rd578, %rd5; + add.s64 %rd580, %rd1, %rd579; + st.global.u8 [%rd580], %rs1089; + add.s32 %r7382, %r7382, 1; + mov.u32 %r7388, 8; + mov.u16 %rs1089, 0; + mov.u32 %r7862, %r7879; + +$L__BB3_1043: + setp.eq.s32 %p1197, %r2656, 1; + mov.u32 %r7879, %r7862; + mov.u32 %r7859, %r2655; + @%p1197 bra $L__BB3_1051; + + add.s32 %r7859, %r7849, -2; + mov.u32 %r5747, 1; + shl.b32 %r5748, %r5747, %r7859; + and.b32 %r5749, %r5748, %r7543; + setp.ne.s32 %p1198, %r5749, 0; + selp.u32 %r5750, 1, 0, %p1198; + cvt.u32.u16 %r5751, %rs1089; + bfi.b32 %r5752, %r5751, %r5750, 1, 8; + cvt.u16.u32 %rs1089, %r5752; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1199, %r7388, 0; + mov.u32 %r7853, %r7862; + @%p1199 bra $L__BB3_1047; + + setp.gt.u32 %p1200, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7853, %r5747; + @%p1200 bra $L__BB3_1047; + + add.s32 %r5755, %r7382, 17477; + cvt.u64.u32 %rd581, %r5755; + add.s64 %rd582, %rd581, %rd5; + add.s64 %rd583, %rd1, %rd582; + and.b16 %rs868, %rs1089, 255; + st.global.u8 [%rd583], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1201, %rs868, 255; + selp.b32 %r7388, 7, 8, %p1201; + mov.u16 %rs1089, 0; + mov.u32 %r7853, %r7862; + +$L__BB3_1047: + setp.eq.s32 %p1202, %r2656, 2; + mov.u32 %r7879, %r7853; + mov.u32 %r7862, %r7853; + @%p1202 bra $L__BB3_1051; + + add.s32 %r7859, %r7849, -3; + mov.u32 %r5756, 1; + shl.b32 %r5757, %r5756, %r7859; + and.b32 %r5758, %r5757, %r7543; + setp.ne.s32 %p1203, %r5758, 0; + selp.u32 %r5759, 1, 0, %p1203; + cvt.u32.u16 %r5760, %rs1089; + bfi.b32 %r5761, %r5760, %r5759, 1, 8; + cvt.u16.u32 %rs1089, %r5761; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1204, %r7388, 0; + mov.u32 %r7879, %r7853; + mov.u32 %r7862, %r7853; + @%p1204 bra $L__BB3_1051; + + setp.gt.u32 %p1205, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7879, %r5756; + mov.u32 %r7862, %r5756; + @%p1205 bra $L__BB3_1051; + + add.s32 %r5766, %r7382, 17477; + cvt.u64.u32 %rd584, %r5766; + add.s64 %rd585, %rd584, %rd5; + add.s64 %rd586, %rd1, %rd585; + and.b16 %rs871, %rs1089, 255; + st.global.u8 [%rd586], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1206, %rs871, 255; + selp.b32 %r7388, 7, 8, %p1206; + mov.u16 %rs1089, 0; + mov.u32 %r7879, %r7853; + mov.u32 %r7862, %r7853; + +$L__BB3_1051: + setp.lt.u32 %p1207, %r2655, 3; + @%p1207 bra $L__BB3_1066; + + mov.u32 %r7879, %r7862; + +$L__BB3_1053: + add.s32 %r5767, %r7859, -1; + mov.u32 %r5768, 1; + shl.b32 %r5769, %r5768, %r5767; + and.b32 %r5770, %r5769, %r7543; + setp.ne.s32 %p1208, %r5770, 0; + selp.u32 %r5771, 1, 0, %p1208; + cvt.u32.u16 %r5772, %rs1089; + bfi.b32 %r7868, %r5772, %r5771, 1, 8; + add.s32 %r7869, %r7388, -1; + setp.ne.s32 %p1209, %r7869, 0; + mov.u32 %r7867, %r7879; + @%p1209 bra $L__BB3_1056; + + setp.gt.u32 %p1210, %r7382, 191; + mov.u32 %r7869, 0; + mov.u32 %r7867, %r5768; + @%p1210 bra $L__BB3_1056; + + cvt.u16.u32 %rs872, %r7868; + and.b16 %rs873, %rs872, 255; + add.s32 %r5776, %r7382, 17477; + cvt.u64.u32 %rd587, %r5776; + add.s64 %rd588, %rd587, %rd5; + add.s64 %rd589, %rd1, %rd588; + st.global.u8 [%rd589], %rs872; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1211, %rs873, 255; + selp.b32 %r7869, 7, 8, %p1211; + mov.u32 %r7868, 0; + mov.u32 %r7867, %r7879; + +$L__BB3_1056: + add.s32 %r5777, %r7859, -2; + shl.b32 %r5779, %r5768, %r5777; + and.b32 %r5780, %r5779, %r7543; + setp.ne.s32 %p1212, %r5780, 0; + and.b32 %r5781, %r7868, 127; + selp.u32 %r5782, 1, 0, %p1212; + bfi.b32 %r7872, %r5781, %r5782, 1, 7; + add.s32 %r7873, %r7869, -1; + setp.ne.s32 %p1213, %r7873, 0; + mov.u32 %r7871, %r7867; + @%p1213 bra $L__BB3_1059; + + setp.gt.u32 %p1214, %r7382, 191; + mov.u32 %r7873, 0; + mov.u32 %r7871, 1; + @%p1214 bra $L__BB3_1059; + + cvt.u16.u32 %rs874, %r7872; + and.b16 %rs875, %rs874, 255; + add.s32 %r5786, %r7382, 17477; + cvt.u64.u32 %rd590, %r5786; + add.s64 %rd591, %rd590, %rd5; + add.s64 %rd592, %rd1, %rd591; + st.global.u8 [%rd592], %rs874; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1215, %rs875, 255; + selp.b32 %r7873, 7, 8, %p1215; + mov.u32 %r7872, 0; + mov.u32 %r7871, %r7867; + +$L__BB3_1059: + add.s32 %r5787, %r7859, -3; + mov.u32 %r5788, 1; + shl.b32 %r5789, %r5788, %r5787; + and.b32 %r5790, %r5789, %r7543; + setp.ne.s32 %p1216, %r5790, 0; + and.b32 %r5791, %r7872, 127; + selp.u32 %r5792, 1, 0, %p1216; + bfi.b32 %r7876, %r5791, %r5792, 1, 7; + add.s32 %r7877, %r7873, -1; + setp.ne.s32 %p1217, %r7877, 0; + mov.u32 %r7875, %r7871; + @%p1217 bra $L__BB3_1062; + + setp.gt.u32 %p1218, %r7382, 191; + mov.u32 %r7877, 0; + mov.u32 %r7875, %r5788; + @%p1218 bra $L__BB3_1062; + + cvt.u16.u32 %rs876, %r7876; + and.b16 %rs877, %rs876, 255; + add.s32 %r5796, %r7382, 17477; + cvt.u64.u32 %rd593, %r5796; + add.s64 %rd594, %rd593, %rd5; + add.s64 %rd595, %rd1, %rd594; + st.global.u8 [%rd595], %rs876; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1219, %rs877, 255; + selp.b32 %r7877, 7, 8, %p1219; + mov.u32 %r7876, 0; + mov.u32 %r7875, %r7871; + +$L__BB3_1062: + add.s32 %r7859, %r7859, -4; + shl.b32 %r5798, %r5788, %r7859; + and.b32 %r5799, %r5798, %r7543; + setp.ne.s32 %p1220, %r5799, 0; + and.b32 %r5800, %r7876, 127; + selp.u32 %r5801, 1, 0, %p1220; + bfi.b32 %r5802, %r5800, %r5801, 1, 15; + cvt.u16.u32 %rs1089, %r5802; + add.s32 %r7388, %r7877, -1; + setp.ne.s32 %p1221, %r7388, 0; + mov.u32 %r7879, %r7875; + @%p1221 bra $L__BB3_1065; + + setp.gt.u32 %p1222, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7879, 1; + @%p1222 bra $L__BB3_1065; + + add.s32 %r5805, %r7382, 17477; + cvt.u64.u32 %rd596, %r5805; + add.s64 %rd597, %rd596, %rd5; + add.s64 %rd598, %rd1, %rd597; + and.b16 %rs879, %rs1089, 255; + st.global.u8 [%rd598], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1223, %rs879, 255; + selp.b32 %r7388, 7, 8, %p1223; + mov.u16 %rs1089, 0; + mov.u32 %r7879, %r7875; + +$L__BB3_1065: + setp.ne.s32 %p1224, %r7859, 0; + @%p1224 bra $L__BB3_1053; + +$L__BB3_1066: + add.s32 %r5807, %r7542, -1; + setp.eq.s32 %p1225, %r7542, 0; + mov.u32 %r7543, 0; + selp.b32 %r7542, 0, %r5807, %p1225; + setp.lt.u32 %p1226, %r7542, 3; + mov.u32 %r7885, %r7543; + @%p1226 bra $L__BB3_1069; + + setp.lt.u32 %p1227, %r7542, 6; + mov.u32 %r7885, 1; + @%p1227 bra $L__BB3_1069; + + setp.lt.u32 %p1228, %r7542, 9; + setp.eq.s32 %p1229, %r7542, 11; + selp.b32 %r5809, 4, 5, %p1229; + setp.lt.u32 %p1230, %r7542, 11; + selp.b32 %r5810, 3, %r5809, %p1230; + selp.b32 %r7885, 2, %r5810, %p1228; + +$L__BB3_1069: + mov.u32 %r5812, 1; + shl.b32 %r7541, %r5812, %r7885; + mov.u32 %r7540, %r7879; + bra.uni $L__BB3_1078; + +$L__BB3_1070: + add.s32 %r7543, %r7543, 1; + setp.lt.u32 %p1231, %r7543, %r7541; + @%p1231 bra $L__BB3_1078; + + shl.b16 %rs880, %rs1089, 1; + or.b16 %rs1089, %rs880, 1; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1232, %r7388, 0; + mov.u32 %r7886, %r7540; + @%p1232 bra $L__BB3_1074; + bra.uni $L__BB3_1072; + +$L__BB3_1074: + add.s32 %r5816, %r7542, 1; + min.u32 %r7542, %r5816, 12; + setp.lt.u32 %p1235, %r7542, 3; + mov.u32 %r7543, 0; + mov.u32 %r7889, %r7543; + @%p1235 bra $L__BB3_1077; + + setp.lt.u32 %p1236, %r7542, 6; + mov.u32 %r7889, 1; + @%p1236 bra $L__BB3_1077; + + setp.lt.u32 %p1237, %r7542, 9; + setp.eq.s32 %p1238, %r7542, 11; + selp.b32 %r5818, 4, 5, %p1238; + setp.lt.u32 %p1239, %r7542, 11; + selp.b32 %r5819, 3, %r5818, %p1239; + selp.b32 %r7889, 2, %r5819, %p1237; + +$L__BB3_1077: + mov.u32 %r5821, 1; + shl.b32 %r7541, %r5821, %r7889; + mov.u32 %r7540, %r7886; + +$L__BB3_1078: + and.b16 %rs883, %rs370, 15; + cvt.u32.u16 %r2739, %rs883; + and.b32 %r5822, %r7822, 1; + setp.eq.b32 %p1240, %r5822, 1; + mov.pred %p1241, 0; + xor.pred %p1242, %p1240, %p1241; + not.pred %p1243, %p1242; + mov.u32 %r7906, %r8093; + @%p1243 bra $L__BB3_1085; + + and.b32 %r5823, %r2739, 1; + sub.s32 %r7896, %r2625, %r5823; + setp.eq.s32 %p1244, %r7896, 0; + mov.u32 %r7906, %r8093; + @%p1244 bra $L__BB3_1085; + + mov.u32 %r5824, -1; + shl.b32 %r5825, %r5824, %r7896; + not.b32 %r5826, %r5825; + and.b32 %r7897, %r7816, %r5826; + +$L__BB3_1081: + setp.gt.u32 %p1245, %r7925, 17476; + mov.u32 %r7906, 1; + @%p1245 bra $L__BB3_1085; + + sub.s32 %r5828, %r7924, %r7923; + min.u32 %r5829, %r5828, %r7896; + setp.eq.s32 %p1246, %r5829, 32; + mov.u32 %r5830, -1; + shl.b32 %r5831, %r5830, %r5829; + not.b32 %r5832, %r5831; + selp.b32 %r5833, -1, %r5832, %p1246; + and.b32 %r5834, %r5833, %r7897; + shl.b32 %r5835, %r5834, %r7923; + or.b32 %r7922, %r5835, %r7922; + add.s32 %r7923, %r5829, %r7923; + shr.u32 %r7897, %r7897, %r5829; + sub.s32 %r7896, %r7896, %r5829; + setp.lt.u32 %p1247, %r7923, %r7924; + @%p1247 bra $L__BB3_1084; + + cvt.u64.u32 %rd599, %r7925; + add.s64 %rd600, %rd599, %rd5; + add.s64 %rd601, %rd1, %rd600; + st.global.u8 [%rd601], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p1248, %r7922, 255; + selp.b32 %r7924, 7, 8, %p1248; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_1084: + setp.ne.s32 %p1249, %r7896, 0; + mov.u32 %r7906, %r8093; + @%p1249 bra $L__BB3_1081; + +$L__BB3_1085: + and.b32 %r2763, %r7822, 2; + setp.eq.s32 %p1250, %r2763, 0; + mov.u32 %r7921, %r7906; + @%p1250 bra $L__BB3_1092; + + shr.u32 %r5838, %r2739, 1; + and.b32 %r5839, %r5838, 1; + sub.s32 %r7911, %r2625, %r5839; + setp.eq.s32 %p1251, %r7911, 0; + mov.u32 %r7921, %r7906; + @%p1251 bra $L__BB3_1092; + + mov.u32 %r5840, -1; + shl.b32 %r5841, %r5840, %r7911; + not.b32 %r5842, %r5841; + and.b32 %r7912, %r7820, %r5842; + +$L__BB3_1088: + setp.gt.u32 %p1252, %r7925, 17476; + mov.u32 %r7921, 1; + @%p1252 bra $L__BB3_1092; + + sub.s32 %r5844, %r7924, %r7923; + min.u32 %r5845, %r5844, %r7911; + setp.eq.s32 %p1253, %r5845, 32; + mov.u32 %r5846, -1; + shl.b32 %r5847, %r5846, %r5845; + not.b32 %r5848, %r5847; + selp.b32 %r5849, -1, %r5848, %p1253; + and.b32 %r5850, %r5849, %r7912; + shl.b32 %r5851, %r5850, %r7923; + or.b32 %r7922, %r5851, %r7922; + add.s32 %r7923, %r5845, %r7923; + shr.u32 %r7912, %r7912, %r5845; + sub.s32 %r7911, %r7911, %r5845; + setp.lt.u32 %p1254, %r7923, %r7924; + @%p1254 bra $L__BB3_1091; + + cvt.u64.u32 %rd602, %r7925; + add.s64 %rd603, %rd602, %rd5; + add.s64 %rd604, %rd1, %rd603; + st.global.u8 [%rd604], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p1255, %r7922, 255; + selp.b32 %r7924, 7, 8, %p1255; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_1091: + setp.ne.s32 %p1256, %r7911, 0; + mov.u32 %r7921, %r7906; + @%p1256 bra $L__BB3_1088; + +$L__BB3_1092: + and.b32 %r2787, %r7822, 4; + setp.eq.s32 %p1257, %r2787, 0; + mov.u32 %r7936, %r7921; + @%p1257 bra $L__BB3_1099; + + shr.u32 %r5854, %r2739, 2; + and.b32 %r5855, %r5854, 1; + sub.s32 %r7926, %r2625, %r5855; + setp.eq.s32 %p1258, %r7926, 0; + mov.u32 %r7936, %r7921; + @%p1258 bra $L__BB3_1099; + + mov.u32 %r5856, -1; + shl.b32 %r5857, %r5856, %r7926; + not.b32 %r5858, %r5857; + and.b32 %r7927, %r7825, %r5858; + +$L__BB3_1095: + setp.gt.u32 %p1259, %r7925, 17476; + mov.u32 %r7936, 1; + @%p1259 bra $L__BB3_1099; + + sub.s32 %r5860, %r7924, %r7923; + min.u32 %r5861, %r5860, %r7926; + setp.eq.s32 %p1260, %r5861, 32; + mov.u32 %r5862, -1; + shl.b32 %r5863, %r5862, %r5861; + not.b32 %r5864, %r5863; + selp.b32 %r5865, -1, %r5864, %p1260; + and.b32 %r5866, %r5865, %r7927; + shl.b32 %r5867, %r5866, %r7923; + or.b32 %r7922, %r5867, %r7922; + add.s32 %r7923, %r5861, %r7923; + shr.u32 %r7927, %r7927, %r5861; + sub.s32 %r7926, %r7926, %r5861; + setp.lt.u32 %p1261, %r7923, %r7924; + @%p1261 bra $L__BB3_1098; + + cvt.u64.u32 %rd605, %r7925; + add.s64 %rd606, %rd605, %rd5; + add.s64 %rd607, %rd1, %rd606; + st.global.u8 [%rd607], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p1262, %r7922, 255; + selp.b32 %r7924, 7, 8, %p1262; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_1098: + setp.ne.s32 %p1263, %r7926, 0; + mov.u32 %r7936, %r7921; + @%p1263 bra $L__BB3_1095; + +$L__BB3_1099: + and.b32 %r2811, %r7822, 8; + setp.eq.s32 %p1264, %r2811, 0; + mov.u32 %r7951, %r7936; + @%p1264 bra $L__BB3_1106; + + shr.u32 %r5870, %r2739, 3; + sub.s32 %r7941, %r2625, %r5870; + setp.eq.s32 %p1265, %r7941, 0; + mov.u32 %r7951, %r7936; + @%p1265 bra $L__BB3_1106; + + mov.u32 %r5871, -1; + shl.b32 %r5872, %r5871, %r7941; + not.b32 %r5873, %r5872; + and.b32 %r7942, %r7830, %r5873; + +$L__BB3_1102: + setp.gt.u32 %p1266, %r7925, 17476; + mov.u32 %r7951, 1; + @%p1266 bra $L__BB3_1106; + + sub.s32 %r5875, %r7924, %r7923; + min.u32 %r5876, %r5875, %r7941; + setp.eq.s32 %p1267, %r5876, 32; + mov.u32 %r5877, -1; + shl.b32 %r5878, %r5877, %r5876; + not.b32 %r5879, %r5878; + selp.b32 %r5880, -1, %r5879, %p1267; + and.b32 %r5881, %r5880, %r7942; + shl.b32 %r5882, %r5881, %r7923; + or.b32 %r7922, %r5882, %r7922; + add.s32 %r7923, %r5876, %r7923; + shr.u32 %r7942, %r7942, %r5876; + sub.s32 %r7941, %r7941, %r5876; + setp.lt.u32 %p1268, %r7923, %r7924; + @%p1268 bra $L__BB3_1105; + + cvt.u64.u32 %rd608, %r7925; + add.s64 %rd609, %rd608, %rd5; + add.s64 %rd610, %rd1, %rd609; + st.global.u8 [%rd610], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p1269, %r7922, 255; + selp.b32 %r7924, 7, 8, %p1269; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_1105: + setp.ne.s32 %p1270, %r7941, 0; + mov.u32 %r7951, %r7936; + @%p1270 bra $L__BB3_1102; + +$L__BB3_1106: + and.b32 %r5886, %r7819, 255; + cvt.u32.u16 %r5887, %rs1153; + and.b32 %r5888, %r5887, 255; + setp.lt.u32 %p1271, %r5886, %r5888; + cvt.u16.u32 %rs884, %r7819; + selp.b16 %rs885, %rs1153, %rs884, %p1271; + add.s32 %r2835, %r4924, %r2563; + mov.u32 %r7957, 0; + st.shared.u8 [%r2835], %rs885; + ld.shared.u8 %rs392, [%r2835+2]; + setp.gt.u16 %p1272, %rs1152, %rs392; + add.s32 %r7797, %r2563, 2; + add.s32 %r5890, %r2563, 1; + selp.b32 %r5891, %r5890, %r7797, %p1272; + add.s32 %r5892, %r4924, %r5891; + ld.shared.u8 %rs393, [%r5892]; + cvt.u16.u32 %rs394, %r7829; + st.shared.u8 [%r2835+1], %r7829; + cvt.u16.u32 %rs887, %r2763; + shr.u16 %rs888, %rs887, 1; + or.b16 %rs889, %rs1151, %rs888; + add.s32 %r2837, %r4928, %r2563; + st.shared.u8 [%r2837], %rs889; + ld.shared.u8 %r2838, [%r2837+2]; + shr.u32 %r2839, %r2811, 3; + st.shared.u8 [%r2837+1], %r2839; + ld.global.u32 %r2840, [%rd689+8]; + setp.eq.s32 %p1273, %r2840, 0; + mov.u32 %r7956, %r7957; + @%p1273 bra $L__BB3_1108; + + and.b32 %r5894, %r2840, -2147483648; + abs.s32 %r5895, %r2840; + shl.b32 %r5896, %r5895, %r1719; + or.b32 %r7956, %r5896, %r5894; + +$L__BB3_1108: + shl.b32 %r5900, %r7956, 1; + shr.u32 %r5901, %r5900, %r1719; + and.b32 %r2843, %r5901, -2; + setp.eq.s32 %p1274, %r2843, 0; + mov.u32 %r7958, %r7957; + mov.u32 %r7964, %r7957; + @%p1274 bra $L__BB3_1110; + + add.s32 %r5903, %r2843, -1; + clz.b32 %r5904, %r5903; + mov.u32 %r5905, 32; + sub.s32 %r7957, %r5905, %r5904; + shr.u32 %r5906, %r7956, 31; + add.s32 %r5907, %r5906, %r2843; + add.s32 %r7958, %r5907, -2; + mov.u32 %r7964, 1; + +$L__BB3_1110: + ld.global.u32 %r2849, [%rd689+264]; + setp.eq.s32 %p1275, %r2849, 0; + mov.u32 %r7961, 0; + mov.u32 %r7960, %r7961; + @%p1275 bra $L__BB3_1112; + + and.b32 %r5909, %r2849, -2147483648; + abs.s32 %r5910, %r2849; + shl.b32 %r5911, %r5910, %r1719; + or.b32 %r7960, %r5911, %r5909; + +$L__BB3_1112: + shl.b32 %r5914, %r7960, 1; + shr.u32 %r5915, %r5914, %r1719; + and.b32 %r2852, %r5915, -2; + setp.eq.s32 %p1276, %r2852, 0; + mov.u32 %r7962, %r7961; + mov.u32 %r7968, %r7957; + @%p1276 bra $L__BB3_1114; + + or.b32 %r7964, %r7964, 2; + add.s32 %r5916, %r2852, -1; + clz.b32 %r5917, %r5916; + mov.u32 %r5918, 32; + sub.s32 %r7961, %r5918, %r5917; + max.s32 %r7968, %r7957, %r7961; + shr.u32 %r5919, %r7960, 31; + add.s32 %r5920, %r5919, %r2852; + add.s32 %r7962, %r5920, -2; + +$L__BB3_1114: + ld.global.u32 %r2861, [%rd689+12]; + setp.eq.s32 %p1277, %r2861, 0; + mov.u32 %r7966, 0; + mov.u32 %r7965, %r7966; + @%p1277 bra $L__BB3_1116; + + and.b32 %r5922, %r2861, -2147483648; + abs.s32 %r5923, %r2861; + shl.b32 %r5924, %r5923, %r1719; + or.b32 %r7965, %r5924, %r5922; + +$L__BB3_1116: + shl.b32 %r5927, %r7965, 1; + shr.u32 %r5928, %r5927, %r1719; + and.b32 %r2864, %r5928, -2; + setp.eq.s32 %p1278, %r2864, 0; + mov.u32 %r7967, %r7966; + @%p1278 bra $L__BB3_1118; + + or.b32 %r7964, %r7964, 4; + add.s32 %r5929, %r2864, -1; + clz.b32 %r5930, %r5929; + mov.u32 %r5931, 32; + sub.s32 %r7966, %r5931, %r5930; + max.s32 %r7968, %r7968, %r7966; + shr.u32 %r5932, %r7965, 31; + add.s32 %r5933, %r5932, %r2864; + add.s32 %r7967, %r5933, -2; + +$L__BB3_1118: + ld.global.u32 %r2873, [%rd689+268]; + setp.eq.s32 %p1279, %r2873, 0; + mov.u32 %r7971, 0; + mov.u32 %r7970, %r7971; + @%p1279 bra $L__BB3_1120; + + and.b32 %r5935, %r2873, -2147483648; + abs.s32 %r5936, %r2873; + shl.b32 %r5937, %r5936, %r1719; + or.b32 %r7970, %r5937, %r5935; + +$L__BB3_1120: + shl.b32 %r5940, %r7970, 1; + shr.u32 %r5941, %r5940, %r1719; + and.b32 %r2876, %r5941, -2; + setp.eq.s32 %p1280, %r2876, 0; + mov.u32 %r7972, %r7971; + @%p1280 bra $L__BB3_1122; + + or.b32 %r7964, %r7964, 8; + add.s32 %r5942, %r2876, -1; + clz.b32 %r5943, %r5942; + mov.u32 %r5944, 32; + sub.s32 %r7971, %r5944, %r5943; + max.s32 %r7968, %r7968, %r7971; + shr.u32 %r5945, %r7970, 31; + add.s32 %r5946, %r5945, %r2876; + add.s32 %r7972, %r5946, -2; + +$L__BB3_1122: + shr.u32 %r5948, %r2811, 2; + shr.u32 %r5949, %r2787, 1; + or.b32 %r5950, %r5948, %r5949; + shl.b32 %r5951, %r2838, 2; + cvt.u32.u16 %r5952, %rs1150; + and.b32 %r5953, %r5952, 255; + add.s32 %r5954, %r5951, %r5953; + or.b32 %r2885, %r5950, %r5954; + add.s32 %r5955, %r7964, -1; + and.b32 %r5956, %r5955, %r7964; + setp.ne.s32 %p1281, %r5956, 0; + mov.u32 %r7975, 0; + setp.gt.u16 %p1282, %rs393, 2; + and.pred %p1283, %p1282, %p1281; + cvt.u32.u16 %r5957, %rs393; + and.b32 %r5958, %r5957, 255; + add.s32 %r5959, %r5958, -1; + selp.b32 %r5960, %r5959, 1, %p1283; + max.s32 %r2886, %r5960, %r7968; + sub.s32 %r2887, %r2886, %r5960; + setp.lt.s32 %p1284, %r2887, 1; + @%p1284 bra $L__BB3_1124; + + setp.eq.s32 %p1285, %r7957, %r7968; + selp.u32 %r5961, 1, 0, %p1285; + setp.eq.s32 %p1286, %r7961, %r7968; + selp.u32 %r5962, -1, 0, %p1286; + bfi.b32 %r5963, %r5962, %r5961, 1, 1; + setp.eq.s32 %p1287, %r7966, %r7968; + selp.u16 %rs891, 1, 0, %p1287; + mul.wide.u16 %r5964, %rs891, 4; + or.b32 %r5965, %r5963, %r5964; + setp.eq.s32 %p1288, %r7971, %r7968; + selp.u16 %rs892, 1, 0, %p1288; + mul.wide.u16 %r5966, %rs892, 8; + or.b32 %r7975, %r5965, %r5966; + +$L__BB3_1124: + shl.b32 %r5967, %r7964, 4; + shl.b32 %r5968, %r2885, 8; + or.b32 %r5969, %r5967, %r5968; + or.b32 %r5970, %r5969, %r7975; + mul.wide.u32 %rd611, %r5970, 2; + add.s64 %rd612, %rd35, %rd611; + ld.global.u16 %rs395, [%rd612]; + shr.u16 %rs893, %rs395, 4; + and.b16 %rs396, %rs893, 7; + setp.eq.s16 %p1289, %rs396, 0; + mov.u32 %r7987, %r7845; + @%p1289 bra $L__BB3_1131; + + cvt.u32.u16 %r7976, %rs396; + shr.u16 %rs894, %rs395, 8; + cvt.u32.u16 %r7977, %rs894; + +$L__BB3_1126: + mov.u32 %r2892, %r7976; + setp.gt.u32 %p1290, %r7779, 2879; + mov.u32 %r7987, 1; + @%p1290 bra $L__BB3_1131; + + mov.u32 %r5972, 8; + sub.s32 %r5973, %r5972, %r7777; + sub.s32 %r5974, %r5973, %r7778; + min.u32 %r5975, %r5974, %r2892; + setp.eq.s32 %p1291, %r5975, 32; + mov.u32 %r5976, -1; + shl.b32 %r5977, %r5976, %r5975; + not.b32 %r5978, %r5977; + selp.b32 %r5979, -1, %r5978, %p1291; + and.b32 %r5980, %r5979, %r7977; + shl.b32 %r5981, %r5980, %r7778; + cvt.u16.u32 %rs895, %r5981; + or.b16 %rs1147, %rs1147, %rs895; + add.s32 %r7778, %r5975, %r7778; + sub.s32 %r7976, %r2892, %r5975; + shr.u32 %r7977, %r7977, %r5975; + setp.gt.u32 %p1292, %r5974, %r2892; + @%p1292 bra $L__BB3_1130; + + setp.ne.s32 %p1293, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs896, %rs1147, 255; + setp.ne.s16 %p1294, %rs896, 127; + and.pred %p1295, %p1293, %p1294; + @%p1295 bra $L__BB3_1130; + + mov.u32 %r5984, 20548; + sub.s32 %r5985, %r5984, %r7779; + cvt.u64.u32 %rd613, %r5985; + add.s64 %rd614, %rd613, %rd5; + add.s64 %rd615, %rd1, %rd614; + st.global.u8 [%rd615], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1296, %rs896, 143; + selp.u32 %r7777, 1, 0, %p1296; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_1130: + setp.ne.s32 %p1297, %r7976, 0; + mov.u32 %r7987, %r7845; + @%p1297 bra $L__BB3_1126; + +$L__BB3_1131: + setp.ne.s32 %p1298, %r2885, 0; + @%p1298 bra $L__BB3_1179; + + setp.eq.s32 %p1299, %r7964, 0; + add.s32 %r5986, %r7382, 17477; + cvt.u64.u32 %rd616, %r5986; + add.s64 %rd617, %rd616, %rd5; + add.s64 %rd40, %rd1, %rd617; + @%p1299 bra $L__BB3_1171; + + shl.b16 %rs1089, %rs1089, 1; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1300, %r7388, 0; + mov.u32 %r8021, %r7540; + @%p1300 bra $L__BB3_1136; + + setp.gt.u32 %p1301, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r8021, 1; + @%p1301 bra $L__BB3_1136; + + st.global.u8 [%rd40], %rs1089; + add.s32 %r7382, %r7382, 1; + mov.u32 %r7388, 8; + mov.u16 %rs1089, 0; + mov.u32 %r8021, %r7540; + +$L__BB3_1136: + setp.lt.u32 %p1302, %r7542, 3; + mov.u32 %r7991, 0; + @%p1302 bra $L__BB3_1139; + + setp.lt.u32 %p1303, %r7542, 6; + mov.u32 %r7991, 1; + @%p1303 bra $L__BB3_1139; + + setp.lt.u32 %p1304, %r7542, 9; + setp.eq.s32 %p1305, %r7542, 11; + selp.b32 %r5992, 4, 5, %p1305; + setp.lt.u32 %p1306, %r7542, 11; + selp.b32 %r5993, 3, %r5992, %p1306; + selp.b32 %r7991, 2, %r5993, %p1304; + +$L__BB3_1139: + setp.eq.s32 %p1307, %r7991, 0; + @%p1307 bra $L__BB3_1167; + + add.s32 %r2916, %r7991, -1; + and.b32 %r2917, %r7991, 3; + setp.eq.s32 %p1308, %r2917, 0; + mov.u32 %r8001, %r7991; + mov.u32 %r8004, %r8021; + @%p1308 bra $L__BB3_1152; + + mov.u32 %r5995, 1; + shl.b32 %r5996, %r5995, %r2916; + and.b32 %r5997, %r5996, %r7543; + setp.ne.s32 %p1309, %r5997, 0; + selp.u32 %r5998, 1, 0, %p1309; + cvt.u32.u16 %r5999, %rs1089; + bfi.b32 %r6000, %r5999, %r5998, 1, 8; + cvt.u16.u32 %rs1089, %r6000; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1310, %r7388, 0; + mov.u32 %r8004, %r8021; + @%p1310 bra $L__BB3_1144; + + setp.gt.u32 %p1311, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r8004, %r5995; + @%p1311 bra $L__BB3_1144; + + add.s32 %r6004, %r7382, 17477; + cvt.u64.u32 %rd618, %r6004; + add.s64 %rd619, %rd618, %rd5; + add.s64 %rd620, %rd1, %rd619; + st.global.u8 [%rd620], %rs1089; + add.s32 %r7382, %r7382, 1; + mov.u32 %r7388, 8; + mov.u16 %rs1089, 0; + mov.u32 %r8004, %r8021; + +$L__BB3_1144: + setp.eq.s32 %p1312, %r2917, 1; + mov.u32 %r8021, %r8004; + mov.u32 %r8001, %r2916; + @%p1312 bra $L__BB3_1152; + + add.s32 %r8001, %r7991, -2; + mov.u32 %r6005, 1; + shl.b32 %r6006, %r6005, %r8001; + and.b32 %r6007, %r6006, %r7543; + setp.ne.s32 %p1313, %r6007, 0; + selp.u32 %r6008, 1, 0, %p1313; + cvt.u32.u16 %r6009, %rs1089; + bfi.b32 %r6010, %r6009, %r6008, 1, 8; + cvt.u16.u32 %rs1089, %r6010; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1314, %r7388, 0; + mov.u32 %r7995, %r8004; + @%p1314 bra $L__BB3_1148; + + setp.gt.u32 %p1315, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r7995, %r6005; + @%p1315 bra $L__BB3_1148; + + add.s32 %r6013, %r7382, 17477; + cvt.u64.u32 %rd621, %r6013; + add.s64 %rd622, %rd621, %rd5; + add.s64 %rd623, %rd1, %rd622; + and.b16 %rs903, %rs1089, 255; + st.global.u8 [%rd623], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1316, %rs903, 255; + selp.b32 %r7388, 7, 8, %p1316; + mov.u16 %rs1089, 0; + mov.u32 %r7995, %r8004; + +$L__BB3_1148: + setp.eq.s32 %p1317, %r2917, 2; + mov.u32 %r8021, %r7995; + mov.u32 %r8004, %r7995; + @%p1317 bra $L__BB3_1152; + + add.s32 %r8001, %r7991, -3; + mov.u32 %r6014, 1; + shl.b32 %r6015, %r6014, %r8001; + and.b32 %r6016, %r6015, %r7543; + setp.ne.s32 %p1318, %r6016, 0; + selp.u32 %r6017, 1, 0, %p1318; + cvt.u32.u16 %r6018, %rs1089; + bfi.b32 %r6019, %r6018, %r6017, 1, 8; + cvt.u16.u32 %rs1089, %r6019; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1319, %r7388, 0; + mov.u32 %r8021, %r7995; + mov.u32 %r8004, %r7995; + @%p1319 bra $L__BB3_1152; + + setp.gt.u32 %p1320, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r8021, %r6014; + mov.u32 %r8004, %r6014; + @%p1320 bra $L__BB3_1152; + + add.s32 %r6024, %r7382, 17477; + cvt.u64.u32 %rd624, %r6024; + add.s64 %rd625, %rd624, %rd5; + add.s64 %rd626, %rd1, %rd625; + and.b16 %rs906, %rs1089, 255; + st.global.u8 [%rd626], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1321, %rs906, 255; + selp.b32 %r7388, 7, 8, %p1321; + mov.u16 %rs1089, 0; + mov.u32 %r8021, %r7995; + mov.u32 %r8004, %r7995; + +$L__BB3_1152: + setp.lt.u32 %p1322, %r2916, 3; + @%p1322 bra $L__BB3_1167; + + mov.u32 %r8021, %r8004; + +$L__BB3_1154: + add.s32 %r6025, %r8001, -1; + mov.u32 %r6026, 1; + shl.b32 %r6027, %r6026, %r6025; + and.b32 %r6028, %r6027, %r7543; + setp.ne.s32 %p1323, %r6028, 0; + selp.u32 %r6029, 1, 0, %p1323; + cvt.u32.u16 %r6030, %rs1089; + bfi.b32 %r8010, %r6030, %r6029, 1, 8; + add.s32 %r8011, %r7388, -1; + setp.ne.s32 %p1324, %r8011, 0; + mov.u32 %r8009, %r8021; + @%p1324 bra $L__BB3_1157; + + setp.gt.u32 %p1325, %r7382, 191; + mov.u32 %r8011, 0; + mov.u32 %r8009, %r6026; + @%p1325 bra $L__BB3_1157; + + cvt.u16.u32 %rs907, %r8010; + and.b16 %rs908, %rs907, 255; + add.s32 %r6034, %r7382, 17477; + cvt.u64.u32 %rd627, %r6034; + add.s64 %rd628, %rd627, %rd5; + add.s64 %rd629, %rd1, %rd628; + st.global.u8 [%rd629], %rs907; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1326, %rs908, 255; + selp.b32 %r8011, 7, 8, %p1326; + mov.u32 %r8010, 0; + mov.u32 %r8009, %r8021; + +$L__BB3_1157: + add.s32 %r6035, %r8001, -2; + shl.b32 %r6037, %r6026, %r6035; + and.b32 %r6038, %r6037, %r7543; + setp.ne.s32 %p1327, %r6038, 0; + and.b32 %r6039, %r8010, 127; + selp.u32 %r6040, 1, 0, %p1327; + bfi.b32 %r8014, %r6039, %r6040, 1, 7; + add.s32 %r8015, %r8011, -1; + setp.ne.s32 %p1328, %r8015, 0; + mov.u32 %r8013, %r8009; + @%p1328 bra $L__BB3_1160; + + setp.gt.u32 %p1329, %r7382, 191; + mov.u32 %r8015, 0; + mov.u32 %r8013, 1; + @%p1329 bra $L__BB3_1160; + + cvt.u16.u32 %rs909, %r8014; + and.b16 %rs910, %rs909, 255; + add.s32 %r6044, %r7382, 17477; + cvt.u64.u32 %rd630, %r6044; + add.s64 %rd631, %rd630, %rd5; + add.s64 %rd632, %rd1, %rd631; + st.global.u8 [%rd632], %rs909; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1330, %rs910, 255; + selp.b32 %r8015, 7, 8, %p1330; + mov.u32 %r8014, 0; + mov.u32 %r8013, %r8009; + +$L__BB3_1160: + add.s32 %r6045, %r8001, -3; + mov.u32 %r6046, 1; + shl.b32 %r6047, %r6046, %r6045; + and.b32 %r6048, %r6047, %r7543; + setp.ne.s32 %p1331, %r6048, 0; + and.b32 %r6049, %r8014, 127; + selp.u32 %r6050, 1, 0, %p1331; + bfi.b32 %r8018, %r6049, %r6050, 1, 7; + add.s32 %r8019, %r8015, -1; + setp.ne.s32 %p1332, %r8019, 0; + mov.u32 %r8017, %r8013; + @%p1332 bra $L__BB3_1163; + + setp.gt.u32 %p1333, %r7382, 191; + mov.u32 %r8019, 0; + mov.u32 %r8017, %r6046; + @%p1333 bra $L__BB3_1163; + + cvt.u16.u32 %rs911, %r8018; + and.b16 %rs912, %rs911, 255; + add.s32 %r6054, %r7382, 17477; + cvt.u64.u32 %rd633, %r6054; + add.s64 %rd634, %rd633, %rd5; + add.s64 %rd635, %rd1, %rd634; + st.global.u8 [%rd635], %rs911; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1334, %rs912, 255; + selp.b32 %r8019, 7, 8, %p1334; + mov.u32 %r8018, 0; + mov.u32 %r8017, %r8013; + +$L__BB3_1163: + add.s32 %r8001, %r8001, -4; + shl.b32 %r6056, %r6046, %r8001; + and.b32 %r6057, %r6056, %r7543; + setp.ne.s32 %p1335, %r6057, 0; + and.b32 %r6058, %r8018, 127; + selp.u32 %r6059, 1, 0, %p1335; + bfi.b32 %r6060, %r6058, %r6059, 1, 15; + cvt.u16.u32 %rs1089, %r6060; + add.s32 %r7388, %r8019, -1; + setp.ne.s32 %p1336, %r7388, 0; + mov.u32 %r8021, %r8017; + @%p1336 bra $L__BB3_1166; + + setp.gt.u32 %p1337, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r8021, 1; + @%p1337 bra $L__BB3_1166; + + add.s32 %r6063, %r7382, 17477; + cvt.u64.u32 %rd636, %r6063; + add.s64 %rd637, %rd636, %rd5; + add.s64 %rd638, %rd1, %rd637; + and.b16 %rs914, %rs1089, 255; + st.global.u8 [%rd638], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1338, %rs914, 255; + selp.b32 %r7388, 7, 8, %p1338; + mov.u16 %rs1089, 0; + mov.u32 %r8021, %r8017; + +$L__BB3_1166: + setp.ne.s32 %p1339, %r8001, 0; + @%p1339 bra $L__BB3_1154; + +$L__BB3_1167: + add.s32 %r6065, %r7542, -1; + setp.eq.s32 %p1340, %r7542, 0; + mov.u32 %r7543, 0; + selp.b32 %r7542, 0, %r6065, %p1340; + setp.lt.u32 %p1341, %r7542, 3; + mov.u32 %r8027, %r7543; + @%p1341 bra $L__BB3_1170; + + setp.lt.u32 %p1342, %r7542, 6; + mov.u32 %r8027, 1; + @%p1342 bra $L__BB3_1170; + + setp.lt.u32 %p1343, %r7542, 9; + setp.eq.s32 %p1344, %r7542, 11; + selp.b32 %r6067, 4, 5, %p1344; + setp.lt.u32 %p1345, %r7542, 11; + selp.b32 %r6068, 3, %r6067, %p1345; + selp.b32 %r8027, 2, %r6068, %p1343; + +$L__BB3_1170: + mov.u32 %r6070, 1; + shl.b32 %r7541, %r6070, %r8027; + mov.u32 %r7540, %r8021; + bra.uni $L__BB3_1179; + +$L__BB3_1171: + add.s32 %r7543, %r7543, 1; + setp.lt.u32 %p1346, %r7543, %r7541; + @%p1346 bra $L__BB3_1179; + + shl.b16 %rs915, %rs1089, 1; + or.b16 %rs1089, %rs915, 1; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1347, %r7388, 0; + mov.u32 %r8028, %r7540; + @%p1347 bra $L__BB3_1175; + bra.uni $L__BB3_1173; + +$L__BB3_1175: + add.s32 %r6074, %r7542, 1; + min.u32 %r7542, %r6074, 12; + setp.lt.u32 %p1350, %r7542, 3; + mov.u32 %r7543, 0; + mov.u32 %r8031, %r7543; + @%p1350 bra $L__BB3_1178; + + setp.lt.u32 %p1351, %r7542, 6; + mov.u32 %r8031, 1; + @%p1351 bra $L__BB3_1178; + + setp.lt.u32 %p1352, %r7542, 9; + setp.eq.s32 %p1353, %r7542, 11; + selp.b32 %r6076, 4, 5, %p1353; + setp.lt.u32 %p1354, %r7542, 11; + selp.b32 %r6077, 3, %r6076, %p1354; + selp.b32 %r8031, 2, %r6077, %p1352; + +$L__BB3_1178: + mov.u32 %r6079, 1; + shl.b32 %r7541, %r6079, %r8031; + mov.u32 %r7540, %r8028; + +$L__BB3_1179: + and.b16 %rs918, %rs395, 15; + cvt.u32.u16 %r3000, %rs918; + and.b32 %r6080, %r7964, 1; + setp.eq.b32 %p1355, %r6080, 1; + mov.pred %p1356, 0; + xor.pred %p1357, %p1355, %p1356; + not.pred %p1358, %p1357; + mov.u32 %r8048, %r7951; + @%p1358 bra $L__BB3_1186; + + and.b32 %r6081, %r3000, 1; + sub.s32 %r8038, %r2886, %r6081; + setp.eq.s32 %p1359, %r8038, 0; + mov.u32 %r8048, %r7951; + @%p1359 bra $L__BB3_1186; + + mov.u32 %r6082, -1; + shl.b32 %r6083, %r6082, %r8038; + not.b32 %r6084, %r6083; + and.b32 %r8039, %r7958, %r6084; + +$L__BB3_1182: + setp.gt.u32 %p1360, %r7925, 17476; + mov.u32 %r8048, 1; + @%p1360 bra $L__BB3_1186; + + sub.s32 %r6086, %r7924, %r7923; + min.u32 %r6087, %r6086, %r8038; + setp.eq.s32 %p1361, %r6087, 32; + mov.u32 %r6088, -1; + shl.b32 %r6089, %r6088, %r6087; + not.b32 %r6090, %r6089; + selp.b32 %r6091, -1, %r6090, %p1361; + and.b32 %r6092, %r6091, %r8039; + shl.b32 %r6093, %r6092, %r7923; + or.b32 %r7922, %r6093, %r7922; + add.s32 %r7923, %r6087, %r7923; + shr.u32 %r8039, %r8039, %r6087; + sub.s32 %r8038, %r8038, %r6087; + setp.lt.u32 %p1362, %r7923, %r7924; + @%p1362 bra $L__BB3_1185; + + cvt.u64.u32 %rd639, %r7925; + add.s64 %rd640, %rd639, %rd5; + add.s64 %rd641, %rd1, %rd640; + st.global.u8 [%rd641], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p1363, %r7922, 255; + selp.b32 %r7924, 7, 8, %p1363; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_1185: + setp.ne.s32 %p1364, %r8038, 0; + mov.u32 %r8048, %r7951; + @%p1364 bra $L__BB3_1182; + +$L__BB3_1186: + and.b32 %r3024, %r7964, 2; + setp.eq.s32 %p1365, %r3024, 0; + mov.u32 %r8063, %r8048; + @%p1365 bra $L__BB3_1193; + + shr.u32 %r6096, %r3000, 1; + and.b32 %r6097, %r6096, 1; + sub.s32 %r8053, %r2886, %r6097; + setp.eq.s32 %p1366, %r8053, 0; + mov.u32 %r8063, %r8048; + @%p1366 bra $L__BB3_1193; + + mov.u32 %r6098, -1; + shl.b32 %r6099, %r6098, %r8053; + not.b32 %r6100, %r6099; + and.b32 %r8054, %r7962, %r6100; + +$L__BB3_1189: + setp.gt.u32 %p1367, %r7925, 17476; + mov.u32 %r8063, 1; + @%p1367 bra $L__BB3_1193; + + sub.s32 %r6102, %r7924, %r7923; + min.u32 %r6103, %r6102, %r8053; + setp.eq.s32 %p1368, %r6103, 32; + mov.u32 %r6104, -1; + shl.b32 %r6105, %r6104, %r6103; + not.b32 %r6106, %r6105; + selp.b32 %r6107, -1, %r6106, %p1368; + and.b32 %r6108, %r6107, %r8054; + shl.b32 %r6109, %r6108, %r7923; + or.b32 %r7922, %r6109, %r7922; + add.s32 %r7923, %r6103, %r7923; + shr.u32 %r8054, %r8054, %r6103; + sub.s32 %r8053, %r8053, %r6103; + setp.lt.u32 %p1369, %r7923, %r7924; + @%p1369 bra $L__BB3_1192; + + cvt.u64.u32 %rd642, %r7925; + add.s64 %rd643, %rd642, %rd5; + add.s64 %rd644, %rd1, %rd643; + st.global.u8 [%rd644], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p1370, %r7922, 255; + selp.b32 %r7924, 7, 8, %p1370; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_1192: + setp.ne.s32 %p1371, %r8053, 0; + mov.u32 %r8063, %r8048; + @%p1371 bra $L__BB3_1189; + +$L__BB3_1193: + and.b32 %r3048, %r7964, 4; + setp.eq.s32 %p1372, %r3048, 0; + mov.u32 %r8078, %r8063; + @%p1372 bra $L__BB3_1200; + + shr.u32 %r6112, %r3000, 2; + and.b32 %r6113, %r6112, 1; + sub.s32 %r8068, %r2886, %r6113; + setp.eq.s32 %p1373, %r8068, 0; + mov.u32 %r8078, %r8063; + @%p1373 bra $L__BB3_1200; + + mov.u32 %r6114, -1; + shl.b32 %r6115, %r6114, %r8068; + not.b32 %r6116, %r6115; + and.b32 %r8069, %r7967, %r6116; + +$L__BB3_1196: + setp.gt.u32 %p1374, %r7925, 17476; + mov.u32 %r8078, 1; + @%p1374 bra $L__BB3_1200; + + sub.s32 %r6118, %r7924, %r7923; + min.u32 %r6119, %r6118, %r8068; + setp.eq.s32 %p1375, %r6119, 32; + mov.u32 %r6120, -1; + shl.b32 %r6121, %r6120, %r6119; + not.b32 %r6122, %r6121; + selp.b32 %r6123, -1, %r6122, %p1375; + and.b32 %r6124, %r6123, %r8069; + shl.b32 %r6125, %r6124, %r7923; + or.b32 %r7922, %r6125, %r7922; + add.s32 %r7923, %r6119, %r7923; + shr.u32 %r8069, %r8069, %r6119; + sub.s32 %r8068, %r8068, %r6119; + setp.lt.u32 %p1376, %r7923, %r7924; + @%p1376 bra $L__BB3_1199; + + cvt.u64.u32 %rd645, %r7925; + add.s64 %rd646, %rd645, %rd5; + add.s64 %rd647, %rd1, %rd646; + st.global.u8 [%rd647], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p1377, %r7922, 255; + selp.b32 %r7924, 7, 8, %p1377; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_1199: + setp.ne.s32 %p1378, %r8068, 0; + mov.u32 %r8078, %r8063; + @%p1378 bra $L__BB3_1196; + +$L__BB3_1200: + and.b32 %r3072, %r7964, 8; + setp.eq.s32 %p1379, %r3072, 0; + mov.u32 %r8093, %r8078; + @%p1379 bra $L__BB3_1207; + + shr.u32 %r6128, %r3000, 3; + sub.s32 %r8083, %r2886, %r6128; + setp.eq.s32 %p1380, %r8083, 0; + mov.u32 %r8093, %r8078; + @%p1380 bra $L__BB3_1207; + + mov.u32 %r6129, -1; + shl.b32 %r6130, %r6129, %r8083; + not.b32 %r6131, %r6130; + and.b32 %r8084, %r7972, %r6131; + +$L__BB3_1203: + setp.gt.u32 %p1381, %r7925, 17476; + mov.u32 %r8093, 1; + @%p1381 bra $L__BB3_1207; + + sub.s32 %r6133, %r7924, %r7923; + min.u32 %r6134, %r6133, %r8083; + setp.eq.s32 %p1382, %r6134, 32; + mov.u32 %r6135, -1; + shl.b32 %r6136, %r6135, %r6134; + not.b32 %r6137, %r6136; + selp.b32 %r6138, -1, %r6137, %p1382; + and.b32 %r6139, %r6138, %r8084; + shl.b32 %r6140, %r6139, %r7923; + or.b32 %r7922, %r6140, %r7922; + add.s32 %r7923, %r6134, %r7923; + shr.u32 %r8084, %r8084, %r6134; + sub.s32 %r8083, %r8083, %r6134; + setp.lt.u32 %p1383, %r7923, %r7924; + @%p1383 bra $L__BB3_1206; + + cvt.u64.u32 %rd648, %r7925; + add.s64 %rd649, %rd648, %rd5; + add.s64 %rd650, %rd1, %rd649; + st.global.u8 [%rd650], %r7922; + add.s32 %r7925, %r7925, 1; + setp.eq.s32 %p1384, %r7922, 255; + selp.b32 %r7924, 7, 8, %p1384; + mov.u32 %r7922, 0; + mov.u32 %r7923, %r7922; + +$L__BB3_1206: + setp.ne.s32 %p1385, %r8083, 0; + mov.u32 %r8093, %r8078; + @%p1385 bra $L__BB3_1203; + +$L__BB3_1207: + and.b32 %r6143, %r7961, 255; + and.b32 %r6144, %r7829, 255; + setp.lt.u32 %p1386, %r6143, %r6144; + cvt.u16.u32 %rs919, %r7961; + selp.b16 %rs920, %rs394, %rs919, %p1386; + st.shared.u8 [%r2835+1], %rs920; + ld.shared.u8 %rs1152, [%r2835+3]; + setp.gt.u16 %p1387, %rs392, %rs1152; + add.s32 %r6145, %r2563, 3; + selp.b32 %r6146, %r7797, %r6145, %p1387; + add.s32 %r6148, %r4924, %r6146; + ld.shared.u8 %rs1154, [%r6148]; + cvt.u16.u32 %rs1153, %r7971; + shr.u32 %r6149, %r3024, 1; + or.b32 %r6150, %r2839, %r6149; + st.shared.u8 [%r2835+2], %r7971; + st.shared.u8 [%r2837+1], %r6150; + ld.shared.u8 %rs1150, [%r2837+3]; + mul.wide.u16 %r6151, %rs1150, 4; + add.s32 %r6152, %r6151, %r2838; + shr.u32 %r6153, %r3072, 3; + cvt.u16.u32 %rs1151, %r6153; + st.shared.u8 [%r2837+2], %r6153; + shr.u32 %r6154, %r3072, 2; + shr.u32 %r6155, %r3048, 1; + or.b32 %r6156, %r6154, %r6155; + or.b32 %r7798, %r6156, %r6152; + mul.lo.s32 %r6157, %r2626, 6; + setp.gt.s32 %p1388, %r2626, 0; + selp.b32 %r6158, %r6157, 0, %p1388; + cvt.u64.u32 %rd651, %r6158; + add.s64 %rd41, %rd34, %rd651; + ld.global.u8 %rs422, [%rd41+1]; + add.s32 %r6159, %r6158, 2; + cvt.u64.u32 %rd652, %r6159; + add.s64 %rd653, %rd34, %rd652; + ld.global.u8 %rs423, [%rd653]; + ld.global.u8 %rs424, [%rd653+1]; + mul.lo.s32 %r6160, %r2887, 6; + setp.gt.s32 %p1389, %r2887, 0; + selp.b32 %r6161, %r6160, 0, %p1389; + cvt.u64.u32 %rd654, %r6161; + add.s64 %rd655, %rd34, %rd654; + ld.global.u8 %rs425, [%rd655]; + ld.global.u8 %rs426, [%rd655+1]; + add.s32 %r6162, %r6161, 2; + cvt.u64.u32 %rd656, %r6162; + add.s64 %rd657, %rd34, %rd656; + ld.global.u8 %rs427, [%rd657]; + ld.global.u8 %rs428, [%rd657+1]; + setp.eq.s16 %p1390, %rs422, 0; + mov.u32 %r8109, %r7987; + @%p1390 bra $L__BB3_1214; + + ld.global.u8 %r8099, [%rd41]; + cvt.u32.u16 %r8098, %rs422; + +$L__BB3_1209: + mov.u32 %r3099, %r8098; + setp.gt.u32 %p1391, %r7779, 2879; + mov.u32 %r8109, 1; + @%p1391 bra $L__BB3_1214; + + mov.u32 %r6164, 8; + sub.s32 %r6165, %r6164, %r7777; + sub.s32 %r6166, %r6165, %r7778; + min.u32 %r6167, %r6166, %r3099; + setp.eq.s32 %p1392, %r6167, 32; + mov.u32 %r6168, -1; + shl.b32 %r6169, %r6168, %r6167; + not.b32 %r6170, %r6169; + selp.b32 %r6171, -1, %r6170, %p1392; + and.b32 %r6172, %r6171, %r8099; + shl.b32 %r6173, %r6172, %r7778; + cvt.u16.u32 %rs921, %r6173; + or.b16 %rs1147, %rs1147, %rs921; + add.s32 %r7778, %r6167, %r7778; + sub.s32 %r8098, %r3099, %r6167; + shr.u32 %r8099, %r8099, %r6167; + setp.gt.u32 %p1393, %r6166, %r3099; + @%p1393 bra $L__BB3_1213; + + setp.ne.s32 %p1394, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs922, %rs1147, 255; + setp.ne.s16 %p1395, %rs922, 127; + and.pred %p1396, %p1394, %p1395; + @%p1396 bra $L__BB3_1213; + + mov.u32 %r6176, 20548; + sub.s32 %r6177, %r6176, %r7779; + cvt.u64.u32 %rd658, %r6177; + add.s64 %rd659, %rd658, %rd5; + add.s64 %rd660, %rd1, %rd659; + st.global.u8 [%rd660], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1397, %rs922, 143; + selp.u32 %r7777, 1, 0, %p1397; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_1213: + setp.ne.s32 %p1398, %r8098, 0; + mov.u32 %r8109, %r7987; + @%p1398 bra $L__BB3_1209; + +$L__BB3_1214: + setp.eq.s16 %p1399, %rs426, 0; + mov.u32 %r8121, %r8109; + @%p1399 bra $L__BB3_1221; + + cvt.u32.u16 %r6178, %rs425; + and.b32 %r8111, %r6178, 255; + cvt.u32.u16 %r6179, %rs426; + and.b32 %r8110, %r6179, 255; + +$L__BB3_1216: + mov.u32 %r3118, %r8110; + setp.gt.u32 %p1400, %r7779, 2879; + mov.u32 %r8121, 1; + @%p1400 bra $L__BB3_1221; + + mov.u32 %r6181, 8; + sub.s32 %r6182, %r6181, %r7777; + sub.s32 %r6183, %r6182, %r7778; + min.u32 %r6184, %r6183, %r3118; + setp.eq.s32 %p1401, %r6184, 32; + mov.u32 %r6185, -1; + shl.b32 %r6186, %r6185, %r6184; + not.b32 %r6187, %r6186; + selp.b32 %r6188, -1, %r6187, %p1401; + and.b32 %r6189, %r6188, %r8111; + shl.b32 %r6190, %r6189, %r7778; + cvt.u16.u32 %rs926, %r6190; + or.b16 %rs1147, %rs1147, %rs926; + add.s32 %r7778, %r6184, %r7778; + sub.s32 %r8110, %r3118, %r6184; + shr.u32 %r8111, %r8111, %r6184; + setp.gt.u32 %p1402, %r6183, %r3118; + @%p1402 bra $L__BB3_1220; + + setp.ne.s32 %p1403, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs927, %rs1147, 255; + setp.ne.s16 %p1404, %rs927, 127; + and.pred %p1405, %p1403, %p1404; + @%p1405 bra $L__BB3_1220; + + mov.u32 %r6193, 20548; + sub.s32 %r6194, %r6193, %r7779; + cvt.u64.u32 %rd661, %r6194; + add.s64 %rd662, %rd661, %rd5; + add.s64 %rd663, %rd1, %rd662; + st.global.u8 [%rd663], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1406, %rs927, 143; + selp.u32 %r7777, 1, 0, %p1406; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_1220: + setp.ne.s32 %p1407, %r8110, 0; + mov.u32 %r8121, %r8109; + @%p1407 bra $L__BB3_1216; + +$L__BB3_1221: + setp.eq.s16 %p1408, %rs424, 0; + mov.u32 %r8133, %r8121; + @%p1408 bra $L__BB3_1228; + + cvt.u32.u16 %r6195, %rs424; + and.b32 %r8122, %r6195, 255; + cvt.u32.u16 %r6196, %rs423; + and.b32 %r8123, %r6196, 255; + +$L__BB3_1223: + mov.u32 %r3137, %r8122; + setp.gt.u32 %p1409, %r7779, 2879; + mov.u32 %r8133, 1; + @%p1409 bra $L__BB3_1228; + + mov.u32 %r6198, 8; + sub.s32 %r6199, %r6198, %r7777; + sub.s32 %r6200, %r6199, %r7778; + min.u32 %r6201, %r6200, %r3137; + setp.eq.s32 %p1410, %r6201, 32; + mov.u32 %r6202, -1; + shl.b32 %r6203, %r6202, %r6201; + not.b32 %r6204, %r6203; + selp.b32 %r6205, -1, %r6204, %p1410; + and.b32 %r6206, %r6205, %r8123; + shl.b32 %r6207, %r6206, %r7778; + cvt.u16.u32 %rs931, %r6207; + or.b16 %rs1147, %rs1147, %rs931; + add.s32 %r7778, %r6201, %r7778; + sub.s32 %r8122, %r3137, %r6201; + shr.u32 %r8123, %r8123, %r6201; + setp.gt.u32 %p1411, %r6200, %r3137; + @%p1411 bra $L__BB3_1227; + + setp.ne.s32 %p1412, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs932, %rs1147, 255; + setp.ne.s16 %p1413, %rs932, 127; + and.pred %p1414, %p1412, %p1413; + @%p1414 bra $L__BB3_1227; + + mov.u32 %r6210, 20548; + sub.s32 %r6211, %r6210, %r7779; + cvt.u64.u32 %rd664, %r6211; + add.s64 %rd665, %rd664, %rd5; + add.s64 %rd666, %rd1, %rd665; + st.global.u8 [%rd666], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1415, %rs932, 143; + selp.u32 %r7777, 1, 0, %p1415; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_1227: + setp.ne.s32 %p1416, %r8122, 0; + mov.u32 %r8133, %r8121; + @%p1416 bra $L__BB3_1223; + +$L__BB3_1228: + setp.eq.s16 %p1417, %rs428, 0; + mov.u32 %r7776, %r8133; + @%p1417 bra $L__BB3_1235; + + cvt.u32.u16 %r6212, %rs427; + and.b32 %r8135, %r6212, 255; + cvt.u32.u16 %r6213, %rs428; + and.b32 %r8134, %r6213, 255; + +$L__BB3_1230: + mov.u32 %r3156, %r8134; + setp.gt.u32 %p1418, %r7779, 2879; + mov.u32 %r7776, 1; + @%p1418 bra $L__BB3_1235; + + mov.u32 %r6215, 8; + sub.s32 %r6216, %r6215, %r7777; + sub.s32 %r6217, %r6216, %r7778; + min.u32 %r6218, %r6217, %r3156; + setp.eq.s32 %p1419, %r6218, 32; + mov.u32 %r6219, -1; + shl.b32 %r6220, %r6219, %r6218; + not.b32 %r6221, %r6220; + selp.b32 %r6222, -1, %r6221, %p1419; + and.b32 %r6223, %r6222, %r8135; + shl.b32 %r6224, %r6223, %r7778; + cvt.u16.u32 %rs936, %r6224; + or.b16 %rs1147, %rs1147, %rs936; + add.s32 %r7778, %r6218, %r7778; + sub.s32 %r8134, %r3156, %r6218; + shr.u32 %r8135, %r8135, %r6218; + setp.gt.u32 %p1420, %r6217, %r3156; + @%p1420 bra $L__BB3_1234; + + setp.ne.s32 %p1421, %r7777, 0; + mov.u32 %r7777, 0; + and.b16 %rs937, %rs1147, 255; + setp.ne.s16 %p1422, %rs937, 127; + and.pred %p1423, %p1421, %p1422; + @%p1423 bra $L__BB3_1234; + + mov.u32 %r6227, 20548; + sub.s32 %r6228, %r6227, %r7779; + cvt.u64.u32 %rd667, %r6228; + add.s64 %rd668, %rd667, %rd5; + add.s64 %rd669, %rd1, %rd668; + st.global.u8 [%rd669], %rs1147; + add.s32 %r7779, %r7779, 1; + setp.gt.u16 %p1424, %rs937, 143; + selp.u32 %r7777, 1, 0, %p1424; + mov.u32 %r7778, 0; + mov.u16 %rs1147, 0; + +$L__BB3_1234: + setp.ne.s32 %p1425, %r8134, 0; + mov.u32 %r7776, %r8133; + @%p1425 bra $L__BB3_1230; + +$L__BB3_1235: + add.s64 %rd689, %rd689, 16; + add.s32 %r7796, %r7796, 4; + setp.lt.u32 %p1426, %r7796, 64; + @%p1426 bra $L__BB3_1005; + + add.s32 %r7780, %r7780, 2; + setp.lt.u32 %p1427, %r7780, 64; + add.s64 %rd688, %rd688, 1; + @%p1427 bra $L__BB3_1004; + + setp.eq.s32 %p1428, %r7543, 0; + mov.u32 %r8146, %r7540; + @%p1428 bra $L__BB3_1241; + + shl.b16 %rs940, %rs1089, 1; + or.b16 %rs1089, %rs940, 1; + add.s32 %r7388, %r7388, -1; + setp.ne.s32 %p1429, %r7388, 0; + mov.u32 %r8146, %r7540; + @%p1429 bra $L__BB3_1241; + + setp.gt.u32 %p1430, %r7382, 191; + mov.u32 %r7388, 0; + mov.u32 %r8146, 1; + @%p1430 bra $L__BB3_1241; + + add.s32 %r6231, %r7382, 17477; + cvt.u64.u32 %rd670, %r6231; + add.s64 %rd671, %rd670, %rd5; + add.s64 %rd672, %rd1, %rd671; + and.b16 %rs942, %rs1089, 255; + st.global.u8 [%rd672], %rs1089; + add.s32 %r7382, %r7382, 1; + setp.eq.s16 %p1431, %rs942, 255; + selp.b32 %r7388, 7, 8, %p1431; + mov.u16 %rs1089, 0; + mov.u32 %r8146, %r7540; + +$L__BB3_1241: + cvt.u32.u16 %r6232, %rs1089; + and.b32 %r6233, %r6232, 255; + shl.b32 %r6234, %r6233, %r7388; + cvt.u16.u32 %rs447, %r6234; + mov.u32 %r6235, -1; + shl.b32 %r6236, %r6235, %r7778; + not.b32 %r6237, %r6236; + mov.u32 %r6238, 255; + and.b32 %r6239, %r6237, 255; + setp.eq.s32 %p1432, %r7778, 0; + selp.b32 %r3181, 0, %r6239, %p1432; + shl.b32 %r3182, %r6238, %r7388; + and.b32 %r6240, %r3182, 255; + or.b32 %r6241, %r6240, %r3181; + setp.eq.s32 %p1433, %r6241, 0; + mov.u32 %r8149, %r8146; + mov.u32 %r8151, %r7776; + @%p1433 bra $L__BB3_1247; + + or.b16 %rs448, %rs1147, %rs447; + and.b16 %rs943, %rs448, 255; + xor.b16 %rs944, %rs448, %rs447; + cvt.u32.u16 %r6242, %rs944; + and.b32 %r6243, %r3182, %r6242; + and.b32 %r6244, %r6243, 255; + xor.b16 %rs945, %rs448, %rs1147; + cvt.u32.u16 %r6245, %rs945; + and.b32 %r6246, %r3181, %r6245; + or.b32 %r6247, %r6244, %r6246; + setp.eq.s32 %p1434, %r6247, 0; + setp.ne.s16 %p1435, %rs943, 255; + and.pred %p1436, %p1435, %p1434; + setp.gt.u32 %p1437, %r7779, 1; + and.pred %p1438, %p1437, %p1436; + add.s32 %r6248, %r7382, 17477; + cvt.u64.u32 %rd673, %r6248; + add.s64 %rd674, %rd673, %rd5; + add.s64 %rd44, %rd1, %rd674; + @%p1438 bra $L__BB3_1245; + bra.uni $L__BB3_1243; + +$L__BB3_1245: + setp.gt.u32 %p1442, %r7382, 191; + mov.u32 %r8149, 1; + mov.u32 %r8151, %r7776; + @%p1442 bra $L__BB3_1247; + + st.global.u8 [%rd44], %rs448; + add.s32 %r7382, %r7382, 1; + mov.u32 %r8149, %r8146; + mov.u32 %r8151, %r7776; + bra.uni $L__BB3_1247; + +$L__BB3_667: + mov.u32 %r4833, 0; + st.global.u32 [%rd6], %r4833; + st.global.u32 [%rd6+4], %r4833; + st.global.u32 [%rd6+8], %r4833; + st.global.u32 [%rd6+12], %r4833; + st.global.u32 [%rd6+16], %r3202; + st.global.u32 [%rd6+20], %r4833; + st.global.u32 [%rd6+24], %r4833; + st.global.u32 [%rd6+28], %r4833; + bra.uni $L__BB3_1261; + +$L__BB3_23: + mov.u32 %r6296, 0; + mov.u32 %r3253, 31; + sub.s32 %r27, %r3253, %r3202; + mov.u16 %rs452, 255; + st.global.u8 [%rd7], %rs452; + add.s32 %r3254, %r3200, 1; + shr.u32 %r28, %r3254, 1; + add.s32 %r3255, %r28, 2; + min.u32 %r29, %r3255, 513; + mov.u32 %r3256, -3; + sub.s32 %r3257, %r3256, %r28; + max.u32 %r3258, %r3257, -514; + mov.u32 %r3259, -2; + sub.s32 %r3260, %r3259, %r3258; + and.b32 %r6298, %r29, 3; + setp.lt.u32 %p33, %r3260, 3; + @%p33 bra $L__BB3_26; + + sub.s32 %r6295, %r29, %r6298; + mov.u32 %r6296, 0; + +$L__BB3_25: + mov.u32 %r3262, _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val; + add.s32 %r3263, %r3262, %r6296; + mov.u16 %rs453, 0; + st.shared.u8 [%r3263], %rs453; + mov.u32 %r3264, _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val; + add.s32 %r3265, %r3264, %r6296; + st.shared.u8 [%r3265], %rs453; + st.shared.u8 [%r3263+1], %rs453; + st.shared.u8 [%r3265+1], %rs453; + st.shared.u8 [%r3263+2], %rs453; + st.shared.u8 [%r3265+2], %rs453; + st.shared.u8 [%r3263+3], %rs453; + st.shared.u8 [%r3265+3], %rs453; + add.s32 %r6296, %r6296, 4; + add.s32 %r6295, %r6295, -4; + setp.ne.s32 %p34, %r6295, 0; + @%p34 bra $L__BB3_25; + +$L__BB3_26: + setp.eq.s32 %p35, %r6298, 0; + @%p35 bra $L__BB3_29; + + mov.u32 %r3266, _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val; + mov.u32 %r3268, _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val; + +$L__BB3_28: + .pragma "nounroll"; + add.s32 %r3267, %r3266, %r6296; + mov.u16 %rs454, 0; + st.shared.u8 [%r3267], %rs454; + add.s32 %r3269, %r3268, %r6296; + st.shared.u8 [%r3269], %rs454; + add.s32 %r6296, %r6296, 1; + add.s32 %r6298, %r6298, -1; + setp.ne.s32 %p36, %r6298, 0; + @%p36 bra $L__BB3_28; + +$L__BB3_29: + mov.u32 %r7016, 0; + mov.u32 %r6829, 1; + mov.u16 %rs956, 0; + mov.u32 %r7017, 8; + mov.u16 %rs1025, 15; + mov.u32 %r6830, 4; + mov.u32 %r7018, %r7016; + mov.u32 %r7019, %r7016; + mov.u32 %r7050, %r7016; + mov.u32 %r6831, %r6829; + mov.u32 %r6832, %r7016; + mov.u32 %r6381, %r7016; + mov.u32 %r6390, %r7017; + mov.u32 %r6595, %r7016; + mov.u32 %r6596, %r7016; + mov.u32 %r6597, %r6829; + mov.u32 %r6598, %r7016; + @%p2 bra $L__BB3_397; + + ld.param.u64 %rd682, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_2]; + cvta.to.global.u64 %rd8, %rd682; + cvta.to.global.u64 %rd9, %rd48; + mov.u32 %r3302, 0; + mov.u32 %r6390, 8; + mov.u32 %r6597, 1; + mov.u32 %r6830, 4; + mov.u16 %rs1025, 15; + mov.u16 %rs956, 0; + mov.u32 %r6299, %r3302; + mov.u32 %r6598, %r3302; + mov.u32 %r6596, %r3302; + mov.u32 %r6595, %r3302; + mov.u32 %r6381, %r3302; + mov.u32 %r6832, %r3302; + mov.u32 %r6831, %r6597; + mov.u32 %r6829, %r6597; + mov.u32 %r7050, %r3302; + mov.u32 %r7019, %r3302; + mov.u32 %r7018, %r3302; + mov.u32 %r7017, %r6390; + mov.u32 %r7016, %r3302; + mov.u32 %r6315, %r3302; + mov.u32 %r6316, %r3302; + bra.uni $L__BB3_31; + +$L__BB3_273: + setp.gt.u32 %p304, %r6381, 191; + mov.u32 %r6694, 1; + mov.u32 %r6390, 0; + @%p304 bra $L__BB3_275; + + st.global.u8 [%rd14], %rs956; + add.s32 %r6381, %r6381, 1; + mov.u16 %rs956, 0; + mov.u32 %r6390, 8; + mov.u32 %r6694, %r6598; + bra.uni $L__BB3_275; + +$L__BB3_197: + setp.gt.u32 %p217, %r6381, 191; + mov.u32 %r6584, 1; + mov.u32 %r6390, 0; + @%p217 bra $L__BB3_199; + + st.global.u8 [%rd13], %rs956; + add.s32 %r6381, %r6381, 1; + mov.u16 %rs956, 0; + mov.u32 %r6390, 8; + mov.u32 %r6584, %r6598; + bra.uni $L__BB3_199; + +$L__BB3_312: + setp.gt.u32 %p351, %r6381, 191; + mov.u32 %r6701, 1; + mov.u32 %r6390, 0; + @%p351 bra $L__BB3_314; + + and.b16 %rs561, %rs956, 255; + st.global.u8 [%rd14], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p352, %rs561, 255; + selp.b32 %r6390, 7, 8, %p352; + mov.u16 %rs956, 0; + mov.u32 %r6701, %r6598; + bra.uni $L__BB3_314; + +$L__BB3_236: + setp.gt.u32 %p264, %r6381, 191; + mov.u32 %r6591, 1; + mov.u32 %r6390, 0; + @%p264 bra $L__BB3_238; + + and.b16 %rs541, %rs956, 255; + st.global.u8 [%rd13], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p265, %rs541, 255; + selp.b32 %r6390, 7, 8, %p265; + mov.u16 %rs956, 0; + mov.u32 %r6591, %r6598; + bra.uni $L__BB3_238; + +$L__BB3_31: + cvt.u64.u32 %rd67, %r6316; + add.s64 %rd68, %rd67, %rd4; + shl.b64 %rd69, %rd68, 2; + add.s64 %rd70, %rd3, %rd69; + ld.global.u32 %r59, [%rd70]; + setp.eq.s32 %p38, %r59, 0; + mov.u32 %r6317, %r3302; + @%p38 bra $L__BB3_33; + + and.b32 %r3304, %r59, -2147483648; + abs.s32 %r3305, %r59; + shl.b32 %r3306, %r3305, %r27; + or.b32 %r6317, %r3306, %r3304; + +$L__BB3_33: + shl.b32 %r3310, %r6317, 1; + shr.u32 %r3311, %r3310, %r27; + and.b32 %r62, %r3311, -2; + setp.eq.s32 %p39, %r62, 0; + mov.u32 %r6321, 0; + mov.u32 %r6318, %r6321; + mov.u32 %r6319, %r6321; + mov.u32 %r6325, %r6321; + @%p39 bra $L__BB3_35; + + add.s32 %r3313, %r62, -1; + clz.b32 %r3314, %r3313; + mov.u32 %r3315, 32; + sub.s32 %r6318, %r3315, %r3314; + shr.u32 %r3316, %r6317, 31; + add.s32 %r3317, %r3316, %r62; + add.s32 %r6319, %r3317, -2; + mov.u32 %r6325, 1; + +$L__BB3_35: + setp.lt.u32 %p40, %r3201, 2; + @%p40 bra $L__BB3_38; + + add.s32 %r3320, %r6316, %r3198; + cvt.u64.u32 %rd71, %r3320; + add.s64 %rd72, %rd71, %rd4; + shl.b64 %rd73, %rd72, 2; + add.s64 %rd74, %rd3, %rd73; + ld.global.u32 %r68, [%rd74]; + setp.eq.s32 %p41, %r68, 0; + @%p41 bra $L__BB3_38; + + and.b32 %r3321, %r68, -2147483648; + abs.s32 %r3322, %r68; + shl.b32 %r3323, %r3322, %r27; + or.b32 %r6321, %r3323, %r3321; + +$L__BB3_38: + shl.b32 %r3326, %r6321, 1; + shr.u32 %r3327, %r3326, %r27; + and.b32 %r71, %r3327, -2; + setp.eq.s32 %p42, %r71, 0; + mov.u32 %r6336, 0; + mov.u32 %r6322, %r6336; + mov.u32 %r6323, %r6336; + mov.u32 %r6341, %r6318; + @%p42 bra $L__BB3_40; + + or.b32 %r6325, %r6325, 2; + add.s32 %r3328, %r71, -1; + clz.b32 %r3329, %r3328; + mov.u32 %r3330, 32; + sub.s32 %r6322, %r3330, %r3329; + max.s32 %r6341, %r6318, %r6322; + shr.u32 %r3331, %r6321, 31; + add.s32 %r3332, %r3331, %r71; + add.s32 %r6323, %r3332, -2; + +$L__BB3_40: + add.s32 %r6340, %r6316, 1; + add.s32 %r3337, %r6299, 1; + setp.ge.u32 %p43, %r3337, %r3200; + mov.u32 %r6337, %r6336; + mov.u32 %r6338, %r6336; + mov.u32 %r6339, %r6336; + @%p43 bra $L__BB3_51; + + cvt.u64.u32 %rd75, %r6340; + add.s64 %rd76, %rd75, %rd4; + shl.b64 %rd77, %rd76, 2; + add.s64 %rd78, %rd3, %rd77; + ld.global.u32 %r81, [%rd78]; + setp.eq.s32 %p44, %r81, 0; + mov.u32 %r6337, 0; + mov.u32 %r6326, %r6337; + @%p44 bra $L__BB3_43; + + and.b32 %r3339, %r81, -2147483648; + abs.s32 %r3340, %r81; + shl.b32 %r3341, %r3340, %r27; + or.b32 %r6326, %r3341, %r3339; + +$L__BB3_43: + shl.b32 %r3344, %r6326, 1; + shr.u32 %r3345, %r3344, %r27; + and.b32 %r84, %r3345, -2; + setp.eq.s32 %p45, %r84, 0; + mov.u32 %r6339, %r6337; + @%p45 bra $L__BB3_45; + + or.b32 %r6325, %r6325, 4; + add.s32 %r3346, %r84, -1; + clz.b32 %r3347, %r3346; + mov.u32 %r3348, 32; + sub.s32 %r6337, %r3348, %r3347; + max.s32 %r6341, %r6341, %r6337; + shr.u32 %r3349, %r6326, 31; + add.s32 %r3350, %r3349, %r84; + add.s32 %r6339, %r3350, -2; + +$L__BB3_45: + mov.u32 %r6336, 0; + mov.u32 %r6331, %r6336; + @%p40 bra $L__BB3_48; + + add.s32 %r3353, %r6340, %r3198; + cvt.u64.u32 %rd79, %r3353; + add.s64 %rd80, %rd79, %rd4; + shl.b64 %rd81, %rd80, 2; + add.s64 %rd82, %rd3, %rd81; + ld.global.u32 %r93, [%rd82]; + setp.eq.s32 %p47, %r93, 0; + @%p47 bra $L__BB3_48; + + and.b32 %r3354, %r93, -2147483648; + abs.s32 %r3355, %r93; + shl.b32 %r3356, %r3355, %r27; + or.b32 %r6331, %r3356, %r3354; + +$L__BB3_48: + shl.b32 %r3359, %r6331, 1; + shr.u32 %r3360, %r3359, %r27; + and.b32 %r96, %r3360, -2; + setp.eq.s32 %p48, %r96, 0; + mov.u32 %r6338, %r6336; + @%p48 bra $L__BB3_50; + + or.b32 %r6325, %r6325, 8; + add.s32 %r3361, %r96, -1; + clz.b32 %r3362, %r3361; + mov.u32 %r3363, 32; + sub.s32 %r6336, %r3363, %r3362; + max.s32 %r6341, %r6341, %r6336; + shr.u32 %r3364, %r6331, 31; + add.s32 %r3365, %r3364, %r96; + add.s32 %r6338, %r3365, -2; + +$L__BB3_50: + add.s32 %r6340, %r6316, 2; + +$L__BB3_51: + mov.u32 %r6316, %r6340; + add.s32 %r3367, %r6341, -1; + setp.lt.s32 %p49, %r6341, 2; + setp.gt.s32 %p50, %r6341, 1; + selp.b32 %r113, %r3367, 0, %p50; + mov.u32 %r6343, 0; + @%p49 bra $L__BB3_53; + + setp.eq.s32 %p51, %r6318, %r6341; + selp.u32 %r3368, 1, 0, %p51; + setp.eq.s32 %p52, %r6322, %r6341; + selp.u32 %r3369, -1, 0, %p52; + bfi.b32 %r3370, %r3369, %r3368, 1, 1; + setp.eq.s32 %p53, %r6337, %r6341; + selp.u16 %rs459, 1, 0, %p53; + mul.wide.u16 %r3371, %rs459, 4; + or.b32 %r3372, %r3370, %r3371; + setp.eq.s32 %p54, %r6336, %r6341; + selp.u16 %rs460, 1, 0, %p54; + mul.wide.u16 %r3373, %rs460, 8; + or.b32 %r6343, %r3372, %r3373; + +$L__BB3_53: + shr.u32 %r3374, %r6299, 1; + mov.u32 %r3375, _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val; + add.s32 %r116, %r3375, %r3374; + ld.shared.u8 %rs461, [%r116]; + cvt.u32.u16 %r3376, %rs461; + and.b32 %r3377, %r3376, 255; + and.b32 %r3378, %r6322, 255; + setp.lt.u32 %p55, %r3378, %r3377; + cvt.u16.u32 %rs462, %r6322; + selp.b16 %rs463, %rs461, %rs462, %p55; + st.shared.u8 [%r116], %rs463; + cvt.u16.u32 %rs3, %r6336; + st.shared.u8 [%r116+1], %rs3; + and.b32 %r117, %r6325, 2; + cvt.u16.u32 %rs464, %r117; + shr.u16 %rs465, %rs464, 1; + mov.u32 %r3379, _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val; + add.s32 %r118, %r3379, %r3374; + ld.shared.u8 %rs466, [%r118]; + or.b16 %rs467, %rs466, %rs465; + st.shared.u8 [%r118], %rs467; + and.b32 %r119, %r6325, 8; + shr.u32 %r120, %r119, 3; + st.shared.u8 [%r118+1], %r120; + shl.b32 %r3380, %r6325, 4; + shl.b32 %r3381, %r6315, 8; + or.b32 %r3382, %r3380, %r3381; + or.b32 %r3383, %r3382, %r6343; + mul.wide.u32 %rd83, %r3383, 2; + add.s64 %rd84, %rd8, %rd83; + ld.global.u16 %rs4, [%rd84]; + shr.u16 %rs468, %rs4, 4; + and.b16 %rs5, %rs468, 7; + setp.eq.s16 %p56, %rs5, 0; + mov.u32 %r6355, %r6832; + @%p56 bra $L__BB3_60; + + cvt.u32.u16 %r6344, %rs5; + shr.u16 %rs469, %rs4, 8; + cvt.u32.u16 %r6345, %rs469; + +$L__BB3_55: + mov.u32 %r123, %r6344; + setp.gt.u32 %p57, %r6829, 2879; + mov.u32 %r6355, 1; + @%p57 bra $L__BB3_60; + + mov.u32 %r3385, 8; + sub.s32 %r3386, %r3385, %r6831; + sub.s32 %r3387, %r3386, %r6830; + min.u32 %r3388, %r3387, %r123; + setp.eq.s32 %p58, %r3388, 32; + mov.u32 %r3389, -1; + shl.b32 %r3390, %r3389, %r3388; + not.b32 %r3391, %r3390; + selp.b32 %r3392, -1, %r3391, %p58; + and.b32 %r3393, %r3392, %r6345; + shl.b32 %r3394, %r3393, %r6830; + cvt.u16.u32 %rs470, %r3394; + or.b16 %rs1025, %rs1025, %rs470; + add.s32 %r6830, %r3388, %r6830; + sub.s32 %r6344, %r123, %r3388; + shr.u32 %r6345, %r6345, %r3388; + setp.gt.u32 %p59, %r3387, %r123; + @%p59 bra $L__BB3_59; + + setp.ne.s32 %p60, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs471, %rs1025, 255; + setp.ne.s16 %p61, %rs471, 127; + and.pred %p62, %p60, %p61; + @%p62 bra $L__BB3_59; + + mov.u32 %r3397, 20548; + sub.s32 %r3398, %r3397, %r6829; + cvt.u64.u32 %rd85, %r3398; + add.s64 %rd86, %rd85, %rd5; + add.s64 %rd87, %rd1, %rd86; + st.global.u8 [%rd87], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p63, %rs471, 143; + selp.u32 %r6831, 1, 0, %p63; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_59: + setp.ne.s32 %p64, %r6344, 0; + mov.u32 %r6355, %r6832; + @%p64 bra $L__BB3_55; + +$L__BB3_60: + setp.ne.s32 %p65, %r6315, 0; + @%p65 bra $L__BB3_108; + + setp.eq.s32 %p66, %r6325, 0; + add.s32 %r3399, %r6381, 17477; + cvt.u64.u32 %rd88, %r3399; + add.s64 %rd89, %rd88, %rd5; + add.s64 %rd10, %rd1, %rd89; + @%p66 bra $L__BB3_100; + + shl.b16 %rs956, %rs956, 1; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p67, %r6390, 0; + mov.u32 %r6391, %r6598; + @%p67 bra $L__BB3_65; + bra.uni $L__BB3_63; + +$L__BB3_65: + setp.lt.u32 %p69, %r6596, 3; + mov.u32 %r6359, 0; + @%p69 bra $L__BB3_68; + + setp.lt.u32 %p70, %r6596, 6; + mov.u32 %r6359, 1; + @%p70 bra $L__BB3_68; + + setp.lt.u32 %p71, %r6596, 9; + setp.eq.s32 %p72, %r6596, 11; + selp.b32 %r3405, 4, 5, %p72; + setp.lt.u32 %p73, %r6596, 11; + selp.b32 %r3406, 3, %r3405, %p73; + selp.b32 %r6359, 2, %r3406, %p71; + +$L__BB3_68: + setp.eq.s32 %p74, %r6359, 0; + @%p74 bra $L__BB3_96; + + add.s32 %r147, %r6359, -1; + and.b32 %r148, %r6359, 3; + setp.eq.s32 %p75, %r148, 0; + mov.u32 %r6369, %r6359; + mov.u32 %r6370, %r6391; + @%p75 bra $L__BB3_81; + + mov.u32 %r3408, 1; + shl.b32 %r3409, %r3408, %r147; + and.b32 %r3410, %r3409, %r6595; + setp.ne.s32 %p76, %r3410, 0; + selp.u32 %r3411, 1, 0, %p76; + cvt.u32.u16 %r3412, %rs956; + bfi.b32 %r3413, %r3412, %r3411, 1, 8; + cvt.u16.u32 %rs956, %r3413; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p77, %r6390, 0; + mov.u32 %r6370, %r6391; + @%p77 bra $L__BB3_73; + + setp.gt.u32 %p78, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r6370, %r3408; + @%p78 bra $L__BB3_73; + + add.s32 %r3417, %r6381, 17477; + cvt.u64.u32 %rd90, %r3417; + add.s64 %rd91, %rd90, %rd5; + add.s64 %rd92, %rd1, %rd91; + st.global.u8 [%rd92], %rs956; + add.s32 %r6381, %r6381, 1; + mov.u16 %rs956, 0; + mov.u32 %r6390, 8; + mov.u32 %r6370, %r6391; + +$L__BB3_73: + setp.eq.s32 %p79, %r148, 1; + mov.u32 %r6391, %r6370; + mov.u32 %r6369, %r147; + @%p79 bra $L__BB3_81; + + add.s32 %r6369, %r6359, -2; + mov.u32 %r3418, 1; + shl.b32 %r3419, %r3418, %r6369; + and.b32 %r3420, %r3419, %r6595; + setp.ne.s32 %p80, %r3420, 0; + selp.u32 %r3421, 1, 0, %p80; + cvt.u32.u16 %r3422, %rs956; + bfi.b32 %r3423, %r3422, %r3421, 1, 8; + cvt.u16.u32 %rs956, %r3423; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p81, %r6390, 0; + mov.u32 %r6365, %r6370; + @%p81 bra $L__BB3_77; + + setp.gt.u32 %p82, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r6365, %r3418; + @%p82 bra $L__BB3_77; + + add.s32 %r3426, %r6381, 17477; + cvt.u64.u32 %rd93, %r3426; + add.s64 %rd94, %rd93, %rd5; + add.s64 %rd95, %rd1, %rd94; + and.b16 %rs478, %rs956, 255; + st.global.u8 [%rd95], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p83, %rs478, 255; + selp.b32 %r6390, 7, 8, %p83; + mov.u16 %rs956, 0; + mov.u32 %r6365, %r6370; + +$L__BB3_77: + setp.eq.s32 %p84, %r148, 2; + mov.u32 %r6391, %r6365; + mov.u32 %r6370, %r6365; + @%p84 bra $L__BB3_81; + + add.s32 %r6369, %r6359, -3; + mov.u32 %r3427, 1; + shl.b32 %r3428, %r3427, %r6369; + and.b32 %r3429, %r3428, %r6595; + setp.ne.s32 %p85, %r3429, 0; + selp.u32 %r3430, 1, 0, %p85; + cvt.u32.u16 %r3431, %rs956; + bfi.b32 %r3432, %r3431, %r3430, 1, 8; + cvt.u16.u32 %rs956, %r3432; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p86, %r6390, 0; + mov.u32 %r6391, %r6365; + mov.u32 %r6370, %r6365; + @%p86 bra $L__BB3_81; + + setp.gt.u32 %p87, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r6391, %r3427; + mov.u32 %r6370, %r3427; + @%p87 bra $L__BB3_81; + + add.s32 %r3437, %r6381, 17477; + cvt.u64.u32 %rd96, %r3437; + add.s64 %rd97, %rd96, %rd5; + add.s64 %rd98, %rd1, %rd97; + and.b16 %rs481, %rs956, 255; + st.global.u8 [%rd98], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p88, %rs481, 255; + selp.b32 %r6390, 7, 8, %p88; + mov.u16 %rs956, 0; + mov.u32 %r6391, %r6365; + mov.u32 %r6370, %r6365; + +$L__BB3_81: + setp.lt.u32 %p89, %r147, 3; + @%p89 bra $L__BB3_96; + + mov.u32 %r6391, %r6370; + +$L__BB3_83: + add.s32 %r3438, %r6369, -1; + mov.u32 %r3439, 1; + shl.b32 %r3440, %r3439, %r3438; + and.b32 %r3441, %r3440, %r6595; + setp.ne.s32 %p90, %r3441, 0; + selp.u32 %r3442, 1, 0, %p90; + cvt.u32.u16 %r3443, %rs956; + bfi.b32 %r6379, %r3443, %r3442, 1, 8; + add.s32 %r6378, %r6390, -1; + setp.ne.s32 %p91, %r6378, 0; + mov.u32 %r6380, %r6391; + @%p91 bra $L__BB3_86; + + setp.gt.u32 %p92, %r6381, 191; + mov.u32 %r6378, 0; + mov.u32 %r6380, %r3439; + @%p92 bra $L__BB3_86; + + cvt.u16.u32 %rs482, %r6379; + and.b16 %rs483, %rs482, 255; + add.s32 %r3447, %r6381, 17477; + cvt.u64.u32 %rd99, %r3447; + add.s64 %rd100, %rd99, %rd5; + add.s64 %rd101, %rd1, %rd100; + st.global.u8 [%rd101], %rs482; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p93, %rs483, 255; + selp.b32 %r6378, 7, 8, %p93; + mov.u32 %r6379, 0; + mov.u32 %r6380, %r6391; + +$L__BB3_86: + add.s32 %r3448, %r6369, -2; + shl.b32 %r3450, %r3439, %r3448; + and.b32 %r3451, %r3450, %r6595; + setp.ne.s32 %p94, %r3451, 0; + and.b32 %r3452, %r6379, 127; + selp.u32 %r3453, 1, 0, %p94; + bfi.b32 %r6383, %r3452, %r3453, 1, 7; + add.s32 %r6382, %r6378, -1; + setp.ne.s32 %p95, %r6382, 0; + mov.u32 %r6384, %r6380; + @%p95 bra $L__BB3_89; + + setp.gt.u32 %p96, %r6381, 191; + mov.u32 %r6384, 1; + mov.u32 %r6382, 0; + @%p96 bra $L__BB3_89; + + cvt.u16.u32 %rs484, %r6383; + and.b16 %rs485, %rs484, 255; + add.s32 %r3457, %r6381, 17477; + cvt.u64.u32 %rd102, %r3457; + add.s64 %rd103, %rd102, %rd5; + add.s64 %rd104, %rd1, %rd103; + st.global.u8 [%rd104], %rs484; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p97, %rs485, 255; + selp.b32 %r6382, 7, 8, %p97; + mov.u32 %r6383, 0; + mov.u32 %r6384, %r6380; + +$L__BB3_89: + add.s32 %r3458, %r6369, -3; + mov.u32 %r3459, 1; + shl.b32 %r3460, %r3459, %r3458; + and.b32 %r3461, %r3460, %r6595; + setp.ne.s32 %p98, %r3461, 0; + and.b32 %r3462, %r6383, 127; + selp.u32 %r3463, 1, 0, %p98; + bfi.b32 %r6387, %r3462, %r3463, 1, 7; + add.s32 %r6386, %r6382, -1; + setp.ne.s32 %p99, %r6386, 0; + mov.u32 %r6388, %r6384; + @%p99 bra $L__BB3_92; + + setp.gt.u32 %p100, %r6381, 191; + mov.u32 %r6386, 0; + mov.u32 %r6388, %r3459; + @%p100 bra $L__BB3_92; + + cvt.u16.u32 %rs486, %r6387; + and.b16 %rs487, %rs486, 255; + add.s32 %r3467, %r6381, 17477; + cvt.u64.u32 %rd105, %r3467; + add.s64 %rd106, %rd105, %rd5; + add.s64 %rd107, %rd1, %rd106; + st.global.u8 [%rd107], %rs486; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p101, %rs487, 255; + selp.b32 %r6386, 7, 8, %p101; + mov.u32 %r6387, 0; + mov.u32 %r6388, %r6384; + +$L__BB3_92: + add.s32 %r6369, %r6369, -4; + shl.b32 %r3469, %r3459, %r6369; + and.b32 %r3470, %r3469, %r6595; + setp.ne.s32 %p102, %r3470, 0; + and.b32 %r3471, %r6387, 127; + selp.u32 %r3472, 1, 0, %p102; + bfi.b32 %r3473, %r3471, %r3472, 1, 15; + cvt.u16.u32 %rs956, %r3473; + add.s32 %r6390, %r6386, -1; + setp.ne.s32 %p103, %r6390, 0; + mov.u32 %r6391, %r6388; + @%p103 bra $L__BB3_95; + + setp.gt.u32 %p104, %r6381, 191; + mov.u32 %r6391, 1; + mov.u32 %r6390, 0; + @%p104 bra $L__BB3_95; + + add.s32 %r3476, %r6381, 17477; + cvt.u64.u32 %rd108, %r3476; + add.s64 %rd109, %rd108, %rd5; + add.s64 %rd110, %rd1, %rd109; + and.b16 %rs489, %rs956, 255; + st.global.u8 [%rd110], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p105, %rs489, 255; + selp.b32 %r6390, 7, 8, %p105; + mov.u16 %rs956, 0; + mov.u32 %r6391, %r6388; + +$L__BB3_95: + setp.ne.s32 %p106, %r6369, 0; + @%p106 bra $L__BB3_83; + +$L__BB3_96: + add.s32 %r3478, %r6596, -1; + setp.eq.s32 %p107, %r6596, 0; + mov.u32 %r6595, 0; + selp.b32 %r6596, 0, %r3478, %p107; + setp.lt.u32 %p108, %r6596, 3; + mov.u32 %r6395, %r6595; + @%p108 bra $L__BB3_99; + + setp.lt.u32 %p109, %r6596, 6; + mov.u32 %r6395, 1; + @%p109 bra $L__BB3_99; + + setp.lt.u32 %p110, %r6596, 9; + setp.eq.s32 %p111, %r6596, 11; + selp.b32 %r3480, 4, 5, %p111; + setp.lt.u32 %p112, %r6596, 11; + selp.b32 %r3481, 3, %r3480, %p112; + selp.b32 %r6395, 2, %r3481, %p110; + +$L__BB3_99: + mov.u32 %r3483, 1; + shl.b32 %r6597, %r3483, %r6395; + mov.u32 %r6598, %r6391; + bra.uni $L__BB3_108; + +$L__BB3_100: + add.s32 %r6595, %r6595, 1; + setp.lt.u32 %p113, %r6595, %r6597; + @%p113 bra $L__BB3_108; + + shl.b16 %rs490, %rs956, 1; + or.b16 %rs956, %rs490, 1; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p114, %r6390, 0; + mov.u32 %r6398, %r6598; + @%p114 bra $L__BB3_104; + + setp.gt.u32 %p115, %r6381, 191; + mov.u32 %r6398, 1; + mov.u32 %r6390, 0; + @%p115 bra $L__BB3_104; + + and.b16 %rs492, %rs956, 255; + st.global.u8 [%rd10], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p116, %rs492, 255; + selp.b32 %r6390, 7, 8, %p116; + mov.u16 %rs956, 0; + mov.u32 %r6398, %r6598; + +$L__BB3_104: + add.s32 %r3487, %r6596, 1; + min.u32 %r6596, %r3487, 12; + setp.lt.u32 %p117, %r6596, 3; + mov.u32 %r6595, 0; + mov.u32 %r6399, %r6595; + @%p117 bra $L__BB3_107; + + setp.lt.u32 %p118, %r6596, 6; + mov.u32 %r6399, 1; + @%p118 bra $L__BB3_107; + + setp.lt.u32 %p119, %r6596, 9; + setp.eq.s32 %p120, %r6596, 11; + selp.b32 %r3489, 4, 5, %p120; + setp.lt.u32 %p121, %r6596, 11; + selp.b32 %r3490, 3, %r3489, %p121; + selp.b32 %r6399, 2, %r3490, %p119; + +$L__BB3_107: + mov.u32 %r3492, 1; + shl.b32 %r6597, %r3492, %r6399; + mov.u32 %r6598, %r6398; + +$L__BB3_108: + max.s32 %r231, %r6341, 1; + and.b16 %rs493, %rs4, 15; + cvt.u32.u16 %r232, %rs493; + and.b32 %r233, %r6325, 1; + setp.eq.s32 %p122, %r233, 0; + mov.u32 %r6420, %r7050; + @%p122 bra $L__BB3_115; + + and.b32 %r3493, %r232, 1; + sub.s32 %r6406, %r231, %r3493; + setp.eq.s32 %p123, %r6406, 0; + mov.u32 %r6420, %r7050; + @%p123 bra $L__BB3_115; + + mov.u32 %r3494, -1; + shl.b32 %r3495, %r3494, %r6406; + not.b32 %r3496, %r3495; + and.b32 %r6407, %r6319, %r3496; + +$L__BB3_111: + setp.gt.u32 %p124, %r7016, 17476; + mov.u32 %r6420, 1; + @%p124 bra $L__BB3_115; + + sub.s32 %r3498, %r7017, %r7018; + min.u32 %r3499, %r3498, %r6406; + setp.eq.s32 %p125, %r3499, 32; + mov.u32 %r3500, -1; + shl.b32 %r3501, %r3500, %r3499; + not.b32 %r3502, %r3501; + selp.b32 %r3503, -1, %r3502, %p125; + and.b32 %r3504, %r3503, %r6407; + shl.b32 %r3505, %r3504, %r7018; + or.b32 %r7019, %r3505, %r7019; + add.s32 %r7018, %r3499, %r7018; + shr.u32 %r6407, %r6407, %r3499; + sub.s32 %r6406, %r6406, %r3499; + setp.lt.u32 %p126, %r7018, %r7017; + @%p126 bra $L__BB3_114; + + cvt.u64.u32 %rd111, %r7016; + add.s64 %rd112, %rd111, %rd5; + add.s64 %rd113, %rd1, %rd112; + st.global.u8 [%rd113], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p127, %r7019, 255; + selp.b32 %r7017, 7, 8, %p127; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_114: + setp.ne.s32 %p128, %r6406, 0; + mov.u32 %r6420, %r7050; + @%p128 bra $L__BB3_111; + +$L__BB3_115: + setp.eq.s32 %p129, %r117, 0; + mov.u32 %r6435, %r6420; + @%p129 bra $L__BB3_122; + + shr.u32 %r3508, %r232, 1; + and.b32 %r3509, %r3508, 1; + sub.s32 %r6421, %r231, %r3509; + setp.eq.s32 %p130, %r6421, 0; + mov.u32 %r6435, %r6420; + @%p130 bra $L__BB3_122; + + mov.u32 %r3510, -1; + shl.b32 %r3511, %r3510, %r6421; + not.b32 %r3512, %r3511; + and.b32 %r6422, %r6323, %r3512; + +$L__BB3_118: + setp.gt.u32 %p131, %r7016, 17476; + mov.u32 %r6435, 1; + @%p131 bra $L__BB3_122; + + sub.s32 %r3514, %r7017, %r7018; + min.u32 %r3515, %r3514, %r6421; + setp.eq.s32 %p132, %r3515, 32; + mov.u32 %r3516, -1; + shl.b32 %r3517, %r3516, %r3515; + not.b32 %r3518, %r3517; + selp.b32 %r3519, -1, %r3518, %p132; + and.b32 %r3520, %r3519, %r6422; + shl.b32 %r3521, %r3520, %r7018; + or.b32 %r7019, %r3521, %r7019; + add.s32 %r7018, %r3515, %r7018; + shr.u32 %r6422, %r6422, %r3515; + sub.s32 %r6421, %r6421, %r3515; + setp.lt.u32 %p133, %r7018, %r7017; + @%p133 bra $L__BB3_121; + + cvt.u64.u32 %rd114, %r7016; + add.s64 %rd115, %rd114, %rd5; + add.s64 %rd116, %rd1, %rd115; + st.global.u8 [%rd116], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p134, %r7019, 255; + selp.b32 %r7017, 7, 8, %p134; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_121: + setp.ne.s32 %p135, %r6421, 0; + mov.u32 %r6435, %r6420; + @%p135 bra $L__BB3_118; + +$L__BB3_122: + and.b32 %r3524, %r6325, 4; + setp.eq.s32 %p136, %r3524, 0; + mov.u32 %r6450, %r6435; + @%p136 bra $L__BB3_129; + + shr.u32 %r3525, %r232, 2; + and.b32 %r3526, %r3525, 1; + sub.s32 %r6436, %r231, %r3526; + setp.eq.s32 %p137, %r6436, 0; + mov.u32 %r6450, %r6435; + @%p137 bra $L__BB3_129; + + mov.u32 %r3527, -1; + shl.b32 %r3528, %r3527, %r6436; + not.b32 %r3529, %r3528; + and.b32 %r6437, %r6339, %r3529; + +$L__BB3_125: + setp.gt.u32 %p138, %r7016, 17476; + mov.u32 %r6450, 1; + @%p138 bra $L__BB3_129; + + sub.s32 %r3531, %r7017, %r7018; + min.u32 %r3532, %r3531, %r6436; + setp.eq.s32 %p139, %r3532, 32; + mov.u32 %r3533, -1; + shl.b32 %r3534, %r3533, %r3532; + not.b32 %r3535, %r3534; + selp.b32 %r3536, -1, %r3535, %p139; + and.b32 %r3537, %r3536, %r6437; + shl.b32 %r3538, %r3537, %r7018; + or.b32 %r7019, %r3538, %r7019; + add.s32 %r7018, %r3532, %r7018; + shr.u32 %r6437, %r6437, %r3532; + sub.s32 %r6436, %r6436, %r3532; + setp.lt.u32 %p140, %r7018, %r7017; + @%p140 bra $L__BB3_128; + + cvt.u64.u32 %rd117, %r7016; + add.s64 %rd118, %rd117, %rd5; + add.s64 %rd119, %rd1, %rd118; + st.global.u8 [%rd119], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p141, %r7019, 255; + selp.b32 %r7017, 7, 8, %p141; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_128: + setp.ne.s32 %p142, %r6436, 0; + mov.u32 %r6450, %r6435; + @%p142 bra $L__BB3_125; + +$L__BB3_129: + setp.eq.s32 %p143, %r119, 0; + mov.u32 %r7050, %r6450; + @%p143 bra $L__BB3_136; + + shr.u32 %r3541, %r232, 3; + sub.s32 %r6451, %r231, %r3541; + setp.eq.s32 %p144, %r6451, 0; + mov.u32 %r7050, %r6450; + @%p144 bra $L__BB3_136; + + mov.u32 %r3542, -1; + shl.b32 %r3543, %r3542, %r6451; + not.b32 %r3544, %r3543; + and.b32 %r6452, %r6338, %r3544; + +$L__BB3_132: + setp.gt.u32 %p145, %r7016, 17476; + mov.u32 %r7050, 1; + @%p145 bra $L__BB3_136; + + sub.s32 %r3546, %r7017, %r7018; + min.u32 %r3547, %r3546, %r6451; + setp.eq.s32 %p146, %r3547, 32; + mov.u32 %r3548, -1; + shl.b32 %r3549, %r3548, %r3547; + not.b32 %r3550, %r3549; + selp.b32 %r3551, -1, %r3550, %p146; + and.b32 %r3552, %r3551, %r6452; + shl.b32 %r3553, %r3552, %r7018; + or.b32 %r7019, %r3553, %r7019; + add.s32 %r7018, %r3547, %r7018; + shr.u32 %r6452, %r6452, %r3547; + sub.s32 %r6451, %r6451, %r3547; + setp.lt.u32 %p147, %r7018, %r7017; + @%p147 bra $L__BB3_135; + + cvt.u64.u32 %rd120, %r7016; + add.s64 %rd121, %rd120, %rd5; + add.s64 %rd122, %rd1, %rd121; + st.global.u8 [%rd122], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p148, %r7019, 255; + selp.b32 %r7017, 7, 8, %p148; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_135: + setp.ne.s32 %p149, %r6451, 0; + mov.u32 %r7050, %r6450; + @%p149 bra $L__BB3_132; + +$L__BB3_136: + add.s32 %r3556, %r6299, 2; + setp.lt.u32 %p150, %r3556, %r3200; + mul.lo.s32 %r326, %r113, 6; + cvt.u64.u32 %rd123, %r326; + add.s64 %rd11, %rd9, %rd123; + add.s32 %r3557, %r326, 2; + cvt.u64.u32 %rd124, %r3557; + add.s64 %rd12, %rd9, %rd124; + @%p150 bra $L__BB3_165; + bra.uni $L__BB3_137; + +$L__BB3_165: + cvt.u64.u32 %rd138, %r6316; + add.s64 %rd139, %rd138, %rd4; + shl.b64 %rd140, %rd139, 2; + add.s64 %rd141, %rd3, %rd140; + ld.global.u32 %r399, [%rd141]; + setp.eq.s32 %p187, %r399, 0; + mov.u32 %r6511, 0; + mov.u32 %r6510, %r6511; + @%p187 bra $L__BB3_167; + + and.b32 %r3628, %r399, -2147483648; + abs.s32 %r3629, %r399; + shl.b32 %r3630, %r3629, %r27; + or.b32 %r6510, %r3630, %r3628; + +$L__BB3_167: + shl.b32 %r3634, %r6510, 1; + shr.u32 %r3635, %r3634, %r27; + and.b32 %r402, %r3635, -2; + setp.eq.s32 %p188, %r402, 0; + mov.u32 %r6512, %r6511; + mov.u32 %r6518, %r6511; + @%p188 bra $L__BB3_169; + + add.s32 %r3637, %r402, -1; + clz.b32 %r3638, %r3637; + mov.u32 %r3639, 32; + sub.s32 %r6511, %r3639, %r3638; + shr.u32 %r3640, %r6510, 31; + add.s32 %r3641, %r3640, %r402; + add.s32 %r6512, %r3641, -2; + mov.u32 %r6518, 1; + +$L__BB3_169: + mov.u32 %r6515, 0; + mov.u32 %r6514, %r6515; + @%p40 bra $L__BB3_172; + + add.s32 %r3644, %r6316, %r3198; + cvt.u64.u32 %rd142, %r3644; + add.s64 %rd143, %rd142, %rd4; + shl.b64 %rd144, %rd143, 2; + add.s64 %rd145, %rd3, %rd144; + ld.global.u32 %r408, [%rd145]; + setp.eq.s32 %p190, %r408, 0; + @%p190 bra $L__BB3_172; + + and.b32 %r3645, %r408, -2147483648; + abs.s32 %r3646, %r408; + shl.b32 %r3647, %r3646, %r27; + or.b32 %r6514, %r3647, %r3645; + +$L__BB3_172: + shl.b32 %r3650, %r6514, 1; + shr.u32 %r3651, %r3650, %r27; + and.b32 %r411, %r3651, -2; + setp.eq.s32 %p191, %r411, 0; + mov.u32 %r6516, %r6515; + mov.u32 %r6534, %r6511; + @%p191 bra $L__BB3_174; + + or.b32 %r6518, %r6518, 2; + add.s32 %r3652, %r411, -1; + clz.b32 %r3653, %r3652; + mov.u32 %r3654, 32; + sub.s32 %r6515, %r3654, %r3653; + max.s32 %r6534, %r6511, %r6515; + shr.u32 %r3655, %r6514, 31; + add.s32 %r3656, %r3655, %r411; + add.s32 %r6516, %r3656, -2; + +$L__BB3_174: + add.s32 %r6533, %r6316, 1; + add.s32 %r3661, %r6299, 3; + setp.ge.u32 %p192, %r3661, %r3200; + mov.u32 %r6536, 0; + mov.u32 %r6529, %r6536; + mov.u32 %r6530, %r6536; + mov.u32 %r6531, %r6536; + mov.u32 %r6532, %r6536; + @%p192 bra $L__BB3_185; + + cvt.u64.u32 %rd146, %r6533; + add.s64 %rd147, %rd146, %rd4; + shl.b64 %rd148, %rd147, 2; + add.s64 %rd149, %rd3, %rd148; + ld.global.u32 %r421, [%rd149]; + setp.eq.s32 %p193, %r421, 0; + mov.u32 %r6530, 0; + mov.u32 %r6519, %r6530; + @%p193 bra $L__BB3_177; + + and.b32 %r3663, %r421, -2147483648; + abs.s32 %r3664, %r421; + shl.b32 %r3665, %r3664, %r27; + or.b32 %r6519, %r3665, %r3663; + +$L__BB3_177: + shl.b32 %r3668, %r6519, 1; + shr.u32 %r3669, %r3668, %r27; + and.b32 %r424, %r3669, -2; + setp.eq.s32 %p194, %r424, 0; + mov.u32 %r6532, %r6530; + @%p194 bra $L__BB3_179; + + or.b32 %r6518, %r6518, 4; + add.s32 %r3670, %r424, -1; + clz.b32 %r3671, %r3670; + mov.u32 %r3672, 32; + sub.s32 %r6530, %r3672, %r3671; + max.s32 %r6534, %r6534, %r6530; + shr.u32 %r3673, %r6519, 31; + add.s32 %r3674, %r3673, %r424; + add.s32 %r6532, %r3674, -2; + +$L__BB3_179: + mov.u32 %r6529, 0; + mov.u32 %r6524, %r6529; + @%p40 bra $L__BB3_182; + + add.s32 %r3677, %r6533, %r3198; + cvt.u64.u32 %rd150, %r3677; + add.s64 %rd151, %rd150, %rd4; + shl.b64 %rd152, %rd151, 2; + add.s64 %rd153, %rd3, %rd152; + ld.global.u32 %r433, [%rd153]; + setp.eq.s32 %p196, %r433, 0; + @%p196 bra $L__BB3_182; + + and.b32 %r3678, %r433, -2147483648; + abs.s32 %r3679, %r433; + shl.b32 %r3680, %r3679, %r27; + or.b32 %r6524, %r3680, %r3678; + +$L__BB3_182: + shl.b32 %r3683, %r6524, 1; + shr.u32 %r3684, %r3683, %r27; + and.b32 %r436, %r3684, -2; + setp.eq.s32 %p197, %r436, 0; + mov.u32 %r6531, %r6529; + @%p197 bra $L__BB3_184; + + or.b32 %r6518, %r6518, 8; + add.s32 %r3685, %r436, -1; + clz.b32 %r3686, %r3685; + mov.u32 %r3687, 32; + sub.s32 %r6529, %r3687, %r3686; + max.s32 %r6534, %r6534, %r6529; + shr.u32 %r3688, %r6524, 31; + add.s32 %r3689, %r3688, %r436; + add.s32 %r6531, %r3689, -2; + +$L__BB3_184: + add.s32 %r6533, %r6316, 2; + +$L__BB3_185: + mov.u32 %r6316, %r6533; + shr.u32 %r3691, %r6325, 1; + or.b32 %r453, %r3691, %r233; + add.s32 %r3692, %r6534, -1; + setp.lt.s32 %p198, %r6534, 2; + setp.gt.s32 %p199, %r6534, 1; + selp.b32 %r454, %r3692, 0, %p199; + @%p198 bra $L__BB3_187; + + setp.eq.s32 %p200, %r6511, %r6534; + selp.u32 %r3693, 1, 0, %p200; + setp.eq.s32 %p201, %r6515, %r6534; + selp.u32 %r3694, -1, 0, %p201; + bfi.b32 %r3695, %r3694, %r3693, 1, 1; + setp.eq.s32 %p202, %r6530, %r6534; + selp.u16 %rs513, 1, 0, %p202; + mul.wide.u16 %r3696, %rs513, 4; + or.b32 %r3697, %r3695, %r3696; + setp.eq.s32 %p203, %r6529, %r6534; + selp.u16 %rs514, 1, 0, %p203; + mul.wide.u16 %r3698, %rs514, 8; + or.b32 %r6536, %r3697, %r3698; + +$L__BB3_187: + and.b32 %r3699, %r6515, 255; + and.b32 %r3700, %r6336, 255; + setp.lt.u32 %p204, %r3699, %r3700; + cvt.u16.u32 %rs515, %r6515; + selp.b16 %rs516, %rs3, %rs515, %p204; + st.shared.u8 [%r116+1], %rs516; + st.shared.u8 [%r116+2], %r6529; + and.b32 %r457, %r6518, 2; + shr.u32 %r3701, %r457, 1; + or.b32 %r3702, %r120, %r3701; + st.shared.u8 [%r118+1], %r3702; + and.b32 %r458, %r6518, 8; + shr.u32 %r3703, %r458, 3; + st.shared.u8 [%r118+2], %r3703; + shl.b32 %r3704, %r6518, 4; + shl.b32 %r3705, %r453, 8; + or.b32 %r3706, %r3704, %r3705; + or.b32 %r3707, %r3706, %r6536; + mul.wide.u32 %rd155, %r3707, 2; + add.s64 %rd156, %rd8, %rd155; + ld.global.u16 %rs48, [%rd156]; + shr.u16 %rs517, %rs48, 4; + and.b16 %rs49, %rs517, 7; + setp.eq.s16 %p205, %rs49, 0; + mov.u32 %r6548, %r6355; + @%p205 bra $L__BB3_194; + + cvt.u32.u16 %r6537, %rs49; + shr.u16 %rs518, %rs48, 8; + cvt.u32.u16 %r6538, %rs518; + +$L__BB3_189: + mov.u32 %r461, %r6537; + setp.gt.u32 %p206, %r6829, 2879; + mov.u32 %r6548, 1; + @%p206 bra $L__BB3_194; + + mov.u32 %r3709, 8; + sub.s32 %r3710, %r3709, %r6831; + sub.s32 %r3711, %r3710, %r6830; + min.u32 %r3712, %r3711, %r461; + setp.eq.s32 %p207, %r3712, 32; + mov.u32 %r3713, -1; + shl.b32 %r3714, %r3713, %r3712; + not.b32 %r3715, %r3714; + selp.b32 %r3716, -1, %r3715, %p207; + and.b32 %r3717, %r3716, %r6538; + shl.b32 %r3718, %r3717, %r6830; + cvt.u16.u32 %rs519, %r3718; + or.b16 %rs1025, %rs1025, %rs519; + add.s32 %r6830, %r3712, %r6830; + sub.s32 %r6537, %r461, %r3712; + shr.u32 %r6538, %r6538, %r3712; + setp.gt.u32 %p208, %r3711, %r461; + @%p208 bra $L__BB3_193; + + setp.ne.s32 %p209, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs520, %rs1025, 255; + setp.ne.s16 %p210, %rs520, 127; + and.pred %p211, %p209, %p210; + @%p211 bra $L__BB3_193; + + mov.u32 %r3721, 20548; + sub.s32 %r3722, %r3721, %r6829; + cvt.u64.u32 %rd157, %r3722; + add.s64 %rd158, %rd157, %rd5; + add.s64 %rd159, %rd1, %rd158; + st.global.u8 [%rd159], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p212, %rs520, 143; + selp.u32 %r6831, 1, 0, %p212; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_193: + setp.ne.s32 %p213, %r6537, 0; + mov.u32 %r6548, %r6355; + @%p213 bra $L__BB3_189; + +$L__BB3_194: + setp.ne.s32 %p214, %r453, 0; + @%p214 bra $L__BB3_242; + + setp.eq.s32 %p215, %r6518, 0; + add.s32 %r3723, %r6381, 17477; + cvt.u64.u32 %rd160, %r3723; + add.s64 %rd161, %rd160, %rd5; + add.s64 %rd13, %rd1, %rd161; + @%p215 bra $L__BB3_234; + + shl.b16 %rs956, %rs956, 1; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p216, %r6390, 0; + mov.u32 %r6584, %r6598; + @%p216 bra $L__BB3_199; + bra.uni $L__BB3_197; + +$L__BB3_199: + setp.lt.u32 %p218, %r6596, 3; + mov.u32 %r6552, 0; + @%p218 bra $L__BB3_202; + + setp.lt.u32 %p219, %r6596, 6; + mov.u32 %r6552, 1; + @%p219 bra $L__BB3_202; + + setp.lt.u32 %p220, %r6596, 9; + setp.eq.s32 %p221, %r6596, 11; + selp.b32 %r3729, 4, 5, %p221; + setp.lt.u32 %p222, %r6596, 11; + selp.b32 %r3730, 3, %r3729, %p222; + selp.b32 %r6552, 2, %r3730, %p220; + +$L__BB3_202: + setp.eq.s32 %p223, %r6552, 0; + @%p223 bra $L__BB3_230; + + add.s32 %r485, %r6552, -1; + and.b32 %r486, %r6552, 3; + setp.eq.s32 %p224, %r486, 0; + mov.u32 %r6562, %r6552; + mov.u32 %r6563, %r6584; + @%p224 bra $L__BB3_215; + + mov.u32 %r3732, 1; + shl.b32 %r3733, %r3732, %r485; + and.b32 %r3734, %r3733, %r6595; + setp.ne.s32 %p225, %r3734, 0; + selp.u32 %r3735, 1, 0, %p225; + cvt.u32.u16 %r3736, %rs956; + bfi.b32 %r3737, %r3736, %r3735, 1, 8; + cvt.u16.u32 %rs956, %r3737; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p226, %r6390, 0; + mov.u32 %r6563, %r6584; + @%p226 bra $L__BB3_207; + + setp.gt.u32 %p227, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r6563, %r3732; + @%p227 bra $L__BB3_207; + + add.s32 %r3741, %r6381, 17477; + cvt.u64.u32 %rd162, %r3741; + add.s64 %rd163, %rd162, %rd5; + add.s64 %rd164, %rd1, %rd163; + st.global.u8 [%rd164], %rs956; + add.s32 %r6381, %r6381, 1; + mov.u16 %rs956, 0; + mov.u32 %r6390, 8; + mov.u32 %r6563, %r6584; + +$L__BB3_207: + setp.eq.s32 %p228, %r486, 1; + mov.u32 %r6584, %r6563; + mov.u32 %r6562, %r485; + @%p228 bra $L__BB3_215; + + add.s32 %r6562, %r6552, -2; + mov.u32 %r3742, 1; + shl.b32 %r3743, %r3742, %r6562; + and.b32 %r3744, %r3743, %r6595; + setp.ne.s32 %p229, %r3744, 0; + selp.u32 %r3745, 1, 0, %p229; + cvt.u32.u16 %r3746, %rs956; + bfi.b32 %r3747, %r3746, %r3745, 1, 8; + cvt.u16.u32 %rs956, %r3747; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p230, %r6390, 0; + mov.u32 %r6558, %r6563; + @%p230 bra $L__BB3_211; + + setp.gt.u32 %p231, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r6558, %r3742; + @%p231 bra $L__BB3_211; + + add.s32 %r3750, %r6381, 17477; + cvt.u64.u32 %rd165, %r3750; + add.s64 %rd166, %rd165, %rd5; + add.s64 %rd167, %rd1, %rd166; + and.b16 %rs527, %rs956, 255; + st.global.u8 [%rd167], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p232, %rs527, 255; + selp.b32 %r6390, 7, 8, %p232; + mov.u16 %rs956, 0; + mov.u32 %r6558, %r6563; + +$L__BB3_211: + setp.eq.s32 %p233, %r486, 2; + mov.u32 %r6584, %r6558; + mov.u32 %r6563, %r6558; + @%p233 bra $L__BB3_215; + + add.s32 %r6562, %r6552, -3; + mov.u32 %r3751, 1; + shl.b32 %r3752, %r3751, %r6562; + and.b32 %r3753, %r3752, %r6595; + setp.ne.s32 %p234, %r3753, 0; + selp.u32 %r3754, 1, 0, %p234; + cvt.u32.u16 %r3755, %rs956; + bfi.b32 %r3756, %r3755, %r3754, 1, 8; + cvt.u16.u32 %rs956, %r3756; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p235, %r6390, 0; + mov.u32 %r6584, %r6558; + mov.u32 %r6563, %r6558; + @%p235 bra $L__BB3_215; + + setp.gt.u32 %p236, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r6584, %r3751; + mov.u32 %r6563, %r3751; + @%p236 bra $L__BB3_215; + + add.s32 %r3761, %r6381, 17477; + cvt.u64.u32 %rd168, %r3761; + add.s64 %rd169, %rd168, %rd5; + add.s64 %rd170, %rd1, %rd169; + and.b16 %rs530, %rs956, 255; + st.global.u8 [%rd170], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p237, %rs530, 255; + selp.b32 %r6390, 7, 8, %p237; + mov.u16 %rs956, 0; + mov.u32 %r6584, %r6558; + mov.u32 %r6563, %r6558; + +$L__BB3_215: + setp.lt.u32 %p238, %r485, 3; + @%p238 bra $L__BB3_230; + + mov.u32 %r6584, %r6563; + +$L__BB3_217: + add.s32 %r3762, %r6562, -1; + mov.u32 %r3763, 1; + shl.b32 %r3764, %r3763, %r3762; + and.b32 %r3765, %r3764, %r6595; + setp.ne.s32 %p239, %r3765, 0; + selp.u32 %r3766, 1, 0, %p239; + cvt.u32.u16 %r3767, %rs956; + bfi.b32 %r6572, %r3767, %r3766, 1, 8; + add.s32 %r6571, %r6390, -1; + setp.ne.s32 %p240, %r6571, 0; + mov.u32 %r6573, %r6584; + @%p240 bra $L__BB3_220; + + setp.gt.u32 %p241, %r6381, 191; + mov.u32 %r6571, 0; + mov.u32 %r6573, %r3763; + @%p241 bra $L__BB3_220; + + cvt.u16.u32 %rs531, %r6572; + and.b16 %rs532, %rs531, 255; + add.s32 %r3771, %r6381, 17477; + cvt.u64.u32 %rd171, %r3771; + add.s64 %rd172, %rd171, %rd5; + add.s64 %rd173, %rd1, %rd172; + st.global.u8 [%rd173], %rs531; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p242, %rs532, 255; + selp.b32 %r6571, 7, 8, %p242; + mov.u32 %r6572, 0; + mov.u32 %r6573, %r6584; + +$L__BB3_220: + add.s32 %r3772, %r6562, -2; + shl.b32 %r3774, %r3763, %r3772; + and.b32 %r3775, %r3774, %r6595; + setp.ne.s32 %p243, %r3775, 0; + and.b32 %r3776, %r6572, 127; + selp.u32 %r3777, 1, 0, %p243; + bfi.b32 %r6576, %r3776, %r3777, 1, 7; + add.s32 %r6575, %r6571, -1; + setp.ne.s32 %p244, %r6575, 0; + mov.u32 %r6577, %r6573; + @%p244 bra $L__BB3_223; + + setp.gt.u32 %p245, %r6381, 191; + mov.u32 %r6577, 1; + mov.u32 %r6575, 0; + @%p245 bra $L__BB3_223; + + cvt.u16.u32 %rs533, %r6576; + and.b16 %rs534, %rs533, 255; + add.s32 %r3781, %r6381, 17477; + cvt.u64.u32 %rd174, %r3781; + add.s64 %rd175, %rd174, %rd5; + add.s64 %rd176, %rd1, %rd175; + st.global.u8 [%rd176], %rs533; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p246, %rs534, 255; + selp.b32 %r6575, 7, 8, %p246; + mov.u32 %r6576, 0; + mov.u32 %r6577, %r6573; + +$L__BB3_223: + add.s32 %r3782, %r6562, -3; + mov.u32 %r3783, 1; + shl.b32 %r3784, %r3783, %r3782; + and.b32 %r3785, %r3784, %r6595; + setp.ne.s32 %p247, %r3785, 0; + and.b32 %r3786, %r6576, 127; + selp.u32 %r3787, 1, 0, %p247; + bfi.b32 %r6580, %r3786, %r3787, 1, 7; + add.s32 %r6579, %r6575, -1; + setp.ne.s32 %p248, %r6579, 0; + mov.u32 %r6581, %r6577; + @%p248 bra $L__BB3_226; + + setp.gt.u32 %p249, %r6381, 191; + mov.u32 %r6579, 0; + mov.u32 %r6581, %r3783; + @%p249 bra $L__BB3_226; + + cvt.u16.u32 %rs535, %r6580; + and.b16 %rs536, %rs535, 255; + add.s32 %r3791, %r6381, 17477; + cvt.u64.u32 %rd177, %r3791; + add.s64 %rd178, %rd177, %rd5; + add.s64 %rd179, %rd1, %rd178; + st.global.u8 [%rd179], %rs535; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p250, %rs536, 255; + selp.b32 %r6579, 7, 8, %p250; + mov.u32 %r6580, 0; + mov.u32 %r6581, %r6577; + +$L__BB3_226: + add.s32 %r6562, %r6562, -4; + shl.b32 %r3793, %r3783, %r6562; + and.b32 %r3794, %r3793, %r6595; + setp.ne.s32 %p251, %r3794, 0; + and.b32 %r3795, %r6580, 127; + selp.u32 %r3796, 1, 0, %p251; + bfi.b32 %r3797, %r3795, %r3796, 1, 15; + cvt.u16.u32 %rs956, %r3797; + add.s32 %r6390, %r6579, -1; + setp.ne.s32 %p252, %r6390, 0; + mov.u32 %r6584, %r6581; + @%p252 bra $L__BB3_229; + + setp.gt.u32 %p253, %r6381, 191; + mov.u32 %r6584, 1; + mov.u32 %r6390, 0; + @%p253 bra $L__BB3_229; + + add.s32 %r3800, %r6381, 17477; + cvt.u64.u32 %rd180, %r3800; + add.s64 %rd181, %rd180, %rd5; + add.s64 %rd182, %rd1, %rd181; + and.b16 %rs538, %rs956, 255; + st.global.u8 [%rd182], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p254, %rs538, 255; + selp.b32 %r6390, 7, 8, %p254; + mov.u16 %rs956, 0; + mov.u32 %r6584, %r6581; + +$L__BB3_229: + setp.ne.s32 %p255, %r6562, 0; + @%p255 bra $L__BB3_217; + +$L__BB3_230: + add.s32 %r3802, %r6596, -1; + setp.eq.s32 %p256, %r6596, 0; + mov.u32 %r6595, 0; + selp.b32 %r6596, 0, %r3802, %p256; + setp.lt.u32 %p257, %r6596, 3; + mov.u32 %r6588, %r6595; + @%p257 bra $L__BB3_233; + + setp.lt.u32 %p258, %r6596, 6; + mov.u32 %r6588, 1; + @%p258 bra $L__BB3_233; + + setp.lt.u32 %p259, %r6596, 9; + setp.eq.s32 %p260, %r6596, 11; + selp.b32 %r3804, 4, 5, %p260; + setp.lt.u32 %p261, %r6596, 11; + selp.b32 %r3805, 3, %r3804, %p261; + selp.b32 %r6588, 2, %r3805, %p259; + +$L__BB3_233: + mov.u32 %r3807, 1; + shl.b32 %r6597, %r3807, %r6588; + mov.u32 %r6598, %r6584; + bra.uni $L__BB3_242; + +$L__BB3_137: + ld.global.u8 %rs26, [%rd11+1]; + ld.global.u8 %rs27, [%rd12]; + ld.global.u8 %rs28, [%rd12+1]; + ld.global.u8 %rs29, [%rd9]; + ld.global.u8 %rs30, [%rd9+1]; + ld.global.u8 %rs31, [%rd9+2]; + ld.global.u8 %rs32, [%rd9+3]; + setp.eq.s16 %p151, %rs26, 0; + mov.u32 %r6477, %r6355; + @%p151 bra $L__BB3_144; + + ld.global.u8 %r6467, [%rd11]; + cvt.u32.u16 %r6466, %rs26; + +$L__BB3_139: + mov.u32 %r329, %r6466; + setp.gt.u32 %p152, %r6829, 2879; + mov.u32 %r6477, 1; + @%p152 bra $L__BB3_144; + + mov.u32 %r3559, 8; + sub.s32 %r3560, %r3559, %r6831; + sub.s32 %r3561, %r3560, %r6830; + min.u32 %r3562, %r3561, %r329; + setp.eq.s32 %p153, %r3562, 32; + mov.u32 %r3563, -1; + shl.b32 %r3564, %r3563, %r3562; + not.b32 %r3565, %r3564; + selp.b32 %r3566, -1, %r3565, %p153; + and.b32 %r3567, %r3566, %r6467; + shl.b32 %r3568, %r3567, %r6830; + cvt.u16.u32 %rs494, %r3568; + or.b16 %rs1025, %rs1025, %rs494; + add.s32 %r6830, %r3562, %r6830; + sub.s32 %r6466, %r329, %r3562; + shr.u32 %r6467, %r6467, %r3562; + setp.gt.u32 %p154, %r3561, %r329; + @%p154 bra $L__BB3_143; + + setp.ne.s32 %p155, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs495, %rs1025, 255; + setp.ne.s16 %p156, %rs495, 127; + and.pred %p157, %p155, %p156; + @%p157 bra $L__BB3_143; + + mov.u32 %r3571, 20548; + sub.s32 %r3572, %r3571, %r6829; + cvt.u64.u32 %rd126, %r3572; + add.s64 %rd127, %rd126, %rd5; + add.s64 %rd128, %rd1, %rd127; + st.global.u8 [%rd128], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p158, %rs495, 143; + selp.u32 %r6831, 1, 0, %p158; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_143: + setp.ne.s32 %p159, %r6466, 0; + mov.u32 %r6477, %r6355; + @%p159 bra $L__BB3_139; + +$L__BB3_144: + setp.eq.s16 %p160, %rs30, 0; + mov.u32 %r6489, %r6477; + @%p160 bra $L__BB3_151; + + cvt.u32.u16 %r3573, %rs29; + and.b32 %r6479, %r3573, 255; + cvt.u32.u16 %r3574, %rs30; + and.b32 %r6478, %r3574, 255; + +$L__BB3_146: + mov.u32 %r348, %r6478; + setp.gt.u32 %p161, %r6829, 2879; + mov.u32 %r6489, 1; + @%p161 bra $L__BB3_151; + + mov.u32 %r3576, 8; + sub.s32 %r3577, %r3576, %r6831; + sub.s32 %r3578, %r3577, %r6830; + min.u32 %r3579, %r3578, %r348; + setp.eq.s32 %p162, %r3579, 32; + mov.u32 %r3580, -1; + shl.b32 %r3581, %r3580, %r3579; + not.b32 %r3582, %r3581; + selp.b32 %r3583, -1, %r3582, %p162; + and.b32 %r3584, %r3583, %r6479; + shl.b32 %r3585, %r3584, %r6830; + cvt.u16.u32 %rs499, %r3585; + or.b16 %rs1025, %rs1025, %rs499; + add.s32 %r6830, %r3579, %r6830; + sub.s32 %r6478, %r348, %r3579; + shr.u32 %r6479, %r6479, %r3579; + setp.gt.u32 %p163, %r3578, %r348; + @%p163 bra $L__BB3_150; + + setp.ne.s32 %p164, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs500, %rs1025, 255; + setp.ne.s16 %p165, %rs500, 127; + and.pred %p166, %p164, %p165; + @%p166 bra $L__BB3_150; + + mov.u32 %r3588, 20548; + sub.s32 %r3589, %r3588, %r6829; + cvt.u64.u32 %rd129, %r3589; + add.s64 %rd130, %rd129, %rd5; + add.s64 %rd131, %rd1, %rd130; + st.global.u8 [%rd131], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p167, %rs500, 143; + selp.u32 %r6831, 1, 0, %p167; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_150: + setp.ne.s32 %p168, %r6478, 0; + mov.u32 %r6489, %r6477; + @%p168 bra $L__BB3_146; + +$L__BB3_151: + setp.eq.s16 %p169, %rs28, 0; + mov.u32 %r6501, %r6489; + @%p169 bra $L__BB3_158; + + cvt.u32.u16 %r3590, %rs28; + and.b32 %r6490, %r3590, 255; + cvt.u32.u16 %r3591, %rs27; + and.b32 %r6491, %r3591, 255; + +$L__BB3_153: + mov.u32 %r367, %r6490; + setp.gt.u32 %p170, %r6829, 2879; + mov.u32 %r6501, 1; + @%p170 bra $L__BB3_158; + + mov.u32 %r3593, 8; + sub.s32 %r3594, %r3593, %r6831; + sub.s32 %r3595, %r3594, %r6830; + min.u32 %r3596, %r3595, %r367; + setp.eq.s32 %p171, %r3596, 32; + mov.u32 %r3597, -1; + shl.b32 %r3598, %r3597, %r3596; + not.b32 %r3599, %r3598; + selp.b32 %r3600, -1, %r3599, %p171; + and.b32 %r3601, %r3600, %r6491; + shl.b32 %r3602, %r3601, %r6830; + cvt.u16.u32 %rs504, %r3602; + or.b16 %rs1025, %rs1025, %rs504; + add.s32 %r6830, %r3596, %r6830; + sub.s32 %r6490, %r367, %r3596; + shr.u32 %r6491, %r6491, %r3596; + setp.gt.u32 %p172, %r3595, %r367; + @%p172 bra $L__BB3_157; + + setp.ne.s32 %p173, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs505, %rs1025, 255; + setp.ne.s16 %p174, %rs505, 127; + and.pred %p175, %p173, %p174; + @%p175 bra $L__BB3_157; + + mov.u32 %r3605, 20548; + sub.s32 %r3606, %r3605, %r6829; + cvt.u64.u32 %rd132, %r3606; + add.s64 %rd133, %rd132, %rd5; + add.s64 %rd134, %rd1, %rd133; + st.global.u8 [%rd134], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p176, %rs505, 143; + selp.u32 %r6831, 1, 0, %p176; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_157: + setp.ne.s32 %p177, %r6490, 0; + mov.u32 %r6501, %r6489; + @%p177 bra $L__BB3_153; + +$L__BB3_158: + setp.eq.s16 %p178, %rs32, 0; + mov.u32 %r6315, 0; + mov.u32 %r6832, %r6501; + @%p178 bra $L__BB3_396; + + cvt.u32.u16 %r3608, %rs31; + and.b32 %r6503, %r3608, 255; + cvt.u32.u16 %r3609, %rs32; + and.b32 %r6502, %r3609, 255; + +$L__BB3_160: + mov.u32 %r386, %r6502; + setp.gt.u32 %p179, %r6829, 2879; + mov.u32 %r6832, 1; + @%p179 bra $L__BB3_396; + + mov.u32 %r3612, 8; + sub.s32 %r3613, %r3612, %r6831; + sub.s32 %r3614, %r3613, %r6830; + min.u32 %r3615, %r3614, %r386; + setp.eq.s32 %p180, %r3615, 32; + mov.u32 %r3616, -1; + shl.b32 %r3617, %r3616, %r3615; + not.b32 %r3618, %r3617; + selp.b32 %r3619, -1, %r3618, %p180; + and.b32 %r3620, %r3619, %r6503; + shl.b32 %r3621, %r3620, %r6830; + cvt.u16.u32 %rs509, %r3621; + or.b16 %rs1025, %rs1025, %rs509; + add.s32 %r6830, %r3615, %r6830; + sub.s32 %r6502, %r386, %r3615; + shr.u32 %r6503, %r6503, %r3615; + setp.gt.u32 %p181, %r3614, %r386; + @%p181 bra $L__BB3_164; + + setp.ne.s32 %p182, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs510, %rs1025, 255; + setp.ne.s16 %p183, %rs510, 127; + and.pred %p184, %p182, %p183; + @%p184 bra $L__BB3_164; + + mov.u32 %r3624, 20548; + sub.s32 %r3625, %r3624, %r6829; + cvt.u64.u32 %rd135, %r3625; + add.s64 %rd136, %rd135, %rd5; + add.s64 %rd137, %rd1, %rd136; + st.global.u8 [%rd137], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p185, %rs510, 143; + selp.u32 %r6831, 1, 0, %p185; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_164: + setp.eq.s32 %p186, %r6502, 0; + mov.u32 %r6832, %r6501; + @%p186 bra $L__BB3_396; + bra.uni $L__BB3_160; + +$L__BB3_63: + setp.gt.u32 %p68, %r6381, 191; + mov.u32 %r6391, 1; + mov.u32 %r6390, 0; + @%p68 bra $L__BB3_65; + + st.global.u8 [%rd10], %rs956; + add.s32 %r6381, %r6381, 1; + mov.u16 %rs956, 0; + mov.u32 %r6390, 8; + mov.u32 %r6391, %r6598; + bra.uni $L__BB3_65; + +$L__BB3_234: + add.s32 %r6595, %r6595, 1; + setp.lt.u32 %p262, %r6595, %r6597; + @%p262 bra $L__BB3_242; + + shl.b16 %rs539, %rs956, 1; + or.b16 %rs956, %rs539, 1; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p263, %r6390, 0; + mov.u32 %r6591, %r6598; + @%p263 bra $L__BB3_238; + bra.uni $L__BB3_236; + +$L__BB3_238: + add.s32 %r3811, %r6596, 1; + min.u32 %r6596, %r3811, 12; + setp.lt.u32 %p266, %r6596, 3; + mov.u32 %r6595, 0; + mov.u32 %r6592, %r6595; + @%p266 bra $L__BB3_241; + + setp.lt.u32 %p267, %r6596, 6; + mov.u32 %r6592, 1; + @%p267 bra $L__BB3_241; + + setp.lt.u32 %p268, %r6596, 9; + setp.eq.s32 %p269, %r6596, 11; + selp.b32 %r3813, 4, 5, %p269; + setp.lt.u32 %p270, %r6596, 11; + selp.b32 %r3814, 3, %r3813, %p270; + selp.b32 %r6592, 2, %r3814, %p268; + +$L__BB3_241: + mov.u32 %r3816, 1; + shl.b32 %r6597, %r3816, %r6592; + mov.u32 %r6598, %r6591; + +$L__BB3_242: + max.s32 %r569, %r6534, 1; + and.b16 %rs542, %rs48, 15; + cvt.u32.u16 %r570, %rs542; + and.b32 %r571, %r6518, 1; + setp.eq.s32 %p271, %r571, 0; + mov.u32 %r6613, %r7050; + @%p271 bra $L__BB3_249; + + and.b32 %r3817, %r570, 1; + sub.s32 %r6599, %r569, %r3817; + setp.eq.s32 %p272, %r6599, 0; + mov.u32 %r6613, %r7050; + @%p272 bra $L__BB3_249; + + mov.u32 %r3818, -1; + shl.b32 %r3819, %r3818, %r6599; + not.b32 %r3820, %r3819; + and.b32 %r6600, %r6512, %r3820; + +$L__BB3_245: + setp.gt.u32 %p273, %r7016, 17476; + mov.u32 %r6613, 1; + @%p273 bra $L__BB3_249; + + sub.s32 %r3822, %r7017, %r7018; + min.u32 %r3823, %r3822, %r6599; + setp.eq.s32 %p274, %r3823, 32; + mov.u32 %r3824, -1; + shl.b32 %r3825, %r3824, %r3823; + not.b32 %r3826, %r3825; + selp.b32 %r3827, -1, %r3826, %p274; + and.b32 %r3828, %r3827, %r6600; + shl.b32 %r3829, %r3828, %r7018; + or.b32 %r7019, %r3829, %r7019; + add.s32 %r7018, %r3823, %r7018; + shr.u32 %r6600, %r6600, %r3823; + sub.s32 %r6599, %r6599, %r3823; + setp.lt.u32 %p275, %r7018, %r7017; + @%p275 bra $L__BB3_248; + + cvt.u64.u32 %rd183, %r7016; + add.s64 %rd184, %rd183, %rd5; + add.s64 %rd185, %rd1, %rd184; + st.global.u8 [%rd185], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p276, %r7019, 255; + selp.b32 %r7017, 7, 8, %p276; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_248: + setp.ne.s32 %p277, %r6599, 0; + mov.u32 %r6613, %r7050; + @%p277 bra $L__BB3_245; + +$L__BB3_249: + setp.eq.s32 %p278, %r457, 0; + mov.u32 %r6628, %r6613; + @%p278 bra $L__BB3_256; + + shr.u32 %r3832, %r570, 1; + and.b32 %r3833, %r3832, 1; + sub.s32 %r6614, %r569, %r3833; + setp.eq.s32 %p279, %r6614, 0; + mov.u32 %r6628, %r6613; + @%p279 bra $L__BB3_256; + + mov.u32 %r3834, -1; + shl.b32 %r3835, %r3834, %r6614; + not.b32 %r3836, %r3835; + and.b32 %r6615, %r6516, %r3836; + +$L__BB3_252: + setp.gt.u32 %p280, %r7016, 17476; + mov.u32 %r6628, 1; + @%p280 bra $L__BB3_256; + + sub.s32 %r3838, %r7017, %r7018; + min.u32 %r3839, %r3838, %r6614; + setp.eq.s32 %p281, %r3839, 32; + mov.u32 %r3840, -1; + shl.b32 %r3841, %r3840, %r3839; + not.b32 %r3842, %r3841; + selp.b32 %r3843, -1, %r3842, %p281; + and.b32 %r3844, %r3843, %r6615; + shl.b32 %r3845, %r3844, %r7018; + or.b32 %r7019, %r3845, %r7019; + add.s32 %r7018, %r3839, %r7018; + shr.u32 %r6615, %r6615, %r3839; + sub.s32 %r6614, %r6614, %r3839; + setp.lt.u32 %p282, %r7018, %r7017; + @%p282 bra $L__BB3_255; + + cvt.u64.u32 %rd186, %r7016; + add.s64 %rd187, %rd186, %rd5; + add.s64 %rd188, %rd1, %rd187; + st.global.u8 [%rd188], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p283, %r7019, 255; + selp.b32 %r7017, 7, 8, %p283; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_255: + setp.ne.s32 %p284, %r6614, 0; + mov.u32 %r6628, %r6613; + @%p284 bra $L__BB3_252; + +$L__BB3_256: + and.b32 %r3848, %r6518, 4; + setp.eq.s32 %p285, %r3848, 0; + mov.u32 %r6643, %r6628; + @%p285 bra $L__BB3_263; + + shr.u32 %r3849, %r570, 2; + and.b32 %r3850, %r3849, 1; + sub.s32 %r6629, %r569, %r3850; + setp.eq.s32 %p286, %r6629, 0; + mov.u32 %r6643, %r6628; + @%p286 bra $L__BB3_263; + + mov.u32 %r3851, -1; + shl.b32 %r3852, %r3851, %r6629; + not.b32 %r3853, %r3852; + and.b32 %r6630, %r6532, %r3853; + +$L__BB3_259: + setp.gt.u32 %p287, %r7016, 17476; + mov.u32 %r6643, 1; + @%p287 bra $L__BB3_263; + + sub.s32 %r3855, %r7017, %r7018; + min.u32 %r3856, %r3855, %r6629; + setp.eq.s32 %p288, %r3856, 32; + mov.u32 %r3857, -1; + shl.b32 %r3858, %r3857, %r3856; + not.b32 %r3859, %r3858; + selp.b32 %r3860, -1, %r3859, %p288; + and.b32 %r3861, %r3860, %r6630; + shl.b32 %r3862, %r3861, %r7018; + or.b32 %r7019, %r3862, %r7019; + add.s32 %r7018, %r3856, %r7018; + shr.u32 %r6630, %r6630, %r3856; + sub.s32 %r6629, %r6629, %r3856; + setp.lt.u32 %p289, %r7018, %r7017; + @%p289 bra $L__BB3_262; + + cvt.u64.u32 %rd189, %r7016; + add.s64 %rd190, %rd189, %rd5; + add.s64 %rd191, %rd1, %rd190; + st.global.u8 [%rd191], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p290, %r7019, 255; + selp.b32 %r7017, 7, 8, %p290; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_262: + setp.ne.s32 %p291, %r6629, 0; + mov.u32 %r6643, %r6628; + @%p291 bra $L__BB3_259; + +$L__BB3_263: + setp.eq.s32 %p292, %r458, 0; + mov.u32 %r7050, %r6643; + @%p292 bra $L__BB3_270; + + shr.u32 %r3865, %r570, 3; + sub.s32 %r6644, %r569, %r3865; + setp.eq.s32 %p293, %r6644, 0; + mov.u32 %r7050, %r6643; + @%p293 bra $L__BB3_270; + + mov.u32 %r3866, -1; + shl.b32 %r3867, %r3866, %r6644; + not.b32 %r3868, %r3867; + and.b32 %r6645, %r6531, %r3868; + +$L__BB3_266: + setp.gt.u32 %p294, %r7016, 17476; + mov.u32 %r7050, 1; + @%p294 bra $L__BB3_270; + + sub.s32 %r3870, %r7017, %r7018; + min.u32 %r3871, %r3870, %r6644; + setp.eq.s32 %p295, %r3871, 32; + mov.u32 %r3872, -1; + shl.b32 %r3873, %r3872, %r3871; + not.b32 %r3874, %r3873; + selp.b32 %r3875, -1, %r3874, %p295; + and.b32 %r3876, %r3875, %r6645; + shl.b32 %r3877, %r3876, %r7018; + or.b32 %r7019, %r3877, %r7019; + add.s32 %r7018, %r3871, %r7018; + shr.u32 %r6645, %r6645, %r3871; + sub.s32 %r6644, %r6644, %r3871; + setp.lt.u32 %p296, %r7018, %r7017; + @%p296 bra $L__BB3_269; + + cvt.u64.u32 %rd192, %r7016; + add.s64 %rd193, %rd192, %rd5; + add.s64 %rd194, %rd1, %rd193; + st.global.u8 [%rd194], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p297, %r7019, 255; + selp.b32 %r7017, 7, 8, %p297; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_269: + setp.ne.s32 %p298, %r6644, 0; + mov.u32 %r7050, %r6643; + @%p298 bra $L__BB3_266; + +$L__BB3_270: + setp.lt.s32 %p299, %r454, 1; + setp.lt.s32 %p300, %r113, 1; + or.pred %p301, %p300, %p299; + @%p301 bra $L__BB3_318; + + min.s32 %r3880, %r113, %r454; + setp.lt.s32 %p302, %r3880, 3; + add.s32 %r3881, %r6381, 17477; + cvt.u64.u32 %rd195, %r3881; + add.s64 %rd196, %rd195, %rd5; + add.s64 %rd14, %rd1, %rd196; + @%p302 bra $L__BB3_310; + bra.uni $L__BB3_272; + +$L__BB3_310: + add.s32 %r6595, %r6595, 1; + setp.lt.u32 %p349, %r6595, %r6597; + @%p349 bra $L__BB3_318; + + shl.b16 %rs559, %rs956, 1; + or.b16 %rs956, %rs559, 1; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p350, %r6390, 0; + mov.u32 %r6701, %r6598; + @%p350 bra $L__BB3_314; + bra.uni $L__BB3_312; + +$L__BB3_314: + add.s32 %r3969, %r6596, 1; + min.u32 %r6596, %r3969, 12; + setp.lt.u32 %p353, %r6596, 3; + mov.u32 %r6595, 0; + mov.u32 %r6702, %r6595; + @%p353 bra $L__BB3_317; + + setp.lt.u32 %p354, %r6596, 6; + mov.u32 %r6702, 1; + @%p354 bra $L__BB3_317; + + setp.lt.u32 %p355, %r6596, 9; + setp.eq.s32 %p356, %r6596, 11; + selp.b32 %r3971, 4, 5, %p356; + setp.lt.u32 %p357, %r6596, 11; + selp.b32 %r3972, 3, %r3971, %p357; + selp.b32 %r6702, 2, %r3972, %p355; + +$L__BB3_317: + mov.u32 %r3974, 1; + shl.b32 %r6597, %r3974, %r6702; + mov.u32 %r6598, %r6701; + bra.uni $L__BB3_318; + +$L__BB3_272: + shl.b16 %rs956, %rs956, 1; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p303, %r6390, 0; + mov.u32 %r6694, %r6598; + @%p303 bra $L__BB3_275; + bra.uni $L__BB3_273; + +$L__BB3_275: + setp.lt.u32 %p305, %r6596, 3; + mov.u32 %r6662, 0; + @%p305 bra $L__BB3_278; + + setp.lt.u32 %p306, %r6596, 6; + mov.u32 %r6662, 1; + @%p306 bra $L__BB3_278; + + setp.lt.u32 %p307, %r6596, 9; + setp.eq.s32 %p308, %r6596, 11; + selp.b32 %r3887, 4, 5, %p308; + setp.lt.u32 %p309, %r6596, 11; + selp.b32 %r3888, 3, %r3887, %p309; + selp.b32 %r6662, 2, %r3888, %p307; + +$L__BB3_278: + setp.eq.s32 %p310, %r6662, 0; + @%p310 bra $L__BB3_306; + + add.s32 %r671, %r6662, -1; + and.b32 %r672, %r6662, 3; + setp.eq.s32 %p311, %r672, 0; + mov.u32 %r6672, %r6662; + mov.u32 %r6673, %r6694; + @%p311 bra $L__BB3_291; + + mov.u32 %r3890, 1; + shl.b32 %r3891, %r3890, %r671; + and.b32 %r3892, %r3891, %r6595; + setp.ne.s32 %p312, %r3892, 0; + selp.u32 %r3893, 1, 0, %p312; + cvt.u32.u16 %r3894, %rs956; + bfi.b32 %r3895, %r3894, %r3893, 1, 8; + cvt.u16.u32 %rs956, %r3895; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p313, %r6390, 0; + mov.u32 %r6673, %r6694; + @%p313 bra $L__BB3_283; + + setp.gt.u32 %p314, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r6673, %r3890; + @%p314 bra $L__BB3_283; + + add.s32 %r3899, %r6381, 17477; + cvt.u64.u32 %rd197, %r3899; + add.s64 %rd198, %rd197, %rd5; + add.s64 %rd199, %rd1, %rd198; + st.global.u8 [%rd199], %rs956; + add.s32 %r6381, %r6381, 1; + mov.u16 %rs956, 0; + mov.u32 %r6390, 8; + mov.u32 %r6673, %r6694; + +$L__BB3_283: + setp.eq.s32 %p315, %r672, 1; + mov.u32 %r6694, %r6673; + mov.u32 %r6672, %r671; + @%p315 bra $L__BB3_291; + + add.s32 %r6672, %r6662, -2; + mov.u32 %r3900, 1; + shl.b32 %r3901, %r3900, %r6672; + and.b32 %r3902, %r3901, %r6595; + setp.ne.s32 %p316, %r3902, 0; + selp.u32 %r3903, 1, 0, %p316; + cvt.u32.u16 %r3904, %rs956; + bfi.b32 %r3905, %r3904, %r3903, 1, 8; + cvt.u16.u32 %rs956, %r3905; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p317, %r6390, 0; + mov.u32 %r6668, %r6673; + @%p317 bra $L__BB3_287; + + setp.gt.u32 %p318, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r6668, %r3900; + @%p318 bra $L__BB3_287; + + add.s32 %r3908, %r6381, 17477; + cvt.u64.u32 %rd200, %r3908; + add.s64 %rd201, %rd200, %rd5; + add.s64 %rd202, %rd1, %rd201; + and.b16 %rs547, %rs956, 255; + st.global.u8 [%rd202], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p319, %rs547, 255; + selp.b32 %r6390, 7, 8, %p319; + mov.u16 %rs956, 0; + mov.u32 %r6668, %r6673; + +$L__BB3_287: + setp.eq.s32 %p320, %r672, 2; + mov.u32 %r6694, %r6668; + mov.u32 %r6673, %r6668; + @%p320 bra $L__BB3_291; + + add.s32 %r6672, %r6662, -3; + mov.u32 %r3909, 1; + shl.b32 %r3910, %r3909, %r6672; + and.b32 %r3911, %r3910, %r6595; + setp.ne.s32 %p321, %r3911, 0; + selp.u32 %r3912, 1, 0, %p321; + cvt.u32.u16 %r3913, %rs956; + bfi.b32 %r3914, %r3913, %r3912, 1, 8; + cvt.u16.u32 %rs956, %r3914; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p322, %r6390, 0; + mov.u32 %r6694, %r6668; + mov.u32 %r6673, %r6668; + @%p322 bra $L__BB3_291; + + setp.gt.u32 %p323, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r6694, %r3909; + mov.u32 %r6673, %r3909; + @%p323 bra $L__BB3_291; + + add.s32 %r3919, %r6381, 17477; + cvt.u64.u32 %rd203, %r3919; + add.s64 %rd204, %rd203, %rd5; + add.s64 %rd205, %rd1, %rd204; + and.b16 %rs550, %rs956, 255; + st.global.u8 [%rd205], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p324, %rs550, 255; + selp.b32 %r6390, 7, 8, %p324; + mov.u16 %rs956, 0; + mov.u32 %r6694, %r6668; + mov.u32 %r6673, %r6668; + +$L__BB3_291: + setp.lt.u32 %p325, %r671, 3; + @%p325 bra $L__BB3_306; + + mov.u32 %r6694, %r6673; + +$L__BB3_293: + add.s32 %r3920, %r6672, -1; + mov.u32 %r3921, 1; + shl.b32 %r3922, %r3921, %r3920; + and.b32 %r3923, %r3922, %r6595; + setp.ne.s32 %p326, %r3923, 0; + selp.u32 %r3924, 1, 0, %p326; + cvt.u32.u16 %r3925, %rs956; + bfi.b32 %r6682, %r3925, %r3924, 1, 8; + add.s32 %r6681, %r6390, -1; + setp.ne.s32 %p327, %r6681, 0; + mov.u32 %r6683, %r6694; + @%p327 bra $L__BB3_296; + + setp.gt.u32 %p328, %r6381, 191; + mov.u32 %r6681, 0; + mov.u32 %r6683, %r3921; + @%p328 bra $L__BB3_296; + + cvt.u16.u32 %rs551, %r6682; + and.b16 %rs552, %rs551, 255; + add.s32 %r3929, %r6381, 17477; + cvt.u64.u32 %rd206, %r3929; + add.s64 %rd207, %rd206, %rd5; + add.s64 %rd208, %rd1, %rd207; + st.global.u8 [%rd208], %rs551; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p329, %rs552, 255; + selp.b32 %r6681, 7, 8, %p329; + mov.u32 %r6682, 0; + mov.u32 %r6683, %r6694; + +$L__BB3_296: + add.s32 %r3930, %r6672, -2; + shl.b32 %r3932, %r3921, %r3930; + and.b32 %r3933, %r3932, %r6595; + setp.ne.s32 %p330, %r3933, 0; + and.b32 %r3934, %r6682, 127; + selp.u32 %r3935, 1, 0, %p330; + bfi.b32 %r6686, %r3934, %r3935, 1, 7; + add.s32 %r6685, %r6681, -1; + setp.ne.s32 %p331, %r6685, 0; + mov.u32 %r6687, %r6683; + @%p331 bra $L__BB3_299; + + setp.gt.u32 %p332, %r6381, 191; + mov.u32 %r6687, 1; + mov.u32 %r6685, 0; + @%p332 bra $L__BB3_299; + + cvt.u16.u32 %rs553, %r6686; + and.b16 %rs554, %rs553, 255; + add.s32 %r3939, %r6381, 17477; + cvt.u64.u32 %rd209, %r3939; + add.s64 %rd210, %rd209, %rd5; + add.s64 %rd211, %rd1, %rd210; + st.global.u8 [%rd211], %rs553; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p333, %rs554, 255; + selp.b32 %r6685, 7, 8, %p333; + mov.u32 %r6686, 0; + mov.u32 %r6687, %r6683; + +$L__BB3_299: + add.s32 %r3940, %r6672, -3; + mov.u32 %r3941, 1; + shl.b32 %r3942, %r3941, %r3940; + and.b32 %r3943, %r3942, %r6595; + setp.ne.s32 %p334, %r3943, 0; + and.b32 %r3944, %r6686, 127; + selp.u32 %r3945, 1, 0, %p334; + bfi.b32 %r6690, %r3944, %r3945, 1, 7; + add.s32 %r6689, %r6685, -1; + setp.ne.s32 %p335, %r6689, 0; + mov.u32 %r6691, %r6687; + @%p335 bra $L__BB3_302; + + setp.gt.u32 %p336, %r6381, 191; + mov.u32 %r6689, 0; + mov.u32 %r6691, %r3941; + @%p336 bra $L__BB3_302; + + cvt.u16.u32 %rs555, %r6690; + and.b16 %rs556, %rs555, 255; + add.s32 %r3949, %r6381, 17477; + cvt.u64.u32 %rd212, %r3949; + add.s64 %rd213, %rd212, %rd5; + add.s64 %rd214, %rd1, %rd213; + st.global.u8 [%rd214], %rs555; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p337, %rs556, 255; + selp.b32 %r6689, 7, 8, %p337; + mov.u32 %r6690, 0; + mov.u32 %r6691, %r6687; + +$L__BB3_302: + add.s32 %r6672, %r6672, -4; + shl.b32 %r3951, %r3941, %r6672; + and.b32 %r3952, %r3951, %r6595; + setp.ne.s32 %p338, %r3952, 0; + and.b32 %r3953, %r6690, 127; + selp.u32 %r3954, 1, 0, %p338; + bfi.b32 %r3955, %r3953, %r3954, 1, 15; + cvt.u16.u32 %rs956, %r3955; + add.s32 %r6390, %r6689, -1; + setp.ne.s32 %p339, %r6390, 0; + mov.u32 %r6694, %r6691; + @%p339 bra $L__BB3_305; + + setp.gt.u32 %p340, %r6381, 191; + mov.u32 %r6694, 1; + mov.u32 %r6390, 0; + @%p340 bra $L__BB3_305; + + add.s32 %r3958, %r6381, 17477; + cvt.u64.u32 %rd215, %r3958; + add.s64 %rd216, %rd215, %rd5; + add.s64 %rd217, %rd1, %rd216; + and.b16 %rs558, %rs956, 255; + st.global.u8 [%rd217], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p341, %rs558, 255; + selp.b32 %r6390, 7, 8, %p341; + mov.u16 %rs956, 0; + mov.u32 %r6694, %r6691; + +$L__BB3_305: + setp.ne.s32 %p342, %r6672, 0; + @%p342 bra $L__BB3_293; + +$L__BB3_306: + add.s32 %r3960, %r6596, -1; + setp.eq.s32 %p343, %r6596, 0; + mov.u32 %r6595, 0; + selp.b32 %r6596, 0, %r3960, %p343; + setp.lt.u32 %p344, %r6596, 3; + mov.u32 %r6698, %r6595; + @%p344 bra $L__BB3_309; + + setp.lt.u32 %p345, %r6596, 6; + mov.u32 %r6698, 1; + @%p345 bra $L__BB3_309; + + setp.lt.u32 %p346, %r6596, 9; + setp.eq.s32 %p347, %r6596, 11; + selp.b32 %r3962, 4, 5, %p347; + setp.lt.u32 %p348, %r6596, 11; + selp.b32 %r3963, 3, %r3962, %p348; + selp.b32 %r6698, 2, %r3963, %p346; + +$L__BB3_309: + mov.u32 %r3965, 1; + shl.b32 %r6597, %r3965, %r6698; + mov.u32 %r6598, %r6694; + +$L__BB3_318: + setp.gt.s32 %p358, %r454, 2; + setp.gt.s32 %p359, %r113, 2; + and.pred %p360, %p359, %p358; + @%p360 bra $L__BB3_367; + bra.uni $L__BB3_319; + +$L__BB3_367: + add.s32 %r4095, %r326, -11; + cvt.u64.u32 %rd247, %r4095; + add.s64 %rd16, %rd9, %rd247; + ld.global.u8 %rs122, [%rd16]; + add.s32 %r4096, %r326, -10; + cvt.u64.u32 %rd249, %r4096; + add.s64 %rd250, %rd9, %rd249; + ld.global.u8 %rs123, [%rd250]; + ld.global.u8 %rs124, [%rd250+1]; + mul.lo.s32 %r4097, %r454, 6; + add.s32 %r4098, %r4097, -12; + cvt.u64.u32 %rd251, %r4098; + add.s64 %rd252, %rd9, %rd251; + ld.global.u8 %rs125, [%rd252]; + ld.global.u8 %rs126, [%rd252+1]; + add.s32 %r4099, %r4097, -10; + cvt.u64.u32 %rd253, %r4099; + add.s64 %rd254, %rd9, %rd253; + ld.global.u8 %rs127, [%rd254]; + ld.global.u8 %rs128, [%rd254+1]; + setp.eq.s16 %p428, %rs122, 0; + mov.u32 %r6796, %r6548; + @%p428 bra $L__BB3_374; + + ld.global.u8 %r6786, [%rd16+-1]; + cvt.u32.u16 %r6785, %rs122; + +$L__BB3_369: + mov.u32 %r881, %r6785; + setp.gt.u32 %p429, %r6829, 2879; + mov.u32 %r6796, 1; + @%p429 bra $L__BB3_374; + + mov.u32 %r4101, 8; + sub.s32 %r4102, %r4101, %r6831; + sub.s32 %r4103, %r4102, %r6830; + min.u32 %r4104, %r4103, %r881; + setp.eq.s32 %p430, %r4104, 32; + mov.u32 %r4105, -1; + shl.b32 %r4106, %r4105, %r4104; + not.b32 %r4107, %r4106; + selp.b32 %r4108, -1, %r4107, %p430; + and.b32 %r4109, %r4108, %r6786; + shl.b32 %r4110, %r4109, %r6830; + cvt.u16.u32 %rs594, %r4110; + or.b16 %rs1025, %rs1025, %rs594; + add.s32 %r6830, %r4104, %r6830; + sub.s32 %r6785, %r881, %r4104; + shr.u32 %r6786, %r6786, %r4104; + setp.gt.u32 %p431, %r4103, %r881; + @%p431 bra $L__BB3_373; + + setp.ne.s32 %p432, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs595, %rs1025, 255; + setp.ne.s16 %p433, %rs595, 127; + and.pred %p434, %p432, %p433; + @%p434 bra $L__BB3_373; + + mov.u32 %r4113, 20548; + sub.s32 %r4114, %r4113, %r6829; + cvt.u64.u32 %rd255, %r4114; + add.s64 %rd256, %rd255, %rd5; + add.s64 %rd257, %rd1, %rd256; + st.global.u8 [%rd257], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p435, %rs595, 143; + selp.u32 %r6831, 1, 0, %p435; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_373: + setp.ne.s32 %p436, %r6785, 0; + mov.u32 %r6796, %r6548; + @%p436 bra $L__BB3_369; + +$L__BB3_374: + setp.eq.s16 %p437, %rs126, 0; + mov.u32 %r6808, %r6796; + @%p437 bra $L__BB3_381; + + cvt.u32.u16 %r4115, %rs125; + and.b32 %r6798, %r4115, 255; + cvt.u32.u16 %r4116, %rs126; + and.b32 %r6797, %r4116, 255; + +$L__BB3_376: + mov.u32 %r900, %r6797; + setp.gt.u32 %p438, %r6829, 2879; + mov.u32 %r6808, 1; + @%p438 bra $L__BB3_381; + + mov.u32 %r4118, 8; + sub.s32 %r4119, %r4118, %r6831; + sub.s32 %r4120, %r4119, %r6830; + min.u32 %r4121, %r4120, %r900; + setp.eq.s32 %p439, %r4121, 32; + mov.u32 %r4122, -1; + shl.b32 %r4123, %r4122, %r4121; + not.b32 %r4124, %r4123; + selp.b32 %r4125, -1, %r4124, %p439; + and.b32 %r4126, %r4125, %r6798; + shl.b32 %r4127, %r4126, %r6830; + cvt.u16.u32 %rs599, %r4127; + or.b16 %rs1025, %rs1025, %rs599; + add.s32 %r6830, %r4121, %r6830; + sub.s32 %r6797, %r900, %r4121; + shr.u32 %r6798, %r6798, %r4121; + setp.gt.u32 %p440, %r4120, %r900; + @%p440 bra $L__BB3_380; + + setp.ne.s32 %p441, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs600, %rs1025, 255; + setp.ne.s16 %p442, %rs600, 127; + and.pred %p443, %p441, %p442; + @%p443 bra $L__BB3_380; + + mov.u32 %r4130, 20548; + sub.s32 %r4131, %r4130, %r6829; + cvt.u64.u32 %rd258, %r4131; + add.s64 %rd259, %rd258, %rd5; + add.s64 %rd260, %rd1, %rd259; + st.global.u8 [%rd260], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p444, %rs600, 143; + selp.u32 %r6831, 1, 0, %p444; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_380: + setp.ne.s32 %p445, %r6797, 0; + mov.u32 %r6808, %r6796; + @%p445 bra $L__BB3_376; + +$L__BB3_381: + setp.eq.s16 %p446, %rs124, 0; + mov.u32 %r6820, %r6808; + @%p446 bra $L__BB3_388; + + cvt.u32.u16 %r4132, %rs124; + and.b32 %r6809, %r4132, 255; + cvt.u32.u16 %r4133, %rs123; + and.b32 %r6810, %r4133, 255; + +$L__BB3_383: + mov.u32 %r919, %r6809; + setp.gt.u32 %p447, %r6829, 2879; + mov.u32 %r6820, 1; + @%p447 bra $L__BB3_388; + + mov.u32 %r4135, 8; + sub.s32 %r4136, %r4135, %r6831; + sub.s32 %r4137, %r4136, %r6830; + min.u32 %r4138, %r4137, %r919; + setp.eq.s32 %p448, %r4138, 32; + mov.u32 %r4139, -1; + shl.b32 %r4140, %r4139, %r4138; + not.b32 %r4141, %r4140; + selp.b32 %r4142, -1, %r4141, %p448; + and.b32 %r4143, %r4142, %r6810; + shl.b32 %r4144, %r4143, %r6830; + cvt.u16.u32 %rs604, %r4144; + or.b16 %rs1025, %rs1025, %rs604; + add.s32 %r6830, %r4138, %r6830; + sub.s32 %r6809, %r919, %r4138; + shr.u32 %r6810, %r6810, %r4138; + setp.gt.u32 %p449, %r4137, %r919; + @%p449 bra $L__BB3_387; + + setp.ne.s32 %p450, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs605, %rs1025, 255; + setp.ne.s16 %p451, %rs605, 127; + and.pred %p452, %p450, %p451; + @%p452 bra $L__BB3_387; + + mov.u32 %r4147, 20548; + sub.s32 %r4148, %r4147, %r6829; + cvt.u64.u32 %rd261, %r4148; + add.s64 %rd262, %rd261, %rd5; + add.s64 %rd263, %rd1, %rd262; + st.global.u8 [%rd263], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p453, %rs605, 143; + selp.u32 %r6831, 1, 0, %p453; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_387: + setp.ne.s32 %p454, %r6809, 0; + mov.u32 %r6820, %r6808; + @%p454 bra $L__BB3_383; + +$L__BB3_388: + setp.eq.s16 %p455, %rs128, 0; + mov.u32 %r6832, %r6820; + @%p455 bra $L__BB3_395; + + cvt.u32.u16 %r4149, %rs127; + and.b32 %r6822, %r4149, 255; + cvt.u32.u16 %r4150, %rs128; + and.b32 %r6821, %r4150, 255; + +$L__BB3_390: + mov.u32 %r938, %r6821; + setp.gt.u32 %p456, %r6829, 2879; + mov.u32 %r6832, 1; + @%p456 bra $L__BB3_395; + + mov.u32 %r4152, 8; + sub.s32 %r4153, %r4152, %r6831; + sub.s32 %r4154, %r4153, %r6830; + min.u32 %r4155, %r4154, %r938; + setp.eq.s32 %p457, %r4155, 32; + mov.u32 %r4156, -1; + shl.b32 %r4157, %r4156, %r4155; + not.b32 %r4158, %r4157; + selp.b32 %r4159, -1, %r4158, %p457; + and.b32 %r4160, %r4159, %r6822; + shl.b32 %r4161, %r4160, %r6830; + cvt.u16.u32 %rs609, %r4161; + or.b16 %rs1025, %rs1025, %rs609; + add.s32 %r6830, %r4155, %r6830; + sub.s32 %r6821, %r938, %r4155; + shr.u32 %r6822, %r6822, %r4155; + setp.gt.u32 %p458, %r4154, %r938; + @%p458 bra $L__BB3_394; + + setp.ne.s32 %p459, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs610, %rs1025, 255; + setp.ne.s16 %p460, %rs610, 127; + and.pred %p461, %p459, %p460; + @%p461 bra $L__BB3_394; + + mov.u32 %r4164, 20548; + sub.s32 %r4165, %r4164, %r6829; + cvt.u64.u32 %rd264, %r4165; + add.s64 %rd265, %rd264, %rd5; + add.s64 %rd266, %rd1, %rd265; + st.global.u8 [%rd266], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p462, %rs610, 143; + selp.u32 %r6831, 1, 0, %p462; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_394: + setp.ne.s32 %p463, %r6821, 0; + mov.u32 %r6832, %r6820; + @%p463 bra $L__BB3_390; + bra.uni $L__BB3_395; + +$L__BB3_319: + setp.gt.s32 %p361, %r454, 0; + and.pred %p363, %p359, %p361; + @%p363 bra $L__BB3_348; + bra.uni $L__BB3_320; + +$L__BB3_348: + ld.global.u8 %rs108, [%rd11+1]; + ld.global.u8 %rs109, [%rd12]; + ld.global.u8 %rs110, [%rd12+1]; + setp.eq.s16 %p402, %rs108, 0; + mov.u32 %r6764, %r6548; + @%p402 bra $L__BB3_355; + + ld.global.u8 %r6754, [%rd11]; + cvt.u32.u16 %r6753, %rs108; + +$L__BB3_350: + mov.u32 %r829, %r6753; + setp.gt.u32 %p403, %r6829, 2879; + mov.u32 %r6764, 1; + @%p403 bra $L__BB3_355; + + mov.u32 %r4047, 8; + sub.s32 %r4048, %r4047, %r6831; + sub.s32 %r4049, %r4048, %r6830; + min.u32 %r4050, %r4049, %r829; + setp.eq.s32 %p404, %r4050, 32; + mov.u32 %r4051, -1; + shl.b32 %r4052, %r4051, %r4050; + not.b32 %r4053, %r4052; + selp.b32 %r4054, -1, %r4053, %p404; + and.b32 %r4055, %r4054, %r6754; + shl.b32 %r4056, %r4055, %r6830; + cvt.u16.u32 %rs581, %r4056; + or.b16 %rs1025, %rs1025, %rs581; + add.s32 %r6830, %r4050, %r6830; + sub.s32 %r6753, %r829, %r4050; + shr.u32 %r6754, %r6754, %r4050; + setp.gt.u32 %p405, %r4049, %r829; + @%p405 bra $L__BB3_354; + + setp.ne.s32 %p406, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs582, %rs1025, 255; + setp.ne.s16 %p407, %rs582, 127; + and.pred %p408, %p406, %p407; + @%p408 bra $L__BB3_354; + + mov.u32 %r4059, 20548; + sub.s32 %r4060, %r4059, %r6829; + cvt.u64.u32 %rd238, %r4060; + add.s64 %rd239, %rd238, %rd5; + add.s64 %rd240, %rd1, %rd239; + st.global.u8 [%rd240], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p409, %rs582, 143; + selp.u32 %r6831, 1, 0, %p409; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_354: + setp.ne.s32 %p410, %r6753, 0; + mov.u32 %r6764, %r6548; + @%p410 bra $L__BB3_350; + +$L__BB3_355: + add.s32 %r6766, %r454, -1; + cvt.u32.u16 %r4062, %rs110; + and.b32 %r6777, %r4062, 255; + cvt.u32.u16 %r4063, %rs109; + and.b32 %r6778, %r4063, 255; + mov.u32 %r4061, 1; + mov.u32 %r6765, %r4061; + +$L__BB3_356: + mov.u32 %r849, %r6765; + setp.gt.u32 %p411, %r6829, 2879; + mov.u32 %r6776, %r4061; + @%p411 bra $L__BB3_361; + + mov.u32 %r4065, 8; + sub.s32 %r4066, %r4065, %r6831; + sub.s32 %r4067, %r4066, %r6830; + min.u32 %r4068, %r4067, %r849; + setp.eq.s32 %p412, %r4068, 32; + mov.u32 %r4069, -1; + shl.b32 %r4070, %r4069, %r4068; + not.b32 %r4071, %r4070; + selp.b32 %r4072, -1, %r4071, %p412; + and.b32 %r4073, %r4072, %r6766; + shl.b32 %r4074, %r4073, %r6830; + cvt.u16.u32 %rs585, %r4074; + or.b16 %rs1025, %rs1025, %rs585; + add.s32 %r6830, %r4068, %r6830; + sub.s32 %r6765, %r849, %r4068; + shr.u32 %r6766, %r6766, %r4068; + setp.gt.u32 %p413, %r4067, %r849; + @%p413 bra $L__BB3_360; + + setp.ne.s32 %p414, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs586, %rs1025, 255; + setp.ne.s16 %p415, %rs586, 127; + and.pred %p416, %p414, %p415; + @%p416 bra $L__BB3_360; + + mov.u32 %r4077, 20548; + sub.s32 %r4078, %r4077, %r6829; + cvt.u64.u32 %rd241, %r4078; + add.s64 %rd242, %rd241, %rd5; + add.s64 %rd243, %rd1, %rd242; + st.global.u8 [%rd243], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p417, %rs586, 143; + selp.u32 %r6831, 1, 0, %p417; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_360: + setp.ne.s32 %p418, %r6765, 0; + mov.u32 %r6776, %r6764; + @%p418 bra $L__BB3_356; + +$L__BB3_361: + setp.eq.s16 %p419, %rs110, 0; + mov.u32 %r6832, %r6776; + @%p419 bra $L__BB3_395; + +$L__BB3_362: + mov.u32 %r866, %r6777; + setp.gt.u32 %p420, %r6829, 2879; + mov.u32 %r6832, 1; + @%p420 bra $L__BB3_395; + + mov.u32 %r4080, 8; + sub.s32 %r4081, %r4080, %r6831; + sub.s32 %r4082, %r4081, %r6830; + min.u32 %r4083, %r4082, %r866; + setp.eq.s32 %p421, %r4083, 32; + mov.u32 %r4084, -1; + shl.b32 %r4085, %r4084, %r4083; + not.b32 %r4086, %r4085; + selp.b32 %r4087, -1, %r4086, %p421; + and.b32 %r4088, %r4087, %r6778; + shl.b32 %r4089, %r4088, %r6830; + cvt.u16.u32 %rs590, %r4089; + or.b16 %rs1025, %rs1025, %rs590; + add.s32 %r6830, %r4083, %r6830; + sub.s32 %r6777, %r866, %r4083; + shr.u32 %r6778, %r6778, %r4083; + setp.gt.u32 %p422, %r4082, %r866; + @%p422 bra $L__BB3_366; + + setp.ne.s32 %p423, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs591, %rs1025, 255; + setp.ne.s16 %p424, %rs591, 127; + and.pred %p425, %p423, %p424; + @%p425 bra $L__BB3_366; + + mov.u32 %r4092, 20548; + sub.s32 %r4093, %r4092, %r6829; + cvt.u64.u32 %rd244, %r4093; + add.s64 %rd245, %rd244, %rd5; + add.s64 %rd246, %rd1, %rd245; + st.global.u8 [%rd246], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p426, %rs591, 143; + selp.u32 %r6831, 1, 0, %p426; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_366: + setp.eq.s32 %p427, %r6777, 0; + mov.u32 %r6832, %r6776; + @%p427 bra $L__BB3_395; + bra.uni $L__BB3_362; + +$L__BB3_320: + setp.gt.s32 %p365, %r113, 0; + selp.b32 %r3975, %r326, 0, %p365; + cvt.u64.u32 %rd218, %r3975; + add.s64 %rd15, %rd9, %rd218; + ld.global.u8 %rs86, [%rd15+1]; + add.s32 %r3976, %r3975, 2; + cvt.u64.u32 %rd220, %r3976; + add.s64 %rd221, %rd9, %rd220; + ld.global.u8 %rs87, [%rd221]; + ld.global.u8 %rs88, [%rd221+1]; + mul.lo.s32 %r3977, %r454, 6; + selp.b32 %r3978, %r3977, 0, %p361; + cvt.u64.u32 %rd222, %r3978; + add.s64 %rd223, %rd9, %rd222; + ld.global.u8 %rs89, [%rd223]; + ld.global.u8 %rs90, [%rd223+1]; + add.s32 %r3979, %r3978, 2; + cvt.u64.u32 %rd224, %r3979; + add.s64 %rd225, %rd9, %rd224; + ld.global.u8 %rs91, [%rd225]; + ld.global.u8 %rs92, [%rd225+1]; + setp.eq.s16 %p366, %rs86, 0; + mov.u32 %r6720, %r6548; + @%p366 bra $L__BB3_327; + + ld.global.u8 %r6710, [%rd15]; + cvt.u32.u16 %r6709, %rs86; + +$L__BB3_322: + mov.u32 %r757, %r6709; + setp.gt.u32 %p367, %r6829, 2879; + mov.u32 %r6720, 1; + @%p367 bra $L__BB3_327; + + mov.u32 %r3981, 8; + sub.s32 %r3982, %r3981, %r6831; + sub.s32 %r3983, %r3982, %r6830; + min.u32 %r3984, %r3983, %r757; + setp.eq.s32 %p368, %r3984, 32; + mov.u32 %r3985, -1; + shl.b32 %r3986, %r3985, %r3984; + not.b32 %r3987, %r3986; + selp.b32 %r3988, -1, %r3987, %p368; + and.b32 %r3989, %r3988, %r6710; + shl.b32 %r3990, %r3989, %r6830; + cvt.u16.u32 %rs562, %r3990; + or.b16 %rs1025, %rs1025, %rs562; + add.s32 %r6830, %r3984, %r6830; + sub.s32 %r6709, %r757, %r3984; + shr.u32 %r6710, %r6710, %r3984; + setp.gt.u32 %p369, %r3983, %r757; + @%p369 bra $L__BB3_326; + + setp.ne.s32 %p370, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs563, %rs1025, 255; + setp.ne.s16 %p371, %rs563, 127; + and.pred %p372, %p370, %p371; + @%p372 bra $L__BB3_326; + + mov.u32 %r3993, 20548; + sub.s32 %r3994, %r3993, %r6829; + cvt.u64.u32 %rd226, %r3994; + add.s64 %rd227, %rd226, %rd5; + add.s64 %rd228, %rd1, %rd227; + st.global.u8 [%rd228], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p373, %rs563, 143; + selp.u32 %r6831, 1, 0, %p373; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_326: + setp.ne.s32 %p374, %r6709, 0; + mov.u32 %r6720, %r6548; + @%p374 bra $L__BB3_322; + +$L__BB3_327: + setp.eq.s16 %p375, %rs90, 0; + mov.u32 %r6732, %r6720; + @%p375 bra $L__BB3_334; + + cvt.u32.u16 %r3995, %rs89; + and.b32 %r6722, %r3995, 255; + cvt.u32.u16 %r3996, %rs90; + and.b32 %r6721, %r3996, 255; + +$L__BB3_329: + mov.u32 %r776, %r6721; + setp.gt.u32 %p376, %r6829, 2879; + mov.u32 %r6732, 1; + @%p376 bra $L__BB3_334; + + mov.u32 %r3998, 8; + sub.s32 %r3999, %r3998, %r6831; + sub.s32 %r4000, %r3999, %r6830; + min.u32 %r4001, %r4000, %r776; + setp.eq.s32 %p377, %r4001, 32; + mov.u32 %r4002, -1; + shl.b32 %r4003, %r4002, %r4001; + not.b32 %r4004, %r4003; + selp.b32 %r4005, -1, %r4004, %p377; + and.b32 %r4006, %r4005, %r6722; + shl.b32 %r4007, %r4006, %r6830; + cvt.u16.u32 %rs567, %r4007; + or.b16 %rs1025, %rs1025, %rs567; + add.s32 %r6830, %r4001, %r6830; + sub.s32 %r6721, %r776, %r4001; + shr.u32 %r6722, %r6722, %r4001; + setp.gt.u32 %p378, %r4000, %r776; + @%p378 bra $L__BB3_333; + + setp.ne.s32 %p379, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs568, %rs1025, 255; + setp.ne.s16 %p380, %rs568, 127; + and.pred %p381, %p379, %p380; + @%p381 bra $L__BB3_333; + + mov.u32 %r4010, 20548; + sub.s32 %r4011, %r4010, %r6829; + cvt.u64.u32 %rd229, %r4011; + add.s64 %rd230, %rd229, %rd5; + add.s64 %rd231, %rd1, %rd230; + st.global.u8 [%rd231], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p382, %rs568, 143; + selp.u32 %r6831, 1, 0, %p382; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_333: + setp.ne.s32 %p383, %r6721, 0; + mov.u32 %r6732, %r6720; + @%p383 bra $L__BB3_329; + +$L__BB3_334: + setp.eq.s16 %p384, %rs88, 0; + mov.u32 %r6744, %r6732; + @%p384 bra $L__BB3_341; + + cvt.u32.u16 %r4012, %rs88; + and.b32 %r6733, %r4012, 255; + cvt.u32.u16 %r4013, %rs87; + and.b32 %r6734, %r4013, 255; + +$L__BB3_336: + mov.u32 %r795, %r6733; + setp.gt.u32 %p385, %r6829, 2879; + mov.u32 %r6744, 1; + @%p385 bra $L__BB3_341; + + mov.u32 %r4015, 8; + sub.s32 %r4016, %r4015, %r6831; + sub.s32 %r4017, %r4016, %r6830; + min.u32 %r4018, %r4017, %r795; + setp.eq.s32 %p386, %r4018, 32; + mov.u32 %r4019, -1; + shl.b32 %r4020, %r4019, %r4018; + not.b32 %r4021, %r4020; + selp.b32 %r4022, -1, %r4021, %p386; + and.b32 %r4023, %r4022, %r6734; + shl.b32 %r4024, %r4023, %r6830; + cvt.u16.u32 %rs572, %r4024; + or.b16 %rs1025, %rs1025, %rs572; + add.s32 %r6830, %r4018, %r6830; + sub.s32 %r6733, %r795, %r4018; + shr.u32 %r6734, %r6734, %r4018; + setp.gt.u32 %p387, %r4017, %r795; + @%p387 bra $L__BB3_340; + + setp.ne.s32 %p388, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs573, %rs1025, 255; + setp.ne.s16 %p389, %rs573, 127; + and.pred %p390, %p388, %p389; + @%p390 bra $L__BB3_340; + + mov.u32 %r4027, 20548; + sub.s32 %r4028, %r4027, %r6829; + cvt.u64.u32 %rd232, %r4028; + add.s64 %rd233, %rd232, %rd5; + add.s64 %rd234, %rd1, %rd233; + st.global.u8 [%rd234], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p391, %rs573, 143; + selp.u32 %r6831, 1, 0, %p391; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_340: + setp.ne.s32 %p392, %r6733, 0; + mov.u32 %r6744, %r6732; + @%p392 bra $L__BB3_336; + +$L__BB3_341: + setp.eq.s16 %p393, %rs92, 0; + mov.u32 %r6832, %r6744; + @%p393 bra $L__BB3_395; + + cvt.u32.u16 %r4029, %rs91; + and.b32 %r6746, %r4029, 255; + cvt.u32.u16 %r4030, %rs92; + and.b32 %r6745, %r4030, 255; + +$L__BB3_343: + mov.u32 %r814, %r6745; + setp.gt.u32 %p394, %r6829, 2879; + mov.u32 %r6832, 1; + @%p394 bra $L__BB3_395; + + mov.u32 %r4032, 8; + sub.s32 %r4033, %r4032, %r6831; + sub.s32 %r4034, %r4033, %r6830; + min.u32 %r4035, %r4034, %r814; + setp.eq.s32 %p395, %r4035, 32; + mov.u32 %r4036, -1; + shl.b32 %r4037, %r4036, %r4035; + not.b32 %r4038, %r4037; + selp.b32 %r4039, -1, %r4038, %p395; + and.b32 %r4040, %r4039, %r6746; + shl.b32 %r4041, %r4040, %r6830; + cvt.u16.u32 %rs577, %r4041; + or.b16 %rs1025, %rs1025, %rs577; + add.s32 %r6830, %r4035, %r6830; + sub.s32 %r6745, %r814, %r4035; + shr.u32 %r6746, %r6746, %r4035; + setp.gt.u32 %p396, %r4034, %r814; + @%p396 bra $L__BB3_347; + + setp.ne.s32 %p397, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs578, %rs1025, 255; + setp.ne.s16 %p398, %rs578, 127; + and.pred %p399, %p397, %p398; + @%p399 bra $L__BB3_347; + + mov.u32 %r4044, 20548; + sub.s32 %r4045, %r4044, %r6829; + cvt.u64.u32 %rd235, %r4045; + add.s64 %rd236, %rd235, %rd5; + add.s64 %rd237, %rd1, %rd236; + st.global.u8 [%rd237], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p400, %rs578, 143; + selp.u32 %r6831, 1, 0, %p400; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_347: + setp.eq.s32 %p401, %r6745, 0; + mov.u32 %r6832, %r6744; + @%p401 bra $L__BB3_395; + bra.uni $L__BB3_343; + +$L__BB3_395: + shr.u32 %r4166, %r6518, 1; + or.b32 %r6315, %r4166, %r571; + +$L__BB3_396: + add.s32 %r6299, %r6299, 4; + setp.lt.u32 %p464, %r6299, %r3200; + @%p464 bra $L__BB3_31; + +$L__BB3_397: + add.s32 %r989, %r28, 1; + setp.gt.u32 %p465, %r989, 512; + @%p465 bra $L__BB3_399; + + mov.u32 %r4167, _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val; + add.s32 %r4168, %r4167, %r989; + mov.u16 %rs613, 0; + st.shared.u8 [%r4168], %rs613; + +$L__BB3_399: + setp.lt.u32 %p466, %r3201, 3; + @%p466 bra $L__BB3_645; + + ld.param.u64 %rd684, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_param_3]; + mov.u32 %r6865, 2; + cvta.to.global.u64 %rd17, %rd48; + cvta.to.global.u64 %rd18, %rd684; + +$L__BB3_401: + ld.shared.u8 %rs151, [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val]; + mov.u16 %rs614, 0; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val], %rs614; + ld.shared.u8 %rs152, [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val]; + st.shared.u8 [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val], %rs614; + @%p2 bra $L__BB3_644; + + mov.u32 %r4172, 0; + ld.shared.u8 %rs615, [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val+1]; + ld.shared.u8 %rs616, [_ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val+1]; + max.u16 %rs618, %rs151, %rs615; + cvt.u32.u16 %r4173, %rs618; + add.s32 %r6900, %r4173, -1; + add.s32 %r1007, %r6865, 1; + mul.lo.s32 %r6898, %r6865, %r3198; + mul.wide.u16 %r4174, %rs616, 4; + cvt.u32.u16 %r4175, %rs152; + and.b32 %r4176, %r4175, 255; + add.s32 %r6897, %r4174, %r4176; + mov.u32 %r6881, %r4172; + mov.u32 %r6899, %r4172; + mov.u32 %r6901, %r4172; + +$L__BB3_403: + cvt.u64.u32 %rd267, %r6898; + add.s64 %rd268, %rd267, %rd4; + shl.b64 %rd269, %rd268, 2; + add.s64 %rd270, %rd3, %rd269; + ld.global.u32 %r1031, [%rd270]; + setp.eq.s32 %p468, %r1031, 0; + mov.u32 %r6902, %r4172; + @%p468 bra $L__BB3_405; + + and.b32 %r4178, %r1031, -2147483648; + abs.s32 %r4179, %r1031; + shl.b32 %r4180, %r4179, %r27; + or.b32 %r6902, %r4180, %r4178; + +$L__BB3_405: + shl.b32 %r4184, %r6902, 1; + shr.u32 %r4185, %r4184, %r27; + and.b32 %r1034, %r4185, -2; + setp.eq.s32 %p469, %r1034, 0; + mov.u32 %r6906, 0; + mov.u32 %r6903, %r6906; + mov.u32 %r6904, %r6906; + mov.u32 %r6910, %r6906; + @%p469 bra $L__BB3_407; + + add.s32 %r4187, %r1034, -1; + clz.b32 %r4188, %r4187; + mov.u32 %r4189, 32; + sub.s32 %r6903, %r4189, %r4188; + shr.u32 %r4190, %r6902, 31; + add.s32 %r4191, %r4190, %r1034; + add.s32 %r6904, %r4191, -2; + mov.u32 %r6910, 1; + +$L__BB3_407: + setp.ge.u32 %p470, %r1007, %r3201; + @%p470 bra $L__BB3_410; + + add.s32 %r4194, %r6898, %r3198; + cvt.u64.u32 %rd271, %r4194; + add.s64 %rd272, %rd271, %rd4; + shl.b64 %rd273, %rd272, 2; + add.s64 %rd274, %rd3, %rd273; + ld.global.u32 %r1040, [%rd274]; + setp.eq.s32 %p471, %r1040, 0; + @%p471 bra $L__BB3_410; + + and.b32 %r4195, %r1040, -2147483648; + abs.s32 %r4196, %r1040; + shl.b32 %r4197, %r4196, %r27; + or.b32 %r6906, %r4197, %r4195; + +$L__BB3_410: + shl.b32 %r4200, %r6906, 1; + shr.u32 %r4201, %r4200, %r27; + and.b32 %r1043, %r4201, -2; + setp.eq.s32 %p472, %r1043, 0; + mov.u32 %r6921, 0; + mov.u32 %r6907, %r6921; + mov.u32 %r6908, %r6921; + mov.u32 %r6926, %r6903; + @%p472 bra $L__BB3_412; + + or.b32 %r6910, %r6910, 2; + add.s32 %r4202, %r1043, -1; + clz.b32 %r4203, %r4202; + mov.u32 %r4204, 32; + sub.s32 %r6907, %r4204, %r4203; + max.s32 %r6926, %r6903, %r6907; + shr.u32 %r4205, %r6906, 31; + add.s32 %r4206, %r4205, %r1043; + add.s32 %r6908, %r4206, -2; + +$L__BB3_412: + add.s32 %r7203, %r6898, 1; + add.s32 %r4211, %r6881, 1; + setp.ge.u32 %p473, %r4211, %r3200; + mov.u32 %r6922, %r6921; + mov.u32 %r6923, %r6921; + mov.u32 %r6924, %r6921; + @%p473 bra $L__BB3_423; + + cvt.u64.u32 %rd275, %r7203; + add.s64 %rd276, %rd275, %rd4; + shl.b64 %rd277, %rd276, 2; + add.s64 %rd278, %rd3, %rd277; + ld.global.u32 %r1053, [%rd278]; + setp.eq.s32 %p474, %r1053, 0; + mov.u32 %r6922, 0; + mov.u32 %r6911, %r6922; + @%p474 bra $L__BB3_415; + + and.b32 %r4213, %r1053, -2147483648; + abs.s32 %r4214, %r1053; + shl.b32 %r4215, %r4214, %r27; + or.b32 %r6911, %r4215, %r4213; + +$L__BB3_415: + shl.b32 %r4218, %r6911, 1; + shr.u32 %r4219, %r4218, %r27; + and.b32 %r1056, %r4219, -2; + setp.eq.s32 %p475, %r1056, 0; + mov.u32 %r6924, %r6922; + @%p475 bra $L__BB3_417; + + or.b32 %r6910, %r6910, 4; + add.s32 %r4220, %r1056, -1; + clz.b32 %r4221, %r4220; + mov.u32 %r4222, 32; + sub.s32 %r6922, %r4222, %r4221; + max.s32 %r6926, %r6926, %r6922; + shr.u32 %r4223, %r6911, 31; + add.s32 %r4224, %r4223, %r1056; + add.s32 %r6924, %r4224, -2; + +$L__BB3_417: + mov.u32 %r6921, 0; + mov.u32 %r6916, %r6921; + @%p470 bra $L__BB3_420; + + add.s32 %r4227, %r7203, %r3198; + cvt.u64.u32 %rd279, %r4227; + add.s64 %rd280, %rd279, %rd4; + shl.b64 %rd281, %rd280, 2; + add.s64 %rd282, %rd3, %rd281; + ld.global.u32 %r1065, [%rd282]; + setp.eq.s32 %p477, %r1065, 0; + @%p477 bra $L__BB3_420; + + and.b32 %r4228, %r1065, -2147483648; + abs.s32 %r4229, %r1065; + shl.b32 %r4230, %r4229, %r27; + or.b32 %r6916, %r4230, %r4228; + +$L__BB3_420: + shl.b32 %r4233, %r6916, 1; + shr.u32 %r4234, %r4233, %r27; + and.b32 %r1068, %r4234, -2; + setp.eq.s32 %p478, %r1068, 0; + mov.u32 %r6923, %r6921; + @%p478 bra $L__BB3_422; + + or.b32 %r6910, %r6910, 8; + add.s32 %r4235, %r1068, -1; + clz.b32 %r4236, %r4235; + mov.u32 %r4237, 32; + sub.s32 %r6921, %r4237, %r4236; + max.s32 %r6926, %r6926, %r6921; + shr.u32 %r4238, %r6916, 31; + add.s32 %r4239, %r4238, %r1068; + add.s32 %r6923, %r4239, -2; + +$L__BB3_422: + add.s32 %r7203, %r6898, 2; + +$L__BB3_423: + add.s32 %r4241, %r6910, -1; + and.b32 %r4242, %r4241, %r6910; + setp.ne.s32 %p479, %r4242, 0; + mov.u32 %r6928, 0; + setp.gt.s32 %p480, %r6900, 1; + and.pred %p481, %p480, %p479; + selp.b32 %r4243, %r6900, 1, %p481; + max.s32 %r1085, %r4243, %r6926; + sub.s32 %r1086, %r1085, %r4243; + setp.lt.s32 %p482, %r1086, 1; + @%p482 bra $L__BB3_425; + + setp.eq.s32 %p483, %r6903, %r6926; + selp.u32 %r4244, 1, 0, %p483; + setp.eq.s32 %p484, %r6907, %r6926; + selp.u32 %r4245, -1, 0, %p484; + bfi.b32 %r4246, %r4245, %r4244, 1, 1; + setp.eq.s32 %p485, %r6922, %r6926; + selp.u16 %rs619, 1, 0, %p485; + mul.wide.u16 %r4247, %rs619, 4; + or.b32 %r4248, %r4246, %r4247; + setp.eq.s32 %p486, %r6921, %r6926; + selp.u16 %rs620, 1, 0, %p486; + mul.wide.u16 %r4249, %rs620, 8; + or.b32 %r6928, %r4248, %r4249; + +$L__BB3_425: + shl.b32 %r4250, %r6910, 4; + shl.b32 %r4251, %r6897, 8; + or.b32 %r4252, %r4250, %r4251; + or.b32 %r4253, %r4252, %r6928; + mul.wide.u32 %rd283, %r4253, 2; + add.s64 %rd284, %rd18, %rd283; + ld.global.u16 %rs155, [%rd284]; + shr.u16 %rs621, %rs155, 4; + and.b16 %rs156, %rs621, 7; + setp.eq.s16 %p487, %rs156, 0; + mov.u32 %r6940, %r6832; + @%p487 bra $L__BB3_432; + + cvt.u32.u16 %r6929, %rs156; + shr.u16 %rs622, %rs155, 8; + cvt.u32.u16 %r6930, %rs622; + +$L__BB3_427: + mov.u32 %r1091, %r6929; + setp.gt.u32 %p488, %r6829, 2879; + mov.u32 %r6940, 1; + @%p488 bra $L__BB3_432; + + mov.u32 %r4255, 8; + sub.s32 %r4256, %r4255, %r6831; + sub.s32 %r4257, %r4256, %r6830; + min.u32 %r4258, %r4257, %r1091; + setp.eq.s32 %p489, %r4258, 32; + mov.u32 %r4259, -1; + shl.b32 %r4260, %r4259, %r4258; + not.b32 %r4261, %r4260; + selp.b32 %r4262, -1, %r4261, %p489; + and.b32 %r4263, %r4262, %r6930; + shl.b32 %r4264, %r4263, %r6830; + cvt.u16.u32 %rs623, %r4264; + or.b16 %rs1025, %rs1025, %rs623; + add.s32 %r6830, %r4258, %r6830; + sub.s32 %r6929, %r1091, %r4258; + shr.u32 %r6930, %r6930, %r4258; + setp.gt.u32 %p490, %r4257, %r1091; + @%p490 bra $L__BB3_431; + + setp.ne.s32 %p491, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs624, %rs1025, 255; + setp.ne.s16 %p492, %rs624, 127; + and.pred %p493, %p491, %p492; + @%p493 bra $L__BB3_431; + + mov.u32 %r4267, 20548; + sub.s32 %r4268, %r4267, %r6829; + cvt.u64.u32 %rd285, %r4268; + add.s64 %rd286, %rd285, %rd5; + add.s64 %rd287, %rd1, %rd286; + st.global.u8 [%rd287], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p494, %rs624, 143; + selp.u32 %r6831, 1, 0, %p494; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_431: + setp.ne.s32 %p495, %r6929, 0; + mov.u32 %r6940, %r6832; + @%p495 bra $L__BB3_427; + +$L__BB3_432: + setp.ne.s32 %p496, %r6897, 0; + @%p496 bra $L__BB3_480; + + setp.eq.s32 %p497, %r6910, 0; + add.s32 %r4269, %r6381, 17477; + cvt.u64.u32 %rd288, %r4269; + add.s64 %rd289, %rd288, %rd5; + add.s64 %rd19, %rd1, %rd289; + @%p497 bra $L__BB3_472; + + shl.b16 %rs956, %rs956, 1; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p498, %r6390, 0; + mov.u32 %r6976, %r6598; + @%p498 bra $L__BB3_437; + bra.uni $L__BB3_435; + +$L__BB3_437: + setp.lt.u32 %p500, %r6596, 3; + mov.u32 %r6944, 0; + @%p500 bra $L__BB3_440; + + setp.lt.u32 %p501, %r6596, 6; + mov.u32 %r6944, 1; + @%p501 bra $L__BB3_440; + + setp.lt.u32 %p502, %r6596, 9; + setp.eq.s32 %p503, %r6596, 11; + selp.b32 %r4275, 4, 5, %p503; + setp.lt.u32 %p504, %r6596, 11; + selp.b32 %r4276, 3, %r4275, %p504; + selp.b32 %r6944, 2, %r4276, %p502; + +$L__BB3_440: + setp.eq.s32 %p505, %r6944, 0; + @%p505 bra $L__BB3_468; + + add.s32 %r1115, %r6944, -1; + and.b32 %r1116, %r6944, 3; + setp.eq.s32 %p506, %r1116, 0; + mov.u32 %r6954, %r6944; + mov.u32 %r6955, %r6976; + @%p506 bra $L__BB3_453; + + mov.u32 %r4278, 1; + shl.b32 %r4279, %r4278, %r1115; + and.b32 %r4280, %r4279, %r6595; + setp.ne.s32 %p507, %r4280, 0; + selp.u32 %r4281, 1, 0, %p507; + cvt.u32.u16 %r4282, %rs956; + bfi.b32 %r4283, %r4282, %r4281, 1, 8; + cvt.u16.u32 %rs956, %r4283; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p508, %r6390, 0; + mov.u32 %r6955, %r6976; + @%p508 bra $L__BB3_445; + + setp.gt.u32 %p509, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r6955, %r4278; + @%p509 bra $L__BB3_445; + + add.s32 %r4287, %r6381, 17477; + cvt.u64.u32 %rd290, %r4287; + add.s64 %rd291, %rd290, %rd5; + add.s64 %rd292, %rd1, %rd291; + st.global.u8 [%rd292], %rs956; + add.s32 %r6381, %r6381, 1; + mov.u16 %rs956, 0; + mov.u32 %r6390, 8; + mov.u32 %r6955, %r6976; + +$L__BB3_445: + setp.eq.s32 %p510, %r1116, 1; + mov.u32 %r6976, %r6955; + mov.u32 %r6954, %r1115; + @%p510 bra $L__BB3_453; + + add.s32 %r6954, %r6944, -2; + mov.u32 %r4288, 1; + shl.b32 %r4289, %r4288, %r6954; + and.b32 %r4290, %r4289, %r6595; + setp.ne.s32 %p511, %r4290, 0; + selp.u32 %r4291, 1, 0, %p511; + cvt.u32.u16 %r4292, %rs956; + bfi.b32 %r4293, %r4292, %r4291, 1, 8; + cvt.u16.u32 %rs956, %r4293; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p512, %r6390, 0; + mov.u32 %r6950, %r6955; + @%p512 bra $L__BB3_449; + + setp.gt.u32 %p513, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r6950, %r4288; + @%p513 bra $L__BB3_449; + + add.s32 %r4296, %r6381, 17477; + cvt.u64.u32 %rd293, %r4296; + add.s64 %rd294, %rd293, %rd5; + add.s64 %rd295, %rd1, %rd294; + and.b16 %rs631, %rs956, 255; + st.global.u8 [%rd295], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p514, %rs631, 255; + selp.b32 %r6390, 7, 8, %p514; + mov.u16 %rs956, 0; + mov.u32 %r6950, %r6955; + +$L__BB3_449: + setp.eq.s32 %p515, %r1116, 2; + mov.u32 %r6976, %r6950; + mov.u32 %r6955, %r6950; + @%p515 bra $L__BB3_453; + + add.s32 %r6954, %r6944, -3; + mov.u32 %r4297, 1; + shl.b32 %r4298, %r4297, %r6954; + and.b32 %r4299, %r4298, %r6595; + setp.ne.s32 %p516, %r4299, 0; + selp.u32 %r4300, 1, 0, %p516; + cvt.u32.u16 %r4301, %rs956; + bfi.b32 %r4302, %r4301, %r4300, 1, 8; + cvt.u16.u32 %rs956, %r4302; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p517, %r6390, 0; + mov.u32 %r6976, %r6950; + mov.u32 %r6955, %r6950; + @%p517 bra $L__BB3_453; + + setp.gt.u32 %p518, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r6976, %r4297; + mov.u32 %r6955, %r4297; + @%p518 bra $L__BB3_453; + + add.s32 %r4307, %r6381, 17477; + cvt.u64.u32 %rd296, %r4307; + add.s64 %rd297, %rd296, %rd5; + add.s64 %rd298, %rd1, %rd297; + and.b16 %rs634, %rs956, 255; + st.global.u8 [%rd298], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p519, %rs634, 255; + selp.b32 %r6390, 7, 8, %p519; + mov.u16 %rs956, 0; + mov.u32 %r6976, %r6950; + mov.u32 %r6955, %r6950; + +$L__BB3_453: + setp.lt.u32 %p520, %r1115, 3; + @%p520 bra $L__BB3_468; + + mov.u32 %r6976, %r6955; + +$L__BB3_455: + add.s32 %r4308, %r6954, -1; + mov.u32 %r4309, 1; + shl.b32 %r4310, %r4309, %r4308; + and.b32 %r4311, %r4310, %r6595; + setp.ne.s32 %p521, %r4311, 0; + selp.u32 %r4312, 1, 0, %p521; + cvt.u32.u16 %r4313, %rs956; + bfi.b32 %r6964, %r4313, %r4312, 1, 8; + add.s32 %r6963, %r6390, -1; + setp.ne.s32 %p522, %r6963, 0; + mov.u32 %r6965, %r6976; + @%p522 bra $L__BB3_458; + + setp.gt.u32 %p523, %r6381, 191; + mov.u32 %r6963, 0; + mov.u32 %r6965, %r4309; + @%p523 bra $L__BB3_458; + + cvt.u16.u32 %rs635, %r6964; + and.b16 %rs636, %rs635, 255; + add.s32 %r4317, %r6381, 17477; + cvt.u64.u32 %rd299, %r4317; + add.s64 %rd300, %rd299, %rd5; + add.s64 %rd301, %rd1, %rd300; + st.global.u8 [%rd301], %rs635; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p524, %rs636, 255; + selp.b32 %r6963, 7, 8, %p524; + mov.u32 %r6964, 0; + mov.u32 %r6965, %r6976; + +$L__BB3_458: + add.s32 %r4318, %r6954, -2; + shl.b32 %r4320, %r4309, %r4318; + and.b32 %r4321, %r4320, %r6595; + setp.ne.s32 %p525, %r4321, 0; + and.b32 %r4322, %r6964, 127; + selp.u32 %r4323, 1, 0, %p525; + bfi.b32 %r6968, %r4322, %r4323, 1, 7; + add.s32 %r6967, %r6963, -1; + setp.ne.s32 %p526, %r6967, 0; + mov.u32 %r6969, %r6965; + @%p526 bra $L__BB3_461; + + setp.gt.u32 %p527, %r6381, 191; + mov.u32 %r6969, 1; + mov.u32 %r6967, 0; + @%p527 bra $L__BB3_461; + + cvt.u16.u32 %rs637, %r6968; + and.b16 %rs638, %rs637, 255; + add.s32 %r4327, %r6381, 17477; + cvt.u64.u32 %rd302, %r4327; + add.s64 %rd303, %rd302, %rd5; + add.s64 %rd304, %rd1, %rd303; + st.global.u8 [%rd304], %rs637; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p528, %rs638, 255; + selp.b32 %r6967, 7, 8, %p528; + mov.u32 %r6968, 0; + mov.u32 %r6969, %r6965; + +$L__BB3_461: + add.s32 %r4328, %r6954, -3; + mov.u32 %r4329, 1; + shl.b32 %r4330, %r4329, %r4328; + and.b32 %r4331, %r4330, %r6595; + setp.ne.s32 %p529, %r4331, 0; + and.b32 %r4332, %r6968, 127; + selp.u32 %r4333, 1, 0, %p529; + bfi.b32 %r6972, %r4332, %r4333, 1, 7; + add.s32 %r6971, %r6967, -1; + setp.ne.s32 %p530, %r6971, 0; + mov.u32 %r6973, %r6969; + @%p530 bra $L__BB3_464; + + setp.gt.u32 %p531, %r6381, 191; + mov.u32 %r6971, 0; + mov.u32 %r6973, %r4329; + @%p531 bra $L__BB3_464; + + cvt.u16.u32 %rs639, %r6972; + and.b16 %rs640, %rs639, 255; + add.s32 %r4337, %r6381, 17477; + cvt.u64.u32 %rd305, %r4337; + add.s64 %rd306, %rd305, %rd5; + add.s64 %rd307, %rd1, %rd306; + st.global.u8 [%rd307], %rs639; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p532, %rs640, 255; + selp.b32 %r6971, 7, 8, %p532; + mov.u32 %r6972, 0; + mov.u32 %r6973, %r6969; + +$L__BB3_464: + add.s32 %r6954, %r6954, -4; + shl.b32 %r4339, %r4329, %r6954; + and.b32 %r4340, %r4339, %r6595; + setp.ne.s32 %p533, %r4340, 0; + and.b32 %r4341, %r6972, 127; + selp.u32 %r4342, 1, 0, %p533; + bfi.b32 %r4343, %r4341, %r4342, 1, 15; + cvt.u16.u32 %rs956, %r4343; + add.s32 %r6390, %r6971, -1; + setp.ne.s32 %p534, %r6390, 0; + mov.u32 %r6976, %r6973; + @%p534 bra $L__BB3_467; + + setp.gt.u32 %p535, %r6381, 191; + mov.u32 %r6976, 1; + mov.u32 %r6390, 0; + @%p535 bra $L__BB3_467; + + add.s32 %r4346, %r6381, 17477; + cvt.u64.u32 %rd308, %r4346; + add.s64 %rd309, %rd308, %rd5; + add.s64 %rd310, %rd1, %rd309; + and.b16 %rs642, %rs956, 255; + st.global.u8 [%rd310], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p536, %rs642, 255; + selp.b32 %r6390, 7, 8, %p536; + mov.u16 %rs956, 0; + mov.u32 %r6976, %r6973; + +$L__BB3_467: + setp.ne.s32 %p537, %r6954, 0; + @%p537 bra $L__BB3_455; + +$L__BB3_468: + add.s32 %r4348, %r6596, -1; + setp.eq.s32 %p538, %r6596, 0; + mov.u32 %r6595, 0; + selp.b32 %r6596, 0, %r4348, %p538; + setp.lt.u32 %p539, %r6596, 3; + mov.u32 %r6980, %r6595; + @%p539 bra $L__BB3_471; + + setp.lt.u32 %p540, %r6596, 6; + mov.u32 %r6980, 1; + @%p540 bra $L__BB3_471; + + setp.lt.u32 %p541, %r6596, 9; + setp.eq.s32 %p542, %r6596, 11; + selp.b32 %r4350, 4, 5, %p542; + setp.lt.u32 %p543, %r6596, 11; + selp.b32 %r4351, 3, %r4350, %p543; + selp.b32 %r6980, 2, %r4351, %p541; + +$L__BB3_471: + mov.u32 %r4353, 1; + shl.b32 %r6597, %r4353, %r6980; + mov.u32 %r6598, %r6976; + bra.uni $L__BB3_480; + +$L__BB3_472: + add.s32 %r6595, %r6595, 1; + setp.lt.u32 %p544, %r6595, %r6597; + @%p544 bra $L__BB3_480; + + shl.b16 %rs643, %rs956, 1; + or.b16 %rs956, %rs643, 1; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p545, %r6390, 0; + mov.u32 %r6983, %r6598; + @%p545 bra $L__BB3_476; + + setp.gt.u32 %p546, %r6381, 191; + mov.u32 %r6983, 1; + mov.u32 %r6390, 0; + @%p546 bra $L__BB3_476; + + and.b16 %rs645, %rs956, 255; + st.global.u8 [%rd19], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p547, %rs645, 255; + selp.b32 %r6390, 7, 8, %p547; + mov.u16 %rs956, 0; + mov.u32 %r6983, %r6598; + +$L__BB3_476: + add.s32 %r4357, %r6596, 1; + min.u32 %r6596, %r4357, 12; + setp.lt.u32 %p548, %r6596, 3; + mov.u32 %r6595, 0; + mov.u32 %r6984, %r6595; + @%p548 bra $L__BB3_479; + + setp.lt.u32 %p549, %r6596, 6; + mov.u32 %r6984, 1; + @%p549 bra $L__BB3_479; + + setp.lt.u32 %p550, %r6596, 9; + setp.eq.s32 %p551, %r6596, 11; + selp.b32 %r4359, 4, 5, %p551; + setp.lt.u32 %p552, %r6596, 11; + selp.b32 %r4360, 3, %r4359, %p552; + selp.b32 %r6984, 2, %r4360, %p550; + +$L__BB3_479: + mov.u32 %r4362, 1; + shl.b32 %r6597, %r4362, %r6984; + mov.u32 %r6598, %r6983; + +$L__BB3_480: + and.b16 %rs646, %rs155, 15; + cvt.u32.u16 %r1199, %rs646; + and.b32 %r4363, %r6910, 1; + setp.eq.b32 %p553, %r4363, 1; + mov.pred %p554, 0; + xor.pred %p555, %p553, %p554; + not.pred %p556, %p555; + mov.u32 %r7005, %r7050; + @%p556 bra $L__BB3_487; + + and.b32 %r4364, %r1199, 1; + sub.s32 %r6991, %r1085, %r4364; + setp.eq.s32 %p557, %r6991, 0; + mov.u32 %r7005, %r7050; + @%p557 bra $L__BB3_487; + + mov.u32 %r4365, -1; + shl.b32 %r4366, %r4365, %r6991; + not.b32 %r4367, %r4366; + and.b32 %r6992, %r6904, %r4367; + +$L__BB3_483: + setp.gt.u32 %p558, %r7016, 17476; + mov.u32 %r7005, 1; + @%p558 bra $L__BB3_487; + + sub.s32 %r4369, %r7017, %r7018; + min.u32 %r4370, %r4369, %r6991; + setp.eq.s32 %p559, %r4370, 32; + mov.u32 %r4371, -1; + shl.b32 %r4372, %r4371, %r4370; + not.b32 %r4373, %r4372; + selp.b32 %r4374, -1, %r4373, %p559; + and.b32 %r4375, %r4374, %r6992; + shl.b32 %r4376, %r4375, %r7018; + or.b32 %r7019, %r4376, %r7019; + add.s32 %r7018, %r4370, %r7018; + shr.u32 %r6992, %r6992, %r4370; + sub.s32 %r6991, %r6991, %r4370; + setp.lt.u32 %p560, %r7018, %r7017; + @%p560 bra $L__BB3_486; + + cvt.u64.u32 %rd311, %r7016; + add.s64 %rd312, %rd311, %rd5; + add.s64 %rd313, %rd1, %rd312; + st.global.u8 [%rd313], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p561, %r7019, 255; + selp.b32 %r7017, 7, 8, %p561; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_486: + setp.ne.s32 %p562, %r6991, 0; + mov.u32 %r7005, %r7050; + @%p562 bra $L__BB3_483; + +$L__BB3_487: + and.b32 %r1223, %r6910, 2; + setp.eq.s32 %p563, %r1223, 0; + mov.u32 %r7020, %r7005; + @%p563 bra $L__BB3_494; + + shr.u32 %r4379, %r1199, 1; + and.b32 %r4380, %r4379, 1; + sub.s32 %r7006, %r1085, %r4380; + setp.eq.s32 %p564, %r7006, 0; + mov.u32 %r7020, %r7005; + @%p564 bra $L__BB3_494; + + mov.u32 %r4381, -1; + shl.b32 %r4382, %r4381, %r7006; + not.b32 %r4383, %r4382; + and.b32 %r7007, %r6908, %r4383; + +$L__BB3_490: + setp.gt.u32 %p565, %r7016, 17476; + mov.u32 %r7020, 1; + @%p565 bra $L__BB3_494; + + sub.s32 %r4385, %r7017, %r7018; + min.u32 %r4386, %r4385, %r7006; + setp.eq.s32 %p566, %r4386, 32; + mov.u32 %r4387, -1; + shl.b32 %r4388, %r4387, %r4386; + not.b32 %r4389, %r4388; + selp.b32 %r4390, -1, %r4389, %p566; + and.b32 %r4391, %r4390, %r7007; + shl.b32 %r4392, %r4391, %r7018; + or.b32 %r7019, %r4392, %r7019; + add.s32 %r7018, %r4386, %r7018; + shr.u32 %r7007, %r7007, %r4386; + sub.s32 %r7006, %r7006, %r4386; + setp.lt.u32 %p567, %r7018, %r7017; + @%p567 bra $L__BB3_493; + + cvt.u64.u32 %rd314, %r7016; + add.s64 %rd315, %rd314, %rd5; + add.s64 %rd316, %rd1, %rd315; + st.global.u8 [%rd316], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p568, %r7019, 255; + selp.b32 %r7017, 7, 8, %p568; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_493: + setp.ne.s32 %p569, %r7006, 0; + mov.u32 %r7020, %r7005; + @%p569 bra $L__BB3_490; + +$L__BB3_494: + and.b32 %r1247, %r6910, 4; + setp.eq.s32 %p570, %r1247, 0; + mov.u32 %r7035, %r7020; + @%p570 bra $L__BB3_501; + + shr.u32 %r4395, %r1199, 2; + and.b32 %r4396, %r4395, 1; + sub.s32 %r7021, %r1085, %r4396; + setp.eq.s32 %p571, %r7021, 0; + mov.u32 %r7035, %r7020; + @%p571 bra $L__BB3_501; + + mov.u32 %r4397, -1; + shl.b32 %r4398, %r4397, %r7021; + not.b32 %r4399, %r4398; + and.b32 %r7022, %r6924, %r4399; + +$L__BB3_497: + setp.gt.u32 %p572, %r7016, 17476; + mov.u32 %r7035, 1; + @%p572 bra $L__BB3_501; + + sub.s32 %r4401, %r7017, %r7018; + min.u32 %r4402, %r4401, %r7021; + setp.eq.s32 %p573, %r4402, 32; + mov.u32 %r4403, -1; + shl.b32 %r4404, %r4403, %r4402; + not.b32 %r4405, %r4404; + selp.b32 %r4406, -1, %r4405, %p573; + and.b32 %r4407, %r4406, %r7022; + shl.b32 %r4408, %r4407, %r7018; + or.b32 %r7019, %r4408, %r7019; + add.s32 %r7018, %r4402, %r7018; + shr.u32 %r7022, %r7022, %r4402; + sub.s32 %r7021, %r7021, %r4402; + setp.lt.u32 %p574, %r7018, %r7017; + @%p574 bra $L__BB3_500; + + cvt.u64.u32 %rd317, %r7016; + add.s64 %rd318, %rd317, %rd5; + add.s64 %rd319, %rd1, %rd318; + st.global.u8 [%rd319], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p575, %r7019, 255; + selp.b32 %r7017, 7, 8, %p575; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_500: + setp.ne.s32 %p576, %r7021, 0; + mov.u32 %r7035, %r7020; + @%p576 bra $L__BB3_497; + +$L__BB3_501: + and.b32 %r1271, %r6910, 8; + setp.eq.s32 %p577, %r1271, 0; + mov.u32 %r7050, %r7035; + @%p577 bra $L__BB3_508; + + shr.u32 %r4411, %r1199, 3; + sub.s32 %r7036, %r1085, %r4411; + setp.eq.s32 %p578, %r7036, 0; + mov.u32 %r7050, %r7035; + @%p578 bra $L__BB3_508; + + mov.u32 %r4412, -1; + shl.b32 %r4413, %r4412, %r7036; + not.b32 %r4414, %r4413; + and.b32 %r7037, %r6923, %r4414; + +$L__BB3_504: + setp.gt.u32 %p579, %r7016, 17476; + mov.u32 %r7050, 1; + @%p579 bra $L__BB3_508; + + sub.s32 %r4416, %r7017, %r7018; + min.u32 %r4417, %r4416, %r7036; + setp.eq.s32 %p580, %r4417, 32; + mov.u32 %r4418, -1; + shl.b32 %r4419, %r4418, %r4417; + not.b32 %r4420, %r4419; + selp.b32 %r4421, -1, %r4420, %p580; + and.b32 %r4422, %r4421, %r7037; + shl.b32 %r4423, %r4422, %r7018; + or.b32 %r7019, %r4423, %r7019; + add.s32 %r7018, %r4417, %r7018; + shr.u32 %r7037, %r7037, %r4417; + sub.s32 %r7036, %r7036, %r4417; + setp.lt.u32 %p581, %r7018, %r7017; + @%p581 bra $L__BB3_507; + + cvt.u64.u32 %rd320, %r7016; + add.s64 %rd321, %rd320, %rd5; + add.s64 %rd322, %rd1, %rd321; + st.global.u8 [%rd322], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p582, %r7019, 255; + selp.b32 %r7017, 7, 8, %p582; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_507: + setp.ne.s32 %p583, %r7036, 0; + mov.u32 %r7050, %r7035; + @%p583 bra $L__BB3_504; + +$L__BB3_508: + mov.u32 %r4428, _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE13cleanup_e_val; + add.s32 %r1295, %r4428, %r6899; + ld.shared.u8 %rs647, [%r1295]; + mov.u32 %r6897, 0; + cvt.u32.u16 %r4429, %rs647; + and.b32 %r4430, %r4429, 255; + and.b32 %r4431, %r6907, 255; + setp.lt.u32 %p584, %r4431, %r4430; + cvt.u16.u32 %rs648, %r6907; + selp.b16 %rs649, %rs647, %rs648, %p584; + st.shared.u8 [%r1295], %rs649; + ld.shared.u8 %rs177, [%r1295+2]; + ld.shared.u8 %rs650, [%r1295+1]; + setp.gt.u16 %p585, %rs650, %rs177; + add.s32 %r7202, %r6899, 1; + add.s32 %r4432, %r6899, 2; + selp.b32 %r4433, %r7202, %r4432, %p585; + add.s32 %r4434, %r4428, %r4433; + ld.shared.u8 %rs178, [%r4434]; + cvt.u32.u16 %r4435, %rs178; + and.b32 %r4436, %r4435, 255; + add.s32 %r6900, %r4436, -1; + cvt.u16.u32 %rs179, %r6921; + cvt.u16.u32 %rs651, %r1223; + shr.u16 %rs652, %rs651, 1; + mov.u32 %r4437, _ZZ52 j2k_htj2k_encode_codeblocks_multi_input_cleanupE14cleanup_cx_val; + add.s32 %r1298, %r4437, %r6901; + st.shared.u8 [%r1295+1], %r6921; + ld.shared.u8 %rs653, [%r1298]; + or.b16 %rs654, %rs653, %rs652; + st.shared.u8 [%r1298], %rs654; + add.s32 %r6901, %r6901, 1; + ld.shared.u8 %rs180, [%r1298+1]; + ld.shared.u8 %r1300, [%r1298+2]; + shr.u32 %r1301, %r1271, 3; + st.shared.u8 [%r1298+1], %r1301; + add.s32 %r4438, %r6881, 2; + setp.ge.u32 %p586, %r4438, %r3200; + mov.u32 %r7220, %r6897; + @%p586 bra $L__BB3_615; + + cvt.u64.u32 %rd323, %r7203; + add.s64 %rd324, %rd323, %rd4; + shl.b64 %rd325, %rd324, 2; + add.s64 %rd326, %rd3, %rd325; + ld.global.u32 %r1302, [%rd326]; + setp.eq.s32 %p587, %r1302, 0; + mov.u32 %r7052, 0; + mov.u32 %r7051, %r7052; + @%p587 bra $L__BB3_511; + + and.b32 %r4440, %r1302, -2147483648; + abs.s32 %r4441, %r1302; + shl.b32 %r4442, %r4441, %r27; + or.b32 %r7051, %r4442, %r4440; + +$L__BB3_511: + shl.b32 %r4446, %r7051, 1; + shr.u32 %r4447, %r4446, %r27; + and.b32 %r1305, %r4447, -2; + setp.eq.s32 %p588, %r1305, 0; + mov.u32 %r7053, %r7052; + mov.u32 %r7059, %r7052; + @%p588 bra $L__BB3_513; + + add.s32 %r4449, %r1305, -1; + clz.b32 %r4450, %r4449; + mov.u32 %r4451, 32; + sub.s32 %r7052, %r4451, %r4450; + shr.u32 %r4452, %r7051, 31; + add.s32 %r4453, %r4452, %r1305; + add.s32 %r7053, %r4453, -2; + mov.u32 %r7059, 1; + +$L__BB3_513: + mov.u32 %r7056, 0; + mov.u32 %r7055, %r7056; + @%p470 bra $L__BB3_516; + + add.s32 %r4456, %r7203, %r3198; + cvt.u64.u32 %rd327, %r4456; + add.s64 %rd328, %rd327, %rd4; + shl.b64 %rd329, %rd328, 2; + add.s64 %rd330, %rd3, %rd329; + ld.global.u32 %r1311, [%rd330]; + setp.eq.s32 %p590, %r1311, 0; + @%p590 bra $L__BB3_516; + + and.b32 %r4457, %r1311, -2147483648; + abs.s32 %r4458, %r1311; + shl.b32 %r4459, %r4458, %r27; + or.b32 %r7055, %r4459, %r4457; + +$L__BB3_516: + shl.b32 %r4462, %r7055, 1; + shr.u32 %r4463, %r4462, %r27; + and.b32 %r1314, %r4463, -2; + setp.eq.s32 %p591, %r1314, 0; + mov.u32 %r7057, %r7056; + mov.u32 %r7075, %r7052; + @%p591 bra $L__BB3_518; + + or.b32 %r7059, %r7059, 2; + add.s32 %r4464, %r1314, -1; + clz.b32 %r4465, %r4464; + mov.u32 %r4466, 32; + sub.s32 %r7056, %r4466, %r4465; + max.s32 %r7075, %r7052, %r7056; + shr.u32 %r4467, %r7055, 31; + add.s32 %r4468, %r4467, %r1314; + add.s32 %r7057, %r4468, -2; + +$L__BB3_518: + add.s32 %r7074, %r7203, 1; + add.s32 %r4473, %r6881, 3; + setp.ge.u32 %p592, %r4473, %r3200; + mov.u32 %r7077, 0; + mov.u32 %r7070, %r7077; + mov.u32 %r7071, %r7077; + mov.u32 %r7072, %r7077; + mov.u32 %r7073, %r7077; + @%p592 bra $L__BB3_529; + + cvt.u64.u32 %rd331, %r7074; + add.s64 %rd332, %rd331, %rd4; + shl.b64 %rd333, %rd332, 2; + add.s64 %rd334, %rd3, %rd333; + ld.global.u32 %r1324, [%rd334]; + setp.eq.s32 %p593, %r1324, 0; + mov.u32 %r7071, 0; + mov.u32 %r7060, %r7071; + @%p593 bra $L__BB3_521; + + and.b32 %r4475, %r1324, -2147483648; + abs.s32 %r4476, %r1324; + shl.b32 %r4477, %r4476, %r27; + or.b32 %r7060, %r4477, %r4475; + +$L__BB3_521: + shl.b32 %r4480, %r7060, 1; + shr.u32 %r4481, %r4480, %r27; + and.b32 %r1327, %r4481, -2; + setp.eq.s32 %p594, %r1327, 0; + mov.u32 %r7073, %r7071; + @%p594 bra $L__BB3_523; + + or.b32 %r7059, %r7059, 4; + add.s32 %r4482, %r1327, -1; + clz.b32 %r4483, %r4482; + mov.u32 %r4484, 32; + sub.s32 %r7071, %r4484, %r4483; + max.s32 %r7075, %r7075, %r7071; + shr.u32 %r4485, %r7060, 31; + add.s32 %r4486, %r4485, %r1327; + add.s32 %r7073, %r4486, -2; + +$L__BB3_523: + mov.u32 %r7070, 0; + mov.u32 %r7065, %r7070; + @%p470 bra $L__BB3_526; + + add.s32 %r4489, %r7074, %r3198; + cvt.u64.u32 %rd335, %r4489; + add.s64 %rd336, %rd335, %rd4; + shl.b64 %rd337, %rd336, 2; + add.s64 %rd338, %rd3, %rd337; + ld.global.u32 %r1336, [%rd338]; + setp.eq.s32 %p596, %r1336, 0; + @%p596 bra $L__BB3_526; + + and.b32 %r4490, %r1336, -2147483648; + abs.s32 %r4491, %r1336; + shl.b32 %r4492, %r4491, %r27; + or.b32 %r7065, %r4492, %r4490; + +$L__BB3_526: + shl.b32 %r4495, %r7065, 1; + shr.u32 %r4496, %r4495, %r27; + and.b32 %r1339, %r4496, -2; + setp.eq.s32 %p597, %r1339, 0; + mov.u32 %r7072, %r7070; + @%p597 bra $L__BB3_528; + + or.b32 %r7059, %r7059, 8; + add.s32 %r4497, %r1339, -1; + clz.b32 %r4498, %r4497; + mov.u32 %r4499, 32; + sub.s32 %r7070, %r4499, %r4498; + max.s32 %r7075, %r7075, %r7070; + shr.u32 %r4500, %r7065, 31; + add.s32 %r4501, %r4500, %r1339; + add.s32 %r7072, %r4501, -2; + +$L__BB3_528: + add.s32 %r7074, %r7203, 2; + +$L__BB3_529: + mov.u32 %r7203, %r7074; + shr.u32 %r4503, %r1271, 2; + shr.u32 %r4504, %r1247, 1; + or.b32 %r4505, %r4503, %r4504; + cvt.u32.u16 %r4506, %rs180; + and.b32 %r4507, %r4506, 255; + shl.b32 %r4508, %r1300, 2; + add.s32 %r4509, %r4508, %r4507; + or.b32 %r1356, %r4505, %r4509; + add.s32 %r4510, %r7059, -1; + and.b32 %r4511, %r4510, %r7059; + setp.ne.s32 %p598, %r4511, 0; + setp.gt.u16 %p599, %rs178, 2; + and.pred %p600, %p599, %p598; + selp.b32 %r4512, %r6900, 1, %p600; + max.s32 %r1357, %r4512, %r7075; + sub.s32 %r7220, %r1357, %r4512; + setp.lt.s32 %p601, %r7220, 1; + @%p601 bra $L__BB3_531; + + setp.eq.s32 %p602, %r7052, %r7075; + selp.u32 %r4513, 1, 0, %p602; + setp.eq.s32 %p603, %r7056, %r7075; + selp.u32 %r4514, -1, 0, %p603; + bfi.b32 %r4515, %r4514, %r4513, 1, 1; + setp.eq.s32 %p604, %r7071, %r7075; + selp.u16 %rs656, 1, 0, %p604; + mul.wide.u16 %r4516, %rs656, 4; + or.b32 %r4517, %r4515, %r4516; + setp.eq.s32 %p605, %r7070, %r7075; + selp.u16 %rs657, 1, 0, %p605; + mul.wide.u16 %r4518, %rs657, 8; + or.b32 %r7077, %r4517, %r4518; + +$L__BB3_531: + shl.b32 %r4519, %r7059, 4; + shl.b32 %r4520, %r1356, 8; + or.b32 %r4521, %r4519, %r4520; + or.b32 %r4522, %r4521, %r7077; + mul.wide.u32 %rd340, %r4522, 2; + add.s64 %rd341, %rd18, %rd340; + ld.global.u16 %rs181, [%rd341]; + shr.u16 %rs658, %rs181, 4; + and.b16 %rs182, %rs658, 7; + setp.eq.s16 %p606, %rs182, 0; + mov.u32 %r7089, %r6940; + @%p606 bra $L__BB3_538; + + cvt.u32.u16 %r7078, %rs182; + shr.u16 %rs659, %rs181, 8; + cvt.u32.u16 %r7079, %rs659; + +$L__BB3_533: + mov.u32 %r1363, %r7078; + setp.gt.u32 %p607, %r6829, 2879; + mov.u32 %r7089, 1; + @%p607 bra $L__BB3_538; + + mov.u32 %r4524, 8; + sub.s32 %r4525, %r4524, %r6831; + sub.s32 %r4526, %r4525, %r6830; + min.u32 %r4527, %r4526, %r1363; + setp.eq.s32 %p608, %r4527, 32; + mov.u32 %r4528, -1; + shl.b32 %r4529, %r4528, %r4527; + not.b32 %r4530, %r4529; + selp.b32 %r4531, -1, %r4530, %p608; + and.b32 %r4532, %r4531, %r7079; + shl.b32 %r4533, %r4532, %r6830; + cvt.u16.u32 %rs660, %r4533; + or.b16 %rs1025, %rs1025, %rs660; + add.s32 %r6830, %r4527, %r6830; + sub.s32 %r7078, %r1363, %r4527; + shr.u32 %r7079, %r7079, %r4527; + setp.gt.u32 %p609, %r4526, %r1363; + @%p609 bra $L__BB3_537; + + setp.ne.s32 %p610, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs661, %rs1025, 255; + setp.ne.s16 %p611, %rs661, 127; + and.pred %p612, %p610, %p611; + @%p612 bra $L__BB3_537; + + mov.u32 %r4536, 20548; + sub.s32 %r4537, %r4536, %r6829; + cvt.u64.u32 %rd342, %r4537; + add.s64 %rd343, %rd342, %rd5; + add.s64 %rd344, %rd1, %rd343; + st.global.u8 [%rd344], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p613, %rs661, 143; + selp.u32 %r6831, 1, 0, %p613; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_537: + setp.ne.s32 %p614, %r7078, 0; + mov.u32 %r7089, %r6940; + @%p614 bra $L__BB3_533; + +$L__BB3_538: + setp.ne.s32 %p615, %r1356, 0; + @%p615 bra $L__BB3_586; + + setp.eq.s32 %p616, %r7059, 0; + add.s32 %r4538, %r6381, 17477; + cvt.u64.u32 %rd345, %r4538; + add.s64 %rd346, %rd345, %rd5; + add.s64 %rd20, %rd1, %rd346; + @%p616 bra $L__BB3_578; + + shl.b16 %rs956, %rs956, 1; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p617, %r6390, 0; + mov.u32 %r7125, %r6598; + @%p617 bra $L__BB3_543; + + setp.gt.u32 %p618, %r6381, 191; + mov.u32 %r7125, 1; + mov.u32 %r6390, 0; + @%p618 bra $L__BB3_543; + + st.global.u8 [%rd20], %rs956; + add.s32 %r6381, %r6381, 1; + mov.u16 %rs956, 0; + mov.u32 %r6390, 8; + mov.u32 %r7125, %r6598; + +$L__BB3_543: + setp.lt.u32 %p619, %r6596, 3; + mov.u32 %r7093, 0; + @%p619 bra $L__BB3_546; + + setp.lt.u32 %p620, %r6596, 6; + mov.u32 %r7093, 1; + @%p620 bra $L__BB3_546; + + setp.lt.u32 %p621, %r6596, 9; + setp.eq.s32 %p622, %r6596, 11; + selp.b32 %r4544, 4, 5, %p622; + setp.lt.u32 %p623, %r6596, 11; + selp.b32 %r4545, 3, %r4544, %p623; + selp.b32 %r7093, 2, %r4545, %p621; + +$L__BB3_546: + setp.eq.s32 %p624, %r7093, 0; + @%p624 bra $L__BB3_574; + + add.s32 %r1387, %r7093, -1; + and.b32 %r1388, %r7093, 3; + setp.eq.s32 %p625, %r1388, 0; + mov.u32 %r7103, %r7093; + mov.u32 %r7104, %r7125; + @%p625 bra $L__BB3_559; + + mov.u32 %r4547, 1; + shl.b32 %r4548, %r4547, %r1387; + and.b32 %r4549, %r4548, %r6595; + setp.ne.s32 %p626, %r4549, 0; + selp.u32 %r4550, 1, 0, %p626; + cvt.u32.u16 %r4551, %rs956; + bfi.b32 %r4552, %r4551, %r4550, 1, 8; + cvt.u16.u32 %rs956, %r4552; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p627, %r6390, 0; + mov.u32 %r7104, %r7125; + @%p627 bra $L__BB3_551; + + setp.gt.u32 %p628, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r7104, %r4547; + @%p628 bra $L__BB3_551; + + add.s32 %r4556, %r6381, 17477; + cvt.u64.u32 %rd347, %r4556; + add.s64 %rd348, %rd347, %rd5; + add.s64 %rd349, %rd1, %rd348; + st.global.u8 [%rd349], %rs956; + add.s32 %r6381, %r6381, 1; + mov.u16 %rs956, 0; + mov.u32 %r6390, 8; + mov.u32 %r7104, %r7125; + +$L__BB3_551: + setp.eq.s32 %p629, %r1388, 1; + mov.u32 %r7125, %r7104; + mov.u32 %r7103, %r1387; + @%p629 bra $L__BB3_559; + + add.s32 %r7103, %r7093, -2; + mov.u32 %r4557, 1; + shl.b32 %r4558, %r4557, %r7103; + and.b32 %r4559, %r4558, %r6595; + setp.ne.s32 %p630, %r4559, 0; + selp.u32 %r4560, 1, 0, %p630; + cvt.u32.u16 %r4561, %rs956; + bfi.b32 %r4562, %r4561, %r4560, 1, 8; + cvt.u16.u32 %rs956, %r4562; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p631, %r6390, 0; + mov.u32 %r7099, %r7104; + @%p631 bra $L__BB3_555; + + setp.gt.u32 %p632, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r7099, %r4557; + @%p632 bra $L__BB3_555; + + add.s32 %r4565, %r6381, 17477; + cvt.u64.u32 %rd350, %r4565; + add.s64 %rd351, %rd350, %rd5; + add.s64 %rd352, %rd1, %rd351; + and.b16 %rs668, %rs956, 255; + st.global.u8 [%rd352], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p633, %rs668, 255; + selp.b32 %r6390, 7, 8, %p633; + mov.u16 %rs956, 0; + mov.u32 %r7099, %r7104; + +$L__BB3_555: + setp.eq.s32 %p634, %r1388, 2; + mov.u32 %r7125, %r7099; + mov.u32 %r7104, %r7099; + @%p634 bra $L__BB3_559; + + add.s32 %r7103, %r7093, -3; + mov.u32 %r4566, 1; + shl.b32 %r4567, %r4566, %r7103; + and.b32 %r4568, %r4567, %r6595; + setp.ne.s32 %p635, %r4568, 0; + selp.u32 %r4569, 1, 0, %p635; + cvt.u32.u16 %r4570, %rs956; + bfi.b32 %r4571, %r4570, %r4569, 1, 8; + cvt.u16.u32 %rs956, %r4571; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p636, %r6390, 0; + mov.u32 %r7125, %r7099; + mov.u32 %r7104, %r7099; + @%p636 bra $L__BB3_559; + + setp.gt.u32 %p637, %r6381, 191; + mov.u32 %r6390, 0; + mov.u32 %r7125, %r4566; + mov.u32 %r7104, %r4566; + @%p637 bra $L__BB3_559; + + add.s32 %r4576, %r6381, 17477; + cvt.u64.u32 %rd353, %r4576; + add.s64 %rd354, %rd353, %rd5; + add.s64 %rd355, %rd1, %rd354; + and.b16 %rs671, %rs956, 255; + st.global.u8 [%rd355], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p638, %rs671, 255; + selp.b32 %r6390, 7, 8, %p638; + mov.u16 %rs956, 0; + mov.u32 %r7125, %r7099; + mov.u32 %r7104, %r7099; + +$L__BB3_559: + setp.lt.u32 %p639, %r1387, 3; + @%p639 bra $L__BB3_574; + + mov.u32 %r7125, %r7104; + +$L__BB3_561: + add.s32 %r4577, %r7103, -1; + mov.u32 %r4578, 1; + shl.b32 %r4579, %r4578, %r4577; + and.b32 %r4580, %r4579, %r6595; + setp.ne.s32 %p640, %r4580, 0; + selp.u32 %r4581, 1, 0, %p640; + cvt.u32.u16 %r4582, %rs956; + bfi.b32 %r7113, %r4582, %r4581, 1, 8; + add.s32 %r7112, %r6390, -1; + setp.ne.s32 %p641, %r7112, 0; + mov.u32 %r7114, %r7125; + @%p641 bra $L__BB3_564; + + setp.gt.u32 %p642, %r6381, 191; + mov.u32 %r7112, 0; + mov.u32 %r7114, %r4578; + @%p642 bra $L__BB3_564; + + cvt.u16.u32 %rs672, %r7113; + and.b16 %rs673, %rs672, 255; + add.s32 %r4586, %r6381, 17477; + cvt.u64.u32 %rd356, %r4586; + add.s64 %rd357, %rd356, %rd5; + add.s64 %rd358, %rd1, %rd357; + st.global.u8 [%rd358], %rs672; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p643, %rs673, 255; + selp.b32 %r7112, 7, 8, %p643; + mov.u32 %r7113, 0; + mov.u32 %r7114, %r7125; + +$L__BB3_564: + add.s32 %r4587, %r7103, -2; + shl.b32 %r4589, %r4578, %r4587; + and.b32 %r4590, %r4589, %r6595; + setp.ne.s32 %p644, %r4590, 0; + and.b32 %r4591, %r7113, 127; + selp.u32 %r4592, 1, 0, %p644; + bfi.b32 %r7117, %r4591, %r4592, 1, 7; + add.s32 %r7116, %r7112, -1; + setp.ne.s32 %p645, %r7116, 0; + mov.u32 %r7118, %r7114; + @%p645 bra $L__BB3_567; + + setp.gt.u32 %p646, %r6381, 191; + mov.u32 %r7118, 1; + mov.u32 %r7116, 0; + @%p646 bra $L__BB3_567; + + cvt.u16.u32 %rs674, %r7117; + and.b16 %rs675, %rs674, 255; + add.s32 %r4596, %r6381, 17477; + cvt.u64.u32 %rd359, %r4596; + add.s64 %rd360, %rd359, %rd5; + add.s64 %rd361, %rd1, %rd360; + st.global.u8 [%rd361], %rs674; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p647, %rs675, 255; + selp.b32 %r7116, 7, 8, %p647; + mov.u32 %r7117, 0; + mov.u32 %r7118, %r7114; + +$L__BB3_567: + add.s32 %r4597, %r7103, -3; + mov.u32 %r4598, 1; + shl.b32 %r4599, %r4598, %r4597; + and.b32 %r4600, %r4599, %r6595; + setp.ne.s32 %p648, %r4600, 0; + and.b32 %r4601, %r7117, 127; + selp.u32 %r4602, 1, 0, %p648; + bfi.b32 %r7121, %r4601, %r4602, 1, 7; + add.s32 %r7120, %r7116, -1; + setp.ne.s32 %p649, %r7120, 0; + mov.u32 %r7122, %r7118; + @%p649 bra $L__BB3_570; + + setp.gt.u32 %p650, %r6381, 191; + mov.u32 %r7120, 0; + mov.u32 %r7122, %r4598; + @%p650 bra $L__BB3_570; + + cvt.u16.u32 %rs676, %r7121; + and.b16 %rs677, %rs676, 255; + add.s32 %r4606, %r6381, 17477; + cvt.u64.u32 %rd362, %r4606; + add.s64 %rd363, %rd362, %rd5; + add.s64 %rd364, %rd1, %rd363; + st.global.u8 [%rd364], %rs676; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p651, %rs677, 255; + selp.b32 %r7120, 7, 8, %p651; + mov.u32 %r7121, 0; + mov.u32 %r7122, %r7118; + +$L__BB3_570: + add.s32 %r7103, %r7103, -4; + shl.b32 %r4608, %r4598, %r7103; + and.b32 %r4609, %r4608, %r6595; + setp.ne.s32 %p652, %r4609, 0; + and.b32 %r4610, %r7121, 127; + selp.u32 %r4611, 1, 0, %p652; + bfi.b32 %r4612, %r4610, %r4611, 1, 15; + cvt.u16.u32 %rs956, %r4612; + add.s32 %r6390, %r7120, -1; + setp.ne.s32 %p653, %r6390, 0; + mov.u32 %r7125, %r7122; + @%p653 bra $L__BB3_573; + + setp.gt.u32 %p654, %r6381, 191; + mov.u32 %r7125, 1; + mov.u32 %r6390, 0; + @%p654 bra $L__BB3_573; + + add.s32 %r4615, %r6381, 17477; + cvt.u64.u32 %rd365, %r4615; + add.s64 %rd366, %rd365, %rd5; + add.s64 %rd367, %rd1, %rd366; + and.b16 %rs679, %rs956, 255; + st.global.u8 [%rd367], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p655, %rs679, 255; + selp.b32 %r6390, 7, 8, %p655; + mov.u16 %rs956, 0; + mov.u32 %r7125, %r7122; + +$L__BB3_573: + setp.ne.s32 %p656, %r7103, 0; + @%p656 bra $L__BB3_561; + +$L__BB3_574: + add.s32 %r4617, %r6596, -1; + setp.eq.s32 %p657, %r6596, 0; + mov.u32 %r6595, 0; + selp.b32 %r6596, 0, %r4617, %p657; + setp.lt.u32 %p658, %r6596, 3; + mov.u32 %r7129, %r6595; + @%p658 bra $L__BB3_577; + + setp.lt.u32 %p659, %r6596, 6; + mov.u32 %r7129, 1; + @%p659 bra $L__BB3_577; + + setp.lt.u32 %p660, %r6596, 9; + setp.eq.s32 %p661, %r6596, 11; + selp.b32 %r4619, 4, 5, %p661; + setp.lt.u32 %p662, %r6596, 11; + selp.b32 %r4620, 3, %r4619, %p662; + selp.b32 %r7129, 2, %r4620, %p660; + +$L__BB3_577: + mov.u32 %r4622, 1; + shl.b32 %r6597, %r4622, %r7129; + mov.u32 %r6598, %r7125; + bra.uni $L__BB3_586; + +$L__BB3_435: + setp.gt.u32 %p499, %r6381, 191; + mov.u32 %r6976, 1; + mov.u32 %r6390, 0; + @%p499 bra $L__BB3_437; + + st.global.u8 [%rd19], %rs956; + add.s32 %r6381, %r6381, 1; + mov.u16 %rs956, 0; + mov.u32 %r6390, 8; + mov.u32 %r6976, %r6598; + bra.uni $L__BB3_437; + +$L__BB3_578: + add.s32 %r6595, %r6595, 1; + setp.lt.u32 %p663, %r6595, %r6597; + @%p663 bra $L__BB3_586; + + shl.b16 %rs680, %rs956, 1; + or.b16 %rs956, %rs680, 1; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p664, %r6390, 0; + mov.u32 %r7132, %r6598; + @%p664 bra $L__BB3_582; + + setp.gt.u32 %p665, %r6381, 191; + mov.u32 %r7132, 1; + mov.u32 %r6390, 0; + @%p665 bra $L__BB3_582; + + and.b16 %rs682, %rs956, 255; + st.global.u8 [%rd20], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p666, %rs682, 255; + selp.b32 %r6390, 7, 8, %p666; + mov.u16 %rs956, 0; + mov.u32 %r7132, %r6598; + +$L__BB3_582: + add.s32 %r4626, %r6596, 1; + min.u32 %r6596, %r4626, 12; + setp.lt.u32 %p667, %r6596, 3; + mov.u32 %r6595, 0; + mov.u32 %r7133, %r6595; + @%p667 bra $L__BB3_585; + + setp.lt.u32 %p668, %r6596, 6; + mov.u32 %r7133, 1; + @%p668 bra $L__BB3_585; + + setp.lt.u32 %p669, %r6596, 9; + setp.eq.s32 %p670, %r6596, 11; + selp.b32 %r4628, 4, 5, %p670; + setp.lt.u32 %p671, %r6596, 11; + selp.b32 %r4629, 3, %r4628, %p671; + selp.b32 %r7133, 2, %r4629, %p669; + +$L__BB3_585: + mov.u32 %r4631, 1; + shl.b32 %r6597, %r4631, %r7133; + mov.u32 %r6598, %r7132; + +$L__BB3_586: + and.b16 %rs683, %rs181, 15; + cvt.u32.u16 %r1471, %rs683; + and.b32 %r4632, %r7059, 1; + setp.eq.b32 %p672, %r4632, 1; + mov.pred %p673, 0; + xor.pred %p674, %p672, %p673; + not.pred %p675, %p674; + mov.u32 %r7154, %r7050; + @%p675 bra $L__BB3_593; + + and.b32 %r4633, %r1471, 1; + sub.s32 %r7140, %r1357, %r4633; + setp.eq.s32 %p676, %r7140, 0; + mov.u32 %r7154, %r7050; + @%p676 bra $L__BB3_593; + + mov.u32 %r4634, -1; + shl.b32 %r4635, %r4634, %r7140; + not.b32 %r4636, %r4635; + and.b32 %r7141, %r7053, %r4636; + +$L__BB3_589: + setp.gt.u32 %p677, %r7016, 17476; + mov.u32 %r7154, 1; + @%p677 bra $L__BB3_593; + + sub.s32 %r4638, %r7017, %r7018; + min.u32 %r4639, %r4638, %r7140; + setp.eq.s32 %p678, %r4639, 32; + mov.u32 %r4640, -1; + shl.b32 %r4641, %r4640, %r4639; + not.b32 %r4642, %r4641; + selp.b32 %r4643, -1, %r4642, %p678; + and.b32 %r4644, %r4643, %r7141; + shl.b32 %r4645, %r4644, %r7018; + or.b32 %r7019, %r4645, %r7019; + add.s32 %r7018, %r4639, %r7018; + shr.u32 %r7141, %r7141, %r4639; + sub.s32 %r7140, %r7140, %r4639; + setp.lt.u32 %p679, %r7018, %r7017; + @%p679 bra $L__BB3_592; + + cvt.u64.u32 %rd368, %r7016; + add.s64 %rd369, %rd368, %rd5; + add.s64 %rd370, %rd1, %rd369; + st.global.u8 [%rd370], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p680, %r7019, 255; + selp.b32 %r7017, 7, 8, %p680; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_592: + setp.ne.s32 %p681, %r7140, 0; + mov.u32 %r7154, %r7050; + @%p681 bra $L__BB3_589; + +$L__BB3_593: + and.b32 %r1495, %r7059, 2; + setp.eq.s32 %p682, %r1495, 0; + mov.u32 %r7169, %r7154; + @%p682 bra $L__BB3_600; + + shr.u32 %r4648, %r1471, 1; + and.b32 %r4649, %r4648, 1; + sub.s32 %r7155, %r1357, %r4649; + setp.eq.s32 %p683, %r7155, 0; + mov.u32 %r7169, %r7154; + @%p683 bra $L__BB3_600; + + mov.u32 %r4650, -1; + shl.b32 %r4651, %r4650, %r7155; + not.b32 %r4652, %r4651; + and.b32 %r7156, %r7057, %r4652; + +$L__BB3_596: + setp.gt.u32 %p684, %r7016, 17476; + mov.u32 %r7169, 1; + @%p684 bra $L__BB3_600; + + sub.s32 %r4654, %r7017, %r7018; + min.u32 %r4655, %r4654, %r7155; + setp.eq.s32 %p685, %r4655, 32; + mov.u32 %r4656, -1; + shl.b32 %r4657, %r4656, %r4655; + not.b32 %r4658, %r4657; + selp.b32 %r4659, -1, %r4658, %p685; + and.b32 %r4660, %r4659, %r7156; + shl.b32 %r4661, %r4660, %r7018; + or.b32 %r7019, %r4661, %r7019; + add.s32 %r7018, %r4655, %r7018; + shr.u32 %r7156, %r7156, %r4655; + sub.s32 %r7155, %r7155, %r4655; + setp.lt.u32 %p686, %r7018, %r7017; + @%p686 bra $L__BB3_599; + + cvt.u64.u32 %rd371, %r7016; + add.s64 %rd372, %rd371, %rd5; + add.s64 %rd373, %rd1, %rd372; + st.global.u8 [%rd373], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p687, %r7019, 255; + selp.b32 %r7017, 7, 8, %p687; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_599: + setp.ne.s32 %p688, %r7155, 0; + mov.u32 %r7169, %r7154; + @%p688 bra $L__BB3_596; + +$L__BB3_600: + and.b32 %r1519, %r7059, 4; + setp.eq.s32 %p689, %r1519, 0; + mov.u32 %r7184, %r7169; + @%p689 bra $L__BB3_607; + + shr.u32 %r4664, %r1471, 2; + and.b32 %r4665, %r4664, 1; + sub.s32 %r7170, %r1357, %r4665; + setp.eq.s32 %p690, %r7170, 0; + mov.u32 %r7184, %r7169; + @%p690 bra $L__BB3_607; + + mov.u32 %r4666, -1; + shl.b32 %r4667, %r4666, %r7170; + not.b32 %r4668, %r4667; + and.b32 %r7171, %r7073, %r4668; + +$L__BB3_603: + setp.gt.u32 %p691, %r7016, 17476; + mov.u32 %r7184, 1; + @%p691 bra $L__BB3_607; + + sub.s32 %r4670, %r7017, %r7018; + min.u32 %r4671, %r4670, %r7170; + setp.eq.s32 %p692, %r4671, 32; + mov.u32 %r4672, -1; + shl.b32 %r4673, %r4672, %r4671; + not.b32 %r4674, %r4673; + selp.b32 %r4675, -1, %r4674, %p692; + and.b32 %r4676, %r4675, %r7171; + shl.b32 %r4677, %r4676, %r7018; + or.b32 %r7019, %r4677, %r7019; + add.s32 %r7018, %r4671, %r7018; + shr.u32 %r7171, %r7171, %r4671; + sub.s32 %r7170, %r7170, %r4671; + setp.lt.u32 %p693, %r7018, %r7017; + @%p693 bra $L__BB3_606; + + cvt.u64.u32 %rd374, %r7016; + add.s64 %rd375, %rd374, %rd5; + add.s64 %rd376, %rd1, %rd375; + st.global.u8 [%rd376], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p694, %r7019, 255; + selp.b32 %r7017, 7, 8, %p694; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_606: + setp.ne.s32 %p695, %r7170, 0; + mov.u32 %r7184, %r7169; + @%p695 bra $L__BB3_603; + +$L__BB3_607: + and.b32 %r1543, %r7059, 8; + setp.eq.s32 %p696, %r1543, 0; + mov.u32 %r7050, %r7184; + @%p696 bra $L__BB3_614; + + shr.u32 %r4680, %r1471, 3; + sub.s32 %r7185, %r1357, %r4680; + setp.eq.s32 %p697, %r7185, 0; + mov.u32 %r7050, %r7184; + @%p697 bra $L__BB3_614; + + mov.u32 %r4681, -1; + shl.b32 %r4682, %r4681, %r7185; + not.b32 %r4683, %r4682; + and.b32 %r7186, %r7072, %r4683; + +$L__BB3_610: + setp.gt.u32 %p698, %r7016, 17476; + mov.u32 %r7050, 1; + @%p698 bra $L__BB3_614; + + sub.s32 %r4685, %r7017, %r7018; + min.u32 %r4686, %r4685, %r7185; + setp.eq.s32 %p699, %r4686, 32; + mov.u32 %r4687, -1; + shl.b32 %r4688, %r4687, %r4686; + not.b32 %r4689, %r4688; + selp.b32 %r4690, -1, %r4689, %p699; + and.b32 %r4691, %r4690, %r7186; + shl.b32 %r4692, %r4691, %r7018; + or.b32 %r7019, %r4692, %r7019; + add.s32 %r7018, %r4686, %r7018; + shr.u32 %r7186, %r7186, %r4686; + sub.s32 %r7185, %r7185, %r4686; + setp.lt.u32 %p700, %r7018, %r7017; + @%p700 bra $L__BB3_613; + + cvt.u64.u32 %rd377, %r7016; + add.s64 %rd378, %rd377, %rd5; + add.s64 %rd379, %rd1, %rd378; + st.global.u8 [%rd379], %r7019; + add.s32 %r7016, %r7016, 1; + setp.eq.s32 %p701, %r7019, 255; + selp.b32 %r7017, 7, 8, %p701; + mov.u32 %r7018, 0; + mov.u32 %r7019, %r7018; + +$L__BB3_613: + setp.ne.s32 %p702, %r7185, 0; + mov.u32 %r7050, %r7184; + @%p702 bra $L__BB3_610; + +$L__BB3_614: + and.b32 %r4695, %r7056, 255; + and.b32 %r4696, %r6921, 255; + setp.lt.u32 %p703, %r4695, %r4696; + cvt.u16.u32 %rs684, %r7056; + selp.b16 %rs685, %rs179, %rs684, %p703; + st.shared.u8 [%r1295+1], %rs685; + ld.shared.u8 %rs686, [%r1295+3]; + setp.gt.u16 %p704, %rs177, %rs686; + add.s32 %r7202, %r7202, 1; + add.s32 %r4697, %r6899, 3; + selp.b32 %r4698, %r7202, %r4697, %p704; + add.s32 %r4700, %r4428, %r4698; + ld.shared.u8 %r4701, [%r4700]; + add.s32 %r6900, %r4701, -1; + shr.u32 %r4702, %r1495, 1; + or.b32 %r4703, %r1301, %r4702; + st.shared.u8 [%r1295+2], %r7070; + st.shared.u8 [%r1298+1], %r4703; + ld.shared.u8 %rs687, [%r1298+3]; + mul.wide.u16 %r4704, %rs687, 4; + add.s32 %r4705, %r4704, %r1300; + shr.u32 %r4706, %r1543, 3; + st.shared.u8 [%r1298+2], %r4706; + shr.u32 %r4707, %r1543, 2; + shr.u32 %r4708, %r1519, 1; + or.b32 %r4709, %r4707, %r4708; + or.b32 %r6897, %r4709, %r4705; + add.s32 %r6901, %r6901, 1; + mov.u32 %r6940, %r7089; + +$L__BB3_615: + mov.u32 %r6898, %r7203; + mov.u32 %r6899, %r7202; + max.s32 %r4710, %r7220, 0; + mul.lo.s32 %r4711, %r1086, 6; + setp.gt.s32 %p705, %r1086, 0; + selp.b32 %r4712, %r4711, 0, %p705; + cvt.u64.u32 %rd380, %r4712; + add.s64 %rd21, %rd17, %rd380; + ld.global.u8 %rs205, [%rd21+1]; + add.s32 %r4713, %r4712, 2; + cvt.u64.u32 %rd381, %r4713; + add.s64 %rd382, %rd17, %rd381; + ld.global.u8 %rs206, [%rd382]; + ld.global.u8 %rs207, [%rd382+1]; + mul.lo.s32 %r4714, %r4710, 6; + cvt.u64.u32 %rd383, %r4714; + add.s64 %rd384, %rd17, %rd383; + ld.global.u8 %rs208, [%rd384]; + ld.global.u8 %rs209, [%rd384+1]; + add.s32 %r4715, %r4714, 2; + cvt.u64.u32 %rd385, %r4715; + add.s64 %rd386, %rd17, %rd385; + ld.global.u8 %rs210, [%rd386]; + ld.global.u8 %rs211, [%rd386+1]; + setp.eq.s16 %p706, %rs205, 0; + mov.u32 %r7232, %r6940; + @%p706 bra $L__BB3_622; + + ld.global.u8 %r7222, [%rd21]; + cvt.u32.u16 %r7221, %rs205; + +$L__BB3_617: + mov.u32 %r1594, %r7221; + setp.gt.u32 %p707, %r6829, 2879; + mov.u32 %r7232, 1; + @%p707 bra $L__BB3_622; + + mov.u32 %r4717, 8; + sub.s32 %r4718, %r4717, %r6831; + sub.s32 %r4719, %r4718, %r6830; + min.u32 %r4720, %r4719, %r1594; + setp.eq.s32 %p708, %r4720, 32; + mov.u32 %r4721, -1; + shl.b32 %r4722, %r4721, %r4720; + not.b32 %r4723, %r4722; + selp.b32 %r4724, -1, %r4723, %p708; + and.b32 %r4725, %r4724, %r7222; + shl.b32 %r4726, %r4725, %r6830; + cvt.u16.u32 %rs688, %r4726; + or.b16 %rs1025, %rs1025, %rs688; + add.s32 %r6830, %r4720, %r6830; + sub.s32 %r7221, %r1594, %r4720; + shr.u32 %r7222, %r7222, %r4720; + setp.gt.u32 %p709, %r4719, %r1594; + @%p709 bra $L__BB3_621; + + setp.ne.s32 %p710, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs689, %rs1025, 255; + setp.ne.s16 %p711, %rs689, 127; + and.pred %p712, %p710, %p711; + @%p712 bra $L__BB3_621; + + mov.u32 %r4729, 20548; + sub.s32 %r4730, %r4729, %r6829; + cvt.u64.u32 %rd387, %r4730; + add.s64 %rd388, %rd387, %rd5; + add.s64 %rd389, %rd1, %rd388; + st.global.u8 [%rd389], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p713, %rs689, 143; + selp.u32 %r6831, 1, 0, %p713; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_621: + setp.ne.s32 %p714, %r7221, 0; + mov.u32 %r7232, %r6940; + @%p714 bra $L__BB3_617; + +$L__BB3_622: + setp.eq.s16 %p715, %rs209, 0; + mov.u32 %r7244, %r7232; + @%p715 bra $L__BB3_629; + + cvt.u32.u16 %r4731, %rs208; + and.b32 %r7234, %r4731, 255; + cvt.u32.u16 %r4732, %rs209; + and.b32 %r7233, %r4732, 255; + +$L__BB3_624: + mov.u32 %r1613, %r7233; + setp.gt.u32 %p716, %r6829, 2879; + mov.u32 %r7244, 1; + @%p716 bra $L__BB3_629; + + mov.u32 %r4734, 8; + sub.s32 %r4735, %r4734, %r6831; + sub.s32 %r4736, %r4735, %r6830; + min.u32 %r4737, %r4736, %r1613; + setp.eq.s32 %p717, %r4737, 32; + mov.u32 %r4738, -1; + shl.b32 %r4739, %r4738, %r4737; + not.b32 %r4740, %r4739; + selp.b32 %r4741, -1, %r4740, %p717; + and.b32 %r4742, %r4741, %r7234; + shl.b32 %r4743, %r4742, %r6830; + cvt.u16.u32 %rs693, %r4743; + or.b16 %rs1025, %rs1025, %rs693; + add.s32 %r6830, %r4737, %r6830; + sub.s32 %r7233, %r1613, %r4737; + shr.u32 %r7234, %r7234, %r4737; + setp.gt.u32 %p718, %r4736, %r1613; + @%p718 bra $L__BB3_628; + + setp.ne.s32 %p719, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs694, %rs1025, 255; + setp.ne.s16 %p720, %rs694, 127; + and.pred %p721, %p719, %p720; + @%p721 bra $L__BB3_628; + + mov.u32 %r4746, 20548; + sub.s32 %r4747, %r4746, %r6829; + cvt.u64.u32 %rd390, %r4747; + add.s64 %rd391, %rd390, %rd5; + add.s64 %rd392, %rd1, %rd391; + st.global.u8 [%rd392], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p722, %rs694, 143; + selp.u32 %r6831, 1, 0, %p722; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_628: + setp.ne.s32 %p723, %r7233, 0; + mov.u32 %r7244, %r7232; + @%p723 bra $L__BB3_624; + +$L__BB3_629: + setp.eq.s16 %p724, %rs207, 0; + mov.u32 %r7256, %r7244; + @%p724 bra $L__BB3_636; + + cvt.u32.u16 %r4748, %rs207; + and.b32 %r7245, %r4748, 255; + cvt.u32.u16 %r4749, %rs206; + and.b32 %r7246, %r4749, 255; + +$L__BB3_631: + mov.u32 %r1632, %r7245; + setp.gt.u32 %p725, %r6829, 2879; + mov.u32 %r7256, 1; + @%p725 bra $L__BB3_636; + + mov.u32 %r4751, 8; + sub.s32 %r4752, %r4751, %r6831; + sub.s32 %r4753, %r4752, %r6830; + min.u32 %r4754, %r4753, %r1632; + setp.eq.s32 %p726, %r4754, 32; + mov.u32 %r4755, -1; + shl.b32 %r4756, %r4755, %r4754; + not.b32 %r4757, %r4756; + selp.b32 %r4758, -1, %r4757, %p726; + and.b32 %r4759, %r4758, %r7246; + shl.b32 %r4760, %r4759, %r6830; + cvt.u16.u32 %rs698, %r4760; + or.b16 %rs1025, %rs1025, %rs698; + add.s32 %r6830, %r4754, %r6830; + sub.s32 %r7245, %r1632, %r4754; + shr.u32 %r7246, %r7246, %r4754; + setp.gt.u32 %p727, %r4753, %r1632; + @%p727 bra $L__BB3_635; + + setp.ne.s32 %p728, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs699, %rs1025, 255; + setp.ne.s16 %p729, %rs699, 127; + and.pred %p730, %p728, %p729; + @%p730 bra $L__BB3_635; + + mov.u32 %r4763, 20548; + sub.s32 %r4764, %r4763, %r6829; + cvt.u64.u32 %rd393, %r4764; + add.s64 %rd394, %rd393, %rd5; + add.s64 %rd395, %rd1, %rd394; + st.global.u8 [%rd395], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p731, %rs699, 143; + selp.u32 %r6831, 1, 0, %p731; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_635: + setp.ne.s32 %p732, %r7245, 0; + mov.u32 %r7256, %r7244; + @%p732 bra $L__BB3_631; + +$L__BB3_636: + setp.eq.s16 %p733, %rs211, 0; + mov.u32 %r6832, %r7256; + @%p733 bra $L__BB3_643; + + cvt.u32.u16 %r4765, %rs210; + and.b32 %r7258, %r4765, 255; + cvt.u32.u16 %r4766, %rs211; + and.b32 %r7257, %r4766, 255; + +$L__BB3_638: + mov.u32 %r1651, %r7257; + setp.gt.u32 %p734, %r6829, 2879; + mov.u32 %r6832, 1; + @%p734 bra $L__BB3_643; + + mov.u32 %r4768, 8; + sub.s32 %r4769, %r4768, %r6831; + sub.s32 %r4770, %r4769, %r6830; + min.u32 %r4771, %r4770, %r1651; + setp.eq.s32 %p735, %r4771, 32; + mov.u32 %r4772, -1; + shl.b32 %r4773, %r4772, %r4771; + not.b32 %r4774, %r4773; + selp.b32 %r4775, -1, %r4774, %p735; + and.b32 %r4776, %r4775, %r7258; + shl.b32 %r4777, %r4776, %r6830; + cvt.u16.u32 %rs703, %r4777; + or.b16 %rs1025, %rs1025, %rs703; + add.s32 %r6830, %r4771, %r6830; + sub.s32 %r7257, %r1651, %r4771; + shr.u32 %r7258, %r7258, %r4771; + setp.gt.u32 %p736, %r4770, %r1651; + @%p736 bra $L__BB3_642; + + setp.ne.s32 %p737, %r6831, 0; + mov.u32 %r6831, 0; + and.b16 %rs704, %rs1025, 255; + setp.ne.s16 %p738, %rs704, 127; + and.pred %p739, %p737, %p738; + @%p739 bra $L__BB3_642; + + mov.u32 %r4780, 20548; + sub.s32 %r4781, %r4780, %r6829; + cvt.u64.u32 %rd396, %r4781; + add.s64 %rd397, %rd396, %rd5; + add.s64 %rd398, %rd1, %rd397; + st.global.u8 [%rd398], %rs1025; + add.s32 %r6829, %r6829, 1; + setp.gt.u16 %p740, %rs704, 143; + selp.u32 %r6831, 1, 0, %p740; + mov.u16 %rs1025, 0; + mov.u32 %r6830, 0; + +$L__BB3_642: + setp.ne.s32 %p741, %r7257, 0; + mov.u32 %r6832, %r7256; + @%p741 bra $L__BB3_638; + +$L__BB3_643: + add.s32 %r6881, %r6881, 4; + setp.lt.u32 %p742, %r6881, %r3200; + @%p742 bra $L__BB3_403; + +$L__BB3_644: + add.s32 %r6865, %r6865, 2; + setp.lt.u32 %p743, %r6865, %r3201; + @%p743 bra $L__BB3_401; + +$L__BB3_645: + setp.eq.s32 %p744, %r6595, 0; + mov.u32 %r7298, %r6598; + @%p744 bra $L__BB3_649; + + shl.b16 %rs707, %rs956, 1; + or.b16 %rs956, %rs707, 1; + add.s32 %r6390, %r6390, -1; + setp.ne.s32 %p745, %r6390, 0; + mov.u32 %r7298, %r6598; + @%p745 bra $L__BB3_649; + + setp.gt.u32 %p746, %r6381, 191; + mov.u32 %r7298, 1; + mov.u32 %r6390, 0; + @%p746 bra $L__BB3_649; + + add.s32 %r4784, %r6381, 17477; + cvt.u64.u32 %rd399, %r4784; + add.s64 %rd400, %rd399, %rd5; + add.s64 %rd401, %rd1, %rd400; + and.b16 %rs709, %rs956, 255; + st.global.u8 [%rd401], %rs956; + add.s32 %r6381, %r6381, 1; + setp.eq.s16 %p747, %rs709, 255; + selp.b32 %r6390, 7, 8, %p747; + mov.u16 %rs956, 0; + mov.u32 %r7298, %r6598; + +$L__BB3_649: + cvt.u32.u16 %r4785, %rs956; + and.b32 %r4786, %r4785, 255; + shl.b32 %r4787, %r4786, %r6390; + cvt.u16.u32 %rs234, %r4787; + mov.u32 %r4788, -1; + shl.b32 %r4789, %r4788, %r6830; + not.b32 %r4790, %r4789; + mov.u32 %r4791, 255; + and.b32 %r4792, %r4790, 255; + setp.eq.s32 %p748, %r6830, 0; + selp.b32 %r1703, 0, %r4792, %p748; + shl.b32 %r1704, %r4791, %r6390; + and.b32 %r4793, %r1704, 255; + or.b32 %r4794, %r4793, %r1703; + setp.eq.s32 %p749, %r4794, 0; + mov.u32 %r7300, %r6832; + mov.u32 %r7302, %r7298; + @%p749 bra $L__BB3_655; + + or.b16 %rs235, %rs1025, %rs234; + and.b16 %rs710, %rs235, 255; + xor.b16 %rs711, %rs235, %rs234; + cvt.u32.u16 %r4795, %rs711; + and.b32 %r4796, %r1704, %r4795; + and.b32 %r4797, %r4796, 255; + xor.b16 %rs712, %rs235, %rs1025; + cvt.u32.u16 %r4798, %rs712; + and.b32 %r4799, %r1703, %r4798; + or.b32 %r4800, %r4797, %r4799; + setp.eq.s32 %p750, %r4800, 0; + setp.ne.s16 %p751, %rs710, 255; + and.pred %p752, %p751, %p750; + setp.gt.u32 %p753, %r6829, 1; + and.pred %p754, %p753, %p752; + add.s32 %r4801, %r6381, 17477; + cvt.u64.u32 %rd402, %r4801; + add.s64 %rd403, %rd402, %rd5; + add.s64 %rd22, %rd1, %rd403; + @%p754 bra $L__BB3_653; + bra.uni $L__BB3_651; + +$L__BB3_653: + setp.gt.u32 %p758, %r6381, 191; + mov.u32 %r7302, 1; + mov.u32 %r7300, %r6832; + @%p758 bra $L__BB3_655; + + st.global.u8 [%rd22], %rs235; + add.s32 %r6381, %r6381, 1; + mov.u32 %r7300, %r6832; + mov.u32 %r7302, %r7298; + bra.uni $L__BB3_655; + +$L__BB3_1243: + setp.gt.u32 %p1439, %r7382, 191; + setp.gt.u32 %p1440, %r7779, 2879; + or.pred %p1441, %p1440, %p1439; + mov.u32 %r8149, 1; + mov.u32 %r8151, %r8149; + @%p1441 bra $L__BB3_1247; + + st.global.u8 [%rd44], %rs447; + add.s32 %r7382, %r7382, 1; + mov.u32 %r6251, 20548; + sub.s32 %r6252, %r6251, %r7779; + cvt.u64.u32 %rd675, %r6252; + add.s64 %rd676, %rd675, %rd5; + add.s64 %rd677, %rd1, %rd676; + st.global.u8 [%rd677], %rs1147; + add.s32 %r7779, %r7779, 1; + mov.u32 %r8149, %r8146; + mov.u32 %r8151, %r7776; + +$L__BB3_1247: + setp.eq.s32 %p1443, %r7923, 0; + @%p1443 bra $L__BB3_1251; + + sub.s32 %r6254, %r7924, %r7923; + mov.u32 %r6255, -1; + shl.b32 %r6256, %r6255, %r6254; + not.b32 %r6257, %r6256; + and.b32 %r6258, %r6257, 255; + shl.b32 %r6259, %r6258, %r7923; + or.b32 %r3190, %r6259, %r7922; + setp.eq.s32 %p1444, %r3190, 255; + mov.u32 %r8153, %r8093; + @%p1444 bra $L__BB3_1253; + + setp.gt.u32 %p1445, %r7925, 17476; + mov.u32 %r8153, 1; + @%p1445 bra $L__BB3_1253; + + cvt.u64.u32 %rd678, %r7925; + add.s64 %rd679, %rd678, %rd5; + add.s64 %rd680, %rd1, %rd679; + st.global.u8 [%rd680], %r3190; + add.s32 %r7925, %r7925, 1; + mov.u32 %r8153, %r8093; + bra.uni $L__BB3_1253; + +$L__BB3_1251: + setp.ne.s32 %p1446, %r7924, 7; + mov.u32 %r8153, %r8093; + @%p1446 bra $L__BB3_1253; + + setp.eq.s32 %p1447, %r7925, 0; + add.s32 %r6261, %r7925, -1; + selp.b32 %r7925, 0, %r6261, %p1447; + mov.u32 %r8153, %r8093; + +$L__BB3_1253: + or.b32 %r6262, %r8151, %r8149; + or.b32 %r6263, %r6262, %r8153; + setp.eq.s32 %p1448, %r6263, 0; + @%p1448 bra $L__BB3_1255; + + mov.u32 %r6264, 1; + st.global.u32 [%rd6], %r6264; + mov.u32 %r6265, 3; + st.global.u32 [%rd6+4], %r6265; + mov.u32 %r6266, 0; + st.global.u32 [%rd6+8], %r6266; + st.global.u32 [%rd6+12], %r6266; + st.global.u32 [%rd6+16], %r6266; + st.global.u32 [%rd6+20], %r6266; + st.global.u32 [%rd6+24], %r6266; + st.global.u32 [%rd6+28], %r6266; + bra.uni $L__BB3_1261; + +$L__BB3_1255: + add.s32 %r6267, %r7779, %r7382; + add.s32 %r3195, %r6267, %r7925; + setp.lt.u32 %p1449, %r3195, 2; + setp.gt.u32 %p1450, %r3195, %r3204; + or.pred %p1451, %p1449, %p1450; + @%p1451 bra $L__BB3_1257; + bra.uni $L__BB3_1256; + +$L__BB3_1257: + mov.u32 %r6274, 1; + st.global.u32 [%rd6], %r6274; + mov.u32 %r6275, 4; + st.global.u32 [%rd6+4], %r6275; + mov.u32 %r6276, 0; + st.global.u32 [%rd6+8], %r6276; + st.global.u32 [%rd6+12], %r6276; + st.global.u32 [%rd6+16], %r6276; + st.global.u32 [%rd6+20], %r6276; + st.global.u32 [%rd6+24], %r6276; + st.global.u32 [%rd6+28], %r6276; + bra.uni $L__BB3_1261; + +$L__BB3_1256: + and.b32 %r6268, %r7382, 32767; + and.b32 %r6269, %r7779, 32767; + bfi.b32 %r6270, %r6269, %r6268, 15, 15; + or.b32 %r6271, %r6270, -2147483648; + mov.u32 %r6272, 0; + st.global.u32 [%rd6], %r6272; + st.global.u32 [%rd6+4], %r6272; + st.global.u32 [%rd6+8], %r3195; + mov.u32 %r6273, 1; + st.global.u32 [%rd6+12], %r6273; + add.s32 %r6285, %r3202, -1; + st.global.u32 [%rd6+16], %r6285; + st.global.u32 [%rd6+20], %r3195; + st.global.u32 [%rd6+24], %r6272; + st.global.u32 [%rd6+28], %r6271; + bra.uni $L__BB3_1261; + +$L__BB3_651: + setp.gt.u32 %p755, %r6381, 191; + setp.gt.u32 %p756, %r6829, 2879; + or.pred %p757, %p756, %p755; + mov.u32 %r7300, 1; + mov.u32 %r7302, %r7300; + @%p757 bra $L__BB3_655; + + st.global.u8 [%rd22], %rs234; + add.s32 %r6381, %r6381, 1; + mov.u32 %r4804, 20548; + sub.s32 %r4805, %r4804, %r6829; + cvt.u64.u32 %rd404, %r4805; + add.s64 %rd405, %rd404, %rd5; + add.s64 %rd406, %rd1, %rd405; + st.global.u8 [%rd406], %rs1025; + add.s32 %r6829, %r6829, 1; + mov.u32 %r7300, %r6832; + mov.u32 %r7302, %r7298; + +$L__BB3_655: + setp.eq.s32 %p759, %r7018, 0; + @%p759 bra $L__BB3_659; + + sub.s32 %r4807, %r7017, %r7018; + mov.u32 %r4808, -1; + shl.b32 %r4809, %r4808, %r4807; + not.b32 %r4810, %r4809; + and.b32 %r4811, %r4810, 255; + shl.b32 %r4812, %r4811, %r7018; + or.b32 %r1712, %r4812, %r7019; + setp.eq.s32 %p760, %r1712, 255; + mov.u32 %r7304, %r7050; + @%p760 bra $L__BB3_661; + + setp.gt.u32 %p761, %r7016, 17476; + mov.u32 %r7304, 1; + @%p761 bra $L__BB3_661; + + cvt.u64.u32 %rd407, %r7016; + add.s64 %rd408, %rd407, %rd5; + add.s64 %rd409, %rd1, %rd408; + st.global.u8 [%rd409], %r1712; + add.s32 %r7016, %r7016, 1; + mov.u32 %r7304, %r7050; + bra.uni $L__BB3_661; + +$L__BB3_659: + setp.ne.s32 %p762, %r7017, 7; + mov.u32 %r7304, %r7050; + @%p762 bra $L__BB3_661; + + setp.eq.s32 %p763, %r7016, 0; + add.s32 %r4814, %r7016, -1; + selp.b32 %r7016, 0, %r4814, %p763; + mov.u32 %r7304, %r7050; + +$L__BB3_661: + or.b32 %r4815, %r7302, %r7300; + or.b32 %r4816, %r4815, %r7304; + setp.eq.s32 %p764, %r4816, 0; + @%p764 bra $L__BB3_663; + + mov.u32 %r4817, 1; + st.global.u32 [%rd6], %r4817; + mov.u32 %r4818, 3; + st.global.u32 [%rd6+4], %r4818; + mov.u32 %r4819, 0; + st.global.u32 [%rd6+8], %r4819; + st.global.u32 [%rd6+12], %r4819; + st.global.u32 [%rd6+16], %r4819; + st.global.u32 [%rd6+20], %r4819; + st.global.u32 [%rd6+24], %r4819; + st.global.u32 [%rd6+28], %r4819; + bra.uni $L__BB3_1261; + +$L__BB3_663: + add.s32 %r4820, %r6381, %r6829; + add.s32 %r1717, %r4820, %r7016; + setp.lt.u32 %p765, %r1717, 2; + setp.gt.u32 %p766, %r1717, %r3204; + or.pred %p767, %p765, %p766; + @%p767 bra $L__BB3_665; + bra.uni $L__BB3_664; + +$L__BB3_665: + mov.u32 %r4827, 1; + st.global.u32 [%rd6], %r4827; + mov.u32 %r4828, 4; + st.global.u32 [%rd6+4], %r4828; + mov.u32 %r4829, 0; + st.global.u32 [%rd6+8], %r4829; + st.global.u32 [%rd6+12], %r4829; + st.global.u32 [%rd6+16], %r4829; + st.global.u32 [%rd6+20], %r4829; + st.global.u32 [%rd6+24], %r4829; + st.global.u32 [%rd6+28], %r4829; + bra.uni $L__BB3_1261; + +$L__BB3_664: + and.b32 %r4821, %r6381, 32767; + and.b32 %r4822, %r6829, 32767; + bfi.b32 %r4823, %r4822, %r4821, 15, 15; + or.b32 %r4824, %r4823, -2147483648; + mov.u32 %r4825, 0; + st.global.u32 [%rd6], %r4825; + st.global.u32 [%rd6+4], %r4825; + st.global.u32 [%rd6+8], %r1717; + mov.u32 %r4826, 1; + st.global.u32 [%rd6+12], %r4826; + add.s32 %r6284, %r3202, -1; + st.global.u32 [%rd6+16], %r6284; + st.global.u32 [%rd6+20], %r1717; + st.global.u32 [%rd6+24], %r4825; + st.global.u32 [%rd6+28], %r4824; + bra.uni $L__BB3_1261; + +} + // .globl j2k_htj2k_encode_codeblocks_multi_input_cleanup_64 +.visible .entry j2k_htj2k_encode_codeblocks_multi_input_cleanup_64( + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_0, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_1, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_2, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_3, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_4, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_5, + .param .u64 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_6 +) +.maxntid 128, 1, 1 +{ + .reg .pred %p<705>; + .reg .b16 %rs<581>; + .reg .b32 %r<3887>; + .reg .b64 %rd<339>; + // demoted variable + .shared .align 4 .b8 _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E9block_max[512]; + // demoted variable + .shared .align 1 .b8 _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val[513]; + // demoted variable + .shared .align 1 .b8 _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val[513]; + + ld.param.u64 %rd33, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_0]; + ld.param.u64 %rd28, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_1]; + ld.param.u64 %rd29, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_2]; + ld.param.u64 %rd31, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_4]; + ld.param.u64 %rd32, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_5]; + ld.param.u64 %rd34, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_6]; + cvta.to.global.u64 %rd1, %rd33; + mov.u32 %r1498, %ctaid.x; + cvt.u64.u32 %rd2, %r1498; + setp.ge.u64 %p1, %rd2, %rd34; + @%p1 bra $L__BB4_605; + + cvta.to.global.u64 %rd35, %rd28; + mul.lo.s64 %rd36, %rd2, 40; + add.s64 %rd37, %rd35, %rd36; + ld.global.u64 %rd38, [%rd37]; + cvta.to.global.u64 %rd3, %rd38; + ld.global.v2.u32 {%r1499, %r1500}, [%rd37+8]; + ld.global.v2.u32 {%r1502, %r1503}, [%rd37+16]; + ld.global.v2.u32 {%r1504, %r1505}, [%rd37+24]; + ld.global.v2.u32 {%r1506, %r1507}, [%rd37+32]; + cvt.u64.u32 %rd4, %r1499; + mov.u32 %r10, %tid.x; + mov.u32 %r3035, 0; + setp.lt.u32 %p2, %r10, 4096; + @%p2 bra $L__BB4_2; + bra.uni $L__BB4_4; + +$L__BB4_2: + mov.u32 %r11, %ntid.x; + mov.u32 %r3033, %r10; + +$L__BB4_3: + cvt.u64.u32 %rd39, %r3033; + add.s64 %rd40, %rd39, %rd4; + shl.b64 %rd41, %rd40, 2; + add.s64 %rd42, %rd3, %rd41; + ld.global.u32 %r1510, [%rd42]; + abs.s32 %r1511, %r1510; + max.u32 %r3035, %r3035, %r1511; + add.s32 %r3033, %r3033, %r11; + setp.lt.u32 %p3, %r3033, 4096; + @%p3 bra $L__BB4_3; + +$L__BB4_4: + shl.b32 %r1512, %r10, 2; + mov.u32 %r1513, _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E9block_max; + add.s32 %r17, %r1513, %r1512; + st.shared.u32 [%r17], %r3035; + bar.sync 0; + mov.u32 %r1514, %ntid.x; + shr.u32 %r3036, %r1514, 1; + setp.eq.s32 %p4, %r3036, 0; + @%p4 bra $L__BB4_8; + +$L__BB4_5: + setp.ge.u32 %p5, %r10, %r3036; + @%p5 bra $L__BB4_7; + + ld.shared.u32 %r1515, [%r17]; + shl.b32 %r1516, %r3036, 2; + add.s32 %r1517, %r17, %r1516; + ld.shared.u32 %r1518, [%r1517]; + setp.gt.u32 %p6, %r1515, %r1518; + add.s32 %r1519, %r3036, %r10; + selp.b32 %r1520, %r10, %r1519, %p6; + shl.b32 %r1521, %r1520, 2; + add.s32 %r1523, %r1513, %r1521; + ld.shared.u32 %r1524, [%r1523]; + st.shared.u32 [%r17], %r1524; + +$L__BB4_7: + bar.sync 0; + shr.u32 %r3036, %r3036, 1; + setp.ne.s32 %p7, %r3036, 0; + @%p7 bra $L__BB4_5; + +$L__BB4_8: + ld.shared.u32 %r21, [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E9block_max]; + setp.ne.s32 %p8, %r10, 0; + @%p8 bra $L__BB4_605; + + mov.u32 %r1525, 1; + cvt.u64.u32 %rd5, %r1505; + cvta.to.global.u64 %rd43, %rd32; + shl.b64 %rd44, %rd2, 5; + add.s64 %rd6, %rd43, %rd44; + st.global.u32 [%rd6], %r1525; + mov.u32 %r1526, 0; + st.global.u32 [%rd6+4], %r1526; + st.global.u32 [%rd6+8], %r1526; + st.global.u32 [%rd6+12], %r1526; + st.global.u32 [%rd6+16], %r1526; + st.global.u32 [%rd6+20], %r1526; + st.global.u32 [%rd6+24], %r1526; + st.global.u32 [%rd6+28], %r1526; + add.s32 %r1527, %r1502, -1; + setp.ge.u32 %p9, %r1527, %r1500; + setp.eq.s32 %p10, %r1503, 0; + or.pred %p11, %p9, %p10; + setp.gt.u32 %p12, %r1502, 1024; + or.pred %p13, %p12, %p11; + @%p13 bra $L__BB4_604; + + cvt.u16.u32 %rs213, %r1502; + mov.u16 %rs214, 4096; + div.u16 %rs215, %rs214, %rs213; + cvt.u32.u16 %r1528, %rs215; + setp.gt.u32 %p14, %r1503, %r1528; + add.s32 %r22, %r1504, -1; + setp.gt.u32 %p15, %r22, 29; + or.pred %p16, %p15, %p14; + setp.lt.u32 %p17, %r1506, 20549; + or.pred %p18, %p17, %p16; + @%p18 bra $L__BB4_604; + bra.uni $L__BB4_11; + +$L__BB4_604: + mov.u32 %r2986, 2; + st.global.u32 [%rd6], %r2986; + st.global.u32 [%rd6+4], %r1525; + st.global.u32 [%rd6+8], %r1526; + st.global.u32 [%rd6+12], %r1526; + st.global.u32 [%rd6+16], %r1526; + st.global.u32 [%rd6+20], %r1526; + st.global.u32 [%rd6+24], %r1526; + st.global.u32 [%rd6+28], %r1526; + +$L__BB4_605: + ret; + +$L__BB4_11: + setp.ne.s32 %p19, %r1502, 64; + setp.ne.s32 %p20, %r1503, 64; + or.pred %p21, %p19, %p20; + setp.ne.s32 %p22, %r1500, 64; + or.pred %p23, %p22, %p21; + @%p23 bra $L__BB4_603; + bra.uni $L__BB4_12; + +$L__BB4_603: + mov.u32 %r2983, 2; + st.global.u32 [%rd6], %r2983; + mov.u32 %r2984, 1; + st.global.u32 [%rd6+4], %r2984; + mov.u32 %r2985, 0; + st.global.u32 [%rd6+8], %r2985; + st.global.u32 [%rd6+12], %r2985; + st.global.u32 [%rd6+16], %r2985; + st.global.u32 [%rd6+20], %r2985; + st.global.u32 [%rd6+24], %r2985; + st.global.u32 [%rd6+28], %r2985; + bra.uni $L__BB4_605; + +$L__BB4_12: + setp.eq.s32 %p24, %r1507, 1; + @%p24 bra $L__BB4_14; + bra.uni $L__BB4_13; + +$L__BB4_14: + setp.eq.s32 %p25, %r21, 0; + @%p25 bra $L__BB4_602; + + clz.b32 %r1532, %r21; + mov.u32 %r1533, 32; + sub.s32 %r1534, %r1533, %r1532; + setp.gt.u32 %p26, %r1534, %r1504; + @%p26 bra $L__BB4_601; + bra.uni $L__BB4_16; + +$L__BB4_601: + mov.u32 %r2979, 1; + st.global.u32 [%rd6], %r2979; + mov.u32 %r2980, 2; + st.global.u32 [%rd6+4], %r2980; + mov.u32 %r2981, 0; + st.global.u32 [%rd6+8], %r2981; + st.global.u32 [%rd6+12], %r2981; + st.global.u32 [%rd6+16], %r2981; + st.global.u32 [%rd6+20], %r2981; + st.global.u32 [%rd6+24], %r2981; + st.global.u32 [%rd6+28], %r2981; + bra.uni $L__BB4_605; + +$L__BB4_13: + mov.u32 %r1529, 2; + st.global.u32 [%rd6], %r1529; + mov.u32 %r1530, 5; + st.global.u32 [%rd6+4], %r1530; + mov.u32 %r1531, 0; + st.global.u32 [%rd6+8], %r1531; + st.global.u32 [%rd6+12], %r1531; + st.global.u32 [%rd6+16], %r1531; + st.global.u32 [%rd6+20], %r1531; + st.global.u32 [%rd6+24], %r1531; + st.global.u32 [%rd6+28], %r1531; + bra.uni $L__BB4_605; + +$L__BB4_602: + mov.u32 %r2982, 0; + st.global.u32 [%rd6], %r2982; + st.global.u32 [%rd6+4], %r2982; + st.global.u32 [%rd6+8], %r2982; + st.global.u32 [%rd6+12], %r2982; + st.global.u32 [%rd6+16], %r1504; + st.global.u32 [%rd6+20], %r2982; + st.global.u32 [%rd6+24], %r2982; + st.global.u32 [%rd6+28], %r2982; + bra.uni $L__BB4_605; + +$L__BB4_16: + add.s64 %rd45, %rd1, %rd5; + mov.u16 %rs218, 255; + st.global.u8 [%rd45+20548], %rs218; + mov.u32 %r1550, 0; + mov.u16 %rs474, 0; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val], %rs474; + mov.u32 %r3276, 1; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+1], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+1], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+2], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+2], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+3], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+3], %rs474; + mov.u32 %r3509, 4; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+4], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+4], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+5], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+5], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+6], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+6], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+7], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+7], %rs474; + mov.u32 %r3120, 8; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+8], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+8], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+9], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+9], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+10], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+10], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+11], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+11], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+12], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+12], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+13], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+13], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+14], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+14], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+15], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+15], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+16], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+16], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+17], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+17], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+18], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+18], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+19], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+19], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+20], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+20], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+21], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+21], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+22], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+22], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+23], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+23], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+24], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+24], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+25], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+25], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+26], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+26], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+27], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+27], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+28], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+28], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+29], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+29], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+30], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+30], %rs474; + mov.u32 %r1552, 31; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+31], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+31], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+32], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+32], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+33], %rs474; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+33], %rs474; + sub.s32 %r23, %r1552, %r1504; + shl.b64 %rd46, %rd4, 2; + add.s64 %rd336, %rd3, %rd46; + mov.u16 %rs532, 15; + mov.u32 %r3037, %r1550; + mov.u32 %r3277, %r1550; + mov.u32 %r3275, %r1550; + mov.u32 %r3274, %r1550; + mov.u32 %r3111, %r1550; + mov.u32 %r3511, %r1550; + mov.u32 %r3510, %r3276; + mov.u32 %r3508, %r3276; + mov.u32 %r3829, %r1550; + mov.u32 %r3656, %r1550; + mov.u32 %r3655, %r1550; + mov.u32 %r3051, %r1550; + mov.u32 %r3654, %r1550; + mov.u32 %r3653, %r3120; + bra.uni $L__BB4_17; + +$L__BB4_45: + setp.gt.u32 %p54, %r3111, 191; + mov.u32 %r3121, 1; + mov.u32 %r3120, 0; + @%p54 bra $L__BB4_47; + + st.global.u8 [%rd9], %rs474; + add.s32 %r3111, %r3111, 1; + mov.u16 %rs474, 0; + mov.u32 %r3120, 8; + mov.u32 %r3121, %r3277; + bra.uni $L__BB4_47; + +$L__BB4_146: + setp.gt.u32 %p163, %r3111, 191; + mov.u32 %r3263, 1; + mov.u32 %r3120, 0; + @%p163 bra $L__BB4_148; + + st.global.u8 [%rd10], %rs474; + add.s32 %r3111, %r3111, 1; + mov.u16 %rs474, 0; + mov.u32 %r3120, 8; + mov.u32 %r3263, %r3277; + bra.uni $L__BB4_148; + +$L__BB4_261: + setp.gt.u32 %p297, %r3111, 191; + mov.u32 %r3380, 1; + mov.u32 %r3120, 0; + @%p297 bra $L__BB4_263; + + and.b16 %rs303, %rs474, 255; + st.global.u8 [%rd11], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p298, %rs303, 255; + selp.b32 %r3120, 7, 8, %p298; + mov.u16 %rs474, 0; + mov.u32 %r3380, %r3277; + bra.uni $L__BB4_263; + +$L__BB4_84: + setp.gt.u32 %p101, %r3111, 191; + mov.u32 %r3128, 1; + mov.u32 %r3120, 0; + @%p101 bra $L__BB4_86; + + and.b16 %rs252, %rs474, 255; + st.global.u8 [%rd9], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p102, %rs252, 255; + selp.b32 %r3120, 7, 8, %p102; + mov.u16 %rs474, 0; + mov.u32 %r3128, %r3277; + bra.uni $L__BB4_86; + +$L__BB4_185: + setp.gt.u32 %p210, %r3111, 191; + mov.u32 %r3270, 1; + mov.u32 %r3120, 0; + @%p210 bra $L__BB4_187; + + and.b16 %rs283, %rs474, 255; + st.global.u8 [%rd10], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p211, %rs283, 255; + selp.b32 %r3120, 7, 8, %p211; + mov.u16 %rs474, 0; + mov.u32 %r3270, %r3277; + bra.uni $L__BB4_187; + +$L__BB4_17: + ld.global.u32 %r41, [%rd336]; + setp.eq.s32 %p27, %r41, 0; + mov.u32 %r3054, %r1550; + @%p27 bra $L__BB4_19; + + and.b32 %r1554, %r41, -2147483648; + abs.s32 %r1555, %r41; + shl.b32 %r1556, %r1555, %r23; + or.b32 %r3054, %r1556, %r1554; + +$L__BB4_19: + shl.b32 %r1560, %r3054, 1; + shr.u32 %r1561, %r1560, %r23; + and.b32 %r44, %r1561, -2; + setp.eq.s32 %p28, %r44, 0; + mov.u32 %r3058, 0; + mov.u32 %r3055, %r3058; + mov.u32 %r3056, %r3058; + mov.u32 %r3062, %r3058; + @%p28 bra $L__BB4_21; + + add.s32 %r1563, %r44, -1; + clz.b32 %r1564, %r1563; + mov.u32 %r1565, 32; + sub.s32 %r3055, %r1565, %r1564; + shr.u32 %r1566, %r3054, 31; + add.s32 %r1567, %r1566, %r44; + add.s32 %r3056, %r1567, -2; + mov.u32 %r3062, 1; + +$L__BB4_21: + ld.global.u32 %r50, [%rd336+256]; + setp.eq.s32 %p29, %r50, 0; + @%p29 bra $L__BB4_23; + + and.b32 %r1569, %r50, -2147483648; + abs.s32 %r1570, %r50; + shl.b32 %r1571, %r1570, %r23; + or.b32 %r3058, %r1571, %r1569; + +$L__BB4_23: + shl.b32 %r1574, %r3058, 1; + shr.u32 %r1575, %r1574, %r23; + and.b32 %r53, %r1575, -2; + setp.eq.s32 %p30, %r53, 0; + mov.u32 %r3063, 0; + mov.u32 %r3059, %r3063; + mov.u32 %r3060, %r3063; + mov.u32 %r3066, %r3055; + @%p30 bra $L__BB4_25; + + or.b32 %r3062, %r3062, 2; + add.s32 %r1576, %r53, -1; + clz.b32 %r1577, %r1576; + mov.u32 %r1578, 32; + sub.s32 %r3059, %r1578, %r1577; + max.s32 %r3066, %r3055, %r3059; + shr.u32 %r1579, %r3058, 31; + add.s32 %r1580, %r1579, %r53; + add.s32 %r3060, %r1580, -2; + +$L__BB4_25: + ld.global.u32 %r62, [%rd336+4]; + setp.eq.s32 %p31, %r62, 0; + @%p31 bra $L__BB4_27; + + and.b32 %r1582, %r62, -2147483648; + abs.s32 %r1583, %r62; + shl.b32 %r1584, %r1583, %r23; + or.b32 %r3063, %r1584, %r1582; + +$L__BB4_27: + shl.b32 %r1587, %r3063, 1; + shr.u32 %r1588, %r1587, %r23; + and.b32 %r65, %r1588, -2; + setp.eq.s32 %p32, %r65, 0; + mov.u32 %r3068, 0; + mov.u32 %r3064, %r3068; + mov.u32 %r3065, %r3068; + @%p32 bra $L__BB4_29; + + or.b32 %r3062, %r3062, 4; + add.s32 %r1589, %r65, -1; + clz.b32 %r1590, %r1589; + mov.u32 %r1591, 32; + sub.s32 %r3064, %r1591, %r1590; + max.s32 %r3066, %r3066, %r3064; + shr.u32 %r1592, %r3063, 31; + add.s32 %r1593, %r1592, %r65; + add.s32 %r3065, %r1593, -2; + +$L__BB4_29: + ld.global.u32 %r74, [%rd336+260]; + setp.eq.s32 %p33, %r74, 0; + @%p33 bra $L__BB4_31; + + and.b32 %r1595, %r74, -2147483648; + abs.s32 %r1596, %r74; + shl.b32 %r1597, %r1596, %r23; + or.b32 %r3068, %r1597, %r1595; + +$L__BB4_31: + shl.b32 %r1600, %r3068, 1; + shr.u32 %r1601, %r1600, %r23; + and.b32 %r77, %r1601, -2; + setp.eq.s32 %p34, %r77, 0; + mov.u32 %r3073, 0; + mov.u32 %r3069, %r3073; + mov.u32 %r3070, %r3073; + @%p34 bra $L__BB4_33; + + or.b32 %r3062, %r3062, 8; + add.s32 %r1602, %r77, -1; + clz.b32 %r1603, %r1602; + mov.u32 %r1604, 32; + sub.s32 %r3069, %r1604, %r1603; + max.s32 %r3066, %r3066, %r3069; + shr.u32 %r1605, %r3068, 31; + add.s32 %r1606, %r1605, %r77; + add.s32 %r3070, %r1606, -2; + +$L__BB4_33: + add.s32 %r1608, %r3066, -1; + setp.lt.s32 %p35, %r3066, 2; + setp.gt.s32 %p36, %r3066, 1; + selp.b32 %r86, %r1608, 0, %p36; + @%p35 bra $L__BB4_35; + + setp.eq.s32 %p37, %r3055, %r3066; + selp.u32 %r1609, 1, 0, %p37; + setp.eq.s32 %p38, %r3059, %r3066; + selp.u32 %r1610, -1, 0, %p38; + bfi.b32 %r1611, %r1610, %r1609, 1, 1; + setp.eq.s32 %p39, %r3064, %r3066; + selp.u16 %rs219, 1, 0, %p39; + mul.wide.u16 %r1612, %rs219, 4; + or.b32 %r1613, %r1611, %r1612; + setp.eq.s32 %p40, %r3069, %r3066; + selp.u16 %rs220, 1, 0, %p40; + mul.wide.u16 %r1614, %rs220, 8; + or.b32 %r3073, %r1613, %r1614; + +$L__BB4_35: + shr.u32 %r1615, %r3037, 1; + mov.u32 %r1616, _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val; + add.s32 %r1617, %r1616, %r1615; + ld.shared.u8 %rs221, [%r1617]; + cvt.u32.u16 %r1618, %rs221; + and.b32 %r1619, %r1618, 255; + and.b32 %r1620, %r3059, 255; + setp.lt.u32 %p41, %r1620, %r1619; + cvt.u16.u32 %rs222, %r3059; + selp.b16 %rs223, %rs221, %rs222, %p41; + st.shared.u8 [%r1617], %rs223; + st.shared.u8 [%r1617+1], %r3069; + mov.u32 %r1621, _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val; + add.s32 %r1622, %r1621, %r1615; + and.b32 %r89, %r3062, 2; + cvt.u16.u32 %rs224, %r89; + shr.u16 %rs225, %rs224, 1; + ld.shared.u8 %rs226, [%r1622]; + or.b16 %rs227, %rs226, %rs225; + st.shared.u8 [%r1622], %rs227; + and.b32 %r90, %r3062, 8; + shr.u32 %r91, %r90, 3; + st.shared.u8 [%r1622+1], %r91; + shl.b32 %r1623, %r3062, 4; + shl.b32 %r1624, %r3051, 8; + or.b32 %r1625, %r1623, %r1624; + or.b32 %r1626, %r1625, %r3073; + cvta.to.global.u64 %rd47, %rd29; + mul.wide.u32 %rd48, %r1626, 2; + add.s64 %rd49, %rd47, %rd48; + ld.global.u16 %rs3, [%rd49]; + shr.u16 %rs228, %rs3, 4; + and.b16 %rs4, %rs228, 7; + setp.eq.s16 %p42, %rs4, 0; + mov.u32 %r3085, %r3511; + @%p42 bra $L__BB4_42; + + cvt.u32.u16 %r3074, %rs4; + shr.u16 %rs229, %rs3, 8; + cvt.u32.u16 %r3075, %rs229; + +$L__BB4_37: + mov.u16 %rs5, %rs532; + mov.u32 %r94, %r3074; + setp.gt.u32 %p43, %r3508, 2879; + mov.u32 %r3085, 1; + @%p43 bra $L__BB4_42; + + mov.u32 %r1628, 8; + sub.s32 %r1629, %r1628, %r3510; + sub.s32 %r1630, %r1629, %r3509; + min.u32 %r1631, %r1630, %r94; + setp.eq.s32 %p44, %r1631, 32; + mov.u32 %r1632, -1; + shl.b32 %r1633, %r1632, %r1631; + not.b32 %r1634, %r1633; + selp.b32 %r1635, -1, %r1634, %p44; + and.b32 %r1636, %r1635, %r3075; + shl.b32 %r1637, %r1636, %r3509; + cvt.u16.u32 %rs230, %r1637; + or.b16 %rs532, %rs5, %rs230; + add.s32 %r3509, %r1631, %r3509; + sub.s32 %r3074, %r94, %r1631; + shr.u32 %r3075, %r3075, %r1631; + setp.gt.u32 %p45, %r1630, %r94; + @%p45 bra $L__BB4_41; + + setp.ne.s32 %p46, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs231, %rs532, 255; + setp.ne.s16 %p47, %rs231, 127; + and.pred %p48, %p46, %p47; + @%p48 bra $L__BB4_41; + + cvt.u16.u32 %rs451, %r1637; + or.b16 %rs450, %rs5, %rs451; + mov.u32 %r1640, 20548; + sub.s32 %r1641, %r1640, %r3508; + cvt.u64.u32 %rd50, %r1641; + add.s64 %rd51, %rd50, %rd5; + add.s64 %rd52, %rd1, %rd51; + st.global.u8 [%rd52], %rs450; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p49, %rs231, 143; + selp.u32 %r3510, 1, 0, %p49; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_41: + setp.ne.s32 %p50, %r3074, 0; + mov.u32 %r3085, %r3511; + @%p50 bra $L__BB4_37; + +$L__BB4_42: + setp.ne.s32 %p51, %r3051, 0; + @%p51 bra $L__BB4_90; + + setp.eq.s32 %p52, %r3062, 0; + add.s32 %r1642, %r3111, 17477; + cvt.u64.u32 %rd53, %r1642; + add.s64 %rd54, %rd53, %rd5; + add.s64 %rd9, %rd1, %rd54; + @%p52 bra $L__BB4_82; + + shl.b16 %rs474, %rs474, 1; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p53, %r3120, 0; + mov.u32 %r3121, %r3277; + @%p53 bra $L__BB4_47; + bra.uni $L__BB4_45; + +$L__BB4_47: + setp.lt.u32 %p55, %r3275, 3; + mov.u32 %r3089, 0; + @%p55 bra $L__BB4_50; + + setp.lt.u32 %p56, %r3275, 6; + mov.u32 %r3089, 1; + @%p56 bra $L__BB4_50; + + setp.lt.u32 %p57, %r3275, 9; + setp.eq.s32 %p58, %r3275, 11; + selp.b32 %r1648, 4, 5, %p58; + setp.lt.u32 %p59, %r3275, 11; + selp.b32 %r1649, 3, %r1648, %p59; + selp.b32 %r3089, 2, %r1649, %p57; + +$L__BB4_50: + setp.eq.s32 %p60, %r3089, 0; + @%p60 bra $L__BB4_78; + + and.b32 %r119, %r3089, 3; + setp.eq.s32 %p61, %r119, 0; + mov.u32 %r3099, %r3089; + mov.u32 %r3100, %r3121; + @%p61 bra $L__BB4_63; + + add.s32 %r3005, %r3089, -1; + mov.u32 %r1651, 1; + shl.b32 %r1652, %r1651, %r3005; + and.b32 %r1653, %r1652, %r3274; + setp.ne.s32 %p62, %r1653, 0; + selp.u32 %r1654, 1, 0, %p62; + cvt.u32.u16 %r1655, %rs474; + bfi.b32 %r1656, %r1655, %r1654, 1, 8; + cvt.u16.u32 %rs474, %r1656; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p63, %r3120, 0; + mov.u32 %r3100, %r3121; + @%p63 bra $L__BB4_55; + + setp.gt.u32 %p64, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3100, %r1651; + @%p64 bra $L__BB4_55; + + add.s32 %r1660, %r3111, 17477; + cvt.u64.u32 %rd55, %r1660; + add.s64 %rd56, %rd55, %rd5; + add.s64 %rd57, %rd1, %rd56; + st.global.u8 [%rd57], %rs474; + add.s32 %r3111, %r3111, 1; + mov.u16 %rs474, 0; + mov.u32 %r3120, 8; + mov.u32 %r3100, %r3121; + +$L__BB4_55: + and.b32 %r3007, %r3089, 3; + add.s32 %r3099, %r3089, -1; + setp.eq.s32 %p65, %r3007, 1; + mov.u32 %r3121, %r3100; + @%p65 bra $L__BB4_63; + + add.s32 %r3099, %r3089, -2; + mov.u32 %r1661, 1; + shl.b32 %r1662, %r1661, %r3099; + and.b32 %r1663, %r1662, %r3274; + setp.ne.s32 %p66, %r1663, 0; + selp.u32 %r1664, 1, 0, %p66; + cvt.u32.u16 %r1665, %rs474; + bfi.b32 %r1666, %r1665, %r1664, 1, 8; + cvt.u16.u32 %rs474, %r1666; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p67, %r3120, 0; + mov.u32 %r3095, %r3100; + @%p67 bra $L__BB4_59; + + setp.gt.u32 %p68, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3095, %r1661; + @%p68 bra $L__BB4_59; + + add.s32 %r1669, %r3111, 17477; + cvt.u64.u32 %rd58, %r1669; + add.s64 %rd59, %rd58, %rd5; + add.s64 %rd60, %rd1, %rd59; + and.b16 %rs238, %rs474, 255; + st.global.u8 [%rd60], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p69, %rs238, 255; + selp.b32 %r3120, 7, 8, %p69; + mov.u16 %rs474, 0; + mov.u32 %r3095, %r3100; + +$L__BB4_59: + and.b32 %r3008, %r3089, 3; + setp.eq.s32 %p70, %r3008, 2; + mov.u32 %r3121, %r3095; + mov.u32 %r3100, %r3095; + @%p70 bra $L__BB4_63; + + add.s32 %r3099, %r3089, -3; + mov.u32 %r1670, 1; + shl.b32 %r1671, %r1670, %r3099; + and.b32 %r1672, %r1671, %r3274; + setp.ne.s32 %p71, %r1672, 0; + selp.u32 %r1673, 1, 0, %p71; + cvt.u32.u16 %r1674, %rs474; + bfi.b32 %r1675, %r1674, %r1673, 1, 8; + cvt.u16.u32 %rs474, %r1675; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p72, %r3120, 0; + mov.u32 %r3121, %r3095; + mov.u32 %r3100, %r3095; + @%p72 bra $L__BB4_63; + + add.s32 %r3099, %r3089, -3; + setp.gt.u32 %p73, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3121, %r1670; + mov.u32 %r3100, %r1670; + @%p73 bra $L__BB4_63; + + add.s32 %r3099, %r3089, -3; + add.s32 %r1680, %r3111, 17477; + cvt.u64.u32 %rd61, %r1680; + add.s64 %rd62, %rd61, %rd5; + add.s64 %rd63, %rd1, %rd62; + and.b16 %rs241, %rs474, 255; + st.global.u8 [%rd63], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p74, %rs241, 255; + selp.b32 %r3120, 7, 8, %p74; + mov.u16 %rs474, 0; + mov.u32 %r3121, %r3095; + mov.u32 %r3100, %r3095; + +$L__BB4_63: + add.s32 %r3009, %r3089, -1; + setp.lt.u32 %p75, %r3009, 3; + @%p75 bra $L__BB4_78; + + mov.u32 %r3121, %r3100; + +$L__BB4_65: + add.s32 %r1681, %r3099, -1; + mov.u32 %r1682, 1; + shl.b32 %r1683, %r1682, %r1681; + and.b32 %r1684, %r1683, %r3274; + setp.ne.s32 %p76, %r1684, 0; + selp.u32 %r1685, 1, 0, %p76; + cvt.u32.u16 %r1686, %rs474; + bfi.b32 %r3109, %r1686, %r1685, 1, 8; + add.s32 %r3108, %r3120, -1; + setp.ne.s32 %p77, %r3108, 0; + mov.u32 %r3110, %r3121; + @%p77 bra $L__BB4_68; + + setp.gt.u32 %p78, %r3111, 191; + mov.u32 %r3108, 0; + mov.u32 %r3110, %r1682; + @%p78 bra $L__BB4_68; + + cvt.u16.u32 %rs242, %r3109; + and.b16 %rs243, %rs242, 255; + add.s32 %r1690, %r3111, 17477; + cvt.u64.u32 %rd64, %r1690; + add.s64 %rd65, %rd64, %rd5; + add.s64 %rd66, %rd1, %rd65; + st.global.u8 [%rd66], %rs242; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p79, %rs243, 255; + selp.b32 %r3108, 7, 8, %p79; + mov.u32 %r3109, 0; + mov.u32 %r3110, %r3121; + +$L__BB4_68: + add.s32 %r1691, %r3099, -2; + shl.b32 %r1693, %r1682, %r1691; + and.b32 %r1694, %r1693, %r3274; + setp.ne.s32 %p80, %r1694, 0; + and.b32 %r1695, %r3109, 127; + selp.u32 %r1696, 1, 0, %p80; + bfi.b32 %r3113, %r1695, %r1696, 1, 7; + add.s32 %r3112, %r3108, -1; + setp.ne.s32 %p81, %r3112, 0; + mov.u32 %r3114, %r3110; + @%p81 bra $L__BB4_71; + + setp.gt.u32 %p82, %r3111, 191; + mov.u32 %r3114, 1; + mov.u32 %r3112, 0; + @%p82 bra $L__BB4_71; + + cvt.u16.u32 %rs244, %r3113; + and.b16 %rs245, %rs244, 255; + add.s32 %r1700, %r3111, 17477; + cvt.u64.u32 %rd67, %r1700; + add.s64 %rd68, %rd67, %rd5; + add.s64 %rd69, %rd1, %rd68; + st.global.u8 [%rd69], %rs244; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p83, %rs245, 255; + selp.b32 %r3112, 7, 8, %p83; + mov.u32 %r3113, 0; + mov.u32 %r3114, %r3110; + +$L__BB4_71: + add.s32 %r1701, %r3099, -3; + mov.u32 %r1702, 1; + shl.b32 %r1703, %r1702, %r1701; + and.b32 %r1704, %r1703, %r3274; + setp.ne.s32 %p84, %r1704, 0; + and.b32 %r1705, %r3113, 127; + selp.u32 %r1706, 1, 0, %p84; + bfi.b32 %r3117, %r1705, %r1706, 1, 7; + add.s32 %r3116, %r3112, -1; + setp.ne.s32 %p85, %r3116, 0; + mov.u32 %r3118, %r3114; + @%p85 bra $L__BB4_74; + + setp.gt.u32 %p86, %r3111, 191; + mov.u32 %r3116, 0; + mov.u32 %r3118, %r1702; + @%p86 bra $L__BB4_74; + + cvt.u16.u32 %rs246, %r3117; + and.b16 %rs247, %rs246, 255; + add.s32 %r1710, %r3111, 17477; + cvt.u64.u32 %rd70, %r1710; + add.s64 %rd71, %rd70, %rd5; + add.s64 %rd72, %rd1, %rd71; + st.global.u8 [%rd72], %rs246; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p87, %rs247, 255; + selp.b32 %r3116, 7, 8, %p87; + mov.u32 %r3117, 0; + mov.u32 %r3118, %r3114; + +$L__BB4_74: + add.s32 %r3099, %r3099, -4; + shl.b32 %r1712, %r1702, %r3099; + and.b32 %r1713, %r1712, %r3274; + setp.ne.s32 %p88, %r1713, 0; + and.b32 %r1714, %r3117, 127; + selp.u32 %r1715, 1, 0, %p88; + bfi.b32 %r1716, %r1714, %r1715, 1, 15; + cvt.u16.u32 %rs474, %r1716; + add.s32 %r3120, %r3116, -1; + setp.ne.s32 %p89, %r3120, 0; + mov.u32 %r3121, %r3118; + @%p89 bra $L__BB4_77; + + setp.gt.u32 %p90, %r3111, 191; + mov.u32 %r3121, 1; + mov.u32 %r3120, 0; + @%p90 bra $L__BB4_77; + + add.s32 %r1719, %r3111, 17477; + cvt.u64.u32 %rd73, %r1719; + add.s64 %rd74, %rd73, %rd5; + add.s64 %rd75, %rd1, %rd74; + and.b16 %rs249, %rs474, 255; + st.global.u8 [%rd75], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p91, %rs249, 255; + selp.b32 %r3120, 7, 8, %p91; + mov.u16 %rs474, 0; + mov.u32 %r3121, %r3118; + +$L__BB4_77: + setp.ne.s32 %p92, %r3099, 0; + @%p92 bra $L__BB4_65; + +$L__BB4_78: + add.s32 %r1721, %r3275, -1; + setp.eq.s32 %p93, %r3275, 0; + mov.u32 %r3274, 0; + selp.b32 %r3275, 0, %r1721, %p93; + setp.lt.u32 %p94, %r3275, 3; + mov.u32 %r3125, %r3274; + @%p94 bra $L__BB4_81; + + setp.lt.u32 %p95, %r3275, 6; + mov.u32 %r3125, 1; + @%p95 bra $L__BB4_81; + + setp.lt.u32 %p96, %r3275, 9; + setp.eq.s32 %p97, %r3275, 11; + selp.b32 %r1723, 4, 5, %p97; + setp.lt.u32 %p98, %r3275, 11; + selp.b32 %r1724, 3, %r1723, %p98; + selp.b32 %r3125, 2, %r1724, %p96; + +$L__BB4_81: + mov.u32 %r1726, 1; + shl.b32 %r3276, %r1726, %r3125; + mov.u32 %r3277, %r3121; + bra.uni $L__BB4_90; + +$L__BB4_82: + add.s32 %r3274, %r3274, 1; + setp.lt.u32 %p99, %r3274, %r3276; + @%p99 bra $L__BB4_90; + + shl.b16 %rs250, %rs474, 1; + or.b16 %rs474, %rs250, 1; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p100, %r3120, 0; + mov.u32 %r3128, %r3277; + @%p100 bra $L__BB4_86; + bra.uni $L__BB4_84; + +$L__BB4_86: + add.s32 %r1730, %r3275, 1; + min.u32 %r3275, %r1730, 12; + setp.lt.u32 %p103, %r3275, 3; + mov.u32 %r3274, 0; + mov.u32 %r3129, %r3274; + @%p103 bra $L__BB4_89; + + setp.lt.u32 %p104, %r3275, 6; + mov.u32 %r3129, 1; + @%p104 bra $L__BB4_89; + + setp.lt.u32 %p105, %r3275, 9; + setp.eq.s32 %p106, %r3275, 11; + selp.b32 %r1732, 4, 5, %p106; + setp.lt.u32 %p107, %r3275, 11; + selp.b32 %r1733, 3, %r1732, %p107; + selp.b32 %r3129, 2, %r1733, %p105; + +$L__BB4_89: + mov.u32 %r1735, 1; + shl.b32 %r3276, %r1735, %r3129; + mov.u32 %r3277, %r3128; + +$L__BB4_90: + max.s32 %r202, %r3066, 1; + and.b16 %rs253, %rs3, 15; + cvt.u32.u16 %r203, %rs253; + and.b32 %r204, %r3062, 1; + setp.eq.s32 %p108, %r204, 0; + mov.u32 %r3150, %r3829; + @%p108 bra $L__BB4_97; + + and.b32 %r1736, %r203, 1; + sub.s32 %r3136, %r202, %r1736; + setp.eq.s32 %p109, %r3136, 0; + mov.u32 %r3150, %r3829; + @%p109 bra $L__BB4_97; + + mov.u32 %r1737, -1; + shl.b32 %r1738, %r1737, %r3136; + not.b32 %r1739, %r1738; + and.b32 %r3137, %r3056, %r1739; + +$L__BB4_93: + setp.gt.u32 %p110, %r3654, 17476; + mov.u32 %r3150, 1; + @%p110 bra $L__BB4_97; + + sub.s32 %r1741, %r3653, %r3655; + min.u32 %r1742, %r1741, %r3136; + setp.eq.s32 %p111, %r1742, 32; + mov.u32 %r1743, -1; + shl.b32 %r1744, %r1743, %r1742; + not.b32 %r1745, %r1744; + selp.b32 %r1746, -1, %r1745, %p111; + and.b32 %r1747, %r1746, %r3137; + shl.b32 %r1748, %r1747, %r3655; + or.b32 %r3656, %r1748, %r3656; + add.s32 %r3655, %r1742, %r3655; + shr.u32 %r3137, %r3137, %r1742; + sub.s32 %r3136, %r3136, %r1742; + setp.lt.u32 %p112, %r3655, %r3653; + @%p112 bra $L__BB4_96; + + cvt.u64.u32 %rd76, %r3654; + add.s64 %rd77, %rd76, %rd5; + add.s64 %rd78, %rd1, %rd77; + st.global.u8 [%rd78], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p113, %r3656, 255; + selp.b32 %r3653, 7, 8, %p113; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_96: + setp.ne.s32 %p114, %r3136, 0; + mov.u32 %r3150, %r3829; + @%p114 bra $L__BB4_93; + +$L__BB4_97: + and.b32 %r2990, %r3062, 2; + setp.eq.s32 %p115, %r2990, 0; + mov.u32 %r3165, %r3150; + @%p115 bra $L__BB4_104; + + shr.u32 %r1751, %r203, 1; + and.b32 %r1752, %r1751, 1; + sub.s32 %r3151, %r202, %r1752; + setp.eq.s32 %p116, %r3151, 0; + mov.u32 %r3165, %r3150; + @%p116 bra $L__BB4_104; + + mov.u32 %r1753, -1; + shl.b32 %r1754, %r1753, %r3151; + not.b32 %r1755, %r1754; + and.b32 %r3152, %r3060, %r1755; + +$L__BB4_100: + setp.gt.u32 %p117, %r3654, 17476; + mov.u32 %r3165, 1; + @%p117 bra $L__BB4_104; + + sub.s32 %r1757, %r3653, %r3655; + min.u32 %r1758, %r1757, %r3151; + setp.eq.s32 %p118, %r1758, 32; + mov.u32 %r1759, -1; + shl.b32 %r1760, %r1759, %r1758; + not.b32 %r1761, %r1760; + selp.b32 %r1762, -1, %r1761, %p118; + and.b32 %r1763, %r1762, %r3152; + shl.b32 %r1764, %r1763, %r3655; + or.b32 %r3656, %r1764, %r3656; + add.s32 %r3655, %r1758, %r3655; + shr.u32 %r3152, %r3152, %r1758; + sub.s32 %r3151, %r3151, %r1758; + setp.lt.u32 %p119, %r3655, %r3653; + @%p119 bra $L__BB4_103; + + cvt.u64.u32 %rd79, %r3654; + add.s64 %rd80, %rd79, %rd5; + add.s64 %rd81, %rd1, %rd80; + st.global.u8 [%rd81], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p120, %r3656, 255; + selp.b32 %r3653, 7, 8, %p120; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_103: + setp.ne.s32 %p121, %r3151, 0; + mov.u32 %r3165, %r3150; + @%p121 bra $L__BB4_100; + +$L__BB4_104: + and.b32 %r1767, %r3062, 4; + setp.eq.s32 %p122, %r1767, 0; + mov.u32 %r3180, %r3165; + @%p122 bra $L__BB4_111; + + shr.u32 %r1768, %r203, 2; + and.b32 %r1769, %r1768, 1; + sub.s32 %r3166, %r202, %r1769; + setp.eq.s32 %p123, %r3166, 0; + mov.u32 %r3180, %r3165; + @%p123 bra $L__BB4_111; + + mov.u32 %r1770, -1; + shl.b32 %r1771, %r1770, %r3166; + not.b32 %r1772, %r1771; + and.b32 %r3167, %r3065, %r1772; + +$L__BB4_107: + setp.gt.u32 %p124, %r3654, 17476; + mov.u32 %r3180, 1; + @%p124 bra $L__BB4_111; + + sub.s32 %r1774, %r3653, %r3655; + min.u32 %r1775, %r1774, %r3166; + setp.eq.s32 %p125, %r1775, 32; + mov.u32 %r1776, -1; + shl.b32 %r1777, %r1776, %r1775; + not.b32 %r1778, %r1777; + selp.b32 %r1779, -1, %r1778, %p125; + and.b32 %r1780, %r1779, %r3167; + shl.b32 %r1781, %r1780, %r3655; + or.b32 %r3656, %r1781, %r3656; + add.s32 %r3655, %r1775, %r3655; + shr.u32 %r3167, %r3167, %r1775; + sub.s32 %r3166, %r3166, %r1775; + setp.lt.u32 %p126, %r3655, %r3653; + @%p126 bra $L__BB4_110; + + cvt.u64.u32 %rd82, %r3654; + add.s64 %rd83, %rd82, %rd5; + add.s64 %rd84, %rd1, %rd83; + st.global.u8 [%rd84], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p127, %r3656, 255; + selp.b32 %r3653, 7, 8, %p127; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_110: + setp.ne.s32 %p128, %r3166, 0; + mov.u32 %r3180, %r3165; + @%p128 bra $L__BB4_107; + +$L__BB4_111: + and.b32 %r2991, %r3062, 8; + setp.eq.s32 %p129, %r2991, 0; + mov.u32 %r3195, %r3180; + @%p129 bra $L__BB4_118; + + shr.u32 %r1784, %r203, 3; + sub.s32 %r3181, %r202, %r1784; + setp.eq.s32 %p130, %r3181, 0; + mov.u32 %r3195, %r3180; + @%p130 bra $L__BB4_118; + + mov.u32 %r1785, -1; + shl.b32 %r1786, %r1785, %r3181; + not.b32 %r1787, %r1786; + and.b32 %r3182, %r3070, %r1787; + +$L__BB4_114: + setp.gt.u32 %p131, %r3654, 17476; + mov.u32 %r3195, 1; + @%p131 bra $L__BB4_118; + + sub.s32 %r1789, %r3653, %r3655; + min.u32 %r1790, %r1789, %r3181; + setp.eq.s32 %p132, %r1790, 32; + mov.u32 %r1791, -1; + shl.b32 %r1792, %r1791, %r1790; + not.b32 %r1793, %r1792; + selp.b32 %r1794, -1, %r1793, %p132; + and.b32 %r1795, %r1794, %r3182; + shl.b32 %r1796, %r1795, %r3655; + or.b32 %r3656, %r1796, %r3656; + add.s32 %r3655, %r1790, %r3655; + shr.u32 %r3182, %r3182, %r1790; + sub.s32 %r3181, %r3181, %r1790; + setp.lt.u32 %p133, %r3655, %r3653; + @%p133 bra $L__BB4_117; + + cvt.u64.u32 %rd85, %r3654; + add.s64 %rd86, %rd85, %rd5; + add.s64 %rd87, %rd1, %rd86; + st.global.u8 [%rd87], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p134, %r3656, 255; + selp.b32 %r3653, 7, 8, %p134; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_117: + setp.ne.s32 %p135, %r3181, 0; + mov.u32 %r3195, %r3180; + @%p135 bra $L__BB4_114; + +$L__BB4_118: + ld.global.u32 %r297, [%rd336+8]; + setp.eq.s32 %p136, %r297, 0; + mov.u32 %r3197, 0; + mov.u32 %r3196, %r3197; + @%p136 bra $L__BB4_120; + + and.b32 %r1800, %r297, -2147483648; + abs.s32 %r1801, %r297; + shl.b32 %r1802, %r1801, %r23; + or.b32 %r3196, %r1802, %r1800; + +$L__BB4_120: + shl.b32 %r1806, %r3196, 1; + shr.u32 %r1807, %r1806, %r23; + and.b32 %r300, %r1807, -2; + setp.eq.s32 %p137, %r300, 0; + mov.u32 %r3198, %r3197; + mov.u32 %r3204, %r3197; + @%p137 bra $L__BB4_122; + + add.s32 %r1809, %r300, -1; + clz.b32 %r1810, %r1809; + mov.u32 %r1811, 32; + sub.s32 %r3197, %r1811, %r1810; + shr.u32 %r1812, %r3196, 31; + add.s32 %r1813, %r1812, %r300; + add.s32 %r3198, %r1813, -2; + mov.u32 %r3204, 1; + +$L__BB4_122: + ld.global.u32 %r306, [%rd336+264]; + setp.eq.s32 %p138, %r306, 0; + mov.u32 %r3201, 0; + mov.u32 %r3200, %r3201; + @%p138 bra $L__BB4_124; + + and.b32 %r1815, %r306, -2147483648; + abs.s32 %r1816, %r306; + shl.b32 %r1817, %r1816, %r23; + or.b32 %r3200, %r1817, %r1815; + +$L__BB4_124: + shl.b32 %r1820, %r3200, 1; + shr.u32 %r1821, %r1820, %r23; + and.b32 %r309, %r1821, -2; + setp.eq.s32 %p139, %r309, 0; + mov.u32 %r3202, %r3201; + mov.u32 %r3208, %r3197; + @%p139 bra $L__BB4_126; + + or.b32 %r3204, %r3204, 2; + add.s32 %r1822, %r309, -1; + clz.b32 %r1823, %r1822; + mov.u32 %r1824, 32; + sub.s32 %r3201, %r1824, %r1823; + max.s32 %r3208, %r3197, %r3201; + shr.u32 %r1825, %r3200, 31; + add.s32 %r1826, %r1825, %r309; + add.s32 %r3202, %r1826, -2; + +$L__BB4_126: + ld.global.u32 %r318, [%rd336+12]; + setp.eq.s32 %p140, %r318, 0; + mov.u32 %r3206, 0; + mov.u32 %r3205, %r3206; + @%p140 bra $L__BB4_128; + + and.b32 %r1828, %r318, -2147483648; + abs.s32 %r1829, %r318; + shl.b32 %r1830, %r1829, %r23; + or.b32 %r3205, %r1830, %r1828; + +$L__BB4_128: + shl.b32 %r1833, %r3205, 1; + shr.u32 %r1834, %r1833, %r23; + and.b32 %r321, %r1834, -2; + setp.eq.s32 %p141, %r321, 0; + mov.u32 %r3207, %r3206; + @%p141 bra $L__BB4_130; + + or.b32 %r3204, %r3204, 4; + add.s32 %r1835, %r321, -1; + clz.b32 %r1836, %r1835; + mov.u32 %r1837, 32; + sub.s32 %r3206, %r1837, %r1836; + max.s32 %r3208, %r3208, %r3206; + shr.u32 %r1838, %r3205, 31; + add.s32 %r1839, %r1838, %r321; + add.s32 %r3207, %r1839, -2; + +$L__BB4_130: + ld.global.u32 %r330, [%rd336+268]; + setp.eq.s32 %p142, %r330, 0; + mov.u32 %r3211, 0; + mov.u32 %r3210, %r3211; + @%p142 bra $L__BB4_132; + + and.b32 %r1841, %r330, -2147483648; + abs.s32 %r1842, %r330; + shl.b32 %r1843, %r1842, %r23; + or.b32 %r3210, %r1843, %r1841; + +$L__BB4_132: + shl.b32 %r1846, %r3210, 1; + shr.u32 %r1847, %r1846, %r23; + and.b32 %r333, %r1847, -2; + setp.eq.s32 %p143, %r333, 0; + mov.u32 %r3212, %r3211; + @%p143 bra $L__BB4_134; + + or.b32 %r3204, %r3204, 8; + add.s32 %r1848, %r333, -1; + clz.b32 %r1849, %r1848; + mov.u32 %r1850, 32; + sub.s32 %r3211, %r1850, %r1849; + max.s32 %r3208, %r3208, %r3211; + shr.u32 %r1851, %r3210, 31; + add.s32 %r1852, %r1851, %r333; + add.s32 %r3212, %r1852, -2; + +$L__BB4_134: + and.b32 %r3012, %r3062, 1; + shr.u32 %r1854, %r3062, 1; + or.b32 %r342, %r1854, %r3012; + add.s32 %r1855, %r3208, -1; + setp.lt.s32 %p144, %r3208, 2; + setp.gt.s32 %p145, %r3208, 1; + selp.b32 %r343, %r1855, 0, %p145; + mov.u32 %r3215, 0; + @%p144 bra $L__BB4_136; + + setp.eq.s32 %p146, %r3197, %r3208; + selp.u32 %r1856, 1, 0, %p146; + setp.eq.s32 %p147, %r3201, %r3208; + selp.u32 %r1857, -1, 0, %p147; + bfi.b32 %r1858, %r1857, %r1856, 1, 1; + setp.eq.s32 %p148, %r3206, %r3208; + selp.u16 %rs254, 1, 0, %p148; + mul.wide.u16 %r1859, %rs254, 4; + or.b32 %r1860, %r1858, %r1859; + setp.eq.s32 %p149, %r3211, %r3208; + selp.u16 %rs255, 1, 0, %p149; + mul.wide.u16 %r1861, %rs255, 8; + or.b32 %r3215, %r1860, %r1861; + +$L__BB4_136: + shr.u32 %r2998, %r3037, 1; + cvta.to.global.u64 %rd328, %rd29; + mov.u32 %r2997, _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val; + add.s32 %r2996, %r2997, %r2998; + and.b32 %r2995, %r3062, 8; + shr.u32 %r2994, %r2995, 3; + mov.u32 %r2993, _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val; + add.s32 %r2992, %r2993, %r2998; + and.b32 %r1862, %r3201, 255; + and.b32 %r1863, %r3069, 255; + setp.lt.u32 %p150, %r1862, %r1863; + cvt.u16.u32 %rs256, %r3069; + cvt.u16.u32 %rs257, %r3201; + selp.b16 %rs258, %rs256, %rs257, %p150; + st.shared.u8 [%r2992+1], %rs258; + st.shared.u8 [%r2992+2], %r3211; + and.b32 %r346, %r3204, 2; + shr.u32 %r1867, %r346, 1; + or.b32 %r1868, %r2994, %r1867; + st.shared.u8 [%r2996+1], %r1868; + and.b32 %r347, %r3204, 8; + shr.u32 %r1871, %r347, 3; + st.shared.u8 [%r2996+2], %r1871; + shl.b32 %r1872, %r3204, 4; + shl.b32 %r1873, %r342, 8; + or.b32 %r1874, %r1872, %r1873; + or.b32 %r1875, %r1874, %r3215; + mul.wide.u32 %rd89, %r1875, 2; + add.s64 %rd90, %rd328, %rd89; + ld.global.u16 %rs25, [%rd90]; + shr.u16 %rs259, %rs25, 4; + and.b16 %rs26, %rs259, 7; + setp.eq.s16 %p151, %rs26, 0; + mov.u32 %r3227, %r3085; + @%p151 bra $L__BB4_143; + + cvt.u32.u16 %r3216, %rs26; + shr.u16 %rs260, %rs25, 8; + cvt.u32.u16 %r3217, %rs260; + +$L__BB4_138: + mov.u16 %rs27, %rs532; + mov.u32 %r350, %r3216; + setp.gt.u32 %p152, %r3508, 2879; + mov.u32 %r3227, 1; + @%p152 bra $L__BB4_143; + + mov.u32 %r1877, 8; + sub.s32 %r1878, %r1877, %r3510; + sub.s32 %r1879, %r1878, %r3509; + min.u32 %r1880, %r1879, %r350; + setp.eq.s32 %p153, %r1880, 32; + mov.u32 %r1881, -1; + shl.b32 %r1882, %r1881, %r1880; + not.b32 %r1883, %r1882; + selp.b32 %r1884, -1, %r1883, %p153; + and.b32 %r1885, %r1884, %r3217; + shl.b32 %r1886, %r1885, %r3509; + cvt.u16.u32 %rs261, %r1886; + or.b16 %rs532, %rs27, %rs261; + add.s32 %r3509, %r1880, %r3509; + sub.s32 %r3216, %r350, %r1880; + shr.u32 %r3217, %r3217, %r1880; + setp.gt.u32 %p154, %r1879, %r350; + @%p154 bra $L__BB4_142; + + setp.ne.s32 %p155, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs262, %rs532, 255; + setp.ne.s16 %p156, %rs262, 127; + and.pred %p157, %p155, %p156; + @%p157 bra $L__BB4_142; + + cvt.u16.u32 %rs453, %r1886; + or.b16 %rs452, %rs27, %rs453; + mov.u32 %r1889, 20548; + sub.s32 %r1890, %r1889, %r3508; + cvt.u64.u32 %rd91, %r1890; + add.s64 %rd92, %rd91, %rd5; + add.s64 %rd93, %rd1, %rd92; + st.global.u8 [%rd93], %rs452; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p158, %rs262, 143; + selp.u32 %r3510, 1, 0, %p158; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_142: + setp.ne.s32 %p159, %r3216, 0; + mov.u32 %r3227, %r3085; + @%p159 bra $L__BB4_138; + +$L__BB4_143: + setp.ne.s32 %p160, %r342, 0; + @%p160 bra $L__BB4_191; + + setp.eq.s32 %p161, %r3204, 0; + add.s32 %r1891, %r3111, 17477; + cvt.u64.u32 %rd94, %r1891; + add.s64 %rd95, %rd94, %rd5; + add.s64 %rd10, %rd1, %rd95; + @%p161 bra $L__BB4_183; + + shl.b16 %rs474, %rs474, 1; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p162, %r3120, 0; + mov.u32 %r3263, %r3277; + @%p162 bra $L__BB4_148; + bra.uni $L__BB4_146; + +$L__BB4_148: + setp.lt.u32 %p164, %r3275, 3; + mov.u32 %r3231, 0; + @%p164 bra $L__BB4_151; + + setp.lt.u32 %p165, %r3275, 6; + mov.u32 %r3231, 1; + @%p165 bra $L__BB4_151; + + setp.lt.u32 %p166, %r3275, 9; + setp.eq.s32 %p167, %r3275, 11; + selp.b32 %r1897, 4, 5, %p167; + setp.lt.u32 %p168, %r3275, 11; + selp.b32 %r1898, 3, %r1897, %p168; + selp.b32 %r3231, 2, %r1898, %p166; + +$L__BB4_151: + setp.eq.s32 %p169, %r3231, 0; + @%p169 bra $L__BB4_179; + + and.b32 %r375, %r3231, 3; + setp.eq.s32 %p170, %r375, 0; + mov.u32 %r3241, %r3231; + mov.u32 %r3242, %r3263; + @%p170 bra $L__BB4_164; + + add.s32 %r3015, %r3231, -1; + mov.u32 %r1900, 1; + shl.b32 %r1901, %r1900, %r3015; + and.b32 %r1902, %r1901, %r3274; + setp.ne.s32 %p171, %r1902, 0; + selp.u32 %r1903, 1, 0, %p171; + cvt.u32.u16 %r1904, %rs474; + bfi.b32 %r1905, %r1904, %r1903, 1, 8; + cvt.u16.u32 %rs474, %r1905; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p172, %r3120, 0; + mov.u32 %r3242, %r3263; + @%p172 bra $L__BB4_156; + + setp.gt.u32 %p173, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3242, %r1900; + @%p173 bra $L__BB4_156; + + add.s32 %r1909, %r3111, 17477; + cvt.u64.u32 %rd96, %r1909; + add.s64 %rd97, %rd96, %rd5; + add.s64 %rd98, %rd1, %rd97; + st.global.u8 [%rd98], %rs474; + add.s32 %r3111, %r3111, 1; + mov.u16 %rs474, 0; + mov.u32 %r3120, 8; + mov.u32 %r3242, %r3263; + +$L__BB4_156: + and.b32 %r3017, %r3231, 3; + add.s32 %r3241, %r3231, -1; + setp.eq.s32 %p174, %r3017, 1; + mov.u32 %r3263, %r3242; + @%p174 bra $L__BB4_164; + + add.s32 %r3241, %r3231, -2; + mov.u32 %r1910, 1; + shl.b32 %r1911, %r1910, %r3241; + and.b32 %r1912, %r1911, %r3274; + setp.ne.s32 %p175, %r1912, 0; + selp.u32 %r1913, 1, 0, %p175; + cvt.u32.u16 %r1914, %rs474; + bfi.b32 %r1915, %r1914, %r1913, 1, 8; + cvt.u16.u32 %rs474, %r1915; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p176, %r3120, 0; + mov.u32 %r3237, %r3242; + @%p176 bra $L__BB4_160; + + setp.gt.u32 %p177, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3237, %r1910; + @%p177 bra $L__BB4_160; + + add.s32 %r1918, %r3111, 17477; + cvt.u64.u32 %rd99, %r1918; + add.s64 %rd100, %rd99, %rd5; + add.s64 %rd101, %rd1, %rd100; + and.b16 %rs269, %rs474, 255; + st.global.u8 [%rd101], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p178, %rs269, 255; + selp.b32 %r3120, 7, 8, %p178; + mov.u16 %rs474, 0; + mov.u32 %r3237, %r3242; + +$L__BB4_160: + and.b32 %r3018, %r3231, 3; + setp.eq.s32 %p179, %r3018, 2; + mov.u32 %r3263, %r3237; + mov.u32 %r3242, %r3237; + @%p179 bra $L__BB4_164; + + add.s32 %r3241, %r3231, -3; + mov.u32 %r1919, 1; + shl.b32 %r1920, %r1919, %r3241; + and.b32 %r1921, %r1920, %r3274; + setp.ne.s32 %p180, %r1921, 0; + selp.u32 %r1922, 1, 0, %p180; + cvt.u32.u16 %r1923, %rs474; + bfi.b32 %r1924, %r1923, %r1922, 1, 8; + cvt.u16.u32 %rs474, %r1924; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p181, %r3120, 0; + mov.u32 %r3263, %r3237; + mov.u32 %r3242, %r3237; + @%p181 bra $L__BB4_164; + + add.s32 %r3241, %r3231, -3; + setp.gt.u32 %p182, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3263, %r1919; + mov.u32 %r3242, %r1919; + @%p182 bra $L__BB4_164; + + add.s32 %r3241, %r3231, -3; + add.s32 %r1929, %r3111, 17477; + cvt.u64.u32 %rd102, %r1929; + add.s64 %rd103, %rd102, %rd5; + add.s64 %rd104, %rd1, %rd103; + and.b16 %rs272, %rs474, 255; + st.global.u8 [%rd104], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p183, %rs272, 255; + selp.b32 %r3120, 7, 8, %p183; + mov.u16 %rs474, 0; + mov.u32 %r3263, %r3237; + mov.u32 %r3242, %r3237; + +$L__BB4_164: + add.s32 %r3019, %r3231, -1; + setp.lt.u32 %p184, %r3019, 3; + @%p184 bra $L__BB4_179; + + mov.u32 %r3263, %r3242; + +$L__BB4_166: + add.s32 %r1930, %r3241, -1; + mov.u32 %r1931, 1; + shl.b32 %r1932, %r1931, %r1930; + and.b32 %r1933, %r1932, %r3274; + setp.ne.s32 %p185, %r1933, 0; + selp.u32 %r1934, 1, 0, %p185; + cvt.u32.u16 %r1935, %rs474; + bfi.b32 %r3251, %r1935, %r1934, 1, 8; + add.s32 %r3250, %r3120, -1; + setp.ne.s32 %p186, %r3250, 0; + mov.u32 %r3252, %r3263; + @%p186 bra $L__BB4_169; + + setp.gt.u32 %p187, %r3111, 191; + mov.u32 %r3250, 0; + mov.u32 %r3252, %r1931; + @%p187 bra $L__BB4_169; + + cvt.u16.u32 %rs273, %r3251; + and.b16 %rs274, %rs273, 255; + add.s32 %r1939, %r3111, 17477; + cvt.u64.u32 %rd105, %r1939; + add.s64 %rd106, %rd105, %rd5; + add.s64 %rd107, %rd1, %rd106; + st.global.u8 [%rd107], %rs273; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p188, %rs274, 255; + selp.b32 %r3250, 7, 8, %p188; + mov.u32 %r3251, 0; + mov.u32 %r3252, %r3263; + +$L__BB4_169: + add.s32 %r1940, %r3241, -2; + shl.b32 %r1942, %r1931, %r1940; + and.b32 %r1943, %r1942, %r3274; + setp.ne.s32 %p189, %r1943, 0; + and.b32 %r1944, %r3251, 127; + selp.u32 %r1945, 1, 0, %p189; + bfi.b32 %r3255, %r1944, %r1945, 1, 7; + add.s32 %r3254, %r3250, -1; + setp.ne.s32 %p190, %r3254, 0; + mov.u32 %r3256, %r3252; + @%p190 bra $L__BB4_172; + + setp.gt.u32 %p191, %r3111, 191; + mov.u32 %r3256, 1; + mov.u32 %r3254, 0; + @%p191 bra $L__BB4_172; + + cvt.u16.u32 %rs275, %r3255; + and.b16 %rs276, %rs275, 255; + add.s32 %r1949, %r3111, 17477; + cvt.u64.u32 %rd108, %r1949; + add.s64 %rd109, %rd108, %rd5; + add.s64 %rd110, %rd1, %rd109; + st.global.u8 [%rd110], %rs275; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p192, %rs276, 255; + selp.b32 %r3254, 7, 8, %p192; + mov.u32 %r3255, 0; + mov.u32 %r3256, %r3252; + +$L__BB4_172: + add.s32 %r1950, %r3241, -3; + mov.u32 %r1951, 1; + shl.b32 %r1952, %r1951, %r1950; + and.b32 %r1953, %r1952, %r3274; + setp.ne.s32 %p193, %r1953, 0; + and.b32 %r1954, %r3255, 127; + selp.u32 %r1955, 1, 0, %p193; + bfi.b32 %r3259, %r1954, %r1955, 1, 7; + add.s32 %r3258, %r3254, -1; + setp.ne.s32 %p194, %r3258, 0; + mov.u32 %r3260, %r3256; + @%p194 bra $L__BB4_175; + + setp.gt.u32 %p195, %r3111, 191; + mov.u32 %r3258, 0; + mov.u32 %r3260, %r1951; + @%p195 bra $L__BB4_175; + + cvt.u16.u32 %rs277, %r3259; + and.b16 %rs278, %rs277, 255; + add.s32 %r1959, %r3111, 17477; + cvt.u64.u32 %rd111, %r1959; + add.s64 %rd112, %rd111, %rd5; + add.s64 %rd113, %rd1, %rd112; + st.global.u8 [%rd113], %rs277; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p196, %rs278, 255; + selp.b32 %r3258, 7, 8, %p196; + mov.u32 %r3259, 0; + mov.u32 %r3260, %r3256; + +$L__BB4_175: + add.s32 %r3241, %r3241, -4; + shl.b32 %r1961, %r1951, %r3241; + and.b32 %r1962, %r1961, %r3274; + setp.ne.s32 %p197, %r1962, 0; + and.b32 %r1963, %r3259, 127; + selp.u32 %r1964, 1, 0, %p197; + bfi.b32 %r1965, %r1963, %r1964, 1, 15; + cvt.u16.u32 %rs474, %r1965; + add.s32 %r3120, %r3258, -1; + setp.ne.s32 %p198, %r3120, 0; + mov.u32 %r3263, %r3260; + @%p198 bra $L__BB4_178; + + setp.gt.u32 %p199, %r3111, 191; + mov.u32 %r3263, 1; + mov.u32 %r3120, 0; + @%p199 bra $L__BB4_178; + + add.s32 %r1968, %r3111, 17477; + cvt.u64.u32 %rd114, %r1968; + add.s64 %rd115, %rd114, %rd5; + add.s64 %rd116, %rd1, %rd115; + and.b16 %rs280, %rs474, 255; + st.global.u8 [%rd116], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p200, %rs280, 255; + selp.b32 %r3120, 7, 8, %p200; + mov.u16 %rs474, 0; + mov.u32 %r3263, %r3260; + +$L__BB4_178: + setp.ne.s32 %p201, %r3241, 0; + @%p201 bra $L__BB4_166; + +$L__BB4_179: + add.s32 %r1970, %r3275, -1; + setp.eq.s32 %p202, %r3275, 0; + mov.u32 %r3274, 0; + selp.b32 %r3275, 0, %r1970, %p202; + setp.lt.u32 %p203, %r3275, 3; + mov.u32 %r3267, %r3274; + @%p203 bra $L__BB4_182; + + setp.lt.u32 %p204, %r3275, 6; + mov.u32 %r3267, 1; + @%p204 bra $L__BB4_182; + + setp.lt.u32 %p205, %r3275, 9; + setp.eq.s32 %p206, %r3275, 11; + selp.b32 %r1972, 4, 5, %p206; + setp.lt.u32 %p207, %r3275, 11; + selp.b32 %r1973, 3, %r1972, %p207; + selp.b32 %r3267, 2, %r1973, %p205; + +$L__BB4_182: + mov.u32 %r1975, 1; + shl.b32 %r3276, %r1975, %r3267; + mov.u32 %r3277, %r3263; + bra.uni $L__BB4_191; + +$L__BB4_183: + add.s32 %r3274, %r3274, 1; + setp.lt.u32 %p208, %r3274, %r3276; + @%p208 bra $L__BB4_191; + + shl.b16 %rs281, %rs474, 1; + or.b16 %rs474, %rs281, 1; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p209, %r3120, 0; + mov.u32 %r3270, %r3277; + @%p209 bra $L__BB4_187; + bra.uni $L__BB4_185; + +$L__BB4_187: + add.s32 %r1979, %r3275, 1; + min.u32 %r3275, %r1979, 12; + setp.lt.u32 %p212, %r3275, 3; + mov.u32 %r3274, 0; + mov.u32 %r3271, %r3274; + @%p212 bra $L__BB4_190; + + setp.lt.u32 %p213, %r3275, 6; + mov.u32 %r3271, 1; + @%p213 bra $L__BB4_190; + + setp.lt.u32 %p214, %r3275, 9; + setp.eq.s32 %p215, %r3275, 11; + selp.b32 %r1981, 4, 5, %p215; + setp.lt.u32 %p216, %r3275, 11; + selp.b32 %r1982, 3, %r1981, %p216; + selp.b32 %r3271, 2, %r1982, %p214; + +$L__BB4_190: + mov.u32 %r1984, 1; + shl.b32 %r3276, %r1984, %r3271; + mov.u32 %r3277, %r3270; + +$L__BB4_191: + max.s32 %r458, %r3208, 1; + and.b16 %rs284, %rs25, 15; + cvt.u32.u16 %r459, %rs284; + and.b32 %r460, %r3204, 1; + setp.eq.s32 %p217, %r460, 0; + mov.u32 %r3292, %r3195; + @%p217 bra $L__BB4_198; + + and.b32 %r1985, %r459, 1; + sub.s32 %r3278, %r458, %r1985; + setp.eq.s32 %p218, %r3278, 0; + mov.u32 %r3292, %r3195; + @%p218 bra $L__BB4_198; + + mov.u32 %r1986, -1; + shl.b32 %r1987, %r1986, %r3278; + not.b32 %r1988, %r1987; + and.b32 %r3279, %r3198, %r1988; + +$L__BB4_194: + setp.gt.u32 %p219, %r3654, 17476; + mov.u32 %r3292, 1; + @%p219 bra $L__BB4_198; + + sub.s32 %r1990, %r3653, %r3655; + min.u32 %r1991, %r1990, %r3278; + setp.eq.s32 %p220, %r1991, 32; + mov.u32 %r1992, -1; + shl.b32 %r1993, %r1992, %r1991; + not.b32 %r1994, %r1993; + selp.b32 %r1995, -1, %r1994, %p220; + and.b32 %r1996, %r1995, %r3279; + shl.b32 %r1997, %r1996, %r3655; + or.b32 %r3656, %r1997, %r3656; + add.s32 %r3655, %r1991, %r3655; + shr.u32 %r3279, %r3279, %r1991; + sub.s32 %r3278, %r3278, %r1991; + setp.lt.u32 %p221, %r3655, %r3653; + @%p221 bra $L__BB4_197; + + cvt.u64.u32 %rd117, %r3654; + add.s64 %rd118, %rd117, %rd5; + add.s64 %rd119, %rd1, %rd118; + st.global.u8 [%rd119], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p222, %r3656, 255; + selp.b32 %r3653, 7, 8, %p222; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_197: + setp.ne.s32 %p223, %r3278, 0; + mov.u32 %r3292, %r3195; + @%p223 bra $L__BB4_194; + +$L__BB4_198: + and.b32 %r3013, %r3204, 2; + setp.eq.s32 %p224, %r3013, 0; + mov.u32 %r3307, %r3292; + @%p224 bra $L__BB4_205; + + shr.u32 %r2000, %r459, 1; + and.b32 %r2001, %r2000, 1; + sub.s32 %r3293, %r458, %r2001; + setp.eq.s32 %p225, %r3293, 0; + mov.u32 %r3307, %r3292; + @%p225 bra $L__BB4_205; + + mov.u32 %r2002, -1; + shl.b32 %r2003, %r2002, %r3293; + not.b32 %r2004, %r2003; + and.b32 %r3294, %r3202, %r2004; + +$L__BB4_201: + setp.gt.u32 %p226, %r3654, 17476; + mov.u32 %r3307, 1; + @%p226 bra $L__BB4_205; + + sub.s32 %r2006, %r3653, %r3655; + min.u32 %r2007, %r2006, %r3293; + setp.eq.s32 %p227, %r2007, 32; + mov.u32 %r2008, -1; + shl.b32 %r2009, %r2008, %r2007; + not.b32 %r2010, %r2009; + selp.b32 %r2011, -1, %r2010, %p227; + and.b32 %r2012, %r2011, %r3294; + shl.b32 %r2013, %r2012, %r3655; + or.b32 %r3656, %r2013, %r3656; + add.s32 %r3655, %r2007, %r3655; + shr.u32 %r3294, %r3294, %r2007; + sub.s32 %r3293, %r3293, %r2007; + setp.lt.u32 %p228, %r3655, %r3653; + @%p228 bra $L__BB4_204; + + cvt.u64.u32 %rd120, %r3654; + add.s64 %rd121, %rd120, %rd5; + add.s64 %rd122, %rd1, %rd121; + st.global.u8 [%rd122], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p229, %r3656, 255; + selp.b32 %r3653, 7, 8, %p229; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_204: + setp.ne.s32 %p230, %r3293, 0; + mov.u32 %r3307, %r3292; + @%p230 bra $L__BB4_201; + +$L__BB4_205: + and.b32 %r2016, %r3204, 4; + setp.eq.s32 %p231, %r2016, 0; + mov.u32 %r3322, %r3307; + @%p231 bra $L__BB4_212; + + shr.u32 %r2017, %r459, 2; + and.b32 %r2018, %r2017, 1; + sub.s32 %r3308, %r458, %r2018; + setp.eq.s32 %p232, %r3308, 0; + mov.u32 %r3322, %r3307; + @%p232 bra $L__BB4_212; + + mov.u32 %r2019, -1; + shl.b32 %r2020, %r2019, %r3308; + not.b32 %r2021, %r2020; + and.b32 %r3309, %r3207, %r2021; + +$L__BB4_208: + setp.gt.u32 %p233, %r3654, 17476; + mov.u32 %r3322, 1; + @%p233 bra $L__BB4_212; + + sub.s32 %r2023, %r3653, %r3655; + min.u32 %r2024, %r2023, %r3308; + setp.eq.s32 %p234, %r2024, 32; + mov.u32 %r2025, -1; + shl.b32 %r2026, %r2025, %r2024; + not.b32 %r2027, %r2026; + selp.b32 %r2028, -1, %r2027, %p234; + and.b32 %r2029, %r2028, %r3309; + shl.b32 %r2030, %r2029, %r3655; + or.b32 %r3656, %r2030, %r3656; + add.s32 %r3655, %r2024, %r3655; + shr.u32 %r3309, %r3309, %r2024; + sub.s32 %r3308, %r3308, %r2024; + setp.lt.u32 %p235, %r3655, %r3653; + @%p235 bra $L__BB4_211; + + cvt.u64.u32 %rd123, %r3654; + add.s64 %rd124, %rd123, %rd5; + add.s64 %rd125, %rd1, %rd124; + st.global.u8 [%rd125], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p236, %r3656, 255; + selp.b32 %r3653, 7, 8, %p236; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_211: + setp.ne.s32 %p237, %r3308, 0; + mov.u32 %r3322, %r3307; + @%p237 bra $L__BB4_208; + +$L__BB4_212: + and.b32 %r3014, %r3204, 8; + setp.eq.s32 %p238, %r3014, 0; + mov.u32 %r3829, %r3322; + @%p238 bra $L__BB4_219; + + shr.u32 %r2033, %r459, 3; + sub.s32 %r3323, %r458, %r2033; + setp.eq.s32 %p239, %r3323, 0; + mov.u32 %r3829, %r3322; + @%p239 bra $L__BB4_219; + + mov.u32 %r2034, -1; + shl.b32 %r2035, %r2034, %r3323; + not.b32 %r2036, %r2035; + and.b32 %r3324, %r3212, %r2036; + +$L__BB4_215: + setp.gt.u32 %p240, %r3654, 17476; + mov.u32 %r3829, 1; + @%p240 bra $L__BB4_219; + + sub.s32 %r2038, %r3653, %r3655; + min.u32 %r2039, %r2038, %r3323; + setp.eq.s32 %p241, %r2039, 32; + mov.u32 %r2040, -1; + shl.b32 %r2041, %r2040, %r2039; + not.b32 %r2042, %r2041; + selp.b32 %r2043, -1, %r2042, %p241; + and.b32 %r2044, %r2043, %r3324; + shl.b32 %r2045, %r2044, %r3655; + or.b32 %r3656, %r2045, %r3656; + add.s32 %r3655, %r2039, %r3655; + shr.u32 %r3324, %r3324, %r2039; + sub.s32 %r3323, %r3323, %r2039; + setp.lt.u32 %p242, %r3655, %r3653; + @%p242 bra $L__BB4_218; + + cvt.u64.u32 %rd126, %r3654; + add.s64 %rd127, %rd126, %rd5; + add.s64 %rd128, %rd1, %rd127; + st.global.u8 [%rd128], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p243, %r3656, 255; + selp.b32 %r3653, 7, 8, %p243; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_218: + setp.ne.s32 %p244, %r3323, 0; + mov.u32 %r3829, %r3322; + @%p244 bra $L__BB4_215; + +$L__BB4_219: + setp.lt.s32 %p245, %r343, 1; + setp.lt.s32 %p246, %r86, 1; + or.pred %p247, %p246, %p245; + @%p247 bra $L__BB4_267; + + min.s32 %r2048, %r86, %r343; + setp.lt.s32 %p248, %r2048, 3; + add.s32 %r2049, %r3111, 17477; + cvt.u64.u32 %rd129, %r2049; + add.s64 %rd130, %rd129, %rd5; + add.s64 %rd11, %rd1, %rd130; + @%p248 bra $L__BB4_259; + bra.uni $L__BB4_221; + +$L__BB4_259: + add.s32 %r3274, %r3274, 1; + setp.lt.u32 %p295, %r3274, %r3276; + @%p295 bra $L__BB4_267; + + shl.b16 %rs301, %rs474, 1; + or.b16 %rs474, %rs301, 1; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p296, %r3120, 0; + mov.u32 %r3380, %r3277; + @%p296 bra $L__BB4_263; + bra.uni $L__BB4_261; + +$L__BB4_263: + add.s32 %r2137, %r3275, 1; + min.u32 %r3275, %r2137, 12; + setp.lt.u32 %p299, %r3275, 3; + mov.u32 %r3274, 0; + mov.u32 %r3381, %r3274; + @%p299 bra $L__BB4_266; + + setp.lt.u32 %p300, %r3275, 6; + mov.u32 %r3381, 1; + @%p300 bra $L__BB4_266; + + setp.lt.u32 %p301, %r3275, 9; + setp.eq.s32 %p302, %r3275, 11; + selp.b32 %r2139, 4, 5, %p302; + setp.lt.u32 %p303, %r3275, 11; + selp.b32 %r2140, 3, %r2139, %p303; + selp.b32 %r3381, 2, %r2140, %p301; + +$L__BB4_266: + mov.u32 %r2142, 1; + shl.b32 %r3276, %r2142, %r3381; + mov.u32 %r3277, %r3380; + bra.uni $L__BB4_267; + +$L__BB4_221: + shl.b16 %rs474, %rs474, 1; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p249, %r3120, 0; + mov.u32 %r3373, %r3277; + @%p249 bra $L__BB4_224; + + setp.gt.u32 %p250, %r3111, 191; + mov.u32 %r3373, 1; + mov.u32 %r3120, 0; + @%p250 bra $L__BB4_224; + + st.global.u8 [%rd11], %rs474; + add.s32 %r3111, %r3111, 1; + mov.u16 %rs474, 0; + mov.u32 %r3120, 8; + mov.u32 %r3373, %r3277; + +$L__BB4_224: + setp.lt.u32 %p251, %r3275, 3; + mov.u32 %r3341, 0; + @%p251 bra $L__BB4_227; + + setp.lt.u32 %p252, %r3275, 6; + mov.u32 %r3341, 1; + @%p252 bra $L__BB4_227; + + setp.lt.u32 %p253, %r3275, 9; + setp.eq.s32 %p254, %r3275, 11; + selp.b32 %r2055, 4, 5, %p254; + setp.lt.u32 %p255, %r3275, 11; + selp.b32 %r2056, 3, %r2055, %p255; + selp.b32 %r3341, 2, %r2056, %p253; + +$L__BB4_227: + setp.eq.s32 %p256, %r3341, 0; + @%p256 bra $L__BB4_255; + + and.b32 %r561, %r3341, 3; + setp.eq.s32 %p257, %r561, 0; + mov.u32 %r3351, %r3341; + mov.u32 %r3352, %r3373; + @%p257 bra $L__BB4_240; + + add.s32 %r3026, %r3341, -1; + mov.u32 %r2058, 1; + shl.b32 %r2059, %r2058, %r3026; + and.b32 %r2060, %r2059, %r3274; + setp.ne.s32 %p258, %r2060, 0; + selp.u32 %r2061, 1, 0, %p258; + cvt.u32.u16 %r2062, %rs474; + bfi.b32 %r2063, %r2062, %r2061, 1, 8; + cvt.u16.u32 %rs474, %r2063; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p259, %r3120, 0; + mov.u32 %r3352, %r3373; + @%p259 bra $L__BB4_232; + + setp.gt.u32 %p260, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3352, %r2058; + @%p260 bra $L__BB4_232; + + add.s32 %r2067, %r3111, 17477; + cvt.u64.u32 %rd131, %r2067; + add.s64 %rd132, %rd131, %rd5; + add.s64 %rd133, %rd1, %rd132; + st.global.u8 [%rd133], %rs474; + add.s32 %r3111, %r3111, 1; + mov.u16 %rs474, 0; + mov.u32 %r3120, 8; + mov.u32 %r3352, %r3373; + +$L__BB4_232: + and.b32 %r3028, %r3341, 3; + add.s32 %r3351, %r3341, -1; + setp.eq.s32 %p261, %r3028, 1; + mov.u32 %r3373, %r3352; + @%p261 bra $L__BB4_240; + + add.s32 %r3351, %r3341, -2; + mov.u32 %r2068, 1; + shl.b32 %r2069, %r2068, %r3351; + and.b32 %r2070, %r2069, %r3274; + setp.ne.s32 %p262, %r2070, 0; + selp.u32 %r2071, 1, 0, %p262; + cvt.u32.u16 %r2072, %rs474; + bfi.b32 %r2073, %r2072, %r2071, 1, 8; + cvt.u16.u32 %rs474, %r2073; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p263, %r3120, 0; + mov.u32 %r3347, %r3352; + @%p263 bra $L__BB4_236; + + setp.gt.u32 %p264, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3347, %r2068; + @%p264 bra $L__BB4_236; + + add.s32 %r2076, %r3111, 17477; + cvt.u64.u32 %rd134, %r2076; + add.s64 %rd135, %rd134, %rd5; + add.s64 %rd136, %rd1, %rd135; + and.b16 %rs289, %rs474, 255; + st.global.u8 [%rd136], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p265, %rs289, 255; + selp.b32 %r3120, 7, 8, %p265; + mov.u16 %rs474, 0; + mov.u32 %r3347, %r3352; + +$L__BB4_236: + and.b32 %r3029, %r3341, 3; + setp.eq.s32 %p266, %r3029, 2; + mov.u32 %r3373, %r3347; + mov.u32 %r3352, %r3347; + @%p266 bra $L__BB4_240; + + add.s32 %r3351, %r3341, -3; + mov.u32 %r2077, 1; + shl.b32 %r2078, %r2077, %r3351; + and.b32 %r2079, %r2078, %r3274; + setp.ne.s32 %p267, %r2079, 0; + selp.u32 %r2080, 1, 0, %p267; + cvt.u32.u16 %r2081, %rs474; + bfi.b32 %r2082, %r2081, %r2080, 1, 8; + cvt.u16.u32 %rs474, %r2082; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p268, %r3120, 0; + mov.u32 %r3373, %r3347; + mov.u32 %r3352, %r3347; + @%p268 bra $L__BB4_240; + + add.s32 %r3351, %r3341, -3; + setp.gt.u32 %p269, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3373, %r2077; + mov.u32 %r3352, %r2077; + @%p269 bra $L__BB4_240; + + add.s32 %r3351, %r3341, -3; + add.s32 %r2087, %r3111, 17477; + cvt.u64.u32 %rd137, %r2087; + add.s64 %rd138, %rd137, %rd5; + add.s64 %rd139, %rd1, %rd138; + and.b16 %rs292, %rs474, 255; + st.global.u8 [%rd139], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p270, %rs292, 255; + selp.b32 %r3120, 7, 8, %p270; + mov.u16 %rs474, 0; + mov.u32 %r3373, %r3347; + mov.u32 %r3352, %r3347; + +$L__BB4_240: + add.s32 %r3030, %r3341, -1; + setp.lt.u32 %p271, %r3030, 3; + @%p271 bra $L__BB4_255; + + mov.u32 %r3373, %r3352; + +$L__BB4_242: + add.s32 %r2088, %r3351, -1; + mov.u32 %r2089, 1; + shl.b32 %r2090, %r2089, %r2088; + and.b32 %r2091, %r2090, %r3274; + setp.ne.s32 %p272, %r2091, 0; + selp.u32 %r2092, 1, 0, %p272; + cvt.u32.u16 %r2093, %rs474; + bfi.b32 %r3361, %r2093, %r2092, 1, 8; + add.s32 %r3360, %r3120, -1; + setp.ne.s32 %p273, %r3360, 0; + mov.u32 %r3362, %r3373; + @%p273 bra $L__BB4_245; + + setp.gt.u32 %p274, %r3111, 191; + mov.u32 %r3360, 0; + mov.u32 %r3362, %r2089; + @%p274 bra $L__BB4_245; + + cvt.u16.u32 %rs293, %r3361; + and.b16 %rs294, %rs293, 255; + add.s32 %r2097, %r3111, 17477; + cvt.u64.u32 %rd140, %r2097; + add.s64 %rd141, %rd140, %rd5; + add.s64 %rd142, %rd1, %rd141; + st.global.u8 [%rd142], %rs293; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p275, %rs294, 255; + selp.b32 %r3360, 7, 8, %p275; + mov.u32 %r3361, 0; + mov.u32 %r3362, %r3373; + +$L__BB4_245: + add.s32 %r2098, %r3351, -2; + shl.b32 %r2100, %r2089, %r2098; + and.b32 %r2101, %r2100, %r3274; + setp.ne.s32 %p276, %r2101, 0; + and.b32 %r2102, %r3361, 127; + selp.u32 %r2103, 1, 0, %p276; + bfi.b32 %r3365, %r2102, %r2103, 1, 7; + add.s32 %r3364, %r3360, -1; + setp.ne.s32 %p277, %r3364, 0; + mov.u32 %r3366, %r3362; + @%p277 bra $L__BB4_248; + + setp.gt.u32 %p278, %r3111, 191; + mov.u32 %r3366, 1; + mov.u32 %r3364, 0; + @%p278 bra $L__BB4_248; + + cvt.u16.u32 %rs295, %r3365; + and.b16 %rs296, %rs295, 255; + add.s32 %r2107, %r3111, 17477; + cvt.u64.u32 %rd143, %r2107; + add.s64 %rd144, %rd143, %rd5; + add.s64 %rd145, %rd1, %rd144; + st.global.u8 [%rd145], %rs295; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p279, %rs296, 255; + selp.b32 %r3364, 7, 8, %p279; + mov.u32 %r3365, 0; + mov.u32 %r3366, %r3362; + +$L__BB4_248: + add.s32 %r2108, %r3351, -3; + mov.u32 %r2109, 1; + shl.b32 %r2110, %r2109, %r2108; + and.b32 %r2111, %r2110, %r3274; + setp.ne.s32 %p280, %r2111, 0; + and.b32 %r2112, %r3365, 127; + selp.u32 %r2113, 1, 0, %p280; + bfi.b32 %r3369, %r2112, %r2113, 1, 7; + add.s32 %r3368, %r3364, -1; + setp.ne.s32 %p281, %r3368, 0; + mov.u32 %r3370, %r3366; + @%p281 bra $L__BB4_251; + + setp.gt.u32 %p282, %r3111, 191; + mov.u32 %r3368, 0; + mov.u32 %r3370, %r2109; + @%p282 bra $L__BB4_251; + + cvt.u16.u32 %rs297, %r3369; + and.b16 %rs298, %rs297, 255; + add.s32 %r2117, %r3111, 17477; + cvt.u64.u32 %rd146, %r2117; + add.s64 %rd147, %rd146, %rd5; + add.s64 %rd148, %rd1, %rd147; + st.global.u8 [%rd148], %rs297; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p283, %rs298, 255; + selp.b32 %r3368, 7, 8, %p283; + mov.u32 %r3369, 0; + mov.u32 %r3370, %r3366; + +$L__BB4_251: + add.s32 %r3351, %r3351, -4; + shl.b32 %r2119, %r2109, %r3351; + and.b32 %r2120, %r2119, %r3274; + setp.ne.s32 %p284, %r2120, 0; + and.b32 %r2121, %r3369, 127; + selp.u32 %r2122, 1, 0, %p284; + bfi.b32 %r2123, %r2121, %r2122, 1, 15; + cvt.u16.u32 %rs474, %r2123; + add.s32 %r3120, %r3368, -1; + setp.ne.s32 %p285, %r3120, 0; + mov.u32 %r3373, %r3370; + @%p285 bra $L__BB4_254; + + setp.gt.u32 %p286, %r3111, 191; + mov.u32 %r3373, 1; + mov.u32 %r3120, 0; + @%p286 bra $L__BB4_254; + + add.s32 %r2126, %r3111, 17477; + cvt.u64.u32 %rd149, %r2126; + add.s64 %rd150, %rd149, %rd5; + add.s64 %rd151, %rd1, %rd150; + and.b16 %rs300, %rs474, 255; + st.global.u8 [%rd151], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p287, %rs300, 255; + selp.b32 %r3120, 7, 8, %p287; + mov.u16 %rs474, 0; + mov.u32 %r3373, %r3370; + +$L__BB4_254: + setp.ne.s32 %p288, %r3351, 0; + @%p288 bra $L__BB4_242; + +$L__BB4_255: + add.s32 %r2128, %r3275, -1; + setp.eq.s32 %p289, %r3275, 0; + mov.u32 %r3274, 0; + selp.b32 %r3275, 0, %r2128, %p289; + setp.lt.u32 %p290, %r3275, 3; + mov.u32 %r3377, %r3274; + @%p290 bra $L__BB4_258; + + setp.lt.u32 %p291, %r3275, 6; + mov.u32 %r3377, 1; + @%p291 bra $L__BB4_258; + + setp.lt.u32 %p292, %r3275, 9; + setp.eq.s32 %p293, %r3275, 11; + selp.b32 %r2130, 4, 5, %p293; + setp.lt.u32 %p294, %r3275, 11; + selp.b32 %r2131, 3, %r2130, %p294; + selp.b32 %r3377, 2, %r2131, %p292; + +$L__BB4_258: + mov.u32 %r2133, 1; + shl.b32 %r3276, %r2133, %r3377; + mov.u32 %r3277, %r3373; + +$L__BB4_267: + setp.gt.s32 %p304, %r343, 2; + setp.gt.s32 %p305, %r86, 2; + and.pred %p306, %p305, %p304; + @%p306 bra $L__BB4_317; + bra.uni $L__BB4_268; + +$L__BB4_317: + mul.lo.s32 %r2263, %r86, 6; + add.s32 %r2264, %r2263, -11; + cvt.u64.u32 %rd185, %r2264; + cvta.to.global.u64 %rd186, %rd31; + add.s64 %rd14, %rd186, %rd185; + ld.global.u8 %rs99, [%rd14]; + add.s32 %r2265, %r2263, -10; + cvt.u64.u32 %rd187, %r2265; + add.s64 %rd188, %rd186, %rd187; + ld.global.u8 %rs100, [%rd188]; + ld.global.u8 %rs101, [%rd188+1]; + mul.lo.s32 %r2266, %r343, 6; + add.s32 %r2267, %r2266, -12; + cvt.u64.u32 %rd189, %r2267; + add.s64 %rd190, %rd186, %rd189; + ld.global.u8 %rs102, [%rd190]; + ld.global.u8 %rs103, [%rd190+1]; + add.s32 %r2268, %r2266, -10; + cvt.u64.u32 %rd191, %r2268; + add.s64 %rd192, %rd186, %rd191; + ld.global.u8 %rs104, [%rd192]; + ld.global.u8 %rs105, [%rd192+1]; + setp.eq.s16 %p374, %rs99, 0; + mov.u32 %r3475, %r3227; + @%p374 bra $L__BB4_324; + + ld.global.u8 %r3465, [%rd14+-1]; + cvt.u32.u16 %r3464, %rs99; + +$L__BB4_319: + mov.u16 %rs106, %rs532; + mov.u32 %r771, %r3464; + setp.gt.u32 %p375, %r3508, 2879; + mov.u32 %r3475, 1; + @%p375 bra $L__BB4_324; + + mov.u32 %r2270, 8; + sub.s32 %r2271, %r2270, %r3510; + sub.s32 %r2272, %r2271, %r3509; + min.u32 %r2273, %r2272, %r771; + setp.eq.s32 %p376, %r2273, 32; + mov.u32 %r2274, -1; + shl.b32 %r2275, %r2274, %r2273; + not.b32 %r2276, %r2275; + selp.b32 %r2277, -1, %r2276, %p376; + and.b32 %r2278, %r2277, %r3465; + shl.b32 %r2279, %r2278, %r3509; + cvt.u16.u32 %rs336, %r2279; + or.b16 %rs532, %rs106, %rs336; + add.s32 %r3509, %r2273, %r3509; + sub.s32 %r3464, %r771, %r2273; + shr.u32 %r3465, %r3465, %r2273; + setp.gt.u32 %p377, %r2272, %r771; + @%p377 bra $L__BB4_323; + + setp.ne.s32 %p378, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs337, %rs532, 255; + setp.ne.s16 %p379, %rs337, 127; + and.pred %p380, %p378, %p379; + @%p380 bra $L__BB4_323; + + cvt.u16.u32 %rs461, %r2279; + or.b16 %rs460, %rs106, %rs461; + mov.u32 %r2282, 20548; + sub.s32 %r2283, %r2282, %r3508; + cvt.u64.u32 %rd193, %r2283; + add.s64 %rd194, %rd193, %rd5; + add.s64 %rd195, %rd1, %rd194; + st.global.u8 [%rd195], %rs460; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p381, %rs337, 143; + selp.u32 %r3510, 1, 0, %p381; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_323: + setp.ne.s32 %p382, %r3464, 0; + mov.u32 %r3475, %r3227; + @%p382 bra $L__BB4_319; + +$L__BB4_324: + setp.eq.s16 %p383, %rs103, 0; + mov.u32 %r3487, %r3475; + @%p383 bra $L__BB4_331; + + cvt.u32.u16 %r2284, %rs102; + and.b32 %r3477, %r2284, 255; + cvt.u32.u16 %r2285, %rs103; + and.b32 %r3476, %r2285, 255; + +$L__BB4_326: + mov.u16 %rs110, %rs532; + mov.u32 %r790, %r3476; + setp.gt.u32 %p384, %r3508, 2879; + mov.u32 %r3487, 1; + @%p384 bra $L__BB4_331; + + mov.u32 %r2287, 8; + sub.s32 %r2288, %r2287, %r3510; + sub.s32 %r2289, %r2288, %r3509; + min.u32 %r2290, %r2289, %r790; + setp.eq.s32 %p385, %r2290, 32; + mov.u32 %r2291, -1; + shl.b32 %r2292, %r2291, %r2290; + not.b32 %r2293, %r2292; + selp.b32 %r2294, -1, %r2293, %p385; + and.b32 %r2295, %r2294, %r3477; + shl.b32 %r2296, %r2295, %r3509; + cvt.u16.u32 %rs341, %r2296; + or.b16 %rs532, %rs110, %rs341; + add.s32 %r3509, %r2290, %r3509; + sub.s32 %r3476, %r790, %r2290; + shr.u32 %r3477, %r3477, %r2290; + setp.gt.u32 %p386, %r2289, %r790; + @%p386 bra $L__BB4_330; + + setp.ne.s32 %p387, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs342, %rs532, 255; + setp.ne.s16 %p388, %rs342, 127; + and.pred %p389, %p387, %p388; + @%p389 bra $L__BB4_330; + + cvt.u16.u32 %rs463, %r2296; + or.b16 %rs462, %rs110, %rs463; + mov.u32 %r2299, 20548; + sub.s32 %r2300, %r2299, %r3508; + cvt.u64.u32 %rd196, %r2300; + add.s64 %rd197, %rd196, %rd5; + add.s64 %rd198, %rd1, %rd197; + st.global.u8 [%rd198], %rs462; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p390, %rs342, 143; + selp.u32 %r3510, 1, 0, %p390; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_330: + setp.ne.s32 %p391, %r3476, 0; + mov.u32 %r3487, %r3475; + @%p391 bra $L__BB4_326; + +$L__BB4_331: + setp.eq.s16 %p392, %rs101, 0; + mov.u32 %r3499, %r3487; + @%p392 bra $L__BB4_338; + + cvt.u32.u16 %r2301, %rs101; + and.b32 %r3488, %r2301, 255; + cvt.u32.u16 %r2302, %rs100; + and.b32 %r3489, %r2302, 255; + +$L__BB4_333: + mov.u32 %r809, %r3488; + setp.gt.u32 %p393, %r3508, 2879; + mov.u32 %r3499, 1; + @%p393 bra $L__BB4_338; + + mov.u32 %r2304, 8; + sub.s32 %r2305, %r2304, %r3510; + sub.s32 %r2306, %r2305, %r3509; + min.u32 %r2307, %r2306, %r809; + setp.eq.s32 %p394, %r2307, 32; + mov.u32 %r2308, -1; + shl.b32 %r2309, %r2308, %r2307; + not.b32 %r2310, %r2309; + selp.b32 %r2311, -1, %r2310, %p394; + and.b32 %r2312, %r2311, %r3489; + shl.b32 %r2313, %r2312, %r3509; + cvt.u16.u32 %rs346, %r2313; + or.b16 %rs532, %rs532, %rs346; + add.s32 %r3509, %r2307, %r3509; + sub.s32 %r3488, %r809, %r2307; + shr.u32 %r3489, %r3489, %r2307; + setp.gt.u32 %p395, %r2306, %r809; + @%p395 bra $L__BB4_337; + + setp.ne.s32 %p396, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs347, %rs532, 255; + setp.ne.s16 %p397, %rs347, 127; + and.pred %p398, %p396, %p397; + @%p398 bra $L__BB4_337; + + mov.u32 %r2316, 20548; + sub.s32 %r2317, %r2316, %r3508; + cvt.u64.u32 %rd199, %r2317; + add.s64 %rd200, %rd199, %rd5; + add.s64 %rd201, %rd1, %rd200; + st.global.u8 [%rd201], %rs532; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p399, %rs347, 143; + selp.u32 %r3510, 1, 0, %p399; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_337: + setp.ne.s32 %p400, %r3488, 0; + mov.u32 %r3499, %r3487; + @%p400 bra $L__BB4_333; + +$L__BB4_338: + setp.eq.s16 %p401, %rs105, 0; + mov.u32 %r3511, %r3499; + @%p401 bra $L__BB4_345; + + cvt.u32.u16 %r2318, %rs104; + and.b32 %r3501, %r2318, 255; + cvt.u32.u16 %r2319, %rs105; + and.b32 %r3500, %r2319, 255; + +$L__BB4_340: + mov.u32 %r828, %r3500; + setp.gt.u32 %p402, %r3508, 2879; + mov.u32 %r3511, 1; + @%p402 bra $L__BB4_345; + + mov.u32 %r2321, 8; + sub.s32 %r2322, %r2321, %r3510; + sub.s32 %r2323, %r2322, %r3509; + min.u32 %r2324, %r2323, %r828; + setp.eq.s32 %p403, %r2324, 32; + mov.u32 %r2325, -1; + shl.b32 %r2326, %r2325, %r2324; + not.b32 %r2327, %r2326; + selp.b32 %r2328, -1, %r2327, %p403; + and.b32 %r2329, %r2328, %r3501; + shl.b32 %r2330, %r2329, %r3509; + cvt.u16.u32 %rs351, %r2330; + or.b16 %rs532, %rs532, %rs351; + add.s32 %r3509, %r2324, %r3509; + sub.s32 %r3500, %r828, %r2324; + shr.u32 %r3501, %r3501, %r2324; + setp.gt.u32 %p404, %r2323, %r828; + @%p404 bra $L__BB4_344; + + setp.ne.s32 %p405, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs352, %rs532, 255; + setp.ne.s16 %p406, %rs352, 127; + and.pred %p407, %p405, %p406; + @%p407 bra $L__BB4_344; + + mov.u32 %r2333, 20548; + sub.s32 %r2334, %r2333, %r3508; + cvt.u64.u32 %rd202, %r2334; + add.s64 %rd203, %rd202, %rd5; + add.s64 %rd204, %rd1, %rd203; + st.global.u8 [%rd204], %rs532; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p408, %rs352, 143; + selp.u32 %r3510, 1, 0, %p408; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_344: + setp.ne.s32 %p409, %r3500, 0; + mov.u32 %r3511, %r3499; + @%p409 bra $L__BB4_340; + bra.uni $L__BB4_345; + +$L__BB4_268: + setp.gt.s32 %p307, %r343, 0; + and.pred %p309, %p305, %p307; + mul.lo.s32 %r644, %r86, 6; + @%p309 bra $L__BB4_297; + bra.uni $L__BB4_269; + +$L__BB4_297: + cvt.u64.u32 %rd172, %r644; + cvta.to.global.u64 %rd173, %rd31; + add.s64 %rd13, %rd173, %rd172; + ld.global.u8 %rs85, [%rd13+1]; + add.s32 %r2214, %r644, 2; + cvt.u64.u32 %rd174, %r2214; + add.s64 %rd175, %rd173, %rd174; + ld.global.u8 %rs86, [%rd175]; + ld.global.u8 %rs87, [%rd175+1]; + setp.eq.s16 %p348, %rs85, 0; + mov.u32 %r3443, %r3227; + @%p348 bra $L__BB4_304; + + ld.global.u8 %r3433, [%rd13]; + cvt.u32.u16 %r3432, %rs85; + +$L__BB4_299: + mov.u16 %rs88, %rs532; + mov.u32 %r719, %r3432; + setp.gt.u32 %p349, %r3508, 2879; + mov.u32 %r3443, 1; + @%p349 bra $L__BB4_304; + + mov.u32 %r2216, 8; + sub.s32 %r2217, %r2216, %r3510; + sub.s32 %r2218, %r2217, %r3509; + min.u32 %r2219, %r2218, %r719; + setp.eq.s32 %p350, %r2219, 32; + mov.u32 %r2220, -1; + shl.b32 %r2221, %r2220, %r2219; + not.b32 %r2222, %r2221; + selp.b32 %r2223, -1, %r2222, %p350; + and.b32 %r2224, %r2223, %r3433; + shl.b32 %r2225, %r2224, %r3509; + cvt.u16.u32 %rs323, %r2225; + or.b16 %rs532, %rs88, %rs323; + add.s32 %r3509, %r2219, %r3509; + sub.s32 %r3432, %r719, %r2219; + shr.u32 %r3433, %r3433, %r2219; + setp.gt.u32 %p351, %r2218, %r719; + @%p351 bra $L__BB4_303; + + setp.ne.s32 %p352, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs324, %rs532, 255; + setp.ne.s16 %p353, %rs324, 127; + and.pred %p354, %p352, %p353; + @%p354 bra $L__BB4_303; + + cvt.u16.u32 %rs459, %r2225; + or.b16 %rs458, %rs88, %rs459; + mov.u32 %r2228, 20548; + sub.s32 %r2229, %r2228, %r3508; + cvt.u64.u32 %rd176, %r2229; + add.s64 %rd177, %rd176, %rd5; + add.s64 %rd178, %rd1, %rd177; + st.global.u8 [%rd178], %rs458; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p355, %rs324, 143; + selp.u32 %r3510, 1, 0, %p355; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_303: + setp.ne.s32 %p356, %r3432, 0; + mov.u32 %r3443, %r3227; + @%p356 bra $L__BB4_299; + +$L__BB4_304: + add.s32 %r3445, %r343, -1; + cvt.u32.u16 %r2232, %rs86; + and.b32 %r3457, %r2232, 255; + mov.u32 %r3444, 1; + +$L__BB4_305: + mov.u32 %r739, %r3444; + mov.u32 %r3455, 1; + setp.gt.u32 %p357, %r3508, 2879; + @%p357 bra $L__BB4_310; + + mov.u32 %r2234, 8; + sub.s32 %r2235, %r2234, %r3510; + sub.s32 %r2236, %r2235, %r3509; + min.u32 %r2237, %r2236, %r739; + setp.eq.s32 %p358, %r2237, 32; + mov.u32 %r2238, -1; + shl.b32 %r2239, %r2238, %r2237; + not.b32 %r2240, %r2239; + selp.b32 %r2241, -1, %r2240, %p358; + and.b32 %r2242, %r2241, %r3445; + shl.b32 %r2243, %r2242, %r3509; + cvt.u16.u32 %rs327, %r2243; + or.b16 %rs532, %rs532, %rs327; + add.s32 %r3509, %r2237, %r3509; + sub.s32 %r3444, %r739, %r2237; + shr.u32 %r3445, %r3445, %r2237; + setp.gt.u32 %p359, %r2236, %r739; + @%p359 bra $L__BB4_309; + + setp.ne.s32 %p360, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs328, %rs532, 255; + setp.ne.s16 %p361, %rs328, 127; + and.pred %p362, %p360, %p361; + @%p362 bra $L__BB4_309; + + mov.u32 %r2246, 20548; + sub.s32 %r2247, %r2246, %r3508; + cvt.u64.u32 %rd179, %r2247; + add.s64 %rd180, %rd179, %rd5; + add.s64 %rd181, %rd1, %rd180; + st.global.u8 [%rd181], %rs532; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p363, %rs328, 143; + selp.u32 %r3510, 1, 0, %p363; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_309: + setp.ne.s32 %p364, %r3444, 0; + mov.u32 %r3455, %r3443; + @%p364 bra $L__BB4_305; + +$L__BB4_310: + setp.eq.s16 %p365, %rs87, 0; + mov.u32 %r3511, %r3455; + @%p365 bra $L__BB4_345; + + cvt.u32.u16 %r3025, %rs87; + and.b32 %r3456, %r3025, 255; + +$L__BB4_312: + mov.u32 %r756, %r3456; + setp.gt.u32 %p366, %r3508, 2879; + mov.u32 %r3511, 1; + @%p366 bra $L__BB4_345; + + mov.u32 %r2249, 8; + sub.s32 %r2250, %r2249, %r3510; + sub.s32 %r2251, %r2250, %r3509; + min.u32 %r2252, %r2251, %r756; + setp.eq.s32 %p367, %r2252, 32; + mov.u32 %r2253, -1; + shl.b32 %r2254, %r2253, %r2252; + not.b32 %r2255, %r2254; + selp.b32 %r2256, -1, %r2255, %p367; + and.b32 %r2257, %r2256, %r3457; + shl.b32 %r2258, %r2257, %r3509; + cvt.u16.u32 %rs332, %r2258; + or.b16 %rs532, %rs532, %rs332; + add.s32 %r3509, %r2252, %r3509; + sub.s32 %r3456, %r756, %r2252; + shr.u32 %r3457, %r3457, %r2252; + setp.gt.u32 %p368, %r2251, %r756; + @%p368 bra $L__BB4_316; + + setp.ne.s32 %p369, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs333, %rs532, 255; + setp.ne.s16 %p370, %rs333, 127; + and.pred %p371, %p369, %p370; + @%p371 bra $L__BB4_316; + + mov.u32 %r2261, 20548; + sub.s32 %r2262, %r2261, %r3508; + cvt.u64.u32 %rd182, %r2262; + add.s64 %rd183, %rd182, %rd5; + add.s64 %rd184, %rd1, %rd183; + st.global.u8 [%rd184], %rs532; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p372, %rs333, 143; + selp.u32 %r3510, 1, 0, %p372; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_316: + setp.eq.s32 %p373, %r3456, 0; + mov.u32 %r3511, %r3455; + @%p373 bra $L__BB4_345; + bra.uni $L__BB4_312; + +$L__BB4_269: + setp.gt.s32 %p311, %r86, 0; + selp.b32 %r2143, %r644, 0, %p311; + cvt.u64.u32 %rd152, %r2143; + cvta.to.global.u64 %rd153, %rd31; + add.s64 %rd12, %rd153, %rd152; + ld.global.u8 %rs63, [%rd12+1]; + add.s32 %r2144, %r2143, 2; + cvt.u64.u32 %rd154, %r2144; + add.s64 %rd155, %rd153, %rd154; + ld.global.u8 %rs64, [%rd155]; + ld.global.u8 %rs65, [%rd155+1]; + mul.lo.s32 %r2145, %r343, 6; + selp.b32 %r2146, %r2145, 0, %p307; + cvt.u64.u32 %rd156, %r2146; + add.s64 %rd157, %rd153, %rd156; + ld.global.u8 %rs66, [%rd157]; + ld.global.u8 %rs67, [%rd157+1]; + add.s32 %r2147, %r2146, 2; + cvt.u64.u32 %rd158, %r2147; + add.s64 %rd159, %rd153, %rd158; + ld.global.u8 %rs68, [%rd159]; + ld.global.u8 %rs69, [%rd159+1]; + setp.eq.s16 %p312, %rs63, 0; + mov.u32 %r3399, %r3227; + @%p312 bra $L__BB4_276; + + ld.global.u8 %r3389, [%rd12]; + cvt.u32.u16 %r3388, %rs63; + +$L__BB4_271: + mov.u16 %rs70, %rs532; + mov.u32 %r647, %r3388; + setp.gt.u32 %p313, %r3508, 2879; + mov.u32 %r3399, 1; + @%p313 bra $L__BB4_276; + + mov.u32 %r2149, 8; + sub.s32 %r2150, %r2149, %r3510; + sub.s32 %r2151, %r2150, %r3509; + min.u32 %r2152, %r2151, %r647; + setp.eq.s32 %p314, %r2152, 32; + mov.u32 %r2153, -1; + shl.b32 %r2154, %r2153, %r2152; + not.b32 %r2155, %r2154; + selp.b32 %r2156, -1, %r2155, %p314; + and.b32 %r2157, %r2156, %r3389; + shl.b32 %r2158, %r2157, %r3509; + cvt.u16.u32 %rs304, %r2158; + or.b16 %rs532, %rs70, %rs304; + add.s32 %r3509, %r2152, %r3509; + sub.s32 %r3388, %r647, %r2152; + shr.u32 %r3389, %r3389, %r2152; + setp.gt.u32 %p315, %r2151, %r647; + @%p315 bra $L__BB4_275; + + setp.ne.s32 %p316, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs305, %rs532, 255; + setp.ne.s16 %p317, %rs305, 127; + and.pred %p318, %p316, %p317; + @%p318 bra $L__BB4_275; + + cvt.u16.u32 %rs455, %r2158; + or.b16 %rs454, %rs70, %rs455; + mov.u32 %r2161, 20548; + sub.s32 %r2162, %r2161, %r3508; + cvt.u64.u32 %rd160, %r2162; + add.s64 %rd161, %rd160, %rd5; + add.s64 %rd162, %rd1, %rd161; + st.global.u8 [%rd162], %rs454; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p319, %rs305, 143; + selp.u32 %r3510, 1, 0, %p319; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_275: + setp.ne.s32 %p320, %r3388, 0; + mov.u32 %r3399, %r3227; + @%p320 bra $L__BB4_271; + +$L__BB4_276: + setp.eq.s16 %p321, %rs67, 0; + mov.u32 %r3411, %r3399; + @%p321 bra $L__BB4_283; + + cvt.u32.u16 %r2163, %rs66; + and.b32 %r3401, %r2163, 255; + cvt.u32.u16 %r2164, %rs67; + and.b32 %r3400, %r2164, 255; + +$L__BB4_278: + mov.u16 %rs74, %rs532; + mov.u32 %r666, %r3400; + setp.gt.u32 %p322, %r3508, 2879; + mov.u32 %r3411, 1; + @%p322 bra $L__BB4_283; + + mov.u32 %r2166, 8; + sub.s32 %r2167, %r2166, %r3510; + sub.s32 %r2168, %r2167, %r3509; + min.u32 %r2169, %r2168, %r666; + setp.eq.s32 %p323, %r2169, 32; + mov.u32 %r2170, -1; + shl.b32 %r2171, %r2170, %r2169; + not.b32 %r2172, %r2171; + selp.b32 %r2173, -1, %r2172, %p323; + and.b32 %r2174, %r2173, %r3401; + shl.b32 %r2175, %r2174, %r3509; + cvt.u16.u32 %rs309, %r2175; + or.b16 %rs532, %rs74, %rs309; + add.s32 %r3509, %r2169, %r3509; + sub.s32 %r3400, %r666, %r2169; + shr.u32 %r3401, %r3401, %r2169; + setp.gt.u32 %p324, %r2168, %r666; + @%p324 bra $L__BB4_282; + + setp.ne.s32 %p325, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs310, %rs532, 255; + setp.ne.s16 %p326, %rs310, 127; + and.pred %p327, %p325, %p326; + @%p327 bra $L__BB4_282; + + cvt.u16.u32 %rs457, %r2175; + or.b16 %rs456, %rs74, %rs457; + mov.u32 %r2178, 20548; + sub.s32 %r2179, %r2178, %r3508; + cvt.u64.u32 %rd163, %r2179; + add.s64 %rd164, %rd163, %rd5; + add.s64 %rd165, %rd1, %rd164; + st.global.u8 [%rd165], %rs456; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p328, %rs310, 143; + selp.u32 %r3510, 1, 0, %p328; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_282: + setp.ne.s32 %p329, %r3400, 0; + mov.u32 %r3411, %r3399; + @%p329 bra $L__BB4_278; + +$L__BB4_283: + setp.eq.s16 %p330, %rs65, 0; + mov.u32 %r3423, %r3411; + @%p330 bra $L__BB4_290; + + cvt.u32.u16 %r2180, %rs65; + and.b32 %r3412, %r2180, 255; + cvt.u32.u16 %r2181, %rs64; + and.b32 %r3413, %r2181, 255; + +$L__BB4_285: + mov.u32 %r685, %r3412; + setp.gt.u32 %p331, %r3508, 2879; + mov.u32 %r3423, 1; + @%p331 bra $L__BB4_290; + + mov.u32 %r2183, 8; + sub.s32 %r2184, %r2183, %r3510; + sub.s32 %r2185, %r2184, %r3509; + min.u32 %r2186, %r2185, %r685; + setp.eq.s32 %p332, %r2186, 32; + mov.u32 %r2187, -1; + shl.b32 %r2188, %r2187, %r2186; + not.b32 %r2189, %r2188; + selp.b32 %r2190, -1, %r2189, %p332; + and.b32 %r2191, %r2190, %r3413; + shl.b32 %r2192, %r2191, %r3509; + cvt.u16.u32 %rs314, %r2192; + or.b16 %rs532, %rs532, %rs314; + add.s32 %r3509, %r2186, %r3509; + sub.s32 %r3412, %r685, %r2186; + shr.u32 %r3413, %r3413, %r2186; + setp.gt.u32 %p333, %r2185, %r685; + @%p333 bra $L__BB4_289; + + setp.ne.s32 %p334, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs315, %rs532, 255; + setp.ne.s16 %p335, %rs315, 127; + and.pred %p336, %p334, %p335; + @%p336 bra $L__BB4_289; + + mov.u32 %r2195, 20548; + sub.s32 %r2196, %r2195, %r3508; + cvt.u64.u32 %rd166, %r2196; + add.s64 %rd167, %rd166, %rd5; + add.s64 %rd168, %rd1, %rd167; + st.global.u8 [%rd168], %rs532; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p337, %rs315, 143; + selp.u32 %r3510, 1, 0, %p337; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_289: + setp.ne.s32 %p338, %r3412, 0; + mov.u32 %r3423, %r3411; + @%p338 bra $L__BB4_285; + +$L__BB4_290: + setp.eq.s16 %p339, %rs69, 0; + mov.u32 %r3511, %r3423; + @%p339 bra $L__BB4_345; + + cvt.u32.u16 %r2197, %rs68; + and.b32 %r3425, %r2197, 255; + cvt.u32.u16 %r2198, %rs69; + and.b32 %r3424, %r2198, 255; + +$L__BB4_292: + mov.u32 %r704, %r3424; + setp.gt.u32 %p340, %r3508, 2879; + mov.u32 %r3511, 1; + @%p340 bra $L__BB4_345; + + mov.u32 %r2200, 8; + sub.s32 %r2201, %r2200, %r3510; + sub.s32 %r2202, %r2201, %r3509; + min.u32 %r2203, %r2202, %r704; + setp.eq.s32 %p341, %r2203, 32; + mov.u32 %r2204, -1; + shl.b32 %r2205, %r2204, %r2203; + not.b32 %r2206, %r2205; + selp.b32 %r2207, -1, %r2206, %p341; + and.b32 %r2208, %r2207, %r3425; + shl.b32 %r2209, %r2208, %r3509; + cvt.u16.u32 %rs319, %r2209; + or.b16 %rs532, %rs532, %rs319; + add.s32 %r3509, %r2203, %r3509; + sub.s32 %r3424, %r704, %r2203; + shr.u32 %r3425, %r3425, %r2203; + setp.gt.u32 %p342, %r2202, %r704; + @%p342 bra $L__BB4_296; + + setp.ne.s32 %p343, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs320, %rs532, 255; + setp.ne.s16 %p344, %rs320, 127; + and.pred %p345, %p343, %p344; + @%p345 bra $L__BB4_296; + + mov.u32 %r2212, 20548; + sub.s32 %r2213, %r2212, %r3508; + cvt.u64.u32 %rd169, %r2213; + add.s64 %rd170, %rd169, %rd5; + add.s64 %rd171, %rd1, %rd170; + st.global.u8 [%rd171], %rs532; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p346, %rs320, 143; + selp.u32 %r3510, 1, 0, %p346; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_296: + setp.eq.s32 %p347, %r3424, 0; + mov.u32 %r3511, %r3423; + @%p347 bra $L__BB4_345; + bra.uni $L__BB4_292; + +$L__BB4_345: + and.b32 %r3022, %r3204, 1; + add.s64 %rd336, %rd336, 16; + shr.u32 %r2335, %r3204, 1; + or.b32 %r3051, %r2335, %r3022; + add.s32 %r3037, %r3037, 4; + setp.lt.u32 %p410, %r3037, 64; + @%p410 bra $L__BB4_17; + + ld.param.u64 %rd329, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_3]; + mov.u16 %rs355, 0; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+33], %rs355; + add.s64 %rd16, %rd4, 128; + cvta.to.global.u64 %rd17, %rd31; + cvta.to.global.u64 %rd18, %rd329; + mov.u32 %r3512, 2; + mov.u64 %rd337, 0; + +$L__BB4_347: + shl.b64 %rd206, %rd337, 7; + add.s64 %rd207, %rd16, %rd206; + shl.b64 %rd208, %rd207, 2; + add.s64 %rd338, %rd3, %rd208; + ld.shared.u8 %rs537, [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val+1]; + mov.u32 %r2338, 0; + ld.shared.u8 %rs358, [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val]; + max.u16 %rs539, %rs358, %rs537; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val], %rs355; + ld.shared.u8 %r2339, [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val]; + ld.shared.u8 %rs535, [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val+1]; + mul.wide.u16 %r2340, %rs535, 4; + add.s32 %r3543, %r2340, %r2339; + st.shared.u8 [_ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val], %rs355; + mov.u16 %rs536, %rs355; + mov.u16 %rs538, %rs355; + mov.u32 %r3528, %r2338; + mov.u32 %r3542, %r2338; + bra.uni $L__BB4_348; + +$L__BB4_376: + setp.gt.u32 %p439, %r3111, 191; + mov.u32 %r3613, 1; + mov.u32 %r3120, 0; + @%p439 bra $L__BB4_378; + + st.global.u8 [%rd22], %rs474; + add.s32 %r3111, %r3111, 1; + mov.u16 %rs474, 0; + mov.u32 %r3120, 8; + mov.u32 %r3613, %r3277; + bra.uni $L__BB4_378; + +$L__BB4_477: + setp.gt.u32 %p554, %r3111, 191; + mov.u32 %r3755, 1; + mov.u32 %r3120, 0; + @%p554 bra $L__BB4_479; + + st.global.u8 [%rd23], %rs474; + add.s32 %r3111, %r3111, 1; + mov.u16 %rs474, 0; + mov.u32 %r3120, 8; + mov.u32 %r3755, %r3277; + bra.uni $L__BB4_479; + +$L__BB4_348: + mov.u32 %r878, %r3542; + ld.global.u32 %r882, [%rd338]; + setp.eq.s32 %p411, %r882, 0; + mov.u32 %r3546, %r2338; + @%p411 bra $L__BB4_350; + + and.b32 %r2342, %r882, -2147483648; + abs.s32 %r2343, %r882; + shl.b32 %r2344, %r2343, %r23; + or.b32 %r3546, %r2344, %r2342; + +$L__BB4_350: + shl.b32 %r2348, %r3546, 1; + shr.u32 %r2349, %r2348, %r23; + and.b32 %r885, %r2349, -2; + setp.eq.s32 %p412, %r885, 0; + mov.u32 %r3550, 0; + mov.u32 %r3547, %r3550; + mov.u32 %r3548, %r3550; + mov.u32 %r3554, %r3550; + @%p412 bra $L__BB4_352; + + add.s32 %r2351, %r885, -1; + clz.b32 %r2352, %r2351; + mov.u32 %r2353, 32; + sub.s32 %r3547, %r2353, %r2352; + shr.u32 %r2354, %r3546, 31; + add.s32 %r2355, %r2354, %r885; + add.s32 %r3548, %r2355, -2; + mov.u32 %r3554, 1; + +$L__BB4_352: + ld.global.u32 %r891, [%rd338+256]; + setp.eq.s32 %p413, %r891, 0; + @%p413 bra $L__BB4_354; + + and.b32 %r2357, %r891, -2147483648; + abs.s32 %r2358, %r891; + shl.b32 %r2359, %r2358, %r23; + or.b32 %r3550, %r2359, %r2357; + +$L__BB4_354: + shl.b32 %r2362, %r3550, 1; + shr.u32 %r2363, %r2362, %r23; + and.b32 %r894, %r2363, -2; + setp.eq.s32 %p414, %r894, 0; + mov.u32 %r3555, 0; + mov.u32 %r3551, %r3555; + mov.u32 %r3552, %r3555; + mov.u32 %r3558, %r3547; + @%p414 bra $L__BB4_356; + + or.b32 %r3554, %r3554, 2; + add.s32 %r2364, %r894, -1; + clz.b32 %r2365, %r2364; + mov.u32 %r2366, 32; + sub.s32 %r3551, %r2366, %r2365; + max.s32 %r3558, %r3547, %r3551; + shr.u32 %r2367, %r3550, 31; + add.s32 %r2368, %r2367, %r894; + add.s32 %r3552, %r2368, -2; + +$L__BB4_356: + ld.global.u32 %r903, [%rd338+4]; + setp.eq.s32 %p415, %r903, 0; + @%p415 bra $L__BB4_358; + + and.b32 %r2370, %r903, -2147483648; + abs.s32 %r2371, %r903; + shl.b32 %r2372, %r2371, %r23; + or.b32 %r3555, %r2372, %r2370; + +$L__BB4_358: + shl.b32 %r2375, %r3555, 1; + shr.u32 %r2376, %r2375, %r23; + and.b32 %r906, %r2376, -2; + setp.eq.s32 %p416, %r906, 0; + mov.u32 %r3560, 0; + mov.u32 %r3556, %r3560; + mov.u32 %r3557, %r3560; + @%p416 bra $L__BB4_360; + + or.b32 %r3554, %r3554, 4; + add.s32 %r2377, %r906, -1; + clz.b32 %r2378, %r2377; + mov.u32 %r2379, 32; + sub.s32 %r3556, %r2379, %r2378; + max.s32 %r3558, %r3558, %r3556; + shr.u32 %r2380, %r3555, 31; + add.s32 %r2381, %r2380, %r906; + add.s32 %r3557, %r2381, -2; + +$L__BB4_360: + ld.global.u32 %r915, [%rd338+260]; + setp.eq.s32 %p417, %r915, 0; + @%p417 bra $L__BB4_362; + + and.b32 %r2383, %r915, -2147483648; + abs.s32 %r2384, %r915; + shl.b32 %r2385, %r2384, %r23; + or.b32 %r3560, %r2385, %r2383; + +$L__BB4_362: + shl.b32 %r2388, %r3560, 1; + shr.u32 %r2389, %r2388, %r23; + and.b32 %r918, %r2389, -2; + setp.eq.s32 %p418, %r918, 0; + mov.u32 %r3565, 0; + mov.u32 %r3561, %r3565; + mov.u32 %r3562, %r3565; + @%p418 bra $L__BB4_364; + + or.b32 %r3554, %r3554, 8; + add.s32 %r2390, %r918, -1; + clz.b32 %r2391, %r2390; + mov.u32 %r2392, 32; + sub.s32 %r3561, %r2392, %r2391; + max.s32 %r3558, %r3558, %r3561; + shr.u32 %r2393, %r3560, 31; + add.s32 %r2394, %r2393, %r918; + add.s32 %r3562, %r2394, -2; + +$L__BB4_364: + add.s32 %r2396, %r3554, -1; + and.b32 %r2397, %r2396, %r3554; + setp.ne.s32 %p419, %r2397, 0; + and.b16 %rs359, %rs539, 255; + setp.gt.u16 %p420, %rs359, 2; + and.pred %p421, %p420, %p419; + cvt.u32.u16 %r2398, %rs539; + and.b32 %r2399, %r2398, 255; + add.s32 %r2400, %r2399, -1; + selp.b32 %r2401, %r2400, 1, %p421; + max.s32 %r927, %r2401, %r3558; + sub.s32 %r928, %r927, %r2401; + setp.lt.s32 %p422, %r928, 1; + @%p422 bra $L__BB4_366; + + setp.eq.s32 %p423, %r3547, %r3558; + selp.u32 %r2402, 1, 0, %p423; + setp.eq.s32 %p424, %r3551, %r3558; + selp.u32 %r2403, -1, 0, %p424; + bfi.b32 %r2404, %r2403, %r2402, 1, 1; + setp.eq.s32 %p425, %r3556, %r3558; + selp.u16 %rs360, 1, 0, %p425; + mul.wide.u16 %r2405, %rs360, 4; + or.b32 %r2406, %r2404, %r2405; + setp.eq.s32 %p426, %r3561, %r3558; + selp.u16 %rs361, 1, 0, %p426; + mul.wide.u16 %r2407, %rs361, 8; + or.b32 %r3565, %r2406, %r2407; + +$L__BB4_366: + shl.b32 %r2408, %r3554, 4; + shl.b32 %r2409, %r3543, 8; + or.b32 %r2410, %r2408, %r2409; + or.b32 %r2411, %r2410, %r3565; + mul.wide.u32 %rd209, %r2411, 2; + add.s64 %rd210, %rd18, %rd209; + ld.global.u16 %rs134, [%rd210]; + shr.u16 %rs362, %rs134, 4; + and.b16 %rs135, %rs362, 7; + setp.eq.s16 %p427, %rs135, 0; + mov.u32 %r3577, %r3511; + @%p427 bra $L__BB4_373; + + cvt.u32.u16 %r3566, %rs135; + shr.u16 %rs363, %rs134, 8; + cvt.u32.u16 %r3567, %rs363; + +$L__BB4_368: + mov.u32 %r933, %r3566; + setp.gt.u32 %p428, %r3508, 2879; + mov.u32 %r3577, 1; + @%p428 bra $L__BB4_373; + + mov.u32 %r2413, 8; + sub.s32 %r2414, %r2413, %r3510; + sub.s32 %r2415, %r2414, %r3509; + min.u32 %r2416, %r2415, %r933; + setp.eq.s32 %p429, %r2416, 32; + mov.u32 %r2417, -1; + shl.b32 %r2418, %r2417, %r2416; + not.b32 %r2419, %r2418; + selp.b32 %r2420, -1, %r2419, %p429; + and.b32 %r2421, %r2420, %r3567; + shl.b32 %r2422, %r2421, %r3509; + cvt.u16.u32 %rs364, %r2422; + or.b16 %rs532, %rs532, %rs364; + add.s32 %r3509, %r2416, %r3509; + sub.s32 %r3566, %r933, %r2416; + shr.u32 %r3567, %r3567, %r2416; + setp.gt.u32 %p430, %r2415, %r933; + @%p430 bra $L__BB4_372; + + setp.ne.s32 %p431, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs365, %rs532, 255; + setp.ne.s16 %p432, %rs365, 127; + and.pred %p433, %p431, %p432; + @%p433 bra $L__BB4_372; + + mov.u32 %r2425, 20548; + sub.s32 %r2426, %r2425, %r3508; + cvt.u64.u32 %rd211, %r2426; + add.s64 %rd212, %rd211, %rd5; + add.s64 %rd213, %rd1, %rd212; + st.global.u8 [%rd213], %rs532; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p434, %rs365, 143; + selp.u32 %r3510, 1, 0, %p434; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_372: + setp.ne.s32 %p435, %r3566, 0; + mov.u32 %r3577, %r3511; + @%p435 bra $L__BB4_368; + +$L__BB4_373: + setp.ne.s32 %p436, %r3543, 0; + @%p436 bra $L__BB4_421; + + setp.eq.s32 %p437, %r3554, 0; + add.s32 %r2427, %r3111, 17477; + cvt.u64.u32 %rd214, %r2427; + add.s64 %rd215, %rd214, %rd5; + add.s64 %rd22, %rd1, %rd215; + @%p437 bra $L__BB4_413; + + shl.b16 %rs474, %rs474, 1; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p438, %r3120, 0; + mov.u32 %r3613, %r3277; + @%p438 bra $L__BB4_378; + bra.uni $L__BB4_376; + +$L__BB4_378: + setp.lt.u32 %p440, %r3275, 3; + mov.u32 %r3581, 0; + @%p440 bra $L__BB4_381; + + setp.lt.u32 %p441, %r3275, 6; + mov.u32 %r3581, 1; + @%p441 bra $L__BB4_381; + + setp.lt.u32 %p442, %r3275, 9; + setp.eq.s32 %p443, %r3275, 11; + selp.b32 %r2433, 4, 5, %p443; + setp.lt.u32 %p444, %r3275, 11; + selp.b32 %r2434, 3, %r2433, %p444; + selp.b32 %r3581, 2, %r2434, %p442; + +$L__BB4_381: + setp.eq.s32 %p445, %r3581, 0; + @%p445 bra $L__BB4_409; + + add.s32 %r957, %r3581, -1; + and.b32 %r958, %r3581, 3; + setp.eq.s32 %p446, %r958, 0; + mov.u32 %r3591, %r3581; + mov.u32 %r3592, %r3613; + @%p446 bra $L__BB4_394; + + mov.u32 %r2436, 1; + shl.b32 %r2437, %r2436, %r957; + and.b32 %r2438, %r2437, %r3274; + setp.ne.s32 %p447, %r2438, 0; + selp.u32 %r2439, 1, 0, %p447; + cvt.u32.u16 %r2440, %rs474; + bfi.b32 %r2441, %r2440, %r2439, 1, 8; + cvt.u16.u32 %rs474, %r2441; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p448, %r3120, 0; + mov.u32 %r3592, %r3613; + @%p448 bra $L__BB4_386; + + setp.gt.u32 %p449, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3592, %r2436; + @%p449 bra $L__BB4_386; + + add.s32 %r2445, %r3111, 17477; + cvt.u64.u32 %rd216, %r2445; + add.s64 %rd217, %rd216, %rd5; + add.s64 %rd218, %rd1, %rd217; + st.global.u8 [%rd218], %rs474; + add.s32 %r3111, %r3111, 1; + mov.u16 %rs474, 0; + mov.u32 %r3120, 8; + mov.u32 %r3592, %r3613; + +$L__BB4_386: + setp.eq.s32 %p450, %r958, 1; + mov.u32 %r3613, %r3592; + mov.u32 %r3591, %r957; + @%p450 bra $L__BB4_394; + + add.s32 %r3591, %r3581, -2; + mov.u32 %r2446, 1; + shl.b32 %r2447, %r2446, %r3591; + and.b32 %r2448, %r2447, %r3274; + setp.ne.s32 %p451, %r2448, 0; + selp.u32 %r2449, 1, 0, %p451; + cvt.u32.u16 %r2450, %rs474; + bfi.b32 %r2451, %r2450, %r2449, 1, 8; + cvt.u16.u32 %rs474, %r2451; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p452, %r3120, 0; + mov.u32 %r3587, %r3592; + @%p452 bra $L__BB4_390; + + setp.gt.u32 %p453, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3587, %r2446; + @%p453 bra $L__BB4_390; + + add.s32 %r2454, %r3111, 17477; + cvt.u64.u32 %rd219, %r2454; + add.s64 %rd220, %rd219, %rd5; + add.s64 %rd221, %rd1, %rd220; + and.b16 %rs372, %rs474, 255; + st.global.u8 [%rd221], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p454, %rs372, 255; + selp.b32 %r3120, 7, 8, %p454; + mov.u16 %rs474, 0; + mov.u32 %r3587, %r3592; + +$L__BB4_390: + setp.eq.s32 %p455, %r958, 2; + mov.u32 %r3613, %r3587; + mov.u32 %r3592, %r3587; + @%p455 bra $L__BB4_394; + + add.s32 %r3591, %r3581, -3; + mov.u32 %r2455, 1; + shl.b32 %r2456, %r2455, %r3591; + and.b32 %r2457, %r2456, %r3274; + setp.ne.s32 %p456, %r2457, 0; + selp.u32 %r2458, 1, 0, %p456; + cvt.u32.u16 %r2459, %rs474; + bfi.b32 %r2460, %r2459, %r2458, 1, 8; + cvt.u16.u32 %rs474, %r2460; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p457, %r3120, 0; + mov.u32 %r3613, %r3587; + mov.u32 %r3592, %r3587; + @%p457 bra $L__BB4_394; + + setp.gt.u32 %p458, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3613, %r2455; + mov.u32 %r3592, %r2455; + @%p458 bra $L__BB4_394; + + add.s32 %r2465, %r3111, 17477; + cvt.u64.u32 %rd222, %r2465; + add.s64 %rd223, %rd222, %rd5; + add.s64 %rd224, %rd1, %rd223; + and.b16 %rs375, %rs474, 255; + st.global.u8 [%rd224], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p459, %rs375, 255; + selp.b32 %r3120, 7, 8, %p459; + mov.u16 %rs474, 0; + mov.u32 %r3613, %r3587; + mov.u32 %r3592, %r3587; + +$L__BB4_394: + setp.lt.u32 %p460, %r957, 3; + @%p460 bra $L__BB4_409; + + mov.u32 %r3613, %r3592; + +$L__BB4_396: + add.s32 %r2466, %r3591, -1; + mov.u32 %r2467, 1; + shl.b32 %r2468, %r2467, %r2466; + and.b32 %r2469, %r2468, %r3274; + setp.ne.s32 %p461, %r2469, 0; + selp.u32 %r2470, 1, 0, %p461; + cvt.u32.u16 %r2471, %rs474; + bfi.b32 %r3601, %r2471, %r2470, 1, 8; + add.s32 %r3600, %r3120, -1; + setp.ne.s32 %p462, %r3600, 0; + mov.u32 %r3602, %r3613; + @%p462 bra $L__BB4_399; + + setp.gt.u32 %p463, %r3111, 191; + mov.u32 %r3600, 0; + mov.u32 %r3602, %r2467; + @%p463 bra $L__BB4_399; + + cvt.u16.u32 %rs376, %r3601; + and.b16 %rs377, %rs376, 255; + add.s32 %r2475, %r3111, 17477; + cvt.u64.u32 %rd225, %r2475; + add.s64 %rd226, %rd225, %rd5; + add.s64 %rd227, %rd1, %rd226; + st.global.u8 [%rd227], %rs376; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p464, %rs377, 255; + selp.b32 %r3600, 7, 8, %p464; + mov.u32 %r3601, 0; + mov.u32 %r3602, %r3613; + +$L__BB4_399: + add.s32 %r2476, %r3591, -2; + shl.b32 %r2478, %r2467, %r2476; + and.b32 %r2479, %r2478, %r3274; + setp.ne.s32 %p465, %r2479, 0; + and.b32 %r2480, %r3601, 127; + selp.u32 %r2481, 1, 0, %p465; + bfi.b32 %r3605, %r2480, %r2481, 1, 7; + add.s32 %r3604, %r3600, -1; + setp.ne.s32 %p466, %r3604, 0; + mov.u32 %r3606, %r3602; + @%p466 bra $L__BB4_402; + + setp.gt.u32 %p467, %r3111, 191; + mov.u32 %r3606, 1; + mov.u32 %r3604, 0; + @%p467 bra $L__BB4_402; + + cvt.u16.u32 %rs378, %r3605; + and.b16 %rs379, %rs378, 255; + add.s32 %r2485, %r3111, 17477; + cvt.u64.u32 %rd228, %r2485; + add.s64 %rd229, %rd228, %rd5; + add.s64 %rd230, %rd1, %rd229; + st.global.u8 [%rd230], %rs378; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p468, %rs379, 255; + selp.b32 %r3604, 7, 8, %p468; + mov.u32 %r3605, 0; + mov.u32 %r3606, %r3602; + +$L__BB4_402: + add.s32 %r2486, %r3591, -3; + mov.u32 %r2487, 1; + shl.b32 %r2488, %r2487, %r2486; + and.b32 %r2489, %r2488, %r3274; + setp.ne.s32 %p469, %r2489, 0; + and.b32 %r2490, %r3605, 127; + selp.u32 %r2491, 1, 0, %p469; + bfi.b32 %r3609, %r2490, %r2491, 1, 7; + add.s32 %r3608, %r3604, -1; + setp.ne.s32 %p470, %r3608, 0; + mov.u32 %r3610, %r3606; + @%p470 bra $L__BB4_405; + + setp.gt.u32 %p471, %r3111, 191; + mov.u32 %r3608, 0; + mov.u32 %r3610, %r2487; + @%p471 bra $L__BB4_405; + + cvt.u16.u32 %rs380, %r3609; + and.b16 %rs381, %rs380, 255; + add.s32 %r2495, %r3111, 17477; + cvt.u64.u32 %rd231, %r2495; + add.s64 %rd232, %rd231, %rd5; + add.s64 %rd233, %rd1, %rd232; + st.global.u8 [%rd233], %rs380; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p472, %rs381, 255; + selp.b32 %r3608, 7, 8, %p472; + mov.u32 %r3609, 0; + mov.u32 %r3610, %r3606; + +$L__BB4_405: + add.s32 %r3591, %r3591, -4; + shl.b32 %r2497, %r2487, %r3591; + and.b32 %r2498, %r2497, %r3274; + setp.ne.s32 %p473, %r2498, 0; + and.b32 %r2499, %r3609, 127; + selp.u32 %r2500, 1, 0, %p473; + bfi.b32 %r2501, %r2499, %r2500, 1, 15; + cvt.u16.u32 %rs474, %r2501; + add.s32 %r3120, %r3608, -1; + setp.ne.s32 %p474, %r3120, 0; + mov.u32 %r3613, %r3610; + @%p474 bra $L__BB4_408; + + setp.gt.u32 %p475, %r3111, 191; + mov.u32 %r3613, 1; + mov.u32 %r3120, 0; + @%p475 bra $L__BB4_408; + + add.s32 %r2504, %r3111, 17477; + cvt.u64.u32 %rd234, %r2504; + add.s64 %rd235, %rd234, %rd5; + add.s64 %rd236, %rd1, %rd235; + and.b16 %rs383, %rs474, 255; + st.global.u8 [%rd236], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p476, %rs383, 255; + selp.b32 %r3120, 7, 8, %p476; + mov.u16 %rs474, 0; + mov.u32 %r3613, %r3610; + +$L__BB4_408: + setp.ne.s32 %p477, %r3591, 0; + @%p477 bra $L__BB4_396; + +$L__BB4_409: + add.s32 %r2506, %r3275, -1; + setp.eq.s32 %p478, %r3275, 0; + mov.u32 %r3274, 0; + selp.b32 %r3275, 0, %r2506, %p478; + setp.lt.u32 %p479, %r3275, 3; + mov.u32 %r3617, %r3274; + @%p479 bra $L__BB4_412; + + setp.lt.u32 %p480, %r3275, 6; + mov.u32 %r3617, 1; + @%p480 bra $L__BB4_412; + + setp.lt.u32 %p481, %r3275, 9; + setp.eq.s32 %p482, %r3275, 11; + selp.b32 %r2508, 4, 5, %p482; + setp.lt.u32 %p483, %r3275, 11; + selp.b32 %r2509, 3, %r2508, %p483; + selp.b32 %r3617, 2, %r2509, %p481; + +$L__BB4_412: + mov.u32 %r2511, 1; + shl.b32 %r3276, %r2511, %r3617; + mov.u32 %r3277, %r3613; + bra.uni $L__BB4_421; + +$L__BB4_413: + add.s32 %r3274, %r3274, 1; + setp.lt.u32 %p484, %r3274, %r3276; + @%p484 bra $L__BB4_421; + + shl.b16 %rs384, %rs474, 1; + or.b16 %rs474, %rs384, 1; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p485, %r3120, 0; + mov.u32 %r3620, %r3277; + @%p485 bra $L__BB4_417; + + setp.gt.u32 %p486, %r3111, 191; + mov.u32 %r3620, 1; + mov.u32 %r3120, 0; + @%p486 bra $L__BB4_417; + + and.b16 %rs386, %rs474, 255; + st.global.u8 [%rd22], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p487, %rs386, 255; + selp.b32 %r3120, 7, 8, %p487; + mov.u16 %rs474, 0; + mov.u32 %r3620, %r3277; + +$L__BB4_417: + add.s32 %r2515, %r3275, 1; + min.u32 %r3275, %r2515, 12; + setp.lt.u32 %p488, %r3275, 3; + mov.u32 %r3274, 0; + mov.u32 %r3621, %r3274; + @%p488 bra $L__BB4_420; + + setp.lt.u32 %p489, %r3275, 6; + mov.u32 %r3621, 1; + @%p489 bra $L__BB4_420; + + setp.lt.u32 %p490, %r3275, 9; + setp.eq.s32 %p491, %r3275, 11; + selp.b32 %r2517, 4, 5, %p491; + setp.lt.u32 %p492, %r3275, 11; + selp.b32 %r2518, 3, %r2517, %p492; + selp.b32 %r3621, 2, %r2518, %p490; + +$L__BB4_420: + mov.u32 %r2520, 1; + shl.b32 %r3276, %r2520, %r3621; + mov.u32 %r3277, %r3620; + +$L__BB4_421: + and.b16 %rs387, %rs134, 15; + cvt.u32.u16 %r1041, %rs387; + and.b32 %r2521, %r3554, 1; + setp.eq.b32 %p493, %r2521, 1; + mov.pred %p494, 0; + xor.pred %p495, %p493, %p494; + not.pred %p496, %p495; + mov.u32 %r3642, %r3829; + @%p496 bra $L__BB4_428; + + and.b32 %r2522, %r1041, 1; + sub.s32 %r3628, %r927, %r2522; + setp.eq.s32 %p497, %r3628, 0; + mov.u32 %r3642, %r3829; + @%p497 bra $L__BB4_428; + + mov.u32 %r2523, -1; + shl.b32 %r2524, %r2523, %r3628; + not.b32 %r2525, %r2524; + and.b32 %r3629, %r3548, %r2525; + +$L__BB4_424: + setp.gt.u32 %p498, %r3654, 17476; + mov.u32 %r3642, 1; + @%p498 bra $L__BB4_428; + + sub.s32 %r2527, %r3653, %r3655; + min.u32 %r2528, %r2527, %r3628; + setp.eq.s32 %p499, %r2528, 32; + mov.u32 %r2529, -1; + shl.b32 %r2530, %r2529, %r2528; + not.b32 %r2531, %r2530; + selp.b32 %r2532, -1, %r2531, %p499; + and.b32 %r2533, %r2532, %r3629; + shl.b32 %r2534, %r2533, %r3655; + or.b32 %r3656, %r2534, %r3656; + add.s32 %r3655, %r2528, %r3655; + shr.u32 %r3629, %r3629, %r2528; + sub.s32 %r3628, %r3628, %r2528; + setp.lt.u32 %p500, %r3655, %r3653; + @%p500 bra $L__BB4_427; + + cvt.u64.u32 %rd237, %r3654; + add.s64 %rd238, %rd237, %rd5; + add.s64 %rd239, %rd1, %rd238; + st.global.u8 [%rd239], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p501, %r3656, 255; + selp.b32 %r3653, 7, 8, %p501; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_427: + setp.ne.s32 %p502, %r3628, 0; + mov.u32 %r3642, %r3829; + @%p502 bra $L__BB4_424; + +$L__BB4_428: + and.b32 %r1065, %r3554, 2; + setp.eq.s32 %p503, %r1065, 0; + mov.u32 %r3657, %r3642; + @%p503 bra $L__BB4_435; + + shr.u32 %r2537, %r1041, 1; + and.b32 %r2538, %r2537, 1; + sub.s32 %r3643, %r927, %r2538; + setp.eq.s32 %p504, %r3643, 0; + mov.u32 %r3657, %r3642; + @%p504 bra $L__BB4_435; + + mov.u32 %r2539, -1; + shl.b32 %r2540, %r2539, %r3643; + not.b32 %r2541, %r2540; + and.b32 %r3644, %r3552, %r2541; + +$L__BB4_431: + setp.gt.u32 %p505, %r3654, 17476; + mov.u32 %r3657, 1; + @%p505 bra $L__BB4_435; + + sub.s32 %r2543, %r3653, %r3655; + min.u32 %r2544, %r2543, %r3643; + setp.eq.s32 %p506, %r2544, 32; + mov.u32 %r2545, -1; + shl.b32 %r2546, %r2545, %r2544; + not.b32 %r2547, %r2546; + selp.b32 %r2548, -1, %r2547, %p506; + and.b32 %r2549, %r2548, %r3644; + shl.b32 %r2550, %r2549, %r3655; + or.b32 %r3656, %r2550, %r3656; + add.s32 %r3655, %r2544, %r3655; + shr.u32 %r3644, %r3644, %r2544; + sub.s32 %r3643, %r3643, %r2544; + setp.lt.u32 %p507, %r3655, %r3653; + @%p507 bra $L__BB4_434; + + cvt.u64.u32 %rd240, %r3654; + add.s64 %rd241, %rd240, %rd5; + add.s64 %rd242, %rd1, %rd241; + st.global.u8 [%rd242], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p508, %r3656, 255; + selp.b32 %r3653, 7, 8, %p508; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_434: + setp.ne.s32 %p509, %r3643, 0; + mov.u32 %r3657, %r3642; + @%p509 bra $L__BB4_431; + +$L__BB4_435: + and.b32 %r1089, %r3554, 4; + setp.eq.s32 %p510, %r1089, 0; + mov.u32 %r3672, %r3657; + @%p510 bra $L__BB4_442; + + shr.u32 %r2553, %r1041, 2; + and.b32 %r2554, %r2553, 1; + sub.s32 %r3658, %r927, %r2554; + setp.eq.s32 %p511, %r3658, 0; + mov.u32 %r3672, %r3657; + @%p511 bra $L__BB4_442; + + mov.u32 %r2555, -1; + shl.b32 %r2556, %r2555, %r3658; + not.b32 %r2557, %r2556; + and.b32 %r3659, %r3557, %r2557; + +$L__BB4_438: + setp.gt.u32 %p512, %r3654, 17476; + mov.u32 %r3672, 1; + @%p512 bra $L__BB4_442; + + sub.s32 %r2559, %r3653, %r3655; + min.u32 %r2560, %r2559, %r3658; + setp.eq.s32 %p513, %r2560, 32; + mov.u32 %r2561, -1; + shl.b32 %r2562, %r2561, %r2560; + not.b32 %r2563, %r2562; + selp.b32 %r2564, -1, %r2563, %p513; + and.b32 %r2565, %r2564, %r3659; + shl.b32 %r2566, %r2565, %r3655; + or.b32 %r3656, %r2566, %r3656; + add.s32 %r3655, %r2560, %r3655; + shr.u32 %r3659, %r3659, %r2560; + sub.s32 %r3658, %r3658, %r2560; + setp.lt.u32 %p514, %r3655, %r3653; + @%p514 bra $L__BB4_441; + + cvt.u64.u32 %rd243, %r3654; + add.s64 %rd244, %rd243, %rd5; + add.s64 %rd245, %rd1, %rd244; + st.global.u8 [%rd245], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p515, %r3656, 255; + selp.b32 %r3653, 7, 8, %p515; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_441: + setp.ne.s32 %p516, %r3658, 0; + mov.u32 %r3672, %r3657; + @%p516 bra $L__BB4_438; + +$L__BB4_442: + and.b32 %r1113, %r3554, 8; + setp.eq.s32 %p517, %r1113, 0; + mov.u32 %r3687, %r3672; + @%p517 bra $L__BB4_449; + + shr.u32 %r2569, %r1041, 3; + sub.s32 %r3673, %r927, %r2569; + setp.eq.s32 %p518, %r3673, 0; + mov.u32 %r3687, %r3672; + @%p518 bra $L__BB4_449; + + mov.u32 %r2570, -1; + shl.b32 %r2571, %r2570, %r3673; + not.b32 %r2572, %r2571; + and.b32 %r3674, %r3562, %r2572; + +$L__BB4_445: + setp.gt.u32 %p519, %r3654, 17476; + mov.u32 %r3687, 1; + @%p519 bra $L__BB4_449; + + sub.s32 %r2574, %r3653, %r3655; + min.u32 %r2575, %r2574, %r3673; + setp.eq.s32 %p520, %r2575, 32; + mov.u32 %r2576, -1; + shl.b32 %r2577, %r2576, %r2575; + not.b32 %r2578, %r2577; + selp.b32 %r2579, -1, %r2578, %p520; + and.b32 %r2580, %r2579, %r3674; + shl.b32 %r2581, %r2580, %r3655; + or.b32 %r3656, %r2581, %r3656; + add.s32 %r3655, %r2575, %r3655; + shr.u32 %r3674, %r3674, %r2575; + sub.s32 %r3673, %r3673, %r2575; + setp.lt.u32 %p521, %r3655, %r3653; + @%p521 bra $L__BB4_448; + + cvt.u64.u32 %rd246, %r3654; + add.s64 %rd247, %rd246, %rd5; + add.s64 %rd248, %rd1, %rd247; + st.global.u8 [%rd248], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p522, %r3656, 255; + selp.b32 %r3653, 7, 8, %p522; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_448: + setp.ne.s32 %p523, %r3673, 0; + mov.u32 %r3687, %r3672; + @%p523 bra $L__BB4_445; + +$L__BB4_449: + mov.u32 %r3000, _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E14cleanup_cx_val; + mov.u32 %r2999, _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val; + and.b32 %r2585, %r3551, 255; + cvt.u32.u16 %r2586, %rs538; + and.b32 %r2587, %r2586, 255; + setp.lt.u32 %p524, %r2585, %r2587; + cvt.u16.u32 %rs388, %r3551; + selp.b16 %rs389, %rs538, %rs388, %p524; + add.s32 %r1137, %r2999, %r878; + mov.u32 %r3689, 0; + st.shared.u8 [%r1137], %rs389; + ld.shared.u8 %rs156, [%r1137+2]; + setp.gt.u16 %p525, %rs537, %rs156; + add.s32 %r3542, %r878, 2; + add.s32 %r2589, %r878, 1; + selp.b32 %r2590, %r2589, %r3542, %p525; + add.s32 %r2591, %r2999, %r2590; + ld.shared.u8 %rs157, [%r2591]; + cvt.u16.u32 %rs158, %r3561; + st.shared.u8 [%r1137+1], %r3561; + cvt.u16.u32 %rs391, %r1065; + shr.u16 %rs392, %rs391, 1; + or.b16 %rs393, %rs536, %rs392; + add.s32 %r1139, %r3000, %r878; + st.shared.u8 [%r1139], %rs393; + ld.shared.u8 %r1140, [%r1139+2]; + shr.u32 %r1141, %r1113, 3; + st.shared.u8 [%r1139+1], %r1141; + ld.global.u32 %r1142, [%rd338+8]; + setp.eq.s32 %p526, %r1142, 0; + mov.u32 %r3688, %r3689; + @%p526 bra $L__BB4_451; + + and.b32 %r2593, %r1142, -2147483648; + abs.s32 %r2594, %r1142; + shl.b32 %r2595, %r2594, %r23; + or.b32 %r3688, %r2595, %r2593; + +$L__BB4_451: + shl.b32 %r2599, %r3688, 1; + shr.u32 %r2600, %r2599, %r23; + and.b32 %r1145, %r2600, -2; + setp.eq.s32 %p527, %r1145, 0; + mov.u32 %r3690, %r3689; + mov.u32 %r3696, %r3689; + @%p527 bra $L__BB4_453; + + add.s32 %r2602, %r1145, -1; + clz.b32 %r2603, %r2602; + mov.u32 %r2604, 32; + sub.s32 %r3689, %r2604, %r2603; + shr.u32 %r2605, %r3688, 31; + add.s32 %r2606, %r2605, %r1145; + add.s32 %r3690, %r2606, -2; + mov.u32 %r3696, 1; + +$L__BB4_453: + ld.global.u32 %r1151, [%rd338+264]; + setp.eq.s32 %p528, %r1151, 0; + mov.u32 %r3693, 0; + mov.u32 %r3692, %r3693; + @%p528 bra $L__BB4_455; + + and.b32 %r2608, %r1151, -2147483648; + abs.s32 %r2609, %r1151; + shl.b32 %r2610, %r2609, %r23; + or.b32 %r3692, %r2610, %r2608; + +$L__BB4_455: + shl.b32 %r2613, %r3692, 1; + shr.u32 %r2614, %r2613, %r23; + and.b32 %r1154, %r2614, -2; + setp.eq.s32 %p529, %r1154, 0; + mov.u32 %r3694, %r3693; + mov.u32 %r3700, %r3689; + @%p529 bra $L__BB4_457; + + or.b32 %r3696, %r3696, 2; + add.s32 %r2615, %r1154, -1; + clz.b32 %r2616, %r2615; + mov.u32 %r2617, 32; + sub.s32 %r3693, %r2617, %r2616; + max.s32 %r3700, %r3689, %r3693; + shr.u32 %r2618, %r3692, 31; + add.s32 %r2619, %r2618, %r1154; + add.s32 %r3694, %r2619, -2; + +$L__BB4_457: + ld.global.u32 %r1163, [%rd338+12]; + setp.eq.s32 %p530, %r1163, 0; + mov.u32 %r3698, 0; + mov.u32 %r3697, %r3698; + @%p530 bra $L__BB4_459; + + and.b32 %r2621, %r1163, -2147483648; + abs.s32 %r2622, %r1163; + shl.b32 %r2623, %r2622, %r23; + or.b32 %r3697, %r2623, %r2621; + +$L__BB4_459: + shl.b32 %r2626, %r3697, 1; + shr.u32 %r2627, %r2626, %r23; + and.b32 %r1166, %r2627, -2; + setp.eq.s32 %p531, %r1166, 0; + mov.u32 %r3699, %r3698; + @%p531 bra $L__BB4_461; + + or.b32 %r3696, %r3696, 4; + add.s32 %r2628, %r1166, -1; + clz.b32 %r2629, %r2628; + mov.u32 %r2630, 32; + sub.s32 %r3698, %r2630, %r2629; + max.s32 %r3700, %r3700, %r3698; + shr.u32 %r2631, %r3697, 31; + add.s32 %r2632, %r2631, %r1166; + add.s32 %r3699, %r2632, -2; + +$L__BB4_461: + ld.global.u32 %r1175, [%rd338+268]; + setp.eq.s32 %p532, %r1175, 0; + mov.u32 %r3703, 0; + mov.u32 %r3702, %r3703; + @%p532 bra $L__BB4_463; + + and.b32 %r2634, %r1175, -2147483648; + abs.s32 %r2635, %r1175; + shl.b32 %r2636, %r2635, %r23; + or.b32 %r3702, %r2636, %r2634; + +$L__BB4_463: + shl.b32 %r2639, %r3702, 1; + shr.u32 %r2640, %r2639, %r23; + and.b32 %r1178, %r2640, -2; + setp.eq.s32 %p533, %r1178, 0; + mov.u32 %r3704, %r3703; + @%p533 bra $L__BB4_465; + + or.b32 %r3696, %r3696, 8; + add.s32 %r2641, %r1178, -1; + clz.b32 %r2642, %r2641; + mov.u32 %r2643, 32; + sub.s32 %r3703, %r2643, %r2642; + max.s32 %r3700, %r3700, %r3703; + shr.u32 %r2644, %r3702, 31; + add.s32 %r2645, %r2644, %r1178; + add.s32 %r3704, %r2645, -2; + +$L__BB4_465: + shr.u32 %r2647, %r1113, 2; + shr.u32 %r2648, %r1089, 1; + or.b32 %r2649, %r2647, %r2648; + shl.b32 %r2650, %r1140, 2; + cvt.u32.u16 %r2651, %rs535; + and.b32 %r2652, %r2651, 255; + add.s32 %r2653, %r2650, %r2652; + or.b32 %r1187, %r2649, %r2653; + add.s32 %r2654, %r3696, -1; + and.b32 %r2655, %r2654, %r3696; + setp.ne.s32 %p534, %r2655, 0; + mov.u32 %r3707, 0; + setp.gt.u16 %p535, %rs157, 2; + and.pred %p536, %p535, %p534; + cvt.u32.u16 %r2656, %rs157; + and.b32 %r2657, %r2656, 255; + add.s32 %r2658, %r2657, -1; + selp.b32 %r2659, %r2658, 1, %p536; + max.s32 %r1188, %r2659, %r3700; + sub.s32 %r1189, %r1188, %r2659; + setp.lt.s32 %p537, %r1189, 1; + @%p537 bra $L__BB4_467; + + setp.eq.s32 %p538, %r3689, %r3700; + selp.u32 %r2660, 1, 0, %p538; + setp.eq.s32 %p539, %r3693, %r3700; + selp.u32 %r2661, -1, 0, %p539; + bfi.b32 %r2662, %r2661, %r2660, 1, 1; + setp.eq.s32 %p540, %r3698, %r3700; + selp.u16 %rs395, 1, 0, %p540; + mul.wide.u16 %r2663, %rs395, 4; + or.b32 %r2664, %r2662, %r2663; + setp.eq.s32 %p541, %r3703, %r3700; + selp.u16 %rs396, 1, 0, %p541; + mul.wide.u16 %r2665, %rs396, 8; + or.b32 %r3707, %r2664, %r2665; + +$L__BB4_467: + shl.b32 %r2666, %r3696, 4; + shl.b32 %r2667, %r1187, 8; + or.b32 %r2668, %r2666, %r2667; + or.b32 %r2669, %r2668, %r3707; + mul.wide.u32 %rd249, %r2669, 2; + add.s64 %rd250, %rd18, %rd249; + ld.global.u16 %rs159, [%rd250]; + shr.u16 %rs397, %rs159, 4; + and.b16 %rs160, %rs397, 7; + setp.eq.s16 %p542, %rs160, 0; + mov.u32 %r3719, %r3577; + @%p542 bra $L__BB4_474; + + cvt.u32.u16 %r3708, %rs160; + shr.u16 %rs398, %rs159, 8; + cvt.u32.u16 %r3709, %rs398; + +$L__BB4_469: + mov.u32 %r1194, %r3708; + setp.gt.u32 %p543, %r3508, 2879; + mov.u32 %r3719, 1; + @%p543 bra $L__BB4_474; + + mov.u32 %r2671, 8; + sub.s32 %r2672, %r2671, %r3510; + sub.s32 %r2673, %r2672, %r3509; + min.u32 %r2674, %r2673, %r1194; + setp.eq.s32 %p544, %r2674, 32; + mov.u32 %r2675, -1; + shl.b32 %r2676, %r2675, %r2674; + not.b32 %r2677, %r2676; + selp.b32 %r2678, -1, %r2677, %p544; + and.b32 %r2679, %r2678, %r3709; + shl.b32 %r2680, %r2679, %r3509; + cvt.u16.u32 %rs399, %r2680; + or.b16 %rs532, %rs532, %rs399; + add.s32 %r3509, %r2674, %r3509; + sub.s32 %r3708, %r1194, %r2674; + shr.u32 %r3709, %r3709, %r2674; + setp.gt.u32 %p545, %r2673, %r1194; + @%p545 bra $L__BB4_473; + + setp.ne.s32 %p546, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs400, %rs532, 255; + setp.ne.s16 %p547, %rs400, 127; + and.pred %p548, %p546, %p547; + @%p548 bra $L__BB4_473; + + mov.u32 %r2683, 20548; + sub.s32 %r2684, %r2683, %r3508; + cvt.u64.u32 %rd251, %r2684; + add.s64 %rd252, %rd251, %rd5; + add.s64 %rd253, %rd1, %rd252; + st.global.u8 [%rd253], %rs532; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p549, %rs400, 143; + selp.u32 %r3510, 1, 0, %p549; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_473: + setp.ne.s32 %p550, %r3708, 0; + mov.u32 %r3719, %r3577; + @%p550 bra $L__BB4_469; + +$L__BB4_474: + setp.ne.s32 %p551, %r1187, 0; + @%p551 bra $L__BB4_522; + + setp.eq.s32 %p552, %r3696, 0; + add.s32 %r2685, %r3111, 17477; + cvt.u64.u32 %rd254, %r2685; + add.s64 %rd255, %rd254, %rd5; + add.s64 %rd23, %rd1, %rd255; + @%p552 bra $L__BB4_514; + + shl.b16 %rs474, %rs474, 1; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p553, %r3120, 0; + mov.u32 %r3755, %r3277; + @%p553 bra $L__BB4_479; + bra.uni $L__BB4_477; + +$L__BB4_479: + setp.lt.u32 %p555, %r3275, 3; + mov.u32 %r3723, 0; + @%p555 bra $L__BB4_482; + + setp.lt.u32 %p556, %r3275, 6; + mov.u32 %r3723, 1; + @%p556 bra $L__BB4_482; + + setp.lt.u32 %p557, %r3275, 9; + setp.eq.s32 %p558, %r3275, 11; + selp.b32 %r2691, 4, 5, %p558; + setp.lt.u32 %p559, %r3275, 11; + selp.b32 %r2692, 3, %r2691, %p559; + selp.b32 %r3723, 2, %r2692, %p557; + +$L__BB4_482: + setp.eq.s32 %p560, %r3723, 0; + @%p560 bra $L__BB4_510; + + add.s32 %r1218, %r3723, -1; + and.b32 %r1219, %r3723, 3; + setp.eq.s32 %p561, %r1219, 0; + mov.u32 %r3733, %r3723; + mov.u32 %r3734, %r3755; + @%p561 bra $L__BB4_495; + + mov.u32 %r2694, 1; + shl.b32 %r2695, %r2694, %r1218; + and.b32 %r2696, %r2695, %r3274; + setp.ne.s32 %p562, %r2696, 0; + selp.u32 %r2697, 1, 0, %p562; + cvt.u32.u16 %r2698, %rs474; + bfi.b32 %r2699, %r2698, %r2697, 1, 8; + cvt.u16.u32 %rs474, %r2699; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p563, %r3120, 0; + mov.u32 %r3734, %r3755; + @%p563 bra $L__BB4_487; + + setp.gt.u32 %p564, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3734, %r2694; + @%p564 bra $L__BB4_487; + + add.s32 %r2703, %r3111, 17477; + cvt.u64.u32 %rd256, %r2703; + add.s64 %rd257, %rd256, %rd5; + add.s64 %rd258, %rd1, %rd257; + st.global.u8 [%rd258], %rs474; + add.s32 %r3111, %r3111, 1; + mov.u16 %rs474, 0; + mov.u32 %r3120, 8; + mov.u32 %r3734, %r3755; + +$L__BB4_487: + setp.eq.s32 %p565, %r1219, 1; + mov.u32 %r3755, %r3734; + mov.u32 %r3733, %r1218; + @%p565 bra $L__BB4_495; + + add.s32 %r3733, %r3723, -2; + mov.u32 %r2704, 1; + shl.b32 %r2705, %r2704, %r3733; + and.b32 %r2706, %r2705, %r3274; + setp.ne.s32 %p566, %r2706, 0; + selp.u32 %r2707, 1, 0, %p566; + cvt.u32.u16 %r2708, %rs474; + bfi.b32 %r2709, %r2708, %r2707, 1, 8; + cvt.u16.u32 %rs474, %r2709; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p567, %r3120, 0; + mov.u32 %r3729, %r3734; + @%p567 bra $L__BB4_491; + + setp.gt.u32 %p568, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3729, %r2704; + @%p568 bra $L__BB4_491; + + add.s32 %r2712, %r3111, 17477; + cvt.u64.u32 %rd259, %r2712; + add.s64 %rd260, %rd259, %rd5; + add.s64 %rd261, %rd1, %rd260; + and.b16 %rs407, %rs474, 255; + st.global.u8 [%rd261], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p569, %rs407, 255; + selp.b32 %r3120, 7, 8, %p569; + mov.u16 %rs474, 0; + mov.u32 %r3729, %r3734; + +$L__BB4_491: + setp.eq.s32 %p570, %r1219, 2; + mov.u32 %r3755, %r3729; + mov.u32 %r3734, %r3729; + @%p570 bra $L__BB4_495; + + add.s32 %r3733, %r3723, -3; + mov.u32 %r2713, 1; + shl.b32 %r2714, %r2713, %r3733; + and.b32 %r2715, %r2714, %r3274; + setp.ne.s32 %p571, %r2715, 0; + selp.u32 %r2716, 1, 0, %p571; + cvt.u32.u16 %r2717, %rs474; + bfi.b32 %r2718, %r2717, %r2716, 1, 8; + cvt.u16.u32 %rs474, %r2718; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p572, %r3120, 0; + mov.u32 %r3755, %r3729; + mov.u32 %r3734, %r3729; + @%p572 bra $L__BB4_495; + + setp.gt.u32 %p573, %r3111, 191; + mov.u32 %r3120, 0; + mov.u32 %r3755, %r2713; + mov.u32 %r3734, %r2713; + @%p573 bra $L__BB4_495; + + add.s32 %r2723, %r3111, 17477; + cvt.u64.u32 %rd262, %r2723; + add.s64 %rd263, %rd262, %rd5; + add.s64 %rd264, %rd1, %rd263; + and.b16 %rs410, %rs474, 255; + st.global.u8 [%rd264], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p574, %rs410, 255; + selp.b32 %r3120, 7, 8, %p574; + mov.u16 %rs474, 0; + mov.u32 %r3755, %r3729; + mov.u32 %r3734, %r3729; + +$L__BB4_495: + setp.lt.u32 %p575, %r1218, 3; + @%p575 bra $L__BB4_510; + + mov.u32 %r3755, %r3734; + +$L__BB4_497: + add.s32 %r2724, %r3733, -1; + mov.u32 %r2725, 1; + shl.b32 %r2726, %r2725, %r2724; + and.b32 %r2727, %r2726, %r3274; + setp.ne.s32 %p576, %r2727, 0; + selp.u32 %r2728, 1, 0, %p576; + cvt.u32.u16 %r2729, %rs474; + bfi.b32 %r3743, %r2729, %r2728, 1, 8; + add.s32 %r3742, %r3120, -1; + setp.ne.s32 %p577, %r3742, 0; + mov.u32 %r3744, %r3755; + @%p577 bra $L__BB4_500; + + setp.gt.u32 %p578, %r3111, 191; + mov.u32 %r3742, 0; + mov.u32 %r3744, %r2725; + @%p578 bra $L__BB4_500; + + cvt.u16.u32 %rs411, %r3743; + and.b16 %rs412, %rs411, 255; + add.s32 %r2733, %r3111, 17477; + cvt.u64.u32 %rd265, %r2733; + add.s64 %rd266, %rd265, %rd5; + add.s64 %rd267, %rd1, %rd266; + st.global.u8 [%rd267], %rs411; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p579, %rs412, 255; + selp.b32 %r3742, 7, 8, %p579; + mov.u32 %r3743, 0; + mov.u32 %r3744, %r3755; + +$L__BB4_500: + add.s32 %r2734, %r3733, -2; + shl.b32 %r2736, %r2725, %r2734; + and.b32 %r2737, %r2736, %r3274; + setp.ne.s32 %p580, %r2737, 0; + and.b32 %r2738, %r3743, 127; + selp.u32 %r2739, 1, 0, %p580; + bfi.b32 %r3747, %r2738, %r2739, 1, 7; + add.s32 %r3746, %r3742, -1; + setp.ne.s32 %p581, %r3746, 0; + mov.u32 %r3748, %r3744; + @%p581 bra $L__BB4_503; + + setp.gt.u32 %p582, %r3111, 191; + mov.u32 %r3748, 1; + mov.u32 %r3746, 0; + @%p582 bra $L__BB4_503; + + cvt.u16.u32 %rs413, %r3747; + and.b16 %rs414, %rs413, 255; + add.s32 %r2743, %r3111, 17477; + cvt.u64.u32 %rd268, %r2743; + add.s64 %rd269, %rd268, %rd5; + add.s64 %rd270, %rd1, %rd269; + st.global.u8 [%rd270], %rs413; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p583, %rs414, 255; + selp.b32 %r3746, 7, 8, %p583; + mov.u32 %r3747, 0; + mov.u32 %r3748, %r3744; + +$L__BB4_503: + add.s32 %r2744, %r3733, -3; + mov.u32 %r2745, 1; + shl.b32 %r2746, %r2745, %r2744; + and.b32 %r2747, %r2746, %r3274; + setp.ne.s32 %p584, %r2747, 0; + and.b32 %r2748, %r3747, 127; + selp.u32 %r2749, 1, 0, %p584; + bfi.b32 %r3751, %r2748, %r2749, 1, 7; + add.s32 %r3750, %r3746, -1; + setp.ne.s32 %p585, %r3750, 0; + mov.u32 %r3752, %r3748; + @%p585 bra $L__BB4_506; + + setp.gt.u32 %p586, %r3111, 191; + mov.u32 %r3750, 0; + mov.u32 %r3752, %r2745; + @%p586 bra $L__BB4_506; + + cvt.u16.u32 %rs415, %r3751; + and.b16 %rs416, %rs415, 255; + add.s32 %r2753, %r3111, 17477; + cvt.u64.u32 %rd271, %r2753; + add.s64 %rd272, %rd271, %rd5; + add.s64 %rd273, %rd1, %rd272; + st.global.u8 [%rd273], %rs415; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p587, %rs416, 255; + selp.b32 %r3750, 7, 8, %p587; + mov.u32 %r3751, 0; + mov.u32 %r3752, %r3748; + +$L__BB4_506: + add.s32 %r3733, %r3733, -4; + shl.b32 %r2755, %r2745, %r3733; + and.b32 %r2756, %r2755, %r3274; + setp.ne.s32 %p588, %r2756, 0; + and.b32 %r2757, %r3751, 127; + selp.u32 %r2758, 1, 0, %p588; + bfi.b32 %r2759, %r2757, %r2758, 1, 15; + cvt.u16.u32 %rs474, %r2759; + add.s32 %r3120, %r3750, -1; + setp.ne.s32 %p589, %r3120, 0; + mov.u32 %r3755, %r3752; + @%p589 bra $L__BB4_509; + + setp.gt.u32 %p590, %r3111, 191; + mov.u32 %r3755, 1; + mov.u32 %r3120, 0; + @%p590 bra $L__BB4_509; + + add.s32 %r2762, %r3111, 17477; + cvt.u64.u32 %rd274, %r2762; + add.s64 %rd275, %rd274, %rd5; + add.s64 %rd276, %rd1, %rd275; + and.b16 %rs418, %rs474, 255; + st.global.u8 [%rd276], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p591, %rs418, 255; + selp.b32 %r3120, 7, 8, %p591; + mov.u16 %rs474, 0; + mov.u32 %r3755, %r3752; + +$L__BB4_509: + setp.ne.s32 %p592, %r3733, 0; + @%p592 bra $L__BB4_497; + +$L__BB4_510: + add.s32 %r2764, %r3275, -1; + setp.eq.s32 %p593, %r3275, 0; + mov.u32 %r3274, 0; + selp.b32 %r3275, 0, %r2764, %p593; + setp.lt.u32 %p594, %r3275, 3; + mov.u32 %r3759, %r3274; + @%p594 bra $L__BB4_513; + + setp.lt.u32 %p595, %r3275, 6; + mov.u32 %r3759, 1; + @%p595 bra $L__BB4_513; + + setp.lt.u32 %p596, %r3275, 9; + setp.eq.s32 %p597, %r3275, 11; + selp.b32 %r2766, 4, 5, %p597; + setp.lt.u32 %p598, %r3275, 11; + selp.b32 %r2767, 3, %r2766, %p598; + selp.b32 %r3759, 2, %r2767, %p596; + +$L__BB4_513: + mov.u32 %r2769, 1; + shl.b32 %r3276, %r2769, %r3759; + mov.u32 %r3277, %r3755; + bra.uni $L__BB4_522; + +$L__BB4_514: + add.s32 %r3274, %r3274, 1; + setp.lt.u32 %p599, %r3274, %r3276; + @%p599 bra $L__BB4_522; + + shl.b16 %rs419, %rs474, 1; + or.b16 %rs474, %rs419, 1; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p600, %r3120, 0; + mov.u32 %r3762, %r3277; + @%p600 bra $L__BB4_518; + + setp.gt.u32 %p601, %r3111, 191; + mov.u32 %r3762, 1; + mov.u32 %r3120, 0; + @%p601 bra $L__BB4_518; + + and.b16 %rs421, %rs474, 255; + st.global.u8 [%rd23], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p602, %rs421, 255; + selp.b32 %r3120, 7, 8, %p602; + mov.u16 %rs474, 0; + mov.u32 %r3762, %r3277; + +$L__BB4_518: + add.s32 %r2773, %r3275, 1; + min.u32 %r3275, %r2773, 12; + setp.lt.u32 %p603, %r3275, 3; + mov.u32 %r3274, 0; + mov.u32 %r3763, %r3274; + @%p603 bra $L__BB4_521; + + setp.lt.u32 %p604, %r3275, 6; + mov.u32 %r3763, 1; + @%p604 bra $L__BB4_521; + + setp.lt.u32 %p605, %r3275, 9; + setp.eq.s32 %p606, %r3275, 11; + selp.b32 %r2775, 4, 5, %p606; + setp.lt.u32 %p607, %r3275, 11; + selp.b32 %r2776, 3, %r2775, %p607; + selp.b32 %r3763, 2, %r2776, %p605; + +$L__BB4_521: + mov.u32 %r2778, 1; + shl.b32 %r3276, %r2778, %r3763; + mov.u32 %r3277, %r3762; + +$L__BB4_522: + and.b16 %rs422, %rs159, 15; + cvt.u32.u16 %r1302, %rs422; + and.b32 %r2779, %r3696, 1; + setp.eq.b32 %p608, %r2779, 1; + mov.pred %p609, 0; + xor.pred %p610, %p608, %p609; + not.pred %p611, %p610; + mov.u32 %r3784, %r3687; + @%p611 bra $L__BB4_529; + + and.b32 %r2780, %r1302, 1; + sub.s32 %r3770, %r1188, %r2780; + setp.eq.s32 %p612, %r3770, 0; + mov.u32 %r3784, %r3687; + @%p612 bra $L__BB4_529; + + mov.u32 %r2781, -1; + shl.b32 %r2782, %r2781, %r3770; + not.b32 %r2783, %r2782; + and.b32 %r3771, %r3690, %r2783; + +$L__BB4_525: + setp.gt.u32 %p613, %r3654, 17476; + mov.u32 %r3784, 1; + @%p613 bra $L__BB4_529; + + sub.s32 %r2785, %r3653, %r3655; + min.u32 %r2786, %r2785, %r3770; + setp.eq.s32 %p614, %r2786, 32; + mov.u32 %r2787, -1; + shl.b32 %r2788, %r2787, %r2786; + not.b32 %r2789, %r2788; + selp.b32 %r2790, -1, %r2789, %p614; + and.b32 %r2791, %r2790, %r3771; + shl.b32 %r2792, %r2791, %r3655; + or.b32 %r3656, %r2792, %r3656; + add.s32 %r3655, %r2786, %r3655; + shr.u32 %r3771, %r3771, %r2786; + sub.s32 %r3770, %r3770, %r2786; + setp.lt.u32 %p615, %r3655, %r3653; + @%p615 bra $L__BB4_528; + + cvt.u64.u32 %rd277, %r3654; + add.s64 %rd278, %rd277, %rd5; + add.s64 %rd279, %rd1, %rd278; + st.global.u8 [%rd279], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p616, %r3656, 255; + selp.b32 %r3653, 7, 8, %p616; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_528: + setp.ne.s32 %p617, %r3770, 0; + mov.u32 %r3784, %r3687; + @%p617 bra $L__BB4_525; + +$L__BB4_529: + and.b32 %r1326, %r3696, 2; + setp.eq.s32 %p618, %r1326, 0; + mov.u32 %r3799, %r3784; + @%p618 bra $L__BB4_536; + + shr.u32 %r2795, %r1302, 1; + and.b32 %r2796, %r2795, 1; + sub.s32 %r3785, %r1188, %r2796; + setp.eq.s32 %p619, %r3785, 0; + mov.u32 %r3799, %r3784; + @%p619 bra $L__BB4_536; + + mov.u32 %r2797, -1; + shl.b32 %r2798, %r2797, %r3785; + not.b32 %r2799, %r2798; + and.b32 %r3786, %r3694, %r2799; + +$L__BB4_532: + setp.gt.u32 %p620, %r3654, 17476; + mov.u32 %r3799, 1; + @%p620 bra $L__BB4_536; + + sub.s32 %r2801, %r3653, %r3655; + min.u32 %r2802, %r2801, %r3785; + setp.eq.s32 %p621, %r2802, 32; + mov.u32 %r2803, -1; + shl.b32 %r2804, %r2803, %r2802; + not.b32 %r2805, %r2804; + selp.b32 %r2806, -1, %r2805, %p621; + and.b32 %r2807, %r2806, %r3786; + shl.b32 %r2808, %r2807, %r3655; + or.b32 %r3656, %r2808, %r3656; + add.s32 %r3655, %r2802, %r3655; + shr.u32 %r3786, %r3786, %r2802; + sub.s32 %r3785, %r3785, %r2802; + setp.lt.u32 %p622, %r3655, %r3653; + @%p622 bra $L__BB4_535; + + cvt.u64.u32 %rd280, %r3654; + add.s64 %rd281, %rd280, %rd5; + add.s64 %rd282, %rd1, %rd281; + st.global.u8 [%rd282], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p623, %r3656, 255; + selp.b32 %r3653, 7, 8, %p623; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_535: + setp.ne.s32 %p624, %r3785, 0; + mov.u32 %r3799, %r3784; + @%p624 bra $L__BB4_532; + +$L__BB4_536: + and.b32 %r1350, %r3696, 4; + setp.eq.s32 %p625, %r1350, 0; + mov.u32 %r3814, %r3799; + @%p625 bra $L__BB4_543; + + shr.u32 %r2811, %r1302, 2; + and.b32 %r2812, %r2811, 1; + sub.s32 %r3800, %r1188, %r2812; + setp.eq.s32 %p626, %r3800, 0; + mov.u32 %r3814, %r3799; + @%p626 bra $L__BB4_543; + + mov.u32 %r2813, -1; + shl.b32 %r2814, %r2813, %r3800; + not.b32 %r2815, %r2814; + and.b32 %r3801, %r3699, %r2815; + +$L__BB4_539: + setp.gt.u32 %p627, %r3654, 17476; + mov.u32 %r3814, 1; + @%p627 bra $L__BB4_543; + + sub.s32 %r2817, %r3653, %r3655; + min.u32 %r2818, %r2817, %r3800; + setp.eq.s32 %p628, %r2818, 32; + mov.u32 %r2819, -1; + shl.b32 %r2820, %r2819, %r2818; + not.b32 %r2821, %r2820; + selp.b32 %r2822, -1, %r2821, %p628; + and.b32 %r2823, %r2822, %r3801; + shl.b32 %r2824, %r2823, %r3655; + or.b32 %r3656, %r2824, %r3656; + add.s32 %r3655, %r2818, %r3655; + shr.u32 %r3801, %r3801, %r2818; + sub.s32 %r3800, %r3800, %r2818; + setp.lt.u32 %p629, %r3655, %r3653; + @%p629 bra $L__BB4_542; + + cvt.u64.u32 %rd283, %r3654; + add.s64 %rd284, %rd283, %rd5; + add.s64 %rd285, %rd1, %rd284; + st.global.u8 [%rd285], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p630, %r3656, 255; + selp.b32 %r3653, 7, 8, %p630; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_542: + setp.ne.s32 %p631, %r3800, 0; + mov.u32 %r3814, %r3799; + @%p631 bra $L__BB4_539; + +$L__BB4_543: + and.b32 %r1374, %r3696, 8; + setp.eq.s32 %p632, %r1374, 0; + mov.u32 %r3829, %r3814; + @%p632 bra $L__BB4_550; + + shr.u32 %r2827, %r1302, 3; + sub.s32 %r3815, %r1188, %r2827; + setp.eq.s32 %p633, %r3815, 0; + mov.u32 %r3829, %r3814; + @%p633 bra $L__BB4_550; + + mov.u32 %r2828, -1; + shl.b32 %r2829, %r2828, %r3815; + not.b32 %r2830, %r2829; + and.b32 %r3816, %r3704, %r2830; + +$L__BB4_546: + setp.gt.u32 %p634, %r3654, 17476; + mov.u32 %r3829, 1; + @%p634 bra $L__BB4_550; + + sub.s32 %r2832, %r3653, %r3655; + min.u32 %r2833, %r2832, %r3815; + setp.eq.s32 %p635, %r2833, 32; + mov.u32 %r2834, -1; + shl.b32 %r2835, %r2834, %r2833; + not.b32 %r2836, %r2835; + selp.b32 %r2837, -1, %r2836, %p635; + and.b32 %r2838, %r2837, %r3816; + shl.b32 %r2839, %r2838, %r3655; + or.b32 %r3656, %r2839, %r3656; + add.s32 %r3655, %r2833, %r3655; + shr.u32 %r3816, %r3816, %r2833; + sub.s32 %r3815, %r3815, %r2833; + setp.lt.u32 %p636, %r3655, %r3653; + @%p636 bra $L__BB4_549; + + cvt.u64.u32 %rd286, %r3654; + add.s64 %rd287, %rd286, %rd5; + add.s64 %rd288, %rd1, %rd287; + st.global.u8 [%rd288], %r3656; + add.s32 %r3654, %r3654, 1; + setp.eq.s32 %p637, %r3656, 255; + selp.b32 %r3653, 7, 8, %p637; + mov.u32 %r3655, 0; + mov.u32 %r3656, %r3655; + +$L__BB4_549: + setp.ne.s32 %p638, %r3815, 0; + mov.u32 %r3829, %r3814; + @%p638 bra $L__BB4_546; + +$L__BB4_550: + mov.u32 %r3001, _ZZ55 j2k_htj2k_encode_codeblocks_multi_input_cleanup_64E13cleanup_e_val; + and.b32 %r2842, %r3693, 255; + and.b32 %r2843, %r3561, 255; + setp.lt.u32 %p639, %r2842, %r2843; + cvt.u16.u32 %rs423, %r3693; + selp.b16 %rs424, %rs158, %rs423, %p639; + st.shared.u8 [%r1137+1], %rs424; + ld.shared.u8 %rs537, [%r1137+3]; + setp.gt.u16 %p640, %rs156, %rs537; + add.s32 %r2844, %r878, 3; + selp.b32 %r2845, %r3542, %r2844, %p640; + add.s32 %r2847, %r3001, %r2845; + ld.shared.u8 %rs539, [%r2847]; + cvt.u16.u32 %rs538, %r3703; + shr.u32 %r2848, %r1326, 1; + or.b32 %r2849, %r1141, %r2848; + st.shared.u8 [%r1137+2], %r3703; + st.shared.u8 [%r1139+1], %r2849; + ld.shared.u8 %rs535, [%r1139+3]; + mul.wide.u16 %r2850, %rs535, 4; + add.s32 %r2851, %r2850, %r1140; + shr.u32 %r2852, %r1374, 3; + cvt.u16.u32 %rs536, %r2852; + st.shared.u8 [%r1139+2], %r2852; + shr.u32 %r2853, %r1374, 2; + shr.u32 %r2854, %r1350, 1; + or.b32 %r2855, %r2853, %r2854; + or.b32 %r3543, %r2855, %r2851; + mul.lo.s32 %r2856, %r928, 6; + setp.gt.s32 %p641, %r928, 0; + selp.b32 %r2857, %r2856, 0, %p641; + cvt.u64.u32 %rd289, %r2857; + add.s64 %rd24, %rd17, %rd289; + ld.global.u8 %rs186, [%rd24+1]; + add.s32 %r2858, %r2857, 2; + cvt.u64.u32 %rd290, %r2858; + add.s64 %rd291, %rd17, %rd290; + ld.global.u8 %rs187, [%rd291]; + ld.global.u8 %rs188, [%rd291+1]; + mul.lo.s32 %r2859, %r1189, 6; + setp.gt.s32 %p642, %r1189, 0; + selp.b32 %r2860, %r2859, 0, %p642; + cvt.u64.u32 %rd292, %r2860; + add.s64 %rd293, %rd17, %rd292; + ld.global.u8 %rs189, [%rd293]; + ld.global.u8 %rs190, [%rd293+1]; + add.s32 %r2861, %r2860, 2; + cvt.u64.u32 %rd294, %r2861; + add.s64 %rd295, %rd17, %rd294; + ld.global.u8 %rs191, [%rd295]; + ld.global.u8 %rs192, [%rd295+1]; + setp.eq.s16 %p643, %rs186, 0; + mov.u32 %r3841, %r3719; + @%p643 bra $L__BB4_557; + + ld.global.u8 %r3831, [%rd24]; + cvt.u32.u16 %r3830, %rs186; + +$L__BB4_552: + mov.u32 %r1401, %r3830; + setp.gt.u32 %p644, %r3508, 2879; + mov.u32 %r3841, 1; + @%p644 bra $L__BB4_557; + + mov.u32 %r2863, 8; + sub.s32 %r2864, %r2863, %r3510; + sub.s32 %r2865, %r2864, %r3509; + min.u32 %r2866, %r2865, %r1401; + setp.eq.s32 %p645, %r2866, 32; + mov.u32 %r2867, -1; + shl.b32 %r2868, %r2867, %r2866; + not.b32 %r2869, %r2868; + selp.b32 %r2870, -1, %r2869, %p645; + and.b32 %r2871, %r2870, %r3831; + shl.b32 %r2872, %r2871, %r3509; + cvt.u16.u32 %rs425, %r2872; + or.b16 %rs532, %rs532, %rs425; + add.s32 %r3509, %r2866, %r3509; + sub.s32 %r3830, %r1401, %r2866; + shr.u32 %r3831, %r3831, %r2866; + setp.gt.u32 %p646, %r2865, %r1401; + @%p646 bra $L__BB4_556; + + setp.ne.s32 %p647, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs426, %rs532, 255; + setp.ne.s16 %p648, %rs426, 127; + and.pred %p649, %p647, %p648; + @%p649 bra $L__BB4_556; + + mov.u32 %r2875, 20548; + sub.s32 %r2876, %r2875, %r3508; + cvt.u64.u32 %rd296, %r2876; + add.s64 %rd297, %rd296, %rd5; + add.s64 %rd298, %rd1, %rd297; + st.global.u8 [%rd298], %rs532; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p650, %rs426, 143; + selp.u32 %r3510, 1, 0, %p650; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_556: + setp.ne.s32 %p651, %r3830, 0; + mov.u32 %r3841, %r3719; + @%p651 bra $L__BB4_552; + +$L__BB4_557: + setp.eq.s16 %p652, %rs190, 0; + mov.u32 %r3853, %r3841; + @%p652 bra $L__BB4_564; + + cvt.u32.u16 %r2877, %rs189; + and.b32 %r3843, %r2877, 255; + cvt.u32.u16 %r2878, %rs190; + and.b32 %r3842, %r2878, 255; + +$L__BB4_559: + mov.u32 %r1420, %r3842; + setp.gt.u32 %p653, %r3508, 2879; + mov.u32 %r3853, 1; + @%p653 bra $L__BB4_564; + + mov.u32 %r2880, 8; + sub.s32 %r2881, %r2880, %r3510; + sub.s32 %r2882, %r2881, %r3509; + min.u32 %r2883, %r2882, %r1420; + setp.eq.s32 %p654, %r2883, 32; + mov.u32 %r2884, -1; + shl.b32 %r2885, %r2884, %r2883; + not.b32 %r2886, %r2885; + selp.b32 %r2887, -1, %r2886, %p654; + and.b32 %r2888, %r2887, %r3843; + shl.b32 %r2889, %r2888, %r3509; + cvt.u16.u32 %rs430, %r2889; + or.b16 %rs532, %rs532, %rs430; + add.s32 %r3509, %r2883, %r3509; + sub.s32 %r3842, %r1420, %r2883; + shr.u32 %r3843, %r3843, %r2883; + setp.gt.u32 %p655, %r2882, %r1420; + @%p655 bra $L__BB4_563; + + setp.ne.s32 %p656, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs431, %rs532, 255; + setp.ne.s16 %p657, %rs431, 127; + and.pred %p658, %p656, %p657; + @%p658 bra $L__BB4_563; + + mov.u32 %r2892, 20548; + sub.s32 %r2893, %r2892, %r3508; + cvt.u64.u32 %rd299, %r2893; + add.s64 %rd300, %rd299, %rd5; + add.s64 %rd301, %rd1, %rd300; + st.global.u8 [%rd301], %rs532; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p659, %rs431, 143; + selp.u32 %r3510, 1, 0, %p659; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_563: + setp.ne.s32 %p660, %r3842, 0; + mov.u32 %r3853, %r3841; + @%p660 bra $L__BB4_559; + +$L__BB4_564: + setp.eq.s16 %p661, %rs188, 0; + mov.u32 %r3865, %r3853; + @%p661 bra $L__BB4_571; + + cvt.u32.u16 %r2894, %rs188; + and.b32 %r3854, %r2894, 255; + cvt.u32.u16 %r2895, %rs187; + and.b32 %r3855, %r2895, 255; + +$L__BB4_566: + mov.u32 %r1439, %r3854; + setp.gt.u32 %p662, %r3508, 2879; + mov.u32 %r3865, 1; + @%p662 bra $L__BB4_571; + + mov.u32 %r2897, 8; + sub.s32 %r2898, %r2897, %r3510; + sub.s32 %r2899, %r2898, %r3509; + min.u32 %r2900, %r2899, %r1439; + setp.eq.s32 %p663, %r2900, 32; + mov.u32 %r2901, -1; + shl.b32 %r2902, %r2901, %r2900; + not.b32 %r2903, %r2902; + selp.b32 %r2904, -1, %r2903, %p663; + and.b32 %r2905, %r2904, %r3855; + shl.b32 %r2906, %r2905, %r3509; + cvt.u16.u32 %rs435, %r2906; + or.b16 %rs532, %rs532, %rs435; + add.s32 %r3509, %r2900, %r3509; + sub.s32 %r3854, %r1439, %r2900; + shr.u32 %r3855, %r3855, %r2900; + setp.gt.u32 %p664, %r2899, %r1439; + @%p664 bra $L__BB4_570; + + setp.ne.s32 %p665, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs436, %rs532, 255; + setp.ne.s16 %p666, %rs436, 127; + and.pred %p667, %p665, %p666; + @%p667 bra $L__BB4_570; + + mov.u32 %r2909, 20548; + sub.s32 %r2910, %r2909, %r3508; + cvt.u64.u32 %rd302, %r2910; + add.s64 %rd303, %rd302, %rd5; + add.s64 %rd304, %rd1, %rd303; + st.global.u8 [%rd304], %rs532; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p668, %rs436, 143; + selp.u32 %r3510, 1, 0, %p668; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_570: + setp.ne.s32 %p669, %r3854, 0; + mov.u32 %r3865, %r3853; + @%p669 bra $L__BB4_566; + +$L__BB4_571: + setp.eq.s16 %p670, %rs192, 0; + mov.u32 %r3511, %r3865; + @%p670 bra $L__BB4_578; + + cvt.u32.u16 %r2911, %rs191; + and.b32 %r3867, %r2911, 255; + cvt.u32.u16 %r2912, %rs192; + and.b32 %r3866, %r2912, 255; + +$L__BB4_573: + mov.u32 %r1458, %r3866; + setp.gt.u32 %p671, %r3508, 2879; + mov.u32 %r3511, 1; + @%p671 bra $L__BB4_578; + + mov.u32 %r2914, 8; + sub.s32 %r2915, %r2914, %r3510; + sub.s32 %r2916, %r2915, %r3509; + min.u32 %r2917, %r2916, %r1458; + setp.eq.s32 %p672, %r2917, 32; + mov.u32 %r2918, -1; + shl.b32 %r2919, %r2918, %r2917; + not.b32 %r2920, %r2919; + selp.b32 %r2921, -1, %r2920, %p672; + and.b32 %r2922, %r2921, %r3867; + shl.b32 %r2923, %r2922, %r3509; + cvt.u16.u32 %rs440, %r2923; + or.b16 %rs532, %rs532, %rs440; + add.s32 %r3509, %r2917, %r3509; + sub.s32 %r3866, %r1458, %r2917; + shr.u32 %r3867, %r3867, %r2917; + setp.gt.u32 %p673, %r2916, %r1458; + @%p673 bra $L__BB4_577; + + setp.ne.s32 %p674, %r3510, 0; + mov.u32 %r3510, 0; + and.b16 %rs441, %rs532, 255; + setp.ne.s16 %p675, %rs441, 127; + and.pred %p676, %p674, %p675; + @%p676 bra $L__BB4_577; + + mov.u32 %r2926, 20548; + sub.s32 %r2927, %r2926, %r3508; + cvt.u64.u32 %rd305, %r2927; + add.s64 %rd306, %rd305, %rd5; + add.s64 %rd307, %rd1, %rd306; + st.global.u8 [%rd307], %rs532; + add.s32 %r3508, %r3508, 1; + setp.gt.u16 %p677, %rs441, 143; + selp.u32 %r3510, 1, 0, %p677; + mov.u16 %rs532, 0; + mov.u32 %r3509, 0; + +$L__BB4_577: + setp.ne.s32 %p678, %r3866, 0; + mov.u32 %r3511, %r3865; + @%p678 bra $L__BB4_573; + +$L__BB4_578: + add.s64 %rd338, %rd338, 16; + add.s32 %r3528, %r3528, 4; + setp.lt.u32 %p679, %r3528, 64; + @%p679 bra $L__BB4_348; + + add.s32 %r3512, %r3512, 2; + setp.lt.u32 %p680, %r3512, 64; + add.s64 %rd337, %rd337, 1; + @%p680 bra $L__BB4_347; + + setp.eq.s32 %p681, %r3274, 0; + mov.u32 %r3880, %r3277; + @%p681 bra $L__BB4_584; + + shl.b16 %rs444, %rs474, 1; + or.b16 %rs474, %rs444, 1; + add.s32 %r3120, %r3120, -1; + setp.ne.s32 %p682, %r3120, 0; + mov.u32 %r3880, %r3277; + @%p682 bra $L__BB4_584; + + setp.gt.u32 %p683, %r3111, 191; + mov.u32 %r3880, 1; + mov.u32 %r3120, 0; + @%p683 bra $L__BB4_584; + + add.s32 %r2930, %r3111, 17477; + cvt.u64.u32 %rd308, %r2930; + add.s64 %rd309, %rd308, %rd5; + add.s64 %rd310, %rd1, %rd309; + and.b16 %rs446, %rs474, 255; + st.global.u8 [%rd310], %rs474; + add.s32 %r3111, %r3111, 1; + setp.eq.s16 %p684, %rs446, 255; + selp.b32 %r3120, 7, 8, %p684; + mov.u16 %rs474, 0; + mov.u32 %r3880, %r3277; + +$L__BB4_584: + cvt.u32.u16 %r2931, %rs474; + and.b32 %r2932, %r2931, 255; + shl.b32 %r2933, %r2932, %r3120; + cvt.u16.u32 %rs211, %r2933; + mov.u32 %r2934, -1; + shl.b32 %r2935, %r2934, %r3509; + not.b32 %r2936, %r2935; + mov.u32 %r2937, 255; + and.b32 %r2938, %r2936, 255; + setp.eq.s32 %p685, %r3509, 0; + selp.b32 %r1483, 0, %r2938, %p685; + shl.b32 %r1484, %r2937, %r3120; + and.b32 %r2939, %r1484, 255; + or.b32 %r2940, %r2939, %r1483; + setp.eq.s32 %p686, %r2940, 0; + mov.u32 %r3882, %r3511; + mov.u32 %r3884, %r3880; + @%p686 bra $L__BB4_590; + + or.b16 %rs212, %rs532, %rs211; + and.b16 %rs447, %rs212, 255; + xor.b16 %rs448, %rs212, %rs211; + cvt.u32.u16 %r2941, %rs448; + and.b32 %r2942, %r1484, %r2941; + and.b32 %r2943, %r2942, 255; + xor.b16 %rs449, %rs212, %rs532; + cvt.u32.u16 %r2944, %rs449; + and.b32 %r2945, %r1483, %r2944; + or.b32 %r2946, %r2943, %r2945; + setp.eq.s32 %p687, %r2946, 0; + setp.ne.s16 %p688, %rs447, 255; + and.pred %p689, %p688, %p687; + setp.gt.u32 %p690, %r3508, 1; + and.pred %p691, %p690, %p689; + add.s32 %r2947, %r3111, 17477; + cvt.u64.u32 %rd311, %r2947; + add.s64 %rd312, %rd311, %rd5; + add.s64 %rd27, %rd1, %rd312; + @%p691 bra $L__BB4_588; + bra.uni $L__BB4_586; + +$L__BB4_588: + setp.gt.u32 %p695, %r3111, 191; + mov.u32 %r3884, 1; + mov.u32 %r3882, %r3511; + @%p695 bra $L__BB4_590; + + st.global.u8 [%rd27], %rs212; + add.s32 %r3111, %r3111, 1; + mov.u32 %r3882, %r3511; + mov.u32 %r3884, %r3880; + bra.uni $L__BB4_590; + +$L__BB4_586: + setp.gt.u32 %p692, %r3111, 191; + setp.gt.u32 %p693, %r3508, 2879; + or.pred %p694, %p693, %p692; + mov.u32 %r3882, 1; + mov.u32 %r3884, %r3882; + @%p694 bra $L__BB4_590; + + st.global.u8 [%rd27], %rs211; + add.s32 %r3111, %r3111, 1; + mov.u32 %r2950, 20548; + sub.s32 %r2951, %r2950, %r3508; + cvt.u64.u32 %rd313, %r2951; + add.s64 %rd314, %rd313, %rd5; + add.s64 %rd315, %rd1, %rd314; + st.global.u8 [%rd315], %rs532; + add.s32 %r3508, %r3508, 1; + mov.u32 %r3882, %r3511; + mov.u32 %r3884, %r3880; + +$L__BB4_590: + setp.eq.s32 %p696, %r3655, 0; + @%p696 bra $L__BB4_594; + + sub.s32 %r2953, %r3653, %r3655; + mov.u32 %r2954, -1; + shl.b32 %r2955, %r2954, %r2953; + not.b32 %r2956, %r2955; + and.b32 %r2957, %r2956, 255; + shl.b32 %r2958, %r2957, %r3655; + or.b32 %r1492, %r2958, %r3656; + setp.eq.s32 %p697, %r1492, 255; + mov.u32 %r3886, %r3829; + @%p697 bra $L__BB4_596; + + setp.gt.u32 %p698, %r3654, 17476; + mov.u32 %r3886, 1; + @%p698 bra $L__BB4_596; + + cvt.u64.u32 %rd316, %r3654; + add.s64 %rd317, %rd316, %rd5; + add.s64 %rd318, %rd1, %rd317; + st.global.u8 [%rd318], %r1492; + add.s32 %r3654, %r3654, 1; + mov.u32 %r3886, %r3829; + bra.uni $L__BB4_596; + +$L__BB4_594: + setp.ne.s32 %p699, %r3653, 7; + mov.u32 %r3886, %r3829; + @%p699 bra $L__BB4_596; + + setp.eq.s32 %p700, %r3654, 0; + add.s32 %r2960, %r3654, -1; + selp.b32 %r3654, 0, %r2960, %p700; + mov.u32 %r3886, %r3829; + +$L__BB4_596: + or.b32 %r2961, %r3884, %r3882; + or.b32 %r2962, %r2961, %r3886; + setp.eq.s32 %p701, %r2962, 0; + @%p701 bra $L__BB4_598; + + ld.param.u64 %rd331, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_5]; + cvta.to.global.u64 %rd330, %rd331; + mov.u32 %r3002, %ctaid.x; + mul.wide.u32 %rd320, %r3002, 32; + add.s64 %rd321, %rd330, %rd320; + mov.u32 %r2964, 1; + st.global.u32 [%rd321], %r2964; + mov.u32 %r2965, 3; + st.global.u32 [%rd321+4], %r2965; + mov.u32 %r2966, 0; + st.global.u32 [%rd321+8], %r2966; + st.global.u32 [%rd321+12], %r2966; + st.global.u32 [%rd321+16], %r2966; + st.global.u32 [%rd321+20], %r2966; + st.global.u32 [%rd321+24], %r2966; + st.global.u32 [%rd321+28], %r2966; + bra.uni $L__BB4_605; + +$L__BB4_598: + add.s32 %r2967, %r3111, %r3508; + add.s32 %r1497, %r2967, %r3654; + setp.lt.u32 %p702, %r1497, 2; + setp.gt.u32 %p703, %r1497, %r1506; + or.pred %p704, %p702, %p703; + @%p704 bra $L__BB4_600; + bra.uni $L__BB4_599; + +$L__BB4_600: + ld.param.u64 %rd335, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_5]; + cvta.to.global.u64 %rd334, %rd335; + mov.u32 %r3004, %ctaid.x; + mul.wide.u32 %rd326, %r3004, 32; + add.s64 %rd327, %rd334, %rd326; + mov.u32 %r2976, 1; + st.global.u32 [%rd327], %r2976; + mov.u32 %r2977, 4; + st.global.u32 [%rd327+4], %r2977; + mov.u32 %r2978, 0; + st.global.u32 [%rd327+8], %r2978; + st.global.u32 [%rd327+12], %r2978; + st.global.u32 [%rd327+16], %r2978; + st.global.u32 [%rd327+20], %r2978; + st.global.u32 [%rd327+24], %r2978; + st.global.u32 [%rd327+28], %r2978; + bra.uni $L__BB4_605; + +$L__BB4_599: + ld.param.u64 %rd333, [ j2k_htj2k_encode_codeblocks_multi_input_cleanup_64_param_5]; + cvta.to.global.u64 %rd332, %rd333; + mov.u32 %r3003, %ctaid.x; + and.b32 %r2968, %r3111, 32767; + and.b32 %r2969, %r3508, 32767; + bfi.b32 %r2970, %r2969, %r2968, 15, 15; + or.b32 %r2971, %r2970, -2147483648; + mul.wide.u32 %rd323, %r3003, 32; + add.s64 %rd324, %rd332, %rd323; + mov.u32 %r2973, 0; + st.global.u32 [%rd324], %r2973; + st.global.u32 [%rd324+4], %r2973; + st.global.u32 [%rd324+8], %r1497; + mov.u32 %r2974, 1; + st.global.u32 [%rd324+12], %r2974; + add.s32 %r2989, %r1504, -1; + st.global.u32 [%rd324+16], %r2989; + st.global.u32 [%rd324+20], %r1497; + st.global.u32 [%rd324+24], %r2973; + st.global.u32 [%rd324+28], %r2971; + bra.uni $L__BB4_605; + +} + // .globl j2k_htj2k_compact_codeblocks +.visible .entry j2k_htj2k_compact_codeblocks( + .param .u64 j2k_htj2k_compact_codeblocks_param_0, + .param .u64 j2k_htj2k_compact_codeblocks_param_1, + .param .u64 j2k_htj2k_compact_codeblocks_param_2, + .param .u64 j2k_htj2k_compact_codeblocks_param_3 +) +{ + .reg .pred %p<16>; + .reg .b16 %rs<11>; + .reg .b32 %r<48>; + .reg .b64 %rd<23>; + + + ld.param.u64 %rd5, [ j2k_htj2k_compact_codeblocks_param_0]; + ld.param.u64 %rd6, [ j2k_htj2k_compact_codeblocks_param_1]; + ld.param.u64 %rd4, [ j2k_htj2k_compact_codeblocks_param_2]; + ld.param.u64 %rd7, [ j2k_htj2k_compact_codeblocks_param_3]; + cvta.to.global.u64 %rd1, %rd6; + cvta.to.global.u64 %rd2, %rd5; + mov.u32 %r32, %ctaid.x; + cvt.u64.u32 %rd3, %r32; + setp.ge.u64 %p1, %rd3, %rd7; + @%p1 bra $L__BB5_21; + + cvta.to.global.u64 %rd8, %rd4; + shl.b64 %rd9, %rd3, 4; + add.s64 %rd10, %rd8, %rd9; + ld.global.u32 %r1, [%rd10]; + ld.global.u32 %r2, [%rd10+4]; + ld.global.u32 %r3, [%rd10+8]; + ld.global.u32 %r4, [%rd10+12]; + setp.lt.s32 %p2, %r4, 0; + @%p2 bra $L__BB5_5; + + mov.u32 %r43, %tid.x; + setp.ge.u32 %p3, %r43, %r3; + @%p3 bra $L__BB5_21; + + mov.u32 %r6, %ntid.x; + +$L__BB5_4: + add.s32 %r33, %r43, %r1; + cvt.u64.u32 %rd11, %r33; + add.s64 %rd12, %rd2, %rd11; + ld.global.u8 %rs3, [%rd12]; + add.s32 %r34, %r43, %r2; + cvt.u64.u32 %rd13, %r34; + add.s64 %rd14, %rd1, %rd13; + st.global.u8 [%rd14], %rs3; + add.s32 %r43, %r43, %r6; + setp.lt.u32 %p4, %r43, %r3; + @%p4 bra $L__BB5_4; + bra.uni $L__BB5_21; + +$L__BB5_5: + and.b32 %r9, %r4, 32767; + shr.u32 %r35, %r4, 15; + and.b32 %r10, %r35, 32767; + add.s32 %r11, %r10, %r9; + setp.lt.u32 %p5, %r3, %r11; + @%p5 bra $L__BB5_21; + + sub.s32 %r12, %r3, %r11; + mov.u32 %r44, %tid.x; + setp.ge.u32 %p6, %r44, %r3; + @%p6 bra $L__BB5_21; + + setp.gt.u32 %p7, %r3, 1; + add.s32 %r14, %r12, %r9; + mov.u32 %r15, %ntid.x; + add.s32 %r36, %r1, 17477; + sub.s32 %r16, %r36, %r12; + add.s32 %r37, %r1, 20549; + sub.s32 %r38, %r37, %r9; + sub.s32 %r39, %r38, %r10; + sub.s32 %r17, %r39, %r12; + @%p7 bra $L__BB5_14; + bra.uni $L__BB5_8; + +$L__BB5_14: + cvt.u16.u32 %rs5, %r11; + and.b16 %rs1, %rs5, 15; + add.s32 %r24, %r3, -2; + shr.u32 %r41, %r11, 4; + cvt.u16.u32 %rs2, %r41; + add.s32 %r25, %r3, -1; + +$L__BB5_15: + setp.lt.u32 %p11, %r44, %r12; + @%p11 bra $L__BB5_19; + bra.uni $L__BB5_16; + +$L__BB5_19: + add.s32 %r47, %r44, %r1; + bra.uni $L__BB5_20; + +$L__BB5_16: + setp.lt.u32 %p12, %r44, %r14; + @%p12 bra $L__BB5_18; + bra.uni $L__BB5_17; + +$L__BB5_18: + add.s32 %r47, %r16, %r44; + bra.uni $L__BB5_20; + +$L__BB5_17: + add.s32 %r47, %r17, %r44; + +$L__BB5_20: + cvt.u64.u32 %rd19, %r47; + add.s64 %rd20, %rd2, %rd19; + ld.global.u8 %rs6, [%rd20]; + and.b16 %rs7, %rs6, 240; + or.b16 %rs8, %rs7, %rs1; + setp.eq.s32 %p13, %r44, %r24; + selp.b16 %rs9, %rs8, %rs6, %p13; + setp.eq.s32 %p14, %r44, %r25; + selp.b16 %rs10, %rs2, %rs9, %p14; + add.s32 %r42, %r44, %r2; + cvt.u64.u32 %rd21, %r42; + add.s64 %rd22, %rd1, %rd21; + st.global.u8 [%rd22], %rs10; + add.s32 %r44, %r44, %r15; + setp.lt.u32 %p15, %r44, %r3; + @%p15 bra $L__BB5_15; + bra.uni $L__BB5_21; + +$L__BB5_8: + setp.lt.u32 %p8, %r44, %r12; + @%p8 bra $L__BB5_12; + bra.uni $L__BB5_9; + +$L__BB5_12: + add.s32 %r45, %r44, %r1; + bra.uni $L__BB5_13; + +$L__BB5_9: + setp.lt.u32 %p9, %r44, %r14; + @%p9 bra $L__BB5_11; + bra.uni $L__BB5_10; + +$L__BB5_11: + add.s32 %r45, %r16, %r44; + bra.uni $L__BB5_13; + +$L__BB5_10: + add.s32 %r45, %r17, %r44; + +$L__BB5_13: + cvt.u64.u32 %rd15, %r45; + add.s64 %rd16, %rd2, %rd15; + ld.global.u8 %rs4, [%rd16]; + add.s32 %r40, %r44, %r2; + cvt.u64.u32 %rd17, %r40; + add.s64 %rd18, %rd1, %rd17; + st.global.u8 [%rd18], %rs4; + add.s32 %r44, %r44, %r15; + setp.lt.u32 %p10, %r44, %r3; + @%p10 bra $L__BB5_8; + +$L__BB5_21: + ret; + +} + // .globl j2k_htj2k_packetize_cleanup +.visible .entry j2k_htj2k_packetize_cleanup( + .param .u64 j2k_htj2k_packetize_cleanup_param_0, + .param .u64 j2k_htj2k_packetize_cleanup_param_1, + .param .u64 j2k_htj2k_packetize_cleanup_param_2, + .param .u64 j2k_htj2k_packetize_cleanup_param_3, + .param .u64 j2k_htj2k_packetize_cleanup_param_4, + .param .u64 j2k_htj2k_packetize_cleanup_param_5, + .param .u64 j2k_htj2k_packetize_cleanup_param_6, + .param .u64 j2k_htj2k_packetize_cleanup_param_7, + .param .u64 j2k_htj2k_packetize_cleanup_param_8, + .param .u64 j2k_htj2k_packetize_cleanup_param_9, + .param .u64 j2k_htj2k_packetize_cleanup_param_10, + .param .u64 j2k_htj2k_packetize_cleanup_param_11 +) +{ + .local .align 16 .b8 __local_depot6[49632]; + .reg .b64 %SP; + .reg .b64 %SPL; + .reg .pred %p<731>; + .reg .b16 %rs<137>; + .reg .b32 %r<3496>; + .reg .b64 %rd<394>; + // demoted variable + .shared .align 4 .u32 _ZZ32 j2k_htj2k_packetize_cleanupE11shared_code; + // demoted variable + .shared .align 4 .u32 _ZZ32 j2k_htj2k_packetize_cleanupE17shared_header_len; + // demoted variable + .shared .align 4 .u32 _ZZ32 j2k_htj2k_packetize_cleanupE15shared_body_len; + + mov.u64 %SPL, __local_depot6; + ld.param.u64 %rd57, [ j2k_htj2k_packetize_cleanup_param_1]; + ld.param.u64 %rd58, [ j2k_htj2k_packetize_cleanup_param_2]; + ld.param.u64 %rd59, [ j2k_htj2k_packetize_cleanup_param_3]; + ld.param.u64 %rd60, [ j2k_htj2k_packetize_cleanup_param_4]; + ld.param.u64 %rd62, [ j2k_htj2k_packetize_cleanup_param_6]; + ld.param.u64 %rd63, [ j2k_htj2k_packetize_cleanup_param_7]; + ld.param.u64 %rd65, [ j2k_htj2k_packetize_cleanup_param_9]; + ld.param.u64 %rd67, [ j2k_htj2k_packetize_cleanup_param_11]; + add.u64 %rd1, %SPL, 64; + add.u64 %rd2, %SPL, 24844; + mov.u32 %r1511, %ctaid.x; + cvt.u64.u32 %rd3, %r1511; + setp.ge.u64 %p1, %rd3, %rd67; + @%p1 bra $L__BB6_608; + + cvta.to.global.u64 %rd70, %rd65; + cvta.to.global.u64 %rd71, %rd58; + mul.lo.s64 %rd72, %rd3, 28; + add.s64 %rd73, %rd71, %rd72; + ld.global.u32 %r1, [%rd73+8]; + ld.global.u32 %r2, [%rd73+12]; + ld.global.u32 %r3, [%rd73+20]; + ld.global.u32 %r4, [%rd73+24]; + ld.global.u32 %rd74, [%rd73+16]; + add.s64 %rd4, %rd70, %rd74; + mov.u32 %r1512, %tid.x; + setp.ne.s32 %p2, %r1512, 0; + @%p2 bra $L__BB6_593; + + setp.eq.s32 %p3, %r2, 0; + mov.u32 %r2752, 0; + @%p3 bra $L__BB6_16; + + cvta.to.global.u64 %rd5, %rd59; + cvta.to.global.u64 %rd6, %rd60; + mov.u32 %r1515, 0; + mov.u32 %r2748, %r1515; + mov.u32 %r2752, %r1515; + +$L__BB6_4: + add.s32 %r1521, %r2748, %r1; + mul.wide.u32 %rd75, %r1521, 16; + add.s64 %rd7, %rd5, %rd75; + ld.global.u32 %r1522, [%rd7+8]; + setp.eq.s32 %p4, %r1522, 0; + ld.global.u32 %r1523, [%rd7+12]; + setp.eq.s32 %p5, %r1523, 0; + or.pred %p6, %p4, %p5; + mul.lo.s32 %r1524, %r1523, %r1522; + ld.global.u32 %r7, [%rd7+4]; + setp.ne.s32 %p7, %r1524, %r7; + mov.u32 %r3482, 7; + mov.u32 %r3481, 1; + or.pred %p8, %p6, %p7; + mov.u32 %r3483, %r1515; + mov.u32 %r3484, %r1515; + mov.u32 %r3485, %r1515; + @%p8 bra $L__BB6_592; + + setp.eq.s32 %p9, %r7, 0; + @%p9 bra $L__BB6_15; + + ld.global.u32 %r8, [%rd7]; + mov.u32 %r3483, 0; + mov.u32 %r2750, %r3483; + +$L__BB6_7: + add.s32 %r1531, %r2750, %r8; + mul.wide.u32 %rd76, %r1531, 36; + add.s64 %rd8, %rd6, %rd76; + ld.global.u32 %rd9, [%rd8+4]; + ld.global.u32 %rd10, [%rd8+8]; + ld.global.u32 %rd11, [%rd8+12]; + ld.global.u32 %r11, [%rd8+16]; + setp.eq.s32 %p10, %r11, 0; + selp.b32 %r2752, %r2752, 1, %p10; + setp.gt.u32 %p11, %r11, 164; + mov.u32 %r3482, 1; + mov.u32 %r3481, 2; + mov.u32 %r3484, %r3483; + mov.u32 %r3485, %r3483; + @%p11 bra $L__BB6_592; + + ld.global.u32 %r1537, [%rd8]; + cvt.u64.u32 %rd77, %r1537; + cvt.u32.u64 %r1538, %rd9; + add.s32 %r1539, %r1538, %r1537; + setp.lt.u32 %p12, %r1539, %r1538; + add.s64 %rd78, %rd9, %rd77; + setp.gt.u64 %p13, %rd78, %rd57; + or.pred %p14, %p12, %p13; + mov.u32 %r3482, 2; + mov.u32 %r3481, 1; + mov.u32 %r3484, %r3483; + mov.u32 %r3485, %r3483; + @%p14 bra $L__BB6_592; + + mov.u32 %r3481, 1; + setp.eq.s32 %p725, %r11, 0; + @%p725 bra $L__BB6_13; + + setp.eq.s32 %p16, %r11, 1; + @%p16 bra $L__BB6_12; + bra.uni $L__BB6_11; + +$L__BB6_12: + cvt.u32.u64 %r2718, %rd9; + cvt.u32.u64 %r1553, %rd10; + setp.eq.s32 %p26, %r1553, 0; + selp.b32 %r1555, %r2718, %r1553, %p26; + setp.ne.s32 %p27, %r1555, %r2718; + cvt.u32.u64 %r1556, %rd11; + setp.ne.s32 %p28, %r1556, 0; + or.pred %p29, %p28, %p27; + mov.u32 %r3482, 11; + mov.u32 %r3484, %r3483; + mov.u32 %r3485, %r3483; + @%p29 bra $L__BB6_592; + bra.uni $L__BB6_14; + +$L__BB6_13: + cvt.u32.u64 %r2720, %rd9; + cvt.u32.u64 %r1563, %rd10; + or.b32 %r1564, %r1563, %r2720; + cvt.u32.u64 %r1565, %rd11; + or.b32 %r1566, %r1564, %r1565; + setp.ne.s32 %p30, %r1566, 0; + mov.u32 %r3482, 10; + mov.u32 %r3484, %r3483; + mov.u32 %r3485, %r3483; + @%p30 bra $L__BB6_592; + bra.uni $L__BB6_14; + +$L__BB6_11: + cvt.u32.u64 %r1545, %rd10; + setp.eq.s32 %p17, %r1545, 0; + cvt.u32.u64 %r1546, %rd11; + setp.eq.s32 %p18, %r1546, 0; + or.pred %p19, %p17, %p18; + add.s64 %rd79, %rd11, %rd10; + setp.ne.s64 %p20, %rd79, %rd9; + or.pred %p21, %p19, %p20; + add.s32 %r1547, %r1545, -2; + setp.gt.u32 %p22, %r1547, 65532; + or.pred %p23, %p22, %p21; + setp.gt.u32 %p24, %r1546, 2046; + or.pred %p25, %p24, %p23; + mov.u32 %r3482, 12; + mov.u32 %r3484, %r3483; + mov.u32 %r3485, %r3483; + @%p25 bra $L__BB6_592; + +$L__BB6_14: + add.s32 %r2750, %r2750, 1; + setp.lt.u32 %p31, %r2750, %r7; + @%p31 bra $L__BB6_7; + +$L__BB6_15: + add.s32 %r2748, %r2748, 1; + setp.lt.u32 %p32, %r2748, %r2; + @%p32 bra $L__BB6_4; + +$L__BB6_16: + setp.eq.s32 %p33, %r2752, 0; + @%p33 bra $L__BB6_590; + + setp.eq.s32 %p726, %r2, 0; + mov.u32 %r3419, 0; + mov.u32 %r3457, 1; + mov.u32 %r3458, %r3457; + mov.u32 %r3422, %r3419; + mov.u32 %r3423, %r3419; + @%p726 bra $L__BB6_242; + + ld.param.u64 %rd383, [ j2k_htj2k_packetize_cleanup_param_5]; + ld.param.u64 %rd382, [ j2k_htj2k_packetize_cleanup_param_3]; + mov.u32 %r2754, 0; + mov.u32 %r3458, 1; + cvta.to.global.u64 %rd80, %rd382; + add.s64 %rd13, %rd1, 24576; + add.s64 %rd14, %rd2, 24576; + cvta.to.global.u64 %rd148, %rd383; + mov.u32 %r3423, %r2754; + mov.u32 %r3422, %r2754; + mov.u32 %r3457, %r3458; + mov.u32 %r3419, %r2754; + +$L__BB6_19: + add.s32 %r1578, %r2754, %r1; + mul.wide.u32 %rd81, %r1578, 16; + add.s64 %rd82, %rd80, %rd81; + ld.global.u32 %r23, [%rd82]; + ld.global.u32 %r24, [%rd82+4]; + ld.global.u32 %r25, [%rd82+8]; + setp.eq.s32 %p35, %r25, 0; + ld.global.u32 %r26, [%rd82+12]; + setp.eq.s32 %p36, %r26, 0; + or.pred %p37, %p35, %p36; + @%p37 bra $L__BB6_229; + bra.uni $L__BB6_20; + +$L__BB6_229: + mov.u32 %r1975, 1; + st.local.u32 [%rd13+200], %r1975; + bra.uni $L__BB6_230; + +$L__BB6_20: + mul.lo.s32 %r2766, %r25, %r26; + div.u32 %r1579, %r2766, %r25; + setp.ne.s32 %p38, %r1579, %r26; + @%p38 bra $L__BB6_81; + + setp.gt.u32 %p39, %r2766, 2048; + @%p39 bra $L__BB6_228; + bra.uni $L__BB6_22; + +$L__BB6_228: + mov.u32 %r1974, 1; + st.local.u32 [%rd13+200], %r1974; + bra.uni $L__BB6_230; + +$L__BB6_22: + st.local.u32 [%rd13], %r25; + st.local.u32 [%rd13+64], %r26; + mov.u32 %r1581, 0; + st.local.u32 [%rd13+128], %r1581; + or.b32 %r1582, %r25, %r26; + and.b32 %r28, %r1582, -2; + setp.eq.s32 %p40, %r28, 0; + mov.u32 %r2760, 1; + mov.u32 %r2761, %r2766; + @%p40 bra $L__BB6_85; + + add.s32 %r1583, %r25, 1; + shr.u32 %r29, %r1583, 1; + add.s32 %r1584, %r26, 1; + shr.u32 %r30, %r1584, 1; + mul.lo.s32 %r31, %r29, %r30; + setp.eq.s32 %p41, %r29, 0; + @%p41 bra $L__BB6_25; + + div.u32 %r1585, %r31, %r29; + setp.ne.s32 %p42, %r1585, %r30; + @%p42 bra $L__BB6_81; + +$L__BB6_25: + add.s32 %r2761, %r2766, %r31; + setp.lt.u32 %p43, %r2761, %r2766; + setp.gt.u32 %p44, %r2761, 2048; + or.pred %p45, %p43, %p44; + @%p45 bra $L__BB6_228; + + st.local.u32 [%rd13+4], %r29; + st.local.u32 [%rd13+68], %r30; + st.local.u32 [%rd13+132], %r2766; + or.b32 %r1587, %r29, %r30; + and.b32 %r1588, %r1587, 2147483646; + setp.eq.s32 %p46, %r1588, 0; + mov.u32 %r2760, 2; + @%p46 bra $L__BB6_85; + + add.s32 %r1589, %r29, 1; + shr.u32 %r33, %r1589, 1; + add.s32 %r1590, %r30, 1; + shr.u32 %r34, %r1590, 1; + mul.lo.s32 %r35, %r33, %r34; + setp.eq.s32 %p47, %r33, 0; + @%p47 bra $L__BB6_29; + + div.u32 %r1591, %r35, %r33; + setp.ne.s32 %p48, %r1591, %r34; + @%p48 bra $L__BB6_81; + +$L__BB6_29: + add.s32 %r36, %r2761, %r35; + setp.lt.u32 %p49, %r36, %r2761; + setp.gt.u32 %p50, %r36, 2048; + or.pred %p51, %p49, %p50; + @%p51 bra $L__BB6_228; + + st.local.u32 [%rd13+8], %r33; + st.local.u32 [%rd13+72], %r34; + st.local.u32 [%rd13+136], %r2761; + or.b32 %r1593, %r33, %r34; + and.b32 %r1594, %r1593, 2147483646; + setp.eq.s32 %p52, %r1594, 0; + mov.u32 %r2760, 3; + mov.u32 %r2761, %r36; + @%p52 bra $L__BB6_85; + + add.s32 %r1595, %r33, 1; + shr.u32 %r37, %r1595, 1; + add.s32 %r1596, %r34, 1; + shr.u32 %r38, %r1596, 1; + mul.lo.s32 %r39, %r37, %r38; + setp.eq.s32 %p53, %r37, 0; + @%p53 bra $L__BB6_33; + + div.u32 %r1597, %r39, %r37; + setp.ne.s32 %p54, %r1597, %r38; + @%p54 bra $L__BB6_81; + +$L__BB6_33: + add.s32 %r2761, %r36, %r39; + setp.lt.u32 %p55, %r2761, %r36; + setp.gt.u32 %p56, %r2761, 2048; + or.pred %p57, %p55, %p56; + @%p57 bra $L__BB6_228; + + st.local.u32 [%rd13+12], %r37; + st.local.u32 [%rd13+76], %r38; + st.local.u32 [%rd13+140], %r36; + or.b32 %r1599, %r37, %r38; + and.b32 %r1600, %r1599, 2147483646; + setp.eq.s32 %p58, %r1600, 0; + mov.u32 %r2760, 4; + @%p58 bra $L__BB6_85; + + add.s32 %r1601, %r37, 1; + shr.u32 %r41, %r1601, 1; + add.s32 %r1602, %r38, 1; + shr.u32 %r42, %r1602, 1; + mul.lo.s32 %r43, %r41, %r42; + setp.eq.s32 %p59, %r41, 0; + @%p59 bra $L__BB6_37; + + div.u32 %r1603, %r43, %r41; + setp.ne.s32 %p60, %r1603, %r42; + @%p60 bra $L__BB6_81; + +$L__BB6_37: + add.s32 %r44, %r2761, %r43; + setp.lt.u32 %p61, %r44, %r2761; + setp.gt.u32 %p62, %r44, 2048; + or.pred %p63, %p61, %p62; + @%p63 bra $L__BB6_228; + + st.local.u32 [%rd13+16], %r41; + st.local.u32 [%rd13+80], %r42; + st.local.u32 [%rd13+144], %r2761; + or.b32 %r1605, %r41, %r42; + and.b32 %r1606, %r1605, 2147483646; + setp.eq.s32 %p64, %r1606, 0; + mov.u32 %r2760, 5; + mov.u32 %r2761, %r44; + @%p64 bra $L__BB6_85; + + add.s32 %r1607, %r41, 1; + shr.u32 %r45, %r1607, 1; + add.s32 %r1608, %r42, 1; + shr.u32 %r46, %r1608, 1; + mul.lo.s32 %r47, %r45, %r46; + setp.eq.s32 %p65, %r45, 0; + @%p65 bra $L__BB6_41; + + div.u32 %r1609, %r47, %r45; + setp.ne.s32 %p66, %r1609, %r46; + @%p66 bra $L__BB6_81; + +$L__BB6_41: + add.s32 %r2761, %r44, %r47; + setp.lt.u32 %p67, %r2761, %r44; + setp.gt.u32 %p68, %r2761, 2048; + or.pred %p69, %p67, %p68; + @%p69 bra $L__BB6_228; + + st.local.u32 [%rd13+20], %r45; + st.local.u32 [%rd13+84], %r46; + st.local.u32 [%rd13+148], %r44; + or.b32 %r1611, %r45, %r46; + and.b32 %r1612, %r1611, 2147483646; + setp.eq.s32 %p70, %r1612, 0; + mov.u32 %r2760, 6; + @%p70 bra $L__BB6_85; + + add.s32 %r1613, %r45, 1; + shr.u32 %r49, %r1613, 1; + add.s32 %r1614, %r46, 1; + shr.u32 %r50, %r1614, 1; + mul.lo.s32 %r51, %r49, %r50; + setp.eq.s32 %p71, %r49, 0; + @%p71 bra $L__BB6_45; + + div.u32 %r1615, %r51, %r49; + setp.ne.s32 %p72, %r1615, %r50; + @%p72 bra $L__BB6_81; + +$L__BB6_45: + add.s32 %r52, %r2761, %r51; + setp.lt.u32 %p73, %r52, %r2761; + setp.gt.u32 %p74, %r52, 2048; + or.pred %p75, %p73, %p74; + @%p75 bra $L__BB6_228; + + st.local.u32 [%rd13+24], %r49; + st.local.u32 [%rd13+88], %r50; + st.local.u32 [%rd13+152], %r2761; + or.b32 %r1617, %r49, %r50; + and.b32 %r1618, %r1617, 2147483646; + setp.eq.s32 %p76, %r1618, 0; + mov.u32 %r2760, 7; + mov.u32 %r2761, %r52; + @%p76 bra $L__BB6_85; + + add.s32 %r1619, %r49, 1; + shr.u32 %r53, %r1619, 1; + add.s32 %r1620, %r50, 1; + shr.u32 %r54, %r1620, 1; + mul.lo.s32 %r55, %r53, %r54; + setp.eq.s32 %p77, %r53, 0; + @%p77 bra $L__BB6_49; + + div.u32 %r1621, %r55, %r53; + setp.ne.s32 %p78, %r1621, %r54; + @%p78 bra $L__BB6_81; + +$L__BB6_49: + add.s32 %r2761, %r52, %r55; + setp.lt.u32 %p79, %r2761, %r52; + setp.gt.u32 %p80, %r2761, 2048; + or.pred %p81, %p79, %p80; + @%p81 bra $L__BB6_228; + + st.local.u32 [%rd13+28], %r53; + st.local.u32 [%rd13+92], %r54; + st.local.u32 [%rd13+156], %r52; + or.b32 %r1623, %r53, %r54; + and.b32 %r1624, %r1623, 2147483646; + setp.eq.s32 %p82, %r1624, 0; + mov.u32 %r2760, 8; + @%p82 bra $L__BB6_85; + + add.s32 %r1625, %r53, 1; + shr.u32 %r57, %r1625, 1; + add.s32 %r1626, %r54, 1; + shr.u32 %r58, %r1626, 1; + mul.lo.s32 %r59, %r57, %r58; + setp.eq.s32 %p83, %r57, 0; + @%p83 bra $L__BB6_53; + + div.u32 %r1627, %r59, %r57; + setp.ne.s32 %p84, %r1627, %r58; + @%p84 bra $L__BB6_81; + +$L__BB6_53: + add.s32 %r60, %r2761, %r59; + setp.lt.u32 %p85, %r60, %r2761; + setp.gt.u32 %p86, %r60, 2048; + or.pred %p87, %p85, %p86; + @%p87 bra $L__BB6_228; + + st.local.u32 [%rd13+32], %r57; + st.local.u32 [%rd13+96], %r58; + st.local.u32 [%rd13+160], %r2761; + or.b32 %r1629, %r57, %r58; + and.b32 %r1630, %r1629, 2147483646; + setp.eq.s32 %p88, %r1630, 0; + mov.u32 %r2760, 9; + mov.u32 %r2761, %r60; + @%p88 bra $L__BB6_85; + + add.s32 %r1631, %r57, 1; + shr.u32 %r61, %r1631, 1; + add.s32 %r1632, %r58, 1; + shr.u32 %r62, %r1632, 1; + mul.lo.s32 %r63, %r61, %r62; + setp.eq.s32 %p89, %r61, 0; + @%p89 bra $L__BB6_57; + + div.u32 %r1633, %r63, %r61; + setp.ne.s32 %p90, %r1633, %r62; + @%p90 bra $L__BB6_81; + +$L__BB6_57: + add.s32 %r2761, %r60, %r63; + setp.lt.u32 %p91, %r2761, %r60; + setp.gt.u32 %p92, %r2761, 2048; + or.pred %p93, %p91, %p92; + @%p93 bra $L__BB6_228; + + st.local.u32 [%rd13+36], %r61; + st.local.u32 [%rd13+100], %r62; + st.local.u32 [%rd13+164], %r60; + or.b32 %r1635, %r61, %r62; + and.b32 %r1636, %r1635, 2147483646; + setp.eq.s32 %p94, %r1636, 0; + mov.u32 %r2760, 10; + @%p94 bra $L__BB6_85; + + add.s32 %r1637, %r61, 1; + shr.u32 %r65, %r1637, 1; + add.s32 %r1638, %r62, 1; + shr.u32 %r66, %r1638, 1; + mul.lo.s32 %r67, %r65, %r66; + setp.eq.s32 %p95, %r65, 0; + @%p95 bra $L__BB6_61; + + div.u32 %r1639, %r67, %r65; + setp.ne.s32 %p96, %r1639, %r66; + @%p96 bra $L__BB6_81; + +$L__BB6_61: + add.s32 %r68, %r2761, %r67; + setp.lt.u32 %p97, %r68, %r2761; + setp.gt.u32 %p98, %r68, 2048; + or.pred %p99, %p97, %p98; + @%p99 bra $L__BB6_228; + + st.local.u32 [%rd13+40], %r65; + st.local.u32 [%rd13+104], %r66; + st.local.u32 [%rd13+168], %r2761; + or.b32 %r1641, %r65, %r66; + and.b32 %r1642, %r1641, 2147483646; + setp.eq.s32 %p100, %r1642, 0; + mov.u32 %r2760, 11; + mov.u32 %r2761, %r68; + @%p100 bra $L__BB6_85; + + add.s32 %r1643, %r65, 1; + shr.u32 %r69, %r1643, 1; + add.s32 %r1644, %r66, 1; + shr.u32 %r70, %r1644, 1; + mul.lo.s32 %r71, %r69, %r70; + setp.eq.s32 %p101, %r69, 0; + @%p101 bra $L__BB6_65; + + div.u32 %r1645, %r71, %r69; + setp.ne.s32 %p102, %r1645, %r70; + @%p102 bra $L__BB6_81; + +$L__BB6_65: + add.s32 %r2761, %r68, %r71; + setp.lt.u32 %p103, %r2761, %r68; + setp.gt.u32 %p104, %r2761, 2048; + or.pred %p105, %p103, %p104; + @%p105 bra $L__BB6_228; + + st.local.u32 [%rd13+44], %r69; + st.local.u32 [%rd13+108], %r70; + st.local.u32 [%rd13+172], %r68; + or.b32 %r1647, %r69, %r70; + and.b32 %r1648, %r1647, 2147483646; + setp.eq.s32 %p106, %r1648, 0; + mov.u32 %r2760, 12; + @%p106 bra $L__BB6_85; + + add.s32 %r1649, %r69, 1; + shr.u32 %r73, %r1649, 1; + add.s32 %r1650, %r70, 1; + shr.u32 %r74, %r1650, 1; + mul.lo.s32 %r75, %r73, %r74; + setp.eq.s32 %p107, %r73, 0; + @%p107 bra $L__BB6_69; + + div.u32 %r1651, %r75, %r73; + setp.ne.s32 %p108, %r1651, %r74; + @%p108 bra $L__BB6_81; + +$L__BB6_69: + add.s32 %r76, %r2761, %r75; + setp.lt.u32 %p109, %r76, %r2761; + setp.gt.u32 %p110, %r76, 2048; + or.pred %p111, %p109, %p110; + @%p111 bra $L__BB6_228; + + st.local.u32 [%rd13+48], %r73; + st.local.u32 [%rd13+112], %r74; + st.local.u32 [%rd13+176], %r2761; + or.b32 %r1653, %r73, %r74; + and.b32 %r1654, %r1653, 2147483646; + setp.eq.s32 %p112, %r1654, 0; + mov.u32 %r2760, 13; + mov.u32 %r2761, %r76; + @%p112 bra $L__BB6_85; + + add.s32 %r1655, %r73, 1; + shr.u32 %r77, %r1655, 1; + add.s32 %r1656, %r74, 1; + shr.u32 %r78, %r1656, 1; + mul.lo.s32 %r79, %r77, %r78; + setp.eq.s32 %p113, %r77, 0; + @%p113 bra $L__BB6_73; + + div.u32 %r1657, %r79, %r77; + setp.ne.s32 %p114, %r1657, %r78; + @%p114 bra $L__BB6_81; + +$L__BB6_73: + add.s32 %r2761, %r76, %r79; + setp.lt.u32 %p115, %r2761, %r76; + setp.gt.u32 %p116, %r2761, 2048; + or.pred %p117, %p115, %p116; + @%p117 bra $L__BB6_228; + + st.local.u32 [%rd13+52], %r77; + st.local.u32 [%rd13+116], %r78; + st.local.u32 [%rd13+180], %r76; + or.b32 %r1659, %r77, %r78; + and.b32 %r1660, %r1659, 2147483646; + setp.eq.s32 %p118, %r1660, 0; + mov.u32 %r2760, 14; + @%p118 bra $L__BB6_85; + + add.s32 %r1661, %r77, 1; + shr.u32 %r81, %r1661, 1; + add.s32 %r1662, %r78, 1; + shr.u32 %r82, %r1662, 1; + mul.lo.s32 %r83, %r81, %r82; + setp.eq.s32 %p119, %r81, 0; + @%p119 bra $L__BB6_77; + + div.u32 %r1663, %r83, %r81; + setp.ne.s32 %p120, %r1663, %r82; + @%p120 bra $L__BB6_81; + +$L__BB6_77: + add.s32 %r84, %r2761, %r83; + setp.lt.u32 %p121, %r84, %r2761; + setp.gt.u32 %p122, %r84, 2048; + or.pred %p123, %p121, %p122; + @%p123 bra $L__BB6_228; + + st.local.u32 [%rd13+56], %r81; + st.local.u32 [%rd13+120], %r82; + st.local.u32 [%rd13+184], %r2761; + or.b32 %r1665, %r81, %r82; + and.b32 %r1666, %r1665, 2147483646; + setp.eq.s32 %p124, %r1666, 0; + mov.u32 %r2760, 15; + mov.u32 %r2761, %r84; + @%p124 bra $L__BB6_85; + + add.s32 %r1667, %r81, 1; + shr.u32 %r85, %r1667, 1; + add.s32 %r1668, %r82, 1; + shr.u32 %r86, %r1668, 1; + mul.lo.s32 %r87, %r85, %r86; + setp.eq.s32 %p125, %r85, 0; + @%p125 bra $L__BB6_82; + + div.u32 %r1669, %r87, %r85; + setp.eq.s32 %p126, %r1669, %r86; + @%p126 bra $L__BB6_82; + bra.uni $L__BB6_81; + +$L__BB6_82: + add.s32 %r2761, %r84, %r87; + setp.lt.u32 %p127, %r2761, %r84; + setp.gt.u32 %p128, %r2761, 2048; + or.pred %p129, %p127, %p128; + @%p129 bra $L__BB6_228; + + st.local.u32 [%rd13+60], %r85; + st.local.u32 [%rd13+124], %r86; + st.local.u32 [%rd13+188], %r84; + or.b32 %r1672, %r85, %r86; + and.b32 %r1673, %r1672, 2147483646; + setp.eq.s32 %p130, %r1673, 0; + mov.u32 %r2760, 16; + @%p130 bra $L__BB6_85; + + mov.u32 %r1674, 1; + st.local.u32 [%rd13+200], %r1674; + bra.uni $L__BB6_230; + +$L__BB6_81: + mov.u32 %r1670, 1; + st.local.u32 [%rd13+200], %r1670; + +$L__BB6_230: + mov.u32 %r1976, 1; + st.local.u32 [%rd13+200], %r1976; + st.local.u32 [%rd14+200], %r1976; + +$L__BB6_231: + ld.local.u32 %r1982, [%rd14+200]; + ld.local.u32 %r1983, [%rd13+200]; + or.b32 %r1984, %r1982, %r1983; + setp.ne.s32 %p287, %r1984, 0; + mov.u32 %r3483, 0; + mov.u32 %r3482, 8; + mov.u32 %r3481, 2; + mov.u32 %r3484, %r3483; + mov.u32 %r3485, %r3483; + @%p287 bra $L__BB6_592; + + setp.eq.s32 %p288, %r24, 0; + @%p288 bra $L__BB6_241; + + mov.u32 %r2804, 0; + +$L__BB6_234: + cvta.to.global.u64 %rd377, %rd60; + add.s32 %r1986, %r2804, %r23; + mul.wide.u32 %rd190, %r1986, 36; + add.s64 %rd191, %rd377, %rd190; + add.s64 %rd33, %rd191, 4; + ld.global.u32 %r271, [%rd191+4]; + ld.global.u32 %r272, [%rd191+8]; + ld.global.u32 %r273, [%rd191+12]; + ld.global.u32 %r274, [%rd191+16]; + ld.global.u32 %r3297, [%rd191+24]; + div.u32 %r2937, %r2804, %r25; + mul.lo.s32 %r1987, %r2937, %r25; + sub.s32 %r2938, %r2804, %r1987; + ld.global.u32 %r1988, [%rd191+28]; + setp.eq.s32 %p289, %r1988, 0; + @%p289 bra $L__BB6_263; + + setp.ne.s32 %p290, %r274, 0; + shl.b32 %r3457, %r3457, 1; + cvt.u64.u32 %rd192, %r3419; + add.s64 %rd34, %rd4, %rd192; + @%p290 bra $L__BB6_259; + bra.uni $L__BB6_236; + +$L__BB6_259: + or.b32 %r3457, %r3457, 1; + setp.eq.s32 %p296, %r3422, 0; + selp.b32 %r289, 8, 7, %p296; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p297, %r3458, %r289; + @%p297 bra $L__BB6_373; + + sub.s32 %r3458, %r3458, %r289; + setp.ge.u32 %p298, %r3419, %r3; + mov.u32 %r2815, 1; + @%p298 bra $L__BB6_262; + + shr.u32 %r1996, %r3457, %r3458; + cvt.u16.u32 %rs3, %r1996; + and.b16 %rs4, %rs3, 255; + st.global.u8 [%rd34], %rs3; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p299, %rs4, 255; + selp.u32 %r3422, 1, 0, %p299; + mov.u32 %r2815, %r3423; + +$L__BB6_262: + mov.u32 %r1997, -1; + shl.b32 %r1998, %r1997, %r3458; + not.b32 %r1999, %r1998; + setp.eq.s32 %p300, %r3458, 0; + selp.b32 %r2000, 0, %r1999, %p300; + and.b32 %r3457, %r2000, %r3457; + mov.u32 %r3423, %r2815; + bra.uni $L__BB6_373; + +$L__BB6_263: + setp.ne.s32 %p301, %r274, 0; + ld.global.u32 %r2006, [%rd33+28]; + setp.ne.s32 %p302, %r2006, %r4; + and.pred %p303, %p301, %p302; + mov.u32 %r3483, 0; + mov.u32 %r3482, 9; + mov.u32 %r3481, 1; + mov.u32 %r3484, %r3483; + mov.u32 %r3485, %r3483; + @%p303 bra $L__BB6_592; + + ld.global.u32 %r298, [%rd33+16]; + ld.local.u32 %r2823, [%rd13+192]; + setp.eq.s32 %p304, %r2823, 0; + @%p304 bra $L__BB6_272; + + add.s32 %r2008, %r2823, -1; + setp.lt.u32 %p305, %r2008, 3; + mov.u32 %r2820, 0; + mov.u32 %r2821, %r2937; + mov.u32 %r2822, %r2938; + @%p305 bra $L__BB6_268; + + and.b32 %r2691, %r2823, 3; + sub.s32 %r2819, %r2823, %r2691; + mov.u32 %r2820, 0; + mov.u32 %r2821, %r2937; + mov.u32 %r2822, %r2938; + +$L__BB6_267: + add.u64 %rd375, %SPL, 0; + mul.wide.u32 %rd194, %r2820, 4; + add.s64 %rd195, %rd1, %rd194; + ld.local.u32 %r2010, [%rd195+24576]; + ld.local.u32 %r2011, [%rd195+24704]; + add.s32 %r2012, %r2011, %r2822; + mad.lo.s32 %r2013, %r2010, %r2821, %r2012; + add.s64 %rd196, %rd375, %rd194; + st.local.u32 [%rd196], %r2013; + ld.local.u32 %r2014, [%rd195+24580]; + shr.u32 %r2015, %r2821, 1; + ld.local.u32 %r2016, [%rd195+24708]; + shr.u32 %r2017, %r2822, 1; + add.s32 %r2018, %r2016, %r2017; + mad.lo.s32 %r2019, %r2014, %r2015, %r2018; + st.local.u32 [%rd196+4], %r2019; + ld.local.u32 %r2020, [%rd195+24584]; + shr.u32 %r2021, %r2821, 2; + ld.local.u32 %r2022, [%rd195+24712]; + shr.u32 %r2023, %r2822, 2; + add.s32 %r2024, %r2022, %r2023; + mad.lo.s32 %r2025, %r2020, %r2021, %r2024; + st.local.u32 [%rd196+8], %r2025; + ld.local.u32 %r2026, [%rd195+24588]; + shr.u32 %r2027, %r2821, 3; + ld.local.u32 %r2028, [%rd195+24716]; + shr.u32 %r2029, %r2822, 3; + add.s32 %r2030, %r2028, %r2029; + mad.lo.s32 %r2031, %r2026, %r2027, %r2030; + st.local.u32 [%rd196+12], %r2031; + shr.u32 %r2822, %r2822, 4; + shr.u32 %r2821, %r2821, 4; + add.s32 %r2820, %r2820, 4; + add.s32 %r2819, %r2819, -4; + setp.ne.s32 %p306, %r2819, 0; + @%p306 bra $L__BB6_267; + +$L__BB6_268: + and.b32 %r2688, %r2823, 3; + setp.eq.s32 %p307, %r2688, 0; + @%p307 bra $L__BB6_272; + + and.b32 %r2689, %r2823, 3; + mul.wide.u32 %rd197, %r2820, 4; + add.s64 %rd198, %rd1, %rd197; + add.s64 %rd36, %rd198, 24704; + ld.local.u32 %r2032, [%rd198+24576]; + ld.local.u32 %r2033, [%rd198+24704]; + add.s32 %r2034, %r2033, %r2822; + mad.lo.s32 %r2035, %r2032, %r2821, %r2034; + add.u64 %rd200, %SPL, 0; + add.s64 %rd37, %rd200, %rd197; + st.local.u32 [%rd37], %r2035; + setp.eq.s32 %p308, %r2689, 1; + @%p308 bra $L__BB6_272; + + and.b32 %r2690, %r2823, 3; + shr.u32 %r2036, %r2821, 1; + ld.local.u32 %r2037, [%rd36+-124]; + ld.local.u32 %r2038, [%rd36+4]; + shr.u32 %r2039, %r2822, 1; + add.s32 %r2040, %r2038, %r2039; + mad.lo.s32 %r2041, %r2037, %r2036, %r2040; + st.local.u32 [%rd37+4], %r2041; + setp.eq.s32 %p309, %r2690, 2; + @%p309 bra $L__BB6_272; + + shr.u32 %r2042, %r2821, 2; + ld.local.u32 %r2043, [%rd36+-120]; + ld.local.u32 %r2044, [%rd36+8]; + shr.u32 %r2045, %r2822, 2; + add.s32 %r2046, %r2044, %r2045; + mad.lo.s32 %r2047, %r2043, %r2042, %r2046; + st.local.u32 [%rd37+8], %r2047; + +$L__BB6_272: + @%p304 bra $L__BB6_318; + + mov.u32 %r2824, 0; + +$L__BB6_274: + add.u64 %rd378, %SPL, 0; + add.s32 %r2823, %r2823, -1; + mul.wide.u32 %rd202, %r2823, 4; + add.s64 %rd203, %rd378, %rd202; + ld.local.u32 %r2049, [%rd203]; + mul.wide.u32 %rd204, %r2049, 4; + add.s64 %rd205, %rd1, %rd204; + add.s64 %rd39, %rd205, 8192; + ld.local.u32 %r2921, [%rd205+8192]; + max.u32 %r322, %r2921, %r2824; + ld.local.u32 %r2050, [%rd205+16384]; + setp.ne.s32 %p311, %r2050, 0; + @%p311 bra $L__BB6_317; + + add.s32 %r2051, %r4, 1; + ld.local.u32 %r323, [%rd39+-8192]; + min.u32 %r324, %r323, %r2051; + setp.le.u32 %p312, %r324, %r322; + mov.u32 %r2869, %r3457; + @%p312 bra $L__BB6_310; + + add.s32 %r2695, %r4, 1; + min.u32 %r2694, %r323, %r2695; + sub.s32 %r2053, %r2694, %r322; + and.b32 %r325, %r2053, 3; + setp.eq.s32 %p313, %r325, 0; + mov.u32 %r2854, %r322; + mov.u32 %r2869, %r3457; + @%p313 bra $L__BB6_292; + + shl.b32 %r2869, %r3457, 1; + setp.eq.s32 %p314, %r3422, 0; + selp.b32 %r327, 8, 7, %p314; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p315, %r3458, %r327; + @%p315 bra $L__BB6_281; + + sub.s32 %r3458, %r3458, %r327; + setp.ge.u32 %p316, %r3419, %r3; + mov.u32 %r2832, 1; + @%p316 bra $L__BB6_280; + + shl.b32 %r2701, %r3457, 1; + shr.u32 %r2055, %r2701, %r3458; + cvt.u16.u32 %rs5, %r2055; + and.b16 %rs6, %rs5, 255; + cvt.u64.u32 %rd206, %r3419; + add.s64 %rd207, %rd4, %rd206; + st.global.u8 [%rd207], %rs5; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p317, %rs6, 255; + selp.u32 %r3422, 1, 0, %p317; + mov.u32 %r2832, %r3423; + +$L__BB6_280: + shl.b32 %r2700, %r3457, 1; + mov.u32 %r2056, -1; + shl.b32 %r2057, %r2056, %r3458; + xor.b32 %r2058, %r2057, -2; + setp.eq.s32 %p318, %r3458, 0; + selp.b32 %r2059, 0, %r2058, %p318; + and.b32 %r2869, %r2059, %r2700; + mov.u32 %r3423, %r2832; + +$L__BB6_281: + add.s32 %r2705, %r4, 1; + min.u32 %r2704, %r323, %r2705; + sub.s32 %r2703, %r2704, %r322; + and.b32 %r2702, %r2703, 3; + add.s32 %r2854, %r322, 1; + setp.eq.s32 %p319, %r2702, 1; + @%p319 bra $L__BB6_292; + + shl.b32 %r2842, %r2869, 1; + setp.eq.s32 %p320, %r3422, 0; + selp.b32 %r343, 8, 7, %p320; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p321, %r3458, %r343; + @%p321 bra $L__BB6_286; + + sub.s32 %r3458, %r3458, %r343; + setp.ge.u32 %p322, %r3419, %r3; + mov.u32 %r2840, 1; + @%p322 bra $L__BB6_285; + + shl.b32 %r2711, %r2869, 1; + shr.u32 %r2061, %r2711, %r3458; + cvt.u16.u32 %rs7, %r2061; + and.b16 %rs8, %rs7, 255; + cvt.u64.u32 %rd208, %r3419; + add.s64 %rd209, %rd4, %rd208; + st.global.u8 [%rd209], %rs7; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p323, %rs8, 255; + selp.u32 %r3422, 1, 0, %p323; + mov.u32 %r2840, %r3423; + +$L__BB6_285: + shl.b32 %r2710, %r2869, 1; + mov.u32 %r2062, -1; + shl.b32 %r2063, %r2062, %r3458; + xor.b32 %r2064, %r2063, -2; + setp.eq.s32 %p324, %r3458, 0; + selp.b32 %r2065, 0, %r2064, %p324; + and.b32 %r2842, %r2065, %r2710; + mov.u32 %r3423, %r2840; + +$L__BB6_286: + add.s32 %r2709, %r4, 1; + min.u32 %r2708, %r323, %r2709; + sub.s32 %r2707, %r2708, %r322; + and.b32 %r2706, %r2707, 3; + add.s32 %r2854, %r322, 2; + setp.eq.s32 %p325, %r2706, 2; + mov.u32 %r2869, %r2842; + @%p325 bra $L__BB6_292; + + shl.b32 %r2869, %r2842, 1; + setp.eq.s32 %p326, %r3422, 0; + selp.b32 %r359, 8, 7, %p326; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p327, %r3458, %r359; + @%p327 bra $L__BB6_291; + + sub.s32 %r3458, %r3458, %r359; + setp.ge.u32 %p328, %r3419, %r3; + mov.u32 %r2848, 1; + @%p328 bra $L__BB6_290; + + shl.b32 %r2713, %r2842, 1; + shr.u32 %r2067, %r2713, %r3458; + cvt.u16.u32 %rs9, %r2067; + and.b16 %rs10, %rs9, 255; + cvt.u64.u32 %rd210, %r3419; + add.s64 %rd211, %rd4, %rd210; + st.global.u8 [%rd211], %rs9; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p329, %rs10, 255; + selp.u32 %r3422, 1, 0, %p329; + mov.u32 %r2848, %r3423; + +$L__BB6_290: + shl.b32 %r2712, %r2842, 1; + mov.u32 %r2068, -1; + shl.b32 %r2069, %r2068, %r3458; + xor.b32 %r2070, %r2069, -2; + setp.eq.s32 %p330, %r3458, 0; + selp.b32 %r2071, 0, %r2070, %p330; + and.b32 %r2869, %r2071, %r2712; + mov.u32 %r3423, %r2848; + +$L__BB6_291: + add.s32 %r2854, %r322, 3; + +$L__BB6_292: + add.s32 %r2697, %r4, 1; + min.u32 %r2696, %r323, %r2697; + not.b32 %r2072, %r322; + add.s32 %r2073, %r2696, %r2072; + setp.lt.u32 %p331, %r2073, 3; + @%p331 bra $L__BB6_310; + +$L__BB6_293: + shl.b32 %r2875, %r2869, 1; + setp.eq.s32 %p332, %r3422, 0; + selp.b32 %r392, 8, 7, %p332; + add.s32 %r2876, %r3458, 1; + setp.lt.u32 %p333, %r2876, %r392; + @%p333 bra $L__BB6_297; + + sub.s32 %r2876, %r2876, %r392; + setp.ge.u32 %p334, %r3419, %r3; + mov.u32 %r2873, 1; + @%p334 bra $L__BB6_296; + + shr.u32 %r2075, %r2875, %r2876; + cvt.u16.u32 %rs11, %r2075; + and.b16 %rs12, %rs11, 255; + cvt.u64.u32 %rd212, %r3419; + add.s64 %rd213, %rd4, %rd212; + st.global.u8 [%rd213], %rs11; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p335, %rs12, 255; + selp.u32 %r3422, 1, 0, %p335; + mov.u32 %r2873, %r3423; + +$L__BB6_296: + mov.u32 %r2076, -1; + shl.b32 %r2077, %r2076, %r2876; + xor.b32 %r2078, %r2077, -2; + setp.eq.s32 %p336, %r2876, 0; + selp.b32 %r2079, 0, %r2078, %p336; + and.b32 %r2875, %r2079, %r2875; + mov.u32 %r3423, %r2873; + +$L__BB6_297: + shl.b32 %r2883, %r2875, 1; + setp.eq.s32 %p337, %r3422, 0; + selp.b32 %r407, 8, 7, %p337; + add.s32 %r2884, %r2876, 1; + setp.lt.u32 %p338, %r2884, %r407; + @%p338 bra $L__BB6_301; + + sub.s32 %r2884, %r2884, %r407; + setp.ge.u32 %p339, %r3419, %r3; + mov.u32 %r2881, 1; + @%p339 bra $L__BB6_300; + + shr.u32 %r2081, %r2883, %r2884; + cvt.u16.u32 %rs13, %r2081; + and.b16 %rs14, %rs13, 255; + cvt.u64.u32 %rd214, %r3419; + add.s64 %rd215, %rd4, %rd214; + st.global.u8 [%rd215], %rs13; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p340, %rs14, 255; + selp.u32 %r3422, 1, 0, %p340; + mov.u32 %r2881, %r3423; + +$L__BB6_300: + mov.u32 %r2082, -1; + shl.b32 %r2083, %r2082, %r2884; + xor.b32 %r2084, %r2083, -2; + setp.eq.s32 %p341, %r2884, 0; + selp.b32 %r2085, 0, %r2084, %p341; + and.b32 %r2883, %r2085, %r2883; + mov.u32 %r3423, %r2881; + +$L__BB6_301: + shl.b32 %r2891, %r2883, 1; + setp.eq.s32 %p342, %r3422, 0; + selp.b32 %r422, 8, 7, %p342; + add.s32 %r2892, %r2884, 1; + setp.lt.u32 %p343, %r2892, %r422; + @%p343 bra $L__BB6_305; + + sub.s32 %r2892, %r2892, %r422; + setp.ge.u32 %p344, %r3419, %r3; + mov.u32 %r2889, 1; + @%p344 bra $L__BB6_304; + + shr.u32 %r2087, %r2891, %r2892; + cvt.u16.u32 %rs15, %r2087; + and.b16 %rs16, %rs15, 255; + cvt.u64.u32 %rd216, %r3419; + add.s64 %rd217, %rd4, %rd216; + st.global.u8 [%rd217], %rs15; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p345, %rs16, 255; + selp.u32 %r3422, 1, 0, %p345; + mov.u32 %r2889, %r3423; + +$L__BB6_304: + mov.u32 %r2088, -1; + shl.b32 %r2089, %r2088, %r2892; + xor.b32 %r2090, %r2089, -2; + setp.eq.s32 %p346, %r2892, 0; + selp.b32 %r2091, 0, %r2090, %p346; + and.b32 %r2891, %r2091, %r2891; + mov.u32 %r3423, %r2889; + +$L__BB6_305: + shl.b32 %r2869, %r2891, 1; + setp.eq.s32 %p347, %r3422, 0; + selp.b32 %r437, 8, 7, %p347; + add.s32 %r3458, %r2892, 1; + setp.lt.u32 %p348, %r3458, %r437; + @%p348 bra $L__BB6_309; + + sub.s32 %r3458, %r3458, %r437; + setp.ge.u32 %p349, %r3419, %r3; + mov.u32 %r2897, 1; + @%p349 bra $L__BB6_308; + + shr.u32 %r2093, %r2869, %r3458; + cvt.u16.u32 %rs17, %r2093; + and.b16 %rs18, %rs17, 255; + cvt.u64.u32 %rd218, %r3419; + add.s64 %rd219, %rd4, %rd218; + st.global.u8 [%rd219], %rs17; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p350, %rs18, 255; + selp.u32 %r3422, 1, 0, %p350; + mov.u32 %r2897, %r3423; + +$L__BB6_308: + mov.u32 %r2094, -1; + shl.b32 %r2095, %r2094, %r3458; + xor.b32 %r2096, %r2095, -2; + setp.eq.s32 %p351, %r3458, 0; + selp.b32 %r2097, 0, %r2096, %p351; + and.b32 %r2869, %r2097, %r2869; + mov.u32 %r3423, %r2897; + +$L__BB6_309: + add.s32 %r2699, %r4, 1; + min.u32 %r2698, %r323, %r2699; + add.s32 %r2854, %r2854, 4; + setp.lt.u32 %p352, %r2854, %r2698; + @%p352 bra $L__BB6_293; + +$L__BB6_310: + setp.ge.u32 %p353, %r323, %r2051; + @%p353 bra $L__BB6_316; + + shl.b32 %r2099, %r2869, 1; + or.b32 %r2912, %r2099, 1; + setp.eq.s32 %p354, %r3422, 0; + selp.b32 %r458, 8, 7, %p354; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p355, %r3458, %r458; + @%p355 bra $L__BB6_315; + + sub.s32 %r3458, %r3458, %r458; + setp.ge.u32 %p356, %r3419, %r3; + mov.u32 %r2910, 1; + @%p356 bra $L__BB6_314; + + shr.u32 %r2101, %r2912, %r3458; + cvt.u16.u32 %rs19, %r2101; + and.b16 %rs20, %rs19, 255; + cvt.u64.u32 %rd220, %r3419; + add.s64 %rd221, %rd4, %rd220; + st.global.u8 [%rd221], %rs19; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p357, %rs20, 255; + selp.u32 %r3422, 1, 0, %p357; + mov.u32 %r2910, %r3423; + +$L__BB6_314: + shl.b32 %r2728, %r2869, 1; + or.b32 %r2727, %r2728, 1; + mov.u32 %r2102, -1; + shl.b32 %r2103, %r2102, %r3458; + not.b32 %r2104, %r2103; + setp.eq.s32 %p358, %r3458, 0; + selp.b32 %r2105, 0, %r2104, %p358; + and.b32 %r2912, %r2105, %r2727; + mov.u32 %r3423, %r2910; + +$L__BB6_315: + mov.u32 %r2106, 1; + st.local.u32 [%rd39+8192], %r2106; + mov.u32 %r2869, %r2912; + +$L__BB6_316: + add.s32 %r2693, %r4, 1; + min.u32 %r2921, %r323, %r2693; + st.local.u32 [%rd39], %r2921; + mov.u32 %r3457, %r2869; + +$L__BB6_317: + setp.ne.s32 %p359, %r2823, 0; + mov.u32 %r2824, %r2921; + @%p359 bra $L__BB6_274; + +$L__BB6_318: + setp.eq.s32 %p360, %r274, 0; + @%p360 bra $L__BB6_240; + + ld.local.u32 %r2939, [%rd14+192]; + setp.eq.s32 %p361, %r2939, 0; + @%p361 bra $L__BB6_327; + + add.s32 %r2108, %r2939, -1; + and.b32 %r490, %r2939, 3; + setp.lt.u32 %p362, %r2108, 3; + mov.u32 %r2936, 0; + @%p362 bra $L__BB6_323; + + sub.s32 %r2935, %r2939, %r490; + mov.u32 %r2936, 0; + +$L__BB6_322: + add.u64 %rd380, %SPL, 0; + mul.wide.u32 %rd223, %r2936, 4; + add.s64 %rd224, %rd2, %rd223; + ld.local.u32 %r2110, [%rd224+24576]; + ld.local.u32 %r2111, [%rd224+24704]; + add.s32 %r2112, %r2111, %r2938; + mad.lo.s32 %r2113, %r2110, %r2937, %r2112; + add.s64 %rd225, %rd380, %rd223; + st.local.u32 [%rd225], %r2113; + ld.local.u32 %r2114, [%rd224+24580]; + shr.u32 %r2115, %r2937, 1; + ld.local.u32 %r2116, [%rd224+24708]; + shr.u32 %r2117, %r2938, 1; + add.s32 %r2118, %r2116, %r2117; + mad.lo.s32 %r2119, %r2114, %r2115, %r2118; + st.local.u32 [%rd225+4], %r2119; + ld.local.u32 %r2120, [%rd224+24584]; + shr.u32 %r2121, %r2937, 2; + ld.local.u32 %r2122, [%rd224+24712]; + shr.u32 %r2123, %r2938, 2; + add.s32 %r2124, %r2122, %r2123; + mad.lo.s32 %r2125, %r2120, %r2121, %r2124; + st.local.u32 [%rd225+8], %r2125; + ld.local.u32 %r2126, [%rd224+24588]; + shr.u32 %r2127, %r2937, 3; + ld.local.u32 %r2128, [%rd224+24716]; + shr.u32 %r2129, %r2938, 3; + add.s32 %r2130, %r2128, %r2129; + mad.lo.s32 %r2131, %r2126, %r2127, %r2130; + st.local.u32 [%rd225+12], %r2131; + shr.u32 %r2938, %r2938, 4; + shr.u32 %r2937, %r2937, 4; + add.s32 %r2936, %r2936, 4; + add.s32 %r2935, %r2935, -4; + setp.ne.s32 %p363, %r2935, 0; + @%p363 bra $L__BB6_322; + +$L__BB6_323: + and.b32 %r2714, %r2939, 3; + setp.eq.s32 %p364, %r2714, 0; + @%p364 bra $L__BB6_327; + + and.b32 %r2715, %r2939, 3; + mul.wide.u32 %rd226, %r2936, 4; + add.s64 %rd227, %rd2, %rd226; + add.s64 %rd41, %rd227, 24704; + ld.local.u32 %r2132, [%rd227+24576]; + ld.local.u32 %r2133, [%rd227+24704]; + add.s32 %r2134, %r2133, %r2938; + mad.lo.s32 %r2135, %r2132, %r2937, %r2134; + add.u64 %rd229, %SPL, 0; + add.s64 %rd42, %rd229, %rd226; + st.local.u32 [%rd42], %r2135; + setp.eq.s32 %p365, %r2715, 1; + @%p365 bra $L__BB6_327; + + and.b32 %r2716, %r2939, 3; + shr.u32 %r2136, %r2937, 1; + ld.local.u32 %r2137, [%rd41+-124]; + ld.local.u32 %r2138, [%rd41+4]; + shr.u32 %r2139, %r2938, 1; + add.s32 %r2140, %r2138, %r2139; + mad.lo.s32 %r2141, %r2137, %r2136, %r2140; + st.local.u32 [%rd42+4], %r2141; + setp.eq.s32 %p366, %r2716, 2; + @%p366 bra $L__BB6_327; + + shr.u32 %r2142, %r2937, 2; + ld.local.u32 %r2143, [%rd41+-120]; + ld.local.u32 %r2144, [%rd41+8]; + shr.u32 %r2145, %r2938, 2; + add.s32 %r2146, %r2144, %r2145; + mad.lo.s32 %r2147, %r2143, %r2142, %r2146; + st.local.u32 [%rd42+8], %r2147; + +$L__BB6_327: + @%p361 bra $L__BB6_373; + + mov.u32 %r2940, 0; + +$L__BB6_329: + add.u64 %rd388, %SPL, 0; + add.s32 %r2939, %r2939, -1; + mul.wide.u32 %rd231, %r2939, 4; + add.s64 %rd232, %rd388, %rd231; + ld.local.u32 %r2149, [%rd232]; + mul.wide.u32 %rd233, %r2149, 4; + add.s64 %rd234, %rd2, %rd233; + add.s64 %rd44, %rd234, 8192; + ld.local.u32 %r3037, [%rd234+8192]; + max.u32 %r512, %r3037, %r2940; + ld.local.u32 %r2150, [%rd234+16384]; + setp.ne.s32 %p368, %r2150, 0; + @%p368 bra $L__BB6_372; + + add.s32 %r2732, %r298, 1; + ld.local.u32 %r513, [%rd44+-8192]; + min.u32 %r514, %r513, %r2732; + setp.le.u32 %p369, %r514, %r512; + mov.u32 %r2985, %r3457; + @%p369 bra $L__BB6_365; + + min.u32 %r2733, %r513, %r2732; + sub.s32 %r2152, %r2733, %r512; + and.b32 %r515, %r2152, 3; + setp.eq.s32 %p370, %r515, 0; + mov.u32 %r2970, %r512; + mov.u32 %r2985, %r3457; + @%p370 bra $L__BB6_347; + + shl.b32 %r2985, %r3457, 1; + setp.eq.s32 %p371, %r3422, 0; + selp.b32 %r517, 8, 7, %p371; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p372, %r3458, %r517; + @%p372 bra $L__BB6_336; + + sub.s32 %r3458, %r3458, %r517; + setp.ge.u32 %p373, %r3419, %r3; + mov.u32 %r2948, 1; + @%p373 bra $L__BB6_335; + + shl.b32 %r2737, %r3457, 1; + shr.u32 %r2154, %r2737, %r3458; + cvt.u16.u32 %rs21, %r2154; + and.b16 %rs22, %rs21, 255; + cvt.u64.u32 %rd235, %r3419; + add.s64 %rd236, %rd4, %rd235; + st.global.u8 [%rd236], %rs21; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p374, %rs22, 255; + selp.u32 %r3422, 1, 0, %p374; + mov.u32 %r2948, %r3423; + +$L__BB6_335: + shl.b32 %r2736, %r3457, 1; + mov.u32 %r2155, -1; + shl.b32 %r2156, %r2155, %r3458; + xor.b32 %r2157, %r2156, -2; + setp.eq.s32 %p375, %r3458, 0; + selp.b32 %r2158, 0, %r2157, %p375; + and.b32 %r2985, %r2158, %r2736; + mov.u32 %r3423, %r2948; + +$L__BB6_336: + add.s32 %r2970, %r512, 1; + setp.eq.s32 %p376, %r515, 1; + @%p376 bra $L__BB6_347; + + shl.b32 %r2958, %r2985, 1; + setp.eq.s32 %p377, %r3422, 0; + selp.b32 %r533, 8, 7, %p377; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p378, %r3458, %r533; + @%p378 bra $L__BB6_341; + + sub.s32 %r3458, %r3458, %r533; + setp.ge.u32 %p379, %r3419, %r3; + mov.u32 %r2956, 1; + @%p379 bra $L__BB6_340; + + shl.b32 %r2739, %r2985, 1; + shr.u32 %r2160, %r2739, %r3458; + cvt.u16.u32 %rs23, %r2160; + and.b16 %rs24, %rs23, 255; + cvt.u64.u32 %rd237, %r3419; + add.s64 %rd238, %rd4, %rd237; + st.global.u8 [%rd238], %rs23; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p380, %rs24, 255; + selp.u32 %r3422, 1, 0, %p380; + mov.u32 %r2956, %r3423; + +$L__BB6_340: + shl.b32 %r2738, %r2985, 1; + mov.u32 %r2161, -1; + shl.b32 %r2162, %r2161, %r3458; + xor.b32 %r2163, %r2162, -2; + setp.eq.s32 %p381, %r3458, 0; + selp.b32 %r2164, 0, %r2163, %p381; + and.b32 %r2958, %r2164, %r2738; + mov.u32 %r3423, %r2956; + +$L__BB6_341: + add.s32 %r2970, %r512, 2; + setp.eq.s32 %p382, %r515, 2; + mov.u32 %r2985, %r2958; + @%p382 bra $L__BB6_347; + + shl.b32 %r2985, %r2958, 1; + setp.eq.s32 %p383, %r3422, 0; + selp.b32 %r549, 8, 7, %p383; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p384, %r3458, %r549; + @%p384 bra $L__BB6_346; + + sub.s32 %r3458, %r3458, %r549; + setp.ge.u32 %p385, %r3419, %r3; + mov.u32 %r2964, 1; + @%p385 bra $L__BB6_345; + + shl.b32 %r2741, %r2958, 1; + shr.u32 %r2166, %r2741, %r3458; + cvt.u16.u32 %rs25, %r2166; + and.b16 %rs26, %rs25, 255; + cvt.u64.u32 %rd239, %r3419; + add.s64 %rd240, %rd4, %rd239; + st.global.u8 [%rd240], %rs25; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p386, %rs26, 255; + selp.u32 %r3422, 1, 0, %p386; + mov.u32 %r2964, %r3423; + +$L__BB6_345: + shl.b32 %r2740, %r2958, 1; + mov.u32 %r2167, -1; + shl.b32 %r2168, %r2167, %r3458; + xor.b32 %r2169, %r2168, -2; + setp.eq.s32 %p387, %r3458, 0; + selp.b32 %r2170, 0, %r2169, %p387; + and.b32 %r2985, %r2170, %r2740; + mov.u32 %r3423, %r2964; + +$L__BB6_346: + add.s32 %r2970, %r512, 3; + +$L__BB6_347: + min.u32 %r2734, %r513, %r2732; + not.b32 %r2171, %r512; + add.s32 %r2172, %r2734, %r2171; + setp.lt.u32 %p388, %r2172, 3; + @%p388 bra $L__BB6_365; + +$L__BB6_348: + shl.b32 %r2991, %r2985, 1; + setp.eq.s32 %p389, %r3422, 0; + selp.b32 %r582, 8, 7, %p389; + add.s32 %r2992, %r3458, 1; + setp.lt.u32 %p390, %r2992, %r582; + @%p390 bra $L__BB6_352; + + sub.s32 %r2992, %r2992, %r582; + setp.ge.u32 %p391, %r3419, %r3; + mov.u32 %r2989, 1; + @%p391 bra $L__BB6_351; + + shr.u32 %r2174, %r2991, %r2992; + cvt.u16.u32 %rs27, %r2174; + and.b16 %rs28, %rs27, 255; + cvt.u64.u32 %rd241, %r3419; + add.s64 %rd242, %rd4, %rd241; + st.global.u8 [%rd242], %rs27; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p392, %rs28, 255; + selp.u32 %r3422, 1, 0, %p392; + mov.u32 %r2989, %r3423; + +$L__BB6_351: + mov.u32 %r2175, -1; + shl.b32 %r2176, %r2175, %r2992; + xor.b32 %r2177, %r2176, -2; + setp.eq.s32 %p393, %r2992, 0; + selp.b32 %r2178, 0, %r2177, %p393; + and.b32 %r2991, %r2178, %r2991; + mov.u32 %r3423, %r2989; + +$L__BB6_352: + shl.b32 %r2999, %r2991, 1; + setp.eq.s32 %p394, %r3422, 0; + selp.b32 %r597, 8, 7, %p394; + add.s32 %r3000, %r2992, 1; + setp.lt.u32 %p395, %r3000, %r597; + @%p395 bra $L__BB6_356; + + sub.s32 %r3000, %r3000, %r597; + setp.ge.u32 %p396, %r3419, %r3; + mov.u32 %r2997, 1; + @%p396 bra $L__BB6_355; + + shr.u32 %r2180, %r2999, %r3000; + cvt.u16.u32 %rs29, %r2180; + and.b16 %rs30, %rs29, 255; + cvt.u64.u32 %rd243, %r3419; + add.s64 %rd244, %rd4, %rd243; + st.global.u8 [%rd244], %rs29; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p397, %rs30, 255; + selp.u32 %r3422, 1, 0, %p397; + mov.u32 %r2997, %r3423; + +$L__BB6_355: + mov.u32 %r2181, -1; + shl.b32 %r2182, %r2181, %r3000; + xor.b32 %r2183, %r2182, -2; + setp.eq.s32 %p398, %r3000, 0; + selp.b32 %r2184, 0, %r2183, %p398; + and.b32 %r2999, %r2184, %r2999; + mov.u32 %r3423, %r2997; + +$L__BB6_356: + shl.b32 %r3007, %r2999, 1; + setp.eq.s32 %p399, %r3422, 0; + selp.b32 %r612, 8, 7, %p399; + add.s32 %r3008, %r3000, 1; + setp.lt.u32 %p400, %r3008, %r612; + @%p400 bra $L__BB6_360; + + sub.s32 %r3008, %r3008, %r612; + setp.ge.u32 %p401, %r3419, %r3; + mov.u32 %r3005, 1; + @%p401 bra $L__BB6_359; + + shr.u32 %r2186, %r3007, %r3008; + cvt.u16.u32 %rs31, %r2186; + and.b16 %rs32, %rs31, 255; + cvt.u64.u32 %rd245, %r3419; + add.s64 %rd246, %rd4, %rd245; + st.global.u8 [%rd246], %rs31; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p402, %rs32, 255; + selp.u32 %r3422, 1, 0, %p402; + mov.u32 %r3005, %r3423; + +$L__BB6_359: + mov.u32 %r2187, -1; + shl.b32 %r2188, %r2187, %r3008; + xor.b32 %r2189, %r2188, -2; + setp.eq.s32 %p403, %r3008, 0; + selp.b32 %r2190, 0, %r2189, %p403; + and.b32 %r3007, %r2190, %r3007; + mov.u32 %r3423, %r3005; + +$L__BB6_360: + shl.b32 %r2985, %r3007, 1; + setp.eq.s32 %p404, %r3422, 0; + selp.b32 %r627, 8, 7, %p404; + add.s32 %r3458, %r3008, 1; + setp.lt.u32 %p405, %r3458, %r627; + @%p405 bra $L__BB6_364; + + sub.s32 %r3458, %r3458, %r627; + setp.ge.u32 %p406, %r3419, %r3; + mov.u32 %r3013, 1; + @%p406 bra $L__BB6_363; + + shr.u32 %r2192, %r2985, %r3458; + cvt.u16.u32 %rs33, %r2192; + and.b16 %rs34, %rs33, 255; + cvt.u64.u32 %rd247, %r3419; + add.s64 %rd248, %rd4, %rd247; + st.global.u8 [%rd248], %rs33; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p407, %rs34, 255; + selp.u32 %r3422, 1, 0, %p407; + mov.u32 %r3013, %r3423; + +$L__BB6_363: + mov.u32 %r2193, -1; + shl.b32 %r2194, %r2193, %r3458; + xor.b32 %r2195, %r2194, -2; + setp.eq.s32 %p408, %r3458, 0; + selp.b32 %r2196, 0, %r2195, %p408; + and.b32 %r2985, %r2196, %r2985; + mov.u32 %r3423, %r3013; + +$L__BB6_364: + min.u32 %r2735, %r513, %r2732; + add.s32 %r2970, %r2970, 4; + setp.lt.u32 %p409, %r2970, %r2735; + @%p409 bra $L__BB6_348; + +$L__BB6_365: + add.s32 %r2729, %r298, 1; + setp.ge.u32 %p410, %r513, %r2729; + @%p410 bra $L__BB6_371; + + shl.b32 %r2197, %r2985, 1; + or.b32 %r3028, %r2197, 1; + setp.eq.s32 %p411, %r3422, 0; + selp.b32 %r648, 8, 7, %p411; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p412, %r3458, %r648; + @%p412 bra $L__BB6_370; + + sub.s32 %r3458, %r3458, %r648; + setp.ge.u32 %p413, %r3419, %r3; + mov.u32 %r3026, 1; + @%p413 bra $L__BB6_369; + + shl.b32 %r2745, %r2985, 1; + or.b32 %r2744, %r2745, 1; + shr.u32 %r2199, %r2744, %r3458; + cvt.u16.u32 %rs35, %r2199; + and.b16 %rs36, %rs35, 255; + cvt.u64.u32 %rd249, %r3419; + add.s64 %rd250, %rd4, %rd249; + st.global.u8 [%rd250], %rs35; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p414, %rs36, 255; + selp.u32 %r3422, 1, 0, %p414; + mov.u32 %r3026, %r3423; + +$L__BB6_369: + shl.b32 %r2743, %r2985, 1; + or.b32 %r2742, %r2743, 1; + mov.u32 %r2200, -1; + shl.b32 %r2201, %r2200, %r3458; + not.b32 %r2202, %r2201; + setp.eq.s32 %p415, %r3458, 0; + selp.b32 %r2203, 0, %r2202, %p415; + and.b32 %r3028, %r2203, %r2742; + mov.u32 %r3423, %r3026; + +$L__BB6_370: + mov.u32 %r2204, 1; + st.local.u32 [%rd44+8192], %r2204; + mov.u32 %r2985, %r3028; + +$L__BB6_371: + add.s32 %r2731, %r298, 1; + min.u32 %r3037, %r513, %r2731; + st.local.u32 [%rd44], %r3037; + mov.u32 %r3457, %r2985; + +$L__BB6_372: + setp.ne.s32 %p416, %r2939, 0; + mov.u32 %r2940, %r3037; + @%p416 bra $L__BB6_329; + +$L__BB6_373: + cvt.u64.u32 %rd251, %r3419; + add.s64 %rd45, %rd4, %rd251; + setp.eq.s32 %p417, %r274, 1; + @%p417 bra $L__BB6_501; + bra.uni $L__BB6_374; + +$L__BB6_501: + shl.b32 %r3285, %r3457, 1; + setp.eq.s32 %p575, %r3422, 0; + selp.b32 %r1129, 8, 7, %p575; + add.s32 %r3286, %r3458, 1; + setp.lt.u32 %p576, %r3286, %r1129; + @%p576 bra $L__BB6_505; + + sub.s32 %r3286, %r3286, %r1129; + setp.ge.u32 %p577, %r3419, %r3; + mov.u32 %r3283, 1; + @%p577 bra $L__BB6_504; + + shr.u32 %r2432, %r3285, %r3286; + cvt.u16.u32 %rs99, %r2432; + and.b16 %rs100, %rs99, 255; + st.global.u8 [%rd45], %rs99; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p578, %rs100, 255; + selp.u32 %r3422, 1, 0, %p578; + mov.u32 %r3283, %r3423; + +$L__BB6_504: + mov.u32 %r2433, -1; + shl.b32 %r2434, %r2433, %r3286; + xor.b32 %r2435, %r2434, -2; + setp.eq.s32 %p579, %r3286, 0; + selp.b32 %r2436, 0, %r2435, %p579; + and.b32 %r3285, %r2436, %r3285; + mov.u32 %r3423, %r3283; + bra.uni $L__BB6_505; + +$L__BB6_374: + setp.eq.s32 %p418, %r274, 2; + @%p418 bra $L__BB6_493; + bra.uni $L__BB6_375; + +$L__BB6_493: + shl.b32 %r2418, %r3457, 1; + or.b32 %r3274, %r2418, 1; + setp.eq.s32 %p565, %r3422, 0; + selp.b32 %r1104, 8, 7, %p565; + add.s32 %r3275, %r3458, 1; + setp.lt.u32 %p566, %r3275, %r1104; + @%p566 bra $L__BB6_497; + + sub.s32 %r3275, %r3275, %r1104; + setp.ge.u32 %p567, %r3419, %r3; + mov.u32 %r3272, 1; + @%p567 bra $L__BB6_496; + + shr.u32 %r2420, %r3274, %r3275; + cvt.u16.u32 %rs95, %r2420; + and.b16 %rs96, %rs95, 255; + st.global.u8 [%rd45], %rs95; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p568, %rs96, 255; + selp.u32 %r3422, 1, 0, %p568; + mov.u32 %r3272, %r3423; + +$L__BB6_496: + mov.u32 %r2421, -1; + shl.b32 %r2422, %r2421, %r3275; + not.b32 %r2423, %r2422; + setp.eq.s32 %p569, %r3275, 0; + selp.b32 %r2424, 0, %r2423, %p569; + and.b32 %r3274, %r3274, %r2424; + mov.u32 %r3423, %r3272; + +$L__BB6_497: + shl.b32 %r3285, %r3274, 1; + setp.eq.s32 %p570, %r3422, 0; + selp.b32 %r1119, 8, 7, %p570; + add.s32 %r3286, %r3275, 1; + setp.lt.u32 %p571, %r3286, %r1119; + @%p571 bra $L__BB6_505; + + sub.s32 %r3286, %r3286, %r1119; + setp.ge.u32 %p572, %r3419, %r3; + mov.u32 %r3280, 1; + @%p572 bra $L__BB6_500; + + shr.u32 %r2426, %r3285, %r3286; + cvt.u16.u32 %rs97, %r2426; + and.b16 %rs98, %rs97, 255; + cvt.u64.u32 %rd304, %r3419; + add.s64 %rd305, %rd4, %rd304; + st.global.u8 [%rd305], %rs97; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p573, %rs98, 255; + selp.u32 %r3422, 1, 0, %p573; + mov.u32 %r3280, %r3423; + +$L__BB6_500: + mov.u32 %r2427, -1; + shl.b32 %r2428, %r2427, %r3286; + not.b32 %r2429, %r2428; + setp.eq.s32 %p574, %r3286, 0; + selp.b32 %r2430, 0, %r2429, %p574; + and.b32 %r3285, %r3285, %r2430; + mov.u32 %r3423, %r3280; + bra.uni $L__BB6_505; + +$L__BB6_236: + add.s32 %r3458, %r3458, 1; + setp.eq.s32 %p291, %r3422, 0; + selp.b32 %r280, 8, 7, %p291; + setp.lt.u32 %p292, %r3458, %r280; + @%p292 bra $L__BB6_240; + + sub.s32 %r3458, %r3458, %r280; + setp.ge.u32 %p293, %r3419, %r3; + mov.u32 %r2812, 1; + @%p293 bra $L__BB6_239; + + shr.u32 %r1990, %r3457, %r3458; + cvt.u16.u32 %rs1, %r1990; + and.b16 %rs2, %rs1, 255; + st.global.u8 [%rd34], %rs1; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p294, %rs2, 255; + selp.u32 %r3422, 1, 0, %p294; + mov.u32 %r2812, %r3423; + +$L__BB6_239: + mov.u32 %r1991, -1; + shl.b32 %r1992, %r1991, %r3458; + xor.b32 %r1993, %r1992, -2; + setp.eq.s32 %p295, %r3458, 0; + selp.b32 %r1994, 0, %r1993, %p295; + and.b32 %r3457, %r1994, %r3457; + mov.u32 %r3423, %r2812; + bra.uni $L__BB6_240; + +$L__BB6_375: + setp.lt.u32 %p419, %r274, 6; + @%p419 bra $L__BB6_477; + bra.uni $L__BB6_376; + +$L__BB6_477: + shl.b32 %r2389, %r3457, 1; + or.b32 %r3247, %r2389, 1; + setp.eq.s32 %p545, %r3422, 0; + selp.b32 %r1048, 8, 7, %p545; + add.s32 %r3248, %r3458, 1; + setp.lt.u32 %p546, %r3248, %r1048; + @%p546 bra $L__BB6_481; + + sub.s32 %r3248, %r3248, %r1048; + setp.ge.u32 %p547, %r3419, %r3; + mov.u32 %r3245, 1; + @%p547 bra $L__BB6_480; + + shr.u32 %r2391, %r3247, %r3248; + cvt.u16.u32 %rs87, %r2391; + and.b16 %rs88, %rs87, 255; + st.global.u8 [%rd45], %rs87; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p548, %rs88, 255; + selp.u32 %r3422, 1, 0, %p548; + mov.u32 %r3245, %r3423; + +$L__BB6_480: + mov.u32 %r2392, -1; + shl.b32 %r2393, %r2392, %r3248; + not.b32 %r2394, %r2393; + setp.eq.s32 %p549, %r3248, 0; + selp.b32 %r2395, 0, %r2394, %p549; + and.b32 %r3247, %r3247, %r2395; + mov.u32 %r3423, %r3245; + +$L__BB6_481: + shl.b32 %r2396, %r3247, 1; + or.b32 %r3255, %r2396, 1; + setp.eq.s32 %p550, %r3422, 0; + selp.b32 %r1063, 8, 7, %p550; + add.s32 %r3256, %r3248, 1; + setp.lt.u32 %p551, %r3256, %r1063; + @%p551 bra $L__BB6_485; + + sub.s32 %r3256, %r3256, %r1063; + setp.ge.u32 %p552, %r3419, %r3; + mov.u32 %r3253, 1; + @%p552 bra $L__BB6_484; + + shr.u32 %r2398, %r3255, %r3256; + cvt.u16.u32 %rs89, %r2398; + and.b16 %rs90, %rs89, 255; + cvt.u64.u32 %rd298, %r3419; + add.s64 %rd299, %rd4, %rd298; + st.global.u8 [%rd299], %rs89; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p553, %rs90, 255; + selp.u32 %r3422, 1, 0, %p553; + mov.u32 %r3253, %r3423; + +$L__BB6_484: + mov.u32 %r2399, -1; + shl.b32 %r2400, %r2399, %r3256; + not.b32 %r2401, %r2400; + setp.eq.s32 %p554, %r3256, 0; + selp.b32 %r2402, 0, %r2401, %p554; + and.b32 %r3255, %r3255, %r2402; + mov.u32 %r3423, %r3253; + +$L__BB6_485: + add.s32 %r1077, %r274, -3; + shr.u32 %r2403, %r1077, 1; + and.b32 %r2404, %r2403, 1; + bfi.b32 %r3263, %r3255, %r2404, 1, 31; + setp.eq.s32 %p555, %r3422, 0; + selp.b32 %r1079, 8, 7, %p555; + add.s32 %r3264, %r3256, 1; + setp.lt.u32 %p556, %r3264, %r1079; + @%p556 bra $L__BB6_489; + + sub.s32 %r3264, %r3264, %r1079; + setp.ge.u32 %p557, %r3419, %r3; + mov.u32 %r3261, 1; + @%p557 bra $L__BB6_488; + + shr.u32 %r2406, %r3263, %r3264; + cvt.u16.u32 %rs91, %r2406; + and.b16 %rs92, %rs91, 255; + cvt.u64.u32 %rd300, %r3419; + add.s64 %rd301, %rd4, %rd300; + st.global.u8 [%rd301], %rs91; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p558, %rs92, 255; + selp.u32 %r3422, 1, 0, %p558; + mov.u32 %r3261, %r3423; + +$L__BB6_488: + mov.u32 %r2407, -1; + shl.b32 %r2408, %r2407, %r3264; + not.b32 %r2409, %r2408; + setp.eq.s32 %p559, %r3264, 0; + selp.b32 %r2410, 0, %r2409, %p559; + and.b32 %r3263, %r3263, %r2410; + mov.u32 %r3423, %r3261; + +$L__BB6_489: + and.b32 %r2411, %r1077, 1; + bfi.b32 %r3285, %r3263, %r2411, 1, 31; + setp.eq.s32 %p560, %r3422, 0; + selp.b32 %r1094, 8, 7, %p560; + add.s32 %r3286, %r3264, 1; + setp.lt.u32 %p561, %r3286, %r1094; + @%p561 bra $L__BB6_505; + + sub.s32 %r3286, %r3286, %r1094; + setp.ge.u32 %p562, %r3419, %r3; + mov.u32 %r3269, 1; + @%p562 bra $L__BB6_492; + + shr.u32 %r2413, %r3285, %r3286; + cvt.u16.u32 %rs93, %r2413; + and.b16 %rs94, %rs93, 255; + cvt.u64.u32 %rd302, %r3419; + add.s64 %rd303, %rd4, %rd302; + st.global.u8 [%rd303], %rs93; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p563, %rs94, 255; + selp.u32 %r3422, 1, 0, %p563; + mov.u32 %r3269, %r3423; + +$L__BB6_492: + mov.u32 %r2414, -1; + shl.b32 %r2415, %r2414, %r3286; + not.b32 %r2416, %r2415; + setp.eq.s32 %p564, %r3286, 0; + selp.b32 %r2417, 0, %r2416, %p564; + and.b32 %r3285, %r3285, %r2417; + mov.u32 %r3423, %r3269; + bra.uni $L__BB6_505; + +$L__BB6_376: + setp.lt.u32 %p420, %r274, 37; + shl.b32 %r2205, %r3457, 1; + or.b32 %r3057, %r2205, 1; + add.s32 %r3058, %r3458, 1; + setp.eq.s32 %p421, %r3422, 0; + selp.b32 %r685, 8, 7, %p421; + @%p420 bra $L__BB6_441; + bra.uni $L__BB6_377; + +$L__BB6_441: + setp.lt.u32 %p501, %r3058, %r685; + @%p501 bra $L__BB6_445; + + sub.s32 %r3058, %r3058, %r685; + setp.ge.u32 %p502, %r3419, %r3; + mov.u32 %r3178, 1; + @%p502 bra $L__BB6_444; + + shr.u32 %r2324, %r3057, %r3058; + cvt.u16.u32 %rs69, %r2324; + and.b16 %rs70, %rs69, 255; + st.global.u8 [%rd45], %rs69; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p503, %rs70, 255; + selp.u32 %r3422, 1, 0, %p503; + mov.u32 %r3178, %r3423; + +$L__BB6_444: + mov.u32 %r2325, -1; + shl.b32 %r2326, %r2325, %r3058; + not.b32 %r2327, %r2326; + setp.eq.s32 %p504, %r3058, 0; + selp.b32 %r2328, 0, %r2327, %p504; + and.b32 %r3057, %r3057, %r2328; + mov.u32 %r3423, %r3178; + +$L__BB6_445: + shl.b32 %r2329, %r3057, 1; + or.b32 %r3188, %r2329, 1; + setp.eq.s32 %p505, %r3422, 0; + selp.b32 %r932, 8, 7, %p505; + add.s32 %r3189, %r3058, 1; + setp.lt.u32 %p506, %r3189, %r932; + @%p506 bra $L__BB6_449; + + sub.s32 %r3189, %r3189, %r932; + setp.ge.u32 %p507, %r3419, %r3; + mov.u32 %r3186, 1; + @%p507 bra $L__BB6_448; + + shr.u32 %r2331, %r3188, %r3189; + cvt.u16.u32 %rs71, %r2331; + and.b16 %rs72, %rs71, 255; + cvt.u64.u32 %rd282, %r3419; + add.s64 %rd283, %rd4, %rd282; + st.global.u8 [%rd283], %rs71; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p508, %rs72, 255; + selp.u32 %r3422, 1, 0, %p508; + mov.u32 %r3186, %r3423; + +$L__BB6_448: + mov.u32 %r2332, -1; + shl.b32 %r2333, %r2332, %r3189; + not.b32 %r2334, %r2333; + setp.eq.s32 %p509, %r3189, 0; + selp.b32 %r2335, 0, %r2334, %p509; + and.b32 %r3188, %r3188, %r2335; + mov.u32 %r3423, %r3186; + +$L__BB6_449: + shl.b32 %r2336, %r3188, 1; + or.b32 %r3196, %r2336, 1; + setp.eq.s32 %p510, %r3422, 0; + selp.b32 %r947, 8, 7, %p510; + add.s32 %r3197, %r3189, 1; + setp.lt.u32 %p511, %r3197, %r947; + @%p511 bra $L__BB6_453; + + sub.s32 %r3197, %r3197, %r947; + setp.ge.u32 %p512, %r3419, %r3; + mov.u32 %r3194, 1; + @%p512 bra $L__BB6_452; + + shr.u32 %r2338, %r3196, %r3197; + cvt.u16.u32 %rs73, %r2338; + and.b16 %rs74, %rs73, 255; + cvt.u64.u32 %rd284, %r3419; + add.s64 %rd285, %rd4, %rd284; + st.global.u8 [%rd285], %rs73; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p513, %rs74, 255; + selp.u32 %r3422, 1, 0, %p513; + mov.u32 %r3194, %r3423; + +$L__BB6_452: + mov.u32 %r2339, -1; + shl.b32 %r2340, %r2339, %r3197; + not.b32 %r2341, %r2340; + setp.eq.s32 %p514, %r3197, 0; + selp.b32 %r2342, 0, %r2341, %p514; + and.b32 %r3196, %r3196, %r2342; + mov.u32 %r3423, %r3194; + +$L__BB6_453: + shl.b32 %r2343, %r3196, 1; + or.b32 %r3204, %r2343, 1; + setp.eq.s32 %p515, %r3422, 0; + selp.b32 %r962, 8, 7, %p515; + add.s32 %r3205, %r3197, 1; + setp.lt.u32 %p516, %r3205, %r962; + @%p516 bra $L__BB6_457; + + sub.s32 %r3205, %r3205, %r962; + setp.ge.u32 %p517, %r3419, %r3; + mov.u32 %r3202, 1; + @%p517 bra $L__BB6_456; + + shr.u32 %r2345, %r3204, %r3205; + cvt.u16.u32 %rs75, %r2345; + and.b16 %rs76, %rs75, 255; + cvt.u64.u32 %rd286, %r3419; + add.s64 %rd287, %rd4, %rd286; + st.global.u8 [%rd287], %rs75; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p518, %rs76, 255; + selp.u32 %r3422, 1, 0, %p518; + mov.u32 %r3202, %r3423; + +$L__BB6_456: + mov.u32 %r2346, -1; + shl.b32 %r2347, %r2346, %r3205; + not.b32 %r2348, %r2347; + setp.eq.s32 %p519, %r3205, 0; + selp.b32 %r2349, 0, %r2348, %p519; + and.b32 %r3204, %r3204, %r2349; + mov.u32 %r3423, %r3202; + +$L__BB6_457: + add.s32 %r976, %r274, -6; + shr.u32 %r2350, %r976, 4; + and.b32 %r2351, %r2350, 1; + bfi.b32 %r3212, %r3204, %r2351, 1, 31; + setp.eq.s32 %p520, %r3422, 0; + selp.b32 %r978, 8, 7, %p520; + add.s32 %r3213, %r3205, 1; + setp.lt.u32 %p521, %r3213, %r978; + @%p521 bra $L__BB6_461; + + sub.s32 %r3213, %r3213, %r978; + setp.ge.u32 %p522, %r3419, %r3; + mov.u32 %r3210, 1; + @%p522 bra $L__BB6_460; + + shr.u32 %r2353, %r3212, %r3213; + cvt.u16.u32 %rs77, %r2353; + and.b16 %rs78, %rs77, 255; + cvt.u64.u32 %rd288, %r3419; + add.s64 %rd289, %rd4, %rd288; + st.global.u8 [%rd289], %rs77; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p523, %rs78, 255; + selp.u32 %r3422, 1, 0, %p523; + mov.u32 %r3210, %r3423; + +$L__BB6_460: + mov.u32 %r2354, -1; + shl.b32 %r2355, %r2354, %r3213; + not.b32 %r2356, %r2355; + setp.eq.s32 %p524, %r3213, 0; + selp.b32 %r2357, 0, %r2356, %p524; + and.b32 %r3212, %r3212, %r2357; + mov.u32 %r3423, %r3210; + +$L__BB6_461: + shr.u32 %r2358, %r976, 3; + and.b32 %r2359, %r2358, 1; + bfi.b32 %r3220, %r3212, %r2359, 1, 31; + setp.eq.s32 %p525, %r3422, 0; + selp.b32 %r993, 8, 7, %p525; + add.s32 %r3221, %r3213, 1; + setp.lt.u32 %p526, %r3221, %r993; + @%p526 bra $L__BB6_465; + + sub.s32 %r3221, %r3221, %r993; + setp.ge.u32 %p527, %r3419, %r3; + mov.u32 %r3218, 1; + @%p527 bra $L__BB6_464; + + shr.u32 %r2361, %r3220, %r3221; + cvt.u16.u32 %rs79, %r2361; + and.b16 %rs80, %rs79, 255; + cvt.u64.u32 %rd290, %r3419; + add.s64 %rd291, %rd4, %rd290; + st.global.u8 [%rd291], %rs79; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p528, %rs80, 255; + selp.u32 %r3422, 1, 0, %p528; + mov.u32 %r3218, %r3423; + +$L__BB6_464: + mov.u32 %r2362, -1; + shl.b32 %r2363, %r2362, %r3221; + not.b32 %r2364, %r2363; + setp.eq.s32 %p529, %r3221, 0; + selp.b32 %r2365, 0, %r2364, %p529; + and.b32 %r3220, %r3220, %r2365; + mov.u32 %r3423, %r3218; + +$L__BB6_465: + shr.u32 %r2366, %r976, 2; + and.b32 %r2367, %r2366, 1; + bfi.b32 %r3228, %r3220, %r2367, 1, 31; + setp.eq.s32 %p530, %r3422, 0; + selp.b32 %r1008, 8, 7, %p530; + add.s32 %r3229, %r3221, 1; + setp.lt.u32 %p531, %r3229, %r1008; + @%p531 bra $L__BB6_469; + + sub.s32 %r3229, %r3229, %r1008; + setp.ge.u32 %p532, %r3419, %r3; + mov.u32 %r3226, 1; + @%p532 bra $L__BB6_468; + + shr.u32 %r2369, %r3228, %r3229; + cvt.u16.u32 %rs81, %r2369; + and.b16 %rs82, %rs81, 255; + cvt.u64.u32 %rd292, %r3419; + add.s64 %rd293, %rd4, %rd292; + st.global.u8 [%rd293], %rs81; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p533, %rs82, 255; + selp.u32 %r3422, 1, 0, %p533; + mov.u32 %r3226, %r3423; + +$L__BB6_468: + mov.u32 %r2370, -1; + shl.b32 %r2371, %r2370, %r3229; + not.b32 %r2372, %r2371; + setp.eq.s32 %p534, %r3229, 0; + selp.b32 %r2373, 0, %r2372, %p534; + and.b32 %r3228, %r3228, %r2373; + mov.u32 %r3423, %r3226; + +$L__BB6_469: + shr.u32 %r2374, %r976, 1; + and.b32 %r2375, %r2374, 1; + bfi.b32 %r3236, %r3228, %r2375, 1, 31; + setp.eq.s32 %p535, %r3422, 0; + selp.b32 %r1023, 8, 7, %p535; + add.s32 %r3237, %r3229, 1; + setp.lt.u32 %p536, %r3237, %r1023; + @%p536 bra $L__BB6_473; + + sub.s32 %r3237, %r3237, %r1023; + setp.ge.u32 %p537, %r3419, %r3; + mov.u32 %r3234, 1; + @%p537 bra $L__BB6_472; + + shr.u32 %r2377, %r3236, %r3237; + cvt.u16.u32 %rs83, %r2377; + and.b16 %rs84, %rs83, 255; + cvt.u64.u32 %rd294, %r3419; + add.s64 %rd295, %rd4, %rd294; + st.global.u8 [%rd295], %rs83; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p538, %rs84, 255; + selp.u32 %r3422, 1, 0, %p538; + mov.u32 %r3234, %r3423; + +$L__BB6_472: + mov.u32 %r2378, -1; + shl.b32 %r2379, %r2378, %r3237; + not.b32 %r2380, %r2379; + setp.eq.s32 %p539, %r3237, 0; + selp.b32 %r2381, 0, %r2380, %p539; + and.b32 %r3236, %r3236, %r2381; + mov.u32 %r3423, %r3234; + +$L__BB6_473: + and.b32 %r2382, %r976, 1; + bfi.b32 %r3285, %r3236, %r2382, 1, 31; + setp.eq.s32 %p540, %r3422, 0; + selp.b32 %r1038, 8, 7, %p540; + add.s32 %r3286, %r3237, 1; + setp.lt.u32 %p541, %r3286, %r1038; + @%p541 bra $L__BB6_505; + + sub.s32 %r3286, %r3286, %r1038; + setp.ge.u32 %p542, %r3419, %r3; + mov.u32 %r3242, 1; + @%p542 bra $L__BB6_476; + + shr.u32 %r2384, %r3285, %r3286; + cvt.u16.u32 %rs85, %r2384; + and.b16 %rs86, %rs85, 255; + cvt.u64.u32 %rd296, %r3419; + add.s64 %rd297, %rd4, %rd296; + st.global.u8 [%rd297], %rs85; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p543, %rs86, 255; + selp.u32 %r3422, 1, 0, %p543; + mov.u32 %r3242, %r3423; + +$L__BB6_476: + mov.u32 %r2385, -1; + shl.b32 %r2386, %r2385, %r3286; + not.b32 %r2387, %r2386; + setp.eq.s32 %p544, %r3286, 0; + selp.b32 %r2388, 0, %r2387, %p544; + and.b32 %r3285, %r3285, %r2388; + mov.u32 %r3423, %r3242; + bra.uni $L__BB6_505; + +$L__BB6_377: + setp.lt.u32 %p422, %r3058, %r685; + @%p422 bra $L__BB6_381; + + sub.s32 %r3058, %r3058, %r685; + setp.ge.u32 %p423, %r3419, %r3; + mov.u32 %r3055, 1; + @%p423 bra $L__BB6_380; + + shr.u32 %r2207, %r3057, %r3058; + cvt.u16.u32 %rs37, %r2207; + and.b16 %rs38, %rs37, 255; + st.global.u8 [%rd45], %rs37; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p424, %rs38, 255; + selp.u32 %r3422, 1, 0, %p424; + mov.u32 %r3055, %r3423; + +$L__BB6_380: + mov.u32 %r2208, -1; + shl.b32 %r2209, %r2208, %r3058; + not.b32 %r2210, %r2209; + setp.eq.s32 %p425, %r3058, 0; + selp.b32 %r2211, 0, %r2210, %p425; + and.b32 %r3057, %r3057, %r2211; + mov.u32 %r3423, %r3055; + +$L__BB6_381: + shl.b32 %r2212, %r3057, 1; + or.b32 %r3065, %r2212, 1; + setp.eq.s32 %p426, %r3422, 0; + selp.b32 %r699, 8, 7, %p426; + add.s32 %r3066, %r3058, 1; + setp.lt.u32 %p427, %r3066, %r699; + @%p427 bra $L__BB6_385; + + sub.s32 %r3066, %r3066, %r699; + setp.ge.u32 %p428, %r3419, %r3; + mov.u32 %r3063, 1; + @%p428 bra $L__BB6_384; + + shr.u32 %r2214, %r3065, %r3066; + cvt.u16.u32 %rs39, %r2214; + and.b16 %rs40, %rs39, 255; + cvt.u64.u32 %rd252, %r3419; + add.s64 %rd253, %rd4, %rd252; + st.global.u8 [%rd253], %rs39; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p429, %rs40, 255; + selp.u32 %r3422, 1, 0, %p429; + mov.u32 %r3063, %r3423; + +$L__BB6_384: + mov.u32 %r2215, -1; + shl.b32 %r2216, %r2215, %r3066; + not.b32 %r2217, %r2216; + setp.eq.s32 %p430, %r3066, 0; + selp.b32 %r2218, 0, %r2217, %p430; + and.b32 %r3065, %r3065, %r2218; + mov.u32 %r3423, %r3063; + +$L__BB6_385: + shl.b32 %r2219, %r3065, 1; + or.b32 %r3073, %r2219, 1; + setp.eq.s32 %p431, %r3422, 0; + selp.b32 %r714, 8, 7, %p431; + add.s32 %r3074, %r3066, 1; + setp.lt.u32 %p432, %r3074, %r714; + @%p432 bra $L__BB6_389; + + sub.s32 %r3074, %r3074, %r714; + setp.ge.u32 %p433, %r3419, %r3; + mov.u32 %r3071, 1; + @%p433 bra $L__BB6_388; + + shr.u32 %r2221, %r3073, %r3074; + cvt.u16.u32 %rs41, %r2221; + and.b16 %rs42, %rs41, 255; + cvt.u64.u32 %rd254, %r3419; + add.s64 %rd255, %rd4, %rd254; + st.global.u8 [%rd255], %rs41; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p434, %rs42, 255; + selp.u32 %r3422, 1, 0, %p434; + mov.u32 %r3071, %r3423; + +$L__BB6_388: + mov.u32 %r2222, -1; + shl.b32 %r2223, %r2222, %r3074; + not.b32 %r2224, %r2223; + setp.eq.s32 %p435, %r3074, 0; + selp.b32 %r2225, 0, %r2224, %p435; + and.b32 %r3073, %r3073, %r2225; + mov.u32 %r3423, %r3071; + +$L__BB6_389: + shl.b32 %r2226, %r3073, 1; + or.b32 %r3081, %r2226, 1; + setp.eq.s32 %p436, %r3422, 0; + selp.b32 %r729, 8, 7, %p436; + add.s32 %r3082, %r3074, 1; + setp.lt.u32 %p437, %r3082, %r729; + @%p437 bra $L__BB6_393; + + sub.s32 %r3082, %r3082, %r729; + setp.ge.u32 %p438, %r3419, %r3; + mov.u32 %r3079, 1; + @%p438 bra $L__BB6_392; + + shr.u32 %r2228, %r3081, %r3082; + cvt.u16.u32 %rs43, %r2228; + and.b16 %rs44, %rs43, 255; + cvt.u64.u32 %rd256, %r3419; + add.s64 %rd257, %rd4, %rd256; + st.global.u8 [%rd257], %rs43; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p439, %rs44, 255; + selp.u32 %r3422, 1, 0, %p439; + mov.u32 %r3079, %r3423; + +$L__BB6_392: + mov.u32 %r2229, -1; + shl.b32 %r2230, %r2229, %r3082; + not.b32 %r2231, %r2230; + setp.eq.s32 %p440, %r3082, 0; + selp.b32 %r2232, 0, %r2231, %p440; + and.b32 %r3081, %r3081, %r2232; + mov.u32 %r3423, %r3079; + +$L__BB6_393: + shl.b32 %r2233, %r3081, 1; + or.b32 %r3089, %r2233, 1; + setp.eq.s32 %p441, %r3422, 0; + selp.b32 %r744, 8, 7, %p441; + add.s32 %r3090, %r3082, 1; + setp.lt.u32 %p442, %r3090, %r744; + @%p442 bra $L__BB6_397; + + sub.s32 %r3090, %r3090, %r744; + setp.ge.u32 %p443, %r3419, %r3; + mov.u32 %r3087, 1; + @%p443 bra $L__BB6_396; + + shr.u32 %r2235, %r3089, %r3090; + cvt.u16.u32 %rs45, %r2235; + and.b16 %rs46, %rs45, 255; + cvt.u64.u32 %rd258, %r3419; + add.s64 %rd259, %rd4, %rd258; + st.global.u8 [%rd259], %rs45; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p444, %rs46, 255; + selp.u32 %r3422, 1, 0, %p444; + mov.u32 %r3087, %r3423; + +$L__BB6_396: + mov.u32 %r2236, -1; + shl.b32 %r2237, %r2236, %r3090; + not.b32 %r2238, %r2237; + setp.eq.s32 %p445, %r3090, 0; + selp.b32 %r2239, 0, %r2238, %p445; + and.b32 %r3089, %r3089, %r2239; + mov.u32 %r3423, %r3087; + +$L__BB6_397: + shl.b32 %r2240, %r3089, 1; + or.b32 %r3097, %r2240, 1; + setp.eq.s32 %p446, %r3422, 0; + selp.b32 %r759, 8, 7, %p446; + add.s32 %r3098, %r3090, 1; + setp.lt.u32 %p447, %r3098, %r759; + @%p447 bra $L__BB6_401; + + sub.s32 %r3098, %r3098, %r759; + setp.ge.u32 %p448, %r3419, %r3; + mov.u32 %r3095, 1; + @%p448 bra $L__BB6_400; + + shr.u32 %r2242, %r3097, %r3098; + cvt.u16.u32 %rs47, %r2242; + and.b16 %rs48, %rs47, 255; + cvt.u64.u32 %rd260, %r3419; + add.s64 %rd261, %rd4, %rd260; + st.global.u8 [%rd261], %rs47; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p449, %rs48, 255; + selp.u32 %r3422, 1, 0, %p449; + mov.u32 %r3095, %r3423; + +$L__BB6_400: + mov.u32 %r2243, -1; + shl.b32 %r2244, %r2243, %r3098; + not.b32 %r2245, %r2244; + setp.eq.s32 %p450, %r3098, 0; + selp.b32 %r2246, 0, %r2245, %p450; + and.b32 %r3097, %r3097, %r2246; + mov.u32 %r3423, %r3095; + +$L__BB6_401: + shl.b32 %r2247, %r3097, 1; + or.b32 %r3105, %r2247, 1; + setp.eq.s32 %p451, %r3422, 0; + selp.b32 %r774, 8, 7, %p451; + add.s32 %r3106, %r3098, 1; + setp.lt.u32 %p452, %r3106, %r774; + @%p452 bra $L__BB6_405; + + sub.s32 %r3106, %r3106, %r774; + setp.ge.u32 %p453, %r3419, %r3; + mov.u32 %r3103, 1; + @%p453 bra $L__BB6_404; + + shr.u32 %r2249, %r3105, %r3106; + cvt.u16.u32 %rs49, %r2249; + and.b16 %rs50, %rs49, 255; + cvt.u64.u32 %rd262, %r3419; + add.s64 %rd263, %rd4, %rd262; + st.global.u8 [%rd263], %rs49; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p454, %rs50, 255; + selp.u32 %r3422, 1, 0, %p454; + mov.u32 %r3103, %r3423; + +$L__BB6_404: + mov.u32 %r2250, -1; + shl.b32 %r2251, %r2250, %r3106; + not.b32 %r2252, %r2251; + setp.eq.s32 %p455, %r3106, 0; + selp.b32 %r2253, 0, %r2252, %p455; + and.b32 %r3105, %r3105, %r2253; + mov.u32 %r3423, %r3103; + +$L__BB6_405: + shl.b32 %r2254, %r3105, 1; + or.b32 %r3113, %r2254, 1; + setp.eq.s32 %p456, %r3422, 0; + selp.b32 %r789, 8, 7, %p456; + add.s32 %r3114, %r3106, 1; + setp.lt.u32 %p457, %r3114, %r789; + @%p457 bra $L__BB6_409; + + sub.s32 %r3114, %r3114, %r789; + setp.ge.u32 %p458, %r3419, %r3; + mov.u32 %r3111, 1; + @%p458 bra $L__BB6_408; + + shr.u32 %r2256, %r3113, %r3114; + cvt.u16.u32 %rs51, %r2256; + and.b16 %rs52, %rs51, 255; + cvt.u64.u32 %rd264, %r3419; + add.s64 %rd265, %rd4, %rd264; + st.global.u8 [%rd265], %rs51; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p459, %rs52, 255; + selp.u32 %r3422, 1, 0, %p459; + mov.u32 %r3111, %r3423; + +$L__BB6_408: + mov.u32 %r2257, -1; + shl.b32 %r2258, %r2257, %r3114; + not.b32 %r2259, %r2258; + setp.eq.s32 %p460, %r3114, 0; + selp.b32 %r2260, 0, %r2259, %p460; + and.b32 %r3113, %r3113, %r2260; + mov.u32 %r3423, %r3111; + +$L__BB6_409: + shl.b32 %r2261, %r3113, 1; + or.b32 %r3121, %r2261, 1; + setp.eq.s32 %p461, %r3422, 0; + selp.b32 %r804, 8, 7, %p461; + add.s32 %r3122, %r3114, 1; + setp.lt.u32 %p462, %r3122, %r804; + @%p462 bra $L__BB6_413; + + sub.s32 %r3122, %r3122, %r804; + setp.ge.u32 %p463, %r3419, %r3; + mov.u32 %r3119, 1; + @%p463 bra $L__BB6_412; + + shr.u32 %r2263, %r3121, %r3122; + cvt.u16.u32 %rs53, %r2263; + and.b16 %rs54, %rs53, 255; + cvt.u64.u32 %rd266, %r3419; + add.s64 %rd267, %rd4, %rd266; + st.global.u8 [%rd267], %rs53; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p464, %rs54, 255; + selp.u32 %r3422, 1, 0, %p464; + mov.u32 %r3119, %r3423; + +$L__BB6_412: + mov.u32 %r2264, -1; + shl.b32 %r2265, %r2264, %r3122; + not.b32 %r2266, %r2265; + setp.eq.s32 %p465, %r3122, 0; + selp.b32 %r2267, 0, %r2266, %p465; + and.b32 %r3121, %r3121, %r2267; + mov.u32 %r3423, %r3119; + +$L__BB6_413: + add.s32 %r818, %r274, -37; + shr.u32 %r2268, %r818, 6; + and.b32 %r2269, %r2268, 1; + bfi.b32 %r3129, %r3121, %r2269, 1, 31; + setp.eq.s32 %p466, %r3422, 0; + selp.b32 %r820, 8, 7, %p466; + add.s32 %r3130, %r3122, 1; + setp.lt.u32 %p467, %r3130, %r820; + @%p467 bra $L__BB6_417; + + sub.s32 %r3130, %r3130, %r820; + setp.ge.u32 %p468, %r3419, %r3; + mov.u32 %r3127, 1; + @%p468 bra $L__BB6_416; + + shr.u32 %r2271, %r3129, %r3130; + cvt.u16.u32 %rs55, %r2271; + and.b16 %rs56, %rs55, 255; + cvt.u64.u32 %rd268, %r3419; + add.s64 %rd269, %rd4, %rd268; + st.global.u8 [%rd269], %rs55; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p469, %rs56, 255; + selp.u32 %r3422, 1, 0, %p469; + mov.u32 %r3127, %r3423; + +$L__BB6_416: + mov.u32 %r2272, -1; + shl.b32 %r2273, %r2272, %r3130; + not.b32 %r2274, %r2273; + setp.eq.s32 %p470, %r3130, 0; + selp.b32 %r2275, 0, %r2274, %p470; + and.b32 %r3129, %r3129, %r2275; + mov.u32 %r3423, %r3127; + +$L__BB6_417: + shr.u32 %r2276, %r818, 5; + and.b32 %r2277, %r2276, 1; + bfi.b32 %r3137, %r3129, %r2277, 1, 31; + setp.eq.s32 %p471, %r3422, 0; + selp.b32 %r835, 8, 7, %p471; + add.s32 %r3138, %r3130, 1; + setp.lt.u32 %p472, %r3138, %r835; + @%p472 bra $L__BB6_421; + + sub.s32 %r3138, %r3138, %r835; + setp.ge.u32 %p473, %r3419, %r3; + mov.u32 %r3135, 1; + @%p473 bra $L__BB6_420; + + shr.u32 %r2279, %r3137, %r3138; + cvt.u16.u32 %rs57, %r2279; + and.b16 %rs58, %rs57, 255; + cvt.u64.u32 %rd270, %r3419; + add.s64 %rd271, %rd4, %rd270; + st.global.u8 [%rd271], %rs57; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p474, %rs58, 255; + selp.u32 %r3422, 1, 0, %p474; + mov.u32 %r3135, %r3423; + +$L__BB6_420: + mov.u32 %r2280, -1; + shl.b32 %r2281, %r2280, %r3138; + not.b32 %r2282, %r2281; + setp.eq.s32 %p475, %r3138, 0; + selp.b32 %r2283, 0, %r2282, %p475; + and.b32 %r3137, %r3137, %r2283; + mov.u32 %r3423, %r3135; + +$L__BB6_421: + shr.u32 %r2284, %r818, 4; + and.b32 %r2285, %r2284, 1; + bfi.b32 %r3145, %r3137, %r2285, 1, 31; + setp.eq.s32 %p476, %r3422, 0; + selp.b32 %r850, 8, 7, %p476; + add.s32 %r3146, %r3138, 1; + setp.lt.u32 %p477, %r3146, %r850; + @%p477 bra $L__BB6_425; + + sub.s32 %r3146, %r3146, %r850; + setp.ge.u32 %p478, %r3419, %r3; + mov.u32 %r3143, 1; + @%p478 bra $L__BB6_424; + + shr.u32 %r2287, %r3145, %r3146; + cvt.u16.u32 %rs59, %r2287; + and.b16 %rs60, %rs59, 255; + cvt.u64.u32 %rd272, %r3419; + add.s64 %rd273, %rd4, %rd272; + st.global.u8 [%rd273], %rs59; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p479, %rs60, 255; + selp.u32 %r3422, 1, 0, %p479; + mov.u32 %r3143, %r3423; + +$L__BB6_424: + mov.u32 %r2288, -1; + shl.b32 %r2289, %r2288, %r3146; + not.b32 %r2290, %r2289; + setp.eq.s32 %p480, %r3146, 0; + selp.b32 %r2291, 0, %r2290, %p480; + and.b32 %r3145, %r3145, %r2291; + mov.u32 %r3423, %r3143; + +$L__BB6_425: + shr.u32 %r2292, %r818, 3; + and.b32 %r2293, %r2292, 1; + bfi.b32 %r3153, %r3145, %r2293, 1, 31; + setp.eq.s32 %p481, %r3422, 0; + selp.b32 %r865, 8, 7, %p481; + add.s32 %r3154, %r3146, 1; + setp.lt.u32 %p482, %r3154, %r865; + @%p482 bra $L__BB6_429; + + sub.s32 %r3154, %r3154, %r865; + setp.ge.u32 %p483, %r3419, %r3; + mov.u32 %r3151, 1; + @%p483 bra $L__BB6_428; + + shr.u32 %r2295, %r3153, %r3154; + cvt.u16.u32 %rs61, %r2295; + and.b16 %rs62, %rs61, 255; + cvt.u64.u32 %rd274, %r3419; + add.s64 %rd275, %rd4, %rd274; + st.global.u8 [%rd275], %rs61; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p484, %rs62, 255; + selp.u32 %r3422, 1, 0, %p484; + mov.u32 %r3151, %r3423; + +$L__BB6_428: + mov.u32 %r2296, -1; + shl.b32 %r2297, %r2296, %r3154; + not.b32 %r2298, %r2297; + setp.eq.s32 %p485, %r3154, 0; + selp.b32 %r2299, 0, %r2298, %p485; + and.b32 %r3153, %r3153, %r2299; + mov.u32 %r3423, %r3151; + +$L__BB6_429: + shr.u32 %r2300, %r818, 2; + and.b32 %r2301, %r2300, 1; + bfi.b32 %r3161, %r3153, %r2301, 1, 31; + setp.eq.s32 %p486, %r3422, 0; + selp.b32 %r880, 8, 7, %p486; + add.s32 %r3162, %r3154, 1; + setp.lt.u32 %p487, %r3162, %r880; + @%p487 bra $L__BB6_433; + + sub.s32 %r3162, %r3162, %r880; + setp.ge.u32 %p488, %r3419, %r3; + mov.u32 %r3159, 1; + @%p488 bra $L__BB6_432; + + shr.u32 %r2303, %r3161, %r3162; + cvt.u16.u32 %rs63, %r2303; + and.b16 %rs64, %rs63, 255; + cvt.u64.u32 %rd276, %r3419; + add.s64 %rd277, %rd4, %rd276; + st.global.u8 [%rd277], %rs63; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p489, %rs64, 255; + selp.u32 %r3422, 1, 0, %p489; + mov.u32 %r3159, %r3423; + +$L__BB6_432: + mov.u32 %r2304, -1; + shl.b32 %r2305, %r2304, %r3162; + not.b32 %r2306, %r2305; + setp.eq.s32 %p490, %r3162, 0; + selp.b32 %r2307, 0, %r2306, %p490; + and.b32 %r3161, %r3161, %r2307; + mov.u32 %r3423, %r3159; + +$L__BB6_433: + shr.u32 %r2308, %r818, 1; + and.b32 %r2309, %r2308, 1; + bfi.b32 %r3169, %r3161, %r2309, 1, 31; + setp.eq.s32 %p491, %r3422, 0; + selp.b32 %r895, 8, 7, %p491; + add.s32 %r3170, %r3162, 1; + setp.lt.u32 %p492, %r3170, %r895; + @%p492 bra $L__BB6_437; + + sub.s32 %r3170, %r3170, %r895; + setp.ge.u32 %p493, %r3419, %r3; + mov.u32 %r3167, 1; + @%p493 bra $L__BB6_436; + + shr.u32 %r2311, %r3169, %r3170; + cvt.u16.u32 %rs65, %r2311; + and.b16 %rs66, %rs65, 255; + cvt.u64.u32 %rd278, %r3419; + add.s64 %rd279, %rd4, %rd278; + st.global.u8 [%rd279], %rs65; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p494, %rs66, 255; + selp.u32 %r3422, 1, 0, %p494; + mov.u32 %r3167, %r3423; + +$L__BB6_436: + mov.u32 %r2312, -1; + shl.b32 %r2313, %r2312, %r3170; + not.b32 %r2314, %r2313; + setp.eq.s32 %p495, %r3170, 0; + selp.b32 %r2315, 0, %r2314, %p495; + and.b32 %r3169, %r3169, %r2315; + mov.u32 %r3423, %r3167; + +$L__BB6_437: + and.b32 %r2316, %r818, 1; + bfi.b32 %r3285, %r3169, %r2316, 1, 31; + setp.eq.s32 %p496, %r3422, 0; + selp.b32 %r910, 8, 7, %p496; + add.s32 %r3286, %r3170, 1; + setp.lt.u32 %p497, %r3286, %r910; + @%p497 bra $L__BB6_505; + + sub.s32 %r3286, %r3286, %r910; + setp.ge.u32 %p498, %r3419, %r3; + mov.u32 %r3175, 1; + @%p498 bra $L__BB6_440; + + shr.u32 %r2318, %r3285, %r3286; + cvt.u16.u32 %rs67, %r2318; + and.b16 %rs68, %rs67, 255; + cvt.u64.u32 %rd280, %r3419; + add.s64 %rd281, %rd4, %rd280; + st.global.u8 [%rd281], %rs67; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p499, %rs68, 255; + selp.u32 %r3422, 1, 0, %p499; + mov.u32 %r3175, %r3423; + +$L__BB6_440: + mov.u32 %r2319, -1; + shl.b32 %r2320, %r2319, %r3286; + not.b32 %r2321, %r2320; + setp.eq.s32 %p500, %r3286, 0; + selp.b32 %r2322, 0, %r2321, %p500; + and.b32 %r3285, %r3285, %r2322; + mov.u32 %r3423, %r3175; + +$L__BB6_505: + setp.eq.s32 %p580, %r272, 0; + mov.u32 %r3291, 0; + and.pred %p582, %p580, %p417; + selp.b32 %r1143, %r271, %r272, %p582; + add.s32 %r2438, %r274, -1; + mul.wide.u32 %rd306, %r2438, -1431655765; + shr.u64 %rd307, %rd306, 33; + cvt.u32.u64 %r2439, %rd307; + mad.lo.s32 %r3290, %r2439, 3, 1; + setp.lt.u32 %p583, %r3290, 2; + setp.eq.s32 %p584, %r274, 0; + or.pred %p585, %p584, %p583; + @%p585 bra $L__BB6_508; + + mov.u32 %r3291, 0; + +$L__BB6_507: + shr.u32 %r1147, %r3290, 1; + add.s32 %r3291, %r3291, 1; + setp.gt.u32 %p586, %r3290, 3; + mov.u32 %r3290, %r1147; + @%p586 bra $L__BB6_507; + +$L__BB6_508: + add.s32 %r3298, %r3291, %r3297; + bra.uni $L__BB6_509; + +$L__BB6_589: + add.s32 %r3297, %r3297, 1; + add.s32 %r3298, %r3298, 1; + +$L__BB6_509: + mov.u32 %r2441, 1; + shl.b32 %r2442, %r2441, %r3298; + setp.le.u32 %p588, %r2442, %r1143; + setp.lt.u32 %p589, %r3298, 32; + and.pred %p590, %p589, %p588; + @%p590 bra $L__BB6_585; + + setp.lt.u32 %p591, %r274, 2; + @%p591 bra $L__BB6_512; + + setp.gt.u32 %p730, %r274, 2; + selp.u32 %r2747, 1, 0, %p730; + add.s32 %r2443, %r3297, %r2747; + setp.lt.u32 %p592, %r2443, 32; + mov.u32 %r2444, 1; + shl.b32 %r2445, %r2444, %r2443; + setp.le.u32 %p593, %r2445, %r273; + and.pred %p594, %p592, %p593; + @%p594 bra $L__BB6_585; + bra.uni $L__BB6_512; + +$L__BB6_585: + shl.b32 %r2572, %r3285, 1; + or.b32 %r3285, %r2572, 1; + setp.eq.s32 %p683, %r3422, 0; + selp.b32 %r1425, 8, 7, %p683; + add.s32 %r3286, %r3286, 1; + setp.lt.u32 %p684, %r3286, %r1425; + @%p684 bra $L__BB6_589; + + sub.s32 %r3286, %r3286, %r1425; + setp.ge.u32 %p685, %r3419, %r3; + mov.u32 %r3450, 1; + @%p685 bra $L__BB6_588; + + cvt.u64.u32 %rd393, %r3419; + add.s64 %rd392, %rd4, %rd393; + shr.u32 %r2574, %r3285, %r3286; + cvt.u16.u32 %rs131, %r2574; + and.b16 %rs132, %rs131, 255; + st.global.u8 [%rd392], %rs131; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p686, %rs132, 255; + selp.u32 %r3422, 1, 0, %p686; + mov.u32 %r3450, %r3423; + +$L__BB6_588: + mov.u32 %r2575, -1; + shl.b32 %r2576, %r2575, %r3286; + not.b32 %r2577, %r2576; + setp.eq.s32 %p687, %r3286, 0; + selp.b32 %r2578, 0, %r2577, %p687; + and.b32 %r3285, %r2578, %r3285; + mov.u32 %r3423, %r3450; + bra.uni $L__BB6_589; + +$L__BB6_512: + shl.b32 %r3457, %r3285, 1; + setp.eq.s32 %p595, %r3422, 0; + selp.b32 %r1160, 8, 7, %p595; + add.s32 %r3458, %r3286, 1; + setp.lt.u32 %p596, %r3458, %r1160; + @%p596 bra $L__BB6_516; + + sub.s32 %r3458, %r3458, %r1160; + setp.ge.u32 %p597, %r3419, %r3; + mov.u32 %r3301, 1; + @%p597 bra $L__BB6_515; + + cvt.u64.u32 %rd391, %r3419; + add.s64 %rd390, %rd4, %rd391; + shr.u32 %r2447, %r3457, %r3458; + cvt.u16.u32 %rs101, %r2447; + and.b16 %rs102, %rs101, 255; + st.global.u8 [%rd390], %rs101; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p598, %rs102, 255; + selp.u32 %r3422, 1, 0, %p598; + mov.u32 %r3301, %r3423; + +$L__BB6_515: + mov.u32 %r2448, -1; + shl.b32 %r2449, %r2448, %r3458; + xor.b32 %r2450, %r2449, -2; + setp.eq.s32 %p599, %r3458, 0; + selp.b32 %r2451, 0, %r2450, %p599; + and.b32 %r3457, %r2451, %r3457; + mov.u32 %r3423, %r3301; + +$L__BB6_516: + setp.eq.s32 %p600, %r3298, 0; + @%p600 bra $L__BB6_550; + + add.s32 %r1174, %r3298, -1; + and.b32 %r1175, %r3298, 3; + setp.eq.s32 %p601, %r1175, 0; + mov.u32 %r3331, %r3298; + @%p601 bra $L__BB6_532; + + shr.u32 %r2453, %r1143, %r1174; + and.b32 %r2454, %r2453, 1; + bfi.b32 %r3457, %r3457, %r2454, 1, 31; + setp.eq.s32 %p602, %r3422, 0; + selp.b32 %r1177, 8, 7, %p602; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p603, %r3458, %r1177; + @%p603 bra $L__BB6_522; + + sub.s32 %r3458, %r3458, %r1177; + setp.ge.u32 %p604, %r3419, %r3; + mov.u32 %r3309, 1; + @%p604 bra $L__BB6_521; + + shr.u32 %r2456, %r3457, %r3458; + cvt.u16.u32 %rs103, %r2456; + and.b16 %rs104, %rs103, 255; + cvt.u64.u32 %rd309, %r3419; + add.s64 %rd310, %rd4, %rd309; + st.global.u8 [%rd310], %rs103; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p605, %rs104, 255; + selp.u32 %r3422, 1, 0, %p605; + mov.u32 %r3309, %r3423; + +$L__BB6_521: + mov.u32 %r2457, -1; + shl.b32 %r2458, %r2457, %r3458; + not.b32 %r2459, %r2458; + setp.eq.s32 %p606, %r3458, 0; + selp.b32 %r2460, 0, %r2459, %p606; + and.b32 %r3457, %r3457, %r2460; + mov.u32 %r3423, %r3309; + +$L__BB6_522: + setp.eq.s32 %p607, %r1175, 1; + mov.u32 %r3331, %r1174; + @%p607 bra $L__BB6_532; + + add.s32 %r3331, %r3298, -2; + shr.u32 %r2461, %r1143, %r3331; + and.b32 %r2462, %r2461, 1; + bfi.b32 %r3457, %r3457, %r2462, 1, 31; + setp.eq.s32 %p608, %r3422, 0; + selp.b32 %r1193, 8, 7, %p608; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p609, %r3458, %r1193; + @%p609 bra $L__BB6_527; + + sub.s32 %r3458, %r3458, %r1193; + setp.ge.u32 %p610, %r3419, %r3; + mov.u32 %r3317, 1; + @%p610 bra $L__BB6_526; + + shr.u32 %r2464, %r3457, %r3458; + cvt.u16.u32 %rs105, %r2464; + and.b16 %rs106, %rs105, 255; + cvt.u64.u32 %rd311, %r3419; + add.s64 %rd312, %rd4, %rd311; + st.global.u8 [%rd312], %rs105; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p611, %rs106, 255; + selp.u32 %r3422, 1, 0, %p611; + mov.u32 %r3317, %r3423; + +$L__BB6_526: + mov.u32 %r2465, -1; + shl.b32 %r2466, %r2465, %r3458; + not.b32 %r2467, %r2466; + setp.eq.s32 %p612, %r3458, 0; + selp.b32 %r2468, 0, %r2467, %p612; + and.b32 %r3457, %r3457, %r2468; + mov.u32 %r3423, %r3317; + +$L__BB6_527: + setp.eq.s32 %p613, %r1175, 2; + @%p613 bra $L__BB6_532; + + add.s32 %r3331, %r3298, -3; + shr.u32 %r2469, %r1143, %r3331; + and.b32 %r2470, %r2469, 1; + bfi.b32 %r3457, %r3457, %r2470, 1, 31; + setp.eq.s32 %p614, %r3422, 0; + selp.b32 %r1209, 8, 7, %p614; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p615, %r3458, %r1209; + @%p615 bra $L__BB6_532; + + sub.s32 %r3458, %r3458, %r1209; + setp.ge.u32 %p616, %r3419, %r3; + mov.u32 %r3325, 1; + @%p616 bra $L__BB6_531; + + shr.u32 %r2472, %r3457, %r3458; + cvt.u16.u32 %rs107, %r2472; + and.b16 %rs108, %rs107, 255; + cvt.u64.u32 %rd313, %r3419; + add.s64 %rd314, %rd4, %rd313; + st.global.u8 [%rd314], %rs107; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p617, %rs108, 255; + selp.u32 %r3422, 1, 0, %p617; + mov.u32 %r3325, %r3423; + +$L__BB6_531: + mov.u32 %r2473, -1; + shl.b32 %r2474, %r2473, %r3458; + not.b32 %r2475, %r2474; + setp.eq.s32 %p618, %r3458, 0; + selp.b32 %r2476, 0, %r2475, %p618; + and.b32 %r3457, %r3457, %r2476; + mov.u32 %r3423, %r3325; + +$L__BB6_532: + setp.lt.u32 %p619, %r1174, 3; + @%p619 bra $L__BB6_550; + +$L__BB6_533: + add.s32 %r2477, %r3331, -1; + shr.u32 %r2478, %r1143, %r2477; + and.b32 %r2479, %r2478, 1; + bfi.b32 %r3347, %r3457, %r2479, 1, 31; + setp.eq.s32 %p620, %r3422, 0; + selp.b32 %r1236, 8, 7, %p620; + add.s32 %r3348, %r3458, 1; + setp.lt.u32 %p621, %r3348, %r1236; + @%p621 bra $L__BB6_537; + + sub.s32 %r3348, %r3348, %r1236; + setp.ge.u32 %p622, %r3419, %r3; + mov.u32 %r3345, 1; + @%p622 bra $L__BB6_536; + + shr.u32 %r2481, %r3347, %r3348; + cvt.u16.u32 %rs109, %r2481; + and.b16 %rs110, %rs109, 255; + cvt.u64.u32 %rd315, %r3419; + add.s64 %rd316, %rd4, %rd315; + st.global.u8 [%rd316], %rs109; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p623, %rs110, 255; + selp.u32 %r3422, 1, 0, %p623; + mov.u32 %r3345, %r3423; + +$L__BB6_536: + mov.u32 %r2482, -1; + shl.b32 %r2483, %r2482, %r3348; + not.b32 %r2484, %r2483; + setp.eq.s32 %p624, %r3348, 0; + selp.b32 %r2485, 0, %r2484, %p624; + and.b32 %r3347, %r3347, %r2485; + mov.u32 %r3423, %r3345; + +$L__BB6_537: + add.s32 %r2486, %r3331, -2; + shr.u32 %r2487, %r1143, %r2486; + and.b32 %r2488, %r2487, 1; + bfi.b32 %r3355, %r3347, %r2488, 1, 31; + setp.eq.s32 %p625, %r3422, 0; + selp.b32 %r1251, 8, 7, %p625; + add.s32 %r3356, %r3348, 1; + setp.lt.u32 %p626, %r3356, %r1251; + @%p626 bra $L__BB6_541; + + sub.s32 %r3356, %r3356, %r1251; + setp.ge.u32 %p627, %r3419, %r3; + mov.u32 %r3353, 1; + @%p627 bra $L__BB6_540; + + shr.u32 %r2490, %r3355, %r3356; + cvt.u16.u32 %rs111, %r2490; + and.b16 %rs112, %rs111, 255; + cvt.u64.u32 %rd317, %r3419; + add.s64 %rd318, %rd4, %rd317; + st.global.u8 [%rd318], %rs111; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p628, %rs112, 255; + selp.u32 %r3422, 1, 0, %p628; + mov.u32 %r3353, %r3423; + +$L__BB6_540: + mov.u32 %r2491, -1; + shl.b32 %r2492, %r2491, %r3356; + not.b32 %r2493, %r2492; + setp.eq.s32 %p629, %r3356, 0; + selp.b32 %r2494, 0, %r2493, %p629; + and.b32 %r3355, %r3355, %r2494; + mov.u32 %r3423, %r3353; + +$L__BB6_541: + add.s32 %r2495, %r3331, -3; + shr.u32 %r2496, %r1143, %r2495; + and.b32 %r2497, %r2496, 1; + bfi.b32 %r3363, %r3355, %r2497, 1, 31; + setp.eq.s32 %p630, %r3422, 0; + selp.b32 %r1266, 8, 7, %p630; + add.s32 %r3364, %r3356, 1; + setp.lt.u32 %p631, %r3364, %r1266; + @%p631 bra $L__BB6_545; + + sub.s32 %r3364, %r3364, %r1266; + setp.ge.u32 %p632, %r3419, %r3; + mov.u32 %r3361, 1; + @%p632 bra $L__BB6_544; + + shr.u32 %r2499, %r3363, %r3364; + cvt.u16.u32 %rs113, %r2499; + and.b16 %rs114, %rs113, 255; + cvt.u64.u32 %rd319, %r3419; + add.s64 %rd320, %rd4, %rd319; + st.global.u8 [%rd320], %rs113; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p633, %rs114, 255; + selp.u32 %r3422, 1, 0, %p633; + mov.u32 %r3361, %r3423; + +$L__BB6_544: + mov.u32 %r2500, -1; + shl.b32 %r2501, %r2500, %r3364; + not.b32 %r2502, %r2501; + setp.eq.s32 %p634, %r3364, 0; + selp.b32 %r2503, 0, %r2502, %p634; + and.b32 %r3363, %r3363, %r2503; + mov.u32 %r3423, %r3361; + +$L__BB6_545: + add.s32 %r3331, %r3331, -4; + shr.u32 %r2504, %r1143, %r3331; + and.b32 %r2505, %r2504, 1; + bfi.b32 %r3457, %r3363, %r2505, 1, 31; + setp.eq.s32 %p635, %r3422, 0; + selp.b32 %r1282, 8, 7, %p635; + add.s32 %r3458, %r3364, 1; + setp.lt.u32 %p636, %r3458, %r1282; + @%p636 bra $L__BB6_549; + + sub.s32 %r3458, %r3458, %r1282; + setp.ge.u32 %p637, %r3419, %r3; + mov.u32 %r3369, 1; + @%p637 bra $L__BB6_548; + + shr.u32 %r2507, %r3457, %r3458; + cvt.u16.u32 %rs115, %r2507; + and.b16 %rs116, %rs115, 255; + cvt.u64.u32 %rd321, %r3419; + add.s64 %rd322, %rd4, %rd321; + st.global.u8 [%rd322], %rs115; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p638, %rs116, 255; + selp.u32 %r3422, 1, 0, %p638; + mov.u32 %r3369, %r3423; + +$L__BB6_548: + mov.u32 %r2508, -1; + shl.b32 %r2509, %r2508, %r3458; + not.b32 %r2510, %r2509; + setp.eq.s32 %p639, %r3458, 0; + selp.b32 %r2511, 0, %r2510, %p639; + and.b32 %r3457, %r3457, %r2511; + mov.u32 %r3423, %r3369; + +$L__BB6_549: + setp.ne.s32 %p640, %r3331, 0; + @%p640 bra $L__BB6_533; + +$L__BB6_550: + setp.lt.u32 %p728, %r274, 2; + @%p728 bra $L__BB6_240; + + setp.gt.u32 %p729, %r274, 2; + selp.u32 %r2746, 1, 0, %p729; + add.s32 %r1301, %r3297, %r2746; + setp.eq.s32 %p642, %r1301, 0; + @%p642 bra $L__BB6_240; + + add.s32 %r1302, %r1301, -1; + and.b32 %r1303, %r1301, 3; + setp.eq.s32 %p643, %r1303, 0; + mov.u32 %r3404, %r1301; + @%p643 bra $L__BB6_567; + + shr.u32 %r2513, %r273, %r1302; + and.b32 %r2514, %r2513, 1; + bfi.b32 %r3457, %r3457, %r2514, 1, 31; + setp.eq.s32 %p644, %r3422, 0; + selp.b32 %r1305, 8, 7, %p644; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p645, %r3458, %r1305; + @%p645 bra $L__BB6_557; + + sub.s32 %r3458, %r3458, %r1305; + setp.ge.u32 %p646, %r3419, %r3; + mov.u32 %r3382, 1; + @%p646 bra $L__BB6_556; + + shr.u32 %r2516, %r3457, %r3458; + cvt.u16.u32 %rs117, %r2516; + and.b16 %rs118, %rs117, 255; + cvt.u64.u32 %rd323, %r3419; + add.s64 %rd324, %rd4, %rd323; + st.global.u8 [%rd324], %rs117; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p647, %rs118, 255; + selp.u32 %r3422, 1, 0, %p647; + mov.u32 %r3382, %r3423; + +$L__BB6_556: + mov.u32 %r2517, -1; + shl.b32 %r2518, %r2517, %r3458; + not.b32 %r2519, %r2518; + setp.eq.s32 %p648, %r3458, 0; + selp.b32 %r2520, 0, %r2519, %p648; + and.b32 %r3457, %r3457, %r2520; + mov.u32 %r3423, %r3382; + +$L__BB6_557: + setp.eq.s32 %p649, %r1303, 1; + mov.u32 %r3404, %r1302; + @%p649 bra $L__BB6_567; + + add.s32 %r3404, %r1301, -2; + shr.u32 %r2521, %r273, %r3404; + and.b32 %r2522, %r2521, 1; + bfi.b32 %r3457, %r3457, %r2522, 1, 31; + setp.eq.s32 %p650, %r3422, 0; + selp.b32 %r1321, 8, 7, %p650; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p651, %r3458, %r1321; + @%p651 bra $L__BB6_562; + + sub.s32 %r3458, %r3458, %r1321; + setp.ge.u32 %p652, %r3419, %r3; + mov.u32 %r3390, 1; + @%p652 bra $L__BB6_561; + + shr.u32 %r2524, %r3457, %r3458; + cvt.u16.u32 %rs119, %r2524; + and.b16 %rs120, %rs119, 255; + cvt.u64.u32 %rd325, %r3419; + add.s64 %rd326, %rd4, %rd325; + st.global.u8 [%rd326], %rs119; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p653, %rs120, 255; + selp.u32 %r3422, 1, 0, %p653; + mov.u32 %r3390, %r3423; + +$L__BB6_561: + mov.u32 %r2525, -1; + shl.b32 %r2526, %r2525, %r3458; + not.b32 %r2527, %r2526; + setp.eq.s32 %p654, %r3458, 0; + selp.b32 %r2528, 0, %r2527, %p654; + and.b32 %r3457, %r3457, %r2528; + mov.u32 %r3423, %r3390; + +$L__BB6_562: + setp.eq.s32 %p655, %r1303, 2; + @%p655 bra $L__BB6_567; + + add.s32 %r3404, %r1301, -3; + shr.u32 %r2529, %r273, %r3404; + and.b32 %r2530, %r2529, 1; + bfi.b32 %r3457, %r3457, %r2530, 1, 31; + setp.eq.s32 %p656, %r3422, 0; + selp.b32 %r1337, 8, 7, %p656; + add.s32 %r3458, %r3458, 1; + setp.lt.u32 %p657, %r3458, %r1337; + @%p657 bra $L__BB6_567; + + sub.s32 %r3458, %r3458, %r1337; + setp.ge.u32 %p658, %r3419, %r3; + mov.u32 %r3398, 1; + @%p658 bra $L__BB6_566; + + shr.u32 %r2532, %r3457, %r3458; + cvt.u16.u32 %rs121, %r2532; + and.b16 %rs122, %rs121, 255; + cvt.u64.u32 %rd327, %r3419; + add.s64 %rd328, %rd4, %rd327; + st.global.u8 [%rd328], %rs121; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p659, %rs122, 255; + selp.u32 %r3422, 1, 0, %p659; + mov.u32 %r3398, %r3423; + +$L__BB6_566: + mov.u32 %r2533, -1; + shl.b32 %r2534, %r2533, %r3458; + not.b32 %r2535, %r2534; + setp.eq.s32 %p660, %r3458, 0; + selp.b32 %r2536, 0, %r2535, %p660; + and.b32 %r3457, %r3457, %r2536; + mov.u32 %r3423, %r3398; + +$L__BB6_567: + setp.lt.u32 %p661, %r1302, 3; + @%p661 bra $L__BB6_240; + +$L__BB6_568: + add.s32 %r2537, %r3404, -1; + shr.u32 %r2538, %r273, %r2537; + and.b32 %r2539, %r2538, 1; + bfi.b32 %r3420, %r3457, %r2539, 1, 31; + setp.eq.s32 %p662, %r3422, 0; + selp.b32 %r1364, 8, 7, %p662; + add.s32 %r3421, %r3458, 1; + setp.lt.u32 %p663, %r3421, %r1364; + @%p663 bra $L__BB6_572; + + sub.s32 %r3421, %r3421, %r1364; + setp.ge.u32 %p664, %r3419, %r3; + mov.u32 %r3418, 1; + @%p664 bra $L__BB6_571; + + shr.u32 %r2541, %r3420, %r3421; + cvt.u16.u32 %rs123, %r2541; + and.b16 %rs124, %rs123, 255; + cvt.u64.u32 %rd329, %r3419; + add.s64 %rd330, %rd4, %rd329; + st.global.u8 [%rd330], %rs123; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p665, %rs124, 255; + selp.u32 %r3422, 1, 0, %p665; + mov.u32 %r3418, %r3423; + +$L__BB6_571: + mov.u32 %r2542, -1; + shl.b32 %r2543, %r2542, %r3421; + not.b32 %r2544, %r2543; + setp.eq.s32 %p666, %r3421, 0; + selp.b32 %r2545, 0, %r2544, %p666; + and.b32 %r3420, %r3420, %r2545; + mov.u32 %r3423, %r3418; + +$L__BB6_572: + add.s32 %r2546, %r3404, -2; + shr.u32 %r2547, %r273, %r2546; + and.b32 %r2548, %r2547, 1; + bfi.b32 %r3428, %r3420, %r2548, 1, 31; + setp.eq.s32 %p667, %r3422, 0; + selp.b32 %r1379, 8, 7, %p667; + add.s32 %r3429, %r3421, 1; + setp.lt.u32 %p668, %r3429, %r1379; + @%p668 bra $L__BB6_576; + + sub.s32 %r3429, %r3429, %r1379; + setp.ge.u32 %p669, %r3419, %r3; + mov.u32 %r3426, 1; + @%p669 bra $L__BB6_575; + + shr.u32 %r2550, %r3428, %r3429; + cvt.u16.u32 %rs125, %r2550; + and.b16 %rs126, %rs125, 255; + cvt.u64.u32 %rd331, %r3419; + add.s64 %rd332, %rd4, %rd331; + st.global.u8 [%rd332], %rs125; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p670, %rs126, 255; + selp.u32 %r3422, 1, 0, %p670; + mov.u32 %r3426, %r3423; + +$L__BB6_575: + mov.u32 %r2551, -1; + shl.b32 %r2552, %r2551, %r3429; + not.b32 %r2553, %r2552; + setp.eq.s32 %p671, %r3429, 0; + selp.b32 %r2554, 0, %r2553, %p671; + and.b32 %r3428, %r3428, %r2554; + mov.u32 %r3423, %r3426; + +$L__BB6_576: + add.s32 %r2555, %r3404, -3; + shr.u32 %r2556, %r273, %r2555; + and.b32 %r2557, %r2556, 1; + bfi.b32 %r3436, %r3428, %r2557, 1, 31; + setp.eq.s32 %p672, %r3422, 0; + selp.b32 %r1394, 8, 7, %p672; + add.s32 %r3437, %r3429, 1; + setp.lt.u32 %p673, %r3437, %r1394; + @%p673 bra $L__BB6_580; + + sub.s32 %r3437, %r3437, %r1394; + setp.ge.u32 %p674, %r3419, %r3; + mov.u32 %r3434, 1; + @%p674 bra $L__BB6_579; + + shr.u32 %r2559, %r3436, %r3437; + cvt.u16.u32 %rs127, %r2559; + and.b16 %rs128, %rs127, 255; + cvt.u64.u32 %rd333, %r3419; + add.s64 %rd334, %rd4, %rd333; + st.global.u8 [%rd334], %rs127; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p675, %rs128, 255; + selp.u32 %r3422, 1, 0, %p675; + mov.u32 %r3434, %r3423; + +$L__BB6_579: + mov.u32 %r2560, -1; + shl.b32 %r2561, %r2560, %r3437; + not.b32 %r2562, %r2561; + setp.eq.s32 %p676, %r3437, 0; + selp.b32 %r2563, 0, %r2562, %p676; + and.b32 %r3436, %r3436, %r2563; + mov.u32 %r3423, %r3434; + +$L__BB6_580: + add.s32 %r3404, %r3404, -4; + shr.u32 %r2564, %r273, %r3404; + and.b32 %r2565, %r2564, 1; + bfi.b32 %r3457, %r3436, %r2565, 1, 31; + setp.eq.s32 %p677, %r3422, 0; + selp.b32 %r1410, 8, 7, %p677; + add.s32 %r3458, %r3437, 1; + setp.lt.u32 %p678, %r3458, %r1410; + @%p678 bra $L__BB6_584; + + sub.s32 %r3458, %r3458, %r1410; + setp.ge.u32 %p679, %r3419, %r3; + mov.u32 %r3442, 1; + @%p679 bra $L__BB6_583; + + shr.u32 %r2567, %r3457, %r3458; + cvt.u16.u32 %rs129, %r2567; + and.b16 %rs130, %rs129, 255; + cvt.u64.u32 %rd335, %r3419; + add.s64 %rd336, %rd4, %rd335; + st.global.u8 [%rd336], %rs129; + add.s32 %r3419, %r3419, 1; + setp.eq.s16 %p680, %rs130, 255; + selp.u32 %r3422, 1, 0, %p680; + mov.u32 %r3442, %r3423; + +$L__BB6_583: + mov.u32 %r2568, -1; + shl.b32 %r2569, %r2568, %r3458; + not.b32 %r2570, %r2569; + setp.eq.s32 %p681, %r3458, 0; + selp.b32 %r2571, 0, %r2570, %p681; + and.b32 %r3457, %r3457, %r2571; + mov.u32 %r3423, %r3442; + +$L__BB6_584: + setp.eq.s32 %p682, %r3404, 0; + @%p682 bra $L__BB6_240; + bra.uni $L__BB6_568; + +$L__BB6_240: + add.s32 %r2804, %r2804, 1; + setp.lt.u32 %p688, %r2804, %r24; + @%p688 bra $L__BB6_234; + +$L__BB6_241: + add.s32 %r2754, %r2754, 1; + setp.lt.u32 %p689, %r2754, %r2; + @%p689 bra $L__BB6_19; + bra.uni $L__BB6_242; + +$L__BB6_85: + st.local.u32 [%rd13+192], %r2760; + st.local.u32 [%rd13+196], %r2761; + st.local.u32 [%rd13+200], %r1581; + setp.eq.s32 %p131, %r2761, 0; + @%p131 bra $L__BB6_93; + + add.s32 %r1677, %r2761, -1; + and.b32 %r91, %r2761, 3; + setp.lt.u32 %p132, %r1677, 3; + mov.u32 %r2764, 0; + @%p132 bra $L__BB6_89; + + sub.s32 %r2763, %r2761, %r91; + mov.u32 %r2764, 0; + +$L__BB6_88: + mov.u32 %r2724, 0; + mul.wide.u32 %rd83, %r2764, 4; + add.s64 %rd84, %rd1, %rd83; + st.local.u32 [%rd84], %r2724; + st.local.u32 [%rd84+8192], %r2724; + st.local.u32 [%rd84+16384], %r2724; + st.local.u32 [%rd84+4], %r2724; + st.local.u32 [%rd84+8196], %r2724; + st.local.u32 [%rd84+16388], %r2724; + st.local.u32 [%rd84+8], %r2724; + st.local.u32 [%rd84+8200], %r2724; + st.local.u32 [%rd84+16392], %r2724; + st.local.u32 [%rd84+12], %r2724; + st.local.u32 [%rd84+8204], %r2724; + st.local.u32 [%rd84+16396], %r2724; + add.s32 %r2764, %r2764, 4; + add.s32 %r2763, %r2763, -4; + setp.ne.s32 %p133, %r2763, 0; + @%p133 bra $L__BB6_88; + +$L__BB6_89: + setp.eq.s32 %p134, %r91, 0; + @%p134 bra $L__BB6_93; + + mul.wide.u32 %rd85, %r2764, 4; + add.s64 %rd15, %rd1, %rd85; + mov.u32 %r1680, 0; + st.local.u32 [%rd15], %r1680; + st.local.u32 [%rd15+8192], %r1680; + st.local.u32 [%rd15+16384], %r1680; + setp.eq.s32 %p135, %r91, 1; + @%p135 bra $L__BB6_93; + + st.local.u32 [%rd15+4], %r1680; + st.local.u32 [%rd15+8196], %r1680; + st.local.u32 [%rd15+16388], %r1680; + setp.eq.s32 %p136, %r91, 2; + @%p136 bra $L__BB6_93; + + mov.u32 %r1682, 0; + st.local.u32 [%rd15+8], %r1682; + st.local.u32 [%rd15+8200], %r1682; + st.local.u32 [%rd15+16392], %r1682; + +$L__BB6_93: + or.b32 %r2723, %r25, %r26; + and.b32 %r2722, %r2723, -2; + setp.eq.s32 %p727, %r2722, 0; + mov.u32 %r1684, 0; + st.local.u32 [%rd14], %r25; + st.local.u32 [%rd14+64], %r26; + st.local.u32 [%rd14+128], %r1684; + mov.u32 %r2765, 1; + @%p727 bra $L__BB6_157; + + add.s32 %r1685, %r25, 1; + shr.u32 %r98, %r1685, 1; + add.s32 %r1686, %r26, 1; + shr.u32 %r99, %r1686, 1; + mul.lo.s32 %r100, %r98, %r99; + setp.eq.s32 %p138, %r98, 0; + @%p138 bra $L__BB6_96; + + div.u32 %r1687, %r100, %r98; + setp.ne.s32 %p139, %r1687, %r99; + @%p139 bra $L__BB6_152; + +$L__BB6_96: + add.s32 %r101, %r2766, %r100; + setp.lt.u32 %p140, %r101, %r2766; + setp.gt.u32 %p141, %r101, 2048; + or.pred %p142, %p140, %p141; + @%p142 bra $L__BB6_156; + bra.uni $L__BB6_97; + +$L__BB6_156: + mov.u32 %r1777, 1; + st.local.u32 [%rd14+200], %r1777; + bra.uni $L__BB6_230; + +$L__BB6_97: + st.local.u32 [%rd14+4], %r98; + st.local.u32 [%rd14+68], %r99; + st.local.u32 [%rd14+132], %r2766; + or.b32 %r1689, %r98, %r99; + and.b32 %r1690, %r1689, 2147483646; + setp.eq.s32 %p143, %r1690, 0; + mov.u32 %r2765, 2; + mov.u32 %r2766, %r101; + @%p143 bra $L__BB6_157; + + add.s32 %r1691, %r98, 1; + shr.u32 %r102, %r1691, 1; + add.s32 %r1692, %r99, 1; + shr.u32 %r103, %r1692, 1; + mul.lo.s32 %r104, %r102, %r103; + setp.eq.s32 %p144, %r102, 0; + @%p144 bra $L__BB6_100; + + div.u32 %r1693, %r104, %r102; + setp.ne.s32 %p145, %r1693, %r103; + @%p145 bra $L__BB6_152; + +$L__BB6_100: + add.s32 %r2766, %r101, %r104; + setp.lt.u32 %p146, %r2766, %r101; + setp.gt.u32 %p147, %r2766, 2048; + or.pred %p148, %p146, %p147; + @%p148 bra $L__BB6_156; + + st.local.u32 [%rd14+8], %r102; + st.local.u32 [%rd14+72], %r103; + st.local.u32 [%rd14+136], %r101; + or.b32 %r1695, %r102, %r103; + and.b32 %r1696, %r1695, 2147483646; + setp.eq.s32 %p149, %r1696, 0; + mov.u32 %r2765, 3; + @%p149 bra $L__BB6_157; + + add.s32 %r1697, %r102, 1; + shr.u32 %r106, %r1697, 1; + add.s32 %r1698, %r103, 1; + shr.u32 %r107, %r1698, 1; + mul.lo.s32 %r108, %r106, %r107; + setp.eq.s32 %p150, %r106, 0; + @%p150 bra $L__BB6_104; + + div.u32 %r1699, %r108, %r106; + setp.ne.s32 %p151, %r1699, %r107; + @%p151 bra $L__BB6_152; + +$L__BB6_104: + add.s32 %r109, %r2766, %r108; + setp.lt.u32 %p152, %r109, %r2766; + setp.gt.u32 %p153, %r109, 2048; + or.pred %p154, %p152, %p153; + @%p154 bra $L__BB6_156; + + st.local.u32 [%rd14+12], %r106; + st.local.u32 [%rd14+76], %r107; + st.local.u32 [%rd14+140], %r2766; + or.b32 %r1701, %r106, %r107; + and.b32 %r1702, %r1701, 2147483646; + setp.eq.s32 %p155, %r1702, 0; + mov.u32 %r2765, 4; + mov.u32 %r2766, %r109; + @%p155 bra $L__BB6_157; + + add.s32 %r1703, %r106, 1; + shr.u32 %r110, %r1703, 1; + add.s32 %r1704, %r107, 1; + shr.u32 %r111, %r1704, 1; + mul.lo.s32 %r112, %r110, %r111; + setp.eq.s32 %p156, %r110, 0; + @%p156 bra $L__BB6_108; + + div.u32 %r1705, %r112, %r110; + setp.ne.s32 %p157, %r1705, %r111; + @%p157 bra $L__BB6_152; + +$L__BB6_108: + add.s32 %r2766, %r109, %r112; + setp.lt.u32 %p158, %r2766, %r109; + setp.gt.u32 %p159, %r2766, 2048; + or.pred %p160, %p158, %p159; + @%p160 bra $L__BB6_156; + + st.local.u32 [%rd14+16], %r110; + st.local.u32 [%rd14+80], %r111; + st.local.u32 [%rd14+144], %r109; + or.b32 %r1707, %r110, %r111; + and.b32 %r1708, %r1707, 2147483646; + setp.eq.s32 %p161, %r1708, 0; + mov.u32 %r2765, 5; + @%p161 bra $L__BB6_157; + + add.s32 %r1709, %r110, 1; + shr.u32 %r114, %r1709, 1; + add.s32 %r1710, %r111, 1; + shr.u32 %r115, %r1710, 1; + mul.lo.s32 %r116, %r114, %r115; + setp.eq.s32 %p162, %r114, 0; + @%p162 bra $L__BB6_112; + + div.u32 %r1711, %r116, %r114; + setp.ne.s32 %p163, %r1711, %r115; + @%p163 bra $L__BB6_152; + +$L__BB6_112: + add.s32 %r117, %r2766, %r116; + setp.lt.u32 %p164, %r117, %r2766; + setp.gt.u32 %p165, %r117, 2048; + or.pred %p166, %p164, %p165; + @%p166 bra $L__BB6_156; + + st.local.u32 [%rd14+20], %r114; + st.local.u32 [%rd14+84], %r115; + st.local.u32 [%rd14+148], %r2766; + or.b32 %r1713, %r114, %r115; + and.b32 %r1714, %r1713, 2147483646; + setp.eq.s32 %p167, %r1714, 0; + mov.u32 %r2765, 6; + mov.u32 %r2766, %r117; + @%p167 bra $L__BB6_157; + + add.s32 %r1715, %r114, 1; + shr.u32 %r118, %r1715, 1; + add.s32 %r1716, %r115, 1; + shr.u32 %r119, %r1716, 1; + mul.lo.s32 %r120, %r118, %r119; + setp.eq.s32 %p168, %r118, 0; + @%p168 bra $L__BB6_116; + + div.u32 %r1717, %r120, %r118; + setp.ne.s32 %p169, %r1717, %r119; + @%p169 bra $L__BB6_152; + +$L__BB6_116: + add.s32 %r2766, %r117, %r120; + setp.lt.u32 %p170, %r2766, %r117; + setp.gt.u32 %p171, %r2766, 2048; + or.pred %p172, %p170, %p171; + @%p172 bra $L__BB6_156; + + st.local.u32 [%rd14+24], %r118; + st.local.u32 [%rd14+88], %r119; + st.local.u32 [%rd14+152], %r117; + or.b32 %r1719, %r118, %r119; + and.b32 %r1720, %r1719, 2147483646; + setp.eq.s32 %p173, %r1720, 0; + mov.u32 %r2765, 7; + @%p173 bra $L__BB6_157; + + add.s32 %r1721, %r118, 1; + shr.u32 %r122, %r1721, 1; + add.s32 %r1722, %r119, 1; + shr.u32 %r123, %r1722, 1; + mul.lo.s32 %r124, %r122, %r123; + setp.eq.s32 %p174, %r122, 0; + @%p174 bra $L__BB6_120; + + div.u32 %r1723, %r124, %r122; + setp.ne.s32 %p175, %r1723, %r123; + @%p175 bra $L__BB6_152; + +$L__BB6_120: + add.s32 %r125, %r2766, %r124; + setp.lt.u32 %p176, %r125, %r2766; + setp.gt.u32 %p177, %r125, 2048; + or.pred %p178, %p176, %p177; + @%p178 bra $L__BB6_156; + + st.local.u32 [%rd14+28], %r122; + st.local.u32 [%rd14+92], %r123; + st.local.u32 [%rd14+156], %r2766; + or.b32 %r1725, %r122, %r123; + and.b32 %r1726, %r1725, 2147483646; + setp.eq.s32 %p179, %r1726, 0; + mov.u32 %r2765, 8; + mov.u32 %r2766, %r125; + @%p179 bra $L__BB6_157; + + add.s32 %r1727, %r122, 1; + shr.u32 %r126, %r1727, 1; + add.s32 %r1728, %r123, 1; + shr.u32 %r127, %r1728, 1; + mul.lo.s32 %r128, %r126, %r127; + setp.eq.s32 %p180, %r126, 0; + @%p180 bra $L__BB6_124; + + div.u32 %r1729, %r128, %r126; + setp.ne.s32 %p181, %r1729, %r127; + @%p181 bra $L__BB6_152; + +$L__BB6_124: + add.s32 %r2766, %r125, %r128; + setp.lt.u32 %p182, %r2766, %r125; + setp.gt.u32 %p183, %r2766, 2048; + or.pred %p184, %p182, %p183; + @%p184 bra $L__BB6_156; + + st.local.u32 [%rd14+32], %r126; + st.local.u32 [%rd14+96], %r127; + st.local.u32 [%rd14+160], %r125; + or.b32 %r1731, %r126, %r127; + and.b32 %r1732, %r1731, 2147483646; + setp.eq.s32 %p185, %r1732, 0; + mov.u32 %r2765, 9; + @%p185 bra $L__BB6_157; + + add.s32 %r1733, %r126, 1; + shr.u32 %r130, %r1733, 1; + add.s32 %r1734, %r127, 1; + shr.u32 %r131, %r1734, 1; + mul.lo.s32 %r132, %r130, %r131; + setp.eq.s32 %p186, %r130, 0; + @%p186 bra $L__BB6_128; + + div.u32 %r1735, %r132, %r130; + setp.ne.s32 %p187, %r1735, %r131; + @%p187 bra $L__BB6_152; + +$L__BB6_128: + add.s32 %r133, %r2766, %r132; + setp.lt.u32 %p188, %r133, %r2766; + setp.gt.u32 %p189, %r133, 2048; + or.pred %p190, %p188, %p189; + @%p190 bra $L__BB6_156; + + st.local.u32 [%rd14+36], %r130; + st.local.u32 [%rd14+100], %r131; + st.local.u32 [%rd14+164], %r2766; + or.b32 %r1737, %r130, %r131; + and.b32 %r1738, %r1737, 2147483646; + setp.eq.s32 %p191, %r1738, 0; + mov.u32 %r2765, 10; + mov.u32 %r2766, %r133; + @%p191 bra $L__BB6_157; + + add.s32 %r1739, %r130, 1; + shr.u32 %r134, %r1739, 1; + add.s32 %r1740, %r131, 1; + shr.u32 %r135, %r1740, 1; + mul.lo.s32 %r136, %r134, %r135; + setp.eq.s32 %p192, %r134, 0; + @%p192 bra $L__BB6_132; + + div.u32 %r1741, %r136, %r134; + setp.ne.s32 %p193, %r1741, %r135; + @%p193 bra $L__BB6_152; + +$L__BB6_132: + add.s32 %r2766, %r133, %r136; + setp.lt.u32 %p194, %r2766, %r133; + setp.gt.u32 %p195, %r2766, 2048; + or.pred %p196, %p194, %p195; + @%p196 bra $L__BB6_156; + + st.local.u32 [%rd14+40], %r134; + st.local.u32 [%rd14+104], %r135; + st.local.u32 [%rd14+168], %r133; + or.b32 %r1743, %r134, %r135; + and.b32 %r1744, %r1743, 2147483646; + setp.eq.s32 %p197, %r1744, 0; + mov.u32 %r2765, 11; + @%p197 bra $L__BB6_157; + + add.s32 %r1745, %r134, 1; + shr.u32 %r138, %r1745, 1; + add.s32 %r1746, %r135, 1; + shr.u32 %r139, %r1746, 1; + mul.lo.s32 %r140, %r138, %r139; + setp.eq.s32 %p198, %r138, 0; + @%p198 bra $L__BB6_136; + + div.u32 %r1747, %r140, %r138; + setp.ne.s32 %p199, %r1747, %r139; + @%p199 bra $L__BB6_152; + +$L__BB6_136: + add.s32 %r141, %r2766, %r140; + setp.lt.u32 %p200, %r141, %r2766; + setp.gt.u32 %p201, %r141, 2048; + or.pred %p202, %p200, %p201; + @%p202 bra $L__BB6_156; + + st.local.u32 [%rd14+44], %r138; + st.local.u32 [%rd14+108], %r139; + st.local.u32 [%rd14+172], %r2766; + or.b32 %r1749, %r138, %r139; + and.b32 %r1750, %r1749, 2147483646; + setp.eq.s32 %p203, %r1750, 0; + mov.u32 %r2765, 12; + mov.u32 %r2766, %r141; + @%p203 bra $L__BB6_157; + + add.s32 %r1751, %r138, 1; + shr.u32 %r142, %r1751, 1; + add.s32 %r1752, %r139, 1; + shr.u32 %r143, %r1752, 1; + mul.lo.s32 %r144, %r142, %r143; + setp.eq.s32 %p204, %r142, 0; + @%p204 bra $L__BB6_140; + + div.u32 %r1753, %r144, %r142; + setp.ne.s32 %p205, %r1753, %r143; + @%p205 bra $L__BB6_152; + +$L__BB6_140: + add.s32 %r2766, %r141, %r144; + setp.lt.u32 %p206, %r2766, %r141; + setp.gt.u32 %p207, %r2766, 2048; + or.pred %p208, %p206, %p207; + @%p208 bra $L__BB6_156; + + st.local.u32 [%rd14+48], %r142; + st.local.u32 [%rd14+112], %r143; + st.local.u32 [%rd14+176], %r141; + or.b32 %r1755, %r142, %r143; + and.b32 %r1756, %r1755, 2147483646; + setp.eq.s32 %p209, %r1756, 0; + mov.u32 %r2765, 13; + @%p209 bra $L__BB6_157; + + add.s32 %r1757, %r142, 1; + shr.u32 %r146, %r1757, 1; + add.s32 %r1758, %r143, 1; + shr.u32 %r147, %r1758, 1; + mul.lo.s32 %r148, %r146, %r147; + setp.eq.s32 %p210, %r146, 0; + @%p210 bra $L__BB6_144; + + div.u32 %r1759, %r148, %r146; + setp.ne.s32 %p211, %r1759, %r147; + @%p211 bra $L__BB6_152; + +$L__BB6_144: + add.s32 %r149, %r2766, %r148; + setp.lt.u32 %p212, %r149, %r2766; + setp.gt.u32 %p213, %r149, 2048; + or.pred %p214, %p212, %p213; + @%p214 bra $L__BB6_156; + + st.local.u32 [%rd14+52], %r146; + st.local.u32 [%rd14+116], %r147; + st.local.u32 [%rd14+180], %r2766; + or.b32 %r1761, %r146, %r147; + and.b32 %r1762, %r1761, 2147483646; + setp.eq.s32 %p215, %r1762, 0; + mov.u32 %r2765, 14; + mov.u32 %r2766, %r149; + @%p215 bra $L__BB6_157; + + add.s32 %r1763, %r146, 1; + shr.u32 %r150, %r1763, 1; + add.s32 %r1764, %r147, 1; + shr.u32 %r151, %r1764, 1; + mul.lo.s32 %r152, %r150, %r151; + setp.eq.s32 %p216, %r150, 0; + @%p216 bra $L__BB6_148; + + div.u32 %r1765, %r152, %r150; + setp.ne.s32 %p217, %r1765, %r151; + @%p217 bra $L__BB6_152; + +$L__BB6_148: + add.s32 %r2766, %r149, %r152; + setp.lt.u32 %p218, %r2766, %r149; + setp.gt.u32 %p219, %r2766, 2048; + or.pred %p220, %p218, %p219; + @%p220 bra $L__BB6_156; + + st.local.u32 [%rd14+56], %r150; + st.local.u32 [%rd14+120], %r151; + st.local.u32 [%rd14+184], %r149; + or.b32 %r1767, %r150, %r151; + and.b32 %r1768, %r1767, 2147483646; + setp.eq.s32 %p221, %r1768, 0; + mov.u32 %r2765, 15; + @%p221 bra $L__BB6_157; + + add.s32 %r1769, %r150, 1; + shr.u32 %r154, %r1769, 1; + add.s32 %r1770, %r151, 1; + shr.u32 %r155, %r1770, 1; + mul.lo.s32 %r156, %r154, %r155; + setp.eq.s32 %p222, %r154, 0; + @%p222 bra $L__BB6_153; + + div.u32 %r1771, %r156, %r154; + setp.eq.s32 %p223, %r1771, %r155; + @%p223 bra $L__BB6_153; + bra.uni $L__BB6_152; + +$L__BB6_153: + add.s32 %r157, %r2766, %r156; + setp.lt.u32 %p224, %r157, %r2766; + setp.gt.u32 %p225, %r157, 2048; + or.pred %p226, %p224, %p225; + @%p226 bra $L__BB6_156; + + st.local.u32 [%rd14+60], %r154; + st.local.u32 [%rd14+124], %r155; + st.local.u32 [%rd14+188], %r2766; + or.b32 %r1774, %r154, %r155; + and.b32 %r1775, %r1774, 2147483646; + setp.eq.s32 %p227, %r1775, 0; + mov.u32 %r2765, 16; + mov.u32 %r2766, %r157; + @%p227 bra $L__BB6_157; + + mov.u32 %r1776, 1; + st.local.u32 [%rd14+200], %r1776; + bra.uni $L__BB6_230; + +$L__BB6_157: + st.local.u32 [%rd14+192], %r2765; + st.local.u32 [%rd14+196], %r2766; + st.local.u32 [%rd14+200], %r1684; + setp.eq.s32 %p228, %r2766, 0; + @%p228 bra $L__BB6_165; + + add.s32 %r1780, %r2766, -1; + and.b32 %r160, %r2766, 3; + setp.lt.u32 %p229, %r1780, 3; + mov.u32 %r2769, 0; + @%p229 bra $L__BB6_161; + + sub.s32 %r2768, %r2766, %r160; + mov.u32 %r1781, 0; + mov.u32 %r2769, %r1781; + +$L__BB6_160: + mul.wide.u32 %rd86, %r2769, 4; + add.s64 %rd87, %rd2, %rd86; + st.local.u32 [%rd87], %r1781; + st.local.u32 [%rd87+8192], %r1781; + st.local.u32 [%rd87+16384], %r1781; + st.local.u32 [%rd87+4], %r1781; + st.local.u32 [%rd87+8196], %r1781; + st.local.u32 [%rd87+16388], %r1781; + st.local.u32 [%rd87+8], %r1781; + st.local.u32 [%rd87+8200], %r1781; + st.local.u32 [%rd87+16392], %r1781; + st.local.u32 [%rd87+12], %r1781; + st.local.u32 [%rd87+8204], %r1781; + st.local.u32 [%rd87+16396], %r1781; + add.s32 %r2769, %r2769, 4; + add.s32 %r2768, %r2768, -4; + setp.ne.s32 %p230, %r2768, 0; + @%p230 bra $L__BB6_160; + +$L__BB6_161: + setp.eq.s32 %p231, %r160, 0; + @%p231 bra $L__BB6_165; + + mul.wide.u32 %rd88, %r2769, 4; + add.s64 %rd16, %rd2, %rd88; + mov.u32 %r1783, 0; + st.local.u32 [%rd16], %r1783; + st.local.u32 [%rd16+8192], %r1783; + st.local.u32 [%rd16+16384], %r1783; + setp.eq.s32 %p232, %r160, 1; + @%p232 bra $L__BB6_165; + + st.local.u32 [%rd16+4], %r1783; + st.local.u32 [%rd16+8196], %r1783; + st.local.u32 [%rd16+16388], %r1783; + setp.eq.s32 %p233, %r160, 2; + @%p233 bra $L__BB6_165; + + mov.u32 %r1785, 0; + st.local.u32 [%rd16+8], %r1785; + st.local.u32 [%rd16+8200], %r1785; + st.local.u32 [%rd16+16392], %r1785; + +$L__BB6_165: + setp.eq.s32 %p234, %r24, 0; + @%p234 bra $L__BB6_173; + + add.s32 %r1787, %r24, -1; + and.b32 %r167, %r24, 3; + setp.lt.u32 %p235, %r1787, 3; + mov.u32 %r2772, 0; + @%p235 bra $L__BB6_169; + + sub.s32 %r2771, %r24, %r167; + mov.u32 %r2772, 0; + +$L__BB6_168: + cvta.to.global.u64 %rd384, %rd60; + add.s32 %r1789, %r2772, %r23; + mul.wide.u32 %rd89, %r1789, 36; + add.s64 %rd90, %rd384, %rd89; + ld.global.u32 %r1790, [%rd90+20]; + ld.global.u32 %r1791, [%rd90+28]; + setp.eq.s32 %p236, %r1791, 0; + ld.global.u32 %r1792, [%rd90+32]; + selp.b32 %r1793, %r1792, 2147483647, %p236; + mul.wide.u32 %rd91, %r2772, 4; + add.s64 %rd92, %rd1, %rd91; + st.local.u32 [%rd92], %r1793; + add.s64 %rd93, %rd2, %rd91; + st.local.u32 [%rd93], %r1790; + add.s32 %r1794, %r1789, 1; + mul.wide.u32 %rd94, %r1794, 36; + add.s64 %rd95, %rd384, %rd94; + ld.global.u32 %r1795, [%rd95+20]; + ld.global.u32 %r1796, [%rd95+28]; + setp.eq.s32 %p237, %r1796, 0; + ld.global.u32 %r1797, [%rd95+32]; + selp.b32 %r1798, %r1797, 2147483647, %p237; + st.local.u32 [%rd92+4], %r1798; + st.local.u32 [%rd93+4], %r1795; + add.s32 %r1799, %r1789, 2; + mul.wide.u32 %rd96, %r1799, 36; + add.s64 %rd97, %rd384, %rd96; + ld.global.u32 %r1800, [%rd97+20]; + ld.global.u32 %r1801, [%rd97+28]; + setp.eq.s32 %p238, %r1801, 0; + ld.global.u32 %r1802, [%rd97+32]; + selp.b32 %r1803, %r1802, 2147483647, %p238; + st.local.u32 [%rd92+8], %r1803; + st.local.u32 [%rd93+8], %r1800; + add.s32 %r1804, %r1789, 3; + mul.wide.u32 %rd98, %r1804, 36; + add.s64 %rd99, %rd384, %rd98; + ld.global.u32 %r1805, [%rd99+20]; + ld.global.u32 %r1806, [%rd99+28]; + setp.eq.s32 %p239, %r1806, 0; + ld.global.u32 %r1807, [%rd99+32]; + selp.b32 %r1808, %r1807, 2147483647, %p239; + st.local.u32 [%rd92+12], %r1808; + st.local.u32 [%rd93+12], %r1805; + add.s32 %r2772, %r2772, 4; + add.s32 %r2771, %r2771, -4; + setp.ne.s32 %p240, %r2771, 0; + @%p240 bra $L__BB6_168; + +$L__BB6_169: + and.b32 %r2725, %r24, 3; + setp.eq.s32 %p241, %r2725, 0; + @%p241 bra $L__BB6_173; + + and.b32 %r2726, %r24, 3; + add.s32 %r1809, %r2772, %r23; + cvta.to.global.u64 %rd100, %rd60; + mul.wide.u32 %rd101, %r1809, 36; + add.s64 %rd102, %rd100, %rd101; + ld.global.u32 %r1810, [%rd102+20]; + ld.global.u32 %r1811, [%rd102+28]; + setp.eq.s32 %p242, %r1811, 0; + ld.global.u32 %r1812, [%rd102+32]; + selp.b32 %r1813, %r1812, 2147483647, %p242; + mul.wide.u32 %rd103, %r2772, 4; + add.s64 %rd18, %rd1, %rd103; + st.local.u32 [%rd18], %r1813; + add.s64 %rd19, %rd2, %rd103; + st.local.u32 [%rd19], %r1810; + setp.eq.s32 %p243, %r2726, 1; + @%p243 bra $L__BB6_173; + + mul.wide.u32 %rd371, %r2772, 4; + add.s64 %rd370, %rd1, %rd371; + add.s32 %r2686, %r2772, %r23; + cvta.to.global.u64 %rd369, %rd60; + and.b32 %r2685, %r24, 3; + add.s32 %r1815, %r2686, 1; + mul.wide.u32 %rd105, %r1815, 36; + add.s64 %rd106, %rd369, %rd105; + ld.global.u32 %r1816, [%rd106+20]; + ld.global.u32 %r1817, [%rd106+28]; + setp.eq.s32 %p244, %r1817, 0; + ld.global.u32 %r1818, [%rd106+32]; + selp.b32 %r1819, %r1818, 2147483647, %p244; + st.local.u32 [%rd370+4], %r1819; + st.local.u32 [%rd19+4], %r1816; + setp.eq.s32 %p245, %r2685, 2; + @%p245 bra $L__BB6_173; + + mul.wide.u32 %rd374, %r2772, 4; + add.s64 %rd373, %rd1, %rd374; + add.s32 %r2687, %r2772, %r23; + cvta.to.global.u64 %rd372, %rd60; + add.s32 %r1821, %r2687, 2; + mul.wide.u32 %rd108, %r1821, 36; + add.s64 %rd109, %rd372, %rd108; + ld.global.u32 %r1822, [%rd109+20]; + ld.global.u32 %r1823, [%rd109+28]; + setp.eq.s32 %p246, %r1823, 0; + ld.global.u32 %r1824, [%rd109+32]; + selp.b32 %r1825, %r1824, 2147483647, %p246; + st.local.u32 [%rd373+8], %r1825; + st.local.u32 [%rd19+8], %r1822; + +$L__BB6_173: + ld.local.u32 %r2786, [%rd13+192]; + setp.lt.u32 %p247, %r2786, 2; + @%p247 bra $L__BB6_193; + + mov.u32 %r2774, 1; + +$L__BB6_175: + add.s32 %r1827, %r2774, -1; + mul.wide.u32 %rd111, %r1827, 4; + add.s64 %rd20, %rd13, %rd111; + ld.local.u32 %r177, [%rd20]; + mul.wide.u32 %rd112, %r2774, 4; + add.s64 %rd22, %rd13, %rd112; + ld.local.u32 %r178, [%rd22]; + ld.local.u32 %r179, [%rd22+64]; + setp.eq.s32 %p248, %r179, 0; + @%p248 bra $L__BB6_192; + + ld.local.u32 %r180, [%rd20+64]; + mov.u32 %r2775, 0; + +$L__BB6_177: + setp.eq.s32 %p249, %r178, 0; + @%p249 bra $L__BB6_190; + + mov.u32 %r2776, 0; + +$L__BB6_179: + shl.b32 %r2659, %r2775, 1; + add.s32 %r2658, %r2659, 2; + min.u32 %r2657, %r2658, %r180; + shl.b32 %r2642, %r2775, 1; + setp.ge.u32 %p250, %r2642, %r2657; + mov.u32 %r2784, -1; + @%p250 bra $L__BB6_189; + + not.b32 %r2645, %r177; + shl.b32 %r2644, %r2776, 1; + shl.b32 %r2777, %r2775, 1; + mov.u32 %r1834, -2; + mov.u32 %r1835, -3; + sub.s32 %r1836, %r1835, %r2644; + max.u32 %r1837, %r1836, %r2645; + not.b32 %r1838, %r1837; + sub.s32 %r1839, %r1838, %r2644; + and.b32 %r190, %r1839, 3; + sub.s32 %r1840, %r1834, %r2644; + sub.s32 %r193, %r1840, %r1837; + mov.u32 %r2784, -1; + +$L__BB6_181: + shl.b32 %r2648, %r2776, 1; + add.s32 %r2647, %r2648, 2; + min.u32 %r2646, %r2647, %r177; + setp.ge.u32 %p251, %r2648, %r2646; + @%p251 bra $L__BB6_188; + + shl.b32 %r2779, %r2776, 1; + setp.eq.s32 %p252, %r190, 0; + ld.local.u32 %r196, [%rd20+128]; + mul.lo.s32 %r197, %r2777, %r177; + @%p252 bra $L__BB6_186; + + add.s32 %r2779, %r2644, 1; + shl.b32 %r2650, %r2776, 1; + setp.eq.s32 %p253, %r190, 1; + add.s32 %r1842, %r2650, %r197; + add.s32 %r1843, %r1842, %r196; + mul.wide.u32 %rd113, %r1843, 4; + add.s64 %rd114, %rd1, %rd113; + ld.local.u32 %r1844, [%rd114]; + min.u32 %r2784, %r2784, %r1844; + @%p253 bra $L__BB6_186; + + shl.b32 %r2631, %r2776, 1; + add.s32 %r2779, %r2631, 2; + add.s32 %r2629, %r2631, 1; + setp.eq.s32 %p254, %r190, 2; + add.s32 %r1845, %r2629, %r197; + add.s32 %r1846, %r1845, %r196; + mul.wide.u32 %rd115, %r1846, 4; + add.s64 %rd116, %rd1, %rd115; + ld.local.u32 %r1847, [%rd116]; + min.u32 %r2784, %r2784, %r1847; + @%p254 bra $L__BB6_186; + + shl.b32 %r2634, %r2776, 1; + add.s32 %r2779, %r2634, 3; + add.s32 %r2632, %r2634, 2; + add.s32 %r1848, %r2632, %r197; + add.s32 %r1849, %r1848, %r196; + mul.wide.u32 %rd117, %r1849, 4; + add.s64 %rd118, %rd1, %rd117; + ld.local.u32 %r1850, [%rd118]; + min.u32 %r2784, %r2784, %r1850; + +$L__BB6_186: + setp.lt.u32 %p255, %r193, 3; + @%p255 bra $L__BB6_188; + +$L__BB6_187: + shl.b32 %r2637, %r2776, 1; + add.s32 %r2636, %r2637, 2; + min.u32 %r2635, %r2636, %r177; + add.s32 %r1851, %r2779, %r197; + add.s32 %r1852, %r1851, %r196; + mul.wide.u32 %rd119, %r1852, 4; + add.s64 %rd120, %rd1, %rd119; + ld.local.u32 %r1853, [%rd120]; + min.u32 %r1854, %r2784, %r1853; + add.s32 %r1855, %r1852, 1; + mul.wide.u32 %rd121, %r1855, 4; + add.s64 %rd122, %rd1, %rd121; + ld.local.u32 %r1856, [%rd122]; + min.u32 %r1857, %r1854, %r1856; + add.s32 %r1858, %r1852, 2; + mul.wide.u32 %rd123, %r1858, 4; + add.s64 %rd124, %rd1, %rd123; + ld.local.u32 %r1859, [%rd124]; + min.u32 %r1860, %r1857, %r1859; + add.s32 %r1861, %r1852, 3; + mul.wide.u32 %rd125, %r1861, 4; + add.s64 %rd126, %rd1, %rd125; + ld.local.u32 %r1862, [%rd126]; + min.u32 %r2784, %r1860, %r1862; + add.s32 %r2779, %r2779, 4; + setp.lt.u32 %p256, %r2779, %r2635; + @%p256 bra $L__BB6_187; + +$L__BB6_188: + shl.b32 %r2654, %r2775, 1; + add.s32 %r2653, %r2654, 2; + min.u32 %r2652, %r2653, %r180; + add.s32 %r2777, %r2777, 1; + setp.lt.u32 %p257, %r2777, %r2652; + @%p257 bra $L__BB6_181; + +$L__BB6_189: + mul.wide.u32 %rd357, %r2774, 4; + add.s64 %rd356, %rd13, %rd357; + mul.lo.s32 %r2638, %r2775, %r178; + add.s32 %r1863, %r2776, %r2638; + ld.local.u32 %r1864, [%rd356+128]; + add.s32 %r1865, %r1863, %r1864; + mul.wide.u32 %rd127, %r1865, 4; + add.s64 %rd128, %rd1, %rd127; + st.local.u32 [%rd128], %r2784; + add.s32 %r2776, %r2776, 1; + setp.lt.u32 %p258, %r2776, %r178; + @%p258 bra $L__BB6_179; + +$L__BB6_190: + add.s32 %r2775, %r2775, 1; + setp.lt.u32 %p259, %r2775, %r179; + @%p259 bra $L__BB6_177; + + ld.local.u32 %r2786, [%rd13+192]; + +$L__BB6_192: + cvt.u64.u32 %rd358, %r2774; + cvt.u32.u64 %r1866, %rd358; + add.s32 %r2774, %r1866, 1; + setp.lt.u32 %p260, %r2774, %r2786; + @%p260 bra $L__BB6_175; + +$L__BB6_193: + ld.local.u32 %r2800, [%rd14+192]; + setp.lt.u32 %p261, %r2800, 2; + @%p261 bra $L__BB6_213; + + mov.u32 %r2788, 1; + +$L__BB6_195: + add.s32 %r1868, %r2788, -1; + mul.wide.u32 %rd130, %r1868, 4; + add.s64 %rd23, %rd14, %rd130; + ld.local.u32 %r219, [%rd23]; + mul.wide.u32 %rd131, %r2788, 4; + add.s64 %rd25, %rd14, %rd131; + ld.local.u32 %r220, [%rd25]; + ld.local.u32 %r221, [%rd25+64]; + setp.eq.s32 %p262, %r221, 0; + @%p262 bra $L__BB6_212; + + ld.local.u32 %r222, [%rd23+64]; + mov.u32 %r2789, 0; + +$L__BB6_197: + setp.eq.s32 %p263, %r220, 0; + @%p263 bra $L__BB6_210; + + mov.u32 %r2790, 0; + +$L__BB6_199: + shl.b32 %r2675, %r2789, 1; + add.s32 %r2674, %r2675, 2; + min.u32 %r2673, %r2674, %r222; + setp.ge.u32 %p264, %r2675, %r2673; + mov.u32 %r2798, -1; + @%p264 bra $L__BB6_209; + + not.b32 %r2678, %r219; + shl.b32 %r2677, %r2790, 1; + shl.b32 %r2791, %r2789, 1; + mov.u32 %r1875, -2; + mov.u32 %r1876, -3; + sub.s32 %r1877, %r1876, %r2677; + max.u32 %r1878, %r1877, %r2678; + not.b32 %r1879, %r1878; + sub.s32 %r1880, %r1879, %r2677; + and.b32 %r232, %r1880, 3; + sub.s32 %r1881, %r1875, %r2677; + sub.s32 %r235, %r1881, %r1878; + mov.u32 %r2798, -1; + +$L__BB6_201: + shl.b32 %r2681, %r2790, 1; + add.s32 %r2680, %r2681, 2; + min.u32 %r2679, %r2680, %r219; + setp.ge.u32 %p265, %r2681, %r2679; + @%p265 bra $L__BB6_208; + + shl.b32 %r2793, %r2790, 1; + setp.eq.s32 %p266, %r232, 0; + ld.local.u32 %r238, [%rd23+128]; + mul.lo.s32 %r239, %r2791, %r219; + @%p266 bra $L__BB6_206; + + add.s32 %r2793, %r2677, 1; + shl.b32 %r2683, %r2790, 1; + setp.eq.s32 %p267, %r232, 1; + add.s32 %r1883, %r2683, %r239; + add.s32 %r1884, %r1883, %r238; + mul.wide.u32 %rd132, %r1884, 4; + add.s64 %rd133, %rd2, %rd132; + ld.local.u32 %r1885, [%rd133]; + min.u32 %r2798, %r2798, %r1885; + @%p267 bra $L__BB6_206; + + shl.b32 %r2662, %r2790, 1; + add.s32 %r2793, %r2662, 2; + add.s32 %r2660, %r2662, 1; + setp.eq.s32 %p268, %r232, 2; + add.s32 %r1886, %r2660, %r239; + add.s32 %r1887, %r1886, %r238; + mul.wide.u32 %rd134, %r1887, 4; + add.s64 %rd135, %rd2, %rd134; + ld.local.u32 %r1888, [%rd135]; + min.u32 %r2798, %r2798, %r1888; + @%p268 bra $L__BB6_206; + + shl.b32 %r2665, %r2790, 1; + add.s32 %r2793, %r2665, 3; + add.s32 %r2663, %r2665, 2; + add.s32 %r1889, %r2663, %r239; + add.s32 %r1890, %r1889, %r238; + mul.wide.u32 %rd136, %r1890, 4; + add.s64 %rd137, %rd2, %rd136; + ld.local.u32 %r1891, [%rd137]; + min.u32 %r2798, %r2798, %r1891; + +$L__BB6_206: + setp.lt.u32 %p269, %r235, 3; + @%p269 bra $L__BB6_208; + +$L__BB6_207: + shl.b32 %r2668, %r2790, 1; + add.s32 %r2667, %r2668, 2; + min.u32 %r2666, %r2667, %r219; + add.s32 %r1892, %r2793, %r239; + add.s32 %r1893, %r1892, %r238; + mul.wide.u32 %rd138, %r1893, 4; + add.s64 %rd139, %rd2, %rd138; + ld.local.u32 %r1894, [%rd139]; + min.u32 %r1895, %r2798, %r1894; + add.s32 %r1896, %r1893, 1; + mul.wide.u32 %rd140, %r1896, 4; + add.s64 %rd141, %rd2, %rd140; + ld.local.u32 %r1897, [%rd141]; + min.u32 %r1898, %r1895, %r1897; + add.s32 %r1899, %r1893, 2; + mul.wide.u32 %rd142, %r1899, 4; + add.s64 %rd143, %rd2, %rd142; + ld.local.u32 %r1900, [%rd143]; + min.u32 %r1901, %r1898, %r1900; + add.s32 %r1902, %r1893, 3; + mul.wide.u32 %rd144, %r1902, 4; + add.s64 %rd145, %rd2, %rd144; + ld.local.u32 %r1903, [%rd145]; + min.u32 %r2798, %r1901, %r1903; + add.s32 %r2793, %r2793, 4; + setp.lt.u32 %p270, %r2793, %r2666; + @%p270 bra $L__BB6_207; + +$L__BB6_208: + shl.b32 %r2671, %r2789, 1; + add.s32 %r2670, %r2671, 2; + min.u32 %r2669, %r2670, %r222; + add.s32 %r2791, %r2791, 1; + setp.lt.u32 %p271, %r2791, %r2669; + @%p271 bra $L__BB6_201; + +$L__BB6_209: + mul.wide.u32 %rd367, %r2788, 4; + add.s64 %rd366, %rd14, %rd367; + mul.lo.s32 %r2672, %r2789, %r220; + add.s32 %r1904, %r2790, %r2672; + ld.local.u32 %r1905, [%rd366+128]; + add.s32 %r1906, %r1904, %r1905; + mul.wide.u32 %rd146, %r1906, 4; + add.s64 %rd147, %rd2, %rd146; + st.local.u32 [%rd147], %r2798; + add.s32 %r2790, %r2790, 1; + setp.lt.u32 %p272, %r2790, %r220; + @%p272 bra $L__BB6_199; + +$L__BB6_210: + add.s32 %r2789, %r2789, 1; + setp.lt.u32 %p273, %r2789, %r221; + @%p273 bra $L__BB6_197; + + ld.local.u32 %r2800, [%rd14+192]; + +$L__BB6_212: + cvt.u64.u32 %rd368, %r2788; + cvt.u32.u64 %r1907, %rd368; + add.s32 %r2788, %r1907, 1; + setp.lt.u32 %p274, %r2788, %r2800; + @%p274 bra $L__BB6_195; + +$L__BB6_213: + setp.eq.s64 %p275, %rd63, 0; + @%p275 bra $L__BB6_231; + + add.s32 %r2655, %r2754, %r1; + cvt.u64.u32 %rd364, %r2655; + setp.lt.u64 %p276, %rd364, %rd63; + @%p276 bra $L__BB6_216; + bra.uni $L__BB6_215; + +$L__BB6_216: + add.s32 %r2656, %r2754, %r1; + cvt.u64.u32 %rd365, %r2656; + shl.b64 %rd149, %rd365, 4; + add.s64 %rd150, %rd148, %rd149; + ld.global.u32 %rd26, [%rd150]; + ld.global.u32 %rd27, [%rd150+4]; + ld.global.u32 %r1909, [%rd150+8]; + cvt.u64.u32 %rd28, %r1909; + ld.local.u32 %r1910, [%rd13+196]; + setp.eq.s32 %p277, %r1909, %r1910; + @%p277 bra $L__BB6_218; + bra.uni $L__BB6_217; + +$L__BB6_218: + ld.param.u64 %rd359, [ j2k_htj2k_packetize_cleanup_param_8]; + add.s64 %rd151, %rd28, %rd26; + setp.gt.u64 %p278, %rd151, %rd359; + add.s64 %rd152, %rd28, %rd27; + setp.gt.u64 %p279, %rd152, %rd359; + or.pred %p280, %p278, %p279; + @%p280 bra $L__BB6_227; + bra.uni $L__BB6_219; + +$L__BB6_227: + mov.u32 %r1973, 1; + st.local.u32 [%rd13+200], %r1973; + st.local.u32 [%rd14+200], %r1973; + bra.uni $L__BB6_231; + +$L__BB6_152: + mov.u32 %r1772, 1; + st.local.u32 [%rd14+200], %r1772; + bra.uni $L__BB6_230; + +$L__BB6_215: + mov.u32 %r1908, 1; + st.local.u32 [%rd13+200], %r1908; + st.local.u32 [%rd14+200], %r1908; + bra.uni $L__BB6_231; + +$L__BB6_217: + mov.u32 %r1911, 1; + st.local.u32 [%rd13+200], %r1911; + st.local.u32 [%rd14+200], %r1911; + bra.uni $L__BB6_231; + +$L__BB6_219: + cvt.u32.u64 %r1912, %rd28; + setp.eq.s32 %p281, %r1912, 0; + @%p281 bra $L__BB6_231; + + add.s32 %r1915, %r1912, -1; + and.b32 %r258, %r1912, 3; + setp.lt.u32 %p282, %r1915, 3; + mov.u32 %r2803, 0; + @%p282 bra $L__BB6_223; + + sub.s32 %r2802, %r1912, %r258; + mov.u32 %r2803, 0; + +$L__BB6_222: + cvta.to.global.u64 %rd385, %rd62; + cvt.u32.u64 %r1918, %rd26; + add.s32 %r1919, %r2803, %r1918; + mul.wide.u32 %rd153, %r1919, 8; + add.s64 %rd154, %rd385, %rd153; + ld.global.u32 %r1920, [%rd154]; + ld.global.u32 %r1921, [%rd154+4]; + cvt.u32.u64 %r1922, %rd27; + add.s32 %r1923, %r2803, %r1922; + mul.wide.u32 %rd155, %r1923, 8; + add.s64 %rd156, %rd385, %rd155; + ld.global.u32 %r1924, [%rd156]; + ld.global.u32 %r1925, [%rd156+4]; + mul.wide.u32 %rd157, %r2803, 4; + add.s64 %rd158, %rd1, %rd157; + st.local.u32 [%rd158+8192], %r1920; + st.local.u32 [%rd158+16384], %r1921; + add.s64 %rd159, %rd2, %rd157; + st.local.u32 [%rd159+8192], %r1924; + st.local.u32 [%rd159+16384], %r1925; + add.s32 %r1926, %r2803, 1; + add.s32 %r1927, %r1926, %r1918; + mul.wide.u32 %rd160, %r1927, 8; + add.s64 %rd161, %rd385, %rd160; + ld.global.u32 %r1928, [%rd161]; + ld.global.u32 %r1929, [%rd161+4]; + add.s32 %r1930, %r1926, %r1922; + mul.wide.u32 %rd162, %r1930, 8; + add.s64 %rd163, %rd385, %rd162; + ld.global.u32 %r1931, [%rd163]; + ld.global.u32 %r1932, [%rd163+4]; + st.local.u32 [%rd158+8196], %r1928; + st.local.u32 [%rd158+16388], %r1929; + st.local.u32 [%rd159+8196], %r1931; + st.local.u32 [%rd159+16388], %r1932; + add.s32 %r1933, %r2803, 2; + add.s32 %r1934, %r1933, %r1918; + mul.wide.u32 %rd164, %r1934, 8; + add.s64 %rd165, %rd385, %rd164; + ld.global.u32 %r1935, [%rd165]; + ld.global.u32 %r1936, [%rd165+4]; + add.s32 %r1937, %r1933, %r1922; + mul.wide.u32 %rd166, %r1937, 8; + add.s64 %rd167, %rd385, %rd166; + ld.global.u32 %r1938, [%rd167]; + ld.global.u32 %r1939, [%rd167+4]; + st.local.u32 [%rd158+8200], %r1935; + st.local.u32 [%rd158+16392], %r1936; + st.local.u32 [%rd159+8200], %r1938; + st.local.u32 [%rd159+16392], %r1939; + add.s32 %r1940, %r2803, 3; + add.s32 %r1941, %r1940, %r1918; + mul.wide.u32 %rd168, %r1941, 8; + add.s64 %rd169, %rd385, %rd168; + ld.global.u32 %r1942, [%rd169]; + ld.global.u32 %r1943, [%rd169+4]; + add.s32 %r1944, %r1940, %r1922; + mul.wide.u32 %rd170, %r1944, 8; + add.s64 %rd171, %rd385, %rd170; + ld.global.u32 %r1945, [%rd171]; + ld.global.u32 %r1946, [%rd171+4]; + st.local.u32 [%rd158+8204], %r1942; + st.local.u32 [%rd158+16396], %r1943; + st.local.u32 [%rd159+8204], %r1945; + st.local.u32 [%rd159+16396], %r1946; + add.s32 %r2803, %r2803, 4; + add.s32 %r2802, %r2802, -4; + setp.ne.s32 %p283, %r2802, 0; + @%p283 bra $L__BB6_222; + +$L__BB6_223: + setp.eq.s32 %p284, %r258, 0; + @%p284 bra $L__BB6_231; + + cvt.u32.u64 %r1947, %rd27; + cvt.u32.u64 %r1948, %rd26; + add.s32 %r1949, %r2803, %r1948; + cvta.to.global.u64 %rd172, %rd62; + mul.wide.u32 %rd173, %r1949, 8; + add.s64 %rd174, %rd172, %rd173; + ld.global.u32 %r1950, [%rd174]; + ld.global.u32 %r1951, [%rd174+4]; + add.s32 %r1952, %r2803, %r1947; + mul.wide.u32 %rd175, %r1952, 8; + add.s64 %rd176, %rd172, %rd175; + ld.global.u32 %r1953, [%rd176]; + ld.global.u32 %r1954, [%rd176+4]; + mul.wide.u32 %rd177, %r2803, 4; + add.s64 %rd178, %rd1, %rd177; + add.s64 %rd30, %rd178, 8192; + st.local.u32 [%rd178+8192], %r1950; + st.local.u32 [%rd178+16384], %r1951; + add.s64 %rd179, %rd2, %rd177; + add.s64 %rd31, %rd179, 8192; + st.local.u32 [%rd179+8192], %r1953; + st.local.u32 [%rd179+16384], %r1954; + setp.eq.s32 %p285, %r258, 1; + @%p285 bra $L__BB6_231; + + cvta.to.global.u64 %rd386, %rd62; + add.s32 %r1955, %r2803, 1; + add.s32 %r1958, %r1955, %r1948; + mul.wide.u32 %rd181, %r1958, 8; + add.s64 %rd182, %rd386, %rd181; + ld.global.u32 %r1959, [%rd182]; + ld.global.u32 %r1960, [%rd182+4]; + add.s32 %r1961, %r1955, %r1947; + mul.wide.u32 %rd183, %r1961, 8; + add.s64 %rd184, %rd386, %rd183; + ld.global.u32 %r1962, [%rd184]; + ld.global.u32 %r1963, [%rd184+4]; + st.local.u32 [%rd30+4], %r1959; + st.local.u32 [%rd30+8196], %r1960; + st.local.u32 [%rd31+4], %r1962; + st.local.u32 [%rd31+8196], %r1963; + setp.eq.s32 %p286, %r258, 2; + @%p286 bra $L__BB6_231; + + cvta.to.global.u64 %rd387, %rd62; + add.s32 %r1964, %r2803, 2; + add.s32 %r1967, %r1964, %r1948; + mul.wide.u32 %rd186, %r1967, 8; + add.s64 %rd187, %rd387, %rd186; + ld.global.u32 %r1968, [%rd187]; + ld.global.u32 %r1969, [%rd187+4]; + add.s32 %r1970, %r1964, %r1947; + mul.wide.u32 %rd188, %r1970, 8; + add.s64 %rd189, %rd387, %rd188; + ld.global.u32 %r1971, [%rd189]; + ld.global.u32 %r1972, [%rd189+4]; + st.local.u32 [%rd30+8], %r1968; + st.local.u32 [%rd30+8200], %r1969; + st.local.u32 [%rd31+8], %r1971; + st.local.u32 [%rd31+8200], %r1972; + bra.uni $L__BB6_231; + +$L__BB6_590: + setp.eq.s32 %p709, %r3, 0; + mov.u32 %r3483, 0; + mov.u32 %r3482, 3; + mov.u32 %r3481, 1; + mov.u32 %r3484, %r3483; + mov.u32 %r3485, %r3483; + @%p709 bra $L__BB6_592; + + mov.u16 %rs135, 0; + st.global.u8 [%rd4], %rs135; + mov.u32 %r3483, 1; + mov.u32 %r3481, 0; + mov.u32 %r3482, %r3481; + mov.u32 %r3484, %r3481; + mov.u32 %r3485, %r3483; + bra.uni $L__BB6_592; + +$L__BB6_242: + setp.eq.s32 %p690, %r3458, 0; + mov.u32 %r3472, %r3423; + @%p690 bra $L__BB6_245; + + setp.eq.s32 %p691, %r3422, 0; + selp.b32 %r2580, 8, 7, %p691; + sub.s32 %r2581, %r2580, %r3458; + shl.b32 %r1458, %r3457, %r2581; + setp.ge.u32 %p692, %r3419, %r3; + mov.u32 %r3472, 1; + @%p692 bra $L__BB6_245; + + cvt.u64.u32 %rd337, %r3419; + add.s64 %rd338, %rd4, %rd337; + st.global.u8 [%rd338], %r1458; + add.s32 %r3419, %r3419, 1; + mov.u32 %r3472, %r3423; + +$L__BB6_245: + setp.ne.s32 %p693, %r3472, 0; + mov.u32 %r3483, 0; + mov.u32 %r3482, 4; + mov.u32 %r3481, 1; + mov.u32 %r3484, %r3483; + mov.u32 %r3485, %r3483; + @%p693 bra $L__BB6_592; + + setp.eq.s32 %p694, %r3419, 0; + mov.u32 %r3478, 0; + mov.u32 %r3473, %r3478; + @%p694 bra $L__BB6_250; + + add.s32 %r2588, %r3419, -1; + cvt.u64.u32 %rd339, %r2588; + add.s64 %rd340, %rd4, %rd339; + ld.global.u8 %rs133, [%rd340]; + setp.ne.s16 %p695, %rs133, 255; + mov.u32 %r3473, %r3419; + @%p695 bra $L__BB6_250; + + setp.ge.u32 %p696, %r3419, %r3; + mov.u32 %r3483, 0; + mov.u32 %r3482, 5; + mov.u32 %r3484, %r3483; + mov.u32 %r3485, %r3483; + @%p696 bra $L__BB6_592; + + cvt.u64.u32 %rd341, %r3419; + add.s64 %rd342, %rd4, %rd341; + mov.u16 %rs134, 0; + st.global.u8 [%rd342], %rs134; + add.s32 %r3473, %r3419, 1; + +$L__BB6_250: + setp.eq.s32 %p724, %r2, 0; + @%p724 bra $L__BB6_258; + + ld.param.u64 %rd363, [ j2k_htj2k_packetize_cleanup_param_3]; + cvta.to.global.u64 %rd47, %rd363; + mov.u32 %r3474, 0; + cvta.to.global.u64 %rd49, %rd60; + mov.u32 %r3478, %r3474; + +$L__BB6_252: + add.s32 %r2597, %r3474, %r1; + mul.wide.u32 %rd343, %r2597, 16; + add.s64 %rd48, %rd47, %rd343; + ld.global.u32 %r1466, [%rd48+4]; + setp.eq.s32 %p698, %r1466, 0; + @%p698 bra $L__BB6_257; + + ld.global.u32 %r1467, [%rd48]; + mov.u32 %r3476, 0; + +$L__BB6_254: + add.s32 %r2599, %r3476, %r1467; + mul.wide.u32 %rd344, %r2599, 36; + add.s64 %rd345, %rd49, %rd344; + ld.global.u32 %r2600, [%rd345+16]; + setp.eq.s32 %p699, %r2600, 0; + ld.global.u32 %r1470, [%rd345+4]; + setp.eq.s32 %p700, %r1470, 0; + or.pred %p701, %p700, %p699; + @%p701 bra $L__BB6_256; + + add.s32 %r3478, %r1470, %r3478; + setp.lt.u32 %p702, %r3478, %r1470; + add.s32 %r2606, %r3478, %r3473; + setp.lt.u32 %p703, %r2606, %r3478; + or.pred %p704, %p702, %p703; + setp.gt.u32 %p705, %r2606, %r3; + or.pred %p706, %p705, %p704; + mov.u32 %r3483, 0; + mov.u32 %r3482, 6; + mov.u32 %r3484, %r3483; + mov.u32 %r3485, %r3483; + @%p706 bra $L__BB6_592; + +$L__BB6_256: + add.s32 %r3476, %r3476, 1; + setp.lt.u32 %p707, %r3476, %r1466; + @%p707 bra $L__BB6_254; + +$L__BB6_257: + add.s32 %r3474, %r3474, 1; + setp.lt.u32 %p708, %r3474, %r2; + @%p708 bra $L__BB6_252; + +$L__BB6_258: + add.s32 %r3485, %r3478, %r3473; + mov.u32 %r3481, 0; + mov.u32 %r3482, %r3481; + mov.u32 %r3483, %r3473; + mov.u32 %r3484, %r3478; + +$L__BB6_592: + mov.u32 %r2639, %ctaid.x; + ld.param.u64 %rd360, [ j2k_htj2k_packetize_cleanup_param_10]; + mov.u32 %r2617, 0; + st.shared.u32 [_ZZ32 j2k_htj2k_packetize_cleanupE11shared_code], %r3481; + st.shared.u32 [_ZZ32 j2k_htj2k_packetize_cleanupE17shared_header_len], %r3483; + st.shared.u32 [_ZZ32 j2k_htj2k_packetize_cleanupE15shared_body_len], %r3484; + cvta.to.global.u64 %rd346, %rd360; + mul.wide.u32 %rd347, %r2639, 16; + add.s64 %rd348, %rd346, %rd347; + st.global.u32 [%rd348], %r3481; + st.global.u32 [%rd348+4], %r3482; + st.global.u32 [%rd348+8], %r3485; + st.global.u32 [%rd348+12], %r2617; + +$L__BB6_593: + bar.sync 0; + ld.shared.u32 %r2619, [_ZZ32 j2k_htj2k_packetize_cleanupE11shared_code]; + setp.ne.s32 %p710, %r2619, 0; + ld.shared.u32 %r1487, [_ZZ32 j2k_htj2k_packetize_cleanupE15shared_body_len]; + setp.eq.s32 %p711, %r1487, 0; + or.pred %p712, %p710, %p711; + @%p712 bra $L__BB6_608; + + mov.u32 %r2640, %tid.x; + setp.ge.u32 %p713, %r2640, %r1487; + @%p713 bra $L__BB6_608; + + setp.eq.s32 %p714, %r2, 0; + mov.u32 %r1488, %ntid.x; + @%p714 bra $L__BB6_606; + + ld.param.u64 %rd362, [ j2k_htj2k_packetize_cleanup_param_0]; + ld.param.u64 %rd361, [ j2k_htj2k_packetize_cleanup_param_3]; + ld.shared.u32 %r1489, [_ZZ32 j2k_htj2k_packetize_cleanupE17shared_header_len]; + mov.u32 %r3486, %tid.x; + cvta.to.global.u64 %rd50, %rd361; + cvta.to.global.u64 %rd53, %rd60; + cvta.to.global.u64 %rd354, %rd362; + +$L__BB6_597: + add.s32 %r2622, %r3486, %r1489; + cvt.u64.u32 %rd349, %r2622; + add.s64 %rd51, %rd4, %rd349; + mov.u32 %r3494, 0; + mov.u32 %r3491, %r3486; + +$L__BB6_598: + add.s32 %r2623, %r3494, %r1; + mul.wide.u32 %rd350, %r2623, 16; + add.s64 %rd52, %rd50, %rd350; + ld.global.u32 %r1494, [%rd52+4]; + setp.eq.s32 %p715, %r1494, 0; + @%p715 bra $L__BB6_604; + + ld.global.u32 %r3489, [%rd52]; + mov.u32 %r3490, 0; + +$L__BB6_600: + cvt.u64.u32 %rd54, %r3489; + mul.wide.u32 %rd351, %r3489, 36; + add.s64 %rd352, %rd53, %rd351; + add.s64 %rd55, %rd352, 4; + ld.global.u32 %r2625, [%rd352+16]; + setp.eq.s32 %p716, %r2625, 0; + ld.global.u32 %r1499, [%rd352+4]; + setp.eq.s32 %p717, %r1499, 0; + or.pred %p718, %p717, %p716; + @%p718 bra $L__BB6_603; + + setp.lt.u32 %p719, %r3491, %r1499; + @%p719 bra $L__BB6_609; + + sub.s32 %r3491, %r3491, %r1499; + +$L__BB6_603: + cvt.u32.u64 %r2628, %rd54; + add.s32 %r3489, %r2628, 1; + add.s32 %r3490, %r3490, 1; + setp.lt.u32 %p720, %r3490, %r1494; + @%p720 bra $L__BB6_600; + bra.uni $L__BB6_604; + +$L__BB6_609: + ld.global.u32 %r2626, [%rd55+-4]; + add.s32 %r2627, %r2626, %r3491; + cvt.u64.u32 %rd353, %r2627; + add.s64 %rd355, %rd354, %rd353; + ld.global.u8 %rs136, [%rd355]; + st.global.u8 [%rd51], %rs136; + mov.u32 %r3494, %r2; + +$L__BB6_604: + add.s32 %r3494, %r3494, 1; + setp.lt.u32 %p721, %r3494, %r2; + @%p721 bra $L__BB6_598; + + add.s32 %r3486, %r3486, %r1488; + setp.lt.u32 %p722, %r3486, %r1487; + @%p722 bra $L__BB6_597; + bra.uni $L__BB6_608; + +$L__BB6_606: + mov.u32 %r3495, %tid.x; + +$L__BB6_607: + add.s32 %r3495, %r3495, %r1488; + setp.lt.u32 %p723, %r3495, %r1487; + @%p723 bra $L__BB6_607; + +$L__BB6_608: + ret; + +} + + \ No newline at end of file diff --git a/crates/j2k-cuda-runtime/src/htj2k_packetize.rs b/crates/j2k-cuda-runtime/src/htj2k_packetize.rs new file mode 100644 index 00000000..25c32426 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/htj2k_packetize.rs @@ -0,0 +1,539 @@ +use crate::{ + build_flags::HTJ2K_ENCODE_PTX_BUILT_FROM_CUDA, + bytes::{ + htj2k_packetization_blocks_as_bytes, htj2k_packetization_packets_as_bytes, + htj2k_packetization_statuses_as_bytes, htj2k_packetization_statuses_as_bytes_mut, + htj2k_packetization_subband_tag_states_as_bytes, htj2k_packetization_subbands_as_bytes, + htj2k_packetization_tag_nodes_as_bytes, + }, + context::CudaContext, + driver::CuFunction, + error::CudaError, + execution::{cuda_kernel_param, CudaExecutionStats}, + htj2k_decode::{HTJ2K_STATUS_OK, HTJ2K_STATUS_UNSUPPORTED}, + kernels::{htj2k_packetize_launch_geometry, CudaKernel}, + memory::CudaDeviceBuffer, +}; + +/// One HTJ2K packet prepared for CUDA Tier-2 packetization. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaHtj2kPacketizationPacket { + /// First block metadata row for this packet. + pub block_start: u32, + /// Number of block metadata rows in this packet. + pub block_count: u32, + /// First subband metadata row for this packet. + pub subband_start: u32, + /// Number of subband metadata rows in this packet. + pub subband_count: u32, + /// Maximum bytes reserved for this packet's header and body. + pub output_capacity: u32, + /// Packet layer index used for first-inclusion tag-tree coding. + pub layer: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct CudaHtj2kPacketizationKernelPacket { + pub(crate) block_start: u32, + pub(crate) block_count: u32, + pub(crate) subband_start: u32, + pub(crate) subband_count: u32, + pub(crate) output_offset: u32, + pub(crate) output_capacity: u32, + pub(crate) layer: u32, +} + +/// One HTJ2K packet subband layout for CUDA packetization. +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaHtj2kPacketizationSubband { + /// First code-block metadata row for this subband. + pub block_start: u32, + /// Number of code-block metadata rows in this subband. + pub block_count: u32, + /// Number of code-blocks in the x direction. + pub num_cbs_x: u32, + /// Number of code-blocks in the y direction. + pub num_cbs_y: u32, +} + +/// Initial tag-tree state for one HTJ2K packet subband. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaHtj2kPacketizationSubbandTagState { + /// First inclusion tag-tree node state row for this packet subband. + pub inclusion_node_start: u32, + /// First zero-bitplane tag-tree node state row for this packet subband. + pub zero_bitplane_node_start: u32, + /// Number of node state rows in each tree. + pub node_count: u32, + /// Reserved for ABI stability. + pub reserved0: u32, +} + +/// Current/known state for one HTJ2K packet tag-tree node. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaHtj2kPacketizationTagNodeState { + /// Tag-tree current value before this packet is emitted. + pub current: u32, + /// Nonzero when this node value is already known before this packet. + pub known: u32, +} + +/// One HTJ2K code-block contribution for CUDA packetization. +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaHtj2kPacketizationBlock { + /// Byte offset into the contiguous encoded code-block payload. + pub data_offset: u32, + /// Encoded code-block payload length in bytes. + pub data_len: u32, + /// HTJ2K cleanup segment length in bytes. + pub cleanup_length: u32, + /// HTJ2K refinement segment length in bytes. + pub refinement_length: u32, + /// Number of coding passes in this contribution. + pub num_coding_passes: u32, + /// Number of zero most-significant bitplanes before first inclusion. + pub num_zero_bitplanes: u32, + /// L-block value for segment-length coding. + pub l_block: u32, + /// Nonzero when this code block was included in an earlier packet for the same packet state. + pub previously_included: u32, + /// First packet layer where this code block is included, or tag-tree infinity when absent. + pub inclusion_layer: u32, +} + +/// Status written by the CUDA HTJ2K packetizer for one packet. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaHtj2kPacketizationStatus { + /// Zero on success; nonzero values are kernel-defined failures. + pub code: u32, + /// Kernel-defined failure detail. + pub detail: u32, + /// Number of packet bytes written into this packet slot. + pub output_len: u32, + /// Reserved for ABI stability. + pub reserved0: u32, +} + +impl CudaHtj2kPacketizationStatus { + /// Return true when the CUDA kernel reported success. + pub fn is_ok(self) -> bool { + self.code == HTJ2K_STATUS_OK + } +} + +/// CUDA event timings for HTJ2K Tier-2 packetization stages. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaHtj2kPacketizationStageTimings { + /// Cleanup packetization dispatch time, in microseconds. + pub packetize_us: u128, +} + +/// Host-visible HTJ2K packet payload produced by the CUDA Tier-2 packetizer. +#[derive(Debug)] +pub struct CudaHtj2kPacketizedTile { + pub(crate) data: Vec, + pub(crate) statuses: Vec, + pub(crate) execution: CudaExecutionStats, + pub(crate) stage_timings: CudaHtj2kPacketizationStageTimings, +} + +impl CudaHtj2kPacketizedTile { + /// Concatenated tile packet payload bytes. + pub fn data(&self) -> &[u8] { + &self.data + } + + /// Per-packet kernel status rows downloaded after dispatch. + pub fn statuses(&self) -> &[CudaHtj2kPacketizationStatus] { + &self.statuses + } + + /// CUDA execution counters for the packetization dispatch. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// CUDA event timings for the packetization dispatch. + pub fn stage_timings(&self) -> CudaHtj2kPacketizationStageTimings { + self.stage_timings + } +} + +impl CudaContext { + /// Packetize HTJ2K code-block payloads with CUDA. + pub fn packetize_htj2k_cleanup_packets( + &self, + payload: &[u8], + packets: &[CudaHtj2kPacketizationPacket], + subbands: &[CudaHtj2kPacketizationSubband], + blocks: &[CudaHtj2kPacketizationBlock], + ) -> Result { + self.packetize_htj2k_cleanup_packets_with_tag_state( + payload, + packets, + subbands, + blocks, + &[], + &[], + ) + } + + /// Packetize HTJ2K code-block payloads with CUDA using caller-provided tag-tree state. + pub fn packetize_htj2k_cleanup_packets_with_tag_state( + &self, + payload: &[u8], + packets: &[CudaHtj2kPacketizationPacket], + subbands: &[CudaHtj2kPacketizationSubband], + blocks: &[CudaHtj2kPacketizationBlock], + subband_tag_states: &[CudaHtj2kPacketizationSubbandTagState], + tag_nodes: &[CudaHtj2kPacketizationTagNodeState], + ) -> Result { + self.inner.set_current()?; + if !HTJ2K_ENCODE_PTX_BUILT_FROM_CUDA + && blocks.iter().any(|block| block.num_coding_passes > 1) + { + return Err(CudaError::InvalidArgument { + message: "multi-pass HTJ2K packetization requires CUDA PTX rebuilt from htj2k_encode_kernels.cu".to_string(), + }); + } + let kernel_packets = + htj2k_packetization_kernel_packets(packets, subbands, blocks, payload.len())?; + validate_htj2k_packetization_tag_state(subbands, subband_tag_states, tag_nodes)?; + let total_output = kernel_packets.iter().try_fold(0usize, |acc, packet| { + let end = usize::try_from(packet.output_offset) + .ok() + .and_then(|offset| offset.checked_add(packet.output_capacity as usize)) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + Ok::(acc.max(end)) + })?; + let output_buffer = self.allocate(total_output)?; + if packets.is_empty() { + return Ok(CudaHtj2kPacketizedTile { + data: Vec::new(), + statuses: Vec::new(), + execution: CudaExecutionStats::default(), + stage_timings: CudaHtj2kPacketizationStageTimings::default(), + }); + } + + let payload_buffer = self.upload_pinned(payload)?; + let packet_buffer = self.upload(htj2k_packetization_packets_as_bytes(&kernel_packets))?; + let subband_buffer = self.upload(htj2k_packetization_subbands_as_bytes(subbands))?; + let block_buffer = self.upload(htj2k_packetization_blocks_as_bytes(blocks))?; + let subband_tag_state_buffer = self.upload( + htj2k_packetization_subband_tag_states_as_bytes(subband_tag_states), + )?; + let tag_node_buffer = self.upload(htj2k_packetization_tag_nodes_as_bytes(tag_nodes))?; + let initial_statuses = vec![ + CudaHtj2kPacketizationStatus { + code: HTJ2K_STATUS_UNSUPPORTED, + ..CudaHtj2kPacketizationStatus::default() + }; + packets.len() + ]; + let status_buffer = + self.upload(htj2k_packetization_statuses_as_bytes(&initial_statuses))?; + + let ((), packetize_us) = + self.time_default_stream_named_us("j2k.htj2k.encode.packetize", || { + self.launch_htj2k_packetize_cleanup( + &payload_buffer, + payload.len(), + &packet_buffer, + &subband_buffer, + &block_buffer, + &subband_tag_state_buffer, + &tag_node_buffer, + subband_tag_states.len(), + tag_nodes.len(), + &output_buffer, + &status_buffer, + packets.len(), + ) + })?; + let stage_timings = CudaHtj2kPacketizationStageTimings { packetize_us }; + + let mut statuses = vec![CudaHtj2kPacketizationStatus::default(); packets.len()]; + status_buffer.copy_to_host(htj2k_packetization_statuses_as_bytes_mut(&mut statuses))?; + if let Some(status) = statuses.iter().copied().find(|status| !status.is_ok()) { + return Err(CudaError::KernelStatus { + kernel: "j2k_htj2k_packetize_cleanup", + code: status.code, + detail: status.detail, + }); + } + + let mut data = Vec::new(); + for (packet, status) in kernel_packets.iter().zip(&statuses) { + if status.output_len > packet.output_capacity { + return Err(CudaError::LengthTooLarge { + len: status.output_len as usize, + }); + } + let start = packet.output_offset as usize; + let end = start + .checked_add(status.output_len as usize) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if end > output_buffer.byte_len() { + return Err(CudaError::LengthTooLarge { len: end }); + } + let previous_len = data.len(); + data.resize(previous_len + status.output_len as usize, 0); + output_buffer.copy_range_to_host(start, &mut data[previous_len..])?; + } + + Ok(CudaHtj2kPacketizedTile { + data, + statuses, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + stage_timings, + }) + } + + #[allow(clippy::too_many_arguments)] + fn launch_htj2k_packetize_cleanup( + &self, + payload: &CudaDeviceBuffer, + payload_len: usize, + packets: &CudaDeviceBuffer, + subbands: &CudaDeviceBuffer, + blocks: &CudaDeviceBuffer, + subband_tag_states: &CudaDeviceBuffer, + tag_nodes: &CudaDeviceBuffer, + subband_tag_state_count: usize, + tag_node_count: usize, + output: &CudaDeviceBuffer, + statuses: &CudaDeviceBuffer, + packet_count: usize, + ) -> Result<(), CudaError> { + let function = self.htj2k_packetize_kernel_function(CudaKernel::Htj2kPacketizeCleanup)?; + let mut payload_ptr = payload.device_ptr(); + let mut payload_len_u64 = u64::try_from(payload_len) + .map_err(|_| CudaError::LengthTooLarge { len: payload_len })?; + let mut packets_ptr = packets.device_ptr(); + let mut subbands_ptr = subbands.device_ptr(); + let mut blocks_ptr = blocks.device_ptr(); + let mut subband_tag_states_ptr = subband_tag_states.device_ptr(); + let mut tag_nodes_ptr = tag_nodes.device_ptr(); + let mut subband_tag_state_count_u64 = + u64::try_from(subband_tag_state_count).map_err(|_| CudaError::LengthTooLarge { + len: subband_tag_state_count, + })?; + let mut tag_node_count_u64 = + u64::try_from(tag_node_count).map_err(|_| CudaError::LengthTooLarge { + len: tag_node_count, + })?; + let mut output_ptr = output.device_ptr(); + let mut statuses_ptr = statuses.device_ptr(); + let mut packet_count_u64 = u64::try_from(packet_count) + .map_err(|_| CudaError::LengthTooLarge { len: packet_count })?; + let mut params = cuda_kernel_params!( + payload_ptr, + payload_len_u64, + packets_ptr, + subbands_ptr, + blocks_ptr, + subband_tag_states_ptr, + tag_nodes_ptr, + subband_tag_state_count_u64, + tag_node_count_u64, + output_ptr, + statuses_ptr, + packet_count_u64 + ); + let geometry = htj2k_packetize_launch_geometry(packet_count) + .ok_or(CudaError::LengthTooLarge { len: packet_count })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn htj2k_packetize_kernel_function(&self, kernel: CudaKernel) -> Result { + #[cfg(feature = "cuda-oxide-j2k-encode")] + { + if crate::build_flags::cuda_oxide_j2k_encode_enabled() + && kernel.is_cuda_oxide_j2k_encode_stage() + { + return self.inner.cuda_oxide_j2k_encode_kernel_function(kernel); + } + } + self.inner.kernel_function(kernel) + } +} + +pub(crate) fn htj2k_packetization_kernel_packets( + packets: &[CudaHtj2kPacketizationPacket], + subbands: &[CudaHtj2kPacketizationSubband], + blocks: &[CudaHtj2kPacketizationBlock], + payload_len: usize, +) -> Result, CudaError> { + let mut output_offset = 0usize; + let mut kernel_packets = Vec::with_capacity(packets.len()); + for packet in packets { + let block_start = packet.block_start as usize; + let block_count = packet.block_count as usize; + let block_end = block_start + .checked_add(block_count) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if block_end > blocks.len() { + return Err(CudaError::LengthTooLarge { len: block_end }); + } + let subband_start = packet.subband_start as usize; + let subband_count = packet.subband_count as usize; + let subband_end = subband_start + .checked_add(subband_count) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if subband_end > subbands.len() { + return Err(CudaError::LengthTooLarge { len: subband_end }); + } + for subband in &subbands[subband_start..subband_end] { + if subband.num_cbs_x == 0 || subband.num_cbs_y == 0 { + return Err(CudaError::LengthTooLarge { len: 0 }); + } + let subband_block_start = subband.block_start as usize; + let subband_block_count = subband.block_count as usize; + let subband_block_end = subband_block_start + .checked_add(subband_block_count) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if subband_block_start < block_start || subband_block_end > block_end { + return Err(CudaError::LengthTooLarge { + len: subband_block_end, + }); + } + let expected_blocks = (subband.num_cbs_x as usize) + .checked_mul(subband.num_cbs_y as usize) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if expected_blocks != subband_block_count { + return Err(CudaError::LengthTooLarge { + len: expected_blocks, + }); + } + } + for block in &blocks[block_start..block_end] { + let data_end = (block.data_offset as usize) + .checked_add(block.data_len as usize) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if data_end > payload_len { + return Err(CudaError::LengthTooLarge { len: data_end }); + } + } + let output_capacity = packet.output_capacity as usize; + let next_output = output_offset + .checked_add(output_capacity) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if next_output > u32::MAX as usize { + return Err(CudaError::LengthTooLarge { len: next_output }); + } + kernel_packets.push(CudaHtj2kPacketizationKernelPacket { + block_start: packet.block_start, + block_count: packet.block_count, + subband_start: packet.subband_start, + subband_count: packet.subband_count, + output_offset: u32::try_from(output_offset) + .map_err(|_| CudaError::LengthTooLarge { len: output_offset })?, + output_capacity: packet.output_capacity, + layer: packet.layer, + }); + output_offset = next_output; + } + Ok(kernel_packets) +} + +pub(crate) fn validate_htj2k_packetization_tag_state( + subbands: &[CudaHtj2kPacketizationSubband], + subband_tag_states: &[CudaHtj2kPacketizationSubbandTagState], + tag_nodes: &[CudaHtj2kPacketizationTagNodeState], +) -> Result<(), CudaError> { + if subband_tag_states.is_empty() { + if tag_nodes.is_empty() { + return Ok(()); + } + return Err(CudaError::InvalidArgument { + message: "HTJ2K packetization tag nodes require subband tag states".to_string(), + }); + } + if subband_tag_states.len() != subbands.len() { + return Err(CudaError::InvalidArgument { + message: "HTJ2K packetization subband tag-state count must match subband count" + .to_string(), + }); + } + for (subband_index, (subband, state)) in subbands.iter().zip(subband_tag_states).enumerate() { + let expected_node_count = + htj2k_packetization_tag_tree_node_count(subband.num_cbs_x, subband.num_cbs_y)?; + if state.node_count as usize != expected_node_count { + return Err(CudaError::InvalidArgument { + message: format!( + "HTJ2K packetization tag-state node count does not match subband {subband_index}" + ), + }); + } + let node_count = state.node_count as usize; + let inclusion_end = (state.inclusion_node_start as usize) + .checked_add(node_count) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + let zero_bitplane_end = (state.zero_bitplane_node_start as usize) + .checked_add(node_count) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if inclusion_end > tag_nodes.len() || zero_bitplane_end > tag_nodes.len() { + return Err(CudaError::InvalidArgument { + message: format!( + "HTJ2K packetization tag-state offsets exceed tag node count at subband {subband_index}" + ), + }); + } + } + Ok(()) +} + +pub(crate) const HTJ2K_PACKET_MAX_TAG_NODES: usize = 2048; + +pub(crate) const HTJ2K_PACKET_MAX_TAG_LEVELS: usize = 16; + +pub(crate) fn htj2k_packetization_tag_tree_node_count( + width: u32, + height: u32, +) -> Result { + if width == 0 || height == 0 { + return Err(CudaError::InvalidArgument { + message: "HTJ2K packetization tag-tree dimensions must be nonzero".to_string(), + }); + } + let mut levels = 0usize; + let mut total = 0usize; + let mut w = width as usize; + let mut h = height as usize; + loop { + if levels >= HTJ2K_PACKET_MAX_TAG_LEVELS { + return Err(CudaError::InvalidArgument { + message: "HTJ2K packetization tag-tree exceeds kernel level bounds".to_string(), + }); + } + let nodes = w + .checked_mul(h) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + total = total + .checked_add(nodes) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if total > HTJ2K_PACKET_MAX_TAG_NODES { + return Err(CudaError::InvalidArgument { + message: "HTJ2K packetization tag-tree exceeds kernel node bounds".to_string(), + }); + } + levels += 1; + if w <= 1 && h <= 1 { + return Ok(total); + } + w = w.div_ceil(2); + h = h.div_ceil(2); + } +} diff --git a/crates/j2k-cuda-runtime/src/j2k_decode.rs b/crates/j2k-cuda-runtime/src/j2k_decode.rs new file mode 100644 index 00000000..497ac88a --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_decode.rs @@ -0,0 +1,2125 @@ +use crate::{ + bytes::{ + idwt_job_as_bytes, idwt_multi_jobs_as_bytes, inverse_mct_job_as_bytes, + store_gray16_job_as_bytes, store_gray8_job_as_bytes, store_rgb16_job_as_bytes, + store_rgb16_mct_job_as_bytes, store_rgb8_job_as_bytes, store_rgb8_mct_batch_jobs_as_bytes, + }, + context::{cuda_idwt_trace_enabled, CudaContext}, + driver::CuDevicePtr, + error::CudaError, + execution::{ + cuda_kernel_param, elapsed_event_us_ceil, CudaExecutionStats, CudaKernelBatchOutput, + CudaKernelContiguousBatchOutput, CudaKernelOutput, CudaLaunchMode, CudaPooledKernelOutput, + CudaQueuedExecution, + }, + kernels::{ + j2k_dwt53_launch_geometry, j2k_forward_rct_launch_geometry, + j2k_idwt_multi_1d_launch_geometry, j2k_idwt_multi_coop_axis_launch_geometry, + j2k_idwt_multi_coop_columns_launch_geometry, j2k_idwt_multi_coop_launch_geometry, + j2k_store_batch_launch_geometry, CudaKernel, + }, + memory::{ + checked_image_words, pooled_device_buffer, CudaBufferPool, CudaDeviceBuffer, + CudaDeviceBufferRange, + }, +}; + +/// CUDA-side integer rectangle for JPEG 2000 direct-plan kernels. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaJ2kRect { + /// Inclusive minimum x coordinate. + pub x0: u32, + /// Inclusive minimum y coordinate. + pub y0: u32, + /// Exclusive maximum x coordinate. + pub x1: u32, + /// Exclusive maximum y coordinate. + pub y1: u32, +} + +/// One single-decomposition inverse DWT dispatch over device coefficient bands. +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaJ2kIdwtJob { + /// Output rectangle produced by the IDWT stage. + pub rect: CudaJ2kRect, + /// LL input band rectangle. + pub ll_rect: CudaJ2kRect, + /// HL input band rectangle. + pub hl_rect: CudaJ2kRect, + /// LH input band rectangle. + pub lh_rect: CudaJ2kRect, + /// HH input band rectangle. + pub hh_rect: CudaJ2kRect, + /// Nonzero for irreversible 9/7; zero for reversible 5/3. + pub irreversible97: u32, +} + +/// One output buffer and input band set for batched inverse DWT. +#[derive(Clone, Copy, Debug)] +pub struct CudaJ2kIdwtTarget<'a> { + /// LL input band. + pub ll: &'a CudaDeviceBuffer, + /// HL input band. + pub hl: &'a CudaDeviceBuffer, + /// LH input band. + pub lh: &'a CudaDeviceBuffer, + /// HH input band. + pub hh: &'a CudaDeviceBuffer, + /// Output buffer for the reconstructed band. + pub output: &'a CudaDeviceBuffer, + /// IDWT geometry and transform metadata. + pub job: CudaJ2kIdwtJob, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug)] +pub(crate) struct CudaJ2kIdwtMultiKernelJob { + pub(crate) ll_ptr: u64, + pub(crate) hl_ptr: u64, + pub(crate) lh_ptr: u64, + pub(crate) hh_ptr: u64, + pub(crate) output_ptr: u64, + pub(crate) job: CudaJ2kIdwtJob, +} + +/// Grayscale store dispatch from f32 component samples to tightly packed Gray8. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CudaJ2kStoreGray8Job { + /// Source component buffer width in samples. + pub input_width: u32, + /// Source x offset in samples. + pub source_x: u32, + /// Source y offset in samples. + pub source_y: u32, + /// Number of samples copied per row. + pub copy_width: u32, + /// Number of rows copied. + pub copy_height: u32, + /// Destination output width in samples. + pub output_width: u32, + /// Destination output height in rows. + pub output_height: u32, + /// Destination x offset in samples. + pub output_x: u32, + /// Destination y offset in samples. + pub output_y: u32, + /// Level-shift addend applied before quantizing to Gray8. + pub addend: f32, + /// Source component bit depth. + pub bit_depth: u32, +} + +/// Grayscale store dispatch from f32 component samples to tightly packed Gray16. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CudaJ2kStoreGray16Job { + /// Source component buffer width in samples. + pub input_width: u32, + /// Source x offset in samples. + pub source_x: u32, + /// Source y offset in samples. + pub source_y: u32, + /// Number of samples copied per row. + pub copy_width: u32, + /// Number of rows copied. + pub copy_height: u32, + /// Destination output width in samples. + pub output_width: u32, + /// Destination output height in rows. + pub output_height: u32, + /// Destination x offset in samples. + pub output_x: u32, + /// Destination y offset in samples. + pub output_y: u32, + /// Level-shift addend applied before quantizing to Gray16. + pub addend: f32, + /// Source component bit depth. + pub bit_depth: u32, +} + +/// In-place inverse MCT dispatch over three device f32 component planes. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CudaJ2kInverseMctJob { + /// Number of samples in each component plane. + pub len: u32, + /// Nonzero for irreversible ICT; zero for reversible RCT. + pub irreversible97: u32, + /// Addend applied to output channel 0 after inverse MCT. + pub addend0: f32, + /// Addend applied to output channel 1 after inverse MCT. + pub addend1: f32, + /// Addend applied to output channel 2 after inverse MCT. + pub addend2: f32, +} + +/// RGB/RGBA store dispatch from three f32 component planes to packed 8-bit pixels. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CudaJ2kStoreRgb8Job { + /// Source width for component 0. + pub input_width0: u32, + /// Source width for component 1. + pub input_width1: u32, + /// Source width for component 2. + pub input_width2: u32, + /// Source x offset for component 0. + pub source_x0: u32, + /// Source y offset for component 0. + pub source_y0: u32, + /// Source x offset for component 1. + pub source_x1: u32, + /// Source y offset for component 1. + pub source_y1: u32, + /// Source x offset for component 2. + pub source_x2: u32, + /// Source y offset for component 2. + pub source_y2: u32, + /// Number of pixels copied per row. + pub copy_width: u32, + /// Number of rows copied. + pub copy_height: u32, + /// Destination output width in pixels. + pub output_width: u32, + /// Destination output height in rows. + pub output_height: u32, + /// Destination x offset. + pub output_x: u32, + /// Destination y offset. + pub output_y: u32, + /// Addend applied to component 0 before quantizing. + pub addend0: f32, + /// Addend applied to component 1 before quantizing. + pub addend1: f32, + /// Addend applied to component 2 before quantizing. + pub addend2: f32, + /// Source bit depth for component 0. + pub bit_depth0: u32, + /// Source bit depth for component 1. + pub bit_depth1: u32, + /// Source bit depth for component 2. + pub bit_depth2: u32, + /// Nonzero to write RGBA8 with opaque alpha; zero writes RGB8. + pub rgba: u32, +} + +/// RGB/RGBA store dispatch from three f32 component planes to packed 16-bit pixels. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CudaJ2kStoreRgb16Job { + /// Source width for component 0. + pub input_width0: u32, + /// Source width for component 1. + pub input_width1: u32, + /// Source width for component 2. + pub input_width2: u32, + /// Source x offset for component 0. + pub source_x0: u32, + /// Source y offset for component 0. + pub source_y0: u32, + /// Source x offset for component 1. + pub source_x1: u32, + /// Source y offset for component 1. + pub source_y1: u32, + /// Source x offset for component 2. + pub source_x2: u32, + /// Source y offset for component 2. + pub source_y2: u32, + /// Number of pixels copied per row. + pub copy_width: u32, + /// Number of rows copied. + pub copy_height: u32, + /// Destination output width in pixels. + pub output_width: u32, + /// Destination output height in rows. + pub output_height: u32, + /// Destination x offset. + pub output_x: u32, + /// Destination y offset. + pub output_y: u32, + /// Addend applied to component 0 before quantizing. + pub addend0: f32, + /// Addend applied to component 1 before quantizing. + pub addend1: f32, + /// Addend applied to component 2 before quantizing. + pub addend2: f32, + /// Source bit depth for component 0. + pub bit_depth0: u32, + /// Source bit depth for component 1. + pub bit_depth1: u32, + /// Source bit depth for component 2. + pub bit_depth2: u32, + /// Nonzero to write RGBA16 with opaque alpha; zero writes RGB16. + pub rgba: u32, +} + +/// Fused inverse RCT/ICT and packed RGB8/RGBA8 store dispatch. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CudaJ2kStoreRgb8MctJob { + /// RGB/RGBA store geometry, addends, bit depths, and alpha mode. + pub store: CudaJ2kStoreRgb8Job, + /// Nonzero for irreversible ICT; zero for reversible RCT. + pub irreversible97: u32, +} + +/// One fused inverse MCT plus RGB8/RGBA8 store item for a batched dispatch. +#[derive(Clone, Copy, Debug)] +pub struct CudaJ2kStoreRgb8MctTarget<'a> { + /// Source component plane 0. + pub plane0: &'a CudaDeviceBuffer, + /// Source component plane 1. + pub plane1: &'a CudaDeviceBuffer, + /// Source component plane 2. + pub plane2: &'a CudaDeviceBuffer, + /// Store geometry and inverse MCT parameters. + pub job: CudaJ2kStoreRgb8MctJob, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct CudaJ2kStoreRgb8MctBatchJob { + pub(crate) plane0_ptr: CuDevicePtr, + pub(crate) plane1_ptr: CuDevicePtr, + pub(crate) plane2_ptr: CuDevicePtr, + pub(crate) output_ptr: CuDevicePtr, + pub(crate) job: CudaJ2kStoreRgb8MctJob, +} + +/// Fused inverse RCT/ICT and packed RGB16/RGBA16 store dispatch. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct CudaJ2kStoreRgb16MctJob { + /// RGB/RGBA store geometry, addends, bit depths, and alpha mode. + pub store: CudaJ2kStoreRgb16Job, + /// Nonzero for irreversible ICT; zero for reversible RCT. + pub irreversible97: u32, +} + +impl CudaContext { + /// Apply one inverse JPEG 2000 DWT decomposition to device coefficient bands. + pub fn j2k_inverse_dwt_single_device( + &self, + ll: &CudaDeviceBuffer, + hl: &CudaDeviceBuffer, + lh: &CudaDeviceBuffer, + hh: &CudaDeviceBuffer, + job: CudaJ2kIdwtJob, + ) -> Result { + self.j2k_inverse_dwt_single_device_impl(ll, hl, lh, hh, job, true) + } + + /// Apply one inverse JPEG 2000 DWT decomposition without per-kernel synchronizes. + pub fn j2k_inverse_dwt_single_device_untimed( + &self, + ll: &CudaDeviceBuffer, + hl: &CudaDeviceBuffer, + lh: &CudaDeviceBuffer, + hh: &CudaDeviceBuffer, + job: CudaJ2kIdwtJob, + ) -> Result { + self.j2k_inverse_dwt_single_device_impl(ll, hl, lh, hh, job, false) + } + + /// Apply one inverse JPEG 2000 DWT decomposition with caller-owned + /// transient buffer reuse. + pub fn j2k_inverse_dwt_single_device_with_pool( + &self, + ll: &CudaDeviceBuffer, + hl: &CudaDeviceBuffer, + lh: &CudaDeviceBuffer, + hh: &CudaDeviceBuffer, + job: CudaJ2kIdwtJob, + pool: &CudaBufferPool, + ) -> Result { + self.j2k_inverse_dwt_single_device_with_pool_impl(ll, hl, lh, hh, job, true, pool) + } + + /// Apply one inverse JPEG 2000 DWT decomposition with caller-owned + /// transient buffer reuse and without per-kernel synchronizes. + pub fn j2k_inverse_dwt_single_device_untimed_with_pool( + &self, + ll: &CudaDeviceBuffer, + hl: &CudaDeviceBuffer, + lh: &CudaDeviceBuffer, + hh: &CudaDeviceBuffer, + job: CudaJ2kIdwtJob, + pool: &CudaBufferPool, + ) -> Result { + self.j2k_inverse_dwt_single_device_with_pool_impl(ll, hl, lh, hh, job, false, pool) + } + + /// Apply inverse JPEG 2000 DWT decompositions for multiple independent + /// targets using one dispatch per parallel stage. + pub fn j2k_inverse_dwt_batch_device_with_pool( + &self, + targets: &[CudaJ2kIdwtTarget<'_>], + pool: &CudaBufferPool, + ) -> Result { + self.j2k_inverse_dwt_batch_device_with_pool_impl(targets, pool, true) + } + + /// Apply inverse JPEG 2000 DWT decompositions for multiple independent + /// targets without per-stage synchronizes. + pub fn j2k_inverse_dwt_batch_device_untimed_with_pool( + &self, + targets: &[CudaJ2kIdwtTarget<'_>], + pool: &CudaBufferPool, + ) -> Result { + self.j2k_inverse_dwt_batch_device_with_pool_impl(targets, pool, false) + } + + /// Enqueue batched inverse JPEG 2000 DWT decompositions without + /// synchronizing. The returned value must be kept live until the default + /// stream has been synchronized by the caller. + pub fn j2k_inverse_dwt_batch_device_enqueue_with_pool( + &self, + targets: &[CudaJ2kIdwtTarget<'_>], + pool: &CudaBufferPool, + ) -> Result { + self.inner.set_current()?; + let kernel_jobs = j2k_idwt_multi_kernel_jobs(targets)?; + if kernel_jobs.is_empty() { + return Ok(CudaQueuedExecution { + resources: Vec::new(), + execution: CudaExecutionStats::default(), + }); + } + let jobs_buffer = pool.upload(idwt_multi_jobs_as_bytes(&kernel_jobs))?; + let jobs_device = pooled_device_buffer(&jobs_buffer)?; + let max_width = kernel_jobs + .iter() + .map(|job| job.job.rect.x1.saturating_sub(job.job.rect.x0)) + .max() + .unwrap_or(0); + let max_height = kernel_jobs + .iter() + .map(|job| job.job.rect.y1.saturating_sub(job.job.rect.y0)) + .max() + .unwrap_or(0); + let kernel_mode = idwt_batch_kernel_mode(&kernel_jobs, max_width, max_height); + let interleave_horizontal_result = match kernel_mode { + CudaJ2kIdwtBatchKernelMode::Cooperative53 => self + .launch_j2k_idwt_interleave_horizontal_53_multi( + jobs_device, + max_height as usize, + kernel_jobs.len(), + false, + ), + CudaJ2kIdwtBatchKernelMode::Cooperative97 => self + .launch_j2k_idwt_interleave_horizontal_97_multi_ptr( + jobs_device.device_ptr(), + max_width as usize, + max_height as usize, + kernel_jobs.len(), + false, + ), + CudaJ2kIdwtBatchKernelMode::Generic => self + .launch_j2k_idwt_interleave_horizontal_multi( + jobs_device, + max_height as usize, + kernel_jobs.len(), + false, + ), + }; + if let Err(error) = interleave_horizontal_result { + let _ = self.synchronize(); + return Err(error); + } + let vertical_result = match kernel_mode { + CudaJ2kIdwtBatchKernelMode::Cooperative53 => self.launch_j2k_idwt_vertical_53_multi( + jobs_device, + max_width as usize, + kernel_jobs.len(), + false, + ), + CudaJ2kIdwtBatchKernelMode::Cooperative97 => self + .launch_j2k_idwt_vertical_97_multi_ptr( + jobs_device.device_ptr(), + max_width as usize, + max_height as usize, + kernel_jobs.len(), + false, + ), + CudaJ2kIdwtBatchKernelMode::Generic => self.launch_j2k_idwt_vertical_multi( + jobs_device, + max_width as usize, + kernel_jobs.len(), + false, + ), + }; + if let Err(error) = vertical_result { + let _ = self.synchronize(); + return Err(error); + } + + Ok(CudaQueuedExecution { + resources: vec![jobs_buffer], + execution: CudaExecutionStats { + kernel_dispatches: 2, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 2, + hardware_decode: false, + }, + }) + } + + /// Enqueue a sequence of batched inverse JPEG 2000 DWT stages while + /// uploading all stage job metadata in one device buffer. The returned + /// value must be kept live until the default stream has been synchronized + /// by the caller. + #[allow(clippy::too_many_lines)] + pub fn j2k_inverse_dwt_batch_sequence_enqueue_with_pool( + &self, + target_batches: &[&[CudaJ2kIdwtTarget<'_>]], + pool: &CudaBufferPool, + ) -> Result { + self.inner.set_current()?; + let mut all_jobs = Vec::new(); + let mut batches = Vec::new(); + for targets in target_batches { + let kernel_jobs = j2k_idwt_multi_kernel_jobs(targets)?; + if kernel_jobs.is_empty() { + continue; + } + let start = all_jobs.len(); + let count = kernel_jobs.len(); + let max_width = kernel_jobs + .iter() + .map(|job| job.job.rect.x1.saturating_sub(job.job.rect.x0)) + .max() + .unwrap_or(0); + let max_height = kernel_jobs + .iter() + .map(|job| job.job.rect.y1.saturating_sub(job.job.rect.y0)) + .max() + .unwrap_or(0); + let kernel_mode = idwt_batch_kernel_mode(&kernel_jobs, max_width, max_height); + all_jobs.extend(kernel_jobs); + batches.push((start, count, max_width, max_height, kernel_mode)); + } + if all_jobs.is_empty() { + return Ok(CudaQueuedExecution { + resources: Vec::new(), + execution: CudaExecutionStats::default(), + }); + } + + let jobs_buffer = pool.upload(idwt_multi_jobs_as_bytes(&all_jobs))?; + let jobs_base = pooled_device_buffer(&jobs_buffer)?.device_ptr(); + let job_size = std::mem::size_of::(); + let mut kernel_dispatches = 0usize; + let trace_enabled = cuda_idwt_trace_enabled(); + for (stage_index, (start, count, max_width, max_height, kernel_mode)) in + batches.into_iter().enumerate() + { + let byte_offset = start + .checked_mul(job_size) + .ok_or(CudaError::LengthTooLarge { len: start })?; + let jobs_ptr = jobs_base + .checked_add(byte_offset as u64) + .ok_or(CudaError::LengthTooLarge { len: byte_offset })?; + let trace_start = if trace_enabled { + let event = self.create_event()?; + event.record_default_stream()?; + Some(event) + } else { + None + }; + let interleave_horizontal_result = match kernel_mode { + CudaJ2kIdwtBatchKernelMode::Cooperative53 => self + .launch_j2k_idwt_interleave_horizontal_53_multi_ptr( + jobs_ptr, + max_height as usize, + count, + false, + ), + CudaJ2kIdwtBatchKernelMode::Cooperative97 => self + .launch_j2k_idwt_interleave_horizontal_97_multi_ptr( + jobs_ptr, + max_width as usize, + max_height as usize, + count, + false, + ), + CudaJ2kIdwtBatchKernelMode::Generic => self + .launch_j2k_idwt_interleave_horizontal_multi_ptr( + jobs_ptr, + max_height as usize, + count, + false, + ), + }; + if let Err(error) = interleave_horizontal_result { + let _ = self.synchronize(); + return Err(error); + } + kernel_dispatches = kernel_dispatches.saturating_add(1); + + let vertical_result = match kernel_mode { + CudaJ2kIdwtBatchKernelMode::Cooperative53 => self + .launch_j2k_idwt_vertical_53_multi_ptr( + jobs_ptr, + max_width as usize, + count, + false, + ), + CudaJ2kIdwtBatchKernelMode::Cooperative97 => self + .launch_j2k_idwt_vertical_97_multi_ptr( + jobs_ptr, + max_width as usize, + max_height as usize, + count, + false, + ), + CudaJ2kIdwtBatchKernelMode::Generic => self.launch_j2k_idwt_vertical_multi_ptr( + jobs_ptr, + max_width as usize, + count, + false, + ), + }; + if let Err(error) = vertical_result { + let _ = self.synchronize(); + return Err(error); + } + kernel_dispatches = kernel_dispatches.saturating_add(1); + if let Some(trace_start) = trace_start { + let trace_end = self.create_event()?; + trace_end.record_default_stream()?; + trace_end.synchronize()?; + let elapsed_us = elapsed_event_us_ceil(&trace_start, &trace_end)?; + let end = start.saturating_add(count); + let row = idwt_batch_trace_row( + stage_index, + &all_jobs[start..end], + max_width, + max_height, + kernel_mode, + elapsed_us, + ); + eprintln!("{}", format_idwt_batch_trace_row(row)); + } + } + + Ok(CudaQueuedExecution { + resources: vec![jobs_buffer], + execution: CudaExecutionStats { + kernel_dispatches, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: kernel_dispatches, + hardware_decode: false, + }, + }) + } + + fn j2k_inverse_dwt_batch_device_with_pool_impl( + &self, + targets: &[CudaJ2kIdwtTarget<'_>], + pool: &CudaBufferPool, + synchronize_each_launch: bool, + ) -> Result { + self.inner.set_current()?; + let kernel_jobs = j2k_idwt_multi_kernel_jobs(targets)?; + if kernel_jobs.is_empty() { + return Ok(CudaExecutionStats::default()); + } + let jobs_buffer = pool.upload(idwt_multi_jobs_as_bytes(&kernel_jobs))?; + let jobs_device = pooled_device_buffer(&jobs_buffer)?; + let max_width = kernel_jobs + .iter() + .map(|job| job.job.rect.x1.saturating_sub(job.job.rect.x0)) + .max() + .unwrap_or(0); + let max_height = kernel_jobs + .iter() + .map(|job| job.job.rect.y1.saturating_sub(job.job.rect.y0)) + .max() + .unwrap_or(0); + let kernel_mode = idwt_batch_kernel_mode(&kernel_jobs, max_width, max_height); + let interleave_horizontal_result = match kernel_mode { + CudaJ2kIdwtBatchKernelMode::Cooperative53 => self + .launch_j2k_idwt_interleave_horizontal_53_multi( + jobs_device, + max_height as usize, + kernel_jobs.len(), + synchronize_each_launch, + ), + CudaJ2kIdwtBatchKernelMode::Cooperative97 => self + .launch_j2k_idwt_interleave_horizontal_97_multi_ptr( + jobs_device.device_ptr(), + max_width as usize, + max_height as usize, + kernel_jobs.len(), + synchronize_each_launch, + ), + CudaJ2kIdwtBatchKernelMode::Generic => self + .launch_j2k_idwt_interleave_horizontal_multi( + jobs_device, + max_height as usize, + kernel_jobs.len(), + synchronize_each_launch, + ), + }; + if let Err(error) = interleave_horizontal_result { + if !synchronize_each_launch { + let _ = self.synchronize(); + } + return Err(error); + } + let vertical_result = match kernel_mode { + CudaJ2kIdwtBatchKernelMode::Cooperative53 => self.launch_j2k_idwt_vertical_53_multi( + jobs_device, + max_width as usize, + kernel_jobs.len(), + synchronize_each_launch, + ), + CudaJ2kIdwtBatchKernelMode::Cooperative97 => self + .launch_j2k_idwt_vertical_97_multi_ptr( + jobs_device.device_ptr(), + max_width as usize, + max_height as usize, + kernel_jobs.len(), + synchronize_each_launch, + ), + CudaJ2kIdwtBatchKernelMode::Generic => self.launch_j2k_idwt_vertical_multi( + jobs_device, + max_width as usize, + kernel_jobs.len(), + synchronize_each_launch, + ), + }; + if let Err(error) = vertical_result { + if !synchronize_each_launch { + let _ = self.synchronize(); + } + return Err(error); + } + if !synchronize_each_launch { + self.synchronize()?; + } + + Ok(CudaExecutionStats { + kernel_dispatches: 2, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 2, + hardware_decode: false, + }) + } + + fn j2k_inverse_dwt_single_device_impl( + &self, + ll: &CudaDeviceBuffer, + hl: &CudaDeviceBuffer, + lh: &CudaDeviceBuffer, + hh: &CudaDeviceBuffer, + job: CudaJ2kIdwtJob, + synchronize_each_launch: bool, + ) -> Result { + let width = job.rect.x1.saturating_sub(job.rect.x0); + let height = job.rect.y1.saturating_sub(job.rect.y0); + let output_words = checked_image_words(width, height, 1)?; + let output = self.allocate(checked_f32_words_byte_len(output_words)?)?; + if output_words == 0 { + return Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats::default(), + }); + } + + let job_buffer = self.upload(idwt_job_as_bytes(&job))?; + let (horizontal_kernel, vertical_kernel) = if job.irreversible97 == 0 { + ( + CudaKernel::J2kIdwtHorizontal53, + CudaKernel::J2kIdwtVertical53, + ) + } else { + ( + CudaKernel::J2kIdwtHorizontal97, + CudaKernel::J2kIdwtVertical97, + ) + }; + if synchronize_each_launch { + self.launch_j2k_idwt_interleave( + [ll, hl, lh, hh], + &output, + &job_buffer, + width, + height, + CudaLaunchMode::Sync, + )?; + self.launch_j2k_idwt_horizontal( + horizontal_kernel, + &output, + &job_buffer, + height as usize, + CudaLaunchMode::Sync, + )?; + self.launch_j2k_idwt_vertical( + vertical_kernel, + &output, + &job_buffer, + width as usize, + CudaLaunchMode::Sync, + )?; + } else { + self.launch_j2k_idwt_interleave( + [ll, hl, lh, hh], + &output, + &job_buffer, + width, + height, + CudaLaunchMode::Async, + )?; + if let Err(error) = self.launch_j2k_idwt_horizontal( + horizontal_kernel, + &output, + &job_buffer, + height as usize, + CudaLaunchMode::Async, + ) { + let _ = self.synchronize(); + return Err(error); + } + if let Err(error) = self.launch_j2k_idwt_vertical( + vertical_kernel, + &output, + &job_buffer, + width as usize, + CudaLaunchMode::Async, + ) { + let _ = self.synchronize(); + return Err(error); + } + self.synchronize()?; + } + Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats { + kernel_dispatches: 3, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 3, + hardware_decode: false, + }, + }) + } + + #[allow(clippy::too_many_arguments)] + fn j2k_inverse_dwt_single_device_with_pool_impl( + &self, + ll: &CudaDeviceBuffer, + hl: &CudaDeviceBuffer, + lh: &CudaDeviceBuffer, + hh: &CudaDeviceBuffer, + job: CudaJ2kIdwtJob, + synchronize_each_launch: bool, + pool: &CudaBufferPool, + ) -> Result { + let width = job.rect.x1.saturating_sub(job.rect.x0); + let height = job.rect.y1.saturating_sub(job.rect.y0); + let output_words = checked_image_words(width, height, 1)?; + let output = pool.take(checked_f32_words_byte_len(output_words)?)?; + let output_buffer = pooled_device_buffer(&output)?; + if output_words == 0 { + return Ok(CudaPooledKernelOutput { + buffer: output, + execution: CudaExecutionStats::default(), + }); + } + + let job_buffer = pool.upload(idwt_job_as_bytes(&job))?; + let job_device_buffer = pooled_device_buffer(&job_buffer)?; + let (horizontal_kernel, vertical_kernel) = if job.irreversible97 == 0 { + ( + CudaKernel::J2kIdwtHorizontal53, + CudaKernel::J2kIdwtVertical53, + ) + } else { + ( + CudaKernel::J2kIdwtHorizontal97, + CudaKernel::J2kIdwtVertical97, + ) + }; + if synchronize_each_launch { + self.launch_j2k_idwt_interleave( + [ll, hl, lh, hh], + output_buffer, + job_device_buffer, + width, + height, + CudaLaunchMode::Sync, + )?; + self.launch_j2k_idwt_horizontal( + horizontal_kernel, + output_buffer, + job_device_buffer, + height as usize, + CudaLaunchMode::Sync, + )?; + self.launch_j2k_idwt_vertical( + vertical_kernel, + output_buffer, + job_device_buffer, + width as usize, + CudaLaunchMode::Sync, + )?; + } else { + self.launch_j2k_idwt_interleave( + [ll, hl, lh, hh], + output_buffer, + job_device_buffer, + width, + height, + CudaLaunchMode::Async, + )?; + if let Err(error) = self.launch_j2k_idwt_horizontal( + horizontal_kernel, + output_buffer, + job_device_buffer, + height as usize, + CudaLaunchMode::Async, + ) { + let _ = self.synchronize(); + return Err(error); + } + if let Err(error) = self.launch_j2k_idwt_vertical( + vertical_kernel, + output_buffer, + job_device_buffer, + width as usize, + CudaLaunchMode::Async, + ) { + let _ = self.synchronize(); + return Err(error); + } + self.synchronize()?; + } + Ok(CudaPooledKernelOutput { + buffer: output, + execution: CudaExecutionStats { + kernel_dispatches: 3, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 3, + hardware_decode: false, + }, + }) + } + + /// Store a device f32 component plane as tightly packed Gray8 pixels. + pub fn j2k_store_gray8_device( + &self, + input: &CudaDeviceBuffer, + job: CudaJ2kStoreGray8Job, + ) -> Result { + let output_words = checked_image_words(job.output_width, job.output_height, 1)?; + let output = self.allocate(output_words)?; + if output_words == 0 { + return Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats::default(), + }); + } + let pixels = checked_image_words(job.copy_width, job.copy_height, 1)?; + if pixels == 0 { + return Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats::default(), + }); + } + validate_store_rgb8_plane( + input, + job.input_width, + job.source_x, + job.source_y, + job.copy_width, + job.copy_height, + )?; + + let job_buffer = self.upload(store_gray8_job_as_bytes(&job))?; + self.launch_j2k_store_gray8(input, &output, &job_buffer, pixels)?; + Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } + + /// Store a device f32 component plane as tightly packed Gray16 pixels. + pub fn j2k_store_gray16_device( + &self, + input: &CudaDeviceBuffer, + job: CudaJ2kStoreGray16Job, + ) -> Result { + let output_words = checked_image_words(job.output_width, job.output_height, 1)?; + let output = self.allocate( + output_words + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: output_words })?, + )?; + if output_words == 0 { + return Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats::default(), + }); + } + let pixels = checked_image_words(job.copy_width, job.copy_height, 1)?; + if pixels == 0 { + return Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats::default(), + }); + } + validate_store_rgb8_plane( + input, + job.input_width, + job.source_x, + job.source_y, + job.copy_width, + job.copy_height, + )?; + + let job_buffer = self.upload(store_gray16_job_as_bytes(&job))?; + self.launch_j2k_store_gray16(input, &output, &job_buffer, pixels)?; + Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } + + /// Apply inverse RCT/ICT in place on three device f32 component planes. + pub fn j2k_inverse_mct_device( + &self, + plane0: &CudaDeviceBuffer, + plane1: &CudaDeviceBuffer, + plane2: &CudaDeviceBuffer, + job: CudaJ2kInverseMctJob, + ) -> Result { + let bytes = (job.len as usize) + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if bytes > plane0.byte_len() || bytes > plane1.byte_len() || bytes > plane2.byte_len() { + return Err(CudaError::LengthTooLarge { len: bytes }); + } + if job.len == 0 { + return Ok(CudaExecutionStats::default()); + } + + let job_buffer = self.upload(inverse_mct_job_as_bytes(&job))?; + self.launch_j2k_inverse_mct(plane0, plane1, plane2, &job_buffer, job.len as usize)?; + Ok(CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }) + } + + /// Store three device f32 component planes as tightly packed RGB8/RGBA8. + pub fn j2k_store_rgb8_device( + &self, + plane0: &CudaDeviceBuffer, + plane1: &CudaDeviceBuffer, + plane2: &CudaDeviceBuffer, + job: CudaJ2kStoreRgb8Job, + ) -> Result { + let channels = if job.rgba == 0 { 3 } else { 4 }; + let output_bytes = checked_image_words(job.output_width, job.output_height, channels)?; + let output = self.allocate(output_bytes)?; + let pixels = checked_image_words(job.copy_width, job.copy_height, 1)?; + if output_bytes == 0 || pixels == 0 { + return Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats::default(), + }); + } + validate_store_rgb8_plane( + plane0, + job.input_width0, + job.source_x0, + job.source_y0, + job.copy_width, + job.copy_height, + )?; + validate_store_rgb8_plane( + plane1, + job.input_width1, + job.source_x1, + job.source_y1, + job.copy_width, + job.copy_height, + )?; + validate_store_rgb8_plane( + plane2, + job.input_width2, + job.source_x2, + job.source_y2, + job.copy_width, + job.copy_height, + )?; + let dst_end = (job.output_x as usize) + .checked_add(job.copy_width as usize) + .zip((job.output_y as usize).checked_add(job.copy_height as usize)) + .ok_or(CudaError::LengthTooLarge { len: output_bytes })?; + if dst_end.0 > job.output_width as usize || dst_end.1 > job.output_height as usize { + return Err(CudaError::LengthTooLarge { len: output_bytes }); + } + + let job_buffer = self.upload(store_rgb8_job_as_bytes(&job))?; + self.launch_j2k_store_rgb8(plane0, plane1, plane2, &output, &job_buffer, pixels)?; + Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } + + /// Store three device f32 component planes as tightly packed RGB16/RGBA16. + pub fn j2k_store_rgb16_device( + &self, + plane0: &CudaDeviceBuffer, + plane1: &CudaDeviceBuffer, + plane2: &CudaDeviceBuffer, + job: CudaJ2kStoreRgb16Job, + ) -> Result { + let channels = if job.rgba == 0 { 3 } else { 4 }; + let output_samples = checked_image_words(job.output_width, job.output_height, channels)?; + let output_bytes = output_samples + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { + len: output_samples, + })?; + let output = self.allocate(output_bytes)?; + let pixels = checked_image_words(job.copy_width, job.copy_height, 1)?; + if output_bytes == 0 || pixels == 0 { + return Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats::default(), + }); + } + validate_store_rgb8_plane( + plane0, + job.input_width0, + job.source_x0, + job.source_y0, + job.copy_width, + job.copy_height, + )?; + validate_store_rgb8_plane( + plane1, + job.input_width1, + job.source_x1, + job.source_y1, + job.copy_width, + job.copy_height, + )?; + validate_store_rgb8_plane( + plane2, + job.input_width2, + job.source_x2, + job.source_y2, + job.copy_width, + job.copy_height, + )?; + let dst_end = (job.output_x as usize) + .checked_add(job.copy_width as usize) + .zip((job.output_y as usize).checked_add(job.copy_height as usize)) + .ok_or(CudaError::LengthTooLarge { len: output_bytes })?; + if dst_end.0 > job.output_width as usize || dst_end.1 > job.output_height as usize { + return Err(CudaError::LengthTooLarge { len: output_bytes }); + } + + let job_buffer = self.upload(store_rgb16_job_as_bytes(&job))?; + self.launch_j2k_store_rgb16(plane0, plane1, plane2, &output, &job_buffer, pixels)?; + Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } + + /// Apply inverse RCT/ICT and store tightly packed RGB8/RGBA8 in one dispatch. + pub fn j2k_store_rgb8_mct_device( + &self, + plane0: &CudaDeviceBuffer, + plane1: &CudaDeviceBuffer, + plane2: &CudaDeviceBuffer, + job: CudaJ2kStoreRgb8MctJob, + ) -> Result { + let batch = self.j2k_store_rgb8_mct_batch_device(&[CudaJ2kStoreRgb8MctTarget { + plane0, + plane1, + plane2, + job, + }])?; + let (mut outputs, execution) = batch.into_parts(); + let buffer = outputs.pop().ok_or_else(|| CudaError::InvalidArgument { + message: "single RGB8 MCT batch store returned no output".to_string(), + })?; + Ok(CudaKernelOutput { buffer, execution }) + } + + /// Apply inverse RCT/ICT and store multiple tightly packed RGB8/RGBA8 images + /// in one dispatch. + pub fn j2k_store_rgb8_mct_batch_device( + &self, + targets: &[CudaJ2kStoreRgb8MctTarget<'_>], + ) -> Result { + if targets.is_empty() { + return Ok(CudaKernelBatchOutput { + outputs: Vec::new(), + execution: CudaExecutionStats::default(), + }); + } + + let mut outputs = Vec::with_capacity(targets.len()); + let mut kernel_jobs = Vec::with_capacity(targets.len()); + let mut max_pixels = 0usize; + for target in targets { + let store = target.job.store; + let channels = if store.rgba == 0 { 3 } else { 4 }; + let output_bytes = + checked_image_words(store.output_width, store.output_height, channels)?; + let output = self.allocate(output_bytes)?; + let pixels = checked_image_words(store.copy_width, store.copy_height, 1)?; + if output_bytes != 0 && pixels != 0 { + validate_store_rgb8_plane( + target.plane0, + store.input_width0, + store.source_x0, + store.source_y0, + store.copy_width, + store.copy_height, + )?; + validate_store_rgb8_plane( + target.plane1, + store.input_width1, + store.source_x1, + store.source_y1, + store.copy_width, + store.copy_height, + )?; + validate_store_rgb8_plane( + target.plane2, + store.input_width2, + store.source_x2, + store.source_y2, + store.copy_width, + store.copy_height, + )?; + let dst_end = (store.output_x as usize) + .checked_add(store.copy_width as usize) + .zip((store.output_y as usize).checked_add(store.copy_height as usize)) + .ok_or(CudaError::LengthTooLarge { len: output_bytes })?; + if dst_end.0 > store.output_width as usize + || dst_end.1 > store.output_height as usize + { + return Err(CudaError::LengthTooLarge { len: output_bytes }); + } + max_pixels = max_pixels.max(pixels); + } + kernel_jobs.push(CudaJ2kStoreRgb8MctBatchJob { + plane0_ptr: target.plane0.device_ptr(), + plane1_ptr: target.plane1.device_ptr(), + plane2_ptr: target.plane2.device_ptr(), + output_ptr: output.device_ptr(), + job: target.job, + }); + outputs.push(output); + } + if max_pixels == 0 { + return Ok(CudaKernelBatchOutput { + outputs, + execution: CudaExecutionStats::default(), + }); + } + + let jobs_buffer = self.upload(store_rgb8_mct_batch_jobs_as_bytes(&kernel_jobs))?; + self.launch_j2k_store_rgb8_mct_batch(&jobs_buffer, max_pixels, kernel_jobs.len())?; + Ok(CudaKernelBatchOutput { + outputs, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } + + /// Apply inverse RCT/ICT and store multiple tightly packed RGB8/RGBA8 images + /// into one contiguous device allocation in one dispatch. + pub fn j2k_store_rgb8_mct_batch_contiguous_device( + &self, + targets: &[CudaJ2kStoreRgb8MctTarget<'_>], + ) -> Result { + let mut ranges = Vec::with_capacity(targets.len()); + let mut total_bytes = 0usize; + let mut max_pixels = 0usize; + for target in targets { + let store = target.job.store; + let channels = if store.rgba == 0 { 3 } else { 4 }; + let output_bytes = + checked_image_words(store.output_width, store.output_height, channels)?; + let pixels = checked_image_words(store.copy_width, store.copy_height, 1)?; + if output_bytes != 0 && pixels != 0 { + validate_store_rgb8_plane( + target.plane0, + store.input_width0, + store.source_x0, + store.source_y0, + store.copy_width, + store.copy_height, + )?; + validate_store_rgb8_plane( + target.plane1, + store.input_width1, + store.source_x1, + store.source_y1, + store.copy_width, + store.copy_height, + )?; + validate_store_rgb8_plane( + target.plane2, + store.input_width2, + store.source_x2, + store.source_y2, + store.copy_width, + store.copy_height, + )?; + let dst_end = (store.output_x as usize) + .checked_add(store.copy_width as usize) + .zip((store.output_y as usize).checked_add(store.copy_height as usize)) + .ok_or(CudaError::LengthTooLarge { len: output_bytes })?; + if dst_end.0 > store.output_width as usize + || dst_end.1 > store.output_height as usize + { + return Err(CudaError::LengthTooLarge { len: output_bytes }); + } + max_pixels = max_pixels.max(pixels); + } + let offset = total_bytes; + total_bytes = total_bytes + .checked_add(output_bytes) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + ranges.push(CudaDeviceBufferRange { + offset, + len: output_bytes, + }); + } + + let output = self.allocate(total_bytes)?; + if targets.is_empty() || max_pixels == 0 { + return Ok(CudaKernelContiguousBatchOutput { + output, + ranges, + execution: CudaExecutionStats::default(), + }); + } + + let base_ptr = output.device_ptr(); + let kernel_jobs = targets + .iter() + .zip(ranges.iter()) + .map(|(target, range)| { + let output_ptr = base_ptr + .checked_add( + u64::try_from(range.offset) + .map_err(|_| CudaError::LengthTooLarge { len: range.offset })?, + ) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + Ok(CudaJ2kStoreRgb8MctBatchJob { + plane0_ptr: target.plane0.device_ptr(), + plane1_ptr: target.plane1.device_ptr(), + plane2_ptr: target.plane2.device_ptr(), + output_ptr, + job: target.job, + }) + }) + .collect::, CudaError>>()?; + let jobs_buffer = self.upload(store_rgb8_mct_batch_jobs_as_bytes(&kernel_jobs))?; + self.launch_j2k_store_rgb8_mct_batch(&jobs_buffer, max_pixels, kernel_jobs.len())?; + Ok(CudaKernelContiguousBatchOutput { + output, + ranges, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } + + /// Apply inverse RCT/ICT and store tightly packed RGB16/RGBA16 in one dispatch. + pub fn j2k_store_rgb16_mct_device( + &self, + plane0: &CudaDeviceBuffer, + plane1: &CudaDeviceBuffer, + plane2: &CudaDeviceBuffer, + job: CudaJ2kStoreRgb16MctJob, + ) -> Result { + let store = job.store; + let channels = if store.rgba == 0 { 3 } else { 4 }; + let output_samples = + checked_image_words(store.output_width, store.output_height, channels)?; + let output_bytes = output_samples + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { + len: output_samples, + })?; + let output = self.allocate(output_bytes)?; + let pixels = checked_image_words(store.copy_width, store.copy_height, 1)?; + if output_bytes == 0 || pixels == 0 { + return Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats::default(), + }); + } + validate_store_rgb8_plane( + plane0, + store.input_width0, + store.source_x0, + store.source_y0, + store.copy_width, + store.copy_height, + )?; + validate_store_rgb8_plane( + plane1, + store.input_width1, + store.source_x1, + store.source_y1, + store.copy_width, + store.copy_height, + )?; + validate_store_rgb8_plane( + plane2, + store.input_width2, + store.source_x2, + store.source_y2, + store.copy_width, + store.copy_height, + )?; + let dst_end = (store.output_x as usize) + .checked_add(store.copy_width as usize) + .zip((store.output_y as usize).checked_add(store.copy_height as usize)) + .ok_or(CudaError::LengthTooLarge { len: output_bytes })?; + if dst_end.0 > store.output_width as usize || dst_end.1 > store.output_height as usize { + return Err(CudaError::LengthTooLarge { len: output_bytes }); + } + + let job_buffer = self.upload(store_rgb16_mct_job_as_bytes(&job))?; + self.launch_j2k_store_rgb16_mct(plane0, plane1, plane2, &output, &job_buffer, pixels)?; + Ok(CudaKernelOutput { + buffer: output, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }, + }) + } + + fn launch_j2k_idwt_interleave( + &self, + bands: [&CudaDeviceBuffer; 4], + output: &CudaDeviceBuffer, + job: &CudaDeviceBuffer, + width: u32, + height: u32, + mode: CudaLaunchMode, + ) -> Result<(), CudaError> { + let function = self.j2k_idwt_kernel_function(CudaKernel::J2kIdwtInterleave)?; + let [ll, hl, lh, hh] = bands; + let mut low_low_ptr = ll.device_ptr(); + let mut high_low_ptr = hl.device_ptr(); + let mut low_high_ptr = lh.device_ptr(); + let mut high_high_ptr = hh.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut job_ptr = job.device_ptr(); + let mut params = cuda_kernel_params!( + low_low_ptr, + high_low_ptr, + low_high_ptr, + high_high_ptr, + output_ptr, + job_ptr + ); + let geometry = + j2k_dwt53_launch_geometry(width, height).ok_or(CudaError::ImageTooLarge { + width, + height, + channels: 1, + })?; + match mode { + CudaLaunchMode::Sync => self.launch_kernel(function, geometry, &mut params), + CudaLaunchMode::Async => self.launch_kernel_async(function, geometry, &mut params), + } + } + + fn launch_j2k_idwt_interleave_horizontal_multi( + &self, + jobs: &CudaDeviceBuffer, + max_rows: usize, + job_count: usize, + synchronize: bool, + ) -> Result<(), CudaError> { + self.launch_j2k_idwt_interleave_horizontal_multi_ptr( + jobs.device_ptr(), + max_rows, + job_count, + synchronize, + ) + } + + fn launch_j2k_idwt_interleave_horizontal_multi_ptr( + &self, + jobs_ptr: CuDevicePtr, + max_rows: usize, + job_count: usize, + synchronize: bool, + ) -> Result<(), CudaError> { + let function = + self.j2k_idwt_kernel_function(CudaKernel::J2kIdwtInterleaveHorizontalMulti)?; + let mut jobs_ptr = jobs_ptr; + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_idwt_multi_1d_launch_geometry(max_rows, job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + if synchronize { + self.launch_kernel(function, geometry, &mut params) + } else { + self.launch_kernel_async(function, geometry, &mut params) + } + } + + fn launch_j2k_idwt_interleave_horizontal_53_multi( + &self, + jobs: &CudaDeviceBuffer, + max_rows: usize, + job_count: usize, + synchronize: bool, + ) -> Result<(), CudaError> { + self.launch_j2k_idwt_interleave_horizontal_53_multi_ptr( + jobs.device_ptr(), + max_rows, + job_count, + synchronize, + ) + } + + fn launch_j2k_idwt_interleave_horizontal_53_multi_ptr( + &self, + jobs_ptr: CuDevicePtr, + max_rows: usize, + job_count: usize, + synchronize: bool, + ) -> Result<(), CudaError> { + let mut jobs_ptr = jobs_ptr; + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_idwt_multi_coop_launch_geometry(max_rows, job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + self.launch_named_kernel( + CudaKernel::J2kIdwtInterleaveHorizontal53Multi, + geometry, + &mut params, + CudaLaunchMode::from_synchronize(synchronize), + ) + } + + fn launch_j2k_idwt_interleave_horizontal_97_multi_ptr( + &self, + jobs_ptr: CuDevicePtr, + max_width: usize, + max_rows: usize, + job_count: usize, + synchronize: bool, + ) -> Result<(), CudaError> { + let mut jobs_ptr = jobs_ptr; + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_idwt_multi_coop_axis_launch_geometry(max_rows, max_width, job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + self.launch_named_kernel( + CudaKernel::J2kIdwtInterleaveHorizontal97Multi, + geometry, + &mut params, + CudaLaunchMode::from_synchronize(synchronize), + ) + } + + fn launch_j2k_idwt_horizontal( + &self, + kernel: CudaKernel, + output: &CudaDeviceBuffer, + job: &CudaDeviceBuffer, + rows: usize, + mode: CudaLaunchMode, + ) -> Result<(), CudaError> { + let function = self.j2k_idwt_kernel_function(kernel)?; + let mut output_ptr = output.device_ptr(); + let mut job_ptr = job.device_ptr(); + let mut params = cuda_kernel_params!(output_ptr, job_ptr); + let geometry = + j2k_forward_rct_launch_geometry(rows).ok_or(CudaError::LengthTooLarge { len: rows })?; + match mode { + CudaLaunchMode::Sync => self.launch_kernel(function, geometry, &mut params), + CudaLaunchMode::Async => self.launch_kernel_async(function, geometry, &mut params), + } + } + + fn launch_j2k_idwt_vertical( + &self, + kernel: CudaKernel, + output: &CudaDeviceBuffer, + job: &CudaDeviceBuffer, + columns: usize, + mode: CudaLaunchMode, + ) -> Result<(), CudaError> { + let function = self.j2k_idwt_kernel_function(kernel)?; + let mut output_ptr = output.device_ptr(); + let mut job_ptr = job.device_ptr(); + let mut params = cuda_kernel_params!(output_ptr, job_ptr); + let geometry = j2k_forward_rct_launch_geometry(columns) + .ok_or(CudaError::LengthTooLarge { len: columns })?; + match mode { + CudaLaunchMode::Sync => self.launch_kernel(function, geometry, &mut params), + CudaLaunchMode::Async => self.launch_kernel_async(function, geometry, &mut params), + } + } + + fn launch_j2k_idwt_vertical_multi( + &self, + jobs: &CudaDeviceBuffer, + max_columns: usize, + job_count: usize, + synchronize: bool, + ) -> Result<(), CudaError> { + self.launch_j2k_idwt_vertical_multi_ptr( + jobs.device_ptr(), + max_columns, + job_count, + synchronize, + ) + } + + fn launch_j2k_idwt_vertical_multi_ptr( + &self, + jobs_ptr: CuDevicePtr, + max_columns: usize, + job_count: usize, + synchronize: bool, + ) -> Result<(), CudaError> { + let function = self.j2k_idwt_kernel_function(CudaKernel::J2kIdwtVerticalMulti)?; + let mut jobs_ptr = jobs_ptr; + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_idwt_multi_1d_launch_geometry(max_columns, job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + if synchronize { + self.launch_kernel(function, geometry, &mut params) + } else { + self.launch_kernel_async(function, geometry, &mut params) + } + } + + fn launch_j2k_idwt_vertical_53_multi( + &self, + jobs: &CudaDeviceBuffer, + max_columns: usize, + job_count: usize, + synchronize: bool, + ) -> Result<(), CudaError> { + self.launch_j2k_idwt_vertical_53_multi_ptr( + jobs.device_ptr(), + max_columns, + job_count, + synchronize, + ) + } + + fn launch_j2k_idwt_vertical_53_multi_ptr( + &self, + jobs_ptr: CuDevicePtr, + max_columns: usize, + job_count: usize, + synchronize: bool, + ) -> Result<(), CudaError> { + let mut jobs_ptr = jobs_ptr; + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_idwt_multi_coop_launch_geometry(max_columns, job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + self.launch_named_kernel( + CudaKernel::J2kIdwtVertical53Multi, + geometry, + &mut params, + CudaLaunchMode::from_synchronize(synchronize), + ) + } + + fn launch_j2k_idwt_vertical_97_multi_ptr( + &self, + jobs_ptr: CuDevicePtr, + max_columns: usize, + max_height: usize, + job_count: usize, + synchronize: bool, + ) -> Result<(), CudaError> { + const COLUMNS_PER_BLOCK: usize = 4; + const MIN_COLS4_JOBS: usize = 64; + let (kernel, geometry) = if job_count >= MIN_COLS4_JOBS && max_height <= 256 { + let geometry = j2k_idwt_multi_coop_columns_launch_geometry( + max_columns, + max_height, + job_count, + COLUMNS_PER_BLOCK, + ) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + (CudaKernel::J2kIdwtVertical97MultiCols4, geometry) + } else { + let geometry = + j2k_idwt_multi_coop_axis_launch_geometry(max_columns, max_height, job_count) + .ok_or(CudaError::LengthTooLarge { len: job_count })?; + (CudaKernel::J2kIdwtVertical97Multi, geometry) + }; + let mut jobs_ptr = jobs_ptr; + let mut params = cuda_kernel_params!(jobs_ptr); + self.launch_named_kernel( + kernel, + geometry, + &mut params, + CudaLaunchMode::from_synchronize(synchronize), + ) + } + + fn launch_j2k_store_gray8( + &self, + input: &CudaDeviceBuffer, + output: &CudaDeviceBuffer, + job: &CudaDeviceBuffer, + pixels: usize, + ) -> Result<(), CudaError> { + let function = self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreGray8)?; + let mut input_ptr = input.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut job_ptr = job.device_ptr(); + let mut params = cuda_kernel_params!(input_ptr, output_ptr, job_ptr); + let geometry = j2k_forward_rct_launch_geometry(pixels) + .ok_or(CudaError::LengthTooLarge { len: pixels })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_j2k_store_gray16( + &self, + input: &CudaDeviceBuffer, + output: &CudaDeviceBuffer, + job: &CudaDeviceBuffer, + pixels: usize, + ) -> Result<(), CudaError> { + let function = self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreGray16)?; + let mut input_ptr = input.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut job_ptr = job.device_ptr(); + let mut params = cuda_kernel_params!(input_ptr, output_ptr, job_ptr); + let geometry = j2k_forward_rct_launch_geometry(pixels) + .ok_or(CudaError::LengthTooLarge { len: pixels })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_j2k_inverse_mct( + &self, + plane0: &CudaDeviceBuffer, + plane1: &CudaDeviceBuffer, + plane2: &CudaDeviceBuffer, + job: &CudaDeviceBuffer, + len: usize, + ) -> Result<(), CudaError> { + let function = self.j2k_decode_store_kernel_function(CudaKernel::J2kInverseMct)?; + let mut plane0_ptr = plane0.device_ptr(); + let mut plane1_ptr = plane1.device_ptr(); + let mut plane2_ptr = plane2.device_ptr(); + let mut job_ptr = job.device_ptr(); + let mut params = cuda_kernel_params!(plane0_ptr, plane1_ptr, plane2_ptr, job_ptr); + let geometry = + j2k_forward_rct_launch_geometry(len).ok_or(CudaError::LengthTooLarge { len })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_j2k_store_rgb8( + &self, + plane0: &CudaDeviceBuffer, + plane1: &CudaDeviceBuffer, + plane2: &CudaDeviceBuffer, + output: &CudaDeviceBuffer, + job: &CudaDeviceBuffer, + pixels: usize, + ) -> Result<(), CudaError> { + let function = self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreRgb8)?; + let mut plane0_ptr = plane0.device_ptr(); + let mut plane1_ptr = plane1.device_ptr(); + let mut plane2_ptr = plane2.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut job_ptr = job.device_ptr(); + let mut params = + cuda_kernel_params!(plane0_ptr, plane1_ptr, plane2_ptr, output_ptr, job_ptr); + let geometry = j2k_forward_rct_launch_geometry(pixels) + .ok_or(CudaError::LengthTooLarge { len: pixels })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_j2k_store_rgb16( + &self, + plane0: &CudaDeviceBuffer, + plane1: &CudaDeviceBuffer, + plane2: &CudaDeviceBuffer, + output: &CudaDeviceBuffer, + job: &CudaDeviceBuffer, + pixels: usize, + ) -> Result<(), CudaError> { + let function = self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreRgb16)?; + let mut plane0_ptr = plane0.device_ptr(); + let mut plane1_ptr = plane1.device_ptr(); + let mut plane2_ptr = plane2.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut job_ptr = job.device_ptr(); + let mut params = + cuda_kernel_params!(plane0_ptr, plane1_ptr, plane2_ptr, output_ptr, job_ptr); + let geometry = j2k_forward_rct_launch_geometry(pixels) + .ok_or(CudaError::LengthTooLarge { len: pixels })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_j2k_store_rgb8_mct_batch( + &self, + jobs: &CudaDeviceBuffer, + max_pixels: usize, + job_count: usize, + ) -> Result<(), CudaError> { + let function = self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreRgb8MctBatch)?; + let mut jobs_ptr = jobs.device_ptr(); + let mut params = cuda_kernel_params!(jobs_ptr); + let geometry = j2k_store_batch_launch_geometry(max_pixels, job_count) + .ok_or(CudaError::LengthTooLarge { len: max_pixels })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_j2k_store_rgb16_mct( + &self, + plane0: &CudaDeviceBuffer, + plane1: &CudaDeviceBuffer, + plane2: &CudaDeviceBuffer, + output: &CudaDeviceBuffer, + job: &CudaDeviceBuffer, + pixels: usize, + ) -> Result<(), CudaError> { + let function = self.j2k_decode_store_kernel_function(CudaKernel::J2kStoreRgb16Mct)?; + let mut plane0_ptr = plane0.device_ptr(); + let mut plane1_ptr = plane1.device_ptr(); + let mut plane2_ptr = plane2.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut job_ptr = job.device_ptr(); + let mut params = + cuda_kernel_params!(plane0_ptr, plane1_ptr, plane2_ptr, output_ptr, job_ptr); + let geometry = j2k_forward_rct_launch_geometry(pixels) + .ok_or(CudaError::LengthTooLarge { len: pixels })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn j2k_decode_store_kernel_function( + &self, + kernel: CudaKernel, + ) -> Result { + #[cfg(feature = "cuda-oxide-j2k-decode-store")] + { + if crate::build_flags::cuda_oxide_j2k_decode_store_enabled() { + return self + .inner + .cuda_oxide_j2k_decode_store_kernel_function(kernel); + } + } + self.inner.kernel_function(kernel) + } + + fn j2k_idwt_kernel_function( + &self, + kernel: CudaKernel, + ) -> Result { + #[cfg(feature = "cuda-oxide-j2k-idwt")] + { + if crate::build_flags::cuda_oxide_j2k_idwt_enabled() && kernel.is_j2k_idwt_stage() { + return self.inner.cuda_oxide_j2k_idwt_kernel_function(kernel); + } + } + self.inner.kernel_function(kernel) + } +} + +/// Device-resident interleaved JPEG 2000 input pixels with row stride metadata. +#[derive(Clone, Copy, Debug)] +pub struct CudaJ2kStridedInterleavedPixels<'a> { + /// Backing CUDA device byte buffer. + pub buffer: &'a CudaDeviceBuffer, + /// Byte offset to the first pixel in `buffer`. + pub byte_offset: usize, + /// Active input width in pixels. + pub width: u32, + /// Active input height in pixels. + pub height: u32, + /// Bytes between the start of consecutive rows. + pub pitch_bytes: usize, + /// Number of interleaved components per pixel. + pub num_components: u8, + /// Integer sample precision. + pub bit_depth: u8, + /// Whether integer samples are signed. + pub signed: bool, +} + +pub(crate) fn active_dwt53_buffers<'a>( + buffer_a: &'a CudaDeviceBuffer, + buffer_b: &'a CudaDeviceBuffer, + active_is_a: bool, +) -> (&'a CudaDeviceBuffer, &'a CudaDeviceBuffer) { + if active_is_a { + (buffer_a, buffer_b) + } else { + (buffer_b, buffer_a) + } +} + +pub(crate) fn j2k_idwt_multi_kernel_jobs( + targets: &[CudaJ2kIdwtTarget<'_>], +) -> Result, CudaError> { + let mut kernel_jobs = Vec::with_capacity(targets.len()); + for target in targets { + let width = target.job.rect.x1.saturating_sub(target.job.rect.x0); + let height = target.job.rect.y1.saturating_sub(target.job.rect.y0); + if width == 0 || height == 0 { + continue; + } + ensure_idwt_buffer_len(target.output, target.job.rect)?; + ensure_idwt_buffer_len(target.ll, target.job.ll_rect)?; + ensure_idwt_buffer_len(target.hl, target.job.hl_rect)?; + ensure_idwt_buffer_len(target.lh, target.job.lh_rect)?; + ensure_idwt_buffer_len(target.hh, target.job.hh_rect)?; + kernel_jobs.push(CudaJ2kIdwtMultiKernelJob { + ll_ptr: target.ll.device_ptr(), + hl_ptr: target.hl.device_ptr(), + lh_ptr: target.lh.device_ptr(), + hh_ptr: target.hh.device_ptr(), + output_ptr: target.output.device_ptr(), + job: target.job, + }); + } + Ok(kernel_jobs) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CudaJ2kIdwtBatchKernelMode { + Generic, + Cooperative53, + Cooperative97, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct CudaJ2kIdwtBatchTraceRow { + pub(crate) stage_index: usize, + pub(crate) mode: CudaJ2kIdwtBatchKernelMode, + pub(crate) job_count: usize, + pub(crate) max_width: u32, + pub(crate) max_height: u32, + pub(crate) min_width: u32, + pub(crate) min_height: u32, + pub(crate) total_pixels: u64, + pub(crate) irreversible_jobs: usize, + pub(crate) elapsed_us: u128, +} + +pub(crate) fn idwt_batch_kernel_mode( + kernel_jobs: &[CudaJ2kIdwtMultiKernelJob], + max_width: u32, + max_height: u32, +) -> CudaJ2kIdwtBatchKernelMode { + const MAX_COOPERATIVE_DIMENSION: u32 = 512; + const MIN_COOPERATIVE_53_DIMENSION: u32 = 128; + const MIN_COOPERATIVE_97_DIMENSION: u32 = 64; + let bounded_cooperative_shape = + max_width <= MAX_COOPERATIVE_DIMENSION && max_height <= MAX_COOPERATIVE_DIMENSION; + if !bounded_cooperative_shape { + return CudaJ2kIdwtBatchKernelMode::Generic; + } + if kernel_jobs.iter().all(|job| job.job.irreversible97 == 0) { + if max_width >= MIN_COOPERATIVE_53_DIMENSION && max_height >= MIN_COOPERATIVE_53_DIMENSION { + CudaJ2kIdwtBatchKernelMode::Cooperative53 + } else { + CudaJ2kIdwtBatchKernelMode::Generic + } + } else if kernel_jobs.iter().all(|job| job.job.irreversible97 != 0) { + if max_width >= MIN_COOPERATIVE_97_DIMENSION && max_height >= MIN_COOPERATIVE_97_DIMENSION { + CudaJ2kIdwtBatchKernelMode::Cooperative97 + } else { + CudaJ2kIdwtBatchKernelMode::Generic + } + } else { + CudaJ2kIdwtBatchKernelMode::Generic + } +} + +pub(crate) fn idwt_batch_trace_row( + stage_index: usize, + kernel_jobs: &[CudaJ2kIdwtMultiKernelJob], + max_width: u32, + max_height: u32, + mode: CudaJ2kIdwtBatchKernelMode, + elapsed_us: u128, +) -> CudaJ2kIdwtBatchTraceRow { + let mut min_width = u32::MAX; + let mut min_height = u32::MAX; + let mut total_pixels = 0u64; + let mut irreversible_jobs = 0usize; + for kernel_job in kernel_jobs { + let width = kernel_job + .job + .rect + .x1 + .saturating_sub(kernel_job.job.rect.x0); + let height = kernel_job + .job + .rect + .y1 + .saturating_sub(kernel_job.job.rect.y0); + min_width = min_width.min(width); + min_height = min_height.min(height); + total_pixels = + total_pixels.saturating_add(u64::from(width).saturating_mul(u64::from(height))); + if kernel_job.job.irreversible97 != 0 { + irreversible_jobs = irreversible_jobs.saturating_add(1); + } + } + if kernel_jobs.is_empty() { + min_width = 0; + min_height = 0; + } + CudaJ2kIdwtBatchTraceRow { + stage_index, + mode, + job_count: kernel_jobs.len(), + max_width, + max_height, + min_width, + min_height, + total_pixels, + irreversible_jobs, + elapsed_us, + } +} + +pub(crate) fn format_idwt_batch_trace_row(row: CudaJ2kIdwtBatchTraceRow) -> String { + format!( + "j2k_profile codec=j2k op=cuda_idwt_batch path=decode \ + stage_index={} mode={:?} job_count={} max_width={} max_height={} \ + min_width={} min_height={} total_pixels={} irreversible_jobs={} elapsed_us={}", + row.stage_index, + row.mode, + row.job_count, + row.max_width, + row.max_height, + row.min_width, + row.min_height, + row.total_pixels, + row.irreversible_jobs, + row.elapsed_us + ) +} + +#[cfg(test)] +pub(crate) fn idwt_batch_uses_cooperative_53( + kernel_jobs: &[CudaJ2kIdwtMultiKernelJob], + max_width: u32, + max_height: u32, +) -> bool { + idwt_batch_kernel_mode(kernel_jobs, max_width, max_height) + == CudaJ2kIdwtBatchKernelMode::Cooperative53 +} + +pub(crate) fn ensure_idwt_buffer_len( + buffer: &CudaDeviceBuffer, + rect: CudaJ2kRect, +) -> Result<(), CudaError> { + let width = rect.x1.saturating_sub(rect.x0); + let height = rect.y1.saturating_sub(rect.y0); + let words = checked_image_words(width, height, 1)?; + let bytes = checked_f32_words_byte_len(words)?; + if bytes > buffer.byte_len() { + return Err(CudaError::OutputTooSmall { + required: bytes, + have: buffer.byte_len(), + }); + } + Ok(()) +} + +pub(crate) fn checked_f32_words_byte_len(words: usize) -> Result { + words + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: words }) +} + +pub(crate) fn validate_store_rgb8_plane( + plane: &CudaDeviceBuffer, + input_width: u32, + source_x: u32, + source_y: u32, + copy_width: u32, + copy_height: u32, +) -> Result<(), CudaError> { + if source_x + .checked_add(copy_width) + .is_none_or(|end_x| end_x > input_width) + { + return Err(CudaError::LengthTooLarge { + len: plane.byte_len(), + }); + } + let last_sample = if copy_height == 0 { + 0 + } else { + (source_y as usize) + .checked_add(copy_height as usize - 1) + .and_then(|row| row.checked_mul(input_width as usize)) + .and_then(|row| row.checked_add(source_x as usize)) + .and_then(|row| row.checked_add(copy_width as usize)) + .ok_or(CudaError::LengthTooLarge { + len: plane.byte_len(), + })? + }; + let required_bytes = + last_sample + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { + len: plane.byte_len(), + })?; + if required_bytes > plane.byte_len() { + return Err(CudaError::LengthTooLarge { + len: required_bytes, + }); + } + Ok(()) +} diff --git a/crates/j2k-cuda-runtime/src/j2k_encode.rs b/crates/j2k-cuda-runtime/src/j2k_encode.rs new file mode 100644 index 00000000..a8f00666 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_encode.rs @@ -0,0 +1,1603 @@ +use crate::{ + bytes::{f32_slice_as_bytes, f32_slice_as_bytes_mut, i32_slice_as_bytes_mut}, + context::CudaContext, + driver::{CuDevicePtr, CuFunction}, + error::CudaError, + execution::{cuda_kernel_param, CudaExecutionStats}, + j2k_decode::{active_dwt53_buffers, CudaJ2kStridedInterleavedPixels}, + kernels::{j2k_dwt53_launch_geometry, j2k_forward_rct_launch_geometry, CudaKernel}, + memory::{checked_image_words, CudaDeviceBuffer}, +}; + +impl CudaContext { + /// Deinterleave interleaved pixel bytes into f32 component planes. + pub fn j2k_deinterleave_to_f32( + &self, + pixels: &[u8], + num_pixels: usize, + num_components: u8, + bit_depth: u8, + signed: bool, + ) -> Result { + let resident = self.j2k_deinterleave_to_f32_resident( + pixels, + num_pixels, + num_components, + bit_depth, + signed, + )?; + let execution = resident.execution(); + let components = resident.download_components()?; + Ok(CudaJ2kDeinterleavedComponents { + components, + execution, + }) + } + + /// Deinterleave interleaved pixel bytes into resident f32 component planes. + pub fn j2k_deinterleave_to_f32_resident( + &self, + pixels: &[u8], + num_pixels: usize, + num_components: u8, + bit_depth: u8, + signed: bool, + ) -> Result { + if num_components == 0 || num_components > 4 { + return Err(CudaError::InvalidArgument { + message: "component count must be between 1 and 4".to_string(), + }); + } + if bit_depth == 0 || bit_depth > 16 { + return Err(CudaError::InvalidArgument { + message: "bit depth must be between 1 and 16".to_string(), + }); + } + let bytes_per_sample = if bit_depth <= 8 { 1usize } else { 2usize }; + let expected_len = num_pixels + .checked_mul(usize::from(num_components)) + .and_then(|len| len.checked_mul(bytes_per_sample)) + .ok_or(CudaError::LengthTooLarge { len: num_pixels })?; + if pixels.len() < expected_len { + return Err(CudaError::InvalidArgument { + message: "pixel buffer is shorter than the requested image".to_string(), + }); + } + + self.inner.set_current()?; + let sample_count = num_pixels + .checked_mul(usize::from(num_components)) + .ok_or(CudaError::LengthTooLarge { len: num_pixels })?; + let output_bytes = sample_count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: sample_count })?; + let output = self.allocate(output_bytes)?; + if num_pixels == 0 { + return Ok(CudaJ2kResidentComponents { + buffer: output, + num_pixels, + num_components, + execution: CudaExecutionStats::default(), + }); + } + + let pixels = self.upload(&pixels[..expected_len])?; + self.launch_j2k_deinterleave_to_f32( + &pixels, + &output, + num_pixels, + num_components, + bit_depth, + signed, + )?; + + Ok(CudaJ2kResidentComponents { + buffer: output, + num_pixels, + num_components, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }) + } + + /// Deinterleave strided device-resident pixel bytes into resident f32 component planes. + pub fn j2k_deinterleave_strided_to_f32_resident( + &self, + image: CudaJ2kStridedInterleavedPixels<'_>, + ) -> Result { + let CudaJ2kStridedInterleavedPixels { + buffer: pixels, + byte_offset, + width, + height, + pitch_bytes, + num_components, + bit_depth, + signed, + } = image; + if width == 0 || height == 0 { + return Err(CudaError::InvalidArgument { + message: "image dimensions must be nonzero".to_string(), + }); + } + if num_components == 0 || num_components > 4 { + return Err(CudaError::InvalidArgument { + message: "component count must be between 1 and 4".to_string(), + }); + } + if bit_depth == 0 || bit_depth > 16 { + return Err(CudaError::InvalidArgument { + message: "bit depth must be between 1 and 16".to_string(), + }); + } + let bytes_per_sample = if bit_depth <= 8 { 1usize } else { 2usize }; + let bytes_per_pixel = usize::from(num_components) + .checked_mul(bytes_per_sample) + .ok_or(CudaError::LengthTooLarge { + len: usize::from(num_components), + })?; + let row_bytes = + (width as usize) + .checked_mul(bytes_per_pixel) + .ok_or(CudaError::ImageTooLarge { + width, + height, + channels: usize::from(num_components), + })?; + if pitch_bytes < row_bytes { + return Err(CudaError::InvalidArgument { + message: "pitch is shorter than one row".to_string(), + }); + } + let required_end = byte_offset + .checked_add( + pitch_bytes + .checked_mul(height.saturating_sub(1) as usize) + .and_then(|prefix| prefix.checked_add(row_bytes)) + .ok_or(CudaError::LengthTooLarge { len: pitch_bytes })?, + ) + .ok_or(CudaError::LengthTooLarge { len: byte_offset })?; + if required_end > pixels.byte_len() { + return Err(CudaError::OutputTooSmall { + required: required_end, + have: pixels.byte_len(), + }); + } + + self.inner.set_current()?; + let num_pixels = + (width as usize) + .checked_mul(height as usize) + .ok_or(CudaError::ImageTooLarge { + width, + height, + channels: usize::from(num_components), + })?; + let sample_count = num_pixels + .checked_mul(usize::from(num_components)) + .ok_or(CudaError::LengthTooLarge { len: num_pixels })?; + let output_bytes = sample_count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: sample_count })?; + let output = self.allocate(output_bytes)?; + self.launch_j2k_deinterleave_strided_to_f32( + pixels, + &output, + width, + height, + byte_offset, + pitch_bytes, + num_components, + bit_depth, + signed, + )?; + + Ok(CudaJ2kResidentComponents { + buffer: output, + num_pixels, + num_components, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }) + } + + /// Run the reversible color transform in place on resident component planes. + /// + /// The transform is applied to the first three planes (R, G, B → Y, Cb, Cr). + /// Any additional plane (e.g. a 4th alpha/auxiliary component) is left + /// untouched, matching the native reference which applies RCT to the first + /// three of `&mut [Vec]` and passes the remainder through unchanged. + pub fn j2k_forward_rct_resident( + &self, + components: &mut CudaJ2kResidentComponents, + ) -> Result { + if components.num_components < 3 { + return Err(CudaError::InvalidArgument { + message: "forward RCT requires at least three resident component planes" + .to_string(), + }); + } + if components.num_pixels == 0 { + return Ok(CudaExecutionStats::default()); + } + + self.inner.set_current()?; + let plane0 = components.component_plane_device_ptr(0)?; + let plane1 = components.component_plane_device_ptr(1)?; + let plane2 = components.component_plane_device_ptr(2)?; + self.launch_j2k_forward_rct_ptrs(plane0, plane1, plane2, components.num_pixels)?; + + Ok(CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }) + } + + /// Run the irreversible color transform in place on resident component planes. + /// + /// The transform is applied to the first three planes (R, G, B → Y, Cb, Cr). + /// Any additional plane is left untouched, matching the native reference + /// which applies ICT to the first three of `&mut [Vec]` and passes the + /// remainder through unchanged. + pub fn j2k_forward_ict_resident( + &self, + components: &mut CudaJ2kResidentComponents, + ) -> Result { + if components.num_components < 3 { + return Err(CudaError::InvalidArgument { + message: "forward ICT requires at least three resident component planes" + .to_string(), + }); + } + if components.num_pixels == 0 { + return Ok(CudaExecutionStats::default()); + } + + self.inner.set_current()?; + let plane0 = components.component_plane_device_ptr(0)?; + let plane1 = components.component_plane_device_ptr(1)?; + let plane2 = components.component_plane_device_ptr(2)?; + self.launch_j2k_forward_ict_ptrs(plane0, plane1, plane2, components.num_pixels)?; + + Ok(CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }) + } + + /// Run the reversible color transform stage on three component planes. + pub fn j2k_forward_rct( + &self, + plane0: &mut [f32], + plane1: &mut [f32], + plane2: &mut [f32], + ) -> Result { + if plane0.len() != plane1.len() || plane0.len() != plane2.len() { + return Err(CudaError::ImageTooLarge { + width: u32::try_from(plane0.len()).unwrap_or(u32::MAX), + height: 1, + channels: 3, + }); + } + if plane0.is_empty() { + return Ok(CudaExecutionStats::default()); + } + + self.inner.set_current()?; + let buffer0 = self.upload(f32_slice_as_bytes(plane0))?; + let buffer1 = self.upload(f32_slice_as_bytes(plane1))?; + let buffer2 = self.upload(f32_slice_as_bytes(plane2))?; + self.launch_j2k_forward_rct_buffers(&buffer0, &buffer1, &buffer2, plane0.len())?; + buffer0.copy_to_host(f32_slice_as_bytes_mut(plane0))?; + buffer1.copy_to_host(f32_slice_as_bytes_mut(plane1))?; + buffer2.copy_to_host(f32_slice_as_bytes_mut(plane2))?; + + Ok(CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }) + } + + /// Run the irreversible color transform stage on three component planes. + pub fn j2k_forward_ict( + &self, + plane0: &mut [f32], + plane1: &mut [f32], + plane2: &mut [f32], + ) -> Result { + if plane0.len() != plane1.len() || plane0.len() != plane2.len() { + return Err(CudaError::ImageTooLarge { + width: u32::try_from(plane0.len()).unwrap_or(u32::MAX), + height: 1, + channels: 3, + }); + } + if plane0.is_empty() { + return Ok(CudaExecutionStats::default()); + } + + self.inner.set_current()?; + let buffer0 = self.upload(f32_slice_as_bytes(plane0))?; + let buffer1 = self.upload(f32_slice_as_bytes(plane1))?; + let buffer2 = self.upload(f32_slice_as_bytes(plane2))?; + self.launch_j2k_forward_ict_buffers(&buffer0, &buffer1, &buffer2, plane0.len())?; + buffer0.copy_to_host(f32_slice_as_bytes_mut(plane0))?; + buffer1.copy_to_host(f32_slice_as_bytes_mut(plane1))?; + buffer2.copy_to_host(f32_slice_as_bytes_mut(plane2))?; + + Ok(CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }) + } + + /// Run the reversible 5/3 forward DWT stage on one component plane. + pub fn j2k_forward_dwt53( + &self, + samples: &[f32], + width: u32, + height: u32, + num_levels: u8, + ) -> Result { + let expected_len = + (width as usize) + .checked_mul(height as usize) + .ok_or(CudaError::ImageTooLarge { + width, + height, + channels: 1, + })?; + if expected_len != samples.len() { + return Err(CudaError::ImageTooLarge { + width, + height, + channels: 1, + }); + } + if samples.is_empty() || num_levels == 0 { + return Ok(CudaDwt53Output { + transformed: samples.to_vec(), + levels: Vec::new(), + ll_width: width, + ll_height: height, + execution: CudaExecutionStats::default(), + }); + } + + self.inner.set_current()?; + let buffer_a = self.upload(f32_slice_as_bytes(samples))?; + let resident = self.j2k_forward_dwt53_resident_buffer( + buffer_a, + samples.len(), + width, + height, + num_levels, + 0, + )?; + let transformed = resident.download_transformed()?; + Ok(CudaDwt53Output { + transformed, + levels: resident.levels().to_vec(), + ll_width: resident.ll_dimensions().0, + ll_height: resident.ll_dimensions().1, + execution: resident.execution(), + }) + } + + /// Run the reversible 5/3 forward DWT on one resident component plane. + pub fn j2k_forward_dwt53_resident_component( + &self, + components: &CudaJ2kResidentComponents, + component: u8, + width: u32, + height: u32, + num_levels: u8, + ) -> Result { + let expected_len = + (width as usize) + .checked_mul(height as usize) + .ok_or(CudaError::ImageTooLarge { + width, + height, + channels: 1, + })?; + if expected_len != components.num_pixels { + return Err(CudaError::ImageTooLarge { + width, + height, + channels: 1, + }); + } + + self.inner.set_current()?; + let plane_ptr = components.component_plane_device_ptr(component)?; + let byte_len = expected_len + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: expected_len })?; + let buffer_a = self.copy_device_ptr_to_device_with_kernel(plane_ptr, byte_len)?; + let copy_dispatches = usize::from(byte_len != 0); + self.j2k_forward_dwt53_resident_buffer( + buffer_a, + expected_len, + width, + height, + num_levels, + copy_dispatches, + ) + } + + fn j2k_forward_dwt53_resident_buffer( + &self, + buffer_a: CudaDeviceBuffer, + sample_count: usize, + width: u32, + height: u32, + num_levels: u8, + initial_copy_dispatches: usize, + ) -> Result { + if sample_count == 0 || num_levels == 0 { + return Ok(CudaResidentDwt53Output { + buffer: buffer_a, + sample_count, + levels: Vec::new(), + ll_width: width, + ll_height: height, + execution: CudaExecutionStats { + kernel_dispatches: initial_copy_dispatches, + copy_kernel_dispatches: initial_copy_dispatches, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }); + } + + let buffer_b = self.allocate( + sample_count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: sample_count })?, + )?; + let mut current_width = width; + let mut current_height = height; + let mut levels = Vec::new(); + let mut dispatches = 0usize; + let mut active_is_a = true; + + for _ in 0..num_levels { + if current_width < 2 && current_height < 2 { + break; + } + let (level_dispatches, level_shape) = self.launch_j2k_forward_dwt53_level( + &buffer_a, + &buffer_b, + &mut active_is_a, + CudaDwt53LevelPass { + full_width: width, + current_width, + current_height, + }, + )?; + dispatches = dispatches.saturating_add(level_dispatches); + levels.push(level_shape); + current_width = level_shape.low_width; + current_height = level_shape.low_height; + } + + let buffer = if active_is_a { buffer_a } else { buffer_b }; + Ok(CudaResidentDwt53Output { + buffer, + sample_count, + levels, + ll_width: current_width, + ll_height: current_height, + execution: CudaExecutionStats { + kernel_dispatches: initial_copy_dispatches.saturating_add(dispatches), + copy_kernel_dispatches: initial_copy_dispatches, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }) + } + + /// Run the irreversible 9/7 forward DWT stage on one component plane. + pub fn j2k_forward_dwt97( + &self, + samples: &[f32], + width: u32, + height: u32, + num_levels: u8, + ) -> Result { + let expected_len = + (width as usize) + .checked_mul(height as usize) + .ok_or(CudaError::ImageTooLarge { + width, + height, + channels: 1, + })?; + if expected_len != samples.len() { + return Err(CudaError::ImageTooLarge { + width, + height, + channels: 1, + }); + } + if samples.is_empty() || num_levels == 0 { + return Ok(CudaDwt97Output { + transformed: samples.to_vec(), + levels: Vec::new(), + ll_width: width, + ll_height: height, + execution: CudaExecutionStats::default(), + }); + } + + self.inner.set_current()?; + let buffer_a = self.upload(f32_slice_as_bytes(samples))?; + let resident = self.j2k_forward_dwt97_resident_buffer( + buffer_a, + samples.len(), + width, + height, + num_levels, + 0, + )?; + let transformed = resident.download_transformed()?; + Ok(CudaDwt97Output { + transformed, + levels: resident.levels().to_vec(), + ll_width: resident.ll_dimensions().0, + ll_height: resident.ll_dimensions().1, + execution: resident.execution(), + }) + } + + /// Run the irreversible 9/7 forward DWT on one resident component plane. + pub fn j2k_forward_dwt97_resident_component( + &self, + components: &CudaJ2kResidentComponents, + component: u8, + width: u32, + height: u32, + num_levels: u8, + ) -> Result { + let expected_len = + (width as usize) + .checked_mul(height as usize) + .ok_or(CudaError::ImageTooLarge { + width, + height, + channels: 1, + })?; + if expected_len != components.num_pixels { + return Err(CudaError::ImageTooLarge { + width, + height, + channels: 1, + }); + } + + self.inner.set_current()?; + let plane_ptr = components.component_plane_device_ptr(component)?; + let byte_len = expected_len + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: expected_len })?; + let buffer_a = self.copy_device_ptr_to_device_with_kernel(plane_ptr, byte_len)?; + let copy_dispatches = usize::from(byte_len != 0); + self.j2k_forward_dwt97_resident_buffer( + buffer_a, + expected_len, + width, + height, + num_levels, + copy_dispatches, + ) + } + + fn j2k_forward_dwt97_resident_buffer( + &self, + buffer_a: CudaDeviceBuffer, + sample_count: usize, + width: u32, + height: u32, + num_levels: u8, + initial_copy_dispatches: usize, + ) -> Result { + if sample_count == 0 || num_levels == 0 { + return Ok(CudaResidentDwt97Output { + buffer: buffer_a, + sample_count, + levels: Vec::new(), + ll_width: width, + ll_height: height, + execution: CudaExecutionStats { + kernel_dispatches: initial_copy_dispatches, + copy_kernel_dispatches: initial_copy_dispatches, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }); + } + + let buffer_b = self.allocate( + sample_count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: sample_count })?, + )?; + let mut current_width = width; + let mut current_height = height; + let mut levels = Vec::new(); + let mut dispatches = 0usize; + let mut active_is_a = true; + + for _ in 0..num_levels { + if current_width < 2 && current_height < 2 { + break; + } + let (level_dispatches, level_shape) = self.launch_j2k_forward_dwt97_level( + &buffer_a, + &buffer_b, + &mut active_is_a, + CudaDwt53LevelPass { + full_width: width, + current_width, + current_height, + }, + )?; + dispatches = dispatches.saturating_add(level_dispatches); + levels.push(level_shape); + current_width = level_shape.low_width; + current_height = level_shape.low_height; + } + + let buffer = if active_is_a { buffer_a } else { buffer_b }; + Ok(CudaResidentDwt97Output { + buffer, + sample_count, + levels, + ll_width: current_width, + ll_height: current_height, + execution: CudaExecutionStats { + kernel_dispatches: initial_copy_dispatches.saturating_add(dispatches), + copy_kernel_dispatches: initial_copy_dispatches, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }) + } + + /// Quantize one JPEG 2000 sub-band on the device. + pub fn j2k_quantize_subband( + &self, + samples: &[f32], + job: CudaJ2kQuantizeJob, + ) -> Result { + let sample_buffer = self.upload(f32_slice_as_bytes(samples))?; + let resident = self.j2k_quantize_subband_resident(&sample_buffer, samples.len(), job)?; + let coefficients = resident.download_coefficients()?; + Ok(CudaJ2kQuantizedSubband { + coefficients, + execution: resident.execution(), + }) + } + + /// Quantize a resident contiguous JPEG 2000 sub-band into resident `i32` coefficients. + pub fn j2k_quantize_subband_resident( + &self, + samples: &CudaDeviceBuffer, + sample_count: usize, + job: CudaJ2kQuantizeJob, + ) -> Result { + if sample_count == 0 { + return Ok(CudaJ2kResidentQuantizedSubband { + coefficients: self.allocate(0)?, + coefficient_count: 0, + execution: CudaExecutionStats::default(), + }); + } + + let available_samples = samples.typed_view::()?.len(); + if available_samples < sample_count { + return Err(CudaError::OutputTooSmall { + required: sample_count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: sample_count })?, + have: samples.byte_len(), + }); + } + + self.inner.set_current()?; + let coefficient_buffer = self.allocate( + sample_count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: sample_count })?, + )?; + self.launch_j2k_quantize_subband(samples, &coefficient_buffer, sample_count, job)?; + + Ok(CudaJ2kResidentQuantizedSubband { + coefficients: coefficient_buffer, + coefficient_count: sample_count, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }) + } + + /// Quantize a resident strided DWT sub-band rectangle into resident `i32` coefficients. + pub fn j2k_quantize_subband_region_resident( + &self, + samples: &CudaDeviceBuffer, + job: CudaJ2kQuantizeSubbandRegionJob, + ) -> Result { + let coefficient_count = checked_image_words(job.width, job.height, 1)?; + if coefficient_count == 0 { + return Ok(CudaJ2kResidentQuantizedSubband { + coefficients: self.allocate(0)?, + coefficient_count: 0, + execution: CudaExecutionStats::default(), + }); + } + + let available_samples = samples.typed_view::()?.len(); + validate_quantize_region(job, available_samples)?; + self.inner.set_current()?; + let coefficient_buffer = self.allocate( + coefficient_count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { + len: coefficient_count, + })?, + )?; + self.launch_j2k_quantize_subband_region(samples, &coefficient_buffer, job)?; + + Ok(CudaJ2kResidentQuantizedSubband { + coefficients: coefficient_buffer, + coefficient_count, + execution: CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }) + } + + fn launch_j2k_forward_dwt53_level( + &self, + buffer_a: &CudaDeviceBuffer, + buffer_b: &CudaDeviceBuffer, + active_is_a: &mut bool, + pass: CudaDwt53LevelPass, + ) -> Result<(usize, CudaDwt53LevelShape), CudaError> { + let low_width = pass.current_width.div_ceil(2); + let low_height = pass.current_height.div_ceil(2); + let mut dispatches = 0usize; + + if pass.current_height >= 2 { + let (input, output) = active_dwt53_buffers(buffer_a, buffer_b, *active_is_a); + self.launch_j2k_forward_dwt53_pass( + CudaKernel::J2kForwardDwt53Vertical, + input, + output, + CudaDwt53Pass { + full_width: pass.full_width, + current_width: pass.current_width, + current_height: pass.current_height, + low_extent: low_height, + }, + )?; + *active_is_a = !*active_is_a; + dispatches = dispatches.saturating_add(1); + } + + if pass.current_width >= 2 { + let (input, output) = active_dwt53_buffers(buffer_a, buffer_b, *active_is_a); + self.launch_j2k_forward_dwt53_pass( + CudaKernel::J2kForwardDwt53Horizontal, + input, + output, + CudaDwt53Pass { + full_width: pass.full_width, + current_width: pass.current_width, + current_height: pass.current_height, + low_extent: low_width, + }, + )?; + *active_is_a = !*active_is_a; + dispatches = dispatches.saturating_add(1); + } + + Ok(( + dispatches, + CudaDwt53LevelShape { + width: pass.current_width, + height: pass.current_height, + low_width, + low_height, + high_width: pass.current_width / 2, + high_height: pass.current_height / 2, + }, + )) + } + + fn launch_j2k_forward_dwt97_level( + &self, + buffer_a: &CudaDeviceBuffer, + buffer_b: &CudaDeviceBuffer, + active_is_a: &mut bool, + pass: CudaDwt53LevelPass, + ) -> Result<(usize, CudaDwt53LevelShape), CudaError> { + let low_width = pass.current_width.div_ceil(2); + let low_height = pass.current_height.div_ceil(2); + let mut dispatches = 0usize; + + if pass.current_height >= 2 { + let (input, output) = active_dwt53_buffers(buffer_a, buffer_b, *active_is_a); + self.launch_j2k_forward_dwt53_pass( + CudaKernel::J2kForwardDwt97Vertical, + input, + output, + CudaDwt53Pass { + full_width: pass.full_width, + current_width: pass.current_width, + current_height: pass.current_height, + low_extent: low_height, + }, + )?; + *active_is_a = !*active_is_a; + dispatches = dispatches.saturating_add(1); + } + + if pass.current_width >= 2 { + let (input, output) = active_dwt53_buffers(buffer_a, buffer_b, *active_is_a); + self.launch_j2k_forward_dwt53_pass( + CudaKernel::J2kForwardDwt97Horizontal, + input, + output, + CudaDwt53Pass { + full_width: pass.full_width, + current_width: pass.current_width, + current_height: pass.current_height, + low_extent: low_width, + }, + )?; + *active_is_a = !*active_is_a; + dispatches = dispatches.saturating_add(1); + } + + Ok(( + dispatches, + CudaDwt53LevelShape { + width: pass.current_width, + height: pass.current_height, + low_width, + low_height, + high_width: pass.current_width / 2, + high_height: pass.current_height / 2, + }, + )) + } + + fn launch_j2k_forward_rct_buffers( + &self, + plane0: &CudaDeviceBuffer, + plane1: &CudaDeviceBuffer, + plane2: &CudaDeviceBuffer, + len: usize, + ) -> Result<(), CudaError> { + self.launch_j2k_forward_rct_ptrs( + plane0.device_ptr(), + plane1.device_ptr(), + plane2.device_ptr(), + len, + ) + } + + fn j2k_encode_kernel_function(&self, kernel: CudaKernel) -> Result { + #[cfg(feature = "cuda-oxide-j2k-encode")] + { + if crate::build_flags::cuda_oxide_j2k_encode_enabled() { + return self.inner.cuda_oxide_j2k_encode_kernel_function(kernel); + } + } + self.inner.kernel_function(kernel) + } + + fn launch_j2k_forward_rct_ptrs( + &self, + plane0: CuDevicePtr, + plane1: CuDevicePtr, + plane2: CuDevicePtr, + len: usize, + ) -> Result<(), CudaError> { + let function = self.j2k_encode_kernel_function(CudaKernel::J2kForwardRct)?; + let mut plane0_ptr = plane0; + let mut plane1_ptr = plane1; + let mut plane2_ptr = plane2; + let mut len_u64 = u64::try_from(len).map_err(|_| CudaError::LengthTooLarge { len })?; + let mut params = cuda_kernel_params!(plane0_ptr, plane1_ptr, plane2_ptr, len_u64); + let geometry = + j2k_forward_rct_launch_geometry(len).ok_or(CudaError::LengthTooLarge { len })?; + + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_j2k_deinterleave_to_f32( + &self, + pixels: &CudaDeviceBuffer, + output: &CudaDeviceBuffer, + num_pixels: usize, + num_components: u8, + bit_depth: u8, + signed: bool, + ) -> Result<(), CudaError> { + let function = self.j2k_encode_kernel_function(CudaKernel::J2kDeinterleaveToF32)?; + let mut pixels_ptr = pixels.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut num_pixels_u64 = + u64::try_from(num_pixels).map_err(|_| CudaError::LengthTooLarge { len: num_pixels })?; + let mut num_components_u32 = u32::from(num_components); + let mut bit_depth_u32 = u32::from(bit_depth); + let mut signed_u32 = u32::from(signed); + let mut params = cuda_kernel_params!( + pixels_ptr, + output_ptr, + num_pixels_u64, + num_components_u32, + bit_depth_u32, + signed_u32 + ); + let geometry = j2k_forward_rct_launch_geometry(num_pixels) + .ok_or(CudaError::LengthTooLarge { len: num_pixels })?; + + self.launch_kernel(function, geometry, &mut params) + } + + #[allow(clippy::too_many_arguments)] + fn launch_j2k_deinterleave_strided_to_f32( + &self, + pixels: &CudaDeviceBuffer, + output: &CudaDeviceBuffer, + width: u32, + height: u32, + byte_offset: usize, + pitch_bytes: usize, + num_components: u8, + bit_depth: u8, + signed: bool, + ) -> Result<(), CudaError> { + let function = self.j2k_encode_kernel_function(CudaKernel::J2kDeinterleaveStridedToF32)?; + let mut pixels_ptr = pixels.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut width_u64 = u64::from(width); + let mut height_u64 = u64::from(height); + let mut byte_offset_u64 = u64::try_from(byte_offset) + .map_err(|_| CudaError::LengthTooLarge { len: byte_offset })?; + let mut pitch_bytes_u64 = u64::try_from(pitch_bytes) + .map_err(|_| CudaError::LengthTooLarge { len: pitch_bytes })?; + let mut num_components_u32 = u32::from(num_components); + let mut bit_depth_u32 = u32::from(bit_depth); + let mut signed_u32 = u32::from(signed); + let mut params = cuda_kernel_params!( + pixels_ptr, + output_ptr, + width_u64, + height_u64, + byte_offset_u64, + pitch_bytes_u64, + num_components_u32, + bit_depth_u32, + signed_u32 + ); + let num_pixels = + (width as usize) + .checked_mul(height as usize) + .ok_or(CudaError::ImageTooLarge { + width, + height, + channels: usize::from(num_components), + })?; + let geometry = j2k_forward_rct_launch_geometry(num_pixels) + .ok_or(CudaError::LengthTooLarge { len: num_pixels })?; + + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_j2k_forward_ict_buffers( + &self, + plane0: &CudaDeviceBuffer, + plane1: &CudaDeviceBuffer, + plane2: &CudaDeviceBuffer, + len: usize, + ) -> Result<(), CudaError> { + self.launch_j2k_forward_ict_ptrs( + plane0.device_ptr(), + plane1.device_ptr(), + plane2.device_ptr(), + len, + ) + } + + fn launch_j2k_forward_ict_ptrs( + &self, + plane0: CuDevicePtr, + plane1: CuDevicePtr, + plane2: CuDevicePtr, + len: usize, + ) -> Result<(), CudaError> { + let function = self.j2k_encode_kernel_function(CudaKernel::J2kForwardIct)?; + let mut plane0_ptr = plane0; + let mut plane1_ptr = plane1; + let mut plane2_ptr = plane2; + let mut len_u64 = u64::try_from(len).map_err(|_| CudaError::LengthTooLarge { len })?; + let mut params = cuda_kernel_params!(plane0_ptr, plane1_ptr, plane2_ptr, len_u64); + let geometry = + j2k_forward_rct_launch_geometry(len).ok_or(CudaError::LengthTooLarge { len })?; + + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_j2k_forward_dwt53_pass( + &self, + kernel: CudaKernel, + input: &CudaDeviceBuffer, + output: &CudaDeviceBuffer, + pass: CudaDwt53Pass, + ) -> Result<(), CudaError> { + let function = self.j2k_encode_kernel_function(kernel)?; + let mut input_ptr = input.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut full_width = pass.full_width; + let mut current_width = pass.current_width; + let mut current_height = pass.current_height; + let mut low_extent = pass.low_extent; + let mut params = cuda_kernel_params!( + input_ptr, + output_ptr, + full_width, + current_width, + current_height, + low_extent + ); + let geometry = j2k_dwt53_launch_geometry(current_width, current_height).ok_or( + CudaError::ImageTooLarge { + width: pass.current_width, + height: pass.current_height, + channels: 1, + }, + )?; + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_j2k_quantize_subband( + &self, + samples: &CudaDeviceBuffer, + coefficients: &CudaDeviceBuffer, + len: usize, + job: CudaJ2kQuantizeJob, + ) -> Result<(), CudaError> { + let function = self.j2k_encode_kernel_function(CudaKernel::J2kQuantizeSubband)?; + let mut samples_ptr = samples.device_ptr(); + let mut coefficients_ptr = coefficients.device_ptr(); + let mut len_u64 = u64::try_from(len).map_err(|_| CudaError::LengthTooLarge { len })?; + let mut step_exponent = u32::from(job.step_exponent); + let mut step_mantissa = u32::from(job.step_mantissa); + let mut range_bits = u32::from(job.range_bits); + let mut reversible = u32::from(job.reversible); + let mut params = cuda_kernel_params!( + samples_ptr, + coefficients_ptr, + len_u64, + step_exponent, + step_mantissa, + range_bits, + reversible + ); + let geometry = + j2k_forward_rct_launch_geometry(len).ok_or(CudaError::LengthTooLarge { len })?; + + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_j2k_quantize_subband_region( + &self, + samples: &CudaDeviceBuffer, + coefficients: &CudaDeviceBuffer, + job: CudaJ2kQuantizeSubbandRegionJob, + ) -> Result<(), CudaError> { + let function = self.j2k_encode_kernel_function(CudaKernel::J2kQuantizeSubbandStrided)?; + let mut samples_ptr = samples.device_ptr(); + let mut coefficients_ptr = coefficients.device_ptr(); + let mut x0 = job.x0; + let mut y0 = job.y0; + let mut width = job.width; + let mut height = job.height; + let mut stride = job.stride; + let mut step_exponent = u32::from(job.quantization.step_exponent); + let mut step_mantissa = u32::from(job.quantization.step_mantissa); + let mut range_bits = u32::from(job.quantization.range_bits); + let mut reversible = u32::from(job.quantization.reversible); + let mut params = cuda_kernel_params!( + samples_ptr, + coefficients_ptr, + x0, + y0, + width, + height, + stride, + step_exponent, + step_mantissa, + range_bits, + reversible + ); + let geometry = + j2k_dwt53_launch_geometry(job.width, job.height).ok_or(CudaError::ImageTooLarge { + width: job.width, + height: job.height, + channels: 1, + })?; + + self.launch_kernel(function, geometry, &mut params) + } +} + +/// Resident f32 component planes produced by CUDA JPEG 2000 encode preparation. +#[derive(Debug)] +pub struct CudaJ2kResidentComponents { + pub(crate) buffer: CudaDeviceBuffer, + pub(crate) num_pixels: usize, + pub(crate) num_components: u8, + pub(crate) execution: CudaExecutionStats, +} + +impl CudaJ2kResidentComponents { + /// Contiguous component-major f32 device buffer. + pub fn buffer(&self) -> &CudaDeviceBuffer { + &self.buffer + } + + /// Number of pixels in each component plane. + pub fn num_pixels(&self) -> usize { + self.num_pixels + } + + /// Number of resident component planes. + pub fn num_components(&self) -> u8 { + self.num_components + } + + /// CUDA execution counters for the producing dispatch. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// Download component planes into host memory for verification or host APIs. + pub fn download_components(&self) -> Result>, CudaError> { + if self.num_pixels == 0 { + return Ok(vec![Vec::new(); usize::from(self.num_components)]); + } + let sample_count = self + .num_pixels + .checked_mul(usize::from(self.num_components)) + .ok_or(CudaError::LengthTooLarge { + len: self.num_pixels, + })?; + let mut flattened = vec![0.0f32; sample_count]; + self.buffer + .copy_to_host(f32_slice_as_bytes_mut(&mut flattened))?; + Ok(flattened + .chunks_exact(self.num_pixels) + .map(<[f32]>::to_vec) + .collect()) + } + + fn component_plane_device_ptr(&self, component: u8) -> Result { + if component >= self.num_components { + return Err(CudaError::InvalidArgument { + message: "component plane index is out of range".to_string(), + }); + } + let plane_bytes = self + .num_pixels + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { + len: self.num_pixels, + })?; + let offset = plane_bytes + .checked_mul(usize::from(component)) + .ok_or(CudaError::LengthTooLarge { len: plane_bytes })?; + let end = offset + .checked_add(plane_bytes) + .ok_or(CudaError::LengthTooLarge { len: offset })?; + if end > self.buffer.byte_len() { + return Err(CudaError::OutputTooSmall { + required: end, + have: self.buffer.byte_len(), + }); + } + let offset = + u64::try_from(offset).map_err(|_| CudaError::LengthTooLarge { len: offset })?; + self.buffer + .device_ptr() + .checked_add(offset) + .ok_or(CudaError::LengthTooLarge { + len: self.buffer.byte_len(), + }) + } +} + +/// Host-visible component planes produced by CUDA pixel deinterleave. +#[derive(Debug)] +pub struct CudaJ2kDeinterleavedComponents { + pub(crate) components: Vec>, + pub(crate) execution: CudaExecutionStats, +} + +impl CudaJ2kDeinterleavedComponents { + /// Per-component f32 sample planes in component order. + pub fn components(&self) -> &[Vec] { + &self.components + } + + /// CUDA execution counters for the deinterleave dispatch. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } + + /// Consume the output and return owned component planes. + pub fn into_components(self) -> Vec> { + self.components + } +} + +/// Forward 5/3 DWT output and level metadata. +#[derive(Debug)] +pub struct CudaDwt53Output { + pub(crate) transformed: Vec, + pub(crate) levels: Vec, + pub(crate) ll_width: u32, + pub(crate) ll_height: u32, + pub(crate) execution: CudaExecutionStats, +} + +impl CudaDwt53Output { + /// Transformed coefficients downloaded to host memory. + pub fn transformed(&self) -> &[f32] { + &self.transformed + } + + /// Per-level DWT shapes. + pub fn levels(&self) -> &[CudaDwt53LevelShape] { + &self.levels + } + + /// Dimensions of the final low-low band. + pub fn ll_dimensions(&self) -> (u32, u32) { + (self.ll_width, self.ll_height) + } + + /// CUDA execution counters for the transform. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } +} + +/// Resident forward 5/3 DWT output and level metadata. +#[derive(Debug)] +pub struct CudaResidentDwt53Output { + pub(crate) buffer: CudaDeviceBuffer, + pub(crate) sample_count: usize, + pub(crate) levels: Vec, + pub(crate) ll_width: u32, + pub(crate) ll_height: u32, + pub(crate) execution: CudaExecutionStats, +} + +impl CudaResidentDwt53Output { + /// Resident component-major transformed coefficient buffer. + pub fn buffer(&self) -> &CudaDeviceBuffer { + &self.buffer + } + + /// Transformed coefficient count. + pub fn sample_count(&self) -> usize { + self.sample_count + } + + /// Download transformed coefficients into host memory. + pub fn download_transformed(&self) -> Result, CudaError> { + let mut transformed = vec![0f32; self.sample_count]; + self.buffer + .copy_to_host(f32_slice_as_bytes_mut(&mut transformed))?; + Ok(transformed) + } + + /// Per-level DWT shapes. + pub fn levels(&self) -> &[CudaDwt53LevelShape] { + &self.levels + } + + /// Dimensions of the final low-low band. + pub fn ll_dimensions(&self) -> (u32, u32) { + (self.ll_width, self.ll_height) + } + + /// CUDA execution counters for the transform. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } +} + +/// Forward 9/7 DWT output and level metadata. +#[derive(Debug)] +pub struct CudaDwt97Output { + pub(crate) transformed: Vec, + pub(crate) levels: Vec, + pub(crate) ll_width: u32, + pub(crate) ll_height: u32, + pub(crate) execution: CudaExecutionStats, +} + +impl CudaDwt97Output { + /// Transformed coefficients downloaded to host memory. + pub fn transformed(&self) -> &[f32] { + &self.transformed + } + + /// Per-level DWT shapes. + pub fn levels(&self) -> &[CudaDwt53LevelShape] { + &self.levels + } + + /// Dimensions of the final low-low band. + pub fn ll_dimensions(&self) -> (u32, u32) { + (self.ll_width, self.ll_height) + } + + /// CUDA execution counters for the transform. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } +} + +/// Resident forward 9/7 DWT output and level metadata. +#[derive(Debug)] +pub struct CudaResidentDwt97Output { + pub(crate) buffer: CudaDeviceBuffer, + pub(crate) sample_count: usize, + pub(crate) levels: Vec, + pub(crate) ll_width: u32, + pub(crate) ll_height: u32, + pub(crate) execution: CudaExecutionStats, +} + +impl CudaResidentDwt97Output { + /// Resident component-major transformed coefficient buffer. + pub fn buffer(&self) -> &CudaDeviceBuffer { + &self.buffer + } + + /// Transformed coefficient count. + pub fn sample_count(&self) -> usize { + self.sample_count + } + + /// Download transformed coefficients into host memory. + pub fn download_transformed(&self) -> Result, CudaError> { + let mut transformed = vec![0f32; self.sample_count]; + self.buffer + .copy_to_host(f32_slice_as_bytes_mut(&mut transformed))?; + Ok(transformed) + } + + /// Per-level DWT shapes. + pub fn levels(&self) -> &[CudaDwt53LevelShape] { + &self.levels + } + + /// Dimensions of the final low-low band. + pub fn ll_dimensions(&self) -> (u32, u32) { + (self.ll_width, self.ll_height) + } + + /// CUDA execution counters for the transform. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } +} + +/// JPEG 2000 sub-band quantization parameters. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaJ2kQuantizeJob { + /// Quantization step-size exponent. + pub step_exponent: u16, + /// Quantization step-size mantissa. + pub step_mantissa: u16, + /// Nominal range bits for this sub-band. + pub range_bits: u8, + /// Whether to use reversible integer quantization. + pub reversible: bool, +} + +/// Resident strided sub-band rectangle and quantization parameters. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaJ2kQuantizeSubbandRegionJob { + /// X offset, in f32 samples, of the sub-band rectangle inside the resident plane. + pub x0: u32, + /// Y offset, in f32 samples, of the sub-band rectangle inside the resident plane. + pub y0: u32, + /// Sub-band rectangle width in samples. + pub width: u32, + /// Sub-band rectangle height in samples. + pub height: u32, + /// Resident source row stride in f32 samples. + pub stride: u32, + /// Quantization parameters applied to every source sample. + pub quantization: CudaJ2kQuantizeJob, +} + +/// Quantized JPEG 2000 sub-band coefficients and execution metadata. +#[derive(Debug)] +pub struct CudaJ2kQuantizedSubband { + pub(crate) coefficients: Vec, + pub(crate) execution: CudaExecutionStats, +} + +impl CudaJ2kQuantizedSubband { + /// Quantized sub-band coefficients downloaded to host memory. + pub fn coefficients(&self) -> &[i32] { + &self.coefficients + } + + /// CUDA execution counters for the quantization stage. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } +} + +/// Device-resident quantized JPEG 2000 sub-band coefficients and execution metadata. +#[derive(Debug)] +pub struct CudaJ2kResidentQuantizedSubband { + pub(crate) coefficients: CudaDeviceBuffer, + pub(crate) coefficient_count: usize, + pub(crate) execution: CudaExecutionStats, +} + +impl CudaJ2kResidentQuantizedSubband { + /// Device buffer containing row-major `i32` coefficients. + pub fn buffer(&self) -> &CudaDeviceBuffer { + &self.coefficients + } + + /// Number of `i32` coefficients in the resident buffer. + pub fn coefficient_count(&self) -> usize { + self.coefficient_count + } + + /// Copy quantized coefficients to host memory. + pub fn download_coefficients(&self) -> Result, CudaError> { + let mut coefficients = vec![0i32; self.coefficient_count]; + self.coefficients + .copy_to_host(i32_slice_as_bytes_mut(&mut coefficients))?; + Ok(coefficients) + } + + /// CUDA execution counters for the quantization stage. + pub fn execution(&self) -> CudaExecutionStats { + self.execution + } +} + +/// Shape metadata for one forward 5/3 DWT level. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaDwt53LevelShape { + /// Input level width. + pub width: u32, + /// Input level height. + pub height: u32, + /// Low-pass width. + pub low_width: u32, + /// Low-pass height. + pub low_height: u32, + /// High-pass width. + pub high_width: u32, + /// High-pass height. + pub high_height: u32, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct CudaDwt53Pass { + pub(crate) full_width: u32, + pub(crate) current_width: u32, + pub(crate) current_height: u32, + pub(crate) low_extent: u32, +} + +#[derive(Clone, Copy, Debug)] +pub(crate) struct CudaDwt53LevelPass { + pub(crate) full_width: u32, + pub(crate) current_width: u32, + pub(crate) current_height: u32, +} + +/// Backend stage timings for a same-geometry 9/7 (or fused code-block) batch. +/// +/// Mirrors `j2k-transcode`'s `Dwt97BatchStageTimings`; kept local because +/// `j2k-cuda-runtime` does not depend on `j2k-transcode`. The dispatch +/// layer maps this onto the transcode type. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CudaDwt97BatchStageTimings { + /// Buffer allocation plus host-to-device block upload time, microseconds. + pub pack_upload_us: u128, + /// IDCT plus horizontal 9/7 row-lift stage time, microseconds. + pub idct_row_lift_us: u128, + /// Vertical 9/7 column-lift stage time, microseconds. + pub column_lift_us: u128, + /// Code-block quantization stage time, microseconds (0 for the band path). + pub quantize_codeblock_us: u128, + /// Resident HT code-block encode time, microseconds. + pub ht_encode_us: u128, + /// Resident HT code-block encode dispatches. + pub ht_codeblock_dispatches: usize, + /// Device-to-host readback and unpack time, microseconds. + pub readback_us: u128, +} + +pub(crate) fn validate_quantize_region( + job: CudaJ2kQuantizeSubbandRegionJob, + available_samples: usize, +) -> Result<(), CudaError> { + if job.width == 0 || job.height == 0 { + return Ok(()); + } + if job.stride == 0 + || job + .x0 + .checked_add(job.width) + .is_none_or(|end_x| end_x > job.stride) + { + return Err(CudaError::LengthTooLarge { + len: available_samples, + }); + } + + let last_sample = (job.y0 as usize) + .checked_add(job.height as usize - 1) + .and_then(|row| row.checked_mul(job.stride as usize)) + .and_then(|row| row.checked_add(job.x0 as usize)) + .and_then(|row| row.checked_add(job.width as usize)) + .ok_or(CudaError::LengthTooLarge { + len: available_samples, + })?; + if last_sample > available_samples { + return Err(CudaError::OutputTooSmall { + required: last_sample + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: last_sample })?, + have: available_samples + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { + len: available_samples, + })?, + }); + } + Ok(()) +} diff --git a/crates/j2k-cuda-runtime/src/j2k_encode_kernels.cu b/crates/j2k-cuda-runtime/src/j2k_encode_kernels.cu new file mode 100644 index 00000000..c53164fd --- /dev/null +++ b/crates/j2k-cuda-runtime/src/j2k_encode_kernels.cu @@ -0,0 +1,506 @@ +#include + +static constexpr float J2K_FDWT97_ALPHA = -1.5861343f; +static constexpr float J2K_FDWT97_BETA = -0.052980117f; +static constexpr float J2K_FDWT97_GAMMA = 0.8829111f; +static constexpr float J2K_FDWT97_DELTA = 0.44350687f; +static constexpr float J2K_FDWT97_KAPPA = 1.2301741f; +static constexpr float J2K_FDWT97_INV_KAPPA = 1.0f / J2K_FDWT97_KAPPA; + +extern "C" __global__ void j2k_deinterleave_to_f32( + const unsigned char *pixels, + float *components, + unsigned long long num_pixels, + unsigned int num_components, + unsigned int bit_depth, + unsigned int is_signed +) { + const unsigned long long idx = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (idx >= num_pixels || num_components == 0u || num_components > 4u) { + return; + } + + const unsigned int bytes_per_sample = bit_depth <= 8u ? 1u : 2u; + const float unsigned_offset = + is_signed != 0u ? 0.0f : float(1u << (bit_depth - 1u)); + const unsigned long long pixel_base = + idx * static_cast(num_components) * bytes_per_sample; + for (unsigned int component = 0u; component < num_components; ++component) { + const unsigned long long sample_base = + pixel_base + static_cast(component) * bytes_per_sample; + float sample; + if (bit_depth <= 8u) { + const unsigned char raw = pixels[sample_base]; + sample = is_signed != 0u + ? float(static_cast(raw)) + : float(raw) - unsigned_offset; + } else { + const unsigned short raw = + static_cast(pixels[sample_base]) + | (static_cast(pixels[sample_base + 1u]) << 8u); + sample = is_signed != 0u + ? float(static_cast(raw)) + : float(raw) - unsigned_offset; + } + components[static_cast(component) * num_pixels + idx] = sample; + } +} + +extern "C" __global__ void j2k_deinterleave_strided_to_f32( + const unsigned char *pixels, + float *components, + unsigned long long width, + unsigned long long height, + unsigned long long byte_offset, + unsigned long long pitch_bytes, + unsigned int num_components, + unsigned int bit_depth, + unsigned int is_signed +) { + const unsigned long long idx = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + const unsigned long long num_pixels = width * height; + if (idx >= num_pixels || num_components == 0u || num_components > 4u) { + return; + } + + const unsigned int bytes_per_sample = bit_depth <= 8u ? 1u : 2u; + const float unsigned_offset = + is_signed != 0u ? 0.0f : float(1u << (bit_depth - 1u)); + const unsigned long long y = idx / width; + const unsigned long long x = idx - y * width; + const unsigned long long pixel_base = + byte_offset + y * pitch_bytes + + x * static_cast(num_components) * bytes_per_sample; + for (unsigned int component = 0u; component < num_components; ++component) { + const unsigned long long sample_base = + pixel_base + static_cast(component) * bytes_per_sample; + float sample; + if (bit_depth <= 8u) { + const unsigned char raw = pixels[sample_base]; + sample = is_signed != 0u + ? float(static_cast(raw)) + : float(raw) - unsigned_offset; + } else { + const unsigned short raw = + static_cast(pixels[sample_base]) + | (static_cast(pixels[sample_base + 1u]) << 8u); + sample = is_signed != 0u + ? float(static_cast(raw)) + : float(raw) - unsigned_offset; + } + components[static_cast(component) * num_pixels + idx] = sample; + } +} + +extern "C" __global__ void j2k_forward_rct( + float *plane0, + float *plane1, + float *plane2, + unsigned long long len +) { + const unsigned long long idx = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (idx >= len) { + return; + } + + const float r = plane0[idx]; + const float g = plane1[idx]; + const float b = plane2[idx]; + plane0[idx] = floorf((r + 2.0f * g + b) * 0.25f); + plane1[idx] = b - g; + plane2[idx] = r - g; +} + +extern "C" __global__ void j2k_forward_ict( + float *plane0, + float *plane1, + float *plane2, + unsigned long long len +) { + const unsigned long long idx = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (idx >= len) { + return; + } + + const float r = plane0[idx]; + const float g = plane1[idx]; + const float b = plane2[idx]; + plane0[idx] = 0.299f * r + 0.587f * g + 0.114f * b; + plane1[idx] = -0.16875f * r - 0.33126f * g + 0.5f * b; + plane2[idx] = 0.5f * r - 0.41869f * g - 0.08131f * b; +} + +__device__ float j2k_fdwt53_predict_row( + const float *src, + unsigned int row_base, + unsigned int width, + unsigned int high_index +) { + const unsigned int odd = high_index * 2u + 1u; + const unsigned int last_even = (width % 2u == 0u) ? width - 2u : width - 1u; + const float left = src[row_base + odd - 1u]; + const float right = (odd + 1u < width) ? src[row_base + odd + 1u] : src[row_base + last_even]; + return src[row_base + odd] - floorf((left + right) * 0.5f); +} + +__device__ float j2k_fdwt53_predict_col( + const float *src, + unsigned int x, + unsigned int full_width, + unsigned int height, + unsigned int high_index +) { + const unsigned int odd = high_index * 2u + 1u; + const unsigned int last_even = (height % 2u == 0u) ? height - 2u : height - 1u; + const float top = src[(odd - 1u) * full_width + x]; + const float bottom = (odd + 1u < height) + ? src[(odd + 1u) * full_width + x] + : src[last_even * full_width + x]; + return src[odd * full_width + x] - floorf((top + bottom) * 0.5f); +} + +extern "C" __global__ void j2k_forward_dwt53_horizontal( + const float *src, + float *dst, + unsigned int full_width, + unsigned int current_width, + unsigned int current_height, + unsigned int low_width +) { + const unsigned int x = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= current_width || y >= current_height) { + return; + } + + const unsigned int row_base = y * full_width; + if (x < low_width) { + const unsigned int even = x * 2u; + const float left = x > 0u + ? j2k_fdwt53_predict_row(src, row_base, current_width, x - 1u) + : j2k_fdwt53_predict_row(src, row_base, current_width, 0u); + const float right = even + 1u < current_width + ? j2k_fdwt53_predict_row(src, row_base, current_width, x) + : left; + dst[row_base + x] = src[row_base + even] + floorf((left + right) * 0.25f + 0.5f); + return; + } + + dst[row_base + x] = j2k_fdwt53_predict_row( + src, + row_base, + current_width, + x - low_width + ); +} + +extern "C" __global__ void j2k_forward_dwt53_vertical( + const float *src, + float *dst, + unsigned int full_width, + unsigned int current_width, + unsigned int current_height, + unsigned int low_height +) { + const unsigned int x = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= current_width || y >= current_height) { + return; + } + + if (y < low_height) { + const unsigned int even = y * 2u; + const float top = y > 0u + ? j2k_fdwt53_predict_col(src, x, full_width, current_height, y - 1u) + : j2k_fdwt53_predict_col(src, x, full_width, current_height, 0u); + const float bottom = even + 1u < current_height + ? j2k_fdwt53_predict_col(src, x, full_width, current_height, y) + : top; + dst[y * full_width + x] = + src[even * full_width + x] + floorf((top + bottom) * 0.25f + 0.5f); + return; + } + + dst[y * full_width + x] = j2k_fdwt53_predict_col( + src, + x, + full_width, + current_height, + y - low_height + ); +} + +__device__ float j2k_fdwt97_high1_row( + const float *src, + unsigned int row_base, + unsigned int width, + unsigned int high_index +) { + const unsigned int odd = high_index * 2u + 1u; + const unsigned int last_even = (width % 2u == 0u) ? width - 2u : width - 1u; + const float left = src[row_base + odd - 1u]; + const float right = (odd + 1u < width) ? src[row_base + odd + 1u] : src[row_base + last_even]; + return src[row_base + odd] + J2K_FDWT97_ALPHA * (left + right); +} + +__device__ float j2k_fdwt97_low1_row( + const float *src, + unsigned int row_base, + unsigned int width, + unsigned int low_index +) { + const unsigned int even = low_index * 2u; + const float left = low_index > 0u + ? j2k_fdwt97_high1_row(src, row_base, width, low_index - 1u) + : j2k_fdwt97_high1_row(src, row_base, width, 0u); + const float right = even + 1u < width + ? j2k_fdwt97_high1_row(src, row_base, width, low_index) + : left; + return src[row_base + even] + J2K_FDWT97_BETA * (left + right); +} + +__device__ float j2k_fdwt97_high2_row( + const float *src, + unsigned int row_base, + unsigned int width, + unsigned int high_index +) { + const unsigned int odd = high_index * 2u + 1u; + const unsigned int last_even = (width % 2u == 0u) ? width - 2u : width - 1u; + const unsigned int last_low = last_even / 2u; + const float left = j2k_fdwt97_low1_row(src, row_base, width, high_index); + const float right = (odd + 1u < width) + ? j2k_fdwt97_low1_row(src, row_base, width, high_index + 1u) + : j2k_fdwt97_low1_row(src, row_base, width, last_low); + return j2k_fdwt97_high1_row(src, row_base, width, high_index) + + J2K_FDWT97_GAMMA * (left + right); +} + +__device__ float j2k_fdwt97_low2_row( + const float *src, + unsigned int row_base, + unsigned int width, + unsigned int low_index +) { + const unsigned int even = low_index * 2u; + const float left = low_index > 0u + ? j2k_fdwt97_high2_row(src, row_base, width, low_index - 1u) + : j2k_fdwt97_high2_row(src, row_base, width, 0u); + const float right = even + 1u < width + ? j2k_fdwt97_high2_row(src, row_base, width, low_index) + : left; + return j2k_fdwt97_low1_row(src, row_base, width, low_index) + + J2K_FDWT97_DELTA * (left + right); +} + +extern "C" __global__ void j2k_forward_dwt97_horizontal( + const float *src, + float *dst, + unsigned int full_width, + unsigned int current_width, + unsigned int current_height, + unsigned int low_width +) { + const unsigned int x = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= current_width || y >= current_height) { + return; + } + + const unsigned int row_base = y * full_width; + if (x < low_width) { + dst[row_base + x] = j2k_fdwt97_low2_row(src, row_base, current_width, x) + * J2K_FDWT97_INV_KAPPA; + return; + } + + dst[row_base + x] = j2k_fdwt97_high2_row( + src, + row_base, + current_width, + x - low_width + ) * J2K_FDWT97_KAPPA; +} + +__device__ float j2k_fdwt97_high1_col( + const float *src, + unsigned int x, + unsigned int full_width, + unsigned int height, + unsigned int high_index +) { + const unsigned int odd = high_index * 2u + 1u; + const unsigned int last_even = (height % 2u == 0u) ? height - 2u : height - 1u; + const float top = src[(odd - 1u) * full_width + x]; + const float bottom = (odd + 1u < height) + ? src[(odd + 1u) * full_width + x] + : src[last_even * full_width + x]; + return src[odd * full_width + x] + J2K_FDWT97_ALPHA * (top + bottom); +} + +__device__ float j2k_fdwt97_low1_col( + const float *src, + unsigned int x, + unsigned int full_width, + unsigned int height, + unsigned int low_index +) { + const unsigned int even = low_index * 2u; + const float top = low_index > 0u + ? j2k_fdwt97_high1_col(src, x, full_width, height, low_index - 1u) + : j2k_fdwt97_high1_col(src, x, full_width, height, 0u); + const float bottom = even + 1u < height + ? j2k_fdwt97_high1_col(src, x, full_width, height, low_index) + : top; + return src[even * full_width + x] + J2K_FDWT97_BETA * (top + bottom); +} + +__device__ float j2k_fdwt97_high2_col( + const float *src, + unsigned int x, + unsigned int full_width, + unsigned int height, + unsigned int high_index +) { + const unsigned int odd = high_index * 2u + 1u; + const unsigned int last_even = (height % 2u == 0u) ? height - 2u : height - 1u; + const unsigned int last_low = last_even / 2u; + const float top = j2k_fdwt97_low1_col(src, x, full_width, height, high_index); + const float bottom = (odd + 1u < height) + ? j2k_fdwt97_low1_col(src, x, full_width, height, high_index + 1u) + : j2k_fdwt97_low1_col(src, x, full_width, height, last_low); + return j2k_fdwt97_high1_col(src, x, full_width, height, high_index) + + J2K_FDWT97_GAMMA * (top + bottom); +} + +__device__ float j2k_fdwt97_low2_col( + const float *src, + unsigned int x, + unsigned int full_width, + unsigned int height, + unsigned int low_index +) { + const unsigned int even = low_index * 2u; + const float top = low_index > 0u + ? j2k_fdwt97_high2_col(src, x, full_width, height, low_index - 1u) + : j2k_fdwt97_high2_col(src, x, full_width, height, 0u); + const float bottom = even + 1u < height + ? j2k_fdwt97_high2_col(src, x, full_width, height, low_index) + : top; + return j2k_fdwt97_low1_col(src, x, full_width, height, low_index) + + J2K_FDWT97_DELTA * (top + bottom); +} + +extern "C" __global__ void j2k_forward_dwt97_vertical( + const float *src, + float *dst, + unsigned int full_width, + unsigned int current_width, + unsigned int current_height, + unsigned int low_height +) { + const unsigned int x = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= current_width || y >= current_height) { + return; + } + + if (y < low_height) { + dst[y * full_width + x] = + j2k_fdwt97_low2_col(src, x, full_width, current_height, y) + * J2K_FDWT97_INV_KAPPA; + return; + } + + dst[y * full_width + x] = j2k_fdwt97_high2_col( + src, + x, + full_width, + current_height, + y - low_height + ) * J2K_FDWT97_KAPPA; +} + +__device__ int j2k_quantize_sample( + float sample, + unsigned int step_exponent, + unsigned int step_mantissa, + unsigned int range_bits, + unsigned int reversible +) { + if (reversible != 0u) { + const float rounded = sample >= 0.0f ? floorf(sample + 0.5f) : -floorf(-sample + 0.5f); + return int(rounded); + } + + const int exponent = int(range_bits) - int(step_exponent); + const float base = ldexpf(1.0f, exponent); + const float delta = base * (1.0f + float(step_mantissa) / 2048.0f); + if (delta <= 0.0f) { + return 0; + } + + const int sign = sample < 0.0f ? -1 : 1; + const int magnitude = int(floorf(fabsf(sample) / delta)); + return sign * magnitude; +} + +extern "C" __global__ void j2k_quantize_subband( + const float *samples, + int *coefficients, + unsigned long long len, + unsigned int step_exponent, + unsigned int step_mantissa, + unsigned int range_bits, + unsigned int reversible +) { + const unsigned long long idx = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (idx >= len) { + return; + } + + coefficients[idx] = j2k_quantize_sample( + samples[idx], + step_exponent, + step_mantissa, + range_bits, + reversible + ); +} + +extern "C" __global__ void j2k_quantize_subband_strided( + const float *samples, + int *coefficients, + unsigned int x0, + unsigned int y0, + unsigned int width, + unsigned int height, + unsigned int stride, + unsigned int step_exponent, + unsigned int step_mantissa, + unsigned int range_bits, + unsigned int reversible +) { + const unsigned int x = blockIdx.x * blockDim.x + threadIdx.x; + const unsigned int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= width || y >= height) { + return; + } + + const unsigned long long source_index = + static_cast(y0 + y) * stride + x0 + x; + const unsigned long long output_index = + static_cast(y) * width + x; + coefficients[output_index] = j2k_quantize_sample( + samples[source_index], + step_exponent, + step_mantissa, + range_bits, + reversible + ); +} diff --git a/crates/signinum-cuda-runtime/src/j2k_encode_kernels.ptx b/crates/j2k-cuda-runtime/src/j2k_encode_kernels.ptx similarity index 88% rename from crates/signinum-cuda-runtime/src/j2k_encode_kernels.ptx rename to crates/j2k-cuda-runtime/src/j2k_encode_kernels.ptx index e8796dbe..57a23447 100644 --- a/crates/signinum-cuda-runtime/src/j2k_encode_kernels.ptx +++ b/crates/j2k-cuda-runtime/src/j2k_encode_kernels.ptx @@ -2,7 +2,19 @@ .target sm_52 .address_size 64 -.visible .entry signinum_j2k_forward_rct( +.visible .entry j2k_deinterleave_to_f32( + .param .u64 pixels, + .param .u64 components, + .param .u64 num_pixels, + .param .u32 num_components, + .param .u32 bit_depth, + .param .u32 is_signed +) +{ + ret; +} + +.visible .entry j2k_forward_rct( .param .u64 plane0, .param .u64 plane1, .param .u64 plane2, @@ -54,7 +66,17 @@ RCT_DONE: ret; } -.visible .entry signinum_j2k_forward_dwt53_horizontal( +.visible .entry j2k_forward_ict( + .param .u64 plane0, + .param .u64 plane1, + .param .u64 plane2, + .param .u64 len +) +{ + ret; +} + +.visible .entry j2k_forward_dwt53_horizontal( .param .u64 src, .param .u64 dst, .param .u32 full_width, @@ -234,7 +256,7 @@ HDWT_DONE: ret; } -.visible .entry signinum_j2k_forward_dwt53_vertical( +.visible .entry j2k_forward_dwt53_vertical( .param .u64 src, .param .u64 dst, .param .u32 full_width, @@ -424,3 +446,57 @@ VDWT_BOTTOM_PREDICT_READY: VDWT_DONE: ret; } + +.visible .entry j2k_forward_dwt97_horizontal( + .param .u64 src, + .param .u64 dst, + .param .u32 full_width, + .param .u32 current_width, + .param .u32 current_height, + .param .u32 low_width +) +{ + ret; +} + +.visible .entry j2k_forward_dwt97_vertical( + .param .u64 src, + .param .u64 dst, + .param .u32 full_width, + .param .u32 current_width, + .param .u32 current_height, + .param .u32 low_height +) +{ + ret; +} + +.visible .entry j2k_quantize_subband( + .param .u64 samples, + .param .u64 coefficients, + .param .u64 len, + .param .u32 step_exponent, + .param .u32 step_mantissa, + .param .u32 range_bits, + .param .u32 reversible +) +{ + ret; +} + +.visible .entry j2k_quantize_subband_strided( + .param .u64 samples, + .param .u64 coefficients, + .param .u32 x0, + .param .u32 y0, + .param .u32 width, + .param .u32 height, + .param .u32 stride, + .param .u32 step_exponent, + .param .u32 step_mantissa, + .param .u32 range_bits, + .param .u32 reversible +) +{ + ret; +} diff --git a/crates/j2k-cuda-runtime/src/jpeg.rs b/crates/j2k-cuda-runtime/src/jpeg.rs new file mode 100644 index 00000000..c70fb5d7 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/jpeg.rs @@ -0,0 +1,982 @@ +#[cfg(j2k_cuda_jpeg_decode_ptx_built)] +use crate::bytes::{ + cuda_jpeg_decode_statuses_as_bytes, cuda_jpeg_decode_statuses_as_bytes_mut, + cuda_jpeg_entropy_checkpoints_as_bytes, cuda_jpeg_entropy_overflow_states_as_bytes, + cuda_jpeg_entropy_overflow_states_as_bytes_mut, cuda_jpeg_entropy_sync_states_as_bytes, + cuda_jpeg_entropy_sync_states_as_bytes_mut, cuda_jpeg_huffman_table_as_bytes, + u16_slice_as_bytes, +}; +use crate::{ + context::CudaContext, + error::CudaError, + execution::{CudaExecutionStats, CudaKernelOutput}, + memory::CudaDeviceBuffer, +}; +#[cfg(j2k_cuda_jpeg_decode_ptx_built)] +use crate::{ + execution::cuda_kernel_param, + kernels::{CudaKernel, CudaLaunchGeometry}, +}; + +/// Prepared baseline JPEG Huffman table for CUDA JPEG decode kernels. +#[repr(C)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaJpegHuffmanTable { + /// Largest Huffman code for each bit length; negative means no codes of that length. + pub max_code: [i32; 17], + /// Value-index offset for each bit length. + pub val_offset: [i32; 17], + /// Huffman values in canonical order. + pub values: [u8; 256], + /// Number of valid entries in `values`. + pub values_len: u32, +} + +impl CudaJpegHuffmanTable { + /// Prepare a CUDA Huffman table from JPEG BITS and HUFFVAL payloads. + pub fn from_jpeg_bits_values( + bits: [u8; 16], + values_len: u16, + values: [u8; 256], + ) -> Result { + let values_len_usize = usize::from(values_len); + let mut huffsize = [0u8; 256]; + let mut huffsize_len = 0usize; + for (len_minus_1, &count) in bits.iter().enumerate() { + let len = u8::try_from(len_minus_1 + 1).map_err(|_| CudaError::InvalidArgument { + message: "JPEG Huffman code length exceeds u8".to_string(), + })?; + for _ in 0..count { + if huffsize_len >= values_len_usize || huffsize_len >= huffsize.len() { + return Err(CudaError::InvalidArgument { + message: "JPEG Huffman BITS exceed values length".to_string(), + }); + } + huffsize[huffsize_len] = len; + huffsize_len += 1; + } + } + if huffsize_len != values_len_usize { + return Err(CudaError::InvalidArgument { + message: "JPEG Huffman BITS do not match values length".to_string(), + }); + } + + let mut huffcode = [0u16; 256]; + let mut code = 0u32; + let mut si = huffsize.first().copied().unwrap_or(0); + for (idx, &size) in huffsize[..huffsize_len].iter().enumerate() { + while size != si { + code <<= 1; + si = si.saturating_add(1); + } + if si > 16 || code >= (1u32 << si) { + return Err(CudaError::InvalidArgument { + message: "JPEG Huffman code overflow".to_string(), + }); + } + huffcode[idx] = u16::try_from(code).map_err(|_| CudaError::InvalidArgument { + message: "JPEG Huffman code exceeds u16".to_string(), + })?; + code = code + .checked_add(1) + .ok_or_else(|| CudaError::InvalidArgument { + message: "JPEG Huffman code overflow".to_string(), + })?; + } + + let mut max_code = [-1i32; 17]; + let mut val_offset = [0i32; 17]; + let mut cursor = 0usize; + for (len_minus_1, &count) in bits.iter().enumerate() { + let len = len_minus_1 + 1; + let count = usize::from(count); + if count == 0 { + continue; + } + let min_code = i32::from(huffcode[cursor]); + max_code[len] = i32::from(huffcode[cursor + count - 1]); + val_offset[len] = i32::try_from(cursor).map_err(|_| CudaError::InvalidArgument { + message: "JPEG Huffman values length exceeds i32".to_string(), + })? - min_code; + cursor += count; + } + + Ok(Self { + max_code, + val_offset, + values, + values_len: u32::from(values_len), + }) + } +} + +/// Entropy resume point for CUDA baseline JPEG decode. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaJpegEntropyCheckpoint { + /// MCU index for this checkpoint. + pub mcu_index: u32, + /// Byte offset into the entropy payload. + pub entropy_pos: u32, + /// Left-aligned buffered entropy bits. + pub bit_acc: u64, + /// Number of valid buffered bits. + pub bit_count: u32, + /// Previous Y DC predictor. + pub y_prev_dc: i32, + /// Previous Cb DC predictor. + pub cb_prev_dc: i32, + /// Previous Cr DC predictor. + pub cr_prev_dc: i32, + /// Reserved for ABI-compatible expansion. + pub reserved: u32, +} + +/// J2K-owned CUDA baseline JPEG RGB8 kernel shape. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum CudaJpegRgb8Sampling { + /// Fast 4:2:0 YCbCr shape: four Y blocks, then Cb and Cr per MCU. + Fast420, + /// Fast 4:2:2 YCbCr shape: two Y blocks, then Cb and Cr per MCU. + Fast422, + /// Fast 4:4:4 YCbCr shape: one Y block, then Cb and Cr per MCU. + Fast444, +} + +/// Experimental JPEG entropy chunking parameters for CUDA self-sync diagnostics. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaJpegChunkedEntropyConfig { + /// Subsequence size in 32-bit words. + pub subsequence_words: u32, + /// Reserved synchronization-sequence length for future grouped scans. + /// + /// The current diagnostic records adjacent-subsequence overflow results for + /// every neighboring pair; this value is validated and passed through the + /// ABI for compatibility with grouped synchronization experiments. + pub sequence_len: u32, + /// Maximum adjacent subsequences an overflow decoder may scan. + pub max_overflow_subsequences: u32, +} + +impl Default for CudaJpegChunkedEntropyConfig { + fn default() -> Self { + Self { + subsequence_words: 1024, + sequence_len: 128, + max_overflow_subsequences: 4, + } + } +} + +impl CudaJpegChunkedEntropyConfig { + /// Return one subsequence size in bits. + pub fn subsequence_bits(self) -> u32 { + self.subsequence_words.saturating_mul(32) + } + + /// Validate parameters before launching diagnostic kernels. + pub fn validate(self) -> Result<(), CudaError> { + if self.subsequence_words == 0 { + return Err(CudaError::InvalidArgument { + message: "JPEG entropy subsequence_words must be nonzero".to_string(), + }); + } + if self.subsequence_words.checked_mul(32).is_none() { + return Err(CudaError::InvalidArgument { + message: "JPEG entropy subsequence_words bit size exceeds u32".to_string(), + }); + } + if self.sequence_len == 0 { + return Err(CudaError::InvalidArgument { + message: "JPEG entropy sequence_len must be nonzero".to_string(), + }); + } + Ok(()) + } + + /// Count fixed-size bit subsequences needed for an entropy payload. + pub fn subsequence_count_for_entropy_bytes( + self, + entropy_len: usize, + ) -> Result { + self.validate()?; + let entropy_bits = entropy_len + .checked_mul(8) + .ok_or(CudaError::LengthTooLarge { len: entropy_len })?; + let bits = self.subsequence_bits() as usize; + Ok(entropy_bits.div_ceil(bits)) + } +} + +#[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] +pub(crate) fn jpeg_entropy_overflow_count(subsequence_count: usize) -> usize { + subsequence_count.saturating_sub(1) +} + +/// Device-written state for one entropy subsequence self-sync diagnostic. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaJpegEntropySyncState { + /// Zero means success; nonzero maps to diagnostic kernel status. + pub code: u32, + /// Subsequence start bit offset. + pub start_bit: u32, + /// Subsequence exclusive end bit offset. + pub end_bit: u32, + /// Decoder bit position after scanning this subsequence. + pub bit_pos: u32, + /// Decoded coefficient-slot count. + pub symbol_count: u32, + /// 4:2:0 block phase: 0..=3 for Y blocks, 4 Cb, 5 Cr. + pub block_phase: u32, + /// Zig-zag coefficient index inside the current block. + pub zigzag_index: u32, + /// Reserved for ABI-compatible expansion. + pub reserved: u32, +} + +/// Device-written overflow result for adjacent subsequence synchronization. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaJpegEntropyOverflowState { + /// Zero means success; nonzero maps to diagnostic kernel status. + pub code: u32, + /// Source subsequence index. + pub from_subsequence: u32, + /// Target subsequence index. + pub to_subsequence: u32, + /// Bits scanned after the target subsequence start before synchronization. + pub overflow_bits: u32, + /// One when synchronization was detected. + pub synchronized: u32, + /// Reserved for ABI-compatible expansion. + pub reserved: [u32; 3], +} + +/// Host-side report returned by experimental JPEG entropy self-sync diagnostics. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct CudaJpegChunkedEntropyReport { + /// Diagnostic chunk configuration. + pub config: CudaJpegChunkedEntropyConfig, + /// Entropy payload length in bytes. + pub entropy_bytes: usize, + /// Per-subsequence first-pass states. + pub states: Vec, + /// Per-adjacent-subsequence overflow states. + pub overflows: Vec, + /// Runtime dispatch stats for diagnostic kernels. + pub execution: CudaExecutionStats, +} + +impl CudaJpegChunkedEntropyReport { + /// Number of subsequences examined. + pub fn subsequence_count(&self) -> usize { + self.states.len() + } + + /// Number of overflow records that synchronized. + pub fn synchronized_overflow_count(&self) -> usize { + self.overflows + .iter() + .filter(|overflow| overflow.synchronized != 0) + .count() + } + + /// Maximum overflow scan length in bits. + pub fn max_overflow_bits(&self) -> Option { + self.overflows + .iter() + .map(|overflow| overflow.overflow_bits) + .max() + } + + /// Number of first-pass states with nonzero status. + pub fn failed_state_count(&self) -> usize { + self.states.iter().filter(|state| state.code != 0).count() + } +} + +/// Experimental J2K-owned CUDA JPEG entropy self-sync diagnostic plan. +#[derive(Debug)] +pub struct CudaJpegChunkedEntropyPlan<'a> { + /// Chunking configuration. + pub config: CudaJpegChunkedEntropyConfig, + /// Entropy-coded scan payload with byte stuffing/restart markers removed. + pub entropy_bytes: &'a [u8], + /// Y DC Huffman table. + pub y_dc_table: CudaJpegHuffmanTable, + /// Y AC Huffman table. + pub y_ac_table: CudaJpegHuffmanTable, + /// Cb DC Huffman table. + pub cb_dc_table: CudaJpegHuffmanTable, + /// Cb AC Huffman table. + pub cb_ac_table: CudaJpegHuffmanTable, + /// Cr DC Huffman table. + pub cr_dc_table: CudaJpegHuffmanTable, + /// Cr AC Huffman table. + pub cr_ac_table: CudaJpegHuffmanTable, +} + +/// J2K-owned CUDA baseline JPEG RGB8 decode plan. +#[derive(Debug)] +pub struct CudaJpegRgb8DecodePlan<'a> { + /// MCU sampling/kernel shape. + pub sampling: CudaJpegRgb8Sampling, + /// Image dimensions as `(width, height)`. + pub dimensions: (u32, u32), + /// Number of MCUs per row. + pub mcus_per_row: u32, + /// Number of MCU rows. + pub mcu_rows: u32, + /// Entropy-coded scan payload with byte stuffing/restart markers removed. + pub entropy_bytes: &'a [u8], + /// Entropy resume checkpoints. + pub entropy_checkpoints: &'a [CudaJpegEntropyCheckpoint], + /// Luma quantization table in JPEG zigzag order. + pub y_quant: [u16; 64], + /// Cb quantization table in JPEG zigzag order. + pub cb_quant: [u16; 64], + /// Cr quantization table in JPEG zigzag order. + pub cr_quant: [u16; 64], + /// Y DC Huffman table. + pub y_dc_table: CudaJpegHuffmanTable, + /// Y AC Huffman table. + pub y_ac_table: CudaJpegHuffmanTable, + /// Cb DC Huffman table. + pub cb_dc_table: CudaJpegHuffmanTable, + /// Cb AC Huffman table. + pub cb_ac_table: CudaJpegHuffmanTable, + /// Cr DC Huffman table. + pub cr_dc_table: CudaJpegHuffmanTable, + /// Cr AC Huffman table. + pub cr_ac_table: CudaJpegHuffmanTable, +} + +/// J2K-owned CUDA baseline JPEG 4:2:0 decode plan. +#[derive(Debug)] +pub struct CudaJpeg420Rgb8DecodePlan<'a> { + /// Image dimensions as `(width, height)`. + pub dimensions: (u32, u32), + /// Number of MCUs per row. + pub mcus_per_row: u32, + /// Number of MCU rows. + pub mcu_rows: u32, + /// Entropy-coded scan payload with byte stuffing/restart markers removed. + pub entropy_bytes: &'a [u8], + /// Entropy resume checkpoints. + pub entropy_checkpoints: &'a [CudaJpegEntropyCheckpoint], + /// Luma quantization table in JPEG zigzag order. + pub y_quant: [u16; 64], + /// Cb quantization table in JPEG zigzag order. + pub cb_quant: [u16; 64], + /// Cr quantization table in JPEG zigzag order. + pub cr_quant: [u16; 64], + /// Y DC Huffman table. + pub y_dc_table: CudaJpegHuffmanTable, + /// Y AC Huffman table. + pub y_ac_table: CudaJpegHuffmanTable, + /// Cb DC Huffman table. + pub cb_dc_table: CudaJpegHuffmanTable, + /// Cb AC Huffman table. + pub cb_ac_table: CudaJpegHuffmanTable, + /// Cr DC Huffman table. + pub cr_dc_table: CudaJpegHuffmanTable, + /// Cr AC Huffman table. + pub cr_ac_table: CudaJpegHuffmanTable, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] +pub(crate) struct CudaJpeg420Params { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) mcus_per_row: u32, + pub(crate) mcu_rows: u32, + pub(crate) entropy_len: u32, + pub(crate) checkpoint_count: u32, + pub(crate) out_stride: u32, + pub(crate) reserved: u32, +} + +// SAFETY: `CudaJpeg420Params` is `#[repr(C)]` and contains only CUDA scalar +// ABI fields passed by value through a kernel-parameter pointer. +unsafe impl crate::execution::CudaKernelParam for CudaJpeg420Params {} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] +pub(crate) struct CudaJpegEntropyChunkParams { + pub(crate) entropy_len: u32, + pub(crate) entropy_bits: u32, + pub(crate) subsequence_bits: u32, + pub(crate) subsequence_count: u32, + pub(crate) sequence_len: u32, + pub(crate) max_overflow_subsequences: u32, + pub(crate) reserved0: u32, + pub(crate) reserved1: u32, +} + +// SAFETY: `CudaJpegEntropyChunkParams` is `#[repr(C)]` and contains only CUDA +// scalar ABI fields passed by value through a kernel-parameter pointer. +unsafe impl crate::execution::CudaKernelParam for CudaJpegEntropyChunkParams {} + +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] +pub(crate) struct CudaJpegDecodeStatus { + pub(crate) code: u32, + pub(crate) detail: u32, + pub(crate) position: u32, + pub(crate) reserved: u32, +} + +#[cfg(j2k_cuda_jpeg_decode_ptx_built)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct CudaJpegRgb8ValidatedPlan { + pub(crate) params: CudaJpeg420Params, + pub(crate) output_len: usize, +} + +#[cfg(j2k_cuda_jpeg_decode_ptx_built)] +pub(crate) fn validate_jpeg_rgb8_plan( + plan: &CudaJpegRgb8DecodePlan<'_>, +) -> Result { + let (width, _) = plan.dimensions; + let out_stride = width.checked_mul(3).ok_or(CudaError::ImageTooLarge { + width, + height: plan.dimensions.1, + channels: 3, + })?; + validate_jpeg_rgb8_plan_with_pitch(plan, out_stride as usize) +} + +#[cfg(j2k_cuda_jpeg_decode_ptx_built)] +pub(crate) fn validate_jpeg_rgb8_plan_with_pitch( + plan: &CudaJpegRgb8DecodePlan<'_>, + pitch_bytes: usize, +) -> Result { + let (width, height) = plan.dimensions; + if width == 0 || height == 0 { + return Err(CudaError::InvalidArgument { + message: "JPEG CUDA decode dimensions must be nonzero".to_string(), + }); + } + if plan.entropy_checkpoints.is_empty() { + return Err(CudaError::InvalidArgument { + message: "JPEG CUDA decode requires at least one entropy checkpoint".to_string(), + }); + } + let entropy_len = + u32::try_from(plan.entropy_bytes.len()).map_err(|_| CudaError::LengthTooLarge { + len: plan.entropy_bytes.len(), + })?; + let checkpoint_count = + u32::try_from(plan.entropy_checkpoints.len()).map_err(|_| CudaError::LengthTooLarge { + len: plan.entropy_checkpoints.len(), + })?; + let row_bytes = width.checked_mul(3).ok_or(CudaError::ImageTooLarge { + width, + height, + channels: 3, + })?; + if pitch_bytes < row_bytes as usize { + return Err(CudaError::InvalidArgument { + message: format!( + "JPEG CUDA decode pitch {pitch_bytes} is smaller than row byte count {row_bytes}" + ), + }); + } + let out_stride = + u32::try_from(pitch_bytes).map_err(|_| CudaError::LengthTooLarge { len: pitch_bytes })?; + let output_len = pitch_bytes + .checked_mul(height as usize - 1) + .and_then(|prefix| prefix.checked_add(row_bytes as usize)) + .ok_or(CudaError::ImageTooLarge { + width, + height, + channels: 3, + })?; + + Ok(CudaJpegRgb8ValidatedPlan { + params: CudaJpeg420Params { + width, + height, + mcus_per_row: plan.mcus_per_row, + mcu_rows: plan.mcu_rows, + entropy_len, + checkpoint_count, + out_stride, + reserved: 0, + }, + output_len, + }) +} + +#[cfg(j2k_cuda_jpeg_decode_ptx_built)] +pub(crate) fn validate_jpeg_entropy_chunk_plan( + plan: &CudaJpegChunkedEntropyPlan<'_>, + subsequences: usize, +) -> Result { + let entropy_len = + u32::try_from(plan.entropy_bytes.len()).map_err(|_| CudaError::LengthTooLarge { + len: plan.entropy_bytes.len(), + })?; + let entropy_bits = entropy_len + .checked_mul(8) + .ok_or(CudaError::LengthTooLarge { + len: plan.entropy_bytes.len(), + })?; + let subsequence_count = + u32::try_from(subsequences).map_err(|_| CudaError::LengthTooLarge { len: subsequences })?; + + Ok(CudaJpegEntropyChunkParams { + entropy_len, + entropy_bits, + subsequence_bits: plan.config.subsequence_bits(), + subsequence_count, + sequence_len: plan.config.sequence_len, + max_overflow_subsequences: plan.config.max_overflow_subsequences, + reserved0: 0, + reserved1: 0, + }) +} + +pub(crate) fn cuda_jpeg_rgb8_plan_from_420<'a>( + plan: &CudaJpeg420Rgb8DecodePlan<'a>, +) -> CudaJpegRgb8DecodePlan<'a> { + CudaJpegRgb8DecodePlan { + sampling: CudaJpegRgb8Sampling::Fast420, + dimensions: plan.dimensions, + mcus_per_row: plan.mcus_per_row, + mcu_rows: plan.mcu_rows, + entropy_bytes: plan.entropy_bytes, + entropy_checkpoints: plan.entropy_checkpoints, + y_quant: plan.y_quant, + cb_quant: plan.cb_quant, + cr_quant: plan.cr_quant, + y_dc_table: plan.y_dc_table, + y_ac_table: plan.y_ac_table, + cb_dc_table: plan.cb_dc_table, + cb_ac_table: plan.cb_ac_table, + cr_dc_table: plan.cr_dc_table, + cr_ac_table: plan.cr_ac_table, + } +} + +#[cfg(j2k_cuda_jpeg_decode_ptx_built)] +pub(crate) fn jpeg_rgb8_kernel(sampling: CudaJpegRgb8Sampling) -> (CudaKernel, &'static str) { + match sampling { + CudaJpegRgb8Sampling::Fast420 => ( + CudaKernel::JpegDecodeFast420Rgb8, + "j2k_jpeg_decode_fast420_rgb8", + ), + CudaJpegRgb8Sampling::Fast422 => ( + CudaKernel::JpegDecodeFast422Rgb8, + "j2k_jpeg_decode_fast422_rgb8", + ), + CudaJpegRgb8Sampling::Fast444 => ( + CudaKernel::JpegDecodeFast444Rgb8, + "j2k_jpeg_decode_fast444_rgb8", + ), + } +} + +impl CudaContext { + /// Run experimental 4:2:0 JPEG entropy self-sync diagnostics. + pub fn diagnose_jpeg_420_entropy_self_sync( + &self, + plan: &CudaJpegChunkedEntropyPlan<'_>, + ) -> Result { + plan.config.validate()?; + let subsequences = plan + .config + .subsequence_count_for_entropy_bytes(plan.entropy_bytes.len())?; + if subsequences == 0 { + return Ok(CudaJpegChunkedEntropyReport { + config: plan.config, + entropy_bytes: plan.entropy_bytes.len(), + states: Vec::new(), + overflows: Vec::new(), + execution: CudaExecutionStats { + kernel_dispatches: 0, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }); + } + + #[cfg(not(j2k_cuda_jpeg_decode_ptx_built))] + { + let _ = subsequences; + Err(CudaError::InvalidArgument { + message: "J2K CUDA JPEG decode PTX was not built from jpeg_decode_kernels.cu" + .to_string(), + }) + } + + #[cfg(j2k_cuda_jpeg_decode_ptx_built)] + { + self.diagnose_jpeg_420_entropy_self_sync_nonempty(plan, subsequences) + } + } + + /// Decode one baseline JPEG 4:2:0 image to device-resident RGB8 using J2K CUDA kernels. + pub fn decode_jpeg_420_rgb8_owned( + &self, + plan: &CudaJpeg420Rgb8DecodePlan<'_>, + ) -> Result { + let plan = cuda_jpeg_rgb8_plan_from_420(plan); + self.decode_jpeg_rgb8_owned(&plan) + } + + /// Decode one baseline JPEG RGB8 image to device-resident RGB8 using J2K CUDA kernels. + pub fn decode_jpeg_rgb8_owned( + &self, + plan: &CudaJpegRgb8DecodePlan<'_>, + ) -> Result { + #[cfg(not(j2k_cuda_jpeg_decode_ptx_built))] + { + let _ = plan; + Err(CudaError::InvalidArgument { + message: "J2K CUDA JPEG decode PTX was not built from jpeg_decode_kernels.cu" + .to_string(), + }) + } + + #[cfg(j2k_cuda_jpeg_decode_ptx_built)] + { + let validated = validate_jpeg_rgb8_plan(plan)?; + self.inner.set_current()?; + let output = self.allocate(validated.output_len)?; + let execution = self.decode_jpeg_rgb8_owned_validated(plan, &output, validated)?; + Ok(CudaKernelOutput { + buffer: output, + execution, + }) + } + } + + /// Decode one baseline JPEG 4:2:0 image into caller-owned CUDA RGB8 memory. + pub fn decode_jpeg_420_rgb8_owned_into( + &self, + plan: &CudaJpeg420Rgb8DecodePlan<'_>, + output: &CudaDeviceBuffer, + pitch_bytes: usize, + ) -> Result { + let plan = cuda_jpeg_rgb8_plan_from_420(plan); + self.decode_jpeg_rgb8_owned_into(&plan, output, pitch_bytes) + } + + /// Decode one baseline JPEG RGB8 image into caller-owned CUDA RGB8 memory. + pub fn decode_jpeg_rgb8_owned_into( + &self, + plan: &CudaJpegRgb8DecodePlan<'_>, + output: &CudaDeviceBuffer, + pitch_bytes: usize, + ) -> Result { + #[cfg(not(j2k_cuda_jpeg_decode_ptx_built))] + { + let _ = (plan, output, pitch_bytes); + Err(CudaError::InvalidArgument { + message: "J2K CUDA JPEG decode PTX was not built from jpeg_decode_kernels.cu" + .to_string(), + }) + } + + #[cfg(j2k_cuda_jpeg_decode_ptx_built)] + { + let validated = validate_jpeg_rgb8_plan_with_pitch(plan, pitch_bytes)?; + if output.byte_len() < validated.output_len { + return Err(CudaError::OutputTooSmall { + required: validated.output_len, + have: output.byte_len(), + }); + } + self.inner.set_current()?; + self.decode_jpeg_rgb8_owned_validated(plan, output, validated) + } + } + + #[cfg(j2k_cuda_jpeg_decode_ptx_built)] + #[allow(clippy::similar_names)] + fn decode_jpeg_rgb8_owned_validated( + &self, + plan: &CudaJpegRgb8DecodePlan<'_>, + output: &CudaDeviceBuffer, + validated: CudaJpegRgb8ValidatedPlan, + ) -> Result { + let (kernel, kernel_name) = jpeg_rgb8_kernel(plan.sampling); + let entropy = self.upload(plan.entropy_bytes)?; + let y_quant = self.upload(u16_slice_as_bytes(&plan.y_quant))?; + let cb_quant = self.upload(u16_slice_as_bytes(&plan.cb_quant))?; + let cr_quant = self.upload(u16_slice_as_bytes(&plan.cr_quant))?; + let y_dc = self.upload(cuda_jpeg_huffman_table_as_bytes(&plan.y_dc_table))?; + let y_ac = self.upload(cuda_jpeg_huffman_table_as_bytes(&plan.y_ac_table))?; + let cb_dc = self.upload(cuda_jpeg_huffman_table_as_bytes(&plan.cb_dc_table))?; + let cb_ac = self.upload(cuda_jpeg_huffman_table_as_bytes(&plan.cb_ac_table))?; + let cr_dc = self.upload(cuda_jpeg_huffman_table_as_bytes(&plan.cr_dc_table))?; + let cr_ac = self.upload(cuda_jpeg_huffman_table_as_bytes(&plan.cr_ac_table))?; + let checkpoints = self.upload(cuda_jpeg_entropy_checkpoints_as_bytes( + plan.entropy_checkpoints, + ))?; + let mut statuses = vec![CudaJpegDecodeStatus::default(); plan.entropy_checkpoints.len()]; + let status_buffer = self.upload(cuda_jpeg_decode_statuses_as_bytes(&statuses))?; + self.launch_jpeg_decode_rgb8( + kernel, + &entropy, + output, + validated.params, + &y_quant, + &cb_quant, + &cr_quant, + &y_dc, + &y_ac, + &cb_dc, + &cb_ac, + &cr_dc, + &cr_ac, + &checkpoints, + &status_buffer, + )?; + status_buffer.copy_to_host(cuda_jpeg_decode_statuses_as_bytes_mut(&mut statuses))?; + for status in statuses { + if status.code != 0 { + return Err(CudaError::KernelStatus { + kernel: kernel_name, + code: status.code, + detail: status.detail, + }); + } + } + Ok(CudaExecutionStats { + kernel_dispatches: 1, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 1, + hardware_decode: false, + }) + } + + #[cfg(j2k_cuda_jpeg_decode_ptx_built)] + #[allow(clippy::similar_names)] + fn diagnose_jpeg_420_entropy_self_sync_nonempty( + &self, + plan: &CudaJpegChunkedEntropyPlan<'_>, + subsequences: usize, + ) -> Result { + let params = validate_jpeg_entropy_chunk_plan(plan, subsequences)?; + self.inner.set_current()?; + let entropy = self.upload_pinned(plan.entropy_bytes)?; + let y_dc = self.upload(cuda_jpeg_huffman_table_as_bytes(&plan.y_dc_table))?; + let y_ac = self.upload(cuda_jpeg_huffman_table_as_bytes(&plan.y_ac_table))?; + let cb_dc = self.upload(cuda_jpeg_huffman_table_as_bytes(&plan.cb_dc_table))?; + let cb_ac = self.upload(cuda_jpeg_huffman_table_as_bytes(&plan.cb_ac_table))?; + let cr_dc = self.upload(cuda_jpeg_huffman_table_as_bytes(&plan.cr_dc_table))?; + let cr_ac = self.upload(cuda_jpeg_huffman_table_as_bytes(&plan.cr_ac_table))?; + + let mut states = vec![CudaJpegEntropySyncState::default(); subsequences]; + let states_buffer = self.upload(cuda_jpeg_entropy_sync_states_as_bytes(&states))?; + self.launch_jpeg_entropy_sync420( + &entropy, + params, + &y_dc, + &y_ac, + &cb_dc, + &cb_ac, + &cr_dc, + &cr_ac, + &states_buffer, + )?; + states_buffer.copy_to_host(cuda_jpeg_entropy_sync_states_as_bytes_mut(&mut states))?; + + let mut overflows = vec![ + CudaJpegEntropyOverflowState::default(); + jpeg_entropy_overflow_count(subsequences) + ]; + if !overflows.is_empty() { + let overflow_buffer = + self.upload(cuda_jpeg_entropy_overflow_states_as_bytes(&overflows))?; + self.launch_jpeg_entropy_overflow420( + &entropy, + params, + &y_dc, + &y_ac, + &cb_dc, + &cb_ac, + &cr_dc, + &cr_ac, + &states_buffer, + &overflow_buffer, + )?; + overflow_buffer.copy_to_host(cuda_jpeg_entropy_overflow_states_as_bytes_mut( + &mut overflows, + ))?; + } + + Ok(CudaJpegChunkedEntropyReport { + config: plan.config, + entropy_bytes: plan.entropy_bytes.len(), + states, + overflows, + execution: CudaExecutionStats { + kernel_dispatches: 1 + usize::from(subsequences > 1), + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }) + } + + #[cfg(j2k_cuda_jpeg_decode_ptx_built)] + #[allow(clippy::similar_names, clippy::too_many_arguments)] + fn launch_jpeg_decode_rgb8( + &self, + kernel: CudaKernel, + entropy: &CudaDeviceBuffer, + output: &CudaDeviceBuffer, + mut params: CudaJpeg420Params, + y_quant: &CudaDeviceBuffer, + cb_quant: &CudaDeviceBuffer, + cr_quant: &CudaDeviceBuffer, + y_dc: &CudaDeviceBuffer, + y_ac: &CudaDeviceBuffer, + cb_dc: &CudaDeviceBuffer, + cb_ac: &CudaDeviceBuffer, + cr_dc: &CudaDeviceBuffer, + cr_ac: &CudaDeviceBuffer, + checkpoints: &CudaDeviceBuffer, + status: &CudaDeviceBuffer, + ) -> Result<(), CudaError> { + let function = self.inner.kernel_function(kernel)?; + let mut entropy_ptr = entropy.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut y_quant_ptr = y_quant.device_ptr(); + let mut cb_quant_ptr = cb_quant.device_ptr(); + let mut cr_quant_ptr = cr_quant.device_ptr(); + let mut y_dc_ptr = y_dc.device_ptr(); + let mut y_ac_ptr = y_ac.device_ptr(); + let mut cb_dc_ptr = cb_dc.device_ptr(); + let mut cb_ac_ptr = cb_ac.device_ptr(); + let mut cr_dc_ptr = cr_dc.device_ptr(); + let mut cr_ac_ptr = cr_ac.device_ptr(); + let mut checkpoints_ptr = checkpoints.device_ptr(); + let mut status_ptr = status.device_ptr(); + let mut kernel_params = cuda_kernel_params!( + entropy_ptr, + output_ptr, + params, + y_quant_ptr, + cb_quant_ptr, + cr_quant_ptr, + y_dc_ptr, + y_ac_ptr, + cb_dc_ptr, + cb_ac_ptr, + cr_dc_ptr, + cr_ac_ptr, + checkpoints_ptr, + status_ptr + ); + let geometry = CudaLaunchGeometry { + grid: (params.checkpoint_count, 1, 1), + block: (1, 1, 1), + }; + + self.launch_kernel(function, geometry, &mut kernel_params) + } + + #[cfg(j2k_cuda_jpeg_decode_ptx_built)] + #[allow(clippy::similar_names, clippy::too_many_arguments)] + fn launch_jpeg_entropy_sync420( + &self, + entropy: &CudaDeviceBuffer, + mut params: CudaJpegEntropyChunkParams, + y_dc: &CudaDeviceBuffer, + y_ac: &CudaDeviceBuffer, + cb_dc: &CudaDeviceBuffer, + cb_ac: &CudaDeviceBuffer, + cr_dc: &CudaDeviceBuffer, + cr_ac: &CudaDeviceBuffer, + states: &CudaDeviceBuffer, + ) -> Result<(), CudaError> { + let function = self.inner.kernel_function(CudaKernel::JpegEntropySync420)?; + let mut entropy_ptr = entropy.device_ptr(); + let mut y_dc_ptr = y_dc.device_ptr(); + let mut y_ac_ptr = y_ac.device_ptr(); + let mut cb_dc_ptr = cb_dc.device_ptr(); + let mut cb_ac_ptr = cb_ac.device_ptr(); + let mut cr_dc_ptr = cr_dc.device_ptr(); + let mut cr_ac_ptr = cr_ac.device_ptr(); + let mut states_ptr = states.device_ptr(); + let mut kernel_params = cuda_kernel_params!( + entropy_ptr, + params, + y_dc_ptr, + y_ac_ptr, + cb_dc_ptr, + cb_ac_ptr, + cr_dc_ptr, + cr_ac_ptr, + states_ptr + ); + let geometry = CudaLaunchGeometry { + grid: (params.subsequence_count.div_ceil(128), 1, 1), + block: (128, 1, 1), + }; + + self.launch_kernel(function, geometry, &mut kernel_params) + } + + #[cfg(j2k_cuda_jpeg_decode_ptx_built)] + #[allow(clippy::similar_names, clippy::too_many_arguments)] + fn launch_jpeg_entropy_overflow420( + &self, + entropy: &CudaDeviceBuffer, + mut params: CudaJpegEntropyChunkParams, + y_dc: &CudaDeviceBuffer, + y_ac: &CudaDeviceBuffer, + cb_dc: &CudaDeviceBuffer, + cb_ac: &CudaDeviceBuffer, + cr_dc: &CudaDeviceBuffer, + cr_ac: &CudaDeviceBuffer, + states: &CudaDeviceBuffer, + overflows: &CudaDeviceBuffer, + ) -> Result<(), CudaError> { + let function = self + .inner + .kernel_function(CudaKernel::JpegEntropyOverflow420)?; + let mut entropy_ptr = entropy.device_ptr(); + let mut y_dc_ptr = y_dc.device_ptr(); + let mut y_ac_ptr = y_ac.device_ptr(); + let mut cb_dc_ptr = cb_dc.device_ptr(); + let mut cb_ac_ptr = cb_ac.device_ptr(); + let mut cr_dc_ptr = cr_dc.device_ptr(); + let mut cr_ac_ptr = cr_ac.device_ptr(); + let mut states_ptr = states.device_ptr(); + let mut overflows_ptr = overflows.device_ptr(); + let mut kernel_params = cuda_kernel_params!( + entropy_ptr, + params, + y_dc_ptr, + y_ac_ptr, + cb_dc_ptr, + cb_ac_ptr, + cr_dc_ptr, + cr_ac_ptr, + states_ptr, + overflows_ptr + ); + let geometry = CudaLaunchGeometry { + grid: ( + (params.subsequence_count.saturating_sub(1)).div_ceil(128), + 1, + 1, + ), + block: (128, 1, 1), + }; + + self.launch_kernel(function, geometry, &mut kernel_params) + } +} diff --git a/crates/j2k-cuda-runtime/src/jpeg_decode_kernels.cu b/crates/j2k-cuda-runtime/src/jpeg_decode_kernels.cu new file mode 100644 index 00000000..b054eb1e --- /dev/null +++ b/crates/j2k-cuda-runtime/src/jpeg_decode_kernels.cu @@ -0,0 +1,1225 @@ +#include + +struct J2KJpeg420Params { + unsigned int width; + unsigned int height; + unsigned int mcus_per_row; + unsigned int mcu_rows; + unsigned int entropy_len; + unsigned int checkpoint_count; + unsigned int out_stride; + unsigned int reserved; +}; + +struct J2KJpegEntropyCheckpoint { + unsigned int mcu_index; + unsigned int entropy_pos; + unsigned long long bit_acc; + unsigned int bit_count; + int y_prev_dc; + int cb_prev_dc; + int cr_prev_dc; + unsigned int reserved; +}; + +struct J2KJpegHuffmanTable { + int max_code[17]; + int val_offset[17]; + unsigned char values[256]; + unsigned int values_len; +}; + +struct J2KJpegDecodeStatus { + unsigned int code; + unsigned int detail; + unsigned int position; + unsigned int reserved; +}; + +struct J2KJpegEntropyChunkParams { + unsigned int entropy_len; + unsigned int entropy_bits; + unsigned int subsequence_bits; + unsigned int subsequence_count; + unsigned int sequence_len; + unsigned int max_overflow_subsequences; + unsigned int reserved0; + unsigned int reserved1; +}; + +struct J2KJpegEntropySyncState { + unsigned int code; + unsigned int start_bit; + unsigned int end_bit; + unsigned int bit_pos; + unsigned int symbol_count; + unsigned int block_phase; + unsigned int zigzag_index; + unsigned int reserved; +}; + +struct J2KJpegEntropyOverflowState { + unsigned int code; + unsigned int from_subsequence; + unsigned int to_subsequence; + unsigned int overflow_bits; + unsigned int synchronized; + unsigned int reserved[3]; +}; + +struct J2KJpegBitReader { + unsigned int pos; + unsigned long long acc; + unsigned int bits; +}; + +static constexpr unsigned int JPEG_STATUS_OK = 0u; +static constexpr unsigned int JPEG_STATUS_TRUNCATED = 1u; +static constexpr unsigned int JPEG_STATUS_HUFFMAN = 2u; + +__device__ __constant__ unsigned char J2K_JPEG_ZIGZAG[64] = { + 0, 1, 8, 16, 9, 2, 3, 10, + 17, 24, 32, 25, 18, 11, 4, 5, + 12, 19, 26, 33, 40, 48, 41, 34, + 27, 20, 13, 6, 7, 14, 21, 28, + 35, 42, 49, 56, 57, 50, 43, 36, + 29, 22, 15, 23, 30, 37, 44, 51, + 58, 59, 52, 45, 38, 31, 39, 46, + 53, 60, 61, 54, 47, 55, 62, 63 +}; + +__device__ unsigned int j2k_jpeg_zigzag(unsigned int k) { + return J2K_JPEG_ZIGZAG[k]; +} + +__device__ void j2k_jpeg_set_error( + J2KJpegDecodeStatus *status, + unsigned int code, + unsigned int detail, + unsigned int position +) { + status->code = code; + status->detail = detail; + status->position = position; +} + +__device__ bool j2k_jpeg_refill_one( + J2KJpegBitReader &reader, + const unsigned char *entropy, + unsigned int entropy_len +) { + if (reader.pos >= entropy_len) { + return false; + } + const unsigned int shift = 64u - 8u - reader.bits; + reader.acc |= static_cast(entropy[reader.pos]) << shift; + reader.pos += 1u; + reader.bits += 8u; + return true; +} + +__device__ bool j2k_jpeg_ensure_bits( + J2KJpegBitReader &reader, + const unsigned char *entropy, + unsigned int entropy_len, + unsigned int wanted +) { + while (reader.bits < wanted) { + if (!j2k_jpeg_refill_one(reader, entropy, entropy_len)) { + return false; + } + } + return true; +} + +__device__ void j2k_jpeg_ensure_bits_padded( + J2KJpegBitReader &reader, + const unsigned char *entropy, + unsigned int entropy_len, + unsigned int wanted +) { + while (reader.bits < wanted) { + if (!j2k_jpeg_refill_one(reader, entropy, entropy_len)) { + reader.acc |= 1ull << (63u - reader.bits); + reader.bits += 1u; + } + } +} + +__device__ unsigned int j2k_jpeg_peek_bits( + const J2KJpegBitReader &reader, + unsigned int count +) { + return count == 0u ? 0u : static_cast(reader.acc >> (64u - count)); +} + +__device__ void j2k_jpeg_consume_bits(J2KJpegBitReader &reader, unsigned int count) { + reader.acc <<= count; + reader.bits -= count; +} + +__device__ J2KJpegBitReader j2k_jpeg_bit_reader_at_bit( + const unsigned char *entropy, + unsigned int entropy_len, + unsigned int bit_pos +) { + J2KJpegBitReader reader; + reader.pos = bit_pos / 8u; + reader.acc = 0ull; + reader.bits = 0u; + const unsigned int skip = bit_pos & 7u; + if (skip != 0u && reader.pos < entropy_len) { + reader.acc = static_cast(entropy[reader.pos]) << 56u; + reader.pos += 1u; + reader.bits = 8u; + j2k_jpeg_consume_bits(reader, skip); + } + return reader; +} + +__device__ bool j2k_jpeg_real_bits_consumed( + const J2KJpegBitReader &reader, + unsigned int before_pos, + unsigned int before_bits, + unsigned int &consumed +) { + const unsigned int loaded_bits = (reader.pos - before_pos) * 8u + before_bits; + if (reader.bits >= loaded_bits) { + consumed = 0u; + return false; + } + consumed = loaded_bits - reader.bits; + return true; +} + +__device__ bool j2k_jpeg_receive_extend( + J2KJpegBitReader &reader, + const unsigned char *entropy, + unsigned int entropy_len, + unsigned int ssss, + J2KJpegDecodeStatus *status, + int &out +) { + if (ssss == 0u) { + out = 0; + return true; + } + if (!j2k_jpeg_ensure_bits(reader, entropy, entropy_len, ssss)) { + j2k_jpeg_set_error(status, JPEG_STATUS_TRUNCATED, ssss, reader.pos); + return false; + } + const int value = static_cast(j2k_jpeg_peek_bits(reader, ssss)); + j2k_jpeg_consume_bits(reader, ssss); + const int threshold = 1 << (ssss - 1u); + out = value < threshold ? value + ((-1) << ssss) + 1 : value; + return true; +} + +__device__ bool j2k_jpeg_decode_symbol( + J2KJpegBitReader &reader, + const unsigned char *entropy, + unsigned int entropy_len, + const J2KJpegHuffmanTable *table, + J2KJpegDecodeStatus *status, + unsigned char &symbol +) { + j2k_jpeg_ensure_bits_padded(reader, entropy, entropy_len, 16u); + const int code16 = static_cast(j2k_jpeg_peek_bits(reader, 16u)); + for (unsigned int len = 1u; len <= 16u; ++len) { + if (table->max_code[len] < 0) { + continue; + } + const int code = code16 >> (16u - len); + if (code <= table->max_code[len]) { + const int idx = code + table->val_offset[len]; + if (idx < 0 || static_cast(idx) >= table->values_len) { + j2k_jpeg_set_error(status, JPEG_STATUS_HUFFMAN, len, reader.pos); + return false; + } + j2k_jpeg_consume_bits(reader, len); + symbol = table->values[idx]; + return true; + } + } + j2k_jpeg_set_error(status, JPEG_STATUS_HUFFMAN, 16u, reader.pos); + return false; +} + +__device__ bool j2k_jpeg_decode_symbol_real( + J2KJpegBitReader &reader, + const unsigned char *entropy, + unsigned int entropy_len, + const J2KJpegHuffmanTable *table, + J2KJpegDecodeStatus *status, + unsigned char &symbol +) { + for (unsigned int len = 1u; len <= 16u; ++len) { + if (!j2k_jpeg_ensure_bits(reader, entropy, entropy_len, len)) { + j2k_jpeg_set_error(status, JPEG_STATUS_TRUNCATED, len, reader.pos); + return false; + } + if (table->max_code[len] < 0) { + continue; + } + const int code = static_cast(j2k_jpeg_peek_bits(reader, len)); + if (code <= table->max_code[len]) { + const int idx = code + table->val_offset[len]; + if (idx < 0 || static_cast(idx) >= table->values_len) { + j2k_jpeg_set_error(status, JPEG_STATUS_HUFFMAN, len, reader.pos); + return false; + } + j2k_jpeg_consume_bits(reader, len); + symbol = table->values[idx]; + return true; + } + } + j2k_jpeg_set_error(status, JPEG_STATUS_HUFFMAN, 16u, reader.pos); + return false; +} + +__device__ bool j2k_jpeg_decode_block( + J2KJpegBitReader &reader, + const unsigned char *entropy, + unsigned int entropy_len, + const J2KJpegHuffmanTable *dc_table, + const J2KJpegHuffmanTable *ac_table, + const unsigned short *quant, + int &prev_dc, + J2KJpegDecodeStatus *status, + int coeffs[64] +) { + for (unsigned int i = 0u; i < 64u; ++i) { + coeffs[i] = 0; + } + + unsigned char ssss = 0u; + if (!j2k_jpeg_decode_symbol(reader, entropy, entropy_len, dc_table, status, ssss)) { + return false; + } + if (ssss > 15u) { + j2k_jpeg_set_error(status, JPEG_STATUS_HUFFMAN, ssss, reader.pos); + return false; + } + int diff = 0; + if (!j2k_jpeg_receive_extend(reader, entropy, entropy_len, ssss, status, diff)) { + return false; + } + prev_dc += diff; + coeffs[0] = prev_dc * static_cast(quant[0]); + + unsigned int k = 1u; + while (k < 64u) { + unsigned char packed = 0u; + if (!j2k_jpeg_decode_symbol(reader, entropy, entropy_len, ac_table, status, packed)) { + return false; + } + const unsigned int run = static_cast(packed >> 4u); + ssss = packed & 0x0Fu; + if (ssss == 0u) { + if (run == 15u) { + k += 16u; + continue; + } + break; + } + k += run; + if (k >= 64u) { + j2k_jpeg_set_error(status, JPEG_STATUS_HUFFMAN, k, reader.pos); + return false; + } + int value = 0; + if (!j2k_jpeg_receive_extend(reader, entropy, entropy_len, ssss, status, value)) { + return false; + } + coeffs[j2k_jpeg_zigzag(k)] = value * static_cast(quant[k]); + k += 1u; + } + return true; +} + +__device__ bool j2k_jpeg_entropy_scan_one_symbol420( + const unsigned char *entropy, + J2KJpegEntropyChunkParams params, + const J2KJpegHuffmanTable *y_dc, + const J2KJpegHuffmanTable *y_ac, + const J2KJpegHuffmanTable *cb_dc, + const J2KJpegHuffmanTable *cb_ac, + const J2KJpegHuffmanTable *cr_dc, + const J2KJpegHuffmanTable *cr_ac, + J2KJpegEntropySyncState &state, + J2KJpegBitReader &reader, + J2KJpegDecodeStatus &status +) { + const bool dc = state.zigzag_index == 0u; + const J2KJpegHuffmanTable *table = + state.block_phase < 4u + ? (dc ? y_dc : y_ac) + : (state.block_phase == 4u ? (dc ? cb_dc : cb_ac) : (dc ? cr_dc : cr_ac)); + unsigned char symbol = 0u; + const unsigned int before_pos = reader.pos; + const unsigned int before_bits = reader.bits; + if (!j2k_jpeg_decode_symbol_real(reader, entropy, params.entropy_len, table, &status, symbol)) { + // Diagnostic self-sync starts at arbitrary bit offsets, so invalid + // prefixes are expected until a candidate stream resynchronizes. + if (status.code == JPEG_STATUS_HUFFMAN) { + if (!j2k_jpeg_ensure_bits(reader, entropy, params.entropy_len, 1u)) { + state.bit_pos = params.entropy_bits; + status.code = JPEG_STATUS_OK; + return true; + } + j2k_jpeg_consume_bits(reader, 1u); + state.bit_pos += 1u; + status.code = JPEG_STATUS_OK; + status.detail = 0u; + status.position = 0u; + return true; + } + if (status.code == JPEG_STATUS_TRUNCATED) { + state.bit_pos = params.entropy_bits; + status.code = JPEG_STATUS_OK; + return true; + } + return false; + } + const unsigned int run = symbol >> 4u; + const unsigned int ssss = symbol & 0x0Fu; + unsigned int coeff_bits = dc ? symbol : (symbol & 0x0Fu); + if (coeff_bits > 15u) { + j2k_jpeg_set_error(&status, JPEG_STATUS_HUFFMAN, coeff_bits, reader.pos); + return false; + } + if (!j2k_jpeg_ensure_bits(reader, entropy, params.entropy_len, coeff_bits)) { + state.bit_pos = params.entropy_bits; + return true; + } + j2k_jpeg_consume_bits(reader, coeff_bits); + unsigned int consumed = 0u; + if (!j2k_jpeg_real_bits_consumed(reader, before_pos, before_bits, consumed)) { + j2k_jpeg_set_error(&status, JPEG_STATUS_TRUNCATED, 0u, reader.pos); + return false; + } + state.bit_pos += consumed; + if (dc) { + state.zigzag_index = 1u; + state.symbol_count += 1u; + return true; + } + if (ssss == 0u && run != 15u) { + state.symbol_count += 64u - state.zigzag_index; + state.zigzag_index = 0u; + state.block_phase = (state.block_phase + 1u) % 6u; + return true; + } + state.zigzag_index += run + 1u; + state.symbol_count += run + 1u; + if (state.zigzag_index >= 64u) { + state.zigzag_index = 0u; + state.block_phase = (state.block_phase + 1u) % 6u; + } + return true; +} + +extern "C" __global__ void j2k_jpeg_entropy_sync420( + const unsigned char *entropy, + J2KJpegEntropyChunkParams params, + const J2KJpegHuffmanTable *y_dc, + const J2KJpegHuffmanTable *y_ac, + const J2KJpegHuffmanTable *cb_dc, + const J2KJpegHuffmanTable *cb_ac, + const J2KJpegHuffmanTable *cr_dc, + const J2KJpegHuffmanTable *cr_ac, + J2KJpegEntropySyncState *states +) { + const unsigned int gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= params.subsequence_count) { + return; + } + + J2KJpegEntropySyncState state; + state.code = JPEG_STATUS_OK; + state.start_bit = gid * params.subsequence_bits; + if (state.start_bit >= params.entropy_bits) { + state.end_bit = params.entropy_bits; + } else { + const unsigned int remaining_bits = params.entropy_bits - state.start_bit; + state.end_bit = state.start_bit + min(params.subsequence_bits, remaining_bits); + } + state.bit_pos = state.start_bit; + state.symbol_count = 0u; + state.block_phase = 0u; + state.zigzag_index = 0u; + state.reserved = 0u; + + J2KJpegBitReader reader = + j2k_jpeg_bit_reader_at_bit(entropy, params.entropy_len, state.start_bit); + J2KJpegDecodeStatus status; + status.code = JPEG_STATUS_OK; + status.detail = 0u; + status.position = 0u; + status.reserved = 0u; + + while (state.bit_pos < state.end_bit && status.code == JPEG_STATUS_OK) { + if (!j2k_jpeg_entropy_scan_one_symbol420( + entropy, + params, + y_dc, + y_ac, + cb_dc, + cb_ac, + cr_dc, + cr_ac, + state, + reader, + status + )) { + break; + } + } + state.code = status.code; + states[gid] = state; +} + +extern "C" __global__ void j2k_jpeg_entropy_overflow420( + const unsigned char *entropy, + J2KJpegEntropyChunkParams params, + const J2KJpegHuffmanTable *y_dc, + const J2KJpegHuffmanTable *y_ac, + const J2KJpegHuffmanTable *cb_dc, + const J2KJpegHuffmanTable *cb_ac, + const J2KJpegHuffmanTable *cr_dc, + const J2KJpegHuffmanTable *cr_ac, + const J2KJpegEntropySyncState *states, + J2KJpegEntropyOverflowState *overflows +) { + const unsigned int gid = blockIdx.x * blockDim.x + threadIdx.x; + if (params.subsequence_count <= 1u) { + return; + } + const unsigned int overflow_count = params.subsequence_count - 1u; + if (gid >= overflow_count) { + return; + } + + J2KJpegEntropyOverflowState out; + out.code = JPEG_STATUS_OK; + out.from_subsequence = gid; + out.to_subsequence = gid + 1u; + out.overflow_bits = 0u; + out.synchronized = 0u; + out.reserved[0] = 0u; + out.reserved[1] = 0u; + out.reserved[2] = 0u; + + const J2KJpegEntropySyncState source = states[gid]; + const J2KJpegEntropySyncState target = states[gid + 1u]; + if (source.code != JPEG_STATUS_OK || target.code != JPEG_STATUS_OK) { + out.code = source.code != JPEG_STATUS_OK ? source.code : target.code; + overflows[gid] = out; + return; + } + + J2KJpegEntropySyncState state = source; + J2KJpegBitReader reader = + j2k_jpeg_bit_reader_at_bit(entropy, params.entropy_len, state.bit_pos); + J2KJpegDecodeStatus status; + status.code = JPEG_STATUS_OK; + status.detail = 0u; + status.position = 0u; + status.reserved = 0u; + + unsigned int stop_bit = state.bit_pos; + if (params.max_overflow_subsequences != 0u + && params.subsequence_bits != 0u + && state.bit_pos < params.entropy_bits) { + const unsigned int remaining_bits = params.entropy_bits - state.bit_pos; + unsigned int overflow_limit = remaining_bits; + if (params.max_overflow_subsequences <= remaining_bits / params.subsequence_bits) { + overflow_limit = params.max_overflow_subsequences * params.subsequence_bits; + } + stop_bit = state.bit_pos + min(overflow_limit, remaining_bits); + } + + if (state.bit_pos == target.bit_pos + && state.block_phase == target.block_phase + && state.zigzag_index == target.zigzag_index) { + out.synchronized = 1u; + out.overflow_bits = state.bit_pos > target.start_bit ? state.bit_pos - target.start_bit : 0u; + } else { + while (state.bit_pos < stop_bit && status.code == JPEG_STATUS_OK) { + if (!j2k_jpeg_entropy_scan_one_symbol420( + entropy, + params, + y_dc, + y_ac, + cb_dc, + cb_ac, + cr_dc, + cr_ac, + state, + reader, + status + )) { + break; + } + if (state.bit_pos == target.bit_pos + && state.block_phase == target.block_phase + && state.zigzag_index == target.zigzag_index) { + out.synchronized = 1u; + out.overflow_bits = state.bit_pos > target.start_bit ? state.bit_pos - target.start_bit : 0u; + break; + } + } + } + + if (status.code != JPEG_STATUS_OK && out.synchronized == 0u) { + out.code = status.code; + } + overflows[gid] = out; +} + +static constexpr int JPEG_CONST_BITS = 13; +static constexpr int JPEG_PASS1_BITS = 2; +static constexpr int JPEG_FIX_0_298631336 = 2446; +static constexpr int JPEG_FIX_0_390180644 = 3196; +static constexpr int JPEG_FIX_0_541196100 = 4433; +static constexpr int JPEG_FIX_0_765366865 = 6270; +static constexpr int JPEG_FIX_0_899976223 = 7373; +static constexpr int JPEG_FIX_1_175875602 = 9633; +static constexpr int JPEG_FIX_1_501321110 = 12299; +static constexpr int JPEG_FIX_1_847759065 = 15137; +static constexpr int JPEG_FIX_1_961570560 = 16069; +static constexpr int JPEG_FIX_2_053119869 = 16819; +static constexpr int JPEG_FIX_2_562915447 = 20995; +static constexpr int JPEG_FIX_3_072711026 = 25172; + +__device__ unsigned char j2k_jpeg_clamp_i32(int value) { + return static_cast(value < 0 ? 0 : (value > 255 ? 255 : value)); +} + +__device__ int j2k_jpeg_descale(int value, int shift) { + return value >> shift; +} + +__device__ unsigned char j2k_jpeg_descale_and_clamp(int value, int shift) { + return j2k_jpeg_clamp_i32((value >> shift) + 128); +} + +__device__ void j2k_jpeg_idct_column(const int input[64], int work[64], unsigned int col) { + const int p0 = input[col]; + const int p1 = input[col + 8u]; + const int p2 = input[col + 16u]; + const int p3 = input[col + 24u]; + const int p4 = input[col + 32u]; + const int p5 = input[col + 40u]; + const int p6 = input[col + 48u]; + const int p7 = input[col + 56u]; + + if (p1 == 0 && p2 == 0 && p3 == 0 && p4 == 0 && p5 == 0 && p6 == 0 && p7 == 0) { + const int dc = p0 << JPEG_PASS1_BITS; + work[col] = dc; + work[col + 8u] = dc; + work[col + 16u] = dc; + work[col + 24u] = dc; + work[col + 32u] = dc; + work[col + 40u] = dc; + work[col + 48u] = dc; + work[col + 56u] = dc; + return; + } + + int z2 = p2; + int z3 = p6; + int z1 = (z2 + z3) * JPEG_FIX_0_541196100; + int tmp2 = z1 - z3 * JPEG_FIX_1_847759065; + int tmp3 = z1 + z2 * JPEG_FIX_0_765366865; + + z2 = p0; + z3 = p4; + int tmp0 = (z2 + z3) << JPEG_CONST_BITS; + int tmp1 = (z2 - z3) << JPEG_CONST_BITS; + + int tmp10 = tmp0 + tmp3; + int tmp13 = tmp0 - tmp3; + int tmp11 = tmp1 + tmp2; + int tmp12 = tmp1 - tmp2; + + tmp0 = p7; + tmp1 = p5; + tmp2 = p3; + tmp3 = p1; + + z1 = tmp0 + tmp3; + z2 = tmp1 + tmp2; + z3 = tmp0 + tmp2; + int z4 = tmp1 + tmp3; + int z5 = (z3 + z4) * JPEG_FIX_1_175875602; + + tmp0 *= JPEG_FIX_0_298631336; + tmp1 *= JPEG_FIX_2_053119869; + tmp2 *= JPEG_FIX_3_072711026; + tmp3 *= JPEG_FIX_1_501321110; + z1 *= -JPEG_FIX_0_899976223; + z2 *= -JPEG_FIX_2_562915447; + z3 *= -JPEG_FIX_1_961570560; + z4 *= -JPEG_FIX_0_390180644; + + z3 += z5; + z4 += z5; + + tmp0 += z1 + z3; + tmp1 += z2 + z4; + tmp2 += z2 + z3; + tmp3 += z1 + z4; + + const int shift = JPEG_CONST_BITS - JPEG_PASS1_BITS; + const int rounding = 1 << (shift - 1); + work[col] = j2k_jpeg_descale(tmp10 + tmp3 + rounding, shift); + work[col + 56u] = j2k_jpeg_descale(tmp10 - tmp3 + rounding, shift); + work[col + 8u] = j2k_jpeg_descale(tmp11 + tmp2 + rounding, shift); + work[col + 48u] = j2k_jpeg_descale(tmp11 - tmp2 + rounding, shift); + work[col + 16u] = j2k_jpeg_descale(tmp12 + tmp1 + rounding, shift); + work[col + 40u] = j2k_jpeg_descale(tmp12 - tmp1 + rounding, shift); + work[col + 24u] = j2k_jpeg_descale(tmp13 + tmp0 + rounding, shift); + work[col + 32u] = j2k_jpeg_descale(tmp13 - tmp0 + rounding, shift); +} + +__device__ void j2k_jpeg_idct_row(const int work[64], unsigned char pixels[64], unsigned int row) { + const unsigned int base = row * 8u; + const int p0 = work[base]; + const int p1 = work[base + 1u]; + const int p2 = work[base + 2u]; + const int p3 = work[base + 3u]; + const int p4 = work[base + 4u]; + const int p5 = work[base + 5u]; + const int p6 = work[base + 6u]; + const int p7 = work[base + 7u]; + + const int shift = JPEG_CONST_BITS + JPEG_PASS1_BITS + 3; + const int rounding = 1 << (shift - 1); + + if (p1 == 0 && p2 == 0 && p3 == 0 && p4 == 0 && p5 == 0 && p6 == 0 && p7 == 0) { + const int dc_shift = JPEG_PASS1_BITS + 3; + const int rounding_dc = 1 << (dc_shift - 1); + const unsigned char pixel = j2k_jpeg_descale_and_clamp(p0 + rounding_dc, dc_shift); + for (unsigned int i = 0u; i < 8u; ++i) { + pixels[base + i] = pixel; + } + return; + } + + int z2 = p2; + int z3 = p6; + int z1 = (z2 + z3) * JPEG_FIX_0_541196100; + int tmp2 = z1 - z3 * JPEG_FIX_1_847759065; + int tmp3 = z1 + z2 * JPEG_FIX_0_765366865; + + int tmp0 = (p0 + p4) << JPEG_CONST_BITS; + int tmp1 = (p0 - p4) << JPEG_CONST_BITS; + + int tmp10 = tmp0 + tmp3; + int tmp13 = tmp0 - tmp3; + int tmp11 = tmp1 + tmp2; + int tmp12 = tmp1 - tmp2; + + tmp0 = p7; + tmp1 = p5; + tmp2 = p3; + tmp3 = p1; + + z1 = tmp0 + tmp3; + z2 = tmp1 + tmp2; + z3 = tmp0 + tmp2; + int z4 = tmp1 + tmp3; + int z5 = (z3 + z4) * JPEG_FIX_1_175875602; + + tmp0 *= JPEG_FIX_0_298631336; + tmp1 *= JPEG_FIX_2_053119869; + tmp2 *= JPEG_FIX_3_072711026; + tmp3 *= JPEG_FIX_1_501321110; + z1 *= -JPEG_FIX_0_899976223; + z2 *= -JPEG_FIX_2_562915447; + z3 *= -JPEG_FIX_1_961570560; + z4 *= -JPEG_FIX_0_390180644; + + z3 += z5; + z4 += z5; + + tmp0 += z1 + z3; + tmp1 += z2 + z4; + tmp2 += z2 + z3; + tmp3 += z1 + z4; + + pixels[base] = j2k_jpeg_descale_and_clamp(tmp10 + tmp3 + rounding, shift); + pixels[base + 7u] = j2k_jpeg_descale_and_clamp(tmp10 - tmp3 + rounding, shift); + pixels[base + 1u] = j2k_jpeg_descale_and_clamp(tmp11 + tmp2 + rounding, shift); + pixels[base + 6u] = j2k_jpeg_descale_and_clamp(tmp11 - tmp2 + rounding, shift); + pixels[base + 2u] = j2k_jpeg_descale_and_clamp(tmp12 + tmp1 + rounding, shift); + pixels[base + 5u] = j2k_jpeg_descale_and_clamp(tmp12 - tmp1 + rounding, shift); + pixels[base + 3u] = j2k_jpeg_descale_and_clamp(tmp13 + tmp0 + rounding, shift); + pixels[base + 4u] = j2k_jpeg_descale_and_clamp(tmp13 - tmp0 + rounding, shift); +} + +__device__ void j2k_jpeg_idct_islow(const int coeffs[64], unsigned char pixels[64]) { + int work[64]; + for (unsigned int col = 0u; col < 8u; ++col) { + j2k_jpeg_idct_column(coeffs, work, col); + } + for (unsigned int row = 0u; row < 8u; ++row) { + j2k_jpeg_idct_row(work, pixels, row); + } +} + +__device__ unsigned char j2k_jpeg_h2v2_sample( + const unsigned char block[64], + unsigned int chroma_cols, + unsigned int chroma_rows, + unsigned int output_x, + unsigned int chroma_y, + bool bottom +) { + const unsigned int n = chroma_cols == 0u ? 1u : chroma_cols; + const unsigned int curr_y = chroma_y < chroma_rows ? chroma_y : chroma_rows - 1u; + const unsigned int near_y = bottom + ? (curr_y + 1u < chroma_rows ? curr_y + 1u : chroma_rows - 1u) + : (curr_y == 0u ? 0u : curr_y - 1u); + const unsigned int sample = min(output_x / 2u, n - 1u); + const unsigned int curr = static_cast(block[curr_y * 8u + sample]); + const unsigned int near = static_cast(block[near_y * 8u + sample]); + const unsigned int this_sum = 3u * curr + near; + if (n == 1u) { + return static_cast((4u * this_sum + 8u) >> 4u); + } + if (output_x == 0u) { + return static_cast((this_sum * 4u + 8u) >> 4u); + } + if (output_x == n * 2u - 1u) { + return static_cast((this_sum * 4u + 7u) >> 4u); + } + if ((output_x & 1u) == 0u) { + const unsigned int last_curr = static_cast(block[curr_y * 8u + sample - 1u]); + const unsigned int last_near = static_cast(block[near_y * 8u + sample - 1u]); + const unsigned int last_sum = 3u * last_curr + last_near; + return static_cast((this_sum * 3u + last_sum + 8u) >> 4u); + } + const unsigned int next_sample = min(sample + 1u, n - 1u); + const unsigned int next_curr = static_cast(block[curr_y * 8u + next_sample]); + const unsigned int next_near = static_cast(block[near_y * 8u + next_sample]); + const unsigned int next_sum = 3u * next_curr + next_near; + return static_cast((this_sum * 3u + next_sum + 7u) >> 4u); +} + +__device__ unsigned char j2k_jpeg_h2v1_sample( + const unsigned char block[64], + unsigned int chroma_cols, + unsigned int output_x, + unsigned int chroma_y +) { + const unsigned int n = chroma_cols == 0u ? 1u : chroma_cols; + const unsigned int row = min(chroma_y, 7u); + const unsigned int base = row * 8u; + if (n == 1u) { + return block[base]; + } + const unsigned int sample = min(output_x / 2u, n - 1u); + if (output_x == 0u) { + return block[base]; + } + if (output_x == n * 2u - 1u) { + return block[base + n - 1u]; + } + const unsigned int curr = static_cast(block[base + sample]); + if ((output_x & 1u) == 0u) { + const unsigned int prev = static_cast(block[base + sample - 1u]); + return static_cast((3u * curr + prev + 2u) >> 2u); + } + const unsigned int next = static_cast(block[base + sample + 1u]); + return static_cast((3u * curr + next + 2u) >> 2u); +} + +__device__ void j2k_jpeg_ycbcr_to_rgb( + unsigned char y, + unsigned char cb, + unsigned char cr, + unsigned char &r, + unsigned char &g, + unsigned char &b +) { + const int yy = static_cast(y); + const int cb_centered = static_cast(cb) - 128; + const int cr_centered = static_cast(cr) - 128; + r = j2k_jpeg_clamp_i32(yy + ((91881 * cr_centered + (1 << 15)) >> 16)); + g = j2k_jpeg_clamp_i32(yy - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); + b = j2k_jpeg_clamp_i32(yy + ((116130 * cb_centered + (1 << 15)) >> 16)); +} + +__device__ void j2k_jpeg_store_rgb420_mcu( + unsigned char *out, + const J2KJpeg420Params ¶ms, + unsigned int mx, + unsigned int my, + const unsigned char y0[64], + const unsigned char y1[64], + const unsigned char y2[64], + const unsigned char y3[64], + const unsigned char cb[64], + const unsigned char cr[64] +) { + const unsigned int base_x = mx * 16u; + const unsigned int base_y = my * 16u; + const unsigned int remaining_x = params.width > base_x ? params.width - base_x : 0u; + const unsigned int remaining_y = params.height > base_y ? params.height - base_y : 0u; + const unsigned int chroma_cols = min(8u, (remaining_x + 1u) / 2u); + const unsigned int chroma_rows = min(8u, (remaining_y + 1u) / 2u); + for (unsigned int yy = 0u; yy < 16u; ++yy) { + const unsigned int py = base_y + yy; + if (py >= params.height) { + continue; + } + for (unsigned int xx = 0u; xx < 16u; ++xx) { + const unsigned int px = base_x + xx; + if (px >= params.width) { + continue; + } + const unsigned char *yb = yy < 8u + ? (xx < 8u ? y0 : y1) + : (xx < 8u ? y2 : y3); + const unsigned int y_idx = (yy & 7u) * 8u + (xx & 7u); + const unsigned int chroma_y = min(yy / 2u, chroma_rows - 1u); + const bool bottom = (yy & 1u) != 0u; + const unsigned char cbv = + j2k_jpeg_h2v2_sample(cb, chroma_cols, chroma_rows, xx, chroma_y, bottom); + const unsigned char crv = + j2k_jpeg_h2v2_sample(cr, chroma_cols, chroma_rows, xx, chroma_y, bottom); + const unsigned int dst = py * params.out_stride + px * 3u; + unsigned char r = 0u; + unsigned char g = 0u; + unsigned char b = 0u; + j2k_jpeg_ycbcr_to_rgb(yb[y_idx], cbv, crv, r, g, b); + out[dst] = r; + out[dst + 1u] = g; + out[dst + 2u] = b; + } + } +} + +__device__ void j2k_jpeg_store_rgb422_mcu( + unsigned char *out, + const J2KJpeg420Params ¶ms, + unsigned int mx, + unsigned int my, + const unsigned char y0[64], + const unsigned char y1[64], + const unsigned char cb[64], + const unsigned char cr[64] +) { + const unsigned int base_x = mx * 16u; + const unsigned int base_y = my * 8u; + const unsigned int remaining_x = params.width > base_x ? params.width - base_x : 0u; + const unsigned int remaining_y = params.height > base_y ? params.height - base_y : 0u; + const unsigned int chroma_cols = min(8u, (remaining_x + 1u) / 2u); + const unsigned int chroma_rows = min(8u, remaining_y); + for (unsigned int yy = 0u; yy < 8u; ++yy) { + const unsigned int py = base_y + yy; + if (py >= params.height) { + continue; + } + const unsigned int chroma_y = min(yy, chroma_rows - 1u); + for (unsigned int xx = 0u; xx < 16u; ++xx) { + const unsigned int px = base_x + xx; + if (px >= params.width) { + continue; + } + const unsigned char *yb = xx < 8u ? y0 : y1; + const unsigned int y_idx = yy * 8u + (xx & 7u); + const unsigned char cbv = j2k_jpeg_h2v1_sample(cb, chroma_cols, xx, chroma_y); + const unsigned char crv = j2k_jpeg_h2v1_sample(cr, chroma_cols, xx, chroma_y); + const unsigned int dst = py * params.out_stride + px * 3u; + unsigned char r = 0u; + unsigned char g = 0u; + unsigned char b = 0u; + j2k_jpeg_ycbcr_to_rgb(yb[y_idx], cbv, crv, r, g, b); + out[dst] = r; + out[dst + 1u] = g; + out[dst + 2u] = b; + } + } +} + +__device__ void j2k_jpeg_store_rgb444_mcu( + unsigned char *out, + const J2KJpeg420Params ¶ms, + unsigned int mx, + unsigned int my, + const unsigned char y[64], + const unsigned char cb[64], + const unsigned char cr[64] +) { + const unsigned int base_x = mx * 8u; + const unsigned int base_y = my * 8u; + for (unsigned int yy = 0u; yy < 8u; ++yy) { + const unsigned int py = base_y + yy; + if (py >= params.height) { + continue; + } + for (unsigned int xx = 0u; xx < 8u; ++xx) { + const unsigned int px = base_x + xx; + if (px >= params.width) { + continue; + } + const unsigned int idx = yy * 8u + xx; + const unsigned int dst = py * params.out_stride + px * 3u; + unsigned char r = 0u; + unsigned char g = 0u; + unsigned char b = 0u; + j2k_jpeg_ycbcr_to_rgb(y[idx], cb[idx], cr[idx], r, g, b); + out[dst] = r; + out[dst + 1u] = g; + out[dst + 2u] = b; + } + } +} + +extern "C" __global__ void j2k_jpeg_decode_fast420_rgb8( + const unsigned char *entropy, + unsigned char *out, + J2KJpeg420Params params, + const unsigned short *y_quant, + const unsigned short *cb_quant, + const unsigned short *cr_quant, + const J2KJpegHuffmanTable *y_dc, + const J2KJpegHuffmanTable *y_ac, + const J2KJpegHuffmanTable *cb_dc, + const J2KJpegHuffmanTable *cb_ac, + const J2KJpegHuffmanTable *cr_dc, + const J2KJpegHuffmanTable *cr_ac, + const J2KJpegEntropyCheckpoint *checkpoints, + J2KJpegDecodeStatus *status +) { + const unsigned int gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= params.checkpoint_count) { + return; + } + J2KJpegDecodeStatus *thread_status = status + gid; + thread_status->code = JPEG_STATUS_OK; + thread_status->detail = 0u; + thread_status->position = 0u; + thread_status->reserved = 0u; + + const unsigned int total_mcus = params.mcus_per_row * params.mcu_rows; + const J2KJpegEntropyCheckpoint checkpoint = checkpoints[gid]; + unsigned int start_mcu = checkpoint.mcu_index; + if (start_mcu >= total_mcus) { + return; + } + unsigned int end_mcu = total_mcus; + if (gid + 1u < params.checkpoint_count) { + end_mcu = checkpoints[gid + 1u].mcu_index; + if (end_mcu > total_mcus) { + end_mcu = total_mcus; + } + } + if (end_mcu <= start_mcu) { + return; + } + + J2KJpegBitReader reader; + reader.pos = checkpoint.entropy_pos; + reader.acc = checkpoint.bit_acc; + reader.bits = checkpoint.bit_count; + int y_prev_dc = checkpoint.y_prev_dc; + int cb_prev_dc = checkpoint.cb_prev_dc; + int cr_prev_dc = checkpoint.cr_prev_dc; + + int coeffs[64]; + unsigned char y0[64]; + unsigned char y1[64]; + unsigned char y2[64]; + unsigned char y3[64]; + unsigned char cb[64]; + unsigned char cr[64]; + + for (unsigned int mcu = start_mcu; mcu < end_mcu; ++mcu) { + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, y0); + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, y1); + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, y2); + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, y3); + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, cb); + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, cr); + const unsigned int mx = mcu - (mcu / params.mcus_per_row) * params.mcus_per_row; + const unsigned int my = mcu / params.mcus_per_row; + j2k_jpeg_store_rgb420_mcu(out, params, mx, my, y0, y1, y2, y3, cb, cr); + } +} + +extern "C" __global__ void j2k_jpeg_decode_fast422_rgb8( + const unsigned char *entropy, + unsigned char *out, + J2KJpeg420Params params, + const unsigned short *y_quant, + const unsigned short *cb_quant, + const unsigned short *cr_quant, + const J2KJpegHuffmanTable *y_dc, + const J2KJpegHuffmanTable *y_ac, + const J2KJpegHuffmanTable *cb_dc, + const J2KJpegHuffmanTable *cb_ac, + const J2KJpegHuffmanTable *cr_dc, + const J2KJpegHuffmanTable *cr_ac, + const J2KJpegEntropyCheckpoint *checkpoints, + J2KJpegDecodeStatus *status +) { + const unsigned int gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= params.checkpoint_count) { + return; + } + J2KJpegDecodeStatus *thread_status = status + gid; + thread_status->code = JPEG_STATUS_OK; + thread_status->detail = 0u; + thread_status->position = 0u; + thread_status->reserved = 0u; + + const unsigned int total_mcus = params.mcus_per_row * params.mcu_rows; + const J2KJpegEntropyCheckpoint checkpoint = checkpoints[gid]; + unsigned int start_mcu = checkpoint.mcu_index; + if (start_mcu >= total_mcus) { + return; + } + unsigned int end_mcu = total_mcus; + if (gid + 1u < params.checkpoint_count) { + end_mcu = checkpoints[gid + 1u].mcu_index; + if (end_mcu > total_mcus) { + end_mcu = total_mcus; + } + } + if (end_mcu <= start_mcu) { + return; + } + + J2KJpegBitReader reader; + reader.pos = checkpoint.entropy_pos; + reader.acc = checkpoint.bit_acc; + reader.bits = checkpoint.bit_count; + int y_prev_dc = checkpoint.y_prev_dc; + int cb_prev_dc = checkpoint.cb_prev_dc; + int cr_prev_dc = checkpoint.cr_prev_dc; + + int coeffs[64]; + unsigned char y0[64]; + unsigned char y1[64]; + unsigned char cb[64]; + unsigned char cr[64]; + + for (unsigned int mcu = start_mcu; mcu < end_mcu; ++mcu) { + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, y0); + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, y1); + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, cb); + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, cr); + const unsigned int mx = mcu - (mcu / params.mcus_per_row) * params.mcus_per_row; + const unsigned int my = mcu / params.mcus_per_row; + j2k_jpeg_store_rgb422_mcu(out, params, mx, my, y0, y1, cb, cr); + } +} + +extern "C" __global__ void j2k_jpeg_decode_fast444_rgb8( + const unsigned char *entropy, + unsigned char *out, + J2KJpeg420Params params, + const unsigned short *y_quant, + const unsigned short *cb_quant, + const unsigned short *cr_quant, + const J2KJpegHuffmanTable *y_dc, + const J2KJpegHuffmanTable *y_ac, + const J2KJpegHuffmanTable *cb_dc, + const J2KJpegHuffmanTable *cb_ac, + const J2KJpegHuffmanTable *cr_dc, + const J2KJpegHuffmanTable *cr_ac, + const J2KJpegEntropyCheckpoint *checkpoints, + J2KJpegDecodeStatus *status +) { + const unsigned int gid = blockIdx.x * blockDim.x + threadIdx.x; + if (gid >= params.checkpoint_count) { + return; + } + J2KJpegDecodeStatus *thread_status = status + gid; + thread_status->code = JPEG_STATUS_OK; + thread_status->detail = 0u; + thread_status->position = 0u; + thread_status->reserved = 0u; + + const unsigned int total_mcus = params.mcus_per_row * params.mcu_rows; + const J2KJpegEntropyCheckpoint checkpoint = checkpoints[gid]; + unsigned int start_mcu = checkpoint.mcu_index; + if (start_mcu >= total_mcus) { + return; + } + unsigned int end_mcu = total_mcus; + if (gid + 1u < params.checkpoint_count) { + end_mcu = checkpoints[gid + 1u].mcu_index; + if (end_mcu > total_mcus) { + end_mcu = total_mcus; + } + } + if (end_mcu <= start_mcu) { + return; + } + + J2KJpegBitReader reader; + reader.pos = checkpoint.entropy_pos; + reader.acc = checkpoint.bit_acc; + reader.bits = checkpoint.bit_count; + int y_prev_dc = checkpoint.y_prev_dc; + int cb_prev_dc = checkpoint.cb_prev_dc; + int cr_prev_dc = checkpoint.cr_prev_dc; + + int coeffs[64]; + unsigned char y[64]; + unsigned char cb[64]; + unsigned char cr[64]; + + for (unsigned int mcu = start_mcu; mcu < end_mcu; ++mcu) { + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, y); + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, cb); + if (!j2k_jpeg_decode_block(reader, entropy, params.entropy_len, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, coeffs)) { + return; + } + j2k_jpeg_idct_islow(coeffs, cr); + const unsigned int mx = mcu - (mcu / params.mcus_per_row) * params.mcus_per_row; + const unsigned int my = mcu / params.mcus_per_row; + j2k_jpeg_store_rgb444_mcu(out, params, mx, my, y, cb, cr); + } +} diff --git a/crates/j2k-cuda-runtime/src/kernels.rs b/crates/j2k-cuda-runtime/src/kernels.rs new file mode 100644 index 00000000..c3c0be5a --- /dev/null +++ b/crates/j2k-cuda-runtime/src/kernels.rs @@ -0,0 +1,1354 @@ +use std::os::raw::c_uint; + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub(crate) enum CudaKernel { + CopyU8, + J2kDeinterleaveToF32, + J2kDeinterleaveStridedToF32, + J2kForwardRct, + J2kForwardIct, + J2kForwardDwt53Horizontal, + J2kForwardDwt53Vertical, + J2kForwardDwt97Horizontal, + J2kForwardDwt97Vertical, + J2kQuantizeSubband, + J2kQuantizeSubbandStrided, + Htj2kDecodeCodeblocks, + Htj2kDecodeCodeblocksMulti, + Htj2kDecodeCodeblocksMultiCleanupOnly, + Htj2kDecodeCodeblocksMultiCleanupDequantize, + J2kDequantizeHtj2kCodeblocks, + J2kDequantizeHtj2kCodeblocksMulti, + J2kDequantizeHtj2kCleanupJobsMulti, + J2kIdwtInterleave, + J2kIdwtInterleaveHorizontalMulti, + J2kIdwtInterleaveHorizontal53Multi, + J2kIdwtInterleaveHorizontal97Multi, + J2kIdwtHorizontal, + J2kIdwtHorizontal53, + J2kIdwtHorizontal97, + J2kIdwtVertical, + J2kIdwtVerticalMulti, + J2kIdwtVertical53Multi, + J2kIdwtVertical97Multi, + J2kIdwtVertical97MultiCols4, + J2kIdwtVertical53, + J2kIdwtVertical97, + Htj2kEncodeCodeblock, + Htj2kEncodeCodeblocks, + Htj2kEncodeCodeblocksMultiInput, + Htj2kEncodeCodeblocksMultiInputCleanup, + Htj2kEncodeCodeblocksMultiInputCleanup64, + Htj2kCompactCodeblocks, + Htj2kPacketizeCleanup, + #[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] + JpegDecodeFast420Rgb8, + #[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] + JpegDecodeFast422Rgb8, + #[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] + JpegDecodeFast444Rgb8, + #[cfg_attr(not(j2k_cuda_jpeg_decode_ptx_built), allow(dead_code))] + JpegEntropySync420, + #[allow(dead_code)] + JpegEntropyOverflow420, + J2kInverseDwtSingle, + J2kInverseMct, + J2kStoreGray16, + J2kStoreGray8, + J2kStoreRgb16, + J2kStoreRgb16Mct, + J2kStoreRgb8, + J2kStoreRgb8Mct, + J2kStoreRgb8MctBatch, + // Coefficient-domain JPEG->HTJ2K transcode (j2k-transcode-cuda). + TranscodeReversible53Idct, + TranscodeReversible53VerticalLow, + TranscodeReversible53VerticalHigh, + TranscodeReversible53HorizontalLow, + TranscodeReversible53HorizontalHigh, + TranscodeDwt97Idct, + TranscodeDwt97RowLift, + TranscodeDwt97ColumnLift, + TranscodeDwt97IdctBatch, + TranscodeDwt97IdctI16Batch, + TranscodeDwt97RowLiftBatch, + TranscodeDwt97RowLiftBatchCoop, + TranscodeDwt97ColumnLiftBatch, + TranscodeDwt97QuantizeCodeblocks, + TranscodeDwt97ColumnLiftQuantizeCodeblocksBatch, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct CudaLaunchGeometry { + pub grid: (c_uint, c_uint, c_uint), + pub block: (c_uint, c_uint, c_uint), +} + +impl CudaKernel { + #[cfg_attr(not(feature = "cuda-oxide-j2k-encode"), allow(dead_code))] + pub(crate) fn is_j2k_encode_stage(self) -> bool { + matches!( + self, + Self::J2kDeinterleaveToF32 + | Self::J2kDeinterleaveStridedToF32 + | Self::J2kForwardRct + | Self::J2kForwardIct + | Self::J2kForwardDwt53Horizontal + | Self::J2kForwardDwt53Vertical + | Self::J2kForwardDwt97Horizontal + | Self::J2kForwardDwt97Vertical + | Self::J2kQuantizeSubband + | Self::J2kQuantizeSubbandStrided + ) + } + + #[cfg_attr(not(feature = "cuda-oxide-j2k-encode"), allow(dead_code))] + pub(crate) fn is_cuda_oxide_j2k_encode_stage(self) -> bool { + self.is_j2k_encode_stage() + || matches!( + self, + Self::Htj2kCompactCodeblocks | Self::Htj2kPacketizeCleanup + ) + } + + #[cfg_attr(not(feature = "cuda-oxide-j2k-decode-store"), allow(dead_code))] + pub(crate) fn is_j2k_decode_store_stage(self) -> bool { + matches!( + self, + Self::J2kInverseMct + | Self::J2kStoreGray16 + | Self::J2kStoreGray8 + | Self::J2kStoreRgb16 + | Self::J2kStoreRgb16Mct + | Self::J2kStoreRgb8 + | Self::J2kStoreRgb8Mct + | Self::J2kStoreRgb8MctBatch + ) + } + + #[cfg_attr(not(feature = "cuda-oxide-j2k-dequantize"), allow(dead_code))] + pub(crate) fn is_j2k_dequantize_stage(self) -> bool { + matches!( + self, + Self::J2kDequantizeHtj2kCodeblocks + | Self::J2kDequantizeHtj2kCodeblocksMulti + | Self::J2kDequantizeHtj2kCleanupJobsMulti + ) + } + + #[cfg_attr(not(feature = "cuda-oxide-j2k-idwt"), allow(dead_code))] + pub(crate) fn is_j2k_idwt_stage(self) -> bool { + matches!( + self, + Self::J2kInverseDwtSingle + | Self::J2kIdwtInterleave + | Self::J2kIdwtInterleaveHorizontalMulti + | Self::J2kIdwtInterleaveHorizontal53Multi + | Self::J2kIdwtInterleaveHorizontal97Multi + | Self::J2kIdwtHorizontal + | Self::J2kIdwtHorizontal53 + | Self::J2kIdwtHorizontal97 + | Self::J2kIdwtVertical + | Self::J2kIdwtVerticalMulti + | Self::J2kIdwtVertical53Multi + | Self::J2kIdwtVertical97Multi + | Self::J2kIdwtVertical97MultiCols4 + | Self::J2kIdwtVertical53 + | Self::J2kIdwtVertical97 + ) + } + + #[cfg_attr(not(feature = "cuda-oxide-transcode"), allow(dead_code))] + pub(crate) fn is_transcode_reversible53_stage(self) -> bool { + matches!( + self, + Self::TranscodeReversible53Idct + | Self::TranscodeReversible53VerticalLow + | Self::TranscodeReversible53VerticalHigh + | Self::TranscodeReversible53HorizontalLow + | Self::TranscodeReversible53HorizontalHigh + ) + } + + #[cfg_attr(not(feature = "cuda-oxide-transcode"), allow(dead_code))] + pub(crate) fn is_transcode_dwt97_single_stage(self) -> bool { + matches!( + self, + Self::TranscodeDwt97Idct | Self::TranscodeDwt97RowLift | Self::TranscodeDwt97ColumnLift + ) + } + + #[cfg_attr(not(feature = "cuda-oxide-transcode"), allow(dead_code))] + pub(crate) fn is_transcode_dwt97_batch_stage(self) -> bool { + matches!( + self, + Self::TranscodeDwt97IdctBatch + | Self::TranscodeDwt97IdctI16Batch + | Self::TranscodeDwt97RowLiftBatch + | Self::TranscodeDwt97RowLiftBatchCoop + | Self::TranscodeDwt97ColumnLiftBatch + | Self::TranscodeDwt97QuantizeCodeblocks + | Self::TranscodeDwt97ColumnLiftQuantizeCodeblocksBatch + ) + } + + #[cfg_attr(not(feature = "cuda-oxide-transcode"), allow(dead_code))] + pub(crate) fn is_cuda_oxide_transcode_stage(self) -> bool { + self.is_transcode_reversible53_stage() + || self.is_transcode_dwt97_single_stage() + || self.is_transcode_dwt97_batch_stage() + } + + pub(crate) fn ptx(self) -> &'static [u8] { + match self { + Self::CopyU8 => COPY_U8_PTX, + Self::J2kDeinterleaveToF32 + | Self::J2kDeinterleaveStridedToF32 + | Self::J2kForwardRct + | Self::J2kForwardIct + | Self::J2kForwardDwt53Horizontal + | Self::J2kForwardDwt53Vertical + | Self::J2kForwardDwt97Horizontal + | Self::J2kForwardDwt97Vertical + | Self::J2kQuantizeSubband + | Self::J2kQuantizeSubbandStrided => J2K_ENCODE_PTX, + Self::Htj2kDecodeCodeblocks + | Self::Htj2kDecodeCodeblocksMulti + | Self::Htj2kDecodeCodeblocksMultiCleanupOnly + | Self::Htj2kDecodeCodeblocksMultiCleanupDequantize + | Self::J2kDequantizeHtj2kCodeblocks + | Self::J2kDequantizeHtj2kCodeblocksMulti + | Self::J2kDequantizeHtj2kCleanupJobsMulti + | Self::J2kIdwtInterleave + | Self::J2kIdwtInterleaveHorizontalMulti + | Self::J2kIdwtInterleaveHorizontal53Multi + | Self::J2kIdwtInterleaveHorizontal97Multi + | Self::J2kIdwtHorizontal + | Self::J2kIdwtHorizontal53 + | Self::J2kIdwtHorizontal97 + | Self::J2kIdwtVertical + | Self::J2kIdwtVerticalMulti + | Self::J2kIdwtVertical53Multi + | Self::J2kIdwtVertical97Multi + | Self::J2kIdwtVertical97MultiCols4 + | Self::J2kIdwtVertical53 + | Self::J2kIdwtVertical97 + | Self::J2kInverseDwtSingle + | Self::J2kInverseMct + | Self::J2kStoreGray16 + | Self::J2kStoreGray8 + | Self::J2kStoreRgb16 + | Self::J2kStoreRgb16Mct + | Self::J2kStoreRgb8 + | Self::J2kStoreRgb8Mct + | Self::J2kStoreRgb8MctBatch => HTJ2K_DECODE_PTX, + Self::Htj2kEncodeCodeblock + | Self::Htj2kEncodeCodeblocks + | Self::Htj2kEncodeCodeblocksMultiInput + | Self::Htj2kEncodeCodeblocksMultiInputCleanup + | Self::Htj2kEncodeCodeblocksMultiInputCleanup64 + | Self::Htj2kCompactCodeblocks + | Self::Htj2kPacketizeCleanup => HTJ2K_ENCODE_PTX, + Self::JpegDecodeFast420Rgb8 + | Self::JpegDecodeFast422Rgb8 + | Self::JpegDecodeFast444Rgb8 + | Self::JpegEntropySync420 + | Self::JpegEntropyOverflow420 => JPEG_DECODE_PTX, + Self::TranscodeReversible53Idct + | Self::TranscodeReversible53VerticalLow + | Self::TranscodeReversible53VerticalHigh + | Self::TranscodeReversible53HorizontalLow + | Self::TranscodeReversible53HorizontalHigh + | Self::TranscodeDwt97Idct + | Self::TranscodeDwt97RowLift + | Self::TranscodeDwt97ColumnLift + | Self::TranscodeDwt97IdctBatch + | Self::TranscodeDwt97IdctI16Batch + | Self::TranscodeDwt97RowLiftBatch + | Self::TranscodeDwt97RowLiftBatchCoop + | Self::TranscodeDwt97ColumnLiftBatch + | Self::TranscodeDwt97QuantizeCodeblocks + | Self::TranscodeDwt97ColumnLiftQuantizeCodeblocksBatch => TRANSCODE_PTX, + } + } + + pub(crate) fn entrypoint(self) -> &'static [u8] { + match self { + Self::CopyU8 => b"j2k_copy_u8\0", + Self::J2kDeinterleaveToF32 => b"j2k_deinterleave_to_f32\0", + Self::J2kDeinterleaveStridedToF32 => b"j2k_deinterleave_strided_to_f32\0", + Self::J2kForwardRct => b"j2k_forward_rct\0", + Self::J2kForwardIct => b"j2k_forward_ict\0", + Self::J2kForwardDwt53Horizontal => b"j2k_forward_dwt53_horizontal\0", + Self::J2kForwardDwt53Vertical => b"j2k_forward_dwt53_vertical\0", + Self::J2kForwardDwt97Horizontal => b"j2k_forward_dwt97_horizontal\0", + Self::J2kForwardDwt97Vertical => b"j2k_forward_dwt97_vertical\0", + Self::J2kQuantizeSubband => b"j2k_quantize_subband\0", + Self::J2kQuantizeSubbandStrided => b"j2k_quantize_subband_strided\0", + Self::Htj2kDecodeCodeblocks => b"j2k_htj2k_decode_codeblocks\0", + Self::Htj2kDecodeCodeblocksMulti => b"j2k_htj2k_decode_codeblocks_multi\0", + Self::Htj2kDecodeCodeblocksMultiCleanupOnly => { + b"j2k_htj2k_decode_codeblocks_multi_cleanup_only\0" + } + Self::Htj2kDecodeCodeblocksMultiCleanupDequantize => { + b"j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize\0" + } + Self::J2kDequantizeHtj2kCodeblocks => b"j2k_dequantize_htj2k_codeblocks\0", + Self::J2kDequantizeHtj2kCodeblocksMulti => b"j2k_dequantize_htj2k_codeblocks_multi\0", + Self::J2kDequantizeHtj2kCleanupJobsMulti => { + b"j2k_dequantize_htj2k_cleanup_jobs_multi\0" + } + Self::J2kIdwtInterleave => b"j2k_idwt_interleave\0", + Self::J2kIdwtInterleaveHorizontalMulti => b"j2k_idwt_interleave_horizontal_multi\0", + Self::J2kIdwtInterleaveHorizontal53Multi => { + b"j2k_idwt_interleave_horizontal_53_multi\0" + } + Self::J2kIdwtInterleaveHorizontal97Multi => { + b"j2k_idwt_interleave_horizontal_97_multi\0" + } + Self::J2kIdwtHorizontal => b"j2k_idwt_horizontal\0", + Self::J2kIdwtHorizontal53 => b"j2k_idwt_horizontal_53\0", + Self::J2kIdwtHorizontal97 => b"j2k_idwt_horizontal_97\0", + Self::J2kIdwtVertical => b"j2k_idwt_vertical\0", + Self::J2kIdwtVerticalMulti => b"j2k_idwt_vertical_multi\0", + Self::J2kIdwtVertical53Multi => b"j2k_idwt_vertical_53_multi\0", + Self::J2kIdwtVertical97Multi => b"j2k_idwt_vertical_97_multi\0", + Self::J2kIdwtVertical97MultiCols4 => b"j2k_idwt_vertical_97_multi_cols4\0", + Self::J2kIdwtVertical53 => b"j2k_idwt_vertical_53\0", + Self::J2kIdwtVertical97 => b"j2k_idwt_vertical_97\0", + Self::Htj2kEncodeCodeblock => b"j2k_htj2k_encode_codeblock\0", + Self::Htj2kEncodeCodeblocks => b"j2k_htj2k_encode_codeblocks\0", + Self::Htj2kEncodeCodeblocksMultiInput => b"j2k_htj2k_encode_codeblocks_multi_input\0", + Self::Htj2kEncodeCodeblocksMultiInputCleanup => { + b"j2k_htj2k_encode_codeblocks_multi_input_cleanup\0" + } + Self::Htj2kEncodeCodeblocksMultiInputCleanup64 => { + b"j2k_htj2k_encode_codeblocks_multi_input_cleanup_64\0" + } + Self::Htj2kCompactCodeblocks => b"j2k_htj2k_compact_codeblocks\0", + Self::Htj2kPacketizeCleanup => b"j2k_htj2k_packetize_cleanup\0", + Self::JpegDecodeFast420Rgb8 => b"j2k_jpeg_decode_fast420_rgb8\0", + Self::JpegDecodeFast422Rgb8 => b"j2k_jpeg_decode_fast422_rgb8\0", + Self::JpegDecodeFast444Rgb8 => b"j2k_jpeg_decode_fast444_rgb8\0", + Self::JpegEntropySync420 => b"j2k_jpeg_entropy_sync420\0", + Self::JpegEntropyOverflow420 => b"j2k_jpeg_entropy_overflow420\0", + Self::J2kInverseDwtSingle => b"j2k_inverse_dwt_single\0", + Self::J2kInverseMct => b"j2k_inverse_mct\0", + Self::J2kStoreGray16 => b"j2k_store_gray16\0", + Self::J2kStoreGray8 => b"j2k_store_gray8\0", + Self::J2kStoreRgb16 => b"j2k_store_rgb16\0", + Self::J2kStoreRgb16Mct => b"j2k_store_rgb16_mct\0", + Self::J2kStoreRgb8 => b"j2k_store_rgb8\0", + Self::J2kStoreRgb8Mct => b"j2k_store_rgb8_mct\0", + Self::J2kStoreRgb8MctBatch => b"j2k_store_rgb8_mct_batch\0", + Self::TranscodeReversible53Idct => b"transcode_reversible53_idct\0", + Self::TranscodeReversible53VerticalLow => b"transcode_reversible53_vertical_low\0", + Self::TranscodeReversible53VerticalHigh => b"transcode_reversible53_vertical_high\0", + Self::TranscodeReversible53HorizontalLow => b"transcode_reversible53_horizontal_low\0", + Self::TranscodeReversible53HorizontalHigh => { + b"transcode_reversible53_horizontal_high\0" + } + Self::TranscodeDwt97Idct => b"transcode_dwt97_idct\0", + Self::TranscodeDwt97RowLift => b"transcode_dwt97_row_lift\0", + Self::TranscodeDwt97ColumnLift => b"transcode_dwt97_column_lift\0", + Self::TranscodeDwt97IdctBatch => b"transcode_dwt97_idct_batch\0", + Self::TranscodeDwt97IdctI16Batch => b"transcode_dwt97_idct_i16_batch\0", + Self::TranscodeDwt97RowLiftBatch => b"transcode_dwt97_row_lift_batch\0", + Self::TranscodeDwt97RowLiftBatchCoop => b"transcode_dwt97_row_lift_batch_coop\0", + Self::TranscodeDwt97ColumnLiftBatch => b"transcode_dwt97_column_lift_batch\0", + Self::TranscodeDwt97QuantizeCodeblocks => b"transcode_dwt97_quantize_codeblocks\0", + Self::TranscodeDwt97ColumnLiftQuantizeCodeblocksBatch => { + b"transcode_dwt97_column_lift_quantize_codeblocks_batch\0" + } + } + } +} + +pub(crate) fn copy_u8_launch_geometry(len: usize) -> Option { + x_blocks_launch_geometry(len, 1, COPY_U8_THREADS) +} + +const COPY_U8_THREADS: usize = 256; +const COPY_U8_THREADS_CUDA: c_uint = 256; +const J2K_IDWT_COOP_THREADS_SMALL_CUDA: c_uint = 256; +const J2K_IDWT_COOP_THREADS_LARGE_CUDA: c_uint = 512; +const J2K_ENCODE_THREADS_X: c_uint = 16; +const J2K_ENCODE_THREADS_Y: c_uint = 16; +const J2K_ENCODE_PTX: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/j2k_encode_kernels.ptx")); +const HTJ2K_DECODE_PTX: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/htj2k_decode_kernels.ptx")); +const HTJ2K_ENCODE_PTX: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/htj2k_encode_kernels.ptx")); +const JPEG_DECODE_PTX: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/jpeg_decode_kernels.ptx")); +// Always resolves: build.rs writes a placeholder empty module when nvcc is +// absent (the dispatch checks `j2k_cuda_transcode_ptx_built` before load). +const TRANSCODE_PTX: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/transcode_kernels.ptx")); +#[cfg(feature = "cuda-oxide-copy-u8")] +const CUDA_OXIDE_COPY_U8_PTX: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/cuda_oxide_copy_u8.ptx")); +#[cfg(feature = "cuda-oxide-j2k-encode")] +const CUDA_OXIDE_J2K_ENCODE_PTX: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/cuda_oxide_j2k_encode.ptx")); +#[cfg(feature = "cuda-oxide-j2k-decode-store")] +const CUDA_OXIDE_J2K_DECODE_STORE_PTX: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/cuda_oxide_j2k_decode_store.ptx")); +#[cfg(feature = "cuda-oxide-j2k-dequantize")] +const CUDA_OXIDE_J2K_DEQUANTIZE_PTX: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/cuda_oxide_j2k_dequantize.ptx")); +#[cfg(feature = "cuda-oxide-j2k-idwt")] +const CUDA_OXIDE_J2K_IDWT_PTX: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/cuda_oxide_j2k_idwt.ptx")); +#[cfg(feature = "cuda-oxide-transcode")] +const CUDA_OXIDE_TRANSCODE_PTX: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/cuda_oxide_transcode.ptx")); +const HTJ2K_DECODE_CODEBLOCK_THREADS: usize = 32; +const HTJ2K_DECODE_CODEBLOCK_THREADS_CUDA: c_uint = 32; +const HTJ2K_DECODE_PACKED_BLOCK_MIN_JOBS: usize = 2_048; +const HTJ2K_ENCODE_CODEBLOCK_THREADS_CUDA: c_uint = 128; + +pub(crate) fn j2k_forward_rct_launch_geometry(len: usize) -> Option { + x_blocks_launch_geometry(len, 1, COPY_U8_THREADS) +} + +pub(crate) fn j2k_dwt53_launch_geometry(width: u32, height: u32) -> Option { + let grid_x = c_uint::try_from(width.div_ceil(J2K_ENCODE_THREADS_X)).ok()?; + let grid_y = c_uint::try_from(height.div_ceil(J2K_ENCODE_THREADS_Y)).ok()?; + Some(CudaLaunchGeometry { + grid: (grid_x, grid_y, 1), + block: (J2K_ENCODE_THREADS_X, J2K_ENCODE_THREADS_Y, 1), + }) +} + +pub(crate) fn j2k_idwt_multi_1d_launch_geometry( + max_len: usize, + job_count: usize, +) -> Option { + x_blocks_launch_geometry(max_len, job_count, COPY_U8_THREADS) +} + +pub(crate) fn j2k_idwt_multi_coop_launch_geometry( + max_len: usize, + job_count: usize, +) -> Option { + let lanes = c_uint::try_from(max_len).ok()?; + let jobs = c_uint::try_from(job_count).ok()?; + let threads = if max_len > COPY_U8_THREADS { + J2K_IDWT_COOP_THREADS_LARGE_CUDA + } else { + J2K_IDWT_COOP_THREADS_SMALL_CUDA + }; + Some(CudaLaunchGeometry { + grid: (lanes, jobs, 1), + block: (threads, 1, 1), + }) +} + +pub(crate) fn j2k_idwt_multi_coop_axis_launch_geometry( + work_items: usize, + lane_count: usize, + job_count: usize, +) -> Option { + let blocks = c_uint::try_from(work_items).ok()?; + let jobs = c_uint::try_from(job_count).ok()?; + let threads = if lane_count > COPY_U8_THREADS { + J2K_IDWT_COOP_THREADS_LARGE_CUDA + } else { + J2K_IDWT_COOP_THREADS_SMALL_CUDA + }; + Some(CudaLaunchGeometry { + grid: (blocks, jobs, 1), + block: (threads, 1, 1), + }) +} + +pub(crate) fn j2k_idwt_multi_coop_columns_launch_geometry( + columns: usize, + rows: usize, + job_count: usize, + columns_per_block: usize, +) -> Option { + if rows == 0 || columns_per_block == 0 || rows.saturating_mul(columns_per_block) > 1024 { + return None; + } + let blocks = c_uint::try_from(columns.div_ceil(columns_per_block)).ok()?; + let jobs = c_uint::try_from(job_count).ok()?; + let block_x = c_uint::try_from(columns_per_block).ok()?; + let block_y = c_uint::try_from(rows).ok()?; + Some(CudaLaunchGeometry { + grid: (blocks, jobs, 1), + block: (block_x, block_y, 1), + }) +} + +pub(crate) fn htj2k_codeblock_launch_geometry(job_count: usize) -> Option { + if job_count >= HTJ2K_DECODE_PACKED_BLOCK_MIN_JOBS { + let jobs = c_uint::try_from(job_count.div_ceil(HTJ2K_DECODE_CODEBLOCK_THREADS)).ok()?; + Some(CudaLaunchGeometry { + grid: (jobs, 1, 1), + block: (HTJ2K_DECODE_CODEBLOCK_THREADS_CUDA, 1, 1), + }) + } else { + let jobs = c_uint::try_from(job_count).ok()?; + Some(CudaLaunchGeometry { + grid: (jobs, 1, 1), + block: (1, 1, 1), + }) + } +} + +pub(crate) fn htj2k_codeblock_sample_launch_geometry( + job_count: usize, +) -> Option { + let jobs = c_uint::try_from(job_count).ok()?; + Some(CudaLaunchGeometry { + grid: (jobs, 1, 1), + block: (COPY_U8_THREADS_CUDA, 1, 1), + }) +} + +pub(crate) fn j2k_store_batch_launch_geometry( + max_pixels: usize, + job_count: usize, +) -> Option { + x_blocks_launch_geometry(max_pixels, job_count, COPY_U8_THREADS) +} + +fn x_blocks_launch_geometry( + work_items: usize, + grid_y: usize, + threads_per_block: usize, +) -> Option { + if threads_per_block == 0 { + return None; + } + let blocks = c_uint::try_from(work_items.div_ceil(threads_per_block)).ok()?; + let grid_y = c_uint::try_from(grid_y).ok()?; + let block_x = c_uint::try_from(threads_per_block).ok()?; + Some(CudaLaunchGeometry { + grid: (blocks, grid_y, 1), + block: (block_x, 1, 1), + }) +} + +pub(crate) fn with_grid_y(base: CudaLaunchGeometry, grid_y: c_uint) -> CudaLaunchGeometry { + CudaLaunchGeometry { + grid: (base.grid.0, grid_y, base.grid.2), + block: base.block, + } +} + +pub(crate) fn with_grid_z(base: CudaLaunchGeometry, grid_z: c_uint) -> CudaLaunchGeometry { + CudaLaunchGeometry { + grid: (base.grid.0, base.grid.1, grid_z), + block: base.block, + } +} + +pub(crate) fn htj2k_encode_codeblock_launch_geometry( + job_count: usize, +) -> Option { + let jobs = c_uint::try_from(job_count).ok()?; + Some(CudaLaunchGeometry { + grid: (jobs, 1, 1), + block: (HTJ2K_ENCODE_CODEBLOCK_THREADS_CUDA, 1, 1), + }) +} + +pub(crate) fn htj2k_packetize_launch_geometry(packet_count: usize) -> Option { + htj2k_codeblock_sample_launch_geometry(packet_count) +} + +#[cfg(feature = "cuda-oxide-copy-u8")] +pub(crate) fn cuda_oxide_copy_u8_ptx() -> &'static [u8] { + CUDA_OXIDE_COPY_U8_PTX +} + +#[cfg(feature = "cuda-oxide-j2k-encode")] +pub(crate) fn cuda_oxide_j2k_encode_ptx() -> &'static [u8] { + CUDA_OXIDE_J2K_ENCODE_PTX +} + +#[cfg(feature = "cuda-oxide-j2k-decode-store")] +pub(crate) fn cuda_oxide_j2k_decode_store_ptx() -> &'static [u8] { + CUDA_OXIDE_J2K_DECODE_STORE_PTX +} + +#[cfg(feature = "cuda-oxide-j2k-dequantize")] +pub(crate) fn cuda_oxide_j2k_dequantize_ptx() -> &'static [u8] { + CUDA_OXIDE_J2K_DEQUANTIZE_PTX +} + +#[cfg(feature = "cuda-oxide-j2k-idwt")] +pub(crate) fn cuda_oxide_j2k_idwt_ptx() -> &'static [u8] { + CUDA_OXIDE_J2K_IDWT_PTX +} + +#[cfg(feature = "cuda-oxide-transcode")] +pub(crate) fn cuda_oxide_transcode_ptx() -> &'static [u8] { + CUDA_OXIDE_TRANSCODE_PTX +} + +const COPY_U8_PTX: &[u8] = concat!( + r" +.version 7.0 +.target sm_52 +.address_size 64 + +.visible .entry j2k_copy_u8( + .param .u64 dst, + .param .u64 src, + .param .u64 len +) +{ + .reg .pred %p; + .reg .b32 %r<5>; + .reg .b64 %rd<7>; + .reg .b16 %u; + + ld.param.u64 %rd1, [dst]; + ld.param.u64 %rd2, [src]; + ld.param.u64 %rd3, [len]; + mov.u32 %r1, %tid.x; + mov.u32 %r2, %ctaid.x; + mov.u32 %r3, %ntid.x; + mad.lo.s32 %r4, %r2, %r3, %r1; + cvt.u64.u32 %rd4, %r4; + setp.ge.u64 %p, %rd4, %rd3; + @%p bra DONE; + add.u64 %rd5, %rd2, %rd4; + ld.global.u8 %u, [%rd5]; + add.u64 %rd6, %rd1, %rd4; + st.global.u8 [%rd6], %u; +DONE: + ret; +} +", + "\0" +) +.as_bytes(); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn copy_u8_kernel_metadata_matches_embedded_ptx() { + let ptx = CudaKernel::CopyU8.ptx(); + assert_eq!(ptx.last(), Some(&0)); + let source = std::str::from_utf8(&ptx[..ptx.len() - 1]).expect("ptx utf8"); + assert!(source.contains(".visible .entry j2k_copy_u8(")); + assert_eq!(CudaKernel::CopyU8.entrypoint(), b"j2k_copy_u8\0"); + } + + #[cfg(all(feature = "cuda-oxide-copy-u8", j2k_cuda_oxide_copy_u8_built))] + #[test] + fn cuda_oxide_copy_u8_kernel_metadata_matches_generated_ptx() { + let ptx = cuda_oxide_copy_u8_ptx(); + assert_eq!(ptx.last(), Some(&0)); + let source = std::str::from_utf8(&ptx[..ptx.len() - 1]).expect("ptx utf8"); + assert!(source.contains(".visible .entry j2k_copy_u8(")); + assert_eq!(CudaKernel::CopyU8.entrypoint(), b"j2k_copy_u8\0"); + } + + #[test] + fn jpeg_decode_kernel_metadata_matches_source_entrypoints() { + assert_eq!( + CudaKernel::JpegEntropySync420.entrypoint(), + b"j2k_jpeg_entropy_sync420\0" + ); + assert_eq!( + CudaKernel::JpegEntropyOverflow420.entrypoint(), + b"j2k_jpeg_entropy_overflow420\0" + ); + + let cuda_source = include_str!("jpeg_decode_kernels.cu"); + assert!(cuda_source.contains("extern \"C\" __global__ void j2k_jpeg_entropy_sync420(")); + assert!(cuda_source.contains("extern \"C\" __global__ void j2k_jpeg_entropy_overflow420(")); + assert!(cuda_source.contains( + "const unsigned int remaining_bits = params.entropy_bits - state.start_bit;" + )); + let scanner_source = cuda_source + .split("__device__ bool j2k_jpeg_entropy_scan_one_symbol420(") + .nth(1) + .expect("entropy scanner source") + .split("extern \"C\" __global__ void j2k_jpeg_entropy_sync420(") + .next() + .expect("entropy scanner source before sync kernel"); + let sync_source = cuda_source + .split("extern \"C\" __global__ void j2k_jpeg_entropy_sync420(") + .nth(1) + .expect("entropy sync source") + .split("extern \"C\" __global__ void j2k_jpeg_entropy_overflow420(") + .next() + .expect("entropy sync source before overflow kernel"); + let overflow_source = cuda_source + .split("extern \"C\" __global__ void j2k_jpeg_entropy_overflow420(") + .nth(1) + .expect("entropy overflow source"); + assert!(sync_source.contains("j2k_jpeg_entropy_scan_one_symbol420(")); + assert!(overflow_source.contains("j2k_jpeg_entropy_scan_one_symbol420(")); + assert!(overflow_source.contains("const unsigned char *entropy,")); + let huffman_recovery = scanner_source + .find("if (status.code == JPEG_STATUS_HUFFMAN) {") + .expect("self-sync Huffman recovery"); + let one_bit_advance = scanner_source + .find("state.bit_pos += 1u;") + .expect("self-sync one-bit recovery advance"); + assert!(huffman_recovery < one_bit_advance); + let truncated_recovery = scanner_source + .find("if (status.code == JPEG_STATUS_TRUNCATED) {") + .expect("self-sync truncated entropy recovery"); + let amplitude_read = scanner_source + .find("if (!j2k_jpeg_ensure_bits(reader, entropy, params.entropy_len, coeff_bits))") + .expect("amplitude bit read"); + assert!(truncated_recovery < amplitude_read); + assert!(!scanner_source.contains("state.zigzag_index + run >= 64u")); + assert!(!scanner_source.contains("run != 0u && run != 15u")); + assert!(cuda_source.contains("__device__ bool j2k_jpeg_decode_symbol_real(")); + assert!(cuda_source.contains("__device__ bool j2k_jpeg_real_bits_consumed(")); + assert!(scanner_source.contains( + "j2k_jpeg_decode_symbol_real(reader, entropy, params.entropy_len, table, &status, symbol)" + )); + let no_progress_guard = scanner_source + .find("if (!j2k_jpeg_real_bits_consumed(reader, before_pos, before_bits, consumed))") + .expect("real-bit progress guard"); + let bit_pos_advance = scanner_source + .find("state.bit_pos += consumed;") + .expect("bit position advance"); + assert!(no_progress_guard < bit_pos_advance); + let coefficient_advance = scanner_source + .find("state.zigzag_index += run + 1u;") + .expect("AC coefficient advance"); + assert!(amplitude_read < coefficient_advance); + let eob_branch = scanner_source + .find("if (ssss == 0u && run != 15u) {") + .expect("EOB branch"); + assert!(eob_branch < coefficient_advance); + assert!(overflow_source + .contains("stop_bit = state.bit_pos + min(overflow_limit, remaining_bits);")); + } + + #[test] + fn htj2k_sample_geometry_uses_threads_with_one_block_per_codeblock() { + let geometry = htj2k_codeblock_sample_launch_geometry(3).expect("geometry"); + assert_eq!(geometry.grid, (3, 1, 1)); + assert_eq!(geometry.block, (COPY_U8_THREADS_CUDA, 1, 1)); + } + + #[test] + fn htj2k_cleanup_decode_geometry_packs_large_batches_into_warps() { + let small_geometry = htj2k_codeblock_launch_geometry(1_200).expect("small geometry"); + assert_eq!(small_geometry.grid, (1_200, 1, 1)); + assert_eq!(small_geometry.block, (1, 1, 1)); + + let large_geometry = htj2k_codeblock_launch_geometry(2_048).expect("large geometry"); + assert_eq!(large_geometry.grid, (64, 1, 1)); + assert_eq!(large_geometry.block, (32, 1, 1)); + } + + #[test] + fn htj2k_encode_geometry_uses_cooperative_threads_per_codeblock() { + let geometry = htj2k_encode_codeblock_launch_geometry(327).expect("geometry"); + assert_eq!(geometry.grid, (327, 1, 1)); + assert_eq!(geometry.block, (128, 1, 1)); + } + + #[test] + fn htj2k_packetize_geometry_uses_cooperative_threads_per_packet() { + let geometry = htj2k_packetize_launch_geometry(5).expect("geometry"); + assert_eq!(geometry.grid, (5, 1, 1)); + assert_eq!(geometry.block, (COPY_U8_THREADS_CUDA, 1, 1)); + } + + #[test] + fn j2k_encode_kernel_metadata_matches_generated_ptx() { + assert_eq!(J2K_ENCODE_PTX.last(), Some(&0)); + assert_eq!( + CudaKernel::J2kDeinterleaveToF32.entrypoint(), + b"j2k_deinterleave_to_f32\0" + ); + assert_eq!(CudaKernel::J2kForwardRct.entrypoint(), b"j2k_forward_rct\0"); + assert_eq!(CudaKernel::J2kForwardIct.entrypoint(), b"j2k_forward_ict\0"); + assert_eq!( + CudaKernel::J2kForwardDwt53Horizontal.entrypoint(), + b"j2k_forward_dwt53_horizontal\0" + ); + assert_eq!( + CudaKernel::J2kForwardDwt53Vertical.entrypoint(), + b"j2k_forward_dwt53_vertical\0" + ); + assert_eq!( + CudaKernel::J2kForwardDwt97Horizontal.entrypoint(), + b"j2k_forward_dwt97_horizontal\0" + ); + assert_eq!( + CudaKernel::J2kForwardDwt97Vertical.entrypoint(), + b"j2k_forward_dwt97_vertical\0" + ); + assert_eq!( + CudaKernel::J2kQuantizeSubband.entrypoint(), + b"j2k_quantize_subband\0" + ); + assert_eq!( + CudaKernel::J2kQuantizeSubbandStrided.entrypoint(), + b"j2k_quantize_subband_strided\0" + ); + let source = + std::str::from_utf8(&J2K_ENCODE_PTX[..J2K_ENCODE_PTX.len() - 1]).expect("ptx utf8"); + assert!(source.contains(".visible .entry j2k_deinterleave_to_f32(")); + assert!(source.contains(".visible .entry j2k_quantize_subband_strided(")); + } + + #[cfg(all(feature = "cuda-oxide-j2k-encode", j2k_cuda_oxide_j2k_encode_built))] + #[test] + fn cuda_oxide_j2k_encode_kernel_metadata_matches_generated_ptx() { + let ptx = cuda_oxide_j2k_encode_ptx(); + assert_eq!(ptx.last(), Some(&0)); + let source = std::str::from_utf8(&ptx[..ptx.len() - 1]).expect("ptx utf8"); + let kernels = [ + CudaKernel::J2kDeinterleaveToF32, + CudaKernel::J2kDeinterleaveStridedToF32, + CudaKernel::J2kForwardRct, + CudaKernel::J2kForwardIct, + CudaKernel::J2kForwardDwt53Horizontal, + CudaKernel::J2kForwardDwt53Vertical, + CudaKernel::J2kForwardDwt97Horizontal, + CudaKernel::J2kForwardDwt97Vertical, + CudaKernel::J2kQuantizeSubband, + CudaKernel::J2kQuantizeSubbandStrided, + CudaKernel::Htj2kCompactCodeblocks, + CudaKernel::Htj2kPacketizeCleanup, + ]; + for kernel in kernels { + assert!(kernel.is_cuda_oxide_j2k_encode_stage()); + let entrypoint = + std::str::from_utf8(&kernel.entrypoint()[..kernel.entrypoint().len() - 1]) + .expect("entrypoint utf8"); + assert!( + source.contains(&format!(".visible .entry {entrypoint}(")), + "missing cuda-oxide J2K encode entrypoint {entrypoint}" + ); + } + } + + #[cfg(all( + feature = "cuda-oxide-j2k-decode-store", + j2k_cuda_oxide_j2k_decode_store_built + ))] + #[test] + fn cuda_oxide_j2k_decode_store_kernel_metadata_matches_generated_ptx() { + let ptx = cuda_oxide_j2k_decode_store_ptx(); + assert_eq!(ptx.last(), Some(&0)); + let source = std::str::from_utf8(&ptx[..ptx.len() - 1]).expect("ptx utf8"); + let kernels = [ + CudaKernel::J2kInverseMct, + CudaKernel::J2kStoreGray8, + CudaKernel::J2kStoreGray16, + CudaKernel::J2kStoreRgb8, + CudaKernel::J2kStoreRgb8Mct, + CudaKernel::J2kStoreRgb8MctBatch, + CudaKernel::J2kStoreRgb16, + CudaKernel::J2kStoreRgb16Mct, + ]; + for kernel in kernels { + assert!(kernel.is_j2k_decode_store_stage()); + let entrypoint = + std::str::from_utf8(&kernel.entrypoint()[..kernel.entrypoint().len() - 1]) + .expect("entrypoint utf8"); + assert!( + source.contains(&format!(".visible .entry {entrypoint}(")), + "missing cuda-oxide J2K decode-store entrypoint {entrypoint}" + ); + } + } + + #[cfg(all( + feature = "cuda-oxide-j2k-dequantize", + j2k_cuda_oxide_j2k_dequantize_built + ))] + #[test] + fn cuda_oxide_j2k_dequantize_kernel_metadata_matches_generated_ptx() { + let ptx = cuda_oxide_j2k_dequantize_ptx(); + assert_eq!(ptx.last(), Some(&0)); + let source = std::str::from_utf8(&ptx[..ptx.len() - 1]).expect("ptx utf8"); + let kernels = [ + CudaKernel::J2kDequantizeHtj2kCodeblocks, + CudaKernel::J2kDequantizeHtj2kCodeblocksMulti, + CudaKernel::J2kDequantizeHtj2kCleanupJobsMulti, + ]; + for kernel in kernels { + assert!(kernel.is_j2k_dequantize_stage()); + let entrypoint = + std::str::from_utf8(&kernel.entrypoint()[..kernel.entrypoint().len() - 1]) + .expect("entrypoint utf8"); + assert!( + source.contains(&format!(".visible .entry {entrypoint}(")), + "missing cuda-oxide J2K dequantize entrypoint {entrypoint}" + ); + } + } + + #[cfg(all(feature = "cuda-oxide-j2k-idwt", j2k_cuda_oxide_j2k_idwt_built))] + #[test] + fn cuda_oxide_j2k_idwt_kernel_metadata_matches_generated_ptx() { + let ptx = cuda_oxide_j2k_idwt_ptx(); + assert_eq!(ptx.last(), Some(&0)); + let source = std::str::from_utf8(&ptx[..ptx.len() - 1]).expect("ptx utf8"); + let kernels = [ + CudaKernel::J2kInverseDwtSingle, + CudaKernel::J2kIdwtInterleave, + CudaKernel::J2kIdwtInterleaveHorizontalMulti, + CudaKernel::J2kIdwtInterleaveHorizontal53Multi, + CudaKernel::J2kIdwtInterleaveHorizontal97Multi, + CudaKernel::J2kIdwtHorizontal, + CudaKernel::J2kIdwtHorizontal53, + CudaKernel::J2kIdwtHorizontal97, + CudaKernel::J2kIdwtVertical, + CudaKernel::J2kIdwtVerticalMulti, + CudaKernel::J2kIdwtVertical53Multi, + CudaKernel::J2kIdwtVertical97Multi, + CudaKernel::J2kIdwtVertical97MultiCols4, + CudaKernel::J2kIdwtVertical53, + CudaKernel::J2kIdwtVertical97, + ]; + for kernel in kernels { + assert!(kernel.is_j2k_idwt_stage()); + let entrypoint = + std::str::from_utf8(&kernel.entrypoint()[..kernel.entrypoint().len() - 1]) + .expect("entrypoint utf8"); + assert!( + source.contains(&format!(".visible .entry {entrypoint}(")), + "missing cuda-oxide J2K IDWT entrypoint {entrypoint}" + ); + } + } + + #[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] + #[test] + fn cuda_oxide_transcode_kernel_metadata_matches_generated_ptx() { + let ptx = cuda_oxide_transcode_ptx(); + assert_eq!(ptx.last(), Some(&0)); + let source = std::str::from_utf8(&ptx[..ptx.len() - 1]).expect("ptx utf8"); + let kernels = [ + CudaKernel::TranscodeReversible53Idct, + CudaKernel::TranscodeReversible53VerticalLow, + CudaKernel::TranscodeReversible53VerticalHigh, + CudaKernel::TranscodeReversible53HorizontalLow, + CudaKernel::TranscodeReversible53HorizontalHigh, + CudaKernel::TranscodeDwt97Idct, + CudaKernel::TranscodeDwt97RowLift, + CudaKernel::TranscodeDwt97ColumnLift, + CudaKernel::TranscodeDwt97IdctBatch, + CudaKernel::TranscodeDwt97IdctI16Batch, + CudaKernel::TranscodeDwt97RowLiftBatch, + CudaKernel::TranscodeDwt97RowLiftBatchCoop, + CudaKernel::TranscodeDwt97ColumnLiftBatch, + CudaKernel::TranscodeDwt97QuantizeCodeblocks, + CudaKernel::TranscodeDwt97ColumnLiftQuantizeCodeblocksBatch, + ]; + for kernel in kernels { + assert!(kernel.is_cuda_oxide_transcode_stage()); + let entrypoint = + std::str::from_utf8(&kernel.entrypoint()[..kernel.entrypoint().len() - 1]) + .expect("entrypoint utf8"); + assert!( + source.contains(&format!(".visible .entry {entrypoint}(")), + "missing cuda-oxide transcode entrypoint {entrypoint}" + ); + } + } + + #[test] + fn j2k_encode_kernel_uses_native_irreversible_delta_formula() { + let cuda_source = include_str!("j2k_encode_kernels.cu"); + assert!(cuda_source.contains("const int exponent = int(range_bits) - int(step_exponent);")); + assert!(!cuda_source.contains("const int exponent = int(step_exponent) - int(range_bits);")); + } + + #[test] + #[allow(clippy::too_many_lines)] + fn htj2k_decode_kernel_metadata_matches_generated_ptx() { + assert_eq!(HTJ2K_DECODE_PTX.last(), Some(&0)); + assert_eq!( + CudaKernel::Htj2kDecodeCodeblocks.entrypoint(), + b"j2k_htj2k_decode_codeblocks\0" + ); + assert_eq!( + CudaKernel::Htj2kDecodeCodeblocksMulti.entrypoint(), + b"j2k_htj2k_decode_codeblocks_multi\0" + ); + assert_eq!( + CudaKernel::Htj2kDecodeCodeblocksMultiCleanupOnly.entrypoint(), + b"j2k_htj2k_decode_codeblocks_multi_cleanup_only\0" + ); + assert_eq!( + CudaKernel::Htj2kDecodeCodeblocksMultiCleanupDequantize.entrypoint(), + b"j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize\0" + ); + assert_eq!( + CudaKernel::J2kDequantizeHtj2kCodeblocks.entrypoint(), + b"j2k_dequantize_htj2k_codeblocks\0" + ); + assert_eq!( + CudaKernel::J2kDequantizeHtj2kCodeblocksMulti.entrypoint(), + b"j2k_dequantize_htj2k_codeblocks_multi\0" + ); + assert_eq!( + CudaKernel::J2kDequantizeHtj2kCleanupJobsMulti.entrypoint(), + b"j2k_dequantize_htj2k_cleanup_jobs_multi\0" + ); + assert_eq!( + CudaKernel::J2kIdwtInterleave.entrypoint(), + b"j2k_idwt_interleave\0" + ); + assert_eq!( + CudaKernel::J2kIdwtInterleaveHorizontalMulti.entrypoint(), + b"j2k_idwt_interleave_horizontal_multi\0" + ); + assert_eq!( + CudaKernel::J2kIdwtInterleaveHorizontal53Multi.entrypoint(), + b"j2k_idwt_interleave_horizontal_53_multi\0" + ); + assert_eq!( + CudaKernel::J2kIdwtInterleaveHorizontal97Multi.entrypoint(), + b"j2k_idwt_interleave_horizontal_97_multi\0" + ); + assert_eq!( + CudaKernel::J2kIdwtHorizontal.entrypoint(), + b"j2k_idwt_horizontal\0" + ); + assert_eq!( + CudaKernel::J2kIdwtVertical.entrypoint(), + b"j2k_idwt_vertical\0" + ); + assert_eq!( + CudaKernel::J2kIdwtVerticalMulti.entrypoint(), + b"j2k_idwt_vertical_multi\0" + ); + assert_eq!( + CudaKernel::J2kIdwtVertical53Multi.entrypoint(), + b"j2k_idwt_vertical_53_multi\0" + ); + assert_eq!( + CudaKernel::J2kIdwtVertical97Multi.entrypoint(), + b"j2k_idwt_vertical_97_multi\0" + ); + assert_eq!( + CudaKernel::J2kIdwtVertical97MultiCols4.entrypoint(), + b"j2k_idwt_vertical_97_multi_cols4\0" + ); + assert_eq!( + CudaKernel::J2kInverseDwtSingle.entrypoint(), + b"j2k_inverse_dwt_single\0" + ); + assert_eq!(CudaKernel::J2kInverseMct.entrypoint(), b"j2k_inverse_mct\0"); + assert_eq!(CudaKernel::J2kStoreGray8.entrypoint(), b"j2k_store_gray8\0"); + assert_eq!( + CudaKernel::J2kStoreGray16.entrypoint(), + b"j2k_store_gray16\0" + ); + assert_eq!(CudaKernel::J2kStoreRgb8.entrypoint(), b"j2k_store_rgb8\0"); + assert_eq!( + CudaKernel::J2kStoreRgb8Mct.entrypoint(), + b"j2k_store_rgb8_mct\0" + ); + assert_eq!( + CudaKernel::J2kStoreRgb8MctBatch.entrypoint(), + b"j2k_store_rgb8_mct_batch\0" + ); + assert_eq!(CudaKernel::J2kStoreRgb16.entrypoint(), b"j2k_store_rgb16\0"); + assert_eq!( + CudaKernel::J2kStoreRgb16Mct.entrypoint(), + b"j2k_store_rgb16_mct\0" + ); + let source = + std::str::from_utf8(&HTJ2K_DECODE_PTX[..HTJ2K_DECODE_PTX.len() - 1]).expect("ptx utf8"); + assert!(source.contains(".visible .entry j2k_htj2k_decode_codeblocks_multi(")); + assert!(source.contains(".visible .entry j2k_htj2k_decode_codeblocks_multi_cleanup_only(")); + assert!(source + .contains(".visible .entry j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize(")); + let cuda_source = include_str!("htj2k_decode_kernels.cu"); + assert!(cuda_source.contains( + "extern \"C\" __global__ void j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize(" + )); + assert!(source.contains(".visible .entry j2k_dequantize_htj2k_codeblocks(")); + assert!(source.contains(".visible .entry j2k_dequantize_htj2k_codeblocks_multi(")); + assert!(source.contains(".visible .entry j2k_dequantize_htj2k_cleanup_jobs_multi(")); + assert!(source.contains(".visible .entry j2k_idwt_interleave(")); + assert!(source.contains(".visible .entry j2k_idwt_interleave_horizontal_multi(")); + assert!(source.contains(".visible .entry j2k_idwt_interleave_horizontal_53_multi(")); + assert!(source.contains(".visible .entry j2k_idwt_interleave_horizontal_97_multi(")); + assert!(source.contains(".visible .entry j2k_idwt_horizontal(")); + assert!(source.contains(".visible .entry j2k_idwt_vertical(")); + assert!(source.contains(".visible .entry j2k_idwt_vertical_multi(")); + assert!(source.contains(".visible .entry j2k_idwt_vertical_53_multi(")); + assert!(source.contains(".visible .entry j2k_idwt_vertical_97_multi(")); + assert!(source.contains(".visible .entry j2k_idwt_horizontal_53(")); + assert!(source.contains(".visible .entry j2k_idwt_vertical_53(")); + assert!(source.contains(".visible .entry j2k_idwt_horizontal_97(")); + assert!(source.contains(".visible .entry j2k_idwt_vertical_97(")); + assert!(source.contains(".visible .entry j2k_store_rgb8_mct(")); + assert!(source.contains(".visible .entry j2k_store_rgb8_mct_batch(")); + assert!(source.contains(".visible .entry j2k_store_rgb16_mct(")); + } + + #[test] + fn htj2k_decode_cleanup_kernels_guard_padded_launch_threads() { + let cuda_source = include_str!("htj2k_decode_kernels.cu"); + assert!(cuda_source.contains("if (gid >= job_count)")); + } + + #[test] + fn htj2k_encode_kernel_metadata_matches_generated_ptx() { + assert_eq!(HTJ2K_ENCODE_PTX.last(), Some(&0)); + assert_eq!( + CudaKernel::Htj2kEncodeCodeblock.entrypoint(), + b"j2k_htj2k_encode_codeblock\0" + ); + assert_eq!( + CudaKernel::Htj2kEncodeCodeblocks.entrypoint(), + b"j2k_htj2k_encode_codeblocks\0" + ); + assert_eq!( + CudaKernel::Htj2kEncodeCodeblocksMultiInput.entrypoint(), + b"j2k_htj2k_encode_codeblocks_multi_input\0" + ); + assert_eq!( + CudaKernel::Htj2kEncodeCodeblocksMultiInputCleanup.entrypoint(), + b"j2k_htj2k_encode_codeblocks_multi_input_cleanup\0" + ); + assert_eq!( + CudaKernel::Htj2kEncodeCodeblocksMultiInputCleanup64.entrypoint(), + b"j2k_htj2k_encode_codeblocks_multi_input_cleanup_64\0" + ); + assert_eq!( + CudaKernel::Htj2kPacketizeCleanup.entrypoint(), + b"j2k_htj2k_packetize_cleanup\0" + ); + let source = + std::str::from_utf8(&HTJ2K_ENCODE_PTX[..HTJ2K_ENCODE_PTX.len() - 1]).expect("ptx utf8"); + assert!(source.contains(".visible .entry j2k_htj2k_encode_codeblocks(")); + assert!(source.contains(".visible .entry j2k_htj2k_encode_codeblocks_multi_input(")); + if cfg!(j2k_cuda_htj2k_encode_ptx_built) { + assert!( + source.contains(".visible .entry j2k_htj2k_encode_codeblocks_multi_input_cleanup(") + ); + assert!(source + .contains(".visible .entry j2k_htj2k_encode_codeblocks_multi_input_cleanup_64(")); + } + assert!(source.contains(".visible .entry j2k_htj2k_packetize_cleanup(")); + let cuda_source = include_str!("htj2k_encode_kernels.cu"); + assert!(cuda_source.contains("j2k_ht_reduce_max_magnitude_cooperative")); + assert!(cuda_source.contains("j2k_packet_copy_body_cooperative")); + } + + #[test] + fn htj2k_encode_kernel_reports_zero_passes_for_all_zero_codeblocks() { + let cuda_source = include_str!("htj2k_encode_kernels.cu"); + assert!(cuda_source.contains( + "j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_OK, 0u, 0u, 0u, params.total_bitplanes);" + )); + assert!(!cuda_source.contains( + "j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_OK, 0u, 0u, 1u, params.total_bitplanes);" + )); + } + + #[test] + fn htj2k_encode_kernel_uses_width_bounded_cleanup_scratch_clear() { + let cuda_source = include_str!("htj2k_encode_kernels.cu"); + assert!(cuda_source.contains("FIXED_64 ? 34u : j2k_ht_cleanup_scratch_entries(width)")); + assert!(cuda_source.contains( + "params.width == 64u && params.height == 64u && params.coefficient_stride == 64u" + )); + assert!(!cuda_source.contains("idx < 513u; ++idx")); + } + + #[test] + fn htj2k_encode_kernel_uses_shared_cleanup_scratch() { + let cuda_source = include_str!("htj2k_encode_kernels.cu"); + assert!(cuda_source.contains("__shared__ uchar cleanup_e_val[J2K_HT_SIGPROP_SCRATCH];")); + assert!(cuda_source.contains("__shared__ uchar cleanup_cx_val[J2K_HT_SIGPROP_SCRATCH];")); + assert!(cuda_source.contains("cleanup_e_val,\n cleanup_cx_val")); + } + + #[test] + fn htj2k_encode_kernel_sizes_max_reduction_for_encode_launch_threads() { + let cuda_source = include_str!("htj2k_encode_kernels.cu"); + assert!(cuda_source.contains("J2K_HT_ENCODE_THREADS = 128u")); + assert!(cuda_source.contains("__shared__ uint block_max[J2K_HT_ENCODE_THREADS];")); + assert!(!cuda_source.contains("__shared__ uint block_max[256];")); + } + + #[test] + fn htj2k_encode_multi_input_kernel_declares_launch_bounds() { + let cuda_source = include_str!("htj2k_encode_kernels.cu"); + let expected = concat!( + "extern \"C\" __global__ void __", + "launch_bounds__(J2K_HT_ENCODE_THREADS) ", + "j2k_htj2k_encode_codeblocks_multi_input" + ); + assert!(cuda_source.contains(expected)); + assert!(cuda_source.contains("j2k_htj2k_encode_codeblocks_multi_input_cleanup")); + } + + #[test] + fn htj2k_encode_kernel_has_contiguous_max_reduction_fast_path() { + let cuda_source = include_str!("htj2k_encode_kernels.cu"); + assert!(cuda_source.contains("if (coefficient_stride == width)")); + assert!(cuda_source.contains("j2k_classic_magnitude(coefficients[sample])")); + } + + #[test] + fn transcode_kernel_entrypoints_match_names() { + assert_eq!( + CudaKernel::TranscodeDwt97Idct.entrypoint(), + b"transcode_dwt97_idct\0" + ); + assert_eq!( + CudaKernel::TranscodeDwt97RowLift.entrypoint(), + b"transcode_dwt97_row_lift\0" + ); + assert_eq!( + CudaKernel::TranscodeDwt97ColumnLift.entrypoint(), + b"transcode_dwt97_column_lift\0" + ); + assert_eq!( + CudaKernel::TranscodeDwt97IdctBatch.entrypoint(), + b"transcode_dwt97_idct_batch\0" + ); + assert_eq!( + CudaKernel::TranscodeDwt97IdctI16Batch.entrypoint(), + b"transcode_dwt97_idct_i16_batch\0" + ); + assert_eq!( + CudaKernel::TranscodeDwt97RowLiftBatch.entrypoint(), + b"transcode_dwt97_row_lift_batch\0" + ); + assert_eq!( + CudaKernel::TranscodeDwt97RowLiftBatchCoop.entrypoint(), + b"transcode_dwt97_row_lift_batch_coop\0" + ); + assert_eq!( + CudaKernel::TranscodeDwt97ColumnLiftBatch.entrypoint(), + b"transcode_dwt97_column_lift_batch\0" + ); + assert_eq!( + CudaKernel::TranscodeDwt97QuantizeCodeblocks.entrypoint(), + b"transcode_dwt97_quantize_codeblocks\0" + ); + assert_eq!( + CudaKernel::TranscodeDwt97ColumnLiftQuantizeCodeblocksBatch.entrypoint(), + b"transcode_dwt97_column_lift_quantize_codeblocks_batch\0" + ); + // All transcode kernels share the one translation unit's PTX. + assert_eq!( + CudaKernel::TranscodeDwt97QuantizeCodeblocks.ptx().as_ptr(), + TRANSCODE_PTX.as_ptr() + ); + + // The placeholder PTX is empty when nvcc is absent; only validate entry + // points are present once the runner actually compiled the kernels. + if cfg!(j2k_cuda_transcode_ptx_built) { + let source = std::str::from_utf8(&TRANSCODE_PTX[..TRANSCODE_PTX.len() - 1]) + .expect("transcode ptx utf8"); + assert!(source.contains(".visible .entry transcode_dwt97_idct_batch(")); + assert!(source.contains(".visible .entry transcode_dwt97_idct_i16_batch(")); + assert!(source.contains(".visible .entry transcode_dwt97_row_lift_batch(")); + assert!(source.contains(".visible .entry transcode_dwt97_row_lift_batch_coop(")); + assert!(source.contains(".visible .entry transcode_dwt97_column_lift_batch(")); + assert!(source.contains(".visible .entry transcode_dwt97_quantize_codeblocks(")); + assert!(source.contains( + ".visible .entry transcode_dwt97_column_lift_quantize_codeblocks_batch(" + )); + } + } + + #[test] + fn transcode_dwt97_idct_uses_precomputed_basis_table() { + let cuda_source = include_str!("transcode_kernels.cu"); + assert!(cuda_source.contains("DWT97_IDCT8_BASIS")); + assert!(cuda_source.contains("DWT97_IDCT8_BASIS[sample_idx * 8 + freq]")); + assert!(!cuda_source.contains("sqrtf(1.0f / 8.0f)")); + assert!(!cuda_source.contains("cosf(angle)")); + } + + #[test] + fn transcode_dwt97_idct_unrolls_fixed_basis_loops() { + let cuda_source = include_str!("transcode_kernels.cu"); + assert!(cuda_source.contains("transcode_dwt97_idct_unroll_guard")); + assert!( + cuda_source.contains("#pragma unroll\n for (int freq_y = 0; freq_y < 8; ++freq_y)") + ); + assert!(cuda_source + .contains("#pragma unroll\n for (int freq_x = 0; freq_x < 8; ++freq_x)")); + } + + #[test] + fn transcode_dwt97_batch_row_lift_has_cooperative_kernel() { + let cuda_source = include_str!("transcode_kernels.cu"); + assert!(cuda_source.contains("transcode_dwt97_row_lift_batch_coop(")); + assert!(cuda_source.contains("DWT97_ROW_LIFT_MAX_WIDTH")); + assert!(cuda_source.contains( + "__shared__ f32 rows[DWT97_ROW_LIFT_ROWS_PER_BLOCK][DWT97_ROW_LIFT_MAX_WIDTH];" + )); + } + + #[test] + fn copy_u8_launch_geometry_rounds_up_to_256_thread_blocks() { + assert_eq!(copy_u8_launch_geometry(1).unwrap().grid, (1, 1, 1)); + assert_eq!(copy_u8_launch_geometry(256).unwrap().grid, (1, 1, 1)); + assert_eq!(copy_u8_launch_geometry(257).unwrap().grid, (2, 1, 1)); + } + + #[test] + fn x_blocks_launch_geometry_rounds_work_items_and_preserves_y_grid() { + let geometry = x_blocks_launch_geometry(513, 7, COPY_U8_THREADS).unwrap(); + + assert_eq!(geometry.grid, (3, 7, 1)); + assert_eq!(geometry.block, (COPY_U8_THREADS_CUDA, 1, 1)); + } + + #[test] + fn x_blocks_launch_geometry_rejects_zero_threads() { + assert_eq!(x_blocks_launch_geometry(513, 7, 0), None); + } + + #[test] + #[cfg(target_pointer_width = "64")] + fn x_blocks_launch_geometry_rejects_grid_dimensions_above_cuda_uint() { + assert_eq!(x_blocks_launch_geometry(usize::MAX, usize::MAX, 1), None); + } + + #[test] + fn with_grid_y_preserves_block_and_other_grid_axes() { + let base = CudaLaunchGeometry { + grid: (2, 3, 4), + block: (16, 8, 1), + }; + + let geometry = with_grid_y(base, 9); + + assert_eq!(geometry.grid, (2, 9, 4)); + assert_eq!(geometry.block, base.block); + } + + #[test] + fn with_grid_z_preserves_block_and_other_grid_axes() { + let base = CudaLaunchGeometry { + grid: (2, 3, 4), + block: (16, 8, 1), + }; + + let geometry = with_grid_z(base, 11); + + assert_eq!(geometry.grid, (2, 3, 11)); + assert_eq!(geometry.block, base.block); + } + + #[test] + fn j2k_dwt53_launch_geometry_uses_16_by_16_thread_blocks() { + let geometry = j2k_dwt53_launch_geometry(17, 33).unwrap(); + assert_eq!(geometry.grid, (2, 3, 1)); + assert_eq!(geometry.block, (16, 16, 1)); + } +} diff --git a/crates/j2k-cuda-runtime/src/lib.rs b/crates/j2k-cuda-runtime/src/lib.rs new file mode 100644 index 00000000..5f4b5cd9 --- /dev/null +++ b/crates/j2k-cuda-runtime/src/lib.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! CUDA codec engine and Driver API runtime used by J2K CUDA adapter crates. + +#![deny(unsafe_op_in_unsafe_fn)] +#![deny(missing_docs)] +#![warn(unreachable_pub)] + +macro_rules! cuda_kernel_params { + ($($arg:ident),+ $(,)?) => { + [$(cuda_kernel_param(&mut $arg)),+] + }; +} + +mod build_flags; +mod bytes; +mod context; +mod driver; +mod error; +mod execution; +mod htj2k_decode; +mod htj2k_encode; +mod htj2k_packetize; +mod j2k_decode; +mod j2k_encode; +mod jpeg; +mod kernels; +mod memory; +#[cfg(test)] +mod tests; +mod transcode; + +pub use build_flags::transcode_kernels_built; +pub use context::{ + CudaContext, CudaHtj2kCompactEncodedCodeBlock, CudaHtj2kCompactEncodedCodeBlocks, + CudaKernelModule, CudaKernelName, +}; +pub use error::CudaError; +pub use execution::{ + CudaEvent, CudaExecutionStats, CudaKernelBatchOutput, CudaKernelContiguousBatchOutput, + CudaKernelOutput, CudaPooledKernelOutput, CudaQueuedExecution, CudaStream, +}; +pub use htj2k_decode::{ + CudaHtj2kCleanupTarget, CudaHtj2kCodeBlockJob, CudaHtj2kDecodeOutput, CudaHtj2kDecodeResources, + CudaHtj2kDecodeStageTimings, CudaHtj2kDecodeTableResources, CudaHtj2kDecodeTables, + CudaHtj2kDequantizeTarget, CudaHtj2kStatus, CudaPooledHtj2kDecodeOutput, + CudaQueuedHtj2kCleanup, +}; +pub use htj2k_encode::{ + CudaHtj2kEncodeCodeBlockJob, CudaHtj2kEncodeCodeBlockRegionJob, CudaHtj2kEncodeResidentTarget, + CudaHtj2kEncodeResources, CudaHtj2kEncodeStageTimings, CudaHtj2kEncodeStatus, + CudaHtj2kEncodeTables, CudaHtj2kEncodedCodeBlock, CudaHtj2kEncodedCodeBlocks, +}; +pub use htj2k_packetize::{ + CudaHtj2kPacketizationBlock, CudaHtj2kPacketizationPacket, CudaHtj2kPacketizationStageTimings, + CudaHtj2kPacketizationStatus, CudaHtj2kPacketizationSubband, + CudaHtj2kPacketizationSubbandTagState, CudaHtj2kPacketizationTagNodeState, + CudaHtj2kPacketizedTile, +}; +pub use j2k_decode::{ + CudaJ2kIdwtJob, CudaJ2kIdwtTarget, CudaJ2kInverseMctJob, CudaJ2kRect, CudaJ2kStoreGray16Job, + CudaJ2kStoreGray8Job, CudaJ2kStoreRgb16Job, CudaJ2kStoreRgb16MctJob, CudaJ2kStoreRgb8Job, + CudaJ2kStoreRgb8MctJob, CudaJ2kStoreRgb8MctTarget, CudaJ2kStridedInterleavedPixels, +}; +pub use j2k_encode::{ + CudaDwt53LevelShape, CudaDwt53Output, CudaDwt97BatchStageTimings, CudaDwt97Output, + CudaJ2kDeinterleavedComponents, CudaJ2kQuantizeJob, CudaJ2kQuantizeSubbandRegionJob, + CudaJ2kQuantizedSubband, CudaJ2kResidentComponents, CudaJ2kResidentQuantizedSubband, + CudaResidentDwt53Output, CudaResidentDwt97Output, +}; +pub use jpeg::{ + CudaJpeg420Rgb8DecodePlan, CudaJpegChunkedEntropyConfig, CudaJpegChunkedEntropyPlan, + CudaJpegChunkedEntropyReport, CudaJpegEntropyCheckpoint, CudaJpegEntropyOverflowState, + CudaJpegEntropySyncState, CudaJpegHuffmanTable, CudaJpegRgb8DecodePlan, CudaJpegRgb8Sampling, +}; +pub use memory::{ + CudaBufferPool, CudaBufferPoolTakeTrace, CudaDeviceBuffer, CudaDeviceBufferRange, + CudaDeviceBufferView, CudaDeviceBufferViewMut, CudaPinnedHostBuffer, CudaPooledDeviceBuffer, +}; +pub use transcode::{ + CudaHtj2k97CodeblockBands, CudaHtj2k97DeviceCodeblockBands, CudaHtj2k97QuantizeParams, + CudaTranscodeDwt97Bands, CudaTranscodeReversible53Bands, +}; + +#[cfg(test)] +pub(crate) use bytes::{ + f32_slice_as_bytes, f32_slice_as_bytes_mut, htj2k_cleanup_multi_jobs_as_bytes, + i32_slice_as_bytes, i32_slice_as_bytes_mut, +}; +#[cfg(test)] +pub(crate) use context::HTJ2K_UVLC_ENCODE_TABLE_BYTES; +#[cfg(test)] +pub(crate) use htj2k_decode::{ + htj2k_decode_multi_cleanup_dequant_kernel_for_jobs, htj2k_decode_multi_kernel_for_jobs, + CudaHtj2kCleanupMultiKernelJob, HTJ2K_STATUS_OK, HTJ2K_STATUS_UNSUPPORTED, +}; +#[cfg(test)] +pub(crate) use htj2k_encode::{ + htj2k_encode_compact_jobs, CudaHtj2kEncodeCompactJob, CudaHtj2kEncodeKernelJob, + HTJ2K_ENCODE_OUTPUT_CAPACITY, +}; +#[cfg(test)] +pub(crate) use j2k_decode::{ + checked_f32_words_byte_len, format_idwt_batch_trace_row, idwt_batch_kernel_mode, + idwt_batch_trace_row, idwt_batch_uses_cooperative_53, CudaJ2kIdwtBatchKernelMode, + CudaJ2kIdwtMultiKernelJob, +}; +#[cfg(test)] +pub(crate) use jpeg::jpeg_entropy_overflow_count; +#[cfg(test)] +pub(crate) use memory::{copy_pooled_bytes_to_vec_uninit, pool_fit_buffer_index_by_len}; +#[cfg(test)] +pub(crate) use transcode::{should_use_pinned_pooled_i16_upload, validate_dct_block_grid}; diff --git a/crates/j2k-cuda-runtime/src/memory.rs b/crates/j2k-cuda-runtime/src/memory.rs new file mode 100644 index 00000000..0b3e9a2b --- /dev/null +++ b/crates/j2k-cuda-runtime/src/memory.rs @@ -0,0 +1,872 @@ +use crate::{ + build_flags::PINNED_UPLOAD_STAGING_POOL_MAX, + bytes::{f32_slice_as_bytes, i16_slice_as_bytes, i32_slice_as_bytes}, + context::{CudaContext, PinnedUploadStaging}, + driver::CuDevicePtr, + error::CudaError, +}; +use std::{ + collections::BTreeMap, + ffi::c_void, + sync::{Arc, Mutex}, +}; + +impl CudaContext { + /// Upload host bytes into a CUDA device buffer. + pub fn upload(&self, bytes: &[u8]) -> Result { + self.inner.set_current()?; + + let mut ptr = 0; + let buffer = if bytes.is_empty() { + CudaDeviceBuffer { + context: self.clone(), + ptr, + len: bytes.len(), + } + } else { + // SAFETY: CUDA writes a device pointer for the requested byte size. + self.inner.driver.check("cuMemAlloc_v2", unsafe { + (self.inner.driver.cu_mem_alloc)(&raw mut ptr, bytes.len()) + })?; + + CudaDeviceBuffer { + context: self.clone(), + ptr, + len: bytes.len(), + } + }; + + if !bytes.is_empty() { + // SAFETY: ptr is a valid device allocation of bytes.len(), and the + // host pointer is valid for bytes.len(). + self.inner.driver.check("cuMemcpyHtoD_v2", unsafe { + (self.inner.driver.cu_memcpy_htod)( + ptr, + bytes.as_ptr().cast::(), + bytes.len(), + ) + })?; + } + + Ok(buffer) + } + + /// Upload host bytes through a temporary page-locked staging buffer. + pub fn upload_pinned(&self, bytes: &[u8]) -> Result { + if bytes.is_empty() { + return self.upload(bytes); + } + let mut staging = self.take_pinned_upload_staging(bytes.len())?; + staging.as_mut_slice()[..bytes.len()].copy_from_slice(bytes); + let upload_result = self.upload(&staging.as_slice()[..bytes.len()]); + let recycle_result = self.recycle_pinned_upload_staging(staging); + match (upload_result, recycle_result) { + (Ok(buffer), Ok(())) => Ok(buffer), + (Err(error), _) | (_, Err(error)) => Err(error), + } + } + + pub(crate) fn take_pinned_upload_staging( + &self, + len: usize, + ) -> Result { + self.inner.set_current()?; + let mut staging = + self.inner + .pinned_upload_staging + .lock() + .map_err(|error| CudaError::StatePoisoned { + message: error.to_string(), + })?; + if let Some(index) = staging.iter().position(|buffer| buffer.len >= len) { + return Ok(staging.swap_remove(index)); + } + drop(staging); + + let mut ptr = std::ptr::null_mut(); + // SAFETY: CUDA writes a page-locked host pointer for the requested byte + // length. The allocation is freed by the context's staging pool cleanup. + self.inner.driver.check("cuMemHostAlloc", unsafe { + (self.inner.driver.cu_mem_host_alloc)(&raw mut ptr, len, 0) + })?; + Ok(PinnedUploadStaging { + ptr: ptr.cast::(), + len, + }) + } + + pub(crate) fn recycle_pinned_upload_staging( + &self, + staging: PinnedUploadStaging, + ) -> Result<(), CudaError> { + let mut pool = + self.inner + .pinned_upload_staging + .lock() + .map_err(|error| CudaError::StatePoisoned { + message: error.to_string(), + })?; + if pool.len() < PINNED_UPLOAD_STAGING_POOL_MAX { + pool.push(staging); + return Ok(()); + } + drop(pool); + self.inner.set_current()?; + staging.free(&self.inner.driver) + } + + /// Upload host `f32` samples into a CUDA device buffer. + pub fn upload_f32(&self, samples: &[f32]) -> Result { + self.upload(f32_slice_as_bytes(samples)) + } + + /// Upload host `f32` samples through a temporary page-locked staging buffer. + pub fn upload_f32_pinned(&self, samples: &[f32]) -> Result { + self.upload_pinned(f32_slice_as_bytes(samples)) + } + + /// Upload host `i32` samples through a temporary page-locked staging buffer. + pub fn upload_i32_pinned(&self, samples: &[i32]) -> Result { + self.upload_pinned(i32_slice_as_bytes(samples)) + } + + /// Allocate an uninitialized CUDA device buffer. + pub fn allocate(&self, len: usize) -> Result { + self.inner.set_current()?; + let mut ptr = 0; + if len != 0 { + // SAFETY: CUDA writes a device pointer for the requested byte size. + self.inner.driver.check("cuMemAlloc_v2", unsafe { + (self.inner.driver.cu_mem_alloc)(&raw mut ptr, len) + })?; + } + Ok(CudaDeviceBuffer { + context: self.clone(), + ptr, + len, + }) + } + + /// Allocate page-locked host memory for host-to-device staging. + pub fn pinned_host_buffer(&self, len: usize) -> Result { + self.inner.set_current()?; + let mut ptr = std::ptr::null_mut(); + if len != 0 { + // SAFETY: CUDA writes a page-locked host pointer for the requested + // byte length. The allocation is freed by CudaPinnedHostBuffer. + self.inner.driver.check("cuMemHostAlloc", unsafe { + (self.inner.driver.cu_mem_host_alloc)(&raw mut ptr, len, 0) + })?; + } + Ok(CudaPinnedHostBuffer { + context: self.clone(), + ptr: ptr.cast::(), + len, + }) + } + + /// Create a reusable device-buffer pool for this context. + pub fn buffer_pool(&self) -> CudaBufferPool { + CudaBufferPool::new(self.clone()) + } + + /// Create a reusable best-fit device-buffer pool for workloads with many + /// same-sized intermediate buffers. + pub fn best_fit_buffer_pool(&self) -> CudaBufferPool { + CudaBufferPool::new_size_buckets(self.clone()) + } +} + +/// Page-locked host staging buffer. +#[derive(Debug)] +pub struct CudaPinnedHostBuffer { + pub(crate) context: CudaContext, + pub(crate) ptr: *mut u8, + pub(crate) len: usize, +} + +impl CudaPinnedHostBuffer { + /// Length in bytes. + pub fn len(&self) -> usize { + self.len + } + + /// Whether this buffer has zero length. + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + /// Immutable byte view of the pinned allocation. + pub fn as_slice(&self) -> &[u8] { + if self.len == 0 { + &[] + } else { + // SAFETY: ptr is a live pinned allocation of len bytes. + unsafe { std::slice::from_raw_parts(self.ptr.cast_const(), self.len) } + } + } + + /// Mutable byte view of the pinned allocation. + pub fn as_mut_slice(&mut self) -> &mut [u8] { + if self.len == 0 { + &mut [] + } else { + // SAFETY: ptr is uniquely borrowed through &mut self and covers len + // bytes allocated by CUDA. + unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) } + } + } +} + +impl Drop for CudaPinnedHostBuffer { + fn drop(&mut self) { + if !self.ptr.is_null() { + let _ = self.context.inner.set_current(); + // SAFETY: ptr was returned by cuMemHostAlloc for this process. + let _ = unsafe { (self.context.inner.driver.cu_mem_free_host)(self.ptr.cast()) }; + } + } +} + +// SAFETY: The pinned allocation is owned by this value and CUDA frees it on +// drop. Mutable access still requires &mut self. +unsafe impl Send for CudaPinnedHostBuffer {} + +/// Owned CUDA device buffer. +#[derive(Debug)] +pub struct CudaDeviceBuffer { + pub(crate) context: CudaContext, + pub(crate) ptr: CuDevicePtr, + pub(crate) len: usize, +} + +/// Typed immutable device buffer view. +#[derive(Clone, Copy, Debug)] +pub struct CudaDeviceBufferView<'a, T> { + pub(crate) ptr: CuDevicePtr, + pub(crate) len: usize, + pub(crate) _marker: std::marker::PhantomData<&'a T>, +} + +impl CudaDeviceBufferView<'_, T> { + /// Raw CUDA device pointer value for kernel argument binding. + pub fn device_ptr(&self) -> u64 { + self.ptr + } + + /// Number of typed elements in this view. + pub fn len(&self) -> usize { + self.len + } + + /// Whether this view has no elements. + pub fn is_empty(&self) -> bool { + self.len == 0 + } +} + +/// Typed mutable device buffer view. +#[derive(Debug)] +pub struct CudaDeviceBufferViewMut<'a, T> { + pub(crate) ptr: CuDevicePtr, + pub(crate) len: usize, + pub(crate) _marker: std::marker::PhantomData<&'a mut T>, +} + +impl CudaDeviceBufferViewMut<'_, T> { + /// Raw CUDA device pointer value for kernel argument binding. + pub fn device_ptr(&self) -> u64 { + self.ptr + } + + /// Number of typed elements in this view. + pub fn len(&self) -> usize { + self.len + } + + /// Whether this view has no elements. + pub fn is_empty(&self) -> bool { + self.len == 0 + } +} + +/// Reusable CUDA device-buffer pool for repeated adapter dispatches. +#[derive(Clone, Debug)] +pub struct CudaBufferPool { + pub(crate) inner: Arc, +} + +#[derive(Debug)] +pub(crate) struct CudaBufferPoolInner { + pub(crate) context: CudaContext, + pub(crate) free: Mutex, +} + +#[derive(Debug)] +pub(crate) enum CudaBufferPoolFree { + FirstFit(Vec), + SizeBuckets(BTreeMap>), +} + +impl CudaBufferPoolInner { + fn recycle_buffer(&self, buffer: CudaDeviceBuffer) -> Result<(), CudaError> { + let mut free = self.free.lock().map_err(|error| CudaError::StatePoisoned { + message: error.to_string(), + })?; + match &mut *free { + CudaBufferPoolFree::FirstFit(free) => free.push(buffer), + CudaBufferPoolFree::SizeBuckets(free) => { + free.entry(buffer.byte_len()).or_default().push(buffer); + } + } + Ok(()) + } +} + +/// Diagnostics for one traced [`CudaBufferPool`] acquisition. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaBufferPoolTakeTrace { + /// Requested byte length for the checkout. + pub requested_len: usize, + /// Number of cached free buffers before the checkout. + pub free_count_before: usize, + /// Number of cached entries examined while finding a reusable buffer or allocating. + pub scanned_count: usize, + /// Whether the checkout reused a cached allocation. + pub reused: bool, + /// Actual allocation byte length backing the checkout. + pub allocation_byte_len: usize, +} + +impl CudaBufferPool { + /// Create a new pool for `context`. + pub fn new(context: CudaContext) -> Self { + Self { + inner: Arc::new(CudaBufferPoolInner { + context, + free: Mutex::new(CudaBufferPoolFree::FirstFit(Vec::new())), + }), + } + } + + fn new_size_buckets(context: CudaContext) -> Self { + Self { + inner: Arc::new(CudaBufferPoolInner { + context, + free: Mutex::new(CudaBufferPoolFree::SizeBuckets(BTreeMap::new())), + }), + } + } + + /// Acquire a device buffer with at least `len` bytes. + pub fn take(&self, len: usize) -> Result { + let mut free = self + .inner + .free + .lock() + .map_err(|error| CudaError::StatePoisoned { + message: error.to_string(), + })?; + let (reusable_buffer, _) = pool_take_fit_buffer(&mut free, len); + let buffer = if let Some(buffer) = reusable_buffer { + buffer + } else { + drop(free); + self.inner.context.allocate(len)? + }; + Ok(CudaPooledDeviceBuffer { + buffer: Some(buffer), + requested_len: len, + pool: self.inner.clone(), + }) + } + + /// Return a raw device buffer to this pool. + pub fn recycle(&self, buffer: CudaDeviceBuffer) -> Result<(), CudaError> { + self.inner.recycle_buffer(buffer) + } + + /// Acquire a device buffer with diagnostics for profiling pool behavior. + pub fn take_with_trace( + &self, + len: usize, + ) -> Result<(CudaPooledDeviceBuffer, CudaBufferPoolTakeTrace), CudaError> { + let mut free = self + .inner + .free + .lock() + .map_err(|error| CudaError::StatePoisoned { + message: error.to_string(), + })?; + let free_count_before = free.cached_count(); + let (reusable_buffer, scanned_count) = pool_take_fit_buffer(&mut free, len); + let reused = reusable_buffer.is_some(); + let buffer = if let Some(buffer) = reusable_buffer { + buffer + } else { + drop(free); + self.inner.context.allocate(len)? + }; + let allocation_byte_len = buffer.byte_len(); + let trace = CudaBufferPoolTakeTrace { + requested_len: len, + free_count_before, + scanned_count, + reused, + allocation_byte_len, + }; + Ok(( + CudaPooledDeviceBuffer { + buffer: Some(buffer), + requested_len: len, + pool: self.inner.clone(), + }, + trace, + )) + } + + /// Upload host bytes into a pooled device buffer. + pub fn upload(&self, bytes: &[u8]) -> Result { + let buffer = self.take(bytes.len())?; + if !bytes.is_empty() { + self.inner.context.inner.set_current()?; + // SAFETY: `buffer` is a live device allocation with at least + // `bytes.len()` bytes for this checkout, and `bytes` is valid for + // that many host bytes. + let result = unsafe { + (self.inner.context.inner.driver.cu_memcpy_htod)( + buffer.device_ptr(), + bytes.as_ptr().cast::(), + bytes.len(), + ) + }; + self.inner + .context + .inner + .driver + .check("cuMemcpyHtoD_v2", result)?; + } + Ok(buffer) + } + + /// Upload host bytes through temporary page-locked staging into a pooled device buffer. + pub fn upload_pinned(&self, bytes: &[u8]) -> Result { + if bytes.is_empty() { + return self.upload(bytes); + } + + let buffer = self.take(bytes.len())?; + let mut staging = self.inner.context.take_pinned_upload_staging(bytes.len())?; + staging.as_mut_slice()[..bytes.len()].copy_from_slice(bytes); + self.inner.context.inner.set_current()?; + // SAFETY: `buffer` is a live device allocation with at least + // `bytes.len()` bytes, and the pinned staging slice covers that range. + let upload_result = unsafe { + (self.inner.context.inner.driver.cu_memcpy_htod)( + buffer.device_ptr(), + staging.as_slice()[..bytes.len()].as_ptr().cast::(), + bytes.len(), + ) + }; + let upload_result = self + .inner + .context + .inner + .driver + .check("cuMemcpyHtoD_v2", upload_result); + let recycle_result = self.inner.context.recycle_pinned_upload_staging(staging); + match (upload_result, recycle_result) { + (Ok(()), Ok(())) => Ok(buffer), + (Err(error), _) | (_, Err(error)) => Err(error), + } + } + + /// Upload host `f32` samples into a pooled device buffer. + pub fn upload_f32(&self, samples: &[f32]) -> Result { + self.upload(f32_slice_as_bytes(samples)) + } + + /// Upload host `f32` samples through pinned staging into a pooled device buffer. + pub fn upload_f32_pinned(&self, samples: &[f32]) -> Result { + self.upload_pinned(f32_slice_as_bytes(samples)) + } + + /// Upload host `i16` samples into a pooled device buffer. + pub fn upload_i16(&self, samples: &[i16]) -> Result { + self.upload(i16_slice_as_bytes(samples)) + } + + /// Upload host `i16` samples through pinned staging into a pooled device buffer. + pub fn upload_i16_pinned(&self, samples: &[i16]) -> Result { + self.upload_pinned(i16_slice_as_bytes(samples)) + } + + /// Number of free buffers currently cached by the pool. + pub fn cached_count(&self) -> Result { + Ok(self + .inner + .free + .lock() + .map_err(|error| CudaError::StatePoisoned { + message: error.to_string(), + })? + .cached_count()) + } +} + +impl CudaBufferPoolFree { + fn cached_count(&self) -> usize { + match self { + Self::FirstFit(free) => free.len(), + Self::SizeBuckets(free) => free.values().map(Vec::len).sum(), + } + } +} + +pub(crate) fn pool_take_fit_buffer( + free: &mut CudaBufferPoolFree, + len: usize, +) -> (Option, usize) { + match free { + CudaBufferPoolFree::FirstFit(free) => pool_take_first_fit_buffer(free, len), + CudaBufferPoolFree::SizeBuckets(free) => pool_take_size_bucket_buffer(free, len), + } +} + +pub(crate) fn pool_take_first_fit_buffer( + free: &mut Vec, + len: usize, +) -> (Option, usize) { + let mut examined = 0usize; + for (index, buffer) in free.iter().enumerate() { + examined = examined.saturating_add(1); + if buffer.byte_len() >= len { + return (Some(free.swap_remove(index)), examined); + } + } + (None, examined) +} + +pub(crate) fn pool_take_size_bucket_buffer( + free: &mut BTreeMap>, + len: usize, +) -> (Option, usize) { + let Some(size) = free.range(len..).next().map(|(size, _)| *size) else { + return (None, usize::from(!free.is_empty())); + }; + let buffer = free + .get_mut(&size) + .expect("selected CUDA buffer pool size bucket must exist") + .pop(); + if free.get(&size).is_some_and(Vec::is_empty) { + free.remove(&size); + } + (buffer, 1) +} + +#[cfg(test)] +pub(crate) fn pool_fit_buffer_index_by_len(lengths: I, len: usize) -> Option +where + I: IntoIterator, +{ + let lengths = lengths.into_iter().collect::>(); + let mut left = 0usize; + let mut right = lengths.len(); + while left < right { + let mid = left + (right - left) / 2; + if lengths[mid].1 < len { + left = mid + 1; + } else { + right = mid; + } + } + (left < lengths.len()).then_some(lengths[left].0) +} + +/// Device buffer borrowed from a [`CudaBufferPool`]. +#[derive(Debug)] +pub struct CudaPooledDeviceBuffer { + pub(crate) buffer: Option, + pub(crate) requested_len: usize, + pub(crate) pool: Arc, +} + +impl CudaPooledDeviceBuffer { + /// Raw CUDA device pointer value for kernel argument binding. + pub fn device_ptr(&self) -> u64 { + self.buffer.as_ref().map_or(0, CudaDeviceBuffer::device_ptr) + } + + /// Requested byte length for the current checkout. + pub fn byte_len(&self) -> usize { + self.requested_len + } + + /// Actual device allocation byte length. + pub fn allocation_byte_len(&self) -> usize { + self.buffer.as_ref().map_or(0, CudaDeviceBuffer::byte_len) + } + + /// Borrow the underlying device buffer while the checkout is live. + pub fn as_device_buffer(&self) -> Option<&CudaDeviceBuffer> { + self.buffer.as_ref() + } + + /// Detach and return the underlying device buffer instead of recycling it + /// when this checkout is dropped. + pub fn into_device_buffer(mut self) -> Result { + self.buffer + .take() + .ok_or_else(|| CudaError::InvalidArgument { + message: "pooled CUDA buffer checkout is empty".to_string(), + }) + } + + /// Copy the requested bytes for this checkout into caller-owned host output. + pub fn copy_to_host(&self, out: &mut [u8]) -> Result<(), CudaError> { + if out.len() < self.requested_len { + return Err(CudaError::OutputTooSmall { + required: self.requested_len, + have: out.len(), + }); + } + if self.requested_len == 0 { + return Ok(()); + } + let buffer = self + .buffer + .as_ref() + .ok_or_else(|| CudaError::InvalidArgument { + message: "pooled CUDA buffer checkout is empty".to_string(), + })?; + buffer.context.inner.set_current()?; + // SAFETY: `buffer.ptr` is a live allocation with at least + // `requested_len` bytes for this checkout, and `out` was validated. + let result = unsafe { + (buffer.context.inner.driver.cu_memcpy_dtoh)( + out.as_mut_ptr().cast::(), + buffer.ptr, + self.requested_len, + ) + }; + buffer + .context + .inner + .driver + .check("cuMemcpyDtoH_v2", result)?; + Ok(()) + } +} + +impl Drop for CudaPooledDeviceBuffer { + fn drop(&mut self) { + if let Some(buffer) = self.buffer.take() { + let _ = self.pool.recycle_buffer(buffer); + } + } +} + +/// One byte range inside a contiguous CUDA batch output allocation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CudaDeviceBufferRange { + /// Byte offset from the start of the contiguous allocation. + pub offset: usize, + /// Byte length for this output item. + pub len: usize, +} + +pub(crate) fn pooled_device_buffer( + buffer: &CudaPooledDeviceBuffer, +) -> Result<&CudaDeviceBuffer, CudaError> { + buffer + .as_device_buffer() + .ok_or_else(|| CudaError::InvalidArgument { + message: "pooled CUDA buffer checkout is empty".to_string(), + }) +} + +pub(crate) fn copy_pooled_bytes_to_vec_uninit( + buffer: &CudaPooledDeviceBuffer, + byte_len: usize, +) -> Result, CudaError> { + let mut out = Vec::with_capacity(byte_len); + pooled_device_buffer(buffer)?.copy_range_to_host_uninit(0, out.spare_capacity_mut())?; + // SAFETY: copy_range_to_host_uninit returned success after writing exactly + // byte_len initialized bytes into the Vec spare capacity. + unsafe { + out.set_len(byte_len); + } + Ok(out) +} + +impl CudaDeviceBuffer { + /// CUDA context that owns this allocation. + pub fn context(&self) -> CudaContext { + self.context.clone() + } + + /// Raw CUDA device pointer value. + pub fn device_ptr(&self) -> u64 { + self.ptr + } + + /// Device allocation length in bytes. + pub fn byte_len(&self) -> usize { + self.len + } + + /// Borrow this allocation as a typed immutable device view. + pub fn typed_view(&self) -> Result, CudaError> { + let element_size = std::mem::size_of::(); + if element_size == 0 || !self.len.is_multiple_of(element_size) { + return Err(CudaError::LengthNotElementAligned { + bytes: self.len, + element_size, + }); + } + Ok(CudaDeviceBufferView { + ptr: self.ptr, + len: self.len / element_size, + _marker: std::marker::PhantomData, + }) + } + + /// Borrow this allocation as a typed mutable device view. + pub fn typed_view_mut(&mut self) -> Result, CudaError> { + let element_size = std::mem::size_of::(); + if element_size == 0 || !self.len.is_multiple_of(element_size) { + return Err(CudaError::LengthNotElementAligned { + bytes: self.len, + element_size, + }); + } + Ok(CudaDeviceBufferViewMut { + ptr: self.ptr, + len: self.len / element_size, + _marker: std::marker::PhantomData, + }) + } + + /// Copy device bytes into caller-owned host output. + pub fn copy_to_host(&self, out: &mut [u8]) -> Result<(), CudaError> { + if out.len() < self.len { + return Err(CudaError::OutputTooSmall { + required: self.len, + have: out.len(), + }); + } + if self.len == 0 { + return Ok(()); + } + + self.context.inner.set_current()?; + // SAFETY: ptr is a live device allocation of self.len bytes, and out is + // valid for at least self.len bytes. + self.context.inner.driver.check("cuMemcpyDtoH_v2", unsafe { + (self.context.inner.driver.cu_memcpy_dtoh)( + out.as_mut_ptr().cast::(), + self.ptr, + self.len, + ) + }) + } + + /// Copy a byte range from this device buffer into caller-owned host output. + pub fn copy_range_to_host(&self, offset: usize, out: &mut [u8]) -> Result<(), CudaError> { + let end = offset + .checked_add(out.len()) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if end > self.len { + return Err(CudaError::OutputTooSmall { + required: end, + have: self.len, + }); + } + if out.is_empty() { + return Ok(()); + } + + self.context.inner.set_current()?; + let source = self + .ptr + .checked_add( + u64::try_from(offset).map_err(|_| CudaError::LengthTooLarge { len: offset })?, + ) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + // SAFETY: `source` is inside this live device allocation, and `out` + // is valid for the requested range length after the bounds check above. + self.context.inner.driver.check("cuMemcpyDtoH_v2", unsafe { + (self.context.inner.driver.cu_memcpy_dtoh)( + out.as_mut_ptr().cast::(), + source, + out.len(), + ) + }) + } + + /// Copy a byte range from this device buffer into uninitialized host output. + pub fn copy_range_to_host_uninit( + &self, + offset: usize, + out: &mut [std::mem::MaybeUninit], + ) -> Result<(), CudaError> { + let end = offset + .checked_add(out.len()) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + if end > self.len { + return Err(CudaError::OutputTooSmall { + required: end, + have: self.len, + }); + } + if out.is_empty() { + return Ok(()); + } + + self.context.inner.set_current()?; + let source = self + .ptr + .checked_add( + u64::try_from(offset).map_err(|_| CudaError::LengthTooLarge { len: offset })?, + ) + .ok_or(CudaError::LengthTooLarge { len: usize::MAX })?; + // SAFETY: `source` is inside this live device allocation, and `out` + // points at writable spare capacity for exactly the requested byte + // count. The caller decides when those bytes become initialized. + self.context.inner.driver.check("cuMemcpyDtoH_v2", unsafe { + (self.context.inner.driver.cu_memcpy_dtoh)( + out.as_mut_ptr().cast::(), + source, + out.len(), + ) + }) + } +} + +impl Drop for CudaDeviceBuffer { + fn drop(&mut self) { + if self.ptr != 0 { + let _ = self.context.inner.set_current(); + // SAFETY: ptr was allocated by this CUDA context. Drop cannot + // surface errors, so failures are ignored during cleanup. + let _ = unsafe { (self.context.inner.driver.cu_mem_free)(self.ptr) }; + } + } +} + +pub(crate) fn checked_image_words( + width: u32, + height: u32, + channels: usize, +) -> Result { + width + .try_into() + .ok() + .and_then(|width: usize| width.checked_mul(height as usize)) + .and_then(|pixels| pixels.checked_mul(channels)) + .ok_or(CudaError::ImageTooLarge { + width, + height, + channels, + }) +} diff --git a/crates/j2k-cuda-runtime/src/tests.rs b/crates/j2k-cuda-runtime/src/tests.rs new file mode 100644 index 00000000..5e4024cc --- /dev/null +++ b/crates/j2k-cuda-runtime/src/tests.rs @@ -0,0 +1,4073 @@ +use super::{ + checked_f32_words_byte_len, f32_slice_as_bytes_mut, format_idwt_batch_trace_row, + idwt_batch_kernel_mode, idwt_batch_trace_row, idwt_batch_uses_cooperative_53, + jpeg_entropy_overflow_count, pool_fit_buffer_index_by_len, validate_dct_block_grid, + CudaContext, CudaError, CudaExecutionStats, CudaHtj2kCleanupMultiKernelJob, + CudaHtj2kCleanupTarget, CudaHtj2kCodeBlockJob, CudaHtj2kDecodeTables, + CudaHtj2kDequantizeTarget, CudaHtj2kEncodeCodeBlockJob, CudaHtj2kEncodeCodeBlockRegionJob, + CudaHtj2kEncodeResidentTarget, CudaHtj2kEncodeTables, CudaJ2kIdwtBatchKernelMode, + CudaJ2kIdwtJob, CudaJ2kIdwtMultiKernelJob, CudaJ2kIdwtTarget, CudaJ2kQuantizeJob, + CudaJ2kQuantizeSubbandRegionJob, CudaJ2kRect, CudaJpegChunkedEntropyConfig, + CudaJpegChunkedEntropyPlan, CudaJpegChunkedEntropyReport, CudaJpegEntropyOverflowState, + CudaJpegEntropySyncState, CudaJpegHuffmanTable, CudaKernelName, CudaQueuedHtj2kCleanup, +}; + +fn cuda_runtime_required() -> bool { + std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_some() +} + +#[test] +fn jpeg_chunked_entropy_config_counts_bit_subsequences() { + let config = CudaJpegChunkedEntropyConfig { + subsequence_words: 4, + sequence_len: 8, + max_overflow_subsequences: 2, + }; + + assert_eq!(config.subsequence_bits(), 128); + assert_eq!(config.subsequence_count_for_entropy_bytes(0).unwrap(), 0); + assert_eq!(config.subsequence_count_for_entropy_bytes(1).unwrap(), 1); + assert_eq!(config.subsequence_count_for_entropy_bytes(16).unwrap(), 1); + assert_eq!(config.subsequence_count_for_entropy_bytes(17).unwrap(), 2); +} + +#[test] +fn checked_f32_words_byte_len_rejects_multiplication_overflow() { + assert_eq!(checked_f32_words_byte_len(2).expect("byte len"), 8); + assert!(matches!( + checked_f32_words_byte_len(usize::MAX), + Err(CudaError::LengthTooLarge { len }) if len == usize::MAX + )); +} + +#[test] +fn validate_dct_block_grid_checks_shape_and_coefficient_count() { + let grid = validate_dct_block_grid(2, 1, 15, 8, 3, 384, "invalid").expect("valid grid"); + + assert_eq!(grid.block_count, 2); + assert_eq!(grid.expected_coeffs, 384); + assert_eq!((grid.low_width, grid.high_width), (8, 7)); + assert_eq!((grid.low_height, grid.high_height), (4, 4)); + assert!(matches!( + validate_dct_block_grid(2, 1, 15, 8, 3, 383, "invalid"), + Err(CudaError::InvalidArgument { .. }) + )); + assert!(matches!( + validate_dct_block_grid(2, 1, 15, 8, 0, 0, "invalid"), + Err(CudaError::InvalidArgument { .. }) + )); + assert!(matches!( + validate_dct_block_grid(usize::MAX, 2, 1, 1, 1, 64, "invalid"), + Err(CudaError::LengthTooLarge { .. }) + )); +} + +#[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] +#[test] +fn cuda_oxide_reversible53_transcode_matches_scalar_fixture_when_required() { + if !cuda_runtime_required() + || std::env::var_os("J2K_CUDA_USE_OXIDE_TRANSCODE").is_none() + || !super::transcode_kernels_built() + { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let mut blocks = [0i16; 64]; + for (index, value) in [ + (0, 80), + (1, -24), + (2, 13), + (3, 5), + (5, -3), + (8, 31), + (9, -11), + (10, 7), + (16, -9), + (17, 4), + (18, 3), + (27, -5), + (36, 6), + (45, -4), + (54, 2), + (63, -1), + ] { + blocks[index] = value; + } + + let bands = context + .j2k_transcode_reversible_dwt53(&blocks, 1, 1, 8, 8) + .expect("cuda-oxide reversible 5/3 transcode"); + + assert_eq!((bands.low_width, bands.low_height), (4, 4)); + assert_eq!((bands.high_width, bands.high_height), (4, 4)); + assert_eq!( + bands.ll.as_slice(), + &[14, 8, 12, 22, 13, 7, 14, 22, 8, 7, 12, 15, 6, 3, 5, 7] + ); + assert_eq!( + bands.hl.as_slice(), + &[2, -1, -1, 5, 1, -4, 2, 0, -1, 1, 0, 3, 3, -3, 2, 0] + ); + assert_eq!( + bands.lh.as_slice(), + &[2, 1, -1, 2, 2, -1, 3, 0, -1, 3, -1, 1, 1, -4, -1, -2] + ); + assert_eq!( + bands.hh.as_slice(), + &[1, 2, -1, -4, 1, -1, 0, 1, -1, -1, 1, -2, -5, 2, -1, -1] + ); +} + +#[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] +#[test] +fn cuda_oxide_dwt97_transcode_matches_scalar_fixture_when_required() { + if !cuda_runtime_required() + || std::env::var_os("J2K_CUDA_USE_OXIDE_TRANSCODE").is_none() + || !super::transcode_kernels_built() + { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let mut blocks = [0.0f32; 64]; + for (index, value) in [ + (0, 80.0), + (1, -24.0), + (2, 13.0), + (3, 5.0), + (5, -3.0), + (8, 31.0), + (9, -11.0), + (10, 7.0), + (16, -9.0), + (17, 4.0), + (18, 3.0), + (27, -5.0), + (36, 6.0), + (45, -4.0), + (54, 2.0), + (63, -1.0), + ] { + blocks[index] = value; + } + + let bands = context + .j2k_transcode_dwt97(&blocks, 1, 1, 8, 8) + .expect("cuda-oxide 9/7 transcode"); + + assert_eq!((bands.low_width, bands.low_height), (4, 4)); + assert_eq!((bands.high_width, bands.high_height), (4, 4)); + assert_f32_slice_close( + &bands.ll, + &[ + 12.144_072, 8.567_899, 11.216_426, 20.388_594, 11.476_019, 7.618_125, 12.952_319, + 19.958_328, 7.468_019, 6.779_34, 10.701_953, 14.315_73, 4.983_001, 3.069_523, + 4.546_064, 6.695_241, + ], + 0.02, + ); + assert_f32_slice_close( + &bands.hl, + &[ + 0.579_117, -0.765_21, -1.113_766, 3.008_691, 1.415_966, -2.878_618, 2.173_036, + -0.629_188, -0.239_748, 0.239_237, -0.885_278, 2.500_556, 1.929_175, -2.255_519, + 1.123_41, 0.191_912, + ], + 0.02, + ); + assert_f32_slice_close( + &bands.lh, + &[ + -0.314_113, 0.534_82, -1.107_942, 1.062_559, 0.976_02, -1.180_377, 1.861_77, + -0.696_248, -1.241_956, 2.006_542, -1.112_403, 0.853_18, 0.104_077, -3.326_791, + 0.079_872, -2.094_714, + ], + 0.02, + ); + assert_f32_slice_close( + &bands.hh, + &[ + -0.434_17, 1.497_277, -0.967_611, -6.657_543, 1.496_545, -1.963_292, -2.252_154, + 3.941_389, -0.968_106, -2.252_748, 1.867_451, -1.252_69, -6.656_182, 3.949_171, + -1.248_663, 0.544_539, + ], + 0.02, + ); +} + +#[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] +#[test] +#[allow(clippy::too_many_lines)] +fn cuda_oxide_dwt97_batch_and_quantize_paths_match_reference_when_required() { + if !cuda_runtime_required() + || std::env::var_os("J2K_CUDA_USE_OXIDE_TRANSCODE").is_none() + || !super::transcode_kernels_built() + { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let first = dwt97_fixture_blocks(1.0); + let second = dwt97_fixture_blocks(-1.0); + let mut blocks = Vec::with_capacity(128); + blocks.extend_from_slice(&first); + blocks.extend_from_slice(&second); + + let expected_first = context + .j2k_transcode_dwt97(&first, 1, 1, 8, 8) + .expect("single first DWT97"); + let expected_second = context + .j2k_transcode_dwt97(&second, 1, 1, 8, 8) + .expect("single second DWT97"); + let (batch, _) = context + .j2k_transcode_dwt97_batch_with_pool(&blocks, 2, 1, 1, 8, 8, &pool) + .expect("cuda-oxide DWT97 batch"); + assert_eq!(batch.len(), 2); + assert_dwt97_bands_close(&batch[0], &expected_first, 0.02); + assert_dwt97_bands_close(&batch[1], &expected_second, 0.02); + + let wide_block_cols = 129; + let wide_width = 1032; + let wide_height = 8; + let mut wide_blocks = vec![0.0f32; wide_block_cols * 64]; + const WIDE_PATTERN: [f32; 17] = [ + -2.0, -1.75, -1.25, -0.5, 0.0, 0.25, 0.75, 1.0, 1.5, 2.0, -2.5, 2.5, -3.0, 3.0, -0.25, 0.5, + 1.25, + ]; + for (index, value) in wide_blocks.iter_mut().enumerate() { + *value = WIDE_PATTERN[index % WIDE_PATTERN.len()]; + } + let wide_expected = context + .j2k_transcode_dwt97(&wide_blocks, wide_block_cols, 1, wide_width, wide_height) + .expect("single wide DWT97"); + let (wide_batch, _) = context + .j2k_transcode_dwt97_batch_with_pool( + &wide_blocks, + 1, + wide_block_cols, + 1, + wide_width, + wide_height, + &pool, + ) + .expect("wide cuda-oxide DWT97 batch"); + assert_eq!(wide_batch.len(), 1); + assert_dwt97_bands_close(&wide_batch[0], &wide_expected, 0.02); + + let params = super::CudaHtj2k97QuantizeParams { + inv_delta_ll: 1.0, + inv_delta_hl: 1.25, + inv_delta_lh: 0.75, + inv_delta_hh: 2.0, + cb_width: 64, + cb_height: 64, + }; + let expected_codeblocks = expected_dwt97_codeblocks(&batch, params); + let (quantized, _) = context + .j2k_transcode_htj2k97_codeblock_batch_with_pool(&blocks, 2, 1, 1, 8, 8, params, &pool) + .expect("cuda-oxide staged DWT97 quantize batch"); + assert_eq!(quantized, expected_codeblocks); + + let first_i16 = dwt97_fixture_i16_blocks(1); + let second_i16 = dwt97_fixture_i16_blocks(-1); + let mut i16_blocks = Vec::with_capacity(128); + i16_blocks.extend_from_slice(&first_i16); + i16_blocks.extend_from_slice(&second_i16); + let (fused, _) = context + .j2k_transcode_htj2k97_codeblock_i16_batch_resident_with_pool( + &i16_blocks, + 2, + 1, + 1, + 8, + 8, + params, + &pool, + ) + .expect("cuda-oxide fused i16 DWT97 quantize batch"); + assert_eq!(download_device_codeblock_bands(&fused), expected_codeblocks); +} + +#[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] +fn assert_f32_slice_close(actual: &[f32], expected: &[f32], tolerance: f32) { + assert_eq!(actual.len(), expected.len()); + for (index, (&actual, &expected)) in actual.iter().zip(expected).enumerate() { + assert!( + (actual - expected).abs() <= tolerance, + "index {index}: actual={actual}, expected={expected}, tolerance={tolerance}" + ); + } +} + +#[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] +fn assert_dwt97_bands_close( + actual: &super::CudaTranscodeDwt97Bands, + expected: &super::CudaTranscodeDwt97Bands, + tolerance: f32, +) { + assert_eq!( + ( + actual.low_width, + actual.low_height, + actual.high_width, + actual.high_height, + ), + ( + expected.low_width, + expected.low_height, + expected.high_width, + expected.high_height, + ) + ); + assert_f32_slice_close(&actual.ll, &expected.ll, tolerance); + assert_f32_slice_close(&actual.hl, &expected.hl, tolerance); + assert_f32_slice_close(&actual.lh, &expected.lh, tolerance); + assert_f32_slice_close(&actual.hh, &expected.hh, tolerance); +} + +#[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] +#[allow(clippy::cast_precision_loss)] +fn dwt97_fixture_blocks(scale: f32) -> [f32; 64] { + let mut blocks = [0.0f32; 64]; + for (index, value) in DWT97_FIXTURE_VALUES { + blocks[index] = f32::from(value) * scale; + } + blocks +} + +#[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] +fn dwt97_fixture_i16_blocks(scale: i16) -> [i16; 64] { + let mut blocks = [0i16; 64]; + for (index, value) in DWT97_FIXTURE_VALUES { + blocks[index] = value * scale; + } + blocks +} + +#[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] +const DWT97_FIXTURE_VALUES: [(usize, i16); 16] = [ + (0, 80), + (1, -24), + (2, 13), + (3, 5), + (5, -3), + (8, 31), + (9, -11), + (10, 7), + (16, -9), + (17, 4), + (18, 3), + (27, -5), + (36, 6), + (45, -4), + (54, 2), + (63, -1), +]; + +#[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] +fn expected_dwt97_codeblocks( + batch: &[super::CudaTranscodeDwt97Bands], + params: super::CudaHtj2k97QuantizeParams, +) -> super::CudaHtj2k97CodeblockBands { + let first = batch.first().expect("non-empty DWT97 batch"); + let mut ll = Vec::new(); + let mut hl = Vec::new(); + let mut lh = Vec::new(); + let mut hh = Vec::new(); + for bands in batch { + ll.extend( + bands + .ll + .iter() + .map(|&value| quantize_dwt97_deadzone(value, params.inv_delta_ll)), + ); + hl.extend( + bands + .hl + .iter() + .map(|&value| quantize_dwt97_deadzone(value, params.inv_delta_hl)), + ); + lh.extend( + bands + .lh + .iter() + .map(|&value| quantize_dwt97_deadzone(value, params.inv_delta_lh)), + ); + hh.extend( + bands + .hh + .iter() + .map(|&value| quantize_dwt97_deadzone(value, params.inv_delta_hh)), + ); + } + super::CudaHtj2k97CodeblockBands { + ll, + hl, + lh, + hh, + item_count: batch.len(), + low_width: first.low_width, + low_height: first.low_height, + high_width: first.high_width, + high_height: first.high_height, + } +} + +#[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] +fn quantize_dwt97_deadzone(value: f32, inv_delta: f32) -> i32 { + let sign = if value < 0.0 { -1 } else { 1 }; + sign * (value.abs() * inv_delta).floor() as i32 +} + +#[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] +fn download_device_codeblock_bands( + bands: &super::CudaHtj2k97DeviceCodeblockBands, +) -> super::CudaHtj2k97CodeblockBands { + let ll_len = bands.item_count * bands.low_width * bands.low_height; + let hl_len = bands.item_count * bands.high_width * bands.low_height; + let lh_len = bands.item_count * bands.low_width * bands.high_height; + let hh_len = bands.item_count * bands.high_width * bands.high_height; + super::CudaHtj2k97CodeblockBands { + ll: download_pooled_i32(&bands.ll, ll_len), + hl: download_pooled_i32(&bands.hl, hl_len), + lh: download_pooled_i32(&bands.lh, lh_len), + hh: download_pooled_i32(&bands.hh, hh_len), + item_count: bands.item_count, + low_width: bands.low_width, + low_height: bands.low_height, + high_width: bands.high_width, + high_height: bands.high_height, + } +} + +#[cfg(all(feature = "cuda-oxide-transcode", j2k_cuda_oxide_transcode_built))] +fn download_pooled_i32(buffer: &super::CudaPooledDeviceBuffer, len: usize) -> Vec { + let mut output = vec![0i32; len]; + buffer + .copy_to_host(super::i32_slice_as_bytes_mut(&mut output)) + .expect("download pooled i32 buffer"); + output +} + +#[test] +fn jpeg_chunked_entropy_report_has_one_less_overflow_than_subsequence_count() { + let config = CudaJpegChunkedEntropyConfig { + subsequence_words: 1, + sequence_len: 8, + max_overflow_subsequences: 2, + }; + let subsequences = config.subsequence_count_for_entropy_bytes(16).unwrap(); + + assert_eq!(subsequences, 4); + assert_eq!(jpeg_entropy_overflow_count(subsequences), 3); + assert_eq!(jpeg_entropy_overflow_count(0), 0); +} + +#[test] +fn jpeg_chunked_entropy_config_rejects_zero_subsequence_or_sequence() { + let zero_words = CudaJpegChunkedEntropyConfig { + subsequence_words: 0, + ..CudaJpegChunkedEntropyConfig::default() + }; + let zero_sequence = CudaJpegChunkedEntropyConfig { + sequence_len: 0, + ..CudaJpegChunkedEntropyConfig::default() + }; + + assert!(zero_words.validate().is_err()); + assert!(zero_sequence.validate().is_err()); +} + +#[test] +fn jpeg_chunked_entropy_config_rejects_subsequence_bit_overflow() { + let config = CudaJpegChunkedEntropyConfig { + subsequence_words: (u32::MAX / 32) + 1, + ..CudaJpegChunkedEntropyConfig::default() + }; + + assert!(config.validate().is_err()); + assert!(config.subsequence_count_for_entropy_bytes(1).is_err()); +} + +#[test] +fn jpeg_chunked_entropy_report_summarizes_sync_quality() { + let report = CudaJpegChunkedEntropyReport { + config: CudaJpegChunkedEntropyConfig { + subsequence_words: 4, + sequence_len: 8, + max_overflow_subsequences: 2, + }, + entropy_bytes: 4096, + states: vec![ + CudaJpegEntropySyncState { + code: 0, + start_bit: 0, + end_bit: 128, + bit_pos: 128, + symbol_count: 10, + block_phase: 0, + zigzag_index: 0, + reserved: 0, + }, + CudaJpegEntropySyncState { + code: 0, + start_bit: 128, + end_bit: 256, + bit_pos: 256, + symbol_count: 9, + block_phase: 3, + zigzag_index: 12, + reserved: 0, + }, + ], + overflows: vec![CudaJpegEntropyOverflowState { + code: 0, + from_subsequence: 0, + to_subsequence: 1, + overflow_bits: 96, + synchronized: 1, + reserved: [0; 3], + }], + execution: CudaExecutionStats { + kernel_dispatches: 2, + copy_kernel_dispatches: 0, + decode_kernel_dispatches: 0, + hardware_decode: false, + }, + }; + + assert_eq!(report.subsequence_count(), 2); + assert_eq!(report.synchronized_overflow_count(), 1); + assert_eq!(report.max_overflow_bits(), Some(96)); + assert_eq!(report.failed_state_count(), 0); +} + +#[test] +fn jpeg_entropy_self_sync_returns_empty_report_for_empty_entropy_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let context = CudaContext::system_default().expect("cuda context"); + let plan = CudaJpegChunkedEntropyPlan { + config: CudaJpegChunkedEntropyConfig::default(), + entropy_bytes: &[], + y_dc_table: CudaJpegHuffmanTable::from_jpeg_bits_values([0; 16], 0, [0; 256]) + .expect("empty huffman table"), + y_ac_table: CudaJpegHuffmanTable::from_jpeg_bits_values([0; 16], 0, [0; 256]) + .expect("empty huffman table"), + cb_dc_table: CudaJpegHuffmanTable::from_jpeg_bits_values([0; 16], 0, [0; 256]) + .expect("empty huffman table"), + cb_ac_table: CudaJpegHuffmanTable::from_jpeg_bits_values([0; 16], 0, [0; 256]) + .expect("empty huffman table"), + cr_dc_table: CudaJpegHuffmanTable::from_jpeg_bits_values([0; 16], 0, [0; 256]) + .expect("empty huffman table"), + cr_ac_table: CudaJpegHuffmanTable::from_jpeg_bits_values([0; 16], 0, [0; 256]) + .expect("empty huffman table"), + }; + + let report = context + .diagnose_jpeg_420_entropy_self_sync(&plan) + .expect("empty diagnostic report"); + assert_eq!(report.subsequence_count(), 0); + assert_eq!(report.overflows.len(), 0); +} + +#[test] +#[allow(clippy::too_many_lines)] +fn runtime_raii_primitives_smoke_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let mut pinned = context.pinned_host_buffer(16).expect("pinned host buffer"); + pinned.as_mut_slice().copy_from_slice(&[7u8; 16]); + assert_eq!(pinned.as_slice(), &[7u8; 16]); + let pinned_upload = context + .upload_pinned(&[1u8, 2, 3, 4]) + .expect("pinned upload"); + let mut uploaded = [0u8; 4]; + pinned_upload + .copy_to_host(&mut uploaded) + .expect("download pinned upload"); + assert_eq!(uploaded, [1, 2, 3, 4]); + let pinned_float_upload = context + .upload_f32_pinned(&[1.25, -2.5]) + .expect("pinned f32 upload"); + let mut downloaded_float_values = [0.0f32; 2]; + pinned_float_upload + .copy_to_host(super::f32_slice_as_bytes_mut(&mut downloaded_float_values)) + .expect("download pinned f32 upload"); + assert!((downloaded_float_values[0] - 1.25).abs() < f32::EPSILON); + assert!((downloaded_float_values[1] + 2.5).abs() < f32::EPSILON); + let pinned_integer_upload = context + .upload_i32_pinned(&[7, -11]) + .expect("pinned i32 upload"); + let mut downloaded_integer_values = [0i32; 2]; + pinned_integer_upload + .copy_to_host(super::i32_slice_as_bytes_mut( + &mut downloaded_integer_values, + )) + .expect("download pinned i32 upload"); + assert_eq!(downloaded_integer_values, [7, -11]); + let ranged_upload = context + .upload(&[9u8, 8, 7, 6, 5, 4]) + .expect("range-copy upload"); + let mut range = [0u8; 3]; + ranged_upload + .copy_range_to_host(2, &mut range) + .expect("copy device range"); + assert_eq!(range, [7, 6, 5]); + let mut uninit_range = Vec::with_capacity(3); + ranged_upload + .copy_range_to_host_uninit(1, uninit_range.spare_capacity_mut()) + .expect("copy device range into spare capacity"); + // SAFETY: copy_range_to_host_uninit returned success after writing + // exactly three bytes into the Vec spare capacity. + unsafe { + uninit_range.set_len(3); + } + assert_eq!(uninit_range, [8, 7, 6]); + let pool = context.buffer_pool(); + let pooled_upload = pool.upload(&[3u8, 1, 4, 1]).expect("pooled upload"); + let pooled_output = super::copy_pooled_bytes_to_vec_uninit(&pooled_upload, 4) + .expect("copy pooled bytes into spare capacity"); + assert_eq!(pooled_output, [3, 1, 4, 1]); + + let module = context + .preload_kernel_module(CudaKernelName::CopyU8) + .expect("preload copy kernel"); + assert_eq!(module.entrypoint(), "j2k_copy_u8"); + + let stream = context.create_stream().expect("CUDA stream"); + let start = context.create_event().expect("start event"); + let end = context.create_event().expect("end event"); + start.record(&stream).expect("record start"); + end.record(&stream).expect("record end"); + end.synchronize().expect("synchronize event"); + let elapsed = super::CudaEvent::elapsed_time_us(&start, &end).expect("elapsed time"); + assert!(elapsed >= 0.0); + + let pool = context.buffer_pool(); + { + let buffer = pool.take(32).expect("pooled buffer"); + assert!(buffer.device_ptr() != 0); + assert_eq!(buffer.byte_len(), 32); + assert!(buffer.allocation_byte_len() >= 32); + } + let cached_count = pool.cached_count().expect("cached count"); + assert_eq!(cached_count, 1); + { + let buffer = pool.take(16).expect("reused pooled buffer"); + assert_eq!(buffer.byte_len(), 16); + assert!(buffer.allocation_byte_len() >= 32); + } + + let samples = [1.25f32, -2.5, 3.75, 4.5]; + { + let buffer = pool.upload_f32(&samples).expect("pooled f32 upload"); + assert_eq!( + buffer.byte_len(), + samples.len() * std::mem::size_of::() + ); + let mut downloaded = vec![0.0f32; samples.len()]; + buffer + .copy_to_host(f32_slice_as_bytes_mut(&mut downloaded)) + .expect("download pooled f32 upload"); + assert_eq!(downloaded, samples); + } + let i16_samples = [-12i16, 7, 19, -4]; + { + let buffer = pool + .upload_i16_pinned(&i16_samples) + .expect("pooled pinned i16 upload"); + assert_eq!( + buffer.byte_len(), + i16_samples.len() * std::mem::size_of::() + ); + let mut downloaded_bytes = vec![0u8; std::mem::size_of_val(&i16_samples)]; + buffer + .copy_to_host(&mut downloaded_bytes) + .expect("download pooled pinned i16 upload"); + let downloaded = downloaded_bytes + .chunks_exact(std::mem::size_of::()) + .map(|chunk| i16::from_ne_bytes([chunk[0], chunk[1]])) + .collect::>(); + assert_eq!(downloaded, i16_samples); + } + let cached_after_upload = pool.cached_count().expect("cached after upload"); + assert!(cached_after_upload >= cached_count); +} + +#[test] +fn pooled_i16_pinned_upload_is_size_gated() { + assert!(super::should_use_pinned_pooled_i16_upload(4 * 1024 * 1024)); + assert!(!super::should_use_pinned_pooled_i16_upload( + 4 * 1024 * 1024 + 1 + )); +} + +#[test] +fn pooled_buffer_selection_uses_smallest_sufficient_fit() { + let buffers = [(1usize, 32usize), (0, 64)]; + + assert_eq!( + pool_fit_buffer_index_by_len(buffers.iter().copied(), 16), + Some(1) + ); + let mut large_pool = (0..1024).map(|index| (index, 8usize)).collect::>(); + large_pool[1022] = (1022, 32); + large_pool[1023] = (1023, 64); + + assert_eq!( + pool_fit_buffer_index_by_len(large_pool.iter().copied(), 16), + Some(1022) + ); + let mut recent_fit_pool = (0..4096).map(|index| (index, 8usize)).collect::>(); + recent_fit_pool[4094] = (4094, 32); + recent_fit_pool[4095] = (4095, 64); + + assert_eq!( + pool_fit_buffer_index_by_len(recent_fit_pool.iter().copied(), 16), + Some(4094) + ); + let fallback_pool = (0..4096) + .map(|index| match index.cmp(&3000) { + std::cmp::Ordering::Less => (index, 8usize), + std::cmp::Ordering::Equal => (index, 32), + std::cmp::Ordering::Greater => (index, 64), + }) + .collect::>(); + + assert_eq!( + pool_fit_buffer_index_by_len(fallback_pool.iter().copied(), 16), + Some(3000) + ); +} + +#[test] +fn pooled_take_with_trace_reports_allocation_and_reuse_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let (fresh, fresh_trace) = pool.take_with_trace(32).expect("fresh traced take"); + + assert_eq!(fresh.byte_len(), 32); + assert_eq!(fresh_trace.requested_len, 32); + assert_eq!(fresh_trace.free_count_before, 0); + assert_eq!(fresh_trace.scanned_count, 0); + assert!(!fresh_trace.reused); + assert!(fresh_trace.allocation_byte_len >= 32); + drop(fresh); + + let (reused, reuse_trace) = pool.take_with_trace(16).expect("reused traced take"); + + assert_eq!(reused.byte_len(), 16); + assert_eq!(reuse_trace.requested_len, 16); + assert_eq!(reuse_trace.free_count_before, 1); + assert_eq!(reuse_trace.scanned_count, 1); + assert!(reuse_trace.reused); + assert!(reuse_trace.allocation_byte_len >= 32); +} + +#[test] +fn pooled_buffer_can_detach_and_recycle_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let raw = pool + .take(32) + .expect("pooled buffer") + .into_device_buffer() + .expect("detach pooled buffer"); + assert_eq!(pool.cached_count().expect("cached after detach"), 0); + + pool.recycle(raw).expect("explicit recycle"); + assert_eq!(pool.cached_count().expect("cached after recycle"), 1); + + let (_reused, trace) = pool.take_with_trace(16).expect("reused traced take"); + assert!(trace.reused); + assert!(trace.allocation_byte_len >= 32); +} + +#[test] +fn htj2k_encoded_codeblock_reports_segment_lengths_from_status() { + let encoded = super::CudaHtj2kEncodedCodeBlock { + data: vec![0u8; 10], + status: super::CudaHtj2kEncodeStatus { + code: super::HTJ2K_STATUS_OK, + detail: 0, + data_len: 10, + number_of_coding_passes: 3, + missing_bit_planes: 4, + reserved0: 7, + reserved1: 3, + reserved2: 0, + }, + execution: super::CudaExecutionStats::default(), + stage_timings: super::CudaHtj2kEncodeStageTimings::default(), + }; + + assert_eq!(encoded.cleanup_length(), 7); + assert_eq!(encoded.refinement_length(), 3); +} + +#[test] +fn htj2k_encode_compact_jobs_pack_actual_payloads() { + let capacity = u32::try_from(super::HTJ2K_ENCODE_OUTPUT_CAPACITY) + .expect("HTJ2K encode output capacity fits u32"); + let double_capacity = capacity + .checked_mul(2) + .expect("test output capacity fits u32"); + let kernel_jobs = [ + super::CudaHtj2kEncodeKernelJob { + coefficient_offset: 0, + coefficient_stride: 64, + width: 64, + height: 64, + total_bitplanes: 8, + output_offset: 0, + output_capacity: capacity, + target_coding_passes: 1, + }, + super::CudaHtj2kEncodeKernelJob { + coefficient_offset: 4096, + coefficient_stride: 64, + width: 64, + height: 64, + total_bitplanes: 8, + output_offset: capacity, + output_capacity: capacity, + target_coding_passes: 1, + }, + super::CudaHtj2kEncodeKernelJob { + coefficient_offset: 8192, + coefficient_stride: 64, + width: 64, + height: 64, + total_bitplanes: 8, + output_offset: double_capacity, + output_capacity: capacity, + target_coding_passes: 1, + }, + ]; + let statuses = [ + super::CudaHtj2kEncodeStatus { + code: super::HTJ2K_STATUS_OK, + data_len: 12, + reserved2: 0x8001_8002, + ..super::CudaHtj2kEncodeStatus::default() + }, + super::CudaHtj2kEncodeStatus { + code: super::HTJ2K_STATUS_OK, + data_len: 0, + ..super::CudaHtj2kEncodeStatus::default() + }, + super::CudaHtj2kEncodeStatus { + code: super::HTJ2K_STATUS_OK, + data_len: 7, + ..super::CudaHtj2kEncodeStatus::default() + }, + ]; + + let (compact_jobs, compact_len) = + super::htj2k_encode_compact_jobs(&statuses, &kernel_jobs).expect("valid compact jobs"); + + assert_eq!(compact_len, 19); + assert_eq!( + compact_jobs, + vec![ + super::CudaHtj2kEncodeCompactJob { + source_offset: 0, + compact_offset: 0, + data_len: 12, + reserved: 0x8001_8002, + }, + super::CudaHtj2kEncodeCompactJob { + source_offset: capacity, + compact_offset: 12, + data_len: 0, + reserved: 0, + }, + super::CudaHtj2kEncodeCompactJob { + source_offset: double_capacity, + compact_offset: 12, + data_len: 7, + reserved: 0, + }, + ] + ); +} + +#[cfg(all(feature = "cuda-oxide-j2k-encode", j2k_cuda_oxide_j2k_encode_built))] +#[test] +fn cuda_oxide_htj2k_compact_codeblocks_assembles_payload_when_required() { + if !cuda_runtime_required() || std::env::var_os("J2K_CUDA_USE_OXIDE_J2K_ENCODE").is_none() { + return; + } + + const J2K_HT_MEL_SIZE: usize = 192; + const J2K_HT_VLC_SIZE: usize = 3072 - J2K_HT_MEL_SIZE; + const J2K_HT_MS_SIZE: usize = ((16384 * 16) + 14) / 15; + const J2K_HT_MEL_OFFSET: usize = J2K_HT_MS_SIZE; + const J2K_HT_VLC_OFFSET: usize = J2K_HT_MS_SIZE + J2K_HT_MEL_SIZE; + const J2K_HT_COMPACT_ASSEMBLE_FLAG: u32 = 0x8000_0000; + + let context = CudaContext::system_default().expect("CUDA context"); + let source_offset = 3usize; + let plain_source_offset = source_offset + J2K_HT_VLC_OFFSET + J2K_HT_VLC_SIZE + 8; + let mut scratch = vec![0u8; plain_source_offset + 4]; + scratch[source_offset..source_offset + 3].copy_from_slice(&[10, 11, 12]); + scratch[source_offset + J2K_HT_MEL_OFFSET..source_offset + J2K_HT_MEL_OFFSET + 2] + .copy_from_slice(&[20, 21]); + let vlc_start = source_offset + J2K_HT_VLC_OFFSET + J2K_HT_VLC_SIZE - 3; + scratch[vlc_start..vlc_start + 3].copy_from_slice(&[30, 31, 32]); + scratch[plain_source_offset..plain_source_offset + 4].copy_from_slice(&[40, 41, 42, 43]); + let jobs = [ + super::CudaHtj2kEncodeCompactJob { + source_offset: u32::try_from(source_offset).expect("source offset fits"), + compact_offset: 0, + data_len: 8, + reserved: J2K_HT_COMPACT_ASSEMBLE_FLAG | 2 | (3 << 15), + }, + super::CudaHtj2kEncodeCompactJob { + source_offset: u32::try_from(plain_source_offset).expect("plain offset fits"), + compact_offset: 8, + data_len: 4, + reserved: 0, + }, + ]; + let expected = [10, 11, 12, 20, 21, 30, 0x15, 0, 40, 41, 42, 43]; + + let scratch_buffer = context.upload(&scratch).expect("scratch upload"); + let compact_buffer = context.allocate(expected.len()).expect("compact output"); + let jobs_buffer = context + .upload(super::bytes::htj2k_encode_compact_jobs_as_bytes(&jobs)) + .expect("compact job upload"); + + context + .launch_htj2k_compact_codeblocks(&scratch_buffer, &compact_buffer, &jobs_buffer, jobs.len()) + .expect("cuda-oxide compact codeblocks"); + let mut actual = vec![0u8; expected.len()]; + compact_buffer + .copy_to_host(&mut actual) + .expect("download compact output"); + + assert_eq!(actual, expected); +} + +#[test] +fn htj2k_encode_resources_feed_resident_region_encode_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let vlc_table0 = [0u16; 2048]; + let vlc_table1 = [0u16; 2048]; + let uvlc_table = vec![0u8; super::HTJ2K_UVLC_ENCODE_TABLE_BYTES]; + let resources = context + .upload_htj2k_encode_resources(CudaHtj2kEncodeTables { + vlc_table0: &vlc_table0, + vlc_table1: &vlc_table1, + uvlc_table: &uvlc_table, + }) + .expect("encode resources"); + let coefficients = context + .upload_i32_pinned(&[0, 0, 0, 0]) + .expect("resident coefficients"); + let jobs = [CudaHtj2kEncodeCodeBlockRegionJob { + coefficient_offset: 0, + coefficient_stride: 2, + width: 2, + height: 2, + total_bitplanes: 1, + target_coding_passes: 1, + }]; + + let encoded = context + .encode_htj2k_codeblock_regions_resident_with_resources(&coefficients, 4, &jobs, &resources) + .expect("resource-backed resident HTJ2K encode"); + + assert_eq!(encoded.execution().kernel_dispatches(), 1); + assert_eq!(encoded.code_blocks().len(), 1); +} + +#[test] +fn htj2k_encode_resident_region_reuses_pool_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let vlc_table0 = [0u16; 2048]; + let vlc_table1 = [0u16; 2048]; + let uvlc_table = vec![0u8; super::HTJ2K_UVLC_ENCODE_TABLE_BYTES]; + let resources = context + .upload_htj2k_encode_resources(CudaHtj2kEncodeTables { + vlc_table0: &vlc_table0, + vlc_table1: &vlc_table1, + uvlc_table: &uvlc_table, + }) + .expect("encode resources"); + let coefficients = context + .upload_i32_pinned(&[0, 0, 0, 0]) + .expect("resident coefficients"); + let jobs = [CudaHtj2kEncodeCodeBlockRegionJob { + coefficient_offset: 0, + coefficient_stride: 2, + width: 2, + height: 2, + total_bitplanes: 1, + target_coding_passes: 1, + }]; + + let encoded = context + .encode_htj2k_codeblock_regions_resident_with_resources_and_pool( + &coefficients, + 4, + &jobs, + &resources, + &pool, + ) + .expect("pooled resource-backed resident HTJ2K encode"); + + assert_eq!(encoded.execution().kernel_dispatches(), 1); + assert_eq!(encoded.code_blocks().len(), 1); + assert!(pool.cached_count().expect("cached pooled encode buffers") >= 3); +} + +#[test] +fn htj2k_encode_codeblocks_resident_reuses_pool_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let vlc_table0 = [0u16; 2048]; + let vlc_table1 = [0u16; 2048]; + let uvlc_table = vec![0u8; super::HTJ2K_UVLC_ENCODE_TABLE_BYTES]; + let resources = context + .upload_htj2k_encode_resources(CudaHtj2kEncodeTables { + vlc_table0: &vlc_table0, + vlc_table1: &vlc_table1, + uvlc_table: &uvlc_table, + }) + .expect("encode resources"); + let coefficients = context + .upload_i32_pinned(&[0, 0, 0, 0]) + .expect("resident coefficients"); + let jobs = [CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 2, + height: 2, + total_bitplanes: 1, + target_coding_passes: 1, + }]; + + let encoded = context + .encode_htj2k_codeblocks_resident_with_resources_and_pool( + &coefficients, + 4, + &jobs, + &resources, + &pool, + ) + .expect("pooled resource-backed resident HTJ2K codeblock encode"); + + assert_eq!(encoded.execution().kernel_dispatches(), 1); + assert_eq!(encoded.code_blocks().len(), 1); + assert!(pool.cached_count().expect("cached pooled encode buffers") >= 3); +} + +#[test] +fn htj2k_encode_multi_resident_inputs_match_separate_batches_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let vlc_table0 = [0u16; 2048]; + let vlc_table1 = [0u16; 2048]; + let uvlc_table = vec![0u8; super::HTJ2K_UVLC_ENCODE_TABLE_BYTES]; + let resources = context + .upload_htj2k_encode_resources(CudaHtj2kEncodeTables { + vlc_table0: &vlc_table0, + vlc_table1: &vlc_table1, + uvlc_table: &uvlc_table, + }) + .expect("encode resources"); + let first = context + .upload_i32_pinned(&[0, 0, 0, 0]) + .expect("first resident coefficients"); + let second = context + .upload_i32_pinned(&[0, 0]) + .expect("second resident coefficients"); + let first_jobs = [CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 2, + height: 2, + total_bitplanes: 1, + target_coding_passes: 1, + }]; + let second_jobs = [CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 2, + height: 1, + total_bitplanes: 1, + target_coding_passes: 1, + }]; + + let first_separate = context + .encode_htj2k_codeblocks_resident_with_resources_and_pool( + &first, + 4, + &first_jobs, + &resources, + &pool, + ) + .expect("first separate resident encode"); + let second_separate = context + .encode_htj2k_codeblocks_resident_with_resources_and_pool( + &second, + 2, + &second_jobs, + &resources, + &pool, + ) + .expect("second separate resident encode"); + + let combined = context + .encode_htj2k_codeblocks_multi_resident_with_resources_and_pool( + &[ + CudaHtj2kEncodeResidentTarget { + coefficients: &first, + coefficient_count: 4, + jobs: &first_jobs, + }, + CudaHtj2kEncodeResidentTarget { + coefficients: &second, + coefficient_count: 2, + jobs: &second_jobs, + }, + ], + &resources, + &pool, + ) + .expect("combined resident encode"); + + assert_eq!(combined.execution().kernel_dispatches(), 1); + assert_eq!(combined.code_blocks().len(), 2); + assert_eq!( + combined.code_blocks()[0].data(), + first_separate.code_blocks()[0].data() + ); + assert_eq!( + combined.code_blocks()[1].data(), + second_separate.code_blocks()[0].data() + ); + let timings = combined.stage_timings(); + assert_eq!( + timings.ht_encode_us, + timings + .ht_kernel_us + .saturating_add(timings.ht_status_readback_us) + .saturating_add(timings.ht_compact_us) + .saturating_add(timings.ht_output_readback_us) + ); + assert!(timings.ht_kernel_us > 0); + assert!(timings.ht_status_readback_us > 0); +} + +#[test] +fn htj2k97_resident_batch_returns_pooled_quantized_bands_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let blocks = vec![0.0f32; 64]; + let params = super::CudaHtj2k97QuantizeParams { + inv_delta_ll: 1.0, + inv_delta_hl: 1.0, + inv_delta_lh: 1.0, + inv_delta_hh: 1.0, + cb_width: 64, + cb_height: 64, + }; + + let (bands, _) = context + .j2k_transcode_htj2k97_codeblock_batch_resident_with_pool( + &blocks, 1, 1, 1, 8, 8, params, &pool, + ) + .expect("resident HTJ2K 9/7 codeblock batch"); + + assert!(bands.ll.as_device_buffer().is_some()); + assert!(bands.hl.as_device_buffer().is_some()); + assert!(bands.lh.as_device_buffer().is_some()); + assert!(bands.hh.as_device_buffer().is_some()); + let cached_while_bands_live = pool.cached_count().expect("cached buffers while live"); + + drop(bands); + + assert!(pool.cached_count().expect("cached buffers after drop") >= cached_while_bands_live + 4); +} + +#[test] +fn htj2k_encode_rejects_unsupported_refinement_pass_count_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let coefficients = [0, 2, -3, 1]; + let jobs = [CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 2, + height: 2, + total_bitplanes: 3, + target_coding_passes: 4, + }]; + + let error = context + .encode_htj2k_codeblocks( + &coefficients, + &jobs, + CudaHtj2kEncodeTables { + vlc_table0: &[0u16; 2048], + vlc_table1: &[0u16; 2048], + uvlc_table: &[0u8; super::HTJ2K_UVLC_ENCODE_TABLE_BYTES], + }, + ) + .expect_err("unsupported HTJ2K encode pass count is explicit"); + + match error { + CudaError::KernelStatus { + kernel, + code, + detail, + } => { + assert_eq!(kernel, "j2k_htj2k_encode_codeblocks"); + assert_eq!(code, super::HTJ2K_STATUS_UNSUPPORTED); + assert_eq!(detail, 5); + } + other => panic!("unexpected CUDA encode error: {other:?}"), + } +} + +#[test] +fn htj2k_encode_rejects_lossy_zero_sigprop_request_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let coefficients = [0, 2, -3, 4]; + let jobs = [CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 2, + height: 2, + total_bitplanes: 3, + target_coding_passes: 2, + }]; + + let error = context + .encode_htj2k_codeblocks( + &coefficients, + &jobs, + CudaHtj2kEncodeTables { + vlc_table0: &[0u16; 2048], + vlc_table1: &[0u16; 2048], + uvlc_table: &[0u8; super::HTJ2K_UVLC_ENCODE_TABLE_BYTES], + }, + ) + .expect_err("target-2 zero SigProp cannot silently drop low coefficient bits"); + + match error { + CudaError::KernelStatus { + kernel, + code, + detail, + } => { + assert_eq!(kernel, "j2k_htj2k_encode_codeblocks"); + assert_eq!(code, super::HTJ2K_STATUS_UNSUPPORTED); + assert_eq!(detail, 6); + } + other => panic!("unexpected CUDA encode error: {other:?}"), + } +} + +#[test] +fn htj2k_encode_rejects_unreachable_target_three_sigprop_coefficients_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let coefficients = [3, 0, 0, 0]; + let jobs = [CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 2, + height: 2, + total_bitplanes: 4, + target_coding_passes: 3, + }]; + + let error = context + .encode_htj2k_codeblocks( + &coefficients, + &jobs, + CudaHtj2kEncodeTables { + vlc_table0: &[0u16; 2048], + vlc_table1: &[0u16; 2048], + uvlc_table: &[0u8; super::HTJ2K_UVLC_ENCODE_TABLE_BYTES], + }, + ) + .expect_err("isolated target-3 SigProp coefficient is explicitly unsupported"); + + match error { + CudaError::KernelStatus { + kernel, + code, + detail, + } => { + assert_eq!(kernel, "j2k_htj2k_encode_codeblocks"); + assert_eq!(code, super::HTJ2K_STATUS_UNSUPPORTED); + assert_eq!(detail, 6); + } + other => panic!("unexpected CUDA encode error: {other:?}"), + } +} + +#[test] +fn htj2k_encode_resources_feed_single_codeblock_encode_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let vlc_table0 = [0u16; 2048]; + let vlc_table1 = [0u16; 2048]; + let uvlc_table = vec![0u8; super::HTJ2K_UVLC_ENCODE_TABLE_BYTES]; + let resources = context + .upload_htj2k_encode_resources(CudaHtj2kEncodeTables { + vlc_table0: &vlc_table0, + vlc_table1: &vlc_table1, + uvlc_table: &uvlc_table, + }) + .expect("encode resources"); + + let encoded = context + .encode_htj2k_codeblock_with_resources(&[0, 0, 0, 0], 2, 2, 1, &resources) + .expect("resource-backed single HTJ2K encode"); + + assert_eq!(encoded.execution().kernel_dispatches(), 1); + // An all-zero codeblock has no significant bitplanes, so the encoder emits zero + // coding passes (matching native ht_block_encode::encode_code_block). + assert_eq!(encoded.num_coding_passes(), 0); + assert_eq!(encoded.cleanup_length(), 0); + assert_eq!(encoded.data().len(), 0); + assert_eq!(encoded.refinement_length(), 0); +} + +#[test] +fn default_stream_timer_reports_elapsed_time_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let input = vec![17u8; 4096]; + let (output, elapsed_us) = context + .time_default_stream_us(|| context.copy_with_kernel(&input)) + .expect("timed CUDA copy kernel"); + + assert_eq!(output.execution().kernel_dispatches(), 1); + assert!(elapsed_us > 0); +} + +#[cfg(all(feature = "cuda-oxide-copy-u8", j2k_cuda_oxide_copy_u8_built))] +#[test] +fn cuda_oxide_copy_u8_matches_builtin_copy_and_cpu_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let input = (0..4099) + .map(|index| ((index * 31 + 17) % 251) as u8) + .collect::>(); + + let builtin = context + .copy_with_kernel(&input) + .expect("builtin CUDA copy kernel"); + let cuda_oxide = context + .copy_with_cuda_oxide_kernel(&input) + .expect("cuda-oxide CUDA copy kernel"); + + let mut builtin_bytes = vec![0u8; input.len()]; + builtin + .buffer() + .copy_to_host(&mut builtin_bytes) + .expect("download builtin CUDA copy"); + let mut cuda_oxide_bytes = vec![0u8; input.len()]; + cuda_oxide + .buffer() + .copy_to_host(&mut cuda_oxide_bytes) + .expect("download cuda-oxide CUDA copy"); + + assert_eq!(builtin.execution().kernel_dispatches(), 1); + assert_eq!(cuda_oxide.execution().kernel_dispatches(), 1); + assert_eq!(builtin_bytes, input); + assert_eq!(cuda_oxide_bytes, input); + assert_eq!(cuda_oxide_bytes, builtin_bytes); +} + +#[test] +fn named_default_stream_timer_is_available_for_profiling_ranges_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let input = vec![23u8; 4096]; + let (output, elapsed_us) = context + .time_default_stream_named_us("j2k.test.copy", || context.copy_with_kernel(&input)) + .expect("named timed CUDA copy kernel"); + + assert_eq!(output.execution().kernel_dispatches(), 1); + assert!(elapsed_us > 0); +} + +#[test] +fn typed_device_view_reports_element_count_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let mut aligned = context.allocate(16).expect("aligned buffer"); + let view = aligned.typed_view::().expect("typed immutable view"); + assert_eq!(view.len(), 4); + let mut_view = aligned.typed_view_mut::().expect("typed mutable view"); + assert_eq!(mut_view.len(), 2); + + let unaligned = context.allocate(3).expect("unaligned buffer"); + let error = unaligned + .typed_view::() + .expect_err("unaligned typed view"); + assert!(matches!( + error, + CudaError::LengthNotElementAligned { + bytes: 3, + element_size: 2 + } + )); +} + +#[test] +#[allow(clippy::too_many_lines)] +fn kernel_module_names_cover_htj2k_decode_and_encode_stages() { + let cases = [ + ( + CudaKernelName::Htj2kDecodeCodeblocks, + "j2k_htj2k_decode_codeblocks", + ), + ( + CudaKernelName::Htj2kDecodeCodeblocksMultiCleanupDequantize, + "j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize", + ), + ( + CudaKernelName::J2kDequantizeHtj2kCodeblocks, + "j2k_dequantize_htj2k_codeblocks", + ), + ( + CudaKernelName::J2kDequantizeHtj2kCodeblocksMulti, + "j2k_dequantize_htj2k_codeblocks_multi", + ), + ( + CudaKernelName::J2kDequantizeHtj2kCleanupJobsMulti, + "j2k_dequantize_htj2k_cleanup_jobs_multi", + ), + (CudaKernelName::J2kIdwtInterleave, "j2k_idwt_interleave"), + ( + CudaKernelName::J2kIdwtInterleaveHorizontal53Multi, + "j2k_idwt_interleave_horizontal_53_multi", + ), + ( + CudaKernelName::J2kIdwtInterleaveHorizontal97Multi, + "j2k_idwt_interleave_horizontal_97_multi", + ), + (CudaKernelName::J2kIdwtHorizontal, "j2k_idwt_horizontal"), + ( + CudaKernelName::J2kIdwtHorizontal53, + "j2k_idwt_horizontal_53", + ), + ( + CudaKernelName::J2kIdwtHorizontal97, + "j2k_idwt_horizontal_97", + ), + (CudaKernelName::J2kIdwtVertical, "j2k_idwt_vertical"), + ( + CudaKernelName::J2kIdwtVertical53Multi, + "j2k_idwt_vertical_53_multi", + ), + ( + CudaKernelName::J2kIdwtVertical97Multi, + "j2k_idwt_vertical_97_multi", + ), + ( + CudaKernelName::J2kIdwtVertical97MultiCols4, + "j2k_idwt_vertical_97_multi_cols4", + ), + (CudaKernelName::J2kIdwtVertical53, "j2k_idwt_vertical_53"), + (CudaKernelName::J2kIdwtVertical97, "j2k_idwt_vertical_97"), + ( + CudaKernelName::J2kInverseDwtSingle, + "j2k_inverse_dwt_single", + ), + (CudaKernelName::J2kInverseMct, "j2k_inverse_mct"), + (CudaKernelName::J2kStoreGray8, "j2k_store_gray8"), + (CudaKernelName::J2kStoreGray16, "j2k_store_gray16"), + (CudaKernelName::J2kStoreRgb8, "j2k_store_rgb8"), + (CudaKernelName::J2kStoreRgb8Mct, "j2k_store_rgb8_mct"), + ( + CudaKernelName::J2kStoreRgb8MctBatch, + "j2k_store_rgb8_mct_batch", + ), + (CudaKernelName::J2kStoreRgb16, "j2k_store_rgb16"), + (CudaKernelName::J2kStoreRgb16Mct, "j2k_store_rgb16_mct"), + ( + CudaKernelName::Htj2kEncodeCodeblock, + "j2k_htj2k_encode_codeblock", + ), + ( + CudaKernelName::Htj2kEncodeCodeblocks, + "j2k_htj2k_encode_codeblocks", + ), + ( + CudaKernelName::Htj2kEncodeCodeblocksMultiInput, + "j2k_htj2k_encode_codeblocks_multi_input", + ), + ( + CudaKernelName::Htj2kEncodeCodeblocksMultiInputCleanup, + "j2k_htj2k_encode_codeblocks_multi_input_cleanup", + ), + ( + CudaKernelName::Htj2kEncodeCodeblocksMultiInputCleanup64, + "j2k_htj2k_encode_codeblocks_multi_input_cleanup_64", + ), + ( + CudaKernelName::Htj2kCompactCodeblocks, + "j2k_htj2k_compact_codeblocks", + ), + ( + CudaKernelName::Htj2kPacketizeCleanup, + "j2k_htj2k_packetize_cleanup", + ), + ]; + + for (kernel, entrypoint) in cases { + assert_eq!(kernel.entrypoint(), entrypoint); + let raw_entrypoint = kernel.kernel().entrypoint(); + assert_eq!( + &raw_entrypoint[..raw_entrypoint.len() - 1], + entrypoint.as_bytes() + ); + assert_eq!(raw_entrypoint.last(), Some(&0)); + } +} + +#[test] +#[allow(clippy::similar_names)] +fn htj2k_empty_codeblock_decode_zero_fills_coefficients_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let first_vlc = [0u16; 1024]; + let later_vlc = [0u16; 1024]; + let first_uvlc = [0u16; 320]; + let later_uvlc = [0u16; 256]; + let output = context + .decode_htj2k_codeblocks( + &[], + &[], + CudaHtj2kDecodeTables { + vlc_table0: &first_vlc, + vlc_table1: &later_vlc, + uvlc_table0: &first_uvlc, + uvlc_table1: &later_uvlc, + }, + 8, + ) + .expect("empty HTJ2K decode"); + let mut actual = vec![f32::NAN; 8]; + output + .coefficients() + .copy_to_host(super::f32_slice_as_bytes_mut(&mut actual)) + .expect("download coefficients"); + + assert_eq!(actual, vec![0.0; 8]); + assert_eq!(output.execution().kernel_dispatches(), 0); +} + +#[test] +#[allow(clippy::similar_names)] +fn htj2k_empty_codeblock_decode_reuses_pool_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let first_vlc = [0u16; 1024]; + let later_vlc = [0u16; 1024]; + let first_uvlc = [0u16; 320]; + let later_uvlc = [0u16; 256]; + let tables = context + .upload_htj2k_decode_table_resources(CudaHtj2kDecodeTables { + vlc_table0: &first_vlc, + vlc_table1: &later_vlc, + uvlc_table0: &first_uvlc, + uvlc_table1: &later_uvlc, + }) + .expect("decode tables"); + let resources = context + .upload_htj2k_decode_resources_with_tables(&[], &tables) + .expect("decode resources"); + + let output = context + .decode_htj2k_codeblocks_with_resources_and_pool(&resources, &[], 8, &pool) + .expect("pooled empty HTJ2K decode"); + let mut actual = vec![f32::NAN; 8]; + output + .coefficients() + .expect("pooled coefficients") + .copy_to_host(super::f32_slice_as_bytes_mut(&mut actual)) + .expect("download coefficients"); + + assert_eq!(actual, vec![0.0; 8]); + assert_eq!(output.execution().kernel_dispatches(), 0); + let cached_while_live = pool.cached_count().expect("cached while live"); + + drop(output); + + assert!(pool.cached_count().expect("cached after drop") > cached_while_live); +} + +#[test] +#[allow(clippy::similar_names)] +fn htj2k_decode_table_resources_feed_multiple_payload_uploads_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let first_vlc = [0u16; 1024]; + let later_vlc = [0u16; 1024]; + let first_uvlc = [0u16; 320]; + let later_uvlc = [0u16; 256]; + let tables = context + .upload_htj2k_decode_table_resources(CudaHtj2kDecodeTables { + vlc_table0: &first_vlc, + vlc_table1: &later_vlc, + uvlc_table0: &first_uvlc, + uvlc_table1: &later_uvlc, + }) + .expect("decode table resources"); + + let first_resources = context + .upload_htj2k_decode_resources_with_tables(&[0xAA, 0x55], &tables) + .expect("first payload resources"); + let second_resources = context + .upload_htj2k_decode_resources_with_tables(&[0x11, 0x22, 0x33], &tables) + .expect("second payload resources"); + + assert!(std::sync::Arc::ptr_eq( + &first_resources.tables.inner, + &second_resources.tables.inner + )); + assert_eq!(first_resources.payload_len, 2); + assert_eq!(second_resources.payload_len, 3); +} + +#[test] +fn j2k_inverse_dwt_single_dispatches_parallel_stages_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let ll = context + .upload(super::f32_slice_as_bytes(&[10.0])) + .expect("upload LL"); + let hl = context + .upload(super::f32_slice_as_bytes(&[2.0])) + .expect("upload HL"); + let lh = context + .upload(super::f32_slice_as_bytes(&[4.0])) + .expect("upload LH"); + let hh = context + .upload(super::f32_slice_as_bytes(&[1.0])) + .expect("upload HH"); + + let output = context + .j2k_inverse_dwt_single_device( + &ll, + &hl, + &lh, + &hh, + CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 2, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + irreversible97: 0, + }, + ) + .expect("CUDA inverse DWT"); + + assert_eq!(output.execution().kernel_dispatches(), 3); + let mut actual = vec![0.0f32; 4]; + output + .buffer() + .copy_to_host(super::f32_slice_as_bytes_mut(&mut actual)) + .expect("download inverse DWT"); + assert_eq!(actual, vec![7.0, 9.0, 10.0, 13.0]); +} + +#[test] +fn j2k_inverse_dwt_single_reuses_pool_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let ll = context + .upload(super::f32_slice_as_bytes(&[10.0])) + .expect("upload LL"); + let hl = context + .upload(super::f32_slice_as_bytes(&[2.0])) + .expect("upload HL"); + let lh = context + .upload(super::f32_slice_as_bytes(&[4.0])) + .expect("upload LH"); + let hh = context + .upload(super::f32_slice_as_bytes(&[1.0])) + .expect("upload HH"); + + let output = context + .j2k_inverse_dwt_single_device_with_pool( + &ll, + &hl, + &lh, + &hh, + CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 2, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + irreversible97: 0, + }, + &pool, + ) + .expect("pooled CUDA inverse DWT"); + + assert_eq!(output.execution().kernel_dispatches(), 3); + let cached_while_live = pool.cached_count().expect("cached while live"); + + drop(output); + + assert!(pool.cached_count().expect("cached after drop") > cached_while_live); +} + +#[test] +fn idwt_cooperative_53_selection_requires_large_reversible_batches() { + let mut kernel_job = CudaJ2kIdwtMultiKernelJob { + ll_ptr: 0, + hl_ptr: 0, + lh_ptr: 0, + hh_ptr: 0, + output_ptr: 0, + job: CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 0, + y1: 0, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 0, + y1: 0, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 0, + y1: 0, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 0, + y1: 0, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 0, + y1: 0, + }, + irreversible97: 0, + }, + }; + + assert!(!idwt_batch_uses_cooperative_53(&[kernel_job], 127, 128)); + assert!(!idwt_batch_uses_cooperative_53(&[kernel_job], 128, 127)); + assert!(idwt_batch_uses_cooperative_53(&[kernel_job], 128, 128)); + assert!(idwt_batch_uses_cooperative_53(&[kernel_job], 512, 512)); + assert!(!idwt_batch_uses_cooperative_53(&[kernel_job], 513, 128)); + kernel_job.job.irreversible97 = 1; + assert!(!idwt_batch_uses_cooperative_53(&[kernel_job], 128, 128)); +} + +#[test] +fn idwt_cooperative_97_selection_requires_large_irreversible_batches() { + let mut kernel_job = CudaJ2kIdwtMultiKernelJob { + ll_ptr: 0, + hl_ptr: 0, + lh_ptr: 0, + hh_ptr: 0, + output_ptr: 0, + job: CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 0, + y1: 0, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 0, + y1: 0, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 0, + y1: 0, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 0, + y1: 0, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 0, + y1: 0, + }, + irreversible97: 1, + }, + }; + + assert_eq!( + idwt_batch_kernel_mode(&[kernel_job], 128, 128), + CudaJ2kIdwtBatchKernelMode::Cooperative97 + ); + assert_eq!( + idwt_batch_kernel_mode(&[kernel_job], 64, 64), + CudaJ2kIdwtBatchKernelMode::Cooperative97 + ); + assert_eq!( + idwt_batch_kernel_mode(&[kernel_job], 512, 512), + CudaJ2kIdwtBatchKernelMode::Cooperative97 + ); + assert_eq!( + idwt_batch_kernel_mode(&[kernel_job], 63, 64), + CudaJ2kIdwtBatchKernelMode::Generic + ); + assert_eq!( + idwt_batch_kernel_mode(&[kernel_job], 513, 128), + CudaJ2kIdwtBatchKernelMode::Generic + ); + kernel_job.job.irreversible97 = 0; + assert_ne!( + idwt_batch_kernel_mode(&[kernel_job], 128, 128), + CudaJ2kIdwtBatchKernelMode::Cooperative97 + ); +} + +#[test] +fn idwt_batch_trace_row_reports_stage_shape_and_mode() { + let kernel_jobs = [ + CudaJ2kIdwtMultiKernelJob { + ll_ptr: 0, + hl_ptr: 0, + lh_ptr: 0, + hh_ptr: 0, + output_ptr: 0, + job: CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 128, + y1: 96, + }, + ll_rect: CudaJ2kRect::default(), + hl_rect: CudaJ2kRect::default(), + lh_rect: CudaJ2kRect::default(), + hh_rect: CudaJ2kRect::default(), + irreversible97: 1, + }, + }, + CudaJ2kIdwtMultiKernelJob { + ll_ptr: 0, + hl_ptr: 0, + lh_ptr: 0, + hh_ptr: 0, + output_ptr: 0, + job: CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 64, + y1: 48, + }, + ll_rect: CudaJ2kRect::default(), + hl_rect: CudaJ2kRect::default(), + lh_rect: CudaJ2kRect::default(), + hh_rect: CudaJ2kRect::default(), + irreversible97: 1, + }, + }, + ]; + + let row = idwt_batch_trace_row( + 3, + &kernel_jobs, + 128, + 96, + CudaJ2kIdwtBatchKernelMode::Cooperative97, + 42, + ); + + assert_eq!( + format_idwt_batch_trace_row(row), + "j2k_profile codec=j2k op=cuda_idwt_batch path=decode stage_index=3 mode=Cooperative97 job_count=2 max_width=128 max_height=96 min_width=64 min_height=48 total_pixels=15360 irreversible_jobs=2 elapsed_us=42" + ); +} + +#[test] +fn j2k_inverse_dwt_batch_empty_uses_no_dispatch_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let execution = context + .j2k_inverse_dwt_batch_device_with_pool(&[] as &[CudaJ2kIdwtTarget<'_>], &pool) + .expect("empty batched CUDA inverse DWT"); + + assert_eq!(execution.kernel_dispatches(), 0); + assert_eq!(execution.decode_kernel_dispatches(), 0); +} + +#[test] +fn j2k_inverse_dwt_batch_matches_expected_outputs_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let ll = context + .upload(super::f32_slice_as_bytes(&[10.0])) + .expect("upload LL"); + let hl = context + .upload(super::f32_slice_as_bytes(&[2.0])) + .expect("upload HL"); + let lh = context + .upload(super::f32_slice_as_bytes(&[4.0])) + .expect("upload LH"); + let hh = context + .upload(super::f32_slice_as_bytes(&[1.0])) + .expect("upload HH"); + let first_output = pool + .take(4 * std::mem::size_of::()) + .expect("first batched IDWT output"); + let second_output = pool + .take(4 * std::mem::size_of::()) + .expect("second batched IDWT output"); + let job = CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 2, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + irreversible97: 0, + }; + + let execution = context + .j2k_inverse_dwt_batch_device_with_pool( + &[ + CudaJ2kIdwtTarget { + ll: &ll, + hl: &hl, + lh: &lh, + hh: &hh, + output: first_output + .as_device_buffer() + .expect("first output device buffer"), + job, + }, + CudaJ2kIdwtTarget { + ll: &ll, + hl: &hl, + lh: &lh, + hh: &hh, + output: second_output + .as_device_buffer() + .expect("second output device buffer"), + job, + }, + ], + &pool, + ) + .expect("batched CUDA inverse DWT"); + assert_eq!(execution.kernel_dispatches(), 2); + + let mut first_actual = vec![0.0f32; 4]; + first_output + .copy_to_host(super::f32_slice_as_bytes_mut(&mut first_actual)) + .expect("download first batched IDWT"); + assert_eq!(first_actual, vec![7.0, 9.0, 10.0, 13.0]); + let mut second_actual = vec![0.0f32; 4]; + second_output + .copy_to_host(super::f32_slice_as_bytes_mut(&mut second_actual)) + .expect("download second batched IDWT"); + assert_eq!(second_actual, vec![7.0, 9.0, 10.0, 13.0]); +} + +#[test] +fn j2k_inverse_dwt_batch_odd_origin_matches_single_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let ll = context + .upload(super::f32_slice_as_bytes(&[10.0])) + .expect("upload odd LL"); + let hl = context + .upload(super::f32_slice_as_bytes(&[2.0, 5.0])) + .expect("upload odd HL"); + let lh = context + .upload(super::f32_slice_as_bytes(&[4.0, 7.0])) + .expect("upload odd LH"); + let hh = context + .upload(super::f32_slice_as_bytes(&[1.0, 3.0, 6.0, 8.0])) + .expect("upload odd HH"); + let job = CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 1, + y0: 1, + x1: 4, + y1: 4, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 1, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 2, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 2, + }, + irreversible97: 0, + }; + + let single = context + .j2k_inverse_dwt_single_device_with_pool(&ll, &hl, &lh, &hh, job, &pool) + .expect("single CUDA inverse DWT"); + assert_eq!(single.execution().kernel_dispatches(), 3); + let batch_output = pool + .take(9 * std::mem::size_of::()) + .expect("odd batched IDWT output"); + let execution = context + .j2k_inverse_dwt_batch_device_with_pool( + &[CudaJ2kIdwtTarget { + ll: &ll, + hl: &hl, + lh: &lh, + hh: &hh, + output: batch_output + .as_device_buffer() + .expect("odd batch output device buffer"), + job, + }], + &pool, + ) + .expect("odd-origin batched CUDA inverse DWT"); + assert_eq!(execution.kernel_dispatches(), 2); + + let mut single_actual = vec![0.0f32; 9]; + single + .buffer() + .expect("single odd output device buffer") + .copy_to_host(super::f32_slice_as_bytes_mut(&mut single_actual)) + .expect("download single odd IDWT"); + let mut batch_actual = vec![0.0f32; 9]; + batch_output + .copy_to_host(super::f32_slice_as_bytes_mut(&mut batch_actual)) + .expect("download batch odd IDWT"); + assert_eq!(batch_actual, single_actual); +} + +#[test] +#[allow(clippy::cast_precision_loss, clippy::similar_names)] +fn j2k_inverse_dwt_batch_large_reversible_matches_single_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let band_len = 64 * 64; + let ll_values: Vec = (0..band_len).map(|idx| (idx % 19) as f32).collect(); + let hl_values: Vec = (0..band_len).map(|idx| ((idx * 3) % 23) as f32).collect(); + let lh_values: Vec = (0..band_len).map(|idx| ((idx * 5) % 29) as f32).collect(); + let hh_values: Vec = (0..band_len).map(|idx| ((idx * 7) % 31) as f32).collect(); + let ll = context + .upload(super::f32_slice_as_bytes(&ll_values)) + .expect("upload large LL"); + let hl = context + .upload(super::f32_slice_as_bytes(&hl_values)) + .expect("upload large HL"); + let lh = context + .upload(super::f32_slice_as_bytes(&lh_values)) + .expect("upload large LH"); + let hh = context + .upload(super::f32_slice_as_bytes(&hh_values)) + .expect("upload large HH"); + let job = CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 128, + y1: 128, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 64, + y1: 64, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 64, + y1: 64, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 64, + y1: 64, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 64, + y1: 64, + }, + irreversible97: 0, + }; + + let single = context + .j2k_inverse_dwt_single_device_with_pool(&ll, &hl, &lh, &hh, job, &pool) + .expect("large single CUDA inverse DWT"); + let batch_output = pool + .take(128 * 128 * std::mem::size_of::()) + .expect("large batched IDWT output"); + let execution = context + .j2k_inverse_dwt_batch_device_with_pool( + &[CudaJ2kIdwtTarget { + ll: &ll, + hl: &hl, + lh: &lh, + hh: &hh, + output: batch_output + .as_device_buffer() + .expect("large batch output device buffer"), + job, + }], + &pool, + ) + .expect("large batched CUDA inverse DWT"); + assert_eq!(execution.kernel_dispatches(), 2); + + let mut single_actual = vec![0.0f32; 128 * 128]; + single + .buffer() + .expect("large single output device buffer") + .copy_to_host(super::f32_slice_as_bytes_mut(&mut single_actual)) + .expect("download large single IDWT"); + let mut batch_actual = vec![0.0f32; 128 * 128]; + batch_output + .copy_to_host(super::f32_slice_as_bytes_mut(&mut batch_actual)) + .expect("download large batch IDWT"); + assert_eq!(batch_actual, single_actual); +} + +#[test] +#[allow(clippy::cast_precision_loss, clippy::similar_names)] +fn j2k_inverse_dwt_batch_large_irreversible_matches_single_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let band_len = 128 * 128; + let ll_values: Vec = (0..band_len) + .map(|idx| ((idx % 43) as f32) * 0.25) + .collect(); + let hl_values: Vec = (0..band_len) + .map(|idx| (((idx * 3) % 47) as f32) * 0.125) + .collect(); + let lh_values: Vec = (0..band_len) + .map(|idx| (((idx * 5) % 53) as f32) * 0.0625) + .collect(); + let hh_values: Vec = (0..band_len) + .map(|idx| (((idx * 7) % 59) as f32) * 0.03125) + .collect(); + let ll = context + .upload(super::f32_slice_as_bytes(&ll_values)) + .expect("upload large irreversible LL"); + let hl = context + .upload(super::f32_slice_as_bytes(&hl_values)) + .expect("upload large irreversible HL"); + let lh = context + .upload(super::f32_slice_as_bytes(&lh_values)) + .expect("upload large irreversible LH"); + let hh = context + .upload(super::f32_slice_as_bytes(&hh_values)) + .expect("upload large irreversible HH"); + let job = CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 256, + y1: 256, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 128, + y1: 128, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 128, + y1: 128, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 128, + y1: 128, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 128, + y1: 128, + }, + irreversible97: 1, + }; + + let single = context + .j2k_inverse_dwt_single_device_with_pool(&ll, &hl, &lh, &hh, job, &pool) + .expect("large irreversible single CUDA inverse DWT"); + let batch_output = pool + .take(256 * 256 * std::mem::size_of::()) + .expect("large irreversible batched IDWT output"); + let execution = context + .j2k_inverse_dwt_batch_device_with_pool( + &[CudaJ2kIdwtTarget { + ll: &ll, + hl: &hl, + lh: &lh, + hh: &hh, + output: batch_output + .as_device_buffer() + .expect("large irreversible batch output device buffer"), + job, + }], + &pool, + ) + .expect("large irreversible batched CUDA inverse DWT"); + assert_eq!(execution.kernel_dispatches(), 2); + + let mut single_actual = vec![0.0f32; 256 * 256]; + single + .buffer() + .expect("large irreversible single output device buffer") + .copy_to_host(super::f32_slice_as_bytes_mut(&mut single_actual)) + .expect("download large irreversible single IDWT"); + let mut batch_actual = vec![0.0f32; 256 * 256]; + batch_output + .copy_to_host(super::f32_slice_as_bytes_mut(&mut batch_actual)) + .expect("download large irreversible batch IDWT"); + assert_eq!(batch_actual, single_actual); +} + +#[test] +#[allow(clippy::cast_precision_loss, clippy::similar_names)] +fn j2k_inverse_dwt_batch_512_reversible_matches_single_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let band_len = 256 * 256; + let ll_values: Vec = (0..band_len).map(|idx| (idx % 43) as f32).collect(); + let hl_values: Vec = (0..band_len).map(|idx| ((idx * 3) % 47) as f32).collect(); + let lh_values: Vec = (0..band_len).map(|idx| ((idx * 5) % 53) as f32).collect(); + let hh_values: Vec = (0..band_len).map(|idx| ((idx * 7) % 59) as f32).collect(); + let ll = context + .upload(super::f32_slice_as_bytes(&ll_values)) + .expect("upload 512 LL"); + let hl = context + .upload(super::f32_slice_as_bytes(&hl_values)) + .expect("upload 512 HL"); + let lh = context + .upload(super::f32_slice_as_bytes(&lh_values)) + .expect("upload 512 LH"); + let hh = context + .upload(super::f32_slice_as_bytes(&hh_values)) + .expect("upload 512 HH"); + let job = CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 512, + y1: 512, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 256, + y1: 256, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 256, + y1: 256, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 256, + y1: 256, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 256, + y1: 256, + }, + irreversible97: 0, + }; + + let single = context + .j2k_inverse_dwt_single_device_with_pool(&ll, &hl, &lh, &hh, job, &pool) + .expect("512 single CUDA inverse DWT"); + let batch_output = pool + .take(512 * 512 * std::mem::size_of::()) + .expect("512 batched IDWT output"); + let execution = context + .j2k_inverse_dwt_batch_device_with_pool( + &[CudaJ2kIdwtTarget { + ll: &ll, + hl: &hl, + lh: &lh, + hh: &hh, + output: batch_output + .as_device_buffer() + .expect("512 batch output device buffer"), + job, + }], + &pool, + ) + .expect("512 batched CUDA inverse DWT"); + assert_eq!(execution.kernel_dispatches(), 2); + + let mut single_actual = vec![0.0f32; 512 * 512]; + single + .buffer() + .expect("512 single output device buffer") + .copy_to_host(super::f32_slice_as_bytes_mut(&mut single_actual)) + .expect("download 512 single IDWT"); + let mut batch_actual = vec![0.0f32; 512 * 512]; + batch_output + .copy_to_host(super::f32_slice_as_bytes_mut(&mut batch_actual)) + .expect("download 512 batch IDWT"); + assert_eq!(batch_actual, single_actual); +} + +#[test] +fn j2k_inverse_dwt_batch_enqueue_matches_expected_outputs_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let ll = context + .upload(super::f32_slice_as_bytes(&[10.0])) + .expect("upload LL"); + let hl = context + .upload(super::f32_slice_as_bytes(&[2.0])) + .expect("upload HL"); + let lh = context + .upload(super::f32_slice_as_bytes(&[4.0])) + .expect("upload LH"); + let hh = context + .upload(super::f32_slice_as_bytes(&[1.0])) + .expect("upload HH"); + let output = pool + .take(4 * std::mem::size_of::()) + .expect("batched IDWT output"); + let job = CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 2, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + irreversible97: 0, + }; + + let queued = context + .j2k_inverse_dwt_batch_device_enqueue_with_pool( + &[CudaJ2kIdwtTarget { + ll: &ll, + hl: &hl, + lh: &lh, + hh: &hh, + output: output.as_device_buffer().expect("output device buffer"), + job, + }], + &pool, + ) + .expect("enqueue batched CUDA inverse DWT"); + assert_eq!(queued.execution().kernel_dispatches(), 2); + context.synchronize().expect("queued IDWT completion"); + drop(queued); + + let mut actual = vec![0.0f32; 4]; + output + .copy_to_host(super::f32_slice_as_bytes_mut(&mut actual)) + .expect("download queued batched IDWT"); + assert_eq!(actual, vec![7.0, 9.0, 10.0, 13.0]); +} + +#[test] +#[allow(clippy::similar_names, clippy::too_many_lines)] +fn j2k_inverse_dwt_batch_sequence_enqueue_matches_two_stage_path_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let ll = context + .upload(super::f32_slice_as_bytes(&[10.0])) + .expect("upload LL"); + let hl = context + .upload(super::f32_slice_as_bytes(&[2.0])) + .expect("upload HL"); + let lh = context + .upload(super::f32_slice_as_bytes(&[4.0])) + .expect("upload LH"); + let hh = context + .upload(super::f32_slice_as_bytes(&[1.0])) + .expect("upload HH"); + let stage2_hl = context + .upload(super::f32_slice_as_bytes(&[0.0, 1.0, 2.0, 3.0])) + .expect("upload stage2 HL"); + let stage2_lh = context + .upload(super::f32_slice_as_bytes(&[4.0, 5.0, 6.0, 7.0])) + .expect("upload stage2 LH"); + let stage2_hh = context + .upload(super::f32_slice_as_bytes(&[8.0, 9.0, 10.0, 11.0])) + .expect("upload stage2 HH"); + let stage1_job = CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 2, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + irreversible97: 0, + }; + let stage2_job = CudaJ2kIdwtJob { + rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 4, + y1: 4, + }, + ll_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 2, + }, + hl_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 2, + }, + lh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 2, + }, + hh_rect: CudaJ2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 2, + }, + irreversible97: 0, + }; + let legacy_stage1 = pool + .take(4 * std::mem::size_of::()) + .expect("legacy stage1 output"); + let legacy_stage2 = pool + .take(16 * std::mem::size_of::()) + .expect("legacy stage2 output"); + let sequence_stage1 = pool + .take(4 * std::mem::size_of::()) + .expect("sequence stage1 output"); + let sequence_stage2 = pool + .take(16 * std::mem::size_of::()) + .expect("sequence stage2 output"); + + context + .j2k_inverse_dwt_batch_device_with_pool( + &[CudaJ2kIdwtTarget { + ll: &ll, + hl: &hl, + lh: &lh, + hh: &hh, + output: legacy_stage1 + .as_device_buffer() + .expect("legacy stage1 device buffer"), + job: stage1_job, + }], + &pool, + ) + .expect("legacy stage1 IDWT"); + context + .j2k_inverse_dwt_batch_device_with_pool( + &[CudaJ2kIdwtTarget { + ll: legacy_stage1 + .as_device_buffer() + .expect("legacy stage1 device buffer"), + hl: &stage2_hl, + lh: &stage2_lh, + hh: &stage2_hh, + output: legacy_stage2 + .as_device_buffer() + .expect("legacy stage2 device buffer"), + job: stage2_job, + }], + &pool, + ) + .expect("legacy stage2 IDWT"); + + let sequence_stage1_targets = [CudaJ2kIdwtTarget { + ll: &ll, + hl: &hl, + lh: &lh, + hh: &hh, + output: sequence_stage1 + .as_device_buffer() + .expect("sequence stage1 device buffer"), + job: stage1_job, + }]; + let sequence_stage2_targets = [CudaJ2kIdwtTarget { + ll: sequence_stage1 + .as_device_buffer() + .expect("sequence stage1 device buffer"), + hl: &stage2_hl, + lh: &stage2_lh, + hh: &stage2_hh, + output: sequence_stage2 + .as_device_buffer() + .expect("sequence stage2 device buffer"), + job: stage2_job, + }]; + let queued = context + .j2k_inverse_dwt_batch_sequence_enqueue_with_pool( + &[&sequence_stage1_targets, &sequence_stage2_targets], + &pool, + ) + .expect("queued IDWT sequence"); + assert_eq!(queued.execution().kernel_dispatches(), 4); + assert_eq!(queued.resource_count(), 1); + context + .synchronize() + .expect("queued IDWT sequence completion"); + drop(queued); + + let mut legacy_actual = vec![0.0f32; 16]; + legacy_stage2 + .copy_to_host(super::f32_slice_as_bytes_mut(&mut legacy_actual)) + .expect("download legacy stage2 IDWT"); + let mut sequence_actual = vec![0.0f32; 16]; + sequence_stage2 + .copy_to_host(super::f32_slice_as_bytes_mut(&mut sequence_actual)) + .expect("download sequence stage2 IDWT"); + assert_eq!(sequence_actual, legacy_actual); +} + +#[test] +fn j2k_store_rgb8_mct_matches_inverse_mct_plus_store_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let plane0 = [16.0f32, 18.0, 21.0, 24.0]; + let plane1 = [-3.0f32, 4.0, 5.0, -6.0]; + let plane2 = [2.0f32, -1.0, 7.0, 3.0]; + let legacy0 = context + .upload(super::f32_slice_as_bytes(&plane0)) + .expect("upload legacy MCT plane 0"); + let legacy1 = context + .upload(super::f32_slice_as_bytes(&plane1)) + .expect("upload legacy MCT plane 1"); + let legacy2 = context + .upload(super::f32_slice_as_bytes(&plane2)) + .expect("upload legacy MCT plane 2"); + let fused0 = context + .upload(super::f32_slice_as_bytes(&plane0)) + .expect("upload fused MCT plane 0"); + let fused1 = context + .upload(super::f32_slice_as_bytes(&plane1)) + .expect("upload fused MCT plane 1"); + let fused2 = context + .upload(super::f32_slice_as_bytes(&plane2)) + .expect("upload fused MCT plane 2"); + let addend = 128.0; + + let mct_stats = context + .j2k_inverse_mct_device( + &legacy0, + &legacy1, + &legacy2, + super::CudaJ2kInverseMctJob { + len: 4, + irreversible97: 0, + addend0: addend, + addend1: addend, + addend2: addend, + }, + ) + .expect("legacy inverse MCT"); + assert_eq!(mct_stats.kernel_dispatches(), 1); + let store_job = super::CudaJ2kStoreRgb8Job { + input_width0: 2, + input_width1: 2, + input_width2: 2, + source_x0: 0, + source_y0: 0, + source_x1: 0, + source_y1: 0, + source_x2: 0, + source_y2: 0, + copy_width: 2, + copy_height: 2, + output_width: 2, + output_height: 2, + output_x: 0, + output_y: 0, + addend0: 0.0, + addend1: 0.0, + addend2: 0.0, + bit_depth0: 8, + bit_depth1: 8, + bit_depth2: 8, + rgba: 1, + }; + let legacy_output = context + .j2k_store_rgb8_device(&legacy0, &legacy1, &legacy2, store_job) + .expect("legacy RGB8 store"); + let fused_output = context + .j2k_store_rgb8_mct_device( + &fused0, + &fused1, + &fused2, + super::CudaJ2kStoreRgb8MctJob { + store: super::CudaJ2kStoreRgb8Job { + addend0: addend, + addend1: addend, + addend2: addend, + ..store_job + }, + irreversible97: 0, + }, + ) + .expect("fused RGB8 MCT store"); + + assert_eq!(legacy_output.execution().kernel_dispatches(), 1); + assert_eq!(fused_output.execution().kernel_dispatches(), 1); + let mut legacy_bytes = vec![0u8; 16]; + legacy_output + .buffer() + .copy_to_host(&mut legacy_bytes) + .expect("download legacy RGB8"); + let mut fused_bytes = vec![0u8; 16]; + fused_output + .buffer() + .copy_to_host(&mut fused_bytes) + .expect("download fused RGB8"); + assert_eq!(fused_bytes, legacy_bytes); +} + +#[test] +#[allow(clippy::similar_names, clippy::too_many_lines)] +fn j2k_store_rgb8_mct_batch_matches_separate_stores_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let plane0_a = [16.0f32, 18.0, 21.0, 24.0]; + let plane1_a = [-3.0f32, 4.0, 5.0, -6.0]; + let plane2_a = [2.0f32, -1.0, 7.0, 3.0]; + let plane0_b = [3.0f32, 7.0, 11.0, 13.0]; + let plane1_b = [5.0f32, -2.0, 9.0, 1.0]; + let plane2_b = [-4.0f32, 6.0, 0.0, 8.0]; + + let plane0_a = context + .upload(super::f32_slice_as_bytes(&plane0_a)) + .expect("upload plane 0 A"); + let plane1_a = context + .upload(super::f32_slice_as_bytes(&plane1_a)) + .expect("upload plane 1 A"); + let plane2_a = context + .upload(super::f32_slice_as_bytes(&plane2_a)) + .expect("upload plane 2 A"); + let plane0_b = context + .upload(super::f32_slice_as_bytes(&plane0_b)) + .expect("upload plane 0 B"); + let plane1_b = context + .upload(super::f32_slice_as_bytes(&plane1_b)) + .expect("upload plane 1 B"); + let plane2_b = context + .upload(super::f32_slice_as_bytes(&plane2_b)) + .expect("upload plane 2 B"); + + let store = super::CudaJ2kStoreRgb8Job { + input_width0: 2, + input_width1: 2, + input_width2: 2, + source_x0: 0, + source_y0: 0, + source_x1: 0, + source_y1: 0, + source_x2: 0, + source_y2: 0, + copy_width: 2, + copy_height: 2, + output_width: 2, + output_height: 2, + output_x: 0, + output_y: 0, + addend0: 128.0, + addend1: 128.0, + addend2: 128.0, + bit_depth0: 8, + bit_depth1: 8, + bit_depth2: 8, + rgba: 1, + }; + let separate_a = context + .j2k_store_rgb8_mct_device( + &plane0_a, + &plane1_a, + &plane2_a, + super::CudaJ2kStoreRgb8MctJob { + store, + irreversible97: 0, + }, + ) + .expect("separate fused store A"); + let separate_b = context + .j2k_store_rgb8_mct_device( + &plane0_b, + &plane1_b, + &plane2_b, + super::CudaJ2kStoreRgb8MctJob { + store, + irreversible97: 0, + }, + ) + .expect("separate fused store B"); + + let batched = context + .j2k_store_rgb8_mct_batch_device(&[ + super::CudaJ2kStoreRgb8MctTarget { + plane0: &plane0_a, + plane1: &plane1_a, + plane2: &plane2_a, + job: super::CudaJ2kStoreRgb8MctJob { + store, + irreversible97: 0, + }, + }, + super::CudaJ2kStoreRgb8MctTarget { + plane0: &plane0_b, + plane1: &plane1_b, + plane2: &plane2_b, + job: super::CudaJ2kStoreRgb8MctJob { + store, + irreversible97: 0, + }, + }, + ]) + .expect("batched fused store"); + + assert_eq!(batched.execution().kernel_dispatches(), 1); + assert_eq!(batched.outputs().len(), 2); + let mut separate_a_bytes = vec![0u8; 16]; + separate_a + .buffer() + .copy_to_host(&mut separate_a_bytes) + .expect("download separate A"); + let mut separate_b_bytes = vec![0u8; 16]; + separate_b + .buffer() + .copy_to_host(&mut separate_b_bytes) + .expect("download separate B"); + let mut batch_a_bytes = vec![0u8; 16]; + batched.outputs()[0] + .copy_to_host(&mut batch_a_bytes) + .expect("download batch A"); + let mut batch_b_bytes = vec![0u8; 16]; + batched.outputs()[1] + .copy_to_host(&mut batch_b_bytes) + .expect("download batch B"); + assert_eq!(batch_a_bytes, separate_a_bytes); + assert_eq!(batch_b_bytes, separate_b_bytes); +} + +#[test] +fn j2k_store_rgb8_mct_single_matches_one_item_batch_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let plane0 = [16.0f32, 18.0, 21.0, 24.0]; + let plane1 = [-3.0f32, 4.0, 5.0, -6.0]; + let plane2 = [2.0f32, -1.0, 7.0, 3.0]; + let single0 = context + .upload(super::f32_slice_as_bytes(&plane0)) + .expect("upload single plane 0"); + let single1 = context + .upload(super::f32_slice_as_bytes(&plane1)) + .expect("upload single plane 1"); + let single2 = context + .upload(super::f32_slice_as_bytes(&plane2)) + .expect("upload single plane 2"); + let batch0 = context + .upload(super::f32_slice_as_bytes(&plane0)) + .expect("upload batch plane 0"); + let batch1 = context + .upload(super::f32_slice_as_bytes(&plane1)) + .expect("upload batch plane 1"); + let batch2 = context + .upload(super::f32_slice_as_bytes(&plane2)) + .expect("upload batch plane 2"); + + let store = super::CudaJ2kStoreRgb8Job { + input_width0: 2, + input_width1: 2, + input_width2: 2, + source_x0: 0, + source_y0: 0, + source_x1: 0, + source_y1: 0, + source_x2: 0, + source_y2: 0, + copy_width: 2, + copy_height: 2, + output_width: 2, + output_height: 2, + output_x: 0, + output_y: 0, + addend0: 128.0, + addend1: 128.0, + addend2: 128.0, + bit_depth0: 8, + bit_depth1: 8, + bit_depth2: 8, + rgba: 1, + }; + let job = super::CudaJ2kStoreRgb8MctJob { + store, + irreversible97: 0, + }; + let single = context + .j2k_store_rgb8_mct_device(&single0, &single1, &single2, job) + .expect("single RGB8 MCT store"); + let batch = context + .j2k_store_rgb8_mct_batch_device(&[super::CudaJ2kStoreRgb8MctTarget { + plane0: &batch0, + plane1: &batch1, + plane2: &batch2, + job, + }]) + .expect("one-item batch RGB8 MCT store"); + + assert_eq!(single.execution().kernel_dispatches(), 1); + assert_eq!(batch.execution().kernel_dispatches(), 1); + let mut single_bytes = vec![0u8; 16]; + single + .buffer() + .copy_to_host(&mut single_bytes) + .expect("download single RGB8 MCT store"); + let mut batch_bytes = vec![0u8; 16]; + batch.outputs()[0] + .copy_to_host(&mut batch_bytes) + .expect("download one-item batch RGB8 MCT store"); + assert_eq!(single_bytes, batch_bytes); +} + +#[test] +fn j2k_store_rgb16_mct_matches_inverse_mct_plus_store_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let plane0 = [40.0f32, 44.0, 52.0, 55.0]; + let plane1 = [-3.5f32, 1.25, 2.75, -4.0]; + let plane2 = [5.0f32, -2.0, 1.5, 6.0]; + let legacy0 = context + .upload(super::f32_slice_as_bytes(&plane0)) + .expect("upload legacy ICT plane 0"); + let legacy1 = context + .upload(super::f32_slice_as_bytes(&plane1)) + .expect("upload legacy ICT plane 1"); + let legacy2 = context + .upload(super::f32_slice_as_bytes(&plane2)) + .expect("upload legacy ICT plane 2"); + let fused0 = context + .upload(super::f32_slice_as_bytes(&plane0)) + .expect("upload fused ICT plane 0"); + let fused1 = context + .upload(super::f32_slice_as_bytes(&plane1)) + .expect("upload fused ICT plane 1"); + let fused2 = context + .upload(super::f32_slice_as_bytes(&plane2)) + .expect("upload fused ICT plane 2"); + let addend = 32768.0; + + context + .j2k_inverse_mct_device( + &legacy0, + &legacy1, + &legacy2, + super::CudaJ2kInverseMctJob { + len: 4, + irreversible97: 1, + addend0: addend, + addend1: addend, + addend2: addend, + }, + ) + .expect("legacy inverse ICT"); + let store_job = super::CudaJ2kStoreRgb16Job { + input_width0: 2, + input_width1: 2, + input_width2: 2, + source_x0: 0, + source_y0: 0, + source_x1: 0, + source_y1: 0, + source_x2: 0, + source_y2: 0, + copy_width: 2, + copy_height: 2, + output_width: 2, + output_height: 2, + output_x: 0, + output_y: 0, + addend0: 0.0, + addend1: 0.0, + addend2: 0.0, + bit_depth0: 16, + bit_depth1: 16, + bit_depth2: 16, + rgba: 0, + }; + let legacy_output = context + .j2k_store_rgb16_device(&legacy0, &legacy1, &legacy2, store_job) + .expect("legacy RGB16 store"); + let fused_output = context + .j2k_store_rgb16_mct_device( + &fused0, + &fused1, + &fused2, + super::CudaJ2kStoreRgb16MctJob { + store: super::CudaJ2kStoreRgb16Job { + addend0: addend, + addend1: addend, + addend2: addend, + ..store_job + }, + irreversible97: 1, + }, + ) + .expect("fused RGB16 MCT store"); + + assert_eq!(legacy_output.execution().kernel_dispatches(), 1); + assert_eq!(fused_output.execution().kernel_dispatches(), 1); + let mut legacy_bytes = vec![0u8; 24]; + legacy_output + .buffer() + .copy_to_host(&mut legacy_bytes) + .expect("download legacy RGB16"); + let mut fused_bytes = vec![0u8; 24]; + fused_output + .buffer() + .copy_to_host(&mut fused_bytes) + .expect("download fused RGB16"); + assert_eq!(fused_bytes, legacy_bytes); +} + +#[test] +fn j2k_dequantize_htj2k_codeblocks_multi_uses_one_dispatch_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let first = context + .upload(super::i32_slice_as_bytes(&[0, 0, 0, 0])) + .expect("upload first coefficients"); + let second = context + .upload(super::i32_slice_as_bytes(&[0, 0])) + .expect("upload second coefficients"); + let first_jobs = [CudaHtj2kCodeBlockJob { + payload_offset: 0, + width: 2, + height: 2, + payload_len: 0, + cleanup_length: 0, + refinement_length: 0, + missing_bit_planes: 0, + num_bitplanes: 1, + number_of_coding_passes: 1, + output_stride: 2, + output_offset: 0, + dequantization_step: 1.0, + stripe_causal: false, + }]; + let second_jobs = [CudaHtj2kCodeBlockJob { + payload_offset: 0, + width: 2, + height: 1, + payload_len: 0, + cleanup_length: 0, + refinement_length: 0, + missing_bit_planes: 0, + num_bitplanes: 1, + number_of_coding_passes: 1, + output_stride: 2, + output_offset: 0, + dequantization_step: 1.0, + stripe_causal: false, + }]; + + let execution = context + .j2k_dequantize_htj2k_codeblocks_multi_device(&[ + CudaHtj2kDequantizeTarget { + coefficients: &first, + jobs: &first_jobs, + output_words: 4, + }, + CudaHtj2kDequantizeTarget { + coefficients: &second, + jobs: &second_jobs, + output_words: 2, + }, + ]) + .expect("multi-buffer HTJ2K dequant"); + assert_eq!(execution.kernel_dispatches(), 1); + + let mut first_actual = vec![f32::NAN; 4]; + first + .copy_to_host(super::f32_slice_as_bytes_mut(&mut first_actual)) + .expect("download first coefficients"); + assert_eq!(first_actual, vec![0.0; 4]); + let mut second_actual = vec![f32::NAN; 2]; + second + .copy_to_host(super::f32_slice_as_bytes_mut(&mut second_actual)) + .expect("download second coefficients"); + assert_eq!(second_actual, vec![0.0; 2]); +} + +#[test] +fn queued_cleanup_metadata_dequantizes_without_second_job_upload_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let first = context + .upload(super::i32_slice_as_bytes(&[1, i32::MIN + 2, 0, 3])) + .expect("upload first coefficients"); + let second = context + .upload(super::i32_slice_as_bytes(&[4, i32::MIN + 5])) + .expect("upload second coefficients"); + let jobs = [ + CudaHtj2kCleanupMultiKernelJob { + output_ptr: first.device_ptr(), + coded_offset: 0, + width: 2, + height: 2, + coded_len: 0, + cleanup_length: 0, + refinement_length: 0, + missing_msbs: 0, + num_bitplanes: 31, + number_of_coding_passes: 1, + output_stride: 2, + output_offset: 0, + dequantization_step: 0.5, + stripe_causal: 0, + }, + CudaHtj2kCleanupMultiKernelJob { + output_ptr: second.device_ptr(), + coded_offset: 0, + width: 2, + height: 1, + coded_len: 0, + cleanup_length: 0, + refinement_length: 0, + missing_msbs: 0, + num_bitplanes: 31, + number_of_coding_passes: 1, + output_stride: 2, + output_offset: 0, + dequantization_step: 0.25, + stripe_causal: 0, + }, + ]; + let jobs_buffer = pool + .upload(super::htj2k_cleanup_multi_jobs_as_bytes(&jobs)) + .expect("upload cleanup metadata"); + let queued = CudaQueuedHtj2kCleanup { + resources: vec![jobs_buffer], + status_buffer: None, + status_count: jobs.len(), + kernel_name: "j2k_htj2k_decode_codeblocks_multi", + execution: CudaExecutionStats::default(), + }; + + let execution = context + .j2k_dequantize_queued_htj2k_cleanup_with_pool(&queued) + .expect("dequant from queued cleanup metadata"); + assert_eq!(execution.kernel_dispatches(), 1); + + let mut first_actual = vec![f32::NAN; 4]; + first + .copy_to_host(super::f32_slice_as_bytes_mut(&mut first_actual)) + .expect("download first coefficients"); + assert_eq!(first_actual, vec![0.5, -1.0, 0.0, 1.5]); + let mut second_actual = vec![f32::NAN; 2]; + second + .copy_to_host(super::f32_slice_as_bytes_mut(&mut second_actual)) + .expect("download second coefficients"); + assert_eq!(second_actual, vec![1.0, -1.25]); +} + +#[test] +fn htj2k_decode_multi_kernel_routes_cleanup_only_jobs() { + let cleanup_job = CudaHtj2kCleanupMultiKernelJob { + output_ptr: 0, + coded_offset: 0, + width: 64, + height: 64, + coded_len: 8, + cleanup_length: 8, + refinement_length: 0, + missing_msbs: 0, + num_bitplanes: 8, + number_of_coding_passes: 1, + output_stride: 64, + output_offset: 0, + dequantization_step: 1.0, + stripe_causal: 0, + }; + let (_, cleanup_kernel_name) = super::htj2k_decode_multi_kernel_for_jobs(&[cleanup_job]); + assert_eq!( + cleanup_kernel_name, + "j2k_htj2k_decode_codeblocks_multi_cleanup_only" + ); + + let mut refinement_job = cleanup_job; + refinement_job.refinement_length = 4; + refinement_job.number_of_coding_passes = 2; + let (_, generic_kernel_name) = super::htj2k_decode_multi_kernel_for_jobs(&[refinement_job]); + assert_eq!(generic_kernel_name, "j2k_htj2k_decode_codeblocks_multi"); +} + +#[test] +fn htj2k_decode_multi_cleanup_dequant_kernel_accepts_cleanup_only_jobs() { + let cleanup_job = CudaHtj2kCleanupMultiKernelJob { + output_ptr: 0, + coded_offset: 0, + width: 64, + height: 64, + coded_len: 8, + cleanup_length: 8, + refinement_length: 0, + missing_msbs: 0, + num_bitplanes: 8, + number_of_coding_passes: 1, + output_stride: 64, + output_offset: 0, + dequantization_step: 1.0, + stripe_causal: 0, + }; + let (_, cleanup_dequant_kernel_name) = + super::htj2k_decode_multi_cleanup_dequant_kernel_for_jobs(&[cleanup_job]) + .expect("cleanup-only jobs use fused cleanup/dequant kernel"); + assert_eq!( + cleanup_dequant_kernel_name, + "j2k_htj2k_decode_codeblocks_multi_cleanup_dequantize" + ); +} + +#[test] +fn htj2k_decode_multi_cleanup_dequant_kernel_rejects_refinement_jobs() { + let mut refinement_job = CudaHtj2kCleanupMultiKernelJob { + output_ptr: 0, + coded_offset: 0, + width: 64, + height: 64, + coded_len: 12, + cleanup_length: 8, + refinement_length: 4, + missing_msbs: 0, + num_bitplanes: 8, + number_of_coding_passes: 2, + output_stride: 64, + output_offset: 0, + dequantization_step: 1.0, + stripe_causal: 0, + }; + assert!(super::htj2k_decode_multi_cleanup_dequant_kernel_for_jobs(&[refinement_job]).is_none()); + + refinement_job.refinement_length = 0; + assert!(super::htj2k_decode_multi_cleanup_dequant_kernel_for_jobs(&[refinement_job]).is_none()); +} + +#[test] +#[allow(clippy::similar_names)] +fn htj2k_cleanup_multi_empty_targets_use_no_dispatch_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let first_vlc = [0u16; 1024]; + let later_vlc = [0u16; 1024]; + let first_uvlc = [0u16; 320]; + let later_uvlc = [0u16; 256]; + let tables = context + .upload_htj2k_decode_table_resources(CudaHtj2kDecodeTables { + vlc_table0: &first_vlc, + vlc_table1: &later_vlc, + uvlc_table0: &first_uvlc, + uvlc_table1: &later_uvlc, + }) + .expect("decode tables"); + let resources = context + .upload_htj2k_decode_resources_with_tables(&[], &tables) + .expect("decode resources"); + + let execution = context + .decode_htj2k_codeblocks_cleanup_multi_with_resources_and_pool( + &resources, + &[] as &[CudaHtj2kCleanupTarget<'_>], + &pool, + ) + .expect("empty cleanup batch"); + + assert_eq!(execution.kernel_dispatches(), 0); + assert_eq!(execution.decode_kernel_dispatches(), 0); +} + +#[test] +#[allow(clippy::similar_names)] +fn htj2k_cleanup_multi_enqueue_empty_targets_finish_with_no_dispatch_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let pool = context.buffer_pool(); + let first_vlc = [0u16; 1024]; + let later_vlc = [0u16; 1024]; + let first_uvlc = [0u16; 320]; + let later_uvlc = [0u16; 256]; + let tables = context + .upload_htj2k_decode_table_resources(CudaHtj2kDecodeTables { + vlc_table0: &first_vlc, + vlc_table1: &later_vlc, + uvlc_table0: &first_uvlc, + uvlc_table1: &later_uvlc, + }) + .expect("decode tables"); + let resources = context + .upload_htj2k_decode_resources_with_tables(&[], &tables) + .expect("decode resources"); + + let queued = context + .decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool( + &resources, + &[] as &[CudaHtj2kCleanupTarget<'_>], + &pool, + ) + .expect("empty queued cleanup batch"); + assert_eq!(queued.execution().kernel_dispatches(), 0); + assert_eq!(queued.execution().decode_kernel_dispatches(), 0); + assert_eq!(queued.resource_count(), 0); + + let execution = queued.finish().expect("finish empty queued cleanup"); + assert_eq!(execution.kernel_dispatches(), 0); + assert_eq!(execution.decode_kernel_dispatches(), 0); +} + +#[test] +fn j2k_forward_rct_matches_cpu_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let mut plane0 = vec![10.0, 1.0, 0.0, 255.0, 128.0]; + let mut plane1 = vec![20.0, 2.0, 255.0, 0.0, 64.0]; + let mut plane2 = vec![30.0, 3.0, 128.0, 127.0, 32.0]; + let mut expected0 = plane0.clone(); + let mut expected1 = plane1.clone(); + let mut expected2 = plane2.clone(); + for ((r, g), b) in expected0 + .iter_mut() + .zip(expected1.iter_mut()) + .zip(expected2.iter_mut()) + { + let r0 = *r; + let g0 = *g; + let b0 = *b; + *r = ((r0 + 2.0_f32 * g0 + b0) * 0.25_f32).floor(); + *g = b0 - g0; + *b = r0 - g0; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let execution = context + .j2k_forward_rct(&mut plane0, &mut plane1, &mut plane2) + .expect("CUDA forward RCT"); + + assert_eq!(execution.kernel_dispatches(), 1); + assert_eq!(plane0, expected0); + assert_eq!(plane1, expected1); + assert_eq!(plane2, expected2); +} + +#[test] +fn j2k_deinterleave_to_f32_matches_cpu_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let pixels = [0u8, 128, 255, 64, 32, 16]; + let context = CudaContext::system_default().expect("CUDA context"); + let output = context + .j2k_deinterleave_to_f32(&pixels, 2, 3, 8, false) + .expect("CUDA deinterleave"); + + assert_eq!(output.execution().kernel_dispatches(), 1); + assert_eq!( + output.components(), + &[vec![-128.0, -64.0], vec![0.0, -96.0], vec![127.0, -112.0],] + ); +} + +#[test] +fn j2k_deinterleave_then_rct_can_stay_resident_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let pixels = [10u8, 20, 30, 40, 50, 60]; + let context = CudaContext::system_default().expect("CUDA context"); + let mut components = context + .j2k_deinterleave_to_f32_resident(&pixels, 2, 3, 8, false) + .expect("resident CUDA deinterleave"); + + assert_eq!(components.num_components(), 3); + assert_eq!(components.num_pixels(), 2); + assert_eq!(components.execution().kernel_dispatches(), 1); + + let rct_execution = context + .j2k_forward_rct_resident(&mut components) + .expect("resident CUDA forward RCT"); + + assert_eq!(rct_execution.kernel_dispatches(), 1); + assert_eq!( + components + .download_components() + .expect("download resident components"), + vec![vec![-108.0, -78.0], vec![10.0, 10.0], vec![-10.0, -10.0]] + ); +} + +#[test] +fn j2k_deinterleave_then_ict_can_stay_resident_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let pixels = [10u8, 20, 30, 40, 50, 60]; + let context = CudaContext::system_default().expect("CUDA context"); + let mut components = context + .j2k_deinterleave_to_f32_resident(&pixels, 2, 3, 8, false) + .expect("resident CUDA deinterleave"); + + let ict_execution = context + .j2k_forward_ict_resident(&mut components) + .expect("resident CUDA forward ICT"); + + assert_eq!(ict_execution.kernel_dispatches(), 1); + let actual = components + .download_components() + .expect("download resident components"); + let expected = [[-118.0f32, -88.0], [-108.0, -78.0], [-98.0, -68.0]]; + for idx in 0..2 { + let r = expected[0][idx]; + let g = expected[1][idx]; + let b = expected[2][idx]; + let expected_y = 0.299 * r + 0.587 * g + 0.114 * b; + let blue_chroma = -0.16875 * r - 0.33126 * g + 0.5 * b; + let red_chroma = 0.5 * r - 0.41869 * g - 0.08131 * b; + assert!((actual[0][idx] - expected_y).abs() < 0.000_1); + assert!((actual[1][idx] - blue_chroma).abs() < 0.000_1); + assert!((actual[2][idx] - red_chroma).abs() < 0.000_1); + } +} + +#[test] +fn j2k_resident_deinterleave_can_feed_resident_dwt53_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let pixels = [0u8, 64, 128, 255]; + let context = CudaContext::system_default().expect("CUDA context"); + let components = context + .j2k_deinterleave_to_f32_resident(&pixels, 4, 1, 8, false) + .expect("resident CUDA deinterleave"); + let host_component = components + .download_components() + .expect("download source component")[0] + .clone(); + let expected = context + .j2k_forward_dwt53(&host_component, 2, 2, 1) + .expect("host-staged CUDA DWT"); + + let resident = context + .j2k_forward_dwt53_resident_component(&components, 0, 2, 2, 1) + .expect("resident CUDA DWT"); + + assert_eq!(resident.levels(), expected.levels()); + assert_eq!(resident.ll_dimensions(), expected.ll_dimensions()); + assert_eq!(resident.execution().copy_kernel_dispatches, 1); + assert_eq!( + resident + .download_transformed() + .expect("download resident DWT"), + expected.transformed() + ); +} + +#[test] +fn j2k_resident_deinterleave_can_feed_resident_dwt97_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let pixels = [0u8, 64, 128, 255]; + let context = CudaContext::system_default().expect("CUDA context"); + let components = context + .j2k_deinterleave_to_f32_resident(&pixels, 4, 1, 8, false) + .expect("resident CUDA deinterleave"); + let host_component = components + .download_components() + .expect("download source component")[0] + .clone(); + let expected = context + .j2k_forward_dwt97(&host_component, 2, 2, 1) + .expect("host-staged CUDA DWT"); + + let resident = context + .j2k_forward_dwt97_resident_component(&components, 0, 2, 2, 1) + .expect("resident CUDA DWT"); + + assert_eq!(resident.levels(), expected.levels()); + assert_eq!(resident.ll_dimensions(), expected.ll_dimensions()); + assert_eq!(resident.execution().copy_kernel_dispatches, 1); + assert_eq!( + resident + .download_transformed() + .expect("download resident DWT"), + expected.transformed() + ); +} + +#[test] +fn j2k_forward_ict_matches_cpu_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let mut plane0 = vec![10.0, 1.0, 0.0, 255.0, 128.0]; + let mut plane1 = vec![20.0, 2.0, 255.0, 0.0, 64.0]; + let mut plane2 = vec![30.0, 3.0, 128.0, 127.0, 32.0]; + let mut expected0 = plane0.clone(); + let mut expected1 = plane1.clone(); + let mut expected2 = plane2.clone(); + for ((r, g), b) in expected0 + .iter_mut() + .zip(expected1.iter_mut()) + .zip(expected2.iter_mut()) + { + let r0 = *r; + let g0 = *g; + let b0 = *b; + *r = 0.299 * r0 + 0.587 * g0 + 0.114 * b0; + *g = -0.16875 * r0 - 0.33126 * g0 + 0.5 * b0; + *b = 0.5 * r0 - 0.41869 * g0 - 0.08131 * b0; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let execution = context + .j2k_forward_ict(&mut plane0, &mut plane1, &mut plane2) + .expect("CUDA forward ICT"); + + assert_eq!(execution.kernel_dispatches(), 1); + for (actual, expected) in plane0.iter().zip(expected0) { + assert!((*actual - expected).abs() < 0.0001); + } + for (actual, expected) in plane1.iter().zip(expected1) { + assert!((*actual - expected).abs() < 0.0001); + } + for (actual, expected) in plane2.iter().zip(expected2) { + assert!((*actual - expected).abs() < 0.0001); + } +} + +#[test] +fn j2k_forward_dwt53_matches_cpu_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let width = 5usize; + let height = 3usize; + let samples: Vec = (0..width * height) + .map(|value| { + let sample = u16::try_from((value * 7 + 3) % 19).expect("sample fits in u16"); + f32::from(sample) + }) + .collect(); + let expected = cpu_forward_dwt53_buffer(&samples, width, height, 1); + + let context = CudaContext::system_default().expect("CUDA context"); + let output = context + .j2k_forward_dwt53( + &samples, + u32::try_from(width).expect("width fits in u32"), + u32::try_from(height).expect("height fits in u32"), + 1, + ) + .expect("CUDA forward 5/3 DWT"); + + assert_eq!(output.execution().kernel_dispatches(), 2); + assert_eq!(output.transformed(), expected.as_slice()); + assert_eq!(output.ll_dimensions(), (3, 2)); +} + +#[test] +fn j2k_forward_dwt97_matches_cpu_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let width = 5usize; + let height = 3usize; + let samples: Vec = (0..width * height) + .map(|value| { + let sample = u16::try_from((value * 11 + 5) % 31).expect("sample fits in u16"); + f32::from(sample) - 12.0 + }) + .collect(); + let expected = cpu_forward_dwt97_buffer(&samples, width, height, 1); + + let context = CudaContext::system_default().expect("CUDA context"); + let output = context + .j2k_forward_dwt97( + &samples, + u32::try_from(width).expect("width fits in u32"), + u32::try_from(height).expect("height fits in u32"), + 1, + ) + .expect("CUDA forward 9/7 DWT"); + + assert_eq!(output.execution().kernel_dispatches(), 2); + for (actual, expected) in output.transformed().iter().zip(expected) { + assert!((*actual - expected).abs() < 0.001); + } + assert_eq!(output.ll_dimensions(), (3, 2)); +} + +#[test] +fn j2k_quantize_subband_matches_cpu_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let samples = [-3.6f32, -2.5, -0.4, 0.0, 0.49, 1.5, 3.2, 9.9]; + let context = CudaContext::system_default().expect("CUDA context"); + let reversible = context + .j2k_quantize_subband( + &samples, + CudaJ2kQuantizeJob { + step_exponent: 8, + step_mantissa: 0, + range_bits: 8, + reversible: true, + }, + ) + .expect("CUDA reversible quantize"); + assert_eq!(reversible.execution().kernel_dispatches(), 1); + assert_eq!(reversible.coefficients(), &[-4, -3, 0, 0, 0, 2, 3, 10]); + + let irreversible = context + .j2k_quantize_subband( + &samples, + CudaJ2kQuantizeJob { + step_exponent: 9, + step_mantissa: 0, + range_bits: 8, + reversible: false, + }, + ) + .expect("CUDA irreversible quantize"); + assert_eq!(irreversible.execution().kernel_dispatches(), 1); + // delta = 2^(range_bits - step_exponent) = 2^(8 - 9) = 0.5, so q = sign*floor(|s|/0.5). + // Matches native QuantStepSize::delta and JPEG 2000 T.800 Annex E. + assert_eq!(irreversible.coefficients(), &[-7, -5, 0, 0, 0, 3, 6, 19]); +} + +#[test] +fn j2k_quantize_strided_resident_subband_matches_contiguous_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let samples: Vec = (0u16..12).map(|value| f32::from(value) - 6.0).collect(); + let context = CudaContext::system_default().expect("CUDA context"); + let sample_buffer = context.upload_f32(&samples).expect("resident samples"); + let quantization = CudaJ2kQuantizeJob { + step_exponent: 8, + step_mantissa: 0, + range_bits: 8, + reversible: true, + }; + let resident = context + .j2k_quantize_subband_region_resident( + &sample_buffer, + CudaJ2kQuantizeSubbandRegionJob { + x0: 1, + y0: 1, + width: 2, + height: 2, + stride: 4, + quantization, + }, + ) + .expect("resident strided quantize"); + let contiguous = [samples[5], samples[6], samples[9], samples[10]]; + let expected = context + .j2k_quantize_subband(&contiguous, quantization) + .expect("contiguous quantize"); + + assert_eq!(resident.coefficient_count(), 4); + assert_eq!(resident.execution().kernel_dispatches(), 1); + assert_eq!( + resident + .download_coefficients() + .expect("download resident quantized coefficients"), + expected.coefficients() + ); +} + +fn cpu_forward_dwt53_buffer(samples: &[f32], width: usize, height: usize, levels: u8) -> Vec { + let mut buffer = samples.to_vec(); + let mut current_width = width; + let mut current_height = height; + + for _ in 0..levels { + if current_width < 2 && current_height < 2 { + break; + } + if current_height >= 2 { + let low_height = current_height.div_ceil(2); + let mut col = vec![0.0; current_height]; + for x in 0..current_width { + for y in 0..current_height { + col[y] = buffer[y * width + x]; + } + forward_lift_53(&mut col); + for y in 0..low_height { + buffer[y * width + x] = col[y * 2]; + } + for y in 0..current_height / 2 { + buffer[(low_height + y) * width + x] = col[y * 2 + 1]; + } + } + } + if current_width >= 2 { + let mut row = vec![0.0; current_width]; + for y in 0..current_height { + let row_start = y * width; + row.copy_from_slice(&buffer[row_start..row_start + current_width]); + forward_lift_53(&mut row); + let low_width = current_width.div_ceil(2); + for x in 0..low_width { + buffer[row_start + x] = row[x * 2]; + } + for x in 0..current_width / 2 { + buffer[row_start + low_width + x] = row[x * 2 + 1]; + } + } + } + current_width = current_width.div_ceil(2); + current_height = current_height.div_ceil(2); + } + + buffer +} + +fn cpu_forward_dwt97_buffer(samples: &[f32], width: usize, height: usize, levels: u8) -> Vec { + let mut buffer = samples.to_vec(); + let mut current_width = width; + let mut current_height = height; + + for _ in 0..levels { + if current_width < 2 && current_height < 2 { + break; + } + if current_height >= 2 { + let low_height = current_height.div_ceil(2); + let mut col = vec![0.0; current_height]; + for x in 0..current_width { + for y in 0..current_height { + col[y] = buffer[y * width + x]; + } + forward_lift_97(&mut col); + for y in 0..low_height { + buffer[y * width + x] = col[y * 2]; + } + for y in 0..current_height / 2 { + buffer[(low_height + y) * width + x] = col[y * 2 + 1]; + } + } + } + if current_width >= 2 { + let mut row = vec![0.0; current_width]; + for y in 0..current_height { + let row_start = y * width; + row.copy_from_slice(&buffer[row_start..row_start + current_width]); + forward_lift_97(&mut row); + let low_width = current_width.div_ceil(2); + for x in 0..low_width { + buffer[row_start + x] = row[x * 2]; + } + for x in 0..current_width / 2 { + buffer[row_start + low_width + x] = row[x * 2 + 1]; + } + } + } + current_width = current_width.div_ceil(2); + current_height = current_height.div_ceil(2); + } + + buffer +} + +fn forward_lift_53(data: &mut [f32]) { + let n = data.len(); + if n < 2 { + return; + } + + let last_even = if n.is_multiple_of(2) { n - 2 } else { n - 1 }; + for i in (1..n).step_by(2) { + let left = data[i - 1]; + let right = if i + 1 < n { + data[i + 1] + } else { + data[last_even] + }; + data[i] -= ((left + right) * 0.5).floor(); + } + + for i in (0..n).step_by(2) { + let left = if i > 0 { data[i - 1] } else { data[1] }; + let right = if i + 1 < n { data[i + 1] } else { left }; + data[i] += ((left + right) * 0.25 + 0.5).floor(); + } +} + +fn forward_lift_97(data: &mut [f32]) { + const ALPHA: f32 = -1.586_134_3; + const BETA: f32 = -0.052_980_117; + const GAMMA: f32 = 0.882_911_1; + const DELTA: f32 = 0.443_506_87; + const KAPPA: f32 = 1.230_174_1; + const INV_KAPPA: f32 = 1.0 / KAPPA; + + let n = data.len(); + if n < 2 { + return; + } + + let last_even = if n.is_multiple_of(2) { n - 2 } else { n - 1 }; + for i in (1..n).step_by(2) { + let left = data[i - 1]; + let right = if i + 1 < n { + data[i + 1] + } else { + data[last_even] + }; + data[i] += ALPHA * (left + right); + } + for i in (0..n).step_by(2) { + let left = if i > 0 { data[i - 1] } else { data[1] }; + let right = if i + 1 < n { data[i + 1] } else { left }; + data[i] += BETA * (left + right); + } + for i in (1..n).step_by(2) { + let left = data[i - 1]; + let right = if i + 1 < n { + data[i + 1] + } else { + data[last_even] + }; + data[i] += GAMMA * (left + right); + } + for i in (0..n).step_by(2) { + let left = if i > 0 { data[i - 1] } else { data[1] }; + let right = if i + 1 < n { data[i + 1] } else { left }; + data[i] += DELTA * (left + right); + } + for i in (0..n).step_by(2) { + data[i] *= INV_KAPPA; + } + for i in (1..n).step_by(2) { + data[i] *= KAPPA; + } +} diff --git a/crates/j2k-cuda-runtime/src/transcode.rs b/crates/j2k-cuda-runtime/src/transcode.rs new file mode 100644 index 00000000..c06e1e3d --- /dev/null +++ b/crates/j2k-cuda-runtime/src/transcode.rs @@ -0,0 +1,1659 @@ +use crate::{ + build_flags::{ + dwt97_fused_column_quantize_disabled, DWT97_ROW_LIFT_COOP_ROWS_PER_BLOCK, + DWT97_ROW_LIFT_COOP_THREADS_X, DWT97_ROW_LIFT_MAX_WIDTH, + PINNED_POOLED_I16_UPLOAD_MAX_BYTES, TRANSCODE_PTX_BUILT_FROM_CUDA, + }, + bytes::i16_slice_as_bytes, + context::CudaContext, + driver::CuFunction, + error::CudaError, + execution::cuda_kernel_param, + j2k_encode::CudaDwt97BatchStageTimings, + kernels::{ + self, copy_u8_launch_geometry, j2k_dwt53_launch_geometry, with_grid_y, with_grid_z, + CudaKernel, + }, + memory::{pooled_device_buffer, CudaBufferPool, CudaDeviceBuffer, CudaPooledDeviceBuffer}, +}; +use std::os::raw::c_uint; + +/// Reversible 5/3 transcode bands downloaded from the device. Layout matches +/// `j2k_transcode::accelerator::ReversibleDwt53FirstLevel`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CudaTranscodeReversible53Bands { + /// Low-horizontal, low-vertical band (`low_width * low_height`). + pub ll: Vec, + /// High-horizontal, low-vertical band (`high_width * low_height`). + pub hl: Vec, + /// Low-horizontal, high-vertical band (`low_width * high_height`). + pub lh: Vec, + /// High-horizontal, high-vertical band (`high_width * high_height`). + pub hh: Vec, + /// Width of horizontally low-pass bands. + pub low_width: usize, + /// Height of vertically low-pass bands. + pub low_height: usize, + /// Width of horizontally high-pass bands. + pub high_width: usize, + /// Height of vertically high-pass bands. + pub high_height: usize, +} + +#[derive(Clone, Copy)] +pub(crate) struct Reversible53Dims { + pub(crate) block_cols: i32, + pub(crate) width: i32, + pub(crate) height: i32, + pub(crate) low_width: i32, + pub(crate) high_width: i32, +} + +#[derive(Clone, Copy)] +pub(crate) struct DctBlockGrid { + pub(crate) block_count: usize, + pub(crate) expected_coeffs: usize, + pub(crate) low_width: usize, + pub(crate) low_height: usize, + pub(crate) high_width: usize, + pub(crate) high_height: usize, + pub(crate) dims: Reversible53Dims, +} + +impl CudaContext { + /// Compute one reversible integer 5/3 level directly from dequantized 8x8 + /// DCT blocks, bit-exact with the `j2k-transcode` scalar oracle. + /// + /// `dequantized_blocks` holds `block_cols * block_rows` natural-order blocks + /// of 64 `i16` coefficients. `width`/`height` are the logical component + /// dimensions (<= `block_cols*8` / `block_rows*8`). + #[allow(clippy::too_many_lines)] + pub fn j2k_transcode_reversible_dwt53( + &self, + dequantized_blocks: &[i16], + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + ) -> Result { + ensure_transcode_runtime_ptx_available()?; + let grid = validate_dct_block_grid( + block_cols, + block_rows, + width, + height, + 1, + dequantized_blocks.len(), + "reversible 5/3 transcode job has unsupported grid geometry", + )?; + let DctBlockGrid { + block_count, + expected_coeffs, + low_width, + low_height, + high_width, + high_height, + dims, + } = grid; + + self.inner.set_current()?; + + let alloc_i32 = |count: usize| -> Result { + let bytes = count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: count })?; + self.allocate(bytes) + }; + let samples = alloc_i32(expected_coeffs)?; + let v_low = alloc_i32(width * low_height)?; + let v_high = alloc_i32(width * high_height)?; + let ll = alloc_i32(low_width * low_height)?; + let hl = alloc_i32(high_width * low_height)?; + let lh = alloc_i32(low_width * high_height)?; + let hh = alloc_i32(high_width * high_height)?; + + // SAFETY: `dequantized_blocks` is a live `&[i16]`; reinterpreting it as a + // byte slice of `len * 2` bytes for upload is a read-only view with the + // same lifetime and no alignment requirement on the destination. + let block_bytes: &[u8] = unsafe { + std::slice::from_raw_parts( + dequantized_blocks.as_ptr().cast::(), + std::mem::size_of_val(dequantized_blocks), + ) + }; + let blocks_dev = self.upload(block_bytes)?; + + self.launch_transcode_reversible53_idct(&blocks_dev, &samples, block_count)?; + if low_height > 0 { + self.launch_transcode_reversible53_vertical( + CudaKernel::TranscodeReversible53VerticalLow, + &samples, + dims, + &v_low, + checked_i32(low_height)?, + )?; + self.launch_transcode_reversible53_horizontal( + CudaKernel::TranscodeReversible53HorizontalLow, + &v_low, + dims, + checked_i32(low_height)?, + &ll, + &hl, + )?; + } + if high_height > 0 { + self.launch_transcode_reversible53_vertical( + CudaKernel::TranscodeReversible53VerticalHigh, + &samples, + dims, + &v_high, + checked_i32(high_height)?, + )?; + self.launch_transcode_reversible53_horizontal( + CudaKernel::TranscodeReversible53HorizontalHigh, + &v_high, + dims, + checked_i32(high_height)?, + &lh, + &hh, + )?; + } + + Ok(CudaTranscodeReversible53Bands { + ll: Self::download_i32_band(&ll, low_width * low_height)?, + hl: Self::download_i32_band(&hl, high_width * low_height)?, + lh: Self::download_i32_band(&lh, low_width * high_height)?, + hh: Self::download_i32_band(&hh, high_width * high_height)?, + low_width, + low_height, + high_width, + high_height, + }) + } + + fn launch_transcode_reversible53_idct( + &self, + blocks: &CudaDeviceBuffer, + samples: &CudaDeviceBuffer, + block_count: usize, + ) -> Result<(), CudaError> { + if block_count == 0 { + return Ok(()); + } + let function = self.transcode_kernel_function(CudaKernel::TranscodeReversible53Idct)?; + let mut blocks_ptr = blocks.device_ptr(); + let mut samples_ptr = samples.device_ptr(); + let mut count = u32::try_from(block_count) + .map_err(|_| CudaError::LengthTooLarge { len: block_count })?; + let mut params = cuda_kernel_params!(blocks_ptr, samples_ptr, count); + let geometry = copy_u8_launch_geometry(block_count) + .ok_or(CudaError::LengthTooLarge { len: block_count })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_transcode_reversible53_vertical( + &self, + kernel: CudaKernel, + samples: &CudaDeviceBuffer, + dims: Reversible53Dims, + out: &CudaDeviceBuffer, + out_rows: i32, + ) -> Result<(), CudaError> { + let function = self.transcode_kernel_function(kernel)?; + let mut samples_ptr = samples.device_ptr(); + let mut block_cols = dims.block_cols; + let mut width = dims.width; + let mut height = dims.height; + let mut out_ptr = out.device_ptr(); + let mut rows = out_rows; + let mut params = cuda_kernel_params!(samples_ptr, block_cols, width, height, out_ptr, rows); + let grid_w = u32::try_from(dims.width).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + let grid_h = u32::try_from(out_rows).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + let geometry = j2k_dwt53_launch_geometry(grid_w, grid_h) + .ok_or(CudaError::LengthTooLarge { len: 0 })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_transcode_reversible53_horizontal( + &self, + kernel: CudaKernel, + rows_buffer: &CudaDeviceBuffer, + dims: Reversible53Dims, + n_rows: i32, + low_out: &CudaDeviceBuffer, + high_out: &CudaDeviceBuffer, + ) -> Result<(), CudaError> { + let row_count = + usize::try_from(n_rows).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + if row_count == 0 { + return Ok(()); + } + let function = self.transcode_kernel_function(kernel)?; + let mut rows_ptr = rows_buffer.device_ptr(); + let mut width = dims.width; + let mut rows = n_rows; + let mut low_width = dims.low_width; + let mut high_width = dims.high_width; + let mut low_ptr = low_out.device_ptr(); + let mut high_ptr = high_out.device_ptr(); + let mut params = + cuda_kernel_params!(rows_ptr, width, rows, low_width, high_width, low_ptr, high_ptr); + let geometry = copy_u8_launch_geometry(row_count) + .ok_or(CudaError::LengthTooLarge { len: row_count })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn transcode_kernel_function(&self, kernel: CudaKernel) -> Result { + #[cfg(feature = "cuda-oxide-transcode")] + { + if crate::build_flags::cuda_oxide_transcode_enabled() + && kernel.is_cuda_oxide_transcode_stage() + { + return self.inner.cuda_oxide_transcode_kernel_function(kernel); + } + } + self.inner.kernel_function(kernel) + } +} + +/// Irreversible single-level 9/7 transcode bands downloaded from the device. +/// Device math is f32; callers widen to f64 (parity is within tolerance). +#[derive(Clone, Debug, PartialEq)] +pub struct CudaTranscodeDwt97Bands { + /// Low-horizontal, low-vertical band (`low_width * low_height`). + pub ll: Vec, + /// High-horizontal, low-vertical band (`high_width * low_height`). + pub hl: Vec, + /// Low-horizontal, high-vertical band (`low_width * high_height`). + pub lh: Vec, + /// High-horizontal, high-vertical band (`high_width * high_height`). + pub hh: Vec, + /// Width of horizontally low-pass bands. + pub low_width: usize, + /// Height of vertically low-pass bands. + pub low_height: usize, + /// Width of horizontally high-pass bands. + pub high_width: usize, + /// Height of vertically high-pass bands. + pub high_height: usize, +} + +/// Per-subband inverse step sizes and code-block geometry for the fused 9/7 +/// code-block quantization batch. The dispatch layer derives the deltas from +/// the `j2k-transcode` code-block oracle so the numbers stay authoritative. +#[derive(Clone, Copy, Debug)] +pub struct CudaHtj2k97QuantizeParams { + /// `1/Δ` for the LL subband. + pub inv_delta_ll: f32, + /// `1/Δ` for the HL subband. + pub inv_delta_hl: f32, + /// `1/Δ` for the LH subband. + pub inv_delta_lh: f32, + /// `1/Δ` for the HH subband. + pub inv_delta_hh: f32, + /// Code-block width in coefficients (`1 << (code_block_width_exp + 2)`). + pub cb_width: usize, + /// Code-block height in coefficients (`1 << (code_block_height_exp + 2)`). + pub cb_height: usize, +} + +#[derive(Clone, Copy)] +pub(crate) struct Dwt97CodeblockBandBuffers<'a> { + pub(crate) ll: &'a CudaDeviceBuffer, + pub(crate) hl: &'a CudaDeviceBuffer, + pub(crate) lh: &'a CudaDeviceBuffer, + pub(crate) hh: &'a CudaDeviceBuffer, +} + +/// Per-item raw code-block-major quantized 9/7 bands from the fused batch. +/// +/// Each band concatenates `item_count` per-item subband buffers in code-block +/// -major order (outer code-block row, inner code-block column, each block +/// row-major), matching the `j2k-transcode` code-block oracle layout. The +/// dispatch layer reslices these into prequantized HTJ2K components. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CudaHtj2k97CodeblockBands { + /// LL subband (`item_count * low_width * low_height`). + pub ll: Vec, + /// HL subband (`item_count * high_width * low_height`). + pub hl: Vec, + /// LH subband (`item_count * low_width * high_height`). + pub lh: Vec, + /// HH subband (`item_count * high_width * high_height`). + pub hh: Vec, + /// Number of items in the batch. + pub item_count: usize, + /// Width of horizontally low-pass bands. + pub low_width: usize, + /// Height of vertically low-pass bands. + pub low_height: usize, + /// Width of horizontally high-pass bands. + pub high_width: usize, + /// Height of vertically high-pass bands. + pub high_height: usize, +} + +/// Device-resident per-item raw code-block-major quantized 9/7 bands from the +/// fused transcode batch. +#[derive(Debug)] +pub struct CudaHtj2k97DeviceCodeblockBands { + /// LL subband (`item_count * low_width * low_height`). + pub ll: CudaPooledDeviceBuffer, + /// HL subband (`item_count * high_width * low_height`). + pub hl: CudaPooledDeviceBuffer, + /// LH subband (`item_count * low_width * high_height`). + pub lh: CudaPooledDeviceBuffer, + /// HH subband (`item_count * high_width * high_height`). + pub hh: CudaPooledDeviceBuffer, + /// Number of items in the batch. + pub item_count: usize, + /// Width of horizontally low-pass bands. + pub low_width: usize, + /// Height of vertically low-pass bands. + pub low_height: usize, + /// Width of horizontally high-pass bands. + pub high_width: usize, + /// Height of vertically high-pass bands. + pub high_height: usize, +} + +/// Device-resident 9/7 batch bands produced by the shared staged pipeline. +pub(crate) struct Dwt97BatchDeviceBands { + pub(crate) ll: CudaPooledDeviceBuffer, + pub(crate) lh: CudaPooledDeviceBuffer, + pub(crate) hl: CudaPooledDeviceBuffer, + pub(crate) hh: CudaPooledDeviceBuffer, + pub(crate) low_width: usize, + pub(crate) low_height: usize, + pub(crate) high_width: usize, + pub(crate) high_height: usize, +} + +#[derive(Clone, Copy)] +pub(crate) enum Dwt97BatchInput<'a> { + F32(&'a [f32]), + I16(&'a [i16]), +} + +fn transcode_runtime_ptx_available() -> bool { + if TRANSCODE_PTX_BUILT_FROM_CUDA { + return true; + } + #[cfg(feature = "cuda-oxide-transcode")] + { + crate::build_flags::cuda_oxide_transcode_enabled() + && crate::build_flags::CUDA_OXIDE_TRANSCODE_PTX_BUILT + } + #[cfg(not(feature = "cuda-oxide-transcode"))] + { + false + } +} + +fn ensure_transcode_runtime_ptx_available() -> Result<(), CudaError> { + if transcode_runtime_ptx_available() { + Ok(()) + } else { + Err(CudaError::InvalidArgument { + message: + "CUDA transcode kernels were not built and cuda-oxide transcode PTX is not enabled/built" + .to_string(), + }) + } +} + +impl Dwt97BatchInput<'_> { + fn len(self) -> usize { + match self { + Self::F32(blocks) => blocks.len(), + Self::I16(blocks) => blocks.len(), + } + } + + fn upload(self, pool: &CudaBufferPool) -> Result { + match self { + Self::F32(blocks) => pool.upload_f32(blocks), + Self::I16(blocks) => { + let bytes = i16_slice_as_bytes(blocks); + if should_use_pinned_pooled_i16_upload(bytes.len()) { + pool.upload_pinned(bytes) + } else { + pool.upload(bytes) + } + } + } + } +} + +pub(crate) fn should_use_pinned_pooled_i16_upload(byte_len: usize) -> bool { + byte_len <= PINNED_POOLED_I16_UPLOAD_MAX_BYTES +} + +pub(crate) fn validate_dct_block_grid( + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + item_count: usize, + coeff_len: usize, + invalid_message: &'static str, +) -> Result { + let block_count = block_cols + .checked_mul(block_rows) + .ok_or(CudaError::LengthTooLarge { len: block_cols })?; + let covered_w = block_cols + .checked_mul(8) + .ok_or(CudaError::LengthTooLarge { len: block_cols })?; + let covered_h = block_rows + .checked_mul(8) + .ok_or(CudaError::LengthTooLarge { len: block_rows })?; + let per_item_coeffs = block_count + .checked_mul(64) + .ok_or(CudaError::LengthTooLarge { len: block_count })?; + let expected_coeffs = + per_item_coeffs + .checked_mul(item_count) + .ok_or(CudaError::LengthTooLarge { + len: per_item_coeffs, + })?; + if item_count == 0 + || width == 0 + || height == 0 + || width > covered_w + || height > covered_h + || coeff_len != expected_coeffs + { + return Err(CudaError::InvalidArgument { + message: invalid_message.to_string(), + }); + } + + let low_width = width.div_ceil(2); + let low_height = height.div_ceil(2); + let high_width = width / 2; + let high_height = height / 2; + Ok(DctBlockGrid { + block_count, + expected_coeffs, + low_width, + low_height, + high_width, + high_height, + dims: Reversible53Dims { + block_cols: checked_i32(block_cols)?, + width: checked_i32(width)?, + height: checked_i32(height)?, + low_width: checked_i32(low_width)?, + high_width: checked_i32(high_width)?, + }, + }) +} + +pub(crate) fn checked_i32(value: usize) -> Result { + i32::try_from(value).map_err(|_| CudaError::LengthTooLarge { len: value }) +} + +impl CudaContext { + /// Compute one irreversible single-level 9/7 transform directly from + /// dequantized 8x8 DCT blocks (`block_cols * block_rows` blocks of 64 `f32` + /// natural-order coefficients), matching the `j2k-transcode` scalar + /// oracle within f32 tolerance. + #[allow(clippy::too_many_lines)] + pub fn j2k_transcode_dwt97( + &self, + blocks: &[f32], + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + ) -> Result { + ensure_transcode_runtime_ptx_available()?; + let grid = validate_dct_block_grid( + block_cols, + block_rows, + width, + height, + 1, + blocks.len(), + "9/7 transcode job has unsupported grid geometry", + )?; + let DctBlockGrid { + expected_coeffs: _, + low_width, + low_height, + high_width, + high_height, + dims, + .. + } = grid; + + self.inner.set_current()?; + + let alloc_f32 = |count: usize| -> Result { + let bytes = count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: count })?; + self.allocate(bytes) + }; + let spatial = alloc_f32(width * height)?; + let row_low = alloc_f32(height * low_width)?; + let row_high = alloc_f32(height * high_width)?; + let ll = alloc_f32(low_width * low_height)?; + let lh = alloc_f32(low_width * high_height)?; + let hl = alloc_f32(high_width * low_height)?; + let hh = alloc_f32(high_width * high_height)?; + + let blocks_dev = self.upload_f32(blocks)?; + + self.launch_transcode_dwt97_idct(dims, &blocks_dev, &spatial)?; + self.launch_transcode_dwt97_row_lift(dims, &spatial, &row_low, &row_high)?; + if dims.low_width > 0 { + self.launch_transcode_dwt97_column_lift( + &row_low, + dims.low_width, + dims.height, + &ll, + &lh, + )?; + } + if dims.high_width > 0 { + self.launch_transcode_dwt97_column_lift( + &row_high, + dims.high_width, + dims.height, + &hl, + &hh, + )?; + } + + Ok(CudaTranscodeDwt97Bands { + ll: Self::download_f32_band(&ll, low_width * low_height)?, + hl: Self::download_f32_band(&hl, high_width * low_height)?, + lh: Self::download_f32_band(&lh, low_width * high_height)?, + hh: Self::download_f32_band(&hh, high_width * high_height)?, + low_width, + low_height, + high_width, + high_height, + }) + } + + fn launch_transcode_dwt97_idct( + &self, + dims: Reversible53Dims, + blocks: &CudaDeviceBuffer, + spatial: &CudaDeviceBuffer, + ) -> Result<(), CudaError> { + let function = self.transcode_kernel_function(CudaKernel::TranscodeDwt97Idct)?; + let mut blocks_ptr = blocks.device_ptr(); + let mut block_cols = dims.block_cols; + let mut width = dims.width; + let mut height = dims.height; + let mut spatial_ptr = spatial.device_ptr(); + let mut params = cuda_kernel_params!(blocks_ptr, block_cols, width, height, spatial_ptr); + let grid_w = u32::try_from(dims.width).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + let grid_h = + u32::try_from(dims.height).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + let geometry = j2k_dwt53_launch_geometry(grid_w, grid_h) + .ok_or(CudaError::LengthTooLarge { len: 0 })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_transcode_dwt97_row_lift( + &self, + dims: Reversible53Dims, + spatial: &CudaDeviceBuffer, + row_low: &CudaDeviceBuffer, + row_high: &CudaDeviceBuffer, + ) -> Result<(), CudaError> { + let function = self.transcode_kernel_function(CudaKernel::TranscodeDwt97RowLift)?; + let mut spatial_ptr = spatial.device_ptr(); + let mut width = dims.width; + let mut height = dims.height; + let mut low_width = dims.low_width; + let mut high_width = dims.high_width; + let mut low_ptr = row_low.device_ptr(); + let mut high_ptr = row_high.device_ptr(); + let mut params = cuda_kernel_params!( + spatial_ptr, + width, + height, + low_width, + high_width, + low_ptr, + high_ptr + ); + let rows = + usize::try_from(dims.height).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + let geometry = + copy_u8_launch_geometry(rows).ok_or(CudaError::LengthTooLarge { len: rows })?; + self.launch_kernel(function, geometry, &mut params) + } + + fn launch_transcode_dwt97_column_lift( + &self, + rows_buffer: &CudaDeviceBuffer, + band_width: i32, + height: i32, + low_out: &CudaDeviceBuffer, + high_out: &CudaDeviceBuffer, + ) -> Result<(), CudaError> { + let columns = + usize::try_from(band_width).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + if columns == 0 { + return Ok(()); + } + let function = self.transcode_kernel_function(CudaKernel::TranscodeDwt97ColumnLift)?; + let mut rows_ptr = rows_buffer.device_ptr(); + let mut band = band_width; + let mut rows = height; + let mut low_ptr = low_out.device_ptr(); + let mut high_ptr = high_out.device_ptr(); + let mut params = cuda_kernel_params!(rows_ptr, band, rows, low_ptr, high_ptr); + let geometry = + copy_u8_launch_geometry(columns).ok_or(CudaError::LengthTooLarge { len: columns })?; + self.launch_kernel(function, geometry, &mut params) + } +} + +impl CudaContext { + /// Compute a same-geometry batch of irreversible single-level 9/7 transforms + /// with one batched launch per stage, returning per-item bands plus real + /// backend stage timings. All jobs must share geometry (`block_cols`, + /// `block_rows`, `width`, `height`); `blocks` is the items' natural-order + /// `f32` coefficients laid out contiguously (`item_count * block_cols * + /// block_rows * 64`). Bit-identical to running `j2k_transcode_dwt97` per item. + #[allow(clippy::similar_names)] + pub fn j2k_transcode_dwt97_batch( + &self, + blocks: &[f32], + item_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + ) -> Result<(Vec, CudaDwt97BatchStageTimings), CudaError> { + let pool = self.buffer_pool(); + self.j2k_transcode_dwt97_batch_with_pool( + blocks, item_count, block_cols, block_rows, width, height, &pool, + ) + } + + /// Compute a same-geometry batch of irreversible single-level 9/7 transforms + /// while reusing device buffers from `pool` for transient stage storage. + #[allow(clippy::too_many_arguments, clippy::similar_names)] + pub fn j2k_transcode_dwt97_batch_with_pool( + &self, + blocks: &[f32], + item_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + pool: &CudaBufferPool, + ) -> Result<(Vec, CudaDwt97BatchStageTimings), CudaError> { + let (bands, pack_upload_us, idct_row_lift_us, column_lift_us) = self + .transcode_dwt97_batch_to_device( + blocks, item_count, block_cols, block_rows, width, height, pool, + )?; + let Dwt97BatchDeviceBands { + ll, + lh, + hl, + hh, + low_width, + low_height, + high_width, + high_height, + } = bands; + + let ll_size = low_width * low_height; + let lh_size = low_width * high_height; + let hl_size = high_width * low_height; + let hh_size = high_width * high_height; + + let (outputs, readback_us) = self.time_default_stream_us(|| { + let ll_all = Self::download_pooled_f32_band(&ll, item_count * ll_size)?; + let lh_all = Self::download_pooled_f32_band(&lh, item_count * lh_size)?; + let hl_all = Self::download_pooled_f32_band(&hl, item_count * hl_size)?; + let hh_all = Self::download_pooled_f32_band(&hh, item_count * hh_size)?; + let mut outputs = Vec::with_capacity(item_count); + for item in 0..item_count { + outputs.push(CudaTranscodeDwt97Bands { + ll: ll_all[item * ll_size..(item + 1) * ll_size].to_vec(), + hl: hl_all[item * hl_size..(item + 1) * hl_size].to_vec(), + lh: lh_all[item * lh_size..(item + 1) * lh_size].to_vec(), + hh: hh_all[item * hh_size..(item + 1) * hh_size].to_vec(), + low_width, + low_height, + high_width, + high_height, + }); + } + Ok(outputs) + })?; + + Ok(( + outputs, + CudaDwt97BatchStageTimings { + pack_upload_us, + idct_row_lift_us, + column_lift_us, + quantize_codeblock_us: 0, + ht_encode_us: 0, + ht_codeblock_dispatches: 0, + readback_us, + }, + )) + } + + /// Compute a same-geometry batch directly into device-resident + /// prequantized HTJ2K code-block coefficients: staged 9/7 followed by + /// per-subband deadzone quantization into code-block-major `i32` layout. + /// `params` carries the per-subband inverse step sizes and the code-block + /// geometry. + #[allow(clippy::too_many_arguments)] + pub fn j2k_transcode_htj2k97_codeblock_batch_resident( + &self, + blocks: &[f32], + item_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + params: CudaHtj2k97QuantizeParams, + ) -> Result<(CudaHtj2k97DeviceCodeblockBands, CudaDwt97BatchStageTimings), CudaError> { + let pool = self.buffer_pool(); + self.j2k_transcode_htj2k97_codeblock_batch_resident_with_pool( + blocks, item_count, block_cols, block_rows, width, height, params, &pool, + ) + } + + /// Compute a same-geometry batch directly into device-resident + /// prequantized HTJ2K code-block coefficients while reusing transient stage + /// buffers from `pool`. + #[allow(clippy::similar_names, clippy::too_many_arguments)] + pub fn j2k_transcode_htj2k97_codeblock_batch_resident_with_pool( + &self, + blocks: &[f32], + item_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + params: CudaHtj2k97QuantizeParams, + pool: &CudaBufferPool, + ) -> Result<(CudaHtj2k97DeviceCodeblockBands, CudaDwt97BatchStageTimings), CudaError> { + let (bands, pack_upload_us, idct_row_lift_us, column_lift_us) = self + .transcode_dwt97_batch_to_device( + blocks, item_count, block_cols, block_rows, width, height, pool, + )?; + let low_width = bands.low_width; + let low_height = bands.low_height; + let high_width = bands.high_width; + let high_height = bands.high_height; + let items = + u32::try_from(item_count).map_err(|_| CudaError::LengthTooLarge { len: item_count })?; + + let alloc_i32 = |count: usize| -> Result { + let bytes = count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: count })?; + pool.take(bytes) + }; + let ll_size = low_width * low_height; + let lh_size = low_width * high_height; + let hl_size = high_width * low_height; + let hh_size = high_width * high_height; + + let ll_q = alloc_i32(item_count * ll_size)?; + let lh_q = alloc_i32(item_count * lh_size)?; + let hl_q = alloc_i32(item_count * hl_size)?; + let hh_q = alloc_i32(item_count * hh_size)?; + + let ((), quantize_codeblock_us) = self.time_default_stream_us(|| { + self.launch_transcode_dwt97_quantize_codeblock_bands( + &bands, + Dwt97CodeblockBandBuffers { + ll: pooled_device_buffer(&ll_q)?, + hl: pooled_device_buffer(&hl_q)?, + lh: pooled_device_buffer(&lh_q)?, + hh: pooled_device_buffer(&hh_q)?, + }, + params, + items, + ) + })?; + + Ok(( + CudaHtj2k97DeviceCodeblockBands { + ll: ll_q, + hl: hl_q, + lh: lh_q, + hh: hh_q, + item_count, + low_width, + low_height, + high_width, + high_height, + }, + CudaDwt97BatchStageTimings { + pack_upload_us, + idct_row_lift_us, + column_lift_us, + quantize_codeblock_us, + ht_encode_us: 0, + ht_codeblock_dispatches: 0, + readback_us: 0, + }, + )) + } + + /// Compute a same-geometry i16 batch directly into device-resident + /// prequantized HTJ2K code-block coefficients while reusing transient stage + /// buffers from `pool`. + #[allow(clippy::similar_names, clippy::too_many_arguments)] + pub fn j2k_transcode_htj2k97_codeblock_i16_batch_resident_with_pool( + &self, + blocks: &[i16], + item_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + params: CudaHtj2k97QuantizeParams, + pool: &CudaBufferPool, + ) -> Result<(CudaHtj2k97DeviceCodeblockBands, CudaDwt97BatchStageTimings), CudaError> { + if !dwt97_fused_column_quantize_disabled() { + return self.j2k_transcode_htj2k97_codeblock_i16_batch_resident_fused_with_pool( + blocks, item_count, block_cols, block_rows, width, height, params, pool, + ); + } + + let (bands, pack_upload_us, idct_row_lift_us, column_lift_us) = self + .transcode_dwt97_i16_batch_to_device( + blocks, item_count, block_cols, block_rows, width, height, pool, + )?; + let low_width = bands.low_width; + let low_height = bands.low_height; + let high_width = bands.high_width; + let high_height = bands.high_height; + let items = + u32::try_from(item_count).map_err(|_| CudaError::LengthTooLarge { len: item_count })?; + + let alloc_i32 = |count: usize| -> Result { + let bytes = count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: count })?; + pool.take(bytes) + }; + let ll_size = low_width * low_height; + let lh_size = low_width * high_height; + let hl_size = high_width * low_height; + let hh_size = high_width * high_height; + + let ll_q = alloc_i32(item_count * ll_size)?; + let lh_q = alloc_i32(item_count * lh_size)?; + let hl_q = alloc_i32(item_count * hl_size)?; + let hh_q = alloc_i32(item_count * hh_size)?; + + let ((), quantize_codeblock_us) = self.time_default_stream_us(|| { + self.launch_transcode_dwt97_quantize_codeblock_bands( + &bands, + Dwt97CodeblockBandBuffers { + ll: pooled_device_buffer(&ll_q)?, + hl: pooled_device_buffer(&hl_q)?, + lh: pooled_device_buffer(&lh_q)?, + hh: pooled_device_buffer(&hh_q)?, + }, + params, + items, + ) + })?; + + Ok(( + CudaHtj2k97DeviceCodeblockBands { + ll: ll_q, + hl: hl_q, + lh: lh_q, + hh: hh_q, + item_count, + low_width, + low_height, + high_width, + high_height, + }, + CudaDwt97BatchStageTimings { + pack_upload_us, + idct_row_lift_us, + column_lift_us, + quantize_codeblock_us, + ht_encode_us: 0, + ht_codeblock_dispatches: 0, + readback_us: 0, + }, + )) + } + + #[allow( + clippy::similar_names, + clippy::too_many_arguments, + clippy::too_many_lines + )] + fn j2k_transcode_htj2k97_codeblock_i16_batch_resident_fused_with_pool( + &self, + blocks: &[i16], + item_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + params: CudaHtj2k97QuantizeParams, + pool: &CudaBufferPool, + ) -> Result<(CudaHtj2k97DeviceCodeblockBands, CudaDwt97BatchStageTimings), CudaError> { + ensure_transcode_runtime_ptx_available()?; + let grid = validate_dct_block_grid( + block_cols, + block_rows, + width, + height, + item_count, + blocks.len(), + "9/7 transcode batch has unsupported grid geometry", + )?; + let DctBlockGrid { + block_count, + low_width, + low_height, + high_width, + high_height, + dims, + .. + } = grid; + let items = + u32::try_from(item_count).map_err(|_| CudaError::LengthTooLarge { len: item_count })?; + let blocks_per_item = checked_i32(block_count)?; + let low_height_i32 = checked_i32(low_height)?; + let high_height_i32 = checked_i32(high_height)?; + let cb_w = checked_i32(params.cb_width)?; + let cb_h = checked_i32(params.cb_height)?; + + self.inner.set_current()?; + + let alloc_f32 = |count: usize| -> Result { + let bytes = count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: count })?; + pool.take(bytes) + }; + let alloc_i32 = |count: usize| -> Result { + let bytes = count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: count })?; + pool.take(bytes) + }; + let (buffers, pack_upload_us) = self.time_default_stream_us(|| { + let spatial = alloc_f32(item_count * width * height)?; + let row_low = alloc_f32(item_count * height * low_width)?; + let row_high = alloc_f32(item_count * height * high_width)?; + let blocks_dev = Dwt97BatchInput::I16(blocks).upload(pool)?; + Ok((spatial, row_low, row_high, blocks_dev)) + })?; + let (spatial, row_low, row_high, blocks_dev) = buffers; + + let ll_size = low_width * low_height; + let lh_size = low_width * high_height; + let hl_size = high_width * low_height; + let hh_size = high_width * high_height; + + let ll_q = alloc_i32(item_count * ll_size)?; + let lh_q = alloc_i32(item_count * lh_size)?; + let hl_q = alloc_i32(item_count * hl_size)?; + let hh_q = alloc_i32(item_count * hh_size)?; + + let ((), idct_row_lift_us) = self.time_default_stream_us(|| { + self.launch_transcode_dwt97_idct_batch_kernel( + CudaKernel::TranscodeDwt97IdctI16Batch, + dims, + blocks_per_item, + items, + pooled_device_buffer(&blocks_dev)?, + pooled_device_buffer(&spatial)?, + )?; + self.launch_transcode_dwt97_row_lift_batch( + dims, + items, + pooled_device_buffer(&spatial)?, + pooled_device_buffer(&row_low)?, + pooled_device_buffer(&row_high)?, + )?; + Ok(()) + })?; + + let ((), column_quantize_us) = self.time_default_stream_us(|| { + if dims.low_width > 0 { + self.launch_transcode_dwt97_column_lift_quantize_codeblocks_batch( + pooled_device_buffer(&row_low)?, + dims.low_width, + dims.height, + low_height_i32, + high_height_i32, + items, + pooled_device_buffer(&ll_q)?, + pooled_device_buffer(&lh_q)?, + cb_w, + cb_h, + params.inv_delta_ll, + params.inv_delta_lh, + )?; + } + if dims.high_width > 0 { + self.launch_transcode_dwt97_column_lift_quantize_codeblocks_batch( + pooled_device_buffer(&row_high)?, + dims.high_width, + dims.height, + low_height_i32, + high_height_i32, + items, + pooled_device_buffer(&hl_q)?, + pooled_device_buffer(&hh_q)?, + cb_w, + cb_h, + params.inv_delta_hl, + params.inv_delta_hh, + )?; + } + Ok(()) + })?; + + Ok(( + CudaHtj2k97DeviceCodeblockBands { + ll: ll_q, + hl: hl_q, + lh: lh_q, + hh: hh_q, + item_count, + low_width, + low_height, + high_width, + high_height, + }, + CudaDwt97BatchStageTimings { + pack_upload_us, + idct_row_lift_us, + column_lift_us: 0, + quantize_codeblock_us: column_quantize_us, + ht_encode_us: 0, + ht_codeblock_dispatches: 0, + readback_us: 0, + }, + )) + } + + /// Compute a same-geometry batch directly into host-owned prequantized + /// HTJ2K code-block coefficients: staged 9/7 followed by per-subband + /// deadzone quantization into code-block-major `i32` layout. + #[allow(clippy::too_many_arguments)] + pub fn j2k_transcode_htj2k97_codeblock_batch( + &self, + blocks: &[f32], + item_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + params: CudaHtj2k97QuantizeParams, + ) -> Result<(CudaHtj2k97CodeblockBands, CudaDwt97BatchStageTimings), CudaError> { + let pool = self.buffer_pool(); + self.j2k_transcode_htj2k97_codeblock_batch_with_pool( + blocks, item_count, block_cols, block_rows, width, height, params, &pool, + ) + } + + /// Compute a same-geometry batch directly into host-owned prequantized + /// HTJ2K code-block coefficients while reusing transient stage buffers + /// from `pool`. + #[allow(clippy::similar_names, clippy::too_many_arguments)] + pub fn j2k_transcode_htj2k97_codeblock_batch_with_pool( + &self, + blocks: &[f32], + item_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + params: CudaHtj2k97QuantizeParams, + pool: &CudaBufferPool, + ) -> Result<(CudaHtj2k97CodeblockBands, CudaDwt97BatchStageTimings), CudaError> { + let (bands, pack_upload_us, idct_row_lift_us, column_lift_us) = self + .transcode_dwt97_batch_to_device( + blocks, item_count, block_cols, block_rows, width, height, pool, + )?; + let low_width = bands.low_width; + let low_height = bands.low_height; + let high_width = bands.high_width; + let high_height = bands.high_height; + let items = + u32::try_from(item_count).map_err(|_| CudaError::LengthTooLarge { len: item_count })?; + + let alloc_i32 = |count: usize| -> Result { + let bytes = count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: count })?; + self.allocate(bytes) + }; + let ll_size = low_width * low_height; + let lh_size = low_width * high_height; + let hl_size = high_width * low_height; + let hh_size = high_width * high_height; + + let ll_q = alloc_i32(item_count * ll_size)?; + let lh_q = alloc_i32(item_count * lh_size)?; + let hl_q = alloc_i32(item_count * hl_size)?; + let hh_q = alloc_i32(item_count * hh_size)?; + + let ((), quantize_codeblock_us) = self.time_default_stream_us(|| { + self.launch_transcode_dwt97_quantize_codeblock_bands( + &bands, + Dwt97CodeblockBandBuffers { + ll: &ll_q, + hl: &hl_q, + lh: &lh_q, + hh: &hh_q, + }, + params, + items, + ) + })?; + + let (codeblocks, readback_us) = self.time_default_stream_us(|| { + Ok(CudaHtj2k97CodeblockBands { + ll: Self::download_i32_band(&ll_q, item_count * ll_size)?, + hl: Self::download_i32_band(&hl_q, item_count * hl_size)?, + lh: Self::download_i32_band(&lh_q, item_count * lh_size)?, + hh: Self::download_i32_band(&hh_q, item_count * hh_size)?, + item_count, + low_width, + low_height, + high_width, + high_height, + }) + })?; + + Ok(( + codeblocks, + CudaDwt97BatchStageTimings { + pack_upload_us, + idct_row_lift_us, + column_lift_us, + quantize_codeblock_us, + ht_encode_us: 0, + ht_codeblock_dispatches: 0, + readback_us, + }, + )) + } + + /// Run the shared staged 9/7 batch pipeline (alloc + upload, batched IDCT + + /// row lift, batched column lift) and return the device-resident bands plus + /// the three pre-readback stage timings. + #[allow(clippy::too_many_lines)] + #[allow(clippy::too_many_arguments)] + fn transcode_dwt97_batch_to_device( + &self, + blocks: &[f32], + item_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + pool: &CudaBufferPool, + ) -> Result<(Dwt97BatchDeviceBands, u128, u128, u128), CudaError> { + self.transcode_dwt97_batch_input_to_device( + Dwt97BatchInput::F32(blocks), + item_count, + block_cols, + block_rows, + width, + height, + pool, + ) + } + + #[allow(clippy::too_many_arguments)] + fn transcode_dwt97_i16_batch_to_device( + &self, + blocks: &[i16], + item_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + pool: &CudaBufferPool, + ) -> Result<(Dwt97BatchDeviceBands, u128, u128, u128), CudaError> { + self.transcode_dwt97_batch_input_to_device( + Dwt97BatchInput::I16(blocks), + item_count, + block_cols, + block_rows, + width, + height, + pool, + ) + } + + #[allow(clippy::too_many_lines)] + #[allow(clippy::too_many_arguments)] + fn transcode_dwt97_batch_input_to_device( + &self, + input: Dwt97BatchInput<'_>, + item_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + pool: &CudaBufferPool, + ) -> Result<(Dwt97BatchDeviceBands, u128, u128, u128), CudaError> { + ensure_transcode_runtime_ptx_available()?; + let grid = validate_dct_block_grid( + block_cols, + block_rows, + width, + height, + item_count, + input.len(), + "9/7 transcode batch has unsupported grid geometry", + )?; + let DctBlockGrid { + block_count, + low_width, + low_height, + high_width, + high_height, + dims, + .. + } = grid; + let items = + u32::try_from(item_count).map_err(|_| CudaError::LengthTooLarge { len: item_count })?; + let blocks_per_item = checked_i32(block_count)?; + let low_height_i32 = checked_i32(low_height)?; + let high_height_i32 = checked_i32(high_height)?; + + self.inner.set_current()?; + + let alloc_f32 = |count: usize| -> Result { + let bytes = count + .checked_mul(std::mem::size_of::()) + .ok_or(CudaError::LengthTooLarge { len: count })?; + pool.take(bytes) + }; + + // Stage: allocate batch buffers and upload all blocks. + let (buffers, pack_upload_us) = self.time_default_stream_us(|| { + let spatial = alloc_f32(item_count * width * height)?; + let row_low = alloc_f32(item_count * height * low_width)?; + let row_high = alloc_f32(item_count * height * high_width)?; + let ll = alloc_f32(item_count * low_width * low_height)?; + let lh = alloc_f32(item_count * low_width * high_height)?; + let hl = alloc_f32(item_count * high_width * low_height)?; + let hh = alloc_f32(item_count * high_width * high_height)?; + let blocks_dev = input.upload(pool)?; + Ok((spatial, row_low, row_high, ll, lh, hl, hh, blocks_dev)) + })?; + let (spatial, row_low, row_high, ll, lh, hl, hh, blocks_dev) = buffers; + + // Stage: batched separable IDCT then horizontal 9/7 row lift. + let ((), idct_row_lift_us) = self.time_default_stream_us(|| { + let idct_kernel = match input { + Dwt97BatchInput::F32(_) => CudaKernel::TranscodeDwt97IdctBatch, + Dwt97BatchInput::I16(_) => CudaKernel::TranscodeDwt97IdctI16Batch, + }; + self.launch_transcode_dwt97_idct_batch_kernel( + idct_kernel, + dims, + blocks_per_item, + items, + pooled_device_buffer(&blocks_dev)?, + pooled_device_buffer(&spatial)?, + )?; + self.launch_transcode_dwt97_row_lift_batch( + dims, + items, + pooled_device_buffer(&spatial)?, + pooled_device_buffer(&row_low)?, + pooled_device_buffer(&row_high)?, + )?; + Ok(()) + })?; + + // Stage: batched vertical 9/7 column lift for both low and high rows. + let ((), column_lift_us) = self.time_default_stream_us(|| { + if dims.low_width > 0 { + self.launch_transcode_dwt97_column_lift_batch( + pooled_device_buffer(&row_low)?, + dims.low_width, + dims.height, + low_height_i32, + high_height_i32, + items, + pooled_device_buffer(&ll)?, + pooled_device_buffer(&lh)?, + )?; + } + if dims.high_width > 0 { + self.launch_transcode_dwt97_column_lift_batch( + pooled_device_buffer(&row_high)?, + dims.high_width, + dims.height, + low_height_i32, + high_height_i32, + items, + pooled_device_buffer(&hl)?, + pooled_device_buffer(&hh)?, + )?; + } + Ok(()) + })?; + + Ok(( + Dwt97BatchDeviceBands { + ll, + lh, + hl, + hh, + low_width, + low_height, + high_width, + high_height, + }, + pack_upload_us, + idct_row_lift_us, + column_lift_us, + )) + } + + fn launch_transcode_dwt97_idct_batch_kernel( + &self, + kernel: CudaKernel, + dims: Reversible53Dims, + blocks_per_item: i32, + items: u32, + blocks: &CudaDeviceBuffer, + spatial: &CudaDeviceBuffer, + ) -> Result<(), CudaError> { + let function = self.transcode_kernel_function(kernel)?; + let mut blocks_ptr = blocks.device_ptr(); + let mut block_cols = dims.block_cols; + let mut width = dims.width; + let mut height = dims.height; + let mut blocks_per_item = blocks_per_item; + let mut spatial_ptr = spatial.device_ptr(); + let mut params = cuda_kernel_params!( + blocks_ptr, + block_cols, + width, + height, + blocks_per_item, + spatial_ptr + ); + let grid_w = u32::try_from(dims.width).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + let grid_h = + u32::try_from(dims.height).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + let base = j2k_dwt53_launch_geometry(grid_w, grid_h) + .ok_or(CudaError::LengthTooLarge { len: 0 })?; + let geometry = with_grid_z(base, items); + self.launch_kernel_async(function, geometry, &mut params) + } + + fn launch_transcode_dwt97_row_lift_batch( + &self, + dims: Reversible53Dims, + items: u32, + spatial: &CudaDeviceBuffer, + row_low: &CudaDeviceBuffer, + row_high: &CudaDeviceBuffer, + ) -> Result<(), CudaError> { + if dims.width <= DWT97_ROW_LIFT_MAX_WIDTH { + return self.launch_transcode_dwt97_row_lift_batch_coop( + dims, items, spatial, row_low, row_high, + ); + } + + let function = self.transcode_kernel_function(CudaKernel::TranscodeDwt97RowLiftBatch)?; + let mut spatial_ptr = spatial.device_ptr(); + let mut width = dims.width; + let mut height = dims.height; + let mut low_width = dims.low_width; + let mut high_width = dims.high_width; + let mut low_ptr = row_low.device_ptr(); + let mut high_ptr = row_high.device_ptr(); + let mut params = cuda_kernel_params!( + spatial_ptr, + width, + height, + low_width, + high_width, + low_ptr, + high_ptr + ); + let rows = + usize::try_from(dims.height).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + let base = copy_u8_launch_geometry(rows).ok_or(CudaError::LengthTooLarge { len: rows })?; + let geometry = with_grid_y(base, items); + self.launch_kernel_async(function, geometry, &mut params) + } + + fn launch_transcode_dwt97_row_lift_batch_coop( + &self, + dims: Reversible53Dims, + items: u32, + spatial: &CudaDeviceBuffer, + row_low: &CudaDeviceBuffer, + row_high: &CudaDeviceBuffer, + ) -> Result<(), CudaError> { + let function = + self.transcode_kernel_function(CudaKernel::TranscodeDwt97RowLiftBatchCoop)?; + let mut spatial_ptr = spatial.device_ptr(); + let mut width = dims.width; + let mut height = dims.height; + let mut low_width = dims.low_width; + let mut high_width = dims.high_width; + let mut low_ptr = row_low.device_ptr(); + let mut high_ptr = row_high.device_ptr(); + let mut params = cuda_kernel_params!( + spatial_ptr, + width, + height, + low_width, + high_width, + low_ptr, + high_ptr + ); + let rows = + usize::try_from(dims.height).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + let rows_per_block = DWT97_ROW_LIFT_COOP_ROWS_PER_BLOCK as usize; + let grid_x = c_uint::try_from(rows.div_ceil(rows_per_block)) + .map_err(|_| CudaError::LengthTooLarge { len: rows })?; + let geometry = kernels::CudaLaunchGeometry { + grid: (grid_x, items, 1), + block: ( + DWT97_ROW_LIFT_COOP_THREADS_X, + DWT97_ROW_LIFT_COOP_ROWS_PER_BLOCK, + 1, + ), + }; + self.launch_kernel_async(function, geometry, &mut params) + } + + #[allow(clippy::too_many_arguments)] + fn launch_transcode_dwt97_column_lift_batch( + &self, + rows_buffer: &CudaDeviceBuffer, + band_width: i32, + height: i32, + low_height: i32, + high_height: i32, + items: u32, + low_out: &CudaDeviceBuffer, + high_out: &CudaDeviceBuffer, + ) -> Result<(), CudaError> { + let columns = + usize::try_from(band_width).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + if columns == 0 { + return Ok(()); + } + let function = self.transcode_kernel_function(CudaKernel::TranscodeDwt97ColumnLiftBatch)?; + let mut rows_ptr = rows_buffer.device_ptr(); + let mut band = band_width; + let mut rows = height; + let mut low_h = low_height; + let mut high_h = high_height; + let mut low_ptr = low_out.device_ptr(); + let mut high_ptr = high_out.device_ptr(); + let mut params = + cuda_kernel_params!(rows_ptr, band, rows, low_h, high_h, low_ptr, high_ptr); + let base = + copy_u8_launch_geometry(columns).ok_or(CudaError::LengthTooLarge { len: columns })?; + let geometry = with_grid_y(base, items); + self.launch_kernel_async(function, geometry, &mut params) + } + + #[allow(clippy::too_many_arguments)] + fn launch_transcode_dwt97_column_lift_quantize_codeblocks_batch( + &self, + rows_buffer: &CudaDeviceBuffer, + band_width: i32, + height: i32, + low_height: i32, + high_height: i32, + items: u32, + low_out: &CudaDeviceBuffer, + high_out: &CudaDeviceBuffer, + cb_width: i32, + cb_height: i32, + inv_delta_low: f32, + inv_delta_high: f32, + ) -> Result<(), CudaError> { + let columns = + usize::try_from(band_width).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + if columns == 0 { + return Ok(()); + } + let function = self.transcode_kernel_function( + CudaKernel::TranscodeDwt97ColumnLiftQuantizeCodeblocksBatch, + )?; + let mut rows_ptr = rows_buffer.device_ptr(); + let mut band = band_width; + let mut rows = height; + let mut low_h = low_height; + let mut high_h = high_height; + let mut low_ptr = low_out.device_ptr(); + let mut high_ptr = high_out.device_ptr(); + let mut cb_w = cb_width; + let mut cb_h = cb_height; + let mut inv_low = inv_delta_low; + let mut inv_high = inv_delta_high; + let mut params = cuda_kernel_params!( + rows_ptr, band, rows, low_h, high_h, low_ptr, high_ptr, cb_w, cb_h, inv_low, inv_high + ); + let base = + copy_u8_launch_geometry(columns).ok_or(CudaError::LengthTooLarge { len: columns })?; + let geometry = with_grid_y(base, items); + self.launch_kernel_async(function, geometry, &mut params) + } + + fn launch_transcode_dwt97_quantize_codeblock_bands( + &self, + bands: &Dwt97BatchDeviceBands, + outputs: Dwt97CodeblockBandBuffers<'_>, + params: CudaHtj2k97QuantizeParams, + items: u32, + ) -> Result<(), CudaError> { + let to_i32 = |value: usize| -> Result { + i32::try_from(value).map_err(|_| CudaError::LengthTooLarge { len: value }) + }; + let low_width = to_i32(bands.low_width)?; + let low_height = to_i32(bands.low_height)?; + let high_width = to_i32(bands.high_width)?; + let high_height = to_i32(bands.high_height)?; + let cb_width = to_i32(params.cb_width)?; + let cb_height = to_i32(params.cb_height)?; + + self.launch_transcode_dwt97_quantize_codeblocks( + pooled_device_buffer(&bands.ll)?, + outputs.ll, + low_width, + low_height, + cb_width, + cb_height, + params.inv_delta_ll, + items, + )?; + self.launch_transcode_dwt97_quantize_codeblocks( + pooled_device_buffer(&bands.hl)?, + outputs.hl, + high_width, + low_height, + cb_width, + cb_height, + params.inv_delta_hl, + items, + )?; + self.launch_transcode_dwt97_quantize_codeblocks( + pooled_device_buffer(&bands.lh)?, + outputs.lh, + low_width, + high_height, + cb_width, + cb_height, + params.inv_delta_lh, + items, + )?; + self.launch_transcode_dwt97_quantize_codeblocks( + pooled_device_buffer(&bands.hh)?, + outputs.hh, + high_width, + high_height, + cb_width, + cb_height, + params.inv_delta_hh, + items, + )?; + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn launch_transcode_dwt97_quantize_codeblocks( + &self, + band: &CudaDeviceBuffer, + output: &CudaDeviceBuffer, + width: i32, + height: i32, + cb_width: i32, + cb_height: i32, + inv_delta: f32, + items: u32, + ) -> Result<(), CudaError> { + if width <= 0 || height <= 0 { + return Ok(()); + } + let function = + self.transcode_kernel_function(CudaKernel::TranscodeDwt97QuantizeCodeblocks)?; + let mut band_ptr = band.device_ptr(); + let mut output_ptr = output.device_ptr(); + let mut width = width; + let mut height = height; + let mut cb_width = cb_width; + let mut cb_height = cb_height; + let mut inv_delta = inv_delta; + let mut params = cuda_kernel_params!( + band_ptr, output_ptr, width, height, cb_width, cb_height, inv_delta + ); + let grid_w = u32::try_from(width).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + let grid_h = u32::try_from(height).map_err(|_| CudaError::LengthTooLarge { len: 0 })?; + let base = j2k_dwt53_launch_geometry(grid_w, grid_h) + .ok_or(CudaError::LengthTooLarge { len: 0 })?; + let geometry = with_grid_z(base, items); + self.launch_kernel_async(function, geometry, &mut params) + } +} diff --git a/crates/j2k-cuda-runtime/src/transcode_kernels.cu b/crates/j2k-cuda-runtime/src/transcode_kernels.cu new file mode 100644 index 00000000..057e311e --- /dev/null +++ b/crates/j2k-cuda-runtime/src/transcode_kernels.cu @@ -0,0 +1,771 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// CUDA kernels for j2k-transcode-cuda: direct DCT-coefficient-domain +// JPEG -> HTJ2K transform stages. These port the j2k-transcode SCALAR +// ORACLE faithfully (they are the parity reference), not a re-derivation: +// +// reversible 5/3: idct_islow (jidctint, 16-bit fixed point) -> -128 level +// shift -> separable reversible integer 5/3 lifting with +// Euclidean floor division, deinterleaved into LL/HL/LH/HH. +// +// Compiled with --fmad=false (see build.rs) so the irreversible 9/7 path (added +// later) keeps bit-identical f32 operation ordering vs the scalar reference. +// +// Integer arithmetic mirrors the scalar oracle's `i32` (Wrapping) math; valid +// JPEG coefficients do not overflow i32 in the islow algorithm by construction. + +typedef short i16; +typedef int i32; +typedef unsigned int u32; + +// ---- jidctint fixed-point constants (match idct/scalar.rs) ----------------- +#define CONST_BITS 13 +#define PASS1_BITS 2 +#define FIX_0_298631336 2446 +#define FIX_0_390180644 3196 +#define FIX_0_541196100 4433 +#define FIX_0_765366865 6270 +#define FIX_0_899976223 7373 +#define FIX_1_175875602 9633 +#define FIX_1_501321110 12299 +#define FIX_1_847759065 15137 +#define FIX_1_961570560 16069 +#define FIX_2_053119869 16819 +#define FIX_2_562915447 20995 +#define FIX_3_072711026 25172 + +// Euclidean floor division by a positive divisor (matches i32::div_euclid). +__device__ __forceinline__ i32 floor_div_pos(i32 a, i32 d) { + i32 q = a / d; + i32 r = a - q * d; + if (r < 0) { + q -= 1; + } + return q; +} + +// One full 8x8 ISLOW inverse DCT (full path; the scalar fast paths are pure +// optimizations and are mathematically identical, so the full path is +// bit-exact for every input). Output: signed samples already level-shifted by +// -128 (i.e. clamp(idct+128,0,255) - 128), matching idct_blocks_to_signed_samples. +__device__ void idct_islow_signed(const i16* in, i32* out64) { + i32 work[64]; + // Column pass. + for (int col = 0; col < 8; ++col) { + i32 p0 = (i32)in[col]; + i32 p1 = (i32)in[col + 8]; + i32 p2 = (i32)in[col + 16]; + i32 p3 = (i32)in[col + 24]; + i32 p4 = (i32)in[col + 32]; + i32 p5 = (i32)in[col + 40]; + i32 p6 = (i32)in[col + 48]; + i32 p7 = (i32)in[col + 56]; + + i32 z2 = p2; + i32 z3 = p6; + i32 z1 = (z2 + z3) * FIX_0_541196100; + i32 tmp2 = z1 + z3 * (-FIX_1_847759065); + i32 tmp3 = z1 + z2 * FIX_0_765366865; + + z2 = p0; + z3 = p4; + i32 tmp0 = (z2 + z3) << CONST_BITS; + i32 tmp1 = (z2 - z3) << CONST_BITS; + + i32 tmp10 = tmp0 + tmp3; + i32 tmp13 = tmp0 - tmp3; + i32 tmp11 = tmp1 + tmp2; + i32 tmp12 = tmp1 - tmp2; + + tmp0 = p7; + tmp1 = p5; + tmp2 = p3; + tmp3 = p1; + + z1 = tmp0 + tmp3; + z2 = tmp1 + tmp2; + z3 = tmp0 + tmp2; + i32 z4 = tmp1 + tmp3; + i32 z5 = (z3 + z4) * FIX_1_175875602; + + tmp0 = tmp0 * FIX_0_298631336; + tmp1 = tmp1 * FIX_2_053119869; + tmp2 = tmp2 * FIX_3_072711026; + tmp3 = tmp3 * FIX_1_501321110; + z1 = z1 * (-FIX_0_899976223); + z2 = z2 * (-FIX_2_562915447); + z3 = z3 * (-FIX_1_961570560); + z4 = z4 * (-FIX_0_390180644); + + z3 += z5; + z4 += z5; + + tmp0 += z1 + z3; + tmp1 += z2 + z4; + tmp2 += z2 + z3; + tmp3 += z1 + z4; + + const int shift = CONST_BITS - PASS1_BITS; + const i32 rounding = (i32)1 << (shift - 1); + work[col] = (tmp10 + tmp3 + rounding) >> shift; + work[col + 56] = (tmp10 - tmp3 + rounding) >> shift; + work[col + 8] = (tmp11 + tmp2 + rounding) >> shift; + work[col + 48] = (tmp11 - tmp2 + rounding) >> shift; + work[col + 16] = (tmp12 + tmp1 + rounding) >> shift; + work[col + 40] = (tmp12 - tmp1 + rounding) >> shift; + work[col + 24] = (tmp13 + tmp0 + rounding) >> shift; + work[col + 32] = (tmp13 - tmp0 + rounding) >> shift; + } + // Row pass. + for (int row = 0; row < 8; ++row) { + const int base = row * 8; + i32 p0 = work[base]; + i32 p1 = work[base + 1]; + i32 p2 = work[base + 2]; + i32 p3 = work[base + 3]; + i32 p4 = work[base + 4]; + i32 p5 = work[base + 5]; + i32 p6 = work[base + 6]; + i32 p7 = work[base + 7]; + + const int shift = CONST_BITS + PASS1_BITS + 3; + const i32 rounding = (i32)1 << (shift - 1); + + i32 z2 = p2; + i32 z3 = p6; + i32 z1 = (z2 + z3) * FIX_0_541196100; + i32 tmp2 = z1 + z3 * (-FIX_1_847759065); + i32 tmp3 = z1 + z2 * FIX_0_765366865; + + i32 tmp0 = (p0 + p4) << CONST_BITS; + i32 tmp1 = (p0 - p4) << CONST_BITS; + + i32 tmp10 = tmp0 + tmp3; + i32 tmp13 = tmp0 - tmp3; + i32 tmp11 = tmp1 + tmp2; + i32 tmp12 = tmp1 - tmp2; + + tmp0 = p7; + tmp1 = p5; + tmp2 = p3; + tmp3 = p1; + + z1 = tmp0 + tmp3; + z2 = tmp1 + tmp2; + z3 = tmp0 + tmp2; + i32 z4 = tmp1 + tmp3; + i32 z5 = (z3 + z4) * FIX_1_175875602; + + tmp0 = tmp0 * FIX_0_298631336; + tmp1 = tmp1 * FIX_2_053119869; + tmp2 = tmp2 * FIX_3_072711026; + tmp3 = tmp3 * FIX_1_501321110; + z1 = z1 * (-FIX_0_899976223); + z2 = z2 * (-FIX_2_562915447); + z3 = z3 * (-FIX_1_961570560); + z4 = z4 * (-FIX_0_390180644); + + z3 += z5; + z4 += z5; + + tmp0 += z1 + z3; + tmp1 += z2 + z4; + tmp2 += z2 + z3; + tmp3 += z1 + z4; + + i32 r0 = (tmp10 + tmp3 + rounding) >> shift; + i32 r7 = (tmp10 - tmp3 + rounding) >> shift; + i32 r1 = (tmp11 + tmp2 + rounding) >> shift; + i32 r6 = (tmp11 - tmp2 + rounding) >> shift; + i32 r2 = (tmp12 + tmp1 + rounding) >> shift; + i32 r5 = (tmp12 - tmp1 + rounding) >> shift; + i32 r3 = (tmp13 + tmp0 + rounding) >> shift; + i32 r4 = (tmp13 - tmp0 + rounding) >> shift; + + // clamp(value+128, 0, 255) then subtract 128 -> signed sample in [-128,127]. + i32 vals[8] = {r0, r1, r2, r3, r4, r5, r6, r7}; + for (int k = 0; k < 8; ++k) { + i32 lv = vals[k] + 128; + if (lv < 0) lv = 0; else if (lv > 255) lv = 255; + out64[base + k] = lv - 128; + } + } +} + +// Kernel 1: one thread per 8x8 DCT block -> block-local signed sample plane. +extern "C" __global__ void transcode_reversible53_idct( + const i16* blocks, // block_count * 64, natural order + i32* samples, // block_count * 64, block-local signed samples + u32 block_count) { + const u32 idx = blockIdx.x * blockDim.x + threadIdx.x; + if (idx >= block_count) { + return; + } + idct_islow_signed(blocks + (size_t)idx * 64, samples + (size_t)idx * 64); +} + +// Block-local sample fetch matching component_sample_i32 (x> 3) * block_cols + (x >> 3); + const int local_idx = (y & 7) * 8 + (x & 7); + return samples[(size_t)block_idx * 64 + local_idx]; +} + +// vertical_high_53_i32_at +__device__ i32 vertical_high( + const i32* samples, int block_cols, int height, int x, int high_idx) { + const int odd_idx = high_idx * 2 + 1; + const i32 current = sample_at(samples, block_cols, x, odd_idx); + const i32 left = sample_at(samples, block_cols, x, odd_idx - 1); + if ((height % 2 == 0) && (odd_idx + 1 == height)) { + return current - left; + } + const int right_idx = (odd_idx + 1 < height) ? (odd_idx + 1) : (height - 1); + const i32 right = sample_at(samples, block_cols, x, right_idx); + return current - floor_div_pos(left + right, 2); +} + +// vertical_low_53_i32_at +__device__ i32 vertical_low( + const i32* samples, int block_cols, int height, int x, int low_idx) { + const int even_idx = low_idx * 2; + const i32 current = sample_at(samples, block_cols, x, even_idx); + if (height < 2) { + return current; + } + if (height % 2 == 0) { + const i32 right = vertical_high(samples, block_cols, height, x, low_idx); + if (low_idx == 0) { + return current + floor_div_pos(right + 1, 2); + } + const i32 left = vertical_high(samples, block_cols, height, x, low_idx - 1); + return current + floor_div_pos(left + right + 2, 4); + } + const int high_len = height / 2; + if (high_len == 0) { + return current; + } + const i32 left = vertical_high(samples, block_cols, height, x, (low_idx > 0) ? (low_idx - 1) : 0); + const i32 right = (low_idx < high_len) + ? vertical_high(samples, block_cols, height, x, low_idx) + : left; + return current + floor_div_pos(left + right + 2, 4); +} + +// Kernel 2: vertical 5/3 low band. One thread per (x, low row). +extern "C" __global__ void transcode_reversible53_vertical_low( + const i32* samples, int block_cols, int width, int height, + i32* v_low, int low_height) { + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int yl = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= width || yl >= low_height) { + return; + } + v_low[(size_t)yl * width + x] = vertical_low(samples, block_cols, height, x, yl); +} + +// Kernel 3: vertical 5/3 high band. One thread per (x, high row). +extern "C" __global__ void transcode_reversible53_vertical_high( + const i32* samples, int block_cols, int width, int height, + i32* v_high, int high_height) { + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int yh = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= width || yh >= high_height) { + return; + } + v_high[(size_t)yh * width + x] = vertical_high(samples, block_cols, height, x, yh); +} + +// reversible_lift_53_i32 applied in place to a contiguous row of `n` i32. +__device__ void reversible_lift_row(i32* v, int n) { + if (n < 2) { + return; + } + if (n % 2 == 0) { + for (int i = 1; i < n - 1; i += 2) { + v[i] -= floor_div_pos(v[i - 1] + v[i + 1], 2); + } + v[n - 1] -= v[n - 2]; + v[0] += floor_div_pos(v[1] + 1, 2); + for (int i = 2; i < n; i += 2) { + v[i] += floor_div_pos(v[i - 1] + v[i + 1] + 2, 4); + } + return; + } + const int last_even = n - 1; + for (int i = 1; i < n; i += 2) { + const i32 right = (i + 1 < n) ? v[i + 1] : v[last_even]; + v[i] -= floor_div_pos(v[i - 1] + right, 2); + } + for (int i = 0; i < n; i += 2) { + const i32 left = (i > 0) ? v[i - 1] : v[1]; + const i32 right = (i + 1 < n) ? v[i + 1] : left; + v[i] += floor_div_pos(left + right + 2, 4); + } +} + +// Kernel 4: horizontal 5/3 lift of each vertically-low row, in place, then +// deinterleave even -> LL, odd -> HL. One thread per low row. +extern "C" __global__ void transcode_reversible53_horizontal_low( + i32* v_low, int width, int low_height, int low_width, int high_width, + i32* ll, i32* hl) { + const int yl = blockIdx.x * blockDim.x + threadIdx.x; + if (yl >= low_height) { + return; + } + i32* row = v_low + (size_t)yl * width; + reversible_lift_row(row, width); + for (int i = 0; i < low_width; ++i) { + ll[(size_t)yl * low_width + i] = row[i * 2]; + } + for (int i = 0; i < high_width; ++i) { + hl[(size_t)yl * high_width + i] = row[i * 2 + 1]; + } +} + +// Kernel 5: horizontal 5/3 lift of each vertically-high row -> LH/HH. +extern "C" __global__ void transcode_reversible53_horizontal_high( + i32* v_high, int width, int high_height, int low_width, int high_width, + i32* lh, i32* hh) { + const int yh = blockIdx.x * blockDim.x + threadIdx.x; + if (yh >= high_height) { + return; + } + i32* row = v_high + (size_t)yh * width; + reversible_lift_row(row, width); + for (int i = 0; i < low_width; ++i) { + lh[(size_t)yh * low_width + i] = row[i * 2]; + } + for (int i = 0; i < high_width; ++i) { + hh[(size_t)yh * high_width + i] = row[i * 2 + 1]; + } +} + +// =========================================================================== +// Irreversible 9/7 path: faithful f32 port of the j2k-transcode scalar +// oracle (dct97_2d.rs). Separable float IDCT (idct8x8_sample) into a spatial +// plane, then the separable single-level 9/7 transform (forward_lift_97): +// rows -> {row_low, row_high}; columns of row_low -> {LL, LH}; columns of +// row_high -> {HL, HH}. Device math is f32 (Metal uses f32 here too); parity +// is asserted within the 2e-2 band tolerance. --fmad=false keeps the lift +// operation ordering close to the scalar reference. +// =========================================================================== + +typedef float f32; + +#define J2K_PI 3.14159265358979323846 +#define DWT97_ALPHA (-1.586134342059924f) +#define DWT97_BETA (-0.052980118572961f) +#define DWT97_GAMMA (0.882911075530934f) +#define DWT97_DELTA (0.443506852043971f) +#define DWT97_KAPPA (1.230174104914001f) +#define DWT97_INV_KAPPA (1.0f / DWT97_KAPPA) + +// idct8_basis(sample_idx, freq): scale * cos((s+0.5)*f*pi/8), scale = sqrt(1/8) +// for freq 0 else sqrt(2/8). Matches idct8_basis_uncached in dct97_2d.rs. +// Precomputed to avoid per-sample sqrtf/cosf in the CUDA hot path. +__device__ __constant__ f32 DWT97_IDCT8_BASIS[64] = { + 0.353553391f, 0.49039264f, 0.461939766f, 0.415734806f, 0.353553391f, 0.277785117f, 0.191341716f, 0.097545161f, + 0.353553391f, 0.415734806f, 0.191341716f, -0.097545161f, -0.353553391f, -0.49039264f, -0.461939766f, -0.277785117f, + 0.353553391f, 0.277785117f, -0.191341716f, -0.49039264f, -0.353553391f, 0.097545161f, 0.461939766f, 0.415734806f, + 0.353553391f, 0.097545161f, -0.461939766f, -0.277785117f, 0.353553391f, 0.415734806f, -0.191341716f, -0.49039264f, + 0.353553391f, -0.097545161f, -0.461939766f, 0.277785117f, 0.353553391f, -0.415734806f, -0.191341716f, 0.49039264f, + 0.353553391f, -0.277785117f, -0.191341716f, 0.49039264f, -0.353553391f, -0.097545161f, 0.461939766f, -0.415734806f, + 0.353553391f, -0.415734806f, 0.191341716f, 0.097545161f, -0.353553391f, 0.49039264f, -0.461939766f, 0.277785117f, + 0.353553391f, -0.49039264f, 0.461939766f, -0.415734806f, 0.353553391f, -0.277785117f, 0.191341716f, -0.097545161f +}; + +__device__ __forceinline__ f32 idct8_basis(int sample_idx, int freq) { + return DWT97_IDCT8_BASIS[sample_idx * 8 + freq]; +} + +// One IDCT sample, matching idct8x8_sample's accumulation order +// (freq_y outer, freq_x inner; coeff * y_basis * x_basis). +__device__ f32 idct8x8_sample(const f32* block, int local_x, int local_y) { + f32 sample = 0.0f; + // transcode_dwt97_idct_unroll_guard: the IDCT loops have fixed 8x8 bounds. +#pragma unroll + for (int freq_y = 0; freq_y < 8; ++freq_y) { + const f32 y_basis = idct8_basis(local_y, freq_y); + const f32* brow = block + freq_y * 8; +#pragma unroll + for (int freq_x = 0; freq_x < 8; ++freq_x) { + sample += brow[freq_x] * y_basis * idct8_basis(local_x, freq_x); + } + } + return sample; +} + +// Same accumulation order as idct8x8_sample, with exact i16->f32 conversion +// performed in-kernel to avoid host-side expansion and halve HtoD traffic. +__device__ f32 idct8x8_sample_i16(const i16* block, int local_x, int local_y) { + f32 sample = 0.0f; +#pragma unroll + for (int freq_y = 0; freq_y < 8; ++freq_y) { + const f32 y_basis = idct8_basis(local_y, freq_y); + const i16* brow = block + freq_y * 8; +#pragma unroll + for (int freq_x = 0; freq_x < 8; ++freq_x) { + sample += ((f32)brow[freq_x]) * y_basis * idct8_basis(local_x, freq_x); + } + } + return sample; +} + +// forward_lift_97 applied with element stride `s` (1 for rows, band width for +// columns), in place over `n` logical samples. Matches forward_lift_97. +__device__ void forward_lift_97(f32* d, int n, int s) { + if (n < 2) { + return; + } + const int last_even = (n % 2 == 0) ? (n - 2) : (n - 1); + for (int i = 1; i < n; i += 2) { + const f32 left = d[(i - 1) * s]; + const f32 right = (i + 1 < n) ? d[(i + 1) * s] : d[last_even * s]; + d[i * s] += DWT97_ALPHA * (left + right); + } + for (int i = 0; i < n; i += 2) { + const f32 left = (i > 0) ? d[(i - 1) * s] : d[1 * s]; + const f32 right = (i + 1 < n) ? d[(i + 1) * s] : left; + d[i * s] += DWT97_BETA * (left + right); + } + for (int i = 1; i < n; i += 2) { + const f32 left = d[(i - 1) * s]; + const f32 right = (i + 1 < n) ? d[(i + 1) * s] : d[last_even * s]; + d[i * s] += DWT97_GAMMA * (left + right); + } + for (int i = 0; i < n; i += 2) { + const f32 left = (i > 0) ? d[(i - 1) * s] : d[1 * s]; + const f32 right = (i + 1 < n) ? d[(i + 1) * s] : left; + d[i * s] += DWT97_DELTA * (left + right); + } + for (int i = 0; i < n; i += 2) { + d[i * s] *= DWT97_INV_KAPPA; + } + for (int i = 1; i < n; i += 2) { + d[i * s] *= DWT97_KAPPA; + } +} + +// Kernel: separable float IDCT into a width*height spatial plane. +// One thread per (x, y). +extern "C" __global__ void transcode_dwt97_idct( + const f32* blocks, int block_cols, int width, int height, f32* spatial) { + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= width || y >= height) { + return; + } + const int block_idx = (y >> 3) * block_cols + (x >> 3); + const f32* block = blocks + (size_t)block_idx * 64; + spatial[(size_t)y * width + x] = idct8x8_sample(block, x & 7, y & 7); +} + +// Kernel: horizontal 9/7 lift per row, then split even -> row_low, odd -> +// row_high. One thread per row. +extern "C" __global__ void transcode_dwt97_row_lift( + f32* spatial, int width, int height, int low_width, int high_width, + f32* row_low, f32* row_high) { + const int y = blockIdx.x * blockDim.x + threadIdx.x; + if (y >= height) { + return; + } + f32* row = spatial + (size_t)y * width; + forward_lift_97(row, width, 1); + for (int i = 0; i < low_width; ++i) { + row_low[(size_t)y * low_width + i] = row[i * 2]; + } + for (int i = 0; i < high_width; ++i) { + row_high[(size_t)y * high_width + i] = row[i * 2 + 1]; + } +} + +// Kernel: vertical 9/7 lift per column (strided) of a band buffer, then split +// even rows -> low_out, odd rows -> high_out. Used for row_low -> {LL, LH} and +// row_high -> {HL, HH}. One thread per column. `band_width` is the column count +// (= the in-place stride). +extern "C" __global__ void transcode_dwt97_column_lift( + f32* rows, int band_width, int height, f32* low_out, f32* high_out) { + const int x = blockIdx.x * blockDim.x + threadIdx.x; + if (x >= band_width) { + return; + } + forward_lift_97(rows + x, height, band_width); + for (int i = 0; i < height; ++i) { + const f32 value = rows[(size_t)i * band_width + x]; + if ((i & 1) == 0) { + low_out[(size_t)(i / 2) * band_width + x] = value; + } else { + high_out[(size_t)(i / 2) * band_width + x] = value; + } + } +} + +// =========================================================================== +// Same-geometry 9/7 batch: per-item replicas of the three single-job kernels +// above, selecting the item from a grid dimension and offsetting every buffer +// by the item's uniform per-item stride. Output is bit-identical to running the +// single-job kernels once per item (same f32 op ordering, same --fmad=false). +// =========================================================================== + +// Kernel: batched separable float IDCT. Grid (x, y, item); thread per sample. +// `blocks_per_item` = block_cols * block_rows; spatial item stride = width*height. +extern "C" __global__ void transcode_dwt97_idct_batch( + const f32* blocks, int block_cols, int width, int height, + int blocks_per_item, f32* spatial) { + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + const int item = blockIdx.z; + if (x >= width || y >= height) { + return; + } + const f32* item_blocks = blocks + (size_t)item * blocks_per_item * 64; + const int block_idx = (y >> 3) * block_cols + (x >> 3); + const f32* block = item_blocks + (size_t)block_idx * 64; + spatial[((size_t)item * height + y) * width + x] = idct8x8_sample(block, x & 7, y & 7); +} + +// Same as transcode_dwt97_idct_batch, but input coefficients stay i16 on the +// host/device boundary and are widened to f32 inside the IDCT sample loop. +extern "C" __global__ void transcode_dwt97_idct_i16_batch( + const i16* blocks, int block_cols, int width, int height, + int blocks_per_item, f32* spatial) { + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + const int item = blockIdx.z; + if (x >= width || y >= height) { + return; + } + const i16* item_blocks = blocks + (size_t)item * blocks_per_item * 64; + const int block_idx = (y >> 3) * block_cols + (x >> 3); + const i16* block = item_blocks + (size_t)block_idx * 64; + spatial[((size_t)item * height + y) * width + x] = idct8x8_sample_i16(block, x & 7, y & 7); +} + +// Kernel: batched horizontal 9/7 row lift + split. Grid (row, item); thread per row. +extern "C" __global__ void transcode_dwt97_row_lift_batch( + f32* spatial, int width, int height, int low_width, int high_width, + f32* row_low, f32* row_high) { + const int y = blockIdx.x * blockDim.x + threadIdx.x; + const int item = blockIdx.y; + if (y >= height) { + return; + } + f32* item_spatial = spatial + (size_t)item * width * height; + f32* item_row_low = row_low + (size_t)item * height * low_width; + f32* item_row_high = row_high + (size_t)item * height * high_width; + f32* row = item_spatial + (size_t)y * width; + forward_lift_97(row, width, 1); + for (int i = 0; i < low_width; ++i) { + item_row_low[(size_t)y * low_width + i] = row[i * 2]; + } + for (int i = 0; i < high_width; ++i) { + item_row_high[(size_t)y * high_width + i] = row[i * 2 + 1]; + } +} + +#define DWT97_ROW_LIFT_MAX_WIDTH 1024 +#define DWT97_ROW_LIFT_ROWS_PER_BLOCK 4 + +// Cooperative row lift for common tile widths. Each block handles up to four +// rows, staging them in shared memory so each lifting phase can run in parallel +// across row positions while preserving the scalar phase order. +extern "C" __global__ void transcode_dwt97_row_lift_batch_coop( + const f32* spatial, int width, int height, int low_width, int high_width, + f32* row_low, f32* row_high) { + __shared__ f32 rows[DWT97_ROW_LIFT_ROWS_PER_BLOCK][DWT97_ROW_LIFT_MAX_WIDTH]; + + const int row_lane = threadIdx.y; + const int tid = threadIdx.x; + const int y = blockIdx.x * DWT97_ROW_LIFT_ROWS_PER_BLOCK + row_lane; + const int item = blockIdx.y; + const bool valid = y < height && width <= DWT97_ROW_LIFT_MAX_WIDTH; + + const f32* item_spatial = spatial + (size_t)item * width * height; + f32* item_row_low = row_low + (size_t)item * height * low_width; + f32* item_row_high = row_high + (size_t)item * height * high_width; + const f32* source = item_spatial + (size_t)y * width; + f32* row = rows[row_lane]; + + if (valid) { + for (int i = tid; i < width; i += blockDim.x) { + row[i] = source[i]; + } + } + __syncthreads(); + + if (width >= 2 && width <= DWT97_ROW_LIFT_MAX_WIDTH) { + const int last_even = (width % 2 == 0) ? (width - 2) : (width - 1); + if (valid) { + for (int i = tid * 2 + 1; i < width; i += blockDim.x * 2) { + const f32 left = row[i - 1]; + const f32 right = (i + 1 < width) ? row[i + 1] : row[last_even]; + row[i] += DWT97_ALPHA * (left + right); + } + } + __syncthreads(); + + if (valid) { + for (int i = tid * 2; i < width; i += blockDim.x * 2) { + const f32 left = (i > 0) ? row[i - 1] : row[1]; + const f32 right = (i + 1 < width) ? row[i + 1] : left; + row[i] += DWT97_BETA * (left + right); + } + } + __syncthreads(); + + if (valid) { + for (int i = tid * 2 + 1; i < width; i += blockDim.x * 2) { + const f32 left = row[i - 1]; + const f32 right = (i + 1 < width) ? row[i + 1] : row[last_even]; + row[i] += DWT97_GAMMA * (left + right); + } + } + __syncthreads(); + + if (valid) { + for (int i = tid * 2; i < width; i += blockDim.x * 2) { + const f32 left = (i > 0) ? row[i - 1] : row[1]; + const f32 right = (i + 1 < width) ? row[i + 1] : left; + row[i] += DWT97_DELTA * (left + right); + } + } + __syncthreads(); + + if (valid) { + for (int i = tid * 2; i < width; i += blockDim.x * 2) { + row[i] *= DWT97_INV_KAPPA; + } + for (int i = tid * 2 + 1; i < width; i += blockDim.x * 2) { + row[i] *= DWT97_KAPPA; + } + } + __syncthreads(); + } + + if (valid) { + for (int i = tid; i < low_width; i += blockDim.x) { + item_row_low[(size_t)y * low_width + i] = row[i * 2]; + } + for (int i = tid; i < high_width; i += blockDim.x) { + item_row_high[(size_t)y * high_width + i] = row[i * 2 + 1]; + } + } +} + +// Kernel: batched vertical 9/7 column lift + split. Grid (column, item); thread +// per column. `low_height`/`high_height` give the per-item output band strides. +extern "C" __global__ void transcode_dwt97_column_lift_batch( + f32* rows, int band_width, int height, int low_height, int high_height, + f32* low_out, f32* high_out) { + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int item = blockIdx.y; + if (x >= band_width) { + return; + } + f32* item_rows = rows + (size_t)item * height * band_width; + f32* item_low = low_out + (size_t)item * low_height * band_width; + f32* item_high = high_out + (size_t)item * high_height * band_width; + forward_lift_97(item_rows + x, height, band_width); + for (int i = 0; i < height; ++i) { + const f32 value = item_rows[(size_t)i * band_width + x]; + if ((i & 1) == 0) { + item_low[(size_t)(i / 2) * band_width + x] = value; + } else { + item_high[(size_t)(i / 2) * band_width + x] = value; + } + } +} + +__device__ __forceinline__ i32 quantize_dwt97_deadzone(f32 value, f32 inv_delta) { + const int sign = (value < 0.0f) ? -1 : 1; + const int magnitude = (int)floorf(fabsf(value) * inv_delta); + return sign * magnitude; +} + +__device__ __forceinline__ size_t dwt97_codeblock_major_offset( + int x, int y, int width, int height, int cb_width, int cb_height) { + if (cb_width == 64 && cb_height == 64) { + const int cbx = x >> 6; + const int cby = y >> 6; + const int local_x = x & 63; + const int local_y = y & 63; + const int block_width = min(64, width - (cbx << 6)); + const int block_height = min(64, height - (cby << 6)); + return (size_t)cby * 64 * width + + (size_t)cbx * 64 * block_height + + (size_t)local_y * block_width + local_x; + } + const int cbx = x / cb_width; + const int cby = y / cb_height; + const int local_x = x - cbx * cb_width; + const int local_y = y - cby * cb_height; + const int block_width = min(cb_width, width - cbx * cb_width); + const int block_height = min(cb_height, height - cby * cb_height); + return (size_t)cby * cb_height * width + + (size_t)cbx * cb_width * block_height + + (size_t)local_y * block_width + local_x; +} + +// Kernel: deadzone-quantize one 9/7 band into code-block-major i32 layout for +// one batch, mirroring the j2k-transcode shared code-block oracle and +// Metal's dct97_quantize_codeblocks_batch. Launched once per subband (its own +// width, height, and inv_delta). Grid (x, y, item); thread per band sample. +// +// q = sign(value) * floor(|value| * inv_delta), sign(0) = +1 +// +// Output offset for (cbx, cby, local): code-block-major (outer cby, inner cbx), +// each block stored row-major; per-item stride = width*height (coeffs preserved). +extern "C" __global__ void transcode_dwt97_quantize_codeblocks( + const f32* band, i32* output, int width, int height, + int cb_width, int cb_height, f32 inv_delta) { + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + const int item = blockIdx.z; + if (x >= width || y >= height) { + return; + } + const size_t item_stride = (size_t)width * height; + const f32 value = band[(size_t)item * item_stride + (size_t)y * width + x]; + const size_t offset = + dwt97_codeblock_major_offset(x, y, width, height, cb_width, cb_height); + output[(size_t)item * item_stride + offset] = + quantize_dwt97_deadzone(value, inv_delta); +} + +// Fused resident HT path stage: vertical 9/7 column lift, even/odd row split, +// and deadzone quantization directly into code-block-major i32 output. This +// keeps the same forward_lift_97 global-memory update order as +// transcode_dwt97_column_lift_batch, then quantizes the f32 value that would +// otherwise have been stored in the intermediate LL/LH/HL/HH band buffer. +extern "C" __global__ void transcode_dwt97_column_lift_quantize_codeblocks_batch( + f32* rows, int band_width, int height, int low_height, int high_height, + i32* low_out, i32* high_out, int cb_width, int cb_height, + f32 inv_delta_low, f32 inv_delta_high) { + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int item = blockIdx.y; + if (x >= band_width) { + return; + } + + f32* item_rows = rows + (size_t)item * height * band_width; + i32* item_low = low_out + (size_t)item * low_height * band_width; + i32* item_high = high_out + (size_t)item * high_height * band_width; + + forward_lift_97(item_rows + x, height, band_width); + for (int i = 0; i < height; ++i) { + const f32 value = item_rows[(size_t)i * band_width + x]; + if ((i & 1) == 0) { + const int y = i / 2; + const size_t offset = dwt97_codeblock_major_offset( + x, y, band_width, low_height, cb_width, cb_height); + item_low[offset] = quantize_dwt97_deadzone(value, inv_delta_low); + } else { + const int y = i / 2; + const size_t offset = dwt97_codeblock_major_offset( + x, y, band_width, high_height, cb_width, cb_height); + item_high[offset] = quantize_dwt97_deadzone(value, inv_delta_high); + } + } +} diff --git a/crates/j2k-cuda/Cargo.toml b/crates/j2k-cuda/Cargo.toml new file mode 100644 index 00000000..cfeb8e35 --- /dev/null +++ b/crates/j2k-cuda/Cargo.toml @@ -0,0 +1,63 @@ +[package] +name = "j2k-cuda" +description = "CUDA decoder and encode-stage adapter for j2k" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +readme = "README.md" + +[package.metadata.docs.rs] +all-features = true +targets = [] + +[lib] +name = "j2k_cuda" +path = "src/lib.rs" + +[features] +default = [] +cuda-runtime = ["dep:j2k-cuda-runtime"] +cuda-profiling = ["cuda-runtime", "j2k-cuda-runtime/cuda-profiling"] + +[dependencies] +j2k-core = { path = "../j2k-core", version = "=0.6.0" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.6.0", optional = true } +j2k = { path = "../j2k", version = "=0.6.0" } +j2k-native = { path = "../j2k-native", version = "=0.6.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.6.0" } +thiserror = { workspace = true } + +[dev-dependencies] +criterion = { workspace = true } +j2k-test-support = { path = "../j2k-test-support", features = ["j2k-native-fixtures"] } + +[[bench]] +name = "encode_stages" +harness = false + +[[bench]] +name = "htj2k_decode" +harness = false + +[[bench]] +name = "htj2k_encode" +harness = false +required-features = ["cuda-runtime"] + +[lints.rust] +unsafe_code = "allow" +unsafe_op_in_unsafe_fn = "deny" +unreachable_pub = "warn" + +[lints.clippy] +pedantic = { level = "warn", priority = -1 } +undocumented_unsafe_blocks = "warn" +module_name_repetitions = "allow" +must_use_candidate = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +too_many_lines = "allow" diff --git a/crates/j2k-cuda/README.md b/crates/j2k-cuda/README.md new file mode 100644 index 00000000..016f9e34 --- /dev/null +++ b/crates/j2k-cuda/README.md @@ -0,0 +1,9 @@ +# j2k-cuda + +CUDA adapter for JPEG 2000 / HTJ2K decode and encode-stage paths. + +The crate provides strict CUDA device-memory decode and encode-stage integration +for supported HTJ2K/J2K workloads using J2K-owned CUDA kernels. +Unsupported explicit CUDA requests return structured errors. + +NVIDIA performance claims require self-hosted benchmark evidence. diff --git a/crates/signinum-j2k-cuda/benches/encode_stages.rs b/crates/j2k-cuda/benches/encode_stages.rs similarity index 79% rename from crates/signinum-j2k-cuda/benches/encode_stages.rs rename to crates/j2k-cuda/benches/encode_stages.rs index 96505f18..762b0c75 100644 --- a/crates/signinum-j2k-cuda/benches/encode_stages.rs +++ b/crates/j2k-cuda/benches/encode_stages.rs @@ -1,8 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; -use signinum_j2k_cuda::CudaEncodeStageAccelerator; -use signinum_j2k_native::{J2kEncodeStageAccelerator, J2kForwardDwt53Job, J2kForwardRctJob}; +use j2k::adapter::encode_stage::{ + J2kEncodeStageAccelerator, J2kForwardDwt53Job, J2kForwardRctJob, J2kQuantizeSubbandJob, +}; +use j2k_cuda::CudaEncodeStageAccelerator; const BENCH_DIMS: &[u32] = &[512, 1024, 2048]; @@ -67,6 +69,35 @@ fn bench_encode_stages(c: &mut Criterion) { } } dwt.finish(); + + let mut quantize = c.benchmark_group("j2k_cuda_quantize_subband"); + for &dim in BENCH_DIMS { + let samples = generate_gray_plane(dim, dim); + quantize.bench_with_input(BenchmarkId::new("cpu", dim), &samples, |b, samples| { + b.iter(|| cpu_quantize_reversible(samples)); + }); + + if cuda_available { + quantize.bench_with_input(BenchmarkId::new("cuda", dim), &samples, |b, samples| { + let mut accelerator = CudaEncodeStageAccelerator::default(); + b.iter(|| { + let output = accelerator + .encode_quantize_subband(J2kQuantizeSubbandJob { + coefficients: samples, + step_exponent: 8, + step_mantissa: 0, + range_bits: 8, + reversible: true, + }) + .expect("CUDA quantize subband") + .expect("CUDA quantize subband dispatch"); + assert_eq!(output.len(), samples.len()); + output + }); + }); + } + } + quantize.finish(); } fn cuda_encode_available() -> bool { @@ -80,15 +111,15 @@ fn cuda_encode_available() -> bool { plane2: &mut plane2, }) { Ok(true) => true, - Ok(false) if std::env::var_os("SIGNINUM_REQUIRE_CUDA_BENCH").is_some() => { - panic!("SIGNINUM_REQUIRE_CUDA_BENCH is set but CUDA encode did not dispatch") + Ok(false) if std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_some() => { + panic!("J2K_REQUIRE_CUDA_BENCH is set but CUDA encode did not dispatch") } Ok(false) => { eprintln!("skipping CUDA encode benches: CUDA runtime feature is disabled"); false } - Err(error) if std::env::var_os("SIGNINUM_REQUIRE_CUDA_BENCH").is_some() => { - panic!("SIGNINUM_REQUIRE_CUDA_BENCH is set but CUDA encode failed: {error}") + Err(error) if std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_some() => { + panic!("J2K_REQUIRE_CUDA_BENCH is set but CUDA encode failed: {error}") } Err(error) => { eprintln!("skipping CUDA encode benches: {error}"); @@ -194,6 +225,11 @@ fn cpu_forward_dwt53(samples: &[f32], width: u32, height: u32, num_levels: u8) - buffer } +#[allow(clippy::cast_possible_truncation)] +fn cpu_quantize_reversible(samples: &[f32]) -> Vec { + samples.iter().map(|sample| sample.round() as i32).collect() +} + fn forward_lift_53(data: &mut [f32]) { let n = data.len(); if n < 2 { diff --git a/crates/j2k-cuda/benches/htj2k_decode.rs b/crates/j2k-cuda/benches/htj2k_decode.rs new file mode 100644 index 00000000..3fcb1658 --- /dev/null +++ b/crates/j2k-cuda/benches/htj2k_decode.rs @@ -0,0 +1,458 @@ +// SPDX-License-Identifier: Apache-2.0 + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use j2k_core::{ + BackendKind, BackendRequest, DecoderContext, DeviceSubmission, DeviceSurface, Downscale, + ImageDecode, ImageDecodeSubmit, PixelFormat, Rect, TileBatchDecodeManyDevice, +}; +use j2k_cuda::{Codec, CudaSession, J2kDecoder, SurfaceResidency}; +use j2k_native::{encode_htj2k, EncodeOptions}; + +const TILE_DIM: u32 = 512; +const BATCH_SIZES: &[usize] = &[8, 16, 32, 64]; + +struct DecodeBenchCase { + id: &'static str, + fixture: Vec, + fmt: PixelFormat, + cuda_available: bool, +} + +fn bench_htj2k_decode(c: &mut Criterion) { + let enabled_cases = enabled_decode_cases(); + let mut cases = Vec::new(); + + if enabled_cases.contains(&"gray8") { + let gray_fixture = htj2k_gray8_fixture(TILE_DIM, TILE_DIM); + cases.push(DecodeBenchCase { + id: "gray8", + cuda_available: cuda_decode_available("gray8", &gray_fixture, PixelFormat::Gray8), + fixture: gray_fixture, + fmt: PixelFormat::Gray8, + }); + } + if enabled_cases + .iter() + .any(|id| matches!(*id, "rgb8" | "rgba8")) + { + let rgb_fixture = htj2k_rgb8_fixture(TILE_DIM, TILE_DIM); + if enabled_cases.contains(&"rgb8") { + cases.push(DecodeBenchCase { + id: "rgb8", + cuda_available: cuda_decode_available("rgb8", &rgb_fixture, PixelFormat::Rgb8), + fixture: rgb_fixture.clone(), + fmt: PixelFormat::Rgb8, + }); + } + if enabled_cases.contains(&"rgba8") { + cases.push(DecodeBenchCase { + id: "rgba8", + cuda_available: cuda_decode_available("rgba8", &rgb_fixture, PixelFormat::Rgba8), + fixture: rgb_fixture, + fmt: PixelFormat::Rgba8, + }); + } + } + + let roi = Rect { + x: TILE_DIM / 4, + y: TILE_DIM / 5, + w: TILE_DIM / 2, + h: TILE_DIM / 2, + }; + let scale = Downscale::Half; + + bench_full_tile(c, &cases); + bench_roi(c, &cases, roi); + bench_scaled(c, &cases, scale); + bench_roi_scaled(c, &cases, roi, scale); + bench_tile_batch(c, &cases); +} + +fn bench_full_tile(c: &mut Criterion, cases: &[DecodeBenchCase]) { + let mut group = c.benchmark_group("j2k_cuda_htj2k_full_tile_decode"); + for case in cases { + let cpu_id = cpu_benchmark_id(case); + group.bench_with_input(BenchmarkId::new(cpu_id, TILE_DIM), case, |b, case| { + b.iter(|| { + let mut decoder = J2kDecoder::new(std::hint::black_box(case.fixture.as_slice())) + .expect("decoder"); + let stride = TILE_DIM as usize * case.fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * TILE_DIM as usize]; + decoder + .decode_into(&mut out, stride, case.fmt) + .expect("CPU HTJ2K decode"); + std::hint::black_box(out) + }); + }); + if case.cuda_available { + let cuda_id = cuda_benchmark_id(case); + group.bench_with_input(BenchmarkId::new(cuda_id, TILE_DIM), case, |b, case| { + let mut session = CudaSession::default(); + b.iter(|| { + let mut decoder = + J2kDecoder::new(std::hint::black_box(case.fixture.as_slice())) + .expect("decoder"); + let surface = decoder + .submit_to_device(&mut session, case.fmt, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K decode submission") + .wait() + .expect("strict CUDA HTJ2K decode"); + assert_cuda_resident_decode(&surface); + std::hint::black_box(surface) + }); + }); + } + } + group.finish(); +} + +fn bench_roi(c: &mut Criterion, cases: &[DecodeBenchCase], roi: Rect) { + let mut group = c.benchmark_group("j2k_cuda_htj2k_roi_decode"); + for case in cases { + let cpu_id = cpu_benchmark_id(case); + group.bench_with_input(BenchmarkId::new(cpu_id, roi.w), case, |b, case| { + b.iter(|| { + let mut decoder = J2kDecoder::new(std::hint::black_box(case.fixture.as_slice())) + .expect("decoder"); + let mut pool = j2k_cuda::J2kScratchPool::new(); + let stride = roi.w as usize * case.fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * roi.h as usize]; + decoder + .decode_region_into(&mut pool, &mut out, stride, case.fmt, roi) + .expect("CPU HTJ2K ROI decode"); + std::hint::black_box(out) + }); + }); + if case.cuda_available { + let cuda_id = cuda_benchmark_id(case); + group.bench_with_input(BenchmarkId::new(cuda_id, roi.w), case, |b, case| { + let mut session = CudaSession::default(); + b.iter(|| { + let mut decoder = + J2kDecoder::new(std::hint::black_box(case.fixture.as_slice())) + .expect("decoder"); + let surface = decoder + .submit_region_to_device(&mut session, case.fmt, roi, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K ROI decode submission") + .wait() + .expect("strict CUDA HTJ2K ROI decode"); + assert_cuda_resident_decode(&surface); + std::hint::black_box(surface) + }); + }); + } + } + group.finish(); +} + +fn bench_scaled(c: &mut Criterion, cases: &[DecodeBenchCase], scale: Downscale) { + let scaled = Rect::full((TILE_DIM, TILE_DIM)).scaled_covering(scale); + let mut group = c.benchmark_group("j2k_cuda_htj2k_scaled_decode"); + for case in cases { + let cpu_id = cpu_benchmark_id(case); + group.bench_with_input(BenchmarkId::new(cpu_id, scaled.w), case, |b, case| { + b.iter(|| { + let mut decoder = J2kDecoder::new(std::hint::black_box(case.fixture.as_slice())) + .expect("decoder"); + let mut pool = j2k_cuda::J2kScratchPool::new(); + let stride = scaled.w as usize * case.fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * scaled.h as usize]; + decoder + .decode_scaled_into(&mut pool, &mut out, stride, case.fmt, scale) + .expect("CPU HTJ2K scaled decode"); + std::hint::black_box(out) + }); + }); + if case.cuda_available { + let cuda_id = cuda_benchmark_id(case); + group.bench_with_input(BenchmarkId::new(cuda_id, scaled.w), case, |b, case| { + let mut session = CudaSession::default(); + b.iter(|| { + let mut decoder = + J2kDecoder::new(std::hint::black_box(case.fixture.as_slice())) + .expect("decoder"); + let surface = decoder + .submit_scaled_to_device( + &mut session, + case.fmt, + scale, + BackendRequest::Cuda, + ) + .expect("strict CUDA HTJ2K scaled decode submission") + .wait() + .expect("strict CUDA HTJ2K scaled decode"); + assert_cuda_resident_decode(&surface); + std::hint::black_box(surface) + }); + }); + } + } + group.finish(); +} + +fn bench_roi_scaled(c: &mut Criterion, cases: &[DecodeBenchCase], roi: Rect, scale: Downscale) { + let scaled = roi.scaled_covering(scale); + let mut group = c.benchmark_group("j2k_cuda_htj2k_roi_scaled_decode"); + for case in cases { + let cpu_id = cpu_benchmark_id(case); + group.bench_with_input(BenchmarkId::new(cpu_id, scaled.w), case, |b, case| { + b.iter(|| { + let mut decoder = J2kDecoder::new(std::hint::black_box(case.fixture.as_slice())) + .expect("decoder"); + let mut pool = j2k_cuda::J2kScratchPool::new(); + let stride = scaled.w as usize * case.fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * scaled.h as usize]; + decoder + .decode_region_scaled_into(&mut pool, &mut out, stride, case.fmt, roi, scale) + .expect("CPU HTJ2K ROI+scaled decode"); + std::hint::black_box(out) + }); + }); + if case.cuda_available { + let cuda_id = cuda_benchmark_id(case); + group.bench_with_input(BenchmarkId::new(cuda_id, scaled.w), case, |b, case| { + let mut session = CudaSession::default(); + b.iter(|| { + let mut decoder = + J2kDecoder::new(std::hint::black_box(case.fixture.as_slice())) + .expect("decoder"); + let surface = decoder + .submit_region_scaled_to_device( + &mut session, + case.fmt, + roi, + scale, + BackendRequest::Cuda, + ) + .expect("strict CUDA HTJ2K ROI+scaled decode submission") + .wait() + .expect("strict CUDA HTJ2K ROI+scaled decode"); + assert_cuda_resident_decode(&surface); + std::hint::black_box(surface) + }); + }); + } + } + group.finish(); +} + +fn bench_tile_batch(c: &mut Criterion, cases: &[DecodeBenchCase]) { + let mut group = c.benchmark_group("j2k_cuda_htj2k_tile_batch_decode"); + let batch_sizes = decode_batch_sizes(); + for case in cases { + for &batch_size in &batch_sizes { + let fixtures = vec![case.fixture.clone(); batch_size]; + let inputs = fixtures.iter().map(Vec::as_slice).collect::>(); + let fmt = case.fmt; + let cpu_id = cpu_benchmark_id(case); + group.bench_with_input( + BenchmarkId::new(cpu_id, batch_size), + &inputs, + |b, inputs| { + b.iter(|| { + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_cuda::J2kScratchPool::new(); + let surfaces = Codec::decode_tiles_to_device( + &mut ctx, + &mut pool, + std::hint::black_box(inputs), + fmt, + BackendRequest::Cpu, + ) + .expect("CPU HTJ2K batch decode"); + std::hint::black_box(surfaces) + }); + }, + ); + if case.cuda_available && cuda_batch_decode_supported(fmt) { + let cuda_id = cuda_benchmark_id(case); + group.bench_with_input( + BenchmarkId::new(cuda_id, batch_size), + &inputs, + |b, inputs| { + let mut session = CudaSession::default(); + b.iter(|| { + let surfaces = J2kDecoder::decode_batch_to_device_with_session( + std::hint::black_box(inputs), + fmt, + &mut session, + ) + .expect("strict CUDA HTJ2K real batch decode"); + assert_eq!(surfaces.len(), inputs.len()); + assert_cuda_resident_batch_decode(&surfaces); + std::hint::black_box(surfaces) + }); + }, + ); + } + } + } + group.finish(); +} + +fn cuda_batch_decode_supported(fmt: PixelFormat) -> bool { + matches!( + fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 | PixelFormat::Rgba16 + ) +} + +fn enabled_decode_cases() -> Vec<&'static str> { + let Some(value) = std::env::var_os("J2K_CUDA_DECODE_FORMATS") else { + return vec!["gray8", "rgb8", "rgba8"]; + }; + let value = value.to_string_lossy(); + let mut cases = Vec::new(); + for raw in value.split(',') { + let id = raw.trim(); + if id.is_empty() { + continue; + } + let id = match id { + "gray8" => "gray8", + "rgb8" => "rgb8", + "rgba8" => "rgba8", + other => panic!( + "unsupported J2K_CUDA_DECODE_FORMATS entry `{other}`; expected gray8,rgb8,rgba8" + ), + }; + if !cases.contains(&id) { + cases.push(id); + } + } + assert!( + !cases.is_empty(), + "J2K_CUDA_DECODE_FORMATS did not contain any decode formats" + ); + cases +} + +fn decode_batch_sizes() -> Vec { + let Some(value) = std::env::var_os("J2K_CUDA_DECODE_BATCH_SIZES") else { + return BATCH_SIZES.to_vec(); + }; + let value = value.to_string_lossy(); + let mut batch_sizes = Vec::new(); + for raw in value.split(',') { + let raw = raw.trim(); + if raw.is_empty() { + continue; + } + let batch_size = raw.parse::().unwrap_or_else(|error| { + panic!("invalid J2K_CUDA_DECODE_BATCH_SIZES entry `{raw}`: {error}") + }); + assert!( + batch_size > 0, + "J2K_CUDA_DECODE_BATCH_SIZES entries must be greater than zero" + ); + if !batch_sizes.contains(&batch_size) { + batch_sizes.push(batch_size); + } + } + assert!( + !batch_sizes.is_empty(), + "J2K_CUDA_DECODE_BATCH_SIZES did not contain any batch sizes" + ); + batch_sizes +} + +fn cpu_benchmark_id(case: &DecodeBenchCase) -> &'static str { + match case.id { + "gray8" => "cpu_gray8", + "rgb8" => "cpu_rgb8", + "rgba8" => "cpu_rgba8", + other => panic!("unknown CPU decode bench case `{other}`"), + } +} + +fn cuda_benchmark_id(case: &DecodeBenchCase) -> &'static str { + match case.id { + "gray8" => "cuda_gray8", + "rgb8" => "cuda_rgb8", + "rgba8" => "cuda_rgba8", + other => panic!("unknown CUDA decode bench case `{other}`"), + } +} + +fn assert_cuda_resident_decode(surface: &j2k_cuda::Surface) { + let cuda = assert_cuda_resident_surface(surface); + assert!(cuda.stats().decode_kernel_dispatches() > 0); +} + +fn assert_cuda_resident_batch_decode(surfaces: &[j2k_cuda::Surface]) { + assert!(!surfaces.is_empty()); + let decode_dispatches = surfaces + .iter() + .map(assert_cuda_resident_surface) + .map(|cuda| cuda.stats().decode_kernel_dispatches()) + .sum::(); + assert!(decode_dispatches > 0); +} + +fn assert_cuda_resident_surface(surface: &j2k_cuda::Surface) -> j2k_cuda::CudaSurface<'_> { + assert_eq!(surface.backend_kind(), BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert!(surface.as_host_bytes().is_none()); + let cuda = surface.cuda_surface().expect("cuda surface"); + assert_ne!(cuda.device_ptr(), 0); + assert_eq!(cuda.stats().copy_kernel_dispatches(), 0); + cuda +} + +fn cuda_decode_available(label: &str, fixture: &[u8], fmt: PixelFormat) -> bool { + let mut session = CudaSession::default(); + let result = J2kDecoder::new(fixture) + .and_then(|mut decoder| decoder.decode_to_device_with_session(fmt, &mut session)); + match result { + Ok(surface) if surface.residency() == SurfaceResidency::CudaResidentDecode => true, + Ok(_) if std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_some() => { + panic!("J2K_REQUIRE_CUDA_BENCH is set but {label} decode was not CUDA resident") + } + Ok(_) => { + eprintln!( + "skipping CUDA HTJ2K {label} decode benches: strict CUDA resident path unavailable" + ); + false + } + Err(error) if std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_some() => { + panic!("J2K_REQUIRE_CUDA_BENCH is set but {label} CUDA decode failed: {error}") + } + Err(error) => { + eprintln!("skipping CUDA HTJ2K {label} decode benches: {error}"); + false + } + } +} + +fn htj2k_gray8_fixture(width: u32, height: u32) -> Vec { + let pixels = (0..width * height) + .map(|idx| u8::try_from((idx * 17 + idx / 3) & 0xff).expect("masked sample fits in u8")) + .collect::>(); + let options = EncodeOptions { + reversible: true, + use_ht_block_coding: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, width, height, 1, 8, false, &options).expect("encode HTJ2K fixture") +} + +fn htj2k_rgb8_fixture(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for idx in 0..width * height { + pixels.push(u8::try_from((idx * 17 + idx / 3) & 0xff).expect("masked red fits")); + pixels.push(u8::try_from((idx * 29 + 7) & 0xff).expect("masked green fits")); + pixels.push(u8::try_from((idx * 43 + 19) & 0xff).expect("masked blue fits")); + } + let options = EncodeOptions { + reversible: true, + use_ht_block_coding: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, width, height, 3, 8, false, &options).expect("encode RGB HTJ2K fixture") +} + +criterion_group!(benches, bench_htj2k_decode); +criterion_main!(benches); diff --git a/crates/j2k-cuda/benches/htj2k_encode.rs b/crates/j2k-cuda/benches/htj2k_encode.rs new file mode 100644 index 00000000..e3967c3f --- /dev/null +++ b/crates/j2k-cuda/benches/htj2k_encode.rs @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: Apache-2.0 + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use j2k::{ + encode_j2k_lossless, EncodeBackendPreference, J2kBlockCodingMode, J2kEncodeValidation, + J2kLosslessEncodeOptions, J2kLosslessSamples, +}; +use j2k_core::BackendKind; +use j2k_cuda::encode_j2k_lossless_with_cuda; +use j2k_cuda_runtime::{ + CudaContext, CudaHtj2kEncodeCodeBlockJob, CudaHtj2kEncodeCodeBlockRegionJob, + CudaHtj2kEncodeTables, CudaHtj2kEncodedCodeBlocks, +}; +use j2k_native::{ + encode_ht_code_block_scalar, ht_uvlc_encode_table, ht_vlc_encode_table0, ht_vlc_encode_table1, +}; + +const TILE_DIM: u32 = 512; +const CODE_BLOCK_DIM: u32 = 64; +const CODE_BLOCK_BATCH: usize = 64; +const REGION_BLOCKS_X: u32 = 8; +const REGION_BLOCKS_Y: u32 = 8; + +fn bench_htj2k_encode(c: &mut Criterion) { + let pixels = generate_gray_tile(TILE_DIM, TILE_DIM); + let coefficients = + generate_codeblock_coefficients(CODE_BLOCK_DIM, CODE_BLOCK_DIM, CODE_BLOCK_BATCH); + let jobs = contiguous_jobs(CODE_BLOCK_DIM, CODE_BLOCK_DIM, CODE_BLOCK_BATCH); + let region_width = CODE_BLOCK_DIM * REGION_BLOCKS_X; + let region_height = CODE_BLOCK_DIM * REGION_BLOCKS_Y; + let region_coefficients = generate_region_coefficients(region_width, region_height); + let region_jobs = strided_region_jobs(CODE_BLOCK_DIM, REGION_BLOCKS_X, REGION_BLOCKS_Y); + let cuda_available = cuda_encode_available(&pixels, &coefficients, &jobs); + + bench_host_input(c, &pixels, cuda_available); + bench_codeblock_microkernels(c, &coefficients, &jobs, cuda_available); + bench_device_input_regions(c, ®ion_coefficients, ®ion_jobs, cuda_available); +} + +fn bench_host_input(c: &mut Criterion, pixels: &[u8], cuda_available: bool) { + let mut group = c.benchmark_group("j2k_cuda_htj2k_host_input_encode"); + group.bench_with_input( + BenchmarkId::new("cpu_gray8", TILE_DIM), + pixels, + |b, pixels| { + let options = cpu_htj2k_options(); + b.iter(|| { + let samples = J2kLosslessSamples::new( + std::hint::black_box(pixels), + TILE_DIM, + TILE_DIM, + 1, + 8, + false, + ) + .expect("valid gray8 samples"); + let encoded = + encode_j2k_lossless(samples, &options).expect("CPU HTJ2K lossless encode"); + assert_eq!(encoded.backend, BackendKind::Cpu); + std::hint::black_box(encoded.codestream.len()) + }); + }, + ); + + if cuda_available { + group.bench_with_input( + BenchmarkId::new("cuda_gray8", TILE_DIM), + pixels, + |b, pixels| { + let options = cuda_htj2k_options(); + b.iter(|| { + let samples = J2kLosslessSamples::new( + std::hint::black_box(pixels), + TILE_DIM, + TILE_DIM, + 1, + 8, + false, + ) + .expect("valid gray8 samples"); + let encoded = encode_j2k_lossless_with_cuda(samples, &options) + .expect("CUDA HTJ2K lossless encode"); + assert_eq!(encoded.backend, BackendKind::Cuda); + std::hint::black_box(encoded.codestream.len()) + }); + }, + ); + } + group.finish(); +} + +fn bench_codeblock_microkernels( + c: &mut Criterion, + coefficients: &[i32], + jobs: &[CudaHtj2kEncodeCodeBlockJob], + cuda_available: bool, +) { + let mut group = c.benchmark_group("j2k_cuda_htj2k_codeblock_microkernel"); + group.bench_with_input( + BenchmarkId::new("cpu_scalar_cleanup", CODE_BLOCK_BATCH), + &(coefficients, jobs), + |b, (coefficients, jobs)| { + b.iter(|| { + let encoded_bytes = jobs + .iter() + .map(|job| { + let coefficients = contiguous_block(coefficients, *job); + encode_ht_code_block_scalar( + std::hint::black_box(coefficients), + job.width, + job.height, + job.total_bitplanes, + ) + .expect("native scalar HT code-block encode") + .data + .len() + }) + .sum::(); + std::hint::black_box(encoded_bytes) + }); + }, + ); + + if cuda_available { + group.bench_with_input( + BenchmarkId::new("cuda_host_staged_cleanup", CODE_BLOCK_BATCH), + &(coefficients, jobs), + |b, (coefficients, jobs)| { + let context = CudaContext::system_default().expect("CUDA context"); + let uvlc_table = uvlc_encode_table_bytes(); + let resources = context + .upload_htj2k_encode_resources(cuda_encode_tables(&uvlc_table)) + .expect("CUDA HTJ2K encode resources"); + b.iter(|| { + let encoded = context + .encode_htj2k_codeblocks_with_resources( + std::hint::black_box(coefficients), + std::hint::black_box(jobs), + &resources, + ) + .expect("CUDA host-staged HTJ2K code-block encode"); + std::hint::black_box(assert_cuda_batch(&encoded)) + }); + }, + ); + + group.bench_with_input( + BenchmarkId::new("cuda_resident_cleanup", CODE_BLOCK_BATCH), + &(coefficients, jobs), + |b, (coefficients, jobs)| { + let context = CudaContext::system_default().expect("CUDA context"); + let coefficient_bytes = coefficients_as_bytes(coefficients); + let resident_coefficients = context + .upload(&coefficient_bytes) + .expect("resident quantized coefficients"); + let uvlc_table = uvlc_encode_table_bytes(); + let resources = context + .upload_htj2k_encode_resources(cuda_encode_tables(&uvlc_table)) + .expect("CUDA HTJ2K encode resources"); + b.iter(|| { + let encoded = context + .encode_htj2k_codeblocks_resident_with_resources( + &resident_coefficients, + coefficients.len(), + std::hint::black_box(jobs), + &resources, + ) + .expect("CUDA resident HTJ2K code-block encode"); + std::hint::black_box(assert_cuda_batch(&encoded)) + }); + }, + ); + } + group.finish(); +} + +fn bench_device_input_regions( + c: &mut Criterion, + coefficients: &[i32], + jobs: &[CudaHtj2kEncodeCodeBlockRegionJob], + cuda_available: bool, +) { + let mut group = c.benchmark_group("j2k_cuda_htj2k_device_input_encode"); + group.bench_with_input( + BenchmarkId::new("cpu_gather_scalar_cleanup", jobs.len()), + &(coefficients, jobs), + |b, (coefficients, jobs)| { + b.iter(|| { + let encoded_bytes = jobs + .iter() + .map(|job| { + let block = gather_region(coefficients, *job); + encode_ht_code_block_scalar( + std::hint::black_box(&block), + job.width, + job.height, + job.total_bitplanes, + ) + .expect("native scalar strided HT encode") + .data + .len() + }) + .sum::(); + std::hint::black_box(encoded_bytes) + }); + }, + ); + + if cuda_available { + group.bench_with_input( + BenchmarkId::new("cuda_resident_strided_cleanup", jobs.len()), + &(coefficients, jobs), + |b, (coefficients, jobs)| { + let context = CudaContext::system_default().expect("CUDA context"); + let coefficient_bytes = coefficients_as_bytes(coefficients); + let resident_coefficients = context + .upload(&coefficient_bytes) + .expect("resident strided quantized coefficients"); + let uvlc_table = uvlc_encode_table_bytes(); + let resources = context + .upload_htj2k_encode_resources(cuda_encode_tables(&uvlc_table)) + .expect("CUDA HTJ2K encode resources"); + b.iter(|| { + let encoded = context + .encode_htj2k_codeblock_regions_resident_with_resources( + &resident_coefficients, + coefficients.len(), + std::hint::black_box(jobs), + &resources, + ) + .expect("CUDA resident strided HTJ2K encode"); + std::hint::black_box(assert_cuda_batch(&encoded)) + }); + }, + ); + } + group.finish(); +} + +fn cuda_encode_available( + pixels: &[u8], + coefficients: &[i32], + jobs: &[CudaHtj2kEncodeCodeBlockJob], +) -> bool { + let samples = J2kLosslessSamples::new(pixels, TILE_DIM, TILE_DIM, 1, 8, false) + .expect("valid gray8 samples"); + let public_result = encode_j2k_lossless_with_cuda(samples, &cuda_htj2k_options()); + match public_result { + Ok(encoded) if encoded.backend == BackendKind::Cuda => {} + Ok(_) if std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_some() => { + panic!("J2K_REQUIRE_CUDA_BENCH is set but CUDA HTJ2K encode was not resident") + } + Ok(_) => { + eprintln!("skipping CUDA HTJ2K encode benches: device encode did not dispatch"); + return false; + } + Err(error) if std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_some() => { + panic!("J2K_REQUIRE_CUDA_BENCH is set but CUDA HTJ2K encode failed: {error}") + } + Err(error) => { + eprintln!("skipping CUDA HTJ2K encode benches: {error}"); + return false; + } + } + + let context = match CudaContext::system_default() { + Ok(context) => context, + Err(error) if std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_some() => { + panic!("J2K_REQUIRE_CUDA_BENCH is set but CUDA context failed: {error}") + } + Err(error) => { + eprintln!("skipping CUDA HTJ2K encode benches: {error}"); + return false; + } + }; + let uvlc_table = uvlc_encode_table_bytes(); + let resources = match context.upload_htj2k_encode_resources(cuda_encode_tables(&uvlc_table)) { + Ok(resources) => resources, + Err(error) if std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_some() => { + panic!("J2K_REQUIRE_CUDA_BENCH is set but CUDA HTJ2K encode resource upload failed: {error}") + } + Err(error) => { + eprintln!("skipping CUDA HTJ2K encode benches: {error}"); + return false; + } + }; + let result = + context.encode_htj2k_codeblocks_with_resources(coefficients, &jobs[..1], &resources); + match result { + Ok(encoded) if encoded.execution().kernel_dispatches() > 0 => true, + Ok(_) if std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_some() => { + panic!( + "J2K_REQUIRE_CUDA_BENCH is set but CUDA HTJ2K code-block encode did not dispatch" + ) + } + Ok(_) => { + eprintln!("skipping CUDA HTJ2K encode benches: code-block kernel did not dispatch"); + false + } + Err(error) if std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_some() => { + panic!("J2K_REQUIRE_CUDA_BENCH is set but CUDA HTJ2K code-block encode failed: {error}") + } + Err(error) => { + eprintln!("skipping CUDA HTJ2K encode benches: {error}"); + false + } + } +} + +fn cpu_htj2k_options() -> J2kLosslessEncodeOptions { + J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(1)) + .with_validation(J2kEncodeValidation::External) +} + +fn cuda_htj2k_options() -> J2kLosslessEncodeOptions { + J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(1)) + .with_validation(J2kEncodeValidation::External) +} + +fn cuda_encode_tables(uvlc_table: &[u8]) -> CudaHtj2kEncodeTables<'_> { + CudaHtj2kEncodeTables { + vlc_table0: ht_vlc_encode_table0(), + vlc_table1: ht_vlc_encode_table1(), + uvlc_table, + } +} + +fn uvlc_encode_table_bytes() -> Vec { + ht_uvlc_encode_table() + .iter() + .flat_map(|entry| { + [ + entry.pre, + entry.pre_len, + entry.suf, + entry.suf_len, + entry.ext, + entry.ext_len, + ] + }) + .collect() +} + +fn assert_cuda_batch(encoded: &CudaHtj2kEncodedCodeBlocks) -> usize { + assert_eq!(encoded.execution().kernel_dispatches(), 1); + assert!(encoded + .code_blocks() + .iter() + .all(|block| block.status().is_ok())); + encoded + .code_blocks() + .iter() + .map(|block| block.data().len()) + .sum() +} + +fn contiguous_block(coefficients: &[i32], job: CudaHtj2kEncodeCodeBlockJob) -> &[i32] { + let start = usize::try_from(job.coefficient_offset).expect("job offset fits usize"); + let len = area_len(job.width, job.height); + &coefficients[start..start + len] +} + +fn gather_region(coefficients: &[i32], job: CudaHtj2kEncodeCodeBlockRegionJob) -> Vec { + let start = usize::try_from(job.coefficient_offset).expect("job offset fits usize"); + let stride = usize::try_from(job.coefficient_stride).expect("job stride fits usize"); + let width = usize::try_from(job.width).expect("job width fits usize"); + let height = usize::try_from(job.height).expect("job height fits usize"); + let mut block = Vec::with_capacity(width * height); + for y in 0..height { + let row_start = start + y * stride; + block.extend_from_slice(&coefficients[row_start..row_start + width]); + } + block +} + +fn generate_gray_tile(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(area_len(width, height)); + for y in 0..height { + for x in 0..width { + let value = (x * 17 + y * 31 + x.wrapping_mul(y) / 11) & 0xff; + pixels.push(u8::try_from(value).expect("masked sample fits in u8")); + } + } + pixels +} + +fn generate_codeblock_coefficients(width: u32, height: u32, batch: usize) -> Vec { + let mut coefficients = Vec::with_capacity(area_len(width, height) * batch); + for block in 0..batch { + let block = u32::try_from(block).expect("bench block index fits u32"); + for y in 0..height { + for x in 0..width { + coefficients.push(patterned_coefficient(x, y, block)); + } + } + } + coefficients +} + +fn generate_region_coefficients(width: u32, height: u32) -> Vec { + let mut coefficients = Vec::with_capacity(area_len(width, height)); + for y in 0..height { + for x in 0..width { + coefficients.push(patterned_coefficient( + x, + y, + x / CODE_BLOCK_DIM + y / CODE_BLOCK_DIM, + )); + } + } + coefficients +} + +fn patterned_coefficient(x: u32, y: u32, block: u32) -> i32 { + if (x + y + block).is_multiple_of(7) { + return 0; + } + let raw = (x * 13 + y * 17 + block * 19 + (x ^ y)) & 0x1ff; + (i32::try_from(raw).expect("masked coefficient fits i32") - 256) / 4 +} + +fn contiguous_jobs(width: u32, height: u32, batch: usize) -> Vec { + let block_len = area_len(width, height); + let mut jobs = Vec::with_capacity(batch); + for block in 0..batch { + let offset = block + .checked_mul(block_len) + .expect("bench coefficient offset fits usize"); + jobs.push(CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: u32::try_from(offset).expect("bench offset fits u32"), + width, + height, + total_bitplanes: 8, + target_coding_passes: 1, + }); + } + jobs +} + +fn strided_region_jobs( + block_dim: u32, + blocks_x: u32, + blocks_y: u32, +) -> Vec { + let stride = block_dim + .checked_mul(blocks_x) + .expect("bench stride fits u32"); + let mut jobs = Vec::with_capacity(area_len(blocks_x, blocks_y)); + for by in 0..blocks_y { + for bx in 0..blocks_x { + let row_offset = by + .checked_mul(block_dim) + .and_then(|value| value.checked_mul(stride)) + .expect("bench row offset fits u32"); + let column_offset = bx + .checked_mul(block_dim) + .expect("bench column offset fits u32"); + jobs.push(CudaHtj2kEncodeCodeBlockRegionJob { + coefficient_offset: row_offset + column_offset, + coefficient_stride: stride, + width: block_dim, + height: block_dim, + total_bitplanes: 8, + target_coding_passes: 1, + }); + } + } + jobs +} + +fn coefficients_as_bytes(coefficients: &[i32]) -> Vec { + let mut bytes = Vec::with_capacity(std::mem::size_of_val(coefficients)); + for coefficient in coefficients { + bytes.extend_from_slice(&coefficient.to_ne_bytes()); + } + bytes +} + +fn area_len(width: u32, height: u32) -> usize { + usize::try_from(width).expect("bench width fits usize") + * usize::try_from(height).expect("bench height fits usize") +} + +criterion_group!(benches, bench_htj2k_encode); +criterion_main!(benches); diff --git a/crates/j2k-cuda/examples/htj2k_decode_profile.rs b/crates/j2k-cuda/examples/htj2k_decode_profile.rs new file mode 100644 index 00000000..1d844f91 --- /dev/null +++ b/crates/j2k-cuda/examples/htj2k_decode_profile.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::time::Instant; + +use j2k_core::PixelFormat; +use j2k_cuda::{CudaSession, J2kDecoder, SurfaceResidency}; +use j2k_native::{encode_htj2k, EncodeOptions}; + +const TILE_DIM: u32 = 512; +const DEFAULT_BATCH_SIZE: usize = 128; +const DEFAULT_ITERATIONS: usize = 4; + +fn main() { + let batch_size = env_usize("J2K_CUDA_PROFILE_BATCH_SIZE", DEFAULT_BATCH_SIZE); + let iterations = env_usize("J2K_CUDA_PROFILE_ITERATIONS", DEFAULT_ITERATIONS); + let fixture = htj2k_rgb8_fixture(TILE_DIM, TILE_DIM); + let fixtures = vec![fixture; batch_size]; + let inputs = fixtures.iter().map(Vec::as_slice).collect::>(); + let mut session = CudaSession::default(); + + let start = Instant::now(); + let mut dispatches = 0usize; + let mut ptr_xor = 0u64; + for _ in 0..iterations { + let surfaces = J2kDecoder::decode_batch_to_device_with_session( + &inputs, + PixelFormat::Rgb8, + &mut session, + ) + .expect("strict CUDA HTJ2K RGB8 batch decode"); + assert_eq!(surfaces.len(), batch_size); + for surface in surfaces { + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + let cuda = surface.cuda_surface().expect("cuda surface"); + dispatches = dispatches.saturating_add(cuda.stats().decode_kernel_dispatches()); + ptr_xor ^= cuda.device_ptr(); + } + } + let elapsed = start.elapsed(); + let tiles = batch_size.saturating_mul(iterations); + let tiles_f64 = f64::from(u32::try_from(tiles).expect("profile tile count fits in u32")); + let seconds = elapsed.as_secs_f64(); + println!( + "mode=batch_no_download tiles={tiles} batch_size={batch_size} iterations={iterations} elapsed_s={seconds:.6} tiles_per_s={:.3} decode_dispatches={dispatches} ptr_xor={ptr_xor}", + tiles_f64 / seconds + ); +} + +fn env_usize(name: &str, default: usize) -> usize { + let Some(value) = std::env::var_os(name) else { + return default; + }; + let value = value.to_string_lossy(); + let parsed = value + .parse::() + .unwrap_or_else(|error| panic!("invalid {name}={value}: {error}")); + assert!(parsed > 0, "{name} must be greater than zero"); + parsed +} + +fn htj2k_rgb8_fixture(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for idx in 0..width * height { + pixels.push(u8::try_from((idx * 17 + idx / 3) & 0xff).expect("masked red fits")); + pixels.push(u8::try_from((idx * 29 + 7) & 0xff).expect("masked green fits")); + pixels.push(u8::try_from((idx * 43 + 19) & 0xff).expect("masked blue fits")); + } + let options = EncodeOptions { + reversible: true, + use_ht_block_coding: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, width, height, 3, 8, false, &options).expect("encode RGB HTJ2K fixture") +} diff --git a/crates/signinum-j2k-cuda/src/codec.rs b/crates/j2k-cuda/src/codec.rs similarity index 54% rename from crates/signinum-j2k-cuda/src/codec.rs rename to crates/j2k-cuda/src/codec.rs index 563c3f78..f05237b7 100644 --- a/crates/signinum-j2k-cuda/src/codec.rs +++ b/crates/j2k-cuda/src/codec.rs @@ -2,19 +2,20 @@ use core::convert::Infallible; -use signinum_core::{ - BackendRequest, Downscale, ImageCodec, PixelFormat, ReadySubmission, Rect, TileBatchDecode, - TileBatchDecodeDevice, TileBatchDecodeManyDevice, TileBatchDecodeSubmit, -}; -use signinum_j2k::{ +use j2k::{ adapter::device_plan::{DeviceDecodePlan, DeviceDecodeRequest}, J2kCodec as CpuCodec, J2kContext as CpuJ2kContext, J2kDecoder as CpuDecoder, J2kScratchPool as CpuJ2kScratchPool, }; +use j2k_core::{ + submit_ready_device, BackendRequest, Downscale, ImageCodec, PixelFormat, ReadySubmission, Rect, + TileBatchDecode, TileBatchDecodeDevice, TileBatchDecodeManyDevice, TileBatchDecodeSubmit, +}; use crate::runtime::{validate_surface_request, wrap_surface}; -use crate::{CudaSession, Error, Surface}; +use crate::{CudaSession, Error, J2kDecoder, Surface}; +/// Marker type implementing tile-batch CUDA surface decode traits. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct Codec; @@ -25,8 +26,33 @@ impl ImageCodec for Codec { } impl Codec { + fn supports_cuda_batch_format(fmt: PixelFormat) -> bool { + matches!( + fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 | PixelFormat::Rgba16 + ) + } + + #[cfg(feature = "cuda-runtime")] + fn decode_tiles_to_cuda_batch( + inputs: &[&[u8]], + fmt: PixelFormat, + session: &mut CudaSession, + ) -> Result, Error> { + J2kDecoder::decode_batch_to_device_with_session(inputs, fmt, session) + } + + #[cfg(not(feature = "cuda-runtime"))] + fn decode_tiles_to_cuda_batch( + _inputs: &[&[u8]], + _fmt: PixelFormat, + _session: &mut CudaSession, + ) -> Result, Error> { + Err(Error::CudaUnavailable) + } + fn decode_tile_to_surface_impl( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, session: &mut CudaSession, pool: &mut CpuJ2kScratchPool, input: &[u8], @@ -34,6 +60,10 @@ impl Codec { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; + if matches!(backend, BackendRequest::Cuda) { + let mut decoder = J2kDecoder::new(input)?; + return decoder.decode_to_device_with_session(fmt, session); + } let dims = CpuDecoder::inspect(input)?.dimensions; let stride = dims.0 as usize * fmt.bytes_per_pixel(); let mut out = vec![0u8; stride * dims.1 as usize]; @@ -42,7 +72,7 @@ impl Codec { } fn decode_tile_region_to_surface_impl( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, session: &mut CudaSession, pool: &mut CpuJ2kScratchPool, input: &[u8], @@ -51,6 +81,10 @@ impl Codec { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; + if matches!(backend, BackendRequest::Cuda) { + let mut decoder = J2kDecoder::new(input)?; + return decoder.decode_region_to_device_with_session(fmt, roi, session); + } let dims = DeviceDecodePlan::for_image( CpuDecoder::inspect(input)?.dimensions, DeviceDecodeRequest::Region { roi }, @@ -63,7 +97,7 @@ impl Codec { } fn decode_tile_scaled_to_surface_impl( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, session: &mut CudaSession, pool: &mut CpuJ2kScratchPool, input: &[u8], @@ -72,6 +106,10 @@ impl Codec { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; + if matches!(backend, BackendRequest::Cuda) { + let mut decoder = J2kDecoder::new(input)?; + return decoder.decode_scaled_to_device_with_session(fmt, scale, session); + } let dims = DeviceDecodePlan::for_image( CpuDecoder::inspect(input)?.dimensions, DeviceDecodeRequest::Scaled { scale }, @@ -85,7 +123,7 @@ impl Codec { #[allow(clippy::too_many_arguments)] fn decode_tile_region_scaled_to_surface_impl( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, session: &mut CudaSession, pool: &mut CpuJ2kScratchPool, input: &[u8], @@ -95,6 +133,10 @@ impl Codec { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; + if matches!(backend, BackendRequest::Cuda) { + let mut decoder = J2kDecoder::new(input)?; + return decoder.decode_region_scaled_to_device_with_session(fmt, roi, scale, session); + } let dims = DeviceDecodePlan::for_image( CpuDecoder::inspect(input)?.dimensions, DeviceDecodeRequest::RegionScaled { roi, scale }, @@ -114,7 +156,7 @@ impl TileBatchDecodeSubmit for Codec { type SubmittedSurface = ReadySubmission; fn submit_tile_to_device( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, session: &mut Self::Session, pool: &mut Self::Pool, input: &[u8], @@ -122,14 +164,13 @@ impl TileBatchDecodeSubmit for Codec { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - Self::decode_tile_to_surface_impl(ctx, session, pool, input, fmt, backend), - )) + Ok(submit_ready_device(session, |session| { + Self::decode_tile_to_surface_impl(ctx, session, pool, input, fmt, backend) + })) } fn submit_tile_region_to_device( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, session: &mut Self::Session, pool: &mut Self::Pool, input: &[u8], @@ -138,14 +179,13 @@ impl TileBatchDecodeSubmit for Codec { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - Self::decode_tile_region_to_surface_impl(ctx, session, pool, input, fmt, roi, backend), - )) + Ok(submit_ready_device(session, |session| { + Self::decode_tile_region_to_surface_impl(ctx, session, pool, input, fmt, roi, backend) + })) } fn submit_tile_scaled_to_device( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, session: &mut Self::Session, pool: &mut Self::Pool, input: &[u8], @@ -154,16 +194,13 @@ impl TileBatchDecodeSubmit for Codec { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - Self::decode_tile_scaled_to_surface_impl( - ctx, session, pool, input, fmt, scale, backend, - ), - )) + Ok(submit_ready_device(session, |session| { + Self::decode_tile_scaled_to_surface_impl(ctx, session, pool, input, fmt, scale, backend) + })) } fn submit_tile_region_scaled_to_device( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, session: &mut Self::Session, pool: &mut Self::Pool, input: &[u8], @@ -173,12 +210,11 @@ impl TileBatchDecodeSubmit for Codec { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( + Ok(submit_ready_device(session, |session| { Self::decode_tile_region_scaled_to_surface_impl( ctx, session, pool, input, fmt, roi, scale, backend, - ), - )) + ) + })) } } @@ -192,14 +228,22 @@ impl TileBatchDecodeManyDevice for Codec { type DeviceSurface = Surface; fn decode_tiles_to_device( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, pool: &mut Self::Pool, inputs: &[&[u8]], fmt: PixelFormat, backend: BackendRequest, ) -> Result, Self::Error> { validate_surface_request(backend)?; + if inputs.is_empty() { + return Ok(Vec::new()); + } + let mut session = CudaSession::default(); + if matches!(backend, BackendRequest::Cuda) && Self::supports_cuda_batch_format(fmt) { + return Self::decode_tiles_to_cuda_batch(inputs, fmt, &mut session); + } + inputs .iter() .map(|input| { @@ -208,3 +252,51 @@ impl TileBatchDecodeManyDevice for Codec { .collect() } } + +#[cfg(all(test, feature = "cuda-runtime"))] +mod tests { + use j2k_core::{BackendRequest, DecoderContext, PixelFormat, TileBatchDecodeManyDevice}; + use j2k_test_support::{cuda_runtime_required, htj2k_rgb8_pattern_fixture}; + + use super::{Codec, CpuJ2kContext, CpuJ2kScratchPool}; + use crate::decoder::{ + testing_cuda_htj2k_batch_decode_calls, testing_reset_cuda_htj2k_batch_decode_calls, + }; + use crate::{Error, SurfaceResidency}; + + #[test] + fn explicit_cuda_rgb_many_decode_uses_batch_api_once() { + testing_reset_cuda_htj2k_batch_decode_calls(); + let fixture = rgb8_htj2k_fixture(32, 32); + let inputs = [fixture.as_slice(), fixture.as_slice()]; + let mut ctx = DecoderContext::::new(); + let mut pool = CpuJ2kScratchPool::new(); + + let result = Codec::decode_tiles_to_device( + &mut ctx, + &mut pool, + &inputs, + PixelFormat::Rgb8, + BackendRequest::Cuda, + ); + + assert_eq!(testing_cuda_htj2k_batch_decode_calls(), 1); + match result { + Ok(surfaces) => { + assert_eq!(surfaces.len(), inputs.len()); + for surface in surfaces { + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + } + } + Err(Error::CudaUnavailable) => { + assert!(!cuda_runtime_required()); + } + Err(error) => panic!("unexpected strict CUDA RGB batch error: {error}"), + } + } + + fn rgb8_htj2k_fixture(width: u32, height: u32) -> Vec { + htj2k_rgb8_pattern_fixture(width, height, 17) + } +} diff --git a/crates/j2k-cuda/src/decoder.rs b/crates/j2k-cuda/src/decoder.rs new file mode 100644 index 00000000..8e213100 --- /dev/null +++ b/crates/j2k-cuda/src/decoder.rs @@ -0,0 +1,3735 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(all(test, feature = "cuda-runtime"))] +use core::cell::Cell; +use core::convert::Infallible; +#[cfg(feature = "cuda-runtime")] +use std::sync::Arc; + +use j2k::{ + adapter::device_plan::{DeviceDecodePlan, DeviceDecodeRequest}, + J2kDecoder as CpuDecoder, J2kError, J2kScratchPool as CpuJ2kScratchPool, J2kView, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_core::BackendKind; +use j2k_core::{ + submit_ready_device, BackendRequest, DecodeOutcome, Downscale, ImageCodec, ImageDecode, + ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, ReadySubmission, Rect, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_cuda_runtime::{ + CudaBufferPool, CudaBufferPoolTakeTrace, CudaDeviceBuffer, CudaError, CudaHtj2kCleanupTarget, + CudaHtj2kCodeBlockJob, CudaHtj2kDecodeResources, CudaHtj2kDecodeTableResources, + CudaHtj2kDequantizeTarget, CudaJ2kIdwtJob, CudaJ2kIdwtTarget, CudaJ2kInverseMctJob, + CudaJ2kRect, CudaJ2kStoreGray16Job, CudaJ2kStoreGray8Job, CudaJ2kStoreRgb16Job, + CudaJ2kStoreRgb16MctJob, CudaJ2kStoreRgb8Job, CudaJ2kStoreRgb8MctJob, + CudaJ2kStoreRgb8MctTarget, CudaPooledDeviceBuffer, CudaQueuedExecution, CudaQueuedHtj2kCleanup, +}; +use j2k_native::{DecodeSettings, DecoderContext as NativeDecoderContext, Image as NativeImage}; + +#[cfg(feature = "cuda-runtime")] +use crate::runtime::cuda_error; +use crate::runtime::{validate_surface_request, wrap_cpu_staged_cuda_surface, wrap_surface}; +#[cfg(feature = "cuda-runtime")] +use crate::surface::{cuda_range_storage, Storage}; +use crate::{ + profile, CudaHtj2kDecodePlan, CudaHtj2kDecodeProfileDetail, CudaHtj2kProfileReport, + CudaSession, Error, Surface, +}; +#[cfg(feature = "cuda-runtime")] +use crate::{ + CudaHtj2kBandId, CudaHtj2kIdwtStep, CudaHtj2kStoreStep, CudaHtj2kTransform, CudaSurfaceStats, + SurfaceResidency, +}; + +#[cfg(feature = "cuda-runtime")] +const CUDA_HTJ2K_KERNELS_NOT_READY: &str = + "strict CUDA HTJ2K resident codestream decode kernels are not available in this build"; +#[cfg(feature = "cuda-runtime")] +const CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED: &str = + "strict CUDA HTJ2K resident decode currently accepts Gray8, Gray16, Rgb8, Rgba8, Rgb16, and Rgba16 output"; +#[cfg(feature = "cuda-runtime")] +const CUDA_HTJ2K_PLAN_INVARIANT_FAILED: &str = + "strict CUDA HTJ2K resident decode plan has invalid internal ranges"; +#[cfg(feature = "cuda-runtime")] +const CUDA_HTJ2K_STORE_UNSUPPORTED: &str = + "strict CUDA HTJ2K resident decode requires a single grayscale store step"; +#[cfg(feature = "cuda-runtime")] +const CUDA_HTJ2K_BATCH_PAYLOAD_TOO_LARGE: &str = + "strict CUDA HTJ2K resident batch decode payload is too large"; +#[cfg(feature = "cuda-runtime")] +const CUDA_IDWT_TRACE_ENV_VAR: &str = "J2K_CUDA_IDWT_TRACE"; + +#[cfg(all(test, feature = "cuda-runtime"))] +std::thread_local! { + static CUDA_HTJ2K_BATCH_DECODE_CALLS: Cell = const { Cell::new(0) }; +} + +#[cfg(all(test, feature = "cuda-runtime"))] +pub(crate) fn testing_reset_cuda_htj2k_batch_decode_calls() { + CUDA_HTJ2K_BATCH_DECODE_CALLS.with(|calls| calls.set(0)); +} + +#[cfg(all(test, feature = "cuda-runtime"))] +pub(crate) fn testing_cuda_htj2k_batch_decode_calls() -> usize { + CUDA_HTJ2K_BATCH_DECODE_CALLS.with(Cell::get) +} + +#[cfg(any(test, feature = "cuda-runtime"))] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct CudaIdwtBatchHostTraceRow { + component_count: usize, + step_count: usize, + output_alloc_us: u128, + target_build_us: u128, + enqueue_us: u128, + output_take_count: usize, + output_pool_reuse_count: usize, + output_pool_alloc_count: usize, + output_pool_scanned_count: usize, + output_pool_max_free_count: usize, + output_requested_bytes: usize, +} + +#[cfg(any(test, feature = "cuda-runtime"))] +fn format_cuda_idwt_batch_host_trace_row(row: CudaIdwtBatchHostTraceRow) -> String { + format!( + "j2k_profile codec=j2k op=cuda_idwt_batch_host path=decode \ + component_count={} step_count={} output_alloc_us={} target_build_us={} enqueue_us={} \ + output_take_count={} output_pool_reuse_count={} output_pool_alloc_count={} \ + output_pool_scanned_count={} output_pool_max_free_count={} output_requested_bytes={}", + row.component_count, + row.step_count, + row.output_alloc_us, + row.target_build_us, + row.enqueue_us, + row.output_take_count, + row.output_pool_reuse_count, + row.output_pool_alloc_count, + row.output_pool_scanned_count, + row.output_pool_max_free_count, + row.output_requested_bytes + ) +} + +#[cfg(feature = "cuda-runtime")] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +struct CudaIdwtOutputPoolTraceTotals { + take_count: usize, + reuse_count: usize, + alloc_count: usize, + scanned_count: usize, + max_free_count: usize, + requested_bytes: usize, +} + +#[cfg(feature = "cuda-runtime")] +impl CudaIdwtOutputPoolTraceTotals { + fn add_take(&mut self, trace: CudaBufferPoolTakeTrace) { + self.take_count = self.take_count.saturating_add(1); + if trace.reused { + self.reuse_count = self.reuse_count.saturating_add(1); + } else { + self.alloc_count = self.alloc_count.saturating_add(1); + } + self.scanned_count = self.scanned_count.saturating_add(trace.scanned_count); + self.max_free_count = self.max_free_count.max(trace.free_count_before); + self.requested_bytes = self.requested_bytes.saturating_add(trace.requested_len); + } +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_idwt_trace_enabled() -> bool { + std::env::var_os(CUDA_IDWT_TRACE_ENV_VAR).is_some() +} + +#[cfg(feature = "cuda-runtime")] +fn elapsed_host_us(start: Option) -> u128 { + start.map_or(0, |start| start.elapsed().as_micros()) +} + +/// CUDA-facing JPEG 2000 decoder wrapper. +pub struct J2kDecoder<'a> { + inner: CpuDecoder<'a>, + pool: CpuJ2kScratchPool, +} + +impl<'a> J2kDecoder<'a> { + /// Create a CUDA-facing decoder from compressed bytes. + pub fn new(input: &'a [u8]) -> Result { + Ok(Self { + inner: CpuDecoder::new(input)?, + pool: CpuJ2kScratchPool::new(), + }) + } + + fn decode_to_surface_impl( + &mut self, + session: &mut CudaSession, + fmt: PixelFormat, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + if matches!(backend, BackendRequest::Cuda) { + return self.decode_to_cuda_resident_surface_impl(session, fmt); + } + let dims = self.inner.info().dimensions; + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + if j2k_profile::gpu_route_profile_enabled() { + let request_s = format!("{backend:?}"); + let fmt_s = format!("{fmt:?}"); + let width_s = dims.0.to_string(); + let height_s = dims.1.to_string(); + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "full"), + ("request", request_s.as_str()), + ("fmt", fmt_s.as_str()), + ("width", width_s.as_str()), + ("height", height_s.as_str()), + ("decision", "cpu_decode_then_wrap"), + ], + ); + } + self.inner + .decode_into_with_scratch(&mut self.pool, &mut out, stride, fmt)?; + wrap_surface(out, dims, fmt, backend, session) + } + + fn decode_to_cuda_resident_surface_impl( + &mut self, + session: &mut CudaSession, + fmt: PixelFormat, + ) -> Result { + decode_to_cuda_resident_surface_impl(self, session, fmt) + } + + fn decode_region_to_cuda_resident_surface_impl( + &mut self, + session: &mut CudaSession, + fmt: PixelFormat, + roi: Rect, + ) -> Result { + decode_region_to_cuda_resident_surface_impl(self, session, fmt, roi) + } + + fn decode_scaled_to_cuda_resident_surface_impl( + &mut self, + session: &mut CudaSession, + fmt: PixelFormat, + scale: Downscale, + ) -> Result { + decode_scaled_to_cuda_resident_surface_impl(self, session, fmt, scale) + } + + fn decode_region_scaled_to_cuda_resident_surface_impl( + &mut self, + session: &mut CudaSession, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + ) -> Result { + decode_region_scaled_to_cuda_resident_surface_impl(self, session, fmt, roi, scale) + } + + fn decode_region_to_surface_impl( + &mut self, + session: &mut CudaSession, + fmt: PixelFormat, + roi: Rect, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + if matches!(backend, BackendRequest::Cuda) { + return self.decode_region_to_cuda_resident_surface_impl(session, fmt, roi); + } + let plan = DeviceDecodePlan::for_image( + self.inner.info().dimensions, + DeviceDecodeRequest::Region { roi }, + )?; + let dims = plan.output_dims(); + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + self.inner + .decode_region_into(&mut self.pool, &mut out, stride, fmt, plan.source_rect())?; + wrap_surface(out, dims, fmt, backend, session) + } + + fn decode_scaled_to_surface_impl( + &mut self, + session: &mut CudaSession, + fmt: PixelFormat, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + if matches!(backend, BackendRequest::Cuda) { + return self.decode_scaled_to_cuda_resident_surface_impl(session, fmt, scale); + } + let dims = DeviceDecodePlan::for_image( + self.inner.info().dimensions, + DeviceDecodeRequest::Scaled { scale }, + )? + .output_dims(); + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + self.inner + .decode_scaled_into(&mut self.pool, &mut out, stride, fmt, scale)?; + wrap_surface(out, dims, fmt, backend, session) + } + + fn decode_region_scaled_to_surface_impl( + &mut self, + session: &mut CudaSession, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + if matches!(backend, BackendRequest::Cuda) { + return self + .decode_region_scaled_to_cuda_resident_surface_impl(session, fmt, roi, scale); + } + let plan = DeviceDecodePlan::for_image( + self.inner.info().dimensions, + DeviceDecodeRequest::RegionScaled { roi, scale }, + )?; + let dims = plan.output_dims(); + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + self.inner.decode_region_scaled_into( + &mut self.pool, + &mut out, + stride, + fmt, + plan.source_rect(), + scale, + )?; + wrap_surface(out, dims, fmt, backend, session) + } + + /// Strictly decode a full HTJ2K image into a CUDA-backed surface using an + /// existing backend session. + pub fn decode_to_device_with_session( + &mut self, + fmt: PixelFormat, + session: &mut CudaSession, + ) -> Result { + self.decode_to_surface_impl(session, fmt, BackendRequest::Cuda) + } + + /// Strictly decode a full HTJ2K image into a CUDA-backed surface and return + /// a structured profile report for CPU planning and CUDA stages. + pub fn decode_to_device_with_session_and_profile( + &mut self, + fmt: PixelFormat, + session: &mut CudaSession, + ) -> Result<(Surface, CudaHtj2kProfileReport), Error> { + decode_to_cuda_resident_surface_with_profile_impl(self, session, fmt) + } + + /// Strictly decode a batch of full HTJ2K images into CUDA-backed surfaces + /// using an existing backend session. + pub fn decode_batch_to_device_with_session( + inputs: &[&[u8]], + fmt: PixelFormat, + session: &mut CudaSession, + ) -> Result, Error> { + decode_batch_to_cuda_resident_surface_with_profile_control(inputs, session, fmt, false) + .map(|(surfaces, _report)| surfaces) + } + + /// Strictly decode a batch of full HTJ2K images into CUDA-backed surfaces + /// and return one aggregate profile report for the shared batch. + pub fn decode_batch_to_device_with_session_and_profile( + inputs: &[&[u8]], + fmt: PixelFormat, + session: &mut CudaSession, + ) -> Result<(Vec, CudaHtj2kProfileReport), Error> { + decode_batch_to_cuda_resident_surface_with_profile_control(inputs, session, fmt, true) + } + + /// Strictly decode a full-resolution HTJ2K region into a CUDA-backed + /// surface using an existing backend session. + pub(crate) fn decode_region_to_device_with_session( + &mut self, + fmt: PixelFormat, + roi: Rect, + session: &mut CudaSession, + ) -> Result { + self.decode_region_to_surface_impl(session, fmt, roi, BackendRequest::Cuda) + } + + /// Strictly decode a reduced-resolution HTJ2K image into a CUDA-backed + /// surface using an existing backend session. + pub(crate) fn decode_scaled_to_device_with_session( + &mut self, + fmt: PixelFormat, + scale: Downscale, + session: &mut CudaSession, + ) -> Result { + self.decode_scaled_to_surface_impl(session, fmt, scale, BackendRequest::Cuda) + } + + /// Strictly decode a reduced-resolution HTJ2K region into a CUDA-backed + /// surface using an existing backend session. + pub(crate) fn decode_region_scaled_to_device_with_session( + &mut self, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + session: &mut CudaSession, + ) -> Result { + self.decode_region_scaled_to_surface_impl(session, fmt, roi, scale, BackendRequest::Cuda) + } + + /// Decode a full image through the CPU path and wrap it as a host surface. + pub fn decode_to_host_surface(&mut self, fmt: PixelFormat) -> Result { + let mut session = CudaSession::default(); + self.decode_to_surface_impl(&mut session, fmt, BackendRequest::Cpu) + } + + /// Build a flat CUDA HTJ2K grayscale decode plan and return stage timings. + pub fn build_cuda_htj2k_grayscale_plan_with_profile( + &mut self, + fmt: PixelFormat, + ) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new(self.inner.bytes(), &DecodeSettings::default()) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_grayscale_plan_with_context(&mut native_context) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let cuda_plan = CudaHtj2kDecodePlan::from_grayscale_direct_plan(&native_plan, fmt, (0, 0))?; + let flatten_us = profile::elapsed_us(flatten_start); + + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count: cuda_plan.code_blocks().len(), + payload_bytes: cuda_plan.payload().len(), + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + Ok((cuda_plan, report)) + } + + /// Build a flat CUDA HTJ2K grayscale region decode plan and return stage timings. + pub fn build_cuda_htj2k_grayscale_region_plan_with_profile( + &mut self, + fmt: PixelFormat, + roi: Rect, + ) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new(self.inner.bytes(), &DecodeSettings::default()) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_grayscale_plan_region_with_context( + &mut native_context, + (roi.x, roi.y, roi.w, roi.h), + ) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let cuda_plan = CudaHtj2kDecodePlan::from_grayscale_direct_plan_region( + &native_plan, + fmt, + (roi.x, roi.y), + (roi.w, roi.h), + )?; + let flatten_us = profile::elapsed_us(flatten_start); + + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count: cuda_plan.code_blocks().len(), + payload_bytes: cuda_plan.payload().len(), + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + Ok((cuda_plan, report)) + } + + /// Build a flat reduced-resolution CUDA HTJ2K grayscale decode plan and + /// return stage timings. + pub fn build_cuda_htj2k_grayscale_scaled_plan_with_profile( + &mut self, + fmt: PixelFormat, + output_dimensions: (u32, u32), + ) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new( + self.inner.bytes(), + &DecodeSettings { + target_resolution: Some(output_dimensions), + ..DecodeSettings::default() + }, + ) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_grayscale_plan_with_context(&mut native_context) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let cuda_plan = CudaHtj2kDecodePlan::from_grayscale_direct_plan(&native_plan, fmt, (0, 0))?; + let flatten_us = profile::elapsed_us(flatten_start); + + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count: cuda_plan.code_blocks().len(), + payload_bytes: cuda_plan.payload().len(), + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + Ok((cuda_plan, report)) + } + + /// Build a flat reduced-resolution CUDA HTJ2K grayscale region decode + /// plan and return stage timings. + pub fn build_cuda_htj2k_grayscale_region_scaled_plan_with_profile( + &mut self, + fmt: PixelFormat, + scaled_roi: Rect, + output_dimensions: (u32, u32), + ) -> Result<(CudaHtj2kDecodePlan, CudaHtj2kProfileReport), Error> { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new( + self.inner.bytes(), + &DecodeSettings { + target_resolution: Some(output_dimensions), + ..DecodeSettings::default() + }, + ) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_grayscale_plan_region_with_context( + &mut native_context, + (scaled_roi.x, scaled_roi.y, scaled_roi.w, scaled_roi.h), + ) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let cuda_plan = CudaHtj2kDecodePlan::from_grayscale_direct_plan_region( + &native_plan, + fmt, + (scaled_roi.x, scaled_roi.y), + (scaled_roi.w, scaled_roi.h), + )?; + let flatten_us = profile::elapsed_us(flatten_start); + + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count: cuda_plan.code_blocks().len(), + payload_bytes: cuda_plan.payload().len(), + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + Ok((cuda_plan, report)) + } + + /// Build flat CUDA HTJ2K RGB component plans and return stage timings. + #[cfg(feature = "cuda-runtime")] + fn build_cuda_htj2k_color_plans_with_profile( + &mut self, + fmt: PixelFormat, + ) -> Result { + let mut native_context = NativeDecoderContext::default(); + build_cuda_htj2k_color_plans_from_bytes_with_profile( + self.inner.bytes(), + fmt, + &mut native_context, + ) + } + + #[cfg(feature = "cuda-runtime")] + fn build_cuda_htj2k_color_scaled_plans_with_profile( + &mut self, + fmt: PixelFormat, + output_dimensions: (u32, u32), + ) -> Result { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new( + self.inner.bytes(), + &DecodeSettings { + target_resolution: Some(output_dimensions), + ..DecodeSettings::default() + }, + ) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_color_plan_with_context(&mut native_context) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let mut payload = Vec::new(); + let mut components = Vec::with_capacity(native_plan.component_plans.len()); + for component_plan in &native_plan.component_plans { + let mut component = + CudaHtj2kDecodePlan::from_grayscale_direct_plan(component_plan, fmt, (0, 0))?; + component.append_payload_to_shared(&mut payload)?; + components.push(component); + } + let flatten_us = profile::elapsed_us(flatten_start); + let block_count = components + .iter() + .map(|plan| plan.code_blocks().len()) + .sum::(); + let payload_bytes = payload.len(); + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count, + payload_bytes, + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + + Ok(CudaHtj2kColorDecodePlans { + dimensions: native_plan.dimensions, + mct_dimensions: native_plan.dimensions, + bit_depths: native_plan.bit_depths, + mct: native_plan.mct, + transform: CudaHtj2kTransform::from_native(native_plan.transform), + payload, + components, + report, + }) + } + + #[cfg(feature = "cuda-runtime")] + fn build_cuda_htj2k_color_region_plans_with_profile( + &mut self, + fmt: PixelFormat, + roi: Rect, + ) -> Result { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new(self.inner.bytes(), &DecodeSettings::default()) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_color_plan_with_context(&mut native_context) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let mut payload = Vec::new(); + let mut components = Vec::with_capacity(native_plan.component_plans.len()); + for component_plan in &native_plan.component_plans { + let mut component = CudaHtj2kDecodePlan::from_grayscale_direct_plan_region( + component_plan, + fmt, + (roi.x, roi.y), + (roi.w, roi.h), + )?; + component.append_payload_to_shared(&mut payload)?; + components.push(component); + } + let flatten_us = profile::elapsed_us(flatten_start); + let block_count = components + .iter() + .map(|plan| plan.code_blocks().len()) + .sum::(); + let payload_bytes = payload.len(); + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count, + payload_bytes, + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + + Ok(CudaHtj2kColorDecodePlans { + dimensions: (roi.w, roi.h), + mct_dimensions: native_plan.dimensions, + bit_depths: native_plan.bit_depths, + mct: native_plan.mct, + transform: CudaHtj2kTransform::from_native(native_plan.transform), + payload, + components, + report, + }) + } + + #[cfg(feature = "cuda-runtime")] + fn build_cuda_htj2k_color_region_scaled_plans_with_profile( + &mut self, + fmt: PixelFormat, + scaled_roi: Rect, + output_dimensions: (u32, u32), + ) -> Result { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new( + self.inner.bytes(), + &DecodeSettings { + target_resolution: Some(output_dimensions), + ..DecodeSettings::default() + }, + ) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let mut native_context = NativeDecoderContext::default(); + let native_plan = image + .build_direct_color_plan_with_context(&mut native_context) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let mut payload = Vec::new(); + let mut components = Vec::with_capacity(native_plan.component_plans.len()); + for component_plan in &native_plan.component_plans { + let mut component = CudaHtj2kDecodePlan::from_grayscale_direct_plan_region( + component_plan, + fmt, + (scaled_roi.x, scaled_roi.y), + (scaled_roi.w, scaled_roi.h), + )?; + component.append_payload_to_shared(&mut payload)?; + components.push(component); + } + let flatten_us = profile::elapsed_us(flatten_start); + let block_count = components + .iter() + .map(|plan| plan.code_blocks().len()) + .sum::(); + let payload_bytes = payload.len(); + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count, + payload_bytes, + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + + Ok(CudaHtj2kColorDecodePlans { + dimensions: (scaled_roi.w, scaled_roi.h), + mct_dimensions: native_plan.dimensions, + bit_depths: native_plan.bit_depths, + mct: native_plan.mct, + transform: CudaHtj2kTransform::from_native(native_plan.transform), + payload, + components, + report, + }) + } + + /// Decode a full image on CPU and upload it into a CUDA buffer using an + /// existing backend session. + pub fn decode_to_cpu_staged_cuda_surface_with_session( + &mut self, + fmt: PixelFormat, + session: &mut CudaSession, + ) -> Result { + let dims = self.inner.info().dimensions; + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + self.inner + .decode_into_with_scratch(&mut self.pool, &mut out, stride, fmt)?; + wrap_cpu_staged_cuda_surface(&out, dims, fmt, session) + } + + /// Decode a region on CPU and upload it into a CUDA buffer using an + /// existing backend session. + pub fn decode_region_to_cpu_staged_cuda_surface_with_session( + &mut self, + fmt: PixelFormat, + roi: Rect, + session: &mut CudaSession, + ) -> Result { + let plan = DeviceDecodePlan::for_image( + self.inner.info().dimensions, + DeviceDecodeRequest::Region { roi }, + )?; + let dims = plan.output_dims(); + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + self.inner + .decode_region_into(&mut self.pool, &mut out, stride, fmt, plan.source_rect())?; + wrap_cpu_staged_cuda_surface(&out, dims, fmt, session) + } + + /// Decode a scaled image on CPU and upload it into a CUDA buffer using an + /// existing backend session. + pub fn decode_scaled_to_cpu_staged_cuda_surface_with_session( + &mut self, + fmt: PixelFormat, + scale: Downscale, + session: &mut CudaSession, + ) -> Result { + let dims = DeviceDecodePlan::for_image( + self.inner.info().dimensions, + DeviceDecodeRequest::Scaled { scale }, + )? + .output_dims(); + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + self.inner + .decode_scaled_into(&mut self.pool, &mut out, stride, fmt, scale)?; + wrap_cpu_staged_cuda_surface(&out, dims, fmt, session) + } + + /// Decode a scaled region on CPU and upload it into a CUDA buffer using an + /// existing backend session. + pub fn decode_region_scaled_to_cpu_staged_cuda_surface_with_session( + &mut self, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + session: &mut CudaSession, + ) -> Result { + let plan = DeviceDecodePlan::for_image( + self.inner.info().dimensions, + DeviceDecodeRequest::RegionScaled { roi, scale }, + )?; + let dims = plan.output_dims(); + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + self.inner.decode_region_scaled_into( + &mut self.pool, + &mut out, + stride, + fmt, + plan.source_rect(), + scale, + )?; + wrap_cpu_staged_cuda_surface(&out, dims, fmt, session) + } +} + +#[cfg(feature = "cuda-runtime")] +struct CudaCoefficientBand { + band_id: CudaHtj2kBandId, + buffer: CudaPooledDeviceBuffer, +} + +#[cfg(feature = "cuda-runtime")] +struct CudaPendingDequantBand { + band_index: usize, + jobs: Vec, + output_words: usize, +} + +#[cfg(feature = "cuda-runtime")] +struct CudaComponentDecodeWork { + bands: Vec, + pending_dequant_bands: Vec, + store: CudaHtj2kStoreStep, + dispatches: usize, + decode_dispatches: usize, + timings: CudaDecodeStageTimings, +} + +#[cfg(feature = "cuda-runtime")] +struct CudaQueuedIdwtBatch { + queued: Vec, + kernel_dispatches: usize, + decode_dispatches: usize, +} + +#[cfg(feature = "cuda-runtime")] +struct CudaDecodedComponent { + buffer: CudaPooledDeviceBuffer, + store: CudaHtj2kStoreStep, + dispatches: usize, + decode_dispatches: usize, + timings: CudaDecodeStageTimings, +} + +#[cfg(feature = "cuda-runtime")] +struct CudaPreparedRgb8MctBatchStore { + color: CudaHtj2kColorDecodePlans, + decoded_components: Vec, + dispatches: usize, + decode_dispatches: usize, + job: CudaJ2kStoreRgb8MctJob, +} + +#[cfg(feature = "cuda-runtime")] +#[derive(Clone, Copy, Debug, Default)] +struct CudaDecodeStageTimings { + h2d: u128, + payload_upload: u128, + status_d2h: u128, + ht_cleanup: u128, + ht_refine: u128, + dequant: u128, + ht_dispatch_count: usize, + idwt: u128, + dequant_dispatch_count: usize, + idwt_dispatch_count: usize, +} + +#[cfg(feature = "cuda-runtime")] +impl CudaDecodeStageTimings { + fn add_to_report(self, report: &mut CudaHtj2kProfileReport) { + report.h2d_us = report.h2d_us.saturating_add(self.h2d); + report.detail.payload_upload_us = report + .detail + .payload_upload_us + .saturating_add(self.payload_upload); + report.detail.status_d2h_us = report.detail.status_d2h_us.saturating_add(self.status_d2h); + report.ht_cleanup_us = report.ht_cleanup_us.saturating_add(self.ht_cleanup); + report.ht_refine_us = report.ht_refine_us.saturating_add(self.ht_refine); + report.dequant_us = report.dequant_us.saturating_add(self.dequant); + report.idwt_us = report.idwt_us.saturating_add(self.idwt); + report.detail.ht_dispatch_count = report + .detail + .ht_dispatch_count + .saturating_add(self.ht_dispatch_count); + report.detail.dequant_dispatch_count = report + .detail + .dequant_dispatch_count + .saturating_add(self.dequant_dispatch_count); + report.detail.idwt_dispatch_count = report + .detail + .idwt_dispatch_count + .saturating_add(self.idwt_dispatch_count); + } +} + +#[cfg(feature = "cuda-runtime")] +struct CudaHtj2kColorDecodePlans { + dimensions: (u32, u32), + mct_dimensions: (u32, u32), + bit_depths: [u8; 3], + mct: bool, + transform: CudaHtj2kTransform, + payload: Vec, + components: Vec, + report: CudaHtj2kProfileReport, +} + +#[cfg(feature = "cuda-runtime")] +fn decode_to_cuda_resident_surface_impl( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, +) -> Result { + decode_to_cuda_resident_surface_with_profile_control(decoder, session, fmt, false) + .map(|(surface, _report)| surface) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_to_cuda_resident_surface_with_profile_impl( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, +) -> Result<(Surface, CudaHtj2kProfileReport), Error> { + decode_to_cuda_resident_surface_with_profile_control(decoder, session, fmt, true) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_to_cuda_resident_surface_with_profile_control( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + collect_stage_timings: bool, +) -> Result<(Surface, CudaHtj2kProfileReport), Error> { + let collect_stage_timings = collect_stage_timings || profile::profile_stages_enabled(); + let wall_started = profile::profile_now(collect_stage_timings); + match fmt { + PixelFormat::Gray8 | PixelFormat::Gray16 => { + decode_grayscale_cuda_resident_surface_with_profile( + decoder, + session, + fmt, + wall_started, + collect_stage_timings, + ) + } + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 | PixelFormat::Rgba16 => { + decode_color_cuda_resident_surface_with_profile( + decoder, + session, + fmt, + wall_started, + collect_stage_timings, + ) + } + _ => Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }), + } +} + +#[cfg(feature = "cuda-runtime")] +fn decode_batch_to_cuda_resident_surface_with_profile_control( + inputs: &[&[u8]], + session: &mut CudaSession, + fmt: PixelFormat, + collect_stage_timings: bool, +) -> Result<(Vec, CudaHtj2kProfileReport), Error> { + #[cfg(all(test, feature = "cuda-runtime"))] + CUDA_HTJ2K_BATCH_DECODE_CALLS.with(|calls| calls.set(calls.get().saturating_add(1))); + + let collect_stage_timings = collect_stage_timings || profile::profile_stages_enabled(); + if inputs.is_empty() { + return Ok(( + Vec::new(), + CudaHtj2kProfileReport { + residency: SurfaceResidency::CudaResidentDecode, + ..CudaHtj2kProfileReport::default() + }, + )); + } + match fmt { + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 | PixelFormat::Rgba16 => { + decode_color_cuda_resident_batch_surfaces_with_profile( + inputs, + session, + fmt, + collect_stage_timings, + ) + } + _ => Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }), + } +} + +#[cfg(feature = "cuda-runtime")] +fn decode_region_to_cuda_resident_surface_impl( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + roi: Rect, +) -> Result { + let plan = DeviceDecodePlan::for_image( + decoder.inner.info().dimensions, + DeviceDecodeRequest::Region { roi }, + )?; + if plan.is_full_frame() { + return decode_to_cuda_resident_surface_impl(decoder, session, fmt); + } + + match fmt { + PixelFormat::Gray8 | PixelFormat::Gray16 => { + decode_grayscale_cuda_resident_region_surface(decoder, session, fmt, plan.source_rect()) + } + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 | PixelFormat::Rgba16 => { + decode_color_cuda_resident_region_surface(decoder, session, fmt, plan.source_rect()) + } + _ => Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }), + } +} + +#[cfg(feature = "cuda-runtime")] +fn decode_scaled_to_cuda_resident_surface_impl( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + scale: Downscale, +) -> Result { + if scale == Downscale::None { + return decode_to_cuda_resident_surface_impl(decoder, session, fmt); + } + let output_dimensions = DeviceDecodePlan::for_image( + decoder.inner.info().dimensions, + DeviceDecodeRequest::Scaled { scale }, + )? + .output_dims(); + + match fmt { + PixelFormat::Gray8 | PixelFormat::Gray16 => { + decode_grayscale_cuda_resident_scaled_surface(decoder, session, fmt, output_dimensions) + } + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 | PixelFormat::Rgba16 => { + decode_color_cuda_resident_scaled_surface(decoder, session, fmt, output_dimensions) + } + _ => Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }), + } +} + +#[cfg(feature = "cuda-runtime")] +fn decode_region_scaled_to_cuda_resident_surface_impl( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, +) -> Result { + if scale == Downscale::None { + return decode_region_to_cuda_resident_surface_impl(decoder, session, fmt, roi); + } + let source_dimensions = decoder.inner.info().dimensions; + let scaled_dimensions = + DeviceDecodePlan::for_image(source_dimensions, DeviceDecodeRequest::Scaled { scale })? + .output_dims(); + let plan = DeviceDecodePlan::for_image( + source_dimensions, + DeviceDecodeRequest::RegionScaled { roi, scale }, + )?; + let scaled_roi = plan.output_rect(); + + match fmt { + PixelFormat::Gray8 | PixelFormat::Gray16 => { + decode_grayscale_cuda_resident_region_scaled_surface( + decoder, + session, + fmt, + scaled_roi, + scaled_dimensions, + ) + } + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 | PixelFormat::Rgba16 => { + decode_color_cuda_resident_region_scaled_surface( + decoder, + session, + fmt, + scaled_roi, + scaled_dimensions, + ) + } + _ => Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED, + }), + } +} + +#[cfg(feature = "cuda-runtime")] +fn decode_grayscale_cuda_resident_surface_with_profile( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + wall_started: Option, + collect_stage_timings: bool, +) -> Result<(Surface, CudaHtj2kProfileReport), Error> { + let (plan, mut report) = decoder.build_cuda_htj2k_grayscale_plan_with_profile(fmt)?; + decode_grayscale_cuda_resident_surface_with_plan_profile( + session, + fmt, + &plan, + &mut report, + wall_started, + collect_stage_timings, + ) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_grayscale_cuda_resident_region_surface( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + roi: Rect, +) -> Result { + let collect_stage_timings = profile::profile_stages_enabled(); + let wall_started = profile::profile_now(collect_stage_timings); + let (plan, mut report) = + decoder.build_cuda_htj2k_grayscale_region_plan_with_profile(fmt, roi)?; + decode_grayscale_cuda_resident_surface_with_plan_profile( + session, + fmt, + &plan, + &mut report, + wall_started, + collect_stage_timings, + ) + .map(|(surface, _report)| surface) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_grayscale_cuda_resident_scaled_surface( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + output_dimensions: (u32, u32), +) -> Result { + let collect_stage_timings = profile::profile_stages_enabled(); + let wall_started = profile::profile_now(collect_stage_timings); + let (plan, mut report) = + decoder.build_cuda_htj2k_grayscale_scaled_plan_with_profile(fmt, output_dimensions)?; + decode_grayscale_cuda_resident_surface_with_plan_profile( + session, + fmt, + &plan, + &mut report, + wall_started, + collect_stage_timings, + ) + .map(|(surface, _report)| surface) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_grayscale_cuda_resident_region_scaled_surface( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + scaled_roi: Rect, + scaled_dimensions: (u32, u32), +) -> Result { + let collect_stage_timings = profile::profile_stages_enabled(); + let wall_started = profile::profile_now(collect_stage_timings); + let (plan, mut report) = decoder.build_cuda_htj2k_grayscale_region_scaled_plan_with_profile( + fmt, + scaled_roi, + scaled_dimensions, + )?; + decode_grayscale_cuda_resident_surface_with_plan_profile( + session, + fmt, + &plan, + &mut report, + wall_started, + collect_stage_timings, + ) + .map(|(surface, _report)| surface) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_grayscale_cuda_resident_surface_with_plan_profile( + session: &mut CudaSession, + fmt: PixelFormat, + plan: &CudaHtj2kDecodePlan, + report: &mut CudaHtj2kProfileReport, + wall_started: Option, + collect_stage_timings: bool, +) -> Result<(Surface, CudaHtj2kProfileReport), Error> { + let context = session.cuda_context()?; + let table_upload_start = profile::profile_now(collect_stage_timings); + let table_resources = session.htj2k_decode_table_resources()?; + let table_upload_us = profile::elapsed_us(table_upload_start); + report.h2d_us = report.h2d_us.saturating_add(table_upload_us); + report.detail.table_upload_us = report + .detail + .table_upload_us + .saturating_add(table_upload_us); + let pool = session.decode_buffer_pool()?; + let component = decode_cuda_component_plan( + &context, + plan, + &table_resources, + &pool, + collect_stage_timings, + )?; + let input_width = component + .store + .input_rect + .x1 + .saturating_sub(component.store.input_rect.x0); + let component_buffer = pooled_cuda_buffer(&component.buffer)?; + let (store_output, store_us) = context + .time_default_stream_named_us_if( + collect_stage_timings, + "j2k.htj2k.decode.store.gray", + || match fmt { + PixelFormat::Gray8 => context.j2k_store_gray8_device( + component_buffer, + CudaJ2kStoreGray8Job { + input_width, + source_x: component.store.source_x, + source_y: component.store.source_y, + copy_width: component.store.copy_width, + copy_height: component.store.copy_height, + output_width: component.store.output_width, + output_height: component.store.output_height, + output_x: component.store.output_x, + output_y: component.store.output_y, + addend: component.store.addend, + bit_depth: u32::from(plan.bit_depth()), + }, + ), + PixelFormat::Gray16 => context.j2k_store_gray16_device( + component_buffer, + CudaJ2kStoreGray16Job { + input_width, + source_x: component.store.source_x, + source_y: component.store.source_y, + copy_width: component.store.copy_width, + copy_height: component.store.copy_height, + output_width: component.store.output_width, + output_height: component.store.output_height, + output_x: component.store.output_x, + output_y: component.store.output_y, + addend: component.store.addend, + bit_depth: u32::from(plan.bit_depth()), + }, + ), + _ => Err(CudaError::InvalidArgument { + message: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED.to_string(), + }), + }, + ) + .map_err(cuda_error)?; + let (surface_buffer, store_stats) = store_output.into_parts(); + let dispatches = component + .dispatches + .saturating_add(store_stats.kernel_dispatches()); + let decode_dispatches = component + .decode_dispatches + .saturating_add(store_stats.decode_kernel_dispatches()); + report.dispatch_count = dispatches; + component.timings.add_to_report(report); + report.store_us = report.store_us.saturating_add(store_us); + report.detail.store_dispatch_count = report + .detail + .store_dispatch_count + .saturating_add(store_stats.kernel_dispatches()); + report.detail.wall_total_us = profile::elapsed_us(wall_started); + profile::finalize_decode_total_us(report); + report.emit("decode"); + + let dimensions = (component.store.output_width, component.store.output_height); + let surface = Surface { + backend: BackendKind::Cuda, + residency: SurfaceResidency::CudaResidentDecode, + dimensions, + fmt, + pitch_bytes: dimensions.0 as usize * fmt.bytes_per_pixel(), + stats: CudaSurfaceStats { + total: dispatches, + copy: 0, + decode: decode_dispatches, + }, + storage: Storage::Cuda(surface_buffer), + }; + Ok((surface, report.clone())) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_color_cuda_resident_surface_with_profile( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + wall_started: Option, + collect_stage_timings: bool, +) -> Result<(Surface, CudaHtj2kProfileReport), Error> { + let color = decoder.build_cuda_htj2k_color_plans_with_profile(fmt)?; + decode_color_cuda_resident_surface_with_plans_profile( + session, + fmt, + color, + wall_started, + collect_stage_timings, + ) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_color_cuda_resident_scaled_surface( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + output_dimensions: (u32, u32), +) -> Result { + let collect_stage_timings = profile::profile_stages_enabled(); + let wall_started = profile::profile_now(collect_stage_timings); + let color = decoder.build_cuda_htj2k_color_scaled_plans_with_profile(fmt, output_dimensions)?; + decode_color_cuda_resident_surface_with_plans_profile( + session, + fmt, + color, + wall_started, + collect_stage_timings, + ) + .map(|(surface, _report)| surface) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_color_cuda_resident_region_surface( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + roi: Rect, +) -> Result { + let collect_stage_timings = profile::profile_stages_enabled(); + let wall_started = profile::profile_now(collect_stage_timings); + let color = decoder.build_cuda_htj2k_color_region_plans_with_profile(fmt, roi)?; + decode_color_cuda_resident_surface_with_plans_profile( + session, + fmt, + color, + wall_started, + collect_stage_timings, + ) + .map(|(surface, _report)| surface) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_color_cuda_resident_region_scaled_surface( + decoder: &mut J2kDecoder<'_>, + session: &mut CudaSession, + fmt: PixelFormat, + scaled_roi: Rect, + scaled_dimensions: (u32, u32), +) -> Result { + let collect_stage_timings = profile::profile_stages_enabled(); + let wall_started = profile::profile_now(collect_stage_timings); + let color = decoder.build_cuda_htj2k_color_region_scaled_plans_with_profile( + fmt, + scaled_roi, + scaled_dimensions, + )?; + decode_color_cuda_resident_surface_with_plans_profile( + session, + fmt, + color, + wall_started, + collect_stage_timings, + ) + .map(|(surface, _report)| surface) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_color_cuda_resident_surface_with_plans_profile( + session: &mut CudaSession, + fmt: PixelFormat, + mut color: CudaHtj2kColorDecodePlans, + wall_started: Option, + collect_stage_timings: bool, +) -> Result<(Surface, CudaHtj2kProfileReport), Error> { + if color.components.len() != 3 { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + let context = session.cuda_context()?; + let pool = session.decode_buffer_pool()?; + let table_upload_start = profile::profile_now(collect_stage_timings); + let table_resources = session.htj2k_decode_table_resources()?; + let table_upload_us = profile::elapsed_us(table_upload_start); + color.report.h2d_us = color.report.h2d_us.saturating_add(table_upload_us); + color.report.detail.table_upload_us = color + .report + .detail + .table_upload_us + .saturating_add(table_upload_us); + let payload_upload_start = profile::profile_now(collect_stage_timings); + let decode_resources = context + .upload_htj2k_decode_resources_with_tables(&color.payload, &table_resources) + .map_err(cuda_error)?; + let payload_upload_us = profile::elapsed_us(payload_upload_start); + profile::add_payload_resource_upload_us(&mut color.report, payload_upload_us); + let mut component_work = Vec::with_capacity(3); + for plan in &color.components { + component_work.push(decode_cuda_component_subbands_with_resources( + &context, + plan, + &pool, + collect_stage_timings, + )?); + } + run_component_cleanup_dequant_batches( + &context, + &decode_resources, + &mut component_work, + &pool, + collect_stage_timings, + )?; + finish_color_cuda_resident_surface_with_component_work( + &context, + &pool, + fmt, + color, + component_work, + wall_started, + collect_stage_timings, + true, + true, + ) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_color_cuda_resident_batch_surfaces_with_profile( + inputs: &[&[u8]], + session: &mut CudaSession, + fmt: PixelFormat, + collect_stage_timings: bool, +) -> Result<(Vec, CudaHtj2kProfileReport), Error> { + let batch_wall_started = profile::profile_now(collect_stage_timings); + let mut colors = Vec::with_capacity(inputs.len()); + let mut shared_payload = Vec::new(); + let mut native_context = NativeDecoderContext::default(); + for input in inputs { + let mut color = + build_cuda_htj2k_color_plans_from_bytes_with_profile(input, fmt, &mut native_context)?; + if color.components.len() != 3 { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + append_color_payload_to_shared(&mut color, &mut shared_payload)?; + colors.push(color); + } + + let context = session.cuda_context()?; + let pool = session.decode_batch_buffer_pool()?; + let table_upload_start = profile::profile_now(collect_stage_timings); + let table_resources = session.htj2k_decode_table_resources()?; + let table_upload_us = profile::elapsed_us(table_upload_start); + let payload_upload_start = profile::profile_now(collect_stage_timings); + let decode_resources = context + .upload_htj2k_decode_resources_with_tables_and_pool( + &shared_payload, + &table_resources, + &pool, + ) + .map_err(cuda_error)?; + let payload_upload_us = profile::elapsed_us(payload_upload_start); + + let component_count = colors + .iter() + .map(|color| color.components.len()) + .sum::(); + let mut all_component_work = Vec::with_capacity(component_count); + for color in &colors { + for plan in &color.components { + all_component_work.push(decode_cuda_component_subbands_with_resources( + &context, + plan, + &pool, + collect_stage_timings, + )?); + } + } + run_component_cleanup_dequant_batches( + &context, + &decode_resources, + &mut all_component_work, + &pool, + collect_stage_timings, + )?; + let batch_components = colors + .iter() + .flat_map(|color| color.components.iter()) + .collect::>(); + let idwt_batched = can_batch_color_idwt(&batch_components); + let pending_idwt_batch = if idwt_batched { + run_color_component_idwt_batches( + &context, + &batch_components, + &mut all_component_work, + &pool, + collect_stage_timings, + )? + } else { + None + }; + drop(batch_components); + + let can_use_batch_store = + idwt_batched && can_batch_rgb8_mct_color_store(fmt, &colors, &all_component_work)?; + let (surfaces, reports) = if can_use_batch_store { + finish_color_cuda_resident_batch_surfaces_with_rgb8_mct_store( + &context, + fmt, + colors, + all_component_work, + collect_stage_timings, + )? + } else { + let mut surfaces = Vec::with_capacity(colors.len()); + let mut reports = Vec::with_capacity(colors.len()); + let mut work_iter = all_component_work.into_iter(); + for color in colors { + let component_count = color.components.len(); + let component_work = work_iter.by_ref().take(component_count).collect::>(); + if component_work.len() != component_count { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + let (surface, report) = finish_color_cuda_resident_surface_with_component_work( + &context, + &pool, + fmt, + color, + component_work, + None, + collect_stage_timings, + !idwt_batched, + false, + )?; + surfaces.push(surface); + reports.push(report); + } + (surfaces, reports) + }; + drop(pending_idwt_batch); + + let aggregate = finalize_color_batch_decode_report( + &reports, + table_upload_us, + payload_upload_us, + batch_wall_started, + ); + aggregate.emit("decode_batch"); + + Ok((surfaces, aggregate)) +} + +#[cfg(feature = "cuda-runtime")] +fn build_cuda_htj2k_color_plans_from_bytes_with_profile<'a>( + input: &'a [u8], + fmt: PixelFormat, + native_context: &mut NativeDecoderContext<'a>, +) -> Result { + let total_start = profile::profile_now(true); + + let parse_start = profile::profile_now(true); + let image = NativeImage::new(input, &DecodeSettings::default()) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let parse_us = profile::elapsed_us(parse_start); + + let plan_start = profile::profile_now(true); + let native_plan = image + .build_direct_color_plan_with_context(native_context) + .map_err(|error| Error::Decode(J2kError::Backend(error.to_string())))?; + let plan_us = profile::elapsed_us(plan_start); + + let flatten_start = profile::profile_now(true); + let mut payload = Vec::new(); + let mut components = Vec::with_capacity(native_plan.component_plans.len()); + for component_plan in &native_plan.component_plans { + let mut component = + CudaHtj2kDecodePlan::from_grayscale_direct_plan(component_plan, fmt, (0, 0))?; + component.append_payload_to_shared(&mut payload)?; + components.push(component); + } + let flatten_us = profile::elapsed_us(flatten_start); + let block_count = components + .iter() + .map(|plan| plan.code_blocks().len()) + .sum::(); + let payload_bytes = payload.len(); + let report = CudaHtj2kProfileReport { + parse_us, + plan_us, + flatten_us, + total_us: profile::elapsed_us(total_start), + block_count, + payload_bytes, + dispatch_count: 0, + residency: crate::SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + ..CudaHtj2kProfileReport::default() + }; + report.emit("plan"); + + Ok(CudaHtj2kColorDecodePlans { + dimensions: native_plan.dimensions, + mct_dimensions: native_plan.dimensions, + bit_depths: native_plan.bit_depths, + mct: native_plan.mct, + transform: CudaHtj2kTransform::from_native(native_plan.transform), + payload, + components, + report, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn finalize_color_batch_decode_report( + reports: &[CudaHtj2kProfileReport], + table_upload_us: u128, + payload_upload_us: u128, + batch_wall_started: Option, +) -> CudaHtj2kProfileReport { + let mut aggregate = aggregate_decode_reports(reports); + aggregate.h2d_us = aggregate + .h2d_us + .saturating_add(table_upload_us) + .saturating_add(payload_upload_us); + aggregate.detail.table_upload_us = aggregate + .detail + .table_upload_us + .saturating_add(table_upload_us); + aggregate.detail.payload_upload_us = aggregate + .detail + .payload_upload_us + .saturating_add(payload_upload_us); + aggregate.detail.wall_total_us = profile::elapsed_us(batch_wall_started); + profile::finalize_decode_total_us(&mut aggregate); + aggregate +} + +#[cfg(feature = "cuda-runtime")] +fn can_batch_rgb8_mct_color_store( + fmt: PixelFormat, + colors: &[CudaHtj2kColorDecodePlans], + all_component_work: &[CudaComponentDecodeWork], +) -> Result { + if !matches!(fmt, PixelFormat::Rgb8 | PixelFormat::Rgba8) { + return Ok(false); + } + + let mut offset = 0usize; + for color in colors { + let component_count = color.components.len(); + if component_count != 3 || offset.saturating_add(component_count) > all_component_work.len() + { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + if !color.mct { + return Ok(false); + } + let component_work = &all_component_work[offset..offset + component_count]; + let stores = [ + &component_work[0].store, + &component_work[1].store, + &component_work[2].store, + ]; + validate_color_stores(stores, color.dimensions)?; + if !can_fuse_mct_store_for_stores(stores) { + return Ok(false); + } + offset = offset.saturating_add(component_count); + } + + if offset != all_component_work.len() { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + Ok(!colors.is_empty()) +} + +#[cfg(feature = "cuda-runtime")] +fn finish_color_cuda_resident_batch_surfaces_with_rgb8_mct_store( + context: &j2k_cuda_runtime::CudaContext, + fmt: PixelFormat, + colors: Vec, + all_component_work: Vec, + collect_stage_timings: bool, +) -> Result<(Vec, Vec), Error> { + let mut prepared = Vec::with_capacity(colors.len()); + let mut work_iter = all_component_work.into_iter(); + for color in colors { + let component_count = color.components.len(); + let component_work = work_iter.by_ref().take(component_count).collect::>(); + if component_work.len() != component_count { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + prepared.push(prepare_rgb8_mct_batch_store(fmt, color, component_work)?); + } + if work_iter.next().is_some() { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + + let targets = prepared + .iter() + .map(rgb8_mct_batch_store_target) + .collect::, Error>>()?; + let (store_output, store_us) = context + .time_default_stream_named_us_if( + collect_stage_timings, + "j2k.htj2k.decode.store.color.batch", + || context.j2k_store_rgb8_mct_batch_contiguous_device(&targets), + ) + .map_err(cuda_error)?; + drop(targets); + let (surface_buffer, surface_ranges, store_stats) = store_output.into_parts(); + if surface_ranges.len() != prepared.len() { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + let shared_surface_buffer = Arc::new(surface_buffer); + + let mut surfaces = Vec::with_capacity(prepared.len()); + let mut reports = Vec::with_capacity(prepared.len()); + let store_dispatches = store_stats.kernel_dispatches(); + let store_decode_dispatches = store_stats.decode_kernel_dispatches(); + for (index, (mut prepared, surface_range)) in + prepared.into_iter().zip(surface_ranges).enumerate() + { + let report_store_dispatches = if index == 0 { store_dispatches } else { 0 }; + let report_store_decode_dispatches = if index == 0 { + store_decode_dispatches + } else { + 0 + }; + let report_store_us = if index == 0 { store_us } else { 0 }; + let dispatches = prepared.dispatches.saturating_add(report_store_dispatches); + let decode_dispatches = prepared + .decode_dispatches + .saturating_add(report_store_decode_dispatches); + prepared.color.report.dispatch_count = dispatches; + prepared.color.report.store_us = prepared + .color + .report + .store_us + .saturating_add(report_store_us); + prepared.color.report.detail.store_dispatch_count = prepared + .color + .report + .detail + .store_dispatch_count + .saturating_add(report_store_dispatches); + profile::finalize_decode_total_us(&mut prepared.color.report); + + let dimensions = prepared.color.dimensions; + surfaces.push(Surface { + backend: BackendKind::Cuda, + residency: SurfaceResidency::CudaResidentDecode, + dimensions, + fmt, + pitch_bytes: dimensions.0 as usize * fmt.bytes_per_pixel(), + stats: CudaSurfaceStats { + total: dispatches, + copy: 0, + decode: decode_dispatches, + }, + storage: cuda_range_storage( + shared_surface_buffer.clone(), + surface_range.offset, + surface_range.len, + ), + }); + reports.push(prepared.color.report); + } + + Ok((surfaces, reports)) +} + +#[cfg(feature = "cuda-runtime")] +fn prepare_rgb8_mct_batch_store( + fmt: PixelFormat, + mut color: CudaHtj2kColorDecodePlans, + component_work: Vec, +) -> Result { + let decoded_components = component_work + .into_iter() + .map(finish_cuda_component_decode) + .collect::, Error>>()?; + let [component0, component1, component2] = decoded_components.as_slice() else { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + }; + let stores = [&component0.store, &component1.store, &component2.store]; + validate_color_stores(stores, color.dimensions)?; + if !color.mct || !can_fuse_mct_store_for_stores(stores) { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + + let dispatches = decoded_components + .iter() + .map(|component| component.dispatches) + .sum::(); + let decode_dispatches = decoded_components + .iter() + .map(|component| component.decode_dispatches) + .sum::(); + for component in &decoded_components { + component.timings.add_to_report(&mut color.report); + } + + let addends = [ + bit_depth_addend(color.bit_depths[0]), + bit_depth_addend(color.bit_depths[1]), + bit_depth_addend(color.bit_depths[2]), + ]; + let job = CudaJ2kStoreRgb8MctJob { + store: CudaJ2kStoreRgb8Job { + input_width0: color_store_input_width(&component0.store), + input_width1: color_store_input_width(&component1.store), + input_width2: color_store_input_width(&component2.store), + source_x0: component0.store.source_x, + source_y0: component0.store.source_y, + source_x1: component1.store.source_x, + source_y1: component1.store.source_y, + source_x2: component2.store.source_x, + source_y2: component2.store.source_y, + copy_width: component0.store.copy_width, + copy_height: component0.store.copy_height, + output_width: component0.store.output_width, + output_height: component0.store.output_height, + output_x: component0.store.output_x, + output_y: component0.store.output_y, + addend0: addends[0], + addend1: addends[1], + addend2: addends[2], + bit_depth0: u32::from(color.bit_depths[0]), + bit_depth1: u32::from(color.bit_depths[1]), + bit_depth2: u32::from(color.bit_depths[2]), + rgba: u32::from(fmt == PixelFormat::Rgba8), + }, + irreversible97: u32::from(color.transform == CudaHtj2kTransform::Irreversible97), + }; + + Ok(CudaPreparedRgb8MctBatchStore { + color, + decoded_components, + dispatches, + decode_dispatches, + job, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn rgb8_mct_batch_store_target( + prepared: &CudaPreparedRgb8MctBatchStore, +) -> Result, Error> { + let [component0, component1, component2] = prepared.decoded_components.as_slice() else { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + }; + Ok(CudaJ2kStoreRgb8MctTarget { + plane0: pooled_cuda_buffer(&component0.buffer)?, + plane1: pooled_cuda_buffer(&component1.buffer)?, + plane2: pooled_cuda_buffer(&component2.buffer)?, + job: prepared.job, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn can_fuse_mct_store_for_stores(stores: [&CudaHtj2kStoreStep; 3]) -> bool { + let input_width0 = color_store_input_width(stores[0]); + let input_width1 = color_store_input_width(stores[1]); + let input_width2 = color_store_input_width(stores[2]); + input_width0 == input_width1 + && input_width0 == input_width2 + && stores[0].source_x == stores[1].source_x + && stores[0].source_x == stores[2].source_x + && stores[0].source_y == stores[1].source_y + && stores[0].source_y == stores[2].source_y +} + +#[cfg(feature = "cuda-runtime")] +fn color_store_input_width(store: &CudaHtj2kStoreStep) -> u32 { + store.input_rect.x1.saturating_sub(store.input_rect.x0) +} + +#[cfg(feature = "cuda-runtime")] +#[allow(clippy::too_many_arguments)] +fn finish_color_cuda_resident_surface_with_component_work( + context: &j2k_cuda_runtime::CudaContext, + pool: &CudaBufferPool, + fmt: PixelFormat, + mut color: CudaHtj2kColorDecodePlans, + mut component_work: Vec, + wall_started: Option, + collect_stage_timings: bool, + run_idwt: bool, + emit_report: bool, +) -> Result<(Surface, CudaHtj2kProfileReport), Error> { + let pending_idwt_batch = if run_idwt { + let batch_components = color.components.iter().collect::>(); + if can_batch_color_idwt(&batch_components) { + run_color_component_idwt_batches( + context, + &batch_components, + &mut component_work, + pool, + collect_stage_timings, + )? + } else { + for (plan, work) in color.components.iter().zip(component_work.iter_mut()) { + run_cuda_component_idwt_steps( + context, + plan.idwt_steps(), + work, + pool, + collect_stage_timings, + )?; + } + None + } + } else { + None + }; + let decoded_components = component_work + .into_iter() + .map(finish_cuda_component_decode) + .collect::, Error>>()?; + let [component0, component1, component2] = decoded_components.as_slice() else { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + }; + validate_color_stores( + [&component0.store, &component1.store, &component2.store], + color.dimensions, + )?; + + let mut dispatches = decoded_components + .iter() + .map(|component| component.dispatches) + .sum::(); + let mut decode_dispatches = decoded_components + .iter() + .map(|component| component.decode_dispatches) + .sum::(); + for component in &decoded_components { + component.timings.add_to_report(&mut color.report); + } + let component0_buffer = pooled_cuda_buffer(&component0.buffer)?; + let component1_buffer = pooled_cuda_buffer(&component1.buffer)?; + let component2_buffer = pooled_cuda_buffer(&component2.buffer)?; + let input_width0 = component0 + .store + .input_rect + .x1 + .saturating_sub(component0.store.input_rect.x0); + let input_width1 = component1 + .store + .input_rect + .x1 + .saturating_sub(component1.store.input_rect.x0); + let input_width2 = component2 + .store + .input_rect + .x1 + .saturating_sub(component2.store.input_rect.x0); + let irreversible97 = u32::from(color.transform == CudaHtj2kTransform::Irreversible97); + let mct_store_addends = [ + bit_depth_addend(color.bit_depths[0]), + bit_depth_addend(color.bit_depths[1]), + bit_depth_addend(color.bit_depths[2]), + ]; + let can_fuse_mct_store = color.mct + && input_width0 == input_width1 + && input_width0 == input_width2 + && component0.store.source_x == component1.store.source_x + && component0.store.source_x == component2.store.source_x + && component0.store.source_y == component1.store.source_y + && component0.store.source_y == component2.store.source_y; + let addends = if color.mct && can_fuse_mct_store { + mct_store_addends + } else if color.mct { + let mct_len = u32::try_from(checked_area( + color.mct_dimensions.0, + color.mct_dimensions.1, + )?) + .map_err(|_| Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + })?; + let stats = context + .time_default_stream_named_us_if(collect_stage_timings, "j2k.htj2k.decode.mct", || { + context.j2k_inverse_mct_device( + component0_buffer, + component1_buffer, + component2_buffer, + CudaJ2kInverseMctJob { + len: mct_len, + irreversible97, + addend0: mct_store_addends[0], + addend1: mct_store_addends[1], + addend2: mct_store_addends[2], + }, + ) + }) + .map_err(cuda_error)?; + let (stats, mct_us) = stats; + dispatches = dispatches.saturating_add(stats.kernel_dispatches()); + decode_dispatches = decode_dispatches.saturating_add(stats.decode_kernel_dispatches()); + color.report.mct_us = color.report.mct_us.saturating_add(mct_us); + color.report.detail.mct_dispatch_count = color + .report + .detail + .mct_dispatch_count + .saturating_add(stats.kernel_dispatches()); + [0.0, 0.0, 0.0] + } else { + [ + component0.store.addend, + component1.store.addend, + component2.store.addend, + ] + }; + let (store_output, store_us) = context + .time_default_stream_named_us_if( + collect_stage_timings, + "j2k.htj2k.decode.store.color", + || match fmt { + PixelFormat::Rgb8 | PixelFormat::Rgba8 => { + let store_job = CudaJ2kStoreRgb8Job { + input_width0, + input_width1, + input_width2, + source_x0: component0.store.source_x, + source_y0: component0.store.source_y, + source_x1: component1.store.source_x, + source_y1: component1.store.source_y, + source_x2: component2.store.source_x, + source_y2: component2.store.source_y, + copy_width: component0.store.copy_width, + copy_height: component0.store.copy_height, + output_width: component0.store.output_width, + output_height: component0.store.output_height, + output_x: component0.store.output_x, + output_y: component0.store.output_y, + addend0: addends[0], + addend1: addends[1], + addend2: addends[2], + bit_depth0: u32::from(color.bit_depths[0]), + bit_depth1: u32::from(color.bit_depths[1]), + bit_depth2: u32::from(color.bit_depths[2]), + rgba: u32::from(fmt == PixelFormat::Rgba8), + }; + if can_fuse_mct_store { + context.j2k_store_rgb8_mct_device( + component0_buffer, + component1_buffer, + component2_buffer, + CudaJ2kStoreRgb8MctJob { + store: store_job, + irreversible97, + }, + ) + } else { + context.j2k_store_rgb8_device( + component0_buffer, + component1_buffer, + component2_buffer, + store_job, + ) + } + } + PixelFormat::Rgb16 | PixelFormat::Rgba16 => { + let store_job = CudaJ2kStoreRgb16Job { + input_width0, + input_width1, + input_width2, + source_x0: component0.store.source_x, + source_y0: component0.store.source_y, + source_x1: component1.store.source_x, + source_y1: component1.store.source_y, + source_x2: component2.store.source_x, + source_y2: component2.store.source_y, + copy_width: component0.store.copy_width, + copy_height: component0.store.copy_height, + output_width: component0.store.output_width, + output_height: component0.store.output_height, + output_x: component0.store.output_x, + output_y: component0.store.output_y, + addend0: addends[0], + addend1: addends[1], + addend2: addends[2], + bit_depth0: u32::from(color.bit_depths[0]), + bit_depth1: u32::from(color.bit_depths[1]), + bit_depth2: u32::from(color.bit_depths[2]), + rgba: u32::from(fmt == PixelFormat::Rgba16), + }; + if can_fuse_mct_store { + context.j2k_store_rgb16_mct_device( + component0_buffer, + component1_buffer, + component2_buffer, + CudaJ2kStoreRgb16MctJob { + store: store_job, + irreversible97, + }, + ) + } else { + context.j2k_store_rgb16_device( + component0_buffer, + component1_buffer, + component2_buffer, + store_job, + ) + } + } + _ => Err(CudaError::InvalidArgument { + message: CUDA_HTJ2K_OUTPUT_FORMAT_UNSUPPORTED.to_string(), + }), + }, + ) + .map_err(cuda_error)?; + drop(pending_idwt_batch); + let (surface_buffer, store_stats) = store_output.into_parts(); + dispatches = dispatches.saturating_add(store_stats.kernel_dispatches()); + decode_dispatches = decode_dispatches.saturating_add(store_stats.decode_kernel_dispatches()); + color.report.dispatch_count = dispatches; + color.report.store_us = color.report.store_us.saturating_add(store_us); + color.report.detail.store_dispatch_count = color + .report + .detail + .store_dispatch_count + .saturating_add(store_stats.kernel_dispatches()); + color.report.detail.wall_total_us = profile::elapsed_us(wall_started); + profile::finalize_decode_total_us(&mut color.report); + if emit_report { + color.report.emit("decode"); + } + + let surface = Surface { + backend: BackendKind::Cuda, + residency: SurfaceResidency::CudaResidentDecode, + dimensions: color.dimensions, + fmt, + pitch_bytes: color.dimensions.0 as usize * fmt.bytes_per_pixel(), + stats: CudaSurfaceStats { + total: dispatches, + copy: 0, + decode: decode_dispatches, + }, + storage: Storage::Cuda(surface_buffer), + }; + Ok((surface, color.report)) +} + +#[cfg(feature = "cuda-runtime")] +fn append_color_payload_to_shared( + color: &mut CudaHtj2kColorDecodePlans, + shared_payload: &mut Vec, +) -> Result<(), Error> { + let base = u64::try_from(shared_payload.len()).map_err(|_| Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_BATCH_PAYLOAD_TOO_LARGE, + })?; + shared_payload + .try_reserve(color.payload.len()) + .map_err(|_| Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_BATCH_PAYLOAD_TOO_LARGE, + })?; + for component in &mut color.components { + component.rebase_payload_offsets(base)?; + } + shared_payload.append(&mut color.payload); + Ok(()) +} + +#[cfg(feature = "cuda-runtime")] +fn aggregate_decode_reports(reports: &[CudaHtj2kProfileReport]) -> CudaHtj2kProfileReport { + let mut aggregate = CudaHtj2kProfileReport { + residency: SurfaceResidency::CudaResidentDecode, + ..CudaHtj2kProfileReport::default() + }; + for report in reports { + add_decode_report(&mut aggregate, report); + } + aggregate +} + +#[cfg(feature = "cuda-runtime")] +fn add_decode_report(aggregate: &mut CudaHtj2kProfileReport, report: &CudaHtj2kProfileReport) { + aggregate.parse_us = aggregate.parse_us.saturating_add(report.parse_us); + aggregate.plan_us = aggregate.plan_us.saturating_add(report.plan_us); + aggregate.flatten_us = aggregate.flatten_us.saturating_add(report.flatten_us); + aggregate.h2d_us = aggregate.h2d_us.saturating_add(report.h2d_us); + aggregate.ht_cleanup_us = aggregate.ht_cleanup_us.saturating_add(report.ht_cleanup_us); + aggregate.ht_refine_us = aggregate.ht_refine_us.saturating_add(report.ht_refine_us); + aggregate.dequant_us = aggregate.dequant_us.saturating_add(report.dequant_us); + aggregate.idwt_us = aggregate.idwt_us.saturating_add(report.idwt_us); + aggregate.mct_us = aggregate.mct_us.saturating_add(report.mct_us); + aggregate.store_us = aggregate.store_us.saturating_add(report.store_us); + aggregate.block_count = aggregate.block_count.saturating_add(report.block_count); + aggregate.payload_bytes = aggregate.payload_bytes.saturating_add(report.payload_bytes); + aggregate.dispatch_count = aggregate + .dispatch_count + .saturating_add(report.dispatch_count); + aggregate.detail.table_upload_us = aggregate + .detail + .table_upload_us + .saturating_add(report.detail.table_upload_us); + aggregate.detail.payload_upload_us = aggregate + .detail + .payload_upload_us + .saturating_add(report.detail.payload_upload_us); + aggregate.detail.job_upload_us = aggregate + .detail + .job_upload_us + .saturating_add(report.detail.job_upload_us); + aggregate.detail.status_d2h_us = aggregate + .detail + .status_d2h_us + .saturating_add(report.detail.status_d2h_us); + aggregate.detail.output_d2h_us = aggregate + .detail + .output_d2h_us + .saturating_add(report.detail.output_d2h_us); + aggregate.detail.ht_dispatch_count = aggregate + .detail + .ht_dispatch_count + .saturating_add(report.detail.ht_dispatch_count); + aggregate.detail.dequant_dispatch_count = aggregate + .detail + .dequant_dispatch_count + .saturating_add(report.detail.dequant_dispatch_count); + aggregate.detail.idwt_dispatch_count = aggregate + .detail + .idwt_dispatch_count + .saturating_add(report.detail.idwt_dispatch_count); + aggregate.detail.mct_dispatch_count = aggregate + .detail + .mct_dispatch_count + .saturating_add(report.detail.mct_dispatch_count); + aggregate.detail.store_dispatch_count = aggregate + .detail + .store_dispatch_count + .saturating_add(report.detail.store_dispatch_count); +} + +#[cfg(not(feature = "cuda-runtime"))] +fn decode_to_cuda_resident_surface_impl( + _decoder: &mut J2kDecoder<'_>, + _session: &mut CudaSession, + _fmt: PixelFormat, +) -> Result { + Err(Error::CudaUnavailable) +} + +#[cfg(not(feature = "cuda-runtime"))] +fn decode_to_cuda_resident_surface_with_profile_impl( + _decoder: &mut J2kDecoder<'_>, + _session: &mut CudaSession, + _fmt: PixelFormat, +) -> Result<(Surface, CudaHtj2kProfileReport), Error> { + Err(Error::CudaUnavailable) +} + +#[cfg(not(feature = "cuda-runtime"))] +fn decode_region_to_cuda_resident_surface_impl( + _decoder: &mut J2kDecoder<'_>, + _session: &mut CudaSession, + _fmt: PixelFormat, + _roi: Rect, +) -> Result { + Err(Error::CudaUnavailable) +} + +#[cfg(not(feature = "cuda-runtime"))] +fn decode_scaled_to_cuda_resident_surface_impl( + _decoder: &mut J2kDecoder<'_>, + _session: &mut CudaSession, + _fmt: PixelFormat, + _scale: Downscale, +) -> Result { + Err(Error::CudaUnavailable) +} + +#[cfg(not(feature = "cuda-runtime"))] +fn decode_region_scaled_to_cuda_resident_surface_impl( + _decoder: &mut J2kDecoder<'_>, + _session: &mut CudaSession, + _fmt: PixelFormat, + _roi: Rect, + _scale: Downscale, +) -> Result { + Err(Error::CudaUnavailable) +} + +#[cfg(not(feature = "cuda-runtime"))] +fn decode_batch_to_cuda_resident_surface_with_profile_control( + _inputs: &[&[u8]], + _session: &mut CudaSession, + _fmt: PixelFormat, + _collect_stage_timings: bool, +) -> Result<(Vec, CudaHtj2kProfileReport), Error> { + Err(Error::CudaUnavailable) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_cuda_component_plan( + context: &j2k_cuda_runtime::CudaContext, + plan: &CudaHtj2kDecodePlan, + tables: &CudaHtj2kDecodeTableResources, + pool: &CudaBufferPool, + collect_stage_timings: bool, +) -> Result { + let resource_upload_start = profile::profile_now(collect_stage_timings); + let decode_resources = context + .upload_htj2k_decode_resources_with_tables(plan.payload(), tables) + .map_err(cuda_error)?; + let resource_upload_us = profile::elapsed_us(resource_upload_start); + let mut component = decode_cuda_component_plan_with_resources( + context, + plan, + &decode_resources, + pool, + collect_stage_timings, + )?; + component.timings.h2d = component.timings.h2d.saturating_add(resource_upload_us); + component.timings.payload_upload = component + .timings + .payload_upload + .saturating_add(resource_upload_us); + Ok(component) +} + +#[cfg(test)] +fn split_htj2k_subband_decode_dispatches(kernel_dispatches: usize) -> (usize, usize) { + if kernel_dispatches == 0 { + return (0, 0); + } + + let dequant_dispatches = usize::from(kernel_dispatches > 1); + ( + kernel_dispatches.saturating_sub(dequant_dispatches), + dequant_dispatches, + ) +} + +#[cfg(feature = "cuda-runtime")] +fn htj2k_batched_cleanup_dispatches(target_count: usize) -> usize { + usize::from(target_count > 0) +} + +#[cfg(any(feature = "cuda-runtime", test))] +fn htj2k_batched_dequant_dispatches(target_count: usize) -> usize { + usize::from(target_count > 0) +} + +#[cfg(feature = "cuda-runtime")] +fn htj2k_batched_cleanup_dequant_dispatches( + target_count: usize, + fused_cleanup_dequant: bool, +) -> (usize, usize) { + if target_count == 0 { + return (0, 0); + } + if fused_cleanup_dequant { + (1, 0) + } else { + (1, 1) + } +} + +#[cfg(feature = "cuda-runtime")] +fn decode_cuda_component_plan_with_resources( + context: &j2k_cuda_runtime::CudaContext, + plan: &CudaHtj2kDecodePlan, + decode_resources: &CudaHtj2kDecodeResources, + pool: &CudaBufferPool, + collect_stage_timings: bool, +) -> Result { + let mut work = + decode_cuda_component_subbands_with_resources(context, plan, pool, collect_stage_timings)?; + run_component_cleanup_dequant_batches( + context, + decode_resources, + std::slice::from_mut(&mut work), + pool, + collect_stage_timings, + )?; + run_cuda_component_idwt_steps( + context, + plan.idwt_steps(), + &mut work, + pool, + collect_stage_timings, + )?; + finish_cuda_component_decode(work) +} + +#[cfg(feature = "cuda-runtime")] +fn decode_cuda_component_subbands_with_resources( + context: &j2k_cuda_runtime::CudaContext, + plan: &CudaHtj2kDecodePlan, + pool: &CudaBufferPool, + collect_stage_timings: bool, +) -> Result { + let mut bands = Vec::with_capacity(plan.subbands().len() + plan.idwt_steps().len()); + let mut pending_dequant_bands = Vec::with_capacity(plan.subbands().len()); + let dispatches = 0usize; + let decode_dispatches = 0usize; + let mut timings = CudaDecodeStageTimings::default(); + + for subband in plan.subbands() { + let start = subband.code_block_start as usize; + let end = start.checked_add(subband.code_block_count as usize).ok_or( + Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + }, + )?; + let code_blocks = + plan.code_blocks() + .get(start..end) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_PLAN_INVARIANT_FAILED, + })?; + let jobs = code_blocks + .iter() + .map(|block| cuda_code_block_job_from_plan_block(block, subband.width)) + .collect::, Error>>()?; + let output_words = checked_area(subband.width, subband.height)?; + let allocate_start = profile::profile_now(collect_stage_timings); + let output = context + .allocate_htj2k_codeblock_coefficients_with_pool(&jobs, output_words, pool) + .map_err(cuda_error)?; + let allocate_wall_us = profile::elapsed_us(allocate_start); + timings.h2d = timings.h2d.saturating_add(allocate_wall_us); + let (buffer, _, _) = output.into_parts(); + let band_index = bands.len(); + bands.push(CudaCoefficientBand { + band_id: subband.band_id, + buffer, + }); + if !jobs.is_empty() { + pending_dequant_bands.push(CudaPendingDequantBand { + band_index, + jobs, + output_words, + }); + } + } + + let [store] = plan.store_steps() else { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_STORE_UNSUPPORTED, + }); + }; + + Ok(CudaComponentDecodeWork { + bands, + pending_dequant_bands, + store: *store, + dispatches, + decode_dispatches, + timings, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn run_component_cleanup_dequant_batches( + context: &j2k_cuda_runtime::CudaContext, + decode_resources: &CudaHtj2kDecodeResources, + component_work: &mut [CudaComponentDecodeWork], + pool: &CudaBufferPool, + collect_stage_timings: bool, +) -> Result<(), Error> { + let pending_count = component_work + .iter() + .map(|work| work.pending_dequant_bands.len()) + .sum::(); + if pending_count == 0 { + return Ok(()); + } + let accounting_index = component_work + .iter() + .position(|work| !work.pending_dequant_bands.is_empty()) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + })?; + + let has_refinement = component_work.iter().any(|work| { + work.pending_dequant_bands.iter().any(|pending| { + pending + .jobs + .iter() + .any(|job| job.refinement_length > 0 || u32::from(job.number_of_coding_passes) > 1) + }) + }); + let cleanup_targets = component_work + .iter() + .flat_map(|work| { + work.pending_dequant_bands + .iter() + .map(move |pending| (work, pending)) + }) + .map(|(work, pending)| { + let coefficients = pooled_cuda_buffer(&work.bands[pending.band_index].buffer)?; + Ok(CudaHtj2kCleanupTarget { + coefficients, + jobs: &pending.jobs, + output_words: pending.output_words, + }) + }) + .collect::, Error>>()?; + if !has_refinement { + let stage_start = profile::profile_now(collect_stage_timings); + let ((stats, runtime_timings), fused_us) = context + .time_default_stream_named_us_if( + collect_stage_timings, + "j2k.htj2k.decode.cleanup_dequantize.batch", + || { + context + .decode_htj2k_codeblocks_cleanup_dequantize_multi_with_resources_and_pool_timed( + decode_resources, + &cleanup_targets, + pool, + collect_stage_timings, + ) + }, + ) + .map_err(cuda_error)?; + let stage_wall_us = profile::elapsed_us(stage_start); + let (cleanup_dispatches, dequant_dispatches) = + htj2k_batched_cleanup_dequant_dispatches(pending_count, true); + { + let accounting = &mut component_work[accounting_index]; + accounting.timings.h2d = accounting + .timings + .h2d + .saturating_add(stage_wall_us.saturating_sub(fused_us)); + accounting.timings.ht_cleanup = accounting.timings.ht_cleanup.saturating_add(fused_us); + accounting.timings.status_d2h = accounting + .timings + .status_d2h + .saturating_add(runtime_timings.status_d2h_us); + accounting.timings.ht_dispatch_count = accounting + .timings + .ht_dispatch_count + .saturating_add(cleanup_dispatches); + accounting.timings.dequant_dispatch_count = accounting + .timings + .dequant_dispatch_count + .saturating_add(dequant_dispatches); + accounting.dispatches = accounting + .dispatches + .saturating_add(stats.kernel_dispatches()); + accounting.decode_dispatches = accounting + .decode_dispatches + .saturating_add(stats.decode_kernel_dispatches()); + } + + for work in component_work { + work.pending_dequant_bands.clear(); + } + return Ok(()); + } + let mut queued_cleanup: Option = None; + let stage_start = profile::profile_now(collect_stage_timings); + let (stats, cleanup_us, status_d2h_us) = if collect_stage_timings { + let ((stats, runtime_timings), cleanup_us) = context + .time_default_stream_named_us_if( + collect_stage_timings, + "j2k.htj2k.decode.cleanup.batch", + || { + context.decode_htj2k_codeblocks_cleanup_multi_with_resources_and_pool_timed( + decode_resources, + &cleanup_targets, + pool, + collect_stage_timings, + ) + }, + ) + .map_err(cuda_error)?; + (stats, cleanup_us, runtime_timings.status_d2h_us) + } else { + let (queued, cleanup_us) = context + .time_default_stream_named_us_if(false, "j2k.htj2k.decode.cleanup.batch", || { + context.decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool( + decode_resources, + &cleanup_targets, + pool, + ) + }) + .map_err(cuda_error)?; + let stats = queued.execution(); + queued_cleanup = Some(queued); + (stats, cleanup_us, 0) + }; + drop(cleanup_targets); + let stage_wall_us = profile::elapsed_us(stage_start); + { + let accounting = &mut component_work[accounting_index]; + accounting.timings.h2d = accounting + .timings + .h2d + .saturating_add(stage_wall_us.saturating_sub(cleanup_us)); + accounting.timings.ht_cleanup = accounting.timings.ht_cleanup.saturating_add(cleanup_us); + accounting.timings.status_d2h = accounting.timings.status_d2h.saturating_add(status_d2h_us); + if has_refinement { + accounting.timings.ht_refine = accounting.timings.ht_refine.saturating_add(cleanup_us); + } + accounting.timings.ht_dispatch_count = accounting + .timings + .ht_dispatch_count + .saturating_add(htj2k_batched_cleanup_dispatches(pending_count)); + accounting.dispatches = accounting + .dispatches + .saturating_add(stats.kernel_dispatches()); + accounting.decode_dispatches = accounting + .decode_dispatches + .saturating_add(stats.decode_kernel_dispatches()); + } + + let stage_start = profile::profile_now(collect_stage_timings); + let (stats, dequant_us, dequant_target_count) = { + let dequant_target_count = pending_count; + let dequant_result = if let Some(queued) = queued_cleanup.as_ref() { + context.time_default_stream_named_us_if( + collect_stage_timings, + "j2k.htj2k.decode.dequantize.batch", + || context.j2k_dequantize_queued_htj2k_cleanup_with_pool(queued), + ) + } else { + let dequant_targets = component_work + .iter() + .flat_map(|work| { + work.pending_dequant_bands + .iter() + .map(move |pending| (work, pending)) + }) + .map(|(work, pending)| { + let coefficients = pooled_cuda_buffer(&work.bands[pending.band_index].buffer)?; + Ok(CudaHtj2kDequantizeTarget { + coefficients, + jobs: &pending.jobs, + output_words: pending.output_words, + }) + }) + .collect::, Error>>()?; + context.time_default_stream_named_us_if( + collect_stage_timings, + "j2k.htj2k.decode.dequantize.batch", + || { + context.j2k_dequantize_htj2k_codeblocks_multi_device_with_pool( + &dequant_targets, + pool, + ) + }, + ) + }; + let (stats, dequant_us) = match dequant_result { + Ok(result) => result, + Err(error) => { + if let Some(queued) = queued_cleanup.take() { + queued.finish().map_err(cuda_error)?; + } + return Err(cuda_error(error)); + } + }; + (stats, dequant_us, dequant_target_count) + }; + let stage_wall_us = profile::elapsed_us(stage_start); + { + let accounting = &mut component_work[accounting_index]; + accounting.timings.h2d = accounting + .timings + .h2d + .saturating_add(stage_wall_us.saturating_sub(dequant_us)); + accounting.timings.dequant = accounting.timings.dequant.saturating_add(dequant_us); + accounting.timings.dequant_dispatch_count = accounting + .timings + .dequant_dispatch_count + .saturating_add(htj2k_batched_dequant_dispatches(dequant_target_count)); + accounting.dispatches = accounting + .dispatches + .saturating_add(stats.kernel_dispatches()); + accounting.decode_dispatches = accounting + .decode_dispatches + .saturating_add(stats.decode_kernel_dispatches()); + } + if let Some(queued) = queued_cleanup.take() { + queued.finish().map_err(cuda_error)?; + } + + for work in component_work { + work.pending_dequant_bands.clear(); + } + Ok(()) +} + +#[cfg(feature = "cuda-runtime")] +fn run_cuda_component_idwt_steps( + context: &j2k_cuda_runtime::CudaContext, + steps: &[CudaHtj2kIdwtStep], + work: &mut CudaComponentDecodeWork, + pool: &CudaBufferPool, + collect_stage_timings: bool, +) -> Result<(), Error> { + for step in steps { + let ll = find_cuda_band(&work.bands, step.ll_band_id)?; + let hl = find_cuda_band(&work.bands, step.hl_band_id)?; + let lh = find_cuda_band(&work.bands, step.lh_band_id)?; + let hh = find_cuda_band(&work.bands, step.hh_band_id)?; + let low_low_device = pooled_cuda_buffer(&ll.buffer)?; + let high_low_device = pooled_cuda_buffer(&hl.buffer)?; + let low_high_device = pooled_cuda_buffer(&lh.buffer)?; + let high_high_device = pooled_cuda_buffer(&hh.buffer)?; + let job = cuda_idwt_job_from_step(step); + let (output, idwt_us) = context + .time_default_stream_named_us_if(collect_stage_timings, "j2k.htj2k.decode.idwt", || { + if collect_stage_timings { + return context.j2k_inverse_dwt_single_device_with_pool( + low_low_device, + high_low_device, + low_high_device, + high_high_device, + job, + pool, + ); + } + context.j2k_inverse_dwt_single_device_untimed_with_pool( + low_low_device, + high_low_device, + low_high_device, + high_high_device, + job, + pool, + ) + }) + .map_err(cuda_error)?; + work.timings.idwt = work.timings.idwt.saturating_add(idwt_us); + let (buffer, stats) = output.into_parts(); + work.dispatches = work.dispatches.saturating_add(stats.kernel_dispatches()); + work.decode_dispatches = work + .decode_dispatches + .saturating_add(stats.decode_kernel_dispatches()); + work.timings.idwt_dispatch_count = work + .timings + .idwt_dispatch_count + .saturating_add(stats.kernel_dispatches()); + work.bands.push(CudaCoefficientBand { + band_id: step.output_band_id, + buffer, + }); + } + Ok(()) +} + +#[cfg(feature = "cuda-runtime")] +fn finish_cuda_component_decode( + mut work: CudaComponentDecodeWork, +) -> Result { + let input_index = work + .bands + .iter() + .position(|band| band.band_id == work.store.input_band_id) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + })?; + let input = work.bands.swap_remove(input_index); + Ok(CudaDecodedComponent { + buffer: input.buffer, + store: work.store, + dispatches: work.dispatches, + decode_dispatches: work.decode_dispatches, + timings: work.timings, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn can_batch_color_idwt(components: &[&CudaHtj2kDecodePlan]) -> bool { + let Some(first) = components.first() else { + return false; + }; + components + .iter() + .all(|component| component.idwt_steps().len() == first.idwt_steps().len()) +} + +#[cfg(feature = "cuda-runtime")] +fn run_color_component_idwt_batches( + context: &j2k_cuda_runtime::CudaContext, + components: &[&CudaHtj2kDecodePlan], + component_work: &mut [CudaComponentDecodeWork], + pool: &CudaBufferPool, + collect_stage_timings: bool, +) -> Result, Error> { + let (queued_batch, idwt_us) = context + .time_default_stream_named_us_if( + collect_stage_timings, + "j2k.htj2k.decode.idwt.batch", + || enqueue_color_component_idwt_batches(context, components, component_work, pool), + ) + .map_err(cuda_error)?; + + if let Some(accounting) = component_work.first_mut() { + accounting.timings.idwt = accounting.timings.idwt.saturating_add(idwt_us); + accounting.dispatches = accounting + .dispatches + .saturating_add(queued_batch.kernel_dispatches); + accounting.decode_dispatches = accounting + .decode_dispatches + .saturating_add(queued_batch.decode_dispatches); + accounting.timings.idwt_dispatch_count = accounting + .timings + .idwt_dispatch_count + .saturating_add(queued_batch.kernel_dispatches); + } + let _queued_resource_count = queued_batch + .queued + .iter() + .map(CudaQueuedExecution::resource_count) + .sum::(); + if collect_stage_timings { + drop(queued_batch); + Ok(None) + } else { + Ok(Some(queued_batch)) + } +} + +#[cfg(feature = "cuda-runtime")] +fn enqueue_color_component_idwt_batches( + context: &j2k_cuda_runtime::CudaContext, + components: &[&CudaHtj2kDecodePlan], + component_work: &mut [CudaComponentDecodeWork], + pool: &CudaBufferPool, +) -> Result { + if components.len() != component_work.len() { + return Err(CudaError::InvalidArgument { + message: CUDA_HTJ2K_KERNELS_NOT_READY.to_string(), + }); + } + let Some(first) = components.first() else { + return Ok(CudaQueuedIdwtBatch { + queued: Vec::new(), + kernel_dispatches: 0, + decode_dispatches: 0, + }); + }; + + let mut queued = Vec::with_capacity(first.idwt_steps().len()); + let mut kernel_dispatches = 0usize; + let mut decode_dispatches = 0usize; + let step_count = first.idwt_steps().len(); + let trace_enabled = cuda_idwt_trace_enabled(); + let enqueue_result = (|| -> Result<(), CudaError> { + let mut output_pool_trace = CudaIdwtOutputPoolTraceTotals::default(); + let output_alloc_start = trace_enabled.then(std::time::Instant::now); + for step_index in 0..step_count { + for (component_index, component) in components.iter().enumerate() { + let step = component.idwt_steps().get(step_index).ok_or_else(|| { + CudaError::InvalidArgument { + message: CUDA_HTJ2K_KERNELS_NOT_READY.to_string(), + } + })?; + let width = step.rect.x1.saturating_sub(step.rect.x0); + let height = step.rect.y1.saturating_sub(step.rect.y0); + let output_words = checked_area(width, height).map_err(cuda_invalid_decode_plan)?; + let output_bytes = output_words + .checked_mul(std::mem::size_of::()) + .ok_or_else(|| CudaError::InvalidArgument { + message: CUDA_HTJ2K_KERNELS_NOT_READY.to_string(), + })?; + let buffer = if trace_enabled { + let (buffer, trace) = pool.take_with_trace(output_bytes)?; + output_pool_trace.add_take(trace); + buffer + } else { + pool.take(output_bytes)? + }; + component_work[component_index] + .bands + .push(CudaCoefficientBand { + band_id: step.output_band_id, + buffer, + }); + } + } + let output_alloc_us = elapsed_host_us(output_alloc_start); + + let target_build_start = trace_enabled.then(std::time::Instant::now); + let mut target_batches = Vec::with_capacity(step_count); + for step_index in 0..step_count { + let targets = components + .iter() + .enumerate() + .map(|(component_index, component)| { + let step = component.idwt_steps().get(step_index).ok_or_else(|| { + CudaError::InvalidArgument { + message: CUDA_HTJ2K_KERNELS_NOT_READY.to_string(), + } + })?; + let work = &component_work[component_index]; + let ll = find_cuda_band(&work.bands, step.ll_band_id) + .map_err(cuda_invalid_decode_plan)?; + let hl = find_cuda_band(&work.bands, step.hl_band_id) + .map_err(cuda_invalid_decode_plan)?; + let lh = find_cuda_band(&work.bands, step.lh_band_id) + .map_err(cuda_invalid_decode_plan)?; + let hh = find_cuda_band(&work.bands, step.hh_band_id) + .map_err(cuda_invalid_decode_plan)?; + let output = find_cuda_band(&work.bands, step.output_band_id) + .map_err(cuda_invalid_decode_plan)?; + Ok(CudaJ2kIdwtTarget { + ll: pooled_cuda_buffer(&ll.buffer).map_err(cuda_invalid_decode_plan)?, + hl: pooled_cuda_buffer(&hl.buffer).map_err(cuda_invalid_decode_plan)?, + lh: pooled_cuda_buffer(&lh.buffer).map_err(cuda_invalid_decode_plan)?, + hh: pooled_cuda_buffer(&hh.buffer).map_err(cuda_invalid_decode_plan)?, + output: pooled_cuda_buffer(&output.buffer) + .map_err(cuda_invalid_decode_plan)?, + job: cuda_idwt_job_from_step(step), + }) + }) + .collect::, CudaError>>()?; + target_batches.push(targets); + } + let target_build_us = elapsed_host_us(target_build_start); + let target_slices = target_batches.iter().map(Vec::as_slice).collect::>(); + let enqueue_start = trace_enabled.then(std::time::Instant::now); + let queued_execution = + context.j2k_inverse_dwt_batch_sequence_enqueue_with_pool(&target_slices, pool)?; + let enqueue_us = elapsed_host_us(enqueue_start); + let execution = queued_execution.execution(); + kernel_dispatches = kernel_dispatches.saturating_add(execution.kernel_dispatches()); + decode_dispatches = decode_dispatches.saturating_add(execution.decode_kernel_dispatches()); + queued.push(queued_execution); + if trace_enabled { + let row = CudaIdwtBatchHostTraceRow { + component_count: components.len(), + step_count, + output_alloc_us, + target_build_us, + enqueue_us, + output_take_count: output_pool_trace.take_count, + output_pool_reuse_count: output_pool_trace.reuse_count, + output_pool_alloc_count: output_pool_trace.alloc_count, + output_pool_scanned_count: output_pool_trace.scanned_count, + output_pool_max_free_count: output_pool_trace.max_free_count, + output_requested_bytes: output_pool_trace.requested_bytes, + }; + eprintln!("{}", format_cuda_idwt_batch_host_trace_row(row)); + } + Ok(()) + })(); + if let Err(error) = enqueue_result { + if !queued.is_empty() { + let _ = context.synchronize(); + } + return Err(error); + } + + Ok(CudaQueuedIdwtBatch { + queued, + kernel_dispatches, + decode_dispatches, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_code_block_job_from_plan_block( + block: &crate::CudaHtj2kCodeBlock, + subband_width: u32, +) -> Result { + let output_offset = block + .output_y + .checked_mul(subband_width) + .and_then(|base| base.checked_add(block.output_x)) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + })?; + Ok(CudaHtj2kCodeBlockJob { + payload_offset: block.payload_offset, + width: block.width, + height: block.height, + payload_len: block.payload_len, + cleanup_length: block.cleanup_length, + refinement_length: block.refinement_length, + missing_bit_planes: block.missing_bit_planes, + num_bitplanes: block.num_bitplanes, + number_of_coding_passes: block.number_of_coding_passes, + output_stride: block.output_stride, + output_offset, + dequantization_step: block.dequantization_step, + stripe_causal: block.stripe_causal != 0, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn validate_color_stores( + stores: [&CudaHtj2kStoreStep; 3], + dimensions: (u32, u32), +) -> Result<(), Error> { + let first = stores[0]; + for store in stores { + let input_width = store.input_rect.x1.saturating_sub(store.input_rect.x0); + let input_height = store.input_rect.y1.saturating_sub(store.input_rect.y0); + let source_end_x = + store + .source_x + .checked_add(store.copy_width) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + })?; + let source_end_y = + store + .source_y + .checked_add(store.copy_height) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + })?; + if store.output_x != 0 + || store.output_y != 0 + || store.copy_width != dimensions.0 + || store.copy_height != dimensions.1 + || store.output_width != dimensions.0 + || store.output_height != dimensions.1 + || source_end_x > input_width + || source_end_y > input_height + || store.source_x != first.source_x + || store.source_y != first.source_y + { + return Err(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }); + } + } + Ok(()) +} + +#[cfg(feature = "cuda-runtime")] +fn bit_depth_addend(bit_depth: u8) -> f32 { + let shift = bit_depth.saturating_sub(1).min(15); + f32::from(1_u16 << shift) +} + +#[cfg(feature = "cuda-runtime")] +fn checked_area(width: u32, height: u32) -> Result { + width + .try_into() + .ok() + .and_then(|width: usize| width.checked_mul(height as usize)) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn find_cuda_band( + bands: &[CudaCoefficientBand], + band_id: CudaHtj2kBandId, +) -> Result<&CudaCoefficientBand, Error> { + bands + .iter() + .find(|band| band.band_id == band_id) + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn pooled_cuda_buffer(buffer: &CudaPooledDeviceBuffer) -> Result<&CudaDeviceBuffer, Error> { + buffer + .as_device_buffer() + .ok_or(Error::UnsupportedCudaRequest { + reason: CUDA_HTJ2K_KERNELS_NOT_READY, + }) +} + +#[cfg(feature = "cuda-runtime")] +#[allow(clippy::needless_pass_by_value)] +fn cuda_invalid_decode_plan(error: Error) -> CudaError { + CudaError::InvalidArgument { + message: error.to_string(), + } +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_runtime_rect(rect: crate::CudaHtj2kRect) -> CudaJ2kRect { + CudaJ2kRect { + x0: rect.x0, + y0: rect.y0, + x1: rect.x1, + y1: rect.y1, + } +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_idwt_job_from_step(step: &CudaHtj2kIdwtStep) -> CudaJ2kIdwtJob { + CudaJ2kIdwtJob { + rect: cuda_runtime_rect(step.rect), + ll_rect: cuda_runtime_rect(step.ll_rect), + hl_rect: cuda_runtime_rect(step.hl_rect), + lh_rect: cuda_runtime_rect(step.lh_rect), + hh_rect: cuda_runtime_rect(step.hh_rect), + irreversible97: u32::from(step.transform == CudaHtj2kTransform::Irreversible97), + } +} + +#[cfg(all(test, feature = "cuda-runtime"))] +mod tests { + use super::{ + build_cuda_htj2k_color_plans_from_bytes_with_profile, can_batch_color_idwt, + cuda_code_block_job_from_plan_block, htj2k_batched_cleanup_dequant_dispatches, + htj2k_batched_cleanup_dispatches, htj2k_batched_dequant_dispatches, CudaDecodeStageTimings, + }; + use j2k_core::PixelFormat; + use j2k_native::{encode_htj2k, DecoderContext as NativeDecoderContext, EncodeOptions}; + + use crate::CudaHtj2kCodeBlock; + + #[test] + fn cuda_runtime_code_block_job_preserves_plan_output_stride() { + let block = CudaHtj2kCodeBlock { + subband_index: 0, + payload_offset: 13, + payload_len: 5, + cleanup_length: 5, + refinement_length: 0, + output_x: 3, + output_y: 2, + width: 4, + height: 5, + output_stride: 99, + missing_bit_planes: 1, + number_of_coding_passes: 1, + num_bitplanes: 8, + stripe_causal: 0, + dequantization_step: 1.0, + }; + + let job = cuda_code_block_job_from_plan_block(&block, 64) + .expect("valid CUDA code-block runtime job"); + + assert_eq!(job.output_offset, 131); + assert_eq!(job.output_stride, 99); + } + + #[test] + fn batched_cleanup_and_dequant_dispatch_helpers_count_one_shared_dispatch() { + assert_eq!(htj2k_batched_cleanup_dispatches(0), 0); + assert_eq!(htj2k_batched_cleanup_dispatches(1), 1); + assert_eq!(htj2k_batched_cleanup_dispatches(3), 1); + assert_eq!(htj2k_batched_dequant_dispatches(0), 0); + assert_eq!(htj2k_batched_dequant_dispatches(1), 1); + assert_eq!(htj2k_batched_dequant_dispatches(3), 1); + assert_eq!(htj2k_batched_cleanup_dequant_dispatches(0, true), (0, 0)); + assert_eq!(htj2k_batched_cleanup_dequant_dispatches(1, true), (1, 0)); + assert_eq!(htj2k_batched_cleanup_dequant_dispatches(3, true), (1, 0)); + assert_eq!(htj2k_batched_cleanup_dequant_dispatches(1, false), (1, 1)); + assert_eq!(htj2k_batched_cleanup_dequant_dispatches(3, false), (1, 1)); + } + + #[test] + fn profiled_cuda_batch_decode_api_accepts_empty_batch() { + let mut session = crate::CudaSession::default(); + let inputs: [&[u8]; 0] = []; + + let (surfaces, report) = + crate::J2kDecoder::decode_batch_to_device_with_session_and_profile( + &inputs, + PixelFormat::Rgb8, + &mut session, + ) + .expect("empty CUDA batch decode"); + + assert!(surfaces.is_empty()); + assert_eq!(report.block_count, 0); + assert_eq!(report.payload_bytes, 0); + } + + #[test] + fn cuda_batch_decode_two_color_images_matches_single_when_runtime_required() { + let pixels_a: Vec = (0u16..16 * 16 * 3) + .map(|idx| u8::try_from((idx * 7 + idx / 5) & 0xff).expect("masked byte")) + .collect(); + let pixels_b: Vec = (0u16..16 * 16 * 3) + .map(|idx| u8::try_from((idx * 11 + 23) & 0xff).expect("masked byte")) + .collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let codestream_a = + encode_htj2k(&pixels_a, 16, 16, 3, 8, false, &options).expect("encode fixture A"); + let codestream_b = + encode_htj2k(&pixels_b, 16, 16, 3, 8, false, &options).expect("encode fixture B"); + let inputs = [codestream_a.as_slice(), codestream_b.as_slice()]; + let mut batch_session = crate::CudaSession::default(); + + let batch = crate::J2kDecoder::decode_batch_to_device_with_session_and_profile( + &inputs, + PixelFormat::Rgb8, + &mut batch_session, + ); + let (surfaces, report) = match batch { + Ok(result) => result, + Err(crate::Error::CudaUnavailable | crate::Error::CudaRuntime { .. }) + if !cuda_runtime_required() => + { + return; + } + Err(error) => panic!("batch CUDA decode failed: {error}"), + }; + + assert_eq!(surfaces.len(), 2); + assert_eq!(report.detail.ht_dispatch_count, 1); + assert_eq!(report.detail.dequant_dispatch_count, 0); + assert_eq!(report.detail.store_dispatch_count, 1); + let batch_pixels_tight = + crate::Surface::download_batch_tight(&surfaces).expect("download tight CUDA batch"); + assert_eq!(batch_pixels_tight.len(), surfaces.len() * 16 * 16 * 3); + for (index, codestream) in inputs.iter().enumerate() { + let mut single_session = crate::CudaSession::default(); + let mut decoder = crate::J2kDecoder::new(codestream).expect("single decoder"); + let single = decoder + .decode_to_device_with_session(PixelFormat::Rgb8, &mut single_session) + .expect("single CUDA decode"); + let mut single_pixels = vec![0u8; 16 * 16 * 3]; + let mut batch_pixels = vec![0u8; 16 * 16 * 3]; + single + .download_into(&mut single_pixels, 16 * 3) + .expect("download single decode"); + surfaces[index] + .download_into(&mut batch_pixels, 16 * 3) + .expect("download batch decode"); + assert_eq!(batch_pixels, single_pixels); + assert_eq!( + &batch_pixels_tight[index * 16 * 16 * 3..(index + 1) * 16 * 16 * 3], + single_pixels.as_slice() + ); + } + } + + #[test] + fn cuda_batch_decode_mixed_idwt_shapes_avoids_fused_batch_store_without_idwt_batch() { + let codestream_a = rgb8_htj2k_fixture(32, 32, 1, 7); + let codestream_b = rgb8_htj2k_fixture(32, 32, 2, 19); + let inputs = [codestream_a.as_slice(), codestream_b.as_slice()]; + let mut batch_session = crate::CudaSession::default(); + + let result = crate::J2kDecoder::decode_batch_to_device_with_session( + &inputs, + PixelFormat::Rgb8, + &mut batch_session, + ); + let surfaces = match result { + Ok(surfaces) => surfaces, + Err(crate::Error::CudaUnavailable | crate::Error::CudaRuntime { .. }) + if !cuda_runtime_required() => + { + return; + } + Err(crate::Error::UnsupportedCudaRequest { .. }) => return, + Err(error) => panic!("mixed-shape batch CUDA decode failed: {error}"), + }; + + assert_eq!(surfaces.len(), inputs.len()); + for (index, codestream) in inputs.iter().enumerate() { + let mut single_session = crate::CudaSession::default(); + let mut decoder = crate::J2kDecoder::new(codestream).expect("single decoder"); + let single = decoder + .decode_to_device_with_session(PixelFormat::Rgb8, &mut single_session) + .expect("single CUDA decode"); + let mut single_pixels = vec![0u8; 32 * 32 * 3]; + let mut batch_pixels = vec![0u8; 32 * 32 * 3]; + single + .download_into(&mut single_pixels, 32 * 3) + .expect("download single decode"); + surfaces[index] + .download_into(&mut batch_pixels, 32 * 3) + .expect("download mixed-shape batch decode"); + assert_eq!(batch_pixels, single_pixels); + } + } + + #[test] + fn decode_stage_timings_report_status_download_detail() { + let mut report = crate::CudaHtj2kProfileReport::default(); + let timings = CudaDecodeStageTimings { + h2d: 17, + status_d2h: 5, + ..CudaDecodeStageTimings::default() + }; + + timings.add_to_report(&mut report); + + assert_eq!(report.h2d_us, 17); + assert_eq!(report.detail.status_d2h_us, 5); + } + + fn cuda_runtime_required() -> bool { + std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_some() + } + + fn rgb8_htj2k_fixture(width: u32, height: u32, levels: u8, seed: u16) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for idx in 0..width * height { + let seed = u32::from(seed); + pixels.push(u8::try_from((idx * seed + idx / 3) & 0xff).expect("red")); + pixels.push(u8::try_from((idx * (seed + 11) + 7) & 0xff).expect("green")); + pixels.push(u8::try_from((idx * (seed + 23) + 19) & 0xff).expect("blue")); + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: levels, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, width, height, 3, 8, false, &options) + .expect("encode RGB HTJ2K fixture") + } + + #[test] + fn color_plan_flattens_one_shared_payload_for_component_decode() { + let pixels: Vec = (0u16..4 * 4 * 3) + .map(|idx| u8::try_from((idx * 13 + idx / 3) & 0xff).expect("masked byte")) + .collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let codestream = + encode_htj2k(&pixels, 4, 4, 3, 8, false, &options).expect("encode HTJ2K RGB fixture"); + let mut decoder = crate::J2kDecoder::new(&codestream).expect("decoder"); + + let color = decoder + .build_cuda_htj2k_color_plans_with_profile(PixelFormat::Rgb8) + .expect("CUDA color plans"); + + assert_eq!(color.components.len(), 3); + assert!(!color.payload.is_empty()); + assert_eq!(color.report.payload_bytes, color.payload.len()); + for component in &color.components { + assert!(component.payload().is_empty()); + for block in component.code_blocks() { + let start = usize::try_from(block.payload_offset).expect("payload offset"); + let end = start + block.payload_len as usize; + assert!(end <= color.payload.len()); + } + } + } + + #[test] + fn byte_color_plan_builder_matches_decoder_color_plan() { + let pixels: Vec = (0u16..8 * 8 * 3) + .map(|idx| u8::try_from((idx * 19 + idx / 5) & 0xff).expect("masked byte")) + .collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let codestream = + encode_htj2k(&pixels, 8, 8, 3, 8, false, &options).expect("encode HTJ2K RGB fixture"); + let mut decoder = crate::J2kDecoder::new(&codestream).expect("decoder"); + let decoder_plan = decoder + .build_cuda_htj2k_color_plans_with_profile(PixelFormat::Rgb8) + .expect("decoder CUDA color plans"); + let mut native_context = NativeDecoderContext::default(); + let byte_plan = build_cuda_htj2k_color_plans_from_bytes_with_profile( + &codestream, + PixelFormat::Rgb8, + &mut native_context, + ) + .expect("byte CUDA color plans"); + + assert_eq!(byte_plan.dimensions, decoder_plan.dimensions); + assert_eq!(byte_plan.mct_dimensions, decoder_plan.mct_dimensions); + assert_eq!(byte_plan.bit_depths, decoder_plan.bit_depths); + assert_eq!(byte_plan.mct, decoder_plan.mct); + assert_eq!(byte_plan.components.len(), decoder_plan.components.len()); + assert_eq!(byte_plan.payload.len(), decoder_plan.payload.len()); + assert_eq!( + byte_plan + .components + .iter() + .map(|component| component.code_blocks().len()) + .collect::>(), + decoder_plan + .components + .iter() + .map(|component| component.code_blocks().len()) + .collect::>() + ); + } + + #[test] + fn multi_image_color_components_can_share_one_idwt_batch() { + let pixels: Vec = (0u16..16 * 16 * 3) + .map(|idx| u8::try_from((idx * 17 + idx / 7) & 0xff).expect("masked byte")) + .collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let codestream = + encode_htj2k(&pixels, 16, 16, 3, 8, false, &options).expect("encode HTJ2K RGB fixture"); + let mut first = crate::J2kDecoder::new(&codestream).expect("first decoder"); + let mut second = crate::J2kDecoder::new(&codestream).expect("second decoder"); + let first = first + .build_cuda_htj2k_color_plans_with_profile(PixelFormat::Rgb8) + .expect("first CUDA color plans"); + let second = second + .build_cuda_htj2k_color_plans_with_profile(PixelFormat::Rgb8) + .expect("second CUDA color plans"); + let components = first + .components + .iter() + .chain(second.components.iter()) + .collect::>(); + + assert_eq!(components.len(), 6); + assert!(can_batch_color_idwt(&components)); + } + + #[test] + fn batched_color_idwt_defers_completion_to_store_sync() { + let source = include_str!("decoder.rs"); + + assert!( + !source.contains( + "if !collect_stage_timings {\n context.synchronize().map_err(cuda_error)?;\n }" + ), + "batched color IDWT should keep queued resources live and let the following store synchronize" + ); + } +} + +impl ImageCodec for J2kDecoder<'_> { + type Error = Error; + type Warning = Infallible; + type Pool = CpuJ2kScratchPool; +} + +impl<'a> ImageDecode<'a> for J2kDecoder<'a> { + type View = J2kView<'a>; + + fn inspect(input: &'a [u8]) -> Result { + Ok(CpuDecoder::inspect(input)?) + } + + fn parse(input: &'a [u8]) -> Result { + Ok(J2kView::parse(input)?) + } + + fn from_view(view: Self::View) -> Result { + Ok(Self { + inner: CpuDecoder::from_view(view)?, + pool: CpuJ2kScratchPool::new(), + }) + } + + fn decode_into( + &mut self, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + ) -> Result, Self::Error> { + Ok(self.inner.decode_into(out, stride, fmt)?) + } + + fn decode_into_with_scratch( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + ) -> Result, Self::Error> { + Ok(self + .inner + .decode_into_with_scratch(pool, out, stride, fmt)?) + } + + fn decode_region_into( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, + ) -> Result, Self::Error> { + Ok(self.inner.decode_region_into(pool, out, stride, fmt, roi)?) + } + + fn decode_scaled_into( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + scale: Downscale, + ) -> Result, Self::Error> { + Ok(self + .inner + .decode_scaled_into(pool, out, stride, fmt, scale)?) + } + + fn decode_region_scaled_into( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + ) -> Result, Self::Error> { + Ok(self + .inner + .decode_region_scaled_into(pool, out, stride, fmt, roi, scale)?) + } +} + +impl<'a> ImageDecodeDevice<'a> for J2kDecoder<'a> { + type DeviceSurface = Surface; +} + +impl<'a> ImageDecodeSubmit<'a> for J2kDecoder<'a> { + type Session = CudaSession; + type DeviceSurface = Surface; + type SubmittedSurface = ReadySubmission; + + fn submit_to_device( + &mut self, + session: &mut Self::Session, + fmt: PixelFormat, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + Ok(submit_ready_device(session, |session| { + self.decode_to_surface_impl(session, fmt, backend) + })) + } + + fn submit_region_to_device( + &mut self, + session: &mut Self::Session, + fmt: PixelFormat, + roi: Rect, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + Ok(submit_ready_device(session, |session| { + self.decode_region_to_surface_impl(session, fmt, roi, backend) + })) + } + + fn submit_scaled_to_device( + &mut self, + session: &mut Self::Session, + fmt: PixelFormat, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + Ok(submit_ready_device(session, |session| { + self.decode_scaled_to_surface_impl(session, fmt, scale, backend) + })) + } + + fn submit_region_scaled_to_device( + &mut self, + session: &mut Self::Session, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + Ok(submit_ready_device(session, |session| { + self.decode_region_scaled_to_surface_impl(session, fmt, roi, scale, backend) + })) + } +} + +#[cfg(test)] +mod dispatch_tests { + use super::{ + format_cuda_idwt_batch_host_trace_row, htj2k_batched_dequant_dispatches, + split_htj2k_subband_decode_dispatches, CudaIdwtBatchHostTraceRow, + }; + + #[test] + fn htj2k_decode_dispatch_split_separates_ht_and_dequant_counts() { + assert_eq!(split_htj2k_subband_decode_dispatches(0), (0, 0)); + assert_eq!(split_htj2k_subband_decode_dispatches(1), (1, 0)); + assert_eq!(split_htj2k_subband_decode_dispatches(2), (1, 1)); + assert_eq!(split_htj2k_subband_decode_dispatches(3), (2, 1)); + } + + #[test] + fn htj2k_batched_dequant_dispatch_count_is_one_for_any_non_empty_batch() { + assert_eq!(htj2k_batched_dequant_dispatches(0), 0); + assert_eq!(htj2k_batched_dequant_dispatches(1), 1); + assert_eq!(htj2k_batched_dequant_dispatches(48), 1); + } + + #[test] + fn cuda_idwt_batch_host_trace_row_reports_host_split() { + let row = CudaIdwtBatchHostTraceRow { + component_count: 327, + step_count: 5, + output_alloc_us: 11, + target_build_us: 22, + enqueue_us: 33, + output_take_count: 1635, + output_pool_reuse_count: 1600, + output_pool_alloc_count: 35, + output_pool_scanned_count: 2400, + output_pool_max_free_count: 1700, + output_requested_bytes: 28, + }; + + assert_eq!( + format_cuda_idwt_batch_host_trace_row(row), + "j2k_profile codec=j2k op=cuda_idwt_batch_host path=decode component_count=327 step_count=5 output_alloc_us=11 target_build_us=22 enqueue_us=33 output_take_count=1635 output_pool_reuse_count=1600 output_pool_alloc_count=35 output_pool_scanned_count=2400 output_pool_max_free_count=1700 output_requested_bytes=28" + ); + } +} diff --git a/crates/j2k-cuda/src/direct_plan.rs b/crates/j2k-cuda/src/direct_plan.rs new file mode 100644 index 00000000..88998cb0 --- /dev/null +++ b/crates/j2k-cuda/src/direct_plan.rs @@ -0,0 +1,1120 @@ +use std::collections::HashMap; + +use j2k_core::PixelFormat; +use j2k_native::{ + idwt_band_index, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kDirectIdwtStep, + J2kDirectStoreStep, J2kRect, J2kWaveletTransform, +}; + +use crate::Error; + +const CLASSIC_J2K_NOT_CUDA_HTJ2K: &str = + "strict CUDA codestream decode only accepts HTJ2K direct-plan subbands"; +const EMPTY_HTJ2K_PLAN: &str = "strict CUDA HTJ2K plan contains no HT code blocks"; +const MIXED_TRANSFORMS_UNSUPPORTED: &str = "strict CUDA HTJ2K plan contains mixed DWT transforms"; +const PLAN_PAYLOAD_TOO_LARGE: &str = "strict CUDA HTJ2K plan payload is too large"; +const PLAN_BLOCK_LENGTH_MISMATCH: &str = + "strict CUDA HTJ2K plan block lengths do not match payload bytes"; +const PLAN_OUTPUT_RECT_MISMATCH: &str = + "strict CUDA HTJ2K plan store does not fit the requested output rectangle"; +const ROI_MAXSHIFT_UNSUPPORTED: &str = + "strict CUDA HTJ2K plan does not support ROI maxshift decode"; + +/// CUDA-side DWT transform selector for a flat HTJ2K plan. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u32)] +pub enum CudaHtj2kTransform { + /// Reversible 5/3 transform. + Reversible53, + /// Irreversible 9/7 transform. + Irreversible97, +} + +/// Stable CUDA-side identifier for a direct-plan coefficient band. +pub type CudaHtj2kBandId = u32; + +impl CudaHtj2kTransform { + pub(crate) fn from_native(value: J2kWaveletTransform) -> Self { + match value { + J2kWaveletTransform::Reversible53 => Self::Reversible53, + J2kWaveletTransform::Irreversible97 => Self::Irreversible97, + } + } +} + +/// Flat POD HTJ2K code-block metadata consumed by CUDA kernels. +#[derive(Debug, Clone, Copy, PartialEq)] +#[repr(C)] +pub struct CudaHtj2kCodeBlock { + /// Index of the parent sub-band in [`CudaHtj2kDecodePlan::subbands`]. + pub subband_index: u32, + /// Byte offset into [`CudaHtj2kDecodePlan::payload`]. + pub payload_offset: u64, + /// Total payload byte length for this code block. + pub payload_len: u32, + /// Cleanup segment length in bytes. + pub cleanup_length: u32, + /// Refinement segment length in bytes. + pub refinement_length: u32, + /// X offset within the target sub-band coefficient buffer. + pub output_x: u32, + /// Y offset within the target sub-band coefficient buffer. + pub output_y: u32, + /// Code-block width in samples. + pub width: u32, + /// Code-block height in samples. + pub height: u32, + /// Output row stride, in samples. + pub output_stride: u32, + /// Missing most-significant bit planes. + pub missing_bit_planes: u8, + /// Number of coding passes present. + pub number_of_coding_passes: u8, + /// Total coded bitplanes for the parent sub-band. + pub num_bitplanes: u8, + /// Nonzero when vertically causal context was enabled. + pub stripe_causal: u8, + /// Dequantization step to apply to decoded coefficients. + pub dequantization_step: f32, +} + +/// Flat POD sub-band geometry consumed by CUDA kernels. +#[derive(Debug, Clone, Copy, PartialEq)] +#[repr(C)] +pub struct CudaHtj2kSubband { + /// Stable CUDA direct-plan band id. + pub band_id: CudaHtj2kBandId, + /// Absolute x0 coordinate in component space. + pub x0: u32, + /// Absolute y0 coordinate in component space. + pub y0: u32, + /// Absolute x1 coordinate in component space. + pub x1: u32, + /// Absolute y1 coordinate in component space. + pub y1: u32, + /// Sub-band width in samples. + pub width: u32, + /// Sub-band height in samples. + pub height: u32, + /// First code-block index for this sub-band. + pub code_block_start: u32, + /// Number of code blocks for this sub-band. + pub code_block_count: u32, +} + +/// Flat POD IDWT step consumed by CUDA kernels. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct CudaHtj2kIdwtStep { + /// Stable identifier of the output coefficient band produced by this step. + pub output_band_id: CudaHtj2kBandId, + /// DWT transform to apply. + pub transform: CudaHtj2kTransform, + /// Output rectangle. + pub rect: CudaHtj2kRect, + /// LL input band id. + pub ll_band_id: CudaHtj2kBandId, + /// LL input rectangle. + pub ll_rect: CudaHtj2kRect, + /// HL input band id. + pub hl_band_id: CudaHtj2kBandId, + /// HL input rectangle. + pub hl_rect: CudaHtj2kRect, + /// LH input band id. + pub lh_band_id: CudaHtj2kBandId, + /// LH input rectangle. + pub lh_rect: CudaHtj2kRect, + /// HH input band id. + pub hh_band_id: CudaHtj2kBandId, + /// HH input rectangle. + pub hh_rect: CudaHtj2kRect, +} + +/// Flat POD store step consumed by CUDA kernels. +#[derive(Debug, Clone, Copy, PartialEq)] +#[repr(C)] +pub struct CudaHtj2kStoreStep { + /// Stable identifier of the input coefficient band. + pub input_band_id: CudaHtj2kBandId, + /// Source rectangle. + pub input_rect: CudaHtj2kRect, + /// Source x offset. + pub source_x: u32, + /// Source y offset. + pub source_y: u32, + /// Number of samples copied per row. + pub copy_width: u32, + /// Number of rows copied. + pub copy_height: u32, + /// Destination row width. + pub output_width: u32, + /// Destination height. + pub output_height: u32, + /// Destination x offset. + pub output_x: u32, + /// Destination y offset. + pub output_y: u32, + /// Constant level-shift addend. + pub addend: f32, +} + +/// Flat POD rectangle used inside CUDA HTJ2K plan metadata. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct CudaHtj2kRect { + /// Inclusive left coordinate. + pub x0: u32, + /// Inclusive top coordinate. + pub y0: u32, + /// Exclusive right coordinate. + pub x1: u32, + /// Exclusive bottom coordinate. + pub y1: u32, +} + +/// Flat CUDA HTJ2K decode plan. +#[derive(Debug, Clone)] +pub struct CudaHtj2kDecodePlan { + dimensions: (u32, u32), + bit_depth: u8, + output_format: PixelFormat, + output_origin: (u32, u32), + transform: CudaHtj2kTransform, + payload: Vec, + code_blocks: Vec, + subbands: Vec, + idwt_steps: Vec, + store_steps: Vec, +} + +impl CudaHtj2kDecodePlan { + pub(crate) fn from_grayscale_direct_plan( + plan: &J2kDirectGrayscalePlan, + output_format: PixelFormat, + output_origin: (u32, u32), + ) -> Result { + Self::from_grayscale_direct_plan_region(plan, output_format, output_origin, plan.dimensions) + } + + pub(crate) fn from_grayscale_direct_plan_region( + plan: &J2kDirectGrayscalePlan, + output_format: PixelFormat, + output_origin: (u32, u32), + output_dimensions: (u32, u32), + ) -> Result { + let capacity_hint = cuda_plan_capacity_hint(plan)?; + let mut payload = Vec::with_capacity(capacity_hint.payload_bytes); + let mut code_blocks = Vec::with_capacity(capacity_hint.code_blocks); + let mut subbands = Vec::with_capacity(capacity_hint.subbands); + let mut idwt_steps = Vec::with_capacity(capacity_hint.idwt_steps); + let mut store_steps = Vec::with_capacity(capacity_hint.store_steps); + let mut transform = None; + let mut saw_classic = false; + let required_regions = if output_origin == (0, 0) && output_dimensions == plan.dimensions { + None + } else { + Some(required_regions_for_direct_plan(plan)?) + }; + + for step in &plan.steps { + match step { + J2kDirectGrayscaleStep::HtSubBand(subband) => { + let subband_index = u32::try_from(subbands.len()).map_err(|_| { + Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + } + })?; + let code_block_start = u32::try_from(code_blocks.len()).map_err(|_| { + Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + } + })?; + for job in &subband.jobs { + let payload_offset = u64::try_from(payload.len()).map_err(|_| { + Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + } + })?; + let payload_len = u32::try_from(job.data.len()).map_err(|_| { + Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + } + })?; + let expected_len = job + .cleanup_length + .checked_add(job.refinement_length) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_BLOCK_LENGTH_MISMATCH, + })?; + if expected_len != payload_len { + return Err(Error::UnsupportedCudaRequest { + reason: PLAN_BLOCK_LENGTH_MISMATCH, + }); + } + let output_stride = u32::try_from(job.output_stride).map_err(|_| { + Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + } + })?; + if let Some(required_regions) = &required_regions { + if !required_regions + .get(&subband.band_id) + .is_some_and(|required| { + required.intersects( + job.output_x, + job.output_y, + job.width, + job.height, + ) + }) + { + continue; + } + } + if job.roi_shift != 0 { + return Err(Error::UnsupportedCudaRequest { + reason: ROI_MAXSHIFT_UNSUPPORTED, + }); + } + payload.extend_from_slice(&job.data); + code_blocks.push(CudaHtj2kCodeBlock { + subband_index, + payload_offset, + payload_len, + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + output_x: job.output_x, + output_y: job.output_y, + width: job.width, + height: job.height, + output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + num_bitplanes: job.num_bitplanes, + stripe_causal: u8::from(job.stripe_causal), + dequantization_step: job.dequantization_step, + }); + } + let code_block_count = u32::try_from( + code_blocks.len() - code_block_start as usize, + ) + .map_err(|_| Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + })?; + subbands.push(CudaHtj2kSubband { + band_id: subband.band_id, + x0: subband.rect.x0, + y0: subband.rect.y0, + x1: subband.rect.x1, + y1: subband.rect.y1, + width: subband.width, + height: subband.height, + code_block_start, + code_block_count, + }); + } + J2kDirectGrayscaleStep::ClassicSubBand(_) => saw_classic = true, + J2kDirectGrayscaleStep::Idwt(step) => { + let step_transform = CudaHtj2kTransform::from_native(step.transform); + match transform { + Some(existing) if existing != step_transform => { + return Err(Error::UnsupportedCudaRequest { + reason: MIXED_TRANSFORMS_UNSUPPORTED, + }); + } + Some(_) => {} + None => transform = Some(step_transform), + } + idwt_steps.push(convert_idwt_step(*step)); + } + J2kDirectGrayscaleStep::Store(step) => { + store_steps.push(convert_store_step(*step, output_origin, output_dimensions)?); + } + } + } + + if saw_classic { + return Err(Error::UnsupportedCudaRequest { + reason: CLASSIC_J2K_NOT_CUDA_HTJ2K, + }); + } + if code_blocks.is_empty() { + return Err(Error::UnsupportedCudaRequest { + reason: EMPTY_HTJ2K_PLAN, + }); + } + + Ok(Self { + dimensions: output_dimensions, + bit_depth: plan.bit_depth, + output_format, + output_origin, + transform: transform.unwrap_or(CudaHtj2kTransform::Reversible53), + payload, + code_blocks, + subbands, + idwt_steps, + store_steps, + }) + } + + /// Output dimensions of the decoded surface. + pub fn dimensions(&self) -> (u32, u32) { + self.dimensions + } + + /// Source component bit depth. + pub fn bit_depth(&self) -> u8 { + self.bit_depth + } + + /// Output pixel format requested by the caller. + pub fn output_format(&self) -> PixelFormat { + self.output_format + } + + /// Destination origin in the caller-visible output surface. + pub fn output_origin(&self) -> (u32, u32) { + self.output_origin + } + + /// DWT transform used by IDWT kernels. + pub fn transform(&self) -> CudaHtj2kTransform { + self.transform + } + + /// Contiguous cleanup/refinement payload bytes. + pub fn payload(&self) -> &[u8] { + &self.payload + } + + #[cfg_attr(not(feature = "cuda-runtime"), allow(dead_code))] + pub(crate) fn append_payload_to_shared( + &mut self, + shared_payload: &mut Vec, + ) -> Result<(), Error> { + let base = + u64::try_from(shared_payload.len()).map_err(|_| Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + })?; + shared_payload + .try_reserve(self.payload.len()) + .map_err(|_| Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + })?; + for block in &mut self.code_blocks { + block.payload_offset = + block + .payload_offset + .checked_add(base) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + })?; + } + shared_payload.append(&mut self.payload); + Ok(()) + } + + #[cfg_attr(not(feature = "cuda-runtime"), allow(dead_code))] + pub(crate) fn rebase_payload_offsets(&mut self, base: u64) -> Result<(), Error> { + for block in &mut self.code_blocks { + block.payload_offset = + block + .payload_offset + .checked_add(base) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + })?; + } + Ok(()) + } + + /// Flat code-block metadata. + pub fn code_blocks(&self) -> &[CudaHtj2kCodeBlock] { + &self.code_blocks + } + + /// Flat sub-band metadata. + pub fn subbands(&self) -> &[CudaHtj2kSubband] { + &self.subbands + } + + /// Flat IDWT step metadata. + pub fn idwt_steps(&self) -> &[CudaHtj2kIdwtStep] { + &self.idwt_steps + } + + /// Flat store step metadata. + pub fn store_steps(&self) -> &[CudaHtj2kStoreStep] { + &self.store_steps + } + + /// Number of per-code-block decode dispatches implied by the plan. + pub fn dispatch_count_hint(&self) -> usize { + self.code_blocks.len() + } +} + +#[derive(Debug, Default)] +struct CudaPlanCapacityHint { + payload_bytes: usize, + code_blocks: usize, + subbands: usize, + idwt_steps: usize, + store_steps: usize, +} + +fn cuda_plan_capacity_hint(plan: &J2kDirectGrayscalePlan) -> Result { + let mut hint = CudaPlanCapacityHint::default(); + for step in &plan.steps { + match step { + J2kDirectGrayscaleStep::HtSubBand(subband) => { + hint.subbands = hint.subbands.saturating_add(1); + hint.code_blocks = hint.code_blocks.checked_add(subband.jobs.len()).ok_or( + Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + }, + )?; + for job in &subband.jobs { + hint.payload_bytes = hint.payload_bytes.checked_add(job.data.len()).ok_or( + Error::UnsupportedCudaRequest { + reason: PLAN_PAYLOAD_TOO_LARGE, + }, + )?; + } + } + J2kDirectGrayscaleStep::ClassicSubBand(_) => {} + J2kDirectGrayscaleStep::Idwt(_) => { + hint.idwt_steps = hint.idwt_steps.saturating_add(1); + } + J2kDirectGrayscaleStep::Store(_) => { + hint.store_steps = hint.store_steps.saturating_add(1); + } + } + } + Ok(hint) +} + +fn convert_idwt_step(step: J2kDirectIdwtStep) -> CudaHtj2kIdwtStep { + CudaHtj2kIdwtStep { + output_band_id: step.output_band_id, + transform: CudaHtj2kTransform::from_native(step.transform), + rect: convert_rect(step.rect), + ll_band_id: step.ll_band_id, + ll_rect: convert_rect(step.ll), + hl_band_id: step.hl_band_id, + hl_rect: convert_rect(step.hl), + lh_band_id: step.lh_band_id, + lh_rect: convert_rect(step.lh), + hh_band_id: step.hh_band_id, + hh_rect: convert_rect(step.hh), + } +} + +#[derive(Clone, Copy, Debug)] +struct RequiredBandRegion { + x0: u32, + y0: u32, + x1: u32, + y1: u32, +} + +impl RequiredBandRegion { + fn new(x0: u32, y0: u32, x1: u32, y1: u32) -> Option { + (x0 < x1 && y0 < y1).then_some(Self { x0, y0, x1, y1 }) + } + + fn expanded(self, margin: u32, width: u32, height: u32) -> Self { + Self { + x0: self.x0.saturating_sub(margin), + y0: self.y0.saturating_sub(margin), + x1: self.x1.saturating_add(margin).min(width), + y1: self.y1.saturating_add(margin).min(height), + } + } + + const fn union(self, other: Self) -> Self { + Self { + x0: if self.x0 < other.x0 { + self.x0 + } else { + other.x0 + }, + y0: if self.y0 < other.y0 { + self.y0 + } else { + other.y0 + }, + x1: if self.x1 > other.x1 { + self.x1 + } else { + other.x1 + }, + y1: if self.y1 > other.y1 { + self.y1 + } else { + other.y1 + }, + } + } + + fn intersects(self, x0: u32, y0: u32, width: u32, height: u32) -> bool { + let x1 = x0.saturating_add(width); + let y1 = y0.saturating_add(height); + self.x0 < x1 && x0 < self.x1 && self.y0 < y1 && y0 < self.y1 + } +} + +fn required_regions_for_direct_plan( + plan: &J2kDirectGrayscalePlan, +) -> Result, Error> { + let mut required = HashMap::::new(); + for step in &plan.steps { + let J2kDirectGrayscaleStep::Store(store) = step else { + continue; + }; + let source_right = + store + .source_x + .checked_add(store.copy_width) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_OUTPUT_RECT_MISMATCH, + })?; + let source_bottom = + store + .source_y + .checked_add(store.copy_height) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_OUTPUT_RECT_MISMATCH, + })?; + if let Some(region) = + RequiredBandRegion::new(store.source_x, store.source_y, source_right, source_bottom) + { + add_required_region(&mut required, store.input_band_id, region); + } + } + + for step in plan.steps.iter().rev() { + let J2kDirectGrayscaleStep::Idwt(idwt) = step else { + continue; + }; + let Some(output_region) = required.get(&idwt.output_band_id).copied() else { + continue; + }; + let expanded = output_region.expanded( + idwt_required_output_margin(idwt.transform), + idwt.rect.width(), + idwt.rect.height(), + ); + add_idwt_input_required_regions(&mut required, idwt, expanded); + } + Ok(required) +} + +fn add_required_region( + required: &mut HashMap, + band_id: CudaHtj2kBandId, + region: RequiredBandRegion, +) { + required + .entry(band_id) + .and_modify(|existing| *existing = existing.union(region)) + .or_insert(region); +} + +const fn idwt_required_output_margin(transform: J2kWaveletTransform) -> u32 { + match transform { + J2kWaveletTransform::Reversible53 => 16, + J2kWaveletTransform::Irreversible97 => 40, + } +} + +fn add_idwt_input_required_regions( + required: &mut HashMap, + idwt: &J2kDirectIdwtStep, + output_region: RequiredBandRegion, +) { + add_required_region( + required, + idwt.ll_band_id, + idwt_input_required_region( + output_region, + idwt.rect.x0, + idwt.rect.y0, + true, + true, + idwt.ll.width(), + idwt.ll.height(), + ), + ); + add_required_region( + required, + idwt.hl_band_id, + idwt_input_required_region( + output_region, + idwt.rect.x0, + idwt.rect.y0, + false, + true, + idwt.hl.width(), + idwt.hl.height(), + ), + ); + add_required_region( + required, + idwt.lh_band_id, + idwt_input_required_region( + output_region, + idwt.rect.x0, + idwt.rect.y0, + true, + false, + idwt.lh.width(), + idwt.lh.height(), + ), + ); + add_required_region( + required, + idwt.hh_band_id, + idwt_input_required_region( + output_region, + idwt.rect.x0, + idwt.rect.y0, + false, + false, + idwt.hh.width(), + idwt.hh.height(), + ), + ); +} + +#[allow(clippy::fn_params_excessive_bools)] +fn idwt_input_required_region( + output_region: RequiredBandRegion, + output_origin_x: u32, + output_origin_y: u32, + low_x: bool, + low_y: bool, + band_width: u32, + band_height: u32, +) -> RequiredBandRegion { + let x0 = idwt_band_index(output_origin_x, output_region.x0, low_x); + let x1 = idwt_band_index(output_origin_x, output_region.x1 - 1, low_x).saturating_add(1); + let y0 = idwt_band_index(output_origin_y, output_region.y0, low_y); + let y1 = idwt_band_index(output_origin_y, output_region.y1 - 1, low_y).saturating_add(1); + RequiredBandRegion { + x0: x0.min(band_width), + y0: y0.min(band_height), + x1: x1.min(band_width), + y1: y1.min(band_height), + } +} + +fn convert_store_step( + step: J2kDirectStoreStep, + output_origin: (u32, u32), + output_dimensions: (u32, u32), +) -> Result { + if output_dimensions.0 == 0 || output_dimensions.1 == 0 { + return Err(Error::UnsupportedCudaRequest { + reason: PLAN_OUTPUT_RECT_MISMATCH, + }); + } + let region_end_x = + output_origin + .0 + .checked_add(output_dimensions.0) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_OUTPUT_RECT_MISMATCH, + })?; + let region_end_y = + output_origin + .1 + .checked_add(output_dimensions.1) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_OUTPUT_RECT_MISMATCH, + })?; + let store_end_x = + step.output_x + .checked_add(step.copy_width) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_OUTPUT_RECT_MISMATCH, + })?; + let store_end_y = + step.output_y + .checked_add(step.copy_height) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_OUTPUT_RECT_MISMATCH, + })?; + if output_origin.0 < step.output_x + || output_origin.1 < step.output_y + || region_end_x > store_end_x + || region_end_y > store_end_y + { + return Err(Error::UnsupportedCudaRequest { + reason: PLAN_OUTPUT_RECT_MISMATCH, + }); + } + let source_x = step + .source_x + .checked_add(output_origin.0 - step.output_x) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_OUTPUT_RECT_MISMATCH, + })?; + let source_y = step + .source_y + .checked_add(output_origin.1 - step.output_y) + .ok_or(Error::UnsupportedCudaRequest { + reason: PLAN_OUTPUT_RECT_MISMATCH, + })?; + Ok(CudaHtj2kStoreStep { + input_band_id: step.input_band_id, + input_rect: convert_rect(step.input_rect), + source_x, + source_y, + copy_width: output_dimensions.0, + copy_height: output_dimensions.1, + output_width: output_dimensions.0, + output_height: output_dimensions.1, + output_x: 0, + output_y: 0, + addend: step.addend, + }) +} + +fn convert_rect(rect: J2kRect) -> CudaHtj2kRect { + CudaHtj2kRect { + x0: rect.x0, + y0: rect.y0, + x1: rect.x1, + y1: rect.y1, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use j2k_core::CodecError; + use j2k_native::{HtOwnedCodeBlockBatchJob, HtOwnedSubBandPlan}; + + fn one_block_direct_plan( + cleanup_length: u32, + refinement_length: u32, + data: Vec, + output_stride: usize, + ) -> J2kDirectGrayscalePlan { + J2kDirectGrayscalePlan { + dimensions: (1, 1), + bit_depth: 8, + steps: vec![ + J2kDirectGrayscaleStep::HtSubBand(HtOwnedSubBandPlan { + band_id: 0, + rect: J2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + width: 1, + height: 1, + jobs: vec![HtOwnedCodeBlockBatchJob { + output_x: 0, + output_y: 0, + data, + cleanup_length, + refinement_length, + width: 1, + height: 1, + output_stride, + missing_bit_planes: 0, + number_of_coding_passes: 1, + num_bitplanes: 8, + roi_shift: 0, + stripe_causal: false, + strict: true, + dequantization_step: 1.0, + }], + }), + J2kDirectGrayscaleStep::Store(J2kDirectStoreStep { + input_band_id: 0, + input_rect: J2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }, + source_x: 0, + source_y: 0, + copy_width: 1, + copy_height: 1, + output_width: 1, + output_height: 1, + output_x: 0, + output_y: 0, + addend: 128.0, + }), + ], + } + } + + fn one_block_plan(data: Vec) -> CudaHtj2kDecodePlan { + let payload_len = u32::try_from(data.len()).expect("fixture payload length"); + let direct = one_block_direct_plan(payload_len, 0, data, 1); + CudaHtj2kDecodePlan::from_grayscale_direct_plan(&direct, PixelFormat::Gray8, (0, 0)) + .expect("CUDA plan") + } + + fn two_block_direct_plan() -> J2kDirectGrayscalePlan { + J2kDirectGrayscalePlan { + dimensions: (2, 1), + bit_depth: 8, + steps: vec![ + J2kDirectGrayscaleStep::HtSubBand(HtOwnedSubBandPlan { + band_id: 0, + rect: J2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 1, + }, + width: 2, + height: 1, + jobs: vec![ + HtOwnedCodeBlockBatchJob { + output_x: 0, + output_y: 0, + data: vec![1], + cleanup_length: 1, + refinement_length: 0, + width: 1, + height: 1, + output_stride: 2, + missing_bit_planes: 0, + number_of_coding_passes: 1, + num_bitplanes: 8, + roi_shift: 0, + stripe_causal: false, + strict: true, + dequantization_step: 1.0, + }, + HtOwnedCodeBlockBatchJob { + output_x: 1, + output_y: 0, + data: vec![2], + cleanup_length: 1, + refinement_length: 0, + width: 1, + height: 1, + output_stride: 2, + missing_bit_planes: 0, + number_of_coding_passes: 1, + num_bitplanes: 8, + roi_shift: 0, + stripe_causal: false, + strict: true, + dequantization_step: 1.0, + }, + ], + }), + J2kDirectGrayscaleStep::Store(J2kDirectStoreStep { + input_band_id: 0, + input_rect: J2kRect { + x0: 0, + y0: 0, + x1: 2, + y1: 1, + }, + source_x: 0, + source_y: 0, + copy_width: 2, + copy_height: 1, + output_width: 2, + output_height: 1, + output_x: 0, + output_y: 0, + addend: 128.0, + }), + ], + } + } + + #[test] + fn append_payload_to_shared_offsets_blocks_and_drains_local_payload() { + let mut first = one_block_plan(vec![1, 2]); + let mut second = one_block_plan(vec![3, 4, 5]); + let mut shared = Vec::new(); + + first + .append_payload_to_shared(&mut shared) + .expect("append first payload"); + second + .append_payload_to_shared(&mut shared) + .expect("append second payload"); + + assert_eq!(shared, vec![1, 2, 3, 4, 5]); + assert!(first.payload().is_empty()); + assert!(second.payload().is_empty()); + assert_eq!(first.code_blocks()[0].payload_offset, 0); + assert_eq!(second.code_blocks()[0].payload_offset, 2); + } + + #[test] + fn rebase_payload_offsets_preserves_shared_payload_for_larger_batch() { + let mut plan = one_block_plan(vec![7, 8]); + let mut shared = Vec::new(); + plan.append_payload_to_shared(&mut shared) + .expect("append local payload"); + + plan.rebase_payload_offsets(4096).expect("rebase payload"); + + assert_eq!(shared, vec![7, 8]); + assert_eq!(plan.code_blocks()[0].payload_offset, 4096); + } + + #[test] + fn full_frame_plan_keeps_all_blocks_while_region_plan_prunes() { + let direct = two_block_direct_plan(); + let full = + CudaHtj2kDecodePlan::from_grayscale_direct_plan(&direct, PixelFormat::Gray8, (0, 0)) + .expect("full CUDA plan"); + let mut region_direct = two_block_direct_plan(); + let J2kDirectGrayscaleStep::Store(store) = &mut region_direct.steps[1] else { + panic!("expected store fixture"); + }; + store.source_x = 1; + store.copy_width = 1; + store.output_x = 1; + let region = CudaHtj2kDecodePlan::from_grayscale_direct_plan_region( + ®ion_direct, + PixelFormat::Gray8, + (1, 0), + (1, 1), + ) + .expect("region CUDA plan"); + + assert_eq!(full.code_blocks().len(), 2); + assert_eq!(region.code_blocks().len(), 1); + assert_eq!(region.code_blocks()[0].output_x, 1); + } + + #[test] + fn rejects_block_length_mismatch() { + let direct = one_block_direct_plan(1, 2, vec![0xAA, 0xBB], 1); + + let error = + CudaHtj2kDecodePlan::from_grayscale_direct_plan(&direct, PixelFormat::Gray8, (0, 0)) + .expect_err("mismatched cleanup/refinement lengths must be rejected"); + + assert!(error.is_unsupported()); + assert!( + error + .to_string() + .contains("block lengths do not match payload bytes"), + "unexpected error: {error}" + ); + } + + #[test] + fn rejects_roi_maxshift_jobs() { + let mut direct = one_block_direct_plan(1, 0, vec![0xAA], 1); + let J2kDirectGrayscaleStep::HtSubBand(subband) = &mut direct.steps[0] else { + panic!("fixture starts with one HT sub-band"); + }; + subband.jobs[0].roi_shift = 7; + + let error = + CudaHtj2kDecodePlan::from_grayscale_direct_plan(&direct, PixelFormat::Gray8, (0, 0)) + .expect_err("ROI maxshift jobs must be rejected"); + + assert!(error.is_unsupported()); + assert!( + error.to_string().contains("ROI maxshift decode"), + "unexpected error: {error}" + ); + } + + #[test] + fn rejects_output_stride_overflow() { + let direct = one_block_direct_plan(1, 0, vec![0xAA], usize::MAX); + + let error = + CudaHtj2kDecodePlan::from_grayscale_direct_plan(&direct, PixelFormat::Gray8, (0, 0)) + .expect_err("unrepresentable output stride must be rejected"); + + assert!(error.is_unsupported()); + } + + #[test] + fn rejects_mixed_idwt_transforms() { + let mut direct = one_block_direct_plan(1, 0, vec![0xAA], 1); + let rect = J2kRect { + x0: 0, + y0: 0, + x1: 1, + y1: 1, + }; + direct.steps.insert( + 1, + J2kDirectGrayscaleStep::Idwt(J2kDirectIdwtStep { + output_band_id: 4, + rect, + transform: J2kWaveletTransform::Reversible53, + ll_band_id: 0, + ll: rect, + hl_band_id: 1, + hl: rect, + lh_band_id: 2, + lh: rect, + hh_band_id: 3, + hh: rect, + }), + ); + direct.steps.insert( + 2, + J2kDirectGrayscaleStep::Idwt(J2kDirectIdwtStep { + output_band_id: 8, + rect, + transform: J2kWaveletTransform::Irreversible97, + ll_band_id: 4, + ll: rect, + hl_band_id: 5, + hl: rect, + lh_band_id: 6, + lh: rect, + hh_band_id: 7, + hh: rect, + }), + ); + + let error = + CudaHtj2kDecodePlan::from_grayscale_direct_plan(&direct, PixelFormat::Gray8, (0, 0)) + .expect_err("mixed transforms must be rejected"); + + assert!(error.is_unsupported()); + assert!( + error.to_string().contains("mixed DWT transforms"), + "unexpected error: {error}" + ); + } + + #[test] + fn region_plan_rejects_store_outside_output_rect() { + let direct = one_block_direct_plan(1, 0, vec![0xAA], 1); + + let error = CudaHtj2kDecodePlan::from_grayscale_direct_plan_region( + &direct, + PixelFormat::Gray8, + (1, 1), + (0, 0), + ) + .expect_err("store outside compact output rectangle must be rejected"); + + assert!(error.is_unsupported()); + assert!( + error + .to_string() + .contains("store does not fit the requested output rectangle"), + "unexpected error: {error}" + ); + } +} diff --git a/crates/j2k-cuda/src/encode.rs b/crates/j2k-cuda/src/encode.rs new file mode 100644 index 00000000..c7b56db3 --- /dev/null +++ b/crates/j2k-cuda/src/encode.rs @@ -0,0 +1,5231 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::adapter::encode_stage::{ + EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, J2kDeinterleaveToF32Job, J2kEncodeDispatchReport, + J2kEncodeStageAccelerator, J2kForwardDwt53Job, J2kForwardDwt53Output, J2kForwardDwt97Job, + J2kForwardDwt97Output, J2kForwardIctJob, J2kForwardRctJob, J2kHtCodeBlockEncodeJob, + J2kHtSubbandEncodeJob, J2kHtj2kTileEncodeJob, J2kPacketizationBlockCodingMode, + J2kPacketizationCodeBlock, J2kPacketizationEncodeJob, J2kPacketizationResolution, + J2kQuantizeSubbandJob, J2kTier1CodeBlockEncodeJob, +}; +#[cfg(feature = "cuda-runtime")] +use j2k::adapter::encode_stage::{ + J2kForwardDwt53Level, J2kForwardDwt97Level, J2kPacketizationPacketDescriptor, + J2kPacketizationSubband, +}; +use j2k_core::BackendKind; +#[cfg(feature = "cuda-runtime")] +use j2k_core::{DeviceSubmission, DeviceSubmitSession, PixelFormat, ReadySubmission}; +#[cfg(feature = "cuda-runtime")] +use j2k_cuda_runtime::{ + CudaContext, CudaDeviceBuffer, CudaDwt53LevelShape, CudaDwt53Output, CudaDwt97Output, + CudaError, CudaHtj2kEncodeCodeBlockJob, CudaHtj2kEncodeCodeBlockRegionJob, + CudaHtj2kEncodeResources, CudaHtj2kEncodeTables, CudaHtj2kPacketizationBlock, + CudaHtj2kPacketizationPacket, CudaHtj2kPacketizationSubband, + CudaHtj2kPacketizationSubbandTagState, CudaHtj2kPacketizationTagNodeState, CudaJ2kQuantizeJob, + CudaJ2kQuantizeSubbandRegionJob, CudaJ2kResidentComponents, CudaJ2kStridedInterleavedPixels, +}; +#[cfg(feature = "cuda-runtime")] +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use j2k_native::packet_math; + +use crate::profile; +#[cfg(feature = "cuda-runtime")] +use crate::{runtime::cuda_error, session::CudaSession}; + +/// Encode lossless JPEG 2000/HTJ2K samples through the CUDA encode-stage adapter. +/// +/// This CUDA-named API is strict: every caller-provided backend preference is +/// treated as `EncodeBackendPreference::RequireDevice`, so unsupported stage +/// coverage returns an error instead of a CPU fallback codestream. +pub fn encode_j2k_lossless_with_cuda( + samples: j2k::J2kLosslessSamples<'_>, + options: &j2k::J2kLosslessEncodeOptions, +) -> Result { + let strict_options = strict_cuda_encode_options(*options); + let profile_enabled = profile::profile_stages_enabled(); + let mut accelerator = CudaEncodeStageAccelerator::with_profile_collection(profile_enabled); + let total_start = profile::profile_now(profile_enabled); + let encoded = j2k::encode_j2k_lossless_with_accelerator( + samples, + &strict_options, + BackendKind::Cuda, + &mut accelerator, + )?; + reject_non_cuda_encode_backend(&encoded)?; + if profile_enabled { + accelerator + .encode_profile_report( + &encoded, + samples.data.len(), + profile::elapsed_us(total_start), + ) + .emit("encode"); + } + Ok(encoded) +} + +/// Encode lossless JPEG 2000/HTJ2K samples through CUDA and return stage timings. +pub fn encode_j2k_lossless_with_cuda_and_profile( + samples: j2k::J2kLosslessSamples<'_>, + options: &j2k::J2kLosslessEncodeOptions, +) -> Result<(j2k::EncodedJ2k, profile::CudaHtj2kEncodeProfileReport), crate::Error> { + let input_bytes = samples.data.len(); + let strict_options = strict_cuda_encode_options(*options); + let mut accelerator = CudaEncodeStageAccelerator::with_profile_collection(true); + let total_start = profile::profile_now(true); + let encoded = j2k::encode_j2k_lossless_with_accelerator( + samples, + &strict_options, + BackendKind::Cuda, + &mut accelerator, + )?; + reject_non_cuda_encode_backend(&encoded)?; + let report = + accelerator.encode_profile_report(&encoded, input_bytes, profile::elapsed_us(total_start)); + report.emit("encode"); + Ok((encoded, report)) +} + +fn strict_cuda_encode_options( + options: j2k::J2kLosslessEncodeOptions, +) -> j2k::J2kLosslessEncodeOptions { + options.with_backend(j2k::EncodeBackendPreference::RequireDevice) +} + +fn reject_non_cuda_encode_backend(encoded: &j2k::EncodedJ2k) -> Result<(), crate::Error> { + if encoded.backend == BackendKind::Cuda { + Ok(()) + } else { + Err(crate::Error::UnsupportedCudaRequest { + reason: "strict CUDA HTJ2K encode did not dispatch all required stages", + }) + } +} + +#[cfg(feature = "cuda-runtime")] +/// CUDA-resident lossless J2K/HTJ2K encode input tile. +#[derive(Debug, Clone, Copy)] +pub struct CudaLosslessEncodeTile<'a> { + /// Source CUDA buffer containing interleaved Gray/RGB/RGBA pixels. + pub buffer: &'a CudaDeviceBuffer, + /// Byte offset of the first source pixel in `buffer`. + pub byte_offset: usize, + /// Width of the valid input region in pixels. + pub width: u32, + /// Height of the valid input region in pixels. + pub height: u32, + /// Number of bytes between consecutive input rows. + pub pitch_bytes: usize, + /// Encoded image width in pixels. + pub output_width: u32, + /// Encoded image height in pixels. + pub output_height: u32, + /// Pixel format of the source buffer. + pub format: PixelFormat, +} + +#[cfg(feature = "cuda-runtime")] +/// Residency decisions used by a lossless CUDA device-buffer encode. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CudaLosslessEncodeResidency { + /// Whether coefficient preparation ran on CUDA. + pub coefficient_prep_used: bool, + /// Whether packetization ran on CUDA. + pub packetization_used: bool, + /// Whether final codestream assembly stayed resident on CUDA. + pub codestream_assembly_used: bool, +} + +#[cfg(feature = "cuda-runtime")] +/// Lossless CUDA device-buffer encode output with host codestream bytes and timings. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CudaLosslessEncodeOutcome { + /// Encoded J2K codestream. + pub encoded: j2k::EncodedJ2k, + /// Whether the input buffer had to be copied or padded. + pub input_copy_used: bool, + /// Residency decisions for encode stages. + pub resident: CudaLosslessEncodeResidency, + /// Time spent copying or padding input. + pub input_copy_duration: Duration, + /// End-to-end encode duration for this tile. + pub encode_duration: Duration, + /// GPU-only duration when timestamp data is available. + pub gpu_duration: Option, + /// Time spent validating encoded output. + pub validation_duration: Duration, + /// Time spent materializing CUDA output into host codestream bytes. + pub host_readback_duration: Duration, + /// CUDA encode stage timing buckets collected for this tile. + pub stage_timings: CudaEncodeStageTimings, +} + +#[cfg(feature = "cuda-runtime")] +/// Submitted single-tile CUDA lossless encode. +#[derive(Debug)] +pub struct SubmittedJ2kLosslessCudaEncode { + inner: ReadySubmission, +} + +#[cfg(feature = "cuda-runtime")] +/// Submitted multi-tile CUDA lossless encode. +#[derive(Debug)] +pub struct SubmittedJ2kLosslessCudaEncodeBatch { + inner: ReadySubmission, crate::Error>, +} + +#[cfg(feature = "cuda-runtime")] +impl DeviceSubmission for SubmittedJ2kLosslessCudaEncode { + type Output = j2k::EncodedJ2k; + type Error = crate::Error; + + fn wait(self) -> Result { + self.inner.wait() + } +} + +#[cfg(feature = "cuda-runtime")] +impl DeviceSubmission for SubmittedJ2kLosslessCudaEncodeBatch { + type Output = Vec; + type Error = crate::Error; + + fn wait(self) -> Result { + self.inner.wait() + } +} + +#[cfg(feature = "cuda-runtime")] +/// Encode one CUDA-resident tile into host codestream bytes. +pub fn encode_lossless_from_cuda_buffer( + tile: CudaLosslessEncodeTile<'_>, + options: &j2k::J2kLosslessEncodeOptions, + session: &mut CudaSession, +) -> Result { + submit_lossless_from_cuda_buffer(tile, options, session)?.wait() +} + +#[cfg(feature = "cuda-runtime")] +/// Submit one CUDA-resident tile encode for later host-byte collection. +pub fn submit_lossless_from_cuda_buffer( + tile: CudaLosslessEncodeTile<'_>, + options: &j2k::J2kLosslessEncodeOptions, + session: &mut CudaSession, +) -> Result { + let result = encode_lossless_from_cuda_buffer_with_report(tile, options, session) + .map(|outcome| outcome.encoded); + Ok(SubmittedJ2kLosslessCudaEncode { + inner: ReadySubmission::from_result(result), + }) +} + +#[cfg(feature = "cuda-runtime")] +/// Encode one CUDA-resident tile and return a host-byte timing report. +pub fn encode_lossless_from_cuda_buffer_with_report( + tile: CudaLosslessEncodeTile<'_>, + options: &j2k::J2kLosslessEncodeOptions, + session: &mut CudaSession, +) -> Result { + validate_cuda_encode_options(*options)?; + validate_cuda_encode_tile(tile)?; + session.record_submit(); + encode_lossless_cuda_tile_with_report(tile, *options) +} + +#[cfg(feature = "cuda-runtime")] +/// Encode multiple CUDA-resident tiles into host codestream bytes. +pub fn encode_lossless_from_cuda_buffers( + tiles: &[CudaLosslessEncodeTile<'_>], + options: &j2k::J2kLosslessEncodeOptions, + session: &mut CudaSession, +) -> Result, crate::Error> { + submit_lossless_from_cuda_buffers(tiles, options, session)?.wait() +} + +#[cfg(feature = "cuda-runtime")] +/// Submit multiple CUDA-resident tile encodes for later host-byte collection. +pub fn submit_lossless_from_cuda_buffers( + tiles: &[CudaLosslessEncodeTile<'_>], + options: &j2k::J2kLosslessEncodeOptions, + session: &mut CudaSession, +) -> Result { + let result = + encode_lossless_from_cuda_buffers_with_report(tiles, options, session).map(|outcomes| { + outcomes + .into_iter() + .map(|outcome| outcome.encoded) + .collect() + }); + Ok(SubmittedJ2kLosslessCudaEncodeBatch { + inner: ReadySubmission::from_result(result), + }) +} + +#[cfg(feature = "cuda-runtime")] +/// Encode multiple CUDA-resident tiles and return host-byte timing reports. +pub fn encode_lossless_from_cuda_buffers_with_report( + tiles: &[CudaLosslessEncodeTile<'_>], + options: &j2k::J2kLosslessEncodeOptions, + session: &mut CudaSession, +) -> Result, crate::Error> { + if tiles.is_empty() { + return Err(crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA encode received an empty tile batch", + }); + } + validate_cuda_encode_options(*options)?; + tiles + .iter() + .copied() + .map(|tile| { + validate_cuda_encode_tile(tile)?; + session.record_submit(); + encode_lossless_cuda_tile_with_report(tile, *options) + }) + .collect() +} + +#[cfg(feature = "cuda-runtime")] +fn validate_cuda_encode_options( + options: j2k::J2kLosslessEncodeOptions, +) -> Result<(), crate::Error> { + if options.block_coding_mode != j2k::J2kBlockCodingMode::HighThroughput { + return Err(crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA device-buffer encode currently requires HTJ2K block coding", + }); + } + if options.validation != j2k::J2kEncodeValidation::External { + return Err(crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA device-buffer encode requires external validation to avoid host input readback", + }); + } + Ok(()) +} + +#[cfg(feature = "cuda-runtime")] +fn validate_cuda_encode_tile(tile: CudaLosslessEncodeTile<'_>) -> Result<(), crate::Error> { + if tile.width == 0 || tile.height == 0 || tile.output_width == 0 || tile.output_height == 0 { + return Err(crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA encode tile dimensions must be nonzero", + }); + } + if tile.width != tile.output_width || tile.height != tile.output_height { + return Err(crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA device-buffer encode does not yet support input padding", + }); + } + let format = cuda_encode_format(tile.format)?; + let row_bytes = (tile.width as usize) + .checked_mul(format.bytes_per_pixel) + .ok_or(crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA encode row byte count overflow", + })?; + if tile.pitch_bytes < row_bytes { + return Err(crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA encode tile pitch is shorter than one row", + }); + } + let required_end = tile + .byte_offset + .checked_add( + tile.pitch_bytes + .checked_mul(tile.height.saturating_sub(1) as usize) + .and_then(|prefix| prefix.checked_add(row_bytes)) + .ok_or(crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA encode input byte range overflow", + })?, + ) + .ok_or(crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA encode input byte range overflow", + })?; + if required_end > tile.buffer.byte_len() { + return Err(crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA encode input byte range exceeds buffer length", + }); + } + Ok(()) +} + +#[cfg(feature = "cuda-runtime")] +#[derive(Debug, Clone, Copy)] +struct CudaEncodeFormat { + components: u8, + bit_depth: u8, + bytes_per_pixel: usize, +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_encode_format(format: PixelFormat) -> Result { + let components = + u8::try_from(format.channels()).map_err(|_| crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA encode received a pixel format with too many components", + })?; + let bit_depth = match format.bytes_per_sample() { + 1 => 8, + 2 => 16, + _ => { + return Err(crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA encode received an unsupported sample width", + }); + } + }; + Ok(CudaEncodeFormat { + components, + bit_depth, + bytes_per_pixel: format.bytes_per_pixel(), + }) +} + +#[cfg(feature = "cuda-runtime")] +fn encode_lossless_cuda_tile_with_report( + tile: CudaLosslessEncodeTile<'_>, + options: j2k::J2kLosslessEncodeOptions, +) -> Result { + let encode_started = Instant::now(); + let format = cuda_encode_format(tile.format)?; + let dummy_len = (tile.output_width as usize) + .checked_mul(tile.output_height as usize) + .and_then(|pixels| pixels.checked_mul(format.bytes_per_pixel)) + .ok_or(crate::Error::UnsupportedCudaRequest { + reason: "J2K CUDA encode sample descriptor length overflow", + })?; + let dummy = vec![0u8; dummy_len]; + let samples = j2k::J2kLosslessSamples::new( + &dummy, + tile.output_width, + tile.output_height, + format.components, + format.bit_depth, + false, + )?; + let context = tile.buffer.context(); + let resources = context + .upload_htj2k_encode_resources(cuda_htj2k_encode_tables()) + .map_err(cuda_error)?; + let mut accelerator = CudaDeviceBufferEncodeAccelerator { + tile, + context, + resources, + dispatch: J2kEncodeDispatchReport::default(), + stage_timings: CudaEncodeStageTimings::default(), + }; + let encoded = j2k::encode_j2k_lossless_with_accelerator( + samples, + &strict_cuda_encode_options(options), + BackendKind::Cuda, + &mut accelerator, + )?; + reject_non_cuda_encode_backend(&encoded)?; + Ok(CudaLosslessEncodeOutcome { + encoded, + input_copy_used: false, + resident: CudaLosslessEncodeResidency { + coefficient_prep_used: accelerator.dispatch.deinterleave > 0, + packetization_used: accelerator.dispatch.packetization > 0, + codestream_assembly_used: false, + }, + input_copy_duration: Duration::ZERO, + encode_duration: encode_started.elapsed(), + gpu_duration: None, + validation_duration: Duration::ZERO, + host_readback_duration: Duration::ZERO, + stage_timings: accelerator.stage_timings, + }) +} + +#[cfg(feature = "cuda-runtime")] +struct CudaDeviceBufferEncodeAccelerator<'a> { + tile: CudaLosslessEncodeTile<'a>, + context: CudaContext, + resources: CudaHtj2kEncodeResources, + dispatch: J2kEncodeDispatchReport, + stage_timings: CudaEncodeStageTimings, +} + +#[cfg(feature = "cuda-runtime")] +impl J2kEncodeStageAccelerator for CudaDeviceBufferEncodeAccelerator<'_> { + fn dispatch_report(&self) -> J2kEncodeDispatchReport { + self.dispatch + } + + fn encode_htj2k_tile( + &mut self, + job: J2kHtj2kTileEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + let Some(encoded) = cuda_encode_htj2k_device_tile_body( + &self.context, + &self.resources, + self.tile, + job, + true, + )? + else { + return Ok(None); + }; + self.dispatch.deinterleave = self + .dispatch + .deinterleave + .saturating_add(encoded.deinterleave_dispatches); + self.dispatch.forward_rct = self + .dispatch + .forward_rct + .saturating_add(encoded.forward_rct_dispatches); + self.dispatch.forward_ict = self + .dispatch + .forward_ict + .saturating_add(encoded.forward_ict_dispatches); + self.dispatch.forward_dwt53 = self + .dispatch + .forward_dwt53 + .saturating_add(encoded.forward_dwt53_dispatches); + self.dispatch.forward_dwt97 = self + .dispatch + .forward_dwt97 + .saturating_add(encoded.forward_dwt97_dispatches); + self.dispatch.quantize_subband = self + .dispatch + .quantize_subband + .saturating_add(encoded.quantize_dispatches); + self.dispatch.ht_code_block = self + .dispatch + .ht_code_block + .saturating_add(encoded.ht_code_block_dispatches); + self.dispatch.packetization = self + .dispatch + .packetization + .saturating_add(encoded.packetization_dispatches); + self.stage_timings = self.stage_timings.saturating_add(encoded.timings); + Ok(Some(encoded.tile_data)) + } +} + +/// CUDA implementation of selected JPEG 2000 encode stages. +#[derive(Debug, Default, Clone)] +#[allow(clippy::struct_excessive_bools)] +pub struct CudaEncodeStageAccelerator { + #[cfg(feature = "cuda-runtime")] + context: Option, + #[cfg(feature = "cuda-runtime")] + encode_resources: Option>, + #[cfg_attr(not(feature = "cuda-runtime"), allow(dead_code))] + collect_profile: bool, + deinterleave_attempts: usize, + forward_rct_attempts: usize, + forward_ict_attempts: usize, + forward_dwt53_attempts: usize, + forward_dwt97_attempts: usize, + htj2k_tile_attempts: usize, + quantize_subband_attempts: usize, + ht_subband_attempts: usize, + tier1_code_block_attempts: usize, + ht_code_block_attempts: usize, + packetization_attempts: usize, + prefer_cpu_forward_rct: bool, + prefer_cpu_ht_subband: bool, + prefer_cpu_quantize_subband: bool, + prefer_cpu_packetization: bool, + deinterleave_dispatches: usize, + forward_rct_dispatches: usize, + forward_ict_dispatches: usize, + forward_dwt53_dispatches: usize, + forward_dwt97_dispatches: usize, + #[cfg_attr(not(feature = "cuda-runtime"), allow(dead_code))] + htj2k_tile_dispatches: usize, + quantize_subband_dispatches: usize, + #[cfg_attr(not(feature = "cuda-runtime"), allow(dead_code))] + ht_subband_dispatches: usize, + tier1_code_block_dispatches: usize, + ht_code_block_dispatches: usize, + packetization_dispatches: usize, + deinterleave_us: u128, + mct_us: u128, + dwt_us: u128, + quantize_us: u128, + ht_encode_us: u128, + packetize_us: u128, +} + +impl CudaEncodeStageAccelerator { + /// Create an encode-stage accelerator with optional CUDA stage timing collection. + #[must_use] + pub fn with_profile_collection(collect_profile: bool) -> Self { + Self { + collect_profile, + ..Self::default() + } + } + + /// Create the measured Auto route for host-output HTJ2K encode. + /// + /// CUDA keeps the DWT and HT code-block stages, while forward RCT and + /// Tier-2 packetization stay on the CPU for the current host-pixel path. + #[must_use] + pub fn for_auto_host_output() -> Self { + Self::default() + .prefer_cpu_forward_rct(true) + .prefer_cpu_packetization(true) + } + + /// Prefer scalar CPU forward RCT while keeping later CUDA stages enabled. + #[must_use] + pub fn prefer_cpu_forward_rct(mut self, prefer_cpu_forward_rct: bool) -> Self { + self.prefer_cpu_forward_rct = prefer_cpu_forward_rct; + self + } + + /// Prefer scalar CPU Tier-2 packetization while keeping CUDA Tier-1/HT block coding enabled. + /// + /// This is useful for batches of many small tiles where launching a CUDA + /// packetization kernel and copying several tiny descriptor buffers per tile + /// costs more than forming the packet body on the host. + #[must_use] + pub fn prefer_cpu_packetization(mut self, prefer_cpu_packetization: bool) -> Self { + self.prefer_cpu_packetization = prefer_cpu_packetization; + self + } + + /// Prefer host sub-band quantization while keeping batched CUDA HT code-block encode enabled. + /// + /// This avoids launching one CUDA quantize/subband path for every prepared + /// subband in multi-resolution precomputed transcode outputs, where the + /// many tiny launches cost more than CPU quantization. + #[must_use] + pub fn prefer_cpu_ht_subband(mut self, prefer_cpu_ht_subband: bool) -> Self { + self.prefer_cpu_ht_subband = prefer_cpu_ht_subband; + self + } + + /// Prefer host sub-band quantization while keeping CUDA HT code-block encode enabled. + /// + /// Multi-resolution transcode workloads can contain thousands of small + /// subbands; for those, CPU quantization plus one batched HT code-block + /// encode per tile is currently faster than launching CUDA quantization for + /// every subband. + #[must_use] + pub fn prefer_cpu_quantize_subband(mut self, prefer_cpu_quantize_subband: bool) -> Self { + self.prefer_cpu_quantize_subband = prefer_cpu_quantize_subband; + self + } + + /// Return cumulative CUDA encode stage timings collected by this accelerator. + #[must_use] + pub const fn collected_stage_timings(&self) -> CudaEncodeStageTimings { + CudaEncodeStageTimings { + deinterleave_us: self.deinterleave_us, + mct_us: self.mct_us, + dwt_us: self.dwt_us, + quantize_us: self.quantize_us, + ht_encode_us: self.ht_encode_us, + packetize_us: self.packetize_us, + } + } + + /// Clear cumulative CUDA encode stage timings without changing dispatch counters. + pub fn reset_collected_stage_timings(&mut self) { + self.deinterleave_us = 0; + self.mct_us = 0; + self.dwt_us = 0; + self.quantize_us = 0; + self.ht_encode_us = 0; + self.packetize_us = 0; + } + + #[cfg(feature = "cuda-runtime")] + fn cuda_context(&mut self) -> core::result::Result, &'static str> { + if self.context.is_none() { + match CudaContext::system_default() { + Ok(context) => self.context = Some(context), + Err(_) if cuda_runtime_required() => return Err("CUDA encode stage unavailable"), + Err(_) => return Ok(None), + } + } + Ok(self.context.clone()) + } + + #[cfg(feature = "cuda-runtime")] + fn cuda_encode_resources( + &mut self, + context: &CudaContext, + ) -> core::result::Result, &'static str> { + if self.encode_resources.is_none() { + let resources = context + .upload_htj2k_encode_resources(cuda_htj2k_encode_tables()) + .map_err(|_| "CUDA HTJ2K encode resource upload failed")?; + self.encode_resources = Some(Arc::new(resources)); + } + self.encode_resources + .clone() + .ok_or("CUDA HTJ2K encode resources unavailable") + } + + fn encode_profile_report( + &self, + encoded: &j2k::EncodedJ2k, + input_bytes: usize, + total_us: u128, + ) -> profile::CudaHtj2kEncodeProfileReport { + profile::CudaHtj2kEncodeProfileReport { + deinterleave_us: self.deinterleave_us, + mct_us: self.mct_us, + dwt_us: self.dwt_us, + quantize_us: self.quantize_us, + ht_encode_us: self.ht_encode_us, + packetize_us: self.packetize_us, + total_us, + input_bytes, + codestream_bytes: encoded.codestream.len(), + block_count: self.ht_code_block_attempts, + dispatch_count: self.dispatch_report().total(), + backend: encoded.backend, + } + } + + /// Number of deinterleave attempts observed. + pub fn deinterleave_attempts(&self) -> usize { + self.deinterleave_attempts + } + + /// Number of forward RCT attempts observed. + pub fn forward_rct_attempts(&self) -> usize { + self.forward_rct_attempts + } + + /// Number of forward ICT attempts observed. + pub fn forward_ict_attempts(&self) -> usize { + self.forward_ict_attempts + } + + /// Number of forward 5/3 DWT attempts observed. + pub fn forward_dwt53_attempts(&self) -> usize { + self.forward_dwt53_attempts + } + + /// Number of forward 9/7 DWT attempts observed. + pub fn forward_dwt97_attempts(&self) -> usize { + self.forward_dwt97_attempts + } + + /// Number of sub-band quantization attempts observed. + pub fn quantize_subband_attempts(&self) -> usize { + self.quantize_subband_attempts + } + + /// Number of classic Tier-1 code-block attempts observed. + pub fn tier1_code_block_attempts(&self) -> usize { + self.tier1_code_block_attempts + } + + /// Number of HT code-block attempts observed. + pub fn ht_code_block_attempts(&self) -> usize { + self.ht_code_block_attempts + } + + /// Number of packetization attempts observed. + pub fn packetization_attempts(&self) -> usize { + self.packetization_attempts + } + + /// Number of deinterleave CUDA dispatches. + pub fn deinterleave_dispatches(&self) -> usize { + self.deinterleave_dispatches + } + + /// Number of forward RCT CUDA dispatches. + pub fn forward_rct_dispatches(&self) -> usize { + self.forward_rct_dispatches + } + + /// Number of forward ICT CUDA dispatches. + pub fn forward_ict_dispatches(&self) -> usize { + self.forward_ict_dispatches + } + + /// Number of forward 5/3 DWT CUDA dispatches. + pub fn forward_dwt53_dispatches(&self) -> usize { + self.forward_dwt53_dispatches + } + + /// Number of forward 9/7 DWT CUDA dispatches. + pub fn forward_dwt97_dispatches(&self) -> usize { + self.forward_dwt97_dispatches + } + + /// Number of sub-band quantization CUDA dispatches. + pub fn quantize_subband_dispatches(&self) -> usize { + self.quantize_subband_dispatches + } + + /// Number of classic Tier-1 CUDA dispatches. + pub fn tier1_code_block_dispatches(&self) -> usize { + self.tier1_code_block_dispatches + } + + /// Number of HT code-block CUDA dispatches. + pub fn ht_code_block_dispatches(&self) -> usize { + self.ht_code_block_dispatches + } + + /// Number of packetization CUDA dispatches. + pub fn packetization_dispatches(&self) -> usize { + self.packetization_dispatches + } +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_runtime_required() -> bool { + std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_some() +} + +#[cfg(feature = "cuda-runtime")] +fn time_cuda_stage( + name: &'static str, + context: &CudaContext, + collect_profile: bool, + work: impl FnOnce() -> core::result::Result, +) -> core::result::Result<(T, u128), CudaError> { + if collect_profile { + context.time_default_stream_named_us(name, work) + } else { + context + .with_nvtx_range(name, work) + .map(|output| (output, 0)) + } +} + +/// Cumulative CUDA encode-stage timings collected by `CudaEncodeStageAccelerator`. +#[allow(clippy::struct_field_names)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct CudaEncodeStageTimings { + /// Pixel deinterleave and level-shift CUDA stage time. + pub deinterleave_us: u128, + /// Forward MCT CUDA stage time. + pub mct_us: u128, + /// Forward DWT CUDA stage time. + pub dwt_us: u128, + /// Quantization CUDA stage time. + pub quantize_us: u128, + /// HT code-block encode CUDA stage time. + pub ht_encode_us: u128, + /// HTJ2K packetization CUDA stage time. + pub packetize_us: u128, +} + +impl CudaEncodeStageTimings { + /// Return field-wise saturating timing sums. + #[must_use] + pub const fn saturating_add(self, other: Self) -> Self { + Self { + deinterleave_us: self.deinterleave_us.saturating_add(other.deinterleave_us), + mct_us: self.mct_us.saturating_add(other.mct_us), + dwt_us: self.dwt_us.saturating_add(other.dwt_us), + quantize_us: self.quantize_us.saturating_add(other.quantize_us), + ht_encode_us: self.ht_encode_us.saturating_add(other.ht_encode_us), + packetize_us: self.packetize_us.saturating_add(other.packetize_us), + } + } + + /// Total collected CUDA encode-stage time. + #[must_use] + pub const fn total_us(self) -> u128 { + self.deinterleave_us + .saturating_add(self.mct_us) + .saturating_add(self.dwt_us) + .saturating_add(self.quantize_us) + .saturating_add(self.ht_encode_us) + .saturating_add(self.packetize_us) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CudaHtj2kPacketizationPlan { + payload: Vec, + packets: Vec, + subbands: Vec, + blocks: Vec, + tag_states: Vec, + tag_nodes: Vec, +} + +struct CudaHtj2kPacketizationPlanSink<'a> { + payload: &'a mut Vec, + packets: &'a mut Vec, + subbands: &'a mut Vec, + blocks: &'a mut Vec, + tag_states: &'a mut Vec, + tag_nodes: &'a mut Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct CudaHtj2kPacketizationPlanPacket { + block_start: u32, + block_count: u32, + subband_start: u32, + subband_count: u32, + output_capacity: u32, + layer: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct CudaHtj2kPacketizationPlanSubband { + block_start: u32, + block_count: u32, + num_cbs_x: u32, + num_cbs_y: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct CudaHtj2kPacketizationPlanBlock { + data_offset: u32, + data_len: u32, + cleanup_length: u32, + refinement_length: u32, + num_coding_passes: u32, + num_zero_bitplanes: u32, + l_block: u32, + previously_included: u32, + inclusion_layer: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct CudaHtj2kPacketizationPlanSubbandTagState { + inclusion_node_start: u32, + zero_bitplane_node_start: u32, + node_count: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct CudaHtj2kPacketizationPlanTagNodeState { + current: u32, + known: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CudaHtj2kPacketizationTagTreeState { + values: Vec, + current: Vec, + known: Vec, + widths: Vec, + heights: Vec, + offsets: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct CudaHtj2kPacketizationBlockState { + previously_included: bool, + l_block: u32, + inclusion_layer: u32, + first_inclusion_zero_bitplanes: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CudaHtj2kPacketizationSubbandState { + num_cbs_x: u32, + num_cbs_y: u32, + inclusion_tree: CudaHtj2kPacketizationTagTreeState, + zero_bitplane_tree: CudaHtj2kPacketizationTagTreeState, + blocks: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct CudaHtj2kPacketizationState { + subbands: Vec, +} + +fn flatten_cuda_htj2k_packetization_job( + job: J2kPacketizationEncodeJob<'_>, +) -> core::result::Result { + if job.resolution_count as usize != job.resolutions.len() { + return Err("CUDA HTJ2K packetization resolution count mismatch"); + } + + let mut payload = Vec::new(); + let mut packets = Vec::new(); + let mut subbands = Vec::new(); + let mut blocks = Vec::new(); + let mut tag_states = Vec::new(); + let mut tag_nodes = Vec::new(); + + { + let mut sink = CudaHtj2kPacketizationPlanSink { + payload: &mut payload, + packets: &mut packets, + subbands: &mut subbands, + blocks: &mut blocks, + tag_states: &mut tag_states, + tag_nodes: &mut tag_nodes, + }; + if job.packet_descriptors.is_empty() { + if job.num_layers != 1 { + return Err( + "CUDA HTJ2K packetization requires explicit descriptors for multiple layers", + ); + } + for packet_index in 0..job.resolutions.len() { + flatten_cuda_htj2k_packet( + job.resolutions + .get(packet_index) + .ok_or("CUDA HTJ2K packet descriptor index out of range")?, + &mut sink, + )?; + } + } else { + let state_count = job + .packet_descriptors + .iter() + .map(|descriptor| descriptor.state_index as usize) + .max() + .map_or(0usize, |max_state| max_state + 1); + let mut states: Vec> = + core::iter::repeat_with(|| None).take(state_count).collect(); + for descriptor in job.packet_descriptors { + if descriptor.layer >= job.num_layers { + return Err("CUDA HTJ2K packetization descriptor layer exceeds layer count"); + } + let resolution = job + .resolutions + .get(descriptor.packet_index as usize) + .ok_or("CUDA HTJ2K packet descriptor index out of range")?; + let state = states + .get_mut(descriptor.state_index as usize) + .ok_or("CUDA HTJ2K packet descriptor state index out of range")?; + if let Some(existing) = state { + validate_cuda_htj2k_packetization_state_layout(existing, resolution)?; + } else { + *state = Some(seed_cuda_htj2k_packetization_state(resolution)?); + } + let state = state + .as_mut() + .ok_or("CUDA HTJ2K packetization state initialization failed")?; + record_cuda_htj2k_packetization_first_inclusion_layers( + state, + resolution, + descriptor.layer, + )?; + } + for state in states.iter_mut().flatten() { + finalize_cuda_htj2k_packetization_tag_trees(state); + } + for descriptor in job.packet_descriptors { + if descriptor.layer >= job.num_layers { + return Err("CUDA HTJ2K packetization descriptor layer exceeds layer count"); + } + let resolution = job + .resolutions + .get(descriptor.packet_index as usize) + .ok_or("CUDA HTJ2K packet descriptor index out of range")?; + let state = states + .get_mut(descriptor.state_index as usize) + .ok_or("CUDA HTJ2K packet descriptor state index out of range")?; + if let Some(existing) = state { + validate_cuda_htj2k_packetization_state_layout(existing, resolution)?; + } else { + *state = Some(seed_cuda_htj2k_packetization_state(resolution)?); + } + let state = state + .as_mut() + .ok_or("CUDA HTJ2K packetization state initialization failed")?; + flatten_cuda_htj2k_packet_with_state( + resolution, + descriptor.layer, + state, + &mut sink, + )?; + } + } + } + + if job.code_block_count as usize != blocks.len() { + return Err("CUDA HTJ2K packetization code-block count mismatch"); + } + + Ok(CudaHtj2kPacketizationPlan { + payload, + packets, + subbands, + blocks, + tag_states, + tag_nodes, + }) +} + +fn seed_cuda_htj2k_packetization_state( + resolution: &J2kPacketizationResolution<'_>, +) -> core::result::Result { + let mut subbands = Vec::with_capacity(resolution.subbands.len()); + for subband in &resolution.subbands { + let block_count = u32::try_from(subband.code_blocks.len()) + .map_err(|_| "CUDA HTJ2K packetization block count exceeds u32")?; + if subband.num_cbs_x == 0 + || subband.num_cbs_y == 0 + || subband.num_cbs_x.saturating_mul(subband.num_cbs_y) != block_count + { + return Err("CUDA HTJ2K packetization subband code-block layout mismatch"); + } + let mut inclusion_tree = + CudaHtj2kPacketizationTagTreeState::new(subband.num_cbs_x, subband.num_cbs_y)?; + let zero_bitplane_tree = + CudaHtj2kPacketizationTagTreeState::new(subband.num_cbs_x, subband.num_cbs_y)?; + for idx in 0..subband.code_blocks.len() { + let (x, y) = cuda_htj2k_packetization_block_xy(idx, subband.num_cbs_x)?; + inclusion_tree.set_leaf_value(x, y, CUDA_HTJ2K_PACKET_TAG_INF); + } + subbands.push(CudaHtj2kPacketizationSubbandState { + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + inclusion_tree, + zero_bitplane_tree, + blocks: subband + .code_blocks + .iter() + .map(|block| CudaHtj2kPacketizationBlockState { + previously_included: block.previously_included, + l_block: block.l_block, + inclusion_layer: CUDA_HTJ2K_PACKET_TAG_INF, + first_inclusion_zero_bitplanes: 0, + }) + .collect(), + }); + } + Ok(CudaHtj2kPacketizationState { subbands }) +} + +fn validate_cuda_htj2k_packetization_state_layout( + state: &CudaHtj2kPacketizationState, + resolution: &J2kPacketizationResolution<'_>, +) -> core::result::Result<(), &'static str> { + if state.subbands.len() != resolution.subbands.len() { + return Err("CUDA HTJ2K packetization state layout mismatch"); + } + for (state_subband, packet_subband) in state.subbands.iter().zip(&resolution.subbands) { + if state_subband.num_cbs_x != packet_subband.num_cbs_x + || state_subband.num_cbs_y != packet_subband.num_cbs_y + || state_subband.blocks.len() != packet_subband.code_blocks.len() + { + return Err("CUDA HTJ2K packetization state layout mismatch"); + } + } + Ok(()) +} + +const CUDA_HTJ2K_PACKET_TAG_INF: u32 = 0x7FFF_FFFF; +const CUDA_HTJ2K_PACKET_MAX_TAG_NODES: usize = 2048; +const CUDA_HTJ2K_PACKET_MAX_TAG_LEVELS: usize = 16; + +fn cuda_htj2k_packetization_block_xy( + index: usize, + num_cbs_x: u32, +) -> core::result::Result<(u32, u32), &'static str> { + let index = + u32::try_from(index).map_err(|_| "CUDA HTJ2K packetization block count exceeds u32")?; + Ok((index % num_cbs_x, index / num_cbs_x)) +} + +impl CudaHtj2kPacketizationTagTreeState { + fn new(width: u32, height: u32) -> core::result::Result { + if width == 0 || height == 0 { + return Err("CUDA HTJ2K packetization subband code-block layout mismatch"); + } + + let mut widths = Vec::new(); + let mut heights = Vec::new(); + let mut offsets = Vec::new(); + let mut total_nodes = 0usize; + let mut w = width; + let mut h = height; + loop { + if widths.len() >= CUDA_HTJ2K_PACKET_MAX_TAG_LEVELS { + return Err("CUDA HTJ2K packetization tag-tree exceeds kernel bounds"); + } + let nodes = (w as usize) + .checked_mul(h as usize) + .ok_or("CUDA HTJ2K packetization tag-tree exceeds kernel bounds")?; + let next_total = total_nodes + .checked_add(nodes) + .ok_or("CUDA HTJ2K packetization tag-tree exceeds kernel bounds")?; + if next_total > CUDA_HTJ2K_PACKET_MAX_TAG_NODES { + return Err("CUDA HTJ2K packetization tag-tree exceeds kernel bounds"); + } + offsets.push(total_nodes); + widths.push(w); + heights.push(h); + total_nodes = next_total; + if w <= 1 && h <= 1 { + break; + } + w = w.div_ceil(2); + h = h.div_ceil(2); + } + + Ok(Self { + values: vec![0; total_nodes], + current: vec![0; total_nodes], + known: vec![0; total_nodes], + widths, + heights, + offsets, + }) + } + + fn set_leaf_value(&mut self, x: u32, y: u32, value: u32) { + let idx = self.offsets[0] + (y * self.widths[0] + x) as usize; + self.values[idx] = value; + } + + #[allow(clippy::similar_names)] + fn propagate(&mut self) { + for level in 1..self.widths.len() { + let prev_w = self.widths[level - 1]; + let prev_h = self.heights[level - 1]; + let curr_w = self.widths[level]; + let curr_h = self.heights[level]; + for cy in 0..curr_h { + for cx in 0..curr_w { + let child_x_start = cx * 2; + let child_y_start = cy * 2; + let child_x_end = ((cx + 1) * 2).min(prev_w); + let child_y_end = ((cy + 1) * 2).min(prev_h); + let mut min_value = u32::MAX; + for child_y in child_y_start..child_y_end { + for child_x in child_x_start..child_x_end { + let child_idx = + self.offsets[level - 1] + (child_y * prev_w + child_x) as usize; + min_value = min_value.min(self.values[child_idx]); + } + } + let parent_idx = self.offsets[level] + (cy * curr_w + cx) as usize; + self.values[parent_idx] = min_value; + } + } + } + } + + fn encode_state_only(&mut self, x: u32, y: u32, max_value: u32) { + let mut path = Vec::with_capacity(self.widths.len()); + let mut cx = x; + let mut cy = y; + for level in 0..self.widths.len() { + path.push(self.offsets[level] + (cy * self.widths[level] + cx) as usize); + cx /= 2; + cy /= 2; + } + + for node_idx in path.into_iter().rev() { + if self.known[node_idx] == 0 { + let target = self.values[node_idx].min(max_value); + if self.values[node_idx] < max_value { + self.known[node_idx] = 1; + } + self.current[node_idx] = target; + } + } + } + + fn append_snapshot( + &self, + out: &mut Vec, + ) -> core::result::Result { + let start = u32::try_from(out.len()) + .map_err(|_| "CUDA HTJ2K packetization tag-state exceeds u32")?; + out.extend( + self.current + .iter() + .copied() + .zip(self.known.iter().copied()) + .map(|(current, known)| CudaHtj2kPacketizationPlanTagNodeState { current, known }), + ); + Ok(start) + } + + fn node_count(&self) -> u32 { + u32::try_from(self.current.len()).expect("tag tree node count was bounded at construction") + } +} + +fn record_cuda_htj2k_packetization_first_inclusion_layers( + state: &mut CudaHtj2kPacketizationState, + resolution: &J2kPacketizationResolution<'_>, + layer: u8, +) -> core::result::Result<(), &'static str> { + validate_cuda_htj2k_packetization_state_layout(state, resolution)?; + for (state_subband, packet_subband) in state.subbands.iter_mut().zip(&resolution.subbands) { + for (idx, (state_block, packet_block)) in state_subband + .blocks + .iter_mut() + .zip(&packet_subband.code_blocks) + .enumerate() + { + if packet_block.num_coding_passes == 0 { + continue; + } + let layer = u32::from(layer); + if layer < state_block.inclusion_layer { + state_block.inclusion_layer = layer; + state_block.first_inclusion_zero_bitplanes = + u32::from(packet_block.num_zero_bitplanes); + let (x, y) = cuda_htj2k_packetization_block_xy(idx, state_subband.num_cbs_x)?; + state_subband.inclusion_tree.set_leaf_value(x, y, layer); + state_subband.zero_bitplane_tree.set_leaf_value( + x, + y, + state_block.first_inclusion_zero_bitplanes, + ); + } + } + } + Ok(()) +} + +fn finalize_cuda_htj2k_packetization_tag_trees(state: &mut CudaHtj2kPacketizationState) { + for subband in &mut state.subbands { + subband.inclusion_tree.propagate(); + subband.zero_bitplane_tree.propagate(); + } +} + +fn append_cuda_htj2k_packetization_tag_state( + state_subband: Option<&CudaHtj2kPacketizationSubbandState>, + num_cbs_x: u32, + num_cbs_y: u32, + tag_states: &mut Vec, + tag_nodes: &mut Vec, +) -> core::result::Result<(), &'static str> { + let (inclusion_node_start, zero_bitplane_node_start, node_count) = + if let Some(state_subband) = state_subband { + let inclusion_start = state_subband.inclusion_tree.append_snapshot(tag_nodes)?; + let zero_bitplane_start = state_subband + .zero_bitplane_tree + .append_snapshot(tag_nodes)?; + ( + inclusion_start, + zero_bitplane_start, + state_subband.inclusion_tree.node_count(), + ) + } else { + let zero_tree = CudaHtj2kPacketizationTagTreeState::new(num_cbs_x, num_cbs_y)?; + let inclusion_start = zero_tree.append_snapshot(tag_nodes)?; + let zero_bitplane_start = zero_tree.append_snapshot(tag_nodes)?; + (inclusion_start, zero_bitplane_start, zero_tree.node_count()) + }; + tag_states.push(CudaHtj2kPacketizationPlanSubbandTagState { + inclusion_node_start, + zero_bitplane_node_start, + node_count, + }); + Ok(()) +} + +fn update_cuda_htj2k_packetization_state_after_block( + state: &mut CudaHtj2kPacketizationState, + subband_index: usize, + block_index: usize, + layer: u8, + code_block: &J2kPacketizationCodeBlock<'_>, + l_block: u32, +) -> core::result::Result<(), &'static str> { + let state_subband = state + .subbands + .get_mut(subband_index) + .ok_or("CUDA HTJ2K packetization state layout mismatch")?; + let (x, y) = cuda_htj2k_packetization_block_xy(block_index, state_subband.num_cbs_x)?; + let previously_included = state_subband + .blocks + .get(block_index) + .ok_or("CUDA HTJ2K packetization state layout mismatch")? + .previously_included; + + if !previously_included { + state_subband + .inclusion_tree + .encode_state_only(x, y, u32::from(layer) + 1); + if code_block.num_coding_passes == 0 { + return Ok(()); + } + state_subband.zero_bitplane_tree.encode_state_only( + x, + y, + u32::from(code_block.num_zero_bitplanes) + 1, + ); + } + + if code_block.num_coding_passes > 0 { + let state_block = state_subband + .blocks + .get_mut(block_index) + .ok_or("CUDA HTJ2K packetization state layout mismatch")?; + let (cleanup_length, refinement_length) = cuda_ht_segment_lengths(code_block)?; + state_block.l_block = updated_ht_l_block( + l_block, + code_block.num_coding_passes, + cleanup_length, + refinement_length, + )?; + state_block.previously_included = true; + } + Ok(()) +} + +fn flatten_cuda_htj2k_packet( + resolution: &J2kPacketizationResolution<'_>, + sink: &mut CudaHtj2kPacketizationPlanSink<'_>, +) -> core::result::Result<(), &'static str> { + flatten_cuda_htj2k_packet_inner(resolution, 0, None, sink) +} + +fn flatten_cuda_htj2k_packet_with_state( + resolution: &J2kPacketizationResolution<'_>, + layer: u8, + state: &mut CudaHtj2kPacketizationState, + sink: &mut CudaHtj2kPacketizationPlanSink<'_>, +) -> core::result::Result<(), &'static str> { + flatten_cuda_htj2k_packet_inner(resolution, layer, Some(state), sink) +} + +fn flatten_cuda_htj2k_packet_inner( + resolution: &J2kPacketizationResolution<'_>, + layer: u8, + mut state: Option<&mut CudaHtj2kPacketizationState>, + sink: &mut CudaHtj2kPacketizationPlanSink<'_>, +) -> core::result::Result<(), &'static str> { + let block_start = u32::try_from(sink.blocks.len()) + .map_err(|_| "CUDA HTJ2K packetization block count exceeds u32")?; + let subband_start = u32::try_from(sink.subbands.len()) + .map_err(|_| "CUDA HTJ2K packetization subband count exceeds u32")?; + let mut body_len = 0usize; + let mut block_count = 0usize; + let packet_has_data = resolution.subbands.iter().any(|subband| { + subband + .code_blocks + .iter() + .any(|block| block.num_coding_passes > 0) + }); + + for (subband_index, subband) in resolution.subbands.iter().enumerate() { + let subband_code_blocks = u32::try_from(subband.code_blocks.len()) + .map_err(|_| "CUDA HTJ2K packetization block count exceeds u32")?; + if subband.num_cbs_x == 0 + || subband.num_cbs_y == 0 + || subband.num_cbs_x.saturating_mul(subband.num_cbs_y) != subband_code_blocks + { + return Err("CUDA HTJ2K packetization subband code-block layout mismatch"); + } + + let subband_block_start = u32::try_from(sink.blocks.len()) + .map_err(|_| "CUDA HTJ2K packetization block count exceeds u32")?; + let state_subband = state + .as_deref() + .and_then(|state| state.subbands.get(subband_index)); + append_cuda_htj2k_packetization_tag_state( + state_subband, + subband.num_cbs_x, + subband.num_cbs_y, + sink.tag_states, + sink.tag_nodes, + )?; + for (block_index, code_block) in subband.code_blocks.iter().enumerate() { + if code_block.block_coding_mode != J2kPacketizationBlockCodingMode::HighThroughput { + return Err("CUDA packetization only supports HTJ2K block-coded packets"); + } + if code_block.num_coding_passes > 164 { + return Err("CUDA HTJ2K packetization coding pass count exceeds JPEG 2000 bounds"); + } + let (previously_included, l_block, inclusion_layer, zero_bitplanes) = + if let Some(state) = state.as_deref() { + let state_block = state + .subbands + .get(subband_index) + .and_then(|state_subband| state_subband.blocks.get(block_index)) + .ok_or("CUDA HTJ2K packetization state layout mismatch")?; + ( + state_block.previously_included, + state_block.l_block, + state_block.inclusion_layer, + state_block.first_inclusion_zero_bitplanes, + ) + } else { + ( + code_block.previously_included, + code_block.l_block, + if code_block.num_coding_passes > 0 { + 0 + } else { + CUDA_HTJ2K_PACKET_TAG_INF + }, + u32::from(code_block.num_zero_bitplanes), + ) + }; + if code_block.num_coding_passes > 0 + && !previously_included + && inclusion_layer != u32::from(layer) + { + return Err( + "CUDA HTJ2K packetization descriptor order does not match first inclusion layer", + ); + } + if state.is_none() && previously_included { + return Err("CUDA HTJ2K packetization requires first-inclusion packets"); + } + if code_block.num_coding_passes == 0 && !code_block.data.is_empty() { + return Err("CUDA HTJ2K packetization empty contributions must not carry payload"); + } + if zero_bitplanes > 31 || l_block > 31 { + return Err("CUDA HTJ2K packetization header fields exceed kernel bounds"); + } + + let data_offset = u32::try_from(sink.payload.len()) + .map_err(|_| "CUDA HTJ2K packetization payload exceeds u32")?; + let data_len = if code_block.num_coding_passes == 0 { + 0 + } else { + u32::try_from(code_block.data.len()) + .map_err(|_| "CUDA HTJ2K packetization code-block payload exceeds u32")? + }; + let (cleanup_length, refinement_length) = cuda_ht_segment_lengths(code_block)?; + if code_block.num_coding_passes > 0 { + sink.payload.extend_from_slice(code_block.data); + body_len = body_len + .checked_add(code_block.data.len()) + .ok_or("CUDA HTJ2K packetization body length overflow")?; + } + sink.blocks.push(CudaHtj2kPacketizationPlanBlock { + data_offset, + data_len, + cleanup_length, + refinement_length, + num_coding_passes: u32::from(code_block.num_coding_passes), + num_zero_bitplanes: zero_bitplanes, + l_block, + previously_included: u32::from(previously_included), + inclusion_layer, + }); + if packet_has_data { + if let Some(state) = state.as_deref_mut() { + update_cuda_htj2k_packetization_state_after_block( + state, + subband_index, + block_index, + layer, + code_block, + l_block, + )?; + } + } + block_count = block_count + .checked_add(1) + .ok_or("CUDA HTJ2K packetization block count overflow")?; + } + sink.subbands.push(CudaHtj2kPacketizationPlanSubband { + block_start: subband_block_start, + block_count: subband_code_blocks, + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + }); + } + + let header_capacity = 256usize + .checked_add( + block_count + .checked_mul(64) + .ok_or("CUDA HTJ2K packetization capacity overflow")?, + ) + .ok_or("CUDA HTJ2K packetization capacity overflow")?; + let output_capacity = body_len + .checked_add(header_capacity) + .ok_or("CUDA HTJ2K packetization capacity overflow")?; + sink.packets.push(CudaHtj2kPacketizationPlanPacket { + block_start, + block_count: u32::try_from(block_count) + .map_err(|_| "CUDA HTJ2K packetization block count exceeds u32")?, + subband_start, + subband_count: u32::try_from(resolution.subbands.len()) + .map_err(|_| "CUDA HTJ2K packetization subband count exceeds u32")?, + output_capacity: u32::try_from(output_capacity) + .map_err(|_| "CUDA HTJ2K packetization packet capacity exceeds u32")?, + layer: u32::from(layer), + }); + Ok(()) +} + +fn updated_ht_l_block( + mut l_block: u32, + num_coding_passes: u8, + cleanup_length: u32, + refinement_length: u32, +) -> core::result::Result { + let mut num_bits = packet_math::bits_for_ht_cleanup_length(l_block, num_coding_passes); + let refinement_extra_bits = u32::from(num_coding_passes > 2); + while !packet_math::value_fits_in_bits(cleanup_length, num_bits) + || (num_coding_passes > 1 + && !packet_math::value_fits_in_bits(refinement_length, l_block + refinement_extra_bits)) + { + l_block = l_block + .checked_add(1) + .ok_or("CUDA HTJ2K packetization L-block overflow")?; + num_bits = num_bits + .checked_add(1) + .ok_or("CUDA HTJ2K packetization L-block overflow")?; + } + Ok(l_block) +} + +fn cuda_ht_segment_lengths( + code_block: &J2kPacketizationCodeBlock<'_>, +) -> core::result::Result<(u32, u32), &'static str> { + packet_math::ht_segment_lengths( + code_block.num_coding_passes, + code_block.data.len(), + code_block.ht_cleanup_length, + code_block.ht_refinement_length, + ) +} + +impl J2kEncodeStageAccelerator for CudaEncodeStageAccelerator { + fn dispatch_report(&self) -> J2kEncodeDispatchReport { + J2kEncodeDispatchReport { + deinterleave: self.deinterleave_dispatches, + forward_rct: self.forward_rct_dispatches, + forward_ict: self.forward_ict_dispatches, + forward_dwt53: self.forward_dwt53_dispatches, + forward_dwt97: self.forward_dwt97_dispatches, + quantize_subband: self.quantize_subband_dispatches, + tier1_code_block: self.tier1_code_block_dispatches, + ht_code_block: self.ht_code_block_dispatches, + packetization: self.packetization_dispatches, + } + } + + fn encode_deinterleave( + &mut self, + job: J2kDeinterleaveToF32Job<'_>, + ) -> core::result::Result>>, &'static str> { + self.deinterleave_attempts = self.deinterleave_attempts.saturating_add(1); + #[cfg(feature = "cuda-runtime")] + if let Some(context) = self.cuda_context()? { + let (output, elapsed_us) = time_cuda_stage( + "j2k.j2k.cuda.encode.deinterleave", + &context, + self.collect_profile, + || { + context.j2k_deinterleave_to_f32( + job.pixels, + job.num_pixels, + job.num_components, + job.bit_depth, + job.signed, + ) + }, + ) + .map_err(|_| "CUDA deinterleave encode kernel failed")?; + let dispatches = output.execution().kernel_dispatches(); + self.deinterleave_dispatches = self.deinterleave_dispatches.saturating_add(dispatches); + self.deinterleave_us = self.deinterleave_us.saturating_add(elapsed_us); + if j2k_profile::gpu_route_profile_enabled() { + let pixels_s = job.num_pixels.to_string(); + let components_s = job.num_components.to_string(); + let dispatches_s = dispatches.to_string(); + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_deinterleave"), + ("decision", "cuda_dispatch"), + ("pixels", pixels_s.as_str()), + ("components", components_s.as_str()), + ("dispatches", dispatches_s.as_str()), + ], + ); + } + return Ok(Some(output.into_components())); + } + #[cfg(not(feature = "cuda-runtime"))] + let _ = job; + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_deinterleave"), + ("decision", "cpu_fallback"), + ("reason", "cuda_unavailable"), + ], + ); + } + Ok(None) + } + + fn encode_forward_rct( + &mut self, + job: J2kForwardRctJob<'_>, + ) -> core::result::Result { + self.forward_rct_attempts = self.forward_rct_attempts.saturating_add(1); + if self.prefer_cpu_forward_rct { + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_forward_rct"), + ("decision", "cpu_fallback"), + ("reason", "prefer_cpu_forward_rct"), + ], + ); + } + let _ = job; + return Ok(false); + } + #[cfg(feature = "cuda-runtime")] + if let Some(context) = self.cuda_context()? { + let (execution, elapsed_us) = time_cuda_stage( + "j2k.j2k.cuda.encode.rct", + &context, + self.collect_profile, + || context.j2k_forward_rct(job.plane0, job.plane1, job.plane2), + ) + .map_err(|_| "CUDA forward RCT encode kernel failed")?; + self.forward_rct_dispatches = self + .forward_rct_dispatches + .saturating_add(execution.kernel_dispatches()); + self.mct_us = self.mct_us.saturating_add(elapsed_us); + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_forward_rct"), + ("decision", "cuda_dispatch"), + ("dispatches", "1"), + ], + ); + } + return Ok(true); + } + #[cfg(not(feature = "cuda-runtime"))] + let _ = job; + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_forward_rct"), + ("decision", "cpu_fallback"), + ("reason", "cuda_unavailable"), + ], + ); + } + Ok(false) + } + + fn encode_forward_ict( + &mut self, + job: J2kForwardIctJob<'_>, + ) -> core::result::Result { + self.forward_ict_attempts = self.forward_ict_attempts.saturating_add(1); + #[cfg(feature = "cuda-runtime")] + if let Some(context) = self.cuda_context()? { + let (execution, elapsed_us) = time_cuda_stage( + "j2k.j2k.cuda.encode.ict", + &context, + self.collect_profile, + || context.j2k_forward_ict(job.plane0, job.plane1, job.plane2), + ) + .map_err(|_| "CUDA forward ICT encode kernel failed")?; + self.forward_ict_dispatches = self + .forward_ict_dispatches + .saturating_add(execution.kernel_dispatches()); + self.mct_us = self.mct_us.saturating_add(elapsed_us); + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_forward_ict"), + ("decision", "cuda_dispatch"), + ("dispatches", "1"), + ], + ); + } + return Ok(true); + } + #[cfg(not(feature = "cuda-runtime"))] + let _ = job; + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_forward_ict"), + ("decision", "cpu_fallback"), + ("reason", "cuda_unavailable"), + ], + ); + } + Ok(false) + } + + fn encode_forward_dwt53( + &mut self, + job: J2kForwardDwt53Job<'_>, + ) -> core::result::Result, &'static str> { + self.forward_dwt53_attempts = self.forward_dwt53_attempts.saturating_add(1); + if job.num_levels == 0 { + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_forward_dwt53"), + ("decision", "cpu_fallback"), + ("reason", "zero_levels"), + ], + ); + } + return Ok(None); + } + #[cfg(feature = "cuda-runtime")] + if let Some(context) = self.cuda_context()? { + let (output, elapsed_us) = time_cuda_stage( + "j2k.j2k.cuda.encode.dwt53", + &context, + self.collect_profile, + || context.j2k_forward_dwt53(job.samples, job.width, job.height, job.num_levels), + ) + .map_err(|_| "CUDA forward 5/3 DWT encode kernel failed")?; + let dispatches = output.execution().kernel_dispatches(); + self.forward_dwt53_dispatches = + self.forward_dwt53_dispatches.saturating_add(dispatches); + self.dwt_us = self.dwt_us.saturating_add(elapsed_us); + if j2k_profile::gpu_route_profile_enabled() { + let width_s = job.width.to_string(); + let height_s = job.height.to_string(); + let levels_s = job.num_levels.to_string(); + let dispatches_s = dispatches.to_string(); + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_forward_dwt53"), + ("decision", "cuda_dispatch"), + ("width", width_s.as_str()), + ("height", height_s.as_str()), + ("levels", levels_s.as_str()), + ("dispatches", dispatches_s.as_str()), + ], + ); + } + return Ok(Some(cuda_dwt53_output_to_j2k(&output)?)); + } + #[cfg(not(feature = "cuda-runtime"))] + let _ = job; + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_forward_dwt53"), + ("decision", "cpu_fallback"), + ("reason", "cuda_unavailable"), + ], + ); + } + Ok(None) + } + + fn encode_forward_dwt97( + &mut self, + job: J2kForwardDwt97Job<'_>, + ) -> core::result::Result, &'static str> { + self.forward_dwt97_attempts = self.forward_dwt97_attempts.saturating_add(1); + if job.num_levels == 0 { + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_forward_dwt97"), + ("decision", "cpu_fallback"), + ("reason", "zero_levels"), + ], + ); + } + return Ok(None); + } + #[cfg(feature = "cuda-runtime")] + if let Some(context) = self.cuda_context()? { + let (output, elapsed_us) = time_cuda_stage( + "j2k.j2k.cuda.encode.dwt97", + &context, + self.collect_profile, + || context.j2k_forward_dwt97(job.samples, job.width, job.height, job.num_levels), + ) + .map_err(|_| "CUDA forward 9/7 DWT encode kernel failed")?; + let dispatches = output.execution().kernel_dispatches(); + self.forward_dwt97_dispatches = + self.forward_dwt97_dispatches.saturating_add(dispatches); + self.dwt_us = self.dwt_us.saturating_add(elapsed_us); + if j2k_profile::gpu_route_profile_enabled() { + let width_s = job.width.to_string(); + let height_s = job.height.to_string(); + let levels_s = job.num_levels.to_string(); + let dispatches_s = dispatches.to_string(); + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_forward_dwt97"), + ("decision", "cuda_dispatch"), + ("width", width_s.as_str()), + ("height", height_s.as_str()), + ("levels", levels_s.as_str()), + ("dispatches", dispatches_s.as_str()), + ], + ); + } + return Ok(Some(cuda_dwt97_output_to_j2k(&output)?)); + } + #[cfg(not(feature = "cuda-runtime"))] + let _ = job; + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_forward_dwt97"), + ("decision", "cpu_fallback"), + ("reason", "cuda_unavailable"), + ], + ); + } + Ok(None) + } + + fn encode_quantize_subband( + &mut self, + job: J2kQuantizeSubbandJob<'_>, + ) -> core::result::Result>, &'static str> { + self.quantize_subband_attempts = self.quantize_subband_attempts.saturating_add(1); + if self.prefer_cpu_quantize_subband { + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_quantize_subband"), + ("decision", "cpu_fallback"), + ("reason", "prefer_cpu_quantize_subband"), + ], + ); + } + let _ = job; + return Ok(None); + } + #[cfg(feature = "cuda-runtime")] + if let Some(context) = self.cuda_context()? { + let (output, elapsed_us) = time_cuda_stage( + "j2k.j2k.cuda.encode.quantize", + &context, + self.collect_profile, + || { + context.j2k_quantize_subband( + job.coefficients, + CudaJ2kQuantizeJob { + step_exponent: job.step_exponent, + step_mantissa: job.step_mantissa, + range_bits: job.range_bits, + reversible: job.reversible, + }, + ) + }, + ) + .map_err(|_| "CUDA quantize subband encode kernel failed")?; + let dispatches = output.execution().kernel_dispatches(); + self.quantize_subband_dispatches = + self.quantize_subband_dispatches.saturating_add(dispatches); + self.quantize_us = self.quantize_us.saturating_add(elapsed_us); + if j2k_profile::gpu_route_profile_enabled() { + let samples_s = job.coefficients.len().to_string(); + let dispatches_s = dispatches.to_string(); + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_quantize_subband"), + ("decision", "cuda_dispatch"), + ("samples", samples_s.as_str()), + ("dispatches", dispatches_s.as_str()), + ], + ); + } + return Ok(Some(output.coefficients().to_vec())); + } + #[cfg(not(feature = "cuda-runtime"))] + let _ = job; + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_quantize_subband"), + ("decision", "cpu_fallback"), + ("reason", "cuda_unavailable"), + ], + ); + } + Ok(None) + } + + fn encode_tier1_code_block( + &mut self, + _job: J2kTier1CodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + self.tier1_code_block_attempts = self.tier1_code_block_attempts.saturating_add(1); + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_tier1_code_block"), + ("decision", "cpu_fallback"), + ("reason", "unsupported_stage"), + ], + ); + } + Ok(None) + } + + fn encode_ht_code_block( + &mut self, + job: J2kHtCodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + self.ht_code_block_attempts = self.ht_code_block_attempts.saturating_add(1); + #[cfg(feature = "cuda-runtime")] + if let Some(context) = self.cuda_context()? { + let resources = self.cuda_encode_resources(&context)?; + let encoded = cuda_encode_ht_code_block(&context, resources.as_ref(), job)?; + let dispatches = encoded.execution().kernel_dispatches(); + let ht_encode_us = encoded.stage_timings().ht_encode_us; + let mut outputs = encoded_ht_code_blocks_from_cuda(&encoded); + let output = outputs + .pop() + .ok_or("CUDA HTJ2K code-block encode returned no output")?; + self.ht_code_block_dispatches = + self.ht_code_block_dispatches.saturating_add(dispatches); + if self.collect_profile { + self.ht_encode_us = self.ht_encode_us.saturating_add(ht_encode_us); + } + if j2k_profile::gpu_route_profile_enabled() { + let width_s = job.width.to_string(); + let height_s = job.height.to_string(); + let dispatches_s = dispatches.to_string(); + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_ht_code_block"), + ("decision", "cuda_dispatch"), + ("width", width_s.as_str()), + ("height", height_s.as_str()), + ("dispatches", dispatches_s.as_str()), + ], + ); + } + return Ok(Some(output)); + } + #[cfg(not(feature = "cuda-runtime"))] + let _ = job; + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_ht_code_block"), + ("decision", "cpu_fallback"), + ("reason", "unsupported_stage"), + ], + ); + } + Ok(None) + } + + fn encode_ht_code_blocks( + &mut self, + jobs: &[J2kHtCodeBlockEncodeJob<'_>], + ) -> core::result::Result>, &'static str> { + self.ht_code_block_attempts = self.ht_code_block_attempts.saturating_add(jobs.len()); + #[cfg(feature = "cuda-runtime")] + if let Some(context) = self.cuda_context()? { + let resources = self.cuda_encode_resources(&context)?; + let encoded = cuda_encode_ht_code_blocks(&context, resources.as_ref(), jobs)?; + let dispatches = encoded.execution().kernel_dispatches(); + let ht_encode_us = encoded.stage_timings().ht_encode_us; + let outputs = encoded_ht_code_blocks_from_cuda(&encoded); + self.ht_code_block_dispatches = + self.ht_code_block_dispatches.saturating_add(dispatches); + if self.collect_profile { + self.ht_encode_us = self.ht_encode_us.saturating_add(ht_encode_us); + } + if j2k_profile::gpu_route_profile_enabled() { + let jobs_s = jobs.len().to_string(); + let dispatches_s = dispatches.to_string(); + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_ht_code_blocks"), + ("decision", "cuda_dispatch"), + ("jobs", jobs_s.as_str()), + ("dispatches", dispatches_s.as_str()), + ], + ); + } + return Ok(Some(outputs)); + } + #[cfg(not(feature = "cuda-runtime"))] + let _ = jobs; + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_ht_code_blocks"), + ("decision", "cpu_fallback"), + ("reason", "cuda_unavailable"), + ], + ); + } + Ok(None) + } + + fn encode_htj2k_tile( + &mut self, + job: J2kHtj2kTileEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + self.htj2k_tile_attempts = self.htj2k_tile_attempts.saturating_add(1); + if self.prefer_cpu_forward_rct || self.prefer_cpu_packetization { + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_htj2k_tile"), + ("decision", "cpu_fallback"), + ("reason", "prefer_stage_hybrid"), + ], + ); + } + let _ = job; + return Ok(None); + } + #[cfg(feature = "cuda-runtime")] + if let Some(context) = self.cuda_context()? { + let resources = self.cuda_encode_resources(&context)?; + let Some(encoded) = cuda_encode_htj2k_tile_body( + &context, + resources.as_ref(), + job, + self.collect_profile, + )? + else { + return Ok(None); + }; + self.htj2k_tile_dispatches = self.htj2k_tile_dispatches.saturating_add(1); + self.deinterleave_attempts = self.deinterleave_attempts.saturating_add(1); + self.deinterleave_dispatches = self + .deinterleave_dispatches + .saturating_add(encoded.deinterleave_dispatches); + if job.use_mct { + if job.reversible { + self.forward_rct_attempts = self.forward_rct_attempts.saturating_add(1); + } else { + self.forward_ict_attempts = self.forward_ict_attempts.saturating_add(1); + } + } + self.forward_rct_dispatches = self + .forward_rct_dispatches + .saturating_add(encoded.forward_rct_dispatches); + self.forward_ict_dispatches = self + .forward_ict_dispatches + .saturating_add(encoded.forward_ict_dispatches); + if job.num_decomposition_levels > 0 { + if job.reversible { + self.forward_dwt53_attempts = self + .forward_dwt53_attempts + .saturating_add(usize::from(job.num_components)); + } else { + self.forward_dwt97_attempts = self + .forward_dwt97_attempts + .saturating_add(usize::from(job.num_components)); + } + } + self.forward_dwt53_dispatches = self + .forward_dwt53_dispatches + .saturating_add(encoded.forward_dwt53_dispatches); + self.forward_dwt97_dispatches = self + .forward_dwt97_dispatches + .saturating_add(encoded.forward_dwt97_dispatches); + self.quantize_subband_attempts = self + .quantize_subband_attempts + .saturating_add(encoded.quantize_jobs); + self.quantize_subband_dispatches = self + .quantize_subband_dispatches + .saturating_add(encoded.quantize_dispatches); + self.ht_code_block_attempts = self + .ht_code_block_attempts + .saturating_add(encoded.ht_code_block_jobs); + self.ht_code_block_dispatches = self + .ht_code_block_dispatches + .saturating_add(encoded.ht_code_block_dispatches); + self.packetization_attempts = self.packetization_attempts.saturating_add(1); + self.packetization_dispatches = self + .packetization_dispatches + .saturating_add(encoded.packetization_dispatches); + if self.collect_profile { + self.deinterleave_us = self + .deinterleave_us + .saturating_add(encoded.timings.deinterleave_us); + self.mct_us = self.mct_us.saturating_add(encoded.timings.mct_us); + self.dwt_us = self.dwt_us.saturating_add(encoded.timings.dwt_us); + self.quantize_us = self.quantize_us.saturating_add(encoded.timings.quantize_us); + self.ht_encode_us = self + .ht_encode_us + .saturating_add(encoded.timings.ht_encode_us); + self.packetize_us = self + .packetize_us + .saturating_add(encoded.timings.packetize_us); + } + if j2k_profile::gpu_route_profile_enabled() { + let components_s = job.num_components.to_string(); + let blocks_s = encoded.ht_code_block_jobs.to_string(); + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_htj2k_tile"), + ("decision", "cuda_dispatch"), + ("components", components_s.as_str()), + ("blocks", blocks_s.as_str()), + ], + ); + } + return Ok(Some(encoded.tile_data)); + } + #[cfg(not(feature = "cuda-runtime"))] + let _ = job; + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_htj2k_tile"), + ("decision", "cpu_fallback"), + ("reason", "cuda_unavailable"), + ], + ); + } + Ok(None) + } + + fn encode_ht_subband( + &mut self, + job: J2kHtSubbandEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + let code_block_count = ht_subband_code_block_count(job)?; + self.ht_subband_attempts = self.ht_subband_attempts.saturating_add(1); + self.quantize_subband_attempts = self.quantize_subband_attempts.saturating_add(1); + self.ht_code_block_attempts = self.ht_code_block_attempts.saturating_add(code_block_count); + if self.prefer_cpu_ht_subband { + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_ht_subband"), + ("decision", "cpu_fallback"), + ("reason", "prefer_cpu_ht_subband"), + ], + ); + } + return Ok(None); + } + #[cfg(feature = "cuda-runtime")] + if let Some(context) = self.cuda_context()? { + let resources = self.cuda_encode_resources(&context)?; + let encoded = + cuda_encode_ht_subband(&context, resources.as_ref(), job, self.collect_profile)?; + let quantize_dispatches = encoded.quantize_dispatches; + let encode_dispatches = encoded.encode.execution().kernel_dispatches(); + let outputs = encoded_ht_code_blocks_from_cuda(&encoded.encode); + self.ht_subband_dispatches = self.ht_subband_dispatches.saturating_add(1); + self.quantize_subband_dispatches = self + .quantize_subband_dispatches + .saturating_add(quantize_dispatches); + self.ht_code_block_dispatches = self + .ht_code_block_dispatches + .saturating_add(encode_dispatches); + if self.collect_profile { + self.quantize_us = self.quantize_us.saturating_add(encoded.timings.quantize_us); + self.ht_encode_us = self + .ht_encode_us + .saturating_add(encoded.timings.ht_encode_us); + } + if j2k_profile::gpu_route_profile_enabled() { + let width_s = job.width.to_string(); + let height_s = job.height.to_string(); + let blocks_s = code_block_count.to_string(); + let quantize_dispatches_s = quantize_dispatches.to_string(); + let encode_dispatches_s = encode_dispatches.to_string(); + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_ht_subband"), + ("decision", "cuda_dispatch"), + ("width", width_s.as_str()), + ("height", height_s.as_str()), + ("blocks", blocks_s.as_str()), + ("quantize_dispatches", quantize_dispatches_s.as_str()), + ("encode_dispatches", encode_dispatches_s.as_str()), + ], + ); + } + return Ok(Some(outputs)); + } + #[cfg(not(feature = "cuda-runtime"))] + let _ = job; + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_ht_subband"), + ("decision", "cpu_fallback"), + ("reason", "cuda_unavailable"), + ], + ); + } + Ok(None) + } + + fn encode_packetization( + &mut self, + job: J2kPacketizationEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + self.packetization_attempts = self.packetization_attempts.saturating_add(1); + if self.prefer_cpu_packetization { + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_packetization"), + ("decision", "cpu_fallback"), + ("reason", "prefer_cpu_packetization"), + ], + ); + } + let _ = job; + return Ok(None); + } + let plan = match flatten_cuda_htj2k_packetization_job(job) { + Ok(plan) => plan, + Err(reason) => { + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_packetization"), + ("decision", "cpu_fallback"), + ("reason", reason), + ], + ); + } + return Ok(None); + } + }; + #[cfg(feature = "cuda-runtime")] + if let Some(context) = self.cuda_context()? { + let packets = cuda_packetization_packets(&plan); + let subbands = cuda_packetization_subbands(&plan); + let blocks = cuda_packetization_blocks(&plan); + let tag_states = cuda_packetization_tag_states(&plan); + let tag_nodes = cuda_packetization_tag_nodes(&plan); + let packetized = context + .packetize_htj2k_cleanup_packets_with_tag_state( + &plan.payload, + &packets, + &subbands, + &blocks, + &tag_states, + &tag_nodes, + ) + .map_err(|_| "CUDA HTJ2K packetization kernel failed")?; + let dispatches = packetized.execution().kernel_dispatches(); + let packetize_us = packetized.stage_timings().packetize_us; + self.packetization_dispatches = + self.packetization_dispatches.saturating_add(dispatches); + if self.collect_profile { + self.packetize_us = self.packetize_us.saturating_add(packetize_us); + } + if j2k_profile::gpu_route_profile_enabled() { + let packets_s = packets.len().to_string(); + let dispatches_s = dispatches.to_string(); + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_packetization"), + ("decision", "cuda_dispatch"), + ("packets", packets_s.as_str()), + ("dispatches", dispatches_s.as_str()), + ], + ); + } + return Ok(Some(packetized.data().to_vec())); + } + #[cfg(not(feature = "cuda-runtime"))] + let _ = plan; + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( + "j2k", + "cuda", + &[ + ("op", "encode_packetization"), + ("decision", "cpu_fallback"), + ("reason", "unsupported_stage"), + ], + ); + } + Ok(None) + } +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_packetization_packets( + plan: &CudaHtj2kPacketizationPlan, +) -> Vec { + plan.packets + .iter() + .map(|packet| CudaHtj2kPacketizationPacket { + block_start: packet.block_start, + block_count: packet.block_count, + subband_start: packet.subband_start, + subband_count: packet.subband_count, + output_capacity: packet.output_capacity, + layer: packet.layer, + }) + .collect() +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_packetization_subbands( + plan: &CudaHtj2kPacketizationPlan, +) -> Vec { + plan.subbands + .iter() + .map(|subband| CudaHtj2kPacketizationSubband { + block_start: subband.block_start, + block_count: subband.block_count, + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + }) + .collect() +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_packetization_blocks( + plan: &CudaHtj2kPacketizationPlan, +) -> Vec { + plan.blocks + .iter() + .map(|block| CudaHtj2kPacketizationBlock { + data_offset: block.data_offset, + data_len: block.data_len, + cleanup_length: block.cleanup_length, + refinement_length: block.refinement_length, + num_coding_passes: block.num_coding_passes, + num_zero_bitplanes: block.num_zero_bitplanes, + l_block: block.l_block, + previously_included: block.previously_included, + inclusion_layer: block.inclusion_layer, + }) + .collect() +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_packetization_tag_states( + plan: &CudaHtj2kPacketizationPlan, +) -> Vec { + plan.tag_states + .iter() + .map(|state| CudaHtj2kPacketizationSubbandTagState { + inclusion_node_start: state.inclusion_node_start, + zero_bitplane_node_start: state.zero_bitplane_node_start, + node_count: state.node_count, + reserved0: 0, + }) + .collect() +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_packetization_tag_nodes( + plan: &CudaHtj2kPacketizationPlan, +) -> Vec { + plan.tag_nodes + .iter() + .map(|node| CudaHtj2kPacketizationTagNodeState { + current: node.current, + known: node.known, + }) + .collect() +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_encode_ht_code_block( + context: &CudaContext, + resources: &CudaHtj2kEncodeResources, + job: J2kHtCodeBlockEncodeJob<'_>, +) -> core::result::Result { + let coefficient_len = (job.width as usize) + .checked_mul(job.height as usize) + .ok_or("CUDA HTJ2K code-block encode job is too large")?; + if coefficient_len != job.coefficients.len() { + return Err("CUDA HTJ2K code-block encode job has invalid coefficient length"); + } + let cuda_jobs = [CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: job.width, + height: job.height, + total_bitplanes: job.total_bitplanes, + target_coding_passes: job.target_coding_passes, + }]; + context + .encode_htj2k_codeblocks_with_resources(job.coefficients, &cuda_jobs, resources) + .map_err(|_| "CUDA HTJ2K code-block encode kernel failed") +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_encode_ht_code_blocks( + context: &CudaContext, + resources: &CudaHtj2kEncodeResources, + jobs: &[J2kHtCodeBlockEncodeJob<'_>], +) -> core::result::Result { + let total_coefficients = jobs.iter().try_fold(0usize, |acc, job| { + let coefficient_len = (job.width as usize) + .checked_mul(job.height as usize) + .ok_or("CUDA HTJ2K code-block batch is too large")?; + if coefficient_len != job.coefficients.len() { + return Err("CUDA HTJ2K code-block encode job has invalid coefficient length"); + } + acc.checked_add(coefficient_len) + .ok_or("CUDA HTJ2K code-block batch is too large") + })?; + let mut coefficients = Vec::with_capacity(total_coefficients); + let mut cuda_jobs = Vec::with_capacity(jobs.len()); + for job in jobs { + let coefficient_offset = u32::try_from(coefficients.len()) + .map_err(|_| "CUDA HTJ2K code-block batch is too large")?; + coefficients.extend_from_slice(job.coefficients); + cuda_jobs.push(CudaHtj2kEncodeCodeBlockJob { + coefficient_offset, + width: job.width, + height: job.height, + total_bitplanes: job.total_bitplanes, + target_coding_passes: job.target_coding_passes, + }); + } + + context + .encode_htj2k_codeblocks_with_resources(&coefficients, &cuda_jobs, resources) + .map_err(|_| "CUDA HTJ2K code-block batch encode kernel failed") +} + +#[cfg(feature = "cuda-runtime")] +struct CudaEncodedHtj2kTile { + tile_data: Vec, + deinterleave_dispatches: usize, + forward_rct_dispatches: usize, + forward_ict_dispatches: usize, + forward_dwt53_dispatches: usize, + forward_dwt97_dispatches: usize, + quantize_jobs: usize, + quantize_dispatches: usize, + ht_code_block_dispatches: usize, + ht_code_block_jobs: usize, + packetization_dispatches: usize, + timings: CudaEncodeStageTimings, +} + +#[cfg(feature = "cuda-runtime")] +#[derive(Default)] +struct CudaHtj2kTileEncodeStats { + collect_profile: bool, + deinterleave_dispatches: usize, + forward_rct_dispatches: usize, + forward_ict_dispatches: usize, + forward_dwt53_dispatches: usize, + forward_dwt97_dispatches: usize, + quantize_jobs: usize, + quantize_dispatches: usize, + ht_code_block_dispatches: usize, + ht_code_block_jobs: usize, + timings: CudaEncodeStageTimings, +} + +#[cfg(feature = "cuda-runtime")] +struct CudaEncodedHtj2kResolution { + subbands: Vec, +} + +#[cfg(feature = "cuda-runtime")] +struct CudaEncodedHtj2kSubband { + code_blocks: Vec, + num_cbs_x: u32, + num_cbs_y: u32, +} + +#[cfg(feature = "cuda-runtime")] +#[derive(Clone, Copy)] +struct CudaTileSubbandRegion { + x0: u32, + y0: u32, + width: u32, + height: u32, + stride: u32, +} + +#[cfg(feature = "cuda-runtime")] +#[derive(Clone, Copy)] +enum CudaTileSubbandKind { + LowLow, + HighLow, + LowHigh, + HighHigh, +} + +#[cfg(feature = "cuda-runtime")] +#[derive(Clone, Copy)] +struct CudaHtj2kEncodeRuntime<'a> { + context: &'a CudaContext, + resources: &'a CudaHtj2kEncodeResources, +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_encode_htj2k_tile_body( + context: &CudaContext, + encode_resources: &CudaHtj2kEncodeResources, + job: J2kHtj2kTileEncodeJob<'_>, + collect_profile: bool, +) -> core::result::Result, &'static str> { + validate_cuda_htj2k_tile_job(job)?; + let num_pixels = (job.width as usize) + .checked_mul(job.height as usize) + .ok_or("CUDA HTJ2K tile dimensions are too large")?; + let (components, deinterleave_us) = time_cuda_stage( + "j2k.htj2k.encode.tile.deinterleave", + context, + collect_profile, + || { + context.j2k_deinterleave_to_f32_resident( + job.pixels, + num_pixels, + job.num_components, + job.bit_depth, + job.signed, + ) + }, + ) + .map_err(|_| "CUDA HTJ2K tile deinterleave failed")?; + cuda_encode_htj2k_resident_components_body( + context, + encode_resources, + job, + components, + deinterleave_us, + collect_profile, + ) +} + +#[cfg(feature = "cuda-runtime")] +fn validate_cuda_htj2k_tile_job( + job: J2kHtj2kTileEncodeJob<'_>, +) -> core::result::Result<(), &'static str> { + if job + .component_sampling + .iter() + .any(|&sampling| sampling != (1, 1)) + { + return Err("CUDA HTJ2K tile encode does not support component subsampling != (1, 1)"); + } + // Native treats `use_mct = options.use_mct && num_components >= 3`, applying the + // color transform to component planes 0,1,2 and passing any 4th plane through + // unchanged. The resident path mirrors this: RCT/ICT runs on the first three + // planes (see `j2k_forward_rct_resident`/`j2k_forward_ict_resident`), and every + // component — including the passthrough 4th — still flows through the per-component + // DWT → quantize → HT code-block → packetization loop below. + // + // Only `{1, 3, 4}` component counts are in scope. Reject any other count with a + // typed hard error rather than `Ok(None)` (a silent CPU fallback is forbidden for + // in-scope inputs). + if !matches!(job.num_components, 1 | 3 | 4) { + return Err("CUDA HTJ2K tile encode supports 1, 3, or 4 components"); + } + if job.use_mct && job.num_components < 3 { + return Err("CUDA HTJ2K tile encode requires at least three components for MCT"); + } + if job.code_block_width == 0 || job.code_block_height == 0 { + return Err("CUDA HTJ2K tile encode job has invalid code-block dimensions"); + } + let expected_quantization_steps = 1usize + .checked_add(usize::from(job.num_decomposition_levels).saturating_mul(3)) + .ok_or("CUDA HTJ2K tile quantization step count overflow")?; + if job.quantization_steps.len() != expected_quantization_steps { + return Err("CUDA HTJ2K tile quantization step count mismatch"); + } + Ok(()) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_encode_htj2k_device_tile_body( + context: &CudaContext, + encode_resources: &CudaHtj2kEncodeResources, + tile: CudaLosslessEncodeTile<'_>, + job: J2kHtj2kTileEncodeJob<'_>, + collect_profile: bool, +) -> core::result::Result, &'static str> { + validate_cuda_htj2k_tile_job(job)?; + let format = cuda_encode_format(tile.format).map_err(|_| "CUDA HTJ2K tile format failed")?; + if job.width != tile.output_width || job.height != tile.output_height { + return Err("CUDA HTJ2K tile encode job dimensions do not match CUDA tile"); + } + if tile.width != tile.output_width || tile.height != tile.output_height { + return Err("CUDA HTJ2K tile encode does not support input padding"); + } + if job.num_components != format.components || job.bit_depth != format.bit_depth || job.signed { + return Err("CUDA HTJ2K tile encode job sample format does not match CUDA tile"); + } + let (components, deinterleave_us) = time_cuda_stage( + "j2k.htj2k.encode.tile.device_deinterleave", + context, + collect_profile, + || { + context.j2k_deinterleave_strided_to_f32_resident(CudaJ2kStridedInterleavedPixels { + buffer: tile.buffer, + byte_offset: tile.byte_offset, + width: tile.width, + height: tile.height, + pitch_bytes: tile.pitch_bytes, + num_components: job.num_components, + bit_depth: job.bit_depth, + signed: job.signed, + }) + }, + ) + .map_err(|_| "CUDA HTJ2K tile device deinterleave failed")?; + cuda_encode_htj2k_resident_components_body( + context, + encode_resources, + job, + components, + deinterleave_us, + collect_profile, + ) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_encode_htj2k_resident_components_body( + context: &CudaContext, + encode_resources: &CudaHtj2kEncodeResources, + job: J2kHtj2kTileEncodeJob<'_>, + mut components: CudaJ2kResidentComponents, + deinterleave_us: u128, + collect_profile: bool, +) -> core::result::Result, &'static str> { + let mut stats = CudaHtj2kTileEncodeStats { + collect_profile, + deinterleave_dispatches: components.execution().kernel_dispatches(), + timings: CudaEncodeStageTimings { + deinterleave_us, + ..CudaEncodeStageTimings::default() + }, + ..CudaHtj2kTileEncodeStats::default() + }; + let runtime = CudaHtj2kEncodeRuntime { + context, + resources: encode_resources, + }; + + if job.use_mct { + let (execution, mct_us) = if job.reversible { + time_cuda_stage( + "j2k.htj2k.encode.tile.rct", + context, + collect_profile, + || context.j2k_forward_rct_resident(&mut components), + ) + .map_err(|_| "CUDA HTJ2K tile RCT failed")? + } else { + time_cuda_stage( + "j2k.htj2k.encode.tile.ict", + context, + collect_profile, + || context.j2k_forward_ict_resident(&mut components), + ) + .map_err(|_| "CUDA HTJ2K tile ICT failed")? + }; + stats.timings.mct_us = stats.timings.mct_us.saturating_add(mct_us); + if job.reversible { + stats.forward_rct_dispatches = execution.kernel_dispatches(); + } else { + stats.forward_ict_dispatches = execution.kernel_dispatches(); + } + } + + let mut component_resolution_packets = Vec::with_capacity(usize::from(job.num_components)); + if job.num_decomposition_levels == 0 { + for component in 0..job.num_components { + let y0 = u32::from(component) + .checked_mul(job.height) + .ok_or("CUDA HTJ2K tile component offset overflow")?; + let subband = cuda_encode_tile_subband_region( + runtime, + components.buffer(), + CudaTileSubbandRegion { + x0: 0, + y0, + width: job.width, + height: job.height, + stride: job.width, + }, + job.quantization_steps[0], + job, + CudaTileSubbandKind::LowLow, + &mut stats, + )?; + component_resolution_packets.push(vec![CudaEncodedHtj2kResolution { + subbands: vec![subband], + }]); + } + } else { + for component in 0..job.num_components { + let packets = if job.reversible { + let (dwt, dwt_us) = time_cuda_stage( + "j2k.htj2k.encode.tile.dwt53", + context, + collect_profile, + || { + context.j2k_forward_dwt53_resident_component( + &components, + component, + job.width, + job.height, + job.num_decomposition_levels, + ) + }, + ) + .map_err(|_| "CUDA HTJ2K tile DWT 5/3 failed")?; + stats.forward_dwt53_dispatches = stats + .forward_dwt53_dispatches + .saturating_add(dwt.execution().kernel_dispatches()); + stats.timings.dwt_us = stats.timings.dwt_us.saturating_add(dwt_us); + cuda_encode_dwt_component_packets( + runtime, + job, + dwt.buffer(), + dwt.levels(), + dwt.ll_dimensions(), + &mut stats, + )? + } else { + let (dwt, dwt_us) = time_cuda_stage( + "j2k.htj2k.encode.tile.dwt97", + context, + collect_profile, + || { + context.j2k_forward_dwt97_resident_component( + &components, + component, + job.width, + job.height, + job.num_decomposition_levels, + ) + }, + ) + .map_err(|_| "CUDA HTJ2K tile DWT 9/7 failed")?; + stats.forward_dwt97_dispatches = stats + .forward_dwt97_dispatches + .saturating_add(dwt.execution().kernel_dispatches()); + stats.timings.dwt_us = stats.timings.dwt_us.saturating_add(dwt_us); + cuda_encode_dwt_component_packets( + runtime, + job, + dwt.buffer(), + dwt.levels(), + dwt.ll_dimensions(), + &mut stats, + )? + }; + component_resolution_packets.push(packets); + } + } + + let resolution_packets = + cuda_order_component_resolution_packets(component_resolution_packets, job.num_components)?; + let (tile_data, packetization_dispatches, packetize_us) = + cuda_packetize_tile_body(context, job, &resolution_packets, stats.ht_code_block_jobs)?; + stats.timings.packetize_us = stats.timings.packetize_us.saturating_add(packetize_us); + Ok(Some(CudaEncodedHtj2kTile { + tile_data, + deinterleave_dispatches: stats.deinterleave_dispatches, + forward_rct_dispatches: stats.forward_rct_dispatches, + forward_ict_dispatches: stats.forward_ict_dispatches, + forward_dwt53_dispatches: stats.forward_dwt53_dispatches, + forward_dwt97_dispatches: stats.forward_dwt97_dispatches, + quantize_jobs: stats.quantize_jobs, + quantize_dispatches: stats.quantize_dispatches, + ht_code_block_dispatches: stats.ht_code_block_dispatches, + ht_code_block_jobs: stats.ht_code_block_jobs, + packetization_dispatches, + timings: stats.timings, + })) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_encode_dwt_component_packets( + runtime: CudaHtj2kEncodeRuntime<'_>, + job: J2kHtj2kTileEncodeJob<'_>, + transformed: &CudaDeviceBuffer, + levels: &[CudaDwt53LevelShape], + ll_dimensions: (u32, u32), + stats: &mut CudaHtj2kTileEncodeStats, +) -> core::result::Result, &'static str> { + if levels.len() != usize::from(job.num_decomposition_levels) { + return Err("CUDA HTJ2K tile DWT level count mismatch"); + } + let (ll_width, ll_height) = ll_dimensions; + let full_width = levels.first().map_or(ll_width, |level| level.width); + let mut packets = Vec::with_capacity(levels.len().saturating_add(1)); + + let ll_subband = cuda_encode_tile_subband_region( + runtime, + transformed, + CudaTileSubbandRegion { + x0: 0, + y0: 0, + width: ll_width, + height: ll_height, + stride: full_width, + }, + job.quantization_steps[0], + job, + CudaTileSubbandKind::LowLow, + stats, + )?; + packets.push(CudaEncodedHtj2kResolution { + subbands: vec![ll_subband], + }); + + for (level_idx, level) in levels.iter().rev().enumerate() { + let step_base = 1usize + .checked_add(level_idx.saturating_mul(3)) + .ok_or("CUDA HTJ2K tile quantization step index overflow")?; + let hl = cuda_encode_tile_subband_region( + runtime, + transformed, + CudaTileSubbandRegion { + x0: level.low_width, + y0: 0, + width: level.high_width, + height: level.low_height, + stride: full_width, + }, + job.quantization_steps[step_base], + job, + CudaTileSubbandKind::HighLow, + stats, + )?; + let lh = cuda_encode_tile_subband_region( + runtime, + transformed, + CudaTileSubbandRegion { + x0: 0, + y0: level.low_height, + width: level.low_width, + height: level.high_height, + stride: full_width, + }, + job.quantization_steps[step_base + 1], + job, + CudaTileSubbandKind::LowHigh, + stats, + )?; + let hh = cuda_encode_tile_subband_region( + runtime, + transformed, + CudaTileSubbandRegion { + x0: level.low_width, + y0: level.low_height, + width: level.high_width, + height: level.high_height, + stride: full_width, + }, + job.quantization_steps[step_base + 2], + job, + CudaTileSubbandKind::HighHigh, + stats, + )?; + packets.push(CudaEncodedHtj2kResolution { + subbands: vec![hl, lh, hh], + }); + } + + Ok(packets) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_encode_tile_subband_region( + runtime: CudaHtj2kEncodeRuntime<'_>, + source: &CudaDeviceBuffer, + region: CudaTileSubbandRegion, + quantization_step: (u16, u16), + job: J2kHtj2kTileEncodeJob<'_>, + subband_kind: CudaTileSubbandKind, + stats: &mut CudaHtj2kTileEncodeStats, +) -> core::result::Result { + if region.width == 0 || region.height == 0 { + return Ok(CudaEncodedHtj2kSubband { + code_blocks: Vec::new(), + num_cbs_x: 0, + num_cbs_y: 0, + }); + } + + let (step_exponent, step_mantissa) = quantization_step; + let step_exponent_u8 = u8::try_from(step_exponent) + .map_err(|_| "CUDA HTJ2K tile quantization exponent exceeds u8")?; + let total_bitplanes = job + .guard_bits + .saturating_add(step_exponent_u8) + .saturating_sub(1); + let (quantized, quantize_us) = time_cuda_stage( + "j2k.htj2k.encode.tile.quantize", + runtime.context, + stats.collect_profile, + || { + runtime.context.j2k_quantize_subband_region_resident( + source, + CudaJ2kQuantizeSubbandRegionJob { + x0: region.x0, + y0: region.y0, + width: region.width, + height: region.height, + stride: region.stride, + quantization: CudaJ2kQuantizeJob { + step_exponent, + step_mantissa, + range_bits: cuda_tile_subband_range_bits(job.bit_depth, subband_kind), + reversible: job.reversible, + }, + }, + ) + }, + ) + .map_err(|_| "CUDA HTJ2K tile quantize failed")?; + stats.quantize_jobs = stats.quantize_jobs.saturating_add(1); + stats.quantize_dispatches = stats + .quantize_dispatches + .saturating_add(quantized.execution().kernel_dispatches()); + stats.timings.quantize_us = stats.timings.quantize_us.saturating_add(quantize_us); + + let region_jobs = cuda_ht_region_jobs( + region.width, + region.height, + job.code_block_width, + job.code_block_height, + total_bitplanes, + )?; + stats.ht_code_block_jobs = stats.ht_code_block_jobs.saturating_add(region_jobs.len()); + let encoded = runtime + .context + .encode_htj2k_codeblock_regions_resident_with_resources( + quantized.buffer(), + quantized.coefficient_count(), + ®ion_jobs, + runtime.resources, + ) + .map_err(|_| "CUDA HTJ2K tile code-block encode failed")?; + stats.ht_code_block_dispatches = stats + .ht_code_block_dispatches + .saturating_add(encoded.execution().kernel_dispatches()); + stats.timings.ht_encode_us = stats + .timings + .ht_encode_us + .saturating_add(encoded.stage_timings().ht_encode_us); + + Ok(CudaEncodedHtj2kSubband { + code_blocks: encoded_ht_code_blocks_from_cuda(&encoded), + num_cbs_x: region.width.div_ceil(job.code_block_width), + num_cbs_y: region.height.div_ceil(job.code_block_height), + }) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_tile_subband_range_bits(bit_depth: u8, subband_kind: CudaTileSubbandKind) -> u8 { + let log_gain = match subband_kind { + CudaTileSubbandKind::LowLow => 0, + CudaTileSubbandKind::HighLow | CudaTileSubbandKind::LowHigh => 1, + CudaTileSubbandKind::HighHigh => 2, + }; + bit_depth.saturating_add(log_gain) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_order_component_resolution_packets( + component_resolution_packets: Vec>, + num_components: u8, +) -> core::result::Result, &'static str> { + if component_resolution_packets.len() != usize::from(num_components) { + return Err("CUDA HTJ2K tile component packet count mismatch"); + } + let resolution_count = component_resolution_packets + .first() + .map_or(0usize, Vec::len); + let mut component_iters: Vec<_> = component_resolution_packets + .into_iter() + .map(Vec::into_iter) + .collect(); + let mut resolution_packets = + Vec::with_capacity(resolution_count.saturating_mul(component_iters.len())); + + for _resolution in 0..resolution_count { + for component in &mut component_iters { + resolution_packets.push( + component + .next() + .ok_or("CUDA HTJ2K tile component resolution count mismatch")?, + ); + } + } + if component_iters + .iter_mut() + .any(|component| component.next().is_some()) + { + return Err("CUDA HTJ2K tile component resolution count mismatch"); + } + + Ok(resolution_packets) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_ht_region_jobs( + width: u32, + height: u32, + code_block_width: u32, + code_block_height: u32, + total_bitplanes: u8, +) -> core::result::Result, &'static str> { + if code_block_width == 0 || code_block_height == 0 { + return Err("CUDA HTJ2K encode job has invalid code-block dimensions"); + } + if width == 0 || height == 0 { + return Ok(Vec::new()); + } + + let num_cbs_x = width.div_ceil(code_block_width); + let num_cbs_y = height.div_ceil(code_block_height); + let count = (num_cbs_x as usize) + .checked_mul(num_cbs_y as usize) + .ok_or("CUDA HTJ2K code-block count overflow")?; + let mut cuda_jobs = Vec::with_capacity(count); + for cby in 0..num_cbs_y { + for cbx in 0..num_cbs_x { + let x0 = cbx + .checked_mul(code_block_width) + .ok_or("CUDA HTJ2K code-block x offset overflow")?; + let y0 = cby + .checked_mul(code_block_height) + .ok_or("CUDA HTJ2K code-block y offset overflow")?; + let block_width = (x0 + code_block_width).min(width) - x0; + let block_height = (y0 + code_block_height).min(height) - y0; + let offset = (y0 as usize) + .checked_mul(width as usize) + .and_then(|row| row.checked_add(x0 as usize)) + .ok_or("CUDA HTJ2K code-block offset overflow")?; + cuda_jobs.push(CudaHtj2kEncodeCodeBlockRegionJob { + coefficient_offset: u32::try_from(offset) + .map_err(|_| "CUDA HTJ2K code-block offset exceeds u32")?, + coefficient_stride: width, + width: block_width, + height: block_height, + total_bitplanes, + target_coding_passes: 1, + }); + } + } + Ok(cuda_jobs) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_packetize_tile_body( + context: &CudaContext, + job: J2kHtj2kTileEncodeJob<'_>, + resolution_packets: &[CudaEncodedHtj2kResolution], + code_block_count: usize, +) -> core::result::Result<(Vec, usize, u128), &'static str> { + let packet_descriptors = + cuda_tile_packet_descriptors(resolution_packets.len(), 1, job.num_components)?; + let resolutions: Vec> = resolution_packets + .iter() + .map(|resolution| J2kPacketizationResolution { + subbands: resolution + .subbands + .iter() + .map(|subband| { + let code_blocks = subband + .code_blocks + .iter() + .map(|block| J2kPacketizationCodeBlock { + data: block.data.as_slice(), + ht_cleanup_length: block.cleanup_length, + ht_refinement_length: block.refinement_length, + num_coding_passes: block.num_coding_passes, + num_zero_bitplanes: block.num_zero_bitplanes, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }) + .collect(); + J2kPacketizationSubband { + code_blocks, + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + } + }) + .collect(), + }) + .collect(); + + let packetization_job = J2kPacketizationEncodeJob { + resolution_count: u32::try_from(resolutions.len()) + .map_err(|_| "CUDA HTJ2K tile resolution count exceeds u32")?, + num_layers: 1, + num_components: job.num_components, + code_block_count: u32::try_from(code_block_count) + .map_err(|_| "CUDA HTJ2K tile code-block count exceeds u32")?, + progression_order: job.progression_order, + packet_descriptors: &packet_descriptors, + resolutions: &resolutions, + }; + let plan = flatten_cuda_htj2k_packetization_job(packetization_job)?; + let packets = cuda_packetization_packets(&plan); + let subbands = cuda_packetization_subbands(&plan); + let blocks = cuda_packetization_blocks(&plan); + let tag_states = cuda_packetization_tag_states(&plan); + let tag_nodes = cuda_packetization_tag_nodes(&plan); + let packetized = context + .packetize_htj2k_cleanup_packets_with_tag_state( + &plan.payload, + &packets, + &subbands, + &blocks, + &tag_states, + &tag_nodes, + ) + .map_err(|_| "CUDA HTJ2K tile packetization failed")?; + Ok(( + packetized.data().to_vec(), + packetized.execution().kernel_dispatches(), + packetized.stage_timings().packetize_us, + )) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_tile_packet_descriptors( + packet_count: usize, + num_layers: u8, + num_components: u8, +) -> core::result::Result, &'static str> { + if num_layers != 1 { + return Err("CUDA HTJ2K tile encode currently prepares one packet layer"); + } + let component_count = usize::from(num_components).max(1); + (0..packet_count) + .map(|packet_index| { + Ok(J2kPacketizationPacketDescriptor { + packet_index: u32::try_from(packet_index) + .map_err(|_| "CUDA HTJ2K tile packet index exceeds u32")?, + state_index: u32::try_from(packet_index) + .map_err(|_| "CUDA HTJ2K tile packet state index exceeds u32")?, + layer: 0, + resolution: u32::try_from(packet_index / component_count) + .map_err(|_| "CUDA HTJ2K tile packet resolution exceeds u32")?, + component: u8::try_from(packet_index % component_count) + .map_err(|_| "CUDA HTJ2K tile packet component exceeds u8")?, + precinct: 0, + }) + }) + .collect() +} + +#[cfg(feature = "cuda-runtime")] +struct CudaEncodedHtSubband { + quantize_dispatches: usize, + encode: j2k_cuda_runtime::CudaHtj2kEncodedCodeBlocks, + timings: CudaEncodeStageTimings, +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_encode_ht_subband( + context: &CudaContext, + encode_resources: &CudaHtj2kEncodeResources, + job: J2kHtSubbandEncodeJob<'_>, + collect_profile: bool, +) -> core::result::Result { + let expected_len = (job.width as usize) + .checked_mul(job.height as usize) + .ok_or("CUDA HTJ2K subband encode dimensions are too large")?; + if expected_len != job.coefficients.len() { + return Err("CUDA HTJ2K subband encode job has invalid coefficient length"); + } + if job.code_block_width == 0 || job.code_block_height == 0 { + return Err("CUDA HTJ2K subband encode job has invalid code-block dimensions"); + } + + let sample_buffer = context + .upload_f32_pinned(job.coefficients) + .map_err(|_| "CUDA HTJ2K subband upload failed")?; + let (quantized, quantize_us) = time_cuda_stage( + "j2k.htj2k.encode.subband.quantize", + context, + collect_profile, + || { + context.j2k_quantize_subband_resident( + &sample_buffer, + job.coefficients.len(), + CudaJ2kQuantizeJob { + step_exponent: job.step_exponent, + step_mantissa: job.step_mantissa, + range_bits: job.range_bits, + reversible: job.reversible, + }, + ) + }, + ) + .map_err(|_| "CUDA quantize subband encode kernel failed")?; + let cuda_jobs = cuda_ht_subband_region_jobs(job)?; + let encoded = context + .encode_htj2k_codeblock_regions_resident_with_resources( + quantized.buffer(), + quantized.coefficient_count(), + &cuda_jobs, + encode_resources, + ) + .map_err(|_| "CUDA HTJ2K resident subband encode kernel failed")?; + + Ok(CudaEncodedHtSubband { + quantize_dispatches: quantized.execution().kernel_dispatches(), + timings: CudaEncodeStageTimings { + quantize_us, + ht_encode_us: encoded.stage_timings().ht_encode_us, + ..CudaEncodeStageTimings::default() + }, + encode: encoded, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_ht_subband_region_jobs( + job: J2kHtSubbandEncodeJob<'_>, +) -> core::result::Result, &'static str> { + cuda_ht_region_jobs( + job.width, + job.height, + job.code_block_width, + job.code_block_height, + job.total_bitplanes, + ) +} + +fn ht_subband_code_block_count( + job: J2kHtSubbandEncodeJob<'_>, +) -> core::result::Result { + if job.code_block_width == 0 || job.code_block_height == 0 { + return Err("CUDA HTJ2K subband encode job has invalid code-block dimensions"); + } + let num_cbs_x = job.width.div_ceil(job.code_block_width); + let num_cbs_y = job.height.div_ceil(job.code_block_height); + (num_cbs_x as usize) + .checked_mul(num_cbs_y as usize) + .ok_or("CUDA HTJ2K subband code-block count overflow") +} + +#[cfg(feature = "cuda-runtime")] +fn encoded_ht_code_block_from_cuda( + encoded: &j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock, +) -> EncodedHtJ2kCodeBlock { + EncodedHtJ2kCodeBlock { + data: encoded.data().to_vec(), + cleanup_length: encoded.cleanup_length(), + refinement_length: encoded.refinement_length(), + num_coding_passes: encoded.num_coding_passes(), + num_zero_bitplanes: encoded.num_zero_bitplanes(), + } +} + +#[cfg(feature = "cuda-runtime")] +fn encoded_ht_code_blocks_from_cuda( + encoded: &j2k_cuda_runtime::CudaHtj2kEncodedCodeBlocks, +) -> Vec { + encoded + .code_blocks() + .iter() + .map(encoded_ht_code_block_from_cuda) + .collect() +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_htj2k_encode_tables() -> CudaHtj2kEncodeTables<'static> { + CudaHtj2kEncodeTables { + vlc_table0: j2k_native::ht_vlc_encode_table0(), + vlc_table1: j2k_native::ht_vlc_encode_table1(), + uvlc_table: j2k_native::ht_uvlc_encode_table_bytes(), + } +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_dwt53_output_to_j2k( + output: &CudaDwt53Output, +) -> core::result::Result { + let (ll_width, ll_height) = output.ll_dimensions(); + let transformed = output.transformed(); + let full_width = output + .levels() + .first() + .map_or(ll_width, |level| level.width) as usize; + let mut ll = Vec::with_capacity((ll_width as usize) * (ll_height as usize)); + for y in 0..ll_height as usize { + let row_start = y + .checked_mul(full_width) + .ok_or("CUDA DWT LL row offset overflow")?; + ll.extend_from_slice(&transformed[row_start..row_start + ll_width as usize]); + } + + let mut levels = Vec::with_capacity(output.levels().len()); + for shape in output.levels() { + levels.push(J2kForwardDwt53Level { + hl: extract_cuda_subband( + transformed, + full_width, + shape.low_width, + 0, + shape.high_width, + shape.low_height, + )?, + lh: extract_cuda_subband( + transformed, + full_width, + 0, + shape.low_height, + shape.low_width, + shape.high_height, + )?, + hh: extract_cuda_subband( + transformed, + full_width, + shape.low_width, + shape.low_height, + shape.high_width, + shape.high_height, + )?, + width: shape.width, + height: shape.height, + low_width: shape.low_width, + low_height: shape.low_height, + high_width: shape.high_width, + high_height: shape.high_height, + }); + } + levels.reverse(); + + Ok(J2kForwardDwt53Output { + ll, + ll_width, + ll_height, + levels, + }) +} + +/// Test-only accessor that converts a CUDA forward 5/3 DWT output into the +/// native `J2kForwardDwt53Output` sub-band representation using the *exact* +/// production reshape (`cuda_dwt53_output_to_j2k`). +/// +/// This is `#[doc(hidden)]` and exists solely so the stage-parity test crate +/// can compare CUDA output against `forward_dwt53_reference` through the same +/// conversion the encoder uses (correct nested-band offsets + finest→coarsest +/// to coarsest→finest level reversal), instead of re-deriving the geometry. +#[cfg(feature = "cuda-runtime")] +#[doc(hidden)] +pub fn cuda_dwt53_output_to_j2k_for_test( + output: &CudaDwt53Output, +) -> core::result::Result { + cuda_dwt53_output_to_j2k(output) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_dwt97_output_to_j2k( + output: &CudaDwt97Output, +) -> core::result::Result { + let (ll_width, ll_height) = output.ll_dimensions(); + let transformed = output.transformed(); + let full_width = output + .levels() + .first() + .map_or(ll_width, |level| level.width) as usize; + let mut ll = Vec::with_capacity((ll_width as usize) * (ll_height as usize)); + for y in 0..ll_height as usize { + let row_start = y + .checked_mul(full_width) + .ok_or("CUDA DWT LL row offset overflow")?; + ll.extend_from_slice(&transformed[row_start..row_start + ll_width as usize]); + } + + let mut levels = Vec::with_capacity(output.levels().len()); + for shape in output.levels() { + levels.push(J2kForwardDwt97Level { + hl: extract_cuda_subband( + transformed, + full_width, + shape.low_width, + 0, + shape.high_width, + shape.low_height, + )?, + lh: extract_cuda_subband( + transformed, + full_width, + 0, + shape.low_height, + shape.low_width, + shape.high_height, + )?, + hh: extract_cuda_subband( + transformed, + full_width, + shape.low_width, + shape.low_height, + shape.high_width, + shape.high_height, + )?, + width: shape.width, + height: shape.height, + low_width: shape.low_width, + low_height: shape.low_height, + high_width: shape.high_width, + high_height: shape.high_height, + }); + } + levels.reverse(); + + Ok(J2kForwardDwt97Output { + ll, + ll_width, + ll_height, + levels, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn extract_cuda_subband( + transformed: &[f32], + full_width: usize, + x0: u32, + y0: u32, + width: u32, + height: u32, +) -> core::result::Result, &'static str> { + let mut out = Vec::with_capacity((width as usize) * (height as usize)); + for y in 0..height as usize { + let row_start = (y0 as usize) + .checked_add(y) + .and_then(|row| row.checked_mul(full_width)) + .and_then(|row| row.checked_add(x0 as usize)) + .ok_or("CUDA DWT subband offset overflow")?; + out.extend_from_slice(&transformed[row_start..row_start + width as usize]); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + #[cfg(feature = "cuda-runtime")] + use super::{cuda_htj2k_encode_tables, cuda_runtime_required}; + use super::{ + encode_j2k_lossless_with_cuda, encode_j2k_lossless_with_cuda_and_profile, + flatten_cuda_htj2k_packetization_job, CudaEncodeStageAccelerator, + CudaHtj2kPacketizationPlanTagNodeState, + }; + use j2k::adapter::encode_stage::NativeEncodeStageAdapter; + #[cfg(feature = "cuda-runtime")] + use j2k::adapter::encode_stage::{J2kDeinterleaveToF32Job, J2kHtCodeBlockEncodeJob}; + use j2k::adapter::encode_stage::{ + J2kEncodeStageAccelerator, J2kHtSubbandEncodeJob, J2kPacketizationBlockCodingMode, + J2kPacketizationCodeBlock, J2kPacketizationEncodeJob, J2kPacketizationPacketDescriptor, + J2kPacketizationProgressionOrder, J2kPacketizationResolution, J2kPacketizationSubband, + J2kQuantizeSubbandJob, + }; + #[cfg(feature = "cuda-runtime")] + use j2k::{encode_j2k_lossy_with_accelerator, J2kLossyEncodeOptions, J2kLossySamples}; + use j2k::{ + EncodeBackendPreference, J2kBlockCodingMode, J2kEncodeValidation, J2kLosslessEncodeOptions, + J2kLosslessSamples, + }; + #[cfg(feature = "cuda-runtime")] + use j2k_core::BackendKind; + use j2k_core::CodecError; + #[cfg(feature = "cuda-runtime")] + use j2k_cuda_runtime::{ + CudaContext, CudaHtj2kEncodeCodeBlockJob, CudaHtj2kEncodeCodeBlockRegionJob, + CudaJ2kQuantizeJob, + }; + use j2k_native::{ + encode_with_accelerator as encode_with_native_accelerator, DecodeSettings, EncodeOptions, + Image, + }; + + fn assert_strict_cuda_classic_tier1_error(err: &E, context: &str) { + assert!(err.is_unsupported()); + let message = err.to_string(); + assert!( + message.contains("tier1_code_block") || message.contains("deinterleave"), + "expected {context} error to mention either the missing classic tier-1 stage or unavailable CUDA deinterleave, got {message}" + ); + } + + #[allow(clippy::too_many_arguments)] + fn encode_with_cuda_test_accelerator( + pixels: &[u8], + width: u32, + height: u32, + components: u8, + bit_depth: u8, + signed: bool, + options: &EncodeOptions, + accelerator: &mut CudaEncodeStageAccelerator, + ) -> core::result::Result, &'static str> { + let mut bridge = NativeEncodeStageAdapter::new(accelerator); + encode_with_native_accelerator( + pixels, + width, + height, + components, + bit_depth, + signed, + options, + &mut bridge, + ) + } + + #[test] + fn cuda_lossless_encode_auto_errors_for_unsupported_classic_tier1() { + let pixels: Vec = (0u32..128 * 128) + .map(|value| u8::try_from((value * 17 + 5) & 0xFF).expect("masked value fits in u8")) + .collect(); + let samples = + J2kLosslessSamples::new(&pixels, 128, 128, 1, 8, false).expect("valid gray8 samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::Auto) + .with_block_coding_mode(J2kBlockCodingMode::Classic) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::CpuRoundTrip); + + let err = encode_j2k_lossless_with_cuda(samples, &options) + .expect_err("CUDA-named encode must not silently return CPU fallback"); + + assert_strict_cuda_classic_tier1_error(&err, "strict CUDA encode"); + } + + #[test] + fn cuda_lossless_encode_profile_auto_errors_for_unsupported_classic_tier1() { + let pixels: Vec = (0u32..128 * 128) + .map(|value| u8::try_from((value * 19 + 7) & 0xFF).expect("masked value fits in u8")) + .collect(); + let samples = + J2kLosslessSamples::new(&pixels, 128, 128, 1, 8, false).expect("valid gray8 samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::Auto) + .with_block_coding_mode(J2kBlockCodingMode::Classic) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::External); + + let err = encode_j2k_lossless_with_cuda_and_profile(samples, &options) + .expect_err("profiled CUDA encode must not silently return CPU fallback"); + + assert_strict_cuda_classic_tier1_error(&err, "profiled strict CUDA encode"); + } + + #[test] + fn cuda_lossless_encode_require_device_errors_for_unsupported_classic_tier1() { + let pixels: Vec = (0u32..128 * 128) + .map(|value| u8::try_from((value * 29 + 11) & 0xFF).expect("masked value fits in u8")) + .collect(); + let samples = + J2kLosslessSamples::new(&pixels, 128, 128, 1, 8, false).expect("valid gray8 samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::Classic) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::External); + + let err = encode_j2k_lossless_with_cuda(samples, &options) + .expect_err("strict CUDA encode must not silently fall back to CPU"); + + assert_strict_cuda_classic_tier1_error(&err, "strict CUDA encode"); + } + + #[test] + fn cuda_packetization_flatten_accepts_cleanup_only_single_block_packet() { + let payload = [0x12, 0x34, 0x56, 0x78]; + let code_block = J2kPacketizationCodeBlock { + data: &payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let subband = J2kPacketizationSubband { + code_blocks: vec![code_block], + num_cbs_x: 1, + num_cbs_y: 1, + }; + let resolution = J2kPacketizationResolution { + subbands: vec![subband], + }; + let descriptor = J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }; + let job = J2kPacketizationEncodeJob { + resolution_count: 1, + num_layers: 1, + num_components: 1, + code_block_count: 1, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &[descriptor], + resolutions: &[resolution], + }; + + let plan = flatten_cuda_htj2k_packetization_job(job).expect("supported CUDA packetization"); + + assert_eq!(plan.payload, payload); + assert_eq!(plan.packets.len(), 1); + assert_eq!(plan.subbands.len(), 1); + assert_eq!(plan.blocks.len(), 1); + assert_eq!(plan.packets[0].block_start, 0); + assert_eq!(plan.packets[0].block_count, 1); + assert_eq!(plan.packets[0].subband_start, 0); + assert_eq!(plan.packets[0].subband_count, 1); + assert_eq!(plan.subbands[0].block_start, 0); + assert_eq!(plan.subbands[0].block_count, 1); + let payload_len = u32::try_from(payload.len()).expect("test payload length fits in u32"); + assert!(plan.packets[0].output_capacity >= payload_len + 256); + assert_eq!(plan.blocks[0].data_offset, 0); + assert_eq!(plan.blocks[0].data_len, payload_len); + assert_eq!(plan.blocks[0].num_coding_passes, 1); + assert_eq!(plan.blocks[0].num_zero_bitplanes, 2); + } + + #[test] + fn cuda_packetization_flatten_accepts_cleanup_only_multi_block_packet() { + let payloads = vec![ + vec![0x10, 0x11, 0x12], + vec![0x20, 0x21], + vec![0x30, 0x31, 0x32, 0x33], + vec![0x40], + ]; + let code_blocks = payloads + .iter() + .enumerate() + .map(|(idx, payload)| J2kPacketizationCodeBlock { + data: payload.as_slice(), + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: u8::try_from(idx + 1).expect("test zbp fits in u8"), + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }) + .collect(); + let subband = J2kPacketizationSubband { + code_blocks, + num_cbs_x: 2, + num_cbs_y: 2, + }; + let resolution = J2kPacketizationResolution { + subbands: vec![subband], + }; + let descriptor = J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }; + let job = J2kPacketizationEncodeJob { + resolution_count: 1, + num_layers: 1, + num_components: 1, + code_block_count: 4, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &[descriptor], + resolutions: &[resolution], + }; + + let plan = + flatten_cuda_htj2k_packetization_job(job).expect("multi-block CUDA packetization"); + + assert_eq!(plan.packets.len(), 1); + assert_eq!(plan.subbands.len(), 1); + assert_eq!(plan.blocks.len(), 4); + assert_eq!(plan.packets[0].block_start, 0); + assert_eq!(plan.packets[0].block_count, 4); + assert_eq!(plan.packets[0].subband_start, 0); + assert_eq!(plan.packets[0].subband_count, 1); + assert_eq!(plan.subbands[0].block_start, 0); + assert_eq!(plan.subbands[0].block_count, 4); + assert_eq!(plan.subbands[0].num_cbs_x, 2); + assert_eq!(plan.subbands[0].num_cbs_y, 2); + assert_eq!( + plan.payload, + payloads.into_iter().flatten().collect::>() + ); + assert_eq!(plan.blocks[2].num_zero_bitplanes, 3); + } + + #[test] + fn cuda_packetization_flatten_accepts_ht_refinement_pass_packet() { + let payload = [0x12, 0x34, 0x56, 0x78, 0x9a]; + let code_block = J2kPacketizationCodeBlock { + data: &payload, + ht_cleanup_length: 3, + ht_refinement_length: 2, + num_coding_passes: 3, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let subband = J2kPacketizationSubband { + code_blocks: vec![code_block], + num_cbs_x: 1, + num_cbs_y: 1, + }; + let resolution = J2kPacketizationResolution { + subbands: vec![subband], + }; + let descriptor = J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }; + let job = J2kPacketizationEncodeJob { + resolution_count: 1, + num_layers: 1, + num_components: 1, + code_block_count: 1, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &[descriptor], + resolutions: &[resolution], + }; + + let plan = flatten_cuda_htj2k_packetization_job(job).expect("HT refinement packetization"); + + assert_eq!(plan.payload, payload); + assert_eq!(plan.blocks.len(), 1); + assert_eq!(plan.blocks[0].num_coding_passes, 3); + assert_eq!( + plan.blocks[0].data_len, + u32::try_from(payload.len()).expect("test payload length fits in u32") + ); + } + + #[test] + fn cuda_packetization_rejects_overflowing_ht_refinement_lengths() { + let payload = [0x12]; + let code_block = J2kPacketizationCodeBlock { + data: &payload, + ht_cleanup_length: u32::MAX, + ht_refinement_length: 1, + num_coding_passes: 3, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + + let err = super::cuda_ht_segment_lengths(&code_block) + .expect_err("overflowing CUDA HT segment lengths rejected"); + + assert_eq!(err, "multi-pass HTJ2K packet contribution length overflow"); + } + + #[test] + fn cuda_packetization_flatten_rejects_out_of_range_ht_pass_count() { + let payload = [0u8; 1]; + let code_block = J2kPacketizationCodeBlock { + data: &payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 165, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let subband = J2kPacketizationSubband { + code_blocks: vec![code_block], + num_cbs_x: 1, + num_cbs_y: 1, + }; + let resolution = J2kPacketizationResolution { + subbands: vec![subband], + }; + let descriptor = J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }; + let job = J2kPacketizationEncodeJob { + resolution_count: 1, + num_layers: 1, + num_components: 1, + code_block_count: 1, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &[descriptor], + resolutions: &[resolution], + }; + + let err = flatten_cuda_htj2k_packetization_job(job) + .expect_err("invalid HT pass count must be rejected before CUDA launch"); + + assert_eq!( + err, + "CUDA HTJ2K packetization coding pass count exceeds JPEG 2000 bounds" + ); + } + + #[test] + fn cuda_packetization_flatten_accepts_previously_included_second_layer_packet() { + let first_payload = [0x11u8; 20]; + let second_payload = [0x22u8; 5]; + let first_block = J2kPacketizationCodeBlock { + data: &first_payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let second_block = J2kPacketizationCodeBlock { + data: &second_payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let first_resolution = J2kPacketizationResolution { + subbands: vec![J2kPacketizationSubband { + code_blocks: vec![first_block], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }; + let second_resolution = J2kPacketizationResolution { + subbands: vec![J2kPacketizationSubband { + code_blocks: vec![second_block], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }; + let descriptors = [ + J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }, + J2kPacketizationPacketDescriptor { + packet_index: 1, + state_index: 0, + layer: 1, + resolution: 0, + component: 0, + precinct: 0, + }, + ]; + let resolutions = [first_resolution, second_resolution]; + let job = J2kPacketizationEncodeJob { + resolution_count: 2, + num_layers: 2, + num_components: 1, + code_block_count: 2, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &descriptors, + resolutions: &resolutions, + }; + + let plan = + flatten_cuda_htj2k_packetization_job(job).expect("stateful CUDA packetization plan"); + + assert_eq!( + plan.payload, + [first_payload.as_slice(), second_payload.as_slice()].concat() + ); + assert_eq!(plan.packets.len(), 2); + assert_eq!(plan.blocks.len(), 2); + assert_eq!(plan.packets[0].layer, 0); + assert_eq!(plan.packets[1].layer, 1); + assert_eq!(plan.blocks[0].l_block, 3); + assert_eq!(plan.blocks[0].previously_included, 0); + assert_eq!(plan.blocks[1].previously_included, 1); + assert_eq!(plan.blocks[0].inclusion_layer, 0); + assert_eq!(plan.blocks[1].inclusion_layer, 0); + assert_eq!( + plan.blocks[1].l_block, 5, + "first layer length must update L-block for later packet state" + ); + } + + #[test] + fn cuda_packetization_flatten_accepts_deferred_first_inclusion_second_layer_packet() { + let payload = [0x44u8; 5]; + let first_block = J2kPacketizationCodeBlock { + data: &[], + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 0, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let second_block = J2kPacketizationCodeBlock { + data: &payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let first_resolution = J2kPacketizationResolution { + subbands: vec![J2kPacketizationSubband { + code_blocks: vec![first_block], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }; + let second_resolution = J2kPacketizationResolution { + subbands: vec![J2kPacketizationSubband { + code_blocks: vec![second_block], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }; + let descriptors = [ + J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }, + J2kPacketizationPacketDescriptor { + packet_index: 1, + state_index: 0, + layer: 1, + resolution: 0, + component: 0, + precinct: 0, + }, + ]; + let resolutions = [first_resolution, second_resolution]; + let job = J2kPacketizationEncodeJob { + resolution_count: 2, + num_layers: 2, + num_components: 1, + code_block_count: 2, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &descriptors, + resolutions: &resolutions, + }; + + let plan = + flatten_cuda_htj2k_packetization_job(job).expect("deferred first inclusion plan"); + + assert_eq!(plan.payload, payload); + assert_eq!(plan.packets.len(), 2); + assert_eq!(plan.blocks.len(), 2); + assert_eq!(plan.packets[0].layer, 0); + assert_eq!(plan.packets[1].layer, 1); + assert_eq!(plan.blocks[0].previously_included, 0); + assert_eq!(plan.blocks[1].previously_included, 0); + assert_eq!(plan.blocks[0].inclusion_layer, 1); + assert_eq!(plan.blocks[1].inclusion_layer, 1); + } + + #[test] + fn cuda_packetization_flatten_accepts_deferred_first_inclusion_after_non_empty_packet() { + let first_payload = [0x11u8; 3]; + let second_payload = [0x22u8; 5]; + let first_resolution = J2kPacketizationResolution { + subbands: vec![J2kPacketizationSubband { + code_blocks: vec![ + J2kPacketizationCodeBlock { + data: &first_payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }, + J2kPacketizationCodeBlock { + data: &[], + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 0, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }, + ], + num_cbs_x: 2, + num_cbs_y: 1, + }], + }; + let second_resolution = J2kPacketizationResolution { + subbands: vec![J2kPacketizationSubband { + code_blocks: vec![ + J2kPacketizationCodeBlock { + data: &[], + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 0, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }, + J2kPacketizationCodeBlock { + data: &second_payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }, + ], + num_cbs_x: 2, + num_cbs_y: 1, + }], + }; + let descriptors = [ + J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }, + J2kPacketizationPacketDescriptor { + packet_index: 1, + state_index: 0, + layer: 1, + resolution: 0, + component: 0, + precinct: 0, + }, + ]; + let resolutions = [first_resolution, second_resolution]; + let job = J2kPacketizationEncodeJob { + resolution_count: 2, + num_layers: 2, + num_components: 1, + code_block_count: 4, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &descriptors, + resolutions: &resolutions, + }; + + let plan = flatten_cuda_htj2k_packetization_job(job) + .expect("persistent tag-tree state is flattened for CUDA packetization"); + + assert_eq!( + plan.payload, + [first_payload.as_slice(), second_payload.as_slice()].concat() + ); + assert_eq!(plan.packets.len(), 2); + assert_eq!(plan.blocks.len(), 4); + assert_eq!(plan.blocks[0].previously_included, 0); + assert_eq!(plan.blocks[1].previously_included, 0); + assert_eq!(plan.blocks[2].previously_included, 1); + assert_eq!(plan.blocks[3].previously_included, 0); + assert_eq!(plan.blocks[0].inclusion_layer, 0); + assert_eq!(plan.blocks[1].inclusion_layer, 1); + assert_eq!(plan.blocks[2].inclusion_layer, 0); + assert_eq!(plan.blocks[3].inclusion_layer, 1); + assert_eq!(plan.tag_states.len(), 2); + assert_eq!(plan.tag_nodes.len(), 12); + assert_eq!(plan.tag_states[1].inclusion_node_start, 6); + assert_eq!(plan.tag_states[1].zero_bitplane_node_start, 9); + assert_eq!( + &plan.tag_nodes[6..9], + &[ + CudaHtj2kPacketizationPlanTagNodeState { + current: 0, + known: 1, + }, + CudaHtj2kPacketizationPlanTagNodeState { + current: 1, + known: 0, + }, + CudaHtj2kPacketizationPlanTagNodeState { + current: 0, + known: 1, + }, + ] + ); + assert_eq!( + &plan.tag_nodes[9..12], + &[ + CudaHtj2kPacketizationPlanTagNodeState { + current: 2, + known: 1, + }, + CudaHtj2kPacketizationPlanTagNodeState { + current: 0, + known: 0, + }, + CudaHtj2kPacketizationPlanTagNodeState { + current: 2, + known: 1, + }, + ] + ); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_lossless_encode_require_device_dispatches_cleanup_packetization_when_runtime_required() + { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u16..8 * 8) + .map(|value| u8::try_from((value * 31 + 7) & 0xFF).expect("masked value fits in u8")) + .collect(); + let samples = + J2kLosslessSamples::new(&pixels, 8, 8, 1, 8, false).expect("valid gray8 samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::CpuRoundTrip); + + let encoded = encode_j2k_lossless_with_cuda(samples, &options) + .expect("strict CUDA single-pass HT encode should dispatch all required stages"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(encoded.backend, BackendKind::Cuda); + assert_eq!(decoded.data, pixels); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_deinterleave_stage_dispatches_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels = [0u8, 128, 255, 64, 32, 16]; + let mut accelerator = CudaEncodeStageAccelerator::default(); + let components = accelerator + .encode_deinterleave(J2kDeinterleaveToF32Job { + pixels: &pixels, + num_pixels: 2, + num_components: 3, + bit_depth: 8, + signed: false, + }) + .expect("CUDA deinterleave hook") + .expect("CUDA deinterleave dispatch"); + + assert_eq!(accelerator.deinterleave_dispatches(), 1); + assert_eq!( + components, + vec![vec![-128.0, -64.0], vec![0.0, -96.0], vec![127.0, -112.0]] + ); + } + + #[test] + fn prefer_cpu_ht_subband_declines_fused_subband_but_counts_attempts() { + let mut accelerator = CudaEncodeStageAccelerator::default() + .prefer_cpu_ht_subband(true) + .prefer_cpu_quantize_subband(true); + let output = accelerator + .encode_ht_subband(J2kHtSubbandEncodeJob { + coefficients: &[0.0; 16], + width: 4, + height: 4, + step_exponent: 8, + step_mantissa: 0, + range_bits: 8, + reversible: false, + code_block_width: 4, + code_block_height: 4, + total_bitplanes: 9, + }) + .expect("subband hook can decline"); + + assert!(output.is_none()); + assert_eq!(accelerator.ht_subband_attempts, 1); + assert_eq!(accelerator.quantize_subband_attempts, 1); + assert_eq!(accelerator.ht_code_block_attempts, 1); + assert_eq!(accelerator.dispatch_report().total(), 0); + + let quantized = accelerator + .encode_quantize_subband(J2kQuantizeSubbandJob { + coefficients: &[0.0; 16], + step_exponent: 8, + step_mantissa: 0, + range_bits: 8, + reversible: false, + }) + .expect("quantize hook can decline"); + assert!(quantized.is_none()); + assert_eq!(accelerator.quantize_subband_attempts, 2); + assert_eq!(accelerator.dispatch_report().total(), 0); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_lossless_encode_require_device_dispatches_multi_block_cleanup_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u32..128 * 128) + .map(|value| u8::try_from((value * 19 + 23) & 0xFF).expect("masked value fits in u8")) + .collect(); + let samples = + J2kLosslessSamples::new(&pixels, 128, 128, 1, 8, false).expect("valid gray8 samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::CpuRoundTrip); + + let encoded = encode_j2k_lossless_with_cuda(samples, &options) + .expect("strict CUDA multi-block cleanup encode should dispatch all required stages"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(encoded.backend, BackendKind::Cuda); + assert_eq!(decoded.data, pixels); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_lossless_encode_require_device_dispatches_dwt53_cleanup_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u32..128 * 128) + .map(|value| u8::try_from((value * 37 + 41) & 0xFF).expect("masked value fits in u8")) + .collect(); + let samples = + J2kLosslessSamples::new(&pixels, 128, 128, 1, 8, false).expect("valid gray8 samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(1)) + .with_validation(J2kEncodeValidation::CpuRoundTrip); + + let encoded = encode_j2k_lossless_with_cuda(samples, &options) + .expect("strict CUDA DWT cleanup encode should dispatch all required stages"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(encoded.backend, BackendKind::Cuda); + assert_eq!(decoded.data, pixels); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_lossless_encode_profile_reports_resident_stage_timings_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u32..128 * 128) + .map(|value| u8::try_from((value * 43 + 29) & 0xFF).expect("masked value fits in u8")) + .collect(); + let samples = + J2kLosslessSamples::new(&pixels, 128, 128, 1, 8, false).expect("valid gray8 samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(1)) + .with_validation(J2kEncodeValidation::CpuRoundTrip); + + let (encoded, report) = encode_j2k_lossless_with_cuda_and_profile(samples, &options) + .expect("strict CUDA profiled DWT cleanup encode should dispatch all required stages"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(encoded.backend, BackendKind::Cuda); + assert_eq!(decoded.data, pixels); + assert_eq!(report.backend, BackendKind::Cuda); + assert_eq!(report.input_bytes, pixels.len()); + assert_eq!(report.codestream_bytes, encoded.codestream.len()); + assert!(report.dispatch_count > 0); + assert!(report.block_count > 0); + assert!(report.deinterleave_us > 0); + assert_eq!(report.mct_us, 0); + assert!(report.dwt_us > 0); + assert!(report.quantize_us > 0); + assert!(report.ht_encode_us > 0); + assert!(report.packetize_us > 0); + assert!(report.total_us > 0); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_lossless_encode_require_device_dispatches_rgb_rct_cleanup_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u32..128 * 128 * 3) + .map(|value| u8::try_from((value * 13 + 71) & 0xFF).expect("masked value fits in u8")) + .collect(); + let samples = + J2kLosslessSamples::new(&pixels, 128, 128, 3, 8, false).expect("valid rgb8 samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(1)) + .with_validation(J2kEncodeValidation::CpuRoundTrip); + + let encoded = encode_j2k_lossless_with_cuda(samples, &options) + .expect("strict CUDA RGB cleanup encode should dispatch all required stages"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(encoded.backend, BackendKind::Cuda); + assert_eq!(decoded.data, pixels); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_lossy_htj2k_facade_require_device_dispatches_supported_stages_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u32..64 * 64) + .map(|value| u8::try_from((value * 41 + 17) & 0xFF).expect("masked value fits in u8")) + .collect(); + let samples = + J2kLossySamples::new(&pixels, 64, 64, 1, 8, false).expect("valid gray8 samples"); + let options = J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(1)) + .with_validation(J2kEncodeValidation::CpuRoundTrip); + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let encoded = encode_j2k_lossy_with_accelerator( + samples, + &options, + BackendKind::Cuda, + &mut accelerator, + ) + .expect("strict CUDA HTJ2K lossy facade encode should dispatch supported stages"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(encoded.backend, BackendKind::Cuda); + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.num_components, 1); + assert_eq!(accelerator.deinterleave_dispatches(), 1); + assert!(accelerator.forward_dwt97_dispatches() > 0); + assert_eq!(accelerator.quantize_subband_dispatches(), 4); + assert_eq!(accelerator.ht_code_block_dispatches(), 4); + assert_eq!(accelerator.packetization_dispatches(), 1); + } + + #[test] + fn cuda_encode_stage_accelerator_preserves_cpu_codestream_validity() { + let pixels: Vec = (0u8..192).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let codestream = encode_with_cuda_test_accelerator( + &pixels, + 8, + 8, + 3, + 8, + false, + &options, + &mut accelerator, + ) + .expect("encode with CUDA stage accelerator"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.width, 8); + assert_eq!(decoded.height, 8); + assert_eq!(decoded.num_components, 3); + assert_eq!(decoded.bit_depth, 8); + assert_eq!(accelerator.forward_rct_attempts(), 1); + assert_eq!(accelerator.forward_dwt53_attempts(), 3); + assert!(accelerator.tier1_code_block_attempts() > 0); + assert_eq!(accelerator.packetization_attempts(), 1); + } + + #[test] + fn cuda_auto_host_output_declines_packetization_before_flattening() { + let mut accelerator = CudaEncodeStageAccelerator::for_auto_host_output(); + let invalid_for_cuda_flattening = J2kPacketizationEncodeJob { + resolution_count: 1, + num_layers: 1, + num_components: 3, + code_block_count: 0, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &[], + resolutions: &[], + }; + + let encoded = J2kEncodeStageAccelerator::encode_packetization( + &mut accelerator, + invalid_for_cuda_flattening, + ) + .expect("Auto host-output CUDA packetization should decline to CPU"); + + assert!(encoded.is_none()); + assert_eq!(accelerator.packetization_attempts(), 1); + assert_eq!(accelerator.packetization_dispatches(), 0); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_forward_rct_dispatches_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u16..7 * 5 * 3) + .map(|i| u8::try_from((i * 17) & 0xFF).expect("masked value fits in u8")) + .collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 0, + ..EncodeOptions::default() + }; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let codestream = encode_with_cuda_test_accelerator( + &pixels, + 7, + 5, + 3, + 8, + false, + &options, + &mut accelerator, + ) + .expect("encode with CUDA forward RCT"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert_eq!(accelerator.forward_rct_attempts(), 1); + assert_eq!(accelerator.forward_rct_dispatches(), 1); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_forward_ict_dispatches_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u32..32 * 32 * 3) + .map(|i| u8::try_from((i * 23 + 19) & 0xFF).expect("masked value fits in u8")) + .collect(); + let options = EncodeOptions { + reversible: false, + use_ht_block_coding: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let codestream = encode_with_cuda_test_accelerator( + &pixels, + 32, + 32, + 3, + 8, + false, + &options, + &mut accelerator, + ) + .expect("encode irreversible RGB with CUDA forward ICT"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data.len(), pixels.len()); + assert_eq!(accelerator.forward_ict_attempts(), 1); + assert_eq!(accelerator.forward_ict_dispatches(), 1); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_forward_dwt53_dispatches_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u16..8 * 8) + .map(|i| u8::try_from((i * 5) & 0xFF).expect("masked value fits in u8")) + .collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let codestream = encode_with_cuda_test_accelerator( + &pixels, + 8, + 8, + 1, + 8, + false, + &options, + &mut accelerator, + ) + .expect("encode with CUDA forward DWT 5/3"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert_eq!(accelerator.forward_dwt53_attempts(), 1); + assert_eq!(accelerator.forward_dwt53_dispatches(), 2); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_forward_dwt97_dispatches_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u16..32 * 32) + .map(|i| u8::try_from((i * 7 + 13) & 0xFF).expect("masked value fits in u8")) + .collect(); + let options = EncodeOptions { + reversible: false, + use_ht_block_coding: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let codestream = encode_with_cuda_test_accelerator( + &pixels, + 32, + 32, + 1, + 8, + false, + &options, + &mut accelerator, + ) + .expect("encode with CUDA forward DWT 9/7"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data.len(), pixels.len()); + assert_eq!(accelerator.forward_dwt97_attempts(), 1); + assert_eq!(accelerator.forward_dwt97_dispatches(), 3); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_quantize_subband_dispatches_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u16..32 * 32) + .map(|i| u8::try_from((i * 19 + 5) & 0xFF).expect("masked value fits in u8")) + .collect(); + let options = EncodeOptions { + reversible: false, + use_ht_block_coding: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let codestream = encode_with_cuda_test_accelerator( + &pixels, + 32, + 32, + 1, + 8, + false, + &options, + &mut accelerator, + ) + .expect("encode with CUDA quantization"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data.len(), pixels.len()); + assert_eq!(accelerator.quantize_subband_attempts(), 4); + assert_eq!(accelerator.quantize_subband_dispatches(), 4); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_encode_uses_resident_tile_body_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u16..32 * 32) + .map(|i| u8::try_from((i * 23 + 11) & 0xFF).expect("masked value fits in u8")) + .collect(); + let options = EncodeOptions { + reversible: true, + use_ht_block_coding: true, + num_decomposition_levels: 0, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let codestream = encode_with_cuda_test_accelerator( + &pixels, + 32, + 32, + 1, + 8, + false, + &options, + &mut accelerator, + ) + .expect("encode HTJ2K through CUDA tile-body hook"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert_eq!(accelerator.htj2k_tile_attempts, 1); + assert_eq!(accelerator.htj2k_tile_dispatches, 1); + assert_eq!(accelerator.ht_subband_attempts, 0); + assert_eq!(accelerator.ht_subband_dispatches, 0); + assert_eq!(accelerator.deinterleave_dispatches(), 1); + assert_eq!(accelerator.quantize_subband_attempts(), 1); + assert_eq!(accelerator.quantize_subband_dispatches(), 1); + assert_eq!(accelerator.ht_code_block_attempts(), 4); + assert_eq!(accelerator.ht_code_block_dispatches(), 1); + assert_eq!(accelerator.packetization_attempts(), 1); + assert_eq!(accelerator.packetization_dispatches(), 1); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_encode_uses_resident_dwt_tile_body_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u16..32 * 32) + .map(|i| u8::try_from((i * 29 + 5) & 0xFF).expect("masked value fits in u8")) + .collect(); + let options = EncodeOptions { + reversible: true, + use_ht_block_coding: true, + num_decomposition_levels: 1, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let codestream = encode_with_cuda_test_accelerator( + &pixels, + 32, + 32, + 1, + 8, + false, + &options, + &mut accelerator, + ) + .expect("encode HTJ2K DWT through CUDA tile-body hook"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert_eq!(accelerator.htj2k_tile_attempts, 1); + assert_eq!(accelerator.htj2k_tile_dispatches, 1); + assert_eq!(accelerator.ht_subband_attempts, 0); + assert_eq!(accelerator.ht_subband_dispatches, 0); + assert_eq!(accelerator.forward_dwt53_attempts(), 1); + assert!(accelerator.forward_dwt53_dispatches() > 0); + assert_eq!(accelerator.quantize_subband_attempts(), 4); + assert_eq!(accelerator.quantize_subband_dispatches(), 4); + assert_eq!(accelerator.ht_code_block_attempts(), 4); + assert_eq!(accelerator.ht_code_block_dispatches(), 4); + assert_eq!(accelerator.packetization_attempts(), 1); + assert_eq!(accelerator.packetization_dispatches(), 1); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_encode_uses_resident_mct_dwt_tile_body_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u16..32 * 32 * 3) + .map(|i| u8::try_from((i * 19 + 17) & 0xFF).expect("masked value fits in u8")) + .collect(); + let options = EncodeOptions { + reversible: true, + use_mct: true, + use_ht_block_coding: true, + num_decomposition_levels: 1, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let codestream = encode_with_cuda_test_accelerator( + &pixels, + 32, + 32, + 3, + 8, + false, + &options, + &mut accelerator, + ) + .expect("encode HTJ2K RGB DWT through CUDA tile-body hook"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert_eq!(accelerator.htj2k_tile_attempts, 1); + assert_eq!(accelerator.htj2k_tile_dispatches, 1); + assert_eq!(accelerator.ht_subband_attempts, 0); + assert_eq!(accelerator.forward_rct_attempts(), 1); + assert_eq!(accelerator.forward_rct_dispatches(), 1); + assert_eq!(accelerator.forward_dwt53_attempts(), 3); + assert!(accelerator.forward_dwt53_dispatches() > 0); + assert_eq!(accelerator.quantize_subband_attempts(), 12); + assert_eq!(accelerator.quantize_subband_dispatches(), 12); + assert_eq!(accelerator.ht_code_block_attempts(), 12); + assert_eq!(accelerator.ht_code_block_dispatches(), 12); + assert_eq!(accelerator.packetization_attempts(), 1); + assert_eq!(accelerator.packetization_dispatches(), 1); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_encode_uses_resident_dwt97_tile_body_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u16..32 * 32) + .map(|i| u8::try_from((i * 31 + 7) & 0xFF).expect("masked value fits in u8")) + .collect(); + let options = EncodeOptions { + reversible: false, + use_ht_block_coding: true, + num_decomposition_levels: 1, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let codestream = encode_with_cuda_test_accelerator( + &pixels, + 32, + 32, + 1, + 8, + false, + &options, + &mut accelerator, + ) + .expect("encode irreversible HTJ2K DWT through CUDA tile-body hook"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.width, 32); + assert_eq!(decoded.height, 32); + assert_eq!(decoded.num_components, 1); + assert_eq!(accelerator.htj2k_tile_attempts, 1); + assert_eq!(accelerator.htj2k_tile_dispatches, 1); + assert_eq!(accelerator.ht_subband_attempts, 0); + assert_eq!(accelerator.forward_dwt97_attempts(), 1); + assert!(accelerator.forward_dwt97_dispatches() > 0); + assert_eq!(accelerator.quantize_subband_attempts(), 4); + assert_eq!(accelerator.quantize_subband_dispatches(), 4); + assert_eq!(accelerator.ht_code_block_attempts(), 4); + assert_eq!(accelerator.ht_code_block_dispatches(), 4); + assert_eq!(accelerator.packetization_attempts(), 1); + assert_eq!(accelerator.packetization_dispatches(), 1); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_htj2k_codeblock_dispatches_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u16..8 * 8) + .map(|i| u8::try_from((i * 11 + 3) & 0xFF).expect("masked value fits in u8")) + .collect(); + let options = EncodeOptions { + reversible: true, + use_ht_block_coding: true, + num_decomposition_levels: 0, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let codestream = encode_with_cuda_test_accelerator( + &pixels, + 8, + 8, + 1, + 8, + false, + &options, + &mut accelerator, + ) + .expect("encode HTJ2K with CUDA HT codeblock kernel"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert!(accelerator.ht_code_block_attempts() > 0); + assert!(accelerator.ht_code_block_dispatches() > 0); + assert!(accelerator.ht_code_block_dispatches() <= accelerator.ht_code_block_attempts()); + assert_eq!( + accelerator.dispatch_report().ht_code_block, + accelerator.ht_code_block_dispatches() + ); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_htj2k_codeblock_preserves_requested_refinement_passes_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let coefficients = [0, 3, -5, 3, 5, 0, -3, 3, 7, -3, 0, 3, 0, 0, 5, -5]; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let encoded = accelerator + .encode_ht_code_block(J2kHtCodeBlockEncodeJob { + coefficients: &coefficients, + width: 4, + height: 4, + total_bitplanes: 4, + target_coding_passes: 2, + }) + .expect("CUDA HTJ2K code-block encode hook") + .expect("CUDA HTJ2K code-block encode output"); + + assert_eq!(encoded.num_coding_passes, 2); + assert_eq!(encoded.num_zero_bitplanes, 2); + assert_eq!(encoded.refinement_length, 1); + assert_eq!( + encoded.cleanup_length + encoded.refinement_length, + u32::try_from(encoded.data.len()).expect("test payload length fits u32") + ); + assert_eq!(accelerator.ht_code_block_dispatches(), 1); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_htj2k_codeblock_batch_uses_single_dispatch_when_runtime_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let pixels: Vec = (0u16..32 * 32) + .map(|i| u8::try_from((i * 17 + 9) & 0xFF).expect("masked value fits in u8")) + .collect(); + let options = EncodeOptions { + reversible: true, + use_ht_block_coding: true, + num_decomposition_levels: 0, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let mut accelerator = CudaEncodeStageAccelerator::default(); + + let codestream = encode_with_cuda_test_accelerator( + &pixels, + 32, + 32, + 1, + 8, + false, + &options, + &mut accelerator, + ) + .expect("encode HTJ2K with CUDA HT batch codeblock kernel"); + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert!(accelerator.ht_code_block_attempts() > 1); + assert_eq!(accelerator.ht_code_block_dispatches(), 1); + assert!( + accelerator.ht_code_block_dispatches() < accelerator.ht_code_block_attempts(), + "batch encode must not launch one kernel per codeblock" + ); + assert_eq!( + accelerator.dispatch_report().ht_code_block, + accelerator.ht_code_block_dispatches() + ); + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_resident_quantized_subband_feeds_resident_ht_batch_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let samples = [-3.6f32, -2.5, -0.4, 0.0, 0.49, 1.5, 3.2, 9.9]; + let context = CudaContext::system_default().expect("CUDA context"); + let sample_buffer = context.upload_f32(&samples).expect("resident samples"); + let quantization = CudaJ2kQuantizeJob { + step_exponent: 8, + step_mantissa: 0, + range_bits: 8, + reversible: true, + }; + let resident_quantized = context + .j2k_quantize_subband_resident(&sample_buffer, samples.len(), quantization) + .expect("resident quantization"); + let host_quantized = context + .j2k_quantize_subband(&samples, quantization) + .expect("host-staged quantization"); + let jobs = [CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 4, + height: 2, + total_bitplanes: 5, + target_coding_passes: 1, + }]; + + let resident_encoded = context + .encode_htj2k_codeblocks_resident( + resident_quantized.buffer(), + resident_quantized.coefficient_count(), + &jobs, + cuda_htj2k_encode_tables(), + ) + .expect("resident HTJ2K encode"); + let staged_encoded = context + .encode_htj2k_codeblocks( + host_quantized.coefficients(), + &jobs, + cuda_htj2k_encode_tables(), + ) + .expect("host-staged HTJ2K encode"); + + assert_eq!(resident_quantized.coefficient_count(), samples.len()); + assert_eq!(resident_encoded.execution().kernel_dispatches(), 1); + assert_eq!( + resident_encoded.code_blocks().len(), + staged_encoded.code_blocks().len() + ); + for (resident, staged) in resident_encoded + .code_blocks() + .iter() + .zip(staged_encoded.code_blocks()) + { + assert_eq!(resident.data(), staged.data()); + assert_eq!(resident.cleanup_length(), staged.cleanup_length()); + assert_eq!(resident.refinement_length(), staged.refinement_length()); + assert_eq!(resident.num_coding_passes(), staged.num_coding_passes()); + assert_eq!(resident.num_zero_bitplanes(), staged.num_zero_bitplanes()); + } + } + + #[cfg(feature = "cuda-runtime")] + #[test] + fn cuda_resident_strided_codeblock_region_matches_host_gather_when_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let samples: Vec = (0u16..16).map(|value| f32::from(value) - 8.0).collect(); + let context = CudaContext::system_default().expect("CUDA context"); + let sample_buffer = context.upload_f32(&samples).expect("resident samples"); + let quantization = CudaJ2kQuantizeJob { + step_exponent: 8, + step_mantissa: 0, + range_bits: 8, + reversible: true, + }; + let resident_quantized = context + .j2k_quantize_subband_resident(&sample_buffer, samples.len(), quantization) + .expect("resident quantization"); + let quantized = resident_quantized + .download_coefficients() + .expect("download quantized coefficients"); + let gathered_codeblock = vec![quantized[5], quantized[6], quantized[9], quantized[10]]; + let region_jobs = [CudaHtj2kEncodeCodeBlockRegionJob { + coefficient_offset: 5, + coefficient_stride: 4, + width: 2, + height: 2, + total_bitplanes: 5, + target_coding_passes: 1, + }]; + let contiguous_jobs = [CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 2, + height: 2, + total_bitplanes: 5, + target_coding_passes: 1, + }]; + + let resident_encoded = context + .encode_htj2k_codeblock_regions_resident( + resident_quantized.buffer(), + resident_quantized.coefficient_count(), + ®ion_jobs, + cuda_htj2k_encode_tables(), + ) + .expect("resident strided HTJ2K encode"); + let staged_encoded = context + .encode_htj2k_codeblocks( + &gathered_codeblock, + &contiguous_jobs, + cuda_htj2k_encode_tables(), + ) + .expect("host-gathered HTJ2K encode"); + + assert_eq!(resident_encoded.execution().kernel_dispatches(), 1); + assert_eq!(resident_encoded.code_blocks().len(), 1); + assert_eq!( + resident_encoded.code_blocks()[0].data(), + staged_encoded.code_blocks()[0].data() + ); + assert_eq!( + resident_encoded.code_blocks()[0].num_zero_bitplanes(), + staged_encoded.code_blocks()[0].num_zero_bitplanes() + ); + } +} diff --git a/crates/j2k-cuda/src/error.rs b/crates/j2k-cuda/src/error.rs new file mode 100644 index 00000000..01fad023 --- /dev/null +++ b/crates/j2k-cuda/src/error.rs @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::J2kError; +use j2k_core::{BackendRequest, BufferError, CodecError}; + +/// Error returned by the CUDA JPEG 2000 adapter. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + /// CPU JPEG 2000 decode failed. + #[error(transparent)] + Decode(#[from] J2kError), + /// Caller-owned output buffers were invalid. + #[error(transparent)] + Buffer(#[from] BufferError), + /// Backend request is unsupported by this adapter. + #[error("backend request {request:?} is not supported by j2k-cuda")] + UnsupportedBackend { + /// Requested backend. + request: BackendRequest, + }, + /// CUDA request is unsupported by the strict CUDA adapter contract. + #[error("unsupported CUDA request: {reason}")] + UnsupportedCudaRequest { + /// Human-readable rejection reason. + reason: &'static str, + }, + /// CUDA runtime or device is unavailable. + #[error("CUDA is unavailable on this host")] + CudaUnavailable, + #[cfg(feature = "cuda-runtime")] + /// CUDA runtime returned an error. + #[error("CUDA runtime error: {message}")] + CudaRuntime { + /// Runtime error message. + message: String, + }, +} + +impl CodecError for Error { + fn is_truncated(&self) -> bool { + matches!(self, Self::Decode(inner) if inner.is_truncated()) + } + + fn is_not_implemented(&self) -> bool { + matches!(self, Self::Decode(inner) if inner.is_not_implemented()) + } + + fn is_unsupported(&self) -> bool { + matches!( + self, + Self::UnsupportedBackend { .. } + | Self::UnsupportedCudaRequest { .. } + | Self::CudaUnavailable + ) || matches!(self, Self::Decode(inner) if inner.is_unsupported()) + } + + fn is_buffer_error(&self) -> bool { + matches!(self, Self::Buffer(_)) + || matches!(self, Self::Decode(inner) if inner.is_buffer_error()) + } +} diff --git a/crates/j2k-cuda/src/lib.rs b/crates/j2k-cuda/src/lib.rs new file mode 100644 index 00000000..d0c112f3 --- /dev/null +++ b/crates/j2k-cuda/src/lib.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! CUDA-facing device-output adapter for `j2k`. +//! +//! This crate intentionally exposes the same backend-selection surface as the +//! Metal adapter. CPU and auto requests return host-backed surfaces. Strict +//! CUDA requests are reserved for CUDA-resident HTJ2K codestream decode and +//! CUDA-resident HTJ2K encode inputs; the CPU-decode-then-upload path is +//! exposed through explicit CPU-staged APIs. + +#![deny(missing_docs)] +#![warn(unreachable_pub)] + +mod codec; +mod decoder; +mod direct_plan; +mod encode; +mod error; +mod profile; +mod runtime; +mod session; +mod surface; + +pub use codec::Codec; +pub use decoder::J2kDecoder; +pub use direct_plan::{ + CudaHtj2kBandId, CudaHtj2kCodeBlock, CudaHtj2kDecodePlan, CudaHtj2kIdwtStep, CudaHtj2kRect, + CudaHtj2kStoreStep, CudaHtj2kSubband, CudaHtj2kTransform, +}; +#[cfg(feature = "cuda-runtime")] +#[doc(hidden)] +pub use encode::cuda_dwt53_output_to_j2k_for_test; +pub use encode::{ + encode_j2k_lossless_with_cuda, encode_j2k_lossless_with_cuda_and_profile, + CudaEncodeStageAccelerator, CudaEncodeStageTimings, +}; +#[cfg(feature = "cuda-runtime")] +pub use encode::{ + encode_lossless_from_cuda_buffer, encode_lossless_from_cuda_buffer_with_report, + encode_lossless_from_cuda_buffers, encode_lossless_from_cuda_buffers_with_report, + submit_lossless_from_cuda_buffer, submit_lossless_from_cuda_buffers, CudaLosslessEncodeOutcome, + CudaLosslessEncodeResidency, CudaLosslessEncodeTile, SubmittedJ2kLosslessCudaEncode, + SubmittedJ2kLosslessCudaEncodeBatch, +}; +pub use error::Error; +pub use j2k::{J2kContext, J2kScratchPool}; +pub use profile::{ + CudaHtj2kDecodeProfileDetail, CudaHtj2kEncodeProfileReport, CudaHtj2kProfileReport, +}; +pub use session::CudaSession; +pub use surface::{CudaSurface, CudaSurfaceStats, Surface, SurfaceResidency}; diff --git a/crates/j2k-cuda/src/profile.rs b/crates/j2k-cuda/src/profile.rs new file mode 100644 index 00000000..408aaca2 --- /dev/null +++ b/crates/j2k-cuda/src/profile.rs @@ -0,0 +1,561 @@ +// SPDX-License-Identifier: Apache-2.0 + +use core::fmt::Write as _; +use std::cell::RefCell; +use std::sync::OnceLock; +use std::time::Instant; + +use j2k_core::BackendKind; +use j2k_profile::{profile_stage_mode_from_env, ProfileStageMode}; + +use crate::SurfaceResidency; + +const PROFILE_ENV_VAR: &str = "J2K_PROFILE_STAGES"; +const CUDA_TRACE_ENV_VAR: &str = "J2K_CUDA_TRACE"; + +thread_local! { + static PROFILE_SUMMARY: RefCell = + RefCell::new(j2k_profile::ProfileSummary::default().emit_on_drop()); +} + +/// Detailed route-overhead timings for strict CUDA HTJ2K decode. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CudaHtj2kDecodeProfileDetail { + /// End-to-end profiled decode wall time. + pub wall_total_us: u128, + /// Sum of the reported decode stage timings. + pub stage_sum_us: u128, + /// CUDA table/resource upload time. + pub table_upload_us: u128, + /// CUDA compressed payload/resource upload time. + /// + /// This includes mixed resource upload calls that contain compressed + /// payload bytes plus decode metadata. Metadata-only job upload is not + /// split out until the CUDA runtime exposes separate timings. + pub payload_upload_us: u128, + /// CUDA decode job upload time, reserved as zero until split runtime timings exist. + pub job_upload_us: u128, + /// CUDA status download time, reserved as zero until split runtime timings exist. + pub status_d2h_us: u128, + /// CUDA output download time, reserved as zero until split runtime timings exist. + pub output_d2h_us: u128, + /// HT cleanup/refinement CUDA dispatch count. + pub ht_dispatch_count: usize, + /// Dequantization CUDA dispatch count. + pub dequant_dispatch_count: usize, + /// Inverse DWT CUDA dispatch count. + pub idwt_dispatch_count: usize, + /// Inverse MCT CUDA dispatch count. + pub mct_dispatch_count: usize, + /// Store/format conversion CUDA dispatch count. + pub store_dispatch_count: usize, +} + +/// Structured stage timings for a strict CUDA HTJ2K operation. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CudaHtj2kProfileReport { + /// CPU marker/box parse time. + pub parse_us: u128, + /// Native direct-plan construction time. + pub plan_us: u128, + /// Flat CUDA plan construction time. + pub flatten_us: u128, + /// Host-to-device upload time for payload and metadata. + pub h2d_us: u128, + /// HT cleanup kernel time. + pub ht_cleanup_us: u128, + /// HT refinement kernel time. + pub ht_refine_us: u128, + /// Dequantization kernel time. + pub dequant_us: u128, + /// Inverse DWT kernel time. + pub idwt_us: u128, + /// Inverse MCT kernel time. + pub mct_us: u128, + /// Store/format conversion kernel time. + pub store_us: u128, + /// Sum of measured decode stages. + /// + /// End-to-end wall time is reported in `detail.wall_total_us`. + pub total_us: u128, + /// Number of HTJ2K code blocks in the flat plan. + pub block_count: usize, + /// Number of compressed payload bytes uploaded to CUDA. + pub payload_bytes: usize, + /// Number of CUDA kernel dispatches. + pub dispatch_count: usize, + /// Surface residency represented by this profile. + pub residency: SurfaceResidency, + /// Detailed route-overhead profile for RCA. + pub detail: CudaHtj2kDecodeProfileDetail, +} + +impl CudaHtj2kProfileReport { + /// Emit the report using `J2K_PROFILE_STAGES`, when enabled. + pub fn emit(&self, path: &str) { + emit_htj2k_profile_row(path, self); + export_trace_if_requested(path, self); + } +} + +/// Structured stage timings for a strict CUDA HTJ2K encode operation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CudaHtj2kEncodeProfileReport { + /// Pixel deinterleave and level-shift CUDA stage time. + pub deinterleave_us: u128, + /// Forward MCT CUDA stage time. + pub mct_us: u128, + /// Forward DWT CUDA stage time. + pub dwt_us: u128, + /// Quantization CUDA stage time. + pub quantize_us: u128, + /// HTJ2K cleanup code-block encode CUDA stage time. + pub ht_encode_us: u128, + /// HTJ2K packetization CUDA stage time. + pub packetize_us: u128, + /// Total wall time for the measured encode call. + pub total_us: u128, + /// Input pixel byte count. + pub input_bytes: usize, + /// Output codestream byte count. + pub codestream_bytes: usize, + /// Number of HTJ2K code blocks encoded. + pub block_count: usize, + /// Number of CUDA kernel dispatches. + pub dispatch_count: usize, + /// Backend that satisfied the encode request. + pub backend: BackendKind, +} + +impl Default for CudaHtj2kEncodeProfileReport { + fn default() -> Self { + Self { + deinterleave_us: 0, + mct_us: 0, + dwt_us: 0, + quantize_us: 0, + ht_encode_us: 0, + packetize_us: 0, + total_us: 0, + input_bytes: 0, + codestream_bytes: 0, + block_count: 0, + dispatch_count: 0, + backend: BackendKind::Cpu, + } + } +} + +impl CudaHtj2kEncodeProfileReport { + /// Emit the report using `J2K_PROFILE_STAGES`, when enabled. + pub fn emit(&self, path: &str) { + emit_htj2k_encode_profile_row(path, self); + export_encode_trace_if_requested(path, self); + } +} + +pub(crate) type ProfileInstant = Instant; + +fn profile_stage_mode() -> ProfileStageMode { + static MODE: OnceLock = OnceLock::new(); + *MODE.get_or_init(|| profile_stage_mode_from_env(PROFILE_ENV_VAR)) +} + +pub(crate) fn profile_stages_enabled() -> bool { + profile_stage_mode() != ProfileStageMode::Disabled +} + +pub(crate) fn profile_now(enabled: bool) -> Option { + enabled.then(Instant::now) +} + +pub(crate) fn elapsed_us(start: Option) -> u128 { + start.map_or(0, |start| start.elapsed().as_micros()) +} + +#[cfg_attr(not(feature = "cuda-runtime"), allow(dead_code))] +pub(crate) fn add_payload_resource_upload_us( + report: &mut CudaHtj2kProfileReport, + elapsed_us: u128, +) { + report.h2d_us = report.h2d_us.saturating_add(elapsed_us); + report.detail.payload_upload_us = report.detail.payload_upload_us.saturating_add(elapsed_us); +} + +#[cfg_attr(not(feature = "cuda-runtime"), allow(dead_code))] +pub(crate) fn finalize_decode_total_us(report: &mut CudaHtj2kProfileReport) { + report.total_us = [ + report.parse_us, + report.plan_us, + report.flatten_us, + report.h2d_us, + report.ht_cleanup_us, + report.ht_refine_us, + report.dequant_us, + report.idwt_us, + report.mct_us, + report.store_us, + ] + .into_iter() + .fold(0u128, u128::saturating_add); + report.detail.stage_sum_us = report.total_us; +} + +pub(crate) fn emit_htj2k_profile_row(path: &str, report: &CudaHtj2kProfileReport) { + let parse_us = report.parse_us.to_string(); + let plan_us = report.plan_us.to_string(); + let flatten_us = report.flatten_us.to_string(); + let h2d_us = report.h2d_us.to_string(); + let ht_cleanup_us = report.ht_cleanup_us.to_string(); + let ht_refine_us = report.ht_refine_us.to_string(); + let dequant_us = report.dequant_us.to_string(); + let idwt_us = report.idwt_us.to_string(); + let mct_us = report.mct_us.to_string(); + let store_us = report.store_us.to_string(); + let total_us = report.total_us.to_string(); + let block_count = report.block_count.to_string(); + let payload_bytes = report.payload_bytes.to_string(); + let dispatch_count = report.dispatch_count.to_string(); + let residency = format!("{:?}", report.residency); + let wall_total_us = report.detail.wall_total_us.to_string(); + let stage_sum_us = report.detail.stage_sum_us.to_string(); + let table_upload_us = report.detail.table_upload_us.to_string(); + let payload_upload_us = report.detail.payload_upload_us.to_string(); + let job_upload_us = report.detail.job_upload_us.to_string(); + let status_d2h_us = report.detail.status_d2h_us.to_string(); + let output_d2h_us = report.detail.output_d2h_us.to_string(); + let ht_dispatch_count = report.detail.ht_dispatch_count.to_string(); + let dequant_dispatch_count = report.detail.dequant_dispatch_count.to_string(); + let idwt_dispatch_count = report.detail.idwt_dispatch_count.to_string(); + let mct_dispatch_count = report.detail.mct_dispatch_count.to_string(); + let store_dispatch_count = report.detail.store_dispatch_count.to_string(); + + j2k_profile::emit_profile_row( + profile_stage_mode(), + &PROFILE_SUMMARY, + "j2k", + "cuda_htj2k", + path, + &[ + ("parse_us", parse_us.as_str()), + ("plan_us", plan_us.as_str()), + ("flatten_us", flatten_us.as_str()), + ("h2d_us", h2d_us.as_str()), + ("ht_cleanup_us", ht_cleanup_us.as_str()), + ("ht_refine_us", ht_refine_us.as_str()), + ("dequant_us", dequant_us.as_str()), + ("idwt_us", idwt_us.as_str()), + ("mct_us", mct_us.as_str()), + ("store_us", store_us.as_str()), + ("total_us", total_us.as_str()), + ("block_count", block_count.as_str()), + ("payload_bytes", payload_bytes.as_str()), + ("dispatch_count", dispatch_count.as_str()), + ("residency", residency.as_str()), + ("wall_total_us", wall_total_us.as_str()), + ("stage_sum_us", stage_sum_us.as_str()), + ("table_upload_us", table_upload_us.as_str()), + ("payload_upload_us", payload_upload_us.as_str()), + ("job_upload_us", job_upload_us.as_str()), + ("status_d2h_us", status_d2h_us.as_str()), + ("output_d2h_us", output_d2h_us.as_str()), + ("ht_dispatch_count", ht_dispatch_count.as_str()), + ("dequant_dispatch_count", dequant_dispatch_count.as_str()), + ("idwt_dispatch_count", idwt_dispatch_count.as_str()), + ("mct_dispatch_count", mct_dispatch_count.as_str()), + ("store_dispatch_count", store_dispatch_count.as_str()), + ], + ); +} + +pub(crate) fn emit_htj2k_encode_profile_row(path: &str, report: &CudaHtj2kEncodeProfileReport) { + let deinterleave_us = report.deinterleave_us.to_string(); + let mct_us = report.mct_us.to_string(); + let dwt_us = report.dwt_us.to_string(); + let quantize_us = report.quantize_us.to_string(); + let ht_encode_us = report.ht_encode_us.to_string(); + let packetize_us = report.packetize_us.to_string(); + let total_us = report.total_us.to_string(); + let input_bytes = report.input_bytes.to_string(); + let codestream_bytes = report.codestream_bytes.to_string(); + let block_count = report.block_count.to_string(); + let dispatch_count = report.dispatch_count.to_string(); + let backend = format!("{:?}", report.backend); + + j2k_profile::emit_profile_row( + profile_stage_mode(), + &PROFILE_SUMMARY, + "j2k", + "cuda_htj2k_encode", + path, + &[ + ("deinterleave_us", deinterleave_us.as_str()), + ("mct_us", mct_us.as_str()), + ("dwt_us", dwt_us.as_str()), + ("quantize_us", quantize_us.as_str()), + ("ht_encode_us", ht_encode_us.as_str()), + ("packetize_us", packetize_us.as_str()), + ("total_us", total_us.as_str()), + ("input_bytes", input_bytes.as_str()), + ("codestream_bytes", codestream_bytes.as_str()), + ("block_count", block_count.as_str()), + ("dispatch_count", dispatch_count.as_str()), + ("backend", backend.as_str()), + ], + ); +} + +fn export_trace_if_requested(path: &str, report: &CudaHtj2kProfileReport) { + let Some(trace_path) = std::env::var_os(CUDA_TRACE_ENV_VAR) else { + return; + }; + let trace = chrome_trace_json(path, report); + if let Err(error) = std::fs::write(&trace_path, trace) { + std::eprintln!("j2k_profile codec=j2k op=cuda_htj2k_trace path=cuda error={error}"); + } +} + +fn chrome_trace_json(path: &str, report: &CudaHtj2kProfileReport) -> String { + let stages = [ + ("parse", report.parse_us), + ("plan", report.plan_us), + ("flatten", report.flatten_us), + ("h2d", report.h2d_us), + ("ht_cleanup", report.ht_cleanup_us), + ("ht_refine", report.ht_refine_us), + ("dequant", report.dequant_us), + ("idwt", report.idwt_us), + ("mct", report.mct_us), + ("store", report.store_us), + ]; + let mut trace = String::from("{\"traceEvents\":["); + let mut ts = 0u128; + for (index, (name, dur)) in stages.iter().enumerate() { + if index != 0 { + trace.push(','); + } + let event_ts = if *name == "ht_refine" { + ts.saturating_sub(report.ht_cleanup_us) + } else { + ts + }; + write!( + trace, + "{{\"name\":\"{name}\",\"cat\":\"{path}\",\"ph\":\"X\",\"pid\":1,\"tid\":1,\"ts\":{event_ts},\"dur\":{dur}}}" + ) + .expect("writing trace JSON to String failed"); + if *name != "ht_refine" { + ts = ts.saturating_add(*dur); + } + } + trace.push_str("]}"); + trace +} + +fn export_encode_trace_if_requested(path: &str, report: &CudaHtj2kEncodeProfileReport) { + let Some(trace_path) = std::env::var_os(CUDA_TRACE_ENV_VAR) else { + return; + }; + let trace = chrome_encode_trace_json(path, report); + if let Err(error) = std::fs::write(&trace_path, trace) { + std::eprintln!("j2k_profile codec=j2k op=cuda_htj2k_encode_trace path=cuda error={error}"); + } +} + +fn chrome_encode_trace_json(path: &str, report: &CudaHtj2kEncodeProfileReport) -> String { + let stages = [ + ("deinterleave", report.deinterleave_us), + ("mct", report.mct_us), + ("dwt", report.dwt_us), + ("quantize", report.quantize_us), + ("ht_encode", report.ht_encode_us), + ("packetize", report.packetize_us), + ]; + let mut trace = String::from("{\"traceEvents\":["); + let mut ts = 0u128; + for (index, (name, dur)) in stages.iter().enumerate() { + if index != 0 { + trace.push(','); + } + write!( + trace, + "{{\"name\":\"{name}\",\"cat\":\"{path}\",\"ph\":\"X\",\"pid\":1,\"tid\":1,\"ts\":{ts},\"dur\":{dur}}}" + ) + .expect("writing trace JSON to String failed"); + ts = ts.saturating_add(*dur); + } + trace.push_str("]}"); + trace +} + +#[cfg(test)] +mod tests { + use super::{ + add_payload_resource_upload_us, chrome_encode_trace_json, chrome_trace_json, + finalize_decode_total_us, CudaHtj2kDecodeProfileDetail, CudaHtj2kEncodeProfileReport, + CudaHtj2kProfileReport, + }; + use j2k_core::BackendKind; + + use crate::SurfaceResidency; + + #[test] + fn finalize_decode_total_us_includes_cpu_and_cuda_stages() { + let mut report = CudaHtj2kProfileReport { + parse_us: 1, + plan_us: 2, + flatten_us: 3, + h2d_us: 4, + ht_cleanup_us: 5, + ht_refine_us: 6, + dequant_us: 7, + idwt_us: 8, + mct_us: 9, + store_us: 10, + total_us: 3, + block_count: 1, + payload_bytes: 2, + dispatch_count: 3, + residency: SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + }; + + finalize_decode_total_us(&mut report); + + assert_eq!(report.total_us, 55); + assert_eq!(report.detail.stage_sum_us, 55); + } + + #[test] + fn detailed_decode_profile_separates_wall_and_stage_sum() { + let mut report = CudaHtj2kProfileReport { + parse_us: 1, + plan_us: 2, + flatten_us: 3, + h2d_us: 4, + ht_cleanup_us: 5, + ht_refine_us: 5, + dequant_us: 6, + idwt_us: 7, + mct_us: 8, + store_us: 9, + total_us: 0, + block_count: 10, + payload_bytes: 11, + dispatch_count: 12, + residency: SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + }; + report.detail.wall_total_us = 100; + report.detail.table_upload_us = 13; + report.detail.payload_upload_us = 17; + report.detail.ht_dispatch_count = 2; + finalize_decode_total_us(&mut report); + + assert_eq!(report.detail.wall_total_us, 100); + assert_eq!(report.detail.stage_sum_us, report.total_us); + assert_eq!(report.detail.ht_dispatch_count, 2); + } + + #[test] + fn payload_resource_upload_detail_does_not_claim_job_status_split() { + let mut report = CudaHtj2kProfileReport::default(); + + add_payload_resource_upload_us(&mut report, 23); + + assert_eq!(report.h2d_us, 23); + assert_eq!(report.detail.payload_upload_us, 23); + assert_eq!(report.detail.job_upload_us, 0); + assert_eq!(report.detail.status_d2h_us, 0); + assert_eq!(report.detail.output_d2h_us, 0); + } + + #[test] + fn decode_trace_json_contains_ordered_stage_spans() { + let report = CudaHtj2kProfileReport { + parse_us: 1, + plan_us: 2, + flatten_us: 3, + h2d_us: 4, + ht_cleanup_us: 5, + ht_refine_us: 6, + dequant_us: 7, + idwt_us: 8, + mct_us: 9, + store_us: 10, + total_us: 55, + block_count: 1, + payload_bytes: 2, + dispatch_count: 3, + residency: SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + }; + + let trace = chrome_trace_json("decode", &report); + + assert!(trace.starts_with("{\"traceEvents\":[")); + assert!(trace.contains("\"name\":\"parse\",\"cat\":\"decode\",\"ph\":\"X\"")); + assert!(trace.contains("\"name\":\"ht_cleanup\",\"cat\":\"decode\",\"ph\":\"X\"")); + assert!(trace.contains("\"name\":\"store\",\"cat\":\"decode\",\"ph\":\"X\"")); + assert!(trace.contains("\"ts\":0,\"dur\":1")); + assert!(trace.contains("\"ts\":39,\"dur\":10")); + assert!(trace.ends_with("]}")); + } + + #[test] + fn decode_trace_json_does_not_advance_time_for_fused_refinement() { + let report = CudaHtj2kProfileReport { + parse_us: 1, + plan_us: 2, + flatten_us: 3, + h2d_us: 4, + ht_cleanup_us: 5, + ht_refine_us: 5, + dequant_us: 6, + idwt_us: 7, + mct_us: 8, + store_us: 9, + total_us: 45, + block_count: 1, + payload_bytes: 2, + dispatch_count: 3, + residency: SurfaceResidency::CudaResidentDecode, + detail: CudaHtj2kDecodeProfileDetail::default(), + }; + + let trace = chrome_trace_json("decode", &report); + + assert!(trace.contains("\"name\":\"ht_refine\",\"cat\":\"decode\",\"ph\":\"X\"")); + assert!(trace.contains("\"name\":\"ht_refine\",\"cat\":\"decode\",\"ph\":\"X\",\"pid\":1,\"tid\":1,\"ts\":10,\"dur\":5")); + assert!(trace.contains("\"name\":\"dequant\",\"cat\":\"decode\",\"ph\":\"X\",\"pid\":1,\"tid\":1,\"ts\":15,\"dur\":6")); + assert!(trace.contains("\"name\":\"store\",\"cat\":\"decode\",\"ph\":\"X\",\"pid\":1,\"tid\":1,\"ts\":36,\"dur\":9")); + } + + #[test] + fn encode_trace_json_contains_ordered_stage_spans() { + let report = CudaHtj2kEncodeProfileReport { + deinterleave_us: 11, + mct_us: 12, + dwt_us: 13, + quantize_us: 14, + ht_encode_us: 15, + packetize_us: 16, + total_us: 81, + input_bytes: 100, + codestream_bytes: 50, + block_count: 4, + dispatch_count: 6, + backend: BackendKind::Cuda, + }; + + let trace = chrome_encode_trace_json("encode", &report); + + assert!(trace.starts_with("{\"traceEvents\":[")); + assert!(trace.contains("\"name\":\"deinterleave\",\"cat\":\"encode\",\"ph\":\"X\"")); + assert!(trace.contains("\"name\":\"ht_encode\",\"cat\":\"encode\",\"ph\":\"X\"")); + assert!(trace.contains("\"name\":\"packetize\",\"cat\":\"encode\",\"ph\":\"X\"")); + assert!(trace.contains("\"ts\":0,\"dur\":11")); + assert!(trace.contains("\"ts\":65,\"dur\":16")); + assert!(trace.ends_with("]}")); + } +} diff --git a/crates/signinum-j2k-cuda/src/runtime.rs b/crates/j2k-cuda/src/runtime.rs similarity index 67% rename from crates/signinum-j2k-cuda/src/runtime.rs rename to crates/j2k-cuda/src/runtime.rs index 857de420..141d438b 100644 --- a/crates/signinum-j2k-cuda/src/runtime.rs +++ b/crates/j2k-cuda/src/runtime.rs @@ -1,11 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_core::{BackendKind, BackendRequest, PixelFormat}; +use j2k_core::{BackendKind, BackendRequest, PixelFormat}; #[cfg(feature = "cuda-runtime")] -use signinum_cuda_runtime::CudaError; +use j2k_cuda_runtime::CudaError; use crate::surface::Storage; -use crate::{profile, CudaSession, CudaSurfaceStats, Error, Surface}; +use crate::{CudaSession, CudaSurfaceStats, Error, Surface, SurfaceResidency}; + +const CPU_STAGED_CUDA_REQUIRES_EXPLICIT_API: &str = + "CPU-staged CUDA upload requires the explicit CPU-staged API; BackendRequest::Cuda only accepts resident CUDA HTJ2K decode"; pub(crate) fn wrap_surface( bytes: Vec, @@ -18,14 +21,13 @@ pub(crate) fn wrap_surface( let pitch_bytes = dimensions.0 as usize * fmt.bytes_per_pixel(); match backend { BackendRequest::Cpu | BackendRequest::Auto => { - if profile::gpu_route_profile_enabled() { + if j2k_profile::gpu_route_profile_enabled() { let request_s = format!("{backend:?}"); let fmt_s = format!("{fmt:?}"); let width_s = dimensions.0.to_string(); let height_s = dimensions.1.to_string(); - profile::emit_gpu_route_profile( + j2k_profile::emit_gpu_route_profile( "j2k", - "gpu_route", "cuda", &[ ("op", "wrap_surface"), @@ -39,6 +41,7 @@ pub(crate) fn wrap_surface( } Ok(Surface { backend: BackendKind::Cpu, + residency: SurfaceResidency::Host, dimensions, fmt, pitch_bytes, @@ -46,16 +49,29 @@ pub(crate) fn wrap_surface( storage: Storage::Host(bytes), }) } - BackendRequest::Cuda => wrap_cuda_surface(&bytes, dimensions, fmt, pitch_bytes, session), + BackendRequest::Cuda => { + let _ = (bytes, session); + Err(Error::UnsupportedCudaRequest { + reason: CPU_STAGED_CUDA_REQUIRES_EXPLICIT_API, + }) + } BackendRequest::Metal => Err(Error::UnsupportedBackend { request: backend }), } } +pub(crate) fn wrap_cpu_staged_cuda_surface( + bytes: &[u8], + dimensions: (u32, u32), + fmt: PixelFormat, + session: &mut CudaSession, +) -> Result { + let pitch_bytes = dimensions.0 as usize * fmt.bytes_per_pixel(); + wrap_cuda_surface(bytes, dimensions, fmt, pitch_bytes, session) +} + pub(crate) fn validate_surface_request(backend: BackendRequest) -> Result<(), Error> { - match backend { - BackendRequest::Cpu | BackendRequest::Auto | BackendRequest::Cuda => Ok(()), - BackendRequest::Metal => Err(Error::UnsupportedBackend { request: backend }), - } + j2k_core::validate_cuda_surface_backend_request(backend) + .map_err(|request| Error::UnsupportedBackend { request }) } #[cfg(feature = "cuda-runtime")] @@ -69,14 +85,13 @@ fn wrap_cuda_surface( let context = session.cuda_context()?; let output = context.copy_with_kernel(bytes).map_err(cuda_error)?; let (buffer, stats) = output.into_parts(); - if profile::gpu_route_profile_enabled() { + if j2k_profile::gpu_route_profile_enabled() { let fmt_s = format!("{fmt:?}"); let width_s = dimensions.0.to_string(); let height_s = dimensions.1.to_string(); let kernel_dispatches_s = stats.kernel_dispatches().to_string(); - profile::emit_gpu_route_profile( + j2k_profile::emit_gpu_route_profile( "j2k", - "gpu_route", "cuda", &[ ("op", "wrap_surface"), @@ -91,11 +106,14 @@ fn wrap_cuda_surface( } Ok(Surface { backend: BackendKind::Cuda, + residency: SurfaceResidency::CpuStagedCudaUpload, dimensions, fmt, pitch_bytes, stats: CudaSurfaceStats { - kernel_dispatches: stats.kernel_dispatches(), + total: stats.kernel_dispatches(), + copy: stats.copy_kernel_dispatches(), + decode: stats.decode_kernel_dispatches(), }, storage: Storage::Cuda(buffer), }) @@ -109,13 +127,12 @@ fn wrap_cuda_surface( _pitch_bytes: usize, _session: &mut CudaSession, ) -> Result { - if profile::gpu_route_profile_enabled() { + if j2k_profile::gpu_route_profile_enabled() { let fmt_s = format!("{fmt:?}"); let width_s = dimensions.0.to_string(); let height_s = dimensions.1.to_string(); - profile::emit_gpu_route_profile( + j2k_profile::emit_gpu_route_profile( "j2k", - "gpu_route", "cuda", &[ ("op", "wrap_surface"), @@ -131,11 +148,13 @@ fn wrap_cuda_surface( } #[cfg(feature = "cuda-runtime")] +#[allow(clippy::needless_pass_by_value)] pub(crate) fn cuda_error(error: CudaError) -> Error { - match error { - CudaError::Unavailable { .. } => Error::CudaUnavailable, - other => Error::CudaRuntime { - message: other.to_string(), - }, + if error.is_unavailable() { + Error::CudaUnavailable + } else { + Error::CudaRuntime { + message: error.to_string(), + } } } diff --git a/crates/j2k-cuda/src/session.rs b/crates/j2k-cuda/src/session.rs new file mode 100644 index 00000000..d9805a68 --- /dev/null +++ b/crates/j2k-cuda/src/session.rs @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(feature = "cuda-runtime")] +use j2k_cuda_runtime::{ + CudaBufferPool, CudaContext, CudaHtj2kDecodeTableResources, CudaHtj2kDecodeTables, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_native::{ht_uvlc_table0, ht_uvlc_table1, ht_vlc_table0, ht_vlc_table1}; +#[cfg(all(test, feature = "cuda-runtime"))] +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[cfg(feature = "cuda-runtime")] +use crate::runtime::cuda_error; +#[cfg(feature = "cuda-runtime")] +use crate::Error; + +#[cfg(all(test, feature = "cuda-runtime"))] +static HTJ2K_DECODE_TABLE_UPLOADS: AtomicUsize = AtomicUsize::new(0); + +/// Mutable CUDA adapter session reused across submissions. +#[derive(Clone, Default)] +pub struct CudaSession { + submissions: u64, + #[cfg(feature = "cuda-runtime")] + context: Option, + #[cfg(feature = "cuda-runtime")] + htj2k_decode_tables: Option, + #[cfg(feature = "cuda-runtime")] + decode_buffer_pool: Option, + #[cfg(feature = "cuda-runtime")] + decode_batch_buffer_pool: Option, +} + +impl CudaSession { + /// Number of submissions recorded by this session. + pub fn submissions(&self) -> u64 { + self.submissions + } + + #[cfg(feature = "cuda-runtime")] + /// True when a CUDA runtime context has been initialized. + pub fn is_runtime_initialized(&self) -> bool { + self.context.is_some() + } + + #[cfg(feature = "cuda-runtime")] + pub(crate) fn cuda_context(&mut self) -> Result { + if self.context.is_none() { + self.context = Some(CudaContext::system_default().map_err(cuda_error)?); + } + self.context.clone().ok_or(Error::CudaUnavailable) + } + + #[cfg(feature = "cuda-runtime")] + pub(crate) fn htj2k_decode_table_resources( + &mut self, + ) -> Result { + if let Some(tables) = &self.htj2k_decode_tables { + return Ok(tables.clone()); + } + + let context = self.cuda_context()?; + let tables = CudaHtj2kDecodeTables { + vlc_table0: ht_vlc_table0(), + vlc_table1: ht_vlc_table1(), + uvlc_table0: ht_uvlc_table0(), + uvlc_table1: ht_uvlc_table1(), + }; + let resources = context + .upload_htj2k_decode_table_resources(tables) + .map_err(cuda_error)?; + #[cfg(test)] + HTJ2K_DECODE_TABLE_UPLOADS.fetch_add(1, Ordering::Relaxed); + self.htj2k_decode_tables = Some(resources.clone()); + Ok(resources) + } + + #[cfg(feature = "cuda-runtime")] + pub(crate) fn decode_buffer_pool(&mut self) -> Result { + if let Some(pool) = &self.decode_buffer_pool { + return Ok(pool.clone()); + } + let context = self.cuda_context()?; + let pool = context.buffer_pool(); + self.decode_buffer_pool = Some(pool.clone()); + Ok(pool) + } + + #[cfg(feature = "cuda-runtime")] + pub(crate) fn decode_batch_buffer_pool(&mut self) -> Result { + if let Some(pool) = &self.decode_batch_buffer_pool { + return Ok(pool.clone()); + } + let context = self.cuda_context()?; + let pool = context.best_fit_buffer_pool(); + self.decode_batch_buffer_pool = Some(pool.clone()); + Ok(pool) + } +} + +impl j2k_core::DeviceSubmitSession for CudaSession { + fn record_submit(&mut self) { + self.submissions = self.submissions.saturating_add(1); + } +} + +impl j2k_core::AcceleratorSession for CudaSession { + fn backend_kind(&self) -> j2k_core::BackendKind { + j2k_core::BackendKind::Cuda + } + + fn execution_stats(&self) -> j2k_core::ExecutionStats { + j2k_core::ExecutionStats { + submissions: self.submissions, + ..j2k_core::ExecutionStats::default() + } + } +} + +#[cfg(all(test, feature = "cuda-runtime"))] +pub(crate) fn reset_htj2k_decode_table_uploads_for_test() { + HTJ2K_DECODE_TABLE_UPLOADS.store(0, Ordering::Relaxed); +} + +#[cfg(all(test, feature = "cuda-runtime"))] +pub(crate) fn htj2k_decode_table_uploads_for_test() -> usize { + HTJ2K_DECODE_TABLE_UPLOADS.load(Ordering::Relaxed) +} + +impl std::fmt::Debug for CudaSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut debug = f.debug_struct("CudaSession"); + debug.field("submissions", &self.submissions); + #[cfg(feature = "cuda-runtime")] + debug.field("runtime_initialized", &self.is_runtime_initialized()); + #[cfg(feature = "cuda-runtime")] + debug.field( + "htj2k_decode_tables_cached", + &self.htj2k_decode_tables.is_some(), + ); + #[cfg(feature = "cuda-runtime")] + debug.field( + "decode_buffer_pool_cached", + &self.decode_buffer_pool.is_some(), + ); + #[cfg(feature = "cuda-runtime")] + debug.field( + "decode_batch_buffer_pool_cached", + &self.decode_batch_buffer_pool.is_some(), + ); + debug.finish_non_exhaustive() + } +} + +#[cfg(all(test, feature = "cuda-runtime"))] +mod tests { + use super::CudaSession; + use crate::Error; + + fn cuda_required() -> bool { + std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_some() + } + + #[test] + fn htj2k_decode_tables_are_uploaded_once_per_session() { + crate::session::reset_htj2k_decode_table_uploads_for_test(); + let mut session = CudaSession::default(); + + let first = session.htj2k_decode_table_resources(); + if matches!( + first, + Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) + ) && !cuda_required() + { + return; + } + first.expect("first HTJ2K decode table upload"); + session + .htj2k_decode_table_resources() + .expect("cached HTJ2K decode tables"); + + assert_eq!(crate::session::htj2k_decode_table_uploads_for_test(), 1); + } + + #[test] + fn cuda_session_reuses_one_decode_buffer_pool_when_required() { + let mut session = CudaSession::default(); + + let first = session.decode_buffer_pool(); + if matches!( + first, + Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) + ) && !cuda_required() + { + return; + } + let first = first.expect("first decode buffer pool"); + let second = session + .decode_buffer_pool() + .expect("cached decode buffer pool"); + { + let buffer = first.take(16).expect("pooled decode buffer"); + assert_eq!(buffer.byte_len(), 16); + } + + assert!(second.cached_count().expect("shared pool cached count") >= 1); + } +} diff --git a/crates/j2k-cuda/src/surface.rs b/crates/j2k-cuda/src/surface.rs new file mode 100644 index 00000000..bc1ac63b --- /dev/null +++ b/crates/j2k-cuda/src/surface.rs @@ -0,0 +1,431 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(feature = "cuda-runtime")] +use std::sync::Arc; + +use j2k_core::{ + copy_tight_pixels_to_strided_output, BackendKind, BufferError, DeviceMemoryRange, + DeviceSurface, ExecutionStats, PixelFormat, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_cuda_runtime::CudaDeviceBuffer; + +#[cfg(feature = "cuda-runtime")] +use crate::runtime::cuda_error; +use crate::Error; + +#[derive(Debug)] +pub(crate) enum Storage { + Host(Vec), + #[cfg(feature = "cuda-runtime")] + Cuda(CudaDeviceBuffer), + #[cfg(feature = "cuda-runtime")] + CudaRange { + buffer: Arc, + offset: usize, + len: usize, + }, +} + +/// CUDA surface execution counters. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CudaSurfaceStats { + pub(crate) total: usize, + pub(crate) copy: usize, + pub(crate) decode: usize, +} + +impl CudaSurfaceStats { + /// Total CUDA kernel dispatches associated with the surface. + pub fn kernel_dispatches(self) -> usize { + self.total + } + + /// CUDA copy/upload kernel dispatches associated with the surface. + pub fn copy_kernel_dispatches(self) -> usize { + self.copy + } + + /// CUDA codestream decode kernel dispatches associated with the surface. + pub fn decode_kernel_dispatches(self) -> usize { + self.decode + } +} + +/// Borrowed view of a CUDA-resident surface. +#[derive(Clone, Copy, Debug)] +pub struct CudaSurface<'a> { + #[cfg(feature = "cuda-runtime")] + buffer: &'a CudaDeviceBuffer, + #[cfg(feature = "cuda-runtime")] + offset: usize, + #[cfg(not(feature = "cuda-runtime"))] + _marker: core::marker::PhantomData<&'a ()>, + pub(crate) stats: CudaSurfaceStats, +} + +impl CudaSurface<'_> { + /// Raw CUDA device pointer value. + pub fn device_ptr(&self) -> u64 { + #[cfg(feature = "cuda-runtime")] + { + self.buffer.device_ptr().saturating_add(self.offset as u64) + } + #[cfg(not(feature = "cuda-runtime"))] + { + unreachable!("CudaSurface cannot be constructed without cuda-runtime support") + } + } + + /// Execution counters for this surface. + pub fn stats(&self) -> CudaSurfaceStats { + self.stats + } +} + +/// Residency of a decoded J2K CUDA adapter surface. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[non_exhaustive] +pub enum SurfaceResidency { + /// Pixels are stored in host memory. + #[default] + Host, + /// Pixels were produced directly by a CUDA codestream decode path. + CudaResidentDecode, + /// Pixels were decoded on CPU and uploaded into a CUDA buffer. + CpuStagedCudaUpload, +} + +/// Host- or CUDA-backed decoded surface. +#[derive(Debug)] +pub struct Surface { + pub(crate) backend: BackendKind, + pub(crate) residency: SurfaceResidency, + pub(crate) dimensions: (u32, u32), + pub(crate) fmt: PixelFormat, + pub(crate) pitch_bytes: usize, + pub(crate) stats: CudaSurfaceStats, + pub(crate) storage: Storage, +} + +impl Surface { + /// Return where the surface's pixels currently reside. + pub fn residency(&self) -> SurfaceResidency { + self.residency + } + + /// Row pitch in bytes. + pub fn pitch_bytes(&self) -> usize { + self.pitch_bytes + } + + /// Borrow host bytes when the surface is host-backed. + pub fn as_host_bytes(&self) -> Option<&[u8]> { + match &self.storage { + Storage::Host(bytes) => Some(bytes), + #[cfg(feature = "cuda-runtime")] + Storage::Cuda(_) | Storage::CudaRange { .. } => None, + } + } + + /// Download or copy the surface into caller-owned strided output. + pub fn download_into(&self, out: &mut [u8], stride: usize) -> Result<(), Error> { + match &self.storage { + Storage::Host(bytes) => { + copy_tight_pixels_to_strided_output(bytes, self.dimensions, self.fmt, out, stride) + .map_err(Error::from) + } + #[cfg(feature = "cuda-runtime")] + Storage::Cuda(buffer) => { + let byte_len = self.byte_len(); + if let Some(len) = + tight_cuda_download_len(byte_len, self.pitch_bytes, stride, out.len()) + { + return buffer.copy_to_host(&mut out[..len]).map_err(cuda_error); + } + let mut tight = vec![0u8; byte_len]; + buffer.copy_to_host(&mut tight).map_err(cuda_error)?; + copy_tight_pixels_to_strided_output(&tight, self.dimensions, self.fmt, out, stride) + .map_err(Error::from) + } + #[cfg(feature = "cuda-runtime")] + Storage::CudaRange { + buffer, + offset, + len, + } => { + let byte_len = self.byte_len(); + debug_assert_eq!(*len, byte_len); + if let Some(len) = + tight_cuda_download_len(byte_len, self.pitch_bytes, stride, out.len()) + { + return buffer + .copy_range_to_host(*offset, &mut out[..len]) + .map_err(cuda_error); + } + let mut tight = vec![0u8; byte_len]; + buffer + .copy_range_to_host(*offset, &mut tight) + .map_err(cuda_error)?; + copy_tight_pixels_to_strided_output(&tight, self.dimensions, self.fmt, out, stride) + .map_err(Error::from) + } + } + } + + /// Download the surface and return elapsed host copy time in microseconds. + pub fn download_into_profiled(&self, out: &mut [u8], stride: usize) -> Result { + let started = std::time::Instant::now(); + self.download_into(out, stride)?; + Ok(started.elapsed().as_micros()) + } + + /// Borrow CUDA metadata when the surface is CUDA-backed. + pub fn cuda_surface(&self) -> Option> { + #[cfg(feature = "cuda-runtime")] + match &self.storage { + Storage::Cuda(buffer) => Some(CudaSurface { + buffer, + offset: 0, + stats: self.stats, + }), + Storage::CudaRange { buffer, offset, .. } => Some(CudaSurface { + buffer, + offset: *offset, + stats: self.stats, + }), + Storage::Host(_) => None, + } + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = self.stats; + None + } + } + + /// Download a sequence of surfaces into a tightly concatenated output buffer. + /// + /// CUDA surfaces produced from one contiguous batch allocation are copied + /// with one device-to-host transfer. Other layouts fall back to downloading + /// each surface tightly in order. + pub fn download_batch_tight(surfaces: &[Self]) -> Result, Error> { + let required = batch_tight_required_len(surfaces)?; + if required == 0 { + return Ok(Vec::new()); + } + + #[cfg(feature = "cuda-runtime")] + if let Some((buffer, offset)) = contiguous_cuda_batch_range(surfaces) { + let mut out = Vec::with_capacity(required); + buffer + .copy_range_to_host_uninit(offset, out.spare_capacity_mut()) + .map_err(cuda_error)?; + // SAFETY: the CUDA copy above initialized exactly `required` + // bytes in this Vec's spare capacity and returned success. + unsafe { + out.set_len(required); + } + return Ok(out); + } + + let mut out = vec![0u8; required]; + Self::download_batch_tight_into(surfaces, &mut out)?; + Ok(out) + } + + /// Download a sequence of surfaces into a tightly concatenated output buffer. + /// + /// CUDA surfaces produced from one contiguous batch allocation are copied + /// with one device-to-host transfer. Other layouts fall back to downloading + /// each surface tightly in order. + pub fn download_batch_tight_into(surfaces: &[Self], out: &mut [u8]) -> Result<(), Error> { + let required = batch_tight_required_len(surfaces)?; + if out.len() < required { + return Err(BufferError::OutputTooSmall { + required, + have: out.len(), + } + .into()); + } + if required == 0 { + return Ok(()); + } + + #[cfg(feature = "cuda-runtime")] + if let Some((buffer, offset)) = contiguous_cuda_batch_range(surfaces) { + return buffer + .copy_range_to_host(offset, &mut out[..required]) + .map_err(cuda_error); + } + + let mut cursor = 0usize; + for surface in surfaces { + let len = surface.byte_len(); + surface.download_into(&mut out[cursor..cursor + len], surface.pitch_bytes)?; + cursor += len; + } + Ok(()) + } +} + +fn batch_tight_required_len(surfaces: &[Surface]) -> Result { + surfaces + .iter() + .try_fold(0usize, |sum, surface| sum.checked_add(surface.byte_len())) + .ok_or(BufferError::SizeOverflow { + what: "tight batch surface output", + }) + .map_err(Error::from) +} + +#[cfg(feature = "cuda-runtime")] +pub(crate) fn cuda_range_storage( + buffer: Arc, + offset: usize, + len: usize, +) -> Storage { + Storage::CudaRange { + buffer, + offset, + len, + } +} + +#[cfg(feature = "cuda-runtime")] +fn contiguous_cuda_batch_range(surfaces: &[Surface]) -> Option<(&CudaDeviceBuffer, usize)> { + let first = surfaces.first()?; + let Storage::CudaRange { + buffer, + offset, + len, + } = &first.storage + else { + return None; + }; + let first_buffer = buffer; + let first_offset = *offset; + let mut expected_offset = first_offset.checked_add(*len)?; + for surface in &surfaces[1..] { + let Storage::CudaRange { + buffer, + offset, + len, + } = &surface.storage + else { + return None; + }; + if !Arc::ptr_eq(first_buffer, buffer) || *offset != expected_offset { + return None; + } + expected_offset = expected_offset.checked_add(*len)?; + } + Some((first_buffer.as_ref(), first_offset)) +} + +#[cfg(any(feature = "cuda-runtime", test))] +fn tight_cuda_download_len( + byte_len: usize, + pitch_bytes: usize, + stride: usize, + out_len: usize, +) -> Option { + (stride == pitch_bytes && out_len >= byte_len).then_some(byte_len) +} + +impl DeviceSurface for Surface { + fn backend_kind(&self) -> BackendKind { + self.backend + } + + fn residency(&self) -> j2k_core::SurfaceResidency { + match self.residency { + SurfaceResidency::Host => j2k_core::SurfaceResidency::Host, + SurfaceResidency::CudaResidentDecode => j2k_core::SurfaceResidency::CudaResidentDecode, + SurfaceResidency::CpuStagedCudaUpload => { + j2k_core::SurfaceResidency::CpuStagedCudaUpload + } + } + } + + fn dimensions(&self) -> (u32, u32) { + self.dimensions + } + + fn pixel_format(&self) -> PixelFormat { + self.fmt + } + + fn byte_len(&self) -> usize { + self.pitch_bytes * self.dimensions.1 as usize + } + + fn execution_stats(&self) -> ExecutionStats { + ExecutionStats { + kernel_dispatches: self.stats.total as u64, + ..ExecutionStats::default() + } + } + + fn memory_range(&self) -> Option { + match &self.storage { + Storage::Host(_) => None, + #[cfg(feature = "cuda-runtime")] + Storage::Cuda(buffer) => Some(DeviceMemoryRange::new( + BackendKind::Cuda, + buffer.device_ptr(), + 0, + self.byte_len(), + )), + #[cfg(feature = "cuda-runtime")] + Storage::CudaRange { + buffer, + offset, + len, + } => Some(DeviceMemoryRange::new( + BackendKind::Cuda, + buffer.device_ptr(), + *offset, + *len, + )), + } + } +} + +#[cfg(test)] +mod tests { + use super::{tight_cuda_download_len, CudaSurfaceStats, Storage, Surface, SurfaceResidency}; + use j2k_core::{BackendKind, PixelFormat}; + + #[test] + fn tight_cuda_download_len_accepts_exact_tight_output() { + assert_eq!(tight_cuda_download_len(32, 8, 8, 32), Some(32)); + } + + #[test] + fn download_batch_tight_returns_tightly_concatenated_host_surfaces() { + let surfaces = [ + Surface { + backend: BackendKind::Cpu, + residency: SurfaceResidency::Host, + dimensions: (2, 1), + fmt: PixelFormat::Gray8, + pitch_bytes: 2, + stats: CudaSurfaceStats::default(), + storage: Storage::Host(vec![1, 2]), + }, + Surface { + backend: BackendKind::Cpu, + residency: SurfaceResidency::Host, + dimensions: (1, 1), + fmt: PixelFormat::Rgb8, + pitch_bytes: 3, + stats: CudaSurfaceStats::default(), + storage: Storage::Host(vec![3, 4, 5]), + }, + ]; + + let tight = Surface::download_batch_tight(&surfaces).expect("batch download"); + + assert_eq!(tight, vec![1, 2, 3, 4, 5]); + } +} diff --git a/crates/j2k-cuda/tests/bench_harness.rs b/crates/j2k-cuda/tests/bench_harness.rs new file mode 100644 index 00000000..49688dcb --- /dev/null +++ b/crates/j2k-cuda/tests/bench_harness.rs @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[test] +fn cuda_htj2k_decode_bench_exposes_gray_rgb_rgba_rows() { + let bench = include_str!("../benches/htj2k_decode.rs"); + + for expected in [ + "cpu_gray8", + "cuda_gray8", + "cpu_rgb8", + "cuda_rgb8", + "cpu_rgba8", + "cuda_rgba8", + "j2k_cuda_htj2k_full_tile_decode", + "j2k_cuda_htj2k_roi_decode", + "j2k_cuda_htj2k_scaled_decode", + "j2k_cuda_htj2k_roi_scaled_decode", + "j2k_cuda_htj2k_tile_batch_decode", + "BATCH_SIZES", + "[8, 16, 32, 64]", + "J2K_CUDA_DECODE_BATCH_SIZES", + "J2K_CUDA_DECODE_FORMATS", + "J2K_REQUIRE_CUDA_BENCH", + ] { + assert!( + bench.contains(expected), + "CUDA HTJ2K decode benchmark is missing `{expected}`" + ); + } +} + +#[test] +fn cuda_htj2k_decode_bench_reuses_session_in_timed_cuda_rows() { + let bench = include_str!("../benches/htj2k_decode.rs"); + + assert!( + bench + .matches("let mut session = CudaSession::default();") + .count() + >= 6, + "CUDA decode benchmarks must create reusable sessions outside timed iterations" + ); + + for forbidden in [ + ".decode_to_device(case.fmt, BackendRequest::Cuda)", + ".decode_region_to_device(case.fmt, roi, BackendRequest::Cuda)", + ".decode_scaled_to_device(case.fmt, scale, BackendRequest::Cuda)", + ".decode_region_scaled_to_device(case.fmt, roi, scale, BackendRequest::Cuda)", + ] { + assert!( + !bench.contains(forbidden), + "CUDA decode benchmark timed row must not call context-creating helper `{forbidden}`" + ); + } + + for expected in [ + ".submit_to_device(", + ".submit_region_to_device(", + ".submit_scaled_to_device(", + ".submit_region_scaled_to_device(", + ] { + assert!( + bench.contains(expected), + "CUDA decode benchmark is missing reusable-session path `{expected}`" + ); + } +} + +#[test] +fn cuda_htj2k_tile_batch_bench_uses_cuda_batch_entrypoint() { + let bench = include_str!("../benches/htj2k_decode.rs"); + let batch_body = extract_function_body(bench, "fn bench_tile_batch"); + let cuda_branch_start = batch_body + .find("if case.cuda_available && cuda_batch_decode_supported(fmt)") + .expect("CUDA batch branch exists"); + let cuda_batch_body = &batch_body[cuda_branch_start..]; + + assert!( + cuda_batch_body.contains("J2kDecoder::decode_batch_to_device_with_session("), + "CUDA HTJ2K tile batch row must use a real batch decode entrypoint" + ); + assert!( + !cuda_batch_body.contains("Codec::submit_tile_to_device("), + "CUDA HTJ2K tile batch row must not submit one tile at a time" + ); +} + +#[test] +fn cuda_htj2k_decode_profile_example_uses_batch_entrypoint() { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/examples/htj2k_decode_profile.rs" + ); + let example = std::fs::read_to_string(path).expect("read CUDA HTJ2K profile example"); + + for expected in [ + "J2K_CUDA_PROFILE_BATCH_SIZE", + "J2K_CUDA_PROFILE_ITERATIONS", + "let mut session = CudaSession::default();", + "decode_batch_to_device_with_session(", + "mode=batch_no_download", + ] { + assert!( + example.contains(expected), + "CUDA HTJ2K profile example is missing `{expected}`" + ); + } + + assert!( + !example.contains(".decode_to_device_with_session(PixelFormat::Rgb8, &mut session)") + && !example.contains("for fixture in &fixtures"), + "CUDA HTJ2K profile example must not report batch_size while looping through single-tile decodes" + ); +} + +#[test] +fn cuda_htj2k_decode_steady_state_uses_untimed_runtime_path() { + let path = concat!(env!("CARGO_MANIFEST_DIR"), "/src/decoder.rs"); + let decoder = std::fs::read_to_string(path).expect("read CUDA HTJ2K decoder"); + + for expected in [ + "decode_to_cuda_resident_surface_with_profile_control(decoder, session, fmt, false)", + "decode_to_cuda_resident_surface_with_profile_impl(self, session, fmt)", + "decode_to_cuda_resident_surface_with_profile_control(decoder, session, fmt, true)", + "collect_stage_timings", + "decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool", + "j2k_inverse_dwt_single_device_untimed_with_pool", + "time_default_stream_named_us_if", + ] { + assert!( + decoder.contains(expected), + "steady-state CUDA HTJ2K decode path is missing `{expected}`" + ); + } +} + +#[test] +fn cuda_runtime_exposes_untimed_htj2k_decode_helpers() { + let runtime = read_cuda_runtime_sources(); + + for expected in [ + "pub fn synchronize(&self) -> Result<(), CudaError>", + "pub fn time_default_stream_named_us_if", + "pub fn decode_htj2k_codeblocks_with_resources_untimed", + "pub fn j2k_inverse_dwt_single_device_untimed", + "pinned_upload_staging", + "take_pinned_upload_staging", + "recycle_pinned_upload_staging", + "enum CudaLaunchMode", + "CudaLaunchMode::Async", + "fn launch_htj2k_decode_codeblocks(", + "fn launch_j2k_dequantize_htj2k_codeblocks(", + "fn launch_j2k_idwt_interleave(", + ] { + assert!( + runtime.contains(expected), + "CUDA runtime is missing steady-state decode helper `{expected}`" + ); + } +} + +fn read_cuda_runtime_sources() -> String { + let src_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../j2k-cuda-runtime/src"); + let mut runtime = String::new(); + + for module in [ + "lib.rs", + "context.rs", + "execution.rs", + "memory.rs", + "htj2k_decode.rs", + "j2k_decode.rs", + ] { + let path = format!("{src_dir}/{module}"); + runtime.push_str(&std::fs::read_to_string(&path).expect("read CUDA runtime module")); + runtime.push('\n'); + } + + runtime +} + +fn extract_function_body<'a>(source: &'a str, signature: &str) -> &'a str { + let start = source.find(signature).expect("function signature exists"); + let function = &source[start..]; + let mut depth = 0usize; + let mut saw_open = false; + for (index, ch) in function.char_indices() { + match ch { + '{' => { + saw_open = true; + depth = depth.saturating_add(1); + } + '}' if saw_open => { + depth = depth.saturating_sub(1); + if depth == 0 { + return &function[..=index]; + } + } + _ => {} + } + } + panic!("function body for `{signature}` is incomplete"); +} diff --git a/crates/j2k-cuda/tests/encode_stage_api.rs b/crates/j2k-cuda/tests/encode_stage_api.rs new file mode 100644 index 00000000..9cfa12ca --- /dev/null +++ b/crates/j2k-cuda/tests/encode_stage_api.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::adapter::encode_stage::{ + J2kEncodeStageAccelerator, J2kPacketizationEncodeJob, J2kPacketizationProgressionOrder, +}; +use j2k_cuda::{CudaEncodeStageAccelerator, CudaEncodeStageTimings}; + +#[cfg(feature = "cuda-runtime")] +use j2k::{ + encode_j2k_lossless, J2kBlockCodingMode, J2kEncodeValidation, J2kLosslessEncodeOptions, + J2kLosslessSamples, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_core::{CodecError, PixelFormat}; +#[cfg(feature = "cuda-runtime")] +use j2k_cuda::{ + encode_lossless_from_cuda_buffer, encode_lossless_from_cuda_buffer_with_report, + encode_lossless_from_cuda_buffers, submit_lossless_from_cuda_buffer, CudaLosslessEncodeTile, + CudaSession, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_cuda_runtime::CudaContext; + +#[test] +fn cuda_encode_stage_timings_are_publicly_readable_and_resettable() { + let mut accelerator = CudaEncodeStageAccelerator::with_profile_collection(true); + + assert_eq!( + accelerator.collected_stage_timings(), + CudaEncodeStageTimings::default() + ); + + accelerator.reset_collected_stage_timings(); + assert_eq!( + accelerator.collected_stage_timings(), + CudaEncodeStageTimings::default() + ); +} + +#[test] +fn cuda_encode_stage_can_prefer_cpu_packetization() { + let mut accelerator = CudaEncodeStageAccelerator::default().prefer_cpu_packetization(true); + let job = J2kPacketizationEncodeJob { + resolution_count: 0, + num_layers: 1, + num_components: 1, + code_block_count: 0, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &[], + resolutions: &[], + }; + + assert!(accelerator.encode_packetization(job).unwrap().is_none()); + assert_eq!(accelerator.packetization_attempts(), 1); + assert_eq!(accelerator.packetization_dispatches(), 0); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_lossless_device_buffer_api_shapes_are_public() { + fn assert_single_fn( + _f: for<'tile, 'options, 'session> fn( + CudaLosslessEncodeTile<'tile>, + &'options J2kLosslessEncodeOptions, + &'session mut CudaSession, + ) -> Result, + ) { + } + fn assert_single_report_fn( + _f: for<'tile, 'options, 'session> fn( + CudaLosslessEncodeTile<'tile>, + &'options J2kLosslessEncodeOptions, + &'session mut CudaSession, + ) -> Result< + j2k_cuda::CudaLosslessEncodeOutcome, + j2k_cuda::Error, + >, + ) { + } + fn assert_submit_fn( + _f: for<'tile, 'options, 'session> fn( + CudaLosslessEncodeTile<'tile>, + &'options J2kLosslessEncodeOptions, + &'session mut CudaSession, + ) -> Result< + j2k_cuda::SubmittedJ2kLosslessCudaEncode, + j2k_cuda::Error, + >, + ) { + } + type BatchEncodeFn = + for<'slice, 'tile, 'options, 'session> fn( + &'slice [CudaLosslessEncodeTile<'tile>], + &'options J2kLosslessEncodeOptions, + &'session mut CudaSession, + ) + -> Result, j2k_cuda::Error>; + fn assert_batch_fn(_f: BatchEncodeFn) {} + + assert_single_fn(encode_lossless_from_cuda_buffer); + assert_single_report_fn(encode_lossless_from_cuda_buffer_with_report); + assert_submit_fn(submit_lossless_from_cuda_buffer); + assert_batch_fn(encode_lossless_from_cuda_buffers); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_lossless_device_buffer_empty_batch_fails_clearly() { + let mut session = CudaSession::default(); + let options = J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_validation(J2kEncodeValidation::External); + + let error = encode_lossless_from_cuda_buffers(&[], &options, &mut session) + .expect_err("empty CUDA encode batch should fail"); + + assert!(error.is_unsupported()); + assert!( + error.to_string().contains("empty"), + "expected an empty-batch error, got {error}" + ); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_lossless_device_buffer_encode_matches_host_htj2k_when_required() { + if std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_none() { + return; + } + + let width = 8; + let height = 8; + let pixels: Vec = (0..width * height * 3) + .map(|i| u8::try_from((i * 17 + 11) % 251).expect("sample fits")) + .collect(); + let context = CudaContext::system_default().expect("CUDA context"); + let buffer = context.upload(&pixels).expect("upload source pixels"); + let tile = CudaLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width, + height, + pitch_bytes: width as usize * PixelFormat::Rgb8.bytes_per_pixel(), + output_width: width, + output_height: height, + format: PixelFormat::Rgb8, + }; + let options = J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_validation(J2kEncodeValidation::External); + let mut session = CudaSession::default(); + + let device = encode_lossless_from_cuda_buffer_with_report(tile, &options, &mut session) + .expect("CUDA device-buffer HTJ2K encode"); + let host = encode_j2k_lossless( + J2kLosslessSamples::new(&pixels, width, height, 3, 8, false).expect("host samples"), + &options, + ) + .expect("host HTJ2K encode"); + + assert_eq!(device.encoded.width, host.width); + assert_eq!(device.encoded.height, host.height); + assert_eq!(device.encoded.components, host.components); + assert_eq!(device.encoded.bit_depth, host.bit_depth); + assert!(device.resident.coefficient_prep_used); + assert!(device.resident.packetization_used); + assert!(!device.input_copy_used); + assert!(!device.encoded.codestream.is_empty()); +} diff --git a/crates/j2k-cuda/tests/fixtures/htj2k/LICENSE.OpenHTJ2K b/crates/j2k-cuda/tests/fixtures/htj2k/LICENSE.OpenHTJ2K new file mode 100644 index 00000000..d896ab36 --- /dev/null +++ b/crates/j2k-cuda/tests/fixtures/htj2k/LICENSE.OpenHTJ2K @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2019 - 2021, Osamu Watanabe +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/crates/j2k-cuda/tests/fixtures/htj2k/README.md b/crates/j2k-cuda/tests/fixtures/htj2k/README.md new file mode 100644 index 00000000..1c5d7e6f --- /dev/null +++ b/crates/j2k-cuda/tests/fixtures/htj2k/README.md @@ -0,0 +1,22 @@ +HTJ2K fixtures for native decoder coverage. + +These fixtures are from the OpenHTJ2K conformance data, copied from OpenHTJ2K +commit `ffe5acf9f1eedb87c36c3fd2134fdc1ddea5e75f`. They are tiny HTONLY +codestreams derived from the JPEG 2000 Part 4 / ITU-T T.803 HTJ2K conformance +set. + +`openhtj2k_ds0_ht_12_b11.j2k` is copied from `ds0_ht_12_b11.j2k`, blob +`cf3fb0bc7e55898b4e6977f38ba0d38d91c359bf`. The native decoder sees 8 HT code +blocks, 2 non-empty refinement jobs, and up to 3 HT coding passes. + +`openhtj2k_ds0_ht_09_b11.j2k` is copied from `ds0_ht_09_b11.j2k`, blob +`d4f2031359c32eb24825d00dde05a92cf3ae451e`. The native decoder sees 14 HT +code blocks, all with non-empty refinement jobs, and up to 3 HT coding passes. + +It is intentionally checked-in test data: tests must not invoke an external +encoder or decoder at runtime. + +The paired `.gray` file contains the expected 8-bit grayscale samples in +row-major order from decoding the checked-in codestream with OpenJPH. + +The OpenHTJ2K source license is retained in `LICENSE.OpenHTJ2K`. diff --git a/crates/j2k-cuda/tests/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray b/crates/j2k-cuda/tests/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray new file mode 100644 index 00000000..115938f1 Binary files /dev/null and b/crates/j2k-cuda/tests/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray differ diff --git a/crates/j2k-cuda/tests/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k b/crates/j2k-cuda/tests/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k new file mode 100644 index 00000000..d4f20313 Binary files /dev/null and b/crates/j2k-cuda/tests/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k differ diff --git a/crates/j2k-cuda/tests/host_surface.rs b/crates/j2k-cuda/tests/host_surface.rs new file mode 100644 index 00000000..bf5289c2 --- /dev/null +++ b/crates/j2k-cuda/tests/host_surface.rs @@ -0,0 +1,1479 @@ +use j2k_core::{ + BackendRequest, CodecError, DecoderContext, DeviceSubmission, DeviceSurface, Downscale, + ImageDecode, ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, Rect, TileBatchDecodeDevice, + TileBatchDecodeManyDevice, +}; +use j2k_cuda::{Codec, CudaSession, Error, J2kDecoder, SurfaceResidency}; +use j2k_native::{encode, EncodeOptions}; +use j2k_test_support::{ + cuda_htj2k_strict_required, cuda_runtime_required, htj2k_gray8_97_fixture, htj2k_gray8_fixture, + htj2k_rgb8_97_fixture, htj2k_rgb8_fixture_with_pixels, htj2k_rgb8_pattern_fixture, + openhtj2k_refinement_odd_fixture, rgb16ne_to_opaque_rgba16ne, +}; + +fn fixture() -> Vec { + let pixels = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]; + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + encode(&pixels, 2, 2, 3, 8, false, &options).expect("encode") +} + +fn fixture_ht_gray8() -> Vec { + htj2k_gray8_fixture(4, 4) +} + +fn fixture_ht_gray8_irreversible_97() -> Vec { + htj2k_gray8_97_fixture(4, 4) +} + +fn fixture_ht_rgb8() -> (Vec, Vec) { + htj2k_rgb8_fixture_with_pixels(4, 4) +} + +fn fixture_ht_rgb8_pattern(width: u32, height: u32, seed: u32) -> Vec { + htj2k_rgb8_pattern_fixture(width, height, seed) +} + +fn fixture_ht_rgb8_irreversible_97() -> Vec { + htj2k_rgb8_97_fixture(4, 4) +} + +fn fixture_openhtj2k_refinement_odd() -> &'static [u8] { + openhtj2k_refinement_odd_fixture() +} + +fn fixture_openhtj2k_refinement_odd_pixels() -> &'static [u8] { + include_bytes!("fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray") +} + +#[derive(Clone, Copy, Debug)] +enum StrictDecodeCase { + Full, + Region(Rect), + Scaled(Downscale), + RegionScaled(Rect, Downscale), +} + +impl StrictDecodeCase { + fn output_dims(self, full_dims: (u32, u32)) -> (u32, u32) { + match self { + Self::Full => full_dims, + Self::Region(roi) => (roi.w, roi.h), + Self::Scaled(scale) => { + let scaled = Rect::full(full_dims).scaled_covering(scale); + (scaled.w, scaled.h) + } + Self::RegionScaled(roi, scale) => { + let scaled = roi.scaled_covering(scale); + (scaled.w, scaled.h) + } + } + } +} + +fn decode_strict_cuda_case( + bytes: &[u8], + format: PixelFormat, + case: StrictDecodeCase, +) -> Result { + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + match case { + StrictDecodeCase::Full => decoder.decode_to_device(format, BackendRequest::Cuda), + StrictDecodeCase::Region(roi) => { + decoder.decode_region_to_device(format, roi, BackendRequest::Cuda) + } + StrictDecodeCase::Scaled(scale) => { + decoder.decode_scaled_to_device(format, scale, BackendRequest::Cuda) + } + StrictDecodeCase::RegionScaled(roi, scale) => { + decoder.decode_region_scaled_to_device(format, roi, scale, BackendRequest::Cuda) + } + } +} + +fn expected_host_decode_case( + bytes: &[u8], + format: PixelFormat, + case: StrictDecodeCase, + full_dims: (u32, u32), +) -> Vec { + if format == PixelFormat::Rgba16 { + let rgb16 = expected_host_decode_case(bytes, PixelFormat::Rgb16, case, full_dims); + return rgb16ne_to_opaque_rgba16ne(&rgb16); + } + + let dims = case.output_dims(full_dims); + let stride = dims.0 as usize * format.bytes_per_pixel(); + let mut expected = vec![0u8; stride * dims.1 as usize]; + let mut decoder = J2kDecoder::new(bytes).expect("host decoder"); + match case { + StrictDecodeCase::Full => decoder + .decode_into(&mut expected, stride, format) + .expect("host full decode"), + StrictDecodeCase::Region(roi) => decoder + .decode_region_into( + &mut j2k_cuda::J2kScratchPool::new(), + &mut expected, + stride, + format, + roi, + ) + .expect("host ROI decode"), + StrictDecodeCase::Scaled(scale) => decoder + .decode_scaled_into( + &mut j2k_cuda::J2kScratchPool::new(), + &mut expected, + stride, + format, + scale, + ) + .expect("host scaled decode"), + StrictDecodeCase::RegionScaled(roi, scale) => decoder + .decode_region_scaled_into( + &mut j2k_cuda::J2kScratchPool::new(), + &mut expected, + stride, + format, + roi, + scale, + ) + .expect("host region+scaled decode"), + }; + expected +} + +fn assert_bytes_within(actual: &[u8], expected: &[u8], tolerance: u8, label: &str) { + assert_eq!(actual.len(), expected.len(), "{label} length"); + let mut max_delta = 0u8; + for (&lhs, &rhs) in actual.iter().zip(expected) { + max_delta = max_delta.max(lhs.abs_diff(rhs)); + } + assert!( + max_delta <= tolerance, + "{label} max byte delta {max_delta} exceeded tolerance {tolerance}" + ); +} + +#[test] +fn auto_falls_back_to_cpu_surface() { + let bytes = fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Auto) + .expect("surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert_eq!(surface.residency(), SurfaceResidency::Host); + assert!(surface.as_host_bytes().is_some()); +} + +#[test] +fn explicit_cuda_classic_j2k_request_rejects_cpu_staged_upload() { + let bytes = fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + + let error = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) + .expect_err("classic J2K must not be CPU-decoded and uploaded"); + + assert!(error.is_unsupported()); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn explicit_cuda_request_validates_decode_before_upload() { + let bytes = fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + + let error = decoder + .decode_to_device(PixelFormat::Rgba16, BackendRequest::Cuda) + .expect_err("unsupported decode"); + assert!(error.is_unsupported()); + assert!(!matches!(error, Error::CudaUnavailable)); +} + +#[test] +fn explicit_cuda_request_returns_cuda_surface_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let bytes = fixture_ht_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), (4, 4)); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda surface"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut expected = [0u8; 16]; + host_decoder + .decode_into(&mut expected, 4, PixelFormat::Gray8) + .expect("host decode"); + assert_eq!(downloaded, expected); +} + +#[test] +fn explicit_cuda_profile_reports_gpu_stage_timings_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let bytes = fixture_ht_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let mut session = CudaSession::default(); + let (surface, report) = decoder + .decode_to_device_with_session_and_profile(PixelFormat::Gray8, &mut session) + .expect("strict CUDA HTJ2K profiled surface"); + + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_resident_cuda_surface(&surface); + assert!(report.dispatch_count > 0); + assert!(report.ht_cleanup_us > 0); + assert_eq!(report.dequant_us, 0); + assert_eq!(report.detail.dequant_dispatch_count, 0); + assert!(report.idwt_us > 0); + assert!(report.store_us > 0); + assert_eq!(report.residency, SurfaceResidency::CudaResidentDecode); +} + +#[test] +fn explicit_cuda_region_surface_matches_host_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let bytes = fixture_ht_gray8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_to_device(PixelFormat::Gray8, roi, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K ROI surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), (roi.w, roi.h)); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda ROI surface"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut expected = vec![0u8; roi.w as usize * roi.h as usize]; + host_decoder + .decode_region_into( + &mut j2k_cuda::J2kScratchPool::new(), + &mut expected, + roi.w as usize, + PixelFormat::Gray8, + roi, + ) + .expect("host ROI decode"); + assert_eq!(downloaded, expected); +} + +#[test] +fn explicit_cuda_scaled_surface_matches_host_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let bytes = fixture_ht_gray8(); + let scale = Downscale::Half; + let scaled = Rect::full((4, 4)).scaled_covering(scale); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_scaled_to_device(PixelFormat::Gray8, scale, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K scaled surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda scaled surface"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut expected = vec![0u8; scaled.w as usize * scaled.h as usize]; + host_decoder + .decode_scaled_into( + &mut j2k_cuda::J2kScratchPool::new(), + &mut expected, + scaled.w as usize, + PixelFormat::Gray8, + scale, + ) + .expect("host scaled decode"); + assert_eq!(downloaded, expected); +} + +#[test] +fn explicit_cuda_rgb8_request_returns_resident_surface_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let (bytes, pixels) = fixture_ht_rgb8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K RGB surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), (4, 4)); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda surface"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut expected = vec![0u8; pixels.len()]; + host_decoder + .decode_into(&mut expected, 4 * 3, PixelFormat::Rgb8) + .expect("host decode"); + assert_eq!(downloaded, expected); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn explicit_cuda_rgb8_region_request_reaches_runtime_boundary() { + let (bytes, _) = fixture_ht_rgb8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + + match decoder.decode_region_to_device(PixelFormat::Rgb8, roi, BackendRequest::Cuda) { + Ok(surface) => { + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.dimensions(), (roi.w, roi.h)); + } + Err(Error::UnsupportedCudaRequest { reason }) => { + panic!("RGB ROI must not stop at strict unsupported before CUDA runtime: {reason}"); + } + Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) => {} + Err(error) => panic!("unexpected RGB ROI strict CUDA error: {error}"), + } +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn explicit_cuda_rgba_requests_reach_runtime_boundary() { + let (bytes, _) = fixture_ht_rgb8(); + + for format in [PixelFormat::Rgba8, PixelFormat::Rgba16] { + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + match decoder.decode_to_device(format, BackendRequest::Cuda) { + Ok(surface) => { + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.dimensions(), (4, 4)); + } + Err(Error::UnsupportedCudaRequest { reason }) => { + panic!( + "{format:?} must not stop at strict unsupported before CUDA runtime: {reason}" + ); + } + Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) => {} + Err(error) => panic!("unexpected {format:?} strict CUDA error: {error}"), + } + } +} + +#[test] +fn explicit_cuda_rgb8_region_surface_matches_host_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let (bytes, _) = fixture_ht_rgb8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_to_device(PixelFormat::Rgb8, roi, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K RGB ROI surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), (roi.w, roi.h)); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda RGB ROI surface"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut expected = vec![0u8; roi.w as usize * roi.h as usize * 3]; + host_decoder + .decode_region_into( + &mut j2k_cuda::J2kScratchPool::new(), + &mut expected, + roi.w as usize * 3, + PixelFormat::Rgb8, + roi, + ) + .expect("host RGB ROI decode"); + assert_eq!(downloaded, expected); +} + +#[test] +fn explicit_cuda_scaled_rgb8_surface_matches_host_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let (bytes, _) = fixture_ht_rgb8(); + let scale = Downscale::Half; + let scaled = Rect::full((4, 4)).scaled_covering(scale); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_scaled_to_device(PixelFormat::Rgb8, scale, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K scaled RGB surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda scaled RGB surface"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut expected = vec![0u8; scaled.w as usize * scaled.h as usize * 3]; + host_decoder + .decode_scaled_into( + &mut j2k_cuda::J2kScratchPool::new(), + &mut expected, + scaled.w as usize * 3, + PixelFormat::Rgb8, + scale, + ) + .expect("host scaled RGB decode"); + assert_eq!(downloaded, expected); +} + +#[test] +fn explicit_cuda_rgb8_region_scaled_surface_matches_host_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let (bytes, _) = fixture_ht_rgb8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Rgb8, roi, scale, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K RGB region+scaled surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda RGB region+scaled surface"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut expected = vec![0u8; scaled.w as usize * scaled.h as usize * 3]; + host_decoder + .decode_region_scaled_into( + &mut j2k_cuda::J2kScratchPool::new(), + &mut expected, + scaled.w as usize * 3, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("host RGB region+scaled decode"); + assert_eq!(downloaded, expected); +} + +#[test] +fn explicit_cuda_gray16_and_rgb16_requests_return_resident_surfaces_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let gray_bytes = fixture_ht_gray8(); + let mut gray_decoder = J2kDecoder::new(&gray_bytes).expect("gray decoder"); + let gray_surface = gray_decoder + .decode_to_device(PixelFormat::Gray16, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K Gray16 surface"); + assert_eq!(gray_surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!( + gray_surface.residency(), + SurfaceResidency::CudaResidentDecode + ); + assert_resident_cuda_surface(&gray_surface); + + let mut gray_downloaded = vec![0u8; gray_surface.byte_len()]; + gray_surface + .download_into(&mut gray_downloaded, gray_surface.pitch_bytes()) + .expect("download Gray16 cuda surface"); + let mut host_gray_decoder = J2kDecoder::new(&gray_bytes).expect("host gray decoder"); + let mut expected_gray = vec![0u8; 4 * 4 * 2]; + host_gray_decoder + .decode_into(&mut expected_gray, 4 * 2, PixelFormat::Gray16) + .expect("host Gray16 decode"); + assert_eq!(gray_downloaded, expected_gray); + + let (rgb_bytes, _) = fixture_ht_rgb8(); + let mut rgb_decoder = J2kDecoder::new(&rgb_bytes).expect("rgb decoder"); + let rgb_surface = rgb_decoder + .decode_to_device(PixelFormat::Rgb16, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K Rgb16 surface"); + assert_eq!(rgb_surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!( + rgb_surface.residency(), + SurfaceResidency::CudaResidentDecode + ); + assert_resident_cuda_surface(&rgb_surface); + + let mut rgb_downloaded = vec![0u8; rgb_surface.byte_len()]; + rgb_surface + .download_into(&mut rgb_downloaded, rgb_surface.pitch_bytes()) + .expect("download Rgb16 cuda surface"); + let mut host_rgb_decoder = J2kDecoder::new(&rgb_bytes).expect("host rgb decoder"); + let mut expected_rgb = vec![0u8; 4 * 4 * 3 * 2]; + host_rgb_decoder + .decode_into(&mut expected_rgb, 4 * 3 * 2, PixelFormat::Rgb16) + .expect("host Rgb16 decode"); + assert_eq!(rgb_downloaded, expected_rgb); +} + +#[test] +fn explicit_cuda_rgba8_and_rgba16_requests_return_resident_surfaces_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let (bytes, _) = fixture_ht_rgb8(); + + let mut rgba8_decoder = J2kDecoder::new(&bytes).expect("rgba8 decoder"); + let rgba8_surface = rgba8_decoder + .decode_to_device(PixelFormat::Rgba8, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K Rgba8 surface"); + assert_eq!(rgba8_surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!( + rgba8_surface.residency(), + SurfaceResidency::CudaResidentDecode + ); + assert_eq!(rgba8_surface.as_host_bytes(), None); + assert_resident_cuda_surface(&rgba8_surface); + assert_eq!(rgba8_surface.dimensions(), (4, 4)); + + let mut rgba8_downloaded = vec![0u8; rgba8_surface.byte_len()]; + rgba8_surface + .download_into(&mut rgba8_downloaded, rgba8_surface.pitch_bytes()) + .expect("download Rgba8 cuda surface"); + let mut host_rgba8_decoder = J2kDecoder::new(&bytes).expect("host rgba8 decoder"); + let mut expected_rgba8 = vec![0u8; 4 * 4 * 4]; + host_rgba8_decoder + .decode_into(&mut expected_rgba8, 4 * 4, PixelFormat::Rgba8) + .expect("host Rgba8 decode"); + assert_eq!(rgba8_downloaded, expected_rgba8); + + let mut rgba16_decoder = J2kDecoder::new(&bytes).expect("rgba16 decoder"); + let rgba16_surface = rgba16_decoder + .decode_to_device(PixelFormat::Rgba16, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K Rgba16 surface"); + assert_eq!(rgba16_surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!( + rgba16_surface.residency(), + SurfaceResidency::CudaResidentDecode + ); + assert_eq!(rgba16_surface.as_host_bytes(), None); + assert_resident_cuda_surface(&rgba16_surface); + assert_eq!(rgba16_surface.dimensions(), (4, 4)); + + let mut rgba16_downloaded = vec![0u8; rgba16_surface.byte_len()]; + rgba16_surface + .download_into(&mut rgba16_downloaded, rgba16_surface.pitch_bytes()) + .expect("download Rgba16 cuda surface"); + let expected_rgba16 = + expected_host_decode_case(&bytes, PixelFormat::Rgba16, StrictDecodeCase::Full, (4, 4)); + assert_eq!(rgba16_downloaded, expected_rgba16); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn explicit_cuda_16bit_and_rgba_region_scaled_requests_reach_runtime_boundary() { + let gray_bytes = fixture_ht_gray8(); + let (rgb_bytes, _) = fixture_ht_rgb8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let cases = [ + StrictDecodeCase::Region(roi), + StrictDecodeCase::Scaled(Downscale::Half), + StrictDecodeCase::RegionScaled(roi, Downscale::Half), + ]; + + for case in cases { + match decode_strict_cuda_case(&gray_bytes, PixelFormat::Gray16, case) { + Ok(surface) => { + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.dimensions(), case.output_dims((4, 4))); + } + Err(Error::UnsupportedCudaRequest { reason }) => { + panic!("Gray16 {case:?} must reach CUDA runtime boundary: {reason}"); + } + Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) => {} + Err(error) => panic!("unexpected Gray16 {case:?} strict CUDA error: {error}"), + } + } + + for format in [PixelFormat::Rgb16, PixelFormat::Rgba8, PixelFormat::Rgba16] { + for case in cases { + match decode_strict_cuda_case(&rgb_bytes, format, case) { + Ok(surface) => { + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.dimensions(), case.output_dims((4, 4))); + } + Err(Error::UnsupportedCudaRequest { reason }) => { + panic!("{format:?} {case:?} must reach CUDA runtime boundary: {reason}"); + } + Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) => {} + Err(error) => panic!("unexpected {format:?} {case:?} strict CUDA error: {error}"), + } + } + } +} + +#[test] +fn explicit_cuda_16bit_and_rgba_region_scaled_surfaces_match_host_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let gray_bytes = fixture_ht_gray8(); + let (rgb_bytes, _) = fixture_ht_rgb8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let cases = [ + StrictDecodeCase::Region(roi), + StrictDecodeCase::Scaled(Downscale::Half), + StrictDecodeCase::RegionScaled(roi, Downscale::Half), + ]; + + for case in cases { + let surface = decode_strict_cuda_case(&gray_bytes, PixelFormat::Gray16, case) + .expect("strict CUDA HTJ2K Gray16 surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), case.output_dims((4, 4))); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download Gray16 cuda surface"); + let expected = expected_host_decode_case(&gray_bytes, PixelFormat::Gray16, case, (4, 4)); + assert_eq!(downloaded, expected, "Gray16 {case:?}"); + } + + for format in [PixelFormat::Rgb16, PixelFormat::Rgba8, PixelFormat::Rgba16] { + for case in cases { + let surface = decode_strict_cuda_case(&rgb_bytes, format, case) + .expect("strict CUDA HTJ2K color surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), case.output_dims((4, 4))); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download color cuda surface"); + let expected = expected_host_decode_case(&rgb_bytes, format, case, (4, 4)); + assert_eq!(downloaded, expected, "{format:?} {case:?}"); + } + } +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn explicit_cuda_irreversible_97_requests_reach_runtime_boundary() { + let gray_bytes = fixture_ht_gray8_irreversible_97(); + let rgb_bytes = fixture_ht_rgb8_irreversible_97(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let cases = [ + StrictDecodeCase::Full, + StrictDecodeCase::Region(roi), + StrictDecodeCase::Scaled(Downscale::Half), + StrictDecodeCase::RegionScaled(roi, Downscale::Half), + ]; + + for case in cases { + match decode_strict_cuda_case(&gray_bytes, PixelFormat::Gray8, case) { + Ok(surface) => { + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.dimensions(), case.output_dims((4, 4))); + } + Err(Error::UnsupportedCudaRequest { reason }) => { + panic!("irreversible Gray8 {case:?} must reach CUDA runtime boundary: {reason}"); + } + Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) => {} + Err(error) => panic!("unexpected irreversible Gray8 {case:?} CUDA error: {error}"), + } + } + + for case in cases { + match decode_strict_cuda_case(&rgb_bytes, PixelFormat::Rgb8, case) { + Ok(surface) => { + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.dimensions(), case.output_dims((4, 4))); + } + Err(Error::UnsupportedCudaRequest { reason }) => { + panic!("irreversible Rgb8 {case:?} must reach CUDA runtime boundary: {reason}"); + } + Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) => {} + Err(error) => panic!("unexpected irreversible Rgb8 {case:?} CUDA error: {error}"), + } + } +} + +#[test] +fn explicit_cuda_irreversible_97_surfaces_match_host_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let gray_bytes = fixture_ht_gray8_irreversible_97(); + let rgb_bytes = fixture_ht_rgb8_irreversible_97(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let cases = [ + StrictDecodeCase::Full, + StrictDecodeCase::Region(roi), + StrictDecodeCase::Scaled(Downscale::Half), + StrictDecodeCase::RegionScaled(roi, Downscale::Half), + ]; + + for case in cases { + let surface = decode_strict_cuda_case(&gray_bytes, PixelFormat::Gray8, case) + .expect("strict CUDA HTJ2K irreversible Gray8 surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), case.output_dims((4, 4))); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download irreversible Gray8 cuda surface"); + let expected = expected_host_decode_case(&gray_bytes, PixelFormat::Gray8, case, (4, 4)); + assert_bytes_within( + &downloaded, + &expected, + 2, + &format!("irreversible Gray8 {case:?}"), + ); + } + + for case in cases { + let surface = decode_strict_cuda_case(&rgb_bytes, PixelFormat::Rgb8, case) + .expect("strict CUDA HTJ2K irreversible Rgb8 surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), case.output_dims((4, 4))); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download irreversible Rgb8 cuda surface"); + let expected = expected_host_decode_case(&rgb_bytes, PixelFormat::Rgb8, case, (4, 4)); + assert_bytes_within( + &downloaded, + &expected, + 2, + &format!("irreversible Rgb8 {case:?}"), + ); + } +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn explicit_cuda_refinement_fixture_request_reaches_runtime_boundary() { + let mut decoder = J2kDecoder::new(fixture_openhtj2k_refinement_odd()).expect("decoder"); + + match decoder.decode_to_device(PixelFormat::Gray8, BackendRequest::Cuda) { + Ok(surface) => { + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.dimensions(), (17, 37)); + } + Err(Error::UnsupportedCudaRequest { reason }) => { + panic!("refinement fixture must reach CUDA runtime boundary: {reason}"); + } + Err(Error::CudaUnavailable | Error::CudaRuntime { .. }) => {} + Err(error) => panic!("unexpected refinement fixture strict CUDA error: {error}"), + } +} + +#[test] +fn explicit_cuda_refinement_fixture_surface_matches_oracle_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let mut decoder = J2kDecoder::new(fixture_openhtj2k_refinement_odd()).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K refinement surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), (17, 37)); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download refinement cuda surface"); + assert_eq!(downloaded, fixture_openhtj2k_refinement_odd_pixels()); +} + +#[test] +fn explicit_cuda_refinement_fixture_profile_reports_refine_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let mut decoder = J2kDecoder::new(fixture_openhtj2k_refinement_odd()).expect("decoder"); + let mut session = CudaSession::default(); + let (surface, report) = decoder + .decode_to_device_with_session_and_profile(PixelFormat::Gray8, &mut session) + .expect("strict CUDA HTJ2K profiled refinement surface"); + + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_resident_cuda_surface(&surface); + assert_eq!(report.block_count, 14); + assert!(report.ht_cleanup_us > 0); + assert!(report.ht_refine_us > 0); + assert!(report.dequant_us > 0); + assert!(report.idwt_us > 0); + assert!(report.store_us > 0); +} + +#[test] +fn explicit_cpu_staged_cuda_api_marks_cpu_upload_residency_when_cuda_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let bytes = fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let mut session = CudaSession::default(); + let surface = decoder + .decode_to_cpu_staged_cuda_surface_with_session(PixelFormat::Rgb8, &mut session) + .expect("CPU-staged CUDA surface"); + + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CpuStagedCudaUpload); + assert_eq!(surface.as_host_bytes(), None); + assert_cpu_staged_cuda_surface(&surface); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download CPU-staged CUDA surface"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut expected = [0u8; 12]; + host_decoder + .decode_into(&mut expected, 6, PixelFormat::Rgb8) + .expect("host decode"); + assert_eq!(downloaded, expected); +} + +#[test] +fn explicit_cuda_region_scaled_surface_matches_host_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let bytes = fixture_ht_gray8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Cuda) + .expect("strict CUDA HTJ2K region+scaled surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda region+scaled surface"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut expected = vec![0u8; scaled.w as usize * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut j2k_cuda::J2kScratchPool::new(), + &mut expected, + scaled.w as usize, + PixelFormat::Gray8, + roi, + scale, + ) + .expect("host region+scaled decode"); + assert_eq!(downloaded, expected); +} + +#[test] +fn explicit_cuda_download_respects_padded_stride_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let bytes = fixture_ht_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Cuda) + .expect("cuda surface"); + assert_resident_cuda_surface(&surface); + let row_bytes = surface.pitch_bytes(); + let stride = row_bytes + 5; + let mut downloaded = vec![0xCD; stride * surface.dimensions().1 as usize]; + surface + .download_into(&mut downloaded, stride) + .expect("download cuda surface"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut expected = [0u8; 16]; + host_decoder + .decode_into(&mut expected, row_bytes, PixelFormat::Gray8) + .expect("host decode"); + for (row, expected_row) in expected.chunks(row_bytes).enumerate() { + let start = row * stride; + assert_eq!(&downloaded[start..start + row_bytes], expected_row); + assert_eq!(&downloaded[start + row_bytes..start + stride], &[0xCD; 5]); + } +} + +fn assert_cpu_staged_cuda_surface(surface: &j2k_cuda::Surface) { + let cuda = surface.cuda_surface().expect("cuda surface"); + assert_ne!(cuda.device_ptr(), 0); + assert!(cuda.stats().kernel_dispatches() > 0); + assert!(cuda.stats().copy_kernel_dispatches() > 0); + assert_eq!(cuda.stats().decode_kernel_dispatches(), 0); +} + +fn assert_resident_cuda_surface(surface: &j2k_cuda::Surface) { + let cuda = surface.cuda_surface().expect("cuda surface"); + assert_ne!(cuda.device_ptr(), 0); + assert!(cuda.stats().kernel_dispatches() > 0); + assert_eq!(cuda.stats().copy_kernel_dispatches(), 0); + assert!(cuda.stats().decode_kernel_dispatches() > 0); +} + +fn assert_cuda_batch_surface(surface: &j2k_cuda::Surface) { + let cuda = surface.cuda_surface().expect("cuda batch surface"); + assert_ne!(cuda.device_ptr(), 0); + assert_eq!(cuda.stats().copy_kernel_dispatches(), 0); +} + +#[test] +fn submit_to_device_auto_falls_back_to_cpu_surface() { + let bytes = fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let mut session = CudaSession::default(); + let surface = as ImageDecodeSubmit<'_>>::submit_to_device( + &mut decoder, + &mut session, + PixelFormat::Rgb8, + BackendRequest::Auto, + ) + .expect("submission") + .wait() + .expect("surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert!(surface.as_host_bytes().is_some()); + assert!(session.submissions() >= 1); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn submit_to_device_auto_does_not_initialize_cuda_runtime() { + let bytes = fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let mut session = CudaSession::default(); + let surface = as ImageDecodeSubmit<'_>>::submit_to_device( + &mut decoder, + &mut session, + PixelFormat::Rgb8, + BackendRequest::Auto, + ) + .expect("submission") + .wait() + .expect("surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert_eq!(session.submissions(), 1); + assert!(!session.is_runtime_initialized()); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn explicit_cuda_submissions_reuse_session_runtime_when_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let bytes = fixture_ht_gray8(); + let mut session = CudaSession::default(); + assert!(!session.is_runtime_initialized()); + + let mut first = J2kDecoder::new(&bytes).expect("decoder"); + let first_surface = as ImageDecodeSubmit<'_>>::submit_to_device( + &mut first, + &mut session, + PixelFormat::Gray8, + BackendRequest::Cuda, + ) + .expect("first submission") + .wait() + .expect("first surface"); + assert_eq!(first_surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_resident_cuda_surface(&first_surface); + assert!(session.is_runtime_initialized()); + + let mut second = J2kDecoder::new(&bytes).expect("decoder"); + let second_surface = as ImageDecodeSubmit<'_>>::submit_to_device( + &mut second, + &mut session, + PixelFormat::Gray8, + BackendRequest::Cuda, + ) + .expect("second submission") + .wait() + .expect("second surface"); + assert_eq!(second_surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_resident_cuda_surface(&second_surface); + assert_eq!(session.submissions(), 2); + assert!(session.is_runtime_initialized()); +} + +#[test] +fn auto_classic_full_frame_surface_matches_host_decode() { + let bytes = fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Auto) + .expect("surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 12]; + host_decoder + .decode_into(&mut host, 6, PixelFormat::Rgb8) + .expect("host decode"); + assert_eq!(surface.as_host_bytes(), Some(host.as_slice())); +} + +#[test] +fn auto_htj2k_full_frame_surface_matches_host_decode() { + let bytes = fixture_ht_gray8(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Auto) + .expect("surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = [0u8; 16]; + host_decoder + .decode_into(&mut host, 4, PixelFormat::Gray8) + .expect("host decode"); + assert_eq!(surface.as_host_bytes(), Some(host.as_slice())); +} + +#[test] +fn auto_region_scaled_surface_matches_host_decode() { + let bytes = fixture_ht_gray8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Auto) + .expect("surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = vec![0u8; scaled.w as usize * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut j2k_cuda::J2kScratchPool::new(), + &mut host, + scaled.w as usize, + PixelFormat::Gray8, + roi, + scale, + ) + .expect("host decode"); + assert_eq!(surface.as_host_bytes(), Some(host.as_slice())); +} + +#[test] +fn tile_batch_region_cuda_surface_matches_host_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let bytes = fixture_ht_gray8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_cuda::J2kScratchPool::new(); + let surface = Codec::decode_tile_region_to_device( + &mut ctx, + &mut pool, + &bytes, + PixelFormat::Gray8, + roi, + BackendRequest::Cuda, + ) + .expect("cuda tile batch ROI surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), (roi.w, roi.h)); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda surface"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut expected = vec![0u8; roi.w as usize * roi.h as usize]; + host_decoder + .decode_region_into( + &mut j2k_cuda::J2kScratchPool::new(), + &mut expected, + roi.w as usize, + PixelFormat::Gray8, + roi, + ) + .expect("host decode"); + assert_eq!(downloaded, expected); +} + +#[test] +fn tile_batch_region_scaled_cuda_surface_matches_host_when_cuda_runtime_required() { + if !cuda_runtime_required() || !cuda_htj2k_strict_required() { + return; + } + + let bytes = fixture_ht_gray8(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_cuda::J2kScratchPool::new(); + let surface = Codec::decode_tile_region_scaled_to_device( + &mut ctx, + &mut pool, + &bytes, + PixelFormat::Gray8, + roi, + scale, + BackendRequest::Cuda, + ) + .expect("cuda tile batch scaled ROI surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda scaled ROI surface"); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut expected = vec![0u8; scaled.w as usize * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut j2k_cuda::J2kScratchPool::new(), + &mut expected, + scaled.w as usize, + PixelFormat::Gray8, + roi, + scale, + ) + .expect("host scaled ROI decode"); + assert_eq!(downloaded, expected); +} + +#[test] +fn decode_tiles_to_device_auto_preserves_order_and_matches_host_bytes() { + let bytes = fixture_ht_gray8(); + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_cuda::J2kScratchPool::new(); + let inputs = [bytes.as_slice(), bytes.as_slice()]; + + let surfaces = Codec::decode_tiles_to_device( + &mut ctx, + &mut pool, + &inputs, + PixelFormat::Gray8, + BackendRequest::Auto, + ) + .expect("batch surfaces"); + + assert_eq!(surfaces.len(), inputs.len()); + let mut expected = [0u8; 16]; + J2kDecoder::new(&bytes) + .expect("host decoder") + .decode_into(&mut expected, 4, PixelFormat::Gray8) + .expect("host decode"); + for surface in surfaces { + assert_eq!(surface.dimensions(), (4, 4)); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert_eq!(surface.residency(), SurfaceResidency::Host); + assert_eq!(surface.as_host_bytes(), Some(expected.as_slice())); + } +} + +#[test] +fn decode_tiles_to_device_cpu_preserves_host_residency() { + let bytes = fixture_ht_gray8(); + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_cuda::J2kScratchPool::new(); + let inputs = [bytes.as_slice(), bytes.as_slice()]; + + let surfaces = Codec::decode_tiles_to_device( + &mut ctx, + &mut pool, + &inputs, + PixelFormat::Gray8, + BackendRequest::Cpu, + ) + .expect("CPU batch surfaces"); + + assert_eq!(surfaces.len(), inputs.len()); + for surface in surfaces { + assert_eq!(surface.dimensions(), (4, 4)); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert_eq!(surface.residency(), SurfaceResidency::Host); + assert!(surface.as_host_bytes().is_some()); + } +} + +#[test] +fn decode_tiles_to_device_explicit_cuda_rgb8_batch_matches_host_bytes() { + let first = fixture_ht_rgb8_pattern(32, 32, 17); + let second = fixture_ht_rgb8_pattern(32, 32, 29); + let inputs = [first.as_slice(), second.as_slice()]; + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_cuda::J2kScratchPool::new(); + + let surfaces = match Codec::decode_tiles_to_device( + &mut ctx, + &mut pool, + &inputs, + PixelFormat::Rgb8, + BackendRequest::Cuda, + ) { + Ok(surfaces) => surfaces, + Err(Error::CudaUnavailable) if !cuda_runtime_required() => return, + #[cfg(feature = "cuda-runtime")] + Err(Error::CudaRuntime { .. }) if !cuda_runtime_required() => return, + Err(error) => panic!("strict CUDA RGB8 batch decode failed: {error}"), + }; + + assert_eq!(surfaces.len(), inputs.len()); + for surface in &surfaces { + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_cuda_batch_surface(surface); + } + + let downloaded = + j2k_cuda::Surface::download_batch_tight(&surfaces).expect("download tight batch"); + let expected = inputs + .iter() + .flat_map(|input| { + let mut out = vec![0u8; 32 * 32 * 3]; + J2kDecoder::new(input) + .expect("host decoder") + .decode_into(&mut out, 32 * 3, PixelFormat::Rgb8) + .expect("host decode"); + out + }) + .collect::>(); + assert_eq!(downloaded, expected); +} + +#[test] +fn decode_tiles_to_device_explicit_cuda_rgba8_batch_matches_host_bytes() { + let first = fixture_ht_rgb8_pattern(32, 32, 31); + let second = fixture_ht_rgb8_pattern(32, 32, 47); + let inputs = [first.as_slice(), second.as_slice()]; + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_cuda::J2kScratchPool::new(); + + let surfaces = match Codec::decode_tiles_to_device( + &mut ctx, + &mut pool, + &inputs, + PixelFormat::Rgba8, + BackendRequest::Cuda, + ) { + Ok(surfaces) => surfaces, + Err(Error::CudaUnavailable) if !cuda_runtime_required() => return, + #[cfg(feature = "cuda-runtime")] + Err(Error::CudaRuntime { .. }) if !cuda_runtime_required() => return, + Err(error) => panic!("strict CUDA Rgba8 batch decode failed: {error}"), + }; + + assert_eq!(surfaces.len(), inputs.len()); + for surface in &surfaces { + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_cuda_batch_surface(surface); + } + + let downloaded = + j2k_cuda::Surface::download_batch_tight(&surfaces).expect("download tight batch"); + let expected = inputs + .iter() + .flat_map(|input| { + let mut out = vec![0u8; 32 * 32 * 4]; + J2kDecoder::new(input) + .expect("host decoder") + .decode_into(&mut out, 32 * 4, PixelFormat::Rgba8) + .expect("host decode"); + out + }) + .collect::>(); + assert_eq!(downloaded, expected); +} + +#[test] +fn decode_tiles_to_device_explicit_cuda_returns_cuda_surfaces_or_clear_unavailable_error() { + let bytes = fixture_ht_gray8(); + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_cuda::J2kScratchPool::new(); + let inputs = [bytes.as_slice(), bytes.as_slice()]; + + match Codec::decode_tiles_to_device( + &mut ctx, + &mut pool, + &inputs, + PixelFormat::Gray8, + BackendRequest::Cuda, + ) { + Ok(surfaces) => { + assert_eq!(surfaces.len(), inputs.len()); + for surface in surfaces { + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.residency(), SurfaceResidency::CudaResidentDecode); + assert_eq!(surface.as_host_bytes(), None); + assert_resident_cuda_surface(&surface); + } + } + Err(error) => assert!(error.is_unsupported()), + } +} diff --git a/crates/j2k-cuda/tests/htj2k_cuda_kernels.rs b/crates/j2k-cuda/tests/htj2k_cuda_kernels.rs new file mode 100644 index 00000000..de20a951 --- /dev/null +++ b/crates/j2k-cuda/tests/htj2k_cuda_kernels.rs @@ -0,0 +1,1709 @@ +#[cfg(feature = "cuda-runtime")] +use j2k_core::PixelFormat; +#[cfg(feature = "cuda-runtime")] +use j2k_cuda::J2kDecoder; +#[cfg(feature = "cuda-runtime")] +use j2k_cuda_runtime::{ + CudaContext, CudaHtj2kCodeBlockJob, CudaHtj2kDecodeTables, CudaHtj2kEncodeCodeBlockJob, + CudaHtj2kEncodeTables, CudaHtj2kPacketizationBlock, CudaHtj2kPacketizationPacket, + CudaHtj2kPacketizationSubband, CudaHtj2kPacketizationSubbandTagState, + CudaHtj2kPacketizationTagNodeState, CudaJ2kInverseMctJob, CudaJ2kStoreGray16Job, + CudaJ2kStoreRgb16Job, CudaJ2kStoreRgb8Job, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_native::{ + decode_ht_code_block_scalar, encode_ht_code_block_scalar, encode_htj2k, + ht_uvlc_encode_table_bytes, ht_uvlc_table0, ht_uvlc_table1, ht_vlc_encode_table0, + ht_vlc_encode_table1, ht_vlc_table0, ht_vlc_table1, EncodeOptions, HtCodeBlockDecodeJob, + J2kPacketizationBlockCodingMode, J2kPacketizationCodeBlock, J2kPacketizationEncodeJob, + J2kPacketizationPacketDescriptor, J2kPacketizationProgressionOrder, J2kPacketizationResolution, + J2kPacketizationSubband, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_test_support::cuda_runtime_required; + +#[cfg(feature = "cuda-runtime")] +fn ht_gray8_fixture() -> Vec { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8") +} + +#[cfg(feature = "cuda-runtime")] +fn openhtj2k_refinement_fixture() -> &'static [u8] { + include_bytes!("fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k") +} + +#[cfg(feature = "cuda-runtime")] +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn rounded_u8(sample: f32) -> u8 { + sample.round().clamp(0.0, 255.0) as u8 +} + +#[cfg(feature = "cuda-runtime")] +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] +fn rounded_u16(sample: f32, bit_depth: u32) -> u16 { + let rounded = sample.round(); + if bit_depth >= 16 { + return rounded.clamp(0.0, f32::from(u16::MAX)) as u16; + } + let shift = u16::try_from(bit_depth.min(15)).expect("bounded bit depth fits in u16"); + let max_value = f32::from((1u16 << shift).saturating_sub(1).max(1)); + ((rounded.clamp(0.0, max_value) / max_value) * f32::from(u16::MAX)).round() as u16 +} + +#[cfg(feature = "cuda-runtime")] +fn push_u16_ne(out: &mut Vec, value: u16) { + out.extend_from_slice(&value.to_ne_bytes()); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_entropy_kernel_matches_native_scalar_codeblock_when_required() { + if !cuda_runtime_required() { + return; + } + + let bytes = ht_gray8_fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let (cuda_plan, _) = decoder + .build_cuda_htj2k_grayscale_plan_with_profile(PixelFormat::Gray8) + .expect("CUDA flat plan"); + let block = cuda_plan + .code_blocks() + .first() + .copied() + .expect("at least one HT block"); + let payload_start = usize::try_from(block.payload_offset).expect("payload offset"); + let payload_end = payload_start + block.payload_len as usize; + let block_payload = &cuda_plan.payload()[payload_start..payload_end]; + + let mut expected = vec![0.0f32; block.width as usize * block.height as usize]; + decode_ht_code_block_scalar( + HtCodeBlockDecodeJob { + data: block_payload, + cleanup_length: block.cleanup_length, + refinement_length: block.refinement_length, + width: block.width, + height: block.height, + output_stride: block.width as usize, + missing_bit_planes: block.missing_bit_planes, + number_of_coding_passes: block.number_of_coding_passes, + num_bitplanes: block.num_bitplanes, + roi_shift: 0, + stripe_causal: block.stripe_causal != 0, + strict: true, + dequantization_step: block.dequantization_step, + }, + &mut expected, + ) + .expect("native scalar HT decode"); + + let context = CudaContext::system_default().expect("CUDA context"); + let output = context + .decode_htj2k_codeblocks( + block_payload, + &[CudaHtj2kCodeBlockJob { + payload_offset: 0, + width: block.width, + height: block.height, + payload_len: block.payload_len, + cleanup_length: block.cleanup_length, + refinement_length: block.refinement_length, + missing_bit_planes: block.missing_bit_planes, + num_bitplanes: block.num_bitplanes, + number_of_coding_passes: block.number_of_coding_passes, + output_stride: block.width, + output_offset: 0, + dequantization_step: block.dequantization_step, + stripe_causal: block.stripe_causal != 0, + }], + CudaHtj2kDecodeTables { + vlc_table0: ht_vlc_table0(), + vlc_table1: ht_vlc_table1(), + uvlc_table0: ht_uvlc_table0(), + uvlc_table1: ht_uvlc_table1(), + }, + expected.len(), + ) + .expect("CUDA HT decode"); + + assert_eq!(output.execution().decode_kernel_dispatches(), 2); + assert!(output.stage_timings().ht_cleanup_us > 0); + assert!(output.stage_timings().dequant_us > 0); + assert!(output.statuses().iter().all(|status| status.is_ok())); + + let mut actual_bytes = vec![0u8; expected.len() * std::mem::size_of::()]; + output + .coefficients() + .copy_to_host(&mut actual_bytes) + .expect("download coefficients"); + let actual = actual_bytes + .chunks_exact(std::mem::size_of::()) + .map(|chunk| f32::from_ne_bytes(chunk.try_into().expect("f32 bytes"))) + .collect::>(); + assert_eq!(actual, expected); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_refinement_kernel_matches_native_scalar_codeblock_when_required() { + if !cuda_runtime_required() { + return; + } + + let mut decoder = J2kDecoder::new(openhtj2k_refinement_fixture()).expect("decoder"); + let (cuda_plan, _) = decoder + .build_cuda_htj2k_grayscale_plan_with_profile(PixelFormat::Gray8) + .expect("CUDA flat plan"); + let block = cuda_plan + .code_blocks() + .iter() + .copied() + .find(|block| block.refinement_length > 0) + .expect("fixture must contain a refinement block"); + let payload_start = usize::try_from(block.payload_offset).expect("payload offset"); + let payload_end = payload_start + block.payload_len as usize; + let block_payload = &cuda_plan.payload()[payload_start..payload_end]; + + let mut expected = vec![0.0f32; block.width as usize * block.height as usize]; + decode_ht_code_block_scalar( + HtCodeBlockDecodeJob { + data: block_payload, + cleanup_length: block.cleanup_length, + refinement_length: block.refinement_length, + width: block.width, + height: block.height, + output_stride: block.width as usize, + missing_bit_planes: block.missing_bit_planes, + number_of_coding_passes: block.number_of_coding_passes, + num_bitplanes: block.num_bitplanes, + roi_shift: 0, + stripe_causal: block.stripe_causal != 0, + strict: true, + dequantization_step: block.dequantization_step, + }, + &mut expected, + ) + .expect("native scalar HT refinement decode"); + + let context = CudaContext::system_default().expect("CUDA context"); + let output = context + .decode_htj2k_codeblocks( + block_payload, + &[CudaHtj2kCodeBlockJob { + payload_offset: 0, + width: block.width, + height: block.height, + payload_len: block.payload_len, + cleanup_length: block.cleanup_length, + refinement_length: block.refinement_length, + missing_bit_planes: block.missing_bit_planes, + num_bitplanes: block.num_bitplanes, + number_of_coding_passes: block.number_of_coding_passes, + output_stride: block.width, + output_offset: 0, + dequantization_step: block.dequantization_step, + stripe_causal: block.stripe_causal != 0, + }], + CudaHtj2kDecodeTables { + vlc_table0: ht_vlc_table0(), + vlc_table1: ht_vlc_table1(), + uvlc_table0: ht_uvlc_table0(), + uvlc_table1: ht_uvlc_table1(), + }, + expected.len(), + ) + .expect("CUDA HT refinement decode"); + + assert_eq!(output.execution().decode_kernel_dispatches(), 2); + assert!(output.stage_timings().ht_cleanup_us > 0); + assert!(output.stage_timings().ht_refine_us > 0); + assert!(output.stage_timings().dequant_us > 0); + assert!(output.statuses().iter().all(|status| status.is_ok())); + + let mut actual_bytes = vec![0u8; expected.len() * std::mem::size_of::()]; + output + .coefficients() + .copy_to_host(&mut actual_bytes) + .expect("download coefficients"); + let actual = actual_bytes + .chunks_exact(std::mem::size_of::()) + .map(|chunk| f32::from_ne_bytes(chunk.try_into().expect("f32 bytes"))) + .collect::>(); + assert_eq!(actual, expected); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_encode_kernel_matches_native_scalar_codeblock_when_required() { + if !cuda_runtime_required() { + return; + } + + let coefficients = [ + 0, 2, -3, 1, 4, 0, -1, 2, 3, -2, 0, 1, 0, 0, 5, -4, 1, 0, -2, 3, 0, 1, 0, -1, 2, -3, 4, 0, + -5, 1, 2, 0, 0, -1, 3, 2, -2, 0, 1, -3, 4, 0, 0, 2, -1, 5, 0, -4, 3, 0, 1, -2, 2, 0, -1, 4, + 0, 3, -3, 1, 0, 2, -4, 5, + ]; + let expected = + encode_ht_code_block_scalar(&coefficients, 8, 8, 8).expect("native scalar HT encode"); + + let context = CudaContext::system_default().expect("CUDA context"); + let uvlc_table = ht_uvlc_encode_table_bytes(); + let encoded = context + .encode_htj2k_codeblock( + &coefficients, + 8, + 8, + 8, + CudaHtj2kEncodeTables { + vlc_table0: ht_vlc_encode_table0(), + vlc_table1: ht_vlc_encode_table1(), + uvlc_table, + }, + ) + .expect("CUDA HT encode"); + + assert_eq!(encoded.execution().kernel_dispatches(), 1); + assert_eq!(encoded.data(), expected.data); + assert_eq!(encoded.cleanup_length(), expected.cleanup_length); + assert_eq!(encoded.refinement_length(), expected.refinement_length); + assert_eq!(encoded.num_coding_passes(), expected.num_coding_passes); + assert_eq!(encoded.num_zero_bitplanes(), expected.num_zero_bitplanes); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_encode_target_two_passes_round_trips_with_sigprop_segment_when_required() { + if !cuda_runtime_required() { + return; + } + + let coefficients = [0, 3, -5, 3, 5, 0, -3, 3, 7, -3, 0, 3, 0, 0, 5, -5]; + + let context = CudaContext::system_default().expect("CUDA context"); + let uvlc_table = ht_uvlc_encode_table_bytes(); + let encoded = context + .encode_htj2k_codeblocks( + &coefficients, + &[CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 4, + height: 4, + total_bitplanes: 4, + target_coding_passes: 2, + }], + CudaHtj2kEncodeTables { + vlc_table0: ht_vlc_encode_table0(), + vlc_table1: ht_vlc_encode_table1(), + uvlc_table, + }, + ) + .expect("CUDA two-pass HT encode"); + let block = encoded + .code_blocks() + .first() + .expect("one encoded code block"); + + assert_eq!(block.num_coding_passes(), 2); + assert_eq!(block.num_zero_bitplanes(), 2); + assert_eq!(block.refinement_length(), 1); + assert_eq!( + block.cleanup_length() + block.refinement_length(), + u32::try_from(block.data().len()).expect("test payload length fits u32") + ); + + let mut decoded = vec![0.0f32; coefficients.len()]; + decode_ht_code_block_scalar( + HtCodeBlockDecodeJob { + data: block.data(), + cleanup_length: block.cleanup_length(), + refinement_length: block.refinement_length(), + width: 4, + height: 4, + output_stride: 4, + missing_bit_planes: block.num_zero_bitplanes(), + number_of_coding_passes: block.num_coding_passes(), + num_bitplanes: 4, + roi_shift: 0, + stripe_causal: false, + strict: true, + dequantization_step: 1.0, + }, + &mut decoded, + ) + .expect("two-pass HT block decodes"); + + assert_eq!( + decoded, + coefficients + .iter() + .map(|value| f32::from(i16::try_from(*value).expect("test coefficient fits i16"))) + .collect::>() + ); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_encode_target_three_passes_round_trips_with_magref_segment_when_required() { + if !cuda_runtime_required() { + return; + } + + let coefficients = [5, -7, 9, -11]; + + let context = CudaContext::system_default().expect("CUDA context"); + let uvlc_table = ht_uvlc_encode_table_bytes(); + let encoded = context + .encode_htj2k_codeblocks( + &coefficients, + &[CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 2, + height: 2, + total_bitplanes: 4, + target_coding_passes: 3, + }], + CudaHtj2kEncodeTables { + vlc_table0: ht_vlc_encode_table0(), + vlc_table1: ht_vlc_encode_table1(), + uvlc_table, + }, + ) + .expect("CUDA three-pass HT encode"); + let block = encoded + .code_blocks() + .first() + .expect("one encoded code block"); + + assert_eq!(block.num_coding_passes(), 3); + assert_eq!(block.num_zero_bitplanes(), 1); + assert_eq!(block.refinement_length(), 2); + assert_eq!( + block.cleanup_length() + block.refinement_length(), + u32::try_from(block.data().len()).expect("test payload length fits u32") + ); + + let mut decoded = vec![0.0f32; coefficients.len()]; + decode_ht_code_block_scalar( + HtCodeBlockDecodeJob { + data: block.data(), + cleanup_length: block.cleanup_length(), + refinement_length: block.refinement_length(), + width: 2, + height: 2, + output_stride: 2, + missing_bit_planes: block.num_zero_bitplanes(), + number_of_coding_passes: block.num_coding_passes(), + num_bitplanes: 4, + roi_shift: 0, + stripe_causal: false, + strict: true, + dequantization_step: 1.0, + }, + &mut decoded, + ) + .expect("three-pass HT block decodes"); + + assert_eq!( + decoded, + coefficients + .iter() + .map(|value| f32::from(i16::try_from(*value).expect("test coefficient fits i16"))) + .collect::>() + ); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_encode_target_three_passes_stuffs_magref_all_one_bytes_when_required() { + if !cuda_runtime_required() { + return; + } + + let coefficients = [7; 8]; + + let context = CudaContext::system_default().expect("CUDA context"); + let uvlc_table = ht_uvlc_encode_table_bytes(); + let encoded = context + .encode_htj2k_codeblocks( + &coefficients, + &[CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 4, + height: 2, + total_bitplanes: 4, + target_coding_passes: 3, + }], + CudaHtj2kEncodeTables { + vlc_table0: ht_vlc_encode_table0(), + vlc_table1: ht_vlc_encode_table1(), + uvlc_table, + }, + ) + .expect("CUDA three-pass HT encode with stuffed MagRef bytes"); + let block = encoded + .code_blocks() + .first() + .expect("one encoded code block"); + + assert_eq!(block.num_coding_passes(), 3); + assert_eq!(block.num_zero_bitplanes(), 1); + assert_eq!(block.refinement_length(), 3); + assert_eq!( + block.cleanup_length() + block.refinement_length(), + u32::try_from(block.data().len()).expect("test payload length fits u32") + ); + + let mut decoded = vec![0.0f32; coefficients.len()]; + decode_ht_code_block_scalar( + HtCodeBlockDecodeJob { + data: block.data(), + cleanup_length: block.cleanup_length(), + refinement_length: block.refinement_length(), + width: 4, + height: 2, + output_stride: 4, + missing_bit_planes: block.num_zero_bitplanes(), + number_of_coding_passes: block.num_coding_passes(), + num_bitplanes: 4, + roi_shift: 0, + stripe_causal: false, + strict: true, + dequantization_step: 1.0, + }, + &mut decoded, + ) + .expect("stuffed MagRef HT block decodes"); + + assert_eq!( + decoded, + coefficients + .iter() + .map(|value| f32::from(i16::try_from(*value).expect("test coefficient fits i16"))) + .collect::>() + ); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_encode_target_three_passes_round_trips_nonzero_sigprop_when_required() { + if !cuda_runtime_required() { + return; + } + + let coefficients = [0, 3, -5, 7]; + + let context = CudaContext::system_default().expect("CUDA context"); + let uvlc_table = ht_uvlc_encode_table_bytes(); + let encoded = context + .encode_htj2k_codeblocks( + &coefficients, + &[CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 2, + height: 2, + total_bitplanes: 4, + target_coding_passes: 3, + }], + CudaHtj2kEncodeTables { + vlc_table0: ht_vlc_encode_table0(), + vlc_table1: ht_vlc_encode_table1(), + uvlc_table, + }, + ) + .expect("CUDA three-pass HT encode with nonzero SigProp"); + let block = encoded + .code_blocks() + .first() + .expect("one encoded code block"); + + assert_eq!(block.num_coding_passes(), 3); + assert_eq!(block.num_zero_bitplanes(), 1); + assert_eq!(block.refinement_length(), 2); + assert_eq!( + block.cleanup_length() + block.refinement_length(), + u32::try_from(block.data().len()).expect("test payload length fits u32") + ); + + let mut decoded = vec![0.0f32; coefficients.len()]; + decode_ht_code_block_scalar( + HtCodeBlockDecodeJob { + data: block.data(), + cleanup_length: block.cleanup_length(), + refinement_length: block.refinement_length(), + width: 2, + height: 2, + output_stride: 2, + missing_bit_planes: block.num_zero_bitplanes(), + number_of_coding_passes: block.num_coding_passes(), + num_bitplanes: 4, + roi_shift: 0, + stripe_causal: false, + strict: true, + dequantization_step: 1.0, + }, + &mut decoded, + ) + .expect("nonzero SigProp HT block decodes"); + + assert_eq!( + decoded, + coefficients + .iter() + .map(|value| f32::from(i16::try_from(*value).expect("test coefficient fits i16"))) + .collect::>() + ); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_batch_encode_kernel_matches_native_scalar_codeblocks_when_required() { + if !cuda_runtime_required() { + return; + } + + let block0 = [ + 0, 2, -3, 1, 4, 0, -1, 2, 3, -2, 0, 1, 0, 0, 5, -4, 1, 0, -2, 3, 0, 1, 0, -1, 2, -3, 4, 0, + -5, 1, 2, 0, 0, -1, 3, 2, -2, 0, 1, -3, 4, 0, 0, 2, -1, 5, 0, -4, 3, 0, 1, -2, 2, 0, -1, 4, + 0, 3, -3, 1, 0, 2, -4, 5, + ]; + let block1 = [1, 0, -1, 2, -2, 3, 0, -3, 4, 0, 2, -1, 0, 5, -4, 1]; + let expected0 = + encode_ht_code_block_scalar(&block0, 8, 8, 8).expect("native scalar HT encode 0"); + let expected1 = + encode_ht_code_block_scalar(&block1, 4, 4, 6).expect("native scalar HT encode 1"); + + let mut coefficients = Vec::with_capacity(block0.len() + block1.len()); + coefficients.extend_from_slice(&block0); + let block1_offset = u32::try_from(coefficients.len()).expect("test offset fits in u32"); + coefficients.extend_from_slice(&block1); + + let context = CudaContext::system_default().expect("CUDA context"); + let uvlc_table = ht_uvlc_encode_table_bytes(); + let encoded = context + .encode_htj2k_codeblocks( + &coefficients, + &[ + CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: 0, + width: 8, + height: 8, + total_bitplanes: 8, + target_coding_passes: 1, + }, + CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: block1_offset, + width: 4, + height: 4, + total_bitplanes: 6, + target_coding_passes: 1, + }, + ], + CudaHtj2kEncodeTables { + vlc_table0: ht_vlc_encode_table0(), + vlc_table1: ht_vlc_encode_table1(), + uvlc_table, + }, + ) + .expect("CUDA batch HT encode"); + + assert_eq!(encoded.execution().kernel_dispatches(), 1); + assert!(encoded.stage_timings().ht_encode_us > 0); + assert_eq!(encoded.code_blocks().len(), 2); + assert_eq!(encoded.code_blocks()[0].data(), expected0.data); + assert_eq!( + encoded.code_blocks()[0].cleanup_length(), + expected0.cleanup_length + ); + assert_eq!( + encoded.code_blocks()[0].refinement_length(), + expected0.refinement_length + ); + assert_eq!( + encoded.code_blocks()[0].num_coding_passes(), + expected0.num_coding_passes + ); + assert_eq!( + encoded.code_blocks()[0].num_zero_bitplanes(), + expected0.num_zero_bitplanes + ); + assert_eq!(encoded.code_blocks()[1].data(), expected1.data); + assert_eq!( + encoded.code_blocks()[1].cleanup_length(), + expected1.cleanup_length + ); + assert_eq!( + encoded.code_blocks()[1].refinement_length(), + expected1.refinement_length + ); + assert_eq!( + encoded.code_blocks()[1].num_coding_passes(), + expected1.num_coding_passes + ); + assert_eq!( + encoded.code_blocks()[1].num_zero_bitplanes(), + expected1.num_zero_bitplanes + ); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_forward_ict_kernel_matches_cpu_transform_when_required() { + if !cuda_runtime_required() { + return; + } + + let mut red = [12.0f32, 25.0, 40.0, 60.0]; + let mut green = [5.0f32, 15.0, 45.0, 100.0]; + let mut blue = [90.0f32, 70.0, 30.0, 10.0]; + let mut expected = Vec::with_capacity(red.len() * 3); + for ((r, g), b) in red.iter().copied().zip(green).zip(blue) { + expected.push(0.299 * r + 0.587 * g + 0.114 * b); + expected.push(-0.16875 * r - 0.33126 * g + 0.5 * b); + expected.push(0.5 * r - 0.41869 * g - 0.08131 * b); + } + + let context = CudaContext::system_default().expect("CUDA context"); + let stats = context + .j2k_forward_ict(&mut red, &mut green, &mut blue) + .expect("CUDA forward ICT"); + + assert_eq!(stats.kernel_dispatches(), 1); + for (((actual_y, actual_cb), actual_cr), expected) in red + .into_iter() + .zip(green) + .zip(blue) + .zip(expected.chunks_exact(3)) + { + assert!((actual_y - expected[0]).abs() < 0.0001); + assert!((actual_cb - expected[1]).abs() < 0.0001); + assert!((actual_cr - expected[2]).abs() < 0.0001); + } +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_packetization_kernel_matches_native_scalar_cleanup_packet_when_required() { + if !cuda_runtime_required() { + return; + } + + let payload = [0x12, 0x34, 0x56, 0x78]; + let code_block = J2kPacketizationCodeBlock { + data: &payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let subband = J2kPacketizationSubband { + code_blocks: vec![code_block], + num_cbs_x: 1, + num_cbs_y: 1, + }; + let resolution = J2kPacketizationResolution { + subbands: vec![subband], + }; + let descriptor = J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }; + let expected = j2k_native::encode_j2k_packetization_scalar(J2kPacketizationEncodeJob { + resolution_count: 1, + num_layers: 1, + num_components: 1, + code_block_count: 1, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &[descriptor], + resolutions: &[resolution], + }) + .expect("native scalar packetization"); + + let context = CudaContext::system_default().expect("CUDA context"); + let payload_len = u32::try_from(payload.len()).expect("test payload length fits in u32"); + let packetized = context + .packetize_htj2k_cleanup_packets( + &payload, + &[CudaHtj2kPacketizationPacket { + block_start: 0, + block_count: 1, + subband_start: 0, + subband_count: 1, + output_capacity: 512, + layer: 0, + }], + &[CudaHtj2kPacketizationSubband { + block_start: 0, + block_count: 1, + num_cbs_x: 1, + num_cbs_y: 1, + }], + &[CudaHtj2kPacketizationBlock { + data_offset: 0, + data_len: payload_len, + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + l_block: 3, + previously_included: 0, + inclusion_layer: 0, + }], + ) + .expect("CUDA packetization"); + + assert_eq!(packetized.execution().kernel_dispatches(), 1); + assert!(packetized.stage_timings().packetize_us > 0); + assert!(packetized.statuses().iter().all(|status| status.is_ok())); + assert_eq!(packetized.data(), expected); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_packetization_kernel_matches_native_scalar_multi_block_packet_when_required() { + if !cuda_runtime_required() { + return; + } + + let payloads = vec![ + vec![0x10, 0x11, 0x12], + vec![0x20, 0x21], + vec![0x30, 0x31, 0x32, 0x33], + vec![0x40], + ]; + let code_blocks = payloads + .iter() + .enumerate() + .map(|(idx, payload)| J2kPacketizationCodeBlock { + data: payload.as_slice(), + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: u8::try_from(idx + 1).expect("test zbp fits in u8"), + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }) + .collect(); + let subband = J2kPacketizationSubband { + code_blocks, + num_cbs_x: 2, + num_cbs_y: 2, + }; + let resolution = J2kPacketizationResolution { + subbands: vec![subband], + }; + let descriptor = J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }; + let expected = j2k_native::encode_j2k_packetization_scalar(J2kPacketizationEncodeJob { + resolution_count: 1, + num_layers: 1, + num_components: 1, + code_block_count: 4, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &[descriptor], + resolutions: &[resolution], + }) + .expect("native scalar packetization"); + + let payload = payloads.into_iter().flatten().collect::>(); + let blocks = [ + CudaHtj2kPacketizationBlock { + data_offset: 0, + data_len: 3, + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 1, + l_block: 3, + previously_included: 0, + inclusion_layer: 0, + }, + CudaHtj2kPacketizationBlock { + data_offset: 3, + data_len: 2, + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + l_block: 3, + previously_included: 0, + inclusion_layer: 0, + }, + CudaHtj2kPacketizationBlock { + data_offset: 5, + data_len: 4, + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 3, + l_block: 3, + previously_included: 0, + inclusion_layer: 0, + }, + CudaHtj2kPacketizationBlock { + data_offset: 9, + data_len: 1, + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 4, + l_block: 3, + previously_included: 0, + inclusion_layer: 0, + }, + ]; + let context = CudaContext::system_default().expect("CUDA context"); + let packetized = context + .packetize_htj2k_cleanup_packets( + &payload, + &[CudaHtj2kPacketizationPacket { + block_start: 0, + block_count: 4, + subband_start: 0, + subband_count: 1, + output_capacity: 512, + layer: 0, + }], + &[CudaHtj2kPacketizationSubband { + block_start: 0, + block_count: 4, + num_cbs_x: 2, + num_cbs_y: 2, + }], + &blocks, + ) + .expect("CUDA multi-block packetization"); + + assert_eq!(packetized.execution().kernel_dispatches(), 1); + assert!(packetized.stage_timings().packetize_us > 0); + assert!(packetized.statuses().iter().all(|status| status.is_ok())); + assert_eq!(packetized.data(), expected); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_packetization_kernel_matches_native_scalar_refinement_pass_packet_when_required() { + if !cuda_runtime_required() { + return; + } + + let payload = [0x12, 0x34, 0x56, 0x78, 0x9a]; + let code_block = J2kPacketizationCodeBlock { + data: &payload, + ht_cleanup_length: 3, + ht_refinement_length: 2, + num_coding_passes: 3, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let subband = J2kPacketizationSubband { + code_blocks: vec![code_block], + num_cbs_x: 1, + num_cbs_y: 1, + }; + let resolution = J2kPacketizationResolution { + subbands: vec![subband], + }; + let descriptor = J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }; + let expected = j2k_native::encode_j2k_packetization_scalar(J2kPacketizationEncodeJob { + resolution_count: 1, + num_layers: 1, + num_components: 1, + code_block_count: 1, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &[descriptor], + resolutions: &[resolution], + }) + .expect("native scalar refinement packetization"); + + let context = CudaContext::system_default().expect("CUDA context"); + let packetized = context + .packetize_htj2k_cleanup_packets( + &payload, + &[CudaHtj2kPacketizationPacket { + block_start: 0, + block_count: 1, + subband_start: 0, + subband_count: 1, + output_capacity: 512, + layer: 0, + }], + &[CudaHtj2kPacketizationSubband { + block_start: 0, + block_count: 1, + num_cbs_x: 1, + num_cbs_y: 1, + }], + &[CudaHtj2kPacketizationBlock { + data_offset: 0, + data_len: u32::try_from(payload.len()).expect("test payload length fits in u32"), + cleanup_length: 3, + refinement_length: 2, + num_coding_passes: 3, + num_zero_bitplanes: 2, + l_block: 3, + previously_included: 0, + inclusion_layer: 0, + }], + ) + .expect("CUDA refinement packetization"); + + assert_eq!(packetized.execution().kernel_dispatches(), 1); + assert!(packetized.stage_timings().packetize_us > 0); + assert!(packetized.statuses().iter().all(|status| status.is_ok())); + assert_eq!(packetized.data(), expected); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_packetization_kernel_matches_native_scalar_previously_included_layer_when_required() { + if !cuda_runtime_required() { + return; + } + + let first_payload = [0x11u8; 20]; + let second_payload = [0x22u8; 5]; + let first_block = J2kPacketizationCodeBlock { + data: &first_payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let second_block = J2kPacketizationCodeBlock { + data: &second_payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let resolutions = [ + J2kPacketizationResolution { + subbands: vec![J2kPacketizationSubband { + code_blocks: vec![first_block], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }, + J2kPacketizationResolution { + subbands: vec![J2kPacketizationSubband { + code_blocks: vec![second_block], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }, + ]; + let descriptors = [ + J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }, + J2kPacketizationPacketDescriptor { + packet_index: 1, + state_index: 0, + layer: 1, + resolution: 0, + component: 0, + precinct: 0, + }, + ]; + let expected = j2k_native::encode_j2k_packetization_scalar(J2kPacketizationEncodeJob { + resolution_count: 2, + num_layers: 2, + num_components: 1, + code_block_count: 2, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &descriptors, + resolutions: &resolutions, + }) + .expect("native scalar stateful packetization"); + + let payload = [first_payload.as_slice(), second_payload.as_slice()].concat(); + let context = CudaContext::system_default().expect("CUDA context"); + let packetized = context + .packetize_htj2k_cleanup_packets( + &payload, + &[ + CudaHtj2kPacketizationPacket { + block_start: 0, + block_count: 1, + subband_start: 0, + subband_count: 1, + output_capacity: 512, + layer: 0, + }, + CudaHtj2kPacketizationPacket { + block_start: 1, + block_count: 1, + subband_start: 1, + subband_count: 1, + output_capacity: 512, + layer: 1, + }, + ], + &[ + CudaHtj2kPacketizationSubband { + block_start: 0, + block_count: 1, + num_cbs_x: 1, + num_cbs_y: 1, + }, + CudaHtj2kPacketizationSubband { + block_start: 1, + block_count: 1, + num_cbs_x: 1, + num_cbs_y: 1, + }, + ], + &[ + CudaHtj2kPacketizationBlock { + data_offset: 0, + data_len: u32::try_from(first_payload.len()) + .expect("test payload length fits in u32"), + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + l_block: 3, + previously_included: 0, + inclusion_layer: 0, + }, + CudaHtj2kPacketizationBlock { + data_offset: u32::try_from(first_payload.len()) + .expect("test payload length fits in u32"), + data_len: u32::try_from(second_payload.len()) + .expect("test payload length fits in u32"), + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + l_block: 5, + previously_included: 1, + inclusion_layer: 0, + }, + ], + ) + .expect("CUDA stateful packetization"); + + assert_eq!(packetized.execution().kernel_dispatches(), 1); + assert!(packetized.stage_timings().packetize_us > 0); + assert!(packetized.statuses().iter().all(|status| status.is_ok())); + assert_eq!(packetized.data(), expected); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_packetization_kernel_matches_native_scalar_deferred_first_inclusion_when_required() { + if !cuda_runtime_required() { + return; + } + + let payload = [0x44u8; 5]; + let first_block = J2kPacketizationCodeBlock { + data: &[], + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 0, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let second_block = J2kPacketizationCodeBlock { + data: &payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let resolutions = [ + J2kPacketizationResolution { + subbands: vec![J2kPacketizationSubband { + code_blocks: vec![first_block], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }, + J2kPacketizationResolution { + subbands: vec![J2kPacketizationSubband { + code_blocks: vec![second_block], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }, + ]; + let descriptors = [ + J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }, + J2kPacketizationPacketDescriptor { + packet_index: 1, + state_index: 0, + layer: 1, + resolution: 0, + component: 0, + precinct: 0, + }, + ]; + let expected = j2k_native::encode_j2k_packetization_scalar(J2kPacketizationEncodeJob { + resolution_count: 2, + num_layers: 2, + num_components: 1, + code_block_count: 2, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &descriptors, + resolutions: &resolutions, + }) + .expect("native scalar deferred inclusion packetization"); + + let context = CudaContext::system_default().expect("CUDA context"); + let packetized = context + .packetize_htj2k_cleanup_packets( + &payload, + &[ + CudaHtj2kPacketizationPacket { + block_start: 0, + block_count: 1, + subband_start: 0, + subband_count: 1, + output_capacity: 512, + layer: 0, + }, + CudaHtj2kPacketizationPacket { + block_start: 1, + block_count: 1, + subband_start: 1, + subband_count: 1, + output_capacity: 512, + layer: 1, + }, + ], + &[ + CudaHtj2kPacketizationSubband { + block_start: 0, + block_count: 1, + num_cbs_x: 1, + num_cbs_y: 1, + }, + CudaHtj2kPacketizationSubband { + block_start: 1, + block_count: 1, + num_cbs_x: 1, + num_cbs_y: 1, + }, + ], + &[ + CudaHtj2kPacketizationBlock { + data_offset: 0, + data_len: 0, + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 0, + num_zero_bitplanes: 2, + l_block: 3, + previously_included: 0, + inclusion_layer: 1, + }, + CudaHtj2kPacketizationBlock { + data_offset: 0, + data_len: u32::try_from(payload.len()) + .expect("test payload length fits in u32"), + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + l_block: 3, + previously_included: 0, + inclusion_layer: 1, + }, + ], + ) + .expect("CUDA deferred inclusion packetization"); + + assert_eq!(packetized.execution().kernel_dispatches(), 1); + assert!(packetized.stage_timings().packetize_us > 0); + assert!(packetized.statuses().iter().all(|status| status.is_ok())); + assert_eq!(packetized.data(), expected); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_packetization_kernel_matches_native_scalar_deferred_first_inclusion_after_non_empty_packet_when_required( +) { + if !cuda_runtime_required() { + return; + } + + let first_payload = [0x11u8; 3]; + let second_payload = [0x22u8; 5]; + let resolutions = [ + J2kPacketizationResolution { + subbands: vec![J2kPacketizationSubband { + code_blocks: vec![ + J2kPacketizationCodeBlock { + data: &first_payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }, + J2kPacketizationCodeBlock { + data: &[], + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 0, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }, + ], + num_cbs_x: 2, + num_cbs_y: 1, + }], + }, + J2kPacketizationResolution { + subbands: vec![J2kPacketizationSubband { + code_blocks: vec![ + J2kPacketizationCodeBlock { + data: &[], + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 0, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }, + J2kPacketizationCodeBlock { + data: &second_payload, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }, + ], + num_cbs_x: 2, + num_cbs_y: 1, + }], + }, + ]; + let descriptors = [ + J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }, + J2kPacketizationPacketDescriptor { + packet_index: 1, + state_index: 0, + layer: 1, + resolution: 0, + component: 0, + precinct: 0, + }, + ]; + let expected = j2k_native::encode_j2k_packetization_scalar(J2kPacketizationEncodeJob { + resolution_count: 2, + num_layers: 2, + num_components: 1, + code_block_count: 4, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &descriptors, + resolutions: &resolutions, + }) + .expect("native scalar deferred first inclusion after non-empty packetization"); + + let payload = [first_payload.as_slice(), second_payload.as_slice()].concat(); + let context = CudaContext::system_default().expect("CUDA context"); + let packetized = context + .packetize_htj2k_cleanup_packets_with_tag_state( + &payload, + &[ + CudaHtj2kPacketizationPacket { + block_start: 0, + block_count: 2, + subband_start: 0, + subband_count: 1, + output_capacity: 512, + layer: 0, + }, + CudaHtj2kPacketizationPacket { + block_start: 2, + block_count: 2, + subband_start: 1, + subband_count: 1, + output_capacity: 512, + layer: 1, + }, + ], + &[ + CudaHtj2kPacketizationSubband { + block_start: 0, + block_count: 2, + num_cbs_x: 2, + num_cbs_y: 1, + }, + CudaHtj2kPacketizationSubband { + block_start: 2, + block_count: 2, + num_cbs_x: 2, + num_cbs_y: 1, + }, + ], + &[ + CudaHtj2kPacketizationBlock { + data_offset: 0, + data_len: u32::try_from(first_payload.len()) + .expect("test payload length fits in u32"), + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + l_block: 3, + previously_included: 0, + inclusion_layer: 0, + }, + CudaHtj2kPacketizationBlock { + data_offset: 0, + data_len: 0, + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 0, + num_zero_bitplanes: 2, + l_block: 3, + previously_included: 0, + inclusion_layer: 1, + }, + CudaHtj2kPacketizationBlock { + data_offset: 0, + data_len: 0, + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 0, + num_zero_bitplanes: 2, + l_block: 3, + previously_included: 1, + inclusion_layer: 0, + }, + CudaHtj2kPacketizationBlock { + data_offset: u32::try_from(first_payload.len()) + .expect("test payload length fits in u32"), + data_len: u32::try_from(second_payload.len()) + .expect("test payload length fits in u32"), + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + l_block: 3, + previously_included: 0, + inclusion_layer: 1, + }, + ], + &[ + CudaHtj2kPacketizationSubbandTagState { + inclusion_node_start: 0, + zero_bitplane_node_start: 3, + node_count: 3, + reserved0: 0, + }, + CudaHtj2kPacketizationSubbandTagState { + inclusion_node_start: 6, + zero_bitplane_node_start: 9, + node_count: 3, + reserved0: 0, + }, + ], + &[ + CudaHtj2kPacketizationTagNodeState { + current: 0, + known: 0, + }, + CudaHtj2kPacketizationTagNodeState { + current: 0, + known: 0, + }, + CudaHtj2kPacketizationTagNodeState { + current: 0, + known: 0, + }, + CudaHtj2kPacketizationTagNodeState { + current: 0, + known: 0, + }, + CudaHtj2kPacketizationTagNodeState { + current: 0, + known: 0, + }, + CudaHtj2kPacketizationTagNodeState { + current: 0, + known: 0, + }, + CudaHtj2kPacketizationTagNodeState { + current: 0, + known: 1, + }, + CudaHtj2kPacketizationTagNodeState { + current: 1, + known: 0, + }, + CudaHtj2kPacketizationTagNodeState { + current: 0, + known: 1, + }, + CudaHtj2kPacketizationTagNodeState { + current: 2, + known: 1, + }, + CudaHtj2kPacketizationTagNodeState { + current: 0, + known: 0, + }, + CudaHtj2kPacketizationTagNodeState { + current: 2, + known: 1, + }, + ], + ) + .expect("CUDA deferred first inclusion after non-empty packetization"); + + assert_eq!(packetized.execution().kernel_dispatches(), 1); + assert!(packetized.stage_timings().packetize_us > 0); + assert!(packetized.statuses().iter().all(|status| status.is_ok())); + assert_eq!(packetized.data(), expected); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_mct_and_rgb_store_kernels_match_reversible_cpu_transform_when_required() { + if !cuda_runtime_required() { + return; + } + + let plane0 = [12.0f32, 25.0, 40.0, 60.0]; + let plane1 = [-3.0f32, 6.0, -10.0, 12.0]; + let plane2 = [5.0f32, -7.0, 11.0, -13.0]; + let mut expected = Vec::with_capacity(plane0.len() * 3); + for ((y0, y1), y2) in plane0 + .iter() + .copied() + .zip(plane1.iter().copied()) + .zip(plane2.iter().copied()) + { + let green = y0 - ((y2 + y1) * 0.25).floor(); + expected.push(rounded_u8(y2 + green + 128.0)); + expected.push(rounded_u8(green + 128.0)); + expected.push(rounded_u8(y1 + green + 128.0)); + } + + let context = CudaContext::system_default().expect("CUDA context"); + let plane0_buffer = context.upload_f32(&plane0).expect("upload plane 0"); + let plane1_buffer = context.upload_f32(&plane1).expect("upload plane 1"); + let plane2_buffer = context.upload_f32(&plane2).expect("upload plane 2"); + let stats = context + .j2k_inverse_mct_device( + &plane0_buffer, + &plane1_buffer, + &plane2_buffer, + CudaJ2kInverseMctJob { + len: u32::try_from(plane0.len()).expect("test plane length fits in u32"), + irreversible97: 0, + addend0: 128.0, + addend1: 128.0, + addend2: 128.0, + }, + ) + .expect("CUDA inverse RCT"); + assert_eq!(stats.kernel_dispatches(), 1); + + let stored = context + .j2k_store_rgb8_device( + &plane0_buffer, + &plane1_buffer, + &plane2_buffer, + CudaJ2kStoreRgb8Job { + input_width0: 2, + input_width1: 2, + input_width2: 2, + source_x0: 0, + source_y0: 0, + source_x1: 0, + source_y1: 0, + source_x2: 0, + source_y2: 0, + copy_width: 2, + copy_height: 2, + output_width: 2, + output_height: 2, + output_x: 0, + output_y: 0, + addend0: 0.0, + addend1: 0.0, + addend2: 0.0, + bit_depth0: 8, + bit_depth1: 8, + bit_depth2: 8, + rgba: 0, + }, + ) + .expect("CUDA RGB store"); + assert_eq!(stored.execution().kernel_dispatches(), 1); + + let mut actual = vec![0u8; expected.len()]; + stored + .buffer() + .copy_to_host(&mut actual) + .expect("download RGB pixels"); + assert_eq!(actual, expected); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_gray16_and_rgb16_store_kernels_match_cpu_scaling_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + let gray = [0.0f32, 128.0, 255.0, 300.0]; + let gray_buffer = context.upload_f32(&gray).expect("upload gray plane"); + let gray_output = context + .j2k_store_gray16_device( + &gray_buffer, + CudaJ2kStoreGray16Job { + input_width: 2, + source_x: 0, + source_y: 0, + copy_width: 2, + copy_height: 2, + output_width: 2, + output_height: 2, + output_x: 0, + output_y: 0, + addend: 0.0, + bit_depth: 8, + }, + ) + .expect("CUDA Gray16 store"); + assert_eq!(gray_output.execution().kernel_dispatches(), 1); + + let mut expected_gray = Vec::with_capacity(gray.len() * 2); + for sample in gray { + push_u16_ne(&mut expected_gray, rounded_u16(sample, 8)); + } + let mut actual_gray = vec![0u8; expected_gray.len()]; + gray_output + .buffer() + .copy_to_host(&mut actual_gray) + .expect("download Gray16 pixels"); + assert_eq!(actual_gray, expected_gray); + + let red = [0.0f32, 64.0, 128.0, 255.0]; + let green = [255.0f32, 128.0, 64.0, 0.0]; + let blue = [12.0f32, 34.0, 56.0, 78.0]; + let red_buffer = context.upload_f32(&red).expect("upload red plane"); + let green_buffer = context.upload_f32(&green).expect("upload green plane"); + let blue_buffer = context.upload_f32(&blue).expect("upload blue plane"); + let rgb_output = context + .j2k_store_rgb16_device( + &red_buffer, + &green_buffer, + &blue_buffer, + CudaJ2kStoreRgb16Job { + input_width0: 2, + input_width1: 2, + input_width2: 2, + source_x0: 0, + source_y0: 0, + source_x1: 0, + source_y1: 0, + source_x2: 0, + source_y2: 0, + copy_width: 2, + copy_height: 2, + output_width: 2, + output_height: 2, + output_x: 0, + output_y: 0, + addend0: 0.0, + addend1: 0.0, + addend2: 0.0, + bit_depth0: 8, + bit_depth1: 8, + bit_depth2: 8, + rgba: 1, + }, + ) + .expect("CUDA RGBA16 store"); + assert_eq!(rgb_output.execution().kernel_dispatches(), 1); + + let mut expected_rgb = Vec::with_capacity(red.len() * 4 * 2); + for ((r, g), b) in red.into_iter().zip(green).zip(blue) { + push_u16_ne(&mut expected_rgb, rounded_u16(r, 8)); + push_u16_ne(&mut expected_rgb, rounded_u16(g, 8)); + push_u16_ne(&mut expected_rgb, rounded_u16(b, 8)); + push_u16_ne(&mut expected_rgb, u16::MAX); + } + let mut actual_rgb = vec![0u8; expected_rgb.len()]; + rgb_output + .buffer() + .copy_to_host(&mut actual_rgb) + .expect("download RGBA16 pixels"); + assert_eq!(actual_rgb, expected_rgb); +} diff --git a/crates/j2k-cuda/tests/htj2k_encode_parity.rs b/crates/j2k-cuda/tests/htj2k_encode_parity.rs new file mode 100644 index 00000000..d1733517 --- /dev/null +++ b/crates/j2k-cuda/tests/htj2k_encode_parity.rs @@ -0,0 +1,781 @@ +// Stage-level byte/value-exact parity tests: CUDA forward-encode stages vs. +// native CPU reference. +// +// Every test gates on `cuda_runtime_required()` and returns early when +// `J2K_REQUIRE_CUDA_RUNTIME` is absent. They run for real only on the +// CI CUDA runner. + +#[cfg(feature = "cuda-runtime")] +use j2k_cuda_runtime::{CudaContext, CudaJ2kQuantizeJob}; +#[cfg(feature = "cuda-runtime")] +use j2k_native::{ + deinterleave_reference, forward_dwt53_reference, forward_rct_reference, + quantize_reversible_reference, +}; + +// --------------------------------------------------------------------------- +// Imports needed only by the facade parity matrix test. +// --------------------------------------------------------------------------- + +#[cfg(feature = "cuda-runtime")] +use j2k::{ + encode_j2k_lossless, EncodeBackendPreference, J2kBlockCodingMode, J2kEncodeValidation, + J2kLosslessEncodeOptions, J2kLosslessSamples, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_cuda::{cuda_dwt53_output_to_j2k_for_test, encode_j2k_lossless_with_cuda}; +#[cfg(feature = "cuda-runtime")] +use j2k_native::{DecodeSettings, Image}; + +// --------------------------------------------------------------------------- +// Gating helper — mirrors the pattern in htj2k_cuda_kernels.rs (line 24–26). +// --------------------------------------------------------------------------- + +#[cfg(feature = "cuda-runtime")] +use j2k_test_support::cuda_runtime_required; + +// --------------------------------------------------------------------------- +// DWT sub-band reshape +// +// The CUDA `j2k_forward_dwt53` output stores coefficients in a single +// "polyphase-flat" plane that preserves the original image stride. For a +// MULTI-level DWT the deeper levels are nested inside the LL quadrant of the +// previous level, so each sub-band must be addressed with an explicit (x0, y0) +// origin — not anchored at the buffer origin. Additionally, CUDA emits its +// `levels()` finest→coarsest, while native `forward_dwt53_reference` returns +// them coarsest→finest (it calls `levels.reverse()`), so a naive same-index +// zip compares mismatched levels. +// +// Rather than re-derive this geometry in the test (the source of the original +// bug), we reuse the EXACT production conversion `cuda_dwt53_output_to_j2k` +// (re-exported as `cuda_dwt53_output_to_j2k_for_test`). It extracts each band +// with explicit offsets (HL at (low_width, 0), LH at (0, low_height), HH at +// (low_width, low_height)) and reverses the level order, producing a +// `J2kForwardDwt53Output` directly comparable to `forward_dwt53_reference`. +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Test 1: forward DWT 5/3 +// --------------------------------------------------------------------------- + +#[cfg(feature = "cuda-runtime")] +fn assert_cuda_forward_dwt53_matches_native(width: u32, height: u32, num_levels: u8) { + // Deterministic signed-ish integer samples in [-128, 127] so the lossless + // path keeps integer coefficients (f32::from is lossless for this range). + let samples: Vec = (0u32..width * height) + .map(|i| { + let v = i16::try_from((i * 7 + 3) % 256).expect("sample fits in i16") - 128; + f32::from(v) + }) + .collect(); + + // Native CPU reference (levels ordered coarsest→finest; LL at deepest level). + let native = forward_dwt53_reference(&samples, width, height, num_levels); + + // CUDA forward DWT (levels ordered finest→coarsest in the flat plane). + let context = CudaContext::system_default().expect("CUDA context"); + let cuda_out = context + .j2k_forward_dwt53(&samples, width, height, num_levels) + .expect("CUDA forward DWT 5/3"); + + assert_eq!( + cuda_out.levels().len(), + native.levels.len(), + "level count (levels={num_levels})" + ); + assert_eq!( + cuda_out.ll_dimensions(), + (native.ll_width, native.ll_height), + "LL dimensions (levels={num_levels})" + ); + + // Convert the flat CUDA plane to the native sub-band representation using + // the SAME production reshape the encoder uses. This handles the nested + // per-level band offsets AND the finest→coarsest to coarsest→finest level + // reversal, so the result lines up index-for-index with `native`. + let cuda_as_native = cuda_dwt53_output_to_j2k_for_test(&cuda_out) + .expect("CUDA DWT output -> native subband reshape"); + + assert_eq!( + cuda_as_native.levels.len(), + native.levels.len(), + "reshaped level count (levels={num_levels})" + ); + assert_eq!( + (cuda_as_native.ll_width, cuda_as_native.ll_height), + (native.ll_width, native.ll_height), + "reshaped LL dimensions (levels={num_levels})" + ); + + // Per-level HL/LH/HH parity (both now coarsest→finest at the same index). + for (level_idx, (cuda_level, native_level)) in cuda_as_native + .levels + .iter() + .zip(native.levels.iter()) + .enumerate() + { + assert_eq!( + cuda_level.hl, native_level.hl, + "levels={num_levels} level {level_idx} HL mismatch" + ); + assert_eq!( + cuda_level.lh, native_level.lh, + "levels={num_levels} level {level_idx} LH mismatch" + ); + assert_eq!( + cuda_level.hh, native_level.hh, + "levels={num_levels} level {level_idx} HH mismatch" + ); + } + + // Deepest LL sub-band parity. + assert_eq!( + cuda_as_native.ll, native.ll, + "levels={num_levels} final LL mismatch" + ); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_forward_dwt53_matches_native_reference_when_required() { + if !cuda_runtime_required() { + return; + } + + // num_levels = 2 exercises the multi-level (nested-band) path that the + // original buggy reshape mishandled. num_levels = 1 and 3 lock the level + // ordering for the single-level and deeper-nesting cases respectively. + // 40×24 stays divisible enough that every level keeps a non-degenerate + // high-pass quadrant. + assert_cuda_forward_dwt53_matches_native(40, 24, 1); + assert_cuda_forward_dwt53_matches_native(40, 24, 2); + assert_cuda_forward_dwt53_matches_native(40, 24, 3); +} + +// --------------------------------------------------------------------------- +// Test 2: forward RCT +// --------------------------------------------------------------------------- + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_forward_rct_matches_native_reference_when_required() { + if !cuda_runtime_required() { + return; + } + + // Three 4×3 = 12-sample planes with RGB-like integer values. + let plane0: Vec = (0u8..12).map(|i| f32::from(i) * 10.0).collect(); + let plane1: Vec = (0u8..12).map(|i| f32::from(i) * 5.0 + 20.0).collect(); + let plane2: Vec = (0u8..12).map(|i| 120.0 - f32::from(i) * 8.0).collect(); + + // Native CPU reference — takes ownership, returns transformed planes. + let native = forward_rct_reference(vec![plane0.clone(), plane1.clone(), plane2.clone()]); + + // CUDA — j2k_forward_rct modifies slices in-place (lib.rs line 2532). + let mut cuda_plane0 = plane0.clone(); + let mut cuda_plane1 = plane1.clone(); + let mut cuda_plane2 = plane2.clone(); + let context = CudaContext::system_default().expect("CUDA context"); + let stats = context + .j2k_forward_rct(&mut cuda_plane0, &mut cuda_plane1, &mut cuda_plane2) + .expect("CUDA forward RCT"); + + assert_eq!(stats.kernel_dispatches(), 1); + assert_eq!(cuda_plane0, native[0], "plane 0 (Y) mismatch"); + assert_eq!(cuda_plane1, native[1], "plane 1 (Cb) mismatch"); + assert_eq!(cuda_plane2, native[2], "plane 2 (Cr) mismatch"); +} + +// --------------------------------------------------------------------------- +// Test 3: reversible sub-band quantization +// --------------------------------------------------------------------------- + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_quantize_reversible_matches_native_reference_when_required() { + if !cuda_runtime_required() { + return; + } + + // Small sub-band with a mix of positive, negative, and near-zero values. + let coefficients: Vec = vec![ + 0.0, 3.7, -8.2, 1.0, -0.5, 10.0, -1.5, 2.5, -3.0, 7.0, -4.4, 0.9, 5.6, -6.1, 1.2, -2.3, + ]; + let step_exponent: u16 = 8; + let step_mantissa: u16 = 0; + let range_bits: u8 = 8; + + // Native CPU reference (reversible = true → integer rounding). + let native = quantize_reversible_reference( + &coefficients, + step_exponent, + step_mantissa, + range_bits, + true, + ); + + // CUDA (lib.rs line 2936). + let context = CudaContext::system_default().expect("CUDA context"); + let cuda_out = context + .j2k_quantize_subband( + &coefficients, + CudaJ2kQuantizeJob { + step_exponent, + step_mantissa, + range_bits, + reversible: true, + }, + ) + .expect("CUDA reversible quantize"); + + assert_eq!(cuda_out.execution().kernel_dispatches(), 1); + assert_eq!(cuda_out.coefficients(), native.as_slice()); +} + +// --------------------------------------------------------------------------- +// Test 4: pixel deinterleave (covers 8-bit unsigned, 8-bit signed, 16-bit unsigned) +// --------------------------------------------------------------------------- + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_deinterleave_matches_native_reference_when_required() { + if !cuda_runtime_required() { + return; + } + + let context = CudaContext::system_default().expect("CUDA context"); + + // --- 4a: 8-bit unsigned RGB, 4 pixels --- + { + let pixels: Vec = vec![ + 10, 20, 30, // pixel 0 + 40, 50, 60, // pixel 1 + 70, 80, 90, // pixel 2 + 100, 110, 120, // pixel 3 + ]; + let num_pixels = 4usize; + let num_components = 3u8; + let bit_depth = 8u8; + let signed = false; + + let native = deinterleave_reference(&pixels, num_pixels, num_components, bit_depth, signed); + let cuda_out = context + .j2k_deinterleave_to_f32(&pixels, num_pixels, num_components, bit_depth, signed) + .expect("CUDA deinterleave 8-bit unsigned RGB"); + + assert_eq!( + cuda_out.execution().kernel_dispatches(), + 1, + "8-bit unsigned kernel_dispatches" + ); + // components() returns one Vec per component (lib.rs line 4550). + assert_eq!( + cuda_out.components(), + native.as_slice(), + "8-bit unsigned deinterleave mismatch" + ); + } + + // --- 4b: 8-bit signed single-component grayscale, 6 pixels --- + { + let pixels: Vec = (0u8..6).collect(); + let num_pixels = 6usize; + let num_components = 1u8; + let bit_depth = 8u8; + let signed = true; + + let native = deinterleave_reference(&pixels, num_pixels, num_components, bit_depth, signed); + let cuda_out = context + .j2k_deinterleave_to_f32(&pixels, num_pixels, num_components, bit_depth, signed) + .expect("CUDA deinterleave 8-bit signed gray"); + + assert_eq!( + cuda_out.components(), + native.as_slice(), + "8-bit signed grayscale deinterleave mismatch" + ); + } + + // --- 4c: 16-bit unsigned RGB, 2 pixels (little-endian pairs) --- + { + // Pack 2 pixels × 3 components × 2 bytes = 12 bytes. + let values: &[u16] = &[0x0010, 0x0080, 0x00FF, 0x0100, 0x0800, 0x0FFF]; + let mut pixels: Vec = Vec::with_capacity(values.len() * 2); + for v in values { + // Native deinterleave reads u16::from_le_bytes and the CUDA kernel + // reads little-endian explicitly, so pack little-endian regardless + // of host endianness. + pixels.extend_from_slice(&v.to_le_bytes()); + } + let num_pixels = 2usize; + let num_components = 3u8; + let bit_depth = 16u8; + let signed = false; + + let native = deinterleave_reference(&pixels, num_pixels, num_components, bit_depth, signed); + let cuda_out = context + .j2k_deinterleave_to_f32(&pixels, num_pixels, num_components, bit_depth, signed) + .expect("CUDA deinterleave 16-bit unsigned RGB"); + + assert_eq!( + cuda_out.components(), + native.as_slice(), + "16-bit unsigned deinterleave mismatch" + ); + } + + // --- 4d: 16-bit signed single-component, 4 pixels --- + { + let values: &[i16] = &[-32768, -1, 0, 32767]; + let mut pixels: Vec = Vec::with_capacity(values.len() * 2); + for v in values { + // Little-endian to match native deinterleave and the CUDA kernel. + pixels.extend_from_slice(&v.to_le_bytes()); + } + let num_pixels = 4usize; + let num_components = 1u8; + let bit_depth = 16u8; + let signed = true; + + let native = deinterleave_reference(&pixels, num_pixels, num_components, bit_depth, signed); + let cuda_out = context + .j2k_deinterleave_to_f32(&pixels, num_pixels, num_components, bit_depth, signed) + .expect("CUDA deinterleave 16-bit signed gray"); + + assert_eq!( + cuda_out.components(), + native.as_slice(), + "16-bit signed deinterleave mismatch" + ); + } +} + +// --------------------------------------------------------------------------- +// Tripwire (ungated — runs on every host, no GPU required) +// --------------------------------------------------------------------------- +// +// If CI sets J2K_REQUIRE_CUDA_RUNTIME but the `cuda-runtime` feature was +// not compiled in, every gated parity test early-returns and false-greens. +// This test is NOT inside any `#[cfg(feature = "cuda-runtime")]` block so it +// is always compiled and always executed regardless of feature flags. + +#[test] +fn cuda_runtime_required_implies_feature_compiled() { + // Fail-closed: if CI asserts the CUDA runtime is required but the cuda-runtime + // feature was not compiled in, every gated parity test would silently early-return + // and masquerade as green. Make that misconfiguration a hard failure instead. + let cuda_runtime_required = std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_some(); + let cuda_runtime_feature_enabled = cfg!(feature = "cuda-runtime"); + assert!( + !cuda_runtime_required || cuda_runtime_feature_enabled, + "J2K_REQUIRE_CUDA_RUNTIME is set but the cuda-runtime feature is not compiled — \ + gated CUDA parity tests would silently skip and false-green" + ); +} + +// --------------------------------------------------------------------------- +// Phase-1 ACCEPTANCE GATE +// Test 5: CUDA facade byte-exact parity matrix vs. CPU reference +// --------------------------------------------------------------------------- +// +// CPU reference rationale +// ----------------------- +// `encode_j2k_lossless_with_cuda` calls `strict_cuda_encode_options(*options)` +// (crates/j2k-cuda/src/encode.rs:88–91), which sets +// `backend = RequireDevice` and otherwise leaves all other fields untouched. +// It then delegates to `j2k::encode_j2k_lossless_with_accelerator` +// (encode.rs:40–45), which internally calls `native_lossless_options` to build +// the `EncodeOptions` for the native encoder (j2k/src/encode.rs:377–392). +// +// The CPU reference here is `j2k::encode_j2k_lossless(samples, &opts)` +// with `backend = CpuOnly` and all other option fields identical to what the +// caller passed to the CUDA facade. `encode_j2k_lossless` calls `encode_cpu` +// (encode.rs:254–270) which calls the same `native_lossless_options` helper, +// producing the same `EncodeOptions` (same reversible transform, block coding +// mode, progression order, decomposition levels, write_tlm flag, use_mct flag). +// The only difference is the dispatch path: CUDA uses the GPU accelerator for +// compute stages; CPU uses the scalar fallback. When the GPU produces a +// bit-identical codestream for supported configurations, `bytes_cuda == +// bytes_native` must hold. +// +// Validation policy for the CPU call is `External` to avoid double round-trip +// overhead in CI; the CUDA facade runs its own validation internally. +// +// Round-trip decode scope (signed vs unsigned) +// -------------------------------------------- +// Every cell asserts CUDA-vs-native CODESTREAM byte parity (the Phase-1 +// deliverable) and that the codestream parses and decodes. The additional +// byte-exact pixel round-trip — decode the codestream, compare to the original +// input — is asserted only for UNSIGNED components. The native DECODER does not +// reconstruct signed samples: it reads the SIZ Ssiz signed bit and ignores it +// (j2k-native/src/j2c/codestream.rs) and unconditionally re-applies the +// unsigned inverse DC level-shift, then clamps negatives +// (decode_native_with_context in j2k-native/src/lib.rs), so signed +// output is offset by +2^(depth-1). This is a SHARED native-decoder limitation — +// the CPU and Metal decode paths are affected identically because decode +// reconstruction is shared — and is independent of the CUDA ENCODER, whose +// codestream is byte-identical to native for signed and unsigned alike (asserted +// for every cell). Signed-source decode round-trip is not a supported contract +// in this repo (e.g. the recode path rejects signed sources outright). Tracked +// as a native-decoder non-goal in the public support policy. + +// Cell descriptor used for matrix iteration and assertion messages. +#[cfg(feature = "cuda-runtime")] +#[derive(Debug, Clone, Copy)] +struct ParityCell { + w: u32, + h: u32, + comps: u8, + depth: u8, + signed: bool, + levels: u8, +} + +/// Build a deterministic byte buffer for the given cell geometry. +/// +/// For 8-bit samples each byte is a single sample. +/// For 16-bit samples each sample is two bytes (little-endian u16), packed +/// interleaved (e.g., `lo_byte`, `hi_byte` per sample per component). The +/// native encoder reads 16-bit samples with `u16::from_le_bytes`, so packing +/// little-endian keeps the input interpretation host-endianness independent. +#[cfg(feature = "cuda-runtime")] +fn synthesize_pixels(cell: &ParityCell) -> Vec { + let npixels = cell.w as usize * cell.h as usize; + let nsamples = npixels * cell.comps as usize; + + if cell.depth <= 8 { + // 8-bit: one byte per sample; deterministic from index. + // The modulus is exactly 256, so truncation is intentional. + #[allow(clippy::cast_possible_truncation)] + (0..nsamples).map(|i| ((i * 31 + 17) % 256) as u8).collect() + } else { + // 16-bit: two bytes per sample (little-endian u16, matching the + // native encoder's u16::from_le_bytes read). + // + // `modulus == 2^depth`; for the in-scope depths (≤ 16) this is ≤ 65536, + // so every `value % modulus` lands in 0..2^16 and the u16 cast is exact. + let modulus: u64 = 1u64 << cell.depth; + let mut buf = Vec::with_capacity(nsamples * 2); + for i in 0..nsamples { + // Mix in u64: the intermediate product `idx * 1_000_003` exceeds + // u32::MAX once idx ≥ 4295 (reached at comps ≥ 3, since + // nsamples = w*h*comps), which previously panicked in debug builds + // with "attempt to multiply with overflow". idx ≤ 12288 here, so + // idx * 1_000_003 + 7 < 2^34 and cannot overflow u64. + let idx = i as u64; + // The modulo keeps the result < modulus ≤ 2^16, so the cast is exact. + #[allow(clippy::cast_possible_truncation)] + let v = ((idx * 1_000_003 + 7) % modulus) as u16; + buf.extend_from_slice(&v.to_le_bytes()); + } + buf + } +} + +/// Build the `J2kLosslessEncodeOptions` for a given cell. +/// +/// We use HTJ2K block coding and LRCP progression (matching the strict CUDA +/// facade's default contract), and pass the cell's `levels` as an explicit +/// `max_decomposition_levels` request. The reversible transform is left at +/// the facade default (`Rct53` for 3-component; the native encoder also applies +/// it to 1-component via a no-op path that does not change the codestream bytes). +#[cfg(feature = "cuda-runtime")] +fn cell_encode_options(cell: &ParityCell) -> J2kLosslessEncodeOptions { + J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(cell.levels)) + .with_validation(J2kEncodeValidation::External) +} + +// Matrix constants live at module scope to avoid the items_after_statements lint. +// comps ∈ {1, 3, 4} (2-component is OUT OF SCOPE — native decoder rejects it). +// 4-component resident MCT is implemented (RCT on planes 0-2, passthrough 4); +// these cells are now asserted byte-exact like every other supported cell. +#[cfg(feature = "cuda-runtime")] +const MATRIX_COMPS: &[u8] = &[1, 3, 4]; +#[cfg(feature = "cuda-runtime")] +const MATRIX_DEPTHS: &[u8] = &[8, 16]; +#[cfg(feature = "cuda-runtime")] +const MATRIX_SIGNED: &[bool] = &[false, true]; +#[cfg(feature = "cuda-runtime")] +const MATRIX_LEVELS: &[u8] = &[0, 1, 3]; + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_facade_byte_matches_native_across_matrix_when_required() { + if !cuda_runtime_required() { + return; + } + + let w: u32 = 64; + let h: u32 = 48; + + let mut failures: Vec = Vec::new(); + + for &comps in MATRIX_COMPS { + for &depth in MATRIX_DEPTHS { + for &signed in MATRIX_SIGNED { + for &levels in MATRIX_LEVELS { + let cell = ParityCell { + w, + h, + comps, + depth, + signed, + levels, + }; + let pixels = synthesize_pixels(&cell); + let opts = cell_encode_options(&cell); + + // Build J2kLosslessSamples through the real public + // constructor for every cell, including 4-component: + // J2kLosslessSamples::new now accepts comps ∈ {1, 3, 4} + // (2-component stays rejected). The CUDA facade supports + // 4-component resident MCT end-to-end. + let samples = J2kLosslessSamples::new( + pixels.as_slice(), + cell.w, + cell.h, + cell.comps, + cell.depth, + cell.signed, + ) + .unwrap_or_else(|err| { + panic!("cell={cell:?}: J2kLosslessSamples::new rejected an in-scope cell: {err}") + }); + + // --- CUDA encode --- + let cuda_result = encode_j2k_lossless_with_cuda(samples, &opts); + + // --- CPU reference encode (configuration-identical) --- + // Uses encode_j2k_lossless with CpuOnly backend so it takes + // the identical native_lossless_options path as the CUDA facade + // (j2k/src/encode.rs:341–355 via encode_cpu). + let cpu_opts = opts.with_backend(EncodeBackendPreference::CpuOnly); + let cpu_result = encode_j2k_lossless(samples, &cpu_opts); + + match (cuda_result, cpu_result) { + (Err(cuda_err), Err(cpu_err)) => { + // Every matrix cell (comps ∈ {1, 3, 4}) is in scope and + // must encode successfully on both paths. + failures.push(format!( + "cell={cell:?}: both encoders rejected an in-scope cell: \ + cuda_err={cuda_err} cpu_err={cpu_err}" + )); + } + (Err(cuda_err), Ok(_)) => { + // CUDA must dispatch for every in-scope cell, including + // 4-component (resident MCT lands in P1-T7). + failures.push(format!( + "cell={cell:?}: CUDA encode failed but CPU succeeded: \ + {cuda_err}" + )); + } + (Ok(_), Err(cpu_err)) => { + failures.push(format!( + "cell={cell:?}: CPU encode failed but CUDA succeeded: \ + {cpu_err}" + )); + } + (Ok(bytes_cuda), Ok(bytes_native)) => { + // --- Byte-exact codestream parity assertion --- + if bytes_cuda.codestream != bytes_native.codestream { + failures.push(format!( + "cell={cell:?}: codestream byte mismatch \ + (cuda={} bytes, cpu={} bytes)", + bytes_cuda.codestream.len(), + bytes_native.codestream.len() + )); + continue; + } + + // --- Round-trip decode of the CUDA codestream --- + // Parse + decode for EVERY cell to prove the (byte-identical) + // codestream is well-formed and decodable. The byte-exact pixel + // comparison is asserted only for UNSIGNED components; see the + // "Round-trip decode scope" note in the acceptance-gate header + // above. In brief: the native decoder does not reconstruct + // signed samples — it reads the SIZ Ssiz signed bit but ignores + // it (j2k-native/src/j2c/codestream.rs) and + // unconditionally applies the unsigned inverse level-shift + + // clamp (decode_native_with_context in + //j2k-native/src/lib.rs), so signed output is offset by + // +2^(depth-1). That is a shared native-decoder limitation + // (CPU and Metal decode are affected identically) and is + // independent of the CUDA *encoder*, whose codestream byte-parity + // is asserted above for signed and unsigned alike. + let image = match Image::new( + &bytes_cuda.codestream, + &DecodeSettings::default(), + ) { + Ok(img) => img, + Err(e) => { + failures.push(format!( + "cell={cell:?}: CUDA codestream parse failed: {e}" + )); + continue; + } + }; + + let decoded = match image.decode_native() { + Ok(bmp) => bmp, + Err(e) => { + failures.push(format!( + "cell={cell:?}: CUDA codestream decode failed: {e}" + )); + continue; + } + }; + + if cell.signed { + // Signed values cannot round-trip through the native decoder + // (see note). Still require the decoded buffer to have the + // correct shape — confirms dimensions, component count, and + // bit depth survived encode + decode. + let expected_len = pixels.len(); + if decoded.data.len() != expected_len { + failures.push(format!( + "cell={cell:?}: signed decode produced wrong-sized \ + buffer (decoded={} bytes, expected={expected_len})", + decoded.data.len() + )); + } + } else if decoded.data != pixels { + failures.push(format!( + "cell={cell:?}: round-trip pixel mismatch \ + (decoded={} bytes, original={} bytes)", + decoded.data.len(), + pixels.len() + )); + } + } + } + } + } + } + } + + assert!( + failures.is_empty(), + "facade parity matrix failures:\n{}", + failures.join("\n") + ); +} + +// --------------------------------------------------------------------------- +// Test 6: subsampling != (1,1) rejection via the accelerator hook (runner-gated) +// +// `cuda_encode_htj2k_tile_body` now returns a typed Err — not Ok(None) — for +// inputs with component subsampling factors != (1, 1). This test verifies the +// rejection at the hook level via `J2kEncodeStageAccelerator::encode_htj2k_tile` +// on the CUDA runner where the subsampling check actually executes. +// +// NOTE: Subsampling != (1,1) is NOT expressible through the public lossless +// facade (`encode_j2k_lossless_with_cuda` / `J2kLosslessEncodeOptions`): +// `native_lossless_options` never sets `EncodeOptions::component_sampling`, +// which defaults to `None` → all (1,1). The rejection here is a defense-in- +// depth contract within `cuda_encode_htj2k_tile_body`, exercised via the +// lower-level hook to confirm the guard exists. +// --------------------------------------------------------------------------- + +#[cfg(feature = "cuda-runtime")] +#[test] +fn cuda_htj2k_tile_encode_hook_rejects_subsampling_with_typed_err_when_cuda_runtime_required() { + use j2k::adapter::encode_stage::{ + J2kEncodeStageAccelerator as _, J2kHtj2kTileEncodeJob, J2kPacketizationProgressionOrder, + }; + use j2k_cuda::CudaEncodeStageAccelerator; + + if !cuda_runtime_required() { + return; + } + + // Minimal 4x4 grayscale HTJ2K tile job with subsampling (2,1) on the single + // component — this is the only path that triggers the subsampling guard in + // cuda_encode_htj2k_tile_body. + let pixels: Vec = (0u8..16).collect(); + let quantization_steps = [(8u16, 0u16)]; // num_decomposition_levels=0 → 1 step + let sampling = [(2u8, 1u8)]; // non-unit subsampling for component 0 + + let mut accelerator = CudaEncodeStageAccelerator::default(); + let result = accelerator.encode_htj2k_tile(J2kHtj2kTileEncodeJob { + pixels: &pixels, + width: 4, + height: 4, + num_components: 1, + bit_depth: 8, + signed: false, + reversible: true, + use_mct: false, + num_decomposition_levels: 0, + guard_bits: 1, + code_block_width: 4, + code_block_height: 4, + quantization_steps: &quantization_steps, + component_sampling: &sampling, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + }); + + let err = result.expect_err( + "encode_htj2k_tile must return Err for subsampling != (1,1) when CUDA is available", + ); + assert!( + err.contains("subsampling"), + "typed rejection must mention subsampling, got: {err}" + ); +} + +// --------------------------------------------------------------------------- +// Test 7 (CPU-only, no runtime gate): assert that the lossless facade's public +// types cannot express subsampling != (1,1). +// +// `J2kLosslessEncodeOptions` has no subsampling field. `native_lossless_options` +// always leaves `EncodeOptions::component_sampling` as `None`, which the native +// encoder resolves to all-(1,1). This structural invariant means the subsampling +// rejection in `cuda_encode_htj2k_tile_body` is unreachable through the facade. +// +// This test encodes a 1-component in-scope cell and asserts the call succeeds — +// proving no silent Ok(None) fallback occurs for in-scope inputs. +// --------------------------------------------------------------------------- + +#[test] +fn lossless_facade_in_scope_input_never_hits_ok_none_fallback() { + use j2k::{ + J2kBlockCodingMode, J2kEncodeValidation, J2kLosslessEncodeOptions, J2kLosslessSamples, + }; + use j2k_core::{BackendKind, CodecError as _}; + use j2k_cuda::encode_j2k_lossless_with_cuda; + + // Small 8x8 single-component 8-bit input — minimal in-scope cell. + let pixels: Vec = (0u8..64).collect(); + let samples = + J2kLosslessSamples::new(&pixels, 8, 8, 1, 8, false).expect("valid in-scope samples"); + let options = J2kLosslessEncodeOptions::default() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::External); + + let result = encode_j2k_lossless_with_cuda(samples, &options); + + // Without CUDA, the facade must return a typed Err (not Ok with CPU backend), + // because strict_cuda_encode_options forces RequireDevice and the missing + // deinterleave dispatch triggers the unsupported-backend error. + // With CUDA, the call succeeds. + // Either way, the result must NOT be Ok with backend == Cpu (silent fallback). + match result { + Ok(encoded) => { + assert_eq!( + encoded.backend, + BackendKind::Cuda, + "encode_j2k_lossless_with_cuda must not silently fall back to CPU backend for in-scope inputs" + ); + } + Err(err) => { + // Strict encode error is expected when CUDA is unavailable. + assert!( + err.is_unsupported(), + "rejection for an in-scope input must be a typed unsupported error, not an internal panic or silent fallback: {err}" + ); + } + } +} diff --git a/crates/j2k-cuda/tests/htj2k_plan.rs b/crates/j2k-cuda/tests/htj2k_plan.rs new file mode 100644 index 00000000..85408590 --- /dev/null +++ b/crates/j2k-cuda/tests/htj2k_plan.rs @@ -0,0 +1,237 @@ +use j2k_core::{CodecError, PixelFormat, Rect}; +use j2k_cuda::{CudaHtj2kTransform, J2kDecoder, SurfaceResidency}; +use j2k_test_support::{ + classic_j2k_gray8_fixture, htj2k_gray8_97_fixture, htj2k_gray8_fixture, + htj2k_gray8_large_fixture, + openhtj2k_refinement_odd_fixture as shared_openhtj2k_refinement_odd_fixture, +}; + +fn ht_gray8_fixture() -> Vec { + htj2k_gray8_fixture(8, 8) +} + +fn ht_gray8_irreversible_97_fixture() -> Vec { + htj2k_gray8_97_fixture(8, 8) +} + +fn ht_gray8_large_fixture() -> Vec { + htj2k_gray8_large_fixture(256, 256) +} + +fn classic_gray8_fixture() -> Vec { + classic_j2k_gray8_fixture(8, 8) +} + +fn openhtj2k_refinement_odd_fixture() -> &'static [u8] { + shared_openhtj2k_refinement_odd_fixture() +} + +#[test] +fn flat_htj2k_plan_contains_payload_offsets_not_pointers() { + let bytes = ht_gray8_fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let (cuda_plan, _) = decoder + .build_cuda_htj2k_grayscale_plan_with_profile(PixelFormat::Gray8) + .expect("CUDA flat plan"); + + assert_eq!(cuda_plan.dimensions(), (8, 8)); + assert_eq!(cuda_plan.output_format(), PixelFormat::Gray8); + assert_eq!(cuda_plan.transform(), CudaHtj2kTransform::Reversible53); + assert!(!cuda_plan.payload().is_empty()); + assert!(!cuda_plan.code_blocks().is_empty()); + assert!(!cuda_plan.subbands().is_empty()); + assert_eq!( + cuda_plan.dispatch_count_hint(), + cuda_plan.code_blocks().len() + ); + + let block_payload_len = cuda_plan + .code_blocks() + .iter() + .map(|block| block.payload_len as usize) + .sum(); + assert_eq!(cuda_plan.payload().len(), block_payload_len); + + for block in cuda_plan.code_blocks() { + let start = usize::try_from(block.payload_offset).expect("payload offset fits usize"); + let end = start + block.payload_len as usize; + assert!(end <= cuda_plan.payload().len()); + assert_eq!( + block.payload_len, + block.cleanup_length + block.refinement_length + ); + } +} + +#[test] +fn flat_htj2k_plan_preserves_refinement_payload_metadata() { + let mut decoder = J2kDecoder::new(openhtj2k_refinement_odd_fixture()).expect("decoder"); + let (cuda_plan, _) = decoder + .build_cuda_htj2k_grayscale_plan_with_profile(PixelFormat::Gray8) + .expect("CUDA refinement flat plan"); + + assert_eq!(cuda_plan.code_blocks().len(), 14); + let mut cursor = 0usize; + let mut refinement_blocks = 0usize; + for block in cuda_plan.code_blocks() { + assert_eq!(block.payload_offset, cursor as u64); + assert_eq!( + block.payload_len, + block.cleanup_length + block.refinement_length + ); + let end = cursor + block.payload_len as usize; + assert!(end <= cuda_plan.payload().len()); + cursor = end; + if block.refinement_length > 0 { + refinement_blocks += 1; + } + } + assert_eq!(cursor, cuda_plan.payload().len()); + assert_eq!(refinement_blocks, 14); +} + +#[test] +fn flat_htj2k_plan_records_irreversible_97_transform() { + let bytes = ht_gray8_irreversible_97_fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let (cuda_plan, _) = decoder + .build_cuda_htj2k_grayscale_plan_with_profile(PixelFormat::Gray8) + .expect("CUDA flat 9/7 plan"); + + assert_eq!(cuda_plan.transform(), CudaHtj2kTransform::Irreversible97); + assert!(!cuda_plan.idwt_steps().is_empty()); + assert!(cuda_plan + .idwt_steps() + .iter() + .all(|step| step.transform == CudaHtj2kTransform::Irreversible97)); +} + +#[test] +fn flat_htj2k_region_plan_stores_compact_output_rect() { + let bytes = ht_gray8_fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let (cuda_plan, _) = decoder + .build_cuda_htj2k_grayscale_region_plan_with_profile( + PixelFormat::Gray8, + Rect { + x: 2, + y: 1, + w: 4, + h: 4, + }, + ) + .expect("CUDA ROI flat plan"); + + assert_eq!(cuda_plan.dimensions(), (4, 4)); + assert_eq!(cuda_plan.output_origin(), (2, 1)); + let [store] = cuda_plan.store_steps() else { + panic!("expected one ROI store"); + }; + assert_eq!(store.output_width, 4); + assert_eq!(store.output_height, 4); + assert_eq!(store.source_x, 2); + assert_eq!(store.source_y, 1); + assert_eq!(store.output_x, 0); + assert_eq!(store.output_y, 0); + assert_eq!(store.copy_width, 4); + assert_eq!(store.copy_height, 4); +} + +#[test] +fn flat_htj2k_region_plan_prunes_code_blocks() { + let bytes = ht_gray8_large_fixture(); + let mut full_decoder = J2kDecoder::new(&bytes).expect("full decoder"); + let (full_cuda_plan, _) = full_decoder + .build_cuda_htj2k_grayscale_scaled_plan_with_profile(PixelFormat::Gray8, (64, 64)) + .expect("CUDA full flat plan"); + + let mut roi_decoder = J2kDecoder::new(&bytes).expect("ROI decoder"); + let (roi_cuda_plan, _) = roi_decoder + .build_cuda_htj2k_grayscale_region_scaled_plan_with_profile( + PixelFormat::Gray8, + Rect { + x: 24, + y: 24, + w: 8, + h: 8, + }, + (64, 64), + ) + .expect("CUDA ROI flat plan"); + + assert!( + roi_cuda_plan.code_blocks().len() < full_cuda_plan.code_blocks().len(), + "ROI should hand CUDA fewer code blocks than a full decode" + ); + assert_eq!(roi_cuda_plan.dimensions(), (8, 8)); + let [store] = roi_cuda_plan.store_steps() else { + panic!("expected one ROI store"); + }; + assert_eq!(store.copy_width, 8); + assert_eq!(store.copy_height, 8); +} + +#[test] +fn flat_htj2k_scaled_region_plan_stores_compact_scaled_rect() { + let bytes = ht_gray8_fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let (cuda_plan, _) = decoder + .build_cuda_htj2k_grayscale_region_scaled_plan_with_profile( + PixelFormat::Gray8, + Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }, + (4, 4), + ) + .expect("CUDA scaled ROI flat plan"); + + assert_eq!(cuda_plan.dimensions(), (2, 3)); + assert_eq!(cuda_plan.output_origin(), (1, 0)); + let [store] = cuda_plan.store_steps() else { + panic!("expected one scaled ROI store"); + }; + assert_eq!(store.output_width, 2); + assert_eq!(store.output_height, 3); + assert_eq!(store.source_x, 1); + assert_eq!(store.source_y, 0); + assert_eq!(store.output_x, 0); + assert_eq!(store.output_y, 0); + assert_eq!(store.copy_width, 2); + assert_eq!(store.copy_height, 3); +} + +#[test] +fn flat_htj2k_plan_rejects_classic_j2k_subband_steps() { + let bytes = classic_gray8_fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let error = decoder + .build_cuda_htj2k_grayscale_plan_with_profile(PixelFormat::Gray8) + .expect_err("classic plans must be rejected"); + + assert!(error.is_unsupported()); +} + +#[test] +fn grayscale_plan_profile_reports_stable_cuda_htj2k_fields() { + let bytes = ht_gray8_fixture(); + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + + let (plan, report) = decoder + .build_cuda_htj2k_grayscale_plan_with_profile(PixelFormat::Gray8) + .expect("profiled plan"); + + assert_eq!(report.block_count, plan.code_blocks().len()); + assert_eq!(report.payload_bytes, plan.payload().len()); + assert_eq!(report.dispatch_count, 0); + assert_eq!(report.residency, SurfaceResidency::CudaResidentDecode); + assert_eq!(report.h2d_us, 0); + assert_eq!(report.ht_cleanup_us, 0); + assert_eq!(report.ht_refine_us, 0); + assert_eq!(report.dequant_us, 0); + assert_eq!(report.idwt_us, 0); + assert_eq!(report.mct_us, 0); + assert_eq!(report.store_us, 0); +} diff --git a/crates/signinum-jpeg-cuda/Cargo.toml b/crates/j2k-jpeg-cuda/Cargo.toml similarity index 54% rename from crates/signinum-jpeg-cuda/Cargo.toml rename to crates/j2k-jpeg-cuda/Cargo.toml index 7074cf8f..02496ff0 100644 --- a/crates/signinum-jpeg-cuda/Cargo.toml +++ b/crates/j2k-jpeg-cuda/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "signinum-jpeg-cuda" -description = "CUDA device-output adapter for signinum-jpeg" -version = "0.4.2" +name = "j2k-jpeg-cuda" +description = "CUDA JPEG decode adapter for j2k-jpeg" +version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true @@ -10,25 +10,28 @@ keywords.workspace = true categories.workspace = true readme = "README.md" +[package.metadata.docs.rs] +all-features = true +targets = [] + [lib] -name = "signinum_jpeg_cuda" +name = "j2k_jpeg_cuda" path = "src/lib.rs" [features] default = [] -cuda-runtime = ["dep:signinum-cuda-runtime"] +cuda-runtime = ["dep:j2k-cuda-runtime"] [dependencies] -signinum-core = { path = "../signinum-core", version = "=0.4.2" } -signinum-cuda-runtime = { path = "../signinum-cuda-runtime", version = "=0.4.2", optional = true } -signinum-jpeg = { path = "../signinum-jpeg", version = "=0.4.2" } -signinum-profile = { path = "../signinum-profile", version = "=0.4.2" } +j2k-core = { path = "../j2k-core", version = "=0.6.0" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.6.0", optional = true } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.6.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.6.0" } thiserror = { workspace = true } [dev-dependencies] criterion = { workspace = true } -jpeg-encoder = "0.7.0" -signinum-test-support = { path = "../signinum-test-support" } +j2k-test-support = { path = "../j2k-test-support" } [[bench]] name = "device_decode" @@ -41,6 +44,7 @@ unreachable_pub = "warn" [lints.clippy] pedantic = { level = "warn", priority = -1 } +undocumented_unsafe_blocks = "warn" module_name_repetitions = "allow" must_use_candidate = "allow" missing_errors_doc = "allow" diff --git a/crates/j2k-jpeg-cuda/README.md b/crates/j2k-jpeg-cuda/README.md new file mode 100644 index 00000000..aed2b9e1 --- /dev/null +++ b/crates/j2k-jpeg-cuda/README.md @@ -0,0 +1,7 @@ +# j2k-jpeg-cuda + +CUDA adapter for J2K JPEG decode surfaces. + +Supported CUDA paths use J2K-owned CUDA kernels and CUDA device memory +outputs. Explicit CUDA requests are strict; unsupported JPEG shapes return +structured errors instead of silently falling back to CPU. diff --git a/crates/j2k-jpeg-cuda/benches/device_decode.rs b/crates/j2k-jpeg-cuda/benches/device_decode.rs new file mode 100644 index 00000000..8fee1adb --- /dev/null +++ b/crates/j2k-jpeg-cuda/benches/device_decode.rs @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: Apache-2.0 + +use criterion::{criterion_group, criterion_main, Criterion}; +use j2k_core::{ + BackendRequest, DeviceSubmission, DeviceSurface, ImageDecodeDevice, ImageDecodeSubmit, + PixelFormat, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_core::{DecoderContext, TileBatchDecodeManyDevice}; +use j2k_jpeg::{ + encode_jpeg_baseline, Decoder as CpuDecoder, JpegBackend, JpegEncodeOptions, JpegSamples, + JpegSubsampling, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_jpeg_cuda::Codec as CudaCodec; +use j2k_jpeg_cuda::{CudaSession, Decoder as CudaDecoder}; + +const DEFAULT_JPEG: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); +const DEFAULT_GENERATED_DIM: u16 = 2048; +#[cfg(feature = "cuda-runtime")] +const DEFAULT_BATCH_DIM: u16 = 1024; +#[cfg(feature = "cuda-runtime")] +const DEFAULT_BATCH_SIZE: usize = 64; + +fn bench_device_decode(c: &mut Criterion) { + let input = bench_input(); + let mut group = c.benchmark_group("jpeg_cuda_device_decode"); + + group.bench_function("cpu_decode_rgb8", |b| { + b.iter(|| { + let decoder = CpuDecoder::new(&input).expect("cpu decoder"); + decoder.decode(PixelFormat::Rgb8).expect("cpu decode") + }); + }); + + match cuda_probe(&input) { + Some(probe) => { + let label = if probe.used_owned_cuda_decode { + "cuda_owned_rgb8_surface" + } else { + "cuda_upload_fallback_rgb8_surface" + }; + group.bench_function(label, |b| { + let mut session = CudaSession::default(); + b.iter(|| { + let mut decoder = CudaDecoder::new(&input).expect("cuda decoder"); + as ImageDecodeSubmit<'_>>::submit_to_device( + &mut decoder, + &mut session, + PixelFormat::Rgb8, + BackendRequest::Cuda, + ) + .expect("cuda submit") + .wait() + .expect("cuda decode") + }); + }); + + let label = if probe.used_owned_cuda_decode { + "cuda_owned_rgb8_download" + } else { + "cuda_upload_fallback_rgb8_download" + }; + group.bench_function(label, |b| { + let mut session = CudaSession::default(); + b.iter(|| { + let mut decoder = CudaDecoder::new(&input).expect("cuda decoder"); + let surface = as ImageDecodeSubmit<'_>>::submit_to_device( + &mut decoder, + &mut session, + PixelFormat::Rgb8, + BackendRequest::Cuda, + ) + .expect("cuda submit") + .wait() + .expect("cuda decode"); + let mut out = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut out, surface.pitch_bytes()) + .expect("cuda download"); + out + }); + }); + } + None if std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_some() => { + panic!("J2K_REQUIRE_CUDA_BENCH is set but CUDA decode is unavailable") + } + None => { + eprintln!("skipping CUDA decode benches: CUDA runtime is unavailable"); + } + } + + group.finish(); + + bench_batch_decode(c); + bench_chunked_entropy_diagnostic(c); +} + +fn bench_input() -> Vec { + let path = + std::env::var_os("J2K_CUDA_BENCH_JPEG").or_else(|| std::env::var_os("J2K_GPU_BENCH_JPEG")); + match path { + Some(path) => std::fs::read(&path).unwrap_or_else(|error| { + panic!( + "failed to read J2K_CUDA_BENCH_JPEG={}: {error}", + path.to_string_lossy() + ) + }), + None if std::env::var_os("J2K_GPU_BENCH_SMALL_FIXTURE").is_some() => DEFAULT_JPEG.to_vec(), + None => { + let (width, height) = generated_dimensions(); + generated_jpeg(width, height) + } + } +} + +fn generated_jpeg(width: u16, height: u16) -> Vec { + let rgb = j2k_test_support::gpu_bench_rgb8(u32::from(width), u32::from(height)); + encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: u32::from(width), + height: u32::from(height), + }, + JpegEncodeOptions { + quality: 90, + subsampling: bench_subsampling(), + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode generated benchmark JPEG") + .data +} + +fn generated_dimensions() -> (u16, u16) { + let Some(value) = std::env::var_os("J2K_GPU_BENCH_DIM") else { + return (DEFAULT_GENERATED_DIM, DEFAULT_GENERATED_DIM); + }; + + parse_dimensions(&value.to_string_lossy()) +} + +fn parse_dimensions(value: &str) -> (u16, u16) { + if let Some((width, height)) = value.split_once('x') { + parse_dimensions_pair(width, height) + } else { + let square = value + .parse::() + .expect("J2K_GPU_BENCH_DIM must be a u16 or WIDTHxHEIGHT"); + assert_in_bench_bounds(square); + (square, square) + } +} + +fn parse_dimensions_pair(width: &str, height: &str) -> (u16, u16) { + let width = width + .trim() + .parse::() + .expect("J2K_GPU_BENCH_DIM must be a u16 or WIDTHxHEIGHT"); + let height = height + .trim() + .parse::() + .expect("J2K_GPU_BENCH_DIM must be a u16 or WIDTHxHEIGHT"); + assert_in_bench_bounds(width); + assert_in_bench_bounds(height); + (width, height) +} + +fn assert_in_bench_bounds(value: u16) { + assert!( + (256..=8192).contains(&value), + "J2K_GPU_BENCH_DIM dimensions must be between 256 and 8192" + ); +} + +fn bench_subsampling() -> JpegSubsampling { + let value = std::env::var("J2K_CUDA_BENCH_SUBSAMPLING") + .or_else(|_| std::env::var("J2K_GPU_BENCH_SUBSAMPLING")) + .unwrap_or_else(|_| "420".to_string()); + parse_subsampling(&value) +} + +fn parse_subsampling(value: &str) -> JpegSubsampling { + match value.trim().to_ascii_lowercase().as_str() { + "420" | "4:2:0" | "ybr420" => JpegSubsampling::Ybr420, + "422" | "4:2:2" | "ybr422" => JpegSubsampling::Ybr422, + "444" | "4:4:4" | "ybr444" => JpegSubsampling::Ybr444, + other => panic!("unsupported JPEG bench subsampling {other}; expected 420, 422, or 444"), + } +} + +#[cfg(feature = "cuda-runtime")] +fn bench_chunked_entropy_diagnostic(c: &mut Criterion) { + let (width, height) = generated_dimensions(); + let input = generated_chunked_entropy_jpeg(width, height); + + if let Err(error) = probe_chunked_entropy_diagnostic(&input) { + assert!( + std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_none(), + "J2K_REQUIRE_CUDA_BENCH is set but CUDA JPEG chunked entropy diagnostic is unavailable: {error}" + ); + eprintln!("skipping CUDA JPEG chunked entropy diagnostic bench: {error}"); + return; + } + + let mut group = c.benchmark_group("jpeg_cuda_chunked_entropy"); + group.sample_size(10); + + group.bench_function("cpu_fast_packet_planning", |b| { + b.iter(|| { + let packet = j2k_jpeg::adapter::build_fast420_packet(&input).expect("fast420 packet"); + std::hint::black_box(packet.entropy_checkpoints.len()) + }); + }); + + group.bench_function("cuda_chunked_entropy_sync", |b| { + let mut session = CudaSession::default(); + b.iter(|| { + let report = CudaCodec::diagnose_tile_rgb8_chunked_entropy_with_session( + &input, + j2k_cuda_runtime::CudaJpegChunkedEntropyConfig::default(), + &mut session, + ) + .expect("chunked entropy diagnostic"); + std::hint::black_box(report.synchronized_overflow_count()) + }); + }); + + group.finish(); +} + +#[cfg(not(feature = "cuda-runtime"))] +fn bench_chunked_entropy_diagnostic(_c: &mut Criterion) {} + +#[cfg(feature = "cuda-runtime")] +fn generated_chunked_entropy_jpeg(width: u16, height: u16) -> Vec { + assert_eq!( + bench_subsampling(), + JpegSubsampling::Ybr420, + "jpeg_cuda_chunked_entropy requires generated 4:2:0 JPEG input; unset J2K_CUDA_BENCH_SUBSAMPLING/J2K_GPU_BENCH_SUBSAMPLING or set it to 420" + ); + generated_jpeg(width, height) +} + +#[cfg(feature = "cuda-runtime")] +fn probe_chunked_entropy_diagnostic(input: &[u8]) -> Result<(), j2k_jpeg_cuda::Error> { + let mut session = CudaSession::default(); + CudaCodec::diagnose_tile_rgb8_chunked_entropy_with_session( + input, + j2k_cuda_runtime::CudaJpegChunkedEntropyConfig::default(), + &mut session, + ) + .map(|_| ()) +} + +#[cfg(feature = "cuda-runtime")] +fn bench_batch_decode(c: &mut Criterion) { + let dim = batch_dim(); + let input = generated_jpeg(dim, dim); + let batch_size = batch_size(); + let batch_refs = vec![input.as_slice(); batch_size]; + + let mut group = c.benchmark_group("jpeg_cuda_batch_decode"); + group.sample_size(10); + + group.bench_function(format!("cpu_decode_rgb8_batch{batch_size}"), |b| { + b.iter(|| { + let mut total = 0usize; + for _ in 0..batch_size { + let decoder = CpuDecoder::new(&input).expect("cpu decoder"); + let decoded_rgb = decoder.decode(PixelFormat::Rgb8).expect("cpu decode"); + total = total.saturating_add(decoded_rgb.0.len()); + std::hint::black_box(decoded_rgb); + } + total + }); + }); + + let mut probe_ctx = DecoderContext::::new(); + let mut probe_pool = j2k_jpeg::ScratchPool::new(); + if let Err(error) = CudaCodec::decode_tiles_to_device( + &mut probe_ctx, + &mut probe_pool, + &batch_refs[..1], + PixelFormat::Rgb8, + BackendRequest::Cuda, + ) { + assert!( + std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_none(), + "J2K_REQUIRE_CUDA_BENCH is set but owned CUDA JPEG batch decode is unavailable: {error}" + ); + eprintln!("skipping CUDA adapter batch decode bench: {error}"); + group.finish(); + return; + } + + group.bench_function( + format!("cuda_adapter_rgb8_batch{batch_size}_surfaces"), + |b| { + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_jpeg::ScratchPool::new(); + b.iter(|| { + let outputs = CudaCodec::decode_tiles_to_device( + &mut ctx, + &mut pool, + &batch_refs, + PixelFormat::Rgb8, + BackendRequest::Cuda, + ) + .expect("cuda adapter batch decode"); + std::hint::black_box(outputs) + }); + }, + ); + + group.finish(); +} + +#[cfg(not(feature = "cuda-runtime"))] +fn bench_batch_decode(_c: &mut Criterion) {} + +#[cfg(feature = "cuda-runtime")] +fn batch_size() -> usize { + let Some(value) = std::env::var_os("J2K_GPU_BENCH_BATCH") else { + return DEFAULT_BATCH_SIZE; + }; + let value = value + .to_string_lossy() + .parse::() + .expect("J2K_GPU_BENCH_BATCH must be a usize"); + assert!( + (1..=256).contains(&value), + "J2K_GPU_BENCH_BATCH must be between 1 and 256" + ); + value +} + +#[cfg(feature = "cuda-runtime")] +fn batch_dim() -> u16 { + let Some(value) = std::env::var_os("J2K_GPU_BENCH_BATCH_DIM") else { + return DEFAULT_BATCH_DIM; + }; + let value = value + .to_string_lossy() + .parse::() + .expect("J2K_GPU_BENCH_BATCH_DIM must be a u16"); + assert!( + (128..=4096).contains(&value), + "J2K_GPU_BENCH_BATCH_DIM must be between 128 and 4096" + ); + value +} + +struct CudaProbe { + used_owned_cuda_decode: bool, +} + +fn cuda_probe(input: &[u8]) -> Option { + let mut decoder = CudaDecoder::new(input).expect("cuda decoder"); + let surface = match decoder.decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) { + Ok(surface) => surface, + Err(error) => { + eprintln!("skipping CUDA decode benches: {error}"); + return None; + } + }; + let stats = surface.cuda_surface().expect("cuda surface").stats(); + if std::env::var_os("J2K_REQUIRE_CUDA_JPEG_HARDWARE_DECODE").is_some() + && !stats.used_owned_cuda_decode() + { + panic!( + "J2K_REQUIRE_CUDA_JPEG_HARDWARE_DECODE is set but owned CUDA JPEG decode was not used" + ); + } + Some(CudaProbe { + used_owned_cuda_decode: stats.used_owned_cuda_decode(), + }) +} + +criterion_group!(benches, bench_device_decode); +criterion_main!(benches); diff --git a/crates/signinum-jpeg-cuda/fixtures/jpeg/baseline_420_16x16.jpg b/crates/j2k-jpeg-cuda/fixtures/jpeg/baseline_420_16x16.jpg similarity index 100% rename from crates/signinum-jpeg-cuda/fixtures/jpeg/baseline_420_16x16.jpg rename to crates/j2k-jpeg-cuda/fixtures/jpeg/baseline_420_16x16.jpg diff --git a/crates/signinum-jpeg-metal/fixtures/jpeg/baseline_422_16x8.jpg b/crates/j2k-jpeg-cuda/fixtures/jpeg/baseline_422_16x8.jpg similarity index 100% rename from crates/signinum-jpeg-metal/fixtures/jpeg/baseline_422_16x8.jpg rename to crates/j2k-jpeg-cuda/fixtures/jpeg/baseline_422_16x8.jpg diff --git a/crates/signinum-jpeg-metal/fixtures/jpeg/baseline_444_8x8.jpg b/crates/j2k-jpeg-cuda/fixtures/jpeg/baseline_444_8x8.jpg similarity index 100% rename from crates/signinum-jpeg-metal/fixtures/jpeg/baseline_444_8x8.jpg rename to crates/j2k-jpeg-cuda/fixtures/jpeg/baseline_444_8x8.jpg diff --git a/crates/j2k-jpeg-cuda/src/codec.rs b/crates/j2k-jpeg-cuda/src/codec.rs new file mode 100644 index 00000000..359795fb --- /dev/null +++ b/crates/j2k-jpeg-cuda/src/codec.rs @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_core::{ + submit_ready_device, BackendRequest, Downscale, ImageCodec, PixelFormat, ReadySubmission, Rect, + TileBatchDecodeDevice, TileBatchDecodeManyDevice, TileBatchDecodeSubmit, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_cuda_runtime::CudaDeviceBuffer; +use j2k_jpeg::{ + decode_tile_into_in_context, decode_tile_region_into_in_context, + decode_tile_region_scaled_into_in_context, decode_tile_scaled_into_in_context, + Decoder as CpuDecoder, DecoderContext as CpuDecoderContext, JpegDecodeOp, JpegDecodeRequest, + JpegResolvedDecode, JpegResolvedDecodePath, ScratchPool as CpuScratchPool, + Warning as CpuWarning, +}; + +use crate::owned_decode::decode_owned_cuda_rgb8; +#[cfg(feature = "cuda-runtime")] +use crate::owned_decode::decode_owned_cuda_rgb8_into; +use crate::runtime::{validate_surface_request, wrap_surface}; +use crate::{CudaSession, Error, Surface}; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +/// JPEG codec marker used by J2K's generic CUDA decode traits. +pub struct Codec; + +impl ImageCodec for Codec { + type Error = Error; + type Warning = CpuWarning; + type Pool = CpuScratchPool; +} + +fn rejected_decode_path_error(backend: BackendRequest, reason: &'static str) -> Error { + match backend { + BackendRequest::Cuda => Error::UnsupportedCudaRequest { reason }, + other => Error::UnsupportedBackend { request: other }, + } +} + +impl Codec { + #[cfg(feature = "cuda-runtime")] + /// Run experimental chunked JPEG entropy self-sync diagnostics for a 4:2:0 RGB8 tile. + /// + /// This does not decode pixels and does not affect production CUDA routing. + pub fn diagnose_tile_rgb8_chunked_entropy_with_session( + input: &[u8], + config: j2k_cuda_runtime::CudaJpegChunkedEntropyConfig, + session: &mut CudaSession, + ) -> Result { + crate::owned_decode::diagnose_owned_cuda_420_entropy(input, config, session) + } + + #[cfg(feature = "cuda-runtime")] + /// Decode one full JPEG tile to caller-owned CUDA RGB8 memory using a session. + /// + /// This is a strict J2K-owned CUDA-kernel path and currently supports + /// full-tile RGB8 fast 4:2:0, 4:2:2, and 4:4:4 YCbCr JPEG inputs. + pub fn decode_tile_rgb8_into_cuda_buffer_with_session( + input: &[u8], + output: &CudaDeviceBuffer, + pitch_bytes: usize, + session: &mut CudaSession, + ) -> Result { + let dimensions = CpuDecoder::inspect(input)?.dimensions; + decode_owned_cuda_rgb8_into(input, dimensions, session, output, pitch_bytes) + } + + /// Decode many JPEG tiles to J2K surfaces using a caller-owned CUDA session. + pub fn decode_tiles_to_device_with_session( + inputs: &[&[u8]], + fmt: PixelFormat, + backend: BackendRequest, + session: &mut CudaSession, + ) -> Result, Error> { + let mut ctx = j2k_core::DecoderContext::::new(); + let mut pool = CpuScratchPool::new(); + Self::decode_tiles_to_device_with_session_in_context( + &mut ctx, &mut pool, inputs, fmt, backend, session, + ) + } + + fn decode_tiles_to_device_with_session_in_context( + ctx: &mut j2k_core::DecoderContext, + pool: &mut CpuScratchPool, + inputs: &[&[u8]], + fmt: PixelFormat, + backend: BackendRequest, + session: &mut CudaSession, + ) -> Result, Error> { + validate_surface_request(backend)?; + if inputs.is_empty() { + return Ok(Vec::new()); + } + + inputs + .iter() + .map(|input| Self::decode_tile_to_surface_impl(ctx, session, pool, input, fmt, backend)) + .collect() + } + + fn decode_tile_to_surface_impl( + ctx: &mut j2k_core::DecoderContext, + session: &mut CudaSession, + pool: &mut CpuScratchPool, + input: &[u8], + fmt: PixelFormat, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + let resolved = JpegResolvedDecode::inspect( + input, + JpegDecodeRequest { + backend, + fmt, + op: JpegDecodeOp::Full, + }, + )?; + if resolved.path == JpegResolvedDecodePath::OwnedCudaRgb8 { + return decode_owned_cuda_rgb8(input, resolved.capabilities.info.dimensions, session); + } + if let JpegResolvedDecodePath::Rejected { backend, reason } = resolved.path { + return Err(rejected_decode_path_error(backend, reason)); + } + let dims = (resolved.output_rect.w, resolved.output_rect.h); + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + decode_tile_into_in_context(input, ctx.codec_mut(), pool, &mut out, stride, fmt)?; + wrap_surface(out, dims, fmt, backend, session) + } + + fn decode_tile_region_to_surface_impl( + ctx: &mut j2k_core::DecoderContext, + session: &mut CudaSession, + pool: &mut CpuScratchPool, + input: &[u8], + fmt: PixelFormat, + roi: Rect, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + if backend == BackendRequest::Cuda { + return Err(Error::UnsupportedCudaRequest { + reason: "J2K CUDA JPEG owned decode does not support region output", + }); + } + let dims = (roi.w, roi.h); + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + decode_tile_region_into_in_context( + input, + ctx.codec_mut(), + pool, + &mut out, + stride, + fmt, + roi.into(), + )?; + wrap_surface(out, dims, fmt, backend, session) + } + + fn decode_tile_scaled_to_surface_impl( + ctx: &mut j2k_core::DecoderContext, + session: &mut CudaSession, + pool: &mut CpuScratchPool, + input: &[u8], + fmt: PixelFormat, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + if backend == BackendRequest::Cuda { + return Err(Error::UnsupportedCudaRequest { + reason: "J2K CUDA JPEG owned decode does not support scaled output", + }); + } + let source_dims = CpuDecoder::inspect(input)?.dimensions; + let dims = ( + source_dims.0.div_ceil(scale.denominator()), + source_dims.1.div_ceil(scale.denominator()), + ); + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + decode_tile_scaled_into_in_context( + input, + ctx.codec_mut(), + pool, + &mut out, + stride, + fmt, + scale, + )?; + wrap_surface(out, dims, fmt, backend, session) + } + + #[allow(clippy::too_many_arguments)] + fn decode_tile_region_scaled_to_surface_impl( + ctx: &mut j2k_core::DecoderContext, + session: &mut CudaSession, + pool: &mut CpuScratchPool, + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + if backend == BackendRequest::Cuda { + return Err(Error::UnsupportedCudaRequest { + reason: "J2K CUDA JPEG owned decode does not support scaled region output", + }); + } + let dims = { + let scaled = roi.scaled_covering(scale); + (scaled.w, scaled.h) + }; + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + decode_tile_region_scaled_into_in_context( + input, + ctx.codec_mut(), + pool, + &mut out, + stride, + fmt, + roi.into(), + scale, + )?; + wrap_surface(out, dims, fmt, backend, session) + } +} + +impl TileBatchDecodeSubmit for Codec { + type Context = CpuDecoderContext; + type Session = CudaSession; + type DeviceSurface = Surface; + type SubmittedSurface = ReadySubmission; + + fn submit_tile_to_device( + ctx: &mut j2k_core::DecoderContext, + session: &mut Self::Session, + pool: &mut Self::Pool, + input: &[u8], + fmt: PixelFormat, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + Ok(submit_ready_device(session, |session| { + Self::decode_tile_to_surface_impl(ctx, session, pool, input, fmt, backend) + })) + } + + fn submit_tile_region_to_device( + ctx: &mut j2k_core::DecoderContext, + session: &mut Self::Session, + pool: &mut Self::Pool, + input: &[u8], + fmt: PixelFormat, + roi: Rect, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + Ok(submit_ready_device(session, |session| { + Self::decode_tile_region_to_surface_impl(ctx, session, pool, input, fmt, roi, backend) + })) + } + + fn submit_tile_scaled_to_device( + ctx: &mut j2k_core::DecoderContext, + session: &mut Self::Session, + pool: &mut Self::Pool, + input: &[u8], + fmt: PixelFormat, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + Ok(submit_ready_device(session, |session| { + Self::decode_tile_scaled_to_surface_impl(ctx, session, pool, input, fmt, scale, backend) + })) + } + + fn submit_tile_region_scaled_to_device( + ctx: &mut j2k_core::DecoderContext, + session: &mut Self::Session, + pool: &mut Self::Pool, + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + validate_surface_request(backend)?; + Ok(submit_ready_device(session, |session| { + Self::decode_tile_region_scaled_to_surface_impl( + ctx, session, pool, input, fmt, roi, scale, backend, + ) + })) + } +} + +impl TileBatchDecodeDevice for Codec { + type Context = CpuDecoderContext; + type DeviceSurface = Surface; +} + +impl TileBatchDecodeManyDevice for Codec { + type Context = CpuDecoderContext; + type DeviceSurface = Surface; + + fn decode_tiles_to_device( + ctx: &mut j2k_core::DecoderContext, + pool: &mut Self::Pool, + inputs: &[&[u8]], + fmt: PixelFormat, + backend: BackendRequest, + ) -> Result, Self::Error> { + let mut session = CudaSession::default(); + Self::decode_tiles_to_device_with_session_in_context( + ctx, + pool, + inputs, + fmt, + backend, + &mut session, + ) + } +} diff --git a/crates/signinum-jpeg-cuda/src/decoder.rs b/crates/j2k-jpeg-cuda/src/decoder.rs similarity index 60% rename from crates/signinum-jpeg-cuda/src/decoder.rs rename to crates/j2k-jpeg-cuda/src/decoder.rs index c4360b0c..92054ada 100644 --- a/crates/signinum-jpeg-cuda/src/decoder.rs +++ b/crates/j2k-jpeg-cuda/src/decoder.rs @@ -1,31 +1,28 @@ // SPDX-License-Identifier: Apache-2.0 -#[cfg(feature = "cuda-runtime")] -use signinum_core::BackendKind; -use signinum_core::{ - BackendRequest, DecodeOutcome, Downscale, ImageCodec, ImageDecode, ImageDecodeDevice, - ImageDecodeSubmit, PixelFormat, ReadySubmission, Rect, +use j2k_core::{ + submit_ready_device, BackendRequest, DecodeOutcome, Downscale, ImageCodec, ImageDecode, + ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, ReadySubmission, Rect, }; #[cfg(feature = "cuda-runtime")] -use signinum_cuda_runtime::CudaError; -#[cfg(feature = "cuda-runtime")] -use signinum_jpeg::adapter::decoder_bytes; -use signinum_jpeg::{ +use j2k_jpeg::adapter::decoder_bytes; +use j2k_jpeg::{ Decoder as CpuDecoder, JpegView, ScratchPool as CpuScratchPool, Warning as CpuWarning, }; #[cfg(feature = "cuda-runtime")] -use crate::runtime::cuda_error; +use crate::owned_decode::decode_owned_cuda_rgb8; +use crate::owned_decode::unsupported_owned_cuda_output_format; use crate::runtime::{validate_surface_request, wrap_surface}; -#[cfg(feature = "cuda-runtime")] -use crate::surface::{CudaSurfaceStats, Storage}; -use crate::{profile, CudaSession, Error, Surface}; +use crate::{CudaSession, Error, Surface}; +/// JPEG decoder that can return host or CUDA-resident surfaces. pub struct Decoder<'a> { inner: CpuDecoder<'a>, } impl<'a> Decoder<'a> { + /// Parse a JPEG byte slice into a CUDA-capable decoder wrapper. pub fn new(input: &'a [u8]) -> Result { Ok(Self { inner: CpuDecoder::new(input)?, @@ -39,14 +36,13 @@ impl<'a> Decoder<'a> { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; - if profile::gpu_route_profile_enabled() { + if j2k_profile::gpu_route_profile_enabled() { let request_s = format!("{backend:?}"); let fmt_s = format!("{fmt:?}"); let width_s = self.inner.info().dimensions.0.to_string(); let height_s = self.inner.info().dimensions.1.to_string(); - profile::emit_gpu_route_profile( + j2k_profile::emit_gpu_route_profile( "jpeg", - "gpu_route", "cuda", &[ ("op", "full"), @@ -58,18 +54,18 @@ impl<'a> Decoder<'a> { ], ); } - if backend == BackendRequest::Cuda && fmt == PixelFormat::Rgb8 { - if let Some(surface) = self.try_decode_cuda_rgb8(session)? { - return Ok(surface); + if backend == BackendRequest::Cuda { + if fmt == PixelFormat::Rgb8 { + return self.decode_cuda_rgb8(session); } + return Err(unsupported_owned_cuda_output_format()); } let (bytes, _outcome) = self.inner.decode(fmt)?; - if profile::gpu_route_profile_enabled() { + if j2k_profile::gpu_route_profile_enabled() { let request_s = format!("{backend:?}"); let fmt_s = format!("{fmt:?}"); - profile::emit_gpu_route_profile( + j2k_profile::emit_gpu_route_profile( "jpeg", - "gpu_route", "cuda", &[ ("op", "full"), @@ -83,100 +79,45 @@ impl<'a> Decoder<'a> { } #[cfg(feature = "cuda-runtime")] - fn try_decode_cuda_rgb8( - &mut self, - session: &mut CudaSession, - ) -> Result, Error> { + fn decode_cuda_rgb8(&mut self, session: &mut CudaSession) -> Result { let dimensions = self.inner.info().dimensions; - let bytes = decoder_bytes(&self.inner); - let context = session.cuda_context()?; - match context.decode_jpeg_rgb8_with_nvjpeg(bytes, dimensions) { - Ok(output) => { - let pitch_bytes = dimensions.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let (buffer, stats) = output.into_parts(); - if profile::gpu_route_profile_enabled() { - let width_s = dimensions.0.to_string(); - let height_s = dimensions.1.to_string(); - let kernel_dispatches_s = stats.kernel_dispatches().to_string(); - let decode_dispatches_s = stats.decode_kernel_dispatches().to_string(); - let hardware_decode_s = stats.used_hardware_decode().to_string(); - profile::emit_gpu_route_profile( - "jpeg", - "gpu_route", - "cuda", - &[ - ("op", "full"), - ("request", "Cuda"), - ("fmt", "Rgb8"), - ("width", width_s.as_str()), - ("height", height_s.as_str()), - ("decision", "nvjpeg"), - ("kernel_dispatches", kernel_dispatches_s.as_str()), - ("decode_kernel_dispatches", decode_dispatches_s.as_str()), - ("hardware_decode", hardware_decode_s.as_str()), - ], - ); - } - Ok(Some(Surface { - backend: BackendKind::Cuda, - dimensions, - fmt: PixelFormat::Rgb8, - pitch_bytes, - stats: CudaSurfaceStats { - kernel_dispatches: stats.kernel_dispatches(), - copy_kernel_dispatches: stats.copy_kernel_dispatches(), - decode_kernel_dispatches: stats.decode_kernel_dispatches(), - hardware_decode: stats.used_hardware_decode(), - }, - storage: Storage::Cuda(buffer), - })) - } - Err( - CudaError::NvjpegUnavailable { .. } - | CudaError::Nvjpeg { .. } - | CudaError::NvjpegDimensions { .. }, - ) => { - if profile::gpu_route_profile_enabled() { - profile::emit_gpu_route_profile( - "jpeg", - "gpu_route", - "cuda", - &[ - ("op", "full"), - ("request", "Cuda"), - ("fmt", "Rgb8"), - ("decision", "nvjpeg_fallback"), - ("reason", "nvjpeg_unavailable_or_rejected"), - ], - ); - } - Ok(None) - } - Err(error) => Err(cuda_error(error)), + let surface = decode_owned_cuda_rgb8(decoder_bytes(&self.inner), dimensions, session)?; + if j2k_profile::gpu_route_profile_enabled() { + let width_s = dimensions.0.to_string(); + let height_s = dimensions.1.to_string(); + j2k_profile::emit_gpu_route_profile( + "jpeg", + "cuda", + &[ + ("op", "full"), + ("request", "Cuda"), + ("fmt", "Rgb8"), + ("width", width_s.as_str()), + ("height", height_s.as_str()), + ("decision", "owned_cuda"), + ], + ); } + Ok(surface) } #[cfg(not(feature = "cuda-runtime"))] #[allow(clippy::unnecessary_wraps, clippy::unused_self)] - fn try_decode_cuda_rgb8( - &mut self, - _session: &mut CudaSession, - ) -> Result, Error> { - if profile::gpu_route_profile_enabled() { - profile::emit_gpu_route_profile( + fn decode_cuda_rgb8(&mut self, _session: &mut CudaSession) -> Result { + if j2k_profile::gpu_route_profile_enabled() { + j2k_profile::emit_gpu_route_profile( "jpeg", - "gpu_route", "cuda", &[ ("op", "full"), ("request", "Cuda"), ("fmt", "Rgb8"), - ("decision", "nvjpeg_fallback"), + ("decision", "owned_cuda_unavailable"), ("reason", "cuda_runtime_feature_disabled"), ], ); } - Ok(None) + Err(Error::CudaUnavailable) } fn decode_region_to_surface_impl( @@ -187,6 +128,11 @@ impl<'a> Decoder<'a> { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; + if backend == BackendRequest::Cuda { + return Err(Error::UnsupportedCudaRequest { + reason: "J2K CUDA JPEG owned decode does not support region output", + }); + } let (bytes, outcome) = self.inner.decode_region(fmt, roi.into())?; wrap_surface( bytes, @@ -205,6 +151,11 @@ impl<'a> Decoder<'a> { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; + if backend == BackendRequest::Cuda { + return Err(Error::UnsupportedCudaRequest { + reason: "J2K CUDA JPEG owned decode does not support scaled output", + }); + } let (bytes, outcome) = self.inner.decode_scaled(fmt, scale)?; wrap_surface( bytes, @@ -224,6 +175,11 @@ impl<'a> Decoder<'a> { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; + if backend == BackendRequest::Cuda { + return Err(Error::UnsupportedCudaRequest { + reason: "J2K CUDA JPEG owned decode does not support scaled region output", + }); + } let (bytes, outcome) = self.inner.decode_region_scaled(fmt, roi.into(), scale)?; wrap_surface( bytes, @@ -244,7 +200,7 @@ impl ImageCodec for Decoder<'_> { impl<'a> ImageDecode<'a> for Decoder<'a> { type View = JpegView<'a>; - fn inspect(input: &'a [u8]) -> Result { + fn inspect(input: &'a [u8]) -> Result { Ok(CpuDecoder::inspect(input)?.to_core_info()) } @@ -340,10 +296,9 @@ impl<'a> ImageDecodeSubmit<'a> for Decoder<'a> { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - self.decode_to_surface_impl(session, fmt, backend), - )) + Ok(submit_ready_device(session, |session| { + self.decode_to_surface_impl(session, fmt, backend) + })) } fn submit_region_to_device( @@ -354,10 +309,9 @@ impl<'a> ImageDecodeSubmit<'a> for Decoder<'a> { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - self.decode_region_to_surface_impl(session, fmt, roi, backend), - )) + Ok(submit_ready_device(session, |session| { + self.decode_region_to_surface_impl(session, fmt, roi, backend) + })) } fn submit_scaled_to_device( @@ -368,10 +322,9 @@ impl<'a> ImageDecodeSubmit<'a> for Decoder<'a> { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - self.decode_scaled_to_surface_impl(session, fmt, scale, backend), - )) + Ok(submit_ready_device(session, |session| { + self.decode_scaled_to_surface_impl(session, fmt, scale, backend) + })) } fn submit_region_scaled_to_device( @@ -383,9 +336,8 @@ impl<'a> ImageDecodeSubmit<'a> for Decoder<'a> { backend: BackendRequest, ) -> Result { validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - self.decode_region_scaled_to_surface_impl(session, fmt, roi, scale, backend), - )) + Ok(submit_ready_device(session, |session| { + self.decode_region_scaled_to_surface_impl(session, fmt, roi, scale, backend) + })) } } diff --git a/crates/j2k-jpeg-cuda/src/error.rs b/crates/j2k-jpeg-cuda/src/error.rs new file mode 100644 index 00000000..e6416274 --- /dev/null +++ b/crates/j2k-jpeg-cuda/src/error.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_core::{BackendRequest, BufferError, CodecError}; +use j2k_jpeg::JpegError; + +#[derive(Debug, thiserror::Error)] +/// Errors returned by the CUDA JPEG adapter. +pub enum Error { + /// Error returned by the CPU JPEG parser or fallback decoder. + #[error(transparent)] + Decode(#[from] JpegError), + /// Output buffer validation failed. + #[error(transparent)] + Buffer(#[from] BufferError), + /// The requested backend is unsupported by this crate. + #[error("backend request {request:?} is not supported by j2k-jpeg-cuda")] + UnsupportedBackend { + /// Backend requested by the caller. + request: BackendRequest, + }, + /// CUDA request is unsupported by the strict CUDA adapter contract. + #[error("unsupported CUDA request: {reason}")] + UnsupportedCudaRequest { + /// Human-readable rejection reason. + reason: &'static str, + }, + /// CUDA is not available on the current host. + #[error("CUDA is unavailable on this host")] + CudaUnavailable, + #[cfg(feature = "cuda-runtime")] + /// CUDA runtime operation failed. + #[error("CUDA runtime error: {message}")] + CudaRuntime { + /// Runtime error message. + message: String, + }, +} + +impl CodecError for Error { + fn is_truncated(&self) -> bool { + matches!(self, Self::Decode(inner) if inner.is_truncated()) + } + + fn is_not_implemented(&self) -> bool { + matches!(self, Self::Decode(inner) if inner.is_not_implemented()) + } + + fn is_unsupported(&self) -> bool { + matches!( + self, + Self::UnsupportedBackend { .. } + | Self::UnsupportedCudaRequest { .. } + | Self::CudaUnavailable + ) || matches!(self, Self::Decode(inner) if inner.is_unsupported()) + } + + fn is_buffer_error(&self) -> bool { + matches!(self, Self::Buffer(_)) + || matches!(self, Self::Decode(inner) if inner.is_buffer_error()) + } +} diff --git a/crates/j2k-jpeg-cuda/src/lib.rs b/crates/j2k-jpeg-cuda/src/lib.rs new file mode 100644 index 00000000..81bf5efa --- /dev/null +++ b/crates/j2k-jpeg-cuda/src/lib.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! CUDA-facing device-output adapter for `j2k-jpeg`. +//! +//! This crate intentionally exposes the same backend-selection surface as the +//! Metal adapter. CPU requests return host-backed surfaces. Scalar auto +//! requests stay on CPU. Explicit CUDA requests use J2K-owned CUDA JPEG +//! decode kernels when the runtime can handle the image, and otherwise return +//! a clear unsupported or unavailable error. + +#![warn(unreachable_pub)] + +mod codec; +mod decoder; +mod error; +mod owned_decode; +mod runtime; +mod session; +mod surface; + +pub use codec::Codec; +pub use decoder::Decoder; +pub use error::Error; +pub use j2k_jpeg::{DecoderContext, ScratchPool}; +pub use session::CudaSession; +pub use surface::{CudaJpegDecodePath, CudaSurface, CudaSurfaceStats, Surface}; diff --git a/crates/j2k-jpeg-cuda/src/owned_decode.rs b/crates/j2k-jpeg-cuda/src/owned_decode.rs new file mode 100644 index 00000000..3362a94b --- /dev/null +++ b/crates/j2k-jpeg-cuda/src/owned_decode.rs @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(feature = "cuda-runtime")] +use std::sync::Arc; + +#[cfg(feature = "cuda-runtime")] +use j2k_core::{BackendKind, PixelFormat}; + +use crate::{CudaSession, Error, Surface}; + +#[cfg(feature = "cuda-runtime")] +use j2k_cuda_runtime::{ + CudaDeviceBuffer, CudaError, CudaJpegEntropyCheckpoint, CudaJpegHuffmanTable, + CudaJpegRgb8DecodePlan, CudaJpegRgb8Sampling, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_jpeg::adapter::{ + JpegEntropyCheckpointV1, JpegFast420PacketV1, JpegFast422PacketV1, JpegFast444PacketV1, + JpegHuffmanTable, +}; +#[cfg(feature = "cuda-runtime")] +use j2k_jpeg::{JpegCapabilityReport, JpegCapabilityRequest, JpegDecodeOp}; + +#[cfg(feature = "cuda-runtime")] +use crate::surface::{CudaJpegDecodePath, CudaSurfaceStats, Storage}; + +pub(crate) fn unsupported_owned_cuda_output_format() -> Error { + Error::UnsupportedCudaRequest { + reason: "J2K CUDA JPEG owned decode currently supports full-frame RGB8 output only", + } +} + +#[cfg(feature = "cuda-runtime")] +const UNSUPPORTED_CHUNKED_ENTROPY_DIAGNOSTIC_INPUT: &str = + "J2K CUDA JPEG chunked entropy diagnostic currently supports baseline 8-bit YCbCr 4:2:0 RGB8 inputs only"; + +#[cfg(feature = "cuda-runtime")] +const INVALID_CHUNKED_ENTROPY_DIAGNOSTIC_ARGUMENT: &str = + "J2K CUDA JPEG chunked entropy diagnostic config or input is invalid"; + +#[cfg(feature = "cuda-runtime")] +pub(crate) fn decode_owned_cuda_rgb8( + bytes: &[u8], + dimensions: (u32, u32), + session: &mut CudaSession, +) -> Result { + let packet = resolve_owned_rgb8_packet(bytes, session)?; + if packet.dimensions() != dimensions { + return Err(Error::UnsupportedCudaRequest { + reason: "J2K CUDA JPEG packet dimensions do not match decoder metadata", + }); + } + + let checkpoints = cuda_entropy_checkpoints(packet.entropy_checkpoints()); + let plan = match &packet { + OwnedFastRgb8Packet::Fast420(packet) => cuda_decode_plan( + CudaJpegRgb8Sampling::Fast420, + packet.as_ref(), + dimensions, + &checkpoints, + )?, + OwnedFastRgb8Packet::Fast422(packet) => cuda_decode_plan( + CudaJpegRgb8Sampling::Fast422, + packet.as_ref(), + dimensions, + &checkpoints, + )?, + OwnedFastRgb8Packet::Fast444(packet) => cuda_decode_plan( + CudaJpegRgb8Sampling::Fast444, + packet.as_ref(), + dimensions, + &checkpoints, + )?, + }; + let context = session.cuda_context()?; + let output = context + .decode_jpeg_rgb8_owned(&plan) + .map_err(cuda_owned_decode_error)?; + let (buffer, stats) = output.into_parts(); + Ok(Surface { + backend: BackendKind::Cuda, + dimensions, + fmt: PixelFormat::Rgb8, + pitch_bytes: dimensions.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(), + stats: CudaSurfaceStats { + kernel_dispatches: stats.kernel_dispatches(), + copy_kernel_dispatches: stats.copy_kernel_dispatches(), + decode_kernel_dispatches: stats.decode_kernel_dispatches(), + hardware_decode: false, + decode_path: CudaJpegDecodePath::OwnedCuda, + }, + storage: Storage::Cuda(buffer), + }) +} + +#[cfg(feature = "cuda-runtime")] +pub(crate) fn decode_owned_cuda_rgb8_into( + bytes: &[u8], + dimensions: (u32, u32), + session: &mut CudaSession, + output: &CudaDeviceBuffer, + pitch_bytes: usize, +) -> Result { + let packet = resolve_owned_rgb8_packet(bytes, session)?; + if packet.dimensions() != dimensions { + return Err(Error::UnsupportedCudaRequest { + reason: "J2K CUDA JPEG packet dimensions do not match decoder metadata", + }); + } + let checkpoints = cuda_entropy_checkpoints(packet.entropy_checkpoints()); + let plan = match &packet { + OwnedFastRgb8Packet::Fast420(packet) => cuda_decode_plan( + CudaJpegRgb8Sampling::Fast420, + packet.as_ref(), + dimensions, + &checkpoints, + )?, + OwnedFastRgb8Packet::Fast422(packet) => cuda_decode_plan( + CudaJpegRgb8Sampling::Fast422, + packet.as_ref(), + dimensions, + &checkpoints, + )?, + OwnedFastRgb8Packet::Fast444(packet) => cuda_decode_plan( + CudaJpegRgb8Sampling::Fast444, + packet.as_ref(), + dimensions, + &checkpoints, + )?, + }; + let context = session.cuda_context()?; + let stats = context + .decode_jpeg_rgb8_owned_into(&plan, output, pitch_bytes) + .map_err(cuda_owned_decode_error)?; + Ok(CudaSurfaceStats { + kernel_dispatches: stats.kernel_dispatches(), + copy_kernel_dispatches: stats.copy_kernel_dispatches(), + decode_kernel_dispatches: stats.decode_kernel_dispatches(), + hardware_decode: false, + decode_path: CudaJpegDecodePath::OwnedCuda, + }) +} + +#[cfg(feature = "cuda-runtime")] +pub(crate) fn diagnose_owned_cuda_420_entropy( + bytes: &[u8], + config: j2k_cuda_runtime::CudaJpegChunkedEntropyConfig, + session: &mut CudaSession, +) -> Result { + validate_chunked_entropy_diagnostic_input(bytes)?; + config + .validate() + .map_err(cuda_chunked_entropy_diagnostic_error)?; + let packet = session.resolve_owned_fast420_packet(bytes)?; + let plan = j2k_cuda_runtime::CudaJpegChunkedEntropyPlan { + config, + entropy_bytes: &packet.entropy_bytes, + y_dc_table: cuda_huffman_table(&packet.y_dc_table)?, + y_ac_table: cuda_huffman_table(&packet.y_ac_table)?, + cb_dc_table: cuda_huffman_table(&packet.cb_dc_table)?, + cb_ac_table: cuda_huffman_table(&packet.cb_ac_table)?, + cr_dc_table: cuda_huffman_table(&packet.cr_dc_table)?, + cr_ac_table: cuda_huffman_table(&packet.cr_ac_table)?, + }; + session + .cuda_context()? + .diagnose_jpeg_420_entropy_self_sync(&plan) + .map_err(cuda_chunked_entropy_diagnostic_error) +} + +#[cfg(not(feature = "cuda-runtime"))] +pub(crate) fn decode_owned_cuda_rgb8( + _bytes: &[u8], + _dimensions: (u32, u32), + _session: &mut CudaSession, +) -> Result { + Err(Error::CudaUnavailable) +} + +#[cfg(feature = "cuda-runtime")] +enum OwnedFastRgb8Packet { + Fast420(Arc), + Fast422(Arc), + Fast444(Arc), +} + +#[cfg(feature = "cuda-runtime")] +impl OwnedFastRgb8Packet { + fn dimensions(&self) -> (u32, u32) { + match self { + Self::Fast420(packet) => packet.dimensions, + Self::Fast422(packet) => packet.dimensions, + Self::Fast444(packet) => packet.dimensions, + } + } + + fn entropy_checkpoints(&self) -> &[JpegEntropyCheckpointV1] { + match self { + Self::Fast420(packet) => &packet.entropy_checkpoints, + Self::Fast422(packet) => &packet.entropy_checkpoints, + Self::Fast444(packet) => &packet.entropy_checkpoints, + } + } +} + +#[cfg(feature = "cuda-runtime")] +trait FastRgb8Packet { + fn mcus_per_row(&self) -> u32; + fn mcu_rows(&self) -> u32; + fn entropy_bytes(&self) -> &[u8]; + fn y_quant(&self) -> [u16; 64]; + fn cb_quant(&self) -> [u16; 64]; + fn cr_quant(&self) -> [u16; 64]; + fn y_dc_table(&self) -> &JpegHuffmanTable; + fn y_ac_table(&self) -> &JpegHuffmanTable; + fn cb_dc_table(&self) -> &JpegHuffmanTable; + fn cb_ac_table(&self) -> &JpegHuffmanTable; + fn cr_dc_table(&self) -> &JpegHuffmanTable; + fn cr_ac_table(&self) -> &JpegHuffmanTable; +} + +#[cfg(feature = "cuda-runtime")] +macro_rules! impl_fast_rgb8_packet { + ($packet:ty) => { + impl FastRgb8Packet for $packet { + fn mcus_per_row(&self) -> u32 { + self.mcus_per_row + } + + fn mcu_rows(&self) -> u32 { + self.mcu_rows + } + + fn entropy_bytes(&self) -> &[u8] { + &self.entropy_bytes + } + + fn y_quant(&self) -> [u16; 64] { + self.y_quant + } + + fn cb_quant(&self) -> [u16; 64] { + self.cb_quant + } + + fn cr_quant(&self) -> [u16; 64] { + self.cr_quant + } + + fn y_dc_table(&self) -> &JpegHuffmanTable { + &self.y_dc_table + } + + fn y_ac_table(&self) -> &JpegHuffmanTable { + &self.y_ac_table + } + + fn cb_dc_table(&self) -> &JpegHuffmanTable { + &self.cb_dc_table + } + + fn cb_ac_table(&self) -> &JpegHuffmanTable { + &self.cb_ac_table + } + + fn cr_dc_table(&self) -> &JpegHuffmanTable { + &self.cr_dc_table + } + + fn cr_ac_table(&self) -> &JpegHuffmanTable { + &self.cr_ac_table + } + } + }; +} + +#[cfg(feature = "cuda-runtime")] +impl_fast_rgb8_packet!(JpegFast420PacketV1); +#[cfg(feature = "cuda-runtime")] +impl_fast_rgb8_packet!(JpegFast422PacketV1); +#[cfg(feature = "cuda-runtime")] +impl_fast_rgb8_packet!(JpegFast444PacketV1); + +#[cfg(feature = "cuda-runtime")] +fn resolve_owned_rgb8_packet( + bytes: &[u8], + session: &mut CudaSession, +) -> Result { + let report = JpegCapabilityReport::inspect( + bytes, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgb8, + }, + )?; + if !report.owned_cuda.eligible { + return Err(Error::UnsupportedCudaRequest { + reason: report.owned_cuda.reason.unwrap_or( + "J2K CUDA JPEG decode currently supports baseline 8-bit YCbCr 4:2:0, 4:2:2, or 4:4:4 RGB8 output", + ), + }); + } + if report.device.matches_fast_420 { + return session + .resolve_owned_fast420_packet(bytes) + .map(OwnedFastRgb8Packet::Fast420); + } + if report.device.matches_fast_422 { + return session + .resolve_owned_fast422_packet(bytes) + .map(OwnedFastRgb8Packet::Fast422); + } + if report.device.matches_fast_444 { + return session + .resolve_owned_fast444_packet(bytes) + .map(OwnedFastRgb8Packet::Fast444); + } + Err(Error::UnsupportedCudaRequest { + reason: "J2K CUDA JPEG decode currently supports baseline 8-bit YCbCr 4:2:0, 4:2:2, or 4:4:4 RGB8 output", + }) +} + +#[cfg(feature = "cuda-runtime")] +fn validate_chunked_entropy_diagnostic_input(bytes: &[u8]) -> Result<(), Error> { + let report = JpegCapabilityReport::inspect( + bytes, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgb8, + }, + )?; + if report.owned_cuda.eligible && report.device.matches_fast_420 { + return Ok(()); + } + Err(Error::UnsupportedCudaRequest { + reason: UNSUPPORTED_CHUNKED_ENTROPY_DIAGNOSTIC_INPUT, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_decode_plan<'a>( + sampling: CudaJpegRgb8Sampling, + packet: &'a impl FastRgb8Packet, + dimensions: (u32, u32), + checkpoints: &'a [CudaJpegEntropyCheckpoint], +) -> Result, Error> { + Ok(CudaJpegRgb8DecodePlan { + sampling, + dimensions, + mcus_per_row: packet.mcus_per_row(), + mcu_rows: packet.mcu_rows(), + entropy_bytes: packet.entropy_bytes(), + entropy_checkpoints: checkpoints, + y_quant: packet.y_quant(), + cb_quant: packet.cb_quant(), + cr_quant: packet.cr_quant(), + y_dc_table: cuda_huffman_table(packet.y_dc_table())?, + y_ac_table: cuda_huffman_table(packet.y_ac_table())?, + cb_dc_table: cuda_huffman_table(packet.cb_dc_table())?, + cb_ac_table: cuda_huffman_table(packet.cb_ac_table())?, + cr_dc_table: cuda_huffman_table(packet.cr_dc_table())?, + cr_ac_table: cuda_huffman_table(packet.cr_ac_table())?, + }) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_entropy_checkpoints( + checkpoints: &[JpegEntropyCheckpointV1], +) -> Vec { + checkpoints + .iter() + .copied() + .map(cuda_entropy_checkpoint) + .collect() +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_huffman_table(table: &JpegHuffmanTable) -> Result { + CudaJpegHuffmanTable::from_jpeg_bits_values(table.bits, table.values_len, table.values) + .map_err(cuda_owned_decode_error) +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_owned_decode_error(error: CudaError) -> Error { + match error { + CudaError::Unavailable { .. } => Error::CudaUnavailable, + CudaError::InvalidArgument { .. } => Error::UnsupportedCudaRequest { + reason: "J2K CUDA JPEG owned decode cannot handle this image or runtime build", + }, + other => Error::CudaRuntime { + message: other.to_string(), + }, + } +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_chunked_entropy_diagnostic_error(error: CudaError) -> Error { + match error { + CudaError::Unavailable { .. } => Error::CudaUnavailable, + CudaError::InvalidArgument { .. } => Error::UnsupportedCudaRequest { + reason: INVALID_CHUNKED_ENTROPY_DIAGNOSTIC_ARGUMENT, + }, + other => Error::CudaRuntime { + message: other.to_string(), + }, + } +} + +#[cfg(feature = "cuda-runtime")] +fn cuda_entropy_checkpoint(value: JpegEntropyCheckpointV1) -> CudaJpegEntropyCheckpoint { + CudaJpegEntropyCheckpoint { + mcu_index: value.mcu_index, + entropy_pos: value.entropy_pos, + bit_acc: value.bit_acc, + bit_count: value.bit_count, + y_prev_dc: value.y_prev_dc, + cb_prev_dc: value.cb_prev_dc, + cr_prev_dc: value.cr_prev_dc, + reserved: value.reserved, + } +} diff --git a/crates/signinum-jpeg-cuda/src/runtime.rs b/crates/j2k-jpeg-cuda/src/runtime.rs similarity index 82% rename from crates/signinum-jpeg-cuda/src/runtime.rs rename to crates/j2k-jpeg-cuda/src/runtime.rs index 9c45fe99..4774abcd 100644 --- a/crates/signinum-jpeg-cuda/src/runtime.rs +++ b/crates/j2k-jpeg-cuda/src/runtime.rs @@ -1,11 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_core::{BackendKind, BackendRequest, PixelFormat}; +use j2k_core::{BackendKind, BackendRequest, PixelFormat}; #[cfg(feature = "cuda-runtime")] -use signinum_cuda_runtime::CudaError; +use j2k_cuda_runtime::CudaError; use crate::surface::Storage; -use crate::{profile, CudaSession, CudaSurfaceStats, Error, Surface}; +use crate::{CudaSession, CudaSurfaceStats, Error, Surface}; pub(crate) fn wrap_surface( bytes: Vec, @@ -18,14 +18,13 @@ pub(crate) fn wrap_surface( let pitch_bytes = dimensions.0 as usize * fmt.bytes_per_pixel(); match backend { BackendRequest::Cpu | BackendRequest::Auto => { - if profile::gpu_route_profile_enabled() { + if j2k_profile::gpu_route_profile_enabled() { let request_s = format!("{backend:?}"); let fmt_s = format!("{fmt:?}"); let width_s = dimensions.0.to_string(); let height_s = dimensions.1.to_string(); - profile::emit_gpu_route_profile( + j2k_profile::emit_gpu_route_profile( "jpeg", - "gpu_route", "cuda", &[ ("op", "wrap_surface"), @@ -52,10 +51,8 @@ pub(crate) fn wrap_surface( } pub(crate) fn validate_surface_request(backend: BackendRequest) -> Result<(), Error> { - match backend { - BackendRequest::Cpu | BackendRequest::Auto | BackendRequest::Cuda => Ok(()), - BackendRequest::Metal => Err(Error::UnsupportedBackend { request: backend }), - } + j2k_core::validate_cuda_surface_backend_request(backend) + .map_err(|request| Error::UnsupportedBackend { request }) } #[cfg(feature = "cuda-runtime")] @@ -69,15 +66,14 @@ fn wrap_cuda_surface( let context = session.cuda_context()?; let output = context.copy_with_kernel(bytes).map_err(cuda_error)?; let (buffer, stats) = output.into_parts(); - if profile::gpu_route_profile_enabled() { + if j2k_profile::gpu_route_profile_enabled() { let fmt_s = format!("{fmt:?}"); let width_s = dimensions.0.to_string(); let height_s = dimensions.1.to_string(); let kernel_dispatches_s = stats.kernel_dispatches().to_string(); let copy_dispatches_s = stats.copy_kernel_dispatches().to_string(); - profile::emit_gpu_route_profile( + j2k_profile::emit_gpu_route_profile( "jpeg", - "gpu_route", "cuda", &[ ("op", "wrap_surface"), @@ -101,6 +97,7 @@ fn wrap_cuda_surface( copy_kernel_dispatches: stats.copy_kernel_dispatches(), decode_kernel_dispatches: stats.decode_kernel_dispatches(), hardware_decode: stats.used_hardware_decode(), + decode_path: crate::surface::CudaJpegDecodePath::None, }, storage: Storage::Cuda(buffer), }) @@ -114,13 +111,12 @@ fn wrap_cuda_surface( _pitch_bytes: usize, _session: &mut CudaSession, ) -> Result { - if profile::gpu_route_profile_enabled() { + if j2k_profile::gpu_route_profile_enabled() { let fmt_s = format!("{fmt:?}"); let width_s = dimensions.0.to_string(); let height_s = dimensions.1.to_string(); - profile::emit_gpu_route_profile( + j2k_profile::emit_gpu_route_profile( "jpeg", - "gpu_route", "cuda", &[ ("op", "wrap_surface"), @@ -136,11 +132,13 @@ fn wrap_cuda_surface( } #[cfg(feature = "cuda-runtime")] +#[allow(clippy::needless_pass_by_value)] pub(crate) fn cuda_error(error: CudaError) -> Error { - match error { - CudaError::Unavailable { .. } => Error::CudaUnavailable, - other => Error::CudaRuntime { - message: other.to_string(), - }, + if error.is_unavailable() { + Error::CudaUnavailable + } else { + Error::CudaRuntime { + message: error.to_string(), + } } } diff --git a/crates/j2k-jpeg-cuda/src/session.rs b/crates/j2k-jpeg-cuda/src/session.rs new file mode 100644 index 00000000..f91f62b4 --- /dev/null +++ b/crates/j2k-jpeg-cuda/src/session.rs @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(feature = "cuda-runtime")] +use std::collections::VecDeque; +#[cfg(feature = "cuda-runtime")] +use std::sync::Arc; + +#[cfg(feature = "cuda-runtime")] +use j2k_cuda_runtime::{CudaBufferPool, CudaContext, CudaDeviceBuffer}; +#[cfg(feature = "cuda-runtime")] +use j2k_jpeg::adapter::{ + build_fast420_packet, build_fast422_packet, build_fast444_packet, FastPacketError, + JpegFast420PacketV1, JpegFast422PacketV1, JpegFast444PacketV1, +}; + +#[cfg(feature = "cuda-runtime")] +use crate::runtime::cuda_error; +#[cfg(feature = "cuda-runtime")] +use crate::Error; + +#[cfg(feature = "cuda-runtime")] +const OWNED_PACKET_CACHE_SLOTS: usize = 8; +#[cfg(feature = "cuda-runtime")] +const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; +#[cfg(feature = "cuda-runtime")] +const FNV_PRIME: u64 = 0x0000_0100_0000_01B3; + +#[cfg(feature = "cuda-runtime")] +#[derive(Clone)] +struct CachedOwnedFastPacket { + digest: u64, + input: Arc<[u8]>, + packet: Arc, +} + +#[derive(Clone, Default)] +/// Reusable CUDA JPEG decode session. +pub struct CudaSession { + submissions: u64, + #[cfg(feature = "cuda-runtime")] + owned_fast420_packets: VecDeque>, + #[cfg(feature = "cuda-runtime")] + owned_fast422_packets: VecDeque>, + #[cfg(feature = "cuda-runtime")] + owned_fast444_packets: VecDeque>, + #[cfg(feature = "cuda-runtime")] + context: Option, + #[cfg(feature = "cuda-runtime")] + owned_output_pool: Option, +} + +impl CudaSession { + /// Number of decode submissions recorded through this session. + pub fn submissions(&self) -> u64 { + self.submissions + } + + /// Number of cached J2K-owned CUDA fast JPEG packets. + pub fn owned_cuda_packet_cache_len(&self) -> usize { + #[cfg(feature = "cuda-runtime")] + { + self.owned_fast420_packets.len() + + self.owned_fast422_packets.len() + + self.owned_fast444_packets.len() + } + #[cfg(not(feature = "cuda-runtime"))] + { + 0 + } + } + + #[cfg(feature = "cuda-runtime")] + /// Whether a CUDA runtime context has been initialized successfully. + pub fn is_runtime_initialized(&self) -> bool { + self.context.is_some() + } + + #[cfg(feature = "cuda-runtime")] + /// Borrow or allocate a reusable CUDA output buffer for owned JPEG decode. + /// + /// Return buffers to the session with + /// [`recycle_owned_cuda_output_buffer`](Self::recycle_owned_cuda_output_buffer). + /// + /// # Errors + /// Returns a CUDA adapter error if the runtime is unavailable or the pool + /// lock is poisoned. + pub fn take_owned_cuda_output_buffer( + &mut self, + byte_len: usize, + ) -> Result { + let buffer = self + .owned_output_pool()? + .take(byte_len) + .map_err(cuda_error)?; + buffer.into_device_buffer().map_err(cuda_error) + } + + #[cfg(feature = "cuda-runtime")] + /// Return a CUDA output buffer to this session's owned JPEG decode pool. + /// + /// # Errors + /// Returns a CUDA adapter error if the pool lock is poisoned. + pub fn recycle_owned_cuda_output_buffer( + &mut self, + buffer: CudaDeviceBuffer, + ) -> Result<(), Error> { + self.owned_output_pool()? + .recycle(buffer) + .map_err(cuda_error) + } + + #[cfg(feature = "cuda-runtime")] + /// Number of reusable owned CUDA output buffers retained by this session. + pub fn retained_owned_cuda_output_buffers(&self) -> Result { + self.owned_output_pool + .as_ref() + .map_or(Ok(0), |pool| pool.cached_count().map_err(cuda_error)) + } + + #[cfg(feature = "cuda-runtime")] + pub(crate) fn resolve_owned_fast420_packet( + &mut self, + input: &[u8], + ) -> Result, Error> { + resolve_owned_packet(&mut self.owned_fast420_packets, input, build_fast420_packet) + } + + #[cfg(feature = "cuda-runtime")] + pub(crate) fn resolve_owned_fast422_packet( + &mut self, + input: &[u8], + ) -> Result, Error> { + resolve_owned_packet(&mut self.owned_fast422_packets, input, build_fast422_packet) + } + + #[cfg(feature = "cuda-runtime")] + pub(crate) fn resolve_owned_fast444_packet( + &mut self, + input: &[u8], + ) -> Result, Error> { + resolve_owned_packet(&mut self.owned_fast444_packets, input, build_fast444_packet) + } + + #[cfg(feature = "cuda-runtime")] + pub(crate) fn cuda_context(&mut self) -> Result { + if self.context.is_none() { + self.context = Some(CudaContext::system_default().map_err(cuda_error)?); + } + self.context.clone().ok_or(Error::CudaUnavailable) + } + + #[cfg(feature = "cuda-runtime")] + fn owned_output_pool(&mut self) -> Result { + if let Some(pool) = &self.owned_output_pool { + return Ok(pool.clone()); + } + let pool = self.cuda_context()?.buffer_pool(); + self.owned_output_pool = Some(pool.clone()); + Ok(pool) + } +} + +impl j2k_core::DeviceSubmitSession for CudaSession { + fn record_submit(&mut self) { + self.submissions = self.submissions.saturating_add(1); + } +} + +impl j2k_core::AcceleratorSession for CudaSession { + fn backend_kind(&self) -> j2k_core::BackendKind { + j2k_core::BackendKind::Cuda + } + + fn execution_stats(&self) -> j2k_core::ExecutionStats { + j2k_core::ExecutionStats { + submissions: self.submissions, + ..j2k_core::ExecutionStats::default() + } + } +} + +impl std::fmt::Debug for CudaSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut debug = f.debug_struct("CudaSession"); + debug.field("submissions", &self.submissions); + debug.field( + "owned_cuda_packet_cache_len", + &self.owned_cuda_packet_cache_len(), + ); + #[cfg(feature = "cuda-runtime")] + debug.field("runtime_initialized", &self.is_runtime_initialized()); + #[cfg(feature = "cuda-runtime")] + debug.field( + "retained_owned_cuda_output_buffers", + &self.retained_owned_cuda_output_buffers().ok(), + ); + debug.finish_non_exhaustive() + } +} + +#[cfg(feature = "cuda-runtime")] +fn owned_packet_error(error: FastPacketError) -> Error { + match error { + FastPacketError::Decode(error) => Error::Decode(error), + _ => Error::UnsupportedCudaRequest { + reason: "J2K CUDA JPEG decode currently supports baseline 8-bit YCbCr 4:2:0, 4:2:2, or 4:4:4 RGB8 output", + }, + } +} + +#[cfg(feature = "cuda-runtime")] +fn resolve_owned_packet( + cache: &mut VecDeque>, + input: &[u8], + build: fn(&[u8]) -> Result, +) -> Result, Error> { + let digest = digest_bytes(input); + if let Some(entry) = cache + .iter() + .find(|entry| entry.digest == digest && entry.input.as_ref() == input) + { + return Ok(Arc::clone(&entry.packet)); + } + + let packet = build(input).map_err(owned_packet_error)?; + let input = Arc::<[u8]>::from(input); + let packet = Arc::new(packet); + if cache.len() == OWNED_PACKET_CACHE_SLOTS { + cache.pop_front(); + } + cache.push_back(CachedOwnedFastPacket { + digest, + input, + packet: Arc::clone(&packet), + }); + Ok(packet) +} + +#[cfg(feature = "cuda-runtime")] +fn digest_bytes(bytes: &[u8]) -> u64 { + let mut hash = FNV_OFFSET; + for &byte in bytes { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(FNV_PRIME); + } + hash +} diff --git a/crates/signinum-jpeg-cuda/src/surface.rs b/crates/j2k-jpeg-cuda/src/surface.rs similarity index 58% rename from crates/signinum-jpeg-cuda/src/surface.rs rename to crates/j2k-jpeg-cuda/src/surface.rs index 5f6a0c93..c0dd3ad8 100644 --- a/crates/signinum-jpeg-cuda/src/surface.rs +++ b/crates/j2k-jpeg-cuda/src/surface.rs @@ -1,8 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_core::{copy_tight_pixels_to_strided_output, BackendKind, DeviceSurface, PixelFormat}; +use j2k_core::{ + copy_tight_pixels_to_strided_output, BackendKind, DeviceMemoryRange, DeviceSurface, + ExecutionStats, PixelFormat, +}; #[cfg(feature = "cuda-runtime")] -use signinum_cuda_runtime::CudaDeviceBuffer; +use j2k_cuda_runtime::CudaDeviceBuffer; #[cfg(feature = "cuda-runtime")] use crate::runtime::cuda_error; @@ -16,32 +19,59 @@ pub(crate) enum Storage { } #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +/// CUDA JPEG decode path used to produce a surface. +pub enum CudaJpegDecodePath { + /// Surface did not use a CUDA JPEG decode kernel or library path. + #[default] + None, + /// Surface was produced by J2K-owned CUDA JPEG kernels. + OwnedCuda, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +/// Dispatch counters and residency metadata for a CUDA JPEG surface. pub struct CudaSurfaceStats { pub(crate) kernel_dispatches: usize, pub(crate) copy_kernel_dispatches: usize, pub(crate) decode_kernel_dispatches: usize, pub(crate) hardware_decode: bool, + pub(crate) decode_path: CudaJpegDecodePath, } impl CudaSurfaceStats { + /// Total CUDA kernel or library dispatches used to produce the surface. pub fn kernel_dispatches(self) -> usize { self.kernel_dispatches } + /// Number of copy-kernel dispatches used for the surface. pub fn copy_kernel_dispatches(self) -> usize { self.copy_kernel_dispatches } + /// Number of decode kernel or library dispatches used for the surface. pub fn decode_kernel_dispatches(self) -> usize { self.decode_kernel_dispatches } + /// CUDA JPEG decode path used for the surface. + pub fn decode_path(self) -> CudaJpegDecodePath { + self.decode_path + } + + /// Whether the J2K-owned CUDA JPEG decode path was used. + pub fn used_owned_cuda_decode(self) -> bool { + self.decode_path == CudaJpegDecodePath::OwnedCuda + } + + /// Whether hardware JPEG decode was used. pub fn used_hardware_decode(self) -> bool { self.hardware_decode } } #[derive(Clone, Copy, Debug)] +/// Borrowed CUDA-resident JPEG decode surface. pub struct CudaSurface<'a> { #[cfg(feature = "cuda-runtime")] buffer: &'a CudaDeviceBuffer, @@ -51,6 +81,7 @@ pub struct CudaSurface<'a> { } impl CudaSurface<'_> { + /// Return the CUDA device pointer for the backing buffer. pub fn device_ptr(&self) -> u64 { #[cfg(feature = "cuda-runtime")] { @@ -62,12 +93,14 @@ impl CudaSurface<'_> { } } + /// Return dispatch statistics for the surface. pub fn stats(&self) -> CudaSurfaceStats { self.stats } } #[derive(Debug)] +/// Decoded JPEG surface returned by the CUDA adapter. pub struct Surface { pub(crate) backend: BackendKind, pub(crate) dimensions: (u32, u32), @@ -78,10 +111,12 @@ pub struct Surface { } impl Surface { + /// Number of bytes between consecutive rows. pub fn pitch_bytes(&self) -> usize { self.pitch_bytes } + /// Borrow host-resident bytes when this surface is not CUDA-backed. pub fn as_host_bytes(&self) -> Option<&[u8]> { match &self.storage { Storage::Host(bytes) => Some(bytes), @@ -90,6 +125,7 @@ impl Surface { } } + /// Copy surface bytes into a caller-provided strided output buffer. pub fn download_into(&self, out: &mut [u8], stride: usize) -> Result<(), Error> { match &self.storage { Storage::Host(bytes) => { @@ -106,6 +142,7 @@ impl Surface { } } + /// Borrow CUDA surface metadata when this surface is CUDA-backed. pub fn cuda_surface(&self) -> Option> { #[cfg(feature = "cuda-runtime")] match &self.storage { @@ -128,6 +165,14 @@ impl DeviceSurface for Surface { self.backend } + fn residency(&self) -> j2k_core::SurfaceResidency { + match &self.storage { + Storage::Host(_) => j2k_core::SurfaceResidency::Host, + #[cfg(feature = "cuda-runtime")] + Storage::Cuda(_) => j2k_core::SurfaceResidency::CudaResidentDecode, + } + } + fn dimensions(&self) -> (u32, u32) { self.dimensions } @@ -139,4 +184,24 @@ impl DeviceSurface for Surface { fn byte_len(&self) -> usize { self.pitch_bytes * self.dimensions.1 as usize } + + fn execution_stats(&self) -> ExecutionStats { + ExecutionStats { + kernel_dispatches: self.stats.kernel_dispatches as u64, + ..ExecutionStats::default() + } + } + + fn memory_range(&self) -> Option { + match &self.storage { + Storage::Host(_) => None, + #[cfg(feature = "cuda-runtime")] + Storage::Cuda(buffer) => Some(DeviceMemoryRange::new( + BackendKind::Cuda, + buffer.device_ptr(), + 0, + self.byte_len(), + )), + } + } } diff --git a/crates/j2k-jpeg-cuda/tests/host_surface.rs b/crates/j2k-jpeg-cuda/tests/host_surface.rs new file mode 100644 index 00000000..d60b8921 --- /dev/null +++ b/crates/j2k-jpeg-cuda/tests/host_surface.rs @@ -0,0 +1,790 @@ +use j2k_core::{ + BackendRequest, CodecError, DecoderContext, DeviceSubmission, DeviceSurface, Downscale, + ImageDecode, ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, Rect, TileBatchDecodeDevice, + TileBatchDecodeManyDevice, +}; +use j2k_jpeg_cuda::{Codec, CudaSession, Decoder, Error}; +use j2k_test_support::{cuda_jpeg_hardware_decode_required, cuda_runtime_required}; + +const BASELINE_420: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); +const BASELINE_422: &[u8] = include_bytes!("../fixtures/jpeg/baseline_422_16x8.jpg"); +const BASELINE_444: &[u8] = include_bytes!("../fixtures/jpeg/baseline_444_8x8.jpg"); +const OWNED_CUDA_RGB8_MAX_CHANNEL_DELTA: u8 = 2; + +#[test] +fn auto_falls_back_to_cpu_surface() { + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Auto) + .expect("surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert!(surface.as_host_bytes().is_some()); +} + +#[test] +fn explicit_cuda_request_returns_cuda_surface_or_clear_unavailable_error() { + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + match decoder.decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) { + Ok(surface) => { + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.as_host_bytes(), None); + #[cfg(feature = "cuda-runtime")] + assert_ne!( + surface.cuda_surface().expect("cuda surface").device_ptr(), + 0 + ); + } + Err(error) => assert!(error.is_unsupported()), + } +} + +#[test] +fn explicit_cuda_request_validates_decode_before_upload() { + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + + let error = decoder + .decode_to_device(PixelFormat::Rgba16, BackendRequest::Cuda) + .expect_err("unsupported decode"); + assert!(error.is_unsupported()); + assert!(!matches!(error, Error::CudaUnavailable)); +} + +#[test] +fn explicit_cuda_gray8_request_fails_without_cpu_upload() { + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + + let error = decoder + .decode_to_device(PixelFormat::Gray8, BackendRequest::Cuda) + .expect_err("strict CUDA Gray8 decode should be unsupported"); + assert!(error.is_unsupported()); + assert!(!matches!(error, Error::CudaUnavailable)); +} + +#[test] +fn explicit_cuda_request_returns_cuda_surface_when_cuda_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) + .expect("cuda surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.as_host_bytes(), None); + assert_cuda_surface(&surface); + assert_eq!(surface.dimensions(), (16, 16)); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda surface"); + + let (expected, _) = j2k_jpeg::Decoder::new(BASELINE_420) + .expect("host decoder") + .decode(PixelFormat::Rgb8) + .expect("host decode"); + assert_surface_bytes_match_or_are_close(&surface, &downloaded, &expected); +} + +#[test] +fn explicit_cuda_region_scaled_surface_fails_without_owned_cuda_path() { + let roi = Rect { + x: 4, + y: 4, + w: 10, + h: 10, + }; + let scale = Downscale::Quarter; + + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + let error = decoder + .decode_region_scaled_to_device(PixelFormat::Rgb8, roi, scale, BackendRequest::Cuda) + .expect_err("strict CUDA region+scaled decode should be unsupported"); + assert!(error.is_unsupported()); +} + +#[test] +fn explicit_cuda_download_respects_padded_stride_when_cuda_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) + .expect("cuda surface"); + assert_cuda_surface(&surface); + let row_bytes = surface.pitch_bytes(); + let stride = row_bytes + 5; + let mut downloaded = vec![0xCD; stride * surface.dimensions().1 as usize]; + surface + .download_into(&mut downloaded, stride) + .expect("download cuda surface"); + + let (expected, _) = j2k_jpeg::Decoder::new(BASELINE_420) + .expect("host decoder") + .decode(PixelFormat::Rgb8) + .expect("host decode"); + for (row, expected_row) in expected.chunks(row_bytes).enumerate() { + let start = row * stride; + assert_surface_bytes_match_or_are_close( + &surface, + &downloaded[start..start + row_bytes], + expected_row, + ); + assert_eq!(&downloaded[start + row_bytes..start + stride], &[0xCD; 5]); + } +} + +#[test] +fn explicit_cuda_full_frame_uses_owned_decode_when_required() { + if !cuda_jpeg_hardware_decode_required() { + return; + } + + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) + .expect("cuda surface"); + let cuda = surface.cuda_surface().expect("cuda surface"); + let stats = cuda.stats(); + assert!( + stats.used_owned_cuda_decode(), + "explicit full-frame RGB8 CUDA decode must use the J2K-owned CUDA JPEG path when required" + ); + assert!( + !stats.used_hardware_decode(), + "strict J2K-owned CUDA JPEG decode must not report external hardware decode" + ); + assert!( + stats.decode_kernel_dispatches() > 0, + "owned CUDA decode path must report decode kernel dispatches" + ); + assert_eq!( + stats.copy_kernel_dispatches(), + 0, + "owned CUDA decode path should not be reported as the CPU decode plus copy fallback" + ); +} + +#[test] +fn explicit_cuda_full_frame_422_uses_owned_decode_when_required() { + assert_full_frame_owned_cuda_decode_when_required(BASELINE_422, (16, 8)); +} + +#[test] +fn explicit_cuda_full_frame_444_uses_owned_decode_when_required() { + assert_full_frame_owned_cuda_decode_when_required(BASELINE_444, (8, 8)); +} + +#[test] +fn cuda_session_owned_decode_cache_starts_empty() { + let session = CudaSession::default(); + + assert_eq!(session.owned_cuda_packet_cache_len(), 0); +} + +#[cfg(feature = "cuda-runtime")] +fn generated_rgb_jpeg(subsampling: j2k_jpeg::JpegSubsampling, width: u32, height: u32) -> Vec { + let rgb = j2k_test_support::gpu_bench_rgb8(width, height); + j2k_jpeg::encode_jpeg_baseline( + j2k_jpeg::JpegSamples::Rgb8 { + data: &rgb, + width, + height, + }, + j2k_jpeg::JpegEncodeOptions { + quality: 90, + subsampling, + restart_interval: None, + backend: j2k_jpeg::JpegBackend::Cpu, + }, + ) + .expect("generated JPEG") + .data +} + +fn assert_cuda_surface(surface: &j2k_jpeg_cuda::Surface) { + let cuda = surface.cuda_surface().expect("cuda surface"); + assert_ne!(cuda.device_ptr(), 0); + assert!(cuda.stats().kernel_dispatches() > 0); +} + +fn assert_surface_bytes_match_or_are_close( + surface: &j2k_jpeg_cuda::Surface, + actual: &[u8], + expected: &[u8], +) { + assert_eq!(actual.len(), expected.len()); + let stats = surface.cuda_surface().expect("cuda surface").stats(); + if stats.used_owned_cuda_decode() { + let max_delta = actual + .iter() + .zip(expected) + .map(|(actual, expected)| actual.abs_diff(*expected)) + .max() + .unwrap_or(0); + assert!( + max_delta <= OWNED_CUDA_RGB8_MAX_CHANNEL_DELTA, + "J2K-owned CUDA decode differed from the CPU reference by max channel delta {max_delta}" + ); + return; + } + assert_eq!(actual, expected); +} + +fn assert_full_frame_owned_cuda_decode_when_required(input: &[u8], dimensions: (u32, u32)) { + if !cuda_jpeg_hardware_decode_required() { + return; + } + + let mut decoder = Decoder::new(input).expect("decoder"); + let surface = decoder + .decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) + .expect("cuda surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.dimensions(), dimensions); + assert_eq!(surface.as_host_bytes(), None); + assert_cuda_surface(&surface); + let stats = surface.cuda_surface().expect("cuda surface").stats(); + assert!(stats.used_owned_cuda_decode()); + assert!(!stats.used_hardware_decode()); + assert_eq!(stats.copy_kernel_dispatches(), 0); + + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda surface"); + let (expected, _) = j2k_jpeg::Decoder::new(input) + .expect("host decoder") + .decode(PixelFormat::Rgb8) + .expect("host decode"); + assert_surface_bytes_match_or_are_close(&surface, &downloaded, &expected); +} + +#[test] +fn submit_to_device_auto_falls_back_to_cpu_surface() { + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut session = CudaSession::default(); + let surface = as ImageDecodeSubmit<'_>>::submit_to_device( + &mut decoder, + &mut session, + PixelFormat::Rgb8, + BackendRequest::Auto, + ) + .expect("submission") + .wait() + .expect("surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert!(surface.as_host_bytes().is_some()); + assert!(session.submissions() >= 1); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn submit_to_device_auto_does_not_initialize_cuda_runtime() { + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut session = CudaSession::default(); + let surface = as ImageDecodeSubmit<'_>>::submit_to_device( + &mut decoder, + &mut session, + PixelFormat::Rgb8, + BackendRequest::Auto, + ) + .expect("submission") + .wait() + .expect("surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert_eq!(session.submissions(), 1); + assert!(!session.is_runtime_initialized()); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn explicit_cuda_submissions_reuse_session_runtime_when_required() { + if !cuda_runtime_required() { + return; + } + + let mut session = CudaSession::default(); + assert!(!session.is_runtime_initialized()); + + let mut first = Decoder::new(BASELINE_420).expect("decoder"); + let first_surface = as ImageDecodeSubmit<'_>>::submit_to_device( + &mut first, + &mut session, + PixelFormat::Rgb8, + BackendRequest::Cuda, + ) + .expect("first submission") + .wait() + .expect("first surface"); + assert_eq!(first_surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_cuda_surface(&first_surface); + assert!(session.is_runtime_initialized()); + + let mut second = Decoder::new(BASELINE_420).expect("decoder"); + let second_surface = as ImageDecodeSubmit<'_>>::submit_to_device( + &mut second, + &mut session, + PixelFormat::Rgb8, + BackendRequest::Cuda, + ) + .expect("second submission") + .wait() + .expect("second surface"); + assert_eq!(second_surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_cuda_surface(&second_surface); + assert_eq!(session.submissions(), 2); + assert!(session.is_runtime_initialized()); +} + +#[test] +fn auto_region_scaled_surface_matches_host_decode() { + let roi = Rect { + x: 4, + y: 4, + w: 10, + h: 10, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Rgb8, roi, scale, BackendRequest::Auto) + .expect("surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + + let mut host_decoder = Decoder::new(BASELINE_420).expect("host decoder"); + let mut host = vec![0u8; scaled.w as usize * scaled.h as usize * 3]; + host_decoder + .decode_region_scaled_into( + &mut j2k_jpeg::ScratchPool::new(), + &mut host, + scaled.w as usize * 3, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("host decode"); + assert_eq!(surface.as_host_bytes(), Some(host.as_slice())); +} + +#[test] +fn tile_batch_region_scaled_auto_surface_matches_host_decode() { + let roi = Rect { + x: 4, + y: 4, + w: 10, + h: 10, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_jpeg::ScratchPool::new(); + let surface = Codec::decode_tile_region_scaled_to_device( + &mut ctx, + &mut pool, + BASELINE_420, + PixelFormat::Rgb8, + roi, + scale, + BackendRequest::Auto, + ) + .expect("surface"); + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cpu); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + + let (expected, _) = j2k_jpeg::Decoder::new(BASELINE_420) + .expect("host decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("host decode"); + assert_eq!(surface.as_host_bytes(), Some(expected.as_slice())); +} + +#[test] +fn tile_batch_region_scaled_cuda_surface_fails_without_owned_cuda_path() { + let roi = Rect { + x: 4, + y: 4, + w: 10, + h: 10, + }; + let scale = Downscale::Quarter; + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_jpeg::ScratchPool::new(); + let error = Codec::decode_tile_region_scaled_to_device( + &mut ctx, + &mut pool, + BASELINE_420, + PixelFormat::Rgb8, + roi, + scale, + BackendRequest::Cuda, + ) + .expect_err("strict CUDA tile-batch region+scaled decode should be unsupported"); + assert!(error.is_unsupported()); +} + +#[test] +fn decode_tiles_to_device_auto_preserves_order_and_matches_host_bytes() { + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_jpeg::ScratchPool::new(); + let inputs = [BASELINE_420, BASELINE_420]; + + let surfaces = Codec::decode_tiles_to_device( + &mut ctx, + &mut pool, + &inputs, + PixelFormat::Rgb8, + BackendRequest::Auto, + ) + .expect("batch surfaces"); + + assert_eq!(surfaces.len(), inputs.len()); + let (expected, _) = j2k_jpeg::Decoder::new(BASELINE_420) + .expect("host decoder") + .decode(PixelFormat::Rgb8) + .expect("host decode"); + for surface in surfaces { + assert_eq!(surface.dimensions(), (16, 16)); + match surface.backend_kind() { + j2k_core::BackendKind::Cpu => { + assert_eq!(surface.as_host_bytes(), Some(expected.as_slice())); + } + j2k_core::BackendKind::Cuda => { + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda surface"); + assert_surface_bytes_match_or_are_close(&surface, &downloaded, &expected); + } + j2k_core::BackendKind::Metal => panic!("JPEG CUDA batch returned Metal surface"), + } + } +} + +#[test] +fn decode_tiles_to_device_with_session_auto_preserves_order_and_matches_host_bytes() { + let inputs = [BASELINE_420, BASELINE_420]; + let mut session = CudaSession::default(); + + let surfaces = Codec::decode_tiles_to_device_with_session( + &inputs, + PixelFormat::Rgb8, + BackendRequest::Auto, + &mut session, + ) + .expect("session-backed batch surfaces"); + + assert_eq!(surfaces.len(), inputs.len()); + let (expected, _) = j2k_jpeg::Decoder::new(BASELINE_420) + .expect("host decoder") + .decode(PixelFormat::Rgb8) + .expect("host decode"); + for surface in surfaces { + assert_eq!(surface.dimensions(), (16, 16)); + match surface.backend_kind() { + j2k_core::BackendKind::Cpu => { + assert_eq!(surface.as_host_bytes(), Some(expected.as_slice())); + } + j2k_core::BackendKind::Cuda => { + let mut downloaded = vec![0u8; surface.byte_len()]; + surface + .download_into(&mut downloaded, surface.pitch_bytes()) + .expect("download cuda surface"); + assert_surface_bytes_match_or_are_close(&surface, &downloaded, &expected); + } + j2k_core::BackendKind::Metal => panic!("JPEG CUDA batch returned Metal surface"), + } + } +} + +#[test] +fn decode_tiles_to_device_explicit_cuda_returns_cuda_surfaces_or_clear_unavailable_error() { + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_jpeg::ScratchPool::new(); + let inputs = [BASELINE_420, BASELINE_420]; + + match Codec::decode_tiles_to_device( + &mut ctx, + &mut pool, + &inputs, + PixelFormat::Rgb8, + BackendRequest::Cuda, + ) { + Ok(surfaces) => { + assert_eq!(surfaces.len(), inputs.len()); + for surface in surfaces { + assert_eq!(surface.backend_kind(), j2k_core::BackendKind::Cuda); + assert_eq!(surface.as_host_bytes(), None); + assert_cuda_surface(&surface); + } + } + Err(error) => assert!(error.is_unsupported()), + } +} + +#[test] +fn decode_tiles_to_device_explicit_cuda_gray8_fails_without_cpu_upload() { + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_jpeg::ScratchPool::new(); + let inputs = [BASELINE_420, BASELINE_420]; + + let error = Codec::decode_tiles_to_device( + &mut ctx, + &mut pool, + &inputs, + PixelFormat::Gray8, + BackendRequest::Cuda, + ) + .expect_err("strict CUDA Gray8 batch decode should be unsupported"); + assert!(error.is_unsupported()); + assert!(!matches!(error, Error::CudaUnavailable)); +} + +#[test] +fn decode_tiles_to_device_explicit_cuda_uses_owned_decode_when_required() { + if !cuda_jpeg_hardware_decode_required() { + return; + } + + let mut ctx = DecoderContext::::new(); + let mut pool = j2k_jpeg::ScratchPool::new(); + let inputs = [BASELINE_420, BASELINE_420]; + + let surfaces = Codec::decode_tiles_to_device( + &mut ctx, + &mut pool, + &inputs, + PixelFormat::Rgb8, + BackendRequest::Cuda, + ) + .expect("cuda batch surfaces"); + + assert_eq!(surfaces.len(), inputs.len()); + for surface in surfaces { + let stats = surface.cuda_surface().expect("cuda surface").stats(); + assert!( + stats.used_owned_cuda_decode(), + "explicit full-tile RGB8 CUDA batch decode must use the J2K-owned CUDA path when required" + ); + assert!( + stats.decode_kernel_dispatches() > 0, + "owned CUDA batch decode path must report decode dispatches" + ); + assert_eq!( + stats.copy_kernel_dispatches(), + 0, + "owned CUDA batch decode path should not be reported as CPU decode plus copy" + ); + } +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn generated_420_chunked_entropy_diagnostic_runs_when_cuda_runtime_required() { + if !cuda_runtime_required() { + return; + } + + let input = generated_rgb_jpeg(j2k_jpeg::JpegSubsampling::Ybr420, 256, 256); + let mut session = CudaSession::default(); + let report = Codec::diagnose_tile_rgb8_chunked_entropy_with_session( + &input, + j2k_cuda_runtime::CudaJpegChunkedEntropyConfig { + subsequence_words: 64, + sequence_len: 32, + max_overflow_subsequences: 4, + }, + &mut session, + ) + .expect("chunked entropy diagnostic"); + + assert!(report.subsequence_count() > 0); + assert_eq!(report.failed_state_count(), 0); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn generated_422_chunked_entropy_diagnostic_returns_diagnostic_420_only_error() { + let input = generated_rgb_jpeg(j2k_jpeg::JpegSubsampling::Ybr422, 256, 256); + let mut session = CudaSession::default(); + let error = Codec::diagnose_tile_rgb8_chunked_entropy_with_session( + &input, + j2k_cuda_runtime::CudaJpegChunkedEntropyConfig { + subsequence_words: 64, + sequence_len: 32, + max_overflow_subsequences: 4, + }, + &mut session, + ) + .expect_err("4:2:2 input should be rejected before diagnostic runtime"); + + assert!(error.is_unsupported()); + match error { + Error::UnsupportedCudaRequest { reason } => { + assert!(reason.contains("chunked entropy diagnostic")); + assert!(reason.contains("4:2:0")); + } + other => panic!("expected unsupported CUDA diagnostic error, got {other:?}"), + } +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn generated_420_chunked_entropy_diagnostic_rejects_invalid_config_before_runtime() { + let input = generated_rgb_jpeg(j2k_jpeg::JpegSubsampling::Ybr420, 256, 256); + let mut session = CudaSession::default(); + let error = Codec::diagnose_tile_rgb8_chunked_entropy_with_session( + &input, + j2k_cuda_runtime::CudaJpegChunkedEntropyConfig { + subsequence_words: 0, + sequence_len: 32, + max_overflow_subsequences: 4, + }, + &mut session, + ) + .expect_err("invalid diagnostic config should be rejected before runtime"); + + assert!(error.is_unsupported()); + match error { + Error::UnsupportedCudaRequest { reason } => { + assert!(reason.contains("chunked entropy diagnostic")); + assert!(reason.contains("config")); + } + other => panic!("expected unsupported CUDA diagnostic config error, got {other:?}"), + } +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn explicit_cuda_session_batch_records_owned_packet_cache_when_required() { + if !cuda_jpeg_hardware_decode_required() { + return; + } + + let inputs = [BASELINE_420, BASELINE_420]; + let mut session = CudaSession::default(); + let surfaces = Codec::decode_tiles_to_device_with_session( + &inputs, + PixelFormat::Rgb8, + BackendRequest::Cuda, + &mut session, + ) + .expect("cuda session batch surfaces"); + + assert_eq!(surfaces.len(), inputs.len()); + assert_eq!(session.owned_cuda_packet_cache_len(), 1); + for surface in surfaces { + let stats = surface.cuda_surface().expect("cuda surface").stats(); + assert!(stats.used_owned_cuda_decode()); + } +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn explicit_cuda_decodes_into_caller_owned_buffer_when_required() { + if !cuda_jpeg_hardware_decode_required() { + return; + } + + let mut session = CudaSession::default(); + let pitch = 16 * PixelFormat::Rgb8.bytes_per_pixel(); + let byte_len = pitch * 16; + let buffer = session + .take_owned_cuda_output_buffer(byte_len) + .expect("device output buffer"); + + let stats = Codec::decode_tile_rgb8_into_cuda_buffer_with_session( + BASELINE_420, + &buffer, + pitch, + &mut session, + ) + .expect("direct owned CUDA decode"); + + assert!(stats.used_owned_cuda_decode()); + assert_eq!(session.owned_cuda_packet_cache_len(), 1); + + let mut downloaded = vec![0u8; byte_len]; + buffer + .copy_to_host(&mut downloaded) + .expect("download buffer"); + let (expected, _) = j2k_jpeg::Decoder::new(BASELINE_420) + .expect("host decoder") + .decode(PixelFormat::Rgb8) + .expect("host decode"); + let max_delta = downloaded + .iter() + .zip(expected) + .map(|(actual, expected)| actual.abs_diff(expected)) + .max() + .unwrap_or(0); + assert!( + max_delta <= OWNED_CUDA_RGB8_MAX_CHANNEL_DELTA, + "direct J2K-owned CUDA decode differed from the CPU reference by max channel delta {max_delta}" + ); +} + +#[cfg(feature = "cuda-runtime")] +#[test] +fn explicit_cuda_decodes_422_and_444_into_caller_owned_buffers_when_required() { + if !cuda_jpeg_hardware_decode_required() { + return; + } + + for (input, dimensions) in [ + (BASELINE_422, (16_u32, 8_u32)), + (BASELINE_444, (8_u32, 8_u32)), + ] { + let mut session = CudaSession::default(); + let pitch = dimensions.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let byte_len = pitch * dimensions.1 as usize; + let buffer = session + .take_owned_cuda_output_buffer(byte_len) + .expect("device output buffer"); + + let stats = Codec::decode_tile_rgb8_into_cuda_buffer_with_session( + input, + &buffer, + pitch, + &mut session, + ) + .expect("direct owned CUDA decode"); + + assert!(stats.used_owned_cuda_decode()); + assert_eq!(session.owned_cuda_packet_cache_len(), 1); + + let mut downloaded = vec![0u8; byte_len]; + buffer + .copy_to_host(&mut downloaded) + .expect("download buffer"); + let (expected, _) = j2k_jpeg::Decoder::new(input) + .expect("host decoder") + .decode(PixelFormat::Rgb8) + .expect("host decode"); + let max_delta = downloaded + .iter() + .zip(expected) + .map(|(actual, expected)| actual.abs_diff(expected)) + .max() + .unwrap_or(0); + assert!( + max_delta <= OWNED_CUDA_RGB8_MAX_CHANNEL_DELTA, + "direct J2K-owned CUDA decode differed from the CPU reference by max channel delta {max_delta}" + ); + } +} diff --git a/crates/signinum-jpeg-metal/Cargo.toml b/crates/j2k-jpeg-metal/Cargo.toml similarity index 61% rename from crates/signinum-jpeg-metal/Cargo.toml rename to crates/j2k-jpeg-metal/Cargo.toml index 1cc2da6a..24be464e 100644 --- a/crates/signinum-jpeg-metal/Cargo.toml +++ b/crates/j2k-jpeg-metal/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "signinum-jpeg-metal" -description = "Metal device-output adapter for signinum-jpeg" -version = "0.4.2" +name = "j2k-jpeg-metal" +description = "Metal JPEG decode and baseline encode adapter for j2k-jpeg" +version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true @@ -10,24 +10,29 @@ keywords.workspace = true categories.workspace = true readme = "README.md" +[package.metadata.docs.rs] +all-features = true +targets = [] + [lib] -name = "signinum_jpeg_metal" +name = "j2k_jpeg_metal" path = "src/lib.rs" [dependencies] -signinum-core = { path = "../signinum-core", version = "=0.4.2" } -signinum-jpeg = { path = "../signinum-jpeg", version = "=0.4.2" } -signinum-profile = { path = "../signinum-profile", version = "=0.4.2" } +j2k-core = { path = "../j2k-core", version = "=0.6.0" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.6.0" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.6.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.6.0" } thiserror = { workspace = true } [target.'cfg(target_os = "macos")'.dependencies] -metal = "0.31" +metal = { workspace = true } [dev-dependencies] criterion = { workspace = true } jpeg-encoder = "0.7.0" jpeg-decoder = { version = "0.3.2", default-features = false } -signinum-test-support = { path = "../signinum-test-support" } +j2k-test-support = { path = "../j2k-test-support" } [[bench]] name = "device_upload" @@ -48,6 +53,7 @@ unreachable_pub = "warn" [lints.clippy] pedantic = { level = "warn", priority = -1 } +undocumented_unsafe_blocks = "warn" module_name_repetitions = "allow" must_use_candidate = "allow" missing_errors_doc = "allow" diff --git a/crates/j2k-jpeg-metal/README.md b/crates/j2k-jpeg-metal/README.md new file mode 100644 index 00000000..00fd5ab5 --- /dev/null +++ b/crates/j2k-jpeg-metal/README.md @@ -0,0 +1,26 @@ +# j2k-jpeg-metal + +Metal adapter for J2K JPEG decode and baseline encode paths on macOS. + +Supported paths return resident Metal outputs or use Metal kernels for selected +adapter stages. Explicit Metal requests are strict and fail for unsupported +shapes. + +## JPEG Decode Scope + +JPEG Metal decode is selective acceleration, not full JPEG feature coverage. It +is intended for fast baseline/checkpointed packet paths, batched WSI-style tile +decode, and resident-output viewport or texture workflows where the output can +stay on Metal. + +Explicit `BackendRequest::Metal` decode accepts only JPEG inputs that can build a +fast 4:2:0, 4:2:2, or 4:4:4 baseline packet and only `Gray8`, `Rgb8`, or +`Rgba8` output. Unsupported sampling families, unsupported color spaces, and +unsupported output formats return `UnsupportedMetalRequest` instead of silently +falling back. + +`BackendRequest::Auto` stays conservative. Single-image decode remains CPU even +when fast-packet capabilities match. Batched and resident-output paths are the +places to look for Metal wins, and any future Auto widening should be backed by +the benchmark groups documented in +[`docs/routing-benchmarks.md`](docs/routing-benchmarks.md). diff --git a/crates/signinum-jpeg-metal/benches/compare.rs b/crates/j2k-jpeg-metal/benches/compare.rs similarity index 77% rename from crates/signinum-jpeg-metal/benches/compare.rs rename to crates/j2k-jpeg-metal/benches/compare.rs index ca1b253b..5735edaa 100644 --- a/crates/signinum-jpeg-metal/benches/compare.rs +++ b/crates/j2k-jpeg-metal/benches/compare.rs @@ -1,27 +1,40 @@ // SPDX-License-Identifier: Apache-2.0 use criterion::{criterion_group, criterion_main, Criterion}; -use signinum_core::{ +use j2k_core::{ BackendRequest, DecoderContext, DeviceSubmission, Downscale, ImageDecodeSubmit, PixelFormat, Rect, TileBatchDecodeSubmit, }; -use signinum_jpeg::{ - adapter::summarize_device_batch, decode_tile_region_scaled_into_in_context, - decode_tile_scaled_into_in_context, Decoder as CpuDecoder, - DecoderContext as JpegDecoderContext, ScratchPool as CpuScratchPool, +use j2k_jpeg::{ + adapter::{ + build_fast420_packet_for_decoder, build_fast422_packet_for_decoder, + build_fast444_packet_for_decoder, summarize_device_batch, + }, + decode_tile_region_scaled_into_in_context, decode_tile_scaled_into_in_context, + Decoder as CpuDecoder, DecoderContext as JpegDecoderContext, ScratchPool as CpuScratchPool, }; -use signinum_jpeg_metal::viewport::{ +use j2k_jpeg_metal::viewport::{ compose_viewport_cpu, compose_viewport_cpu_to_surface, compose_viewport_hybrid, decode_viewport_region_cpu, decode_viewport_region_cpu_to_surface, decode_viewport_region_hybrid, decode_viewport_to_surface, suggest_viewport_workload, ViewportTile, ViewportWorkload, }; -use signinum_jpeg_metal::{Codec, Decoder, MetalSession, ScratchPool}; +#[cfg(target_os = "macos")] +use j2k_jpeg_metal::viewport::{ + decode_viewport_to_resizable_metal_buffer_with_decoder_session, + decode_viewport_to_resizable_metal_textures_with_decoder_session, +}; +use j2k_jpeg_metal::{Codec, Decoder, MetalSession, ScratchPool}; +#[cfg(target_os = "macos")] +use j2k_jpeg_metal::{MetalBackendSession, MetalBatchOutputBuffer, MetalBatchTextureOutput}; +use jpeg_encoder::{ColorType, Encoder, SamplingFactor}; use std::collections::HashSet; use std::fs; use std::path::{Path, PathBuf}; const FULL_FRAME_MAX_OUTPUT_BYTES: usize = 512 * 1024 * 1024; +#[cfg(target_os = "macos")] +const RESIDENT_TEXTURE_BATCH_SIZES: [usize; 3] = [16, 64, 256]; #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum DecodeMode { @@ -60,6 +73,19 @@ struct DistinctTileBatch<'a> { tiles: Vec<&'a BenchInput>, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct FastPacketPlan { + matches_fast_420: bool, + matches_fast_422: bool, + matches_fast_444: bool, +} + +impl FastPacketPlan { + fn has_fast_packet(self) -> bool { + self.matches_fast_420 || self.matches_fast_422 || self.matches_fast_444 + } +} + fn load_bench_inputs() -> Vec { let mut inputs = vec![ BenchInput { @@ -69,6 +95,20 @@ fn load_bench_inputs() -> Vec { mode: DecodeMode::Rgb, input_class: CorpusInputClass::BoundedFullFrame, }, + BenchInput { + name: "repo/baseline_422_16x8".to_string(), + bytes: include_bytes!("../fixtures/jpeg/baseline_422_16x8.jpg").to_vec(), + dimensions: (16, 8), + mode: DecodeMode::Rgb, + input_class: CorpusInputClass::BoundedFullFrame, + }, + BenchInput { + name: "repo/baseline_444_8x8".to_string(), + bytes: include_bytes!("../fixtures/jpeg/baseline_444_8x8.jpg").to_vec(), + dimensions: (8, 8), + mode: DecodeMode::Rgb, + input_class: CorpusInputClass::BoundedFullFrame, + }, BenchInput { name: "repo/grayscale_8x8".to_string(), bytes: include_bytes!("../fixtures/jpeg/grayscale_8x8.jpg").to_vec(), @@ -76,21 +116,67 @@ fn load_bench_inputs() -> Vec { mode: DecodeMode::Gray, input_class: CorpusInputClass::BoundedFullFrame, }, + BenchInput { + name: "generated/fast420_256x256".to_string(), + bytes: generated_rgb_jpeg(256, 256, SamplingFactor::F_2_2, None), + dimensions: (256, 256), + mode: DecodeMode::Rgb, + input_class: CorpusInputClass::BoundedFullFrame, + }, + BenchInput { + name: "generated/fast420_restart2_256x256".to_string(), + bytes: generated_rgb_jpeg(256, 256, SamplingFactor::F_2_2, Some(2)), + dimensions: (256, 256), + mode: DecodeMode::Rgb, + input_class: CorpusInputClass::BoundedFullFrame, + }, + BenchInput { + name: "generated/fast422_256x256".to_string(), + bytes: generated_rgb_jpeg(256, 256, SamplingFactor::F_2_1, None), + dimensions: (256, 256), + mode: DecodeMode::Rgb, + input_class: CorpusInputClass::BoundedFullFrame, + }, + BenchInput { + name: "generated/fast444_256x256".to_string(), + bytes: generated_rgb_jpeg(256, 256, SamplingFactor::F_1_1, None), + dimensions: (256, 256), + mode: DecodeMode::Rgb, + input_class: CorpusInputClass::BoundedFullFrame, + }, ]; let mut seen = inputs .iter() .map(|input| input.name.clone()) .collect::>(); - for path in - std::env::split_paths(&std::env::var_os("SIGNINUM_BENCH_INPUTS").unwrap_or_default()) - { + for path in std::env::split_paths(&std::env::var_os("J2K_BENCH_INPUTS").unwrap_or_default()) { collect_jpegs(&path, &mut inputs, &mut seen); } inputs.sort_by(|a, b| a.name.cmp(&b.name)); inputs } +fn generated_rgb_jpeg( + width: u16, + height: u16, + sampling: SamplingFactor, + restart_interval: Option, +) -> Vec { + let rgb = j2k_test_support::gpu_bench_rgb8(u32::from(width), u32::from(height)); + + let mut jpeg = Vec::new(); + let mut encoder = Encoder::new(&mut jpeg, 90); + encoder.set_sampling_factor(sampling); + if let Some(interval) = restart_interval { + encoder.set_restart_interval(interval); + } + encoder + .encode(&rgb, width, height, ColorType::Rgb) + .expect("encode generated benchmark JPEG"); + jpeg +} + fn collect_jpegs(path: &Path, inputs: &mut Vec, seen: &mut Vec) { if path.is_file() { push_jpeg(path, inputs, seen); @@ -162,11 +248,11 @@ fn relative_name(path: &Path) -> String { absolute.display().to_string() } -fn color_space_mode(color_space: signinum_jpeg::ColorSpace) -> Option { +fn color_space_mode(color_space: j2k_jpeg::ColorSpace) -> Option { match color_space { - signinum_jpeg::ColorSpace::Grayscale => Some(DecodeMode::Gray), - signinum_jpeg::ColorSpace::YCbCr | signinum_jpeg::ColorSpace::Rgb => Some(DecodeMode::Rgb), - signinum_jpeg::ColorSpace::Cmyk | signinum_jpeg::ColorSpace::Ycck => None, + j2k_jpeg::ColorSpace::Grayscale => Some(DecodeMode::Gray), + j2k_jpeg::ColorSpace::YCbCr | j2k_jpeg::ColorSpace::Rgb => Some(DecodeMode::Rgb), + j2k_jpeg::ColorSpace::Cmyk | j2k_jpeg::ColorSpace::Ycck => None, } } @@ -177,11 +263,7 @@ fn classify_corpus_input(dimensions: (u32, u32), mode: DecodeMode) -> CorpusInpu }; let bytes = usize::try_from(dimensions.0) .ok() - .and_then(|width| { - usize::try_from(dimensions.1) - .ok() - .map(|height| (width, height)) - }) + .zip(usize::try_from(dimensions.1).ok()) .and_then(|(width, height)| width.checked_mul(height)) .and_then(|pixels| pixels.checked_mul(bpp)); match bytes { @@ -215,6 +297,71 @@ fn device_batch_key(input: &BenchInput) -> Option { }) } +fn fast_packet_plan(bytes: &[u8]) -> Option { + let decoder = CpuDecoder::new(bytes).ok()?; + Some(FastPacketPlan { + matches_fast_444: build_fast444_packet_for_decoder(&decoder).is_ok(), + matches_fast_422: build_fast422_packet_for_decoder(&decoder).is_ok(), + matches_fast_420: build_fast420_packet_for_decoder(&decoder).is_ok(), + }) +} + +fn fast_packet_family_label(plan: FastPacketPlan) -> &'static str { + if plan.matches_fast_420 { + "fast420" + } else if plan.matches_fast_422 { + "fast422" + } else if plan.matches_fast_444 { + "fast444" + } else { + "no_fast_packet" + } +} + +fn checkpoint_label(key: DeviceBatchKey) -> &'static str { + if key.restart_interval.is_some() { + "restart" + } else if key.checkpoint_count > 0 { + "checkpointed" + } else { + "unchunked" + } +} + +fn fast_packet_scope_label(input: &BenchInput) -> String { + let Some(plan) = fast_packet_plan(&input.bytes) else { + return "reject/not_jpeg".to_string(); + }; + let checkpoint = device_batch_key(input).map_or("unknown", checkpoint_label); + + let family = fast_packet_family_label(plan); + let disposition = if plan.has_fast_packet() { + "accept" + } else { + "reject" + }; + format!("{disposition}/{family}/{checkpoint}") +} + +fn bench_fast_packet_planning(c: &mut Criterion, inputs: &[BenchInput]) { + let mut group = c.benchmark_group("jpeg_metal_fast_packet_planning"); + for input in inputs + .iter() + .filter(|input| input.input_class == CorpusInputClass::BoundedFullFrame) + { + let scope = fast_packet_scope_label(input); + group.bench_function(format!("{scope}/{}", input.name), |b| { + b.iter(|| { + std::hint::black_box( + fast_packet_plan(std::hint::black_box(input.bytes.as_slice())) + .expect("fast packet plan"), + ); + }); + }); + } + group.finish(); +} + fn digest_bytes(bytes: &[u8]) -> u64 { const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; const FNV_PRIME: u64 = 0x0000_0100_0000_01B3; @@ -302,8 +449,8 @@ fn centered_roi((width, height): (u32, u32), side: u32) -> Rect { } } -fn to_jpeg_rect(rect: Rect) -> signinum_jpeg::Rect { - signinum_jpeg::Rect { +fn to_jpeg_rect(rect: Rect) -> j2k_jpeg::Rect { + j2k_jpeg::Rect { x: rect.x, y: rect.y, w: rect.w, @@ -317,7 +464,7 @@ fn cpu_decode_tile_batch(bytes: &[u8], batch_size: usize) { let mut out = Vec::new(); for _ in 0..batch_size { let decoder = CpuDecoder::from_view_in_context( - signinum_jpeg::JpegView::parse(bytes).expect("view"), + j2k_jpeg::JpegView::parse(bytes).expect("view"), &mut ctx, ) .expect("decoder"); @@ -382,7 +529,7 @@ fn cpu_decode_region(bytes: &[u8], side: u32) { let (out, _) = decoder .decode_region( PixelFormat::Rgb8, - signinum_jpeg::Rect { + j2k_jpeg::Rect { x: roi.x, y: roi.y, w: roi.w, @@ -496,6 +643,179 @@ fn metal_decode_tile_batch(bytes: &[u8], batch_size: usize) { device_decode_tile_batch(bytes, batch_size, BackendRequest::Metal); } +#[cfg(target_os = "macos")] +fn supports_resident_texture_batch(input: &BenchInput) -> bool { + device_batch_key(input) + .is_some_and(|key| key.matches_fast_420 || key.matches_fast_422 || key.matches_fast_444) +} + +#[cfg(target_os = "macos")] +fn bench_resident_texture_batches(c: &mut Criterion, inputs: &[BenchInput], has_metal: bool) { + if !has_metal { + return; + } + + let mut group = c.benchmark_group("wsi_tile_batch_rgba_textures"); + for &batch_size in &RESIDENT_TEXTURE_BATCH_SIZES { + for input in inputs + .iter() + .filter(|input| input.mode == DecodeMode::Rgb && supports_resident_texture_batch(input)) + { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, input.dimensions, batch_size) + .expect("texture output"); + let decoders = (0..batch_size) + .map(|_| Decoder::new(input.bytes.as_slice()).expect("metal decoder")) + .collect::>(); + group.bench_function( + format!( + "batch{batch_size}/warm_session_reused_textures/{}", + input.name + ), + move |b| { + let decoder_refs = decoders.iter().collect::>(); + b.iter(|| { + let tiles = Codec::decode_rgb8_decoder_batch_into_resizable_metal_textures_with_session( + &decoder_refs, + &mut output, + &session, + ) + .expect("resident texture batch decode"); + std::hint::black_box(tiles); + }); + }, + ); + } + } + group.finish(); +} + +#[cfg(not(target_os = "macos"))] +fn bench_resident_texture_batches(_c: &mut Criterion, _inputs: &[BenchInput], _has_metal: bool) {} + +#[cfg(target_os = "macos")] +fn bench_resident_viewport_outputs(c: &mut Criterion, inputs: &[BenchInput], has_metal: bool) { + if !has_metal { + return; + } + + { + let mut buffer_group = c.benchmark_group("viewer_resident_viewport_rgb_buffer_warm"); + for input in inputs.iter().filter(|input| { + input.mode == DecodeMode::Rgb + && input.input_class == CorpusInputClass::BoundedFullFrame + && suggest_viewport_workload(input.dimensions).is_some() + }) { + let contiguous_workload = + suggest_viewport_workload(input.dimensions).expect("viewport workload"); + bench_resident_viewport_buffer_case( + &mut buffer_group, + input, + "contiguous", + contiguous_workload.clone(), + ); + + if let Some(sparse_workload) = sparse_viewport_workload(&contiguous_workload) { + bench_resident_viewport_buffer_case( + &mut buffer_group, + input, + "sparse", + sparse_workload, + ); + } + } + buffer_group.finish(); + } + + { + let mut texture_group = c.benchmark_group("viewer_resident_viewport_rgba_texture_warm"); + for input in inputs.iter().filter(|input| { + input.mode == DecodeMode::Rgb + && input.input_class == CorpusInputClass::BoundedFullFrame + && suggest_viewport_workload(input.dimensions).is_some() + }) { + let contiguous_workload = + suggest_viewport_workload(input.dimensions).expect("viewport workload"); + bench_resident_viewport_texture_case( + &mut texture_group, + input, + "contiguous", + contiguous_workload.clone(), + ); + + if let Some(sparse_workload) = sparse_viewport_workload(&contiguous_workload) { + bench_resident_viewport_texture_case( + &mut texture_group, + input, + "sparse", + sparse_workload, + ); + } + } + texture_group.finish(); + } +} + +#[cfg(not(target_os = "macos"))] +fn bench_resident_viewport_outputs(_c: &mut Criterion, _inputs: &[BenchInput], _has_metal: bool) {} + +#[cfg(target_os = "macos")] +fn bench_resident_viewport_buffer_case( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + input: &BenchInput, + shape: &'static str, + workload: ViewportWorkload, +) { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("buffer output"); + let bytes = input.bytes.clone(); + group.bench_function(format!("{shape}/{}", input.name), move |b| { + let decoder = Decoder::new(&bytes).expect("metal decoder"); + let mut pool = CpuScratchPool::new(); + b.iter(|| { + let surface = decode_viewport_to_resizable_metal_buffer_with_decoder_session( + &decoder, + &mut pool, + &workload, + &mut output, + &session, + ) + .expect("resident viewport buffer"); + std::hint::black_box(surface); + }); + }); +} + +#[cfg(target_os = "macos")] +fn bench_resident_viewport_texture_case( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + input: &BenchInput, + shape: &'static str, + workload: ViewportWorkload, +) { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + let bytes = input.bytes.clone(); + group.bench_function(format!("{shape}/{}", input.name), move |b| { + let decoder = Decoder::new(&bytes).expect("metal decoder"); + let mut pool = CpuScratchPool::new(); + b.iter(|| { + let tile = decode_viewport_to_resizable_metal_textures_with_decoder_session( + &decoder, + &mut pool, + &workload, + &mut output, + &session, + ) + .expect("resident viewport texture"); + std::hint::black_box(tile); + }); + }); +} + fn auto_decode_tile_batch(bytes: &[u8], batch_size: usize) { device_decode_tile_batch(bytes, batch_size, BackendRequest::Auto); } @@ -588,11 +908,11 @@ fn metal_decode_tile_batch_region_scaled( std::hint::black_box(submission.wait().expect("surface")); } assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 1, "coalesced region+scaled tile batch should flush once" ); - std::hint::black_box(session.submissions()); + std::hint::black_box(session.submissions().expect("session submissions")); } fn metal_decode_distinct_tile_batch_region_scaled( @@ -623,7 +943,7 @@ fn metal_decode_distinct_tile_batch_region_scaled( for submission in submissions { std::hint::black_box(submission.wait().expect("surface")); } - std::hint::black_box(session.submissions()); + std::hint::black_box(session.submissions().expect("session submissions")); } fn metal_decode_region(bytes: &[u8], side: u32) { @@ -743,7 +1063,7 @@ fn hybrid_viewport_composite_device(bytes: &[u8], dimensions: (u32, u32)) { fn scheduled_viewport_surface( decoder: &CpuDecoder<'_>, pool: &mut CpuScratchPool, - workload: &signinum_jpeg_metal::viewport::ViewportWorkload, + workload: &j2k_jpeg_metal::viewport::ViewportWorkload, backend: BackendRequest, ) { let surface = decode_viewport_to_surface(decoder, pool, workload, backend).expect("viewport"); @@ -777,8 +1097,8 @@ fn metal_available() -> bool { #[cfg(not(target_os = "macos"))] { assert!( - std::env::var_os("SIGNINUM_REQUIRE_METAL_BENCH").is_none(), - "SIGNINUM_REQUIRE_METAL_BENCH is set but this is not a Metal host" + std::env::var_os("J2K_REQUIRE_METAL_BENCH").is_none(), + "J2K_REQUIRE_METAL_BENCH is set but this is not a Metal host" ); false } @@ -790,6 +1110,8 @@ fn bench_compare(c: &mut Criterion) { let coalesced_hit_rate = coalesce_hit_rate_label(63, 64); let has_metal = metal_available(); + bench_fast_packet_planning(c, &inputs); + let mut decode_rgb = c.benchmark_group("decode_rgb"); for input in inputs.iter().filter(|input| { input.mode == DecodeMode::Rgb && input.input_class == CorpusInputClass::BoundedFullFrame @@ -821,6 +1143,9 @@ fn bench_compare(c: &mut Criterion) { } wsi_tile_batch_rgb.finish(); + bench_resident_texture_batches(c, &inputs, has_metal); + bench_resident_viewport_outputs(c, &inputs, has_metal); + let mut wsi_region_rgb = c.benchmark_group("wsi_region_rgb"); for input in inputs.iter().filter(|input| { input.mode == DecodeMode::Rgb && input.input_class == CorpusInputClass::BoundedFullFrame diff --git a/crates/signinum-jpeg-metal/benches/device_upload.rs b/crates/j2k-jpeg-metal/benches/device_upload.rs similarity index 64% rename from crates/signinum-jpeg-metal/benches/device_upload.rs rename to crates/j2k-jpeg-metal/benches/device_upload.rs index 087290c5..67ec1326 100644 --- a/crates/signinum-jpeg-metal/benches/device_upload.rs +++ b/crates/j2k-jpeg-metal/benches/device_upload.rs @@ -1,13 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 use criterion::{criterion_group, criterion_main, Criterion}; -use jpeg_encoder::{ColorType, Encoder, SamplingFactor}; -use signinum_core::{ +use j2k_core::{ BackendRequest, DecoderContext, DeviceSubmission, ImageDecodeDevice, PixelFormat, TileBatchDecodeSubmit, }; -use signinum_jpeg::{Decoder as CpuDecoder, DecoderContext as JpegDecoderContext}; -use signinum_jpeg_metal::{Codec, Decoder as MetalDecoder, MetalSession, ScratchPool}; +use j2k_jpeg::{Decoder as CpuDecoder, DecoderContext as JpegDecoderContext}; +use j2k_jpeg_metal::{Codec, Decoder as MetalDecoder, MetalSession, ScratchPool}; +use jpeg_encoder::{ColorType, Encoder, SamplingFactor}; const BASELINE_420: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); const DEFAULT_GENERATED_DIM: u16 = 2048; @@ -47,30 +47,31 @@ fn metal_decode_available() -> bool { #[cfg(not(target_os = "macos"))] { assert!( - std::env::var_os("SIGNINUM_REQUIRE_METAL_BENCH").is_none(), - "SIGNINUM_REQUIRE_METAL_BENCH is set but this is not a Metal host" + std::env::var_os("J2K_REQUIRE_METAL_BENCH").is_none(), + "J2K_REQUIRE_METAL_BENCH is set but this is not a Metal host" ); false } } fn bench_input() -> Vec { - match std::env::var_os("SIGNINUM_GPU_BENCH_JPEG") { + match std::env::var_os("J2K_GPU_BENCH_JPEG") { Some(path) => std::fs::read(&path).unwrap_or_else(|error| { panic!( - "failed to read SIGNINUM_GPU_BENCH_JPEG={}: {error}", + "failed to read J2K_GPU_BENCH_JPEG={}: {error}", path.to_string_lossy() ) }), - None if std::env::var_os("SIGNINUM_GPU_BENCH_SMALL_FIXTURE").is_some() => { - BASELINE_420.to_vec() + None if std::env::var_os("J2K_GPU_BENCH_SMALL_FIXTURE").is_some() => BASELINE_420.to_vec(), + None => { + let (width, height) = generated_dimensions(); + generated_jpeg(width, height) } - None => generated_jpeg(generated_dim()), } } -fn generated_jpeg(dim: u16) -> Vec { - let rgb = signinum_test_support::gpu_bench_rgb8(u32::from(dim), u32::from(dim)); +fn generated_jpeg(width: u16, height: u16) -> Vec { + let rgb = j2k_test_support::gpu_bench_rgb8(u32::from(width), u32::from(height)); let mut jpeg = Vec::new(); let mut encoder = Encoder::new(&mut jpeg, 90); @@ -79,32 +80,57 @@ fn generated_jpeg(dim: u16) -> Vec { encoder.set_restart_interval(interval); } encoder - .encode(&rgb, dim, dim, ColorType::Rgb) + .encode(&rgb, width, height, ColorType::Rgb) .expect("encode generated benchmark JPEG"); jpeg } -fn generated_dim() -> u16 { - let Some(value) = std::env::var_os("SIGNINUM_GPU_BENCH_DIM") else { - return DEFAULT_GENERATED_DIM; +fn generated_dimensions() -> (u16, u16) { + let Some(value) = std::env::var_os("J2K_GPU_BENCH_DIM") else { + return (DEFAULT_GENERATED_DIM, DEFAULT_GENERATED_DIM); }; - let value = value - .to_string_lossy() + parse_dimensions(&value.to_string_lossy()) +} + +fn parse_dimensions(value: &str) -> (u16, u16) { + if let Some((width, height)) = value.split_once('x') { + parse_dimensions_pair(width, height) + } else { + let square = value + .parse::() + .expect("J2K_GPU_BENCH_DIM must be a u16 or WIDTHxHEIGHT"); + assert_in_bench_bounds(square); + (square, square) + } +} + +fn parse_dimensions_pair(width: &str, height: &str) -> (u16, u16) { + let width = width + .trim() + .parse::() + .expect("J2K_GPU_BENCH_DIM must be a u16 or WIDTHxHEIGHT"); + let height = height + .trim() .parse::() - .expect("SIGNINUM_GPU_BENCH_DIM must be a u16"); + .expect("J2K_GPU_BENCH_DIM must be a u16 or WIDTHxHEIGHT"); + assert_in_bench_bounds(width); + assert_in_bench_bounds(height); + (width, height) +} + +fn assert_in_bench_bounds(value: u16) { assert!( (256..=8192).contains(&value), - "SIGNINUM_GPU_BENCH_DIM must be between 256 and 8192" + "J2K_GPU_BENCH_DIM dimensions must be between 256 and 8192" ); - value } fn restart_interval() -> Option { - let value = std::env::var_os("SIGNINUM_GPU_BENCH_RESTART_INTERVAL")?; + let value = std::env::var_os("J2K_GPU_BENCH_RESTART_INTERVAL")?; let value = value .to_string_lossy() .parse::() - .expect("SIGNINUM_GPU_BENCH_RESTART_INTERVAL must be a u16"); + .expect("J2K_GPU_BENCH_RESTART_INTERVAL must be a u16"); if value == 0 { None } else { @@ -114,7 +140,7 @@ fn restart_interval() -> Option { fn bench_batch_decode(c: &mut Criterion) { let dim = batch_dim(); - let input = generated_jpeg(dim); + let input = generated_jpeg(dim, dim); let batch_size = batch_size(); let mut group = c.benchmark_group("jpeg_metal_batch_decode"); @@ -173,31 +199,31 @@ fn device_decode_tile_batch(input: &[u8], batch_size: usize, backend: BackendReq } fn batch_size() -> usize { - let Some(value) = std::env::var_os("SIGNINUM_GPU_BENCH_BATCH") else { + let Some(value) = std::env::var_os("J2K_GPU_BENCH_BATCH") else { return DEFAULT_BATCH_SIZE; }; let value = value .to_string_lossy() .parse::() - .expect("SIGNINUM_GPU_BENCH_BATCH must be a usize"); + .expect("J2K_GPU_BENCH_BATCH must be a usize"); assert!( - (1..=256).contains(&value), - "SIGNINUM_GPU_BENCH_BATCH must be between 1 and 256" + (1..=512).contains(&value), + "J2K_GPU_BENCH_BATCH must be between 1 and 512" ); value } fn batch_dim() -> u16 { - let Some(value) = std::env::var_os("SIGNINUM_GPU_BENCH_BATCH_DIM") else { + let Some(value) = std::env::var_os("J2K_GPU_BENCH_BATCH_DIM") else { return DEFAULT_BATCH_DIM; }; let value = value .to_string_lossy() .parse::() - .expect("SIGNINUM_GPU_BENCH_BATCH_DIM must be a u16"); + .expect("J2K_GPU_BENCH_BATCH_DIM must be a u16"); assert!( (128..=4096).contains(&value), - "SIGNINUM_GPU_BENCH_BATCH_DIM must be between 128 and 4096" + "J2K_GPU_BENCH_BATCH_DIM must be between 128 and 4096" ); value } diff --git a/crates/signinum-jpeg-metal/benches/encode_baseline.rs b/crates/j2k-jpeg-metal/benches/encode_baseline.rs similarity index 87% rename from crates/signinum-jpeg-metal/benches/encode_baseline.rs rename to crates/j2k-jpeg-metal/benches/encode_baseline.rs index eea5ebf6..7bebd08d 100644 --- a/crates/signinum-jpeg-metal/benches/encode_baseline.rs +++ b/crates/j2k-jpeg-metal/benches/encode_baseline.rs @@ -8,12 +8,12 @@ use criterion::{criterion_group, criterion_main, Criterion, Throughput}; #[cfg(target_os = "macos")] -use signinum_core::PixelFormat; -use signinum_jpeg::{ +use j2k_core::PixelFormat; +use j2k_jpeg::{ encode_jpeg_baseline, JpegBackend, JpegEncodeOptions, JpegSamples, JpegSubsampling, }; #[cfg(target_os = "macos")] -use signinum_jpeg_metal::{ +use j2k_jpeg_metal::{ encode_jpeg_baseline_batch_from_metal_buffers, encode_jpeg_baseline_from_metal_buffer, JpegBaselineMetalEncodeTile, MetalBackendSession, }; @@ -27,7 +27,7 @@ fn bench_encode_baseline(c: &mut Criterion) { let dim = bench_dim(); let batch_size = bench_batch_size(); let tile_bytes = dim as usize * dim as usize * 3; - let rgb = signinum_test_support::patterned_rgb8_tiles(dim, dim, batch_size); + let rgb = j2k_test_support::patterned_rgb8_tiles(dim, dim, batch_size); let cpu_options = options(JpegBackend::Cpu); #[cfg(target_os = "macos")] let metal_options = options(JpegBackend::Metal); @@ -153,46 +153,46 @@ fn options(backend: JpegBackend) -> JpegEncodeOptions { } fn bench_dim() -> u32 { - let Some(value) = std::env::var_os("SIGNINUM_JPEG_ENCODE_BENCH_DIM") else { + let Some(value) = std::env::var_os("J2K_JPEG_ENCODE_BENCH_DIM") else { return DEFAULT_DIM; }; let value = value .to_string_lossy() .parse::() - .expect("SIGNINUM_JPEG_ENCODE_BENCH_DIM must be a u32"); + .expect("J2K_JPEG_ENCODE_BENCH_DIM must be a u32"); assert!( (64..=4096).contains(&value), - "SIGNINUM_JPEG_ENCODE_BENCH_DIM must be between 64 and 4096" + "J2K_JPEG_ENCODE_BENCH_DIM must be between 64 and 4096" ); value } fn bench_batch_size() -> usize { - let Some(value) = std::env::var_os("SIGNINUM_JPEG_ENCODE_BENCH_BATCH") else { + let Some(value) = std::env::var_os("J2K_JPEG_ENCODE_BENCH_BATCH") else { return DEFAULT_BATCH_SIZE; }; let value = value .to_string_lossy() .parse::() - .expect("SIGNINUM_JPEG_ENCODE_BENCH_BATCH must be a usize"); + .expect("J2K_JPEG_ENCODE_BENCH_BATCH must be a usize"); assert!( (1..=128).contains(&value), - "SIGNINUM_JPEG_ENCODE_BENCH_BATCH must be between 1 and 128" + "J2K_JPEG_ENCODE_BENCH_BATCH must be between 1 and 128" ); value } fn bench_quality() -> u8 { - let Some(value) = std::env::var_os("SIGNINUM_JPEG_ENCODE_BENCH_QUALITY") else { + let Some(value) = std::env::var_os("J2K_JPEG_ENCODE_BENCH_QUALITY") else { return DEFAULT_QUALITY; }; let value = value .to_string_lossy() .parse::() - .expect("SIGNINUM_JPEG_ENCODE_BENCH_QUALITY must be a u8"); + .expect("J2K_JPEG_ENCODE_BENCH_QUALITY must be a u8"); assert!( (1..=100).contains(&value), - "SIGNINUM_JPEG_ENCODE_BENCH_QUALITY must be between 1 and 100" + "J2K_JPEG_ENCODE_BENCH_QUALITY must be between 1 and 100" ); value } diff --git a/crates/j2k-jpeg-metal/docs/routing-benchmarks.md b/crates/j2k-jpeg-metal/docs/routing-benchmarks.md new file mode 100644 index 00000000..b12302ce --- /dev/null +++ b/crates/j2k-jpeg-metal/docs/routing-benchmarks.md @@ -0,0 +1,80 @@ +# JPEG Metal Routing and Benchmarks + +JPEG Metal decode should stay selective. The current Metal paths are for fast +baseline/checkpointed packets, coalesced batches, and resident outputs. They are +not a claim of full JPEG entropy decode coverage. + +## Routing Contract + +Explicit `BackendRequest::Metal` is strict: + +- Accept candidates: fast baseline 4:2:0, 4:2:2, or 4:4:4 packets produced by + the `j2k-jpeg` fast packet builders. +- Output formats: `Gray8`, `Rgb8`, and `Rgba8`. +- Rejections: unsupported packet shape, unsupported output format, unsupported + backend, or unavailable Metal runtime. Unsupported explicit Metal requests + must return a structured error before launching kernels. + +`BackendRequest::Auto` is deliberately narrower: + +- Single-image full, region, scaled, and region-scaled requests stay on CPU even + when a fast packet exists. +- Small restart-coded tile batches stay CPU. +- Existing restart-coded batch threshold tests cover the current macOS Auto path + that can use Metal for coalesced WSI-style batches. +- Sparse viewport workloads stay CPU for scheduled surface output; contiguous + restart-coded viewports may use the hybrid path on macOS. Reusable resident + viewport outputs may use direct contiguous decode or resident composition. + +## Benchmark Map + +Run: + +```sh +cargo bench -p j2k-jpeg-metal --bench compare +``` + +The harness also accepts extra JPEG corpus roots through `J2K_BENCH_INPUTS`. +Inputs are grouped by dimensions, restart/checkpoint shape, and fast packet +family so batch results make coalescing visible. + +Use these groups to decide where Metal makes sense: + +- `decode_rgb`: cold single full-frame CPU vs explicit Metal. This is the loss + check for CPU-preferred single decode. +- `wsi_tile_batch_rgb` and `wsi_tile_batch_scaled_rgb_q4`: repeated tile batches + comparing CPU, explicit Metal, and Auto. +- `wsi_tile_batch_region_scaled_coalesced_rgb_q4`: coalesced region+scaled batch + candidate where Metal can amortize setup. +- `wsi_tile_batch_region_scaled_distinct_rgb_q4`: low-coalescing control case. + Treat Metal wins here as evidence, not an assumption. +- `wsi_tile_batch_rgba_textures`: resident texture batches that avoid host + downloads. +- `viewer_region_scaled_composite_rgb*`: CPU/hybrid viewport comparisons, with + warm variants separating setup from repeated viewer work. +- `viewer_resident_viewport_rgb_buffer_warm` and + `viewer_resident_viewport_rgba_texture_warm`: resident-output viewport cases. +- `jpeg_metal_fast_packet_planning`: route-discovery overhead for accepted and + rejected fast packet families. + +The benchmark surface intentionally includes both likely wins and likely losses. +Do not broaden Auto routing unless the relevant group shows repeatable wins for +the workload class being changed. + +## Applied Rust Guidance + +- Rust API Guidelines: the routing contract keeps errors meaningful and + documented, validates request shape before dispatch, and links the benchmark + evidence to the public crate docs. + +- Clippy docs: this crate keeps the existing targeted `pedantic` setup and does + not enable broad `restriction` or `nursery` groups for benchmark-only code. + +- Cargo features: no new feature flag is added for this routing change; macOS + Metal availability remains target-gated so feature unification cannot widen + JPEG Metal behavior accidentally. + +- Unsafe Code Guidelines: no new unsafe code is required for the routing or + benchmark documentation; the Metal path continues to use the existing runtime + wrappers and tests around resident surfaces. + diff --git a/crates/signinum-jpeg-metal/fixtures/jpeg/baseline_420_16x16.jpg b/crates/j2k-jpeg-metal/fixtures/jpeg/baseline_420_16x16.jpg similarity index 100% rename from crates/signinum-jpeg-metal/fixtures/jpeg/baseline_420_16x16.jpg rename to crates/j2k-jpeg-metal/fixtures/jpeg/baseline_420_16x16.jpg diff --git a/crates/signinum-jpeg-metal/fixtures/jpeg/baseline_420_restart_32x16.jpg b/crates/j2k-jpeg-metal/fixtures/jpeg/baseline_420_restart_32x16.jpg similarity index 100% rename from crates/signinum-jpeg-metal/fixtures/jpeg/baseline_420_restart_32x16.jpg rename to crates/j2k-jpeg-metal/fixtures/jpeg/baseline_420_restart_32x16.jpg diff --git a/crates/signinum-jpeg/fixtures/conformance/baseline_422_16x8.jpg b/crates/j2k-jpeg-metal/fixtures/jpeg/baseline_422_16x8.jpg similarity index 100% rename from crates/signinum-jpeg/fixtures/conformance/baseline_422_16x8.jpg rename to crates/j2k-jpeg-metal/fixtures/jpeg/baseline_422_16x8.jpg diff --git a/crates/signinum-jpeg/fixtures/conformance/baseline_444_8x8.jpg b/crates/j2k-jpeg-metal/fixtures/jpeg/baseline_444_8x8.jpg similarity index 100% rename from crates/signinum-jpeg/fixtures/conformance/baseline_444_8x8.jpg rename to crates/j2k-jpeg-metal/fixtures/jpeg/baseline_444_8x8.jpg diff --git a/crates/signinum-jpeg-metal/fixtures/jpeg/grayscale_8x8.jpg b/crates/j2k-jpeg-metal/fixtures/jpeg/grayscale_8x8.jpg similarity index 100% rename from crates/signinum-jpeg-metal/fixtures/jpeg/grayscale_8x8.jpg rename to crates/j2k-jpeg-metal/fixtures/jpeg/grayscale_8x8.jpg diff --git a/crates/j2k-jpeg-metal/src/abi.rs b/crates/j2k-jpeg-metal/src/abi.rs new file mode 100644 index 00000000..0bd676d6 --- /dev/null +++ b/crates/j2k-jpeg-metal/src/abi.rs @@ -0,0 +1,488 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(target_os = "macos")] +use j2k_jpeg::adapter::{JpegEntropyCheckpointV1, JpegHuffmanTable as PacketHuffmanTable}; +#[cfg(target_os = "macos")] +use metal::Buffer; + +#[cfg(target_os = "macos")] +pub(crate) const MODE_GRAY: u32 = 0; +#[cfg(target_os = "macos")] +pub(crate) const MODE_YCBCR: u32 = 1; +#[cfg(target_os = "macos")] +pub(crate) const MODE_RGB: u32 = 2; + +#[cfg(target_os = "macos")] +pub(crate) const OUT_GRAY: u32 = 0; +#[cfg(target_os = "macos")] +pub(crate) const OUT_RGB: u32 = 1; +#[cfg(target_os = "macos")] +pub(crate) const OUT_RGBA: u32 = 2; + +#[cfg(target_os = "macos")] +pub(crate) const JPEG_BASELINE_ENCODE_FORMAT_GRAY8: u32 = 0; +#[cfg(target_os = "macos")] +pub(crate) const JPEG_BASELINE_ENCODE_FORMAT_RGB8: u32 = 1; +#[cfg(target_os = "macos")] +pub(crate) const JPEG_BASELINE_ENCODE_STATUS_OK: u32 = 0; +#[cfg(target_os = "macos")] +pub(crate) const JPEG_BASELINE_ENCODE_STATUS_OVERFLOW: u32 = 1; +#[cfg(target_os = "macos")] +pub(crate) const JPEG_BASELINE_ENCODE_STATUS_MISSING_HUFFMAN: u32 = 2; +#[cfg(target_os = "macos")] +pub(crate) const JPEG_BASELINE_ENCODE_STATUS_INVALID_PARAMS: u32 = 3; + +#[cfg(target_os = "macos")] +pub(crate) const FAST420_STATUS_OK: u32 = 0; +#[cfg(target_os = "macos")] +pub(crate) const FAST420_STATUS_TRUNCATED: u32 = 1; +#[cfg(target_os = "macos")] +pub(crate) const FAST420_STATUS_HUFFMAN: u32 = 2; + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct JpegPackParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) out_stride: u32, + pub(crate) alpha: u32, + pub(crate) mode: u32, + pub(crate) out_format: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct JpegBaselineEncodeParams { + pub(crate) input_offset_bytes: u32, + pub(crate) input_width: u32, + pub(crate) input_height: u32, + pub(crate) output_width: u32, + pub(crate) output_height: u32, + pub(crate) pitch_bytes: u32, + pub(crate) mcus_per_row: u32, + pub(crate) mcu_rows: u32, + pub(crate) restart_interval_mcus: u32, + pub(crate) format: u32, + pub(crate) components: u32, + pub(crate) max_h: u32, + pub(crate) max_v: u32, + pub(crate) h0: u32, + pub(crate) v0: u32, + pub(crate) h1: u32, + pub(crate) v1: u32, + pub(crate) h2: u32, + pub(crate) v2: u32, + pub(crate) entropy_offset_bytes: u32, + pub(crate) entropy_capacity: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct JpegBaselineEncodeHuffmanTable { + pub(crate) codes: [u16; 256], + pub(crate) lens: [u8; 256], +} + +#[cfg(target_os = "macos")] +impl Default for JpegBaselineEncodeHuffmanTable { + fn default() -> Self { + Self { + codes: [0; 256], + lens: [0; 256], + } + } +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct JpegBaselineEncodeStatus { + pub(crate) code: u32, + pub(crate) entropy_len: u32, + pub(crate) detail: u32, + pub(crate) reserved: u32, +} + +#[cfg(target_os = "macos")] +pub(crate) struct JpegBaselineEntropyEncodeJob<'a> { + pub(crate) input: &'a Buffer, + pub(crate) input_offset: usize, + pub(crate) params: JpegBaselineEncodeParams, + pub(crate) q_luma: [u8; 64], + pub(crate) q_chroma: [u8; 64], + pub(crate) huff_dc_luma: JpegBaselineEncodeHuffmanTable, + pub(crate) huff_ac_luma: JpegBaselineEncodeHuffmanTable, + pub(crate) huff_dc_chroma: JpegBaselineEncodeHuffmanTable, + pub(crate) huff_ac_chroma: JpegBaselineEncodeHuffmanTable, + pub(crate) entropy_capacity: usize, +} + +#[cfg(target_os = "macos")] +pub(crate) struct JpegBaselineEntropyEncodeBatchJob<'a> { + pub(crate) input: &'a Buffer, + pub(crate) params: Vec, + pub(crate) q_luma: [u8; 64], + pub(crate) q_chroma: [u8; 64], + pub(crate) huff_dc_luma: JpegBaselineEncodeHuffmanTable, + pub(crate) huff_ac_luma: JpegBaselineEncodeHuffmanTable, + pub(crate) huff_dc_chroma: JpegBaselineEncodeHuffmanTable, + pub(crate) huff_ac_chroma: JpegBaselineEncodeHuffmanTable, + pub(crate) entropy_capacity: usize, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct JpegFast420Params { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) chroma_width: u32, + pub(crate) chroma_height: u32, + pub(crate) mcus_per_row: u32, + pub(crate) mcu_rows: u32, + pub(crate) restart_interval_mcus: u32, + pub(crate) restart_offset_count: u32, + pub(crate) restart_start_mcu: u32, + pub(crate) entropy_len: u32, + pub(crate) out_stride: u32, + pub(crate) alpha: u32, + pub(crate) out_format: u32, + pub(crate) origin_x: u32, + pub(crate) origin_y: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct JpegFast420ScaledParams { + pub(crate) scaled_width: u32, + pub(crate) scaled_height: u32, + pub(crate) chroma_width: u32, + pub(crate) chroma_height: u32, + pub(crate) mcus_per_row: u32, + pub(crate) mcu_rows: u32, + pub(crate) restart_interval_mcus: u32, + pub(crate) restart_offset_count: u32, + pub(crate) restart_start_mcu: u32, + pub(crate) entropy_len: u32, + pub(crate) scale_shift: u32, + pub(crate) origin_x: u32, + pub(crate) origin_y: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct JpegFast444Params { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) mcus_per_row: u32, + pub(crate) mcu_rows: u32, + pub(crate) restart_interval_mcus: u32, + pub(crate) restart_offset_count: u32, + pub(crate) restart_start_mcu: u32, + pub(crate) entropy_len: u32, + pub(crate) origin_x: u32, + pub(crate) origin_y: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct JpegFast444ScaledParams { + pub(crate) scaled_width: u32, + pub(crate) scaled_height: u32, + pub(crate) mcus_per_row: u32, + pub(crate) mcu_rows: u32, + pub(crate) restart_interval_mcus: u32, + pub(crate) restart_offset_count: u32, + pub(crate) restart_start_mcu: u32, + pub(crate) entropy_len: u32, + pub(crate) scale_shift: u32, + pub(crate) origin_x: u32, + pub(crate) origin_y: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct JpegFast420WindowedPackParams { + pub(crate) src_width: u32, + pub(crate) src_height: u32, + pub(crate) chroma_width: u32, + pub(crate) chroma_height: u32, + pub(crate) src_x: u32, + pub(crate) src_y: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) out_stride: u32, + pub(crate) alpha: u32, + pub(crate) out_format: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct JpegFast420BatchParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) chroma_width: u32, + pub(crate) chroma_height: u32, + pub(crate) mcus_per_row: u32, + pub(crate) mcu_rows: u32, + pub(crate) segment_count: u32, + pub(crate) tile_count: u32, + pub(crate) out_stride: u32, + pub(crate) alpha: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct JpegFastRegionScaledBatchParams { + pub(crate) scaled_width: u32, + pub(crate) scaled_height: u32, + pub(crate) chroma_width: u32, + pub(crate) chroma_height: u32, + pub(crate) mcus_per_row: u32, + pub(crate) mcu_rows: u32, + pub(crate) segment_count: u32, + pub(crate) tile_count: u32, + pub(crate) scale_shift: u32, + pub(crate) origin_x: u32, + pub(crate) origin_y: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct JpegFast444TextureBatchParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) mcus_per_row: u32, + pub(crate) mcu_rows: u32, + pub(crate) segment_count: u32, + pub(crate) tile_index: u32, + pub(crate) alpha: u32, + pub(crate) mode: u32, +} + +#[cfg(target_os = "macos")] +pub(crate) const FAST422_TEXTURE_BOUNDARY_META_WORDS: usize = 4; +#[cfg(target_os = "macos")] +pub(crate) const FAST422_TEXTURE_BOUNDARY_SAMPLE_BYTES: usize = 48; +#[cfg(target_os = "macos")] +pub(crate) const FAST420_TEXTURE_BOUNDARY_META_WORDS: usize = 4; +#[cfg(target_os = "macos")] +pub(crate) const FAST420_TEXTURE_BOUNDARY_SAMPLE_BYTES: usize = 64; +#[cfg(target_os = "macos")] +pub(crate) const FAST420_TEXTURE_VERTICAL_META_WORDS: usize = 4; +#[cfg(target_os = "macos")] +pub(crate) const FAST420_TEXTURE_VERTICAL_SAMPLE_BYTES: usize = 64; + +/// Direct-to-texture decode params shared by the 4:2:0 and 4:2:2 texture +/// kernels (identical layout; both kernel families read the same fields). +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct JpegFast420TextureBatchParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) chroma_width: u32, + pub(crate) chroma_height: u32, + pub(crate) mcus_per_row: u32, + pub(crate) mcu_rows: u32, + pub(crate) segment_count: u32, + pub(crate) tile_index: u32, + pub(crate) alpha: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) struct JpegWindowedPackBatchParams { + pub(crate) src_width: u32, + pub(crate) src_height: u32, + pub(crate) chroma_width: u32, + pub(crate) chroma_height: u32, + pub(crate) src_x: u32, + pub(crate) src_y: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) tile_count: u32, + pub(crate) out_stride: u32, + pub(crate) alpha: u32, + pub(crate) mode: u32, + pub(crate) out_format: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct JpegWindowedTexturePackBatchParams { + pub(crate) src_width: u32, + pub(crate) src_height: u32, + pub(crate) chroma_width: u32, + pub(crate) chroma_height: u32, + pub(crate) src_x: u32, + pub(crate) src_y: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) tile_index: u32, + pub(crate) alpha: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct JpegTexturePackBatchParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) chroma_width: u32, + pub(crate) chroma_height: u32, + pub(crate) tile_index: u32, + pub(crate) alpha: u32, + pub(crate) mode: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct JpegRgb8ToRgbaTextureParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) in_stride: u32, + pub(crate) alpha: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct PreparedHuffmanHost { + pub(crate) min_code: [i32; 17], + pub(crate) max_code: [i32; 17], + pub(crate) val_offset: [i32; 17], + pub(crate) values: [u8; 256], + pub(crate) fast_symbol: [u8; 512], + pub(crate) fast_len: [u8; 512], + pub(crate) values_len: u16, + pub(crate) reserved: u16, +} + +#[cfg(target_os = "macos")] +impl From<&PacketHuffmanTable> for PreparedHuffmanHost { + fn from(value: &PacketHuffmanTable) -> Self { + let mut min_code = [i32::MAX; 17]; + let mut max_code = [-1i32; 17]; + let mut val_offset = [0i32; 17]; + let mut values = [0u8; 256]; + let mut fast_symbol = [0u8; 512]; + let mut fast_len = [0u8; 512]; + let values_len = usize::from(value.values_len); + values[..values_len].copy_from_slice(&value.values[..values_len]); + + let mut huffsize = [0u8; 256]; + let mut huffsize_len = 0usize; + for (len_minus_1, &count) in value.bits.iter().enumerate() { + let len = u8::try_from(len_minus_1 + 1).expect("JPEG Huffman code length fits in u8"); + for _ in 0..count { + huffsize[huffsize_len] = len; + huffsize_len += 1; + } + } + + let mut huffcode = [0u16; 256]; + let mut code = 0u32; + let mut si = huffsize.first().copied().unwrap_or(0); + for (idx, &size) in huffsize[..huffsize_len].iter().enumerate() { + while size != si { + code <<= 1; + si += 1; + } + huffcode[idx] = u16::try_from(code).expect("JPEG Huffman code fits in u16"); + code += 1; + } + + let mut idx = 0usize; + for (len_minus_1, &count) in value.bits.iter().enumerate() { + let len = len_minus_1 + 1; + let count = usize::from(count); + if count == 0 { + continue; + } + min_code[len] = i32::from(huffcode[idx]); + max_code[len] = i32::from(huffcode[idx + count - 1]); + val_offset[len] = + i32::try_from(idx).expect("JPEG Huffman value index fits in i32") - min_code[len]; + idx += count; + } + + for idx in 0..huffsize_len { + let len = usize::from(huffsize[idx]); + if len == 0 || len > 9 { + continue; + } + let code = usize::from(huffcode[idx]); + let prefix = code << (9 - len); + let fill = 1usize << (9 - len); + for suffix in 0..fill { + fast_symbol[prefix | suffix] = values[idx]; + fast_len[prefix | suffix] = huffsize[idx]; + } + } + + Self { + min_code, + max_code, + val_offset, + values, + fast_symbol, + fast_len, + values_len: value.values_len, + reserved: 0, + } + } +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct JpegDecodeStatus { + pub(crate) code: u32, + pub(crate) detail: u32, + pub(crate) position: u32, + pub(crate) reserved: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct JpegEntropyCheckpointHost { + pub(crate) mcu_index: u32, + pub(crate) entropy_pos: u32, + pub(crate) bit_acc: u64, + pub(crate) bit_count: u32, + pub(crate) y_prev_dc: i32, + pub(crate) cb_prev_dc: i32, + pub(crate) cr_prev_dc: i32, + pub(crate) reserved: u32, +} + +#[cfg(target_os = "macos")] +impl From for JpegEntropyCheckpointHost { + fn from(value: JpegEntropyCheckpointV1) -> Self { + Self { + mcu_index: value.mcu_index, + entropy_pos: value.entropy_pos, + bit_acc: value.bit_acc, + bit_count: value.bit_count, + y_prev_dc: value.y_prev_dc, + cb_prev_dc: value.cb_prev_dc, + cr_prev_dc: value.cr_prev_dc, + reserved: value.reserved, + } + } +} diff --git a/crates/signinum-jpeg-metal/src/batch.rs b/crates/j2k-jpeg-metal/src/batch.rs similarity index 88% rename from crates/signinum-jpeg-metal/src/batch.rs rename to crates/j2k-jpeg-metal/src/batch.rs index d96f0225..7a40ce8a 100644 --- a/crates/signinum-jpeg-metal/src/batch.rs +++ b/crates/j2k-jpeg-metal/src/batch.rs @@ -3,10 +3,8 @@ use std::collections::HashMap; use std::sync::Arc; -use signinum_core::{BackendRequest, DeviceSubmission, Downscale, PixelFormat, Rect}; -use signinum_jpeg::adapter::{ - JpegMetalFast420PacketV1, JpegMetalFast422PacketV1, JpegMetalFast444PacketV1, -}; +use j2k_core::{BackendRequest, DeviceSubmission, Downscale, PixelFormat, Rect}; +use j2k_jpeg::adapter::{JpegFast420PacketV1, JpegFast422PacketV1, JpegFast444PacketV1}; use crate::{session::SharedSession, Error, Surface}; @@ -56,9 +54,9 @@ pub(crate) struct QueuedRequest { pub(crate) fmt: PixelFormat, pub(crate) backend: BackendRequest, pub(crate) op: BatchOp, - pub(crate) fast444_packet: Option>, - pub(crate) fast422_packet: Option>, - pub(crate) fast420_packet: Option>, + pub(crate) fast444_packet: Option>, + pub(crate) fast422_packet: Option>, + pub(crate) fast420_packet: Option>, pub(crate) output_slot: usize, } @@ -69,9 +67,9 @@ impl QueuedRequest { fmt: PixelFormat, backend: BackendRequest, op: BatchOp, - fast444_packet: Option, - fast422_packet: Option, - fast420_packet: Option, + fast444_packet: Option, + fast422_packet: Option, + fast420_packet: Option, ) -> Self { Self { input, @@ -90,9 +88,9 @@ impl QueuedRequest { fmt: PixelFormat, backend: BackendRequest, op: BatchOp, - fast444_packet: Option>, - fast422_packet: Option>, - fast420_packet: Option>, + fast444_packet: Option>, + fast422_packet: Option>, + fast420_packet: Option>, ) -> Self { Self { input, @@ -147,7 +145,7 @@ impl DeviceSubmission for MetalSubmission { type Error = Error; fn wait(self) -> Result { - let mut session = self.session.0.lock().expect("metal session"); + let mut session = self.session.lock()?; flush_if_needed(&mut session); take_surface(&mut session, self.slot) } @@ -161,7 +159,7 @@ pub(crate) fn flush_if_needed(session: &mut crate::session::SessionState) { let batches = group_compatible_requests(std::mem::take(&mut session.queued), session); for batch in batches { session.submissions = session.submissions.saturating_add(1); - match crate::decode_compatible_batch(&batch) { + match crate::decode_compatible_batch_with_session(&batch, session) { Ok(Some(results)) => { for (request, result) in batch.into_iter().zip(results) { session.completed[request.output_slot] = Some(result); @@ -195,6 +193,10 @@ fn batched_decode_error(err: &Error) -> Error { Error::MetalUnavailable => Error::MetalUnavailable, Error::UnsupportedBackend { request } => Error::UnsupportedBackend { request: *request }, Error::UnsupportedMetalRequest { reason } => Error::UnsupportedMetalRequest { reason }, + Error::MetalRuntime { message } => Error::MetalRuntime { + message: message.clone(), + }, + Error::MetalStatePoisoned { state } => Error::MetalStatePoisoned { state }, _ => Error::MetalKernel { message: format!("batched JPEG Metal decode failed: {err}"), }, diff --git a/crates/signinum-jpeg-metal/src/bin/viewport_report.rs b/crates/j2k-jpeg-metal/src/bin/viewport_report.rs similarity index 93% rename from crates/signinum-jpeg-metal/src/bin/viewport_report.rs rename to crates/j2k-jpeg-metal/src/bin/viewport_report.rs index f277a750..e4a86ab7 100644 --- a/crates/signinum-jpeg-metal/src/bin/viewport_report.rs +++ b/crates/j2k-jpeg-metal/src/bin/viewport_report.rs @@ -4,9 +4,9 @@ use std::fs; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; -use signinum_core::BackendRequest; -use signinum_jpeg::{ColorSpace, Decoder as CpuDecoder, ScratchPool}; -use signinum_jpeg_metal::viewport::{decode_viewport_to_surface, suggest_viewport_workload}; +use j2k_core::BackendRequest; +use j2k_jpeg::{ColorSpace, Decoder as CpuDecoder, ScratchPool}; +use j2k_jpeg_metal::viewport::{decode_viewport_to_surface, suggest_viewport_workload}; const FULL_FRAME_MAX_OUTPUT_BYTES: usize = 512 * 1024 * 1024; const DEFAULT_WARMUP_ITERS: usize = 3; @@ -56,7 +56,7 @@ fn main() { if inputs.is_empty() { eprintln!( - "viewport_report: no eligible JPEG inputs found; set SIGNINUM_BENCH_INPUTS to extracted JPEG tiles or levels" + "viewport_report: no eligible JPEG inputs found; set J2K_BENCH_INPUTS to extracted JPEG tiles or levels" ); std::process::exit(2); } @@ -150,9 +150,7 @@ fn load_bench_inputs() -> Vec { .iter() .map(|input| input.name.clone()) .collect::>(); - for path in - std::env::split_paths(&std::env::var_os("SIGNINUM_BENCH_INPUTS").unwrap_or_default()) - { + for path in std::env::split_paths(&std::env::var_os("J2K_BENCH_INPUTS").unwrap_or_default()) { collect_jpegs(&path, &mut inputs, &mut seen); } inputs.sort_by(|a, b| a.name.cmp(&b.name)); @@ -245,11 +243,7 @@ fn classify_corpus_input(dimensions: (u32, u32), mode: DecodeMode) -> CorpusInpu }; let bytes = usize::try_from(dimensions.0) .ok() - .and_then(|width| { - usize::try_from(dimensions.1) - .ok() - .map(|height| (width, height)) - }) + .zip(usize::try_from(dimensions.1).ok()) .and_then(|(width, height)| width.checked_mul(height)) .and_then(|pixels| pixels.checked_mul(bpp)); match bytes { diff --git a/crates/j2k-jpeg-metal/src/buffers.rs b/crates/j2k-jpeg-metal/src/buffers.rs new file mode 100644 index 00000000..99f9ab4b --- /dev/null +++ b/crates/j2k-jpeg-metal/src/buffers.rs @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(test)] +use std::cell::Cell; +use std::mem::size_of_val; + +use j2k_metal_support::{private_buffer, shared_buffer, shared_buffer_with_bytes}; +use metal::{Buffer, Device}; + +#[cfg(test)] +std::thread_local! { + static JPEG_PRIVATE_BUFFER_ALLOCATIONS: Cell = const { Cell::new(0) }; + static JPEG_SHARED_BUFFER_ALLOCATIONS: Cell = const { Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_jpeg_private_buffer_allocations_for_test() { + JPEG_PRIVATE_BUFFER_ALLOCATIONS.with(|allocations| allocations.set(0)); +} + +#[cfg(test)] +pub(crate) fn reset_jpeg_shared_buffer_allocations_for_test() { + JPEG_SHARED_BUFFER_ALLOCATIONS.with(|allocations| allocations.set(0)); +} + +#[cfg(test)] +pub(crate) fn jpeg_private_buffer_allocations_for_test() -> usize { + JPEG_PRIVATE_BUFFER_ALLOCATIONS.with(Cell::get) +} + +#[cfg(test)] +pub(crate) fn jpeg_shared_buffer_allocations_for_test() -> usize { + JPEG_SHARED_BUFFER_ALLOCATIONS.with(Cell::get) +} + +pub(crate) fn new_shared_buffer(device: &Device, bytes: usize) -> Buffer { + #[cfg(test)] + JPEG_SHARED_BUFFER_ALLOCATIONS.with(|allocations| allocations.set(allocations.get() + 1)); + shared_buffer(device, bytes) +} + +pub(crate) fn new_shared_buffer_with_data(device: &Device, bytes: &[u8]) -> Buffer { + #[cfg(test)] + JPEG_SHARED_BUFFER_ALLOCATIONS.with(|allocations| allocations.set(allocations.get() + 1)); + shared_buffer_with_bytes(device, bytes) +} + +pub(crate) fn new_private_buffer(device: &Device, bytes: usize) -> Buffer { + #[cfg(test)] + JPEG_PRIVATE_BUFFER_ALLOCATIONS.with(|allocations| allocations.set(allocations.get() + 1)); + private_buffer(device, bytes) +} + +pub(crate) fn new_decode_plane_buffer( + device: &Device, + bytes: usize, + returned_publicly: bool, +) -> Buffer { + if returned_publicly { + new_shared_buffer(device, bytes) + } else { + new_private_buffer(device, bytes) + } +} + +struct ReusablePrivateBuffer { + key: &'static str, + capacity: usize, + buffer: Buffer, +} + +struct ReusableSharedBuffer { + key: &'static str, + capacity: usize, + buffer: Buffer, +} + +#[derive(Default)] +pub(crate) struct MetalBatchScratch { + private_buffers: Vec, + shared_buffers: Vec, +} + +impl MetalBatchScratch { + pub(crate) fn private_buffer( + &mut self, + device: &Device, + key: &'static str, + bytes: usize, + ) -> Buffer { + let bytes = bytes.max(1); + if let Some(entry) = self + .private_buffers + .iter() + .find(|entry| entry.key == key && entry.capacity >= bytes) + { + return entry.buffer.clone(); + } + + let buffer = new_private_buffer(device, bytes); + if let Some(entry) = self + .private_buffers + .iter_mut() + .find(|entry| entry.key == key) + { + entry.capacity = bytes; + entry.buffer = buffer.clone(); + } else { + self.private_buffers.push(ReusablePrivateBuffer { + key, + capacity: bytes, + buffer: buffer.clone(), + }); + } + buffer + } + + pub(crate) fn shared_buffer_with_bytes( + &mut self, + device: &Device, + key: &'static str, + bytes: &[u8], + ) -> Buffer { + let capacity = bytes.len().max(1); + let buffer = if let Some(entry) = self + .shared_buffers + .iter() + .find(|entry| entry.key == key && entry.capacity >= capacity) + { + entry.buffer.clone() + } else { + let buffer = new_shared_buffer(device, capacity); + if let Some(entry) = self + .shared_buffers + .iter_mut() + .find(|entry| entry.key == key) + { + entry.capacity = capacity; + entry.buffer = buffer.clone(); + } else { + self.shared_buffers.push(ReusableSharedBuffer { + key, + capacity, + buffer: buffer.clone(), + }); + } + buffer + }; + + if !bytes.is_empty() { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + unsafe { + core::ptr::copy_nonoverlapping( + bytes.as_ptr(), + buffer.contents().cast::(), + bytes.len(), + ); + } + } + buffer + } + + pub(crate) fn shared_buffer_with_slice( + &mut self, + device: &Device, + key: &'static str, + values: &[T], + ) -> Buffer { + // SAFETY: The immutable slice is reinterpreted as its initialized byte range. + let bytes = unsafe { + core::slice::from_raw_parts(values.as_ptr().cast::(), size_of_val(values)) + }; + self.shared_buffer_with_bytes(device, key, bytes) + } +} diff --git a/crates/j2k-jpeg-metal/src/compute.rs b/crates/j2k-jpeg-metal/src/compute.rs new file mode 100644 index 00000000..75e18cdf --- /dev/null +++ b/crates/j2k-jpeg-metal/src/compute.rs @@ -0,0 +1,8501 @@ +// SPDX-License-Identifier: Apache-2.0 + +#![allow(clippy::similar_names)] + +#[cfg(all(target_os = "macos", test))] +use j2k_metal_support::system_default_device; +#[cfg(all(target_os = "macos", test))] +use metal::foreign_types::ForeignType; +#[cfg(target_os = "macos")] +use std::{ + cell::RefCell, + mem::{size_of, size_of_val}, + sync::{Mutex, MutexGuard}, + time::Instant, +}; + +#[cfg(test)] +use j2k_core::BackendRequest; +use j2k_core::{BufferError, PixelFormat, Rect}; +use j2k_jpeg::{ + adapter::{ + JpegEntropyCheckpointV1, JpegFast420PacketV1, JpegFast422PacketV1, JpegFast444PacketV1, + JpegHuffmanTable as PacketHuffmanTable, + }, + ColorSpace as JpegColorSpace, Decoder as CpuDecoder, +}; +#[cfg(target_os = "macos")] +use j2k_metal_support::{checked_command_queue, MetalPipelineLoader, MetalSupportError}; +#[cfg(target_os = "macos")] +use metal::{ + Buffer, CommandBuffer, CommandBufferRef, CommandQueue, ComputePipelineState, Device, + MTLPixelFormat, MTLResourceOptions, MTLSize, +}; + +#[cfg(target_os = "macos")] +pub(crate) use crate::abi::*; + +#[cfg(target_os = "macos")] +use crate::buffers::{ + new_decode_plane_buffer, new_private_buffer, new_shared_buffer_with_data, MetalBatchScratch, +}; +use crate::{batch, Error, Surface}; + +mod batch_plan; +#[cfg(target_os = "macos")] +mod batch_support; +#[cfg(target_os = "macos")] +mod kernel_helpers; +#[cfg(target_os = "macos")] +mod region_scaled_plan; +#[cfg(target_os = "macos")] +mod status; +mod viewport_cache; +#[cfg(target_os = "macos")] +mod viewport_compose; +use self::batch_plan::{ + batched_fast_packets, core_rect_to_jpeg, BatchDeviceBufferCache, BatchedDecodeItem, + BatchedFastPacket, +}; +#[cfg(all(test, target_os = "macos"))] +use self::batch_support::fast420_batch_timing_value_enabled; +#[cfg(target_os = "macos")] +use self::batch_support::{ + batch_entropy_buffers, fast420_batch_timing_enabled, fast_batch_decode_mode, + region_scaled_batch_error_results, texture_batch_error_results, BatchEntropyBufferKeys, + FastBatchDecodeMode, FastBatchTiming, +}; +#[cfg(all(test, target_os = "macos"))] +use self::kernel_helpers::choose_1d_threadgroup_width; +#[cfg(target_os = "macos")] +use self::kernel_helpers::{ + bind_fast_decode_entropy_inputs, bind_three_plane_pack, dispatch_1d_pipeline, + dispatch_2d_pipeline, dispatch_3d_pipeline, packed_pair_extent, pixel_format_to_out_format, + plane_mode_to_u32, +}; +#[cfg(target_os = "macos")] +use self::region_scaled_plan::{ + fast444_full_rgb_batch_groups, fast444_packets_share_region_scaled_batch_shape, + fast444_region_scaled_batch_groups, fast_subsampled_full_rgb_batch_groups, + fast_subsampled_packets_share_full_rgb_batch_shape, fast_subsampled_region_scaled_batch_groups, + fast_subsampled_region_scaled_batch_plan, windowed_texture_pack_params, +}; +#[cfg(target_os = "macos")] +use self::status::{ + decode_error_from_cpu, decode_status_buffer, fast422_status_error, first_decode_error_status, + jpeg_baseline_encode_status_error, +}; +use self::viewport_cache::{cached_plane_stage, CachedViewportPlanes, PlaneMode, PlaneStage}; +#[cfg(target_os = "macos")] +pub(crate) use self::viewport_compose::{ + compose_rgb_viewport_from_regions, compose_rgb_viewport_from_regions_into_output_with_session, + compose_rgb_viewport_from_regions_into_textures_with_session, +}; + +#[cfg(all(target_os = "macos", test))] +pub(crate) use crate::buffers::{ + jpeg_private_buffer_allocations_for_test, jpeg_shared_buffer_allocations_for_test, + reset_jpeg_private_buffer_allocations_for_test, reset_jpeg_shared_buffer_allocations_for_test, +}; + +#[cfg(target_os = "macos")] +const SHADER_SOURCE: &str = include_str!("shaders.metal"); + +#[cfg(target_os = "macos")] +const REGION_SCALED_BATCH_CHUNK: usize = 8; + +#[cfg(target_os = "macos")] +struct FastRgbDecodeBuffer { + buffer: Buffer, + dimensions: (u32, u32), + status_buffer: Buffer, + command_buffer: CommandBuffer, +} + +#[cfg(target_os = "macos")] +fn private_jpeg_tile_from_fast_rgb_buffer( + decoded: FastRgbDecodeBuffer, +) -> crate::ResidentPrivateJpegTile { + crate::ResidentPrivateJpegTile { + buffer: decoded.buffer, + byte_offset: 0, + dimensions: decoded.dimensions, + pixel_format: PixelFormat::Rgb8, + pitch_bytes: decoded.dimensions.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(), + status_buffer: decoded.status_buffer, + command_buffer: decoded.command_buffer, + } +} + +#[cfg(target_os = "macos")] +thread_local! { + static DEFAULT_METAL_SESSION: RefCell>> = const { RefCell::new(None) }; +} + +#[cfg(target_os = "macos")] +pub(crate) struct MetalRuntime { + device: Device, + queue: CommandQueue, + pack_pipeline: ComputePipelineState, + jpeg_baseline_encode_pipeline: ComputePipelineState, + jpeg_baseline_encode_batch_pipeline: ComputePipelineState, + pack_420_pipeline: ComputePipelineState, + pack_420_rgb_pipeline: ComputePipelineState, + pack_420_rgba_pipeline: ComputePipelineState, + pack_420_rgb_batch_pipeline: ComputePipelineState, + pack_420_rgba_texture_pipeline: ComputePipelineState, + pack_420_windowed_rgb_batch_pipeline: ComputePipelineState, + pack_420_windowed_rgba_texture_pipeline: ComputePipelineState, + pack_422_rgb_pipeline: ComputePipelineState, + pack_422_rgba_pipeline: ComputePipelineState, + pack_422_rgb_batch_pipeline: ComputePipelineState, + pack_422_rgba_texture_pipeline: ComputePipelineState, + pack_422_windowed_rgb_batch_pipeline: ComputePipelineState, + pack_422_windowed_rgba_texture_pipeline: ComputePipelineState, + pack_444_rgb_batch_pipeline: ComputePipelineState, + pack_444_rgba_texture_pipeline: ComputePipelineState, + pack_422_windowed_pipeline: ComputePipelineState, + pack_422_windowed_rgb_pipeline: ComputePipelineState, + pack_422_windowed_rgba_pipeline: ComputePipelineState, + pack_420_windowed_pipeline: ComputePipelineState, + pack_420_windowed_rgb_pipeline: ComputePipelineState, + pack_420_windowed_rgba_pipeline: ComputePipelineState, + fast420_decode_pipeline: ComputePipelineState, + fast420_batch_decode_pipeline: ComputePipelineState, + #[cfg(test)] + fast420_batch_coeffs_decode_pipeline: ComputePipelineState, + #[cfg(test)] + fast420_batch_idct_deposit_pipeline: ComputePipelineState, + fast420_scaled_region_batch_decode_pipeline: ComputePipelineState, + fast420_rgba_texture_batch_decode_pipeline: ComputePipelineState, + fast420_rgba_texture_boundary_pipeline: ComputePipelineState, + fast420_rgba_texture_vertical_boundary_pipeline: ComputePipelineState, + fast420_rgba_texture_corner_pipeline: ComputePipelineState, + fast422_decode_pipeline: ComputePipelineState, + fast422_batch_decode_pipeline: ComputePipelineState, + fast422_scaled_region_batch_decode_pipeline: ComputePipelineState, + fast422_rgba_texture_batch_decode_pipeline: ComputePipelineState, + fast422_rgba_texture_boundary_pipeline: ComputePipelineState, + fast422_region_decode_pipeline: ComputePipelineState, + fast422_scaled_decode_pipeline: ComputePipelineState, + fast422_scaled_region_decode_pipeline: ComputePipelineState, + fast420_region_decode_pipeline: ComputePipelineState, + fast420_scaled_decode_pipeline: ComputePipelineState, + fast420_scaled_region_decode_pipeline: ComputePipelineState, + fast444_decode_pipeline: ComputePipelineState, + fast444_region_decode_pipeline: ComputePipelineState, + fast444_scaled_decode_pipeline: ComputePipelineState, + fast444_scaled_region_decode_pipeline: ComputePipelineState, + fast444_scaled_region_batch_decode_pipeline: ComputePipelineState, + fast444_rgba_texture_batch_decode_pipeline: ComputePipelineState, + rgb8_to_rgba_texture_pipeline: ComputePipelineState, + batch_scratch: Mutex, + viewport_plane_cache: Mutex>, +} + +#[cfg(target_os = "macos")] +impl MetalRuntime { + #[cfg(test)] + fn new() -> Result { + let device = system_default_device()?; + Self::new_with_device(device) + } + + pub(crate) fn new_with_device(device: Device) -> Result { + let loader = MetalPipelineLoader::new(&device, SHADER_SOURCE)?; + let pipeline = |name: &str| loader.pipeline(name); + let queue = checked_command_queue(&device)?; + Ok(Self { + device, + queue, + pack_pipeline: pipeline("jpeg_pack")?, + jpeg_baseline_encode_pipeline: pipeline("jpeg_encode_baseline_entropy")?, + jpeg_baseline_encode_batch_pipeline: pipeline("jpeg_encode_baseline_entropy_batch")?, + pack_420_pipeline: pipeline("jpeg_pack_420")?, + pack_420_rgb_pipeline: pipeline("jpeg_pack_420_rgb")?, + pack_420_rgba_pipeline: pipeline("jpeg_pack_420_rgba")?, + pack_420_rgb_batch_pipeline: pipeline("jpeg_pack_420_rgb_batch")?, + pack_420_rgba_texture_pipeline: pipeline("jpeg_pack_420_rgba_texture")?, + pack_420_windowed_rgb_batch_pipeline: pipeline("jpeg_pack_420_windowed_rgb_batch")?, + pack_420_windowed_rgba_texture_pipeline: pipeline( + "jpeg_pack_420_windowed_rgba_texture", + )?, + pack_422_rgb_pipeline: pipeline("jpeg_pack_422_rgb")?, + pack_422_rgba_pipeline: pipeline("jpeg_pack_422_rgba")?, + pack_422_rgb_batch_pipeline: pipeline("jpeg_pack_422_rgb_batch")?, + pack_422_rgba_texture_pipeline: pipeline("jpeg_pack_422_rgba_texture")?, + pack_422_windowed_rgb_batch_pipeline: pipeline("jpeg_pack_422_windowed_rgb_batch")?, + pack_422_windowed_rgba_texture_pipeline: pipeline( + "jpeg_pack_422_windowed_rgba_texture", + )?, + pack_444_rgb_batch_pipeline: pipeline("jpeg_pack_444_rgb_batch")?, + pack_444_rgba_texture_pipeline: pipeline("jpeg_pack_444_rgba_texture")?, + pack_422_windowed_pipeline: pipeline("jpeg_pack_422_windowed")?, + pack_422_windowed_rgb_pipeline: pipeline("jpeg_pack_422_windowed_rgb")?, + pack_422_windowed_rgba_pipeline: pipeline("jpeg_pack_422_windowed_rgba")?, + pack_420_windowed_pipeline: pipeline("jpeg_pack_420_windowed")?, + pack_420_windowed_rgb_pipeline: pipeline("jpeg_pack_420_windowed_rgb")?, + pack_420_windowed_rgba_pipeline: pipeline("jpeg_pack_420_windowed_rgba")?, + fast420_decode_pipeline: pipeline("jpeg_decode_fast420")?, + fast420_batch_decode_pipeline: pipeline("jpeg_decode_fast420_batch")?, + #[cfg(test)] + fast420_batch_coeffs_decode_pipeline: pipeline("jpeg_decode_fast420_batch_coeffs")?, + #[cfg(test)] + fast420_batch_idct_deposit_pipeline: pipeline("jpeg_idct_deposit_fast420_batch")?, + fast420_scaled_region_batch_decode_pipeline: pipeline( + "jpeg_decode_fast420_scaled_region_batch", + )?, + fast420_rgba_texture_batch_decode_pipeline: pipeline( + "jpeg_decode_fast420_rgba_texture_batch", + )?, + fast420_rgba_texture_boundary_pipeline: pipeline( + "jpeg_resolve_fast420_rgba_texture_boundaries", + )?, + fast420_rgba_texture_vertical_boundary_pipeline: pipeline( + "jpeg_resolve_fast420_rgba_texture_vertical_boundaries", + )?, + fast420_rgba_texture_corner_pipeline: pipeline( + "jpeg_resolve_fast420_rgba_texture_corners", + )?, + fast422_decode_pipeline: pipeline("jpeg_decode_fast422")?, + fast422_batch_decode_pipeline: pipeline("jpeg_decode_fast422_batch")?, + fast422_scaled_region_batch_decode_pipeline: pipeline( + "jpeg_decode_fast422_scaled_region_batch", + )?, + fast422_rgba_texture_batch_decode_pipeline: pipeline( + "jpeg_decode_fast422_rgba_texture_batch", + )?, + fast422_rgba_texture_boundary_pipeline: pipeline( + "jpeg_resolve_fast422_rgba_texture_boundaries", + )?, + fast422_region_decode_pipeline: pipeline("jpeg_decode_fast422_region")?, + fast422_scaled_decode_pipeline: pipeline("jpeg_decode_fast422_scaled")?, + fast422_scaled_region_decode_pipeline: pipeline("jpeg_decode_fast422_scaled_region")?, + fast420_region_decode_pipeline: pipeline("jpeg_decode_fast420_region")?, + fast420_scaled_decode_pipeline: pipeline("jpeg_decode_fast420_scaled")?, + fast420_scaled_region_decode_pipeline: pipeline("jpeg_decode_fast420_scaled_region")?, + fast444_decode_pipeline: pipeline("jpeg_decode_fast444")?, + fast444_region_decode_pipeline: pipeline("jpeg_decode_fast444_region")?, + fast444_scaled_decode_pipeline: pipeline("jpeg_decode_fast444_scaled")?, + fast444_scaled_region_decode_pipeline: pipeline("jpeg_decode_fast444_scaled_region")?, + fast444_scaled_region_batch_decode_pipeline: pipeline( + "jpeg_decode_fast444_scaled_region_batch", + )?, + fast444_rgba_texture_batch_decode_pipeline: pipeline( + "jpeg_decode_fast444_rgba_texture_batch", + )?, + rgb8_to_rgba_texture_pipeline: pipeline("jpeg_copy_rgb8_to_rgba_texture")?, + batch_scratch: Mutex::new(MetalBatchScratch::default()), + viewport_plane_cache: Mutex::new(None), + }) + } + + fn batch_scratch(&self) -> Result, Error> { + self.batch_scratch + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "JPEG Metal batch scratch", + }) + } + + fn viewport_plane_cache(&self) -> Result>, Error> { + self.viewport_plane_cache + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "JPEG Metal viewport plane cache", + }) + } + + #[cfg(test)] + fn viewport_plane_cache_id_for_test(&self) -> Result, Error> { + Ok(self + .viewport_plane_cache()? + .as_ref() + .map(|cached| cached.plane0.as_ptr() as usize)) + } +} + +#[cfg(target_os = "macos")] +fn pack_420_pipeline_for_format(runtime: &MetalRuntime, fmt: PixelFormat) -> &ComputePipelineState { + match fmt { + PixelFormat::Rgb8 => &runtime.pack_420_rgb_pipeline, + PixelFormat::Rgba8 => &runtime.pack_420_rgba_pipeline, + _ => &runtime.pack_420_pipeline, + } +} + +#[cfg(target_os = "macos")] +fn pack_420_windowed_pipeline_for_format( + runtime: &MetalRuntime, + fmt: PixelFormat, +) -> &ComputePipelineState { + match fmt { + PixelFormat::Rgb8 => &runtime.pack_420_windowed_rgb_pipeline, + PixelFormat::Rgba8 => &runtime.pack_420_windowed_rgba_pipeline, + _ => &runtime.pack_420_windowed_pipeline, + } +} + +#[cfg(target_os = "macos")] +fn pack_422_pipeline_for_format( + runtime: &MetalRuntime, + fmt: PixelFormat, +) -> Option<&ComputePipelineState> { + match fmt { + PixelFormat::Rgb8 => Some(&runtime.pack_422_rgb_pipeline), + PixelFormat::Rgba8 => Some(&runtime.pack_422_rgba_pipeline), + _ => None, + } +} + +#[cfg(target_os = "macos")] +fn pack_422_windowed_pipeline_for_format( + runtime: &MetalRuntime, + fmt: PixelFormat, +) -> &ComputePipelineState { + match fmt { + PixelFormat::Rgb8 => &runtime.pack_422_windowed_rgb_pipeline, + PixelFormat::Rgba8 => &runtime.pack_422_windowed_rgba_pipeline, + _ => &runtime.pack_422_windowed_pipeline, + } +} + +#[cfg(target_os = "macos")] +fn with_runtime(f: impl FnOnce(&MetalRuntime) -> Result) -> Result { + DEFAULT_METAL_SESSION.with(|session| { + let mut session = session.borrow_mut(); + if session.is_none() { + *session = Some( + j2k_metal_support::system_default_device().map(crate::MetalBackendSession::new), + ); + } + let Some(session) = session.as_ref() else { + return Err(Error::MetalRuntime { + message: "JPEG Metal default session was not initialized".to_string(), + }); + }; + match session { + Ok(session) => with_runtime_for_session(session, f), + Err(error) => Err(runtime_initialization_error(error)), + } + }) +} + +#[cfg(target_os = "macos")] +fn with_runtime_for_session( + session: &crate::MetalBackendSession, + f: impl FnOnce(&MetalRuntime) -> Result, +) -> Result { + let runtime = session + .runtime + .get_or_init(|| MetalRuntime::new_with_device(session.device.clone())); + match runtime { + Ok(runtime) => f(runtime), + Err(error) => Err(runtime_initialization_error(error)), + } +} + +#[cfg(target_os = "macos")] +pub(crate) fn runtime_initialization_error(error: &MetalSupportError) -> Error { + if error.is_unavailable() { + Error::MetalUnavailable + } else { + Error::MetalRuntime { + message: error.to_string(), + } + } +} + +#[cfg(target_os = "macos")] +pub(crate) fn encode_jpeg_baseline_entropy_with_session( + session: &crate::MetalBackendSession, + job: &JpegBaselineEntropyEncodeJob<'_>, +) -> Result, Error> { + with_runtime_for_session(session, |runtime| { + let entropy_buffer = runtime.device.new_buffer( + job.entropy_capacity as u64, + MTLResourceOptions::StorageModeShared, + ); + let status = JpegBaselineEncodeStatus::default(); + let status_buffer = runtime.device.new_buffer_with_data( + (&raw const status).cast(), + size_of::() as u64, + MTLResourceOptions::StorageModeShared, + ); + + let command_buffer = runtime.queue.new_command_buffer(); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.jpeg_baseline_encode_pipeline); + encoder.set_buffer(0, Some(job.input), job.input_offset as u64); + encoder.set_buffer(1, Some(&entropy_buffer), 0); + encoder.set_buffer(2, Some(&status_buffer), 0); + encoder.set_bytes( + 3, + size_of::() as u64, + (&raw const job.params).cast(), + ); + encoder.set_bytes( + 4, + size_of_val(&job.q_luma) as u64, + job.q_luma.as_ptr().cast(), + ); + encoder.set_bytes( + 5, + size_of_val(&job.q_chroma) as u64, + job.q_chroma.as_ptr().cast(), + ); + encoder.set_bytes( + 6, + size_of::() as u64, + (&raw const job.huff_dc_luma).cast(), + ); + encoder.set_bytes( + 7, + size_of::() as u64, + (&raw const job.huff_ac_luma).cast(), + ); + encoder.set_bytes( + 8, + size_of::() as u64, + (&raw const job.huff_dc_chroma).cast(), + ); + encoder.set_bytes( + 9, + size_of::() as u64, + (&raw const job.huff_ac_chroma).cast(), + ); + encoder.dispatch_threads( + MTLSize { + width: 1, + height: 1, + depth: 1, + }, + MTLSize { + width: 1, + height: 1, + depth: 1, + }, + ); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let status = unsafe { *(status_buffer.contents().cast::()) }; + if status.code != JPEG_BASELINE_ENCODE_STATUS_OK { + return Err(jpeg_baseline_encode_status_error(status)); + } + let entropy_len = usize::try_from(status.entropy_len).map_err(|_| Error::MetalKernel { + message: "JPEG Baseline Metal encode entropy length exceeds usize".to_string(), + })?; + if entropy_len > job.entropy_capacity { + return Err(Error::MetalKernel { + message: "JPEG Baseline Metal encode reported length exceeds output capacity" + .to_string(), + }); + } + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let entropy = unsafe { + core::slice::from_raw_parts(entropy_buffer.contents().cast::(), entropy_len) + }; + Ok(entropy.to_vec()) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn encode_jpeg_baseline_entropy_batch_with_session( + session: &crate::MetalBackendSession, + job: &JpegBaselineEntropyEncodeBatchJob<'_>, +) -> Result>, Error> { + if job.params.is_empty() { + return Ok(Vec::new()); + } + with_runtime_for_session(session, |runtime| { + let entropy_buffer = runtime.device.new_buffer( + job.entropy_capacity as u64, + MTLResourceOptions::StorageModeShared, + ); + let statuses = vec![JpegBaselineEncodeStatus::default(); job.params.len()]; + let status_buffer = runtime.device.new_buffer_with_data( + statuses.as_ptr().cast(), + size_of::() as u64 * statuses.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let params_buffer = runtime.device.new_buffer_with_data( + job.params.as_ptr().cast(), + size_of::() as u64 * job.params.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let tile_count = u32::try_from(job.params.len()).map_err(|_| Error::MetalKernel { + message: "JPEG Baseline Metal batch tile count exceeds u32".to_string(), + })?; + + let command_buffer = runtime.queue.new_command_buffer(); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.jpeg_baseline_encode_batch_pipeline); + encoder.set_buffer(0, Some(job.input), 0); + encoder.set_buffer(1, Some(&entropy_buffer), 0); + encoder.set_buffer(2, Some(&status_buffer), 0); + encoder.set_buffer(3, Some(¶ms_buffer), 0); + encoder.set_bytes( + 4, + size_of_val(&job.q_luma) as u64, + job.q_luma.as_ptr().cast(), + ); + encoder.set_bytes( + 5, + size_of_val(&job.q_chroma) as u64, + job.q_chroma.as_ptr().cast(), + ); + encoder.set_bytes( + 6, + size_of::() as u64, + (&raw const job.huff_dc_luma).cast(), + ); + encoder.set_bytes( + 7, + size_of::() as u64, + (&raw const job.huff_ac_luma).cast(), + ); + encoder.set_bytes( + 8, + size_of::() as u64, + (&raw const job.huff_dc_chroma).cast(), + ); + encoder.set_bytes( + 9, + size_of::() as u64, + (&raw const job.huff_ac_chroma).cast(), + ); + encoder.set_bytes(10, size_of::() as u64, (&raw const tile_count).cast()); + encoder.dispatch_threads( + MTLSize { + width: u64::from(tile_count), + height: 1, + depth: 1, + }, + MTLSize { + width: 1, + height: 1, + depth: 1, + }, + ); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let status_slice = unsafe { + core::slice::from_raw_parts( + status_buffer.contents().cast::(), + job.params.len(), + ) + }; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let entropy_bytes = unsafe { + core::slice::from_raw_parts( + entropy_buffer.contents().cast::(), + job.entropy_capacity, + ) + }; + let mut out = Vec::with_capacity(job.params.len()); + for (status, params) in status_slice.iter().copied().zip(job.params.iter()) { + if status.code != JPEG_BASELINE_ENCODE_STATUS_OK { + return Err(jpeg_baseline_encode_status_error(status)); + } + let entropy_len = + usize::try_from(status.entropy_len).map_err(|_| Error::MetalKernel { + message: "JPEG Baseline Metal encode entropy length exceeds usize".to_string(), + })?; + let offset = + usize::try_from(params.entropy_offset_bytes).map_err(|_| Error::MetalKernel { + message: "JPEG Baseline Metal batch entropy offset exceeds usize".to_string(), + })?; + let capacity = + usize::try_from(params.entropy_capacity).map_err(|_| Error::MetalKernel { + message: "JPEG Baseline Metal batch entropy capacity exceeds usize".to_string(), + })?; + if entropy_len > capacity { + return Err(Error::MetalKernel { + message: + "JPEG Baseline Metal encode reported length exceeds tile output capacity" + .to_string(), + }); + } + let end = offset + .checked_add(entropy_len) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Baseline Metal batch entropy range overflow".to_string(), + })?; + if end > entropy_bytes.len() { + return Err(Error::MetalKernel { + message: "JPEG Baseline Metal batch entropy range exceeds buffer".to_string(), + }); + } + out.push(entropy_bytes[offset..end].to_vec()); + } + Ok(out) + }) +} + +#[cfg(target_os = "macos")] +/// Field access over the sampling-family fast packets, which are +/// field-identical; the sampling factor is carried by the packet type. +trait FastPacketAccess { + /// Family name used in diagnostics ("fast420" / "fast422" / "fast444"). + const FAMILY_NAME: &'static str; + + fn dimensions(&self) -> (u32, u32); + fn mcus_per_row(&self) -> u32; + fn mcu_rows(&self) -> u32; + fn restart_interval_mcus(&self) -> u32; + fn restart_offsets(&self) -> &[u32]; + fn entropy_checkpoints(&self) -> &[JpegEntropyCheckpointV1]; + fn entropy_bytes(&self) -> &[u8]; + fn y_quant(&self) -> &[u16; 64]; + fn cb_quant(&self) -> &[u16; 64]; + fn cr_quant(&self) -> &[u16; 64]; + fn y_dc_table(&self) -> &PacketHuffmanTable; + fn y_ac_table(&self) -> &PacketHuffmanTable; + fn cb_dc_table(&self) -> &PacketHuffmanTable; + fn cb_ac_table(&self) -> &PacketHuffmanTable; + fn cr_dc_table(&self) -> &PacketHuffmanTable; + fn cr_ac_table(&self) -> &PacketHuffmanTable; +} + +macro_rules! impl_fast_packet_access { + ($packet:ty, $name:literal) => { + impl FastPacketAccess for $packet { + const FAMILY_NAME: &'static str = $name; + + fn dimensions(&self) -> (u32, u32) { + self.dimensions + } + fn mcus_per_row(&self) -> u32 { + self.mcus_per_row + } + fn mcu_rows(&self) -> u32 { + self.mcu_rows + } + fn restart_interval_mcus(&self) -> u32 { + self.restart_interval_mcus + } + fn restart_offsets(&self) -> &[u32] { + &self.restart_offsets + } + fn entropy_checkpoints(&self) -> &[JpegEntropyCheckpointV1] { + &self.entropy_checkpoints + } + fn entropy_bytes(&self) -> &[u8] { + &self.entropy_bytes + } + fn y_quant(&self) -> &[u16; 64] { + &self.y_quant + } + fn cb_quant(&self) -> &[u16; 64] { + &self.cb_quant + } + fn cr_quant(&self) -> &[u16; 64] { + &self.cr_quant + } + fn y_dc_table(&self) -> &PacketHuffmanTable { + &self.y_dc_table + } + fn y_ac_table(&self) -> &PacketHuffmanTable { + &self.y_ac_table + } + fn cb_dc_table(&self) -> &PacketHuffmanTable { + &self.cb_dc_table + } + fn cb_ac_table(&self) -> &PacketHuffmanTable { + &self.cb_ac_table + } + fn cr_dc_table(&self) -> &PacketHuffmanTable { + &self.cr_dc_table + } + fn cr_ac_table(&self) -> &PacketHuffmanTable { + &self.cr_ac_table + } + } + }; +} + +impl_fast_packet_access!(JpegFast420PacketV1, "fast420"); +impl_fast_packet_access!(JpegFast422PacketV1, "fast422"); +impl_fast_packet_access!(JpegFast444PacketV1, "fast444"); + +/// Chroma geometry for the subsampled families that share the +/// `JpegFast420Params` kernel ABI (4:2:0 halves chroma rows, 4:2:2 keeps +/// them; both halve chroma columns). +trait FastSubsampledPacket: FastPacketAccess { + /// Luma MCU width in pixels. + const MCU_WIDTH: u32; + /// Luma MCU height in pixels (4:2:0 MCUs are 16 rows, 4:2:2 are 8). + const MCU_HEIGHT: u32; + /// Whether the full-RGB batch path may group restart-interval packets. + const FULL_RGB_BATCH_SUPPORTS_RESTART: bool; + const ENTROPY_PAYLOAD_CTX: &'static str; + const REGION_SCALED_BATCH_OUT_STRIDE_CTX: &'static str; + const OUTPUT_STRIDE_CTX: &'static str; + const REGION_OUTPUT_STRIDE_CTX: &'static str; + const SCALED_ENTROPY_PAYLOAD_CTX: &'static str; + /// Blocks per MCU when the full-RGB batch path needs block-count + /// validation for the split coeff/IDCT debug mode (4:2:0 only). + const FULL_RGB_BATCH_BLOCKS_PER_MCU: Option; + + fn chroma_height(height: u32) -> u32; + /// Vertical dispatch extent for the full-frame pack kernels: 4:2:0 packs + /// 2x2 pixel quads per thread, 4:2:2 packs 2x1 pairs (full-height rows). + fn packed_height_extent(height: u32) -> u32; +} + +impl FastSubsampledPacket for JpegFast420PacketV1 { + const MCU_WIDTH: u32 = 16; + const MCU_HEIGHT: u32 = 16; + const FULL_RGB_BATCH_SUPPORTS_RESTART: bool = true; + const ENTROPY_PAYLOAD_CTX: &'static str = "fast420 entropy payload"; + const REGION_SCALED_BATCH_OUT_STRIDE_CTX: &'static str = + "fast420 region scaled batch output stride"; + const OUTPUT_STRIDE_CTX: &'static str = "fast420 output stride"; + const REGION_OUTPUT_STRIDE_CTX: &'static str = "fast420 region output stride"; + const SCALED_ENTROPY_PAYLOAD_CTX: &'static str = "fast420 scaled entropy payload"; + const FULL_RGB_BATCH_BLOCKS_PER_MCU: Option = Some(6); + + fn chroma_height(height: u32) -> u32 { + height.div_ceil(2) + } + fn packed_height_extent(height: u32) -> u32 { + height.div_ceil(2).max(1) + } +} + +impl FastSubsampledPacket for JpegFast422PacketV1 { + const MCU_WIDTH: u32 = 16; + const MCU_HEIGHT: u32 = 8; + const FULL_RGB_BATCH_SUPPORTS_RESTART: bool = false; + const ENTROPY_PAYLOAD_CTX: &'static str = "fast422 entropy payload"; + const REGION_SCALED_BATCH_OUT_STRIDE_CTX: &'static str = + "fast422 region scaled batch output stride"; + const OUTPUT_STRIDE_CTX: &'static str = "fast422 output stride"; + const REGION_OUTPUT_STRIDE_CTX: &'static str = "fast422 region output stride"; + const SCALED_ENTROPY_PAYLOAD_CTX: &'static str = "fast422 scaled entropy payload"; + const FULL_RGB_BATCH_BLOCKS_PER_MCU: Option = None; + + fn chroma_height(height: u32) -> u32 { + height + } + fn packed_height_extent(height: u32) -> u32 { + height + } +} + +/// Scratch-pool cache keys for one batch driver's buffers; keys stay +/// per-family so pooled buffers are never shared across kernel families. +#[cfg(target_os = "macos")] +struct FastScratchKeys { + y: &'static str, + cb: &'static str, + cr: &'static str, + entropy: &'static str, + entropy_offsets: &'static str, + entropy_lens: &'static str, + entropy_checkpoints: &'static str, + status: &'static str, +} + +/// Per-family vertical chroma-repair scratch layout for the direct-to-texture +/// full-frame path (4:2:0 only; 4:2:2 has no vertical MCU chroma boundary). +#[cfg(target_os = "macos")] +struct FastVerticalRepairSpec { + meta_words: usize, + sample_bytes: usize, + meta_key: &'static str, + samples_key: &'static str, +} + +/// Metal-side hooks for the subsampled families: per-family pipelines, +/// scratch keys, and batched-packet extraction. +#[cfg(target_os = "macos")] +trait FastSubsampledMetal: FastSubsampledPacket { + const REGION_SCALED_KEYS: FastScratchKeys; + const REGION_SCALED_TEXTURE_KEYS: FastScratchKeys; + /// Scratch keys for the full-frame RGB batch driver. + const FULL_BATCH_KEYS: FastScratchKeys; + /// Scratch keys for the full-frame RGBA texture batch driver. + const TEXTURE_KEYS: FastScratchKeys; + /// Profile tag for the full-RGB batch timing rows. + const FULL_RGB_BATCH_TIMING_TAG: &'static str; + /// Boundary chroma-repair record layout for the direct-to-texture path. + const TEXTURE_BOUNDARY_META_WORDS: usize; + const TEXTURE_BOUNDARY_SAMPLE_BYTES: usize; + const TEXTURE_BOUNDARY_META_KEY: &'static str; + const TEXTURE_BOUNDARY_SAMPLES_KEY: &'static str; + const TEXTURE_VERTICAL_REPAIR: Option; + + fn from_batched<'a>(packet: &BatchedFastPacket<'a>) -> Option<&'a Self>; + fn to_batched(&self) -> BatchedFastPacket<'_>; + fn decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; + fn region_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; + fn scaled_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; + fn scaled_region_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; + /// Full-frame pack pipeline for the requested output format, or `None` + /// when the family has no kernel for that format (4:2:2 packs only RGB + /// and RGBA; 4:2:0 falls back to its generic pack kernel). + fn pack_pipeline_for_format( + runtime: &MetalRuntime, + fmt: PixelFormat, + ) -> Option<&ComputePipelineState>; + fn pack_windowed_pipeline_for_format( + runtime: &MetalRuntime, + fmt: PixelFormat, + ) -> &ComputePipelineState; + fn scaled_region_batch_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; + fn pack_windowed_rgb_batch_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; + fn pack_windowed_rgba_texture_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; + fn full_rgb_batch_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; + fn pack_full_rgb_batch_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; + fn pack_rgba_texture_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; + fn rgba_texture_batch_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; + fn rgba_texture_boundary_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; + /// Whether the full-RGB batch driver should record per-stage timing rows. + fn full_rgb_batch_timing_enabled() -> bool; + /// Per-MCU dispatch width for the texture repair passes (4:2:0 only; + /// 4:2:2 repairs per entropy segment instead). + fn texture_mcu_dispatch_threads(total_mcus: usize) -> Result, Error>; + /// Repair record count for the direct-to-texture boundary scratch. + fn texture_repair_record_count( + tile_count: usize, + total_mcus: usize, + total_decode_threads: u32, + ) -> Result; + /// Thread count for the horizontal boundary repair pass, or `None` when + /// the pass is not needed for this batch shape. + fn horizontal_repair_threads( + first: &Self, + segment_count_u32: u32, + mcu_threads: Option, + ) -> Option; + /// Encode any family-specific repair passes beyond the shared horizontal + /// pass (4:2:0 adds vertical and corner passes). + fn encode_extra_texture_repair_passes( + runtime: &MetalRuntime, + ctx: &FastTextureRepairCtx<'_>, + ) -> Result<(), Error>; + /// Pipelines for the split coeff/IDCT debug decode mode, when supported. + #[cfg(test)] + fn split_coeff_idct_pipelines( + runtime: &MetalRuntime, + ) -> Option<(&ComputePipelineState, &ComputePipelineState)>; + /// Scratch keys (coeff blocks, DC-only flags) for the split decode mode + /// in the texture driver. + #[cfg(test)] + const SPLIT_TEXTURE_SCRATCH_KEYS: (&'static str, &'static str); +} + +/// Shared context handed to the family-specific texture repair hooks. +#[cfg(target_os = "macos")] +struct FastTextureRepairCtx<'a> { + command_buffer: &'a CommandBufferRef, + output: &'a crate::MetalBatchTextureOutput, + boundary_meta_buffer: &'a Buffer, + vertical_buffers: Option<&'a (Buffer, Buffer)>, + decode_params: JpegFast420TextureBatchParams, + tile_count: usize, + mcu_threads: Option, + tile_index_ctx: &'a str, +} + +#[cfg(target_os = "macos")] +impl FastSubsampledMetal for JpegFast420PacketV1 { + const REGION_SCALED_KEYS: FastScratchKeys = FastScratchKeys { + y: "fast420_region_scaled_y", + cb: "fast420_region_scaled_cb", + cr: "fast420_region_scaled_cr", + entropy: "fast420_region_scaled_entropy", + entropy_offsets: "fast420_region_scaled_entropy_offsets", + entropy_lens: "fast420_region_scaled_entropy_lens", + entropy_checkpoints: "fast420_region_scaled_entropy_checkpoints", + status: "fast420_region_scaled_status", + }; + const REGION_SCALED_TEXTURE_KEYS: FastScratchKeys = FastScratchKeys { + y: "fast420_region_scaled_texture_y", + cb: "fast420_region_scaled_texture_cb", + cr: "fast420_region_scaled_texture_cr", + entropy: "fast420_region_scaled_texture_entropy", + entropy_offsets: "fast420_region_scaled_texture_entropy_offsets", + entropy_lens: "fast420_region_scaled_texture_entropy_lens", + entropy_checkpoints: "fast420_region_scaled_texture_entropy_checkpoints", + status: "fast420_region_scaled_texture_status", + }; + + const FULL_BATCH_KEYS: FastScratchKeys = FastScratchKeys { + y: "fast420_full_y", + cb: "fast420_full_cb", + cr: "fast420_full_cr", + entropy: "fast420_full_entropy", + entropy_offsets: "fast420_full_entropy_offsets", + entropy_lens: "fast420_full_entropy_lens", + entropy_checkpoints: "fast420_full_entropy_checkpoints", + status: "fast420_full_status", + }; + const TEXTURE_KEYS: FastScratchKeys = FastScratchKeys { + y: "fast420_texture_y", + cb: "fast420_texture_cb", + cr: "fast420_texture_cr", + entropy: "fast420_texture_entropy", + entropy_offsets: "fast420_texture_entropy_offsets", + entropy_lens: "fast420_texture_entropy_lens", + entropy_checkpoints: "fast420_texture_entropy_checkpoints", + status: "fast420_texture_status", + }; + const FULL_RGB_BATCH_TIMING_TAG: &'static str = "metal_fast420_batch"; + const TEXTURE_BOUNDARY_META_WORDS: usize = FAST420_TEXTURE_BOUNDARY_META_WORDS; + const TEXTURE_BOUNDARY_SAMPLE_BYTES: usize = FAST420_TEXTURE_BOUNDARY_SAMPLE_BYTES; + const TEXTURE_BOUNDARY_META_KEY: &'static str = "fast420_texture_boundary_meta"; + const TEXTURE_BOUNDARY_SAMPLES_KEY: &'static str = "fast420_texture_boundary_samples"; + const TEXTURE_VERTICAL_REPAIR: Option = Some(FastVerticalRepairSpec { + meta_words: FAST420_TEXTURE_VERTICAL_META_WORDS, + sample_bytes: FAST420_TEXTURE_VERTICAL_SAMPLE_BYTES, + meta_key: "fast420_texture_vertical_meta", + samples_key: "fast420_texture_vertical_samples", + }); + + fn from_batched<'a>(packet: &BatchedFastPacket<'a>) -> Option<&'a Self> { + match packet { + BatchedFastPacket::Fast420(packet) => Some(packet), + _ => None, + } + } + fn to_batched(&self) -> BatchedFastPacket<'_> { + BatchedFastPacket::Fast420(self) + } + fn decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast420_decode_pipeline + } + fn region_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast420_region_decode_pipeline + } + fn scaled_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast420_scaled_decode_pipeline + } + fn scaled_region_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast420_scaled_region_decode_pipeline + } + fn pack_pipeline_for_format( + runtime: &MetalRuntime, + fmt: PixelFormat, + ) -> Option<&ComputePipelineState> { + Some(pack_420_pipeline_for_format(runtime, fmt)) + } + fn pack_windowed_pipeline_for_format( + runtime: &MetalRuntime, + fmt: PixelFormat, + ) -> &ComputePipelineState { + pack_420_windowed_pipeline_for_format(runtime, fmt) + } + fn scaled_region_batch_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast420_scaled_region_batch_decode_pipeline + } + fn pack_windowed_rgb_batch_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.pack_420_windowed_rgb_batch_pipeline + } + fn pack_windowed_rgba_texture_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.pack_420_windowed_rgba_texture_pipeline + } + fn full_rgb_batch_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast420_batch_decode_pipeline + } + fn pack_full_rgb_batch_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.pack_420_rgb_batch_pipeline + } + fn pack_rgba_texture_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.pack_420_rgba_texture_pipeline + } + fn rgba_texture_batch_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast420_rgba_texture_batch_decode_pipeline + } + fn rgba_texture_boundary_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast420_rgba_texture_boundary_pipeline + } + fn full_rgb_batch_timing_enabled() -> bool { + fast420_batch_timing_enabled() + } + fn texture_mcu_dispatch_threads(total_mcus: usize) -> Result, Error> { + checked_u32(total_mcus, "fast420 texture batch MCU count").map(Some) + } + fn texture_repair_record_count( + tile_count: usize, + total_mcus: usize, + _total_decode_threads: u32, + ) -> Result { + tile_count + .checked_mul(total_mcus) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal fast420 texture repair record count overflowed".to_string(), + }) + } + fn horizontal_repair_threads( + first: &Self, + _segment_count_u32: u32, + mcu_threads: Option, + ) -> Option { + mcu_threads.filter(|_| first.mcus_per_row > 1) + } + fn encode_extra_texture_repair_passes( + runtime: &MetalRuntime, + ctx: &FastTextureRepairCtx<'_>, + ) -> Result<(), Error> { + let (vertical_meta_buffer, vertical_samples_buffer) = + ctx.vertical_buffers.ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal fast420 texture vertical repair scratch was missing" + .to_string(), + })?; + let Some(mcu_threads) = ctx.mcu_threads else { + return Err(Error::MetalKernel { + message: "JPEG Metal fast420 texture MCU dispatch width was missing".to_string(), + }); + }; + if ctx.decode_params.mcu_rows > 1 { + for index in 0..ctx.tile_count { + let texture = ctx + .output + .texture(index) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?; + let decode_params = JpegFast420TextureBatchParams { + tile_index: checked_u32(index, ctx.tile_index_ctx)?, + ..ctx.decode_params + }; + let boundary_encoder = ctx.command_buffer.new_compute_command_encoder(); + boundary_encoder.set_compute_pipeline_state( + &runtime.fast420_rgba_texture_vertical_boundary_pipeline, + ); + boundary_encoder.set_buffer(0, Some(vertical_meta_buffer), 0); + boundary_encoder.set_buffer(1, Some(vertical_samples_buffer), 0); + boundary_encoder.set_bytes( + 2, + size_of::() as u64, + (&raw const decode_params).cast(), + ); + boundary_encoder.set_texture(0, Some(texture)); + dispatch_1d_pipeline( + boundary_encoder, + &runtime.fast420_rgba_texture_vertical_boundary_pipeline, + mcu_threads, + ); + boundary_encoder.end_encoding(); + } + } + if ctx.decode_params.mcus_per_row > 1 && ctx.decode_params.mcu_rows > 1 { + for index in 0..ctx.tile_count { + let texture = ctx + .output + .texture(index) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?; + let decode_params = JpegFast420TextureBatchParams { + tile_index: checked_u32(index, ctx.tile_index_ctx)?, + ..ctx.decode_params + }; + let corner_encoder = ctx.command_buffer.new_compute_command_encoder(); + corner_encoder + .set_compute_pipeline_state(&runtime.fast420_rgba_texture_corner_pipeline); + corner_encoder.set_buffer(0, Some(ctx.boundary_meta_buffer), 0); + corner_encoder.set_buffer(1, Some(vertical_meta_buffer), 0); + corner_encoder.set_buffer(2, Some(vertical_samples_buffer), 0); + corner_encoder.set_bytes( + 3, + size_of::() as u64, + (&raw const decode_params).cast(), + ); + corner_encoder.set_texture(0, Some(texture)); + dispatch_1d_pipeline( + corner_encoder, + &runtime.fast420_rgba_texture_corner_pipeline, + mcu_threads, + ); + corner_encoder.end_encoding(); + } + } + Ok(()) + } + #[cfg(test)] + fn split_coeff_idct_pipelines( + runtime: &MetalRuntime, + ) -> Option<(&ComputePipelineState, &ComputePipelineState)> { + Some(( + &runtime.fast420_batch_coeffs_decode_pipeline, + &runtime.fast420_batch_idct_deposit_pipeline, + )) + } + #[cfg(test)] + const SPLIT_TEXTURE_SCRATCH_KEYS: (&'static str, &'static str) = ( + "fast420_texture_coeff_blocks", + "fast420_texture_dc_only_flags", + ); +} + +#[cfg(target_os = "macos")] +impl FastSubsampledMetal for JpegFast422PacketV1 { + const REGION_SCALED_KEYS: FastScratchKeys = FastScratchKeys { + y: "fast422_region_scaled_y", + cb: "fast422_region_scaled_cb", + cr: "fast422_region_scaled_cr", + entropy: "fast422_region_scaled_entropy", + entropy_offsets: "fast422_region_scaled_entropy_offsets", + entropy_lens: "fast422_region_scaled_entropy_lens", + entropy_checkpoints: "fast422_region_scaled_entropy_checkpoints", + status: "fast422_region_scaled_status", + }; + const REGION_SCALED_TEXTURE_KEYS: FastScratchKeys = FastScratchKeys { + y: "fast422_region_scaled_texture_y", + cb: "fast422_region_scaled_texture_cb", + cr: "fast422_region_scaled_texture_cr", + entropy: "fast422_region_scaled_texture_entropy", + entropy_offsets: "fast422_region_scaled_texture_entropy_offsets", + entropy_lens: "fast422_region_scaled_texture_entropy_lens", + entropy_checkpoints: "fast422_region_scaled_texture_entropy_checkpoints", + status: "fast422_region_scaled_texture_status", + }; + + const FULL_BATCH_KEYS: FastScratchKeys = FastScratchKeys { + y: "fast422_full_y", + cb: "fast422_full_cb", + cr: "fast422_full_cr", + entropy: "fast422_full_entropy", + entropy_offsets: "fast422_full_entropy_offsets", + entropy_lens: "fast422_full_entropy_lens", + entropy_checkpoints: "fast422_full_entropy_checkpoints", + status: "fast422_full_status", + }; + const TEXTURE_KEYS: FastScratchKeys = FastScratchKeys { + y: "fast422_texture_y", + cb: "fast422_texture_cb", + cr: "fast422_texture_cr", + entropy: "fast422_texture_entropy", + entropy_offsets: "fast422_texture_entropy_offsets", + entropy_lens: "fast422_texture_entropy_lens", + entropy_checkpoints: "fast422_texture_entropy_checkpoints", + status: "fast422_texture_status", + }; + const FULL_RGB_BATCH_TIMING_TAG: &'static str = "metal_fast422_batch"; + const TEXTURE_BOUNDARY_META_WORDS: usize = FAST422_TEXTURE_BOUNDARY_META_WORDS; + const TEXTURE_BOUNDARY_SAMPLE_BYTES: usize = FAST422_TEXTURE_BOUNDARY_SAMPLE_BYTES; + const TEXTURE_BOUNDARY_META_KEY: &'static str = "fast422_texture_boundary_meta"; + const TEXTURE_BOUNDARY_SAMPLES_KEY: &'static str = "fast422_texture_boundary_samples"; + const TEXTURE_VERTICAL_REPAIR: Option = None; + + fn from_batched<'a>(packet: &BatchedFastPacket<'a>) -> Option<&'a Self> { + match packet { + BatchedFastPacket::Fast422(packet) => Some(packet), + _ => None, + } + } + fn to_batched(&self) -> BatchedFastPacket<'_> { + BatchedFastPacket::Fast422(self) + } + fn decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast422_decode_pipeline + } + fn region_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast422_region_decode_pipeline + } + fn scaled_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast422_scaled_decode_pipeline + } + fn scaled_region_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast422_scaled_region_decode_pipeline + } + fn pack_pipeline_for_format( + runtime: &MetalRuntime, + fmt: PixelFormat, + ) -> Option<&ComputePipelineState> { + pack_422_pipeline_for_format(runtime, fmt) + } + fn pack_windowed_pipeline_for_format( + runtime: &MetalRuntime, + fmt: PixelFormat, + ) -> &ComputePipelineState { + pack_422_windowed_pipeline_for_format(runtime, fmt) + } + fn scaled_region_batch_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast422_scaled_region_batch_decode_pipeline + } + fn pack_windowed_rgb_batch_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.pack_422_windowed_rgb_batch_pipeline + } + fn pack_windowed_rgba_texture_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.pack_422_windowed_rgba_texture_pipeline + } + fn full_rgb_batch_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast422_batch_decode_pipeline + } + fn pack_full_rgb_batch_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.pack_422_rgb_batch_pipeline + } + fn pack_rgba_texture_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.pack_422_rgba_texture_pipeline + } + fn rgba_texture_batch_decode_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast422_rgba_texture_batch_decode_pipeline + } + fn rgba_texture_boundary_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.fast422_rgba_texture_boundary_pipeline + } + fn full_rgb_batch_timing_enabled() -> bool { + false + } + fn texture_mcu_dispatch_threads(_total_mcus: usize) -> Result, Error> { + Ok(None) + } + fn texture_repair_record_count( + _tile_count: usize, + _total_mcus: usize, + total_decode_threads: u32, + ) -> Result { + Ok(total_decode_threads as usize) + } + fn horizontal_repair_threads( + _first: &Self, + segment_count_u32: u32, + _mcu_threads: Option, + ) -> Option { + (segment_count_u32 > 1).then_some(segment_count_u32) + } + fn encode_extra_texture_repair_passes( + _runtime: &MetalRuntime, + _ctx: &FastTextureRepairCtx<'_>, + ) -> Result<(), Error> { + Ok(()) + } + #[cfg(test)] + fn split_coeff_idct_pipelines( + _runtime: &MetalRuntime, + ) -> Option<(&ComputePipelineState, &ComputePipelineState)> { + None + } + #[cfg(test)] + const SPLIT_TEXTURE_SCRATCH_KEYS: (&'static str, &'static str) = ( + "fast422_texture_coeff_blocks", + "fast422_texture_dc_only_flags", + ); +} + +fn fast_subsampled_params( + packet: &P, + fmt: PixelFormat, +) -> Result { + let out_format = pixel_format_to_out_format(fmt).ok_or_else(|| Error::MetalKernel { + message: format!( + "unsupported JPEG Metal {} pixel format {fmt:?}", + P::FAMILY_NAME + ), + })?; + let out_stride = packet.dimensions().0 as usize * fmt.bytes_per_pixel(); + Ok(JpegFast420Params { + width: packet.dimensions().0, + height: packet.dimensions().1, + chroma_width: packet.dimensions().0.div_ceil(2), + chroma_height: P::chroma_height(packet.dimensions().1), + mcus_per_row: packet.mcus_per_row(), + mcu_rows: packet.mcu_rows(), + restart_interval_mcus: packet.restart_interval_mcus(), + restart_offset_count: checked_entropy_segment_count( + packet.restart_interval_mcus(), + packet.restart_offsets().len(), + packet.entropy_checkpoints().len(), + )?, + restart_start_mcu: 0, + entropy_len: checked_u32(packet.entropy_bytes().len(), P::ENTROPY_PAYLOAD_CTX)?, + out_stride: checked_u32(out_stride, P::OUTPUT_STRIDE_CTX)?, + alpha: u32::from(u8::MAX), + out_format, + origin_x: 0, + origin_y: 0, + }) +} + +fn fast_subsampled_region_params( + packet: &P, + fmt: PixelFormat, + source_window: j2k_jpeg::Rect, +) -> Result { + let out_format = pixel_format_to_out_format(fmt).ok_or_else(|| Error::MetalKernel { + message: format!( + "unsupported JPEG Metal {} pixel format {fmt:?}", + P::FAMILY_NAME + ), + })?; + let out_stride = source_window.w as usize * fmt.bytes_per_pixel(); + Ok(JpegFast420Params { + width: source_window.w, + height: source_window.h, + chroma_width: source_window.w.div_ceil(2), + chroma_height: P::chroma_height(source_window.h), + mcus_per_row: packet.mcus_per_row(), + mcu_rows: packet.mcu_rows(), + restart_interval_mcus: packet.restart_interval_mcus(), + restart_offset_count: checked_entropy_segment_count( + packet.restart_interval_mcus(), + packet.restart_offsets().len(), + packet.entropy_checkpoints().len(), + )?, + restart_start_mcu: 0, + entropy_len: checked_u32(packet.entropy_bytes().len(), P::ENTROPY_PAYLOAD_CTX)?, + out_stride: checked_u32(out_stride, P::REGION_OUTPUT_STRIDE_CTX)?, + alpha: u32::from(u8::MAX), + out_format, + origin_x: source_window.x, + origin_y: source_window.y, + }) +} + +fn fast_subsampled_scaled_params( + packet: &P, + scale: j2k_core::Downscale, +) -> Option { + let scale_shift = match scale { + j2k_core::Downscale::Half => 1, + j2k_core::Downscale::Quarter => 2, + j2k_core::Downscale::Eighth => 3, + _ => return None, + }; + let denom = 1u32 << scale_shift; + let scaled_width = packet.dimensions().0.div_ceil(denom); + let scaled_height = packet.dimensions().1.div_ceil(denom); + Some(JpegFast420ScaledParams { + scaled_width, + scaled_height, + chroma_width: scaled_width.div_ceil(2), + chroma_height: P::chroma_height(scaled_height), + mcus_per_row: packet.mcus_per_row(), + mcu_rows: packet.mcu_rows(), + restart_interval_mcus: packet.restart_interval_mcus(), + restart_offset_count: optional_entropy_segment_count( + packet.restart_interval_mcus(), + packet.restart_offsets().len(), + packet.entropy_checkpoints().len(), + )?, + restart_start_mcu: 0, + entropy_len: checked_u32(packet.entropy_bytes().len(), P::SCALED_ENTROPY_PAYLOAD_CTX) + .ok()?, + scale_shift, + origin_x: 0, + origin_y: 0, + }) +} + +fn fast_subsampled_scaled_region_params( + packet: &P, + scale: j2k_core::Downscale, + source_window: j2k_jpeg::Rect, +) -> Option { + let full = fast_subsampled_scaled_params(packet, scale)?; + Some(JpegFast420ScaledParams { + scaled_width: source_window.w, + scaled_height: source_window.h, + chroma_width: source_window.w.div_ceil(2), + chroma_height: P::chroma_height(source_window.h), + origin_x: source_window.x, + origin_y: source_window.y, + ..full + }) +} + +fn fast_subsampled_full_mcu_window( + dims: (u32, u32), + roi: j2k_jpeg::Rect, +) -> j2k_jpeg::Rect { + let x0 = (roi.x / P::MCU_WIDTH) * P::MCU_WIDTH; + let y0 = (roi.y / P::MCU_HEIGHT) * P::MCU_HEIGHT; + let x1 = (roi.x + roi.w).div_ceil(P::MCU_WIDTH) * P::MCU_WIDTH; + let y1 = (roi.y + roi.h).div_ceil(P::MCU_HEIGHT) * P::MCU_HEIGHT; + j2k_jpeg::Rect { + x: x0, + y: y0, + w: x1.min(dims.0).saturating_sub(x0), + h: y1.min(dims.1).saturating_sub(y0), + } +} + +fn fast_subsampled_full_mcu_scaled_window( + scaled_dims: (u32, u32), + roi: j2k_jpeg::Rect, + scale_shift: u32, +) -> j2k_jpeg::Rect { + let mcu_width = P::MCU_WIDTH >> scale_shift; + let mcu_height = P::MCU_HEIGHT >> scale_shift; + let x0 = (roi.x / mcu_width) * mcu_width; + let y0 = (roi.y / mcu_height) * mcu_height; + let x1 = (roi.x + roi.w).div_ceil(mcu_width) * mcu_width; + let y1 = (roi.y + roi.h).div_ceil(mcu_height) * mcu_height; + j2k_jpeg::Rect { + x: x0, + y: y0, + w: x1.min(scaled_dims.0).saturating_sub(x0), + h: y1.min(scaled_dims.1).saturating_sub(y0), + } +} + +#[cfg(target_os = "macos")] +fn fast444_params(packet: &JpegFast444PacketV1) -> Result { + Ok(JpegFast444Params { + width: packet.dimensions.0, + height: packet.dimensions.1, + mcus_per_row: packet.mcus_per_row, + mcu_rows: packet.mcu_rows, + restart_interval_mcus: packet.restart_interval_mcus, + restart_offset_count: checked_entropy_segment_count( + packet.restart_interval_mcus, + packet.restart_offsets.len(), + packet.entropy_checkpoints.len(), + )?, + restart_start_mcu: 0, + entropy_len: checked_u32(packet.entropy_bytes.len(), "fast444 entropy payload")?, + origin_x: 0, + origin_y: 0, + }) +} + +#[cfg(target_os = "macos")] +fn fast444_region_params( + packet: &JpegFast444PacketV1, + roi: j2k_jpeg::Rect, +) -> Result { + Ok(JpegFast444Params { + width: roi.w, + height: roi.h, + origin_x: roi.x, + origin_y: roi.y, + ..fast444_params(packet)? + }) +} + +#[cfg(target_os = "macos")] +fn fast444_scaled_params( + packet: &JpegFast444PacketV1, + scale: j2k_core::Downscale, +) -> Option { + let scale_shift = match scale { + j2k_core::Downscale::Half => 1, + j2k_core::Downscale::Quarter => 2, + j2k_core::Downscale::Eighth => 3, + _ => return None, + }; + let denom = 1u32 << scale_shift; + Some(JpegFast444ScaledParams { + scaled_width: packet.dimensions.0.div_ceil(denom), + scaled_height: packet.dimensions.1.div_ceil(denom), + mcus_per_row: packet.mcus_per_row, + mcu_rows: packet.mcu_rows, + restart_interval_mcus: packet.restart_interval_mcus, + restart_offset_count: optional_entropy_segment_count( + packet.restart_interval_mcus, + packet.restart_offsets.len(), + packet.entropy_checkpoints.len(), + )?, + restart_start_mcu: 0, + entropy_len: checked_u32(packet.entropy_bytes.len(), "fast444 scaled entropy payload") + .ok()?, + scale_shift, + origin_x: 0, + origin_y: 0, + }) +} + +#[cfg(target_os = "macos")] +fn fast444_scaled_region_params( + packet: &JpegFast444PacketV1, + scale: j2k_core::Downscale, + roi: j2k_jpeg::Rect, +) -> Option { + Some(JpegFast444ScaledParams { + scaled_width: roi.w, + scaled_height: roi.h, + origin_x: roi.x, + origin_y: roi.y, + ..fast444_scaled_params(packet, scale)? + }) +} + +#[cfg(target_os = "macos")] +fn fast_subsampled_windowed_pack_params_for_dims( + dims: (u32, u32), + fmt: PixelFormat, + roi: j2k_jpeg::Rect, +) -> Result { + let out_format = pixel_format_to_out_format(fmt).ok_or_else(|| Error::MetalKernel { + message: format!( + "unsupported JPEG Metal {} pixel format {fmt:?}", + P::FAMILY_NAME + ), + })?; + let out_stride = roi.w as usize * fmt.bytes_per_pixel(); + Ok(JpegFast420WindowedPackParams { + src_width: dims.0, + src_height: dims.1, + chroma_width: dims.0.div_ceil(2), + chroma_height: P::chroma_height(dims.1), + src_x: roi.x, + src_y: roi.y, + width: roi.w, + height: roi.h, + out_stride: checked_u32( + out_stride, + &format!("{} windowed output stride", P::FAMILY_NAME), + )?, + alpha: u32::from(u8::MAX), + out_format, + }) +} + +#[cfg(target_os = "macos")] +fn restart_offsets_buffer(device: &Device, restart_offsets: &[u32]) -> Result { + if restart_offsets.is_empty() { + return Err(Error::MetalKernel { + message: "JPEG Metal restart offsets must contain at least one entry".to_string(), + }); + } + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let bytes = unsafe { + core::slice::from_raw_parts( + restart_offsets.as_ptr().cast::(), + size_of_val(restart_offsets), + ) + }; + Ok(new_shared_buffer_with_data(device, bytes)) +} + +#[cfg(target_os = "macos")] +fn entropy_checkpoints_buffer( + device: &Device, + entropy_checkpoints: &[JpegEntropyCheckpointV1], +) -> Result { + if entropy_checkpoints.is_empty() { + return Err(Error::MetalKernel { + message: "JPEG Metal entropy checkpoints must contain at least one entry".to_string(), + }); + } + let checkpoints = entropy_checkpoint_hosts(entropy_checkpoints)?; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let bytes = unsafe { + core::slice::from_raw_parts( + checkpoints.as_ptr().cast::(), + size_of_val(checkpoints.as_slice()), + ) + }; + Ok(new_shared_buffer_with_data(device, bytes)) +} + +#[cfg(target_os = "macos")] +fn entropy_checkpoint_hosts( + entropy_checkpoints: &[JpegEntropyCheckpointV1], +) -> Result, Error> { + if entropy_checkpoints.is_empty() { + return Err(Error::MetalKernel { + message: "JPEG Metal entropy checkpoints must contain at least one entry".to_string(), + }); + } + Ok(entropy_checkpoints + .iter() + .copied() + .map(JpegEntropyCheckpointHost::from) + .collect::>()) +} + +#[cfg(target_os = "macos")] +fn entropy_segment_count( + restart_interval_mcus: u32, + restart_offsets_len: usize, + entropy_checkpoints_len: usize, +) -> u32 { + let len = if restart_interval_mcus == 0 { + entropy_checkpoints_len + } else { + restart_offsets_len + }; + u32::try_from(len) + .expect("JPEG Metal entropy segment count fits in u32") + .max(1) +} + +#[cfg(target_os = "macos")] +fn optional_entropy_segment_count( + restart_interval_mcus: u32, + restart_offsets_len: usize, + entropy_checkpoints_len: usize, +) -> Option { + let len = if restart_interval_mcus == 0 { + entropy_checkpoints_len + } else { + restart_offsets_len + }; + u32::try_from(len).ok().map(|count| count.max(1)) +} + +#[cfg(target_os = "macos")] +fn checked_entropy_segment_count( + restart_interval_mcus: u32, + restart_offsets_len: usize, + entropy_checkpoints_len: usize, +) -> Result { + optional_entropy_segment_count( + restart_interval_mcus, + restart_offsets_len, + entropy_checkpoints_len, + ) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal entropy segment count does not fit in u32".to_string(), + }) +} + +#[cfg(target_os = "macos")] +fn restart_work_for_mcu_range( + restart_offsets: &[u32], + restart_interval_mcus: u32, + total_mcus: u32, + first_mcu: u32, + end_mcu: u32, +) -> (u32, &[u32]) { + if restart_interval_mcus == 0 || restart_offsets.len() <= 1 { + return (0, restart_offsets); + } + + let first_mcu = first_mcu.min(total_mcus); + let end_mcu = end_mcu.min(total_mcus).max(first_mcu + 1); + let restart_offset_count = + u32::try_from(restart_offsets.len()).expect("JPEG Metal restart offsets fit in u32"); + let first_segment = (first_mcu / restart_interval_mcus).min(restart_offset_count - 1); + let end_segment = end_mcu + .div_ceil(restart_interval_mcus) + .min(restart_offset_count) + .max(first_segment + 1); + ( + first_segment * restart_interval_mcus, + &restart_offsets[first_segment as usize..end_segment as usize], + ) +} + +#[cfg(target_os = "macos")] +fn mcu_range_for_rect( + rect: j2k_jpeg::Rect, + mcus_per_row: u32, + mcu_rows: u32, + mcu_width: u32, + mcu_height: u32, +) -> (u32, u32) { + if rect.w == 0 || rect.h == 0 || mcus_per_row == 0 || mcu_rows == 0 { + return (0, 0); + } + + let max_col = mcus_per_row - 1; + let max_row = mcu_rows - 1; + let last_x = rect.x.saturating_add(rect.w).saturating_sub(1); + let last_y = rect.y.saturating_add(rect.h).saturating_sub(1); + let first_col = (rect.x / mcu_width).min(max_col); + let last_col = (last_x / mcu_width).min(max_col); + let first_row = (rect.y / mcu_height).min(max_row); + let last_row = (last_y / mcu_height).min(max_row); + let first_mcu = first_row * mcus_per_row + first_col; + let end_mcu = last_row * mcus_per_row + last_col + 1; + (first_mcu, end_mcu) +} + +#[cfg(target_os = "macos")] +fn entropy_decode_thread_count( + restart_interval_mcus: u32, + restart_offsets_len: usize, + entropy_checkpoints_len: usize, +) -> u32 { + entropy_segment_count( + restart_interval_mcus, + restart_offsets_len, + entropy_checkpoints_len, + ) +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn encode_jpeg_pack_to_surface_in_command_buffer( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + plane0: &Buffer, + plane1: Option<&Buffer>, + plane2: Option<&Buffer>, + dims: (u32, u32), + mode: PlaneMode, + fmt: PixelFormat, +) -> Result { + match (mode, fmt) { + (PlaneMode::Gray | PlaneMode::YCbCr, PixelFormat::Gray8) => { + return Ok(Surface::from_metal_buffer(plane0.clone(), dims, fmt)); + } + ( + PlaneMode::Gray | PlaneMode::YCbCr | PlaneMode::Rgb, + PixelFormat::Rgb8 | PixelFormat::Rgba8, + ) + | (PlaneMode::Rgb, PixelFormat::Gray8) => {} + _ => { + return Err(Error::MetalKernel { + message: format!("unsupported JPEG Metal pixel format {fmt:?}"), + }); + } + } + + let pitch_bytes = dims.0 as usize * fmt.bytes_per_pixel(); + let out_buffer = runtime.device.new_buffer( + (pitch_bytes * dims.1 as usize) as u64, + MTLResourceOptions::StorageModeShared, + ); + let params = JpegPackParams { + width: dims.0, + height: dims.1, + out_stride: u32::try_from(pitch_bytes).expect("JPEG Metal output stride fits in u32"), + alpha: u32::from(u8::MAX), + mode: match mode { + PlaneMode::Gray => MODE_GRAY, + PlaneMode::YCbCr => MODE_YCBCR, + PlaneMode::Rgb => MODE_RGB, + }, + out_format: match fmt { + PixelFormat::Gray8 => OUT_GRAY, + PixelFormat::Rgb8 => OUT_RGB, + PixelFormat::Rgba8 => OUT_RGBA, + _ => unreachable!("validated by caller"), + }, + }; + + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.pack_pipeline); + encoder.set_buffer(0, Some(plane0), 0); + encoder.set_buffer(1, plane1.map(std::convert::AsRef::as_ref), 0); + encoder.set_buffer(2, plane2.map(std::convert::AsRef::as_ref), 0); + encoder.set_buffer(3, Some(&out_buffer), 0); + encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_2d_pipeline(encoder, &runtime.pack_pipeline, dims); + encoder.end_encoding(); + + Ok(Surface::from_metal_buffer(out_buffer, dims, fmt)) +} + +#[cfg(target_os = "macos")] +fn encode_fast_subsampled_region_batch_item( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + request_index: usize, + packet: &P, + fmt: PixelFormat, + roi: Rect, +) -> Result { + let roi = core_rect_to_jpeg(roi); + let source_window = fast_subsampled_full_mcu_window::

(packet.dimensions(), roi); + let mut params = fast_subsampled_region_params(packet, fmt, source_window)?; + let (first_mcu, end_mcu) = mcu_range_for_rect( + source_window, + packet.mcus_per_row(), + packet.mcu_rows(), + P::MCU_WIDTH, + P::MCU_HEIGHT, + ); + let total_mcus = packet.mcus_per_row() * packet.mcu_rows(); + let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( + packet.restart_offsets(), + packet.restart_interval_mcus(), + total_mcus, + first_mcu, + end_mcu, + ); + params.restart_start_mcu = restart_start_mcu; + params.restart_offset_count = checked_entropy_segment_count( + packet.restart_interval_mcus(), + restart_offsets.len(), + packet.entropy_checkpoints().len(), + )?; + + let local_roi = j2k_jpeg::Rect { + x: roi.x - source_window.x, + y: roi.y - source_window.y, + w: roi.w, + h: roi.h, + }; + let pack_params = fast_subsampled_windowed_pack_params_for_dims::

( + (source_window.w, source_window.h), + fmt, + local_roi, + )?; + let y_len = source_window.w as usize * source_window.h as usize; + let chroma_len = + source_window.w.div_ceil(2) as usize * P::chroma_height(source_window.h) as usize; + let y_plane = new_decode_plane_buffer(&runtime.device, y_len, false); + let cb_plane = new_private_buffer(&runtime.device, chroma_len); + let cr_plane = new_private_buffer(&runtime.device, chroma_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus(), + restart_offsets.len(), + packet.entropy_checkpoints().len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes().as_ptr().cast(), + packet.entropy_bytes().len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, packet.entropy_checkpoints())?; + + let dc_tables = [ + PreparedHuffmanHost::from(packet.y_dc_table()), + PreparedHuffmanHost::from(packet.cb_dc_table()), + PreparedHuffmanHost::from(packet.cr_dc_table()), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(packet.y_ac_table()), + PreparedHuffmanHost::from(packet.cb_ac_table()), + PreparedHuffmanHost::from(packet.cr_ac_table()), + ]; + + let decode_pipeline = P::region_decode_pipeline(runtime); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &cb_plane, &cr_plane], + ¶ms, + [packet.y_quant(), packet.cb_quant(), packet.cr_quant()], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline(decoder_encoder, decode_pipeline, decode_threads); + decoder_encoder.end_encoding(); + + let out_buffer = runtime.device.new_buffer( + (pack_params.out_stride as usize * roi.h as usize) as u64, + MTLResourceOptions::StorageModeShared, + ); + let pack_encoder = command_buffer.new_compute_command_encoder(); + let pack_pipeline = P::pack_windowed_pipeline_for_format(runtime, fmt); + pack_encoder.set_compute_pipeline_state(pack_pipeline); + bind_three_plane_pack::( + pack_encoder, + [Some(&y_plane), Some(&cb_plane), Some(&cr_plane)], + &out_buffer, + &pack_params, + ); + dispatch_2d_pipeline(pack_encoder, pack_pipeline, (roi.w, roi.h)); + pack_encoder.end_encoding(); + + Ok(BatchedDecodeItem { + request_index, + surface: Surface::from_metal_buffer(out_buffer, (roi.w, roi.h), fmt), + status_buffer: status_buffer.clone(), + decode_threads, + _decode_resources: vec![ + y_plane, + cb_plane, + cr_plane, + entropy_buffer, + restart_offsets_buffer, + entropy_checkpoints_buffer, + status_buffer, + ], + }) +} + +#[cfg(target_os = "macos")] +fn encode_fast_subsampled_scaled_batch_item( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + request_index: usize, + packet: &P, + fmt: PixelFormat, + scale: j2k_core::Downscale, +) -> Result { + let Some(params) = fast_subsampled_scaled_params(packet, scale) else { + return Err(Error::MetalKernel { + message: format!("unsupported JPEG Metal {} scale {scale:?}", P::FAMILY_NAME), + }); + }; + + let y_len = params.scaled_width as usize * params.scaled_height as usize; + let chroma_len = params.chroma_width as usize * params.chroma_height as usize; + let y_plane = new_decode_plane_buffer(&runtime.device, y_len, fmt == PixelFormat::Gray8); + let cb_plane = new_private_buffer(&runtime.device, chroma_len); + let cr_plane = new_private_buffer(&runtime.device, chroma_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus(), + packet.restart_offsets().len(), + packet.entropy_checkpoints().len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes().as_ptr().cast(), + packet.entropy_bytes().len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, packet.restart_offsets())?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, packet.entropy_checkpoints())?; + + let dc_tables = [ + PreparedHuffmanHost::from(packet.y_dc_table()), + PreparedHuffmanHost::from(packet.cb_dc_table()), + PreparedHuffmanHost::from(packet.cr_dc_table()), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(packet.y_ac_table()), + PreparedHuffmanHost::from(packet.cb_ac_table()), + PreparedHuffmanHost::from(packet.cr_ac_table()), + ]; + + let decode_pipeline = P::scaled_decode_pipeline(runtime); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &cb_plane, &cr_plane], + ¶ms, + [packet.y_quant(), packet.cb_quant(), packet.cr_quant()], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline(decoder_encoder, decode_pipeline, decode_threads); + decoder_encoder.end_encoding(); + + let out_buffer = (fmt != PixelFormat::Gray8).then(|| { + runtime.device.new_buffer( + (params.scaled_width as usize * fmt.bytes_per_pixel() * params.scaled_height as usize) + as u64, + MTLResourceOptions::StorageModeShared, + ) + }); + + if let Some(out_buffer) = out_buffer.as_ref() { + let pack_params = JpegFast420Params { + width: params.scaled_width, + height: params.scaled_height, + chroma_width: params.chroma_width, + chroma_height: params.chroma_height, + mcus_per_row: params.mcus_per_row, + mcu_rows: params.mcu_rows, + restart_interval_mcus: params.restart_interval_mcus, + restart_offset_count: params.restart_offset_count, + restart_start_mcu: params.restart_start_mcu, + entropy_len: params.entropy_len, + out_stride: checked_u32( + params.scaled_width as usize * fmt.bytes_per_pixel(), + "scaled output stride", + )?, + alpha: u32::from(u8::MAX), + out_format: pixel_format_to_out_format(fmt).ok_or_else(|| Error::MetalKernel { + message: format!("unsupported JPEG Metal pixel format {fmt:?}"), + })?, + origin_x: 0, + origin_y: 0, + }; + let Some(pack_pipeline) = P::pack_pipeline_for_format(runtime, fmt) else { + return Err(Error::MetalKernel { + message: format!( + "unsupported JPEG Metal {} pixel format {fmt:?}", + P::FAMILY_NAME + ), + }); + }; + let pack_encoder = command_buffer.new_compute_command_encoder(); + pack_encoder.set_compute_pipeline_state(pack_pipeline); + pack_encoder.set_buffer(0, Some(&y_plane), 0); + pack_encoder.set_buffer(1, Some(&cb_plane), 0); + pack_encoder.set_buffer(2, Some(&cr_plane), 0); + pack_encoder.set_buffer(3, Some(out_buffer), 0); + pack_encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const pack_params).cast(), + ); + dispatch_2d_pipeline( + pack_encoder, + pack_pipeline, + (params.scaled_width, params.scaled_height), + ); + pack_encoder.end_encoding(); + } + + let surface = match out_buffer { + Some(out_buffer) => { + Surface::from_metal_buffer(out_buffer, (params.scaled_width, params.scaled_height), fmt) + } + None => Surface::from_metal_buffer( + y_plane.clone(), + (params.scaled_width, params.scaled_height), + fmt, + ), + }; + + Ok(BatchedDecodeItem { + request_index, + surface, + status_buffer: status_buffer.clone(), + decode_threads, + _decode_resources: vec![ + y_plane, + cb_plane, + cr_plane, + entropy_buffer, + restart_offsets_buffer, + entropy_checkpoints_buffer, + status_buffer, + ], + }) +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn encode_fast_subsampled_scaled_region_batch_item( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + device_buffer_cache: &mut BatchDeviceBufferCache, + request_index: usize, + packet: &P, + fmt: PixelFormat, + roi: Rect, + scale: j2k_core::Downscale, +) -> Result { + let Some(full_params) = fast_subsampled_scaled_params(packet, scale) else { + return Err(Error::MetalKernel { + message: format!("unsupported JPEG Metal {} scale {scale:?}", P::FAMILY_NAME), + }); + }; + let scaled_roi = roi.scaled_covering(scale); + let scaled_roi = j2k_jpeg::Rect { + x: scaled_roi.x, + y: scaled_roi.y, + w: scaled_roi.w, + h: scaled_roi.h, + }; + let source_window = fast_subsampled_full_mcu_scaled_window::

( + (full_params.scaled_width, full_params.scaled_height), + scaled_roi, + full_params.scale_shift, + ); + let Some(mut decode_params) = + fast_subsampled_scaled_region_params(packet, scale, source_window) + else { + return Err(Error::MetalKernel { + message: format!( + "unsupported JPEG Metal {} scaled region {scale:?}", + P::FAMILY_NAME + ), + }); + }; + let mcu_width = P::MCU_WIDTH >> decode_params.scale_shift; + let mcu_height = P::MCU_HEIGHT >> decode_params.scale_shift; + let (first_mcu, end_mcu) = mcu_range_for_rect( + source_window, + packet.mcus_per_row(), + packet.mcu_rows(), + mcu_width, + mcu_height, + ); + let total_mcus = packet.mcus_per_row() * packet.mcu_rows(); + let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( + packet.restart_offsets(), + packet.restart_interval_mcus(), + total_mcus, + first_mcu, + end_mcu, + ); + decode_params.restart_start_mcu = restart_start_mcu; + decode_params.restart_offset_count = checked_entropy_segment_count( + packet.restart_interval_mcus(), + restart_offsets.len(), + packet.entropy_checkpoints().len(), + )?; + let local_roi = j2k_jpeg::Rect { + x: scaled_roi.x - source_window.x, + y: scaled_roi.y - source_window.y, + w: scaled_roi.w, + h: scaled_roi.h, + }; + let pack_params = fast_subsampled_windowed_pack_params_for_dims::

( + (source_window.w, source_window.h), + fmt, + local_roi, + )?; + let y_len = source_window.w as usize * source_window.h as usize; + let chroma_len = + source_window.w.div_ceil(2) as usize * P::chroma_height(source_window.h) as usize; + let y_plane = new_decode_plane_buffer(&runtime.device, y_len, false); + let cb_plane = new_private_buffer(&runtime.device, chroma_len); + let cr_plane = new_private_buffer(&runtime.device, chroma_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus(), + restart_offsets.len(), + packet.entropy_checkpoints().len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; + let (entropy_buffer, entropy_checkpoints_buffer) = device_buffer_cache.packet_buffers( + runtime, + packet.entropy_bytes(), + packet.entropy_checkpoints(), + )?; + + let dc_tables = [ + PreparedHuffmanHost::from(packet.y_dc_table()), + PreparedHuffmanHost::from(packet.cb_dc_table()), + PreparedHuffmanHost::from(packet.cr_dc_table()), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(packet.y_ac_table()), + PreparedHuffmanHost::from(packet.cb_ac_table()), + PreparedHuffmanHost::from(packet.cr_ac_table()), + ]; + + let decode_pipeline = P::scaled_region_decode_pipeline(runtime); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &cb_plane, &cr_plane], + &decode_params, + [packet.y_quant(), packet.cb_quant(), packet.cr_quant()], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline(decoder_encoder, decode_pipeline, decode_threads); + decoder_encoder.end_encoding(); + + let out_buffer = runtime.device.new_buffer( + (pack_params.out_stride as usize * scaled_roi.h as usize) as u64, + MTLResourceOptions::StorageModeShared, + ); + let pack_encoder = command_buffer.new_compute_command_encoder(); + let pack_pipeline = P::pack_windowed_pipeline_for_format(runtime, fmt); + pack_encoder.set_compute_pipeline_state(pack_pipeline); + bind_three_plane_pack::( + pack_encoder, + [Some(&y_plane), Some(&cb_plane), Some(&cr_plane)], + &out_buffer, + &pack_params, + ); + dispatch_2d_pipeline(pack_encoder, pack_pipeline, (scaled_roi.w, scaled_roi.h)); + pack_encoder.end_encoding(); + + Ok(BatchedDecodeItem { + request_index, + surface: Surface::from_metal_buffer(out_buffer, (scaled_roi.w, scaled_roi.h), fmt), + status_buffer: status_buffer.clone(), + decode_threads, + _decode_resources: vec![ + y_plane, + cb_plane, + cr_plane, + entropy_buffer, + restart_offsets_buffer, + entropy_checkpoints_buffer, + status_buffer, + ], + }) +} + +#[cfg(target_os = "macos")] +fn encode_fast_subsampled_batch_item( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + request_index: usize, + packet: &P, + fmt: PixelFormat, +) -> Result { + let params = fast_subsampled_params(packet, fmt)?; + let y_len = params.width as usize * params.height as usize; + let chroma_len = params.chroma_width as usize * params.chroma_height as usize; + let y_plane = new_decode_plane_buffer(&runtime.device, y_len, fmt == PixelFormat::Gray8); + let cb_plane = new_private_buffer(&runtime.device, chroma_len); + let cr_plane = new_private_buffer(&runtime.device, chroma_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus(), + packet.restart_offsets().len(), + packet.entropy_checkpoints().len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes().as_ptr().cast(), + packet.entropy_bytes().len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, packet.restart_offsets())?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, packet.entropy_checkpoints())?; + + let dc_tables = [ + PreparedHuffmanHost::from(packet.y_dc_table()), + PreparedHuffmanHost::from(packet.cb_dc_table()), + PreparedHuffmanHost::from(packet.cr_dc_table()), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(packet.y_ac_table()), + PreparedHuffmanHost::from(packet.cb_ac_table()), + PreparedHuffmanHost::from(packet.cr_ac_table()), + ]; + + let decode_pipeline = P::decode_pipeline(runtime); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &cb_plane, &cr_plane], + ¶ms, + [packet.y_quant(), packet.cb_quant(), packet.cr_quant()], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline(decoder_encoder, decode_pipeline, decode_threads); + decoder_encoder.end_encoding(); + + let surface = if fmt == PixelFormat::Gray8 { + Surface::from_metal_buffer(y_plane.clone(), packet.dimensions(), fmt) + } else { + let Some(pack_pipeline) = P::pack_pipeline_for_format(runtime, fmt) else { + return Err(Error::MetalKernel { + message: format!( + "unsupported JPEG Metal {} pixel format {fmt:?}", + P::FAMILY_NAME + ), + }); + }; + let out_buffer = runtime.device.new_buffer( + (params.out_stride as usize * params.height as usize) as u64, + MTLResourceOptions::StorageModeShared, + ); + let pack_encoder = command_buffer.new_compute_command_encoder(); + pack_encoder.set_compute_pipeline_state(pack_pipeline); + bind_three_plane_pack::( + pack_encoder, + [Some(&y_plane), Some(&cb_plane), Some(&cr_plane)], + &out_buffer, + ¶ms, + ); + dispatch_2d_pipeline(pack_encoder, pack_pipeline, packet.dimensions()); + pack_encoder.end_encoding(); + Surface::from_metal_buffer(out_buffer, packet.dimensions(), fmt) + }; + + Ok(BatchedDecodeItem { + request_index, + surface, + status_buffer: status_buffer.clone(), + decode_threads, + _decode_resources: vec![ + y_plane, + cb_plane, + cr_plane, + entropy_buffer, + restart_offsets_buffer, + entropy_checkpoints_buffer, + status_buffer, + ], + }) +} + +/// Route one batch request to the family's encode item for its op. +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn encode_fast_subsampled_op_batch_item( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + device_buffer_cache: &mut BatchDeviceBufferCache, + request_index: usize, + packet: &P, + fmt: PixelFormat, + op: batch::BatchOp, +) -> Result { + match op { + batch::BatchOp::Full => { + encode_fast_subsampled_batch_item(runtime, command_buffer, request_index, packet, fmt) + } + batch::BatchOp::Region(roi) => encode_fast_subsampled_region_batch_item( + runtime, + command_buffer, + request_index, + packet, + fmt, + roi, + ), + batch::BatchOp::Scaled(scale) => encode_fast_subsampled_scaled_batch_item( + runtime, + command_buffer, + request_index, + packet, + fmt, + scale, + ), + batch::BatchOp::RegionScaled { roi, scale } => { + encode_fast_subsampled_scaled_region_batch_item( + runtime, + command_buffer, + device_buffer_cache, + request_index, + packet, + fmt, + roi, + scale, + ) + } + } +} + +#[cfg(target_os = "macos")] +fn encode_fast444_region_batch_item( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + request_index: usize, + packet: &JpegFast444PacketV1, + mode: PlaneMode, + fmt: PixelFormat, + roi: Rect, +) -> Result { + let roi = core_rect_to_jpeg(roi); + let mut params = fast444_region_params(packet, roi)?; + let (first_mcu, end_mcu) = mcu_range_for_rect(roi, packet.mcus_per_row, packet.mcu_rows, 8, 8); + let total_mcus = packet.mcus_per_row * packet.mcu_rows; + let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( + &packet.restart_offsets, + packet.restart_interval_mcus, + total_mcus, + first_mcu, + end_mcu, + ); + params.restart_start_mcu = restart_start_mcu; + params.restart_offset_count = checked_entropy_segment_count( + packet.restart_interval_mcus, + restart_offsets.len(), + packet.entropy_checkpoints.len(), + )?; + + let plane_len = params.width as usize * params.height as usize; + let y_plane = new_decode_plane_buffer( + &runtime.device, + plane_len, + fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, + ); + let cb_plane = new_private_buffer(&runtime.device, plane_len); + let cr_plane = new_private_buffer(&runtime.device, plane_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus, + restart_offsets.len(), + packet.entropy_checkpoints.len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes.as_ptr().cast(), + packet.entropy_bytes.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; + + let dc_tables = [ + PreparedHuffmanHost::from(&packet.y_dc_table), + PreparedHuffmanHost::from(&packet.cb_dc_table), + PreparedHuffmanHost::from(&packet.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&packet.y_ac_table), + PreparedHuffmanHost::from(&packet.cb_ac_table), + PreparedHuffmanHost::from(&packet.cr_ac_table), + ]; + + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(&runtime.fast444_region_decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &cb_plane, &cr_plane], + ¶ms, + [&packet.y_quant, &packet.cb_quant, &packet.cr_quant], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_region_decode_pipeline, + decode_threads, + ); + decoder_encoder.end_encoding(); + + let surface = encode_jpeg_pack_to_surface_in_command_buffer( + runtime, + command_buffer, + &y_plane, + Some(&cb_plane), + Some(&cr_plane), + (roi.w, roi.h), + mode, + fmt, + )?; + + Ok(BatchedDecodeItem { + request_index, + surface, + status_buffer: status_buffer.clone(), + decode_threads, + _decode_resources: vec![ + y_plane, + cb_plane, + cr_plane, + entropy_buffer, + restart_offsets_buffer, + entropy_checkpoints_buffer, + status_buffer, + ], + }) +} + +#[cfg(target_os = "macos")] +fn encode_fast444_scaled_batch_item( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + request_index: usize, + packet: &JpegFast444PacketV1, + mode: PlaneMode, + fmt: PixelFormat, + scale: j2k_core::Downscale, +) -> Result { + let Some(params) = fast444_scaled_params(packet, scale) else { + return Err(Error::MetalKernel { + message: format!("unsupported JPEG Metal fast444 scale {scale:?}"), + }); + }; + + let plane_len = params.scaled_width as usize * params.scaled_height as usize; + let y_plane = new_decode_plane_buffer( + &runtime.device, + plane_len, + fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, + ); + let cb_plane = new_private_buffer(&runtime.device, plane_len); + let cr_plane = new_private_buffer(&runtime.device, plane_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus, + packet.restart_offsets.len(), + packet.entropy_checkpoints.len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes.as_ptr().cast(), + packet.entropy_bytes.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; + + let dc_tables = [ + PreparedHuffmanHost::from(&packet.y_dc_table), + PreparedHuffmanHost::from(&packet.cb_dc_table), + PreparedHuffmanHost::from(&packet.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&packet.y_ac_table), + PreparedHuffmanHost::from(&packet.cb_ac_table), + PreparedHuffmanHost::from(&packet.cr_ac_table), + ]; + + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(&runtime.fast444_scaled_decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &cb_plane, &cr_plane], + ¶ms, + [&packet.y_quant, &packet.cb_quant, &packet.cr_quant], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_scaled_decode_pipeline, + decode_threads, + ); + decoder_encoder.end_encoding(); + + let surface = encode_jpeg_pack_to_surface_in_command_buffer( + runtime, + command_buffer, + &y_plane, + Some(&cb_plane), + Some(&cr_plane), + (params.scaled_width, params.scaled_height), + mode, + fmt, + )?; + + Ok(BatchedDecodeItem { + request_index, + surface, + status_buffer: status_buffer.clone(), + decode_threads, + _decode_resources: vec![ + y_plane, + cb_plane, + cr_plane, + entropy_buffer, + restart_offsets_buffer, + entropy_checkpoints_buffer, + status_buffer, + ], + }) +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn encode_fast444_scaled_region_batch_item( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + device_buffer_cache: &mut BatchDeviceBufferCache, + request_index: usize, + packet: &JpegFast444PacketV1, + mode: PlaneMode, + fmt: PixelFormat, + roi: Rect, + scale: j2k_core::Downscale, +) -> Result { + let scaled_roi = roi.scaled_covering(scale); + let scaled_roi = j2k_jpeg::Rect { + x: scaled_roi.x, + y: scaled_roi.y, + w: scaled_roi.w, + h: scaled_roi.h, + }; + let Some(mut params) = fast444_scaled_region_params(packet, scale, scaled_roi) else { + return Err(Error::MetalKernel { + message: format!("unsupported JPEG Metal fast444 scaled region {scale:?}"), + }); + }; + let mcu_size = 8u32 >> params.scale_shift; + let (first_mcu, end_mcu) = mcu_range_for_rect( + scaled_roi, + packet.mcus_per_row, + packet.mcu_rows, + mcu_size, + mcu_size, + ); + let total_mcus = packet.mcus_per_row * packet.mcu_rows; + let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( + &packet.restart_offsets, + packet.restart_interval_mcus, + total_mcus, + first_mcu, + end_mcu, + ); + params.restart_start_mcu = restart_start_mcu; + params.restart_offset_count = checked_entropy_segment_count( + packet.restart_interval_mcus, + restart_offsets.len(), + packet.entropy_checkpoints.len(), + )?; + + let plane_len = params.scaled_width as usize * params.scaled_height as usize; + let y_plane = new_decode_plane_buffer( + &runtime.device, + plane_len, + fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, + ); + let cb_plane = new_private_buffer(&runtime.device, plane_len); + let cr_plane = new_private_buffer(&runtime.device, plane_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus, + restart_offsets.len(), + packet.entropy_checkpoints.len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; + let (entropy_buffer, entropy_checkpoints_buffer) = device_buffer_cache.packet_buffers( + runtime, + &packet.entropy_bytes, + &packet.entropy_checkpoints, + )?; + + let dc_tables = [ + PreparedHuffmanHost::from(&packet.y_dc_table), + PreparedHuffmanHost::from(&packet.cb_dc_table), + PreparedHuffmanHost::from(&packet.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&packet.y_ac_table), + PreparedHuffmanHost::from(&packet.cb_ac_table), + PreparedHuffmanHost::from(&packet.cr_ac_table), + ]; + + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(&runtime.fast444_scaled_region_decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &cb_plane, &cr_plane], + ¶ms, + [&packet.y_quant, &packet.cb_quant, &packet.cr_quant], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_scaled_region_decode_pipeline, + decode_threads, + ); + decoder_encoder.end_encoding(); + + let surface = encode_jpeg_pack_to_surface_in_command_buffer( + runtime, + command_buffer, + &y_plane, + Some(&cb_plane), + Some(&cr_plane), + (scaled_roi.w, scaled_roi.h), + mode, + fmt, + )?; + + Ok(BatchedDecodeItem { + request_index, + surface, + status_buffer: status_buffer.clone(), + decode_threads, + _decode_resources: vec![ + y_plane, + cb_plane, + cr_plane, + entropy_buffer, + restart_offsets_buffer, + entropy_checkpoints_buffer, + status_buffer, + ], + }) +} + +#[cfg(target_os = "macos")] +fn encode_fast444_batch_item( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + request_index: usize, + packet: &JpegFast444PacketV1, + mode: PlaneMode, + fmt: PixelFormat, +) -> Result { + let params = fast444_params(packet)?; + let plane_len = params.width as usize * params.height as usize; + let y_plane = new_decode_plane_buffer( + &runtime.device, + plane_len, + fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, + ); + let cb_plane = new_private_buffer(&runtime.device, plane_len); + let cr_plane = new_private_buffer(&runtime.device, plane_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus, + packet.restart_offsets.len(), + packet.entropy_checkpoints.len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes.as_ptr().cast(), + packet.entropy_bytes.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; + + let dc_tables = [ + PreparedHuffmanHost::from(&packet.y_dc_table), + PreparedHuffmanHost::from(&packet.cb_dc_table), + PreparedHuffmanHost::from(&packet.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&packet.y_ac_table), + PreparedHuffmanHost::from(&packet.cb_ac_table), + PreparedHuffmanHost::from(&packet.cr_ac_table), + ]; + + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(&runtime.fast444_decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &cb_plane, &cr_plane], + ¶ms, + [&packet.y_quant, &packet.cb_quant, &packet.cr_quant], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_decode_pipeline, + decode_threads, + ); + decoder_encoder.end_encoding(); + + let surface = encode_jpeg_pack_to_surface_in_command_buffer( + runtime, + command_buffer, + &y_plane, + Some(&cb_plane), + Some(&cr_plane), + packet.dimensions, + mode, + fmt, + )?; + + Ok(BatchedDecodeItem { + request_index, + surface, + status_buffer: status_buffer.clone(), + decode_threads, + _decode_resources: vec![ + y_plane, + cb_plane, + cr_plane, + entropy_buffer, + restart_offsets_buffer, + entropy_checkpoints_buffer, + status_buffer, + ], + }) +} + +#[cfg(target_os = "macos")] +fn checked_u32(value: usize, label: &str) -> Result { + u32::try_from(value).map_err(|_| Error::MetalKernel { + message: format!("JPEG Metal {label} does not fit in u32"), + }) +} + +#[cfg(target_os = "macos")] +fn batch_output_buffer_or_new( + runtime: &MetalRuntime, + output: Option<&crate::MetalBatchOutputBuffer>, + dimensions: (u32, u32), + tile_count: usize, + out_stride: usize, + out_tile_len: usize, +) -> Result { + let Some(output) = output else { + let byte_len = out_tile_len + .checked_mul(tile_count) + .ok_or(BufferError::SizeOverflow { + what: "JPEG Metal batch output bytes", + })?; + let byte_len_u64 = u64::try_from(byte_len).map_err(|_| BufferError::SizeOverflow { + what: "JPEG Metal batch output bytes", + })?; + return Ok(runtime + .device + .new_buffer(byte_len_u64, MTLResourceOptions::StorageModeShared)); + }; + + if output.dimensions() != dimensions + || output.pixel_format() != PixelFormat::Rgb8 + || output.pitch_bytes() != out_stride + || output.tile_stride_bytes() < out_tile_len + { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal batch output buffer shape does not match requested RGB8 tiles", + }); + } + if output.tile_capacity() < tile_count { + return Err(BufferError::OutputTooSmall { + required: output.tile_stride_bytes().checked_mul(tile_count).ok_or( + BufferError::SizeOverflow { + what: "JPEG Metal batch output bytes", + }, + )?, + have: output.byte_len(), + } + .into()); + } + + Ok(output.clone_buffer()) +} + +#[cfg(target_os = "macos")] +type GroupedSurfaceResult = (usize, Result); + +#[cfg(target_os = "macos")] +type GroupedTextureResult = (usize, Result); + +#[cfg(target_os = "macos")] +fn copy_grouped_surfaces_to_output( + runtime: &MetalRuntime, + output: &crate::MetalBatchOutputBuffer, + dimensions: (u32, u32), + out_tile_len: usize, + group_indices: &[usize], + group_results: Vec>, +) -> Result, Error> { + if group_results.len() != group_indices.len() { + return Err(Error::MetalKernel { + message: "JPEG Metal grouped buffer result count mismatch".to_string(), + }); + } + + let output_buffer = output.clone_buffer(); + let mut copies = Vec::<(Buffer, usize, usize)>::new(); + let mut mapped_results = Vec::with_capacity(group_indices.len()); + for (original_index, result) in group_indices.iter().copied().zip(group_results) { + match result { + Ok(surface) => { + let (source, source_offset) = + surface.metal_buffer().ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal grouped buffer source was not Metal-backed" + .to_string(), + })?; + let destination_offset = original_index + .checked_mul(output.tile_stride_bytes()) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal grouped buffer destination offset overflowed" + .to_string(), + })?; + copies.push((source.clone(), source_offset, destination_offset)); + mapped_results.push(( + original_index, + Ok(Surface::from_metal_buffer_offset( + output_buffer.clone(), + dimensions, + PixelFormat::Rgb8, + destination_offset, + )), + )); + } + Err(error) => mapped_results.push((original_index, Err(error))), + } + } + + if !copies.is_empty() { + let command_buffer = runtime.queue.new_command_buffer(); + let blit = command_buffer.new_blit_command_encoder(); + for (source, source_offset, destination_offset) in copies { + blit.copy_from_buffer( + &source, + u64::try_from(source_offset).map_err(|_| Error::MetalKernel { + message: "JPEG Metal grouped buffer source offset exceeds u64".to_string(), + })?, + &output_buffer, + u64::try_from(destination_offset).map_err(|_| Error::MetalKernel { + message: "JPEG Metal grouped buffer destination offset exceeds u64".to_string(), + })?, + u64::try_from(out_tile_len).map_err(|_| Error::MetalKernel { + message: "JPEG Metal grouped buffer copy size exceeds u64".to_string(), + })?, + ); + } + blit.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + } + + Ok(mapped_results) +} + +#[cfg(target_os = "macos")] +fn validate_rgba_texture_batch_output( + output: &crate::MetalBatchTextureOutput, + dimensions: (u32, u32), + tile_count: usize, + out_tile_len: usize, +) -> Result<(), Error> { + if output.dimensions() != dimensions + || output.pixel_format() != PixelFormat::Rgba8 + || output.metal_pixel_format() != MTLPixelFormat::RGBA8Unorm + { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal batch texture output shape does not match requested RGBA8 tiles", + }); + } + if output.tile_capacity() < tile_count { + return Err(BufferError::OutputTooSmall { + required: out_tile_len + .checked_mul(tile_count) + .ok_or(BufferError::SizeOverflow { + what: "JPEG Metal batch texture output bytes", + })?, + have: out_tile_len.checked_mul(output.tile_capacity()).ok_or( + BufferError::SizeOverflow { + what: "JPEG Metal batch texture output bytes", + }, + )?, + } + .into()); + } + + for index in 0..tile_count { + let Some(texture) = output.texture(index) else { + return Err(Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + }); + }; + if texture.width() != u64::from(dimensions.0) + || texture.height() != u64::from(dimensions.1) + || texture.pixel_format() != MTLPixelFormat::RGBA8Unorm + { + return Err(Error::UnsupportedMetalRequest { + reason: + "JPEG Metal batch texture output texture does not match requested RGBA8 tiles", + }); + } + } + + Ok(()) +} + +#[cfg(target_os = "macos")] +fn texture_batch_success_results( + output: &crate::MetalBatchTextureOutput, + dimensions: (u32, u32), + tile_count: usize, +) -> Result>, Error> { + let mut results = Vec::with_capacity(tile_count); + for index in 0..tile_count { + let texture = output + .clone_texture(index) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?; + results.push(Ok(crate::MetalTextureTile::new( + texture, + dimensions, + PixelFormat::Rgba8, + ))); + } + Ok(results) +} + +#[cfg(target_os = "macos")] +fn copy_rgb8_surfaces_to_rgba_textures( + runtime: &MetalRuntime, + output: &crate::MetalBatchTextureOutput, + dimensions: (u32, u32), + tile_count: usize, + group_indices: &[usize], + group_results: Vec>, +) -> Result, Error> { + if group_results.len() != group_indices.len() { + return Err(Error::MetalKernel { + message: "JPEG Metal grouped texture result count mismatch".to_string(), + }); + } + let out_tile_len = dimensions + .0 + .checked_mul(dimensions.1) + .and_then(|pixels| { + pixels.checked_mul(u32::try_from(PixelFormat::Rgba8.bytes_per_pixel()).ok()?) + }) + .ok_or(BufferError::SizeOverflow { + what: "JPEG Metal batch texture output bytes", + })? as usize; + validate_rgba_texture_batch_output(output, dimensions, tile_count, out_tile_len)?; + + let in_stride = dimensions + .0 + .checked_mul( + u32::try_from(PixelFormat::Rgb8.bytes_per_pixel()).map_err(|_| { + BufferError::SizeOverflow { + what: "JPEG Metal RGB texture copy input stride", + } + })?, + ) + .ok_or(BufferError::SizeOverflow { + what: "JPEG Metal RGB texture copy input stride", + })?; + let params = JpegRgb8ToRgbaTextureParams { + width: dimensions.0, + height: dimensions.1, + in_stride, + alpha: u32::from(u8::MAX), + }; + let mut copies = Vec::<(usize, Buffer, usize)>::new(); + let mut mapped_results = Vec::with_capacity(group_indices.len()); + for (original_index, result) in group_indices.iter().copied().zip(group_results) { + match result { + Ok(surface) => { + if surface.dimensions != dimensions || surface.fmt != PixelFormat::Rgb8 { + return Err(Error::MetalKernel { + message: "JPEG Metal texture copy source shape mismatch".to_string(), + }); + } + let (source, source_offset) = + surface.metal_buffer().ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal texture copy source was not Metal-backed".to_string(), + })?; + let texture = + output + .clone_texture(original_index) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?; + copies.push((original_index, source.clone(), source_offset)); + mapped_results.push(( + original_index, + Ok(crate::MetalTextureTile::new( + texture, + dimensions, + PixelFormat::Rgba8, + )), + )); + } + Err(error) => mapped_results.push((original_index, Err(error))), + } + } + + if !copies.is_empty() { + let command_buffer = runtime.queue.new_command_buffer(); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.rgb8_to_rgba_texture_pipeline); + for (original_index, source, source_offset) in copies { + let texture = output + .texture(original_index) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?; + encoder.set_buffer( + 0, + Some(&source), + u64::try_from(source_offset).map_err(|_| Error::MetalKernel { + message: "JPEG Metal texture copy source offset exceeds u64".to_string(), + })?, + ); + encoder.set_bytes( + 1, + size_of::() as u64, + (&raw const params).cast(), + ); + encoder.set_texture(0, Some(texture)); + dispatch_2d_pipeline(encoder, &runtime.rgb8_to_rgba_texture_pipeline, dimensions); + } + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + } + + Ok(mapped_results) +} + +#[cfg(target_os = "macos")] +fn dispatch_rgba_texture_pack( + command_buffer: &CommandBufferRef, + pipeline: &ComputePipelineState, + planes: (&Buffer, &Buffer, &Buffer), + output: &crate::MetalBatchTextureOutput, + params: JpegTexturePackBatchParams, + tile_count: usize, + dispatch_dims: (u32, u32), +) -> Result<(), Error> { + let pack_encoder = command_buffer.new_compute_command_encoder(); + pack_encoder.set_compute_pipeline_state(pipeline); + pack_encoder.set_buffer(0, Some(planes.0), 0); + pack_encoder.set_buffer(1, Some(planes.1), 0); + pack_encoder.set_buffer(2, Some(planes.2), 0); + for index in 0..tile_count { + let texture = output.texture(index).ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?; + let mut params = params; + params.tile_index = checked_u32(index, "texture batch tile index")?; + pack_encoder.set_texture(0, Some(texture)); + pack_encoder.set_bytes( + 3, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_2d_pipeline(pack_encoder, pipeline, dispatch_dims); + } + pack_encoder.end_encoding(); + Ok(()) +} + +#[cfg(target_os = "macos")] +fn dispatch_windowed_rgba_texture_pack( + command_buffer: &CommandBufferRef, + pipeline: &ComputePipelineState, + planes: (&Buffer, &Buffer, &Buffer), + output: &crate::MetalBatchTextureOutput, + params: JpegWindowedTexturePackBatchParams, + tile_count: usize, + dispatch_dims: (u32, u32), +) -> Result<(), Error> { + let pack_encoder = command_buffer.new_compute_command_encoder(); + pack_encoder.set_compute_pipeline_state(pipeline); + pack_encoder.set_buffer(0, Some(planes.0), 0); + pack_encoder.set_buffer(1, Some(planes.1), 0); + pack_encoder.set_buffer(2, Some(planes.2), 0); + for index in 0..tile_count { + let texture = output.texture(index).ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?; + let mut params = params; + params.tile_index = checked_u32(index, "windowed texture batch tile index")?; + pack_encoder.set_texture(0, Some(texture)); + pack_encoder.set_bytes( + 3, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_2d_pipeline(pack_encoder, pipeline, dispatch_dims); + } + pack_encoder.end_encoding(); + Ok(()) +} + +/// Encode the split coeff-decode + IDCT-deposit passes shared by the surfaces +/// and texture drivers' `SplitCoeffIdct` debug mode. +#[cfg(all(target_os = "macos", test))] +#[allow(clippy::too_many_arguments)] +fn encode_split_coeff_idct_passes( + command_buffer: &CommandBufferRef, + pipelines: (&ComputePipelineState, &ComputePipelineState), + params: &JpegFast420BatchParams, + quants: [&[u16; 64]; 3], + dc_tables: &[PreparedHuffmanHost; 3], + ac_tables: &[PreparedHuffmanHost; 3], + entropy: (&Buffer, &Buffer, &Buffer, &Buffer), + status_buffer: &Buffer, + planes: [&Buffer; 3], + scratch: (&Buffer, &Buffer), + total_decode_threads: u32, + idct_grid: (u32, u32, u32), +) { + let (coeffs_pipeline, idct_pipeline) = pipelines; + let (entropy_payload, entropy_offsets, entropy_lens, entropy_checkpoints) = entropy; + let (coeff_blocks, dc_only_flags) = scratch; + + let coeff_encoder = command_buffer.new_compute_command_encoder(); + coeff_encoder.set_compute_pipeline_state(coeffs_pipeline); + coeff_encoder.set_buffer(0, Some(entropy_payload), 0); + coeff_encoder.set_buffer(1, Some(coeff_blocks), 0); + coeff_encoder.set_buffer(2, Some(dc_only_flags), 0); + coeff_encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const *params).cast(), + ); + coeff_encoder.set_bytes(5, size_of::<[u16; 64]>() as u64, quants[0].as_ptr().cast()); + coeff_encoder.set_bytes(6, size_of::<[u16; 64]>() as u64, quants[1].as_ptr().cast()); + coeff_encoder.set_bytes(7, size_of::<[u16; 64]>() as u64, quants[2].as_ptr().cast()); + coeff_encoder.set_bytes( + 8, + size_of::() as u64, + (&raw const dc_tables[0]).cast(), + ); + coeff_encoder.set_bytes( + 9, + size_of::() as u64, + (&raw const ac_tables[0]).cast(), + ); + coeff_encoder.set_bytes( + 10, + size_of::() as u64, + (&raw const dc_tables[1]).cast(), + ); + coeff_encoder.set_bytes( + 11, + size_of::() as u64, + (&raw const ac_tables[1]).cast(), + ); + coeff_encoder.set_bytes( + 12, + size_of::() as u64, + (&raw const dc_tables[2]).cast(), + ); + coeff_encoder.set_bytes( + 13, + size_of::() as u64, + (&raw const ac_tables[2]).cast(), + ); + coeff_encoder.set_buffer(14, Some(entropy_offsets), 0); + coeff_encoder.set_buffer(15, Some(entropy_lens), 0); + coeff_encoder.set_buffer(16, Some(status_buffer), 0); + coeff_encoder.set_buffer(17, Some(entropy_checkpoints), 0); + dispatch_1d_pipeline(coeff_encoder, coeffs_pipeline, total_decode_threads); + coeff_encoder.end_encoding(); + + let idct_encoder = command_buffer.new_compute_command_encoder(); + idct_encoder.set_compute_pipeline_state(idct_pipeline); + idct_encoder.set_buffer(0, Some(coeff_blocks), 0); + idct_encoder.set_buffer(1, Some(dc_only_flags), 0); + idct_encoder.set_buffer(2, Some(planes[0]), 0); + idct_encoder.set_buffer(3, Some(planes[1]), 0); + idct_encoder.set_buffer(4, Some(planes[2]), 0); + idct_encoder.set_bytes( + 5, + size_of::() as u64, + (&raw const *params).cast(), + ); + dispatch_3d_pipeline(idct_encoder, idct_pipeline, idct_grid); + idct_encoder.end_encoding(); +} + +#[cfg(target_os = "macos")] +fn try_decode_fast_subsampled_full_rgb_batch_to_surfaces( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], +) -> Result>>, Error> { + try_decode_fast_subsampled_full_rgb_batch_to_surfaces_with_mode_and_output::

( + runtime, + requests, + packets, + fast_batch_decode_mode(), + None, + ) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast_subsampled_full_rgb_batch_to_surfaces_into_output( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchOutputBuffer, +) -> Result>>, Error> { + try_decode_fast_subsampled_full_rgb_batch_to_surfaces_with_mode_and_output::

( + runtime, + requests, + packets, + fast_batch_decode_mode(), + Some(output), + ) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast_subsampled_full_rgb_batch_to_surfaces_with_mode_and_output< + P: FastSubsampledMetal, +>( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + decode_mode: FastBatchDecodeMode, + output: Option<&crate::MetalBatchOutputBuffer>, +) -> Result>>, Error> { + let timing_enabled = + decode_mode == FastBatchDecodeMode::Fused && P::full_rgb_batch_timing_enabled(); + let timing_total_start = timing_enabled.then(Instant::now); + let mut timing = FastBatchTiming::default(); + + if requests.is_empty() + || requests + .iter() + .any(|request| request.op != batch::BatchOp::Full || request.fmt != PixelFormat::Rgb8) + { + return Ok(None); + } + + let mut family_packets = Vec::with_capacity(packets.len()); + for packet in packets { + let Some(packet) = P::from_batched(packet) else { + return Ok(None); + }; + family_packets.push(packet); + } + + let Some(first) = family_packets.first().copied() else { + return Ok(None); + }; + if (!P::FULL_RGB_BATCH_SUPPORTS_RESTART && first.restart_interval_mcus() != 0) + || first.entropy_checkpoints().is_empty() + { + return Ok(None); + } + + let Some(groups) = fast_subsampled_full_rgb_batch_groups(&family_packets) else { + return Ok(None); + }; + if groups.len() > 1 { + return try_decode_grouped_fast_subsampled_full_rgb_batch_to_surfaces_with_output::

( + runtime, + requests, + &family_packets, + decode_mode, + output, + groups, + ); + } + + let segment_count = first.entropy_checkpoints().len(); + if !family_packets.iter().all(|packet| { + fast_subsampled_packets_share_full_rgb_batch_shape(first, packet, segment_count) + }) { + return Ok(None); + } + + let tile_count = family_packets.len(); + let tile_count_u32 = checked_u32(tile_count, "batch tile count")?; + let segment_count_u32 = checked_u32(segment_count, "batch segment count")?; + let total_decode_threads = checked_u32( + tile_count + .checked_mul(segment_count) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch decode thread count overflowed".to_string(), + })?, + "batch decode thread count", + )?; + + let width = first.dimensions().0; + let height = first.dimensions().1; + let chroma_width = width.div_ceil(2); + let chroma_height = P::chroma_height(height); + let y_len = width as usize * height as usize; + let chroma_len = chroma_width as usize * chroma_height as usize; + let out_stride = width as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let out_tile_len = out_stride * height as usize; + #[cfg_attr(not(test), allow(unused_variables))] + let total_blocks = match P::FULL_RGB_BATCH_BLOCKS_PER_MCU { + Some(blocks_per_mcu) => { + let total_mcus = first.mcus_per_row() as usize * first.mcu_rows() as usize; + let blocks_per_tile = + total_mcus + .checked_mul(blocks_per_mcu) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal {} batch block count overflowed", + P::FAMILY_NAME + ), + })?; + let total_blocks = + blocks_per_tile + .checked_mul(tile_count) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal {} batch total block count overflowed", + P::FAMILY_NAME + ), + })?; + let _total_blocks_u32 = checked_u32( + total_blocks, + &format!("{} batch block count", P::FAMILY_NAME), + )?; + Some(total_blocks) + } + None => None, + }; + + let params = JpegFast420BatchParams { + width, + height, + chroma_width, + chroma_height, + mcus_per_row: first.mcus_per_row(), + mcu_rows: first.mcu_rows(), + segment_count: segment_count_u32, + tile_count: tile_count_u32, + out_stride: checked_u32(out_stride, "batch output stride")?, + alpha: u32::from(u8::MAX), + }; + if timing_enabled { + timing.accepted = timing_total_start + .expect("timing start is set when timing is enabled") + .elapsed(); + } + + let timing_entropy_start = timing_enabled.then(Instant::now); + let total_entropy_len = family_packets + .iter() + .map(|packet| packet.entropy_bytes().len()) + .try_fold(0usize, usize::checked_add) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch entropy length overflowed".to_string(), + })?; + if total_entropy_len == 0 { + return Ok(None); + } + + let mut entropy_bytes = Vec::with_capacity(total_entropy_len); + let mut entropy_offsets = Vec::with_capacity(tile_count); + let mut entropy_lens = Vec::with_capacity(tile_count); + let mut entropy_checkpoints = Vec::with_capacity(tile_count * segment_count); + for packet in &family_packets { + entropy_offsets.push(checked_u32(entropy_bytes.len(), "batch entropy offset")?); + entropy_lens.push(checked_u32( + packet.entropy_bytes().len(), + "batch entropy length", + )?); + entropy_bytes.extend_from_slice(packet.entropy_bytes()); + entropy_checkpoints.extend(packet.entropy_checkpoints().iter().copied()); + } + if timing_enabled { + timing.entropy_concat = timing_entropy_start + .expect("timing start is set when timing is enabled") + .elapsed(); + } + + let timing_buffer_start = timing_enabled.then(Instant::now); + let mut batch_scratch = runtime.batch_scratch()?; + let y_plane = + batch_scratch.private_buffer(&runtime.device, P::FULL_BATCH_KEYS.y, y_len * tile_count); + let cb_plane = batch_scratch.private_buffer( + &runtime.device, + P::FULL_BATCH_KEYS.cb, + chroma_len * tile_count, + ); + let cr_plane = batch_scratch.private_buffer( + &runtime.device, + P::FULL_BATCH_KEYS.cr, + chroma_len * tile_count, + ); + let out_buffer = batch_output_buffer_or_new( + runtime, + output, + first.dimensions(), + tile_count, + out_stride, + out_tile_len, + )?; + let statuses = vec![JpegDecodeStatus::default(); total_decode_threads as usize]; + let checkpoint_hosts = entropy_checkpoint_hosts(&entropy_checkpoints)?; + let status_buffer = batch_scratch.shared_buffer_with_slice( + &runtime.device, + P::FULL_BATCH_KEYS.status, + &statuses, + ); + let entropy_buffer = batch_scratch.shared_buffer_with_bytes( + &runtime.device, + P::FULL_BATCH_KEYS.entropy, + &entropy_bytes, + ); + let entropy_offsets_buffer = batch_scratch.shared_buffer_with_slice( + &runtime.device, + P::FULL_BATCH_KEYS.entropy_offsets, + &entropy_offsets, + ); + let entropy_lens_buffer = batch_scratch.shared_buffer_with_slice( + &runtime.device, + P::FULL_BATCH_KEYS.entropy_lens, + &entropy_lens, + ); + let entropy_checkpoints_buffer = batch_scratch.shared_buffer_with_slice( + &runtime.device, + P::FULL_BATCH_KEYS.entropy_checkpoints, + &checkpoint_hosts, + ); + if timing_enabled { + timing.buffer_alloc = timing_buffer_start + .expect("timing start is set when timing is enabled") + .elapsed(); + } + + let dc_tables = [ + PreparedHuffmanHost::from(first.y_dc_table()), + PreparedHuffmanHost::from(first.cb_dc_table()), + PreparedHuffmanHost::from(first.cr_dc_table()), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(first.y_ac_table()), + PreparedHuffmanHost::from(first.cb_ac_table()), + PreparedHuffmanHost::from(first.cr_ac_table()), + ]; + + let mut command_buffer = runtime.queue.new_command_buffer(); + #[cfg(test)] + let mut split_scratch: Option<(Buffer, Buffer)> = None; + match decode_mode { + FastBatchDecodeMode::Fused => { + let timing_encode_start = timing_enabled.then(Instant::now); + let decode_pipeline = P::full_rgb_batch_decode_pipeline(runtime); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &cb_plane, &cr_plane], + ¶ms, + [first.y_quant(), first.cb_quant(), first.cr_quant()], + &dc_tables, + &ac_tables, + &entropy_offsets_buffer, + &entropy_lens_buffer, + &status_buffer, + ); + decoder_encoder.set_buffer(17, Some(&entropy_checkpoints_buffer), 0); + dispatch_1d_pipeline(decoder_encoder, decode_pipeline, total_decode_threads); + decoder_encoder.end_encoding(); + if timing_enabled { + timing.encode_decode = timing_encode_start + .expect("timing start is set when timing is enabled") + .elapsed(); + command_buffer.commit(); + let timing_wait_start = Instant::now(); + command_buffer.wait_until_completed(); + timing.wait_decode = timing_wait_start.elapsed(); + command_buffer = runtime.queue.new_command_buffer(); + } + } + #[cfg(test)] + FastBatchDecodeMode::SplitCoeffIdct => { + let Some((split, total_blocks)) = + P::split_coeff_idct_pipelines(runtime).zip(total_blocks) + else { + return Err(Error::MetalKernel { + message: format!( + "JPEG Metal {} batch split coeff/IDCT decode mode is unsupported", + P::FAMILY_NAME + ), + }); + }; + let coeff_bytes = total_blocks + .checked_mul(64) + .and_then(|bytes| bytes.checked_mul(size_of::())) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal {} batch coefficient scratch overflowed", + P::FAMILY_NAME + ), + })?; + let idct_component_depth = + tile_count_u32 + .checked_mul(6) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal {} batch IDCT dispatch overflowed", + P::FAMILY_NAME + ), + })?; + let coeff_blocks = runtime + .device + .new_buffer(coeff_bytes as u64, MTLResourceOptions::StorageModePrivate); + let dc_only_flags = runtime + .device + .new_buffer(total_blocks as u64, MTLResourceOptions::StorageModePrivate); + + encode_split_coeff_idct_passes( + command_buffer, + split, + ¶ms, + [first.y_quant(), first.cb_quant(), first.cr_quant()], + &dc_tables, + &ac_tables, + ( + &entropy_buffer, + &entropy_offsets_buffer, + &entropy_lens_buffer, + &entropy_checkpoints_buffer, + ), + &status_buffer, + [&y_plane, &cb_plane, &cr_plane], + (&coeff_blocks, &dc_only_flags), + total_decode_threads, + (first.mcus_per_row(), first.mcu_rows(), idct_component_depth), + ); + + split_scratch = Some((coeff_blocks, dc_only_flags)); + } + } + + let timing_pack_encode_start = timing_enabled.then(Instant::now); + let pack_pipeline = P::pack_full_rgb_batch_pipeline(runtime); + let pack_encoder = command_buffer.new_compute_command_encoder(); + pack_encoder.set_compute_pipeline_state(pack_pipeline); + bind_three_plane_pack::( + pack_encoder, + [Some(&y_plane), Some(&cb_plane), Some(&cr_plane)], + &out_buffer, + ¶ms, + ); + dispatch_3d_pipeline( + pack_encoder, + pack_pipeline, + ( + packed_pair_extent(width), + P::packed_height_extent(height), + tile_count_u32, + ), + ); + pack_encoder.end_encoding(); + if timing_enabled { + timing.encode_pack = timing_pack_encode_start + .expect("timing start is set when timing is enabled") + .elapsed(); + } + + command_buffer.commit(); + if timing_enabled { + let timing_wait_start = Instant::now(); + command_buffer.wait_until_completed(); + timing.wait_pack = timing_wait_start.elapsed(); + timing.total = timing_total_start + .expect("timing start is set when timing is enabled") + .elapsed(); + timing.log( + P::FULL_RGB_BATCH_TIMING_TAG, + "fused-stages", + tile_count, + first.dimensions(), + segment_count, + ); + } else { + command_buffer.wait_until_completed(); + } + #[cfg(test)] + drop(split_scratch); + drop(batch_scratch); + + if let Some(status) = first_decode_error_status(&status_buffer, total_decode_threads) { + let mut results = Vec::with_capacity(requests.len()); + for request in requests { + let decoder = CpuDecoder::new(request.input.as_ref())?; + results.push(Err(decode_error_from_cpu(&decoder, request.fmt, status))); + } + return Ok(Some(results)); + } + + let mut results = Vec::with_capacity(requests.len()); + for index in 0..requests.len() { + results.push(Ok(Surface::from_metal_buffer_offset( + out_buffer.clone(), + first.dimensions(), + PixelFormat::Rgb8, + index * out_tile_len, + ))); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_grouped_fast_subsampled_full_rgb_batch_to_surfaces_with_output< + P: FastSubsampledMetal, +>( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + family_packets: &[&P], + decode_mode: FastBatchDecodeMode, + output: Option<&crate::MetalBatchOutputBuffer>, + groups: Vec>, +) -> Result>>, Error> { + if let Some(output) = output { + for packet in family_packets { + let out_stride = packet.dimensions().0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let out_tile_len = out_stride * packet.dimensions().1 as usize; + batch_output_buffer_or_new( + runtime, + Some(output), + packet.dimensions(), + requests.len(), + out_stride, + out_tile_len, + )?; + } + } + + let mut merged_results: Vec>> = + (0..requests.len()).map(|_| None).collect(); + for group_indices in groups { + let group_requests = group_indices + .iter() + .map(|&index| requests[index].clone()) + .collect::>(); + let group_packets = group_indices + .iter() + .map(|&index| family_packets[index].to_batched()) + .collect::>(); + + let Some(group_results) = + try_decode_fast_subsampled_full_rgb_batch_to_surfaces_with_mode_and_output::

( + runtime, + &group_requests, + &group_packets, + decode_mode, + None, + )? + else { + return Ok(None); + }; + + if let Some(output) = output { + let Some(&first_group_index) = group_indices.first() else { + continue; + }; + let packet = family_packets[first_group_index]; + let out_stride = packet.dimensions().0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let out_tile_len = out_stride * packet.dimensions().1 as usize; + for (original_index, result) in copy_grouped_surfaces_to_output( + runtime, + output, + packet.dimensions(), + out_tile_len, + &group_indices, + group_results, + )? { + merged_results[original_index] = Some(result); + } + } else { + if group_results.len() != group_indices.len() { + return Err(Error::MetalKernel { + message: format!( + "JPEG Metal grouped {} buffer result count mismatch", + P::FAMILY_NAME + ), + }); + } + for (original_index, result) in group_indices.into_iter().zip(group_results) { + merged_results[original_index] = Some(result); + } + } + } + + let mut results = Vec::with_capacity(requests.len()); + for (index, result) in merged_results.into_iter().enumerate() { + results.push(result.ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal grouped {} buffer result for tile {index} was missing", + P::FAMILY_NAME + ), + })?); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast_subsampled_full_rgba_batch_to_textures( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchTextureOutput, + decode_mode: FastBatchDecodeMode, +) -> Result>>, Error> { + if requests.is_empty() + || requests + .iter() + .any(|request| request.op != batch::BatchOp::Full || request.fmt != PixelFormat::Rgb8) + { + return Ok(None); + } + + let mut family_packets = Vec::with_capacity(packets.len()); + for packet in packets { + let Some(packet) = P::from_batched(packet) else { + return Ok(None); + }; + family_packets.push(packet); + } + + let Some(first) = family_packets.first().copied() else { + return Ok(None); + }; + if (!P::FULL_RGB_BATCH_SUPPORTS_RESTART && first.restart_interval_mcus() != 0) + || first.entropy_checkpoints().is_empty() + { + return Ok(None); + } + + let Some(groups) = fast_subsampled_full_rgb_batch_groups(&family_packets) else { + return Ok(None); + }; + if groups.len() > 1 { + return try_decode_grouped_fast_subsampled_full_rgba_batch_to_textures::

( + runtime, + requests, + &family_packets, + output, + decode_mode, + groups, + ); + } + + let segment_count = first.entropy_checkpoints().len(); + let tile_count = family_packets.len(); + let tile_count_u32 = checked_u32( + tile_count, + &format!("{} texture batch tile count", P::FAMILY_NAME), + )?; + let segment_count_u32 = checked_u32( + segment_count, + &format!("{} texture batch segment count", P::FAMILY_NAME), + )?; + let total_decode_threads = checked_u32( + tile_count + .checked_mul(segment_count) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal {} texture batch decode thread count overflowed", + P::FAMILY_NAME + ), + })?, + &format!("{} texture batch decode thread count", P::FAMILY_NAME), + )?; + + let width = first.dimensions().0; + let height = first.dimensions().1; + let chroma_width = width.div_ceil(2); + let chroma_height = P::chroma_height(height); + let y_len = width as usize * height as usize; + let chroma_len = chroma_width as usize * chroma_height as usize; + let out_stride = width as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let out_tile_len = out_stride * height as usize; + validate_rgba_texture_batch_output(output, first.dimensions(), tile_count, out_tile_len)?; + + let total_mcus = first.mcus_per_row() as usize * first.mcu_rows() as usize; + let mcu_threads = P::texture_mcu_dispatch_threads(total_mcus)?; + #[cfg(test)] + let total_blocks = match P::FULL_RGB_BATCH_BLOCKS_PER_MCU { + Some(blocks_per_mcu) => { + let blocks_per_tile = + total_mcus + .checked_mul(blocks_per_mcu) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal {} texture batch block count overflowed", + P::FAMILY_NAME + ), + })?; + Some( + blocks_per_tile + .checked_mul(tile_count) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal {} texture batch total block count overflowed", + P::FAMILY_NAME + ), + })?, + ) + } + None => None, + }; + + let params = JpegFast420BatchParams { + width, + height, + chroma_width, + chroma_height, + mcus_per_row: first.mcus_per_row(), + mcu_rows: first.mcu_rows(), + segment_count: segment_count_u32, + tile_count: tile_count_u32, + out_stride: checked_u32( + out_stride, + &format!("{} texture batch output stride", P::FAMILY_NAME), + )?, + alpha: u32::from(u8::MAX), + }; + + let mut batch_scratch = runtime.batch_scratch()?; + let Some(entropy_buffers) = batch_entropy_buffers( + runtime, + &mut batch_scratch, + BatchEntropyBufferKeys { + payload: P::TEXTURE_KEYS.entropy, + offsets: P::TEXTURE_KEYS.entropy_offsets, + lens: P::TEXTURE_KEYS.entropy_lens, + checkpoints: P::TEXTURE_KEYS.entropy_checkpoints, + }, + family_packets.iter().map(|packet| packet.entropy_bytes()), + family_packets + .iter() + .map(|packet| packet.entropy_checkpoints()), + tile_count, + segment_count, + )? + else { + return Ok(None); + }; + + // Chroma reconstruction needs neighboring samples at MCU boundaries (4:2:0 + // repairs both axes with per-MCU records, 4:2:2 repairs horizontal + // boundaries per entropy segment). The fused path carries same-segment + // boundaries in-thread and resolves cross-segment boundaries from compact + // shared records before returning the caller-owned texture. + if decode_mode == FastBatchDecodeMode::Fused { + let statuses = vec![JpegDecodeStatus::default(); total_decode_threads as usize]; + let status_buffer = batch_scratch.shared_buffer_with_slice( + &runtime.device, + P::TEXTURE_KEYS.status, + &statuses, + ); + let total_repair_records = + P::texture_repair_record_count(tile_count, total_mcus, total_decode_threads)?; + let boundary_meta = vec![0u32; total_repair_records * P::TEXTURE_BOUNDARY_META_WORDS]; + let boundary_samples = vec![0u8; total_repair_records * P::TEXTURE_BOUNDARY_SAMPLE_BYTES]; + let boundary_meta_buffer = batch_scratch.shared_buffer_with_slice( + &runtime.device, + P::TEXTURE_BOUNDARY_META_KEY, + &boundary_meta, + ); + let boundary_samples_buffer = batch_scratch.shared_buffer_with_bytes( + &runtime.device, + P::TEXTURE_BOUNDARY_SAMPLES_KEY, + &boundary_samples, + ); + let vertical_buffers = match &P::TEXTURE_VERTICAL_REPAIR { + Some(spec) => { + let vertical_meta = vec![0u32; total_repair_records * spec.meta_words]; + let vertical_samples = vec![0u8; total_repair_records * spec.sample_bytes]; + let vertical_meta_buffer = batch_scratch.shared_buffer_with_slice( + &runtime.device, + spec.meta_key, + &vertical_meta, + ); + let vertical_samples_buffer = batch_scratch.shared_buffer_with_bytes( + &runtime.device, + spec.samples_key, + &vertical_samples, + ); + Some((vertical_meta_buffer, vertical_samples_buffer)) + } + None => None, + }; + let dc_tables = [ + PreparedHuffmanHost::from(first.y_dc_table()), + PreparedHuffmanHost::from(first.cb_dc_table()), + PreparedHuffmanHost::from(first.cr_dc_table()), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(first.y_ac_table()), + PreparedHuffmanHost::from(first.cb_ac_table()), + PreparedHuffmanHost::from(first.cr_ac_table()), + ]; + + let tile_index_ctx = format!("{} texture batch tile index", P::FAMILY_NAME); + let texture_decode_pipeline = P::rgba_texture_batch_decode_pipeline(runtime); + let command_buffer = runtime.queue.new_command_buffer(); + for index in 0..tile_count { + let texture = output.texture(index).ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?; + let decode_params = JpegFast420TextureBatchParams { + width, + height, + chroma_width, + chroma_height, + mcus_per_row: first.mcus_per_row(), + mcu_rows: first.mcu_rows(), + segment_count: segment_count_u32, + tile_index: checked_u32(index, &tile_index_ctx)?, + alpha: u32::from(u8::MAX), + }; + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(texture_decode_pipeline); + decoder_encoder.set_buffer(0, Some(&entropy_buffers.payload), 0); + decoder_encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const decode_params).cast(), + ); + decoder_encoder.set_bytes( + 5, + size_of::<[u16; 64]>() as u64, + first.y_quant().as_ptr().cast(), + ); + decoder_encoder.set_bytes( + 6, + size_of::<[u16; 64]>() as u64, + first.cb_quant().as_ptr().cast(), + ); + decoder_encoder.set_bytes( + 7, + size_of::<[u16; 64]>() as u64, + first.cr_quant().as_ptr().cast(), + ); + decoder_encoder.set_bytes( + 8, + size_of::() as u64, + (&raw const dc_tables[0]).cast(), + ); + decoder_encoder.set_bytes( + 9, + size_of::() as u64, + (&raw const ac_tables[0]).cast(), + ); + decoder_encoder.set_bytes( + 10, + size_of::() as u64, + (&raw const dc_tables[1]).cast(), + ); + decoder_encoder.set_bytes( + 11, + size_of::() as u64, + (&raw const ac_tables[1]).cast(), + ); + decoder_encoder.set_bytes( + 12, + size_of::() as u64, + (&raw const dc_tables[2]).cast(), + ); + decoder_encoder.set_bytes( + 13, + size_of::() as u64, + (&raw const ac_tables[2]).cast(), + ); + decoder_encoder.set_buffer(14, Some(&entropy_buffers.offsets), 0); + decoder_encoder.set_buffer(15, Some(&entropy_buffers.lens), 0); + decoder_encoder.set_buffer(16, Some(&status_buffer), 0); + decoder_encoder.set_buffer(17, Some(&entropy_buffers.checkpoints), 0); + decoder_encoder.set_buffer(18, Some(&boundary_meta_buffer), 0); + decoder_encoder.set_buffer(19, Some(&boundary_samples_buffer), 0); + if let Some((vertical_meta_buffer, vertical_samples_buffer)) = &vertical_buffers { + decoder_encoder.set_buffer(20, Some(vertical_meta_buffer), 0); + decoder_encoder.set_buffer(21, Some(vertical_samples_buffer), 0); + } + decoder_encoder.set_texture(0, Some(texture)); + dispatch_1d_pipeline(decoder_encoder, texture_decode_pipeline, segment_count_u32); + decoder_encoder.end_encoding(); + } + if let Some(repair_threads) = + P::horizontal_repair_threads(first, segment_count_u32, mcu_threads) + { + let boundary_pipeline = P::rgba_texture_boundary_pipeline(runtime); + for index in 0..tile_count { + let texture = output.texture(index).ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?; + let decode_params = JpegFast420TextureBatchParams { + width, + height, + chroma_width, + chroma_height, + mcus_per_row: first.mcus_per_row(), + mcu_rows: first.mcu_rows(), + segment_count: segment_count_u32, + tile_index: checked_u32(index, &tile_index_ctx)?, + alpha: u32::from(u8::MAX), + }; + let boundary_encoder = command_buffer.new_compute_command_encoder(); + boundary_encoder.set_compute_pipeline_state(boundary_pipeline); + boundary_encoder.set_buffer(0, Some(&boundary_meta_buffer), 0); + boundary_encoder.set_buffer(1, Some(&boundary_samples_buffer), 0); + boundary_encoder.set_bytes( + 2, + size_of::() as u64, + (&raw const decode_params).cast(), + ); + boundary_encoder.set_texture(0, Some(texture)); + dispatch_1d_pipeline(boundary_encoder, boundary_pipeline, repair_threads); + boundary_encoder.end_encoding(); + } + } + P::encode_extra_texture_repair_passes( + runtime, + &FastTextureRepairCtx { + command_buffer, + output, + boundary_meta_buffer: &boundary_meta_buffer, + vertical_buffers: vertical_buffers.as_ref(), + decode_params: JpegFast420TextureBatchParams { + width, + height, + chroma_width, + chroma_height, + mcus_per_row: first.mcus_per_row(), + mcu_rows: first.mcu_rows(), + segment_count: segment_count_u32, + tile_index: 0, + alpha: u32::from(u8::MAX), + }, + tile_count, + mcu_threads, + tile_index_ctx: &tile_index_ctx, + }, + )?; + + command_buffer.commit(); + command_buffer.wait_until_completed(); + drop(batch_scratch); + + if let Some(results) = + texture_batch_error_results(requests, &status_buffer, total_decode_threads)? + { + return Ok(Some(results)); + } + + return Ok(Some(texture_batch_success_results( + output, + first.dimensions(), + requests.len(), + )?)); + } + + let y_plane = + batch_scratch.private_buffer(&runtime.device, P::TEXTURE_KEYS.y, y_len * tile_count); + let cb_plane = + batch_scratch.private_buffer(&runtime.device, P::TEXTURE_KEYS.cb, chroma_len * tile_count); + let cr_plane = + batch_scratch.private_buffer(&runtime.device, P::TEXTURE_KEYS.cr, chroma_len * tile_count); + let statuses = vec![JpegDecodeStatus::default(); total_decode_threads as usize]; + let status_buffer = + batch_scratch.shared_buffer_with_slice(&runtime.device, P::TEXTURE_KEYS.status, &statuses); + let dc_tables = [ + PreparedHuffmanHost::from(first.y_dc_table()), + PreparedHuffmanHost::from(first.cb_dc_table()), + PreparedHuffmanHost::from(first.cr_dc_table()), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(first.y_ac_table()), + PreparedHuffmanHost::from(first.cb_ac_table()), + PreparedHuffmanHost::from(first.cr_ac_table()), + ]; + + let command_buffer = runtime.queue.new_command_buffer(); + match decode_mode { + FastBatchDecodeMode::Fused => { + let decode_pipeline = P::full_rgb_batch_decode_pipeline(runtime); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffers.payload, + [&y_plane, &cb_plane, &cr_plane], + ¶ms, + [first.y_quant(), first.cb_quant(), first.cr_quant()], + &dc_tables, + &ac_tables, + &entropy_buffers.offsets, + &entropy_buffers.lens, + &status_buffer, + ); + decoder_encoder.set_buffer(17, Some(&entropy_buffers.checkpoints), 0); + dispatch_1d_pipeline(decoder_encoder, decode_pipeline, total_decode_threads); + decoder_encoder.end_encoding(); + } + #[cfg(test)] + FastBatchDecodeMode::SplitCoeffIdct => { + let Some((split, total_blocks)) = + P::split_coeff_idct_pipelines(runtime).zip(total_blocks) + else { + return Err(Error::MetalKernel { + message: format!( + "JPEG Metal {} texture batch split coeff/IDCT decode mode is unsupported", + P::FAMILY_NAME + ), + }); + }; + let coeff_bytes = total_blocks + .checked_mul(64) + .and_then(|bytes| bytes.checked_mul(size_of::())) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal {} texture batch coefficient scratch overflowed", + P::FAMILY_NAME + ), + })?; + let idct_component_depth = + tile_count_u32 + .checked_mul(6) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal {} texture batch IDCT dispatch overflowed", + P::FAMILY_NAME + ), + })?; + let coeff_blocks = batch_scratch.private_buffer( + &runtime.device, + P::SPLIT_TEXTURE_SCRATCH_KEYS.0, + coeff_bytes, + ); + let dc_only_flags = batch_scratch.private_buffer( + &runtime.device, + P::SPLIT_TEXTURE_SCRATCH_KEYS.1, + total_blocks, + ); + + encode_split_coeff_idct_passes( + command_buffer, + split, + ¶ms, + [first.y_quant(), first.cb_quant(), first.cr_quant()], + &dc_tables, + &ac_tables, + ( + &entropy_buffers.payload, + &entropy_buffers.offsets, + &entropy_buffers.lens, + &entropy_buffers.checkpoints, + ), + &status_buffer, + [&y_plane, &cb_plane, &cr_plane], + (&coeff_blocks, &dc_only_flags), + total_decode_threads, + (first.mcus_per_row(), first.mcu_rows(), idct_component_depth), + ); + } + } + + let pack_params = JpegTexturePackBatchParams { + width, + height, + chroma_width, + chroma_height, + tile_index: 0, + alpha: u32::from(u8::MAX), + mode: MODE_YCBCR, + }; + dispatch_rgba_texture_pack( + command_buffer, + P::pack_rgba_texture_pipeline(runtime), + (&y_plane, &cb_plane, &cr_plane), + output, + pack_params, + tile_count, + (packed_pair_extent(width), P::packed_height_extent(height)), + )?; + + command_buffer.commit(); + command_buffer.wait_until_completed(); + drop(batch_scratch); + + if let Some(results) = + texture_batch_error_results(requests, &status_buffer, total_decode_threads)? + { + return Ok(Some(results)); + } + + Ok(Some(texture_batch_success_results( + output, + first.dimensions(), + requests.len(), + )?)) +} + +#[cfg(target_os = "macos")] +fn try_decode_grouped_fast_subsampled_full_rgba_batch_to_textures( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + family_packets: &[&P], + output: &crate::MetalBatchTextureOutput, + decode_mode: FastBatchDecodeMode, + groups: Vec>, +) -> Result>>, Error> { + for packet in family_packets { + let out_stride = packet.dimensions().0 as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let out_tile_len = out_stride * packet.dimensions().1 as usize; + validate_rgba_texture_batch_output( + output, + packet.dimensions(), + requests.len(), + out_tile_len, + )?; + } + + let mut merged_results: Vec>> = + (0..requests.len()).map(|_| None).collect(); + for group_indices in groups { + let group_output = output.clone_slots(&group_indices)?; + let group_requests = group_indices + .iter() + .map(|&index| requests[index].clone()) + .collect::>(); + let group_packets = group_indices + .iter() + .map(|&index| family_packets[index].to_batched()) + .collect::>(); + + let Some(group_results) = try_decode_fast_subsampled_full_rgba_batch_to_textures::

( + runtime, + &group_requests, + &group_packets, + &group_output, + decode_mode, + )? + else { + return Ok(None); + }; + if group_results.len() != group_indices.len() { + return Err(Error::MetalKernel { + message: format!( + "JPEG Metal grouped {} texture result count mismatch", + P::FAMILY_NAME + ), + }); + } + for (original_index, result) in group_indices.into_iter().zip(group_results) { + merged_results[original_index] = Some(result); + } + } + + let mut results = Vec::with_capacity(requests.len()); + for (index, result) in merged_results.into_iter().enumerate() { + results.push(result.ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal grouped {} texture result for tile {index} was missing", + P::FAMILY_NAME + ), + })?); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_full_rgb_batch_to_surfaces( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], +) -> Result>>, Error> { + try_decode_fast444_full_rgb_batch_to_surfaces_with_output(runtime, requests, packets, None) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_full_rgb_batch_to_surfaces_into_output( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchOutputBuffer, +) -> Result>>, Error> { + try_decode_fast444_full_rgb_batch_to_surfaces_with_output( + runtime, + requests, + packets, + Some(output), + ) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_full_rgb_batch_to_surfaces_with_output( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: Option<&crate::MetalBatchOutputBuffer>, +) -> Result>>, Error> { + if requests.is_empty() + || requests + .iter() + .any(|request| request.op != batch::BatchOp::Full || request.fmt != PixelFormat::Rgb8) + { + return Ok(None); + } + + let mut fast444_packets = Vec::with_capacity(packets.len()); + for packet in packets { + let BatchedFastPacket::Fast444(packet, mode) = packet else { + return Ok(None); + }; + fast444_packets.push((*packet, *mode)); + } + + let Some((first, first_mode)) = fast444_packets.first().copied() else { + return Ok(None); + }; + if first.restart_interval_mcus != 0 || first.entropy_checkpoints.is_empty() { + return Ok(None); + } + + let Some(groups) = fast444_full_rgb_batch_groups(&fast444_packets) else { + return Ok(None); + }; + if groups.len() > 1 { + return try_decode_grouped_fast444_full_rgb_batch_to_surfaces_with_output( + runtime, + requests, + &fast444_packets, + output, + groups, + ); + } + + let segment_count = first.entropy_checkpoints.len(); + if !fast444_packets.iter().all(|(packet, mode)| { + *mode == first_mode + && fast444_packets_share_region_scaled_batch_shape(first, packet, segment_count) + }) { + return Ok(None); + } + + let tile_count = fast444_packets.len(); + let tile_count_u32 = checked_u32(tile_count, "fast444 batch tile count")?; + let segment_count_u32 = checked_u32(segment_count, "fast444 batch segment count")?; + let total_decode_threads = checked_u32( + tile_count + .checked_mul(segment_count) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal fast444 batch decode thread count overflowed".to_string(), + })?, + "fast444 batch decode thread count", + )?; + + let width = first.dimensions.0; + let height = first.dimensions.1; + let out_stride = width as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let out_tile_len = out_stride * height as usize; + let plane_len = width as usize * height as usize; + let decode_params = JpegFastRegionScaledBatchParams { + scaled_width: width, + scaled_height: height, + chroma_width: width, + chroma_height: height, + mcus_per_row: first.mcus_per_row, + mcu_rows: first.mcu_rows, + segment_count: segment_count_u32, + tile_count: tile_count_u32, + scale_shift: 0, + origin_x: 0, + origin_y: 0, + }; + let pack_params = JpegWindowedPackBatchParams { + src_width: width, + src_height: height, + chroma_width: width, + chroma_height: height, + src_x: 0, + src_y: 0, + width, + height, + tile_count: tile_count_u32, + out_stride: checked_u32(out_stride, "fast444 batch output stride")?, + alpha: u32::from(u8::MAX), + mode: plane_mode_to_u32(first_mode), + out_format: OUT_RGB, + }; + + let mut batch_scratch = runtime.batch_scratch()?; + let Some(entropy_buffers) = batch_entropy_buffers( + runtime, + &mut batch_scratch, + BatchEntropyBufferKeys { + payload: "fast444_full_entropy", + offsets: "fast444_full_entropy_offsets", + lens: "fast444_full_entropy_lens", + checkpoints: "fast444_full_entropy_checkpoints", + }, + fast444_packets + .iter() + .map(|(packet, _)| packet.entropy_bytes.as_slice()), + fast444_packets + .iter() + .map(|(packet, _)| packet.entropy_checkpoints.as_slice()), + tile_count, + segment_count, + )? + else { + return Ok(None); + }; + + let y_plane = + batch_scratch.private_buffer(&runtime.device, "fast444_full_y", plane_len * tile_count); + let cb_plane = + batch_scratch.private_buffer(&runtime.device, "fast444_full_cb", plane_len * tile_count); + let cr_plane = + batch_scratch.private_buffer(&runtime.device, "fast444_full_cr", plane_len * tile_count); + let out_buffer = batch_output_buffer_or_new( + runtime, + output, + first.dimensions, + tile_count, + out_stride, + out_tile_len, + )?; + let statuses = vec![JpegDecodeStatus::default(); total_decode_threads as usize]; + let status_buffer = + batch_scratch.shared_buffer_with_slice(&runtime.device, "fast444_full_status", &statuses); + let dc_tables = [ + PreparedHuffmanHost::from(&first.y_dc_table), + PreparedHuffmanHost::from(&first.cb_dc_table), + PreparedHuffmanHost::from(&first.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&first.y_ac_table), + PreparedHuffmanHost::from(&first.cb_ac_table), + PreparedHuffmanHost::from(&first.cr_ac_table), + ]; + + let command_buffer = runtime.queue.new_command_buffer(); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder + .set_compute_pipeline_state(&runtime.fast444_scaled_region_batch_decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffers.payload, + [&y_plane, &cb_plane, &cr_plane], + &decode_params, + [&first.y_quant, &first.cb_quant, &first.cr_quant], + &dc_tables, + &ac_tables, + &entropy_buffers.offsets, + &entropy_buffers.lens, + &status_buffer, + ); + decoder_encoder.set_buffer(17, Some(&entropy_buffers.checkpoints), 0); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_scaled_region_batch_decode_pipeline, + total_decode_threads, + ); + decoder_encoder.end_encoding(); + + let pack_encoder = command_buffer.new_compute_command_encoder(); + pack_encoder.set_compute_pipeline_state(&runtime.pack_444_rgb_batch_pipeline); + bind_three_plane_pack::( + pack_encoder, + [Some(&y_plane), Some(&cb_plane), Some(&cr_plane)], + &out_buffer, + &pack_params, + ); + dispatch_3d_pipeline( + pack_encoder, + &runtime.pack_444_rgb_batch_pipeline, + (width, height, tile_count_u32), + ); + pack_encoder.end_encoding(); + + command_buffer.commit(); + command_buffer.wait_until_completed(); + drop(batch_scratch); + + if let Some(results) = + region_scaled_batch_error_results(requests, &status_buffer, total_decode_threads)? + { + return Ok(Some(results)); + } + + let mut results = Vec::with_capacity(requests.len()); + for index in 0..requests.len() { + results.push(Ok(Surface::from_metal_buffer_offset( + out_buffer.clone(), + first.dimensions, + PixelFormat::Rgb8, + index * out_tile_len, + ))); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_grouped_fast444_full_rgb_batch_to_surfaces_with_output( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + fast444_packets: &[(&JpegFast444PacketV1, PlaneMode)], + output: Option<&crate::MetalBatchOutputBuffer>, + groups: Vec>, +) -> Result>>, Error> { + if let Some(output) = output { + for (packet, _) in fast444_packets { + let out_stride = packet.dimensions.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let out_tile_len = out_stride * packet.dimensions.1 as usize; + batch_output_buffer_or_new( + runtime, + Some(output), + packet.dimensions, + requests.len(), + out_stride, + out_tile_len, + )?; + } + } + + let mut merged_results: Vec>> = + (0..requests.len()).map(|_| None).collect(); + for group_indices in groups { + let group_requests = group_indices + .iter() + .map(|&index| requests[index].clone()) + .collect::>(); + let group_packets = group_indices + .iter() + .map(|&index| { + let (packet, mode) = fast444_packets[index]; + BatchedFastPacket::Fast444(packet, mode) + }) + .collect::>(); + + let Some(group_results) = try_decode_fast444_full_rgb_batch_to_surfaces_with_output( + runtime, + &group_requests, + &group_packets, + None, + )? + else { + return Ok(None); + }; + + if let Some(output) = output { + let Some(&first_group_index) = group_indices.first() else { + continue; + }; + let (packet, _) = fast444_packets[first_group_index]; + let out_stride = packet.dimensions.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let out_tile_len = out_stride * packet.dimensions.1 as usize; + for (original_index, result) in copy_grouped_surfaces_to_output( + runtime, + output, + packet.dimensions, + out_tile_len, + &group_indices, + group_results, + )? { + merged_results[original_index] = Some(result); + } + } else { + if group_results.len() != group_indices.len() { + return Err(Error::MetalKernel { + message: "JPEG Metal grouped fast444 buffer result count mismatch".to_string(), + }); + } + for (original_index, result) in group_indices.into_iter().zip(group_results) { + merged_results[original_index] = Some(result); + } + } + } + + let mut results = Vec::with_capacity(requests.len()); + for (index, result) in merged_results.into_iter().enumerate() { + results.push(result.ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal grouped fast444 buffer result for tile {index} was missing" + ), + })?); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_full_rgba_batch_to_textures( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchTextureOutput, +) -> Result>>, Error> { + if requests.is_empty() + || requests + .iter() + .any(|request| request.op != batch::BatchOp::Full || request.fmt != PixelFormat::Rgb8) + { + return Ok(None); + } + + let mut fast444_packets = Vec::with_capacity(packets.len()); + for packet in packets { + let BatchedFastPacket::Fast444(packet, mode) = packet else { + return Ok(None); + }; + fast444_packets.push((*packet, *mode)); + } + + let Some((first, first_mode)) = fast444_packets.first().copied() else { + return Ok(None); + }; + if first.restart_interval_mcus != 0 || first.entropy_checkpoints.is_empty() { + return Ok(None); + } + + let Some(groups) = fast444_full_rgb_batch_groups(&fast444_packets) else { + return Ok(None); + }; + if groups.len() > 1 { + return try_decode_grouped_fast444_full_rgba_batch_to_textures( + runtime, + requests, + &fast444_packets, + output, + groups, + ); + } + + let segment_count = first.entropy_checkpoints.len(); + let tile_count = fast444_packets.len(); + let width = first.dimensions.0; + let height = first.dimensions.1; + let out_stride = width as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let out_tile_len = out_stride * height as usize; + validate_rgba_texture_batch_output(output, first.dimensions, tile_count, out_tile_len)?; + + let segment_count_u32 = checked_u32(segment_count, "fast444 batch segment count")?; + let total_decode_threads = checked_u32( + tile_count + .checked_mul(segment_count) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal fast444 texture batch decode thread count overflowed" + .to_string(), + })?, + "fast444 texture batch decode thread count", + )?; + + let mut batch_scratch = runtime.batch_scratch()?; + let Some(entropy_buffers) = batch_entropy_buffers( + runtime, + &mut batch_scratch, + BatchEntropyBufferKeys { + payload: "fast444_texture_entropy", + offsets: "fast444_texture_entropy_offsets", + lens: "fast444_texture_entropy_lens", + checkpoints: "fast444_texture_entropy_checkpoints", + }, + fast444_packets + .iter() + .map(|(packet, _)| packet.entropy_bytes.as_slice()), + fast444_packets + .iter() + .map(|(packet, _)| packet.entropy_checkpoints.as_slice()), + tile_count, + segment_count, + )? + else { + return Ok(None); + }; + + let statuses = vec![JpegDecodeStatus::default(); total_decode_threads as usize]; + let status_buffer = batch_scratch.shared_buffer_with_slice( + &runtime.device, + "fast444_texture_status", + &statuses, + ); + let dc_tables = [ + PreparedHuffmanHost::from(&first.y_dc_table), + PreparedHuffmanHost::from(&first.cb_dc_table), + PreparedHuffmanHost::from(&first.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&first.y_ac_table), + PreparedHuffmanHost::from(&first.cb_ac_table), + PreparedHuffmanHost::from(&first.cr_ac_table), + ]; + + let command_buffer = runtime.queue.new_command_buffer(); + for index in 0..tile_count { + let texture = output.texture(index).ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?; + let decode_params = JpegFast444TextureBatchParams { + width, + height, + mcus_per_row: first.mcus_per_row, + mcu_rows: first.mcu_rows, + segment_count: segment_count_u32, + tile_index: checked_u32(index, "fast444 texture batch tile index")?, + alpha: u32::from(u8::MAX), + mode: plane_mode_to_u32(first_mode), + }; + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder + .set_compute_pipeline_state(&runtime.fast444_rgba_texture_batch_decode_pipeline); + decoder_encoder.set_buffer(0, Some(&entropy_buffers.payload), 0); + decoder_encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const decode_params).cast(), + ); + decoder_encoder.set_bytes( + 5, + size_of::<[u16; 64]>() as u64, + first.y_quant.as_ptr().cast(), + ); + decoder_encoder.set_bytes( + 6, + size_of::<[u16; 64]>() as u64, + first.cb_quant.as_ptr().cast(), + ); + decoder_encoder.set_bytes( + 7, + size_of::<[u16; 64]>() as u64, + first.cr_quant.as_ptr().cast(), + ); + decoder_encoder.set_bytes( + 8, + size_of::() as u64, + (&raw const dc_tables[0]).cast(), + ); + decoder_encoder.set_bytes( + 9, + size_of::() as u64, + (&raw const ac_tables[0]).cast(), + ); + decoder_encoder.set_bytes( + 10, + size_of::() as u64, + (&raw const dc_tables[1]).cast(), + ); + decoder_encoder.set_bytes( + 11, + size_of::() as u64, + (&raw const ac_tables[1]).cast(), + ); + decoder_encoder.set_bytes( + 12, + size_of::() as u64, + (&raw const dc_tables[2]).cast(), + ); + decoder_encoder.set_bytes( + 13, + size_of::() as u64, + (&raw const ac_tables[2]).cast(), + ); + decoder_encoder.set_buffer(14, Some(&entropy_buffers.offsets), 0); + decoder_encoder.set_buffer(15, Some(&entropy_buffers.lens), 0); + decoder_encoder.set_buffer(16, Some(&status_buffer), 0); + decoder_encoder.set_buffer(17, Some(&entropy_buffers.checkpoints), 0); + decoder_encoder.set_texture(0, Some(texture)); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_rgba_texture_batch_decode_pipeline, + segment_count_u32, + ); + decoder_encoder.end_encoding(); + } + + command_buffer.commit(); + command_buffer.wait_until_completed(); + drop(batch_scratch); + + if let Some(results) = + texture_batch_error_results(requests, &status_buffer, total_decode_threads)? + { + return Ok(Some(results)); + } + + Ok(Some(texture_batch_success_results( + output, + first.dimensions, + requests.len(), + )?)) +} + +#[cfg(target_os = "macos")] +fn try_decode_grouped_fast444_full_rgba_batch_to_textures( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + fast444_packets: &[(&JpegFast444PacketV1, PlaneMode)], + output: &crate::MetalBatchTextureOutput, + groups: Vec>, +) -> Result>>, Error> { + for (packet, _) in fast444_packets { + let out_stride = packet.dimensions.0 as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let out_tile_len = out_stride * packet.dimensions.1 as usize; + validate_rgba_texture_batch_output( + output, + packet.dimensions, + requests.len(), + out_tile_len, + )?; + } + + let mut merged_results: Vec>> = + (0..requests.len()).map(|_| None).collect(); + for group_indices in groups { + let group_output = output.clone_slots(&group_indices)?; + let group_requests = group_indices + .iter() + .map(|&index| requests[index].clone()) + .collect::>(); + let group_packets = group_indices + .iter() + .map(|&index| { + let (packet, mode) = fast444_packets[index]; + BatchedFastPacket::Fast444(packet, mode) + }) + .collect::>(); + + let Some(group_results) = try_decode_fast444_full_rgba_batch_to_textures( + runtime, + &group_requests, + &group_packets, + &group_output, + )? + else { + return Ok(None); + }; + if group_results.len() != group_indices.len() { + return Err(Error::MetalKernel { + message: "JPEG Metal grouped fast444 texture result count mismatch".to_string(), + }); + } + for (original_index, result) in group_indices.into_iter().zip(group_results) { + merged_results[original_index] = Some(result); + } + } + + let mut results = Vec::with_capacity(requests.len()); + for (index, result) in merged_results.into_iter().enumerate() { + results.push(result.ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal grouped fast444 texture result for tile {index} was missing" + ), + })?); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_region_scaled_rgb_batch_to_surfaces( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], +) -> Result>>, Error> { + try_decode_fast444_region_scaled_rgb_batch_to_surfaces_with_output( + runtime, requests, packets, None, + ) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_region_scaled_rgb_batch_to_surfaces_into_output( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchOutputBuffer, +) -> Result>>, Error> { + try_decode_fast444_region_scaled_rgb_batch_to_surfaces_with_output( + runtime, + requests, + packets, + Some(output), + ) +} + +#[cfg(target_os = "macos")] +fn fast444_region_scaled_rgb_output_shape( + packet: &JpegFast444PacketV1, + roi: Rect, + scale: j2k_core::Downscale, +) -> Option<((u32, u32), usize, usize)> { + let scaled = roi.scaled_covering(scale); + let scaled_roi = j2k_jpeg::Rect { + x: scaled.x, + y: scaled.y, + w: scaled.w, + h: scaled.h, + }; + let params = fast444_scaled_region_params(packet, scale, scaled_roi)?; + let out_dims = (params.scaled_width, params.scaled_height); + let out_stride = out_dims.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let out_tile_len = out_stride * out_dims.1 as usize; + Some((out_dims, out_stride, out_tile_len)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_restart_region_scaled_rgb_batch_to_surfaces_with_output( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + fast444_packets: &[(&JpegFast444PacketV1, PlaneMode)], + output: Option<&crate::MetalBatchOutputBuffer>, +) -> Result>>, Error> { + if !fast444_packets + .iter() + .any(|(packet, _)| packet.restart_interval_mcus != 0) + { + return Ok(None); + } + if fast444_packets + .iter() + .any(|(packet, _)| packet.entropy_bytes.is_empty() || packet.entropy_checkpoints.is_empty()) + { + return Ok(None); + } + + let mut first_shape = None; + if output.is_some() { + for (request, (packet, _)) in requests.iter().zip(fast444_packets.iter().copied()) { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + let Some((out_dims, out_stride, out_tile_len)) = + fast444_region_scaled_rgb_output_shape(packet, roi, scale) + else { + return Ok(None); + }; + batch_output_buffer_or_new( + runtime, + output, + out_dims, + requests.len(), + out_stride, + out_tile_len, + )?; + first_shape.get_or_insert((out_dims, out_tile_len)); + } + } + + let mut results = Vec::with_capacity(requests.len()); + for (request, (packet, mode)) in requests.iter().zip(fast444_packets.iter().copied()) { + let decoder = CpuDecoder::new(request.input.as_ref())?; + let batched_packet = BatchedFastPacket::Fast444(packet, mode); + results.push(decode_region_scaled_packet_surface( + runtime, + &decoder, + request, + &batched_packet, + )); + } + + let Some(output) = output else { + return Ok(Some(results)); + }; + let Some((out_dims, out_tile_len)) = first_shape else { + return Ok(Some(results)); + }; + let group_indices = (0..requests.len()).collect::>(); + let copied = copy_grouped_surfaces_to_output( + runtime, + output, + out_dims, + out_tile_len, + &group_indices, + results, + )?; + let mut merged_results: Vec>> = + (0..requests.len()).map(|_| None).collect(); + for (index, result) in copied { + merged_results[index] = Some(result); + } + + let mut results = Vec::with_capacity(requests.len()); + for (index, result) in merged_results.into_iter().enumerate() { + results.push(result.ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal restart fast444 region scaled buffer result for tile {index} was missing" + ), + })?); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_region_scaled_rgb_batch_to_surfaces_with_output( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: Option<&crate::MetalBatchOutputBuffer>, +) -> Result>>, Error> { + if requests.is_empty() + || requests + .iter() + .any(|request| request.fmt != PixelFormat::Rgb8) + { + return Ok(None); + } + + let mut fast444_packets = Vec::with_capacity(packets.len()); + for packet in packets { + let BatchedFastPacket::Fast444(packet, mode) = packet else { + return Ok(None); + }; + fast444_packets.push((*packet, *mode)); + } + + let Some((first, first_mode)) = fast444_packets.first().copied() else { + return Ok(None); + }; + let batch::BatchOp::RegionScaled { + roi: first_roi, + scale: first_scale, + } = requests[0].op + else { + return Ok(None); + }; + if fast444_packets + .iter() + .any(|(packet, _)| packet.restart_interval_mcus != 0) + { + return try_decode_fast444_restart_region_scaled_rgb_batch_to_surfaces_with_output( + runtime, + requests, + &fast444_packets, + output, + ); + } + if first.restart_interval_mcus != 0 || first.entropy_checkpoints.is_empty() { + return Ok(None); + } + + let Some(groups) = fast444_region_scaled_batch_groups(requests, &fast444_packets) else { + return Ok(None); + }; + if groups.len() > 1 { + return try_decode_grouped_fast444_region_scaled_rgb_batch_to_surfaces_with_output( + runtime, + requests, + &fast444_packets, + output, + groups, + ); + } + + let first_scaled = first_roi.scaled_covering(first_scale); + let first_scaled_roi = j2k_jpeg::Rect { + x: first_scaled.x, + y: first_scaled.y, + w: first_scaled.w, + h: first_scaled.h, + }; + let Some(first_decode_params) = + fast444_scaled_region_params(first, first_scale, first_scaled_roi) + else { + return Ok(None); + }; + + let segment_count = first.entropy_checkpoints.len(); + let tile_count = fast444_packets.len(); + let tile_count_u32 = checked_u32(tile_count, "region scaled batch tile count")?; + let segment_count_u32 = checked_u32(segment_count, "region scaled batch segment count")?; + let total_decode_threads = checked_u32( + tile_count + .checked_mul(segment_count) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal region scaled batch decode thread count overflowed" + .to_string(), + })?, + "region scaled batch decode thread count", + )?; + + for (request, (packet, mode)) in requests.iter().zip(fast444_packets.iter().copied()) { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + if scale != first_scale + || mode != first_mode + || !fast444_packets_share_region_scaled_batch_shape(first, packet, segment_count) + { + return Ok(None); + } + let scaled = roi.scaled_covering(scale); + let scaled_roi = j2k_jpeg::Rect { + x: scaled.x, + y: scaled.y, + w: scaled.w, + h: scaled.h, + }; + if fast444_scaled_region_params(packet, scale, scaled_roi) != Some(first_decode_params) { + return Ok(None); + } + } + + let out_stride = + first_decode_params.scaled_width as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let out_tile_len = out_stride * first_decode_params.scaled_height as usize; + + let plane_len = + first_decode_params.scaled_width as usize * first_decode_params.scaled_height as usize; + let decode_params = JpegFastRegionScaledBatchParams { + scaled_width: first_decode_params.scaled_width, + scaled_height: first_decode_params.scaled_height, + chroma_width: first_decode_params.scaled_width, + chroma_height: first_decode_params.scaled_height, + mcus_per_row: first_decode_params.mcus_per_row, + mcu_rows: first_decode_params.mcu_rows, + segment_count: segment_count_u32, + tile_count: tile_count_u32, + scale_shift: first_decode_params.scale_shift, + origin_x: first_decode_params.origin_x, + origin_y: first_decode_params.origin_y, + }; + let pack_params = JpegWindowedPackBatchParams { + src_width: first_decode_params.scaled_width, + src_height: first_decode_params.scaled_height, + chroma_width: first_decode_params.scaled_width, + chroma_height: first_decode_params.scaled_height, + src_x: 0, + src_y: 0, + width: first_decode_params.scaled_width, + height: first_decode_params.scaled_height, + tile_count: tile_count_u32, + out_stride: checked_u32(out_stride, "region scaled batch output stride")?, + alpha: u32::from(u8::MAX), + mode: plane_mode_to_u32(first_mode), + out_format: OUT_RGB, + }; + + let mut batch_scratch = runtime.batch_scratch()?; + let Some(entropy_buffers) = batch_entropy_buffers( + runtime, + &mut batch_scratch, + BatchEntropyBufferKeys { + payload: "fast444_region_scaled_entropy", + offsets: "fast444_region_scaled_entropy_offsets", + lens: "fast444_region_scaled_entropy_lens", + checkpoints: "fast444_region_scaled_entropy_checkpoints", + }, + fast444_packets + .iter() + .map(|(packet, _)| packet.entropy_bytes.as_slice()), + fast444_packets + .iter() + .map(|(packet, _)| packet.entropy_checkpoints.as_slice()), + tile_count, + segment_count, + )? + else { + return Ok(None); + }; + + let y_plane = batch_scratch.private_buffer( + &runtime.device, + "fast444_region_scaled_y", + plane_len * tile_count, + ); + let cb_plane = batch_scratch.private_buffer( + &runtime.device, + "fast444_region_scaled_cb", + plane_len * tile_count, + ); + let cr_plane = batch_scratch.private_buffer( + &runtime.device, + "fast444_region_scaled_cr", + plane_len * tile_count, + ); + let out_buffer = batch_output_buffer_or_new( + runtime, + output, + ( + first_decode_params.scaled_width, + first_decode_params.scaled_height, + ), + tile_count, + out_stride, + out_tile_len, + )?; + let statuses = vec![JpegDecodeStatus::default(); total_decode_threads as usize]; + let status_buffer = batch_scratch.shared_buffer_with_slice( + &runtime.device, + "fast444_region_scaled_status", + &statuses, + ); + let dc_tables = [ + PreparedHuffmanHost::from(&first.y_dc_table), + PreparedHuffmanHost::from(&first.cb_dc_table), + PreparedHuffmanHost::from(&first.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&first.y_ac_table), + PreparedHuffmanHost::from(&first.cb_ac_table), + PreparedHuffmanHost::from(&first.cr_ac_table), + ]; + + let command_buffer = runtime.queue.new_command_buffer(); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder + .set_compute_pipeline_state(&runtime.fast444_scaled_region_batch_decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffers.payload, + [&y_plane, &cb_plane, &cr_plane], + &decode_params, + [&first.y_quant, &first.cb_quant, &first.cr_quant], + &dc_tables, + &ac_tables, + &entropy_buffers.offsets, + &entropy_buffers.lens, + &status_buffer, + ); + decoder_encoder.set_buffer(17, Some(&entropy_buffers.checkpoints), 0); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_scaled_region_batch_decode_pipeline, + total_decode_threads, + ); + decoder_encoder.end_encoding(); + + let pack_encoder = command_buffer.new_compute_command_encoder(); + pack_encoder.set_compute_pipeline_state(&runtime.pack_444_rgb_batch_pipeline); + bind_three_plane_pack::( + pack_encoder, + [Some(&y_plane), Some(&cb_plane), Some(&cr_plane)], + &out_buffer, + &pack_params, + ); + dispatch_3d_pipeline( + pack_encoder, + &runtime.pack_444_rgb_batch_pipeline, + ( + first_decode_params.scaled_width, + first_decode_params.scaled_height, + tile_count_u32, + ), + ); + pack_encoder.end_encoding(); + + command_buffer.commit(); + command_buffer.wait_until_completed(); + drop(batch_scratch); + + if let Some(results) = + region_scaled_batch_error_results(requests, &status_buffer, total_decode_threads)? + { + return Ok(Some(results)); + } + + let mut results = Vec::with_capacity(requests.len()); + for index in 0..requests.len() { + results.push(Ok(Surface::from_metal_buffer_offset( + out_buffer.clone(), + ( + first_decode_params.scaled_width, + first_decode_params.scaled_height, + ), + PixelFormat::Rgb8, + index * out_tile_len, + ))); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_grouped_fast444_region_scaled_rgb_batch_to_surfaces_with_output( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + fast444_packets: &[(&JpegFast444PacketV1, PlaneMode)], + output: Option<&crate::MetalBatchOutputBuffer>, + groups: Vec>, +) -> Result>>, Error> { + if let Some(output) = output { + for (request, (packet, _)) in requests.iter().zip(fast444_packets.iter().copied()) { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + let scaled = roi.scaled_covering(scale); + let scaled_roi = j2k_jpeg::Rect { + x: scaled.x, + y: scaled.y, + w: scaled.w, + h: scaled.h, + }; + let Some(params) = fast444_scaled_region_params(packet, scale, scaled_roi) else { + return Ok(None); + }; + let out_dims = (params.scaled_width, params.scaled_height); + let out_stride = out_dims.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let out_tile_len = out_stride * out_dims.1 as usize; + batch_output_buffer_or_new( + runtime, + Some(output), + out_dims, + requests.len(), + out_stride, + out_tile_len, + )?; + } + } + + let mut merged_results: Vec>> = + (0..requests.len()).map(|_| None).collect(); + for group_indices in groups { + let group_requests = group_indices + .iter() + .map(|&index| requests[index].clone()) + .collect::>(); + let group_packets = group_indices + .iter() + .map(|&index| { + let (packet, mode) = fast444_packets[index]; + BatchedFastPacket::Fast444(packet, mode) + }) + .collect::>(); + + let Some(group_results) = + try_decode_fast444_region_scaled_rgb_batch_to_surfaces_with_output( + runtime, + &group_requests, + &group_packets, + None, + )? + else { + return Ok(None); + }; + + if let Some(output) = output { + let Some(&first_group_index) = group_indices.first() else { + continue; + }; + let batch::BatchOp::RegionScaled { roi, scale } = requests[first_group_index].op else { + return Ok(None); + }; + let (packet, _) = fast444_packets[first_group_index]; + let scaled = roi.scaled_covering(scale); + let scaled_roi = j2k_jpeg::Rect { + x: scaled.x, + y: scaled.y, + w: scaled.w, + h: scaled.h, + }; + let Some(params) = fast444_scaled_region_params(packet, scale, scaled_roi) else { + return Ok(None); + }; + let out_dims = (params.scaled_width, params.scaled_height); + let out_tile_len = + out_dims.0 as usize * out_dims.1 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + for (original_index, result) in copy_grouped_surfaces_to_output( + runtime, + output, + out_dims, + out_tile_len, + &group_indices, + group_results, + )? { + merged_results[original_index] = Some(result); + } + } else { + if group_results.len() != group_indices.len() { + return Err(Error::MetalKernel { + message: + "JPEG Metal grouped fast444 region scaled buffer result count mismatch" + .to_string(), + }); + } + for (original_index, result) in group_indices.into_iter().zip(group_results) { + merged_results[original_index] = Some(result); + } + } + } + + let mut results = Vec::with_capacity(requests.len()); + for (index, result) in merged_results.into_iter().enumerate() { + results.push(result.ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal grouped fast444 region scaled buffer result for tile {index} was missing" + ), + })?); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_restart_region_scaled_rgba_batch_to_textures( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + fast444_packets: &[(&JpegFast444PacketV1, PlaneMode)], + output: &crate::MetalBatchTextureOutput, +) -> Result>>, Error> { + if !fast444_packets + .iter() + .any(|(packet, _)| packet.restart_interval_mcus != 0) + { + return Ok(None); + } + if fast444_packets + .iter() + .any(|(packet, _)| packet.entropy_bytes.is_empty() || packet.entropy_checkpoints.is_empty()) + { + return Ok(None); + } + + let mut first_shape = None; + for (request, (packet, _)) in requests.iter().zip(fast444_packets.iter().copied()) { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + let Some((out_dims, _, _)) = fast444_region_scaled_rgb_output_shape(packet, roi, scale) + else { + return Ok(None); + }; + let out_tile_len = + out_dims.0 as usize * out_dims.1 as usize * PixelFormat::Rgba8.bytes_per_pixel(); + validate_rgba_texture_batch_output(output, out_dims, requests.len(), out_tile_len)?; + first_shape.get_or_insert(out_dims); + } + + let Some(out_dims) = first_shape else { + return Ok(Some(Vec::new())); + }; + let mut surfaces = Vec::with_capacity(requests.len()); + for (request, (packet, mode)) in requests.iter().zip(fast444_packets.iter().copied()) { + let decoder = CpuDecoder::new(request.input.as_ref())?; + let batched_packet = BatchedFastPacket::Fast444(packet, mode); + surfaces.push(decode_region_scaled_packet_surface( + runtime, + &decoder, + request, + &batched_packet, + )); + } + + let group_indices = (0..requests.len()).collect::>(); + let copied = copy_rgb8_surfaces_to_rgba_textures( + runtime, + output, + out_dims, + requests.len(), + &group_indices, + surfaces, + )?; + let mut merged_results: Vec>> = + (0..requests.len()).map(|_| None).collect(); + for (index, result) in copied { + merged_results[index] = Some(result); + } + + let mut results = Vec::with_capacity(requests.len()); + for (index, result) in merged_results.into_iter().enumerate() { + results.push(result.ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal restart fast444 region scaled texture result for tile {index} was missing" + ), + })?); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_region_scaled_rgba_batch_to_textures( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchTextureOutput, +) -> Result>>, Error> { + if requests.is_empty() + || requests + .iter() + .any(|request| request.fmt != PixelFormat::Rgb8) + { + return Ok(None); + } + + let mut fast444_packets = Vec::with_capacity(packets.len()); + for packet in packets { + let BatchedFastPacket::Fast444(packet, mode) = packet else { + return Ok(None); + }; + fast444_packets.push((*packet, *mode)); + } + + let Some((first, first_mode)) = fast444_packets.first().copied() else { + return Ok(None); + }; + let batch::BatchOp::RegionScaled { + roi: first_roi, + scale: first_scale, + } = requests[0].op + else { + return Ok(None); + }; + if fast444_packets + .iter() + .any(|(packet, _)| packet.restart_interval_mcus != 0) + { + return try_decode_fast444_restart_region_scaled_rgba_batch_to_textures( + runtime, + requests, + &fast444_packets, + output, + ); + } + if first.restart_interval_mcus != 0 || first.entropy_checkpoints.is_empty() { + return Ok(None); + } + + let Some(groups) = fast444_region_scaled_batch_groups(requests, &fast444_packets) else { + return Ok(None); + }; + if groups.len() > 1 { + return try_decode_grouped_fast444_region_scaled_rgba_batch_to_textures( + runtime, + requests, + &fast444_packets, + output, + groups, + ); + } + + let first_scaled = first_roi.scaled_covering(first_scale); + let first_scaled_roi = j2k_jpeg::Rect { + x: first_scaled.x, + y: first_scaled.y, + w: first_scaled.w, + h: first_scaled.h, + }; + let Some(first_decode_params) = + fast444_scaled_region_params(first, first_scale, first_scaled_roi) + else { + return Ok(None); + }; + + let segment_count = first.entropy_checkpoints.len(); + let tile_count = fast444_packets.len(); + let tile_count_u32 = checked_u32(tile_count, "region scaled texture batch tile count")?; + let segment_count_u32 = + checked_u32(segment_count, "region scaled texture batch segment count")?; + let total_decode_threads = checked_u32( + tile_count + .checked_mul(segment_count) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal region scaled texture batch decode thread count overflowed" + .to_string(), + })?, + "region scaled texture batch decode thread count", + )?; + + for (request, (packet, mode)) in requests.iter().zip(fast444_packets.iter().copied()) { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + if scale != first_scale + || mode != first_mode + || !fast444_packets_share_region_scaled_batch_shape(first, packet, segment_count) + { + return Ok(None); + } + let scaled = roi.scaled_covering(scale); + let scaled_roi = j2k_jpeg::Rect { + x: scaled.x, + y: scaled.y, + w: scaled.w, + h: scaled.h, + }; + if fast444_scaled_region_params(packet, scale, scaled_roi) != Some(first_decode_params) { + return Ok(None); + } + } + + let out_dims = ( + first_decode_params.scaled_width, + first_decode_params.scaled_height, + ); + let out_tile_len = + out_dims.0 as usize * out_dims.1 as usize * PixelFormat::Rgba8.bytes_per_pixel(); + validate_rgba_texture_batch_output(output, out_dims, tile_count, out_tile_len)?; + + let plane_len = + first_decode_params.scaled_width as usize * first_decode_params.scaled_height as usize; + let decode_params = JpegFastRegionScaledBatchParams { + scaled_width: first_decode_params.scaled_width, + scaled_height: first_decode_params.scaled_height, + chroma_width: first_decode_params.scaled_width, + chroma_height: first_decode_params.scaled_height, + mcus_per_row: first_decode_params.mcus_per_row, + mcu_rows: first_decode_params.mcu_rows, + segment_count: segment_count_u32, + tile_count: tile_count_u32, + scale_shift: first_decode_params.scale_shift, + origin_x: first_decode_params.origin_x, + origin_y: first_decode_params.origin_y, + }; + + let mut batch_scratch = runtime.batch_scratch()?; + let Some(entropy_buffers) = batch_entropy_buffers( + runtime, + &mut batch_scratch, + BatchEntropyBufferKeys { + payload: "fast444_region_scaled_texture_entropy", + offsets: "fast444_region_scaled_texture_entropy_offsets", + lens: "fast444_region_scaled_texture_entropy_lens", + checkpoints: "fast444_region_scaled_texture_entropy_checkpoints", + }, + fast444_packets + .iter() + .map(|(packet, _)| packet.entropy_bytes.as_slice()), + fast444_packets + .iter() + .map(|(packet, _)| packet.entropy_checkpoints.as_slice()), + tile_count, + segment_count, + )? + else { + return Ok(None); + }; + + let y_plane = batch_scratch.private_buffer( + &runtime.device, + "fast444_region_scaled_texture_y", + plane_len * tile_count, + ); + let cb_plane = batch_scratch.private_buffer( + &runtime.device, + "fast444_region_scaled_texture_cb", + plane_len * tile_count, + ); + let cr_plane = batch_scratch.private_buffer( + &runtime.device, + "fast444_region_scaled_texture_cr", + plane_len * tile_count, + ); + let statuses = vec![JpegDecodeStatus::default(); total_decode_threads as usize]; + let status_buffer = batch_scratch.shared_buffer_with_slice( + &runtime.device, + "fast444_region_scaled_texture_status", + &statuses, + ); + let dc_tables = [ + PreparedHuffmanHost::from(&first.y_dc_table), + PreparedHuffmanHost::from(&first.cb_dc_table), + PreparedHuffmanHost::from(&first.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&first.y_ac_table), + PreparedHuffmanHost::from(&first.cb_ac_table), + PreparedHuffmanHost::from(&first.cr_ac_table), + ]; + + let command_buffer = runtime.queue.new_command_buffer(); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder + .set_compute_pipeline_state(&runtime.fast444_scaled_region_batch_decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffers.payload, + [&y_plane, &cb_plane, &cr_plane], + &decode_params, + [&first.y_quant, &first.cb_quant, &first.cr_quant], + &dc_tables, + &ac_tables, + &entropy_buffers.offsets, + &entropy_buffers.lens, + &status_buffer, + ); + decoder_encoder.set_buffer(17, Some(&entropy_buffers.checkpoints), 0); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_scaled_region_batch_decode_pipeline, + total_decode_threads, + ); + decoder_encoder.end_encoding(); + + let pack_params = JpegTexturePackBatchParams { + width: out_dims.0, + height: out_dims.1, + chroma_width: out_dims.0, + chroma_height: out_dims.1, + tile_index: 0, + alpha: u32::from(u8::MAX), + mode: plane_mode_to_u32(first_mode), + }; + dispatch_rgba_texture_pack( + command_buffer, + &runtime.pack_444_rgba_texture_pipeline, + (&y_plane, &cb_plane, &cr_plane), + output, + pack_params, + tile_count, + out_dims, + )?; + + command_buffer.commit(); + command_buffer.wait_until_completed(); + drop(batch_scratch); + + if let Some(results) = + texture_batch_error_results(requests, &status_buffer, total_decode_threads)? + { + return Ok(Some(results)); + } + + Ok(Some(texture_batch_success_results( + output, + out_dims, + requests.len(), + )?)) +} + +#[cfg(target_os = "macos")] +fn try_decode_grouped_fast444_region_scaled_rgba_batch_to_textures( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + fast444_packets: &[(&JpegFast444PacketV1, PlaneMode)], + output: &crate::MetalBatchTextureOutput, + groups: Vec>, +) -> Result>>, Error> { + for (request, (packet, _)) in requests.iter().zip(fast444_packets.iter().copied()) { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + let scaled = roi.scaled_covering(scale); + let scaled_roi = j2k_jpeg::Rect { + x: scaled.x, + y: scaled.y, + w: scaled.w, + h: scaled.h, + }; + let Some(params) = fast444_scaled_region_params(packet, scale, scaled_roi) else { + return Ok(None); + }; + let out_dims = (params.scaled_width, params.scaled_height); + let out_tile_len = + out_dims.0 as usize * out_dims.1 as usize * PixelFormat::Rgba8.bytes_per_pixel(); + validate_rgba_texture_batch_output(output, out_dims, requests.len(), out_tile_len)?; + } + + let mut merged_results: Vec>> = + (0..requests.len()).map(|_| None).collect(); + for group_indices in groups { + let group_output = output.clone_slots(&group_indices)?; + let group_requests = group_indices + .iter() + .map(|&index| requests[index].clone()) + .collect::>(); + let group_packets = group_indices + .iter() + .map(|&index| { + let (packet, mode) = fast444_packets[index]; + BatchedFastPacket::Fast444(packet, mode) + }) + .collect::>(); + + let Some(group_results) = try_decode_fast444_region_scaled_rgba_batch_to_textures( + runtime, + &group_requests, + &group_packets, + &group_output, + )? + else { + return Ok(None); + }; + if group_results.len() != group_indices.len() { + return Err(Error::MetalKernel { + message: "JPEG Metal grouped fast444 region scaled texture result count mismatch" + .to_string(), + }); + } + for (original_index, result) in group_indices.into_iter().zip(group_results) { + merged_results[original_index] = Some(result); + } + } + + let mut results = Vec::with_capacity(requests.len()); + for (index, result) in merged_results.into_iter().enumerate() { + results.push(result.ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal grouped fast444 region scaled texture result for tile {index} was missing" + ), + })?); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast420_region_scaled_rgb_batch_to_surfaces( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], +) -> Result>>, Error> { + try_decode_fast420_region_scaled_rgb_batch_to_surfaces_with_output( + runtime, requests, packets, None, + ) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast420_region_scaled_rgb_batch_to_surfaces_into_output( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchOutputBuffer, +) -> Result>>, Error> { + try_decode_fast420_region_scaled_rgb_batch_to_surfaces_with_output( + runtime, + requests, + packets, + Some(output), + ) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast_subsampled_restart_region_scaled_rgb_batch_to_surfaces_with_output< + P: FastSubsampledMetal, +>( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + family_packets: &[&P], + output: Option<&crate::MetalBatchOutputBuffer>, +) -> Result>>, Error> { + if !family_packets + .iter() + .any(|packet| packet.restart_interval_mcus() != 0) + { + return Ok(None); + } + if family_packets + .iter() + .any(|packet| packet.entropy_bytes().is_empty() || packet.entropy_checkpoints().is_empty()) + { + return Ok(None); + } + + let mut first_plan = None; + if output.is_some() { + for (request, packet) in requests.iter().zip(family_packets.iter().copied()) { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + let segment_count_u32 = checked_u32( + packet.entropy_checkpoints().len(), + &format!( + "{} restart region scaled buffer segment count", + P::FAMILY_NAME + ), + )?; + let Some(plan) = + fast_subsampled_region_scaled_batch_plan(packet, roi, scale, 1, segment_count_u32) + else { + return Ok(None); + }; + batch_output_buffer_or_new( + runtime, + output, + plan.out_dims, + requests.len(), + plan.pack_params.out_stride as usize, + plan.out_tile_len, + )?; + first_plan.get_or_insert(plan); + } + } + + let mut results = Vec::with_capacity(requests.len()); + for (request, packet) in requests.iter().zip(family_packets.iter().copied()) { + let decoder = CpuDecoder::new(request.input.as_ref())?; + let batched_packet = packet.to_batched(); + results.push(decode_region_scaled_packet_surface( + runtime, + &decoder, + request, + &batched_packet, + )); + } + + let Some(output) = output else { + return Ok(Some(results)); + }; + let Some(plan) = first_plan else { + return Ok(Some(results)); + }; + let group_indices = (0..requests.len()).collect::>(); + let copied = copy_grouped_surfaces_to_output( + runtime, + output, + plan.out_dims, + plan.out_tile_len, + &group_indices, + results, + )?; + let mut merged_results: Vec>> = + (0..requests.len()).map(|_| None).collect(); + for (index, result) in copied { + merged_results[index] = Some(result); + } + + let mut results = Vec::with_capacity(requests.len()); + for (index, result) in merged_results.into_iter().enumerate() { + results.push(result.ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal restart {} region scaled buffer result for tile {index} was missing", + P::FAMILY_NAME + ), + })?); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast_subsampled_region_scaled_rgb_batch_to_surfaces_with_output< + P: FastSubsampledMetal, +>( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: Option<&crate::MetalBatchOutputBuffer>, +) -> Result>>, Error> { + if requests.is_empty() + || requests + .iter() + .any(|request| request.fmt != PixelFormat::Rgb8) + { + return Ok(None); + } + + let mut family_packets = Vec::with_capacity(packets.len()); + for packet in packets { + let Some(packet) = P::from_batched(packet) else { + return Ok(None); + }; + family_packets.push(packet); + } + + let Some(first) = family_packets.first().copied() else { + return Ok(None); + }; + let batch::BatchOp::RegionScaled { + roi: first_roi, + scale: first_scale, + } = requests[0].op + else { + return Ok(None); + }; + if family_packets + .iter() + .any(|packet| packet.restart_interval_mcus() != 0) + { + return try_decode_fast_subsampled_restart_region_scaled_rgb_batch_to_surfaces_with_output( + runtime, + requests, + &family_packets, + output, + ); + } + if first.restart_interval_mcus() != 0 || first.entropy_checkpoints().is_empty() { + return Ok(None); + } + + let Some(groups) = fast_subsampled_region_scaled_batch_groups(requests, &family_packets)? + else { + return Ok(None); + }; + if groups.len() > 1 { + return try_decode_grouped_fast_subsampled_region_scaled_rgb_batch_to_surfaces_with_output( + runtime, + requests, + &family_packets, + output, + groups, + ); + } + + let segment_count = first.entropy_checkpoints().len(); + let tile_count = family_packets.len(); + let tile_count_u32 = checked_u32(tile_count, "region scaled batch tile count")?; + let segment_count_u32 = checked_u32(segment_count, "region scaled batch segment count")?; + let Some(first_plan) = fast_subsampled_region_scaled_batch_plan( + first, + first_roi, + first_scale, + tile_count_u32, + segment_count_u32, + ) else { + return Ok(None); + }; + + let total_decode_threads = checked_u32( + tile_count + .checked_mul(segment_count) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal {} region scaled batch decode thread count overflowed", + P::FAMILY_NAME + ), + })?, + &format!("{} region scaled batch decode thread count", P::FAMILY_NAME), + )?; + + for (request, packet) in requests.iter().zip(family_packets.iter().copied()) { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + if scale != first_scale + || !fast_subsampled_packets_share_full_rgb_batch_shape(first, packet, segment_count) + || fast_subsampled_region_scaled_batch_plan( + packet, + roi, + scale, + tile_count_u32, + segment_count_u32, + ) != Some(first_plan) + { + return Ok(None); + } + } + + let mut batch_scratch = runtime.batch_scratch()?; + let Some(entropy_buffers) = batch_entropy_buffers( + runtime, + &mut batch_scratch, + BatchEntropyBufferKeys { + payload: P::REGION_SCALED_KEYS.entropy, + offsets: P::REGION_SCALED_KEYS.entropy_offsets, + lens: P::REGION_SCALED_KEYS.entropy_lens, + checkpoints: P::REGION_SCALED_KEYS.entropy_checkpoints, + }, + family_packets.iter().map(|packet| packet.entropy_bytes()), + family_packets + .iter() + .map(|packet| packet.entropy_checkpoints()), + tile_count, + segment_count, + )? + else { + return Ok(None); + }; + + let y_plane = batch_scratch.private_buffer( + &runtime.device, + P::REGION_SCALED_KEYS.y, + first_plan.y_len * tile_count, + ); + let cb_plane = batch_scratch.private_buffer( + &runtime.device, + P::REGION_SCALED_KEYS.cb, + first_plan.chroma_len * tile_count, + ); + let cr_plane = batch_scratch.private_buffer( + &runtime.device, + P::REGION_SCALED_KEYS.cr, + first_plan.chroma_len * tile_count, + ); + let out_buffer = batch_output_buffer_or_new( + runtime, + output, + first_plan.out_dims, + tile_count, + first_plan.pack_params.out_stride as usize, + first_plan.out_tile_len, + )?; + let statuses = vec![JpegDecodeStatus::default(); total_decode_threads as usize]; + let status_buffer = batch_scratch.shared_buffer_with_slice( + &runtime.device, + P::REGION_SCALED_KEYS.status, + &statuses, + ); + let dc_tables = [ + PreparedHuffmanHost::from(first.y_dc_table()), + PreparedHuffmanHost::from(first.cb_dc_table()), + PreparedHuffmanHost::from(first.cr_dc_table()), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(first.y_ac_table()), + PreparedHuffmanHost::from(first.cb_ac_table()), + PreparedHuffmanHost::from(first.cr_ac_table()), + ]; + + let command_buffer = runtime.queue.new_command_buffer(); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(P::scaled_region_batch_decode_pipeline(runtime)); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffers.payload, + [&y_plane, &cb_plane, &cr_plane], + &first_plan.decode_params, + [first.y_quant(), first.cb_quant(), first.cr_quant()], + &dc_tables, + &ac_tables, + &entropy_buffers.offsets, + &entropy_buffers.lens, + &status_buffer, + ); + decoder_encoder.set_buffer(17, Some(&entropy_buffers.checkpoints), 0); + dispatch_1d_pipeline( + decoder_encoder, + P::scaled_region_batch_decode_pipeline(runtime), + total_decode_threads, + ); + decoder_encoder.end_encoding(); + + let pack_encoder = command_buffer.new_compute_command_encoder(); + pack_encoder.set_compute_pipeline_state(P::pack_windowed_rgb_batch_pipeline(runtime)); + bind_three_plane_pack::( + pack_encoder, + [Some(&y_plane), Some(&cb_plane), Some(&cr_plane)], + &out_buffer, + &first_plan.pack_params, + ); + dispatch_3d_pipeline( + pack_encoder, + P::pack_windowed_rgb_batch_pipeline(runtime), + (first_plan.out_dims.0, first_plan.out_dims.1, tile_count_u32), + ); + pack_encoder.end_encoding(); + + command_buffer.commit(); + command_buffer.wait_until_completed(); + drop(batch_scratch); + + if let Some(results) = + region_scaled_batch_error_results(requests, &status_buffer, total_decode_threads)? + { + return Ok(Some(results)); + } + + let mut results = Vec::with_capacity(requests.len()); + for index in 0..requests.len() { + results.push(Ok(Surface::from_metal_buffer_offset( + out_buffer.clone(), + first_plan.out_dims, + PixelFormat::Rgb8, + index * first_plan.out_tile_len, + ))); + } + Ok(Some(results)) +} + +fn try_decode_fast420_region_scaled_rgb_batch_to_surfaces_with_output( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: Option<&crate::MetalBatchOutputBuffer>, +) -> Result>>, Error> { + try_decode_fast_subsampled_region_scaled_rgb_batch_to_surfaces_with_output::( + runtime, requests, packets, output, + ) +} + +#[cfg(target_os = "macos")] +fn try_decode_grouped_fast_subsampled_region_scaled_rgb_batch_to_surfaces_with_output< + P: FastSubsampledMetal, +>( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + family_packets: &[&P], + output: Option<&crate::MetalBatchOutputBuffer>, + groups: Vec>, +) -> Result>>, Error> { + if let Some(output) = output { + for (request, packet) in requests.iter().zip(family_packets.iter().copied()) { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + let segment_count_u32 = checked_u32( + packet.entropy_checkpoints().len(), + &format!( + "{} grouped region scaled buffer segment count", + P::FAMILY_NAME + ), + )?; + let Some(plan) = + fast_subsampled_region_scaled_batch_plan(packet, roi, scale, 1, segment_count_u32) + else { + return Ok(None); + }; + batch_output_buffer_or_new( + runtime, + Some(output), + plan.out_dims, + requests.len(), + plan.pack_params.out_stride as usize, + plan.out_tile_len, + )?; + } + } + + let mut merged_results: Vec>> = + (0..requests.len()).map(|_| None).collect(); + for group_indices in groups { + let group_requests = group_indices + .iter() + .map(|&index| requests[index].clone()) + .collect::>(); + let group_packets = group_indices + .iter() + .map(|&index| family_packets[index].to_batched()) + .collect::>(); + + let Some(group_results) = + try_decode_fast_subsampled_region_scaled_rgb_batch_to_surfaces_with_output::

( + runtime, + &group_requests, + &group_packets, + None, + )? + else { + return Ok(None); + }; + + if let Some(output) = output { + let Some(&first_group_index) = group_indices.first() else { + continue; + }; + let batch::BatchOp::RegionScaled { roi, scale } = requests[first_group_index].op else { + return Ok(None); + }; + let packet = family_packets[first_group_index]; + let segment_count_u32 = checked_u32( + packet.entropy_checkpoints().len(), + &format!( + "{} grouped region scaled buffer segment count", + P::FAMILY_NAME + ), + )?; + let Some(plan) = + fast_subsampled_region_scaled_batch_plan(packet, roi, scale, 1, segment_count_u32) + else { + return Ok(None); + }; + for (original_index, result) in copy_grouped_surfaces_to_output( + runtime, + output, + plan.out_dims, + plan.out_tile_len, + &group_indices, + group_results, + )? { + merged_results[original_index] = Some(result); + } + } else { + if group_results.len() != group_indices.len() { + return Err(Error::MetalKernel { + message: format!( + "JPEG Metal grouped {} region scaled buffer result count mismatch", + P::FAMILY_NAME + ), + }); + } + for (original_index, result) in group_indices.into_iter().zip(group_results) { + merged_results[original_index] = Some(result); + } + } + } + + let mut results = Vec::with_capacity(requests.len()); + for (index, result) in merged_results.into_iter().enumerate() { + results.push(result.ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal grouped {} region scaled buffer result for tile {index} was missing", + P::FAMILY_NAME + ), + })?); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast_subsampled_restart_region_scaled_rgba_batch_to_textures< + P: FastSubsampledMetal, +>( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + family_packets: &[&P], + output: &crate::MetalBatchTextureOutput, +) -> Result>>, Error> { + if !family_packets + .iter() + .any(|packet| packet.restart_interval_mcus() != 0) + { + return Ok(None); + } + if family_packets + .iter() + .any(|packet| packet.entropy_bytes().is_empty() || packet.entropy_checkpoints().is_empty()) + { + return Ok(None); + } + + let mut first_plan = None; + for (request, packet) in requests.iter().zip(family_packets.iter().copied()) { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + let segment_count_u32 = checked_u32( + packet.entropy_checkpoints().len(), + &format!( + "{} restart region scaled texture segment count", + P::FAMILY_NAME + ), + )?; + let Some(plan) = + fast_subsampled_region_scaled_batch_plan(packet, roi, scale, 1, segment_count_u32) + else { + return Ok(None); + }; + let out_tile_len = plan.out_dims.0 as usize + * plan.out_dims.1 as usize + * PixelFormat::Rgba8.bytes_per_pixel(); + validate_rgba_texture_batch_output(output, plan.out_dims, requests.len(), out_tile_len)?; + first_plan.get_or_insert(plan); + } + + let Some(plan) = first_plan else { + return Ok(Some(Vec::new())); + }; + let mut surfaces = Vec::with_capacity(requests.len()); + for (request, packet) in requests.iter().zip(family_packets.iter().copied()) { + let decoder = CpuDecoder::new(request.input.as_ref())?; + let batched_packet = packet.to_batched(); + surfaces.push(decode_region_scaled_packet_surface( + runtime, + &decoder, + request, + &batched_packet, + )); + } + + let group_indices = (0..requests.len()).collect::>(); + let copied = copy_rgb8_surfaces_to_rgba_textures( + runtime, + output, + plan.out_dims, + requests.len(), + &group_indices, + surfaces, + )?; + let mut merged_results: Vec>> = + (0..requests.len()).map(|_| None).collect(); + for (index, result) in copied { + merged_results[index] = Some(result); + } + + let mut results = Vec::with_capacity(requests.len()); + for (index, result) in merged_results.into_iter().enumerate() { + results.push(result.ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal restart {} region scaled texture result for tile {index} was missing", + P::FAMILY_NAME + ), + })?); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast_subsampled_region_scaled_rgba_batch_to_textures( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchTextureOutput, +) -> Result>>, Error> { + if requests.is_empty() + || requests + .iter() + .any(|request| request.fmt != PixelFormat::Rgb8) + { + return Ok(None); + } + + let mut family_packets = Vec::with_capacity(packets.len()); + for packet in packets { + let Some(packet) = P::from_batched(packet) else { + return Ok(None); + }; + family_packets.push(packet); + } + + let Some(first) = family_packets.first().copied() else { + return Ok(None); + }; + let batch::BatchOp::RegionScaled { + roi: first_roi, + scale: first_scale, + } = requests[0].op + else { + return Ok(None); + }; + if family_packets + .iter() + .any(|packet| packet.restart_interval_mcus() != 0) + { + return try_decode_fast_subsampled_restart_region_scaled_rgba_batch_to_textures( + runtime, + requests, + &family_packets, + output, + ); + } + if first.restart_interval_mcus() != 0 || first.entropy_checkpoints().is_empty() { + return Ok(None); + } + + let Some(groups) = fast_subsampled_region_scaled_batch_groups(requests, &family_packets)? + else { + return Ok(None); + }; + if groups.len() > 1 { + return try_decode_grouped_fast_subsampled_region_scaled_rgba_batch_to_textures( + runtime, + requests, + &family_packets, + output, + groups, + ); + } + + let segment_count = first.entropy_checkpoints().len(); + let tile_count = family_packets.len(); + let tile_count_u32 = checked_u32(tile_count, "region scaled texture batch tile count")?; + let segment_count_u32 = + checked_u32(segment_count, "region scaled texture batch segment count")?; + let Some(first_plan) = fast_subsampled_region_scaled_batch_plan( + first, + first_roi, + first_scale, + tile_count_u32, + segment_count_u32, + ) else { + return Ok(None); + }; + + let total_decode_threads = checked_u32( + tile_count + .checked_mul(segment_count) + .ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal {} region scaled texture decode thread count overflowed", + P::FAMILY_NAME + ), + })?, + &format!( + "{} region scaled texture decode thread count", + P::FAMILY_NAME + ), + )?; + + for (request, packet) in requests.iter().zip(family_packets.iter().copied()) { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + if scale != first_scale + || !fast_subsampled_packets_share_full_rgb_batch_shape(first, packet, segment_count) + || fast_subsampled_region_scaled_batch_plan( + packet, + roi, + scale, + tile_count_u32, + segment_count_u32, + ) != Some(first_plan) + { + return Ok(None); + } + } + + let out_tile_len = first_plan.out_dims.0 as usize + * first_plan.out_dims.1 as usize + * PixelFormat::Rgba8.bytes_per_pixel(); + validate_rgba_texture_batch_output(output, first_plan.out_dims, tile_count, out_tile_len)?; + + let mut batch_scratch = runtime.batch_scratch()?; + let Some(entropy_buffers) = batch_entropy_buffers( + runtime, + &mut batch_scratch, + BatchEntropyBufferKeys { + payload: P::REGION_SCALED_TEXTURE_KEYS.entropy, + offsets: P::REGION_SCALED_TEXTURE_KEYS.entropy_offsets, + lens: P::REGION_SCALED_TEXTURE_KEYS.entropy_lens, + checkpoints: P::REGION_SCALED_TEXTURE_KEYS.entropy_checkpoints, + }, + family_packets.iter().map(|packet| packet.entropy_bytes()), + family_packets + .iter() + .map(|packet| packet.entropy_checkpoints()), + tile_count, + segment_count, + )? + else { + return Ok(None); + }; + + let y_plane = batch_scratch.private_buffer( + &runtime.device, + P::REGION_SCALED_TEXTURE_KEYS.y, + first_plan.y_len * tile_count, + ); + let cb_plane = batch_scratch.private_buffer( + &runtime.device, + P::REGION_SCALED_TEXTURE_KEYS.cb, + first_plan.chroma_len * tile_count, + ); + let cr_plane = batch_scratch.private_buffer( + &runtime.device, + P::REGION_SCALED_TEXTURE_KEYS.cr, + first_plan.chroma_len * tile_count, + ); + let statuses = vec![JpegDecodeStatus::default(); total_decode_threads as usize]; + let status_buffer = batch_scratch.shared_buffer_with_slice( + &runtime.device, + P::REGION_SCALED_TEXTURE_KEYS.status, + &statuses, + ); + let dc_tables = [ + PreparedHuffmanHost::from(first.y_dc_table()), + PreparedHuffmanHost::from(first.cb_dc_table()), + PreparedHuffmanHost::from(first.cr_dc_table()), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(first.y_ac_table()), + PreparedHuffmanHost::from(first.cb_ac_table()), + PreparedHuffmanHost::from(first.cr_ac_table()), + ]; + + let command_buffer = runtime.queue.new_command_buffer(); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(P::scaled_region_batch_decode_pipeline(runtime)); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffers.payload, + [&y_plane, &cb_plane, &cr_plane], + &first_plan.decode_params, + [first.y_quant(), first.cb_quant(), first.cr_quant()], + &dc_tables, + &ac_tables, + &entropy_buffers.offsets, + &entropy_buffers.lens, + &status_buffer, + ); + decoder_encoder.set_buffer(17, Some(&entropy_buffers.checkpoints), 0); + dispatch_1d_pipeline( + decoder_encoder, + P::scaled_region_batch_decode_pipeline(runtime), + total_decode_threads, + ); + decoder_encoder.end_encoding(); + + dispatch_windowed_rgba_texture_pack( + command_buffer, + P::pack_windowed_rgba_texture_pipeline(runtime), + (&y_plane, &cb_plane, &cr_plane), + output, + windowed_texture_pack_params(first_plan), + tile_count, + first_plan.out_dims, + )?; + + command_buffer.commit(); + command_buffer.wait_until_completed(); + drop(batch_scratch); + + if let Some(results) = + texture_batch_error_results(requests, &status_buffer, total_decode_threads)? + { + return Ok(Some(results)); + } + + Ok(Some(texture_batch_success_results( + output, + first_plan.out_dims, + requests.len(), + )?)) +} + +fn try_decode_fast420_region_scaled_rgba_batch_to_textures( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchTextureOutput, +) -> Result>>, Error> { + try_decode_fast_subsampled_region_scaled_rgba_batch_to_textures::( + runtime, requests, packets, output, + ) +} + +#[cfg(target_os = "macos")] +fn try_decode_grouped_fast_subsampled_region_scaled_rgba_batch_to_textures< + P: FastSubsampledMetal, +>( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + family_packets: &[&P], + output: &crate::MetalBatchTextureOutput, + groups: Vec>, +) -> Result>>, Error> { + for (request, packet) in requests.iter().zip(family_packets.iter().copied()) { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + let segment_count_u32 = checked_u32( + packet.entropy_checkpoints().len(), + &format!( + "{} grouped region scaled texture batch segment count", + P::FAMILY_NAME + ), + )?; + let Some(plan) = + fast_subsampled_region_scaled_batch_plan(packet, roi, scale, 1, segment_count_u32) + else { + return Ok(None); + }; + let out_tile_len = plan.out_dims.0 as usize + * plan.out_dims.1 as usize + * PixelFormat::Rgba8.bytes_per_pixel(); + validate_rgba_texture_batch_output(output, plan.out_dims, requests.len(), out_tile_len)?; + } + + let mut merged_results: Vec>> = + (0..requests.len()).map(|_| None).collect(); + for group_indices in groups { + let group_output = output.clone_slots(&group_indices)?; + let group_requests = group_indices + .iter() + .map(|&index| requests[index].clone()) + .collect::>(); + let group_packets = group_indices + .iter() + .map(|&index| family_packets[index].to_batched()) + .collect::>(); + + let Some(group_results) = try_decode_fast_subsampled_region_scaled_rgba_batch_to_textures::< + P, + >( + runtime, &group_requests, &group_packets, &group_output + )? + else { + return Ok(None); + }; + if group_results.len() != group_indices.len() { + return Err(Error::MetalKernel { + message: format!( + "JPEG Metal grouped {} region scaled texture result count mismatch", + P::FAMILY_NAME + ), + }); + } + for (original_index, result) in group_indices.into_iter().zip(group_results) { + merged_results[original_index] = Some(result); + } + } + + let mut results = Vec::with_capacity(requests.len()); + for (index, result) in merged_results.into_iter().enumerate() { + results.push(result.ok_or_else(|| Error::MetalKernel { + message: format!( + "JPEG Metal grouped {} region scaled texture result for tile {index} was missing", + P::FAMILY_NAME + ), + })?); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast422_region_scaled_rgb_batch_to_surfaces( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], +) -> Result>>, Error> { + try_decode_fast422_region_scaled_rgb_batch_to_surfaces_with_output( + runtime, requests, packets, None, + ) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast422_region_scaled_rgb_batch_to_surfaces_into_output( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchOutputBuffer, +) -> Result>>, Error> { + try_decode_fast422_region_scaled_rgb_batch_to_surfaces_with_output( + runtime, + requests, + packets, + Some(output), + ) +} + +fn try_decode_fast422_region_scaled_rgb_batch_to_surfaces_with_output( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: Option<&crate::MetalBatchOutputBuffer>, +) -> Result>>, Error> { + try_decode_fast_subsampled_region_scaled_rgb_batch_to_surfaces_with_output::( + runtime, requests, packets, output, + ) +} + +fn try_decode_fast422_region_scaled_rgba_batch_to_textures( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchTextureOutput, +) -> Result>>, Error> { + try_decode_fast_subsampled_region_scaled_rgba_batch_to_textures::( + runtime, requests, packets, output, + ) +} + +#[cfg(target_os = "macos")] +fn requests_share_one_input(requests: &[batch::QueuedRequest]) -> bool { + let Some(first) = requests.first() else { + return false; + }; + requests.iter().all(|request| { + request.input.as_ptr() == first.input.as_ptr() && request.input.len() == first.input.len() + }) +} + +#[cfg(target_os = "macos")] +fn requests_share_one_region_scaled_work(requests: &[batch::QueuedRequest]) -> bool { + let Some(first) = requests.first() else { + return false; + }; + requests_share_one_input(requests) + && requests.iter().all(|request| { + request.fmt == first.fmt && request.backend == first.backend && request.op == first.op + }) +} + +#[cfg(target_os = "macos")] +fn decode_region_scaled_packet_surface( + runtime: &MetalRuntime, + decoder: &CpuDecoder<'_>, + request: &batch::QueuedRequest, + packet: &BatchedFastPacket<'_>, +) -> Result { + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Err(Error::MetalKernel { + message: "JPEG Metal expected a region scaled batch request".to_string(), + }); + }; + let scaled = roi.scaled_covering(scale); + let scaled_roi = j2k_jpeg::Rect { + x: scaled.x, + y: scaled.y, + w: scaled.w, + h: scaled.h, + }; + match packet { + BatchedFastPacket::Fast420(packet) => try_decode_fast420_scaled_region_to_surface( + runtime, + decoder, + Some(packet), + request.fmt, + scaled_roi, + scale, + ), + BatchedFastPacket::Fast422(packet) => try_decode_fast422_scaled_region_to_surface( + runtime, + Some(packet), + request.fmt, + scaled_roi, + scale, + ), + BatchedFastPacket::Fast444(packet, _) => try_decode_fast444_scaled_region_to_surface( + runtime, + decoder, + Some(packet), + request.fmt, + scaled_roi, + scale, + ), + } + .and_then(|surface| { + surface.ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal repeated region scaled batch was not packet-decodable".to_string(), + }) + }) +} + +#[cfg(target_os = "macos")] +fn try_decode_repeated_region_scaled_batch_to_surfaces( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], +) -> Result>>, Error> { + if requests.len() <= REGION_SCALED_BATCH_CHUNK + || !requests_share_one_input(requests) + || !requests + .iter() + .all(|request| matches!(request.op, batch::BatchOp::RegionScaled { .. })) + { + return Ok(None); + } + + let decoder = CpuDecoder::new(requests[0].input.as_ref())?; + if requests_share_one_region_scaled_work(requests) { + let surface = + decode_region_scaled_packet_surface(runtime, &decoder, &requests[0], &packets[0])?; + return Ok(Some( + (0..requests.len()) + .map(|_| Ok(surface.clone())) + .collect::>(), + )); + } + + let mut results = Vec::with_capacity(requests.len()); + for (request, packet) in requests.iter().zip(packets.iter()) { + results.push(decode_region_scaled_packet_surface( + runtime, &decoder, request, packet, + )); + } + + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +#[cfg_attr(not(test), allow(dead_code))] +pub(crate) fn decode_full_batch_to_surfaces( + requests: &[batch::QueuedRequest], +) -> Result>>, Error> { + let Some(packets) = batched_fast_packets(requests)? else { + return Ok(None); + }; + + with_runtime(|runtime| decode_full_batch_to_surfaces_with_runtime(runtime, requests, &packets)) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_full_batch_to_surfaces_with_session( + requests: &[batch::QueuedRequest], + session: &crate::MetalBackendSession, +) -> Result>>, Error> { + let Some(packets) = batched_fast_packets(requests)? else { + return Ok(None); + }; + + with_runtime_for_session(session, |runtime| { + decode_full_batch_to_surfaces_with_runtime(runtime, requests, &packets) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_full_batch_to_surfaces_with_session_state( + requests: &[batch::QueuedRequest], + session: &mut crate::session::SessionState, +) -> Result>>, Error> { + let Some(packets) = batched_fast_packets(requests)? else { + return Ok(None); + }; + + let backend_session = session.backend_session()?; + with_runtime_for_session(backend_session, |runtime| { + decode_full_batch_to_surfaces_with_runtime(runtime, requests, &packets) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_full_rgb8_batch_into_output_with_session( + requests: &[batch::QueuedRequest], + output: &crate::MetalBatchOutputBuffer, + session: &crate::MetalBackendSession, +) -> Result>>, Error> { + let Some(packets) = batched_fast_packets(requests)? else { + return Ok(None); + }; + + with_runtime_for_session(session, |runtime| { + decode_full_rgb8_batch_into_output_with_runtime(runtime, requests, &packets, output) + }) +} + +#[cfg(target_os = "macos")] +fn decode_full_rgb8_batch_into_output_with_runtime( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchOutputBuffer, +) -> Result>>, Error> { + if let Some(results) = try_decode_fast_subsampled_full_rgb_batch_to_surfaces_into_output::< + JpegFast420PacketV1, + >(runtime, requests, packets, output)? + { + return Ok(Some(results)); + } + if let Some(results) = try_decode_fast_subsampled_full_rgb_batch_to_surfaces_into_output::< + JpegFast422PacketV1, + >(runtime, requests, packets, output)? + { + return Ok(Some(results)); + } + if let Some(results) = try_decode_fast444_full_rgb_batch_to_surfaces_into_output( + runtime, requests, packets, output, + )? { + return Ok(Some(results)); + } + + Ok(None) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_full_rgb8_batch_into_textures_with_session( + requests: &[batch::QueuedRequest], + output: &crate::MetalBatchTextureOutput, + session: &crate::MetalBackendSession, +) -> Result>>, Error> { + let Some(packets) = batched_fast_packets(requests)? else { + return Ok(None); + }; + + with_runtime_for_session(session, |runtime| { + decode_full_rgb8_batch_into_textures_with_runtime(runtime, requests, &packets, output) + }) +} + +#[cfg(target_os = "macos")] +fn decode_full_rgb8_batch_into_textures_with_runtime( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchTextureOutput, +) -> Result>>, Error> { + if let Some(results) = try_decode_fast_subsampled_full_rgba_batch_to_textures::< + JpegFast420PacketV1, + >(runtime, requests, packets, output, fast_batch_decode_mode())? + { + return Ok(Some(results)); + } + if let Some(results) = try_decode_fast_subsampled_full_rgba_batch_to_textures::< + JpegFast422PacketV1, + >(runtime, requests, packets, output, fast_batch_decode_mode())? + { + return Ok(Some(results)); + } + if let Some(results) = + try_decode_fast444_full_rgba_batch_to_textures(runtime, requests, packets, output)? + { + return Ok(Some(results)); + } + + Ok(None) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_region_scaled_rgb8_batch_into_output_with_session( + requests: &[batch::QueuedRequest], + output: &crate::MetalBatchOutputBuffer, + session: &crate::MetalBackendSession, +) -> Result>>, Error> { + let Some(packets) = batched_fast_packets(requests)? else { + return Ok(None); + }; + + with_runtime_for_session(session, |runtime| { + decode_region_scaled_rgb8_batch_into_output_with_runtime( + runtime, requests, &packets, output, + ) + }) +} + +#[cfg(target_os = "macos")] +fn decode_region_scaled_rgb8_batch_into_output_with_runtime( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchOutputBuffer, +) -> Result>>, Error> { + if let Some(results) = try_decode_fast444_region_scaled_rgb_batch_to_surfaces_into_output( + runtime, requests, packets, output, + )? { + return Ok(Some(results)); + } + if let Some(results) = try_decode_fast420_region_scaled_rgb_batch_to_surfaces_into_output( + runtime, requests, packets, output, + )? { + return Ok(Some(results)); + } + if let Some(results) = try_decode_fast422_region_scaled_rgb_batch_to_surfaces_into_output( + runtime, requests, packets, output, + )? { + return Ok(Some(results)); + } + + Ok(None) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_region_scaled_rgb8_batch_into_textures_with_session( + requests: &[batch::QueuedRequest], + output: &crate::MetalBatchTextureOutput, + session: &crate::MetalBackendSession, +) -> Result>>, Error> { + let Some(packets) = batched_fast_packets(requests)? else { + return Ok(None); + }; + + with_runtime_for_session(session, |runtime| { + decode_region_scaled_rgb8_batch_into_textures_with_runtime( + runtime, requests, &packets, output, + ) + }) +} + +#[cfg(target_os = "macos")] +fn decode_region_scaled_rgb8_batch_into_textures_with_runtime( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], + output: &crate::MetalBatchTextureOutput, +) -> Result>>, Error> { + if let Some(results) = + try_decode_fast444_region_scaled_rgba_batch_to_textures(runtime, requests, packets, output)? + { + return Ok(Some(results)); + } + if let Some(results) = + try_decode_fast420_region_scaled_rgba_batch_to_textures(runtime, requests, packets, output)? + { + return Ok(Some(results)); + } + if let Some(results) = + try_decode_fast422_region_scaled_rgba_batch_to_textures(runtime, requests, packets, output)? + { + return Ok(Some(results)); + } + + Ok(None) +} + +#[cfg(target_os = "macos")] +fn decode_full_batch_to_surfaces_with_runtime( + runtime: &MetalRuntime, + requests: &[batch::QueuedRequest], + packets: &[BatchedFastPacket<'_>], +) -> Result>>, Error> { + if let Some(results) = try_decode_fast_subsampled_full_rgb_batch_to_surfaces::< + JpegFast420PacketV1, + >(runtime, requests, packets)? + { + return Ok(Some(results)); + } + if let Some(results) = try_decode_fast_subsampled_full_rgb_batch_to_surfaces::< + JpegFast422PacketV1, + >(runtime, requests, packets)? + { + return Ok(Some(results)); + } + if let Some(results) = + try_decode_fast444_full_rgb_batch_to_surfaces(runtime, requests, packets)? + { + return Ok(Some(results)); + } + if let Some(results) = + try_decode_repeated_region_scaled_batch_to_surfaces(runtime, requests, packets)? + { + return Ok(Some(results)); + } + if let Some(results) = + try_decode_fast444_region_scaled_rgb_batch_to_surfaces(runtime, requests, packets)? + { + return Ok(Some(results)); + } + if let Some(results) = + try_decode_fast420_region_scaled_rgb_batch_to_surfaces(runtime, requests, packets)? + { + return Ok(Some(results)); + } + if let Some(results) = + try_decode_fast422_region_scaled_rgb_batch_to_surfaces(runtime, requests, packets)? + { + return Ok(Some(results)); + } + + let mut results = Vec::with_capacity(requests.len()); + let has_region_scaled = requests + .iter() + .any(|request| matches!(request.op, batch::BatchOp::RegionScaled { .. })); + let chunk_size = if has_region_scaled { + REGION_SCALED_BATCH_CHUNK + } else { + requests.len().max(1) + }; + for chunk_start in (0..requests.len()).step_by(chunk_size) { + let chunk_end = (chunk_start + chunk_size).min(requests.len()); + let command_buffer = runtime.queue.new_command_buffer(); + let mut encoded = Vec::with_capacity(chunk_end - chunk_start); + let mut device_buffer_cache = BatchDeviceBufferCache::default(); + for index in chunk_start..chunk_end { + let request = &requests[index]; + let packet = &packets[index]; + let item = match packet { + BatchedFastPacket::Fast420(packet) => encode_fast_subsampled_op_batch_item( + runtime, + command_buffer, + &mut device_buffer_cache, + index, + *packet, + request.fmt, + request.op, + )?, + BatchedFastPacket::Fast422(packet) => encode_fast_subsampled_op_batch_item( + runtime, + command_buffer, + &mut device_buffer_cache, + index, + *packet, + request.fmt, + request.op, + )?, + BatchedFastPacket::Fast444(packet, mode) => match request.op { + batch::BatchOp::Full => encode_fast444_batch_item( + runtime, + command_buffer, + index, + packet, + *mode, + request.fmt, + )?, + batch::BatchOp::Region(roi) => encode_fast444_region_batch_item( + runtime, + command_buffer, + index, + packet, + *mode, + request.fmt, + roi, + )?, + batch::BatchOp::Scaled(scale) => encode_fast444_scaled_batch_item( + runtime, + command_buffer, + index, + packet, + *mode, + request.fmt, + scale, + )?, + batch::BatchOp::RegionScaled { roi, scale } => { + encode_fast444_scaled_region_batch_item( + runtime, + command_buffer, + &mut device_buffer_cache, + index, + packet, + *mode, + request.fmt, + roi, + scale, + )? + } + }, + }; + encoded.push(item); + } + + command_buffer.commit(); + command_buffer.wait_until_completed(); + + for item in encoded { + if let Some(status) = + first_decode_error_status(&item.status_buffer, item.decode_threads) + { + let request = &requests[item.request_index]; + let decoder = CpuDecoder::new(request.input.as_ref())?; + results.push(Err(decode_error_from_cpu(&decoder, request.fmt, status))); + } else { + results.push(Ok(item.surface)); + } + } + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast422_to_surface( + runtime: &MetalRuntime, + packet: Option<&JpegFast422PacketV1>, + fmt: PixelFormat, +) -> Result, Error> { + try_decode_fast_subsampled_to_surface(runtime, packet, fmt, fast422_status_error) +} + +#[cfg(target_os = "macos")] +fn decode_fast422_to_rgb_buffer( + runtime: &MetalRuntime, + packet: Option<&JpegFast422PacketV1>, + fmt: PixelFormat, + output_storage: MTLResourceOptions, +) -> Result, Error> { + decode_fast_subsampled_to_rgb_buffer(runtime, packet, fmt, output_storage, fast422_status_error) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast_subsampled_to_surface( + runtime: &MetalRuntime, + packet: Option<&P>, + fmt: PixelFormat, + map_status: impl Fn(JpegDecodeStatus) -> Error, +) -> Result, Error> { + let Some(decoded) = decode_fast_subsampled_to_rgb_buffer( + runtime, + packet, + fmt, + MTLResourceOptions::StorageModeShared, + map_status, + )? + else { + return Ok(None); + }; + Ok(Some(Surface::from_metal_buffer( + decoded.buffer, + decoded.dimensions, + fmt, + ))) +} + +#[cfg(target_os = "macos")] +fn decode_fast_subsampled_to_rgb_buffer( + runtime: &MetalRuntime, + packet: Option<&P>, + fmt: PixelFormat, + output_storage: MTLResourceOptions, + map_status: impl Fn(JpegDecodeStatus) -> Error, +) -> Result, Error> { + let Some(packet) = packet else { + return Ok(None); + }; + let Some(_out_format) = pixel_format_to_out_format(fmt) else { + return Ok(None); + }; + + let params = fast_subsampled_params(packet, fmt)?; + let y_len = params.width as usize * params.height as usize; + let chroma_len = params.chroma_width as usize * params.chroma_height as usize; + let y_plane = new_decode_plane_buffer(&runtime.device, y_len, fmt == PixelFormat::Gray8); + let cb_plane = new_private_buffer(&runtime.device, chroma_len); + let cr_plane = new_private_buffer(&runtime.device, chroma_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus(), + packet.restart_offsets().len(), + packet.entropy_checkpoints().len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes().as_ptr().cast(), + packet.entropy_bytes().len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, packet.restart_offsets())?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, packet.entropy_checkpoints())?; + + let dc_tables = [ + PreparedHuffmanHost::from(packet.y_dc_table()), + PreparedHuffmanHost::from(packet.cb_dc_table()), + PreparedHuffmanHost::from(packet.cr_dc_table()), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(packet.y_ac_table()), + PreparedHuffmanHost::from(packet.cb_ac_table()), + PreparedHuffmanHost::from(packet.cr_ac_table()), + ]; + + let out_buffer = (fmt != PixelFormat::Gray8).then(|| { + runtime.device.new_buffer( + (params.out_stride as usize * params.height as usize) as u64, + output_storage, + ) + }); + + let decode_pipeline = P::decode_pipeline(runtime); + let command_buffer = runtime.queue.new_command_buffer(); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &cb_plane, &cr_plane], + ¶ms, + [packet.y_quant(), packet.cb_quant(), packet.cr_quant()], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline(decoder_encoder, decode_pipeline, decode_threads); + decoder_encoder.end_encoding(); + + if let Some(out_buffer) = out_buffer.as_ref() { + let Some(pack_pipeline) = P::pack_pipeline_for_format(runtime, fmt) else { + return Ok(None); + }; + let pack_encoder = command_buffer.new_compute_command_encoder(); + pack_encoder.set_compute_pipeline_state(pack_pipeline); + pack_encoder.set_buffer(0, Some(&y_plane), 0); + pack_encoder.set_buffer(1, Some(&cb_plane), 0); + pack_encoder.set_buffer(2, Some(&cr_plane), 0); + pack_encoder.set_buffer(3, Some(out_buffer), 0); + pack_encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_2d_pipeline(pack_encoder, pack_pipeline, packet.dimensions()); + pack_encoder.end_encoding(); + } + + command_buffer.commit(); + command_buffer.wait_until_completed(); + let command_buffer = command_buffer.to_owned(); + + if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { + return Err(map_status(status)); + } + + Ok(Some(FastRgbDecodeBuffer { + buffer: out_buffer.unwrap_or(y_plane), + dimensions: packet.dimensions(), + status_buffer, + command_buffer, + })) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast_subsampled_region_to_surface( + runtime: &MetalRuntime, + packet: Option<&P>, + fmt: PixelFormat, + roi: j2k_jpeg::Rect, + map_status: impl Fn(JpegDecodeStatus) -> Error, +) -> Result, Error> { + let Some(packet) = packet else { + return Ok(None); + }; + let Some(_) = pixel_format_to_out_format(fmt) else { + return Ok(None); + }; + + let command_buffer = runtime.queue.new_command_buffer(); + let item = encode_fast_subsampled_region_batch_item( + runtime, + command_buffer, + 0, + packet, + fmt, + Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + )?; + command_buffer.commit(); + command_buffer.wait_until_completed(); + + if let Some(status) = first_decode_error_status(&item.status_buffer, item.decode_threads) { + return Err(map_status(status)); + } + + Ok(Some(item.surface)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast_subsampled_scaled_to_surface( + runtime: &MetalRuntime, + packet: Option<&P>, + fmt: PixelFormat, + scale: j2k_core::Downscale, + map_status: impl Fn(JpegDecodeStatus) -> Error, +) -> Result, Error> { + let Some(packet) = packet else { + return Ok(None); + }; + let Some(_) = pixel_format_to_out_format(fmt) else { + return Ok(None); + }; + if fast_subsampled_scaled_params(packet, scale).is_none() { + return Ok(None); + } + + let command_buffer = runtime.queue.new_command_buffer(); + let item = + encode_fast_subsampled_scaled_batch_item(runtime, command_buffer, 0, packet, fmt, scale)?; + command_buffer.commit(); + command_buffer.wait_until_completed(); + + if let Some(status) = first_decode_error_status(&item.status_buffer, item.decode_threads) { + return Err(map_status(status)); + } + + Ok(Some(item.surface)) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast422_region_to_surface( + runtime: &MetalRuntime, + packet: Option<&JpegFast422PacketV1>, + fmt: PixelFormat, + roi: j2k_jpeg::Rect, +) -> Result, Error> { + try_decode_fast_subsampled_region_to_surface(runtime, packet, fmt, roi, fast422_status_error) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast422_scaled_to_surface( + runtime: &MetalRuntime, + packet: Option<&JpegFast422PacketV1>, + fmt: PixelFormat, + scale: j2k_core::Downscale, +) -> Result, Error> { + try_decode_fast_subsampled_scaled_to_surface(runtime, packet, fmt, scale, fast422_status_error) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast422_scaled_region_to_surface( + runtime: &MetalRuntime, + packet: Option<&JpegFast422PacketV1>, + fmt: PixelFormat, + scaled_roi: j2k_jpeg::Rect, + scale: j2k_core::Downscale, +) -> Result, Error> { + try_decode_fast_subsampled_scaled_region_to_surface( + runtime, + packet, + fmt, + scaled_roi, + scale, + fast422_status_error, + ) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast_subsampled_scaled_region_to_surface( + runtime: &MetalRuntime, + packet: Option<&P>, + fmt: PixelFormat, + scaled_roi: j2k_jpeg::Rect, + scale: j2k_core::Downscale, + map_status: impl Fn(JpegDecodeStatus) -> Error, +) -> Result, Error> { + let Some(packet) = packet else { + return Ok(None); + }; + let Some(_) = pixel_format_to_out_format(fmt) else { + return Ok(None); + }; + let Some(full_params) = fast_subsampled_scaled_params(packet, scale) else { + return Ok(None); + }; + let source_window = fast_subsampled_full_mcu_scaled_window::

( + (full_params.scaled_width, full_params.scaled_height), + scaled_roi, + full_params.scale_shift, + ); + let Some(mut decode_params) = + fast_subsampled_scaled_region_params(packet, scale, source_window) + else { + return Ok(None); + }; + let mcu_width = P::MCU_WIDTH >> decode_params.scale_shift; + let mcu_height = P::MCU_HEIGHT >> decode_params.scale_shift; + let (first_mcu, end_mcu) = mcu_range_for_rect( + source_window, + packet.mcus_per_row(), + packet.mcu_rows(), + mcu_width, + mcu_height, + ); + let total_mcus = packet.mcus_per_row() * packet.mcu_rows(); + let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( + packet.restart_offsets(), + packet.restart_interval_mcus(), + total_mcus, + first_mcu, + end_mcu, + ); + decode_params.restart_start_mcu = restart_start_mcu; + decode_params.restart_offset_count = checked_entropy_segment_count( + packet.restart_interval_mcus(), + restart_offsets.len(), + packet.entropy_checkpoints().len(), + )?; + let local_roi = j2k_jpeg::Rect { + x: scaled_roi.x - source_window.x, + y: scaled_roi.y - source_window.y, + w: scaled_roi.w, + h: scaled_roi.h, + }; + let pack_params = fast_subsampled_windowed_pack_params_for_dims::

( + (source_window.w, source_window.h), + fmt, + local_roi, + )?; + let y_len = source_window.w as usize * source_window.h as usize; + let chroma_len = + source_window.w.div_ceil(2) as usize * P::chroma_height(source_window.h) as usize; + let y_plane = new_decode_plane_buffer(&runtime.device, y_len, false); + let cb_plane = new_private_buffer(&runtime.device, chroma_len); + let cr_plane = new_private_buffer(&runtime.device, chroma_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus(), + restart_offsets.len(), + packet.entropy_checkpoints().len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes().as_ptr().cast(), + packet.entropy_bytes().len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, packet.entropy_checkpoints())?; + + let dc_tables = [ + PreparedHuffmanHost::from(packet.y_dc_table()), + PreparedHuffmanHost::from(packet.cb_dc_table()), + PreparedHuffmanHost::from(packet.cr_dc_table()), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(packet.y_ac_table()), + PreparedHuffmanHost::from(packet.cb_ac_table()), + PreparedHuffmanHost::from(packet.cr_ac_table()), + ]; + + let out_buffer = runtime.device.new_buffer( + (pack_params.out_stride as usize * scaled_roi.h as usize) as u64, + MTLResourceOptions::StorageModeShared, + ); + + let decode_pipeline = P::scaled_region_decode_pipeline(runtime); + let command_buffer = runtime.queue.new_command_buffer(); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &cb_plane, &cr_plane], + &decode_params, + [packet.y_quant(), packet.cb_quant(), packet.cr_quant()], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline(decoder_encoder, decode_pipeline, decode_threads); + decoder_encoder.end_encoding(); + + let pack_encoder = command_buffer.new_compute_command_encoder(); + let pack_pipeline = P::pack_windowed_pipeline_for_format(runtime, fmt); + pack_encoder.set_compute_pipeline_state(pack_pipeline); + bind_three_plane_pack::( + pack_encoder, + [Some(&y_plane), Some(&cb_plane), Some(&cr_plane)], + &out_buffer, + &pack_params, + ); + dispatch_2d_pipeline(pack_encoder, pack_pipeline, (scaled_roi.w, scaled_roi.h)); + pack_encoder.end_encoding(); + + command_buffer.commit(); + command_buffer.wait_until_completed(); + + if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { + return Err(map_status(status)); + } + + Ok(Some(Surface::from_metal_buffer( + out_buffer, + (scaled_roi.w, scaled_roi.h), + fmt, + ))) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast420_to_surface( + runtime: &MetalRuntime, + decoder: &CpuDecoder<'_>, + packet: Option<&JpegFast420PacketV1>, + fmt: PixelFormat, +) -> Result, Error> { + try_decode_fast_subsampled_to_surface(runtime, packet, fmt, |status| { + decode_error_from_cpu(decoder, fmt, status) + }) +} + +#[cfg(target_os = "macos")] +fn decode_fast420_to_rgb_buffer( + runtime: &MetalRuntime, + decoder: &CpuDecoder<'_>, + packet: Option<&JpegFast420PacketV1>, + fmt: PixelFormat, + output_storage: MTLResourceOptions, +) -> Result, Error> { + decode_fast_subsampled_to_rgb_buffer(runtime, packet, fmt, output_storage, |status| { + decode_error_from_cpu(decoder, fmt, status) + }) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast420_region_to_surface( + runtime: &MetalRuntime, + decoder: &CpuDecoder<'_>, + packet: Option<&JpegFast420PacketV1>, + fmt: PixelFormat, + roi: j2k_jpeg::Rect, +) -> Result, Error> { + try_decode_fast_subsampled_region_to_surface(runtime, packet, fmt, roi, |status| { + decode_error_from_cpu(decoder, fmt, status) + }) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast420_scaled_to_surface( + runtime: &MetalRuntime, + decoder: &CpuDecoder<'_>, + packet: Option<&JpegFast420PacketV1>, + fmt: PixelFormat, + scale: j2k_core::Downscale, +) -> Result, Error> { + try_decode_fast_subsampled_scaled_to_surface(runtime, packet, fmt, scale, |status| { + decode_error_from_cpu(decoder, fmt, status) + }) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast420_scaled_region_to_surface( + runtime: &MetalRuntime, + decoder: &CpuDecoder<'_>, + packet: Option<&JpegFast420PacketV1>, + fmt: PixelFormat, + scaled_roi: j2k_jpeg::Rect, + scale: j2k_core::Downscale, +) -> Result, Error> { + try_decode_fast_subsampled_scaled_region_to_surface( + runtime, + packet, + fmt, + scaled_roi, + scale, + |status| decode_error_from_cpu(decoder, fmt, status), + ) +} + +#[cfg(target_os = "macos")] +fn fast444_plane_mode(decoder: &CpuDecoder<'_>) -> PlaneMode { + match decoder.info().color_space { + JpegColorSpace::Rgb => PlaneMode::Rgb, + _ => PlaneMode::YCbCr, + } +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_to_surface( + runtime: &MetalRuntime, + decoder: &CpuDecoder<'_>, + packet: Option<&JpegFast444PacketV1>, + fmt: PixelFormat, +) -> Result, Error> { + let Some(packet) = packet else { + return Ok(None); + }; + let Some(_) = pixel_format_to_out_format(fmt) else { + return Ok(None); + }; + + let params = fast444_params(packet)?; + let mode = fast444_plane_mode(decoder); + let plane_len = params.width as usize * params.height as usize; + let y_plane = new_decode_plane_buffer( + &runtime.device, + plane_len, + fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, + ); + let chroma_blue_plane = new_private_buffer(&runtime.device, plane_len); + let chroma_red_plane = new_private_buffer(&runtime.device, plane_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus, + packet.restart_offsets.len(), + packet.entropy_checkpoints.len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes.as_ptr().cast(), + packet.entropy_bytes.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; + + let dc_tables = [ + PreparedHuffmanHost::from(&packet.y_dc_table), + PreparedHuffmanHost::from(&packet.cb_dc_table), + PreparedHuffmanHost::from(&packet.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&packet.y_ac_table), + PreparedHuffmanHost::from(&packet.cb_ac_table), + PreparedHuffmanHost::from(&packet.cr_ac_table), + ]; + + let command_buffer = runtime.queue.new_command_buffer(); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(&runtime.fast444_decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &chroma_blue_plane, &chroma_red_plane], + ¶ms, + [&packet.y_quant, &packet.cb_quant, &packet.cr_quant], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_decode_pipeline, + decode_threads, + ); + decoder_encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { + return Err(decode_error_from_cpu(decoder, fmt, status)); + } + + PlaneStage { + dims: packet.dimensions, + mode, + plane0: y_plane, + plane1: Some(chroma_blue_plane), + plane2: Some(chroma_red_plane), + } + .finish_resident_with_runtime(runtime, fmt) + .map(Some) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_to_private_rgb8_tile( + runtime: &MetalRuntime, + decoder: &CpuDecoder<'_>, + packet: Option<&JpegFast444PacketV1>, +) -> Result, Error> { + let Some(packet) = packet else { + return Ok(None); + }; + + let params = fast444_params(packet)?; + let mode = fast444_plane_mode(decoder); + let plane_len = params.width as usize * params.height as usize; + let y_plane = new_private_buffer(&runtime.device, plane_len); + let chroma_blue_plane = new_private_buffer(&runtime.device, plane_len); + let chroma_red_plane = new_private_buffer(&runtime.device, plane_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus, + packet.restart_offsets.len(), + packet.entropy_checkpoints.len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes.as_ptr().cast(), + packet.entropy_bytes.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; + + let dc_tables = [ + PreparedHuffmanHost::from(&packet.y_dc_table), + PreparedHuffmanHost::from(&packet.cb_dc_table), + PreparedHuffmanHost::from(&packet.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&packet.y_ac_table), + PreparedHuffmanHost::from(&packet.cb_ac_table), + PreparedHuffmanHost::from(&packet.cr_ac_table), + ]; + + let command_buffer = runtime.queue.new_command_buffer(); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(&runtime.fast444_decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &chroma_blue_plane, &chroma_red_plane], + ¶ms, + [&packet.y_quant, &packet.cb_quant, &packet.cr_quant], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_decode_pipeline, + decode_threads, + ); + decoder_encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { + return Err(decode_error_from_cpu(decoder, PixelFormat::Rgb8, status)); + } + + Ok(Some( + PlaneStage { + dims: packet.dimensions, + mode, + plane0: y_plane, + plane1: Some(chroma_blue_plane), + plane2: Some(chroma_red_plane), + } + .dispatch_private_rgb8_with_runtime(runtime, status_buffer), + )) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_region_to_surface( + runtime: &MetalRuntime, + decoder: &CpuDecoder<'_>, + packet: Option<&JpegFast444PacketV1>, + fmt: PixelFormat, + roi: j2k_jpeg::Rect, +) -> Result, Error> { + let Some(packet) = packet else { + return Ok(None); + }; + let Some(_) = pixel_format_to_out_format(fmt) else { + return Ok(None); + }; + + let mut params = fast444_region_params(packet, roi)?; + let (first_mcu, end_mcu) = mcu_range_for_rect(roi, packet.mcus_per_row, packet.mcu_rows, 8, 8); + let total_mcus = packet.mcus_per_row * packet.mcu_rows; + let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( + &packet.restart_offsets, + packet.restart_interval_mcus, + total_mcus, + first_mcu, + end_mcu, + ); + params.restart_start_mcu = restart_start_mcu; + params.restart_offset_count = checked_entropy_segment_count( + packet.restart_interval_mcus, + restart_offsets.len(), + packet.entropy_checkpoints.len(), + )?; + let mode = fast444_plane_mode(decoder); + let plane_len = params.width as usize * params.height as usize; + let y_plane = new_decode_plane_buffer( + &runtime.device, + plane_len, + fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, + ); + let chroma_blue_plane = new_private_buffer(&runtime.device, plane_len); + let chroma_red_plane = new_private_buffer(&runtime.device, plane_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus, + restart_offsets.len(), + packet.entropy_checkpoints.len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes.as_ptr().cast(), + packet.entropy_bytes.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; + + let dc_tables = [ + PreparedHuffmanHost::from(&packet.y_dc_table), + PreparedHuffmanHost::from(&packet.cb_dc_table), + PreparedHuffmanHost::from(&packet.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&packet.y_ac_table), + PreparedHuffmanHost::from(&packet.cb_ac_table), + PreparedHuffmanHost::from(&packet.cr_ac_table), + ]; + + let command_buffer = runtime.queue.new_command_buffer(); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(&runtime.fast444_region_decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &chroma_blue_plane, &chroma_red_plane], + ¶ms, + [&packet.y_quant, &packet.cb_quant, &packet.cr_quant], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_region_decode_pipeline, + decode_threads, + ); + decoder_encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { + return Err(decode_error_from_cpu(decoder, fmt, status)); + } + + PlaneStage { + dims: (roi.w, roi.h), + mode, + plane0: y_plane, + plane1: Some(chroma_blue_plane), + plane2: Some(chroma_red_plane), + } + .finish_resident_with_runtime(runtime, fmt) + .map(Some) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_scaled_to_surface( + runtime: &MetalRuntime, + decoder: &CpuDecoder<'_>, + packet: Option<&JpegFast444PacketV1>, + fmt: PixelFormat, + scale: j2k_core::Downscale, +) -> Result, Error> { + let Some(packet) = packet else { + return Ok(None); + }; + let Some(_) = pixel_format_to_out_format(fmt) else { + return Ok(None); + }; + let Some(params) = fast444_scaled_params(packet, scale) else { + return Ok(None); + }; + + let mode = fast444_plane_mode(decoder); + let plane_len = params.scaled_width as usize * params.scaled_height as usize; + let y_plane = new_decode_plane_buffer( + &runtime.device, + plane_len, + fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, + ); + let chroma_blue_plane = new_private_buffer(&runtime.device, plane_len); + let chroma_red_plane = new_private_buffer(&runtime.device, plane_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus, + packet.restart_offsets.len(), + packet.entropy_checkpoints.len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes.as_ptr().cast(), + packet.entropy_bytes.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; + + let dc_tables = [ + PreparedHuffmanHost::from(&packet.y_dc_table), + PreparedHuffmanHost::from(&packet.cb_dc_table), + PreparedHuffmanHost::from(&packet.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&packet.y_ac_table), + PreparedHuffmanHost::from(&packet.cb_ac_table), + PreparedHuffmanHost::from(&packet.cr_ac_table), + ]; + + let command_buffer = runtime.queue.new_command_buffer(); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(&runtime.fast444_scaled_decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &chroma_blue_plane, &chroma_red_plane], + ¶ms, + [&packet.y_quant, &packet.cb_quant, &packet.cr_quant], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_scaled_decode_pipeline, + decode_threads, + ); + decoder_encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { + return Err(decode_error_from_cpu(decoder, fmt, status)); + } + + PlaneStage { + dims: (params.scaled_width, params.scaled_height), + mode, + plane0: y_plane, + plane1: Some(chroma_blue_plane), + plane2: Some(chroma_red_plane), + } + .finish_resident_with_runtime(runtime, fmt) + .map(Some) +} + +#[cfg(target_os = "macos")] +fn try_decode_fast444_scaled_region_to_surface( + runtime: &MetalRuntime, + decoder: &CpuDecoder<'_>, + packet: Option<&JpegFast444PacketV1>, + fmt: PixelFormat, + scaled_roi: j2k_jpeg::Rect, + scale: j2k_core::Downscale, +) -> Result, Error> { + let Some(packet) = packet else { + return Ok(None); + }; + let Some(_) = pixel_format_to_out_format(fmt) else { + return Ok(None); + }; + let Some(mut params) = fast444_scaled_region_params(packet, scale, scaled_roi) else { + return Ok(None); + }; + let mcu_size = 8u32 >> params.scale_shift; + let (first_mcu, end_mcu) = mcu_range_for_rect( + scaled_roi, + packet.mcus_per_row, + packet.mcu_rows, + mcu_size, + mcu_size, + ); + let total_mcus = packet.mcus_per_row * packet.mcu_rows; + let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( + &packet.restart_offsets, + packet.restart_interval_mcus, + total_mcus, + first_mcu, + end_mcu, + ); + params.restart_start_mcu = restart_start_mcu; + params.restart_offset_count = checked_entropy_segment_count( + packet.restart_interval_mcus, + restart_offsets.len(), + packet.entropy_checkpoints.len(), + )?; + + let mode = fast444_plane_mode(decoder); + let plane_len = params.scaled_width as usize * params.scaled_height as usize; + let y_plane = new_decode_plane_buffer( + &runtime.device, + plane_len, + fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, + ); + let chroma_blue_plane = new_private_buffer(&runtime.device, plane_len); + let chroma_red_plane = new_private_buffer(&runtime.device, plane_len); + let decode_threads = entropy_decode_thread_count( + packet.restart_interval_mcus, + restart_offsets.len(), + packet.entropy_checkpoints.len(), + ); + let status_buffer = decode_status_buffer(&runtime.device, decode_threads); + let entropy_buffer = runtime.device.new_buffer_with_data( + packet.entropy_bytes.as_ptr().cast(), + packet.entropy_bytes.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; + + let dc_tables = [ + PreparedHuffmanHost::from(&packet.y_dc_table), + PreparedHuffmanHost::from(&packet.cb_dc_table), + PreparedHuffmanHost::from(&packet.cr_dc_table), + ]; + let ac_tables = [ + PreparedHuffmanHost::from(&packet.y_ac_table), + PreparedHuffmanHost::from(&packet.cb_ac_table), + PreparedHuffmanHost::from(&packet.cr_ac_table), + ]; + + let command_buffer = runtime.queue.new_command_buffer(); + let decoder_encoder = command_buffer.new_compute_command_encoder(); + decoder_encoder.set_compute_pipeline_state(&runtime.fast444_scaled_region_decode_pipeline); + bind_fast_decode_entropy_inputs::( + decoder_encoder, + &entropy_buffer, + [&y_plane, &chroma_blue_plane, &chroma_red_plane], + ¶ms, + [&packet.y_quant, &packet.cb_quant, &packet.cr_quant], + &dc_tables, + &ac_tables, + &restart_offsets_buffer, + &status_buffer, + &entropy_checkpoints_buffer, + ); + dispatch_1d_pipeline( + decoder_encoder, + &runtime.fast444_scaled_region_decode_pipeline, + decode_threads, + ); + decoder_encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { + return Err(decode_error_from_cpu(decoder, fmt, status)); + } + + PlaneStage { + dims: (scaled_roi.w, scaled_roi.h), + mode, + plane0: y_plane, + plane1: Some(chroma_blue_plane), + plane2: Some(chroma_red_plane), + } + .finish_resident_with_runtime(runtime, fmt) + .map(Some) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_to_surface( + decoder: &CpuDecoder<'_>, + pool: &mut j2k_jpeg::ScratchPool, + fmt: PixelFormat, + fast444_packet: Option<&JpegFast444PacketV1>, + fast422_packet: Option<&JpegFast422PacketV1>, + fast420_packet: Option<&JpegFast420PacketV1>, +) -> Result { + with_runtime(|runtime| { + decode_to_surface_with_runtime( + runtime, + decoder, + pool, + fmt, + fast444_packet, + fast422_packet, + fast420_packet, + ) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_to_surface_with_session( + decoder: &CpuDecoder<'_>, + pool: &mut j2k_jpeg::ScratchPool, + fmt: PixelFormat, + fast444_packet: Option<&JpegFast444PacketV1>, + fast422_packet: Option<&JpegFast422PacketV1>, + fast420_packet: Option<&JpegFast420PacketV1>, + session: &crate::MetalBackendSession, +) -> Result { + with_runtime_for_session(session, |runtime| { + decode_to_surface_with_runtime( + runtime, + decoder, + pool, + fmt, + fast444_packet, + fast422_packet, + fast420_packet, + ) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_private_rgb8_tile_with_session( + decoder: &CpuDecoder<'_>, + fast444_packet: Option<&JpegFast444PacketV1>, + fast422_packet: Option<&JpegFast422PacketV1>, + fast420_packet: Option<&JpegFast420PacketV1>, + session: &crate::MetalBackendSession, +) -> Result { + with_runtime_for_session(session, |runtime| { + if let Some(tile) = + try_decode_fast444_to_private_rgb8_tile(runtime, decoder, fast444_packet)? + { + return Ok(tile); + } + if let Some(decoded) = decode_fast422_to_rgb_buffer( + runtime, + fast422_packet, + PixelFormat::Rgb8, + MTLResourceOptions::StorageModePrivate, + )? { + return Ok(private_jpeg_tile_from_fast_rgb_buffer(decoded)); + } + if let Some(decoded) = decode_fast420_to_rgb_buffer( + runtime, + decoder, + fast420_packet, + PixelFormat::Rgb8, + MTLResourceOptions::StorageModePrivate, + )? { + return Ok(private_jpeg_tile_from_fast_rgb_buffer(decoded)); + } + Err(Error::UnsupportedMetalRequest { + reason: + "private JPEG Metal output supports only fast baseline 4:4:4, 4:2:2, or 4:2:0 RGB8 full-tile decode", + }) + }) +} + +#[cfg(target_os = "macos")] +fn decode_to_surface_with_runtime( + runtime: &MetalRuntime, + decoder: &CpuDecoder<'_>, + pool: &mut j2k_jpeg::ScratchPool, + fmt: PixelFormat, + fast444_packet: Option<&JpegFast444PacketV1>, + fast422_packet: Option<&JpegFast422PacketV1>, + fast420_packet: Option<&JpegFast420PacketV1>, +) -> Result { + if let Some(surface) = try_decode_fast444_to_surface(runtime, decoder, fast444_packet, fmt)? { + return Ok(surface); + } + if let Some(surface) = try_decode_fast422_to_surface(runtime, fast422_packet, fmt)? { + return Ok(surface); + } + if let Some(surface) = try_decode_fast420_to_surface(runtime, decoder, fast420_packet, fmt)? { + return Ok(surface); + } + let mut stage = PlaneStage::new( + &runtime.device, + decoder.info().color_space, + decoder.info().dimensions, + )?; + decoder.decode_component_rows_with_scratch(pool, &mut stage)?; + stage.finish_with_runtime(runtime, fmt) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_region_to_surface( + decoder: &CpuDecoder<'_>, + pool: &mut j2k_jpeg::ScratchPool, + fmt: PixelFormat, + roi: j2k_jpeg::Rect, + fast444_packet: Option<&JpegFast444PacketV1>, + fast422_packet: Option<&JpegFast422PacketV1>, + fast420_packet: Option<&JpegFast420PacketV1>, +) -> Result { + with_runtime(|runtime| { + if let Some(surface) = + try_decode_fast444_region_to_surface(runtime, decoder, fast444_packet, fmt, roi)? + { + return Ok(surface); + } + if let Some(surface) = + try_decode_fast422_region_to_surface(runtime, fast422_packet, fmt, roi)? + { + return Ok(surface); + } + if let Some(surface) = + try_decode_fast420_region_to_surface(runtime, decoder, fast420_packet, fmt, roi)? + { + return Ok(surface); + } + let dims = (roi.w, roi.h); + let mut stage = cached_plane_stage(runtime, decoder.info().color_space, dims)?; + decoder.decode_region_component_rows_with_scratch( + pool, + &mut stage, + roi, + j2k_core::Downscale::None, + )?; + stage.finish_with_runtime(runtime, fmt) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_scaled_to_surface( + decoder: &CpuDecoder<'_>, + pool: &mut j2k_jpeg::ScratchPool, + fmt: PixelFormat, + scale: j2k_core::Downscale, + fast444_packet: Option<&JpegFast444PacketV1>, + fast422_packet: Option<&JpegFast422PacketV1>, + fast420_packet: Option<&JpegFast420PacketV1>, +) -> Result { + with_runtime(|runtime| { + if let Some(surface) = + try_decode_fast444_scaled_to_surface(runtime, decoder, fast444_packet, fmt, scale)? + { + return Ok(surface); + } + if let Some(surface) = + try_decode_fast422_scaled_to_surface(runtime, fast422_packet, fmt, scale)? + { + return Ok(surface); + } + if let Some(surface) = + try_decode_fast420_scaled_to_surface(runtime, decoder, fast420_packet, fmt, scale)? + { + return Ok(surface); + } + let full = decoder.info().dimensions; + let roi = j2k_jpeg::Rect { + x: 0, + y: 0, + w: full.0, + h: full.1, + }; + let scaled = (Rect { + x: 0, + y: 0, + w: full.0, + h: full.1, + }) + .scaled_covering(scale); + let mut stage = + cached_plane_stage(runtime, decoder.info().color_space, (scaled.w, scaled.h))?; + decoder.decode_region_component_rows_with_scratch(pool, &mut stage, roi, scale)?; + stage.finish_with_runtime(runtime, fmt) + }) +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +pub(crate) fn decode_region_scaled_to_surface( + decoder: &CpuDecoder<'_>, + pool: &mut j2k_jpeg::ScratchPool, + fmt: PixelFormat, + roi: j2k_jpeg::Rect, + scale: j2k_core::Downscale, + fast444_packet: Option<&JpegFast444PacketV1>, + fast422_packet: Option<&JpegFast422PacketV1>, + fast420_packet: Option<&JpegFast420PacketV1>, +) -> Result { + with_runtime(|runtime| { + let scaled_roi = (Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }) + .scaled_covering(scale); + if let Some(surface) = try_decode_fast444_scaled_region_to_surface( + runtime, + decoder, + fast444_packet, + fmt, + j2k_jpeg::Rect { + x: scaled_roi.x, + y: scaled_roi.y, + w: scaled_roi.w, + h: scaled_roi.h, + }, + scale, + )? { + return Ok(surface); + } + if let Some(surface) = try_decode_fast422_scaled_region_to_surface( + runtime, + fast422_packet, + fmt, + j2k_jpeg::Rect { + x: scaled_roi.x, + y: scaled_roi.y, + w: scaled_roi.w, + h: scaled_roi.h, + }, + scale, + )? { + return Ok(surface); + } + if let Some(surface) = try_decode_fast420_scaled_region_to_surface( + runtime, + decoder, + fast420_packet, + fmt, + j2k_jpeg::Rect { + x: scaled_roi.x, + y: scaled_roi.y, + w: scaled_roi.w, + h: scaled_roi.h, + }, + scale, + )? { + return Ok(surface); + } + let scaled = (Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }) + .scaled_covering(scale); + let mut stage = + cached_plane_stage(runtime, decoder.info().color_space, (scaled.w, scaled.h))?; + decoder.decode_region_component_rows_with_scratch(pool, &mut stage, roi, scale)?; + stage.finish_with_runtime(runtime, fmt) + }) +} + +#[cfg(all(test, target_os = "macos"))] +mod tests; diff --git a/crates/j2k-jpeg-metal/src/compute/batch_plan.rs b/crates/j2k-jpeg-metal/src/compute/batch_plan.rs new file mode 100644 index 00000000..129fb274 --- /dev/null +++ b/crates/j2k-jpeg-metal/src/compute/batch_plan.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_core::{BackendRequest, Rect}; +use j2k_jpeg::{ + adapter::{ + JpegEntropyCheckpointV1, JpegFast420PacketV1, JpegFast422PacketV1, JpegFast444PacketV1, + }, + Decoder as CpuDecoder, +}; +use metal::{Buffer, MTLResourceOptions}; + +use super::{entropy_checkpoints_buffer, fast444_plane_mode, pixel_format_to_out_format}; +use super::{MetalRuntime, PlaneMode}; +use crate::{batch, Error, Surface}; + +const AUTO_METAL_MIN_BATCH_REQUESTS: usize = 8; +const AUTO_METAL_MIN_BATCH_EDGE: u32 = 512; + +pub(super) enum BatchedFastPacket<'a> { + Fast420(&'a JpegFast420PacketV1), + Fast422(&'a JpegFast422PacketV1), + Fast444(&'a JpegFast444PacketV1, PlaneMode), +} + +pub(super) struct BatchedDecodeItem { + pub(super) request_index: usize, + pub(super) surface: Surface, + pub(super) status_buffer: Buffer, + pub(super) decode_threads: u32, + pub(super) _decode_resources: Vec, +} + +#[derive(Default)] +pub(super) struct BatchDeviceBufferCache { + packet_buffers: Vec, +} + +struct SharedPacketDeviceBuffers { + entropy_ptr: usize, + entropy_len: usize, + checkpoints_ptr: usize, + checkpoints_len: usize, + entropy_buffer: Buffer, + entropy_checkpoints_buffer: Buffer, +} + +impl BatchDeviceBufferCache { + pub(super) fn packet_buffers( + &mut self, + runtime: &MetalRuntime, + entropy_bytes: &[u8], + entropy_checkpoints: &[JpegEntropyCheckpointV1], + ) -> Result<(Buffer, Buffer), Error> { + let entropy_ptr = entropy_bytes.as_ptr() as usize; + let entropy_len = entropy_bytes.len(); + let checkpoints_ptr = entropy_checkpoints.as_ptr() as usize; + let checkpoints_len = entropy_checkpoints.len(); + if let Some(entry) = self.packet_buffers.iter().find(|entry| { + entry.entropy_ptr == entropy_ptr + && entry.entropy_len == entropy_len + && entry.checkpoints_ptr == checkpoints_ptr + && entry.checkpoints_len == checkpoints_len + }) { + return Ok(( + entry.entropy_buffer.clone(), + entry.entropy_checkpoints_buffer.clone(), + )); + } + + let entropy_buffer = runtime.device.new_buffer_with_data( + entropy_bytes.as_ptr().cast(), + entropy_bytes.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let entropy_checkpoints_buffer = + entropy_checkpoints_buffer(&runtime.device, entropy_checkpoints)?; + self.packet_buffers.push(SharedPacketDeviceBuffers { + entropy_ptr, + entropy_len, + checkpoints_ptr, + checkpoints_len, + entropy_buffer: entropy_buffer.clone(), + entropy_checkpoints_buffer: entropy_checkpoints_buffer.clone(), + }); + Ok((entropy_buffer, entropy_checkpoints_buffer)) + } +} + +fn request_allows_batched_packet( + requests: &[batch::QueuedRequest], + request: &batch::QueuedRequest, + restart_interval_mcus: u32, + dimensions: (u32, u32), +) -> bool { + match request.backend { + BackendRequest::Metal => true, + BackendRequest::Auto => match request.op { + batch::BatchOp::RegionScaled { .. } => false, + _ => { + requests.len() >= AUTO_METAL_MIN_BATCH_REQUESTS + && (restart_interval_mcus != 0 + || auto_batch_work_is_large_enough(request, dimensions)) + } + }, + BackendRequest::Cpu | BackendRequest::Cuda => false, + } +} + +fn auto_batch_work_is_large_enough(request: &batch::QueuedRequest, dimensions: (u32, u32)) -> bool { + let dims = match request.op { + batch::BatchOp::Full | batch::BatchOp::Scaled(_) => dimensions, + batch::BatchOp::Region(roi) => (roi.w, roi.h), + batch::BatchOp::RegionScaled { .. } => return false, + }; + dims.0 >= AUTO_METAL_MIN_BATCH_EDGE && dims.1 >= AUTO_METAL_MIN_BATCH_EDGE +} + +pub(super) fn batched_fast_packets( + requests: &[batch::QueuedRequest], +) -> Result>>, Error> { + if requests.is_empty() { + return Ok(None); + } + + let mut packets = Vec::with_capacity(requests.len()); + for request in requests { + let batchable_op = match request.op { + batch::BatchOp::Full + | batch::BatchOp::Region(_) + | batch::BatchOp::Scaled( + j2k_core::Downscale::Half + | j2k_core::Downscale::Quarter + | j2k_core::Downscale::Eighth, + ) + | batch::BatchOp::RegionScaled { + scale: + j2k_core::Downscale::Half + | j2k_core::Downscale::Quarter + | j2k_core::Downscale::Eighth, + .. + } => true, + batch::BatchOp::Scaled(_) | batch::BatchOp::RegionScaled { .. } => false, + }; + if !batchable_op + || !matches!( + request.backend, + BackendRequest::Auto | BackendRequest::Metal + ) + || pixel_format_to_out_format(request.fmt).is_none() + { + return Ok(None); + } + + if let Some(packet) = request.fast420_packet.as_deref() { + if !request_allows_batched_packet( + requests, + request, + packet.restart_interval_mcus, + packet.dimensions, + ) { + return Ok(None); + } + packets.push(BatchedFastPacket::Fast420(packet)); + continue; + } + + if let Some(packet) = request.fast422_packet.as_deref() { + if !request_allows_batched_packet( + requests, + request, + packet.restart_interval_mcus, + packet.dimensions, + ) { + return Ok(None); + } + packets.push(BatchedFastPacket::Fast422(packet)); + continue; + } + + if let Some(packet) = request.fast444_packet.as_deref() { + if !request_allows_batched_packet( + requests, + request, + packet.restart_interval_mcus, + packet.dimensions, + ) { + return Ok(None); + } + let decoder = CpuDecoder::new(request.input.as_ref())?; + packets.push(BatchedFastPacket::Fast444( + packet, + fast444_plane_mode(&decoder), + )); + continue; + } + + return Ok(None); + } + + Ok(Some(packets)) +} + +pub(super) fn core_rect_to_jpeg(rect: Rect) -> j2k_jpeg::Rect { + j2k_jpeg::Rect { + x: rect.x, + y: rect.y, + w: rect.w, + h: rect.h, + } +} diff --git a/crates/j2k-jpeg-metal/src/compute/batch_support.rs b/crates/j2k-jpeg-metal/src/compute/batch_support.rs new file mode 100644 index 00000000..1abdc535 --- /dev/null +++ b/crates/j2k-jpeg-metal/src/compute/batch_support.rs @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::{ffi::OsStr, time::Duration}; + +use j2k_jpeg::adapter::JpegEntropyCheckpointV1; +use j2k_jpeg::Decoder as CpuDecoder; +use metal::Buffer; + +use crate::buffers::MetalBatchScratch; +use crate::{batch, Error, Surface}; + +use super::{ + checked_u32, decode_error_from_cpu, entropy_checkpoint_hosts, first_decode_error_status, + MetalRuntime, +}; + +const FAST420_BATCH_TIMING_ENV: &str = "J2K_JPEG_METAL_FAST420_BATCH_TIMING"; + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum FastBatchDecodeMode { + Fused, + #[cfg(test)] + SplitCoeffIdct, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug, Default)] +pub(super) struct FastBatchTiming { + pub(super) accepted: Duration, + pub(super) entropy_concat: Duration, + pub(super) buffer_alloc: Duration, + pub(super) encode_decode: Duration, + pub(super) wait_decode: Duration, + pub(super) encode_pack: Duration, + pub(super) wait_pack: Duration, + pub(super) total: Duration, +} + +#[cfg(target_os = "macos")] +impl FastBatchTiming { + fn micros(duration: Duration) -> u128 { + duration.as_micros() + } + + pub(super) fn log( + self, + tag: &'static str, + label: &str, + tile_count: usize, + dimensions: (u32, u32), + segment_count: usize, + ) { + j2k_profile::emit_profile_row_now( + "jpeg", + "decode", + tag, + &[ + ("mode", label.to_string()), + ("tiles", tile_count.to_string()), + ("dimensions", format!("{}x{}", dimensions.0, dimensions.1)), + ("segments", segment_count.to_string()), + ("accepted_us", Self::micros(self.accepted).to_string()), + ( + "entropy_concat_us", + Self::micros(self.entropy_concat).to_string(), + ), + ( + "buffer_alloc_us", + Self::micros(self.buffer_alloc).to_string(), + ), + ( + "encode_decode_us", + Self::micros(self.encode_decode).to_string(), + ), + ("wait_decode_us", Self::micros(self.wait_decode).to_string()), + ("encode_pack_us", Self::micros(self.encode_pack).to_string()), + ("wait_pack_us", Self::micros(self.wait_pack).to_string()), + ("total_us", Self::micros(self.total).to_string()), + ], + ); + } +} + +#[cfg(target_os = "macos")] +pub(super) fn fast_batch_decode_mode() -> FastBatchDecodeMode { + FastBatchDecodeMode::Fused +} + +#[cfg(target_os = "macos")] +pub(super) fn fast420_batch_timing_enabled() -> bool { + fast420_batch_timing_value_enabled(std::env::var_os(FAST420_BATCH_TIMING_ENV).as_deref()) +} + +#[cfg(target_os = "macos")] +pub(super) fn fast420_batch_timing_value_enabled(value: Option<&OsStr>) -> bool { + value.is_some_and(|value| value == OsStr::new("1")) +} + +#[cfg(target_os = "macos")] +pub(super) struct BatchEntropyBuffers { + pub(super) payload: Buffer, + pub(super) offsets: Buffer, + pub(super) lens: Buffer, + pub(super) checkpoints: Buffer, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +pub(super) struct BatchEntropyBufferKeys { + pub(super) payload: &'static str, + pub(super) offsets: &'static str, + pub(super) lens: &'static str, + pub(super) checkpoints: &'static str, +} + +#[cfg(target_os = "macos")] +pub(super) fn batch_entropy_buffers<'a>( + runtime: &MetalRuntime, + scratch: &mut MetalBatchScratch, + keys: BatchEntropyBufferKeys, + entropy_bytes_iter: impl Iterator + Clone, + entropy_checkpoints_iter: impl Iterator + Clone, + tile_count: usize, + segment_count: usize, +) -> Result, Error> { + let total_entropy_len = entropy_bytes_iter + .clone() + .map(<[u8]>::len) + .try_fold(0usize, usize::checked_add) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal region scaled batch entropy length overflowed".to_string(), + })?; + if total_entropy_len == 0 { + return Ok(None); + } + + let mut entropy_bytes = Vec::with_capacity(total_entropy_len); + let mut entropy_offsets = Vec::with_capacity(tile_count); + let mut entropy_lens = Vec::with_capacity(tile_count); + let mut entropy_checkpoints = Vec::with_capacity(tile_count * segment_count); + for (bytes, checkpoints) in entropy_bytes_iter.zip(entropy_checkpoints_iter) { + entropy_offsets.push(checked_u32( + entropy_bytes.len(), + "region scaled batch entropy offset", + )?); + entropy_lens.push(checked_u32( + bytes.len(), + "region scaled batch entropy length", + )?); + entropy_bytes.extend_from_slice(bytes); + entropy_checkpoints.extend(checkpoints.iter().copied()); + } + + let checkpoints = entropy_checkpoint_hosts(&entropy_checkpoints)?; + Ok(Some(BatchEntropyBuffers { + payload: scratch.shared_buffer_with_bytes(&runtime.device, keys.payload, &entropy_bytes), + offsets: scratch.shared_buffer_with_slice(&runtime.device, keys.offsets, &entropy_offsets), + lens: scratch.shared_buffer_with_slice(&runtime.device, keys.lens, &entropy_lens), + checkpoints: scratch.shared_buffer_with_slice( + &runtime.device, + keys.checkpoints, + &checkpoints, + ), + })) +} + +#[cfg(target_os = "macos")] +pub(super) fn region_scaled_batch_error_results( + requests: &[batch::QueuedRequest], + status_buffer: &Buffer, + total_decode_threads: u32, +) -> Result>>, Error> { + let Some(status) = first_decode_error_status(status_buffer, total_decode_threads) else { + return Ok(None); + }; + let mut results = Vec::with_capacity(requests.len()); + for request in requests { + let decoder = CpuDecoder::new(request.input.as_ref())?; + results.push(Err(decode_error_from_cpu(&decoder, request.fmt, status))); + } + Ok(Some(results)) +} + +#[cfg(target_os = "macos")] +pub(super) fn texture_batch_error_results( + requests: &[batch::QueuedRequest], + status_buffer: &Buffer, + total_decode_threads: u32, +) -> Result>>, Error> { + let Some(status) = first_decode_error_status(status_buffer, total_decode_threads) else { + return Ok(None); + }; + let mut results = Vec::with_capacity(requests.len()); + for request in requests { + let decoder = CpuDecoder::new(request.input.as_ref())?; + results.push(Err(decode_error_from_cpu(&decoder, request.fmt, status))); + } + Ok(Some(results)) +} diff --git a/crates/j2k-jpeg-metal/src/compute/kernel_helpers.rs b/crates/j2k-jpeg-metal/src/compute/kernel_helpers.rs new file mode 100644 index 00000000..3bc45a99 --- /dev/null +++ b/crates/j2k-jpeg-metal/src/compute/kernel_helpers.rs @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::mem::size_of; + +use j2k_core::PixelFormat; +use metal::{Buffer, ComputePipelineState, MTLSize}; + +use super::{ + PlaneMode, PreparedHuffmanHost, MODE_GRAY, MODE_RGB, MODE_YCBCR, OUT_GRAY, OUT_RGB, OUT_RGBA, +}; + +#[cfg(target_os = "macos")] +#[cfg(target_os = "macos")] +/// Bind the shared fast-decode entropy kernel inputs at slots 0-16: entropy +/// bytes, the three component planes, the family params struct, the three +/// quantization tables, the per-component DC/AC Huffman table pairs, restart +/// offsets, decode status, and entropy checkpoints. +#[allow(clippy::too_many_arguments)] +pub(super) fn bind_fast_decode_entropy_inputs

( + encoder: &metal::ComputeCommandEncoderRef, + entropy_buffer: &Buffer, + planes: [&Buffer; 3], + params: &P, + quants: [&[u16; 64]; 3], + dc_tables: &[PreparedHuffmanHost; 3], + ac_tables: &[PreparedHuffmanHost; 3], + restart_offsets_buffer: &Buffer, + status_buffer: &Buffer, + entropy_checkpoints_buffer: &Buffer, +) { + encoder.set_buffer(0, Some(entropy_buffer), 0); + encoder.set_buffer(1, Some(planes[0]), 0); + encoder.set_buffer(2, Some(planes[1]), 0); + encoder.set_buffer(3, Some(planes[2]), 0); + encoder.set_bytes(4, size_of::

() as u64, (&raw const *params).cast()); + for (slot, quant) in (5u64..).zip(quants) { + encoder.set_bytes(slot, size_of::<[u16; 64]>() as u64, quant.as_ptr().cast()); + } + for (index, (dc, ac)) in dc_tables.iter().zip(ac_tables.iter()).enumerate() { + let slot = 8 + 2 * index as u64; + encoder.set_bytes( + slot, + size_of::() as u64, + (&raw const *dc).cast(), + ); + encoder.set_bytes( + slot + 1, + size_of::() as u64, + (&raw const *ac).cast(), + ); + } + encoder.set_buffer(14, Some(restart_offsets_buffer), 0); + encoder.set_buffer(15, Some(status_buffer), 0); + encoder.set_buffer(16, Some(entropy_checkpoints_buffer), 0); +} + +/// Bind the shared three-plane pack kernel layout at slots 0-4: the component +/// planes, the packed output buffer, and the pack params struct. +pub(super) fn bind_three_plane_pack

( + encoder: &metal::ComputeCommandEncoderRef, + planes: [Option<&Buffer>; 3], + out_buffer: &Buffer, + params: &P, +) { + encoder.set_buffer(0, planes[0].map(std::convert::AsRef::as_ref), 0); + encoder.set_buffer(1, planes[1].map(std::convert::AsRef::as_ref), 0); + encoder.set_buffer(2, planes[2].map(std::convert::AsRef::as_ref), 0); + encoder.set_buffer(3, Some(out_buffer), 0); + encoder.set_bytes(4, size_of::

() as u64, (&raw const *params).cast()); +} + +pub(super) fn dispatch_2d_pipeline( + encoder: &metal::ComputeCommandEncoderRef, + pipeline: &ComputePipelineState, + dims: (u32, u32), +) { + let width = pipeline.thread_execution_width().max(1); + let max_threads = pipeline.max_total_threads_per_threadgroup().max(width); + let height = (max_threads / width).max(1); + encoder.dispatch_threads( + MTLSize { + width: u64::from(dims.0), + height: u64::from(dims.1), + depth: 1, + }, + MTLSize { + width, + height, + depth: 1, + }, + ); +} + +#[cfg(target_os = "macos")] +pub(super) fn dispatch_3d_pipeline( + encoder: &metal::ComputeCommandEncoderRef, + pipeline: &ComputePipelineState, + dims: (u32, u32, u32), +) { + let width = pipeline.thread_execution_width().max(1); + let max_threads = pipeline.max_total_threads_per_threadgroup().max(width); + let height = (max_threads / width).max(1); + encoder.dispatch_threads( + MTLSize { + width: u64::from(dims.0), + height: u64::from(dims.1), + depth: u64::from(dims.2), + }, + MTLSize { + width, + height, + depth: 1, + }, + ); +} + +#[cfg(target_os = "macos")] +pub(super) fn packed_pair_extent(value: u32) -> u32 { + value.div_ceil(2).max(1) +} + +#[cfg(target_os = "macos")] +pub(super) fn dispatch_1d_pipeline( + encoder: &metal::ComputeCommandEncoderRef, + pipeline: &ComputePipelineState, + threads: u32, +) { + let threadgroup_width = choose_1d_threadgroup_width( + pipeline.thread_execution_width(), + pipeline.max_total_threads_per_threadgroup(), + threads, + ); + encoder.dispatch_threads( + MTLSize { + width: u64::from(threads.max(1)), + height: 1, + depth: 1, + }, + MTLSize { + width: threadgroup_width, + height: 1, + depth: 1, + }, + ); +} + +#[cfg(target_os = "macos")] +pub(super) fn choose_1d_threadgroup_width(simd_width: u64, max_threads: u64, threads: u32) -> u64 { + let simd_width = simd_width.max(1); + let max_threads = max_threads.max(simd_width); + let requested = u64::from(threads.max(1)); + let rounded = requested.div_ceil(simd_width) * simd_width; + rounded.clamp(simd_width, max_threads.min(256).max(simd_width)) +} + +#[cfg(target_os = "macos")] +pub(super) fn pixel_format_to_out_format(fmt: PixelFormat) -> Option { + match fmt { + PixelFormat::Gray8 => Some(OUT_GRAY), + PixelFormat::Rgb8 => Some(OUT_RGB), + PixelFormat::Rgba8 => Some(OUT_RGBA), + _ => None, + } +} + +#[cfg(target_os = "macos")] +pub(super) fn plane_mode_to_u32(mode: PlaneMode) -> u32 { + match mode { + PlaneMode::Gray => MODE_GRAY, + PlaneMode::YCbCr => MODE_YCBCR, + PlaneMode::Rgb => MODE_RGB, + } +} diff --git a/crates/j2k-jpeg-metal/src/compute/region_scaled_plan.rs b/crates/j2k-jpeg-metal/src/compute/region_scaled_plan.rs new file mode 100644 index 00000000..b1ec26fa --- /dev/null +++ b/crates/j2k-jpeg-metal/src/compute/region_scaled_plan.rs @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_core::{PixelFormat, Rect}; +use j2k_jpeg::adapter::JpegFast444PacketV1; + +use crate::{batch, Error}; + +use super::{ + checked_u32, fast444_scaled_region_params, fast_subsampled_full_mcu_scaled_window, + fast_subsampled_scaled_params, fast_subsampled_scaled_region_params, + fast_subsampled_windowed_pack_params_for_dims, FastSubsampledMetal, FastSubsampledPacket, + JpegFast444ScaledParams, JpegFastRegionScaledBatchParams, JpegWindowedPackBatchParams, + JpegWindowedTexturePackBatchParams, PlaneMode, MODE_YCBCR, OUT_RGB, +}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) struct RegionScaledBatchPlan { + pub(super) decode_params: JpegFastRegionScaledBatchParams, + pub(super) pack_params: JpegWindowedPackBatchParams, + pub(super) y_len: usize, + pub(super) chroma_len: usize, + pub(super) out_tile_len: usize, + pub(super) out_dims: (u32, u32), +} + +pub(super) fn windowed_texture_pack_params( + plan: RegionScaledBatchPlan, +) -> JpegWindowedTexturePackBatchParams { + JpegWindowedTexturePackBatchParams { + src_width: plan.pack_params.src_width, + src_height: plan.pack_params.src_height, + chroma_width: plan.pack_params.chroma_width, + chroma_height: plan.pack_params.chroma_height, + src_x: plan.pack_params.src_x, + src_y: plan.pack_params.src_y, + width: plan.pack_params.width, + height: plan.pack_params.height, + tile_index: 0, + alpha: u32::from(u8::MAX), + } +} + +pub(super) fn fast_subsampled_packets_share_full_rgb_batch_shape( + first: &P, + packet: &P, + segment_count: usize, +) -> bool { + (P::FULL_RGB_BATCH_SUPPORTS_RESTART || packet.restart_interval_mcus() == 0) + && packet.dimensions() == first.dimensions() + && packet.mcus_per_row() == first.mcus_per_row() + && packet.mcu_rows() == first.mcu_rows() + && packet.entropy_checkpoints().len() == segment_count + && packet.y_quant() == first.y_quant() + && packet.cb_quant() == first.cb_quant() + && packet.cr_quant() == first.cr_quant() + && packet.y_dc_table() == first.y_dc_table() + && packet.y_ac_table() == first.y_ac_table() + && packet.cb_dc_table() == first.cb_dc_table() + && packet.cb_ac_table() == first.cb_ac_table() + && packet.cr_dc_table() == first.cr_dc_table() + && packet.cr_ac_table() == first.cr_ac_table() +} + +pub(super) fn fast_subsampled_full_rgb_batch_groups( + packets: &[&P], +) -> Option>> { + let mut groups = Vec::>::new(); + 'packet: for (index, packet) in packets.iter().copied().enumerate() { + if packet.entropy_bytes().is_empty() || packet.entropy_checkpoints().is_empty() { + return None; + } + + for group in &mut groups { + let first = packets[group[0]]; + if fast_subsampled_packets_share_full_rgb_batch_shape( + first, + packet, + first.entropy_checkpoints().len(), + ) { + group.push(index); + continue 'packet; + } + } + groups.push(vec![index]); + } + Some(groups) +} + +pub(super) fn fast_subsampled_region_scaled_batch_plan( + packet: &P, + roi: Rect, + scale: j2k_core::Downscale, + tile_count: u32, + segment_count: u32, +) -> Option { + let full_params = fast_subsampled_scaled_params(packet, scale)?; + let scaled_roi = roi.scaled_covering(scale); + let scaled_roi = j2k_jpeg::Rect { + x: scaled_roi.x, + y: scaled_roi.y, + w: scaled_roi.w, + h: scaled_roi.h, + }; + let source_window = fast_subsampled_full_mcu_scaled_window::

( + (full_params.scaled_width, full_params.scaled_height), + scaled_roi, + full_params.scale_shift, + ); + let decode_params = fast_subsampled_scaled_region_params(packet, scale, source_window)?; + let local_roi = j2k_jpeg::Rect { + x: scaled_roi.x - source_window.x, + y: scaled_roi.y - source_window.y, + w: scaled_roi.w, + h: scaled_roi.h, + }; + let pack_params = fast_subsampled_windowed_pack_params_for_dims::

( + (source_window.w, source_window.h), + PixelFormat::Rgb8, + local_roi, + ) + .ok()?; + let out_stride = scaled_roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + Some(RegionScaledBatchPlan { + decode_params: JpegFastRegionScaledBatchParams { + scaled_width: decode_params.scaled_width, + scaled_height: decode_params.scaled_height, + chroma_width: decode_params.chroma_width, + chroma_height: decode_params.chroma_height, + mcus_per_row: decode_params.mcus_per_row, + mcu_rows: decode_params.mcu_rows, + segment_count, + tile_count, + scale_shift: decode_params.scale_shift, + origin_x: decode_params.origin_x, + origin_y: decode_params.origin_y, + }, + pack_params: JpegWindowedPackBatchParams { + src_width: pack_params.src_width, + src_height: pack_params.src_height, + chroma_width: pack_params.chroma_width, + chroma_height: pack_params.chroma_height, + src_x: pack_params.src_x, + src_y: pack_params.src_y, + width: pack_params.width, + height: pack_params.height, + tile_count, + out_stride: checked_u32(out_stride, P::REGION_SCALED_BATCH_OUT_STRIDE_CTX).ok()?, + alpha: u32::from(u8::MAX), + mode: MODE_YCBCR, + out_format: OUT_RGB, + }, + y_len: source_window.w as usize * source_window.h as usize, + chroma_len: source_window.w.div_ceil(2) as usize + * P::chroma_height(source_window.h) as usize, + out_tile_len: out_stride * scaled_roi.h as usize, + out_dims: (scaled_roi.w, scaled_roi.h), + }) +} + +struct FastRegionScaledGroup { + indices: Vec, + scale: j2k_core::Downscale, + plan: RegionScaledBatchPlan, +} + +pub(super) fn fast_subsampled_region_scaled_batch_groups( + requests: &[batch::QueuedRequest], + packets: &[&P], +) -> Result>>, Error> { + let mut groups = Vec::::new(); + 'packet: for (index, (request, packet)) in + requests.iter().zip(packets.iter().copied()).enumerate() + { + if packet.restart_interval_mcus() != 0 + || packet.entropy_bytes().is_empty() + || packet.entropy_checkpoints().is_empty() + { + return Ok(None); + } + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return Ok(None); + }; + let segment_count = packet.entropy_checkpoints().len(); + let segment_count_u32 = checked_u32( + segment_count, + &format!( + "{} region scaled texture batch segment count", + P::FAMILY_NAME + ), + )?; + let Some(plan) = + fast_subsampled_region_scaled_batch_plan(packet, roi, scale, 1, segment_count_u32) + else { + return Ok(None); + }; + + for group in &mut groups { + let first = packets[group.indices[0]]; + let first_segment_count = first.entropy_checkpoints().len(); + if scale == group.scale + && plan == group.plan + && fast_subsampled_packets_share_full_rgb_batch_shape( + first, + packet, + first_segment_count, + ) + { + group.indices.push(index); + continue 'packet; + } + } + groups.push(FastRegionScaledGroup { + indices: vec![index], + scale, + plan, + }); + } + Ok(Some( + groups.into_iter().map(|group| group.indices).collect(), + )) +} + +pub(super) fn fast444_packets_share_region_scaled_batch_shape( + first: &JpegFast444PacketV1, + packet: &JpegFast444PacketV1, + segment_count: usize, +) -> bool { + packet.restart_interval_mcus == 0 + && packet.dimensions == first.dimensions + && packet.mcus_per_row == first.mcus_per_row + && packet.mcu_rows == first.mcu_rows + && packet.entropy_checkpoints.len() == segment_count + && packet.y_quant == first.y_quant + && packet.cb_quant == first.cb_quant + && packet.cr_quant == first.cr_quant + && packet.y_dc_table == first.y_dc_table + && packet.y_ac_table == first.y_ac_table + && packet.cb_dc_table == first.cb_dc_table + && packet.cb_ac_table == first.cb_ac_table + && packet.cr_dc_table == first.cr_dc_table + && packet.cr_ac_table == first.cr_ac_table +} + +pub(super) fn fast444_full_rgb_batch_groups( + packets: &[(&JpegFast444PacketV1, PlaneMode)], +) -> Option>> { + let mut groups = Vec::>::new(); + 'packet: for (index, (packet, mode)) in packets.iter().copied().enumerate() { + if packet.restart_interval_mcus != 0 + || packet.entropy_bytes.is_empty() + || packet.entropy_checkpoints.is_empty() + { + return None; + } + + for group in &mut groups { + let (first, first_mode) = packets[group[0]]; + if mode == first_mode + && fast444_packets_share_region_scaled_batch_shape( + first, + packet, + first.entropy_checkpoints.len(), + ) + { + group.push(index); + continue 'packet; + } + } + groups.push(vec![index]); + } + Some(groups) +} + +struct Fast444RegionScaledGroup { + indices: Vec, + mode: PlaneMode, + scale: j2k_core::Downscale, + params: JpegFast444ScaledParams, +} + +pub(super) fn fast444_region_scaled_batch_groups( + requests: &[batch::QueuedRequest], + packets: &[(&JpegFast444PacketV1, PlaneMode)], +) -> Option>> { + let mut groups = Vec::::new(); + 'packet: for (index, (request, (packet, mode))) in + requests.iter().zip(packets.iter().copied()).enumerate() + { + if packet.restart_interval_mcus != 0 + || packet.entropy_bytes.is_empty() + || packet.entropy_checkpoints.is_empty() + { + return None; + } + let batch::BatchOp::RegionScaled { roi, scale } = request.op else { + return None; + }; + let scaled = roi.scaled_covering(scale); + let scaled_roi = j2k_jpeg::Rect { + x: scaled.x, + y: scaled.y, + w: scaled.w, + h: scaled.h, + }; + let params = fast444_scaled_region_params(packet, scale, scaled_roi)?; + + for group in &mut groups { + let (first, _) = packets[group.indices[0]]; + if mode == group.mode + && scale == group.scale + && params == group.params + && fast444_packets_share_region_scaled_batch_shape( + first, + packet, + first.entropy_checkpoints.len(), + ) + { + group.indices.push(index); + continue 'packet; + } + } + groups.push(Fast444RegionScaledGroup { + indices: vec![index], + mode, + scale, + params, + }); + } + Some(groups.into_iter().map(|group| group.indices).collect()) +} diff --git a/crates/j2k-jpeg-metal/src/compute/status.rs b/crates/j2k-jpeg-metal/src/compute/status.rs new file mode 100644 index 00000000..2f7450d9 --- /dev/null +++ b/crates/j2k-jpeg-metal/src/compute/status.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::mem::size_of_val; + +use j2k_core::PixelFormat; +use j2k_jpeg::Decoder as CpuDecoder; +use metal::{Buffer, Device}; + +use crate::{ + abi::{ + JpegBaselineEncodeStatus, JpegDecodeStatus, FAST420_STATUS_HUFFMAN, FAST420_STATUS_OK, + FAST420_STATUS_TRUNCATED, JPEG_BASELINE_ENCODE_STATUS_INVALID_PARAMS, + JPEG_BASELINE_ENCODE_STATUS_MISSING_HUFFMAN, JPEG_BASELINE_ENCODE_STATUS_OVERFLOW, + }, + buffers::new_shared_buffer_with_data, + Error, +}; + +pub(super) fn jpeg_baseline_encode_status_error(status: JpegBaselineEncodeStatus) -> Error { + let message = match status.code { + JPEG_BASELINE_ENCODE_STATUS_OVERFLOW => { + "JPEG Baseline Metal encode entropy output exceeded capacity".to_string() + } + JPEG_BASELINE_ENCODE_STATUS_MISSING_HUFFMAN => format!( + "JPEG Baseline Metal encode missing Huffman code for symbol {}", + status.detail + ), + JPEG_BASELINE_ENCODE_STATUS_INVALID_PARAMS => { + "JPEG Baseline Metal encode received invalid kernel parameters".to_string() + } + other => format!("JPEG Baseline Metal encode failed with status {other}"), + }; + Error::MetalKernel { message } +} + +pub(super) fn decode_error_from_cpu( + decoder: &CpuDecoder<'_>, + fmt: PixelFormat, + status: JpegDecodeStatus, +) -> Error { + if let Err(err) = decoder.decode(fmt) { + Error::Decode(err) + } else { + let reason = match status.code { + FAST420_STATUS_TRUNCATED => "truncated entropy stream", + FAST420_STATUS_HUFFMAN => "invalid Huffman stream", + _ => "unexpected Metal fast420 failure", + }; + Error::MetalKernel { + message: format!("{reason} at entropy byte {}", status.position), + } + } +} + +pub(super) fn decode_status_buffer(device: &Device, count: u32) -> Buffer { + let statuses = vec![JpegDecodeStatus::default(); count as usize]; + // SAFETY: The status Vec is initialized and viewed as immutable bytes for an immediate Metal + // buffer upload with the exact slice byte length. + let bytes = unsafe { + core::slice::from_raw_parts( + statuses.as_ptr().cast::(), + size_of_val(statuses.as_slice()), + ) + }; + new_shared_buffer_with_data(device, bytes) +} + +pub(super) fn first_decode_error_status(buffer: &Buffer, count: u32) -> Option { + // SAFETY: Decode status buffers are allocated for `count` JpegDecodeStatus entries before + // dispatch and are read only after the producing command buffer has completed. + let statuses = unsafe { + core::slice::from_raw_parts(buffer.contents().cast::(), count as usize) + }; + statuses + .iter() + .copied() + .find(|status| status.code != FAST420_STATUS_OK) +} + +pub(super) fn fast422_status_error(status: JpegDecodeStatus) -> Error { + Error::MetalKernel { + message: format!( + "unexpected Metal fast422 failure at entropy byte {}", + status.position + ), + } +} diff --git a/crates/j2k-jpeg-metal/src/compute/tests.rs b/crates/j2k-jpeg-metal/src/compute/tests.rs new file mode 100644 index 00000000..0c238c38 --- /dev/null +++ b/crates/j2k-jpeg-metal/src/compute/tests.rs @@ -0,0 +1,1612 @@ +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::Storage; +use std::sync::Arc; + +const BASELINE_420: &[u8] = include_bytes!("../../fixtures/jpeg/baseline_420_16x16.jpg"); +const BASELINE_420_RESTART: &[u8] = + include_bytes!("../../fixtures/jpeg/baseline_420_restart_32x16.jpg"); +const BASELINE_422: &[u8] = include_bytes!("../../fixtures/jpeg/baseline_422_16x8.jpg"); +const BASELINE_444: &[u8] = include_bytes!("../../fixtures/jpeg/baseline_444_8x8.jpg"); + +#[test] +fn mcu_range_for_rect_covers_only_touched_rows_and_columns() { + let roi = j2k_jpeg::Rect { + x: 19, + y: 35, + w: 11, + h: 34, + }; + + assert_eq!(mcu_range_for_rect(roi, 8, 6, 16, 16), (17, 34)); +} + +#[test] +fn restart_work_for_mcu_range_slices_to_overlapping_restart_segments() { + let restart_offsets = [0, 10, 20, 30, 40, 50]; + + let (restart_start_mcu, offsets) = restart_work_for_mcu_range(&restart_offsets, 4, 24, 9, 15); + + assert_eq!(restart_start_mcu, 8); + assert_eq!(offsets, &[20, 30]); +} + +#[test] +fn runtime_initialization_error_classifies_device_unavailable() { + assert!(matches!( + runtime_initialization_error(&MetalSupportError::MetalUnavailable), + Error::MetalUnavailable + )); + assert!(matches!( + runtime_initialization_error(&MetalSupportError::ShaderLibrary { + message: "failed to compile Metal library".to_string() + }), + Error::MetalRuntime { .. } + )); +} + +#[test] +fn fast420_params_rejects_output_stride_overflow_without_panic() { + let packet = minimal_fast420_packet((u32::MAX, 1)); + + let Err(err) = fast_subsampled_params(&packet, PixelFormat::Rgba8) else { + panic!("expected stride overflow"); + }; + + assert!(matches!(err, Error::MetalKernel { .. })); + assert!( + err.to_string().contains("fast420 output stride"), + "unexpected error: {err}" + ); +} + +#[test] +fn fast422_region_params_rejects_output_stride_overflow_without_panic() { + let packet = minimal_fast422_packet((16, 8)); + let source_window = j2k_jpeg::Rect { + x: 0, + y: 0, + w: u32::MAX, + h: 1, + }; + + let Err(err) = fast_subsampled_region_params(&packet, PixelFormat::Rgba8, source_window) else { + panic!("expected stride overflow"); + }; + + assert!(matches!(err, Error::MetalKernel { .. })); + assert!( + err.to_string().contains("fast422 region output stride"), + "unexpected error: {err}" + ); +} + +#[test] +fn fast444_params_accepts_minimal_packet() { + let packet = minimal_fast444_packet((8, 8)); + + let params = fast444_params(&packet).expect("fast444 params"); + + assert_eq!(params.width, 8); + assert_eq!(params.height, 8); + assert_eq!(params.restart_offset_count, 1); + assert_eq!(params.entropy_len, 0); +} + +#[test] +fn viewport_plane_cache_is_runtime_local() { + let runtime_a = MetalRuntime::new().expect("Metal runtime"); + let runtime_b = MetalRuntime::new_with_device(runtime_a.device.clone()).expect("Metal runtime"); + + let stage_a = cached_plane_stage(&runtime_a, JpegColorSpace::YCbCr, (8, 8)).expect("stage"); + let stage_b = cached_plane_stage(&runtime_b, JpegColorSpace::YCbCr, (8, 8)).expect("stage"); + drop((stage_a, stage_b)); + + assert_ne!( + runtime_a + .viewport_plane_cache_id_for_test() + .expect("cache") + .expect("cache entry"), + runtime_b + .viewport_plane_cache_id_for_test() + .expect("cache") + .expect("cache entry") + ); +} + +#[test] +fn shader_decode_block_clears_coefficients_with_vector_stores() { + assert!( + SHADER_SOURCE.contains("thread short4 *coeff_chunks"), + "decode_block should clear coeffs with packed short4 stores" + ); + assert!( + SHADER_SOURCE.contains("coeff_chunks[i] = short4(0);"), + "decode_block should zero each packed coefficient chunk" + ); +} + +#[test] +fn shader_source_keeps_entropy_fast_paths() { + assert!(SHADER_SOURCE.contains("inline bool refill_four_bytes(")); + assert!(SHADER_SOURCE.contains("return refill_four_bytes(br, bytes, len) || refill_one_byte")); + assert!(SHADER_SOURCE.contains("ensure_bits_padded(br, bytes, len, 9)")); + assert!(SHADER_SOURCE.contains("table.fast_len[fast_index]")); + assert!(SHADER_SOURCE.contains("inline bool decode_block_skip(")); + assert!(SHADER_SOURCE.contains("skip_receive_extend(br, bytes, len, ssss, status)")); + assert!(SHADER_SOURCE.contains("inline bool configure_batch_entropy_thread(")); +} + +#[test] +fn shader_kernels_use_incremental_mx_my() { + assert!( + SHADER_SOURCE.contains("inline void init_mcu_cursor("), + "fast decode kernels should seed mx/my via init_mcu_cursor instead of dividing per MCU" + ); + assert!( + SHADER_SOURCE.contains("inline void advance_mcu_cursor("), + "fast decode kernels should carry mx/my via advance_mcu_cursor instead of dividing per MCU" + ); + assert!( + !SHADER_SOURCE.contains("mcu_index / params.mcus_per_row"), + "no fast kernel should still divide mcu_index by mcus_per_row inside the MCU loop" + ); + assert!( + !SHADER_SOURCE.contains("mcu_index % params.mcus_per_row"), + "no fast kernel should still modulo mcu_index by mcus_per_row inside the MCU loop" + ); +} + +#[test] +fn fast420_batch_timing_env_requires_explicit_one() { + assert!(fast420_batch_timing_value_enabled(Some( + std::ffi::OsStr::new("1") + ))); + assert!(!fast420_batch_timing_value_enabled(Some( + std::ffi::OsStr::new("true") + ))); + assert!(!fast420_batch_timing_value_enabled(None)); +} + +#[test] +fn one_dimensional_dispatch_width_tracks_work_without_full_threadgroup_waste() { + assert_eq!(choose_1d_threadgroup_width(32, 1024, 1), 32); + assert_eq!(choose_1d_threadgroup_width(32, 1024, 33), 64); + assert_eq!(choose_1d_threadgroup_width(32, 1024, 256), 256); + assert_eq!(choose_1d_threadgroup_width(32, 1024, 257), 256); +} + +#[test] +fn auto_batched_packets_skip_distinct_region_scaled_requests() { + let packet = j2k_jpeg::adapter::build_fast420_packet(BASELINE_420_RESTART).expect("packet"); + let roi = Rect { + x: 0, + y: 0, + w: 16, + h: 16, + }; + let requests = vec![ + batch::QueuedRequest::new( + Arc::<[u8]>::from(BASELINE_420_RESTART), + PixelFormat::Rgb8, + BackendRequest::Auto, + batch::BatchOp::RegionScaled { + roi, + scale: j2k_core::Downscale::Quarter, + }, + None, + None, + Some(packet.clone()), + ), + batch::QueuedRequest::new( + Arc::<[u8]>::from(BASELINE_420_RESTART), + PixelFormat::Rgb8, + BackendRequest::Auto, + batch::BatchOp::RegionScaled { + roi, + scale: j2k_core::Downscale::Quarter, + }, + None, + None, + Some(packet), + ), + ]; + + assert!(batched_fast_packets(&requests) + .expect("packet lookup") + .is_none()); +} + +#[test] +fn auto_batched_packets_keep_large_repeated_region_scaled_requests_off_metal() { + let input = Arc::<[u8]>::from(BASELINE_420); + let packet = j2k_jpeg::adapter::build_fast420_packet(BASELINE_420).expect("packet"); + let roi = Rect { + x: 0, + y: 0, + w: 16, + h: 16, + }; + let requests = (0..=REGION_SCALED_BATCH_CHUNK) + .map(|_| { + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Auto, + batch::BatchOp::RegionScaled { + roi, + scale: j2k_core::Downscale::Quarter, + }, + None, + None, + Some(packet.clone()), + ) + }) + .collect::>(); + + assert!(batched_fast_packets(&requests) + .expect("packet lookup") + .is_none()); +} + +#[test] +fn auto_batched_packets_require_wsi_batch_threshold() { + let input = Arc::<[u8]>::from(BASELINE_420_RESTART); + let packet = j2k_jpeg::adapter::build_fast420_packet(BASELINE_420_RESTART).expect("packet"); + let requests = (0..7) + .map(|_| { + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Auto, + batch::BatchOp::Full, + None, + None, + Some(packet.clone()), + ) + }) + .collect::>(); + + assert!(batched_fast_packets(&requests) + .expect("packet lookup") + .is_none()); +} + +#[test] +fn auto_batched_packets_accept_restart_wsi_batch_at_threshold() { + let input = Arc::<[u8]>::from(BASELINE_420_RESTART); + let packet = j2k_jpeg::adapter::build_fast420_packet(BASELINE_420_RESTART).expect("packet"); + let requests = (0..8) + .map(|_| { + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Auto, + batch::BatchOp::Full, + None, + None, + Some(packet.clone()), + ) + }) + .collect::>(); + + assert!(batched_fast_packets(&requests) + .expect("packet lookup") + .is_some()); +} + +#[test] +fn auto_batched_packets_accept_large_nonrestart_wsi_batch_at_threshold() { + let input = Arc::<[u8]>::from(generated_rgb_jpeg(512)); + let fast444_packet = j2k_jpeg::adapter::build_fast444_packet(input.as_ref()).ok(); + let fast422_packet = j2k_jpeg::adapter::build_fast422_packet(input.as_ref()).ok(); + let fast420_packet = j2k_jpeg::adapter::build_fast420_packet(input.as_ref()).ok(); + assert!( + fast444_packet.is_some() || fast422_packet.is_some() || fast420_packet.is_some(), + "generated JPEG must be packet-decodable" + ); + let requests = (0..8) + .map(|_| { + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Auto, + batch::BatchOp::Full, + fast444_packet.clone(), + fast422_packet.clone(), + fast420_packet.clone(), + ) + }) + .collect::>(); + + assert!(batched_fast_packets(&requests) + .expect("packet lookup") + .is_some()); +} + +fn generated_rgb_jpeg(dim: u16) -> Vec { + let mut rgb = Vec::with_capacity(dim as usize * dim as usize * 3); + for y in 0..dim { + for x in 0..dim { + let xf = u32::from(x); + let yf = u32::from(y); + rgb.push(((xf * 13 + yf * 3) & 0xff) as u8); + rgb.push(((xf * 5 + yf * 11 + (xf ^ yf)) & 0xff) as u8); + rgb.push(((xf * 7 + yf * 17 + (xf.wrapping_mul(yf) >> 5)) & 0xff) as u8); + } + } + + let mut jpeg = Vec::new(); + let mut encoder = jpeg_encoder::Encoder::new(&mut jpeg, 90); + encoder.set_sampling_factor(jpeg_encoder::SamplingFactor::F_2_2); + encoder + .encode(&rgb, dim, dim, jpeg_encoder::ColorType::Rgb) + .expect("encode generated JPEG"); + jpeg +} + +#[test] +fn fast420_packet_scaled_decode_matches_cpu_scaled_bytes() { + let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); + let packet = j2k_jpeg::adapter::build_fast420_packet(BASELINE_420).expect("packet"); + for scale in [j2k_core::Downscale::Half, j2k_core::Downscale::Quarter] { + let (expected, _) = decoder + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled"); + + let surface = with_runtime(|runtime| { + let surface = try_decode_fast420_scaled_to_surface( + runtime, + &decoder, + Some(&packet), + PixelFormat::Rgb8, + scale, + )? + .expect("fast420 scaled surface"); + Ok::<_, Error>(surface) + }) + .expect("metal scaled"); + + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast420_packet_region_decode_matches_cpu_region_bytes() { + let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); + let packet = j2k_jpeg::adapter::build_fast420_packet(BASELINE_420).expect("packet"); + let roi = j2k_jpeg::Rect { + x: 3, + y: 2, + w: 9, + h: 10, + }; + let (expected, _) = decoder + .decode_region(PixelFormat::Rgb8, roi) + .expect("cpu region"); + + let surface = with_runtime(|runtime| { + let surface = try_decode_fast420_region_to_surface( + runtime, + &decoder, + Some(&packet), + PixelFormat::Rgb8, + roi, + )? + .expect("fast420 region surface"); + Ok::<_, Error>(surface) + }) + .expect("metal region"); + + assert_eq!(surface.dimensions, (roi.w, roi.h)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); +} + +#[test] +fn fast420_region_batch_decode_matches_cpu_region_bytes() { + let input = Arc::<[u8]>::from(BASELINE_420); + let packet = j2k_jpeg::adapter::build_fast420_packet(BASELINE_420).expect("packet"); + let roi = Rect { + x: 4, + y: 4, + w: 8, + h: 8, + }; + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Region(roi), + None, + None, + Some(packet.clone()), + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Region(roi), + None, + None, + Some(packet), + ), + ]; + let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); + let (expected, _) = decoder + .decode_region( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + ) + .expect("cpu region"); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("region batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.dimensions, (roi.w, roi.h)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast420_full_batch_decode_uses_shared_surface_offsets() { + let input = Arc::<[u8]>::from(BASELINE_420); + let packet = j2k_jpeg::adapter::build_fast420_packet(BASELINE_420).expect("packet"); + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Full, + None, + None, + Some(packet.clone()), + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Full, + None, + None, + Some(packet), + ), + ]; + let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); + let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("fast420 full batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for (index, result) in results.iter().enumerate() { + let surface = result.as_ref().expect("surface"); + assert_eq!(surface.dimensions, (16, 16)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); + let Storage::Metal { offset, .. } = &surface.storage else { + panic!("expected Metal storage"); + }; + assert_eq!(*offset, index * expected.len()); + } +} + +#[test] +fn fast420_split_full_batch_decode_matches_cpu_bytes() { + let jpeg = generated_rgb_jpeg(32); + let input = Arc::<[u8]>::from(jpeg.into_boxed_slice()); + let packet = j2k_jpeg::adapter::build_fast420_packet(input.as_ref()).expect("packet"); + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Full, + None, + None, + Some(packet.clone()), + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Full, + None, + None, + Some(packet), + ), + ]; + let packets = batched_fast_packets(&requests) + .expect("packet lookup") + .expect("packets"); + let decoder = CpuDecoder::new(input.as_ref()).expect("decoder"); + let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); + + let results = with_runtime(|runtime| { + try_decode_fast_subsampled_full_rgb_batch_to_surfaces_with_mode_and_output::< + JpegFast420PacketV1, + >( + runtime, + &requests, + &packets, + FastBatchDecodeMode::SplitCoeffIdct, + None, + ) + }) + .expect("batch result") + .expect("split fast420 full batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.dimensions, (32, 32)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast420_batch_clears_high_ac_before_dc_only_blocks() { + let input = Arc::<[u8]>::from(fast420_high_ac_then_dc_only_jpeg(1)); + let packet = j2k_jpeg::adapter::build_fast420_packet(input.as_ref()).expect("packet"); + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Full, + None, + None, + Some(packet.clone()), + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Full, + None, + None, + Some(packet), + ), + ]; + let decoder = CpuDecoder::new(input.as_ref()).expect("decoder"); + let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("fast420 full batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.dimensions, (16, 16)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast420_batch_matches_cpu_for_high_ac_overflow_coefficients() { + let input = Arc::<[u8]>::from(fast420_high_ac_then_dc_only_jpeg(u8::MAX)); + let packet = j2k_jpeg::adapter::build_fast420_packet(input.as_ref()).expect("packet"); + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Full, + None, + None, + Some(packet.clone()), + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Full, + None, + None, + Some(packet), + ), + ]; + let decoder = CpuDecoder::new(input.as_ref()).expect("decoder"); + let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("fast420 full batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.dimensions, (16, 16)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast420_packet_region_scaled_decode_matches_cpu_region_scaled_bytes() { + let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); + let packet = j2k_jpeg::adapter::build_fast420_packet(BASELINE_420).expect("packet"); + let roi = j2k_jpeg::Rect { + x: 3, + y: 2, + w: 9, + h: 10, + }; + let scale = j2k_core::Downscale::Quarter; + let (expected, _) = decoder + .decode_region_scaled(PixelFormat::Rgb8, roi, scale) + .expect("cpu region scaled"); + let scaled_roi = j2k_jpeg::Rect { + x: roi.x / 4, + y: roi.y / 4, + w: (roi.x + roi.w).div_ceil(4) - (roi.x / 4), + h: (roi.y + roi.h).div_ceil(4) - (roi.y / 4), + }; + + let surface = with_runtime(|runtime| { + let surface = try_decode_fast420_scaled_region_to_surface( + runtime, + &decoder, + Some(&packet), + PixelFormat::Rgb8, + scaled_roi, + scale, + )? + .expect("fast420 scaled region surface"); + Ok::<_, Error>(surface) + }) + .expect("metal region scaled"); + + assert_eq!(surface.dimensions, (scaled_roi.w, scaled_roi.h)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); +} + +#[test] +fn fast420_region_scaled_batch_decode_matches_cpu_region_scaled_bytes() { + let input = Arc::<[u8]>::from(BASELINE_420); + let packet = j2k_jpeg::adapter::build_fast420_packet(BASELINE_420).expect("packet"); + let roi = Rect { + x: 3, + y: 2, + w: 9, + h: 10, + }; + let scale = j2k_core::Downscale::Quarter; + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::RegionScaled { roi, scale }, + None, + None, + Some(packet.clone()), + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::RegionScaled { roi, scale }, + None, + None, + Some(packet), + ), + ]; + let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); + let (expected, _) = decoder + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region scaled"); + let scaled = roi.scaled_covering(scale); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("region scaled batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.dimensions, (scaled.w, scaled.h)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast420_scaled_batch_decode_matches_cpu_scaled_bytes() { + let input = Arc::<[u8]>::from(BASELINE_420); + let packet = j2k_jpeg::adapter::build_fast420_packet(BASELINE_420).expect("packet"); + let scale = j2k_core::Downscale::Quarter; + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Scaled(scale), + None, + None, + Some(packet.clone()), + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Scaled(scale), + None, + None, + Some(packet), + ), + ]; + let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); + let (expected, _) = decoder + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled"); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("scaled batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.dimensions, (4, 4)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast422_packet_full_decode_matches_cpu_bytes() { + let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); + let packet = j2k_jpeg::adapter::build_fast422_packet(BASELINE_422).expect("packet"); + let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); + + let surface = with_runtime(|runtime| { + let surface = try_decode_fast422_to_surface(runtime, Some(&packet), PixelFormat::Rgb8)? + .expect("fast422 surface"); + Ok::<_, Error>(surface) + }) + .expect("metal full decode"); + + assert_eq!(surface.as_bytes(), expected.as_slice()); +} + +#[test] +fn fast422_full_batch_decode_matches_cpu_bytes() { + let input = Arc::<[u8]>::from(BASELINE_422); + let packet = j2k_jpeg::adapter::build_fast422_packet(BASELINE_422).expect("packet"); + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Full, + None, + Some(packet.clone()), + None, + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Full, + None, + Some(packet), + None, + ), + ]; + let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); + let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("fast422 batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for (index, result) in results.iter().enumerate() { + let surface = result.as_ref().expect("surface"); + assert_eq!(surface.dimensions, (16, 8)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); + let Storage::Metal { offset, .. } = &surface.storage else { + panic!("expected Metal storage"); + }; + assert_eq!(*offset, index * expected.len()); + } +} + +#[test] +fn fast422_packet_region_decode_matches_cpu_region_bytes() { + let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); + let packet = j2k_jpeg::adapter::build_fast422_packet(BASELINE_422).expect("packet"); + let roi = j2k_jpeg::Rect { + x: 3, + y: 1, + w: 9, + h: 5, + }; + let (expected, _) = decoder + .decode_region(PixelFormat::Rgb8, roi) + .expect("cpu region"); + + let surface = with_runtime(|runtime| { + let surface = + try_decode_fast422_region_to_surface(runtime, Some(&packet), PixelFormat::Rgb8, roi)? + .expect("fast422 region surface"); + Ok::<_, Error>(surface) + }) + .expect("metal region"); + + assert_eq!(surface.dimensions, (roi.w, roi.h)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); +} + +#[test] +fn fast422_region_batch_decode_matches_cpu_region_bytes() { + let input = Arc::<[u8]>::from(BASELINE_422); + let packet = j2k_jpeg::adapter::build_fast422_packet(BASELINE_422).expect("packet"); + let roi = Rect { + x: 3, + y: 1, + w: 9, + h: 5, + }; + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Region(roi), + None, + Some(packet.clone()), + None, + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Region(roi), + None, + Some(packet), + None, + ), + ]; + let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); + let (expected, _) = decoder + .decode_region( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + ) + .expect("cpu region"); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("fast422 region batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.dimensions, (roi.w, roi.h)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast422_packet_scaled_decode_matches_cpu_scaled_bytes() { + let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); + let packet = j2k_jpeg::adapter::build_fast422_packet(BASELINE_422).expect("packet"); + for (scale, dims) in [ + (j2k_core::Downscale::Half, (8, 4)), + (j2k_core::Downscale::Quarter, (4, 2)), + ] { + let (expected, _) = decoder + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled"); + + let surface = with_runtime(|runtime| { + let surface = try_decode_fast422_scaled_to_surface( + runtime, + Some(&packet), + PixelFormat::Rgb8, + scale, + )? + .expect("fast422 scaled surface"); + Ok::<_, Error>(surface) + }) + .expect("metal scaled"); + + assert_eq!(surface.dimensions, dims); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast422_scaled_batch_decode_matches_cpu_scaled_bytes() { + let input = Arc::<[u8]>::from(BASELINE_422); + let packet = j2k_jpeg::adapter::build_fast422_packet(BASELINE_422).expect("packet"); + let scale = j2k_core::Downscale::Quarter; + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Scaled(scale), + None, + Some(packet.clone()), + None, + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Scaled(scale), + None, + Some(packet), + None, + ), + ]; + let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); + let (expected, _) = decoder + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled"); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("fast422 scaled batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.dimensions, (4, 2)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast422_packet_region_scaled_decode_matches_cpu_region_scaled_bytes() { + let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); + let packet = j2k_jpeg::adapter::build_fast422_packet(BASELINE_422).expect("packet"); + let roi = j2k_jpeg::Rect { + x: 3, + y: 1, + w: 9, + h: 5, + }; + let scale = j2k_core::Downscale::Half; + let (expected, _) = decoder + .decode_region_scaled(PixelFormat::Rgb8, roi, scale) + .expect("cpu region scaled"); + let scaled_roi = j2k_jpeg::Rect { + x: roi.x / 2, + y: roi.y / 2, + w: (roi.x + roi.w).div_ceil(2) - (roi.x / 2), + h: (roi.y + roi.h).div_ceil(2) - (roi.y / 2), + }; + + let surface = with_runtime(|runtime| { + let surface = try_decode_fast422_scaled_region_to_surface( + runtime, + Some(&packet), + PixelFormat::Rgb8, + scaled_roi, + scale, + )? + .expect("fast422 scaled region surface"); + Ok::<_, Error>(surface) + }) + .expect("metal region scaled"); + + assert_eq!(surface.dimensions, (scaled_roi.w, scaled_roi.h)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); +} + +#[test] +fn fast422_region_scaled_batch_decode_matches_cpu_region_scaled_bytes() { + let input = Arc::<[u8]>::from(BASELINE_422); + let packet = j2k_jpeg::adapter::build_fast422_packet(BASELINE_422).expect("packet"); + let roi = Rect { + x: 3, + y: 1, + w: 9, + h: 5, + }; + let scale = j2k_core::Downscale::Half; + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::RegionScaled { roi, scale }, + None, + Some(packet.clone()), + None, + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::RegionScaled { roi, scale }, + None, + Some(packet), + None, + ), + ]; + let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); + let (expected, _) = decoder + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region scaled"); + let scaled = roi.scaled_covering(scale); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("fast422 region scaled batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.dimensions, (scaled.w, scaled.h)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast444_packet_full_decode_matches_cpu_bytes() { + let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); + let packet = j2k_jpeg::adapter::build_fast444_packet(BASELINE_444).expect("packet"); + let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); + + let surface = with_runtime(|runtime| { + let surface = + try_decode_fast444_to_surface(runtime, &decoder, Some(&packet), PixelFormat::Rgb8)? + .expect("fast444 surface"); + Ok::<_, Error>(surface) + }) + .expect("metal full decode"); + + assert_eq!( + surface.residency(), + crate::SurfaceResidency::MetalResidentDecode + ); + assert_eq!(surface.as_bytes(), expected.as_slice()); +} + +#[test] +fn fast444_packet_scaled_decode_matches_cpu_scaled_bytes() { + let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); + let packet = j2k_jpeg::adapter::build_fast444_packet(BASELINE_444).expect("packet"); + for scale in [j2k_core::Downscale::Half, j2k_core::Downscale::Quarter] { + let (expected, _) = decoder + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled"); + + let surface = with_runtime(|runtime| { + let surface = try_decode_fast444_scaled_to_surface( + runtime, + &decoder, + Some(&packet), + PixelFormat::Rgb8, + scale, + )? + .expect("fast444 scaled surface"); + Ok::<_, Error>(surface) + }) + .expect("metal scaled"); + + assert_eq!( + surface.residency(), + crate::SurfaceResidency::MetalResidentDecode + ); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast444_packet_region_decode_matches_cpu_region_bytes() { + let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); + let packet = j2k_jpeg::adapter::build_fast444_packet(BASELINE_444).expect("packet"); + let roi = j2k_jpeg::Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let (expected, _) = decoder + .decode_region(PixelFormat::Rgb8, roi) + .expect("cpu region"); + + let surface = with_runtime(|runtime| { + let surface = try_decode_fast444_region_to_surface( + runtime, + &decoder, + Some(&packet), + PixelFormat::Rgb8, + roi, + )? + .expect("fast444 region surface"); + Ok::<_, Error>(surface) + }) + .expect("metal region"); + + assert_eq!(surface.dimensions, (roi.w, roi.h)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!( + surface.residency(), + crate::SurfaceResidency::MetalResidentDecode + ); + assert_eq!(surface.as_bytes(), expected.as_slice()); +} + +#[test] +fn fast444_region_batch_decode_matches_cpu_region_bytes() { + let input = Arc::<[u8]>::from(BASELINE_444); + let packet = j2k_jpeg::adapter::build_fast444_packet(BASELINE_444).expect("packet"); + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Region(roi), + Some(packet.clone()), + None, + None, + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Region(roi), + Some(packet), + None, + None, + ), + ]; + let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); + let (expected, _) = decoder + .decode_region( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + ) + .expect("cpu region"); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("region batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.dimensions, (roi.w, roi.h)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!( + surface.residency(), + crate::SurfaceResidency::MetalResidentDecode + ); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast444_packet_region_scaled_decode_matches_cpu_region_scaled_bytes() { + let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); + let packet = j2k_jpeg::adapter::build_fast444_packet(BASELINE_444).expect("packet"); + let roi = j2k_jpeg::Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let scale = j2k_core::Downscale::Quarter; + let (expected, _) = decoder + .decode_region_scaled(PixelFormat::Rgb8, roi, scale) + .expect("cpu region scaled"); + let scaled_roi = j2k_jpeg::Rect { + x: roi.x / 4, + y: roi.y / 4, + w: (roi.x + roi.w).div_ceil(4) - (roi.x / 4), + h: (roi.y + roi.h).div_ceil(4) - (roi.y / 4), + }; + + let surface = with_runtime(|runtime| { + let surface = try_decode_fast444_scaled_region_to_surface( + runtime, + &decoder, + Some(&packet), + PixelFormat::Rgb8, + scaled_roi, + scale, + )? + .expect("fast444 scaled region surface"); + Ok::<_, Error>(surface) + }) + .expect("metal region scaled"); + + assert_eq!(surface.dimensions, (scaled_roi.w, scaled_roi.h)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!( + surface.residency(), + crate::SurfaceResidency::MetalResidentDecode + ); + assert_eq!(surface.as_bytes(), expected.as_slice()); +} + +#[test] +fn fast444_region_scaled_batch_decode_matches_cpu_region_scaled_bytes() { + let input = Arc::<[u8]>::from(BASELINE_444); + let packet = j2k_jpeg::adapter::build_fast444_packet(BASELINE_444).expect("packet"); + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let scale = j2k_core::Downscale::Quarter; + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::RegionScaled { roi, scale }, + Some(packet.clone()), + None, + None, + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::RegionScaled { roi, scale }, + Some(packet), + None, + None, + ), + ]; + let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); + let (expected, _) = decoder + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region scaled"); + let scaled = roi.scaled_covering(scale); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("region scaled batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.dimensions, (scaled.w, scaled.h)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!( + surface.residency(), + crate::SurfaceResidency::MetalResidentDecode + ); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +#[test] +fn fast444_scaled_batch_decode_matches_cpu_scaled_bytes() { + let input = Arc::<[u8]>::from(BASELINE_444); + let packet = j2k_jpeg::adapter::build_fast444_packet(BASELINE_444).expect("packet"); + let scale = j2k_core::Downscale::Quarter; + let requests = vec![ + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Scaled(scale), + Some(packet.clone()), + None, + None, + ), + batch::QueuedRequest::new( + Arc::clone(&input), + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Scaled(scale), + Some(packet), + None, + None, + ), + ]; + let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); + let (expected, _) = decoder + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled"); + + let results = decode_full_batch_to_surfaces(&requests) + .expect("batch result") + .expect("scaled batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.dimensions, (2, 2)); + assert_eq!(surface.fmt, PixelFormat::Rgb8); + assert_eq!( + surface.residency(), + crate::SurfaceResidency::MetalResidentDecode + ); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } +} + +fn fast420_high_ac_then_dc_only_jpeg(ac_quant: u8) -> Vec { + assert!(ac_quant > 0, "JPEG quant entries must be nonzero"); + + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + + let mut quant = [1u8; 64]; + quant[63] = ac_quant; + let mut dqt = Vec::with_capacity(65); + dqt.push(0x00); + dqt.extend_from_slice(&quant); + append_jpeg_segment(&mut bytes, 0xdb, &dqt); + + append_jpeg_segment( + &mut bytes, + 0xc0, + &[ + 8, + 0, + 16, + 0, + 16, + 3, + 1, + (2 << 4) | 2, + 0, + 2, + (1 << 4) | 1, + 0, + 3, + (1 << 4) | 1, + 0, + ], + ); + + let mut dc_bits = [0u8; 16]; + dc_bits[0] = 1; + let mut dht_dc = Vec::with_capacity(18); + dht_dc.push(0x00); + dht_dc.extend_from_slice(&dc_bits); + dht_dc.push(0x00); + append_jpeg_segment(&mut bytes, 0xc4, &dht_dc); + + let mut ac_bits = [0u8; 16]; + ac_bits[1] = 3; + let mut dht_ac = Vec::with_capacity(20); + dht_ac.push(0x10); + dht_ac.extend_from_slice(&ac_bits); + dht_ac.extend_from_slice(&[0x00, 0xf0, 0xea]); + append_jpeg_segment(&mut bytes, 0xc4, &dht_ac); + + append_jpeg_segment(&mut bytes, 0xda, &[3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0]); + + bytes.extend_from_slice(&fast420_high_ac_entropy()); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn append_jpeg_segment(bytes: &mut Vec, marker: u8, payload: &[u8]) { + bytes.extend_from_slice(&[0xff, marker]); + let len = u16::try_from(payload.len() + 2).expect("JPEG segment length fits in u16"); + bytes.extend_from_slice(&len.to_be_bytes()); + bytes.extend_from_slice(payload); +} + +fn minimal_fast420_packet(dimensions: (u32, u32)) -> JpegFast420PacketV1 { + let [y_dc_table, y_ac_table, cb_dc_table, cb_ac_table, cr_dc_table, cr_ac_table] = + empty_packet_huffman_tables(); + JpegFast420PacketV1 { + dimensions, + mcus_per_row: 1, + mcu_rows: 1, + restart_interval_mcus: 0, + restart_offsets: vec![0], + entropy_checkpoints: vec![empty_entropy_checkpoint()], + y_quant: [1; 64], + cb_quant: [1; 64], + cr_quant: [1; 64], + y_dc_table, + y_ac_table, + cb_dc_table, + cb_ac_table, + cr_dc_table, + cr_ac_table, + entropy_bytes: Vec::new(), + } +} + +fn minimal_fast422_packet(dimensions: (u32, u32)) -> JpegFast422PacketV1 { + let [y_dc_table, y_ac_table, cb_dc_table, cb_ac_table, cr_dc_table, cr_ac_table] = + empty_packet_huffman_tables(); + JpegFast422PacketV1 { + dimensions, + mcus_per_row: 1, + mcu_rows: 1, + restart_interval_mcus: 0, + restart_offsets: vec![0], + entropy_checkpoints: vec![empty_entropy_checkpoint()], + y_quant: [1; 64], + cb_quant: [1; 64], + cr_quant: [1; 64], + y_dc_table, + y_ac_table, + cb_dc_table, + cb_ac_table, + cr_dc_table, + cr_ac_table, + entropy_bytes: Vec::new(), + } +} + +fn minimal_fast444_packet(dimensions: (u32, u32)) -> JpegFast444PacketV1 { + let [y_dc_table, y_ac_table, cb_dc_table, cb_ac_table, cr_dc_table, cr_ac_table] = + empty_packet_huffman_tables(); + JpegFast444PacketV1 { + dimensions, + mcus_per_row: 1, + mcu_rows: 1, + restart_interval_mcus: 0, + restart_offsets: vec![0], + entropy_checkpoints: vec![empty_entropy_checkpoint()], + y_quant: [1; 64], + cb_quant: [1; 64], + cr_quant: [1; 64], + y_dc_table, + y_ac_table, + cb_dc_table, + cb_ac_table, + cr_dc_table, + cr_ac_table, + entropy_bytes: Vec::new(), + } +} + +fn empty_packet_huffman_tables() -> [PacketHuffmanTable; 6] { + std::array::from_fn(|_| PacketHuffmanTable { + bits: [0; 16], + values_len: 0, + values: [0; 256], + }) +} + +fn empty_entropy_checkpoint() -> JpegEntropyCheckpointV1 { + JpegEntropyCheckpointV1 { + mcu_index: 0, + entropy_pos: 0, + bit_acc: 0, + bit_count: 0, + y_prev_dc: 0, + cb_prev_dc: 0, + cr_prev_dc: 0, + reserved: 0, + } +} + +fn fast420_high_ac_entropy() -> Vec { + let mut writer = EntropyBitWriter::default(); + emit_high_ac_block(&mut writer); + for _ in 0..5 { + emit_dc_only_block(&mut writer); + } + writer.finish() +} + +fn emit_high_ac_block(writer: &mut EntropyBitWriter) { + writer.push_bits(0, 1); + for _ in 0..3 { + writer.push_bits(0b01, 2); + } + writer.push_bits(0b10, 2); + writer.push_bits(0b11_1111_1111, 10); +} + +fn emit_dc_only_block(writer: &mut EntropyBitWriter) { + writer.push_bits(0, 1); + writer.push_bits(0b00, 2); +} + +#[derive(Default)] +struct EntropyBitWriter { + bytes: Vec, + current: u8, + bit_count: u8, +} + +impl EntropyBitWriter { + fn push_bits(&mut self, bits: u16, len: u8) { + for shift in (0..len).rev() { + let bit = u8::from(((bits >> shift) & 1) != 0); + self.current = (self.current << 1) | bit; + self.bit_count += 1; + if self.bit_count == 8 { + self.push_current_byte(); + } + } + } + + fn finish(mut self) -> Vec { + if self.bit_count != 0 { + let pad_bits = 8 - self.bit_count; + self.current = (self.current << pad_bits) | ((1u8 << pad_bits) - 1); + self.push_current_byte(); + } + self.bytes + } + + fn push_current_byte(&mut self) { + self.bytes.push(self.current); + if self.current == 0xff { + self.bytes.push(0x00); + } + self.current = 0; + self.bit_count = 0; + } +} diff --git a/crates/j2k-jpeg-metal/src/compute/viewport_cache.rs b/crates/j2k-jpeg-metal/src/compute/viewport_cache.rs new file mode 100644 index 00000000..29634b89 --- /dev/null +++ b/crates/j2k-jpeg-metal/src/compute/viewport_cache.rs @@ -0,0 +1,636 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::mem::size_of; + +use j2k_core::{PixelFormat, Rect}; +use j2k_jpeg::{ColorSpace as JpegColorSpace, ComponentRowWriter}; +use j2k_metal_support::dispatch_2d_pipeline; +use metal::{Buffer, Device, MTLResourceOptions}; + +use crate::buffers::new_private_buffer; +use crate::{Error, Surface}; + +use super::{ + batch_output_buffer_or_new, bind_three_plane_pack, validate_rgba_texture_batch_output, + JpegPackParams, JpegRgb8ToRgbaTextureParams, MetalRuntime, MODE_GRAY, MODE_RGB, MODE_YCBCR, + OUT_GRAY, OUT_RGB, OUT_RGBA, +}; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum PlaneMode { + Gray, + YCbCr, + Rgb, +} + +#[cfg(target_os = "macos")] +pub(super) struct PlaneStage { + pub(super) dims: (u32, u32), + pub(super) mode: PlaneMode, + pub(super) plane0: Buffer, + pub(super) plane1: Option, + pub(super) plane2: Option, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +enum PlaneStageResidency { + CpuStagedMetalUpload, + MetalResidentDecode, +} + +#[cfg(target_os = "macos")] +pub(super) struct ViewportPlaneWriter<'a> { + pub(super) stage: &'a mut PlaneStage, + pub(super) dest: Rect, +} + +#[cfg(target_os = "macos")] +pub(super) struct CachedViewportPlanes { + pub(super) dims: (u32, u32), + pub(super) mode: PlaneMode, + pub(super) plane0: Buffer, + pub(super) plane1: Option, + pub(super) plane2: Option, +} + +#[cfg(target_os = "macos")] +impl PlaneStage { + pub(super) fn new( + device: &Device, + color_space: JpegColorSpace, + dims: (u32, u32), + ) -> Result { + let len = dims.0 as usize * dims.1 as usize; + let plane0 = device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared); + let (mode, plane1, plane2) = match color_space { + JpegColorSpace::Grayscale => (PlaneMode::Gray, None, None), + JpegColorSpace::YCbCr => ( + PlaneMode::YCbCr, + Some(device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared)), + Some(device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared)), + ), + JpegColorSpace::Rgb => ( + PlaneMode::Rgb, + Some(device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared)), + Some(device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared)), + ), + JpegColorSpace::Cmyk | JpegColorSpace::Ycck => { + return Err(Error::MetalKernel { + message: "Metal compute path does not support CMYK/YCCK JPEG output" + .to_string(), + }) + } + }; + + Ok(Self { + dims, + mode, + plane0, + plane1, + plane2, + }) + } + + pub(super) fn finish_with_runtime( + self, + runtime: &MetalRuntime, + fmt: PixelFormat, + ) -> Result { + self.finish_with_runtime_and_residency( + runtime, + fmt, + PlaneStageResidency::CpuStagedMetalUpload, + ) + } + + pub(super) fn finish_resident_with_runtime( + self, + runtime: &MetalRuntime, + fmt: PixelFormat, + ) -> Result { + self.finish_with_runtime_and_residency( + runtime, + fmt, + PlaneStageResidency::MetalResidentDecode, + ) + } + + fn finish_with_runtime_and_residency( + self, + runtime: &MetalRuntime, + fmt: PixelFormat, + residency: PlaneStageResidency, + ) -> Result { + match (self.mode, fmt) { + (PlaneMode::Gray | PlaneMode::YCbCr, PixelFormat::Gray8) => Ok( + surface_from_plane_buffer(self.plane0, self.dims, fmt, residency), + ), + ( + PlaneMode::Gray | PlaneMode::YCbCr | PlaneMode::Rgb, + PixelFormat::Rgb8 | PixelFormat::Rgba8, + ) + | (PlaneMode::Rgb, PixelFormat::Gray8) => { + Ok(self.dispatch_with_runtime(runtime, fmt, residency)) + } + _ => Err(Error::MetalKernel { + message: format!("unsupported JPEG Metal pixel format {fmt:?}"), + }), + } + } + + fn dispatch_with_runtime( + self, + runtime: &MetalRuntime, + fmt: PixelFormat, + residency: PlaneStageResidency, + ) -> Surface { + let pitch_bytes = self.dims.0 as usize * fmt.bytes_per_pixel(); + let out_buffer = runtime.device.new_buffer( + (pitch_bytes * self.dims.1 as usize) as u64, + MTLResourceOptions::StorageModeShared, + ); + let params = JpegPackParams { + width: self.dims.0, + height: self.dims.1, + out_stride: u32::try_from(pitch_bytes).expect("JPEG Metal output stride fits in u32"), + alpha: u32::from(u8::MAX), + mode: match self.mode { + PlaneMode::Gray => MODE_GRAY, + PlaneMode::YCbCr => MODE_YCBCR, + PlaneMode::Rgb => MODE_RGB, + }, + out_format: match fmt { + PixelFormat::Gray8 => OUT_GRAY, + PixelFormat::Rgb8 => OUT_RGB, + PixelFormat::Rgba8 => OUT_RGBA, + _ => unreachable!("validated by finish"), + }, + }; + + let command_buffer = runtime.queue.new_command_buffer(); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.pack_pipeline); + bind_three_plane_pack::( + encoder, + [ + Some(&self.plane0), + self.plane1.as_ref(), + self.plane2.as_ref(), + ], + &out_buffer, + ¶ms, + ); + dispatch_2d_pipeline(encoder, &runtime.pack_pipeline, self.dims); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + surface_from_plane_buffer(out_buffer, self.dims, fmt, residency) + } + + pub(super) fn finish_rgb8_into_output_with_runtime( + self, + runtime: &MetalRuntime, + output: &crate::MetalBatchOutputBuffer, + ) -> Result { + let fmt = PixelFormat::Rgb8; + let pitch_bytes = self.dims.0 as usize * fmt.bytes_per_pixel(); + let tile_len = pitch_bytes * self.dims.1 as usize; + let out_buffer = + batch_output_buffer_or_new(runtime, Some(output), self.dims, 1, pitch_bytes, tile_len)?; + let params = JpegPackParams { + width: self.dims.0, + height: self.dims.1, + out_stride: u32::try_from(pitch_bytes).expect("JPEG Metal output stride fits in u32"), + alpha: u32::from(u8::MAX), + mode: match self.mode { + PlaneMode::Gray => MODE_GRAY, + PlaneMode::YCbCr => MODE_YCBCR, + PlaneMode::Rgb => MODE_RGB, + }, + out_format: OUT_RGB, + }; + + let command_buffer = runtime.queue.new_command_buffer(); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.pack_pipeline); + bind_three_plane_pack::( + encoder, + [ + Some(&self.plane0), + self.plane1.as_ref(), + self.plane2.as_ref(), + ], + &out_buffer, + ¶ms, + ); + dispatch_2d_pipeline(encoder, &runtime.pack_pipeline, self.dims); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + Ok(Surface::from_metal_buffer_offset( + out_buffer, self.dims, fmt, 0, + )) + } + + pub(super) fn finish_rgba8_into_texture_output_with_runtime( + self, + runtime: &MetalRuntime, + output: &crate::MetalBatchTextureOutput, + ) -> Result { + let rgb_pitch_bytes = self.dims.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let rgb_tile_len = rgb_pitch_bytes * self.dims.1 as usize; + let rgba_tile_len = + self.dims.0 as usize * self.dims.1 as usize * PixelFormat::Rgba8.bytes_per_pixel(); + validate_rgba_texture_batch_output(output, self.dims, 1, rgba_tile_len)?; + let out_buffer = { + let mut batch_scratch = runtime.batch_scratch()?; + batch_scratch.private_buffer( + &runtime.device, + "viewport_sparse_rgba_texture_rgb8", + rgb_tile_len, + ) + }; + let texture = output.texture(0).ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?; + let pack_params = JpegPackParams { + width: self.dims.0, + height: self.dims.1, + out_stride: u32::try_from(rgb_pitch_bytes) + .expect("JPEG Metal output stride fits in u32"), + alpha: u32::from(u8::MAX), + mode: match self.mode { + PlaneMode::Gray => MODE_GRAY, + PlaneMode::YCbCr => MODE_YCBCR, + PlaneMode::Rgb => MODE_RGB, + }, + out_format: OUT_RGB, + }; + let texture_params = JpegRgb8ToRgbaTextureParams { + width: self.dims.0, + height: self.dims.1, + in_stride: u32::try_from(rgb_pitch_bytes) + .expect("JPEG Metal RGB texture input stride fits in u32"), + alpha: u32::from(u8::MAX), + }; + + let command_buffer = runtime.queue.new_command_buffer(); + let pack_encoder = command_buffer.new_compute_command_encoder(); + pack_encoder.set_compute_pipeline_state(&runtime.pack_pipeline); + bind_three_plane_pack::( + pack_encoder, + [ + Some(&self.plane0), + self.plane1.as_ref(), + self.plane2.as_ref(), + ], + &out_buffer, + &pack_params, + ); + dispatch_2d_pipeline(pack_encoder, &runtime.pack_pipeline, self.dims); + pack_encoder.end_encoding(); + + let texture_encoder = command_buffer.new_compute_command_encoder(); + texture_encoder.set_compute_pipeline_state(&runtime.rgb8_to_rgba_texture_pipeline); + texture_encoder.set_buffer(0, Some(&out_buffer), 0); + texture_encoder.set_bytes( + 1, + size_of::() as u64, + (&raw const texture_params).cast(), + ); + texture_encoder.set_texture(0, Some(texture)); + dispatch_2d_pipeline( + texture_encoder, + &runtime.rgb8_to_rgba_texture_pipeline, + self.dims, + ); + texture_encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + let texture = output.clone_texture(0).ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?; + Ok(crate::MetalTextureTile::new( + texture, + self.dims, + PixelFormat::Rgba8, + )) + } + + pub(super) fn dispatch_private_rgb8_with_runtime( + self, + runtime: &MetalRuntime, + status_buffer: Buffer, + ) -> crate::ResidentPrivateJpegTile { + let fmt = PixelFormat::Rgb8; + let pitch_bytes = self.dims.0 as usize * fmt.bytes_per_pixel(); + let out_buffer = new_private_buffer(&runtime.device, pitch_bytes * self.dims.1 as usize); + let params = JpegPackParams { + width: self.dims.0, + height: self.dims.1, + out_stride: u32::try_from(pitch_bytes).expect("JPEG Metal output stride fits in u32"), + alpha: u32::from(u8::MAX), + mode: match self.mode { + PlaneMode::Gray => MODE_GRAY, + PlaneMode::YCbCr => MODE_YCBCR, + PlaneMode::Rgb => MODE_RGB, + }, + out_format: OUT_RGB, + }; + + let command_buffer = runtime.queue.new_command_buffer(); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.pack_pipeline); + bind_three_plane_pack::( + encoder, + [ + Some(&self.plane0), + self.plane1.as_ref(), + self.plane2.as_ref(), + ], + &out_buffer, + ¶ms, + ); + dispatch_2d_pipeline(encoder, &runtime.pack_pipeline, self.dims); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + let command_buffer = command_buffer.to_owned(); + + crate::ResidentPrivateJpegTile { + buffer: out_buffer, + byte_offset: 0, + dimensions: self.dims, + pixel_format: fmt, + pitch_bytes, + status_buffer, + command_buffer, + } + } +} + +#[cfg(target_os = "macos")] +fn surface_from_plane_buffer( + buffer: Buffer, + dims: (u32, u32), + fmt: PixelFormat, + residency: PlaneStageResidency, +) -> Surface { + match residency { + PlaneStageResidency::CpuStagedMetalUpload => { + Surface::from_cpu_staged_metal_buffer(buffer, dims, fmt) + } + PlaneStageResidency::MetalResidentDecode => Surface::from_metal_buffer(buffer, dims, fmt), + } +} + +#[cfg(target_os = "macos")] +impl ComponentRowWriter for PlaneStage { + fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), j2k_jpeg::JpegError> { + let width = self.dims.0 as usize; + write_row_u8(&self.plane0, y, width, gray_row); + Ok(()) + } + + fn write_ycbcr_row( + &mut self, + y: u32, + y_row: &[u8], + chroma_blue_row: &[u8], + chroma_red_row: &[u8], + ) -> Result<(), j2k_jpeg::JpegError> { + let width = self.dims.0 as usize; + write_row_u8(&self.plane0, y, width, y_row); + write_row_u8( + self.plane1.as_ref().expect("Cb plane"), + y, + width, + chroma_blue_row, + ); + write_row_u8( + self.plane2.as_ref().expect("Cr plane"), + y, + width, + chroma_red_row, + ); + Ok(()) + } + + fn write_rgb_row( + &mut self, + y: u32, + r_row: &[u8], + g_row: &[u8], + b_row: &[u8], + ) -> Result<(), j2k_jpeg::JpegError> { + let width = self.dims.0 as usize; + write_row_u8(&self.plane0, y, width, r_row); + write_row_u8(self.plane1.as_ref().expect("G plane"), y, width, g_row); + write_row_u8(self.plane2.as_ref().expect("B plane"), y, width, b_row); + Ok(()) + } +} + +#[cfg(target_os = "macos")] +fn write_row_u8(buffer: &Buffer, y: u32, width: usize, src: &[u8]) { + let row_start = y as usize * width; + let row_end = row_start + width; + let len = width * (y as usize + 1); + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let dst = unsafe { + core::slice::from_raw_parts_mut(buffer.contents().cast::(), len.max(row_end)) + }; + dst[row_start..row_end].copy_from_slice(&src[..width]); +} + +#[cfg(target_os = "macos")] +fn write_row_u8_at(buffer: &Buffer, y: u32, x: u32, full_width: usize, src: &[u8]) { + let row_start = y as usize * full_width + x as usize; + let row_end = row_start + src.len(); + let len = full_width * (y as usize + 1); + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let dst = unsafe { + core::slice::from_raw_parts_mut(buffer.contents().cast::(), len.max(row_end)) + }; + dst[row_start..row_end].copy_from_slice(src); +} + +#[cfg(target_os = "macos")] +fn plane_mode_for_color_space(color_space: JpegColorSpace) -> Result { + match color_space { + JpegColorSpace::Grayscale => Ok(PlaneMode::Gray), + JpegColorSpace::YCbCr => Ok(PlaneMode::YCbCr), + JpegColorSpace::Rgb => Ok(PlaneMode::Rgb), + JpegColorSpace::Cmyk | JpegColorSpace::Ycck => Err(Error::MetalKernel { + message: "Metal compute path does not support CMYK/YCCK JPEG output".to_string(), + }), + } +} + +#[cfg(target_os = "macos")] +fn clear_buffer(buffer: &Buffer, len: usize) { + fill_buffer(buffer, len, 0); +} + +#[cfg(target_os = "macos")] +fn fill_buffer(buffer: &Buffer, len: usize, value: u8) { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + unsafe { + core::ptr::write_bytes(buffer.contents().cast::(), value, len); + } +} + +#[cfg(target_os = "macos")] +pub(super) fn cached_plane_stage( + runtime: &MetalRuntime, + color_space: JpegColorSpace, + dims: (u32, u32), +) -> Result { + let mode = plane_mode_for_color_space(color_space)?; + let mut slot = runtime.viewport_plane_cache()?; + let len = dims.0 as usize * dims.1 as usize; + let refresh = slot + .as_ref() + .is_none_or(|cached| cached.dims != dims || cached.mode != mode); + if refresh { + let plane0 = runtime + .device + .new_buffer(len as u64, MTLResourceOptions::StorageModeShared); + let (plane1, plane2) = match mode { + PlaneMode::Gray => (None, None), + PlaneMode::YCbCr | PlaneMode::Rgb => ( + Some( + runtime + .device + .new_buffer(len as u64, MTLResourceOptions::StorageModeShared), + ), + Some( + runtime + .device + .new_buffer(len as u64, MTLResourceOptions::StorageModeShared), + ), + ), + }; + *slot = Some(CachedViewportPlanes { + dims, + mode, + plane0, + plane1, + plane2, + }); + } + + let cached = slot.as_ref().expect("viewport plane cache"); + let stage = PlaneStage { + dims, + mode, + plane0: cached.plane0.clone(), + plane1: cached.plane1.clone(), + plane2: cached.plane2.clone(), + }; + drop(slot); + + clear_buffer(&stage.plane0, len); + match stage.mode { + PlaneMode::Gray => {} + PlaneMode::YCbCr => { + if let Some(plane1) = &stage.plane1 { + fill_buffer(plane1, len, 128); + } + if let Some(plane2) = &stage.plane2 { + fill_buffer(plane2, len, 128); + } + } + PlaneMode::Rgb => { + if let Some(plane1) = &stage.plane1 { + clear_buffer(plane1, len); + } + if let Some(plane2) = &stage.plane2 { + clear_buffer(plane2, len); + } + } + } + Ok(stage) +} + +#[cfg(target_os = "macos")] +impl ComponentRowWriter for ViewportPlaneWriter<'_> { + fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), j2k_jpeg::JpegError> { + write_row_u8_at( + &self.stage.plane0, + self.dest.y + y, + self.dest.x, + self.stage.dims.0 as usize, + gray_row, + ); + Ok(()) + } + + fn write_ycbcr_row( + &mut self, + y: u32, + y_row: &[u8], + chroma_blue_row: &[u8], + chroma_red_row: &[u8], + ) -> Result<(), j2k_jpeg::JpegError> { + let width = self.stage.dims.0 as usize; + write_row_u8_at( + &self.stage.plane0, + self.dest.y + y, + self.dest.x, + width, + y_row, + ); + write_row_u8_at( + self.stage.plane1.as_ref().expect("Cb plane"), + self.dest.y + y, + self.dest.x, + width, + chroma_blue_row, + ); + write_row_u8_at( + self.stage.plane2.as_ref().expect("Cr plane"), + self.dest.y + y, + self.dest.x, + width, + chroma_red_row, + ); + Ok(()) + } + + fn write_rgb_row( + &mut self, + y: u32, + r_row: &[u8], + g_row: &[u8], + b_row: &[u8], + ) -> Result<(), j2k_jpeg::JpegError> { + let width = self.stage.dims.0 as usize; + write_row_u8_at( + &self.stage.plane0, + self.dest.y + y, + self.dest.x, + width, + r_row, + ); + write_row_u8_at( + self.stage.plane1.as_ref().expect("G plane"), + self.dest.y + y, + self.dest.x, + width, + g_row, + ); + write_row_u8_at( + self.stage.plane2.as_ref().expect("B plane"), + self.dest.y + y, + self.dest.x, + width, + b_row, + ); + Ok(()) + } +} diff --git a/crates/j2k-jpeg-metal/src/compute/viewport_compose.rs b/crates/j2k-jpeg-metal/src/compute/viewport_compose.rs new file mode 100644 index 00000000..cd442e90 --- /dev/null +++ b/crates/j2k-jpeg-metal/src/compute/viewport_compose.rs @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_core::PixelFormat; +use j2k_jpeg::Decoder as CpuDecoder; + +use crate::viewport::ViewportTile; +use crate::{Error, Surface}; + +use super::viewport_cache::{cached_plane_stage, ViewportPlaneWriter}; +use super::{with_runtime, with_runtime_for_session}; + +pub(crate) fn compose_rgb_viewport_from_regions( + decoder: &CpuDecoder<'_>, + pool: &mut j2k_jpeg::ScratchPool, + scale: j2k_core::Downscale, + viewport_dims: (u32, u32), + tiles: &[ViewportTile], +) -> Result { + with_runtime(|runtime| { + let mut stage = cached_plane_stage(runtime, decoder.info().color_space, viewport_dims)?; + for tile in tiles { + let dims = tile.source_roi.scaled_covering(scale); + if (dims.w, dims.h) != (tile.dest.w, tile.dest.h) { + return Err(Error::MetalKernel { + message: format!( + "viewport tile dims {:?} do not match destination rect {:?}", + (dims.w, dims.h), + tile.dest + ), + }); + } + let mut writer = ViewportPlaneWriter { + stage: &mut stage, + dest: tile.dest, + }; + decoder.decode_region_component_rows_with_scratch( + pool, + &mut writer, + j2k_jpeg::Rect { + x: tile.source_roi.x, + y: tile.source_roi.y, + w: tile.source_roi.w, + h: tile.source_roi.h, + }, + scale, + )?; + } + stage.finish_with_runtime(runtime, PixelFormat::Rgb8) + }) +} + +pub(crate) fn compose_rgb_viewport_from_regions_into_output_with_session( + decoder: &CpuDecoder<'_>, + pool: &mut j2k_jpeg::ScratchPool, + scale: j2k_core::Downscale, + viewport_dims: (u32, u32), + tiles: &[ViewportTile], + output: &crate::MetalBatchOutputBuffer, + session: &crate::MetalBackendSession, +) -> Result { + with_runtime_for_session(session, |runtime| { + let mut stage = cached_plane_stage(runtime, decoder.info().color_space, viewport_dims)?; + for tile in tiles { + let dims = tile.source_roi.scaled_covering(scale); + if (dims.w, dims.h) != (tile.dest.w, tile.dest.h) { + return Err(Error::MetalKernel { + message: format!( + "viewport tile dims {:?} do not match destination rect {:?}", + (dims.w, dims.h), + tile.dest + ), + }); + } + let mut writer = ViewportPlaneWriter { + stage: &mut stage, + dest: tile.dest, + }; + decoder.decode_region_component_rows_with_scratch( + pool, + &mut writer, + j2k_jpeg::Rect { + x: tile.source_roi.x, + y: tile.source_roi.y, + w: tile.source_roi.w, + h: tile.source_roi.h, + }, + scale, + )?; + } + stage.finish_rgb8_into_output_with_runtime(runtime, output) + }) +} + +pub(crate) fn compose_rgb_viewport_from_regions_into_textures_with_session( + decoder: &CpuDecoder<'_>, + pool: &mut j2k_jpeg::ScratchPool, + scale: j2k_core::Downscale, + viewport_dims: (u32, u32), + tiles: &[ViewportTile], + output: &crate::MetalBatchTextureOutput, + session: &crate::MetalBackendSession, +) -> Result { + with_runtime_for_session(session, |runtime| { + let mut stage = cached_plane_stage(runtime, decoder.info().color_space, viewport_dims)?; + for tile in tiles { + let dims = tile.source_roi.scaled_covering(scale); + if (dims.w, dims.h) != (tile.dest.w, tile.dest.h) { + return Err(Error::MetalKernel { + message: format!( + "viewport tile dims {:?} do not match destination rect {:?}", + (dims.w, dims.h), + tile.dest + ), + }); + } + let mut writer = ViewportPlaneWriter { + stage: &mut stage, + dest: tile.dest, + }; + decoder.decode_region_component_rows_with_scratch( + pool, + &mut writer, + j2k_jpeg::Rect { + x: tile.source_roi.x, + y: tile.source_roi.y, + w: tile.source_roi.w, + h: tile.source_roi.h, + }, + scale, + )?; + } + stage.finish_rgba8_into_texture_output_with_runtime(runtime, output) + }) +} diff --git a/crates/signinum-jpeg-metal/src/encode.rs b/crates/j2k-jpeg-metal/src/encode.rs similarity index 91% rename from crates/signinum-jpeg-metal/src/encode.rs rename to crates/j2k-jpeg-metal/src/encode.rs index e21f95ae..c31ab80c 100644 --- a/crates/signinum-jpeg-metal/src/encode.rs +++ b/crates/j2k-jpeg-metal/src/encode.rs @@ -3,41 +3,52 @@ #![allow(clippy::similar_names)] #[cfg(target_os = "macos")] -use metal::Buffer; -#[cfg(target_os = "macos")] -use signinum_core::PixelFormat; +use j2k_core::PixelFormat; #[cfg(target_os = "macos")] -use signinum_jpeg::adapter::{ +use j2k_jpeg::adapter::{ assemble_jpeg_baseline_frame, baseline_encode_tables, jpeg_baseline_entropy_capacity_bytes, validate_jpeg_baseline_dimensions, JpegBaselineHuffmanTable, JpegBaselineSampling, }; -use signinum_jpeg::{EncodedJpeg, JpegEncodeOptions}; +use j2k_jpeg::{EncodedJpeg, JpegEncodeOptions}; #[cfg(target_os = "macos")] -use signinum_jpeg::{JpegBackend, JpegEncodeError, JpegSubsampling}; +use j2k_jpeg::{JpegBackend, JpegEncodeError, JpegSubsampling}; +#[cfg(target_os = "macos")] +use metal::Buffer; #[cfg(target_os = "macos")] use crate::compute; #[cfg(target_os = "macos")] #[derive(Debug, Clone, Copy)] +/// Metal buffer and layout metadata for one baseline JPEG encode tile. pub struct JpegBaselineMetalEncodeTile<'a> { + /// Source Metal buffer containing RGB8 or Gray8 pixels. pub buffer: &'a Buffer, + /// Byte offset of the first source pixel in `buffer`. pub byte_offset: usize, + /// Width of the valid input region in pixels. pub width: u32, + /// Height of the valid input region in pixels. pub height: u32, + /// Number of bytes between consecutive input rows. pub pitch_bytes: usize, + /// Encoded frame width in pixels. pub output_width: u32, + /// Encoded frame height in pixels. pub output_height: u32, + /// Pixel format of the source buffer. pub format: PixelFormat, } #[cfg(not(target_os = "macos"))] #[derive(Debug, Clone, Copy)] +/// Placeholder encode tile type for non-macOS builds. pub struct JpegBaselineMetalEncodeTile<'a> { _private: core::marker::PhantomData<&'a ()>, } #[cfg(target_os = "macos")] +/// Encode one Metal-resident tile as a baseline JPEG frame. pub fn encode_jpeg_baseline_from_metal_buffer( tile: JpegBaselineMetalEncodeTile<'_>, options: JpegEncodeOptions, @@ -80,6 +91,11 @@ pub fn encode_jpeg_baseline_from_metal_buffer( } #[cfg(target_os = "macos")] +/// Encode multiple Metal-resident tiles as baseline JPEG frames. +/// +/// Consecutive tiles that share a source Metal buffer are submitted through a +/// single entropy-kernel batch where possible. The returned frames preserve the +/// input order. pub fn encode_jpeg_baseline_batch_from_metal_buffers( tiles: &[JpegBaselineMetalEncodeTile<'_>], options: JpegEncodeOptions, @@ -172,6 +188,7 @@ pub fn encode_jpeg_baseline_batch_from_metal_buffers( } #[cfg(not(target_os = "macos"))] +/// Return `Error::MetalUnavailable` for batch Metal encode requests on non-macOS hosts. pub fn encode_jpeg_baseline_batch_from_metal_buffers( tiles: &[JpegBaselineMetalEncodeTile<'_>], options: JpegEncodeOptions, @@ -182,6 +199,7 @@ pub fn encode_jpeg_baseline_batch_from_metal_buffers( } #[cfg(not(target_os = "macos"))] +/// Return `Error::MetalUnavailable` for Metal encode requests on non-macOS hosts. pub fn encode_jpeg_baseline_from_metal_buffer( tile: JpegBaselineMetalEncodeTile<'_>, options: JpegEncodeOptions, diff --git a/crates/j2k-jpeg-metal/src/lib.rs b/crates/j2k-jpeg-metal/src/lib.rs new file mode 100644 index 00000000..afa30a2d --- /dev/null +++ b/crates/j2k-jpeg-metal/src/lib.rs @@ -0,0 +1,7376 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Metal-backed JPEG decode and encode adapters. +//! +//! The crate exposes the same CPU-visible JPEG decode surface as +//! `j2k-jpeg`, with optional Metal-resident surfaces and batch submission +//! helpers on macOS. Non-macOS builds keep the public API available but return +//! `Error::MetalUnavailable` for explicit Metal-only work. + +#![deny(unsafe_op_in_unsafe_fn)] +#![warn(unreachable_pub)] + +#[cfg(target_os = "macos")] +mod abi; +mod batch; +#[cfg(target_os = "macos")] +mod buffers; +#[cfg(target_os = "macos")] +mod compute; +mod encode; +mod routing; +mod session; +/// Viewport planning and composition helpers for JPEG decode surfaces. +pub mod viewport; + +use std::sync::Arc; +#[cfg(target_os = "macos")] +use std::sync::Mutex; +#[cfg(target_os = "macos")] +use std::sync::OnceLock; + +use j2k_core::{ + copy_tight_pixels_to_strided_output, BackendKind, BackendRequest, BufferError, CodecError, + DecodeOutcome, DeviceMemoryRange, DeviceSubmission, DeviceSurface, Downscale, ImageCodec, + ImageDecode, ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, Rect, TileBatchDecodeDevice, + TileBatchDecodeManyDevice, TileBatchDecodeSubmit, +}; +use j2k_jpeg::{ + adapter::{ + build_fast420_packet, build_fast420_packet_for_decoder, build_fast422_packet, + build_fast422_packet_for_decoder, build_fast444_packet, build_fast444_packet_for_decoder, + decoder_bytes, JpegFast420PacketV1, JpegFast422PacketV1, JpegFast444PacketV1, + }, + Decoder as CpuDecoder, DecoderContext as CpuDecoderContext, JpegError, JpegView, + ScratchPool as CpuScratchPool, Warning as CpuWarning, +}; +use j2k_metal_support::{ + cpu_host_route, metal_kernel_route, metal_unavailable_route, reject_explicit_metal_route, + reject_unsupported_backend_route, MetalRouteProfileLabels, +}; +#[cfg(target_os = "macos")] +use j2k_metal_support::{system_default_device, MetalSupportError}; + +pub use encode::{ + encode_jpeg_baseline_batch_from_metal_buffers, encode_jpeg_baseline_from_metal_buffer, + JpegBaselineMetalEncodeTile, +}; + +#[cfg(target_os = "macos")] +use metal::foreign_types::ForeignType; +#[cfg(target_os = "macos")] +use metal::{ + Buffer, BufferRef, CommandBuffer, Device, MTLPixelFormat, MTLResourceOptions, MTLStorageMode, + MTLTextureType, MTLTextureUsage, Texture, TextureDescriptor, TextureRef, +}; + +#[derive(Debug, thiserror::Error)] +/// Errors returned by the Metal JPEG backend. +pub enum Error { + /// Error returned by the CPU JPEG parser or fallback decoder. + #[error(transparent)] + Decode(#[from] JpegError), + /// Error returned while assembling a baseline JPEG encode result. + #[error(transparent)] + Encode(#[from] j2k_jpeg::JpegEncodeError), + /// Output buffer validation failed. + #[error(transparent)] + Buffer(#[from] BufferError), + /// The requested backend is not supported by this crate. + #[error("backend request {request:?} is not supported by j2k-jpeg-metal")] + UnsupportedBackend { + /// Backend requested by the caller. + request: BackendRequest, + }, + /// A Metal-specific request is structurally unsupported. + #[error("unsupported JPEG Metal request: {reason}")] + UnsupportedMetalRequest { + /// Static reason describing the rejected request. + reason: &'static str, + }, + /// Metal is not available on the current host. + #[error("Metal is unavailable on this host")] + MetalUnavailable, + /// Metal runtime creation or device setup failed. + #[error("Metal runtime error: {message}")] + MetalRuntime { + /// Runtime error message. + message: String, + }, + /// Metal kernel launch, validation, or completion failed. + #[error("Metal kernel error: {message}")] + MetalKernel { + /// Kernel error message. + message: String, + }, + /// Shared Metal backend state was poisoned by a prior panic. + #[error("Metal state `{state}` is poisoned")] + MetalStatePoisoned { + /// Name of the poisoned state. + state: &'static str, + }, +} + +impl CodecError for Error { + fn is_truncated(&self) -> bool { + matches!(self, Self::Decode(inner) if inner.is_truncated()) + } + + fn is_not_implemented(&self) -> bool { + matches!(self, Self::Decode(inner) if inner.is_not_implemented()) + } + + fn is_unsupported(&self) -> bool { + matches!( + self, + Self::UnsupportedBackend { .. } + | Self::MetalUnavailable + | Self::UnsupportedMetalRequest { .. } + ) || matches!(self, Self::Decode(inner) if inner.is_unsupported()) + } + + fn is_buffer_error(&self) -> bool { + matches!(self, Self::Buffer(_)) + || matches!(self, Self::Decode(inner) if inner.is_buffer_error()) + } +} + +#[derive(Clone)] +pub(crate) enum Storage { + Host(Vec), + #[cfg(target_os = "macos")] + Metal { + buffer: Buffer, + offset: usize, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Where a decoded surface is currently resident. +pub enum SurfaceResidency { + /// Pixel bytes are resident in host memory. + Host, + /// Pixel bytes were produced directly by a Metal decode kernel. + MetalResidentDecode, + /// Pixel bytes were decoded on CPU and uploaded into a Metal buffer. + CpuStagedMetalUpload, +} + +#[derive(Clone)] +/// Decoded image surface returned by the JPEG Metal backend. +pub struct Surface { + backend: BackendKind, + residency: SurfaceResidency, + dimensions: (u32, u32), + fmt: PixelFormat, + pitch_bytes: usize, + storage: Storage, +} + +impl Surface { + /// Number of bytes between consecutive rows. + pub fn pitch_bytes(&self) -> usize { + self.pitch_bytes + } + + /// Current residency for the surface bytes. + pub fn residency(&self) -> SurfaceResidency { + self.residency + } + + /// Return the tightly packed surface bytes. + pub fn as_bytes(&self) -> &[u8] { + match &self.storage { + Storage::Host(bytes) => bytes, + #[cfg(target_os = "macos")] + Storage::Metal { buffer, offset } => { + let len = self.byte_len(); + // SAFETY: Metal surface byte views are bounded by validated dimensions and formats. + unsafe { + core::slice::from_raw_parts(buffer.contents().cast::().add(*offset), len) + } + } + } + } + + /// Copy the tightly packed surface into a caller-provided strided buffer. + pub fn download_into(&self, out: &mut [u8], stride: usize) -> Result<(), Error> { + copy_tight_pixels_to_strided_output(self.as_bytes(), self.dimensions, self.fmt, out, stride) + .map_err(Error::from) + } + + #[cfg(target_os = "macos")] + /// Return the Metal buffer and byte offset when the surface is Metal-backed. + pub fn metal_buffer(&self) -> Option<(&Buffer, usize)> { + match &self.storage { + Storage::Metal { buffer, offset } => Some((buffer, *offset)), + Storage::Host(_) => None, + } + } + + #[cfg(target_os = "macos")] + pub(crate) fn from_metal_buffer( + buffer: Buffer, + dimensions: (u32, u32), + fmt: PixelFormat, + ) -> Self { + Self::from_metal_buffer_offset(buffer, dimensions, fmt, 0) + } + + #[cfg(target_os = "macos")] + pub(crate) fn from_metal_buffer_offset( + buffer: Buffer, + dimensions: (u32, u32), + fmt: PixelFormat, + offset: usize, + ) -> Self { + Self { + backend: BackendKind::Metal, + residency: SurfaceResidency::MetalResidentDecode, + dimensions, + fmt, + pitch_bytes: dimensions.0 as usize * fmt.bytes_per_pixel(), + storage: Storage::Metal { buffer, offset }, + } + } + + #[cfg(target_os = "macos")] + pub(crate) fn from_cpu_staged_metal_buffer( + buffer: Buffer, + dimensions: (u32, u32), + fmt: PixelFormat, + ) -> Self { + Self::from_cpu_staged_metal_buffer_offset(buffer, dimensions, fmt, 0) + } + + #[cfg(target_os = "macos")] + pub(crate) fn from_cpu_staged_metal_buffer_offset( + buffer: Buffer, + dimensions: (u32, u32), + fmt: PixelFormat, + offset: usize, + ) -> Self { + Self { + backend: BackendKind::Metal, + residency: SurfaceResidency::CpuStagedMetalUpload, + dimensions, + fmt, + pitch_bytes: dimensions.0 as usize * fmt.bytes_per_pixel(), + storage: Storage::Metal { buffer, offset }, + } + } +} + +impl DeviceSurface for Surface { + fn backend_kind(&self) -> BackendKind { + self.backend + } + + fn residency(&self) -> j2k_core::SurfaceResidency { + match self.residency { + SurfaceResidency::Host => j2k_core::SurfaceResidency::Host, + SurfaceResidency::MetalResidentDecode => { + j2k_core::SurfaceResidency::MetalResidentDecode + } + SurfaceResidency::CpuStagedMetalUpload => { + j2k_core::SurfaceResidency::CpuStagedMetalUpload + } + } + } + + fn dimensions(&self) -> (u32, u32) { + self.dimensions + } + + fn pixel_format(&self) -> PixelFormat { + self.fmt + } + + fn byte_len(&self) -> usize { + self.pitch_bytes * self.dimensions.1 as usize + } + + fn memory_range(&self) -> Option { + match &self.storage { + Storage::Host(_) => None, + #[cfg(target_os = "macos")] + Storage::Metal { buffer, offset } => Some(DeviceMemoryRange::new( + BackendKind::Metal, + u64::try_from(buffer.as_ptr() as usize).ok()?, + *offset, + self.byte_len(), + )), + } + } +} + +#[cfg(target_os = "macos")] +#[doc(hidden)] +#[derive(Clone)] +pub struct ResidentPrivateJpegTile { + pub buffer: Buffer, + pub byte_offset: usize, + pub dimensions: (u32, u32), + pub pixel_format: PixelFormat, + pub pitch_bytes: usize, + pub status_buffer: Buffer, + pub command_buffer: CommandBuffer, +} + +#[cfg(target_os = "macos")] +#[derive(Clone)] +/// Reusable caller-owned Metal buffer for full-tile JPEG batch output. +pub struct MetalBatchOutputBuffer { + buffer: Buffer, + dimensions: (u32, u32), + fmt: PixelFormat, + pitch_bytes: usize, + tile_stride_bytes: usize, + tile_capacity: usize, +} + +#[cfg(target_os = "macos")] +impl MetalBatchOutputBuffer { + /// Allocate a reusable RGB8 output buffer for `tile_capacity` full-size tiles. + pub fn new_rgb8_tiles( + session: &MetalBackendSession, + dimensions: (u32, u32), + tile_capacity: usize, + ) -> Result { + Self::new_tiles(session, dimensions, PixelFormat::Rgb8, tile_capacity) + } + + /// Ensure this output buffer can hold `tile_capacity` RGB8 tiles with `dimensions`. + /// + /// The existing allocation is retained when it already has the requested + /// layout and at least the requested capacity. Otherwise the buffer is + /// replaced with a new allocation. + pub fn ensure_rgb8_tiles( + &mut self, + session: &MetalBackendSession, + dimensions: (u32, u32), + tile_capacity: usize, + ) -> Result<(), Error> { + if self.dimensions == dimensions + && self.fmt == PixelFormat::Rgb8 + && self.tile_capacity >= tile_capacity + { + return Ok(()); + } + + *self = Self::new_rgb8_tiles(session, dimensions, tile_capacity)?; + Ok(()) + } + + /// Ensure this output buffer fits a full-image scaled RGB8 batch. + pub fn ensure_rgb8_scaled_tiles( + &mut self, + session: &MetalBackendSession, + full_dimensions: (u32, u32), + scale: Downscale, + tile_capacity: usize, + ) -> Result<(), Error> { + self.ensure_rgb8_tiles(session, scaled_dims(full_dimensions, scale), tile_capacity) + } + + /// Ensure this output buffer fits a region-scaled RGB8 batch. + pub fn ensure_rgb8_region_scaled_tiles( + &mut self, + session: &MetalBackendSession, + roi: Rect, + scale: Downscale, + tile_capacity: usize, + ) -> Result<(), Error> { + let scaled = roi.scaled_covering(scale); + self.ensure_rgb8_tiles(session, (scaled.w, scaled.h), tile_capacity) + } + + /// Ensure this output buffer fits a preflighted RGB8 Metal resident batch. + /// + /// Ineligible reports return an error without replacing the existing + /// allocation. Eligible empty reports are a no-op. + pub fn ensure_rgb8_batch_report( + &mut self, + session: &MetalBackendSession, + report: &JpegMetalResidentBatchReport, + ) -> Result<(), Error> { + let Some(dimensions) = report_required_output_dimensions(report)? else { + return Ok(()); + }; + self.ensure_rgb8_tiles(session, dimensions, report.required_tile_capacity()) + } + + fn new_tiles( + session: &MetalBackendSession, + dimensions: (u32, u32), + fmt: PixelFormat, + tile_capacity: usize, + ) -> Result { + if dimensions.0 == 0 || dimensions.1 == 0 || tile_capacity == 0 { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal batch output requires nonzero dimensions and tile capacity", + }); + } + let row_bytes = dimensions + .0 + .checked_mul(u32::try_from(fmt.bytes_per_pixel()).map_err(|_| { + BufferError::SizeOverflow { + what: "JPEG Metal output row bytes", + } + })?) + .ok_or(BufferError::SizeOverflow { + what: "JPEG Metal output row bytes", + })? as usize; + let tile_stride_bytes = + row_bytes + .checked_mul(dimensions.1 as usize) + .ok_or(BufferError::SizeOverflow { + what: "JPEG Metal output tile bytes", + })?; + let byte_len = + tile_stride_bytes + .checked_mul(tile_capacity) + .ok_or(BufferError::SizeOverflow { + what: "JPEG Metal batch output bytes", + })?; + let byte_len_u64 = u64::try_from(byte_len).map_err(|_| BufferError::SizeOverflow { + what: "JPEG Metal batch output bytes", + })?; + let buffer = session + .device() + .new_buffer(byte_len_u64, MTLResourceOptions::StorageModeShared); + Ok(Self { + buffer, + dimensions, + fmt, + pitch_bytes: row_bytes, + tile_stride_bytes, + tile_capacity, + }) + } + + /// Backing Metal buffer. + pub fn buffer(&self) -> &BufferRef { + self.buffer.as_ref() + } + + /// Tile dimensions for this output allocation. + pub fn dimensions(&self) -> (u32, u32) { + self.dimensions + } + + /// Pixel format for this output allocation. + pub fn pixel_format(&self) -> PixelFormat { + self.fmt + } + + /// Number of reusable tile slots in the buffer. + pub fn tile_capacity(&self) -> usize { + self.tile_capacity + } + + /// Number of bytes between rows in one tile. + pub fn pitch_bytes(&self) -> usize { + self.pitch_bytes + } + + /// Number of bytes reserved for each tile slot. + pub fn tile_stride_bytes(&self) -> usize { + self.tile_stride_bytes + } + + /// Total byte length of the backing allocation. + pub fn byte_len(&self) -> usize { + self.tile_stride_bytes * self.tile_capacity + } + + pub(crate) fn clone_buffer(&self) -> Buffer { + self.buffer.clone() + } +} + +#[cfg(target_os = "macos")] +#[derive(Clone)] +/// Reusable caller-owned Metal textures for full-tile JPEG batch output. +pub struct MetalBatchTextureOutput { + textures: Vec, + dimensions: (u32, u32), + fmt: PixelFormat, + metal_fmt: MTLPixelFormat, +} + +#[cfg(target_os = "macos")] +impl MetalBatchTextureOutput { + /// Allocate reusable private RGBA8 textures for `tile_capacity` full-size tiles. + pub fn new_rgba8_tiles( + session: &MetalBackendSession, + dimensions: (u32, u32), + tile_capacity: usize, + ) -> Result { + if dimensions.0 == 0 || dimensions.1 == 0 || tile_capacity == 0 { + return Err(Error::UnsupportedMetalRequest { + reason: + "JPEG Metal batch texture output requires nonzero dimensions and tile capacity", + }); + } + + let descriptor = TextureDescriptor::new(); + descriptor.set_texture_type(MTLTextureType::D2); + descriptor.set_pixel_format(MTLPixelFormat::RGBA8Unorm); + descriptor.set_width(u64::from(dimensions.0)); + descriptor.set_height(u64::from(dimensions.1)); + descriptor.set_depth(1); + descriptor.set_mipmap_level_count(1); + descriptor.set_sample_count(1); + descriptor.set_storage_mode(MTLStorageMode::Private); + descriptor.set_usage(MTLTextureUsage::ShaderRead | MTLTextureUsage::ShaderWrite); + + let mut textures = Vec::with_capacity(tile_capacity); + for _ in 0..tile_capacity { + textures.push(session.device().new_texture(&descriptor)); + } + + Ok(Self { + textures, + dimensions, + fmt: PixelFormat::Rgba8, + metal_fmt: MTLPixelFormat::RGBA8Unorm, + }) + } + + /// Ensure this output set can hold `tile_capacity` RGBA8 textures with `dimensions`. + /// + /// Existing textures are retained when they already have the requested + /// layout and at least the requested capacity. Otherwise the texture set is + /// replaced with new private RGBA8 textures. + pub fn ensure_rgba8_tiles( + &mut self, + session: &MetalBackendSession, + dimensions: (u32, u32), + tile_capacity: usize, + ) -> Result<(), Error> { + if self.dimensions == dimensions + && self.fmt == PixelFormat::Rgba8 + && self.metal_fmt == MTLPixelFormat::RGBA8Unorm + && self.tile_capacity() >= tile_capacity + { + return Ok(()); + } + + *self = Self::new_rgba8_tiles(session, dimensions, tile_capacity)?; + Ok(()) + } + + /// Ensure this output set fits a full-image scaled RGBA8 texture batch. + pub fn ensure_rgba8_scaled_tiles( + &mut self, + session: &MetalBackendSession, + full_dimensions: (u32, u32), + scale: Downscale, + tile_capacity: usize, + ) -> Result<(), Error> { + self.ensure_rgba8_tiles(session, scaled_dims(full_dimensions, scale), tile_capacity) + } + + /// Ensure this output set fits a region-scaled RGBA8 texture batch. + pub fn ensure_rgba8_region_scaled_tiles( + &mut self, + session: &MetalBackendSession, + roi: Rect, + scale: Downscale, + tile_capacity: usize, + ) -> Result<(), Error> { + let scaled = roi.scaled_covering(scale); + self.ensure_rgba8_tiles(session, (scaled.w, scaled.h), tile_capacity) + } + + /// Ensure this texture set fits a preflighted RGB8 Metal resident batch. + /// + /// Ineligible reports return an error without replacing the existing + /// textures. Eligible empty reports are a no-op. + pub fn ensure_rgba8_batch_report( + &mut self, + session: &MetalBackendSession, + report: &JpegMetalResidentBatchReport, + ) -> Result<(), Error> { + let Some(dimensions) = report_required_output_dimensions(report)? else { + return Ok(()); + }; + self.ensure_rgba8_tiles(session, dimensions, report.required_tile_capacity()) + } + + /// Tile dimensions for this output allocation. + pub fn dimensions(&self) -> (u32, u32) { + self.dimensions + } + + /// Pixel format for this output allocation. + pub fn pixel_format(&self) -> PixelFormat { + self.fmt + } + + /// Metal pixel format for each backing texture. + pub fn metal_pixel_format(&self) -> MTLPixelFormat { + self.metal_fmt + } + + /// Number of reusable tile texture slots. + pub fn tile_capacity(&self) -> usize { + self.textures.len() + } + + /// Return a reusable output texture by tile slot. + pub fn texture(&self, index: usize) -> Option<&TextureRef> { + self.textures.get(index).map(std::convert::AsRef::as_ref) + } + + pub(crate) fn clone_texture(&self, index: usize) -> Option { + self.textures.get(index).cloned() + } + + pub(crate) fn clone_slots(&self, indices: &[usize]) -> Result { + let mut textures = Vec::with_capacity(indices.len()); + for &index in indices { + textures.push( + self.clone_texture(index) + .ok_or_else(|| Error::MetalKernel { + message: "JPEG Metal batch texture output slot was missing".to_string(), + })?, + ); + } + Ok(Self { + textures, + dimensions: self.dimensions, + fmt: self.fmt, + metal_fmt: self.metal_fmt, + }) + } +} + +#[cfg(target_os = "macos")] +#[derive(Clone)] +/// One decoded JPEG tile resident in a caller-owned Metal texture. +pub struct MetalTextureTile { + texture: Texture, + dimensions: (u32, u32), + fmt: PixelFormat, +} + +#[cfg(target_os = "macos")] +impl MetalTextureTile { + pub(crate) fn new(texture: Texture, dimensions: (u32, u32), fmt: PixelFormat) -> Self { + Self { + texture, + dimensions, + fmt, + } + } + + /// Backing Metal texture containing the decoded tile. + pub fn texture(&self) -> &TextureRef { + self.texture.as_ref() + } + + /// Decoded tile dimensions. + pub fn dimensions(&self) -> (u32, u32) { + self.dimensions + } + + /// Decoded tile pixel format. + pub fn pixel_format(&self) -> PixelFormat { + self.fmt + } +} + +#[cfg(target_os = "macos")] +#[derive(Clone)] +/// Reusable Metal device session for decode and encode submissions. +pub struct MetalBackendSession { + device: Device, + runtime: Arc>>, +} + +#[cfg(target_os = "macos")] +impl MetalBackendSession { + /// Create a session bound to an existing Metal device. + pub fn new(device: Device) -> Self { + Self { + device, + runtime: Arc::new(OnceLock::new()), + } + } + + /// Create a session from the system default Metal device. + pub fn system_default() -> Result { + system_default_device() + .map(Self::new) + .map_err(|error| compute::runtime_initialization_error(&error)) + } + + /// Metal device used by this session. + pub fn device(&self) -> &metal::DeviceRef { + self.device.as_ref() + } +} + +#[cfg(target_os = "macos")] +impl j2k_core::AcceleratorSession for MetalBackendSession { + fn backend_kind(&self) -> BackendKind { + BackendKind::Metal + } +} + +#[cfg(target_os = "macos")] +impl core::fmt::Debug for MetalBackendSession { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("MetalBackendSession") + .field("device", &self.device.name()) + .field("runtime_initialized", &self.runtime.get().is_some()) + .finish() + } +} + +#[cfg(not(target_os = "macos"))] +#[derive(Clone, Copy, Debug, Default)] +/// Placeholder Metal session for non-macOS builds. +pub struct MetalBackendSession { + _private: (), +} + +#[cfg(not(target_os = "macos"))] +impl MetalBackendSession { + /// Return `Error::MetalUnavailable` on hosts without Metal support. + pub fn system_default() -> Result { + Err(Error::MetalUnavailable) + } +} + +#[derive(Default)] +/// Shared batching session used by `JpegTileBatch` and submit APIs. +pub struct MetalSession { + shared: session::SharedSession, +} + +impl MetalSession { + /// Create a tile batching session that reuses an existing Metal backend session. + #[cfg(target_os = "macos")] + pub fn with_backend_session(backend_session: MetalBackendSession) -> Self { + Self { + shared: session::SharedSession(Arc::new(Mutex::new( + session::SessionState::with_backend_session(backend_session), + ))), + } + } + + /// Number of Metal or emulated submissions flushed through this session. + pub fn submissions(&self) -> Result { + Ok(self.shared.lock()?.submissions) + } +} + +impl core::fmt::Debug for MetalSession { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("MetalSession") + .field("submissions", &self.submissions()) + .finish() + } +} + +/// Convenience wrapper for submitting a group of JPEG tiles to one decoder +/// session. +/// +/// The batch preserves submission order and lets compatible requests share a +/// Metal submission. Callers still own slide metadata, level selection, cache +/// policy, and viewport planning. +#[derive(Default)] +pub struct JpegTileBatch { + session: MetalSession, + submissions: Vec, +} + +impl JpegTileBatch { + /// Create an empty tile batch. + pub fn new() -> Self { + Self::default() + } + + /// Create an empty tile batch with capacity for `capacity` submissions. + pub fn with_capacity(capacity: usize) -> Self { + Self { + submissions: Vec::with_capacity(capacity), + ..Self::default() + } + } + + /// Number of queued tile requests. + pub fn len(&self) -> usize { + self.submissions.len() + } + + /// Whether the batch has no queued tile requests. + pub fn is_empty(&self) -> bool { + self.submissions.is_empty() + } + + /// Number of Metal session submissions already flushed. + /// + /// Queued requests normally do not increment this until `decode_all` waits + /// on the first result. + pub fn submissions(&self) -> Result { + self.session.submissions() + } + + /// Queue a full-tile decode request, copying the compressed tile bytes into + /// the batch. + pub fn push_tile( + &mut self, + input: &[u8], + fmt: PixelFormat, + backend: BackendRequest, + ) -> Result { + self.push_shared_tile(Arc::<[u8]>::from(input), fmt, backend) + } + + /// Queue a full-tile decode request backed by shared compressed tile bytes. + pub fn push_shared_tile( + &mut self, + input: Arc<[u8]>, + fmt: PixelFormat, + backend: BackendRequest, + ) -> Result { + self.push_shared_request(input, fmt, backend, batch::BatchOp::Full) + } + + /// Queue a region decode request, copying the compressed tile bytes into + /// the batch. + pub fn push_tile_region( + &mut self, + input: &[u8], + fmt: PixelFormat, + roi: Rect, + backend: BackendRequest, + ) -> Result { + self.push_shared_tile_region(Arc::<[u8]>::from(input), fmt, roi, backend) + } + + /// Queue a region decode request backed by shared compressed tile bytes. + pub fn push_shared_tile_region( + &mut self, + input: Arc<[u8]>, + fmt: PixelFormat, + roi: Rect, + backend: BackendRequest, + ) -> Result { + self.push_shared_request(input, fmt, backend, batch::BatchOp::Region(roi)) + } + + /// Queue a scaled decode request, copying the compressed tile bytes into + /// the batch. + pub fn push_tile_scaled( + &mut self, + input: &[u8], + fmt: PixelFormat, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + self.push_shared_tile_scaled(Arc::<[u8]>::from(input), fmt, scale, backend) + } + + /// Queue a scaled decode request backed by shared compressed tile bytes. + pub fn push_shared_tile_scaled( + &mut self, + input: Arc<[u8]>, + fmt: PixelFormat, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + self.push_shared_request(input, fmt, backend, batch::BatchOp::Scaled(scale)) + } + + /// Queue a region decode at reduced resolution, copying the compressed tile + /// bytes into the batch. + pub fn push_tile_region_scaled( + &mut self, + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + self.push_shared_tile_region_scaled(Arc::<[u8]>::from(input), fmt, roi, scale, backend) + } + + /// Queue a region decode at reduced resolution backed by shared compressed + /// tile bytes. + pub fn push_shared_tile_region_scaled( + &mut self, + input: Arc<[u8]>, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + self.push_shared_request( + input, + fmt, + backend, + batch::BatchOp::RegionScaled { roi, scale }, + ) + } + + /// Decode all queued tile requests and return surfaces in submission order. + pub fn decode_all(self) -> Result, Error> { + let mut surfaces = Vec::with_capacity(self.submissions.len()); + for submission in self.submissions { + surfaces.push(submission.wait()?); + } + Ok(surfaces) + } + + fn push_shared_request( + &mut self, + input: Arc<[u8]>, + fmt: PixelFormat, + backend: BackendRequest, + op: batch::BatchOp, + ) -> Result { + let slot = self.submissions.len(); + let submission = { + let mut state = self.session.shared.lock()?; + let (fast444_packet, fast422_packet, fast420_packet) = + state.resolve_fast_packets(&input, backend); + let slot = state.queue_request(batch::QueuedRequest::new_shared( + input, + fmt, + backend, + op, + fast444_packet, + fast422_packet, + fast420_packet, + )); + batch::MetalSubmission { + session: self.session.shared.clone(), + slot, + } + }; + self.submissions.push(submission); + Ok(slot) + } +} + +/// JPEG decoder that can return host or Metal-resident surfaces. +pub struct Decoder<'a> { + inner: CpuDecoder<'a>, + source: Arc<[u8]>, + fast444_packet: Option>, + fast422_packet: Option>, + fast420_packet: Option>, +} + +impl<'a> Decoder<'a> { + /// Parse a JPEG byte slice into a decoder with any available Metal packets. + pub fn new(input: &'a [u8]) -> Result { + let inner = CpuDecoder::new(input)?; + Ok(Self { + fast444_packet: build_fast444_packet(input).ok().map(Arc::new), + fast422_packet: build_fast422_packet(input).ok().map(Arc::new), + fast420_packet: build_fast420_packet(input).ok().map(Arc::new), + inner, + source: Arc::<[u8]>::from(input), + }) + } + + /// Create a decoder from an already parsed JPEG view. + pub fn from_view(view: JpegView<'a>) -> Result { + let inner = CpuDecoder::from_view(view)?; + let source = Arc::<[u8]>::from(decoder_bytes(&inner)); + let fast444_packet = build_fast444_packet_for_decoder(&inner).ok().map(Arc::new); + let fast422_packet = build_fast422_packet_for_decoder(&inner).ok().map(Arc::new); + let fast420_packet = build_fast420_packet_for_decoder(&inner).ok().map(Arc::new); + Ok(Self { + inner, + source, + fast444_packet, + fast422_packet, + fast420_packet, + }) + } + + /// Borrow the underlying CPU JPEG decoder. + pub fn inner(&self) -> &CpuDecoder<'a> { + &self.inner + } + + #[cfg(target_os = "macos")] + pub(crate) fn fast444_packet(&self) -> Option<&JpegFast444PacketV1> { + self.fast444_packet.as_deref() + } + + #[cfg(target_os = "macos")] + pub(crate) fn fast422_packet(&self) -> Option<&JpegFast422PacketV1> { + self.fast422_packet.as_deref() + } + + #[cfg(target_os = "macos")] + pub(crate) fn fast420_packet(&self) -> Option<&JpegFast420PacketV1> { + self.fast420_packet.as_deref() + } + + #[cfg(target_os = "macos")] + pub(crate) fn rgb8_region_scaled_metal_request( + &self, + roi: Rect, + scale: Downscale, + ) -> batch::QueuedRequest { + self.rgb8_metal_request(batch::BatchOp::RegionScaled { roi, scale }) + } + + #[cfg(target_os = "macos")] + pub(crate) fn rgb8_metal_request(&self, op: batch::BatchOp) -> batch::QueuedRequest { + batch::QueuedRequest::new_shared( + Arc::clone(&self.source), + PixelFormat::Rgb8, + BackendRequest::Metal, + op, + self.fast444_packet.clone(), + self.fast422_packet.clone(), + self.fast420_packet.clone(), + ) + } + + /// Consume this wrapper and return the underlying CPU JPEG decoder. + pub fn into_inner(self) -> CpuDecoder<'a> { + self.inner + } + + /// Decode a region at the requested scale into a device surface when possible. + pub fn decode_region_scaled_to_device( + &mut self, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + let mut pool = CpuScratchPool::new(); + decode_region_scaled_surface_from_decoder( + &self.inner, + &mut pool, + fmt, + roi, + scale, + backend, + self.fast444_packet.as_deref(), + self.fast422_packet.as_deref(), + self.fast420_packet.as_deref(), + ) + } + + /// Decode a full image into a device surface using a reusable Metal session. + pub fn decode_to_device_with_session( + &mut self, + fmt: PixelFormat, + session: &MetalBackendSession, + ) -> Result { + #[cfg(target_os = "macos")] + { + let mut pool = CpuScratchPool::new(); + let decision = choose_route( + &self.inner, + BackendRequest::Metal, + fmt, + batch::BatchOp::Full, + self.fast444_packet.as_deref(), + self.fast422_packet.as_deref(), + self.fast420_packet.as_deref(), + ); + if let Some(err) = routing::decision_error(decision) { + return Err(err); + } + match decision { + routing::RouteDecision::MetalKernel => { + reject_cpu_staged_metal_upload(compute::decode_to_surface_with_session( + &self.inner, + &mut pool, + fmt, + self.fast444_packet.as_deref(), + self.fast422_packet.as_deref(), + self.fast420_packet.as_deref(), + session, + )?) + } + routing::RouteDecision::CpuHost + | routing::RouteDecision::RejectExplicitMetal { .. } + | routing::RouteDecision::RejectUnsupportedBackend { .. } + | routing::RouteDecision::MetalUnavailable => unreachable!("handled above"), + } + } + #[cfg(not(target_os = "macos"))] + { + let _ = session; + let decision = choose_route( + &self.inner, + BackendRequest::Metal, + fmt, + batch::BatchOp::Full, + self.fast444_packet.as_deref(), + self.fast422_packet.as_deref(), + self.fast420_packet.as_deref(), + ); + if let Some(err) = routing::decision_error(decision) { + return Err(err); + } + Err(Error::MetalUnavailable) + } + } + + #[cfg(target_os = "macos")] + #[doc(hidden)] + pub fn decode_private_rgb8_tile_with_session( + &mut self, + session: &MetalBackendSession, + ) -> Result { + let decision = choose_route( + &self.inner, + BackendRequest::Metal, + PixelFormat::Rgb8, + batch::BatchOp::Full, + self.fast444_packet.as_deref(), + self.fast422_packet.as_deref(), + self.fast420_packet.as_deref(), + ); + if let Some(err) = routing::decision_error(decision) { + return Err(err); + } + match decision { + routing::RouteDecision::MetalKernel => compute::decode_private_rgb8_tile_with_session( + &self.inner, + self.fast444_packet.as_deref(), + self.fast422_packet.as_deref(), + self.fast420_packet.as_deref(), + session, + ), + routing::RouteDecision::CpuHost + | routing::RouteDecision::RejectExplicitMetal { .. } + | routing::RouteDecision::RejectUnsupportedBackend { .. } + | routing::RouteDecision::MetalUnavailable => unreachable!("handled above"), + } + } +} + +impl ImageCodec for Decoder<'_> { + type Error = Error; + type Warning = CpuWarning; + type Pool = CpuScratchPool; +} + +impl<'a> ImageDecode<'a> for Decoder<'a> { + type View = JpegView<'a>; + + fn inspect(input: &'a [u8]) -> Result { + Ok(CpuDecoder::inspect(input)?.to_core_info()) + } + + fn parse(input: &'a [u8]) -> Result { + Ok(JpegView::parse(input)?) + } + + fn from_view(view: Self::View) -> Result { + Self::from_view(view) + } + + fn decode_into( + &mut self, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + ) -> Result, Self::Error> { + Ok(self.inner.decode_into(out, stride, fmt)?.into()) + } + + fn decode_into_with_scratch( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + ) -> Result, Self::Error> { + Ok(self + .inner + .decode_into_with_scratch(pool, out, stride, fmt)? + .into()) + } + + fn decode_region_into( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, + ) -> Result, Self::Error> { + Ok(self + .inner + .decode_region_into_with_scratch(pool, out, stride, fmt, roi.into())? + .into()) + } + + fn decode_scaled_into( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + scale: Downscale, + ) -> Result, Self::Error> { + Ok(self + .inner + .decode_scaled_into_with_scratch(pool, out, stride, fmt, scale)? + .into()) + } + + fn decode_region_scaled_into( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + ) -> Result, Self::Error> { + Ok(self + .inner + .decode_region_scaled_into_with_scratch(pool, out, stride, fmt, roi.into(), scale)? + .into()) + } +} + +impl<'a> ImageDecodeDevice<'a> for Decoder<'a> { + type DeviceSurface = Surface; +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +/// JPEG codec marker used by J2K's generic decode traits. +pub struct Codec; + +#[cfg(target_os = "macos")] +struct Rgb8MetalBatchPlan { + requests: Vec, + output_dimensions: Option<(u32, u32)>, +} + +#[cfg(target_os = "macos")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Preflight report for RGB8 JPEG Metal resident decoder batches. +pub struct JpegMetalResidentBatchReport { + /// Requested decode operation. + pub op: j2k_jpeg::JpegDecodeOp, + /// Number of decoder tiles in the batch. + pub tile_count: usize, + /// Required output dimensions when the batch is eligible and shape-compatible. + pub output_dimensions: Option<(u32, u32)>, + /// Whether the batch can use reusable RGB8 Metal resident output. + pub eligibility: j2k_jpeg::JpegBackendEligibility, +} + +#[cfg(target_os = "macos")] +impl JpegMetalResidentBatchReport { + /// Required number of tile slots in caller-owned Metal output. + #[must_use] + pub fn required_tile_capacity(&self) -> usize { + self.tile_count + } +} + +#[cfg(target_os = "macos")] +fn report_required_output_dimensions( + report: &JpegMetalResidentBatchReport, +) -> Result, Error> { + if !report.eligibility.eligible { + return Err(Error::UnsupportedMetalRequest { + reason: report + .eligibility + .reason + .unwrap_or("JPEG Metal resident batch report is not eligible"), + }); + } + if report.tile_count == 0 { + return Ok(None); + } + report + .output_dimensions + .ok_or(Error::UnsupportedMetalRequest { + reason: "JPEG Metal resident batch report is missing output dimensions", + }) + .map(Some) +} + +#[cfg(target_os = "macos")] +fn rgb8_metal_output_dimensions_for_op( + full_dimensions: (u32, u32), + op: j2k_jpeg::JpegDecodeOp, +) -> Option<(u32, u32)> { + match op { + j2k_jpeg::JpegDecodeOp::Full => Some(full_dimensions), + j2k_jpeg::JpegDecodeOp::Scaled(scale) => Some(scaled_dims(full_dimensions, scale)), + j2k_jpeg::JpegDecodeOp::RegionScaled { roi, scale } => { + let scaled = Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + } + .scaled_covering(scale); + Some((scaled.w, scaled.h)) + } + j2k_jpeg::JpegDecodeOp::Region(_) => None, + } +} + +#[cfg(target_os = "macos")] +fn decoder_resident_sampling_family(decoder: &Decoder<'_>) -> batch::SamplingFamily { + if decoder.fast420_packet().is_some() { + batch::SamplingFamily::Fast420 + } else if decoder.fast422_packet().is_some() { + batch::SamplingFamily::Fast422 + } else if decoder.fast444_packet().is_some() { + batch::SamplingFamily::Fast444 + } else { + batch::SamplingFamily::Other + } +} + +#[cfg(target_os = "macos")] +fn decoder_resident_restart_interval_mcus(decoder: &Decoder<'_>) -> u32 { + if let Some(packet) = decoder.fast420_packet() { + packet.restart_interval_mcus + } else if let Some(packet) = decoder.fast422_packet() { + packet.restart_interval_mcus + } else if let Some(packet) = decoder.fast444_packet() { + packet.restart_interval_mcus + } else { + 0 + } +} + +impl ImageCodec for Codec { + type Error = Error; + type Warning = CpuWarning; + type Pool = CpuScratchPool; +} + +/// Inputs for a batched RGB8 Metal decode: raw JPEG bytes or pre-parsed +/// decoders that carry cached Metal fast-packet state. +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +pub enum Rgb8MetalBatchSource<'a, 'b> { + /// Raw JPEG byte streams, parsed per call. + Bytes(&'a [&'a [u8]]), + /// Already parsed `Decoder` wrappers; reuses their cached Metal + /// fast-packet state when building the resident batch request. + Decoders(&'a [&'a Decoder<'b>]), +} + +#[cfg(target_os = "macos")] +impl Rgb8MetalBatchSource<'_, '_> { + fn is_empty(&self) -> bool { + match self { + Rgb8MetalBatchSource::Bytes(inputs) => inputs.is_empty(), + Rgb8MetalBatchSource::Decoders(decoders) => decoders.is_empty(), + } + } +} + +/// Geometry op applied to every tile of a batched RGB8 Metal decode. +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Rgb8MetalBatchOp { + /// Full-tile decode at native dimensions. + Full, + /// Whole-tile downscale (half, quarter, or eighth). + Scaled(Downscale), + /// Scaled decode of one region, shared by every tile in the batch. + RegionScaled { + /// Region of interest to decode from every source tile. + roi: Rect, + /// Downscale factor applied to the selected region. + scale: Downscale, + }, +} + +/// A batched RGB8 Metal decode request: what to decode and how. +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +pub struct Rgb8MetalBatchRequest<'a, 'b> { + /// Source JPEG bytes or prepared decoders for the batch. + pub source: Rgb8MetalBatchSource<'a, 'b>, + /// Geometry operation applied to each source tile. + pub op: Rgb8MetalBatchOp, +} + +/// Caller-owned Metal buffer target for a batched RGB8 decode. +#[cfg(target_os = "macos")] +pub enum MetalBufferBatchTarget<'a> { + /// Reuse the buffer as-is; its shape must already fit the batch. + Reusable(&'a MetalBatchOutputBuffer), + /// Grow the buffer to fit the batch before decoding. + Resizable(&'a mut MetalBatchOutputBuffer), +} + +/// Caller-owned Metal RGBA8 texture target for a batched RGB8 decode. +#[cfg(target_os = "macos")] +pub enum MetalTextureBatchTarget<'a> { + /// Reuse the texture set as-is; its shape must already fit the batch. + Reusable(&'a MetalBatchTextureOutput), + /// Grow the texture set to fit the batch before decoding. + Resizable(&'a mut MetalBatchTextureOutput), +} + +impl Codec { + #[cfg(target_os = "macos")] + /// Inspect a cached RGB8 decoder batch for reusable Metal resident output. + /// + /// The report exposes whether the batch is resident-output eligible and, + /// when eligible, the exact output dimensions and tile capacity callers + /// should allocate before dispatch. + pub fn inspect_rgb8_decoder_batch_metal_output( + decoders: &[&Decoder<'_>], + op: j2k_jpeg::JpegDecodeOp, + ) -> JpegMetalResidentBatchReport { + if decoders.is_empty() { + return JpegMetalResidentBatchReport { + op, + tile_count: 0, + output_dimensions: None, + eligibility: j2k_jpeg::JpegBackendEligibility { + eligible: true, + reason: None, + }, + }; + } + + let mut output_dimensions = None; + let mut sampling_family = None; + for decoder in decoders { + let request = j2k_jpeg::JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb8, + }; + let report = j2k_jpeg::JpegCapabilityReport::for_decoder(decoder.inner(), request); + let eligibility = report.metal_resident_rgb8_batch_output(); + if !eligibility.eligible { + return JpegMetalResidentBatchReport { + op, + tile_count: decoders.len(), + output_dimensions: None, + eligibility, + }; + } + + if decoder.fast444_packet().is_none() + && decoder.fast422_packet().is_none() + && decoder.fast420_packet().is_none() + { + return JpegMetalResidentBatchReport { + op, + tile_count: decoders.len(), + output_dimensions: None, + eligibility: j2k_jpeg::JpegBackendEligibility { + eligible: false, + reason: Some( + "JPEG Metal reusable resident batch output requires cached fast-packet state", + ), + }, + }; + } + + let Some(dimensions) = + rgb8_metal_output_dimensions_for_op(decoder.inner().info().dimensions, op) + else { + return JpegMetalResidentBatchReport { + op, + tile_count: decoders.len(), + output_dimensions: None, + eligibility, + }; + }; + if let Some(first) = output_dimensions { + if first != dimensions { + return JpegMetalResidentBatchReport { + op, + tile_count: decoders.len(), + output_dimensions: None, + eligibility: j2k_jpeg::JpegBackendEligibility { + eligible: false, + reason: Some( + "JPEG Metal reusable RGB8 batch output requires matching output dimensions", + ), + }, + }; + } + } else { + output_dimensions = Some(dimensions); + } + + let decoder_sampling_family = decoder_resident_sampling_family(decoder); + if let Some(first) = sampling_family { + if first != decoder_sampling_family { + return JpegMetalResidentBatchReport { + op, + tile_count: decoders.len(), + output_dimensions: None, + eligibility: j2k_jpeg::JpegBackendEligibility { + eligible: false, + reason: Some( + "JPEG Metal reusable resident batch output requires one batch to use the same fast-packet sampling family", + ), + }, + }; + } + } else { + sampling_family = Some(decoder_sampling_family); + } + + if op == j2k_jpeg::JpegDecodeOp::Full + && matches!( + decoder_sampling_family, + batch::SamplingFamily::Fast422 | batch::SamplingFamily::Fast444 + ) + && decoder_resident_restart_interval_mcus(decoder) != 0 + { + return JpegMetalResidentBatchReport { + op, + tile_count: decoders.len(), + output_dimensions: None, + eligibility: j2k_jpeg::JpegBackendEligibility { + eligible: false, + reason: Some( + "JPEG Metal reusable resident batch output does not support restart-coded full-tile 4:2:2 or 4:4:4 batches", + ), + }, + }; + } + } + + JpegMetalResidentBatchReport { + op, + tile_count: decoders.len(), + output_dimensions, + eligibility: j2k_jpeg::JpegBackendEligibility { + eligible: true, + reason: None, + }, + } + } + + #[cfg(target_os = "macos")] + fn observe_rgb8_batch_output_dimensions( + first_output_dimensions: &mut Option<(u32, u32)>, + output_dimensions: (u32, u32), + ) -> Result<(), Error> { + if let Some(first) = *first_output_dimensions { + if first != output_dimensions { + return Err(Error::UnsupportedMetalRequest { + reason: + "JPEG Metal reusable RGB8 batch output requires matching output dimensions", + }); + } + } else { + *first_output_dimensions = Some(output_dimensions); + } + Ok(()) + } + + #[cfg(target_os = "macos")] + fn rgb8_metal_batch_requests( + inputs: &[&[u8]], + mut op_for_decoder: impl FnMut(&CpuDecoder<'_>) -> batch::BatchOp, + ) -> Result, Error> { + let plan = Self::rgb8_metal_batch_requests_with_output_dimensions(inputs, |decoder| { + (op_for_decoder(decoder), decoder.info().dimensions) + })?; + Ok(plan.requests) + } + + #[cfg(target_os = "macos")] + fn rgb8_metal_batch_requests_with_output_dimensions( + inputs: &[&[u8]], + mut op_and_dimensions_for_decoder: impl FnMut(&CpuDecoder<'_>) -> (batch::BatchOp, (u32, u32)), + ) -> Result { + let mut state = session::SessionState::default(); + let mut requests = Vec::with_capacity(inputs.len()); + let mut first_output_dimensions = None; + for input in inputs { + let decoder = CpuDecoder::new(input)?; + let (op, output_dimensions) = op_and_dimensions_for_decoder(&decoder); + Self::observe_rgb8_batch_output_dimensions( + &mut first_output_dimensions, + output_dimensions, + )?; + let input = state.intern_input_slice(input); + let (fast444_packet, fast422_packet, fast420_packet) = + state.resolve_fast_packets(&input, BackendRequest::Metal); + requests.push(batch::QueuedRequest::new_shared( + input, + PixelFormat::Rgb8, + BackendRequest::Metal, + op, + fast444_packet, + fast422_packet, + fast420_packet, + )); + } + Ok(Rgb8MetalBatchPlan { + requests, + output_dimensions: first_output_dimensions, + }) + } + + #[cfg(target_os = "macos")] + fn rgb8_metal_decoder_batch_requests_with_output_dimensions( + decoders: &[&Decoder<'_>], + mut op_and_dimensions_for_decoder: impl FnMut(&Decoder<'_>) -> (batch::BatchOp, (u32, u32)), + ) -> Result { + let mut requests = Vec::with_capacity(decoders.len()); + let mut first_output_dimensions = None; + for decoder in decoders { + let (op, output_dimensions) = op_and_dimensions_for_decoder(decoder); + Self::observe_rgb8_batch_output_dimensions( + &mut first_output_dimensions, + output_dimensions, + )?; + requests.push(decoder.rgb8_metal_request(op)); + } + Ok(Rgb8MetalBatchPlan { + requests, + output_dimensions: first_output_dimensions, + }) + } + #[cfg(target_os = "macos")] + fn rgb8_batch_op_and_dimensions( + op: Rgb8MetalBatchOp, + dimensions: (u32, u32), + ) -> (batch::BatchOp, (u32, u32)) { + match op { + Rgb8MetalBatchOp::Full => (batch::BatchOp::Full, dimensions), + Rgb8MetalBatchOp::Scaled(scale) => { + let (w, h) = dimensions; + ( + batch::BatchOp::RegionScaled { + roi: Rect { x: 0, y: 0, w, h }, + scale, + }, + scaled_dims((w, h), scale), + ) + } + Rgb8MetalBatchOp::RegionScaled { roi, scale } => { + let scaled = roi.scaled_covering(scale); + ( + batch::BatchOp::RegionScaled { roi, scale }, + (scaled.w, scaled.h), + ) + } + } + } + + #[cfg(target_os = "macos")] + fn rgb8_batch_jpeg_decode_op(op: Rgb8MetalBatchOp) -> j2k_jpeg::JpegDecodeOp { + match op { + Rgb8MetalBatchOp::Full => j2k_jpeg::JpegDecodeOp::Full, + Rgb8MetalBatchOp::Scaled(scale) => j2k_jpeg::JpegDecodeOp::Scaled(scale), + Rgb8MetalBatchOp::RegionScaled { roi, scale } => j2k_jpeg::JpegDecodeOp::RegionScaled { + roi: roi.into(), + scale, + }, + } + } + + #[cfg(target_os = "macos")] + fn plan_rgb8_metal_batch( + source: Rgb8MetalBatchSource<'_, '_>, + op: Rgb8MetalBatchOp, + track_output_dimensions: bool, + ) -> Result<(Rgb8MetalBatchPlan, usize), Error> { + match source { + Rgb8MetalBatchSource::Bytes(inputs) => { + if track_output_dimensions { + Self::rgb8_metal_batch_requests_with_output_dimensions(inputs, |decoder| { + Self::rgb8_batch_op_and_dimensions(op, decoder.info().dimensions) + }) + .map(|plan| (plan, inputs.len())) + } else { + Self::rgb8_metal_batch_requests(inputs, |decoder| { + Self::rgb8_batch_op_and_dimensions(op, decoder.info().dimensions).0 + }) + .map(|requests| { + ( + Rgb8MetalBatchPlan { + requests, + output_dimensions: None, + }, + inputs.len(), + ) + }) + } + } + Rgb8MetalBatchSource::Decoders(decoders) => { + Self::rgb8_metal_decoder_batch_requests_with_output_dimensions( + decoders, + |decoder| { + Self::rgb8_batch_op_and_dimensions(op, decoder.inner().info().dimensions) + }, + ) + .map(|plan| (plan, decoders.len())) + } + } + } + + #[cfg(target_os = "macos")] + const fn rgb8_buffer_batch_unsupported_reason(op: Rgb8MetalBatchOp) -> &'static str { + match op { + Rgb8MetalBatchOp::Full => { + "JPEG Metal reusable batch output currently supports batchable full-tile RGB8 fast 4:2:0, 4:2:2, or 4:4:4 inputs" + } + Rgb8MetalBatchOp::Scaled(_) => { + "JPEG Metal reusable scaled batch output currently supports batchable RGB8 fast 4:2:0, 4:2:2, or 4:4:4 inputs with half, quarter, or eighth scaling" + } + Rgb8MetalBatchOp::RegionScaled { .. } => { + "JPEG Metal reusable region-scaled batch output currently supports batchable RGB8 fast 4:2:0, 4:2:2, or 4:4:4 inputs with matching output shapes" + } + } + } + + #[cfg(target_os = "macos")] + const fn rgb8_texture_batch_unsupported_reason(op: Rgb8MetalBatchOp) -> &'static str { + match op { + Rgb8MetalBatchOp::Full => { + "JPEG Metal texture batch output currently supports batchable full-tile RGB8 fast 4:2:0, 4:2:2, or 4:4:4 inputs" + } + Rgb8MetalBatchOp::Scaled(_) => { + "JPEG Metal texture scaled batch output currently supports batchable RGB8 fast 4:2:0, 4:2:2, or 4:4:4 inputs with half, quarter, or eighth scaling" + } + Rgb8MetalBatchOp::RegionScaled { .. } => { + "JPEG Metal texture region-scaled batch output currently supports batchable RGB8 fast 4:2:0, 4:2:2, or 4:4:4 inputs with matching output shapes" + } + } + } + + #[cfg(target_os = "macos")] + /// Decode a batched RGB8 JPEG request into a caller-owned Metal buffer. + /// + /// This is the single buffer-output entry point for full, scaled, and + /// region-scaled batches sourced from raw bytes or pre-parsed decoders; + /// `MetalBufferBatchTarget::Resizable` grows the buffer to fit before + /// decoding. + pub fn decode_rgb8_batch_into_buffer_with_session( + request: Rgb8MetalBatchRequest<'_, '_>, + target: MetalBufferBatchTarget<'_>, + session: &MetalBackendSession, + ) -> Result>, Error> { + if request.source.is_empty() { + return Ok(Vec::new()); + } + + let resizable = matches!(target, MetalBufferBatchTarget::Resizable(_)); + let (plan, tile_count) = + Self::plan_rgb8_metal_batch(request.source, request.op, resizable)?; + let output: &MetalBatchOutputBuffer = match target { + MetalBufferBatchTarget::Reusable(output) => output, + MetalBufferBatchTarget::Resizable(output) => { + if let Rgb8MetalBatchSource::Decoders(decoders) = request.source { + let report = Self::inspect_rgb8_decoder_batch_metal_output( + decoders, + Self::rgb8_batch_jpeg_decode_op(request.op), + ); + output.ensure_rgb8_batch_report(session, &report)?; + } + let Some(output_dimensions) = plan.output_dimensions else { + return Ok(Vec::new()); + }; + output.ensure_rgb8_tiles(session, output_dimensions, tile_count)?; + output + } + }; + + let results = match request.op { + Rgb8MetalBatchOp::Full => compute::decode_full_rgb8_batch_into_output_with_session( + &plan.requests, + output, + session, + )?, + Rgb8MetalBatchOp::Scaled(_) | Rgb8MetalBatchOp::RegionScaled { .. } => { + compute::decode_region_scaled_rgb8_batch_into_output_with_session( + &plan.requests, + output, + session, + )? + } + }; + results.ok_or(Error::UnsupportedMetalRequest { + reason: Self::rgb8_buffer_batch_unsupported_reason(request.op), + }) + } + + #[cfg(target_os = "macos")] + /// Decode a batched RGB8 JPEG request into caller-owned Metal RGBA8 textures. + /// + /// This is the single texture-output entry point for full, scaled, and + /// region-scaled batches sourced from raw bytes or pre-parsed decoders; + /// `MetalTextureBatchTarget::Resizable` grows the texture set to fit + /// before decoding. + pub fn decode_rgb8_batch_into_textures_with_session( + request: Rgb8MetalBatchRequest<'_, '_>, + target: MetalTextureBatchTarget<'_>, + session: &MetalBackendSession, + ) -> Result>, Error> { + if request.source.is_empty() { + return Ok(Vec::new()); + } + + let resizable = matches!(target, MetalTextureBatchTarget::Resizable(_)); + let (plan, tile_count) = + Self::plan_rgb8_metal_batch(request.source, request.op, resizable)?; + let output: &MetalBatchTextureOutput = match target { + MetalTextureBatchTarget::Reusable(output) => output, + MetalTextureBatchTarget::Resizable(output) => { + if let Rgb8MetalBatchSource::Decoders(decoders) = request.source { + let report = Self::inspect_rgb8_decoder_batch_metal_output( + decoders, + Self::rgb8_batch_jpeg_decode_op(request.op), + ); + output.ensure_rgba8_batch_report(session, &report)?; + } + let Some(output_dimensions) = plan.output_dimensions else { + return Ok(Vec::new()); + }; + output.ensure_rgba8_tiles(session, output_dimensions, tile_count)?; + output + } + }; + + let results = match request.op { + Rgb8MetalBatchOp::Full => compute::decode_full_rgb8_batch_into_textures_with_session( + &plan.requests, + output, + session, + )?, + Rgb8MetalBatchOp::Scaled(_) | Rgb8MetalBatchOp::RegionScaled { .. } => { + compute::decode_region_scaled_rgb8_batch_into_textures_with_session( + &plan.requests, + output, + session, + )? + } + }; + results.ok_or(Error::UnsupportedMetalRequest { + reason: Self::rgb8_texture_batch_unsupported_reason(request.op), + }) + } + + #[cfg(target_os = "macos")] + /// Decode a full-tile RGB8 JPEG decoder batch into resizable caller-owned + /// Metal RGBA8 textures. + /// + /// Convenience wrapper over [`Codec::decode_rgb8_batch_into_textures_with_session`] + /// for the resident whole-slide tile path. + pub fn decode_rgb8_decoder_batch_into_resizable_metal_textures_with_session( + decoders: &[&Decoder<'_>], + output: &mut MetalBatchTextureOutput, + session: &MetalBackendSession, + ) -> Result>, Error> { + Self::decode_rgb8_batch_into_textures_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(decoders), + op: Rgb8MetalBatchOp::Full, + }, + MetalTextureBatchTarget::Resizable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + /// Decode a region-scaled RGB8 JPEG batch into a resizable caller-owned + /// Metal buffer. + /// + /// Convenience wrapper over [`Codec::decode_rgb8_batch_into_buffer_with_session`] + /// for the viewport composition path. + pub fn decode_rgb8_region_scaled_batch_into_resizable_metal_buffer_with_session( + inputs: &[&[u8]], + roi: Rect, + scale: Downscale, + output: &mut MetalBatchOutputBuffer, + session: &MetalBackendSession, + ) -> Result>, Error> { + Self::decode_rgb8_batch_into_buffer_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Bytes(inputs), + op: Rgb8MetalBatchOp::RegionScaled { roi, scale }, + }, + MetalBufferBatchTarget::Resizable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + /// Decode a region-scaled RGB8 JPEG batch into resizable caller-owned + /// Metal RGBA8 textures. + /// + /// Convenience wrapper over [`Codec::decode_rgb8_batch_into_textures_with_session`] + /// for the viewport composition path. + pub fn decode_rgb8_region_scaled_batch_into_resizable_metal_textures_with_session( + inputs: &[&[u8]], + roi: Rect, + scale: Downscale, + output: &mut MetalBatchTextureOutput, + session: &MetalBackendSession, + ) -> Result>, Error> { + Self::decode_rgb8_batch_into_textures_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Bytes(inputs), + op: Rgb8MetalBatchOp::RegionScaled { roi, scale }, + }, + MetalTextureBatchTarget::Resizable(output), + session, + ) + } + + #[allow(clippy::too_many_arguments)] + /// Submit a scaled region tile decode into a reusable Metal session. + pub fn submit_tile_region_scaled_to_device( + ctx: &mut j2k_core::DecoderContext, + session: &mut MetalSession, + pool: &mut CpuScratchPool, + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + backend: BackendRequest, + ) -> Result<::SubmittedSurface, Error> { + let _ = (ctx, pool); + let slot = { + let mut state = session.shared.lock()?; + let input = state.intern_input_slice(input); + let (fast444_packet, fast422_packet, fast420_packet) = + state.resolve_fast_packets(&input, backend); + state.queue_request(batch::QueuedRequest::new_shared( + input, + fmt, + backend, + batch::BatchOp::RegionScaled { roi, scale }, + fast444_packet, + fast422_packet, + fast420_packet, + )) + }; + Ok(batch::MetalSubmission { + session: session.shared.clone(), + slot, + }) + } +} + +impl<'a> ImageDecodeSubmit<'a> for Decoder<'a> { + type Session = MetalSession; + type DeviceSurface = Surface; + type SubmittedSurface = batch::MetalSubmission; + + fn submit_to_device( + &mut self, + session: &mut Self::Session, + fmt: PixelFormat, + backend: BackendRequest, + ) -> Result { + let fast444_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { + self.fast444_packet.clone() + } else { + None + }; + let fast422_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { + self.fast422_packet.clone() + } else { + None + }; + let fast420_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { + self.fast420_packet.clone() + } else { + None + }; + let slot = session + .shared + .lock()? + .queue_request(batch::QueuedRequest::new_shared( + Arc::clone(&self.source), + fmt, + backend, + batch::BatchOp::Full, + fast444_packet, + fast422_packet, + fast420_packet, + )); + Ok(batch::MetalSubmission { + session: session.shared.clone(), + slot, + }) + } + + fn submit_region_to_device( + &mut self, + session: &mut Self::Session, + fmt: PixelFormat, + roi: Rect, + backend: BackendRequest, + ) -> Result { + let fast444_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { + self.fast444_packet.clone() + } else { + None + }; + let fast422_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { + self.fast422_packet.clone() + } else { + None + }; + let fast420_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { + self.fast420_packet.clone() + } else { + None + }; + let slot = session + .shared + .lock()? + .queue_request(batch::QueuedRequest::new_shared( + Arc::clone(&self.source), + fmt, + backend, + batch::BatchOp::Region(roi), + fast444_packet, + fast422_packet, + fast420_packet, + )); + Ok(batch::MetalSubmission { + session: session.shared.clone(), + slot, + }) + } + + fn submit_scaled_to_device( + &mut self, + session: &mut Self::Session, + fmt: PixelFormat, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + let fast444_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { + self.fast444_packet.clone() + } else { + None + }; + let fast422_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { + self.fast422_packet.clone() + } else { + None + }; + let fast420_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { + self.fast420_packet.clone() + } else { + None + }; + let slot = session + .shared + .lock()? + .queue_request(batch::QueuedRequest::new_shared( + Arc::clone(&self.source), + fmt, + backend, + batch::BatchOp::Scaled(scale), + fast444_packet, + fast422_packet, + fast420_packet, + )); + Ok(batch::MetalSubmission { + session: session.shared.clone(), + slot, + }) + } + + fn submit_region_scaled_to_device( + &mut self, + session: &mut Self::Session, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + let fast444_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { + self.fast444_packet.clone() + } else { + None + }; + let fast422_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { + self.fast422_packet.clone() + } else { + None + }; + let fast420_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { + self.fast420_packet.clone() + } else { + None + }; + let slot = session + .shared + .lock()? + .queue_request(batch::QueuedRequest::new_shared( + Arc::clone(&self.source), + fmt, + backend, + batch::BatchOp::RegionScaled { roi, scale }, + fast444_packet, + fast422_packet, + fast420_packet, + )); + Ok(batch::MetalSubmission { + session: session.shared.clone(), + slot, + }) + } +} + +impl TileBatchDecodeSubmit for Codec { + type Context = CpuDecoderContext; + type Session = MetalSession; + type DeviceSurface = Surface; + type SubmittedSurface = batch::MetalSubmission; + + fn submit_tile_to_device( + ctx: &mut j2k_core::DecoderContext, + session: &mut Self::Session, + pool: &mut Self::Pool, + input: &[u8], + fmt: PixelFormat, + backend: BackendRequest, + ) -> Result { + let _ = (ctx, pool); + let slot = { + let mut state = session.shared.lock()?; + let input = state.intern_input_slice(input); + let (fast444_packet, fast422_packet, fast420_packet) = + state.resolve_fast_packets(&input, backend); + state.queue_request(batch::QueuedRequest::new_shared( + input, + fmt, + backend, + batch::BatchOp::Full, + fast444_packet, + fast422_packet, + fast420_packet, + )) + }; + Ok(batch::MetalSubmission { + session: session.shared.clone(), + slot, + }) + } + + fn submit_tile_region_to_device( + ctx: &mut j2k_core::DecoderContext, + session: &mut Self::Session, + pool: &mut Self::Pool, + input: &[u8], + fmt: PixelFormat, + roi: Rect, + backend: BackendRequest, + ) -> Result { + let _ = (ctx, pool); + let slot = { + let mut state = session.shared.lock()?; + let input = state.intern_input_slice(input); + let (fast444_packet, fast422_packet, fast420_packet) = + state.resolve_fast_packets(&input, backend); + state.queue_request(batch::QueuedRequest::new_shared( + input, + fmt, + backend, + batch::BatchOp::Region(roi), + fast444_packet, + fast422_packet, + fast420_packet, + )) + }; + Ok(batch::MetalSubmission { + session: session.shared.clone(), + slot, + }) + } + + fn submit_tile_scaled_to_device( + ctx: &mut j2k_core::DecoderContext, + session: &mut Self::Session, + pool: &mut Self::Pool, + input: &[u8], + fmt: PixelFormat, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + let _ = (ctx, pool); + let slot = { + let mut state = session.shared.lock()?; + let input = state.intern_input_slice(input); + let (fast444_packet, fast422_packet, fast420_packet) = + state.resolve_fast_packets(&input, backend); + state.queue_request(batch::QueuedRequest::new_shared( + input, + fmt, + backend, + batch::BatchOp::Scaled(scale), + fast444_packet, + fast422_packet, + fast420_packet, + )) + }; + Ok(batch::MetalSubmission { + session: session.shared.clone(), + slot, + }) + } + + fn submit_tile_region_scaled_to_device( + ctx: &mut j2k_core::DecoderContext, + session: &mut Self::Session, + pool: &mut Self::Pool, + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + backend: BackendRequest, + ) -> Result { + Codec::submit_tile_region_scaled_to_device( + ctx, session, pool, input, fmt, roi, scale, backend, + ) + } +} + +impl TileBatchDecodeDevice for Codec { + type Context = CpuDecoderContext; + type DeviceSurface = Surface; +} + +impl TileBatchDecodeManyDevice for Codec { + type Context = CpuDecoderContext; + type DeviceSurface = Surface; + + fn decode_tiles_to_device( + ctx: &mut j2k_core::DecoderContext, + pool: &mut Self::Pool, + inputs: &[&[u8]], + fmt: PixelFormat, + backend: BackendRequest, + ) -> Result, Self::Error> { + if inputs.is_empty() { + return Ok(Vec::new()); + } + + let mut session = MetalSession::default(); + let submissions = inputs + .iter() + .map(|input| { + ::submit_tile_to_device( + ctx, + &mut session, + pool, + input, + fmt, + backend, + ) + }) + .collect::, _>>()?; + + submissions + .into_iter() + .map(DeviceSubmission::wait) + .collect() + } +} + +pub(crate) fn decode_surface_from_bytes( + input: &[u8], + fmt: PixelFormat, + backend: BackendRequest, + op: batch::BatchOp, + fast444_packet: Option>, + fast422_packet: Option>, + fast420_packet: Option>, +) -> Result { + let decoder = CpuDecoder::new(input)?; + let mut pool = CpuScratchPool::new(); + let build_auto_packets = + matches!(backend, BackendRequest::Auto) && decoder.info().restart_interval.is_some(); + let build_metal_packets = matches!(backend, BackendRequest::Metal); + let fast444_packet = if build_auto_packets || build_metal_packets { + fast444_packet.or_else(|| { + build_fast444_packet_for_decoder(&decoder) + .ok() + .map(Arc::new) + }) + } else { + None + }; + let fast422_packet = if build_auto_packets || build_metal_packets { + fast422_packet.or_else(|| { + build_fast422_packet_for_decoder(&decoder) + .ok() + .map(Arc::new) + }) + } else { + None + }; + let fast420_packet = if build_auto_packets || build_metal_packets { + fast420_packet.or_else(|| { + build_fast420_packet_for_decoder(&decoder) + .ok() + .map(Arc::new) + }) + } else { + None + }; + decode_surface_from_decoder( + &decoder, + &mut pool, + fmt, + backend, + op, + fast444_packet.as_deref(), + fast422_packet.as_deref(), + fast420_packet.as_deref(), + ) +} + +#[cfg(not(target_os = "macos"))] +#[allow(clippy::unnecessary_wraps)] +pub(crate) fn decode_compatible_batch( + requests: &[batch::QueuedRequest], +) -> Result>>, Error> { + let _ = requests; + Ok(None) +} + +#[allow(clippy::unnecessary_wraps)] +pub(crate) fn decode_compatible_batch_with_session( + requests: &[batch::QueuedRequest], + session: &mut session::SessionState, +) -> Result>>, Error> { + #[cfg(target_os = "macos")] + { + compute::decode_full_batch_to_surfaces_with_session_state(requests, session) + } + #[cfg(not(target_os = "macos"))] + { + let _ = session; + decode_compatible_batch(requests) + } +} + +#[cfg(target_os = "macos")] +#[doc(hidden)] +pub fn decode_rgb8_batch_to_device_with_session( + inputs: &[&[u8]], + session: &MetalBackendSession, +) -> Result>>, Error> { + if inputs.len() < 2 { + return Ok(None); + } + + let mut state = session::SessionState::default(); + let mut requests = Vec::with_capacity(inputs.len()); + for input in inputs { + let input = state.intern_input_slice(input); + let (fast444_packet, fast422_packet, fast420_packet) = + state.resolve_fast_packets(&input, BackendRequest::Metal); + requests.push(batch::QueuedRequest::new_shared( + input, + PixelFormat::Rgb8, + BackendRequest::Metal, + batch::BatchOp::Full, + fast444_packet, + fast422_packet, + fast420_packet, + )); + } + + compute::decode_full_batch_to_surfaces_with_session(&requests, session) +} + +#[allow(clippy::too_many_arguments)] +fn decode_surface_from_decoder( + decoder: &CpuDecoder<'_>, + pool: &mut CpuScratchPool, + fmt: PixelFormat, + backend: BackendRequest, + op: batch::BatchOp, + fast444_packet: Option<&JpegFast444PacketV1>, + fast422_packet: Option<&JpegFast422PacketV1>, + fast420_packet: Option<&JpegFast420PacketV1>, +) -> Result { + match op { + batch::BatchOp::Full => match backend { + BackendRequest::Cpu => decode_full_cpu_upload(decoder, pool, fmt), + BackendRequest::Auto | BackendRequest::Metal => { + let decision = choose_route( + decoder, + backend, + fmt, + op, + fast444_packet, + fast422_packet, + fast420_packet, + ); + if let Some(err) = routing::decision_error(decision) { + return Err(err); + } + match decision { + routing::RouteDecision::CpuHost => decode_full_cpu_upload(decoder, pool, fmt), + routing::RouteDecision::MetalKernel => { + #[cfg(target_os = "macos")] + { + reject_cpu_staged_metal_upload(compute::decode_to_surface( + decoder, + pool, + fmt, + fast444_packet, + fast422_packet, + fast420_packet, + )?) + } + #[cfg(not(target_os = "macos"))] + { + let _ = ( + decoder, + pool, + fmt, + fast444_packet, + fast422_packet, + fast420_packet, + ); + Err(Error::MetalUnavailable) + } + } + routing::RouteDecision::RejectExplicitMetal { .. } + | routing::RouteDecision::RejectUnsupportedBackend { .. } + | routing::RouteDecision::MetalUnavailable => unreachable!("handled above"), + } + } + BackendRequest::Cuda => Err(Error::UnsupportedBackend { request: backend }), + }, + batch::BatchOp::Region(roi) => match backend { + BackendRequest::Cpu => decode_region_cpu_upload(decoder, pool, fmt, roi), + BackendRequest::Auto | BackendRequest::Metal => { + let decision = choose_route( + decoder, + backend, + fmt, + op, + fast444_packet, + fast422_packet, + fast420_packet, + ); + if let Some(err) = routing::decision_error(decision) { + return Err(err); + } + match decision { + routing::RouteDecision::CpuHost => { + decode_region_cpu_upload(decoder, pool, fmt, roi) + } + routing::RouteDecision::MetalKernel => { + #[cfg(target_os = "macos")] + { + reject_cpu_staged_metal_upload(compute::decode_region_to_surface( + decoder, + pool, + fmt, + roi.into(), + fast444_packet, + fast422_packet, + fast420_packet, + )?) + } + #[cfg(not(target_os = "macos"))] + { + let _ = ( + decoder, + pool, + fmt, + roi, + fast444_packet, + fast422_packet, + fast420_packet, + ); + Err(Error::MetalUnavailable) + } + } + routing::RouteDecision::RejectExplicitMetal { .. } + | routing::RouteDecision::RejectUnsupportedBackend { .. } + | routing::RouteDecision::MetalUnavailable => unreachable!("handled above"), + } + } + BackendRequest::Cuda => Err(Error::UnsupportedBackend { request: backend }), + }, + batch::BatchOp::Scaled(scale) => match backend { + BackendRequest::Cpu => decode_scaled_cpu_upload(decoder, pool, fmt, scale), + BackendRequest::Auto | BackendRequest::Metal => { + let decision = choose_route( + decoder, + backend, + fmt, + op, + fast444_packet, + fast422_packet, + fast420_packet, + ); + if let Some(err) = routing::decision_error(decision) { + return Err(err); + } + match decision { + routing::RouteDecision::CpuHost => { + decode_scaled_cpu_upload(decoder, pool, fmt, scale) + } + routing::RouteDecision::MetalKernel => { + #[cfg(target_os = "macos")] + { + reject_cpu_staged_metal_upload(compute::decode_scaled_to_surface( + decoder, + pool, + fmt, + scale, + fast444_packet, + fast422_packet, + fast420_packet, + )?) + } + #[cfg(not(target_os = "macos"))] + { + let _ = ( + decoder, + pool, + fmt, + scale, + fast444_packet, + fast422_packet, + fast420_packet, + ); + Err(Error::MetalUnavailable) + } + } + routing::RouteDecision::RejectExplicitMetal { .. } + | routing::RouteDecision::RejectUnsupportedBackend { .. } + | routing::RouteDecision::MetalUnavailable => unreachable!("handled above"), + } + } + BackendRequest::Cuda => Err(Error::UnsupportedBackend { request: backend }), + }, + batch::BatchOp::RegionScaled { roi, scale } => decode_region_scaled_surface_from_decoder( + decoder, + pool, + fmt, + roi, + scale, + backend, + fast444_packet, + fast422_packet, + fast420_packet, + ), + } +} + +fn decode_full_cpu_upload( + decoder: &CpuDecoder<'_>, + pool: &mut CpuScratchPool, + fmt: PixelFormat, +) -> Result { + let dims = decoder.info().dimensions; + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + decoder.decode_into_with_scratch(pool, &mut out, stride, fmt)?; + upload_surface(out, dims, fmt, BackendRequest::Cpu) +} + +fn decode_region_cpu_upload( + decoder: &CpuDecoder<'_>, + pool: &mut CpuScratchPool, + fmt: PixelFormat, + roi: Rect, +) -> Result { + let dims = (roi.w, roi.h); + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + decoder.decode_region_into_with_scratch(pool, &mut out, stride, fmt, roi.into())?; + upload_surface(out, dims, fmt, BackendRequest::Cpu) +} + +fn decode_scaled_cpu_upload( + decoder: &CpuDecoder<'_>, + pool: &mut CpuScratchPool, + fmt: PixelFormat, + scale: Downscale, +) -> Result { + let dims = scaled_dims(decoder.info().dimensions, scale); + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + decoder.decode_scaled_into_with_scratch(pool, &mut out, stride, fmt, scale)?; + upload_surface(out, dims, fmt, BackendRequest::Cpu) +} + +#[allow(clippy::too_many_arguments)] +fn decode_region_scaled_surface_from_decoder( + decoder: &CpuDecoder<'_>, + pool: &mut CpuScratchPool, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + backend: BackendRequest, + fast444_packet: Option<&JpegFast444PacketV1>, + fast422_packet: Option<&JpegFast422PacketV1>, + fast420_packet: Option<&JpegFast420PacketV1>, +) -> Result { + match backend { + BackendRequest::Cpu => { + decode_region_scaled_cpu_upload(decoder, pool, fmt, roi, scale, BackendRequest::Cpu) + } + BackendRequest::Auto | BackendRequest::Metal => { + let decision = choose_route( + decoder, + backend, + fmt, + batch::BatchOp::RegionScaled { roi, scale }, + fast444_packet, + fast422_packet, + fast420_packet, + ); + if let Some(err) = routing::decision_error(decision) { + return Err(err); + } + match decision { + routing::RouteDecision::CpuHost => decode_region_scaled_cpu_upload( + decoder, + pool, + fmt, + roi, + scale, + BackendRequest::Cpu, + ), + routing::RouteDecision::MetalKernel => { + #[cfg(target_os = "macos")] + { + reject_cpu_staged_metal_upload(compute::decode_region_scaled_to_surface( + decoder, + pool, + fmt, + roi.into(), + scale, + fast444_packet, + fast422_packet, + fast420_packet, + )?) + } + #[cfg(not(target_os = "macos"))] + { + let _ = ( + decoder, + pool, + fmt, + roi, + scale, + fast444_packet, + fast422_packet, + fast420_packet, + ); + Err(Error::MetalUnavailable) + } + } + routing::RouteDecision::RejectExplicitMetal { .. } + | routing::RouteDecision::RejectUnsupportedBackend { .. } + | routing::RouteDecision::MetalUnavailable => unreachable!("handled above"), + } + } + BackendRequest::Cuda => Err(Error::UnsupportedBackend { request: backend }), + } +} + +#[cfg(target_os = "macos")] +fn reject_cpu_staged_metal_upload(surface: Surface) -> Result { + if surface.residency() == SurfaceResidency::CpuStagedMetalUpload { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal explicit device decode requires a direct resident Metal decode; use the CPU path for CPU-staged output", + }); + } + Ok(surface) +} + +#[allow(clippy::too_many_arguments)] +fn choose_route( + decoder: &CpuDecoder<'_>, + backend: BackendRequest, + fmt: PixelFormat, + op: batch::BatchOp, + fast444_packet: Option<&JpegFast444PacketV1>, + fast422_packet: Option<&JpegFast422PacketV1>, + fast420_packet: Option<&JpegFast420PacketV1>, +) -> routing::RouteDecision { + let capabilities = routing::JpegMetalCapabilities::for_request( + decoder, + fmt, + op, + fast444_packet, + fast422_packet, + fast420_packet, + ); + let decision = routing::decide_route(backend, capabilities); + if j2k_profile::gpu_route_profile_enabled() { + let request_s = format!("{backend:?}"); + let fmt_s = format!("{fmt:?}"); + let has_fast_packet_s = capabilities.has_fast_packet().to_string(); + let supports_format_s = capabilities.supports_output_format().to_string(); + let labels = jpeg_route_decision_profile(decision); + j2k_profile::emit_gpu_route_profile( + "jpeg", + "metal", + &[ + ("request", request_s.as_str()), + ("fmt", fmt_s.as_str()), + ("op", jpeg_batch_op_profile(op)), + ("has_fast_packet", has_fast_packet_s.as_str()), + ("supports_output_format", supports_format_s.as_str()), + ("decision", labels.decision), + ("reason", labels.reason), + ], + ); + } + decision +} + +fn jpeg_batch_op_profile(op: batch::BatchOp) -> &'static str { + match op { + batch::BatchOp::Full => "full", + batch::BatchOp::Region(_) => "region", + batch::BatchOp::Scaled(_) => "scaled", + batch::BatchOp::RegionScaled { .. } => "region_scaled", + } +} + +fn jpeg_route_decision_profile(decision: routing::RouteDecision) -> MetalRouteProfileLabels { + match decision { + routing::RouteDecision::CpuHost => cpu_host_route(), + routing::RouteDecision::MetalKernel => metal_kernel_route(), + routing::RouteDecision::RejectExplicitMetal { reason } => { + let reason_code = if reason.contains("fast") { + "no_fast_packet" + } else { + "unsupported_format" + }; + reject_explicit_metal_route(reason_code) + } + routing::RouteDecision::RejectUnsupportedBackend { .. } => { + reject_unsupported_backend_route() + } + routing::RouteDecision::MetalUnavailable => metal_unavailable_route(), + } +} + +fn decode_region_scaled_cpu_upload( + decoder: &CpuDecoder<'_>, + pool: &mut CpuScratchPool, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + backend: BackendRequest, +) -> Result { + let scaled = roi.scaled_covering(scale); + let dims = (scaled.w, scaled.h); + let stride = dims.0 as usize * fmt.bytes_per_pixel(); + let mut out = vec![0u8; stride * dims.1 as usize]; + decoder.decode_region_scaled_into_with_scratch( + pool, + &mut out, + stride, + fmt, + roi.into(), + scale, + )?; + upload_surface(out, dims, fmt, backend) +} + +fn scaled_dims(full: (u32, u32), scale: Downscale) -> (u32, u32) { + ( + full.0.div_ceil(scale.denominator()), + full.1.div_ceil(scale.denominator()), + ) +} + +pub(crate) fn upload_surface( + bytes: Vec, + dimensions: (u32, u32), + fmt: PixelFormat, + backend: BackendRequest, +) -> Result { + let pitch_bytes = dimensions.0 as usize * fmt.bytes_per_pixel(); + match backend { + BackendRequest::Cpu => Ok(Surface { + backend: BackendKind::Cpu, + residency: SurfaceResidency::Host, + dimensions, + fmt, + pitch_bytes, + storage: Storage::Host(bytes), + }), + BackendRequest::Auto | BackendRequest::Metal => { + #[cfg(target_os = "macos")] + { + let device = Device::system_default().ok_or(Error::MetalUnavailable)?; + let buffer = device.new_buffer_with_data( + bytes.as_ptr().cast(), + bytes.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + Ok(Surface { + backend: BackendKind::Metal, + residency: SurfaceResidency::CpuStagedMetalUpload, + dimensions, + fmt, + pitch_bytes, + storage: Storage::Metal { buffer, offset: 0 }, + }) + } + #[cfg(not(target_os = "macos"))] + { + if matches!(backend, BackendRequest::Auto) { + Ok(Surface { + backend: BackendKind::Cpu, + residency: SurfaceResidency::Host, + dimensions, + fmt, + pitch_bytes, + storage: Storage::Host(bytes), + }) + } else { + Err(Error::MetalUnavailable) + } + } + } + BackendRequest::Cuda => Err(Error::UnsupportedBackend { request: backend }), + } +} + +pub use j2k_jpeg::{ + DecoderContext, Downscale as JpegDownscale, PixelFormat as JpegPixelFormat, ScratchPool, +}; +pub use j2k_jpeg::{Info, Rect as JpegRectPublic}; + +#[cfg(test)] +mod tests { + use super::*; + + // Shims over the collapsed batch API so every legacy entry shape + // (source x op x target) keeps device coverage. + #[cfg(target_os = "macos")] + fn decode_rgb8_batch_into_metal_buffer_with_session( + inputs: &[&[u8]], + output: &MetalBatchOutputBuffer, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_buffer_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Bytes(inputs), + op: Rgb8MetalBatchOp::Full, + }, + MetalBufferBatchTarget::Reusable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_batch_into_metal_textures_with_session( + inputs: &[&[u8]], + output: &MetalBatchTextureOutput, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_textures_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Bytes(inputs), + op: Rgb8MetalBatchOp::Full, + }, + MetalTextureBatchTarget::Reusable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_decoder_batch_into_metal_buffer_with_session( + decoders: &[&Decoder<'_>], + output: &MetalBatchOutputBuffer, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_buffer_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(decoders), + op: Rgb8MetalBatchOp::Full, + }, + MetalBufferBatchTarget::Reusable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_decoder_batch_into_metal_textures_with_session( + decoders: &[&Decoder<'_>], + output: &MetalBatchTextureOutput, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_textures_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(decoders), + op: Rgb8MetalBatchOp::Full, + }, + MetalTextureBatchTarget::Reusable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_decoder_batch_into_resizable_metal_buffer_with_session( + decoders: &[&Decoder<'_>], + output: &mut MetalBatchOutputBuffer, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_buffer_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(decoders), + op: Rgb8MetalBatchOp::Full, + }, + MetalBufferBatchTarget::Resizable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_scaled_batch_into_metal_buffer_with_session( + inputs: &[&[u8]], + scale: Downscale, + output: &MetalBatchOutputBuffer, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_buffer_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Bytes(inputs), + op: Rgb8MetalBatchOp::Scaled(scale), + }, + MetalBufferBatchTarget::Reusable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_scaled_batch_into_metal_textures_with_session( + inputs: &[&[u8]], + scale: Downscale, + output: &MetalBatchTextureOutput, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_textures_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Bytes(inputs), + op: Rgb8MetalBatchOp::Scaled(scale), + }, + MetalTextureBatchTarget::Reusable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_scaled_batch_into_resizable_metal_textures_with_session( + inputs: &[&[u8]], + scale: Downscale, + output: &mut MetalBatchTextureOutput, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_textures_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Bytes(inputs), + op: Rgb8MetalBatchOp::Scaled(scale), + }, + MetalTextureBatchTarget::Resizable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_decoder_scaled_batch_into_metal_buffer_with_session( + decoders: &[&Decoder<'_>], + scale: Downscale, + output: &MetalBatchOutputBuffer, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_buffer_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(decoders), + op: Rgb8MetalBatchOp::Scaled(scale), + }, + MetalBufferBatchTarget::Reusable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_decoder_scaled_batch_into_metal_textures_with_session( + decoders: &[&Decoder<'_>], + scale: Downscale, + output: &MetalBatchTextureOutput, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_textures_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(decoders), + op: Rgb8MetalBatchOp::Scaled(scale), + }, + MetalTextureBatchTarget::Reusable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_decoder_scaled_batch_into_resizable_metal_buffer_with_session( + decoders: &[&Decoder<'_>], + scale: Downscale, + output: &mut MetalBatchOutputBuffer, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_buffer_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(decoders), + op: Rgb8MetalBatchOp::Scaled(scale), + }, + MetalBufferBatchTarget::Resizable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_decoder_scaled_batch_into_resizable_metal_textures_with_session( + decoders: &[&Decoder<'_>], + scale: Downscale, + output: &mut MetalBatchTextureOutput, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_textures_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(decoders), + op: Rgb8MetalBatchOp::Scaled(scale), + }, + MetalTextureBatchTarget::Resizable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_region_scaled_batch_into_metal_buffer_with_session( + inputs: &[&[u8]], + roi: Rect, + scale: Downscale, + output: &MetalBatchOutputBuffer, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_buffer_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Bytes(inputs), + op: Rgb8MetalBatchOp::RegionScaled { roi, scale }, + }, + MetalBufferBatchTarget::Reusable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_region_scaled_batch_into_metal_textures_with_session( + inputs: &[&[u8]], + roi: Rect, + scale: Downscale, + output: &MetalBatchTextureOutput, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_textures_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Bytes(inputs), + op: Rgb8MetalBatchOp::RegionScaled { roi, scale }, + }, + MetalTextureBatchTarget::Reusable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_decoder_region_scaled_batch_into_metal_buffer_with_session( + decoders: &[&Decoder<'_>], + roi: Rect, + scale: Downscale, + output: &MetalBatchOutputBuffer, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_buffer_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(decoders), + op: Rgb8MetalBatchOp::RegionScaled { roi, scale }, + }, + MetalBufferBatchTarget::Reusable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_decoder_region_scaled_batch_into_metal_textures_with_session( + decoders: &[&Decoder<'_>], + roi: Rect, + scale: Downscale, + output: &MetalBatchTextureOutput, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_textures_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(decoders), + op: Rgb8MetalBatchOp::RegionScaled { roi, scale }, + }, + MetalTextureBatchTarget::Reusable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_decoder_region_scaled_batch_into_resizable_metal_buffer_with_session( + decoders: &[&Decoder<'_>], + roi: Rect, + scale: Downscale, + output: &mut MetalBatchOutputBuffer, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_buffer_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(decoders), + op: Rgb8MetalBatchOp::RegionScaled { roi, scale }, + }, + MetalBufferBatchTarget::Resizable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + fn decode_rgb8_decoder_region_scaled_batch_into_resizable_metal_textures_with_session( + decoders: &[&Decoder<'_>], + roi: Rect, + scale: Downscale, + output: &mut MetalBatchTextureOutput, + session: &MetalBackendSession, + ) -> Result>, Error> { + Codec::decode_rgb8_batch_into_textures_with_session( + Rgb8MetalBatchRequest { + source: Rgb8MetalBatchSource::Decoders(decoders), + op: Rgb8MetalBatchOp::RegionScaled { roi, scale }, + }, + MetalTextureBatchTarget::Resizable(output), + session, + ) + } + + #[cfg(target_os = "macos")] + use j2k_jpeg::adapter::build_fast422_packet; + use j2k_jpeg::adapter::{build_fast420_packet, build_fast444_packet}; + #[cfg(target_os = "macos")] + use j2k_jpeg::{ + encode_jpeg_baseline, JpegBackend, JpegEncodeOptions, JpegSamples, JpegSubsampling, + }; + + const BASELINE_420: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); + const BASELINE_420_RESTART: &[u8] = + include_bytes!("../fixtures/jpeg/baseline_420_restart_32x16.jpg"); + #[cfg(target_os = "macos")] + const BASELINE_422: &[u8] = include_bytes!("../fixtures/jpeg/baseline_422_16x8.jpg"); + const BASELINE_444: &[u8] = include_bytes!("../fixtures/jpeg/baseline_444_8x8.jpg"); + #[cfg(not(target_os = "macos"))] + const GRAYSCALE: &[u8] = include_bytes!("../fixtures/jpeg/grayscale_8x8.jpg"); + + #[test] + fn metal_runtime_failures_are_not_unsupported_errors() { + for err in [ + Error::MetalRuntime { + message: "runtime".to_string(), + }, + Error::MetalKernel { + message: "kernel".to_string(), + }, + Error::MetalStatePoisoned { + state: "JPEG Metal session", + }, + ] { + assert!(!err.is_unsupported(), "{err:?}"); + } + } + + #[test] + fn auto_route_prefers_cpu_host_for_nonrestart_packets() { + let decoder_420 = CpuDecoder::new(BASELINE_420).expect("420 decoder"); + let packet_420 = build_fast420_packet(BASELINE_420).expect("420 packet"); + assert_eq!( + choose_route( + &decoder_420, + BackendRequest::Auto, + PixelFormat::Rgb8, + batch::BatchOp::Full, + None, + None, + Some(&packet_420), + ), + routing::RouteDecision::CpuHost + ); + + let decoder_444 = CpuDecoder::new(BASELINE_444).expect("444 decoder"); + let packet_444 = build_fast444_packet(BASELINE_444).expect("444 packet"); + assert_eq!( + choose_route( + &decoder_444, + BackendRequest::Auto, + PixelFormat::Rgb8, + batch::BatchOp::Scaled(Downscale::Quarter), + Some(&packet_444), + None, + None, + ), + routing::RouteDecision::CpuHost + ); + } + + #[test] + fn auto_route_keeps_small_single_restart_packets_on_cpu_host() { + let decoder = CpuDecoder::new(BASELINE_420_RESTART).expect("restart decoder"); + let packet = build_fast420_packet(BASELINE_420_RESTART).expect("restart packet"); + + assert_eq!( + choose_route( + &decoder, + BackendRequest::Auto, + PixelFormat::Rgb8, + batch::BatchOp::Full, + None, + None, + Some(&packet) + ), + routing::RouteDecision::CpuHost + ); + assert_eq!( + choose_route( + &decoder, + BackendRequest::Auto, + PixelFormat::Rgb8, + batch::BatchOp::Region(Rect { + x: 0, + y: 0, + w: 16, + h: 16, + }), + None, + None, + Some(&packet), + ), + routing::RouteDecision::CpuHost + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn metal_backend_session_reuses_compiled_runtime() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + assert!(session.runtime.get().is_none()); + + let mut first = Decoder::new(BASELINE_420).expect("first decoder"); + let first_surface = first + .decode_to_device_with_session(PixelFormat::Rgb8, &session) + .expect("first session decode"); + assert_eq!( + first_surface.residency(), + SurfaceResidency::MetalResidentDecode + ); + let first_runtime = session + .runtime + .get() + .and_then(|runtime| runtime.as_ref().ok()) + .map(std::ptr::from_ref::) + .expect("session runtime after first decode"); + + let mut second = Decoder::new(BASELINE_420).expect("second decoder"); + second + .decode_to_device_with_session(PixelFormat::Rgb8, &session) + .expect("second session decode"); + let second_runtime = session + .runtime + .get() + .and_then(|runtime| runtime.as_ref().ok()) + .map(std::ptr::from_ref::) + .expect("session runtime after second decode"); + + assert_eq!(first_runtime, second_runtime); + } + + #[cfg(target_os = "macos")] + #[test] + fn jpeg_rgb8_batch_decode_uses_backend_session_runtime() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + assert!(session.runtime.get().is_none()); + + let inputs = [BASELINE_420, BASELINE_420]; + let results = decode_rgb8_batch_to_device_with_session(&inputs, &session) + .expect("session batch decode") + .expect("baseline JPEG batch should use Metal batch path"); + + assert_eq!(results.len(), 2); + assert!(session.runtime.get().is_some()); + for result in results { + let surface = result.expect("surface"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (16, 16)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn queued_jpeg_batch_decode_uses_metal_session_runtime() { + use j2k_core::DeviceSubmission as _; + + let backend_session = MetalBackendSession::system_default().expect("Metal backend session"); + assert!(backend_session.runtime.get().is_none()); + let mut session = MetalSession::with_backend_session(backend_session.clone()); + let mut ctx = j2k_core::DecoderContext::::new(); + let mut pool = ScratchPool::new(); + + let submissions = (0..2) + .map(|_| { + ::submit_tile_to_device( + &mut ctx, + &mut session, + &mut pool, + BASELINE_420, + PixelFormat::Rgb8, + BackendRequest::Metal, + ) + .expect("queued Metal tile submit") + }) + .collect::>(); + + for submission in submissions { + let surface = submission.wait().expect("queued Metal surface"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (16, 16)); + } + + assert_eq!(session.submissions().expect("session submissions"), 1); + assert!( + backend_session.runtime.get().is_some(), + "queued MetalSession batch decode should reuse its backend runtime" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn default_queued_jpeg_batch_decode_lazily_initializes_backend_session() { + use j2k_core::DeviceSubmission as _; + + let mut session = MetalSession::default(); + assert!(session + .shared + .0 + .lock() + .expect("metal session") + .backend_session + .is_none()); + let mut ctx = j2k_core::DecoderContext::::new(); + let mut pool = ScratchPool::new(); + + let submissions = (0..2) + .map(|_| { + ::submit_tile_to_device( + &mut ctx, + &mut session, + &mut pool, + BASELINE_420, + PixelFormat::Rgb8, + BackendRequest::Metal, + ) + .expect("queued Metal tile submit") + }) + .collect::>(); + + for submission in submissions { + let surface = submission.wait().expect("queued Metal surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + } + + let runtime_initialized = session + .shared + .0 + .lock() + .expect("metal session") + .backend_session + .as_ref() + .and_then(|backend| backend.runtime.get()) + .is_some(); + assert!(runtime_initialized); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_batch_decode_can_write_into_reusable_metal_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (16, 16), 2).expect("output buffer"); + let inputs = [BASELINE_420, BASELINE_420]; + let (expected, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + + let surfaces = decode_rgb8_batch_into_metal_buffer_with_session(&inputs, &output, &session) + .expect("decode into reusable output"); + + assert_eq!(surfaces.len(), 2); + assert_eq!(output.tile_capacity(), 2); + assert_eq!( + output.tile_stride_bytes(), + 16 * 16 * PixelFormat::Rgb8.bytes_per_pixel() + ); + for (index, result) in surfaces.into_iter().enumerate() { + let surface = result.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (16, 16)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_batch_resizes_reusable_metal_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let (expected, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + + let surfaces = decode_rgb8_decoder_batch_into_resizable_metal_buffer_with_session( + &decoders, + &mut output, + &session, + ) + .expect("decode cached decoder batch into resizable reusable output"); + + assert_eq!(output.dimensions(), (16, 16)); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(surfaces.len(), 2); + for (index, result) in surfaces.into_iter().enumerate() { + let surface = result.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (16, 16)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_batch_can_write_into_fixed_metal_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (16, 16), 2).expect("output buffer"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let (expected, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + + let surfaces = + decode_rgb8_decoder_batch_into_metal_buffer_with_session(&decoders, &output, &session) + .expect("decode cached decoder batch into fixed reusable output"); + + assert_eq!(surfaces.len(), 2); + assert_eq!(output.dimensions(), (16, 16)); + assert_eq!(output.tile_capacity(), 2); + for (index, result) in surfaces.into_iter().enumerate() { + let surface = result.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (16, 16)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_batch_rejects_mixed_output_dimensions_without_resizing_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_444).expect("second decoder"); + let decoders = [&first, &second]; + + let Err(err) = decode_rgb8_decoder_batch_into_resizable_metal_buffer_with_session( + &decoders, + &mut output, + &session, + ) else { + panic!("mixed output dimensions should be rejected"); + }; + + assert!(matches!(err, Error::UnsupportedMetalRequest { .. })); + assert_eq!(output.dimensions(), (1, 1)); + assert_eq!(output.tile_capacity(), 1); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_batch_rejects_mixed_sampling_without_resizing_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + let rgb = j2k_test_support::patterned_rgb8(16, 16); + let fast420 = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: 16, + height: 16, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode fast420 jpeg"); + let fast444 = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: 16, + height: 16, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr444, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode fast444 jpeg"); + let first = Decoder::new(&fast420.data).expect("first decoder"); + let second = Decoder::new(&fast444.data).expect("second decoder"); + let decoders = [&first, &second]; + + let Err(err) = decode_rgb8_decoder_batch_into_resizable_metal_buffer_with_session( + &decoders, + &mut output, + &session, + ) else { + panic!("mixed sampling should be rejected"); + }; + + assert!(matches!( + err, + Error::UnsupportedMetalRequest { reason } + if reason.contains("same fast-packet sampling family") + )); + assert_eq!(output.dimensions(), (1, 1)); + assert_eq!(output.tile_capacity(), 1); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_batch_metal_report_exposes_required_output_shape() { + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + + let full = + Codec::inspect_rgb8_decoder_batch_metal_output(&decoders, j2k_jpeg::JpegDecodeOp::Full); + let scaled = Codec::inspect_rgb8_decoder_batch_metal_output( + &decoders, + j2k_jpeg::JpegDecodeOp::Scaled(Downscale::Quarter), + ); + let roi = Rect { + x: 1, + y: 2, + w: 10, + h: 9, + }; + let region_scaled = Codec::inspect_rgb8_decoder_batch_metal_output( + &decoders, + j2k_jpeg::JpegDecodeOp::RegionScaled { + roi: j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale: Downscale::Quarter, + }, + ); + + assert!(full.eligibility.eligible); + assert_eq!(full.tile_count, 2); + assert_eq!(full.output_dimensions, Some((16, 16))); + assert_eq!(full.required_tile_capacity(), 2); + + assert!(scaled.eligibility.eligible); + assert_eq!(scaled.output_dimensions, Some((4, 4))); + + assert!(region_scaled.eligibility.eligible); + let expected = roi.scaled_covering(Downscale::Quarter); + assert_eq!( + region_scaled.output_dimensions, + Some((expected.w, expected.h)) + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_batch_metal_report_rejects_incompatible_batches_without_launch() { + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_444).expect("second decoder"); + let decoders = [&first, &second]; + + let mixed = + Codec::inspect_rgb8_decoder_batch_metal_output(&decoders, j2k_jpeg::JpegDecodeOp::Full); + let region = Codec::inspect_rgb8_decoder_batch_metal_output( + &[&first], + j2k_jpeg::JpegDecodeOp::Region(j2k_jpeg::Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }), + ); + + assert!(!mixed.eligibility.eligible); + assert_eq!(mixed.output_dimensions, None); + assert!(mixed + .eligibility + .reason + .expect("mixed rejection") + .contains("matching output dimensions")); + + assert!(!region.eligibility.eligible); + assert!(region + .eligibility + .reason + .expect("region rejection") + .contains("full, scaled, or region-scaled")); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_batch_metal_report_rejects_mixed_sampling_family() { + let rgb = j2k_test_support::patterned_rgb8(16, 16); + let fast420 = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: 16, + height: 16, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode fast420 jpeg"); + let fast444 = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: 16, + height: 16, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr444, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode fast444 jpeg"); + let first = Decoder::new(&fast420.data).expect("first decoder"); + let second = Decoder::new(&fast444.data).expect("second decoder"); + let decoders = [&first, &second]; + + let report = + Codec::inspect_rgb8_decoder_batch_metal_output(&decoders, j2k_jpeg::JpegDecodeOp::Full); + + assert!(!report.eligibility.eligible); + assert_eq!(report.output_dimensions, None); + assert!(report + .eligibility + .reason + .expect("mixed sampling rejection") + .contains("same fast-packet sampling family")); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_batch_metal_report_rejects_restart_fast422_full_tiles() { + let rgb = j2k_test_support::patterned_rgb8(64, 32); + let jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: 64, + height: 32, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr422, + restart_interval: Some(4), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode restart fast422 jpeg"); + let packet = build_fast422_packet(&jpeg.data).expect("restart fast422 packet"); + assert_ne!(packet.restart_interval_mcus, 0); + let first = Decoder::new(&jpeg.data).expect("first decoder"); + let second = Decoder::new(&jpeg.data).expect("second decoder"); + let decoders = [&first, &second]; + + let full = + Codec::inspect_rgb8_decoder_batch_metal_output(&decoders, j2k_jpeg::JpegDecodeOp::Full); + let scaled = Codec::inspect_rgb8_decoder_batch_metal_output( + &decoders, + j2k_jpeg::JpegDecodeOp::Scaled(Downscale::Half), + ); + + assert!(!full.eligibility.eligible); + assert_eq!(full.output_dimensions, None); + assert!(full + .eligibility + .reason + .expect("restart fast422 full rejection") + .contains("restart-coded full-tile 4:2:2 or 4:4:4")); + + assert!(scaled.eligibility.eligible); + assert_eq!(scaled.output_dimensions, Some((32, 16))); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_fast444_batch_decode_can_write_into_reusable_metal_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (8, 8), 2).expect("output buffer"); + let inputs = [BASELINE_444, BASELINE_444]; + let (expected, _) = CpuDecoder::new(BASELINE_444) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + + let surfaces = decode_rgb8_batch_into_metal_buffer_with_session(&inputs, &output, &session) + .expect("decode into reusable output"); + + assert_eq!(surfaces.len(), 2); + for (index, result) in surfaces.into_iter().enumerate() { + let surface = result.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (8, 8)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + fn assert_table_mixed_full_buffer_groups_resident( + subsampling: JpegSubsampling, + dimensions: (u32, u32), + first_quality: u8, + second_quality: u8, + ) { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let rgb_a = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_b = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_c = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + for (index, pixel) in rgb_b.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(43) + .wrapping_add(17); + pixel[0] ^= delta.rotate_left(1); + pixel[1] = pixel[1].wrapping_sub(delta); + pixel[2] = pixel[2].wrapping_add(delta.rotate_right(2)); + } + for (index, pixel) in rgb_c.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(47) + .wrapping_add(23); + pixel[0] = pixel[0].wrapping_add(delta.rotate_left(2)); + pixel[1] ^= delta.rotate_right(1); + pixel[2] = pixel[2].wrapping_sub(delta); + } + + let jpeg_a = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_a, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: first_quality, + subsampling, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode first table-mixed full buffer jpeg"); + let jpeg_b = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_b, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: second_quality, + subsampling, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode second table-mixed full buffer jpeg"); + let jpeg_c = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_c, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: first_quality, + subsampling, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode third table-mixed full buffer jpeg"); + + match subsampling { + JpegSubsampling::Ybr420 => { + let packet_a = build_fast420_packet(&jpeg_a.data).expect("first packet"); + let packet_b = build_fast420_packet(&jpeg_b.data).expect("second packet"); + let packet_c = build_fast420_packet(&jpeg_c.data).expect("third packet"); + assert_eq!(packet_a.y_quant, packet_c.y_quant); + assert_eq!(packet_a.y_dc_table, packet_c.y_dc_table); + assert_eq!( + packet_a.entropy_checkpoints.len(), + packet_c.entropy_checkpoints.len() + ); + assert_ne!(packet_a.y_quant, packet_b.y_quant); + } + JpegSubsampling::Ybr422 => { + let packet_a = build_fast422_packet(&jpeg_a.data).expect("first packet"); + let packet_b = build_fast422_packet(&jpeg_b.data).expect("second packet"); + let packet_c = build_fast422_packet(&jpeg_c.data).expect("third packet"); + assert_eq!(packet_a.y_quant, packet_c.y_quant); + assert_eq!(packet_a.y_dc_table, packet_c.y_dc_table); + assert_eq!( + packet_a.entropy_checkpoints.len(), + packet_c.entropy_checkpoints.len() + ); + assert_ne!(packet_a.y_quant, packet_b.y_quant); + } + JpegSubsampling::Ybr444 => { + let packet_a = build_fast444_packet(&jpeg_a.data).expect("first packet"); + let packet_b = build_fast444_packet(&jpeg_b.data).expect("second packet"); + let packet_c = build_fast444_packet(&jpeg_c.data).expect("third packet"); + assert_eq!(packet_a.y_quant, packet_c.y_quant); + assert_eq!(packet_a.y_dc_table, packet_c.y_dc_table); + assert_eq!( + packet_a.entropy_checkpoints.len(), + packet_c.entropy_checkpoints.len() + ); + assert_ne!(packet_a.y_quant, packet_b.y_quant); + } + JpegSubsampling::Gray => panic!("table-mixed buffer helper expects YCbCr sampling"), + } + + let output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, dimensions, 3).expect("output buffer"); + let inputs = [ + jpeg_a.data.as_slice(), + jpeg_b.data.as_slice(), + jpeg_c.data.as_slice(), + ]; + let expected_tiles = inputs + .iter() + .map(|input| { + CpuDecoder::new(input) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode") + .0 + }) + .collect::>(); + assert_ne!(expected_tiles[0], expected_tiles[1]); + assert_ne!(expected_tiles[0], expected_tiles[2]); + assert_ne!(expected_tiles[1], expected_tiles[2]); + + let surfaces = decode_rgb8_batch_into_metal_buffer_with_session(&inputs, &output, &session) + .expect("decode table-mixed full tiles into reusable output buffer"); + + assert_eq!(surfaces.len(), 3); + for (index, surface) in surfaces.into_iter().enumerate() { + let surface = surface.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), dimensions); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected_tiles[index].as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_table_mixed_fast420_buffer_batch_groups_resident_dispatches() { + assert_table_mixed_full_buffer_groups_resident(JpegSubsampling::Ybr420, (128, 96), 90, 72); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_table_mixed_fast422_buffer_batch_groups_resident_dispatches() { + assert_table_mixed_full_buffer_groups_resident(JpegSubsampling::Ybr422, (128, 96), 91, 73); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_table_mixed_fast444_buffer_batch_groups_resident_dispatches() { + assert_table_mixed_full_buffer_groups_resident(JpegSubsampling::Ybr444, (96, 96), 92, 74); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_scaled_batch_decode_can_write_into_reusable_metal_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let scale = Downscale::Quarter; + let output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (4, 4), 2).expect("output buffer"); + let inputs = [BASELINE_420, BASELINE_420]; + let (expected, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled decode"); + + let surfaces = decode_rgb8_scaled_batch_into_metal_buffer_with_session( + &inputs, scale, &output, &session, + ) + .expect("decode scaled into reusable output"); + + assert_eq!(surfaces.len(), 2); + for (index, result) in surfaces.into_iter().enumerate() { + let surface = result.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (4, 4)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_scaled_batch_resizes_reusable_metal_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let scale = Downscale::Quarter; + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let (expected, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled decode"); + + let surfaces = decode_rgb8_decoder_scaled_batch_into_resizable_metal_buffer_with_session( + &decoders, + scale, + &mut output, + &session, + ) + .expect("decode cached decoder scaled batch into resizable reusable output"); + + assert_eq!(output.dimensions(), (4, 4)); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(surfaces.len(), 2); + for (index, result) in surfaces.into_iter().enumerate() { + let surface = result.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (4, 4)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_scaled_batch_can_write_into_fixed_metal_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let scale = Downscale::Quarter; + let output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (4, 4), 2).expect("output buffer"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let (expected, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled decode"); + + let surfaces = decode_rgb8_decoder_scaled_batch_into_metal_buffer_with_session( + &decoders, scale, &output, &session, + ) + .expect("decode cached decoder scaled batch into fixed reusable output"); + + assert_eq!(surfaces.len(), 2); + assert_eq!(output.dimensions(), (4, 4)); + assert_eq!(output.tile_capacity(), 2); + for (index, result) in surfaces.into_iter().enumerate() { + let surface = result.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (4, 4)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_region_scaled_batch_decode_can_write_into_reusable_metal_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let output = MetalBatchOutputBuffer::new_rgb8_tiles(&session, (scaled.w, scaled.h), 2) + .expect("output buffer"); + let inputs = [BASELINE_444, BASELINE_444]; + let (expected, _) = CpuDecoder::new(BASELINE_444) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region scaled decode"); + + let surfaces = decode_rgb8_region_scaled_batch_into_metal_buffer_with_session( + &inputs, roi, scale, &output, &session, + ) + .expect("decode region scaled into reusable output"); + + assert_eq!(surfaces.len(), 2); + for (index, result) in surfaces.into_iter().enumerate() { + let surface = result.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_region_scaled_batch_decode_resizes_reusable_metal_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + let inputs = [BASELINE_444, BASELINE_444]; + let (expected, _) = CpuDecoder::new(BASELINE_444) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region scaled decode"); + + let surfaces = + Codec::decode_rgb8_region_scaled_batch_into_resizable_metal_buffer_with_session( + &inputs, + roi, + scale, + &mut output, + &session, + ) + .expect("decode region scaled into resizable reusable output"); + + assert_eq!(output.dimensions(), (scaled.w, scaled.h)); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(surfaces.len(), 2); + for (index, result) in surfaces.into_iter().enumerate() { + let surface = result.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_region_scaled_batch_resizes_reusable_metal_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 1, + y: 2, + w: 10, + h: 9, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let (expected, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region scaled decode"); + + let surfaces = + decode_rgb8_decoder_region_scaled_batch_into_resizable_metal_buffer_with_session( + &decoders, + roi, + scale, + &mut output, + &session, + ) + .expect("decode cached decoder batch into resizable reusable output"); + + assert_eq!(output.dimensions(), (scaled.w, scaled.h)); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(surfaces.len(), 2); + for (index, result) in surfaces.into_iter().enumerate() { + let surface = result.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_region_scaled_batch_can_write_into_fixed_metal_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 1, + y: 2, + w: 10, + h: 9, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let output = MetalBatchOutputBuffer::new_rgb8_tiles(&session, (scaled.w, scaled.h), 2) + .expect("output buffer"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let (expected, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region scaled decode"); + + let surfaces = decode_rgb8_decoder_region_scaled_batch_into_metal_buffer_with_session( + &decoders, roi, scale, &output, &session, + ) + .expect("decode cached decoder region-scaled batch into fixed reusable output"); + + assert_eq!(surfaces.len(), 2); + assert_eq!(output.dimensions(), (scaled.w, scaled.h)); + assert_eq!(output.tile_capacity(), 2); + for (index, result) in surfaces.into_iter().enumerate() { + let surface = result.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_restart_fast420_region_scaled_batch_decode_writes_reusable_metal_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (128, 128); + let roi = Rect { + x: 9, + y: 11, + w: 73, + h: 67, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: Some(4), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode restart-coded fast420 region-scaled jpeg"); + let packet = build_fast420_packet(&jpeg.data).expect("restart fast420 packet"); + assert_ne!(packet.restart_interval_mcus, 0); + assert!(!packet.restart_offsets.is_empty()); + + let output = MetalBatchOutputBuffer::new_rgb8_tiles(&session, (scaled.w, scaled.h), 2) + .expect("output buffer"); + let inputs = [jpeg.data.as_slice(), jpeg.data.as_slice()]; + let (expected, _) = CpuDecoder::new(&jpeg.data) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region-scaled decode"); + + let surfaces = decode_rgb8_region_scaled_batch_into_metal_buffer_with_session( + &inputs, roi, scale, &output, &session, + ) + .expect("decode restart-coded region-scaled tiles into reusable output buffer"); + + assert_eq!(surfaces.len(), 2); + for (index, surface) in surfaces.into_iter().enumerate() { + let surface = surface.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + fn assert_restart_region_scaled_buffer_batch_writes_reusable_metal_output( + subsampling: JpegSubsampling, + dimensions: (u32, u32), + ) { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 0, + y: 0, + w: dimensions.0, + h: dimensions.1, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling, + restart_interval: Some(256), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode restart-coded region-scaled jpeg"); + match subsampling { + JpegSubsampling::Ybr422 => { + let packet = build_fast422_packet(&jpeg.data).expect("restart fast422 packet"); + assert_ne!(packet.restart_interval_mcus, 0); + assert!(!packet.restart_offsets.is_empty()); + } + JpegSubsampling::Ybr444 => { + let packet = build_fast444_packet(&jpeg.data).expect("restart fast444 packet"); + assert_ne!(packet.restart_interval_mcus, 0); + assert!(!packet.restart_offsets.is_empty()); + } + _ => panic!("restart region-scaled buffer helper expects fast422 or fast444"), + } + + let output = MetalBatchOutputBuffer::new_rgb8_tiles(&session, (scaled.w, scaled.h), 2) + .expect("output buffer"); + let inputs = [jpeg.data.as_slice(), jpeg.data.as_slice()]; + let (expected, _) = CpuDecoder::new(&jpeg.data) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region-scaled decode"); + + let surfaces = decode_rgb8_region_scaled_batch_into_metal_buffer_with_session( + &inputs, roi, scale, &output, &session, + ) + .expect("decode restart-coded region-scaled tiles into reusable output buffer"); + + assert_eq!(surfaces.len(), 2); + for (index, surface) in surfaces.into_iter().enumerate() { + let surface = surface.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_restart_fast422_region_scaled_batch_decode_writes_reusable_metal_output_buffer() { + assert_restart_region_scaled_buffer_batch_writes_reusable_metal_output( + JpegSubsampling::Ybr422, + (128, 96), + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_restart_fast444_region_scaled_batch_decode_writes_reusable_metal_output_buffer() { + assert_restart_region_scaled_buffer_batch_writes_reusable_metal_output( + JpegSubsampling::Ybr444, + (96, 96), + ); + } + + #[cfg(target_os = "macos")] + fn assert_table_mixed_region_scaled_buffer_groups_resident( + subsampling: JpegSubsampling, + dimensions: (u32, u32), + first_quality: u8, + second_quality: u8, + ) { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 0, + y: 0, + w: dimensions.0, + h: dimensions.1, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let rgb_a = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_b = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_c = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + for (index, pixel) in rgb_b.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(37) + .wrapping_add(19); + pixel[0] = pixel[0].wrapping_add(delta.rotate_left(1)); + pixel[1] ^= delta; + pixel[2] = pixel[2].wrapping_sub(delta.rotate_right(2)); + } + for (index, pixel) in rgb_c.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(53) + .wrapping_add(11); + pixel[0] ^= delta.rotate_right(1); + pixel[1] = pixel[1].wrapping_sub(delta.rotate_left(2)); + pixel[2] = pixel[2].wrapping_add(delta); + } + + let jpeg_a = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_a, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: first_quality, + subsampling, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode first table-mixed region-scaled buffer jpeg"); + let jpeg_b = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_b, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: second_quality, + subsampling, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode second table-mixed region-scaled buffer jpeg"); + let jpeg_c = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_c, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: first_quality, + subsampling, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode third table-mixed region-scaled buffer jpeg"); + + match subsampling { + JpegSubsampling::Ybr420 => { + let packet_a = build_fast420_packet(&jpeg_a.data).expect("first packet"); + let packet_b = build_fast420_packet(&jpeg_b.data).expect("second packet"); + let packet_c = build_fast420_packet(&jpeg_c.data).expect("third packet"); + assert_eq!(packet_a.y_quant, packet_c.y_quant); + assert_eq!(packet_a.y_dc_table, packet_c.y_dc_table); + assert_eq!( + packet_a.entropy_checkpoints.len(), + packet_c.entropy_checkpoints.len() + ); + assert_ne!(packet_a.y_quant, packet_b.y_quant); + } + JpegSubsampling::Ybr422 => { + let packet_a = build_fast422_packet(&jpeg_a.data).expect("first packet"); + let packet_b = build_fast422_packet(&jpeg_b.data).expect("second packet"); + let packet_c = build_fast422_packet(&jpeg_c.data).expect("third packet"); + assert_eq!(packet_a.y_quant, packet_c.y_quant); + assert_eq!(packet_a.y_dc_table, packet_c.y_dc_table); + assert_eq!( + packet_a.entropy_checkpoints.len(), + packet_c.entropy_checkpoints.len() + ); + assert_ne!(packet_a.y_quant, packet_b.y_quant); + } + JpegSubsampling::Ybr444 => { + let packet_a = build_fast444_packet(&jpeg_a.data).expect("first packet"); + let packet_b = build_fast444_packet(&jpeg_b.data).expect("second packet"); + let packet_c = build_fast444_packet(&jpeg_c.data).expect("third packet"); + assert_eq!(packet_a.y_quant, packet_c.y_quant); + assert_eq!(packet_a.y_dc_table, packet_c.y_dc_table); + assert_eq!( + packet_a.entropy_checkpoints.len(), + packet_c.entropy_checkpoints.len() + ); + assert_ne!(packet_a.y_quant, packet_b.y_quant); + } + JpegSubsampling::Gray => panic!("table-mixed buffer helper expects YCbCr sampling"), + } + + let output = MetalBatchOutputBuffer::new_rgb8_tiles(&session, (scaled.w, scaled.h), 3) + .expect("output buffer"); + let inputs = [ + jpeg_a.data.as_slice(), + jpeg_b.data.as_slice(), + jpeg_c.data.as_slice(), + ]; + let expected_tiles = inputs + .iter() + .map(|input| { + CpuDecoder::new(input) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region-scaled decode") + .0 + }) + .collect::>(); + assert_ne!(expected_tiles[0], expected_tiles[1]); + assert_ne!(expected_tiles[0], expected_tiles[2]); + assert_ne!(expected_tiles[1], expected_tiles[2]); + + let surfaces = decode_rgb8_region_scaled_batch_into_metal_buffer_with_session( + &inputs, roi, scale, &output, &session, + ) + .expect("decode table-mixed region-scaled tiles into reusable output buffer"); + + assert_eq!(surfaces.len(), 3); + for (index, surface) in surfaces.into_iter().enumerate() { + let surface = surface.expect("surface"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, index * output.tile_stride_bytes()); + assert_eq!(surface.as_bytes(), expected_tiles[index].as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_table_mixed_fast420_region_scaled_buffer_batch_groups_resident_dispatches() { + assert_table_mixed_region_scaled_buffer_groups_resident( + JpegSubsampling::Ybr420, + (128, 96), + 90, + 72, + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_table_mixed_fast422_region_scaled_buffer_batch_groups_resident_dispatches() { + assert_table_mixed_region_scaled_buffer_groups_resident( + JpegSubsampling::Ybr422, + (128, 96), + 91, + 73, + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_table_mixed_fast444_region_scaled_buffer_batch_groups_resident_dispatches() { + assert_table_mixed_region_scaled_buffer_groups_resident( + JpegSubsampling::Ybr444, + (96, 96), + 92, + 74, + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_fast444_region_scaled_batch_decode_can_write_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (scaled.w, scaled.h), 2) + .expect("texture output"); + let inputs = [BASELINE_444, BASELINE_444]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_444) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region scaled decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_region_scaled_batch_into_metal_textures_with_session( + &inputs, roi, scale, &output, &session, + ) + .expect("decode region scaled into reusable textures"); + + assert_eq!(tiles.len(), 2); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(output.dimensions(), (scaled.w, scaled.h)); + assert_eq!(output.pixel_format(), PixelFormat::Rgba8); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (scaled.w, scaled.h)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn metal_batch_output_buffer_ensure_reuses_matching_allocation_and_grows_capacity() { + use metal::foreign_types::ForeignTypeRef; + + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (16, 16), 2).expect("output buffer"); + let original_buffer = output.buffer().as_ptr(); + + output + .ensure_rgb8_tiles(&session, (16, 16), 1) + .expect("ensure smaller matching output"); + assert_eq!(output.buffer().as_ptr(), original_buffer); + assert_eq!(output.dimensions(), (16, 16)); + assert_eq!(output.tile_capacity(), 2); + + output + .ensure_rgb8_tiles(&session, (16, 16), 3) + .expect("ensure larger output"); + assert_ne!(output.buffer().as_ptr(), original_buffer); + assert_eq!(output.dimensions(), (16, 16)); + assert_eq!(output.tile_capacity(), 3); + assert_eq!( + output.byte_len(), + 16 * 16 * PixelFormat::Rgb8.bytes_per_pixel() * 3 + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn metal_batch_texture_output_ensure_reuses_matching_textures_and_grows_capacity() { + use metal::foreign_types::ForeignTypeRef; + + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (16, 16), 2) + .expect("texture output"); + let original_texture = output.texture(0).expect("texture").as_ptr(); + + output + .ensure_rgba8_tiles(&session, (16, 16), 1) + .expect("ensure smaller matching texture output"); + assert_eq!( + output.texture(0).expect("texture").as_ptr(), + original_texture + ); + assert_eq!(output.dimensions(), (16, 16)); + assert_eq!(output.tile_capacity(), 2); + + output + .ensure_rgba8_tiles(&session, (16, 16), 3) + .expect("ensure larger texture output"); + assert_ne!( + output.texture(0).expect("texture").as_ptr(), + original_texture + ); + assert_eq!(output.dimensions(), (16, 16)); + assert_eq!(output.tile_capacity(), 3); + assert_eq!(output.pixel_format(), PixelFormat::Rgba8); + } + + #[cfg(target_os = "macos")] + #[test] + fn metal_batch_output_buffer_ensure_region_scaled_tiles_uses_scaled_roi_shape() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 4, + y: 4, + w: 10, + h: 10, + }; + let scaled = roi.scaled_covering(Downscale::Quarter); + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + + output + .ensure_rgb8_region_scaled_tiles(&session, roi, Downscale::Quarter, 2) + .expect("ensure region-scaled output"); + + assert_eq!(output.dimensions(), (scaled.w, scaled.h)); + assert_eq!(output.tile_capacity(), 2); + assert_eq!( + output.tile_stride_bytes(), + scaled.w as usize * scaled.h as usize * PixelFormat::Rgb8.bytes_per_pixel() + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn metal_batch_texture_output_ensure_scaled_tiles_uses_scaled_full_shape() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + + output + .ensure_rgba8_scaled_tiles(&session, (16, 16), Downscale::Quarter, 2) + .expect("ensure scaled texture output"); + + assert_eq!(output.dimensions(), (4, 4)); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(output.pixel_format(), PixelFormat::Rgba8); + } + + #[cfg(target_os = "macos")] + #[test] + fn metal_batch_outputs_can_ensure_from_resident_batch_report() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let report = Codec::inspect_rgb8_decoder_batch_metal_output( + &decoders, + j2k_jpeg::JpegDecodeOp::Scaled(Downscale::Quarter), + ); + assert!(report.eligibility.eligible); + + let mut buffer = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + let mut textures = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + + buffer + .ensure_rgb8_batch_report(&session, &report) + .expect("ensure buffer from report"); + textures + .ensure_rgba8_batch_report(&session, &report) + .expect("ensure textures from report"); + + assert_eq!(buffer.dimensions(), (4, 4)); + assert_eq!(buffer.tile_capacity(), 2); + assert_eq!( + buffer.tile_stride_bytes(), + 4 * 4 * PixelFormat::Rgb8.bytes_per_pixel() + ); + assert_eq!(textures.dimensions(), (4, 4)); + assert_eq!(textures.tile_capacity(), 2); + assert_eq!(textures.pixel_format(), PixelFormat::Rgba8); + } + + #[cfg(target_os = "macos")] + #[test] + fn metal_batch_outputs_reject_ineligible_report_without_resizing() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_444).expect("second decoder"); + let decoders = [&first, &second]; + let report = + Codec::inspect_rgb8_decoder_batch_metal_output(&decoders, j2k_jpeg::JpegDecodeOp::Full); + assert!(!report.eligibility.eligible); + + let mut buffer = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + let mut textures = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + + let buffer_err = buffer + .ensure_rgb8_batch_report(&session, &report) + .expect_err("ineligible report should reject buffer ensure"); + let texture_err = textures + .ensure_rgba8_batch_report(&session, &report) + .expect_err("ineligible report should reject texture ensure"); + + assert!(matches!( + buffer_err, + Error::UnsupportedMetalRequest { reason } + if reason.contains("matching output dimensions") + )); + assert!(matches!( + texture_err, + Error::UnsupportedMetalRequest { reason } + if reason.contains("matching output dimensions") + )); + assert_eq!(buffer.dimensions(), (1, 1)); + assert_eq!(buffer.tile_capacity(), 1); + assert_eq!(textures.dimensions(), (1, 1)); + assert_eq!(textures.tile_capacity(), 1); + } + + #[cfg(target_os = "macos")] + #[test] + fn warm_session_reuses_private_intermediate_buffers_for_reusable_output_batches() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (16, 16), 2).expect("output buffer"); + let inputs = [BASELINE_420, BASELINE_420]; + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let first = decode_rgb8_batch_into_metal_buffer_with_session(&inputs, &output, &session) + .expect("first decode"); + for surface in first { + assert_eq!( + surface.expect("surface").residency(), + SurfaceResidency::MetalResidentDecode + ); + } + let allocations_after_first = compute::jpeg_private_buffer_allocations_for_test(); + + let second = decode_rgb8_batch_into_metal_buffer_with_session(&inputs, &output, &session) + .expect("second decode"); + for surface in second { + assert_eq!( + surface.expect("surface").residency(), + SurfaceResidency::MetalResidentDecode + ); + } + + assert!( + allocations_after_first > 0, + "first batch should allocate private intermediate buffers" + ); + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + allocations_after_first, + "warm session batch should reuse private intermediate buffers" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn warm_session_reuses_shared_upload_buffers_for_reusable_output_batches() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (16, 16), 2).expect("output buffer"); + let inputs = [BASELINE_420, BASELINE_420]; + + compute::reset_jpeg_shared_buffer_allocations_for_test(); + decode_rgb8_batch_into_metal_buffer_with_session(&inputs, &output, &session) + .expect("first decode"); + let allocations_after_first = compute::jpeg_shared_buffer_allocations_for_test(); + + decode_rgb8_batch_into_metal_buffer_with_session(&inputs, &output, &session) + .expect("second decode"); + + assert!( + allocations_after_first > 0, + "first batch should allocate shared upload/status buffers" + ); + assert_eq!( + compute::jpeg_shared_buffer_allocations_for_test(), + allocations_after_first, + "warm session batch should reuse shared upload/status buffers" + ); + } + + #[cfg(target_os = "macos")] + fn patterned_index_byte(index: usize) -> u8 { + u8::try_from(index % 256).expect("modulo 256 fits in u8") + } + + #[cfg(target_os = "macos")] + fn rgb_to_rgba_opaque(rgb: &[u8]) -> Vec { + let mut rgba = Vec::with_capacity(rgb.len() / 3 * 4); + for pixel in rgb.chunks_exact(3) { + rgba.extend_from_slice(pixel); + rgba.push(u8::MAX); + } + rgba + } + + #[cfg(target_os = "macos")] + fn download_rgba8_texture( + session: &MetalBackendSession, + texture: &metal::TextureRef, + dimensions: (u32, u32), + ) -> Vec { + let row_bytes = dimensions.0 as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let byte_len = row_bytes * dimensions.1 as usize; + let buffer = session.device().new_buffer( + byte_len as u64, + metal::MTLResourceOptions::StorageModeShared, + ); + let queue = session.device().new_command_queue(); + let command_buffer = queue.new_command_buffer(); + let blit = command_buffer.new_blit_command_encoder(); + blit.copy_from_texture_to_buffer( + texture, + 0, + 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + metal::MTLSize::new(u64::from(dimensions.0), u64::from(dimensions.1), 1), + &buffer, + 0, + row_bytes as u64, + byte_len as u64, + metal::MTLBlitOption::None, + ); + blit.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + // SAFETY: Metal surface byte views are bounded by validated dimensions and formats. + unsafe { core::slice::from_raw_parts(buffer.contents().cast::(), byte_len).to_vec() } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_fast444_batch_decode_can_write_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (8, 8), 2).expect("texture output"); + let inputs = [BASELINE_444, BASELINE_444]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_444) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + + assert_eq!(tiles.len(), 2); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(output.dimensions(), (8, 8)); + assert_eq!(output.pixel_format(), PixelFormat::Rgba8); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (8, 8)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_batch_resizes_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = Codec::decode_rgb8_decoder_batch_into_resizable_metal_textures_with_session( + &decoders, + &mut output, + &session, + ) + .expect("decode cached decoder batch into resizable reusable textures"); + + assert_eq!(output.dimensions(), (16, 16)); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (16, 16)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_batch_can_write_into_fixed_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (16, 16), 2) + .expect("texture output"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_decoder_batch_into_metal_textures_with_session( + &decoders, &output, &session, + ) + .expect("decode cached decoder batch into fixed reusable textures"); + + assert_eq!(tiles.len(), 2); + assert_eq!(output.dimensions(), (16, 16)); + assert_eq!(output.tile_capacity(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (16, 16)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_batch_rejects_mixed_output_dimensions_without_resizing_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_444).expect("second decoder"); + let decoders = [&first, &second]; + + let Err(err) = Codec::decode_rgb8_decoder_batch_into_resizable_metal_textures_with_session( + &decoders, + &mut output, + &session, + ) else { + panic!("mixed output dimensions should be rejected"); + }; + + assert!(matches!(err, Error::UnsupportedMetalRequest { .. })); + assert_eq!(output.dimensions(), (1, 1)); + assert_eq!(output.tile_capacity(), 1); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_batch_rejects_mixed_sampling_without_resizing_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + let rgb = j2k_test_support::patterned_rgb8(16, 16); + let fast420 = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: 16, + height: 16, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode fast420 jpeg"); + let fast444 = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: 16, + height: 16, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr444, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode fast444 jpeg"); + let first = Decoder::new(&fast420.data).expect("first decoder"); + let second = Decoder::new(&fast444.data).expect("second decoder"); + let decoders = [&first, &second]; + + let Err(err) = Codec::decode_rgb8_decoder_batch_into_resizable_metal_textures_with_session( + &decoders, + &mut output, + &session, + ) else { + panic!("mixed sampling should be rejected"); + }; + + assert!(matches!( + err, + Error::UnsupportedMetalRequest { reason } + if reason.contains("same fast-packet sampling family") + )); + assert_eq!(output.dimensions(), (1, 1)); + assert_eq!(output.tile_capacity(), 1); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_scaled_batch_decode_can_write_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let scale = Downscale::Quarter; + let output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (4, 4), 2).expect("texture output"); + let inputs = [BASELINE_420, BASELINE_420]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_scaled_batch_into_metal_textures_with_session( + &inputs, scale, &output, &session, + ) + .expect("decode scaled into reusable textures"); + + assert_eq!(tiles.len(), 2); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(output.dimensions(), (4, 4)); + assert_eq!(output.pixel_format(), PixelFormat::Rgba8); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (4, 4)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_scaled_batch_decode_resizes_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let scale = Downscale::Quarter; + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + let inputs = [BASELINE_420, BASELINE_420]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_scaled_batch_into_resizable_metal_textures_with_session( + &inputs, + scale, + &mut output, + &session, + ) + .expect("decode scaled into resizable reusable textures"); + + assert_eq!(output.dimensions(), (4, 4)); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (4, 4)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_scaled_batch_resizes_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let scale = Downscale::Quarter; + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_decoder_scaled_batch_into_resizable_metal_textures_with_session( + &decoders, + scale, + &mut output, + &session, + ) + .expect("decode cached decoder scaled batch into resizable reusable textures"); + + assert_eq!(output.dimensions(), (4, 4)); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (4, 4)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_scaled_batch_can_write_into_fixed_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let scale = Downscale::Quarter; + let output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (4, 4), 2).expect("texture output"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode_scaled(PixelFormat::Rgb8, scale) + .expect("cpu scaled decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_decoder_scaled_batch_into_metal_textures_with_session( + &decoders, scale, &output, &session, + ) + .expect("decode cached decoder scaled batch into fixed reusable textures"); + + assert_eq!(tiles.len(), 2); + assert_eq!(output.dimensions(), (4, 4)); + assert_eq!(output.tile_capacity(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (4, 4)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_fast422_region_scaled_batch_decode_can_write_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 1, + y: 1, + w: 9, + h: 6, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (scaled.w, scaled.h), 2) + .expect("texture output"); + let inputs = [BASELINE_422, BASELINE_422]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_422) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region scaled decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_region_scaled_batch_into_metal_textures_with_session( + &inputs, roi, scale, &output, &session, + ) + .expect("decode region scaled into reusable textures"); + + assert_eq!(tiles.len(), 2); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(output.dimensions(), (scaled.w, scaled.h)); + assert_eq!(output.pixel_format(), PixelFormat::Rgba8); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (scaled.w, scaled.h)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_table_mixed_fast422_region_scaled_texture_batch_groups_resident_dispatches() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (128, 96); + let roi = Rect { + x: 0, + y: 0, + w: dimensions.0, + h: dimensions.1, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let rgb_a = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_b = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_c = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + for (index, pixel) in rgb_b.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(41) + .wrapping_add(29); + pixel[0] ^= delta.rotate_left(1); + pixel[1] = pixel[1].wrapping_add(delta); + pixel[2] = pixel[2].wrapping_sub(delta.rotate_right(2)); + } + for (index, pixel) in rgb_c.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index).wrapping_mul(59).wrapping_add(3); + pixel[0] = pixel[0].wrapping_sub(delta.rotate_left(2)); + pixel[1] ^= delta.rotate_right(1); + pixel[2] = pixel[2].wrapping_add(delta); + } + + let jpeg_a = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_a, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr422, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode first fast422 region-scaled table group jpeg"); + let jpeg_b = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_b, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 71, + subsampling: JpegSubsampling::Ybr422, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode second fast422 region-scaled table group jpeg"); + let jpeg_c = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_c, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr422, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode third fast422 region-scaled table group jpeg"); + let packet_a = build_fast422_packet(&jpeg_a.data).expect("first fast422 packet"); + let packet_b = build_fast422_packet(&jpeg_b.data).expect("second fast422 packet"); + let packet_c = build_fast422_packet(&jpeg_c.data).expect("third fast422 packet"); + assert_eq!(packet_a.y_quant, packet_c.y_quant); + assert_eq!(packet_a.cb_quant, packet_c.cb_quant); + assert_eq!(packet_a.cr_quant, packet_c.cr_quant); + assert_eq!(packet_a.y_dc_table, packet_c.y_dc_table); + assert_eq!(packet_a.y_ac_table, packet_c.y_ac_table); + assert_eq!( + packet_a.entropy_checkpoints.len(), + packet_c.entropy_checkpoints.len() + ); + assert_ne!(packet_a.y_quant, packet_b.y_quant); + + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (scaled.w, scaled.h), 3) + .expect("texture output"); + let inputs = [ + jpeg_a.data.as_slice(), + jpeg_b.data.as_slice(), + jpeg_c.data.as_slice(), + ]; + let (expected_rgb_a, _) = CpuDecoder::new(&jpeg_a.data) + .expect("first cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("first cpu region scaled decode"); + let (expected_rgb_b, _) = CpuDecoder::new(&jpeg_b.data) + .expect("second cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("second cpu region scaled decode"); + let (expected_rgb_c, _) = CpuDecoder::new(&jpeg_c.data) + .expect("third cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("third cpu region scaled decode"); + let expected_tiles = [ + rgb_to_rgba_opaque(&expected_rgb_a), + rgb_to_rgba_opaque(&expected_rgb_b), + rgb_to_rgba_opaque(&expected_rgb_c), + ]; + assert_ne!(expected_tiles[0], expected_tiles[1]); + assert_ne!(expected_tiles[0], expected_tiles[2]); + assert_ne!(expected_tiles[1], expected_tiles[2]); + + let tiles = decode_rgb8_region_scaled_batch_into_metal_textures_with_session( + &inputs, roi, scale, &output, &session, + ) + .expect("decode table-mixed fast422 region-scaled tiles into reusable textures"); + + assert_eq!(tiles.len(), 3); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (scaled.w, scaled.h)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + let actual_rgba = download_rgba8_texture(&session, tile.texture(), tile.dimensions()); + assert_eq!(actual_rgba.as_slice(), expected_tiles[index].as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_table_mixed_fast444_region_scaled_texture_batch_groups_resident_dispatches() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (96, 96); + let roi = Rect { + x: 0, + y: 0, + w: dimensions.0, + h: dimensions.1, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let rgb_a = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_b = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_c = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + for (index, pixel) in rgb_b.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(61) + .wrapping_add(13); + pixel[0] = pixel[0].wrapping_add(delta); + pixel[1] ^= delta.rotate_left(1); + pixel[2] = pixel[2].wrapping_sub(delta.rotate_right(2)); + } + for (index, pixel) in rgb_c.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(67) + .wrapping_add(31); + pixel[0] = pixel[0].wrapping_sub(delta.rotate_left(2)); + pixel[1] = pixel[1].wrapping_add(delta.rotate_right(1)); + pixel[2] ^= delta; + } + + let jpeg_a = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_a, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 91, + subsampling: JpegSubsampling::Ybr444, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode first fast444 region-scaled table group jpeg"); + let jpeg_b = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_b, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 70, + subsampling: JpegSubsampling::Ybr444, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode second fast444 region-scaled table group jpeg"); + let jpeg_c = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_c, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 91, + subsampling: JpegSubsampling::Ybr444, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode third fast444 region-scaled table group jpeg"); + let packet_a = build_fast444_packet(&jpeg_a.data).expect("first fast444 packet"); + let packet_b = build_fast444_packet(&jpeg_b.data).expect("second fast444 packet"); + let packet_c = build_fast444_packet(&jpeg_c.data).expect("third fast444 packet"); + assert_eq!(packet_a.y_quant, packet_c.y_quant); + assert_eq!(packet_a.cb_quant, packet_c.cb_quant); + assert_eq!(packet_a.cr_quant, packet_c.cr_quant); + assert_eq!(packet_a.y_dc_table, packet_c.y_dc_table); + assert_eq!(packet_a.y_ac_table, packet_c.y_ac_table); + assert_eq!( + packet_a.entropy_checkpoints.len(), + packet_c.entropy_checkpoints.len() + ); + assert_ne!(packet_a.y_quant, packet_b.y_quant); + + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (scaled.w, scaled.h), 3) + .expect("texture output"); + let inputs = [ + jpeg_a.data.as_slice(), + jpeg_b.data.as_slice(), + jpeg_c.data.as_slice(), + ]; + let (expected_rgb_a, _) = CpuDecoder::new(&jpeg_a.data) + .expect("first cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("first cpu region scaled decode"); + let (expected_rgb_b, _) = CpuDecoder::new(&jpeg_b.data) + .expect("second cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("second cpu region scaled decode"); + let (expected_rgb_c, _) = CpuDecoder::new(&jpeg_c.data) + .expect("third cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("third cpu region scaled decode"); + let expected_tiles = [ + rgb_to_rgba_opaque(&expected_rgb_a), + rgb_to_rgba_opaque(&expected_rgb_b), + rgb_to_rgba_opaque(&expected_rgb_c), + ]; + assert_ne!(expected_tiles[0], expected_tiles[1]); + assert_ne!(expected_tiles[0], expected_tiles[2]); + assert_ne!(expected_tiles[1], expected_tiles[2]); + + let tiles = decode_rgb8_region_scaled_batch_into_metal_textures_with_session( + &inputs, roi, scale, &output, &session, + ) + .expect("decode table-mixed fast444 region-scaled tiles into reusable textures"); + + assert_eq!(tiles.len(), 3); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (scaled.w, scaled.h)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + let actual_rgba = download_rgba8_texture(&session, tile.texture(), tile.dimensions()); + assert_eq!(actual_rgba.as_slice(), expected_tiles[index].as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_fast420_region_scaled_batch_decode_can_write_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 1, + y: 2, + w: 10, + h: 9, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (scaled.w, scaled.h), 2) + .expect("texture output"); + let inputs = [BASELINE_420, BASELINE_420]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region scaled decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_region_scaled_batch_into_metal_textures_with_session( + &inputs, roi, scale, &output, &session, + ) + .expect("decode region scaled into reusable textures"); + + assert_eq!(tiles.len(), 2); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(output.dimensions(), (scaled.w, scaled.h)); + assert_eq!(output.pixel_format(), PixelFormat::Rgba8); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (scaled.w, scaled.h)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_region_scaled_batch_resizes_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 1, + y: 2, + w: 10, + h: 9, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region scaled decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = + decode_rgb8_decoder_region_scaled_batch_into_resizable_metal_textures_with_session( + &decoders, + roi, + scale, + &mut output, + &session, + ) + .expect("decode cached decoder batch into resizable reusable textures"); + + assert_eq!(output.dimensions(), (scaled.w, scaled.h)); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (scaled.w, scaled.h)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_decoder_region_scaled_batch_can_write_into_fixed_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 1, + y: 2, + w: 10, + h: 9, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (scaled.w, scaled.h), 2) + .expect("texture output"); + let first = Decoder::new(BASELINE_420).expect("first decoder"); + let second = Decoder::new(BASELINE_420).expect("second decoder"); + let decoders = [&first, &second]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region scaled decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_decoder_region_scaled_batch_into_metal_textures_with_session( + &decoders, roi, scale, &output, &session, + ) + .expect("decode cached decoder region-scaled batch into fixed reusable textures"); + + assert_eq!(tiles.len(), 2); + assert_eq!(output.dimensions(), (scaled.w, scaled.h)); + assert_eq!(output.tile_capacity(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (scaled.w, scaled.h)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_restart_fast420_region_scaled_batch_decode_writes_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (128, 128); + let roi = Rect { + x: 9, + y: 11, + w: 73, + h: 67, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: Some(4), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode restart-coded fast420 region-scaled texture jpeg"); + let packet = build_fast420_packet(&jpeg.data).expect("restart fast420 packet"); + assert_ne!(packet.restart_interval_mcus, 0); + assert!(!packet.restart_offsets.is_empty()); + + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (scaled.w, scaled.h), 2) + .expect("texture output"); + let inputs = [jpeg.data.as_slice(), jpeg.data.as_slice()]; + let (expected_rgb, _) = CpuDecoder::new(&jpeg.data) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region-scaled decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_region_scaled_batch_into_metal_textures_with_session( + &inputs, roi, scale, &output, &session, + ) + .expect("decode restart-coded region-scaled tiles into reusable textures"); + + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (scaled.w, scaled.h)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + fn assert_restart_region_scaled_texture_batch_writes_reusable_metal_output( + subsampling: JpegSubsampling, + dimensions: (u32, u32), + ) { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let roi = Rect { + x: 0, + y: 0, + w: dimensions.0, + h: dimensions.1, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling, + restart_interval: Some(256), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode restart-coded region-scaled texture jpeg"); + match subsampling { + JpegSubsampling::Ybr422 => { + let packet = build_fast422_packet(&jpeg.data).expect("restart fast422 packet"); + assert_ne!(packet.restart_interval_mcus, 0); + assert!(!packet.restart_offsets.is_empty()); + } + JpegSubsampling::Ybr444 => { + let packet = build_fast444_packet(&jpeg.data).expect("restart fast444 packet"); + assert_ne!(packet.restart_interval_mcus, 0); + assert!(!packet.restart_offsets.is_empty()); + } + _ => panic!("restart region-scaled texture helper expects fast422 or fast444"), + } + + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (scaled.w, scaled.h), 2) + .expect("texture output"); + let inputs = [jpeg.data.as_slice(), jpeg.data.as_slice()]; + let (expected_rgb, _) = CpuDecoder::new(&jpeg.data) + .expect("cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("cpu region-scaled decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_region_scaled_batch_into_metal_textures_with_session( + &inputs, roi, scale, &output, &session, + ) + .expect("decode restart-coded region-scaled tiles into reusable textures"); + + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (scaled.w, scaled.h)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_restart_fast422_region_scaled_batch_decode_writes_reusable_metal_textures() { + assert_restart_region_scaled_texture_batch_writes_reusable_metal_output( + JpegSubsampling::Ybr422, + (128, 96), + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_restart_fast444_region_scaled_batch_decode_writes_reusable_metal_textures() { + assert_restart_region_scaled_texture_batch_writes_reusable_metal_output( + JpegSubsampling::Ybr444, + (96, 96), + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_table_mixed_fast420_region_scaled_texture_batch_groups_resident_dispatches() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (128, 128); + let roi = Rect { + x: 9, + y: 11, + w: 77, + h: 65, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let rgb_a = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_b = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_c = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + for (index, pixel) in rgb_b.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(43) + .wrapping_add(19); + pixel[0] = pixel[0].wrapping_add(delta.rotate_left(1)); + pixel[1] = pixel[1].wrapping_sub(delta); + pixel[2] ^= delta.rotate_right(2); + } + for (index, pixel) in rgb_c.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(47) + .wrapping_add(23); + pixel[0] ^= delta.rotate_left(2); + pixel[1] = pixel[1].wrapping_add(delta.rotate_right(1)); + pixel[2] = pixel[2].wrapping_sub(delta); + } + + let jpeg_a = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_a, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode first fast420 region-scaled table group jpeg"); + let jpeg_b = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_b, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 72, + subsampling: JpegSubsampling::Ybr420, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode second fast420 region-scaled table group jpeg"); + let jpeg_c = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_c, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode third fast420 region-scaled table group jpeg"); + let packet_a = build_fast420_packet(&jpeg_a.data).expect("first fast420 packet"); + let packet_b = build_fast420_packet(&jpeg_b.data).expect("second fast420 packet"); + let packet_c = build_fast420_packet(&jpeg_c.data).expect("third fast420 packet"); + assert_eq!(packet_a.y_quant, packet_c.y_quant); + assert_eq!(packet_a.cb_quant, packet_c.cb_quant); + assert_eq!(packet_a.cr_quant, packet_c.cr_quant); + assert_eq!(packet_a.y_dc_table, packet_c.y_dc_table); + assert_eq!(packet_a.y_ac_table, packet_c.y_ac_table); + assert_eq!( + packet_a.entropy_checkpoints.len(), + packet_c.entropy_checkpoints.len() + ); + assert_ne!(packet_a.y_quant, packet_b.y_quant); + + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (scaled.w, scaled.h), 3) + .expect("texture output"); + let inputs = [ + jpeg_a.data.as_slice(), + jpeg_b.data.as_slice(), + jpeg_c.data.as_slice(), + ]; + let (expected_rgb_a, _) = CpuDecoder::new(&jpeg_a.data) + .expect("first cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("first cpu region scaled decode"); + let (expected_rgb_b, _) = CpuDecoder::new(&jpeg_b.data) + .expect("second cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("second cpu region scaled decode"); + let (expected_rgb_c, _) = CpuDecoder::new(&jpeg_c.data) + .expect("third cpu decoder") + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + scale, + ) + .expect("third cpu region scaled decode"); + let expected_tiles = [ + rgb_to_rgba_opaque(&expected_rgb_a), + rgb_to_rgba_opaque(&expected_rgb_b), + rgb_to_rgba_opaque(&expected_rgb_c), + ]; + assert_ne!(expected_tiles[0], expected_tiles[1]); + assert_ne!(expected_tiles[0], expected_tiles[2]); + assert_ne!(expected_tiles[1], expected_tiles[2]); + + let tiles = decode_rgb8_region_scaled_batch_into_metal_textures_with_session( + &inputs, roi, scale, &output, &session, + ) + .expect("decode table-mixed fast420 region-scaled tiles into reusable textures"); + + assert_eq!(tiles.len(), 3); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (scaled.w, scaled.h)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + let actual_rgba = download_rgba8_texture(&session, tile.texture(), tile.dimensions()); + assert_eq!(actual_rgba.as_slice(), expected_tiles[index].as_slice()); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_fast420_batch_decode_can_write_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (16, 16), 2) + .expect("texture output"); + let inputs = [BASELINE_420, BASELINE_420]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + + assert_eq!(tiles.len(), 2); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(output.dimensions(), (16, 16)); + assert_eq!(output.pixel_format(), PixelFormat::Rgba8); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (16, 16)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_fast422_batch_decode_can_write_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (16, 8), 2).expect("texture output"); + let inputs = [BASELINE_422, BASELINE_422]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_422) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + + assert_eq!(tiles.len(), 2); + assert_eq!(output.tile_capacity(), 2); + assert_eq!(output.dimensions(), (16, 8)); + assert_eq!(output.pixel_format(), PixelFormat::Rgba8); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (16, 8)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_texture_batch_decode_avoids_private_rgba_staging_buffers() { + let cases = [ + (BASELINE_420, (16, 16), 0), + (BASELINE_422, (16, 8), 0), + (BASELINE_444, (8, 8), 0), + ]; + + for (input, dimensions, expected_private_allocations) in cases { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, 2) + .expect("texture output"); + let inputs = [input, input]; + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = + decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + assert_eq!(tiles.len(), 2); + for tile in tiles { + assert_eq!( + tile.expect("texture tile").pixel_format(), + PixelFormat::Rgba8 + ); + } + + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + expected_private_allocations, + "texture batch decode should not allocate a private RGBA staging buffer for {dimensions:?}" + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_fast444_texture_batch_decode_fuses_directly_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (8, 8), 2).expect("texture output"); + let inputs = [BASELINE_444, BASELINE_444]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_444) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (8, 8)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "fused 4:4:4 texture batch decode should not allocate private Y/Cb/Cr staging planes" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_table_mixed_fast444_texture_batch_groups_resident_dispatches() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (64, 64); + let rgb_a = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_b = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_c = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + for (index, pixel) in rgb_b.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index).wrapping_mul(31).wrapping_add(5); + pixel[0] = pixel[0].wrapping_sub(delta); + pixel[1] = pixel[1].wrapping_add(delta.rotate_left(1)); + pixel[2] ^= delta.rotate_right(2); + } + for (index, pixel) in rgb_c.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(37) + .wrapping_add(17); + pixel[0] ^= delta.rotate_left(3); + pixel[1] = pixel[1].wrapping_sub(delta.rotate_right(1)); + pixel[2] = pixel[2].wrapping_add(delta); + } + + let jpeg_a = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_a, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 92, + subsampling: JpegSubsampling::Ybr444, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode first fast444 table group jpeg"); + let jpeg_b = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_b, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 71, + subsampling: JpegSubsampling::Ybr444, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode second fast444 table group jpeg"); + let jpeg_c = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_c, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 92, + subsampling: JpegSubsampling::Ybr444, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode third fast444 table group jpeg"); + let packet_a = build_fast444_packet(&jpeg_a.data).expect("first fast444 packet"); + let packet_b = build_fast444_packet(&jpeg_b.data).expect("second fast444 packet"); + let packet_c = build_fast444_packet(&jpeg_c.data).expect("third fast444 packet"); + assert_eq!(packet_a.y_quant, packet_c.y_quant); + assert_eq!(packet_a.cb_quant, packet_c.cb_quant); + assert_eq!(packet_a.cr_quant, packet_c.cr_quant); + assert_eq!(packet_a.y_dc_table, packet_c.y_dc_table); + assert_eq!(packet_a.y_ac_table, packet_c.y_ac_table); + assert_eq!( + packet_a.entropy_checkpoints.len(), + packet_c.entropy_checkpoints.len() + ); + assert_ne!(packet_a.y_quant, packet_b.y_quant); + + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, 3) + .expect("texture output"); + let inputs = [ + jpeg_a.data.as_slice(), + jpeg_b.data.as_slice(), + jpeg_c.data.as_slice(), + ]; + let (expected_rgb_a, _) = CpuDecoder::new(&jpeg_a.data) + .expect("first cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("first cpu decode"); + let (expected_rgb_b, _) = CpuDecoder::new(&jpeg_b.data) + .expect("second cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("second cpu decode"); + let (expected_rgb_c, _) = CpuDecoder::new(&jpeg_c.data) + .expect("third cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("third cpu decode"); + let expected_tiles = [ + rgb_to_rgba_opaque(&expected_rgb_a), + rgb_to_rgba_opaque(&expected_rgb_b), + rgb_to_rgba_opaque(&expected_rgb_c), + ]; + assert_ne!(expected_tiles[0], expected_tiles[1]); + assert_ne!(expected_tiles[0], expected_tiles[2]); + assert_ne!(expected_tiles[1], expected_tiles[2]); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode table-mixed fast444 tiles into reusable textures"); + + assert_eq!(tiles.len(), 3); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), dimensions); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + let actual_rgba = download_rgba8_texture(&session, tile.texture(), tile.dimensions()); + assert_eq!(actual_rgba.as_slice(), expected_tiles[index].as_slice()); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "table-mixed resident 4:4:4 texture dispatches should not allocate private Y/Cb/Cr staging planes" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_fast422_texture_batch_decode_fuses_directly_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (16, 8), 2).expect("texture output"); + let inputs = [BASELINE_422, BASELINE_422]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_422) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (16, 8)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "fused 4:2:2 texture batch decode should not allocate private Y/Cb/Cr staging planes" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_wide_fast422_texture_batch_decode_fuses_directly_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (48, 16); + let rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 92, + subsampling: JpegSubsampling::Ybr422, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode 4:2:2 source jpeg"); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, 2) + .expect("texture output"); + let inputs = [jpeg.data.as_slice(), jpeg.data.as_slice()]; + let (expected_rgb, _) = CpuDecoder::new(&jpeg.data) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), dimensions); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "wide fused 4:2:2 texture batch decode should not allocate private Y/Cb/Cr staging planes" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_table_mixed_fast422_texture_batch_groups_resident_dispatches() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (96, 48); + let rgb_a = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_b = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_c = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + for (index, pixel) in rgb_b.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(23) + .wrapping_add(11); + pixel[0] = pixel[0].wrapping_add(delta.rotate_left(1)); + pixel[1] ^= delta; + pixel[2] = pixel[2].wrapping_sub(delta.rotate_right(2)); + } + for (index, pixel) in rgb_c.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(19) + .wrapping_add(53); + pixel[0] ^= delta.rotate_left(2); + pixel[1] = pixel[1].wrapping_sub(delta); + pixel[2] = pixel[2].wrapping_add(delta.rotate_right(1)); + } + + let jpeg_a = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_a, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 91, + subsampling: JpegSubsampling::Ybr422, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode first fast422 table group jpeg"); + let jpeg_b = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_b, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 73, + subsampling: JpegSubsampling::Ybr422, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode second fast422 table group jpeg"); + let jpeg_c = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_c, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 91, + subsampling: JpegSubsampling::Ybr422, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode third fast422 table group jpeg"); + let packet_a = build_fast422_packet(&jpeg_a.data).expect("first fast422 packet"); + let packet_b = build_fast422_packet(&jpeg_b.data).expect("second fast422 packet"); + let packet_c = build_fast422_packet(&jpeg_c.data).expect("third fast422 packet"); + assert_eq!(packet_a.y_quant, packet_c.y_quant); + assert_eq!(packet_a.cb_quant, packet_c.cb_quant); + assert_eq!(packet_a.cr_quant, packet_c.cr_quant); + assert_eq!(packet_a.y_dc_table, packet_c.y_dc_table); + assert_eq!(packet_a.y_ac_table, packet_c.y_ac_table); + assert_eq!( + packet_a.entropy_checkpoints.len(), + packet_c.entropy_checkpoints.len() + ); + assert_ne!(packet_a.y_quant, packet_b.y_quant); + + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, 3) + .expect("texture output"); + let inputs = [ + jpeg_a.data.as_slice(), + jpeg_b.data.as_slice(), + jpeg_c.data.as_slice(), + ]; + let (expected_rgb_a, _) = CpuDecoder::new(&jpeg_a.data) + .expect("first cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("first cpu decode"); + let (expected_rgb_b, _) = CpuDecoder::new(&jpeg_b.data) + .expect("second cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("second cpu decode"); + let (expected_rgb_c, _) = CpuDecoder::new(&jpeg_c.data) + .expect("third cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("third cpu decode"); + let expected_tiles = [ + rgb_to_rgba_opaque(&expected_rgb_a), + rgb_to_rgba_opaque(&expected_rgb_b), + rgb_to_rgba_opaque(&expected_rgb_c), + ]; + assert_ne!(expected_tiles[0], expected_tiles[1]); + assert_ne!(expected_tiles[0], expected_tiles[2]); + assert_ne!(expected_tiles[1], expected_tiles[2]); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode table-mixed fast422 tiles into reusable textures"); + + assert_eq!(tiles.len(), 3); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), dimensions); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + let actual_rgba = download_rgba8_texture(&session, tile.texture(), tile.dimensions()); + assert_eq!(actual_rgba.as_slice(), expected_tiles[index].as_slice()); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "table-mixed resident 4:2:2 texture dispatches should not allocate private Y/Cb/Cr staging planes" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_fast420_texture_batch_decode_fuses_directly_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, (16, 16), 2) + .expect("texture output"); + let inputs = [BASELINE_420, BASELINE_420]; + let (expected_rgb, _) = CpuDecoder::new(BASELINE_420) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), (16, 16)); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "fused 4:2:0 texture batch decode should not allocate private Y/Cb/Cr staging planes" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_wide_row_fast420_texture_batch_decode_fuses_directly_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (32, 16); + let rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 92, + subsampling: JpegSubsampling::Ybr420, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode 4:2:0 source jpeg"); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, 2) + .expect("texture output"); + let inputs = [jpeg.data.as_slice(), jpeg.data.as_slice()]; + let (expected_rgb, _) = CpuDecoder::new(&jpeg.data) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), dimensions); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "wide-row fused 4:2:0 texture batch decode should not allocate private Y/Cb/Cr staging planes" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_multi_row_fast420_texture_batch_decode_fuses_directly_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (16, 32); + let rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 92, + subsampling: JpegSubsampling::Ybr420, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode 4:2:0 source jpeg"); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, 2) + .expect("texture output"); + let inputs = [jpeg.data.as_slice(), jpeg.data.as_slice()]; + let (expected_rgb, _) = CpuDecoder::new(&jpeg.data) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), dimensions); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "multi-row fused 4:2:0 texture batch decode should not allocate private Y/Cb/Cr staging planes" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_multi_axis_fast420_texture_batch_decode_fuses_directly_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + for dimensions in [(32, 32), (48, 48)] { + let rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 92, + subsampling: JpegSubsampling::Ybr420, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode 4:2:0 source jpeg"); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, 2) + .expect("texture output"); + let inputs = [jpeg.data.as_slice(), jpeg.data.as_slice()]; + let (expected_rgb, _) = CpuDecoder::new(&jpeg.data) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = + decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), dimensions); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "multi-axis fused 4:2:0 texture batch decode should not allocate private Y/Cb/Cr staging planes for {dimensions:?}" + ); + } + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_chunked_multi_axis_fast420_texture_batch_decode_fuses_directly_into_reusable_metal_textures( + ) { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (736, 720); + let rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode chunked 4:2:0 source jpeg"); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, 2) + .expect("texture output"); + let inputs = [jpeg.data.as_slice(), jpeg.data.as_slice()]; + let (expected_rgb, _) = CpuDecoder::new(&jpeg.data) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), dimensions); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "chunked multi-axis fused 4:2:0 texture batch decode should not allocate private Y/Cb/Cr staging planes" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_restart_fast420_texture_batch_decode_fuses_directly_into_reusable_metal_textures() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (48, 48); + let rgb = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: Some(2), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode restart 4:2:0 source jpeg"); + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, 2) + .expect("texture output"); + let inputs = [jpeg.data.as_slice(), jpeg.data.as_slice()]; + let (expected_rgb, _) = CpuDecoder::new(&jpeg.data) + .expect("cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("cpu decode"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode into reusable textures"); + + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), dimensions); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "restart fused 4:2:0 texture batch decode should not allocate private Y/Cb/Cr staging planes" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_distinct_restart_fast420_texture_batch_decode_fuses_directly_into_reusable_metal_textures( + ) { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (128, 128); + let rgb_a = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_b = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + for (index, pixel) in rgb_b.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(17) + .wrapping_add(31); + pixel[0] = pixel[0].wrapping_add(delta); + pixel[1] = pixel[1].wrapping_sub(delta.rotate_left(1)); + pixel[2] ^= delta.rotate_right(1); + } + assert_ne!(rgb_a, rgb_b); + + let jpeg_a = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_a, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: Some(4), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode first restart 4:2:0 source jpeg"); + let jpeg_b = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_b, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: Some(4), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode second restart 4:2:0 source jpeg"); + assert_ne!(jpeg_a.data, jpeg_b.data); + + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, 2) + .expect("texture output"); + let inputs = [jpeg_a.data.as_slice(), jpeg_b.data.as_slice()]; + let (expected_rgb_a, _) = CpuDecoder::new(&jpeg_a.data) + .expect("first cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("first cpu decode"); + let (expected_rgb_b, _) = CpuDecoder::new(&jpeg_b.data) + .expect("second cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("second cpu decode"); + let expected_tiles = [ + rgb_to_rgba_opaque(&expected_rgb_a), + rgb_to_rgba_opaque(&expected_rgb_b), + ]; + assert_ne!(expected_tiles[0], expected_tiles[1]); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode distinct restart tiles into reusable textures"); + + assert_eq!(tiles.len(), 2); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), dimensions); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + let actual_rgba = download_rgba8_texture(&session, tile.texture(), tile.dimensions()); + assert_eq!(actual_rgba.as_slice(), expected_tiles[index].as_slice()); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "distinct restart fused 4:2:0 texture batch decode should not allocate private Y/Cb/Cr staging planes" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rgb8_table_mixed_restart_fast420_texture_batch_groups_resident_dispatches() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let dimensions = (128, 128); + let rgb_a = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_b = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + let mut rgb_c = j2k_test_support::patterned_rgb8(dimensions.0, dimensions.1); + for (index, pixel) in rgb_b.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index).wrapping_mul(29).wrapping_add(7); + pixel[0] ^= delta; + pixel[1] = pixel[1].wrapping_add(delta.rotate_left(2)); + pixel[2] = pixel[2].wrapping_sub(delta.rotate_right(2)); + } + for (index, pixel) in rgb_c.chunks_exact_mut(3).enumerate() { + let delta = patterned_index_byte(index) + .wrapping_mul(13) + .wrapping_add(41); + pixel[0] = pixel[0].wrapping_sub(delta.rotate_left(1)); + pixel[1] ^= delta.rotate_right(3); + pixel[2] = pixel[2].wrapping_add(delta); + } + + let jpeg_a = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_a, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: Some(4), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode first table group jpeg"); + let jpeg_b = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_b, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 74, + subsampling: JpegSubsampling::Ybr420, + restart_interval: Some(4), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode second table group jpeg"); + let jpeg_c = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb_c, + width: dimensions.0, + height: dimensions.1, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Ybr420, + restart_interval: Some(4), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode third table group jpeg"); + let packet_a = build_fast420_packet(&jpeg_a.data).expect("first fast420 packet"); + let packet_b = build_fast420_packet(&jpeg_b.data).expect("second fast420 packet"); + let packet_c = build_fast420_packet(&jpeg_c.data).expect("third fast420 packet"); + assert_eq!(packet_a.y_quant, packet_c.y_quant); + assert_eq!(packet_a.cb_quant, packet_c.cb_quant); + assert_eq!(packet_a.cr_quant, packet_c.cr_quant); + assert_eq!(packet_a.y_dc_table, packet_c.y_dc_table); + assert_eq!(packet_a.y_ac_table, packet_c.y_ac_table); + assert_eq!( + packet_a.entropy_checkpoints.len(), + packet_c.entropy_checkpoints.len() + ); + assert_ne!(packet_a.y_quant, packet_b.y_quant); + + let output = MetalBatchTextureOutput::new_rgba8_tiles(&session, dimensions, 3) + .expect("texture output"); + let inputs = [ + jpeg_a.data.as_slice(), + jpeg_b.data.as_slice(), + jpeg_c.data.as_slice(), + ]; + let (expected_rgb_a, _) = CpuDecoder::new(&jpeg_a.data) + .expect("first cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("first cpu decode"); + let (expected_rgb_b, _) = CpuDecoder::new(&jpeg_b.data) + .expect("second cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("second cpu decode"); + let (expected_rgb_c, _) = CpuDecoder::new(&jpeg_c.data) + .expect("third cpu decoder") + .decode(PixelFormat::Rgb8) + .expect("third cpu decode"); + let expected_tiles = [ + rgb_to_rgba_opaque(&expected_rgb_a), + rgb_to_rgba_opaque(&expected_rgb_b), + rgb_to_rgba_opaque(&expected_rgb_c), + ]; + assert_ne!(expected_tiles[0], expected_tiles[1]); + assert_ne!(expected_tiles[0], expected_tiles[2]); + assert_ne!(expected_tiles[1], expected_tiles[2]); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let tiles = decode_rgb8_batch_into_metal_textures_with_session(&inputs, &output, &session) + .expect("decode table-mixed restart tiles into reusable textures"); + + assert_eq!(tiles.len(), 3); + for (index, tile) in tiles.into_iter().enumerate() { + let tile = tile.expect("texture tile"); + assert_eq!(tile.dimensions(), dimensions); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(index).expect("output texture") + )); + let actual_rgba = download_rgba8_texture(&session, tile.texture(), tile.dimensions()); + assert_eq!(actual_rgba.as_slice(), expected_tiles[index].as_slice()); + } + assert_eq!( + compute::jpeg_private_buffer_allocations_for_test(), + 0, + "table-mixed resident 4:2:0 texture dispatches should not allocate private Y/Cb/Cr staging planes" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn jpeg_device_decode_uses_private_internal_planes() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + + compute::reset_jpeg_private_buffer_allocations_for_test(); + let surface = decoder + .decode_to_device_with_session(PixelFormat::Rgb8, &session) + .expect("resident JPEG Metal decode"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert!( + compute::jpeg_private_buffer_allocations_for_test() > 0, + "resident JPEG Metal decode should use Private internal planes" + ); + let _ = surface.as_bytes(); + } + + #[cfg(target_os = "macos")] + #[test] + fn jpeg_private_rgb8_tile_uses_private_output_buffer() { + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + + let tile = decoder + .decode_private_rgb8_tile_with_session(&session) + .expect("resident private JPEG Metal decode"); + + assert_eq!(tile.dimensions, (16, 16)); + assert_eq!(tile.pixel_format, PixelFormat::Rgb8); + assert_eq!(tile.pitch_bytes, 16 * PixelFormat::Rgb8.bytes_per_pixel()); + assert_eq!(tile.byte_offset, 0); + assert_eq!(tile.buffer.storage_mode(), metal::MTLStorageMode::Private); + assert!(tile.status_buffer.length() > 0); + } + + #[cfg(target_os = "macos")] + #[test] + fn jpeg_gray_region_decode_uses_private_internal_planes() { + let roi = Rect { + x: 4, + y: 4, + w: 8, + h: 8, + }; + let mut expected_decoder = Decoder::new(BASELINE_420).expect("expected decoder"); + let mut expected = vec![0; roi.w as usize * roi.h as usize]; + expected_decoder + .decode_region_into( + &mut CpuScratchPool::new(), + &mut expected, + roi.w as usize, + PixelFormat::Gray8, + roi, + ) + .expect("expected CPU region decode"); + + let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); + compute::reset_jpeg_private_buffer_allocations_for_test(); + let surface = decoder + .decode_region_to_device(PixelFormat::Gray8, roi, BackendRequest::Metal) + .expect("resident JPEG Metal region decode"); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert!( + compute::jpeg_private_buffer_allocations_for_test() >= 3, + "resident Gray8 region decode should keep decoded Y/Cb/Cr planes Private" + ); + assert_eq!(surface.as_bytes(), expected.as_slice()); + } + + #[cfg(target_os = "macos")] + #[test] + fn uploaded_metal_surface_is_marked_cpu_staged() { + let surface = upload_surface( + vec![1, 2, 3], + (1, 1), + PixelFormat::Rgb8, + BackendRequest::Metal, + ) + .expect("CPU staged Metal upload"); + + assert_eq!(surface.residency(), SurfaceResidency::CpuStagedMetalUpload); + } + + #[test] + fn auto_route_prefers_cpu_host_for_region_scaled_even_with_restart_packets() { + let decoder = CpuDecoder::new(BASELINE_420_RESTART).expect("restart decoder"); + let packet = build_fast420_packet(BASELINE_420_RESTART).expect("restart packet"); + + assert_eq!( + choose_route( + &decoder, + BackendRequest::Auto, + PixelFormat::Rgb8, + batch::BatchOp::RegionScaled { + roi: Rect { + x: 0, + y: 0, + w: 16, + h: 16, + }, + scale: Downscale::Quarter, + }, + None, + None, + Some(&packet), + ), + routing::RouteDecision::CpuHost + ); + } + + #[cfg(not(target_os = "macos"))] + #[test] + fn session_decode_rejects_unsupported_shape_before_host_unavailability() { + let mut decoder = Decoder::new(GRAYSCALE).expect("decoder"); + let session = MetalBackendSession::default(); + + assert!(matches!( + decoder.decode_to_device_with_session(PixelFormat::Gray8, &session), + Err(Error::UnsupportedMetalRequest { .. }) + )); + } +} diff --git a/crates/signinum-jpeg-metal/src/routing.rs b/crates/j2k-jpeg-metal/src/routing.rs similarity index 67% rename from crates/signinum-jpeg-metal/src/routing.rs rename to crates/j2k-jpeg-metal/src/routing.rs index 73f48c73..9ccc321a 100644 --- a/crates/signinum-jpeg-metal/src/routing.rs +++ b/crates/j2k-jpeg-metal/src/routing.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_core::{BackendRequest, PixelFormat}; -use signinum_jpeg::{ - adapter::{JpegMetalFast420PacketV1, JpegMetalFast422PacketV1, JpegMetalFast444PacketV1}, +use j2k_core::{BackendRequest, PixelFormat}; +use j2k_jpeg::{ + adapter::{JpegFast420PacketV1, JpegFast422PacketV1, JpegFast444PacketV1}, Decoder as CpuDecoder, }; @@ -42,9 +42,9 @@ impl JpegMetalCapabilities { _decoder: &CpuDecoder<'_>, fmt: PixelFormat, _op: BatchOp, - fast444_packet: Option<&JpegMetalFast444PacketV1>, - fast422_packet: Option<&JpegMetalFast422PacketV1>, - fast420_packet: Option<&JpegMetalFast420PacketV1>, + fast444_packet: Option<&JpegFast444PacketV1>, + fast422_packet: Option<&JpegFast422PacketV1>, + fast420_packet: Option<&JpegFast420PacketV1>, ) -> Self { let has_fast_packet = fast444_packet.is_some() || fast422_packet.is_some() || fast420_packet.is_some(); @@ -117,6 +117,31 @@ pub(crate) fn decision_error(decision: RouteDecision) -> Option { #[cfg(test)] mod tests { use super::*; + use j2k_jpeg::adapter::{ + build_fast420_packet_for_decoder, build_fast422_packet_for_decoder, + build_fast444_packet_for_decoder, + }; + + const BASELINE_420: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); + const BASELINE_422: &[u8] = include_bytes!("../fixtures/jpeg/baseline_422_16x8.jpg"); + const BASELINE_444: &[u8] = include_bytes!("../fixtures/jpeg/baseline_444_8x8.jpg"); + const GRAYSCALE: &[u8] = include_bytes!("../fixtures/jpeg/grayscale_8x8.jpg"); + + fn capabilities_for(bytes: &[u8], fmt: PixelFormat) -> JpegMetalCapabilities { + let decoder = CpuDecoder::new(bytes).expect("decoder"); + let fast444_packet = build_fast444_packet_for_decoder(&decoder).ok(); + let fast422_packet = build_fast422_packet_for_decoder(&decoder).ok(); + let fast420_packet = build_fast420_packet_for_decoder(&decoder).ok(); + + JpegMetalCapabilities::for_request( + &decoder, + fmt, + BatchOp::Full, + fast444_packet.as_ref(), + fast422_packet.as_ref(), + fast420_packet.as_ref(), + ) + } #[test] fn cuda_route_reports_unsupported_backend() { @@ -153,6 +178,40 @@ mod tests { )); } + #[test] + fn explicit_metal_accepts_fast_baseline_sampling_families() { + for (name, bytes) in [ + ("fast420", BASELINE_420), + ("fast422", BASELINE_422), + ("fast444", BASELINE_444), + ] { + let capabilities = capabilities_for(bytes, PixelFormat::Rgb8); + + assert!(capabilities.has_fast_packet(), "{name}"); + assert!(capabilities.supports_output_format(), "{name}"); + assert!( + matches!( + decide_route(BackendRequest::Metal, capabilities), + RouteDecision::MetalKernel | RouteDecision::MetalUnavailable + ), + "{name}" + ); + } + } + + #[test] + fn explicit_metal_rejects_supported_output_when_fast_packet_is_missing() { + let capabilities = capabilities_for(GRAYSCALE, PixelFormat::Gray8); + + assert!(!capabilities.has_fast_packet()); + assert!(capabilities.supports_output_format()); + assert!(matches!( + decide_route(BackendRequest::Metal, capabilities), + RouteDecision::RejectExplicitMetal { reason } + if reason.contains("fast 4:2:0, 4:2:2, or 4:4:4 baseline packets") + )); + } + #[test] fn auto_routes_single_request_to_cpu_even_when_metal_capabilities_match() { let capabilities = JpegMetalCapabilities { diff --git a/crates/signinum-jpeg-metal/src/session.rs b/crates/j2k-jpeg-metal/src/session.rs similarity index 70% rename from crates/signinum-jpeg-metal/src/session.rs rename to crates/j2k-jpeg-metal/src/session.rs index fafddb9b..d515b411 100644 --- a/crates/signinum-jpeg-metal/src/session.rs +++ b/crates/j2k-jpeg-metal/src/session.rs @@ -1,12 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 use std::collections::VecDeque; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, MutexGuard}; -use signinum_core::BackendRequest; -use signinum_jpeg::adapter::{ - build_metal_fast420_packet, build_metal_fast422_packet, build_metal_fast444_packet, - JpegMetalFast420PacketV1, JpegMetalFast422PacketV1, JpegMetalFast444PacketV1, +use j2k_core::BackendRequest; +use j2k_jpeg::adapter::{ + build_fast420_packet, build_fast422_packet, build_fast444_packet, JpegFast420PacketV1, + JpegFast422PacketV1, JpegFast444PacketV1, }; use crate::{batch, Error}; @@ -18,9 +18,9 @@ const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; const FNV_PRIME: u64 = 0x0000_0100_0000_01B3; pub(crate) type SharedFastPackets = ( - Option>, - Option>, - Option>, + Option>, + Option>, + Option>, ); #[derive(Clone)] @@ -34,15 +34,16 @@ pub(crate) struct CachedBatchShape { pub(crate) struct CachedFastPackets { digest: u64, input: Arc<[u8]>, - fast444_packet: Option>, - fast422_packet: Option>, - fast420_packet: Option>, + fast444_packet: Option>, + fast422_packet: Option>, + fast420_packet: Option>, } #[derive(Clone)] struct CachedInputAlias { source_ptr: usize, source_len: usize, + digest: u64, input: Arc<[u8]>, } @@ -51,12 +52,22 @@ pub(crate) struct SessionState { pub(crate) submissions: u64, pub(crate) queued: Vec, pub(crate) completed: Vec>>, + #[cfg(target_os = "macos")] + pub(crate) backend_session: Option, batch_shapes: VecDeque, fast_packets: VecDeque, input_aliases: VecDeque, } impl SessionState { + #[cfg(target_os = "macos")] + pub(crate) fn with_backend_session(backend_session: crate::MetalBackendSession) -> Self { + Self { + backend_session: Some(backend_session), + ..Self::default() + } + } + pub(crate) fn queue_request(&mut self, request: crate::batch::QueuedRequest) -> usize { let slot = self.completed.len(); self.completed.push(None); @@ -67,12 +78,22 @@ impl SessionState { pub(crate) fn intern_input_slice(&mut self, input: &[u8]) -> Arc<[u8]> { let source_ptr = input.as_ptr() as usize; let source_len = input.len(); + // Pointer identity alone is unsound: a caller may reuse one allocation + // for different payloads, so every hit is verified by digest plus byte + // equality, mirroring resolve_batch_shape/resolve_fast_packets. + let digest = digest_bytes(input); if let Some(entry) = self .input_aliases - .iter() + .iter_mut() .find(|entry| entry.source_ptr == source_ptr && entry.source_len == source_len) { - return Arc::clone(&entry.input); + if entry.digest == digest && entry.input.as_ref() == input { + return Arc::clone(&entry.input); + } + let refreshed = Arc::<[u8]>::from(input); + entry.digest = digest; + entry.input = Arc::clone(&refreshed); + return refreshed; } let input = Arc::<[u8]>::from(input); @@ -82,6 +103,7 @@ impl SessionState { self.input_aliases.push_back(CachedInputAlias { source_ptr, source_len, + digest, input: Arc::clone(&input), }); input @@ -131,8 +153,8 @@ impl SessionState { return Ok(entry.shape); } - let decoder = signinum_jpeg::Decoder::new(input.as_ref())?; - let summary = signinum_jpeg::adapter::summarize_device_batch(&decoder, 4); + let decoder = j2k_jpeg::Decoder::new(input.as_ref())?; + let summary = j2k_jpeg::adapter::summarize_device_batch(&decoder, 4); let shape = batch::BatchShape { restart_interval: summary.restart_interval, checkpoint_count: summary.checkpoint_count, @@ -193,15 +215,9 @@ impl SessionState { ); } - let fast444_packet = build_metal_fast444_packet(input.as_ref()) - .ok() - .map(Arc::new); - let fast422_packet = build_metal_fast422_packet(input.as_ref()) - .ok() - .map(Arc::new); - let fast420_packet = build_metal_fast420_packet(input.as_ref()) - .ok() - .map(Arc::new); + let fast444_packet = build_fast444_packet(input.as_ref()).ok().map(Arc::new); + let fast422_packet = build_fast422_packet(input.as_ref()).ok().map(Arc::new); + let fast420_packet = build_fast420_packet(input.as_ref()).ok().map(Arc::new); if self.fast_packets.len() == FAST_PACKET_CACHE_SLOTS { self.fast_packets.pop_front(); } @@ -215,11 +231,30 @@ impl SessionState { (fast444_packet, fast422_packet, fast420_packet) } + + #[cfg(target_os = "macos")] + pub(crate) fn backend_session(&mut self) -> Result<&crate::MetalBackendSession, Error> { + if self.backend_session.is_none() { + self.backend_session = Some(crate::MetalBackendSession::system_default()?); + } + Ok(self + .backend_session + .as_ref() + .expect("backend session initialized")) + } } #[derive(Clone, Default)] pub(crate) struct SharedSession(pub(crate) Arc>); +impl SharedSession { + pub(crate) fn lock(&self) -> Result, Error> { + self.0.lock().map_err(|_| Error::MetalStatePoisoned { + state: "JPEG Metal session", + }) + } +} + fn digest_bytes(bytes: &[u8]) -> u64 { let mut hash = FNV_OFFSET; for &byte in bytes { @@ -279,6 +314,37 @@ mod tests { assert_eq!(shape.sampling_family, batch::SamplingFamily::Fast422); } + #[test] + fn intern_input_slice_refreshes_when_buffer_is_overwritten() { + let mut session = SessionState::default(); + let mut buffer = vec![0xAAu8; 64]; + + let first = session.intern_input_slice(&buffer); + assert_eq!(first.as_ref(), [0xAAu8; 64].as_slice()); + + // Reuse the same allocation (same pointer, same length) with new contents. + buffer.fill(0xBB); + let second = session.intern_input_slice(&buffer); + assert_eq!( + second.as_ref(), + [0xBBu8; 64].as_slice(), + "input-alias cache returned stale bytes for a reused buffer" + ); + assert_eq!(session.input_aliases.len(), 1); + } + + #[test] + fn intern_input_slice_reuses_interned_arc_for_unchanged_buffer() { + let mut session = SessionState::default(); + let buffer = vec![0x42u8; 64]; + + let first = session.intern_input_slice(&buffer); + let second = session.intern_input_slice(&buffer); + + assert!(Arc::ptr_eq(&first, &second)); + assert_eq!(session.input_aliases.len(), 1); + } + #[cfg(not(target_os = "macos"))] #[test] fn non_macos_auto_and_metal_shape_resolution_stays_unparsed() { diff --git a/crates/signinum-jpeg-metal/src/shaders.metal b/crates/j2k-jpeg-metal/src/shaders.metal similarity index 73% rename from crates/signinum-jpeg-metal/src/shaders.metal rename to crates/j2k-jpeg-metal/src/shaders.metal index b8217cde..f78dc698 100644 --- a/crates/signinum-jpeg-metal/src/shaders.metal +++ b/crates/j2k-jpeg-metal/src/shaders.metal @@ -155,6 +155,41 @@ struct JpegFastRegionScaledBatchParams { uint origin_y; }; +struct JpegFast444TextureBatchParams { + uint width; + uint height; + uint mcus_per_row; + uint mcu_rows; + uint segment_count; + uint tile_index; + uint alpha; + uint mode; +}; + +struct JpegFast422TextureBatchParams { + uint width; + uint height; + uint chroma_width; + uint chroma_height; + uint mcus_per_row; + uint mcu_rows; + uint segment_count; + uint tile_index; + uint alpha; +}; + +struct JpegFast420TextureBatchParams { + uint width; + uint height; + uint chroma_width; + uint chroma_height; + uint mcus_per_row; + uint mcu_rows; + uint segment_count; + uint tile_index; + uint alpha; +}; + struct JpegWindowedPackBatchParams { uint src_width; uint src_height; @@ -171,6 +206,36 @@ struct JpegWindowedPackBatchParams { uint out_format; }; +struct JpegWindowedTexturePackBatchParams { + uint src_width; + uint src_height; + uint chroma_width; + uint chroma_height; + uint src_x; + uint src_y; + uint width; + uint height; + uint tile_index; + uint alpha; +}; + +struct JpegTexturePackBatchParams { + uint width; + uint height; + uint chroma_width; + uint chroma_height; + uint tile_index; + uint alpha; + uint mode; +}; + +struct JpegRgb8ToRgbaTextureParams { + uint width; + uint height; + uint in_stride; + uint alpha; +}; + struct JpegDecodeStatus { uint code; uint detail; @@ -189,7 +254,7 @@ struct JpegEntropyCheckpoint { uint reserved; }; -struct MetalHuffmanTable { +struct JpegHuffmanTable { uchar bits[16]; ushort values_len; ushort reserved; @@ -1083,7 +1148,7 @@ inline bool configure_batch_entropy_thread( } inline void prepare_huffman( - constant MetalHuffmanTable &raw, + constant JpegHuffmanTable &raw, thread PreparedHuffman &out ) { uchar huffsize[256]; @@ -1989,7 +2054,15 @@ inline void deposit_scaled_block_region( thread const short coeffs[64], bool dc_only ) { - if (scale_shift == 1u) { + if (scale_shift == 0u) { + thread uchar pixels8[64]; + if (dc_only) { + idct_islow_dc_only(coeffs[0], pixels8); + } else { + idct_islow(coeffs, pixels8); + } + deposit_block_region(plane, stride, width, height, origin_x, origin_y, x, y, pixels8); + } else if (scale_shift == 1u) { thread uchar pixels4[16]; if (dc_only) { const uchar pixel = idct_islow_1x1(coeffs); @@ -2049,6 +2122,143 @@ inline uchar h2v2_sample( return uchar((this_sum * 3u + next_sum + 7u) >> 4); } +inline uchar h2v2_sample_thread( + thread const uchar *near_row, + thread const uchar *curr_row, + uint n, + uint x +) { + if (n == 0) { + return 0; + } + const uint sample = min(x / 2, n - 1); + const uint this_sum = 3u * uint(curr_row[sample]) + uint(near_row[sample]); + if (n == 1) { + return uchar((4u * this_sum + 8u) >> 4); + } + if (x == 0) { + return uchar((this_sum * 4u + 8u) >> 4); + } + if (x == n * 2u - 1u) { + return uchar((this_sum * 4u + 7u) >> 4); + } + if ((x & 1u) == 0u) { + const uint last_sum = 3u * uint(curr_row[sample - 1]) + uint(near_row[sample - 1]); + return uchar((this_sum * 3u + last_sum + 8u) >> 4); + } + const uint next_sum = 3u * uint(curr_row[sample + 1]) + uint(near_row[sample + 1]); + return uchar((this_sum * 3u + next_sum + 7u) >> 4); +} + +inline uchar h2v2_sample_thread_local( + thread const uchar *near_row, + thread const uchar *curr_row, + uint n, + uint x, + uint local_sample_base, + uchar left_near_sample, + uchar left_curr_sample +) { + if (n == 0) { + return 0; + } + const uint sample = min(x / 2, n - 1); + const uint local_sample = sample - local_sample_base; + const uint this_sum = 3u * uint(curr_row[local_sample]) + uint(near_row[local_sample]); + if (n == 1) { + return uchar((4u * this_sum + 8u) >> 4); + } + if (x == 0) { + return uchar((this_sum * 4u + 8u) >> 4); + } + if (x == n * 2u - 1u) { + return uchar((this_sum * 4u + 7u) >> 4); + } + if ((x & 1u) == 0u) { + const uint last_sum = local_sample == 0u + ? 3u * uint(left_curr_sample) + uint(left_near_sample) + : 3u * uint(curr_row[local_sample - 1]) + uint(near_row[local_sample - 1]); + return uchar((this_sum * 3u + last_sum + 8u) >> 4); + } + const uint next_sum = 3u * uint(curr_row[local_sample + 1]) + uint(near_row[local_sample + 1]); + return uchar((this_sum * 3u + next_sum + 7u) >> 4); +} + +inline uint fast420_boundary_meta_base(uint record_index) { + return record_index * 4u; +} + +inline uint fast420_boundary_sample_base(uint record_index) { + return record_index * 64u; +} + +inline uint fast420_vertical_meta_base(uint record_index) { + return record_index * 4u; +} + +inline uint fast420_vertical_sample_base(uint record_index) { + return record_index * 64u; +} + +inline uchar h2v2_sample_device_local( + device const uchar *near_row, + device const uchar *curr_row, + uint n, + uint x, + uint local_sample_base +) { + if (n == 0) { + return 0; + } + const uint sample = min(x / 2u, n - 1u); + const uint local_sample = sample - local_sample_base; + const uint this_sum = 3u * uint(curr_row[local_sample]) + uint(near_row[local_sample]); + if (n == 1) { + return uchar((4u * this_sum + 8u) >> 4); + } + if (x == 0u) { + return uchar((this_sum * 4u + 8u) >> 4); + } + if (x == n * 2u - 1u) { + return uchar((this_sum * 4u + 7u) >> 4); + } + if ((x & 1u) == 0u) { + const uint last_sum = 3u * uint(curr_row[local_sample - 1u]) + + uint(near_row[local_sample - 1u]); + return uchar((this_sum * 3u + last_sum + 8u) >> 4); + } + const uint next_sum = 3u * uint(curr_row[local_sample + 1u]) + + uint(near_row[local_sample + 1u]); + return uchar((this_sum * 3u + next_sum + 7u) >> 4); +} + +inline uchar h2v2_boundary_left_from_sums(uint left_sum, uint right_sum) { + return uchar((left_sum * 3u + right_sum + 7u) >> 4); +} + +inline uchar h2v2_boundary_right_from_sums(uint left_sum, uint right_sum) { + return uchar((right_sum * 3u + left_sum + 8u) >> 4); +} + +inline uchar h2v2_corner_sample( + uchar top_left, + uchar top_right, + uchar bottom_left, + uchar bottom_right, + bool bottom_row, + bool right_column +) { + const uint left_sum = bottom_row + ? 3u * uint(bottom_left) + uint(top_left) + : 3u * uint(top_left) + uint(bottom_left); + const uint right_sum = bottom_row + ? 3u * uint(bottom_right) + uint(top_right) + : 3u * uint(top_right) + uint(bottom_right); + return right_column + ? h2v2_boundary_right_from_sums(left_sum, right_sum) + : h2v2_boundary_left_from_sums(left_sum, right_sum); +} + inline uchar h2v1_sample( device const uchar *row, uint n, @@ -2077,6 +2287,73 @@ inline uchar h2v1_sample( return uchar((3u * curr + next + 2u) >> 2); } +inline uchar h2v1_sample_thread( + thread const uchar *row, + uint n, + uint x +) { + if (n == 0) { + return 0; + } + if (n == 1) { + return row[0]; + } + const uint sample = min(x / 2u, n - 1u); + if (x == 0u) { + return row[0]; + } + if (x == n * 2u - 1u) { + return row[n - 1u]; + } + if ((x & 1u) == 0u) { + const uint prev = uint(row[sample - 1u]); + const uint curr = uint(row[sample]); + return uchar((3u * curr + prev + 2u) >> 2); + } + const uint curr = uint(row[sample]); + const uint next = uint(row[sample + 1u]); + return uchar((3u * curr + next + 2u) >> 2); +} + +inline uchar h2v1_sample_thread_local( + thread const uchar *row, + uint n, + uint x, + uint local_sample_base, + uchar left_sample +) { + if (n == 0) { + return 0; + } + if (n == 1) { + return row[0]; + } + const uint sample = min(x / 2u, n - 1u); + const uint local_sample = sample - local_sample_base; + if (x == 0u) { + return row[0]; + } + if (x == n * 2u - 1u) { + return row[local_sample]; + } + if ((x & 1u) == 0u) { + const uint prev = local_sample == 0u ? uint(left_sample) : uint(row[local_sample - 1u]); + const uint curr = uint(row[local_sample]); + return uchar((3u * curr + prev + 2u) >> 2); + } + const uint curr = uint(row[local_sample]); + const uint next = uint(row[local_sample + 1u]); + return uchar((3u * curr + next + 2u) >> 2); +} + +inline uint fast422_boundary_meta_base(uint record_index) { + return record_index * 4u; +} + +inline uint fast422_boundary_sample_base(uint record_index) { + return record_index * 48u; +} + inline void h2v2_sample_even_pair( device const uchar *near_row, device const uchar *curr_row, @@ -2134,6 +2411,47 @@ inline void h2v1_sample_even_pair( right = uchar((3u * curr + next + 2u) >> 2); } +inline void jpeg_sample_420_chroma( + device const uchar *cb_plane, + device const uchar *cr_plane, + uint chroma_width, + uint chroma_height, + uint x, + uint y, + thread uchar &cb, + thread uchar &cr +) { + const uint chroma_y = min(y / 2u, chroma_height - 1u); + const uint near_y = (y & 1u) == 0u + ? (chroma_y == 0u ? 0u : chroma_y - 1u) + : min(chroma_y + 1u, chroma_height - 1u); + device const uchar *curr_cb = cb_plane + chroma_y * chroma_width; + device const uchar *near_cb = cb_plane + near_y * chroma_width; + device const uchar *curr_cr = cr_plane + chroma_y * chroma_width; + device const uchar *near_cr = cr_plane + near_y * chroma_width; + + cb = h2v2_sample(near_cb, curr_cb, chroma_width, x); + cr = h2v2_sample(near_cr, curr_cr, chroma_width, x); +} + +inline void jpeg_sample_422_chroma( + device const uchar *cb_plane, + device const uchar *cr_plane, + uint chroma_width, + uint chroma_height, + uint x, + uint y, + thread uchar &cb, + thread uchar &cr +) { + const uint chroma_y = min(y, chroma_height - 1u); + device const uchar *curr_cb = cb_plane + chroma_y * chroma_width; + device const uchar *curr_cr = cr_plane + chroma_y * chroma_width; + + cb = h2v1_sample(curr_cb, chroma_width, x); + cr = h2v1_sample(curr_cr, chroma_width, x); +} + inline void store_rgb_ycbcr( device uchar *out, uint out_idx, @@ -2149,6 +2467,42 @@ inline void store_rgb_ycbcr( out[out_idx + 2u] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); } +inline void store_rgba_ycbcr( + device uchar *out, + uint out_idx, + uchar y_value, + uchar cb_value, + uchar cr_value, + uint alpha +) { + store_rgb_ycbcr(out, out_idx, y_value, cb_value, cr_value); + out[out_idx + 3u] = uchar(alpha); +} + +inline float4 rgba_float_ycbcr( + uchar y_value, + uchar cb_value, + uchar cr_value, + uint alpha +) { + const int y = int(y_value); + const int cb_centered = int(cb_value) - 128; + const int cr_centered = int(cr_value) - 128; + const uchar r = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); + const uchar g = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); + const uchar b = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); + return float4(float(r), float(g), float(b), float(uchar(alpha))) / 255.0f; +} + +inline float4 rgba_float_direct( + uchar r, + uchar g, + uchar b, + uint alpha +) { + return float4(float(r), float(g), float(b), float(uchar(alpha))) / 255.0f; +} + kernel void jpeg_pack( device const uchar *plane0 [[buffer(0)]], device const uchar *plane1 [[buffer(1)]], @@ -2190,12 +2544,7 @@ kernel void jpeg_pack( out[out_idx + 1] = plane1[idx]; out[out_idx + 2] = plane2[idx]; } else { - const int y = int(plane0[idx]); - const int cb = int(plane1[idx]) - 128; - const int cr = int(plane2[idx]) - 128; - out[out_idx] = clamp_u8(y + ((91881 * cr + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb + 46802 * cr + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb + (1 << 15)) >> 16)); + store_rgb_ycbcr(out, out_idx, plane0[idx], plane1[idx], plane2[idx]); } if (params.out_format == OUT_RGBA) { @@ -2225,7 +2574,8 @@ kernel void jpeg_pack_444_rgb_batch( const uint idx = plane_base + src_y * params.src_width + src_x; const uint out_base = gid.z * params.out_stride * params.height; - const uint out_idx = out_base + gid.y * params.out_stride + gid.x * 3u; + const uint out_idx = + out_base + gid.y * params.out_stride + gid.x * (params.out_format == OUT_RGB ? 3u : 4u); if (params.mode == MODE_GRAY) { const uchar gray = plane0[idx]; @@ -2237,12 +2587,40 @@ kernel void jpeg_pack_444_rgb_batch( out[out_idx + 1] = plane1[idx]; out[out_idx + 2] = plane2[idx]; } else { - const int y = int(plane0[idx]); - const int cb = int(plane1[idx]) - 128; - const int cr = int(plane2[idx]) - 128; - out[out_idx] = clamp_u8(y + ((91881 * cr + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb + 46802 * cr + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb + (1 << 15)) >> 16)); + store_rgb_ycbcr(out, out_idx, plane0[idx], plane1[idx], plane2[idx]); + } + + if (params.out_format == OUT_RGBA) { + out[out_idx + 3] = uchar(params.alpha); + } +} + +kernel void jpeg_pack_444_rgba_texture( + device const uchar *plane0 [[buffer(0)]], + device const uchar *plane1 [[buffer(1)]], + device const uchar *plane2 [[buffer(2)]], + constant JpegTexturePackBatchParams ¶ms [[buffer(3)]], + texture2d out [[texture(0)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height) { + return; + } + + const uint plane_len = params.width * params.height; + const uint plane_base = params.tile_index * plane_len; + const uint idx = plane_base + gid.y * params.width + gid.x; + + if (params.mode == MODE_GRAY) { + const uchar gray = plane0[idx]; + out.write(rgba_float_direct(gray, gray, gray, params.alpha), gid); + } else if (params.mode == MODE_RGB) { + out.write( + rgba_float_direct(plane0[idx], plane1[idx], plane2[idx], params.alpha), + gid + ); + } else { + out.write(rgba_float_ycbcr(plane0[idx], plane1[idx], plane2[idx], params.alpha), gid); } } @@ -2514,15 +2892,693 @@ kernel void jpeg_decode_fast420_batch( } } -inline uint fast420_total_mcus(constant JpegFast420BatchParams ¶ms) { - return params.mcus_per_row * params.mcu_rows; -} - -inline uint fast420_y_blocks_per_tile(constant JpegFast420BatchParams ¶ms) { - return fast420_total_mcus(params) * 4u; -} - -inline uint fast420_blocks_per_tile(constant JpegFast420BatchParams ¶ms) { +kernel void jpeg_decode_fast420_rgba_texture_batch( + device const uchar *entropy [[buffer(0)]], + constant JpegFast420TextureBatchParams ¶ms [[buffer(4)]], + constant ushort *y_quant [[buffer(5)]], + constant ushort *cb_quant [[buffer(6)]], + constant ushort *cr_quant [[buffer(7)]], + constant PreparedHuffman &y_dc [[buffer(8)]], + constant PreparedHuffman &y_ac [[buffer(9)]], + constant PreparedHuffman &cb_dc [[buffer(10)]], + constant PreparedHuffman &cb_ac [[buffer(11)]], + constant PreparedHuffman &cr_dc [[buffer(12)]], + constant PreparedHuffman &cr_ac [[buffer(13)]], + device const uint *entropy_offsets [[buffer(14)]], + device const uint *entropy_lens [[buffer(15)]], + device JpegDecodeStatus *status [[buffer(16)]], + device const JpegEntropyCheckpoint *entropy_checkpoints [[buffer(17)]], + device uint *boundary_meta [[buffer(18)]], + device uchar *boundary_samples [[buffer(19)]], + device uint *vertical_meta [[buffer(20)]], + device uchar *vertical_samples [[buffer(21)]], + texture2d out [[texture(0)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= params.segment_count) { + return; + } + + const uint total_mcus = params.mcus_per_row * params.mcu_rows; + const uint status_index = params.tile_index * params.segment_count + gid; + device JpegDecodeStatus *thread_status = status + status_index; + thread_status->code = FAST420_STATUS_OK; + thread_status->detail = 0; + thread_status->position = 0; + thread_status->reserved = 0; + + const uint checkpoint_base = params.tile_index * params.segment_count; + const JpegEntropyCheckpoint checkpoint = entropy_checkpoints[checkpoint_base + gid]; + uint start_mcu = checkpoint.mcu_index; + if (start_mcu >= total_mcus) { + return; + } + uint end_mcu = total_mcus; + if (gid + 1u < params.segment_count) { + end_mcu = min(total_mcus, entropy_checkpoints[checkpoint_base + gid + 1u].mcu_index); + } + if (end_mcu <= start_mcu) { + return; + } + + const uint entropy_base = entropy_offsets[params.tile_index]; + const uint entropy_end = entropy_base + entropy_lens[params.tile_index]; + thread BitReader br; + br.pos = entropy_base + checkpoint.entropy_pos; + br.acc = checkpoint.bit_acc; + br.bits = checkpoint.bit_count; + + int y_prev_dc = checkpoint.y_prev_dc; + int cb_prev_dc = checkpoint.cb_prev_dc; + int cr_prev_dc = checkpoint.cr_prev_dc; + + thread short coeffs[64]; + thread uchar y00_pixels[64]; + thread uchar y10_pixels[64]; + thread uchar y01_pixels[64]; + thread uchar y11_pixels[64]; + thread uchar cb_pixels[64]; + thread uchar cr_pixels[64]; + thread uchar prev_y10_pixels[64]; + thread uchar prev_y11_pixels[64]; + thread uchar prev_cb_pixels[64]; + thread uchar prev_cr_pixels[64]; + bool have_prev_horizontal = false; + thread uchar prev_vertical_y01_pixels[64]; + thread uchar prev_vertical_y11_pixels[64]; + thread uchar prev_vertical_cb_pixels[64]; + thread uchar prev_vertical_cr_pixels[64]; + bool have_prev_vertical = false; + + uint mx = 0u; + uint my = 0u; + init_mcu_cursor(start_mcu, params.mcus_per_row, mx, my); + for (uint mcu_index = start_mcu; mcu_index < end_mcu; ++mcu_index) { + const uint y_x = mx * 16u; + const uint y_y = my * 16u; + bool dc_only = false; + + if (!decode_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], y00_pixels); + } else { + idct_islow(coeffs, y00_pixels); + } + + if (!decode_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], y10_pixels); + } else { + idct_islow(coeffs, y10_pixels); + } + + if (!decode_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], y01_pixels); + } else { + idct_islow(coeffs, y01_pixels); + } + + if (!decode_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], y11_pixels); + } else { + idct_islow(coeffs, y11_pixels); + } + + if (!decode_block(br, entropy, entropy_end, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], cb_pixels); + } else { + idct_islow(coeffs, cb_pixels); + } + + if (!decode_block(br, entropy, entropy_end, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], cr_pixels); + } else { + idct_islow(coeffs, cr_pixels); + } + + const uint copy_width = min(16u, params.width - min(y_x, params.width)); + const uint copy_height = min(16u, params.height - min(y_y, params.height)); + const uint chroma_y_base = my * 8u; + const uint copy_chroma_height = min(8u, params.chroma_height - min(chroma_y_base, params.chroma_height)); + const bool starts_mid_row = mcu_index == start_mcu && mx > 0u; + const bool has_left_mcu = mx > 0u; + const bool has_right_mcu = mx + 1u < params.mcus_per_row; + const bool has_top_mcu = my > 0u; + const bool has_bottom_mcu = my + 1u < params.mcu_rows; + const uint repair_record_index = params.tile_index * total_mcus + mcu_index; + const uint boundary_meta_base = fast420_boundary_meta_base(repair_record_index); + boundary_meta[boundary_meta_base] = 0u; + boundary_meta[boundary_meta_base + 1u] = 0u; + boundary_meta[boundary_meta_base + 2u] = 0u; + boundary_meta[boundary_meta_base + 3u] = 0u; + const uint vertical_meta_base = fast420_vertical_meta_base(repair_record_index); + vertical_meta[vertical_meta_base] = 0u; + vertical_meta[vertical_meta_base + 1u] = 0u; + vertical_meta[vertical_meta_base + 2u] = 0u; + vertical_meta[vertical_meta_base + 3u] = 0u; + const uint boundary_sample_base = fast420_boundary_sample_base(repair_record_index); + const uint vertical_sample_base = fast420_vertical_sample_base(repair_record_index); + const uint local_sample_base = mx * 8u; + if (has_left_mcu) { + boundary_meta[boundary_meta_base] = y_x; + boundary_meta[boundary_meta_base + 1u] = y_y; + boundary_meta[boundary_meta_base + 2u] = 1u; + for (uint by = 0u; by < copy_height; ++by) { + boundary_samples[boundary_sample_base + by] = by < 8u + ? y00_pixels[by * 8u] + : y01_pixels[(by - 8u) * 8u]; + } + for (uint cy = 0u; cy < copy_chroma_height; ++cy) { + boundary_samples[boundary_sample_base + 16u + cy] = cb_pixels[cy * 8u]; + boundary_samples[boundary_sample_base + 24u + cy] = cr_pixels[cy * 8u]; + } + } + if (has_right_mcu) { + boundary_meta[boundary_meta_base + 3u] = 1u; + for (uint by = 0u; by < copy_height; ++by) { + boundary_samples[boundary_sample_base + 32u + by] = by < 8u + ? y10_pixels[by * 8u + 7u] + : y11_pixels[(by - 8u) * 8u + 7u]; + } + for (uint cy = 0u; cy < copy_chroma_height; ++cy) { + boundary_samples[boundary_sample_base + 48u + cy] = cb_pixels[cy * 8u + 7u]; + boundary_samples[boundary_sample_base + 56u + cy] = cr_pixels[cy * 8u + 7u]; + } + } + if (has_top_mcu) { + vertical_meta[vertical_meta_base] = y_x; + vertical_meta[vertical_meta_base + 1u] = y_y; + vertical_meta[vertical_meta_base + 2u] = 1u; + for (uint bx = 0u; bx < copy_width; ++bx) { + vertical_samples[vertical_sample_base + bx] = bx < 8u + ? y00_pixels[bx] + : y10_pixels[bx - 8u]; + } + for (uint cx = 0u; cx < min(8u, params.chroma_width - min(mx * 8u, params.chroma_width)); ++cx) { + vertical_samples[vertical_sample_base + 16u + cx] = cb_pixels[cx]; + vertical_samples[vertical_sample_base + 24u + cx] = cr_pixels[cx]; + } + } + if (has_bottom_mcu) { + vertical_meta[vertical_meta_base + 3u] = 1u; + for (uint bx = 0u; bx < copy_width; ++bx) { + vertical_samples[vertical_sample_base + 32u + bx] = bx < 8u + ? y01_pixels[7u * 8u + bx] + : y11_pixels[7u * 8u + (bx - 8u)]; + } + for (uint cx = 0u; cx < min(8u, params.chroma_width - min(mx * 8u, params.chroma_width)); ++cx) { + vertical_samples[vertical_sample_base + 48u + cx] = cb_pixels[7u * 8u + cx]; + vertical_samples[vertical_sample_base + 56u + cx] = cr_pixels[7u * 8u + cx]; + } + } + if (have_prev_vertical && params.mcus_per_row == 1u && has_top_mcu) { + const uint top_y = y_y; + const uint bottom_y = y_y - 1u; + for (uint bx = 0u; bx < copy_width; ++bx) { + const uint out_x = y_x + bx; + const uchar top_y_value = bx < 8u ? y00_pixels[bx] : y10_pixels[bx - 8u]; + const uchar bottom_y_value = bx < 8u + ? prev_vertical_y01_pixels[7u * 8u + bx] + : prev_vertical_y11_pixels[7u * 8u + (bx - 8u)]; + out.write( + rgba_float_ycbcr( + bottom_y_value, + h2v2_sample_thread_local( + cb_pixels, + prev_vertical_cb_pixels + 7u * 8u, + params.chroma_width, + out_x, + local_sample_base, + cb_pixels[0], + prev_vertical_cb_pixels[7u * 8u] + ), + h2v2_sample_thread_local( + cr_pixels, + prev_vertical_cr_pixels + 7u * 8u, + params.chroma_width, + out_x, + local_sample_base, + cr_pixels[0], + prev_vertical_cr_pixels[7u * 8u] + ), + params.alpha + ), + uint2(out_x, bottom_y) + ); + out.write( + rgba_float_ycbcr( + top_y_value, + h2v2_sample_thread_local( + prev_vertical_cb_pixels + 7u * 8u, + cb_pixels, + params.chroma_width, + out_x, + local_sample_base, + prev_vertical_cb_pixels[7u * 8u], + cb_pixels[0] + ), + h2v2_sample_thread_local( + prev_vertical_cr_pixels + 7u * 8u, + cr_pixels, + params.chroma_width, + out_x, + local_sample_base, + prev_vertical_cr_pixels[7u * 8u], + cr_pixels[0] + ), + params.alpha + ), + uint2(out_x, top_y) + ); + } + } + if (have_prev_horizontal && mx > 0u) { + const uint left_x = y_x - 1u; + for (uint by = 0u; by < copy_height; ++by) { + const uint out_y = y_y + by; + if (params.mcu_rows > 1u && has_top_mcu && by == 0u) { + continue; + } + if (params.mcu_rows > 1u && has_bottom_mcu && by + 1u == copy_height) { + continue; + } + const uint chroma_y = min(out_y / 2u, params.chroma_height - 1u); + const uint near_y = (out_y & 1u) == 0u + ? (chroma_y == 0u ? 0u : chroma_y - 1u) + : min(chroma_y + 1u, params.chroma_height - 1u); + const uint local_chroma_y = chroma_y - chroma_y_base; + const uint local_near_y = near_y - chroma_y_base; + const uint left_cb_sum = 3u * uint(prev_cb_pixels[local_chroma_y * 8u + 7u]) + + uint(prev_cb_pixels[local_near_y * 8u + 7u]); + const uint left_cr_sum = 3u * uint(prev_cr_pixels[local_chroma_y * 8u + 7u]) + + uint(prev_cr_pixels[local_near_y * 8u + 7u]); + const uint right_cb_sum = 3u * uint(cb_pixels[local_chroma_y * 8u]) + + uint(cb_pixels[local_near_y * 8u]); + const uint right_cr_sum = 3u * uint(cr_pixels[local_chroma_y * 8u]) + + uint(cr_pixels[local_near_y * 8u]); + const uchar left_y = by < 8u + ? prev_y10_pixels[by * 8u + 7u] + : prev_y11_pixels[(by - 8u) * 8u + 7u]; + const uchar right_y = by < 8u + ? y00_pixels[by * 8u] + : y01_pixels[(by - 8u) * 8u]; + out.write( + rgba_float_ycbcr( + left_y, + h2v2_boundary_left_from_sums(left_cb_sum, right_cb_sum), + h2v2_boundary_left_from_sums(left_cr_sum, right_cr_sum), + params.alpha + ), + uint2(left_x, out_y) + ); + out.write( + rgba_float_ycbcr( + right_y, + h2v2_boundary_right_from_sums(left_cb_sum, right_cb_sum), + h2v2_boundary_right_from_sums(left_cr_sum, right_cr_sum), + params.alpha + ), + uint2(y_x, out_y) + ); + } + } + + const uint last_chroma_x = params.chroma_width * 2u - 1u; + for (uint by = 0u; by < copy_height; ++by) { + const uint out_y = y_y + by; + if (has_top_mcu && by == 0u) { + continue; + } + if (has_bottom_mcu && by + 1u == copy_height) { + continue; + } + const uint chroma_y = min(out_y / 2u, params.chroma_height - 1u); + const uint near_y = (out_y & 1u) == 0u + ? (chroma_y == 0u ? 0u : chroma_y - 1u) + : min(chroma_y + 1u, params.chroma_height - 1u); + const uint local_chroma_y = chroma_y - chroma_y_base; + const uint local_near_y = near_y - chroma_y_base; + thread const uchar *curr_cb = cb_pixels + local_chroma_y * 8u; + thread const uchar *curr_cr = cr_pixels + local_chroma_y * 8u; + thread const uchar *near_cb = cb_pixels + local_near_y * 8u; + thread const uchar *near_cr = cr_pixels + local_near_y * 8u; + const uchar left_curr_cb = mx == 0u || starts_mid_row + ? curr_cb[0] + : prev_cb_pixels[local_chroma_y * 8u + 7u]; + const uchar left_curr_cr = mx == 0u || starts_mid_row + ? curr_cr[0] + : prev_cr_pixels[local_chroma_y * 8u + 7u]; + const uchar left_near_cb = mx == 0u || starts_mid_row + ? near_cb[0] + : prev_cb_pixels[local_near_y * 8u + 7u]; + const uchar left_near_cr = mx == 0u || starts_mid_row + ? near_cr[0] + : prev_cr_pixels[local_near_y * 8u + 7u]; + for (uint bx = 0u; bx < copy_width; ++bx) { + const uint out_x = y_x + bx; + if (mx > 0u && bx == 0u) { + continue; + } + if (has_right_mcu && bx == 15u && out_x != last_chroma_x) { + continue; + } + uchar y_value; + if (by < 8u) { + y_value = bx < 8u + ? y00_pixels[by * 8u + bx] + : y10_pixels[by * 8u + (bx - 8u)]; + } else { + y_value = bx < 8u + ? y01_pixels[(by - 8u) * 8u + bx] + : y11_pixels[(by - 8u) * 8u + (bx - 8u)]; + } + const uchar cb_value = h2v2_sample_thread_local( + near_cb, + curr_cb, + params.chroma_width, + out_x, + local_sample_base, + left_near_cb, + left_curr_cb + ); + const uchar cr_value = h2v2_sample_thread_local( + near_cr, + curr_cr, + params.chroma_width, + out_x, + local_sample_base, + left_near_cr, + left_curr_cr + ); + out.write( + rgba_float_ycbcr(y_value, cb_value, cr_value, params.alpha), + uint2(out_x, out_y) + ); + } + } + + for (uint i = 0u; i < 64u; ++i) { + prev_y10_pixels[i] = y10_pixels[i]; + prev_y11_pixels[i] = y11_pixels[i]; + prev_cb_pixels[i] = cb_pixels[i]; + prev_cr_pixels[i] = cr_pixels[i]; + prev_vertical_y01_pixels[i] = y01_pixels[i]; + prev_vertical_y11_pixels[i] = y11_pixels[i]; + prev_vertical_cb_pixels[i] = cb_pixels[i]; + prev_vertical_cr_pixels[i] = cr_pixels[i]; + } + have_prev_horizontal = true; + have_prev_vertical = true; + advance_mcu_cursor(mx, my, params.mcus_per_row); + } +} + +kernel void jpeg_resolve_fast420_rgba_texture_boundaries( + device const uint *boundary_meta [[buffer(0)]], + device const uchar *boundary_samples [[buffer(1)]], + constant JpegFast420TextureBatchParams ¶ms [[buffer(2)]], + texture2d out [[texture(0)]], + uint gid [[thread_position_in_grid]] +) { + const uint total_mcus = params.mcus_per_row * params.mcu_rows; + if (gid >= total_mcus || params.mcus_per_row <= 1u) { + return; + } + const uint mx = gid % params.mcus_per_row; + if (mx == 0u) { + return; + } + + const uint record_index = params.tile_index * total_mcus + gid; + const uint previous_record_index = record_index - 1u; + const uint meta_base = fast420_boundary_meta_base(record_index); + const uint previous_meta_base = fast420_boundary_meta_base(previous_record_index); + if (boundary_meta[meta_base + 2u] == 0u || boundary_meta[previous_meta_base + 3u] == 0u) { + return; + } + + const uint x = boundary_meta[meta_base]; + const uint y = boundary_meta[meta_base + 1u]; + if (x == 0u || x >= params.width || y >= params.height) { + return; + } + + const uint sample_base = fast420_boundary_sample_base(record_index); + const uint previous_sample_base = fast420_boundary_sample_base(previous_record_index); + const uint copy_height = min(16u, params.height - y); + const uint chroma_y_base = y / 2u; + const bool has_top_row = y > 0u; + const bool has_bottom_row = y + copy_height < params.height; + for (uint by = 0u; by < copy_height; ++by) { + const uint out_y = y + by; + if (params.mcu_rows > 1u && has_top_row && by == 0u) { + continue; + } + if (params.mcu_rows > 1u && has_bottom_row && by + 1u == copy_height) { + continue; + } + const uint chroma_y = min(out_y / 2u, params.chroma_height - 1u); + const uint near_y = (out_y & 1u) == 0u + ? (chroma_y == 0u ? 0u : chroma_y - 1u) + : min(chroma_y + 1u, params.chroma_height - 1u); + const uint local_chroma_y = chroma_y - chroma_y_base; + const uint local_near_y = near_y - chroma_y_base; + const uint left_cb_sum = 3u * uint(boundary_samples[previous_sample_base + 48u + local_chroma_y]) + + uint(boundary_samples[previous_sample_base + 48u + local_near_y]); + const uint left_cr_sum = 3u * uint(boundary_samples[previous_sample_base + 56u + local_chroma_y]) + + uint(boundary_samples[previous_sample_base + 56u + local_near_y]); + const uint right_cb_sum = 3u * uint(boundary_samples[sample_base + 16u + local_chroma_y]) + + uint(boundary_samples[sample_base + 16u + local_near_y]); + const uint right_cr_sum = 3u * uint(boundary_samples[sample_base + 24u + local_chroma_y]) + + uint(boundary_samples[sample_base + 24u + local_near_y]); + const uchar left_y = boundary_samples[previous_sample_base + 32u + by]; + const uchar right_y = boundary_samples[sample_base + by]; + out.write( + rgba_float_ycbcr( + left_y, + h2v2_boundary_left_from_sums(left_cb_sum, right_cb_sum), + h2v2_boundary_left_from_sums(left_cr_sum, right_cr_sum), + params.alpha + ), + uint2(x - 1u, out_y) + ); + out.write( + rgba_float_ycbcr( + right_y, + h2v2_boundary_right_from_sums(left_cb_sum, right_cb_sum), + h2v2_boundary_right_from_sums(left_cr_sum, right_cr_sum), + params.alpha + ), + uint2(x, out_y) + ); + } +} + +kernel void jpeg_resolve_fast420_rgba_texture_vertical_boundaries( + device const uint *vertical_meta [[buffer(0)]], + device const uchar *vertical_samples [[buffer(1)]], + constant JpegFast420TextureBatchParams ¶ms [[buffer(2)]], + texture2d out [[texture(0)]], + uint gid [[thread_position_in_grid]] +) { + const uint total_mcus = params.mcus_per_row * params.mcu_rows; + if (gid >= total_mcus || params.mcu_rows <= 1u) { + return; + } + const uint my = gid / params.mcus_per_row; + if (my == 0u) { + return; + } + + const uint record_index = params.tile_index * total_mcus + gid; + const uint previous_record_index = record_index - params.mcus_per_row; + const uint meta_base = fast420_vertical_meta_base(record_index); + const uint previous_meta_base = fast420_vertical_meta_base(previous_record_index); + if (vertical_meta[meta_base + 2u] == 0u || vertical_meta[previous_meta_base + 3u] == 0u) { + return; + } + + const uint x = vertical_meta[meta_base]; + const uint y = vertical_meta[meta_base + 1u]; + if (x >= params.width || y == 0u || y >= params.height) { + return; + } + + const uint sample_base = fast420_vertical_sample_base(record_index); + const uint previous_sample_base = fast420_vertical_sample_base(previous_record_index); + const uint copy_width = min(16u, params.width - x); + const uint local_sample_base = x / 2u; + device const uchar *top_cb = vertical_samples + sample_base + 16u; + device const uchar *top_cr = vertical_samples + sample_base + 24u; + device const uchar *bottom_cb = vertical_samples + previous_sample_base + 48u; + device const uchar *bottom_cr = vertical_samples + previous_sample_base + 56u; + const bool has_left_column = params.mcus_per_row > 1u && x > 0u; + const bool has_right_column = params.mcus_per_row > 1u && x + copy_width < params.width; + for (uint bx = 0u; bx < copy_width; ++bx) { + const uint out_x = x + bx; + if (has_left_column && bx == 0u) { + continue; + } + if (has_right_column && bx + 1u == copy_width) { + continue; + } + const uchar bottom_y = vertical_samples[previous_sample_base + 32u + bx]; + const uchar top_y = vertical_samples[sample_base + bx]; + out.write( + rgba_float_ycbcr( + bottom_y, + h2v2_sample_device_local(top_cb, bottom_cb, params.chroma_width, out_x, local_sample_base), + h2v2_sample_device_local(top_cr, bottom_cr, params.chroma_width, out_x, local_sample_base), + params.alpha + ), + uint2(out_x, y - 1u) + ); + out.write( + rgba_float_ycbcr( + top_y, + h2v2_sample_device_local(bottom_cb, top_cb, params.chroma_width, out_x, local_sample_base), + h2v2_sample_device_local(bottom_cr, top_cr, params.chroma_width, out_x, local_sample_base), + params.alpha + ), + uint2(out_x, y) + ); + } +} + +kernel void jpeg_resolve_fast420_rgba_texture_corners( + device const uint *boundary_meta [[buffer(0)]], + device const uint *vertical_meta [[buffer(1)]], + device const uchar *vertical_samples [[buffer(2)]], + constant JpegFast420TextureBatchParams ¶ms [[buffer(3)]], + texture2d out [[texture(0)]], + uint gid [[thread_position_in_grid]] +) { + if (params.mcus_per_row <= 1u || params.mcu_rows <= 1u) { + return; + } + const uint total_mcus = params.mcus_per_row * params.mcu_rows; + if (gid >= total_mcus) { + return; + } + const uint mx = gid % params.mcus_per_row; + const uint my = gid / params.mcus_per_row; + if (mx == 0u || my == 0u) { + return; + } + + const uint br_record = params.tile_index * total_mcus + gid; + const uint bl_record = br_record - 1u; + const uint tr_record = br_record - params.mcus_per_row; + const uint tl_record = tr_record - 1u; + const uint br_boundary_meta_base = fast420_boundary_meta_base(br_record); + const uint bl_boundary_meta_base = fast420_boundary_meta_base(bl_record); + const uint tr_boundary_meta_base = fast420_boundary_meta_base(tr_record); + const uint tl_boundary_meta_base = fast420_boundary_meta_base(tl_record); + const uint br_vertical_meta_base = fast420_vertical_meta_base(br_record); + const uint bl_vertical_meta_base = fast420_vertical_meta_base(bl_record); + const uint tr_vertical_meta_base = fast420_vertical_meta_base(tr_record); + const uint tl_vertical_meta_base = fast420_vertical_meta_base(tl_record); + if ( + boundary_meta[br_boundary_meta_base + 2u] == 0u || + boundary_meta[bl_boundary_meta_base + 3u] == 0u || + boundary_meta[tr_boundary_meta_base + 2u] == 0u || + boundary_meta[tl_boundary_meta_base + 3u] == 0u || + vertical_meta[br_vertical_meta_base + 2u] == 0u || + vertical_meta[bl_vertical_meta_base + 2u] == 0u || + vertical_meta[tr_vertical_meta_base + 3u] == 0u || + vertical_meta[tl_vertical_meta_base + 3u] == 0u + ) { + return; + } + + const uint x = vertical_meta[br_vertical_meta_base]; + const uint y = vertical_meta[br_vertical_meta_base + 1u]; + if (x == 0u || y == 0u || x >= params.width || y >= params.height) { + return; + } + + const uint br_sample_base = fast420_vertical_sample_base(br_record); + const uint bl_sample_base = fast420_vertical_sample_base(bl_record); + const uint tr_sample_base = fast420_vertical_sample_base(tr_record); + const uint tl_sample_base = fast420_vertical_sample_base(tl_record); + + const uchar tl_y = vertical_samples[tl_sample_base + 32u + 15u]; + const uchar tr_y = vertical_samples[tr_sample_base + 32u]; + const uchar bl_y = vertical_samples[bl_sample_base + 15u]; + const uchar br_y = vertical_samples[br_sample_base]; + + const uchar tl_cb = vertical_samples[tl_sample_base + 48u + 7u]; + const uchar tr_cb = vertical_samples[tr_sample_base + 48u]; + const uchar bl_cb = vertical_samples[bl_sample_base + 16u + 7u]; + const uchar br_cb = vertical_samples[br_sample_base + 16u]; + const uchar tl_cr = vertical_samples[tl_sample_base + 56u + 7u]; + const uchar tr_cr = vertical_samples[tr_sample_base + 56u]; + const uchar bl_cr = vertical_samples[bl_sample_base + 24u + 7u]; + const uchar br_cr = vertical_samples[br_sample_base + 24u]; + + out.write( + rgba_float_ycbcr( + tl_y, + h2v2_corner_sample(tl_cb, tr_cb, bl_cb, br_cb, false, false), + h2v2_corner_sample(tl_cr, tr_cr, bl_cr, br_cr, false, false), + params.alpha + ), + uint2(x - 1u, y - 1u) + ); + out.write( + rgba_float_ycbcr( + tr_y, + h2v2_corner_sample(tl_cb, tr_cb, bl_cb, br_cb, false, true), + h2v2_corner_sample(tl_cr, tr_cr, bl_cr, br_cr, false, true), + params.alpha + ), + uint2(x, y - 1u) + ); + out.write( + rgba_float_ycbcr( + bl_y, + h2v2_corner_sample(tl_cb, tr_cb, bl_cb, br_cb, true, false), + h2v2_corner_sample(tl_cr, tr_cr, bl_cr, br_cr, true, false), + params.alpha + ), + uint2(x - 1u, y) + ); + out.write( + rgba_float_ycbcr( + br_y, + h2v2_corner_sample(tl_cb, tr_cb, bl_cb, br_cb, true, true), + h2v2_corner_sample(tl_cr, tr_cr, bl_cr, br_cr, true, true), + params.alpha + ), + uint2(x, y) + ); +} + +inline uint fast420_total_mcus(constant JpegFast420BatchParams ¶ms) { + return params.mcus_per_row * params.mcu_rows; +} + +inline uint fast420_y_blocks_per_tile(constant JpegFast420BatchParams ¶ms) { + return fast420_total_mcus(params) * 4u; +} + +inline uint fast420_blocks_per_tile(constant JpegFast420BatchParams ¶ms) { return fast420_total_mcus(params) * 6u; } @@ -2968,12 +4024,9 @@ kernel void jpeg_decode_fast422_batch( } } -kernel void jpeg_decode_fast422_region( +kernel void jpeg_decode_fast422_rgba_texture_batch( device const uchar *entropy [[buffer(0)]], - device uchar *y_plane [[buffer(1)]], - device uchar *cb_plane [[buffer(2)]], - device uchar *cr_plane [[buffer(3)]], - constant JpegFast420Params ¶ms [[buffer(4)]], + constant JpegFast422TextureBatchParams ¶ms [[buffer(4)]], constant ushort *y_quant [[buffer(5)]], constant ushort *cb_quant [[buffer(6)]], constant ushort *cr_quant [[buffer(7)]], @@ -2983,36 +4036,301 @@ kernel void jpeg_decode_fast422_region( constant PreparedHuffman &cb_ac [[buffer(11)]], constant PreparedHuffman &cr_dc [[buffer(12)]], constant PreparedHuffman &cr_ac [[buffer(13)]], - device const uint *restart_offsets [[buffer(14)]], - device JpegDecodeStatus *status [[buffer(15)]], - device const JpegEntropyCheckpoint *entropy_checkpoints [[buffer(16)]], + device const uint *entropy_offsets [[buffer(14)]], + device const uint *entropy_lens [[buffer(15)]], + device JpegDecodeStatus *status [[buffer(16)]], + device const JpegEntropyCheckpoint *entropy_checkpoints [[buffer(17)]], + device uint *boundary_meta [[buffer(18)]], + device uchar *boundary_samples [[buffer(19)]], + texture2d out [[texture(0)]], uint gid [[thread_position_in_grid]] ) { - const uint total_mcus = params.mcus_per_row * params.mcu_rows; - thread BitReader br; - uint start_mcu = 0u; - uint end_mcu = 0u; - int y_prev_dc = 0; - int cb_prev_dc = 0; - int cr_prev_dc = 0; - if (!configure_entropy_thread( - gid, - total_mcus, - params.restart_interval_mcus, - params.restart_offset_count, - params.restart_start_mcu, - restart_offsets, - entropy_checkpoints, - br, - start_mcu, - end_mcu, - y_prev_dc, - cb_prev_dc, - cr_prev_dc - )) { + if (gid >= params.segment_count) { return; } - device JpegDecodeStatus *thread_status = status + gid; + + const uint total_mcus = params.mcus_per_row * params.mcu_rows; + const uint status_index = params.tile_index * params.segment_count + gid; + device JpegDecodeStatus *thread_status = status + status_index; + thread_status->code = FAST420_STATUS_OK; + thread_status->detail = 0; + thread_status->position = 0; + thread_status->reserved = 0; + const uint boundary_meta_base = fast422_boundary_meta_base(status_index); + boundary_meta[boundary_meta_base] = 0u; + boundary_meta[boundary_meta_base + 1u] = 0u; + boundary_meta[boundary_meta_base + 2u] = 0u; + boundary_meta[boundary_meta_base + 3u] = 0u; + + const uint checkpoint_base = params.tile_index * params.segment_count; + const JpegEntropyCheckpoint checkpoint = entropy_checkpoints[checkpoint_base + gid]; + uint start_mcu = checkpoint.mcu_index; + if (start_mcu >= total_mcus) { + return; + } + uint end_mcu = total_mcus; + if (gid + 1u < params.segment_count) { + end_mcu = min(total_mcus, entropy_checkpoints[checkpoint_base + gid + 1u].mcu_index); + } + if (end_mcu <= start_mcu) { + return; + } + + const uint entropy_base = entropy_offsets[params.tile_index]; + const uint entropy_end = entropy_base + entropy_lens[params.tile_index]; + thread BitReader br; + br.pos = entropy_base + checkpoint.entropy_pos; + br.acc = checkpoint.bit_acc; + br.bits = checkpoint.bit_count; + + int y_prev_dc = checkpoint.y_prev_dc; + int cb_prev_dc = checkpoint.cb_prev_dc; + int cr_prev_dc = checkpoint.cr_prev_dc; + + thread short coeffs[64]; + thread uchar y_left_pixels[64]; + thread uchar y_right_pixels[64]; + thread uchar cb_pixels[64]; + thread uchar cr_pixels[64]; + thread uchar prev_y_right_pixels[64]; + thread uchar prev_cb_pixels[64]; + thread uchar prev_cr_pixels[64]; + bool have_prev_horizontal = false; + + uint mx = 0u; + uint my = 0u; + init_mcu_cursor(start_mcu, params.mcus_per_row, mx, my); + for (uint mcu_index = start_mcu; mcu_index < end_mcu; ++mcu_index) { + const uint y_x = mx * 16u; + const uint y_y = my * 8u; + bool dc_only = false; + + if (!decode_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], y_left_pixels); + } else { + idct_islow(coeffs, y_left_pixels); + } + + if (!decode_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], y_right_pixels); + } else { + idct_islow(coeffs, y_right_pixels); + } + + if (!decode_block(br, entropy, entropy_end, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], cb_pixels); + } else { + idct_islow(coeffs, cb_pixels); + } + + if (!decode_block(br, entropy, entropy_end, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], cr_pixels); + } else { + idct_islow(coeffs, cr_pixels); + } + + const uint copy_width = min(16u, params.width - min(y_x, params.width)); + const uint copy_height = min(8u, params.height - min(y_y, params.height)); + const bool starts_mid_row = mcu_index == start_mcu && mx > 0u; + const bool ends_mid_row = mcu_index + 1u >= end_mcu && mx + 1u < params.mcus_per_row; + const uint boundary_sample_base = fast422_boundary_sample_base(status_index); + if (starts_mid_row) { + boundary_meta[boundary_meta_base] = y_x; + boundary_meta[boundary_meta_base + 1u] = y_y; + boundary_meta[boundary_meta_base + 2u] = 1u; + for (uint by = 0u; by < copy_height; ++by) { + boundary_samples[boundary_sample_base + by] = y_left_pixels[by * 8u]; + boundary_samples[boundary_sample_base + 8u + by] = cb_pixels[by * 8u]; + boundary_samples[boundary_sample_base + 16u + by] = cr_pixels[by * 8u]; + } + } + if (ends_mid_row) { + boundary_meta[boundary_meta_base + 3u] = 1u; + for (uint by = 0u; by < copy_height; ++by) { + boundary_samples[boundary_sample_base + 24u + by] = y_right_pixels[by * 8u + 7u]; + boundary_samples[boundary_sample_base + 32u + by] = cb_pixels[by * 8u + 7u]; + boundary_samples[boundary_sample_base + 40u + by] = cr_pixels[by * 8u + 7u]; + } + } + if (have_prev_horizontal && mx > 0u) { + const uint prev_x = y_x - 1u; + for (uint by = 0u; by < copy_height; ++by) { + thread const uchar *cb_row = cb_pixels + by * 8u; + thread const uchar *cr_row = cr_pixels + by * 8u; + thread const uchar *prev_cb_row = prev_cb_pixels + by * 8u; + thread const uchar *prev_cr_row = prev_cr_pixels + by * 8u; + const uchar y_value = prev_y_right_pixels[by * 8u + 7u]; + const uchar cb_value = uchar((3u * uint(prev_cb_row[7]) + uint(cb_row[0]) + 2u) >> 2); + const uchar cr_value = uchar((3u * uint(prev_cr_row[7]) + uint(cr_row[0]) + 2u) >> 2); + out.write( + rgba_float_ycbcr(y_value, cb_value, cr_value, params.alpha), + uint2(prev_x, y_y + by) + ); + } + } + + const uint local_sample_base = mx * 8u; + const bool has_right_mcu = mx + 1u < params.mcus_per_row; + const uint last_chroma_x = params.chroma_width * 2u - 1u; + for (uint by = 0u; by < copy_height; ++by) { + thread const uchar *cb_row = cb_pixels + by * 8u; + thread const uchar *cr_row = cr_pixels + by * 8u; + const uchar left_cb = mx == 0u || starts_mid_row + ? cb_row[0] + : prev_cb_pixels[by * 8u + 7u]; + const uchar left_cr = mx == 0u || starts_mid_row + ? cr_row[0] + : prev_cr_pixels[by * 8u + 7u]; + for (uint bx = 0u; bx < copy_width; ++bx) { + const uint x = y_x + bx; + if (starts_mid_row && bx == 0u) { + continue; + } + if (has_right_mcu && bx == 15u && (x & 1u) == 1u && x != last_chroma_x) { + continue; + } + const uchar y_value = bx < 8u + ? y_left_pixels[by * 8u + bx] + : y_right_pixels[by * 8u + (bx - 8u)]; + const uchar cb_value = h2v1_sample_thread_local( + cb_row, + params.chroma_width, + x, + local_sample_base, + left_cb + ); + const uchar cr_value = h2v1_sample_thread_local( + cr_row, + params.chroma_width, + x, + local_sample_base, + left_cr + ); + out.write( + rgba_float_ycbcr(y_value, cb_value, cr_value, params.alpha), + uint2(x, y_y + by) + ); + } + } + + for (uint i = 0u; i < 64u; ++i) { + prev_y_right_pixels[i] = y_right_pixels[i]; + prev_cb_pixels[i] = cb_pixels[i]; + prev_cr_pixels[i] = cr_pixels[i]; + } + have_prev_horizontal = true; + advance_mcu_cursor(mx, my, params.mcus_per_row); + } +} + +kernel void jpeg_resolve_fast422_rgba_texture_boundaries( + device const uint *boundary_meta [[buffer(0)]], + device const uchar *boundary_samples [[buffer(1)]], + constant JpegFast422TextureBatchParams ¶ms [[buffer(2)]], + texture2d out [[texture(0)]], + uint gid [[thread_position_in_grid]] +) { + if (gid == 0u || gid >= params.segment_count) { + return; + } + + const uint record_index = params.tile_index * params.segment_count + gid; + const uint previous_record_index = record_index - 1u; + const uint meta_base = fast422_boundary_meta_base(record_index); + const uint previous_meta_base = fast422_boundary_meta_base(previous_record_index); + if (boundary_meta[meta_base + 2u] == 0u || boundary_meta[previous_meta_base + 3u] == 0u) { + return; + } + + const uint x = boundary_meta[meta_base]; + const uint y = boundary_meta[meta_base + 1u]; + if (x == 0u || x >= params.width || y >= params.height) { + return; + } + + const uint sample_base = fast422_boundary_sample_base(record_index); + const uint previous_sample_base = fast422_boundary_sample_base(previous_record_index); + const uint copy_height = min(8u, params.height - y); + for (uint by = 0u; by < copy_height; ++by) { + const uchar left_y = boundary_samples[previous_sample_base + 24u + by]; + const uchar left_cb = boundary_samples[previous_sample_base + 32u + by]; + const uchar left_cr = boundary_samples[previous_sample_base + 40u + by]; + const uchar right_y = boundary_samples[sample_base + by]; + const uchar right_cb = boundary_samples[sample_base + 8u + by]; + const uchar right_cr = boundary_samples[sample_base + 16u + by]; + const uchar resolved_left_cb = uchar((3u * uint(left_cb) + uint(right_cb) + 2u) >> 2); + const uchar resolved_left_cr = uchar((3u * uint(left_cr) + uint(right_cr) + 2u) >> 2); + const uchar resolved_right_cb = uchar((3u * uint(right_cb) + uint(left_cb) + 2u) >> 2); + const uchar resolved_right_cr = uchar((3u * uint(right_cr) + uint(left_cr) + 2u) >> 2); + const uint row_y = y + by; + out.write( + rgba_float_ycbcr(left_y, resolved_left_cb, resolved_left_cr, params.alpha), + uint2(x - 1u, row_y) + ); + out.write( + rgba_float_ycbcr(right_y, resolved_right_cb, resolved_right_cr, params.alpha), + uint2(x, row_y) + ); + } +} + +kernel void jpeg_decode_fast422_region( + device const uchar *entropy [[buffer(0)]], + device uchar *y_plane [[buffer(1)]], + device uchar *cb_plane [[buffer(2)]], + device uchar *cr_plane [[buffer(3)]], + constant JpegFast420Params ¶ms [[buffer(4)]], + constant ushort *y_quant [[buffer(5)]], + constant ushort *cb_quant [[buffer(6)]], + constant ushort *cr_quant [[buffer(7)]], + constant PreparedHuffman &y_dc [[buffer(8)]], + constant PreparedHuffman &y_ac [[buffer(9)]], + constant PreparedHuffman &cb_dc [[buffer(10)]], + constant PreparedHuffman &cb_ac [[buffer(11)]], + constant PreparedHuffman &cr_dc [[buffer(12)]], + constant PreparedHuffman &cr_ac [[buffer(13)]], + device const uint *restart_offsets [[buffer(14)]], + device JpegDecodeStatus *status [[buffer(15)]], + device const JpegEntropyCheckpoint *entropy_checkpoints [[buffer(16)]], + uint gid [[thread_position_in_grid]] +) { + const uint total_mcus = params.mcus_per_row * params.mcu_rows; + thread BitReader br; + uint start_mcu = 0u; + uint end_mcu = 0u; + int y_prev_dc = 0; + int cb_prev_dc = 0; + int cr_prev_dc = 0; + if (!configure_entropy_thread( + gid, + total_mcus, + params.restart_interval_mcus, + params.restart_offset_count, + params.restart_start_mcu, + restart_offsets, + entropy_checkpoints, + br, + start_mcu, + end_mcu, + y_prev_dc, + cb_prev_dc, + cr_prev_dc + )) { + return; + } + device JpegDecodeStatus *thread_status = status + gid; thread_status->code = FAST420_STATUS_OK; thread_status->detail = 0; @@ -5299,6 +6617,127 @@ kernel void jpeg_decode_fast444_scaled_region_batch( } } +kernel void jpeg_decode_fast444_rgba_texture_batch( + device const uchar *entropy [[buffer(0)]], + constant JpegFast444TextureBatchParams ¶ms [[buffer(4)]], + constant ushort *y_quant [[buffer(5)]], + constant ushort *cb_quant [[buffer(6)]], + constant ushort *cr_quant [[buffer(7)]], + constant PreparedHuffman &y_dc [[buffer(8)]], + constant PreparedHuffman &y_ac [[buffer(9)]], + constant PreparedHuffman &cb_dc [[buffer(10)]], + constant PreparedHuffman &cb_ac [[buffer(11)]], + constant PreparedHuffman &cr_dc [[buffer(12)]], + constant PreparedHuffman &cr_ac [[buffer(13)]], + device const uint *entropy_offsets [[buffer(14)]], + device const uint *entropy_lens [[buffer(15)]], + device JpegDecodeStatus *status [[buffer(16)]], + device const JpegEntropyCheckpoint *entropy_checkpoints [[buffer(17)]], + texture2d out [[texture(0)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= params.segment_count) { + return; + } + + const uint total_mcus = params.mcus_per_row * params.mcu_rows; + const uint status_index = params.tile_index * params.segment_count + gid; + device JpegDecodeStatus *thread_status = status + status_index; + thread_status->code = FAST420_STATUS_OK; + thread_status->detail = 0; + thread_status->position = 0; + thread_status->reserved = 0; + + const uint checkpoint_base = params.tile_index * params.segment_count; + const JpegEntropyCheckpoint checkpoint = entropy_checkpoints[checkpoint_base + gid]; + uint start_mcu = checkpoint.mcu_index; + if (start_mcu >= total_mcus) { + return; + } + uint end_mcu = total_mcus; + if (gid + 1u < params.segment_count) { + end_mcu = min(total_mcus, entropy_checkpoints[checkpoint_base + gid + 1u].mcu_index); + } + if (end_mcu <= start_mcu) { + return; + } + + const uint entropy_base = entropy_offsets[params.tile_index]; + const uint entropy_end = entropy_base + entropy_lens[params.tile_index]; + thread BitReader br; + br.pos = entropy_base + checkpoint.entropy_pos; + br.acc = checkpoint.bit_acc; + br.bits = checkpoint.bit_count; + + int y_prev_dc = checkpoint.y_prev_dc; + int cb_prev_dc = checkpoint.cb_prev_dc; + int cr_prev_dc = checkpoint.cr_prev_dc; + + thread short coeffs[64]; + thread uchar y_pixels[64]; + thread uchar cb_pixels[64]; + thread uchar cr_pixels[64]; + uint mx = 0u; + uint my = 0u; + init_mcu_cursor(start_mcu, params.mcus_per_row, mx, my); + for (uint mcu_index = start_mcu; mcu_index < end_mcu; ++mcu_index) { + const uint block_x = mx * 8u; + const uint block_y = my * 8u; + bool dc_only = false; + + if (!decode_block(br, entropy, entropy_end, y_dc, y_ac, y_quant, y_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], y_pixels); + } else { + idct_islow(coeffs, y_pixels); + } + + if (!decode_block(br, entropy, entropy_end, cb_dc, cb_ac, cb_quant, cb_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], cb_pixels); + } else { + idct_islow(coeffs, cb_pixels); + } + + if (!decode_block(br, entropy, entropy_end, cr_dc, cr_ac, cr_quant, cr_prev_dc, thread_status, coeffs, dc_only)) { + return; + } + if (dc_only) { + idct_islow_dc_only(coeffs[0], cr_pixels); + } else { + idct_islow(coeffs, cr_pixels); + } + + const uint copy_width = min(8u, params.width - min(block_x, params.width)); + const uint copy_height = min(8u, params.height - min(block_y, params.height)); + for (uint by = 0u; by < copy_height; ++by) { + for (uint bx = 0u; bx < copy_width; ++bx) { + const uint idx = by * 8u + bx; + const uint2 pos = uint2(block_x + bx, block_y + by); + if (params.mode == MODE_GRAY) { + const uchar gray = y_pixels[idx]; + out.write(rgba_float_direct(gray, gray, gray, params.alpha), pos); + } else if (params.mode == MODE_RGB) { + out.write( + rgba_float_direct(y_pixels[idx], cb_pixels[idx], cr_pixels[idx], params.alpha), + pos + ); + } else { + out.write( + rgba_float_ycbcr(y_pixels[idx], cb_pixels[idx], cr_pixels[idx], params.alpha), + pos + ); + } + } + } + advance_mcu_cursor(mx, my, params.mcus_per_row); + } +} + kernel void jpeg_pack_420( device const uchar *y_plane [[buffer(0)]], device const uchar *cb_plane [[buffer(1)]], @@ -5317,25 +6756,21 @@ kernel void jpeg_pack_420( return; } - const uint chroma_y = min(gid.y / 2u, params.chroma_height - 1u); - const uint near_y = (gid.y & 1u) == 0u - ? (chroma_y == 0u ? 0u : chroma_y - 1u) - : min(chroma_y + 1u, params.chroma_height - 1u); - device const uchar *curr_cb = cb_plane + chroma_y * params.chroma_width; - device const uchar *near_cb = cb_plane + near_y * params.chroma_width; - device const uchar *curr_cr = cr_plane + chroma_y * params.chroma_width; - device const uchar *near_cr = cr_plane + near_y * params.chroma_width; - - const uchar cb = h2v2_sample(near_cb, curr_cb, params.chroma_width, gid.x); - const uchar cr = h2v2_sample(near_cr, curr_cr, params.chroma_width, gid.x); - const int y = int(y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb; + uchar cr; + jpeg_sample_420_chroma( + cb_plane, + cr_plane, + params.chroma_width, + params.chroma_height, + gid.x, + gid.y, + cb, + cr + ); uint out_idx = gid.y * params.out_stride + gid.x * (params.out_format == OUT_RGB ? 3u : 4u); - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); + store_rgb_ycbcr(out, out_idx, y_plane[y_idx], cb, cr); if (params.out_format == OUT_RGBA) { out[out_idx + 3] = uchar(params.alpha); } @@ -5354,25 +6789,21 @@ kernel void jpeg_pack_420_rgb( } const uint y_idx = gid.y * params.width + gid.x; - const uint chroma_y = min(gid.y / 2u, params.chroma_height - 1u); - const uint near_y = (gid.y & 1u) == 0u - ? (chroma_y == 0u ? 0u : chroma_y - 1u) - : min(chroma_y + 1u, params.chroma_height - 1u); - device const uchar *curr_cb = cb_plane + chroma_y * params.chroma_width; - device const uchar *near_cb = cb_plane + near_y * params.chroma_width; - device const uchar *curr_cr = cr_plane + chroma_y * params.chroma_width; - device const uchar *near_cr = cr_plane + near_y * params.chroma_width; - - const uchar cb = h2v2_sample(near_cb, curr_cb, params.chroma_width, gid.x); - const uchar cr = h2v2_sample(near_cr, curr_cr, params.chroma_width, gid.x); - const int y = int(y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb; + uchar cr; + jpeg_sample_420_chroma( + cb_plane, + cr_plane, + params.chroma_width, + params.chroma_height, + gid.x, + gid.y, + cb, + cr + ); const uint out_idx = gid.y * params.out_stride + gid.x * 3u; - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); + store_rgb_ycbcr(out, out_idx, y_plane[y_idx], cb, cr); } kernel void jpeg_pack_420_rgb_batch( @@ -5459,26 +6890,100 @@ kernel void jpeg_pack_420_rgba( } const uint y_idx = gid.y * params.width + gid.x; - const uint chroma_y = min(gid.y / 2u, params.chroma_height - 1u); - const uint near_y = (gid.y & 1u) == 0u + uchar cb; + uchar cr; + jpeg_sample_420_chroma( + cb_plane, + cr_plane, + params.chroma_width, + params.chroma_height, + gid.x, + gid.y, + cb, + cr + ); + + const uint out_idx = gid.y * params.out_stride + gid.x * 4u; + store_rgba_ycbcr(out, out_idx, y_plane[y_idx], cb, cr, params.alpha); +} + +kernel void jpeg_pack_420_rgba_texture( + device const uchar *y_plane [[buffer(0)]], + device const uchar *cb_plane [[buffer(1)]], + device const uchar *cr_plane [[buffer(2)]], + constant JpegTexturePackBatchParams ¶ms [[buffer(3)]], + texture2d out [[texture(0)]], + uint2 gid [[thread_position_in_grid]] +) { + const uint x0 = gid.x * 2u; + const uint y0 = gid.y * 2u; + if (x0 >= params.width || y0 >= params.height) { + return; + } + + const uint y_plane_base = params.tile_index * params.width * params.height; + const uint chroma_plane_base = params.tile_index * params.chroma_width * params.chroma_height; + device const uchar *tile_y_plane = y_plane + y_plane_base; + device const uchar *tile_cb_plane = cb_plane + chroma_plane_base; + device const uchar *tile_cr_plane = cr_plane + chroma_plane_base; + + const uint chroma_y = min(y0 / 2u, params.chroma_height - 1u); + const uint near_y = (y0 & 1u) == 0u ? (chroma_y == 0u ? 0u : chroma_y - 1u) : min(chroma_y + 1u, params.chroma_height - 1u); - device const uchar *curr_cb = cb_plane + chroma_y * params.chroma_width; - device const uchar *near_cb = cb_plane + near_y * params.chroma_width; - device const uchar *curr_cr = cr_plane + chroma_y * params.chroma_width; - device const uchar *near_cr = cr_plane + near_y * params.chroma_width; + device const uchar *curr_cb = tile_cb_plane + chroma_y * params.chroma_width; + device const uchar *near_cb = tile_cb_plane + near_y * params.chroma_width; + device const uchar *curr_cr = tile_cr_plane + chroma_y * params.chroma_width; + device const uchar *near_cr = tile_cr_plane + near_y * params.chroma_width; - const uchar cb = h2v2_sample(near_cb, curr_cb, params.chroma_width, gid.x); - const uchar cr = h2v2_sample(near_cr, curr_cr, params.chroma_width, gid.x); - const int y = int(y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb0; + uchar cb1; + uchar cr0; + uchar cr1; + h2v2_sample_even_pair(near_cb, curr_cb, params.chroma_width, x0, cb0, cb1); + h2v2_sample_even_pair(near_cr, curr_cr, params.chroma_width, x0, cr0, cr1); - const uint out_idx = gid.y * params.out_stride + gid.x * 4u; - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); - out[out_idx + 3] = uchar(params.alpha); + const uint y_idx0 = y0 * params.width + x0; + out.write( + rgba_float_ycbcr(tile_y_plane[y_idx0], cb0, cr0, params.alpha), + uint2(x0, y0) + ); + const uint x1 = x0 + 1u; + if (x1 < params.width) { + out.write( + rgba_float_ycbcr(tile_y_plane[y_idx0 + 1u], cb1, cr1, params.alpha), + uint2(x1, y0) + ); + } + + const uint y1 = y0 + 1u; + if (y1 >= params.height) { + return; + } + + const uint chroma_y1 = min(y1 / 2u, params.chroma_height - 1u); + const uint near_y1 = (y1 & 1u) == 0u + ? (chroma_y1 == 0u ? 0u : chroma_y1 - 1u) + : min(chroma_y1 + 1u, params.chroma_height - 1u); + device const uchar *curr_cb1 = tile_cb_plane + chroma_y1 * params.chroma_width; + device const uchar *near_cb1 = tile_cb_plane + near_y1 * params.chroma_width; + device const uchar *curr_cr1 = tile_cr_plane + chroma_y1 * params.chroma_width; + device const uchar *near_cr1 = tile_cr_plane + near_y1 * params.chroma_width; + + h2v2_sample_even_pair(near_cb1, curr_cb1, params.chroma_width, x0, cb0, cb1); + h2v2_sample_even_pair(near_cr1, curr_cr1, params.chroma_width, x0, cr0, cr1); + + const uint y_idx1 = y1 * params.width + x0; + out.write( + rgba_float_ycbcr(tile_y_plane[y_idx1], cb0, cr0, params.alpha), + uint2(x0, y1) + ); + if (x1 < params.width) { + out.write( + rgba_float_ycbcr(tile_y_plane[y_idx1 + 1u], cb1, cr1, params.alpha), + uint2(x1, y1) + ); + } } kernel void jpeg_pack_422_rgb( @@ -5494,20 +6999,21 @@ kernel void jpeg_pack_422_rgb( } const uint y_idx = gid.y * params.width + gid.x; - const uint chroma_y = min(gid.y, params.chroma_height - 1u); - device const uchar *curr_cb = cb_plane + chroma_y * params.chroma_width; - device const uchar *curr_cr = cr_plane + chroma_y * params.chroma_width; - - const uchar cb = h2v1_sample(curr_cb, params.chroma_width, gid.x); - const uchar cr = h2v1_sample(curr_cr, params.chroma_width, gid.x); - const int y = int(y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb; + uchar cr; + jpeg_sample_422_chroma( + cb_plane, + cr_plane, + params.chroma_width, + params.chroma_height, + gid.x, + gid.y, + cb, + cr + ); const uint out_idx = gid.y * params.out_stride + gid.x * 3u; - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); + store_rgb_ycbcr(out, out_idx, y_plane[y_idx], cb, cr); } kernel void jpeg_pack_422_rgb_batch( @@ -5550,6 +7056,50 @@ kernel void jpeg_pack_422_rgb_batch( } } +kernel void jpeg_pack_422_rgba_texture( + device const uchar *y_plane [[buffer(0)]], + device const uchar *cb_plane [[buffer(1)]], + device const uchar *cr_plane [[buffer(2)]], + constant JpegTexturePackBatchParams ¶ms [[buffer(3)]], + texture2d out [[texture(0)]], + uint2 gid [[thread_position_in_grid]] +) { + const uint x0 = gid.x * 2u; + if (x0 >= params.width || gid.y >= params.height) { + return; + } + + const uint y_plane_base = params.tile_index * params.width * params.height; + const uint chroma_plane_base = params.tile_index * params.chroma_width * params.chroma_height; + device const uchar *tile_y_plane = y_plane + y_plane_base; + device const uchar *tile_cb_plane = cb_plane + chroma_plane_base; + device const uchar *tile_cr_plane = cr_plane + chroma_plane_base; + + const uint y_idx = gid.y * params.width + x0; + const uint chroma_y = min(gid.y, params.chroma_height - 1u); + device const uchar *curr_cb = tile_cb_plane + chroma_y * params.chroma_width; + device const uchar *curr_cr = tile_cr_plane + chroma_y * params.chroma_width; + + uchar cb0; + uchar cb1; + uchar cr0; + uchar cr1; + h2v1_sample_even_pair(curr_cb, params.chroma_width, x0, cb0, cb1); + h2v1_sample_even_pair(curr_cr, params.chroma_width, x0, cr0, cr1); + + out.write( + rgba_float_ycbcr(tile_y_plane[y_idx], cb0, cr0, params.alpha), + uint2(x0, gid.y) + ); + const uint x1 = x0 + 1u; + if (x1 < params.width) { + out.write( + rgba_float_ycbcr(tile_y_plane[y_idx + 1u], cb1, cr1, params.alpha), + uint2(x1, gid.y) + ); + } +} + kernel void jpeg_pack_422_rgba( device const uchar *y_plane [[buffer(0)]], device const uchar *cb_plane [[buffer(1)]], @@ -5563,21 +7113,21 @@ kernel void jpeg_pack_422_rgba( } const uint y_idx = gid.y * params.width + gid.x; - const uint chroma_y = min(gid.y, params.chroma_height - 1u); - device const uchar *curr_cb = cb_plane + chroma_y * params.chroma_width; - device const uchar *curr_cr = cr_plane + chroma_y * params.chroma_width; - - const uchar cb = h2v1_sample(curr_cb, params.chroma_width, gid.x); - const uchar cr = h2v1_sample(curr_cr, params.chroma_width, gid.x); - const int y = int(y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb; + uchar cr; + jpeg_sample_422_chroma( + cb_plane, + cr_plane, + params.chroma_width, + params.chroma_height, + gid.x, + gid.y, + cb, + cr + ); const uint out_idx = gid.y * params.out_stride + gid.x * 4u; - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); - out[out_idx + 3] = uchar(params.alpha); + store_rgba_ycbcr(out, out_idx, y_plane[y_idx], cb, cr, params.alpha); } kernel void jpeg_pack_422_windowed( @@ -5604,20 +7154,21 @@ kernel void jpeg_pack_422_windowed( return; } - const uint chroma_y = min(src_y, params.chroma_height - 1u); - device const uchar *curr_cb = cb_plane + chroma_y * params.chroma_width; - device const uchar *curr_cr = cr_plane + chroma_y * params.chroma_width; - - const uchar cb = h2v1_sample(curr_cb, params.chroma_width, src_x); - const uchar cr = h2v1_sample(curr_cr, params.chroma_width, src_x); - const int y = int(y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb; + uchar cr; + jpeg_sample_422_chroma( + cb_plane, + cr_plane, + params.chroma_width, + params.chroma_height, + src_x, + src_y, + cb, + cr + ); uint out_idx = gid.y * params.out_stride + gid.x * (params.out_format == OUT_RGB ? 3u : 4u); - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); + store_rgb_ycbcr(out, out_idx, y_plane[y_idx], cb, cr); if (params.out_format == OUT_RGBA) { out[out_idx + 3] = uchar(params.alpha); } @@ -5642,20 +7193,21 @@ kernel void jpeg_pack_422_windowed_rgb( } const uint y_idx = src_y * params.src_width + src_x; - const uint chroma_y = min(src_y, params.chroma_height - 1u); - device const uchar *curr_cb = cb_plane + chroma_y * params.chroma_width; - device const uchar *curr_cr = cr_plane + chroma_y * params.chroma_width; - - const uchar cb = h2v1_sample(curr_cb, params.chroma_width, src_x); - const uchar cr = h2v1_sample(curr_cr, params.chroma_width, src_x); - const int y = int(y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb; + uchar cr; + jpeg_sample_422_chroma( + cb_plane, + cr_plane, + params.chroma_width, + params.chroma_height, + src_x, + src_y, + cb, + cr + ); const uint out_idx = gid.y * params.out_stride + gid.x * 3u; - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); + store_rgb_ycbcr(out, out_idx, y_plane[y_idx], cb, cr); } kernel void jpeg_pack_422_windowed_rgb_batch( @@ -5683,21 +7235,62 @@ kernel void jpeg_pack_422_windowed_rgb_batch( device const uchar *tile_cr_plane = cr_plane + chroma_plane_base; const uint y_idx = src_y * params.src_width + src_x; - const uint chroma_y = min(src_y, params.chroma_height - 1u); - device const uchar *curr_cb = tile_cb_plane + chroma_y * params.chroma_width; - device const uchar *curr_cr = tile_cr_plane + chroma_y * params.chroma_width; - - const uchar cb = h2v1_sample(curr_cb, params.chroma_width, src_x); - const uchar cr = h2v1_sample(curr_cr, params.chroma_width, src_x); - const int y = int(tile_y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb; + uchar cr; + jpeg_sample_422_chroma( + tile_cb_plane, + tile_cr_plane, + params.chroma_width, + params.chroma_height, + src_x, + src_y, + cb, + cr + ); const uint out_base = gid.z * params.out_stride * params.height; const uint out_idx = out_base + gid.y * params.out_stride + gid.x * 3u; - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); + store_rgb_ycbcr(out, out_idx, tile_y_plane[y_idx], cb, cr); +} + +kernel void jpeg_pack_422_windowed_rgba_texture( + device const uchar *y_plane [[buffer(0)]], + device const uchar *cb_plane [[buffer(1)]], + device const uchar *cr_plane [[buffer(2)]], + constant JpegWindowedTexturePackBatchParams ¶ms [[buffer(3)]], + texture2d out [[texture(0)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height) { + return; + } + + const uint src_x = gid.x + params.src_x; + const uint src_y = gid.y + params.src_y; + if (src_x >= params.src_width || src_y >= params.src_height) { + return; + } + + const uint y_plane_base = params.tile_index * params.src_width * params.src_height; + const uint chroma_plane_base = params.tile_index * params.chroma_width * params.chroma_height; + device const uchar *tile_y_plane = y_plane + y_plane_base; + device const uchar *tile_cb_plane = cb_plane + chroma_plane_base; + device const uchar *tile_cr_plane = cr_plane + chroma_plane_base; + + const uint y_idx = src_y * params.src_width + src_x; + uchar cb; + uchar cr; + jpeg_sample_422_chroma( + tile_cb_plane, + tile_cr_plane, + params.chroma_width, + params.chroma_height, + src_x, + src_y, + cb, + cr + ); + out.write(rgba_float_ycbcr(tile_y_plane[y_idx], cb, cr, params.alpha), gid); } kernel void jpeg_pack_422_windowed_rgba( @@ -5719,21 +7312,21 @@ kernel void jpeg_pack_422_windowed_rgba( } const uint y_idx = src_y * params.src_width + src_x; - const uint chroma_y = min(src_y, params.chroma_height - 1u); - device const uchar *curr_cb = cb_plane + chroma_y * params.chroma_width; - device const uchar *curr_cr = cr_plane + chroma_y * params.chroma_width; - - const uchar cb = h2v1_sample(curr_cb, params.chroma_width, src_x); - const uchar cr = h2v1_sample(curr_cr, params.chroma_width, src_x); - const int y = int(y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb; + uchar cr; + jpeg_sample_422_chroma( + cb_plane, + cr_plane, + params.chroma_width, + params.chroma_height, + src_x, + src_y, + cb, + cr + ); const uint out_idx = gid.y * params.out_stride + gid.x * 4u; - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); - out[out_idx + 3] = uchar(params.alpha); + store_rgba_ycbcr(out, out_idx, y_plane[y_idx], cb, cr, params.alpha); } kernel void jpeg_pack_420_windowed( @@ -5760,25 +7353,21 @@ kernel void jpeg_pack_420_windowed( return; } - const uint chroma_y = min(src_y / 2u, params.chroma_height - 1u); - const uint near_y = (src_y & 1u) == 0u - ? (chroma_y == 0u ? 0u : chroma_y - 1u) - : min(chroma_y + 1u, params.chroma_height - 1u); - device const uchar *curr_cb = cb_plane + chroma_y * params.chroma_width; - device const uchar *near_cb = cb_plane + near_y * params.chroma_width; - device const uchar *curr_cr = cr_plane + chroma_y * params.chroma_width; - device const uchar *near_cr = cr_plane + near_y * params.chroma_width; - - const uchar cb = h2v2_sample(near_cb, curr_cb, params.chroma_width, src_x); - const uchar cr = h2v2_sample(near_cr, curr_cr, params.chroma_width, src_x); - const int y = int(y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb; + uchar cr; + jpeg_sample_420_chroma( + cb_plane, + cr_plane, + params.chroma_width, + params.chroma_height, + src_x, + src_y, + cb, + cr + ); uint out_idx = gid.y * params.out_stride + gid.x * (params.out_format == OUT_RGB ? 3u : 4u); - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); + store_rgb_ycbcr(out, out_idx, y_plane[y_idx], cb, cr); if (params.out_format == OUT_RGBA) { out[out_idx + 3] = uchar(params.alpha); } @@ -5803,25 +7392,21 @@ kernel void jpeg_pack_420_windowed_rgb( } const uint y_idx = src_y * params.src_width + src_x; - const uint chroma_y = min(src_y / 2u, params.chroma_height - 1u); - const uint near_y = (src_y & 1u) == 0u - ? (chroma_y == 0u ? 0u : chroma_y - 1u) - : min(chroma_y + 1u, params.chroma_height - 1u); - device const uchar *curr_cb = cb_plane + chroma_y * params.chroma_width; - device const uchar *near_cb = cb_plane + near_y * params.chroma_width; - device const uchar *curr_cr = cr_plane + chroma_y * params.chroma_width; - device const uchar *near_cr = cr_plane + near_y * params.chroma_width; - - const uchar cb = h2v2_sample(near_cb, curr_cb, params.chroma_width, src_x); - const uchar cr = h2v2_sample(near_cr, curr_cr, params.chroma_width, src_x); - const int y = int(y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb; + uchar cr; + jpeg_sample_420_chroma( + cb_plane, + cr_plane, + params.chroma_width, + params.chroma_height, + src_x, + src_y, + cb, + cr + ); const uint out_idx = gid.y * params.out_stride + gid.x * 3u; - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); + store_rgb_ycbcr(out, out_idx, y_plane[y_idx], cb, cr); } kernel void jpeg_pack_420_windowed_rgb_batch( @@ -5849,26 +7434,81 @@ kernel void jpeg_pack_420_windowed_rgb_batch( device const uchar *tile_cr_plane = cr_plane + chroma_plane_base; const uint y_idx = src_y * params.src_width + src_x; - const uint chroma_y = min(src_y / 2u, params.chroma_height - 1u); - const uint near_y = (src_y & 1u) == 0u - ? (chroma_y == 0u ? 0u : chroma_y - 1u) - : min(chroma_y + 1u, params.chroma_height - 1u); - device const uchar *curr_cb = tile_cb_plane + chroma_y * params.chroma_width; - device const uchar *near_cb = tile_cb_plane + near_y * params.chroma_width; - device const uchar *curr_cr = tile_cr_plane + chroma_y * params.chroma_width; - device const uchar *near_cr = tile_cr_plane + near_y * params.chroma_width; - - const uchar cb = h2v2_sample(near_cb, curr_cb, params.chroma_width, src_x); - const uchar cr = h2v2_sample(near_cr, curr_cr, params.chroma_width, src_x); - const int y = int(tile_y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb; + uchar cr; + jpeg_sample_420_chroma( + tile_cb_plane, + tile_cr_plane, + params.chroma_width, + params.chroma_height, + src_x, + src_y, + cb, + cr + ); const uint out_base = gid.z * params.out_stride * params.height; const uint out_idx = out_base + gid.y * params.out_stride + gid.x * 3u; - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); + store_rgb_ycbcr(out, out_idx, tile_y_plane[y_idx], cb, cr); +} + +kernel void jpeg_pack_420_windowed_rgba_texture( + device const uchar *y_plane [[buffer(0)]], + device const uchar *cb_plane [[buffer(1)]], + device const uchar *cr_plane [[buffer(2)]], + constant JpegWindowedTexturePackBatchParams ¶ms [[buffer(3)]], + texture2d out [[texture(0)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height) { + return; + } + + const uint src_x = gid.x + params.src_x; + const uint src_y = gid.y + params.src_y; + if (src_x >= params.src_width || src_y >= params.src_height) { + return; + } + + const uint y_plane_base = params.tile_index * params.src_width * params.src_height; + const uint chroma_plane_base = params.tile_index * params.chroma_width * params.chroma_height; + device const uchar *tile_y_plane = y_plane + y_plane_base; + device const uchar *tile_cb_plane = cb_plane + chroma_plane_base; + device const uchar *tile_cr_plane = cr_plane + chroma_plane_base; + + const uint y_idx = src_y * params.src_width + src_x; + uchar cb; + uchar cr; + jpeg_sample_420_chroma( + tile_cb_plane, + tile_cr_plane, + params.chroma_width, + params.chroma_height, + src_x, + src_y, + cb, + cr + ); + out.write(rgba_float_ycbcr(tile_y_plane[y_idx], cb, cr, params.alpha), gid); +} + +kernel void jpeg_copy_rgb8_to_rgba_texture( + device const uchar *rgb [[buffer(0)]], + constant JpegRgb8ToRgbaTextureParams ¶ms [[buffer(1)]], + texture2d out [[texture(0)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height) { + return; + } + + const uint idx = gid.y * params.in_stride + gid.x * 3u; + out.write(float4( + float(rgb[idx]) / 255.0f, + float(rgb[idx + 1u]) / 255.0f, + float(rgb[idx + 2u]) / 255.0f, + float(params.alpha) / 255.0f + ), gid); } kernel void jpeg_pack_420_windowed_rgba( @@ -5890,24 +7530,19 @@ kernel void jpeg_pack_420_windowed_rgba( } const uint y_idx = src_y * params.src_width + src_x; - const uint chroma_y = min(src_y / 2u, params.chroma_height - 1u); - const uint near_y = (src_y & 1u) == 0u - ? (chroma_y == 0u ? 0u : chroma_y - 1u) - : min(chroma_y + 1u, params.chroma_height - 1u); - device const uchar *curr_cb = cb_plane + chroma_y * params.chroma_width; - device const uchar *near_cb = cb_plane + near_y * params.chroma_width; - device const uchar *curr_cr = cr_plane + chroma_y * params.chroma_width; - device const uchar *near_cr = cr_plane + near_y * params.chroma_width; - - const uchar cb = h2v2_sample(near_cb, curr_cb, params.chroma_width, src_x); - const uchar cr = h2v2_sample(near_cr, curr_cr, params.chroma_width, src_x); - const int y = int(y_plane[y_idx]); - const int cb_centered = int(cb) - 128; - const int cr_centered = int(cr) - 128; + uchar cb; + uchar cr; + jpeg_sample_420_chroma( + cb_plane, + cr_plane, + params.chroma_width, + params.chroma_height, + src_x, + src_y, + cb, + cr + ); const uint out_idx = gid.y * params.out_stride + gid.x * 4u; - out[out_idx] = clamp_u8(y + ((91881 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 1] = clamp_u8(y - ((22554 * cb_centered + 46802 * cr_centered + (1 << 15)) >> 16)); - out[out_idx + 2] = clamp_u8(y + ((116130 * cb_centered + (1 << 15)) >> 16)); - out[out_idx + 3] = uchar(params.alpha); + store_rgba_ycbcr(out, out_idx, y_plane[y_idx], cb, cr, params.alpha); } diff --git a/crates/signinum-jpeg-metal/src/viewport.rs b/crates/j2k-jpeg-metal/src/viewport.rs similarity index 50% rename from crates/signinum-jpeg-metal/src/viewport.rs rename to crates/j2k-jpeg-metal/src/viewport.rs index 4f5eb779..3a7cb27a 100644 --- a/crates/signinum-jpeg-metal/src/viewport.rs +++ b/crates/j2k-jpeg-metal/src/viewport.rs @@ -1,38 +1,71 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_core::{BackendRequest, Downscale, PixelFormat, Rect}; -use signinum_jpeg::adapter::{ - build_metal_fast420_packet_for_decoder, build_metal_fast422_packet_for_decoder, - build_metal_fast444_packet_for_decoder, +use j2k_core::{BackendRequest, Downscale, PixelFormat, Rect}; +#[cfg(target_os = "macos")] +use j2k_jpeg::adapter::decoder_bytes; +use j2k_jpeg::adapter::{ + build_fast420_packet_for_decoder, build_fast422_packet_for_decoder, + build_fast444_packet_for_decoder, JpegFast420PacketV1, JpegFast422PacketV1, + JpegFast444PacketV1, }; -use signinum_jpeg::{Decoder as CpuDecoder, Rect as JpegRect, ScratchPool}; +#[cfg(target_os = "macos")] +use j2k_jpeg::ColorSpace as JpegColorSpace; +use j2k_jpeg::{Decoder as CpuDecoder, Rect as JpegRect, ScratchPool}; use crate::{batch, routing, Error, Surface}; +#[cfg(target_os = "macos")] +use crate::{ + Codec, MetalBackendSession, MetalBatchOutputBuffer, MetalBatchTextureOutput, MetalTextureTile, +}; const VIEWPORT_TILE_EDGE: u32 = 96; const VIEWPORT_TILE_COLS: u32 = 6; const VIEWPORT_TILE_ROWS: u32 = 2; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// One source-to-destination region in a composed viewport. pub struct ViewportTile { + /// Source region in the JPEG image before downscaling. pub source_roi: Rect, + /// Destination rectangle in the viewport after downscaling. pub dest: Rect, } #[derive(Debug, Clone, PartialEq, Eq)] +/// Planned viewport decode made of one or more source tiles. pub struct ViewportWorkload { + /// Downscale factor applied to every source tile. pub scale: Downscale, + /// Output viewport dimensions in pixels. pub viewport_dims: (u32, u32), + /// Tiles to decode and place into the viewport. pub tiles: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Execution strategy selected for a viewport decode. pub enum ViewportSurfaceStrategy { + /// Decode each tile on CPU and composite into a host viewport. CpuComposite, + /// Decode one contiguous source region on CPU. CpuContiguous, + /// Decode or upload through Metal while compositing multiple source tiles. HybridComposite, + /// Decode one contiguous source region through the Metal path. HybridContiguous, } +#[cfg(target_os = "macos")] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Resident Metal output strategy selected for a reusable viewport decode. +pub enum ViewportResidentOutputStrategy { + /// Decode the contiguous source bounds through the direct resident batch path. + DirectContiguous, + /// Decode component rows into resident planes and pack the composed viewport. + Composite, +} + +/// Compute the bounding source rectangle covering all tiles in a workload. pub fn viewport_source_bounds(workload: &ViewportWorkload) -> Rect { let mut min_x = u32::MAX; let mut min_y = u32::MAX; @@ -53,6 +86,7 @@ pub fn viewport_source_bounds(workload: &ViewportWorkload) -> Rect { } } +/// Return whether the workload covers a contiguous viewport without overlaps. pub fn is_contiguous_viewport_workload(workload: &ViewportWorkload) -> bool { if workload.tiles.is_empty() { return false; @@ -106,6 +140,7 @@ pub fn is_contiguous_viewport_workload(workload: &ViewportWorkload) -> bool { area_sum == viewport_area } +/// Choose the backend strategy for a workload without inspecting JPEG capabilities. pub fn choose_viewport_surface_strategy( workload: &ViewportWorkload, backend: BackendRequest, @@ -186,19 +221,35 @@ fn choose_viewport_surface_strategy_for_decoder( #[cfg(target_os = "macos")] fn has_direct_viewport_packet(decoder: &CpuDecoder<'_>) -> bool { - build_metal_fast444_packet_for_decoder(decoder).is_ok() - || build_metal_fast422_packet_for_decoder(decoder).is_ok() - || build_metal_fast420_packet_for_decoder(decoder).is_ok() + build_fast444_packet_for_decoder(decoder).is_ok() + || build_fast422_packet_for_decoder(decoder).is_ok() + || build_fast420_packet_for_decoder(decoder).is_ok() } fn validate_explicit_metal_viewport_request( decoder: &CpuDecoder<'_>, workload: &ViewportWorkload, +) -> Result<(), Error> { + let fast444_packet = build_fast444_packet_for_decoder(decoder).ok(); + let fast422_packet = build_fast422_packet_for_decoder(decoder).ok(); + let fast420_packet = build_fast420_packet_for_decoder(decoder).ok(); + validate_explicit_metal_viewport_request_with_packets( + decoder, + workload, + fast444_packet.as_ref(), + fast422_packet.as_ref(), + fast420_packet.as_ref(), + ) +} + +fn validate_explicit_metal_viewport_request_with_packets( + decoder: &CpuDecoder<'_>, + workload: &ViewportWorkload, + fast444_packet: Option<&JpegFast444PacketV1>, + fast422_packet: Option<&JpegFast422PacketV1>, + fast420_packet: Option<&JpegFast420PacketV1>, ) -> Result<(), Error> { let source = viewport_source_bounds(workload); - let fast444_packet = build_metal_fast444_packet_for_decoder(decoder).ok(); - let fast422_packet = build_metal_fast422_packet_for_decoder(decoder).ok(); - let fast420_packet = build_metal_fast420_packet_for_decoder(decoder).ok(); let capabilities = routing::JpegMetalCapabilities::for_request( decoder, PixelFormat::Rgb8, @@ -206,9 +257,9 @@ fn validate_explicit_metal_viewport_request( roi: source, scale: workload.scale, }, - fast444_packet.as_ref(), - fast422_packet.as_ref(), - fast420_packet.as_ref(), + fast444_packet, + fast422_packet, + fast420_packet, ); let decision = routing::decide_route(BackendRequest::Metal, capabilities); if let Some(err) = routing::decision_error(decision) { @@ -218,6 +269,85 @@ fn validate_explicit_metal_viewport_request( Ok(()) } +#[cfg(target_os = "macos")] +fn validate_resident_viewport_composition_request( + decoder: &CpuDecoder<'_>, + workload: &ViewportWorkload, +) -> Result<(), Error> { + if workload.tiles.is_empty() { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal resident viewport output requires at least one viewport tile", + }); + } + if matches!( + decoder.info().color_space, + JpegColorSpace::Cmyk | JpegColorSpace::Ycck + ) { + return Err(Error::UnsupportedMetalRequest { + reason: + "JPEG Metal resident viewport composition does not support CMYK/YCCK JPEG output", + }); + } + + for tile in &workload.tiles { + let dims = tile.source_roi.scaled_covering(workload.scale); + if (dims.w, dims.h) != (tile.dest.w, tile.dest.h) { + return Err(Error::UnsupportedMetalRequest { + reason: + "JPEG Metal resident viewport tile dimensions do not match destination rect", + }); + } + if tile.dest.x.saturating_add(tile.dest.w) > workload.viewport_dims.0 + || tile.dest.y.saturating_add(tile.dest.h) > workload.viewport_dims.1 + { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal resident viewport destination exceeds viewport dimensions", + }); + } + } + + Ok(()) +} + +#[cfg(target_os = "macos")] +/// Choose the resident Metal strategy for a reusable viewport output request. +pub fn choose_resizable_metal_viewport_strategy( + decoder: &CpuDecoder<'_>, + workload: &ViewportWorkload, +) -> Result { + if is_contiguous_viewport_workload(workload) + && validate_explicit_metal_viewport_request(decoder, workload).is_ok() + { + return Ok(ViewportResidentOutputStrategy::DirectContiguous); + } + + validate_resident_viewport_composition_request(decoder, workload)?; + Ok(ViewportResidentOutputStrategy::Composite) +} + +#[cfg(target_os = "macos")] +fn choose_resizable_metal_viewport_strategy_for_decoder( + decoder: &crate::Decoder<'_>, + workload: &ViewportWorkload, +) -> Result { + if is_contiguous_viewport_workload(workload) + && validate_explicit_metal_viewport_request_with_packets( + decoder.inner(), + workload, + decoder.fast444_packet(), + decoder.fast422_packet(), + decoder.fast420_packet(), + ) + .is_ok() + { + return Ok(ViewportResidentOutputStrategy::DirectContiguous); + } + + validate_resident_viewport_composition_request(decoder.inner(), workload)?; + Ok(ViewportResidentOutputStrategy::Composite) +} + +/// Suggest a fixed-size centered viewport workload for an image. pub fn suggest_viewport_workload(dimensions: (u32, u32)) -> Option { let scales = [ Downscale::Eighth, @@ -280,6 +410,7 @@ pub fn suggest_viewport_workload(dimensions: (u32, u32)) -> Option, pool: &mut ScratchPool, @@ -326,6 +457,7 @@ pub fn compose_viewport_cpu( Ok(viewport) } +/// Decode the contiguous source region for a workload into host bytes. pub fn decode_viewport_region_cpu( decoder: &CpuDecoder<'_>, pool: &mut ScratchPool, @@ -346,6 +478,7 @@ pub fn decode_viewport_region_cpu( Ok(viewport) } +/// Decode a viewport workload into a surface using the requested backend policy. pub fn decode_viewport_to_surface( decoder: &CpuDecoder<'_>, pool: &mut ScratchPool, @@ -377,6 +510,7 @@ pub fn decode_viewport_to_surface( } #[cfg(target_os = "macos")] +/// Decode the contiguous source region on CPU and upload it to a surface. pub fn decode_viewport_region_cpu_to_surface( decoder: &CpuDecoder<'_>, pool: &mut ScratchPool, @@ -387,11 +521,12 @@ pub fn decode_viewport_region_cpu_to_surface( bytes, workload.viewport_dims, PixelFormat::Rgb8, - signinum_core::BackendRequest::Cpu, + j2k_core::BackendRequest::Cpu, ) } #[cfg(not(target_os = "macos"))] +/// Decode the contiguous source region on CPU and return a host-backed surface. pub fn decode_viewport_region_cpu_to_surface( decoder: &CpuDecoder<'_>, pool: &mut ScratchPool, @@ -402,11 +537,12 @@ pub fn decode_viewport_region_cpu_to_surface( bytes, workload.viewport_dims, PixelFormat::Rgb8, - signinum_core::BackendRequest::Cpu, + j2k_core::BackendRequest::Cpu, ) } #[cfg(target_os = "macos")] +/// Decode and composite viewport tiles on CPU, then upload to a surface. pub fn compose_viewport_cpu_to_surface( decoder: &CpuDecoder<'_>, pool: &mut ScratchPool, @@ -426,11 +562,12 @@ pub fn compose_viewport_cpu_to_surface( bytes, viewport_dims, PixelFormat::Rgb8, - signinum_core::BackendRequest::Cpu, + j2k_core::BackendRequest::Cpu, ) } #[cfg(not(target_os = "macos"))] +/// Decode and composite viewport tiles on CPU into a host-backed surface. pub fn compose_viewport_cpu_to_surface( decoder: &CpuDecoder<'_>, pool: &mut ScratchPool, @@ -450,11 +587,12 @@ pub fn compose_viewport_cpu_to_surface( bytes, viewport_dims, PixelFormat::Rgb8, - signinum_core::BackendRequest::Cpu, + j2k_core::BackendRequest::Cpu, ) } #[cfg(target_os = "macos")] +/// Compose a multi-tile viewport through the Metal hybrid path. pub fn compose_viewport_hybrid( decoder: &CpuDecoder<'_>, pool: &mut ScratchPool, @@ -466,6 +604,268 @@ pub fn compose_viewport_hybrid( } #[cfg(target_os = "macos")] +/// Compose a viewport workload into a reusable caller-owned Metal buffer. +/// +/// This path supports sparse and non-contiguous workloads. It decodes component +/// rows into reusable Metal plane buffers, resizes `output` to one RGB8 viewport +/// slot, and packs the composed viewport directly into that caller-owned buffer. +pub fn compose_viewport_to_resizable_metal_buffer_with_session( + decoder: &CpuDecoder<'_>, + pool: &mut ScratchPool, + workload: &ViewportWorkload, + output: &mut MetalBatchOutputBuffer, + session: &MetalBackendSession, +) -> Result { + validate_resident_viewport_composition_request(decoder, workload)?; + output.ensure_rgb8_tiles(session, workload.viewport_dims, 1)?; + crate::compute::compose_rgb_viewport_from_regions_into_output_with_session( + decoder, + pool, + workload.scale, + workload.viewport_dims, + &workload.tiles, + output, + session, + ) +} + +#[cfg(target_os = "macos")] +/// Compose a viewport workload into a reusable caller-owned Metal texture. +/// +/// This path supports sparse and non-contiguous workloads. It decodes component +/// rows into reusable Metal plane buffers, resizes `output` to one RGBA8 +/// viewport slot, and packs the composed viewport directly into that +/// caller-owned texture. +pub fn compose_viewport_to_resizable_metal_textures_with_session( + decoder: &CpuDecoder<'_>, + pool: &mut ScratchPool, + workload: &ViewportWorkload, + output: &mut MetalBatchTextureOutput, + session: &MetalBackendSession, +) -> Result { + validate_resident_viewport_composition_request(decoder, workload)?; + output.ensure_rgba8_tiles(session, workload.viewport_dims, 1)?; + crate::compute::compose_rgb_viewport_from_regions_into_textures_with_session( + decoder, + pool, + workload.scale, + workload.viewport_dims, + &workload.tiles, + output, + session, + ) +} + +#[cfg(target_os = "macos")] +/// Decode any viewport workload into a reusable caller-owned Metal buffer. +/// +/// Contiguous workloads use the direct resident region-scaled batch path when +/// eligible. Sparse or unsupported direct shapes use resident component-row +/// composition into the same caller-owned RGB8 buffer. +pub fn decode_viewport_to_resizable_metal_buffer_with_session( + decoder: &CpuDecoder<'_>, + pool: &mut ScratchPool, + workload: &ViewportWorkload, + output: &mut MetalBatchOutputBuffer, + session: &MetalBackendSession, +) -> Result { + match choose_resizable_metal_viewport_strategy(decoder, workload)? { + ViewportResidentOutputStrategy::DirectContiguous => { + decode_viewport_region_to_resizable_metal_buffer_with_session( + decoder_bytes(decoder), + workload, + output, + session, + ) + } + ViewportResidentOutputStrategy::Composite => { + compose_viewport_to_resizable_metal_buffer_with_session( + decoder, pool, workload, output, session, + ) + } + } +} + +#[cfg(target_os = "macos")] +/// Decode any viewport workload into reusable caller-owned Metal textures. +/// +/// Contiguous workloads use the direct resident region-scaled texture batch path +/// when eligible. Sparse or unsupported direct shapes use resident component-row +/// composition into the same caller-owned RGBA8 texture. +pub fn decode_viewport_to_resizable_metal_textures_with_session( + decoder: &CpuDecoder<'_>, + pool: &mut ScratchPool, + workload: &ViewportWorkload, + output: &mut MetalBatchTextureOutput, + session: &MetalBackendSession, +) -> Result { + match choose_resizable_metal_viewport_strategy(decoder, workload)? { + ViewportResidentOutputStrategy::DirectContiguous => { + decode_viewport_region_to_resizable_metal_textures_with_session( + decoder_bytes(decoder), + workload, + output, + session, + ) + } + ViewportResidentOutputStrategy::Composite => { + compose_viewport_to_resizable_metal_textures_with_session( + decoder, pool, workload, output, session, + ) + } + } +} + +#[cfg(target_os = "macos")] +/// Decode any viewport workload into a reusable caller-owned Metal buffer using +/// an already parsed Metal decoder wrapper. +/// +/// Contiguous workloads use the wrapper's cached fast-packet state for direct +/// resident region-scaled decode. Sparse or unsupported direct shapes use +/// resident component-row composition through the wrapper's CPU decoder. +pub fn decode_viewport_to_resizable_metal_buffer_with_decoder_session( + decoder: &crate::Decoder<'_>, + pool: &mut ScratchPool, + workload: &ViewportWorkload, + output: &mut MetalBatchOutputBuffer, + session: &MetalBackendSession, +) -> Result { + match choose_resizable_metal_viewport_strategy_for_decoder(decoder, workload)? { + ViewportResidentOutputStrategy::DirectContiguous => { + decode_viewport_region_to_resizable_metal_buffer_with_decoder_session( + decoder, workload, output, session, + ) + } + ViewportResidentOutputStrategy::Composite => { + compose_viewport_to_resizable_metal_buffer_with_session( + decoder.inner(), + pool, + workload, + output, + session, + ) + } + } +} + +#[cfg(target_os = "macos")] +/// Decode any viewport workload into reusable caller-owned Metal textures using +/// an already parsed Metal decoder wrapper. +/// +/// Contiguous workloads use the wrapper's cached fast-packet state for direct +/// resident region-scaled decode. Sparse or unsupported direct shapes use +/// resident component-row composition through the wrapper's CPU decoder. +pub fn decode_viewport_to_resizable_metal_textures_with_decoder_session( + decoder: &crate::Decoder<'_>, + pool: &mut ScratchPool, + workload: &ViewportWorkload, + output: &mut MetalBatchTextureOutput, + session: &MetalBackendSession, +) -> Result { + match choose_resizable_metal_viewport_strategy_for_decoder(decoder, workload)? { + ViewportResidentOutputStrategy::DirectContiguous => { + decode_viewport_region_to_resizable_metal_textures_with_decoder_session( + decoder, workload, output, session, + ) + } + ViewportResidentOutputStrategy::Composite => { + compose_viewport_to_resizable_metal_textures_with_session( + decoder.inner(), + pool, + workload, + output, + session, + ) + } + } +} + +#[cfg(target_os = "macos")] +fn decode_viewport_region_to_resizable_metal_buffer_with_decoder_session( + decoder: &crate::Decoder<'_>, + workload: &ViewportWorkload, + output: &mut MetalBatchOutputBuffer, + session: &MetalBackendSession, +) -> Result { + validate_explicit_metal_viewport_request_with_packets( + decoder.inner(), + workload, + decoder.fast444_packet(), + decoder.fast422_packet(), + decoder.fast420_packet(), + )?; + if !is_contiguous_viewport_workload(workload) { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal reusable viewport output currently requires a contiguous viewport workload", + }); + } + + let source = viewport_source_bounds(workload); + let scaled = source.scaled_covering(workload.scale); + output.ensure_rgb8_tiles(session, (scaled.w, scaled.h), 1)?; + let request = decoder + .rgb8_region_scaled_metal_request(source, workload.scale) + .with_output_slot(0); + let requests = [request]; + let mut surfaces = crate::compute::decode_region_scaled_rgb8_batch_into_output_with_session( + &requests, output, session, + )? + .ok_or(Error::UnsupportedMetalRequest { + reason: "JPEG Metal reusable viewport output currently supports RGB8 fast 4:2:0, 4:2:2, or 4:4:4 inputs", + })?; + let Some(surface) = surfaces.pop() else { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal reusable viewport output did not produce a surface", + }); + }; + debug_assert!(surfaces.is_empty()); + surface +} + +#[cfg(target_os = "macos")] +fn decode_viewport_region_to_resizable_metal_textures_with_decoder_session( + decoder: &crate::Decoder<'_>, + workload: &ViewportWorkload, + output: &mut MetalBatchTextureOutput, + session: &MetalBackendSession, +) -> Result { + validate_explicit_metal_viewport_request_with_packets( + decoder.inner(), + workload, + decoder.fast444_packet(), + decoder.fast422_packet(), + decoder.fast420_packet(), + )?; + if !is_contiguous_viewport_workload(workload) { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal reusable viewport texture output currently requires a contiguous viewport workload", + }); + } + + let source = viewport_source_bounds(workload); + let scaled = source.scaled_covering(workload.scale); + output.ensure_rgba8_tiles(session, (scaled.w, scaled.h), 1)?; + let request = decoder + .rgb8_region_scaled_metal_request(source, workload.scale) + .with_output_slot(0); + let requests = [request]; + let mut tiles = crate::compute::decode_region_scaled_rgb8_batch_into_textures_with_session( + &requests, output, session, + )? + .ok_or(Error::UnsupportedMetalRequest { + reason: "JPEG Metal reusable viewport texture output currently supports RGB8 fast 4:2:0, 4:2:2, or 4:4:4 inputs", + })?; + let Some(tile) = tiles.pop() else { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal reusable viewport texture output did not produce a tile", + }); + }; + debug_assert!(tiles.is_empty()); + tile +} + +#[cfg(target_os = "macos")] +/// Decode a contiguous viewport region through the Metal hybrid path. pub fn decode_viewport_region_hybrid( decoder: &CpuDecoder<'_>, pool: &mut ScratchPool, @@ -473,13 +873,13 @@ pub fn decode_viewport_region_hybrid( ) -> Result { let use_direct_kernel = decoder.info().restart_interval.is_some(); let fast444_packet = use_direct_kernel - .then(|| build_metal_fast444_packet_for_decoder(decoder).ok()) + .then(|| build_fast444_packet_for_decoder(decoder).ok()) .flatten(); let fast422_packet = use_direct_kernel - .then(|| build_metal_fast422_packet_for_decoder(decoder).ok()) + .then(|| build_fast422_packet_for_decoder(decoder).ok()) .flatten(); let fast420_packet = use_direct_kernel - .then(|| build_metal_fast420_packet_for_decoder(decoder).ok()) + .then(|| build_fast420_packet_for_decoder(decoder).ok()) .flatten(); crate::compute::decode_region_scaled_to_surface( decoder, @@ -494,6 +894,7 @@ pub fn decode_viewport_region_hybrid( } #[cfg(not(target_os = "macos"))] +/// Return `Error::MetalUnavailable` for hybrid viewport decode requests. pub fn decode_viewport_region_hybrid( _decoder: &CpuDecoder<'_>, _pool: &mut ScratchPool, @@ -502,7 +903,84 @@ pub fn decode_viewport_region_hybrid( Err(Error::MetalUnavailable) } +#[cfg(target_os = "macos")] +/// Decode a contiguous viewport workload into a reusable caller-owned Metal buffer. +/// +/// This is the resident-output counterpart to `decode_viewport_region_hybrid`: +/// it rejects non-contiguous workloads instead of compositing, resizes `output` +/// to one RGB8 viewport slot, and returns a `MetalResidentDecode` surface +/// backed by that caller-owned allocation. +pub fn decode_viewport_region_to_resizable_metal_buffer_with_session( + input: &[u8], + workload: &ViewportWorkload, + output: &mut MetalBatchOutputBuffer, + session: &MetalBackendSession, +) -> Result { + let decoder = CpuDecoder::new(input)?; + validate_explicit_metal_viewport_request(&decoder, workload)?; + if !is_contiguous_viewport_workload(workload) { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal reusable viewport output currently requires a contiguous viewport workload", + }); + } + + let mut surfaces = + Codec::decode_rgb8_region_scaled_batch_into_resizable_metal_buffer_with_session( + &[input], + viewport_source_bounds(workload), + workload.scale, + output, + session, + )?; + let Some(surface) = surfaces.pop() else { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal reusable viewport output did not produce a surface", + }); + }; + debug_assert!(surfaces.is_empty()); + surface +} + +#[cfg(target_os = "macos")] +/// Decode a contiguous viewport workload into reusable caller-owned Metal textures. +/// +/// This is the texture-output counterpart to +/// `decode_viewport_region_to_resizable_metal_buffer_with_session`: it rejects +/// non-contiguous workloads, resizes `output` to one RGBA8 viewport slot, and +/// returns a tile backed by that caller-owned texture. +pub fn decode_viewport_region_to_resizable_metal_textures_with_session( + input: &[u8], + workload: &ViewportWorkload, + output: &mut MetalBatchTextureOutput, + session: &MetalBackendSession, +) -> Result { + let decoder = CpuDecoder::new(input)?; + validate_explicit_metal_viewport_request(&decoder, workload)?; + if !is_contiguous_viewport_workload(workload) { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal reusable viewport texture output currently requires a contiguous viewport workload", + }); + } + + let mut tiles = + Codec::decode_rgb8_region_scaled_batch_into_resizable_metal_textures_with_session( + &[input], + viewport_source_bounds(workload), + workload.scale, + output, + session, + )?; + let Some(tile) = tiles.pop() else { + return Err(Error::UnsupportedMetalRequest { + reason: "JPEG Metal reusable viewport texture output did not produce a tile", + }); + }; + debug_assert!(tiles.is_empty()); + tile +} + #[cfg(not(target_os = "macos"))] +/// Return `Error::MetalUnavailable` for hybrid viewport composition requests. pub fn compose_viewport_hybrid( _decoder: &CpuDecoder<'_>, _pool: &mut ScratchPool, diff --git a/crates/signinum-jpeg-metal/tests/batch.rs b/crates/j2k-jpeg-metal/tests/batch.rs similarity index 93% rename from crates/signinum-jpeg-metal/tests/batch.rs rename to crates/j2k-jpeg-metal/tests/batch.rs index 235301ca..b3a2cfc5 100644 --- a/crates/signinum-jpeg-metal/tests/batch.rs +++ b/crates/j2k-jpeg-metal/tests/batch.rs @@ -1,11 +1,11 @@ -use signinum_core::{ +use j2k_core::{ BackendKind, BackendRequest, CodecError, DecoderContext, DeviceSubmission, DeviceSurface, Downscale, PixelFormat, Rect, TileBatchDecodeDevice, TileBatchDecodeSubmit, }; -use signinum_jpeg::{Decoder as CpuDecoder, DecoderContext as JpegDecoderContext}; +use j2k_jpeg::{Decoder as CpuDecoder, DecoderContext as JpegDecoderContext}; #[cfg(target_os = "macos")] -use signinum_jpeg_metal::JpegTileBatch; -use signinum_jpeg_metal::{Codec, MetalSession, ScratchPool}; +use j2k_jpeg_metal::JpegTileBatch; +use j2k_jpeg_metal::{Codec, MetalSession, ScratchPool}; const BASELINE_420: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); const BASELINE_420_RESTART: &[u8] = @@ -77,7 +77,7 @@ fn tile_region_device_decode_has_expected_dimensions() { .expect("cpu decoder") .decode_region_scaled( PixelFormat::Rgb8, - signinum_jpeg::Rect { + j2k_jpeg::Rect { x: roi.x, y: roi.y, w: roi.w, @@ -131,7 +131,7 @@ fn compatible_tile_submits_flush_once() { assert_eq!(surface.as_bytes(), expected.as_slice()); } - assert_eq!(session.submissions(), 1); + assert_eq!(session.submissions().expect("session submissions"), 1); } #[cfg(target_os = "macos")] @@ -157,7 +157,7 @@ fn jpeg_tile_batch_api_decodes_full_tiles_in_submission_order() { 1 ); assert_eq!(batch.len(), 2); - assert_eq!(batch.submissions(), 0); + assert_eq!(batch.submissions().expect("batch submissions"), 0); let surfaces = batch.decode_all().expect("decode JPEG tile batch"); @@ -199,7 +199,7 @@ fn auto_small_restart_tile_batch_stays_cpu_surface() { assert_eq!(surface.as_bytes(), expected.as_slice()); } - assert_eq!(session.submissions(), 1); + assert_eq!(session.submissions().expect("session submissions"), 1); } #[cfg(target_os = "macos")] @@ -233,7 +233,7 @@ fn auto_restart_wsi_tile_batch_uses_metal_at_threshold() { assert_eq!(surface.as_bytes(), expected.as_slice()); } - assert_eq!(session.submissions(), 1); + assert_eq!(session.submissions().expect("session submissions"), 1); } #[cfg(target_os = "macos")] @@ -253,7 +253,7 @@ fn compatible_region_scaled_tile_submits_flush_once() { .expect("cpu decoder") .decode_region_scaled( PixelFormat::Rgb8, - signinum_jpeg::Rect { + j2k_jpeg::Rect { x: roi.x, y: roi.y, w: roi.w, @@ -286,7 +286,7 @@ fn compatible_region_scaled_tile_submits_flush_once() { assert_eq!(surface.as_bytes(), expected.as_slice()); } - assert_eq!(session.submissions(), 1); + assert_eq!(session.submissions().expect("session submissions"), 1); } #[test] @@ -330,7 +330,7 @@ fn explicit_metal_tile_unsupported_shape_is_rejected() { ); match result { - Err(signinum_jpeg_metal::Error::UnsupportedMetalRequest { reason }) => { + Err(j2k_jpeg_metal::Error::UnsupportedMetalRequest { reason }) => { assert!(reason.contains("JPEG Metal")); } Err(other) => panic!("unexpected explicit Metal tile error: {other:?}"), @@ -375,10 +375,10 @@ fn cuda_tile_request_remains_unsupported_backend() { ); match result { - Err(signinum_jpeg_metal::Error::UnsupportedBackend { + Err(j2k_jpeg_metal::Error::UnsupportedBackend { request: BackendRequest::Cuda, }) => {} - Err(signinum_jpeg_metal::Error::UnsupportedMetalRequest { reason }) => { + Err(j2k_jpeg_metal::Error::UnsupportedMetalRequest { reason }) => { panic!("CUDA must not be reported as a Metal request: {reason}") } Err(other) => panic!("unexpected CUDA tile error: {other:?}"), @@ -404,7 +404,7 @@ fn non_macos_explicit_metal_tile_decode_is_unavailable() { assert!(matches!( result, - Err(signinum_jpeg_metal::Error::MetalUnavailable) + Err(j2k_jpeg_metal::Error::MetalUnavailable) )); } @@ -438,5 +438,5 @@ fn incompatible_shapes_split_batches() { let _ = full.wait().expect("full wait"); let _ = scaled.wait().expect("scaled wait"); - assert_eq!(session.submissions(), 2); + assert_eq!(session.submissions().expect("session submissions"), 2); } diff --git a/crates/signinum-jpeg-metal/tests/core_traits.rs b/crates/j2k-jpeg-metal/tests/core_traits.rs similarity index 89% rename from crates/signinum-jpeg-metal/tests/core_traits.rs rename to crates/j2k-jpeg-metal/tests/core_traits.rs index 32cde5fa..d4101246 100644 --- a/crates/signinum-jpeg-metal/tests/core_traits.rs +++ b/crates/j2k-jpeg-metal/tests/core_traits.rs @@ -1,11 +1,12 @@ #![cfg(target_os = "macos")] -use signinum_core::{ +use j2k_core::{ BackendKind, BackendRequest, CodecError, DeviceSubmission, DeviceSurface, Downscale, ImageDecode, ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, Rect, + TileBatchDecodeManyDevice, }; -use signinum_jpeg_metal::{ - Decoder, Error, MetalBackendSession, MetalSession, ScratchPool, SurfaceResidency, +use j2k_jpeg_metal::{ + Codec, Decoder, Error, MetalBackendSession, MetalSession, ScratchPool, SurfaceResidency, }; const BASELINE_420: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); @@ -126,9 +127,34 @@ fn fast422_decode_to_metal_matches_cpu_decode_bytes() { assert_eq!(surface.as_bytes(), host.as_slice()); } +#[test] +fn codec_many_device_decode_batches_full_tiles_to_metal_surfaces() { + let mut ctx = j2k_core::DecoderContext::new(); + let mut pool = ScratchPool::new(); + let inputs = [BASELINE_420, BASELINE_422]; + + let surfaces = ::decode_tiles_to_device( + &mut ctx, + &mut pool, + &inputs, + PixelFormat::Rgb8, + BackendRequest::Metal, + ) + .expect("full-tile Metal batch decode"); + + assert_eq!(surfaces.len(), 2); + assert_eq!(surfaces[0].dimensions(), (16, 16)); + assert_eq!(surfaces[1].dimensions(), (16, 8)); + assert!(surfaces.iter().all(|surface| { + surface.backend_kind() == BackendKind::Metal + && surface.residency() == SurfaceResidency::MetalResidentDecode + && surface.pixel_format() == PixelFormat::Rgb8 + })); +} + #[test] fn region_and_scaled_metal_bytes_match_cpu_decode() { - let roi = signinum_core::Rect { + let roi = j2k_core::Rect { x: 4, y: 4, w: 8, @@ -156,7 +182,7 @@ fn region_and_scaled_metal_bytes_match_cpu_decode() { let scaled_surface = metal_decoder .decode_scaled_to_device( PixelFormat::Rgb8, - signinum_core::Downscale::Quarter, + j2k_core::Downscale::Quarter, BackendRequest::Metal, ) .expect("scaled surface"); @@ -167,7 +193,7 @@ fn region_and_scaled_metal_bytes_match_cpu_decode() { &mut scaled_host, 4 * 3, PixelFormat::Rgb8, - signinum_core::Downscale::Quarter, + j2k_core::Downscale::Quarter, ) .expect("cpu scaled"); assert_eq!(scaled_surface.as_bytes(), scaled_host.as_slice()); @@ -203,7 +229,7 @@ fn region_scaled_metal_bytes_match_cpu_decode() { &mut host, scaled.w as usize * 3, PixelFormat::Rgb8, - signinum_jpeg::Rect { + j2k_jpeg::Rect { x: roi.x, y: roi.y, w: roi.w, @@ -243,7 +269,7 @@ fn region_scaled_submit_trait_returns_metal_surface() { assert_eq!(surface.backend_kind(), BackendKind::Metal); assert_eq!(surface.dimensions(), (3, 3)); - assert!(session.submissions() >= 1); + assert!(session.submissions().expect("session submissions") >= 1); } #[test] @@ -274,11 +300,11 @@ fn auto_region_scaled_unsupported_metal_shape_returns_cpu_surface() { fn auto_viewport_cpu_fallback_returns_cpu_surface() { let decoder = Decoder::new(BASELINE_420).expect("decoder"); let mut pool = ScratchPool::new(); - let workload = signinum_jpeg_metal::viewport::ViewportWorkload { + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { scale: Downscale::None, viewport_dims: (16, 16), tiles: vec![ - signinum_jpeg_metal::viewport::ViewportTile { + j2k_jpeg_metal::viewport::ViewportTile { source_roi: Rect { x: 0, y: 0, @@ -292,7 +318,7 @@ fn auto_viewport_cpu_fallback_returns_cpu_surface() { h: 8, }, }, - signinum_jpeg_metal::viewport::ViewportTile { + j2k_jpeg_metal::viewport::ViewportTile { source_roi: Rect { x: 8, y: 8, @@ -309,7 +335,7 @@ fn auto_viewport_cpu_fallback_returns_cpu_surface() { ], }; - let surface = signinum_jpeg_metal::viewport::decode_viewport_to_surface( + let surface = j2k_jpeg_metal::viewport::decode_viewport_to_surface( decoder.inner(), &mut pool, &workload, @@ -406,7 +432,7 @@ fn submit_to_device_returns_surface_and_updates_session() { .expect("submission"); let surface = submission.wait().expect("surface"); assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert!(session.submissions() >= 1); + assert!(session.submissions().expect("session submissions") >= 1); } #[test] @@ -435,5 +461,5 @@ fn multiple_submits_share_one_session_flush() { assert_eq!(second_surface.backend_kind(), BackendKind::Metal); assert_eq!(first_surface.backend_kind(), BackendKind::Metal); - assert_eq!(session.submissions(), 1); + assert_eq!(session.submissions().expect("session submissions"), 1); } diff --git a/crates/signinum-jpeg-metal/tests/encode.rs b/crates/j2k-jpeg-metal/tests/encode.rs similarity index 92% rename from crates/signinum-jpeg-metal/tests/encode.rs rename to crates/j2k-jpeg-metal/tests/encode.rs index 25572e8f..03509856 100644 --- a/crates/signinum-jpeg-metal/tests/encode.rs +++ b/crates/j2k-jpeg-metal/tests/encode.rs @@ -30,15 +30,15 @@ fn assert_independent_decoder_accepts( #[cfg(target_os = "macos")] #[test] fn metal_baseline_encoder_round_trips_rgb_422() { - use signinum_core::PixelFormat; - use signinum_jpeg::{DecodeOptions, Decoder, JpegBackend, JpegEncodeOptions, JpegSubsampling}; - use signinum_jpeg_metal::{ + use j2k_core::PixelFormat; + use j2k_jpeg::{DecodeOptions, Decoder, JpegBackend, JpegEncodeOptions, JpegSubsampling}; + use j2k_jpeg_metal::{ encode_jpeg_baseline_from_metal_buffer, JpegBaselineMetalEncodeTile, MetalBackendSession, }; let width = 19u32; let height = 17u32; - let rgb = signinum_test_support::patterned_rgb8(width, height); + let rgb = j2k_test_support::patterned_rgb8(width, height); let session = MetalBackendSession::system_default().expect("Metal backend session"); let buffer = session.device().new_buffer_with_data( @@ -91,9 +91,9 @@ fn metal_baseline_encoder_round_trips_rgb_422() { #[cfg(target_os = "macos")] #[test] fn metal_baseline_encoder_round_trips_all_rgb_subsampling_modes() { - use signinum_core::PixelFormat; - use signinum_jpeg::{DecodeOptions, Decoder, JpegBackend, JpegEncodeOptions, JpegSubsampling}; - use signinum_jpeg_metal::{ + use j2k_core::PixelFormat; + use j2k_jpeg::{DecodeOptions, Decoder, JpegBackend, JpegEncodeOptions, JpegSubsampling}; + use j2k_jpeg_metal::{ encode_jpeg_baseline_from_metal_buffer, JpegBaselineMetalEncodeTile, MetalBackendSession, }; @@ -162,9 +162,9 @@ fn metal_baseline_encoder_round_trips_all_rgb_subsampling_modes() { #[cfg(target_os = "macos")] #[test] fn metal_baseline_encoder_round_trips_gray_with_padded_output() { - use signinum_core::PixelFormat; - use signinum_jpeg::{DecodeOptions, Decoder, JpegBackend, JpegEncodeOptions, JpegSubsampling}; - use signinum_jpeg_metal::{ + use j2k_core::PixelFormat; + use j2k_jpeg::{DecodeOptions, Decoder, JpegBackend, JpegEncodeOptions, JpegSubsampling}; + use j2k_jpeg_metal::{ encode_jpeg_baseline_from_metal_buffer, JpegBaselineMetalEncodeTile, MetalBackendSession, }; @@ -172,7 +172,7 @@ fn metal_baseline_encoder_round_trips_gray_with_padded_output() { let height = 5u32; let output_width = 13u32; let output_height = 11u32; - let gray = signinum_test_support::patterned_gray8(width, height); + let gray = j2k_test_support::patterned_gray8(width, height); let session = MetalBackendSession::system_default().expect("Metal backend session"); let buffer = session.device().new_buffer_with_data( @@ -228,9 +228,9 @@ fn metal_baseline_encoder_round_trips_gray_with_padded_output() { #[cfg(target_os = "macos")] #[test] fn metal_baseline_batch_encoder_round_trips_multiple_rgb_tiles() { - use signinum_core::PixelFormat; - use signinum_jpeg::{DecodeOptions, Decoder, JpegBackend, JpegEncodeOptions, JpegSubsampling}; - use signinum_jpeg_metal::{ + use j2k_core::PixelFormat; + use j2k_jpeg::{DecodeOptions, Decoder, JpegBackend, JpegEncodeOptions, JpegSubsampling}; + use j2k_jpeg_metal::{ encode_jpeg_baseline_batch_from_metal_buffers, JpegBaselineMetalEncodeTile, MetalBackendSession, }; diff --git a/crates/j2k-jpeg-metal/tests/shader_integrity.rs b/crates/j2k-jpeg-metal/tests/shader_integrity.rs new file mode 100644 index 00000000..d2d926ca --- /dev/null +++ b/crates/j2k-jpeg-metal/tests/shader_integrity.rs @@ -0,0 +1,14 @@ +use j2k_test_support::unwired_metal_kernels; + +const SHADER_SOURCE: &str = include_str!("../src/shaders.metal"); +const COMPUTE_SOURCE: &str = include_str!("../src/compute.rs"); + +#[test] +fn metal_kernels_are_wired_to_host_pipelines() { + let unused = unwired_metal_kernels([SHADER_SOURCE], COMPUTE_SOURCE); + + assert!( + unused.is_empty(), + "Metal kernels must be compiled by host pipeline setup or removed: {unused:?}" + ); +} diff --git a/crates/j2k-jpeg-metal/tests/viewport.rs b/crates/j2k-jpeg-metal/tests/viewport.rs new file mode 100644 index 00000000..b8bf2216 --- /dev/null +++ b/crates/j2k-jpeg-metal/tests/viewport.rs @@ -0,0 +1,1090 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(target_os = "macos")] +use j2k_core::DeviceSurface; +use j2k_core::{BackendRequest, Downscale, PixelFormat, Rect}; +use j2k_jpeg::{Decoder, ScratchPool}; +#[cfg(target_os = "macos")] +use j2k_jpeg_metal::viewport::{ + choose_resizable_metal_viewport_strategy, compose_viewport_hybrid, + compose_viewport_to_resizable_metal_buffer_with_session, + compose_viewport_to_resizable_metal_textures_with_session, decode_viewport_region_hybrid, + decode_viewport_region_to_resizable_metal_buffer_with_session, + decode_viewport_region_to_resizable_metal_textures_with_session, + decode_viewport_to_resizable_metal_buffer_with_decoder_session, + decode_viewport_to_resizable_metal_buffer_with_session, + decode_viewport_to_resizable_metal_textures_with_decoder_session, + decode_viewport_to_resizable_metal_textures_with_session, ViewportResidentOutputStrategy, +}; +use j2k_jpeg_metal::viewport::{ + choose_viewport_surface_strategy, compose_viewport_cpu, decode_viewport_region_cpu, + decode_viewport_to_surface, is_contiguous_viewport_workload, suggest_viewport_workload, + viewport_source_bounds, ViewportSurfaceStrategy, ViewportTile, +}; +#[cfg(target_os = "macos")] +use j2k_jpeg_metal::{ + Decoder as MetalDecoder, MetalBackendSession, MetalBatchOutputBuffer, MetalBatchTextureOutput, + SurfaceResidency, +}; + +const BASELINE_420: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); +const GRAYSCALE: &[u8] = include_bytes!("../fixtures/jpeg/grayscale_8x8.jpg"); + +fn quadrant_tiles() -> [ViewportTile; 4] { + [ + ViewportTile { + source_roi: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + dest: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + }, + ViewportTile { + source_roi: Rect { + x: 8, + y: 0, + w: 8, + h: 8, + }, + dest: Rect { + x: 8, + y: 0, + w: 8, + h: 8, + }, + }, + ViewportTile { + source_roi: Rect { + x: 0, + y: 8, + w: 8, + h: 8, + }, + dest: Rect { + x: 0, + y: 8, + w: 8, + h: 8, + }, + }, + ViewportTile { + source_roi: Rect { + x: 8, + y: 8, + w: 8, + h: 8, + }, + dest: Rect { + x: 8, + y: 8, + w: 8, + h: 8, + }, + }, + ] +} + +#[cfg(target_os = "macos")] +fn rgb_to_rgba_opaque(rgb: &[u8]) -> Vec { + let mut rgba = Vec::with_capacity(rgb.len() / 3 * 4); + for pixel in rgb.chunks_exact(3) { + rgba.extend_from_slice(pixel); + rgba.push(u8::MAX); + } + rgba +} + +#[cfg(target_os = "macos")] +fn download_rgba8_texture( + session: &MetalBackendSession, + texture: &metal::TextureRef, + dimensions: (u32, u32), +) -> Vec { + let row_bytes = dimensions.0 as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let byte_len = row_bytes * dimensions.1 as usize; + let buffer = session.device().new_buffer( + byte_len as u64, + metal::MTLResourceOptions::StorageModeShared, + ); + let queue = session.device().new_command_queue(); + let command_buffer = queue.new_command_buffer(); + let blit = command_buffer.new_blit_command_encoder(); + blit.copy_from_texture_to_buffer( + texture, + 0, + 0, + metal::MTLOrigin { x: 0, y: 0, z: 0 }, + metal::MTLSize::new(u64::from(dimensions.0), u64::from(dimensions.1), 1), + &buffer, + 0, + row_bytes as u64, + byte_len as u64, + metal::MTLBlitOption::None, + ); + blit.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + // SAFETY: The test buffer size is computed from the validated viewport dimensions. + unsafe { core::slice::from_raw_parts(buffer.contents().cast::(), byte_len).to_vec() } +} + +#[test] +fn cpu_viewport_quadrants_match_full_decode() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut pool = ScratchPool::new(); + + let actual = compose_viewport_cpu( + &decoder, + &mut pool, + PixelFormat::Rgb8, + Downscale::None, + (16, 16), + &quadrant_tiles(), + ) + .expect("viewport"); + let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("full decode"); + + assert_eq!(actual, expected); +} + +#[test] +fn suggested_viewport_workload_is_fixed_for_macro_like_input() { + let workload = suggest_viewport_workload((1_191, 408)).expect("workload"); + + assert_eq!(workload.scale, Downscale::Half); + assert_eq!(workload.viewport_dims, (576, 192)); + assert_eq!(workload.tiles.len(), 12); + assert_eq!( + workload.tiles.first(), + Some(&ViewportTile { + source_roi: Rect { + x: 18, + y: 12, + w: 192, + h: 192, + }, + dest: Rect { + x: 0, + y: 0, + w: 96, + h: 96, + }, + }) + ); + assert_eq!( + workload.tiles.last(), + Some(&ViewportTile { + source_roi: Rect { + x: 978, + y: 204, + w: 192, + h: 192, + }, + dest: Rect { + x: 480, + y: 96, + w: 96, + h: 96, + }, + }) + ); + assert!(is_contiguous_viewport_workload(&workload)); +} + +#[test] +fn cpu_viewport_misaligned_scaled_tile_matches_direct_decode() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut cpu_pool = ScratchPool::new(); + let roi = Rect { + x: 1, + y: 1, + w: 10, + h: 10, + }; + let tiles = [ViewportTile { + source_roi: roi, + dest: Rect { + x: 0, + y: 0, + w: 6, + h: 6, + }, + }]; + + let viewport = compose_viewport_cpu( + &decoder, + &mut cpu_pool, + PixelFormat::Rgb8, + Downscale::Half, + (6, 6), + &tiles, + ) + .expect("cpu viewport"); + let (expected, _outcome) = decoder + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + }, + Downscale::Half, + ) + .expect("direct decode"); + + assert_eq!(expected.len(), 6 * 6 * 3); + assert_eq!(viewport, expected); +} + +#[test] +fn cpu_contiguous_viewport_region_matches_direct_decode() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut pool = ScratchPool::new(); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (16, 16), + tiles: quadrant_tiles().to_vec(), + }; + + let actual = decode_viewport_region_cpu(&decoder, &mut pool, PixelFormat::Rgb8, &workload) + .expect("cpu viewport region"); + let (expected, _) = decoder + .decode_region_scaled( + PixelFormat::Rgb8, + j2k_jpeg::Rect { + x: viewport_source_bounds(&workload).x, + y: viewport_source_bounds(&workload).y, + w: viewport_source_bounds(&workload).w, + h: viewport_source_bounds(&workload).h, + }, + workload.scale, + ) + .expect("direct decode"); + + assert_eq!(actual, expected); +} + +#[test] +fn gapped_tiles_are_not_contiguous() { + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (16, 16), + tiles: vec![ + ViewportTile { + source_roi: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + dest: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + }, + ViewportTile { + source_roi: Rect { + x: 8, + y: 8, + w: 8, + h: 8, + }, + dest: Rect { + x: 8, + y: 8, + w: 8, + h: 8, + }, + }, + ], + }; + + assert!(!is_contiguous_viewport_workload(&workload)); + assert_eq!( + choose_viewport_surface_strategy(&workload, BackendRequest::Cpu).expect("cpu strategy"), + ViewportSurfaceStrategy::CpuComposite + ); +} + +#[test] +fn cpu_auto_strategy_prefers_contiguous_when_available() { + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (16, 16), + tiles: quadrant_tiles().to_vec(), + }; + + assert!(is_contiguous_viewport_workload(&workload)); + assert_eq!( + choose_viewport_surface_strategy(&workload, BackendRequest::Cpu).expect("cpu strategy"), + ViewportSurfaceStrategy::CpuContiguous + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn reusable_metal_viewport_strategy_reports_direct_contiguous_workload() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (16, 16), + tiles: quadrant_tiles().to_vec(), + }; + + assert_eq!( + choose_resizable_metal_viewport_strategy(&decoder, &workload) + .expect("resident viewport strategy"), + ViewportResidentOutputStrategy::DirectContiguous + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn reusable_metal_viewport_strategy_reports_sparse_composition_workload() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (16, 16), + tiles: vec![ + ViewportTile { + source_roi: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + dest: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + }, + ViewportTile { + source_roi: Rect { + x: 8, + y: 8, + w: 8, + h: 8, + }, + dest: Rect { + x: 8, + y: 8, + w: 8, + h: 8, + }, + }, + ], + }; + + assert_eq!( + choose_resizable_metal_viewport_strategy(&decoder, &workload) + .expect("resident viewport strategy"), + ViewportResidentOutputStrategy::Composite + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn hybrid_viewport_quadrants_match_cpu_viewport() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut cpu_pool = ScratchPool::new(); + let mut hybrid_pool = ScratchPool::new(); + + let expected = compose_viewport_cpu( + &decoder, + &mut cpu_pool, + PixelFormat::Rgb8, + Downscale::None, + (16, 16), + &quadrant_tiles(), + ) + .expect("cpu viewport"); + let actual = compose_viewport_hybrid( + &decoder, + &mut hybrid_pool, + Downscale::None, + (16, 16), + &quadrant_tiles(), + ) + .expect("hybrid viewport"); + + assert_eq!(actual.as_bytes(), expected.as_slice()); +} + +#[cfg(target_os = "macos")] +#[test] +fn hybrid_viewport_misaligned_scaled_tile_matches_cpu_viewport() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut cpu_pool = ScratchPool::new(); + let mut hybrid_pool = ScratchPool::new(); + let tiles = [ViewportTile { + source_roi: Rect { + x: 1, + y: 1, + w: 10, + h: 10, + }, + dest: Rect { + x: 0, + y: 0, + w: 6, + h: 6, + }, + }]; + + let expected = compose_viewport_cpu( + &decoder, + &mut cpu_pool, + PixelFormat::Rgb8, + Downscale::Half, + (6, 6), + &tiles, + ) + .expect("cpu viewport"); + let actual = + compose_viewport_hybrid(&decoder, &mut hybrid_pool, Downscale::Half, (6, 6), &tiles) + .expect("hybrid viewport"); + + assert_eq!(actual.as_bytes(), expected.as_slice()); +} + +#[cfg(target_os = "macos")] +#[test] +fn sparse_viewport_composition_resizes_reusable_metal_output_buffer() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut cpu_pool = ScratchPool::new(); + let mut metal_pool = ScratchPool::new(); + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (16, 16), + tiles: vec![ + ViewportTile { + source_roi: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + dest: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + }, + ViewportTile { + source_roi: Rect { + x: 8, + y: 8, + w: 8, + h: 8, + }, + dest: Rect { + x: 8, + y: 8, + w: 8, + h: 8, + }, + }, + ], + }; + assert!(!is_contiguous_viewport_workload(&workload)); + let expected = compose_viewport_cpu( + &decoder, + &mut cpu_pool, + PixelFormat::Rgb8, + workload.scale, + workload.viewport_dims, + &workload.tiles, + ) + .expect("cpu viewport"); + + let surface = compose_viewport_to_resizable_metal_buffer_with_session( + &decoder, + &mut metal_pool, + &workload, + &mut output, + &session, + ) + .expect("resident sparse viewport"); + + assert_eq!(output.dimensions(), workload.viewport_dims); + assert_eq!(output.tile_capacity(), 1); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), workload.viewport_dims); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, 0); + assert_eq!(surface.as_bytes(), expected.as_slice()); +} + +#[cfg(target_os = "macos")] +#[test] +fn reusable_metal_viewport_buffer_helper_routes_sparse_workload() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut cpu_pool = ScratchPool::new(); + let mut metal_pool = ScratchPool::new(); + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (16, 16), + tiles: vec![ + ViewportTile { + source_roi: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + dest: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + }, + ViewportTile { + source_roi: Rect { + x: 8, + y: 8, + w: 8, + h: 8, + }, + dest: Rect { + x: 8, + y: 8, + w: 8, + h: 8, + }, + }, + ], + }; + assert!(!is_contiguous_viewport_workload(&workload)); + let expected = compose_viewport_cpu( + &decoder, + &mut cpu_pool, + PixelFormat::Rgb8, + workload.scale, + workload.viewport_dims, + &workload.tiles, + ) + .expect("cpu viewport"); + + let surface = decode_viewport_to_resizable_metal_buffer_with_session( + &decoder, + &mut metal_pool, + &workload, + &mut output, + &session, + ) + .expect("resident sparse viewport"); + + assert_eq!(output.dimensions(), workload.viewport_dims); + assert_eq!(output.tile_capacity(), 1); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), workload.viewport_dims); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, 0); + assert_eq!(surface.as_bytes(), expected.as_slice()); +} + +#[cfg(target_os = "macos")] +#[test] +fn hybrid_contiguous_viewport_region_matches_cpu_region() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut cpu_pool = ScratchPool::new(); + let mut hybrid_pool = ScratchPool::new(); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (16, 16), + tiles: quadrant_tiles().to_vec(), + }; + + let expected = + decode_viewport_region_cpu(&decoder, &mut cpu_pool, PixelFormat::Rgb8, &workload) + .expect("cpu viewport region"); + let actual = decode_viewport_region_hybrid(&decoder, &mut hybrid_pool, &workload) + .expect("hybrid viewport region"); + + assert_eq!(actual.as_bytes(), expected.as_slice()); +} + +#[cfg(target_os = "macos")] +#[test] +fn contiguous_viewport_region_resizes_reusable_metal_output_buffer() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut cpu_pool = ScratchPool::new(); + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + let roi = Rect { + x: 1, + y: 2, + w: 10, + h: 9, + }; + let scaled = roi.scaled_covering(Downscale::Quarter); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::Quarter, + viewport_dims: (scaled.w, scaled.h), + tiles: vec![ViewportTile { + source_roi: roi, + dest: Rect { + x: 0, + y: 0, + w: scaled.w, + h: scaled.h, + }, + }], + }; + let expected = + decode_viewport_region_cpu(&decoder, &mut cpu_pool, PixelFormat::Rgb8, &workload) + .expect("cpu viewport region"); + + let surface = decode_viewport_region_to_resizable_metal_buffer_with_session( + BASELINE_420, + &workload, + &mut output, + &session, + ) + .expect("resident viewport region"); + + assert_eq!(output.dimensions(), workload.viewport_dims); + assert_eq!(output.tile_capacity(), 1); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), workload.viewport_dims); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, 0); + assert_eq!(surface.as_bytes(), expected.as_slice()); +} + +#[cfg(target_os = "macos")] +#[test] +fn contiguous_viewport_region_resizes_reusable_metal_textures() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut cpu_pool = ScratchPool::new(); + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + let roi = Rect { + x: 1, + y: 2, + w: 10, + h: 9, + }; + let scaled = roi.scaled_covering(Downscale::Quarter); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::Quarter, + viewport_dims: (scaled.w, scaled.h), + tiles: vec![ViewportTile { + source_roi: roi, + dest: Rect { + x: 0, + y: 0, + w: scaled.w, + h: scaled.h, + }, + }], + }; + let expected_rgb = + decode_viewport_region_cpu(&decoder, &mut cpu_pool, PixelFormat::Rgb8, &workload) + .expect("cpu viewport region"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tile = decode_viewport_region_to_resizable_metal_textures_with_session( + BASELINE_420, + &workload, + &mut output, + &session, + ) + .expect("resident viewport texture"); + + assert_eq!(output.dimensions(), workload.viewport_dims); + assert_eq!(output.tile_capacity(), 1); + assert_eq!(tile.dimensions(), workload.viewport_dims); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(0).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn reusable_metal_viewport_decoder_helper_routes_contiguous_workload_to_buffer() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let metal_decoder = MetalDecoder::new(BASELINE_420).expect("metal decoder"); + let mut cpu_pool = ScratchPool::new(); + let mut metal_pool = ScratchPool::new(); + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchOutputBuffer::new_rgb8_tiles(&session, (1, 1), 1).expect("output buffer"); + let roi = Rect { + x: 1, + y: 2, + w: 10, + h: 9, + }; + let scaled = roi.scaled_covering(Downscale::Quarter); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::Quarter, + viewport_dims: (scaled.w, scaled.h), + tiles: vec![ViewportTile { + source_roi: roi, + dest: Rect { + x: 0, + y: 0, + w: scaled.w, + h: scaled.h, + }, + }], + }; + assert!(is_contiguous_viewport_workload(&workload)); + let expected = + decode_viewport_region_cpu(&decoder, &mut cpu_pool, PixelFormat::Rgb8, &workload) + .expect("cpu viewport region"); + + let surface = decode_viewport_to_resizable_metal_buffer_with_decoder_session( + &metal_decoder, + &mut metal_pool, + &workload, + &mut output, + &session, + ) + .expect("resident viewport buffer"); + + assert_eq!(output.dimensions(), workload.viewport_dims); + assert_eq!(output.tile_capacity(), 1); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), workload.viewport_dims); + assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); + let (buffer, offset) = surface.metal_buffer().expect("metal buffer"); + assert!(std::ptr::eq(buffer.as_ref(), output.buffer())); + assert_eq!(offset, 0); + assert_eq!(surface.as_bytes(), expected.as_slice()); +} + +#[cfg(target_os = "macos")] +#[test] +fn reusable_metal_viewport_decoder_helper_routes_contiguous_workload_to_textures() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let metal_decoder = MetalDecoder::new(BASELINE_420).expect("metal decoder"); + let mut cpu_pool = ScratchPool::new(); + let mut metal_pool = ScratchPool::new(); + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + let roi = Rect { + x: 1, + y: 2, + w: 10, + h: 9, + }; + let scaled = roi.scaled_covering(Downscale::Quarter); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::Quarter, + viewport_dims: (scaled.w, scaled.h), + tiles: vec![ViewportTile { + source_roi: roi, + dest: Rect { + x: 0, + y: 0, + w: scaled.w, + h: scaled.h, + }, + }], + }; + assert!(is_contiguous_viewport_workload(&workload)); + let expected_rgb = + decode_viewport_region_cpu(&decoder, &mut cpu_pool, PixelFormat::Rgb8, &workload) + .expect("cpu viewport region"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tile = decode_viewport_to_resizable_metal_textures_with_decoder_session( + &metal_decoder, + &mut metal_pool, + &workload, + &mut output, + &session, + ) + .expect("resident viewport texture"); + + assert_eq!(output.dimensions(), workload.viewport_dims); + assert_eq!(output.tile_capacity(), 1); + assert_eq!(tile.dimensions(), workload.viewport_dims); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(0).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn reusable_metal_viewport_texture_helper_routes_contiguous_workload() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut cpu_pool = ScratchPool::new(); + let mut metal_pool = ScratchPool::new(); + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + let roi = Rect { + x: 1, + y: 2, + w: 10, + h: 9, + }; + let scaled = roi.scaled_covering(Downscale::Quarter); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::Quarter, + viewport_dims: (scaled.w, scaled.h), + tiles: vec![ViewportTile { + source_roi: roi, + dest: Rect { + x: 0, + y: 0, + w: scaled.w, + h: scaled.h, + }, + }], + }; + assert!(is_contiguous_viewport_workload(&workload)); + let expected_rgb = + decode_viewport_region_cpu(&decoder, &mut cpu_pool, PixelFormat::Rgb8, &workload) + .expect("cpu viewport region"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tile = decode_viewport_to_resizable_metal_textures_with_session( + &decoder, + &mut metal_pool, + &workload, + &mut output, + &session, + ) + .expect("resident viewport texture"); + + assert_eq!(output.dimensions(), workload.viewport_dims); + assert_eq!(output.tile_capacity(), 1); + assert_eq!(tile.dimensions(), workload.viewport_dims); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(0).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn sparse_viewport_composition_resizes_reusable_metal_texture_output() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut cpu_pool = ScratchPool::new(); + let mut metal_pool = ScratchPool::new(); + let session = MetalBackendSession::system_default().expect("Metal backend session"); + let mut output = + MetalBatchTextureOutput::new_rgba8_tiles(&session, (1, 1), 1).expect("texture output"); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (16, 16), + tiles: vec![ + ViewportTile { + source_roi: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + dest: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + }, + ViewportTile { + source_roi: Rect { + x: 8, + y: 8, + w: 8, + h: 8, + }, + dest: Rect { + x: 8, + y: 8, + w: 8, + h: 8, + }, + }, + ], + }; + assert!(!is_contiguous_viewport_workload(&workload)); + let expected_rgb = compose_viewport_cpu( + &decoder, + &mut cpu_pool, + PixelFormat::Rgb8, + workload.scale, + workload.viewport_dims, + &workload.tiles, + ) + .expect("cpu viewport"); + let expected_rgba = rgb_to_rgba_opaque(&expected_rgb); + + let tile = compose_viewport_to_resizable_metal_textures_with_session( + &decoder, + &mut metal_pool, + &workload, + &mut output, + &session, + ) + .expect("resident sparse viewport texture"); + + assert_eq!(output.dimensions(), workload.viewport_dims); + assert_eq!(output.tile_capacity(), 1); + assert_eq!(tile.dimensions(), workload.viewport_dims); + assert_eq!(tile.pixel_format(), PixelFormat::Rgba8); + assert!(std::ptr::eq( + tile.texture(), + output.texture(0).expect("output texture") + )); + assert_eq!( + download_rgba8_texture(&session, tile.texture(), tile.dimensions()), + expected_rgba + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn auto_viewport_surface_path_prefers_cpu_for_small_contiguous_workloads() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut direct_pool = ScratchPool::new(); + let mut auto_pool = ScratchPool::new(); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (16, 16), + tiles: quadrant_tiles().to_vec(), + }; + + let expected = j2k_jpeg_metal::viewport::decode_viewport_region_cpu_to_surface( + &decoder, + &mut direct_pool, + &workload, + ) + .expect("cpu viewport surface"); + let actual = + decode_viewport_to_surface(&decoder, &mut auto_pool, &workload, BackendRequest::Auto) + .expect("auto viewport surface"); + + assert_eq!(actual.as_bytes(), expected.as_bytes()); +} + +#[cfg(not(target_os = "macos"))] +#[test] +fn non_macos_auto_viewport_surface_returns_cpu_surface() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut pool = ScratchPool::new(); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (16, 16), + tiles: quadrant_tiles().to_vec(), + }; + + let surface = decode_viewport_to_surface(&decoder, &mut pool, &workload, BackendRequest::Auto) + .expect("auto viewport surface"); + + assert_eq!( + j2k_core::DeviceSurface::backend_kind(&surface), + j2k_core::BackendKind::Cpu + ); +} + +#[cfg(not(target_os = "macos"))] +#[test] +fn non_macos_explicit_metal_viewport_surface_is_unavailable() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let mut pool = ScratchPool::new(); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (16, 16), + tiles: quadrant_tiles().to_vec(), + }; + + let result = decode_viewport_to_surface(&decoder, &mut pool, &workload, BackendRequest::Metal); + assert!(matches!( + result, + Err(j2k_jpeg_metal::Error::MetalUnavailable) + )); +} + +#[test] +fn explicit_metal_viewport_unsupported_shape_is_rejected() { + let decoder = Decoder::new(GRAYSCALE).expect("decoder"); + let mut pool = ScratchPool::new(); + let workload = j2k_jpeg_metal::viewport::ViewportWorkload { + scale: Downscale::None, + viewport_dims: (8, 8), + tiles: vec![ViewportTile { + source_roi: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + dest: Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }, + }], + }; + + let result = decode_viewport_to_surface(&decoder, &mut pool, &workload, BackendRequest::Metal); + + match result { + Err(j2k_jpeg_metal::Error::UnsupportedMetalRequest { reason }) => { + assert!(reason.contains("JPEG Metal")); + } + #[cfg(not(target_os = "macos"))] + Err(j2k_jpeg_metal::Error::MetalUnavailable) => { + panic!("unsupported shape should be rejected before host availability") + } + Err(other) => panic!("unexpected explicit Metal viewport error: {other:?}"), + Ok(surface) => panic!( + "explicit Metal viewport must not fall back; got {:?}", + j2k_core::DeviceSurface::backend_kind(&surface) + ), + } +} diff --git a/crates/signinum-jpeg/Cargo.toml b/crates/j2k-jpeg/Cargo.toml similarity index 66% rename from crates/signinum-jpeg/Cargo.toml rename to crates/j2k-jpeg/Cargo.toml index e81b69f3..7c9a7c8f 100644 --- a/crates/signinum-jpeg/Cargo.toml +++ b/crates/j2k-jpeg/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "signinum-jpeg" -description = "JPEG decoder optimized for whole-slide images (WSI)" -version = "0.4.2" +name = "j2k-jpeg" +description = "JPEG inspect/decode and fallback encode support for j2k" +version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true @@ -9,26 +9,34 @@ repository.workspace = true keywords.workspace = true categories.workspace = true readme = "README.md" -build = "build.rs" + +[package.metadata.docs.rs] +all-features = true +targets = [] [lib] -name = "signinum_jpeg" +name = "j2k_jpeg" path = "src/lib.rs" [features] scalar-only = [] # retained for fuzz/reference decode until native SIMD backends land +# Opt-in for the comparison benches only: probes pkg-config for +# libturbojpeg/libjpeg and emits link directives. Off by default so library +# consumers never link a system JPEG. +bench-libjpeg-turbo = [] +bench-internals = [] [dependencies] -signinum-core = { path = "../signinum-core", version = "=0.4.2" } -signinum-profile = { path = "../signinum-profile", version = "=0.4.2" } +j2k-core = { path = "../j2k-core", version = "=0.6.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.6.0" } thiserror = { workspace = true } -memchr = { version = "2.7.6", default-features = false } -rayon = "1" +memchr = { workspace = true } +rayon = { workspace = true } [dev-dependencies] proptest = { workspace = true } criterion = { workspace = true } -signinum-test-support = { path = "../signinum-test-support" } +j2k-test-support = { path = "../j2k-test-support" } jpeg-decoder = { version = "0.3.2", default-features = false } zune-core = "0.5.1" zune-jpeg = "0.5.15" @@ -36,19 +44,38 @@ zune-jpeg = "0.5.15" [[bench]] name = "compare" harness = false +required-features = ["bench-libjpeg-turbo"] [[bench]] name = "micro" harness = false +required-features = ["bench-internals"] [[bench]] name = "fast420_breakdown" harness = false +required-features = ["bench-libjpeg-turbo", "bench-internals"] [[bench]] name = "corpus_report" harness = false +[[bench]] +name = "encode_cpu" +harness = false + +[[test]] +name = "fast420_profile" +required-features = ["bench-internals"] + +[[test]] +name = "idct_parity" +required-features = ["bench-internals"] + +[[test]] +name = "neon_hot_paths" +required-features = ["bench-internals"] + [lints.rust] # SIMD backends use tightly-scoped intrinsics in `src/backend/`; keep unsafe # explicit there while the rest of the crate remains safe Rust. @@ -61,6 +88,7 @@ unreachable_pub = "warn" [lints.clippy] pedantic = { level = "warn", priority = -1 } +undocumented_unsafe_blocks = "warn" module_name_repetitions = "allow" must_use_candidate = "allow" # Style-only lints that conflict with the plan's explicit-over-idiomatic diff --git a/crates/j2k-jpeg/README.md b/crates/j2k-jpeg/README.md new file mode 100644 index 00000000..af66cce0 --- /dev/null +++ b/crates/j2k-jpeg/README.md @@ -0,0 +1,11 @@ +# j2k-jpeg + +Pure-Rust JPEG inspect/decode crate for J2K transcode and codec pipelines. + +CPU decode is the correctness baseline. Supported JPEG classes are covered by +tests and capability reports; unsupported classes return structured errors. +The crate also contains fixture/fallback baseline encode support used by tests +and adapters. + +Use this crate directly for JPEG input; use `j2k` for JPEG 2000 / HTJ2K and +`j2k-transcode` for JPEG-to-J2K/HTJ2K transcode paths. diff --git a/crates/signinum-jpeg/benches/common/classification.rs b/crates/j2k-jpeg/benches/common/classification.rs similarity index 92% rename from crates/signinum-jpeg/benches/common/classification.rs rename to crates/j2k-jpeg/benches/common/classification.rs index 028bb48c..66ca584e 100644 --- a/crates/signinum-jpeg/benches/common/classification.rs +++ b/crates/j2k-jpeg/benches/common/classification.rs @@ -2,7 +2,7 @@ #![allow(dead_code)] -use signinum_jpeg::ColorSpace; +use j2k_jpeg::ColorSpace; pub(crate) const FULL_FRAME_MAX_OUTPUT_BYTES: usize = 512 * 1024 * 1024; @@ -87,7 +87,7 @@ pub(crate) fn should_compare_full_frame_for_policy( } fn force_full_frame_compare_from_env() -> bool { - std::env::var_os("SIGNINUM_FORCE_FULL_FRAME") + std::env::var_os("J2K_FORCE_FULL_FRAME") .is_some_and(|value| !matches!(value.to_str(), Some("0" | "false" | "FALSE" | "False"))) } @@ -98,11 +98,7 @@ fn full_frame_output_len(dimensions: (u32, u32), mode: DecodeMode) -> Option Result { + // SAFETY: Benchmark FFI calls use a live libjpeg-turbo handle and sized outputs. let handle = unsafe { tj3Init(TJINIT_DECOMPRESS) }; let Some(handle) = NonNull::new(handle) else { return Err("tj3Init returned null".to_string()); @@ -93,6 +94,34 @@ mod imp { self.decode(bytes, TJPF_GRAY, None, Downscale::None) } + pub(crate) fn prepare_rgb(&mut self, bytes: &[u8]) -> Result { + let header = self.read_header(bytes)?; + self.set_scaling(TJUNSCALED)?; + self.set_crop(TJUNCROPPED)?; + Ok(header) + } + + pub(crate) fn decode_rgb_into( + &mut self, + bytes: &[u8], + out: &mut [u8], + pitch: usize, + ) -> Result<(), String> { + self.decode_into(bytes, TJPF_RGB, Downscale::None, out, pitch) + } + + pub(crate) fn decode_prepared_rgb_into( + &mut self, + bytes: &[u8], + out: &mut [u8], + pitch: usize, + width: usize, + height: usize, + ) -> Result<(), String> { + validate_output_buffer(width, height, TJPF_RGB, out, pitch)?; + self.decompress(bytes, out, pitch, TJPF_RGB) + } + pub(crate) fn decode_scaled_rgb( &mut self, bytes: &[u8], @@ -168,15 +197,39 @@ mod imp { Ok(out) } + fn decode_into( + &mut self, + bytes: &[u8], + pixel_format: c_int, + factor: Downscale, + out: &mut [u8], + pitch: usize, + ) -> Result<(), String> { + let header = self.read_header(bytes)?; + let scale = scaling_factor(factor); + self.set_scaling(scale)?; + self.set_crop(TJUNCROPPED)?; + + let out_width = scaled_dimension(header.width, scale) as usize; + let out_height = scaled_dimension(header.height, scale) as usize; + validate_output_buffer(out_width, out_height, pixel_format, out, pitch)?; + + self.decompress(bytes, out, pitch, pixel_format) + } + fn read_header(&mut self, bytes: &[u8]) -> Result { let rc = + // SAFETY: Benchmark FFI calls use a live libjpeg-turbo handle and sized outputs. unsafe { tj3DecompressHeader(self.handle.as_ptr(), bytes.as_ptr(), bytes.len()) }; if rc != 0 { return Err(self.error_string()); } + // SAFETY: Benchmark FFI calls use a live libjpeg-turbo handle and sized outputs. let width = unsafe { tj3Get(self.handle.as_ptr(), TJPARAM_JPEGWIDTH) }; + // SAFETY: Benchmark FFI calls use a live libjpeg-turbo handle and sized outputs. let height = unsafe { tj3Get(self.handle.as_ptr(), TJPARAM_JPEGHEIGHT) }; + // SAFETY: Benchmark FFI calls use a live libjpeg-turbo handle and sized outputs. let subsamp = unsafe { tj3Get(self.handle.as_ptr(), TJPARAM_SUBSAMP) }; if width < 0 || height < 0 || subsamp < 0 { return Err("tj3Get returned incomplete header parameters".to_string()); @@ -190,6 +243,7 @@ mod imp { } fn set_scaling(&mut self, scale: TjScalingFactor) -> Result<(), String> { + // SAFETY: Benchmark FFI calls use a live libjpeg-turbo handle and sized outputs. let rc = unsafe { tj3SetScalingFactor(self.handle.as_ptr(), scale) }; if rc != 0 { return Err(self.error_string()); @@ -198,6 +252,7 @@ mod imp { } fn set_crop(&mut self, region: TjRegion) -> Result<(), String> { + // SAFETY: Benchmark FFI calls use a live libjpeg-turbo handle and sized outputs. let rc = unsafe { tj3SetCroppingRegion(self.handle.as_ptr(), region) }; if rc != 0 { return Err(self.error_string()); @@ -212,6 +267,7 @@ mod imp { pitch: usize, pixel_format: c_int, ) -> Result<(), String> { + // SAFETY: Benchmark FFI calls use a live libjpeg-turbo handle and sized outputs. let rc = unsafe { tj3Decompress8( self.handle.as_ptr(), @@ -229,18 +285,60 @@ mod imp { } fn error_string(&self) -> String { + // SAFETY: Benchmark FFI calls use a live libjpeg-turbo handle and sized outputs. let ptr = unsafe { tj3GetErrorStr(self.handle.as_ptr()) }; if ptr.is_null() { return "libjpeg-turbo error".to_string(); } + // SAFETY: Benchmark FFI calls use a live libjpeg-turbo handle and sized outputs. unsafe { CStr::from_ptr(ptr) } .to_string_lossy() .into_owned() } } + fn validate_output_buffer( + width: usize, + height: usize, + pixel_format: c_int, + out: &[u8], + pitch: usize, + ) -> Result<(), String> { + let row_bytes = width + .checked_mul(bytes_per_pixel(pixel_format)) + .ok_or("libjpeg-turbo output row size overflow")?; + if pitch < row_bytes { + return Err(format!( + "libjpeg-turbo output pitch {pitch} is smaller than row bytes {row_bytes}" + )); + } + let required_len = required_strided_len(height, pitch, row_bytes)?; + if out.len() < required_len { + return Err(format!( + "libjpeg-turbo output buffer too small: need {required_len}, got {}", + out.len() + )); + } + Ok(()) + } + + fn required_strided_len( + height: usize, + pitch: usize, + row_bytes: usize, + ) -> Result { + if height == 0 { + return Ok(0); + } + pitch + .checked_mul(height - 1) + .and_then(|prefix| prefix.checked_add(row_bytes)) + .ok_or_else(|| "libjpeg-turbo output buffer size overflow".to_string()) + } + impl Drop for TurboJpegDecoder { fn drop(&mut self) { + // SAFETY: Benchmark FFI calls use a live libjpeg-turbo handle and sized outputs. unsafe { tj3Destroy(self.handle.as_ptr()) }; } } @@ -287,13 +385,7 @@ mod imp { } fn scaled_rect(rect: Rect, factor: Downscale) -> Rect { - let denom = match factor { - Downscale::None => 1, - Downscale::Half => 2, - Downscale::Quarter => 4, - Downscale::Eighth => 8, - _ => unreachable!("unsupported Downscale variant"), - }; + let denom = factor.denominator(); let x_end = rect.x + rect.w; let y_end = rect.y + rect.h; Rect { @@ -360,6 +452,33 @@ mod imp { Err("libjpeg-turbo not available".to_string()) } + pub(crate) fn prepare_rgb(&mut self, _bytes: &[u8]) -> Result { + let _ = self; + Err("libjpeg-turbo not available".to_string()) + } + + pub(crate) fn decode_rgb_into( + &mut self, + _bytes: &[u8], + _out: &mut [u8], + _pitch: usize, + ) -> Result<(), String> { + let _ = self; + Err("libjpeg-turbo not available".to_string()) + } + + pub(crate) fn decode_prepared_rgb_into( + &mut self, + _bytes: &[u8], + _out: &mut [u8], + _pitch: usize, + _width: usize, + _height: usize, + ) -> Result<(), String> { + let _ = self; + Err("libjpeg-turbo not available".to_string()) + } + pub(crate) fn decode_scaled_rgb( &mut self, _bytes: &[u8], diff --git a/crates/signinum-jpeg/benches/common/mod.rs b/crates/j2k-jpeg/benches/common/mod.rs similarity index 61% rename from crates/signinum-jpeg/benches/common/mod.rs rename to crates/j2k-jpeg/benches/common/mod.rs index bc236e06..476eff84 100644 --- a/crates/signinum-jpeg/benches/common/mod.rs +++ b/crates/j2k-jpeg/benches/common/mod.rs @@ -4,17 +4,19 @@ pub(crate) mod classification; mod libjpeg_turbo; +pub(crate) mod report; pub(crate) use self::classification::DecodeMode; use self::classification::{classify_corpus_input, color_space_mode, CorpusInputClass}; pub(crate) use self::libjpeg_turbo::TurboJpegDecoder; -use signinum_core::tile_batch_worker_count; -use signinum_jpeg::{ +use j2k_core::tile_batch_worker_count; +use j2k_jpeg::{ decode_tiles_into, decode_tiles_region_scaled_into, decode_tiles_scaled_into, Decoder, - DecoderContext, Downscale, JpegError, PixelFormat, Rect, RowSink, ScratchPool, - TileBatchOptions, TileDecodeJob, TileRegionScaledDecodeJob, TileScaledDecodeJob, + DecoderContext, Downscale, JpegBatchSession, JpegError, JpegOutputBuffer, PixelFormat, Rect, + RowSink, ScratchPool, TileBatchOptions, TileDecodeJob, TileRegionScaledDecodeJob, + TileScaledDecodeJob, }; -use std::fs; +use j2k_test_support::{JPEG_BASELINE_420_16X16, JPEG_GRAYSCALE_8X8}; use std::path::{Path, PathBuf}; use zune_core::bytestream::ZCursor; use zune_core::colorspace::ColorSpace as ZuneColorSpace; @@ -35,14 +37,14 @@ pub(crate) fn load_bench_inputs() -> Vec { let mut inputs = vec![ BenchInput { name: "repo/baseline_420_16x16".to_string(), - bytes: include_bytes!("../../fixtures/conformance/baseline_420_16x16.jpg").to_vec(), + bytes: JPEG_BASELINE_420_16X16.to_vec(), dimensions: (16, 16), mode: DecodeMode::Rgb, input_class: CorpusInputClass::BoundedFullFrame, }, BenchInput { name: "repo/grayscale_8x8".to_string(), - bytes: include_bytes!("../../fixtures/conformance/grayscale_8x8.jpg").to_vec(), + bytes: JPEG_GRAYSCALE_8X8.to_vec(), dimensions: (8, 8), mode: DecodeMode::Gray, input_class: CorpusInputClass::BoundedFullFrame, @@ -53,9 +55,7 @@ pub(crate) fn load_bench_inputs() -> Vec { .iter() .map(|input| input.name.clone()) .collect::>(); - for path in - std::env::split_paths(&std::env::var_os("SIGNINUM_BENCH_INPUTS").unwrap_or_default()) - { + for path in j2k_test_support::paths_from_env("J2K_BENCH_INPUTS") { collect_jpegs(&path, &mut inputs, &mut seen); } @@ -64,35 +64,13 @@ pub(crate) fn load_bench_inputs() -> Vec { } fn collect_jpegs(path: &Path, inputs: &mut Vec, seen: &mut Vec) { - if path.is_file() { - push_jpeg(path, inputs, seen); - return; - } - if !path.is_dir() { - return; - } - - let mut stack = vec![path.to_path_buf()]; - while let Some(dir) = stack.pop() { - let Ok(entries) = fs::read_dir(&dir) else { - continue; - }; - for entry in entries.flatten() { - let child = entry.path(); - if child.is_dir() { - stack.push(child); - } else { - push_jpeg(&child, inputs, seen); - } - } + for path in j2k_test_support::collect_jpeg_paths(path) { + push_jpeg(&path, inputs, seen); } } fn push_jpeg(path: &Path, inputs: &mut Vec, seen: &mut Vec) { - if !is_jpeg(path) { - return; - } - let Ok(bytes) = fs::read(path) else { + let Ok(bytes) = std::fs::read(path) else { return; }; let Ok(dec) = Decoder::new(&bytes) else { @@ -129,14 +107,8 @@ fn relative_name(path: &Path) -> String { absolute.display().to_string() } -fn is_jpeg(path: &Path) -> bool { - path.extension() - .and_then(|ext| ext.to_str()) - .is_some_and(|ext| matches!(ext.to_ascii_lowercase().as_str(), "jpg" | "jpeg")) -} - -pub(crate) fn signinum_inspect(bytes: &[u8]) { - let info = Decoder::inspect(bytes).expect("signinum inspect"); +pub(crate) fn j2k_inspect(bytes: &[u8]) { + let info = Decoder::inspect(bytes).expect("j2k inspect"); std::hint::black_box(info); } @@ -164,13 +136,13 @@ pub(crate) fn zune_inspect(bytes: &[u8]) { std::hint::black_box(decoder.info()); } -pub(crate) fn signinum_decode(bytes: &[u8], mode: DecodeMode) { - let dec = Decoder::new(bytes).expect("signinum decoder"); +pub(crate) fn j2k_decode(bytes: &[u8], mode: DecodeMode) { + let dec = Decoder::new(bytes).expect("j2k decoder"); let fmt = match mode { DecodeMode::Gray => PixelFormat::Gray8, DecodeMode::Rgb => PixelFormat::Rgb8, }; - let (out, _) = dec.decode(fmt).expect("signinum decode"); + let (out, _) = dec.decode(fmt).expect("j2k decode"); std::hint::black_box(out); } @@ -203,15 +175,15 @@ pub(crate) fn output_geometry(dec: &Decoder<'_>, mode: DecodeMode) -> (PixelForm /// buffer. Isolates pure decode cost from `Decoder::new` + output allocation — /// the realistic WSI tile-batch shape. Competitor crates are not called from /// this helper because neither `zune-jpeg` nor `jpeg-decoder` expose a reusable -/// decoder; fairness is preserved by keeping this group signinum-only. -pub(crate) fn signinum_decode_reused( +/// decoder; fairness is preserved by keeping this group j2k-only. +pub(crate) fn j2k_decode_reused( dec: &Decoder<'_>, out: &mut [u8], stride: usize, fmt: PixelFormat, ) { dec.decode_into(out, stride, fmt) - .expect("signinum decode (reused)"); + .expect("j2k decode (reused)"); std::hint::black_box(&*out); } @@ -219,7 +191,7 @@ pub(crate) fn signinum_decode_reused( /// pre-allocated `ScratchPool`. The pool amortizes stripe-buffer and /// DC-predictor allocations across many tiles — the shape every Phase 3 /// WSI benchmark is trying to surface. -pub(crate) fn signinum_decode_with_scratch( +pub(crate) fn j2k_decode_with_scratch( dec: &Decoder<'_>, pool: &mut ScratchPool, out: &mut [u8], @@ -227,7 +199,7 @@ pub(crate) fn signinum_decode_with_scratch( fmt: PixelFormat, ) { dec.decode_into_with_scratch(pool, out, stride, fmt) - .expect("signinum decode (scratch)"); + .expect("j2k decode (scratch)"); std::hint::black_box(&*out); } @@ -242,19 +214,29 @@ impl RowSink for NullSink { } } -pub(crate) struct SigninumTileBatchRgbScratch { +pub(crate) struct J2KTileBatchRgbScratch { outputs: Vec>, stride: usize, + options: TileBatchOptions, } -impl SigninumTileBatchRgbScratch { +impl J2KTileBatchRgbScratch { pub(crate) fn new(bytes: &[u8], batch_size: usize) -> Self { - let info = Decoder::inspect(bytes).expect("signinum inspect tile batch"); + Self::new_with_options(bytes, batch_size, TileBatchOptions::default()) + } + + pub(crate) fn new_with_options( + bytes: &[u8], + batch_size: usize, + options: TileBatchOptions, + ) -> Self { + let info = Decoder::inspect(bytes).expect("j2k inspect tile batch"); let stride = info.dimensions.0 as usize * 3; let len = stride * info.dimensions.1 as usize; Self { outputs: (0..batch_size).map(|_| vec![0u8; len]).collect(), stride, + options, } } @@ -269,24 +251,205 @@ impl SigninumTileBatchRgbScratch { stride: self.stride, }) .collect::>(); - decode_tiles_into(&mut jobs, PixelFormat::Rgb8, TileBatchOptions::default()) - .expect("signinum production tile batch") + decode_tiles_into(&mut jobs, PixelFormat::Rgb8, self.options) + .expect("j2k production tile batch") }; std::hint::black_box(&outcomes); std::hint::black_box(&self.outputs); } } -pub(crate) fn signinum_decode_rows(bytes: &[u8]) { - let dec = Decoder::new(bytes).expect("signinum decoder"); +pub(crate) struct J2KTileBatchRgbSession { + outputs: Vec>, + stride: usize, + session: JpegBatchSession, +} + +impl J2KTileBatchRgbSession { + pub(crate) fn new(bytes: &[u8], batch_size: usize) -> Self { + let info = Decoder::inspect(bytes).expect("j2k inspect tile batch"); + let stride = info.dimensions.0 as usize * 3; + let len = stride * info.dimensions.1 as usize; + let mut this = Self { + outputs: (0..batch_size).map(|_| vec![0u8; len]).collect(), + stride, + session: JpegBatchSession::default(), + }; + this.run(bytes); + this + } + + pub(crate) fn run(&mut self, bytes: &[u8]) { + let outcomes = { + let mut jobs = self + .outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes, + out: out.as_mut_slice(), + stride: self.stride, + }) + .collect::>(); + self.session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb8) + .expect("j2k session tile batch") + }; + std::hint::black_box(outcomes); + std::hint::black_box(&self.outputs); + } +} + +pub(crate) struct J2KTileBatchRgbOutputBuffers { + outputs: Vec, + session: JpegBatchSession, +} + +impl J2KTileBatchRgbOutputBuffers { + pub(crate) fn new(bytes: &[u8], batch_size: usize) -> Self { + let info = Decoder::inspect(bytes).expect("j2k inspect tile batch"); + let mut this = Self { + outputs: (0..batch_size) + .map(|_| { + JpegOutputBuffer::new(info.dimensions, PixelFormat::Rgb8) + .expect("JPEG output buffer") + }) + .collect(), + session: JpegBatchSession::default(), + }; + this.run(bytes); + this + } + + pub(crate) fn run(&mut self, bytes: &[u8]) { + let outcomes = { + let mut jobs = self + .outputs + .iter_mut() + .map(|out| { + let stride = out.stride(); + TileDecodeJob { + input: bytes, + out: out.as_mut_slice(), + stride, + } + }) + .collect::>(); + self.session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb8) + .expect("j2k session tile batch with output buffers") + }; + std::hint::black_box(outcomes); + std::hint::black_box(&self.outputs); + } +} + +pub(crate) struct J2KTileBatchScaledRgbSession { + outputs: Vec, + scale: Downscale, + session: JpegBatchSession, +} + +impl J2KTileBatchScaledRgbSession { + pub(crate) fn new(bytes: &[u8], batch_size: usize, scale: Downscale) -> Self { + let info = Decoder::inspect(bytes).expect("j2k inspect scaled tile batch"); + let dims = scaled_dims(info.dimensions, scale); + let mut this = Self { + outputs: (0..batch_size) + .map(|_| JpegOutputBuffer::new(dims, PixelFormat::Rgb8).expect("output buffer")) + .collect(), + scale, + session: JpegBatchSession::default(), + }; + this.run(bytes); + this + } + + pub(crate) fn run(&mut self, bytes: &[u8]) { + let outcomes = { + let mut jobs = self + .outputs + .iter_mut() + .map(|out| { + let stride = out.stride(); + TileScaledDecodeJob { + input: bytes, + out: out.as_mut_slice(), + stride, + scale: self.scale, + } + }) + .collect::>(); + self.session + .decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgb8) + .expect("j2k session scaled tile batch") + }; + std::hint::black_box(outcomes); + std::hint::black_box(&self.outputs); + } +} + +pub(crate) struct J2KTileBatchRegionScaledRgbSession { + outputs: Vec, + roi: Rect, + scale: Downscale, + session: JpegBatchSession, +} + +impl J2KTileBatchRegionScaledRgbSession { + pub(crate) fn new(bytes: &[u8], batch_size: usize, side: u32, scale: Downscale) -> Self { + let info = Decoder::inspect(bytes).expect("j2k inspect region-scaled tile batch"); + let roi = centered_roi(info.dimensions, side); + let dims = { + let scaled = scaled_rect(roi, scale); + (scaled.w, scaled.h) + }; + let mut this = Self { + outputs: (0..batch_size) + .map(|_| JpegOutputBuffer::new(dims, PixelFormat::Rgb8).expect("output buffer")) + .collect(), + roi, + scale, + session: JpegBatchSession::default(), + }; + this.run(bytes); + this + } + + pub(crate) fn run(&mut self, bytes: &[u8]) { + let outcomes = { + let mut jobs = self + .outputs + .iter_mut() + .map(|out| { + let stride = out.stride(); + TileRegionScaledDecodeJob { + input: bytes, + out: out.as_mut_slice(), + stride, + roi: self.roi.into(), + scale: self.scale, + } + }) + .collect::>(); + self.session + .decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8) + .expect("j2k session region-scaled tile batch") + }; + std::hint::black_box(outcomes); + std::hint::black_box(&self.outputs); + } +} + +pub(crate) fn j2k_decode_rows(bytes: &[u8]) { + let dec = Decoder::new(bytes).expect("j2k decoder"); let mut sink = NullSink; - dec.decode_rows(&mut sink).expect("signinum decode_rows"); + dec.decode_rows(&mut sink).expect("j2k decode_rows"); } -pub(crate) fn signinum_decode_tile_batch(bytes: &[u8], batch_size: usize) { - let worker_count = signinum_tile_batch_worker_count(batch_size); +pub(crate) fn j2k_decode_tile_batch(bytes: &[u8], batch_size: usize) { + let worker_count = j2k_tile_batch_worker_count(batch_size); if worker_count == 1 { - signinum_decode_tile_batch_sequential(bytes, batch_size); + j2k_decode_tile_batch_sequential(bytes, batch_size); return; } @@ -296,23 +459,23 @@ pub(crate) fn signinum_decode_tile_batch(bytes: &[u8], batch_size: usize) { let extra_tiles = batch_size % worker_count; for worker in 0..worker_count { let tile_count = base_tiles + usize::from(worker < extra_tiles); - handles.push(scope.spawn(move || signinum_decode_tile_batch_worker(bytes, tile_count))); + handles.push(scope.spawn(move || j2k_decode_tile_batch_worker(bytes, tile_count))); } for handle in handles { handle .join() - .expect("signinum decode_tile worker panicked") - .expect("signinum decode_tile batch"); + .expect("j2k decode_tile worker panicked") + .expect("j2k decode_tile batch"); } }); } -pub(crate) fn signinum_decode_tile_batch_sequential(bytes: &[u8], batch_size: usize) { - signinum_decode_tile_batch_worker(bytes, batch_size).expect("signinum decode_tile batch"); +pub(crate) fn j2k_decode_tile_batch_sequential(bytes: &[u8], batch_size: usize) { + j2k_decode_tile_batch_worker(bytes, batch_size).expect("j2k decode_tile batch"); } -fn signinum_tile_batch_worker_count(batch_size: usize) -> usize { - let configured = std::env::var("SIGNINUM_JPEG_BATCH_THREADS") +fn j2k_tile_batch_worker_count(batch_size: usize) -> usize { + let configured = std::env::var("J2K_JPEG_BATCH_THREADS") .ok() .and_then(|value| value.parse::().ok()) .and_then(std::num::NonZeroUsize::new); @@ -325,7 +488,7 @@ fn signinum_tile_batch_worker_count(batch_size: usize) -> usize { ) } -fn signinum_decode_tile_batch_worker(bytes: &[u8], tile_count: usize) -> Result<(), JpegError> { +fn j2k_decode_tile_batch_worker(bytes: &[u8], tile_count: usize) -> Result<(), JpegError> { let mut ctx = DecoderContext::new(); let mut pool = ScratchPool::new(); let mut sink = NullSink; @@ -346,14 +509,49 @@ pub(crate) fn libjpeg_turbo_decode_batch( } } -pub(crate) fn signinum_decode_tile_batch_scaled( - bytes: &[u8], - batch_size: usize, - factor: Downscale, -) { - let info = Decoder::inspect(bytes).expect("signinum inspect"); - let out_width = info.dimensions.0.div_ceil(scale_denominator(factor)); - let out_height = info.dimensions.1.div_ceil(scale_denominator(factor)); +pub(crate) struct TurboJpegBatchRgbOutputBuffers { + decoder: TurboJpegDecoder, + outputs: Vec>, + stride: usize, + dimensions: (usize, usize), +} + +impl TurboJpegBatchRgbOutputBuffers { + pub(crate) fn new(bytes: &[u8], batch_size: usize) -> Self { + let mut decoder = TurboJpegDecoder::new().expect("libjpeg-turbo decoder"); + let info = decoder.prepare_rgb(bytes).expect("libjpeg-turbo prepare"); + let stride = info.width as usize * 3; + let len = stride * info.height as usize; + let mut this = Self { + decoder, + outputs: (0..batch_size).map(|_| vec![0u8; len]).collect(), + stride, + dimensions: (info.width as usize, info.height as usize), + }; + this.run(bytes); + this + } + + pub(crate) fn run(&mut self, bytes: &[u8]) { + for out in &mut self.outputs { + self.decoder + .decode_prepared_rgb_into( + bytes, + out, + self.stride, + self.dimensions.0, + self.dimensions.1, + ) + .expect("libjpeg-turbo preallocated decode"); + } + std::hint::black_box(&self.outputs); + } +} + +pub(crate) fn j2k_decode_tile_batch_scaled(bytes: &[u8], batch_size: usize, factor: Downscale) { + let info = Decoder::inspect(bytes).expect("j2k inspect"); + let out_width = info.dimensions.0.div_ceil(factor.denominator()); + let out_height = info.dimensions.1.div_ceil(factor.denominator()); let stride = out_width as usize * 3; let len = stride * out_height as usize; let mut outputs = (0..batch_size).map(|_| vec![0u8; len]).collect::>(); @@ -368,19 +566,19 @@ pub(crate) fn signinum_decode_tile_batch_scaled( }) .collect::>(); decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgb8, TileBatchOptions::default()) - .expect("signinum production scaled tile batch") + .expect("j2k production scaled tile batch") }; std::hint::black_box(outcomes); std::hint::black_box(outputs); } -pub(crate) fn signinum_decode_tile_batch_region_scaled( +pub(crate) fn j2k_decode_tile_batch_region_scaled( bytes: &[u8], batch_size: usize, side: u32, factor: Downscale, ) { - let info = Decoder::inspect(bytes).expect("signinum inspect"); + let info = Decoder::inspect(bytes).expect("j2k inspect"); let roi = centered_roi(info.dimensions, side); let scaled = scaled_rect(roi, factor); let stride = scaled.w as usize * 3; @@ -393,23 +591,23 @@ pub(crate) fn signinum_decode_tile_batch_region_scaled( input: bytes, out: out.as_mut_slice(), stride, - roi, + roi: roi.into(), scale: factor, }) .collect::>(); decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8, TileBatchOptions::default()) - .expect("signinum production region-scaled tile batch") + .expect("j2k production region-scaled tile batch") }; std::hint::black_box(outcomes); std::hint::black_box(outputs); } -pub(crate) fn signinum_decode_region(bytes: &[u8], side: u32) { - let dec = Decoder::new(bytes).expect("signinum decoder"); +pub(crate) fn j2k_decode_region(bytes: &[u8], side: u32) { + let dec = Decoder::new(bytes).expect("j2k decoder"); let roi = centered_roi(dec.info().dimensions, side); let (out, _) = dec .decode_region(PixelFormat::Rgb8, roi) - .expect("signinum region decode"); + .expect("j2k region decode"); std::hint::black_box(out); } @@ -420,11 +618,11 @@ pub(crate) fn libjpeg_turbo_decode_region(decoder: &mut TurboJpegDecoder, bytes: std::hint::black_box(out); } -pub(crate) fn signinum_decode_scaled(bytes: &[u8], factor: Downscale) { - let dec = Decoder::new(bytes).expect("signinum decoder"); +pub(crate) fn j2k_decode_scaled(bytes: &[u8], factor: Downscale) { + let dec = Decoder::new(bytes).expect("j2k decoder"); let (out, _) = dec .decode_scaled(PixelFormat::Rgb8, factor) - .expect("signinum scaled decode"); + .expect("j2k scaled decode"); std::hint::black_box(out); } @@ -439,12 +637,12 @@ pub(crate) fn libjpeg_turbo_decode_scaled( std::hint::black_box(out); } -pub(crate) fn signinum_decode_region_scaled(bytes: &[u8], side: u32, factor: Downscale) { - let dec = Decoder::new(bytes).expect("signinum decoder"); +pub(crate) fn j2k_decode_region_scaled(bytes: &[u8], side: u32, factor: Downscale) { + let dec = Decoder::new(bytes).expect("j2k decoder"); let roi = centered_roi(dec.info().dimensions, side); let (out, _) = dec .decode_region_scaled(PixelFormat::Rgb8, roi, factor) - .expect("signinum scaled region decode"); + .expect("j2k scaled region decode"); std::hint::black_box(out); } @@ -483,7 +681,7 @@ pub(crate) fn jpeg_decoder_decode_scaled(bytes: &[u8], factor: Downscale) { &out, info.width as usize, info.height as usize, - scale_denominator(factor) as usize, + factor.denominator() as usize, ); std::hint::black_box(scaled); } @@ -498,7 +696,7 @@ pub(crate) fn jpeg_decoder_decode_region_scaled(bytes: &[u8], side: u32, factor: &cropped, roi.w as usize, roi.h as usize, - scale_denominator(factor) as usize, + factor.denominator() as usize, ); std::hint::black_box(scaled); } @@ -546,7 +744,7 @@ pub(crate) fn zune_decode_scaled(bytes: &[u8], factor: Downscale) { &out, info.width.into(), info.height.into(), - scale_denominator(factor) as usize, + factor.denominator() as usize, ); std::hint::black_box(scaled); } @@ -567,7 +765,7 @@ pub(crate) fn zune_decode_region_scaled(bytes: &[u8], side: u32, factor: Downsca &cropped, roi.w as usize, roi.h as usize, - scale_denominator(factor) as usize, + factor.denominator() as usize, ); std::hint::black_box(scaled); } @@ -647,7 +845,7 @@ pub(crate) fn centered_roi((width, height): (u32, u32), side: u32) -> Rect { } pub(crate) fn scaled_rect(rect: Rect, factor: Downscale) -> Rect { - let denom = scale_denominator(factor); + let denom = factor.denominator(); let x_end = rect.x + rect.w; let y_end = rect.y + rect.h; Rect { @@ -658,14 +856,9 @@ pub(crate) fn scaled_rect(rect: Rect, factor: Downscale) -> Rect { } } -fn scale_denominator(factor: Downscale) -> u32 { - match factor { - Downscale::None => 1, - Downscale::Half => 2, - Downscale::Quarter => 4, - Downscale::Eighth => 8, - _ => unreachable!("unsupported Downscale variant"), - } +fn scaled_dims((width, height): (u32, u32), factor: Downscale) -> (u32, u32) { + let denom = factor.denominator(); + (width.div_ceil(denom), height.div_ceil(denom)) } fn crop_rgb(full: &[u8], width: usize, roi: Rect) -> Vec { diff --git a/crates/j2k-jpeg/benches/common/report.rs b/crates/j2k-jpeg/benches/common/report.rs new file mode 100644 index 00000000..68a7df6a --- /dev/null +++ b/crates/j2k-jpeg/benches/common/report.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +pub(crate) fn report_iterations(default_iters: usize) -> usize { + std::env::var("J2K_REPORT_ITERS") + .ok() + .and_then(|raw| raw.parse::().ok()) + .filter(|&iters| iters > 0) + .unwrap_or(default_iters) +} + +pub(crate) fn median_ns(iterations: usize, mut f: impl FnMut()) -> u128 { + f(); + let mut samples = Vec::with_capacity(iterations); + for _ in 0..iterations { + let start = Instant::now(); + f(); + samples.push(start.elapsed().as_nanos()); + } + samples.sort_unstable(); + samples[samples.len() / 2] +} + +pub(crate) fn format_ns(ns: u128) -> String { + if ns >= 1_000_000 { + format!("{:.3} ms", nanos_as_secs(ns) * 1_000.0) + } else if ns >= 1_000 { + format!("{:.3} µs", nanos_as_secs(ns) * 1_000_000.0) + } else { + format!("{ns} ns") + } +} + +pub(crate) fn format_ms(ns: u128) -> String { + format!("{:.3} ms", nanos_as_secs(ns) * 1_000.0) +} + +fn nanos_as_secs(ns: u128) -> f64 { + let capped = u64::try_from(ns).unwrap_or(u64::MAX); + Duration::from_nanos(capped).as_secs_f64() +} + +pub(crate) struct ReportPaths { + pub(crate) csv: PathBuf, + pub(crate) markdown: PathBuf, +} + +pub(crate) fn write_reports( + report_dir: impl AsRef, + stem: &str, + csv: &str, + markdown: &str, +) -> ReportPaths { + let report_dir = report_dir.as_ref(); + fs::create_dir_all(report_dir).expect("create benchmark report directory"); + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock after unix epoch") + .as_secs(); + let csv_path = report_dir.join(format!("{stem}-{timestamp}.csv")); + let md_path = report_dir.join(format!("{stem}-{timestamp}.md")); + + fs::write(&csv_path, csv).expect("write CSV report"); + fs::write(&md_path, markdown).expect("write Markdown report"); + fs::write(report_dir.join(format!("{stem}-latest.csv")), csv).expect("write latest CSV report"); + fs::write(report_dir.join(format!("{stem}-latest.md")), markdown) + .expect("write latest Markdown report"); + + ReportPaths { + csv: csv_path, + markdown: md_path, + } +} diff --git a/crates/signinum-jpeg/benches/compare.rs b/crates/j2k-jpeg/benches/compare.rs similarity index 52% rename from crates/signinum-jpeg/benches/compare.rs rename to crates/j2k-jpeg/benches/compare.rs index ce393804..b3aac332 100644 --- a/crates/signinum-jpeg/benches/compare.rs +++ b/crates/j2k-jpeg/benches/compare.rs @@ -2,33 +2,36 @@ mod common; +use std::num::NonZeroUsize; + use common::{ centered_roi, classification::{should_bench_decode_rows_rgb, should_compare_full_frame, CorpusInputClass}, - jpeg_decoder_decode, jpeg_decoder_decode_batch_region_scaled, jpeg_decoder_decode_batch_scaled, + j2k_decode, j2k_decode_region, j2k_decode_region_scaled, j2k_decode_reused, j2k_decode_rows, + j2k_decode_scaled, j2k_decode_tile_batch_region_scaled, j2k_decode_tile_batch_scaled, + j2k_decode_with_scratch, j2k_inspect, jpeg_decoder_decode, + jpeg_decoder_decode_batch_region_scaled, jpeg_decoder_decode_batch_scaled, jpeg_decoder_decode_region, jpeg_decoder_decode_region_scaled, jpeg_decoder_decode_scaled, jpeg_decoder_inspect, libjpeg_turbo_available, libjpeg_turbo_decode, libjpeg_turbo_decode_batch, libjpeg_turbo_decode_batch_region_scaled, libjpeg_turbo_decode_batch_scaled, libjpeg_turbo_decode_region, libjpeg_turbo_decode_region_scaled, libjpeg_turbo_decode_scaled, libjpeg_turbo_inspect, - load_bench_inputs, output_geometry, signinum_decode, signinum_decode_region, - signinum_decode_region_scaled, signinum_decode_reused, signinum_decode_rows, - signinum_decode_scaled, signinum_decode_tile_batch_region_scaled, - signinum_decode_tile_batch_scaled, signinum_decode_with_scratch, signinum_inspect, zune_decode, - zune_decode_batch_region_scaled, zune_decode_batch_scaled, zune_decode_region, - zune_decode_region_scaled, zune_decode_scaled, zune_inspect, DecodeMode, - SigninumTileBatchRgbScratch, TurboJpegDecoder, + load_bench_inputs, output_geometry, zune_decode, zune_decode_batch_region_scaled, + zune_decode_batch_scaled, zune_decode_region, zune_decode_region_scaled, zune_decode_scaled, + zune_inspect, DecodeMode, J2KTileBatchRegionScaledRgbSession, J2KTileBatchRgbOutputBuffers, + J2KTileBatchRgbScratch, J2KTileBatchRgbSession, J2KTileBatchScaledRgbSession, + TurboJpegBatchRgbOutputBuffers, TurboJpegDecoder, }; use criterion::{criterion_group, criterion_main, Criterion}; -use signinum_jpeg::{Decoder, Downscale, ScratchPool}; +use j2k_jpeg::{Decoder, Downscale, ScratchPool, TileBatchOptions}; fn bench_compare(c: &mut Criterion) { let inputs = load_bench_inputs(); let mut inspect = c.benchmark_group("inspect"); for input in &inputs { - inspect.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_inspect(&input.bytes)); + inspect.bench_function(format!("j2k/{}", input.name), |b| { + b.iter(|| j2k_inspect(&input.bytes)); }); inspect.bench_function(format!("jpeg-decoder/{}", input.name), |b| { b.iter(|| jpeg_decoder_inspect(&input.bytes)); @@ -49,8 +52,8 @@ fn bench_compare(c: &mut Criterion) { for input in inputs.iter().filter(|input| { input.mode == DecodeMode::Rgb && should_compare_full_frame(input.mode, input.input_class) }) { - decode_rgb.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_decode(&input.bytes, DecodeMode::Rgb)); + decode_rgb.bench_function(format!("j2k/{}", input.name), |b| { + b.iter(|| j2k_decode(&input.bytes, DecodeMode::Rgb)); }); decode_rgb.bench_function(format!("jpeg-decoder/{}", input.name), |b| { b.iter(|| jpeg_decoder_decode(&input.bytes)); @@ -71,8 +74,8 @@ fn bench_compare(c: &mut Criterion) { for input in inputs.iter().filter(|input| { input.mode == DecodeMode::Gray && should_compare_full_frame(input.mode, input.input_class) }) { - decode_gray.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_decode(&input.bytes, DecodeMode::Gray)); + decode_gray.bench_function(format!("j2k/{}", input.name), |b| { + b.iter(|| j2k_decode(&input.bytes, DecodeMode::Gray)); }); decode_gray.bench_function(format!("jpeg-decoder/{}", input.name), |b| { b.iter(|| jpeg_decoder_decode(&input.bytes)); @@ -93,11 +96,11 @@ fn bench_compare(c: &mut Criterion) { for input in inputs.iter().filter(|input| { input.mode == DecodeMode::Rgb && input.input_class == CorpusInputClass::BoundedFullFrame }) { - let dec = Decoder::new(&input.bytes).expect("signinum decoder (reused-setup)"); + let dec = Decoder::new(&input.bytes).expect("j2k decoder (reused-setup)"); let (fmt, stride, len) = output_geometry(&dec, DecodeMode::Rgb); let mut out = vec![0u8; len]; - decode_reused_rgb.bench_function(format!("signinum_reused/{}", input.name), |b| { - b.iter(|| signinum_decode_reused(&dec, &mut out, stride, fmt)); + decode_reused_rgb.bench_function(format!("j2k_reused/{}", input.name), |b| { + b.iter(|| j2k_decode_reused(&dec, &mut out, stride, fmt)); }); } decode_reused_rgb.finish(); @@ -106,11 +109,11 @@ fn bench_compare(c: &mut Criterion) { for input in inputs.iter().filter(|input| { input.mode == DecodeMode::Gray && input.input_class == CorpusInputClass::BoundedFullFrame }) { - let dec = Decoder::new(&input.bytes).expect("signinum decoder (reused-setup)"); + let dec = Decoder::new(&input.bytes).expect("j2k decoder (reused-setup)"); let (fmt, stride, len) = output_geometry(&dec, DecodeMode::Gray); let mut out = vec![0u8; len]; - decode_reused_gray.bench_function(format!("signinum_reused/{}", input.name), |b| { - b.iter(|| signinum_decode_reused(&dec, &mut out, stride, fmt)); + decode_reused_gray.bench_function(format!("j2k_reused/{}", input.name), |b| { + b.iter(|| j2k_decode_reused(&dec, &mut out, stride, fmt)); }); } decode_reused_gray.finish(); @@ -119,14 +122,14 @@ fn bench_compare(c: &mut Criterion) { for input in inputs.iter().filter(|input| { input.mode == DecodeMode::Rgb && input.input_class == CorpusInputClass::BoundedFullFrame }) { - let dec = Decoder::new(&input.bytes).expect("signinum decoder (scratch-setup)"); + let dec = Decoder::new(&input.bytes).expect("j2k decoder (scratch-setup)"); let (fmt, stride, len) = output_geometry(&dec, DecodeMode::Rgb); let mut out = vec![0u8; len]; let mut pool = ScratchPool::new(); // Warm the pool once so iteration 1 pays zero allocation cost. - signinum_decode_with_scratch(&dec, &mut pool, &mut out, stride, fmt); - decode_scratch_rgb.bench_function(format!("signinum_scratch/{}", input.name), |b| { - b.iter(|| signinum_decode_with_scratch(&dec, &mut pool, &mut out, stride, fmt)); + j2k_decode_with_scratch(&dec, &mut pool, &mut out, stride, fmt); + decode_scratch_rgb.bench_function(format!("j2k_scratch/{}", input.name), |b| { + b.iter(|| j2k_decode_with_scratch(&dec, &mut pool, &mut out, stride, fmt)); }); } decode_scratch_rgb.finish(); @@ -135,13 +138,13 @@ fn bench_compare(c: &mut Criterion) { for input in inputs.iter().filter(|input| { input.mode == DecodeMode::Gray && input.input_class == CorpusInputClass::BoundedFullFrame }) { - let dec = Decoder::new(&input.bytes).expect("signinum decoder (scratch-setup)"); + let dec = Decoder::new(&input.bytes).expect("j2k decoder (scratch-setup)"); let (fmt, stride, len) = output_geometry(&dec, DecodeMode::Gray); let mut out = vec![0u8; len]; let mut pool = ScratchPool::new(); - signinum_decode_with_scratch(&dec, &mut pool, &mut out, stride, fmt); - decode_scratch_gray.bench_function(format!("signinum_scratch/{}", input.name), |b| { - b.iter(|| signinum_decode_with_scratch(&dec, &mut pool, &mut out, stride, fmt)); + j2k_decode_with_scratch(&dec, &mut pool, &mut out, stride, fmt); + decode_scratch_gray.bench_function(format!("j2k_scratch/{}", input.name), |b| { + b.iter(|| j2k_decode_with_scratch(&dec, &mut pool, &mut out, stride, fmt)); }); } decode_scratch_gray.finish(); @@ -153,8 +156,8 @@ fn bench_compare(c: &mut Criterion) { .iter() .filter(|input| should_bench_decode_rows_rgb(input.mode, input.input_class)) { - decode_rows_rgb.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_decode_rows(&input.bytes)); + decode_rows_rgb.bench_function(format!("j2k/{}", input.name), |b| { + b.iter(|| j2k_decode_rows(&input.bytes)); }); } decode_rows_rgb.finish(); @@ -163,9 +166,9 @@ fn bench_compare(c: &mut Criterion) { let mut wsi_tile_batch_rgb = c.benchmark_group("wsi_tile_batch_rgb"); for input in inputs.iter().filter(|input| input.mode == DecodeMode::Rgb) { let bytes = &input.bytes; - let mut signinum_batch = SigninumTileBatchRgbScratch::new(bytes, batch_size); - wsi_tile_batch_rgb.bench_function(format!("signinum/{}", input.name), move |b| { - b.iter(|| signinum_batch.run(bytes)); + let mut j2k_batch = J2KTileBatchRgbScratch::new(bytes, batch_size); + wsi_tile_batch_rgb.bench_function(format!("j2k/{}", input.name), move |b| { + b.iter(|| j2k_batch.run(bytes)); }); if libjpeg_turbo_available() { let mut turbo = TurboJpegDecoder::new().expect("libjpeg-turbo decoder"); @@ -176,13 +179,110 @@ fn bench_compare(c: &mut Criterion) { } wsi_tile_batch_rgb.finish(); + let mut wsi_tile_batch_session_rgb = c.benchmark_group("wsi_tile_batch_session_rgb"); + for batch_size in [16usize, 64, 256] { + for input in inputs.iter().filter(|input| input.mode == DecodeMode::Rgb) { + let bytes = &input.bytes; + let mut current_batch = J2KTileBatchRgbScratch::new(bytes, batch_size); + wsi_tile_batch_session_rgb.bench_function( + format!("current_free_batch/{}/{}", batch_size, input.name), + move |b| b.iter(|| current_batch.run(bytes)), + ); + + for workers in [1usize, 2, 4] { + let bytes = &input.bytes; + let options = TileBatchOptions { + workers: NonZeroUsize::new(workers), + }; + let mut fixed_worker_batch = + J2KTileBatchRgbScratch::new_with_options(bytes, batch_size, options); + wsi_tile_batch_session_rgb.bench_function( + format!( + "current_free_batch_workers_{workers}/{}/{}", + batch_size, input.name + ), + move |b| b.iter(|| fixed_worker_batch.run(bytes)), + ); + } + + let bytes = &input.bytes; + let mut session_batch = J2KTileBatchRgbSession::new(bytes, batch_size); + wsi_tile_batch_session_rgb.bench_function( + format!("warm_session/{}/{}", batch_size, input.name), + move |b| b.iter(|| session_batch.run(bytes)), + ); + + let bytes = &input.bytes; + let mut output_session = J2KTileBatchRgbOutputBuffers::new(bytes, batch_size); + wsi_tile_batch_session_rgb.bench_function( + format!("warm_session_output_buffers/{}/{}", batch_size, input.name), + move |b| b.iter(|| output_session.run(bytes)), + ); + + if libjpeg_turbo_available() { + let bytes = &input.bytes; + let mut turbo_output_session = + TurboJpegBatchRgbOutputBuffers::new(bytes, batch_size); + wsi_tile_batch_session_rgb.bench_function( + format!("libjpeg-turbo_prealloc/{}/{}", batch_size, input.name), + move |b| b.iter(|| turbo_output_session.run(bytes)), + ); + } + + let bytes = &input.bytes; + wsi_tile_batch_session_rgb.bench_function( + format!("current_scaled_q4/{}/{}", batch_size, input.name), + move |b| { + b.iter(|| { + j2k_decode_tile_batch_scaled(bytes, batch_size, Downscale::Quarter); + }); + }, + ); + + let bytes = &input.bytes; + let mut scaled_session = + J2KTileBatchScaledRgbSession::new(bytes, batch_size, Downscale::Quarter); + wsi_tile_batch_session_rgb.bench_function( + format!("warm_session_scaled_q4/{}/{}", batch_size, input.name), + move |b| b.iter(|| scaled_session.run(bytes)), + ); + + let bytes = &input.bytes; + wsi_tile_batch_session_rgb.bench_function( + format!("current_region_scaled_q4/{}/{}", batch_size, input.name), + move |b| { + b.iter(|| { + j2k_decode_tile_batch_region_scaled( + bytes, + batch_size, + 256, + Downscale::Quarter, + ); + }); + }, + ); + + let bytes = &input.bytes; + let mut region_scaled_session = + J2KTileBatchRegionScaledRgbSession::new(bytes, batch_size, 256, Downscale::Quarter); + wsi_tile_batch_session_rgb.bench_function( + format!( + "warm_session_region_scaled_q4/{}/{}", + batch_size, input.name + ), + move |b| b.iter(|| region_scaled_session.run(bytes)), + ); + } + } + wsi_tile_batch_session_rgb.finish(); + let mut wsi_region_rgb = c.benchmark_group("wsi_region_rgb"); for input in inputs.iter().filter(|input| { input.mode == DecodeMode::Rgb && input.input_class == CorpusInputClass::BoundedFullFrame }) { let roi = centered_roi(input.dimensions, 256); - wsi_region_rgb.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_decode_region(&input.bytes, 256)); + wsi_region_rgb.bench_function(format!("j2k/{}", input.name), |b| { + b.iter(|| j2k_decode_region(&input.bytes, 256)); }); wsi_region_rgb.bench_function(format!("jpeg-decoder/{}", input.name), |b| { b.iter(|| jpeg_decoder_decode_region(&input.bytes, 256)); @@ -199,127 +299,72 @@ fn bench_compare(c: &mut Criterion) { } wsi_region_rgb.finish(); - let mut wsi_scaled_rgb_q4 = c.benchmark_group("wsi_scaled_rgb_q4"); - for input in inputs.iter().filter(|input| { - input.mode == DecodeMode::Rgb && input.input_class == CorpusInputClass::BoundedFullFrame - }) { - wsi_scaled_rgb_q4.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_decode_scaled(&input.bytes, Downscale::Quarter)); - }); - wsi_scaled_rgb_q4.bench_function(format!("jpeg-decoder/{}", input.name), |b| { - b.iter(|| jpeg_decoder_decode_scaled(&input.bytes, Downscale::Quarter)); - }); - wsi_scaled_rgb_q4.bench_function(format!("zune-jpeg/{}", input.name), |b| { - b.iter(|| zune_decode_scaled(&input.bytes, Downscale::Quarter)); - }); - if libjpeg_turbo_available() { - let mut turbo = TurboJpegDecoder::new().expect("libjpeg-turbo decoder"); - wsi_scaled_rgb_q4.bench_function(format!("libjpeg-turbo/{}", input.name), move |b| { - b.iter(|| { - libjpeg_turbo_decode_scaled(&mut turbo, &input.bytes, Downscale::Quarter); - }); + for (group_name, scale) in [ + ("wsi_scaled_rgb_q4", Downscale::Quarter), + ("wsi_scaled_rgb_q8", Downscale::Eighth), + ] { + let mut group = c.benchmark_group(group_name); + for input in inputs.iter().filter(|input| { + input.mode == DecodeMode::Rgb && input.input_class == CorpusInputClass::BoundedFullFrame + }) { + group.bench_function(format!("j2k/{}", input.name), |b| { + b.iter(|| j2k_decode_scaled(&input.bytes, scale)); }); - } - } - wsi_scaled_rgb_q4.finish(); - - let mut wsi_scaled_rgb_q8 = c.benchmark_group("wsi_scaled_rgb_q8"); - for input in inputs.iter().filter(|input| { - input.mode == DecodeMode::Rgb && input.input_class == CorpusInputClass::BoundedFullFrame - }) { - wsi_scaled_rgb_q8.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_decode_scaled(&input.bytes, Downscale::Eighth)); - }); - wsi_scaled_rgb_q8.bench_function(format!("jpeg-decoder/{}", input.name), |b| { - b.iter(|| jpeg_decoder_decode_scaled(&input.bytes, Downscale::Eighth)); - }); - wsi_scaled_rgb_q8.bench_function(format!("zune-jpeg/{}", input.name), |b| { - b.iter(|| zune_decode_scaled(&input.bytes, Downscale::Eighth)); - }); - if libjpeg_turbo_available() { - let mut turbo = TurboJpegDecoder::new().expect("libjpeg-turbo decoder"); - wsi_scaled_rgb_q8.bench_function(format!("libjpeg-turbo/{}", input.name), move |b| { - b.iter(|| libjpeg_turbo_decode_scaled(&mut turbo, &input.bytes, Downscale::Eighth)); + group.bench_function(format!("jpeg-decoder/{}", input.name), |b| { + b.iter(|| jpeg_decoder_decode_scaled(&input.bytes, scale)); }); - } - } - wsi_scaled_rgb_q8.finish(); - - let mut wsi_region_scaled_rgb_q4 = c.benchmark_group("wsi_region_scaled_rgb_q4"); - for input in inputs.iter().filter(|input| { - input.mode == DecodeMode::Rgb && input.input_class == CorpusInputClass::BoundedFullFrame - }) { - let roi = centered_roi(input.dimensions, 256); - wsi_region_scaled_rgb_q4.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_decode_region_scaled(&input.bytes, 256, Downscale::Quarter)); - }); - wsi_region_scaled_rgb_q4.bench_function(format!("jpeg-decoder/{}", input.name), |b| { - b.iter(|| { - jpeg_decoder_decode_region_scaled(&input.bytes, 256, Downscale::Quarter); + group.bench_function(format!("zune-jpeg/{}", input.name), |b| { + b.iter(|| zune_decode_scaled(&input.bytes, scale)); }); - }); - wsi_region_scaled_rgb_q4.bench_function(format!("zune-jpeg/{}", input.name), |b| { - b.iter(|| zune_decode_region_scaled(&input.bytes, 256, Downscale::Quarter)); - }); - if libjpeg_turbo_available() { - let mut turbo = TurboJpegDecoder::new().expect("libjpeg-turbo decoder"); - wsi_region_scaled_rgb_q4.bench_function( - format!("libjpeg-turbo/{}", input.name), - move |b| { + if libjpeg_turbo_available() { + let mut turbo = TurboJpegDecoder::new().expect("libjpeg-turbo decoder"); + group.bench_function(format!("libjpeg-turbo/{}", input.name), move |b| { b.iter(|| { - libjpeg_turbo_decode_region_scaled( - &mut turbo, - &input.bytes, - roi, - Downscale::Quarter, - ); + libjpeg_turbo_decode_scaled(&mut turbo, &input.bytes, scale); }); - }, - ); + }); + } } + group.finish(); } - wsi_region_scaled_rgb_q4.finish(); - let mut wsi_region_scaled_rgb_q8 = c.benchmark_group("wsi_region_scaled_rgb_q8"); - for input in inputs.iter().filter(|input| { - input.mode == DecodeMode::Rgb && input.input_class == CorpusInputClass::BoundedFullFrame - }) { - let roi = centered_roi(input.dimensions, 256); - wsi_region_scaled_rgb_q8.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_decode_region_scaled(&input.bytes, 256, Downscale::Eighth)); - }); - wsi_region_scaled_rgb_q8.bench_function(format!("jpeg-decoder/{}", input.name), |b| { - b.iter(|| { - jpeg_decoder_decode_region_scaled(&input.bytes, 256, Downscale::Eighth); + for (group_name, scale) in [ + ("wsi_region_scaled_rgb_q4", Downscale::Quarter), + ("wsi_region_scaled_rgb_q8", Downscale::Eighth), + ] { + let mut group = c.benchmark_group(group_name); + for input in inputs.iter().filter(|input| { + input.mode == DecodeMode::Rgb && input.input_class == CorpusInputClass::BoundedFullFrame + }) { + let roi = centered_roi(input.dimensions, 256); + group.bench_function(format!("j2k/{}", input.name), |b| { + b.iter(|| j2k_decode_region_scaled(&input.bytes, 256, scale)); }); - }); - wsi_region_scaled_rgb_q8.bench_function(format!("zune-jpeg/{}", input.name), |b| { - b.iter(|| zune_decode_region_scaled(&input.bytes, 256, Downscale::Eighth)); - }); - if libjpeg_turbo_available() { - let mut turbo = TurboJpegDecoder::new().expect("libjpeg-turbo decoder"); - wsi_region_scaled_rgb_q8.bench_function( - format!("libjpeg-turbo/{}", input.name), - move |b| { + group.bench_function(format!("jpeg-decoder/{}", input.name), |b| { + b.iter(|| { + jpeg_decoder_decode_region_scaled(&input.bytes, 256, scale); + }); + }); + group.bench_function(format!("zune-jpeg/{}", input.name), |b| { + b.iter(|| zune_decode_region_scaled(&input.bytes, 256, scale)); + }); + if libjpeg_turbo_available() { + let mut turbo = TurboJpegDecoder::new().expect("libjpeg-turbo decoder"); + group.bench_function(format!("libjpeg-turbo/{}", input.name), move |b| { b.iter(|| { - libjpeg_turbo_decode_region_scaled( - &mut turbo, - &input.bytes, - roi, - Downscale::Eighth, - ); + libjpeg_turbo_decode_region_scaled(&mut turbo, &input.bytes, roi, scale); }); - }, - ); + }); + } } + group.finish(); } - wsi_region_scaled_rgb_q8.finish(); let mut wsi_tile_batch_scaled_rgb_q4 = c.benchmark_group("wsi_tile_batch_scaled_rgb_q4"); for input in inputs.iter().filter(|input| input.mode == DecodeMode::Rgb) { - wsi_tile_batch_scaled_rgb_q4.bench_function(format!("signinum/{}", input.name), |b| { + wsi_tile_batch_scaled_rgb_q4.bench_function(format!("j2k/{}", input.name), |b| { b.iter(|| { - signinum_decode_tile_batch_scaled(&input.bytes, 64, Downscale::Quarter); + j2k_decode_tile_batch_scaled(&input.bytes, 64, Downscale::Quarter); }); }); wsi_tile_batch_scaled_rgb_q4.bench_function(format!("jpeg-decoder/{}", input.name), |b| { @@ -351,19 +396,11 @@ fn bench_compare(c: &mut Criterion) { c.benchmark_group("wsi_tile_batch_region_scaled_rgb_q4"); for input in inputs.iter().filter(|input| input.mode == DecodeMode::Rgb) { let roi = centered_roi(input.dimensions, 256); - wsi_tile_batch_region_scaled_rgb_q4.bench_function( - format!("signinum/{}", input.name), - |b| { - b.iter(|| { - signinum_decode_tile_batch_region_scaled( - &input.bytes, - 64, - 256, - Downscale::Quarter, - ); - }); - }, - ); + wsi_tile_batch_region_scaled_rgb_q4.bench_function(format!("j2k/{}", input.name), |b| { + b.iter(|| { + j2k_decode_tile_batch_region_scaled(&input.bytes, 64, 256, Downscale::Quarter); + }); + }); wsi_tile_batch_region_scaled_rgb_q4.bench_function( format!("jpeg-decoder/{}", input.name), |b| { @@ -407,7 +444,7 @@ fn bench_compare(c: &mut Criterion) { } fn wsi_tile_batch_size() -> usize { - std::env::var("SIGNINUM_JPEG_TILE_BATCH_SIZE") + std::env::var("J2K_JPEG_TILE_BATCH_SIZE") .ok() .and_then(|value| value.parse::().ok()) .filter(|&value| value > 0) diff --git a/crates/signinum-jpeg/benches/corpus_report.rs b/crates/j2k-jpeg/benches/corpus_report.rs similarity index 81% rename from crates/signinum-jpeg/benches/corpus_report.rs rename to crates/j2k-jpeg/benches/corpus_report.rs index 8fb15daa..11ede674 100644 --- a/crates/signinum-jpeg/benches/corpus_report.rs +++ b/crates/j2k-jpeg/benches/corpus_report.rs @@ -14,21 +14,21 @@ mod common; use common::{ centered_roi, classification::{should_compare_full_frame, CorpusInputClass}, + j2k_decode, j2k_decode_region, j2k_decode_region_scaled, j2k_decode_rows, j2k_decode_scaled, + j2k_decode_tile_batch_region_scaled, j2k_decode_tile_batch_scaled, j2k_inspect, jpeg_decoder_decode, jpeg_decoder_decode_batch_region_scaled, jpeg_decoder_decode_batch_scaled, jpeg_decoder_decode_region, jpeg_decoder_decode_region_scaled, jpeg_decoder_decode_scaled, - jpeg_decoder_inspect, load_bench_inputs, scaled_rect, signinum_decode, signinum_decode_region, - signinum_decode_region_scaled, signinum_decode_rows, signinum_decode_scaled, - signinum_decode_tile_batch_region_scaled, signinum_decode_tile_batch_scaled, signinum_inspect, - zune_decode, zune_decode_batch_region_scaled, zune_decode_batch_scaled, zune_decode_region, - zune_decode_region_scaled, zune_decode_scaled, zune_inspect, BenchInput, DecodeMode, + jpeg_decoder_inspect, load_bench_inputs, + report::{format_ns, report_iterations, write_reports}, + scaled_rect, zune_decode, zune_decode_batch_region_scaled, zune_decode_batch_scaled, + zune_decode_region, zune_decode_region_scaled, zune_decode_scaled, zune_inspect, BenchInput, + DecodeMode, }; -use signinum_jpeg::{Downscale, Rect}; +use j2k_jpeg::{Downscale, Rect}; use std::cmp::Ordering; use std::collections::BTreeMap; -use std::fs; use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::path::PathBuf; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use std::time::Instant; const ROI_SIDE: u32 = 256; const TILE_BATCH: usize = 64; @@ -37,7 +37,7 @@ const TIE_THRESHOLD: f64 = 0.01; fn main() { let mut inputs = load_bench_inputs(); - if std::env::var_os("SIGNINUM_BENCH_INPUTS").is_some() { + if std::env::var_os("J2K_BENCH_INPUTS").is_some() { inputs.retain(|input| !input.name.starts_with("repo/")); } inputs.sort_by(|lhs, rhs| { @@ -46,38 +46,19 @@ fn main() { .then_with(|| lhs.mode.cmp(&rhs.mode)) .then_with(|| lhs.name.cmp(&rhs.name)) }); - let iterations = std::env::var("SIGNINUM_REPORT_ITERS") - .ok() - .and_then(|raw| raw.parse::().ok()) - .filter(|&iters| iters > 0) - .unwrap_or(DEFAULT_ITERS); + let iterations = report_iterations(DEFAULT_ITERS); let mut rows = Vec::new(); for input in &inputs { rows.extend(run_input(input, iterations)); } - let report_dir = PathBuf::from("target/bench-reports"); - fs::create_dir_all(&report_dir).expect("create target/bench-reports"); - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock after unix epoch") - .as_secs(); - let csv_path = report_dir.join(format!("corpus-report-{timestamp}.csv")); - let md_path = report_dir.join(format!("corpus-report-{timestamp}.md")); - let latest_csv = report_dir.join("corpus-report-latest.csv"); - let latest_md = report_dir.join("corpus-report-latest.md"); - let csv = render_csv(&rows); let markdown = render_markdown(&rows, iterations); + let paths = write_reports("target/bench-reports", "corpus-report", &csv, &markdown); - fs::write(&csv_path, &csv).expect("write CSV report"); - fs::write(&md_path, &markdown).expect("write Markdown report"); - fs::write(&latest_csv, &csv).expect("write latest CSV report"); - fs::write(&latest_md, &markdown).expect("write latest Markdown report"); - - println!("Wrote {}", csv_path.display()); - println!("Wrote {}", md_path.display()); + println!("Wrote {}", paths.csv.display()); + println!("Wrote {}", paths.markdown.display()); println!(); println!("{markdown}"); } @@ -117,7 +98,7 @@ impl Operation { #[derive(Clone, Copy)] enum Library { - Signinum, + J2K, JpegDecoder, Zune, } @@ -125,7 +106,7 @@ enum Library { impl Library { fn as_str(self) -> &'static str { match self { - Self::Signinum => "signinum", + Self::J2K => "j2k", Self::JpegDecoder => "jpeg-decoder", Self::Zune => "zune-jpeg", } @@ -166,7 +147,7 @@ struct ReportRow { mode: DecodeMode, input_class: CorpusInputClass, operation: Operation, - signinum: Measurement, + j2k: Measurement, jpeg_decoder: Measurement, zune: Measurement, } @@ -235,19 +216,19 @@ fn run_input(input: &BenchInput, iterations: usize) -> Vec { Operation::DecodeRgb, iterations_for(input, Operation::DecodeRgb, iterations), )); - rows.push(run_signinum_only_row( + rows.push(run_j2k_only_row( input, Operation::DecodeRowsRgb, iterations_for(input, Operation::DecodeRowsRgb, iterations), - "comparator skipped for very large RGB input; report uses signinum decode_rows", + "comparator skipped for very large RGB input; report uses j2k decode_rows", )); } (DecodeMode::Rgb, CorpusInputClass::VeryLarge) => { - rows.push(run_signinum_only_row( + rows.push(run_j2k_only_row( input, Operation::DecodeRowsRgb, iterations_for(input, Operation::DecodeRowsRgb, iterations), - "comparator skipped for very large RGB input; report uses signinum decode_rows", + "comparator skipped for very large RGB input; report uses j2k decode_rows", )); } (DecodeMode::Gray, CorpusInputClass::BoundedFullFrame) => { @@ -360,7 +341,7 @@ fn estimated_output_bytes(input: &BenchInput, operation: Operation) -> Option (u32, u32) { } fn run_compare_row(input: &BenchInput, operation: Operation, iterations: usize) -> ReportRow { - let (signinum, jpeg_decoder, zune) = run_compare_measurements(operation, input, iterations); + let (j2k, jpeg_decoder, zune) = run_compare_measurements(operation, input, iterations); ReportRow { input_name: input.name.clone(), mode: input.mode, input_class: input.input_class, operation, - signinum, + j2k, jpeg_decoder, zune, } } -fn run_signinum_only_row( +fn run_j2k_only_row( input: &BenchInput, operation: Operation, iterations: usize, @@ -393,7 +374,7 @@ fn run_signinum_only_row( mode: input.mode, input_class: input.input_class, operation, - signinum: run_measurement(Library::Signinum, operation, input, iterations), + j2k: run_measurement(Library::J2K, operation, input, iterations), jpeg_decoder: Measurement::skipped(skip_reason), zune: Measurement::skipped(skip_reason), } @@ -430,7 +411,7 @@ fn run_measurement( fn is_supported(library: Library, operation: Operation, input: &BenchInput) -> bool { matches!( (library, operation, input.mode, input.input_class), - (Library::Signinum, Operation::Inspect, _, _) + (Library::J2K, Operation::Inspect, _, _) | (Library::JpegDecoder, Operation::Inspect, _, _) | (Library::Zune, Operation::Inspect, _, _) | ( @@ -485,7 +466,7 @@ fn is_supported(library: Library, operation: Operation, input: &BenchInput) -> b _ ) | ( - Library::Signinum, + Library::J2K, Operation::DecodeRowsRgb, DecodeMode::Rgb, CorpusInputClass::VeryLarge @@ -498,9 +479,9 @@ fn run_compare_measurements( input: &BenchInput, iterations: usize, ) -> (Measurement, Measurement, Measurement) { - let mut signinum = MeasurementState::new( - Library::Signinum, - is_supported(Library::Signinum, operation, input), + let mut j2k = MeasurementState::new( + Library::J2K, + is_supported(Library::J2K, operation, input), iterations, ); let mut jpeg_decoder = MeasurementState::new( @@ -513,7 +494,7 @@ fn run_compare_measurements( is_supported(Library::Zune, operation, input), iterations, ); - let mut states = [&mut signinum, &mut jpeg_decoder, &mut zune]; + let mut states = [&mut j2k, &mut jpeg_decoder, &mut zune]; let inner_loops = inner_loops_for(input, operation); for state in &mut states { @@ -527,7 +508,7 @@ fn run_compare_measurements( } } - (signinum.finish(), jpeg_decoder.finish(), zune.finish()) + (j2k.finish(), jpeg_decoder.finish(), zune.finish()) } fn time_operation( @@ -552,27 +533,23 @@ fn time_operation( fn run_operation(library: Library, operation: Operation, input: &BenchInput) { match (library, operation) { - (Library::Signinum, Operation::Inspect) => signinum_inspect(&input.bytes), + (Library::J2K, Operation::Inspect) => j2k_inspect(&input.bytes), (Library::JpegDecoder, Operation::Inspect) => jpeg_decoder_inspect(&input.bytes), (Library::Zune, Operation::Inspect) => zune_inspect(&input.bytes), - (Library::Signinum, Operation::DecodeRgb) => signinum_decode(&input.bytes, DecodeMode::Rgb), + (Library::J2K, Operation::DecodeRgb) => j2k_decode(&input.bytes, DecodeMode::Rgb), (Library::JpegDecoder, Operation::DecodeRgb) => jpeg_decoder_decode(&input.bytes), (Library::Zune, Operation::DecodeRgb) => zune_decode(&input.bytes, DecodeMode::Rgb), - (Library::Signinum, Operation::DecodeGray) => { - signinum_decode(&input.bytes, DecodeMode::Gray) - } + (Library::J2K, Operation::DecodeGray) => j2k_decode(&input.bytes, DecodeMode::Gray), (Library::JpegDecoder, Operation::DecodeGray) => jpeg_decoder_decode(&input.bytes), (Library::Zune, Operation::DecodeGray) => zune_decode(&input.bytes, DecodeMode::Gray), - (Library::Signinum, Operation::DecodeRowsRgb) => signinum_decode_rows(&input.bytes), - (Library::Signinum, Operation::WsiRegionRgb) => { - signinum_decode_region(&input.bytes, ROI_SIDE) - } + (Library::J2K, Operation::DecodeRowsRgb) => j2k_decode_rows(&input.bytes), + (Library::J2K, Operation::WsiRegionRgb) => j2k_decode_region(&input.bytes, ROI_SIDE), (Library::JpegDecoder, Operation::WsiRegionRgb) => { jpeg_decoder_decode_region(&input.bytes, ROI_SIDE) } (Library::Zune, Operation::WsiRegionRgb) => zune_decode_region(&input.bytes, ROI_SIDE), - (Library::Signinum, Operation::WsiScaledRgbQ4) => { - signinum_decode_scaled(&input.bytes, Downscale::Quarter); + (Library::J2K, Operation::WsiScaledRgbQ4) => { + j2k_decode_scaled(&input.bytes, Downscale::Quarter); } (Library::JpegDecoder, Operation::WsiScaledRgbQ4) => { jpeg_decoder_decode_scaled(&input.bytes, Downscale::Quarter); @@ -580,8 +557,8 @@ fn run_operation(library: Library, operation: Operation, input: &BenchInput) { (Library::Zune, Operation::WsiScaledRgbQ4) => { zune_decode_scaled(&input.bytes, Downscale::Quarter); } - (Library::Signinum, Operation::WsiScaledRgbQ8) => { - signinum_decode_scaled(&input.bytes, Downscale::Eighth); + (Library::J2K, Operation::WsiScaledRgbQ8) => { + j2k_decode_scaled(&input.bytes, Downscale::Eighth); } (Library::JpegDecoder, Operation::WsiScaledRgbQ8) => { jpeg_decoder_decode_scaled(&input.bytes, Downscale::Eighth); @@ -589,8 +566,8 @@ fn run_operation(library: Library, operation: Operation, input: &BenchInput) { (Library::Zune, Operation::WsiScaledRgbQ8) => { zune_decode_scaled(&input.bytes, Downscale::Eighth); } - (Library::Signinum, Operation::WsiRegionScaledRgbQ4) => { - signinum_decode_region_scaled(&input.bytes, ROI_SIDE, Downscale::Quarter); + (Library::J2K, Operation::WsiRegionScaledRgbQ4) => { + j2k_decode_region_scaled(&input.bytes, ROI_SIDE, Downscale::Quarter); } (Library::JpegDecoder, Operation::WsiRegionScaledRgbQ4) => { jpeg_decoder_decode_region_scaled(&input.bytes, ROI_SIDE, Downscale::Quarter); @@ -598,8 +575,8 @@ fn run_operation(library: Library, operation: Operation, input: &BenchInput) { (Library::Zune, Operation::WsiRegionScaledRgbQ4) => { zune_decode_region_scaled(&input.bytes, ROI_SIDE, Downscale::Quarter); } - (Library::Signinum, Operation::WsiRegionScaledRgbQ8) => { - signinum_decode_region_scaled(&input.bytes, ROI_SIDE, Downscale::Eighth); + (Library::J2K, Operation::WsiRegionScaledRgbQ8) => { + j2k_decode_region_scaled(&input.bytes, ROI_SIDE, Downscale::Eighth); } (Library::JpegDecoder, Operation::WsiRegionScaledRgbQ8) => { jpeg_decoder_decode_region_scaled(&input.bytes, ROI_SIDE, Downscale::Eighth); @@ -607,8 +584,8 @@ fn run_operation(library: Library, operation: Operation, input: &BenchInput) { (Library::Zune, Operation::WsiRegionScaledRgbQ8) => { zune_decode_region_scaled(&input.bytes, ROI_SIDE, Downscale::Eighth); } - (Library::Signinum, Operation::WsiTileBatchScaledRgbQ4) => { - signinum_decode_tile_batch_scaled(&input.bytes, TILE_BATCH, Downscale::Quarter); + (Library::J2K, Operation::WsiTileBatchScaledRgbQ4) => { + j2k_decode_tile_batch_scaled(&input.bytes, TILE_BATCH, Downscale::Quarter); } (Library::JpegDecoder, Operation::WsiTileBatchScaledRgbQ4) => { jpeg_decoder_decode_batch_scaled(&input.bytes, TILE_BATCH, Downscale::Quarter); @@ -616,8 +593,8 @@ fn run_operation(library: Library, operation: Operation, input: &BenchInput) { (Library::Zune, Operation::WsiTileBatchScaledRgbQ4) => { zune_decode_batch_scaled(&input.bytes, TILE_BATCH, Downscale::Quarter); } - (Library::Signinum, Operation::WsiTileBatchRegionScaledRgbQ4) => { - signinum_decode_tile_batch_region_scaled( + (Library::J2K, Operation::WsiTileBatchRegionScaledRgbQ4) => { + j2k_decode_tile_batch_region_scaled( &input.bytes, TILE_BATCH, ROI_SIDE, @@ -651,7 +628,7 @@ fn panic_message(payload: Box) -> String { fn render_csv(rows: &[ReportRow]) -> String { let mut csv = String::from( - "input,mode,class,operation,signinum_ns,jpeg_decoder_ns,zune_ns,signinum_error,jpeg_decoder_error,zune_error,fastest\n", + "input,mode,class,operation, j2k_ns,jpeg_decoder_ns,zune_ns, j2k_error,jpeg_decoder_error,zune_error,fastest\n", ); for row in rows { let fastest = fastest_label(row).unwrap_or("n/a"); @@ -661,10 +638,10 @@ fn render_csv(rows: &[ReportRow]) -> String { row.mode.as_str(), row.input_class.as_str(), row.operation.as_str(), - render_ns(&row.signinum), + render_ns(&row.j2k), render_ns(&row.jpeg_decoder), render_ns(&row.zune), - escape_csv(row.signinum.error.as_deref().unwrap_or("")), + escape_csv(row.j2k.error.as_deref().unwrap_or("")), escape_csv(row.jpeg_decoder.error.as_deref().unwrap_or("")), escape_csv(row.zune.error.as_deref().unwrap_or("")), fastest @@ -683,7 +660,7 @@ fn render_markdown(rows: &[ReportRow], iterations: usize) -> String { } let mut md = String::new(); - md.push_str("# Signinum JPEG corpus report\n\n"); + md.push_str("# J2K JPEG corpus report\n\n"); md.push_str(&format!( "- inputs: {}\n", rows.iter() @@ -698,13 +675,13 @@ fn render_markdown(rows: &[ReportRow], iterations: usize) -> String { TIE_THRESHOLD * 100.0 )); md.push_str("## Summary by operation\n\n"); - md.push_str("| operation | signinum fastest | vs jpeg wins | vs jpeg losses | vs zune wins | vs zune losses | failures |\n"); + md.push_str("| operation | j2k fastest | vs jpeg wins | vs jpeg losses | vs zune wins | vs zune losses | failures |\n"); md.push_str("|---|---:|---:|---:|---:|---:|---:|\n"); for (operation, stats) in &summary { md.push_str(&format!( "| {} | {} | {} | {} | {} | {} | {} |\n", operation, - stats.signinum_fastest, + stats.j2k_fastest, stats.vs_jpeg_wins, stats.vs_jpeg_losses, stats.vs_zune_wins, @@ -712,13 +689,13 @@ fn render_markdown(rows: &[ReportRow], iterations: usize) -> String { stats.failures, )); } - md.push_str("\n## Rows where signinum is not fastest\n\n"); - md.push_str("| input | operation | signinum | jpeg-decoder | zune-jpeg | fastest |\n"); + md.push_str("\n## Rows where j2k is not fastest\n\n"); + md.push_str("| input | operation | j2k | jpeg-decoder | zune-jpeg | fastest |\n"); md.push_str("|---|---|---:|---:|---:|---|\n"); let mut any_slower = false; for row in rows { let fastest = fastest_label(row); - if fastest == Some("signinum") || fastest.is_none() { + if fastest == Some("j2k") || fastest.is_none() { continue; } any_slower = true; @@ -726,7 +703,7 @@ fn render_markdown(rows: &[ReportRow], iterations: usize) -> String { "| {} | {} | {} | {} | {} | {} |\n", row.input_name, row.operation.as_str(), - format_measurement(&row.signinum), + format_measurement(&row.j2k), format_measurement(&row.jpeg_decoder), format_measurement(&row.zune), fastest.unwrap_or("n/a"), @@ -737,14 +714,11 @@ fn render_markdown(rows: &[ReportRow], iterations: usize) -> String { } md.push_str("\n## Failures / skips\n\n"); - md.push_str("| input | operation | signinum | jpeg-decoder | zune-jpeg |\n"); + md.push_str("| input | operation | j2k | jpeg-decoder | zune-jpeg |\n"); md.push_str("|---|---|---|---|---|\n"); let mut any_failures = false; for row in rows { - if row.signinum.error.is_none() - && row.jpeg_decoder.error.is_none() - && row.zune.error.is_none() - { + if row.j2k.error.is_none() && row.jpeg_decoder.error.is_none() && row.zune.error.is_none() { continue; } any_failures = true; @@ -752,7 +726,7 @@ fn render_markdown(rows: &[ReportRow], iterations: usize) -> String { "| {} | {} | {} | {} | {} |\n", row.input_name, row.operation.as_str(), - row.signinum.error.as_deref().unwrap_or("ok"), + row.j2k.error.as_deref().unwrap_or("ok"), row.jpeg_decoder.error.as_deref().unwrap_or("ok"), row.zune.error.as_deref().unwrap_or("ok"), )); @@ -766,7 +740,7 @@ fn render_markdown(rows: &[ReportRow], iterations: usize) -> String { #[derive(Default)] struct Summary { - signinum_fastest: usize, + j2k_fastest: usize, vs_jpeg_wins: usize, vs_jpeg_losses: usize, vs_zune_wins: usize, @@ -839,23 +813,20 @@ impl MeasurementState { impl Summary { fn accumulate(&mut self, row: &ReportRow) { - if fastest_label(row) == Some("signinum") { - self.signinum_fastest += 1; + if fastest_label(row) == Some("j2k") { + self.j2k_fastest += 1; } - match compare_measurements(&row.signinum, &row.jpeg_decoder) { + match compare_measurements(&row.j2k, &row.jpeg_decoder) { Some(Ordering::Less) => self.vs_jpeg_wins += 1, Some(Ordering::Greater) => self.vs_jpeg_losses += 1, _ => {} } - match compare_measurements(&row.signinum, &row.zune) { + match compare_measurements(&row.j2k, &row.zune) { Some(Ordering::Less) => self.vs_zune_wins += 1, Some(Ordering::Greater) => self.vs_zune_losses += 1, _ => {} } - if row.signinum.error.is_some() - || row.jpeg_decoder.error.is_some() - || row.zune.error.is_some() - { + if row.j2k.error.is_some() || row.jpeg_decoder.error.is_some() || row.zune.error.is_some() { self.failures += 1; } } @@ -863,18 +834,18 @@ impl Summary { fn fastest_label(row: &ReportRow) -> Option<&'static str> { let mut best_ns: Option = None; - for measurement in [&row.signinum, &row.jpeg_decoder, &row.zune] { + for measurement in [&row.j2k, &row.jpeg_decoder, &row.zune] { let Some(ns) = measurement.ns else { continue; }; best_ns = Some(best_ns.map_or(ns, |best| best.min(ns))); } let best_ns = best_ns?; - if row.signinum.ns.is_some_and(|ns| { + if row.j2k.ns.is_some_and(|ns| { let max_ns = ns.max(best_ns) as f64; max_ns > 0.0 && ((ns as f64 - best_ns as f64).abs() / max_ns) <= TIE_THRESHOLD }) { - return Some("signinum"); + return Some("j2k"); } if row.jpeg_decoder.ns.is_some_and(|ns| ns == best_ns) { return Some("jpeg-decoder"); @@ -911,16 +882,6 @@ fn render_ns(measurement: &Measurement) -> String { measurement.ns.map_or_else(String::new, |ns| ns.to_string()) } -fn format_ns(ns: u128) -> String { - if ns >= 1_000_000 { - format!("{:.3} ms", ns as f64 / 1_000_000.0) - } else if ns >= 1_000 { - format!("{:.3} µs", ns as f64 / 1_000.0) - } else { - format!("{ns} ns") - } -} - fn escape_csv(raw: &str) -> String { raw.replace('"', "\"\"") } diff --git a/crates/j2k-jpeg/benches/encode_cpu.rs b/crates/j2k-jpeg/benches/encode_cpu.rs new file mode 100644 index 00000000..29491ca6 --- /dev/null +++ b/crates/j2k-jpeg/benches/encode_cpu.rs @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: Apache-2.0 + +use core::alloc::{GlobalAlloc, Layout}; +use core::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use criterion::{criterion_group, criterion_main, Criterion}; +use j2k_jpeg::{ + encode_jpeg_baseline, JpegBackend, JpegEncodeOptions, JpegSamples, JpegSubsampling, +}; +use j2k_test_support::{patterned_gray8, patterned_rgb8}; + +#[global_allocator] +static ALLOCATOR: CountingAllocator = + CountingAllocator::new(std::alloc::System); + +#[derive(Debug, Clone, Copy)] +struct AllocationStats { + allocations: usize, + allocated_bytes: usize, +} + +struct CountingAllocator { + inner: A, + enabled: AtomicBool, + allocations: AtomicUsize, + allocated_bytes: AtomicUsize, +} + +impl CountingAllocator { + const fn new(inner: A) -> Self { + Self { + inner, + enabled: AtomicBool::new(false), + allocations: AtomicUsize::new(0), + allocated_bytes: AtomicUsize::new(0), + } + } + + fn measure(&self, f: impl FnOnce() -> R) -> (R, AllocationStats) { + self.reset(); + self.enabled.store(true, Ordering::SeqCst); + let result = f(); + self.enabled.store(false, Ordering::SeqCst); + (result, self.stats()) + } + + fn reset(&self) { + self.allocations.store(0, Ordering::SeqCst); + self.allocated_bytes.store(0, Ordering::SeqCst); + } + + fn stats(&self) -> AllocationStats { + AllocationStats { + allocations: self.allocations.load(Ordering::SeqCst), + allocated_bytes: self.allocated_bytes.load(Ordering::SeqCst), + } + } + + fn record_alloc(&self, size: usize) { + self.allocations.fetch_add(1, Ordering::Relaxed); + self.allocated_bytes.fetch_add(size, Ordering::Relaxed); + } + + fn metering_enabled(&self) -> bool { + self.enabled.load(Ordering::Relaxed) + } +} + +// SAFETY: The wrapper forwards allocator contracts unchanged and only records counts. +unsafe impl GlobalAlloc for CountingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // SAFETY: The wrapper forwards allocator contracts unchanged and only records counts. + let ptr = unsafe { self.inner.alloc(layout) }; + if self.metering_enabled() && !ptr.is_null() { + self.record_alloc(layout.size()); + } + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + // SAFETY: The wrapper forwards allocator contracts unchanged and only records counts. + unsafe { self.inner.dealloc(ptr, layout) }; + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + // SAFETY: The wrapper forwards allocator contracts unchanged and only records counts. + let new_ptr = unsafe { self.inner.realloc(ptr, layout, new_size) }; + if self.metering_enabled() && !new_ptr.is_null() { + self.record_alloc(new_size); + } + new_ptr + } +} + +#[derive(Clone, Copy)] +enum CaseKind { + Gray, + Rgb, +} + +struct EncodeCase { + name: &'static str, + width: u32, + height: u32, + kind: CaseKind, + data: Vec, + options: JpegEncodeOptions, +} + +impl EncodeCase { + fn samples(&self) -> JpegSamples<'_> { + match self.kind { + CaseKind::Gray => JpegSamples::Gray8 { + data: &self.data, + width: self.width, + height: self.height, + }, + CaseKind::Rgb => JpegSamples::Rgb8 { + data: &self.data, + width: self.width, + height: self.height, + }, + } + } + + fn encode(&self) -> usize { + let encoded = + encode_jpeg_baseline(self.samples(), self.options).expect("JPEG CPU encode bench"); + let output_bytes = encoded.data.len(); + std::hint::black_box(encoded.backend); + std::hint::black_box(output_bytes) + } +} + +fn gray_case(name: &'static str, width: u32, height: u32, quality: u8) -> EncodeCase { + EncodeCase { + name, + width, + height, + kind: CaseKind::Gray, + data: patterned_gray8(width, height), + options: JpegEncodeOptions { + quality, + subsampling: JpegSubsampling::Gray, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + } +} + +fn rgb_case( + name: &'static str, + width: u32, + height: u32, + quality: u8, + subsampling: JpegSubsampling, + restart_interval: Option, +) -> EncodeCase { + EncodeCase { + name, + width, + height, + kind: CaseKind::Rgb, + data: patterned_rgb8(width, height), + options: JpegEncodeOptions { + quality, + subsampling, + restart_interval, + backend: JpegBackend::Cpu, + }, + } +} + +fn encode_cases() -> Vec { + vec![ + gray_case("gray8_256_default", 256, 256, 90), + gray_case("gray8_512_default", 512, 512, 90), + gray_case("gray8_257x263_default", 257, 263, 90), + rgb_case( + "rgb8_256_444_default", + 256, + 256, + 90, + JpegSubsampling::Ybr444, + None, + ), + rgb_case( + "rgb8_256_422_default", + 256, + 256, + 90, + JpegSubsampling::Ybr422, + None, + ), + rgb_case( + "rgb8_256_420_default", + 256, + 256, + 90, + JpegSubsampling::Ybr420, + None, + ), + rgb_case( + "rgb8_512_444_default", + 512, + 512, + 90, + JpegSubsampling::Ybr444, + None, + ), + rgb_case( + "rgb8_512_422_default", + 512, + 512, + 90, + JpegSubsampling::Ybr422, + None, + ), + rgb_case( + "rgb8_512_420_default", + 512, + 512, + 90, + JpegSubsampling::Ybr420, + None, + ), + rgb_case( + "rgb8_257x263_420_default", + 257, + 263, + 90, + JpegSubsampling::Ybr420, + None, + ), + rgb_case( + "rgb8_512_420_high_quality", + 512, + 512, + 98, + JpegSubsampling::Ybr420, + None, + ), + rgb_case( + "rgb8_512_420_restart_64", + 512, + 512, + 90, + JpegSubsampling::Ybr420, + Some(64), + ), + ] +} + +fn report_allocations(cases: &[EncodeCase]) { + if std::env::var_os("J2K_ALLOC_REPORT").is_none() { + return; + } + + for case in cases { + std::hint::black_box(case.encode()); + } + + for case in cases { + let (output_bytes, stats) = ALLOCATOR.measure(|| case.encode()); + println!( + "j2k_alloc case={} allocations={} allocated_bytes={} output_bytes={}", + case.name, stats.allocations, stats.allocated_bytes, output_bytes + ); + } +} + +fn bench_encode_cpu(c: &mut Criterion) { + let cases = encode_cases(); + report_allocations(&cases); + + let mut group = c.benchmark_group("jpeg_cpu_encode_runtime"); + for case in &cases { + group.bench_function(case.name, |b| { + b.iter(|| { + std::hint::black_box(case.encode()); + }); + }); + } + group.finish(); +} + +criterion_group!(benches, bench_encode_cpu); +criterion_main!(benches); diff --git a/crates/signinum-jpeg/benches/fast420_breakdown.rs b/crates/j2k-jpeg/benches/fast420_breakdown.rs similarity index 62% rename from crates/signinum-jpeg/benches/fast420_breakdown.rs rename to crates/j2k-jpeg/benches/fast420_breakdown.rs index b50d8039..95869121 100644 --- a/crates/signinum-jpeg/benches/fast420_breakdown.rs +++ b/crates/j2k-jpeg/benches/fast420_breakdown.rs @@ -9,35 +9,30 @@ mod common; use common::{ - libjpeg_turbo_available, libjpeg_turbo_decode_batch, load_bench_inputs, - signinum_decode_tile_batch_sequential, TurboJpegDecoder, + j2k_decode_tile_batch_sequential, libjpeg_turbo_available, libjpeg_turbo_decode_batch, + load_bench_inputs, + report::{format_ms, median_ns, report_iterations, write_reports}, + TurboJpegDecoder, }; -use signinum_jpeg::bench_support::{bench_profile_fast420_tile_batch, BenchFast420Profile}; -use std::fs; -use std::path::PathBuf; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use j2k_jpeg::bench_support::{bench_profile_fast420_tile_batch, BenchFast420Profile}; const TILE_BATCH: usize = 64; const DEFAULT_ITERS: usize = 3; struct BreakdownRow { input_name: String, - signinum_ns: u128, + j2k_ns: u128, turbo_ns: Option, profile: BenchFast420Profile, } fn main() { let mut inputs = load_bench_inputs(); - if std::env::var_os("SIGNINUM_BENCH_INPUTS").is_some() { + if std::env::var_os("J2K_BENCH_INPUTS").is_some() { inputs.retain(|input| !input.name.starts_with("repo/")); } - let iterations = std::env::var("SIGNINUM_REPORT_ITERS") - .ok() - .and_then(|raw| raw.parse::().ok()) - .filter(|&iters| iters > 0) - .unwrap_or(DEFAULT_ITERS); + let iterations = report_iterations(DEFAULT_ITERS); let mut turbo = if libjpeg_turbo_available() { Some(TurboJpegDecoder::new().expect("create libjpeg-turbo decoder")) @@ -53,8 +48,8 @@ fn main() { continue; }; - let signinum_ns = median_ns(iterations, || { - signinum_decode_tile_batch_sequential(&input.bytes, TILE_BATCH); + let j2k_ns = median_ns(iterations, || { + j2k_decode_tile_batch_sequential(&input.bytes, TILE_BATCH); }); let turbo_ns = turbo.as_mut().map(|decoder| { median_ns(iterations, || { @@ -64,61 +59,35 @@ fn main() { rows.push(BreakdownRow { input_name: input.name.clone(), - signinum_ns, + j2k_ns, turbo_ns, profile, }); } - let report_dir = PathBuf::from("target/bench-reports"); - fs::create_dir_all(&report_dir).expect("create target/bench-reports"); - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("system clock after unix epoch") - .as_secs(); - let csv = render_csv(&rows); let markdown = render_markdown(&rows, iterations); - let csv_path = report_dir.join(format!("fast420-breakdown-{timestamp}.csv")); - let md_path = report_dir.join(format!("fast420-breakdown-{timestamp}.md")); - fs::write(&csv_path, &csv).expect("write CSV report"); - fs::write(&md_path, &markdown).expect("write Markdown report"); - fs::write(report_dir.join("fast420-breakdown-latest.csv"), &csv) - .expect("write latest CSV report"); - fs::write(report_dir.join("fast420-breakdown-latest.md"), &markdown) - .expect("write latest Markdown report"); - - println!("Wrote {}", csv_path.display()); - println!("Wrote {}", md_path.display()); + let paths = write_reports("target/bench-reports", "fast420-breakdown", &csv, &markdown); + + println!("Wrote {}", paths.csv.display()); + println!("Wrote {}", paths.markdown.display()); println!(); println!("{markdown}"); } -fn median_ns(iterations: usize, mut f: impl FnMut()) -> u128 { - f(); - let mut samples = Vec::with_capacity(iterations); - for _ in 0..iterations { - let start = Instant::now(); - f(); - samples.push(start.elapsed().as_nanos()); - } - samples.sort_unstable(); - samples[samples.len() / 2] -} - fn render_csv(rows: &[BreakdownRow]) -> String { let mut csv = String::from( - "input,signinum_ns,turbo_ns,turbo_speedup,profile_total_ns,parse_plan_ns,mcu_decode_ns,rgb_emit_ns,finish_ns,total_blocks,dc_only_blocks,bottom_half_zero_blocks,general_blocks\n", + "input, j2k_ns,turbo_ns,turbo_speedup,profile_total_ns,parse_plan_ns,mcu_decode_ns,rgb_emit_ns,finish_ns,total_blocks,dc_only_blocks,bottom_half_zero_blocks,general_blocks\n", ); for row in rows { let counts = row.profile.block_activity_counts(); csv.push_str(&format!( "{},{},{},{},{},{},{},{},{},{},{},{},{}\n", row.input_name, - row.signinum_ns, + row.j2k_ns, row.turbo_ns.map_or_else(String::new, |ns| ns.to_string()), row.turbo_ns - .map_or_else(String::new, |turbo| ratio(row.signinum_ns, turbo)), + .map_or_else(String::new, |turbo| ratio(row.j2k_ns, turbo)), row.profile.total_ns(), row.profile.parse_plan_ns(), row.profile.mcu_decode_ns(), @@ -151,45 +120,45 @@ fn render_markdown(rows: &[BreakdownRow], iterations: usize) -> String { let (parse, mcu, rgb, finish, total) = aggregate_stage_ns(rows); md.push_str(&format!( "| profile parse/plan | {} ({:.1}%) |\n", - fmt_ms(parse), + format_ms(parse), pct(parse, total) )); md.push_str(&format!( "| profile MCU decode | {} ({:.1}%) |\n", - fmt_ms(mcu), + format_ms(mcu), pct(mcu, total) )); md.push_str(&format!( "| profile RGB emit | {} ({:.1}%) |\n", - fmt_ms(rgb), + format_ms(rgb), pct(rgb, total) )); md.push_str(&format!( "| profile finish | {} ({:.1}%) |\n", - fmt_ms(finish), + format_ms(finish), pct(finish, total) )); md.push('\n'); md.push_str("## Inputs\n\n"); - md.push_str("| input | signinum | turbo | turbo speedup | parse/plan | MCU decode | RGB emit | blocks dc/bhz/general |\n"); + md.push_str("| input | j2k | turbo | turbo speedup | parse/plan | MCU decode | RGB emit | blocks dc/bhz/general |\n"); md.push_str("|---|---:|---:|---:|---:|---:|---:|---:|\n"); for row in rows { let counts = row.profile.block_activity_counts(); md.push_str(&format!( "| {} | {} | {} | {} | {} ({:.1}%) | {} ({:.1}%) | {} ({:.1}%) | {}/{}/{} |\n", row.input_name, - fmt_ms(row.signinum_ns), - row.turbo_ns.map_or_else(|| "n/a".to_string(), fmt_ms), + format_ms(row.j2k_ns), + row.turbo_ns.map_or_else(|| "n/a".to_string(), format_ms), row.turbo_ns.map_or_else( || "n/a".to_string(), - |turbo| format!("{}x", ratio(row.signinum_ns, turbo)) + |turbo| format!("{}x", ratio(row.j2k_ns, turbo)) ), - fmt_ms(row.profile.parse_plan_ns()), + format_ms(row.profile.parse_plan_ns()), pct(row.profile.parse_plan_ns(), row.profile.total_ns()), - fmt_ms(row.profile.mcu_decode_ns()), + format_ms(row.profile.mcu_decode_ns()), pct(row.profile.mcu_decode_ns(), row.profile.total_ns()), - fmt_ms(row.profile.rgb_emit_ns()), + format_ms(row.profile.rgb_emit_ns()), pct(row.profile.rgb_emit_ns(), row.profile.total_ns()), counts.dc_only_blocks(), counts.bottom_half_zero_blocks(), @@ -216,7 +185,7 @@ fn mean_turbo_speedup(rows: &[BreakdownRow]) -> Option { let mut total = 0.0f64; for row in rows { if let Some(turbo) = row.turbo_ns { - total += row.signinum_ns as f64 / turbo as f64; + total += row.j2k_ns as f64 / turbo as f64; count += 1; } } @@ -234,7 +203,3 @@ fn pct(part: u128, total: u128) -> f64 { part as f64 * 100.0 / total as f64 } } - -fn fmt_ms(ns: u128) -> String { - format!("{:.3} ms", ns as f64 / 1_000_000.0) -} diff --git a/crates/signinum-jpeg/benches/micro.rs b/crates/j2k-jpeg/benches/micro.rs similarity index 92% rename from crates/signinum-jpeg/benches/micro.rs rename to crates/j2k-jpeg/benches/micro.rs index 27b99ab6..df43f150 100644 --- a/crates/signinum-jpeg/benches/micro.rs +++ b/crates/j2k-jpeg/benches/micro.rs @@ -1,18 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 use criterion::{criterion_group, criterion_main, Criterion}; -use signinum_jpeg::bench_support::{ +use j2k_jpeg::bench_support::{ bench_idct_reference_block, BenchColorRowScratch, BenchHuffmanState, BenchRgb420RowPairScratch, BenchUpsampleH2V2Scratch, }; -use signinum_jpeg::Decoder; +use j2k_jpeg::Decoder; +use j2k_test_support::JPEG_BASELINE_420_16X16; fn bench_micro(c: &mut Criterion) { - let small = include_bytes!("../fixtures/conformance/baseline_420_16x16.jpg"); + let small = JPEG_BASELINE_420_16X16; c.bench_function("micro/inspect_small", |b| { b.iter(|| { - let info = Decoder::inspect(small).expect("signinum inspect"); + let info = Decoder::inspect(small).expect("j2k inspect"); std::hint::black_box(info); }); }); @@ -57,7 +58,7 @@ fn bench_micro(c: &mut Criterion) { bottom_half_zero[24] = 11; { - use signinum_jpeg::bench_support::{ + use j2k_jpeg::bench_support::{ bench_idct_dc_only_block_with, bench_idct_reference_block_with, }; c.bench_function("micro/idct_islow_scalar_block", |b| { @@ -86,7 +87,7 @@ fn bench_micro(c: &mut Criterion) { } { - use signinum_jpeg::bench_support::bench_idct_reduced_2x2_block_with; + use j2k_jpeg::bench_support::bench_idct_reduced_2x2_block_with; c.bench_function("micro/idct_islow_2x2_scalar_block", |b| { let mut out = [0u8; 4]; b.iter(|| { @@ -98,7 +99,7 @@ fn bench_micro(c: &mut Criterion) { #[cfg(target_arch = "aarch64")] { - use signinum_jpeg::bench_support::bench_idct_neon_block; + use j2k_jpeg::bench_support::bench_idct_neon_block; c.bench_function("micro/idct_islow_neon_block", |b| { let mut out = [0u8; 64]; b.iter(|| { @@ -119,7 +120,7 @@ fn bench_micro(c: &mut Criterion) { #[cfg(target_arch = "x86_64")] { if std::is_x86_feature_detected!("avx2") { - use signinum_jpeg::bench_support::bench_idct_avx2_block; + use j2k_jpeg::bench_support::bench_idct_avx2_block; c.bench_function("micro/idct_islow_avx2_block", |b| { let mut out = [0u8; 64]; b.iter(|| { diff --git a/crates/signinum-jpeg/build.rs b/crates/j2k-jpeg/build.rs similarity index 73% rename from crates/signinum-jpeg/build.rs rename to crates/j2k-jpeg/build.rs index a6551087..a790b69b 100644 --- a/crates/signinum-jpeg/build.rs +++ b/crates/j2k-jpeg/build.rs @@ -1,10 +1,18 @@ // SPDX-License-Identifier: Apache-2.0 +use std::env; use std::process::Command; fn main() { println!("cargo:rustc-check-cfg=cfg(has_libjpeg_turbo)"); println!("cargo:rerun-if-changed=build.rs"); + + // Probing pkg-config and linking system libjpeg-turbo is exclusively for + // the comparison benches; library consumers must never pick up a system + // JPEG link just because pkg-config can find one. + if env::var_os("CARGO_FEATURE_BENCH_LIBJPEG_TURBO").is_none() { + return; + } println!("cargo:rerun-if-env-changed=PKG_CONFIG_PATH"); let Ok(output) = Command::new("pkg-config") diff --git a/crates/signinum-jpeg/examples/decode_region.rs b/crates/j2k-jpeg/examples/decode_region.rs similarity index 72% rename from crates/signinum-jpeg/examples/decode_region.rs rename to crates/j2k-jpeg/examples/decode_region.rs index 39141c76..93e80309 100644 --- a/crates/signinum-jpeg/examples/decode_region.rs +++ b/crates/j2k-jpeg/examples/decode_region.rs @@ -3,11 +3,12 @@ //! Decode a source-coordinate ROI from a JPEG tile. //! //! Run with: -//! `cargo run -p signinum-jpeg --example decode_region` +//! `cargo run -p j2k-jpeg --example decode_region` -use signinum_jpeg::{Decoder, PixelFormat, Rect}; +use j2k_jpeg::{Decoder, PixelFormat, Rect}; +use j2k_test_support::JPEG_BASELINE_420_16X16; -const TILE: &[u8] = include_bytes!("../fixtures/conformance/baseline_420_16x16.jpg"); +const TILE: &[u8] = JPEG_BASELINE_420_16X16; fn main() -> Result<(), Box> { let decoder = Decoder::new(TILE)?; diff --git a/crates/j2k-jpeg/examples/dump_bmp.rs b/crates/j2k-jpeg/examples/dump_bmp.rs new file mode 100644 index 00000000..973eaa0f --- /dev/null +++ b/crates/j2k-jpeg/examples/dump_bmp.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Decode a JPEG through `j2k-jpeg` and write a viewable 24-bit BMP. +//! +//! Run with: +//! `cargo run -p j2k-jpeg --example dump_bmp` +//! +//! Or pass an input and output path: +//! `cargo run -p j2k-jpeg --example dump_bmp -- input.jpg output.bmp` + +use std::{ + env, fs, + io::{BufWriter, Write}, + path::{Path, PathBuf}, +}; + +use j2k_jpeg::{Decoder, PixelFormat}; +use j2k_test_support::JPEG_BASELINE_420_16X16; + +const DEFAULT_OUTPUT: &str = "target/j2k-jpeg-baseline-420.bmp"; + +fn main() -> Result<(), Box> { + let args = env::args().skip(1).collect::>(); + let (input_label, bytes, output_path) = match args.as_slice() { + [] => ( + "baseline_420_16x16 fixture".to_string(), + JPEG_BASELINE_420_16X16.to_vec(), + PathBuf::from(DEFAULT_OUTPUT), + ), + [input, output] => (input.clone(), fs::read(input)?, PathBuf::from(output)), + _ => { + return Err( + "usage: cargo run -p j2k-jpeg --example dump_bmp -- [input.jpg output.bmp]".into(), + ) + } + }; + + let decoder = Decoder::new(&bytes)?; + let (rgb, outcome) = decoder.decode(PixelFormat::Rgb8)?; + let width = outcome.decoded.w as usize; + let height = outcome.decoded.h as usize; + + write_rgb8_bmp(&output_path, width, height, &rgb)?; + println!( + "decoded {input_label} as {}x{} Rgb8 and wrote {}", + width, + height, + output_path.display() + ); + Ok(()) +} + +fn write_rgb8_bmp( + path: &Path, + width: usize, + height: usize, + rgb: &[u8], +) -> Result<(), Box> { + let expected_len = width + .checked_mul(height) + .and_then(|pixels| pixels.checked_mul(3)) + .ok_or("decoded image dimensions overflow")?; + if rgb.len() != expected_len { + return Err(format!("expected {expected_len} RGB bytes, got {}", rgb.len()).into()); + } + + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent)?; + } + + let row_stride = width + .checked_mul(3) + .and_then(|row| row.checked_add(3)) + .map(|row| row & !3) + .ok_or("BMP row stride overflow")?; + let pixel_bytes = row_stride + .checked_mul(height) + .ok_or("BMP pixel buffer size overflow")?; + let file_size = 54usize + .checked_add(pixel_bytes) + .ok_or("BMP file size overflow")?; + + let mut out = BufWriter::new(fs::File::create(path)?); + out.write_all(b"BM")?; + write_u32(&mut out, u32::try_from(file_size)?)?; + write_u16(&mut out, 0)?; + write_u16(&mut out, 0)?; + write_u32(&mut out, 54)?; + + write_u32(&mut out, 40)?; + write_i32(&mut out, i32::try_from(width)?)?; + write_i32(&mut out, i32::try_from(height)?)?; + write_u16(&mut out, 1)?; + write_u16(&mut out, 24)?; + write_u32(&mut out, 0)?; + write_u32(&mut out, u32::try_from(pixel_bytes)?)?; + write_i32(&mut out, 0)?; + write_i32(&mut out, 0)?; + write_u32(&mut out, 0)?; + write_u32(&mut out, 0)?; + + let padding = vec![0_u8; row_stride - width * 3]; + for y in (0..height).rev() { + let row = &rgb[y * width * 3..(y + 1) * width * 3]; + for pixel in row.chunks_exact(3) { + out.write_all(&[pixel[2], pixel[1], pixel[0]])?; + } + out.write_all(&padding)?; + } + Ok(()) +} + +fn write_u16(out: &mut dyn Write, value: u16) -> Result<(), std::io::Error> { + out.write_all(&value.to_le_bytes()) +} + +fn write_u32(out: &mut dyn Write, value: u32) -> Result<(), std::io::Error> { + out.write_all(&value.to_le_bytes()) +} + +fn write_i32(out: &mut dyn Write, value: i32) -> Result<(), std::io::Error> { + out.write_all(&value.to_le_bytes()) +} diff --git a/crates/signinum-jpeg/examples/inspect.rs b/crates/j2k-jpeg/examples/inspect.rs similarity index 94% rename from crates/signinum-jpeg/examples/inspect.rs rename to crates/j2k-jpeg/examples/inspect.rs index 8ce9ef25..28fbc0b7 100644 --- a/crates/signinum-jpeg/examples/inspect.rs +++ b/crates/j2k-jpeg/examples/inspect.rs @@ -21,7 +21,7 @@ fn main() -> ExitCode { return ExitCode::from(1); } }; - match signinum_jpeg::Decoder::inspect(&bytes) { + match j2k_jpeg::Decoder::inspect(&bytes) { Ok(info) => { println!("{info:#?}"); ExitCode::SUCCESS diff --git a/crates/signinum-jpeg/fuzz/.gitignore b/crates/j2k-jpeg/fuzz/.gitignore similarity index 100% rename from crates/signinum-jpeg/fuzz/.gitignore rename to crates/j2k-jpeg/fuzz/.gitignore diff --git a/crates/signinum-j2k/fuzz/Cargo.lock b/crates/j2k-jpeg/fuzz/Cargo.lock similarity index 65% rename from crates/signinum-j2k/fuzz/Cargo.lock rename to crates/j2k-jpeg/fuzz/Cargo.lock index dc687e11..1f0fb84d 100644 --- a/crates/signinum-j2k/fuzz/Cargo.lock +++ b/crates/j2k-jpeg/fuzz/Cargo.lock @@ -8,17 +8,11 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -[[package]] -name = "bytemuck" -version = "1.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" - [[package]] name = "cc" -version = "1.2.60" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", "jobserver", @@ -33,14 +27,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] -name = "fearless_simd" -version = "0.3.0" +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb2907d1f08b2b316b9223ced5b0e89d87028ba8deae9764741dba8ff7f3903" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "bytemuck", + "crossbeam-utils", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -59,6 +75,36 @@ dependencies = [ "wasip2", ] +[[package]] +name = "j2k-core" +version = "0.6.0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "j2k-jpeg" +version = "0.6.0" +dependencies = [ + "j2k-core", + "j2k-profile", + "memchr", + "rayon", + "thiserror", +] + +[[package]] +name = "j2k-jpeg-fuzz" +version = "0.1.0" +dependencies = [ + "j2k-jpeg", + "libfuzzer-sys", +] + +[[package]] +name = "j2k-profile" +version = "0.6.0" + [[package]] name = "jobserver" version = "0.1.34" @@ -71,25 +117,25 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libfuzzer-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12a681b7dd8ce12bff52488013ba614b869148d54dd79836ab85aafdd53f08d" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" dependencies = [ "arbitrary", "cc", ] [[package]] -name = "libm" -version = "0.2.16" +name = "memchr" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "proc-macro2" @@ -116,48 +162,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] -name = "shlex" -version = "1.3.0" +name = "rayon" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signinum-core" -version = "1.0.0" -dependencies = [ - "thiserror", -] - -[[package]] -name = "signinum-j2k" -version = "1.0.0" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ - "signinum-core", - "signinum-j2k-native", - "thiserror", + "either", + "rayon-core", ] [[package]] -name = "signinum-j2k-fuzz" -version = "0.1.0" +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" dependencies = [ - "libfuzzer-sys", - "signinum-j2k", + "crossbeam-deque", + "crossbeam-utils", ] [[package]] -name = "signinum-j2k-native" -version = "0.2.0" -dependencies = [ - "fearless_simd", - "libm", -] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -192,9 +226,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen", ] diff --git a/crates/signinum-jpeg/fuzz/Cargo.toml b/crates/j2k-jpeg/fuzz/Cargo.toml similarity index 83% rename from crates/signinum-jpeg/fuzz/Cargo.toml rename to crates/j2k-jpeg/fuzz/Cargo.toml index 51423102..65288e7e 100644 --- a/crates/signinum-jpeg/fuzz/Cargo.toml +++ b/crates/j2k-jpeg/fuzz/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "signinum-jpeg-fuzz" +name = "j2k-jpeg-fuzz" version = "0.1.0" edition = "2021" publish = false @@ -8,8 +8,8 @@ publish = false cargo-fuzz = true [dependencies] -libfuzzer-sys = "0.4" -signinum-jpeg = { path = "..", features = ["scalar-only"] } +libfuzzer-sys = "0.4" +j2k-jpeg = { path = ".." } [[bin]] name = "parse_fuzz" diff --git a/crates/signinum-jpeg/fuzz/fuzz_targets/decode_fuzz.rs b/crates/j2k-jpeg/fuzz/fuzz_targets/decode_fuzz.rs similarity index 96% rename from crates/signinum-jpeg/fuzz/fuzz_targets/decode_fuzz.rs rename to crates/j2k-jpeg/fuzz/fuzz_targets/decode_fuzz.rs index fb067d60..8413c21e 100644 --- a/crates/signinum-jpeg/fuzz/fuzz_targets/decode_fuzz.rs +++ b/crates/j2k-jpeg/fuzz/fuzz_targets/decode_fuzz.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 #![no_main] +use j2k_jpeg::{Decoder, PixelFormat}; use libfuzzer_sys::fuzz_target; -use signinum_jpeg::{Decoder, PixelFormat}; /// The invariant: on any arbitrary byte input, `Decoder::new` + `decode_into` /// either succeeds with a filled buffer or returns a typed `JpegError` — and diff --git a/crates/signinum-jpeg/fuzz/fuzz_targets/parse_fuzz.rs b/crates/j2k-jpeg/fuzz/fuzz_targets/parse_fuzz.rs similarity index 84% rename from crates/signinum-jpeg/fuzz/fuzz_targets/parse_fuzz.rs rename to crates/j2k-jpeg/fuzz/fuzz_targets/parse_fuzz.rs index d2c335a1..587e77dc 100644 --- a/crates/signinum-jpeg/fuzz/fuzz_targets/parse_fuzz.rs +++ b/crates/j2k-jpeg/fuzz/fuzz_targets/parse_fuzz.rs @@ -5,5 +5,5 @@ libfuzzer_sys::fuzz_target!(|data: &[u8]| { // Invariant: never panic. Every input must produce either `Ok(Info)` // or `Err(JpegError)`. libfuzzer aborts on panic, which is the detection // mechanism. - let _ = signinum_jpeg::Decoder::inspect(data); + let _ = j2k_jpeg::Decoder::inspect(data); }); diff --git a/crates/signinum-jpeg/src/adapter/baseline_encode.rs b/crates/j2k-jpeg/src/adapter/baseline_encode.rs similarity index 84% rename from crates/signinum-jpeg/src/adapter/baseline_encode.rs rename to crates/j2k-jpeg/src/adapter/baseline_encode.rs index 3595b3eb..f5028a8e 100644 --- a/crates/signinum-jpeg/src/adapter/baseline_encode.rs +++ b/crates/j2k-jpeg/src/adapter/baseline_encode.rs @@ -7,31 +7,49 @@ use crate::encoder::{ }; #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Baseline JPEG component sampling parameters. pub struct JpegBaselineSampling { + /// Number of encoded components. pub components: u8, + /// Horizontal sampling factor per component. pub h: [u8; 3], + /// Vertical sampling factor per component. pub v: [u8; 3], + /// Maximum horizontal sampling factor across components. pub max_h: u8, + /// Maximum vertical sampling factor across components. pub max_v: u8, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Canonical Huffman lookup table for encoding. pub struct JpegBaselineHuffmanTable { + /// Huffman code value by symbol. pub codes: [u16; 256], + /// Huffman code length by symbol. pub lens: [u8; 256], } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Tables needed to assemble and entropy-code a baseline JPEG frame. pub struct JpegBaselineEncodeTables { + /// Component sampling metadata. pub sampling: JpegBaselineSampling, + /// Luma quantization table in natural order. pub q_luma: [u8; 64], + /// Chroma quantization table in natural order. pub q_chroma: [u8; 64], + /// Luma DC Huffman table. pub huff_dc_luma: JpegBaselineHuffmanTable, + /// Luma AC Huffman table. pub huff_ac_luma: JpegBaselineHuffmanTable, + /// Chroma DC Huffman table. pub huff_dc_chroma: JpegBaselineHuffmanTable, + /// Chroma AC Huffman table. pub huff_ac_chroma: JpegBaselineHuffmanTable, } +/// Build quantization, sampling, and Huffman tables for baseline encoding. pub fn baseline_encode_tables( options: JpegEncodeOptions, ) -> Result { @@ -47,6 +65,7 @@ pub fn baseline_encode_tables( }) } +/// Validate that dimensions can be represented in baseline JPEG markers. pub fn validate_jpeg_baseline_dimensions(width: u32, height: u32) -> Result<(), JpegEncodeError> { if width == 0 || height == 0 { return Err(JpegEncodeError::EmptyDimensions); @@ -57,6 +76,7 @@ pub fn validate_jpeg_baseline_dimensions(width: u32, height: u32) -> Result<(), Ok(()) } +/// Validate a user-provided restart interval. pub fn validate_jpeg_baseline_restart_interval( restart_interval: Option, ) -> Result<(), JpegEncodeError> { @@ -66,6 +86,7 @@ pub fn validate_jpeg_baseline_restart_interval( Ok(()) } +/// Return JPEG component sampling factors for a public subsampling mode. pub fn jpeg_baseline_sampling_for(subsampling: JpegSubsampling) -> JpegBaselineSampling { match subsampling { JpegSubsampling::Gray => JpegBaselineSampling { @@ -99,6 +120,7 @@ pub fn jpeg_baseline_sampling_for(subsampling: JpegSubsampling) -> JpegBaselineS } } +/// Conservative upper bound for entropy bytes produced by the CPU encoder. pub fn jpeg_baseline_entropy_capacity_bytes( width: u32, height: u32, @@ -130,6 +152,7 @@ pub fn jpeg_baseline_entropy_capacity_bytes( .map_err(|_| JpegEncodeError::Internal("JPEG entropy capacity exceeds usize".into())) } +/// Assemble a complete baseline JPEG codestream from entropy bytes and tables. pub fn assemble_jpeg_baseline_frame( entropy: &[u8], width: u32, @@ -164,6 +187,41 @@ pub fn assemble_jpeg_baseline_frame( Ok(EncodedJpeg { data: out, backend }) } +pub(crate) fn assemble_jpeg_baseline_frame_with_quant_tables( + entropy: &[u8], + width: u32, + height: u32, + sampling: JpegBaselineSampling, + q_luma: &[u8; 64], + q_chroma: Option<&[u8; 64]>, + backend: JpegBackend, +) -> Result { + validate_jpeg_baseline_dimensions(width, height)?; + + let mut out = Vec::with_capacity(768usize.saturating_add(entropy.len())); + write_marker(&mut out, 0xD8); + write_dqt(&mut out, 0, q_luma)?; + if sampling.components == 3 { + let q_chroma = q_chroma.ok_or_else(|| { + JpegEncodeError::Internal("three-component DCT JPEG requires chroma quant table".into()) + })?; + write_dqt(&mut out, 1, q_chroma)?; + } + write_sof0(&mut out, width, height, sampling)?; + write_dht(&mut out, 0, 0, &STD_LUMA_DC_BITS, &STD_LUMA_DC_VALUES)?; + write_dht(&mut out, 1, 0, &STD_LUMA_AC_BITS, &STD_LUMA_AC_VALUES)?; + if sampling.components == 3 { + write_dht(&mut out, 0, 1, &STD_CHROMA_DC_BITS, &STD_CHROMA_DC_VALUES)?; + write_dht(&mut out, 1, 1, &STD_CHROMA_AC_BITS, &STD_CHROMA_AC_VALUES)?; + } + write_sos(&mut out, sampling.components)?; + out.extend_from_slice(entropy); + write_marker(&mut out, 0xD9); + + Ok(EncodedJpeg { data: out, backend }) +} + +/// JPEG zigzag coefficient order used by baseline entropy coding. pub const JPEG_BASELINE_ZIGZAG: [u8; 64] = [ 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59, diff --git a/crates/signinum-jpeg/src/adapter/device_plan.rs b/crates/j2k-jpeg/src/adapter/device_plan.rs similarity index 79% rename from crates/signinum-jpeg/src/adapter/device_plan.rs rename to crates/j2k-jpeg/src/adapter/device_plan.rs index 1fc3480c..f7586bb0 100644 --- a/crates/signinum-jpeg/src/adapter/device_plan.rs +++ b/crates/j2k-jpeg/src/adapter/device_plan.rs @@ -9,35 +9,57 @@ use alloc::borrow::Cow; use alloc::vec::Vec; #[derive(Debug, Clone, PartialEq, Eq)] +/// Component metadata needed by device-side JPEG decoders. pub struct DeviceComponentPlan { + /// Horizontal sampling factor. pub h: u8, + /// Vertical sampling factor. pub v: u8, + /// Output component index in decoded pixel/component order. pub output_index: usize, } #[derive(Debug, Clone, PartialEq, Eq)] +/// Validated decode plan and entropy payload for device-side decoders. pub struct DeviceDecodePlan<'a> { + /// Image dimensions as `(width, height)` in pixels. pub dimensions: (u32, u32), + /// Header-derived JPEG color space. pub color_space: ColorSpace, + /// Restart interval in MCUs, if restart markers are present. pub restart_interval: Option, + /// Non-fatal warnings collected while building the plan. pub warnings: Vec, + /// Entropy scan bytes with device-compatible marker handling. pub scan_bytes: Cow<'a, [u8]>, + /// Component sampling and output placement. pub components: Vec, + /// Entropy checkpoints for restart or synthetic resume points. pub checkpoints: Vec, + /// True when the tile matches the Metal/CUDA fast 4:2:0 path. pub matches_fast_420: bool, + /// True when the tile matches the Metal/CUDA fast 4:2:2 path. pub matches_fast_422: bool, + /// True when the tile matches the Metal/CUDA fast 4:4:4 path. pub matches_fast_444: bool, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Lightweight summary used for batching without materializing scan bytes. pub struct DeviceBatchSummary { + /// Restart interval in MCUs, if restart markers are present. pub restart_interval: Option, + /// Number of checkpoints the full device plan would contain. pub checkpoint_count: usize, + /// True when the tile matches the fast 4:2:0 device path. pub matches_fast_420: bool, + /// True when the tile matches the fast 4:2:2 device path. pub matches_fast_422: bool, + /// True when the tile matches the fast 4:4:4 device path. pub matches_fast_444: bool, } +/// Build a full device decode plan for an inspected decoder. pub fn build_device_plan<'a>( decoder: &'a Decoder<'a>, cadence_mcus: u32, @@ -82,6 +104,7 @@ pub fn build_device_plan<'a>( }) } +/// Summarize device planning properties without copying entropy data. pub fn summarize_device_batch(decoder: &Decoder<'_>, cadence_mcus: u32) -> DeviceBatchSummary { if !matches!( decoder.info().sof_kind, diff --git a/crates/j2k-jpeg/src/adapter/fast_packet.rs b/crates/j2k-jpeg/src/adapter/fast_packet.rs new file mode 100644 index 00000000..7d9cf6df --- /dev/null +++ b/crates/j2k-jpeg/src/adapter/fast_packet.rs @@ -0,0 +1,684 @@ +// SPDX-License-Identifier: Apache-2.0 + +use crate::decoder::Decoder; +use crate::error::{JpegError, MarkerKind}; +use crate::info::{ColorSpace, SamplingFactors, SofKind}; +use crate::internal::checkpoint::{build_checkpoint_plan, DeviceCheckpoint}; +use crate::parse::header::parse_header; +use crate::parse::scan::ScanComponent; +use crate::parse::tables::RawHuffmanTable; +use alloc::vec::Vec; + +const MAX_NONRESTART_ENTROPY_CHECKPOINTS: u32 = 2048; + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Error while building a backend fast-path JPEG packet. +pub enum FastPacketError { + /// Header or entropy decode failed. + Decode(JpegError), + /// JPEG SOF kind is not supported by the fast path. + UnsupportedSof(SofKind), + /// JPEG color space is not supported by the selected fast path. + UnsupportedColorSpace(ColorSpace), + /// JPEG component sampling does not match the selected fast path. + UnsupportedSampling, + /// Scan component order does not match SOF component order. + UnsupportedComponentOrder, + /// Stream does not contain a scan payload. + MissingScan, + /// Referenced quantization table is absent. + MissingQuantTable { + /// Quantization table slot. + slot: u8, + }, + /// Referenced Huffman table is absent. + MissingHuffmanTable { + /// Huffman table class. + kind: TableKind, + /// Huffman table slot. + slot: u8, + }, + /// Entropy payload contains a marker unsupported by the fast path. + EntropyMarkerUnsupported { + /// Raw marker byte following `0xff`. + marker: u8, + }, + /// Entropy payload ended before the packet could be built. + TruncatedEntropy, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Huffman table class used by fast-path packet builders. +pub enum TableKind { + /// DC Huffman table. + Dc, + /// AC Huffman table. + Ac, +} + +impl From for FastPacketError { + fn from(value: JpegError) -> Self { + Self::Decode(value) + } +} + +#[repr(C)] +#[derive(Debug, Clone, PartialEq, Eq)] +/// Huffman table payload copied into backend-compatible packet structs. +pub struct JpegHuffmanTable { + /// JPEG BITS counts for code lengths 1 through 16. + pub bits: [u8; 16], + /// Number of populated entries in `values`. + pub values_len: u16, + /// JPEG HUFFVAL symbols padded to fixed capacity. + pub values: [u8; 256], +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Entropy decoder resume point for fast-path packet decoding. +pub struct JpegEntropyCheckpointV1 { + /// MCU index for this checkpoint. + pub mcu_index: u32, + /// Byte offset into the entropy payload. + pub entropy_pos: u32, + /// Buffered entropy bits. + pub bit_acc: u64, + /// Number of valid bits in `bit_acc`. + pub bit_count: u32, + /// Previous Y DC predictor. + pub y_prev_dc: i32, + /// Previous Cb DC predictor. + pub cb_prev_dc: i32, + /// Previous Cr DC predictor. + pub cr_prev_dc: i32, + /// Reserved for ABI-compatible future expansion. + pub reserved: u32, +} + +impl JpegHuffmanTable { + fn from_raw(raw: &RawHuffmanTable) -> Self { + let mut values = [0u8; 256]; + let slice = raw.values.as_slice(); + values[..slice.len()].copy_from_slice(slice); + Self { + bits: raw.bits, + values_len: slice.len() as u16, + values, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, PartialEq, Eq)] +/// Backend fast-path packet for 8-bit 4:2:0 JPEG tiles. +pub struct JpegFast420PacketV1 { + /// Image dimensions as `(width, height)` in pixels. + pub dimensions: (u32, u32), + /// Number of MCUs per row. + pub mcus_per_row: u32, + /// Number of MCU rows. + pub mcu_rows: u32, + /// Restart interval in MCUs, or zero when absent. + pub restart_interval_mcus: u32, + /// Byte offsets of restart-addressable entropy segments. + pub restart_offsets: Vec, + /// Entropy resume checkpoints. + pub entropy_checkpoints: Vec, + /// Y quantization table in natural order. + pub y_quant: [u16; 64], + /// Cb quantization table in natural order. + pub cb_quant: [u16; 64], + /// Cr quantization table in natural order. + pub cr_quant: [u16; 64], + /// Y DC Huffman table. + pub y_dc_table: JpegHuffmanTable, + /// Y AC Huffman table. + pub y_ac_table: JpegHuffmanTable, + /// Cb DC Huffman table. + pub cb_dc_table: JpegHuffmanTable, + /// Cb AC Huffman table. + pub cb_ac_table: JpegHuffmanTable, + /// Cr DC Huffman table. + pub cr_dc_table: JpegHuffmanTable, + /// Cr AC Huffman table. + pub cr_ac_table: JpegHuffmanTable, + /// Entropy-coded scan bytes. + pub entropy_bytes: Vec, +} + +#[repr(C)] +#[derive(Debug, Clone, PartialEq, Eq)] +/// Backend fast-path packet for 8-bit 4:2:2 JPEG tiles. +pub struct JpegFast422PacketV1 { + /// Image dimensions as `(width, height)` in pixels. + pub dimensions: (u32, u32), + /// Number of MCUs per row. + pub mcus_per_row: u32, + /// Number of MCU rows. + pub mcu_rows: u32, + /// Restart interval in MCUs, or zero when absent. + pub restart_interval_mcus: u32, + /// Byte offsets of restart-addressable entropy segments. + pub restart_offsets: Vec, + /// Entropy resume checkpoints. + pub entropy_checkpoints: Vec, + /// Y quantization table in natural order. + pub y_quant: [u16; 64], + /// Cb quantization table in natural order. + pub cb_quant: [u16; 64], + /// Cr quantization table in natural order. + pub cr_quant: [u16; 64], + /// Y DC Huffman table. + pub y_dc_table: JpegHuffmanTable, + /// Y AC Huffman table. + pub y_ac_table: JpegHuffmanTable, + /// Cb DC Huffman table. + pub cb_dc_table: JpegHuffmanTable, + /// Cb AC Huffman table. + pub cb_ac_table: JpegHuffmanTable, + /// Cr DC Huffman table. + pub cr_dc_table: JpegHuffmanTable, + /// Cr AC Huffman table. + pub cr_ac_table: JpegHuffmanTable, + /// Entropy-coded scan bytes. + pub entropy_bytes: Vec, +} + +#[repr(C)] +#[derive(Debug, Clone, PartialEq, Eq)] +/// Backend fast-path packet for 8-bit 4:4:4 JPEG tiles. +pub struct JpegFast444PacketV1 { + /// Image dimensions as `(width, height)` in pixels. + pub dimensions: (u32, u32), + /// Number of MCUs per row. + pub mcus_per_row: u32, + /// Number of MCU rows. + pub mcu_rows: u32, + /// Restart interval in MCUs, or zero when absent. + pub restart_interval_mcus: u32, + /// Byte offsets of restart-addressable entropy segments. + pub restart_offsets: Vec, + /// Entropy resume checkpoints. + pub entropy_checkpoints: Vec, + /// Y quantization table in natural order. + pub y_quant: [u16; 64], + /// Cb quantization table in natural order. + pub cb_quant: [u16; 64], + /// Cr quantization table in natural order. + pub cr_quant: [u16; 64], + /// Y DC Huffman table. + pub y_dc_table: JpegHuffmanTable, + /// Y AC Huffman table. + pub y_ac_table: JpegHuffmanTable, + /// Cb DC Huffman table. + pub cb_dc_table: JpegHuffmanTable, + /// Cb AC Huffman table. + pub cb_ac_table: JpegHuffmanTable, + /// Cr DC Huffman table. + pub cr_dc_table: JpegHuffmanTable, + /// Cr AC Huffman table. + pub cr_ac_table: JpegHuffmanTable, + /// Entropy-coded scan bytes. + pub entropy_bytes: Vec, +} + +#[repr(C)] +#[derive(Debug, Clone, PartialEq, Eq)] +/// Backend fast-path packet for 8-bit grayscale JPEG tiles. +pub struct JpegGrayPacketV1 { + /// Image dimensions as `(width, height)` in pixels. + pub dimensions: (u32, u32), + /// Number of MCUs per row. + pub mcus_per_row: u32, + /// Number of MCU rows. + pub mcu_rows: u32, + /// Restart interval in MCUs, or zero when absent. + pub restart_interval_mcus: u32, + /// Byte offsets of restart-addressable entropy segments. + pub restart_offsets: Vec, + /// Y quantization table in natural order. + pub y_quant: [u16; 64], + /// Y DC Huffman table. + pub y_dc_table: JpegHuffmanTable, + /// Y AC Huffman table. + pub y_ac_table: JpegHuffmanTable, + /// Entropy-coded scan bytes. + pub entropy_bytes: Vec, +} + +#[derive(Debug, Clone, Copy)] +struct FastLayout { + sampling: &'static [(u8, u8)], + allow_rgb: bool, + mcu_width: u32, + mcu_height: u32, +} + +const FAST420_LAYOUT: FastLayout = FastLayout { + sampling: &[(2, 2), (1, 1), (1, 1)], + allow_rgb: false, + mcu_width: 16, + mcu_height: 16, +}; + +const FAST422_LAYOUT: FastLayout = FastLayout { + sampling: &[(2, 1), (1, 1), (1, 1)], + allow_rgb: false, + mcu_width: 16, + mcu_height: 8, +}; + +const FAST444_LAYOUT: FastLayout = FastLayout { + sampling: &[(1, 1), (1, 1), (1, 1)], + allow_rgb: true, + mcu_width: 8, + mcu_height: 8, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ColorFastPacketParts { + dimensions: (u32, u32), + mcus_per_row: u32, + mcu_rows: u32, + restart_interval_mcus: u32, + restart_offsets: Vec, + entropy_checkpoints: Vec, + y_quant: [u16; 64], + cb_quant: [u16; 64], + cr_quant: [u16; 64], + y_dc_table: JpegHuffmanTable, + y_ac_table: JpegHuffmanTable, + cb_dc_table: JpegHuffmanTable, + cb_ac_table: JpegHuffmanTable, + cr_dc_table: JpegHuffmanTable, + cr_ac_table: JpegHuffmanTable, + entropy_bytes: Vec, +} + +macro_rules! impl_from_color_fast_packet_parts { + ($packet:ty) => { + impl From for $packet { + fn from(parts: ColorFastPacketParts) -> Self { + Self { + dimensions: parts.dimensions, + mcus_per_row: parts.mcus_per_row, + mcu_rows: parts.mcu_rows, + restart_interval_mcus: parts.restart_interval_mcus, + restart_offsets: parts.restart_offsets, + entropy_checkpoints: parts.entropy_checkpoints, + y_quant: parts.y_quant, + cb_quant: parts.cb_quant, + cr_quant: parts.cr_quant, + y_dc_table: parts.y_dc_table, + y_ac_table: parts.y_ac_table, + cb_dc_table: parts.cb_dc_table, + cb_ac_table: parts.cb_ac_table, + cr_dc_table: parts.cr_dc_table, + cr_ac_table: parts.cr_ac_table, + entropy_bytes: parts.entropy_bytes, + } + } + } + }; +} + +impl_from_color_fast_packet_parts!(JpegFast420PacketV1); +impl_from_color_fast_packet_parts!(JpegFast422PacketV1); +impl_from_color_fast_packet_parts!(JpegFast444PacketV1); + +#[derive(Debug, Clone, PartialEq, Eq)] +struct EntropySegments { + entropy_bytes: Vec, + restart_offsets: Vec, +} + +/// Build a 4:2:0 fast-path packet from JPEG bytes. +pub fn build_fast420_packet(bytes: &[u8]) -> Result { + build_color_fast_packet(bytes, FAST420_LAYOUT).map(Into::into) +} + +/// Build a 4:4:4 fast-path packet from JPEG bytes. +pub fn build_fast444_packet(bytes: &[u8]) -> Result { + build_color_fast_packet(bytes, FAST444_LAYOUT).map(Into::into) +} + +/// Build a 4:2:2 fast-path packet from JPEG bytes. +pub fn build_fast422_packet(bytes: &[u8]) -> Result { + build_color_fast_packet(bytes, FAST422_LAYOUT).map(Into::into) +} + +fn build_color_fast_packet( + bytes: &[u8], + layout: FastLayout, +) -> Result { + let decoder = Decoder::new(bytes)?; + let header = parse_header(bytes)?; + if !matches!(header.sof_kind, SofKind::Baseline8 | SofKind::Extended8) { + return Err(FastPacketError::UnsupportedSof(header.sof_kind)); + } + if header.bit_depth != 8 { + return Err(FastPacketError::Decode(JpegError::UnsupportedBitDepth { + depth: header.bit_depth, + })); + } + let color_space = header.color_space(); + if color_space != ColorSpace::YCbCr && !(layout.allow_rgb && color_space == ColorSpace::Rgb) { + return Err(FastPacketError::UnsupportedColorSpace(header.color_space())); + } + if header.sampling != SamplingFactors::from_validated_components(layout.sampling) { + return Err(FastPacketError::UnsupportedSampling); + } + let scan = header.scan.as_ref().ok_or(FastPacketError::MissingScan)?; + let [y_scan, cb_scan, cr_scan] = ordered_scan_triplet(&header.component_ids, &scan.components)?; + + let y_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 0)?; + let cb_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 1)?; + let cr_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 2)?; + let y_dc_table = huffman_table(&header.huffman_tables.dc, TableKind::Dc, y_scan.dc_table)?; + let y_ac_table = huffman_table(&header.huffman_tables.ac, TableKind::Ac, y_scan.ac_table)?; + let cb_dc_table = huffman_table(&header.huffman_tables.dc, TableKind::Dc, cb_scan.dc_table)?; + let cb_ac_table = huffman_table(&header.huffman_tables.ac, TableKind::Ac, cb_scan.ac_table)?; + let cr_dc_table = huffman_table(&header.huffman_tables.dc, TableKind::Dc, cr_scan.dc_table)?; + let cr_ac_table = huffman_table(&header.huffman_tables.ac, TableKind::Ac, cr_scan.ac_table)?; + + let entropy_offset = header.sos_offset.ok_or(FastPacketError::MissingScan)?; + let restart_interval_mcus = u32::from(header.restart_interval.unwrap_or(0)); + let EntropySegments { + entropy_bytes, + restart_offsets, + } = extract_entropy_segments(&bytes[entropy_offset..], header.restart_interval)?; + let (width, height) = header.dimensions; + let mcus_per_row = width.div_ceil(layout.mcu_width); + let mcu_rows = height.div_ceil(layout.mcu_height); + let total_mcus = mcus_per_row + .checked_mul(mcu_rows) + .ok_or(FastPacketError::Decode(JpegError::DimensionOverflow { + width, + height, + }))?; + let entropy_checkpoints = + build_fast_entropy_checkpoints(&decoder, &bytes[entropy_offset..], total_mcus)?; + + Ok(ColorFastPacketParts { + dimensions: header.dimensions, + mcus_per_row, + mcu_rows, + restart_interval_mcus, + restart_offsets, + entropy_checkpoints, + y_quant, + cb_quant, + cr_quant, + y_dc_table, + y_ac_table, + cb_dc_table, + cb_ac_table, + cr_dc_table, + cr_ac_table, + entropy_bytes, + }) +} + +/// Build a grayscale fast-path packet from JPEG bytes. +pub fn build_gray_packet(bytes: &[u8]) -> Result { + let header = parse_header(bytes)?; + if !matches!(header.sof_kind, SofKind::Baseline8 | SofKind::Extended8) { + return Err(FastPacketError::UnsupportedSof(header.sof_kind)); + } + if header.bit_depth != 8 { + return Err(FastPacketError::Decode(JpegError::UnsupportedBitDepth { + depth: header.bit_depth, + })); + } + if header.color_space() != ColorSpace::Grayscale { + return Err(FastPacketError::UnsupportedColorSpace(header.color_space())); + } + if header.sampling != SamplingFactors::from_validated_components(&[(1, 1)]) { + return Err(FastPacketError::UnsupportedSampling); + } + + let scan = header.scan.as_ref().ok_or(FastPacketError::MissingScan)?; + if header.component_ids.len() != 1 || scan.components.len() != 1 { + return Err(FastPacketError::UnsupportedComponentOrder); + } + if scan.components[0].id != header.component_ids[0] { + return Err(FastPacketError::UnsupportedComponentOrder); + } + + let y_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 0)?; + let y_dc_table = huffman_table( + &header.huffman_tables.dc, + TableKind::Dc, + scan.components[0].dc_table, + )?; + let y_ac_table = huffman_table( + &header.huffman_tables.ac, + TableKind::Ac, + scan.components[0].ac_table, + )?; + + let entropy_offset = header.sos_offset.ok_or(FastPacketError::MissingScan)?; + let restart_interval_mcus = u32::from(header.restart_interval.unwrap_or(0)); + let EntropySegments { + entropy_bytes, + restart_offsets, + } = extract_entropy_segments(&bytes[entropy_offset..], header.restart_interval)?; + let (width, height) = header.dimensions; + + Ok(JpegGrayPacketV1 { + dimensions: header.dimensions, + mcus_per_row: width.div_ceil(8), + mcu_rows: height.div_ceil(8), + restart_interval_mcus, + restart_offsets, + y_quant, + y_dc_table, + y_ac_table, + entropy_bytes, + }) +} + +/// Build a 4:2:0 fast-path packet from an inspected decoder. +pub fn build_fast420_packet_for_decoder( + decoder: &crate::decoder::Decoder<'_>, +) -> Result { + build_fast420_packet(decoder.bytes) +} + +/// Build a 4:4:4 fast-path packet from an inspected decoder. +pub fn build_fast444_packet_for_decoder( + decoder: &crate::decoder::Decoder<'_>, +) -> Result { + build_fast444_packet(decoder.bytes) +} + +/// Build a 4:2:2 fast-path packet from an inspected decoder. +pub fn build_fast422_packet_for_decoder( + decoder: &crate::decoder::Decoder<'_>, +) -> Result { + build_fast422_packet(decoder.bytes) +} + +/// Build a grayscale fast-path packet from an inspected decoder. +pub fn build_gray_packet_for_decoder( + decoder: &crate::decoder::Decoder<'_>, +) -> Result { + build_gray_packet(decoder.bytes) +} + +fn quant_for_component( + quant_table_ids: &[u8], + tables: &[Option<[u16; 64]>; 4], + component_idx: usize, +) -> Result<[u16; 64], FastPacketError> { + let slot = *quant_table_ids + .get(component_idx) + .ok_or(FastPacketError::UnsupportedComponentOrder)?; + tables[slot as usize].ok_or(FastPacketError::MissingQuantTable { slot }) +} + +fn ordered_scan_triplet( + component_ids: &[u8], + scan_components: &[ScanComponent], +) -> Result<[ScanComponent; 3], FastPacketError> { + if component_ids.len() != 3 || scan_components.len() != 3 { + return Err(FastPacketError::UnsupportedComponentOrder); + } + + let mut ordered = [None; 3]; + for (index, &component_id) in component_ids.iter().enumerate() { + let Some(component) = scan_components + .iter() + .copied() + .find(|component| component.id == component_id) + else { + return Err(FastPacketError::UnsupportedComponentOrder); + }; + ordered[index] = Some(component); + } + + match ordered { + [Some(first), Some(second), Some(third)] => Ok([first, second, third]), + _ => Err(FastPacketError::UnsupportedComponentOrder), + } +} + +fn huffman_table( + tables: &[Option; 4], + kind: TableKind, + slot: u8, +) -> Result { + let raw = tables[slot as usize] + .as_ref() + .ok_or(FastPacketError::MissingHuffmanTable { kind, slot })?; + Ok(JpegHuffmanTable::from_raw(raw)) +} + +fn nonrestart_entropy_chunk_mcus(total_mcus: u32) -> u32 { + total_mcus + .div_ceil(MAX_NONRESTART_ENTROPY_CHECKPOINTS) + .max(1) +} + +fn build_fast_entropy_checkpoints( + decoder: &Decoder<'_>, + scan_bytes: &[u8], + total_mcus: u32, +) -> Result, FastPacketError> { + let device_checkpoints = build_checkpoint_plan( + &decoder.plan, + scan_bytes, + nonrestart_entropy_chunk_mcus(total_mcus), + )?; + device_checkpoints + .iter() + .map(|checkpoint| packet_checkpoint_from_device(checkpoint, scan_bytes)) + .collect() +} + +fn packet_checkpoint_from_device( + checkpoint: &DeviceCheckpoint, + scan_bytes: &[u8], +) -> Result { + Ok(JpegEntropyCheckpointV1 { + mcu_index: checkpoint.mcu_index, + entropy_pos: destuffed_entropy_offset(scan_bytes, checkpoint.scan_offset)?, + bit_acc: checkpoint.bit_accumulator, + bit_count: u32::from(checkpoint.bits_buffered), + y_prev_dc: checkpoint.prev_dc[0], + cb_prev_dc: checkpoint.prev_dc[1], + cr_prev_dc: checkpoint.prev_dc[2], + reserved: 0, + }) +} + +fn destuffed_entropy_offset(scan_bytes: &[u8], target: usize) -> Result { + if target > scan_bytes.len() { + return Err(FastPacketError::TruncatedEntropy); + } + + let mut pos = 0usize; + let mut destuffed = 0usize; + while pos < target { + if scan_bytes[pos] != 0xff { + pos += 1; + destuffed += 1; + continue; + } + + let marker = *scan_bytes + .get(pos + 1) + .ok_or(FastPacketError::TruncatedEntropy)?; + if pos + 2 > target { + return Err(FastPacketError::TruncatedEntropy); + } + match marker { + 0x00 => { + pos += 2; + destuffed += 1; + } + 0xd0..=0xd7 | 0xd9 => { + pos += 2; + } + marker => return Err(FastPacketError::EntropyMarkerUnsupported { marker }), + } + } + + if pos != target { + return Err(FastPacketError::TruncatedEntropy); + } + u32::try_from(destuffed).map_err(|_| FastPacketError::TruncatedEntropy) +} + +fn extract_entropy_segments( + bytes: &[u8], + restart_interval: Option, +) -> Result { + let mut out = Vec::with_capacity(bytes.len()); + let mut restart_offsets = vec![0u32]; + let mut pos = 0usize; + let mut expected_rst = 0xD0u8; + while pos < bytes.len() { + let byte = bytes[pos]; + if byte != 0xFF { + out.push(byte); + pos += 1; + continue; + } + let next = *bytes + .get(pos + 1) + .ok_or(FastPacketError::TruncatedEntropy)?; + match next { + 0x00 => { + out.push(0xFF); + pos += 2; + } + 0xD9 => { + return Ok(EntropySegments { + entropy_bytes: out, + restart_offsets, + }); + } + 0xD0..=0xD7 if restart_interval.unwrap_or(0) != 0 => { + if next != expected_rst { + return Err(FastPacketError::EntropyMarkerUnsupported { marker: next }); + } + restart_offsets + .push(u32::try_from(out.len()).map_err(|_| FastPacketError::TruncatedEntropy)?); + expected_rst = if expected_rst == 0xD7 { + 0xD0 + } else { + expected_rst + 1 + }; + pos += 2; + } + marker => { + return Err(FastPacketError::EntropyMarkerUnsupported { marker }); + } + } + } + Err(FastPacketError::Decode(JpegError::MissingMarker { + marker: MarkerKind::Eoi, + })) +} diff --git a/crates/signinum-jpeg/src/adapter/mod.rs b/crates/j2k-jpeg/src/adapter/mod.rs similarity index 59% rename from crates/signinum-jpeg/src/adapter/mod.rs rename to crates/j2k-jpeg/src/adapter/mod.rs index 28043754..c5ae305b 100644 --- a/crates/signinum-jpeg/src/adapter/mod.rs +++ b/crates/j2k-jpeg/src/adapter/mod.rs @@ -7,11 +7,13 @@ mod baseline_encode; mod device_plan; -pub mod metal_fast420; +/// Backend-neutral fast-path packet builders and packet types. +pub mod fast_packet; use crate::Decoder; pub use crate::internal::checkpoint::DeviceCheckpoint; +pub(crate) use baseline_encode::assemble_jpeg_baseline_frame_with_quant_tables; pub use baseline_encode::{ assemble_jpeg_baseline_frame, baseline_encode_tables, jpeg_baseline_entropy_capacity_bytes, jpeg_baseline_sampling_for, validate_jpeg_baseline_dimensions, @@ -22,15 +24,15 @@ pub use device_plan::{ build_device_plan, summarize_device_batch, DeviceBatchSummary, DeviceComponentPlan, DeviceDecodePlan, }; -pub use metal_fast420::{ - build_metal_fast420_packet, build_metal_fast420_packet_for_decoder, build_metal_fast422_packet, - build_metal_fast422_packet_for_decoder, build_metal_fast444_packet, - build_metal_fast444_packet_for_decoder, build_metal_gray_packet, - build_metal_gray_packet_for_decoder, JpegMetalEntropyCheckpointV1, JpegMetalFast420PacketV1, - JpegMetalFast422PacketV1, JpegMetalFast444PacketV1, JpegMetalGrayPacketV1, - MetalFast420PacketError, MetalHuffmanTable, +pub use fast_packet::{ + build_fast420_packet, build_fast420_packet_for_decoder, build_fast422_packet, + build_fast422_packet_for_decoder, build_fast444_packet, build_fast444_packet_for_decoder, + build_gray_packet, build_gray_packet_for_decoder, FastPacketError, JpegEntropyCheckpointV1, + JpegFast420PacketV1, JpegFast422PacketV1, JpegFast444PacketV1, JpegGrayPacketV1, + JpegHuffmanTable, }; +/// Return the original JPEG byte stream owned by a decoder. pub fn decoder_bytes<'a>(decoder: &'a Decoder<'a>) -> &'a [u8] { decoder.bytes } diff --git a/crates/signinum-jpeg/src/backend/mod.rs b/crates/j2k-jpeg/src/backend/mod.rs similarity index 75% rename from crates/signinum-jpeg/src/backend/mod.rs rename to crates/j2k-jpeg/src/backend/mod.rs index c86b83c1..598910cc 100644 --- a/crates/signinum-jpeg/src/backend/mod.rs +++ b/crates/j2k-jpeg/src/backend/mod.rs @@ -4,9 +4,9 @@ //! inverse DCT. use crate::idct; -use signinum_core::CpuFeatures; +use j2k_core::CpuFeatures; -mod scalar; +pub(crate) mod scalar; #[cfg(target_arch = "x86_64")] mod x86; @@ -100,6 +100,56 @@ impl Backend { } } + pub(crate) fn fill_rgba_row_from_gray(self, gray_row: &[u8], dst: &mut [u8], alpha: u8) { + match self.kind { + BackendKind::Scalar => scalar::fill_rgba_row_from_gray(gray_row, dst, alpha), + #[cfg(target_arch = "x86_64")] + BackendKind::Avx2 => scalar::fill_rgba_row_from_gray(gray_row, dst, alpha), + #[cfg(target_arch = "aarch64")] + BackendKind::Neon => scalar::fill_rgba_row_from_gray(gray_row, dst, alpha), + } + } + + pub(crate) fn fill_rgba_row_from_rgb( + self, + r_row: &[u8], + g_row: &[u8], + b_row: &[u8], + dst: &mut [u8], + alpha: u8, + ) { + match self.kind { + BackendKind::Scalar => scalar::fill_rgba_row_from_rgb(r_row, g_row, b_row, dst, alpha), + #[cfg(target_arch = "x86_64")] + BackendKind::Avx2 => scalar::fill_rgba_row_from_rgb(r_row, g_row, b_row, dst, alpha), + #[cfg(target_arch = "aarch64")] + BackendKind::Neon => scalar::fill_rgba_row_from_rgb(r_row, g_row, b_row, dst, alpha), + } + } + + pub(crate) fn fill_rgba_row_from_ycbcr( + self, + y_row: &[u8], + cb_row: &[u8], + cr_row: &[u8], + dst: &mut [u8], + alpha: u8, + ) { + match self.kind { + BackendKind::Scalar => { + scalar::fill_rgba_row_from_ycbcr(y_row, cb_row, cr_row, dst, alpha); + } + #[cfg(target_arch = "x86_64")] + BackendKind::Avx2 => { + scalar::fill_rgba_row_from_ycbcr(y_row, cb_row, cr_row, dst, alpha); + } + #[cfg(target_arch = "aarch64")] + BackendKind::Neon => { + scalar::fill_rgba_row_from_ycbcr(y_row, cb_row, cr_row, dst, alpha); + } + } + } + #[allow(clippy::too_many_arguments)] pub(crate) fn fill_rgb_row_pair_from_420( self, @@ -186,8 +236,10 @@ impl Backend { match self.kind { BackendKind::Scalar => idct::scalar::idct_islow(input, output), #[cfg(target_arch = "x86_64")] + // SAFETY: Backend selection guarantees the SIMD target feature for this call. BackendKind::Avx2 => unsafe { idct::avx2::idct_islow(input, output) }, #[cfg(target_arch = "aarch64")] + // SAFETY: Backend selection guarantees the SIMD target feature for this call. BackendKind::Neon => unsafe { idct::neon::idct_islow(input, output) }, } } @@ -196,8 +248,10 @@ impl Backend { match self.kind { BackendKind::Scalar => idct::scalar::idct_islow_bottom_half_zero(input, output), #[cfg(target_arch = "x86_64")] + // SAFETY: Backend selection guarantees the SIMD target feature for this call. BackendKind::Avx2 => unsafe { idct::avx2::idct_islow(input, output) }, #[cfg(target_arch = "aarch64")] + // SAFETY: Backend selection guarantees the SIMD target feature for this call. BackendKind::Neon => unsafe { idct::neon::idct_islow_bottom_half_zero(input, output) }, } } diff --git a/crates/signinum-jpeg/src/backend/neon.rs b/crates/j2k-jpeg/src/backend/neon.rs similarity index 77% rename from crates/signinum-jpeg/src/backend/neon.rs rename to crates/j2k-jpeg/src/backend/neon.rs index b262432d..49870ea2 100644 --- a/crates/signinum-jpeg/src/backend/neon.rs +++ b/crates/j2k-jpeg/src/backend/neon.rs @@ -8,10 +8,19 @@ use core::arch::aarch64::{ }; use super::scalar; -use crate::color::ycbcr::ycbcr_to_rgb; +use crate::color::upsample::h2v2_fancy_sample_for_width; +use crate::color::ycbcr::{ + ycbcr_to_rgb, FIX_0_34414, FIX_0_71414, FIX_1_40200, FIX_1_77200, ROUND, +}; pub(crate) fn fill_rgb_row_from_gray(gray_row: &[u8], dst: &mut [u8]) { + let width = gray_row.len().min(dst.len() / 3); + let gray_row = &gray_row[..width]; + let dst = &mut dst[..width * 3]; debug_assert_eq!(dst.len(), gray_row.len() * 3); + // SAFETY: NEON is mandatory on supported aarch64 targets for this backend, + // and the wrapper narrows source and destination slices to one pixel count. + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_from_gray_neon(gray_row, dst); } @@ -22,7 +31,9 @@ unsafe fn fill_rgb_row_from_gray_neon(gray_row: &[u8], dst: &mut [u8]) { let width = gray_row.len(); let mut offset = 0; while offset + LANES <= width { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let g = unsafe { vld1_u8(gray_row.as_ptr().add(offset)) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { vst3_u8(dst.as_mut_ptr().add(offset * 3), uint8x8x3_t(g, g, g)); } @@ -34,9 +45,21 @@ unsafe fn fill_rgb_row_from_gray_neon(gray_row: &[u8], dst: &mut [u8]) { } pub(crate) fn fill_rgb_row_from_rgb(r_row: &[u8], g_row: &[u8], b_row: &[u8], dst: &mut [u8]) { + let width = r_row + .len() + .min(g_row.len()) + .min(b_row.len()) + .min(dst.len() / 3); + let r_row = &r_row[..width]; + let g_row = &g_row[..width]; + let b_row = &b_row[..width]; + let dst = &mut dst[..width * 3]; debug_assert_eq!(r_row.len(), g_row.len()); debug_assert_eq!(r_row.len(), b_row.len()); debug_assert_eq!(dst.len(), r_row.len() * 3); + // SAFETY: NEON is mandatory on supported aarch64 targets for this backend, + // and all source rows plus the destination share the same bounded width. + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_from_rgb_neon(r_row, g_row, b_row, dst); } @@ -47,9 +70,13 @@ unsafe fn fill_rgb_row_from_rgb_neon(r_row: &[u8], g_row: &[u8], b_row: &[u8], d let width = r_row.len(); let mut offset = 0; while offset + LANES <= width { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let r = unsafe { vld1_u8(r_row.as_ptr().add(offset)) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let g = unsafe { vld1_u8(g_row.as_ptr().add(offset)) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let b = unsafe { vld1_u8(b_row.as_ptr().add(offset)) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { vst3_u8(dst.as_mut_ptr().add(offset * 3), uint8x8x3_t(r, g, b)); } @@ -65,18 +92,25 @@ unsafe fn fill_rgb_row_from_rgb_neon(r_row: &[u8], g_row: &[u8], b_row: &[u8], d } } -const FIX_1_40200: i32 = 91_881; -const FIX_0_34414: i32 = 22_554; -const FIX_0_71414: i32 = 46_802; -const FIX_1_77200: i32 = 116_130; -const ROUND: i32 = 1 << 15; const LANES: usize = 8; const UPSAMPLED_LANES: usize = LANES * 2; pub(crate) fn fill_rgb_row_from_ycbcr(y_row: &[u8], cb_row: &[u8], cr_row: &[u8], dst: &mut [u8]) { + let width = y_row + .len() + .min(cb_row.len()) + .min(cr_row.len()) + .min(dst.len() / 3); + let y_row = &y_row[..width]; + let cb_row = &cb_row[..width]; + let cr_row = &cr_row[..width]; + let dst = &mut dst[..width * 3]; debug_assert_eq!(y_row.len(), cb_row.len()); debug_assert_eq!(y_row.len(), cr_row.len()); debug_assert_eq!(dst.len(), y_row.len() * 3); + // SAFETY: NEON is mandatory on supported aarch64 targets for this backend, + // and all source rows plus the destination share the same bounded width. + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_from_ycbcr_neon(y_row, cb_row, cr_row, dst); } @@ -105,6 +139,35 @@ pub(crate) fn fill_rgb_row_pair_from_420( dst_top: &mut [u8], dst_bottom: Option<&mut [u8]>, ) { + let chroma_width = prev_cb + .len() + .min(curr_cb.len()) + .min(next_cb.len()) + .min(prev_cr.len()) + .min(curr_cr.len()) + .min(next_cr.len()); + let bottom_width = match (y_bottom.as_ref(), dst_bottom.as_ref()) { + (Some(row), Some(dst)) => row.len().min(dst.len() / 3), + _ => usize::MAX, + }; + let width = y_top + .len() + .min(dst_top.len() / 3) + .min(bottom_width) + .min(chroma_width.saturating_mul(2)); + if width == 0 { + return; + } + let y_top = &y_top[..width]; + let y_bottom = y_bottom.and_then(|row| row.get(..width)); + let prev_cb = &prev_cb[..chroma_width]; + let curr_cb = &curr_cb[..chroma_width]; + let next_cb = &next_cb[..chroma_width]; + let prev_cr = &prev_cr[..chroma_width]; + let curr_cr = &curr_cr[..chroma_width]; + let next_cr = &next_cr[..chroma_width]; + let dst_top = &mut dst_top[..width * 3]; + let dst_bottom = dst_bottom.and_then(|row| row.get_mut(..width * 3)); debug_assert_eq!(dst_top.len(), y_top.len() * 3); debug_assert!(y_bottom.is_none_or(|row| row.len() == y_top.len())); debug_assert!(dst_bottom @@ -114,6 +177,10 @@ pub(crate) fn fill_rgb_row_pair_from_420( debug_assert_eq!(prev_cb.len(), next_cb.len()); debug_assert_eq!(prev_cr.len(), curr_cr.len()); debug_assert_eq!(prev_cr.len(), next_cr.len()); + // SAFETY: NEON is mandatory on supported aarch64 targets for this backend. + // The wrapper clamps luma, chroma, and destination slices so upsampled reads + // and RGB writes stay within the passed rows. + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_neon( y_top, y_bottom, prev_cb, curr_cb, next_cb, prev_cr, curr_cr, next_cr, dst_top, @@ -137,17 +204,53 @@ pub(crate) fn fill_rgb_row_pair_from_420_cropped( dst_top: &mut [u8], dst_bottom: Option<&mut [u8]>, ) { - let crop_end = crop_start + crop_width; + let chroma_width = prev_cb + .len() + .min(curr_cb.len()) + .min(next_cb.len()) + .min(prev_cr.len()) + .min(curr_cr.len()) + .min(next_cr.len()); + let available_chroma = chroma_width.saturating_mul(2).saturating_sub(crop_start); + let available_top = y_top.len().saturating_sub(crop_start); + let bottom_available = match (y_bottom.as_ref(), dst_bottom.as_ref()) { + (Some(row), Some(dst)) => row.len().saturating_sub(crop_start).min(dst.len() / 3), + _ => usize::MAX, + }; + let width = crop_width + .min(available_top) + .min(dst_top.len() / 3) + .min(bottom_available) + .min(available_chroma); + if width == 0 { + return; + } + let Some(crop_end) = crop_start.checked_add(width) else { + return; + }; + if y_top.get(crop_start..crop_end).is_none() { + return; + } + let y_bottom = y_bottom.and_then(|row| row.get(..)); + let prev_cb = &prev_cb[..chroma_width]; + let curr_cb = &curr_cb[..chroma_width]; + let next_cb = &next_cb[..chroma_width]; + let prev_cr = &prev_cr[..chroma_width]; + let curr_cr = &curr_cr[..chroma_width]; + let next_cr = &next_cr[..chroma_width]; + let dst_top = &mut dst_top[..width * 3]; + let dst_bottom = dst_bottom.and_then(|row| row.get_mut(..width * 3)); debug_assert!(crop_end <= y_top.len()); - debug_assert_eq!(dst_top.len(), crop_width * 3); - debug_assert!(y_bottom.is_none_or(|row| row.len() == y_top.len())); - debug_assert!(dst_bottom - .as_ref() - .is_none_or(|row| row.len() == crop_width * 3)); + debug_assert_eq!(dst_top.len(), width * 3); + debug_assert!(y_bottom.is_none_or(|row| crop_end <= row.len())); + debug_assert!(dst_bottom.as_ref().is_none_or(|row| row.len() == width * 3)); debug_assert_eq!(prev_cb.len(), curr_cb.len()); debug_assert_eq!(prev_cb.len(), next_cb.len()); debug_assert_eq!(prev_cr.len(), curr_cr.len()); debug_assert_eq!(prev_cr.len(), next_cr.len()); + // SAFETY: NEON is mandatory on supported aarch64 targets for this backend. + // The crop range and output rows are clamped to validated luma/chroma spans. + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_cropped_neon( y_top, y_bottom, prev_cb, curr_cb, next_cb, prev_cr, curr_cr, next_cr, crop_start, @@ -173,6 +276,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon( let width = y_top.len(); let chroma_width = curr_cb.len(); if let (Some(y_bottom), Some(dst_bottom)) = (y_bottom, dst_bottom) { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_neon_dual( y_top, y_bottom, prev_cb, curr_cb, next_cb, prev_cr, curr_cr, next_cr, dst_top, @@ -180,6 +284,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon( ); } } else { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_neon_top_only( y_top, @@ -212,6 +317,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon( dst_bottom: Option<&mut [u8]>, ) { if let (Some(y_bottom), Some(dst_bottom)) = (y_bottom, dst_bottom) { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_cropped_neon_dual( y_top, y_bottom, prev_cb, curr_cb, next_cb, prev_cr, curr_cr, next_cr, crop_start, @@ -219,6 +325,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon( ); } } else { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_cropped_neon_top_only( y_top, prev_cb, curr_cb, prev_cr, curr_cr, crop_start, crop_width, dst_top, @@ -271,6 +378,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_dual( if copy_width >= LANES && can_vectorize_cropped_420_chunk(y_top.len(), curr_cb.len(), aligned_x) { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_cropped_partial_chunk16_dual( y_top, @@ -314,6 +422,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_dual( break; } + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_chunk16_interior_neon( &y_top[x..x + UPSAMPLED_LANES], @@ -336,6 +445,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_dual( if remaining >= LANES { let x = crop_start + out_x; if can_vectorize_cropped_420_chunk(y_top.len(), curr_cb.len(), x) { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_cropped_partial_chunk16_dual( y_top, @@ -415,6 +525,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_top_only( if copy_width >= LANES && can_vectorize_cropped_420_chunk(y_top.len(), curr_cb.len(), aligned_x) { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_cropped_partial_chunk16_top_only( y_top, @@ -454,6 +565,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_top_only( break; } + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_from_420_chunk16_interior_neon( &y_top[x..x + UPSAMPLED_LANES], @@ -472,6 +584,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_neon_top_only( if remaining >= LANES { let x = crop_start + out_x; if can_vectorize_cropped_420_chunk(y_top.len(), curr_cb.len(), x) { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_cropped_partial_chunk16_top_only( y_top, @@ -534,6 +647,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_partial_chunk16_dual( debug_assert!(copy_width <= UPSAMPLED_LANES); let mut tmp_top = [0u8; UPSAMPLED_LANES * 3]; let mut tmp_bottom = [0u8; UPSAMPLED_LANES * 3]; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_chunk16_interior_neon( &y_top[aligned_x..aligned_x + UPSAMPLED_LANES], @@ -571,6 +685,7 @@ unsafe fn fill_rgb_row_pair_from_420_cropped_partial_chunk16_top_only( debug_assert!(src_skip + copy_width <= UPSAMPLED_LANES); debug_assert!(copy_width <= UPSAMPLED_LANES); let mut tmp_top = [0u8; UPSAMPLED_LANES * 3]; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_from_420_chunk16_interior_neon( &y_top[aligned_x..aligned_x + UPSAMPLED_LANES], @@ -614,6 +729,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon_dual( let chunk_width = (width - x).min(chunk_samples * 2); if can_vectorize_420_chunk(chroma_width, sample, chunk_width) { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_chunk16_interior_neon( &y_top[x..x + UPSAMPLED_LANES], @@ -634,6 +750,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon_dual( } if sample == 0 { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_edge_neon_dual( y_top, @@ -651,7 +768,8 @@ unsafe fn fill_rgb_row_pair_from_420_neon_dual( ); } } else if can_use_tail_420_chunk(chroma_width, sample, chunk_width) { - crate::bench_support::record_420_dispatch_neon_tail_chunk(); + record_420_dispatch_neon_tail_chunk(); + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_tail_neon_dual( y_top, @@ -671,12 +789,13 @@ unsafe fn fill_rgb_row_pair_from_420_neon_dual( ); } } else { - crate::bench_support::record_420_dispatch_scalar_chunk(); + record_420_dispatch_scalar_chunk(); let mut cb_top = [0u8; UPSAMPLED_LANES]; let mut cb_bot = [0u8; UPSAMPLED_LANES]; let mut cr_top = [0u8; UPSAMPLED_LANES]; let mut cr_bot = [0u8; UPSAMPLED_LANES]; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_upsampled_420_chunk( prev_cb, @@ -748,6 +867,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon_top_only( let chunk_width = (width - x).min(chunk_samples * 2); if can_vectorize_420_chunk(chroma_width, sample, chunk_width) { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_from_420_chunk16_interior_neon( &y_top[x..x + UPSAMPLED_LANES], @@ -764,6 +884,7 @@ unsafe fn fill_rgb_row_pair_from_420_neon_top_only( } if sample == 0 { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_edge_neon_top_only( y_top, @@ -777,7 +898,8 @@ unsafe fn fill_rgb_row_pair_from_420_neon_top_only( ); } } else if can_use_tail_420_chunk(chroma_width, sample, chunk_width) { - crate::bench_support::record_420_dispatch_neon_tail_chunk(); + record_420_dispatch_neon_tail_chunk(); + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_pair_from_420_tail_neon_top_only( y_top, @@ -793,9 +915,10 @@ unsafe fn fill_rgb_row_pair_from_420_neon_top_only( ); } } else { - crate::bench_support::record_420_dispatch_scalar_chunk(); + record_420_dispatch_scalar_chunk(); let mut cb_top = [0u8; UPSAMPLED_LANES]; let mut cr_top = [0u8; UPSAMPLED_LANES]; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_upsampled_420_chunk( prev_cb, @@ -850,13 +973,19 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_dual( let next_cr_head = load_head_window(next_cr, TAIL_WINDOW); let (cb_top, cb_bottom) = + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { upsampled_420_chunk16_pair_u16(&prev_cb_head, &curr_cb_head, &next_cb_head, 1) }; let (cr_top, cr_bottom) = + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { upsampled_420_chunk16_pair_u16(&prev_cr_head, &curr_cr_head, &next_cr_head, 1) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_top_lo = unsafe { load_eight(&y_top_tail, 0) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_top_hi = unsafe { load_eight(&y_top_tail, LANES) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_bottom_lo = unsafe { load_eight(&y_bottom_tail, 0) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_bottom_hi = unsafe { load_eight(&y_bottom_tail, LANES) }; let top_cb = ((u32::from(prev_cb[0]) + 3 * u32::from(curr_cb[0])) * 4 + 8) >> 4; @@ -868,6 +997,7 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_dual( ycbcr_to_rgb(y_bottom[0], bottom_cb as u8, bottom_cr as u8); if chunk_width == UPSAMPLED_LANES { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk_from_vectors_u16(y_top_lo, cb_top.0, cr_top.0, &mut dst_top[..LANES * 3]); fill_chunk_from_vectors_u16( @@ -894,6 +1024,7 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_dual( } else { let mut rgb_top = [0u8; UPSAMPLED_LANES * 3]; let mut rgb_bottom = [0u8; UPSAMPLED_LANES * 3]; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk_from_vectors_u16(y_top_lo, cb_top.0, cr_top.0, &mut rgb_top[..LANES * 3]); fill_chunk_from_vectors_u16(y_top_hi, cb_top.1, cr_top.1, &mut rgb_top[LANES * 3..]); @@ -935,9 +1066,13 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_top_only( let prev_cr_head = load_head_window(prev_cr, TAIL_WINDOW); let curr_cr_head = load_head_window(curr_cr, TAIL_WINDOW); + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let cb = unsafe { upsampled_420_chunk16_u16(&prev_cb_head, &curr_cb_head, 1) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let cr = unsafe { upsampled_420_chunk16_u16(&prev_cr_head, &curr_cr_head, 1) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_lo = unsafe { load_eight(&y_top_tail, 0) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_hi = unsafe { load_eight(&y_top_tail, LANES) }; let cb0 = ((u32::from(prev_cb[0]) + 3 * u32::from(curr_cb[0])) * 4 + 8) >> 4; @@ -945,6 +1080,7 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_top_only( let (r, g, b) = ycbcr_to_rgb(y_top[0], cb0 as u8, cr0 as u8); if chunk_width == UPSAMPLED_LANES { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk_from_vectors_u16(y_lo, cb.0, cr.0, &mut dst_top[..LANES * 3]); fill_chunk_from_vectors_u16( @@ -957,6 +1093,7 @@ unsafe fn fill_rgb_row_pair_from_420_edge_neon_top_only( dst_top[..3].copy_from_slice(&[r, g, b]); } else { let mut rgb = [0u8; UPSAMPLED_LANES * 3]; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk_from_vectors_u16(y_lo, cb.0, cr.0, &mut rgb[..LANES * 3]); fill_chunk_from_vectors_u16(y_hi, cb.1, cr.1, &mut rgb[LANES * 3..]); @@ -994,16 +1131,23 @@ unsafe fn fill_rgb_row_pair_from_420_tail_neon_dual( let next_cr_tail = load_tail_window(next_cr, sample_offset - 1, TAIL_WINDOW); let (cb_top, cb_bottom) = + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { upsampled_420_chunk16_pair_u16(&prev_cb_tail, &curr_cb_tail, &next_cb_tail, 1) }; let (cr_top, cr_bottom) = + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { upsampled_420_chunk16_pair_u16(&prev_cr_tail, &curr_cr_tail, &next_cr_tail, 1) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_top_lo = unsafe { load_eight(&y_top_tail, 0) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_top_hi = unsafe { load_eight(&y_top_tail, LANES) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_bottom_lo = unsafe { load_eight(&y_bottom_tail, 0) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_bottom_hi = unsafe { load_eight(&y_bottom_tail, LANES) }; if chunk_width == UPSAMPLED_LANES { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk_from_vectors_u16( y_top_lo, @@ -1051,6 +1195,7 @@ unsafe fn fill_rgb_row_pair_from_420_tail_neon_dual( } else { let mut rgb_top = [0u8; UPSAMPLED_LANES * 3]; let mut rgb_bottom = [0u8; UPSAMPLED_LANES * 3]; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk_from_vectors_u16(y_top_lo, cb_top.0, cr_top.0, &mut rgb_top[..LANES * 3]); fill_chunk_from_vectors_u16(y_top_hi, cb_top.1, cr_top.1, &mut rgb_top[LANES * 3..]); @@ -1112,12 +1257,17 @@ unsafe fn fill_rgb_row_pair_from_420_tail_neon_top_only( let prev_cr_tail = load_tail_window(prev_cr, sample_offset - 1, TAIL_WINDOW); let curr_cr_tail = load_tail_window(curr_cr, sample_offset - 1, TAIL_WINDOW); + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let cb = unsafe { upsampled_420_chunk16_u16(&prev_cb_tail, &curr_cb_tail, 1) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let cr = unsafe { upsampled_420_chunk16_u16(&prev_cr_tail, &curr_cr_tail, 1) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_lo = unsafe { load_eight(&y_top_tail, 0) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_hi = unsafe { load_eight(&y_top_tail, LANES) }; if chunk_width == UPSAMPLED_LANES { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk_from_vectors_u16(y_lo, cb.0, cr.0, &mut dst_top[x * 3..x * 3 + LANES * 3]); fill_chunk_from_vectors_u16( @@ -1140,6 +1290,7 @@ unsafe fn fill_rgb_row_pair_from_420_tail_neon_top_only( } } else { let mut rgb = [0u8; UPSAMPLED_LANES * 3]; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk_from_vectors_u16(y_lo, cb.0, cr.0, &mut rgb[..LANES * 3]); fill_chunk_from_vectors_u16(y_hi, cb.1, cr.1, &mut rgb[LANES * 3..]); @@ -1212,6 +1363,16 @@ fn can_use_tail_420_chunk(chroma_width: usize, sample_offset: usize, out_len: us out_len <= UPSAMPLED_LANES && sample_offset > 0 && sample_offset + LANES >= chroma_width } +fn record_420_dispatch_neon_tail_chunk() { + #[cfg(feature = "bench-internals")] + crate::bench_support::record_420_dispatch_neon_tail_chunk(); +} + +fn record_420_dispatch_scalar_chunk() { + #[cfg(feature = "bench-internals")] + crate::bench_support::record_420_dispatch_scalar_chunk(); +} + #[target_feature(enable = "neon")] unsafe fn fill_rgb_row_from_420_chunk16_interior_neon( y_row: &[u8], @@ -1225,10 +1386,15 @@ unsafe fn fill_rgb_row_from_420_chunk16_interior_neon( debug_assert_eq!(y_row.len(), UPSAMPLED_LANES); debug_assert_eq!(dst.len(), UPSAMPLED_LANES * 3); + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let cb = unsafe { upsampled_420_chunk16_u16(near_cb, curr_cb, sample_offset) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let cr = unsafe { upsampled_420_chunk16_u16(near_cr, curr_cr, sample_offset) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_lo = unsafe { load_eight(y_row, 0) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_hi = unsafe { load_eight(y_row, LANES) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk_from_vectors_u16(y_lo, cb.0, cr.0, &mut dst[..LANES * 3]); fill_chunk_from_vectors_u16(y_hi, cb.1, cr.1, &mut dst[LANES * 3..]); @@ -1256,14 +1422,21 @@ unsafe fn fill_rgb_row_pair_from_420_chunk16_interior_neon( debug_assert_eq!(dst_bottom.len(), UPSAMPLED_LANES * 3); let (cb_top, cb_bottom) = + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { upsampled_420_chunk16_pair_u16(prev_cb, curr_cb, next_cb, sample_offset) }; let (cr_top, cr_bottom) = + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { upsampled_420_chunk16_pair_u16(prev_cr, curr_cr, next_cr, sample_offset) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_top_lo = unsafe { load_eight(y_top, 0) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_top_hi = unsafe { load_eight(y_top, LANES) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_bottom_lo = unsafe { load_eight(y_bottom, 0) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_bottom_hi = unsafe { load_eight(y_bottom, LANES) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk_from_vectors_u16(y_top_lo, cb_top.0, cr_top.0, &mut dst_top[..LANES * 3]); fill_chunk_from_vectors_u16(y_top_hi, cb_top.1, cr_top.1, &mut dst_top[LANES * 3..]); @@ -1288,6 +1461,7 @@ unsafe fn fill_rgb_row_from_ycbcr_neon(y_row: &[u8], cb_row: &[u8], cr_row: &[u8 let mut offset = 0; while offset + UPSAMPLED_LANES <= width { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_rgb_row_from_ycbcr_chunk16_neon( &y_row[offset..offset + UPSAMPLED_LANES], @@ -1306,6 +1480,7 @@ unsafe fn fill_rgb_row_from_ycbcr_neon(y_row: &[u8], cb_row: &[u8], cr_row: &[u8 } while offset + LANES <= width { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk( y_row, @@ -1338,9 +1513,13 @@ unsafe fn fill_chunk( ) { debug_assert_eq!(dst_chunk.len(), LANES * 3); + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y = unsafe { load_eight(y_row, offset) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let cb = unsafe { load_eight(cb_row, offset) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let cr = unsafe { load_eight(cr_row, offset) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk_from_vectors(y, cb, cr, dst_chunk) }; } @@ -1352,30 +1531,10 @@ unsafe fn fill_chunk_from_vectors( dst_chunk: &mut [u8], ) { debug_assert_eq!(dst_chunk.len(), LANES * 3); - let y16 = vmovl_u8(y); let cb16 = vmovl_u8(cb); let cr16 = vmovl_u8(cr); - - let y_lo = widen_low(y16); - let y_hi = widen_high(y16); - let cb_lo = subtract_bias(widen_low(cb16)); - let cb_hi = subtract_bias(widen_high(cb16)); - let cr_lo = subtract_bias(widen_low(cr16)); - let cr_hi = subtract_bias(widen_high(cr16)); - - let (r_lo, g_lo, b_lo) = convert_half(y_lo, cb_lo, cr_lo); - let (r_hi, g_hi, b_hi) = convert_half(y_hi, cb_hi, cr_hi); - - let r_bytes = pack_eight_u8(r_lo, r_hi); - let g_bytes = pack_eight_u8(g_lo, g_hi); - let b_bytes = pack_eight_u8(b_lo, b_hi); - - unsafe { - vst3_u8( - dst_chunk.as_mut_ptr(), - uint8x8x3_t(r_bytes, g_bytes, b_bytes), - ); - } + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. + unsafe { fill_chunk_from_vectors_u16(y, cb16, cr16, dst_chunk) }; } #[target_feature(enable = "neon")] @@ -1402,6 +1561,7 @@ unsafe fn fill_chunk_from_vectors_u16( let g_bytes = pack_eight_u8(g_lo, g_hi); let b_bytes = pack_eight_u8(b_lo, b_hi); + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { vst3_u8( dst_chunk.as_mut_ptr(), @@ -1420,8 +1580,11 @@ unsafe fn fill_rgb_row_from_ycbcr_chunk16_neon( debug_assert_eq!(y_row.len(), UPSAMPLED_LANES); debug_assert_eq!(dst.len(), UPSAMPLED_LANES * 3); + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_lo = unsafe { load_eight(y_row, 0) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let y_hi = unsafe { load_eight(y_row, LANES) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_chunk_from_vectors( y_lo, @@ -1440,6 +1603,10 @@ unsafe fn fill_rgb_row_from_ycbcr_chunk16_neon( #[target_feature(enable = "neon")] unsafe fn load_eight(src: &[u8], offset: usize) -> uint8x8_t { + debug_assert!(offset <= src.len().saturating_sub(LANES)); + // SAFETY: callers guarantee there are at least eight readable bytes at + // `offset`; `vld1_u8` accepts unaligned loads. + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { vld1_u8(src.as_ptr().add(offset)) } } @@ -1498,6 +1665,7 @@ unsafe fn fill_upsampled_420_chunk( out: &mut [u8], ) { if can_vectorize_420_chunk(curr.len(), sample_offset, out.len()) { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { fill_upsampled_420_chunk_neon(near, curr, sample_offset, out); } @@ -1519,27 +1687,9 @@ fn fill_upsampled_420_chunk_scalar( return; } - let colsum = |i: usize| 3 * u32::from(curr[i]) + u32::from(near[i]); - for sample in 0..out.len().div_ceil(2) { - let global_sample = sample_offset + sample; - let this = colsum(global_sample); - let x = global_sample * 2; - let local_x = sample * 2; - out[local_x] = if x == 0 { - ((this * 4 + 8) >> 4) as u8 - } else { - let last = colsum(global_sample - 1); - ((this * 3 + last + 8) >> 4) as u8 - }; - if local_x + 1 >= out.len() { - break; - } - out[local_x + 1] = if x + 1 == output_width - 1 { - ((this * 4 + 7) >> 4) as u8 - } else { - let next = colsum((global_sample + 1).min(n - 1)); - ((this * 3 + next + 7) >> 4) as u8 - }; + let output_x = sample_offset * 2; + for (local_x, slot) in out.iter_mut().enumerate() { + *slot = h2v2_fancy_sample_for_width(near, curr, output_width, output_x + local_x); } } @@ -1559,6 +1709,7 @@ unsafe fn fill_upsampled_420_chunk_neon( sample_offset, out.len() )); + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. unsafe { vst1q_u8( out.as_mut_ptr(), @@ -1569,6 +1720,7 @@ unsafe fn fill_upsampled_420_chunk_neon( #[target_feature(enable = "neon")] unsafe fn upsampled_420_chunk16(near: &[u8], curr: &[u8], sample_offset: usize) -> uint8x16_t { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let lanes = unsafe { upsampled_420_chunk16_u16(near, curr, sample_offset) }; let even8 = vqmovn_u16(lanes.0); let odd8 = vqmovn_u16(lanes.1); @@ -1582,8 +1734,11 @@ unsafe fn upsampled_420_chunk16_u16( curr: &[u8], sample_offset: usize, ) -> core::arch::aarch64::uint16x8x2_t { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let this = unsafe { colsum_eight(near, curr, sample_offset) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let prev = unsafe { colsum_eight(near, curr, sample_offset - 1) }; + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let next = unsafe { colsum_eight(near, curr, sample_offset + 1) }; let three_this = vaddq_u16(this, vaddq_u16(this, this)); @@ -1602,8 +1757,11 @@ unsafe fn upsampled_420_chunk16_pair_u16( core::arch::aarch64::uint16x8x2_t, core::arch::aarch64::uint16x8x2_t, ) { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let curr_prev = vmovl_u8(unsafe { vld1_u8(curr.as_ptr().add(sample_offset - 1)) }); + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let curr_this = vmovl_u8(unsafe { vld1_u8(curr.as_ptr().add(sample_offset)) }); + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let curr_next = vmovl_u8(unsafe { vld1_u8(curr.as_ptr().add(sample_offset + 1)) }); let three_prev = vaddq_u16(curr_prev, vaddq_u16(curr_prev, curr_prev)); @@ -1612,27 +1770,33 @@ unsafe fn upsampled_420_chunk16_pair_u16( let top_prev = vaddq_u16( three_prev, + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. vmovl_u8(unsafe { vld1_u8(top_near.as_ptr().add(sample_offset - 1)) }), ); let top_this = vaddq_u16( three_this, + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. vmovl_u8(unsafe { vld1_u8(top_near.as_ptr().add(sample_offset)) }), ); let top_next = vaddq_u16( three_next, + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. vmovl_u8(unsafe { vld1_u8(top_near.as_ptr().add(sample_offset + 1)) }), ); let bottom_prev = vaddq_u16( three_prev, + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. vmovl_u8(unsafe { vld1_u8(bottom_near.as_ptr().add(sample_offset - 1)) }), ); let bottom_this = vaddq_u16( three_this, + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. vmovl_u8(unsafe { vld1_u8(bottom_near.as_ptr().add(sample_offset)) }), ); let bottom_next = vaddq_u16( three_next, + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. vmovl_u8(unsafe { vld1_u8(bottom_near.as_ptr().add(sample_offset + 1)) }), ); @@ -1664,7 +1828,9 @@ unsafe fn upsampled_420_chunk16_pair_u16( #[target_feature(enable = "neon")] unsafe fn colsum_eight(near: &[u8], curr: &[u8], sample_offset: usize) -> uint16x8_t { + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let near16 = vmovl_u8(unsafe { vld1_u8(near.as_ptr().add(sample_offset)) }); + // SAFETY: NEON pointer uses are bounded by row slicing, lane strides, or helper preconditions. let curr16 = vmovl_u8(unsafe { vld1_u8(curr.as_ptr().add(sample_offset)) }); vaddq_u16(vaddq_u16(curr16, curr16), vaddq_u16(curr16, near16)) } diff --git a/crates/signinum-jpeg/src/backend/scalar.rs b/crates/j2k-jpeg/src/backend/scalar.rs similarity index 68% rename from crates/signinum-jpeg/src/backend/scalar.rs rename to crates/j2k-jpeg/src/backend/scalar.rs index e25c7750..bdc6f9aa 100644 --- a/crates/signinum-jpeg/src/backend/scalar.rs +++ b/crates/j2k-jpeg/src/backend/scalar.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +use crate::color::upsample::h2v2_fancy_sample; use crate::color::ycbcr::ycbcr_to_rgb; pub(crate) fn fill_rgb_row_from_gray(gray_row: &[u8], dst: &mut [u8]) { @@ -37,6 +38,54 @@ pub(crate) fn fill_rgb_row_from_ycbcr(y_row: &[u8], cb_row: &[u8], cr_row: &[u8] } } +pub(crate) fn fill_rgba_row_from_gray(gray_row: &[u8], dst: &mut [u8], alpha: u8) { + for (&gray, pixel) in gray_row.iter().zip(dst.chunks_exact_mut(4)) { + write_rgba_pixel(pixel, gray, gray, gray, alpha); + } +} + +pub(crate) fn fill_rgba_row_from_rgb( + r_row: &[u8], + g_row: &[u8], + b_row: &[u8], + dst: &mut [u8], + alpha: u8, +) { + for (((&r, &g), &b), pixel) in r_row + .iter() + .zip(g_row.iter()) + .zip(b_row.iter()) + .zip(dst.chunks_exact_mut(4)) + { + write_rgba_pixel(pixel, r, g, b, alpha); + } +} + +pub(crate) fn fill_rgba_row_from_ycbcr( + y_row: &[u8], + cb_row: &[u8], + cr_row: &[u8], + dst: &mut [u8], + alpha: u8, +) { + for (((&y_sample, &cb_sample), &cr_sample), pixel) in y_row + .iter() + .zip(cb_row.iter()) + .zip(cr_row.iter()) + .zip(dst.chunks_exact_mut(4)) + { + let (r, g, b) = ycbcr_to_rgb(y_sample, cb_sample, cr_sample); + write_rgba_pixel(pixel, r, g, b, alpha); + } +} + +fn write_rgba_pixel(pixel: &mut [u8], r: u8, g: u8, b: u8, alpha: u8) { + pixel[0] = r; + pixel[1] = g; + pixel[2] = b; + pixel[3] = alpha; +} + #[allow(clippy::too_many_arguments)] pub(crate) fn fill_rgb_row_pair_from_420( y_top: &[u8], @@ -56,8 +105,8 @@ pub(crate) fn fill_rgb_row_pair_from_420( debug_assert!(dst_bottom.as_ref().is_none_or(|row| row.len() == width * 3)); for (x, pixel) in dst_top.chunks_exact_mut(3).enumerate() { - let cb = h2v2_sample(prev_cb, curr_cb, x); - let cr = h2v2_sample(prev_cr, curr_cr, x); + let cb = h2v2_fancy_sample(prev_cb, curr_cb, x); + let cr = h2v2_fancy_sample(prev_cr, curr_cr, x); let (r, g, b) = ycbcr_to_rgb(y_top[x], cb, cr); pixel[0] = r; pixel[1] = g; @@ -66,8 +115,8 @@ pub(crate) fn fill_rgb_row_pair_from_420( if let (Some(y_bottom), Some(dst_bottom)) = (y_bottom, dst_bottom) { for (x, pixel) in dst_bottom.chunks_exact_mut(3).enumerate() { - let cb = h2v2_sample(next_cb, curr_cb, x); - let cr = h2v2_sample(next_cr, curr_cr, x); + let cb = h2v2_fancy_sample(next_cb, curr_cb, x); + let cr = h2v2_fancy_sample(next_cr, curr_cr, x); let (r, g, b) = ycbcr_to_rgb(y_bottom[x], cb, cr); pixel[0] = r; pixel[1] = g; @@ -105,8 +154,8 @@ pub(crate) fn fill_rgb_row_pair_from_420_cropped( for (local_x, pixel) in dst_top.chunks_exact_mut(3).enumerate() { let x = crop_start + local_x; - let cb = h2v2_sample(prev_cb, curr_cb, x); - let cr = h2v2_sample(prev_cr, curr_cr, x); + let cb = h2v2_fancy_sample(prev_cb, curr_cb, x); + let cr = h2v2_fancy_sample(prev_cr, curr_cr, x); let (r, g, b) = ycbcr_to_rgb(y_top[x], cb, cr); pixel[0] = r; pixel[1] = g; @@ -116,8 +165,8 @@ pub(crate) fn fill_rgb_row_pair_from_420_cropped( if let (Some(y_bottom), Some(dst_bottom)) = (y_bottom, dst_bottom) { for (local_x, pixel) in dst_bottom.chunks_exact_mut(3).enumerate() { let x = crop_start + local_x; - let cb = h2v2_sample(next_cb, curr_cb, x); - let cr = h2v2_sample(next_cr, curr_cr, x); + let cb = h2v2_fancy_sample(next_cb, curr_cb, x); + let cr = h2v2_fancy_sample(next_cr, curr_cr, x); let (r, g, b) = ycbcr_to_rgb(y_bottom[x], cb, cr); pixel[0] = r; pixel[1] = g; @@ -125,30 +174,3 @@ pub(crate) fn fill_rgb_row_pair_from_420_cropped( } } } - -fn h2v2_sample(near: &[u8], curr: &[u8], x: usize) -> u8 { - debug_assert_eq!(near.len(), curr.len()); - let n = curr.len(); - if n == 0 { - return 0; - } - let sample = (x / 2).min(n - 1); - let colsum = |idx: usize| 3 * u32::from(curr[idx]) + u32::from(near[idx]); - if n == 1 { - return ((4 * colsum(0) + 8) >> 4) as u8; - } - - let this = colsum(sample); - match x { - 0 => ((this * 4 + 8) >> 4) as u8, - _ if x == n * 2 - 1 => ((this * 4 + 7) >> 4) as u8, - _ if x.is_multiple_of(2) => { - let last = colsum(sample - 1); - ((this * 3 + last + 8) >> 4) as u8 - } - _ => { - let next = colsum(sample + 1); - ((this * 3 + next + 7) >> 4) as u8 - } - } -} diff --git a/crates/signinum-jpeg/src/backend/tests.rs b/crates/j2k-jpeg/src/backend/tests.rs similarity index 87% rename from crates/signinum-jpeg/src/backend/tests.rs rename to crates/j2k-jpeg/src/backend/tests.rs index c36f4074..ceb8d993 100644 --- a/crates/signinum-jpeg/src/backend/tests.rs +++ b/crates/j2k-jpeg/src/backend/tests.rs @@ -364,6 +364,30 @@ fn avx2_ycbcr_rows_match_scalar_reference_for_tail_widths() { assert_eq!(actual, expected); } +#[cfg(target_arch = "x86_64")] +#[test] +fn avx2_ycbcr_rows_use_shortest_safe_prefix() { + if !std::is_x86_feature_detected!("avx2") { + return; + } + + let y = [16u8, 40, 90, 200, 12, 24, 48, 96]; + let cb = [128u8, 100, 200, 180, 90]; + let cr = [128u8, 220, 10, 90, 70, 60, 50]; + let mut expected = vec![0xAAu8; y.len() * 3]; + let mut actual = expected.clone(); + + scalar::fill_rgb_row_from_ycbcr( + &y[..cb.len()], + &cb, + &cr[..cb.len()], + &mut expected[..cb.len() * 3], + ); + super::x86::fill_rgb_row_from_ycbcr_for_test(&y, &cb, &cr, &mut actual); + + assert_eq!(actual, expected); +} + #[cfg(target_arch = "x86_64")] #[test] fn avx2_gray_rows_match_scalar_reference() { @@ -657,6 +681,60 @@ fn avx2_420_cropped_row_pair_matches_scalar_reference() { assert_eq!(actual_bot, expected_bot); } +#[cfg(target_arch = "x86_64")] +#[test] +fn avx2_420_row_pair_uses_shortest_safe_prefix() { + if !std::is_x86_feature_detected!("avx2") { + return; + } + + let backend = super::Backend { + kind: super::BackendKind::Avx2, + }; + let y_top = [16u8, 24, 32, 40, 48, 56, 64, 72, 80]; + let y_bot = [80u8, 88, 96, 104]; + let prev_cb = [120u8, 100, 140]; + let curr_cb = [110u8, 90, 130]; + let next_cb = [100u8, 80, 120]; + let prev_cr = [130u8, 150, 170]; + let curr_cr = [140u8, 160, 180]; + let next_cr = [150u8, 170, 190]; + + let safe_width = y_bot.len(); + let mut expected_top = vec![0xAAu8; y_top.len() * 3]; + let mut expected_bot = vec![0xAAu8; y_bot.len() * 3]; + scalar::fill_rgb_row_pair_from_420( + &y_top[..safe_width], + Some(&y_bot), + &prev_cb, + &curr_cb, + &next_cb, + &prev_cr, + &curr_cr, + &next_cr, + &mut expected_top[..safe_width * 3], + Some(&mut expected_bot), + ); + + let mut actual_top = vec![0xAAu8; y_top.len() * 3]; + let mut actual_bot = vec![0xAAu8; y_bot.len() * 3]; + backend.fill_rgb_row_pair_from_420( + &y_top, + Some(&y_bot), + &prev_cb, + &curr_cb, + &next_cb, + &prev_cr, + &curr_cr, + &next_cr, + &mut actual_top, + Some(&mut actual_bot), + ); + + assert_eq!(actual_top, expected_top); + assert_eq!(actual_bot, expected_bot); +} + #[cfg(target_arch = "x86_64")] #[test] fn avx2_backend_prefers_cropped_420_region_when_available() { @@ -686,6 +764,26 @@ fn neon_ycbcr_rows_match_scalar_reference_for_tail_widths() { assert_eq!(actual, expected); } +#[cfg(target_arch = "aarch64")] +#[test] +fn neon_ycbcr_rows_use_shortest_safe_prefix() { + let y = [16u8, 40, 90, 200, 12, 24, 48, 96]; + let cb = [128u8, 100, 200, 180, 90]; + let cr = [128u8, 220, 10, 90, 70, 60, 50]; + let mut expected = vec![0xAAu8; y.len() * 3]; + let mut actual = expected.clone(); + + scalar::fill_rgb_row_from_ycbcr( + &y[..cb.len()], + &cb, + &cr[..cb.len()], + &mut expected[..cb.len() * 3], + ); + super::neon::fill_rgb_row_from_ycbcr_for_test(&y, &cb, &cr, &mut actual); + + assert_eq!(actual, expected); +} + #[cfg(target_arch = "aarch64")] #[test] fn neon_ycbcr_rows_match_scalar_reference_across_multiple_chunks() { @@ -849,6 +947,56 @@ fn neon_420_row_pair_matches_scalar_reference_across_multiple_chunks() { assert_eq!(actual_bot, expected_bot); } +#[cfg(target_arch = "aarch64")] +#[test] +fn neon_420_row_pair_uses_shortest_safe_prefix() { + let backend = super::Backend { + kind: super::BackendKind::Neon, + }; + let y_top = [16u8, 24, 32, 40, 48, 56, 64, 72, 80]; + let y_bot = [80u8, 88, 96, 104]; + let prev_cb = [120u8, 100, 140]; + let curr_cb = [110u8, 90, 130]; + let next_cb = [100u8, 80, 120]; + let prev_cr = [130u8, 150, 170]; + let curr_cr = [140u8, 160, 180]; + let next_cr = [150u8, 170, 190]; + + let safe_width = y_bot.len(); + let mut expected_top = vec![0xAAu8; y_top.len() * 3]; + let mut expected_bot = vec![0xAAu8; y_bot.len() * 3]; + scalar::fill_rgb_row_pair_from_420( + &y_top[..safe_width], + Some(&y_bot), + &prev_cb, + &curr_cb, + &next_cb, + &prev_cr, + &curr_cr, + &next_cr, + &mut expected_top[..safe_width * 3], + Some(&mut expected_bot), + ); + + let mut actual_top = vec![0xAAu8; y_top.len() * 3]; + let mut actual_bot = vec![0xAAu8; y_bot.len() * 3]; + backend.fill_rgb_row_pair_from_420( + &y_top, + Some(&y_bot), + &prev_cb, + &curr_cb, + &next_cb, + &prev_cr, + &curr_cr, + &next_cr, + &mut actual_top, + Some(&mut actual_bot), + ); + + assert_eq!(actual_top, expected_top); + assert_eq!(actual_bot, expected_bot); +} + #[cfg(target_arch = "aarch64")] #[test] fn neon_420_row_pair_handles_missing_bottom_row() { diff --git a/crates/signinum-jpeg/src/backend/x86.rs b/crates/j2k-jpeg/src/backend/x86.rs similarity index 64% rename from crates/signinum-jpeg/src/backend/x86.rs rename to crates/j2k-jpeg/src/backend/x86.rs index 405da25f..95377682 100644 --- a/crates/signinum-jpeg/src/backend/x86.rs +++ b/crates/j2k-jpeg/src/backend/x86.rs @@ -8,15 +8,13 @@ use core::arch::x86_64::{ }; use core::cell::RefCell; -use crate::color::upsample::{upsample_h2v2_fancy_row, upsample_h2v2_fancy_rows}; +use crate::color::upsample::{ + h2v2_fancy_sample, upsample_h2v2_fancy_row, upsample_h2v2_fancy_rows, +}; +use crate::color::ycbcr::{FIX_0_34414, FIX_0_71414, FIX_1_40200, FIX_1_77200, ROUND}; use super::scalar; -const FIX_1_40200: i32 = 91_881; -const FIX_0_34414: i32 = 22_554; -const FIX_0_71414: i32 = 46_802; -const FIX_1_77200: i32 = 116_130; -const ROUND: i32 = 1 << 15; const LANES: usize = 8; const RGB_UNROLL: usize = 8; @@ -42,6 +40,9 @@ std::thread_local! { } pub(crate) fn fill_rgb_row_from_gray(gray_row: &[u8], dst: &mut [u8]) { + let width = gray_row.len().min(dst.len() / 3); + let gray_row = &gray_row[..width]; + let dst = &mut dst[..width * 3]; debug_assert_eq!(dst.len(), gray_row.len() * 3); let mut offset = 0; while offset + RGB_UNROLL <= gray_row.len() { @@ -60,6 +61,15 @@ pub(crate) fn fill_rgb_row_from_gray(gray_row: &[u8], dst: &mut [u8]) { } pub(crate) fn fill_rgb_row_from_rgb(r_row: &[u8], g_row: &[u8], b_row: &[u8], dst: &mut [u8]) { + let width = r_row + .len() + .min(g_row.len()) + .min(b_row.len()) + .min(dst.len() / 3); + let r_row = &r_row[..width]; + let g_row = &g_row[..width]; + let b_row = &b_row[..width]; + let dst = &mut dst[..width * 3]; debug_assert_eq!(r_row.len(), g_row.len()); debug_assert_eq!(r_row.len(), b_row.len()); debug_assert_eq!(dst.len(), r_row.len() * 3); @@ -92,9 +102,20 @@ pub(crate) fn fill_rgb_row_from_rgb(r_row: &[u8], g_row: &[u8], b_row: &[u8], ds } pub(crate) fn fill_rgb_row_from_ycbcr(y_row: &[u8], cb_row: &[u8], cr_row: &[u8], dst: &mut [u8]) { + let width = y_row + .len() + .min(cb_row.len()) + .min(cr_row.len()) + .min(dst.len() / 3); + let y_row = &y_row[..width]; + let cb_row = &cb_row[..width]; + let cr_row = &cr_row[..width]; + let dst = &mut dst[..width * 3]; debug_assert_eq!(y_row.len(), cb_row.len()); debug_assert_eq!(y_row.len(), cr_row.len()); debug_assert_eq!(dst.len(), y_row.len() * 3); + // SAFETY: Backend dispatch selects this path only when AVX2 is available. + // All source rows and the destination are narrowed to the same pixel count. unsafe { fill_rgb_row_from_ycbcr_avx2(y_row, cb_row, cr_row, dst); } @@ -113,6 +134,35 @@ pub(crate) fn fill_rgb_row_pair_from_420( dst_top: &mut [u8], dst_bottom: Option<&mut [u8]>, ) { + let chroma_width = prev_cb + .len() + .min(curr_cb.len()) + .min(next_cb.len()) + .min(prev_cr.len()) + .min(curr_cr.len()) + .min(next_cr.len()); + let bottom_width = match (y_bottom.as_ref(), dst_bottom.as_ref()) { + (Some(row), Some(dst)) => row.len().min(dst.len() / 3), + _ => usize::MAX, + }; + let width = y_top + .len() + .min(dst_top.len() / 3) + .min(bottom_width) + .min(chroma_width.saturating_mul(2)); + if width == 0 { + return; + } + let y_top = &y_top[..width]; + let y_bottom = y_bottom.and_then(|row| row.get(..width)); + let prev_cb = &prev_cb[..chroma_width]; + let curr_cb = &curr_cb[..chroma_width]; + let next_cb = &next_cb[..chroma_width]; + let prev_cr = &prev_cr[..chroma_width]; + let curr_cr = &curr_cr[..chroma_width]; + let next_cr = &next_cr[..chroma_width]; + let dst_top = &mut dst_top[..width * 3]; + let dst_bottom = dst_bottom.and_then(|row| row.get_mut(..width * 3)); debug_assert_eq!(dst_top.len(), y_top.len() * 3); debug_assert!(y_bottom.is_none_or(|row| row.len() == y_top.len())); debug_assert!(dst_bottom @@ -126,6 +176,9 @@ pub(crate) fn fill_rgb_row_pair_from_420( ROW_PAIR_SCRATCH.with(|scratch| { let mut scratch = scratch.borrow_mut(); scratch.ensure_width(y_top.len()); + // SAFETY: Backend dispatch selects this path only when AVX2 is + // available. The wrapper clamps luma, chroma, and destination rows so + // all upsampled reads and RGB writes fit the passed slices. unsafe { fill_rgb_row_pair_from_420_avx2( y_top, @@ -159,13 +212,45 @@ pub(crate) fn fill_rgb_row_pair_from_420_cropped( dst_top: &mut [u8], dst_bottom: Option<&mut [u8]>, ) { - let crop_end = crop_start + crop_width; - debug_assert!(crop_end <= y_top.len()); - debug_assert_eq!(dst_top.len(), crop_width * 3); - debug_assert!(y_bottom.is_none_or(|row| row.len() == y_top.len())); - debug_assert!(dst_bottom - .as_ref() - .is_none_or(|row| row.len() == crop_width * 3)); + let chroma_width = prev_cb + .len() + .min(curr_cb.len()) + .min(next_cb.len()) + .min(prev_cr.len()) + .min(curr_cr.len()) + .min(next_cr.len()); + let available_chroma = chroma_width.saturating_mul(2).saturating_sub(crop_start); + let available_top = y_top.len().saturating_sub(crop_start); + let bottom_available = match (y_bottom.as_ref(), dst_bottom.as_ref()) { + (Some(row), Some(dst)) => row.len().saturating_sub(crop_start).min(dst.len() / 3), + _ => usize::MAX, + }; + let width = crop_width + .min(available_top) + .min(dst_top.len() / 3) + .min(bottom_available) + .min(available_chroma); + if width == 0 { + return; + } + let Some(crop_end) = crop_start.checked_add(width) else { + return; + }; + let Some(y_top_crop) = y_top.get(crop_start..crop_end) else { + return; + }; + let y_bottom = y_bottom.and_then(|row| row.get(crop_start..crop_end)); + let prev_cb = &prev_cb[..chroma_width]; + let curr_cb = &curr_cb[..chroma_width]; + let next_cb = &next_cb[..chroma_width]; + let prev_cr = &prev_cr[..chroma_width]; + let curr_cr = &curr_cr[..chroma_width]; + let next_cr = &next_cr[..chroma_width]; + let dst_top = &mut dst_top[..width * 3]; + let dst_bottom = dst_bottom.and_then(|row| row.get_mut(..width * 3)); + debug_assert_eq!(dst_top.len(), width * 3); + debug_assert!(y_bottom.is_none_or(|row| row.len() == width)); + debug_assert!(dst_bottom.as_ref().is_none_or(|row| row.len() == width * 3)); debug_assert_eq!(prev_cb.len(), curr_cb.len()); debug_assert_eq!(prev_cb.len(), next_cb.len()); debug_assert_eq!(prev_cr.len(), curr_cr.len()); @@ -173,33 +258,34 @@ pub(crate) fn fill_rgb_row_pair_from_420_cropped( ROW_PAIR_SCRATCH.with(|scratch| { let mut scratch = scratch.borrow_mut(); - scratch.ensure_width(crop_width); + scratch.ensure_width(width); let RowPairScratch { cb_top, cb_bottom, cr_top, cr_bottom, } = &mut *scratch; - let cb_top = &mut cb_top[..crop_width]; - let cr_top = &mut cr_top[..crop_width]; + let cb_top = &mut cb_top[..width]; + let cr_top = &mut cr_top[..width]; fill_cropped_h2v2_row(prev_cb, curr_cb, crop_start, cb_top); fill_cropped_h2v2_row(prev_cr, curr_cr, crop_start, cr_top); + // SAFETY: Backend dispatch selects this path only when AVX2 is + // available. `y_top_crop`, scratch chroma rows, and destination slices + // all have the same bounded pixel width. unsafe { - fill_rgb_row_from_ycbcr_avx2(&y_top[crop_start..crop_end], cb_top, cr_top, dst_top); + fill_rgb_row_from_ycbcr_avx2(y_top_crop, cb_top, cr_top, dst_top); } if let (Some(y_bottom), Some(dst_bottom)) = (y_bottom, dst_bottom) { - let cb_bottom = &mut cb_bottom[..crop_width]; - let cr_bottom = &mut cr_bottom[..crop_width]; + let cb_bottom = &mut cb_bottom[..width]; + let cr_bottom = &mut cr_bottom[..width]; fill_cropped_h2v2_row(next_cb, curr_cb, crop_start, cb_bottom); fill_cropped_h2v2_row(next_cr, curr_cr, crop_start, cr_bottom); + // SAFETY: Backend dispatch selects this path only when AVX2 is + // available. The bottom luma, scratch chroma, and destination + // slices were clamped to the same bounded pixel width. unsafe { - fill_rgb_row_from_ycbcr_avx2( - &y_bottom[crop_start..crop_end], - cb_bottom, - cr_bottom, - dst_bottom, - ); + fill_rgb_row_from_ycbcr_avx2(y_bottom, cb_bottom, cr_bottom, dst_bottom); } } }); @@ -207,34 +293,7 @@ pub(crate) fn fill_rgb_row_pair_from_420_cropped( fn fill_cropped_h2v2_row(near: &[u8], curr: &[u8], crop_start: usize, out: &mut [u8]) { for (local_x, slot) in out.iter_mut().enumerate() { - *slot = h2v2_sample(near, curr, crop_start + local_x); - } -} - -fn h2v2_sample(near: &[u8], curr: &[u8], x: usize) -> u8 { - debug_assert_eq!(near.len(), curr.len()); - let n = curr.len(); - if n == 0 { - return 0; - } - let sample = (x / 2).min(n - 1); - let colsum = |idx: usize| 3 * u32::from(curr[idx]) + u32::from(near[idx]); - if n == 1 { - return ((4 * colsum(0) + 8) >> 4) as u8; - } - - let this = colsum(sample); - match x { - 0 => ((this * 4 + 8) >> 4) as u8, - _ if x == n * 2 - 1 => ((this * 4 + 7) >> 4) as u8, - _ if x.is_multiple_of(2) => { - let last = colsum(sample - 1); - ((this * 3 + last + 8) >> 4) as u8 - } - _ => { - let next = colsum(sample + 1); - ((this * 3 + next + 7) >> 4) as u8 - } + *slot = h2v2_fancy_sample(near, curr, crop_start + local_x); } } @@ -267,6 +326,9 @@ unsafe fn fill_rgb_row_pair_from_420_avx2( let cr_bottom = &mut cr_bottom[..width]; upsample_h2v2_fancy_rows(prev_cb, curr_cb, next_cb, width, cb_top, cb_bottom); upsample_h2v2_fancy_rows(prev_cr, curr_cr, next_cr, width, cr_top, cr_bottom); + // SAFETY: This AVX2 helper is reached through the safe wrapper, which + // clamps both luma rows, both scratch chroma rows, and both RGB + // destination rows to the same pixel width. unsafe { fill_rgb_row_from_ycbcr_avx2(y_top, cb_top, cr_top, dst_top); fill_rgb_row_from_ycbcr_avx2(y_bottom, cb_bottom, cr_bottom, dst_bottom); @@ -274,6 +336,9 @@ unsafe fn fill_rgb_row_pair_from_420_avx2( } else { upsample_h2v2_fancy_row(prev_cb, curr_cb, next_cb, width, false, cb_top); upsample_h2v2_fancy_row(prev_cr, curr_cr, next_cr, width, false, cr_top); + // SAFETY: This AVX2 helper is reached through the safe wrapper, which + // clamps the luma row, scratch chroma rows, and RGB destination row to + // the same pixel width. unsafe { fill_rgb_row_from_ycbcr_avx2(y_top, cb_top, cr_top, dst_top); } @@ -311,6 +376,8 @@ unsafe fn fill_rgb_row_from_ycbcr_avx2(y_row: &[u8], cb_row: &[u8], cr_row: &[u8 let mut offset = 0; while offset + (LANES * 2) <= width { + // SAFETY: The safe wrapper slices all input rows and `dst` to the same + // pixel count, and this loop only passes full eight-pixel chunks. unsafe { fill_chunk( y_row, @@ -331,6 +398,8 @@ unsafe fn fill_rgb_row_from_ycbcr_avx2(y_row: &[u8], cb_row: &[u8], cr_row: &[u8 } while offset + LANES <= width { + // SAFETY: The safe wrapper slices all input rows and `dst` to the same + // pixel count, and this loop only passes a full eight-pixel chunk. unsafe { fill_chunk( y_row, @@ -363,8 +432,11 @@ unsafe fn fill_chunk( ) { debug_assert_eq!(dst_chunk.len(), LANES * 3); + // SAFETY: callers prove `offset + LANES <= row.len()` for each source row. let y = unsafe { load_eight(y_row, offset) }; + // SAFETY: callers prove `offset + LANES <= row.len()` for each source row. let cb = unsafe { load_eight(cb_row, offset) }; + // SAFETY: callers prove `offset + LANES <= row.len()` for each source row. let cr = unsafe { load_eight(cr_row, offset) }; let bias = _mm256_set1_epi32(128); @@ -388,6 +460,7 @@ unsafe fn fill_chunk( ); let b = _mm256_add_epi32(y32, fixed_mul_shift(cb32, FIX_1_77200)); + // SAFETY: `dst_chunk` is narrowed by the caller to exactly one RGB chunk. unsafe { store_rgb_chunk(dst_chunk, r, g, b); } @@ -395,6 +468,9 @@ unsafe fn fill_chunk( #[target_feature(enable = "avx2")] unsafe fn load_eight(src: &[u8], offset: usize) -> __m128i { + debug_assert!(offset <= src.len().saturating_sub(LANES)); + // SAFETY: the caller guarantees there are at least eight readable bytes at + // `offset`; `_mm_loadl_epi64` accepts unaligned loads. unsafe { _mm_loadl_epi64(src.as_ptr().add(offset).cast()) } } @@ -411,8 +487,12 @@ fn fixed_mul_shift(values: __m256i, coefficient: i32) -> __m256i { #[target_feature(enable = "avx2")] unsafe fn store_rgb_chunk(dst_chunk: &mut [u8], r: __m256i, g: __m256i, b: __m256i) { + debug_assert_eq!(dst_chunk.len(), LANES * 3); + // SAFETY: packing only rearranges register values and does not dereference. let r_bytes = unsafe { pack_eight_u8(r) }; + // SAFETY: packing only rearranges register values and does not dereference. let g_bytes = unsafe { pack_eight_u8(g) }; + // SAFETY: packing only rearranges register values and does not dereference. let b_bytes = unsafe { pack_eight_u8(b) }; for ((((r, g), b), pixel), _) in r_bytes diff --git a/crates/j2k-jpeg/src/batch_session.rs b/crates/j2k-jpeg/src/batch_session.rs new file mode 100644 index 00000000..7a403b72 --- /dev/null +++ b/crates/j2k-jpeg/src/batch_session.rs @@ -0,0 +1,534 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Persistent JPEG batch decode session. + +use alloc::vec::Vec; +use core::num::NonZeroUsize; +use j2k_core::{ + collect_indexed_batch_results, tile_batch_worker_count, CodecContext, IndexedBatchResult, + PixelFormat, ScratchPool as CoreScratchPool, TileBatchOptions, +}; +use rayon::prelude::*; +use std::sync::{Mutex, MutexGuard}; + +use crate::context::DecoderContext; +use crate::decoder::{ + decode_prepared_jpeg_tile_rgb8_in_context, decode_tile_into_in_context_with_options, + decode_tile_region_scaled_into_in_context_with_options, + decode_tile_scaled_into_in_context_with_options, DecodeOutcome, DecodedTile, + PreparedJpegTileJob, TileBatchError, TileDecodeJob, TileRegionScaledDecodeJob, + TileScaledDecodeJob, +}; +use crate::error::JpegError; +use crate::info::DecodeOptions; +use crate::internal::scratch::ScratchPool; + +const SMALL_OUTPUT_DEFAULT_WORKER_CAP: usize = 4; +const SMALL_OUTPUT_BYTES: usize = 32 * 1024; + +#[derive(Debug, Default)] +struct WorkerSlot { + ctx: DecoderContext, + pool: ScratchPool, +} + +impl WorkerSlot { + fn decode_tile_job_chunk( + &mut self, + start_index: usize, + jobs: &mut [TileDecodeJob<'_, '_>], + fmt: PixelFormat, + options: DecodeOptions, + ) -> Vec> { + let mut results = Vec::with_capacity(jobs.len()); + for (local_index, job) in jobs.iter_mut().enumerate() { + let outcome = decode_tile_into_in_context_with_options( + job.input, + &mut self.ctx, + &mut self.pool, + job.out, + job.stride, + fmt, + options, + ); + results.push((start_index + local_index, outcome)); + } + results + } + + fn decode_prepared_tile_job_chunk( + &mut self, + start_index: usize, + jobs: &mut [PreparedJpegTileJob<'_, '_>], + ) -> Vec> { + let mut results = Vec::with_capacity(jobs.len()); + for (local_index, job) in jobs.iter_mut().enumerate() { + let outcome = decode_prepared_jpeg_tile_rgb8_in_context( + &job.input, + &mut self.ctx, + &mut self.pool, + job.out, + job.stride, + job.options, + ); + results.push((start_index + local_index, outcome)); + } + results + } + + fn decode_tile_scaled_job_chunk( + &mut self, + start_index: usize, + jobs: &mut [TileScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + options: DecodeOptions, + ) -> Vec> { + let mut results = Vec::with_capacity(jobs.len()); + for (local_index, job) in jobs.iter_mut().enumerate() { + let outcome = decode_tile_scaled_into_in_context_with_options( + job.input, + &mut self.ctx, + &mut self.pool, + job.out, + job.stride, + fmt, + job.scale, + options, + ); + results.push((start_index + local_index, outcome)); + } + results + } + + fn decode_tile_region_scaled_job_chunk( + &mut self, + start_index: usize, + jobs: &mut [TileRegionScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + options: DecodeOptions, + ) -> Vec> { + let mut results = Vec::with_capacity(jobs.len()); + for (local_index, job) in jobs.iter_mut().enumerate() { + let outcome = decode_tile_region_scaled_into_in_context_with_options( + job.input, + &mut self.ctx, + &mut self.pool, + job.out, + job.stride, + fmt, + job.roi.into(), + job.scale, + options, + ); + results.push((start_index + local_index, outcome)); + } + results + } + + fn reset(&mut self) { + self.ctx.clear(); + self.pool.reset(); + } +} + +/// Reusable JPEG tile-batch runtime for WSI viewport loops. +/// +/// The session keeps one decoder context and scratch pool per active worker. +/// Reusing it across calls amortizes table/plan caches and heap scratch while +/// callers continue to own compressed inputs and decoded output buffers. +#[derive(Debug)] +pub struct JpegBatchSession { + options: TileBatchOptions, + workers: Vec>, + active_workers: usize, + cap_small_output_default_workers: bool, +} + +impl Default for JpegBatchSession { + fn default() -> Self { + Self::new(TileBatchOptions::default()) + } +} + +impl JpegBatchSession { + /// Create a session using the given CPU batch worker options. + #[must_use] + pub fn new(options: TileBatchOptions) -> Self { + Self { + options, + workers: Vec::new(), + active_workers: 0, + cap_small_output_default_workers: false, + } + } + + pub(crate) fn new_one_shot(options: TileBatchOptions) -> Self { + Self { + cap_small_output_default_workers: true, + ..Self::new(options) + } + } + + /// Return current worker options. + #[must_use] + pub fn options(&self) -> TileBatchOptions { + self.options + } + + /// Replace worker options for future decode calls. + pub fn set_options(&mut self, options: TileBatchOptions) { + self.options = options; + } + + /// Number of active workers used by the most recent non-empty decode call. + #[must_use] + pub fn worker_count(&self) -> usize { + self.active_workers + } + + /// Number of worker slots retained by the session. + #[must_use] + pub fn retained_worker_slots(&self) -> usize { + self.workers.len() + } + + /// Clear worker-local decode contexts and scratch pools while retaining slots. + pub fn reset(&mut self) { + for slot in &self.workers { + lock_worker(slot).reset(); + } + self.active_workers = 0; + } + + /// Decode full JPEG tiles into caller-owned output buffers. + /// + /// # Errors + /// Returns [`TileBatchError`] with the first failing tile index in input order. + pub fn decode_tiles_into( + &mut self, + jobs: &mut [TileDecodeJob<'_, '_>], + fmt: PixelFormat, + ) -> Result, TileBatchError> { + self.decode_tiles_into_with_options(jobs, fmt, DecodeOptions::default()) + } + + /// Decode full JPEG tiles with explicit JPEG decode options. + /// + /// # Errors + /// Returns [`TileBatchError`] with the first failing tile index in input order. + pub fn decode_tiles_into_with_options( + &mut self, + jobs: &mut [TileDecodeJob<'_, '_>], + fmt: PixelFormat, + decode_options: DecodeOptions, + ) -> Result, TileBatchError> { + if jobs.is_empty() { + return Ok(Vec::new()); + } + let job_count = jobs.len(); + let chunk_size = self.prepare_chunks(job_count, min_output_len(jobs)); + let decode_chunk = |worker: &mut WorkerSlot, start_index, chunk: &mut [_]| { + worker.decode_tile_job_chunk(start_index, chunk, fmt, decode_options) + }; + let results = if self.options.workers.is_some() { + self.run_chunks_scoped(jobs, chunk_size, decode_chunk) + } else { + self.run_chunks_rayon(jobs, chunk_size, decode_chunk) + }; + collect_results(job_count, results) + } + + /// Decode prepared TIFF/WSI JPEG tiles into caller-owned RGB8 buffers. + /// + /// Results preserve the caller's input order and retain each tile's error + /// independently instead of collapsing the batch to the first failure. + #[must_use] + pub fn decode_prepared_jpeg_tiles_rgb8( + &mut self, + jobs: &mut [PreparedJpegTileJob<'_, '_>], + ) -> Vec> { + if jobs.is_empty() { + return Vec::new(); + } + let job_count = jobs.len(); + let chunk_size = self.prepare_chunks(job_count, min_output_len(jobs)); + let decode_chunk = |worker: &mut WorkerSlot, start_index, chunk: &mut [_]| { + worker.decode_prepared_tile_job_chunk(start_index, chunk) + }; + let results = if self.options.workers.is_some() { + self.run_chunks_scoped(jobs, chunk_size, decode_chunk) + } else { + self.run_chunks_rayon(jobs, chunk_size, decode_chunk) + }; + collect_per_tile_results(job_count, results) + } + + /// Decode scaled JPEG tiles into caller-owned output buffers. + /// + /// # Errors + /// Returns [`TileBatchError`] with the first failing tile index in input order. + pub fn decode_tiles_scaled_into( + &mut self, + jobs: &mut [TileScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + ) -> Result, TileBatchError> { + self.decode_tiles_scaled_into_with_options(jobs, fmt, DecodeOptions::default()) + } + + /// Decode scaled JPEG tiles with explicit JPEG decode options. + /// + /// # Errors + /// Returns [`TileBatchError`] with the first failing tile index in input order. + pub fn decode_tiles_scaled_into_with_options( + &mut self, + jobs: &mut [TileScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + decode_options: DecodeOptions, + ) -> Result, TileBatchError> { + if jobs.is_empty() { + return Ok(Vec::new()); + } + let job_count = jobs.len(); + let chunk_size = self.prepare_chunks(job_count, min_output_len(jobs)); + let decode_chunk = |worker: &mut WorkerSlot, start_index, chunk: &mut [_]| { + worker.decode_tile_scaled_job_chunk(start_index, chunk, fmt, decode_options) + }; + let results = if self.options.workers.is_some() { + self.run_chunks_scoped(jobs, chunk_size, decode_chunk) + } else { + self.run_chunks_rayon(jobs, chunk_size, decode_chunk) + }; + collect_results(job_count, results) + } + + /// Decode scaled JPEG tile regions into caller-owned output buffers. + /// + /// # Errors + /// Returns [`TileBatchError`] with the first failing tile index in input order. + pub fn decode_tiles_region_scaled_into( + &mut self, + jobs: &mut [TileRegionScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + ) -> Result, TileBatchError> { + self.decode_tiles_region_scaled_into_with_options(jobs, fmt, DecodeOptions::default()) + } + + /// Decode scaled JPEG tile regions with explicit JPEG decode options. + /// + /// # Errors + /// Returns [`TileBatchError`] with the first failing tile index in input order. + pub fn decode_tiles_region_scaled_into_with_options( + &mut self, + jobs: &mut [TileRegionScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + decode_options: DecodeOptions, + ) -> Result, TileBatchError> { + if jobs.is_empty() { + return Ok(Vec::new()); + } + let job_count = jobs.len(); + let chunk_size = self.prepare_chunks(job_count, min_output_len(jobs)); + let decode_chunk = |worker: &mut WorkerSlot, start_index, chunk: &mut [_]| { + worker.decode_tile_region_scaled_job_chunk(start_index, chunk, fmt, decode_options) + }; + let results = if self.options.workers.is_some() { + self.run_chunks_scoped(jobs, chunk_size, decode_chunk) + } else { + self.run_chunks_rayon(jobs, chunk_size, decode_chunk) + }; + collect_results(job_count, results) + } + + fn prepare_chunks(&mut self, job_count: usize, min_output_len: usize) -> usize { + let mut worker_count = + tile_batch_worker_count(job_count, self.options, available_tile_batch_workers()); + let small_output_default_batch = self.cap_small_output_default_workers + && self.options.workers.is_none() + && min_output_len <= SMALL_OUTPUT_BYTES; + if small_output_default_batch { + worker_count = worker_count.min(SMALL_OUTPUT_DEFAULT_WORKER_CAP); + } + self.ensure_worker_slots(worker_count); + self.active_workers = worker_count; + job_count.div_ceil(worker_count) + } + + fn ensure_worker_slots(&mut self, worker_count: usize) { + while self.workers.len() < worker_count { + self.workers.push(Mutex::new(WorkerSlot::default())); + } + } + + fn run_chunks_rayon( + &self, + jobs: &mut [T], + chunk_size: usize, + decode_chunk: F, + ) -> Vec> + where + T: Send, + R: Send, + F: Fn(&mut WorkerSlot, usize, &mut [T]) -> Vec> + Sync, + { + jobs.par_chunks_mut(chunk_size) + .enumerate() + .flat_map_iter(|(chunk_index, chunk)| { + let start_index = chunk_index * chunk_size; + decode_chunk( + &mut lock_worker(&self.workers[chunk_index]), + start_index, + chunk, + ) + }) + .collect() + } + + fn run_chunks_scoped( + &self, + jobs: &mut [T], + chunk_size: usize, + decode_chunk: F, + ) -> Vec> + where + T: Send, + R: Send, + F: Fn(&mut WorkerSlot, usize, &mut [T]) -> Vec> + Sync, + { + if jobs.len() <= chunk_size { + return decode_chunk(&mut lock_worker(&self.workers[0]), 0, jobs); + } + + let decode_chunk = &decode_chunk; + std::thread::scope(|scope| { + let mut handles = Vec::new(); + for (chunk_index, chunk) in jobs.chunks_mut(chunk_size).enumerate() { + let start_index = chunk_index * chunk_size; + let worker = &self.workers[chunk_index]; + handles.push( + scope.spawn(move || decode_chunk(&mut lock_worker(worker), start_index, chunk)), + ); + } + collect_joined_batch_results(handles) + }) + } +} + +fn available_tile_batch_workers() -> usize { + std::thread::available_parallelism().map_or(1, NonZeroUsize::get) +} + +fn lock_worker(slot: &Mutex) -> MutexGuard<'_, WorkerSlot> { + slot.lock().expect("JPEG batch worker slot poisoned") +} + +trait BatchJobOutput { + fn out_len(&self) -> usize; +} + +impl BatchJobOutput for TileDecodeJob<'_, '_> { + fn out_len(&self) -> usize { + self.out.len() + } +} + +impl BatchJobOutput for PreparedJpegTileJob<'_, '_> { + fn out_len(&self) -> usize { + self.out.len() + } +} + +impl BatchJobOutput for TileScaledDecodeJob<'_, '_> { + fn out_len(&self) -> usize { + self.out.len() + } +} + +impl BatchJobOutput for TileRegionScaledDecodeJob<'_, '_> { + fn out_len(&self) -> usize { + self.out.len() + } +} + +fn min_output_len(jobs: &[T]) -> usize { + jobs.iter() + .map(BatchJobOutput::out_len) + .min() + .expect("non-empty batch has an output buffer") +} + +fn collect_results( + job_count: usize, + results: Vec>, +) -> Result, TileBatchError> { + collect_indexed_batch_results(job_count, results, |index, source| TileBatchError { + index, + source, + }) +} + +fn collect_per_tile_results( + job_count: usize, + results: Vec>, +) -> Vec> { + let mut ordered = core::iter::repeat_with(|| None) + .take(job_count) + .collect::>(); + for (index, result) in results { + ordered[index] = Some(result); + } + ordered + .into_iter() + .map(|slot| slot.expect("JPEG prepared batch worker omitted a tile result")) + .collect() +} + +fn collect_joined_batch_results( + handles: Vec>>>, +) -> Vec> { + let mut results = Vec::new(); + for handle in handles { + match handle.join() { + Ok(chunk_results) => results.extend(chunk_results), + Err(payload) => std::panic::resume_unwind(payload), + } + } + results +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::decoder::Decoder; + use j2k_test_support::JPEG_BASELINE_420_16X16; + + #[test] + fn one_shot_session_caps_default_workers_for_small_outputs() { + const JOBS: usize = 64; + let info = Decoder::inspect(JPEG_BASELINE_420_16X16).expect("fixture inspect"); + let stride = info.dimensions.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let len = stride * info.dimensions.1 as usize; + let mut outputs = (0..JOBS).map(|_| vec![0u8; len]).collect::>(); + let mut session = JpegBatchSession::new_one_shot(TileBatchOptions::default()); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: JPEG_BASELINE_420_16X16, + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb8) + .expect("one-shot session decode") + }; + + let available = available_tile_batch_workers(); + assert_eq!(outcomes.len(), JOBS); + assert_eq!( + session.worker_count(), + available.min(SMALL_OUTPUT_DEFAULT_WORKER_CAP).min(JOBS) + ); + } +} diff --git a/crates/signinum-jpeg/src/bench_support.rs b/crates/j2k-jpeg/src/bench_support.rs similarity index 97% rename from crates/signinum-jpeg/src/bench_support.rs rename to crates/j2k-jpeg/src/bench_support.rs index e88d0abb..8a0b8921 100644 --- a/crates/signinum-jpeg/src/bench_support.rs +++ b/crates/j2k-jpeg/src/bench_support.rs @@ -2,6 +2,7 @@ //! Hidden helpers used by Criterion benches. +use crate::backend::scalar; use crate::backend::Backend; use crate::color::upsample::upsample_h2v2_fancy_rows; use crate::color::ycbcr::ycbcr_to_rgb; @@ -22,14 +23,6 @@ use core::cell::Cell; use core::ptr; use std::time::Instant; -// `crate::backend::scalar` is intentionally private. Reuse the production -// source file here so bench/test helpers call the real scalar row-pair kernel -// without carrying a second handwritten copy of the algorithm. -#[allow(dead_code)] -#[allow(clippy::duplicate_mod)] -#[path = "backend/scalar.rs"] -mod bench_scalar_backend; - #[doc(hidden)] #[derive(Default, Debug, Clone)] pub struct Bench420DispatchStats { @@ -192,6 +185,7 @@ pub(crate) fn record_420_dispatch_scalar_chunk() { BENCH_420_DISPATCH_STATS.with(|slot| { let stats = slot.get(); if !stats.is_null() { + // SAFETY: Benchmark helpers preserve production buffer sizing and backend feature checks. unsafe { (*stats).record_scalar_chunk(); } @@ -204,6 +198,7 @@ pub(crate) fn record_420_dispatch_neon_tail_chunk() { BENCH_420_DISPATCH_STATS.with(|slot| { let stats = slot.get(); if !stats.is_null() { + // SAFETY: Benchmark helpers preserve production buffer sizing and backend feature checks. unsafe { (*stats).record_neon_tail_chunk(); } @@ -404,6 +399,7 @@ pub fn bench_idct_reduced_2x2_block_with(input: &[i16; 64], output: &mut [u8; 4] #[cfg(target_arch = "aarch64")] #[doc(hidden)] pub fn bench_idct_neon_block(input: &[i16; 64], output: &mut [u8; 64]) { + // SAFETY: Benchmark helpers preserve production buffer sizing and backend feature checks. unsafe { crate::idct::neon::idct_islow(input, output) }; } @@ -412,6 +408,7 @@ pub fn bench_idct_neon_block(input: &[i16; 64], output: &mut [u8; 64]) { #[cfg(target_arch = "x86_64")] #[doc(hidden)] pub fn bench_idct_avx2_block(input: &[i16; 64], output: &mut [u8; 64]) { + // SAFETY: Benchmark helpers preserve production buffer sizing and backend feature checks. unsafe { crate::idct::avx2::idct_islow(input, output) }; } @@ -548,7 +545,7 @@ pub fn bench_rgb_row_pair_from_420_reference( dst_top: &mut [u8], dst_bottom: Option<&mut [u8]>, ) { - bench_scalar_backend::fill_rgb_row_pair_from_420( + scalar::fill_rgb_row_pair_from_420( y_top, y_bottom, prev_cb, curr_cb, next_cb, prev_cr, curr_cr, next_cr, dst_top, dst_bottom, ); } diff --git a/crates/j2k-jpeg/src/capabilities.rs b/crates/j2k-jpeg/src/capabilities.rs new file mode 100644 index 00000000..b8815302 --- /dev/null +++ b/crates/j2k-jpeg/src/capabilities.rs @@ -0,0 +1,649 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Public JPEG capability introspection for backend routing. + +use crate::adapter::{summarize_device_batch, DeviceBatchSummary}; +use crate::decoder::{Decoder, JpegView}; +use crate::error::JpegError; +use crate::info::{ColorSpace, Info, Rect, SofKind}; +use j2k_core::{BackendRequest, Downscale, PixelFormat}; + +/// JPEG decode operation shape for capability routing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JpegDecodeOp { + /// Full-image/tile decode. + Full, + /// Source-coordinate region decode. + Region(Rect), + /// Full-image/tile decode at reduced resolution. + Scaled(Downscale), + /// Source-coordinate region decode at reduced resolution. + RegionScaled { + /// Source-coordinate region. + roi: Rect, + /// Reduced-resolution factor. + scale: Downscale, + }, +} + +impl JpegDecodeOp { + fn scale(self) -> Downscale { + match self { + Self::Full | Self::Region(_) => Downscale::None, + Self::Scaled(scale) | Self::RegionScaled { scale, .. } => scale, + } + } +} + +/// Capability request for a JPEG decode route. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JpegCapabilityRequest { + /// Decode operation shape. + pub op: JpegDecodeOp, + /// Requested output pixel format. + pub fmt: PixelFormat, +} + +/// Complete JPEG decode request used by backend path resolution. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JpegDecodeRequest { + /// Requested backend policy. + pub backend: BackendRequest, + /// Requested output pixel format. + pub fmt: PixelFormat, + /// Decode operation shape. + pub op: JpegDecodeOp, +} + +impl JpegDecodeRequest { + /// Return the capability-only portion of the request. + #[must_use] + pub const fn capability(self) -> JpegCapabilityRequest { + JpegCapabilityRequest { + op: self.op, + fmt: self.fmt, + } + } +} + +/// Normalized JPEG decode path selected for a request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JpegResolvedDecodePath { + /// Portable CPU host decode. + CpuHost, + /// J2K-owned CUDA RGB8 decode path. + OwnedCudaRgb8, + /// J2K Metal fast-packet decode path. + MetalFast, + /// Request cannot be satisfied by this path resolver. + Rejected { + /// Backend requested by the caller. + backend: BackendRequest, + /// Stable rejection reason. + reason: &'static str, + }, +} + +/// Parsed JPEG metadata plus the selected backend path. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JpegResolvedDecode { + /// Original decode request. + pub request: JpegDecodeRequest, + /// Capability report used for the decision. + pub capabilities: JpegCapabilityReport, + /// Output rectangle after ROI and scale are applied. + pub output_rect: Rect, + /// Selected backend path. + pub path: JpegResolvedDecodePath, +} + +impl JpegResolvedDecode { + /// Inspect JPEG bytes and resolve the requested backend path. + pub fn inspect(input: &[u8], request: JpegDecodeRequest) -> Result { + let capabilities = JpegCapabilityReport::inspect(input, request.capability())?; + Ok(Self::from_capabilities(capabilities, request)) + } + + /// Resolve a path from an existing capability report. + #[must_use] + pub fn from_capabilities( + capabilities: JpegCapabilityReport, + request: JpegDecodeRequest, + ) -> Self { + let output_rect = output_rect_for_request(&capabilities.info, request.op); + let path = capabilities.resolve_path(request.backend); + Self { + request, + capabilities, + output_rect, + path, + } + } +} + +/// Backend eligibility result with a stable rejection reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JpegBackendEligibility { + /// Whether this backend can handle the requested decode shape. + pub eligible: bool, + /// Static rejection reason when `eligible` is false. + pub reason: Option<&'static str>, +} + +impl JpegBackendEligibility { + const fn eligible() -> Self { + Self { + eligible: true, + reason: None, + } + } + + const fn rejected(reason: &'static str) -> Self { + Self { + eligible: false, + reason: Some(reason), + } + } +} + +/// Parsed JPEG metadata and backend eligibility for one request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JpegCapabilityReport { + /// Original capability request. + pub request: JpegCapabilityRequest, + /// Public JPEG metadata. + pub info: Info, + /// Device batch summary derived from J2K's parser/planner. + pub device: DeviceBatchSummary, + /// Portable CPU decode eligibility. + pub cpu: JpegBackendEligibility, + /// J2K-owned CUDA-kernel eligibility. + pub owned_cuda: JpegBackendEligibility, + /// Metal fast-packet shape eligibility. + pub metal_fast: JpegBackendEligibility, +} + +impl JpegCapabilityReport { + /// Inspect JPEG bytes and report decode-route eligibility. + /// + /// # Errors + /// Returns [`JpegError`] when JPEG header parsing fails or planner + /// validation finds malformed decode-table state. Parseable JPEG classes + /// that J2K has not implemented yet still return a report with + /// rejected backend eligibility. + pub fn inspect(input: &[u8], request: JpegCapabilityRequest) -> Result { + let view = JpegView::parse(input)?; + let info = view.info().clone(); + let has_lossless_subsampled_color_capability_shape = + view.has_lossless_subsampled_color_capability_shape(); + match Decoder::from_view(view) { + Ok(decoder) => Ok(Self::for_decoder(&decoder, request)), + Err(err) + if can_report_from_parsed_info( + &err, + has_lossless_subsampled_color_capability_shape, + ) => + { + Ok(Self::for_planner_rejected_info(info, request, &err)) + } + Err(err) => Err(err), + } + } + + /// Build a capability report from an already parsed decoder. + #[must_use] + pub fn for_decoder(decoder: &Decoder<'_>, request: JpegCapabilityRequest) -> Self { + let info = decoder.info().clone(); + let device = summarize_device_batch(decoder, 4); + Self { + request, + info: info.clone(), + device, + cpu: cpu_eligibility(&info, request), + owned_cuda: owned_cuda_eligibility(&info, device, request), + metal_fast: metal_fast_eligibility(&info, device, request), + } + } + + fn for_parsed_info(info: Info, request: JpegCapabilityRequest) -> Self { + let device = unavailable_device_summary(&info); + Self { + request, + info: info.clone(), + device, + cpu: cpu_eligibility(&info, request), + owned_cuda: owned_cuda_eligibility(&info, device, request), + metal_fast: metal_fast_eligibility(&info, device, request), + } + } + + fn for_planner_rejected_info( + info: Info, + request: JpegCapabilityRequest, + err: &JpegError, + ) -> Self { + let mut report = Self::for_parsed_info(info, request); + if report.cpu.eligible && matches!(err, JpegError::NotImplemented { .. }) { + report.cpu = JpegBackendEligibility::rejected( + "JPEG CPU decode planner rejected this stream shape before decode", + ); + } + report + } + + /// Eligibility for explicit reusable RGB8 Metal batch outputs. + /// + /// This is narrower than [`Self::metal_fast`]: it describes the current + /// caller-owned Metal buffer/texture batch APIs, not every Metal-capable + /// surface decode shape. + #[must_use] + pub fn metal_resident_rgb8_batch_output(&self) -> JpegBackendEligibility { + metal_resident_rgb8_batch_output_eligibility(self.device, self.request) + } + + /// Resolve a backend request using this report's eligibility results. + #[must_use] + pub fn resolve_path(&self, backend: BackendRequest) -> JpegResolvedDecodePath { + match backend { + BackendRequest::Cpu => { + if self.cpu.eligible { + JpegResolvedDecodePath::CpuHost + } else { + JpegResolvedDecodePath::Rejected { + backend, + reason: self + .cpu + .reason + .unwrap_or("JPEG CPU decode rejected this request"), + } + } + } + BackendRequest::Auto => JpegResolvedDecodePath::CpuHost, + BackendRequest::Cuda => { + if self.owned_cuda.eligible { + JpegResolvedDecodePath::OwnedCudaRgb8 + } else { + JpegResolvedDecodePath::Rejected { + backend, + reason: self + .owned_cuda + .reason + .unwrap_or("J2K-owned CUDA JPEG decode rejected this request"), + } + } + } + BackendRequest::Metal => { + if self.metal_fast.eligible { + JpegResolvedDecodePath::MetalFast + } else { + JpegResolvedDecodePath::Rejected { + backend, + reason: self + .metal_fast + .reason + .unwrap_or("JPEG Metal fast path rejected this request"), + } + } + } + } + } +} + +fn output_rect_for_request(info: &Info, op: JpegDecodeOp) -> Rect { + match op { + JpegDecodeOp::Full => Rect::full(info.dimensions), + JpegDecodeOp::Region(roi) => roi, + JpegDecodeOp::Scaled(scale) => scaled_rect(Rect::full(info.dimensions), scale), + JpegDecodeOp::RegionScaled { roi, scale } => scaled_rect(roi, scale), + } +} + +fn scaled_rect(rect: Rect, scale: Downscale) -> Rect { + let denom = scale.denominator(); + let x_end = rect.x.saturating_add(rect.w); + let y_end = rect.y.saturating_add(rect.h); + let x0 = rect.x / denom; + let y0 = rect.y / denom; + let x1 = x_end.div_ceil(denom); + let y1 = y_end.div_ceil(denom); + Rect { + x: x0, + y: y0, + w: x1.saturating_sub(x0), + h: y1.saturating_sub(y0), + } +} + +fn cpu_eligibility(info: &Info, request: JpegCapabilityRequest) -> JpegBackendEligibility { + match info.sof_kind { + SofKind::Extended12 if is_twelve_bit_output_request(request.fmt) => { + return twelve_bit_eligibility(info, request.fmt, TwelveBitSof::Extended); + } + SofKind::Progressive12 if is_twelve_bit_output_request(request.fmt) => { + return twelve_bit_eligibility(info, request.fmt, TwelveBitSof::Progressive); + } + SofKind::Extended12 | SofKind::Progressive12 => { + return JpegBackendEligibility::rejected( + "JPEG CPU decode does not yet support this 12-bit JPEG output", + ) + } + SofKind::Lossless + if matches!( + request.fmt, + PixelFormat::Gray8 + | PixelFormat::Gray16 + | PixelFormat::Rgb8 + | PixelFormat::Rgba8 + | PixelFormat::Rgb16 + | PixelFormat::Rgba16 + ) => + { + return match (info.color_space, info.bit_depth, request.fmt) { + (ColorSpace::Grayscale, 8, PixelFormat::Gray8) + | (ColorSpace::Grayscale, 16, PixelFormat::Gray16) => { + JpegBackendEligibility::eligible() + } + ( + ColorSpace::Rgb | ColorSpace::YCbCr, + 8, + PixelFormat::Rgb8 | PixelFormat::Rgba8, + ) + | ( + ColorSpace::Rgb | ColorSpace::YCbCr, + 16, + PixelFormat::Rgb16 | PixelFormat::Rgba16, + ) + if is_supported_lossless_color_sampling(info) => + { + JpegBackendEligibility::eligible() + } + (ColorSpace::Rgb, 8, PixelFormat::Rgb8 | PixelFormat::Rgba8) + | (ColorSpace::Rgb, 16, PixelFormat::Rgb16 | PixelFormat::Rgba16) => JpegBackendEligibility::rejected( + "JPEG CPU lossless SOF3 APP14 RGB decode currently supports 4:4:4 sampling, even-width 8/16-bit 4:2:2 sampling, or even-dimension 8/16-bit 4:2:0 sampling", + ), + (ColorSpace::YCbCr, 8, PixelFormat::Rgb8 | PixelFormat::Rgba8) + | (ColorSpace::YCbCr, 16, PixelFormat::Rgb16 | PixelFormat::Rgba16) => JpegBackendEligibility::rejected( + "JPEG CPU lossless SOF3 YCbCr decode currently supports 4:4:4 sampling, even-width 8/16-bit 4:2:2 sampling, or even-dimension 8/16-bit 4:2:0 sampling", + ), + _ => JpegBackendEligibility::rejected( + "JPEG CPU lossless SOF3 decode currently supports 8-bit Gray8, 16-bit Gray16, 8-bit YCbCr Rgb8/Rgba8 including even-width 4:2:2 and even-dimension 4:2:0, 16-bit YCbCr Rgb16/Rgba16 including even-width 4:2:2 and even-dimension 4:2:0, 8-bit APP14 RGB Rgb8/Rgba8 including even-width 4:2:2 and even-dimension 4:2:0, or 16-bit APP14 RGB Rgb16/Rgba16 including even-width 4:2:2 and even-dimension 4:2:0 output only", + ), + }; + } + SofKind::Lossless => { + return JpegBackendEligibility::rejected( + "JPEG CPU decode does not yet support lossless SOF3 JPEG", + ) + } + SofKind::Baseline8 | SofKind::Extended8 | SofKind::Progressive8 => {} + } + + match (request.fmt, request.op.scale()) { + (PixelFormat::Rgb8 | PixelFormat::Gray8, _) => JpegBackendEligibility::eligible(), + (PixelFormat::Rgba8, _) => JpegBackendEligibility::eligible(), + (PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16, _) => { + JpegBackendEligibility::rejected("JPEG CPU decode does not support 16-bit output") + } + _ => JpegBackendEligibility::rejected("unsupported JPEG CPU output format"), + } +} + +#[derive(Debug, Clone, Copy)] +enum TwelveBitSof { + Extended, + Progressive, +} + +impl TwelveBitSof { + const fn ycbcr_sampling_reason(self) -> &'static str { + match self { + Self::Extended => { + "JPEG CPU 12-bit extended YCbCr decode currently supports 4:4:4, 4:2:2, or 4:2:0 sampling only" + } + Self::Progressive => { + "JPEG CPU 12-bit progressive YCbCr decode currently supports 4:4:4, 4:2:2, or 4:2:0 sampling only" + } + } + } + + const fn rgb_sampling_reason(self) -> &'static str { + match self { + Self::Extended => { + "JPEG CPU 12-bit extended RGB decode currently supports 4:4:4, 4:2:2, or 4:2:0 sampling only" + } + Self::Progressive => { + "JPEG CPU 12-bit progressive RGB decode currently supports 4:4:4, 4:2:2, or 4:2:0 sampling only" + } + } + } + + const fn four_component_sampling_reason(self) -> &'static str { + match self { + Self::Extended => { + "JPEG CPU 12-bit extended four-component CMYK/YCCK decode currently supports 4:4:4, 4:2:2, or 4:2:0 sampling only" + } + Self::Progressive => { + "JPEG CPU 12-bit progressive four-component CMYK/YCCK decode currently supports 4:4:4, 4:2:2, or 4:2:0 sampling only" + } + } + } + + const fn output_reason(self) -> &'static str { + match self { + Self::Extended => { + "JPEG CPU 12-bit extended decode currently supports grayscale Gray16/Rgb16/Rgba16, APP14 RGB 4:4:4/4:2:2/4:2:0 Rgb16/Rgba16, YCbCr 4:4:4/4:2:2/4:2:0 Rgb16/Rgba16, or CMYK/YCCK 4:4:4/4:2:2/4:2:0 Rgb16/Rgba16 only" + } + Self::Progressive => { + "JPEG CPU 12-bit progressive decode currently supports grayscale Gray16/Rgb16/Rgba16, APP14 RGB 4:4:4/4:2:2/4:2:0 Rgb16/Rgba16, YCbCr 4:4:4/4:2:2/4:2:0 Rgb16/Rgba16, or CMYK/YCCK 4:4:4/4:2:2/4:2:0 Rgb16/Rgba16 only" + } + } + } +} + +fn is_twelve_bit_output_request(fmt: PixelFormat) -> bool { + matches!( + fmt, + PixelFormat::Gray16 | PixelFormat::Rgb16 | PixelFormat::Rgba16 + ) +} + +fn twelve_bit_eligibility( + info: &Info, + fmt: PixelFormat, + sof: TwelveBitSof, +) -> JpegBackendEligibility { + match (info.color_space, fmt) { + (ColorSpace::Grayscale, PixelFormat::Gray16 | PixelFormat::Rgb16 | PixelFormat::Rgba16) => { + JpegBackendEligibility::eligible() + } + (ColorSpace::Rgb | ColorSpace::YCbCr, PixelFormat::Rgb16 | PixelFormat::Rgba16) + if is_supported_12bit_three_component_sampling(info) => + { + JpegBackendEligibility::eligible() + } + (ColorSpace::Cmyk | ColorSpace::Ycck, PixelFormat::Rgb16 | PixelFormat::Rgba16) + if is_supported_extended12_four_component_sampling(info) => + { + JpegBackendEligibility::eligible() + } + (ColorSpace::YCbCr, PixelFormat::Rgb16 | PixelFormat::Rgba16) => { + JpegBackendEligibility::rejected(sof.ycbcr_sampling_reason()) + } + (ColorSpace::Rgb, PixelFormat::Rgb16 | PixelFormat::Rgba16) => { + JpegBackendEligibility::rejected(sof.rgb_sampling_reason()) + } + (ColorSpace::Cmyk | ColorSpace::Ycck, PixelFormat::Rgb16 | PixelFormat::Rgba16) => { + JpegBackendEligibility::rejected(sof.four_component_sampling_reason()) + } + _ => JpegBackendEligibility::rejected(sof.output_reason()), + } +} + +fn is_supported_extended12_four_component_sampling(info: &Info) -> bool { + info.sampling.len() == 4 + && matches!( + ( + info.sampling.max_h, + info.sampling.max_v, + info.sampling.components() + ), + (1, 1, [(1, 1), (1, 1), (1, 1), (1, 1)]) + | (2, 1, [(2, 1), (1, 1), (1, 1), (1, 1)]) + | (2, 2, [(2, 2), (1, 1), (1, 1), (1, 1)]) + ) +} + +fn is_supported_12bit_three_component_sampling(info: &Info) -> bool { + info.sampling.len() == 3 + && matches!( + ( + info.sampling.max_h, + info.sampling.max_v, + info.sampling.components() + ), + (1, 1, [(1, 1), (1, 1), (1, 1)]) + | (2, 1, [(2, 1), (1, 1), (1, 1)]) + | (2, 2, [(2, 2), (1, 1), (1, 1)]) + ) +} + +fn is_supported_lossless_color_sampling(info: &Info) -> bool { + info.sampling.len() == 3 + && matches!( + ( + info.bit_depth, + info.dimensions.0.is_multiple_of(2), + info.dimensions.1.is_multiple_of(2), + info.sampling.max_h, + info.sampling.max_v, + info.sampling.components() + ), + (_, _, _, 1, 1, [(1, 1), (1, 1), (1, 1)]) + | (8 | 16, true, _, 2, 1, [(2, 1), (1, 1), (1, 1)]) + | (8 | 16, true, true, 2, 2, [(2, 2), (1, 1), (1, 1)]) + ) +} + +fn owned_cuda_eligibility( + info: &Info, + device: DeviceBatchSummary, + request: JpegCapabilityRequest, +) -> JpegBackendEligibility { + if request.op != JpegDecodeOp::Full || request.fmt != PixelFormat::Rgb8 { + return JpegBackendEligibility::rejected( + "J2K-owned CUDA JPEG decode currently supports full-tile RGB8 fast 4:2:0, 4:2:2, or 4:4:4 only", + ); + } + if !matches!(info.sof_kind, SofKind::Baseline8 | SofKind::Extended8) { + return JpegBackendEligibility::rejected( + "J2K-owned CUDA JPEG decode supports baseline/extended 8-bit sequential JPEG only", + ); + } + if info.color_space != ColorSpace::YCbCr + || !(device.matches_fast_420 || device.matches_fast_422 || device.matches_fast_444) + { + return JpegBackendEligibility::rejected( + "J2K-owned CUDA JPEG decode currently requires a YCbCr 4:2:0, 4:2:2, or 4:4:4 fast packet shape", + ); + } + JpegBackendEligibility::eligible() +} + +fn metal_fast_eligibility( + info: &Info, + device: DeviceBatchSummary, + request: JpegCapabilityRequest, +) -> JpegBackendEligibility { + if !matches!( + request.fmt, + PixelFormat::Gray8 | PixelFormat::Rgb8 | PixelFormat::Rgba8 + ) { + return JpegBackendEligibility::rejected( + "JPEG Metal fast path supports Gray8, Rgb8, or Rgba8 output formats", + ); + } + if !matches!(info.sof_kind, SofKind::Baseline8 | SofKind::Extended8) { + return JpegBackendEligibility::rejected( + "JPEG Metal fast path currently supports baseline/extended 8-bit sequential JPEG only", + ); + } + if !matches!( + info.color_space, + ColorSpace::Grayscale | ColorSpace::YCbCr | ColorSpace::Rgb + ) { + return JpegBackendEligibility::rejected( + "JPEG Metal fast path requires grayscale, YCbCr, or RGB input color", + ); + } + if device.matches_fast_420 || device.matches_fast_422 || device.matches_fast_444 { + JpegBackendEligibility::eligible() + } else { + JpegBackendEligibility::rejected( + "JPEG Metal fast path requires a fast 4:2:0, 4:2:2, or 4:4:4 packet shape", + ) + } +} + +fn metal_resident_rgb8_batch_output_eligibility( + device: DeviceBatchSummary, + request: JpegCapabilityRequest, +) -> JpegBackendEligibility { + if request.fmt != PixelFormat::Rgb8 { + return JpegBackendEligibility::rejected( + "JPEG Metal reusable resident batch output currently supports RGB8 output only", + ); + } + if !(device.matches_fast_420 || device.matches_fast_422 || device.matches_fast_444) { + return JpegBackendEligibility::rejected( + "JPEG Metal reusable resident batch output requires a fast 4:2:0, 4:2:2, or 4:4:4 packet shape", + ); + } + + match request.op { + JpegDecodeOp::Full => JpegBackendEligibility::eligible(), + JpegDecodeOp::Scaled(scale) | JpegDecodeOp::RegionScaled { scale, .. } + if supports_metal_resident_batch_scale(scale) => + { + JpegBackendEligibility::eligible() + } + JpegDecodeOp::Scaled(_) | JpegDecodeOp::RegionScaled { .. } => { + JpegBackendEligibility::rejected( + "JPEG Metal reusable resident batch output currently supports half, quarter, or eighth scaling", + ) + } + JpegDecodeOp::Region(_) => JpegBackendEligibility::rejected( + "JPEG Metal reusable resident batch output currently supports full, scaled, or region-scaled decode shapes", + ), + } +} + +fn supports_metal_resident_batch_scale(scale: Downscale) -> bool { + matches!( + scale, + Downscale::Half | Downscale::Quarter | Downscale::Eighth + ) +} + +fn can_report_from_parsed_info( + err: &JpegError, + has_lossless_subsampled_color_capability_shape: bool, +) -> bool { + match err { + JpegError::UnsupportedColorSpace { .. } => true, + JpegError::NotImplemented { sof } if *sof != SofKind::Lossless => true, + JpegError::NotImplemented { + sof: SofKind::Lossless, + } => has_lossless_subsampled_color_capability_shape, + _ => false, + } +} + +fn unavailable_device_summary(info: &Info) -> DeviceBatchSummary { + DeviceBatchSummary { + restart_interval: info.restart_interval, + checkpoint_count: 0, + matches_fast_420: false, + matches_fast_422: false, + matches_fast_444: false, + } +} diff --git a/crates/j2k-jpeg/src/color/cmyk.rs b/crates/j2k-jpeg/src/color/cmyk.rs new file mode 100644 index 00000000..de6a3e72 --- /dev/null +++ b/crates/j2k-jpeg/src/color/cmyk.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Adobe-style CMYK/YCCK to RGB conversion. + +use crate::color::ycbcr::ycbcr12_to_rgb16; +use crate::color::ycbcr::ycbcr_to_rgb; + +fn multiply_u8(a: u8, b: u8) -> u8 { + ((u16::from(a) * u16::from(b) + 127) / 255) as u8 +} + +fn multiply_u12(a: u16, b: u16) -> u16 { + ((u32::from(a) * u32::from(b) + 2047) / 4095) as u16 +} + +/// Convert complemented Adobe CMYK samples to RGB. +/// +/// Adobe CMYK JPEG stores samples in the complemented domain, so each channel +/// acts like the amount of display light remaining after ink coverage. +pub(crate) fn inverted_cmyk_to_rgb(c: u8, m: u8, y: u8, k: u8) -> (u8, u8, u8) { + (multiply_u8(c, k), multiply_u8(m, k), multiply_u8(y, k)) +} + +/// Convert 12-bit complemented Adobe CMYK samples to native-range RGB16. +pub(crate) fn inverted_cmyk12_to_rgb16(c: u16, m: u16, y: u16, k: u16) -> (u16, u16, u16) { + (multiply_u12(c, k), multiply_u12(m, k), multiply_u12(y, k)) +} + +/// Convert Adobe YCCK samples directly to RGB. +pub(crate) fn ycck_to_rgb(y: u8, cb: u8, cr: u8, k: u8) -> (u8, u8, u8) { + let (c, m, y) = ycbcr_to_rgb(y, cb, cr); + inverted_cmyk_to_rgb(c, m, y, k) +} + +/// Convert 12-bit Adobe YCCK samples directly to native-range RGB16. +pub(crate) fn ycck12_to_rgb16(y: u16, cb: u16, cr: u16, k: u16) -> (u16, u16, u16) { + let (c, m, y) = ycbcr12_to_rgb16(y, cb, cr); + inverted_cmyk12_to_rgb16(c, m, y, k) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn inverted_cmyk_multiplies_color_channels_by_black_channel() { + assert_eq!(inverted_cmyk_to_rgb(255, 255, 255, 255), (255, 255, 255)); + assert_eq!(inverted_cmyk_to_rgb(255, 0, 0, 255), (255, 0, 0)); + assert_eq!(inverted_cmyk_to_rgb(128, 128, 128, 128), (64, 64, 64)); + } + + #[test] + fn ycck_neutral_samples_match_inverted_cmyk_neutral() { + assert_eq!(ycck_to_rgb(128, 128, 128, 128), (64, 64, 64)); + } + + #[test] + fn cmyk12_and_ycck12_neutral_samples_use_native_12_bit_range() { + assert_eq!( + inverted_cmyk12_to_rgb16(2048, 2048, 2048, 2048), + (1024, 1024, 1024) + ); + assert_eq!(ycck12_to_rgb16(2048, 2048, 2048, 2048), (1024, 1024, 1024)); + } +} diff --git a/crates/signinum-jpeg/src/color/mod.rs b/crates/j2k-jpeg/src/color/mod.rs similarity index 87% rename from crates/signinum-jpeg/src/color/mod.rs rename to crates/j2k-jpeg/src/color/mod.rs index c833a1d6..53dcb1a2 100644 --- a/crates/signinum-jpeg/src/color/mod.rs +++ b/crates/j2k-jpeg/src/color/mod.rs @@ -2,5 +2,6 @@ //! Color conversion + chroma upsampling. Scalar-only in M1b. +pub(crate) mod cmyk; pub(crate) mod upsample; pub(crate) mod ycbcr; diff --git a/crates/signinum-jpeg/src/color/upsample.rs b/crates/j2k-jpeg/src/color/upsample.rs similarity index 84% rename from crates/signinum-jpeg/src/color/upsample.rs rename to crates/j2k-jpeg/src/color/upsample.rs index 4edd6a46..c41c59b8 100644 --- a/crates/signinum-jpeg/src/color/upsample.rs +++ b/crates/j2k-jpeg/src/color/upsample.rs @@ -143,32 +143,62 @@ pub(crate) fn upsample_h2v2_fancy_row( /// below (for the bottom output) the current chroma row. fn emit_h2v2_row(near: &[u8], curr: &[u8], output_width: usize, out: &mut [u8]) { let n = curr.len(); + for (x, slot) in out.iter_mut().enumerate().take(output_width) { + *slot = h2v2_fancy_sample_with_len(near, curr, n, n * 2, x); + } +} + +#[inline] +pub(crate) fn h2v2_fancy_sample(near: &[u8], curr: &[u8], x: usize) -> u8 { + debug_assert_eq!(near.len(), curr.len()); + h2v2_fancy_sample_with_len(near, curr, curr.len(), curr.len() * 2, x) +} + +// Only the NEON row filler samples by explicit output width; the scalar and +// x86 paths go through h2v2_fancy_sample. +#[cfg(target_arch = "aarch64")] +#[inline] +pub(crate) fn h2v2_fancy_sample_for_width( + near: &[u8], + curr: &[u8], + output_width: usize, + x: usize, +) -> u8 { + debug_assert_eq!(near.len(), curr.len()); + h2v2_fancy_sample_with_len(near, curr, curr.len(), output_width, x) +} + +#[inline] +fn h2v2_fancy_sample_with_len( + near: &[u8], + curr: &[u8], + n: usize, + output_width: usize, + x: usize, +) -> u8 { + if n == 0 { + return 0; + } + let sample = (x / 2).min(n - 1); // Column sums: `colsum[i] = 3 * curr[i] + near[i]`. libjpeg-turbo streams // these as `this/next/last` without materializing the whole array. - let colsum = |i: usize| 3 * curr[i] as u32 + near[i] as u32; - + let colsum = |i: usize| 3 * u32::from(curr[i]) + u32::from(near[i]); if n == 1 { - // Degenerate edge: just replicate. - let v = (4 * colsum(0) + 8) >> 4; - out[..output_width].fill(v as u8); - return; + return ((4 * colsum(0) + 8) >> 4) as u8; } - for (x, slot) in out.iter_mut().enumerate().take(output_width) { - let sample = x / 2; - let this = colsum(sample); - *slot = match x { - 0 => ((this * 4 + 8) >> 4) as u8, - _ if x == n * 2 - 1 => ((this * 4 + 7) >> 4) as u8, - _ if x.is_multiple_of(2) => { - let last = colsum(sample - 1); - ((this * 3 + last + 8) >> 4) as u8 - } - _ => { - let next = colsum(sample + 1); - ((this * 3 + next + 7) >> 4) as u8 - } - }; + let this = colsum(sample); + match x { + 0 => ((this * 4 + 8) >> 4) as u8, + _ if x == output_width - 1 => ((this * 4 + 7) >> 4) as u8, + _ if x.is_multiple_of(2) => { + let last = colsum(sample - 1); + ((this * 3 + last + 8) >> 4) as u8 + } + _ => { + let next = colsum(sample + 1); + ((this * 3 + next + 7) >> 4) as u8 + } } } diff --git a/crates/j2k-jpeg/src/color/ycbcr.rs b/crates/j2k-jpeg/src/color/ycbcr.rs new file mode 100644 index 00000000..f8954677 --- /dev/null +++ b/crates/j2k-jpeg/src/color/ycbcr.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! YCbCr → RGB conversion. Scalar implementation uses libjpeg-turbo's 16-bit +//! fixed-point coefficients (`jdcolor.c`), so outputs match their ISLOW path +//! byte-for-byte. +//! +//! Coefficients (all × 2^16, rounded): +//! R = Y + 1.40200 * (Cr - 128) +//! G = Y - 0.34414 * (Cb - 128) - 0.71414 * (Cr - 128) +//! B = Y + 1.77200 * (Cb - 128) + +pub(crate) const FIX_1_40200: i32 = 91_881; // (int)(1.40200 * 65536 + 0.5) +pub(crate) const FIX_0_34414: i32 = 22_554; // (int)(0.34414 * 65536 + 0.5) +pub(crate) const FIX_0_71414: i32 = 46_802; // (int)(0.71414 * 65536 + 0.5) +pub(crate) const FIX_1_77200: i32 = 116_130; // (int)(1.77200 * 65536 + 0.5) +pub(crate) const ROUND: i32 = 1 << 15; // 0.5 in 16-bit fixed point + +const fn clamp_to_u8(v: i32) -> u8 { + if v < 0 { + 0 + } else if v > 255 { + 255 + } else { + v as u8 + } +} + +const fn clamp_to_12bit(v: i32) -> u16 { + if v < 0 { + 0 + } else if v > 4095 { + 4095 + } else { + v as u16 + } +} + +const fn clamp_to_u16(v: i64) -> u16 { + if v < 0 { + 0 + } else if v > u16::MAX as i64 { + u16::MAX + } else { + v as u16 + } +} + +/// Convert one YCbCr pixel to RGB. `y`, `cb`, `cr` are the 8-bit component +/// values as read from the decoded block after IDCT and upsample. +/// +/// Returns `(R, G, B)` clamped to `[0, 255]`. +pub(crate) fn ycbcr_to_rgb(y: u8, cb: u8, cr: u8) -> (u8, u8, u8) { + let y = y as i32; + let cb_centered = cb as i32 - 128; + let cr_centered = cr as i32 - 128; + let r = y + ((FIX_1_40200 * cr_centered + ROUND) >> 16); + let g = y - ((FIX_0_34414 * cb_centered + FIX_0_71414 * cr_centered + ROUND) >> 16); + let b = y + ((FIX_1_77200 * cb_centered + ROUND) >> 16); + + (clamp_to_u8(r), clamp_to_u8(g), clamp_to_u8(b)) +} + +/// Convert one 12-bit YCbCr pixel to RGB samples stored in `Rgb16` output. +/// +/// Returned values are clamped to the native 12-bit range `[0, 4095]`, not +/// scaled to the full `u16` range. +pub(crate) fn ycbcr12_to_rgb16(y: u16, cb: u16, cr: u16) -> (u16, u16, u16) { + let y = i32::from(y); + let cb_centered = i32::from(cb) - 2048; + let cr_centered = i32::from(cr) - 2048; + let r = y + ((FIX_1_40200 * cr_centered + ROUND) >> 16); + let g = y - ((FIX_0_34414 * cb_centered + FIX_0_71414 * cr_centered + ROUND) >> 16); + let b = y + ((FIX_1_77200 * cb_centered + ROUND) >> 16); + + (clamp_to_12bit(r), clamp_to_12bit(g), clamp_to_12bit(b)) +} + +/// Convert one 16-bit lossless YCbCr pixel to RGB samples stored in `Rgb16` +/// output. +/// +/// Returned values are clamped to the native 16-bit range `[0, 65535]`. +pub(crate) fn ycbcr16_to_rgb16(y: u16, cb: u16, cr: u16) -> (u16, u16, u16) { + let y = i64::from(y); + let cb_centered = i64::from(cb) - 32768; + let cr_centered = i64::from(cr) - 32768; + let r = y + ((i64::from(FIX_1_40200) * cr_centered + i64::from(ROUND)) >> 16); + let g = y + - ((i64::from(FIX_0_34414) * cb_centered + + i64::from(FIX_0_71414) * cr_centered + + i64::from(ROUND)) + >> 16); + let b = y + ((i64::from(FIX_1_77200) * cb_centered + i64::from(ROUND)) >> 16); + + (clamp_to_u16(r), clamp_to_u16(g), clamp_to_u16(b)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn neutral_gray_roundtrips_to_equal_rgb_channels() { + let (r, g, b) = ycbcr_to_rgb(128, 128, 128); + assert_eq!((r, g, b), (128, 128, 128)); + } + + #[test] + fn bright_red_maps_to_high_r_low_gb() { + // libjpeg-turbo: Y=76 Cb=85 Cr=255 ≈ pure red (255, 0, 0). + let (r, g, b) = ycbcr_to_rgb(76, 85, 255); + assert!(r > 240 && g < 15 && b < 15, "got ({r}, {g}, {b})"); + } + + #[test] + fn clamps_out_of_range_arithmetic_to_0_255() { + // Y=255, large Cr pushes R arithmetic above 255 → saturate high. + let (r, _, _) = ycbcr_to_rgb(255, 128, 255); + assert_eq!(r, 255, "R must saturate at 255"); + // Y=255, large Cb pushes B arithmetic above 255 → saturate high. + let (_, _, b) = ycbcr_to_rgb(255, 255, 128); + assert_eq!(b, 255, "B must saturate at 255"); + // Y=0, small Cr pushes R arithmetic below 0 → saturate low. + let (r, _, _) = ycbcr_to_rgb(0, 128, 0); + assert_eq!(r, 0, "R must saturate at 0"); + } + + #[test] + fn matches_libjpeg_turbo_fixed_point_expectations() { + // Sampled checks against libjpeg-turbo jdcolor.c computed values. + // Y=100 Cb=150 Cr=200 → R=201, G=41, B=139 (16-bit fixed point). + let (r, g, b) = ycbcr_to_rgb(100, 150, 200); + assert!((r as i32 - 201).abs() <= 1, "R={r}, expected ≈201"); + assert!((g as i32 - 41).abs() <= 1, "G={g}, expected ≈41"); + assert!((b as i32 - 139).abs() <= 1, "B={b}, expected ≈139"); + } + + #[test] + fn ycbcr12_to_rgb16_uses_native_12_bit_range() { + assert_eq!(ycbcr12_to_rgb16(2048, 2048, 2048), (2048, 2048, 2048)); + assert_eq!(ycbcr12_to_rgb16(2064, 2072, 2032), (2042, 2067, 2107)); + assert_eq!(ycbcr12_to_rgb16(4095, 2048, 4095).0, 4095); + assert_eq!(ycbcr12_to_rgb16(0, 2048, 0).0, 0); + } + + #[test] + fn ycbcr16_to_rgb16_uses_native_16_bit_range() { + assert_eq!(ycbcr16_to_rgb16(32768, 32768, 32768), (32768, 32768, 32768)); + assert_eq!(ycbcr16_to_rgb16(33000, 35000, 40000), (43139, 27067, 36955)); + assert_eq!(ycbcr16_to_rgb16(u16::MAX, 32768, u16::MAX).0, u16::MAX); + assert_eq!(ycbcr16_to_rgb16(0, 32768, 0).0, 0); + } +} diff --git a/crates/signinum-jpeg/src/context.rs b/crates/j2k-jpeg/src/context.rs similarity index 66% rename from crates/signinum-jpeg/src/context.rs rename to crates/j2k-jpeg/src/context.rs index f1f73987..6cf01d10 100644 --- a/crates/signinum-jpeg/src/context.rs +++ b/crates/j2k-jpeg/src/context.rs @@ -9,7 +9,7 @@ use crate::error::Warning; use crate::info::Info; use crate::parse::tables::RawHuffmanTable; use alloc::sync::Arc; -use signinum_core::{CacheStats, CodecContext}; +use j2k_core::{CacheStats, CodecContext}; const QUANT_CACHE_SLOTS: usize = 8; const HUFFMAN_CACHE_SLOTS: usize = 8; @@ -26,6 +26,7 @@ struct CachedQuantTable { #[derive(Debug, Clone)] struct CachedHuffmanTable { digest: u64, + raw: RawHuffmanTable, table: Arc, } @@ -48,6 +49,9 @@ pub struct DecoderContext { quant_tables: [Option; QUANT_CACHE_SLOTS], huffman_tables: [Option; HUFFMAN_CACHE_SLOTS], decode_plans: [Option; PLAN_CACHE_SLOTS], + cache_hits: u64, + cache_misses: u64, + cache_evictions: u64, } impl DecoderContext { @@ -58,22 +62,33 @@ impl DecoderContext { quant_tables: core::array::from_fn(|_| None), huffman_tables: core::array::from_fn(|_| None), decode_plans: core::array::from_fn(|_| None), + cache_hits: 0, + cache_misses: 0, + cache_evictions: 0, } } pub(crate) fn resolve_quant_table(&mut self, table: [u16; 64]) -> Arc<[u16; 64]> { let digest = digest_quant_table(&table); + self.resolve_quant_table_with_digest(table, digest) + } + + fn resolve_quant_table_with_digest(&mut self, table: [u16; 64], digest: u64) -> Arc<[u16; 64]> { let start = (digest as usize) % self.quant_tables.len(); for probe in 0..self.quant_tables.len() { let slot = (start + probe) % self.quant_tables.len(); match &self.quant_tables[slot] { - Some(cached) if cached.digest == digest => return Arc::clone(&cached.table), + Some(cached) if cached.digest == digest && cached.table.as_ref() == &table => { + self.cache_hits = self.cache_hits.saturating_add(1); + return Arc::clone(&cached.table); + } None => { let table = Arc::new(table); self.quant_tables[slot] = Some(CachedQuantTable { digest, table: Arc::clone(&table), }); + self.cache_misses = self.cache_misses.saturating_add(1); return table; } Some(_) => {} @@ -86,6 +101,8 @@ impl DecoderContext { digest, table: Arc::clone(&table), }); + self.cache_misses = self.cache_misses.saturating_add(1); + self.cache_evictions = self.cache_evictions.saturating_add(1); table } @@ -94,17 +111,30 @@ impl DecoderContext { raw: &RawHuffmanTable, ) -> Result, JpegError> { let digest = digest_huffman_table(raw); + self.resolve_huffman_table_with_digest(raw, digest) + } + + fn resolve_huffman_table_with_digest( + &mut self, + raw: &RawHuffmanTable, + digest: u64, + ) -> Result, JpegError> { let start = (digest as usize) % self.huffman_tables.len(); for probe in 0..self.huffman_tables.len() { let slot = (start + probe) % self.huffman_tables.len(); match &self.huffman_tables[slot] { - Some(cached) if cached.digest == digest => return Ok(Arc::clone(&cached.table)), + Some(cached) if cached.digest == digest && &cached.raw == raw => { + self.cache_hits = self.cache_hits.saturating_add(1); + return Ok(Arc::clone(&cached.table)); + } None => { let table = Arc::new(HuffmanTable::from_raw(raw)?); self.huffman_tables[slot] = Some(CachedHuffmanTable { digest, + raw: raw.clone(), table: Arc::clone(&table), }); + self.cache_misses = self.cache_misses.saturating_add(1); return Ok(table); } Some(_) => {} @@ -115,8 +145,11 @@ impl DecoderContext { let table = Arc::new(HuffmanTable::from_raw(raw)?); self.huffman_tables[slot] = Some(CachedHuffmanTable { digest, + raw: raw.clone(), table: Arc::clone(&table), }); + self.cache_misses = self.cache_misses.saturating_add(1); + self.cache_evictions = self.cache_evictions.saturating_add(1); Ok(table) } @@ -138,6 +171,7 @@ impl DecoderContext { if cached.digest == digest && cached.header_prefix.as_ref() == header_prefix => { + self.cache_hits = self.cache_hits.saturating_add(1); return Ok(( cached.info.clone(), Arc::clone(&cached.warnings), @@ -161,8 +195,31 @@ impl DecoderContext { warnings: Arc::clone(&built.1), plan: built.2.clone(), }); + self.cache_misses = self.cache_misses.saturating_add(1); + if empty_slot.is_none() { + self.cache_evictions = self.cache_evictions.saturating_add(1); + } Ok(built) } + + fn occupied_cache_slots(&self) -> u64 { + let occupied = self + .quant_tables + .iter() + .filter(|slot| slot.is_some()) + .count() + + self + .huffman_tables + .iter() + .filter(|slot| slot.is_some()) + .count() + + self + .decode_plans + .iter() + .filter(|slot| slot.is_some()) + .count(); + occupied as u64 + } } impl CodecContext for DecoderContext { @@ -171,7 +228,12 @@ impl CodecContext for DecoderContext { } fn cache_stats(&self) -> CacheStats { - CacheStats::default() + CacheStats::with_slots( + self.cache_hits, + self.cache_misses, + self.occupied_cache_slots(), + self.cache_evictions, + ) } } @@ -216,6 +278,12 @@ mod tests { let first = ctx.resolve_quant_table([7; 64]); let second = ctx.resolve_quant_table([7; 64]); assert!(Arc::ptr_eq(&first, &second)); + + let stats = ctx.cache_stats(); + assert_eq!(stats.hits, 1); + assert_eq!(stats.misses, 1); + assert_eq!(stats.occupied_slots, 1); + assert_eq!(stats.evictions, 0); } #[test] @@ -230,6 +298,41 @@ mod tests { assert!(Arc::ptr_eq(&first, &second)); } + #[test] + fn quant_table_digest_collision_compares_full_table_contents() { + let mut ctx = DecoderContext::new(); + let first = ctx.resolve_quant_table_with_digest([7; 64], 0); + let second = ctx.resolve_quant_table_with_digest([8; 64], 0); + + assert!(!Arc::ptr_eq(&first, &second)); + assert_eq!(*first, [7; 64]); + assert_eq!(*second, [8; 64]); + assert_eq!(ctx.cache_stats().misses, 2); + } + + #[test] + fn huffman_table_digest_collision_compares_full_raw_table_contents() { + let first_raw = RawHuffmanTable { + bits: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + values: crate::parse::tables::HuffmanValues::from_slice(&[0]), + }; + let second_raw = RawHuffmanTable { + bits: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + values: crate::parse::tables::HuffmanValues::from_slice(&[1]), + }; + let mut ctx = DecoderContext::new(); + + let first = ctx + .resolve_huffman_table_with_digest(&first_raw, 0) + .unwrap(); + let second = ctx + .resolve_huffman_table_with_digest(&second_raw, 0) + .unwrap(); + + assert!(!Arc::ptr_eq(&first, &second)); + assert_eq!(ctx.cache_stats().misses, 2); + } + #[test] fn prepared_plan_cache_hits_skip_rebuild() { let mut ctx = DecoderContext::new(); @@ -244,7 +347,11 @@ mod tests { Info { dimensions: (16, 16), color_space: ColorSpace::YCbCr, - sampling: SamplingFactors::from_components(&[(2, 2), (1, 1), (1, 1)]), + sampling: SamplingFactors::from_validated_components(&[ + (2, 2), + (1, 1), + (1, 1), + ]), sof_kind: SofKind::Baseline8, bit_depth: 8, restart_interval: None, @@ -260,7 +367,11 @@ mod tests { Arc::clone(&warnings), PreparedDecodePlan { components: vec![], - sampling: SamplingFactors::from_components(&[(2, 2), (1, 1), (1, 1)]), + sampling: SamplingFactors::from_validated_components(&[ + (2, 2), + (1, 1), + (1, 1), + ]), color_space: ColorSpace::YCbCr, restart_interval: None, dimensions: (16, 16), diff --git a/crates/j2k-jpeg/src/decoder.rs b/crates/j2k-jpeg/src/decoder.rs new file mode 100644 index 00000000..c7d51537 --- /dev/null +++ b/crates/j2k-jpeg/src/decoder.rs @@ -0,0 +1,7422 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Public [`Decoder`] entry points. + +use crate::backend::Backend; +use crate::context::DecoderContext; +use crate::entropy::block::{decode_block_with_activity, BlockActivity, CoefficientBlock}; +use crate::entropy::huffman::HuffmanTable; +use crate::entropy::progressive::{ + decode_progressive, decode_progressive_dct_blocks, PreparedProgressiveComponentPlan, + PreparedProgressivePlan, PreparedProgressiveScan, PreparedProgressiveScanComponent, +}; +use crate::entropy::sequential::{ + decode_scan_baseline, decode_scan_baseline_rgb, decode_scan_fast_rgb_444, + decode_scan_fast_tile_rgb, decode_scan_fast_tile_rgb_region, + decode_scan_fast_tile_rgb_region_scaled, fast_tile_region_first_decode_mcu, finish_scan, + stripe_region_layout, PreparedComponentPlan, PreparedDecodePlan, +}; +use crate::entropy::ZIGZAG; +use crate::error::{JpegError, MarkerKind, Warning}; +use crate::info::{ + ColorSpace, DecodeOptions, DownscaleFactor, Info, OutputFormat, Rect, RestartIndex, + RestartSegment, SofKind, +}; +use crate::internal::bit_reader::BitReader; +use crate::internal::checkpoint::{checkpoint_before_mcu, CpuCheckpointCache, DeviceCheckpoint}; +use crate::internal::scratch::{ScratchPool, SinkRows}; +use crate::lossless::{lossless_predict, LosslessSample}; +use crate::output::{ + validate_buffer, Gray8Writer, InterleavedRgbWriter, OutputWriter, Rgb8Writer, Rgba8Writer, +}; +use crate::parse::header::{parse_header, parse_info, ParsedHeader}; +use crate::parse::tables::{HuffmanValues, RawHuffmanTable}; +use crate::profile::{duration_us_string, emit_jpeg_profile_row, jpeg_profile_stages_enabled}; +use crate::segment::PreparedJpeg; +use crate::JpegCodec; +use alloc::sync::Arc; +use alloc::vec::Vec; +use core::cell::RefCell; +pub use j2k_core::TileBatchOptions; +use j2k_core::{ + CompressedPayloadKind, CompressedTransferSyntax, DecodeOutcome as CoreDecodeOutcome, + DecodeRowsError, DecoderContext as CoreDecoderContext, Downscale, ImageCodec, ImageDecode, + ImageDecodeRows, PassthroughCandidate, PixelFormat, RowSink, TileBatchDecode, +}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +const DEFAULT_MAX_DECODE_BYTES: usize = 512 * 1024 * 1024; +const CPU_ROI_CHECKPOINT_CADENCE_MCUS: u32 = 1024; +const CPU_ROI_CHECKPOINT_MIN_TARGET_MCUS: u32 = 4096; + +std::thread_local! { + static DEFAULT_SCRATCH: RefCell = RefCell::new(ScratchPool::new()); + static DEFAULT_CONTEXT: RefCell = RefCell::new(DecoderContext::new()); +} + +/// Non-fatal outcome of a successful decode. See spec Section 2. +/// +/// `DecodeOutcome` lives on `decoder.rs` rather than `info.rs` because it +/// carries `Warning` values from `error.rs`, and moving it into `info` would +/// create a `info → error` cycle (see `info.rs` header note). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecodeOutcome { + /// The rectangle actually written to the output buffer. For `decode_into` + /// this is always `Rect::full(info.dimensions)`; later milestones add + /// `decode_region_into` which can return a narrower rect. + pub decoded: Rect, + /// Warnings emitted during parse or decode. Empty when the stream is + /// syntactically clean and every capability was exercised without fallback. + pub warnings: Vec, +} + +impl From for CoreDecodeOutcome { + fn from(outcome: DecodeOutcome) -> Self { + Self { + decoded: outcome.decoded.into(), + warnings: outcome.warnings, + } + } +} + +/// One tile decode request for [`decode_tiles_into`]. +pub type TileDecodeJob<'i, 'o> = j2k_core::TileDecodeJob<'i, 'o>; + +/// One decode request for a JPEG tile already normalized by +/// [`prepare_tiff_jpeg_tile`](crate::prepare_tiff_jpeg_tile). +pub struct PreparedJpegTileJob<'i, 'o> { + /// Decode-ready prepared JPEG bytes. + pub input: PreparedJpeg<'i>, + /// Caller-owned RGB8 output buffer for this tile. + pub out: &'o mut [u8], + /// Distance in bytes between output rows. + pub stride: usize, + /// Per-job JPEG decode options. + pub options: DecodeOptions, +} + +/// Result for one successful prepared JPEG tile decode. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecodedTile { + /// Tile dimensions reported by the prepared JPEG header. + pub dimensions: (u32, u32), + /// Rectangle written into the output buffer. + pub decoded: Rect, + /// Non-fatal warnings emitted during parse or decode. + pub warnings: Vec, +} + +/// One scaled tile decode request for [`decode_tiles_scaled_into`]. +pub type TileScaledDecodeJob<'i, 'o> = j2k_core::TileScaledDecodeJob<'i, 'o>; + +/// One ROI+scaled tile decode request for +/// [`decode_tiles_region_scaled_into`]. +pub type TileRegionScaledDecodeJob<'i, 'o> = j2k_core::TileRegionScaledDecodeJob<'i, 'o>; + +/// Error returned by [`decode_tiles_into`], annotated with the failing tile +/// index from the caller's input order. +pub type TileBatchError = j2k_core::TileBatchError; + +/// Receives decoded component rows before they are packed into the final +/// interleaved pixel format. +pub trait ComponentRowWriter { + /// Receive one grayscale row. + fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError>; + + /// Receive one full-width Y/Cb/Cr row. + fn write_ycbcr_row( + &mut self, + y: u32, + y_row: &[u8], + cb_row: &[u8], + cr_row: &[u8], + ) -> Result<(), JpegError>; + + /// Receive one full-width planar RGB row. + fn write_rgb_row( + &mut self, + y: u32, + r_row: &[u8], + g_row: &[u8], + b_row: &[u8], + ) -> Result<(), JpegError>; +} + +/// A parsed borrowed view of a JPEG stream. +#[derive(Debug)] +pub struct JpegView<'a> { + bytes: &'a [u8], + header: ParsedHeader, + info: Info, + options: DecodeOptions, +} + +impl<'a> JpegView<'a> { + /// Parse the stream into a borrowed view that can later build a decoder. + pub fn parse(input: &'a [u8]) -> Result { + Self::parse_with_options(input, DecodeOptions::default()) + } + + /// Parse the stream with explicit decode options. + pub fn parse_with_options(input: &'a [u8], options: DecodeOptions) -> Result { + let header = parse_header(input)?; + let mut info = header.info(); + options.apply_to_info(&mut info); + Ok(Self { + bytes: input, + header, + info, + options, + }) + } + + /// Header-derived metadata for the parsed stream. + pub fn info(&self) -> &Info { + &self.info + } + + /// Original compressed bytes backing this view. + pub fn bytes(&self) -> &'a [u8] { + self.bytes + } + + /// Return a byte-preserving passthrough candidate for active DICOM/WSI + /// transfer syntaxes. + /// + /// Progressive JPEG is intentionally not exposed here because the active + /// conversion path should transcode it rather than introduce a retired or + /// unsupported destination syntax. + pub fn passthrough_candidate(&self) -> Option> { + jpeg_passthrough_syntax(&self.info).map(|transfer_syntax| { + PassthroughCandidate::new( + self.bytes, + transfer_syntax, + CompressedPayloadKind::JpegInterchange, + self.info.to_core_info(), + ) + }) + } + + /// Build a restart-marker byte-offset index for the first scan. + /// + /// Offsets are absolute byte positions in the original JPEG byte slice. + /// Returns `Ok(None)` when the stream has no non-zero DRI marker. + pub fn restart_index(&self) -> Result, JpegError> { + restart_index_for_stream( + self.bytes, + self.header.sos_offset, + &self.info, + self.info.restart_interval, + ) + } + + pub(crate) fn has_lossless_subsampled_color_capability_shape(&self) -> bool { + if self.info.sof_kind != SofKind::Lossless + || !matches!(self.info.color_space, ColorSpace::Rgb | ColorSpace::YCbCr) + || !matches!(self.info.bit_depth, 8 | 16) + || self.info.sampling.len() != 3 + || !self + .info + .sampling + .components() + .iter() + .any(|&(h, v)| h != 1 || v != 1) + || self.header.scan_count != 1 + { + return false; + } + + let Some(scan) = self.header.scan.as_ref() else { + return false; + }; + if !(1..=7).contains(&scan.ss) + || scan.se != 0 + || scan.ah != 0 + || scan.al != 0 + || scan.components.len() != 3 + { + return false; + } + + scan.components.iter().all(|scan_component| { + find_component_index(&self.header.component_ids, scan_component.id).is_some() + && self.header.huffman_tables.dc[scan_component.dc_table as usize].is_some() + }) + } +} + +/// A borrowed view of a JPEG stream ready to decode. Constructed via +/// [`Decoder::new`] or [`Decoder::from_view`]. `Decoder<'a>: Send + Sync`. +#[derive(Debug)] +pub struct Decoder<'a> { + pub(crate) bytes: &'a [u8], + pub(crate) info: Info, + pub(crate) warnings: Arc<[Warning]>, + pub(crate) backend: Backend, + pub(crate) plan: PreparedDecodePlan, + pub(crate) progressive_plan: Option, + lossless_plan: Option, + pub(crate) cpu_entropy_checkpoints: Mutex, +} + +#[derive(Debug, Clone)] +struct PreparedLosslessPlan { + predictor: u8, + bit_depth: u8, + dc_table: Arc, + dimensions: (u32, u32), + scan_offset: usize, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum LosslessColorSampling { + S444, + S422, + S420, +} + +impl<'a> Decoder<'a> { + /// Parse the headers without decoding pixels. The parser walks headers up + /// to the first SOS and then performs a lightweight marker scan so + /// `Info::scan_count` reflects all scans in the file. + /// + /// # Errors + /// Returns any structural, unsupported-SOF, or sanity-check error + /// encountered before the Start-of-Scan marker. See [`JpegError`]. + pub fn inspect(input: &'a [u8]) -> Result { + Self::inspect_with_options(input, DecodeOptions::default()) + } + + /// Parse headers with explicit decode options, without decoding pixels. + /// + /// The options are applied to the returned [`Info`] exactly as they would + /// be for [`Self::new_with_options`]. + pub fn inspect_with_options( + input: &'a [u8], + options: DecodeOptions, + ) -> Result { + let mut info = parse_info(input)?; + options.apply_to_info(&mut info); + Ok(info) + } + + /// Build a decoder with explicit decode options. + pub fn new_with_options(input: &'a [u8], options: DecodeOptions) -> Result { + let view = JpegView::parse_with_options(input, options)?; + DEFAULT_CONTEXT.with(|ctx| Self::from_view_in_context(view, &mut ctx.borrow_mut())) + } + + /// Build a decoder ready for `decode_into`. Parses the full header, pre- + /// builds every referenced Huffman table, and validates that the stream is + /// one of the SOFs this release implements. + /// + /// # Errors + /// - Any parse error encountered before SOS (see [`Self::inspect`]). + /// - [`JpegError::NotImplemented`] for SOFs that parse but are not yet + /// decodable for the requested shape (for example Progressive12 or + /// unsupported Lossless predictors). + /// - [`JpegError::MissingHuffmanTable`] if the scan references a DC/AC + /// table slot that was never defined by a DHT segment. + pub fn new(input: &'a [u8]) -> Result { + Self::new_with_options(input, DecodeOptions::default()) + } + + /// Build a decoder from a previously parsed [`JpegView`]. + pub fn from_view(view: JpegView<'a>) -> Result { + DEFAULT_CONTEXT.with(|ctx| Self::from_view_in_context(view, &mut ctx.borrow_mut())) + } + + /// Build a decoder from a previously parsed [`JpegView`], reusing shared + /// compiled DHT/DQT state from `ctx` when table contents repeat. + pub fn from_view_in_context( + view: JpegView<'a>, + ctx: &mut DecoderContext, + ) -> Result { + let JpegView { + bytes, + header, + info, + options, + } = view; + let backend = Backend::detect(); + let (info, warnings, plan, progressive_plan, lossless_plan) = if matches!( + info.sof_kind, + SofKind::Progressive8 | SofKind::Progressive12 + ) { + let progressive_plan = Self::build_progressive_plan(&header, &info, ctx)?; + let plan = Self::build_progressive_host_output_plan(&header, &info, ctx)?; + ( + info, + Arc::<[Warning]>::from(header.warnings.as_slice()), + plan, + Some(progressive_plan), + None, + ) + } else if info.sof_kind == SofKind::Lossless { + let (plan, lossless_plan) = Self::build_lossless_plan(&header, &info, ctx)?; + ( + info, + Arc::<[Warning]>::from(header.warnings.as_slice()), + plan, + None, + Some(lossless_plan), + ) + } else if options == DecodeOptions::default() { + if let Some(scan_offset) = header.sos_offset { + let header_prefix = &bytes[..scan_offset]; + let (info, warnings, plan) = ctx.resolve_decode_plan(header_prefix, |ctx| { + let plan = Self::build_prepared_plan(&header, &info, ctx)?; + Ok(( + info.clone(), + Arc::<[Warning]>::from(header.warnings.as_slice()), + plan, + )) + })?; + (info, warnings, plan, None, None) + } else { + let plan = Self::build_prepared_plan(&header, &info, ctx)?; + ( + info, + Arc::<[Warning]>::from(header.warnings.as_slice()), + plan, + None, + None, + ) + } + } else { + let plan = Self::build_prepared_plan(&header, &info, ctx)?; + ( + info, + Arc::<[Warning]>::from(header.warnings.as_slice()), + plan, + None, + None, + ) + }; + Ok(Self { + bytes, + info, + warnings, + backend, + plan, + progressive_plan, + lossless_plan, + cpu_entropy_checkpoints: Mutex::new(CpuCheckpointCache::default()), + }) + } + + fn build_prepared_plan( + header: &ParsedHeader, + info: &Info, + ctx: &mut DecoderContext, + ) -> Result { + match info.sof_kind { + SofKind::Baseline8 | SofKind::Extended8 | SofKind::Extended12 => {} + other => return Err(JpegError::NotImplemented { sof: other }), + } + if info.sof_kind == SofKind::Extended12 + && !matches!( + info.color_space, + ColorSpace::Grayscale + | ColorSpace::YCbCr + | ColorSpace::Rgb + | ColorSpace::Cmyk + | ColorSpace::Ycck + ) + { + return Err(JpegError::NotImplemented { sof: info.sof_kind }); + } + match info.color_space { + ColorSpace::Grayscale + | ColorSpace::YCbCr + | ColorSpace::Rgb + | ColorSpace::Cmyk + | ColorSpace::Ycck => {} + } + + let mut dc_tables: [Option>; 4] = Default::default(); + let mut ac_tables: [Option>; 4] = Default::default(); + let scan = header.scan.as_ref().ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sos, + })?; + if header.scan_count != 1 { + return Err(JpegError::InvalidSequentialScanCount { + sof: info.sof_kind, + count: header.scan_count, + }); + } + validate_leading_component_sampling(header, info)?; + // Every component must declare H,V in 1..=4 per T.81 §B.2.2, and max_h + // must actually divide every component's H (same for V). Malformed + // streams can set H=0 (div-by-zero in upsample ratio), non-divisors + // (arbitrary ratios M2 handles), or ratios that don't produce planes + // that cover the image width. + for (h, v) in header.sampling.iter() { + if h == 0 || v == 0 || h > 4 || v > 4 { + return Err(JpegError::NotImplemented { sof: info.sof_kind }); + } + if !header.sampling.max_h.is_multiple_of(h) || !header.sampling.max_v.is_multiple_of(v) + { + return Err(JpegError::NotImplemented { sof: info.sof_kind }); + } + } + for comp in &scan.components { + let di = comp.dc_table as usize; + let ai = comp.ac_table as usize; + if dc_tables[di].is_none() { + let raw = header.huffman_tables.dc[di].as_ref().ok_or( + JpegError::MissingHuffmanTable { + component: comp.id, + class: 0, + id: comp.dc_table, + }, + )?; + dc_tables[di] = Some(ctx.resolve_huffman_table(raw)?); + } + if ac_tables[ai].is_none() { + let raw = header.huffman_tables.ac[ai].as_ref().ok_or( + JpegError::MissingHuffmanTable { + component: comp.id, + class: 1, + id: comp.ac_table, + }, + )?; + ac_tables[ai] = Some(ctx.resolve_huffman_table(raw)?); + } + } + + build_decode_plan(header, info, &dc_tables, &ac_tables, ctx) + } + + fn build_lossless_plan( + header: &ParsedHeader, + info: &Info, + ctx: &mut DecoderContext, + ) -> Result<(PreparedDecodePlan, PreparedLosslessPlan), JpegError> { + if info.sof_kind != SofKind::Lossless { + return Err(JpegError::NotImplemented { sof: info.sof_kind }); + } + if header.scan_count != 1 { + return Err(JpegError::NotImplemented { sof: info.sof_kind }); + } + let scan = header.scan.as_ref().ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sos, + })?; + if !(1..=7).contains(&scan.ss) { + return Err(JpegError::UnsupportedPredictor { predictor: scan.ss }); + } + if scan.se != 0 || scan.ah != 0 || scan.al != 0 { + return Err(JpegError::NotImplemented { sof: info.sof_kind }); + } + let expected_components = match (info.color_space, info.bit_depth) { + (ColorSpace::Grayscale, 8 | 16) => 1, + (ColorSpace::Rgb, 8 | 16) => 3, + (ColorSpace::YCbCr, 8 | 16) => 3, + _ => return Err(JpegError::NotImplemented { sof: info.sof_kind }), + }; + if scan.components.len() != expected_components { + return Err(JpegError::UnsupportedComponentCount { + count: scan.components.len() as u8, + }); + } + let empty_raw = RawHuffmanTable { + bits: [0; 16], + values: HuffmanValues::default(), + }; + let empty_huffman = ctx.resolve_huffman_table(&empty_raw)?; + let mut components = Vec::with_capacity(scan.components.len()); + let mut first_dc_table = None; + for scan_component in scan.components.iter().copied() { + let component_index = find_component_index(&header.component_ids, scan_component.id) + .ok_or(JpegError::UnknownScanComponent { + offset: header.sos_offset.unwrap_or_default(), + component: scan_component.id, + })?; + let (h, v) = + header + .sampling + .component(component_index) + .ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sof, + })?; + let raw_dc = header.huffman_tables.dc[scan_component.dc_table as usize] + .as_ref() + .ok_or(JpegError::MissingHuffmanTable { + component: scan_component.id, + class: 0, + id: scan_component.dc_table, + })?; + let dc_table = ctx.resolve_huffman_table(raw_dc)?; + first_dc_table.get_or_insert_with(|| Arc::clone(&dc_table)); + components.push(PreparedComponentPlan { + h, + v, + output_index: component_index, + quant: ctx.resolve_quant_table([1; 64]), + dc_table, + ac_table: Arc::clone(&empty_huffman), + }); + } + if matches!(info.color_space, ColorSpace::Rgb | ColorSpace::YCbCr) + && lossless_color_sampling(info).is_none() + { + return Err(JpegError::NotImplemented { sof: info.sof_kind }); + } + let plan = PreparedDecodePlan { + components, + sampling: info.sampling, + color_space: info.color_space, + restart_interval: header.restart_interval, + dimensions: info.dimensions, + scan_offset: header.sos_offset.ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sos, + })?, + scratch_bytes: compute_lossless_scratch_bytes(info, DEFAULT_MAX_DECODE_BYTES)?, + }; + let lossless = PreparedLosslessPlan { + predictor: scan.ss, + bit_depth: info.bit_depth, + dc_table: first_dc_table.ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sos, + })?, + dimensions: info.dimensions, + scan_offset: plan.scan_offset, + }; + Ok((plan, lossless)) + } + + fn build_progressive_plan( + header: &ParsedHeader, + info: &Info, + ctx: &mut DecoderContext, + ) -> Result { + if !matches!( + info.sof_kind, + SofKind::Progressive8 | SofKind::Progressive12 + ) { + return Err(JpegError::NotImplemented { sof: info.sof_kind }); + } + match (info.sof_kind, info.color_space) { + ( + SofKind::Progressive8 | SofKind::Progressive12, + ColorSpace::Grayscale | ColorSpace::YCbCr | ColorSpace::Rgb, + ) => {} + (SofKind::Progressive12, ColorSpace::Cmyk | ColorSpace::Ycck) => {} + (_, color_space) => return Err(JpegError::UnsupportedColorSpace { color_space }), + } + validate_sampling_factors(header, info)?; + if header.progressive_scans.is_empty() { + return Err(JpegError::MissingMarker { + marker: MarkerKind::Sos, + }); + } + + let max_h = u32::from(header.sampling.max_h); + let max_v = u32::from(header.sampling.max_v); + let mcu_cols = info.dimensions.0.div_ceil(8 * max_h); + let mcu_rows = info.dimensions.1.div_ceil(8 * max_v); + let mut components = Vec::with_capacity(header.component_ids.len()); + for (output_index, &id) in header.component_ids.iter().enumerate() { + let (h, v) = + header + .sampling + .component(output_index) + .ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sof, + })?; + let quant_id = + *header + .quant_table_ids + .get(output_index) + .ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sof, + })? as usize; + let quant = *header + .quant_tables + .entries + .get(quant_id) + .and_then(|q| q.as_ref()) + .ok_or(JpegError::MissingQuantTable { + component: id, + table_id: quant_id as u8, + })?; + components.push(PreparedProgressiveComponentPlan { + h, + v, + output_index, + quant: ctx.resolve_quant_table(quant), + block_cols: mcu_cols * u32::from(h), + block_rows: mcu_rows * u32::from(v), + sample_width: info + .dimensions + .0 + .saturating_mul(u32::from(h)) + .div_ceil(max_h), + sample_height: info + .dimensions + .1 + .saturating_mul(u32::from(v)) + .div_ceil(max_v), + }); + } + + let mut scans = Vec::with_capacity(header.progressive_scans.len()); + for parsed in &header.progressive_scans { + let mut scan_components = Vec::with_capacity(parsed.scan.components.len()); + for component in &parsed.scan.components { + let component_index = find_component_index(&header.component_ids, component.id) + .ok_or(JpegError::UnknownScanComponent { + offset: parsed.entropy_offset, + component: component.id, + })?; + let quant_id = *header.quant_table_ids.get(component_index).ok_or( + JpegError::MissingMarker { + marker: MarkerKind::Sof, + }, + )?; + let _ = parsed + .quant_tables + .entries + .get(quant_id as usize) + .and_then(|q| q.as_ref()) + .ok_or(JpegError::MissingQuantTable { + component: component.id, + table_id: quant_id, + })?; + let dc_table = if parsed.scan.ss == 0 { + Some(resolve_progressive_huffman( + ctx, + &parsed.huffman_tables.dc, + component.id, + 0, + component.dc_table, + )?) + } else { + None + }; + let ac_table = if parsed.scan.ss > 0 { + Some(resolve_progressive_huffman( + ctx, + &parsed.huffman_tables.ac, + component.id, + 1, + component.ac_table, + )?) + } else { + None + }; + scan_components.push(PreparedProgressiveScanComponent { + component_index, + dc_table, + ac_table, + }); + } + scans.push(PreparedProgressiveScan { + components: scan_components, + ss: parsed.scan.ss, + se: parsed.scan.se, + ah: parsed.scan.ah, + al: parsed.scan.al, + entropy_offset: parsed.entropy_offset, + restart_interval: parsed.restart_interval, + }); + } + + let scratch_bytes = + compute_progressive_scratch_bytes(&components, info.dimensions.0 as usize)?; + Ok(PreparedProgressivePlan { + components, + scans, + sampling: info.sampling, + color_space: info.color_space, + dimensions: info.dimensions, + mcu_cols, + mcu_rows, + scratch_bytes, + }) + } + + fn build_progressive_host_output_plan( + header: &ParsedHeader, + info: &Info, + ctx: &mut DecoderContext, + ) -> Result { + let empty_raw = RawHuffmanTable { + bits: [0; 16], + values: HuffmanValues::default(), + }; + let empty_huffman = ctx.resolve_huffman_table(&empty_raw)?; + let mut components = Vec::with_capacity(header.component_ids.len()); + for (output_index, &id) in header.component_ids.iter().enumerate() { + let (h, v) = + header + .sampling + .component(output_index) + .ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sof, + })?; + let quant_id = + *header + .quant_table_ids + .get(output_index) + .ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sof, + })? as usize; + let quant = *header + .quant_tables + .entries + .get(quant_id) + .and_then(|q| q.as_ref()) + .ok_or(JpegError::MissingQuantTable { + component: id, + table_id: quant_id as u8, + })?; + components.push(PreparedComponentPlan { + h, + v, + output_index, + quant: ctx.resolve_quant_table(quant), + dc_table: Arc::clone(&empty_huffman), + ac_table: Arc::clone(&empty_huffman), + }); + } + Ok(PreparedDecodePlan { + components, + sampling: info.sampling, + color_space: info.color_space, + restart_interval: header.restart_interval, + dimensions: info.dimensions, + scan_offset: header.sos_offset.ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sos, + })?, + scratch_bytes: compute_decode_scratch_bytes( + info.dimensions, + info.sampling, + DEFAULT_MAX_DECODE_BYTES, + )?, + }) + } + + /// The parsed header as a public [`Info`]. + pub fn info(&self) -> &Info { + &self.info + } + + /// Return a byte-preserving passthrough candidate for this decoded stream. + pub fn passthrough_candidate(&self) -> Option> { + jpeg_passthrough_syntax(&self.info).map(|transfer_syntax| { + PassthroughCandidate::new( + self.bytes, + transfer_syntax, + CompressedPayloadKind::JpegInterchange, + self.info.to_core_info(), + ) + }) + } + + /// Build a restart-marker byte-offset index for the first scan. + /// + /// Offsets are absolute byte positions in the original JPEG byte slice. + /// Returns `Ok(None)` when the stream has no non-zero DRI marker. + pub fn restart_index(&self) -> Result, JpegError> { + restart_index_for_stream( + self.bytes, + Some(self.plan.scan_offset), + &self.info, + self.plan.restart_interval, + ) + } + + /// Decode the full image into the caller's buffer. + /// + /// # Errors + /// - [`JpegError::OutputBufferTooSmall`] or [`JpegError::InvalidStride`] + /// if the provided buffer/stride cannot hold the image at `fmt`. + /// - [`JpegError::NotImplemented`] if `fmt` requests a raw output the + /// current release does not emit (e.g. `RawYCbCr8`). + /// - Any entropy- or structural-decode error from the scan walker. + pub fn decode_into( + &self, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + ) -> Result { + DEFAULT_SCRATCH + .with(|pool| self.decode_into_with_scratch(&mut pool.borrow_mut(), out, stride, fmt)) + } + + /// Decode the full image into a freshly allocated tightly-packed buffer. + /// + /// This is the owned-output counterpart to [`Self::decode_into`]. It + /// avoids pre-zeroing the destination buffer, which matters on WSI-sized + /// RGB outputs where the allocation itself can otherwise dominate the + /// benchmark. + pub fn decode(&self, fmt: PixelFormat) -> Result<(Vec, DecodeOutcome), JpegError> { + DEFAULT_SCRATCH.with(|pool| self.decode_with_scratch(&mut pool.borrow_mut(), fmt)) + } + + /// Decode the full image into the caller's buffer using the core + /// `PixelFormat` + `Downscale` contract. + pub fn decode_scaled_into( + &self, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + scale: Downscale, + ) -> Result { + DEFAULT_SCRATCH.with(|pool| { + self.decode_scaled_into_with_scratch(&mut pool.borrow_mut(), out, stride, fmt, scale) + }) + } + + /// Decode the full image into a freshly allocated tightly-packed buffer + /// using the core `PixelFormat` + `Downscale` contract. + pub fn decode_scaled( + &self, + fmt: PixelFormat, + scale: Downscale, + ) -> Result<(Vec, DecodeOutcome), JpegError> { + DEFAULT_SCRATCH + .with(|pool| self.decode_scaled_with_scratch(&mut pool.borrow_mut(), fmt, scale)) + } + + /// [`Self::decode`] with caller-owned scratch. + pub fn decode_with_scratch( + &self, + pool: &mut ScratchPool, + fmt: PixelFormat, + ) -> Result<(Vec, DecodeOutcome), JpegError> { + self.decode_scaled_with_scratch(pool, fmt, Downscale::None) + } + + /// [`Self::decode_scaled`] with caller-owned scratch. + pub fn decode_scaled_with_scratch( + &self, + pool: &mut ScratchPool, + fmt: PixelFormat, + scale: Downscale, + ) -> Result<(Vec, DecodeOutcome), JpegError> { + let legacy = output_format_from_parts(self.info.sof_kind, fmt, scale)?; + let downscale = legacy.downscale(); + let (width, height) = scaled_dimensions(self.info.dimensions, downscale); + let (stride, len) = checked_output_geometry(width, height, legacy.bytes_per_pixel())?; + let mut out = allocate_output_buffer(len); + let outcome = self.decode_scaled_into_with_scratch(pool, &mut out, stride, fmt, scale)?; + Ok((out, outcome)) + } + + /// Decode the full image into the caller's buffer, reusing the supplied + /// [`ScratchPool`]. On a long-running tile batch this eliminates the + /// per-tile allocation of stripe buffers, the DC predictor, and the + /// chroma upsample rows — the realistic WSI reader shape. The first + /// call against a fresh pool does the allocation; subsequent calls at + /// the same-or-smaller shape reuse the underlying `Vec`s. + /// + /// # Errors + /// Identical to [`Self::decode_into`]. + pub fn decode_into_with_scratch( + &self, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + ) -> Result { + self.decode_scaled_into_with_scratch(pool, out, stride, fmt, Downscale::None) + } + + fn decode_into_output_format_with_scratch( + &self, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: OutputFormat, + ) -> Result { + let profile_enabled = jpeg_profile_stages_enabled(); + let total_start = profile_enabled.then(Instant::now); + let downscale = fmt.downscale(); + let (w, h) = scaled_dimensions(self.info.dimensions, downscale); + let scratch_bytes = self.decode_scratch_bytes(DEFAULT_MAX_DECODE_BYTES)?; + let bpp = fmt.bytes_per_pixel(); + validate_buffer(out, stride, w, h, bpp)?; + let decode_start = profile_enabled.then(Instant::now); + let result = match fmt { + OutputFormat::Rgb8 | OutputFormat::Rgb8Scaled { .. } => { + if self.lossless_plan.is_some() { + return match self.info.color_space { + ColorSpace::YCbCr => self.decode_lossless_ycbcr8_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ), + _ => self.decode_lossless_rgb8_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ), + }; + } + let mut writer = Rgb8Writer::new_with_backend(out, stride, w, self.backend); + self.decode_rgb_with_writer( + pool, + &mut writer, + downscale, + Rect::full(self.info.dimensions), + ) + } + OutputFormat::Rgba8 { alpha } | OutputFormat::Rgba8Scaled { alpha, .. } => { + if self.lossless_plan.is_some() { + return self.decode_lossless_rgba8_region_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + alpha, + ); + } + let mut writer = Rgba8Writer::new_with_backend(out, stride, w, alpha, self.backend); + self.decode_with_writer( + pool, + &mut writer, + downscale, + Rect::full(self.info.dimensions), + ) + } + OutputFormat::Gray8 | OutputFormat::Gray8Scaled { .. } => { + if self.lossless_plan.is_some() { + return self.decode_lossless_gray8_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ); + } + let mut writer = Gray8Writer::new(out, stride, w); + self.decode_with_writer( + pool, + &mut writer, + downscale, + Rect::full(self.info.dimensions), + ) + } + OutputFormat::Gray16 => { + if self.lossless_plan.is_some() { + return self.decode_lossless_gray16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ); + } + if self.info.sof_kind == SofKind::Progressive12 { + return self.decode_progressive12_gray16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ); + } + self.decode_extended12_gray16_into(out, stride) + } + OutputFormat::Gray16Scaled { .. } => { + if self.lossless_plan.is_some() { + return self.decode_lossless_gray16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ); + } + if self.info.sof_kind == SofKind::Progressive12 { + return self.decode_progressive12_gray16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ); + } + self.decode_extended12_gray16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ) + } + OutputFormat::Rgb16 => { + if self.lossless_plan.is_some() { + return match self.info.color_space { + ColorSpace::YCbCr => self.decode_lossless_ycbcr16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ), + _ => self.decode_lossless_rgb16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ), + }; + } + if self.info.sof_kind == SofKind::Progressive12 { + return self.decode_progressive12_rgb16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ); + } + self.decode_extended12_rgb16_into(out, stride) + } + OutputFormat::Rgb16Scaled { .. } => { + if self.lossless_plan.is_some() { + return match self.info.color_space { + ColorSpace::YCbCr => self.decode_lossless_ycbcr16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ), + _ => self.decode_lossless_rgb16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ), + }; + } + if self.info.sof_kind == SofKind::Progressive12 { + return self.decode_progressive12_rgb16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ); + } + self.decode_extended12_rgb16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + ) + } + OutputFormat::Rgba16 { alpha } | OutputFormat::Rgba16Scaled { alpha, .. } => { + if self.lossless_plan.is_some() { + return self.decode_lossless_rgba16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + alpha, + ); + } + if matches!( + self.info.sof_kind, + SofKind::Extended12 | SofKind::Progressive12 + ) { + return self.decode_12bit_rgba16_region_scaled_into( + out, + stride, + Rect::full(self.info.dimensions), + downscale, + alpha, + ); + } + Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }) + } + }; + if let (Some(total_start), Some(decode_start), Ok(outcome)) = + (total_start, decode_start, &result) + { + let source_width_s = self.info.dimensions.0.to_string(); + let source_height_s = self.info.dimensions.1.to_string(); + let output_width_s = w.to_string(); + let output_height_s = h.to_string(); + let stride_s = stride.to_string(); + let bpp_s = bpp.to_string(); + let output_bytes_s = stride.saturating_mul(h as usize).to_string(); + let scratch_bytes_s = scratch_bytes.to_string(); + let warning_count_s = outcome.warnings.len().to_string(); + let decode_us = duration_us_string(decode_start.elapsed()); + let total_us = duration_us_string(total_start.elapsed()); + emit_jpeg_profile_row( + "decode", + "cpu", + &[ + ("mode", "full"), + ("fmt", output_format_profile_name(fmt)), + ("downscale", downscale_profile_name(downscale)), + ("source_width", source_width_s.as_str()), + ("source_height", source_height_s.as_str()), + ("output_width", output_width_s.as_str()), + ("output_height", output_height_s.as_str()), + ("stride", stride_s.as_str()), + ("bpp", bpp_s.as_str()), + ("scratch_bytes", scratch_bytes_s.as_str()), + ("output_bytes", output_bytes_s.as_str()), + ("decode_us", decode_us.as_str()), + ("total_us", total_us.as_str()), + ("warnings", warning_count_s.as_str()), + ], + ); + } + result + } + + /// [`Self::decode_scaled_into`] with caller-owned scratch. + pub fn decode_scaled_into_with_scratch( + &self, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + scale: Downscale, + ) -> Result { + self.decode_into_output_format_with_scratch( + pool, + out, + stride, + output_format_from_parts(self.info.sof_kind, fmt, scale)?, + ) + } + + /// Decode the full image into rows delivered to `sink`. + /// + /// DCT-backed and 8-bit lossless color paths emit interleaved RGB8 rows. + /// Lossless 16-bit grayscale SOF3 emits little-endian Gray16 rows, and + /// supported lossless 16-bit color SOF3 emits little-endian Rgb16 rows. + pub fn decode_rows(&self, sink: &mut S) -> Result + where + S: RowSink, + { + DEFAULT_SCRATCH.with(|pool| self.decode_rows_with_scratch(&mut pool.borrow_mut(), sink)) + } + + /// [`Self::decode_rows`] with caller-owned scratch. See + /// [`Self::decode_into_with_scratch`] for the reuse contract. + pub fn decode_rows_with_scratch( + &self, + pool: &mut ScratchPool, + sink: &mut S, + ) -> Result + where + S: RowSink, + { + if self.lossless_plan.is_some() { + return self.decode_lossless_rows_with_scratch(pool, sink); + } + let width = self.info.dimensions.0 as usize; + let rows = pool.take_sink_rows(width); + let mut writer = SinkWriter::new(sink, rows, self.backend); + let result = self.decode_rgb_with_writer( + pool, + &mut writer, + DownscaleFactor::Full, + Rect::full(self.info.dimensions), + ); + pool.restore_sink_rows(writer.into_rows()); + result + } + + fn decode_lossless_rows_with_scratch( + &self, + pool: &mut ScratchPool, + sink: &mut S, + ) -> Result + where + S: RowSink, + { + let plan = self + .lossless_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: self.info.sof_kind, + })?; + if !(1..=7).contains(&plan.predictor) { + return Err(JpegError::UnsupportedPredictor { + predictor: plan.predictor, + }); + } + + let width = self.info.dimensions.0 as usize; + match (self.info.color_space, plan.bit_depth) { + (ColorSpace::Grayscale, 8) => { + let mut rows = pool.take_sink_rows(width); + let result = self.decode_lossless_gray8_rows( + sink, + &mut pool.lossless_prev_row, + &mut pool.lossless_curr_row, + &mut rows.top_row, + ); + pool.restore_sink_rows(rows); + result + } + (ColorSpace::Grayscale, 16) => self.decode_lossless_gray16_rows( + sink, + &mut pool.lossless_prev_row, + &mut pool.lossless_curr_row, + ), + (ColorSpace::Rgb, 8) => self.decode_lossless_color8_rows( + sink, + &mut pool.lossless_prev_row, + &mut pool.lossless_curr_row, + None, + ColorSpace::Rgb, + ), + (ColorSpace::YCbCr, 8) => { + let mut rows = pool.take_sink_rows(width); + let result = self.decode_lossless_color8_rows( + sink, + &mut pool.lossless_prev_row, + &mut pool.lossless_curr_row, + Some(&mut rows.top_row), + ColorSpace::YCbCr, + ); + pool.restore_sink_rows(rows); + result + } + (ColorSpace::Rgb, 16) => self.decode_lossless_color16_rows( + sink, + &mut pool.lossless_prev_row, + &mut pool.lossless_curr_row, + None, + ColorSpace::Rgb, + ), + (ColorSpace::YCbCr, 16) => { + let mut rows = pool.take_sink_rows(width); + rows.top_row.resize(width.saturating_mul(6), 0); + let result = self.decode_lossless_color16_rows( + sink, + &mut pool.lossless_prev_row, + &mut pool.lossless_curr_row, + Some(&mut rows.top_row), + ColorSpace::YCbCr, + ); + pool.restore_sink_rows(rows); + result + } + (_, depth) if depth != 8 && depth != 16 => { + Err(JpegError::UnsupportedBitDepth { depth }) + } + _ => Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }), + } + } + + /// Decode the full image into component rows. + pub fn decode_component_rows_with_scratch( + &self, + pool: &mut ScratchPool, + writer: &mut W, + ) -> Result + where + W: ComponentRowWriter, + { + self.decode_region_component_rows_with_scratch( + pool, + writer, + Rect::full(self.info.dimensions), + Downscale::None, + ) + } + + /// Decode `roi` into component rows, optionally at a reduced scale. + pub fn decode_region_component_rows_with_scratch( + &self, + pool: &mut ScratchPool, + writer: &mut W, + roi: Rect, + scale: Downscale, + ) -> Result + where + W: ComponentRowWriter, + { + if !roi.is_within(self.info.dimensions) { + return Err(JpegError::RectOutOfBounds { + rect: roi, + width: self.info.dimensions.0, + height: self.info.dimensions.1, + }); + } + + let downscale = jpeg_downscale(scale); + let scaled_roi = scaled_rect_covering(roi, downscale)?; + let mut adapter = ComponentWriterAdapter { inner: writer }; + + if roi == Rect::full(self.info.dimensions) { + self.decode_with_writer(pool, &mut adapter, downscale, roi) + } else { + let (source_x0, source_width) = + self.source_window_for_output_rect(downscale, scaled_roi); + let mut cropped = CroppedWriter::new(adapter, scaled_roi, source_x0, source_width); + self.decode_with_writer(pool, &mut cropped, downscale, roi) + } + } + + /// Decode a rectangular region of the image into the caller's buffer. + /// + /// `roi` is expressed in source-image coordinates. If `fmt` requests a + /// downscaled output, the written pixels cover the corresponding bounding + /// box in the scaled image grid. + pub fn decode_region_into( + &self, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, + ) -> Result { + DEFAULT_SCRATCH.with(|pool| { + self.decode_region_into_with_scratch(&mut pool.borrow_mut(), out, stride, fmt, roi) + }) + } + + /// Decode `roi` into a freshly allocated tightly-packed buffer. + pub fn decode_region( + &self, + fmt: PixelFormat, + roi: Rect, + ) -> Result<(Vec, DecodeOutcome), JpegError> { + DEFAULT_SCRATCH + .with(|pool| self.decode_region_with_scratch(&mut pool.borrow_mut(), fmt, roi)) + } + + /// Decode `roi` into a freshly allocated tightly-packed buffer using the + /// core `PixelFormat` + `Downscale` contract. + pub fn decode_region_scaled( + &self, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + ) -> Result<(Vec, DecodeOutcome), JpegError> { + DEFAULT_SCRATCH.with(|pool| { + self.decode_region_scaled_with_scratch(&mut pool.borrow_mut(), fmt, roi, scale) + }) + } + + /// [`Self::decode_region`] with caller-owned scratch. + pub fn decode_region_with_scratch( + &self, + pool: &mut ScratchPool, + fmt: PixelFormat, + roi: Rect, + ) -> Result<(Vec, DecodeOutcome), JpegError> { + self.decode_region_scaled_with_scratch(pool, fmt, roi, Downscale::None) + } + + /// [`Self::decode_region_scaled`] with caller-owned scratch. + pub fn decode_region_scaled_with_scratch( + &self, + pool: &mut ScratchPool, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + ) -> Result<(Vec, DecodeOutcome), JpegError> { + let legacy = output_format_from_parts(self.info.sof_kind, fmt, scale)?; + let scaled_roi = scaled_rect_covering(roi, legacy.downscale())?; + let (stride, len) = + checked_output_geometry(scaled_roi.w, scaled_roi.h, legacy.bytes_per_pixel())?; + let mut out = allocate_output_buffer(len); + let outcome = + self.decode_region_scaled_into_with_scratch(pool, &mut out, stride, fmt, roi, scale)?; + Ok((out, outcome)) + } + + /// [`Self::decode_region_into`] with caller-owned scratch. + pub fn decode_region_into_with_scratch( + &self, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, + ) -> Result { + self.decode_region_scaled_into_with_scratch(pool, out, stride, fmt, roi, Downscale::None) + } + + fn decode_region_into_output_format_with_scratch( + &self, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: OutputFormat, + roi: Rect, + ) -> Result { + let profile_enabled = jpeg_profile_stages_enabled(); + let total_start = profile_enabled.then(Instant::now); + if !roi.is_within(self.info.dimensions) { + return Err(JpegError::RectOutOfBounds { + rect: roi, + width: self.info.dimensions.0, + height: self.info.dimensions.1, + }); + } + + if roi == Rect::full(self.info.dimensions) { + return self.decode_into_output_format_with_scratch(pool, out, stride, fmt); + } + + let downscale = fmt.downscale(); + let scaled_roi = scaled_rect_covering(roi, downscale)?; + let scratch_bytes = self.decode_scratch_bytes(DEFAULT_MAX_DECODE_BYTES)?; + validate_buffer( + out, + stride, + scaled_roi.w, + scaled_roi.h, + fmt.bytes_per_pixel(), + )?; + + let decode_start = profile_enabled.then(Instant::now); + let result = match fmt { + OutputFormat::Rgb8 | OutputFormat::Rgb8Scaled { .. } => { + if self.lossless_plan.is_some() { + return match self.info.color_space { + ColorSpace::YCbCr => self + .decode_lossless_ycbcr8_region_scaled_into(out, stride, roi, downscale), + _ => self + .decode_lossless_rgb8_region_scaled_into(out, stride, roi, downscale), + }; + } + if fmt == OutputFormat::Rgb8 + && downscale == DownscaleFactor::Full + && self.progressive_plan.is_none() + && self.plan.matches_fast_tile_shape() + { + let mut writer = + Rgb8Writer::new_with_backend(out, stride, scaled_roi.w, self.backend); + let scan_bytes = &self.bytes[self.plan.scan_offset..]; + let checkpoint = self.checkpoint_for_mcu( + scan_bytes, + fast_tile_region_first_decode_mcu(&self.plan, roi, DownscaleFactor::Full), + )?; + let scan_warnings = decode_scan_fast_tile_rgb_region( + &self.plan, + self.backend, + scan_bytes, + pool, + &mut writer, + roi, + checkpoint.as_ref(), + )?; + Ok(DecodeOutcome { + decoded: roi, + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } else if matches!(fmt, OutputFormat::Rgb8Scaled { .. }) + && self.progressive_plan.is_none() + && self.plan.matches_fast_tile_shape() + { + let mut writer = + Rgb8Writer::new_with_backend(out, stride, scaled_roi.w, self.backend); + let scan_bytes = &self.bytes[self.plan.scan_offset..]; + let checkpoint = self.checkpoint_for_mcu( + scan_bytes, + fast_tile_region_first_decode_mcu(&self.plan, scaled_roi, downscale), + )?; + let scan_warnings = decode_scan_fast_tile_rgb_region_scaled( + &self.plan, + self.backend, + scan_bytes, + pool, + &mut writer, + scaled_roi, + downscale, + checkpoint.as_ref(), + )?; + Ok(DecodeOutcome { + decoded: scaled_roi, + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } else { + let base = + Rgb8Writer::new_with_backend(out, stride, scaled_roi.w, self.backend); + let (source_x0, source_width) = + self.source_window_for_output_rect(downscale, scaled_roi); + let mut writer = CroppedWriter::new(base, scaled_roi, source_x0, source_width); + self.decode_rgb_with_writer(pool, &mut writer, downscale, roi) + } + } + OutputFormat::Rgba8 { alpha } | OutputFormat::Rgba8Scaled { alpha, .. } => { + if self.lossless_plan.is_some() { + return self + .decode_lossless_rgba8_region_into(out, stride, roi, downscale, alpha); + } + let base = + Rgba8Writer::new_with_backend(out, stride, scaled_roi.w, alpha, self.backend); + let (source_x0, source_width) = + self.source_window_for_output_rect(downscale, scaled_roi); + let mut writer = CroppedWriter::new(base, scaled_roi, source_x0, source_width); + self.decode_with_writer(pool, &mut writer, downscale, roi) + } + OutputFormat::Gray8 | OutputFormat::Gray8Scaled { .. } => { + if self.lossless_plan.is_some() { + return self + .decode_lossless_gray8_region_scaled_into(out, stride, roi, downscale); + } + let base = Gray8Writer::new(out, stride, scaled_roi.w); + let (source_x0, source_width) = + self.source_window_for_output_rect(downscale, scaled_roi); + let mut writer = CroppedWriter::new(base, scaled_roi, source_x0, source_width); + self.decode_with_writer(pool, &mut writer, downscale, roi) + } + OutputFormat::Gray16 => { + if self.lossless_plan.is_some() { + return self + .decode_lossless_gray16_region_scaled_into(out, stride, roi, downscale); + } + if self.info.sof_kind == SofKind::Progressive12 { + return self.decode_progressive12_gray16_region_scaled_into( + out, stride, roi, downscale, + ); + } + self.decode_extended12_gray16_region_into(out, stride, roi) + } + OutputFormat::Gray16Scaled { .. } => { + if self.lossless_plan.is_some() { + return self + .decode_lossless_gray16_region_scaled_into(out, stride, roi, downscale); + } + if self.info.sof_kind == SofKind::Progressive12 { + return self.decode_progressive12_gray16_region_scaled_into( + out, stride, roi, downscale, + ); + } + self.decode_extended12_gray16_region_scaled_into(out, stride, roi, downscale) + } + OutputFormat::Rgb16 => { + if self.lossless_plan.is_some() { + return match self.info.color_space { + ColorSpace::YCbCr => self.decode_lossless_ycbcr16_region_scaled_into( + out, stride, roi, downscale, + ), + _ => self + .decode_lossless_rgb16_region_scaled_into(out, stride, roi, downscale), + }; + } + if self.info.sof_kind == SofKind::Progressive12 { + return self.decode_progressive12_rgb16_region_scaled_into( + out, stride, roi, downscale, + ); + } + self.decode_extended12_rgb16_region_into(out, stride, roi) + } + OutputFormat::Rgb16Scaled { .. } => { + if self.lossless_plan.is_some() { + return match self.info.color_space { + ColorSpace::YCbCr => self.decode_lossless_ycbcr16_region_scaled_into( + out, stride, roi, downscale, + ), + _ => self + .decode_lossless_rgb16_region_scaled_into(out, stride, roi, downscale), + }; + } + if self.info.sof_kind == SofKind::Progressive12 { + return self.decode_progressive12_rgb16_region_scaled_into( + out, stride, roi, downscale, + ); + } + self.decode_extended12_rgb16_region_scaled_into(out, stride, roi, downscale) + } + OutputFormat::Rgba16 { alpha } | OutputFormat::Rgba16Scaled { alpha, .. } => { + if self.lossless_plan.is_some() { + return self.decode_lossless_rgba16_region_scaled_into( + out, stride, roi, downscale, alpha, + ); + } + if matches!( + self.info.sof_kind, + SofKind::Extended12 | SofKind::Progressive12 + ) { + return self.decode_12bit_rgba16_region_scaled_into( + out, stride, roi, downscale, alpha, + ); + } + Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }) + } + }; + if let (Some(total_start), Some(decode_start), Ok(outcome)) = + (total_start, decode_start, &result) + { + let source_width_s = self.info.dimensions.0.to_string(); + let source_height_s = self.info.dimensions.1.to_string(); + let roi_x_s = roi.x.to_string(); + let roi_y_s = roi.y.to_string(); + let roi_w_s = roi.w.to_string(); + let roi_h_s = roi.h.to_string(); + let output_width_s = scaled_roi.w.to_string(); + let output_height_s = scaled_roi.h.to_string(); + let stride_s = stride.to_string(); + let bpp_s = fmt.bytes_per_pixel().to_string(); + let output_bytes_s = stride.saturating_mul(scaled_roi.h as usize).to_string(); + let scratch_bytes_s = scratch_bytes.to_string(); + let warning_count_s = outcome.warnings.len().to_string(); + let decode_us = duration_us_string(decode_start.elapsed()); + let total_us = duration_us_string(total_start.elapsed()); + let mode = if downscale == DownscaleFactor::Full { + "region" + } else { + "region_scaled" + }; + emit_jpeg_profile_row( + "decode", + "cpu", + &[ + ("mode", mode), + ("fmt", output_format_profile_name(fmt)), + ("downscale", downscale_profile_name(downscale)), + ("source_width", source_width_s.as_str()), + ("source_height", source_height_s.as_str()), + ("roi_x", roi_x_s.as_str()), + ("roi_y", roi_y_s.as_str()), + ("roi_w", roi_w_s.as_str()), + ("roi_h", roi_h_s.as_str()), + ("output_width", output_width_s.as_str()), + ("output_height", output_height_s.as_str()), + ("stride", stride_s.as_str()), + ("bpp", bpp_s.as_str()), + ("scratch_bytes", scratch_bytes_s.as_str()), + ("output_bytes", output_bytes_s.as_str()), + ("decode_us", decode_us.as_str()), + ("total_us", total_us.as_str()), + ("warnings", warning_count_s.as_str()), + ], + ); + } + result + } + + /// Decode `roi` into the caller's buffer using the core `PixelFormat` + + /// `Downscale` contract. + pub fn decode_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + ) -> Result { + DEFAULT_SCRATCH.with(|pool| { + self.decode_region_scaled_into_with_scratch( + &mut pool.borrow_mut(), + out, + stride, + fmt, + roi, + scale, + ) + }) + } + + /// [`Self::decode_region_scaled_into`] with caller-owned scratch. + pub fn decode_region_scaled_into_with_scratch( + &self, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + ) -> Result { + self.decode_region_into_output_format_with_scratch( + pool, + out, + stride, + output_format_from_parts(self.info.sof_kind, fmt, scale)?, + roi, + ) + } + + /// Decode the full image into RGBA with a caller-chosen alpha byte. + pub fn decode_rgba8_into_with_alpha( + &self, + out: &mut [u8], + stride: usize, + alpha: u8, + ) -> Result { + DEFAULT_SCRATCH.with(|pool| { + self.decode_rgba8_into_with_alpha_with_scratch( + &mut pool.borrow_mut(), + out, + stride, + alpha, + ) + }) + } + + /// [`Self::decode_rgba8_into_with_alpha`] with caller-owned scratch. + pub fn decode_rgba8_into_with_alpha_with_scratch( + &self, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + alpha: u8, + ) -> Result { + self.decode_into_output_format_with_scratch( + pool, + out, + stride, + OutputFormat::Rgba8 { alpha }, + ) + } + + /// Decode a region into RGBA with a caller-chosen alpha byte. + pub fn decode_region_rgba8_into_with_alpha( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + alpha: u8, + ) -> Result { + DEFAULT_SCRATCH.with(|pool| { + self.decode_region_rgba8_into_with_alpha_with_scratch( + &mut pool.borrow_mut(), + out, + stride, + roi, + alpha, + ) + }) + } + + /// [`Self::decode_region_rgba8_into_with_alpha`] with caller-owned scratch. + pub fn decode_region_rgba8_into_with_alpha_with_scratch( + &self, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + roi: Rect, + alpha: u8, + ) -> Result { + self.decode_region_into_output_format_with_scratch( + pool, + out, + stride, + OutputFormat::Rgba8 { alpha }, + roi, + ) + } +} + +/// One-shot parse-plus-decode of an independent JPEG tile into the caller's +/// buffer, reusing a pre-allocated [`ScratchPool`]. This is the primitive +/// WSI tile-batch readers want: one function call per tile, with all +/// heap state external. +/// +/// Parallelism is the caller's responsibility for this primitive. For +/// production batch decode, use [`decode_tiles_into`]. +/// +/// # Example +/// +/// ```no_run +/// use j2k_jpeg::{decode_tile_into, PixelFormat, ScratchPool}; +/// +/// let bytes: &[u8] = &[]; +/// let mut out = vec![0u8; 256 * 256 * 3]; +/// let mut pool = ScratchPool::new(); +/// decode_tile_into(bytes, &mut pool, &mut out, 256 * 3, PixelFormat::Rgb8)?; +/// # Ok::<(), j2k_jpeg::JpegError>(()) +/// ``` +/// +/// # Errors +/// Forwarded from [`Decoder::new`] (parse) and +/// [`Decoder::decode_into_with_scratch`] (decode). +pub fn decode_tile_into( + bytes: &[u8], + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, +) -> Result { + DEFAULT_CONTEXT.with(|ctx| { + decode_tile_into_in_context(bytes, &mut ctx.borrow_mut(), pool, out, stride, fmt) + }) +} + +/// One-shot parse-plus-decode of an independent JPEG tile into the caller's +/// buffer, reusing both caller-owned [`DecoderContext`] and caller-owned +/// [`ScratchPool`]. +pub fn decode_tile_into_in_context( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, +) -> Result { + decode_tile_into_in_context_with_options( + bytes, + ctx, + pool, + out, + stride, + fmt, + DecodeOptions::default(), + ) +} + +/// One-shot parse-plus-decode of an independent JPEG tile into the caller's +/// buffer, reusing both caller-owned [`DecoderContext`] and caller-owned +/// [`ScratchPool`], with explicit JPEG decode options. +pub fn decode_tile_into_in_context_with_options( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + options: DecodeOptions, +) -> Result { + let dec = Decoder::from_view_in_context(JpegView::parse_with_options(bytes, options)?, ctx)?; + dec.decode_into_with_scratch(pool, out, stride, fmt) +} + +pub(crate) fn decode_prepared_jpeg_tile_rgb8_in_context( + input: &PreparedJpeg<'_>, + ctx: &mut DecoderContext, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + options: DecodeOptions, +) -> Result { + let view = JpegView::parse_with_options(input.as_bytes(), options)?; + let dimensions = view.info().dimensions; + let dec = Decoder::from_view_in_context(view, ctx)?; + let outcome = dec.decode_into_with_scratch(pool, out, stride, PixelFormat::Rgb8)?; + Ok(DecodedTile { + dimensions, + decoded: outcome.decoded, + warnings: outcome.warnings, + }) +} + +/// Decode prepared TIFF/WSI JPEG tiles into caller-owned RGB8 output buffers. +/// +/// Returned results preserve the caller's input order. Each job carries its +/// own [`DecodeOptions`], allowing container metadata to resolve RGB/YCbCr +/// interpretation independently per tile. +#[must_use] +pub fn decode_prepared_jpeg_tiles_rgb8( + jobs: &mut [PreparedJpegTileJob<'_, '_>], +) -> Vec> { + crate::JpegBatchSession::new_one_shot(TileBatchOptions::default()) + .decode_prepared_jpeg_tiles_rgb8(jobs) +} + +/// Decode independent JPEG tiles into caller-owned output buffers using a +/// scoped CPU worker pool. +/// +/// Each worker owns one [`DecoderContext`] and one [`ScratchPool`], so repeated +/// tiles reuse parsed table state and heap scratch within that worker without +/// sharing mutable decoder state across threads. Returned outcomes preserve +/// the caller's input order. +/// +/// # Errors +/// Returns [`TileBatchError`] with the first failing tile index in input order. +pub fn decode_tiles_into( + jobs: &mut [TileDecodeJob<'_, '_>], + fmt: PixelFormat, + options: TileBatchOptions, +) -> Result, TileBatchError> { + decode_tiles_into_with_options(jobs, fmt, DecodeOptions::default(), options) +} + +/// Decode independent JPEG tiles into caller-owned output buffers using a +/// scoped CPU worker pool and explicit JPEG decode options. +/// +/// Use this variant when container metadata has already resolved ambiguous +/// three-component JPEG data to RGB or YCbCr via [`DecodeOptions`]. +/// +/// # Errors +/// Returns [`TileBatchError`] with the first failing tile index in input order. +pub fn decode_tiles_into_with_options( + jobs: &mut [TileDecodeJob<'_, '_>], + fmt: PixelFormat, + decode_options: DecodeOptions, + options: TileBatchOptions, +) -> Result, TileBatchError> { + crate::JpegBatchSession::new_one_shot(options).decode_tiles_into_with_options( + jobs, + fmt, + decode_options, + ) +} + +/// Decode independent JPEG tiles at reduced resolution into caller-owned +/// output buffers using a scoped CPU worker pool. +/// +/// Each worker owns one [`DecoderContext`] and one [`ScratchPool`], so repeated +/// tiles reuse parsed table state and heap scratch within that worker without +/// sharing mutable decoder state across threads. Returned outcomes preserve +/// the caller's input order. +/// +/// # Errors +/// Returns [`TileBatchError`] with the first failing tile index in input order. +pub fn decode_tiles_scaled_into( + jobs: &mut [TileScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + options: TileBatchOptions, +) -> Result, TileBatchError> { + decode_tiles_scaled_into_with_options(jobs, fmt, DecodeOptions::default(), options) +} + +/// Decode independent JPEG tiles at reduced resolution into caller-owned +/// output buffers using a scoped CPU worker pool and explicit JPEG decode +/// options. +/// +/// # Errors +/// Returns [`TileBatchError`] with the first failing tile index in input order. +pub fn decode_tiles_scaled_into_with_options( + jobs: &mut [TileScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + decode_options: DecodeOptions, + options: TileBatchOptions, +) -> Result, TileBatchError> { + crate::JpegBatchSession::new_one_shot(options).decode_tiles_scaled_into_with_options( + jobs, + fmt, + decode_options, + ) +} + +/// Decode independent JPEG tile regions at reduced resolution into +/// caller-owned output buffers using a scoped CPU worker pool. +/// +/// Each worker owns one [`DecoderContext`] and one [`ScratchPool`], so repeated +/// tiles reuse parsed table state and heap scratch within that worker without +/// sharing mutable decoder state across threads. Returned outcomes preserve +/// the caller's input order. +/// +/// # Errors +/// Returns [`TileBatchError`] with the first failing tile index in input order. +pub fn decode_tiles_region_scaled_into( + jobs: &mut [TileRegionScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + options: TileBatchOptions, +) -> Result, TileBatchError> { + decode_tiles_region_scaled_into_with_options(jobs, fmt, DecodeOptions::default(), options) +} + +/// Decode independent JPEG tile regions at reduced resolution into +/// caller-owned output buffers using a scoped CPU worker pool and explicit JPEG +/// decode options. +/// +/// # Errors +/// Returns [`TileBatchError`] with the first failing tile index in input order. +pub fn decode_tiles_region_scaled_into_with_options( + jobs: &mut [TileRegionScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + decode_options: DecodeOptions, + options: TileBatchOptions, +) -> Result, TileBatchError> { + crate::JpegBatchSession::new_one_shot(options).decode_tiles_region_scaled_into_with_options( + jobs, + fmt, + decode_options, + ) +} + +/// One-shot parse-plus-region-decode of an independent JPEG tile into the +/// caller's buffer, reusing both caller-owned [`DecoderContext`] and +/// caller-owned [`ScratchPool`]. +pub fn decode_tile_region_into_in_context( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, +) -> Result { + decode_tile_region_into_in_context_with_options( + bytes, + ctx, + pool, + out, + stride, + fmt, + roi, + DecodeOptions::default(), + ) +} + +/// One-shot parse-plus-region-decode of an independent JPEG tile into the +/// caller's buffer, reusing caller-owned state and explicit JPEG decode +/// options. +#[allow(clippy::too_many_arguments)] +pub fn decode_tile_region_into_in_context_with_options( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, + options: DecodeOptions, +) -> Result { + let dec = Decoder::from_view_in_context(JpegView::parse_with_options(bytes, options)?, ctx)?; + dec.decode_region_into_with_scratch(pool, out, stride, fmt, roi) +} + +/// One-shot parse-plus-scaled-decode of an independent JPEG tile into the +/// caller's buffer, reusing both caller-owned [`DecoderContext`] and +/// caller-owned [`ScratchPool`]. +pub fn decode_tile_scaled_into_in_context( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + scale: Downscale, +) -> Result { + decode_tile_scaled_into_in_context_with_options( + bytes, + ctx, + pool, + out, + stride, + fmt, + scale, + DecodeOptions::default(), + ) +} + +/// One-shot parse-plus-scaled-decode of an independent JPEG tile into the +/// caller's buffer, reusing caller-owned state and explicit JPEG decode +/// options. +#[allow(clippy::too_many_arguments)] +pub fn decode_tile_scaled_into_in_context_with_options( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + scale: Downscale, + options: DecodeOptions, +) -> Result { + let dec = Decoder::from_view_in_context(JpegView::parse_with_options(bytes, options)?, ctx)?; + dec.decode_scaled_into_with_scratch(pool, out, stride, fmt, scale) +} + +/// One-shot parse-plus-region-scaled-decode of an independent JPEG tile into +/// the caller's buffer, reusing both caller-owned [`DecoderContext`] and +/// caller-owned [`ScratchPool`]. +#[allow(clippy::too_many_arguments)] +pub fn decode_tile_region_scaled_into_in_context( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, +) -> Result { + decode_tile_region_scaled_into_in_context_with_options( + bytes, + ctx, + pool, + out, + stride, + fmt, + roi, + scale, + DecodeOptions::default(), + ) +} + +/// One-shot parse-plus-region-scaled-decode of an independent JPEG tile into +/// the caller's buffer, reusing caller-owned state and explicit JPEG decode +/// options. +#[allow(clippy::too_many_arguments)] +pub fn decode_tile_region_scaled_into_in_context_with_options( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut ScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + options: DecodeOptions, +) -> Result { + let dec = Decoder::from_view_in_context(JpegView::parse_with_options(bytes, options)?, ctx)?; + dec.decode_region_scaled_into_with_scratch(pool, out, stride, fmt, roi, scale) +} + +impl Decoder<'_> { + /// One-shot parse-plus-row-decode of a JPEG tile using caller-owned shared + /// table context and caller-owned scratch. + pub fn decode_tile( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut ScratchPool, + sink: &mut S, + ) -> Result + where + S: RowSink, + { + let dec = Decoder::from_view_in_context(JpegView::parse(bytes)?, ctx)?; + dec.decode_rows_with_scratch(pool, sink) + } +} + +impl Decoder<'_> { + fn decode_scratch_bytes(&self, cap: usize) -> Result { + let scratch_bytes = self + .progressive_plan + .as_ref() + .map_or(self.plan.scratch_bytes, |plan| plan.scratch_bytes); + if scratch_bytes > cap { + return Err(JpegError::MemoryCapExceeded { + requested: scratch_bytes, + cap, + }); + } + Ok(scratch_bytes) + } + + fn checkpoint_for_mcu( + &self, + scan_bytes: &[u8], + target_mcu: u32, + ) -> Result, JpegError> { + if self.plan.restart_interval.is_some() || target_mcu < CPU_ROI_CHECKPOINT_MIN_TARGET_MCUS { + return Ok(None); + } + + let mut cache = self + .cpu_entropy_checkpoints + .lock() + .expect("CPU entropy checkpoint cache mutex poisoned"); + checkpoint_before_mcu( + &self.plan, + scan_bytes, + CPU_ROI_CHECKPOINT_CADENCE_MCUS, + target_mcu, + &mut cache, + ) + } + + fn source_window_for_output_rect( + &self, + downscale: DownscaleFactor, + output_rect: Rect, + ) -> (u32, u32) { + if self.progressive_plan.is_some() { + return (0, scaled_dimensions(self.info.dimensions, downscale).0); + } + let layout = stripe_region_layout(&self.plan, downscale, output_rect); + (layout.source_x0, layout.source_width) + } + + fn decode_with_writer( + &self, + pool: &mut ScratchPool, + writer: &mut W, + downscale: DownscaleFactor, + decoded: Rect, + ) -> Result { + let _ = self.decode_scratch_bytes(DEFAULT_MAX_DECODE_BYTES)?; + let profile_enabled = jpeg_profile_stages_enabled(); + if let Some(plan) = &self.progressive_plan { + let scan_start = profile_enabled.then(Instant::now); + let scan_warnings = if downscale == DownscaleFactor::Full { + decode_progressive(plan, self.backend, self.bytes, writer)? + } else { + let mut scaled = + ProgressiveDownscaleWriter::new(writer, downscale, self.info.dimensions); + decode_progressive(plan, self.backend, self.bytes, &mut scaled)? + }; + if let Some(start) = scan_start { + emit_decode_scan_profile( + "progressive", + self.info.dimensions, + decoded, + downscale, + start.elapsed(), + ); + } + return Ok(DecodeOutcome { + decoded, + warnings: merged_warnings(&self.warnings, scan_warnings), + }); + } + let output_rect = scaled_rect_covering(decoded, downscale)?; + let scan_bytes = &self.bytes[self.plan.scan_offset..]; + let scan_start = profile_enabled.then(Instant::now); + let scan_warnings = decode_scan_baseline( + &self.plan, + self.backend, + scan_bytes, + pool, + writer, + downscale, + output_rect, + )?; + if let Some(start) = scan_start { + emit_decode_scan_profile( + "baseline", + self.info.dimensions, + decoded, + downscale, + start.elapsed(), + ); + } + Ok(DecodeOutcome { + decoded, + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } + + fn decode_rgb_with_writer( + &self, + pool: &mut ScratchPool, + writer: &mut W, + downscale: DownscaleFactor, + decoded: Rect, + ) -> Result { + let _ = self.decode_scratch_bytes(DEFAULT_MAX_DECODE_BYTES)?; + let profile_enabled = jpeg_profile_stages_enabled(); + if let Some(plan) = &self.progressive_plan { + let scan_start = profile_enabled.then(Instant::now); + let scan_warnings = if downscale == DownscaleFactor::Full { + decode_progressive(plan, self.backend, self.bytes, writer)? + } else { + let mut scaled = + ProgressiveDownscaleWriter::new(writer, downscale, self.info.dimensions); + decode_progressive(plan, self.backend, self.bytes, &mut scaled)? + }; + if let Some(start) = scan_start { + emit_decode_scan_profile( + "progressive_rgb", + self.info.dimensions, + decoded, + downscale, + start.elapsed(), + ); + } + return Ok(DecodeOutcome { + decoded, + warnings: merged_warnings(&self.warnings, scan_warnings), + }); + } + let output_rect = scaled_rect_covering(decoded, downscale)?; + let scan_bytes = &self.bytes[self.plan.scan_offset..]; + let scan_start = profile_enabled.then(Instant::now); + let (scan_path, scan_warnings) = + if downscale == DownscaleFactor::Full && self.plan.matches_fast_tile_shape() { + ( + "fast420_rgb", + decode_scan_fast_tile_rgb(&self.plan, self.backend, scan_bytes, pool, writer)?, + ) + } else if downscale == DownscaleFactor::Full + && decoded == Rect::full(self.info.dimensions) + && self.plan.matches_fast_rgb444_shape() + { + ( + "fast444_rgb", + decode_scan_fast_rgb_444(&self.plan, self.backend, scan_bytes, pool, writer)?, + ) + } else { + ( + "baseline_rgb", + decode_scan_baseline_rgb( + &self.plan, + self.backend, + scan_bytes, + pool, + writer, + downscale, + output_rect, + )?, + ) + }; + if let Some(start) = scan_start { + emit_decode_scan_profile( + scan_path, + self.info.dimensions, + decoded, + downscale, + start.elapsed(), + ); + } + Ok(DecodeOutcome { + decoded, + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } + + fn decode_lossless_gray8_into( + &self, + out: &mut [u8], + stride: usize, + ) -> Result { + let plan = self + .lossless_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: self.info.sof_kind, + })?; + if plan.bit_depth != 8 { + return Err(JpegError::UnsupportedBitDepth { + depth: plan.bit_depth, + }); + } + if !(1..=7).contains(&plan.predictor) { + return Err(JpegError::UnsupportedPredictor { + predictor: plan.predictor, + }); + } + + let (width, height) = plan.dimensions; + let scan_bytes = &self.bytes[plan.scan_offset..]; + let mut br = BitReader::new(scan_bytes); + let restart = u32::from(self.plan.restart_interval.unwrap_or(0)); + let total_samples = width.saturating_mul(height); + let mut samples_since_restart = 0u32; + let mut expected_rst = 0u8; + for y in 0..height as usize { + for x in 0..width as usize { + let sample_index = y as u32 * width + x as u32; + if restart > 0 && samples_since_restart == restart { + consume_lossless_restart( + &mut br, + sample_index, + total_samples, + &mut expected_rst, + )?; + samples_since_restart = 0; + } + let predictor = if restart > 0 && samples_since_restart == 0 { + 128 + } else { + lossless_predictor_value(plan.predictor, out, stride, x, y) + }; + let diff = plan.dc_table.decode_fast_dc(&mut br)?; + let sample = ::from_i32(predictor + diff)?; + out[y * stride + x] = sample; + samples_since_restart += 1; + } + } + + let scan_warnings = finish_scan(&mut br, true)?; + Ok(DecodeOutcome { + decoded: Rect::full(self.info.dimensions), + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } + + fn decode_lossless_gray8_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + ) -> Result { + if roi == Rect::full(self.info.dimensions) && downscale == DownscaleFactor::Full { + return self.decode_lossless_gray8_into(out, stride); + } + + let (width, height) = self.info.dimensions; + let full_stride = width as usize; + let mut full = + allocate_output_buffer(checked_scratch_len(&[full_stride, height as usize])?); + let mut outcome = self.decode_lossless_gray8_into(&mut full, full_stride)?; + let output_rect = scaled_rect_covering(roi, downscale)?; + copy_gray8_scaled_rect( + &full, + (width, height), + output_rect, + downscale.denominator(), + out, + stride, + ); + outcome.decoded = roi; + Ok(outcome) + } + + fn decode_lossless_gray_rows( + &self, + sink: &mut S, + prev_row: &mut Vec, + curr_row: &mut Vec, + mut emit_row: impl FnMut(&mut S, u32, &[u8]) -> Result<(), JpegError>, + ) -> Result + where + P: LosslessSample, + S: RowSink, + { + let plan = self + .lossless_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: self.info.sof_kind, + })?; + if plan.bit_depth != P::BIT_DEPTH { + return Err(JpegError::UnsupportedBitDepth { + depth: plan.bit_depth, + }); + } + if self.info.color_space != ColorSpace::Grayscale { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + if !(1..=7).contains(&plan.predictor) { + return Err(JpegError::UnsupportedPredictor { + predictor: plan.predictor, + }); + } + + let (width, height) = plan.dimensions; + let width = width as usize; + let row_len = width.saturating_mul(P::BYTES); + prev_row.resize(row_len, 0); + curr_row.resize(row_len, 0); + + let scan_bytes = &self.bytes[plan.scan_offset..]; + let mut br = BitReader::new(scan_bytes); + let restart = u32::from(self.plan.restart_interval.unwrap_or(0)); + let total_samples = plan.dimensions.0.saturating_mul(height); + let mut samples_since_restart = 0u32; + let mut expected_rst = 0u8; + for y in 0..height as usize { + for x in 0..width { + let sample_index = y as u32 * plan.dimensions.0 + x as u32; + if restart > 0 && samples_since_restart == restart { + consume_lossless_restart( + &mut br, + sample_index, + total_samples, + &mut expected_rst, + )?; + samples_since_restart = 0; + } + let predictor = if restart > 0 && samples_since_restart == 0 { + P::RESTART_PREDICTOR + } else { + lossless_predictor_gray_rows::

(plan.predictor, curr_row, prev_row, x, y) + }; + let diff = plan.dc_table.decode_fast_dc(&mut br)?; + let sample = P::from_i32(predictor + diff)?; + sample.write_le(&mut curr_row[x * P::BYTES..]); + samples_since_restart += 1; + } + emit_row(sink, y as u32, &curr_row[..row_len])?; + core::mem::swap(prev_row, curr_row); + } + + let scan_warnings = finish_scan(&mut br, true)?; + Ok(DecodeOutcome { + decoded: Rect::full(self.info.dimensions), + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } + + fn decode_lossless_gray8_rows( + &self, + sink: &mut S, + prev_row: &mut Vec, + curr_row: &mut Vec, + rgb_row: &mut [u8], + ) -> Result + where + S: RowSink, + { + self.decode_lossless_gray_rows::(sink, prev_row, curr_row, |sink, y, gray_row| { + let rgb_len = gray_row.len().saturating_mul(3); + if rgb_row.len() < rgb_len { + return Err(JpegError::OutputBufferTooSmall { + required: rgb_len, + provided: rgb_row.len(), + }); + } + for (pixel, &sample) in rgb_row[..rgb_len].chunks_exact_mut(3).zip(gray_row.iter()) { + pixel.copy_from_slice(&[sample, sample, sample]); + } + sink.write_row(y, &rgb_row[..rgb_len]) + }) + } + + fn decode_lossless_rgb8_into( + &self, + out: &mut [u8], + stride: usize, + ) -> Result { + match lossless_color_sampling(&self.info) { + Some(LosslessColorSampling::S444) => { + self.decode_lossless_color8_components_into(out, stride, ColorSpace::Rgb) + } + Some(LosslessColorSampling::S422 | LosslessColorSampling::S420) => { + self.decode_lossless_color8_sampled_into(out, stride, ColorSpace::Rgb) + } + None => Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }), + } + } + + fn decode_lossless_color_components_into

( + &self, + out: &mut [u8], + stride: usize, + color_space: ColorSpace, + ) -> Result + where + P: LosslessSample, + { + let plan = self + .lossless_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: self.info.sof_kind, + })?; + if plan.bit_depth != P::BIT_DEPTH { + return Err(JpegError::UnsupportedBitDepth { + depth: plan.bit_depth, + }); + } + if self.info.color_space != color_space { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + if self.plan.components.len() != 3 { + return Err(JpegError::UnsupportedComponentCount { + count: self.plan.components.len() as u8, + }); + } + if !(1..=7).contains(&plan.predictor) { + return Err(JpegError::UnsupportedPredictor { + predictor: plan.predictor, + }); + } + + let (width, height) = plan.dimensions; + let scan_bytes = &self.bytes[plan.scan_offset..]; + let mut br = BitReader::new(scan_bytes); + let restart = u32::from(self.plan.restart_interval.unwrap_or(0)); + let total_pixels = width.saturating_mul(height); + let mut pixels_since_restart = 0u32; + let mut expected_rst = 0u8; + for y in 0..height as usize { + for x in 0..width as usize { + let pixel_index = y as u32 * width + x as u32; + if restart > 0 && pixels_since_restart == restart { + consume_lossless_restart( + &mut br, + pixel_index, + total_pixels, + &mut expected_rst, + )?; + pixels_since_restart = 0; + } + for component in &self.plan.components { + if component.output_index >= 3 { + return Err(JpegError::UnsupportedComponentCount { + count: self.plan.components.len() as u8, + }); + } + let predictor = if restart > 0 && pixels_since_restart == 0 { + P::RESTART_PREDICTOR + } else { + lossless_predictor_color_into::

( + plan.predictor, + out, + stride, + x, + y, + component.output_index, + ) + }; + let diff = component.dc_table.decode_fast_dc(&mut br)?; + let sample = P::from_i32(predictor + diff)?; + let offset = y * stride + (x * 3 + component.output_index) * P::BYTES; + sample.write_le(&mut out[offset..]); + } + pixels_since_restart += 1; + } + } + + let scan_warnings = finish_scan(&mut br, true)?; + Ok(DecodeOutcome { + decoded: Rect::full(self.info.dimensions), + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } + + fn decode_lossless_color8_components_into( + &self, + out: &mut [u8], + stride: usize, + color_space: ColorSpace, + ) -> Result { + self.decode_lossless_color_components_into::(out, stride, color_space) + } + + fn decode_lossless_color_sampled_into

( + &self, + out: &mut [u8], + stride: usize, + color_space: ColorSpace, + write_output: impl FnOnce( + &mut [u8], + usize, + ColorSpace, + LosslessColorSampling, + (usize, usize), + LosslessColorPlanes<'_, P>, + ), + ) -> Result + where + P: LosslessSample, + { + let plan = self + .lossless_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: self.info.sof_kind, + })?; + if plan.bit_depth != P::BIT_DEPTH { + return Err(JpegError::UnsupportedBitDepth { + depth: plan.bit_depth, + }); + } + if self.info.color_space != color_space { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + let sampling = lossless_color_sampling(&self.info).ok_or(JpegError::NotImplemented { + sof: self.info.sof_kind, + })?; + if !matches!( + sampling, + LosslessColorSampling::S422 | LosslessColorSampling::S420 + ) { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + if self.plan.components.len() != 3 { + return Err(JpegError::UnsupportedComponentCount { + count: self.plan.components.len() as u8, + }); + } + if !(1..=7).contains(&plan.predictor) { + return Err(JpegError::UnsupportedPredictor { + predictor: plan.predictor, + }); + } + + let (width, height) = plan.dimensions; + let width = width as usize; + let height = height as usize; + let chroma_width = width.div_ceil(self.info.sampling.max_h as usize); + let chroma_height = height.div_ceil(self.info.sampling.max_v as usize); + let mut c0 = vec![P::default(); width * height]; + let mut c1 = vec![P::default(); chroma_width * chroma_height]; + let mut c2 = vec![P::default(); chroma_width * chroma_height]; + + let scan_bytes = &self.bytes[plan.scan_offset..]; + let mut br = BitReader::new(scan_bytes); + let restart = u32::from(self.plan.restart_interval.unwrap_or(0)); + let total_mcus = (chroma_width * chroma_height) as u32; + let mut mcus_since_restart = 0u32; + let mut expected_rst = 0u8; + for mcu_y in 0..chroma_height { + for mcu_x in 0..chroma_width { + let mcu_index = (mcu_y * chroma_width + mcu_x) as u32; + if restart > 0 && mcus_since_restart == restart { + consume_lossless_restart(&mut br, mcu_index, total_mcus, &mut expected_rst)?; + mcus_since_restart = 0; + } + for component in &self.plan.components { + let (plane, plane_width, plane_height) = match component.output_index { + 0 => (&mut c0, width, height), + 1 => (&mut c1, chroma_width, chroma_height), + 2 => (&mut c2, chroma_width, chroma_height), + _ => { + return Err(JpegError::UnsupportedComponentCount { + count: self.plan.components.len() as u8, + }); + } + }; + for local_y in 0..component.v as usize { + for local_x in 0..component.h as usize { + let x = mcu_x * component.h as usize + local_x; + let y = mcu_y * component.v as usize + local_y; + if x >= plane_width || y >= plane_height { + continue; + } + decode_lossless_plane_sample( + &mut br, + &component.dc_table, + plan.predictor, + plane, + plane_width, + LosslessPlaneSample { + x, + y, + restart_first_sample: restart > 0 + && mcus_since_restart == 0 + && local_x == 0 + && local_y == 0, + }, + )?; + } + } + } + mcus_since_restart += 1; + } + } + + let scan_warnings = finish_scan(&mut br, true)?; + write_output( + out, + stride, + color_space, + sampling, + (width, height), + LosslessColorPlanes { + c0: &c0, + c1: &c1, + c2: &c2, + }, + ); + Ok(DecodeOutcome { + decoded: Rect::full(self.info.dimensions), + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } + + fn decode_lossless_color8_sampled_into( + &self, + out: &mut [u8], + stride: usize, + color_space: ColorSpace, + ) -> Result { + self.decode_lossless_color_sampled_into::( + out, + stride, + color_space, + write_lossless_color8_sampled_output, + ) + } + + fn decode_lossless_ycbcr8_into( + &self, + out: &mut [u8], + stride: usize, + ) -> Result { + match lossless_color_sampling(&self.info) { + Some(LosslessColorSampling::S444) => { + let outcome = + self.decode_lossless_color8_components_into(out, stride, ColorSpace::YCbCr)?; + convert_ycbcr8_to_rgb8_in_place(out, stride, self.info.dimensions); + Ok(outcome) + } + Some(LosslessColorSampling::S422 | LosslessColorSampling::S420) => { + self.decode_lossless_color8_sampled_into(out, stride, ColorSpace::YCbCr) + } + None => Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }), + } + } + + fn decode_lossless_rgb8_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + ) -> Result { + if roi == Rect::full(self.info.dimensions) && downscale == DownscaleFactor::Full { + return self.decode_lossless_rgb8_into(out, stride); + } + + let (width, height) = self.info.dimensions; + let full_stride = width as usize * 3; + let mut full = + allocate_output_buffer(checked_scratch_len(&[full_stride, height as usize])?); + let mut outcome = self.decode_lossless_rgb8_into(&mut full, full_stride)?; + let output_rect = scaled_rect_covering(roi, downscale)?; + copy_rgb8_scaled_rect( + &full, + (width, height), + output_rect, + downscale.denominator(), + out, + stride, + ); + outcome.decoded = roi; + Ok(outcome) + } + + fn decode_lossless_ycbcr8_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + ) -> Result { + if roi == Rect::full(self.info.dimensions) && downscale == DownscaleFactor::Full { + return self.decode_lossless_ycbcr8_into(out, stride); + } + + let (width, height) = self.info.dimensions; + let full_stride = width as usize * 3; + let mut full = + allocate_output_buffer(checked_scratch_len(&[full_stride, height as usize])?); + let mut outcome = self.decode_lossless_ycbcr8_into(&mut full, full_stride)?; + let output_rect = scaled_rect_covering(roi, downscale)?; + copy_rgb8_scaled_rect( + &full, + (width, height), + output_rect, + downscale.denominator(), + out, + stride, + ); + outcome.decoded = roi; + Ok(outcome) + } + + fn decode_lossless_rgba8_region_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + alpha: u8, + ) -> Result { + let output_rect = scaled_rect_covering(roi, downscale)?; + let rgb_stride = output_rect.w as usize * 3; + let mut rgb = + allocate_output_buffer(checked_scratch_len(&[rgb_stride, output_rect.h as usize])?); + let outcome = match self.info.color_space { + ColorSpace::YCbCr => { + self.decode_lossless_ycbcr8_region_scaled_into(&mut rgb, rgb_stride, roi, downscale) + } + _ => self.decode_lossless_rgb8_region_scaled_into(&mut rgb, rgb_stride, roi, downscale), + }?; + copy_rgb8_to_rgba8( + &rgb, + rgb_stride, + output_rect.w, + output_rect.h, + out, + stride, + alpha, + ); + Ok(outcome) + } + + fn decode_lossless_color_rows( + &self, + sink: &mut S, + prev_row: &mut Vec, + curr_row: &mut Vec, + conversion_row: Option<&mut [u8]>, + color_space: ColorSpace, + convert_row: impl Fn(&[u8], &mut [u8]), + ) -> Result + where + P: LosslessSample, + S: RowSink, + { + let plan = self + .lossless_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: self.info.sof_kind, + })?; + if plan.bit_depth != P::BIT_DEPTH { + return Err(JpegError::UnsupportedBitDepth { + depth: plan.bit_depth, + }); + } + if self.info.color_space != color_space { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + if self.plan.components.len() != 3 { + return Err(JpegError::UnsupportedComponentCount { + count: self.plan.components.len() as u8, + }); + } + if !(1..=7).contains(&plan.predictor) { + return Err(JpegError::UnsupportedPredictor { + predictor: plan.predictor, + }); + } + + let (width, height) = plan.dimensions; + let width = width as usize; + let row_len = width.saturating_mul(3 * P::BYTES); + prev_row.resize(row_len, 0); + curr_row.resize(row_len, 0); + + let scan_bytes = &self.bytes[plan.scan_offset..]; + let mut br = BitReader::new(scan_bytes); + let restart = u32::from(self.plan.restart_interval.unwrap_or(0)); + let total_pixels = plan.dimensions.0.saturating_mul(height); + let mut pixels_since_restart = 0u32; + let mut expected_rst = 0u8; + let mut conversion_row = conversion_row; + for y in 0..height as usize { + for x in 0..width { + let pixel_index = y as u32 * plan.dimensions.0 + x as u32; + if restart > 0 && pixels_since_restart == restart { + consume_lossless_restart( + &mut br, + pixel_index, + total_pixels, + &mut expected_rst, + )?; + pixels_since_restart = 0; + } + for component in &self.plan.components { + if component.output_index >= 3 { + return Err(JpegError::UnsupportedComponentCount { + count: self.plan.components.len() as u8, + }); + } + let predictor = if restart > 0 && pixels_since_restart == 0 { + P::RESTART_PREDICTOR + } else { + lossless_predictor_color_rows::

( + plan.predictor, + curr_row, + prev_row, + x, + y, + component.output_index, + ) + }; + let diff = component.dc_table.decode_fast_dc(&mut br)?; + let sample = P::from_i32(predictor + diff)?; + let offset = (x * 3 + component.output_index) * P::BYTES; + sample.write_le(&mut curr_row[offset..]); + } + pixels_since_restart += 1; + } + let row = if color_space == ColorSpace::YCbCr { + let row = conversion_row + .as_deref_mut() + .ok_or(JpegError::OutputBufferTooSmall { + required: row_len, + provided: 0, + })?; + if row.len() < row_len { + return Err(JpegError::OutputBufferTooSmall { + required: row_len, + provided: row.len(), + }); + } + convert_row(&curr_row[..row_len], &mut row[..row_len]); + &row[..row_len] + } else { + &curr_row[..row_len] + }; + sink.write_row(y as u32, row)?; + core::mem::swap(prev_row, curr_row); + } + + let scan_warnings = finish_scan(&mut br, true)?; + Ok(DecodeOutcome { + decoded: Rect::full(self.info.dimensions), + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } + + fn decode_lossless_color8_rows( + &self, + sink: &mut S, + prev_row: &mut Vec, + curr_row: &mut Vec, + conversion_row: Option<&mut [u8]>, + color_space: ColorSpace, + ) -> Result + where + S: RowSink, + { + self.decode_lossless_color_rows::( + sink, + prev_row, + curr_row, + conversion_row, + color_space, + copy_ycbcr8_row_to_rgb8, + ) + } + + fn decode_lossless_gray16_rows( + &self, + sink: &mut S, + prev_row: &mut Vec, + curr_row: &mut Vec, + ) -> Result + where + S: RowSink, + { + self.decode_lossless_gray_rows::(sink, prev_row, curr_row, |sink, y, row| { + sink.write_row(y, row) + }) + } + + fn decode_lossless_color16_rows( + &self, + sink: &mut S, + prev_row: &mut Vec, + curr_row: &mut Vec, + conversion_row: Option<&mut [u8]>, + color_space: ColorSpace, + ) -> Result + where + S: RowSink, + { + self.decode_lossless_color_rows::( + sink, + prev_row, + curr_row, + conversion_row, + color_space, + copy_ycbcr16_row_to_rgb16, + ) + } + + fn decode_lossless_rgb16_into( + &self, + out: &mut [u8], + stride: usize, + ) -> Result { + match lossless_color_sampling(&self.info) { + Some(LosslessColorSampling::S444) => { + self.decode_lossless_color16_components_into(out, stride, ColorSpace::Rgb) + } + Some(LosslessColorSampling::S422) => { + self.decode_lossless_color16_sampled_into(out, stride, ColorSpace::Rgb) + } + Some(LosslessColorSampling::S420) => { + self.decode_lossless_color16_sampled_into(out, stride, ColorSpace::Rgb) + } + None => Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }), + } + } + + fn decode_lossless_color16_components_into( + &self, + out: &mut [u8], + stride: usize, + color_space: ColorSpace, + ) -> Result { + self.decode_lossless_color_components_into::(out, stride, color_space) + } + + fn decode_lossless_color16_sampled_into( + &self, + out: &mut [u8], + stride: usize, + color_space: ColorSpace, + ) -> Result { + self.decode_lossless_color_sampled_into::( + out, + stride, + color_space, + write_lossless_color16_sampled_output, + ) + } + + fn decode_lossless_ycbcr16_into( + &self, + out: &mut [u8], + stride: usize, + ) -> Result { + match lossless_color_sampling(&self.info) { + Some(LosslessColorSampling::S444) => { + let outcome = + self.decode_lossless_color16_components_into(out, stride, ColorSpace::YCbCr)?; + convert_ycbcr16_to_rgb16_in_place(out, stride, self.info.dimensions); + Ok(outcome) + } + Some(LosslessColorSampling::S422) => { + self.decode_lossless_color16_sampled_into(out, stride, ColorSpace::YCbCr) + } + Some(LosslessColorSampling::S420) => { + self.decode_lossless_color16_sampled_into(out, stride, ColorSpace::YCbCr) + } + None => Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }), + } + } + + fn decode_lossless_rgb16_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + ) -> Result { + if roi == Rect::full(self.info.dimensions) && downscale == DownscaleFactor::Full { + return self.decode_lossless_rgb16_into(out, stride); + } + + let (width, height) = self.info.dimensions; + let full_stride = width as usize * 6; + let mut full = + allocate_output_buffer(checked_scratch_len(&[full_stride, height as usize])?); + let mut outcome = self.decode_lossless_rgb16_into(&mut full, full_stride)?; + let output_rect = scaled_rect_covering(roi, downscale)?; + copy_rgb16_scaled_rect( + &full, + (width, height), + output_rect, + downscale.denominator(), + out, + stride, + ); + outcome.decoded = roi; + Ok(outcome) + } + + fn decode_lossless_ycbcr16_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + ) -> Result { + if roi == Rect::full(self.info.dimensions) && downscale == DownscaleFactor::Full { + return self.decode_lossless_ycbcr16_into(out, stride); + } + + let (width, height) = self.info.dimensions; + let full_stride = width as usize * 6; + let mut full = + allocate_output_buffer(checked_scratch_len(&[full_stride, height as usize])?); + let mut outcome = self.decode_lossless_ycbcr16_into(&mut full, full_stride)?; + let output_rect = scaled_rect_covering(roi, downscale)?; + copy_rgb16_scaled_rect( + &full, + (width, height), + output_rect, + downscale.denominator(), + out, + stride, + ); + outcome.decoded = roi; + Ok(outcome) + } + + fn decode_lossless_rgba16_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + alpha: u16, + ) -> Result { + let output_rect = scaled_rect_covering(roi, downscale)?; + let rgb_stride = output_rect.w as usize * 6; + let mut rgb = + allocate_output_buffer(checked_scratch_len(&[rgb_stride, output_rect.h as usize])?); + let outcome = match self.info.color_space { + ColorSpace::YCbCr => self + .decode_lossless_ycbcr16_region_scaled_into(&mut rgb, rgb_stride, roi, downscale), + _ => { + self.decode_lossless_rgb16_region_scaled_into(&mut rgb, rgb_stride, roi, downscale) + } + }?; + copy_rgb16_to_rgba16( + &rgb, + rgb_stride, + output_rect.w, + output_rect.h, + out, + stride, + alpha, + ); + Ok(outcome) + } + + fn decode_12bit_rgba16_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + alpha: u16, + ) -> Result { + let output_rect = scaled_rect_covering(roi, downscale)?; + let rgb_stride = output_rect.w as usize * 6; + let mut rgb = + allocate_output_buffer(checked_scratch_len(&[rgb_stride, output_rect.h as usize])?); + let outcome = if self.info.sof_kind == SofKind::Progressive12 { + self.decode_progressive12_rgb16_region_scaled_into(&mut rgb, rgb_stride, roi, downscale) + } else { + self.decode_extended12_rgb16_region_scaled_into(&mut rgb, rgb_stride, roi, downscale) + }?; + copy_rgb16_to_rgba16( + &rgb, + rgb_stride, + output_rect.w, + output_rect.h, + out, + stride, + alpha, + ); + Ok(outcome) + } + + fn decode_lossless_gray16_into( + &self, + out: &mut [u8], + stride: usize, + ) -> Result { + let plan = self + .lossless_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: self.info.sof_kind, + })?; + if plan.bit_depth != 16 { + return Err(JpegError::UnsupportedBitDepth { + depth: plan.bit_depth, + }); + } + if !(1..=7).contains(&plan.predictor) { + return Err(JpegError::UnsupportedPredictor { + predictor: plan.predictor, + }); + } + + let (width, height) = plan.dimensions; + let scan_bytes = &self.bytes[plan.scan_offset..]; + let mut br = BitReader::new(scan_bytes); + let restart = u32::from(self.plan.restart_interval.unwrap_or(0)); + let total_samples = width.saturating_mul(height); + let mut samples_since_restart = 0u32; + let mut expected_rst = 0u8; + for y in 0..height as usize { + for x in 0..width as usize { + let sample_index = y as u32 * width + x as u32; + if restart > 0 && samples_since_restart == restart { + consume_lossless_restart( + &mut br, + sample_index, + total_samples, + &mut expected_rst, + )?; + samples_since_restart = 0; + } + let predictor = if restart > 0 && samples_since_restart == 0 { + 32768 + } else { + lossless_predictor_value_u16(plan.predictor, out, stride, x, y) + }; + let diff = plan.dc_table.decode_fast_dc(&mut br)?; + let sample = ::from_i32(predictor + diff)?; + let offset = y * stride + x * 2; + sample.write_le(&mut out[offset..offset + 2]); + samples_since_restart += 1; + } + } + + let scan_warnings = finish_scan(&mut br, true)?; + Ok(DecodeOutcome { + decoded: Rect::full(self.info.dimensions), + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } + + fn decode_lossless_gray16_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + ) -> Result { + if roi == Rect::full(self.info.dimensions) && downscale == DownscaleFactor::Full { + return self.decode_lossless_gray16_into(out, stride); + } + + let (width, height) = self.info.dimensions; + let full_stride = width as usize * 2; + let mut full = + allocate_output_buffer(checked_scratch_len(&[full_stride, height as usize])?); + let mut outcome = self.decode_lossless_gray16_into(&mut full, full_stride)?; + let output_rect = scaled_rect_covering(roi, downscale)?; + copy_gray16_scaled_rect( + &full, + (width, height), + output_rect, + downscale.denominator(), + out, + stride, + ); + outcome.decoded = roi; + Ok(outcome) + } + + fn decode_progressive12_gray16_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + ) -> Result { + self.decode_progressive12_region_into(out, stride, roi, downscale, Extended12Output::Gray16) + } + + fn decode_progressive12_rgb16_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + ) -> Result { + self.decode_progressive12_region_into(out, stride, roi, downscale, Extended12Output::Rgb16) + } + + fn decode_progressive12_region_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + output: Extended12Output, + ) -> Result { + let plan = self + .progressive_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: self.info.sof_kind, + })?; + if self.info.sof_kind != SofKind::Progressive12 + || !matches!( + self.info.color_space, + ColorSpace::Grayscale + | ColorSpace::YCbCr + | ColorSpace::Rgb + | ColorSpace::Cmyk + | ColorSpace::Ycck + ) + { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + if !roi.is_within(self.info.dimensions) { + return Err(JpegError::RectOutOfBounds { + rect: roi, + width: self.info.dimensions.0, + height: self.info.dimensions.1, + }); + } + if matches!(output, Extended12Output::Rgb16) { + match self.info.color_space { + ColorSpace::Rgb => { + let sampling = progressive_color_sampling(plan, self.info.sof_kind)?; + return match sampling { + Extended12ColorSampling::S444 => self + .decode_progressive12_color444_region_into( + out, + stride, + roi, + downscale, + Extended12RgbProjection::Identity, + ), + Extended12ColorSampling::S422 | Extended12ColorSampling::S420 => self + .decode_progressive12_color_subsampled_region_into( + out, + stride, + roi, + downscale, + sampling, + Extended12RgbProjection::Identity, + ), + }; + } + ColorSpace::YCbCr => { + let sampling = progressive_color_sampling(plan, self.info.sof_kind)?; + return match sampling { + Extended12ColorSampling::S444 => self + .decode_progressive12_color444_region_into( + out, + stride, + roi, + downscale, + Extended12RgbProjection::YCbCr, + ), + Extended12ColorSampling::S422 | Extended12ColorSampling::S420 => self + .decode_progressive12_color_subsampled_region_into( + out, + stride, + roi, + downscale, + sampling, + Extended12RgbProjection::YCbCr, + ), + }; + } + ColorSpace::Cmyk | ColorSpace::Ycck => { + let sampling = progressive_four_component_sampling(plan, self.info.sof_kind)?; + return self.decode_progressive12_four_component_region_into( + out, stride, roi, downscale, sampling, + ); + } + ColorSpace::Grayscale => {} + } + } + if self.info.color_space != ColorSpace::Grayscale || plan.components.len() != 1 { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + + let output_rect = scaled_rect_covering(roi, downscale)?; + let dct_blocks = decode_progressive_dct_blocks(plan, self.bytes)?; + let component = &plan.components[0]; + let component_coeffs = &dct_blocks.quantized[0]; + let (width, height) = self.info.dimensions; + let mut dequant = [0i16; 64]; + let mut pixels = [0u16; 64]; + let write_region = Extended12WriteRegion { + output_rect, + dimensions: (width, height), + downscale, + output, + }; + + for block_y in 0..component.block_rows as usize { + for block_x in 0..component.block_cols as usize { + let block_index = block_y * component.block_cols as usize + block_x; + dequantize_progressive12_block( + &component_coeffs[block_index], + &component.quant, + &mut dequant, + ); + if dequant[1..].iter().all(|&coeff| coeff == 0) { + pixels.fill(crate::idct::idct_islow_12bit_dc_only_sample(dequant[0])); + } else { + crate::idct::idct_islow_12bit(&dequant, &mut pixels); + } + write_extended12_block_region( + out, + stride, + write_region, + ((block_x as u32) * 8, (block_y as u32) * 8), + &pixels, + ); + } + } + + Ok(DecodeOutcome { + decoded: roi, + warnings: self.warnings.to_vec(), + }) + } + + fn decode_progressive12_color444_region_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + projection: Extended12RgbProjection, + ) -> Result { + let plan = self + .progressive_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: self.info.sof_kind, + })?; + if plan.components.len() != 3 + || plan.sampling.max_h != 1 + || plan.sampling.max_v != 1 + || plan + .components + .iter() + .any(|component| component.h != 1 || component.v != 1 || component.output_index > 2) + { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + + let output_rect = scaled_rect_covering(roi, downscale)?; + let dct_blocks = decode_progressive_dct_blocks(plan, self.bytes)?; + let (width, height) = self.info.dimensions; + let component_indices = progressive_color_component_indices(plan)?; + let block_cols = plan.components[component_indices[0]].block_cols as usize; + let block_rows = plan.components[component_indices[0]].block_rows as usize; + let mut dequant = [[0i16; 64]; 3]; + let mut pixels = [[0u16; 64]; 3]; + let write_region = Extended12WriteRegion { + output_rect, + dimensions: (width, height), + downscale, + output: Extended12Output::Rgb16, + }; + + for block_y in 0..block_rows { + for block_x in 0..block_cols { + for output_index in 0..3 { + let component_index = component_indices[output_index]; + let component = &plan.components[component_index]; + let component_coeffs = &dct_blocks.quantized[component_index]; + let block_index = block_y * component.block_cols as usize + block_x; + dequantize_progressive12_block( + &component_coeffs[block_index], + &component.quant, + &mut dequant[output_index], + ); + if dequant[output_index][1..].iter().all(|&coeff| coeff == 0) { + pixels[output_index].fill(crate::idct::idct_islow_12bit_dc_only_sample( + dequant[output_index][0], + )); + } else { + crate::idct::idct_islow_12bit( + &dequant[output_index], + &mut pixels[output_index], + ); + } + } + write_extended12_rgb_block_region( + out, + stride, + write_region, + projection, + ((block_x as u32) * 8, (block_y as u32) * 8), + &pixels, + ); + } + } + + Ok(DecodeOutcome { + decoded: roi, + warnings: self.warnings.to_vec(), + }) + } + + fn decode_progressive12_color_subsampled_region_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + sampling: Extended12ColorSampling, + projection: Extended12RgbProjection, + ) -> Result { + let plan = self + .progressive_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: self.info.sof_kind, + })?; + debug_assert!(matches!( + sampling, + Extended12ColorSampling::S422 | Extended12ColorSampling::S420 + )); + if progressive_color_sampling(plan, self.info.sof_kind)? != sampling { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + + let output_rect = scaled_rect_covering(roi, downscale)?; + let dct_blocks = decode_progressive_dct_blocks(plan, self.bytes)?; + let planes = render_progressive12_color_planes(plan, &dct_blocks.quantized)?; + let write_region = Extended12WriteRegion { + output_rect, + dimensions: self.info.dimensions, + downscale, + output: Extended12Output::Rgb16, + }; + match sampling { + Extended12ColorSampling::S444 => unreachable!("4:4:4 path is handled directly"), + Extended12ColorSampling::S422 => write_extended12_color422_planes_region( + out, + stride, + write_region, + projection, + &planes, + ), + Extended12ColorSampling::S420 => write_extended12_color420_planes_region( + out, + stride, + write_region, + projection, + &planes, + ), + } + + Ok(DecodeOutcome { + decoded: roi, + warnings: self.warnings.to_vec(), + }) + } + + fn decode_progressive12_four_component_region_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + sampling: Extended12ColorSampling, + ) -> Result { + let plan = self + .progressive_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: self.info.sof_kind, + })?; + if progressive_four_component_sampling(plan, self.info.sof_kind)? != sampling { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + + let output_rect = scaled_rect_covering(roi, downscale)?; + let dct_blocks = decode_progressive_dct_blocks(plan, self.bytes)?; + let planes = render_progressive12_four_component_planes(plan, &dct_blocks.quantized)?; + write_extended12_four_component_planes_region( + out, + stride, + Extended12WriteRegion { + output_rect, + dimensions: self.info.dimensions, + downscale, + output: Extended12Output::Rgb16, + }, + self.info.color_space, + sampling, + &planes, + ); + + Ok(DecodeOutcome { + decoded: roi, + warnings: self.warnings.to_vec(), + }) + } + + fn decode_extended12_gray16_into( + &self, + out: &mut [u8], + stride: usize, + ) -> Result { + self.decode_extended12_region_into( + out, + stride, + Rect::full(self.info.dimensions), + DownscaleFactor::Full, + Extended12Output::Gray16, + ) + } + + fn decode_extended12_gray16_region_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + ) -> Result { + self.decode_extended12_region_into( + out, + stride, + roi, + DownscaleFactor::Full, + Extended12Output::Gray16, + ) + } + + fn decode_extended12_gray16_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + ) -> Result { + self.decode_extended12_region_into(out, stride, roi, downscale, Extended12Output::Gray16) + } + + fn decode_extended12_rgb16_into( + &self, + out: &mut [u8], + stride: usize, + ) -> Result { + self.decode_extended12_region_into( + out, + stride, + Rect::full(self.info.dimensions), + DownscaleFactor::Full, + Extended12Output::Rgb16, + ) + } + + fn decode_extended12_rgb16_region_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + ) -> Result { + self.decode_extended12_region_into( + out, + stride, + roi, + DownscaleFactor::Full, + Extended12Output::Rgb16, + ) + } + + fn decode_extended12_rgb16_region_scaled_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + ) -> Result { + self.decode_extended12_region_into(out, stride, roi, downscale, Extended12Output::Rgb16) + } + + fn decode_extended12_region_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + output: Extended12Output, + ) -> Result { + if self.info.sof_kind != SofKind::Extended12 { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + if !roi.is_within(self.info.dimensions) { + return Err(JpegError::RectOutOfBounds { + rect: roi, + width: self.info.dimensions.0, + height: self.info.dimensions.1, + }); + } + if matches!(output, Extended12Output::Rgb16) { + match self.info.color_space { + ColorSpace::Rgb => { + let sampling = extended12_color_sampling(&self.plan, self.info.sof_kind)?; + return match sampling { + Extended12ColorSampling::S444 => self + .decode_extended12_color444_region_into( + out, + stride, + roi, + downscale, + Extended12RgbProjection::Identity, + ), + Extended12ColorSampling::S422 | Extended12ColorSampling::S420 => self + .decode_extended12_color_subsampled_region_into( + out, + stride, + roi, + downscale, + sampling, + Extended12RgbProjection::Identity, + ), + }; + } + ColorSpace::YCbCr => { + let sampling = extended12_color_sampling(&self.plan, self.info.sof_kind)?; + return match sampling { + Extended12ColorSampling::S444 => self + .decode_extended12_color444_region_into( + out, + stride, + roi, + downscale, + Extended12RgbProjection::YCbCr, + ), + Extended12ColorSampling::S422 | Extended12ColorSampling::S420 => self + .decode_extended12_color_subsampled_region_into( + out, + stride, + roi, + downscale, + sampling, + Extended12RgbProjection::YCbCr, + ), + }; + } + ColorSpace::Cmyk | ColorSpace::Ycck => { + let sampling = + extended12_four_component_sampling(&self.plan, self.info.sof_kind)?; + return match sampling { + Extended12ColorSampling::S444 => self + .decode_extended12_four_component444_region_into( + out, stride, roi, downscale, + ), + Extended12ColorSampling::S422 | Extended12ColorSampling::S420 => self + .decode_extended12_four_component_subsampled_region_into( + out, stride, roi, downscale, sampling, + ), + }; + } + ColorSpace::Grayscale => {} + } + } + if self.info.color_space != ColorSpace::Grayscale || self.plan.components.len() != 1 { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + + let output_rect = scaled_rect_covering(roi, downscale)?; + let scan_bytes = &self.bytes[self.plan.scan_offset..]; + let component = &self.plan.components[0]; + let (width, height) = self.info.dimensions; + let mcu_cols = width.div_ceil(8); + let mcu_rows = height.div_ceil(8); + let mut br = BitReader::new(scan_bytes); + let mut prev_dc = 0i32; + let mut coeff = CoefficientBlock::default(); + let mut pixels = [0u16; 64]; + let restart = u32::from(self.plan.restart_interval.unwrap_or(0)); + let mut mcus_since_restart = 0u32; + let mut expected_rst = 0u8; + let total_mcus = mcu_cols * mcu_rows; + let write_region = Extended12WriteRegion { + output_rect, + dimensions: (width, height), + downscale, + output, + }; + + for mcu_y in 0..mcu_rows { + for mcu_x in 0..mcu_cols { + let current_mcu = mcu_y * mcu_cols + mcu_x; + if restart > 0 && mcus_since_restart == restart { + consume_extended12_restart( + &mut br, + current_mcu, + total_mcus, + &mut expected_rst, + )?; + prev_dc = 0; + mcus_since_restart = 0; + } + decode_extended12_block_pixels( + &mut br, + component, + &mut prev_dc, + &mut coeff, + &mut pixels, + )?; + write_extended12_block_region( + out, + stride, + write_region, + (mcu_x * 8, mcu_y * 8), + &pixels, + ); + mcus_since_restart += 1; + } + } + + let scan_warnings = finish_scan(&mut br, true)?; + Ok(DecodeOutcome { + decoded: roi, + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } + + fn decode_extended12_color444_region_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + projection: Extended12RgbProjection, + ) -> Result { + validate_extended12_color444_plan(&self.plan, self.info.sof_kind)?; + + let output_rect = scaled_rect_covering(roi, downscale)?; + let scan_bytes = &self.bytes[self.plan.scan_offset..]; + let (width, height) = self.info.dimensions; + let mcu_cols = width.div_ceil(8); + let mcu_rows = height.div_ceil(8); + let mut br = BitReader::new(scan_bytes); + let mut prev_dc = [0i32; 3]; + let mut coeffs: [CoefficientBlock; 3] = + core::array::from_fn(|_| CoefficientBlock::default()); + let mut pixels = [[0u16; 64]; 3]; + let restart = u32::from(self.plan.restart_interval.unwrap_or(0)); + let mut mcus_since_restart = 0u32; + let mut expected_rst = 0u8; + let total_mcus = mcu_cols * mcu_rows; + let write_region = Extended12WriteRegion { + output_rect, + dimensions: (width, height), + downscale, + output: Extended12Output::Rgb16, + }; + + for mcu_y in 0..mcu_rows { + for mcu_x in 0..mcu_cols { + let current_mcu = mcu_y * mcu_cols + mcu_x; + if restart > 0 && mcus_since_restart == restart { + consume_extended12_restart( + &mut br, + current_mcu, + total_mcus, + &mut expected_rst, + )?; + prev_dc.fill(0); + mcus_since_restart = 0; + } + for component in &self.plan.components { + let output_index = component.output_index; + decode_extended12_block_pixels( + &mut br, + component, + &mut prev_dc[output_index], + &mut coeffs[output_index], + &mut pixels[output_index], + )?; + } + write_extended12_rgb_block_region( + out, + stride, + write_region, + projection, + (mcu_x * 8, mcu_y * 8), + &pixels, + ); + mcus_since_restart += 1; + } + } + + let scan_warnings = finish_scan(&mut br, true)?; + Ok(DecodeOutcome { + decoded: roi, + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } + + fn decode_extended12_color_subsampled_region_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + sampling: Extended12ColorSampling, + projection: Extended12RgbProjection, + ) -> Result { + debug_assert!(matches!( + sampling, + Extended12ColorSampling::S422 | Extended12ColorSampling::S420 + )); + if extended12_color_sampling(&self.plan, self.info.sof_kind)? != sampling { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + + let output_rect = scaled_rect_covering(roi, downscale)?; + let scan_bytes = &self.bytes[self.plan.scan_offset..]; + let (planes, scan_warnings) = + decode_extended12_color_planes(&self.plan, scan_bytes, self.info.sof_kind)?; + let write_region = Extended12WriteRegion { + output_rect, + dimensions: self.info.dimensions, + downscale, + output: Extended12Output::Rgb16, + }; + match sampling { + Extended12ColorSampling::S444 => unreachable!("4:4:4 path is handled directly"), + Extended12ColorSampling::S422 => write_extended12_color422_planes_region( + out, + stride, + write_region, + projection, + &planes, + ), + Extended12ColorSampling::S420 => write_extended12_color420_planes_region( + out, + stride, + write_region, + projection, + &planes, + ), + } + + Ok(DecodeOutcome { + decoded: roi, + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } + + fn decode_extended12_four_component444_region_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + ) -> Result { + validate_extended12_four_component444_plan(&self.plan, self.info.sof_kind)?; + + let output_rect = scaled_rect_covering(roi, downscale)?; + let scan_bytes = &self.bytes[self.plan.scan_offset..]; + let (width, height) = self.info.dimensions; + let mcu_cols = width.div_ceil(8); + let mcu_rows = height.div_ceil(8); + let mut br = BitReader::new(scan_bytes); + let mut prev_dc = [0i32; 4]; + let mut coeffs: [CoefficientBlock; 4] = + core::array::from_fn(|_| CoefficientBlock::default()); + let mut pixels = [[0u16; 64]; 4]; + let restart = u32::from(self.plan.restart_interval.unwrap_or(0)); + let mut mcus_since_restart = 0u32; + let mut expected_rst = 0u8; + let total_mcus = mcu_cols * mcu_rows; + let write_region = Extended12WriteRegion { + output_rect, + dimensions: (width, height), + downscale, + output: Extended12Output::Rgb16, + }; + + for mcu_y in 0..mcu_rows { + for mcu_x in 0..mcu_cols { + let current_mcu = mcu_y * mcu_cols + mcu_x; + if restart > 0 && mcus_since_restart == restart { + consume_extended12_restart( + &mut br, + current_mcu, + total_mcus, + &mut expected_rst, + )?; + prev_dc.fill(0); + mcus_since_restart = 0; + } + for component in &self.plan.components { + let output_index = component.output_index; + decode_extended12_block_pixels( + &mut br, + component, + &mut prev_dc[output_index], + &mut coeffs[output_index], + &mut pixels[output_index], + )?; + } + write_extended12_four_component_block_region( + out, + stride, + write_region, + self.info.color_space, + (mcu_x * 8, mcu_y * 8), + &pixels, + ); + mcus_since_restart += 1; + } + } + + let scan_warnings = finish_scan(&mut br, true)?; + Ok(DecodeOutcome { + decoded: roi, + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } + + fn decode_extended12_four_component_subsampled_region_into( + &self, + out: &mut [u8], + stride: usize, + roi: Rect, + downscale: DownscaleFactor, + sampling: Extended12ColorSampling, + ) -> Result { + debug_assert!(matches!( + sampling, + Extended12ColorSampling::S422 | Extended12ColorSampling::S420 + )); + if extended12_four_component_sampling(&self.plan, self.info.sof_kind)? != sampling { + return Err(JpegError::NotImplemented { + sof: self.info.sof_kind, + }); + } + + let output_rect = scaled_rect_covering(roi, downscale)?; + let scan_bytes = &self.bytes[self.plan.scan_offset..]; + let (planes, scan_warnings) = + decode_extended12_four_component_planes(&self.plan, scan_bytes, self.info.sof_kind)?; + write_extended12_four_component_planes_region( + out, + stride, + Extended12WriteRegion { + output_rect, + dimensions: self.info.dimensions, + downscale, + output: Extended12Output::Rgb16, + }, + self.info.color_space, + sampling, + &planes, + ); + + Ok(DecodeOutcome { + decoded: roi, + warnings: merged_warnings(&self.warnings, scan_warnings), + }) + } +} + +#[derive(Debug, Clone, Copy)] +enum Extended12Output { + Gray16, + Rgb16, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Extended12ColorSampling { + S444, + S422, + S420, +} + +fn lossless_color_sampling(info: &Info) -> Option { + if info.sampling.len() != 3 { + return None; + } + match ( + info.sampling.max_h, + info.sampling.max_v, + info.sampling.components(), + ) { + (1, 1, &[(1, 1), (1, 1), (1, 1)]) => Some(LosslessColorSampling::S444), + (2, 1, &[(2, 1), (1, 1), (1, 1)]) + if matches!(info.bit_depth, 8 | 16) && info.dimensions.0.is_multiple_of(2) => + { + Some(LosslessColorSampling::S422) + } + (2, 2, &[(2, 2), (1, 1), (1, 1)]) + if matches!(info.bit_depth, 8 | 16) + && info.dimensions.0.is_multiple_of(2) + && info.dimensions.1.is_multiple_of(2) => + { + Some(LosslessColorSampling::S420) + } + _ => None, + } +} + +#[derive(Debug, Clone, Copy)] +enum Extended12RgbProjection { + Identity, + YCbCr, +} + +#[derive(Debug, Clone, Copy)] +struct Extended12WriteRegion { + output_rect: Rect, + dimensions: (u32, u32), + downscale: DownscaleFactor, + output: Extended12Output, +} + +struct Extended12Plane { + pixels: Vec, + stride: usize, + width: usize, +} + +fn decode_extended12_block_pixels( + br: &mut BitReader<'_>, + component: &PreparedComponentPlan, + prev_dc: &mut i32, + coeff: &mut CoefficientBlock, + pixels: &mut [u16; 64], +) -> Result<(), JpegError> { + let activity = decode_block_with_activity( + br, + &component.dc_table, + &component.ac_table, + prev_dc, + component.quant.as_ref(), + coeff, + )?; + match activity { + BlockActivity::DcOnly => { + pixels.fill(crate::idct::idct_islow_12bit_dc_only_sample( + coeff.dc_coeff(), + )); + } + BlockActivity::BottomHalfZero | BlockActivity::General => { + crate::idct::idct_islow_12bit(coeff.coefficients(), pixels); + } + } + Ok(()) +} + +fn decode_extended12_color_planes( + plan: &PreparedDecodePlan, + scan_bytes: &[u8], + sof: SofKind, +) -> Result<([Extended12Plane; 3], Vec), JpegError> { + if plan.components.len() != 3 { + return Err(JpegError::NotImplemented { sof }); + } + let mut planes = extended12_planes_for_sequential_plan(plan, sof)?; + let mut br = BitReader::new(scan_bytes); + let mut prev_dc = [0i32; 3]; + let mut coeffs: [CoefficientBlock; 3] = core::array::from_fn(|_| CoefficientBlock::default()); + let mut pixels = [[0u16; 64]; 3]; + let mcu_cols = plan + .dimensions + .0 + .div_ceil(u32::from(plan.sampling.max_h) * 8); + let mcu_rows = plan + .dimensions + .1 + .div_ceil(u32::from(plan.sampling.max_v) * 8); + let restart = u32::from(plan.restart_interval.unwrap_or(0)); + let mut mcus_since_restart = 0u32; + let mut expected_rst = 0u8; + let total_mcus = mcu_cols * mcu_rows; + + for mcu_y in 0..mcu_rows { + for mcu_x in 0..mcu_cols { + let current_mcu = mcu_y * mcu_cols + mcu_x; + if restart > 0 && mcus_since_restart == restart { + consume_extended12_restart(&mut br, current_mcu, total_mcus, &mut expected_rst)?; + prev_dc.fill(0); + mcus_since_restart = 0; + } + for component in &plan.components { + let output_index = component.output_index; + if output_index > 2 { + return Err(JpegError::NotImplemented { sof }); + } + for by in 0..u32::from(component.v) { + for bx in 0..u32::from(component.h) { + decode_extended12_block_pixels( + &mut br, + component, + &mut prev_dc[output_index], + &mut coeffs[output_index], + &mut pixels[output_index], + )?; + deposit_extended12_block( + &mut planes[output_index], + (mcu_x * u32::from(component.h) + bx) as usize * 8, + (mcu_y * u32::from(component.v) + by) as usize * 8, + &pixels[output_index], + ); + } + } + } + mcus_since_restart += 1; + } + } + + let scan_warnings = finish_scan(&mut br, true)?; + Ok((planes, scan_warnings)) +} + +fn decode_extended12_four_component_planes( + plan: &PreparedDecodePlan, + scan_bytes: &[u8], + sof: SofKind, +) -> Result<([Extended12Plane; 4], Vec), JpegError> { + if plan.components.len() != 4 { + return Err(JpegError::NotImplemented { sof }); + } + let mut planes = extended12_four_component_planes_for_sequential_plan(plan, sof)?; + let mut br = BitReader::new(scan_bytes); + let mut prev_dc = [0i32; 4]; + let mut coeffs: [CoefficientBlock; 4] = core::array::from_fn(|_| CoefficientBlock::default()); + let mut pixels = [[0u16; 64]; 4]; + let mcu_cols = plan + .dimensions + .0 + .div_ceil(u32::from(plan.sampling.max_h) * 8); + let mcu_rows = plan + .dimensions + .1 + .div_ceil(u32::from(plan.sampling.max_v) * 8); + let restart = u32::from(plan.restart_interval.unwrap_or(0)); + let mut mcus_since_restart = 0u32; + let mut expected_rst = 0u8; + let total_mcus = mcu_cols * mcu_rows; + + for mcu_y in 0..mcu_rows { + for mcu_x in 0..mcu_cols { + let current_mcu = mcu_y * mcu_cols + mcu_x; + if restart > 0 && mcus_since_restart == restart { + consume_extended12_restart(&mut br, current_mcu, total_mcus, &mut expected_rst)?; + prev_dc.fill(0); + mcus_since_restart = 0; + } + for component in &plan.components { + let output_index = component.output_index; + if output_index > 3 { + return Err(JpegError::NotImplemented { sof }); + } + for by in 0..u32::from(component.v) { + for bx in 0..u32::from(component.h) { + decode_extended12_block_pixels( + &mut br, + component, + &mut prev_dc[output_index], + &mut coeffs[output_index], + &mut pixels[output_index], + )?; + deposit_extended12_block( + &mut planes[output_index], + (mcu_x * u32::from(component.h) + bx) as usize * 8, + (mcu_y * u32::from(component.v) + by) as usize * 8, + &pixels[output_index], + ); + } + } + } + mcus_since_restart += 1; + } + } + + let scan_warnings = finish_scan(&mut br, true)?; + Ok((planes, scan_warnings)) +} + +fn extended12_planes_for_sequential_plan( + plan: &PreparedDecodePlan, + sof: SofKind, +) -> Result<[Extended12Plane; 3], JpegError> { + let mcu_cols = plan + .dimensions + .0 + .div_ceil(u32::from(plan.sampling.max_h) * 8); + let mcu_rows = plan + .dimensions + .1 + .div_ceil(u32::from(plan.sampling.max_v) * 8); + let mut widths = [0usize; 3]; + let mut strides = [0usize; 3]; + let mut heights = [0usize; 3]; + let mut lens = [0usize; 3]; + for component in &plan.components { + if component.output_index > 2 { + return Err(JpegError::NotImplemented { sof }); + } + widths[component.output_index] = + plan.dimensions + .0 + .saturating_mul(u32::from(component.h)) + .div_ceil(u32::from(plan.sampling.max_h)) as usize; + strides[component.output_index] = + checked_scratch_len(&[mcu_cols as usize, usize::from(component.h), 8])?; + heights[component.output_index] = + checked_scratch_len(&[mcu_rows as usize, usize::from(component.v), 8])?; + lens[component.output_index] = checked_scratch_len(&[ + strides[component.output_index], + heights[component.output_index], + core::mem::size_of::(), + ])? / core::mem::size_of::(); + } + Ok(core::array::from_fn(|index| Extended12Plane { + pixels: vec![0u16; lens[index]], + stride: strides[index], + width: widths[index], + })) +} + +fn extended12_four_component_planes_for_sequential_plan( + plan: &PreparedDecodePlan, + sof: SofKind, +) -> Result<[Extended12Plane; 4], JpegError> { + let mcu_cols = plan + .dimensions + .0 + .div_ceil(u32::from(plan.sampling.max_h) * 8); + let mcu_rows = plan + .dimensions + .1 + .div_ceil(u32::from(plan.sampling.max_v) * 8); + let mut widths = [0usize; 4]; + let mut strides = [0usize; 4]; + let mut heights = [0usize; 4]; + let mut lens = [0usize; 4]; + for component in &plan.components { + if component.output_index > 3 { + return Err(JpegError::NotImplemented { sof }); + } + widths[component.output_index] = + plan.dimensions + .0 + .saturating_mul(u32::from(component.h)) + .div_ceil(u32::from(plan.sampling.max_h)) as usize; + strides[component.output_index] = + checked_scratch_len(&[mcu_cols as usize, usize::from(component.h), 8])?; + heights[component.output_index] = + checked_scratch_len(&[mcu_rows as usize, usize::from(component.v), 8])?; + lens[component.output_index] = checked_scratch_len(&[ + strides[component.output_index], + heights[component.output_index], + core::mem::size_of::(), + ])? / core::mem::size_of::(); + } + Ok(core::array::from_fn(|index| Extended12Plane { + pixels: vec![0u16; lens[index]], + stride: strides[index], + width: widths[index], + })) +} + +fn render_progressive12_color_planes( + plan: &PreparedProgressivePlan, + coeffs: &[Vec<[i32; 64]>], +) -> Result<[Extended12Plane; 3], JpegError> { + let mut planes = progressive12_color_planes(plan)?; + let mut dequant = [0i16; 64]; + let mut pixels = [0u16; 64]; + for (component_index, component) in plan.components.iter().enumerate() { + let output_index = component.output_index; + if output_index > 2 { + return Err(JpegError::NotImplemented { + sof: SofKind::Progressive12, + }); + } + for by in 0..component.block_rows as usize { + for bx in 0..component.block_cols as usize { + let block_index = by * component.block_cols as usize + bx; + dequantize_progressive12_block( + &coeffs[component_index][block_index], + &component.quant, + &mut dequant, + ); + if dequant[1..].iter().all(|&coeff| coeff == 0) { + pixels.fill(crate::idct::idct_islow_12bit_dc_only_sample(dequant[0])); + } else { + crate::idct::idct_islow_12bit(&dequant, &mut pixels); + } + deposit_extended12_block(&mut planes[output_index], bx * 8, by * 8, &pixels); + } + } + } + Ok(planes) +} + +fn render_progressive12_four_component_planes( + plan: &PreparedProgressivePlan, + coeffs: &[Vec<[i32; 64]>], +) -> Result<[Extended12Plane; 4], JpegError> { + let mut planes = progressive12_four_component_planes(plan)?; + let mut dequant = [0i16; 64]; + let mut pixels = [0u16; 64]; + for (component_index, component) in plan.components.iter().enumerate() { + let output_index = component.output_index; + if output_index > 3 { + return Err(JpegError::NotImplemented { + sof: SofKind::Progressive12, + }); + } + for by in 0..component.block_rows as usize { + for bx in 0..component.block_cols as usize { + let block_index = by * component.block_cols as usize + bx; + dequantize_progressive12_block( + &coeffs[component_index][block_index], + &component.quant, + &mut dequant, + ); + if dequant[1..].iter().all(|&coeff| coeff == 0) { + pixels.fill(crate::idct::idct_islow_12bit_dc_only_sample(dequant[0])); + } else { + crate::idct::idct_islow_12bit(&dequant, &mut pixels); + } + deposit_extended12_block(&mut planes[output_index], bx * 8, by * 8, &pixels); + } + } + } + Ok(planes) +} + +fn progressive12_color_planes( + plan: &PreparedProgressivePlan, +) -> Result<[Extended12Plane; 3], JpegError> { + let mut widths = [0usize; 3]; + let mut strides = [0usize; 3]; + let mut heights = [0usize; 3]; + for component in &plan.components { + if component.output_index > 2 { + return Err(JpegError::NotImplemented { + sof: SofKind::Progressive12, + }); + } + widths[component.output_index] = component.sample_width as usize; + strides[component.output_index] = component.block_cols as usize * 8; + heights[component.output_index] = component.block_rows as usize * 8; + } + Ok(core::array::from_fn(|index| Extended12Plane { + pixels: vec![0u16; strides[index] * heights[index]], + stride: strides[index], + width: widths[index], + })) +} + +fn progressive12_four_component_planes( + plan: &PreparedProgressivePlan, +) -> Result<[Extended12Plane; 4], JpegError> { + let mut widths = [0usize; 4]; + let mut strides = [0usize; 4]; + let mut heights = [0usize; 4]; + for component in &plan.components { + if component.output_index > 3 { + return Err(JpegError::NotImplemented { + sof: SofKind::Progressive12, + }); + } + widths[component.output_index] = component.sample_width as usize; + strides[component.output_index] = component.block_cols as usize * 8; + heights[component.output_index] = component.block_rows as usize * 8; + } + Ok(core::array::from_fn(|index| Extended12Plane { + pixels: vec![0u16; strides[index] * heights[index]], + stride: strides[index], + width: widths[index], + })) +} + +fn deposit_extended12_block(plane: &mut Extended12Plane, x: usize, y: usize, block: &[u16; 64]) { + for row in 0..8 { + let dst_start = (y + row) * plane.stride + x; + let src_start = row * 8; + plane.pixels[dst_start..dst_start + 8].copy_from_slice(&block[src_start..src_start + 8]); + } +} + +fn validate_extended12_color444_plan( + plan: &PreparedDecodePlan, + sof: SofKind, +) -> Result<(), JpegError> { + if plan.components.len() != 3 || plan.sampling.max_h != 1 || plan.sampling.max_v != 1 { + return Err(JpegError::NotImplemented { sof }); + } + let mut seen = [false; 3]; + for component in &plan.components { + if component.h != 1 || component.v != 1 || component.output_index > 2 { + return Err(JpegError::NotImplemented { sof }); + } + if seen[component.output_index] { + return Err(JpegError::NotImplemented { sof }); + } + seen[component.output_index] = true; + } + if seen.iter().any(|&present| !present) { + return Err(JpegError::NotImplemented { sof }); + } + Ok(()) +} + +fn validate_extended12_four_component444_plan( + plan: &PreparedDecodePlan, + sof: SofKind, +) -> Result<(), JpegError> { + if extended12_four_component_sampling(plan, sof)? != Extended12ColorSampling::S444 { + return Err(JpegError::NotImplemented { sof }); + } + Ok(()) +} + +fn extended12_color_sampling( + plan: &PreparedDecodePlan, + sof: SofKind, +) -> Result { + if plan.components.len() != 3 { + return Err(JpegError::NotImplemented { sof }); + } + let components = color_component_sampling_from_sequential(plan, sof)?; + color_sampling_from_components(plan.sampling.max_h, plan.sampling.max_v, components, sof) +} + +fn extended12_four_component_sampling( + plan: &PreparedDecodePlan, + sof: SofKind, +) -> Result { + if plan.components.len() != 4 { + return Err(JpegError::NotImplemented { sof }); + } + let components = four_component_sampling_from_sequential(plan, sof)?; + four_component_sampling_from_components( + plan.sampling.max_h, + plan.sampling.max_v, + components, + sof, + ) +} + +fn color_component_sampling_from_sequential( + plan: &PreparedDecodePlan, + sof: SofKind, +) -> Result<[(u8, u8); 3], JpegError> { + let mut components = [(0u8, 0u8); 3]; + let mut seen = [false; 3]; + for component in &plan.components { + if component.output_index > 2 || seen[component.output_index] { + return Err(JpegError::NotImplemented { sof }); + } + seen[component.output_index] = true; + components[component.output_index] = (component.h, component.v); + } + if seen.iter().any(|&present| !present) { + return Err(JpegError::NotImplemented { sof }); + } + Ok(components) +} + +fn four_component_sampling_from_sequential( + plan: &PreparedDecodePlan, + sof: SofKind, +) -> Result<[(u8, u8); 4], JpegError> { + let mut components = [(0u8, 0u8); 4]; + let mut seen = [false; 4]; + for component in &plan.components { + if component.output_index > 3 || seen[component.output_index] { + return Err(JpegError::NotImplemented { sof }); + } + seen[component.output_index] = true; + components[component.output_index] = (component.h, component.v); + } + if seen.iter().any(|&present| !present) { + return Err(JpegError::NotImplemented { sof }); + } + Ok(components) +} + +fn progressive_color_sampling( + plan: &PreparedProgressivePlan, + sof: SofKind, +) -> Result { + if plan.components.len() != 3 { + return Err(JpegError::NotImplemented { sof }); + } + let components = color_component_sampling_from_progressive(plan, sof)?; + color_sampling_from_components(plan.sampling.max_h, plan.sampling.max_v, components, sof) +} + +fn progressive_four_component_sampling( + plan: &PreparedProgressivePlan, + sof: SofKind, +) -> Result { + if plan.components.len() != 4 { + return Err(JpegError::NotImplemented { sof }); + } + let components = four_component_sampling_from_progressive(plan, sof)?; + four_component_sampling_from_components( + plan.sampling.max_h, + plan.sampling.max_v, + components, + sof, + ) +} + +fn color_component_sampling_from_progressive( + plan: &PreparedProgressivePlan, + sof: SofKind, +) -> Result<[(u8, u8); 3], JpegError> { + let mut components = [(0u8, 0u8); 3]; + let mut seen = [false; 3]; + for component in &plan.components { + if component.output_index > 2 || seen[component.output_index] { + return Err(JpegError::NotImplemented { sof }); + } + seen[component.output_index] = true; + components[component.output_index] = (component.h, component.v); + } + if seen.iter().any(|&present| !present) { + return Err(JpegError::NotImplemented { sof }); + } + Ok(components) +} + +fn four_component_sampling_from_progressive( + plan: &PreparedProgressivePlan, + sof: SofKind, +) -> Result<[(u8, u8); 4], JpegError> { + let mut components = [(0u8, 0u8); 4]; + let mut seen = [false; 4]; + for component in &plan.components { + if component.output_index > 3 || seen[component.output_index] { + return Err(JpegError::NotImplemented { sof }); + } + seen[component.output_index] = true; + components[component.output_index] = (component.h, component.v); + } + if seen.iter().any(|&present| !present) { + return Err(JpegError::NotImplemented { sof }); + } + Ok(components) +} + +fn color_sampling_from_components( + max_h: u8, + max_v: u8, + components: [(u8, u8); 3], + sof: SofKind, +) -> Result { + match (max_h, max_v, components) { + (1, 1, [(1, 1), (1, 1), (1, 1)]) => Ok(Extended12ColorSampling::S444), + (2, 1, [(2, 1), (1, 1), (1, 1)]) => Ok(Extended12ColorSampling::S422), + (2, 2, [(2, 2), (1, 1), (1, 1)]) => Ok(Extended12ColorSampling::S420), + _ => Err(JpegError::NotImplemented { sof }), + } +} + +fn four_component_sampling_from_components( + max_h: u8, + max_v: u8, + components: [(u8, u8); 4], + sof: SofKind, +) -> Result { + match (max_h, max_v, components) { + (1, 1, [(1, 1), (1, 1), (1, 1), (1, 1)]) => Ok(Extended12ColorSampling::S444), + (2, 1, [(2, 1), (1, 1), (1, 1), (1, 1)]) => Ok(Extended12ColorSampling::S422), + (2, 2, [(2, 2), (1, 1), (1, 1), (1, 1)]) => Ok(Extended12ColorSampling::S420), + _ => Err(JpegError::NotImplemented { sof }), + } +} + +fn progressive_color_component_indices( + plan: &PreparedProgressivePlan, +) -> Result<[usize; 3], JpegError> { + let mut indices = [usize::MAX; 3]; + for (component_index, component) in plan.components.iter().enumerate() { + if component.output_index < 3 { + if indices[component.output_index] != usize::MAX { + return Err(JpegError::NotImplemented { + sof: SofKind::Progressive12, + }); + } + indices[component.output_index] = component_index; + } + } + if indices.contains(&usize::MAX) { + return Err(JpegError::NotImplemented { + sof: SofKind::Progressive12, + }); + } + Ok(indices) +} + +fn dequantize_progressive12_block(coeffs: &[i32; 64], quant: &[u16; 64], out: &mut [i16; 64]) { + out.fill(0); + for k in 0..64 { + let natural_idx = usize::from(ZIGZAG[k]); + let value = coeffs[natural_idx].wrapping_mul(i32::from(quant[k])); + out[natural_idx] = value.clamp(i16::MIN as i32, i16::MAX as i32) as i16; + } +} + +fn write_extended12_rgb_block_region( + out: &mut [u8], + stride: usize, + region: Extended12WriteRegion, + projection: Extended12RgbProjection, + block_origin: (u32, u32), + pixels: &[[u16; 64]; 3], +) { + let (width, height) = region.dimensions; + let (x0, y0) = block_origin; + let block_x1 = (x0 + 8).min(width); + let block_y1 = (y0 + 8).min(height); + let denom = region.downscale.denominator(); + let output_rect = region.output_rect; + for output_y in output_rect.y..output_rect.y + output_rect.h { + let source_y = output_y.saturating_mul(denom).min(height - 1); + if source_y < y0 || source_y >= block_y1 { + continue; + } + let src_row = (source_y - y0) as usize; + let dst_row = (output_y - output_rect.y) as usize; + for output_x in output_rect.x..output_rect.x + output_rect.w { + let source_x = output_x.saturating_mul(denom).min(width - 1); + if source_x < x0 || source_x >= block_x1 { + continue; + } + let src_col = (source_x - x0) as usize; + let src_index = src_row * 8 + src_col; + let dst_col = (output_x - output_rect.x) as usize; + let dst_start = dst_row * stride + dst_col * 6; + let dst = &mut out[dst_start..dst_start + 6]; + let (r, g, b) = match projection { + Extended12RgbProjection::Identity => ( + pixels[0][src_index], + pixels[1][src_index], + pixels[2][src_index], + ), + Extended12RgbProjection::YCbCr => crate::color::ycbcr::ycbcr12_to_rgb16( + pixels[0][src_index], + pixels[1][src_index], + pixels[2][src_index], + ), + }; + dst[0..2].copy_from_slice(&r.to_le_bytes()); + dst[2..4].copy_from_slice(&g.to_le_bytes()); + dst[4..6].copy_from_slice(&b.to_le_bytes()); + } + } +} + +fn write_extended12_four_component_block_region( + out: &mut [u8], + stride: usize, + region: Extended12WriteRegion, + color_space: ColorSpace, + block_origin: (u32, u32), + pixels: &[[u16; 64]; 4], +) { + let (width, height) = region.dimensions; + let (x0, y0) = block_origin; + let block_x1 = (x0 + 8).min(width); + let block_y1 = (y0 + 8).min(height); + let denom = region.downscale.denominator(); + let output_rect = region.output_rect; + for output_y in output_rect.y..output_rect.y + output_rect.h { + let source_y = output_y.saturating_mul(denom).min(height - 1); + if source_y < y0 || source_y >= block_y1 { + continue; + } + let src_row = (source_y - y0) as usize; + let dst_row = (output_y - output_rect.y) as usize; + for output_x in output_rect.x..output_rect.x + output_rect.w { + let source_x = output_x.saturating_mul(denom).min(width - 1); + if source_x < x0 || source_x >= block_x1 { + continue; + } + let src_col = (source_x - x0) as usize; + let src_index = src_row * 8 + src_col; + let (r, g, b) = match color_space { + ColorSpace::Cmyk => crate::color::cmyk::inverted_cmyk12_to_rgb16( + pixels[0][src_index], + pixels[1][src_index], + pixels[2][src_index], + pixels[3][src_index], + ), + ColorSpace::Ycck => crate::color::cmyk::ycck12_to_rgb16( + pixels[0][src_index], + pixels[1][src_index], + pixels[2][src_index], + pixels[3][src_index], + ), + _ => unreachable!("12-bit four-component path only accepts CMYK/YCCK"), + }; + let dst_col = (output_x - output_rect.x) as usize; + let dst_start = dst_row * stride + dst_col * 6; + let dst = &mut out[dst_start..dst_start + 6]; + dst[0..2].copy_from_slice(&r.to_le_bytes()); + dst[2..4].copy_from_slice(&g.to_le_bytes()); + dst[4..6].copy_from_slice(&b.to_le_bytes()); + } + } +} + +fn write_extended12_color422_planes_region( + out: &mut [u8], + stride: usize, + region: Extended12WriteRegion, + projection: Extended12RgbProjection, + planes: &[Extended12Plane; 3], +) { + let (width, height) = region.dimensions; + let denom = region.downscale.denominator(); + let output_rect = region.output_rect; + for output_y in output_rect.y..output_rect.y + output_rect.h { + let source_y = output_y.saturating_mul(denom).min(height - 1) as usize; + let dst_row = (output_y - output_rect.y) as usize; + for output_x in output_rect.x..output_rect.x + output_rect.w { + let source_x = output_x.saturating_mul(denom).min(width - 1) as usize; + let y = planes[0].pixels[source_y * planes[0].stride + source_x]; + let chroma_y = source_y.min(planes[1].pixels.len() / planes[1].stride - 1); + let cb_row = &planes[1].pixels + [chroma_y * planes[1].stride..chroma_y * planes[1].stride + planes[1].width]; + let cr_row = &planes[2].pixels + [chroma_y * planes[2].stride..chroma_y * planes[2].stride + planes[2].width]; + let c1 = upsample_h2v1_12bit_at(cb_row, source_x); + let c2 = upsample_h2v1_12bit_at(cr_row, source_x); + let (r, g, b) = match projection { + Extended12RgbProjection::Identity => (y, c1, c2), + Extended12RgbProjection::YCbCr => crate::color::ycbcr::ycbcr12_to_rgb16(y, c1, c2), + }; + let dst_col = (output_x - output_rect.x) as usize; + let dst_start = dst_row * stride + dst_col * 6; + let dst = &mut out[dst_start..dst_start + 6]; + dst[0..2].copy_from_slice(&r.to_le_bytes()); + dst[2..4].copy_from_slice(&g.to_le_bytes()); + dst[4..6].copy_from_slice(&b.to_le_bytes()); + } + } +} + +fn write_extended12_color420_planes_region( + out: &mut [u8], + stride: usize, + region: Extended12WriteRegion, + projection: Extended12RgbProjection, + planes: &[Extended12Plane; 3], +) { + let (width, height) = region.dimensions; + let denom = region.downscale.denominator(); + let output_rect = region.output_rect; + for output_y in output_rect.y..output_rect.y + output_rect.h { + let source_y = output_y.saturating_mul(denom).min(height - 1) as usize; + let dst_row = (output_y - output_rect.y) as usize; + for output_x in output_rect.x..output_rect.x + output_rect.w { + let source_x = output_x.saturating_mul(denom).min(width - 1) as usize; + let y = planes[0].pixels[source_y * planes[0].stride + source_x]; + let chroma_height = planes[1].pixels.len() / planes[1].stride; + let chroma_y = (source_y / 2).min(chroma_height - 1); + let prev_y = chroma_y.saturating_sub(1); + let next_y = (chroma_y + 1).min(chroma_height - 1); + let c1 = upsample_h2v2_12bit_at( + extended12_plane_row(&planes[1], prev_y), + extended12_plane_row(&planes[1], chroma_y), + extended12_plane_row(&planes[1], next_y), + source_x, + !source_y.is_multiple_of(2), + ); + let c2 = upsample_h2v2_12bit_at( + extended12_plane_row(&planes[2], prev_y), + extended12_plane_row(&planes[2], chroma_y), + extended12_plane_row(&planes[2], next_y), + source_x, + !source_y.is_multiple_of(2), + ); + let (r, g, b) = match projection { + Extended12RgbProjection::Identity => (y, c1, c2), + Extended12RgbProjection::YCbCr => crate::color::ycbcr::ycbcr12_to_rgb16(y, c1, c2), + }; + let dst_col = (output_x - output_rect.x) as usize; + let dst_start = dst_row * stride + dst_col * 6; + let dst = &mut out[dst_start..dst_start + 6]; + dst[0..2].copy_from_slice(&r.to_le_bytes()); + dst[2..4].copy_from_slice(&g.to_le_bytes()); + dst[4..6].copy_from_slice(&b.to_le_bytes()); + } + } +} + +fn write_extended12_four_component_planes_region( + out: &mut [u8], + stride: usize, + region: Extended12WriteRegion, + color_space: ColorSpace, + sampling: Extended12ColorSampling, + planes: &[Extended12Plane; 4], +) { + let (width, height) = region.dimensions; + let denom = region.downscale.denominator(); + let output_rect = region.output_rect; + for output_y in output_rect.y..output_rect.y + output_rect.h { + let source_y = output_y.saturating_mul(denom).min(height - 1) as usize; + let dst_row = (output_y - output_rect.y) as usize; + for output_x in output_rect.x..output_rect.x + output_rect.w { + let source_x = output_x.saturating_mul(denom).min(width - 1) as usize; + let c0 = planes[0].pixels[source_y * planes[0].stride + source_x]; + let (c1, c2, c3) = match sampling { + Extended12ColorSampling::S444 => ( + sample_extended12_plane_at(&planes[1], source_x, source_y), + sample_extended12_plane_at(&planes[2], source_x, source_y), + sample_extended12_plane_at(&planes[3], source_x, source_y), + ), + Extended12ColorSampling::S422 => ( + upsample_extended12_plane_h2v1_at(&planes[1], source_x, source_y), + upsample_extended12_plane_h2v1_at(&planes[2], source_x, source_y), + upsample_extended12_plane_h2v1_at(&planes[3], source_x, source_y), + ), + Extended12ColorSampling::S420 => ( + upsample_extended12_plane_h2v2_at(&planes[1], source_x, source_y), + upsample_extended12_plane_h2v2_at(&planes[2], source_x, source_y), + upsample_extended12_plane_h2v2_at(&planes[3], source_x, source_y), + ), + }; + let (r, g, b) = match color_space { + ColorSpace::Cmyk => crate::color::cmyk::inverted_cmyk12_to_rgb16(c0, c1, c2, c3), + ColorSpace::Ycck => crate::color::cmyk::ycck12_to_rgb16(c0, c1, c2, c3), + _ => unreachable!("12-bit four-component plane path only accepts CMYK/YCCK"), + }; + let dst_col = (output_x - output_rect.x) as usize; + let dst_start = dst_row * stride + dst_col * 6; + let dst = &mut out[dst_start..dst_start + 6]; + dst[0..2].copy_from_slice(&r.to_le_bytes()); + dst[2..4].copy_from_slice(&g.to_le_bytes()); + dst[4..6].copy_from_slice(&b.to_le_bytes()); + } + } +} + +fn sample_extended12_plane_at(plane: &Extended12Plane, source_x: usize, source_y: usize) -> u16 { + let height = plane.pixels.len() / plane.stride; + let y = source_y.min(height - 1); + let x = source_x.min(plane.width - 1); + plane.pixels[y * plane.stride + x] +} + +fn upsample_extended12_plane_h2v1_at( + plane: &Extended12Plane, + source_x: usize, + source_y: usize, +) -> u16 { + let height = plane.pixels.len() / plane.stride; + let y = source_y.min(height - 1); + upsample_h2v1_12bit_at(extended12_plane_row(plane, y), source_x) +} + +fn upsample_extended12_plane_h2v2_at( + plane: &Extended12Plane, + source_x: usize, + source_y: usize, +) -> u16 { + let height = plane.pixels.len() / plane.stride; + let chroma_y = (source_y / 2).min(height - 1); + let prev_y = chroma_y.saturating_sub(1); + let next_y = (chroma_y + 1).min(height - 1); + upsample_h2v2_12bit_at( + extended12_plane_row(plane, prev_y), + extended12_plane_row(plane, chroma_y), + extended12_plane_row(plane, next_y), + source_x, + !source_y.is_multiple_of(2), + ) +} + +fn extended12_plane_row(plane: &Extended12Plane, y: usize) -> &[u16] { + let row_start = y * plane.stride; + &plane.pixels[row_start..row_start + plane.width] +} + +fn upsample_h2v1_12bit_at(row: &[u16], output_x: usize) -> u16 { + debug_assert!(!row.is_empty()); + if row.len() == 1 { + return row[0]; + } + let sample = output_x / 2; + if output_x == 0 { + row[0] + } else if output_x == row.len() * 2 - 1 { + row[row.len() - 1] + } else if output_x.is_multiple_of(2) { + ((3 * u32::from(row[sample]) + u32::from(row[sample - 1]) + 2) / 4) as u16 + } else { + ((3 * u32::from(row[sample]) + u32::from(row[sample + 1]) + 2) / 4) as u16 + } +} + +fn upsample_h2v2_12bit_at( + prev: &[u16], + curr: &[u16], + next: &[u16], + output_x: usize, + output_is_bottom: bool, +) -> u16 { + debug_assert!(!curr.is_empty()); + debug_assert_eq!(prev.len(), curr.len()); + debug_assert_eq!(next.len(), curr.len()); + let near = if output_is_bottom { next } else { prev }; + let colsum = |index: usize| 3 * u32::from(curr[index]) + u32::from(near[index]); + if curr.len() == 1 { + return ((4 * colsum(0) + 8) >> 4) as u16; + } + + let sample = output_x / 2; + let this = colsum(sample); + match output_x { + 0 => ((this * 4 + 8) >> 4) as u16, + _ if output_x == curr.len() * 2 - 1 => ((this * 4 + 7) >> 4) as u16, + _ if output_x.is_multiple_of(2) => { + let last = colsum(sample - 1); + ((this * 3 + last + 8) >> 4) as u16 + } + _ => { + let next = colsum(sample + 1); + ((this * 3 + next + 7) >> 4) as u16 + } + } +} + +fn write_extended12_block_region( + out: &mut [u8], + stride: usize, + region: Extended12WriteRegion, + block_origin: (u32, u32), + pixels: &[u16; 64], +) { + let (width, height) = region.dimensions; + let (x0, y0) = block_origin; + let block_x1 = (x0 + 8).min(width); + let block_y1 = (y0 + 8).min(height); + let denom = region.downscale.denominator(); + let bytes_per_pixel = match region.output { + Extended12Output::Gray16 => 2, + Extended12Output::Rgb16 => 6, + }; + let output_rect = region.output_rect; + for output_y in output_rect.y..output_rect.y + output_rect.h { + let source_y = output_y.saturating_mul(denom).min(height - 1); + if source_y < y0 || source_y >= block_y1 { + continue; + } + let src_row = (source_y - y0) as usize; + let dst_row = (output_y - output_rect.y) as usize; + for output_x in output_rect.x..output_rect.x + output_rect.w { + let source_x = output_x.saturating_mul(denom).min(width - 1); + if source_x < x0 || source_x >= block_x1 { + continue; + } + let src_col = (source_x - x0) as usize; + let sample = pixels[src_row * 8 + src_col].to_le_bytes(); + let dst_col = (output_x - output_rect.x) as usize; + let dst_start = dst_row * stride + dst_col * bytes_per_pixel; + let dst = &mut out[dst_start..dst_start + bytes_per_pixel]; + match region.output { + Extended12Output::Gray16 => { + dst.copy_from_slice(&sample); + } + Extended12Output::Rgb16 => { + dst[0..2].copy_from_slice(&sample); + dst[2..4].copy_from_slice(&sample); + dst[4..6].copy_from_slice(&sample); + } + } + } + } +} + +fn restart_index_for_stream( + bytes: &[u8], + scan_data_offset: Option, + info: &Info, + restart_interval: Option, +) -> Result, JpegError> { + let Some(interval_mcus) = restart_interval + .filter(|&interval| interval > 0) + .map(u32::from) + else { + return Ok(None); + }; + let scan_data_offset = scan_data_offset.ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sos, + })?; + if !matches!(info.sof_kind, SofKind::Baseline8 | SofKind::Extended8) || info.scan_count != 1 { + return Err(JpegError::NotImplemented { sof: info.sof_kind }); + } + let total_mcus = info.mcu_geometry.count; + let expected_restarts = total_mcus.saturating_sub(1) / interval_mcus; + let mut segments = Vec::new(); + segments.push(RestartSegment { + start_mcu: 0, + entropy_offset: scan_data_offset, + marker_offset: None, + marker: None, + }); + + let mut found_restarts = 0u32; + let mut expected_rst = 0xd0u8; + let mut pos = scan_data_offset; + while pos < bytes.len() { + if bytes[pos] != 0xff { + pos += 1; + continue; + } + + let mut marker_code_pos = pos + 1; + while marker_code_pos < bytes.len() && bytes[marker_code_pos] == 0xff { + marker_code_pos += 1; + } + if marker_code_pos >= bytes.len() { + return Err(JpegError::Truncated { + offset: pos, + expected: 1, + }); + } + + let marker = bytes[marker_code_pos]; + let marker_offset = marker_code_pos - 1; + match marker { + 0x00 => pos = marker_code_pos + 1, + 0xd0..=0xd7 => { + if found_restarts >= expected_restarts { + return Err(JpegError::UnexpectedMarker { + offset: marker_offset, + expected: MarkerKind::Eoi, + found: marker, + }); + } + if marker != expected_rst { + return Err(JpegError::RestartMismatch { + offset: marker_offset, + expected: expected_rst & 0x07, + found: marker, + }); + } + found_restarts += 1; + segments.push(RestartSegment { + start_mcu: found_restarts.saturating_mul(interval_mcus), + entropy_offset: marker_code_pos + 1, + marker_offset: Some(marker_offset), + marker: Some(marker), + }); + expected_rst = if expected_rst == 0xd7 { + 0xd0 + } else { + expected_rst + 1 + }; + pos = marker_code_pos + 1; + } + 0xd9 => { + if found_restarts != expected_restarts { + return Err(JpegError::UnexpectedEoi { + mcu_at: found_restarts + .saturating_add(1) + .saturating_mul(interval_mcus), + mcu_total: total_mcus, + }); + } + return Ok(Some(RestartIndex { + scan_data_offset, + interval_mcus, + segments, + })); + } + found => { + return Err(JpegError::UnexpectedMarker { + offset: marker_offset, + expected: MarkerKind::Eoi, + found, + }); + } + } + } + + Err(JpegError::MissingMarker { + marker: MarkerKind::Eoi, + }) +} + +fn output_format_profile_name(fmt: OutputFormat) -> &'static str { + match fmt { + OutputFormat::Rgb8 | OutputFormat::Rgb8Scaled { .. } => "Rgb8", + OutputFormat::Rgba8 { .. } | OutputFormat::Rgba8Scaled { .. } => "Rgba8", + OutputFormat::Gray8 | OutputFormat::Gray8Scaled { .. } => "Gray8", + OutputFormat::Gray16 | OutputFormat::Gray16Scaled { .. } => "Gray16", + OutputFormat::Rgb16 | OutputFormat::Rgb16Scaled { .. } => "Rgb16", + OutputFormat::Rgba16 { .. } | OutputFormat::Rgba16Scaled { .. } => "Rgba16", + } +} + +fn downscale_profile_name(downscale: DownscaleFactor) -> &'static str { + match downscale { + DownscaleFactor::Full => "full", + DownscaleFactor::Half => "half", + DownscaleFactor::Quarter => "quarter", + DownscaleFactor::Eighth => "eighth", + } +} + +fn emit_decode_scan_profile( + scan_path: &str, + dimensions: (u32, u32), + decoded: Rect, + downscale: DownscaleFactor, + elapsed: Duration, +) { + let source_width_s = dimensions.0.to_string(); + let source_height_s = dimensions.1.to_string(); + let decoded_x_s = decoded.x.to_string(); + let decoded_y_s = decoded.y.to_string(); + let decoded_w_s = decoded.w.to_string(); + let decoded_h_s = decoded.h.to_string(); + let scan_us = duration_us_string(elapsed); + emit_jpeg_profile_row( + "decode_scan", + "cpu", + &[ + ("scan_path", scan_path), + ("downscale", downscale_profile_name(downscale)), + ("source_width", source_width_s.as_str()), + ("source_height", source_height_s.as_str()), + ("decoded_x", decoded_x_s.as_str()), + ("decoded_y", decoded_y_s.as_str()), + ("decoded_w", decoded_w_s.as_str()), + ("decoded_h", decoded_h_s.as_str()), + ("scan_us", scan_us.as_str()), + ], + ); +} + +fn consume_lossless_restart( + br: &mut BitReader<'_>, + sample_index: u32, + total_samples: u32, + expected_rst: &mut u8, +) -> Result<(), JpegError> { + br.reset_at_restart(); + let _ = br.ensure_bits(1); + let marker = br.take_marker().ok_or(JpegError::UnexpectedEoi { + mcu_at: sample_index, + mcu_total: total_samples, + })?; + let expected = 0xD0 | *expected_rst; + if marker != expected { + return Err(JpegError::RestartMismatch { + offset: br.position(), + expected: *expected_rst, + found: marker, + }); + } + *expected_rst = (*expected_rst + 1) & 0x07; + br.reset_at_restart(); + Ok(()) +} + +fn consume_extended12_restart( + br: &mut BitReader<'_>, + mcu_index: u32, + total_mcus: u32, + expected_rst: &mut u8, +) -> Result<(), JpegError> { + let _ = br.ensure_bits(1); + let marker = br.take_marker().ok_or(JpegError::UnexpectedEoi { + mcu_at: mcu_index, + mcu_total: total_mcus, + })?; + let expected = 0xD0 | *expected_rst; + if marker != expected { + return Err(JpegError::RestartMismatch { + offset: br.position(), + expected: *expected_rst, + found: marker, + }); + } + *expected_rst = (*expected_rst + 1) & 0x07; + br.reset_at_restart(); + Ok(()) +} + +fn lossless_predictor_value(predictor: u8, out: &[u8], stride: usize, x: usize, y: usize) -> i32 { + lossless_predict(predictor, 128, x, y, |sx, sy| { + i32::from(out[sy * stride + sx]) + }) +} + +fn lossless_predictor_color_into( + predictor: u8, + out: &[u8], + stride: usize, + x: usize, + y: usize, + component: usize, +) -> i32 { + lossless_predict(predictor, P::RESTART_PREDICTOR, x, y, |sx, sy| { + P::read_le(&out[sy * stride + (sx * 3 + component) * P::BYTES..]) + }) +} + +fn lossless_predictor_gray_rows( + predictor: u8, + curr_row: &[u8], + prev_row: &[u8], + x: usize, + y: usize, +) -> i32 { + lossless_predict(predictor, P::RESTART_PREDICTOR, x, y, |sx, sy| { + let row = if sy == y { curr_row } else { prev_row }; + P::read_le(&row[sx * P::BYTES..]) + }) +} + +fn lossless_predictor_color_rows( + predictor: u8, + curr_row: &[u8], + prev_row: &[u8], + x: usize, + y: usize, + component: usize, +) -> i32 { + lossless_predict(predictor, P::RESTART_PREDICTOR, x, y, |sx, sy| { + let row = if sy == y { curr_row } else { prev_row }; + P::read_le(&row[(sx * 3 + component) * P::BYTES..]) + }) +} + +#[derive(Clone, Copy)] +struct LosslessPlaneSample { + x: usize, + y: usize, + restart_first_sample: bool, +} + +fn decode_lossless_plane_sample( + br: &mut BitReader<'_>, + table: &HuffmanTable, + predictor: u8, + plane: &mut [P], + width: usize, + sample: LosslessPlaneSample, +) -> Result<(), JpegError> { + let predicted = if sample.restart_first_sample { + P::RESTART_PREDICTOR + } else { + lossless_predictor_plane(predictor, plane, width, sample.x, sample.y) + }; + let diff = table.decode_fast_dc(br)?; + plane[sample.y * width + sample.x] = P::from_i32(predicted + diff)?; + Ok(()) +} + +fn lossless_predictor_plane( + predictor: u8, + plane: &[P], + width: usize, + x: usize, + y: usize, +) -> i32 { + lossless_predict(predictor, P::RESTART_PREDICTOR, x, y, |sx, sy| { + plane[sy * width + sx].into() + }) +} + +struct LosslessColorPlanes<'a, P> { + c0: &'a [P], + c1: &'a [P], + c2: &'a [P], +} + +fn write_lossless_color8_sampled_output( + out: &mut [u8], + stride: usize, + color_space: ColorSpace, + sampling: LosslessColorSampling, + dimensions: (usize, usize), + planes: LosslessColorPlanes<'_, u8>, +) { + let (width, height) = dimensions; + let chroma_width = width.div_ceil(2); + let chroma_height = match sampling { + LosslessColorSampling::S422 => height, + LosslessColorSampling::S420 => height.div_ceil(2), + LosslessColorSampling::S444 => unreachable!("sampled writer is not used for 4:4:4"), + }; + for y in 0..height { + for x in 0..width { + let c0_sample = planes.c0[y * width + x]; + let (c1_sample, c2_sample) = match sampling { + LosslessColorSampling::S422 => { + let c1_row = &planes.c1[y * chroma_width..(y + 1) * chroma_width]; + let c2_row = &planes.c2[y * chroma_width..(y + 1) * chroma_width]; + ( + upsample_h2v1_u8_at(c1_row, x), + upsample_h2v1_u8_at(c2_row, x), + ) + } + LosslessColorSampling::S420 => ( + upsample_h2v2_u8_at(planes.c1, chroma_width, chroma_height, width, x, y), + upsample_h2v2_u8_at(planes.c2, chroma_width, chroma_height, width, x, y), + ), + LosslessColorSampling::S444 => unreachable!("sampled writer is not used for 4:4:4"), + }; + let (r, g, b) = match color_space { + ColorSpace::Rgb => (c0_sample, c1_sample, c2_sample), + ColorSpace::YCbCr => { + crate::color::ycbcr::ycbcr_to_rgb(c0_sample, c1_sample, c2_sample) + } + _ => unreachable!("lossless sampled color path only accepts RGB/YCbCr"), + }; + let dst = y * stride + x * 3; + out[dst..dst + 3].copy_from_slice(&[r, g, b]); + } + } +} + +fn write_lossless_color16_sampled_output( + out: &mut [u8], + stride: usize, + color_space: ColorSpace, + sampling: LosslessColorSampling, + dimensions: (usize, usize), + planes: LosslessColorPlanes<'_, u16>, +) { + let (width, height) = dimensions; + let chroma_width = width.div_ceil(2); + let chroma_height = match sampling { + LosslessColorSampling::S422 => height, + LosslessColorSampling::S420 => height.div_ceil(2), + LosslessColorSampling::S444 => unreachable!("sampled writer is not used for 4:4:4"), + }; + for y in 0..height { + for x in 0..width { + let c0_sample = planes.c0[y * width + x]; + let (c1_sample, c2_sample) = match sampling { + LosslessColorSampling::S422 => { + let c1_row = &planes.c1[y * chroma_width..(y + 1) * chroma_width]; + let c2_row = &planes.c2[y * chroma_width..(y + 1) * chroma_width]; + ( + upsample_h2v1_u16_at(c1_row, x), + upsample_h2v1_u16_at(c2_row, x), + ) + } + LosslessColorSampling::S420 => ( + upsample_h2v2_u16_at(planes.c1, chroma_width, chroma_height, width, x, y), + upsample_h2v2_u16_at(planes.c2, chroma_width, chroma_height, width, x, y), + ), + LosslessColorSampling::S444 => unreachable!("sampled writer is not used for 4:4:4"), + }; + let (r, g, b) = match color_space { + ColorSpace::Rgb => (c0_sample, c1_sample, c2_sample), + ColorSpace::YCbCr => { + crate::color::ycbcr::ycbcr16_to_rgb16(c0_sample, c1_sample, c2_sample) + } + _ => unreachable!("lossless 4:2:2 color path only accepts RGB/YCbCr"), + }; + let dst = y * stride + x * 6; + out[dst..dst + 2].copy_from_slice(&r.to_le_bytes()); + out[dst + 2..dst + 4].copy_from_slice(&g.to_le_bytes()); + out[dst + 4..dst + 6].copy_from_slice(&b.to_le_bytes()); + } + } +} + +fn upsample_h2v2_u8_at( + plane: &[u8], + chroma_width: usize, + chroma_height: usize, + output_width: usize, + output_x: usize, + output_y: usize, +) -> u8 { + debug_assert!(!plane.is_empty()); + debug_assert!(chroma_width > 0); + debug_assert!(chroma_height > 0); + let chroma_y = output_y / 2; + let current = &plane[chroma_y * chroma_width..(chroma_y + 1) * chroma_width]; + let near_y = if output_y.is_multiple_of(2) { + chroma_y.saturating_sub(1) + } else { + (chroma_y + 1).min(chroma_height - 1) + }; + let near = &plane[near_y * chroma_width..(near_y + 1) * chroma_width]; + let colsum = |index: usize| 3 * u32::from(current[index]) + u32::from(near[index]); + if chroma_width == 1 { + return ((4 * colsum(0) + 8) >> 4) as u8; + } + + let sample = output_x / 2; + let this = colsum(sample); + match output_x { + 0 => ((this * 4 + 8) >> 4) as u8, + _ if output_x == output_width - 1 => ((this * 4 + 7) >> 4) as u8, + _ if output_x.is_multiple_of(2) => { + let last = colsum(sample - 1); + ((this * 3 + last + 8) >> 4) as u8 + } + _ => { + let next = colsum(sample + 1); + ((this * 3 + next + 7) >> 4) as u8 + } + } +} + +fn upsample_h2v1_u8_at(row: &[u8], output_x: usize) -> u8 { + debug_assert!(!row.is_empty()); + if row.len() == 1 { + return row[0]; + } + let sample = output_x / 2; + if output_x == 0 { + row[0] + } else if output_x == row.len() * 2 - 1 { + row[row.len() - 1] + } else if output_x.is_multiple_of(2) { + ((3 * u32::from(row[sample]) + u32::from(row[sample - 1]) + 2) / 4) as u8 + } else { + ((3 * u32::from(row[sample]) + u32::from(row[sample + 1]) + 2) / 4) as u8 + } +} + +fn upsample_h2v2_u16_at( + plane: &[u16], + chroma_width: usize, + chroma_height: usize, + output_width: usize, + output_x: usize, + output_y: usize, +) -> u16 { + debug_assert!(!plane.is_empty()); + debug_assert!(chroma_width > 0); + debug_assert!(chroma_height > 0); + let chroma_y = output_y / 2; + let current = &plane[chroma_y * chroma_width..(chroma_y + 1) * chroma_width]; + let near_y = if output_y.is_multiple_of(2) { + chroma_y.saturating_sub(1) + } else { + (chroma_y + 1).min(chroma_height - 1) + }; + let near = &plane[near_y * chroma_width..(near_y + 1) * chroma_width]; + let colsum = |index: usize| 3 * u32::from(current[index]) + u32::from(near[index]); + if chroma_width == 1 { + return ((4 * colsum(0) + 8) >> 4) as u16; + } + + let sample = output_x / 2; + let this = colsum(sample); + match output_x { + 0 => ((this * 4 + 8) >> 4) as u16, + _ if output_x == output_width - 1 => ((this * 4 + 7) >> 4) as u16, + _ if output_x.is_multiple_of(2) => { + let last = colsum(sample - 1); + ((this * 3 + last + 8) >> 4) as u16 + } + _ => { + let next = colsum(sample + 1); + ((this * 3 + next + 7) >> 4) as u16 + } + } +} + +fn upsample_h2v1_u16_at(row: &[u16], output_x: usize) -> u16 { + debug_assert!(!row.is_empty()); + if row.len() == 1 { + return row[0]; + } + let sample = output_x / 2; + if output_x == 0 { + row[0] + } else if output_x == row.len() * 2 - 1 { + row[row.len() - 1] + } else if output_x.is_multiple_of(2) { + ((3 * u32::from(row[sample]) + u32::from(row[sample - 1]) + 2) / 4) as u16 + } else { + ((3 * u32::from(row[sample]) + u32::from(row[sample + 1]) + 2) / 4) as u16 + } +} + +fn lossless_predictor_value_u16( + predictor: u8, + out: &[u8], + stride: usize, + x: usize, + y: usize, +) -> i32 { + lossless_predict(predictor, 32768, x, y, |sx, sy| { + i32::from(read_gray16_sample(out, sy * stride + sx * 2)) + }) +} + +fn read_gray16_sample(out: &[u8], offset: usize) -> u16 { + u16::from_le_bytes([out[offset], out[offset + 1]]) +} + +fn merged_warnings(header_warnings: &[Warning], scan_warnings: Vec) -> Vec { + if header_warnings.is_empty() { + return scan_warnings; + } + if scan_warnings.is_empty() { + return header_warnings.to_vec(); + } + let mut warnings = Vec::with_capacity(header_warnings.len() + scan_warnings.len()); + warnings.extend_from_slice(header_warnings); + warnings.extend(scan_warnings); + warnings +} + +fn copy_gray8_scaled_rect( + full: &[u8], + dimensions: (u32, u32), + output_rect: Rect, + denom: u32, + out: &mut [u8], + stride: usize, +) { + let (width, height) = dimensions; + for output_y in output_rect.y..output_rect.y + output_rect.h { + let source_y = output_y.saturating_mul(denom).min(height - 1); + let dst_row = (output_y - output_rect.y) as usize; + let dst_start = dst_row * stride; + let dst = &mut out[dst_start..dst_start + output_rect.w as usize]; + for (dst_px, output_x) in (output_rect.x..output_rect.x + output_rect.w).enumerate() { + let source_x = output_x.saturating_mul(denom).min(width - 1); + let src = source_y as usize * width as usize + source_x as usize; + dst[dst_px] = full[src]; + } + } +} + +fn copy_rgb8_scaled_rect( + full: &[u8], + dimensions: (u32, u32), + output_rect: Rect, + denom: u32, + out: &mut [u8], + stride: usize, +) { + let (width, height) = dimensions; + for output_y in output_rect.y..output_rect.y + output_rect.h { + let source_y = output_y.saturating_mul(denom).min(height - 1); + let dst_row = (output_y - output_rect.y) as usize; + for output_x in output_rect.x..output_rect.x + output_rect.w { + let source_x = output_x.saturating_mul(denom).min(width - 1); + let src = (source_y as usize * width as usize + source_x as usize) * 3; + let dst = dst_row * stride + (output_x - output_rect.x) as usize * 3; + out[dst..dst + 3].copy_from_slice(&full[src..src + 3]); + } + } +} + +fn convert_ycbcr8_to_rgb8_in_place(out: &mut [u8], stride: usize, dimensions: (u32, u32)) { + let (width, height) = dimensions; + let row_bytes = width as usize * 3; + for y in 0..height as usize { + let row = &mut out[y * stride..y * stride + row_bytes]; + for pixel in row.chunks_exact_mut(3) { + let (r, g, b) = crate::color::ycbcr::ycbcr_to_rgb(pixel[0], pixel[1], pixel[2]); + pixel.copy_from_slice(&[r, g, b]); + } + } +} + +fn copy_ycbcr8_row_to_rgb8(src: &[u8], dst: &mut [u8]) { + debug_assert_eq!(src.len(), dst.len()); + for (source, target) in src.chunks_exact(3).zip(dst.chunks_exact_mut(3)) { + let (r, g, b) = crate::color::ycbcr::ycbcr_to_rgb(source[0], source[1], source[2]); + target.copy_from_slice(&[r, g, b]); + } +} + +fn copy_rgb8_to_rgba8( + src: &[u8], + src_stride: usize, + width: u32, + height: u32, + dst: &mut [u8], + dst_stride: usize, + alpha: u8, +) { + let src_row_bytes = width as usize * 3; + let dst_row_bytes = width as usize * 4; + for y in 0..height as usize { + let src_row = &src[y * src_stride..y * src_stride + src_row_bytes]; + let dst_row = &mut dst[y * dst_stride..y * dst_stride + dst_row_bytes]; + for (source, target) in src_row.chunks_exact(3).zip(dst_row.chunks_exact_mut(4)) { + target.copy_from_slice(&[source[0], source[1], source[2], alpha]); + } + } +} + +fn copy_rgb16_to_rgba16( + src: &[u8], + src_stride: usize, + width: u32, + height: u32, + dst: &mut [u8], + dst_stride: usize, + alpha: u16, +) { + let src_row_bytes = width as usize * 6; + let dst_row_bytes = width as usize * 8; + let alpha = alpha.to_le_bytes(); + for y in 0..height as usize { + let src_row = &src[y * src_stride..y * src_stride + src_row_bytes]; + let dst_row = &mut dst[y * dst_stride..y * dst_stride + dst_row_bytes]; + for (source, target) in src_row.chunks_exact(6).zip(dst_row.chunks_exact_mut(8)) { + target[..6].copy_from_slice(source); + target[6..8].copy_from_slice(&alpha); + } + } +} + +fn convert_ycbcr16_to_rgb16_in_place(out: &mut [u8], stride: usize, dimensions: (u32, u32)) { + let (width, height) = dimensions; + let row_bytes = width as usize * 6; + for y in 0..height as usize { + let row = &mut out[y * stride..y * stride + row_bytes]; + for pixel in row.chunks_exact_mut(6) { + let y = u16::from_le_bytes([pixel[0], pixel[1]]); + let cb = u16::from_le_bytes([pixel[2], pixel[3]]); + let cr = u16::from_le_bytes([pixel[4], pixel[5]]); + let (r, g, b) = crate::color::ycbcr::ycbcr16_to_rgb16(y, cb, cr); + pixel[0..2].copy_from_slice(&r.to_le_bytes()); + pixel[2..4].copy_from_slice(&g.to_le_bytes()); + pixel[4..6].copy_from_slice(&b.to_le_bytes()); + } + } +} + +fn copy_ycbcr16_row_to_rgb16(src: &[u8], dst: &mut [u8]) { + debug_assert_eq!(src.len(), dst.len()); + for (source, target) in src.chunks_exact(6).zip(dst.chunks_exact_mut(6)) { + let y = u16::from_le_bytes([source[0], source[1]]); + let cb = u16::from_le_bytes([source[2], source[3]]); + let cr = u16::from_le_bytes([source[4], source[5]]); + let (r, g, b) = crate::color::ycbcr::ycbcr16_to_rgb16(y, cb, cr); + target[0..2].copy_from_slice(&r.to_le_bytes()); + target[2..4].copy_from_slice(&g.to_le_bytes()); + target[4..6].copy_from_slice(&b.to_le_bytes()); + } +} + +fn copy_rgb16_scaled_rect( + full: &[u8], + dimensions: (u32, u32), + output_rect: Rect, + denom: u32, + out: &mut [u8], + stride: usize, +) { + let (width, height) = dimensions; + let full_stride = width as usize * 6; + for output_y in output_rect.y..output_rect.y + output_rect.h { + let source_y = output_y.saturating_mul(denom).min(height - 1); + let dst_row = (output_y - output_rect.y) as usize; + for output_x in output_rect.x..output_rect.x + output_rect.w { + let source_x = output_x.saturating_mul(denom).min(width - 1); + let src = source_y as usize * full_stride + source_x as usize * 6; + let dst = dst_row * stride + (output_x - output_rect.x) as usize * 6; + out[dst..dst + 6].copy_from_slice(&full[src..src + 6]); + } + } +} + +fn copy_gray16_scaled_rect( + full: &[u8], + dimensions: (u32, u32), + output_rect: Rect, + denom: u32, + out: &mut [u8], + stride: usize, +) { + let (width, height) = dimensions; + let full_stride = width as usize * 2; + for output_y in output_rect.y..output_rect.y + output_rect.h { + let source_y = output_y.saturating_mul(denom).min(height - 1); + let dst_row = (output_y - output_rect.y) as usize; + for output_x in output_rect.x..output_rect.x + output_rect.w { + let source_x = output_x.saturating_mul(denom).min(width - 1); + let src = source_y as usize * full_stride + source_x as usize * 2; + let dst = dst_row * stride + (output_x - output_rect.x) as usize * 2; + out[dst..dst + 2].copy_from_slice(&full[src..src + 2]); + } + } +} + +fn jpeg_passthrough_syntax(info: &Info) -> Option { + match info.sof_kind { + SofKind::Baseline8 if info.bit_depth == 8 => Some(CompressedTransferSyntax::JpegBaseline8), + SofKind::Extended8 | SofKind::Extended12 => { + Some(CompressedTransferSyntax::JpegExtendedSequential) + } + SofKind::Baseline8 | SofKind::Progressive8 | SofKind::Progressive12 | SofKind::Lossless => { + None + } + } +} + +fn core_outcome(outcome: DecodeOutcome) -> CoreDecodeOutcome { + outcome.into() +} + +fn jpeg_downscale(scale: Downscale) -> DownscaleFactor { + match scale { + Downscale::None => DownscaleFactor::Full, + Downscale::Half => DownscaleFactor::Half, + Downscale::Quarter => DownscaleFactor::Quarter, + Downscale::Eighth => DownscaleFactor::Eighth, + _ => unreachable!("unsupported Downscale variant"), + } +} + +fn output_format_from_parts( + sof_kind: SofKind, + fmt: PixelFormat, + scale: Downscale, +) -> Result { + if matches!(sof_kind, SofKind::Extended12 | SofKind::Progressive12) { + return match (sof_kind, fmt, scale) { + (SofKind::Extended12, PixelFormat::Gray16, Downscale::None) => Ok(OutputFormat::Gray16), + (SofKind::Extended12, PixelFormat::Gray16, scale) => Ok(OutputFormat::Gray16Scaled { + factor: jpeg_downscale(scale), + }), + (SofKind::Extended12, PixelFormat::Rgb16, Downscale::None) => Ok(OutputFormat::Rgb16), + (SofKind::Extended12, PixelFormat::Rgb16, scale) => Ok(OutputFormat::Rgb16Scaled { + factor: jpeg_downscale(scale), + }), + (SofKind::Extended12, PixelFormat::Rgba16, Downscale::None) => { + Ok(OutputFormat::Rgba16 { alpha: u16::MAX }) + } + (SofKind::Extended12, PixelFormat::Rgba16, scale) => Ok(OutputFormat::Rgba16Scaled { + alpha: u16::MAX, + factor: jpeg_downscale(scale), + }), + (SofKind::Progressive12, PixelFormat::Gray16, Downscale::None) => { + Ok(OutputFormat::Gray16) + } + (SofKind::Progressive12, PixelFormat::Gray16, scale) => { + Ok(OutputFormat::Gray16Scaled { + factor: jpeg_downscale(scale), + }) + } + (SofKind::Progressive12, PixelFormat::Rgb16, Downscale::None) => { + Ok(OutputFormat::Rgb16) + } + (SofKind::Progressive12, PixelFormat::Rgb16, scale) => Ok(OutputFormat::Rgb16Scaled { + factor: jpeg_downscale(scale), + }), + (SofKind::Progressive12, PixelFormat::Rgba16, Downscale::None) => { + Ok(OutputFormat::Rgba16 { alpha: u16::MAX }) + } + (SofKind::Progressive12, PixelFormat::Rgba16, scale) => { + Ok(OutputFormat::Rgba16Scaled { + alpha: u16::MAX, + factor: jpeg_downscale(scale), + }) + } + (_, PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16, _) => { + Err(JpegError::NotImplemented { sof: sof_kind }) + } + _ => Err(JpegError::UnsupportedBitDepth { depth: 12 }), + }; + } + if sof_kind == SofKind::Lossless { + return match (fmt, scale) { + (PixelFormat::Gray8, Downscale::None) => Ok(OutputFormat::Gray8), + (PixelFormat::Gray8, scale) => Ok(OutputFormat::Gray8Scaled { + factor: jpeg_downscale(scale), + }), + (PixelFormat::Gray16, Downscale::None) => Ok(OutputFormat::Gray16), + (PixelFormat::Gray16, scale) => Ok(OutputFormat::Gray16Scaled { + factor: jpeg_downscale(scale), + }), + (PixelFormat::Rgb8, Downscale::None) => Ok(OutputFormat::Rgb8), + (PixelFormat::Rgb8, scale) => Ok(OutputFormat::Rgb8Scaled { + factor: jpeg_downscale(scale), + }), + (PixelFormat::Rgba8, Downscale::None) => Ok(OutputFormat::Rgba8 { alpha: 255 }), + (PixelFormat::Rgba8, scale) => Ok(OutputFormat::Rgba8Scaled { + alpha: 255, + factor: jpeg_downscale(scale), + }), + (PixelFormat::Rgb16, Downscale::None) => Ok(OutputFormat::Rgb16), + (PixelFormat::Rgb16, scale) => Ok(OutputFormat::Rgb16Scaled { + factor: jpeg_downscale(scale), + }), + (PixelFormat::Rgba16, Downscale::None) => Ok(OutputFormat::Rgba16 { alpha: u16::MAX }), + (PixelFormat::Rgba16, scale) => Ok(OutputFormat::Rgba16Scaled { + alpha: u16::MAX, + factor: jpeg_downscale(scale), + }), + _ => Err(JpegError::NotImplemented { sof: sof_kind }), + }; + } + + match (fmt, scale) { + (PixelFormat::Rgb8, Downscale::None) => Ok(OutputFormat::Rgb8), + (PixelFormat::Rgb8, scale) => Ok(OutputFormat::Rgb8Scaled { + factor: jpeg_downscale(scale), + }), + (PixelFormat::Gray8, Downscale::None) => Ok(OutputFormat::Gray8), + (PixelFormat::Gray8, scale) => Ok(OutputFormat::Gray8Scaled { + factor: jpeg_downscale(scale), + }), + (PixelFormat::Rgba8, Downscale::None) => Ok(OutputFormat::Rgba8 { alpha: 255 }), + (PixelFormat::Rgba8, scale) => Ok(OutputFormat::Rgba8Scaled { + alpha: 255, + factor: jpeg_downscale(scale), + }), + (PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16, _) => { + Err(JpegError::UnsupportedBitDepth { depth: 16 }) + } + _ => Err(JpegError::DownscaleUnsupported { sof: sof_kind }), + } +} + +impl ImageCodec for JpegCodec { + type Error = JpegError; + type Warning = Warning; + type Pool = ScratchPool; +} + +impl ImageCodec for Decoder<'_> { + type Error = JpegError; + type Warning = Warning; + type Pool = ScratchPool; +} + +impl<'a> ImageDecode<'a> for Decoder<'a> { + type View = JpegView<'a>; + + fn inspect(input: &'a [u8]) -> Result { + Ok(Decoder::inspect(input)?.to_core_info()) + } + + fn parse(input: &'a [u8]) -> Result { + JpegView::parse(input) + } + + fn from_view(view: Self::View) -> Result { + Decoder::from_view(view) + } + + fn decode_into( + &mut self, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + ) -> Result, Self::Error> { + Decoder::decode_into(self, out, stride, fmt).map(core_outcome) + } + + fn decode_into_with_scratch( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + ) -> Result, Self::Error> { + Decoder::decode_into_with_scratch(self, pool, out, stride, fmt).map(core_outcome) + } + + fn decode_region_into( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: j2k_core::Rect, + ) -> Result, Self::Error> { + Decoder::decode_region_into_with_scratch(self, pool, out, stride, fmt, roi.into()) + .map(core_outcome) + } + + fn decode_scaled_into( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + scale: Downscale, + ) -> Result, Self::Error> { + Decoder::decode_scaled_into_with_scratch(self, pool, out, stride, fmt, scale) + .map(core_outcome) + } + + fn decode_region_scaled_into( + &mut self, + pool: &mut Self::Pool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: j2k_core::Rect, + scale: Downscale, + ) -> Result, Self::Error> { + Decoder::decode_region_scaled_into_with_scratch( + self, + pool, + out, + stride, + fmt, + roi.into(), + scale, + ) + .map(core_outcome) + } +} + +struct CoreRowSinkAdapter<'a, R: RowSink> { + sink: &'a mut R, + sink_error: Option, +} + +impl> RowSink for CoreRowSinkAdapter<'_, R> { + type Error = JpegError; + + fn write_row(&mut self, y: u32, row: &[u8]) -> Result<(), JpegError> { + match self.sink.write_row(y, row) { + Ok(()) => Ok(()), + Err(err) => { + self.sink_error = Some(err); + Err(JpegError::RowSinkAborted) + } + } + } +} + +impl<'a> ImageDecodeRows<'a, u8> for Decoder<'a> { + fn decode_rows>( + &mut self, + sink: &mut R, + ) -> Result, DecodeRowsError> { + let mut adapter = CoreRowSinkAdapter { + sink, + sink_error: None, + }; + match Decoder::decode_rows(self, &mut adapter) { + Ok(outcome) => Ok(core_outcome(outcome)), + Err(JpegError::RowSinkAborted) => Err(DecodeRowsError::Sink( + adapter + .sink_error + .expect("row sink abort stores the original sink error"), + )), + Err(err) => Err(DecodeRowsError::Decode(err)), + } + } +} + +impl TileBatchDecode for JpegCodec { + type Context = DecoderContext; + + fn decode_tile( + ctx: &mut CoreDecoderContext, + pool: &mut Self::Pool, + input: &[u8], + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + ) -> Result, Self::Error> { + let dec = Decoder::from_view_in_context(JpegView::parse(input)?, ctx.codec_mut())?; + dec.decode_into_with_scratch(pool, out, stride, fmt) + .map(core_outcome) + } + + fn decode_tile_region( + ctx: &mut CoreDecoderContext, + pool: &mut Self::Pool, + input: &[u8], + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: j2k_core::Rect, + ) -> Result, Self::Error> { + let dec = Decoder::from_view_in_context(JpegView::parse(input)?, ctx.codec_mut())?; + dec.decode_region_into_with_scratch(pool, out, stride, fmt, roi.into()) + .map(core_outcome) + } + + fn decode_tile_scaled( + ctx: &mut CoreDecoderContext, + pool: &mut Self::Pool, + input: &[u8], + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + scale: Downscale, + ) -> Result, Self::Error> { + let dec = Decoder::from_view_in_context(JpegView::parse(input)?, ctx.codec_mut())?; + dec.decode_scaled_into_with_scratch(pool, out, stride, fmt, scale) + .map(core_outcome) + } + + fn decode_tile_region_scaled( + ctx: &mut CoreDecoderContext, + pool: &mut Self::Pool, + input: &[u8], + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: j2k_core::Rect, + scale: Downscale, + ) -> Result, Self::Error> { + let dec = Decoder::from_view_in_context(JpegView::parse(input)?, ctx.codec_mut())?; + dec.decode_region_scaled_into_with_scratch(pool, out, stride, fmt, roi.into(), scale) + .map(core_outcome) + } +} + +#[allow(clippy::uninit_vec)] +fn allocate_output_buffer(len: usize) -> Vec { + let mut out = Vec::with_capacity(len); + // Safety: all owned-output entrypoints use tight row strides, and the + // decode writers fully initialize every byte in the destination on success. + // If decode returns an error, dropping a Vec with uninitialized bytes is + // still sound because `u8` has no drop glue. + unsafe { + out.set_len(len); + } + out +} + +fn scaled_dimensions(dims: (u32, u32), factor: DownscaleFactor) -> (u32, u32) { + let denom = factor.denominator(); + (dims.0.div_ceil(denom), dims.1.div_ceil(denom)) +} + +fn scaled_rect_covering(rect: Rect, factor: DownscaleFactor) -> Result { + let denom = factor.denominator(); + let x_end = rect + .x + .checked_add(rect.w) + .ok_or(JpegError::RectOutOfBounds { + rect, + width: u32::MAX, + height: u32::MAX, + })?; + let y_end = rect + .y + .checked_add(rect.h) + .ok_or(JpegError::RectOutOfBounds { + rect, + width: u32::MAX, + height: u32::MAX, + })?; + let x0 = rect.x / denom; + let y0 = rect.y / denom; + let x1 = x_end.div_ceil(denom); + let y1 = y_end.div_ceil(denom); + Ok(Rect { + x: x0, + y: y0, + w: x1.saturating_sub(x0), + h: y1.saturating_sub(y0), + }) +} + +struct CroppedWriter { + inner: W, + rect: Rect, + source_x0: u32, + source_width: u32, + top_row: Vec, + bottom_row: Vec, +} + +struct ProgressiveDownscaleWriter<'a, W> { + inner: &'a mut W, + denom: u32, + scaled_width: usize, + r: Vec, + g: Vec, + b: Vec, +} + +impl<'a, W> ProgressiveDownscaleWriter<'a, W> { + fn new(inner: &'a mut W, downscale: DownscaleFactor, dimensions: (u32, u32)) -> Self { + let denom = downscale.denominator(); + let scaled_width = dimensions.0.div_ceil(denom) as usize; + Self { + inner, + denom, + scaled_width, + r: Vec::new(), + g: Vec::new(), + b: Vec::new(), + } + } + + fn should_emit(&self, y: u32) -> bool { + y.is_multiple_of(self.denom) + } + + fn sample_row(src: &[u8], denom: u32, width: usize, dst: &mut Vec) { + dst.resize(width, 0); + for (x, out) in dst.iter_mut().enumerate() { + let src_x = (x as u32) + .saturating_mul(denom) + .min(src.len().saturating_sub(1) as u32); + *out = src[src_x as usize]; + } + } +} + +impl OutputWriter for ProgressiveDownscaleWriter<'_, W> { + fn write_rgb_row( + &mut self, + y: u32, + r_row: &[u8], + g_row: &[u8], + b_row: &[u8], + ) -> Result<(), JpegError> { + if !self.should_emit(y) { + return Ok(()); + } + Self::sample_row(r_row, self.denom, self.scaled_width, &mut self.r); + Self::sample_row(g_row, self.denom, self.scaled_width, &mut self.g); + Self::sample_row(b_row, self.denom, self.scaled_width, &mut self.b); + self.inner + .write_rgb_row(y / self.denom, &self.r, &self.g, &self.b) + } + + fn write_ycbcr_row( + &mut self, + y: u32, + y_row: &[u8], + cb_row: &[u8], + cr_row: &[u8], + ) -> Result<(), JpegError> { + if !self.should_emit(y) { + return Ok(()); + } + Self::sample_row(y_row, self.denom, self.scaled_width, &mut self.r); + Self::sample_row(cb_row, self.denom, self.scaled_width, &mut self.g); + Self::sample_row(cr_row, self.denom, self.scaled_width, &mut self.b); + self.inner + .write_ycbcr_row(y / self.denom, &self.r, &self.g, &self.b) + } + + fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError> { + if !self.should_emit(y) { + return Ok(()); + } + Self::sample_row(gray_row, self.denom, self.scaled_width, &mut self.r); + self.inner.write_gray_row(y / self.denom, &self.r) + } +} + +struct ComponentWriterAdapter<'a, W> { + inner: &'a mut W, +} + +impl OutputWriter for ComponentWriterAdapter<'_, W> { + fn write_rgb_row( + &mut self, + y: u32, + r_row: &[u8], + g_row: &[u8], + b_row: &[u8], + ) -> Result<(), JpegError> { + self.inner.write_rgb_row(y, r_row, g_row, b_row) + } + + fn write_ycbcr_row( + &mut self, + y: u32, + y_row: &[u8], + cb_row: &[u8], + cr_row: &[u8], + ) -> Result<(), JpegError> { + self.inner.write_ycbcr_row(y, y_row, cb_row, cr_row) + } + + fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError> { + self.inner.write_gray_row(y, gray_row) + } +} + +impl CroppedWriter { + fn new(inner: W, rect: Rect, source_x0: u32, source_width: u32) -> Self { + let row_len = source_width as usize * 3; + Self { + inner, + rect, + source_x0, + source_width, + top_row: vec![0; row_len], + bottom_row: vec![0; row_len], + } + } +} + +impl OutputWriter for CroppedWriter { + fn write_rgb_row( + &mut self, + y: u32, + r_row: &[u8], + g_row: &[u8], + b_row: &[u8], + ) -> Result<(), JpegError> { + if y < self.rect.y || y >= self.rect.y + self.rect.h { + return Ok(()); + } + let x0 = self + .rect + .x + .checked_sub(self.source_x0) + .expect("crop window must cover requested rect") as usize; + let x1 = x0 + self.rect.w as usize; + self.inner.write_rgb_row( + y - self.rect.y, + &r_row[x0..x1], + &g_row[x0..x1], + &b_row[x0..x1], + ) + } + + fn write_ycbcr_row( + &mut self, + y: u32, + y_row: &[u8], + cb_row: &[u8], + cr_row: &[u8], + ) -> Result<(), JpegError> { + if y < self.rect.y || y >= self.rect.y + self.rect.h { + return Ok(()); + } + let x0 = self + .rect + .x + .checked_sub(self.source_x0) + .expect("crop window must cover requested rect") as usize; + let x1 = x0 + self.rect.w as usize; + self.inner.write_ycbcr_row( + y - self.rect.y, + &y_row[x0..x1], + &cb_row[x0..x1], + &cr_row[x0..x1], + ) + } + + fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError> { + if y < self.rect.y || y >= self.rect.y + self.rect.h { + return Ok(()); + } + let x0 = self + .rect + .x + .checked_sub(self.source_x0) + .expect("crop window must cover requested rect") as usize; + let x1 = x0 + self.rect.w as usize; + self.inner + .write_gray_row(y - self.rect.y, &gray_row[x0..x1]) + } +} + +impl InterleavedRgbWriter for CroppedWriter { + fn with_rgb_rows(&mut self, y: u32, row_count: usize, fill: F) -> Result + where + F: FnOnce(&mut [u8], Option<&mut [u8]>) -> Result, + { + let row_len = self.source_width as usize * 3; + if self.top_row.len() != row_len { + self.top_row.resize(row_len, 0); + self.bottom_row.resize(row_len, 0); + } + + let result = match row_count { + 1 => fill(&mut self.top_row, None)?, + 2 => fill(&mut self.top_row, Some(&mut self.bottom_row))?, + _ => unreachable!("CroppedWriter only supports one or two rows"), + }; + + let top_in = y >= self.rect.y && y < self.rect.y + self.rect.h; + let bottom_y = y + 1; + let bottom_in = + row_count == 2 && bottom_y >= self.rect.y && bottom_y < self.rect.y + self.rect.h; + let x0 = self + .rect + .x + .checked_sub(self.source_x0) + .expect("crop window must cover requested rect") as usize + * 3; + let x1 = x0 + self.rect.w as usize * 3; + + match (top_in, bottom_in) { + (false, false) => {} + (true, false) => { + self.inner.with_rgb_rows(y - self.rect.y, 1, |dst, _| { + dst.copy_from_slice(&self.top_row[x0..x1]); + Ok(()) + })?; + } + (false, true) => { + self.inner + .with_rgb_rows(bottom_y - self.rect.y, 1, |dst, _| { + dst.copy_from_slice(&self.bottom_row[x0..x1]); + Ok(()) + })?; + } + (true, true) => { + self.inner + .with_rgb_rows(y - self.rect.y, 2, |dst_top, dst_bottom| { + dst_top.copy_from_slice(&self.top_row[x0..x1]); + dst_bottom + .expect("row_count=2 supplies bottom row") + .copy_from_slice(&self.bottom_row[x0..x1]); + Ok(()) + })?; + } + } + + Ok(result) + } +} + +fn build_decode_plan( + header: &ParsedHeader, + info: &Info, + dc_tables: &[Option>; 4], + ac_tables: &[Option>; 4], + ctx: &mut DecoderContext, +) -> Result { + let scan = header.scan.as_ref().ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sos, + })?; + let scan_offset = header.sos_offset.ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sos, + })?; + + let mut components = Vec::with_capacity(scan.components.len()); + for scan_comp in scan.components.iter().copied() { + let output_index = find_component_index(&header.component_ids, scan_comp.id).ok_or( + JpegError::UnknownScanComponent { + offset: scan_offset, + component: scan_comp.id, + }, + )?; + let (h, v) = header + .sampling + .component(output_index) + .ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sof, + })?; + let quant_id = + *header + .quant_table_ids + .get(output_index) + .ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sof, + })? as usize; + let quant = *header + .quant_tables + .entries + .get(quant_id) + .and_then(|q| q.as_ref()) + .ok_or(JpegError::MissingQuantTable { + component: scan_comp.id, + table_id: quant_id as u8, + })?; + let dc_table = dc_tables[scan_comp.dc_table as usize].as_ref().ok_or( + JpegError::MissingHuffmanTable { + component: scan_comp.id, + class: 0, + id: scan_comp.dc_table, + }, + )?; + let ac_table = ac_tables[scan_comp.ac_table as usize].as_ref().ok_or( + JpegError::MissingHuffmanTable { + component: scan_comp.id, + class: 1, + id: scan_comp.ac_table, + }, + )?; + components.push(PreparedComponentPlan { + h, + v, + output_index, + quant: ctx.resolve_quant_table(quant), + dc_table: Arc::clone(dc_table), + ac_table: Arc::clone(ac_table), + }); + } + + let mut scratch_bytes = + compute_decode_scratch_bytes(info.dimensions, info.sampling, DEFAULT_MAX_DECODE_BYTES)?; + if info.sof_kind == SofKind::Extended12 { + // The sequential 12-bit paths render through full-frame u16 component + // planes, which dwarf the stripe-based estimate above. + scratch_bytes = scratch_bytes.max(compute_extended12_planes_scratch_bytes( + &components, + info.dimensions, + info.sampling, + DEFAULT_MAX_DECODE_BYTES, + )?); + } + + Ok(PreparedDecodePlan { + components, + sampling: info.sampling, + color_space: info.color_space, + restart_interval: header.restart_interval, + dimensions: info.dimensions, + scan_offset, + scratch_bytes, + }) +} + +fn validate_sampling_factors(header: &ParsedHeader, info: &Info) -> Result<(), JpegError> { + validate_leading_component_sampling(header, info)?; + for (h, v) in header.sampling.iter() { + if h == 0 || v == 0 || h > 4 || v > 4 { + return Err(JpegError::NotImplemented { sof: info.sof_kind }); + } + if !header.sampling.max_h.is_multiple_of(h) || !header.sampling.max_v.is_multiple_of(v) { + return Err(JpegError::NotImplemented { sof: info.sof_kind }); + } + } + Ok(()) +} + +fn validate_leading_component_sampling( + header: &ParsedHeader, + info: &Info, +) -> Result<(), JpegError> { + if !matches!(info.color_space, ColorSpace::YCbCr) { + return Ok(()); + } + if let Some((h, v)) = header.sampling.component(0) { + if h != header.sampling.max_h || v != header.sampling.max_v { + return Err(JpegError::NotImplemented { sof: info.sof_kind }); + } + } + Ok(()) +} + +fn resolve_progressive_huffman( + ctx: &mut DecoderContext, + tables: &[Option; 4], + component: u8, + class: u8, + id: u8, +) -> Result, JpegError> { + let raw = tables + .get(id as usize) + .and_then(|table| table.as_ref()) + .ok_or(JpegError::MissingHuffmanTable { + component, + class, + id, + })?; + ctx.resolve_huffman_table(raw) +} + +fn compute_progressive_scratch_bytes( + components: &[PreparedProgressiveComponentPlan], + output_width: usize, +) -> Result { + let cap = DEFAULT_MAX_DECODE_BYTES; + let mut total = 0usize; + for component in components { + let blocks = checked_usize_product( + &[component.block_cols as usize, component.block_rows as usize], + cap, + )?; + let coeffs = checked_usize_product(&[blocks, 64, core::mem::size_of::()], cap)?; + total = total + .checked_add(coeffs) + .ok_or(JpegError::MemoryCapExceeded { + requested: usize::MAX, + cap, + })?; + + let plane = checked_usize_product( + &[ + component.block_cols as usize, + component.block_rows as usize, + 64, + ], + cap, + )?; + total = total + .checked_add(plane) + .ok_or(JpegError::MemoryCapExceeded { + requested: usize::MAX, + cap, + })?; + if total > cap { + return Err(JpegError::MemoryCapExceeded { + requested: total, + cap, + }); + } + } + total = + total + .checked_add(output_width.saturating_mul(3)) + .ok_or(JpegError::MemoryCapExceeded { + requested: usize::MAX, + cap, + })?; + if total > cap { + return Err(JpegError::MemoryCapExceeded { + requested: total, + cap, + }); + } + Ok(total) +} + +struct SinkWriter<'a, S> { + sink: &'a mut S, + rows: SinkRows, + backend: Backend, +} + +impl<'a, S> SinkWriter<'a, S> { + fn new(sink: &'a mut S, rows: SinkRows, backend: Backend) -> Self { + debug_assert_eq!(rows.top_row.len(), rows.bottom_row.len()); + Self { + sink, + rows, + backend, + } + } + + fn into_rows(self) -> SinkRows { + self.rows + } +} + +impl InterleavedRgbWriter for SinkWriter<'_, S> +where + S: RowSink, +{ + fn with_rgb_rows(&mut self, y: u32, row_count: usize, fill: F) -> Result + where + F: FnOnce(&mut [u8], Option<&mut [u8]>) -> Result, + { + let result = match row_count { + 1 => fill(&mut self.rows.top_row, None), + 2 => fill(&mut self.rows.top_row, Some(&mut self.rows.bottom_row)), + _ => unreachable!("SinkWriter only supports one or two rows"), + }?; + self.sink.write_row(y, &self.rows.top_row)?; + if row_count == 2 { + self.sink.write_row(y + 1, &self.rows.bottom_row)?; + } + Ok(result) + } +} + +impl OutputWriter for SinkWriter<'_, S> +where + S: RowSink, +{ + fn write_rgb_row( + &mut self, + y: u32, + r_row: &[u8], + g_row: &[u8], + b_row: &[u8], + ) -> Result<(), JpegError> { + self.backend + .fill_rgb_row_from_rgb(r_row, g_row, b_row, &mut self.rows.top_row); + self.sink.write_row(y, &self.rows.top_row) + } + + fn write_ycbcr_row( + &mut self, + y: u32, + y_row: &[u8], + cb_row: &[u8], + cr_row: &[u8], + ) -> Result<(), JpegError> { + self.backend + .fill_rgb_row_from_ycbcr(y_row, cb_row, cr_row, &mut self.rows.top_row); + self.sink.write_row(y, &self.rows.top_row) + } + + fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError> { + self.backend + .fill_rgb_row_from_gray(gray_row, &mut self.rows.top_row); + self.sink.write_row(y, &self.rows.top_row) + } +} + +fn find_component_index(component_ids: &[u8], id: u8) -> Option { + component_ids + .iter() + .position(|&component_id| component_id == id) +} + +fn compute_decode_scratch_bytes( + (width, height): (u32, u32), + sampling: crate::info::SamplingFactors, + cap: usize, +) -> Result { + let max_h = u32::from(sampling.max_h); + let max_v = u32::from(sampling.max_v); + let mcu_width = 8u32 + .checked_mul(max_h) + .ok_or(JpegError::MemoryCapExceeded { + requested: usize::MAX, + cap, + })?; + let mcu_height = 8u32 + .checked_mul(max_v) + .ok_or(JpegError::MemoryCapExceeded { + requested: usize::MAX, + cap, + })?; + let mcus_per_row = width.div_ceil(mcu_width); + let _mcu_rows = height.div_ceil(mcu_height); + + let mut stripe_total = 0usize; + for (h, v) in sampling.iter() { + let cols = checked_usize_product(&[mcus_per_row as usize, usize::from(h), 8usize], cap)?; + let rows = checked_usize_product(&[usize::from(v), 8usize], cap)?; + let plane = cols.checked_mul(rows).ok_or(JpegError::MemoryCapExceeded { + requested: usize::MAX, + cap, + })?; + stripe_total = stripe_total + .checked_add(plane) + .ok_or(JpegError::MemoryCapExceeded { + requested: usize::MAX, + cap, + })?; + if stripe_total > cap { + return Err(JpegError::MemoryCapExceeded { + requested: stripe_total, + cap, + }); + } + } + + let stripe_buffers = checked_usize_product(&[stripe_total, 3], cap)?; + let row_scratch = checked_usize_product(&[width as usize, 7], cap)?; + let total = stripe_buffers + .checked_add(row_scratch) + .ok_or(JpegError::MemoryCapExceeded { + requested: usize::MAX, + cap, + })?; + if total > cap { + return Err(JpegError::MemoryCapExceeded { + requested: total, + cap, + }); + } + + Ok(total) +} + +fn checked_usize_product(factors: &[usize], cap: usize) -> Result { + let mut value = 1usize; + for factor in factors { + value = value + .checked_mul(*factor) + .ok_or(JpegError::MemoryCapExceeded { + requested: usize::MAX, + cap, + })?; + } + Ok(value) +} + +/// Checked size for a transient full-frame intermediate buffer, enforcing the +/// decode memory cap at the allocation site. +fn checked_scratch_len(factors: &[usize]) -> Result { + let cap = DEFAULT_MAX_DECODE_BYTES; + let len = checked_usize_product(factors, cap)?; + if len > cap { + return Err(JpegError::MemoryCapExceeded { + requested: len, + cap, + }); + } + Ok(len) +} + +fn output_cap_error(requested: usize) -> JpegError { + JpegError::MemoryCapExceeded { + requested, + cap: DEFAULT_MAX_DECODE_BYTES, + } +} + +#[inline] +fn checked_output_geometry( + width: u32, + height: u32, + bytes_per_pixel: usize, +) -> Result<(usize, usize), JpegError> { + #[cfg(target_pointer_width = "64")] + { + // SOF parsing caps JPEG dimensions at 65_500, so these products cannot + // overflow usize on 64-bit targets. Keep the hot path to one cap check. + let stride = width as usize * bytes_per_pixel; + let len = stride * height as usize; + if len > DEFAULT_MAX_DECODE_BYTES { + return Err(output_cap_error(len)); + } + Ok((stride, len)) + } + + #[cfg(not(target_pointer_width = "64"))] + { + let stride = checked_output_product(width as usize, bytes_per_pixel)?; + let len = checked_output_product(stride, height as usize)?; + Ok((stride, len)) + } +} + +#[cfg(not(target_pointer_width = "64"))] +#[inline] +fn checked_output_product(left: usize, right: usize) -> Result { + let len = left + .checked_mul(right) + .ok_or_else(|| output_cap_error(usize::MAX))?; + if len > DEFAULT_MAX_DECODE_BYTES { + return Err(output_cap_error(len)); + } + Ok(len) +} + +fn compute_lossless_scratch_bytes(info: &Info, cap: usize) -> Result { + // Only the sampled-color lossless paths materialize full-frame component + // planes; grayscale and 4:4:4 decode stream straight into caller buffers. + // Region/scaled lossless intermediates are capped at their allocation + // sites via checked_scratch_len. + if !matches!( + lossless_color_sampling(info), + Some(LosslessColorSampling::S422 | LosslessColorSampling::S420) + ) { + return Ok(0); + } + let width = info.dimensions.0 as usize; + let height = info.dimensions.1 as usize; + let bytes_per_sample: usize = if info.bit_depth > 8 { 2 } else { 1 }; + let chroma_width = width.div_ceil(usize::from(info.sampling.max_h)); + let chroma_height = height.div_ceil(usize::from(info.sampling.max_v)); + let luma = checked_usize_product(&[width, height, bytes_per_sample], cap)?; + let chroma = checked_usize_product(&[chroma_width, chroma_height, bytes_per_sample, 2], cap)?; + let total = luma + .checked_add(chroma) + .ok_or(JpegError::MemoryCapExceeded { + requested: usize::MAX, + cap, + })?; + if total > cap { + return Err(JpegError::MemoryCapExceeded { + requested: total, + cap, + }); + } + Ok(total) +} + +fn compute_extended12_planes_scratch_bytes( + components: &[PreparedComponentPlan], + (width, height): (u32, u32), + sampling: crate::info::SamplingFactors, + cap: usize, +) -> Result { + let mcu_cols = width.div_ceil(u32::from(sampling.max_h) * 8) as usize; + let mcu_rows = height.div_ceil(u32::from(sampling.max_v) * 8) as usize; + let mut total = 0usize; + for component in components { + let stride = checked_usize_product(&[mcu_cols, usize::from(component.h), 8], cap)?; + let rows = checked_usize_product(&[mcu_rows, usize::from(component.v), 8], cap)?; + let plane = checked_usize_product(&[stride, rows, core::mem::size_of::()], cap)?; + total = total + .checked_add(plane) + .ok_or(JpegError::MemoryCapExceeded { + requested: usize::MAX, + cap, + })?; + } + if total > cap { + return Err(JpegError::MemoryCapExceeded { + requested: total, + cap, + }); + } + Ok(total) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::error::Warning; + use crate::output::OutputWriter; + use alloc::vec; + use alloc::vec::Vec; + + fn minimal_baseline_jpeg() -> Vec { + let mut v = Vec::new(); + v.extend_from_slice(&[0xFF, 0xD8]); + v.extend_from_slice(&[0xFF, 0xDB, 0x00, 67, 0x00]); + v.extend(core::iter::repeat_n(1u8, 64)); + v.extend_from_slice(&[ + 0xFF, + 0xC0, + 0x00, + 17, + 8, + 0, + 16, + 0, + 16, + 3, + 1, + (2 << 4) | 2, + 0, + 2, + (1 << 4) | 1, + 0, + 3, + (1 << 4) | 1, + 0, + ]); + v.extend_from_slice(&[ + 0xFF, 0xC4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xAA, + ]); + v.extend_from_slice(&[ + 0xFF, 0xC4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xBB, + ]); + v.extend_from_slice(&[0xFF, 0xDA, 0x00, 12, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0]); + v.extend_from_slice(&[0x00, 0xFF, 0xD9]); + v + } + + fn baseline_jpeg_with_dimensions(width: u16, height: u16) -> Vec { + let mut bytes = minimal_baseline_jpeg(); + let sof = bytes + .windows(2) + .position(|w| w == [0xFF, 0xC0]) + .expect("SOF0 marker"); + bytes[sof + 5..sof + 7].copy_from_slice(&height.to_be_bytes()); + bytes[sof + 7..sof + 9].copy_from_slice(&width.to_be_bytes()); + bytes + } + + fn dc_only_420_jpeg(width: u16, height: u16) -> Vec { + let mut v = Vec::new(); + v.extend_from_slice(&[0xFF, 0xD8]); + v.extend_from_slice(&[0xFF, 0xDB, 0x00, 67, 0x00]); + v.extend(core::iter::repeat_n(1u8, 64)); + v.extend_from_slice(&[ + 0xFF, + 0xC0, + 0x00, + 17, + 8, + (height >> 8) as u8, + height as u8, + (width >> 8) as u8, + width as u8, + 3, + 1, + (2 << 4) | 2, + 0, + 2, + (1 << 4) | 1, + 0, + 3, + (1 << 4) | 1, + 0, + ]); + v.extend_from_slice(&[ + 0xFF, 0xC4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + v.extend_from_slice(&[ + 0xFF, 0xC4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + v.extend_from_slice(&[0xFF, 0xDA, 0x00, 12, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0]); + + let mcus_per_row = u32::from(width).div_ceil(16); + let mcu_rows = u32::from(height).div_ceil(16); + let entropy_bits = mcus_per_row * mcu_rows * 12; + let entropy_bytes = (entropy_bits as usize).div_ceil(8) + 8; + v.extend(core::iter::repeat_n(0u8, entropy_bytes)); + v.extend_from_slice(&[0xFF, 0xD9]); + v + } + + #[test] + fn decoder_new_succeeds_on_baseline_stream() { + let bytes = minimal_baseline_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + assert_eq!(dec.info().dimensions, (16, 16)); + } + + #[test] + fn owned_baseline_decode_with_huge_dimensions_errors_before_allocating() { + let bytes = baseline_jpeg_with_dimensions(65_500, 65_500); + let dec = Decoder::new(&bytes).expect("huge baseline header should parse"); + + let err = dec.decode(PixelFormat::Rgb8).unwrap_err(); + + assert!( + matches!(err, JpegError::MemoryCapExceeded { .. }), + "expected MemoryCapExceeded before owned output allocation, got {err:?}" + ); + } + + #[test] + fn owned_baseline_region_decode_with_huge_dimensions_errors_before_allocating() { + let bytes = baseline_jpeg_with_dimensions(65_500, 65_500); + let dec = Decoder::new(&bytes).expect("huge baseline header should parse"); + + let err = dec + .decode_region(PixelFormat::Rgb8, Rect::full(dec.info().dimensions)) + .unwrap_err(); + + assert!( + matches!(err, JpegError::MemoryCapExceeded { .. }), + "expected MemoryCapExceeded before region output allocation, got {err:?}" + ); + } + + fn minimal_lossless_jpeg( + width: u16, + height: u16, + precision: u8, + sampling_420: bool, + ) -> Vec { + let mut v = Vec::new(); + v.extend_from_slice(&[0xFF, 0xD8]); + let first_sampling = if sampling_420 { + (2 << 4) | 2 + } else { + (1 << 4) | 1 + }; + v.extend_from_slice(&[ + 0xFF, + 0xC3, + 0x00, + 17, + precision, + (height >> 8) as u8, + height as u8, + (width >> 8) as u8, + width as u8, + 3, + 1, + first_sampling, + 0, + 2, + (1 << 4) | 1, + 0, + 3, + (1 << 4) | 1, + 0, + ]); + v.extend_from_slice(&[ + 0xFF, 0xC4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x00, + ]); + // SOS: predictor 1 (ss=1), point transform 0. + v.extend_from_slice(&[0xFF, 0xDA, 0x00, 12, 3, 1, 0x00, 2, 0x00, 3, 0x00, 1, 0, 0]); + v.extend_from_slice(&[0x00, 0x00, 0xFF, 0xD9]); + v + } + + #[test] + fn lossless_sampled_plan_with_huge_dimensions_is_rejected_by_memory_cap() { + // 65500x65500 lossless 16-bit 4:2:0: the sampled-color decode path + // would allocate ~12.9 GB of full-frame planes. Plan build must refuse. + let bytes = minimal_lossless_jpeg(65_500, 65_500, 16, true); + match Decoder::new(&bytes) { + Err(JpegError::MemoryCapExceeded { .. }) => {} + other => panic!("expected MemoryCapExceeded at plan build, got {other:?}"), + } + } + + #[test] + fn lossless_sampled_plan_with_small_dimensions_still_parses() { + let bytes = minimal_lossless_jpeg(16, 16, 16, true); + let dec = Decoder::new(&bytes).expect("small lossless stream should parse"); + assert_eq!(dec.info().dimensions, (16, 16)); + assert!( + dec.plan.scratch_bytes > 0, + "sampled lossless plan must report scratch" + ); + } + + #[test] + fn extended12_plan_with_huge_dimensions_is_rejected_by_memory_cap() { + // 65500x65500 Extended12 4:2:0: the sequential 12-bit path allocates + // full-frame u16 planes (~13 GB). Plan build must refuse. + let mut bytes = minimal_baseline_jpeg(); + let p = bytes.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap(); + bytes[p + 1] = 0xC1; + bytes[p + 4] = 12; + bytes[p + 5] = (65_500u16 >> 8) as u8; + bytes[p + 6] = 65_500u16 as u8; + bytes[p + 7] = (65_500u16 >> 8) as u8; + bytes[p + 8] = 65_500u16 as u8; + match Decoder::new(&bytes) { + Err(JpegError::MemoryCapExceeded { .. }) => {} + other => panic!("expected MemoryCapExceeded at plan build, got {other:?}"), + } + } + + #[test] + fn lossless_gray16_region_scaled_with_huge_dimensions_errors_before_allocating() { + // Grayscale direct decode streams to the caller's buffer, so plan + // scratch stays 0 — but the region/scaled path materializes the full + // frame (~8.6 GB here) and must be stopped at the allocation site. + let mut bytes = minimal_lossless_jpeg(65_500, 65_500, 16, false); + let p = bytes.windows(2).position(|w| w == [0xFF, 0xC3]).unwrap(); + // Rewrite SOF to a single grayscale component. + bytes[p + 3] = 11; + bytes[p + 9] = 1; + bytes[p + 10] = 1; + bytes[p + 11] = (1 << 4) | 1; + bytes[p + 12] = 0; + bytes.splice(p + 13..p + 19, core::iter::empty()); + let p = bytes.windows(2).position(|w| w == [0xFF, 0xDA]).unwrap(); + bytes.splice( + p..p + 14, + [0xFF, 0xDA, 0x00, 8, 1, 1, 0x00, 1, 0, 0].iter().copied(), + ); + let dec = Decoder::new(&bytes).expect("huge lossless gray stream should parse"); + let roi = Rect { + x: 0, + y: 0, + w: 16, + h: 16, + }; + let mut out = vec![0u8; 16 * 16 * 2]; + let err = dec + .decode_region_scaled_into(&mut out, 16 * 2, PixelFormat::Gray16, roi, Downscale::Half) + .unwrap_err(); + assert!( + matches!(err, JpegError::MemoryCapExceeded { .. }), + "expected MemoryCapExceeded, got {err:?}" + ); + } + + #[test] + fn decode_into_rejects_unsupported_extended12_ycbcr_sampling_with_not_implemented() { + let mut bytes = minimal_baseline_jpeg(); + let p = bytes.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap(); + bytes[p + 1] = 0xC1; + bytes[p + 4] = 12; + bytes[p + 11] = (1 << 4) | 2; + let dec = Decoder::new(&bytes).expect("unsupported Extended12 YCbCr sampling should parse"); + let stride = dec.info().dimensions.0 as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut out = vec![0u8; stride * dec.info().dimensions.1 as usize]; + + let err = dec + .decode_into(&mut out, stride, PixelFormat::Rgb16) + .unwrap_err(); + + assert!(err.is_not_implemented()); + } + + #[test] + fn decoder_new_rejects_arithmetic_as_unsupported() { + let mut bytes = minimal_baseline_jpeg(); + let p = bytes.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap(); + bytes[p + 1] = 0xC9; + let err = Decoder::new(&bytes).unwrap_err(); + assert!(err.is_unsupported()); + } + + #[test] + fn decode_outcome_carries_rect_and_warnings() { + let outcome = DecodeOutcome { + decoded: Rect { + x: 0, + y: 0, + w: 16, + h: 16, + }, + warnings: vec![Warning::MissingEoi], + }; + assert_eq!(outcome.decoded.w, 16); + assert_eq!(outcome.warnings.len(), 1); + } + + #[test] + fn decode_into_rejects_undersized_buffer() { + let bytes = minimal_baseline_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let mut buf = vec![0u8; 4]; + let err = dec + .decode_into(&mut buf, 48, PixelFormat::Rgb8) + .unwrap_err(); + assert!(matches!(err, JpegError::OutputBufferTooSmall { .. })); + } + + #[test] + fn decode_into_rejects_invalid_stride() { + let bytes = minimal_baseline_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let mut buf = vec![0u8; 16 * 16 * 3]; + let err = dec + .decode_into(&mut buf, 10, PixelFormat::Rgb8) + .unwrap_err(); + assert!(matches!(err, JpegError::InvalidStride { .. })); + } + + #[test] + fn large_fast_420_region_decode_populates_cpu_entropy_checkpoints() { + let bytes = dc_only_420_jpeg(1024, 2048); + let dec = Decoder::new(&bytes).expect("decoder"); + assert!(dec.plan.matches_fast_tile_shape()); + + let roi = Rect { + x: 64, + y: 1536, + w: 64, + h: 64, + }; + let mut out = vec![0u8; roi.w as usize * roi.h as usize * 3]; + let mut pool = ScratchPool::new(); + dec.decode_region_into_with_scratch( + &mut pool, + &mut out, + roi.w as usize * 3, + PixelFormat::Rgb8, + roi, + ) + .expect("deep ROI decode"); + + let cache = dec + .cpu_entropy_checkpoints + .lock() + .expect("checkpoint cache mutex"); + assert!(cache + .checkpoints + .iter() + .any(|checkpoint| checkpoint.mcu_index >= CPU_ROI_CHECKPOINT_MIN_TARGET_MCUS)); + } + + #[derive(Default)] + struct GrayRows { + rows: Vec<(u32, Vec)>, + } + + impl OutputWriter for GrayRows { + fn write_rgb_row( + &mut self, + _y: u32, + _r_row: &[u8], + _g_row: &[u8], + _b_row: &[u8], + ) -> Result<(), JpegError> { + unreachable!("gray test writer should not receive rgb rows"); + } + + fn write_ycbcr_row( + &mut self, + _y: u32, + _y_row: &[u8], + _cb_row: &[u8], + _cr_row: &[u8], + ) -> Result<(), JpegError> { + unreachable!("gray test writer should not receive ycbcr rows"); + } + + fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError> { + self.rows.push((y, gray_row.to_vec())); + Ok(()) + } + } + + #[test] + fn cropped_writer_honors_source_window_origin() { + let inner = GrayRows::default(); + let rect = Rect { + x: 6, + y: 1, + w: 2, + h: 1, + }; + let mut writer = CroppedWriter::new(inner, rect, 4, 4); + + writer + .write_gray_row(1, &[10, 20, 30, 40]) + .expect("crop write must succeed"); + + assert_eq!(writer.inner.rows, vec![(0, vec![30, 40])]); + } +} diff --git a/crates/signinum-jpeg/src/encoder.rs b/crates/j2k-jpeg/src/encoder.rs similarity index 80% rename from crates/signinum-jpeg/src/encoder.rs rename to crates/j2k-jpeg/src/encoder.rs index c3d97049..52116417 100644 --- a/crates/signinum-jpeg/src/encoder.rs +++ b/crates/j2k-jpeg/src/encoder.rs @@ -24,25 +24,39 @@ use crate::profile::{duration_us_string, emit_jpeg_profile_row, jpeg_profile_sta use std::time::{Duration, Instant}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// JPEG encoder backend selector. pub enum JpegBackend { + /// Choose the best available backend for the platform. Auto, + /// Use the portable CPU encoder. Cpu, + /// Use a Metal encoder when called through the Metal integration. Metal, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// JPEG baseline chroma subsampling mode. pub enum JpegSubsampling { + /// Single-component grayscale. Gray, + /// Three-component YBR/RGB 4:4:4 sampling. Ybr444, + /// Three-component YBR/RGB 4:2:2 sampling. Ybr422, + /// Three-component YBR/RGB 4:2:0 sampling. Ybr420, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Options controlling baseline JPEG encoding. pub struct JpegEncodeOptions { + /// JPEG quality in the conventional 1..=100 range. pub quality: u8, + /// Output component sampling. pub subsampling: JpegSubsampling, + /// Optional restart interval in MCUs. pub restart_interval: Option, + /// Requested encoder backend. pub backend: JpegBackend, } @@ -58,51 +72,94 @@ impl Default for JpegEncodeOptions { } #[derive(Debug, Clone, Copy)] +/// Borrowed input samples for baseline JPEG encoding. pub enum JpegSamples<'a> { + /// Interleaved 8-bit grayscale samples. Gray8 { + /// Pixel data, one byte per pixel. data: &'a [u8], + /// Image width in pixels. width: u32, + /// Image height in pixels. height: u32, }, + /// Interleaved 8-bit RGB samples. Rgb8 { + /// Pixel data, three bytes per pixel. data: &'a [u8], + /// Image width in pixels. width: u32, + /// Image height in pixels. height: u32, }, } #[derive(Debug, Clone, PartialEq, Eq)] +/// Encoded baseline JPEG bytes and the backend that produced them. pub struct EncodedJpeg { + /// Complete JPEG codestream. pub data: Vec, + /// Backend used to encode the codestream. pub backend: JpegBackend, } #[derive(Debug, Error)] +/// Errors produced by baseline JPEG encoding. pub enum JpegEncodeError { #[error("JPEG encode requires nonzero dimensions")] + /// Width or height was zero. EmptyDimensions, #[error("JPEG baseline dimensions must fit in u16, got {width}x{height}")] - DimensionsTooLarge { width: u32, height: u32 }, + /// JPEG baseline SOF dimensions exceed the 16-bit marker fields. + DimensionsTooLarge { + /// Requested width in pixels. + width: u32, + /// Requested height in pixels. + height: u32, + }, #[error("JPEG sample buffer length mismatch: expected {expected}, got {actual}")] - SampleLength { expected: usize, actual: usize }, + /// Input sample buffer length does not match width, height, and format. + SampleLength { + /// Required byte count. + expected: usize, + /// Supplied byte count. + actual: usize, + }, #[error("JPEG subsampling {subsampling:?} is incompatible with {samples}")] + /// Requested subsampling is incompatible with the supplied sample format. IncompatibleSubsampling { + /// Requested output sampling. subsampling: JpegSubsampling, + /// Human-readable sample format name. samples: &'static str, }, #[error("JPEG restart interval must be nonzero when provided")] + /// Restart interval was explicitly set to zero. InvalidRestartInterval, - #[error("JPEG encode backend {backend:?} is unavailable in signinum-jpeg CPU crate")] - UnsupportedBackend { backend: JpegBackend }, + #[error("JPEG encode backend {backend:?} is unavailable in j2k-jpeg CPU crate")] + /// Requested backend is not available in this crate. + UnsupportedBackend { + /// Requested backend. + backend: JpegBackend, + }, #[error("JPEG encoded marker segment is too large: {name}")] - SegmentTooLarge { name: &'static str }, + /// A marker segment would exceed the JPEG 16-bit length field. + SegmentTooLarge { + /// Marker segment name. + name: &'static str, + }, #[error("JPEG entropy symbol has no Huffman code: {symbol}")] - MissingHuffmanCode { symbol: u8 }, + /// Encoder attempted to emit a symbol absent from the active Huffman table. + MissingHuffmanCode { + /// Missing entropy symbol. + symbol: u8, + }, #[error("JPEG encode failed: {0}")] + /// Internal encoder failure with diagnostic text. Internal(String), } -struct BitWriter { +pub(crate) struct BitWriter { bytes: Vec, current: u8, used: u8, @@ -118,7 +175,7 @@ struct JpegEncodeProfile { } impl BitWriter { - fn new() -> Self { + pub(crate) fn new() -> Self { Self { bytes: Vec::new(), current: 0, @@ -151,13 +208,7 @@ impl BitWriter { self.used = 0; } - fn push_restart_marker(&mut self, rst: u8) { - self.align_with_ones(); - self.bytes.push(0xFF); - self.bytes.push(0xD0 + (rst & 0x07)); - } - - fn into_bytes(mut self) -> Vec { + pub(crate) fn into_bytes(mut self) -> Vec { self.align_with_ones(); self.bytes } @@ -170,6 +221,7 @@ impl BitWriter { } } +/// Encode grayscale or RGB samples as a baseline JPEG codestream. pub fn encode_jpeg_baseline( samples: JpegSamples<'_>, options: JpegEncodeOptions, @@ -417,59 +469,56 @@ fn encode_entropy_serial( cosine: &[[f64; 8]; 8], restart_interval: Option, ) -> Result, JpegEncodeError> { - let mcu_width = u32::from(sampling.max_h) * 8; - let mcu_height = u32::from(sampling.max_v) * 8; - let mcus_per_row = width.div_ceil(mcu_width); - let mcu_rows = height.div_ceil(mcu_height); - let mut writer = BitWriter::new(); - let mut prev_dc = [0i32; 3]; - let mut mcus_since_restart = 0u16; - let mut rst = 0u8; - - for mcu_y in 0..mcu_rows { - for mcu_x in 0..mcus_per_row { - if let Some(interval) = restart_interval { - if mcus_since_restart == interval { - writer.push_restart_marker(rst); - rst = (rst + 1) & 7; - prev_dc = [0; 3]; - mcus_since_restart = 0; - } - } - for component in 0..sampling.components as usize { - let quant = if component == 0 { q_luma } else { q_chroma }; - let dc_table = if component == 0 { - dc_tables[0] - } else { - dc_tables[1] - }; - let ac_table = if component == 0 { - ac_tables[0] - } else { - ac_tables[1] - }; - for block_y in 0..sampling.v[component] { - for block_x in 0..sampling.h[component] { - let block = sample_block( - planes, width, height, sampling, component, mcu_x, mcu_y, block_x, - block_y, - ); - let coeffs = fdct_quantize(&block, quant, cosine); - encode_block( - &coeffs, - &mut prev_dc[component], - dc_table, - ac_table, - &mut writer, - )?; - } - } + let (mcus_per_row, total_mcus) = entropy_mcu_layout(width, height, sampling)?; + if total_mcus == 0 { + return Ok(Vec::new()); + } + if let Some(restart_interval) = restart_interval { + if restart_interval == 0 { + return Err(JpegEncodeError::InvalidRestartInterval); + } + let restart_interval = u32::from(restart_interval); + let mut out = Vec::new(); + let mut rst = 0u8; + for start_mcu in (0..total_mcus).step_by(restart_interval as usize) { + if start_mcu > 0 { + out.push(0xFF); + out.push(0xD0 + rst); + rst = (rst + 1) & 7; } - mcus_since_restart = mcus_since_restart.saturating_add(1); + let end_mcu = start_mcu.saturating_add(restart_interval).min(total_mcus); + out.extend_from_slice(&encode_entropy_mcu_range( + planes, + width, + height, + sampling, + q_luma, + q_chroma, + dc_tables, + ac_tables, + cosine, + mcus_per_row, + start_mcu, + end_mcu, + )?); } + return Ok(out); } - Ok(writer.into_bytes()) + encode_entropy_mcu_range( + planes, + width, + height, + sampling, + q_luma, + q_chroma, + dc_tables, + ac_tables, + cosine, + mcus_per_row, + 0, + total_mcus, + ) } #[allow(clippy::too_many_arguments)] @@ -488,13 +537,7 @@ fn encode_entropy_restart_segments( if restart_interval == 0 { return Err(JpegEncodeError::InvalidRestartInterval); } - let mcu_width = u32::from(sampling.max_h) * 8; - let mcu_height = u32::from(sampling.max_v) * 8; - let mcus_per_row = width.div_ceil(mcu_width); - let mcu_rows = height.div_ceil(mcu_height); - let total_mcus = mcus_per_row - .checked_mul(mcu_rows) - .ok_or_else(|| JpegEncodeError::Internal("JPEG MCU count overflow".into()))?; + let (mcus_per_row, total_mcus) = entropy_mcu_layout(width, height, sampling)?; if total_mcus == 0 { return Ok(Vec::new()); } @@ -553,7 +596,7 @@ fn encode_entropy_mcu_range( for mcu_index in start_mcu..end_mcu { let mcu_y = mcu_index / mcus_per_row; let mcu_x = mcu_index % mcus_per_row; - for component in 0..sampling.components as usize { + for_each_mcu_block(sampling, |component, block_x, block_y| { let quant = if component == 0 { q_luma } else { q_chroma }; let dc_table = if component == 0 { dc_tables[0] @@ -565,24 +608,52 @@ fn encode_entropy_mcu_range( } else { ac_tables[1] }; - for block_y in 0..sampling.v[component] { - for block_x in 0..sampling.h[component] { - let block = sample_block( - planes, width, height, sampling, component, mcu_x, mcu_y, block_x, block_y, - ); - let coeffs = fdct_quantize(&block, quant, cosine); - encode_block( - &coeffs, - &mut prev_dc[component], - dc_table, - ac_table, - &mut writer, - )?; - } + let block = sample_block( + planes, width, height, sampling, component, mcu_x, mcu_y, block_x, block_y, + ); + let coeffs = fdct_quantize(&block, quant, cosine); + encode_block( + &coeffs, + &mut prev_dc[component], + dc_table, + ac_table, + &mut writer, + ) + })?; + } + Ok(writer.into_bytes()) +} + +fn entropy_mcu_layout( + width: u32, + height: u32, + sampling: JpegBaselineSampling, +) -> Result<(u32, u32), JpegEncodeError> { + let mcu_width = u32::from(sampling.max_h) * 8; + let mcu_height = u32::from(sampling.max_v) * 8; + let mcus_per_row = width.div_ceil(mcu_width); + let mcu_rows = height.div_ceil(mcu_height); + let total_mcus = mcus_per_row + .checked_mul(mcu_rows) + .ok_or_else(|| JpegEncodeError::Internal("JPEG MCU count overflow".into()))?; + Ok((mcus_per_row, total_mcus)) +} + +fn for_each_mcu_block( + sampling: JpegBaselineSampling, + mut visit: F, +) -> Result<(), JpegEncodeError> +where + F: FnMut(usize, u8, u8) -> Result<(), JpegEncodeError>, +{ + for component in 0..sampling.components as usize { + for block_y in 0..sampling.v[component] { + for block_x in 0..sampling.h[component] { + visit(component, block_x, block_y)?; } } } - Ok(writer.into_bytes()) + Ok(()) } #[allow(clippy::too_many_arguments)] @@ -662,7 +733,7 @@ fn fdct_quantize(block: &[u8; 64], quant: &[u8; 64], cosine: &[[f64; 8]; 8]) -> coeffs } -fn encode_block( +pub(crate) fn encode_block( coeffs: &[i32; 64], prev_dc: &mut i32, dc_table: &JpegBaselineHuffmanTable, diff --git a/crates/signinum-jpeg/src/entropy/block.rs b/crates/j2k-jpeg/src/entropy/block.rs similarity index 81% rename from crates/signinum-jpeg/src/entropy/block.rs rename to crates/j2k-jpeg/src/entropy/block.rs index 57a46ed0..720db2fd 100644 --- a/crates/signinum-jpeg/src/entropy/block.rs +++ b/crates/j2k-jpeg/src/entropy/block.rs @@ -17,7 +17,10 @@ //! Produces a 64-entry array in row-major (natural) order, suitable for //! direct consumption by the IDCT. -use crate::entropy::huffman::{AcDecoded, AcSkipDecoded, HuffmanTable}; +use crate::entropy::huffman::{ + ac_decoded_run, ac_decoded_value, HuffmanTable, AC_FAST_EOB, AC_FAST_KIND_MASK, AC_FAST_VALUE, + AC_FAST_ZRL, +}; use crate::entropy::ZIGZAG; use crate::error::{HuffmanFailure, JpegError}; use crate::internal::bit_reader::BitReader; @@ -163,34 +166,79 @@ pub(crate) fn decode_block_with_activity( let dc_dequant = (*prev_dc).wrapping_mul(quant[0] as i32); block.store(0, clamp_i16(dc_dequant)); - // AC. - let mut k: usize = 1; let mut activity = BlockActivity::DcOnly; - while k < 64 { - match ac_table.decode_fast_ac(br)? { - AcDecoded::Eob => break, - AcDecoded::Zrl => { - // ZRL — 16 zeros, continue. - k += 16; - } - AcDecoded::Value { run, value } => { - k += run; - if k >= 64 { - return Err(JpegError::HuffmanDecode { - mcu: 0, - reason: HuffmanFailure::InvalidSymbol, - }); - } - let natural_idx = ZIGZAG[k] as usize; - // Quant table entries are stored in zigzag order per T.81 §B.2.4.1, - // so `quant[k]` is the matching coefficient (not `quant[natural_idx]`). - let dequant = value.wrapping_mul(quant[k] as i32); - block.store(natural_idx, clamp_i16(dequant)); - activity = extend_activity(activity, natural_idx); - k += 1; - } - } - } + drive_ac_fast::(br, ac_table, |k, ac| { + let natural_idx = ZIGZAG[k] as usize; + // Quant table entries are stored in zigzag order per T.81 §B.2.4.1, + // so `quant[k]` is the matching coefficient (not `quant[natural_idx]`). + let value = ac_decoded_value(ac); + let dequant = value.wrapping_mul(quant[k] as i32); + block.store(natural_idx, clamp_i16(dequant)); + activity = extend_activity(activity, natural_idx); + Ok(()) + })?; + Ok(activity) +} + +/// Decode one dequantized block directly into a freshly zeroed output block. +/// +/// The caller must pass an output block that is all zeroes; this function only +/// writes non-zero decoded coefficients. +#[inline(always)] +pub(crate) fn decode_block_dequantized_into( + br: &mut BitReader<'_>, + dc_table: &HuffmanTable, + ac_table: &HuffmanTable, + prev_dc: &mut i32, + quant: &[u16; 64], + out: &mut [i16; 64], +) -> Result<(), JpegError> { + // DC. + let diff = dc_table.decode_fast_dc(br)?; + *prev_dc = prev_dc.wrapping_add(diff); + let dc_dequant = (*prev_dc).wrapping_mul(quant[0] as i32); + out[0] = clamp_i16(dc_dequant); + + drive_ac_fast::(br, ac_table, |k, ac| { + let natural_idx = ZIGZAG[k] as usize; + let value = ac_decoded_value(ac); + let dequant = value.wrapping_mul(quant[k] as i32); + out[natural_idx] = clamp_i16(dequant); + Ok(()) + })?; + Ok(()) +} + +#[inline(always)] +pub(crate) fn decode_block_quantized_and_dequantized_with_activity( + br: &mut BitReader<'_>, + dc_table: &HuffmanTable, + ac_table: &HuffmanTable, + prev_dc: &mut i32, + quant: &[u16; 64], + quantized_block: &mut CoefficientBlock, + dequantized_block: &mut CoefficientBlock, +) -> Result { + quantized_block.clear_touched(); + dequantized_block.clear_touched(); + + // DC. + let diff = dc_table.decode_fast_dc(br)?; + *prev_dc = prev_dc.wrapping_add(diff); + quantized_block.store(0, clamp_i16(*prev_dc)); + let dc_dequant = (*prev_dc).wrapping_mul(quant[0] as i32); + dequantized_block.store(0, clamp_i16(dc_dequant)); + + let mut activity = BlockActivity::DcOnly; + drive_ac_fast::(br, ac_table, |k, ac| { + let natural_idx = ZIGZAG[k] as usize; + let value = ac_decoded_value(ac); + quantized_block.store(natural_idx, clamp_i16(value)); + let dequant = value.wrapping_mul(quant[k] as i32); + dequantized_block.store(natural_idx, clamp_i16(dequant)); + activity = extend_activity(activity, natural_idx); + Ok(()) + })?; Ok(activity) } @@ -211,30 +259,15 @@ pub(crate) fn decode_block_with_dc_status( let dc_dequant = (*prev_dc).wrapping_mul(quant[0] as i32); block.store(0, clamp_i16(dc_dequant)); - let mut k: usize = 1; let mut dc_only = true; - while k < 64 { - match ac_table.decode_fast_ac(br)? { - AcDecoded::Eob => break, - AcDecoded::Zrl => { - k += 16; - } - AcDecoded::Value { run, value } => { - k += run; - if k >= 64 { - return Err(JpegError::HuffmanDecode { - mcu: 0, - reason: HuffmanFailure::InvalidSymbol, - }); - } - let natural_idx = ZIGZAG[k] as usize; - let dequant = value.wrapping_mul(quant[k] as i32); - block.store(natural_idx, clamp_i16(dequant)); - dc_only = false; - k += 1; - } - } - } + drive_ac_fast::(br, ac_table, |k, ac| { + let natural_idx = ZIGZAG[k] as usize; + let value = ac_decoded_value(ac); + let dequant = value.wrapping_mul(quant[k] as i32); + block.store(natural_idx, clamp_i16(dequant)); + dc_only = false; + Ok(()) + })?; Ok(dc_only) } @@ -255,32 +288,17 @@ pub(crate) fn decode_block_for_reduced_idct( let dc_dequant = (*prev_dc).wrapping_mul(quant[0] as i32); block.store(0, clamp_i16(dc_dequant)); - let mut k: usize = 1; let mut dc_only_for_reduced_idct = true; - while k < 64 { - match ac_table.decode_fast_ac(br)? { - AcDecoded::Eob => break, - AcDecoded::Zrl => { - k += 16; - } - AcDecoded::Value { run, value } => { - k += run; - if k >= 64 { - return Err(JpegError::HuffmanDecode { - mcu: 0, - reason: HuffmanFailure::InvalidSymbol, - }); - } - let natural_idx = ZIGZAG[k] as usize; - if keep.keeps(natural_idx) { - let dequant = value.wrapping_mul(quant[k] as i32); - block.store(natural_idx, clamp_i16(dequant)); - dc_only_for_reduced_idct = false; - } - k += 1; - } + drive_ac_fast::(br, ac_table, |k, ac| { + let natural_idx = ZIGZAG[k] as usize; + if keep.keeps(natural_idx) { + let value = ac_decoded_value(ac); + let dequant = value.wrapping_mul(quant[k] as i32); + block.store(natural_idx, clamp_i16(dequant)); + dc_only_for_reduced_idct = false; } - } + Ok(()) + })?; Ok(dc_only_for_reduced_idct) } @@ -300,25 +318,7 @@ pub(crate) fn decode_block_for_1x1_idct( let dc_dequant = (*prev_dc).wrapping_mul(quant[0] as i32); block.store(0, clamp_i16(dc_dequant)); - let mut k: usize = 1; - while k < 64 { - match ac_table.skip_fast_ac(br)? { - AcSkipDecoded::Eob => break, - AcSkipDecoded::Zrl => { - k += 16; - } - AcSkipDecoded::Value { run } => { - k += run; - if k >= 64 { - return Err(JpegError::HuffmanDecode { - mcu: 0, - reason: HuffmanFailure::InvalidSymbol, - }); - } - k += 1; - } - } - } + drive_ac_fast::(br, ac_table, |_, _| Ok(()))?; Ok(()) } @@ -332,30 +332,59 @@ pub(crate) fn skip_block( let diff = dc_table.decode_fast_dc(br)?; *prev_dc = prev_dc.wrapping_add(diff); + drive_ac_fast::(br, ac_table, |_, _| Ok(()))?; + Ok(()) +} + +#[inline(always)] +fn drive_ac_fast( + br: &mut BitReader<'_>, + ac_table: &HuffmanTable, + mut on_value: F, +) -> Result<(), JpegError> +where + F: FnMut(usize, u32) -> Result<(), JpegError>, +{ let mut k: usize = 1; while k < 64 { - match ac_table.skip_fast_ac(br)? { - AcSkipDecoded::Eob => break, - AcSkipDecoded::Zrl => { - k += 16; - } - AcSkipDecoded::Value { run } => { - k += run; + let ac = if SKIP_VALUES { + ac_table.skip_fast_ac(br)? + } else { + ac_table.decode_fast_ac(br)? + }; + match ac & AC_FAST_KIND_MASK { + AC_FAST_EOB => break, + AC_FAST_ZRL => k += 16, + AC_FAST_VALUE => { + k += ac_decoded_run(ac); if k >= 64 { - return Err(JpegError::HuffmanDecode { - mcu: 0, - reason: HuffmanFailure::InvalidSymbol, - }); + return Err(invalid_huffman_symbol()); } + on_value(k, ac)?; k += 1; } + _ => unreachable!("invalid AC fast-table tag"), } } Ok(()) } +#[inline(always)] +fn invalid_huffman_symbol() -> JpegError { + JpegError::HuffmanDecode { + mcu: 0, + reason: HuffmanFailure::InvalidSymbol, + } +} + fn clamp_i16(v: i32) -> i16 { - v.clamp(i16::MIN as i32, i16::MAX as i32) as i16 + if v > i16::MAX as i32 { + i16::MAX + } else if v < i16::MIN as i32 { + i16::MIN + } else { + v as i16 + } } #[cfg(test)] diff --git a/crates/signinum-jpeg/src/entropy/huffman.rs b/crates/j2k-jpeg/src/entropy/huffman.rs similarity index 90% rename from crates/signinum-jpeg/src/entropy/huffman.rs rename to crates/j2k-jpeg/src/entropy/huffman.rs index 3519e82d..3a04f352 100644 --- a/crates/signinum-jpeg/src/entropy/huffman.rs +++ b/crates/j2k-jpeg/src/entropy/huffman.rs @@ -22,25 +22,11 @@ use alloc::boxed::Box; const FAST_BITS: u8 = 12; const FAST_ENTRIES: usize = 1 << FAST_BITS; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum AcDecoded { - Eob, - Zrl, - Value { run: usize, value: i32 }, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum AcSkipDecoded { - Eob, - Zrl, - Value { run: usize }, -} - const AC_FAST_KIND_SHIFT: u32 = 28; -const AC_FAST_KIND_MASK: u32 = 0xF << AC_FAST_KIND_SHIFT; -const AC_FAST_VALUE: u32 = 1 << AC_FAST_KIND_SHIFT; -const AC_FAST_EOB: u32 = 2 << AC_FAST_KIND_SHIFT; -const AC_FAST_ZRL: u32 = 3 << AC_FAST_KIND_SHIFT; +pub(crate) const AC_FAST_KIND_MASK: u32 = 0xF << AC_FAST_KIND_SHIFT; +pub(crate) const AC_FAST_VALUE: u32 = 1 << AC_FAST_KIND_SHIFT; +pub(crate) const AC_FAST_EOB: u32 = 2 << AC_FAST_KIND_SHIFT; +pub(crate) const AC_FAST_ZRL: u32 = 3 << AC_FAST_KIND_SHIFT; const AC_FAST_LEN_MASK: u32 = 0x0F; const AC_FAST_RUN_MASK: u32 = 0xF0; const AC_FAST_VALUE_SHIFT: u32 = 8; @@ -268,21 +254,13 @@ impl HuffmanTable { } #[inline(always)] - pub(crate) fn decode_fast_ac(&self, br: &mut BitReader<'_>) -> Result { + pub(crate) fn decode_fast_ac(&self, br: &mut BitReader<'_>) -> Result { br.ensure_bits_padded(FAST_BITS)?; let peek = br.peek_bits(FAST_BITS) as usize; let packed = self.fast_ac[peek]; if packed != 0 { br.consume_bits((packed & AC_FAST_LEN_MASK) as u8); - return Ok(match packed & AC_FAST_KIND_MASK { - AC_FAST_VALUE => AcDecoded::Value { - run: ((packed & AC_FAST_RUN_MASK) >> 4) as usize, - value: i32::from(((packed >> AC_FAST_VALUE_SHIFT) & 0xFFFF) as u16 as i16), - }, - AC_FAST_EOB => AcDecoded::Eob, - AC_FAST_ZRL => AcDecoded::Zrl, - _ => unreachable!("invalid AC fast-table tag"), - }); + return Ok(packed); } let (sym, len) = self.fast[peek]; @@ -293,35 +271,28 @@ impl HuffmanTable { self.decode(br)? }; - let run = (sym >> 4) as usize; + let run = sym >> 4; let ssss = sym & 0x0F; if ssss == 0 { return Ok(if run == 15 { - AcDecoded::Zrl + pack_ac_zrl(0) } else { - AcDecoded::Eob + pack_ac_eob(0) }); } let value = br.receive_extend(ssss)?; - Ok(AcDecoded::Value { run, value }) + Ok(pack_ac_value(0, run, value as i16)) } #[inline(always)] - pub(crate) fn skip_fast_ac(&self, br: &mut BitReader<'_>) -> Result { + pub(crate) fn skip_fast_ac(&self, br: &mut BitReader<'_>) -> Result { br.ensure_bits_padded(FAST_BITS)?; let peek = br.peek_bits(FAST_BITS) as usize; let packed = self.fast_ac[peek]; if packed != 0 { br.consume_bits((packed & AC_FAST_LEN_MASK) as u8); - return Ok(match packed & AC_FAST_KIND_MASK { - AC_FAST_VALUE => AcSkipDecoded::Value { - run: ((packed & AC_FAST_RUN_MASK) >> 4) as usize, - }, - AC_FAST_EOB => AcSkipDecoded::Eob, - AC_FAST_ZRL => AcSkipDecoded::Zrl, - _ => unreachable!("invalid AC fast-table tag"), - }); + return Ok(packed); } let (sym, len) = self.fast[peek]; @@ -332,22 +303,32 @@ impl HuffmanTable { self.decode(br)? }; - let run = (sym >> 4) as usize; + let run = sym >> 4; let ssss = sym & 0x0F; if ssss == 0 { return Ok(if run == 15 { - AcSkipDecoded::Zrl + pack_ac_zrl(0) } else { - AcSkipDecoded::Eob + pack_ac_eob(0) }); } br.ensure_bits(ssss)?; br.consume_bits(ssss); - Ok(AcSkipDecoded::Value { run }) + Ok(pack_ac_value(0, run, 0)) } } +#[inline(always)] +pub(crate) fn ac_decoded_run(packed: u32) -> usize { + ((packed & AC_FAST_RUN_MASK) >> 4) as usize +} + +#[inline(always)] +pub(crate) fn ac_decoded_value(packed: u32) -> i32 { + i32::from(((packed >> AC_FAST_VALUE_SHIFT) & 0xFFFF) as u16 as i16) +} + #[inline] fn pack_ac_value(total_len: u8, run: u8, value: i16) -> u32 { AC_FAST_VALUE @@ -407,7 +388,7 @@ mod tests { #[test] fn widened_fast_table_covers_9_bit_luma_dc_code() { let table = HuffmanTable::from_raw(&luma_dc_raw()).unwrap(); - let idx = ((0b1_1111_1110u32) << 3) as usize; + let idx = 0b1_1111_1110usize << usize::from(FAST_BITS - 9); let (sym, len) = table.fast.get(idx).copied().unwrap_or((0, 0)); assert_eq!((sym, len), (11, 9)); } diff --git a/crates/signinum-jpeg/src/entropy/mod.rs b/crates/j2k-jpeg/src/entropy/mod.rs similarity index 100% rename from crates/signinum-jpeg/src/entropy/mod.rs rename to crates/j2k-jpeg/src/entropy/mod.rs diff --git a/crates/signinum-jpeg/src/entropy/progressive.rs b/crates/j2k-jpeg/src/entropy/progressive.rs similarity index 97% rename from crates/signinum-jpeg/src/entropy/progressive.rs rename to crates/j2k-jpeg/src/entropy/progressive.rs index ed75ece3..17c45b6e 100644 --- a/crates/signinum-jpeg/src/entropy/progressive.rs +++ b/crates/j2k-jpeg/src/entropy/progressive.rs @@ -55,6 +55,11 @@ pub(crate) struct PreparedProgressivePlan { pub(crate) scratch_bytes: usize, } +#[derive(Debug, Clone)] +pub(crate) struct ProgressiveDctBlocks { + pub(crate) quantized: Vec>, +} + #[derive(Debug)] struct ComponentImage { plane: Vec, @@ -67,13 +72,22 @@ pub(crate) fn decode_progressive( bytes: &[u8], writer: &mut W, ) -> Result, JpegError> { + let dct_blocks = decode_progressive_dct_blocks(plan, bytes)?; + let coeffs = dct_blocks.quantized; + let images = render_component_images(plan, backend, &coeffs); + emit_component_images(plan, &images, writer)?; + Ok(Vec::new()) +} + +pub(crate) fn decode_progressive_dct_blocks( + plan: &PreparedProgressivePlan, + bytes: &[u8], +) -> Result { let mut coeffs = allocate_coefficients(plan)?; for scan in &plan.scans { decode_progressive_scan(plan, scan, bytes, &mut coeffs)?; } - let images = render_component_images(plan, backend, &coeffs); - emit_component_images(plan, &images, writer)?; - Ok(Vec::new()) + Ok(ProgressiveDctBlocks { quantized: coeffs }) } fn allocate_coefficients(plan: &PreparedProgressivePlan) -> Result>, JpegError> { diff --git a/crates/signinum-jpeg/src/entropy/sequential.rs b/crates/j2k-jpeg/src/entropy/sequential.rs similarity index 89% rename from crates/signinum-jpeg/src/entropy/sequential.rs rename to crates/j2k-jpeg/src/entropy/sequential.rs index 5a8774aa..74366fca 100644 --- a/crates/signinum-jpeg/src/entropy/sequential.rs +++ b/crates/j2k-jpeg/src/entropy/sequential.rs @@ -5,16 +5,19 @@ //! color conversion. use crate::backend::Backend; +#[cfg(feature = "bench-internals")] use crate::bench_support::{BenchBlockActivityCounts, BenchFast420Profile}; +use crate::color::cmyk::{inverted_cmyk_to_rgb, ycck_to_rgb}; use crate::color::upsample::{ upsample_1x1, upsample_h2v1_fancy_row, upsample_h2v2_fancy_row, upsample_h2v2_fancy_rows, }; use crate::entropy::block::{ - decode_block_for_1x1_idct, decode_block_for_reduced_idct, decode_block_with_activity, - skip_block, BlockActivity, CoefficientBlock, ReducedIdctCoefficients, + decode_block_dequantized_into, decode_block_for_1x1_idct, decode_block_for_reduced_idct, + decode_block_quantized_and_dequantized_with_activity, decode_block_with_activity, skip_block, + BlockActivity, CoefficientBlock, ReducedIdctCoefficients, }; use crate::entropy::huffman::HuffmanTable; -use crate::error::{HuffmanFailure, JpegError, Warning}; +use crate::error::{JpegError, Warning}; use crate::idct::downscale; use crate::info::{ColorSpace, DownscaleFactor, Rect, SamplingFactors}; use crate::internal::bit_reader::{BitReader, BitReaderSnapshot}; @@ -26,8 +29,32 @@ use crate::output::{InterleavedRgbWriter, OutputWriter}; use alloc::sync::Arc; use alloc::vec::Vec; use core::ptr; +#[cfg(feature = "bench-internals")] use std::time::Instant; +trait Fast420Profiler { + fn record_activity(&mut self, activity: BlockActivity); +} + +struct NoopFast420Profiler; + +impl Fast420Profiler for NoopFast420Profiler { + #[inline] + fn record_activity(&mut self, _activity: BlockActivity) {} +} + +#[cfg(feature = "bench-internals")] +impl Fast420Profiler for BenchBlockActivityCounts { + #[inline] + fn record_activity(&mut self, activity: BlockActivity) { + match activity { + BlockActivity::DcOnly => self.record_dc_only(), + BlockActivity::BottomHalfZero => self.record_bottom_half_zero(), + BlockActivity::General => self.record_general(), + } + } +} + /// Per-component decode context. One entry per component declared in the /// SOF, in scan order. #[derive(Debug, Clone)] @@ -209,7 +236,7 @@ pub(crate) fn decode_scan_baseline( ColorSpace::YCbCr if is_ycbcr_420(plan) => OutputScratch::YCbCr420(ycbcr_420_rows), ColorSpace::YCbCr => OutputScratch::YCbCrGeneric(ycbcr_generic_rows), ColorSpace::Rgb => OutputScratch::RgbGeneric(rgb_generic_rows), - ColorSpace::Cmyk | ColorSpace::Ycck => OutputScratch::Grayscale, + ColorSpace::Cmyk | ColorSpace::Ycck => OutputScratch::RgbGeneric(rgb_generic_rows), }; let mut prev_stripe: &mut StripeBuffer = stripe_a; let mut curr_stripe: &mut StripeBuffer = stripe_b; @@ -364,7 +391,7 @@ pub(crate) fn decode_scan_baseline_rgb( ColorSpace::YCbCr if is_ycbcr_420(plan) => RgbOutputScratch::YCbCr420, ColorSpace::YCbCr => RgbOutputScratch::YCbCrGeneric(ycbcr_generic_rows), ColorSpace::Rgb => RgbOutputScratch::RgbGeneric(rgb_generic_rows), - ColorSpace::Cmyk | ColorSpace::Ycck => RgbOutputScratch::None, + ColorSpace::Cmyk | ColorSpace::Ycck => RgbOutputScratch::RgbGeneric(rgb_generic_rows), }; let mut prev_stripe: &mut StripeBuffer = stripe_a; let mut curr_stripe: &mut StripeBuffer = stripe_b; @@ -535,6 +562,7 @@ pub(crate) fn decode_scan_fast_tile_rgb( let mut curr_stripe: &mut StripeBuffer = stripe_b; let mut next_stripe: &mut StripeBuffer = stripe_c; let mut output_scratch = RgbOutputScratch::YCbCr420; + let mut profiler = NoopFast420Profiler; decode_mcu_row_fast_tile_420( y_comp, @@ -551,6 +579,7 @@ pub(crate) fn decode_scan_fast_tile_rgb( 0, mcus_per_row, curr_stripe, + &mut profiler, )?; let mut has_prev = false; @@ -570,6 +599,7 @@ pub(crate) fn decode_scan_fast_tile_rgb( 0, mcus_per_row, next_stripe, + &mut profiler, )?; emit_stripe_rgb( plan, @@ -604,6 +634,7 @@ pub(crate) fn decode_scan_fast_tile_rgb( finish_fast_tile_scan(&mut br) } +#[cfg(feature = "bench-internals")] pub(crate) fn decode_scan_fast_tile_rgb_profiled( plan: &PreparedDecodePlan, backend: Backend, @@ -648,7 +679,7 @@ pub(crate) fn decode_scan_fast_tile_rgb_profiled y0 < rect_y1 && y1 > rect.y } -fn finish_scan(br: &mut BitReader<'_>, validate_eoi: bool) -> Result, JpegError> { +pub(crate) fn finish_scan( + br: &mut BitReader<'_>, + validate_eoi: bool, +) -> Result, JpegError> { if !validate_eoi { return Ok(Vec::new()); } @@ -1003,6 +1037,130 @@ fn skip_mcu( Ok(()) } +pub(crate) fn decode_scan_dct_blocks( + plan: &PreparedDecodePlan, + scan_bytes: &[u8], + retain_quantized_blocks: bool, +) -> Result { + let max_h = u32::from(plan.sampling.max_h); + let max_v = u32::from(plan.sampling.max_v); + let mcu_width_px = 8 * max_h; + let mcu_height_px = 8 * max_v; + let mcus_per_row = plan.dimensions.0.div_ceil(mcu_width_px); + let mcu_rows = plan.dimensions.1.div_ceil(mcu_height_px); + let component_count = plan.sampling.len(); + + let mut block_cols_by_component = vec![0_u32; component_count]; + let mut block_rows_by_component = vec![0_u32; component_count]; + for component in &plan.components { + block_cols_by_component[component.output_index] = mcus_per_row * u32::from(component.h); + block_rows_by_component[component.output_index] = mcu_rows * u32::from(component.v); + } + + let mut quantized_blocks = block_cols_by_component + .iter() + .zip(block_rows_by_component.iter()) + .map(|(&cols, &rows)| { + if retain_quantized_blocks { + vec![[0_i16; 64]; (cols * rows) as usize] + } else { + Vec::new() + } + }) + .collect::>(); + let mut dequantized_blocks = block_cols_by_component + .iter() + .zip(block_rows_by_component.iter()) + .map(|(&cols, &rows)| vec![[0_i16; 64]; (cols * rows) as usize]) + .collect::>(); + + let mut br = BitReader::new(scan_bytes); + let mut prev_dc = vec![0_i32; component_count]; + let mut quantized_coeff = CoefficientBlock::default(); + let mut dequantized_coeff = CoefficientBlock::default(); + let restart = plan.restart_interval.unwrap_or(0); + let mut mcus_since_restart = 0_u32; + let mut expected_rst = 0_u8; + let total_mcus = mcu_rows * mcus_per_row; + + for mcu_y in 0..mcu_rows { + for mcu_x in 0..mcus_per_row { + let current_mcu = mcu_y * mcus_per_row + mcu_x; + if restart > 0 && mcus_since_restart == u32::from(restart) { + let _ = br.ensure_bits(1); + let marker = br.take_marker().ok_or(JpegError::UnexpectedEoi { + mcu_at: current_mcu, + mcu_total: total_mcus, + })?; + let expected = 0xD0 | expected_rst; + if marker != expected { + return Err(JpegError::RestartMismatch { + offset: br.position(), + expected: expected_rst, + found: marker, + }); + } + expected_rst = (expected_rst + 1) & 0x07; + br.reset_at_restart(); + prev_dc.fill(0); + mcus_since_restart = 0; + } + + for component in &plan.components { + let plane_idx = component.output_index; + let block_cols = block_cols_by_component[plane_idx]; + for vy in 0..u32::from(component.v) { + for vx in 0..u32::from(component.h) { + let block_x = mcu_x * u32::from(component.h) + vx; + let block_y = mcu_y * u32::from(component.v) + vy; + let block_idx = (block_y * block_cols + block_x) as usize; + if retain_quantized_blocks { + decode_block_quantized_and_dequantized_with_activity( + &mut br, + &component.dc_table, + &component.ac_table, + &mut prev_dc[plane_idx], + &component.quant, + &mut quantized_coeff, + &mut dequantized_coeff, + )?; + quantized_blocks[plane_idx][block_idx] = + *quantized_coeff.coefficients(); + } else { + decode_block_dequantized_into( + &mut br, + &component.dc_table, + &component.ac_table, + &mut prev_dc[plane_idx], + &component.quant, + &mut dequantized_blocks[plane_idx][block_idx], + )?; + } + if retain_quantized_blocks { + dequantized_blocks[plane_idx][block_idx] = + *dequantized_coeff.coefficients(); + } + } + } + } + + mcus_since_restart += 1; + } + } + + finish_scan(&mut br, true)?; + Ok(DecodedDctBlocks { + quantized: quantized_blocks, + dequantized: dequantized_blocks, + }) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct DecodedDctBlocks { + pub(crate) quantized: Vec>, + pub(crate) dequantized: Vec>, +} + #[allow(clippy::too_many_arguments)] fn skip_to_mcu( plan: &PreparedDecodePlan, @@ -1151,6 +1309,7 @@ pub(crate) fn decode_scan_fast_tile_rgb_region= needed_cols + && stride + .checked_mul(needed_rows) + .is_some_and(|needed| len >= needed), + "stripe plane {plane_idx} ({len} bytes, stride {stride}) is not sized for \ + {stripe_mcus_per_row} MCUs of {h}x{v} blocks of {block_size}px; \ + StripeBuffer::resize_for must run first" + ); +} + fn deposit_block(plane: &mut [u8], stride: usize, x: u32, y: u32, block: &[u8; 64]) { let dst_row_start = (y as usize) * stride + (x as usize); debug_assert!(x as usize + 8 <= stride); debug_assert!(plane.len() >= dst_row_start + stride.saturating_mul(7) + 8); + // SAFETY: `plane`/`stride` come from a StripeBuffer sized by `resize_for`, + // and the caller's per-stripe `assert_stripe_deposit_capacity` check + // guarantees in release builds that every in-stripe (x, y) block start + // leaves room for 8 rows of 8 bytes at this stride. + // SAFETY: Destination pointers are derived from validated row starts and row widths. let mut dst = unsafe { plane.as_mut_ptr().add(dst_row_start) }; let mut src = block.as_ptr(); for _ in 0..8 { + // SAFETY: Destination pointers are derived from validated row starts and row widths. unsafe { ptr::copy_nonoverlapping(src, dst, 8); dst = dst.add(stride); @@ -1400,8 +1596,12 @@ fn deposit_dc_block(plane: &mut [u8], stride: usize, x: u32, y: u32, pixel: u8) let dst_row_start = (y as usize) * stride + (x as usize); debug_assert!(x as usize + 8 <= stride); debug_assert!(plane.len() >= dst_row_start + stride.saturating_mul(7) + 8); + // SAFETY: same contract as `deposit_block` — upheld by `resize_for` plus + // the caller's per-stripe `assert_stripe_deposit_capacity` check. + // SAFETY: Destination pointers are derived from validated row starts and row widths. let mut dst = unsafe { plane.as_mut_ptr().add(dst_row_start) }; for _ in 0..8 { + // SAFETY: Destination pointers are derived from validated row starts and row widths. unsafe { ptr::write_bytes(dst, pixel, 8); dst = dst.add(stride); @@ -1669,6 +1869,16 @@ fn decode_mcu_row( ) -> Result<(), JpegError> { let stripe_mcu_end = stripe_mcu_start + stripe_mcus_per_row; let block_size = downscale.output_block_size(); + for comp in &plan.components { + assert_stripe_deposit_capacity( + stripe, + comp.output_index, + u32::from(comp.h), + u32::from(comp.v), + stripe_mcus_per_row, + block_size, + ); + } let mut pixels_4x4 = [0u8; 16]; let mut pixels_2x2 = [0u8; 4]; for mx in 0..mcus_per_row { @@ -1920,6 +2130,9 @@ fn decode_mcu_row_fast_rgb_444( mcus_since_restart: &mut u32, expected_rst: &mut u8, ) -> Result<(), JpegError> { + for plane_idx in 0..3 { + assert_stripe_deposit_capacity(stripe, plane_idx, 1, 1, mcus_per_row, 8); + } for mx in 0..mcus_per_row { if restart > 0 && *mcus_since_restart == u32::from(restart) { let _ = br.ensure_bits(1); @@ -2385,8 +2598,12 @@ fn decode_mcu_row_fast_tile_420( stripe_mcu_start: u32, stripe_mcus_per_row: u32, stripe: &mut StripeBuffer, + profiler: &mut impl Fast420Profiler, ) -> Result<(), JpegError> { let stripe_mcu_end = stripe_mcu_start + stripe_mcus_per_row; + assert_stripe_deposit_capacity(stripe, 0, 2, 2, stripe_mcus_per_row, 8); + assert_stripe_deposit_capacity(stripe, 1, 1, 1, stripe_mcus_per_row, 8); + assert_stripe_deposit_capacity(stripe, 2, 1, 1, stripe_mcus_per_row, 8); for mx in 0..mcus_per_row { let in_region = mx >= stripe_mcu_start && mx < stripe_mcu_end; if !in_region { @@ -2410,6 +2627,7 @@ fn decode_mcu_row_fast_tile_420( y_comp.quant.as_ref(), coeff, )?; + profiler.record_activity(y0_activity); idct_deposit_fast_tile_block( y0_activity, backend, @@ -2429,6 +2647,7 @@ fn decode_mcu_row_fast_tile_420( y_comp.quant.as_ref(), coeff, )?; + profiler.record_activity(y1_activity); idct_deposit_fast_tile_block( y1_activity, backend, @@ -2448,6 +2667,7 @@ fn decode_mcu_row_fast_tile_420( y_comp.quant.as_ref(), coeff, )?; + profiler.record_activity(y2_activity); idct_deposit_fast_tile_block( y2_activity, backend, @@ -2467,6 +2687,7 @@ fn decode_mcu_row_fast_tile_420( y_comp.quant.as_ref(), coeff, )?; + profiler.record_activity(y3_activity); idct_deposit_fast_tile_block( y3_activity, backend, @@ -2486,6 +2707,7 @@ fn decode_mcu_row_fast_tile_420( cb_comp.quant.as_ref(), coeff, )?; + profiler.record_activity(cb_activity); idct_deposit_fast_tile_block( cb_activity, backend, @@ -2505,6 +2727,7 @@ fn decode_mcu_row_fast_tile_420( cr_comp.quant.as_ref(), coeff, )?; + profiler.record_activity(cr_activity); idct_deposit_fast_tile_block( cr_activity, backend, @@ -2520,172 +2743,6 @@ fn decode_mcu_row_fast_tile_420( Ok(()) } -#[allow(clippy::too_many_arguments)] -fn decode_mcu_row_fast_tile_420_profiled( - y_comp: &PreparedComponentPlan, - cb_comp: &PreparedComponentPlan, - cr_comp: &PreparedComponentPlan, - backend: Backend, - br: &mut BitReader<'_>, - y_dc: &mut i32, - cb_dc: &mut i32, - cr_dc: &mut i32, - coeff: &mut CoefficientBlock, - pixels: &mut [u8; 64], - mcus_per_row: u32, - stripe_mcu_start: u32, - stripe_mcus_per_row: u32, - stripe: &mut StripeBuffer, - counts: &mut BenchBlockActivityCounts, -) -> Result<(), JpegError> { - let stripe_mcu_end = stripe_mcu_start + stripe_mcus_per_row; - for mx in 0..mcus_per_row { - let in_region = mx >= stripe_mcu_start && mx < stripe_mcu_end; - if !in_region { - for _ in 0..4 { - skip_block(br, &y_comp.dc_table, &y_comp.ac_table, y_dc)?; - } - skip_block(br, &cb_comp.dc_table, &cb_comp.ac_table, cb_dc)?; - skip_block(br, &cr_comp.dc_table, &cr_comp.ac_table, cr_dc)?; - continue; - } - - let local_mx = mx - stripe_mcu_start; - let y_x = local_mx * 16; - let c_x = local_mx * 8; - - let y0_activity = decode_block_with_activity( - br, - &y_comp.dc_table, - &y_comp.ac_table, - y_dc, - y_comp.quant.as_ref(), - coeff, - )?; - record_profiled_activity(counts, y0_activity); - idct_deposit_fast_tile_block( - y0_activity, - backend, - coeff, - pixels, - &mut stripe.planes[0], - stripe.plane_strides[0], - y_x, - 0, - ); - - let y1_activity = decode_block_with_activity( - br, - &y_comp.dc_table, - &y_comp.ac_table, - y_dc, - y_comp.quant.as_ref(), - coeff, - )?; - record_profiled_activity(counts, y1_activity); - idct_deposit_fast_tile_block( - y1_activity, - backend, - coeff, - pixels, - &mut stripe.planes[0], - stripe.plane_strides[0], - y_x + 8, - 0, - ); - - let y2_activity = decode_block_with_activity( - br, - &y_comp.dc_table, - &y_comp.ac_table, - y_dc, - y_comp.quant.as_ref(), - coeff, - )?; - record_profiled_activity(counts, y2_activity); - idct_deposit_fast_tile_block( - y2_activity, - backend, - coeff, - pixels, - &mut stripe.planes[0], - stripe.plane_strides[0], - y_x, - 8, - ); - - let y3_activity = decode_block_with_activity( - br, - &y_comp.dc_table, - &y_comp.ac_table, - y_dc, - y_comp.quant.as_ref(), - coeff, - )?; - record_profiled_activity(counts, y3_activity); - idct_deposit_fast_tile_block( - y3_activity, - backend, - coeff, - pixels, - &mut stripe.planes[0], - stripe.plane_strides[0], - y_x + 8, - 8, - ); - - let cb_activity = decode_block_with_activity( - br, - &cb_comp.dc_table, - &cb_comp.ac_table, - cb_dc, - cb_comp.quant.as_ref(), - coeff, - )?; - record_profiled_activity(counts, cb_activity); - idct_deposit_fast_tile_block( - cb_activity, - backend, - coeff, - pixels, - &mut stripe.planes[1], - stripe.plane_strides[1], - c_x, - 0, - ); - - let cr_activity = decode_block_with_activity( - br, - &cr_comp.dc_table, - &cr_comp.ac_table, - cr_dc, - cr_comp.quant.as_ref(), - coeff, - )?; - record_profiled_activity(counts, cr_activity); - idct_deposit_fast_tile_block( - cr_activity, - backend, - coeff, - pixels, - &mut stripe.planes[2], - stripe.plane_strides[2], - c_x, - 0, - ); - } - - Ok(()) -} - -fn record_profiled_activity(counts: &mut BenchBlockActivityCounts, activity: BlockActivity) { - match activity { - BlockActivity::DcOnly => counts.record_dc_only(), - BlockActivity::BottomHalfZero => counts.record_bottom_half_zero(), - BlockActivity::General => counts.record_general(), - } -} - #[allow(clippy::too_many_arguments)] fn decode_mcu_row_fast_tile_420_eighth( y_comp: &PreparedComponentPlan, @@ -3098,10 +3155,26 @@ fn emit_stripe( } } ColorSpace::Cmyk | ColorSpace::Ycck => { - return Err(JpegError::HuffmanDecode { - mcu: 0, - reason: HuffmanFailure::InvalidSymbol, - }); + let OutputScratch::RgbGeneric(scratch) = output_scratch else { + unreachable!("CMYK/YCCK decode requires reusable row scratch"); + }; + for local_y in 0..stripe_rows { + fill_four_component_rgb_row( + plan, + prev, + curr, + next, + local_y as u32, + width, + scratch, + )?; + writer.write_rgb_row( + y_start + local_y as u32, + &scratch.r[..width], + &scratch.g[..width], + &scratch.b[..width], + )?; + } } } Ok(()) @@ -3325,16 +3398,137 @@ fn emit_stripe_rgb( } } ColorSpace::Cmyk | ColorSpace::Ycck => { - return Err(JpegError::HuffmanDecode { - mcu: 0, - reason: HuffmanFailure::InvalidSymbol, - }); + let RgbOutputScratch::RgbGeneric(scratch) = output_scratch else { + unreachable!("CMYK/YCCK RGB output requires reusable row scratch"); + }; + for local_y in 0..stripe_rows { + fill_four_component_rgb_row( + plan, + prev, + curr, + next, + local_y as u32, + width, + scratch, + )?; + writer.with_rgb_rows(y_start + local_y as u32, 1, |dst, _| { + backend.fill_rgb_row_from_rgb( + &scratch.r[..width], + &scratch.g[..width], + &scratch.b[..width], + dst, + ); + Ok(()) + })?; + } } } Ok(()) } +#[allow(clippy::too_many_arguments)] +fn fill_four_component_rgb_row( + plan: &PreparedDecodePlan, + prev: Option<&StripeBuffer>, + curr: &StripeBuffer, + next: Option<&StripeBuffer>, + local_y: u32, + width: usize, + scratch: &mut RgbGenericRows, +) -> Result<(), JpegError> { + let (c0_h, c0_v) = plan + .sampling + .component(0) + .map(|(h, v)| (u32::from(h), u32::from(v))) + .ok_or(JpegError::UnsupportedComponentCount { count: 0 })?; + let (c1_h, c1_v) = plan + .sampling + .component(1) + .map(|(h, v)| (u32::from(h), u32::from(v))) + .ok_or(JpegError::UnsupportedComponentCount { count: 1 })?; + let (c2_h, c2_v) = plan + .sampling + .component(2) + .map(|(h, v)| (u32::from(h), u32::from(v))) + .ok_or(JpegError::UnsupportedComponentCount { count: 2 })?; + let (k_h, k_v) = plan + .sampling + .component(3) + .map(|(h, v)| (u32::from(h), u32::from(v))) + .ok_or(JpegError::UnsupportedComponentCount { count: 3 })?; + let max_h = u32::from(plan.sampling.max_h); + let max_v = u32::from(plan.sampling.max_v); + + upsample_component_row_stripe( + prev, + curr, + next, + 0, + c0_h, + c0_v, + max_h, + max_v, + local_y, + width, + &mut scratch.r, + ); + upsample_component_row_stripe( + prev, + curr, + next, + 1, + c1_h, + c1_v, + max_h, + max_v, + local_y, + width, + &mut scratch.g, + ); + upsample_component_row_stripe( + prev, + curr, + next, + 2, + c2_h, + c2_v, + max_h, + max_v, + local_y, + width, + &mut scratch.b, + ); + upsample_component_row_stripe( + prev, + curr, + next, + 3, + k_h, + k_v, + max_h, + max_v, + local_y, + width, + &mut scratch.k, + ); + + for x in 0..width { + let (r, g, b) = match plan.color_space { + ColorSpace::Cmyk => { + inverted_cmyk_to_rgb(scratch.r[x], scratch.g[x], scratch.b[x], scratch.k[x]) + } + ColorSpace::Ycck => ycck_to_rgb(scratch.r[x], scratch.g[x], scratch.b[x], scratch.k[x]), + _ => unreachable!("four-component conversion requires CMYK/YCCK input"), + }; + scratch.r[x] = r; + scratch.g[x] = g; + scratch.b[x] = b; + } + + Ok(()) +} + fn component_row_triplet<'a>( prev: Option>, curr: StripePlane<'a>, @@ -3504,13 +3698,11 @@ mod tests { use crate::Decoder; use alloc::sync::Arc; use alloc::vec; - - const BASELINE_420_JPG: &[u8] = - include_bytes!("../../fixtures/conformance/baseline_420_16x16.jpg"); + use j2k_test_support::JPEG_BASELINE_420_16X16; #[test] fn fast_tile_rgb_matches_generic_baseline_decode() { - let dec = Decoder::new(BASELINE_420_JPG).expect("fixture must parse"); + let dec = Decoder::new(JPEG_BASELINE_420_16X16).expect("fixture must parse"); assert!(dec.plan.matches_fast_tile_shape()); let mut generic = vec![0u8; (dec.info.dimensions.0 * dec.info.dimensions.1 * 3) as usize]; @@ -3591,6 +3783,7 @@ mod tests { let mut y_dc = 0; let mut cb_dc = 0; let mut cr_dc = 0; + let mut profiler = NoopFast420Profiler; decode_mcu_row_fast_tile_420( &y_comp, @@ -3607,6 +3800,7 @@ mod tests { 0, 1, &mut stripe, + &mut profiler, ) .expect("second stripe row must still decode within stripe-local buffers"); @@ -3815,7 +4009,7 @@ mod tests { let plan = PreparedDecodePlan { components: vec![], - sampling: SamplingFactors::from_components(&[(1, 1), (1, 1), (1, 1)]), + sampling: SamplingFactors::from_validated_components(&[(1, 1), (1, 1), (1, 1)]), color_space: ColorSpace::YCbCr, restart_interval: None, dimensions: (width as u32, height), diff --git a/crates/signinum-jpeg/src/error.rs b/crates/j2k-jpeg/src/error.rs similarity index 51% rename from crates/signinum-jpeg/src/error.rs rename to crates/j2k-jpeg/src/error.rs index cc0145a7..e24fc720 100644 --- a/crates/signinum-jpeg/src/error.rs +++ b/crates/j2k-jpeg/src/error.rs @@ -3,7 +3,7 @@ //! Typed error and warning taxonomy. See spec Section 6. use crate::info::{ColorSpace, Rect, SofKind}; -use signinum_core::CodecError; +use j2k_core::CodecError; /// A category of JPEG marker. Carried in [`JpegError::UnexpectedMarker`] and /// related variants so callers can branch on marker class without parsing the @@ -31,176 +31,420 @@ pub enum MarkerKind { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Reason an otherwise recognized SOF marker cannot be decoded. pub enum UnsupportedReason { + /// Stream uses arithmetic entropy coding. ArithmeticCoding, + /// Stream uses hierarchical JPEG coding. Hierarchical, + /// Stream combines arithmetic entropy coding and hierarchical coding. ArithmeticAndHierarchical, + /// Stream uses a differential baseline SOF. DifferentialBaseline, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Huffman entropy decoder failure category. pub enum HuffmanFailure { + /// A Huffman code exceeded the representable code space. CodeOverflow, + /// A decoded symbol is invalid for its context. InvalidSymbol, + /// The entropy stream ended before a symbol could be decoded. TableExhausted, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Invalid decoder-builder input configuration. pub enum BuilderConflictReason { + /// No input source was provided. NoInput, + /// Raw input bytes and scan fragments were both provided. InputAndScanFragments, + /// Scan-fragment mode was selected without any fragments. ScanFragmentsEmpty, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// JPEG table class used in diagnostics. pub enum TableKind { + /// Quantization table. Quant, + /// AC Huffman table. HuffmanAc, + /// DC Huffman table. HuffmanDc, } +/// Fatal JPEG decode or API error. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] #[non_exhaustive] pub enum JpegError { #[error("JPEG truncated at offset {offset}: expected {expected} more bytes")] - Truncated { offset: usize, expected: usize }, + /// Input ended before the requested bytes could be read. + Truncated { + /// Byte offset where decoding stopped. + offset: usize, + /// Additional byte count required. + expected: usize, + }, #[error("invalid marker FF{marker:02X} at offset {offset}")] - InvalidMarker { offset: usize, marker: u8 }, + /// Marker byte is not legal in the current JPEG position. + InvalidMarker { + /// Byte offset of the marker. + offset: usize, + /// Raw marker byte following `0xff`. + marker: u8, + }, #[error("expected {expected:?}, found FF{found:02X} at offset {offset}")] + /// A different marker kind was required at this position. UnexpectedMarker { + /// Byte offset of the unexpected marker. offset: usize, + /// Marker kind expected by the parser. expected: MarkerKind, + /// Raw marker byte that was found. found: u8, }, #[error("missing required marker {marker:?}")] - MissingMarker { marker: MarkerKind }, + /// Required marker was absent from the stream. + MissingMarker { + /// Missing marker kind. + marker: MarkerKind, + }, #[error("duplicate {marker:?} at offset {offset}")] - DuplicateMarker { offset: usize, marker: MarkerKind }, + /// A marker appeared more than once where only one is legal. + DuplicateMarker { + /// Byte offset of the duplicate marker. + offset: usize, + /// Duplicated marker kind. + marker: MarkerKind, + }, #[error("invalid length {length} for marker FF{marker:02X} at offset {offset}")] + /// Marker segment length is invalid for the marker kind. InvalidSegmentLength { + /// Byte offset of the segment. offset: usize, + /// Raw marker byte following `0xff`. marker: u8, + /// Declared segment length. length: u16, }, + #[error("conflicting duplicate JPEG table {table:?} id={id} at offset {offset}")] + /// Duplicate DQT/DHT table has different bytes from an earlier definition. + ConflictingDuplicateTable { + /// Byte offset of the conflicting table segment. + offset: usize, + /// Table class. + table: TableKind, + /// Table id. + id: u8, + }, + + #[error("expected dimensions are required to repair zero SOF dimensions at offset {offset}")] + /// TIFF/NDPI metadata did not provide dimensions needed to repair a zero SOF. + ExpectedDimensionsRequired { + /// Byte offset of the SOF marker. + offset: usize, + }, + + #[error( + "expected dimensions {expected:?} conflict with SOF dimensions {actual:?} at offset {offset}" + )] + /// Container-provided dimensions conflict with non-zero SOF dimensions. + ConflictingExpectedDimensions { + /// Byte offset of the SOF marker. + offset: usize, + /// Expected dimensions supplied by the caller. + expected: (u16, u16), + /// Dimensions declared by the SOF marker. + actual: (u16, u16), + }, + + #[error("invalid TIFF JPEG assembly at offset {offset}: {reason}")] + /// TIFF/JPEGTables assembly cannot produce a valid JPEG interchange stream. + InvalidJpegAssembly { + /// Byte offset of the assembly problem. + offset: usize, + /// Static diagnostic reason. + reason: &'static str, + }, + + #[error("conflicting DRI values at offset {offset}: existing {existing}, new {new}")] + /// Duplicate DRI marker conflicts with an earlier DRI value. + ConflictingDri { + /// Byte offset of the conflicting DRI segment. + offset: usize, + /// Existing non-zero restart interval. + existing: u16, + /// New non-zero restart interval. + new: u16, + }, + /// Unsupported SOF variant. Carries the raw marker byte (e.g. `0xC9` for /// arithmetic extended-sequential) so callers routing to a fallback /// decoder can distinguish FFC5 from FFC9 without relying on `reason`. #[error("unsupported SOF marker FF{marker:02X} ({reason:?})")] + /// SOF marker class is outside the decoder's supported JPEG subset. UnsupportedSof { + /// Raw SOF marker byte. marker: u8, + /// Unsupported feature category. reason: UnsupportedReason, }, #[error("unsupported component count: {count}")] - UnsupportedComponentCount { count: u8 }, + /// Component count is outside the supported range. + UnsupportedComponentCount { + /// Declared component count. + count: u8, + }, #[error("unsupported color space for decode: {color_space:?}")] - UnsupportedColorSpace { color_space: ColorSpace }, + /// Color space cannot be produced by the requested decode path. + UnsupportedColorSpace { + /// Header-derived color space. + color_space: ColorSpace, + }, #[error("unsupported bit depth: {depth}")] - UnsupportedBitDepth { depth: u8 }, + /// Sample precision is not supported by this decoder path. + UnsupportedBitDepth { + /// Declared sample precision in bits. + depth: u8, + }, #[error("unsupported lossless predictor: {predictor}")] - UnsupportedPredictor { predictor: u8 }, + /// Lossless predictor selection is unsupported. + UnsupportedPredictor { + /// Predictor value from the scan header. + predictor: u8, + }, #[error("zero dimension in SOF: {width}×{height}")] - ZeroDimension { width: u16, height: u16 }, + /// SOF declares a zero width or height. + ZeroDimension { + /// Declared width. + width: u16, + /// Declared height. + height: u16, + }, #[error("dimension overflow: {width}×{height} exceeds 65500")] - DimensionOverflow { width: u32, height: u32 }, + /// Dimensions exceed this crate's safe decode bounds. + DimensionOverflow { + /// Declared width. + width: u32, + /// Declared height. + height: u32, + }, #[error("invalid sampling ({h}×{v}) for component {component}")] - InvalidSampling { component: u8, h: u8, v: u8 }, + /// Component sampling factors are outside the JPEG legal range. + InvalidSampling { + /// Component id. + component: u8, + /// Horizontal sampling factor. + h: u8, + /// Vertical sampling factor. + v: u8, + }, #[error("missing quantization table {table_id} for component {component}")] - MissingQuantTable { component: u8, table_id: u8 }, + /// Component references a quantization table that was not defined. + MissingQuantTable { + /// Component id. + component: u8, + /// Referenced quantization table id. + table_id: u8, + }, #[error("missing Huffman table class={class} id={id} for component {component}")] - MissingHuffmanTable { component: u8, class: u8, id: u8 }, + /// Scan references a Huffman table that was not defined. + MissingHuffmanTable { + /// Component id. + component: u8, + /// Huffman table class. + class: u8, + /// Huffman table id. + id: u8, + }, #[error( "invalid sequential scan parameters at offset {offset}: Ss={ss} Se={se} Ah={ah} Al={al}" )] + /// Scan spectral selection or approximation parameters are invalid. InvalidScanParameters { + /// Byte offset of the scan header. offset: usize, + /// Start of spectral selection. ss: u8, + /// End of spectral selection. se: u8, + /// Successive approximation high bit. ah: u8, + /// Successive approximation low bit. al: u8, }, #[error("unknown scan component id {component} at offset {offset}")] - UnknownScanComponent { offset: usize, component: u8 }, + /// Scan references a component id not declared in the SOF. + UnknownScanComponent { + /// Byte offset of the scan header. + offset: usize, + /// Unknown component id. + component: u8, + }, #[error("duplicate scan component id {component} at offset {offset}")] - DuplicateScanComponent { offset: usize, component: u8 }, + /// Scan lists the same component more than once. + DuplicateScanComponent { + /// Byte offset of the scan header. + offset: usize, + /// Duplicated component id. + component: u8, + }, #[error( "invalid sequential scan component set at offset {offset}: expected {expected} components, found {found}" )] + /// Sequential scan does not contain the expected component set. InvalidSequentialComponentSet { + /// Byte offset of the scan header. offset: usize, + /// Expected component count. expected: u8, + /// Found component count. found: u8, }, #[error("invalid sequential scan count for {sof:?}: expected 1, found {count}")] - InvalidSequentialScanCount { sof: SofKind, count: u16 }, + /// Sequential SOF contained an invalid number of scans. + InvalidSequentialScanCount { + /// SOF kind being decoded. + sof: SofKind, + /// Observed scan count. + count: u16, + }, - #[error("Huffman decode failed at MCU {mcu}: {reason:?}")] - HuffmanDecode { mcu: u32, reason: HuffmanFailure }, + #[error("Huffman decode failed near MCU {mcu}: {reason:?}")] + /// Huffman entropy decoding failed. `mcu` is the current MCU when the + /// caller has MCU progress, or `0` for table/bitstream contexts that do + /// not track image position. + HuffmanDecode { + /// Current MCU index, or `0` when the decoder context has no MCU index. + mcu: u32, + /// Failure category. + reason: HuffmanFailure, + }, #[error("restart mismatch at offset {offset}: expected RST{expected}, found FF{found:02X}")] + /// Restart marker sequence did not match the expected RST index. RestartMismatch { + /// Byte offset of the marker. offset: usize, + /// Expected RST index. expected: u8, + /// Found raw marker byte. found: u8, }, #[error("unexpected EOI at MCU {mcu_at}/{mcu_total}")] - UnexpectedEoi { mcu_at: u32, mcu_total: u32 }, + /// EOI was reached before all MCUs were decoded. + UnexpectedEoi { + /// MCU index where EOI was found. + mcu_at: u32, + /// Total MCU count expected for the image. + mcu_total: u32, + }, #[error("coefficient overflow at MCU {mcu}, component {component}")] - CoefficientOverflow { mcu: u32, component: u8 }, + /// Decoded coefficient exceeded the representable range. + CoefficientOverflow { + /// MCU index. + mcu: u32, + /// Component index. + component: u8, + }, #[error("decode size {requested} bytes exceeds cap {cap} bytes")] - MemoryCapExceeded { requested: usize, cap: usize }, + /// Requested decode allocation exceeds the configured memory cap. + MemoryCapExceeded { + /// Requested byte count. + requested: usize, + /// Configured byte cap. + cap: usize, + }, #[error("output buffer too small: need {required} bytes, got {provided}")] - OutputBufferTooSmall { required: usize, provided: usize }, + /// Caller-provided output buffer is too small. + OutputBufferTooSmall { + /// Required byte count. + required: usize, + /// Provided byte count. + provided: usize, + }, #[error("stride {stride} smaller than row width {row}")] - InvalidStride { stride: usize, row: usize }, + /// Output stride is smaller than the decoded row size. + InvalidStride { + /// Caller-provided stride. + stride: usize, + /// Minimum row byte count. + row: usize, + }, #[error("rect {rect:?} out of image bounds ({width}×{height})")] - RectOutOfBounds { rect: Rect, width: u32, height: u32 }, + /// Requested decode rectangle is outside image bounds. + RectOutOfBounds { + /// Requested rectangle. + rect: Rect, + /// Image width in pixels. + width: u32, + /// Image height in pixels. + height: u32, + }, #[error("downscale not supported for {sof:?} streams")] - DownscaleUnsupported { sof: SofKind }, + /// Requested downscale is not supported for the SOF kind. + DownscaleUnsupported { + /// SOF kind being decoded. + sof: SofKind, + }, #[error("scan fragments overlap at MCU {mcu}")] - ScanFragmentsOverlap { mcu: u32 }, + /// Builder-provided scan fragments overlap in MCU space. + ScanFragmentsOverlap { + /// First overlapping MCU index. + mcu: u32, + }, #[error("builder input configuration conflict: {reason:?}")] - BuilderConflict { reason: BuilderConflictReason }, + /// Decoder builder inputs conflict. + BuilderConflict { + /// Conflict category. + reason: BuilderConflictReason, + }, - /// Transient pre-1.0 gap: the SOF is parseable and will eventually be + /// Transient pre-1.0 gap: the SOF is parseable and may eventually be /// supported by the decoder, but the current release does not implement - /// it yet. M3 removes this variant by implementing Extended12, - /// Progressive12, and Lossless. Distinct from `UnsupportedSof` because - /// callers routing - /// to a fallback decoder on `is_unsupported()` should NOT reroute streams - /// that a newer version of signinum will decode natively. + /// the requested shape yet. Distinct from `UnsupportedSof` because callers + /// routing to a fallback decoder on `is_unsupported()` should NOT reroute + /// streams that a newer version of j2k will decode natively. #[error("decode not yet implemented for {sof:?} — see CHANGELOG for milestone")] - NotImplemented { sof: SofKind }, + NotImplemented { + /// SOF kind awaiting implementation. + sof: SofKind, + }, #[error("row sink aborted decode")] + /// Row sink returned an error and aborted row-based decoding. RowSinkAborted, } @@ -237,7 +481,7 @@ impl JpegError { } /// True if the error is a transient "not yet implemented" gap — the stream - /// is valid and will decode on a future signinum release, so callers + /// is valid and will decode on a future j2k release, so callers /// should *not* reroute to a different decoder permanently. See /// [`Self::is_unsupported`] for errors that are permanent routing decisions. pub fn is_not_implemented(&self) -> bool { @@ -289,16 +533,55 @@ impl CodecError for JpegError { #[derive(Debug, Clone, PartialEq, Eq)] #[non_exhaustive] pub enum Warning { + /// Stream ended without an EOI marker after otherwise decodable entropy. MissingEoi, - SofDimensionsPatched { from: (u16, u16), to: (u16, u16) }, + /// SOF dimensions were repaired from external context. + SofDimensionsPatched { + /// Original SOF dimensions. + from: (u16, u16), + /// Replacement dimensions. + to: (u16, u16), + }, + /// Stream uses nonstandard but decodable table layout. NonstandardTables, - AdobeApp14Ambiguous { raw_transform: u8 }, - IccProfileIgnored { size: usize }, - UnknownAppMarker { marker: u8, size: usize }, - RestartRecovered { offset: usize }, - PrecisionClamped { from_bits: u8, to_bits: u8 }, + /// Adobe APP14 transform value could not unambiguously define color. + AdobeApp14Ambiguous { + /// Raw APP14 transform byte. + raw_transform: u8, + }, + /// ICC profile was present but ignored by this decoder. + IccProfileIgnored { + /// ICC payload size in bytes. + size: usize, + }, + /// Unknown APP marker was skipped. + UnknownAppMarker { + /// Raw APP marker byte. + marker: u8, + /// Segment payload size in bytes. + size: usize, + }, + /// Decoder recovered at a restart marker. + RestartRecovered { + /// Byte offset near the recovered restart marker. + offset: usize, + }, + /// Higher-precision samples were clamped to a lower output precision. + PrecisionClamped { + /// Source precision in bits. + from_bits: u8, + /// Output precision in bits. + to_bits: u8, + }, + /// Color profile metadata was present but unrecognized. UnknownColorProfile, - TableCacheMismatch { which: TableKind, id: u8 }, + /// Cached table metadata disagreed with the active stream tables. + TableCacheMismatch { + /// Table class. + which: TableKind, + /// Table id. + id: u8, + }, } impl core::fmt::Display for Warning { diff --git a/crates/signinum-jpeg/src/idct/avx2.rs b/crates/j2k-jpeg/src/idct/avx2.rs similarity index 85% rename from crates/signinum-jpeg/src/idct/avx2.rs rename to crates/j2k-jpeg/src/idct/avx2.rs index 3d1bf344..fe578e7b 100644 --- a/crates/signinum-jpeg/src/idct/avx2.rs +++ b/crates/j2k-jpeg/src/idct/avx2.rs @@ -12,10 +12,9 @@ #![allow(clippy::cast_possible_truncation, clippy::cast_lossless)] use core::arch::x86_64::{ - __m128i, _mm_add_epi32, _mm_cvtepi16_epi32, _mm_loadl_epi64, _mm_mullo_epi32, _mm_packs_epi32, - _mm_packus_epi16, _mm_set1_epi32, _mm_slli_epi32, _mm_srai_epi32, _mm_srli_si128, - _mm_storel_epi64, _mm_sub_epi32, _mm_unpackhi_epi32, _mm_unpackhi_epi64, _mm_unpacklo_epi32, - _mm_unpacklo_epi64, + __m128i, _mm_add_epi32, _mm_cvtepi16_epi32, _mm_mullo_epi32, _mm_packs_epi32, _mm_packus_epi16, + _mm_set1_epi32, _mm_slli_epi32, _mm_srai_epi32, _mm_srli_si128, _mm_storel_epi64, + _mm_sub_epi32, _mm_unpackhi_epi32, _mm_unpackhi_epi64, _mm_unpacklo_epi32, _mm_unpacklo_epi64, }; const CONST_BITS: i32 = 13; @@ -49,14 +48,29 @@ pub(crate) unsafe fn idct_islow(input: &[i16; 64], output: &mut [u8; 64]) { const PASS2_SHIFT: i32 = CONST_BITS + PASS1_BITS + 3; let src = input.as_ptr(); - let (r0l, r0h) = unsafe { widen(src.add(0)) }; - let (r1l, r1h) = unsafe { widen(src.add(8)) }; - let (r2l, r2h) = unsafe { widen(src.add(16)) }; - let (r3l, r3h) = unsafe { widen(src.add(24)) }; - let (r4l, r4h) = unsafe { widen(src.add(32)) }; - let (r5l, r5h) = unsafe { widen(src.add(40)) }; - let (r6l, r6h) = unsafe { widen(src.add(48)) }; - let (r7l, r7h) = unsafe { widen(src.add(56)) }; + // SAFETY: `input` contains exactly 64 i16 coefficients. The offsets below + // load eight coefficients each and stay within that fixed block. + let ( + (r0l, r0h), + (r1l, r1h), + (r2l, r2h), + (r3l, r3h), + (r4l, r4h), + (r5l, r5h), + (r6l, r6h), + (r7l, r7h), + ) = unsafe { + ( + widen(src.add(0)), + widen(src.add(8)), + widen(src.add(16)), + widen(src.add(24)), + widen(src.add(32)), + widen(src.add(40)), + widen(src.add(48)), + widen(src.add(56)), + ) + }; let round1 = _mm_set1_epi32(1 << (PASS1_SHIFT - 1)); let cw_lo = idct_1d_x4::(r0l, r1l, r2l, r3l, r4l, r5l, r6l, r7l, round1); @@ -98,6 +112,8 @@ pub(crate) unsafe fn idct_islow(input: &[i16; 64], output: &mut [u8; 64]) { ); let store = output.as_mut_ptr(); + // SAFETY: `output` contains 64 writable bytes, and each store writes one + // eight-byte row at offsets 0, 8, ..., 56. unsafe { store_row(store, fll0, flh0); store_row(store.add(8), fll1, flh1); @@ -115,10 +131,9 @@ pub(crate) unsafe fn idct_islow(input: &[i16; 64], output: &mut [u8; 64]) { #[inline] #[target_feature(enable = "avx2")] unsafe fn widen(src: *const i16) -> (__m128i, __m128i) { - let vec = unsafe { _mm_loadl_epi64(src.cast()) }; // load 8 bytes = 4 i16 into lower 64 - // Actually the row is 8 i16 = 16 bytes, not 8. Use full 128-bit load. + // SAFETY: callers pass a pointer to at least eight readable i16 values; + // unaligned loads are intentional for JPEG coefficient blocks. let full = unsafe { core::ptr::read_unaligned(src.cast::<__m128i>()) }; - let _ = vec; let lo = _mm_cvtepi16_epi32(full); let hi_shuffled = _mm_srli_si128::<8>(full); let hi = _mm_cvtepi16_epi32(hi_shuffled); @@ -129,8 +144,12 @@ unsafe fn widen(src: *const i16) -> (__m128i, __m128i) { #[inline] #[target_feature(enable = "avx2")] unsafe fn store_row(dst: *mut u8, lo: __m128i, hi: __m128i) { - let i16_packed = _mm_packs_epi32(lo, hi); // [lo0..3, hi0..3] as i16 - let u8_packed = _mm_packus_epi16(i16_packed, i16_packed); // low 8 lanes are our u8s + // Lanes are [lo0..3, hi0..3] as i16. + let i16_packed = _mm_packs_epi32(lo, hi); + // The low eight lanes are the saturated output row. + let u8_packed = _mm_packus_epi16(i16_packed, i16_packed); + // SAFETY: callers pass a pointer to eight writable bytes; the store writes + // only the low 64 bits and does not require alignment. unsafe { _mm_storel_epi64(dst.cast(), u8_packed); } @@ -225,6 +244,7 @@ mod tests { idct_scalar(input, &mut scalar_out); let mut avx_out = [0u8; 64]; if std::is_x86_feature_detected!("avx2") { + // SAFETY: the runtime guard proves the required AVX2 feature. unsafe { idct_islow(input, &mut avx_out) }; } else { // Running the test on a non-AVX2 host: copy scalar output so diff --git a/crates/signinum-jpeg/src/idct/downscale.rs b/crates/j2k-jpeg/src/idct/downscale.rs similarity index 100% rename from crates/signinum-jpeg/src/idct/downscale.rs rename to crates/j2k-jpeg/src/idct/downscale.rs diff --git a/crates/signinum-jpeg/src/idct/mod.rs b/crates/j2k-jpeg/src/idct/mod.rs similarity index 83% rename from crates/signinum-jpeg/src/idct/mod.rs rename to crates/j2k-jpeg/src/idct/mod.rs index eba55eac..ce27a2ad 100644 --- a/crates/signinum-jpeg/src/idct/mod.rs +++ b/crates/j2k-jpeg/src/idct/mod.rs @@ -14,5 +14,7 @@ pub(crate) mod neon; pub(crate) mod avx2; pub(crate) use scalar::idct_islow; +pub(crate) use scalar::idct_islow_12bit; +pub(crate) use scalar::idct_islow_12bit_dc_only_sample; pub(crate) use scalar::idct_islow_dc_only; pub(crate) use scalar::idct_islow_dc_only_pixel; diff --git a/crates/signinum-jpeg/src/idct/neon.rs b/crates/j2k-jpeg/src/idct/neon.rs similarity index 92% rename from crates/signinum-jpeg/src/idct/neon.rs rename to crates/j2k-jpeg/src/idct/neon.rs index 843caba1..011fdd5e 100644 --- a/crates/signinum-jpeg/src/idct/neon.rs +++ b/crates/j2k-jpeg/src/idct/neon.rs @@ -48,16 +48,25 @@ pub(crate) unsafe fn idct_islow(input: &[i16; 64], output: &mut [u8; 64]) { // Load 8 rows as int16x8_t so the common bottom-half-zero shortcut can // reuse the tail rows instead of rescanning coefficients scalar-by-scalar. let src = input.as_ptr(); + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. let row0 = unsafe { vld1q_s16(src) }; + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. let row1 = unsafe { vld1q_s16(src.add(8)) }; + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. let row2 = unsafe { vld1q_s16(src.add(16)) }; + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. let row3 = unsafe { vld1q_s16(src.add(24)) }; + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. let row4 = unsafe { vld1q_s16(src.add(32)) }; + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. let row5 = unsafe { vld1q_s16(src.add(40)) }; + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. let row6 = unsafe { vld1q_s16(src.add(48)) }; + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. let row7 = unsafe { vld1q_s16(src.add(56)) }; let bottom_half_zero = bottom_half_rows_are_zero(row4, row5, row6, row7); if bottom_half_zero { + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. unsafe { idct_islow_bottom_half_zero_rows(row0, row1, row2, row3, output); } @@ -135,6 +144,7 @@ pub(crate) unsafe fn idct_islow(input: &[i16; 64], output: &mut [u8; 64]) { // `fhl_r` = row r (4..7), cols 0..3. // `fhh_r` = row r (4..7), cols 4..7. let store = output.as_mut_ptr(); + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. unsafe { store_row(store, fll0, flh0); store_row(store.add(8), fll1, flh1); @@ -151,6 +161,7 @@ pub(crate) unsafe fn idct_islow(input: &[i16; 64], output: &mut [u8; 64]) { #[target_feature(enable = "neon")] pub(crate) unsafe fn idct_islow_bottom_half_zero(input: &[i16; 64], output: &mut [u8; 64]) { let src = input.as_ptr(); + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. unsafe { idct_islow_bottom_half_zero_rows( vld1q_s16(src), @@ -217,6 +228,7 @@ unsafe fn idct_islow_bottom_half_zero_rows( ); let store = output.as_mut_ptr(); + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. unsafe { store_row(store, fll0, flh0); store_row(store.add(8), fll1, flh1); @@ -232,7 +244,9 @@ unsafe fn idct_islow_bottom_half_zero_rows( #[inline] #[cfg(test)] fn bottom_half_is_zero(input: &[i16; 64]) -> bool { + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. let tail = unsafe { input.as_ptr().add(32) }; + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. unsafe { bottom_half_rows_are_zero( vld1q_s16(tail), @@ -266,6 +280,7 @@ fn bottom_half_rows_are_zero( #[target_feature(enable = "neon")] unsafe fn store_row(dst: *mut u8, lo: int32x4_t, hi: int32x4_t) { let packed_i16: int16x8_t = vcombine_s16(vqmovn_s32(lo), vqmovn_s32(hi)); + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. unsafe { vst1_u8(dst, vqmovun_s16(packed_i16)); } @@ -402,6 +417,7 @@ mod tests { let mut scalar_out = [0u8; 64]; idct_scalar(input, &mut scalar_out); let mut neon_out = [0u8; 64]; + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. unsafe { idct_islow(input, &mut neon_out) }; (scalar_out, neon_out) } @@ -469,6 +485,7 @@ mod tests { let mut scalar_out = [0u8; 64]; idct_scalar(&input, &mut scalar_out); let mut neon_out = [0u8; 64]; + // SAFETY: IDCT pointers address fixed 8x8 arrays and NEON dispatch preconditions hold. unsafe { idct_islow_bottom_half_zero(&input, &mut neon_out) }; assert_eq!(scalar_out, neon_out); } diff --git a/crates/signinum-jpeg/src/idct/scalar.rs b/crates/j2k-jpeg/src/idct/scalar.rs similarity index 63% rename from crates/signinum-jpeg/src/idct/scalar.rs rename to crates/j2k-jpeg/src/idct/scalar.rs index f6aa812a..14d6b44e 100644 --- a/crates/signinum-jpeg/src/idct/scalar.rs +++ b/crates/j2k-jpeg/src/idct/scalar.rs @@ -44,7 +44,7 @@ pub(crate) fn idct_islow(input: &[i16; 64], output: &mut [u8; 64]) { } } for row in 0..8 { - idct_1d_row(&work, output, row); + idct_1d_row::(&work, output, row); } } @@ -55,7 +55,7 @@ pub(crate) fn idct_islow_bottom_half_zero(input: &[i16; 64], output: &mut [u8; 6 idct_1d_column_bottom_half_zero(input, &mut work, col); } for row in 0..8 { - idct_1d_row(&work, output, row); + idct_1d_row::(&work, output, row); } } @@ -67,9 +67,59 @@ pub(crate) fn idct_islow_dc_only(dc_coeff: i16, output: &mut [u8; 64]) { /// Return the uniform output sample produced by the DC-only ISLOW path. pub(crate) fn idct_islow_dc_only_pixel(dc_coeff: i16) -> u8 { - ((i32::from(dc_coeff) + 4) >> 3) - .wrapping_add(128) - .clamp(0, 255) as u8 + idct_islow_dc_only_sample::(dc_coeff) +} + +/// Inverse DCT of a 12-bit JPEG block, returning native 0..4095 sample values. +pub(crate) fn idct_islow_12bit(input: &[i16; 64], output: &mut [u16; 64]) { + let mut work = [Wrapping(0i32); 64]; + if input[32..].iter().all(|&coeff| coeff == 0) { + for col in 0..8 { + idct_1d_column_bottom_half_zero(input, &mut work, col); + } + } else { + for col in 0..8 { + idct_1d_column(input, &mut work, col); + } + } + for row in 0..8 { + idct_1d_row::(&work, output, row); + } +} + +/// Return the uniform native 12-bit sample produced by the DC-only ISLOW path. +pub(crate) fn idct_islow_12bit_dc_only_sample(dc_coeff: i16) -> u16 { + idct_islow_dc_only_sample::(dc_coeff) +} + +trait IdctSample: Copy { + const LEVEL_SHIFT: i32; + const MAX: i32; + + fn from_clamped_i32(value: i32) -> Self; +} + +impl IdctSample for u8 { + const LEVEL_SHIFT: i32 = 128; + const MAX: i32 = 255; + + fn from_clamped_i32(value: i32) -> Self { + value as Self + } +} + +impl IdctSample for u16 { + const LEVEL_SHIFT: i32 = 2048; + const MAX: i32 = 4095; + + fn from_clamped_i32(value: i32) -> Self { + value as Self + } +} + +fn idct_islow_dc_only_sample(dc_coeff: i16) -> T { + let level_shifted = ((i32::from(dc_coeff) + 4) >> 3).wrapping_add(T::LEVEL_SHIFT); + T::from_clamped_i32(level_shifted.clamp(0, T::MAX)) } fn idct_1d_column(input: &[i16; 64], work: &mut [Wrapping; 64], col: usize) { @@ -209,7 +259,7 @@ fn descale(v: Wrapping, shift: usize) -> Wrapping { Wrapping(v.0 >> shift) } -fn idct_1d_row(work: &[Wrapping; 64], output: &mut [u8; 64], row: usize) { +fn idct_1d_row(work: &[Wrapping; 64], output: &mut [T; 64], row: usize) { let base = row * 8; let p0 = work[base]; let p1 = work[base + 1]; @@ -226,7 +276,7 @@ fn idct_1d_row(work: &[Wrapping; 64], output: &mut [u8; 64], row: usize) { if p1.0 == 0 && p2.0 == 0 && p3.0 == 0 && p4.0 == 0 && p5.0 == 0 && p6.0 == 0 && p7.0 == 0 { let dc_shift = PASS1_BITS + 3; let rounding_dc = Wrapping(1i32 << (dc_shift - 1)); - let pixel = descale_and_clamp(p0 + rounding_dc, dc_shift); + let pixel = descale_and_clamp::(p0 + rounding_dc, dc_shift); for i in 0..8 { output[base + i] = pixel; } @@ -275,20 +325,20 @@ fn idct_1d_row(work: &[Wrapping; 64], output: &mut [u8; 64], row: usize) { let tmp2 = tmp2 + z2 + z3; let tmp3 = tmp3 + z1 + z4; - output[base] = descale_and_clamp(tmp10 + tmp3 + rounding, shift); - output[base + 7] = descale_and_clamp(tmp10 - tmp3 + rounding, shift); - output[base + 1] = descale_and_clamp(tmp11 + tmp2 + rounding, shift); - output[base + 6] = descale_and_clamp(tmp11 - tmp2 + rounding, shift); - output[base + 2] = descale_and_clamp(tmp12 + tmp1 + rounding, shift); - output[base + 5] = descale_and_clamp(tmp12 - tmp1 + rounding, shift); - output[base + 3] = descale_and_clamp(tmp13 + tmp0 + rounding, shift); - output[base + 4] = descale_and_clamp(tmp13 - tmp0 + rounding, shift); + output[base] = descale_and_clamp::(tmp10 + tmp3 + rounding, shift); + output[base + 7] = descale_and_clamp::(tmp10 - tmp3 + rounding, shift); + output[base + 1] = descale_and_clamp::(tmp11 + tmp2 + rounding, shift); + output[base + 6] = descale_and_clamp::(tmp11 - tmp2 + rounding, shift); + output[base + 2] = descale_and_clamp::(tmp12 + tmp1 + rounding, shift); + output[base + 5] = descale_and_clamp::(tmp12 - tmp1 + rounding, shift); + output[base + 3] = descale_and_clamp::(tmp13 + tmp0 + rounding, shift); + output[base + 4] = descale_and_clamp::(tmp13 - tmp0 + rounding, shift); } -fn descale_and_clamp(value: Wrapping, shift: usize) -> u8 { +fn descale_and_clamp(value: Wrapping, shift: usize) -> T { let shifted = value.0 >> shift; - let level_shifted = shifted.wrapping_add(128); - level_shifted.clamp(0, 255) as u8 + let level_shifted = shifted.wrapping_add(T::LEVEL_SHIFT); + T::from_clamped_i32(level_shifted.clamp(0, T::MAX)) } #[cfg(test)] @@ -316,6 +366,108 @@ mod tests { } } + // ------------------------------------------------------------------------- + // Ground truth: exact mathematical inverse DCT. + // + // ISLOW is a fixed-point (Chen-Wang) approximation; nothing previously + // pinned it to the *defining* inverse DCT, so a wrong constant or butterfly + // could pass the sanity checks (and be faithfully copied by the CUDA port). + // Compare every pixel to the exact double-precision IDCT. ISLOW meets the + // IEEE-1180 peak-error bound, so a 1-LSB tolerance is the correct gate. + + /// Exact 8x8 inverse DCT (DCT-III, orthonormal) of natural-order signed + /// coefficients, before the +128 level shift. `coeffs[v * 8 + u]` is the + /// coefficient at horizontal frequency `u`, vertical frequency `v`. + fn exact_idct_pixel(coeffs: &[i16; 64], x: usize, y: usize) -> f64 { + use core::f64::consts::PI; + let alpha = |k: usize| { + if k == 0 { + (1.0_f64 / 8.0).sqrt() + } else { + (2.0_f64 / 8.0).sqrt() + } + }; + let cos_term = |sample: usize, freq: usize| -> f64 { + let s = f64::from(u8::try_from(sample).unwrap()); + let f = f64::from(u8::try_from(freq).unwrap()); + (((2.0 * s) + 1.0) * f * PI / 16.0).cos() + }; + let mut acc = 0.0; + for v in 0..8 { + for u in 0..8 { + acc += alpha(u) + * alpha(v) + * f64::from(coeffs[v * 8 + u]) + * cos_term(x, u) + * cos_term(y, v); + } + } + acc + } + + /// The exact IDCT pixels with JPEG's +128 level shift and `[0, 255]` clamp. + fn exact_islow_reference(coeffs: &[i16; 64]) -> [u8; 64] { + let mut out = [0u8; 64]; + for y in 0..8 { + for x in 0..8 { + let value = exact_idct_pixel(coeffs, x, y) + 128.0; + out[y * 8 + x] = value.round().clamp(0.0, 255.0) as u8; + } + } + out + } + + fn next_coeff(state: &mut u64) -> i16 { + *state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + // Modest dequantized magnitudes so most pixels stay inside 8-bit range. + ((*state >> 40) & 0x1ff) as i32 as i16 - 256 + } + + #[test] + fn islow_matches_exact_idct_within_one_lsb() { + let mut state = 0x0bad_c0de_1234_5678u64; + for _ in 0..256 { + let mut coeffs = [0i16; 64]; + for c in &mut coeffs { + *c = next_coeff(&mut state); + } + // Keep DC moderate so the field is centered in range. + coeffs[0] = i16::try_from(i32::from(coeffs[0]) / 4).unwrap(); + + let mut got = [0u8; 64]; + idct_islow(&coeffs, &mut got); + let want = exact_islow_reference(&coeffs); + for i in 0..64 { + assert!( + (i32::from(got[i]) - i32::from(want[i])).abs() <= 1, + "pixel {i}: islow={} exact={} (coeffs {coeffs:?})", + got[i], + want[i] + ); + } + } + } + + #[test] + fn islow_dc_only_matches_closed_form() { + // DC-only block: every pixel is F(0,0)/8 + 128. + for dc in [-512i16, -200, -64, 8, 64, 200, 512] { + let mut coeffs = [0i16; 64]; + coeffs[0] = dc; + let mut got = [0u8; 64]; + idct_islow(&coeffs, &mut got); + let expected = ((f64::from(dc) / 8.0) + 128.0).round().clamp(0.0, 255.0) as u8; + for &px in &got { + assert!( + (i32::from(px) - i32::from(expected)).abs() <= 1, + "dc={dc}: px={px} expected={expected}" + ); + } + } + } + #[test] fn dc_only_helper_matches_full_idct() { let mut input = [0i16; 64]; diff --git a/crates/signinum-jpeg/src/info.rs b/crates/j2k-jpeg/src/info.rs similarity index 62% rename from crates/signinum-jpeg/src/info.rs rename to crates/j2k-jpeg/src/info.rs index c4864c97..6abc4cce 100644 --- a/crates/signinum-jpeg/src/info.rs +++ b/crates/j2k-jpeg/src/info.rs @@ -29,10 +29,15 @@ pub enum SofKind { /// Declared input color space after APP14 detection. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ColorSpace { + /// Single-component grayscale. Grayscale, + /// Three-component luma/chroma JPEG data. YCbCr, + /// Three-component RGB JPEG data. Rgb, + /// Four-component CMYK JPEG data. Cmyk, + /// Four-component YCCK JPEG data. Ycck, } @@ -47,12 +52,64 @@ pub struct SamplingFactors { pub max_v: u8, } +/// Error returned when constructing [`SamplingFactors`] from caller input. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum SamplingFactorsError { + /// At least one component is required. + #[error("sampling metadata must contain at least one component")] + Empty, + /// This crate stores at most four component sampling entries. + #[error("sampling metadata supports at most four components, got {count}")] + TooManyComponents { + /// Supplied component count. + count: usize, + }, + /// Component sampling factors are outside the JPEG legal range. + #[error("invalid sampling ({h}x{v}) for component {component}")] + InvalidSampling { + /// Component index in declaration order. + component: usize, + /// Horizontal sampling factor. + h: u8, + /// Vertical sampling factor. + v: u8, + }, +} + impl SamplingFactors { - pub fn from_components(components: &[(u8, u8)]) -> Self { - assert!( - components.len() <= 4, - "sampling metadata supports at most four components" - ); + /// Build sampling metadata from component `(H, V)` factors. + /// + /// # Errors + /// Returns [`SamplingFactorsError`] when no components are supplied, more + /// than four components are supplied, or any sampling factor is outside + /// the JPEG legal range `1..=4`. + pub fn from_components(components: &[(u8, u8)]) -> Result { + if components.is_empty() { + return Err(SamplingFactorsError::Empty); + } + if components.len() > 4 { + return Err(SamplingFactorsError::TooManyComponents { + count: components.len(), + }); + } + for (idx, &(h, v)) in components.iter().enumerate() { + if !(1..=4).contains(&h) || !(1..=4).contains(&v) { + return Err(SamplingFactorsError::InvalidSampling { + component: idx, + h, + v, + }); + } + } + Ok(Self::from_validated_components(components)) + } + + pub(crate) fn from_validated_components(components: &[(u8, u8)]) -> Self { + debug_assert!(!components.is_empty()); + debug_assert!(components.len() <= 4); + debug_assert!(components + .iter() + .all(|&(h, v)| (1..=4).contains(&h) && (1..=4).contains(&v))); let mut packed = [(0u8, 0u8); 4]; let mut max_h = 0u8; let mut max_v = 0u8; @@ -69,18 +126,22 @@ impl SamplingFactors { } } + /// Number of declared components. pub fn len(&self) -> usize { self.component_count as usize } + /// True when no components were declared. pub fn is_empty(&self) -> bool { self.component_count == 0 } + /// Sampling factors for a component by declaration index. pub fn component(&self, index: usize) -> Option<(u8, u8)> { self.components().get(index).copied() } + /// Sampling factors in component declaration order. pub fn components(&self) -> &[(u8, u8)] { &self.components[..self.component_count as usize] } @@ -148,9 +209,13 @@ impl McuGeometry { /// Inclusive axis-aligned rectangle in image coordinates. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Rect { + /// Left coordinate in pixels. pub x: u32, + /// Top coordinate in pixels. pub y: u32, + /// Width in pixels. pub w: u32, + /// Height in pixels. pub h: u32, } @@ -173,8 +238,8 @@ impl Rect { } } -impl From for Rect { - fn from(rect: signinum_core::Rect) -> Self { +impl From for Rect { + fn from(rect: j2k_core::Rect) -> Self { Self { x: rect.x, y: rect.y, @@ -184,7 +249,7 @@ impl From for Rect { } } -impl From for signinum_core::Rect { +impl From for j2k_core::Rect { fn from(rect: Rect) -> Self { Self { x: rect.x, @@ -202,23 +267,43 @@ pub(crate) enum OutputFormat { Rgb8, Rgb8Scaled { factor: DownscaleFactor }, Rgba8 { alpha: u8 }, + Rgba8Scaled { alpha: u8, factor: DownscaleFactor }, Gray8, Gray8Scaled { factor: DownscaleFactor }, + Gray16, + Gray16Scaled { factor: DownscaleFactor }, + Rgb16, + Rgb16Scaled { factor: DownscaleFactor }, + Rgba16 { alpha: u16 }, + Rgba16Scaled { alpha: u16, factor: DownscaleFactor }, } impl OutputFormat { pub(crate) fn bytes_per_pixel(self) -> usize { match self { Self::Rgb8 | Self::Rgb8Scaled { .. } => 3, - Self::Rgba8 { .. } => 4, + Self::Rgba8 { .. } | Self::Rgba8Scaled { .. } => 4, Self::Gray8 | Self::Gray8Scaled { .. } => 1, + Self::Gray16 | Self::Gray16Scaled { .. } => 2, + Self::Rgb16 | Self::Rgb16Scaled { .. } => 6, + Self::Rgba16 { .. } | Self::Rgba16Scaled { .. } => 8, } } pub(crate) fn downscale(self) -> DownscaleFactor { match self { - Self::Rgb8 | Self::Rgba8 { .. } | Self::Gray8 => DownscaleFactor::Full, - Self::Rgb8Scaled { factor } | Self::Gray8Scaled { factor } => factor, + Self::Rgb8 + | Self::Rgba8 { .. } + | Self::Gray8 + | Self::Gray16 + | Self::Rgb16 + | Self::Rgba16 { .. } => DownscaleFactor::Full, + Self::Rgb8Scaled { factor } + | Self::Rgba8Scaled { factor, .. } + | Self::Gray8Scaled { factor } + | Self::Gray16Scaled { factor } + | Self::Rgb16Scaled { factor } + | Self::Rgba16Scaled { factor, .. } => factor, } } } @@ -256,8 +341,11 @@ impl DownscaleFactor { /// Override for APP14 color-transform detection. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ColorTransform { + /// Detect the transform from APP14 metadata and component layout. Auto, + /// Treat three-component data as RGB regardless of APP14 metadata. ForceRgb, + /// Treat three-component data as YCbCr regardless of APP14 metadata. ForceYCbCr, } @@ -309,25 +397,34 @@ impl DecodeOptions { /// count of refinement passes. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Info { + /// Image dimensions as `(width, height)` in pixels. pub dimensions: (u32, u32), + /// Header-derived color space after APP14 transform handling. pub color_space: ColorSpace, + /// Per-component sampling factors from the SOF marker. pub sampling: SamplingFactors, + /// Start-of-frame variant that selects the decode pipeline. pub sof_kind: SofKind, + /// Sample precision in bits. pub bit_depth: u8, + /// Restart interval in MCUs, if a DRI marker was present. pub restart_interval: Option, + /// Derived MCU geometry for the image. pub mcu_geometry: McuGeometry, + /// Number of SOS markers observed in the stream. pub scan_count: u16, } impl Info { - pub fn to_core_info(&self) -> signinum_core::Info { - signinum_core::Info { + /// Convert JPEG metadata into the codec-neutral core metadata type. + pub fn to_core_info(&self) -> j2k_core::Info { + j2k_core::Info { dimensions: self.dimensions, components: self.sampling.len() as u8, colorspace: core_colorspace(self.color_space), bit_depth: self.bit_depth, tile_layout: None, - coded_unit_layout: Some(signinum_core::CodedUnitLayout { + coded_unit_layout: Some(j2k_core::CodedUnitLayout { unit_width: self.mcu_geometry.width, unit_height: self.mcu_geometry.height, units_x: self.mcu_geometry.columns, @@ -339,13 +436,13 @@ impl Info { } } -fn core_colorspace(color_space: ColorSpace) -> signinum_core::Colorspace { +fn core_colorspace(color_space: ColorSpace) -> j2k_core::Colorspace { match color_space { - ColorSpace::Grayscale => signinum_core::Colorspace::Grayscale, - ColorSpace::YCbCr => signinum_core::Colorspace::YCbCr, - ColorSpace::Rgb => signinum_core::Colorspace::Rgb, - ColorSpace::Cmyk => signinum_core::Colorspace::Cmyk, - ColorSpace::Ycck => signinum_core::Colorspace::Ycck, + ColorSpace::Grayscale => j2k_core::Colorspace::Grayscale, + ColorSpace::YCbCr => j2k_core::Colorspace::YCbCr, + ColorSpace::Rgb => j2k_core::Colorspace::Rgb, + ColorSpace::Cmyk => j2k_core::Colorspace::Cmyk, + ColorSpace::Ycck => j2k_core::Colorspace::Ycck, } } @@ -414,6 +511,14 @@ mod tests { 3 ); assert_eq!(OutputFormat::Rgba8 { alpha: 255 }.bytes_per_pixel(), 4); + assert_eq!( + OutputFormat::Rgba8Scaled { + alpha: 255, + factor: DownscaleFactor::Half, + } + .bytes_per_pixel(), + 4 + ); assert_eq!(OutputFormat::Gray8.bytes_per_pixel(), 1); assert_eq!( OutputFormat::Gray8Scaled { @@ -422,11 +527,40 @@ mod tests { .bytes_per_pixel(), 1 ); + assert_eq!(OutputFormat::Gray16.bytes_per_pixel(), 2); + assert_eq!( + OutputFormat::Gray16Scaled { + factor: DownscaleFactor::Half + } + .bytes_per_pixel(), + 2 + ); + assert_eq!(OutputFormat::Rgb16.bytes_per_pixel(), 6); + assert_eq!( + OutputFormat::Rgb16Scaled { + factor: DownscaleFactor::Half + } + .bytes_per_pixel(), + 6 + ); + assert_eq!( + OutputFormat::Rgba16 { alpha: u16::MAX }.bytes_per_pixel(), + 8 + ); + assert_eq!( + OutputFormat::Rgba16Scaled { + alpha: u16::MAX, + factor: DownscaleFactor::Half + } + .bytes_per_pixel(), + 8 + ); } #[test] fn sampling_factors_store_components_without_heap_state() { - let sampling = SamplingFactors::from_components(&[(2, 2), (1, 1), (1, 1)]); + let sampling = + SamplingFactors::from_components(&[(2, 2), (1, 1), (1, 1)]).expect("sampling"); assert_eq!(sampling.len(), 3); assert_eq!(sampling.component(0), Some((2, 2))); assert_eq!(sampling.component(1), Some((1, 1))); @@ -436,12 +570,62 @@ mod tests { assert_eq!(sampling.max_v, 2); } + #[test] + fn sampling_factors_reject_empty_component_list() { + assert!(matches!( + SamplingFactors::from_components(&[]), + Err(SamplingFactorsError::Empty) + )); + } + + #[test] + fn sampling_factors_accept_supported_component_counts() { + for components in [ + &[(1, 1)][..], + &[(2, 2), (1, 1), (1, 1)][..], + &[(1, 1), (1, 1), (1, 1), (1, 1)][..], + ] { + let sampling = SamplingFactors::from_components(components).expect("sampling"); + assert_eq!(sampling.len(), components.len()); + assert_eq!(sampling.components(), components); + } + } + + #[test] + fn sampling_factors_reject_invalid_factors() { + assert!(matches!( + SamplingFactors::from_components(&[(0, 1)]), + Err(SamplingFactorsError::InvalidSampling { + component: 0, + h: 0, + v: 1 + }) + )); + assert!(matches!( + SamplingFactors::from_components(&[(1, 5)]), + Err(SamplingFactorsError::InvalidSampling { + component: 0, + h: 1, + v: 5 + }) + )); + } + + #[test] + fn sampling_factors_reject_more_than_four_components_without_panic() { + assert!(matches!( + SamplingFactors::from_components(&[(1, 1); 5]), + Err(SamplingFactorsError::TooManyComponents { count: 5 }) + )); + } + #[test] fn info_to_core_info_preserves_metadata_for_device_adapters() { let info = Info { dimensions: (32, 16), color_space: ColorSpace::YCbCr, - sampling: SamplingFactors::from_components(&[(2, 2), (1, 1), (1, 1)]), + sampling: SamplingFactors::from_components(&[(2, 2), (1, 1), (1, 1)]) + .expect("sampling"), sof_kind: SofKind::Baseline8, bit_depth: 8, restart_interval: Some(2), @@ -459,12 +643,12 @@ mod tests { assert_eq!(core.dimensions, (32, 16)); assert_eq!(core.components, 3); - assert_eq!(core.colorspace, signinum_core::Colorspace::YCbCr); + assert_eq!(core.colorspace, j2k_core::Colorspace::YCbCr); assert_eq!(core.bit_depth, 8); assert_eq!(core.tile_layout, None); assert_eq!( core.coded_unit_layout, - Some(signinum_core::CodedUnitLayout { + Some(j2k_core::CodedUnitLayout { unit_width: 16, unit_height: 16, units_x: 2, @@ -478,13 +662,14 @@ mod tests { #[test] fn info_to_core_info_preserves_four_component_colorspaces() { for (color_space, core_colorspace) in [ - (ColorSpace::Cmyk, signinum_core::Colorspace::Cmyk), - (ColorSpace::Ycck, signinum_core::Colorspace::Ycck), + (ColorSpace::Cmyk, j2k_core::Colorspace::Cmyk), + (ColorSpace::Ycck, j2k_core::Colorspace::Ycck), ] { let info = Info { dimensions: (64, 32), color_space, - sampling: SamplingFactors::from_components(&[(1, 1), (1, 1), (1, 1), (1, 1)]), + sampling: SamplingFactors::from_components(&[(1, 1), (1, 1), (1, 1), (1, 1)]) + .expect("sampling"), sof_kind: SofKind::Baseline8, bit_depth: 8, restart_interval: None, @@ -504,7 +689,7 @@ mod tests { assert_eq!(core.colorspace, core_colorspace); assert_eq!( core.coded_unit_layout, - Some(signinum_core::CodedUnitLayout { + Some(j2k_core::CodedUnitLayout { unit_width: 8, unit_height: 8, units_x: 8, diff --git a/crates/signinum-jpeg/src/internal/bit_reader.rs b/crates/j2k-jpeg/src/internal/bit_reader.rs similarity index 100% rename from crates/signinum-jpeg/src/internal/bit_reader.rs rename to crates/j2k-jpeg/src/internal/bit_reader.rs diff --git a/crates/signinum-jpeg/src/internal/checkpoint.rs b/crates/j2k-jpeg/src/internal/checkpoint.rs similarity index 97% rename from crates/signinum-jpeg/src/internal/checkpoint.rs rename to crates/j2k-jpeg/src/internal/checkpoint.rs index e18fe68a..63f55086 100644 --- a/crates/signinum-jpeg/src/internal/checkpoint.rs +++ b/crates/j2k-jpeg/src/internal/checkpoint.rs @@ -11,12 +11,19 @@ pub(crate) struct CpuCheckpointCache { } #[derive(Debug, Clone, PartialEq, Eq)] +/// Entropy decoder resume point for device-side JPEG decode. pub struct DeviceCheckpoint { + /// MCU index where decoding can resume. pub mcu_index: u32, + /// Byte offset into scan entropy data. pub scan_offset: usize, + /// Buffered entropy bits at the checkpoint. pub bit_accumulator: u64, + /// Number of valid bits in `bit_accumulator`. pub bits_buffered: u8, + /// Previous DC predictor per component. pub prev_dc: [i32; 4], + /// Next expected restart marker index. pub expected_rst: u8, } diff --git a/crates/signinum-jpeg/src/internal/mod.rs b/crates/j2k-jpeg/src/internal/mod.rs similarity index 100% rename from crates/signinum-jpeg/src/internal/mod.rs rename to crates/j2k-jpeg/src/internal/mod.rs diff --git a/crates/signinum-jpeg/src/internal/scratch.rs b/crates/j2k-jpeg/src/internal/scratch.rs similarity index 92% rename from crates/signinum-jpeg/src/internal/scratch.rs rename to crates/j2k-jpeg/src/internal/scratch.rs index 20867356..53c72ca1 100644 --- a/crates/signinum-jpeg/src/internal/scratch.rs +++ b/crates/j2k-jpeg/src/internal/scratch.rs @@ -5,7 +5,7 @@ //! A [`ScratchPool`] owns every `Vec` that the sequential scan decoder //! would otherwise allocate on each call: the three rolling MCU stripe //! buffers, the per-component DC predictor, the chroma upsample rows, and -//! the RGB row buffers used by [`signinum_core::RowSink`] drivers. +//! the RGB row buffers used by [`j2k_core::RowSink`] drivers. //! //! Use [`Decoder::decode_into_with_scratch`](crate::Decoder::decode_into_with_scratch) //! / [`decode_rows_with_scratch`](crate::Decoder::decode_rows_with_scratch) @@ -14,7 +14,7 @@ use crate::entropy::sequential::{PreparedDecodePlan, StripeBuffer}; use alloc::vec::Vec; -use signinum_core::ScratchPool as CoreScratchPool; +use j2k_core::ScratchPool as CoreScratchPool; #[derive(Debug, Default)] pub(crate) struct YCbCr420Rows { @@ -51,6 +51,7 @@ pub(crate) struct RgbGenericRows { pub(crate) r: Vec, pub(crate) g: Vec, pub(crate) b: Vec, + pub(crate) k: Vec, } impl RgbGenericRows { @@ -58,6 +59,7 @@ impl RgbGenericRows { self.r.resize(width, 0); self.g.resize(width, 0); self.b.resize(width, 0); + self.k.resize(width, 0); } } @@ -88,6 +90,8 @@ pub struct ScratchPool { pub(crate) ycbcr_420_rows: YCbCr420Rows, pub(crate) ycbcr_generic_rows: YCbCrGenericRows, pub(crate) rgb_generic_rows: RgbGenericRows, + pub(crate) lossless_prev_row: Vec, + pub(crate) lossless_curr_row: Vec, sink_rows: SinkRows, } @@ -168,6 +172,9 @@ impl CoreScratchPool for ScratchPool { .saturating_add(vec_bytes(&self.rgb_generic_rows.r)) .saturating_add(vec_bytes(&self.rgb_generic_rows.g)) .saturating_add(vec_bytes(&self.rgb_generic_rows.b)) + .saturating_add(vec_bytes(&self.rgb_generic_rows.k)) + .saturating_add(vec_bytes(&self.lossless_prev_row)) + .saturating_add(vec_bytes(&self.lossless_curr_row)) .saturating_add(vec_bytes(&self.sink_rows.top_row)) .saturating_add(vec_bytes(&self.sink_rows.bottom_row)); total @@ -195,6 +202,9 @@ impl CoreScratchPool for ScratchPool { self.rgb_generic_rows.r.clear(); self.rgb_generic_rows.g.clear(); self.rgb_generic_rows.b.clear(); + self.rgb_generic_rows.k.clear(); + self.lossless_prev_row.clear(); + self.lossless_curr_row.clear(); self.sink_rows.top_row.clear(); self.sink_rows.bottom_row.clear(); } diff --git a/crates/signinum-jpeg/src/lib.rs b/crates/j2k-jpeg/src/lib.rs similarity index 59% rename from crates/signinum-jpeg/src/lib.rs rename to crates/j2k-jpeg/src/lib.rs index f984f242..faa06afa 100644 --- a/crates/signinum-jpeg/src/lib.rs +++ b/crates/j2k-jpeg/src/lib.rs @@ -10,16 +10,16 @@ // `missing_docs` remains staged crate-by-crate; see Cargo.toml for rationale. #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] -compile_error!("signinum-jpeg currently supports only x86_64 and aarch64 targets"); +compile_error!("j2k-jpeg currently supports only x86_64 and aarch64 targets"); extern crate alloc; pub mod info; pub use info::{ ColorSpace, ColorTransform, DecodeOptions, Info, McuGeometry, Rect, RestartIndex, - RestartSegment, SamplingFactors, SofKind, + RestartSegment, SamplingFactors, SamplingFactorsError, SofKind, }; -pub use signinum_core::{ +pub use j2k_core::{ CacheStats, CodecContext, CompressedPayloadKind, CompressedTransferSyntax, DecodeRowsError, Downscale, ImageCodec, ImageDecode, ImageDecodeRows, PassthroughCandidate, PassthroughDecision, PassthroughRejectReason, PassthroughRequirements, PixelFormat, PixelLayout, RowSink, Sample, @@ -29,6 +29,25 @@ pub use signinum_core::{ pub mod context; pub use context::DecoderContext; +pub mod batch_session; +pub use batch_session::JpegBatchSession; + +pub mod capabilities; +pub use capabilities::{ + JpegBackendEligibility, JpegCapabilityReport, JpegCapabilityRequest, JpegDecodeOp, + JpegDecodeRequest, JpegResolvedDecode, JpegResolvedDecodePath, +}; + +pub mod output_buffer; +pub use output_buffer::JpegOutputBuffer; + +pub mod segment; +pub use segment::{ + find_scan_ranges, is_sof_marker, iter_segments, parse_dri, parse_sof_info, + prepare_tiff_jpeg_tile, rewrite_sof_dimensions, DuplicateTablePolicy, JpegScanRanges, + JpegSegment, JpegSegmentIter, JpegSofInfo, JpegTilePrepareOptions, PreparedJpeg, +}; + pub mod adapter; pub mod error; @@ -43,6 +62,8 @@ pub(crate) mod entropy; pub(crate) mod idct; +pub(crate) mod lossless; + pub(crate) mod internal; pub(crate) mod color; @@ -53,30 +74,35 @@ pub(crate) mod output; pub(crate) mod profile; +/// Baseline JPEG encoder API. pub mod encoder; pub use encoder::{ encode_jpeg_baseline, EncodedJpeg, JpegBackend, JpegEncodeError, JpegEncodeOptions, JpegSamples, JpegSubsampling, }; +pub mod transcode; + pub mod decoder; pub use decoder::{ - decode_tile_into, decode_tile_into_in_context, decode_tile_into_in_context_with_options, - decode_tile_region_into_in_context, decode_tile_region_into_in_context_with_options, - decode_tile_region_scaled_into_in_context, + decode_prepared_jpeg_tiles_rgb8, decode_tile_into, decode_tile_into_in_context, + decode_tile_into_in_context_with_options, decode_tile_region_into_in_context, + decode_tile_region_into_in_context_with_options, decode_tile_region_scaled_into_in_context, decode_tile_region_scaled_into_in_context_with_options, decode_tile_scaled_into_in_context, decode_tile_scaled_into_in_context_with_options, decode_tiles_into, decode_tiles_into_with_options, decode_tiles_region_scaled_into, decode_tiles_region_scaled_into_with_options, decode_tiles_scaled_into, - decode_tiles_scaled_into_with_options, ComponentRowWriter, DecodeOutcome, Decoder, JpegView, - TileBatchError, TileBatchOptions, TileDecodeJob, TileRegionScaledDecodeJob, - TileScaledDecodeJob, + decode_tiles_scaled_into_with_options, ComponentRowWriter, DecodeOutcome, DecodedTile, Decoder, + JpegView, PreparedJpegTileJob, TileBatchError, TileBatchOptions, TileDecodeJob, + TileRegionScaledDecodeJob, TileScaledDecodeJob, }; pub use internal::scratch::ScratchPool; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +/// JPEG codec marker type for `j2k-core` trait integrations. pub struct JpegCodec; +#[cfg(feature = "bench-internals")] #[doc(hidden)] pub mod bench_support; diff --git a/crates/j2k-jpeg/src/lossless/mod.rs b/crates/j2k-jpeg/src/lossless/mod.rs new file mode 100644 index 00000000..8a3cd2b4 --- /dev/null +++ b/crates/j2k-jpeg/src/lossless/mod.rs @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Shared lossless JPEG decode helpers. + +use crate::error::{HuffmanFailure, JpegError}; + +pub(crate) trait LosslessSample: Copy + Default + Into { + const RESTART_PREDICTOR: i32; + const BIT_DEPTH: u8; + const BYTES: usize; + + fn from_i32(value: i32) -> Result; + + fn read_le(src: &[u8]) -> i32; + + fn write_le(self, dst: &mut [u8]); +} + +impl LosslessSample for u8 { + const RESTART_PREDICTOR: i32 = 128; + const BIT_DEPTH: u8 = 8; + const BYTES: usize = 1; + + fn from_i32(value: i32) -> Result { + u8::try_from(value).map_err(|_| invalid_lossless_symbol()) + } + + fn read_le(src: &[u8]) -> i32 { + i32::from(src[0]) + } + + fn write_le(self, dst: &mut [u8]) { + dst[0] = self; + } +} + +impl LosslessSample for u16 { + const RESTART_PREDICTOR: i32 = 32_768; + const BIT_DEPTH: u8 = 16; + const BYTES: usize = 2; + + fn from_i32(value: i32) -> Result { + u16::try_from(value).map_err(|_| invalid_lossless_symbol()) + } + + fn read_le(src: &[u8]) -> i32 { + i32::from(u16::from_le_bytes([src[0], src[1]])) + } + + fn write_le(self, dst: &mut [u8]) { + dst[..2].copy_from_slice(&self.to_le_bytes()); + } +} + +fn invalid_lossless_symbol() -> JpegError { + JpegError::HuffmanDecode { + mcu: 0, + reason: HuffmanFailure::InvalidSymbol, + } +} + +/// Spec predictor (ITU-T T.81 H.1.2.1) over a sample accessor. +/// +/// Edge rules: the first sample predicts `bias` (1 << (P - 1)); the first row +/// predicts Ra; the first column predicts Rb. `at(x, y)` must return the +/// reconstructed sample at the given coordinates; only `(x-1, y)`, `(x, y-1)` +/// and `(x-1, y-1)` are ever requested. +#[allow(clippy::inline_always)] // per-sample hot path: keep the accessor closure monomorphized +#[inline(always)] +pub(crate) fn lossless_predict( + predictor: u8, + bias: i32, + x: usize, + y: usize, + at: impl Fn(usize, usize) -> i32, +) -> i32 { + if x == 0 && y == 0 { + return bias; + } + if y == 0 { + return at(x - 1, 0); + } + if x == 0 { + return at(0, y - 1); + } + let ra = at(x - 1, y); + let rb = at(x, y - 1); + let rc = at(x - 1, y - 1); + match predictor { + 1 => ra, + 2 => rb, + 3 => rc, + 4 => ra + rb - rc, + 5 => ra + ((rb - rc) >> 1), + 6 => rb + ((ra - rc) >> 1), + 7 => (ra + rb) >> 1, + _ => bias, + } +} diff --git a/crates/signinum-jpeg/src/output/gray8.rs b/crates/j2k-jpeg/src/output/gray8.rs similarity index 100% rename from crates/signinum-jpeg/src/output/gray8.rs rename to crates/j2k-jpeg/src/output/gray8.rs diff --git a/crates/signinum-jpeg/src/output/mod.rs b/crates/j2k-jpeg/src/output/mod.rs similarity index 62% rename from crates/signinum-jpeg/src/output/mod.rs rename to crates/j2k-jpeg/src/output/mod.rs index a6c46f45..1b475699 100644 --- a/crates/signinum-jpeg/src/output/mod.rs +++ b/crates/j2k-jpeg/src/output/mod.rs @@ -5,7 +5,7 @@ //! each call site so there is no dynamic dispatch on the per-pixel hot path. use crate::error::JpegError; -use signinum_core::{validate_strided_output_buffer, BufferError, PixelFormat}; +use j2k_core::{validate_strided_output_buffer, BufferError, PixelFormat}; pub(crate) mod gray8; pub(crate) mod rgb8; @@ -57,39 +57,54 @@ pub(crate) fn validate_buffer( image_height: u32, bytes_per_pixel: usize, ) -> Result<(), JpegError> { - let fmt = match bytes_per_pixel { - 1 => PixelFormat::Gray8, - 3 => PixelFormat::Rgb8, - 4 => PixelFormat::Rgba8, - _ => { - return Err(JpegError::OutputBufferTooSmall { - required: usize::MAX, - provided: out.len(), - }) - } + let Some(fmt) = pixel_format_for_bytes_per_pixel(bytes_per_pixel) else { + return Err(JpegError::OutputBufferTooSmall { + required: usize::MAX, + provided: out.len(), + }); }; - validate_strided_output_buffer((image_width, image_height), out.len(), stride, fmt).map_err( - |err| match err { - BufferError::StrideTooSmall { row_bytes, stride } => JpegError::InvalidStride { - stride, - row: row_bytes, - }, - BufferError::OutputTooSmall { required, have } => JpegError::OutputBufferTooSmall { - required, - provided: have, - }, - BufferError::SizeOverflow { .. } => JpegError::OutputBufferTooSmall { - required: usize::MAX, - provided: out.len(), - }, - BufferError::InputTooSmall { .. } - | BufferError::StrideNotAligned { .. } - | BufferError::SampleTypeMismatch { .. } => JpegError::OutputBufferTooSmall { - required: usize::MAX, - provided: out.len(), - }, + validate_strided_output_buffer((image_width, image_height), out.len(), stride, fmt) + .map_err(|err| jpeg_buffer_error(err, out.len())) +} + +pub(crate) const fn pixel_format_for_bytes_per_pixel( + bytes_per_pixel: usize, +) -> Option { + match bytes_per_pixel { + 1 => Some(PixelFormat::Gray8), + 2 => Some(PixelFormat::Gray16), + 3 => Some(PixelFormat::Rgb8), + 4 => Some(PixelFormat::Rgba8), + 6 => Some(PixelFormat::Rgb16), + 8 => Some(PixelFormat::Rgba16), + _ => None, + } +} + +pub(crate) fn jpeg_buffer_error(error: BufferError, provided_len: usize) -> JpegError { + match error { + BufferError::StrideTooSmall { row_bytes, stride } => JpegError::InvalidStride { + stride, + row: row_bytes, + }, + BufferError::OutputTooSmall { required, have } => JpegError::OutputBufferTooSmall { + required, + provided: have, + }, + BufferError::AllocationTooLarge { requested, cap, .. } => { + JpegError::MemoryCapExceeded { requested, cap } + } + BufferError::SizeOverflow { .. } => JpegError::OutputBufferTooSmall { + required: usize::MAX, + provided: provided_len, + }, + BufferError::InputTooSmall { .. } + | BufferError::StrideNotAligned { .. } + | BufferError::SampleTypeMismatch { .. } => JpegError::OutputBufferTooSmall { + required: usize::MAX, + provided: provided_len, }, - ) + } } #[cfg(test)] diff --git a/crates/signinum-jpeg/src/output/rgb8.rs b/crates/j2k-jpeg/src/output/rgb8.rs similarity index 82% rename from crates/signinum-jpeg/src/output/rgb8.rs rename to crates/j2k-jpeg/src/output/rgb8.rs index 507a3378..3d2cae94 100644 --- a/crates/signinum-jpeg/src/output/rgb8.rs +++ b/crates/j2k-jpeg/src/output/rgb8.rs @@ -2,7 +2,7 @@ //! `Rgb8Writer` — 3-byte-per-pixel RGB output. -use crate::color::ycbcr::ycbcr_to_rgb; +use crate::backend::Backend; use crate::error::JpegError; use crate::output::{InterleavedRgbWriter, OutputWriter}; @@ -10,11 +10,27 @@ pub(crate) struct Rgb8Writer<'o> { out: &'o mut [u8], stride: usize, width: u32, + backend: Backend, } impl<'o> Rgb8Writer<'o> { + #[cfg(test)] pub(crate) fn new(out: &'o mut [u8], stride: usize, width: u32) -> Self { - Self { out, stride, width } + Self::new_with_backend(out, stride, width, Backend::detect()) + } + + pub(crate) fn new_with_backend( + out: &'o mut [u8], + stride: usize, + width: u32, + backend: Backend, + ) -> Self { + Self { + out, + stride, + width, + backend, + } } fn row_mut(&mut self, y: u32) -> &mut [u8] { @@ -60,16 +76,7 @@ impl OutputWriter for Rgb8Writer<'_> { let dst_start = (y as usize) * self.stride; let width = self.width as usize; let dst = &mut self.out[dst_start..dst_start + width * 3]; - for (((&r, &g), &b), pixel) in r_row - .iter() - .zip(g_row.iter()) - .zip(b_row.iter()) - .zip(dst.chunks_exact_mut(3)) - { - pixel[0] = r; - pixel[1] = g; - pixel[2] = b; - } + self.backend.fill_rgb_row_from_rgb(r_row, g_row, b_row, dst); Ok(()) } @@ -83,17 +90,8 @@ impl OutputWriter for Rgb8Writer<'_> { let dst_start = (y as usize) * self.stride; let width = self.width as usize; let dst = &mut self.out[dst_start..dst_start + width * 3]; - for (((&y_sample, &cb_sample), &cr_sample), pixel) in y_row - .iter() - .zip(cb_row.iter()) - .zip(cr_row.iter()) - .zip(dst.chunks_exact_mut(3)) - { - let (r, g, b) = ycbcr_to_rgb(y_sample, cb_sample, cr_sample); - pixel[0] = r; - pixel[1] = g; - pixel[2] = b; - } + self.backend + .fill_rgb_row_from_ycbcr(y_row, cb_row, cr_row, dst); Ok(()) } @@ -101,11 +99,7 @@ impl OutputWriter for Rgb8Writer<'_> { let dst_start = (y as usize) * self.stride; let width = self.width as usize; let dst = &mut self.out[dst_start..dst_start + width * 3]; - for (&gray, pixel) in gray_row.iter().zip(dst.chunks_exact_mut(3)) { - pixel[0] = gray; - pixel[1] = gray; - pixel[2] = gray; - } + self.backend.fill_rgb_row_from_gray(gray_row, dst); Ok(()) } } diff --git a/crates/signinum-jpeg/src/output/rgba8.rs b/crates/j2k-jpeg/src/output/rgba8.rs similarity index 76% rename from crates/signinum-jpeg/src/output/rgba8.rs rename to crates/j2k-jpeg/src/output/rgba8.rs index 39c03185..1f385ec0 100644 --- a/crates/signinum-jpeg/src/output/rgba8.rs +++ b/crates/j2k-jpeg/src/output/rgba8.rs @@ -3,7 +3,7 @@ //! `Rgba8Writer` — 4-byte-per-pixel RGBA output. Alpha is a constant captured //! at construction time and written verbatim into every pixel's A channel. -use crate::color::ycbcr::ycbcr_to_rgb; +use crate::backend::Backend; use crate::error::JpegError; use crate::output::OutputWriter; @@ -12,15 +12,28 @@ pub(crate) struct Rgba8Writer<'o> { stride: usize, width: u32, alpha: u8, + backend: Backend, } impl<'o> Rgba8Writer<'o> { + #[cfg(test)] pub(crate) fn new(out: &'o mut [u8], stride: usize, width: u32, alpha: u8) -> Self { + Self::new_with_backend(out, stride, width, alpha, Backend::detect()) + } + + pub(crate) fn new_with_backend( + out: &'o mut [u8], + stride: usize, + width: u32, + alpha: u8, + backend: Backend, + ) -> Self { Self { out, stride, width, alpha, + backend, } } } @@ -36,13 +49,8 @@ impl OutputWriter for Rgba8Writer<'_> { let dst_start = (y as usize) * self.stride; let width = self.width as usize; let dst = &mut self.out[dst_start..dst_start + width * 4]; - let alpha = self.alpha; - for i in 0..width { - dst[i * 4] = r_row[i]; - dst[i * 4 + 1] = g_row[i]; - dst[i * 4 + 2] = b_row[i]; - dst[i * 4 + 3] = alpha; - } + self.backend + .fill_rgba_row_from_rgb(r_row, g_row, b_row, dst, self.alpha); Ok(()) } @@ -56,14 +64,8 @@ impl OutputWriter for Rgba8Writer<'_> { let dst_start = (y as usize) * self.stride; let width = self.width as usize; let dst = &mut self.out[dst_start..dst_start + width * 4]; - let alpha = self.alpha; - for i in 0..width { - let (r, g, b) = ycbcr_to_rgb(y_row[i], cb_row[i], cr_row[i]); - dst[i * 4] = r; - dst[i * 4 + 1] = g; - dst[i * 4 + 2] = b; - dst[i * 4 + 3] = alpha; - } + self.backend + .fill_rgba_row_from_ycbcr(y_row, cb_row, cr_row, dst, self.alpha); Ok(()) } @@ -71,13 +73,8 @@ impl OutputWriter for Rgba8Writer<'_> { let dst_start = (y as usize) * self.stride; let width = self.width as usize; let dst = &mut self.out[dst_start..dst_start + width * 4]; - let alpha = self.alpha; - for i in 0..width { - dst[i * 4] = gray_row[i]; - dst[i * 4 + 1] = gray_row[i]; - dst[i * 4 + 2] = gray_row[i]; - dst[i * 4 + 3] = alpha; - } + self.backend + .fill_rgba_row_from_gray(gray_row, dst, self.alpha); Ok(()) } } diff --git a/crates/j2k-jpeg/src/output_buffer.rs b/crates/j2k-jpeg/src/output_buffer.rs new file mode 100644 index 00000000..9c982bba --- /dev/null +++ b/crates/j2k-jpeg/src/output_buffer.rs @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Reusable host output buffers for JPEG tile decode. + +use alloc::vec::Vec; +use j2k_core::{ + strided_output_len_capped, validate_strided_output_buffer, BufferError, PixelFormat, + DEFAULT_MAX_HOST_ALLOCATION_BYTES, +}; + +/// Caller-owned reusable host pixel buffer. +/// +/// The buffer uses a tight stride by default and can be resized across viewport +/// reads. Resizing to a same-or-smaller byte requirement keeps existing vector +/// capacity, so callers can reuse allocations while still passing ordinary +/// `&mut [u8]` slices into decode APIs. +#[derive(Debug, Clone)] +pub struct JpegOutputBuffer { + bytes: Vec, + dimensions: (u32, u32), + stride: usize, + fmt: PixelFormat, +} + +impl JpegOutputBuffer { + /// Create a tightly packed output buffer for `dimensions` and `fmt`. + /// + /// Uses the shared default host allocation cap. + /// + /// # Errors + /// Returns [`BufferError`] if the requested shape overflows byte counts or + /// exceeds the default host allocation cap. + pub fn new(dimensions: (u32, u32), fmt: PixelFormat) -> Result { + Self::new_with_max_bytes(dimensions, fmt, DEFAULT_MAX_HOST_ALLOCATION_BYTES) + } + + /// Create a tightly packed output buffer with an explicit allocation cap. + /// + /// # Errors + /// Returns [`BufferError`] if the requested shape overflows byte counts or + /// exceeds `max_bytes`. + pub fn new_with_max_bytes( + dimensions: (u32, u32), + fmt: PixelFormat, + max_bytes: usize, + ) -> Result { + let stride = tight_stride(dimensions.0, fmt)?; + Self::with_stride_with_max_bytes(dimensions, stride, fmt, max_bytes) + } + + /// Create an output buffer with an explicit row stride. + /// + /// Uses the shared default host allocation cap. + /// + /// # Errors + /// Returns [`BufferError`] if the stride is too small, sizes overflow, or the + /// allocation exceeds the default host allocation cap. + pub fn with_stride( + dimensions: (u32, u32), + stride: usize, + fmt: PixelFormat, + ) -> Result { + Self::with_stride_with_max_bytes(dimensions, stride, fmt, DEFAULT_MAX_HOST_ALLOCATION_BYTES) + } + + /// Create an output buffer with explicit row stride and allocation cap. + /// + /// # Errors + /// Returns [`BufferError`] if the stride is too small, sizes overflow, or the + /// allocation exceeds `max_bytes`. + pub fn with_stride_with_max_bytes( + dimensions: (u32, u32), + stride: usize, + fmt: PixelFormat, + max_bytes: usize, + ) -> Result { + let len = + strided_output_len_capped(dimensions, stride, fmt, max_bytes, "JPEG output buffer")?; + validate_strided_output_buffer(dimensions, len, stride, fmt)?; + Ok(Self { + bytes: alloc::vec![0; len], + dimensions, + stride, + fmt, + }) + } + + /// Resize to a tightly packed output shape. + /// + /// Uses the shared default host allocation cap. + /// + /// # Errors + /// Returns [`BufferError`] if the requested shape overflows byte counts or + /// exceeds the default host allocation cap. + pub fn resize(&mut self, dimensions: (u32, u32), fmt: PixelFormat) -> Result<(), BufferError> { + self.resize_with_max_bytes(dimensions, fmt, DEFAULT_MAX_HOST_ALLOCATION_BYTES) + } + + /// Resize to a tightly packed output shape with an explicit allocation cap. + /// + /// # Errors + /// Returns [`BufferError`] if the requested shape overflows byte counts or + /// exceeds `max_bytes`. + pub fn resize_with_max_bytes( + &mut self, + dimensions: (u32, u32), + fmt: PixelFormat, + max_bytes: usize, + ) -> Result<(), BufferError> { + let stride = tight_stride(dimensions.0, fmt)?; + self.resize_with_stride_with_max_bytes(dimensions, stride, fmt, max_bytes) + } + + /// Resize with an explicit row stride. + /// + /// Uses the shared default host allocation cap. + /// + /// # Errors + /// Returns [`BufferError`] if the stride is too small, sizes overflow, or the + /// allocation exceeds the default host allocation cap. + pub fn resize_with_stride( + &mut self, + dimensions: (u32, u32), + stride: usize, + fmt: PixelFormat, + ) -> Result<(), BufferError> { + self.resize_with_stride_with_max_bytes( + dimensions, + stride, + fmt, + DEFAULT_MAX_HOST_ALLOCATION_BYTES, + ) + } + + /// Resize with explicit row stride and allocation cap. + /// + /// # Errors + /// Returns [`BufferError`] if the stride is too small, sizes overflow, or the + /// allocation exceeds `max_bytes`. + pub fn resize_with_stride_with_max_bytes( + &mut self, + dimensions: (u32, u32), + stride: usize, + fmt: PixelFormat, + max_bytes: usize, + ) -> Result<(), BufferError> { + let len = + strided_output_len_capped(dimensions, stride, fmt, max_bytes, "JPEG output buffer")?; + validate_strided_output_buffer(dimensions, len, stride, fmt)?; + self.bytes.resize(len, 0); + self.dimensions = dimensions; + self.stride = stride; + self.fmt = fmt; + Ok(()) + } + + /// Borrow the decoded pixel bytes. + #[must_use] + pub fn as_slice(&self) -> &[u8] { + &self.bytes + } + + /// Mutably borrow the decoded pixel bytes. + #[must_use] + pub fn as_mut_slice(&mut self) -> &mut [u8] { + &mut self.bytes + } + + /// Current dimensions in pixels. + #[must_use] + pub fn dimensions(&self) -> (u32, u32) { + self.dimensions + } + + /// Current row stride in bytes. + #[must_use] + pub fn stride(&self) -> usize { + self.stride + } + + /// Current pixel format. + #[must_use] + pub fn pixel_format(&self) -> PixelFormat { + self.fmt + } + + /// Current logical byte length. + #[must_use] + pub fn len(&self) -> usize { + self.bytes.len() + } + + /// Whether the logical byte length is zero. + #[must_use] + pub fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + /// Retained vector capacity in bytes. + #[must_use] + pub fn capacity(&self) -> usize { + self.bytes.capacity() + } +} + +fn tight_stride(width: u32, fmt: PixelFormat) -> Result { + (width as usize) + .checked_mul(fmt.bytes_per_pixel()) + .ok_or(BufferError::SizeOverflow { + what: "tight JPEG output stride", + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + const HUGE_DIMENSIONS: (u32, u32) = (65_500, 65_500); + + fn assert_allocation_too_large(error: BufferError) { + assert!( + matches!( + error, + BufferError::AllocationTooLarge { + requested, + cap: DEFAULT_MAX_HOST_ALLOCATION_BYTES, + what: "JPEG output buffer", + } if requested > DEFAULT_MAX_HOST_ALLOCATION_BYTES + ), + "expected AllocationTooLarge, got {error:?}" + ); + } + + #[test] + fn new_rejects_huge_output_before_allocation() { + let err = JpegOutputBuffer::new(HUGE_DIMENSIONS, PixelFormat::Rgba16) + .expect_err("huge output must be capped"); + assert_allocation_too_large(err); + } + + #[test] + fn with_stride_rejects_huge_output_before_allocation() { + let stride = HUGE_DIMENSIONS.0 as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let err = JpegOutputBuffer::with_stride(HUGE_DIMENSIONS, stride, PixelFormat::Rgba16) + .expect_err("huge output must be capped"); + assert_allocation_too_large(err); + } + + #[test] + fn resize_rejects_huge_output_before_allocation() { + let mut buffer = + JpegOutputBuffer::new((1, 1), PixelFormat::Rgba8).expect("small output buffer"); + let err = buffer + .resize(HUGE_DIMENSIONS, PixelFormat::Rgba16) + .expect_err("huge output must be capped"); + assert_allocation_too_large(err); + assert_eq!(buffer.dimensions(), (1, 1)); + } + + #[test] + fn resize_with_stride_rejects_huge_output_before_allocation() { + let mut buffer = + JpegOutputBuffer::new((1, 1), PixelFormat::Rgba8).expect("small output buffer"); + let stride = HUGE_DIMENSIONS.0 as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let err = buffer + .resize_with_stride(HUGE_DIMENSIONS, stride, PixelFormat::Rgba16) + .expect_err("huge output must be capped"); + assert_allocation_too_large(err); + assert_eq!(buffer.dimensions(), (1, 1)); + } + + #[test] + fn explicit_max_bytes_allows_callers_to_choose_smaller_caps() { + let err = JpegOutputBuffer::new_with_max_bytes((2, 2), PixelFormat::Rgba8, 15) + .expect_err("caller cap should be enforced"); + assert!(matches!( + err, + BufferError::AllocationTooLarge { + requested: 16, + cap: 15, + what: "JPEG output buffer", + } + )); + } +} diff --git a/crates/signinum-jpeg/src/parse/adobe_app14.rs b/crates/j2k-jpeg/src/parse/adobe_app14.rs similarity index 100% rename from crates/signinum-jpeg/src/parse/adobe_app14.rs rename to crates/j2k-jpeg/src/parse/adobe_app14.rs diff --git a/crates/signinum-jpeg/src/parse/header.rs b/crates/j2k-jpeg/src/parse/header.rs similarity index 97% rename from crates/signinum-jpeg/src/parse/header.rs rename to crates/j2k-jpeg/src/parse/header.rs index 34802dec..2a85c3ef 100644 --- a/crates/signinum-jpeg/src/parse/header.rs +++ b/crates/j2k-jpeg/src/parse/header.rs @@ -6,7 +6,7 @@ use crate::error::{JpegError, MarkerKind, Warning}; use crate::info::{ColorSpace, Info, McuGeometry, SamplingFactors, SofKind}; use crate::parse::adobe_app14::{parse_adobe_app14, AdobeTransform}; -use crate::parse::markers::MarkerWalker; +use crate::parse::markers::{next_marker_after_entropy, MarkerWalker}; use crate::parse::scan::{parse_scan_header, ParsedScan}; use crate::parse::sof::parse_sof; use crate::parse::tables::{parse_dht, parse_dqt, HuffmanTables, QuantTables}; @@ -535,26 +535,6 @@ fn collect_progressive_scans( Ok(scans) } -fn next_marker_after_entropy(bytes: &[u8], mut pos: usize) -> Option<(usize, u8)> { - while pos < bytes.len() { - let ff_rel = memchr(0xFF, &bytes[pos..])?; - let marker_prefix = pos + ff_rel; - let mut code_pos = marker_prefix + 1; - while code_pos < bytes.len() && bytes[code_pos] == 0xFF { - code_pos += 1; - } - if code_pos >= bytes.len() { - return None; - } - let code = bytes[code_pos]; - match code { - 0x00 | 0xD0..=0xD7 => pos = code_pos + 1, - _ => return Some((code_pos - 1, code)), - } - } - None -} - fn marker_payload( bytes: &[u8], marker_offset: usize, @@ -753,7 +733,7 @@ mod tests { let mut bytes = minimal_baseline_jpeg(); // Insert a second SOF0 before SOS. Find SOS offset and splice. let sos_pos = bytes.windows(2).position(|w| w == [0xFF, 0xDA]).unwrap(); - let second_sof = vec![ + let second_sof = [ 0xFF, 0xC0, 0x00, @@ -825,7 +805,7 @@ mod tests { let first_sos_pos = bytes.windows(2).position(|w| w == [0xFF, 0xDA]).unwrap(); bytes[first_sos_pos + 12] = 0; let eoi_pos = bytes.windows(2).rposition(|w| w == [0xFF, 0xD9]).unwrap(); - let second_scan = vec![ + let second_scan = [ 0xFF, 0xDA, 0x00, 12, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 0, 0x10, 0x00, ]; bytes.splice(eoi_pos..eoi_pos, second_scan.iter().copied()); diff --git a/crates/signinum-jpeg/src/parse/markers.rs b/crates/j2k-jpeg/src/parse/markers.rs similarity index 88% rename from crates/signinum-jpeg/src/parse/markers.rs rename to crates/j2k-jpeg/src/parse/markers.rs index 57fd174f..36bfc20e 100644 --- a/crates/signinum-jpeg/src/parse/markers.rs +++ b/crates/j2k-jpeg/src/parse/markers.rs @@ -11,6 +11,7 @@ //! next non-RST marker; progressive streams resume marker enumeration there. use crate::error::{JpegError, MarkerKind}; +use memchr::memchr; /// One parsed marker plus a borrow of its payload bytes. For stand-alone /// markers (SOI, EOI, RSTn, TEM) the payload slice is empty. @@ -28,6 +29,26 @@ pub(crate) struct MarkerWalker<'a> { soi_seen: bool, } +pub(crate) fn next_marker_after_entropy(bytes: &[u8], mut pos: usize) -> Option<(usize, u8)> { + while pos < bytes.len() { + let ff_rel = memchr(0xFF, &bytes[pos..])?; + let marker_prefix = pos + ff_rel; + let mut code_pos = marker_prefix + 1; + while code_pos < bytes.len() && bytes[code_pos] == 0xFF { + code_pos += 1; + } + if code_pos >= bytes.len() { + return None; + } + let code = bytes[code_pos]; + match code { + 0x00 | 0xD0..=0xD7 => pos = code_pos + 1, + _ => return Some((code_pos - 1, code)), + } + } + None +} + impl<'a> MarkerWalker<'a> { pub(crate) fn new(bytes: &'a [u8]) -> Self { Self { @@ -266,6 +287,27 @@ mod tests { assert!(m.payload.is_empty()); } + #[test] + fn entropy_marker_scan_skips_stuffed_bytes_and_restart_markers() { + let bytes = &[0x11, 0xFF, 0x00, 0x22, 0xFF, 0xD3, 0x33, 0xFF, 0xD9]; + + assert_eq!(next_marker_after_entropy(bytes, 0), Some((7, 0xD9))); + } + + #[test] + fn entropy_marker_scan_skips_fill_bytes_before_marker() { + let bytes = &[0x11, 0xFF, 0xFF, 0xFF, 0xDA]; + + assert_eq!(next_marker_after_entropy(bytes, 0), Some((3, 0xDA))); + } + + #[test] + fn entropy_marker_scan_returns_none_for_trailing_fill() { + let bytes = &[0x11, 0x22, 0xFF, 0xFF]; + + assert_eq!(next_marker_after_entropy(bytes, 0), None); + } + #[test] fn next_marker_rejects_non_ff_byte_in_marker_position() { // After SOI, the parser expects 0xFF. A stray 0x00 must be an error, diff --git a/crates/signinum-jpeg/src/parse/mod.rs b/crates/j2k-jpeg/src/parse/mod.rs similarity index 100% rename from crates/signinum-jpeg/src/parse/mod.rs rename to crates/j2k-jpeg/src/parse/mod.rs diff --git a/crates/signinum-jpeg/src/parse/scan.rs b/crates/j2k-jpeg/src/parse/scan.rs similarity index 100% rename from crates/signinum-jpeg/src/parse/scan.rs rename to crates/j2k-jpeg/src/parse/scan.rs diff --git a/crates/signinum-jpeg/src/parse/sof.rs b/crates/j2k-jpeg/src/parse/sof.rs similarity index 90% rename from crates/signinum-jpeg/src/parse/sof.rs rename to crates/j2k-jpeg/src/parse/sof.rs index c598e35f..fbba6603 100644 --- a/crates/signinum-jpeg/src/parse/sof.rs +++ b/crates/j2k-jpeg/src/parse/sof.rs @@ -60,8 +60,15 @@ pub(crate) fn parse_sof( (0xC2, 8) => SofKind::Progressive8, (0xC2, 12) => SofKind::Progressive12, (0xC3, 2..=16) => SofKind::Lossless, - // Differential / hierarchical - (0xC5 | 0xC6 | 0xC7, _) => { + // Differential sequential, currently unsupported. + (0xC5, _) => { + return Err(JpegError::UnsupportedSof { + marker: marker_code, + reason: UnsupportedReason::DifferentialBaseline, + }); + } + // Hierarchical/differential progressive/lossless, currently unsupported. + (0xC6 | 0xC7, _) => { return Err(JpegError::UnsupportedSof { marker: marker_code, reason: UnsupportedReason::Hierarchical, @@ -130,7 +137,7 @@ pub(crate) fn parse_sof( bit_depth: precision, width, height, - sampling: SamplingFactors::from_components(&components), + sampling: SamplingFactors::from_validated_components(&components), component_ids, quant_table_ids, }) @@ -209,8 +216,20 @@ mod tests { } #[test] - fn rejects_sof5_hierarchical() { + fn rejects_sof5_differential_baseline() { let err = parse_sof(0xC5, &sof0_420_payload(), 0).unwrap_err(); + assert!(matches!( + err, + JpegError::UnsupportedSof { + reason: UnsupportedReason::DifferentialBaseline, + .. + } + )); + } + + #[test] + fn rejects_sof6_hierarchical() { + let err = parse_sof(0xC6, &sof0_420_payload(), 0).unwrap_err(); assert!(matches!( err, JpegError::UnsupportedSof { diff --git a/crates/signinum-jpeg/src/parse/tables.rs b/crates/j2k-jpeg/src/parse/tables.rs similarity index 100% rename from crates/signinum-jpeg/src/parse/tables.rs rename to crates/j2k-jpeg/src/parse/tables.rs diff --git a/crates/signinum-jpeg/src/profile.rs b/crates/j2k-jpeg/src/profile.rs similarity index 75% rename from crates/signinum-jpeg/src/profile.rs rename to crates/j2k-jpeg/src/profile.rs index 5366d632..e6f939e3 100644 --- a/crates/signinum-jpeg/src/profile.rs +++ b/crates/j2k-jpeg/src/profile.rs @@ -3,15 +3,15 @@ use std::cell::RefCell; use std::sync::OnceLock; -pub(crate) use signinum_profile::duration_us_string; -use signinum_profile::{ +pub(crate) use j2k_profile::duration_us_string; +use j2k_profile::{ profile_stage_mode_from_env, same_summary_labels, ProfileStageMode, SummaryLabel, }; #[cfg(test)] -pub(crate) use signinum_profile::ProfileSummary; +pub(crate) use j2k_profile::ProfileSummary; -const JPEG_PROFILE_STAGES_ENV: &str = "SIGNINUM_JPEG_PROFILE_STAGES"; +const JPEG_PROFILE_STAGES_ENV: &str = "J2K_JPEG_PROFILE_STAGES"; const SUMMARY_LABEL_FIELD_KEYS: &[&str] = &["mode", "fmt", "downscale", "scan_path"]; pub(crate) fn jpeg_profile_stages_enabled() -> bool { @@ -24,7 +24,7 @@ fn jpeg_profile_stage_mode() -> ProfileStageMode { } pub(crate) fn emit_jpeg_profile_row(op: &str, path: &str, fields: &[(&str, &str)]) { - signinum_profile::emit_profile_row_with_timing_summary( + j2k_profile::emit_profile_row_with_timing_summary( jpeg_profile_stage_mode(), &PROFILE_SUMMARY, "jpeg", @@ -36,8 +36,8 @@ pub(crate) fn emit_jpeg_profile_row(op: &str, path: &str, fields: &[(&str, &str) } thread_local! { - static PROFILE_SUMMARY: RefCell = RefCell::new( - signinum_profile::ProfileSummary::new(summary_label_fields().iter().cloned()) + static PROFILE_SUMMARY: RefCell = RefCell::new( + j2k_profile::ProfileSummary::new(summary_label_fields().iter().cloned()).emit_on_drop() ); } @@ -54,7 +54,7 @@ mod tests { #[test] fn profile_summary_groups_rows_and_averages_timing_fields() { let mut summary = ProfileSummary::new(summary_label_fields().iter().cloned()); - signinum_profile::record_timing_summary_str( + j2k_profile::record_timing_summary_str( &mut summary, "jpeg", "decode", @@ -69,7 +69,7 @@ mod tests { ], SUMMARY_LABEL_FIELD_KEYS, ); - signinum_profile::record_timing_summary_str( + j2k_profile::record_timing_summary_str( &mut summary, "jpeg", "decode", @@ -88,7 +88,7 @@ mod tests { assert_eq!( summary.format_rows(), vec![ - "signinum_profile_summary codec=jpeg op=decode path=cpu mode=full fmt=Rgb8 count=2 decode_us_sum=12 decode_us_avg=6 total_us_sum=16 total_us_avg=8" + "j2k_profile_summary codec=jpeg op=decode path=cpu mode=full fmt=Rgb8 count=2 decode_us_sum=12 decode_us_avg=6 total_us_sum=16 total_us_avg=8" ] ); } diff --git a/crates/j2k-jpeg/src/segment.rs b/crates/j2k-jpeg/src/segment.rs new file mode 100644 index 00000000..f2cf75ac --- /dev/null +++ b/crates/j2k-jpeg/src/segment.rs @@ -0,0 +1,838 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Public JPEG marker-level utilities. + +use crate::error::{JpegError, MarkerKind, TableKind, UnsupportedReason}; +use crate::info::{SamplingFactors, SofKind}; +use crate::parse::markers::next_marker_after_entropy; +use alloc::vec::Vec; +use core::ops::Range; +use memchr::memchr; + +/// One marker segment in a JPEG byte stream. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JpegSegment<'a> { + /// Raw marker byte after the `0xff` prefix. + pub marker: u8, + /// Offset of the marker prefix byte. + pub marker_offset: usize, + /// Offset of the segment payload. Standalone markers use the byte after the marker. + pub payload_offset: usize, + /// Segment payload excluding marker and length bytes. + pub payload: &'a [u8], +} + +/// Iterator over marker segments in a JPEG byte stream. +#[derive(Debug)] +pub struct JpegSegmentIter<'a> { + input: &'a [u8], + pos: usize, + started: bool, + finished: bool, + scan_entropy: bool, +} + +/// Parsed Start-of-Frame facts. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JpegSofInfo { + /// Raw SOF marker byte. + pub marker: u8, + /// Supported SOF kind classification. + pub sof_kind: SofKind, + /// Component sample precision. + pub bit_depth: u8, + /// Width and height from the SOF payload. + pub dimensions: (u16, u16), + /// Component identifiers in declaration order. + pub component_ids: Vec, + /// Component sampling factors in declaration order. + pub sampling: SamplingFactors, + /// Quantization-table selectors in declaration order. + pub quant_table_ids: Vec, +} + +/// Byte ranges around the first Start-of-Scan marker. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JpegScanRanges { + /// Offset of the SOS marker prefix byte. + pub sos_marker_offset: usize, + /// Payload range of the SOS segment, excluding marker and length bytes. + pub sos_payload_range: Range, + /// Entropy-coded scan data range after SOS and before EOI or the next marker. + pub entropy_range: Range, + /// Offset of EOI when present. + pub eoi_marker_offset: Option, +} + +/// Options for preparing TIFF/WSI JPEG tile payloads. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JpegTilePrepareOptions { + /// Container-derived expected tile dimensions. + pub expected_dimensions: Option<(u16, u16)>, + /// Duplicate DQT/DHT handling policy. + pub duplicate_table_policy: DuplicateTablePolicy, + /// Repair zero SOF dimensions using `expected_dimensions`. + pub repair_zero_sof_dimensions: bool, + /// Validate restart marker order in scan data. + pub validate_restart_markers: bool, +} + +impl Default for JpegTilePrepareOptions { + fn default() -> Self { + Self { + expected_dimensions: None, + duplicate_table_policy: DuplicateTablePolicy::RejectConflicting, + repair_zero_sof_dimensions: false, + validate_restart_markers: false, + } + } +} + +/// Duplicate JPEG table handling policy. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DuplicateTablePolicy { + /// Accept byte-identical duplicate table definitions. + AllowIdentical, + /// Reject conflicting duplicate table definitions. + RejectConflicting, +} + +/// Prepared JPEG bytes, borrowed when unchanged and owned when normalized. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PreparedJpeg<'a> { + /// Original tile bytes can be decoded directly. + Borrowed(&'a [u8]), + /// Preparation changed the byte stream. + Owned(Vec), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TableKey { + Quant(u8), + HuffmanDc(u8), + HuffmanAc(u8), + Dri, +} + +#[derive(Debug, Clone)] +struct SegmentBytes { + offset: usize, + bytes: Vec, + key: Option, +} + +impl PreparedJpeg<'_> { + /// Return decode-ready JPEG bytes. + #[must_use] + pub fn as_bytes(&self) -> &[u8] { + match self { + Self::Borrowed(bytes) => bytes, + Self::Owned(bytes) => bytes, + } + } +} + +impl AsRef<[u8]> for PreparedJpeg<'_> { + fn as_ref(&self) -> &[u8] { + self.as_bytes() + } +} + +/// Iterate over JPEG marker segments. +#[must_use] +pub fn iter_segments(input: &[u8]) -> JpegSegmentIter<'_> { + JpegSegmentIter { + input, + pos: 0, + started: false, + finished: false, + scan_entropy: false, + } +} + +/// Return true for JPEG SOF marker classes. +#[must_use] +pub const fn is_sof_marker(marker: u8) -> bool { + matches!( + marker, + 0xc0..=0xc3 | 0xc5..=0xc7 | 0xc9..=0xcb | 0xcd..=0xcf + ) +} + +/// Parse a Start-of-Frame payload. +pub fn parse_sof_info(marker: u8, payload: &[u8]) -> Result { + parse_sof_info_at(marker, payload, 0, false) +} + +pub(crate) fn parse_sof_info_allowing_zero_dimensions( + marker: u8, + payload: &[u8], + payload_offset: usize, +) -> Result { + parse_sof_info_at(marker, payload, payload_offset, true) +} + +/// Parse a Define Restart Interval payload. +pub fn parse_dri(payload: &[u8]) -> Result, JpegError> { + if payload.len() != 2 { + return Err(JpegError::InvalidSegmentLength { + offset: 0, + marker: 0xdd, + length: (payload.len() + 2) as u16, + }); + } + let interval = u16::from_be_bytes([payload[0], payload[1]]); + Ok((interval > 0).then_some(interval)) +} + +/// Find the first SOS header and scan data ranges. +pub fn find_scan_ranges(input: &[u8]) -> Result { + for segment in iter_segments(input) { + let segment = segment?; + if segment.marker == 0xda { + let entropy_start = segment.payload_offset + segment.payload.len(); + let next_marker = next_marker_after_entropy(input, entropy_start); + let (entropy_end, eoi_marker_offset) = match next_marker { + Some((marker_offset, 0xd9)) => (marker_offset, Some(marker_offset)), + Some((marker_offset, _)) => (marker_offset, None), + None => (input.len(), None), + }; + return Ok(JpegScanRanges { + sos_marker_offset: segment.marker_offset, + sos_payload_range: segment.payload_offset..entropy_start, + entropy_range: entropy_start..entropy_end, + eoi_marker_offset, + }); + } + } + Err(JpegError::MissingMarker { + marker: MarkerKind::Sos, + }) +} + +/// Return a copy of `input` with the first SOF dimensions rewritten. +pub fn rewrite_sof_dimensions(input: &[u8], dimensions: (u16, u16)) -> Result, JpegError> { + if dimensions.0 == 0 || dimensions.1 == 0 { + return Err(JpegError::ZeroDimension { + width: dimensions.0, + height: dimensions.1, + }); + } + for segment in iter_segments(input) { + let segment = segment?; + if is_sof_marker(segment.marker) { + if segment.payload.len() < 5 { + return Err(JpegError::Truncated { + offset: segment.payload_offset + segment.payload.len(), + expected: 5 - segment.payload.len(), + }); + } + let mut out = input.to_vec(); + let width = dimensions.0.to_be_bytes(); + let height = dimensions.1.to_be_bytes(); + out[segment.payload_offset + 1] = height[0]; + out[segment.payload_offset + 2] = height[1]; + out[segment.payload_offset + 3] = width[0]; + out[segment.payload_offset + 4] = width[1]; + return Ok(out); + } + } + Err(JpegError::MissingMarker { + marker: MarkerKind::Sof, + }) +} + +/// Prepare a TIFF/WSI JPEG tile for decode. +pub fn prepare_tiff_jpeg_tile<'a>( + tile: &'a [u8], + tables: Option<&'a [u8]>, + opts: JpegTilePrepareOptions, +) -> Result, JpegError> { + if is_complete_jpeg(tile) { + validate_complete_tile(tile, opts) + } else { + assemble_abbreviated_tile(tile, tables, opts) + } +} + +fn is_complete_jpeg(input: &[u8]) -> bool { + input.len() >= 4 + && input[0] == 0xff + && input[1] == 0xd8 + && input[input.len() - 2] == 0xff + && input[input.len() - 1] == 0xd9 +} + +fn validate_complete_tile( + tile: &[u8], + opts: JpegTilePrepareOptions, +) -> Result, JpegError> { + if let Some(repaired) = finalize_prepared_bytes(tile, opts)? { + return Ok(PreparedJpeg::Owned(repaired)); + } + Ok(PreparedJpeg::Borrowed(tile)) +} + +fn finalize_prepared_bytes( + bytes: &[u8], + opts: JpegTilePrepareOptions, +) -> Result>, JpegError> { + let repaired = repair_or_validate_dimensions(bytes, opts)?; + let validation_input = repaired.as_deref().unwrap_or(bytes); + let _ = find_scan_ranges(validation_input)?; + if opts.validate_restart_markers { + validate_restart_markers(validation_input)?; + } + Ok(repaired) +} + +fn repair_or_validate_dimensions( + bytes: &[u8], + opts: JpegTilePrepareOptions, +) -> Result>, JpegError> { + let mut saw_sof = false; + for segment in iter_segments(bytes) { + let segment = segment?; + if segment.marker == 0xda { + break; + } + if is_sof_marker(segment.marker) { + saw_sof = true; + let sof = parse_sof_info_allowing_zero_dimensions( + segment.marker, + segment.payload, + segment.payload_offset, + )?; + if sof.dimensions.0 == 0 || sof.dimensions.1 == 0 { + let Some(expected) = opts.expected_dimensions else { + return Err(JpegError::ExpectedDimensionsRequired { + offset: segment.marker_offset, + }); + }; + if !opts.repair_zero_sof_dimensions { + return Err(JpegError::ZeroDimension { + width: sof.dimensions.0, + height: sof.dimensions.1, + }); + } + let repaired = rewrite_sof_dimensions(bytes, expected)?; + validate_nonzero_sof_dimensions(&repaired, opts)?; + return Ok(Some(repaired)); + } + if let Some(expected) = opts.expected_dimensions { + if expected != sof.dimensions { + return Err(JpegError::ConflictingExpectedDimensions { + offset: segment.marker_offset, + expected, + actual: sof.dimensions, + }); + } + } + } + } + if !saw_sof { + return Err(JpegError::MissingMarker { + marker: MarkerKind::Sof, + }); + } + Ok(None) +} + +fn validate_nonzero_sof_dimensions( + bytes: &[u8], + opts: JpegTilePrepareOptions, +) -> Result<(), JpegError> { + let mut saw_sof = false; + for segment in iter_segments(bytes) { + let segment = segment?; + if is_sof_marker(segment.marker) { + saw_sof = true; + let sof = parse_sof_info(segment.marker, segment.payload)?; + if let Some(expected) = opts.expected_dimensions { + if expected != sof.dimensions { + return Err(JpegError::ConflictingExpectedDimensions { + offset: segment.marker_offset, + expected, + actual: sof.dimensions, + }); + } + } + } + } + if !saw_sof { + return Err(JpegError::MissingMarker { + marker: MarkerKind::Sof, + }); + } + Ok(()) +} + +fn validate_restart_markers(bytes: &[u8]) -> Result<(), JpegError> { + let ranges = find_scan_ranges(bytes)?; + let mut expected = 0u8; + let mut pos = ranges.entropy_range.start; + while pos < ranges.entropy_range.end { + let Some(relative) = memchr(0xff, &bytes[pos..ranges.entropy_range.end]) else { + break; + }; + let prefix = pos + relative; + let mut marker_pos = prefix + 1; + while marker_pos < ranges.entropy_range.end && bytes[marker_pos] == 0xff { + marker_pos += 1; + } + if marker_pos >= ranges.entropy_range.end { + return Err(JpegError::Truncated { + offset: prefix, + expected: 1, + }); + } + let marker = bytes[marker_pos]; + match marker { + 0x00 => pos = marker_pos + 1, + 0xd0..=0xd7 => { + let found = marker & 0x07; + if found != expected { + return Err(JpegError::RestartMismatch { + offset: marker_pos - 1, + expected, + found: marker, + }); + } + expected = (expected + 1) & 0x07; + pos = marker_pos + 1; + } + 0xd9 => break, + _ => { + return Err(JpegError::UnexpectedMarker { + offset: marker_pos - 1, + expected: MarkerKind::Eoi, + found: marker, + }); + } + } + } + Ok(()) +} + +fn assemble_abbreviated_tile<'a>( + tile: &'a [u8], + tables: Option<&'a [u8]>, + opts: JpegTilePrepareOptions, +) -> Result, JpegError> { + let Some(tables) = tables else { + return Err(JpegError::InvalidJpegAssembly { + offset: 0, + reason: "abbreviated JPEG tile requires JPEGTables", + }); + }; + let mut out = Vec::new(); + out.extend_from_slice(&[0xff, 0xd8]); + let mut keyed_segments = Vec::<(TableKey, Vec)>::new(); + for segment in collect_normalized_segments(tables)? { + push_segment_dedup(&mut out, &mut keyed_segments, segment)?; + } + + let tile_body = normalized_abbreviated_tile_body(tile)?; + out.extend_from_slice(tile_body); + if !out.ends_with(&[0xff, 0xd9]) { + out.extend_from_slice(&[0xff, 0xd9]); + } + if let Some(repaired) = finalize_prepared_bytes(&out, opts)? { + out = repaired; + } + Ok(PreparedJpeg::Owned(out)) +} + +fn collect_normalized_segments(input: &[u8]) -> Result, JpegError> { + let mut segments = Vec::new(); + for segment in iter_segments(input) { + let segment = segment?; + if matches!(segment.marker, 0xd8 | 0xd9) { + continue; + } + let total_end = segment.payload_offset + segment.payload.len(); + let key = table_key(segment.marker, segment.payload)?; + segments.push(SegmentBytes { + offset: segment.marker_offset, + bytes: input[segment.marker_offset..total_end].to_vec(), + key, + }); + } + Ok(segments) +} + +fn table_key(marker: u8, payload: &[u8]) -> Result, JpegError> { + match marker { + 0xdb => { + let Some(first) = payload.first() else { + return Err(JpegError::InvalidSegmentLength { + offset: 0, + marker, + length: 2, + }); + }; + Ok(Some(TableKey::Quant(first & 0x0f))) + } + 0xc4 => { + let Some(first) = payload.first() else { + return Err(JpegError::InvalidSegmentLength { + offset: 0, + marker, + length: 2, + }); + }; + let class = first >> 4; + let id = first & 0x0f; + Ok(Some(if class == 0 { + TableKey::HuffmanDc(id) + } else { + TableKey::HuffmanAc(id) + })) + } + 0xdd => Ok(Some(TableKey::Dri)), + _ => Ok(None), + } +} + +fn push_segment_dedup( + out: &mut Vec, + keyed_segments: &mut Vec<(TableKey, Vec)>, + segment: SegmentBytes, +) -> Result<(), JpegError> { + let Some(key) = segment.key else { + out.extend_from_slice(&segment.bytes); + return Ok(()); + }; + if let Some((_, existing)) = keyed_segments + .iter() + .find(|(existing_key, _)| *existing_key == key) + { + if existing == &segment.bytes { + return Ok(()); + } + return match key { + TableKey::Quant(id) => Err(JpegError::ConflictingDuplicateTable { + offset: segment.offset, + table: TableKind::Quant, + id, + }), + TableKey::HuffmanDc(id) => Err(JpegError::ConflictingDuplicateTable { + offset: segment.offset, + table: TableKind::HuffmanDc, + id, + }), + TableKey::HuffmanAc(id) => Err(JpegError::ConflictingDuplicateTable { + offset: segment.offset, + table: TableKind::HuffmanAc, + id, + }), + TableKey::Dri => { + let existing = parse_dri_payload_from_segment(existing).unwrap_or(0); + let new = parse_dri_payload_from_segment(&segment.bytes).unwrap_or(0); + Err(JpegError::ConflictingDri { + offset: segment.offset, + existing, + new, + }) + } + }; + } + keyed_segments.push((key, segment.bytes.clone())); + out.extend_from_slice(&segment.bytes); + Ok(()) +} + +fn parse_dri_payload_from_segment(segment: &[u8]) -> Option { + if segment.len() < 6 || segment[0] != 0xff || segment[1] != 0xdd { + return None; + } + Some(u16::from_be_bytes([segment[4], segment[5]])) +} + +fn normalized_abbreviated_tile_body(tile: &[u8]) -> Result<&[u8], JpegError> { + let start = if tile.starts_with(&[0xff, 0xd8]) { + 2 + } else { + 0 + }; + let end = if tile.len() >= start + 2 && tile[tile.len() - 2..] == [0xff, 0xd9] { + tile.len() - 2 + } else { + tile.len() + }; + if start >= end { + return Err(JpegError::InvalidJpegAssembly { + offset: 0, + reason: "abbreviated JPEG tile is empty", + }); + } + Ok(&tile[start..end]) +} + +impl<'a> Iterator for JpegSegmentIter<'a> { + type Item = Result, JpegError>; + + fn next(&mut self) -> Option { + if self.finished { + return None; + } + if !self.started { + self.started = true; + return Some(self.read_soi()); + } + if self.scan_entropy { + if let Some((marker_offset, marker)) = next_marker_after_entropy(self.input, self.pos) { + self.pos = marker_offset; + self.scan_entropy = false; + if marker == 0xd9 { + return Some(self.read_standalone_marker()); + } + } else { + self.finished = true; + return None; + } + } + Some(self.read_segment()) + } +} + +impl<'a> JpegSegmentIter<'a> { + fn read_soi(&mut self) -> Result, JpegError> { + if self.input.len() < 2 { + return Err(JpegError::Truncated { + offset: 0, + expected: 2 - self.input.len(), + }); + } + if self.input[0] != 0xff || self.input[1] != 0xd8 { + return Err(JpegError::UnexpectedMarker { + offset: 0, + expected: MarkerKind::Soi, + found: self.input.get(1).copied().unwrap_or(0), + }); + } + self.pos = 2; + Ok(JpegSegment { + marker: 0xd8, + marker_offset: 0, + payload_offset: 2, + payload: &[], + }) + } + + fn read_segment(&mut self) -> Result, JpegError> { + if self.pos >= self.input.len() { + return Err(JpegError::Truncated { + offset: self.pos, + expected: 2, + }); + } + if self.input[self.pos] != 0xff { + return Err(JpegError::InvalidMarker { + offset: self.pos, + marker: self.input[self.pos], + }); + } + while self.pos < self.input.len() && self.input[self.pos] == 0xff { + self.pos += 1; + } + if self.pos >= self.input.len() { + return Err(JpegError::Truncated { + offset: self.pos, + expected: 1, + }); + } + let marker = self.input[self.pos]; + let marker_offset = self.pos - 1; + self.pos += 1; + + match marker { + 0x01 | 0xd0..=0xd9 => { + if marker == 0xd9 { + self.finished = true; + } + Ok(JpegSegment { + marker, + marker_offset, + payload_offset: self.pos, + payload: &[], + }) + } + 0x00 => Err(JpegError::InvalidMarker { + offset: marker_offset, + marker, + }), + _ => { + if self.pos + 2 > self.input.len() { + return Err(JpegError::Truncated { + offset: self.pos, + expected: self.pos + 2 - self.input.len(), + }); + } + let length = u16::from_be_bytes([self.input[self.pos], self.input[self.pos + 1]]); + if length < 2 { + return Err(JpegError::InvalidSegmentLength { + offset: self.pos, + marker, + length, + }); + } + let payload_offset = self.pos + 2; + let payload_end = self.pos.checked_add(usize::from(length)).ok_or( + JpegError::InvalidSegmentLength { + offset: self.pos, + marker, + length, + }, + )?; + if payload_end > self.input.len() { + return Err(JpegError::Truncated { + offset: payload_offset, + expected: payload_end - self.input.len(), + }); + } + self.pos = payload_end; + if marker == 0xda { + self.scan_entropy = true; + } + Ok(JpegSegment { + marker, + marker_offset, + payload_offset, + payload: &self.input[payload_offset..payload_end], + }) + } + } + } + + fn read_standalone_marker(&mut self) -> Result, JpegError> { + let marker_offset = self.pos; + if self.pos + 1 >= self.input.len() { + return Err(JpegError::Truncated { + offset: self.pos, + expected: self.pos + 2 - self.input.len(), + }); + } + let marker = self.input[self.pos + 1]; + self.pos += 2; + if marker == 0xd9 { + self.finished = true; + } + Ok(JpegSegment { + marker, + marker_offset, + payload_offset: self.pos, + payload: &[], + }) + } +} + +fn parse_sof_info_at( + marker: u8, + payload: &[u8], + payload_offset: usize, + allow_zero_dimensions: bool, +) -> Result { + if payload.len() < 8 { + return Err(JpegError::Truncated { + offset: payload_offset + payload.len(), + expected: 8 - payload.len(), + }); + } + + let bit_depth = payload[0]; + let height = u16::from_be_bytes([payload[1], payload[2]]); + let width = u16::from_be_bytes([payload[3], payload[4]]); + let component_count = payload[5]; + let expected_len = 6 + usize::from(component_count) * 3; + if payload.len() < expected_len { + return Err(JpegError::Truncated { + offset: payload_offset + payload.len(), + expected: expected_len - payload.len(), + }); + } + + let sof_kind = match (marker, bit_depth) { + (0xc0, 8) => SofKind::Baseline8, + (0xc1, 8) => SofKind::Extended8, + (0xc1, 12) => SofKind::Extended12, + (0xc2, 8) => SofKind::Progressive8, + (0xc2, 12) => SofKind::Progressive12, + (0xc3, 2..=16) => SofKind::Lossless, + (0xc5, _) => { + return Err(JpegError::UnsupportedSof { + marker, + reason: UnsupportedReason::DifferentialBaseline, + }); + } + (0xc6 | 0xc7, _) => { + return Err(JpegError::UnsupportedSof { + marker, + reason: UnsupportedReason::Hierarchical, + }); + } + (0xc9 | 0xca | 0xcb, _) => { + return Err(JpegError::UnsupportedSof { + marker, + reason: UnsupportedReason::ArithmeticCoding, + }); + } + (0xcd | 0xce | 0xcf, _) => { + return Err(JpegError::UnsupportedSof { + marker, + reason: UnsupportedReason::ArithmeticAndHierarchical, + }); + } + (_, bad_precision) => { + return Err(JpegError::UnsupportedBitDepth { + depth: bad_precision, + }) + } + }; + + if !allow_zero_dimensions && (width == 0 || height == 0) { + return Err(JpegError::ZeroDimension { width, height }); + } + if width > 65_500 || height > 65_500 { + return Err(JpegError::DimensionOverflow { + width: u32::from(width), + height: u32::from(height), + }); + } + if !matches!(component_count, 1 | 3 | 4) { + return Err(JpegError::UnsupportedComponentCount { + count: component_count, + }); + } + + let mut sampling = Vec::with_capacity(usize::from(component_count)); + let mut component_ids = Vec::with_capacity(usize::from(component_count)); + let mut quant_table_ids = Vec::with_capacity(usize::from(component_count)); + for i in 0..usize::from(component_count) { + let base = 6 + i * 3; + let component_id = payload[base]; + let sampling_byte = payload[base + 1]; + let h = sampling_byte >> 4; + let v = sampling_byte & 0x0f; + if !(1..=4).contains(&h) || !(1..=4).contains(&v) { + return Err(JpegError::InvalidSampling { + component: i as u8, + h, + v, + }); + } + component_ids.push(component_id); + sampling.push((h, v)); + quant_table_ids.push(payload[base + 2]); + } + + Ok(JpegSofInfo { + marker, + sof_kind, + bit_depth, + dimensions: (width, height), + component_ids, + sampling: SamplingFactors::from_validated_components(&sampling), + quant_table_ids, + }) +} diff --git a/crates/j2k-jpeg/src/transcode.rs b/crates/j2k-jpeg/src/transcode.rs new file mode 100644 index 00000000..e352aeee --- /dev/null +++ b/crates/j2k-jpeg/src/transcode.rs @@ -0,0 +1,523 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Experimental JPEG coefficient extraction for transcode pipelines. + +use crate::adapter::{ + assemble_jpeg_baseline_frame_with_quant_tables, baseline_encode_tables, JpegBaselineSampling, +}; +use crate::decoder::Decoder; +use crate::encoder::{ + encode_block, BitWriter, JpegBackend, JpegEncodeError, JpegEncodeOptions, JpegSubsampling, +}; +use crate::entropy::progressive::{decode_progressive_dct_blocks, ProgressiveDctBlocks}; +use crate::entropy::sequential::{decode_scan_dct_blocks, DecodedDctBlocks}; +use crate::entropy::ZIGZAG; +use crate::error::{JpegError, MarkerKind}; +use crate::info::{ColorSpace, RestartIndex, SofKind}; +use alloc::vec::Vec; + +/// Options for experimental DCT block extraction. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct DctExtractOptions { + /// Whether to retain quantized DCT blocks in the extracted image. + /// + /// This defaults to true because JPEG DCT re-emission needs quantized + /// coefficients. Coefficient-domain transcode paths that only need + /// dequantized coefficients can disable this to avoid extra block writes. + pub retain_quantized_blocks: bool, +} + +impl DctExtractOptions { + /// Extract only dequantized DCT blocks. + #[must_use] + pub const fn dequantized_only() -> Self { + Self { + retain_quantized_blocks: false, + } + } +} + +impl Default for DctExtractOptions { + fn default() -> Self { + Self { + retain_quantized_blocks: true, + } + } +} + +/// JPEG image represented as entropy-decoded DCT blocks. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JpegDctImage { + /// Reference-grid image width. + pub width: u32, + /// Reference-grid image height. + pub height: u32, + /// JPEG color space after APP14 detection. + pub color_space: ColorSpace, + /// Entropy coding mode that produced the extracted DCT coefficients. + pub coding_mode: JpegDctCodingMode, + /// Number of SOS marker segments parsed for this image. + pub scan_count: u16, + /// Components in SOF declaration order, each at native resolution. + pub components: Vec, + /// Restart-marker metadata when the stream uses a non-zero DRI interval. + pub restart_index: Option, +} + +/// JPEG DCT entropy coding mode represented by [`JpegDctImage`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JpegDctCodingMode { + /// SOF0 baseline sequential Huffman DCT. + BaselineSequential, + /// SOF2 progressive Huffman DCT with accumulated scan coefficients. + Progressive, +} + +/// One JPEG component's natural-order DCT blocks. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JpegDctComponent { + /// Component index in SOF declaration order. + pub component_index: usize, + /// Native component width in samples. + pub width: u32, + /// Native component height in samples. + pub height: u32, + /// Horizontal JPEG sampling factor. + pub h_samp: u8, + /// Vertical JPEG sampling factor. + pub v_samp: u8, + /// Number of 8x8 blocks per component row, including padded edge blocks. + pub block_cols: u32, + /// Number of 8x8 block rows, including padded edge blocks. + pub block_rows: u32, + /// Quantization table used by this component, in JPEG zigzag table order. + pub quant_table: [u16; 64], + /// Quantized DCT blocks in natural row-major coefficient order. + pub quantized_blocks: Vec<[i16; 64]>, + /// Dequantized DCT blocks in natural row-major coefficient order. + pub dequantized_blocks: Vec<[i16; 64]>, +} + +/// Extract quantized and dequantized natural-order DCT blocks from a baseline +/// sequential JPEG. +/// +/// The returned data remains in JPEG component space. No IDCT, chroma upsample, +/// RGB conversion, or color transform is performed. +pub fn extract_dct_blocks( + bytes: &[u8], + options: DctExtractOptions, +) -> Result { + let decoder = Decoder::new(bytes)?; + match decoder.info().color_space { + ColorSpace::Grayscale | ColorSpace::YCbCr | ColorSpace::Rgb => {} + color_space => return Err(JpegError::UnsupportedColorSpace { color_space }), + } + + let (coding_mode, components) = match decoder.info().sof_kind { + SofKind::Baseline8 => { + let scan_bytes = &decoder.bytes[decoder.plan.scan_offset..]; + let decoded_blocks = + decode_scan_dct_blocks(&decoder.plan, scan_bytes, options.retain_quantized_blocks)?; + ( + JpegDctCodingMode::BaselineSequential, + build_sequential_components(&decoder, decoded_blocks)?, + ) + } + SofKind::Progressive8 => { + let progressive_plan = + decoder + .progressive_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: SofKind::Progressive8, + })?; + let decoded_blocks = decode_progressive_dct_blocks(progressive_plan, decoder.bytes)?; + ( + JpegDctCodingMode::Progressive, + build_progressive_components(&decoder, decoded_blocks)?, + ) + } + other => return Err(JpegError::NotImplemented { sof: other }), + }; + let restart_index = decoder.restart_index()?; + + Ok(JpegDctImage { + width: decoder.info().dimensions.0, + height: decoder.info().dimensions.1, + color_space: decoder.info().color_space, + coding_mode, + scan_count: decoder.info().scan_count, + components, + restart_index, + }) +} + +/// Re-emit a baseline JPEG from quantized DCT blocks. +/// +/// This path writes a fresh JPEG header and entropy stream, but it does not +/// perform IDCT, chroma upsampling, color conversion, or pixel-domain JPEG +/// encoding. DC deltas are recalculated from each component's quantized DC +/// sequence as the entropy stream is written. +pub fn encode_baseline_dct_image(image: &JpegDctImage) -> Result, JpegEncodeError> { + if image.coding_mode != JpegDctCodingMode::BaselineSequential { + return Err(JpegEncodeError::Internal( + "DCT JPEG re-emission supports baseline sequential input only".into(), + )); + } + let component_count = image.components.len(); + if component_count != 1 && component_count != 3 { + return Err(JpegEncodeError::Internal(format!( + "DCT JPEG re-emission supports 1 or 3 components, got {component_count}" + ))); + } + let max_h = image + .components + .iter() + .map(|component| component.h_samp) + .max() + .unwrap_or(1); + let max_v = image + .components + .iter() + .map(|component| component.v_samp) + .max() + .unwrap_or(1); + if max_h == 0 || max_v == 0 { + return Err(JpegEncodeError::Internal( + "DCT JPEG re-emission requires nonzero sampling factors".into(), + )); + } + let mut sampling = JpegBaselineSampling { + components: component_count as u8, + h: [0; 3], + v: [0; 3], + max_h, + max_v, + }; + for (idx, component) in image.components.iter().enumerate() { + if component.component_index != idx { + return Err(JpegEncodeError::Internal( + "DCT JPEG components must be in SOF declaration order".into(), + )); + } + sampling.h[idx] = component.h_samp; + sampling.v[idx] = component.v_samp; + } + validate_dct_component_grids(image, sampling)?; + + let luma_quant = zigzag_quant_to_natural_u8(&image.components[0].quant_table)?; + let chroma_quant = if component_count == 3 { + if image.components[1].quant_table != image.components[2].quant_table { + return Err(JpegEncodeError::Internal( + "DCT JPEG re-emission supports one shared chroma quant table".into(), + )); + } + Some(zigzag_quant_to_natural_u8( + &image.components[1].quant_table, + )?) + } else { + None + }; + let huffman_tables = baseline_encode_tables(JpegEncodeOptions { + quality: 90, + subsampling: if component_count == 1 { + JpegSubsampling::Gray + } else { + JpegSubsampling::Ybr420 + }, + restart_interval: None, + backend: JpegBackend::Cpu, + })?; + let dc_tables = [&huffman_tables.huff_dc_luma, &huffman_tables.huff_dc_chroma]; + let ac_tables = [&huffman_tables.huff_ac_luma, &huffman_tables.huff_ac_chroma]; + let entropy = encode_dct_entropy(image, sampling, dc_tables, ac_tables)?; + let encoded = assemble_jpeg_baseline_frame_with_quant_tables( + &entropy, + image.width, + image.height, + sampling, + &luma_quant, + chroma_quant.as_ref(), + JpegBackend::Cpu, + )?; + Ok(encoded.data) +} + +/// Run the scalar ISLOW IDCT oracle on one dequantized natural-order DCT block. +/// +/// The output matches j2k-jpeg's scalar decode semantics for one component +/// block, including JPEG's unsigned sample level shift and clamping. +#[must_use] +pub fn idct_islow_block(block: &[i16; 64]) -> [u8; 64] { + let mut output = [0; 64]; + crate::idct::idct_islow(block, &mut output); + output +} + +fn validate_dct_component_grids( + image: &JpegDctImage, + sampling: JpegBaselineSampling, +) -> Result<(), JpegEncodeError> { + let mcu_cols = image.width.div_ceil(u32::from(sampling.max_h) * 8); + let mcu_rows = image.height.div_ceil(u32::from(sampling.max_v) * 8); + for (idx, component) in image.components.iter().enumerate() { + let expected_block_cols = mcu_cols * u32::from(sampling.h[idx]); + let expected_block_rows = mcu_rows * u32::from(sampling.v[idx]); + let expected_blocks = expected_block_cols + .checked_mul(expected_block_rows) + .ok_or_else(|| JpegEncodeError::Internal("DCT block count overflow".into()))?; + if component.block_cols != expected_block_cols + || component.block_rows != expected_block_rows + || component.quantized_blocks.len() != expected_blocks as usize + { + return Err(JpegEncodeError::Internal(format!( + "DCT component {idx} grid is {}x{} blocks with {} blocks, expected {}x{} and {} blocks", + component.block_cols, + component.block_rows, + component.quantized_blocks.len(), + expected_block_cols, + expected_block_rows, + expected_blocks + ))); + } + } + Ok(()) +} + +fn zigzag_quant_to_natural_u8(quant: &[u16; 64]) -> Result<[u8; 64], JpegEncodeError> { + let mut natural = [0u8; 64]; + for (zigzag_idx, &natural_idx) in ZIGZAG.iter().enumerate() { + natural[usize::from(natural_idx)] = u8::try_from(quant[zigzag_idx]).map_err(|_| { + JpegEncodeError::Internal( + "DCT JPEG re-emission supports 8-bit quant tables only".into(), + ) + })?; + } + Ok(natural) +} + +fn encode_dct_entropy( + image: &JpegDctImage, + sampling: JpegBaselineSampling, + dc_tables: [&crate::adapter::JpegBaselineHuffmanTable; 2], + ac_tables: [&crate::adapter::JpegBaselineHuffmanTable; 2], +) -> Result, JpegEncodeError> { + let mcu_cols = image.width.div_ceil(u32::from(sampling.max_h) * 8); + let mcu_rows = image.height.div_ceil(u32::from(sampling.max_v) * 8); + let mut writer = BitWriter::new(); + let mut prev_dc = [0i32; 3]; + for mcu_y in 0..mcu_rows { + for mcu_x in 0..mcu_cols { + for (component_idx, prev_dc_component) in prev_dc + .iter_mut() + .enumerate() + .take(sampling.components as usize) + { + let component = &image.components[component_idx]; + let table_idx = usize::from(component_idx != 0); + for block_y in 0..sampling.v[component_idx] { + for block_x in 0..sampling.h[component_idx] { + let source_block_x = + mcu_x * u32::from(sampling.h[component_idx]) + u32::from(block_x); + let source_block_y = + mcu_y * u32::from(sampling.v[component_idx]) + u32::from(block_y); + let block_idx = + (source_block_y * component.block_cols + source_block_x) as usize; + let mut coeffs = [0i32; 64]; + for (dst, &src) in coeffs + .iter_mut() + .zip(component.quantized_blocks[block_idx].iter()) + { + *dst = i32::from(src); + } + encode_block( + &coeffs, + prev_dc_component, + dc_tables[table_idx], + ac_tables[table_idx], + &mut writer, + )?; + } + } + } + } + } + Ok(writer.into_bytes()) +} + +fn build_sequential_components( + decoder: &Decoder<'_>, + mut decoded_blocks: DecodedDctBlocks, +) -> Result, JpegError> { + let dimensions = decoder.info().dimensions; + let sampling = decoder.info().sampling; + let max_h = u32::from(sampling.max_h); + let max_v = u32::from(sampling.max_v); + let mcu_cols = dimensions.0.div_ceil(8 * max_h); + let mcu_rows = dimensions.1.div_ceil(8 * max_v); + + let mut components = Vec::with_capacity(sampling.len()); + for (component_index, &(h_samp, v_samp)) in sampling.components().iter().enumerate() { + let plan_component = decoder + .plan + .components + .iter() + .find(|component| component.output_index == component_index) + .ok_or(JpegError::InvalidSequentialComponentSet { + offset: decoder.plan.scan_offset, + expected: sampling.len() as u8, + found: decoder.plan.components.len() as u8, + })?; + let quantized_blocks = decoded_blocks + .quantized + .get_mut(component_index) + .map(core::mem::take) + .ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sos, + })?; + let dequantized_blocks = decoded_blocks + .dequantized + .get_mut(component_index) + .map(core::mem::take) + .ok_or(JpegError::MissingMarker { + marker: MarkerKind::Sos, + })?; + + components.push(JpegDctComponent { + component_index, + width: dimensions + .0 + .saturating_mul(u32::from(h_samp)) + .div_ceil(max_h), + height: dimensions + .1 + .saturating_mul(u32::from(v_samp)) + .div_ceil(max_v), + h_samp, + v_samp, + block_cols: mcu_cols * u32::from(h_samp), + block_rows: mcu_rows * u32::from(v_samp), + quant_table: *plan_component.quant.as_ref(), + quantized_blocks, + dequantized_blocks, + }); + } + + Ok(components) +} + +fn build_progressive_components( + decoder: &Decoder<'_>, + decoded_blocks: ProgressiveDctBlocks, +) -> Result, JpegError> { + let plan = decoder + .progressive_plan + .as_ref() + .ok_or(JpegError::NotImplemented { + sof: SofKind::Progressive8, + })?; + let mut components = Vec::with_capacity(plan.components.len()); + for component in &plan.components { + let quantized_i32 = decoded_blocks.quantized.get(component.output_index).ok_or( + JpegError::MissingMarker { + marker: MarkerKind::Sos, + }, + )?; + let mut quantized_blocks = Vec::with_capacity(quantized_i32.len()); + let mut dequantized_blocks = Vec::with_capacity(quantized_i32.len()); + for block in quantized_i32 { + let quantized = quantized_i16_block(block); + let dequantized = dequantize_progressive_block(block, &component.quant); + quantized_blocks.push(quantized); + dequantized_blocks.push(dequantized); + } + + components.push(JpegDctComponent { + component_index: component.output_index, + width: component.sample_width, + height: component.sample_height, + h_samp: component.h, + v_samp: component.v, + block_cols: component.block_cols, + block_rows: component.block_rows, + quant_table: *component.quant.as_ref(), + quantized_blocks, + dequantized_blocks, + }); + } + + Ok(components) +} + +fn quantized_i16_block(block: &[i32; 64]) -> [i16; 64] { + let mut out = [0i16; 64]; + for (dst, &value) in out.iter_mut().zip(block.iter()) { + *dst = value.clamp(i16::MIN as i32, i16::MAX as i32) as i16; + } + out +} + +fn dequantize_progressive_block(block: &[i32; 64], quant: &[u16; 64]) -> [i16; 64] { + let mut out = [0i16; 64]; + for (zigzag_idx, &natural_idx) in ZIGZAG.iter().enumerate() { + let value = block[usize::from(natural_idx)].wrapping_mul(i32::from(quant[zigzag_idx])); + out[usize::from(natural_idx)] = value.clamp(i16::MIN as i32, i16::MAX as i32) as i16; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + encode_jpeg_baseline, JpegBackend, JpegEncodeOptions, JpegSamples, JpegSubsampling, + }; + + #[test] + fn reemits_baseline_jpeg_from_extracted_quantized_dct_blocks() { + let width = 32; + let height = 24; + let mut rgb = Vec::with_capacity(width * height * 3); + for y in 0..height { + for x in 0..width { + rgb.push(((x * 7 + y * 3) & 0xff) as u8); + rgb.push(((x * 5 + y * 11) & 0xff) as u8); + rgb.push(((x * 13 + y * 2) & 0xff) as u8); + } + } + let encoded = encode_jpeg_baseline( + JpegSamples::Rgb8 { + width: width as u32, + height: height as u32, + data: &rgb, + }, + JpegEncodeOptions { + quality: 83, + subsampling: JpegSubsampling::Ybr420, + restart_interval: Some(2), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode source jpeg"); + let source = extract_dct_blocks(&encoded.data, DctExtractOptions::default()) + .expect("extract source dct"); + + let reemitted = encode_baseline_dct_image(&source).expect("re-emit dct jpeg"); + let actual = extract_dct_blocks(&reemitted, DctExtractOptions::default()) + .expect("extract re-emitted dct"); + + assert_eq!(actual.width, source.width); + assert_eq!(actual.height, source.height); + assert_eq!(actual.color_space, source.color_space); + assert_eq!(actual.components.len(), source.components.len()); + for (actual, expected) in actual.components.iter().zip(source.components.iter()) { + assert_eq!(actual.width, expected.width); + assert_eq!(actual.height, expected.height); + assert_eq!(actual.h_samp, expected.h_samp); + assert_eq!(actual.v_samp, expected.v_samp); + assert_eq!(actual.quant_table, expected.quant_table); + assert_eq!(actual.quantized_blocks, expected.quantized_blocks); + } + } +} diff --git a/crates/signinum-jpeg/tests/baseline_encode_helpers.rs b/crates/j2k-jpeg/tests/baseline_encode_helpers.rs similarity index 93% rename from crates/signinum-jpeg/tests/baseline_encode_helpers.rs rename to crates/j2k-jpeg/tests/baseline_encode_helpers.rs index 9f9b0c4e..3db17c87 100644 --- a/crates/signinum-jpeg/tests/baseline_encode_helpers.rs +++ b/crates/j2k-jpeg/tests/baseline_encode_helpers.rs @@ -1,11 +1,11 @@ -use signinum_jpeg::adapter::{assemble_jpeg_baseline_frame, baseline_encode_tables}; -use signinum_jpeg::{ +use j2k_jpeg::adapter::{assemble_jpeg_baseline_frame, baseline_encode_tables}; +use j2k_jpeg::{ encode_jpeg_baseline, JpegBackend, JpegEncodeOptions, JpegSamples, JpegSubsampling, }; #[test] fn baseline_encode_helper_assembles_same_frame_as_cpu_encoder() { - let pixels = signinum_test_support::patterned_rgb8(2, 2); + let pixels = j2k_test_support::patterned_rgb8(2, 2); let options = JpegEncodeOptions { quality: 87, subsampling: JpegSubsampling::Ybr444, diff --git a/crates/j2k-jpeg/tests/batch.rs b/crates/j2k-jpeg/tests/batch.rs new file mode 100644 index 00000000..795d9b91 --- /dev/null +++ b/crates/j2k-jpeg/tests/batch.rs @@ -0,0 +1,2180 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Batch decode via [`Decoder::decode_tile`]: sequential output must match +//! parallel output byte-for-byte across a worker pool. Validates the +//! Phase 5 tile primitive under `std::thread::scope`. + +use fixtures::{ + cmyk_16x16_420_jpeg, cmyk_16x8_422_jpeg, cmyk_8x8_jpeg, extended_12bit_cmyk_16x16_420_jpeg, + extended_12bit_cmyk_16x8_422_jpeg, extended_12bit_cmyk_420_restart_32x16_jpeg, + extended_12bit_cmyk_422_restart_32x8_jpeg, extended_12bit_cmyk_8x8_jpeg, + extended_12bit_cmyk_restart_16x8_jpeg, extended_12bit_rgb_32x32_rgb16, + extended_12bit_rgb_32x8_rgb16, extended_12bit_rgb_420_32x32_jpeg, + extended_12bit_rgb_422_32x8_jpeg, extended_12bit_rgb_8x8_jpeg, extended_12bit_rgb_8x8_rgb16, + extended_12bit_ycbcr_420_32x32_jpeg, extended_12bit_ycbcr_420_32x32_rgb16, + extended_12bit_ycbcr_420_restart_32x32_jpeg, extended_12bit_ycbcr_420_restart_32x32_rgb16, + extended_12bit_ycbcr_422_32x8_jpeg, extended_12bit_ycbcr_422_32x8_rgb16, + extended_12bit_ycbcr_8x8_jpeg, extended_12bit_ycbcr_8x8_rgb16, + extended_12bit_ycck_16x16_420_jpeg, extended_12bit_ycck_16x8_422_jpeg, + extended_12bit_ycck_420_restart_32x16_jpeg, extended_12bit_ycck_422_restart_32x8_jpeg, + extended_12bit_ycck_8x8_jpeg, extended_12bit_ycck_restart_16x8_jpeg, + four_component_12bit_16x16_rgb16, four_component_12bit_16x8_rgb16, + four_component_12bit_32x16_rgb16, four_component_12bit_32x8_rgb16, + four_component_12bit_8x8_rgb16, four_component_16x16_rgb, four_component_16x8_rgb, + four_component_8x8_rgb, lossless_predictor_rgb_16bit_3x3_jpeg, + lossless_predictor_ycbcr_16bit_3x3_jpeg, lossless_predictor_ycbcr_3x3_jpeg, + lossless_restart_predictor_rgb_16bit_3x3_jpeg, lossless_restart_predictor_ycbcr_16bit_3x3_jpeg, + lossless_restart_predictor_ycbcr_3x3_jpeg, lossless_rgb_16bit_420_4x4_jpeg, + lossless_rgb_16bit_420_4x4_rgb16, lossless_rgb_16bit_420_restart_4x4_jpeg, + lossless_rgb_16bit_422_4x2_jpeg, lossless_rgb_16bit_422_4x2_rgb16, + lossless_rgb_16bit_422_restart_4x2_jpeg, lossless_rgb_8bit_420_4x4_jpeg, + lossless_rgb_8bit_420_4x4_rgb8, lossless_rgb_8bit_420_restart_4x4_jpeg, + lossless_rgb_8bit_422_4x2_jpeg, lossless_rgb_8bit_422_4x2_rgb8, + lossless_rgb_8bit_422_restart_4x2_jpeg, lossless_ycbcr_16bit_3x3_rgb16, + lossless_ycbcr_16bit_420_4x4_jpeg, lossless_ycbcr_16bit_420_4x4_rgb16, + lossless_ycbcr_16bit_420_restart_4x4_jpeg, lossless_ycbcr_16bit_422_4x2_jpeg, + lossless_ycbcr_16bit_422_4x2_rgb16, lossless_ycbcr_16bit_422_restart_4x2_jpeg, + lossless_ycbcr_3x3_rgb8, lossless_ycbcr_8bit_420_4x4_jpeg, lossless_ycbcr_8bit_420_4x4_rgb8, + lossless_ycbcr_8bit_420_restart_4x4_jpeg, lossless_ycbcr_8bit_422_4x2_jpeg, + lossless_ycbcr_8bit_422_4x2_rgb8, lossless_ycbcr_8bit_422_restart_4x2_jpeg, + progressive_12bit_cmyk_16x16_420_jpeg, progressive_12bit_cmyk_16x8_422_jpeg, + progressive_12bit_cmyk_420_restart_32x16_jpeg, progressive_12bit_cmyk_422_restart_32x8_jpeg, + progressive_12bit_cmyk_8x8_jpeg, progressive_12bit_cmyk_restart_16x8_jpeg, + progressive_12bit_grayscale_8x8_jpeg, progressive_12bit_rgb_420_32x32_jpeg, + progressive_12bit_rgb_422_32x8_jpeg, progressive_12bit_rgb_8x8_jpeg, + progressive_12bit_ycbcr_420_32x32_jpeg, progressive_12bit_ycbcr_422_32x8_jpeg, + progressive_12bit_ycbcr_8x8_jpeg, progressive_12bit_ycck_16x16_420_jpeg, + progressive_12bit_ycck_16x8_422_jpeg, progressive_12bit_ycck_420_restart_32x16_jpeg, + progressive_12bit_ycck_422_restart_32x8_jpeg, progressive_12bit_ycck_8x8_jpeg, + progressive_12bit_ycck_restart_16x8_jpeg, progressive_8x8_jpeg, ycck_16x16_420_jpeg, + ycck_16x8_422_jpeg, ycck_8x8_jpeg, LOSSLESS_RGB_16BIT_3X3_PIXELS, +}; +use j2k_jpeg::{ + decode_prepared_jpeg_tiles_rgb8, decode_tile_into_in_context, + decode_tile_into_in_context_with_options, decode_tile_region_scaled_into_in_context, + decode_tiles_into, decode_tiles_into_with_options, decode_tiles_region_scaled_into, + decode_tiles_scaled_into, decode_tiles_scaled_into_with_options, prepare_tiff_jpeg_tile, + ColorTransform, DecodeOptions, Decoder, DecoderContext, Downscale, JpegBatchSession, JpegError, + JpegOutputBuffer, JpegTilePrepareOptions, PixelFormat, PreparedJpeg, PreparedJpegTileJob, Rect, + RowSink, ScratchPool, TileBatchOptions, TileDecodeJob, TileRegionScaledDecodeJob, + TileScaledDecodeJob, +}; +use j2k_test_support as fixtures; +use j2k_test_support::{ + rgb16le_to_rgba16le, rgb8_to_rgba8, u16_samples_to_le_bytes, JPEG_BASELINE_420_16X16, +}; +use std::num::NonZeroUsize; +use std::thread; + +const BASELINE_420: &[u8] = JPEG_BASELINE_420_16X16; + +const BATCH_SIZE: usize = 100; + +#[derive(Default)] +struct CollectRows { + rows: Vec, +} + +impl RowSink for CollectRows { + type Error = j2k_jpeg::JpegError; + + fn write_row(&mut self, _y: u32, row: &[u8]) -> Result<(), j2k_jpeg::JpegError> { + self.rows.extend_from_slice(row); + Ok(()) + } +} + +fn decode_tile_bytes(bytes: &[u8], ctx: &mut DecoderContext, pool: &mut ScratchPool) -> Vec { + let mut sink = CollectRows::default(); + Decoder::decode_tile(bytes, ctx, pool, &mut sink).expect("Decoder::decode_tile"); + sink.rows +} + +fn decode_tile_rgb8_reference(bytes: &[u8]) -> (Vec, usize) { + let dec = Decoder::new(bytes).expect("fixture decoder"); + let (width, height) = dec.info().dimensions; + let stride = width as usize * 3; + let mut out = vec![0u8; stride * height as usize]; + dec.decode_into(&mut out, stride, PixelFormat::Rgb8) + .expect("fixture decode_into"); + (out, stride) +} + +fn scaled_rect(rect: Rect, scale: Downscale) -> Rect { + let denom = scale.denominator(); + Rect { + x: rect.x / denom, + y: rect.y / denom, + w: (rect.x + rect.w).div_ceil(denom) - rect.x / denom, + h: (rect.y + rect.h).div_ceil(denom) - rect.y / denom, + } +} + +fn scaled_dims((width, height): (u32, u32), scale: Downscale) -> (u32, u32) { + let denom = scale.denominator(); + (width.div_ceil(denom), height.div_ceil(denom)) +} + +fn assert_session_batch_decode( + session: &mut JpegBatchSession, + input: &[u8], + fmt: PixelFormat, + stride: usize, + expected: &[u8], + context: &str, +) { + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input, + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, fmt) + .unwrap_or_else(|err| panic!("{context}: {err}")) + }; + + assert_eq!(outcomes.len(), outputs.len(), "{context}"); + for output in outputs { + assert_eq!(output, expected, "{context}"); + } +} + +#[test] +fn prepared_jpeg_batch_decode_returns_ordered_per_tile_results() { + let (expected, stride) = decode_tile_rgb8_reference(BASELINE_420); + let prepared_a = prepare_tiff_jpeg_tile(BASELINE_420, None, JpegTilePrepareOptions::default()) + .expect("prepared A"); + let prepared_c = prepare_tiff_jpeg_tile(BASELINE_420, None, JpegTilePrepareOptions::default()) + .expect("prepared C"); + let mut out_a = vec![0u8; expected.len()]; + let mut out_bad = vec![0u8; expected.len()]; + let mut out_c = vec![0u8; expected.len()]; + + let results = { + let mut jobs = vec![ + PreparedJpegTileJob { + input: prepared_a, + out: out_a.as_mut_slice(), + stride, + options: DecodeOptions::default(), + }, + PreparedJpegTileJob { + input: PreparedJpeg::Borrowed(&[]), + out: out_bad.as_mut_slice(), + stride, + options: DecodeOptions::default(), + }, + PreparedJpegTileJob { + input: prepared_c, + out: out_c.as_mut_slice(), + stride, + options: DecodeOptions::default(), + }, + ]; + decode_prepared_jpeg_tiles_rgb8(&mut jobs) + }; + + assert_eq!(results.len(), 3); + assert!(results[0].is_ok()); + assert!(matches!(results[1], Err(JpegError::Truncated { .. }))); + assert!(results[2].is_ok()); + assert_eq!( + results[0].as_ref().expect("first result").dimensions, + (16, 16) + ); + assert_eq!( + results[2].as_ref().expect("third result").dimensions, + (16, 16) + ); + assert_eq!(out_a, expected); + assert_eq!(out_c, expected); +} + +#[test] +fn prepared_jpeg_batch_decode_applies_per_job_decode_options() { + let prepared_rgb = + prepare_tiff_jpeg_tile(BASELINE_420, None, JpegTilePrepareOptions::default()) + .expect("prepared RGB"); + let prepared_ycbcr = + prepare_tiff_jpeg_tile(BASELINE_420, None, JpegTilePrepareOptions::default()) + .expect("prepared YCbCr"); + let stride = 16 * 3; + let mut expected_rgb = vec![0u8; stride * 16]; + let mut expected_ycbcr = vec![0u8; stride * 16]; + let mut ctx = DecoderContext::new(); + let mut pool = ScratchPool::new(); + decode_tile_into_in_context_with_options( + BASELINE_420, + &mut ctx, + &mut pool, + &mut expected_rgb, + stride, + PixelFormat::Rgb8, + DecodeOptions::default().with_color_transform(ColorTransform::ForceRgb), + ) + .expect("force RGB reference"); + decode_tile_into_in_context_with_options( + BASELINE_420, + &mut ctx, + &mut pool, + &mut expected_ycbcr, + stride, + PixelFormat::Rgb8, + DecodeOptions::default().with_color_transform(ColorTransform::ForceYCbCr), + ) + .expect("force YCbCr reference"); + assert_ne!(expected_rgb, expected_ycbcr); + + let mut out_rgb = vec![0u8; expected_rgb.len()]; + let mut out_ycbcr = vec![0u8; expected_ycbcr.len()]; + { + let mut jobs = vec![ + PreparedJpegTileJob { + input: prepared_rgb, + out: out_rgb.as_mut_slice(), + stride, + options: DecodeOptions::default().with_color_transform(ColorTransform::ForceRgb), + }, + PreparedJpegTileJob { + input: prepared_ycbcr, + out: out_ycbcr.as_mut_slice(), + stride, + options: DecodeOptions::default().with_color_transform(ColorTransform::ForceYCbCr), + }, + ]; + let results = decode_prepared_jpeg_tiles_rgb8(&mut jobs); + assert!(results.iter().all(Result::is_ok)); + } + + assert_eq!(out_rgb, expected_rgb); + assert_eq!(out_ycbcr, expected_ycbcr); +} + +#[test] +fn production_batch_decode_empty_input_succeeds() { + let mut jobs: Vec> = Vec::new(); + + let outcomes = decode_tiles_into(&mut jobs, PixelFormat::Rgb8, TileBatchOptions::default()) + .expect("empty batch succeeds"); + + assert!(outcomes.is_empty()); +} + +#[test] +fn production_batch_decode_worker_one_matches_single_tile_decode() { + let (expected, stride) = decode_tile_rgb8_reference(BASELINE_420); + let mut actual = vec![0u8; expected.len()]; + let options = TileBatchOptions { + workers: NonZeroUsize::new(1), + }; + + let outcomes = { + let mut jobs = vec![TileDecodeJob { + input: BASELINE_420, + out: actual.as_mut_slice(), + stride, + }]; + decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect("batch decode") + }; + + assert_eq!(outcomes.len(), 1); + assert_eq!(actual, expected); +} + +#[test] +fn production_batch_decode_progressive8_matches_single_tile_decode() { + let bytes = progressive_8x8_jpeg(); + let (expected, stride) = decode_tile_rgb8_reference(&bytes); + let mut actual = vec![0u8; expected.len()]; + + let outcomes = { + let mut jobs = vec![TileDecodeJob { + input: &bytes, + out: actual.as_mut_slice(), + stride, + }]; + decode_tiles_into( + &mut jobs, + PixelFormat::Rgb8, + TileBatchOptions { + workers: NonZeroUsize::new(1), + }, + ) + .expect("progressive batch decode") + }; + + assert_eq!(outcomes.len(), 1); + assert_eq!(actual, expected); +} + +#[test] +fn session_batch_scaled_and_region_scaled_progressive8_matches_single_tile_decode() { + let bytes = progressive_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("progressive decoder"); + let scale = Downscale::Half; + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_full = scaled_dims(dec.info().dimensions, scale); + let scaled_roi = scaled_rect(roi, scale); + let scaled_stride = scaled_full.0 as usize * 3; + let region_stride = scaled_roi.w as usize * 3; + let mut expected_scaled = vec![0u8; scaled_stride * scaled_full.1 as usize]; + let mut expected_region = vec![0u8; region_stride * scaled_roi.h as usize]; + dec.decode_scaled_into( + &mut expected_scaled, + scaled_stride, + PixelFormat::Rgb8, + scale, + ) + .expect("progressive scaled reference"); + dec.decode_region_scaled_into( + &mut expected_region, + region_stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("progressive region-scaled reference"); + + let mut actual_scaled = vec![0u8; expected_scaled.len()]; + let mut actual_region = vec![0u8; expected_region.len()]; + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + { + let mut jobs = vec![TileScaledDecodeJob { + input: &bytes, + out: actual_scaled.as_mut_slice(), + stride: scaled_stride, + scale, + }]; + session + .decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgb8) + .expect("progressive session scaled batch"); + } + { + let mut jobs = vec![TileRegionScaledDecodeJob { + input: &bytes, + out: actual_region.as_mut_slice(), + stride: region_stride, + roi: roi.into(), + scale, + }]; + session + .decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8) + .expect("progressive session region-scaled batch"); + } + + assert_eq!(actual_scaled, expected_scaled); + assert_eq!(actual_region, expected_region); +} + +#[test] +fn session_batch_scaled_and_region_scaled_progressive12_matches_single_tile_decode() { + let bytes = progressive_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive decoder"); + let scale = Downscale::Half; + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_full = scaled_dims(dec.info().dimensions, scale); + let scaled_roi = scaled_rect(roi, scale); + let scaled_stride = scaled_full.0 as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let region_stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut expected_scaled = vec![0u8; scaled_stride * scaled_full.1 as usize]; + let mut expected_region = vec![0u8; region_stride * scaled_roi.h as usize]; + dec.decode_scaled_into( + &mut expected_scaled, + scaled_stride, + PixelFormat::Rgb16, + scale, + ) + .expect("12-bit progressive scaled reference"); + dec.decode_region_scaled_into( + &mut expected_region, + region_stride, + PixelFormat::Rgb16, + roi, + scale, + ) + .expect("12-bit progressive region-scaled reference"); + + let mut actual_scaled = vec![0u8; expected_scaled.len()]; + let mut actual_region = vec![0u8; expected_region.len()]; + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + { + let mut jobs = vec![TileScaledDecodeJob { + input: &bytes, + out: actual_scaled.as_mut_slice(), + stride: scaled_stride, + scale, + }]; + session + .decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgb16) + .expect("12-bit progressive session scaled batch"); + } + { + let mut jobs = vec![TileRegionScaledDecodeJob { + input: &bytes, + out: actual_region.as_mut_slice(), + stride: region_stride, + roi: roi.into(), + scale, + }]; + session + .decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb16) + .expect("12-bit progressive session region-scaled batch"); + } + + assert_eq!(actual_scaled, expected_scaled); + assert_eq!(actual_region, expected_region); +} + +#[test] +fn session_batch_decode_extended12_app14_rgb_matches_single_tile_decode() { + let bytes = extended_12bit_rgb_8x8_jpeg(); + let expected = extended_12bit_rgb_8x8_rgb16(); + let stride = 8 * PixelFormat::Rgb16.bytes_per_pixel(); + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + assert_session_batch_decode( + &mut session, + &bytes, + PixelFormat::Rgb16, + stride, + &expected, + "12-bit APP14 RGB session batch decode", + ); +} + +#[test] +fn session_batch_decode_progressive12_app14_rgb_matches_single_tile_decode() { + let bytes = progressive_12bit_rgb_8x8_jpeg(); + let expected = extended_12bit_rgb_8x8_rgb16(); + let stride = 8 * PixelFormat::Rgb16.bytes_per_pixel(); + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + assert_session_batch_decode( + &mut session, + &bytes, + PixelFormat::Rgb16, + stride, + &expected, + "12-bit progressive APP14 RGB session batch decode", + ); +} + +#[test] +fn session_batch_decode_lossless_app14_rgb16_matches_single_tile_decode() { + let expected = u16_samples_to_le_bytes(&LOSSLESS_RGB_16BIT_3X3_PIXELS); + let stride = 3 * PixelFormat::Rgb16.bytes_per_pixel(); + let expected_rgba = rgb16le_to_rgba16le(&expected, u16::MAX); + let rgba_stride = 3 * PixelFormat::Rgba16.bytes_per_pixel(); + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + for bytes in [ + lossless_predictor_rgb_16bit_3x3_jpeg(1), + lossless_restart_predictor_rgb_16bit_3x3_jpeg(1), + ] { + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let mut rgba_outputs = vec![ + vec![0u8; expected_rgba.len()], + vec![0u8; expected_rgba.len()], + ]; + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb16) + .expect("lossless SOF3 APP14 RGB16 session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in outputs { + assert_eq!(output, expected); + } + + let outcomes = { + let mut jobs = rgba_outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride: rgba_stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgba16) + .expect("lossless SOF3 APP14 RGBA16 session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in rgba_outputs { + assert_eq!(output, expected_rgba); + } + } +} + +#[test] +fn session_batch_decode_lossless_ycbcr_matches_single_tile_decode() { + let expected = lossless_ycbcr_3x3_rgb8(); + let stride = 3 * PixelFormat::Rgb8.bytes_per_pixel(); + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + for bytes in [ + lossless_predictor_ycbcr_3x3_jpeg(1), + lossless_restart_predictor_ycbcr_3x3_jpeg(1), + ] { + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb8) + .expect("lossless SOF3 YCbCr session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in outputs { + assert_eq!(output, expected); + } + } +} + +#[test] +fn session_batch_decode_lossless_ycbcr16_matches_single_tile_decode() { + let expected = lossless_ycbcr_16bit_3x3_rgb16(); + let stride = 3 * PixelFormat::Rgb16.bytes_per_pixel(); + let expected_rgba = rgb16le_to_rgba16le(&expected, u16::MAX); + let rgba_stride = 3 * PixelFormat::Rgba16.bytes_per_pixel(); + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + for bytes in [ + lossless_predictor_ycbcr_16bit_3x3_jpeg(1), + lossless_restart_predictor_ycbcr_16bit_3x3_jpeg(1), + ] { + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let mut rgba_outputs = vec![ + vec![0u8; expected_rgba.len()], + vec![0u8; expected_rgba.len()], + ]; + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb16) + .expect("lossless SOF3 16-bit YCbCr session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in outputs { + assert_eq!(output, expected); + } + + let outcomes = { + let mut jobs = rgba_outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride: rgba_stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgba16) + .expect("lossless SOF3 16-bit YCbCr RGBA16 session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in rgba_outputs { + assert_eq!(output, expected_rgba); + } + } +} + +#[test] +fn session_batch_decode_lossless_422_rgb16_matches_single_tile_decode() { + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + for (bytes, expected, label) in [ + ( + lossless_rgb_16bit_422_4x2_jpeg(4), + lossless_rgb_16bit_422_4x2_rgb16(), + "APP14 RGB", + ), + ( + lossless_rgb_16bit_422_restart_4x2_jpeg(4), + lossless_rgb_16bit_422_4x2_rgb16(), + "APP14 RGB restart", + ), + ( + lossless_ycbcr_16bit_422_4x2_jpeg(4), + lossless_ycbcr_16bit_422_4x2_rgb16(), + "YCbCr", + ), + ( + lossless_ycbcr_16bit_422_restart_4x2_jpeg(4), + lossless_ycbcr_16bit_422_4x2_rgb16(), + "YCbCr restart", + ), + ] { + let stride = 4 * PixelFormat::Rgb16.bytes_per_pixel(); + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb16) + .unwrap_or_else(|err| { + panic!("lossless SOF3 16-bit 4:2:2 {label} session batch decode: {err}") + }) + }; + + assert_eq!(outcomes.len(), 2, "{label}"); + for output in outputs { + assert_eq!(output, expected, "{label}"); + } + } +} + +#[test] +fn session_batch_decode_lossless_8bit_sampled_rgb8_matches_single_tile_decode() { + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + for (bytes, expected, label) in [ + ( + lossless_rgb_8bit_422_4x2_jpeg(4), + lossless_rgb_8bit_422_4x2_rgb8(), + "4:2:2 APP14 RGB", + ), + ( + lossless_rgb_8bit_422_restart_4x2_jpeg(4), + lossless_rgb_8bit_422_4x2_rgb8(), + "4:2:2 APP14 RGB restart", + ), + ( + lossless_ycbcr_8bit_422_4x2_jpeg(4), + lossless_ycbcr_8bit_422_4x2_rgb8(), + "4:2:2 YCbCr", + ), + ( + lossless_ycbcr_8bit_422_restart_4x2_jpeg(4), + lossless_ycbcr_8bit_422_4x2_rgb8(), + "4:2:2 YCbCr restart", + ), + ( + lossless_rgb_8bit_420_4x4_jpeg(4), + lossless_rgb_8bit_420_4x4_rgb8(), + "4:2:0 APP14 RGB", + ), + ( + lossless_rgb_8bit_420_restart_4x4_jpeg(4), + lossless_rgb_8bit_420_4x4_rgb8(), + "4:2:0 APP14 RGB restart", + ), + ( + lossless_ycbcr_8bit_420_4x4_jpeg(4), + lossless_ycbcr_8bit_420_4x4_rgb8(), + "4:2:0 YCbCr", + ), + ( + lossless_ycbcr_8bit_420_restart_4x4_jpeg(4), + lossless_ycbcr_8bit_420_4x4_rgb8(), + "4:2:0 YCbCr restart", + ), + ] { + let stride = 4 * PixelFormat::Rgb8.bytes_per_pixel(); + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb8) + .unwrap_or_else(|err| { + panic!("lossless SOF3 8-bit sampled {label} session batch decode: {err}") + }) + }; + + assert_eq!(outcomes.len(), 2, "{label}"); + for output in outputs { + assert_eq!(output, expected, "{label}"); + } + } +} + +#[test] +fn session_batch_decode_extended12_ycbcr444_matches_single_tile_decode() { + let bytes = extended_12bit_ycbcr_8x8_jpeg(); + let expected = extended_12bit_ycbcr_8x8_rgb16(); + let stride = 8 * PixelFormat::Rgb16.bytes_per_pixel(); + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb16) + .expect("12-bit YCbCr session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in outputs { + assert_eq!(output, expected); + } +} + +#[test] +fn session_batch_decode_progressive12_ycbcr444_matches_single_tile_decode() { + let bytes = progressive_12bit_ycbcr_8x8_jpeg(); + let expected = extended_12bit_ycbcr_8x8_rgb16(); + let stride = 8 * PixelFormat::Rgb16.bytes_per_pixel(); + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb16) + .expect("12-bit progressive YCbCr session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in outputs { + assert_eq!(output, expected); + } +} + +#[test] +fn session_batch_decode_lossless_420_rgb16_matches_single_tile_decode() { + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + for (bytes, expected, label) in [ + ( + lossless_rgb_16bit_420_4x4_jpeg(4), + lossless_rgb_16bit_420_4x4_rgb16(), + "APP14 RGB", + ), + ( + lossless_rgb_16bit_420_restart_4x4_jpeg(4), + lossless_rgb_16bit_420_4x4_rgb16(), + "APP14 RGB restart", + ), + ( + lossless_ycbcr_16bit_420_4x4_jpeg(4), + lossless_ycbcr_16bit_420_4x4_rgb16(), + "YCbCr", + ), + ( + lossless_ycbcr_16bit_420_restart_4x4_jpeg(4), + lossless_ycbcr_16bit_420_4x4_rgb16(), + "YCbCr restart", + ), + ] { + let stride = 4 * PixelFormat::Rgb16.bytes_per_pixel(); + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb16) + .unwrap_or_else(|err| { + panic!("lossless SOF3 16-bit 4:2:0 {label} session batch decode: {err}") + }) + }; + + assert_eq!(outcomes.len(), 2, "{label}"); + for output in outputs { + assert_eq!(output, expected, "{label}"); + } + } +} + +#[test] +fn session_batch_decode_extended12_ycbcr422_matches_single_tile_decode() { + let bytes = extended_12bit_ycbcr_422_32x8_jpeg(); + let expected = extended_12bit_ycbcr_422_32x8_rgb16(); + let stride = 32 * PixelFormat::Rgb16.bytes_per_pixel(); + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb16) + .expect("12-bit YCbCr 4:2:2 session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in outputs { + assert_eq!(output, expected); + } +} + +#[test] +fn session_batch_decode_progressive12_ycbcr422_matches_single_tile_decode() { + let bytes = progressive_12bit_ycbcr_422_32x8_jpeg(); + let expected = extended_12bit_ycbcr_422_32x8_rgb16(); + let stride = 32 * PixelFormat::Rgb16.bytes_per_pixel(); + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb16) + .expect("12-bit progressive YCbCr 4:2:2 session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in outputs { + assert_eq!(output, expected); + } +} + +#[test] +fn session_batch_decode_extended12_ycbcr420_matches_single_tile_decode() { + let bytes = extended_12bit_ycbcr_420_32x32_jpeg(); + let expected = extended_12bit_ycbcr_420_32x32_rgb16(); + let stride = 32 * PixelFormat::Rgb16.bytes_per_pixel(); + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb16) + .expect("12-bit YCbCr 4:2:0 session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in outputs { + assert_eq!(output, expected); + } +} + +#[test] +fn session_batch_decode_extended12_restart_ycbcr420_matches_single_tile_decode() { + let bytes = extended_12bit_ycbcr_420_restart_32x32_jpeg(); + let expected = extended_12bit_ycbcr_420_restart_32x32_rgb16(); + let stride = 32 * PixelFormat::Rgb16.bytes_per_pixel(); + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb16) + .expect("12-bit restart YCbCr 4:2:0 session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in outputs { + assert_eq!(output, expected); + } +} + +#[test] +fn session_batch_decode_progressive12_ycbcr420_matches_single_tile_decode() { + let bytes = progressive_12bit_ycbcr_420_32x32_jpeg(); + let expected = extended_12bit_ycbcr_420_32x32_rgb16(); + let stride = 32 * PixelFormat::Rgb16.bytes_per_pixel(); + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb16) + .expect("12-bit progressive YCbCr 4:2:0 session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in outputs { + assert_eq!(output, expected); + } +} + +#[test] +fn session_batch_decode_12bit_rgba16_matches_single_tile_decode() { + for (bytes, expected_rgb, width, label) in [ + ( + extended_12bit_rgb_8x8_jpeg(), + extended_12bit_rgb_8x8_rgb16(), + 8, + "12-bit extended APP14 RGB", + ), + ( + extended_12bit_rgb_422_32x8_jpeg(), + extended_12bit_rgb_32x8_rgb16(), + 32, + "12-bit extended APP14 RGB 4:2:2", + ), + ( + extended_12bit_rgb_420_32x32_jpeg(), + extended_12bit_rgb_32x32_rgb16(), + 32, + "12-bit extended APP14 RGB 4:2:0", + ), + ( + progressive_12bit_rgb_422_32x8_jpeg(), + extended_12bit_rgb_32x8_rgb16(), + 32, + "12-bit progressive APP14 RGB 4:2:2", + ), + ( + progressive_12bit_rgb_420_32x32_jpeg(), + extended_12bit_rgb_32x32_rgb16(), + 32, + "12-bit progressive APP14 RGB 4:2:0", + ), + ( + progressive_12bit_ycbcr_420_32x32_jpeg(), + extended_12bit_ycbcr_420_32x32_rgb16(), + 32, + "12-bit progressive YCbCr 4:2:0", + ), + ] { + let expected = rgb16le_to_rgba16le(&expected_rgb, u16::MAX); + let stride = width * PixelFormat::Rgba16.bytes_per_pixel(); + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: bytes.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgba16) + .unwrap_or_else(|err| panic!("{label} RGBA16 session batch decode: {err}")) + }; + + assert_eq!(outcomes.len(), 2, "{label}"); + for output in outputs { + assert_eq!(output, expected, "{label}"); + } + } +} + +#[test] +fn session_batch_decode_12bit_cmyk_ycck_rgba16_matches_expected_pixels() { + let cases = [ + ( + extended_12bit_cmyk_8x8_jpeg(), + four_component_12bit_8x8_rgb16(), + 8, + ), + ( + extended_12bit_ycck_8x8_jpeg(), + four_component_12bit_8x8_rgb16(), + 8, + ), + ( + extended_12bit_cmyk_restart_16x8_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + ), + ( + extended_12bit_ycck_restart_16x8_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + ), + ( + extended_12bit_cmyk_16x8_422_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + ), + ( + extended_12bit_ycck_16x8_422_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + ), + ( + extended_12bit_cmyk_422_restart_32x8_jpeg(), + four_component_12bit_32x8_rgb16(), + 32, + ), + ( + extended_12bit_ycck_422_restart_32x8_jpeg(), + four_component_12bit_32x8_rgb16(), + 32, + ), + ( + extended_12bit_cmyk_16x16_420_jpeg(), + four_component_12bit_16x16_rgb16(), + 16, + ), + ( + extended_12bit_ycck_16x16_420_jpeg(), + four_component_12bit_16x16_rgb16(), + 16, + ), + ( + extended_12bit_cmyk_420_restart_32x16_jpeg(), + four_component_12bit_32x16_rgb16(), + 32, + ), + ( + extended_12bit_ycck_420_restart_32x16_jpeg(), + four_component_12bit_32x16_rgb16(), + 32, + ), + ( + progressive_12bit_cmyk_8x8_jpeg(), + four_component_12bit_8x8_rgb16(), + 8, + ), + ( + progressive_12bit_ycck_8x8_jpeg(), + four_component_12bit_8x8_rgb16(), + 8, + ), + ( + progressive_12bit_cmyk_restart_16x8_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + ), + ( + progressive_12bit_ycck_restart_16x8_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + ), + ( + progressive_12bit_cmyk_16x8_422_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + ), + ( + progressive_12bit_ycck_16x8_422_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + ), + ( + progressive_12bit_cmyk_422_restart_32x8_jpeg(), + four_component_12bit_32x8_rgb16(), + 32, + ), + ( + progressive_12bit_ycck_422_restart_32x8_jpeg(), + four_component_12bit_32x8_rgb16(), + 32, + ), + ( + progressive_12bit_cmyk_16x16_420_jpeg(), + four_component_12bit_16x16_rgb16(), + 16, + ), + ( + progressive_12bit_ycck_16x16_420_jpeg(), + four_component_12bit_16x16_rgb16(), + 16, + ), + ( + progressive_12bit_cmyk_420_restart_32x16_jpeg(), + four_component_12bit_32x16_rgb16(), + 32, + ), + ( + progressive_12bit_ycck_420_restart_32x16_jpeg(), + four_component_12bit_32x16_rgb16(), + 32, + ), + ]; + let expected = cases + .iter() + .map(|(_, expected_rgb, _)| rgb16le_to_rgba16le(expected_rgb, u16::MAX)) + .collect::>(); + let mut outputs = expected + .iter() + .map(|expected| vec![0u8; expected.len()]) + .collect::>(); + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + let outcomes = { + let mut jobs = cases + .iter() + .zip(outputs.iter_mut()) + .map(|((input, _, width), out)| TileDecodeJob { + input: input.as_slice(), + out: out.as_mut_slice(), + stride: *width * PixelFormat::Rgba16.bytes_per_pixel(), + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgba16) + .expect("12-bit CMYK/YCCK RGBA16 session batch decode") + }; + + assert_eq!(outcomes.len(), cases.len()); + for (output, expected) in outputs.iter().zip(expected.iter()) { + assert_eq!(output, expected); + } +} + +#[test] +fn session_batch_decode_converts_cmyk_and_ycck() { + let inputs = [cmyk_8x8_jpeg(), ycck_8x8_jpeg()]; + let expected = four_component_8x8_rgb(); + let expected_rgba = rgb8_to_rgba8(&expected, 255); + let stride = 8 * 3; + let rgba_stride = 8 * PixelFormat::Rgba8.bytes_per_pixel(); + let mut outputs = vec![vec![0u8; expected.len()], vec![0u8; expected.len()]]; + let mut rgba_outputs = vec![ + vec![0u8; expected_rgba.len()], + vec![0u8; expected_rgba.len()], + ]; + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + let outcomes = { + let mut jobs = inputs + .iter() + .zip(outputs.iter_mut()) + .map(|(input, out)| TileDecodeJob { + input, + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb8) + .expect("CMYK/YCCK session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in outputs { + assert_eq!(output, expected); + } + + let outcomes = { + let mut jobs = inputs + .iter() + .zip(rgba_outputs.iter_mut()) + .map(|(input, out)| TileDecodeJob { + input, + out: out.as_mut_slice(), + stride: rgba_stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgba8) + .expect("CMYK/YCCK RGBA8 session batch decode") + }; + + assert_eq!(outcomes.len(), 2); + for output in rgba_outputs { + assert_eq!(output, expected_rgba); + } +} + +#[test] +fn session_batch_decode_subsampled_cmyk_ycck_matches_expected_pixels() { + let cases = [ + ( + cmyk_16x8_422_jpeg(), + four_component_16x8_rgb(), + 16, + "CMYK 4:2:2", + ), + ( + ycck_16x8_422_jpeg(), + four_component_16x8_rgb(), + 16, + "YCCK 4:2:2", + ), + ( + cmyk_16x16_420_jpeg(), + four_component_16x16_rgb(), + 16, + "CMYK 4:2:0", + ), + ( + ycck_16x16_420_jpeg(), + four_component_16x16_rgb(), + 16, + "YCCK 4:2:0", + ), + ]; + let mut rgb_outputs = cases + .iter() + .map(|(_, expected, _, _)| vec![0u8; expected.len()]) + .collect::>(); + let mut rgba_outputs = cases + .iter() + .map(|(_, expected, _, _)| vec![0u8; expected.len() / 3 * 4]) + .collect::>(); + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + let outcomes = { + let mut jobs = cases + .iter() + .zip(rgb_outputs.iter_mut()) + .map(|((input, _, width, _), out)| TileDecodeJob { + input, + out: out.as_mut_slice(), + stride: *width * PixelFormat::Rgb8.bytes_per_pixel(), + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb8) + .expect("subsampled CMYK/YCCK RGB8 session batch decode") + }; + + assert_eq!(outcomes.len(), cases.len()); + for ((_, expected, _, label), output) in cases.iter().zip(rgb_outputs.iter()) { + assert_eq!(output, expected, "{label}"); + } + + let outcomes = { + let mut jobs = cases + .iter() + .zip(rgba_outputs.iter_mut()) + .map(|((input, _, width, _), out)| TileDecodeJob { + input, + out: out.as_mut_slice(), + stride: *width * PixelFormat::Rgba8.bytes_per_pixel(), + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgba8) + .expect("subsampled CMYK/YCCK RGBA8 session batch decode") + }; + + assert_eq!(outcomes.len(), cases.len()); + for ((_, expected, _, label), output) in cases.iter().zip(rgba_outputs.iter()) { + assert_eq!(output, &rgb8_to_rgba8(expected, 255), "{label}"); + } +} + +#[test] +fn session_batch_scaled_and_region_scaled_cmyk_ycck_match_free_batch() { + let inputs = [cmyk_8x8_jpeg(), ycck_8x8_jpeg()]; + let scale = Downscale::Half; + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_full = (4, 4); + let scaled_roi = Rect { + x: roi.x / 2, + y: roi.y / 2, + w: (roi.x + roi.w).div_ceil(2) - roi.x / 2, + h: (roi.y + roi.h).div_ceil(2) - roi.y / 2, + }; + let scaled_stride = scaled_full.0 * PixelFormat::Rgb8.bytes_per_pixel(); + let region_stride = scaled_roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut scaled_outputs = inputs + .iter() + .map(|_| vec![0u8; scaled_stride * scaled_full.1]) + .collect::>(); + let mut scaled_expected = scaled_outputs.clone(); + let mut region_outputs = inputs + .iter() + .map(|_| vec![0u8; region_stride * scaled_roi.h as usize]) + .collect::>(); + let mut region_expected = region_outputs.clone(); + let options = TileBatchOptions { + workers: NonZeroUsize::new(2), + }; + let mut session = JpegBatchSession::new(options); + + { + let mut jobs = inputs + .iter() + .zip(scaled_expected.iter_mut()) + .map(|(input, out)| TileScaledDecodeJob { + input, + out: out.as_mut_slice(), + stride: scaled_stride, + scale, + }) + .collect::>(); + decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgb8, options) + .expect("free CMYK/YCCK scaled batch decode"); + } + { + let mut jobs = inputs + .iter() + .zip(scaled_outputs.iter_mut()) + .map(|(input, out)| TileScaledDecodeJob { + input, + out: out.as_mut_slice(), + stride: scaled_stride, + scale, + }) + .collect::>(); + session + .decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgb8) + .expect("session CMYK/YCCK scaled batch decode"); + } + { + let mut jobs = inputs + .iter() + .zip(region_expected.iter_mut()) + .map(|(input, out)| TileRegionScaledDecodeJob { + input, + out: out.as_mut_slice(), + stride: region_stride, + roi: roi.into(), + scale, + }) + .collect::>(); + decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8, options) + .expect("free CMYK/YCCK region-scaled batch decode"); + } + { + let mut jobs = inputs + .iter() + .zip(region_outputs.iter_mut()) + .map(|(input, out)| TileRegionScaledDecodeJob { + input, + out: out.as_mut_slice(), + stride: region_stride, + roi: roi.into(), + scale, + }) + .collect::>(); + session + .decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8) + .expect("session CMYK/YCCK region-scaled batch decode"); + } + + assert_eq!(scaled_outputs, scaled_expected); + assert_eq!(region_outputs, region_expected); +} + +#[test] +fn session_batch_rgba8_scaled_and_region_scaled_cmyk_ycck_match_free_batch() { + let inputs = [cmyk_8x8_jpeg(), ycck_8x8_jpeg()]; + let scale = Downscale::Half; + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_full = (4, 4); + let scaled_roi = Rect { + x: roi.x / 2, + y: roi.y / 2, + w: (roi.x + roi.w).div_ceil(2) - roi.x / 2, + h: (roi.y + roi.h).div_ceil(2) - roi.y / 2, + }; + let scaled_stride = scaled_full.0 * PixelFormat::Rgba8.bytes_per_pixel(); + let region_stride = scaled_roi.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let mut scaled_outputs = inputs + .iter() + .map(|_| vec![0u8; scaled_stride * scaled_full.1]) + .collect::>(); + let mut scaled_expected = scaled_outputs.clone(); + let mut region_outputs = inputs + .iter() + .map(|_| vec![0u8; region_stride * scaled_roi.h as usize]) + .collect::>(); + let mut region_expected = region_outputs.clone(); + let options = TileBatchOptions { + workers: NonZeroUsize::new(2), + }; + let mut session = JpegBatchSession::new(options); + + { + let mut jobs = inputs + .iter() + .zip(scaled_expected.iter_mut()) + .map(|(input, out)| TileScaledDecodeJob { + input, + out: out.as_mut_slice(), + stride: scaled_stride, + scale, + }) + .collect::>(); + decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgba8, options) + .expect("free CMYK/YCCK RGBA8 scaled batch decode"); + } + { + let mut jobs = inputs + .iter() + .zip(scaled_outputs.iter_mut()) + .map(|(input, out)| TileScaledDecodeJob { + input, + out: out.as_mut_slice(), + stride: scaled_stride, + scale, + }) + .collect::>(); + session + .decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgba8) + .expect("session CMYK/YCCK RGBA8 scaled batch decode"); + } + { + let mut jobs = inputs + .iter() + .zip(region_expected.iter_mut()) + .map(|(input, out)| TileRegionScaledDecodeJob { + input, + out: out.as_mut_slice(), + stride: region_stride, + roi: roi.into(), + scale, + }) + .collect::>(); + decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgba8, options) + .expect("free CMYK/YCCK RGBA8 region-scaled batch decode"); + } + { + let mut jobs = inputs + .iter() + .zip(region_outputs.iter_mut()) + .map(|(input, out)| TileRegionScaledDecodeJob { + input, + out: out.as_mut_slice(), + stride: region_stride, + roi: roi.into(), + scale, + }) + .collect::>(); + session + .decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgba8) + .expect("session CMYK/YCCK RGBA8 region-scaled batch decode"); + } + + assert_eq!(scaled_outputs, scaled_expected); + assert_eq!(region_outputs, region_expected); +} + +#[test] +fn production_batch_decode_parallel_preserves_order_and_output() { + const JOBS: usize = 32; + let (expected, stride) = decode_tile_rgb8_reference(BASELINE_420); + let mut outputs = (0..JOBS) + .map(|_| vec![0u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions { + workers: NonZeroUsize::new(4), + }; + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: BASELINE_420, + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect("batch decode") + }; + + assert_eq!(outcomes.len(), JOBS); + for (index, out) in outputs.iter().enumerate() { + assert_eq!(out, &expected, "tile {index} output diverged"); + } +} + +#[test] +fn session_batch_decode_reuses_worker_state_across_calls_and_matches_free_batch() { + const JOBS: usize = 32; + let (expected, stride) = decode_tile_rgb8_reference(BASELINE_420); + let options = TileBatchOptions { + workers: NonZeroUsize::new(4), + }; + let mut session = JpegBatchSession::new(options); + let mut outputs = (0..JOBS) + .map(|_| vec![0u8; expected.len()]) + .collect::>(); + + for pass in 0..3 { + for output in &mut outputs { + output.fill(pass as u8); + } + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: BASELINE_420, + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb8) + .expect("session batch decode") + }; + + assert_eq!(outcomes.len(), JOBS); + assert_eq!(session.worker_count(), 4); + for (index, out) in outputs.iter().enumerate() { + assert_eq!(out, &expected, "pass {pass} tile {index} output diverged"); + } + } +} + +#[test] +fn default_session_uses_available_workers_for_small_outputs() { + const JOBS: usize = 64; + let (expected, stride) = decode_tile_rgb8_reference(BASELINE_420); + let mut outputs = (0..JOBS) + .map(|_| vec![0u8; expected.len()]) + .collect::>(); + let mut session = JpegBatchSession::default(); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: BASELINE_420, + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb8) + .expect("session batch decode") + }; + + let available = thread::available_parallelism().map_or(1, NonZeroUsize::get); + assert_eq!(outcomes.len(), JOBS); + assert_eq!(session.worker_count(), available.min(JOBS)); +} + +#[test] +fn explicit_session_worker_count_is_not_small_output_capped() { + const JOBS: usize = 32; + let (expected, stride) = decode_tile_rgb8_reference(BASELINE_420); + let mut outputs = (0..JOBS) + .map(|_| vec![0u8; expected.len()]) + .collect::>(); + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(8), + }); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: BASELINE_420, + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb8) + .expect("session batch decode") + }; + + assert_eq!(outcomes.len(), JOBS); + assert_eq!(session.worker_count(), 8); +} + +#[test] +fn session_batch_decode_reports_first_failing_tile_index() { + let (expected, stride) = decode_tile_rgb8_reference(BASELINE_420); + let mut outputs = (0..3) + .map(|_| vec![0u8; expected.len()]) + .collect::>(); + let mut session = JpegBatchSession::new(TileBatchOptions { + workers: NonZeroUsize::new(2), + }); + + let err = { + let inputs: [&[u8]; 3] = [BASELINE_420, b"not a jpeg", BASELINE_420]; + let mut jobs = inputs + .into_iter() + .zip(outputs.iter_mut()) + .map(|(input, out)| TileDecodeJob { + input, + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + session + .decode_tiles_into(&mut jobs, PixelFormat::Rgb8) + .expect_err("bad tile fails") + }; + + assert_eq!(err.index, 1); +} + +#[test] +fn session_batch_scaled_and_region_scaled_decode_match_existing_batch_api() { + const JOBS: usize = 16; + let dec = Decoder::new(BASELINE_420).expect("fixture decoder"); + let scale = Downscale::Quarter; + let roi = Rect { + x: 4, + y: 4, + w: 8, + h: 8, + }; + let scaled_full = ( + dec.info().dimensions.0.div_ceil(4), + dec.info().dimensions.1.div_ceil(4), + ); + let scaled_roi = Rect { + x: roi.x / 4, + y: roi.y / 4, + w: (roi.x + roi.w).div_ceil(4) - roi.x / 4, + h: (roi.y + roi.h).div_ceil(4) - roi.y / 4, + }; + let scaled_stride = scaled_full.0 as usize * 3; + let region_stride = scaled_roi.w as usize * 3; + let mut scaled_outputs = (0..JOBS) + .map(|_| vec![0u8; scaled_stride * scaled_full.1 as usize]) + .collect::>(); + let mut scaled_expected = scaled_outputs.clone(); + let mut region_outputs = (0..JOBS) + .map(|_| vec![0u8; region_stride * scaled_roi.h as usize]) + .collect::>(); + let mut region_expected = region_outputs.clone(); + let options = TileBatchOptions { + workers: NonZeroUsize::new(4), + }; + let mut session = JpegBatchSession::new(options); + + { + let mut jobs = scaled_expected + .iter_mut() + .map(|out| TileScaledDecodeJob { + input: BASELINE_420, + out: out.as_mut_slice(), + stride: scaled_stride, + scale, + }) + .collect::>(); + decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgb8, options) + .expect("free scaled batch decode"); + } + { + let mut jobs = scaled_outputs + .iter_mut() + .map(|out| TileScaledDecodeJob { + input: BASELINE_420, + out: out.as_mut_slice(), + stride: scaled_stride, + scale, + }) + .collect::>(); + session + .decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgb8) + .expect("session scaled batch decode"); + } + { + let mut jobs = region_expected + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: BASELINE_420, + out: out.as_mut_slice(), + stride: region_stride, + roi: roi.into(), + scale, + }) + .collect::>(); + decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8, options) + .expect("free region-scaled batch decode"); + } + { + let mut jobs = region_outputs + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: BASELINE_420, + out: out.as_mut_slice(), + stride: region_stride, + roi: roi.into(), + scale, + }) + .collect::>(); + session + .decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8) + .expect("session region-scaled batch decode"); + } + + assert_eq!(scaled_outputs, scaled_expected); + assert_eq!(region_outputs, region_expected); +} + +#[test] +fn jpeg_output_buffer_resizes_without_reallocating_for_same_or_smaller_shape() { + let mut buffer = JpegOutputBuffer::new((16, 16), PixelFormat::Rgb8).expect("output buffer"); + let initial_capacity = buffer.capacity(); + assert_eq!(buffer.dimensions(), (16, 16)); + assert_eq!(buffer.stride(), 16 * 3); + assert_eq!(buffer.as_mut_slice().len(), 16 * 16 * 3); + + buffer + .resize((8, 8), PixelFormat::Rgb8) + .expect("smaller resize"); + + assert_eq!(buffer.capacity(), initial_capacity); + assert_eq!(buffer.dimensions(), (8, 8)); + assert_eq!(buffer.as_mut_slice().len(), 8 * 8 * 3); +} + +#[test] +fn production_batch_decode_with_options_preserves_forced_color_transform() { + const JOBS: usize = 8; + let decode_options = DecodeOptions::default().with_color_transform(ColorTransform::ForceRgb); + let dec = Decoder::new_with_options(BASELINE_420, decode_options).expect("fixture decoder"); + let (width, height) = dec.info().dimensions; + let stride = width as usize * 3; + let mut expected = vec![0u8; stride * height as usize]; + dec.decode_into(&mut expected, stride, PixelFormat::Rgb8) + .expect("reference forced-RGB decode"); + let mut outputs = (0..JOBS) + .map(|_| vec![0u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions { + workers: NonZeroUsize::new(2), + }; + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: BASELINE_420, + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + decode_tiles_into_with_options(&mut jobs, PixelFormat::Rgb8, decode_options, options) + .expect("batch decode with options") + }; + + assert_eq!(outcomes.len(), JOBS); + for (index, out) in outputs.iter().enumerate() { + assert_eq!(out, &expected, "tile {index} output diverged"); + } +} + +#[test] +fn production_batch_decode_reports_first_failing_tile_index() { + let (expected, stride) = decode_tile_rgb8_reference(BASELINE_420); + let mut outputs = (0..3) + .map(|_| vec![0u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions { + workers: NonZeroUsize::new(2), + }; + + let err = { + let inputs: [&[u8]; 3] = [BASELINE_420, b"not a jpeg", BASELINE_420]; + let mut jobs = inputs + .into_iter() + .zip(outputs.iter_mut()) + .map(|(input, out)| TileDecodeJob { + input, + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect_err("bad tile fails") + }; + + assert_eq!(err.index, 1); +} + +#[test] +fn sequential_and_parallel_batch_produce_identical_output() { + let tiles: Vec<&[u8]> = (0..BATCH_SIZE).map(|_| BASELINE_420).collect(); + + let sequential: Vec> = { + let mut pool = ScratchPool::new(); + let mut ctx = DecoderContext::new(); + tiles + .iter() + .map(|bytes| decode_tile_bytes(bytes, &mut ctx, &mut pool)) + .collect() + }; + + let parallel: Vec> = thread::scope(|scope| { + const WORKERS: usize = 4; + let chunk_size = tiles.len().div_ceil(WORKERS); + let handles: Vec<_> = tiles + .chunks(chunk_size) + .map(|chunk| { + scope.spawn(|| { + let mut pool = ScratchPool::new(); + let mut ctx = DecoderContext::new(); + chunk + .iter() + .map(|bytes| decode_tile_bytes(bytes, &mut ctx, &mut pool)) + .collect::>() + }) + }) + .collect(); + handles + .into_iter() + .flat_map(|h| h.join().expect("worker panicked")) + .collect() + }); + + assert_eq!(sequential.len(), parallel.len()); + for (i, (seq, par)) in sequential.iter().zip(parallel.iter()).enumerate() { + assert_eq!( + seq, par, + "tile {i} diverged between sequential and parallel" + ); + } +} + +#[test] +fn pool_reuse_across_batch_matches_fresh_pool() { + let mut reused_pool = ScratchPool::new(); + let mut reused_ctx = DecoderContext::new(); + let reused_outputs: Vec> = (0..BATCH_SIZE) + .map(|_| decode_tile_bytes(BASELINE_420, &mut reused_ctx, &mut reused_pool)) + .collect(); + + let fresh_outputs: Vec> = (0..BATCH_SIZE) + .map(|_| { + let mut pool = ScratchPool::new(); + let mut ctx = DecoderContext::new(); + decode_tile_bytes(BASELINE_420, &mut ctx, &mut pool) + }) + .collect(); + + for (i, (reused, fresh)) in reused_outputs.iter().zip(fresh_outputs.iter()).enumerate() { + assert_eq!(reused, fresh, "iter {i} reused-pool output diverged"); + } +} + +#[test] +fn tile_buffer_decode_matches_decoder_decode_into() { + let dec = Decoder::new(BASELINE_420).expect("fixture decoder"); + let (width, height) = dec.info().dimensions; + let stride = width as usize * 3; + let mut expected = vec![0u8; stride * height as usize]; + let mut actual = vec![0u8; expected.len()]; + dec.decode_into(&mut expected, stride, PixelFormat::Rgb8) + .expect("baseline decode_into"); + + let mut ctx = DecoderContext::new(); + let mut pool = ScratchPool::new(); + decode_tile_into_in_context( + BASELINE_420, + &mut ctx, + &mut pool, + &mut actual, + stride, + PixelFormat::Rgb8, + ) + .expect("tile decode_into_in_context"); + + assert_eq!(actual, expected); +} + +#[test] +fn tile_region_scaled_decode_matches_decoder_region_decode() { + let dec = Decoder::new(BASELINE_420).expect("fixture decoder"); + let roi = Rect { + x: 4, + y: 4, + w: 8, + h: 8, + }; + let denom = 4; + let scaled_w = (roi.x + roi.w).div_ceil(denom) - roi.x / denom; + let scaled_h = (roi.y + roi.h).div_ceil(denom) - roi.y / denom; + let stride = scaled_w as usize * 3; + let mut expected = vec![0u8; stride * scaled_h as usize]; + let mut actual = vec![0u8; expected.len()]; + dec.decode_region_scaled_into( + &mut expected, + stride, + PixelFormat::Rgb8, + roi, + Downscale::Quarter, + ) + .expect("core region-scaled decode"); + + let mut ctx = DecoderContext::new(); + let mut pool = ScratchPool::new(); + decode_tile_region_scaled_into_in_context( + BASELINE_420, + &mut ctx, + &mut pool, + &mut actual, + stride, + PixelFormat::Rgb8, + roi, + Downscale::Quarter, + ) + .expect("tile region decode_into_in_context"); + + assert_eq!(actual, expected); +} + +#[test] +fn production_batch_region_scaled_decode_parallel_preserves_order_and_output() { + const JOBS: usize = 32; + let dec = Decoder::new(BASELINE_420).expect("fixture decoder"); + let roi = Rect { + x: 4, + y: 4, + w: 8, + h: 8, + }; + let denom = 4; + let scaled_w = (roi.x + roi.w).div_ceil(denom) - roi.x / denom; + let scaled_h = (roi.y + roi.h).div_ceil(denom) - roi.y / denom; + let stride = scaled_w as usize * 3; + let mut expected = vec![0u8; stride * scaled_h as usize]; + dec.decode_region_scaled_into( + &mut expected, + stride, + PixelFormat::Rgb8, + roi, + Downscale::Quarter, + ) + .expect("reference region-scaled decode"); + let mut outputs = (0..JOBS) + .map(|_| vec![0u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions { + workers: NonZeroUsize::new(4), + }; + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: BASELINE_420, + out: out.as_mut_slice(), + stride, + roi: roi.into(), + scale: Downscale::Quarter, + }) + .collect::>(); + decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8, options) + .expect("batch region-scaled decode") + }; + + assert_eq!(outcomes.len(), JOBS); + for (index, out) in outputs.iter().enumerate() { + assert_eq!(out, &expected, "tile {index} output diverged"); + } +} + +#[test] +fn production_batch_scaled_decode_parallel_preserves_order_and_output() { + const JOBS: usize = 32; + let dec = Decoder::new(BASELINE_420).expect("fixture decoder"); + let scale = Downscale::Quarter; + let denom = 4; + let (width, height) = dec.info().dimensions; + let scaled_w = width.div_ceil(denom); + let scaled_h = height.div_ceil(denom); + let stride = scaled_w as usize * 3; + let mut expected = vec![0u8; stride * scaled_h as usize]; + dec.decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, scale) + .expect("reference scaled decode"); + let mut outputs = (0..JOBS) + .map(|_| vec![0u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions { + workers: NonZeroUsize::new(4), + }; + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileScaledDecodeJob { + input: BASELINE_420, + out: out.as_mut_slice(), + stride, + scale, + }) + .collect::>(); + decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgb8, options) + .expect("batch scaled decode") + }; + + assert_eq!(outcomes.len(), JOBS); + for (index, out) in outputs.iter().enumerate() { + assert_eq!(out, &expected, "tile {index} output diverged"); + } +} + +#[test] +fn production_batch_scaled_decode_with_options_preserves_forced_color_transform() { + const JOBS: usize = 8; + let decode_options = DecodeOptions::default().with_color_transform(ColorTransform::ForceRgb); + let dec = Decoder::new_with_options(BASELINE_420, decode_options).expect("fixture decoder"); + let scale = Downscale::Quarter; + let denom = 4; + let (width, height) = dec.info().dimensions; + let scaled_w = width.div_ceil(denom); + let scaled_h = height.div_ceil(denom); + let stride = scaled_w as usize * 3; + let mut expected = vec![0u8; stride * scaled_h as usize]; + dec.decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, scale) + .expect("reference scaled forced-RGB decode"); + let mut outputs = (0..JOBS) + .map(|_| vec![0u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions { + workers: NonZeroUsize::new(2), + }; + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileScaledDecodeJob { + input: BASELINE_420, + out: out.as_mut_slice(), + stride, + scale, + }) + .collect::>(); + decode_tiles_scaled_into_with_options(&mut jobs, PixelFormat::Rgb8, decode_options, options) + .expect("batch scaled decode with options") + }; + + assert_eq!(outcomes.len(), JOBS); + for (index, out) in outputs.iter().enumerate() { + assert_eq!(out, &expected, "tile {index} output diverged"); + } +} diff --git a/crates/signinum-jpeg/tests/component_rows.rs b/crates/j2k-jpeg/tests/component_rows.rs similarity index 89% rename from crates/signinum-jpeg/tests/component_rows.rs rename to crates/j2k-jpeg/tests/component_rows.rs index 084b430e..a699dc46 100644 --- a/crates/signinum-jpeg/tests/component_rows.rs +++ b/crates/j2k-jpeg/tests/component_rows.rs @@ -1,7 +1,7 @@ -use signinum_core::Downscale; -use signinum_jpeg::{ComponentRowWriter, Decoder, Rect, ScratchPool}; +use j2k_core::Downscale; +use j2k_jpeg::{ComponentRowWriter, Decoder, Rect, ScratchPool}; -mod fixtures; +use j2k_test_support as fixtures; #[derive(Default)] struct CollectRows { @@ -11,7 +11,7 @@ struct CollectRows { } impl ComponentRowWriter for CollectRows { - fn write_gray_row(&mut self, _y: u32, gray_row: &[u8]) -> Result<(), signinum_jpeg::JpegError> { + fn write_gray_row(&mut self, _y: u32, gray_row: &[u8]) -> Result<(), j2k_jpeg::JpegError> { self.gray.push(gray_row.to_vec()); Ok(()) } @@ -22,7 +22,7 @@ impl ComponentRowWriter for CollectRows { y_row: &[u8], cb_row: &[u8], cr_row: &[u8], - ) -> Result<(), signinum_jpeg::JpegError> { + ) -> Result<(), j2k_jpeg::JpegError> { self.ycbcr .push((y_row.to_vec(), cb_row.to_vec(), cr_row.to_vec())); Ok(()) @@ -34,7 +34,7 @@ impl ComponentRowWriter for CollectRows { r_row: &[u8], g_row: &[u8], b_row: &[u8], - ) -> Result<(), signinum_jpeg::JpegError> { + ) -> Result<(), j2k_jpeg::JpegError> { self.rgb .push((r_row.to_vec(), g_row.to_vec(), b_row.to_vec())); Ok(()) diff --git a/crates/signinum-jpeg/tests/conversions.rs b/crates/j2k-jpeg/tests/conversions.rs similarity index 79% rename from crates/signinum-jpeg/tests/conversions.rs rename to crates/j2k-jpeg/tests/conversions.rs index edb6c1ec..f23c6acf 100644 --- a/crates/signinum-jpeg/tests/conversions.rs +++ b/crates/j2k-jpeg/tests/conversions.rs @@ -1,5 +1,5 @@ -use signinum_core::DecodeOutcome as CoreDecodeOutcome; -use signinum_jpeg::{DecodeOutcome, Rect, Warning}; +use j2k_core::DecodeOutcome as CoreDecodeOutcome; +use j2k_jpeg::{DecodeOutcome, Rect, Warning}; #[test] fn rect_conversions_preserve_coordinates_in_both_directions() { @@ -10,10 +10,10 @@ fn rect_conversions_preserve_coordinates_in_both_directions() { h: 11, }; - let core: signinum_core::Rect = jpeg.into(); + let core: j2k_core::Rect = jpeg.into(); assert_eq!( core, - signinum_core::Rect { + j2k_core::Rect { x: 3, y: 5, w: 7, @@ -38,7 +38,7 @@ fn decode_outcome_conversion_preserves_rect_and_warnings() { let core: CoreDecodeOutcome = outcome.into(); assert_eq!( core.decoded, - signinum_core::Rect { + j2k_core::Rect { x: 1, y: 2, w: 3, diff --git a/crates/signinum-jpeg/tests/core_traits.rs b/crates/j2k-jpeg/tests/core_traits.rs similarity index 87% rename from crates/signinum-jpeg/tests/core_traits.rs rename to crates/j2k-jpeg/tests/core_traits.rs index 8e2b33e3..6e2b01ad 100644 --- a/crates/signinum-jpeg/tests/core_traits.rs +++ b/crates/j2k-jpeg/tests/core_traits.rs @@ -1,8 +1,9 @@ -use signinum_core::{ +use j2k_core::{ CodedUnitLayout, DecoderContext as CoreDecoderContext, Downscale, ImageDecode, ImageDecodeRows, PixelFormat, RowSink, TileBatchDecode, }; -use signinum_jpeg::{Decoder, DecoderContext, JpegCodec, JpegError, ScratchPool}; +use j2k_jpeg::{Decoder, DecoderContext, JpegCodec, JpegError, ScratchPool}; +use j2k_test_support::{JPEG_BASELINE_420_16X16, JPEG_BASELINE_420_RESTART_32X16}; struct CollectRows { rows: Vec>, @@ -19,7 +20,7 @@ impl RowSink for CollectRows { #[test] fn decoder_implements_core_traits_for_rgb_decode() { - let bytes = include_bytes!("../fixtures/conformance/baseline_420_16x16.jpg"); + let bytes = JPEG_BASELINE_420_16X16; let mut dec = as ImageDecode<'_>>::from_view( as ImageDecode<'_>>::parse(bytes).expect("parse"), ) @@ -50,7 +51,7 @@ fn decoder_implements_core_traits_for_rgb_decode() { #[test] fn core_inspect_exposes_restart_interval_and_coded_units_for_wsi_planning() { - let bytes = include_bytes!("../fixtures/conformance/baseline_420_restart_32x16.jpg"); + let bytes = JPEG_BASELINE_420_RESTART_32X16; let info = as ImageDecode<'_>>::inspect(bytes).expect("inspect"); assert_eq!(info.dimensions, (32, 16)); @@ -68,7 +69,7 @@ fn core_inspect_exposes_restart_interval_and_coded_units_for_wsi_planning() { #[test] fn row_and_tile_core_traits_are_callable() { - let bytes = include_bytes!("../fixtures/conformance/baseline_420_16x16.jpg"); + let bytes = JPEG_BASELINE_420_16X16; let mut dec = Decoder::new(bytes).expect("decoder"); let mut sink = CollectRows { rows: Vec::new() }; as ImageDecodeRows<'_, u8>>::decode_rows(&mut dec, &mut sink) diff --git a/crates/j2k-jpeg/tests/decode_into.rs b/crates/j2k-jpeg/tests/decode_into.rs new file mode 100644 index 00000000..7e5ef486 --- /dev/null +++ b/crates/j2k-jpeg/tests/decode_into.rs @@ -0,0 +1,3993 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Integration tests for `Decoder::decode_into`. + +use j2k_jpeg::{Decoder, Downscale, JpegError, PixelFormat, Rect, SofKind}; + +use fixtures::{ + cmyk_16x16_420_jpeg, cmyk_16x8_422_jpeg, cmyk_16x8_nonleading_max_422_jpeg, cmyk_8x8_jpeg, + extended_12bit_cmyk_8x8_jpeg, extended_12bit_cmyk_nonconstant_8x8_jpeg, + extended_12bit_grayscale_8x8_jpeg, extended_12bit_grayscale_restart_16x8_jpeg, + extended_12bit_ycck_8x8_jpeg, extended_12bit_ycck_nonconstant_8x8_jpeg, + four_component_12bit_16x16_rgb16, four_component_12bit_16x8_rgb16, + four_component_12bit_32x16_rgb16, four_component_12bit_32x8_rgb16, + four_component_12bit_8x8_cmyk_nonconstant_rgb16, four_component_12bit_8x8_rgb16, + four_component_12bit_8x8_ycck_nonconstant_rgb16, four_component_16x16_rgb, + four_component_16x8_rgb, four_component_8x8_rgb, grayscale_8x8_jpeg, + lossless_predictor_grayscale_16bit_3x3_jpeg, lossless_predictor_grayscale_3x3_jpeg, + lossless_predictor_rgb_16bit_3x3_jpeg, lossless_predictor_rgb_3x3_jpeg, + lossless_predictor_ycbcr_16bit_3x3_jpeg, lossless_predictor_ycbcr_3x3_jpeg, + lossless_restart_predictor_grayscale_16bit_3x3_jpeg, + lossless_restart_predictor_grayscale_3x3_jpeg, lossless_restart_predictor_rgb_16bit_3x3_jpeg, + lossless_restart_predictor_rgb_3x3_jpeg, lossless_restart_predictor_ycbcr_16bit_3x3_jpeg, + lossless_restart_predictor_ycbcr_3x3_jpeg, lossless_rgb_16bit_420_4x4_jpeg, + lossless_rgb_16bit_420_4x4_rgb16, lossless_rgb_16bit_420_restart_4x4_jpeg, + lossless_rgb_16bit_422_4x2_jpeg, lossless_rgb_16bit_422_4x2_rgb16, + lossless_rgb_16bit_422_restart_4x2_jpeg, lossless_rgb_8bit_420_4x4_jpeg, + lossless_rgb_8bit_420_4x4_rgb8, lossless_rgb_8bit_420_restart_4x4_jpeg, + lossless_rgb_8bit_422_4x2_jpeg, lossless_rgb_8bit_422_4x2_rgb8, + lossless_rgb_8bit_422_restart_4x2_jpeg, lossless_ycbcr_16bit_3x3_rgb16, + lossless_ycbcr_16bit_420_4x4_jpeg, lossless_ycbcr_16bit_420_4x4_rgb16, + lossless_ycbcr_16bit_420_restart_4x4_jpeg, lossless_ycbcr_16bit_422_4x2_jpeg, + lossless_ycbcr_16bit_422_4x2_rgb16, lossless_ycbcr_16bit_422_restart_4x2_jpeg, + lossless_ycbcr_3x3_rgb8, lossless_ycbcr_8bit_420_4x4_jpeg, lossless_ycbcr_8bit_420_4x4_rgb8, + lossless_ycbcr_8bit_420_restart_4x4_jpeg, lossless_ycbcr_8bit_422_4x2_jpeg, + lossless_ycbcr_8bit_422_4x2_rgb8, lossless_ycbcr_8bit_422_restart_4x2_jpeg, + malformed_cmyk_nondivisible_sampling_jpeg, minimal_baseline_420_jpeg, + progressive_12bit_cmyk_nonconstant_8x8_jpeg, progressive_12bit_grayscale_8x8_jpeg, + progressive_12bit_rgb_8x8_jpeg, progressive_12bit_ycck_nonconstant_8x8_jpeg, + progressive_8x8_jpeg, rgb_app14_8x8_jpeg, rgb_app14_8x8_rgb, ycck_16x16_420_jpeg, + ycck_16x8_422_jpeg, ycck_16x8_nonleading_max_422_jpeg, ycck_8x8_jpeg, + LOSSLESS_GRAYSCALE_16BIT_3X3_PIXELS, LOSSLESS_GRAYSCALE_3X3_PIXELS, + LOSSLESS_RGB_16BIT_3X3_PIXELS, LOSSLESS_RGB_3X3_PIXELS, +}; +use fixtures::{ + extended_12bit_cmyk_16x16_420_jpeg, extended_12bit_cmyk_16x8_422_jpeg, + extended_12bit_cmyk_420_restart_32x16_jpeg, extended_12bit_cmyk_422_restart_32x8_jpeg, + extended_12bit_cmyk_restart_16x8_jpeg, extended_12bit_rgb_32x32_rgb16, + extended_12bit_rgb_32x8_rgb16, extended_12bit_rgb_420_32x32_jpeg, + extended_12bit_rgb_422_32x8_jpeg, extended_12bit_rgb_8x8_jpeg, extended_12bit_rgb_8x8_rgb16, + extended_12bit_rgb_restart_16x8_jpeg, extended_12bit_rgb_restart_16x8_rgb16, + extended_12bit_ycbcr_420_32x32_jpeg, extended_12bit_ycbcr_420_32x32_rgb16, + extended_12bit_ycbcr_420_restart_32x32_jpeg, extended_12bit_ycbcr_420_restart_32x32_rgb16, + extended_12bit_ycbcr_422_32x8_jpeg, extended_12bit_ycbcr_422_32x8_rgb16, + extended_12bit_ycbcr_422_restart_32x8_jpeg, extended_12bit_ycbcr_422_restart_32x8_rgb16, + extended_12bit_ycbcr_8x8_jpeg, extended_12bit_ycbcr_8x8_rgb16, + extended_12bit_ycbcr_restart_16x8_jpeg, extended_12bit_ycbcr_restart_16x8_rgb16, + extended_12bit_ycck_16x16_420_jpeg, extended_12bit_ycck_16x8_422_jpeg, + extended_12bit_ycck_420_restart_32x16_jpeg, extended_12bit_ycck_422_restart_32x8_jpeg, + extended_12bit_ycck_restart_16x8_jpeg, progressive_12bit_cmyk_16x16_420_jpeg, + progressive_12bit_cmyk_16x8_422_jpeg, progressive_12bit_cmyk_420_restart_32x16_jpeg, + progressive_12bit_cmyk_422_restart_32x8_jpeg, progressive_12bit_cmyk_8x8_jpeg, + progressive_12bit_cmyk_restart_16x8_jpeg, progressive_12bit_rgb_420_32x32_jpeg, + progressive_12bit_rgb_422_32x8_jpeg, progressive_12bit_ycbcr_420_32x32_jpeg, + progressive_12bit_ycbcr_422_32x8_jpeg, progressive_12bit_ycbcr_8x8_jpeg, + progressive_12bit_ycck_16x16_420_jpeg, progressive_12bit_ycck_16x8_422_jpeg, + progressive_12bit_ycck_420_restart_32x16_jpeg, progressive_12bit_ycck_422_restart_32x8_jpeg, + progressive_12bit_ycck_8x8_jpeg, progressive_12bit_ycck_restart_16x8_jpeg, +}; +use j2k_test_support as fixtures; +use j2k_test_support::{ + crop_interleaved_u16, crop_interleaved_u8, project_scaled_interleaved_u16, + project_scaled_interleaved_u8, rgb8_to_rgba8, scaled_rect_covering, PixelRect, +}; + +#[test] +fn decode_into_rgb8_returns_decoded_rect_full_image() { + let bytes = minimal_baseline_420_jpeg(); + let dec = Decoder::new(&bytes).expect("baseline 4:2:0 must construct"); + let (w, h) = dec.info().dimensions; + let mut buf = vec![0u8; (w * h * 3) as usize]; + let outcome = dec + .decode_into(&mut buf, (w * 3) as usize, PixelFormat::Rgb8) + .expect("baseline 4:2:0 decode must succeed"); + assert_eq!(outcome.decoded.w, w); + assert_eq!(outcome.decoded.h, h); +} + +#[test] +fn decode_owned_rgb8_matches_decode_into() { + let bytes = minimal_baseline_420_jpeg(); + let dec = Decoder::new(&bytes).expect("baseline 4:2:0 must construct"); + let (w, h) = dec.info().dimensions; + let mut expected = vec![0u8; (w * h * 3) as usize]; + let expected_outcome = dec + .decode_into(&mut expected, (w * 3) as usize, PixelFormat::Rgb8) + .expect("baseline 4:2:0 decode must succeed"); + + let (owned, outcome) = dec.decode(PixelFormat::Rgb8).unwrap(); + assert_eq!(owned, expected); + assert_eq!(outcome, expected_outcome); +} + +#[test] +fn decode_into_rgba8_writes_alpha_byte() { + let bytes = minimal_baseline_420_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let (w, h) = dec.info().dimensions; + let mut buf = vec![0u8; (w * h * 4) as usize]; + dec.decode_rgba8_into_with_alpha(&mut buf, (w * 4) as usize, 200) + .unwrap(); + for y in 0..h as usize { + for x in 0..w as usize { + let idx = (y * w as usize + x) * 4; + assert_eq!(buf[idx + 3], 200, "pixel ({x},{y}) alpha"); + } + } +} + +#[test] +fn decode_into_rgba8_defaults_alpha_to_255() { + let bytes = minimal_baseline_420_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let (w, h) = dec.info().dimensions; + let mut buf = vec![0u8; (w * h * 4) as usize]; + dec.decode_into(&mut buf, (w * 4) as usize, PixelFormat::Rgba8) + .unwrap(); + for y in 0..h as usize { + for x in 0..w as usize { + let idx = (y * w as usize + x) * 4; + assert_eq!(buf[idx + 3], 255, "pixel ({x},{y}) alpha"); + } + } +} + +#[test] +fn decode_owned_region_scaled_matches_decode_region_into() { + let bytes = rgb_app14_8x8_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let roi = Rect { + x: 2, + y: 2, + w: 4, + h: 4, + }; + let mut expected = vec![0u8; 2 * 2 * 3]; + let expected_outcome = dec + .decode_region_scaled_into( + &mut expected, + 2 * 3, + PixelFormat::Rgb8, + roi, + Downscale::Half, + ) + .unwrap(); + + let (owned, outcome) = dec + .decode_region_scaled(PixelFormat::Rgb8, roi, Downscale::Half) + .unwrap(); + assert_eq!(owned, expected); + assert_eq!(outcome, expected_outcome); +} + +#[test] +fn decode_owned_scaled_matches_decode_scaled_into() { + let bytes = rgb_app14_8x8_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let mut expected = vec![0u8; 4 * 4 * 3]; + let expected_outcome = dec + .decode_scaled_into(&mut expected, 4 * 3, PixelFormat::Rgb8, Downscale::Half) + .unwrap(); + + let (owned, outcome) = dec + .decode_scaled(PixelFormat::Rgb8, Downscale::Half) + .unwrap(); + assert_eq!(owned, expected); + assert_eq!(outcome, expected_outcome); +} + +#[test] +fn full_tile_region_scaled_matches_full_scaled_decode() { + let bytes = minimal_baseline_420_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let (w, h) = dec.info().dimensions; + let roi = Rect { x: 0, y: 0, w, h }; + let stride = w.div_ceil(4) as usize * 3; + let mut expected = vec![0u8; stride * h.div_ceil(4) as usize]; + let expected_outcome = dec + .decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, Downscale::Quarter) + .unwrap(); + let mut actual = vec![0u8; expected.len()]; + + let actual_outcome = dec + .decode_region_scaled_into( + &mut actual, + stride, + PixelFormat::Rgb8, + roi, + Downscale::Quarter, + ) + .unwrap(); + + assert_eq!(actual, expected); + assert_eq!(actual_outcome, expected_outcome); + assert_eq!(actual_outcome.decoded, roi); +} + +#[test] +fn decode_into_gray8_produces_single_byte_per_pixel() { + let bytes = grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let (w, h) = dec.info().dimensions; + assert_eq!((w, h), (8, 8)); + let mut buf = vec![0u8; (w * h) as usize]; + let outcome = dec + .decode_into(&mut buf, w as usize, PixelFormat::Gray8) + .unwrap(); + assert_eq!(outcome.decoded.w, 8); + assert!(buf.iter().any(|&b| b != 0), "expected non-zero pixels"); +} + +#[test] +fn decode_into_gray16_accepts_extended12_grayscale_samples() { + let bytes = extended_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended grayscale JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Gray16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Gray16) + .expect("12-bit grayscale decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + for sample in buf.chunks_exact(2) { + assert_eq!(u16::from_le_bytes([sample[0], sample[1]]), 2048); + } +} + +#[test] +fn decode_region_into_gray16_crops_extended12_grayscale_samples() { + let bytes = extended_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended grayscale JPEG must construct"); + let roi = Rect { + x: 2, + y: 1, + w: 3, + h: 4, + }; + let stride = roi.w as usize * PixelFormat::Gray16.bytes_per_pixel() + 4; + let mut buf = vec![0xaau8; stride * roi.h as usize]; + + let outcome = dec + .decode_region_into(&mut buf, stride, PixelFormat::Gray16, roi) + .expect("12-bit grayscale ROI decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for sample in row[..roi.w as usize * 2].chunks_exact(2) { + assert_eq!(u16::from_le_bytes([sample[0], sample[1]]), 2048); + } + assert_eq!(&row[roi.w as usize * 2..], &[0xaa; 4]); + } +} + +#[test] +fn decode_scaled_into_gray16_projects_extended12_grayscale_samples() { + let bytes = extended_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended grayscale JPEG must construct"); + let scale = Downscale::Half; + let scaled_w = 4; + let scaled_h = 4; + let stride = scaled_w as usize * PixelFormat::Gray16.bytes_per_pixel() + 4; + let mut buf = vec![0xaau8; stride * scaled_h as usize]; + + let outcome = dec + .decode_scaled_into(&mut buf, stride, PixelFormat::Gray16, scale) + .expect("12-bit grayscale scaled decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full(dec.info().dimensions)); + for row in buf.chunks_exact(stride) { + for sample in row[..scaled_w as usize * 2].chunks_exact(2) { + assert_eq!(u16::from_le_bytes([sample[0], sample[1]]), 2048); + } + assert_eq!(&row[scaled_w as usize * 2..], &[0xaa; 4]); + } +} + +#[test] +fn decode_into_gray16_accepts_extended12_restart_grayscale_samples() { + let bytes = extended_12bit_grayscale_restart_16x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended restart grayscale JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Gray16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Gray16) + .expect("12-bit restart grayscale decode must succeed"); + + assert_eq!(dec.info().restart_interval, Some(1)); + assert_eq!(outcome.decoded, Rect::full((w, h))); + for sample in buf.chunks_exact(2) { + assert_eq!(u16::from_le_bytes([sample[0], sample[1]]), 2048); + } +} + +#[test] +fn decode_region_scaled_into_gray16_accepts_extended12_restart_grayscale_samples() { + let bytes = extended_12bit_grayscale_restart_16x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended restart grayscale JPEG must construct"); + let roi = Rect { + x: 2, + y: 1, + w: 12, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Gray16.bytes_per_pixel() + 4; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Gray16, roi, Downscale::Half) + .expect("12-bit restart grayscale region-scaled decode must succeed"); + + assert_eq!(dec.info().restart_interval, Some(1)); + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for sample in row[..scaled_roi.w as usize * 2].chunks_exact(2) { + assert_eq!(u16::from_le_bytes([sample[0], sample[1]]), 2048); + } + assert_eq!(&row[scaled_roi.w as usize * 2..], &[0xaa; 4]); + } +} + +#[test] +fn decode_region_scaled_into_rgb16_accepts_extended12_restart_grayscale_samples() { + let bytes = extended_12bit_grayscale_restart_16x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended restart grayscale JPEG must construct"); + let roi = Rect { + x: 2, + y: 1, + w: 12, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit restart grayscale region-scaled Rgb16 decode must succeed"); + + assert_eq!(dec.info().restart_interval, Some(1)); + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for pixel in row[..scaled_roi.w as usize * 6].chunks_exact(6) { + let channels = [ + u16::from_le_bytes([pixel[0], pixel[1]]), + u16::from_le_bytes([pixel[2], pixel[3]]), + u16::from_le_bytes([pixel[4], pixel[5]]), + ]; + assert_eq!(channels, [2048; 3]); + } + assert_eq!(&row[scaled_roi.w as usize * 6..], &[0xaa; 6]); + } +} + +#[test] +fn decode_into_gray16_accepts_progressive12_grayscale_samples() { + let bytes = progressive_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive grayscale JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Gray16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Gray16) + .expect("12-bit progressive Gray16 decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + for sample in buf.chunks_exact(2) { + assert_eq!(u16::from_le_bytes([sample[0], sample[1]]), 2048); + } +} + +#[test] +fn decode_region_into_gray16_crops_progressive12_grayscale_samples() { + let bytes = progressive_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive grayscale JPEG must construct"); + let roi = Rect { + x: 2, + y: 1, + w: 3, + h: 4, + }; + let stride = roi.w as usize * PixelFormat::Gray16.bytes_per_pixel() + 4; + let mut buf = vec![0xaau8; stride * roi.h as usize]; + + let outcome = dec + .decode_region_into(&mut buf, stride, PixelFormat::Gray16, roi) + .expect("12-bit progressive Gray16 ROI decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for sample in row[..roi.w as usize * 2].chunks_exact(2) { + assert_eq!(u16::from_le_bytes([sample[0], sample[1]]), 2048); + } + assert_eq!(&row[roi.w as usize * 2..], &[0xaa; 4]); + } +} + +#[test] +fn decode_scaled_into_gray16_projects_progressive12_grayscale_samples() { + let bytes = progressive_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive grayscale JPEG must construct"); + let scaled_w = 4; + let scaled_h = 4; + let stride = scaled_w as usize * PixelFormat::Gray16.bytes_per_pixel() + 4; + let mut buf = vec![0xaau8; stride * scaled_h as usize]; + + let outcome = dec + .decode_scaled_into(&mut buf, stride, PixelFormat::Gray16, Downscale::Half) + .expect("12-bit progressive Gray16 scaled decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full(dec.info().dimensions)); + for row in buf.chunks_exact(stride) { + for sample in row[..scaled_w as usize * 2].chunks_exact(2) { + assert_eq!(u16::from_le_bytes([sample[0], sample[1]]), 2048); + } + assert_eq!(&row[scaled_w as usize * 2..], &[0xaa; 4]); + } +} + +#[test] +fn decode_region_scaled_into_gray16_projects_progressive12_grayscale_samples() { + let bytes = progressive_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive grayscale JPEG must construct"); + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Gray16.bytes_per_pixel() + 4; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Gray16, roi, Downscale::Half) + .expect("12-bit progressive Gray16 region-scaled decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for sample in row[..scaled_roi.w as usize * 2].chunks_exact(2) { + assert_eq!(u16::from_le_bytes([sample[0], sample[1]]), 2048); + } + assert_eq!(&row[scaled_roi.w as usize * 2..], &[0xaa; 4]); + } +} + +#[test] +fn decode_into_rgb16_expands_progressive12_grayscale_samples() { + let bytes = progressive_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive grayscale JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit progressive Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + for pixel in buf.chunks_exact(6) { + let channels = [ + u16::from_le_bytes([pixel[0], pixel[1]]), + u16::from_le_bytes([pixel[2], pixel[3]]), + u16::from_le_bytes([pixel[4], pixel[5]]), + ]; + assert_eq!(channels, [2048; 3]); + } +} + +#[test] +fn decode_region_scaled_into_rgb16_projects_progressive12_grayscale_samples() { + let bytes = progressive_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive grayscale JPEG must construct"); + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit progressive region-scaled Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for pixel in row[..scaled_roi.w as usize * 6].chunks_exact(6) { + let channels = [ + u16::from_le_bytes([pixel[0], pixel[1]]), + u16::from_le_bytes([pixel[2], pixel[3]]), + u16::from_le_bytes([pixel[4], pixel[5]]), + ]; + assert_eq!(channels, [2048; 3]); + } + assert_eq!(&row[scaled_roi.w as usize * 6..], &[0xaa; 6]); + } +} + +#[test] +fn decode_region_scaled_into_rgba16_projects_12bit_grayscale_samples() { + for (bytes, roi, label) in [ + ( + extended_12bit_grayscale_restart_16x8_jpeg(), + Rect { + x: 2, + y: 1, + w: 12, + h: 6, + }, + "extended restart", + ), + ( + progressive_12bit_grayscale_8x8_jpeg(), + Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }, + "progressive", + ), + ] { + let dec = Decoder::new(&bytes) + .unwrap_or_else(|err| panic!("12-bit {label} grayscale JPEG must construct: {err}")); + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let row_bytes = scaled_roi.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let stride = row_bytes + 8; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgba16, roi, Downscale::Half) + .unwrap_or_else(|err| { + panic!("12-bit {label} grayscale RGBA16 region-scaled decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, roi, "{label}"); + for row in buf.chunks_exact(stride) { + for pixel in row[..row_bytes].chunks_exact(8) { + let channels = [ + u16::from_le_bytes([pixel[0], pixel[1]]), + u16::from_le_bytes([pixel[2], pixel[3]]), + u16::from_le_bytes([pixel[4], pixel[5]]), + u16::from_le_bytes([pixel[6], pixel[7]]), + ]; + assert_eq!(channels, [2048, 2048, 2048, u16::MAX], "{label}"); + } + assert_eq!(&row[row_bytes..], &[0xaa; 8], "{label}"); + } + } +} + +#[test] +fn decode_into_rgb16_accepts_progressive12_app14_rgb_samples() { + let bytes = progressive_12bit_rgb_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive APP14 RGB JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit progressive APP14 RGB decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, extended_12bit_rgb_8x8_rgb16()); +} + +#[test] +fn decode_region_scaled_into_rgb16_projects_progressive12_app14_rgb_samples() { + let bytes = progressive_12bit_rgb_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive APP14 RGB JPEG must construct"); + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + let expected_pixel = [2064u16, 2072, 2032] + .into_iter() + .flat_map(u16::to_le_bytes) + .collect::>(); + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit progressive APP14 RGB region-scaled decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for pixel in row[..scaled_roi.w as usize * 6].chunks_exact(6) { + assert_eq!(pixel, expected_pixel.as_slice()); + } + assert_eq!(&row[scaled_roi.w as usize * 6..], &[0xaa; 6]); + } +} + +#[test] +fn decode_into_rgb16_converts_progressive12_ycbcr444_samples() { + let bytes = progressive_12bit_ycbcr_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive YCbCr JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit progressive YCbCr Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, extended_12bit_ycbcr_8x8_rgb16()); +} + +#[test] +fn decode_region_scaled_into_rgb16_converts_progressive12_ycbcr444_samples() { + let bytes = progressive_12bit_ycbcr_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive YCbCr JPEG must construct"); + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + let expected_pixel = [2042u16, 2067, 2107] + .into_iter() + .flat_map(u16::to_le_bytes) + .collect::>(); + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit progressive YCbCr region-scaled Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for pixel in row[..scaled_roi.w as usize * 6].chunks_exact(6) { + assert_eq!(pixel, expected_pixel.as_slice()); + } + assert_eq!(&row[scaled_roi.w as usize * 6..], &[0xaa; 6]); + } +} + +#[test] +fn decode_into_rgb16_converts_progressive12_ycbcr422_samples() { + let bytes = progressive_12bit_ycbcr_422_32x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive YCbCr 4:2:2 JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit progressive YCbCr 4:2:2 Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, extended_12bit_ycbcr_422_32x8_rgb16()); +} + +#[test] +fn decode_region_scaled_into_rgb16_converts_progressive12_ycbcr422_samples() { + let bytes = progressive_12bit_ycbcr_422_32x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive YCbCr 4:2:2 JPEG must construct"); + let roi = Rect { + x: 13, + y: 0, + w: 8, + h: 4, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + let full = extended_12bit_ycbcr_422_32x8_rgb16(); + let expected_pixels = expected_scaled_rgb16_pixels(&full, 32, roi, 2); + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit progressive YCbCr 4:2:2 region-scaled Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, roi); + assert_padded_rgb16_rows(&buf, stride, scaled_roi.w as usize, &expected_pixels); +} + +#[test] +fn decode_into_rgb16_converts_progressive12_ycbcr420_samples() { + let bytes = progressive_12bit_ycbcr_420_32x32_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive YCbCr 4:2:0 JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit progressive YCbCr 4:2:0 Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_rgb16_image_eq(&buf, &extended_12bit_ycbcr_420_32x32_rgb16(), w as usize); +} + +#[test] +fn decode_region_scaled_into_rgb16_converts_progressive12_ycbcr420_samples() { + let bytes = progressive_12bit_ycbcr_420_32x32_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit progressive YCbCr 4:2:0 JPEG must construct"); + let roi = Rect { + x: 13, + y: 14, + w: 10, + h: 10, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + let full = extended_12bit_ycbcr_420_32x32_rgb16(); + let expected_pixels = expected_scaled_rgb16_pixels(&full, 32, roi, 2); + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit progressive YCbCr 4:2:0 region-scaled Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, roi); + assert_padded_rgb16_rows(&buf, stride, scaled_roi.w as usize, &expected_pixels); +} + +#[test] +fn decode_region_scaled_into_rgba16_projects_progressive12_color_samples() { + for (bytes, full, full_width, roi, label) in [ + ( + progressive_12bit_rgb_8x8_jpeg(), + extended_12bit_rgb_8x8_rgb16(), + 8, + Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }, + "APP14 RGB", + ), + ( + progressive_12bit_ycbcr_420_32x32_jpeg(), + extended_12bit_ycbcr_420_32x32_rgb16(), + 32, + Rect { + x: 13, + y: 14, + w: 10, + h: 10, + }, + "YCbCr 4:2:0", + ), + ] { + let dec = Decoder::new(&bytes) + .unwrap_or_else(|err| panic!("12-bit progressive {label} JPEG must construct: {err}")); + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let row_bytes = scaled_roi.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let stride = row_bytes + 8; + let expected_rgb = expected_scaled_rgb16_pixels(&full, full_width, roi, 2); + let expected = rgb16_to_rgba16(&expected_rgb, u16::MAX); + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgba16, roi, Downscale::Half) + .unwrap_or_else(|err| { + panic!("12-bit progressive {label} RGBA16 region-scaled decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, roi, "{label}"); + assert_padded_rgba16_rows(&buf, stride, scaled_roi.w as usize, &expected); + } +} + +#[test] +fn decode_into_rgb16_expands_extended12_grayscale_samples() { + let bytes = extended_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended grayscale JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit grayscale Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + for pixel in buf.chunks_exact(6) { + let channels = [ + u16::from_le_bytes([pixel[0], pixel[1]]), + u16::from_le_bytes([pixel[2], pixel[3]]), + u16::from_le_bytes([pixel[4], pixel[5]]), + ]; + assert_eq!(channels, [2048; 3]); + } +} + +#[test] +fn decode_region_into_rgb16_crops_extended12_grayscale_samples() { + let bytes = extended_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended grayscale JPEG must construct"); + let roi = Rect { + x: 1, + y: 2, + w: 4, + h: 3, + }; + let stride = roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * roi.h as usize]; + + let outcome = dec + .decode_region_into(&mut buf, stride, PixelFormat::Rgb16, roi) + .expect("12-bit grayscale Rgb16 ROI decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for pixel in row[..roi.w as usize * 6].chunks_exact(6) { + let channels = [ + u16::from_le_bytes([pixel[0], pixel[1]]), + u16::from_le_bytes([pixel[2], pixel[3]]), + u16::from_le_bytes([pixel[4], pixel[5]]), + ]; + assert_eq!(channels, [2048; 3]); + } + assert_eq!(&row[roi.w as usize * 6..], &[0xaa; 6]); + } +} + +#[test] +fn decode_region_scaled_into_rgb16_projects_extended12_grayscale_samples() { + let bytes = extended_12bit_grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended grayscale JPEG must construct"); + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit grayscale region-scaled Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for pixel in row[..scaled_roi.w as usize * 6].chunks_exact(6) { + let channels = [ + u16::from_le_bytes([pixel[0], pixel[1]]), + u16::from_le_bytes([pixel[2], pixel[3]]), + u16::from_le_bytes([pixel[4], pixel[5]]), + ]; + assert_eq!(channels, [2048; 3]); + } + assert_eq!(&row[scaled_roi.w as usize * 6..], &[0xaa; 6]); + } +} + +#[test] +fn decode_into_rgba16_accepts_extended12_color_samples() { + for (bytes, expected_rgb, label) in [ + ( + extended_12bit_rgb_8x8_jpeg(), + extended_12bit_rgb_8x8_rgb16(), + "APP14 RGB", + ), + ( + extended_12bit_ycbcr_420_32x32_jpeg(), + extended_12bit_ycbcr_420_32x32_rgb16(), + "YCbCr 4:2:0", + ), + ] { + let dec = Decoder::new(&bytes) + .unwrap_or_else(|err| panic!("12-bit extended {label} JPEG must construct: {err}")); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgba16) + .unwrap_or_else(|err| { + panic!("12-bit extended {label} RGBA16 decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, Rect::full((w, h)), "{label}"); + assert_eq!(buf, rgb16_to_rgba16(&expected_rgb, u16::MAX), "{label}"); + } +} + +#[test] +fn decode_into_rgb16_accepts_extended12_app14_rgb_samples() { + let bytes = extended_12bit_rgb_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended APP14 RGB JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit APP14 RGB decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, extended_12bit_rgb_8x8_rgb16()); +} + +#[test] +fn decode_into_rgb16_accepts_extended12_restart_app14_rgb_samples() { + let bytes = extended_12bit_rgb_restart_16x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended restart APP14 RGB JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit restart APP14 RGB decode must succeed"); + + assert_eq!(dec.info().restart_interval, Some(1)); + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, extended_12bit_rgb_restart_16x8_rgb16()); +} + +#[test] +fn decode_region_scaled_into_rgb16_projects_extended12_app14_rgb_samples() { + let bytes = extended_12bit_rgb_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended APP14 RGB JPEG must construct"); + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + let expected_pixel = [2064u16, 2072, 2032] + .into_iter() + .flat_map(u16::to_le_bytes) + .collect::>(); + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit APP14 RGB region-scaled decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for pixel in row[..scaled_roi.w as usize * 6].chunks_exact(6) { + assert_eq!(pixel, expected_pixel.as_slice()); + } + assert_eq!(&row[scaled_roi.w as usize * 6..], &[0xaa; 6]); + } +} + +#[test] +fn decode_region_scaled_into_rgba16_projects_extended12_restart_color_samples() { + for (bytes, full, full_width, roi, label) in [ + ( + extended_12bit_rgb_restart_16x8_jpeg(), + extended_12bit_rgb_restart_16x8_rgb16(), + 16, + Rect { + x: 2, + y: 1, + w: 12, + h: 6, + }, + "APP14 RGB", + ), + ( + extended_12bit_ycbcr_420_restart_32x32_jpeg(), + extended_12bit_ycbcr_420_restart_32x32_rgb16(), + 32, + Rect { + x: 13, + y: 14, + w: 10, + h: 10, + }, + "YCbCr 4:2:0", + ), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("12-bit extended restart {label} JPEG must construct: {err}") + }); + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let row_bytes = scaled_roi.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let stride = row_bytes + 8; + let expected_rgb = expected_scaled_rgb16_pixels(&full, full_width, roi, 2); + let expected = rgb16_to_rgba16(&expected_rgb, u16::MAX); + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgba16, roi, Downscale::Half) + .unwrap_or_else(|err| { + panic!( + "12-bit extended restart {label} RGBA16 region-scaled decode must succeed: {err}" + ) + }); + + assert_eq!(dec.info().restart_interval, Some(1), "{label}"); + assert_eq!(outcome.decoded, roi, "{label}"); + assert_padded_rgba16_rows(&buf, stride, scaled_roi.w as usize, &expected); + } +} + +#[test] +fn decode_region_scaled_into_rgb16_projects_extended12_restart_app14_rgb_samples() { + let bytes = extended_12bit_rgb_restart_16x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended restart APP14 RGB JPEG must construct"); + let roi = Rect { + x: 2, + y: 1, + w: 12, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + let full = extended_12bit_rgb_restart_16x8_rgb16(); + let expected_pixels = expected_scaled_rgb16_pixels(&full, 16, roi, 2); + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit restart APP14 RGB region-scaled decode must succeed"); + + assert_eq!(dec.info().restart_interval, Some(1)); + assert_eq!(outcome.decoded, roi); + assert_padded_rgb16_rows(&buf, stride, scaled_roi.w as usize, &expected_pixels); +} + +#[test] +fn decode_12bit_app14_rgb_subsampled_full_roi_scaled_and_region_scaled_outputs() { + for (bytes, expected_full, width, height, label) in [ + ( + extended_12bit_rgb_422_32x8_jpeg(), + extended_12bit_rgb_32x8_rgb16(), + 32, + 8, + "12-bit extended APP14 RGB 4:2:2", + ), + ( + extended_12bit_rgb_420_32x32_jpeg(), + extended_12bit_rgb_32x32_rgb16(), + 32, + 32, + "12-bit extended APP14 RGB 4:2:0", + ), + ( + progressive_12bit_rgb_422_32x8_jpeg(), + extended_12bit_rgb_32x8_rgb16(), + 32, + 8, + "12-bit progressive APP14 RGB 4:2:2", + ), + ( + progressive_12bit_rgb_420_32x32_jpeg(), + extended_12bit_rgb_32x32_rgb16(), + 32, + 32, + "12-bit progressive APP14 RGB 4:2:0", + ), + ] { + let dec = Decoder::new(&bytes) + .unwrap_or_else(|err| panic!("{label} decoder should construct: {err}")); + let full_rect = Rect::full((width, height)); + + let mut full = + vec![0u8; width as usize * height as usize * PixelFormat::Rgb16.bytes_per_pixel()]; + let outcome = dec + .decode_into( + &mut full, + width as usize * PixelFormat::Rgb16.bytes_per_pixel(), + PixelFormat::Rgb16, + ) + .unwrap_or_else(|err| panic!("{label} RGB16 full decode should succeed: {err}")); + assert_eq!(outcome.decoded, full_rect, "{label}"); + assert_eq!(full, expected_full, "{label}"); + + let roi = Rect { + x: 3, + y: 1, + w: 11, + h: 6, + }; + let expected_roi = crop_rgb16_bytes(&expected_full, width as usize, roi); + let mut roi_buf = vec![0u8; roi.w as usize * roi.h as usize * 6]; + let outcome = dec + .decode_region_into( + &mut roi_buf, + roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel(), + PixelFormat::Rgb16, + roi, + ) + .unwrap_or_else(|err| panic!("{label} RGB16 ROI decode should succeed: {err}")); + assert_eq!(outcome.decoded, roi, "{label}"); + assert_eq!(roi_buf, expected_roi, "{label}"); + + let scaled_rect = scaled_rect_covering_for_test(full_rect, 2); + let scaled_row_bytes = scaled_rect.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let scaled_stride = scaled_row_bytes + 8; + let expected_scaled = rgb16_to_rgba16( + &expected_scaled_rgb16_pixels(&expected_full, width as usize, full_rect, 2), + u16::MAX, + ); + let mut scaled = vec![0xaau8; scaled_stride * scaled_rect.h as usize]; + let outcome = dec + .decode_scaled_into( + &mut scaled, + scaled_stride, + PixelFormat::Rgba16, + Downscale::Half, + ) + .unwrap_or_else(|err| panic!("{label} RGBA16 scaled decode should succeed: {err}")); + assert_eq!(outcome.decoded, full_rect, "{label}"); + assert_padded_rgba16_rows( + &scaled, + scaled_stride, + scaled_rect.w as usize, + &expected_scaled, + ); + + let region_scaled = scaled_rect_covering_for_test(roi, 2); + let region_scaled_row_bytes = + region_scaled.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let region_scaled_stride = region_scaled_row_bytes + 8; + let expected_region_scaled = rgb16_to_rgba16( + &expected_scaled_rgb16_pixels(&expected_full, width as usize, roi, 2), + u16::MAX, + ); + let mut region_scaled_buf = vec![0xaau8; region_scaled_stride * region_scaled.h as usize]; + let outcome = dec + .decode_region_scaled_into( + &mut region_scaled_buf, + region_scaled_stride, + PixelFormat::Rgba16, + roi, + Downscale::Half, + ) + .unwrap_or_else(|err| { + panic!("{label} RGBA16 region-scaled decode should succeed: {err}") + }); + assert_eq!(outcome.decoded, roi, "{label}"); + assert_padded_rgba16_rows( + ®ion_scaled_buf, + region_scaled_stride, + region_scaled.w as usize, + &expected_region_scaled, + ); + } +} + +#[test] +fn decode_into_rgb16_converts_extended12_ycbcr444_samples() { + let bytes = extended_12bit_ycbcr_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended YCbCr JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit YCbCr Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, extended_12bit_ycbcr_8x8_rgb16()); +} + +#[test] +fn decode_into_rgb16_converts_extended12_restart_ycbcr444_samples() { + let bytes = extended_12bit_ycbcr_restart_16x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended restart YCbCr JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit restart YCbCr Rgb16 decode must succeed"); + + assert_eq!(dec.info().restart_interval, Some(1)); + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, extended_12bit_ycbcr_restart_16x8_rgb16()); +} + +#[test] +fn decode_region_scaled_into_rgb16_converts_extended12_ycbcr444_samples() { + let bytes = extended_12bit_ycbcr_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended YCbCr JPEG must construct"); + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + let expected_pixel = [2042u16, 2067, 2107] + .into_iter() + .flat_map(u16::to_le_bytes) + .collect::>(); + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit YCbCr region-scaled Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for pixel in row[..scaled_roi.w as usize * 6].chunks_exact(6) { + assert_eq!(pixel, expected_pixel.as_slice()); + } + assert_eq!(&row[scaled_roi.w as usize * 6..], &[0xaa; 6]); + } +} + +#[test] +fn decode_into_rgb16_converts_extended12_ycbcr422_samples() { + let bytes = extended_12bit_ycbcr_422_32x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended YCbCr 4:2:2 JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit YCbCr 4:2:2 Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, extended_12bit_ycbcr_422_32x8_rgb16()); +} + +#[test] +fn decode_into_rgb16_converts_extended12_restart_ycbcr422_samples() { + let bytes = extended_12bit_ycbcr_422_restart_32x8_jpeg(); + let dec = + Decoder::new(&bytes).expect("12-bit extended restart YCbCr 4:2:2 JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit restart YCbCr 4:2:2 Rgb16 decode must succeed"); + + assert_eq!(dec.info().restart_interval, Some(1)); + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, extended_12bit_ycbcr_422_restart_32x8_rgb16()); +} + +#[test] +fn decode_region_scaled_into_rgb16_converts_extended12_ycbcr422_samples() { + let bytes = extended_12bit_ycbcr_422_32x8_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended YCbCr 4:2:2 JPEG must construct"); + let roi = Rect { + x: 13, + y: 0, + w: 8, + h: 4, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + let full = extended_12bit_ycbcr_422_32x8_rgb16(); + let expected_pixels = expected_scaled_rgb16_pixels(&full, 32, roi, 2); + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit YCbCr 4:2:2 region-scaled Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, roi); + assert_padded_rgb16_rows(&buf, stride, scaled_roi.w as usize, &expected_pixels); +} + +#[test] +fn decode_into_rgb16_converts_extended12_ycbcr420_samples() { + let bytes = extended_12bit_ycbcr_420_32x32_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended YCbCr 4:2:0 JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit YCbCr 4:2:0 Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_rgb16_image_eq(&buf, &extended_12bit_ycbcr_420_32x32_rgb16(), w as usize); +} + +#[test] +fn decode_into_rgb16_converts_extended12_restart_ycbcr420_samples() { + let bytes = extended_12bit_ycbcr_420_restart_32x32_jpeg(); + let dec = + Decoder::new(&bytes).expect("12-bit extended restart YCbCr 4:2:0 JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .expect("12-bit restart YCbCr 4:2:0 Rgb16 decode must succeed"); + + assert_eq!(dec.info().restart_interval, Some(1)); + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_rgb16_image_eq( + &buf, + &extended_12bit_ycbcr_420_restart_32x32_rgb16(), + w as usize, + ); +} + +#[test] +fn decode_region_scaled_into_rgb16_converts_extended12_ycbcr420_samples() { + let bytes = extended_12bit_ycbcr_420_32x32_jpeg(); + let dec = Decoder::new(&bytes).expect("12-bit extended YCbCr 4:2:0 JPEG must construct"); + let roi = Rect { + x: 13, + y: 14, + w: 10, + h: 10, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + let full = extended_12bit_ycbcr_420_32x32_rgb16(); + let expected_pixels = expected_scaled_rgb16_pixels(&full, 32, roi, 2); + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit YCbCr 4:2:0 region-scaled Rgb16 decode must succeed"); + + assert_eq!(outcome.decoded, roi); + assert_padded_rgb16_rows(&buf, stride, scaled_roi.w as usize, &expected_pixels); +} + +#[test] +fn decode_region_scaled_into_rgb16_converts_extended12_restart_ycbcr420_samples() { + let bytes = extended_12bit_ycbcr_420_restart_32x32_jpeg(); + let dec = + Decoder::new(&bytes).expect("12-bit extended restart YCbCr 4:2:0 JPEG must construct"); + let roi = Rect { + x: 13, + y: 14, + w: 10, + h: 10, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + let full = extended_12bit_ycbcr_420_restart_32x32_rgb16(); + let expected_pixels = expected_scaled_rgb16_pixels(&full, 32, roi, 2); + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("12-bit restart YCbCr 4:2:0 region-scaled Rgb16 decode must succeed"); + + assert_eq!(dec.info().restart_interval, Some(1)); + assert_eq!(outcome.decoded, roi); + assert_padded_rgb16_rows(&buf, stride, scaled_roi.w as usize, &expected_pixels); +} + +#[test] +fn decode_into_gray8_accepts_lossless_grayscale_common_predictors() { + for predictor in 1..=7 { + let bytes = lossless_predictor_grayscale_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless predictor-{predictor} grayscale JPEG must construct: {err}") + }); + let (w, h) = dec.info().dimensions; + let mut buf = vec![0u8; (w * h) as usize]; + + let outcome = dec.decode_into(&mut buf, w as usize, PixelFormat::Gray8); + + assert_eq!( + outcome + .unwrap_or_else(|err| { + panic!("lossless predictor-{predictor} grayscale decode must succeed: {err}") + }) + .decoded, + Rect::full((w, h)) + ); + assert_eq!(buf, LOSSLESS_GRAYSCALE_3X3_PIXELS, "predictor {predictor}"); + } +} + +#[test] +fn decode_into_gray8_accepts_restart_coded_lossless_grayscale() { + for predictor in 1..=7 { + let bytes = lossless_restart_predictor_grayscale_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!( + "restart-coded lossless predictor-{predictor} grayscale JPEG must construct: {err}" + ) + }); + assert_eq!(dec.info().restart_interval, Some(3)); + let (w, h) = dec.info().dimensions; + let mut buf = vec![0u8; (w * h) as usize]; + + let outcome = dec + .decode_into(&mut buf, w as usize, PixelFormat::Gray8) + .unwrap_or_else(|err| { + panic!( + "restart-coded lossless predictor-{predictor} grayscale decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, LOSSLESS_GRAYSCALE_3X3_PIXELS, "predictor {predictor}"); + } +} + +#[test] +fn decode_into_rgb8_accepts_lossless_app14_rgb_common_predictors() { + for predictor in 1..=7 { + let bytes = lossless_predictor_rgb_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless predictor-{predictor} APP14 RGB JPEG must construct: {err}") + }); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb8) + .unwrap_or_else(|err| { + panic!("lossless predictor-{predictor} APP14 RGB decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, LOSSLESS_RGB_3X3_PIXELS, "predictor {predictor}"); + } +} + +#[test] +fn decode_into_rgb8_accepts_restart_coded_lossless_app14_rgb() { + for predictor in 1..=7 { + let bytes = lossless_restart_predictor_rgb_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!( + "restart-coded lossless predictor-{predictor} APP14 RGB JPEG must construct: {err}" + ) + }); + assert_eq!(dec.info().restart_interval, Some(3)); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb8) + .unwrap_or_else(|err| { + panic!( + "restart-coded lossless predictor-{predictor} APP14 RGB decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, LOSSLESS_RGB_3X3_PIXELS, "predictor {predictor}"); + } +} + +#[test] +fn decode_into_rgb8_converts_lossless_ycbcr_common_predictors() { + let expected = lossless_ycbcr_3x3_rgb8(); + for predictor in 1..=7 { + let bytes = lossless_predictor_ycbcr_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless predictor-{predictor} YCbCr JPEG must construct: {err}") + }); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb8) + .unwrap_or_else(|err| { + panic!("lossless predictor-{predictor} YCbCr decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, expected, "predictor {predictor}"); + } +} + +#[test] +fn decode_into_rgb8_converts_restart_coded_lossless_ycbcr() { + let expected = lossless_ycbcr_3x3_rgb8(); + for predictor in 1..=7 { + let bytes = lossless_restart_predictor_ycbcr_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("restart-coded lossless predictor-{predictor} YCbCr JPEG must construct: {err}") + }); + assert_eq!(dec.info().restart_interval, Some(3)); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb8) + .unwrap_or_else(|err| { + panic!( + "restart-coded lossless predictor-{predictor} YCbCr decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(buf, expected, "predictor {predictor}"); + } +} + +#[test] +fn decode_into_rgba8_accepts_lossless_color_common_predictors() { + for predictor in 1..=7 { + for (bytes, expected_rgb, label) in [ + ( + lossless_predictor_rgb_3x3_jpeg(predictor), + LOSSLESS_RGB_3X3_PIXELS.to_vec(), + "APP14 RGB", + ), + ( + lossless_predictor_ycbcr_3x3_jpeg(predictor), + lossless_ycbcr_3x3_rgb8(), + "YCbCr", + ), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless predictor-{predictor} {label} JPEG must construct: {err}") + }); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgba8) + .unwrap_or_else(|err| { + panic!( + "lossless predictor-{predictor} {label} RGBA8 decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!( + buf, + rgb8_to_rgba8(&expected_rgb, 255), + "{label} predictor {predictor}" + ); + } + } +} + +#[test] +fn decode_region_into_rgba8_crops_restart_coded_lossless_color() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let stride = roi.w as usize * PixelFormat::Rgba8.bytes_per_pixel() + 4; + + for (bytes, expected_rgb, label) in [ + ( + lossless_restart_predictor_rgb_3x3_jpeg(4), + LOSSLESS_RGB_3X3_PIXELS.to_vec(), + "APP14 RGB", + ), + ( + lossless_restart_predictor_ycbcr_3x3_jpeg(4), + lossless_ycbcr_3x3_rgb8(), + "YCbCr", + ), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("restart-coded lossless {label} JPEG must construct: {err}") + }); + let expected = rgb8_to_rgba8(&crop_rgb(&expected_rgb, 3, roi), 255); + let row_bytes = roi.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let mut buf = vec![0xaau8; stride * roi.h as usize]; + + let outcome = dec + .decode_region_into(&mut buf, stride, PixelFormat::Rgba8, roi) + .unwrap_or_else(|err| { + panic!("restart-coded lossless {label} RGBA8 ROI decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, roi); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(row_bytes)) + { + assert_eq!(&row[..row_bytes], expected_row); + assert_eq!(&row[row_bytes..], &[0xaa; 4]); + } + } +} + +#[test] +fn decode_scaled_into_rgba8_projects_lossless_color() { + let scaled = Rect { + x: 0, + y: 0, + w: 2, + h: 2, + }; + + for (bytes, expected_rgb, label) in [ + ( + lossless_predictor_rgb_3x3_jpeg(5), + LOSSLESS_RGB_3X3_PIXELS.to_vec(), + "APP14 RGB", + ), + ( + lossless_predictor_ycbcr_3x3_jpeg(5), + lossless_ycbcr_3x3_rgb8(), + "YCbCr", + ), + ] { + let dec = Decoder::new(&bytes) + .unwrap_or_else(|err| panic!("lossless {label} JPEG must construct: {err}")); + let stride = scaled.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let mut buf = vec![0u8; stride * scaled.h as usize]; + let expected = rgb8_to_rgba8(&project_scaled_rgb(&expected_rgb, 3, 3, scaled, 2), 255); + + let outcome = dec + .decode_scaled_into(&mut buf, stride, PixelFormat::Rgba8, Downscale::Half) + .unwrap_or_else(|err| { + panic!("lossless {label} RGBA8 scaled decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, Rect::full((3, 3))); + assert_eq!(buf, expected); + } +} + +#[test] +fn decode_region_scaled_into_rgba8_projects_restart_coded_lossless_color() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let row_bytes = scaled_roi.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let stride = row_bytes + 4; + + for (bytes, expected_rgb, label) in [ + ( + lossless_restart_predictor_rgb_3x3_jpeg(5), + LOSSLESS_RGB_3X3_PIXELS.to_vec(), + "APP14 RGB", + ), + ( + lossless_restart_predictor_ycbcr_3x3_jpeg(5), + lossless_ycbcr_3x3_rgb8(), + "YCbCr", + ), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("restart-coded lossless {label} JPEG must construct: {err}") + }); + let expected = rgb8_to_rgba8(&project_scaled_rgb(&expected_rgb, 3, 3, scaled_roi, 2), 255); + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgba8, roi, Downscale::Half) + .unwrap_or_else(|err| { + panic!( + "restart-coded lossless {label} RGBA8 region-scaled decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, roi); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(row_bytes)) + { + assert_eq!(&row[..row_bytes], expected_row); + assert_eq!(&row[row_bytes..], &[0xaa; 4]); + } + } +} + +#[test] +fn decode_into_rgb16_accepts_lossless_app14_rgb16_common_predictors() { + let expected = rgb16_samples_to_le_bytes(&LOSSLESS_RGB_16BIT_3X3_PIXELS); + for predictor in 1..=7 { + let bytes = lossless_predictor_rgb_16bit_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless 16-bit predictor-{predictor} APP14 RGB JPEG must construct: {err}") + }); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .unwrap_or_else(|err| { + panic!("lossless 16-bit predictor-{predictor} APP14 RGB decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_rgb16_image_eq(&buf, &expected, w as usize); + } +} + +#[test] +fn decode_into_rgb16_accepts_restart_coded_lossless_app14_rgb16() { + let expected = rgb16_samples_to_le_bytes(&LOSSLESS_RGB_16BIT_3X3_PIXELS); + for predictor in 1..=7 { + let bytes = lossless_restart_predictor_rgb_16bit_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!( + "restart-coded lossless 16-bit predictor-{predictor} APP14 RGB JPEG must construct: {err}" + ) + }); + assert_eq!(dec.info().restart_interval, Some(3)); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .unwrap_or_else(|err| { + panic!( + "restart-coded lossless 16-bit predictor-{predictor} APP14 RGB decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_rgb16_image_eq(&buf, &expected, w as usize); + } +} + +#[test] +fn decode_into_rgb16_converts_lossless_ycbcr16_common_predictors() { + let expected = lossless_ycbcr_16bit_3x3_rgb16(); + for predictor in 1..=7 { + let bytes = lossless_predictor_ycbcr_16bit_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless 16-bit predictor-{predictor} YCbCr JPEG must construct: {err}") + }); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .unwrap_or_else(|err| { + panic!("lossless 16-bit predictor-{predictor} YCbCr decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_rgb16_image_eq(&buf, &expected, w as usize); + } +} + +#[test] +fn decode_8bit_lossless_422_color_full_roi_scaled_and_region_scaled_outputs() { + let cases = [ + ( + "APP14 RGB", + lossless_rgb_8bit_422_4x2_jpeg(4), + lossless_rgb_8bit_422_4x2_rgb8(), + ), + ( + "APP14 RGB restart", + lossless_rgb_8bit_422_restart_4x2_jpeg(4), + lossless_rgb_8bit_422_4x2_rgb8(), + ), + ( + "YCbCr", + lossless_ycbcr_8bit_422_4x2_jpeg(4), + lossless_ycbcr_8bit_422_4x2_rgb8(), + ), + ( + "YCbCr restart", + lossless_ycbcr_8bit_422_restart_4x2_jpeg(4), + lossless_ycbcr_8bit_422_4x2_rgb8(), + ), + ]; + + for (label, bytes, expected_full) in cases { + assert_8bit_lossless_sampled_color_decode( + label, + &bytes, + &expected_full, + (4, 2), + &[(2, 1), (1, 1), (1, 1)], + Rect { + x: 1, + y: 0, + w: 2, + h: 2, + }, + ); + } +} + +#[test] +fn decode_8bit_lossless_420_color_full_roi_scaled_and_region_scaled_outputs() { + let cases = [ + ( + "APP14 RGB", + lossless_rgb_8bit_420_4x4_jpeg(4), + lossless_rgb_8bit_420_4x4_rgb8(), + ), + ( + "APP14 RGB restart", + lossless_rgb_8bit_420_restart_4x4_jpeg(4), + lossless_rgb_8bit_420_4x4_rgb8(), + ), + ( + "YCbCr", + lossless_ycbcr_8bit_420_4x4_jpeg(4), + lossless_ycbcr_8bit_420_4x4_rgb8(), + ), + ( + "YCbCr restart", + lossless_ycbcr_8bit_420_restart_4x4_jpeg(4), + lossless_ycbcr_8bit_420_4x4_rgb8(), + ), + ]; + + for (label, bytes, expected_full) in cases { + assert_8bit_lossless_sampled_color_decode( + label, + &bytes, + &expected_full, + (4, 4), + &[(2, 2), (1, 1), (1, 1)], + Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + ); + } +} + +fn assert_8bit_lossless_sampled_color_decode( + label: &str, + bytes: &[u8], + expected_full: &[u8], + dimensions: (u32, u32), + sampling: &[(u8, u8)], + roi: Rect, +) { + let dec = Decoder::new(bytes) + .unwrap_or_else(|err| panic!("8-bit lossless sampled {label} JPEG must construct: {err}")); + let (w, h) = dec.info().dimensions; + assert_eq!((w, h), dimensions, "{label}"); + assert_eq!(dec.info().sampling.components(), sampling, "{label}"); + + let stride = w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut full = vec![0u8; stride * h as usize]; + let outcome = dec + .decode_into(&mut full, stride, PixelFormat::Rgb8) + .unwrap_or_else(|err| panic!("8-bit lossless sampled {label} full decode: {err}")); + assert_eq!(outcome.decoded, Rect::full((w, h)), "{label}"); + assert_eq!(full, expected_full, "{label}"); + + let roi_stride = roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel() + 3; + let mut roi_buf = vec![0xaau8; roi_stride * roi.h as usize]; + let expected_roi = crop_rgb(expected_full, w, roi); + let outcome = dec + .decode_region_into(&mut roi_buf, roi_stride, PixelFormat::Rgb8, roi) + .unwrap_or_else(|err| panic!("8-bit lossless sampled {label} ROI decode: {err}")); + assert_eq!(outcome.decoded, roi, "{label}"); + let roi_row_bytes = roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + for (row, expected_row) in roi_buf + .chunks_exact(roi_stride) + .zip(expected_roi.chunks_exact(roi_row_bytes)) + { + assert_eq!(&row[..roi_row_bytes], expected_row, "{label}"); + assert_eq!(&row[roi_row_bytes..], &[0xaa; 3], "{label}"); + } + + let scaled = scaled_rect_covering_for_test(Rect::full((w, h)), 2); + let scaled_stride = scaled.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let mut scaled_buf = vec![0u8; scaled_stride * scaled.h as usize]; + let expected_scaled = rgb8_to_rgba8(&project_scaled_rgb(expected_full, w, h, scaled, 2), 255); + let outcome = dec + .decode_scaled_into( + &mut scaled_buf, + scaled_stride, + PixelFormat::Rgba8, + Downscale::Half, + ) + .unwrap_or_else(|err| panic!("8-bit lossless sampled {label} scaled decode: {err}")); + assert_eq!(outcome.decoded, Rect::full((w, h)), "{label}"); + assert_eq!(scaled_buf, expected_scaled, "{label}"); + + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let scaled_roi_stride = scaled_roi.w as usize * PixelFormat::Rgba8.bytes_per_pixel() + 4; + let scaled_roi_row_bytes = scaled_roi.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let mut scaled_roi_buf = vec![0xaau8; scaled_roi_stride * scaled_roi.h as usize]; + let expected_scaled_roi = + rgb8_to_rgba8(&project_scaled_rgb(expected_full, w, h, scaled_roi, 2), 255); + let outcome = dec + .decode_region_scaled_into( + &mut scaled_roi_buf, + scaled_roi_stride, + PixelFormat::Rgba8, + roi, + Downscale::Half, + ) + .unwrap_or_else(|err| panic!("8-bit lossless sampled {label} region-scaled decode: {err}")); + assert_eq!(outcome.decoded, roi, "{label}"); + for (row, expected_row) in scaled_roi_buf + .chunks_exact(scaled_roi_stride) + .zip(expected_scaled_roi.chunks_exact(scaled_roi_row_bytes)) + { + assert_eq!(&row[..scaled_roi_row_bytes], expected_row, "{label}"); + assert_eq!(&row[scaled_roi_row_bytes..], &[0xaa; 4], "{label}"); + } +} + +#[test] +fn decode_16bit_lossless_422_color_full_roi_scaled_and_region_scaled_outputs() { + let cases = [ + ( + "APP14 RGB", + lossless_rgb_16bit_422_4x2_jpeg(4), + lossless_rgb_16bit_422_4x2_rgb16(), + ), + ( + "APP14 RGB restart", + lossless_rgb_16bit_422_restart_4x2_jpeg(4), + lossless_rgb_16bit_422_4x2_rgb16(), + ), + ( + "YCbCr", + lossless_ycbcr_16bit_422_4x2_jpeg(4), + lossless_ycbcr_16bit_422_4x2_rgb16(), + ), + ( + "YCbCr restart", + lossless_ycbcr_16bit_422_restart_4x2_jpeg(4), + lossless_ycbcr_16bit_422_4x2_rgb16(), + ), + ]; + + for (label, bytes, expected_full) in cases { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("16-bit lossless 4:2:2 {label} JPEG must construct: {err}") + }); + let (w, h) = dec.info().dimensions; + assert_eq!((w, h), (4, 2), "{label}"); + assert_eq!(dec.info().sampling.max_h, 2, "{label}"); + assert_eq!(dec.info().sampling.max_v, 1, "{label}"); + assert_eq!( + dec.info().sampling.components(), + &[(2, 1), (1, 1), (1, 1)], + "{label}" + ); + + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut full = vec![0u8; stride * h as usize]; + let outcome = dec + .decode_into(&mut full, stride, PixelFormat::Rgb16) + .unwrap_or_else(|err| panic!("16-bit lossless 4:2:2 {label} full decode: {err}")); + assert_eq!(outcome.decoded, Rect::full((w, h)), "{label}"); + assert_rgb16_image_eq(&full, &expected_full, w as usize); + + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 2, + }; + let roi_stride = roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 4; + let mut roi_buf = vec![0xaau8; roi_stride * roi.h as usize]; + let expected_roi = crop_rgb16_bytes(&expected_full, w as usize, roi); + let outcome = dec + .decode_region_into(&mut roi_buf, roi_stride, PixelFormat::Rgb16, roi) + .unwrap_or_else(|err| panic!("16-bit lossless 4:2:2 {label} ROI decode: {err}")); + assert_eq!(outcome.decoded, roi, "{label}"); + let roi_row_bytes = roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + for (row, expected_row) in roi_buf + .chunks_exact(roi_stride) + .zip(expected_roi.chunks_exact(roi_row_bytes)) + { + assert_eq!(&row[..roi_row_bytes], expected_row, "{label}"); + assert_eq!(&row[roi_row_bytes..], &[0xaa; 4], "{label}"); + } + + let scaled = scaled_rect_covering_for_test(Rect::full((w, h)), 2); + let scaled_stride = scaled.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let mut scaled_buf = vec![0u8; scaled_stride * scaled.h as usize]; + let expected_scaled = rgb16_to_rgba16( + &expected_scaled_rgb16_pixels(&expected_full, w as usize, Rect::full((w, h)), 2), + u16::MAX, + ); + let outcome = dec + .decode_scaled_into( + &mut scaled_buf, + scaled_stride, + PixelFormat::Rgba16, + Downscale::Half, + ) + .unwrap_or_else(|err| { + panic!("16-bit lossless 4:2:2 {label} scaled RGBA16 decode: {err}") + }); + assert_eq!(outcome.decoded, Rect::full((w, h)), "{label}"); + assert_eq!(scaled_buf, expected_scaled, "{label}"); + + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let scaled_roi_stride = scaled_roi.w as usize * PixelFormat::Rgba16.bytes_per_pixel() + 8; + let scaled_roi_row_bytes = scaled_roi.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let mut scaled_roi_buf = vec![0xaau8; scaled_roi_stride * scaled_roi.h as usize]; + let expected_scaled_roi = rgb16_to_rgba16( + &expected_scaled_rgb16_pixels(&expected_full, w as usize, roi, 2), + u16::MAX, + ); + let outcome = dec + .decode_region_scaled_into( + &mut scaled_roi_buf, + scaled_roi_stride, + PixelFormat::Rgba16, + roi, + Downscale::Half, + ) + .unwrap_or_else(|err| { + panic!("16-bit lossless 4:2:2 {label} region-scaled RGBA16 decode: {err}") + }); + assert_eq!(outcome.decoded, roi, "{label}"); + for (row, expected_row) in scaled_roi_buf + .chunks_exact(scaled_roi_stride) + .zip(expected_scaled_roi.chunks_exact(scaled_roi_row_bytes)) + { + assert_eq!(&row[..scaled_roi_row_bytes], expected_row, "{label}"); + assert_eq!(&row[scaled_roi_row_bytes..], &[0xaa; 8], "{label}"); + } + } +} + +#[test] +fn decode_16bit_lossless_420_color_full_roi_scaled_and_region_scaled_outputs() { + let cases = [ + ( + "APP14 RGB", + lossless_rgb_16bit_420_4x4_jpeg(4), + lossless_rgb_16bit_420_4x4_rgb16(), + ), + ( + "APP14 RGB restart", + lossless_rgb_16bit_420_restart_4x4_jpeg(4), + lossless_rgb_16bit_420_4x4_rgb16(), + ), + ( + "YCbCr", + lossless_ycbcr_16bit_420_4x4_jpeg(4), + lossless_ycbcr_16bit_420_4x4_rgb16(), + ), + ( + "YCbCr restart", + lossless_ycbcr_16bit_420_restart_4x4_jpeg(4), + lossless_ycbcr_16bit_420_4x4_rgb16(), + ), + ]; + + for (label, bytes, expected_full) in cases { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("16-bit lossless 4:2:0 {label} JPEG must construct: {err}") + }); + let (w, h) = dec.info().dimensions; + assert_eq!((w, h), (4, 4), "{label}"); + assert_eq!(dec.info().sampling.max_h, 2, "{label}"); + assert_eq!(dec.info().sampling.max_v, 2, "{label}"); + assert_eq!( + dec.info().sampling.components(), + &[(2, 2), (1, 1), (1, 1)], + "{label}" + ); + + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut full = vec![0u8; stride * h as usize]; + let outcome = dec + .decode_into(&mut full, stride, PixelFormat::Rgb16) + .unwrap_or_else(|err| panic!("16-bit lossless 4:2:0 {label} full decode: {err}")); + assert_eq!(outcome.decoded, Rect::full((w, h)), "{label}"); + assert_rgb16_image_eq(&full, &expected_full, w as usize); + + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let roi_stride = roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 4; + let mut roi_buf = vec![0xaau8; roi_stride * roi.h as usize]; + let expected_roi = crop_rgb16_bytes(&expected_full, w as usize, roi); + let outcome = dec + .decode_region_into(&mut roi_buf, roi_stride, PixelFormat::Rgb16, roi) + .unwrap_or_else(|err| panic!("16-bit lossless 4:2:0 {label} ROI decode: {err}")); + assert_eq!(outcome.decoded, roi, "{label}"); + let roi_row_bytes = roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + for (row, expected_row) in roi_buf + .chunks_exact(roi_stride) + .zip(expected_roi.chunks_exact(roi_row_bytes)) + { + assert_eq!(&row[..roi_row_bytes], expected_row, "{label}"); + assert_eq!(&row[roi_row_bytes..], &[0xaa; 4], "{label}"); + } + + let scaled = scaled_rect_covering_for_test(Rect::full((w, h)), 2); + let scaled_stride = scaled.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let mut scaled_buf = vec![0u8; scaled_stride * scaled.h as usize]; + let expected_scaled = rgb16_to_rgba16( + &expected_scaled_rgb16_pixels(&expected_full, w as usize, Rect::full((w, h)), 2), + u16::MAX, + ); + let outcome = dec + .decode_scaled_into( + &mut scaled_buf, + scaled_stride, + PixelFormat::Rgba16, + Downscale::Half, + ) + .unwrap_or_else(|err| { + panic!("16-bit lossless 4:2:0 {label} scaled RGBA16 decode: {err}") + }); + assert_eq!(outcome.decoded, Rect::full((w, h)), "{label}"); + assert_eq!(scaled_buf, expected_scaled, "{label}"); + + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let scaled_roi_stride = scaled_roi.w as usize * PixelFormat::Rgba16.bytes_per_pixel() + 8; + let scaled_roi_row_bytes = scaled_roi.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let mut scaled_roi_buf = vec![0xaau8; scaled_roi_stride * scaled_roi.h as usize]; + let expected_scaled_roi = rgb16_to_rgba16( + &expected_scaled_rgb16_pixels(&expected_full, w as usize, roi, 2), + u16::MAX, + ); + let outcome = dec + .decode_region_scaled_into( + &mut scaled_roi_buf, + scaled_roi_stride, + PixelFormat::Rgba16, + roi, + Downscale::Half, + ) + .unwrap_or_else(|err| { + panic!("16-bit lossless 4:2:0 {label} region-scaled RGBA16 decode: {err}") + }); + assert_eq!(outcome.decoded, roi, "{label}"); + for (row, expected_row) in scaled_roi_buf + .chunks_exact(scaled_roi_stride) + .zip(expected_scaled_roi.chunks_exact(scaled_roi_row_bytes)) + { + assert_eq!(&row[..scaled_roi_row_bytes], expected_row, "{label}"); + assert_eq!(&row[scaled_roi_row_bytes..], &[0xaa; 8], "{label}"); + } + } +} + +#[test] +fn decode_into_rgb16_converts_restart_coded_lossless_ycbcr16() { + let expected = lossless_ycbcr_16bit_3x3_rgb16(); + for predictor in 1..=7 { + let bytes = lossless_restart_predictor_ycbcr_16bit_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!( + "restart-coded lossless 16-bit predictor-{predictor} YCbCr JPEG must construct: {err}" + ) + }); + assert_eq!(dec.info().restart_interval, Some(3)); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgb16) + .unwrap_or_else(|err| { + panic!( + "restart-coded lossless 16-bit predictor-{predictor} YCbCr decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_rgb16_image_eq(&buf, &expected, w as usize); + } +} + +#[test] +fn decode_into_rgba16_accepts_lossless_color_common_predictors() { + for predictor in 1..=7 { + for (bytes, expected_rgb, label) in [ + ( + lossless_predictor_rgb_16bit_3x3_jpeg(predictor), + rgb16_samples_to_le_bytes(&LOSSLESS_RGB_16BIT_3X3_PIXELS), + "APP14 RGB", + ), + ( + lossless_predictor_ycbcr_16bit_3x3_jpeg(predictor), + lossless_ycbcr_16bit_3x3_rgb16(), + "YCbCr", + ), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless 16-bit predictor-{predictor} {label} JPEG must construct: {err}") + }); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Rgba16) + .unwrap_or_else(|err| { + panic!( + "lossless 16-bit predictor-{predictor} {label} RGBA16 decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!( + buf, + rgb16_to_rgba16(&expected_rgb, u16::MAX), + "{label} predictor {predictor}" + ); + } + } +} + +#[test] +fn decode_region_into_rgba16_crops_restart_coded_lossless_color() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let row_bytes = roi.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let stride = row_bytes + 8; + + for (bytes, expected_rgb, label) in [ + ( + lossless_restart_predictor_rgb_16bit_3x3_jpeg(4), + rgb16_samples_to_le_bytes(&LOSSLESS_RGB_16BIT_3X3_PIXELS), + "APP14 RGB", + ), + ( + lossless_restart_predictor_ycbcr_16bit_3x3_jpeg(4), + lossless_ycbcr_16bit_3x3_rgb16(), + "YCbCr", + ), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("restart-coded lossless 16-bit {label} JPEG must construct: {err}") + }); + let cropped = crop_rgb16_bytes(&expected_rgb, 3, roi); + let expected = rgb16_to_rgba16(&cropped, u16::MAX); + let mut buf = vec![0xaau8; stride * roi.h as usize]; + + let outcome = dec + .decode_region_into(&mut buf, stride, PixelFormat::Rgba16, roi) + .unwrap_or_else(|err| { + panic!( + "restart-coded lossless 16-bit {label} RGBA16 ROI decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, roi); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(row_bytes)) + { + assert_eq!(&row[..row_bytes], expected_row); + assert_eq!(&row[row_bytes..], &[0xaa; 8]); + } + } +} + +#[test] +fn decode_scaled_into_rgba16_projects_lossless_color() { + let roi = Rect::full((3, 3)); + let scaled = scaled_rect_covering_for_test(roi, 2); + + for (bytes, expected_rgb, label) in [ + ( + lossless_predictor_rgb_16bit_3x3_jpeg(5), + rgb16_samples_to_le_bytes(&LOSSLESS_RGB_16BIT_3X3_PIXELS), + "APP14 RGB", + ), + ( + lossless_predictor_ycbcr_16bit_3x3_jpeg(5), + lossless_ycbcr_16bit_3x3_rgb16(), + "YCbCr", + ), + ] { + let dec = Decoder::new(&bytes) + .unwrap_or_else(|err| panic!("lossless 16-bit {label} JPEG must construct: {err}")); + let stride = scaled.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * scaled.h as usize]; + let expected = rgb16_to_rgba16( + &expected_scaled_rgb16_pixels(&expected_rgb, 3, roi, 2), + u16::MAX, + ); + + let outcome = dec + .decode_scaled_into(&mut buf, stride, PixelFormat::Rgba16, Downscale::Half) + .unwrap_or_else(|err| { + panic!("lossless 16-bit {label} RGBA16 scaled decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, roi); + assert_eq!(buf, expected); + } +} + +#[test] +fn decode_region_scaled_into_rgba16_projects_restart_coded_lossless_color() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let row_bytes = scaled_roi.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let stride = row_bytes + 8; + + for (bytes, expected_rgb, label) in [ + ( + lossless_restart_predictor_rgb_16bit_3x3_jpeg(5), + rgb16_samples_to_le_bytes(&LOSSLESS_RGB_16BIT_3X3_PIXELS), + "APP14 RGB", + ), + ( + lossless_restart_predictor_ycbcr_16bit_3x3_jpeg(5), + lossless_ycbcr_16bit_3x3_rgb16(), + "YCbCr", + ), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("restart-coded lossless 16-bit {label} JPEG must construct: {err}") + }); + let expected = rgb16_to_rgba16( + &expected_scaled_rgb16_pixels(&expected_rgb, 3, roi, 2), + u16::MAX, + ); + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgba16, roi, Downscale::Half) + .unwrap_or_else(|err| { + panic!( + "restart-coded lossless 16-bit {label} RGBA16 region-scaled decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, roi); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(row_bytes)) + { + assert_eq!(&row[..row_bytes], expected_row); + assert_eq!(&row[row_bytes..], &[0xaa; 8]); + } + } +} + +#[test] +fn decode_region_into_gray8_crops_lossless_grayscale_common_predictors() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let expected = crop_gray(&LOSSLESS_GRAYSCALE_3X3_PIXELS, 3, roi); + for predictor in 1..=7 { + let bytes = lossless_predictor_grayscale_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless predictor-{predictor} grayscale JPEG must construct: {err}") + }); + let stride = roi.w as usize + 2; + let mut buf = vec![0xaau8; stride * roi.h as usize]; + + let outcome = dec + .decode_region_into(&mut buf, stride, PixelFormat::Gray8, roi) + .unwrap_or_else(|err| { + panic!("lossless predictor-{predictor} grayscale ROI decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, roi); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(roi.w as usize)) + { + assert_eq!(&row[..roi.w as usize], expected_row); + assert_eq!(&row[roi.w as usize..], &[0xaa; 2]); + } + } +} + +#[test] +fn decode_scaled_into_gray8_projects_lossless_grayscale_common_predictors() { + let scale = Downscale::Half; + let scaled_w = 2; + let scaled_h = 2; + let expected = project_scaled_gray( + &LOSSLESS_GRAYSCALE_3X3_PIXELS, + 3, + 3, + Rect { + x: 0, + y: 0, + w: scaled_w, + h: scaled_h, + }, + 2, + ); + for predictor in 1..=7 { + let bytes = lossless_predictor_grayscale_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless predictor-{predictor} grayscale JPEG must construct: {err}") + }); + let stride = scaled_w as usize + 2; + let mut buf = vec![0xaau8; stride * scaled_h as usize]; + + let outcome = dec + .decode_scaled_into(&mut buf, stride, PixelFormat::Gray8, scale) + .unwrap_or_else(|err| { + panic!("lossless predictor-{predictor} grayscale scaled decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, Rect::full(dec.info().dimensions)); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(scaled_w as usize)) + { + assert_eq!(&row[..scaled_w as usize], expected_row); + assert_eq!(&row[scaled_w as usize..], &[0xaa; 2]); + } + } +} + +#[test] +fn decode_region_scaled_into_gray8_projects_lossless_grayscale_common_predictors() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let expected = project_scaled_gray(&LOSSLESS_GRAYSCALE_3X3_PIXELS, 3, 3, scaled_roi, 2); + for predictor in 1..=7 { + let bytes = lossless_predictor_grayscale_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless predictor-{predictor} grayscale JPEG must construct: {err}") + }); + let stride = scaled_roi.w as usize + 2; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Gray8, roi, Downscale::Half) + .unwrap_or_else(|err| { + panic!( + "lossless predictor-{predictor} grayscale region-scaled decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, roi); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(scaled_roi.w as usize)) + { + assert_eq!(&row[..scaled_roi.w as usize], expected_row); + assert_eq!(&row[scaled_roi.w as usize..], &[0xaa; 2]); + } + } +} + +#[test] +fn decode_region_scaled_into_gray8_projects_restart_coded_lossless_grayscale() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let expected = project_scaled_gray(&LOSSLESS_GRAYSCALE_3X3_PIXELS, 3, 3, scaled_roi, 2); + let bytes = lossless_restart_predictor_grayscale_3x3_jpeg(1); + let dec = Decoder::new(&bytes).expect("restart-coded lossless grayscale JPEG must construct"); + let stride = scaled_roi.w as usize + 2; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Gray8, roi, Downscale::Half) + .expect("restart-coded lossless grayscale region-scaled decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(scaled_roi.w as usize)) + { + assert_eq!(&row[..scaled_roi.w as usize], expected_row); + assert_eq!(&row[scaled_roi.w as usize..], &[0xaa; 2]); + } +} + +#[test] +fn decode_region_scaled_into_rgb8_projects_lossless_app14_rgb() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let expected = project_scaled_rgb(&LOSSLESS_RGB_3X3_PIXELS, 3, 3, scaled_roi, 2); + let bytes = lossless_predictor_rgb_3x3_jpeg(1); + let dec = Decoder::new(&bytes).expect("lossless APP14 RGB JPEG must construct"); + let stride = scaled_roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel() + 3; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb8, roi, Downscale::Half) + .expect("lossless APP14 RGB region-scaled decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(scaled_roi.w as usize * 3)) + { + assert_eq!(&row[..scaled_roi.w as usize * 3], expected_row); + assert_eq!(&row[scaled_roi.w as usize * 3..], &[0xaa; 3]); + } +} + +#[test] +fn decode_region_scaled_into_rgb8_projects_restart_coded_lossless_app14_rgb() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let expected = project_scaled_rgb(&LOSSLESS_RGB_3X3_PIXELS, 3, 3, scaled_roi, 2); + let bytes = lossless_restart_predictor_rgb_3x3_jpeg(1); + let dec = Decoder::new(&bytes).expect("restart-coded lossless APP14 RGB JPEG must construct"); + let stride = scaled_roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel() + 3; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb8, roi, Downscale::Half) + .expect("restart-coded lossless APP14 RGB region-scaled decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(scaled_roi.w as usize * 3)) + { + assert_eq!(&row[..scaled_roi.w as usize * 3], expected_row); + assert_eq!(&row[scaled_roi.w as usize * 3..], &[0xaa; 3]); + } +} + +#[test] +fn decode_region_scaled_into_rgb8_projects_lossless_ycbcr() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let full = lossless_ycbcr_3x3_rgb8(); + let expected = project_scaled_rgb(&full, 3, 3, scaled_roi, 2); + let bytes = lossless_predictor_ycbcr_3x3_jpeg(1); + let dec = Decoder::new(&bytes).expect("lossless YCbCr JPEG must construct"); + let stride = scaled_roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel() + 3; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb8, roi, Downscale::Half) + .expect("lossless YCbCr region-scaled decode must succeed"); + + assert_eq!(outcome.decoded, roi); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(scaled_roi.w as usize * 3)) + { + assert_eq!(&row[..scaled_roi.w as usize * 3], expected_row); + assert_eq!(&row[scaled_roi.w as usize * 3..], &[0xaa; 3]); + } +} + +#[test] +fn decode_region_scaled_into_rgb16_projects_lossless_app14_rgb16() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let full = rgb16_samples_to_le_bytes(&LOSSLESS_RGB_16BIT_3X3_PIXELS); + let expected = expected_scaled_rgb16_pixels(&full, 3, roi, 2); + let bytes = lossless_predictor_rgb_16bit_3x3_jpeg(1); + let dec = Decoder::new(&bytes).expect("lossless 16-bit APP14 RGB JPEG must construct"); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("lossless 16-bit APP14 RGB region-scaled decode must succeed"); + + assert_eq!(outcome.decoded, roi); + assert_padded_rgb16_rows(&buf, stride, scaled_roi.w as usize, &expected); +} + +#[test] +fn decode_region_scaled_into_rgb16_projects_lossless_ycbcr16() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let full = lossless_ycbcr_16bit_3x3_rgb16(); + let expected = expected_scaled_rgb16_pixels(&full, 3, roi, 2); + let bytes = lossless_predictor_ycbcr_16bit_3x3_jpeg(1); + let dec = Decoder::new(&bytes).expect("lossless 16-bit YCbCr JPEG must construct"); + let stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel() + 6; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb16, roi, Downscale::Half) + .expect("lossless 16-bit YCbCr region-scaled decode must succeed"); + + assert_eq!(outcome.decoded, roi); + assert_padded_rgb16_rows(&buf, stride, scaled_roi.w as usize, &expected); +} + +#[test] +fn decode_into_gray16_accepts_lossless_16bit_grayscale_common_predictors() { + for predictor in 1..=7 { + let bytes = lossless_predictor_grayscale_16bit_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless 16-bit predictor-{predictor} grayscale JPEG must construct: {err}") + }); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Gray16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Gray16) + .unwrap_or_else(|err| { + panic!("lossless 16-bit predictor-{predictor} Gray16 decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_gray16_samples( + &buf, + stride, + w, + &LOSSLESS_GRAYSCALE_16BIT_3X3_PIXELS, + predictor, + ); + } +} + +#[test] +fn decode_into_gray16_accepts_restart_coded_lossless_grayscale() { + for predictor in 1..=7 { + let bytes = lossless_restart_predictor_grayscale_16bit_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!( + "restart-coded 16-bit lossless predictor-{predictor} grayscale JPEG must construct: {err}" + ) + }); + assert_eq!(dec.info().restart_interval, Some(3)); + let (w, h) = dec.info().dimensions; + let stride = w as usize * PixelFormat::Gray16.bytes_per_pixel(); + let mut buf = vec![0u8; stride * h as usize]; + + let outcome = dec + .decode_into(&mut buf, stride, PixelFormat::Gray16) + .unwrap_or_else(|err| { + panic!( + "restart-coded 16-bit lossless predictor-{predictor} Gray16 decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_gray16_samples( + &buf, + stride, + w, + &LOSSLESS_GRAYSCALE_16BIT_3X3_PIXELS, + predictor, + ); + } +} + +#[test] +fn decode_region_into_gray16_crops_lossless_16bit_grayscale_common_predictors() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let expected = crop_gray16(&LOSSLESS_GRAYSCALE_16BIT_3X3_PIXELS, 3, roi); + for predictor in 1..=7 { + let bytes = lossless_predictor_grayscale_16bit_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless 16-bit predictor-{predictor} grayscale JPEG must construct: {err}") + }); + let stride = roi.w as usize * PixelFormat::Gray16.bytes_per_pixel() + 4; + let mut buf = vec![0xaau8; stride * roi.h as usize]; + + let outcome = dec + .decode_region_into(&mut buf, stride, PixelFormat::Gray16, roi) + .unwrap_or_else(|err| { + panic!( + "lossless 16-bit predictor-{predictor} Gray16 ROI decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, roi); + assert_gray16_rows_with_padding(&buf, stride, roi.w, &expected, 4, predictor); + } +} + +#[test] +fn decode_scaled_into_gray16_projects_lossless_16bit_grayscale_common_predictors() { + let scaled_w = 2; + let scaled_h = 2; + let expected = project_scaled_gray16( + &LOSSLESS_GRAYSCALE_16BIT_3X3_PIXELS, + 3, + 3, + Rect { + x: 0, + y: 0, + w: scaled_w, + h: scaled_h, + }, + 2, + ); + for predictor in 1..=7 { + let bytes = lossless_predictor_grayscale_16bit_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless 16-bit predictor-{predictor} grayscale JPEG must construct: {err}") + }); + let stride = scaled_w as usize * PixelFormat::Gray16.bytes_per_pixel() + 4; + let mut buf = vec![0xaau8; stride * scaled_h as usize]; + + let outcome = dec + .decode_scaled_into(&mut buf, stride, PixelFormat::Gray16, Downscale::Half) + .unwrap_or_else(|err| { + panic!("lossless 16-bit predictor-{predictor} Gray16 scaled decode must succeed: {err}") + }); + + assert_eq!(outcome.decoded, Rect::full(dec.info().dimensions)); + assert_gray16_rows_with_padding(&buf, stride, scaled_w, &expected, 4, predictor); + } +} + +#[test] +fn decode_region_scaled_into_gray16_projects_lossless_16bit_grayscale_common_predictors() { + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let expected = project_scaled_gray16(&LOSSLESS_GRAYSCALE_16BIT_3X3_PIXELS, 3, 3, scaled_roi, 2); + for predictor in 1..=7 { + let bytes = lossless_predictor_grayscale_16bit_3x3_jpeg(predictor); + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("lossless 16-bit predictor-{predictor} grayscale JPEG must construct: {err}") + }); + let stride = scaled_roi.w as usize * PixelFormat::Gray16.bytes_per_pixel() + 4; + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into( + &mut buf, + stride, + PixelFormat::Gray16, + roi, + Downscale::Half, + ) + .unwrap_or_else(|err| { + panic!( + "lossless 16-bit predictor-{predictor} Gray16 region-scaled decode must succeed: {err}" + ) + }); + + assert_eq!(outcome.decoded, roi); + assert_gray16_rows_with_padding(&buf, stride, scaled_roi.w, &expected, 4, predictor); + } +} + +#[test] +fn decode_into_rejects_undersized_buffer_with_api_misuse_error() { + let bytes = minimal_baseline_420_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let mut buf = vec![0u8; 4]; + let err = dec + .decode_into(&mut buf, 48, PixelFormat::Rgb8) + .unwrap_err(); + assert!(err.is_api_misuse()); + assert!(matches!(err, JpegError::OutputBufferTooSmall { .. })); +} + +#[test] +fn decode_into_rejects_stride_narrower_than_row_width() { + let bytes = minimal_baseline_420_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let mut buf = vec![0u8; 16 * 16 * 3]; + let err = dec + .decode_into(&mut buf, 10, PixelFormat::Rgb8) + .unwrap_err(); + assert!(err.is_api_misuse()); + assert!(matches!(err, JpegError::InvalidStride { .. })); +} + +#[test] +fn decode_into_tolerates_padded_stride() { + let bytes = minimal_baseline_420_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let (w, h) = dec.info().dimensions; + let padded_stride = (w as usize * 3) + 32; + let mut buf = vec![0xAAu8; padded_stride * h as usize]; + dec.decode_into(&mut buf, padded_stride, PixelFormat::Rgb8) + .unwrap(); + let last_row_start = (h as usize - 1) * padded_stride; + let last_row_end = last_row_start + w as usize * 3; + assert_eq!( + &buf[last_row_end..last_row_end + 16], + &[0xAA; 16], + "stride padding must not be overwritten" + ); +} + +#[test] +fn decode_into_rgb8_preserves_app14_rgb_pixels() { + let bytes = rgb_app14_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("APP14 RGB fixture must construct"); + let (w, h) = dec.info().dimensions; + assert_eq!((w, h), (8, 8)); + let mut buf = vec![0u8; (w * h * 3) as usize]; + dec.decode_into(&mut buf, (w * 3) as usize, PixelFormat::Rgb8) + .expect("APP14 RGB decode must succeed"); + assert_eq!(buf, rgb_app14_8x8_rgb()); +} + +#[test] +fn decode_into_rgb8_scaled_preserves_constant_app14_rgb_pixels() { + let bytes = rgb_app14_8x8_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + + for (factor, dims) in [ + (Downscale::Half, (4u32, 4u32)), + (Downscale::Quarter, (2u32, 2u32)), + (Downscale::Eighth, (1u32, 1u32)), + ] { + let mut buf = vec![0u8; dims.0 as usize * dims.1 as usize * 3]; + dec.decode_scaled_into(&mut buf, dims.0 as usize * 3, PixelFormat::Rgb8, factor) + .unwrap(); + let mut expected = Vec::with_capacity(buf.len()); + for _ in 0..(dims.0 * dims.1) { + expected.extend_from_slice(&[200, 20, 10]); + } + assert_eq!(buf, expected, "factor={factor:?}"); + } +} + +#[test] +fn decode_into_gray8_scaled_projects_constant_app14_rgb_pixels() { + let bytes = rgb_app14_8x8_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let expected = ((77 * 200 + 150 * 20 + 29 * 10 + 128) >> 8) as u8; + + for (factor, dims) in [ + (Downscale::Half, (4u32, 4u32)), + (Downscale::Quarter, (2u32, 2u32)), + (Downscale::Eighth, (1u32, 1u32)), + ] { + let mut buf = vec![0u8; dims.0 as usize * dims.1 as usize]; + dec.decode_scaled_into(&mut buf, dims.0 as usize, PixelFormat::Gray8, factor) + .unwrap(); + assert!(buf.iter().all(|&px| px == expected), "factor={factor:?}"); + } +} + +#[test] +fn decoder_new_accepts_progressive8() { + let bytes = progressive_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("progressive 8-bit JPEG must construct"); + + assert_eq!(dec.info().dimensions, (8, 8)); +} + +#[test] +fn decode_progressive8_rgb8_matches_jpeg_decoder_reference() { + let bytes = progressive_8x8_jpeg(); + let mut reference_decoder = jpeg_decoder::Decoder::new(std::io::Cursor::new(&bytes)); + let reference = reference_decoder + .decode() + .expect("jpeg-decoder reference decode"); + let info = reference_decoder.info().expect("jpeg-decoder info"); + assert_eq!((info.width, info.height), (8, 8)); + + let dec = Decoder::new(&bytes).expect("progressive 8-bit JPEG must construct"); + let (w, h) = dec.info().dimensions; + let mut actual = vec![0u8; (w * h * 3) as usize]; + dec.decode_into(&mut actual, (w * 3) as usize, PixelFormat::Rgb8) + .expect("progressive decode must succeed"); + + assert_eq!(actual.len(), reference.len()); + let max_delta = actual + .iter() + .zip(reference.iter()) + .map(|(&a, &b)| a.abs_diff(b)) + .max() + .unwrap_or(0); + assert!( + max_delta <= 3, + "progressive RGB max channel delta {max_delta} exceeds tolerance" + ); +} + +#[test] +fn decode_region_into_rgb8_crops_progressive8_pixels() { + let bytes = progressive_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("progressive 8-bit JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * 3; + let mut full = vec![0u8; stride * h as usize]; + dec.decode_into(&mut full, stride, PixelFormat::Rgb8) + .expect("full progressive decode must succeed"); + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let mut actual = vec![0u8; roi.w as usize * roi.h as usize * 3]; + + let outcome = dec + .decode_region_into(&mut actual, roi.w as usize * 3, PixelFormat::Rgb8, roi) + .expect("progressive ROI decode must succeed"); + + assert_eq!(outcome.decoded, roi); + assert_eq!(actual, crop_rgb(&full, w, roi)); +} + +#[test] +fn decode_scaled_into_rgb8_projects_progressive8_pixels() { + let bytes = progressive_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("progressive 8-bit JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * 3; + let mut full = vec![0u8; stride * h as usize]; + dec.decode_into(&mut full, stride, PixelFormat::Rgb8) + .expect("full progressive decode must succeed"); + let scale = Downscale::Half; + let scaled_w = w.div_ceil(2); + let scaled_h = h.div_ceil(2); + let scaled_rect = Rect { + x: 0, + y: 0, + w: scaled_w, + h: scaled_h, + }; + let mut actual = vec![0u8; scaled_w as usize * scaled_h as usize * 3]; + + let outcome = dec + .decode_scaled_into(&mut actual, scaled_w as usize * 3, PixelFormat::Rgb8, scale) + .expect("progressive scaled decode must succeed"); + + assert_eq!(outcome.decoded, Rect::full((w, h))); + assert_eq!(actual, project_scaled_rgb(&full, w, h, scaled_rect, 2)); +} + +#[test] +fn decode_region_scaled_into_rgb8_projects_progressive8_pixels() { + let bytes = progressive_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("progressive 8-bit JPEG must construct"); + let (w, h) = dec.info().dimensions; + let stride = w as usize * 3; + let mut full = vec![0u8; stride * h as usize]; + dec.decode_into(&mut full, stride, PixelFormat::Rgb8) + .expect("full progressive decode must succeed"); + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let mut actual = vec![0u8; scaled_roi.w as usize * scaled_roi.h as usize * 3]; + + let outcome = dec + .decode_region_scaled_into( + &mut actual, + scaled_roi.w as usize * 3, + PixelFormat::Rgb8, + roi, + Downscale::Half, + ) + .expect("progressive region-scaled decode must succeed"); + + assert_eq!(outcome.decoded, roi); + assert_eq!(actual, project_scaled_rgb(&full, w, h, scaled_roi, 2)); +} + +#[test] +fn decode_region_into_rgb8_crops_constant_app14_rgb_pixels() { + let bytes = rgb_app14_8x8_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let roi = Rect { + x: 2, + y: 1, + w: 3, + h: 4, + }; + let mut buf = vec![0u8; roi.w as usize * roi.h as usize * 3]; + let outcome = dec + .decode_region_into(&mut buf, roi.w as usize * 3, PixelFormat::Rgb8, roi) + .unwrap(); + assert_eq!(outcome.decoded, roi); + let mut expected = Vec::with_capacity(buf.len()); + for _ in 0..(roi.w * roi.h) { + expected.extend_from_slice(&[200, 20, 10]); + } + assert_eq!(buf, expected); +} + +fn crop_rgb(full: &[u8], width: u32, roi: Rect) -> Vec { + crop_interleaved_u8(full, width as usize, 3, pixel_rect(roi)) +} + +fn crop_gray(full: &[u8], width: u32, roi: Rect) -> Vec { + crop_interleaved_u8(full, width as usize, 1, pixel_rect(roi)) +} + +fn crop_gray16(full: &[u16], width: u32, roi: Rect) -> Vec { + crop_interleaved_u16(full, width as usize, 1, pixel_rect(roi)) +} + +fn project_scaled_rgb( + full: &[u8], + width: u32, + height: u32, + output_rect: Rect, + denom: u32, +) -> Vec { + project_scaled_interleaved_u8(full, width, height, 3, pixel_rect(output_rect), denom) +} + +fn project_scaled_gray( + full: &[u8], + width: u32, + height: u32, + output_rect: Rect, + denom: u32, +) -> Vec { + project_scaled_interleaved_u8(full, width, height, 1, pixel_rect(output_rect), denom) +} + +fn project_scaled_gray16( + full: &[u16], + width: u32, + height: u32, + output_rect: Rect, + denom: u32, +) -> Vec { + project_scaled_interleaved_u16(full, width, height, 1, pixel_rect(output_rect), denom) +} + +fn assert_gray16_samples(buf: &[u8], stride: usize, width: u32, expected: &[u16], context: u8) { + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(width as usize)) + { + for (sample, expected) in row[..width as usize * 2] + .chunks_exact(2) + .zip(expected_row.iter().copied()) + { + assert_eq!( + u16::from_le_bytes([sample[0], sample[1]]), + expected, + "predictor {context}" + ); + } + } +} + +fn assert_gray16_rows_with_padding( + buf: &[u8], + stride: usize, + width: u32, + expected: &[u16], + pad: usize, + context: u8, +) { + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(width as usize)) + { + for (sample, expected) in row[..width as usize * 2] + .chunks_exact(2) + .zip(expected_row.iter().copied()) + { + assert_eq!( + u16::from_le_bytes([sample[0], sample[1]]), + expected, + "predictor {context}" + ); + } + assert_eq!(&row[width as usize * 2..], vec![0xaa; pad].as_slice()); + } +} + +fn scaled_rect_covering_for_test(rect: Rect, denom: u32) -> Rect { + jpeg_rect(scaled_rect_covering(pixel_rect(rect), denom)) +} + +fn pixel_rect(rect: Rect) -> PixelRect { + PixelRect::new(rect.x, rect.y, rect.w, rect.h) +} + +fn jpeg_rect(rect: PixelRect) -> Rect { + Rect { + x: rect.x, + y: rect.y, + w: rect.w, + h: rect.h, + } +} + +fn expected_scaled_rgb16_pixels(full: &[u8], full_width: usize, roi: Rect, denom: u32) -> Vec { + let scaled = scaled_rect_covering_for_test(roi, denom); + let mut expected = Vec::with_capacity(scaled.w as usize * scaled.h as usize * 6); + for out_y in scaled.y..scaled.y + scaled.h { + let src_y = out_y.saturating_mul(denom).min(roi.y + roi.h - 1); + for out_x in scaled.x..scaled.x + scaled.w { + let src_x = out_x.saturating_mul(denom).min(roi.x + roi.w - 1); + let start = (src_y as usize * full_width + src_x as usize) * 6; + expected.extend_from_slice(&full[start..start + 6]); + } + } + expected +} + +fn assert_padded_rgb16_rows(buf: &[u8], stride: usize, width: usize, expected: &[u8]) { + let row_bytes = width * PixelFormat::Rgb16.bytes_per_pixel(); + for (row_index, row) in buf.chunks_exact(stride).enumerate() { + let start = row_index * row_bytes; + assert_rgb16_image_eq( + &row[..row_bytes], + &expected[start..start + row_bytes], + width, + ); + assert_eq!(&row[row_bytes..], &[0xaa; 6]); + } +} + +fn assert_padded_rgba16_rows(buf: &[u8], stride: usize, width: usize, expected: &[u8]) { + let row_bytes = width * PixelFormat::Rgba16.bytes_per_pixel(); + for (row_index, row) in buf.chunks_exact(stride).enumerate() { + let start = row_index * row_bytes; + assert_eq!(&row[..row_bytes], &expected[start..start + row_bytes]); + assert_eq!(&row[row_bytes..], &[0xaa; 8]); + } +} + +fn rgb16_samples_to_le_bytes(samples: &[u16]) -> Vec { + let mut out = Vec::with_capacity(samples.len() * 2); + for sample in samples { + out.extend_from_slice(&sample.to_le_bytes()); + } + out +} + +fn rgb16_to_rgba16(rgb: &[u8], alpha: u16) -> Vec { + let mut out = Vec::with_capacity(rgb.len() / 6 * 8); + let alpha = alpha.to_le_bytes(); + for pixel in rgb.chunks_exact(6) { + out.extend_from_slice(pixel); + out.extend_from_slice(&alpha); + } + out +} + +fn crop_rgb16_bytes(full: &[u8], width: usize, roi: Rect) -> Vec { + let mut out = Vec::with_capacity(roi.w as usize * roi.h as usize * 6); + for y in roi.y..roi.y + roi.h { + let row = y as usize * width * 6; + let start = row + roi.x as usize * 6; + let end = start + roi.w as usize * 6; + out.extend_from_slice(&full[start..end]); + } + out +} + +fn assert_rgb16_image_eq(actual: &[u8], expected: &[u8], width: usize) { + assert_eq!(actual.len(), expected.len()); + for (pixel_index, (actual_pixel, expected_pixel)) in actual + .chunks_exact(6) + .zip(expected.chunks_exact(6)) + .enumerate() + { + if actual_pixel != expected_pixel { + let actual_rgb = rgb16_pixel(actual_pixel); + let expected_rgb = rgb16_pixel(expected_pixel); + panic!( + "RGB16 pixel mismatch at ({}, {}): actual {:?}, expected {:?}", + pixel_index % width, + pixel_index / width, + actual_rgb, + expected_rgb + ); + } + } +} + +fn rgb16_pixel(pixel: &[u8]) -> [u16; 3] { + [ + u16::from_le_bytes([pixel[0], pixel[1]]), + u16::from_le_bytes([pixel[2], pixel[3]]), + u16::from_le_bytes([pixel[4], pixel[5]]), + ] +} + +#[test] +fn decode_region_into_rgb8_scaled_crops_constant_app14_rgb_pixels() { + let bytes = rgb_app14_8x8_jpeg(); + let dec = Decoder::new(&bytes).unwrap(); + let roi = Rect { + x: 2, + y: 2, + w: 4, + h: 4, + }; + let mut buf = vec![0u8; 2 * 2 * 3]; + let outcome = dec + .decode_region_scaled_into(&mut buf, 2 * 3, PixelFormat::Rgb8, roi, Downscale::Half) + .unwrap(); + assert_eq!(outcome.decoded, roi); + let mut expected = Vec::with_capacity(buf.len()); + for _ in 0..4 { + expected.extend_from_slice(&[200, 20, 10]); + } + assert_eq!(buf, expected); +} + +#[test] +fn decode_into_rgb8_converts_cmyk_and_ycck() { + for bytes in [cmyk_8x8_jpeg(), ycck_8x8_jpeg()] { + let dec = Decoder::new(&bytes).expect("four-component baseline JPEG should construct"); + let (w, h) = dec.info().dimensions; + let mut buf = vec![0u8; (w * h * 3) as usize]; + + dec.decode_into(&mut buf, (w * 3) as usize, PixelFormat::Rgb8) + .expect("CMYK/YCCK to RGB8 decode should succeed"); + + assert_eq!(buf, four_component_8x8_rgb()); + } +} + +#[test] +fn decode_into_rgba8_converts_cmyk_and_ycck_with_alpha() { + for bytes in [cmyk_8x8_jpeg(), ycck_8x8_jpeg()] { + let dec = Decoder::new(&bytes).expect("four-component baseline JPEG should construct"); + let (w, h) = dec.info().dimensions; + let mut buf = vec![0u8; (w * h * 4) as usize]; + + dec.decode_into(&mut buf, (w * 4) as usize, PixelFormat::Rgba8) + .expect("CMYK/YCCK to RGBA8 decode should succeed"); + + for pixel in buf.chunks_exact(4) { + assert_eq!(pixel, &[64, 64, 64, 255]); + } + } +} + +#[test] +fn decode_region_into_rgb8_crops_cmyk_and_ycck() { + let roi = Rect { + x: 2, + y: 1, + w: 5, + h: 4, + }; + let expected = crop_rgb(&four_component_8x8_rgb(), 8, roi); + + for bytes in [cmyk_8x8_jpeg(), ycck_8x8_jpeg()] { + let dec = Decoder::new(&bytes).expect("four-component baseline JPEG should construct"); + let mut buf = vec![0u8; roi.w as usize * roi.h as usize * 3]; + + let outcome = dec + .decode_region_into(&mut buf, roi.w as usize * 3, PixelFormat::Rgb8, roi) + .expect("CMYK/YCCK RGB8 ROI decode should succeed"); + + assert_eq!(outcome.decoded, roi); + assert_eq!(buf, expected); + } +} + +#[test] +fn decode_scaled_into_rgb8_projects_cmyk_and_ycck() { + let expected = project_scaled_rgb( + &four_component_8x8_rgb(), + 8, + 8, + Rect { + x: 0, + y: 0, + w: 4, + h: 4, + }, + 2, + ); + + for bytes in [cmyk_8x8_jpeg(), ycck_8x8_jpeg()] { + let dec = Decoder::new(&bytes).expect("four-component baseline JPEG should construct"); + let mut buf = vec![0u8; expected.len()]; + + let outcome = dec + .decode_scaled_into(&mut buf, 4 * 3, PixelFormat::Rgb8, Downscale::Half) + .expect("CMYK/YCCK RGB8 scaled decode should succeed"); + + assert_eq!(outcome.decoded, Rect::full((8, 8))); + assert_eq!(buf, expected); + } +} + +#[test] +fn decode_scaled_into_rgba8_projects_cmyk_and_ycck() { + let expected = rgb8_to_rgba8( + &project_scaled_rgb( + &four_component_8x8_rgb(), + 8, + 8, + Rect { + x: 0, + y: 0, + w: 4, + h: 4, + }, + 2, + ), + 255, + ); + + for bytes in [cmyk_8x8_jpeg(), ycck_8x8_jpeg()] { + let dec = Decoder::new(&bytes).expect("four-component baseline JPEG should construct"); + let mut buf = vec![0u8; expected.len()]; + + let outcome = dec + .decode_scaled_into(&mut buf, 4 * 4, PixelFormat::Rgba8, Downscale::Half) + .expect("CMYK/YCCK RGBA8 scaled decode should succeed"); + + assert_eq!(outcome.decoded, Rect::full((8, 8))); + assert_eq!(buf, expected); + } +} + +#[test] +fn decode_region_scaled_into_rgb8_projects_cmyk_and_ycck_with_padding() { + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let expected = project_scaled_rgb(&four_component_8x8_rgb(), 8, 8, scaled_roi, 2); + let row_bytes = scaled_roi.w as usize * 3; + let stride = row_bytes + 5; + + for bytes in [cmyk_8x8_jpeg(), ycck_8x8_jpeg()] { + let dec = Decoder::new(&bytes).expect("four-component baseline JPEG should construct"); + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgb8, roi, Downscale::Half) + .expect("CMYK/YCCK RGB8 region-scaled decode should succeed"); + + assert_eq!(outcome.decoded, roi); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(row_bytes)) + { + assert_eq!(&row[..row_bytes], expected_row); + assert_eq!(&row[row_bytes..], &[0xaa; 5]); + } + } +} + +#[test] +fn decode_region_scaled_into_rgba8_projects_cmyk_and_ycck_with_padding() { + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let expected = rgb8_to_rgba8( + &project_scaled_rgb(&four_component_8x8_rgb(), 8, 8, scaled_roi, 2), + 255, + ); + let row_bytes = scaled_roi.w as usize * 4; + let stride = row_bytes + 4; + + for bytes in [cmyk_8x8_jpeg(), ycck_8x8_jpeg()] { + let dec = Decoder::new(&bytes).expect("four-component baseline JPEG should construct"); + let mut buf = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into(&mut buf, stride, PixelFormat::Rgba8, roi, Downscale::Half) + .expect("CMYK/YCCK RGBA8 region-scaled decode should succeed"); + + assert_eq!(outcome.decoded, roi); + for (row, expected_row) in buf + .chunks_exact(stride) + .zip(expected.chunks_exact(row_bytes)) + { + assert_eq!(&row[..row_bytes], expected_row); + assert_eq!(&row[row_bytes..], &[0xaa; 4]); + } + } +} + +#[test] +fn decode_subsampled_cmyk_ycck_full_and_region_scaled_outputs() { + for (bytes, expected, width, height, label) in [ + ( + cmyk_16x8_422_jpeg(), + four_component_16x8_rgb(), + 16, + 8, + "CMYK 4:2:2", + ), + ( + ycck_16x8_422_jpeg(), + four_component_16x8_rgb(), + 16, + 8, + "YCCK 4:2:2", + ), + ( + cmyk_16x16_420_jpeg(), + four_component_16x16_rgb(), + 16, + 16, + "CMYK 4:2:0", + ), + ( + ycck_16x16_420_jpeg(), + four_component_16x16_rgb(), + 16, + 16, + "YCCK 4:2:0", + ), + ] { + let dec = Decoder::new(&bytes) + .unwrap_or_else(|err| panic!("{label} four-component JPEG should construct: {err}")); + let mut full = vec![0u8; expected.len()]; + + let outcome = dec + .decode_into( + &mut full, + width * PixelFormat::Rgb8.bytes_per_pixel(), + PixelFormat::Rgb8, + ) + .unwrap_or_else(|err| panic!("{label} full RGB8 decode should succeed: {err}")); + + assert_eq!( + outcome.decoded, + Rect::full((width as u32, height as u32)), + "{label}" + ); + assert_eq!(full, expected, "{label}"); + + let roi = Rect { + x: width as u32 / 4, + y: height as u32 / 4, + w: width as u32 / 2, + h: height as u32 / 2, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let row_bytes = scaled_roi.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let stride = row_bytes + 4; + let expected_rgba = rgb8_to_rgba8( + &project_scaled_rgb(&expected, width as u32, height as u32, scaled_roi, 2), + 255, + ); + let mut region = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into( + &mut region, + stride, + PixelFormat::Rgba8, + roi, + Downscale::Half, + ) + .unwrap_or_else(|err| { + panic!("{label} region-scaled RGBA8 decode should succeed: {err}") + }); + + assert_eq!(outcome.decoded, roi, "{label}"); + for (row, expected_row) in region + .chunks_exact(stride) + .zip(expected_rgba.chunks_exact(row_bytes)) + { + assert_eq!(&row[..row_bytes], expected_row, "{label}"); + assert_eq!(&row[row_bytes..], &[0xaa; 4], "{label}"); + } + } +} + +#[test] +fn decode_12bit_cmyk_ycck_full_roi_scaled_and_region_scaled_outputs() { + for (bytes, expected_full, width, height, label) in [ + ( + extended_12bit_cmyk_8x8_jpeg(), + four_component_12bit_8x8_rgb16(), + 8, + 8, + "12-bit CMYK 4:4:4", + ), + ( + extended_12bit_ycck_8x8_jpeg(), + four_component_12bit_8x8_rgb16(), + 8, + 8, + "12-bit YCCK 4:4:4", + ), + ( + extended_12bit_cmyk_restart_16x8_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + 8, + "restart-coded 12-bit CMYK 4:4:4", + ), + ( + extended_12bit_ycck_restart_16x8_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + 8, + "restart-coded 12-bit YCCK 4:4:4", + ), + ( + extended_12bit_cmyk_16x8_422_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + 8, + "12-bit CMYK 4:2:2", + ), + ( + extended_12bit_ycck_16x8_422_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + 8, + "12-bit YCCK 4:2:2", + ), + ( + extended_12bit_cmyk_422_restart_32x8_jpeg(), + four_component_12bit_32x8_rgb16(), + 32, + 8, + "restart-coded 12-bit CMYK 4:2:2", + ), + ( + extended_12bit_ycck_422_restart_32x8_jpeg(), + four_component_12bit_32x8_rgb16(), + 32, + 8, + "restart-coded 12-bit YCCK 4:2:2", + ), + ( + extended_12bit_cmyk_16x16_420_jpeg(), + four_component_12bit_16x16_rgb16(), + 16, + 16, + "12-bit CMYK 4:2:0", + ), + ( + extended_12bit_ycck_16x16_420_jpeg(), + four_component_12bit_16x16_rgb16(), + 16, + 16, + "12-bit YCCK 4:2:0", + ), + ( + extended_12bit_cmyk_420_restart_32x16_jpeg(), + four_component_12bit_32x16_rgb16(), + 32, + 16, + "restart-coded 12-bit CMYK 4:2:0", + ), + ( + extended_12bit_ycck_420_restart_32x16_jpeg(), + four_component_12bit_32x16_rgb16(), + 32, + 16, + "restart-coded 12-bit YCCK 4:2:0", + ), + ( + progressive_12bit_cmyk_8x8_jpeg(), + four_component_12bit_8x8_rgb16(), + 8, + 8, + "progressive 12-bit CMYK 4:4:4", + ), + ( + progressive_12bit_ycck_8x8_jpeg(), + four_component_12bit_8x8_rgb16(), + 8, + 8, + "progressive 12-bit YCCK 4:4:4", + ), + ( + progressive_12bit_cmyk_restart_16x8_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + 8, + "restart-coded progressive 12-bit CMYK 4:4:4", + ), + ( + progressive_12bit_ycck_restart_16x8_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + 8, + "restart-coded progressive 12-bit YCCK 4:4:4", + ), + ( + progressive_12bit_cmyk_16x8_422_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + 8, + "progressive 12-bit CMYK 4:2:2", + ), + ( + progressive_12bit_ycck_16x8_422_jpeg(), + four_component_12bit_16x8_rgb16(), + 16, + 8, + "progressive 12-bit YCCK 4:2:2", + ), + ( + progressive_12bit_cmyk_422_restart_32x8_jpeg(), + four_component_12bit_32x8_rgb16(), + 32, + 8, + "restart-coded progressive 12-bit CMYK 4:2:2", + ), + ( + progressive_12bit_ycck_422_restart_32x8_jpeg(), + four_component_12bit_32x8_rgb16(), + 32, + 8, + "restart-coded progressive 12-bit YCCK 4:2:2", + ), + ( + progressive_12bit_cmyk_16x16_420_jpeg(), + four_component_12bit_16x16_rgb16(), + 16, + 16, + "progressive 12-bit CMYK 4:2:0", + ), + ( + progressive_12bit_ycck_16x16_420_jpeg(), + four_component_12bit_16x16_rgb16(), + 16, + 16, + "progressive 12-bit YCCK 4:2:0", + ), + ( + progressive_12bit_cmyk_420_restart_32x16_jpeg(), + four_component_12bit_32x16_rgb16(), + 32, + 16, + "restart-coded progressive 12-bit CMYK 4:2:0", + ), + ( + progressive_12bit_ycck_420_restart_32x16_jpeg(), + four_component_12bit_32x16_rgb16(), + 32, + 16, + "restart-coded progressive 12-bit YCCK 4:2:0", + ), + ] { + let dec = Decoder::new(&bytes) + .unwrap_or_else(|err| panic!("{label} decoder should construct: {err}")); + let full_rect = Rect::full((width, height)); + + let mut full = + vec![0u8; width as usize * height as usize * PixelFormat::Rgb16.bytes_per_pixel()]; + let outcome = dec + .decode_into( + &mut full, + width as usize * PixelFormat::Rgb16.bytes_per_pixel(), + PixelFormat::Rgb16, + ) + .unwrap_or_else(|err| panic!("{label} RGB16 full decode should succeed: {err}")); + assert_eq!(outcome.decoded, full_rect, "{label}"); + assert_eq!(full, expected_full, "{label}"); + + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let expected_roi = crop_rgb16_bytes(&expected_full, width as usize, roi); + let mut roi_buf = vec![0u8; roi.w as usize * roi.h as usize * 6]; + let outcome = dec + .decode_region_into( + &mut roi_buf, + roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel(), + PixelFormat::Rgb16, + roi, + ) + .unwrap_or_else(|err| panic!("{label} RGB16 ROI decode should succeed: {err}")); + assert_eq!(outcome.decoded, roi, "{label}"); + assert_eq!(roi_buf, expected_roi, "{label}"); + + let scaled_rect = scaled_rect_covering_for_test(full_rect, 2); + let scaled_row_bytes = scaled_rect.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let scaled_stride = scaled_row_bytes + 8; + let expected_scaled = rgb16_to_rgba16( + &expected_scaled_rgb16_pixels(&expected_full, width as usize, full_rect, 2), + u16::MAX, + ); + let mut scaled = vec![0xaau8; scaled_stride * scaled_rect.h as usize]; + let outcome = dec + .decode_scaled_into( + &mut scaled, + scaled_stride, + PixelFormat::Rgba16, + Downscale::Half, + ) + .unwrap_or_else(|err| panic!("{label} RGBA16 scaled decode should succeed: {err}")); + assert_eq!(outcome.decoded, full_rect, "{label}"); + assert_padded_rgba16_rows( + &scaled, + scaled_stride, + scaled_rect.w as usize, + &expected_scaled, + ); + + let region_scaled = scaled_rect_covering_for_test(roi, 2); + let region_scaled_row_bytes = + region_scaled.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let region_scaled_stride = region_scaled_row_bytes + 8; + let expected_region_scaled = rgb16_to_rgba16( + &expected_scaled_rgb16_pixels(&expected_full, width as usize, roi, 2), + u16::MAX, + ); + let mut region_scaled_buf = vec![0xaau8; region_scaled_stride * region_scaled.h as usize]; + let outcome = dec + .decode_region_scaled_into( + &mut region_scaled_buf, + region_scaled_stride, + PixelFormat::Rgba16, + roi, + Downscale::Half, + ) + .unwrap_or_else(|err| { + panic!("{label} RGBA16 region-scaled decode should succeed: {err}") + }); + assert_eq!(outcome.decoded, roi, "{label}"); + assert_padded_rgba16_rows( + ®ion_scaled_buf, + region_scaled_stride, + region_scaled.w as usize, + &expected_region_scaled, + ); + } +} + +#[test] +fn decode_12bit_cmyk_ycck_nonconstant_full_and_region_scaled_outputs() { + for (bytes, expected_full, label) in [ + ( + extended_12bit_cmyk_nonconstant_8x8_jpeg(), + four_component_12bit_8x8_cmyk_nonconstant_rgb16(), + "12-bit extended CMYK non-constant", + ), + ( + extended_12bit_ycck_nonconstant_8x8_jpeg(), + four_component_12bit_8x8_ycck_nonconstant_rgb16(), + "12-bit extended YCCK non-constant", + ), + ( + progressive_12bit_cmyk_nonconstant_8x8_jpeg(), + four_component_12bit_8x8_cmyk_nonconstant_rgb16(), + "12-bit progressive CMYK non-constant", + ), + ( + progressive_12bit_ycck_nonconstant_8x8_jpeg(), + four_component_12bit_8x8_ycck_nonconstant_rgb16(), + "12-bit progressive YCCK non-constant", + ), + ] { + let dec = Decoder::new(&bytes) + .unwrap_or_else(|err| panic!("{label} decoder should construct: {err}")); + let mut full = vec![0u8; expected_full.len()]; + + dec.decode_into(&mut full, 8 * 6, PixelFormat::Rgb16) + .unwrap_or_else(|err| panic!("{label} full RGB16 decode should succeed: {err}")); + + assert_eq!(full, expected_full, "{label}"); + + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + let region_scaled = scaled_rect_covering_for_test(roi, 2); + let row_bytes = region_scaled.w as usize * PixelFormat::Rgba16.bytes_per_pixel(); + let stride = row_bytes + 8; + let expected_region_scaled = rgb16_to_rgba16( + &expected_scaled_rgb16_pixels(&expected_full, 8, roi, 2), + u16::MAX, + ); + let mut region_scaled_buf = vec![0xaau8; stride * region_scaled.h as usize]; + + let outcome = dec + .decode_region_scaled_into( + &mut region_scaled_buf, + stride, + PixelFormat::Rgba16, + roi, + Downscale::Half, + ) + .unwrap_or_else(|err| { + panic!("{label} region-scaled RGBA16 decode should succeed: {err}") + }); + + assert_eq!(outcome.decoded, roi, "{label}"); + assert_padded_rgba16_rows( + ®ion_scaled_buf, + stride, + region_scaled.w as usize, + &expected_region_scaled, + ); + } +} + +#[test] +fn decode_nonleading_max_four_component_sampling_uses_generic_upsample() { + for (bytes, label) in [ + (cmyk_16x8_nonleading_max_422_jpeg(), "non-leading-max CMYK"), + (ycck_16x8_nonleading_max_422_jpeg(), "non-leading-max YCCK"), + ] { + let expected = four_component_16x8_rgb(); + let dec = Decoder::new(&bytes) + .unwrap_or_else(|err| panic!("{label} sampling should use generic upsample: {err}")); + let mut full = vec![0u8; expected.len()]; + + dec.decode_into(&mut full, 16 * 3, PixelFormat::Rgb8) + .unwrap_or_else(|err| panic!("{label} full decode should succeed: {err}")); + + assert_eq!(full, expected, "{label}"); + + let roi = Rect { + x: 4, + y: 2, + w: 8, + h: 4, + }; + let scaled_roi = scaled_rect_covering_for_test(roi, 2); + let row_bytes = scaled_roi.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let stride = row_bytes + 4; + let expected_rgba = + rgb8_to_rgba8(&project_scaled_rgb(&expected, 16, 8, scaled_roi, 2), 255); + let mut region = vec![0xaau8; stride * scaled_roi.h as usize]; + + let outcome = dec + .decode_region_scaled_into( + &mut region, + stride, + PixelFormat::Rgba8, + roi, + Downscale::Half, + ) + .unwrap_or_else(|err| panic!("{label} region-scaled decode should succeed: {err}")); + + assert_eq!(outcome.decoded, roi, "{label}"); + for (row, expected_row) in region + .chunks_exact(stride) + .zip(expected_rgba.chunks_exact(row_bytes)) + { + assert_eq!(&row[..row_bytes], expected_row, "{label}"); + assert_eq!(&row[row_bytes..], &[0xaa; 4], "{label}"); + } + } +} + +#[test] +fn decoder_new_rejects_malformed_four_component_sampling_shape() { + let input = malformed_cmyk_nondivisible_sampling_jpeg(); + let Err(err) = Decoder::new(&input) else { + panic!("malformed four-component sampling should reject construction"); + }; + + assert!( + matches!( + err, + JpegError::NotImplemented { + sof: SofKind::Baseline8 + } + ), + "{err}" + ); +} + +#[test] +fn decode_region_into_rgba8_crops_cmyk_and_ycck_with_alpha() { + let roi = Rect { + x: 3, + y: 2, + w: 3, + h: 4, + }; + let stride = roi.w as usize * 4 + 4; + + for bytes in [cmyk_8x8_jpeg(), ycck_8x8_jpeg()] { + let dec = Decoder::new(&bytes).expect("four-component baseline JPEG should construct"); + let mut buf = vec![0xaau8; stride * roi.h as usize]; + + let outcome = dec + .decode_region_into(&mut buf, stride, PixelFormat::Rgba8, roi) + .expect("CMYK/YCCK RGBA8 ROI decode should succeed"); + + assert_eq!(outcome.decoded, roi); + for row in buf.chunks_exact(stride) { + for pixel in row[..roi.w as usize * 4].chunks_exact(4) { + assert_eq!(pixel, &[64, 64, 64, 255]); + } + assert_eq!(&row[roi.w as usize * 4..], &[0xaa; 4]); + } + } +} + +#[test] +fn decoder_new_rejects_invalid_sequential_scan_parameters() { + let mut bytes = minimal_baseline_420_jpeg(); + let sos = bytes + .windows(2) + .position(|w| w == [0xff, 0xda]) + .expect("fixture SOS"); + bytes[sos + 2 + 2 + 1 + 3 * 2] = 1; + + let err = Decoder::new(&bytes).expect_err("baseline Ss=1 must be rejected"); + assert!(matches!( + err, + JpegError::InvalidScanParameters { + ss: 1, + se: 63, + ah: 0, + al: 0, + .. + } + )); +} diff --git a/crates/j2k-jpeg/tests/device_plan.rs b/crates/j2k-jpeg/tests/device_plan.rs new file mode 100644 index 00000000..e910be52 --- /dev/null +++ b/crates/j2k-jpeg/tests/device_plan.rs @@ -0,0 +1,3030 @@ +use std::borrow::Cow; + +use j2k_jpeg::{ + ColorSpace, Decoder, Downscale, JpegCapabilityReport, JpegCapabilityRequest, JpegDecodeOp, + JpegError, PixelFormat, Rect, SofKind, UnsupportedReason, Warning, +}; +use j2k_test_support::{ + baseline_grayscale_jpeg, restart_coded_grayscale_jpeg, JPEG_BASELINE_420_16X16, + JPEG_BASELINE_422_16X8, JPEG_BASELINE_444_8X8, +}; + +use fixtures::{ + cmyk_16x16_420_jpeg, cmyk_16x8_422_jpeg, cmyk_16x8_nonleading_max_422_jpeg, cmyk_8x8_jpeg, + extended_12bit_cmyk_16x16_420_jpeg, extended_12bit_cmyk_16x8_422_jpeg, + extended_12bit_cmyk_420_restart_32x16_jpeg, extended_12bit_cmyk_422_restart_32x8_jpeg, + extended_12bit_cmyk_8x8_jpeg, extended_12bit_cmyk_restart_16x8_jpeg, + extended_12bit_grayscale_restart_16x8_jpeg, extended_12bit_rgb_420_32x32_jpeg, + extended_12bit_rgb_422_32x8_jpeg, extended_12bit_rgb_8x8_jpeg, + extended_12bit_rgb_restart_16x8_jpeg, extended_12bit_ycbcr_420_32x32_jpeg, + extended_12bit_ycbcr_420_restart_32x32_jpeg, extended_12bit_ycbcr_422_32x8_jpeg, + extended_12bit_ycbcr_422_restart_32x8_jpeg, extended_12bit_ycbcr_8x8_jpeg, + extended_12bit_ycbcr_restart_16x8_jpeg, extended_12bit_ycck_16x16_420_jpeg, + extended_12bit_ycck_16x8_422_jpeg, extended_12bit_ycck_420_restart_32x16_jpeg, + extended_12bit_ycck_422_restart_32x8_jpeg, extended_12bit_ycck_8x8_jpeg, + extended_12bit_ycck_restart_16x8_jpeg, lossless_predictor_grayscale_16bit_3x3_jpeg, + lossless_predictor_grayscale_3x3_jpeg, lossless_predictor_rgb_16bit_3x3_jpeg, + lossless_predictor_rgb_3x3_jpeg, lossless_predictor_ycbcr_16bit_3x3_jpeg, + lossless_predictor_ycbcr_3x3_jpeg, lossless_restart_predictor_grayscale_16bit_3x3_jpeg, + lossless_restart_predictor_grayscale_3x3_jpeg, lossless_restart_predictor_rgb_16bit_3x3_jpeg, + lossless_restart_predictor_rgb_3x3_jpeg, lossless_restart_predictor_ycbcr_16bit_3x3_jpeg, + lossless_restart_predictor_ycbcr_3x3_jpeg, lossless_rgb_16bit_420_4x4_jpeg, + lossless_rgb_16bit_420_restart_4x4_jpeg, lossless_rgb_16bit_422_4x2_jpeg, + lossless_rgb_16bit_422_restart_4x2_jpeg, lossless_rgb_8bit_420_4x4_jpeg, + lossless_rgb_8bit_420_restart_4x4_jpeg, lossless_rgb_8bit_422_4x2_jpeg, + lossless_rgb_8bit_422_restart_4x2_jpeg, lossless_ycbcr_16bit_420_4x4_jpeg, + lossless_ycbcr_16bit_420_restart_4x4_jpeg, lossless_ycbcr_16bit_422_3x3_jpeg, + lossless_ycbcr_16bit_422_4x2_jpeg, lossless_ycbcr_16bit_422_restart_4x2_jpeg, + lossless_ycbcr_8bit_420_4x4_jpeg, lossless_ycbcr_8bit_420_restart_4x4_jpeg, + lossless_ycbcr_8bit_422_4x2_jpeg, lossless_ycbcr_8bit_422_restart_4x2_jpeg, + malformed_cmyk_nondivisible_sampling_jpeg, progressive_12bit_cmyk_16x16_420_jpeg, + progressive_12bit_cmyk_16x8_422_jpeg, progressive_12bit_cmyk_420_restart_32x16_jpeg, + progressive_12bit_cmyk_422_restart_32x8_jpeg, progressive_12bit_cmyk_8x8_jpeg, + progressive_12bit_cmyk_restart_16x8_jpeg, progressive_12bit_grayscale_8x8_jpeg, + progressive_12bit_rgb_420_32x32_jpeg, progressive_12bit_rgb_422_32x8_jpeg, + progressive_12bit_rgb_8x8_jpeg, progressive_12bit_ycbcr_420_32x32_jpeg, + progressive_12bit_ycbcr_422_32x8_jpeg, progressive_12bit_ycbcr_8x8_jpeg, + progressive_12bit_ycck_16x16_420_jpeg, progressive_12bit_ycck_16x8_422_jpeg, + progressive_12bit_ycck_420_restart_32x16_jpeg, progressive_12bit_ycck_422_restart_32x8_jpeg, + progressive_12bit_ycck_8x8_jpeg, progressive_12bit_ycck_restart_16x8_jpeg, + progressive_8x8_jpeg, ycck_16x16_420_jpeg, ycck_16x8_422_jpeg, + ycck_16x8_nonleading_max_422_jpeg, ycck_8x8_jpeg, +}; +use j2k_test_support as fixtures; + +const BASELINE_420: &[u8] = JPEG_BASELINE_420_16X16; +const BASELINE_422: &[u8] = JPEG_BASELINE_422_16X8; +const BASELINE_444: &[u8] = JPEG_BASELINE_444_8X8; + +fn baseline_420_with_sof_marker(marker: u8) -> Vec { + let mut bytes = BASELINE_420.to_vec(); + let pos = bytes + .windows(2) + .position(|window| window == [0xff, 0xc0]) + .expect("baseline fixture has SOF0 marker"); + bytes[pos + 1] = marker; + bytes +} + +fn standard_ops(region_roi: Rect, scaled_roi: Rect) -> [JpegDecodeOp; 4] { + [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(region_roi), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: scaled_roi, + scale: Downscale::Half, + }, + ] +} + +fn inspect_capability( + input: &[u8], + op: JpegDecodeOp, + fmt: PixelFormat, + context: &str, +) -> JpegCapabilityReport { + JpegCapabilityReport::inspect(input, JpegCapabilityRequest { op, fmt }) + .unwrap_or_else(|err| panic!("{context}: {err}")) +} + +fn assert_cpu_only(report: &JpegCapabilityReport, context: &str) { + assert!(report.cpu.eligible, "{context}"); + assert!(!report.owned_cuda.eligible, "{context}"); + assert!(!report.metal_fast.eligible, "{context}"); +} + +#[test] +fn adapter_device_plan_exposes_scan_metadata() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let plan = j2k_jpeg::adapter::build_device_plan(&decoder, 4).expect("device plan"); + + assert_eq!(plan.dimensions, (16, 16)); + assert_eq!(plan.color_space, ColorSpace::YCbCr); + assert_eq!(plan.components.len(), 3); + assert_eq!(plan.checkpoints[0].mcu_index, 0); + assert!(!plan.scan_bytes.is_empty()); +} + +#[test] +fn adapter_device_plan_borrows_scan_bytes_for_well_formed_streams() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let plan = j2k_jpeg::adapter::build_device_plan(&decoder, 4).expect("device plan"); + + assert!(matches!(plan.scan_bytes, Cow::Borrowed(_))); +} + +#[test] +fn adapter_device_plan_keeps_fast_420_shape_information() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let plan = j2k_jpeg::adapter::build_device_plan(&decoder, 4).expect("device plan"); + + assert!(plan.matches_fast_420); + assert!(!plan.matches_fast_422); + assert!(!plan.matches_fast_444); +} + +#[test] +fn adapter_device_plan_keeps_fast_422_shape_information() { + let decoder = Decoder::new(BASELINE_422).expect("decoder"); + let plan = j2k_jpeg::adapter::build_device_plan(&decoder, 4).expect("device plan"); + + assert!(!plan.matches_fast_420); + assert!(plan.matches_fast_422); + assert!(!plan.matches_fast_444); +} + +#[test] +fn adapter_device_plan_keeps_fast_444_shape_information() { + let decoder = Decoder::new(BASELINE_444).expect("decoder"); + let plan = j2k_jpeg::adapter::build_device_plan(&decoder, 4).expect("device plan"); + + assert!(!plan.matches_fast_420); + assert!(!plan.matches_fast_422); + assert!(plan.matches_fast_444); +} + +#[test] +fn capability_report_exposes_metadata_and_fast_backend_eligibility() { + let report = JpegCapabilityReport::inspect( + BASELINE_420, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgb8, + }, + ) + .expect("capability report"); + + assert_eq!(report.info.dimensions, (16, 16)); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert_eq!(report.info.sof_kind, j2k_jpeg::SofKind::Baseline8); + assert!(report.cpu.eligible); + assert!(report.owned_cuda.eligible); + assert!(report.metal_fast.eligible); + assert!(report.device.matches_fast_420); + assert!(!report.device.matches_fast_422); + assert!(!report.device.matches_fast_444); +} + +#[test] +fn capability_report_marks_cmyk_and_ycck_cpu_rgb8_rgba8_eligible() { + for (input, expected_color) in [ + (cmyk_8x8_jpeg(), ColorSpace::Cmyk), + (ycck_8x8_jpeg(), ColorSpace::Ycck), + ] { + for op in standard_ops( + Rect { + x: 2, + y: 1, + w: 5, + h: 4, + }, + Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }, + ) { + let report = inspect_capability( + &input, + op, + PixelFormat::Rgb8, + "capability report should parse four-component color metadata", + ); + + assert_eq!(report.info.sof_kind, SofKind::Baseline8); + assert_eq!(report.info.color_space, expected_color); + assert_cpu_only(&report, &format!("{expected_color:?} {op:?}")); + assert!(report + .metal_fast + .reason + .expect("Metal rejection reason") + .contains("YCbCr")); + } + + for op in standard_ops( + Rect { + x: 3, + y: 2, + w: 3, + h: 4, + }, + Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }, + ) { + let report = inspect_capability( + &input, + op, + PixelFormat::Rgba8, + "capability report should parse four-component color metadata", + ); + + assert_eq!(report.info.sof_kind, SofKind::Baseline8); + assert_eq!(report.info.color_space, expected_color); + assert_cpu_only(&report, &format!("{expected_color:?} {op:?}")); + } + } +} + +#[test] +fn capability_report_marks_subsampled_cmyk_and_ycck_cpu_rgb8_rgba8_eligible() { + for (name, input, expected_color, expected_dimensions, expected_sampling) in [ + ( + "CMYK 4:2:2", + cmyk_16x8_422_jpeg(), + ColorSpace::Cmyk, + (16, 8), + (2, 1), + ), + ( + "YCCK 4:2:2", + ycck_16x8_422_jpeg(), + ColorSpace::Ycck, + (16, 8), + (2, 1), + ), + ( + "CMYK 4:2:0", + cmyk_16x16_420_jpeg(), + ColorSpace::Cmyk, + (16, 16), + (2, 2), + ), + ( + "YCCK 4:2:0", + ycck_16x16_420_jpeg(), + ColorSpace::Ycck, + (16, 16), + (2, 2), + ), + ] { + for fmt in [PixelFormat::Rgb8, PixelFormat::Rgba8] { + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: expected_dimensions.0 / 4, + y: expected_dimensions.1 / 4, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: expected_dimensions.0 / 4, + y: expected_dimensions.1 / 4, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }, + scale: Downscale::Half, + }, + ] { + let report = + JpegCapabilityReport::inspect(&input, JpegCapabilityRequest { op, fmt }) + .unwrap_or_else(|err| { + panic!("capability report should parse {name} metadata: {err}") + }); + + assert_eq!(report.info.sof_kind, SofKind::Baseline8, "{name}"); + assert_eq!(report.info.dimensions, expected_dimensions, "{name}"); + assert_eq!(report.info.color_space, expected_color, "{name}"); + assert_eq!(report.info.sampling.max_h, expected_sampling.0, "{name}"); + assert_eq!(report.info.sampling.max_v, expected_sampling.1, "{name}"); + assert_eq!(report.info.sampling.components().len(), 4, "{name}"); + assert!(report.cpu.eligible, "{name} {fmt:?} {op:?}"); + assert!(!report.owned_cuda.eligible, "{name}"); + let owned_cuda_reason = report + .owned_cuda + .reason + .expect("owned CUDA rejection reason"); + if op == JpegDecodeOp::Full && fmt == PixelFormat::Rgb8 { + assert!(owned_cuda_reason.contains("YCbCr"), "{name}"); + } else { + assert!(owned_cuda_reason.contains("full-tile RGB8"), "{name}"); + } + assert!(!report.metal_fast.eligible, "{name}"); + assert!(report + .metal_fast + .reason + .expect("Metal rejection reason") + .contains("YCbCr")); + } + } + } +} + +#[test] +fn capability_report_marks_12bit_four_component_cpu_eligible() { + for (name, input, expected_sof, expected_color, expected_dimensions, expected_sampling) in [ + ( + "12-bit CMYK 4:4:4", + extended_12bit_cmyk_8x8_jpeg(), + SofKind::Extended12, + ColorSpace::Cmyk, + (8, 8), + [(1, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "12-bit YCCK 4:4:4", + extended_12bit_ycck_8x8_jpeg(), + SofKind::Extended12, + ColorSpace::Ycck, + (8, 8), + [(1, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "12-bit CMYK 4:2:2", + extended_12bit_cmyk_16x8_422_jpeg(), + SofKind::Extended12, + ColorSpace::Cmyk, + (16, 8), + [(2, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "12-bit YCCK 4:2:2", + extended_12bit_ycck_16x8_422_jpeg(), + SofKind::Extended12, + ColorSpace::Ycck, + (16, 8), + [(2, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "12-bit CMYK 4:2:0", + extended_12bit_cmyk_16x16_420_jpeg(), + SofKind::Extended12, + ColorSpace::Cmyk, + (16, 16), + [(2, 2), (1, 1), (1, 1), (1, 1)], + ), + ( + "12-bit YCCK 4:2:0", + extended_12bit_ycck_16x16_420_jpeg(), + SofKind::Extended12, + ColorSpace::Ycck, + (16, 16), + [(2, 2), (1, 1), (1, 1), (1, 1)], + ), + ( + "progressive 12-bit CMYK 4:4:4", + progressive_12bit_cmyk_8x8_jpeg(), + SofKind::Progressive12, + ColorSpace::Cmyk, + (8, 8), + [(1, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "progressive 12-bit YCCK 4:4:4", + progressive_12bit_ycck_8x8_jpeg(), + SofKind::Progressive12, + ColorSpace::Ycck, + (8, 8), + [(1, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "progressive 12-bit CMYK 4:2:2", + progressive_12bit_cmyk_16x8_422_jpeg(), + SofKind::Progressive12, + ColorSpace::Cmyk, + (16, 8), + [(2, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "progressive 12-bit YCCK 4:2:2", + progressive_12bit_ycck_16x8_422_jpeg(), + SofKind::Progressive12, + ColorSpace::Ycck, + (16, 8), + [(2, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "progressive 12-bit CMYK 4:2:0", + progressive_12bit_cmyk_16x16_420_jpeg(), + SofKind::Progressive12, + ColorSpace::Cmyk, + (16, 16), + [(2, 2), (1, 1), (1, 1), (1, 1)], + ), + ( + "progressive 12-bit YCCK 4:2:0", + progressive_12bit_ycck_16x16_420_jpeg(), + SofKind::Progressive12, + ColorSpace::Ycck, + (16, 16), + [(2, 2), (1, 1), (1, 1), (1, 1)], + ), + ] { + for fmt in [PixelFormat::Rgb16, PixelFormat::Rgba16] { + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: expected_dimensions.0 / 4, + y: expected_dimensions.1 / 4, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: expected_dimensions.0 / 4, + y: expected_dimensions.1 / 4, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }, + scale: Downscale::Half, + }, + ] { + let report = + JpegCapabilityReport::inspect(&input, JpegCapabilityRequest { op, fmt }) + .unwrap_or_else(|err| { + panic!("capability report should parse {name} metadata: {err}") + }); + + assert_eq!(report.info.sof_kind, expected_sof, "{name}"); + assert_eq!(report.info.bit_depth, 12, "{name}"); + assert_eq!(report.info.dimensions, expected_dimensions, "{name}"); + assert_eq!(report.info.color_space, expected_color, "{name}"); + assert_eq!(report.info.sampling.components().len(), 4, "{name}"); + assert_eq!( + report.info.sampling.components(), + &expected_sampling, + "{name}" + ); + assert!(report.cpu.eligible, "{name} {fmt:?} {op:?}"); + assert_eq!(report.cpu.reason, None, "{name} {fmt:?} {op:?}"); + assert!(!report.owned_cuda.eligible, "{name}"); + assert!(!report.metal_fast.eligible, "{name}"); + } + } + } +} + +#[test] +fn capability_report_marks_restart_coded_12bit_four_component_cpu_eligible() { + for (name, input, expected_sof, expected_color, expected_dimensions, expected_sampling) in [ + ( + "restart-coded 12-bit CMYK 4:4:4", + extended_12bit_cmyk_restart_16x8_jpeg(), + SofKind::Extended12, + ColorSpace::Cmyk, + (16, 8), + [(1, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "restart-coded 12-bit YCCK 4:4:4", + extended_12bit_ycck_restart_16x8_jpeg(), + SofKind::Extended12, + ColorSpace::Ycck, + (16, 8), + [(1, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "restart-coded 12-bit CMYK 4:2:2", + extended_12bit_cmyk_422_restart_32x8_jpeg(), + SofKind::Extended12, + ColorSpace::Cmyk, + (32, 8), + [(2, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "restart-coded 12-bit YCCK 4:2:2", + extended_12bit_ycck_422_restart_32x8_jpeg(), + SofKind::Extended12, + ColorSpace::Ycck, + (32, 8), + [(2, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "restart-coded 12-bit CMYK 4:2:0", + extended_12bit_cmyk_420_restart_32x16_jpeg(), + SofKind::Extended12, + ColorSpace::Cmyk, + (32, 16), + [(2, 2), (1, 1), (1, 1), (1, 1)], + ), + ( + "restart-coded 12-bit YCCK 4:2:0", + extended_12bit_ycck_420_restart_32x16_jpeg(), + SofKind::Extended12, + ColorSpace::Ycck, + (32, 16), + [(2, 2), (1, 1), (1, 1), (1, 1)], + ), + ( + "restart-coded progressive 12-bit CMYK 4:4:4", + progressive_12bit_cmyk_restart_16x8_jpeg(), + SofKind::Progressive12, + ColorSpace::Cmyk, + (16, 8), + [(1, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "restart-coded progressive 12-bit YCCK 4:4:4", + progressive_12bit_ycck_restart_16x8_jpeg(), + SofKind::Progressive12, + ColorSpace::Ycck, + (16, 8), + [(1, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "restart-coded progressive 12-bit CMYK 4:2:2", + progressive_12bit_cmyk_422_restart_32x8_jpeg(), + SofKind::Progressive12, + ColorSpace::Cmyk, + (32, 8), + [(2, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "restart-coded progressive 12-bit YCCK 4:2:2", + progressive_12bit_ycck_422_restart_32x8_jpeg(), + SofKind::Progressive12, + ColorSpace::Ycck, + (32, 8), + [(2, 1), (1, 1), (1, 1), (1, 1)], + ), + ( + "restart-coded progressive 12-bit CMYK 4:2:0", + progressive_12bit_cmyk_420_restart_32x16_jpeg(), + SofKind::Progressive12, + ColorSpace::Cmyk, + (32, 16), + [(2, 2), (1, 1), (1, 1), (1, 1)], + ), + ( + "restart-coded progressive 12-bit YCCK 4:2:0", + progressive_12bit_ycck_420_restart_32x16_jpeg(), + SofKind::Progressive12, + ColorSpace::Ycck, + (32, 16), + [(2, 2), (1, 1), (1, 1), (1, 1)], + ), + ] { + for fmt in [PixelFormat::Rgb16, PixelFormat::Rgba16] { + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: expected_dimensions.0 / 4, + y: expected_dimensions.1 / 4, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: expected_dimensions.0 / 4, + y: expected_dimensions.1 / 4, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }, + scale: Downscale::Half, + }, + ] { + let report = + JpegCapabilityReport::inspect(&input, JpegCapabilityRequest { op, fmt }) + .unwrap_or_else(|err| { + panic!("capability report should parse {name} metadata: {err}") + }); + + assert_eq!(report.info.sof_kind, expected_sof, "{name}"); + assert_eq!(report.info.bit_depth, 12, "{name}"); + assert_eq!(report.info.restart_interval, Some(1), "{name}"); + assert_eq!(report.info.dimensions, expected_dimensions, "{name}"); + assert_eq!(report.info.color_space, expected_color, "{name}"); + assert_eq!( + report.info.sampling.components(), + &expected_sampling, + "{name}" + ); + assert!(report.cpu.eligible, "{name} {fmt:?} {op:?}"); + assert_eq!(report.cpu.reason, None, "{name} {fmt:?} {op:?}"); + assert!(!report.owned_cuda.eligible, "{name}"); + assert!(!report.metal_fast.eligible, "{name}"); + } + } + } +} + +#[test] +fn capability_report_marks_nonleading_max_four_component_sampling_cpu_eligible() { + for (input, color_space, label) in [ + ( + cmyk_16x8_nonleading_max_422_jpeg(), + ColorSpace::Cmyk, + "CMYK", + ), + ( + ycck_16x8_nonleading_max_422_jpeg(), + ColorSpace::Ycck, + "YCCK", + ), + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgb8, + }, + ) + .unwrap_or_else(|err| { + panic!("{label} capability report should parse legal non-leading-max metadata: {err}") + }); + + assert_eq!(report.info.sof_kind, SofKind::Baseline8, "{label}"); + assert_eq!(report.info.color_space, color_space, "{label}"); + assert_eq!( + report.info.sampling.components(), + &[(1, 1), (2, 1), (1, 1), (1, 1)], + "{label}" + ); + assert!(report.cpu.eligible, "{label}"); + assert_eq!(report.cpu.reason, None, "{label}"); + assert!(!report.owned_cuda.eligible, "{label}"); + assert!(!report.metal_fast.eligible, "{label}"); + } +} + +#[test] +fn capability_report_rejects_malformed_four_component_sampling_shape() { + let input = malformed_cmyk_nondivisible_sampling_jpeg(); + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgb8, + }, + ) + .expect("capability report should parse malformed four-component metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Baseline8); + assert_eq!(report.info.color_space, ColorSpace::Cmyk); + assert_eq!(report.info.sampling.max_h, 3); + assert_eq!(report.info.sampling.max_v, 1); + assert_eq!(report.info.sampling.component(0), Some((3, 1))); + assert!(!report.cpu.eligible); + assert!(report + .cpu + .reason + .expect("CPU rejection reason") + .contains("planner rejected")); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); +} + +#[test] +fn capability_report_marks_progressive_roi_and_scaled_cpu_eligible() { + let input = progressive_8x8_jpeg(); + let full = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgb8, + }, + ) + .expect("progressive full capability report"); + let roi_scaled = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::RegionScaled { + roi: Rect { + x: 0, + y: 0, + w: 4, + h: 4, + }, + scale: Downscale::Half, + }, + fmt: PixelFormat::Rgb8, + }, + ) + .expect("progressive region-scaled capability report"); + + assert_eq!(full.info.sof_kind, SofKind::Progressive8); + assert!(full.cpu.eligible); + assert!(roi_scaled.cpu.eligible); + assert!(!roi_scaled.owned_cuda.eligible); + assert!(!roi_scaled.metal_fast.eligible); +} + +#[test] +fn capability_report_inspects_12_bit_and_lossless_sof_without_building_decoder() { + for (input, expected_sof, expected_bits, expected_dimensions, expected_reason) in [ + ( + grayscale_sof_jpeg(0xc1, 12), + SofKind::Extended12, + 12, + (8, 8), + "12-bit", + ), + ( + progressive_12_bit_jpeg(), + SofKind::Progressive12, + 12, + (8, 8), + "12-bit", + ), + ( + lossless_predictor_grayscale_3x3_jpeg(1), + SofKind::Lossless, + 8, + (3, 3), + "lossless SOF3", + ), + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgba8, + }, + ) + .expect("capability report should parse unsupported SOF metadata"); + + assert_eq!(report.info.sof_kind, expected_sof); + assert_eq!(report.info.bit_depth, expected_bits); + assert_eq!(report.info.dimensions, expected_dimensions); + assert!(!report.cpu.eligible); + assert!(report + .cpu + .reason + .expect("CPU rejection reason") + .contains(expected_reason)); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_rejects_future_sof_classes_with_typed_errors() { + for (marker, expected_reason) in [ + (0xc9, UnsupportedReason::ArithmeticCoding), + (0xc5, UnsupportedReason::DifferentialBaseline), + (0xc6, UnsupportedReason::Hierarchical), + (0xcd, UnsupportedReason::ArithmeticAndHierarchical), + ] { + let input = baseline_420_with_sof_marker(marker); + let err = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgb8, + }, + ) + .expect_err("future SOF classes should stay explicit unsupported errors"); + + assert!(matches!( + err, + JpegError::UnsupportedSof { + marker: got_marker, + reason, + } if got_marker == marker && reason == expected_reason + )); + assert!(err.is_unsupported()); + } +} + +#[test] +fn capability_report_marks_extended12_gray16_full_cpu_eligible() { + let input = grayscale_sof_jpeg(0xc1, 12); + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Gray16, + }, + ) + .expect("capability report should parse 12-bit SOF1 metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Extended12); + assert_eq!(report.info.bit_depth, 12); + assert!(report.cpu.eligible); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); +} + +#[test] +fn capability_report_marks_extended12_gray16_region_cpu_eligible() { + let input = grayscale_sof_jpeg(0xc1, 12); + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::Region(Rect { + x: 2, + y: 1, + w: 3, + h: 4, + }), + fmt: PixelFormat::Gray16, + }, + ) + .expect("capability report should parse 12-bit SOF1 metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Extended12); + assert!(report.cpu.eligible); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); +} + +#[test] +fn capability_report_marks_extended12_rgb16_full_and_region_cpu_eligible() { + let input = grayscale_sof_jpeg(0xc1, 12); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 2, + w: 4, + h: 3, + }), + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb16, + }, + ) + .expect("capability report should parse 12-bit SOF1 metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Extended12); + assert!(report.cpu.eligible, "op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_extended12_gray16_and_rgb16_scaled_cpu_eligible() { + let input = grayscale_sof_jpeg(0xc1, 12); + for fmt in [PixelFormat::Gray16, PixelFormat::Rgb16] { + for op in [ + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect(&input, JpegCapabilityRequest { op, fmt }) + .expect("capability report should parse 12-bit SOF1 metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Extended12); + assert!(report.cpu.eligible, "fmt {fmt:?} op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_extended12_app14_rgb_rgb16_cpu_eligible() { + let input = extended_12bit_rgb_8x8_jpeg(); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 2, + y: 1, + w: 3, + h: 4, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb16, + }, + ) + .expect("capability report should parse 12-bit SOF1 APP14 RGB metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Extended12); + assert_eq!(report.info.bit_depth, 12); + assert_eq!(report.info.color_space, ColorSpace::Rgb); + assert!(report.cpu.eligible, "op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_12bit_subsampled_app14_rgb_cpu_eligible() { + for (name, input, expected_sof, expected_dimensions, expected_sampling) in [ + ( + "12-bit extended APP14 RGB 4:2:2", + extended_12bit_rgb_422_32x8_jpeg(), + SofKind::Extended12, + (32, 8), + [(2, 1), (1, 1), (1, 1)], + ), + ( + "12-bit extended APP14 RGB 4:2:0", + extended_12bit_rgb_420_32x32_jpeg(), + SofKind::Extended12, + (32, 32), + [(2, 2), (1, 1), (1, 1)], + ), + ( + "12-bit progressive APP14 RGB 4:2:2", + progressive_12bit_rgb_422_32x8_jpeg(), + SofKind::Progressive12, + (32, 8), + [(2, 1), (1, 1), (1, 1)], + ), + ( + "12-bit progressive APP14 RGB 4:2:0", + progressive_12bit_rgb_420_32x32_jpeg(), + SofKind::Progressive12, + (32, 32), + [(2, 2), (1, 1), (1, 1)], + ), + ] { + for fmt in [PixelFormat::Rgb16, PixelFormat::Rgba16] { + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: expected_dimensions.0 / 4, + y: expected_dimensions.1 / 4, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: expected_dimensions.0 / 4, + y: expected_dimensions.1 / 4, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }, + scale: Downscale::Half, + }, + ] { + let report = + JpegCapabilityReport::inspect(&input, JpegCapabilityRequest { op, fmt }) + .unwrap_or_else(|err| { + panic!("capability report should parse {name} metadata: {err}") + }); + + assert_eq!(report.info.sof_kind, expected_sof, "{name}"); + assert_eq!(report.info.bit_depth, 12, "{name}"); + assert_eq!(report.info.dimensions, expected_dimensions, "{name}"); + assert_eq!(report.info.color_space, ColorSpace::Rgb, "{name}"); + assert_eq!( + report.info.sampling.components(), + &expected_sampling, + "{name}" + ); + assert!(report.cpu.eligible, "{name} {fmt:?} {op:?}"); + assert_eq!(report.cpu.reason, None, "{name} {fmt:?} {op:?}"); + assert!(!report.owned_cuda.eligible, "{name}"); + assert!(!report.metal_fast.eligible, "{name}"); + } + } + } +} + +#[test] +fn capability_report_marks_extended12_ycbcr444_rgb16_cpu_eligible() { + let input = extended_12bit_ycbcr_8x8_jpeg(); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 2, + y: 1, + w: 3, + h: 4, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb16, + }, + ) + .expect("capability report should parse 12-bit SOF1 YCbCr metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Extended12); + assert_eq!(report.info.bit_depth, 12); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert!(report.cpu.eligible, "op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_extended12_ycbcr422_rgb16_cpu_eligible() { + let input = extended_12bit_ycbcr_422_32x8_jpeg(); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 13, + y: 0, + w: 8, + h: 4, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 13, + y: 0, + w: 8, + h: 4, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb16, + }, + ) + .expect("capability report should parse 12-bit SOF1 YCbCr 4:2:2 metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Extended12); + assert_eq!(report.info.bit_depth, 12); + assert_eq!(report.info.dimensions, (32, 8)); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert_eq!(report.info.sampling.max_h, 2); + assert_eq!(report.info.sampling.max_v, 1); + assert!(report.cpu.eligible, "op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_extended12_ycbcr420_rgb16_cpu_eligible() { + let input = extended_12bit_ycbcr_420_32x32_jpeg(); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 13, + y: 14, + w: 10, + h: 10, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 13, + y: 14, + w: 10, + h: 10, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb16, + }, + ) + .expect("capability report should parse 12-bit SOF1 YCbCr 4:2:0 metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Extended12); + assert_eq!(report.info.bit_depth, 12); + assert_eq!(report.info.dimensions, (32, 32)); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert_eq!(report.info.sampling.max_h, 2); + assert_eq!(report.info.sampling.max_v, 2); + assert!(report.cpu.eligible, "op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_extended12_color_restart_rgb16_cpu_eligible() { + for (name, input, expected_color, expected_dimensions, expected_sampling) in [ + ( + "APP14 RGB 4:4:4", + extended_12bit_rgb_restart_16x8_jpeg(), + ColorSpace::Rgb, + (16, 8), + (1, 1), + ), + ( + "YCbCr 4:4:4", + extended_12bit_ycbcr_restart_16x8_jpeg(), + ColorSpace::YCbCr, + (16, 8), + (1, 1), + ), + ( + "YCbCr 4:2:2", + extended_12bit_ycbcr_422_restart_32x8_jpeg(), + ColorSpace::YCbCr, + (32, 8), + (2, 1), + ), + ( + "YCbCr 4:2:0", + extended_12bit_ycbcr_420_restart_32x32_jpeg(), + ColorSpace::YCbCr, + (32, 32), + (2, 2), + ), + ] { + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb16, + }, + ) + .unwrap_or_else(|err| { + panic!("capability report should parse 12-bit restart {name} metadata: {err}") + }); + + assert_eq!(report.info.sof_kind, SofKind::Extended12, "{name}"); + assert_eq!(report.info.restart_interval, Some(1), "{name}"); + assert_eq!(report.info.dimensions, expected_dimensions, "{name}"); + assert_eq!(report.info.color_space, expected_color, "{name}"); + assert_eq!(report.info.sampling.max_h, expected_sampling.0, "{name}"); + assert_eq!(report.info.sampling.max_v, expected_sampling.1, "{name}"); + assert!(report.cpu.eligible, "{name} op {op:?}"); + assert!(!report.owned_cuda.eligible, "{name}"); + assert!(!report.metal_fast.eligible, "{name}"); + } + } +} + +#[test] +fn capability_report_marks_extended12_rgba16_cpu_eligible() { + for (name, input, expected_color, expected_dimensions, expected_sampling) in [ + ( + "grayscale", + grayscale_sof_jpeg(0xc1, 12), + ColorSpace::Grayscale, + (8, 8), + (1, 1), + ), + ( + "APP14 RGB 4:4:4", + extended_12bit_rgb_8x8_jpeg(), + ColorSpace::Rgb, + (8, 8), + (1, 1), + ), + ( + "YCbCr 4:4:4", + extended_12bit_ycbcr_8x8_jpeg(), + ColorSpace::YCbCr, + (8, 8), + (1, 1), + ), + ( + "YCbCr 4:2:2", + extended_12bit_ycbcr_422_32x8_jpeg(), + ColorSpace::YCbCr, + (32, 8), + (2, 1), + ), + ( + "YCbCr 4:2:0", + extended_12bit_ycbcr_420_32x32_jpeg(), + ColorSpace::YCbCr, + (32, 32), + (2, 2), + ), + ] { + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgba16, + }, + ) + .unwrap_or_else(|err| { + panic!("capability report should parse 12-bit extended {name} metadata: {err}") + }); + + assert_eq!(report.info.sof_kind, SofKind::Extended12, "{name}"); + assert_eq!(report.info.bit_depth, 12, "{name}"); + assert_eq!(report.info.dimensions, expected_dimensions, "{name}"); + assert_eq!(report.info.color_space, expected_color, "{name}"); + assert_eq!(report.info.sampling.max_h, expected_sampling.0, "{name}"); + assert_eq!(report.info.sampling.max_v, expected_sampling.1, "{name}"); + assert!(report.cpu.eligible, "{name} op {op:?}"); + assert!(!report.owned_cuda.eligible, "{name}"); + assert!(!report.metal_fast.eligible, "{name}"); + } + } +} + +#[test] +fn capability_report_marks_progressive12_gray16_and_rgb16_cpu_eligible() { + let input = progressive_12bit_grayscale_8x8_jpeg(); + for fmt in [PixelFormat::Gray16, PixelFormat::Rgb16] { + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 2, + y: 1, + w: 3, + h: 4, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect(&input, JpegCapabilityRequest { op, fmt }) + .expect("capability report should parse 12-bit SOF2 grayscale metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Progressive12); + assert_eq!(report.info.bit_depth, 12); + assert_eq!(report.info.dimensions, (8, 8)); + assert!(report.cpu.eligible, "fmt {fmt:?} op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_progressive12_rgba16_cpu_eligible() { + for (name, input, expected_color, expected_dimensions, expected_sampling) in [ + ( + "grayscale", + progressive_12bit_grayscale_8x8_jpeg(), + ColorSpace::Grayscale, + (8, 8), + (1, 1), + ), + ( + "APP14 RGB 4:4:4", + progressive_12bit_rgb_8x8_jpeg(), + ColorSpace::Rgb, + (8, 8), + (1, 1), + ), + ( + "YCbCr 4:4:4", + progressive_12bit_ycbcr_8x8_jpeg(), + ColorSpace::YCbCr, + (8, 8), + (1, 1), + ), + ( + "YCbCr 4:2:2", + progressive_12bit_ycbcr_422_32x8_jpeg(), + ColorSpace::YCbCr, + (32, 8), + (2, 1), + ), + ( + "YCbCr 4:2:0", + progressive_12bit_ycbcr_420_32x32_jpeg(), + ColorSpace::YCbCr, + (32, 32), + (2, 2), + ), + ] { + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: expected_dimensions.0 / 2, + h: expected_dimensions.1 / 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgba16, + }, + ) + .unwrap_or_else(|err| { + panic!("capability report should parse 12-bit progressive {name} metadata: {err}") + }); + + assert_eq!(report.info.sof_kind, SofKind::Progressive12, "{name}"); + assert_eq!(report.info.bit_depth, 12, "{name}"); + assert_eq!(report.info.dimensions, expected_dimensions, "{name}"); + assert_eq!(report.info.color_space, expected_color, "{name}"); + assert_eq!(report.info.sampling.max_h, expected_sampling.0, "{name}"); + assert_eq!(report.info.sampling.max_v, expected_sampling.1, "{name}"); + assert!(report.cpu.eligible, "{name} op {op:?}"); + assert!(!report.owned_cuda.eligible, "{name}"); + assert!(!report.metal_fast.eligible, "{name}"); + } + } +} + +#[test] +fn capability_report_marks_progressive12_app14_rgb_rgb16_cpu_eligible() { + let input = progressive_12bit_rgb_8x8_jpeg(); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 2, + y: 1, + w: 3, + h: 4, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb16, + }, + ) + .expect("capability report should parse 12-bit SOF2 APP14 RGB metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Progressive12); + assert_eq!(report.info.bit_depth, 12); + assert_eq!(report.info.color_space, ColorSpace::Rgb); + assert!(report.cpu.eligible, "op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_progressive12_ycbcr444_rgb16_cpu_eligible() { + let input = progressive_12bit_ycbcr_8x8_jpeg(); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 2, + y: 1, + w: 3, + h: 4, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb16, + }, + ) + .expect("capability report should parse 12-bit SOF2 YCbCr metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Progressive12); + assert_eq!(report.info.bit_depth, 12); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert!(report.cpu.eligible, "op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_progressive12_ycbcr422_rgb16_cpu_eligible() { + let input = progressive_12bit_ycbcr_422_32x8_jpeg(); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 13, + y: 0, + w: 8, + h: 4, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 13, + y: 0, + w: 8, + h: 4, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb16, + }, + ) + .expect("capability report should parse 12-bit SOF2 YCbCr 4:2:2 metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Progressive12); + assert_eq!(report.info.bit_depth, 12); + assert_eq!(report.info.dimensions, (32, 8)); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert_eq!(report.info.sampling.max_h, 2); + assert_eq!(report.info.sampling.max_v, 1); + assert!(report.cpu.eligible, "op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_progressive12_ycbcr420_rgb16_cpu_eligible() { + let input = progressive_12bit_ycbcr_420_32x32_jpeg(); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 13, + y: 14, + w: 10, + h: 10, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 13, + y: 14, + w: 10, + h: 10, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb16, + }, + ) + .expect("capability report should parse 12-bit SOF2 YCbCr 4:2:0 metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Progressive12); + assert_eq!(report.info.bit_depth, 12); + assert_eq!(report.info.dimensions, (32, 32)); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert_eq!(report.info.sampling.max_h, 2); + assert_eq!(report.info.sampling.max_v, 2); + assert!(report.cpu.eligible, "op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_extended12_restart_grayscale_cpu_eligible() { + let input = extended_12bit_grayscale_restart_16x8_jpeg(); + for fmt in [PixelFormat::Gray16, PixelFormat::Rgb16] { + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 2, + y: 1, + w: 12, + h: 6, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 2, + y: 1, + w: 12, + h: 6, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect(&input, JpegCapabilityRequest { op, fmt }) + .expect("capability report should parse 12-bit restart metadata"); + + assert_eq!(report.info.sof_kind, SofKind::Extended12); + assert_eq!(report.info.restart_interval, Some(1)); + assert_eq!(report.info.dimensions, (16, 8)); + assert_eq!(report.info.color_space, ColorSpace::Grayscale); + assert!(report.cpu.eligible, "fmt {fmt:?}, op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_lossless_common_predictor_gray8_full_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_predictor_grayscale_3x3_jpeg(predictor); + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Gray8, + }, + ) + .unwrap_or_else(|err| { + panic!("capability report should parse SOF3 predictor-{predictor} metadata: {err}") + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 8); + assert!(report.cpu.eligible, "predictor {predictor}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_lossless_common_predictor_gray8_roi_and_scaled_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_predictor_grayscale_3x3_jpeg(predictor); + for op in [ + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Gray8, + }, + ) + .unwrap_or_else(|err| { + panic!("capability report should parse SOF3 predictor-{predictor} metadata: {err}") + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 8); + assert!(report.cpu.eligible, "predictor {predictor} op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_lossless_16bit_gray16_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_predictor_grayscale_16bit_3x3_jpeg(predictor); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Gray16, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse 16-bit SOF3 predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 16); + assert!(report.cpu.eligible, "predictor {predictor} op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_restart_coded_lossless_grayscale_cpu_eligible() { + for predictor in 1..=7 { + let cases = [ + ( + lossless_restart_predictor_grayscale_3x3_jpeg(predictor), + PixelFormat::Gray8, + 8, + ), + ( + lossless_restart_predictor_grayscale_16bit_3x3_jpeg(predictor), + PixelFormat::Gray16, + 16, + ), + ]; + + for (input, fmt, bit_depth) in cases { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + fmt, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse restart-coded SOF3 predictor-{predictor} grayscale metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, bit_depth); + assert_eq!(report.info.restart_interval, Some(3)); + assert!(report.cpu.eligible, "predictor {predictor}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_lossless_app14_rgb8_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_predictor_rgb_3x3_jpeg(predictor); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb8, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse SOF3 APP14 RGB predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 8); + assert_eq!(report.info.color_space, ColorSpace::Rgb); + assert!(report.cpu.eligible, "predictor {predictor} op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_lossless_app14_rgb8_rgba8_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_predictor_rgb_3x3_jpeg(predictor); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgba8, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse SOF3 APP14 RGB RGBA8 predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 8); + assert_eq!(report.info.color_space, ColorSpace::Rgb); + assert!(report.cpu.eligible, "predictor {predictor} op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_lossless_app14_rgb16_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_predictor_rgb_16bit_3x3_jpeg(predictor); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb16, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse 16-bit SOF3 APP14 RGB predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 16); + assert_eq!(report.info.color_space, ColorSpace::Rgb); + assert!(report.cpu.eligible, "predictor {predictor} op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_lossless_app14_rgb16_rgba16_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_predictor_rgb_16bit_3x3_jpeg(predictor); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgba16, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse 16-bit SOF3 APP14 RGB RGBA16 predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 16); + assert_eq!(report.info.color_space, ColorSpace::Rgb); + assert!(report.cpu.eligible, "predictor {predictor} op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_restart_coded_lossless_app14_rgb8_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_restart_predictor_rgb_3x3_jpeg(predictor); + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + fmt: PixelFormat::Rgb8, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse restart-coded SOF3 APP14 RGB predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 8); + assert_eq!(report.info.color_space, ColorSpace::Rgb); + assert_eq!(report.info.restart_interval, Some(3)); + assert!(report.cpu.eligible, "predictor {predictor}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_lossless_ycbcr_rgb8_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_predictor_ycbcr_3x3_jpeg(predictor); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb8, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse SOF3 YCbCr predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 8); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert!(report.cpu.eligible, "predictor {predictor} op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_lossless_ycbcr_rgb8_rgba8_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_predictor_ycbcr_3x3_jpeg(predictor); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgba8, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse SOF3 YCbCr RGBA8 predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 8); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert!(report.cpu.eligible, "predictor {predictor} op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_restart_coded_lossless_ycbcr_rgb8_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_restart_predictor_ycbcr_3x3_jpeg(predictor); + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + fmt: PixelFormat::Rgb8, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse restart-coded SOF3 YCbCr predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 8); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert_eq!(report.info.restart_interval, Some(3)); + assert!(report.cpu.eligible, "predictor {predictor}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_lossless_ycbcr16_rgb16_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_predictor_ycbcr_16bit_3x3_jpeg(predictor); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgb16, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse SOF3 16-bit YCbCr predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 16); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert!(report.cpu.eligible, "predictor {predictor} op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_lossless_ycbcr16_rgba16_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_predictor_ycbcr_16bit_3x3_jpeg(predictor); + for op in [ + JpegDecodeOp::Full, + JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }), + JpegDecodeOp::Scaled(Downscale::Half), + JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + ] { + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op, + fmt: PixelFormat::Rgba16, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse SOF3 16-bit YCbCr RGBA16 predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 16); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert!(report.cpu.eligible, "predictor {predictor} op {op:?}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } + } +} + +#[test] +fn capability_report_marks_restart_coded_lossless_ycbcr16_rgb16_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_restart_predictor_ycbcr_16bit_3x3_jpeg(predictor); + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + fmt: PixelFormat::Rgb16, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse restart-coded SOF3 16-bit YCbCr predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 16); + assert_eq!(report.info.color_space, ColorSpace::YCbCr); + assert_eq!(report.info.restart_interval, Some(3)); + assert!(report.cpu.eligible, "predictor {predictor}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_marks_restart_coded_lossless_app14_rgb16_cpu_eligible() { + for predictor in 1..=7 { + let input = lossless_restart_predictor_rgb_16bit_3x3_jpeg(predictor); + let report = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + fmt: PixelFormat::Rgb16, + }, + ) + .unwrap_or_else(|err| { + panic!( + "capability report should parse restart-coded 16-bit SOF3 APP14 RGB predictor-{predictor} metadata: {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless); + assert_eq!(report.info.bit_depth, 16); + assert_eq!(report.info.color_space, ColorSpace::Rgb); + assert_eq!(report.info.restart_interval, Some(3)); + assert!(report.cpu.eligible, "predictor {predictor}"); + assert!(!report.owned_cuda.eligible); + assert!(!report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_rejects_unsupported_lossless_predictor_explicitly() { + let input = lossless_predictor_grayscale_3x3_jpeg(8); + let err = JpegCapabilityReport::inspect( + &input, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Gray8, + }, + ) + .expect_err("unsupported SOF3 predictor should not infer CPU eligibility from parsed info"); + + assert!(matches!( + err, + JpegError::UnsupportedPredictor { predictor: 8 } + )); +} + +#[test] +fn capability_report_rejects_unsupported_lossless_scan_shapes_without_info_fallback() { + let mut invalid_scan_params = lossless_predictor_grayscale_3x3_jpeg(1); + let sos = invalid_scan_params + .windows(2) + .position(|w| w == [0xff, 0xda]) + .expect("fixture has SOS"); + invalid_scan_params[sos + 8] = 1; + + let err = JpegCapabilityReport::inspect( + &invalid_scan_params, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Gray8, + }, + ) + .expect_err("unsupported SOF3 scan shape should not infer eligibility from parsed info"); + + assert!(matches!( + err, + JpegError::NotImplemented { + sof: SofKind::Lossless + } + )); +} + +#[test] +fn capability_report_marks_lossless_8bit_sampled_color_cpu_eligible() { + let requests = [ + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgb8, + }, + JpegCapabilityRequest { + op: JpegDecodeOp::Region(Rect { + x: 1, + y: 0, + w: 2, + h: 2, + }), + fmt: PixelFormat::Rgb8, + }, + JpegCapabilityRequest { + op: JpegDecodeOp::Scaled(Downscale::Half), + fmt: PixelFormat::Rgba8, + }, + JpegCapabilityRequest { + op: JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 0, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + fmt: PixelFormat::Rgba8, + }, + ]; + + for (input, color_space, dimensions, sampling, label) in [ + ( + lossless_rgb_8bit_422_4x2_jpeg(4), + ColorSpace::Rgb, + (4, 2), + [(2, 1), (1, 1), (1, 1)], + "4:2:2 APP14 RGB", + ), + ( + lossless_rgb_8bit_422_restart_4x2_jpeg(4), + ColorSpace::Rgb, + (4, 2), + [(2, 1), (1, 1), (1, 1)], + "4:2:2 APP14 RGB restart", + ), + ( + lossless_ycbcr_8bit_422_4x2_jpeg(4), + ColorSpace::YCbCr, + (4, 2), + [(2, 1), (1, 1), (1, 1)], + "4:2:2 YCbCr", + ), + ( + lossless_ycbcr_8bit_422_restart_4x2_jpeg(4), + ColorSpace::YCbCr, + (4, 2), + [(2, 1), (1, 1), (1, 1)], + "4:2:2 YCbCr restart", + ), + ( + lossless_rgb_8bit_420_4x4_jpeg(4), + ColorSpace::Rgb, + (4, 4), + [(2, 2), (1, 1), (1, 1)], + "4:2:0 APP14 RGB", + ), + ( + lossless_rgb_8bit_420_restart_4x4_jpeg(4), + ColorSpace::Rgb, + (4, 4), + [(2, 2), (1, 1), (1, 1)], + "4:2:0 APP14 RGB restart", + ), + ( + lossless_ycbcr_8bit_420_4x4_jpeg(4), + ColorSpace::YCbCr, + (4, 4), + [(2, 2), (1, 1), (1, 1)], + "4:2:0 YCbCr", + ), + ( + lossless_ycbcr_8bit_420_restart_4x4_jpeg(4), + ColorSpace::YCbCr, + (4, 4), + [(2, 2), (1, 1), (1, 1)], + "4:2:0 YCbCr restart", + ), + ] { + for request in requests { + let report = JpegCapabilityReport::inspect(&input, request).unwrap_or_else(|err| { + panic!( + "lossless SOF3 8-bit sampled {label} should report CPU-eligible capability metadata, got {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless, "{label}"); + assert_eq!(report.info.bit_depth, 8, "{label}"); + assert_eq!(report.info.dimensions, dimensions, "{label}"); + assert!( + matches!(report.info.restart_interval, None | Some(2)), + "{label}" + ); + assert_eq!(report.info.color_space, color_space, "{label}"); + assert_eq!(report.info.sampling.components(), &sampling, "{label}"); + assert!(report.cpu.eligible, "{label} {request:?}"); + assert_eq!(report.cpu.reason, None, "{label} {request:?}"); + assert!(!report.owned_cuda.eligible, "{label}"); + assert!(!report.metal_fast.eligible, "{label}"); + } + } +} + +#[test] +fn capability_report_marks_lossless_16bit_422_color_cpu_eligible() { + let requests = [ + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgb16, + }, + JpegCapabilityRequest { + op: JpegDecodeOp::Region(Rect { + x: 1, + y: 0, + w: 2, + h: 2, + }), + fmt: PixelFormat::Rgb16, + }, + JpegCapabilityRequest { + op: JpegDecodeOp::Scaled(Downscale::Half), + fmt: PixelFormat::Rgba16, + }, + JpegCapabilityRequest { + op: JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 0, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + fmt: PixelFormat::Rgba16, + }, + ]; + + for (input, color_space, label) in [ + ( + lossless_rgb_16bit_422_4x2_jpeg(4), + ColorSpace::Rgb, + "APP14 RGB", + ), + ( + lossless_rgb_16bit_422_restart_4x2_jpeg(4), + ColorSpace::Rgb, + "APP14 RGB restart", + ), + ( + lossless_ycbcr_16bit_422_4x2_jpeg(4), + ColorSpace::YCbCr, + "YCbCr", + ), + ( + lossless_ycbcr_16bit_422_restart_4x2_jpeg(4), + ColorSpace::YCbCr, + "YCbCr restart", + ), + ] { + for request in requests { + let report = JpegCapabilityReport::inspect(&input, request).unwrap_or_else(|err| { + panic!( + "lossless SOF3 16-bit 4:2:2 {label} should report CPU-eligible capability metadata, got {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless, "{label}"); + assert_eq!(report.info.bit_depth, 16, "{label}"); + assert_eq!(report.info.dimensions, (4, 2), "{label}"); + assert!( + matches!(report.info.restart_interval, None | Some(2)), + "{label}" + ); + assert_eq!(report.info.color_space, color_space, "{label}"); + assert_eq!(report.info.sampling.max_h, 2, "{label}"); + assert_eq!(report.info.sampling.max_v, 1, "{label}"); + assert_eq!( + report.info.sampling.components(), + &[(2, 1), (1, 1), (1, 1)], + "{label}" + ); + assert!(report.cpu.eligible, "{label} {request:?}"); + assert_eq!(report.cpu.reason, None, "{label} {request:?}"); + assert!(!report.owned_cuda.eligible, "{label}"); + assert!(!report.metal_fast.eligible, "{label}"); + } + } +} + +#[test] +fn capability_report_marks_lossless_16bit_420_color_cpu_eligible() { + let requests = [ + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgb16, + }, + JpegCapabilityRequest { + op: JpegDecodeOp::Region(Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }), + fmt: PixelFormat::Rgb16, + }, + JpegCapabilityRequest { + op: JpegDecodeOp::Scaled(Downscale::Half), + fmt: PixelFormat::Rgba16, + }, + JpegCapabilityRequest { + op: JpegDecodeOp::RegionScaled { + roi: Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }, + scale: Downscale::Half, + }, + fmt: PixelFormat::Rgba16, + }, + ]; + + for (input, color_space, label) in [ + ( + lossless_rgb_16bit_420_4x4_jpeg(4), + ColorSpace::Rgb, + "APP14 RGB", + ), + ( + lossless_rgb_16bit_420_restart_4x4_jpeg(4), + ColorSpace::Rgb, + "APP14 RGB restart", + ), + ( + lossless_ycbcr_16bit_420_4x4_jpeg(4), + ColorSpace::YCbCr, + "YCbCr", + ), + ( + lossless_ycbcr_16bit_420_restart_4x4_jpeg(4), + ColorSpace::YCbCr, + "YCbCr restart", + ), + ] { + for request in requests { + let report = JpegCapabilityReport::inspect(&input, request).unwrap_or_else(|err| { + panic!( + "lossless SOF3 16-bit 4:2:0 {label} should report CPU-eligible capability metadata, got {err}" + ) + }); + + assert_eq!(report.info.sof_kind, SofKind::Lossless, "{label}"); + assert_eq!(report.info.bit_depth, 16, "{label}"); + assert_eq!(report.info.dimensions, (4, 4), "{label}"); + assert!( + matches!(report.info.restart_interval, None | Some(2)), + "{label}" + ); + assert_eq!(report.info.color_space, color_space, "{label}"); + assert_eq!(report.info.sampling.max_h, 2, "{label}"); + assert_eq!(report.info.sampling.max_v, 2, "{label}"); + assert_eq!( + report.info.sampling.components(), + &[(2, 2), (1, 1), (1, 1)], + "{label}" + ); + assert!(report.cpu.eligible, "{label} {request:?}"); + assert_eq!(report.cpu.reason, None, "{label} {request:?}"); + assert!(!report.owned_cuda.eligible, "{label}"); + assert!(!report.metal_fast.eligible, "{label}"); + } + } +} + +#[test] +fn capability_report_rejects_malformed_subsampled_lossless_scan_params_without_info_fallback() { + let mut invalid_scan_params = lossless_ycbcr_16bit_422_3x3_jpeg(); + let sos = invalid_scan_params + .windows(2) + .position(|w| w == [0xff, 0xda]) + .expect("fixture has SOS"); + let scan_component_count = usize::from(invalid_scan_params[sos + 4]); + let se_offset = sos + 6 + scan_component_count * 2; + invalid_scan_params[se_offset] = 1; + + let err = JpegCapabilityReport::inspect( + &invalid_scan_params, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgb16, + }, + ) + .expect_err( + "malformed subsampled SOF3 scan shape should not infer eligibility from parsed info", + ); + + assert!(matches!( + err, + JpegError::NotImplemented { + sof: SofKind::Lossless + } + )); +} + +#[test] +fn capability_report_marks_owned_cuda_eligible_for_fast_422_and_444_rgb8() { + for (input, expected_dimensions, expected_shape) in [ + (BASELINE_422, (16, 8), "4:2:2"), + (BASELINE_444, (8, 8), "4:4:4"), + ] { + let report = JpegCapabilityReport::inspect( + input, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Rgb8, + }, + ) + .expect("capability report"); + + assert_eq!(report.info.dimensions, expected_dimensions); + assert!( + report.owned_cuda.eligible, + "owned CUDA must be eligible for full-tile RGB8 fast {expected_shape}" + ); + assert!(report.metal_fast.eligible); + } +} + +#[test] +fn capability_report_rejects_owned_cuda_for_scaled_or_non_rgb8_requests() { + let scaled = JpegCapabilityReport::inspect( + BASELINE_420, + JpegCapabilityRequest { + op: JpegDecodeOp::Scaled(Downscale::Quarter), + fmt: PixelFormat::Rgb8, + }, + ) + .expect("scaled capability report"); + let gray = JpegCapabilityReport::inspect( + BASELINE_420, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Gray8, + }, + ) + .expect("gray capability report"); + + assert!(!scaled.owned_cuda.eligible); + assert!(scaled + .owned_cuda + .reason + .expect("scaled cuda rejection") + .contains("full-tile RGB8")); + assert!(!gray.owned_cuda.eligible); + assert!(gray + .owned_cuda + .reason + .expect("gray cuda rejection") + .contains("full-tile RGB8")); +} + +#[test] +fn capability_report_keeps_roi_shape_visible_for_statumen_routing() { + let roi = Rect { + x: 4, + y: 4, + w: 8, + h: 8, + }; + let report = JpegCapabilityReport::inspect( + BASELINE_420, + JpegCapabilityRequest { + op: JpegDecodeOp::RegionScaled { + roi, + scale: Downscale::Quarter, + }, + fmt: PixelFormat::Rgb8, + }, + ) + .expect("roi capability report"); + + assert_eq!( + report.request.op, + JpegDecodeOp::RegionScaled { + roi, + scale: Downscale::Quarter, + } + ); + assert!(report.cpu.eligible); + assert!(!report.owned_cuda.eligible); +} + +#[test] +fn capability_report_exposes_resident_metal_rgb8_batch_output_eligibility() { + let roi = Rect { + x: 4, + y: 4, + w: 8, + h: 8, + }; + let report = JpegCapabilityReport::inspect( + BASELINE_420, + JpegCapabilityRequest { + op: JpegDecodeOp::RegionScaled { + roi, + scale: Downscale::Quarter, + }, + fmt: PixelFormat::Rgb8, + }, + ) + .expect("region-scaled capability report"); + + assert!(report.metal_fast.eligible); + assert!(report.metal_resident_rgb8_batch_output().eligible); +} + +#[test] +fn capability_report_distinguishes_metal_fast_shape_from_reusable_rgb8_output() { + let gray = JpegCapabilityReport::inspect( + BASELINE_420, + JpegCapabilityRequest { + op: JpegDecodeOp::Full, + fmt: PixelFormat::Gray8, + }, + ) + .expect("gray capability report"); + let region = JpegCapabilityReport::inspect( + BASELINE_420, + JpegCapabilityRequest { + op: JpegDecodeOp::Region(Rect { + x: 0, + y: 0, + w: 8, + h: 8, + }), + fmt: PixelFormat::Rgb8, + }, + ) + .expect("region capability report"); + + assert!(gray.metal_fast.eligible); + assert!(!gray.metal_resident_rgb8_batch_output().eligible); + assert!(gray + .metal_resident_rgb8_batch_output() + .reason + .expect("gray rejection") + .contains("RGB8")); + + assert!(region.metal_fast.eligible); + assert!(!region.metal_resident_rgb8_batch_output().eligible); + assert!(region + .metal_resident_rgb8_batch_output() + .reason + .expect("region rejection") + .contains("full, scaled, or region-scaled")); +} + +#[test] +fn adapter_device_plan_scan_bytes_keep_terminal_eoi() { + let decoder = Decoder::new(BASELINE_420).expect("decoder"); + let plan = j2k_jpeg::adapter::build_device_plan(&decoder, 4).expect("device plan"); + + assert!(plan.scan_bytes.ends_with(&[0xff, 0xd9])); +} + +#[test] +fn adapter_device_plan_checkpoint_cadence_handles_multi_mcu_inputs() { + let bytes = baseline_grayscale_jpeg(24, 24); + let decoder = Decoder::new(&bytes).expect("grayscale decoder"); + + let cadence_zero = + j2k_jpeg::adapter::build_device_plan(&decoder, 0).expect("zero-cadence plan"); + let cadence_two = j2k_jpeg::adapter::build_device_plan(&decoder, 2).expect("cadence-two plan"); + + assert_eq!( + cadence_zero + .checkpoints + .iter() + .map(|checkpoint| checkpoint.mcu_index) + .collect::>(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8] + ); + let zero_offsets = cadence_zero + .checkpoints + .iter() + .map(|checkpoint| checkpoint.scan_offset) + .collect::>(); + assert_eq!(zero_offsets.first(), Some(&0)); + assert!(zero_offsets.windows(2).all(|pair| pair[0] <= pair[1])); + assert_eq!( + cadence_two + .checkpoints + .iter() + .map(|checkpoint| checkpoint.mcu_index) + .collect::>(), + vec![0, 2, 4, 6, 8] + ); + let cadence_two_offsets = cadence_two + .checkpoints + .iter() + .map(|checkpoint| checkpoint.scan_offset) + .collect::>(); + assert_eq!(cadence_two_offsets.first(), Some(&0)); + assert!(cadence_two_offsets + .windows(2) + .all(|pair| pair[0] <= pair[1])); + assert!(cadence_two + .checkpoints + .iter() + .all(|checkpoint| checkpoint.bits_buffered <= 64 && checkpoint.expected_rst == 0)); +} + +#[test] +fn adapter_device_plan_restart_checkpoints_capture_resume_state() { + let bytes = restart_coded_grayscale_jpeg(24, 24); + let decoder = Decoder::new(&bytes).expect("restart-coded decoder"); + let plan = j2k_jpeg::adapter::build_device_plan(&decoder, 2).expect("device plan"); + + assert_eq!( + plan.checkpoints + .iter() + .map(|checkpoint| checkpoint.mcu_index) + .collect::>(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 8] + ); + assert_eq!( + plan.checkpoints + .iter() + .map(|checkpoint| checkpoint.scan_offset) + .collect::>(), + vec![0, 3, 6, 9, 12, 15, 18, 21, 24] + ); + assert_eq!( + plan.checkpoints + .iter() + .map(|checkpoint| checkpoint.expected_rst) + .collect::>(), + vec![0, 1, 2, 3, 4, 5, 6, 7, 0] + ); + assert!(plan + .checkpoints + .iter() + .all(|checkpoint| checkpoint.bits_buffered == 0 && checkpoint.prev_dc == [0; 4])); +} + +#[test] +fn adapter_device_plan_treats_dri_zero_as_non_restart_fast_path() { + let bytes = insert_restart_interval(BASELINE_420.to_vec(), 0); + let decoder = Decoder::new(&bytes).expect("decoder"); + let plan = j2k_jpeg::adapter::build_device_plan(&decoder, 2).expect("device plan"); + + assert_eq!(plan.restart_interval, None); + assert!(plan.matches_fast_420); + assert_eq!( + plan.checkpoints + .iter() + .map(|checkpoint| checkpoint.expected_rst) + .collect::>(), + vec![0; plan.checkpoints.len()] + ); +} + +#[test] +fn adapter_device_plan_handles_restart_after_partial_entropy_byte() { + let bytes = restart_coded_grayscale_jpeg(16, 8); + let decoder = Decoder::new(&bytes).expect("restart-coded decoder"); + let plan = j2k_jpeg::adapter::build_device_plan(&decoder, 2).expect("device plan"); + + assert_eq!(plan.checkpoints.len(), 2); + assert_eq!(plan.checkpoints[1].mcu_index, 1); + assert_eq!(plan.checkpoints[1].scan_offset, 3); + assert_eq!(plan.checkpoints[1].expected_rst, 1); +} + +#[test] +fn adapter_device_plan_surfaces_missing_eoi_warning() { + let mut bytes = baseline_grayscale_jpeg(24, 24); + bytes.truncate(bytes.len() - 2); + + let decoder = Decoder::new(&bytes).expect("decoder"); + let plan = j2k_jpeg::adapter::build_device_plan(&decoder, 2) + .expect("missing EOI should remain decodable"); + + assert!(plan.warnings.contains(&Warning::MissingEoi)); +} + +#[test] +fn adapter_device_plan_treats_trailing_ff_as_missing_eoi() { + let mut bytes = baseline_grayscale_jpeg(24, 24); + bytes.truncate(bytes.len() - 1); + + let decoder = Decoder::new(&bytes).expect("decoder"); + let plan = j2k_jpeg::adapter::build_device_plan(&decoder, 2) + .expect("trailing FF should remain decodable"); + + assert!(plan.warnings.contains(&Warning::MissingEoi)); + assert_eq!(plan.scan_bytes.last(), Some(&0xff)); +} + +#[test] +fn adapter_device_plan_rejects_non_eoi_marker_after_entropy() { + let mut bytes = restart_coded_grayscale_jpeg(24, 24); + let marker = bytes + .windows(2) + .position(|window| matches!(window, [0xff, 0xd0..=0xd7])) + .expect("restart marker"); + bytes[marker + 1] = 0xe0; + + let decoder = Decoder::new(&bytes).expect("restart-coded decoder"); + let err = j2k_jpeg::adapter::build_device_plan(&decoder, 2) + .expect_err("unexpected marker should fail"); + + assert!(matches!( + err, + j2k_jpeg::JpegError::UnexpectedMarker { + expected: j2k_jpeg::MarkerKind::Eoi, + found: 0xe0, + .. + } + )); +} + +#[test] +fn adapter_device_plan_rejects_restart_marker_without_dri() { + let bytes = insert_entropy_marker(BASELINE_420.to_vec(), 0xd0); + let decoder = Decoder::new(&bytes).expect("decoder"); + let err = j2k_jpeg::adapter::build_device_plan(&decoder, 2) + .expect_err("restart marker without DRI must fail"); + + assert!(matches!( + err, + j2k_jpeg::JpegError::UnexpectedMarker { + expected: j2k_jpeg::MarkerKind::Eoi, + found: 0xd0, + .. + } + )); +} + +#[test] +fn adapter_device_plan_rejects_doubled_ff_before_terminal_eoi() { + let mut bytes = baseline_grayscale_jpeg(24, 24); + bytes.insert(bytes.len() - 1, 0xff); + + let decoder = Decoder::new(&bytes).expect("decoder"); + let err = j2k_jpeg::adapter::build_device_plan(&decoder, 2) + .expect_err("double-FF terminal marker should fail"); + + assert!(matches!( + err, + j2k_jpeg::JpegError::UnexpectedMarker { + expected: j2k_jpeg::MarkerKind::Eoi, + found: 0xff, + .. + } + )); +} + +fn grayscale_sof_jpeg(marker: u8, precision: u8) -> Vec { + let mut bytes = baseline_grayscale_jpeg(8, 8); + let sof = bytes + .windows(2) + .position(|window| window == [0xff, 0xc0]) + .expect("SOF0 marker"); + bytes[sof + 1] = marker; + bytes[sof + 4] = precision; + bytes +} + +fn progressive_12_bit_jpeg() -> Vec { + let mut bytes = progressive_8x8_jpeg(); + let sof = bytes + .windows(2) + .position(|window| window == [0xff, 0xc2]) + .expect("SOF2 marker"); + bytes[sof + 4] = 12; + bytes +} + +fn insert_restart_interval(mut bytes: Vec, interval: u16) -> Vec { + let sos = bytes + .windows(2) + .position(|window| window == [0xff, 0xda]) + .expect("SOS marker"); + bytes.splice( + sos..sos, + [ + 0xff, + 0xdd, + 0x00, + 0x04, + (interval >> 8) as u8, + interval as u8, + ], + ); + bytes +} + +fn insert_entropy_marker(mut bytes: Vec, marker: u8) -> Vec { + bytes.splice(bytes.len() - 2..bytes.len() - 2, [0xff, marker]); + bytes +} diff --git a/crates/signinum-jpeg/tests/encode_baseline.rs b/crates/j2k-jpeg/tests/encode_baseline.rs similarity index 97% rename from crates/signinum-jpeg/tests/encode_baseline.rs rename to crates/j2k-jpeg/tests/encode_baseline.rs index 4642266e..fa806142 100644 --- a/crates/signinum-jpeg/tests/encode_baseline.rs +++ b/crates/j2k-jpeg/tests/encode_baseline.rs @@ -1,8 +1,8 @@ -use signinum_jpeg::{ +use j2k_jpeg::{ encode_jpeg_baseline, DecodeOptions, Decoder, EncodedJpeg, JpegBackend, JpegEncodeOptions, JpegSamples, JpegSubsampling, PixelFormat, }; -use signinum_test_support::{patterned_gray8, patterned_rgb8}; +use j2k_test_support::{patterned_gray8, patterned_rgb8}; use std::io::Cursor; fn encode_rgb(subsampling: JpegSubsampling) -> EncodedJpeg { diff --git a/crates/signinum-jpeg/tests/external_wsi.rs b/crates/j2k-jpeg/tests/external_wsi.rs similarity index 93% rename from crates/signinum-jpeg/tests/external_wsi.rs rename to crates/j2k-jpeg/tests/external_wsi.rs index 6c36a48e..4c004640 100644 --- a/crates/signinum-jpeg/tests/external_wsi.rs +++ b/crates/j2k-jpeg/tests/external_wsi.rs @@ -5,7 +5,7 @@ #[path = "../benches/common/classification.rs"] mod classification; -use signinum_jpeg::{ColorSpace, Decoder, Downscale, JpegError, PixelFormat, RowSink}; +use j2k_jpeg::{ColorSpace, Decoder, Downscale, JpegError, PixelFormat, RowSink}; use std::fs; use std::path::{Path, PathBuf}; @@ -126,11 +126,11 @@ fn force_full_frame_policy_disables_large_rgb_skip() { #[test] fn extracted_wsi_jpegs_decode_when_local_corpus_is_available() { - let require_corpus = std::env::var_os("SIGNINUM_REQUIRE_WSI_ROOT").is_some(); - let Some(root) = std::env::var_os("SIGNINUM_WSI_ROOT") else { + let require_corpus = std::env::var_os("J2K_REQUIRE_WSI_ROOT").is_some(); + let Some(root) = std::env::var_os("J2K_WSI_ROOT") else { assert!( !require_corpus, - "SIGNINUM_REQUIRE_WSI_ROOT is set but SIGNINUM_WSI_ROOT is not configured" + "J2K_REQUIRE_WSI_ROOT is set but J2K_WSI_ROOT is not configured" ); return; }; @@ -139,7 +139,7 @@ fn extracted_wsi_jpegs_decode_when_local_corpus_is_available() { files.sort(); assert!( !require_corpus || !files.is_empty(), - "SIGNINUM_REQUIRE_WSI_ROOT is set but SIGNINUM_WSI_ROOT did not contain any JPEG files" + "J2K_REQUIRE_WSI_ROOT is set but J2K_WSI_ROOT did not contain any JPEG files" ); for path in files { diff --git a/crates/signinum-jpeg/tests/fast420_profile.rs b/crates/j2k-jpeg/tests/fast420_profile.rs similarity index 65% rename from crates/signinum-jpeg/tests/fast420_profile.rs rename to crates/j2k-jpeg/tests/fast420_profile.rs index 79644f60..89d64c9c 100644 --- a/crates/signinum-jpeg/tests/fast420_profile.rs +++ b/crates/j2k-jpeg/tests/fast420_profile.rs @@ -1,12 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_jpeg::bench_support::bench_profile_fast420_tile_batch; +use j2k_jpeg::bench_support::bench_profile_fast420_tile_batch; +use j2k_test_support::{JPEG_BASELINE_420_16X16, JPEG_GRAYSCALE_8X8}; #[test] fn profiles_baseline_420_fast_tile_batch() { - let bytes = include_bytes!("../fixtures/conformance/baseline_420_16x16.jpg"); - - let profile = bench_profile_fast420_tile_batch(bytes, 3) + let profile = bench_profile_fast420_tile_batch(JPEG_BASELINE_420_16X16, 3) .expect("profile should not fail") .expect("baseline 4:2:0 fixture should use fast tile path"); @@ -26,9 +25,8 @@ fn profiles_baseline_420_fast_tile_batch() { #[test] fn skips_non_fast_tile_inputs_without_error() { - let bytes = include_bytes!("../fixtures/conformance/grayscale_8x8.jpg"); - - let profile = bench_profile_fast420_tile_batch(bytes, 1).expect("profile should not fail"); + let profile = + bench_profile_fast420_tile_batch(JPEG_GRAYSCALE_8X8, 1).expect("profile should not fail"); assert!(profile.is_none()); } diff --git a/crates/signinum-jpeg/tests/metal_fast420_packet.rs b/crates/j2k-jpeg/tests/fast_packet.rs similarity index 70% rename from crates/signinum-jpeg/tests/metal_fast420_packet.rs rename to crates/j2k-jpeg/tests/fast_packet.rs index d3feb564..31876711 100644 --- a/crates/signinum-jpeg/tests/metal_fast420_packet.rs +++ b/crates/j2k-jpeg/tests/fast_packet.rs @@ -1,9 +1,15 @@ -mod fixtures; +use j2k_test_support as fixtures; -use signinum_jpeg::adapter::metal_fast420::{ - build_metal_fast420_packet, build_metal_fast422_packet, build_metal_fast444_packet, - build_metal_gray_packet, MetalFast420PacketError, +use j2k_jpeg::adapter::{ + build_device_plan, + fast_packet::{ + build_fast420_packet, build_fast422_packet, build_fast444_packet, build_gray_packet, + FastPacketError, JpegEntropyCheckpointV1, + }, }; +use j2k_jpeg::Decoder; + +const FAST_PACKET_MAX_NONRESTART_CHECKPOINTS: u32 = 2048; fn rewrite_three_component_ids(mut bytes: Vec, component_ids: [u8; 3]) -> Vec { assert_eq!(&bytes[..2], &[0xff, 0xd8], "fixture must start with SOI"); @@ -98,10 +104,64 @@ fn strip_dri_and_restart_markers(bytes: &[u8]) -> Vec { out } +fn assert_fast_checkpoints_match_device_plan( + bytes: &[u8], + packet_total_mcus: u32, + packet_checkpoints: &[JpegEntropyCheckpointV1], +) { + let decoder = Decoder::new(bytes).expect("device-plan decoder"); + let cadence = packet_total_mcus + .div_ceil(FAST_PACKET_MAX_NONRESTART_CHECKPOINTS) + .max(1); + let device_plan = build_device_plan(&decoder, cadence).expect("device plan"); + + assert_eq!(packet_checkpoints.len(), device_plan.checkpoints.len()); + for (packet, device) in packet_checkpoints + .iter() + .zip(device_plan.checkpoints.iter()) + { + assert_eq!(packet.mcu_index, device.mcu_index); + assert_eq!( + packet.entropy_pos, + destuffed_entropy_offset(&device_plan.scan_bytes, device.scan_offset) + ); + assert_eq!(packet.bit_acc, device.bit_accumulator); + assert_eq!(packet.bit_count, u32::from(device.bits_buffered)); + assert_eq!(packet.y_prev_dc, device.prev_dc[0]); + assert_eq!(packet.cb_prev_dc, device.prev_dc[1]); + assert_eq!(packet.cr_prev_dc, device.prev_dc[2]); + } +} + +fn destuffed_entropy_offset(scan_bytes: &[u8], target: usize) -> u32 { + let mut pos = 0usize; + let mut destuffed = 0u32; + while pos < target { + if scan_bytes[pos] != 0xff { + pos += 1; + destuffed += 1; + continue; + } + let marker = scan_bytes[pos + 1]; + match marker { + 0x00 => { + pos += 2; + destuffed += 1; + } + 0xd0..=0xd7 | 0xd9 => { + pos += 2; + } + _ => panic!("unexpected entropy marker ff{marker:02x}"), + } + } + assert_eq!(pos, target, "device checkpoint split a marker pair"); + destuffed +} + #[test] fn baseline_420_fixture_builds_fast420_packet() { let bytes = fixtures::minimal_baseline_420_jpeg(); - let packet = build_metal_fast420_packet(&bytes).expect("fast420 packet"); + let packet = build_fast420_packet(&bytes).expect("fast420 packet"); assert_eq!(packet.dimensions, (16, 16)); assert_eq!(packet.mcus_per_row, 1); @@ -121,7 +181,7 @@ fn baseline_420_fixture_builds_fast420_packet() { #[test] fn baseline_420_restart_fixture_builds_fast420_packet() { let bytes = fixtures::baseline_420_restart_32x16_jpeg(); - let packet = build_metal_fast420_packet(&bytes).expect("restart fast420 packet"); + let packet = build_fast420_packet(&bytes).expect("restart fast420 packet"); assert_eq!(packet.dimensions, (32, 16)); assert_eq!(packet.mcus_per_row, 2); @@ -137,12 +197,17 @@ fn baseline_420_restart_fixture_builds_fast420_packet() { assert_eq!(packet.entropy_checkpoints[0].entropy_pos, 0); assert_eq!(packet.entropy_checkpoints[0].bit_acc, 0); assert_eq!(packet.entropy_checkpoints[0].bit_count, 0); + assert_fast_checkpoints_match_device_plan( + &bytes, + packet.mcus_per_row * packet.mcu_rows, + &packet.entropy_checkpoints, + ); } #[test] fn stripped_restart_fixture_builds_nonrestart_entropy_checkpoints() { let bytes = strip_dri_and_restart_markers(&fixtures::baseline_420_restart_32x16_jpeg()); - let packet = build_metal_fast420_packet(&bytes).expect("nonrestart fast420 packet"); + let packet = build_fast420_packet(&bytes).expect("nonrestart fast420 packet"); assert_eq!(packet.restart_interval_mcus, 0); assert_eq!(packet.restart_offsets, vec![0]); @@ -151,12 +216,17 @@ fn stripped_restart_fixture_builds_nonrestart_entropy_checkpoints() { assert_eq!(packet.entropy_checkpoints[0].entropy_pos, 0); assert_eq!(packet.entropy_checkpoints[1].mcu_index, 1); assert!(packet.entropy_checkpoints[1].entropy_pos > packet.entropy_checkpoints[0].entropy_pos); + assert_fast_checkpoints_match_device_plan( + &bytes, + packet.mcus_per_row * packet.mcu_rows, + &packet.entropy_checkpoints, + ); } #[test] fn baseline_420_packet_accepts_zero_based_component_ids() { let bytes = rewrite_three_component_ids(fixtures::minimal_baseline_420_jpeg(), [0, 1, 2]); - let packet = build_metal_fast420_packet(&bytes).expect("fast420 packet"); + let packet = build_fast420_packet(&bytes).expect("fast420 packet"); assert_eq!(packet.dimensions, (16, 16)); assert_eq!(packet.mcus_per_row, 1); @@ -166,7 +236,7 @@ fn baseline_420_packet_accepts_zero_based_component_ids() { #[test] fn baseline_444_fixture_builds_fast444_packet() { let bytes = fixtures::baseline_444_8x8_jpeg(); - let packet = build_metal_fast444_packet(&bytes).expect("fast444 packet"); + let packet = build_fast444_packet(&bytes).expect("fast444 packet"); assert_eq!(packet.dimensions, (8, 8)); assert_eq!(packet.mcus_per_row, 1); @@ -187,7 +257,7 @@ fn baseline_444_fixture_builds_fast444_packet() { #[test] fn baseline_444_packet_accepts_zero_based_component_ids() { let bytes = rewrite_three_component_ids(fixtures::baseline_444_8x8_jpeg(), [0, 1, 2]); - let packet = build_metal_fast444_packet(&bytes).expect("fast444 packet"); + let packet = build_fast444_packet(&bytes).expect("fast444 packet"); assert_eq!(packet.dimensions, (8, 8)); assert_eq!(packet.mcus_per_row, 1); @@ -197,7 +267,7 @@ fn baseline_444_packet_accepts_zero_based_component_ids() { #[test] fn baseline_422_fixture_builds_fast422_packet() { let bytes = fixtures::baseline_422_16x8_jpeg(); - let packet = build_metal_fast422_packet(&bytes).expect("fast422 packet"); + let packet = build_fast422_packet(&bytes).expect("fast422 packet"); assert_eq!(packet.dimensions, (16, 8)); assert_eq!(packet.mcus_per_row, 1); @@ -217,7 +287,7 @@ fn baseline_422_fixture_builds_fast422_packet() { #[test] fn baseline_422_packet_accepts_zero_based_component_ids() { let bytes = rewrite_three_component_ids(fixtures::baseline_422_16x8_jpeg(), [0, 1, 2]); - let packet = build_metal_fast422_packet(&bytes).expect("fast422 packet"); + let packet = build_fast422_packet(&bytes).expect("fast422 packet"); assert_eq!(packet.dimensions, (16, 8)); assert_eq!(packet.mcus_per_row, 1); @@ -227,19 +297,18 @@ fn baseline_422_packet_accepts_zero_based_component_ids() { #[test] fn grayscale_fixture_is_rejected_for_fast420_subset() { let bytes = fixtures::grayscale_8x8_jpeg(); - let error = build_metal_fast420_packet(&bytes).expect_err("grayscale must be rejected"); + let error = build_fast420_packet(&bytes).expect_err("grayscale must be rejected"); assert!(matches!( error, - MetalFast420PacketError::UnsupportedColorSpace(_) - | MetalFast420PacketError::UnsupportedSampling + FastPacketError::UnsupportedColorSpace(_) | FastPacketError::UnsupportedSampling )); } #[test] fn grayscale_fixture_builds_gray_packet() { let bytes = fixtures::grayscale_8x8_jpeg(); - let packet = build_metal_gray_packet(&bytes).expect("gray packet"); + let packet = build_gray_packet(&bytes).expect("gray packet"); assert_eq!(packet.dimensions, (8, 8)); assert_eq!(packet.mcus_per_row, 1); @@ -256,7 +325,7 @@ fn grayscale_fixture_builds_gray_packet() { #[test] fn progressive_fixture_is_rejected_for_fast420_subset() { let bytes = fixtures::progressive_8x8_jpeg(); - let error = build_metal_fast420_packet(&bytes).expect_err("progressive must be rejected"); + let error = build_fast420_packet(&bytes).expect_err("progressive must be rejected"); - assert!(matches!(error, MetalFast420PacketError::UnsupportedSof(_))); + assert!(matches!(error, FastPacketError::UnsupportedSof(_))); } diff --git a/crates/signinum-jpeg/tests/idct_parity.proptest-regressions b/crates/j2k-jpeg/tests/idct_parity.proptest-regressions similarity index 100% rename from crates/signinum-jpeg/tests/idct_parity.proptest-regressions rename to crates/j2k-jpeg/tests/idct_parity.proptest-regressions diff --git a/crates/signinum-jpeg/tests/idct_parity.rs b/crates/j2k-jpeg/tests/idct_parity.rs similarity index 95% rename from crates/signinum-jpeg/tests/idct_parity.rs rename to crates/j2k-jpeg/tests/idct_parity.rs index f94a7ac7..ea59a09d 100644 --- a/crates/signinum-jpeg/tests/idct_parity.rs +++ b/crates/j2k-jpeg/tests/idct_parity.rs @@ -13,7 +13,7 @@ use proptest::prelude::*; fn scalar(input: &[i16; 64]) -> [u8; 64] { let mut out = [0u8; 64]; - signinum_jpeg::bench_support::bench_idct_reference_block_with(input, &mut out); + j2k_jpeg::bench_support::bench_idct_reference_block_with(input, &mut out); out } @@ -21,7 +21,7 @@ fn scalar(input: &[i16; 64]) -> [u8; 64] { fn assert_neon_matches_scalar(input: &[i16; 64]) { let scalar_out = scalar(input); let mut neon_out = [0u8; 64]; - signinum_jpeg::bench_support::bench_idct_neon_block(input, &mut neon_out); + j2k_jpeg::bench_support::bench_idct_neon_block(input, &mut neon_out); assert_eq!( scalar_out, neon_out, "NEON IDCT diverged from scalar on input {input:?}" @@ -35,7 +35,7 @@ fn assert_avx2_matches_scalar(input: &[i16; 64]) { } let scalar_out = scalar(input); let mut avx_out = [0u8; 64]; - signinum_jpeg::bench_support::bench_idct_avx2_block(input, &mut avx_out); + j2k_jpeg::bench_support::bench_idct_avx2_block(input, &mut avx_out); assert_eq!( scalar_out, avx_out, "AVX2 IDCT diverged from scalar on input {input:?}" diff --git a/crates/j2k-jpeg/tests/inspect.rs b/crates/j2k-jpeg/tests/inspect.rs new file mode 100644 index 00000000..15392ac5 --- /dev/null +++ b/crates/j2k-jpeg/tests/inspect.rs @@ -0,0 +1,639 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Integration tests for `Decoder::inspect`. + +use j2k_jpeg::{ + find_scan_ranges, is_sof_marker, iter_segments, parse_dri, parse_sof_info, + prepare_tiff_jpeg_tile, rewrite_sof_dimensions, ColorSpace, ColorTransform, DecodeOptions, + Decoder, DuplicateTablePolicy, JpegError, JpegTilePrepareOptions, JpegView, McuGeometry, + PreparedJpeg, RestartSegment, SofKind, UnsupportedReason, +}; +use j2k_jpeg::{ + CompressedPayloadKind, CompressedTransferSyntax, PassthroughDecision, PassthroughRequirements, +}; + +use fixtures::progressive_8x8_jpeg; +use j2k_test_support as fixtures; +use j2k_test_support::restart_coded_grayscale_jpeg; + +fn minimal_baseline_jpeg() -> Vec { + // Same construction as parse::header::tests — duplicated here because + // integration tests cannot access pub(crate) helpers. + let mut v = Vec::new(); + v.extend_from_slice(&[0xFF, 0xD8]); + v.extend_from_slice(&[0xFF, 0xDB, 0x00, 67, 0x00]); + v.extend(core::iter::repeat_n(1u8, 64)); + v.extend_from_slice(&[ + 0xFF, + 0xC0, + 0x00, + 17, + 8, + 0, + 16, + 0, + 16, + 3, + 1, + (2 << 4) | 2, + 0, + 2, + (1 << 4) | 1, + 0, + 3, + (1 << 4) | 1, + 0, + ]); + // DHT length = 2 (length field) + 1 (Tc/Th) + 16 (bits[]) + 1 (value) = 20 + v.extend_from_slice(&[ + 0xFF, 0xC4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xAA, + ]); + v.extend_from_slice(&[ + 0xFF, 0xC4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xBB, + ]); + v.extend_from_slice(&[0xFF, 0xDA, 0x00, 12, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0]); + v.extend_from_slice(&[0x00, 0xFF, 0xD9]); + v +} + +fn minimal_baseline_jpeg_with_restart_interval(interval: u16) -> Vec { + let mut bytes = minimal_baseline_jpeg(); + let sos_pos = bytes + .windows(2) + .position(|window| window == [0xff, 0xda]) + .expect("SOS marker"); + bytes.splice( + sos_pos..sos_pos, + [ + 0xff, + 0xdd, + 0x00, + 0x04, + (interval >> 8) as u8, + interval as u8, + ], + ); + bytes +} + +fn minimal_jpeg_with_sof_marker(marker: u8) -> Vec { + let mut bytes = minimal_baseline_jpeg(); + let pos = bytes + .windows(2) + .position(|window| window == [0xff, 0xc0]) + .expect("minimal fixture has SOF0 marker"); + bytes[pos + 1] = marker; + bytes +} + +fn scan_data_offset(bytes: &[u8]) -> usize { + let sos_pos = bytes + .windows(2) + .position(|window| window == [0xff, 0xda]) + .expect("SOS marker"); + let len = u16::from_be_bytes([bytes[sos_pos + 2], bytes[sos_pos + 3]]) as usize; + sos_pos + 2 + len +} + +fn restart_marker_offsets(bytes: &[u8]) -> Vec { + bytes + .windows(2) + .enumerate() + .filter_map(|(offset, window)| { + (window[0] == 0xff && (0xd0..=0xd7).contains(&window[1])).then_some(offset) + }) + .collect() +} + +fn prepare_options() -> JpegTilePrepareOptions { + JpegTilePrepareOptions { + expected_dimensions: None, + duplicate_table_policy: DuplicateTablePolicy::RejectConflicting, + repair_zero_sof_dimensions: false, + validate_restart_markers: false, + } +} + +fn zero_sof_jpeg() -> Vec { + let mut bytes = minimal_baseline_jpeg(); + let sof = bytes + .windows(2) + .position(|window| window == [0xff, 0xc0]) + .expect("SOF"); + bytes[sof + 5] = 0; + bytes[sof + 6] = 0; + bytes[sof + 7] = 0; + bytes[sof + 8] = 0; + bytes +} + +fn split_tables_and_scan(bytes: &[u8]) -> (Vec, Vec) { + let ranges = find_scan_ranges(bytes).expect("scan ranges"); + let mut tables = Vec::new(); + tables.extend_from_slice(&[0xff, 0xd8]); + tables.extend_from_slice(&bytes[2..ranges.sos_marker_offset]); + tables.extend_from_slice(&[0xff, 0xd9]); + + let mut tile = Vec::new(); + tile.extend_from_slice(&bytes[ranges.sos_marker_offset..]); + (tables, tile) +} + +fn mutate_first_dqt_value(tables: &[u8]) -> Vec { + let mut out = tables.to_vec(); + let dqt = out + .windows(2) + .position(|window| window == [0xff, 0xdb]) + .expect("DQT"); + out[dqt + 5] ^= 0x7f; + out +} + +fn restart_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[0xff, 0xc0, 0x00, 11, 8, 0, 8, 0, 16, 1, 1, 0x11, 0]); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); + bytes.extend_from_slice(&[0x00, 0xff, 0xd0, 0x00, 0xff, 0xd9]); + bytes +} + +fn segment_payload(bytes: &[u8], marker: u8) -> &[u8] { + iter_segments(bytes) + .find_map(|segment| { + let segment = segment.expect("segment parses"); + (segment.marker == marker).then_some(segment.payload) + }) + .expect("marker present") +} + +#[test] +fn public_segment_iterator_reports_header_markers_without_stuffed_entropy_markers() { + let mut bytes = minimal_baseline_jpeg(); + let eoi = bytes.len() - 2; + bytes.splice(eoi..eoi, [0xff, 0x00, 0x7f]); + let markers = iter_segments(&bytes) + .map(|segment| segment.expect("segment parses").marker) + .collect::>(); + + assert_eq!(markers, vec![0xd8, 0xdb, 0xc0, 0xc4, 0xc4, 0xda, 0xd9]); +} + +#[test] +fn public_sof_and_dri_helpers_report_marker_facts() { + for marker in [ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf, + ] { + assert!(is_sof_marker(marker), "FF{marker:02X}"); + } + for marker in [0xc4, 0xd8, 0xd9, 0xda, 0xdb, 0xdd, 0xee] { + assert!(!is_sof_marker(marker), "FF{marker:02X}"); + } + + let bytes = minimal_baseline_jpeg(); + let sof = parse_sof_info(0xc0, segment_payload(&bytes, 0xc0)).expect("SOF parses"); + assert_eq!(sof.sof_kind, SofKind::Baseline8); + assert_eq!(sof.dimensions, (16, 16)); + assert_eq!(sof.sampling.components(), &[(2, 2), (1, 1), (1, 1)]); + assert_eq!(sof.component_ids, vec![1, 2, 3]); + assert_eq!(sof.quant_table_ids, vec![0, 0, 0]); + + assert_eq!(parse_dri(&[0x00, 0x00]).expect("zero DRI"), None); + assert_eq!(parse_dri(&[0x00, 0x08]).expect("nonzero DRI"), Some(8)); +} + +#[test] +fn public_scan_ranges_and_sof_rewrite_helpers_use_absolute_offsets() { + let bytes = minimal_baseline_jpeg(); + let ranges = find_scan_ranges(&bytes).expect("scan ranges"); + let sos = bytes + .windows(2) + .position(|window| window == [0xff, 0xda]) + .expect("SOS"); + assert_eq!(ranges.sos_marker_offset, sos); + assert_eq!(ranges.sos_payload_range, sos + 4..sos + 14); + assert_eq!(ranges.entropy_range, sos + 14..bytes.len() - 2); + assert_eq!(ranges.eoi_marker_offset, Some(bytes.len() - 2)); + + let rewritten = rewrite_sof_dimensions(&bytes, (32, 24)).expect("rewrite"); + let sof = parse_sof_info(0xc0, segment_payload(&rewritten, 0xc0)).expect("SOF parses"); + assert_eq!(sof.dimensions, (32, 24)); + assert_eq!(rewritten.len(), bytes.len()); +} + +#[test] +fn public_scan_ranges_accepts_multiscan_progressive_entropy_boundaries() { + let bytes = progressive_8x8_jpeg(); + let ranges = find_scan_ranges(&bytes).expect("progressive scan ranges"); + + assert_eq!(ranges.sos_marker_offset, 223); + assert_eq!(ranges.entropy_range, 237..241); + assert_eq!(ranges.eoi_marker_offset, None); +} + +#[test] +fn complete_tiff_jpeg_tile_preparation_returns_borrowed_bytes() { + let bytes = minimal_baseline_jpeg(); + let prepared = prepare_tiff_jpeg_tile(&bytes, None, prepare_options()).expect("prepared"); + + match prepared { + PreparedJpeg::Borrowed(slice) => assert!(std::ptr::eq(slice.as_ptr(), bytes.as_ptr())), + PreparedJpeg::Owned(_) => panic!("complete JPEG tile should stay borrowed"), + } +} + +#[test] +fn prepared_tiff_jpeg_bytes_decode_complete_tile() { + let bytes = minimal_baseline_jpeg(); + let prepared = prepare_tiff_jpeg_tile(&bytes, None, prepare_options()).expect("prepared"); + let info = Decoder::inspect(prepared.as_bytes()).expect("inspect prepared"); + + assert_eq!(info.dimensions, (16, 16)); +} + +#[test] +fn prepared_tiff_jpeg_bytes_accept_multiscan_progressive_complete_tile() { + let bytes = progressive_8x8_jpeg(); + let prepared = prepare_tiff_jpeg_tile(&bytes, None, prepare_options()).expect("prepared"); + let info = Decoder::inspect(prepared.as_bytes()).expect("progressive inspect"); + + assert_eq!(info.sof_kind, SofKind::Progressive8); + assert_eq!(info.scan_count, 10); + assert!(matches!(prepared, PreparedJpeg::Borrowed(_))); +} + +#[test] +fn zero_sof_tiff_jpeg_dimensions_without_expected_dimensions_are_rejected() { + let err = prepare_tiff_jpeg_tile(&zero_sof_jpeg(), None, prepare_options()).unwrap_err(); + + assert!(matches!( + err, + JpegError::ZeroDimension { + width: 0, + height: 0 + } | JpegError::ExpectedDimensionsRequired { .. } + )); +} + +#[test] +fn tiff_jpeg_preparation_rejects_scan_without_sof() { + let tile = [ + 0xff, 0xd8, 0xff, 0xda, 0x00, 0x08, 1, 1, 0, 0, 63, 0, 0, 0xff, 0xd9, + ]; + let err = prepare_tiff_jpeg_tile(&tile, None, prepare_options()).unwrap_err(); + + assert!(matches!( + err, + JpegError::MissingMarker { .. } | JpegError::InvalidJpegAssembly { .. } + )); +} + +#[test] +fn abbreviated_tiff_jpeg_tile_with_jpeg_tables_assembles_decode_ready_stream() { + let full = minimal_baseline_jpeg(); + let (tables, tile) = split_tables_and_scan(&full); + let prepared = + prepare_tiff_jpeg_tile(&tile, Some(&tables), prepare_options()).expect("prepared"); + + assert!(matches!(prepared, PreparedJpeg::Owned(_))); + let info = Decoder::inspect(prepared.as_bytes()).expect("assembled inspect"); + assert_eq!(info.dimensions, (16, 16)); + assert!(prepared.as_bytes().starts_with(&[0xff, 0xd8])); + assert!(prepared.as_bytes().ends_with(&[0xff, 0xd9])); +} + +#[test] +fn jpeg_tables_soi_and_eoi_are_normalized_to_one_interchange_stream() { + let full = minimal_baseline_jpeg(); + let (tables, tile) = split_tables_and_scan(&full); + let prepared = + prepare_tiff_jpeg_tile(&tile, Some(&tables), prepare_options()).expect("prepared"); + let soi_count = prepared + .as_bytes() + .windows(2) + .filter(|window| *window == [0xff, 0xd8]) + .count(); + let eoi_count = prepared + .as_bytes() + .windows(2) + .filter(|window| *window == [0xff, 0xd9]) + .count(); + + assert_eq!(soi_count, 1); + assert_eq!(eoi_count, 1); +} + +#[test] +fn identical_duplicate_jpeg_tables_are_deduplicated_under_allow_identical() { + let full = minimal_baseline_jpeg(); + let (mut tables, tile) = split_tables_and_scan(&full); + let dqt = tables + .windows(2) + .position(|window| window == [0xff, 0xdb]) + .expect("DQT"); + let dqt_len = u16::from_be_bytes([tables[dqt + 2], tables[dqt + 3]]) as usize + 2; + let duplicate = tables[dqt..dqt + dqt_len].to_vec(); + tables.splice(dqt..dqt, duplicate); + let mut opts = prepare_options(); + opts.duplicate_table_policy = DuplicateTablePolicy::AllowIdentical; + + let prepared = prepare_tiff_jpeg_tile(&tile, Some(&tables), opts).expect("prepared"); + let dqt_count = prepared + .as_bytes() + .windows(2) + .filter(|window| *window == [0xff, 0xdb]) + .count(); + + assert_eq!(dqt_count, 1); +} + +#[test] +fn conflicting_duplicate_jpeg_tables_are_rejected() { + let full = minimal_baseline_jpeg(); + let (tables, tile) = split_tables_and_scan(&full); + let mut conflicting = mutate_first_dqt_value(&tables); + let dqt = tables + .windows(2) + .position(|window| window == [0xff, 0xdb]) + .expect("DQT"); + let dqt_len = u16::from_be_bytes([tables[dqt + 2], tables[dqt + 3]]) as usize + 2; + conflicting.splice(2..2, tables[dqt..dqt + dqt_len].iter().copied()); + + let err = prepare_tiff_jpeg_tile(&tile, Some(&conflicting), prepare_options()).unwrap_err(); + assert!(matches!(err, JpegError::ConflictingDuplicateTable { .. })); +} + +#[test] +fn zero_sof_dimensions_are_repaired_with_expected_dimensions() { + let bytes = zero_sof_jpeg(); + let mut opts = prepare_options(); + opts.expected_dimensions = Some((16, 16)); + opts.repair_zero_sof_dimensions = true; + let prepared = prepare_tiff_jpeg_tile(&bytes, None, opts).expect("prepared"); + let info = Decoder::inspect(prepared.as_bytes()).expect("inspect repaired"); + + assert_eq!(info.dimensions, (16, 16)); + assert!(matches!(prepared, PreparedJpeg::Owned(_))); +} + +#[test] +fn nonzero_sof_dimensions_conflicting_with_expected_dimensions_are_rejected() { + let bytes = minimal_baseline_jpeg(); + let mut opts = prepare_options(); + opts.expected_dimensions = Some((32, 16)); + + let err = prepare_tiff_jpeg_tile(&bytes, None, opts).unwrap_err(); + assert!(matches!( + err, + JpegError::ConflictingExpectedDimensions { .. } + )); +} + +#[test] +fn dri_survives_preparation_and_restart_validation_accepts_ordered_rst_markers() { + let bytes = restart_jpeg(); + let mut opts = prepare_options(); + opts.validate_restart_markers = true; + let prepared = prepare_tiff_jpeg_tile(&bytes, None, opts).expect("prepared"); + let info = Decoder::inspect(prepared.as_bytes()).expect("inspect"); + + assert_eq!(info.restart_interval, Some(1)); +} + +#[test] +fn restart_validation_rejects_out_of_order_rst_marker() { + let mut bytes = restart_jpeg(); + let rst = bytes + .windows(2) + .position(|window| window == [0xff, 0xd0]) + .expect("RST0"); + bytes[rst + 1] = 0xd3; + let mut opts = prepare_options(); + opts.validate_restart_markers = true; + + let err = prepare_tiff_jpeg_tile(&bytes, None, opts).unwrap_err(); + assert!(matches!(err, JpegError::RestartMismatch { .. })); +} + +#[test] +fn inspect_returns_info_for_valid_baseline_jpeg() { + let info = Decoder::inspect(&minimal_baseline_jpeg()).unwrap(); + assert_eq!(info.dimensions, (16, 16)); + assert_eq!(info.sof_kind, SofKind::Baseline8); + assert_eq!(info.color_space, ColorSpace::YCbCr); + assert_eq!(info.bit_depth, 8); + assert!(info.restart_interval.is_none()); + assert_eq!( + info.mcu_geometry, + McuGeometry { + width: 16, + height: 16, + columns: 1, + rows: 1, + count: 1, + } + ); + assert_eq!(info.scan_count, 1, "single SOS → scan_count must be 1"); +} + +#[test] +fn decode_options_color_transform_setter_round_trips() { + let mut options = DecodeOptions::default(); + options.set_color_transform(ColorTransform::ForceRgb); + assert!(matches!( + options.color_transform(), + ColorTransform::ForceRgb + )); +} + +#[test] +fn inspect_with_options_forces_three_component_color_space() { + let bytes = minimal_baseline_jpeg(); + let auto = Decoder::inspect(&bytes).unwrap(); + assert_eq!(auto.color_space, ColorSpace::YCbCr); + + let force_rgb = Decoder::inspect_with_options( + &bytes, + DecodeOptions::default().with_color_transform(ColorTransform::ForceRgb), + ) + .unwrap(); + assert_eq!(force_rgb.color_space, ColorSpace::Rgb); + + let force_ycbcr = Decoder::inspect_with_options( + &bytes, + DecodeOptions::default().with_color_transform(ColorTransform::ForceYCbCr), + ) + .unwrap(); + assert_eq!(force_ycbcr.color_space, ColorSpace::YCbCr); +} + +#[test] +fn inspect_returns_typed_error_for_empty_input() { + let err = Decoder::inspect(&[]).unwrap_err(); + assert!(matches!(err, JpegError::Truncated { .. })); +} + +#[test] +fn inspect_returns_typed_error_for_missing_sof() { + // SOI + EOI, nothing between + let bytes = &[0xFF, 0xD8, 0xFF, 0xD9]; + let err = Decoder::inspect(bytes).unwrap_err(); + assert!(matches!(err, JpegError::MissingMarker { .. })); +} + +#[test] +fn inspect_returns_typed_error_for_future_sof_classes() { + for (marker, expected_reason) in [ + (0xc9, UnsupportedReason::ArithmeticCoding), + (0xc5, UnsupportedReason::DifferentialBaseline), + (0xc6, UnsupportedReason::Hierarchical), + (0xcd, UnsupportedReason::ArithmeticAndHierarchical), + ] { + let bytes = minimal_jpeg_with_sof_marker(marker); + let err = Decoder::inspect(&bytes).unwrap_err(); + assert!(matches!( + err, + JpegError::UnsupportedSof { marker: got_marker, reason } + if got_marker == marker && reason == expected_reason + )); + assert!(err.is_unsupported()); + } +} + +#[test] +fn inspect_is_api_misuse_predicate_negative_for_all_parse_errors() { + // Parse errors are never API misuse. + let err = Decoder::inspect(&[]).unwrap_err(); + assert!(!err.is_api_misuse()); +} + +#[test] +fn inspect_reports_all_progressive_scans() { + let info = Decoder::inspect(&progressive_8x8_jpeg()).unwrap(); + assert_eq!(info.sof_kind, SofKind::Progressive8); + assert_eq!(info.scan_count, 10); +} + +#[test] +fn inspect_treats_dri_zero_as_no_restart_interval() { + let info = Decoder::inspect(&minimal_baseline_jpeg_with_restart_interval(0)).unwrap(); + assert!(info.restart_interval.is_none()); +} + +#[test] +fn inspect_reports_restart_interval_and_mcu_geometry_for_wsi_planning() { + let info = Decoder::inspect(&fixtures::baseline_420_restart_32x16_jpeg()).unwrap(); + + assert_eq!(info.dimensions, (32, 16)); + assert_eq!(info.restart_interval, Some(2)); + assert_eq!( + info.mcu_geometry, + McuGeometry { + width: 16, + height: 16, + columns: 2, + rows: 1, + count: 2, + } + ); +} + +#[test] +fn jpeg_view_restart_index_reports_original_byte_offsets() { + let bytes = restart_coded_grayscale_jpeg(24, 8); + let view = JpegView::parse(&bytes).expect("view"); + let index = view + .restart_index() + .expect("restart index") + .expect("DRI should produce an index"); + let scan_data_offset = scan_data_offset(&bytes); + let rst_offsets = restart_marker_offsets(&bytes); + + assert_eq!(index.scan_data_offset, scan_data_offset); + assert_eq!(index.interval_mcus, 1); + assert_eq!( + index.segments, + vec![ + RestartSegment { + start_mcu: 0, + entropy_offset: scan_data_offset, + marker_offset: None, + marker: None, + }, + RestartSegment { + start_mcu: 1, + entropy_offset: rst_offsets[0] + 2, + marker_offset: Some(rst_offsets[0]), + marker: Some(0xd0), + }, + RestartSegment { + start_mcu: 2, + entropy_offset: rst_offsets[1] + 2, + marker_offset: Some(rst_offsets[1]), + marker: Some(0xd1), + }, + ] + ); + + let decoder_index = Decoder::new(&bytes) + .expect("decoder") + .restart_index() + .expect("decoder restart index"); + assert_eq!(decoder_index, Some(index)); +} + +#[test] +fn restart_index_is_none_without_dri() { + let bytes = minimal_baseline_jpeg(); + let view = JpegView::parse(&bytes).expect("view"); + assert_eq!(view.restart_index().expect("restart index"), None); +} + +#[test] +fn jpeg_view_exposes_baseline_passthrough_candidate_with_original_bytes() { + let bytes = minimal_baseline_jpeg(); + let view = JpegView::parse(&bytes).expect("view"); + let candidate = view + .passthrough_candidate() + .expect("baseline JPEG passthrough candidate"); + let requirements = PassthroughRequirements::new( + CompressedTransferSyntax::JpegBaseline8, + CompressedPayloadKind::JpegInterchange, + ) + .with_dimensions((16, 16)) + .with_components(3) + .with_bit_depth(8); + + assert_eq!(view.bytes(), bytes.as_slice()); + assert_eq!( + candidate.transfer_syntax(), + CompressedTransferSyntax::JpegBaseline8 + ); + assert_eq!( + candidate.payload_kind(), + CompressedPayloadKind::JpegInterchange + ); + assert_eq!( + candidate.evaluate(&requirements), + PassthroughDecision::Copy { + bytes: bytes.as_slice() + } + ); +} + +#[test] +fn jpeg_progressive_is_not_offered_as_active_passthrough_candidate() { + let bytes = progressive_8x8_jpeg(); + let view = JpegView::parse(&bytes).expect("progressive view"); + + assert!(view.passthrough_candidate().is_none()); +} diff --git a/crates/signinum-jpeg/tests/libjpeg_turbo_compare.rs b/crates/j2k-jpeg/tests/libjpeg_turbo_compare.rs similarity index 77% rename from crates/signinum-jpeg/tests/libjpeg_turbo_compare.rs rename to crates/j2k-jpeg/tests/libjpeg_turbo_compare.rs index 183a3b3d..fdf48c39 100644 --- a/crates/signinum-jpeg/tests/libjpeg_turbo_compare.rs +++ b/crates/j2k-jpeg/tests/libjpeg_turbo_compare.rs @@ -3,34 +3,35 @@ #[path = "../benches/common/libjpeg_turbo.rs"] mod libjpeg_turbo; -use signinum_jpeg::{Decoder, Downscale, PixelFormat, Rect}; +use j2k_jpeg::{Decoder, Downscale, PixelFormat, Rect}; +use j2k_test_support::JPEG_BASELINE_420_16X16; #[test] -fn turbojpeg_rgb_and_region_match_signinum_fixture() { - let require_turbo = std::env::var_os("SIGNINUM_REQUIRE_LIBJPEG_TURBO").is_some(); +fn turbojpeg_rgb_and_region_match_j2k_fixture() { + let require_turbo = std::env::var_os("J2K_REQUIRE_LIBJPEG_TURBO").is_some(); let turbo_available = libjpeg_turbo::is_available(); assert!( !require_turbo || turbo_available, - "SIGNINUM_REQUIRE_LIBJPEG_TURBO is set but libjpeg-turbo is unavailable" + "J2K_REQUIRE_LIBJPEG_TURBO is set but libjpeg-turbo is unavailable" ); if !turbo_available { return; } - let bytes = include_bytes!("../fixtures/conformance/baseline_420_16x16.jpg"); - let dec = Decoder::new(bytes).expect("signinum decoder"); + let bytes = JPEG_BASELINE_420_16X16; + let dec = Decoder::new(bytes).expect("j2k decoder"); let mut turbo = libjpeg_turbo::TurboJpegDecoder::new().expect("turbojpeg decoder"); let info = turbo.inspect(bytes).expect("turbojpeg inspect"); assert_eq!((info.width, info.height), (16, 16)); - let (rgb, _) = dec.decode(PixelFormat::Rgb8).expect("signinum rgb"); + let (rgb, _) = dec.decode(PixelFormat::Rgb8).expect("j2k rgb"); let turbo_rgb = turbo.decode_rgb(bytes).expect("turbojpeg rgb"); assert_eq!(turbo_rgb, rgb); let (scaled, _) = dec .decode_scaled(PixelFormat::Rgb8, Downscale::Quarter) - .expect("signinum scaled"); + .expect("j2k scaled"); let turbo_scaled = turbo .decode_scaled_rgb(bytes, Downscale::Quarter) .expect("turbojpeg scaled"); diff --git a/crates/signinum-jpeg/tests/ndpi_passthrough.rs b/crates/j2k-jpeg/tests/ndpi_passthrough.rs similarity index 97% rename from crates/signinum-jpeg/tests/ndpi_passthrough.rs rename to crates/j2k-jpeg/tests/ndpi_passthrough.rs index 4d6c403a..fe318297 100644 --- a/crates/signinum-jpeg/tests/ndpi_passthrough.rs +++ b/crates/j2k-jpeg/tests/ndpi_passthrough.rs @@ -2,13 +2,13 @@ //! Optional local NDPI passthrough coverage. //! -//! Set `SIGNINUM_NDPI_PATH=/path/to/slide.ndpi` to run this test against a +//! Set `J2K_NDPI_PATH=/path/to/slide.ndpi` to run this test against a //! local slide. The test reads TIFF directories and compressed JPEG payloads; -//! it does not decode the whole-slide image. Set `SIGNINUM_NDPI_TILE_LIMIT=0` +//! it does not decode the whole-slide image. Set `J2K_NDPI_TILE_LIMIT=0` //! to validate every JPEG payload in the container. -use signinum_core::{CodedUnitLayout, Colorspace, Info as CoreInfo}; -use signinum_jpeg::{ +use j2k_core::{CodedUnitLayout, Colorspace, Info as CoreInfo}; +use j2k_jpeg::{ CompressedPayloadKind, CompressedTransferSyntax, JpegError, JpegView, PassthroughCandidate, PassthroughDecision, PassthroughRequirements, }; @@ -21,10 +21,10 @@ use std::{ time::Instant, }; -const NDPI_PATH_ENV: &str = "SIGNINUM_NDPI_PATH"; -const REQUIRE_NDPI_ENV: &str = "SIGNINUM_REQUIRE_NDPI"; -const TILE_LIMIT_ENV: &str = "SIGNINUM_NDPI_TILE_LIMIT"; -const MAX_PAYLOAD_BYTES_ENV: &str = "SIGNINUM_NDPI_MAX_PAYLOAD_BYTES"; +const NDPI_PATH_ENV: &str = "J2K_NDPI_PATH"; +const REQUIRE_NDPI_ENV: &str = "J2K_REQUIRE_NDPI"; +const TILE_LIMIT_ENV: &str = "J2K_NDPI_TILE_LIMIT"; +const MAX_PAYLOAD_BYTES_ENV: &str = "J2K_NDPI_MAX_PAYLOAD_BYTES"; const DEFAULT_TILE_LIMIT: usize = 8; const DEFAULT_BOUNDED_MAX_PAYLOAD_BYTES: u64 = 64 * 1024 * 1024; diff --git a/crates/signinum-jpeg/tests/neon_hot_paths.rs b/crates/j2k-jpeg/tests/neon_hot_paths.rs similarity index 99% rename from crates/signinum-jpeg/tests/neon_hot_paths.rs rename to crates/j2k-jpeg/tests/neon_hot_paths.rs index f97483eb..a6b9c574 100644 --- a/crates/signinum-jpeg/tests/neon_hot_paths.rs +++ b/crates/j2k-jpeg/tests/neon_hot_paths.rs @@ -2,7 +2,7 @@ #![cfg(target_arch = "aarch64")] -use signinum_jpeg::bench_support::{ +use j2k_jpeg::bench_support::{ bench_rgb_row_pair_from_420, bench_rgb_row_pair_from_420_reference, bench_rgb_row_pair_from_420_with_stats, Bench420DispatchStats, }; diff --git a/crates/signinum-jpeg/tests/parse_errors.rs b/crates/j2k-jpeg/tests/parse_errors.rs similarity index 98% rename from crates/signinum-jpeg/tests/parse_errors.rs rename to crates/j2k-jpeg/tests/parse_errors.rs index 05a85854..bf7b89c8 100644 --- a/crates/signinum-jpeg/tests/parse_errors.rs +++ b/crates/j2k-jpeg/tests/parse_errors.rs @@ -4,8 +4,8 @@ //! must always return a typed `JpegError`, and must not consume unbounded //! memory for crafted headers. +use j2k_jpeg::Decoder; use proptest::prelude::*; -use signinum_jpeg::Decoder; proptest! { #![proptest_config(ProptestConfig { cases: 4096, .. ProptestConfig::default() })] diff --git a/crates/signinum-jpeg/tests/regressions.rs b/crates/j2k-jpeg/tests/regressions.rs similarity index 81% rename from crates/signinum-jpeg/tests/regressions.rs rename to crates/j2k-jpeg/tests/regressions.rs index 95d8dcef..db26fc46 100644 --- a/crates/signinum-jpeg/tests/regressions.rs +++ b/crates/j2k-jpeg/tests/regressions.rs @@ -2,10 +2,11 @@ //! Regression coverage for structural decode bugs and allocation guardrails. -use signinum_jpeg::{Decoder, Downscale, JpegError, PixelFormat, RowSink}; +use j2k_jpeg::{Decoder, Downscale, JpegError, PixelFormat, RowSink}; +use j2k_test_support::{minimal_grayscale_jpeg_with_dimensions, restart_coded_grayscale_jpeg}; -mod fixtures; use fixtures::minimal_baseline_420_jpeg; +use j2k_test_support as fixtures; const R_ID: u8 = 0x52; const G_ID: u8 = 0x47; @@ -133,7 +134,7 @@ impl RowSink for NullSink { #[test] fn decode_rows_does_not_use_full_image_scratch_cap() { - let bytes = minimal_grayscale_jpeg((65_000, 65_000)); + let bytes = minimal_grayscale_jpeg_with_dimensions(65_000, 65_000); let dec = Decoder::new(&bytes).expect("header must parse before row decode"); let err = dec @@ -147,7 +148,7 @@ fn decode_rows_does_not_use_full_image_scratch_cap() { #[test] fn decode_into_handles_restart_marker_after_partial_entropy_byte() { - let bytes = grayscale_restart_jpeg(); + let bytes = restart_coded_grayscale_jpeg(16, 8); let dec = Decoder::new(&bytes).expect("restart fixture must parse"); let (width, height) = dec.info().dimensions; let stride = width as usize; @@ -228,55 +229,6 @@ fn minimal_baseline_jpeg((width, height): (u16, u16)) -> Vec { bytes } -fn minimal_grayscale_jpeg((width, height): (u16, u16)) -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0xff, 0xd8]); - bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); - bytes.extend(std::iter::repeat_n(16u8, 64)); - bytes.extend_from_slice(&[ - 0xff, - 0xc0, - 0x00, - 11, - 8, - (height >> 8) as u8, - height as u8, - (width >> 8) as u8, - width as u8, - 1, - 1, - 0x11, - 0, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); - bytes.extend_from_slice(&[0x00, 0xff, 0xd9]); - bytes -} - -fn grayscale_restart_jpeg() -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0xff, 0xd8]); - bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); - bytes.extend(std::iter::repeat_n(16u8, 64)); - bytes.extend_from_slice(&[0xff, 0xc0, 0x00, 11, 8, 0, 8, 0, 16, 1, 1, 0x11, 0]); - bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); - bytes.extend_from_slice(&[0x00, 0xff, 0xd0, 0x00, 0xff, 0xd9]); - bytes -} - fn rgb_app14_constant_jpeg(scan_order: [u8; 3]) -> Vec { let mut bytes = Vec::new(); bytes.extend_from_slice(&[0xff, 0xd8]); diff --git a/crates/signinum-jpeg/tests/scratch_reuse.rs b/crates/j2k-jpeg/tests/scratch_reuse.rs similarity index 95% rename from crates/signinum-jpeg/tests/scratch_reuse.rs rename to crates/j2k-jpeg/tests/scratch_reuse.rs index e8dfc522..fdf49112 100644 --- a/crates/signinum-jpeg/tests/scratch_reuse.rs +++ b/crates/j2k-jpeg/tests/scratch_reuse.rs @@ -3,10 +3,11 @@ //! Reusing a `ScratchPool` across many decodes must produce byte-identical //! output on every iteration. Regression guard for Phase 3. -use signinum_jpeg::{Decoder, Downscale, PixelFormat, Rect, ScratchPool}; +use j2k_jpeg::{Decoder, Downscale, PixelFormat, Rect, ScratchPool}; +use j2k_test_support::{JPEG_BASELINE_420_16X16, JPEG_GRAYSCALE_8X8}; -const BASELINE_420: &[u8] = include_bytes!("../fixtures/conformance/baseline_420_16x16.jpg"); -const GRAYSCALE_8X8: &[u8] = include_bytes!("../fixtures/conformance/grayscale_8x8.jpg"); +const BASELINE_420: &[u8] = JPEG_BASELINE_420_16X16; +const GRAYSCALE_8X8: &[u8] = JPEG_GRAYSCALE_8X8; #[test] fn rgb8_decode_is_byte_stable_across_pool_reuse() { diff --git a/crates/j2k-jpeg/tests/transcode_blocks.rs b/crates/j2k-jpeg/tests/transcode_blocks.rs new file mode 100644 index 00000000..6a400123 --- /dev/null +++ b/crates/j2k-jpeg/tests/transcode_blocks.rs @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_test_support as fixtures; + +use j2k_jpeg::transcode::{ + extract_dct_blocks, idct_islow_block, DctExtractOptions, JpegDctCodingMode, +}; + +#[test] +fn extracts_grayscale_dct_blocks() { + let image = extract_dct_blocks( + &fixtures::grayscale_8x8_jpeg(), + DctExtractOptions::default(), + ) + .expect("extract grayscale DCT blocks"); + + assert_eq!((image.width, image.height), (8, 8)); + assert_eq!(image.components.len(), 1); + assert_component(&image.components[0], (8, 8), (1, 1), (1, 1), 1); +} + +#[test] +fn exposes_quantized_and_dequantized_natural_order_blocks() { + let image = extract_dct_blocks( + &fixtures::baseline_444_8x8_jpeg(), + DctExtractOptions::default(), + ) + .expect("extract 4:4:4 DCT blocks"); + let component = &image.components[0]; + + assert_eq!( + component.quantized_blocks.len(), + component.dequantized_blocks.len() + ); + assert!(component + .quantized_blocks + .iter() + .any(|block| block.iter().any(|&coefficient| coefficient != 0))); + assert_ne!( + component.quantized_blocks[0], + component.dequantized_blocks[0] + ); + + for (quantized, dequantized) in component + .quantized_blocks + .iter() + .zip(component.dequantized_blocks.iter()) + { + for (zigzag_idx, &natural_idx) in JPEG_ZIGZAG.iter().enumerate() { + let expected = + i32::from(quantized[natural_idx]) * i32::from(component.quant_table[zigzag_idx]); + assert_eq!( + i32::from(dequantized[natural_idx]), + expected, + "dequantized coefficient at natural index {natural_idx}" + ); + } + } +} + +#[test] +fn dequantized_only_extraction_omits_quantized_blocks() { + let default_image = extract_dct_blocks( + &fixtures::baseline_422_16x8_jpeg(), + DctExtractOptions::default(), + ) + .expect("extract default DCT blocks"); + let dequantized_only = extract_dct_blocks( + &fixtures::baseline_422_16x8_jpeg(), + DctExtractOptions::dequantized_only(), + ) + .expect("extract dequantized-only DCT blocks"); + + assert_eq!( + dequantized_only.components.len(), + default_image.components.len() + ); + for (actual, expected) in dequantized_only + .components + .iter() + .zip(default_image.components.iter()) + { + assert!(actual.quantized_blocks.is_empty()); + assert_eq!(actual.dequantized_blocks, expected.dequantized_blocks); + assert_eq!(actual.block_cols, expected.block_cols); + assert_eq!(actual.block_rows, expected.block_rows); + } +} + +#[test] +fn extracts_ycbcr_444_dct_blocks() { + let image = extract_dct_blocks( + &fixtures::baseline_444_8x8_jpeg(), + DctExtractOptions::default(), + ) + .expect("extract 4:4:4 DCT blocks"); + + assert_eq!((image.width, image.height), (8, 8)); + assert_eq!(image.components.len(), 3); + for component in &image.components { + assert_component(component, (8, 8), (1, 1), (1, 1), 1); + } +} + +#[test] +fn extracts_ycbcr_422_dct_blocks_at_native_component_resolution() { + let image = extract_dct_blocks( + &fixtures::baseline_422_16x8_jpeg(), + DctExtractOptions::default(), + ) + .expect("extract 4:2:2 DCT blocks"); + + assert_eq!((image.width, image.height), (16, 8)); + assert_eq!(image.components.len(), 3); + assert_component(&image.components[0], (16, 8), (2, 1), (2, 1), 2); + assert_component(&image.components[1], (8, 8), (1, 1), (1, 1), 1); + assert_component(&image.components[2], (8, 8), (1, 1), (1, 1), 1); +} + +#[test] +fn extracts_ycbcr_420_dct_blocks_at_native_component_resolution() { + let image = extract_dct_blocks( + &fixtures::minimal_baseline_420_jpeg(), + DctExtractOptions::default(), + ) + .expect("extract 4:2:0 DCT blocks"); + + assert_eq!((image.width, image.height), (16, 16)); + assert_eq!(image.components.len(), 3); + assert_component(&image.components[0], (16, 16), (2, 2), (2, 2), 4); + assert_component(&image.components[1], (8, 8), (1, 1), (1, 1), 1); + assert_component(&image.components[2], (8, 8), (1, 1), (1, 1), 1); +} + +#[test] +fn extracts_restart_coded_ycbcr_420_blocks_and_restart_metadata() { + let image = extract_dct_blocks( + &fixtures::baseline_420_restart_32x16_jpeg(), + DctExtractOptions::default(), + ) + .expect("extract restart-coded 4:2:0 DCT blocks"); + + assert_eq!((image.width, image.height), (32, 16)); + assert_eq!( + image.restart_index.as_ref().map(|idx| idx.interval_mcus), + Some(2) + ); + assert_component(&image.components[0], (32, 16), (2, 2), (4, 2), 8); + assert_component(&image.components[1], (16, 8), (1, 1), (2, 1), 2); + assert_component(&image.components[2], (16, 8), (1, 1), (2, 1), 2); +} + +#[test] +fn extracts_progressive_ycbcr_420_dct_blocks_at_native_component_resolution() { + let image = extract_dct_blocks( + &fixtures::progressive_8x8_jpeg(), + DctExtractOptions::default(), + ) + .expect("extract progressive 4:2:0 DCT blocks"); + + assert_eq!((image.width, image.height), (8, 8)); + assert_eq!(image.coding_mode, JpegDctCodingMode::Progressive); + assert_eq!(image.scan_count, 10); + assert_eq!(image.components.len(), 3); + assert_component(&image.components[0], (8, 8), (2, 2), (2, 2), 4); + assert_component(&image.components[1], (4, 4), (1, 1), (1, 1), 1); + assert_component(&image.components[2], (4, 4), (1, 1), (1, 1), 1); + assert!(image.restart_index.is_none()); +} + +#[test] +fn exposes_scalar_islow_block_idct_for_transcode_oracles() { + let image = extract_dct_blocks( + &fixtures::grayscale_8x8_jpeg(), + DctExtractOptions::default(), + ) + .expect("extract grayscale DCT blocks"); + + let samples = idct_islow_block(&image.components[0].dequantized_blocks[0]); + + assert_eq!(samples.len(), 64); + assert!(samples.iter().any(|&sample| sample != 128)); +} + +fn assert_component( + component: &j2k_jpeg::transcode::JpegDctComponent, + sample_dimensions: (u32, u32), + sampling: (u8, u8), + block_grid: (u32, u32), + block_count: usize, +) { + assert_eq!( + (component.width, component.height), + sample_dimensions, + "component dimensions" + ); + assert_eq!((component.h_samp, component.v_samp), sampling, "sampling"); + assert_eq!( + (component.block_cols, component.block_rows), + block_grid, + "block grid" + ); + assert_eq!(component.dequantized_blocks.len(), block_count); + assert_eq!(component.quantized_blocks.len(), block_count); + assert!(component + .dequantized_blocks + .iter() + .any(|block| block.iter().any(|&coefficient| coefficient != 0))); + assert!(component + .quantized_blocks + .iter() + .any(|block| block.iter().any(|&coefficient| coefficient != 0))); +} + +const JPEG_ZIGZAG: [usize; 64] = [ + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, + 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59, + 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63, +]; diff --git a/crates/j2k-jpeg/tests/view_and_rows.rs b/crates/j2k-jpeg/tests/view_and_rows.rs new file mode 100644 index 00000000..5117b0c5 --- /dev/null +++ b/crates/j2k-jpeg/tests/view_and_rows.rs @@ -0,0 +1,465 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Integration tests for the parsed-view API and row-streaming decode surface. + +use j2k_jpeg::{ + ComponentRowWriter, Decoder, Downscale, JpegError, JpegView, PixelFormat, Rect, RowSink, + ScratchPool, +}; +use j2k_test_support::restart_coded_grayscale_jpeg; + +use fixtures::{ + cmyk_8x8_jpeg, grayscale_8x8_jpeg, lossless_predictor_grayscale_16bit_3x3_jpeg, + lossless_predictor_grayscale_3x3_jpeg, lossless_predictor_rgb_16bit_3x3_jpeg, + lossless_predictor_rgb_3x3_jpeg, lossless_predictor_ycbcr_16bit_3x3_jpeg, + lossless_predictor_ycbcr_3x3_jpeg, lossless_restart_predictor_grayscale_16bit_3x3_jpeg, + lossless_restart_predictor_grayscale_3x3_jpeg, lossless_restart_predictor_rgb_16bit_3x3_jpeg, + lossless_restart_predictor_rgb_3x3_jpeg, lossless_restart_predictor_ycbcr_16bit_3x3_jpeg, + lossless_restart_predictor_ycbcr_3x3_jpeg, lossless_ycbcr_16bit_3x3_rgb16, + lossless_ycbcr_3x3_rgb8, minimal_baseline_420_jpeg, rgb_app14_8x8_jpeg, ycck_8x8_jpeg, + LOSSLESS_GRAYSCALE_16BIT_3X3_PIXELS, LOSSLESS_GRAYSCALE_3X3_PIXELS, + LOSSLESS_RGB_16BIT_3X3_PIXELS, LOSSLESS_RGB_3X3_PIXELS, +}; +use j2k_test_support as fixtures; + +#[derive(Default)] +struct CollectRows { + rows: Vec<(u32, Vec)>, +} + +impl RowSink for CollectRows { + type Error = JpegError; + + fn write_row(&mut self, y: u32, row: &[u8]) -> Result<(), JpegError> { + self.rows.push((y, row.to_vec())); + Ok(()) + } +} + +#[derive(Default)] +struct CollectGrayComponentRows { + rows: Vec<(u32, Vec)>, +} + +impl ComponentRowWriter for CollectGrayComponentRows { + fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError> { + self.rows.push((y, gray_row.to_vec())); + Ok(()) + } + + fn write_ycbcr_row( + &mut self, + _y: u32, + _y_row: &[u8], + _cb_row: &[u8], + _cr_row: &[u8], + ) -> Result<(), JpegError> { + unreachable!("grayscale test writer should not receive ycbcr rows"); + } + + fn write_rgb_row( + &mut self, + _y: u32, + _r_row: &[u8], + _g_row: &[u8], + _b_row: &[u8], + ) -> Result<(), JpegError> { + unreachable!("grayscale test writer should not receive rgb rows"); + } +} + +#[derive(Default)] +struct CollectRgbComponentRows { + rows: Vec<(u32, Vec)>, +} + +impl ComponentRowWriter for CollectRgbComponentRows { + fn write_gray_row(&mut self, _y: u32, _gray_row: &[u8]) -> Result<(), JpegError> { + unreachable!("RGB component-row test writer should not receive gray rows"); + } + + fn write_ycbcr_row( + &mut self, + _y: u32, + _y_row: &[u8], + _cb_row: &[u8], + _cr_row: &[u8], + ) -> Result<(), JpegError> { + unreachable!("RGB component-row test writer should not receive ycbcr rows"); + } + + fn write_rgb_row( + &mut self, + y: u32, + r_row: &[u8], + g_row: &[u8], + b_row: &[u8], + ) -> Result<(), JpegError> { + let mut row = Vec::with_capacity(r_row.len() * 3); + for ((r, g), b) in r_row.iter().zip(g_row).zip(b_row) { + row.extend_from_slice(&[*r, *g, *b]); + } + self.rows.push((y, row)); + Ok(()) + } +} + +#[test] +fn jpeg_view_parse_matches_decoder_inspect() { + let bytes = minimal_baseline_420_jpeg(); + let view = JpegView::parse(&bytes).expect("parsed view must construct"); + let info = Decoder::inspect(&bytes).expect("inspect must succeed"); + assert_eq!(view.info(), &info); +} + +#[test] +fn decoder_from_view_matches_decoder_new_rgb_output() { + let bytes = rgb_app14_8x8_jpeg(); + let dec_from_new = Decoder::new(&bytes).expect("decoder::new must succeed"); + let dec_from_view = Decoder::from_view(JpegView::parse(&bytes).unwrap()) + .expect("decoder::from_view must succeed"); + + let (w, h) = dec_from_new.info().dimensions; + let stride = (w * 3) as usize; + let mut new_out = vec![0u8; stride * h as usize]; + let mut view_out = vec![0u8; stride * h as usize]; + + dec_from_new + .decode_scaled_into(&mut new_out, stride, PixelFormat::Rgb8, Downscale::None) + .unwrap(); + dec_from_view + .decode_scaled_into(&mut view_out, stride, PixelFormat::Rgb8, Downscale::None) + .unwrap(); + + assert_eq!(view_out, new_out); +} + +#[test] +fn decode_rows_matches_decode_into_rgb8() { + let bytes = minimal_baseline_420_jpeg(); + let dec = Decoder::new(&bytes).expect("decoder::new must succeed"); + let (w, h) = dec.info().dimensions; + let stride = (w * 3) as usize; + + let mut expected = vec![0u8; stride * h as usize]; + dec.decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, Downscale::None) + .unwrap(); + + let mut sink = CollectRows::default(); + dec.decode_rows(&mut sink) + .expect("decode_rows must succeed"); + + assert_eq!(sink.rows.len(), h as usize); + for (row_idx, (y, row)) in sink.rows.iter().enumerate() { + assert_eq!(*y as usize, row_idx); + assert_eq!(row.len(), stride); + assert_eq!( + row.as_slice(), + &expected[row_idx * stride..(row_idx + 1) * stride] + ); + } +} + +#[test] +fn decode_rows_matches_decode_into_rgb8_for_grayscale_input() { + let bytes = grayscale_8x8_jpeg(); + let dec = Decoder::new(&bytes).expect("decoder::new must succeed"); + let (w, h) = dec.info().dimensions; + let stride = (w * 3) as usize; + + let mut expected = vec![0u8; stride * h as usize]; + dec.decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, Downscale::None) + .unwrap(); + + let mut sink = CollectRows::default(); + dec.decode_rows(&mut sink) + .expect("decode_rows must succeed"); + + assert_eq!(sink.rows.len(), h as usize); + for (row_idx, (y, row)) in sink.rows.iter().enumerate() { + assert_eq!(*y as usize, row_idx); + assert_eq!( + row.as_slice(), + &expected[row_idx * stride..(row_idx + 1) * stride] + ); + } +} + +#[test] +fn decode_rows_matches_decode_into_rgb8_for_cmyk_and_ycck() { + for bytes in [cmyk_8x8_jpeg(), ycck_8x8_jpeg()] { + let dec = Decoder::new(&bytes).expect("CMYK/YCCK decoder must construct"); + let (w, h) = dec.info().dimensions; + let stride = (w * 3) as usize; + + let mut expected = vec![0u8; stride * h as usize]; + dec.decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, Downscale::None) + .expect("full CMYK/YCCK RGB8 decode must succeed"); + + let mut sink = CollectRows::default(); + dec.decode_rows(&mut sink) + .expect("CMYK/YCCK decode_rows must succeed"); + + assert_eq!(sink.rows.len(), h as usize); + for (row_idx, (y, row)) in sink.rows.iter().enumerate() { + assert_eq!(*y as usize, row_idx); + assert_eq!(row.len(), stride); + assert_eq!( + row.as_slice(), + &expected[row_idx * stride..(row_idx + 1) * stride] + ); + } + } +} + +#[test] +fn decode_rows_expands_lossless_gray8_common_predictors() { + let expected = expand_gray_to_rgb(&LOSSLESS_GRAYSCALE_3X3_PIXELS); + for predictor in 1..=7 { + for bytes in [ + lossless_predictor_grayscale_3x3_jpeg(predictor), + lossless_restart_predictor_grayscale_3x3_jpeg(predictor), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("SOF3 grayscale predictor-{predictor} decoder must construct: {err}") + }); + let mut sink = CollectRows::default(); + + dec.decode_rows(&mut sink).unwrap_or_else(|err| { + panic!("SOF3 grayscale predictor-{predictor} decode_rows must succeed: {err}") + }); + + assert_eq!(flatten_rows(&sink.rows, 3 * 3), expected); + } + } +} + +#[test] +fn decode_rows_streams_lossless_gray16_common_predictors() { + let expected = gray16_samples_to_le_bytes(&LOSSLESS_GRAYSCALE_16BIT_3X3_PIXELS); + for predictor in 1..=7 { + for bytes in [ + lossless_predictor_grayscale_16bit_3x3_jpeg(predictor), + lossless_restart_predictor_grayscale_16bit_3x3_jpeg(predictor), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("SOF3 Gray16 predictor-{predictor} decoder must construct: {err}") + }); + let mut sink = CollectRows::default(); + + dec.decode_rows(&mut sink).unwrap_or_else(|err| { + panic!("SOF3 Gray16 predictor-{predictor} decode_rows must succeed: {err}") + }); + + assert_eq!(flatten_rows(&sink.rows, 3 * 2), expected); + } + } +} + +#[test] +fn decode_rows_matches_lossless_app14_rgb_common_predictors() { + for predictor in 1..=7 { + for bytes in [ + lossless_predictor_rgb_3x3_jpeg(predictor), + lossless_restart_predictor_rgb_3x3_jpeg(predictor), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("SOF3 APP14 RGB predictor-{predictor} decoder must construct: {err}") + }); + let mut sink = CollectRows::default(); + + dec.decode_rows(&mut sink).unwrap_or_else(|err| { + panic!("SOF3 APP14 RGB predictor-{predictor} decode_rows must succeed: {err}") + }); + + assert_eq!(flatten_rows(&sink.rows, 3 * 3), LOSSLESS_RGB_3X3_PIXELS); + } + } +} + +#[test] +fn decode_rows_matches_lossless_ycbcr_rgb8_common_predictors() { + let expected = lossless_ycbcr_3x3_rgb8(); + for predictor in 1..=7 { + for bytes in [ + lossless_predictor_ycbcr_3x3_jpeg(predictor), + lossless_restart_predictor_ycbcr_3x3_jpeg(predictor), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("SOF3 YCbCr predictor-{predictor} decoder must construct: {err}") + }); + let mut sink = CollectRows::default(); + + dec.decode_rows(&mut sink).unwrap_or_else(|err| { + panic!("SOF3 YCbCr predictor-{predictor} decode_rows must succeed: {err}") + }); + + assert_eq!(flatten_rows(&sink.rows, 3 * 3), expected); + } + } +} + +#[test] +fn decode_rows_matches_lossless_app14_rgb16_common_predictors() { + let expected = rgb16_samples_to_le_bytes(&LOSSLESS_RGB_16BIT_3X3_PIXELS); + for predictor in 1..=7 { + for bytes in [ + lossless_predictor_rgb_16bit_3x3_jpeg(predictor), + lossless_restart_predictor_rgb_16bit_3x3_jpeg(predictor), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("SOF3 APP14 RGB16 predictor-{predictor} decoder must construct: {err}") + }); + let mut sink = CollectRows::default(); + + dec.decode_rows(&mut sink).unwrap_or_else(|err| { + panic!("SOF3 APP14 RGB16 predictor-{predictor} decode_rows must succeed: {err}") + }); + + assert_eq!(flatten_rows(&sink.rows, 3 * 6), expected); + } + } +} + +#[test] +fn decode_rows_matches_lossless_ycbcr16_rgb16_common_predictors() { + let expected = lossless_ycbcr_16bit_3x3_rgb16(); + for predictor in 1..=7 { + for bytes in [ + lossless_predictor_ycbcr_16bit_3x3_jpeg(predictor), + lossless_restart_predictor_ycbcr_16bit_3x3_jpeg(predictor), + ] { + let dec = Decoder::new(&bytes).unwrap_or_else(|err| { + panic!("SOF3 YCbCr16 predictor-{predictor} decoder must construct: {err}") + }); + let mut sink = CollectRows::default(); + + dec.decode_rows(&mut sink).unwrap_or_else(|err| { + panic!("SOF3 YCbCr16 predictor-{predictor} decode_rows must succeed: {err}") + }); + + assert_eq!(flatten_rows(&sink.rows, 3 * 6), expected); + } + } +} + +#[test] +fn decode_rows_matches_decode_into_rgb8_for_restart_coded_grayscale_wsi_shape() { + let bytes = restart_coded_grayscale_jpeg(24, 24); + let dec = Decoder::new(&bytes).expect("restart-coded grayscale fixture must parse"); + let (w, h) = dec.info().dimensions; + let stride = (w * 3) as usize; + + let mut expected = vec![0u8; stride * h as usize]; + dec.decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, Downscale::None) + .expect("full decode must succeed"); + + let mut sink = CollectRows::default(); + dec.decode_rows(&mut sink) + .expect("decode_rows must succeed on restart-coded input"); + + assert_eq!(sink.rows.len(), h as usize); + for (row_idx, (y, row)) in sink.rows.iter().enumerate() { + assert_eq!(*y as usize, row_idx); + assert_eq!(row.len(), stride); + assert_eq!( + row.as_slice(), + &expected[row_idx * stride..(row_idx + 1) * stride] + ); + } +} + +fn flatten_rows(rows: &[(u32, Vec)], stride: usize) -> Vec { + let mut out = Vec::with_capacity(rows.len() * stride); + for (row_idx, (y, row)) in rows.iter().enumerate() { + assert_eq!(*y as usize, row_idx); + assert_eq!(row.len(), stride); + out.extend_from_slice(row); + } + out +} + +fn expand_gray_to_rgb(gray: &[u8]) -> Vec { + let mut out = Vec::with_capacity(gray.len() * 3); + for &sample in gray { + out.extend_from_slice(&[sample, sample, sample]); + } + out +} + +fn gray16_samples_to_le_bytes(samples: &[u16]) -> Vec { + let mut out = Vec::with_capacity(samples.len() * 2); + for sample in samples { + out.extend_from_slice(&sample.to_le_bytes()); + } + out +} + +fn rgb16_samples_to_le_bytes(samples: &[u16]) -> Vec { + let mut out = Vec::with_capacity(samples.len() * 2); + for sample in samples { + out.extend_from_slice(&sample.to_le_bytes()); + } + out +} + +#[test] +fn region_component_rows_scaled_matches_gray_region_decode_for_restart_fixture() { + let bytes = restart_coded_grayscale_jpeg(24, 24); + let dec = Decoder::new(&bytes).expect("restart-coded grayscale fixture must parse"); + let roi = Rect { + x: 5, + y: 6, + w: 11, + h: 10, + }; + + let mut pool = ScratchPool::new(); + let mut sink = CollectGrayComponentRows::default(); + dec.decode_region_component_rows_with_scratch(&mut pool, &mut sink, roi, Downscale::Half) + .expect("scaled region component rows must decode"); + + let expected = dec + .decode_region_scaled(PixelFormat::Gray8, roi, Downscale::Half) + .expect("scaled region decode must succeed") + .0; + + let mut collected = Vec::new(); + for (row_idx, (y, row)) in sink.rows.iter().enumerate() { + assert_eq!(*y as usize, row_idx); + collected.extend_from_slice(row); + } + + assert_eq!(collected, expected); +} + +#[test] +fn region_component_rows_scaled_match_cmyk_ycck_region_decode() { + let roi = Rect { + x: 1, + y: 1, + w: 6, + h: 6, + }; + + for bytes in [cmyk_8x8_jpeg(), ycck_8x8_jpeg()] { + let dec = Decoder::new(&bytes).expect("CMYK/YCCK decoder must construct"); + let mut pool = ScratchPool::new(); + let mut sink = CollectRgbComponentRows::default(); + dec.decode_region_component_rows_with_scratch(&mut pool, &mut sink, roi, Downscale::Half) + .expect("CMYK/YCCK scaled region component rows must decode"); + + let expected = dec + .decode_region_scaled(PixelFormat::Rgb8, roi, Downscale::Half) + .expect("CMYK/YCCK scaled region decode must succeed") + .0; + + let mut collected = Vec::new(); + for (row_idx, (y, row)) in sink.rows.iter().enumerate() { + assert_eq!(*y as usize, row_idx); + collected.extend_from_slice(row); + } + + assert_eq!(collected, expected); + } +} diff --git a/crates/signinum-jpeg/tests/wsi_parity.rs b/crates/j2k-jpeg/tests/wsi_parity.rs similarity index 61% rename from crates/signinum-jpeg/tests/wsi_parity.rs rename to crates/j2k-jpeg/tests/wsi_parity.rs index b7996d54..fa7425e9 100644 --- a/crates/signinum-jpeg/tests/wsi_parity.rs +++ b/crates/j2k-jpeg/tests/wsi_parity.rs @@ -2,13 +2,18 @@ //! Bit-exact parity against libjpeg-turbo's ISLOW path. -use signinum_jpeg::{Decoder, Downscale, PixelFormat, Rect}; +use j2k_jpeg::{Decoder, Downscale, PixelFormat, Rect}; +use j2k_test_support::{ + crop_interleaved_bytes, crop_interleaved_u8, restart_coded_grayscale_jpeg, + scaled_rect_covering, PixelRect, JPEG_BASELINE_420_16X16, JPEG_BASELINE_420_16X16_RGB, + JPEG_GRAYSCALE_8X8, JPEG_GRAYSCALE_8X8_GRAY, +}; -const BASELINE_420_JPG: &[u8] = include_bytes!("../fixtures/conformance/baseline_420_16x16.jpg"); -const BASELINE_420_RGB: &[u8] = include_bytes!("../fixtures/conformance/baseline_420_16x16.rgb"); +const BASELINE_420_JPG: &[u8] = JPEG_BASELINE_420_16X16; +const BASELINE_420_RGB: &[u8] = JPEG_BASELINE_420_16X16_RGB; -const GRAYSCALE_8X8_JPG: &[u8] = include_bytes!("../fixtures/conformance/grayscale_8x8.jpg"); -const GRAYSCALE_8X8_GRAY: &[u8] = include_bytes!("../fixtures/conformance/grayscale_8x8.gray"); +const GRAYSCALE_8X8_JPG: &[u8] = JPEG_GRAYSCALE_8X8; +const GRAYSCALE_8X8_GRAY: &[u8] = JPEG_GRAYSCALE_8X8_GRAY; #[test] fn baseline_420_16x16_matches_libjpeg_turbo_bit_exact() { @@ -132,79 +137,23 @@ fn decode_full_rgb(dec: &Decoder<'_>) -> Vec { } fn crop_rgb8(full: &[u8], width: usize, roi: Rect) -> Vec { - let mut out = Vec::with_capacity((roi.w * roi.h * 3) as usize); - let row_stride = width * 3; - let crop_stride = roi.w as usize * 3; - for y in roi.y as usize..(roi.y + roi.h) as usize { - let row = &full[y * row_stride..(y + 1) * row_stride]; - let x0 = roi.x as usize * 3; - out.extend_from_slice(&row[x0..x0 + crop_stride]); - } - out + crop_interleaved_bytes(full, width, 3, pixel_rect(roi)) } fn crop_gray8(full: &[u8], width: usize, roi: Rect) -> Vec { - let mut out = Vec::with_capacity((roi.w * roi.h) as usize); - for y in roi.y as usize..(roi.y + roi.h) as usize { - let row = &full[y * width..(y + 1) * width]; - let x0 = roi.x as usize; - out.extend_from_slice(&row[x0..x0 + roi.w as usize]); - } - out + crop_interleaved_u8(full, width, 1, pixel_rect(roi)) } fn scaled_rect_covering_half(roi: Rect) -> Rect { - let x0 = roi.x / 2; - let y0 = roi.y / 2; - let x1 = (roi.x + roi.w).div_ceil(2); - let y1 = (roi.y + roi.h).div_ceil(2); + let scaled = scaled_rect_covering(pixel_rect(roi), 2); Rect { - x: x0, - y: y0, - w: x1 - x0, - h: y1 - y0, + x: scaled.x, + y: scaled.y, + w: scaled.w, + h: scaled.h, } } -fn restart_coded_grayscale_jpeg(width: u16, height: u16) -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0xff, 0xd8]); - bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); - bytes.extend(std::iter::repeat_n(16u8, 64)); - bytes.extend_from_slice(&[ - 0xff, - 0xc0, - 0x00, - 11, - 8, - (height >> 8) as u8, - height as u8, - (width >> 8) as u8, - width as u8, - 1, - 1, - 0x11, - 0, - ]); - bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); - - let mcu_cols = u32::from(width).div_ceil(8); - let mcu_rows = u32::from(height).div_ceil(8); - let mcu_count = (mcu_cols * mcu_rows) as usize; - for mcu in 0..mcu_count { - bytes.push(0x00); - if mcu + 1 != mcu_count { - bytes.extend_from_slice(&[0xff, 0xd0 | ((mcu as u8) & 0x07)]); - } - } - - bytes.extend_from_slice(&[0xff, 0xd9]); - bytes +fn pixel_rect(roi: Rect) -> PixelRect { + PixelRect::new(roi.x, roi.y, roi.w, roi.h) } diff --git a/crates/j2k-metal-support/Cargo.toml b/crates/j2k-metal-support/Cargo.toml new file mode 100644 index 00000000..d3e401b1 --- /dev/null +++ b/crates/j2k-metal-support/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "j2k-metal-support" +description = "Shared Metal runtime setup helpers for J2K adapters" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +readme = "README.md" + +[package.metadata.docs.rs] +all-features = true +targets = [] + +[lib] +name = "j2k_metal_support" +path = "src/lib.rs" + +[dependencies] +j2k-core = { path = "../j2k-core", version = "=0.6.0" } + +[target.'cfg(target_os = "macos")'.dependencies] +metal = { workspace = true } + +[lints.rust] +unsafe_code = "allow" +unsafe_op_in_unsafe_fn = "deny" +unreachable_pub = "warn" + +[lints.clippy] +pedantic = { level = "warn", priority = -1 } +undocumented_unsafe_blocks = "warn" +missing_errors_doc = "allow" +missing_panics_doc = "allow" diff --git a/crates/j2k-metal-support/README.md b/crates/j2k-metal-support/README.md new file mode 100644 index 00000000..99531fd2 --- /dev/null +++ b/crates/j2k-metal-support/README.md @@ -0,0 +1,7 @@ +# j2k-metal-support + +Shared Metal runtime setup helpers for J2K Metal adapters. + +The crate centralizes system device lookup, checked command-queue creation, +shader-library compilation, named pipeline loading, and stable route labels. +Codec-specific kernels stay in the codec adapter crates. diff --git a/crates/j2k-metal-support/src/lib.rs b/crates/j2k-metal-support/src/lib.rs new file mode 100644 index 00000000..bf9f3723 --- /dev/null +++ b/crates/j2k-metal-support/src/lib.rs @@ -0,0 +1,639 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Shared Metal runtime setup helpers for J2K adapter crates. + +#![warn(unreachable_pub)] + +use core::fmt; + +/// Stable profile labels for a Metal backend route decision. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MetalRouteProfileLabels { + /// Route decision label emitted in GPU route profiles. + pub decision: &'static str, + /// Route reason label emitted in GPU route profiles. + pub reason: &'static str, +} + +impl MetalRouteProfileLabels { + /// Construct route profile labels from stable string values. + #[must_use] + pub const fn new(decision: &'static str, reason: &'static str) -> Self { + Self { decision, reason } + } +} + +/// Route profile labels for CPU host execution. +#[must_use] +pub const fn cpu_host_route() -> MetalRouteProfileLabels { + MetalRouteProfileLabels::new("cpu_host", "none") +} + +/// Route profile labels for Metal kernel execution. +#[must_use] +pub const fn metal_kernel_route() -> MetalRouteProfileLabels { + MetalRouteProfileLabels::new("metal_kernel", "none") +} + +/// Route profile labels for an explicit Metal request rejected by codec policy. +#[must_use] +pub const fn reject_explicit_metal_route(reason: &'static str) -> MetalRouteProfileLabels { + MetalRouteProfileLabels::new("reject_explicit_metal", reason) +} + +/// Route profile labels for backend requests unsupported by the Metal adapter. +#[must_use] +pub const fn reject_unsupported_backend_route() -> MetalRouteProfileLabels { + MetalRouteProfileLabels::new("reject_unsupported_backend", "unsupported_backend") +} + +/// Route profile labels for hosts without an available Metal runtime. +#[must_use] +pub const fn metal_unavailable_route() -> MetalRouteProfileLabels { + MetalRouteProfileLabels::new("metal_unavailable", "metal_unavailable") +} + +/// Errors returned by shared Metal runtime setup helpers. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MetalSupportError { + /// The host does not expose a system default Metal device. + MetalUnavailable, + /// Metal returned a null command queue for the selected device. + CommandQueueUnavailable, + /// Metal command queue creation failed before returning a queue. + CommandQueue { + /// Error reported by the Objective-C message send. + message: String, + }, + /// Metal shader source compilation failed. + ShaderLibrary { + /// Compiler error reported by Metal. + message: String, + }, + /// A named function was not present in a compiled Metal library. + PipelineFunction { + /// Requested Metal function name. + function_name: String, + /// Error reported by Metal. + message: String, + }, + /// Compute pipeline creation failed for a Metal function. + PipelineState { + /// Requested Metal function name. + function_name: String, + /// Error reported by Metal. + message: String, + }, + /// A requested CPU-visible buffer range does not fit in the Metal buffer. + BufferBounds { + /// Byte offset requested by the caller. + offset_bytes: usize, + /// Number of bytes requested by the caller. + byte_len: usize, + /// Metal buffer length in bytes. + buffer_len: usize, + }, + /// A requested typed buffer view is not aligned for the element type. + BufferAlignment { + /// Byte offset requested by the caller. + offset_bytes: usize, + /// Required alignment in bytes. + align: usize, + }, + /// The Metal buffer is not CPU-visible through `contents()`. + BufferContentsUnavailable, +} + +impl MetalSupportError { + /// Return true when the error represents an unavailable Metal backend. + #[must_use] + pub const fn is_unavailable(&self) -> bool { + matches!(self, Self::MetalUnavailable | Self::CommandQueueUnavailable) + } +} + +impl fmt::Display for MetalSupportError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MetalUnavailable => f.write_str("Metal is unavailable on this host"), + Self::CommandQueueUnavailable => { + f.write_str("Metal command queue is unavailable on this host") + } + Self::CommandQueue { message } => { + write!(f, "Metal command queue creation failed: {message}") + } + Self::ShaderLibrary { message } => { + write!(f, "Metal shader library compilation failed: {message}") + } + Self::PipelineFunction { + function_name, + message, + } => write!( + f, + "Metal pipeline function `{function_name}` lookup failed: {message}" + ), + Self::PipelineState { + function_name, + message, + } => write!( + f, + "Metal compute pipeline `{function_name}` creation failed: {message}" + ), + Self::BufferBounds { + offset_bytes, + byte_len, + buffer_len, + } => write!( + f, + "Metal buffer range offset {offset_bytes} length {byte_len} exceeds buffer length {buffer_len}" + ), + Self::BufferAlignment { + offset_bytes, + align, + } => write!( + f, + "Metal buffer range offset {offset_bytes} is not aligned to {align} bytes" + ), + Self::BufferContentsUnavailable => { + f.write_str("Metal buffer contents are not CPU-visible") + } + } + } +} + +impl std::error::Error for MetalSupportError {} + +#[cfg(target_os = "macos")] +use j2k_core::GpuAbi; +#[cfg(target_os = "macos")] +use metal::{ + foreign_types::ForeignType, + objc::{runtime::Sel, Message}, + Buffer, CommandQueue, CompileOptions, ComputeCommandEncoderRef, ComputePipelineState, Device, + Library, MTLCommandQueue, MTLResourceOptions, MTLSize, +}; + +#[cfg(target_os = "macos")] +/// Return the system default Metal device, or a stable error message. +pub fn system_default_device() -> Result { + Device::system_default().ok_or(MetalSupportError::MetalUnavailable) +} + +#[cfg(target_os = "macos")] +/// Create a command queue and surface null-queue failures explicitly. +pub fn checked_command_queue(device: &Device) -> Result { + // SAFETY: Objective-C/Metal pointers are null-checked or range-validated before wrapping. + let queue: *mut MTLCommandQueue = unsafe { + device + .as_ref() + .send_message(Sel::register("newCommandQueue"), ()) + .map_err(|error| MetalSupportError::CommandQueue { + message: error.to_string(), + })? + }; + if queue.is_null() { + Err(MetalSupportError::CommandQueueUnavailable) + } else { + // SAFETY: Objective-C/Metal pointers are null-checked or range-validated before wrapping. + Ok(unsafe { CommandQueue::from_ptr(queue) }) + } +} + +#[cfg(target_os = "macos")] +/// Compile a Metal shader source string with default compile options. +pub fn shader_library(device: &Device, source: &str) -> Result { + let options = CompileOptions::new(); + device + .new_library_with_source(source, &options) + .map_err(|message| MetalSupportError::ShaderLibrary { message }) +} + +#[cfg(target_os = "macos")] +/// Load a named compute pipeline from an already compiled shader library. +pub fn named_pipeline( + device: &Device, + library: &Library, + function_name: &str, +) -> Result { + let function = library + .get_function(function_name, None) + .map_err(|message| MetalSupportError::PipelineFunction { + function_name: function_name.to_string(), + message, + })?; + device + .new_compute_pipeline_state_with_function(&function) + .map_err(|message| MetalSupportError::PipelineState { + function_name: function_name.to_string(), + message, + }) +} + +#[cfg(target_os = "macos")] +/// Caller-owned Metal device session shared by adapter crates. +#[derive(Clone)] +pub struct MetalDeviceSession { + device: Device, +} + +#[cfg(target_os = "macos")] +impl MetalDeviceSession { + /// Create a session bound to an existing Metal device. + #[must_use] + pub fn new(device: Device) -> Self { + Self { device } + } + + /// Create a session from the system default Metal device. + pub fn system_default() -> Result { + system_default_device().map(Self::new) + } + + /// Metal device used by this session. + #[must_use] + pub fn device(&self) -> &metal::DeviceRef { + self.device.as_ref() + } +} + +#[cfg(target_os = "macos")] +impl core::fmt::Debug for MetalDeviceSession { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("MetalDeviceSession") + .field("device", &self.device.name()) + .finish() + } +} + +#[cfg(target_os = "macos")] +/// Allocate a shared Metal buffer, clamping zero-length requests to one byte. +#[must_use] +pub fn shared_buffer(device: &Device, bytes: usize) -> Buffer { + device.new_buffer(bytes.max(1) as u64, MTLResourceOptions::StorageModeShared) +} + +#[cfg(target_os = "macos")] +/// Allocate a private Metal buffer, clamping zero-length requests to one byte. +#[must_use] +pub fn private_buffer(device: &Device, bytes: usize) -> Buffer { + device.new_buffer(bytes.max(1) as u64, MTLResourceOptions::StorageModePrivate) +} + +#[cfg(target_os = "macos")] +/// Allocate a shared Metal buffer initialized from bytes. +#[must_use] +pub fn shared_buffer_with_bytes(device: &Device, bytes: &[u8]) -> Buffer { + if bytes.is_empty() { + return shared_buffer(device, 1); + } + device.new_buffer_with_data( + bytes.as_ptr().cast(), + bytes.len() as u64, + MTLResourceOptions::StorageModeShared, + ) +} + +#[cfg(target_os = "macos")] +/// Allocate a shared Metal buffer initialized from GPU ABI values. +#[must_use] +pub fn shared_buffer_with_slice(device: &Device, values: &[T]) -> Buffer { + shared_buffer_with_bytes(device, T::slice_as_bytes(values)) +} + +#[cfg(target_os = "macos")] +/// Allocate a shared Metal buffer large enough for `len` GPU ABI values. +#[must_use] +pub fn shared_buffer_for_len(device: &Device, len: usize) -> Buffer { + shared_buffer(device, len.saturating_mul(core::mem::size_of::())) +} + +#[cfg(target_os = "macos")] +fn checked_buffer_contents_ptr( + buffer: &Buffer, + offset_bytes: usize, + len: usize, +) -> Result<*mut T, MetalSupportError> { + let buffer_len = usize::try_from(buffer.length()).unwrap_or(usize::MAX); + let byte_len = + len.checked_mul(core::mem::size_of::()) + .ok_or(MetalSupportError::BufferBounds { + offset_bytes, + byte_len: usize::MAX, + buffer_len, + })?; + let end = offset_bytes + .checked_add(byte_len) + .ok_or(MetalSupportError::BufferBounds { + offset_bytes, + byte_len, + buffer_len, + })?; + if end > buffer_len { + return Err(MetalSupportError::BufferBounds { + offset_bytes, + byte_len, + buffer_len, + }); + } + + let base = buffer.contents().cast::(); + if base.is_null() { + return Err(MetalSupportError::BufferContentsUnavailable); + } + let address = + (base as usize) + .checked_add(offset_bytes) + .ok_or(MetalSupportError::BufferBounds { + offset_bytes, + byte_len, + buffer_len, + })?; + let align = core::mem::align_of::(); + if align > 1 && !address.is_multiple_of(align) { + return Err(MetalSupportError::BufferAlignment { + offset_bytes, + align, + }); + } + + // SAFETY: bounds and alignment were validated above. + Ok(unsafe { base.add(offset_bytes).cast::() }) +} + +#[cfg(target_os = "macos")] +/// Checked typed borrow from a CPU-visible shared Metal buffer. +/// +/// Validates offset arithmetic, typed alignment, requested byte length, and the +/// Metal buffer length before constructing the returned slice. +pub fn checked_buffer_contents_slice( + buffer: &Buffer, + offset_bytes: usize, + len: usize, +) -> Result<&[T], MetalSupportError> { + let ptr = checked_buffer_contents_ptr::(buffer, offset_bytes, len)?; + // SAFETY: `checked_buffer_contents_ptr` validated the byte range and typed + // alignment for `len` elements in this CPU-visible buffer. + // SAFETY: Objective-C/Metal pointers are null-checked or range-validated before wrapping. + Ok(unsafe { core::slice::from_raw_parts(ptr.cast_const(), len) }) +} + +#[cfg(target_os = "macos")] +/// Checked mutable typed borrow from a CPU-visible shared Metal buffer. +/// +/// Validates offset arithmetic, typed alignment, requested byte length, and the +/// Metal buffer length before constructing the returned slice. +pub fn checked_buffer_contents_slice_mut( + buffer: &mut Buffer, + offset_bytes: usize, + len: usize, +) -> Result<&mut [T], MetalSupportError> { + let ptr = checked_buffer_contents_ptr::(buffer, offset_bytes, len)?; + // SAFETY: `checked_buffer_contents_ptr` validated the byte range and typed + // alignment for `len` elements in this CPU-visible buffer. The mutable + // borrow of `buffer` prevents another safe mutable slice from this helper. + // SAFETY: Objective-C/Metal pointers are null-checked or range-validated before wrapping. + Ok(unsafe { core::slice::from_raw_parts_mut(ptr, len) }) +} + +#[cfg(target_os = "macos")] +/// Borrow typed contents from a shared Metal buffer. +/// +/// # Safety +/// The caller must ensure the buffer is CPU-visible, contains at least +/// `offset_bytes + len * size_of::()` initialized bytes, and is not mutably +/// aliased for the returned lifetime. +#[must_use] +pub unsafe fn buffer_contents_slice( + buffer: &Buffer, + offset_bytes: usize, + len: usize, +) -> &[T] { + checked_buffer_contents_slice(buffer, offset_bytes, len) + .expect("Metal buffer contents slice violates unsafe API contract") +} + +#[cfg(target_os = "macos")] +/// Mutably borrow typed contents from a shared Metal buffer. +/// +/// # Safety +/// The caller must ensure the buffer is CPU-visible, contains at least +/// `offset_bytes + len * size_of::()` initialized bytes, and no other alias +/// can read or write the same memory for the returned lifetime. +#[must_use] +pub unsafe fn buffer_contents_slice_mut( + buffer: &mut Buffer, + offset_bytes: usize, + len: usize, +) -> &mut [T] { + checked_buffer_contents_slice_mut(buffer, offset_bytes, len) + .expect("Metal mutable buffer contents slice violates unsafe API contract") +} + +#[cfg(target_os = "macos")] +/// Construct a Metal dispatch size. +#[must_use] +pub const fn mtl_size(width: u64, height: u64, depth: u64) -> MTLSize { + MTLSize { + width, + height, + depth, + } +} + +#[cfg(target_os = "macos")] +/// One-dimensional thread-group size with empty SIMD widths clamped to one. +#[must_use] +pub const fn one_d_threads_per_group(simd_width: u64) -> MTLSize { + mtl_size(if simd_width == 0 { 1 } else { simd_width }, 1, 1) +} + +#[cfg(target_os = "macos")] +/// Two-dimensional thread-group size preserving SIMD width and filling height. +#[must_use] +pub const fn two_d_threads_per_group(simd_width: u64, max_threads: u64) -> MTLSize { + let width = if simd_width == 0 { 1 } else { simd_width }; + let max_threads = if max_threads < width { + width + } else { + max_threads + }; + mtl_size(width, max_threads / width, 1) +} + +#[cfg(target_os = "macos")] +/// Dispatch a one-dimensional compute workload with one SIMD group per threadgroup. +pub fn dispatch_1d_pipeline( + encoder: &ComputeCommandEncoderRef, + pipeline: &ComputePipelineState, + width: u64, +) { + encoder.dispatch_threads( + mtl_size(width, 1, 1), + one_d_threads_per_group(pipeline.thread_execution_width()), + ); +} + +#[cfg(target_os = "macos")] +/// Dispatch a single compute thread. +pub fn dispatch_single_thread(encoder: &ComputeCommandEncoderRef) { + encoder.dispatch_threads(mtl_size(1, 1, 1), mtl_size(1, 1, 1)); +} + +#[cfg(target_os = "macos")] +/// Dispatch a two-dimensional compute workload using the pipeline's SIMD width. +pub fn dispatch_2d_pipeline( + encoder: &ComputeCommandEncoderRef, + pipeline: &ComputePipelineState, + dims: (u32, u32), +) { + encoder.dispatch_threads( + mtl_size(u64::from(dims.0), u64::from(dims.1), 1), + two_d_threads_per_group( + pipeline.thread_execution_width(), + pipeline.max_total_threads_per_threadgroup(), + ), + ); +} + +#[cfg(target_os = "macos")] +/// Dispatch a three-dimensional compute workload using a 2D threadgroup shape. +pub fn dispatch_3d_pipeline( + encoder: &ComputeCommandEncoderRef, + pipeline: &ComputePipelineState, + dims: (u32, u32, u32), +) { + encoder.dispatch_threads( + mtl_size(u64::from(dims.0), u64::from(dims.1), u64::from(dims.2)), + two_d_threads_per_group( + pipeline.thread_execution_width(), + pipeline.max_total_threads_per_threadgroup(), + ), + ); +} + +#[cfg(target_os = "macos")] +/// Convenience loader for many pipelines from one Metal shader library. +pub struct MetalPipelineLoader { + device: Device, + library: Library, +} + +#[cfg(target_os = "macos")] +impl MetalPipelineLoader { + /// Compile `source` and keep the resulting library for named pipeline loads. + pub fn new(device: &Device, source: &str) -> Result { + Ok(Self { + device: device.clone(), + library: shader_library(device, source)?, + }) + } + + /// Load one named compute pipeline from the cached shader library. + pub fn pipeline(&self, function_name: &str) -> Result { + named_pipeline(&self.device, &self.library, function_name) + } + + /// Borrow the compiled shader library. + #[must_use] + pub fn library(&self) -> &Library { + &self.library + } +} + +#[cfg(all(test, target_os = "macos"))] +mod tests { + use super::{ + checked_buffer_contents_slice, checked_buffer_contents_slice_mut, one_d_threads_per_group, + shared_buffer_for_len, shared_buffer_with_slice, system_default_device, + two_d_threads_per_group, MetalSupportError, + }; + + #[test] + fn two_d_threads_per_group_clamps_empty_pipeline_limits() { + let threads = two_d_threads_per_group(0, 0); + + assert_eq!((threads.width, threads.height, threads.depth), (1, 1, 1)); + } + + #[test] + fn one_d_threads_per_group_clamps_empty_pipeline_width() { + let threads = one_d_threads_per_group(0); + + assert_eq!((threads.width, threads.height, threads.depth), (1, 1, 1)); + } + + #[test] + fn two_d_threads_per_group_preserves_simd_width_and_derives_height() { + let threads = two_d_threads_per_group(32, 1024); + + assert_eq!((threads.width, threads.height, threads.depth), (32, 32, 1)); + } + + #[test] + fn buffer_contents_slice_reads_typed_shared_buffer_values() { + let Ok(device) = system_default_device() else { + eprintln!("skipping shared buffer slice test: no Metal device"); + return; + }; + let buffer = shared_buffer_with_slice(&device, &[3_u32, 5, 8, 13]); + + let values = checked_buffer_contents_slice::(&buffer, 0, 4).expect("checked slice"); + + assert_eq!(values, &[3, 5, 8, 13]); + } + + #[test] + fn buffer_contents_slice_mut_writes_typed_shared_buffer_values() { + let Ok(device) = system_default_device() else { + eprintln!("skipping mutable shared buffer slice test: no Metal device"); + return; + }; + let mut buffer = shared_buffer_for_len::(&device, 3); + + let values = + checked_buffer_contents_slice_mut::(&mut buffer, 0, 3).expect("checked slice"); + values.copy_from_slice(&[21, 34, 55]); + let values = checked_buffer_contents_slice::(&buffer, 0, 3).expect("checked slice"); + + assert_eq!(values, &[21, 34, 55]); + } + + #[test] + fn checked_buffer_contents_slice_rejects_out_of_bounds_range() { + let Ok(device) = system_default_device() else { + eprintln!("skipping shared buffer bounds test: no Metal device"); + return; + }; + let buffer = shared_buffer_with_slice(&device, &[1_u32]); + + let err = checked_buffer_contents_slice::(&buffer, 0, 2).expect_err("bounds error"); + + assert!(matches!( + err, + MetalSupportError::BufferBounds { + offset_bytes: 0, + byte_len: 8, + buffer_len: 4, + } + )); + } + + #[test] + fn checked_buffer_contents_slice_rejects_unaligned_range() { + let Ok(device) = system_default_device() else { + eprintln!("skipping shared buffer alignment test: no Metal device"); + return; + }; + let buffer = shared_buffer_with_slice(&device, &[1_u32, 2]); + + let err = checked_buffer_contents_slice::(&buffer, 1, 1).expect_err("alignment error"); + + assert!(matches!( + err, + MetalSupportError::BufferAlignment { + offset_bytes: 1, + align: 4, + } + )); + } +} diff --git a/crates/j2k-metal/Cargo.toml b/crates/j2k-metal/Cargo.toml new file mode 100644 index 00000000..7c75fa1d --- /dev/null +++ b/crates/j2k-metal/Cargo.toml @@ -0,0 +1,58 @@ +[package] +name = "j2k-metal" +description = "Metal decoder and encode-stage adapter for j2k" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +build = "build.rs" +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +readme = "README.md" + +[package.metadata.docs.rs] +all-features = true +targets = [] + +[lib] +name = "j2k_metal" +path = "src/lib.rs" + +[dependencies] +j2k-core = { path = "../j2k-core", version = "=0.6.0" } +j2k = { path = "../j2k", version = "=0.6.0" } +j2k-native = { path = "../j2k-native", version = "=0.6.0" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.6.0" } +j2k-profile = { path = "../j2k-profile", version = "=0.6.0" } +thiserror = { workspace = true } + +[target.'cfg(target_os = "macos")'.dependencies] +libc = { workspace = true } +metal = { workspace = true } +rayon = { workspace = true } + +[dev-dependencies] +rayon = { workspace = true } +j2k-test-support = { path = "../j2k-test-support" } + +[build-dependencies] +cc = { workspace = true } + +[lints.rust] +unsafe_code = "allow" +unsafe_op_in_unsafe_fn = "deny" +unreachable_pub = "warn" + +[lints.clippy] +pedantic = { level = "warn", priority = -1 } +undocumented_unsafe_blocks = "warn" +module_name_repetitions = "allow" +must_use_candidate = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +too_many_lines = "allow" +cast_possible_truncation = "allow" +cast_sign_loss = "allow" +cast_possible_wrap = "allow" +doc_markdown = "allow" diff --git a/crates/j2k-metal/README.md b/crates/j2k-metal/README.md new file mode 100644 index 00000000..8cbd9cda --- /dev/null +++ b/crates/j2k-metal/README.md @@ -0,0 +1,22 @@ +# j2k-metal + +Metal adapter for JPEG 2000 / HTJ2K decode and encode-stage paths on macOS. + +The crate provides resident Metal decode and encode-stage integration for +supported workloads. It uses `j2k-metal-support` for runtime setup while +keeping codec-specific kernels local. + +Encode support is stage-oriented unless a documented resident path accepts the +shape. `Auto` host-output encode may dispatch benchmark-gated coefficient-prep +stages for 512 x 512 and larger stage inputs, and the resident HTJ2K RGB8 +lossless shortcut is gated to 1,024 x 1,024 and larger tiles. Explicit Metal +requests are strict: supported shapes dispatch, and unsupported direct Metal +requests return `UnsupportedMetalRequest` instead of silently changing backend. + +Metal routing is deliberately selective. `Auto` may decline small tiles, +irregular packet shapes, or stages where host/device transfer and dispatch +overhead dominate. Stage-by-stage host-output `Auto` currently limits Metal to +deinterleave, forward RCT/ICT, forward 5/3 and 9/7 DWT, and subband +quantization. Classic Tier-1, HT code-block encode, packetization, and +codestream assembly stay CPU for that route unless a documented resident path +supports the shape with parity and benchmark evidence. diff --git a/crates/j2k-metal/build.rs b/crates/j2k-metal/build.rs new file mode 100644 index 00000000..307a6290 --- /dev/null +++ b/crates/j2k-metal/build.rs @@ -0,0 +1,8 @@ +fn main() { + println!("cargo:rerun-if-changed=src/signpost.c"); + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") { + cc::Build::new() + .file("src/signpost.c") + .compile("j2k_metal_signpost"); + } +} diff --git a/crates/j2k-metal/src/batch.rs b/crates/j2k-metal/src/batch.rs new file mode 100644 index 00000000..dca53a20 --- /dev/null +++ b/crates/j2k-metal/src/batch.rs @@ -0,0 +1,1543 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + cell::OnceCell, + collections::hash_map::DefaultHasher, + hash::{Hash, Hasher}, + sync::{Arc, Mutex, MutexGuard}, +}; + +use j2k::{ + decode_tiles_into, decode_tiles_region_scaled_into, TileBatchOptions, TileDecodeJob, + TileRegionScaledDecodeJob, +}; +use j2k_core::{BackendKind, BackendRequest, DeviceSubmission, Downscale, PixelFormat, Rect}; + +use crate::{profile, Error, J2kDecoder, MetalSession, Storage, Surface, SurfaceResidency}; + +const AUTO_REGION_SCALED_DIRECT_BATCH64_MIN_DIM: u32 = 512; +const AUTO_REGION_SCALED_DIRECT_BATCH64_MIN_COUNT: usize = 64; +const AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM: u32 = 512; +const AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT: usize = 2; +const AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM: u32 = 1024; +const AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT: usize = 16; +const REGION_SCALED_DIRECT_FORMATS: [PixelFormat; 5] = [ + PixelFormat::Gray8, + PixelFormat::Gray16, + PixelFormat::Rgb8, + PixelFormat::Rgba8, + PixelFormat::Rgb16, +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum BatchOp { + Full, + Region(Rect), + Scaled(Downscale), + RegionScaled { roi: Rect, scale: Downscale }, +} + +#[derive(Clone)] +struct QueuedRequest { + input: Arc<[u8]>, + fmt: PixelFormat, + backend: BackendRequest, + op: BatchOp, + output_slot: usize, + max_image_dim: OnceCell>, + input_fingerprint: OnceCell, +} + +impl QueuedRequest { + fn max_image_dim(&self) -> Option { + *self.max_image_dim.get_or_init(|| { + let decoder = J2kDecoder::new(self.input.as_ref()).ok()?; + let dims = decoder.inner.info().dimensions; + Some(dims.0.max(dims.1)) + }) + } + + fn input_fingerprint(&self) -> u64 { + *self.input_fingerprint.get_or_init(|| { + let mut hasher = DefaultHasher::new(); + self.input.len().hash(&mut hasher); + if !self.input.is_empty() { + let len = self.input.len(); + for offset in [0, len / 4, len / 2, len.saturating_sub(8)] { + let end = offset.saturating_add(8).min(len); + self.input[offset..end].hash(&mut hasher); + } + } + hasher.finish() + }) + } + + #[cfg(test)] + fn max_image_dim_cache_filled_for_test(&self) -> bool { + self.max_image_dim.get().is_some() + } + + #[cfg(test)] + fn input_fingerprint_cache_filled_for_test(&self) -> bool { + self.input_fingerprint.get().is_some() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BatchRoute { + Generic, + AutoRegionScaledDirectCpu, + AutoRegionScaledDirectMetal, + AutoRepeatedRegionScaledDirectMetal, +} + +fn profile_route_label(route: BatchRoute) -> &'static str { + match route { + BatchRoute::Generic => "generic", + BatchRoute::AutoRegionScaledDirectCpu => "auto_region_scaled_direct_cpu", + BatchRoute::AutoRegionScaledDirectMetal => "auto_region_scaled_direct_metal", + BatchRoute::AutoRepeatedRegionScaledDirectMetal => { + "auto_repeated_region_scaled_direct_metal" + } + } +} + +struct GroupedRequests { + route: BatchRoute, + requests: Vec, +} + +fn batch_scheduler_invariant(message: &'static str) -> Error { + Error::MetalKernel { + message: format!("internal J2K Metal batch scheduler error: {message}"), + } +} + +impl GroupedRequests { + fn generic(requests: Vec) -> Self { + Self { + route: BatchRoute::Generic, + requests, + } + } +} + +#[doc(hidden)] +pub struct BenchmarkGroupedRequests { + pub batch_count: usize, + pub max_batch_len: usize, +} + +#[doc(hidden)] +pub fn benchmark_group_region_scaled_requests( + inputs: &[Arc<[u8]>], + fmt: PixelFormat, + backend: BackendRequest, + roi: Rect, + scale: Downscale, +) -> BenchmarkGroupedRequests { + let queued = inputs + .iter() + .enumerate() + .map(|(output_slot, input)| QueuedRequest { + input: input.clone(), + fmt, + backend, + op: BatchOp::RegionScaled { roi, scale }, + output_slot, + max_image_dim: OnceCell::new(), + input_fingerprint: OnceCell::new(), + }) + .collect::>(); + let batches = group_metal_requests(queued); + BenchmarkGroupedRequests { + batch_count: batches.len(), + max_batch_len: batches + .iter() + .map(|batch| batch.requests.len()) + .max() + .unwrap_or(0), + } +} + +#[derive(Default)] +pub(crate) struct SessionState { + pub(crate) submissions: u64, + queued: Vec, + completed: Vec>>, +} + +#[derive(Clone, Default)] +pub(crate) struct SharedSession(pub(crate) Arc>); + +impl SharedSession { + pub(crate) fn lock(&self) -> Result, Error> { + self.0.lock().map_err(|_| Error::MetalStatePoisoned { + state: "J2K Metal session", + }) + } +} + +pub struct MetalSubmission { + session: SharedSession, + slot: usize, +} + +impl DeviceSubmission for MetalSubmission { + type Output = Surface; + type Error = Error; + + fn wait(self) -> Result { + let mut session = self.session.lock()?; + flush_if_needed(&mut session); + take_surface(&mut session, self.slot) + } +} + +pub(crate) fn queue_tile_request( + session: &mut MetalSession, + input: &[u8], + fmt: PixelFormat, + backend: BackendRequest, + op: BatchOp, +) -> Result { + queue_tile_request_shared(session, Arc::<[u8]>::from(input), fmt, backend, op) +} + +pub(crate) fn queue_tile_request_shared( + session: &mut MetalSession, + input: Arc<[u8]>, + fmt: PixelFormat, + backend: BackendRequest, + op: BatchOp, +) -> Result { + let mut state = session.shared.lock()?; + let slot = state.completed.len(); + state.completed.push(None); + state.queued.push(QueuedRequest { + input, + fmt, + backend, + op, + output_slot: slot, + max_image_dim: OnceCell::new(), + input_fingerprint: OnceCell::new(), + }); + Ok(MetalSubmission { + session: session.shared.clone(), + slot, + }) +} + +fn flush_if_needed(session: &mut SessionState) { + if session.queued.is_empty() { + return; + } + + let profile_enabled = profile::metal_profile_stages_enabled(); + let queued = std::mem::take(&mut session.queued); + let request_count = queued.len(); + let group_started = profile::profile_now(profile_enabled); + let batches = group_metal_requests(queued); + if profile_enabled { + profile::emit_metal_batch_profile_row( + "decode", + &profile::MetalBatchProfileRow { + slice: "decode_batch", + stage: "group", + pipeline: "metal_cpu_hybrid", + processor: "scheduler", + route: "all", + backend: "mixed", + fmt: "mixed", + request_count, + output_count: batches.len(), + elapsed_us: profile::elapsed_us(group_started), + outcome: "grouped", + }, + ); + } + + for batch in batches { + process_batch(session, batch); + } +} + +fn group_metal_requests(queued: Vec) -> Vec { + coalesce_cpu_host_batches(coalesce_distinct_region_scaled_direct_metal_requests( + coalesce_distinct_full_color_metal_requests( + coalesce_distinct_full_grayscale_metal_requests(group_repeated_full_metal_requests( + queued, + )), + ), + )) +} + +fn group_repeated_full_metal_requests(queued: Vec) -> Vec { + let mut batches: Vec = Vec::new(); + for request in queued { + if let Some(batch) = batches.iter_mut().find(|batch| { + batch.route == BatchRoute::Generic + && can_decode_as_repeated_full_metal_batch(&batch.requests[0], &request) + }) { + batch.requests.push(request); + } else { + batches.push(GroupedRequests::generic(vec![request])); + } + } + batches +} + +fn coalesce_distinct_full_grayscale_metal_requests( + repeated_batches: Vec, +) -> Vec { + let mut batches = Vec::new(); + let mut gray8 = Vec::new(); + let mut gray16 = Vec::new(); + + for batch in repeated_batches { + if batch.route == BatchRoute::Generic + && batch.requests.len() == 1 + && is_distinct_full_grayscale_metal_candidate(&batch.requests[0]) + { + let request = batch + .requests + .into_iter() + .next() + .expect("single-entry batch has request"); + match request.fmt { + PixelFormat::Gray8 => gray8.push(request), + PixelFormat::Gray16 => gray16.push(request), + _ => batches.push(GroupedRequests::generic(vec![request])), + } + } else { + batches.push(batch); + } + } + + push_coalesced_or_single(&mut batches, gray8); + push_coalesced_or_single(&mut batches, gray16); + batches +} + +fn coalesce_distinct_region_scaled_direct_metal_requests( + repeated_batches: Vec, +) -> Vec { + let mut batches = Vec::new(); + let mut metal_by_format: [Vec; REGION_SCALED_DIRECT_FORMATS.len()] = + std::array::from_fn(|_| Vec::new()); + let mut auto_by_format: [Vec; REGION_SCALED_DIRECT_FORMATS.len()] = + std::array::from_fn(|_| Vec::new()); + + for batch in repeated_batches { + if batch.route == BatchRoute::Generic + && batch.requests.len() == 1 + && is_region_scaled_direct_batch_candidate(&batch.requests[0]) + { + let request = batch + .requests + .into_iter() + .next() + .expect("single-entry batch has request"); + let Some(format_idx) = region_scaled_direct_format_index(request.fmt) else { + batches.push(GroupedRequests::generic(vec![request])); + continue; + }; + match request.backend { + BackendRequest::Metal => metal_by_format[format_idx].push(request), + BackendRequest::Auto => auto_by_format[format_idx].push(request), + _ => batches.push(GroupedRequests::generic(vec![request])), + } + } else { + batches.push(batch); + } + } + + for requests in metal_by_format { + push_coalesced_or_single(&mut batches, requests); + } + for requests in auto_by_format { + push_auto_region_scaled_direct_batches(&mut batches, requests); + } + batches +} + +fn push_coalesced_or_single(batches: &mut Vec, requests: Vec) { + push_coalesced_or_single_with_route(batches, requests, BatchRoute::Generic); +} + +fn push_coalesced_or_single_with_route( + batches: &mut Vec, + requests: Vec, + route: BatchRoute, +) { + if requests.is_empty() { + return; + } + if requests.len() == 1 { + batches.extend(requests.into_iter().map(|request| GroupedRequests { + route, + requests: vec![request], + })); + } else { + batches.push(GroupedRequests { route, requests }); + } +} + +fn push_auto_region_scaled_direct_batches( + batches: &mut Vec, + requests: Vec, +) { + let Some(classification) = auto_region_scaled_direct_metal_classification(&requests) else { + push_coalesced_or_single_with_route( + batches, + requests, + BatchRoute::AutoRegionScaledDirectCpu, + ); + return; + }; + + let mut metal_requests = Vec::new(); + let mut cpu_requests = Vec::new(); + for request in requests { + if request + .max_image_dim() + .is_some_and(|max_dim| max_dim >= classification.min_dim) + { + metal_requests.push(request); + } else { + cpu_requests.push(request); + } + } + push_coalesced_or_single_with_route(batches, metal_requests, classification.route); + push_coalesced_or_single_with_route( + batches, + cpu_requests, + BatchRoute::AutoRegionScaledDirectCpu, + ); +} + +#[allow(clippy::similar_names)] +fn coalesce_distinct_full_color_metal_requests( + repeated_batches: Vec, +) -> Vec { + let mut batches = Vec::new(); + let mut rgb8 = Vec::new(); + let mut rgba8 = Vec::new(); + let mut rgb16 = Vec::new(); + + for batch in repeated_batches { + if batch.route == BatchRoute::Generic + && batch.requests.len() == 1 + && is_distinct_full_color_metal_candidate(&batch.requests[0]) + { + let request = batch + .requests + .into_iter() + .next() + .expect("single-entry batch has request"); + match request.fmt { + PixelFormat::Rgb8 => rgb8.push(request), + PixelFormat::Rgba8 => rgba8.push(request), + PixelFormat::Rgb16 => rgb16.push(request), + _ => batches.push(GroupedRequests::generic(vec![request])), + } + } else { + batches.push(batch); + } + } + + push_coalesced_or_single(&mut batches, rgb8); + push_coalesced_or_single(&mut batches, rgba8); + push_coalesced_or_single(&mut batches, rgb16); + batches +} + +fn coalesce_cpu_host_batches(batches: Vec) -> Vec { + let mut coalesced: Vec = Vec::new(); + let mut cpu_groups: Vec> = Vec::new(); + for batch in batches { + if batch.route == BatchRoute::Generic + && batch.requests.len() == 1 + && is_cpu_host_batch_candidate(&batch.requests[0]) + { + let request = batch + .requests + .into_iter() + .next() + .expect("single-entry batch has request"); + if let Some(existing) = cpu_groups + .iter_mut() + .find(|existing| can_coalesce_cpu_host_batch(&existing[0], &request)) + { + existing.push(request); + } else { + cpu_groups.push(vec![request]); + } + } else { + coalesced.push(batch); + } + } + coalesced.extend(cpu_groups.into_iter().map(GroupedRequests::generic)); + coalesced +} + +fn is_cpu_host_batch_candidate(request: &QueuedRequest) -> bool { + matches!(request.op, BatchOp::Full | BatchOp::RegionScaled { .. }) + && matches!(request.backend, BackendRequest::Cpu | BackendRequest::Auto) +} + +fn can_coalesce_cpu_host_batch(first: &QueuedRequest, next: &QueuedRequest) -> bool { + is_cpu_host_batch_candidate(first) + && is_cpu_host_batch_candidate(next) + && first.fmt == next.fmt + && matches!( + (&first.op, &next.op), + (BatchOp::Full, BatchOp::Full) + | (BatchOp::RegionScaled { .. }, BatchOp::RegionScaled { .. }) + ) +} + +fn can_decode_as_repeated_full_grayscale_batch( + first: &QueuedRequest, + next: &QueuedRequest, +) -> bool { + is_repeated_full_grayscale_candidate(first) + && is_repeated_full_grayscale_candidate(next) + && first.fmt == next.fmt + && first.backend == next.backend + && same_input_bytes(first, next) +} + +fn can_decode_as_repeated_full_color_batch(first: &QueuedRequest, next: &QueuedRequest) -> bool { + is_repeated_full_color_candidate(first) + && is_repeated_full_color_candidate(next) + && first.fmt == next.fmt + && first.backend == next.backend + && same_input_bytes(first, next) +} + +fn same_input_bytes(first: &QueuedRequest, next: &QueuedRequest) -> bool { + if Arc::ptr_eq(&first.input, &next.input) { + return true; + } + if first.input.len() != next.input.len() { + return false; + } + if first.input_fingerprint() != next.input_fingerprint() { + return false; + } + first.input.as_ref() == next.input.as_ref() +} + +fn can_decode_as_repeated_full_metal_batch(first: &QueuedRequest, next: &QueuedRequest) -> bool { + can_decode_as_repeated_full_grayscale_batch(first, next) + || can_decode_as_repeated_full_color_batch(first, next) +} + +fn is_repeated_full_grayscale_candidate(request: &QueuedRequest) -> bool { + matches!(request.op, BatchOp::Full) + && matches!(request.fmt, PixelFormat::Gray8 | PixelFormat::Gray16) + && matches!( + request.backend, + BackendRequest::Auto | BackendRequest::Metal + ) +} + +fn is_repeated_full_color_candidate(request: &QueuedRequest) -> bool { + matches!(request.op, BatchOp::Full) + && matches!( + request.fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 + ) + && request.backend == BackendRequest::Metal +} + +fn is_distinct_full_grayscale_metal_candidate(request: &QueuedRequest) -> bool { + matches!(request.op, BatchOp::Full) + && matches!(request.fmt, PixelFormat::Gray8 | PixelFormat::Gray16) + && request.backend == BackendRequest::Metal +} + +fn is_distinct_full_color_metal_candidate(request: &QueuedRequest) -> bool { + matches!(request.op, BatchOp::Full) + && matches!( + request.fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 + ) + && request.backend == BackendRequest::Metal +} + +fn is_region_scaled_direct_batch_candidate(request: &QueuedRequest) -> bool { + matches!(request.op, BatchOp::RegionScaled { .. }) + && region_scaled_direct_format_index(request.fmt).is_some() + && matches!( + request.backend, + BackendRequest::Auto | BackendRequest::Metal + ) +} + +fn region_scaled_direct_format_index(fmt: PixelFormat) -> Option { + REGION_SCALED_DIRECT_FORMATS + .iter() + .position(|candidate| *candidate == fmt) +} + +fn should_auto_use_metal_for_region_scaled_direct_batch(requests: &[QueuedRequest]) -> bool { + auto_region_scaled_direct_metal_min_dim(requests).is_some() +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct AutoRegionScaledDirectMetalClassification { + min_dim: u32, + route: BatchRoute, +} + +fn auto_region_scaled_direct_metal_min_dim(requests: &[QueuedRequest]) -> Option { + auto_region_scaled_direct_metal_classification(requests) + .map(|classification| classification.min_dim) +} + +fn auto_region_scaled_direct_metal_classification( + requests: &[QueuedRequest], +) -> Option { + let first = requests.first()?; + let is_repeated_rgb = matches!( + first.fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 + ) && can_decode_requests_as_repeated_region_scaled_batch(requests); + if matches!( + first.fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 + ) { + if !is_repeated_rgb { + return None; + } + let repeated_rgb_eligible = requests + .iter() + .filter(|request| { + request.max_image_dim().is_some_and(|max_dim| { + max_dim >= AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM + }) + }) + .count(); + if repeated_rgb_eligible >= AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT { + return Some(AutoRegionScaledDirectMetalClassification { + min_dim: AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM, + route: BatchRoute::AutoRepeatedRegionScaledDirectMetal, + }); + } + } + + let mut count_512_class = 0usize; + let mut count_1024_class = 0usize; + for request in requests { + let Some(max_dim) = request.max_image_dim() else { + continue; + }; + if max_dim >= AUTO_REGION_SCALED_DIRECT_BATCH64_MIN_DIM { + count_512_class += 1; + } + if max_dim >= AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM { + count_1024_class += 1; + } + } + + if count_512_class >= AUTO_REGION_SCALED_DIRECT_BATCH64_MIN_COUNT { + Some(AutoRegionScaledDirectMetalClassification { + min_dim: AUTO_REGION_SCALED_DIRECT_BATCH64_MIN_DIM, + route: BatchRoute::AutoRegionScaledDirectMetal, + }) + } else if count_1024_class >= AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT { + Some(AutoRegionScaledDirectMetalClassification { + min_dim: AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM, + route: BatchRoute::AutoRegionScaledDirectMetal, + }) + } else { + None + } +} + +fn can_decode_requests_as_repeated_region_scaled_batch(requests: &[QueuedRequest]) -> bool { + let Some((first, rest)) = requests.split_first() else { + return false; + }; + !rest.is_empty() + && rest.iter().all(|request| { + is_region_scaled_direct_batch_candidate(first) + && is_region_scaled_direct_batch_candidate(request) + && first.fmt == request.fmt + && first.backend == request.backend + && same_input_bytes(first, request) + && matches!( + (first.op, request.op), + ( + BatchOp::RegionScaled { + roi: first_roi, + scale: first_scale + }, + BatchOp::RegionScaled { + roi: request_roi, + scale: request_scale + } + ) if first_roi == request_roi && first_scale == request_scale + ) + }) +} + +fn can_decode_requests_as_repeated_full_grayscale_batch(requests: &[QueuedRequest]) -> bool { + let Some((first, rest)) = requests.split_first() else { + return false; + }; + !rest.is_empty() + && rest + .iter() + .all(|request| can_decode_as_repeated_full_grayscale_batch(first, request)) +} + +fn can_decode_requests_as_repeated_full_color_batch(requests: &[QueuedRequest]) -> bool { + let Some((first, rest)) = requests.split_first() else { + return false; + }; + !rest.is_empty() + && rest + .iter() + .all(|request| can_decode_as_repeated_full_color_batch(first, request)) +} + +fn complete_cpu_host_fallback(session: &mut SessionState, requests: Vec) { + if requests.len() > 1 { + if let Some(Ok(surfaces)) = decode_cpu_host_batch(&requests) { + if surfaces.len() == requests.len() { + session.submissions = session.submissions.saturating_add(1); + for (request, surface) in requests.into_iter().zip(surfaces) { + session.completed[request.output_slot] = Some(Ok(surface)); + } + return; + } + } + } + for request in requests { + session.submissions = session.submissions.saturating_add(1); + session.completed[request.output_slot] = Some(decode_individual(&request)); + } +} + +fn process_batch(session: &mut SessionState, grouped: GroupedRequests) { + let GroupedRequests { route, requests } = grouped; + let profile_enabled = profile::metal_profile_stages_enabled(); + let started = profile::profile_now(profile_enabled); + let request_count = requests.len(); + let slots = if profile_enabled { + requests + .iter() + .map(|request| request.output_slot) + .collect::>() + } else { + Vec::new() + }; + let backend = profile_backend_label(&requests); + let fmt = profile_format_label(&requests); + + process_batch_inner(session, route, requests); + + if profile_enabled { + profile::emit_metal_batch_profile_row( + "decode", + &profile::MetalBatchProfileRow { + slice: "decode_batch", + stage: "execute", + pipeline: "metal_cpu_hybrid", + processor: "hybrid", + route: profile_route_label(route), + backend: &backend, + fmt: &fmt, + request_count, + output_count: profile_completed_output_count(session, &slots), + elapsed_us: profile::elapsed_us(started), + outcome: profile_completed_outcome(session, &slots), + }, + ); + } +} + +fn process_batch_inner( + session: &mut SessionState, + route: BatchRoute, + requests: Vec, +) { + if route == BatchRoute::AutoRegionScaledDirectCpu { + complete_cpu_host_fallback(session, requests); + return; + } + + if matches!( + route, + BatchRoute::AutoRegionScaledDirectMetal | BatchRoute::AutoRepeatedRegionScaledDirectMetal + ) && requests.len() > 1 + { + let decoded = if route == BatchRoute::AutoRepeatedRegionScaledDirectMetal { + decode_repeated_region_scaled_direct_batch_prechecked(&requests) + } else { + decode_distinct_region_scaled_direct_batch_prechecked(&requests) + }; + if let Some(Ok(surfaces)) = decoded { + if surfaces.len() == requests.len() { + session.submissions = session.submissions.saturating_add(1); + for (request, surface) in requests.into_iter().zip(surfaces) { + session.completed[request.output_slot] = Some(Ok(surface)); + } + return; + } + } + complete_cpu_host_fallback(session, requests); + return; + } + + if can_decode_requests_as_repeated_full_grayscale_batch(&requests) { + if let Some(Ok(surfaces)) = decode_repeated_full_grayscale(&requests[0], requests.len()) { + if surfaces.len() == requests.len() { + session.submissions = session.submissions.saturating_add(1); + for (request, surface) in requests.into_iter().zip(surfaces) { + session.completed[request.output_slot] = Some(Ok(surface)); + } + return; + } + } + } + + if can_decode_requests_as_repeated_full_color_batch(&requests) { + if let Some(Ok(surfaces)) = decode_repeated_full_color(&requests[0], requests.len()) { + if surfaces.len() == requests.len() { + session.submissions = session.submissions.saturating_add(1); + for (request, surface) in requests.into_iter().zip(surfaces) { + session.completed[request.output_slot] = Some(Ok(surface)); + } + return; + } + } + } + + if requests.len() > 1 { + if let Some(Ok(surfaces)) = decode_distinct_full_grayscale_batch(&requests) { + if surfaces.len() == requests.len() { + session.submissions = session.submissions.saturating_add(1); + for (request, surface) in requests.into_iter().zip(surfaces) { + session.completed[request.output_slot] = Some(Ok(surface)); + } + return; + } + } + } + + if requests.len() > 1 { + if let Some(result) = decode_distinct_full_color_batch(&requests) { + match result { + Ok(surfaces) if surfaces.len() == requests.len() => { + session.submissions = session.submissions.saturating_add(1); + for (request, surface) in requests.into_iter().zip(surfaces) { + session.completed[request.output_slot] = Some(Ok(surface)); + } + return; + } + Ok(_) | Err(_) => {} + } + } + } + + if requests.len() > 1 { + if let Some(Ok(surfaces)) = decode_distinct_region_scaled_direct_batch(&requests) { + if surfaces.len() == requests.len() { + session.submissions = session.submissions.saturating_add(1); + for (request, surface) in requests.into_iter().zip(surfaces) { + session.completed[request.output_slot] = Some(Ok(surface)); + } + return; + } + } + } + + if requests.len() > 1 { + if let Some(Ok(surfaces)) = decode_cpu_host_batch(&requests) { + if surfaces.len() == requests.len() { + session.submissions = session.submissions.saturating_add(1); + for (request, surface) in requests.into_iter().zip(surfaces) { + session.completed[request.output_slot] = Some(Ok(surface)); + } + return; + } + } + } + + for request in requests { + session.submissions = session.submissions.saturating_add(1); + session.completed[request.output_slot] = Some(decode_individual(&request)); + } +} + +fn profile_backend_label(requests: &[QueuedRequest]) -> String { + let Some(first) = requests.first() else { + return "none".to_string(); + }; + if requests + .iter() + .all(|request| request.backend == first.backend) + { + format!("{:?}", first.backend) + } else { + "mixed".to_string() + } +} + +fn profile_format_label(requests: &[QueuedRequest]) -> String { + let Some(first) = requests.first() else { + return "none".to_string(); + }; + if requests.iter().all(|request| request.fmt == first.fmt) { + format!("{:?}", first.fmt) + } else { + "mixed".to_string() + } +} + +fn profile_completed_output_count(session: &SessionState, slots: &[usize]) -> usize { + slots + .iter() + .filter(|&&slot| session.completed.get(slot).is_some_and(Option::is_some)) + .count() +} + +fn profile_completed_outcome(session: &SessionState, slots: &[usize]) -> &'static str { + let mut ok_count = 0usize; + let mut err_count = 0usize; + let mut cpu_count = 0usize; + let mut metal_count = 0usize; + let mut pending_count = 0usize; + + for &slot in slots { + match session.completed.get(slot).and_then(Option::as_ref) { + Some(Ok(surface)) => { + ok_count = ok_count.saturating_add(1); + match surface.backend { + BackendKind::Cpu => cpu_count = cpu_count.saturating_add(1), + BackendKind::Metal => metal_count = metal_count.saturating_add(1), + BackendKind::Cuda => {} + } + } + Some(Err(_)) => err_count = err_count.saturating_add(1), + None => pending_count = pending_count.saturating_add(1), + } + } + + if pending_count > 0 { + "pending" + } else if err_count > 0 && ok_count == 0 { + "error" + } else if err_count > 0 { + "mixed_error" + } else if metal_count == ok_count && ok_count > 0 { + "metal_surface" + } else if cpu_count == ok_count && ok_count > 0 { + "cpu_surface" + } else if ok_count > 0 { + "mixed" + } else { + "none" + } +} + +fn decode_cpu_host_batch(requests: &[QueuedRequest]) -> Option, Error>> { + decode_cpu_full_batch(requests).or_else(|| decode_cpu_region_scaled_batch(requests)) +} + +fn decode_cpu_full_batch(requests: &[QueuedRequest]) -> Option, Error>> { + let first = requests.first()?; + if requests.len() <= 1 + || !requests + .iter() + .all(|request| is_cpu_host_full_batch_candidate(request) && request.fmt == first.fmt) + { + return None; + } + + Some(decode_cpu_full_batch_inner(requests, first.fmt)) +} + +fn is_cpu_host_full_batch_candidate(request: &QueuedRequest) -> bool { + matches!(request.op, BatchOp::Full) + && matches!(request.backend, BackendRequest::Cpu | BackendRequest::Auto) +} + +fn decode_cpu_full_batch_inner( + requests: &[QueuedRequest], + fmt: PixelFormat, +) -> Result, Error> { + let mut dims = Vec::with_capacity(requests.len()); + let mut outputs = Vec::with_capacity(requests.len()); + for request in requests { + let decoder = J2kDecoder::new(request.input.as_ref())?; + let tile_dims = decoder.inner.info().dimensions; + let stride = tile_dims.0 as usize * fmt.bytes_per_pixel(); + dims.push(tile_dims); + outputs.push(vec![0_u8; stride * tile_dims.1 as usize]); + } + + { + let mut jobs = requests + .iter() + .zip(dims.iter()) + .zip(outputs.iter_mut()) + .map(|((request, dims), out)| TileDecodeJob { + input: request.input.as_ref(), + out: out.as_mut_slice(), + stride: dims.0 as usize * fmt.bytes_per_pixel(), + }) + .collect::>(); + decode_tiles_into(&mut jobs, fmt, TileBatchOptions::default()) + .map_err(|err| Error::Decode(err.source))?; + } + + Ok(outputs + .into_iter() + .zip(dims) + .map(|(bytes, dimensions)| host_surface(bytes, dimensions, fmt)) + .collect()) +} + +fn decode_cpu_region_scaled_batch( + requests: &[QueuedRequest], +) -> Option, Error>> { + let first = requests.first()?; + if requests.len() <= 1 + || !requests.iter().all(|request| { + is_cpu_host_region_scaled_batch_candidate(request) && request.fmt == first.fmt + }) + { + return None; + } + + Some(decode_cpu_region_scaled_batch_inner(requests, first.fmt)) +} + +fn is_cpu_host_region_scaled_batch_candidate(request: &QueuedRequest) -> bool { + matches!(request.op, BatchOp::RegionScaled { .. }) + && matches!(request.backend, BackendRequest::Cpu | BackendRequest::Auto) +} + +fn decode_cpu_region_scaled_batch_inner( + requests: &[QueuedRequest], + fmt: PixelFormat, +) -> Result, Error> { + let mut dims = Vec::with_capacity(requests.len()); + let mut outputs = Vec::with_capacity(requests.len()); + for request in requests { + let BatchOp::RegionScaled { roi, scale } = request.op else { + return Err(batch_scheduler_invariant( + "CPU region-scaled batch contains a non-region-scaled request", + )); + }; + let dimensions = roi.scaled_covering(scale); + let stride = dimensions.w as usize * fmt.bytes_per_pixel(); + dims.push((dimensions.w, dimensions.h)); + outputs.push(vec![0_u8; stride * dimensions.h as usize]); + } + + { + let mut jobs = Vec::with_capacity(requests.len()); + for (request, out) in requests.iter().zip(outputs.iter_mut()) { + let BatchOp::RegionScaled { roi, scale } = request.op else { + return Err(batch_scheduler_invariant( + "CPU region-scaled job creation received a non-region-scaled request", + )); + }; + let dimensions = roi.scaled_covering(scale); + jobs.push(TileRegionScaledDecodeJob { + input: request.input.as_ref(), + out: out.as_mut_slice(), + stride: dimensions.w as usize * fmt.bytes_per_pixel(), + roi, + scale, + }); + } + decode_tiles_region_scaled_into(&mut jobs, fmt, TileBatchOptions::default()) + .map_err(|err| Error::Decode(err.source))?; + } + + Ok(outputs + .into_iter() + .zip(dims) + .map(|(bytes, dimensions)| host_surface(bytes, dimensions, fmt)) + .collect()) +} + +fn host_surface(bytes: Vec, dimensions: (u32, u32), fmt: PixelFormat) -> Surface { + Surface { + backend: BackendKind::Cpu, + residency: SurfaceResidency::Host, + dimensions, + fmt, + pitch_bytes: dimensions.0 as usize * fmt.bytes_per_pixel(), + byte_offset: 0, + storage: Storage::Host(bytes), + } +} + +fn decode_repeated_full_grayscale( + request: &QueuedRequest, + count: usize, +) -> Option, Error>> { + if !is_repeated_full_grayscale_candidate(request) || count <= 1 { + return None; + } + + #[cfg(target_os = "macos")] + { + let result = + J2kDecoder::new(request.input.as_ref()).and_then(|mut decoder| match request.backend { + BackendRequest::Auto => { + decoder.decode_repeated_grayscale_auto_to_device(request.fmt, count) + } + BackendRequest::Metal => { + decoder.decode_repeated_grayscale_direct_to_device(request.fmt, count) + } + _ => Err(batch_scheduler_invariant( + "repeated grayscale batch contains an unsupported backend", + )), + }); + Some(result) + } + + #[cfg(not(target_os = "macos"))] + { + None + } +} + +fn decode_repeated_full_color( + request: &QueuedRequest, + count: usize, +) -> Option, Error>> { + if !is_repeated_full_color_candidate(request) || count <= 1 { + return None; + } + + #[cfg(target_os = "macos")] + { + let result = J2kDecoder::new(request.input.as_ref()).and_then(|mut decoder| { + decoder.decode_repeated_color_direct_to_device(request.fmt, count) + }); + Some(result) + } + + #[cfg(not(target_os = "macos"))] + { + None + } +} + +fn decode_distinct_full_grayscale_batch( + requests: &[QueuedRequest], +) -> Option, Error>> { + let first = requests.first()?; + if requests.len() <= 1 + || !requests.iter().all(|request| { + is_distinct_full_grayscale_metal_candidate(request) && request.fmt == first.fmt + }) + { + return None; + } + + #[cfg(target_os = "macos")] + { + let inputs = requests + .iter() + .map(|request| request.input.clone()) + .collect::>(); + Some(crate::decode_full_grayscale_batch_direct_to_device( + &inputs, first.fmt, + )) + } + + #[cfg(not(target_os = "macos"))] + { + None + } +} + +fn decode_distinct_full_color_batch( + requests: &[QueuedRequest], +) -> Option, Error>> { + let first = requests.first()?; + if requests.len() <= 1 + || !requests.iter().all(|request| { + is_distinct_full_color_metal_candidate(request) && request.fmt == first.fmt + }) + { + return None; + } + + #[cfg(target_os = "macos")] + { + let inputs = requests + .iter() + .map(|request| request.input.clone()) + .collect::>(); + Some(crate::decode_full_color_batch_direct_to_device( + &inputs, first.fmt, + )) + } + + #[cfg(not(target_os = "macos"))] + { + None + } +} + +fn decode_distinct_region_scaled_direct_batch( + requests: &[QueuedRequest], +) -> Option, Error>> { + decode_distinct_region_scaled_direct_batch_inner(requests, false) +} + +fn decode_repeated_region_scaled_direct_batch_prechecked( + requests: &[QueuedRequest], +) -> Option, Error>> { + let first = requests.first()?; + if requests.len() <= 1 { + return None; + } + if !matches!(first.op, BatchOp::RegionScaled { .. }) { + return None; + } + + #[cfg(target_os = "macos")] + { + let BatchOp::RegionScaled { roi, scale } = first.op else { + return Some(Err(batch_scheduler_invariant( + "repeated direct batch is missing its region-scaled operation", + ))); + }; + let result = match first.fmt { + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 => { + crate::hybrid::decode_repeated_region_scaled_color_batch_direct_to_device( + first.input.as_ref(), + roi, + scale, + first.fmt, + requests.len(), + ) + } + _ => return None, + }; + Some(result) + } + + #[cfg(not(target_os = "macos"))] + { + None + } +} + +fn decode_distinct_region_scaled_direct_batch_prechecked( + requests: &[QueuedRequest], +) -> Option, Error>> { + decode_distinct_region_scaled_direct_batch_inner(requests, true) +} + +fn decode_distinct_region_scaled_direct_batch_inner( + requests: &[QueuedRequest], + auto_metal_prechecked: bool, +) -> Option, Error>> { + let first = requests.first()?; + if requests.len() <= 1 + || !requests.iter().all(|request| { + is_region_scaled_direct_batch_candidate(request) + && request.fmt == first.fmt + && request.backend == first.backend + }) + { + return None; + } + if first.backend == BackendRequest::Auto + && !auto_metal_prechecked + && !should_auto_use_metal_for_region_scaled_direct_batch(requests) + { + return None; + } + + #[cfg(target_os = "macos")] + { + let mut request_specs = Vec::with_capacity(requests.len()); + for request in requests { + match request.op { + BatchOp::RegionScaled { roi, scale } => { + request_specs.push((request.input.clone(), roi, scale)); + } + _ => { + return Some(Err(batch_scheduler_invariant( + "direct region-scaled batch contains a non-region-scaled request", + ))); + } + } + } + let result = match first.fmt { + PixelFormat::Gray8 | PixelFormat::Gray16 => { + crate::hybrid::decode_region_scaled_grayscale_batch_direct_to_device( + &request_specs, + first.fmt, + ) + } + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 => { + crate::hybrid::decode_region_scaled_color_batch_direct_to_device( + &request_specs, + first.fmt, + ) + } + _ => Err(batch_scheduler_invariant( + "direct region-scaled batch contains an unsupported pixel format", + )), + }; + Some(result) + } + + #[cfg(not(target_os = "macos"))] + { + None + } +} + +fn decode_individual(request: &QueuedRequest) -> Result { + let mut decoder = J2kDecoder::new(request.input.as_ref())?; + match request.op { + BatchOp::Full => decoder.decode_to_surface_impl(request.fmt, request.backend), + BatchOp::Region(roi) => { + decoder.decode_region_to_surface_impl(request.fmt, roi, request.backend) + } + BatchOp::Scaled(scale) => { + decoder.decode_scaled_to_surface_impl(request.fmt, scale, request.backend) + } + BatchOp::RegionScaled { roi, scale } => { + decoder.decode_region_scaled_to_surface_impl(request.fmt, roi, scale, request.backend) + } + } +} + +fn take_surface(session: &mut SessionState, slot: usize) -> Result { + session + .completed + .get_mut(slot) + .and_then(Option::take) + .ok_or_else(|| Error::MetalKernel { + message: format!("missing queued J2K Metal surface for slot {slot}"), + })? +} + +#[cfg(test)] +mod tests { + use super::*; + + fn auto_rgb_region_scaled_request(input: Arc<[u8]>) -> QueuedRequest { + QueuedRequest { + input, + fmt: PixelFormat::Rgb8, + backend: BackendRequest::Auto, + op: BatchOp::RegionScaled { + roi: Rect { + x: 128, + y: 128, + w: 512, + h: 256, + }, + scale: Downscale::Quarter, + }, + output_slot: 0, + max_image_dim: OnceCell::new(), + input_fingerprint: OnceCell::new(), + } + } + + fn auto_rgb_region_scaled_request_with_max_dim( + input: Arc<[u8]>, + max_image_dim: u32, + ) -> QueuedRequest { + let request = auto_rgb_region_scaled_request(input); + request.max_image_dim.set(Some(max_image_dim)).ok(); + request + } + + #[test] + fn auto_region_scaled_rgb_threshold_requires_repeated_inputs() { + let requests = (0..AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT) + .map(|idx| auto_rgb_region_scaled_request(Arc::from([idx as u8]))) + .collect::>(); + + assert!(!can_decode_requests_as_repeated_region_scaled_batch( + &requests + )); + assert_eq!( + auto_region_scaled_direct_metal_min_dim(&requests), + None, + "distinct RGB ROI+scaled Auto batches must stay CPU until hybrid wins for distinct inputs" + ); + + let shared = Arc::<[u8]>::from([1_u8]); + let repeated = (0..AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT) + .map(|_| auto_rgb_region_scaled_request(shared.clone())) + .collect::>(); + assert!(can_decode_requests_as_repeated_region_scaled_batch( + &repeated + )); + } + + #[test] + fn auto_region_scaled_repeated_rgb_uses_measured_batch_two_metal_threshold() { + let shared = Arc::<[u8]>::from([1_u8]); + let repeated = (0..2) + .map(|_| auto_rgb_region_scaled_request_with_max_dim(shared.clone(), 512)) + .collect::>(); + + assert_eq!( + auto_region_scaled_direct_metal_min_dim(&repeated), + Some(512), + "measured repeated RGB ROI+scaled batches should route to Metal from batch 2 at 512px" + ); + + let single = vec![auto_rgb_region_scaled_request_with_max_dim(shared, 512)]; + assert_eq!(auto_region_scaled_direct_metal_min_dim(&single), None); + } + + #[test] + fn queued_request_caches_image_dimension_probe() { + let request = auto_rgb_region_scaled_request(Arc::from([0_u8])); + + assert!(!request.max_image_dim_cache_filled_for_test()); + assert_eq!(request.max_image_dim(), None); + assert!(request.max_image_dim_cache_filled_for_test()); + assert_eq!(request.max_image_dim(), None); + } + + #[test] + fn repeated_input_check_uses_pointer_identity_before_fingerprint() { + let shared = Arc::<[u8]>::from([1_u8, 2, 3, 4]); + let first = auto_rgb_region_scaled_request(shared.clone()); + let next = auto_rgb_region_scaled_request(shared); + + assert!(same_input_bytes(&first, &next)); + assert!(!first.input_fingerprint_cache_filled_for_test()); + assert!(!next.input_fingerprint_cache_filled_for_test()); + } + + #[test] + fn auto_region_scaled_grouping_preserves_repeated_rgb_metal_decision() { + let shared = Arc::<[u8]>::from([1_u8, 2, 3, 4]); + let requests = (0..AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT) + .map(|_| { + auto_rgb_region_scaled_request_with_max_dim( + shared.clone(), + AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM, + ) + }) + .collect::>(); + + let grouped = group_metal_requests(requests); + + assert_eq!(grouped.len(), 1); + assert_eq!( + grouped[0].route, + BatchRoute::AutoRepeatedRegionScaledDirectMetal + ); + assert_eq!( + grouped[0].requests.len(), + AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT + ); + assert!( + grouped[0] + .requests + .iter() + .all(|request| !request.input_fingerprint_cache_filled_for_test()), + "shared repeated inputs should be classified by Arc identity without fingerprinting" + ); + } + + #[test] + fn auto_region_scaled_distinct_rgb_grouping_preserves_cpu_decision() { + let requests = (0..AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT) + .map(|idx| { + auto_rgb_region_scaled_request_with_max_dim( + Arc::from([idx as u8]), + AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_DIM, + ) + }) + .collect::>(); + + let grouped = group_metal_requests(requests); + + assert_eq!(grouped.len(), 1); + assert_eq!(grouped[0].route, BatchRoute::AutoRegionScaledDirectCpu); + assert_eq!( + grouped[0].requests.len(), + AUTO_REGION_SCALED_DIRECT_BATCH16_MIN_COUNT + ); + } + + #[test] + fn profile_route_labels_are_stable_for_decode_batch_slices() { + assert_eq!(profile_route_label(BatchRoute::Generic), "generic"); + assert_eq!( + profile_route_label(BatchRoute::AutoRegionScaledDirectCpu), + "auto_region_scaled_direct_cpu" + ); + assert_eq!( + profile_route_label(BatchRoute::AutoRegionScaledDirectMetal), + "auto_region_scaled_direct_metal" + ); + assert_eq!( + profile_route_label(BatchRoute::AutoRepeatedRegionScaledDirectMetal), + "auto_repeated_region_scaled_direct_metal" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn auto_region_scaled_prechecked_error_does_not_retry_generic_direct_path() { + let _guard = crate::hybrid::region_scaled_color_plan_test_lock_for_test(); + crate::hybrid::reset_region_scaled_color_plan_builds_for_test(); + let shared = Arc::<[u8]>::from([1_u8, 2, 3, 4]); + let requests = (0..AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_COUNT) + .map(|slot| { + let mut request = auto_rgb_region_scaled_request_with_max_dim( + shared.clone(), + AUTO_REGION_SCALED_DIRECT_REPEATED_RGB_MIN_DIM, + ); + request.output_slot = slot; + request + }) + .collect::>(); + let mut session = SessionState { + submissions: 0, + queued: Vec::new(), + completed: (0..requests.len()).map(|_| None).collect(), + }; + + process_batch( + &mut session, + GroupedRequests { + route: BatchRoute::AutoRepeatedRegionScaledDirectMetal, + requests, + }, + ); + + assert_eq!( + crate::hybrid::region_scaled_color_plan_builds_for_test(), + 1, + "failed prechecked Auto Metal routing should fall back to CPU without retrying generic direct Metal" + ); + assert!( + session + .completed + .iter() + .all(|result| matches!(result, Some(Err(_)))), + "invalid inputs should be surfaced on every fallback request" + ); + } +} diff --git a/crates/j2k-metal/src/buffer_pool.rs b/crates/j2k-metal/src/buffer_pool.rs new file mode 100644 index 00000000..d89f49a4 --- /dev/null +++ b/crates/j2k-metal/src/buffer_pool.rs @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(test)] +use std::cell::Cell; +use std::{collections::HashMap, sync::Mutex}; + +use j2k_metal_support::{private_buffer, shared_buffer}; +use metal::{Buffer, Device}; + +use crate::Error; + +#[cfg(test)] +std::thread_local! { + static PRIVATE_BUFFER_POOL_MISSES: Cell = const { Cell::new(0) }; + static SHARED_BUFFER_POOL_MISSES: Cell = const { Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn reset_private_buffer_pool_misses_for_test() { + PRIVATE_BUFFER_POOL_MISSES.with(|misses| misses.set(0)); +} + +#[cfg(test)] +pub(crate) fn private_buffer_pool_misses_for_test() -> usize { + PRIVATE_BUFFER_POOL_MISSES.with(Cell::get) +} + +#[cfg(test)] +pub(crate) fn reset_shared_buffer_pool_misses_for_test() { + SHARED_BUFFER_POOL_MISSES.with(|misses| misses.set(0)); +} + +#[cfg(test)] +pub(crate) fn shared_buffer_pool_misses_for_test() -> usize { + SHARED_BUFFER_POOL_MISSES.with(Cell::get) +} + +#[cfg(test)] +fn record_private_buffer_pool_miss_for_test() { + PRIVATE_BUFFER_POOL_MISSES.with(|misses| misses.set(misses.get() + 1)); +} + +#[cfg(test)] +fn record_shared_buffer_pool_miss_for_test() { + SHARED_BUFFER_POOL_MISSES.with(|misses| misses.set(misses.get() + 1)); +} + +pub(crate) struct MetalBufferPools { + private: Mutex>>, + shared: Mutex>>, +} + +impl MetalBufferPools { + pub(crate) fn new() -> Self { + Self { + private: Mutex::new(HashMap::new()), + shared: Mutex::new(HashMap::new()), + } + } + + pub(crate) fn take_private(&self, device: &Device, bytes: usize) -> Result { + let bytes = bytes.max(1); + let mut pool = self.private.lock().map_err(|_| Error::MetalStatePoisoned { + state: "j2k metal private buffer pool", + })?; + if let Some(buffer) = pool.get_mut(&bytes).and_then(Vec::pop) { + Ok(buffer) + } else { + #[cfg(test)] + record_private_buffer_pool_miss_for_test(); + Ok(private_buffer(device, bytes)) + } + } + + pub(crate) fn recycle_private(&self, bytes: usize, buffer: Buffer) -> Result<(), Error> { + let bytes = bytes.max(1); + self.private + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "j2k metal private buffer pool", + })? + .entry(bytes) + .or_default() + .push(buffer); + Ok(()) + } + + pub(crate) fn take_shared(&self, device: &Device, bytes: usize) -> Result { + let bytes = bytes.max(1); + let mut pool = self.shared.lock().map_err(|_| Error::MetalStatePoisoned { + state: "j2k metal shared buffer pool", + })?; + if let Some(buffer) = pool.get_mut(&bytes).and_then(Vec::pop) { + Ok(buffer) + } else { + #[cfg(test)] + record_shared_buffer_pool_miss_for_test(); + Ok(shared_buffer(device, bytes)) + } + } + + pub(crate) fn recycle_shared(&self, bytes: usize, buffer: Buffer) -> Result<(), Error> { + let bytes = bytes.max(1); + self.shared + .lock() + .map_err(|_| Error::MetalStatePoisoned { + state: "j2k metal shared buffer pool", + })? + .entry(bytes) + .or_default() + .push(buffer); + Ok(()) + } +} diff --git a/crates/signinum-j2k-metal/src/classic.metal b/crates/j2k-metal/src/classic.metal similarity index 99% rename from crates/signinum-j2k-metal/src/classic.metal rename to crates/j2k-metal/src/classic.metal index c03d6b5f..c3687b28 100644 --- a/crates/signinum-j2k-metal/src/classic.metal +++ b/crates/j2k-metal/src/classic.metal @@ -12,6 +12,7 @@ struct J2kClassicCleanupBatchJob { uint output_offset; uint missing_msbs; uint total_bitplanes; + uint roi_shift; uint number_of_coding_passes; uint sub_band_type; uint style_flags; diff --git a/crates/signinum-j2k-metal/src/classic.rs b/crates/j2k-metal/src/classic.rs similarity index 96% rename from crates/signinum-j2k-metal/src/classic.rs rename to crates/j2k-metal/src/classic.rs index 56f38216..3f1a6988 100644 --- a/crates/signinum-j2k-metal/src/classic.rs +++ b/crates/j2k-metal/src/classic.rs @@ -3,8 +3,8 @@ #[cfg(target_os = "macos")] use crate::compute; #[cfg(target_os = "macos")] -use signinum_j2k_native::DecodingError; -use signinum_j2k_native::{ +use j2k_native::DecodingError; +use j2k_native::{ decode_ht_code_block_scalar, decode_j2k_code_block_scalar, decode_j2k_sub_band_scalar, HtCodeBlockDecodeJob, HtCodeBlockDecoder, J2kCodeBlockDecodeJob, J2kSubBandDecodeJob, Result, }; @@ -109,6 +109,9 @@ fn supports_metal_classic_kernel(job: &J2kCodeBlockDecodeJob<'_>) -> bool { if job.number_of_coding_passes == 0 { return false; } + if job.roi_shift != 0 { + return false; + } if job.data.is_empty() { return false; } @@ -185,7 +188,7 @@ mod tests { use super::MetalClassicBlockDecoder; #[cfg(target_os = "macos")] use crate::compute; - use signinum_j2k_native::{ + use j2k_native::{ decode_j2k_code_block_scalar, encode, ColorSpace, DecodeSettings, DecoderContext, EncodeOptions, HtCodeBlockDecodeJob, HtCodeBlockDecoder, Image, J2kCodeBlockDecodeJob, J2kCodeBlockSegment, @@ -194,15 +197,16 @@ mod tests { #[derive(Clone)] struct OwnedClassicJob { data: Vec, - segments: Vec, + segments: Vec, width: u32, height: u32, output_stride: usize, missing_bit_planes: u8, number_of_coding_passes: u8, total_bitplanes: u8, - sub_band_type: signinum_j2k_native::J2kSubBandType, - style: signinum_j2k_native::J2kCodeBlockStyle, + roi_shift: u8, + sub_band_type: j2k_native::J2kSubBandType, + style: j2k_native::J2kCodeBlockStyle, strict: bool, dequantization_step: f32, } @@ -218,6 +222,7 @@ mod tests { missing_bit_planes: self.missing_bit_planes, number_of_coding_passes: self.number_of_coding_passes, total_bitplanes: self.total_bitplanes, + roi_shift: self.roi_shift, sub_band_type: self.sub_band_type, style: self.style, strict: self.strict, @@ -279,7 +284,7 @@ mod tests { &mut self, job: J2kCodeBlockDecodeJob<'_>, output: &mut [f32], - ) -> signinum_j2k_native::Result { + ) -> j2k_native::Result { if self.first.is_none() { self.first = Some(OwnedClassicJob { data: job.data.to_vec(), @@ -290,6 +295,7 @@ mod tests { missing_bit_planes: job.missing_bit_planes, number_of_coding_passes: job.number_of_coding_passes, total_bitplanes: job.total_bitplanes, + roi_shift: job.roi_shift, sub_band_type: job.sub_band_type, style: job.style, strict: job.strict, @@ -304,8 +310,8 @@ mod tests { &mut self, job: HtCodeBlockDecodeJob<'_>, output: &mut [f32], - ) -> signinum_j2k_native::Result<()> { - signinum_j2k_native::decode_ht_code_block_scalar(job, output) + ) -> j2k_native::Result<()> { + j2k_native::decode_ht_code_block_scalar(job, output) } } @@ -519,13 +525,13 @@ mod tests { decode_j2k_code_block_scalar(job.as_job(), &mut original_coefficients) .expect("decode original cleanup job"); let original_coefficients = integral_coefficients(&original_coefficients); - let encoded = signinum_j2k_native::encode_j2k_code_block_scalar_with_style( + let encoded = j2k_native::encode_j2k_code_block_scalar_with_style( &original_coefficients, job.width, job.height, job.sub_band_type, job.total_bitplanes, - signinum_j2k_native::J2kCodeBlockStyle { + j2k_native::J2kCodeBlockStyle { segmentation_symbols: true, ..job.style }, @@ -587,13 +593,13 @@ mod tests { decode_j2k_code_block_scalar(job.as_job(), &mut original_coefficients) .expect("decode original multi-pass job"); let original_coefficients = integral_coefficients(&original_coefficients); - let encoded = signinum_j2k_native::encode_j2k_code_block_scalar_with_style( + let encoded = j2k_native::encode_j2k_code_block_scalar_with_style( &original_coefficients, job.width, job.height, job.sub_band_type, job.total_bitplanes, - signinum_j2k_native::J2kCodeBlockStyle { + j2k_native::J2kCodeBlockStyle { reset_context_probabilities: true, ..job.style }, @@ -632,13 +638,13 @@ mod tests { decode_j2k_code_block_scalar(job.as_job(), &mut original_coefficients) .expect("decode original tall job"); let original_coefficients = integral_coefficients(&original_coefficients); - let encoded = signinum_j2k_native::encode_j2k_code_block_scalar_with_style( + let encoded = j2k_native::encode_j2k_code_block_scalar_with_style( &original_coefficients, job.width, job.height, job.sub_band_type, job.total_bitplanes, - signinum_j2k_native::J2kCodeBlockStyle { + j2k_native::J2kCodeBlockStyle { vertically_causal_context: true, ..job.style }, @@ -677,13 +683,13 @@ mod tests { decode_j2k_code_block_scalar(job.as_job(), &mut original_coefficients) .expect("decode original multi-pass job"); let original_coefficients = integral_coefficients(&original_coefficients); - let encoded = signinum_j2k_native::encode_j2k_code_block_scalar_with_style( + let encoded = j2k_native::encode_j2k_code_block_scalar_with_style( &original_coefficients, job.width, job.height, job.sub_band_type, job.total_bitplanes, - signinum_j2k_native::J2kCodeBlockStyle { + j2k_native::J2kCodeBlockStyle { termination_on_each_pass: true, ..job.style }, @@ -729,13 +735,13 @@ mod tests { decode_j2k_code_block_scalar(job.as_job(), &mut original_coefficients) .expect("decode original bypass job"); let original_coefficients = integral_coefficients(&original_coefficients); - let encoded = signinum_j2k_native::encode_j2k_code_block_scalar_with_style( + let encoded = j2k_native::encode_j2k_code_block_scalar_with_style( &original_coefficients, job.width, job.height, job.sub_band_type, job.total_bitplanes, - signinum_j2k_native::J2kCodeBlockStyle { + j2k_native::J2kCodeBlockStyle { selective_arithmetic_coding_bypass: true, ..job.style }, diff --git a/crates/signinum-j2k-metal/src/compute.rs b/crates/j2k-metal/src/compute.rs similarity index 54% rename from crates/signinum-j2k-metal/src/compute.rs rename to crates/j2k-metal/src/compute.rs index 0a905078..c371b7c5 100644 --- a/crates/signinum-j2k-metal/src/compute.rs +++ b/crates/j2k-metal/src/compute.rs @@ -1,3936 +1,4791 @@ // SPDX-License-Identifier: Apache-2.0 -#[cfg(all(target_os = "macos", test))] -use std::cell::Cell; -#[cfg(all(target_os = "macos", test))] -use std::sync::atomic::{AtomicUsize, Ordering}; #[cfg(target_os = "macos")] use std::{ cell::RefCell, collections::HashMap, mem::{size_of, size_of_val}, - sync::{Arc, Mutex, OnceLock}, - time::Duration, + sync::Arc, + time::{Duration, Instant}, }; +use j2k_core::{PixelFormat, Rect}; +#[cfg(test)] +use j2k_metal_support::system_default_device; #[cfg(target_os = "macos")] -use metal::{ - foreign_types::ForeignType, - objc::{runtime::Sel, Message}, - Buffer, CommandBuffer, CommandBufferRef, CommandQueue, CompileOptions, - ComputeCommandEncoderRef, ComputePipelineState, Device, MTLCommandQueue, MTLResourceOptions, - MTLSize, +use j2k_metal_support::{ + checked_command_queue, dispatch_1d_pipeline, dispatch_2d_pipeline, dispatch_3d_pipeline, + dispatch_single_thread, MetalPipelineLoader, MetalSupportError, }; -use signinum_core::{PixelFormat, Rect}; #[cfg(target_os = "macos")] -use signinum_j2k_native::HtCodeBlockDecoder; -use signinum_j2k_native::{ +use j2k_native::{ ht_uvlc_encode_table, ht_uvlc_table0, ht_uvlc_table1, ht_vlc_encode_table0, - ht_vlc_encode_table1, ht_vlc_table0, ht_vlc_table1, ColorSpace as NativeColorSpace, + ht_vlc_encode_table1, ht_vlc_table0, ht_vlc_table1, + pack_j2k_code_block_scalar_from_tier1_tokens, ColorSpace as NativeColorSpace, DecodedComponents as NativeDecodedComponents, EncodeProgressionOrder, EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, HtCodeBlockDecodeJob, HtSubBandDecodeJob, J2kCodeBlockDecodeJob, - J2kCodeBlockSegment, J2kDirectBandId, J2kDirectColorPlan, J2kDirectGrayscalePlan, - J2kDirectGrayscaleStep, J2kDirectIdwtStep, J2kDirectStoreStep, J2kForwardDwt53Level, - J2kForwardDwt53Output, J2kHtCodeBlockEncodeJob, J2kInverseMctJob, - J2kPacketizationBlockCodingMode, J2kPacketizationEncodeJob, J2kPacketizationPacketDescriptor, + J2kCodeBlockSegment, J2kDeinterleaveToF32Job, J2kDirectBandId, J2kDirectColorPlan, + J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kDirectIdwtStep, J2kDirectStoreStep, + J2kForwardDwt53Level, J2kForwardDwt53Output, J2kForwardDwt97Level, J2kForwardDwt97Output, + J2kHtCodeBlockEncodeJob, J2kInverseMctJob, J2kPacketizationBlockCodingMode, + J2kPacketizationEncodeJob, J2kPacketizationPacketDescriptor, J2kQuantizeSubbandJob, J2kSingleDecompositionIdwtJob, J2kStoreComponentJob, J2kSubBandDecodeJob, - J2kTier1CodeBlockEncodeJob, J2kWaveletTransform, + J2kTier1CodeBlockEncodeJob, J2kTier1TokenSegment, J2kWaveletTransform, }; #[cfg(target_os = "macos")] -use signinum_j2k_native::{ - DecodeSettings as NativeDecodeSettings, DecoderContext as NativeDecoderContext, - Image as NativeImage, +use metal::{ + foreign_types::ForeignType, Buffer, CommandBuffer, CommandBufferRef, CommandQueue, + ComputeCommandEncoderRef, ComputePipelineState, Device, MTLResourceOptions, MTLSize, }; +#[cfg(target_os = "macos")] +use rayon::prelude::*; #[cfg(target_os = "macos")] use crate::{ - classic::MetalClassicBlockDecoder, ht::MetalHtBlockDecoder, idwt::MetalIdwtDecoder, - mct::MetalMctDecoder, store::MetalStoreDecoder, + buffer_pool::MetalBufferPools, + profile_env::{ + classic_tier1_gpu_token_pack_requested, classic_tier1_split_gpu_token_pack_requested, + classic_tier1_split_mq_byte_gpu_token_pack_disabled, + classic_tier1_split_mq_byte_gpu_token_pack_requested, hybrid_stage_signpost, + label_command_buffer, label_compute_encoder, + metal_profile_classic_tier1_arithmetic_pack_enabled, + metal_profile_classic_tier1_density_enabled, metal_profile_classic_tier1_pass_plan_enabled, + metal_profile_classic_tier1_raw_pack_enabled, + metal_profile_classic_tier1_split_token_emit_enabled, + metal_profile_classic_tier1_symbol_plan_enabled, + metal_profile_classic_tier1_token_emit_enabled, + metal_profile_classic_tier1_token_pack_enabled, + metal_profile_coefficient_prep_split_commands_enabled, + metal_profile_decode_split_commands_enabled, HybridSignpostName, + SIGNPOST_DECODE_HYBRID_COEFFICIENT_UPLOAD, SIGNPOST_DECODE_HYBRID_COMMAND_WAIT, + SIGNPOST_DECODE_HYBRID_CPU_TIER1, SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE, + SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE, + SIGNPOST_DECODE_HYBRID_STORE_COMMAND_ENCODE, + SIGNPOST_ENCODE_HYBRID_CLASSIC_CODESTREAM_ASSEMBLY_COMMAND_ENCODE, + SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKETIZATION_COMMAND_ENCODE, + SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_BUFFER_SETUP, + SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_PLAN, + SIGNPOST_ENCODE_HYBRID_CLASSIC_PAYLOAD_COPY_COMMAND_ENCODE, + SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_COMMAND_ENCODE, + SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_SETUP, SIGNPOST_ENCODE_HYBRID_COMMAND_WAIT, + SIGNPOST_ENCODE_HYBRID_HT_CODESTREAM_ASSEMBLY_COMMAND_ENCODE, + SIGNPOST_ENCODE_HYBRID_HT_PACKETIZATION_COMMAND_ENCODE, + SIGNPOST_ENCODE_HYBRID_HT_PACKET_BLOCK_PREP_COMMAND_ENCODE, + SIGNPOST_ENCODE_HYBRID_HT_PACKET_BUFFER_SETUP, SIGNPOST_ENCODE_HYBRID_HT_PACKET_PLAN, + SIGNPOST_ENCODE_HYBRID_HT_PAYLOAD_COPY_COMMAND_ENCODE, + SIGNPOST_ENCODE_HYBRID_HT_TIER1_COMMAND_ENCODE, SIGNPOST_ENCODE_HYBRID_HT_TIER1_SETUP, + SIGNPOST_ENCODE_HYBRID_RESULT_HARVEST, + }, }; use crate::{Error, Surface}; +mod abi; +pub(crate) use self::abi::*; +#[cfg(target_os = "macos")] +mod buffer_validation; +#[cfg(target_os = "macos")] +pub(crate) use self::buffer_validation::{ + copy_interleaved_padded_to_shared_buffer, validate_metal_buffer_matches_bytes, + validate_metal_buffers_match, +}; +#[cfg(target_os = "macos")] +mod classic_encode_pipeline; +#[cfg(target_os = "macos")] +use self::classic_encode_pipeline::{ + classic_cod_block_style_from_flags, classic_encode_code_blocks_pipeline, + classic_encode_code_blocks_pipeline_kind, classic_resident_style_flags_from_env, + classic_tier1_gpu_token_pack_supported, J2kClassicEncodePipelineKind, +}; +#[cfg(target_os = "macos")] +mod classic_tier1_stats; +#[cfg(target_os = "macos")] +use self::classic_tier1_stats::{ + accumulate_classic_tier1_scan_estimates, classic_tier1_pass_class_counts, +}; +#[cfg(target_os = "macos")] +mod code_block_decoder; +#[cfg(target_os = "macos")] +use self::code_block_decoder::MetalCodeBlockDecoder; +mod direct_cache; +use self::direct_cache::CpuTier1CoefficientCache; +#[cfg(target_os = "macos")] +mod direct_buffers; +#[cfg(target_os = "macos")] +use self::direct_buffers::{ + borrow_mut_slice_buffer, borrow_slice_buffer, copied_recyclable_shared_slice_buffer, + copied_slice_buffer, owned_slice_buffer, take_classic_coefficients_scratch_buffer, + take_classic_states_scratch_buffer, wrap_f32_output_buffer, zeroed_recyclable_shared_buffer, +}; +#[cfg(target_os = "macos")] +mod direct_commands; +#[cfg(target_os = "macos")] +use self::direct_commands::{ + DecodeHybridSplitCommandBuffers, DirectColorBatchCommandBuffers, DirectIdwtCommandBuffers, +}; +#[cfg(target_os = "macos")] +mod direct_cpu; #[cfg(all(target_os = "macos", test))] -static HT_BATCH_COEFFICIENT_COPY_BLITS: AtomicUsize = AtomicUsize::new(0); -#[cfg(all(target_os = "macos", test))] -std::thread_local! { - static PRIVATE_BUFFER_POOL_MISSES: Cell = const { Cell::new(0) }; - static HT_SIMD_PROTOTYPE_DISPATCHES: Cell = const { Cell::new(0) }; - static HT_SIMD_PROTOTYPE_ROUTE_OVERRIDE: Cell> = const { Cell::new(None) }; -} - -#[cfg(all(target_os = "macos", test))] -pub(crate) fn reset_ht_batch_coefficient_copy_blits_for_test() { - HT_BATCH_COEFFICIENT_COPY_BLITS.store(0, Ordering::Relaxed); -} - -#[cfg(all(target_os = "macos", test))] -pub(crate) fn ht_batch_coefficient_copy_blits_for_test() -> usize { - HT_BATCH_COEFFICIENT_COPY_BLITS.load(Ordering::Relaxed) -} - -#[cfg(all(target_os = "macos", test))] -pub(crate) fn reset_private_buffer_pool_misses_for_test() { - PRIVATE_BUFFER_POOL_MISSES.with(|misses| misses.set(0)); -} - -#[cfg(all(target_os = "macos", test))] -pub(crate) fn private_buffer_pool_misses_for_test() -> usize { - PRIVATE_BUFFER_POOL_MISSES.with(Cell::get) -} - +use self::direct_cpu::decode_prepared_classic_sub_band_on_cpu; +#[cfg(target_os = "macos")] +use self::direct_cpu::{ + decode_classic_inputs_on_cpu_with_plan_cache, decode_ht_inputs_on_cpu_with_plan_cache, + decode_prepared_classic_jobs_on_cpu_with_scratch, + decode_prepared_classic_jobs_on_cpu_with_scratch_profiled, + decode_prepared_classic_sub_band_group_on_cpu_profile, + decode_prepared_classic_sub_band_on_cpu_profile, decode_prepared_ht_jobs_on_cpu_with_workspace, + decode_prepared_ht_jobs_on_cpu_with_workspace_profiled, + decode_prepared_ht_sub_band_group_on_cpu_profile, decode_prepared_ht_sub_band_on_cpu_profile, + ClassicCpuDecodeInput, ClassicCpuDecodeScratch, HtCpuDecodeInput, +}; +#[cfg(target_os = "macos")] +mod direct_flattened; #[cfg(all(target_os = "macos", test))] -pub(crate) fn reset_ht_simd_prototype_dispatches_for_test() { - HT_SIMD_PROTOTYPE_DISPATCHES.with(|dispatches| dispatches.set(0)); -} +use self::direct_flattened::hybrid_cpu_decode_worker_count; +#[cfg(target_os = "macos")] +use self::direct_flattened::{ + build_flattened_cpu_tier1_cache, packed_cpu_decode_coefficients, packed_cpu_decode_output_len, + FlattenedCpuTier1Cache, +}; +mod direct_profile; +#[cfg(target_os = "macos")] +use self::direct_profile::record_completed_decode_split_gpu_stages; +use self::direct_profile::{ + elapsed_us, emit_direct_hybrid_stage_timings, CpuTier1DecodeSubstageCounters, + DirectHybridStageTimings, +}; +mod direct_plan_support; +#[cfg(test)] +use self::direct_plan_support::prepared_direct_color_tier1_input_count; +use self::direct_plan_support::{ + classic_group_shapes_match, classic_sub_band_shapes_match, direct_preflight_invariant, + ht_group_shapes_match, ht_sub_band_shapes_match, idwt_shapes_match, + prepared_direct_color_plan_supports_runtime, store_shapes_match, +}; +mod direct_scratch; +use self::direct_scratch::{ + recycle_private_buffers, recycle_scratch_buffers, recycle_shared_buffers, + take_f32_scratch_buffer, take_recyclable_private_buffer, DirectScratchBuffer, +}; +#[cfg(target_os = "macos")] +mod direct_status; +#[cfg(target_os = "macos")] +use self::direct_status::{ + decode_classic_status_error, decode_ht_status_error, decode_idwt_status_error, + decode_mct_status_error, validate_direct_status, DirectStatusCheck, +}; +#[cfg(target_os = "macos")] +mod direct_tier1; +#[cfg(target_os = "macos")] +use self::direct_tier1::{ + flattened_hybrid_cpu_tier1_enabled, prepare_direct_tier1_input_buffer, + record_flattened_hybrid_cpu_decode_batch, record_hybrid_cpu_decode_inputs, + record_hybrid_cpu_decode_worker_init, record_hybrid_repeated_output_blit, + record_hybrid_stacked_component_batch, should_flatten_hybrid_cpu_tier1_color_batch, + DirectTier1Mode, HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK, +}; +mod pack_params; +use self::pack_params::{j2k_pack_scale_arrays, j2k_scalar_pack_params, j2k_u32_param}; +mod encode_capacity; +use self::encode_capacity::{ + classic_encode_output_capacity, classic_encode_output_capacity_for_mode, + classic_encode_segment_capacity, classic_packet_output_capacity, + codestream_progression_order_code, ht_encode_output_capacity, + ht_packet_output_capacity_for_mode, lossless_codestream_assembly_capacity, + packet_tree_node_count, +}; +pub(crate) use self::encode_capacity::{ + ht_packet_output_capacity_mode_from_env, J2kClassicEncodeOutputCapacityMode, + J2kHtPacketOutputCapacityMode, +}; +mod resident_packet_plan; +use self::resident_packet_plan::{ + build_resident_batch_packet_plan, prepared_lossless_batch_tiles, ResidentBatchPacketPlan, + ResidentBatchPacketPlanParams, +}; +mod resident_types; +pub(crate) use self::resident_types::{ + J2kPendingResidentLosslessCodestream, J2kResidentBatchEncodeItem, J2kResidentEncodeStageStats, + J2kResidentLosslessCodestream, J2kResidentLosslessCodestreamBatchResult, +}; +#[cfg(target_os = "macos")] +mod gpu_timing; +#[cfg(target_os = "macos")] +use self::gpu_timing::{ + completed_command_buffer_gpu_duration, completed_command_buffers_gpu_duration, + completed_command_buffers_gpu_duration_and_elapsed_window, +}; +#[cfg(target_os = "macos")] +mod resident_stage_timing; +#[cfg(target_os = "macos")] +use self::resident_stage_timing::{ + duration_share, finish_resident_encode_split_command_buffer, + finish_resident_encode_split_command_buffer_timed, new_resident_encode_command_buffer, + record_completed_resident_encode_gpu_stages, J2kResidentEncodeGpuStage, + J2kResidentEncodeGpuStageCommandBuffer, +}; +#[cfg(target_os = "macos")] +mod shader_source; +#[cfg(target_os = "macos")] +use self::shader_source::SHADER_SOURCE; +#[cfg(test)] +mod test_counters; +#[cfg(test)] +pub(crate) use self::test_counters::{ + classic_gpu_token_pack_dispatches_for_test, + classic_split_mq_byte_gpu_token_pack_dispatches_for_test, + direct_tier1_input_buffer_prepares_for_test, flattened_hybrid_cpu_decode_batches_for_test, + ht_batch_coefficient_copy_blits_for_test, hybrid_cpu_decode_inputs_for_test, + hybrid_cpu_decode_worker_inits_for_test, hybrid_repeated_output_blits_for_test, + hybrid_stacked_component_batches_for_test, lossless_deinterleave_rct_fused_dispatches_for_test, + reset_classic_gpu_token_pack_dispatches_for_test, + reset_classic_split_mq_byte_gpu_token_pack_dispatches_for_test, + reset_direct_tier1_input_buffer_prepares_for_test, + reset_flattened_hybrid_cpu_decode_batches_for_test, + reset_ht_batch_coefficient_copy_blits_for_test, reset_hybrid_cpu_decode_inputs_for_test, + reset_hybrid_cpu_decode_worker_inits_for_test, reset_hybrid_repeated_output_blits_for_test, + reset_hybrid_stacked_component_batches_for_test, + reset_lossless_deinterleave_rct_fused_dispatches_for_test, + reset_resident_codestream_command_buffer_waits_for_test, + reset_resident_gpu_timestamp_queries_for_test, + resident_codestream_command_buffer_waits_for_test, resident_gpu_timestamp_queries_for_test, +}; -#[cfg(all(target_os = "macos", test))] -pub(crate) fn ht_simd_prototype_dispatches_for_test() -> usize { - HT_SIMD_PROTOTYPE_DISPATCHES.with(Cell::get) -} +#[cfg(target_os = "macos")] +pub(crate) use crate::profile_env::metal_profile_stages_enabled; #[cfg(all(target_os = "macos", test))] -pub(crate) struct HtSimdPrototypeRouteOverrideGuard { - previous: Option, -} +pub(crate) use crate::buffer_pool::{ + private_buffer_pool_misses_for_test, reset_private_buffer_pool_misses_for_test, + reset_shared_buffer_pool_misses_for_test, shared_buffer_pool_misses_for_test, +}; #[cfg(all(target_os = "macos", test))] -impl Drop for HtSimdPrototypeRouteOverrideGuard { - fn drop(&mut self) { - HT_SIMD_PROTOTYPE_ROUTE_OVERRIDE.with(|route| route.set(self.previous)); - } -} +pub(crate) use crate::profile_env::{ + force_classic_gpu_token_pack_route_for_test, force_metal_profile_stages_for_test, +}; -#[cfg(all(target_os = "macos", test))] -pub(crate) fn force_ht_simd_prototype_route_for_test( - enabled: bool, -) -> HtSimdPrototypeRouteOverrideGuard { - let previous = HT_SIMD_PROTOTYPE_ROUTE_OVERRIDE.with(|route| route.replace(Some(enabled))); - HtSimdPrototypeRouteOverrideGuard { previous } +#[cfg(target_os = "macos")] +thread_local! { + static DEFAULT_METAL_SESSION: RefCell>> = const { RefCell::new(None) }; + static METAL_RUNTIME_OVERRIDE: RefCell>> = const { RefCell::new(None) }; } #[cfg(target_os = "macos")] -#[derive(Default)] -struct MetalCodeBlockDecoder { - classic: MetalClassicBlockDecoder, - ht: MetalHtBlockDecoder, - idwt: MetalIdwtDecoder, - mct: MetalMctDecoder, - store: MetalStoreDecoder, +pub(crate) struct MetalRuntime { + device: Device, + queue: CommandQueue, + zero_u32_buffer: ComputePipelineState, + validate_bytes_equal: ComputePipelineState, + copy_interleaved_padded: ComputePipelineState, + lossless_deinterleave_to_planes: ComputePipelineState, + lossless_deinterleave_rct_rgb8_to_planes: ComputePipelineState, + lossless_extract_coefficients: ComputePipelineState, + pack_gray8: ComputePipelineState, + pack_rgb8: ComputePipelineState, + pack_mct_rgb8: ComputePipelineState, + pack_mct_rgb8_batched: ComputePipelineState, + pack_rgb_opaque_rgba8: ComputePipelineState, + pack_rgba8: ComputePipelineState, + pack_gray16: ComputePipelineState, + pack_rgb16: ComputePipelineState, + pack_u8_repeated_gray: ComputePipelineState, + pack_u16_repeated_gray: ComputePipelineState, + classic_cleanup_plain_batched: ComputePipelineState, + classic_cleanup_batched: ComputePipelineState, + classic_cleanup_plain_repeated_batched: ComputePipelineState, + classic_cleanup_plain_dev_repeated_batched: ComputePipelineState, + classic_cleanup_repeated_batched: ComputePipelineState, + classic_store_repeated_batched: ComputePipelineState, + idwt_interleave: ComputePipelineState, + idwt_reversible53_horizontal: ComputePipelineState, + idwt_reversible53_vertical: ComputePipelineState, + idwt_interleave_batched: ComputePipelineState, + idwt_reversible53_horizontal_batched: ComputePipelineState, + idwt_reversible53_vertical_batched: ComputePipelineState, + idwt_irreversible97_single_decomposition: ComputePipelineState, + fdwt53_horizontal: ComputePipelineState, + fdwt53_vertical: ComputePipelineState, + fdwt53_horizontal_batched: ComputePipelineState, + fdwt53_vertical_batched: ComputePipelineState, + fdwt97_lift_horizontal: ComputePipelineState, + fdwt97_lift_vertical: ComputePipelineState, + fdwt97_deinterleave_horizontal: ComputePipelineState, + fdwt97_deinterleave_vertical: ComputePipelineState, + inverse_mct: ComputePipelineState, + forward_rct: ComputePipelineState, + forward_ict: ComputePipelineState, + quantize_subband: ComputePipelineState, + store_component: ComputePipelineState, + store_component_repeated: ComputePipelineState, + store_component_repeated_gray_u8: ComputePipelineState, + store_component_repeated_gray_u16: ComputePipelineState, + store_component_repeated_gray_u8_contiguous: ComputePipelineState, + store_component_repeated_gray_u16_contiguous: ComputePipelineState, + store_component_gray_u8: ComputePipelineState, + store_component_gray_u16: ComputePipelineState, + ht_cleanup: ComputePipelineState, + ht_cleanup_batched: ComputePipelineState, + ht_cleanup_repeated_batched: ComputePipelineState, + classic_encode_code_block: ComputePipelineState, + classic_encode_code_blocks: ComputePipelineState, + classic_encode_code_blocks_32: ComputePipelineState, + classic_encode_code_blocks_bypass_32: ComputePipelineState, + classic_encode_code_blocks_bypass_u16_32: ComputePipelineState, + classic_tier1_density_bypass_u16_32: ComputePipelineState, + classic_tier1_raw_pack_bypass_u16_32: ComputePipelineState, + classic_tier1_arithmetic_pack_bypass_u16_32: ComputePipelineState, + classic_tier1_symbol_plan_bypass_u16_32: ComputePipelineState, + classic_tier1_pass_plan_bypass_u16_32: ComputePipelineState, + classic_tier1_token_emit_bypass_u16_32: ComputePipelineState, + classic_tier1_split_token_emit_bypass_u16_32: ComputePipelineState, + classic_tier1_split_mq_byte_token_emit_bypass_u16_32: ComputePipelineState, + classic_tier1_token_pack_bypass_u16_32: ComputePipelineState, + classic_tier1_split_token_pack_bypass_u16_32: ComputePipelineState, + classic_encode_code_blocks_style0: ComputePipelineState, + classic_encode_code_blocks_style0_32: ComputePipelineState, + ht_encode_code_block: ComputePipelineState, + ht_encode_code_blocks: ComputePipelineState, + packet_block_prepare_resident_classic: ComputePipelineState, + packet_block_prepare_resident_ht: ComputePipelineState, + packet_encode: ComputePipelineState, + packet_encode_batched: ComputePipelineState, + packet_encode_resident_classic_batched: ComputePipelineState, + packet_payload_copy_batched: ComputePipelineState, + lossless_codestream_assemble: ComputePipelineState, + lossless_codestream_assemble_batched: ComputePipelineState, + ht_vlc_table0: Buffer, + ht_vlc_table1: Buffer, + ht_uvlc_table0: Buffer, + ht_uvlc_table1: Buffer, + ht_vlc_encode_table0: Buffer, + ht_vlc_encode_table1: Buffer, + ht_uvlc_encode_table: Buffer, + tier1_dummy_buffer: Buffer, + buffer_pools: MetalBufferPools, } #[cfg(target_os = "macos")] -impl HtCodeBlockDecoder for MetalCodeBlockDecoder { - fn decode_j2k_sub_band( - &mut self, - job: J2kSubBandDecodeJob<'_>, - output: &mut [f32], - ) -> signinum_j2k_native::Result { - self.classic.decode_j2k_sub_band(job, output) - } - - fn decode_j2k_code_block( - &mut self, - job: signinum_j2k_native::J2kCodeBlockDecodeJob<'_>, - output: &mut [f32], - ) -> signinum_j2k_native::Result { - self.classic.decode_j2k_code_block(job, output) +impl MetalRuntime { + #[cfg(test)] + fn new() -> Result { + let device = system_default_device()?; + Self::new_with_device(&device) } - fn decode_sub_band( - &mut self, - job: HtSubBandDecodeJob<'_>, - output: &mut [f32], - ) -> signinum_j2k_native::Result { - self.ht.decode_sub_band(job, output) + pub(crate) fn new_with_device(device: &Device) -> Result { + let loader = MetalPipelineLoader::new(device, SHADER_SOURCE)?; + let pipeline = |name: &str| loader.pipeline(name); + let queue = checked_command_queue(device)?; + Ok(Self { + device: device.clone(), + queue, + zero_u32_buffer: pipeline("j2k_zero_u32_buffer")?, + validate_bytes_equal: pipeline("j2k_validate_bytes_equal")?, + copy_interleaved_padded: pipeline("j2k_copy_interleaved_padded")?, + lossless_deinterleave_to_planes: pipeline("j2k_lossless_deinterleave_to_planes")?, + lossless_deinterleave_rct_rgb8_to_planes: pipeline( + "j2k_lossless_deinterleave_rct_rgb8_to_planes", + )?, + lossless_extract_coefficients: pipeline("j2k_lossless_extract_coefficients")?, + pack_gray8: pipeline("j2k_pack_gray8")?, + pack_rgb8: pipeline("j2k_pack_rgb8")?, + pack_mct_rgb8: pipeline("j2k_pack_mct_rgb8")?, + pack_mct_rgb8_batched: pipeline("j2k_pack_mct_rgb8_batched")?, + pack_rgb_opaque_rgba8: pipeline("j2k_pack_rgb_opaque_rgba8")?, + pack_rgba8: pipeline("j2k_pack_rgba8")?, + pack_gray16: pipeline("j2k_pack_gray16")?, + pack_rgb16: pipeline("j2k_pack_rgb16")?, + pack_u8_repeated_gray: pipeline("j2k_pack_u8_repeated_gray")?, + pack_u16_repeated_gray: pipeline("j2k_pack_u16_repeated_gray")?, + classic_cleanup_plain_batched: pipeline("j2k_decode_classic_cleanup_plain_batched")?, + classic_cleanup_batched: pipeline("j2k_decode_classic_cleanup_batched")?, + classic_cleanup_plain_repeated_batched: pipeline( + "j2k_decode_classic_cleanup_plain_repeated_batched", + )?, + classic_cleanup_plain_dev_repeated_batched: pipeline( + "j2k_decode_classic_cleanup_plain_dev_repeated_batched", + )?, + classic_cleanup_repeated_batched: pipeline( + "j2k_decode_classic_cleanup_repeated_batched", + )?, + classic_store_repeated_batched: pipeline("j2k_store_classic_repeated_batched")?, + idwt_interleave: pipeline("j2k_idwt_interleave")?, + idwt_reversible53_horizontal: pipeline("j2k_idwt_reversible53_horizontal_pass")?, + idwt_reversible53_vertical: pipeline("j2k_idwt_reversible53_vertical_pass")?, + idwt_interleave_batched: pipeline("j2k_idwt_interleave_batched")?, + idwt_reversible53_horizontal_batched: pipeline( + "j2k_idwt_reversible53_horizontal_pass_batched", + )?, + idwt_reversible53_vertical_batched: pipeline( + "j2k_idwt_reversible53_vertical_pass_batched", + )?, + idwt_irreversible97_single_decomposition: pipeline( + "j2k_idwt_irreversible97_single_decomposition", + )?, + fdwt53_horizontal: pipeline("j2k_forward_dwt53_horizontal")?, + fdwt53_vertical: pipeline("j2k_forward_dwt53_vertical")?, + fdwt53_horizontal_batched: pipeline("j2k_forward_dwt53_horizontal_batched")?, + fdwt53_vertical_batched: pipeline("j2k_forward_dwt53_vertical_batched")?, + fdwt97_lift_horizontal: pipeline("j2k_forward_dwt97_lift_horizontal")?, + fdwt97_lift_vertical: pipeline("j2k_forward_dwt97_lift_vertical")?, + fdwt97_deinterleave_horizontal: pipeline("j2k_forward_dwt97_deinterleave_horizontal")?, + fdwt97_deinterleave_vertical: pipeline("j2k_forward_dwt97_deinterleave_vertical")?, + inverse_mct: pipeline("j2k_inverse_mct")?, + forward_rct: pipeline("j2k_forward_rct")?, + forward_ict: pipeline("j2k_forward_ict")?, + quantize_subband: pipeline("j2k_quantize_subband")?, + store_component: pipeline("j2k_store_component")?, + store_component_repeated: pipeline("j2k_store_component_repeated")?, + store_component_repeated_gray_u8: pipeline("j2k_store_component_repeated_gray_u8")?, + store_component_repeated_gray_u16: pipeline("j2k_store_component_repeated_gray_u16")?, + store_component_repeated_gray_u8_contiguous: pipeline( + "j2k_store_component_repeated_gray_u8_contiguous", + )?, + store_component_repeated_gray_u16_contiguous: pipeline( + "j2k_store_component_repeated_gray_u16_contiguous", + )?, + store_component_gray_u8: pipeline("j2k_store_component_gray_u8")?, + store_component_gray_u16: pipeline("j2k_store_component_gray_u16")?, + ht_cleanup: pipeline("j2k_decode_ht_cleanup")?, + ht_cleanup_batched: pipeline("j2k_decode_ht_cleanup_batched")?, + ht_cleanup_repeated_batched: pipeline("j2k_decode_ht_cleanup_repeated_batched")?, + classic_encode_code_block: pipeline("j2k_encode_classic_code_block")?, + classic_encode_code_blocks: pipeline("j2k_encode_classic_code_blocks")?, + classic_encode_code_blocks_32: pipeline("j2k_encode_classic_code_blocks_32")?, + classic_encode_code_blocks_bypass_32: pipeline( + "j2k_encode_classic_code_blocks_bypass_32", + )?, + classic_encode_code_blocks_bypass_u16_32: pipeline( + "j2k_encode_classic_code_blocks_bypass_u16_32", + )?, + classic_tier1_density_bypass_u16_32: pipeline( + "j2k_profile_classic_tier1_density_bypass_u16_32", + )?, + classic_tier1_raw_pack_bypass_u16_32: pipeline( + "j2k_profile_classic_tier1_raw_pack_bypass_u16_32", + )?, + classic_tier1_arithmetic_pack_bypass_u16_32: pipeline( + "j2k_profile_classic_tier1_arithmetic_pack_bypass_u16_32", + )?, + classic_tier1_symbol_plan_bypass_u16_32: pipeline( + "j2k_plan_classic_tier1_symbols_bypass_u16_32", + )?, + classic_tier1_pass_plan_bypass_u16_32: pipeline( + "j2k_plan_classic_tier1_passes_bypass_u16_32", + )?, + classic_tier1_token_emit_bypass_u16_32: pipeline( + "j2k_emit_classic_tier1_tokens_bypass_u16_32", + )?, + classic_tier1_split_token_emit_bypass_u16_32: pipeline( + "j2k_emit_classic_tier1_split_tokens_bypass_u16_32", + )?, + classic_tier1_split_mq_byte_token_emit_bypass_u16_32: pipeline( + "j2k_emit_classic_tier1_split_mq_byte_raw_tokens_bypass_u16_32", + )?, + classic_tier1_token_pack_bypass_u16_32: pipeline( + "j2k_pack_classic_tier1_tokens_bypass_u16_32", + )?, + classic_tier1_split_token_pack_bypass_u16_32: pipeline( + "j2k_pack_classic_tier1_split_tokens_bypass_u16_32", + )?, + classic_encode_code_blocks_style0: pipeline("j2k_encode_classic_code_blocks_style0")?, + classic_encode_code_blocks_style0_32: pipeline( + "j2k_encode_classic_code_blocks_style0_32", + )?, + ht_encode_code_block: pipeline("j2k_encode_ht_code_block")?, + ht_encode_code_blocks: pipeline("j2k_encode_ht_code_blocks")?, + packet_block_prepare_resident_classic: pipeline( + "j2k_prepare_packet_blocks_from_classic_status", + )?, + packet_block_prepare_resident_ht: pipeline("j2k_prepare_packet_blocks_from_ht_status")?, + packet_encode: pipeline("j2k_encode_packetization")?, + packet_encode_batched: pipeline("j2k_encode_packetization_batched")?, + packet_encode_resident_classic_batched: pipeline( + "j2k_encode_packetization_resident_classic_batched", + )?, + packet_payload_copy_batched: pipeline("j2k_copy_packet_payload_batched")?, + lossless_codestream_assemble: pipeline("j2k_assemble_lossless_classic_codestream")?, + lossless_codestream_assemble_batched: pipeline( + "j2k_assemble_lossless_codestream_batched", + )?, + ht_vlc_table0: device.new_buffer_with_data( + ht_vlc_table0().as_ptr().cast(), + size_of_val(ht_vlc_table0()) as u64, + MTLResourceOptions::StorageModeShared, + ), + ht_vlc_table1: device.new_buffer_with_data( + ht_vlc_table1().as_ptr().cast(), + size_of_val(ht_vlc_table1()) as u64, + MTLResourceOptions::StorageModeShared, + ), + ht_uvlc_table0: device.new_buffer_with_data( + ht_uvlc_table0().as_ptr().cast(), + size_of_val(ht_uvlc_table0()) as u64, + MTLResourceOptions::StorageModeShared, + ), + ht_uvlc_table1: device.new_buffer_with_data( + ht_uvlc_table1().as_ptr().cast(), + size_of_val(ht_uvlc_table1()) as u64, + MTLResourceOptions::StorageModeShared, + ), + ht_vlc_encode_table0: device.new_buffer_with_data( + ht_vlc_encode_table0().as_ptr().cast(), + size_of_val(ht_vlc_encode_table0()) as u64, + MTLResourceOptions::StorageModeShared, + ), + ht_vlc_encode_table1: device.new_buffer_with_data( + ht_vlc_encode_table1().as_ptr().cast(), + size_of_val(ht_vlc_encode_table1()) as u64, + MTLResourceOptions::StorageModeShared, + ), + ht_uvlc_encode_table: device.new_buffer_with_data( + ht_uvlc_encode_table().as_ptr().cast(), + size_of_val(ht_uvlc_encode_table()) as u64, + MTLResourceOptions::StorageModeShared, + ), + tier1_dummy_buffer: device.new_buffer(1, MTLResourceOptions::StorageModeShared), + buffer_pools: MetalBufferPools::new(), + }) } - fn decode_code_block( - &mut self, - job: HtCodeBlockDecodeJob<'_>, - output: &mut [f32], - ) -> signinum_j2k_native::Result<()> { - self.ht.decode_code_block(job, output) + fn take_private_buffer(&self, bytes: usize) -> Result { + self.buffer_pools.take_private(&self.device, bytes) } - fn decode_single_decomposition_idwt( - &mut self, - job: J2kSingleDecompositionIdwtJob<'_>, - output: &mut [f32], - ) -> signinum_j2k_native::Result { - self.idwt.decode_single_decomposition_idwt(job, output) + fn recycle_private_buffer(&self, bytes: usize, buffer: Buffer) -> Result<(), Error> { + self.buffer_pools.recycle_private(bytes, buffer) } - fn decode_inverse_mct( - &mut self, - job: J2kInverseMctJob<'_>, - ) -> signinum_j2k_native::Result { - self.mct.decode_inverse_mct(job) + fn take_shared_buffer(&self, bytes: usize) -> Result { + self.buffer_pools.take_shared(&self.device, bytes) } - fn decode_store_component( - &mut self, - job: J2kStoreComponentJob<'_>, - ) -> signinum_j2k_native::Result { - self.store.decode_store_component(job) + fn recycle_shared_buffer(&self, bytes: usize, buffer: Buffer) -> Result<(), Error> { + self.buffer_pools.recycle_shared(bytes, buffer) } } #[cfg(target_os = "macos")] -const SHADER_SOURCE: &str = concat!( - r#" -#include -using namespace metal; - -kernel void j2k_zero_u32_buffer( - device uint *buffer [[buffer(0)]], - constant uint &word_count [[buffer(1)]], - uint gid [[thread_position_in_grid]] -) { - if (gid >= word_count) { - return; +fn with_runtime(f: impl FnOnce(&MetalRuntime) -> Result) -> Result { + let override_runtime = METAL_RUNTIME_OVERRIDE.with(|slot| slot.borrow().clone()); + if let Some(runtime) = override_runtime { + return f(&runtime); } - buffer[gid] = 0u; + DEFAULT_METAL_SESSION.with(|session| { + let mut session = session.borrow_mut(); + if session.is_none() { + *session = Some( + j2k_metal_support::system_default_device().map(crate::MetalBackendSession::new), + ); + } + let Some(session) = session.as_ref() else { + return Err(Error::MetalRuntime { + message: "J2K Metal default session was not initialized".to_string(), + }); + }; + match session { + Ok(session) => with_runtime_for_session(session, f), + Err(error) => Err(runtime_initialization_error(error)), + } + }) } -struct J2kValidateBytesParams { - uint byte_len; -}; +#[cfg(target_os = "macos")] +pub(crate) fn runtime_initialization_error(error: &MetalSupportError) -> Error { + if error.is_unavailable() { + Error::MetalUnavailable + } else { + Error::MetalRuntime { + message: error.to_string(), + } + } +} -struct J2kValidateBytesStatus { - uint code; - uint index; - uint expected; - uint actual; -}; - -kernel void j2k_validate_bytes_equal( - device const uchar *actual [[buffer(0)]], - device const uchar *expected [[buffer(1)]], - device J2kValidateBytesStatus *status [[buffer(2)]], - constant J2kValidateBytesParams ¶ms [[buffer(3)]], - uint gid [[thread_position_in_grid]] -) { - if (gid != 0u) { - return; - } - - status[0].code = 0u; - status[0].index = 0u; - status[0].expected = 0u; - status[0].actual = 0u; - - for (uint i = 0u; i < params.byte_len; ++i) { - const uchar actual_byte = actual[i]; - const uchar expected_byte = expected[i]; - if (actual_byte != expected_byte) { - status[0].code = 1u; - status[0].index = i; - status[0].expected = uint(expected_byte); - status[0].actual = uint(actual_byte); - return; - } - } -} - -struct J2kCopyInterleavedParams { - uint src_width; - uint src_height; - uint src_stride; - uint dst_width; - uint dst_height; - uint dst_stride; - uint bytes_per_pixel; -}; - -kernel void j2k_copy_interleaved_padded( - device const uchar *src [[buffer(0)]], - device uchar *dst [[buffer(1)]], - constant J2kCopyInterleavedParams ¶ms [[buffer(2)]], - uint2 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.dst_width || gid.y >= params.dst_height) { - return; - } - - const uint dst_idx = gid.y * params.dst_stride + gid.x * params.bytes_per_pixel; - const bool inside_src = gid.x < params.src_width && gid.y < params.src_height; - const uint src_idx = gid.y * params.src_stride + gid.x * params.bytes_per_pixel; - for (uint byte_idx = 0u; byte_idx < params.bytes_per_pixel; ++byte_idx) { - dst[dst_idx + byte_idx] = inside_src ? src[src_idx + byte_idx] : uchar(0); - } -} - -struct J2kLosslessDeinterleaveParams { - uint src_width; - uint src_height; - uint src_stride; - uint dst_width; - uint dst_height; - uint components; - uint bytes_per_sample; - uint sample_offset; -}; - -inline float j2k_lossless_load_sample( - device const uchar *src, - uint base, - uint component, - uint components, - uint bytes_per_sample, - uint sample_offset, - bool inside_src -) { - if (!inside_src) { - return -float(int(sample_offset)); - } - if (bytes_per_sample == 1u) { - return float(int(src[base + component]) - int(sample_offset)); - } - const uint byte_offset = base + component * 2u; - const uint raw = uint(src[byte_offset]) | (uint(src[byte_offset + 1u]) << 8u); - return float(int(raw) - int(sample_offset)); -} - -kernel void j2k_lossless_deinterleave_to_planes( - device const uchar *src [[buffer(0)]], - device float *plane0 [[buffer(1)]], - device float *plane1 [[buffer(2)]], - device float *plane2 [[buffer(3)]], - constant J2kLosslessDeinterleaveParams ¶ms [[buffer(4)]], - uint2 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.dst_width || gid.y >= params.dst_height) { - return; - } - - const bool inside_src = gid.x < params.src_width && gid.y < params.src_height; - const uint src_base = gid.y * params.src_stride - + gid.x * params.components * params.bytes_per_sample; - const uint dst_idx = gid.y * params.dst_width + gid.x; - plane0[dst_idx] = j2k_lossless_load_sample( - src, - src_base, - 0u, - params.components, - params.bytes_per_sample, - params.sample_offset, - inside_src - ); - if (params.components >= 3u) { - plane1[dst_idx] = j2k_lossless_load_sample( - src, - src_base, - 1u, - params.components, - params.bytes_per_sample, - params.sample_offset, - inside_src - ); - plane2[dst_idx] = j2k_lossless_load_sample( - src, - src_base, - 2u, - params.components, - params.bytes_per_sample, - params.sample_offset, - inside_src - ); - } -} - -struct J2kLosslessCoefficientJob { - uint coefficient_offset; - uint component; - uint subband_x; - uint subband_y; - uint block_x; - uint block_y; - uint block_width; - uint block_height; - uint full_width; -}; - -kernel void j2k_lossless_extract_coefficients( - device const float *plane0 [[buffer(0)]], - device const float *plane1 [[buffer(1)]], - device const float *plane2 [[buffer(2)]], - device int *coefficients [[buffer(3)]], - constant J2kLosslessCoefficientJob *jobs [[buffer(4)]], - constant uint &job_count [[buffer(5)]], - uint3 gid [[thread_position_in_grid]] -) { - if (gid.z >= job_count) { - return; - } - constant J2kLosslessCoefficientJob &job = jobs[gid.z]; - if (gid.x >= job.block_width || gid.y >= job.block_height) { - return; - } - - device const float *plane = plane0; - if (job.component == 1u) { - plane = plane1; - } else if (job.component == 2u) { - plane = plane2; - } - const uint src_x = job.subband_x + job.block_x + gid.x; - const uint src_y = job.subband_y + job.block_y + gid.y; - const uint src_idx = src_y * job.full_width + src_x; - const uint dst_idx = job.coefficient_offset + gid.y * job.block_width + gid.x; - coefficients[dst_idx] = int(round(plane[src_idx])); -} - -struct J2kPackParams { - uint width; - uint height; - uint out_stride; - uint output_channels; - uint opaque_alpha; - float max_values[4]; - float u8_scales[4]; - float u16_scales[4]; -}; - -struct J2kMctRgb8PackParams { - uint width; - uint height; - uint out_stride; - uint transform; - float addends[3]; - float max_values[3]; - float u8_scales[3]; -}; - -struct J2kBatchedMctRgb8PackParams { - uint width; - uint height; - uint out_stride; - uint transform; - uint batch_count; - uint plane_stride; - uint output_stride; - float addends[3]; - float max_values[3]; - float u8_scales[3]; -}; - -inline uchar scale_to_u8(float sample, float max_value, float scale) { - const float clamped = clamp(sample, 0.0f, max_value); - return uchar(min(floor(clamped * scale + 0.5f), 255.0f)); -} - -inline ushort pack_to_u16(float sample, float max_value, float scale) { - const float clamped = clamp(sample, 0.0f, max_value); - return ushort(min(floor(clamped * scale + 0.5f), 65535.0f)); -} - -kernel void j2k_pack_gray8( - device const float *plane0 [[buffer(0)]], - device const float *plane1 [[buffer(1)]], - device const float *plane2 [[buffer(2)]], - device const float *plane3 [[buffer(3)]], - device uchar *out [[buffer(4)]], - constant J2kPackParams ¶ms [[buffer(5)]], - uint2 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.width || gid.y >= params.height) { - return; - } - - const uint idx = gid.y * params.width + gid.x; - const uint out_idx = gid.y * params.out_stride + gid.x; - out[out_idx] = scale_to_u8(plane0[idx], params.max_values[0], params.u8_scales[0]); -} - -kernel void j2k_pack_rgb8( - device const float *plane0 [[buffer(0)]], - device const float *plane1 [[buffer(1)]], - device const float *plane2 [[buffer(2)]], - device const float *plane3 [[buffer(3)]], - device uchar *out [[buffer(4)]], - constant J2kPackParams ¶ms [[buffer(5)]], - uint2 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.width || gid.y >= params.height) { - return; - } - - const uint idx = gid.y * params.width + gid.x; - const uint out_idx = gid.y * params.out_stride + gid.x * 3u; - out[out_idx] = scale_to_u8(plane0[idx], params.max_values[0], params.u8_scales[0]); - out[out_idx + 1] = scale_to_u8(plane1[idx], params.max_values[1], params.u8_scales[1]); - out[out_idx + 2] = scale_to_u8(plane2[idx], params.max_values[2], params.u8_scales[2]); -} - -kernel void j2k_pack_mct_rgb8( - device const float *plane0 [[buffer(0)]], - device const float *plane1 [[buffer(1)]], - device const float *plane2 [[buffer(2)]], - device uchar *out [[buffer(3)]], - constant J2kMctRgb8PackParams ¶ms [[buffer(4)]], - uint2 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.width || gid.y >= params.height) { - return; - } - - const uint idx = gid.y * params.width + gid.x; - const float y0 = plane0[idx]; - const float y1 = plane1[idx]; - const float y2 = plane2[idx]; - float rgb0; - float rgb1; - float rgb2; - - if (params.transform == 0u) { - const float i1 = y0 - floor((y2 + y1) * 0.25f); - rgb0 = y2 + i1 + params.addends[0]; - rgb1 = i1 + params.addends[1]; - rgb2 = y1 + i1 + params.addends[2]; - } else { - rgb0 = y2 * 1.402f + y0 + params.addends[0]; - rgb1 = y2 * -0.71414f + y1 * -0.34413f + y0 + params.addends[1]; - rgb2 = y1 * 1.772f + y0 + params.addends[2]; - } - - const uint out_idx = gid.y * params.out_stride + gid.x * 3u; - out[out_idx] = scale_to_u8(rgb0, params.max_values[0], params.u8_scales[0]); - out[out_idx + 1] = scale_to_u8(rgb1, params.max_values[1], params.u8_scales[1]); - out[out_idx + 2] = scale_to_u8(rgb2, params.max_values[2], params.u8_scales[2]); -} - -kernel void j2k_pack_mct_rgb8_batched( - device const float *plane0 [[buffer(0)]], - device const float *plane1 [[buffer(1)]], - device const float *plane2 [[buffer(2)]], - device uchar *out [[buffer(3)]], - constant J2kBatchedMctRgb8PackParams ¶ms [[buffer(4)]], - uint3 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.width || gid.y >= params.height || gid.z >= params.batch_count) { - return; - } - - const uint plane_base = gid.z * params.plane_stride; - const uint idx = plane_base + gid.y * params.width + gid.x; - const float y0 = plane0[idx]; - const float y1 = plane1[idx]; - const float y2 = plane2[idx]; - float rgb0; - float rgb1; - float rgb2; - - if (params.transform == 0u) { - const float i1 = y0 - floor((y2 + y1) * 0.25f); - rgb0 = y2 + i1 + params.addends[0]; - rgb1 = i1 + params.addends[1]; - rgb2 = y1 + i1 + params.addends[2]; - } else { - rgb0 = y2 * 1.402f + y0 + params.addends[0]; - rgb1 = y2 * -0.71414f + y1 * -0.34413f + y0 + params.addends[1]; - rgb2 = y1 * 1.772f + y0 + params.addends[2]; - } - - const uint out_idx = gid.z * params.output_stride + gid.y * params.out_stride + gid.x * 3u; - out[out_idx] = scale_to_u8(rgb0, params.max_values[0], params.u8_scales[0]); - out[out_idx + 1] = scale_to_u8(rgb1, params.max_values[1], params.u8_scales[1]); - out[out_idx + 2] = scale_to_u8(rgb2, params.max_values[2], params.u8_scales[2]); -} - -kernel void j2k_pack_rgb_opaque_rgba8( - device const float *plane0 [[buffer(0)]], - device const float *plane1 [[buffer(1)]], - device const float *plane2 [[buffer(2)]], - device const float *plane3 [[buffer(3)]], - device uchar *out [[buffer(4)]], - constant J2kPackParams ¶ms [[buffer(5)]], - uint2 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.width || gid.y >= params.height) { - return; - } - - const uint idx = gid.y * params.width + gid.x; - const uint out_idx = gid.y * params.out_stride + gid.x * 4u; - out[out_idx] = scale_to_u8(plane0[idx], params.max_values[0], params.u8_scales[0]); - out[out_idx + 1] = scale_to_u8(plane1[idx], params.max_values[1], params.u8_scales[1]); - out[out_idx + 2] = scale_to_u8(plane2[idx], params.max_values[2], params.u8_scales[2]); - out[out_idx + 3] = uchar(255); -} - -kernel void j2k_pack_rgba8( - device const float *plane0 [[buffer(0)]], - device const float *plane1 [[buffer(1)]], - device const float *plane2 [[buffer(2)]], - device const float *plane3 [[buffer(3)]], - device uchar *out [[buffer(4)]], - constant J2kPackParams ¶ms [[buffer(5)]], - uint2 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.width || gid.y >= params.height) { - return; - } - - const uint idx = gid.y * params.width + gid.x; - const uint out_idx = gid.y * params.out_stride + gid.x * 4u; - out[out_idx] = scale_to_u8(plane0[idx], params.max_values[0], params.u8_scales[0]); - out[out_idx + 1] = scale_to_u8(plane1[idx], params.max_values[1], params.u8_scales[1]); - out[out_idx + 2] = scale_to_u8(plane2[idx], params.max_values[2], params.u8_scales[2]); - out[out_idx + 3] = scale_to_u8(plane3[idx], params.max_values[3], params.u8_scales[3]); +#[cfg(target_os = "macos")] +struct RuntimeOverrideGuard { + previous: Option>, } -kernel void j2k_pack_gray16( - device const float *plane0 [[buffer(0)]], - device const float *plane1 [[buffer(1)]], - device const float *plane2 [[buffer(2)]], - device const float *plane3 [[buffer(3)]], - device ushort *out [[buffer(4)]], - constant J2kPackParams ¶ms [[buffer(5)]], - uint2 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.width || gid.y >= params.height) { - return; +#[cfg(target_os = "macos")] +impl Drop for RuntimeOverrideGuard { + fn drop(&mut self) { + let previous = self.previous.take(); + METAL_RUNTIME_OVERRIDE.with(|slot| { + slot.replace(previous); + }); } - - const uint idx = gid.y * params.width + gid.x; - const uint out_idx = (gid.y * params.out_stride) / 2u + gid.x; - out[out_idx] = pack_to_u16(plane0[idx], params.max_values[0], params.u16_scales[0]); } -kernel void j2k_pack_rgb16( - device const float *plane0 [[buffer(0)]], - device const float *plane1 [[buffer(1)]], - device const float *plane2 [[buffer(2)]], - device const float *plane3 [[buffer(3)]], - device ushort *out [[buffer(4)]], - constant J2kPackParams ¶ms [[buffer(5)]], - uint2 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.width || gid.y >= params.height) { - return; - } - - const uint idx = gid.y * params.width + gid.x; - const uint out_idx = (gid.y * params.out_stride) / 2u + gid.x * 3u; - out[out_idx] = pack_to_u16(plane0[idx], params.max_values[0], params.u16_scales[0]); - out[out_idx + 1] = pack_to_u16(plane1[idx], params.max_values[1], params.u16_scales[1]); - out[out_idx + 2] = pack_to_u16(plane2[idx], params.max_values[2], params.u16_scales[2]); +#[cfg(target_os = "macos")] +pub(crate) fn with_runtime_for_session( + session: &crate::MetalBackendSession, + f: impl FnOnce(&MetalRuntime) -> Result, +) -> Result { + let runtime = session.runtime()?; + let previous = METAL_RUNTIME_OVERRIDE.with(|slot| slot.replace(Some(runtime.clone()))); + let _guard = RuntimeOverrideGuard { previous }; + f(&runtime) } -struct J2kRepeatedGrayPackParams { - uint width; - uint height; - uint out_stride; - uint batch_count; - float max_value; - float u8_scale; - float u16_scale; -}; - -kernel void j2k_pack_u8_repeated_gray( - device const float *plane0 [[buffer(0)]], - device uchar *out [[buffer(1)]], - constant J2kRepeatedGrayPackParams ¶ms [[buffer(2)]], - uint3 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.width || gid.y >= params.height || gid.z >= params.batch_count) { - return; +#[cfg(target_os = "macos")] +fn with_runtime_for_device( + device: &Device, + f: impl FnOnce(&MetalRuntime) -> Result, +) -> Result { + let override_runtime = METAL_RUNTIME_OVERRIDE.with(|slot| slot.borrow().clone()); + if let Some(runtime) = override_runtime { + if runtime.device.as_ptr() == device.as_ptr() { + return f(&runtime); + } } - const uint plane_base = gid.z * params.width * params.height; - const uint out_base = gid.z * params.out_stride * params.height; - const uint plane_idx = plane_base + gid.y * params.width + gid.x; - const uint out_idx = out_base + gid.y * params.out_stride + gid.x; - out[out_idx] = scale_to_u8(plane0[plane_idx], params.max_value, params.u8_scale); + let session = crate::MetalBackendSession::new(device.clone()); + with_runtime_for_session(&session, f) } -kernel void j2k_pack_u16_repeated_gray( - device const float *plane0 [[buffer(0)]], - device ushort *out [[buffer(1)]], - constant J2kRepeatedGrayPackParams ¶ms [[buffer(2)]], - uint3 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.width || gid.y >= params.height || gid.z >= params.batch_count) { - return; - } - - const uint plane_base = gid.z * params.width * params.height; - const uint out_base = (gid.z * params.out_stride * params.height) / 2u; - const uint plane_idx = plane_base + gid.y * params.width + gid.x; - const uint out_idx = out_base + gid.y * (params.out_stride / 2u) + gid.x; - out[out_idx] = pack_to_u16(plane0[plane_idx], params.max_value, params.u16_scale); -} -"#, - "\n", - include_str!("classic.metal"), - "\n", - include_str!("encode_bitstream.metal"), - "\n", - include_str!("idwt.metal"), - "\n", - include_str!("fdwt.metal"), - "\n", - include_str!("mct.metal"), - "\n", - include_str!("store.metal"), - "\n", - include_str!("ht_cleanup.metal"), -); - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kValidateBytesParams { - byte_len: u32, +#[cfg(all(target_os = "macos", test))] +pub(crate) fn with_isolated_runtime_for_device_for_test( + device: &Device, + f: impl FnOnce() -> Result, +) -> Result { + let runtime = Arc::new( + MetalRuntime::new_with_device(device) + .map_err(|error| runtime_initialization_error(&error))?, + ); + let previous = METAL_RUNTIME_OVERRIDE.with(|slot| slot.replace(Some(runtime))); + let _guard = RuntimeOverrideGuard { previous }; + f() } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kValidateBytesStatus { - code: u32, - index: u32, - expected: u32, - actual: u32, +#[derive(Clone)] +pub(crate) struct PreparedDirectGrayscalePlan { + dimensions: (u32, u32), + bit_depth: u8, + tier1_prepare_mode: DirectTier1Mode, + steps: Vec, + classic_groups: Vec, + ht_groups: Vec, + cpu_tier1_cache: Arc, } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kCopyInterleavedParams { - src_width: u32, - src_height: u32, - src_stride: u32, - dst_width: u32, - dst_height: u32, - dst_stride: u32, - bytes_per_pixel: u32, +#[derive(Clone)] +pub(crate) struct PreparedDirectColorPlan { + dimensions: (u32, u32), + bit_depths: [u8; 3], + mct: bool, + transform: J2kWaveletTransform, + component_plans: Vec, } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kLosslessDeinterleaveParams { - src_width: u32, - src_height: u32, - src_stride: u32, - dst_width: u32, - dst_height: u32, - components: u32, - bytes_per_sample: u32, - sample_offset: u32, +#[derive(Clone)] +enum PreparedDirectGrayscaleStep { + ClassicSubBand(PreparedClassicSubBand), + HtSubBand(PreparedHtSubBand), + Idwt(PreparedDirectIdwt), + Store(J2kDirectStoreStep), } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kLosslessCoefficientJob { - coefficient_offset: u32, - component: u32, - subband_x: u32, - subband_y: u32, - block_x: u32, - block_y: u32, - block_width: u32, - block_height: u32, - full_width: u32, +#[derive(Clone)] +struct PreparedDirectIdwt { + step: J2kDirectIdwtStep, + output_window: BandRequiredRegion, } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kPackParams { +#[derive(Clone)] +struct PreparedClassicSubBand { + band_id: J2kDirectBandId, width: u32, height: u32, - out_stride: u32, - output_channels: u32, - opaque_alpha: u32, - max_values: [f32; 4], - u8_scales: [f32; 4], - u16_scales: [f32; 4], + zero_fill: bool, + coded_data: Vec, + coded_buffer: Buffer, + jobs: Vec, + jobs_buffer: Buffer, + segments: Vec, + segments_buffer: Buffer, } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kMctRgb8PackParams { - width: u32, - height: u32, - out_stride: u32, - transform: u32, - addends: [f32; 3], - max_values: [f32; 3], - u8_scales: [f32; 3], +#[derive(Clone)] +struct PreparedClassicSubBandGroup { + start_step: usize, + end_step: usize, + total_coefficients: usize, + zero_fill: bool, + coded_data: Vec, + coded_buffer: Buffer, + jobs: Vec, + jobs_buffer: Buffer, + segments: Vec, + segments_buffer: Buffer, + members: Vec, } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kBatchedMctRgb8PackParams { - width: u32, - height: u32, - out_stride: u32, - transform: u32, - batch_count: u32, - plane_stride: u32, - output_stride: u32, - addends: [f32; 3], - max_values: [f32; 3], - u8_scales: [f32; 3], +#[derive(Clone)] +struct PreparedClassicSubBandGroupMember { + band_id: J2kDirectBandId, + offset_elements: usize, + window: BandRequiredRegion, } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kRepeatedGrayPackParams { +#[derive(Clone)] +struct PreparedHtSubBand { + band_id: J2kDirectBandId, width: u32, height: u32, - out_stride: u32, - batch_count: u32, - max_value: f32, - u8_scale: f32, - u16_scale: f32, + coded_data: Vec, + coded_buffer: Option, + jobs: Vec, + jobs_buffer: Option, } #[cfg(target_os = "macos")] -#[derive(Clone, Copy)] -struct J2kScalarPackParams { - max_value: f32, - u8_scale: f32, - u16_scale: f32, +#[derive(Clone)] +struct HtCodedArena { + data: Vec, + buffer: Buffer, } #[cfg(target_os = "macos")] -fn j2k_scalar_pack_params(bit_depth: u32) -> J2kScalarPackParams { - let clamped = bit_depth.min(16); - let max_value_u16 = u16::try_from(((1u32 << clamped) - 1).max(1)) - .expect("clamped J2K bit depth max fits in u16"); - let max_value = f32::from(max_value_u16); - let u8_scale = 255.0 / max_value; - let u16_scale = if bit_depth <= 8 { - 65_535.0 / max_value - } else { - 1.0 - }; - J2kScalarPackParams { - max_value, - u8_scale, - u16_scale, - } +#[derive(Clone)] +struct PreparedHtSubBandGroup { + start_step: usize, + end_step: usize, + total_coefficients: usize, + coded_arena: HtCodedArena, + jobs: Vec, + jobs_buffer: Buffer, + members: Vec, } #[cfg(target_os = "macos")] -fn j2k_pack_scale_arrays(bit_depths: [u32; 4]) -> ([f32; 4], [f32; 4], [f32; 4]) { - let mut max_values = [1.0f32; 4]; - let mut u8_scales = [255.0f32; 4]; - let mut u16_scales = [65_535.0f32; 4]; - for (index, bit_depth) in bit_depths.into_iter().enumerate() { - let params = j2k_scalar_pack_params(bit_depth); - max_values[index] = params.max_value; - u8_scales[index] = params.u8_scale; - u16_scales[index] = params.u16_scale; - } - (max_values, u8_scales, u16_scales) +#[derive(Clone)] +struct PreparedHtSubBandGroupMember { + band_id: J2kDirectBandId, + offset_elements: usize, + window: BandRequiredRegion, } #[cfg(target_os = "macos")] -const J2K_CLASSIC_STATUS_OK: u32 = 0; -#[cfg(target_os = "macos")] -const J2K_CLASSIC_STATUS_FAIL: u32 = 1; -#[cfg(target_os = "macos")] -const J2K_CLASSIC_STATUS_UNSUPPORTED: u32 = 2; -#[cfg(target_os = "macos")] -const J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES: u32 = 1 << 0; -#[cfg(target_os = "macos")] -const J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS: u32 = 1 << 1; -#[cfg(target_os = "macos")] -const J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT: u32 = 1 << 2; -#[cfg(target_os = "macos")] -const J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS: u32 = 1 << 3; -#[cfg(target_os = "macos")] -const J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS: u32 = 1 << 4; -#[cfg(target_os = "macos")] -const J2K_CLASSIC_MAX_WIDTH: u32 = 64; -#[cfg(target_os = "macos")] -const J2K_CLASSIC_MAX_HEIGHT: u32 = 64; -#[cfg(target_os = "macos")] -const J2K_CLASSIC_MAX_COEFF_COUNT: usize = - (J2K_CLASSIC_MAX_WIDTH as usize + 2) * (J2K_CLASSIC_MAX_HEIGHT as usize + 2); - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kClassicCleanupBatchJob { - coded_offset: u32, - coded_len: u32, - segment_offset: u32, - segment_count: u32, - width: u32, - height: u32, - output_stride: u32, - output_offset: u32, - missing_msbs: u32, - total_bitplanes: u32, - number_of_coding_passes: u32, - sub_band_type: u32, - style_flags: u32, - strict: u32, - dequantization_step: f32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kClassicSegment { - data_offset: u32, - data_length: u32, - start_coding_pass: u32, - end_coding_pass: u32, - use_arithmetic: u32, +struct PlaneStage { + dims: (u32, u32), + plane_count: usize, + color_space: NativeColorSpace, + has_alpha: bool, + bit_depths: [u32; 4], + planes: [Option; 4], } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kClassicStatus { - code: u32, - detail: u32, - reserved0: u32, - reserved1: u32, -} +impl PlaneStage { + fn from_planes( + device: &Device, + decoded: &NativeDecodedComponents<'_>, + roi: Option, + ) -> Result { + let full_dims = decoded.dimensions(); + let roi = roi.unwrap_or(Rect { + x: 0, + y: 0, + w: full_dims.0, + h: full_dims.1, + }); + let dims = (roi.w, roi.h); + let plane_count = decoded.planes().len(); + if plane_count == 0 || plane_count > 4 { + return Err(Error::MetalKernel { + message: format!("unsupported J2K plane count {plane_count}"), + }); + } -#[cfg(target_os = "macos")] -const J2K_IDWT_STATUS_OK: u32 = 0; -#[cfg(target_os = "macos")] -const J2K_IDWT_STATUS_FAIL: u32 = 1; + let mut bit_depths = [0u32; 4]; + let mut planes: [Option; 4] = [None, None, None, None]; + for (index, plane) in decoded.planes().iter().enumerate() { + bit_depths[index] = u32::from(plane.bit_depth()); + let len = dims.0 as usize * dims.1 as usize; + let buffer = device.new_buffer( + (len * size_of::()) as u64, + MTLResourceOptions::StorageModeShared, + ); + copy_plane_samples(&buffer, plane.samples(), full_dims.0 as usize, roi); + planes[index] = Some(buffer); + } -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kIdwtSingleDecompositionParams { - x0: u32, - y0: u32, - output_x: u32, - output_y: u32, - width: u32, - height: u32, - ll_x: u32, - ll_y: u32, - ll_width: u32, - ll_height: u32, - hl_x: u32, - hl_y: u32, - hl_width: u32, - hl_height: u32, - lh_x: u32, - lh_y: u32, - lh_width: u32, - lh_height: u32, - hh_x: u32, - hh_y: u32, - hh_width: u32, - hh_height: u32, -} + Ok(Self { + dims, + plane_count, + color_space: decoded.color_space().clone(), + has_alpha: decoded.has_alpha(), + bit_depths, + planes, + }) + } -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kRepeatedIdwtSingleDecompositionParams { - x0: u32, - y0: u32, - output_x: u32, - output_y: u32, - width: u32, - height: u32, - ll_x: u32, - ll_y: u32, - ll_width: u32, - ll_height: u32, - hl_x: u32, - hl_y: u32, - hl_width: u32, - hl_height: u32, - lh_x: u32, - lh_y: u32, - lh_width: u32, - lh_height: u32, - hh_x: u32, - hh_y: u32, - hh_width: u32, - hh_height: u32, - ll_instance_stride: u32, - hl_instance_stride: u32, - lh_instance_stride: u32, - hh_instance_stride: u32, - batch_count: u32, -} + fn from_captured_planes( + decoded: &NativeDecodedComponents<'_>, + captured_planes: Vec, + ) -> Option { + let plane_count = decoded.planes().len(); + let supported_shape = matches!( + (decoded.color_space(), decoded.has_alpha(), plane_count), + (NativeColorSpace::Gray, false, 1) | (NativeColorSpace::RGB, false, 3) + ); + if !supported_shape { + return None; + } + if captured_planes.len() != plane_count || plane_count == 0 || plane_count > 4 { + return None; + } -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kIdwtStatus { - code: u32, - detail: u32, - reserved0: u32, - reserved1: u32, -} + let mut bit_depths = [0u32; 4]; + let mut planes: [Option; 4] = [None, None, None, None]; + for (index, (plane, buffer)) in decoded.planes().iter().zip(captured_planes).enumerate() { + bit_depths[index] = u32::from(plane.bit_depth()); + planes[index] = Some(buffer); + } -#[cfg(target_os = "macos")] -const J2K_MCT_STATUS_OK: u32 = 0; -#[cfg(target_os = "macos")] -const J2K_MCT_STATUS_FAIL: u32 = 1; + Some(Self { + dims: decoded.dimensions(), + plane_count, + color_space: decoded.color_space().clone(), + has_alpha: decoded.has_alpha(), + bit_depths, + planes, + }) + } -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kInverseMctParams { - _len: u32, - _transform: u32, - _addend0: f32, - _addend1: f32, - _addend2: f32, + fn finish_with_runtime( + self, + runtime: &MetalRuntime, + fmt: PixelFormat, + ) -> Result { + let command_buffer = runtime.queue.new_command_buffer(); + let surface = + encode_plane_stage_to_surface_in_command_buffer(runtime, command_buffer, &self, fmt)?; + command_buffer.commit(); + command_buffer.wait_until_completed(); + Ok(surface) + } } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kForwardRctParams { - _len: u32, - _reserved0: u32, - _reserved1: u32, - _reserved2: u32, -} +fn encode_plane_stage_to_surface_in_command_buffer( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + stage: &PlaneStage, + fmt: PixelFormat, +) -> Result { + let pitch_bytes = stage.dims.0 as usize * fmt.bytes_per_pixel(); + let out_buffer = runtime.device.new_buffer( + (pitch_bytes * stage.dims.1 as usize) as u64, + MTLResourceOptions::StorageModeShared, + ); + let (output_channels, opaque_alpha, pipeline) = output_shape_for( + &stage.color_space, + stage.has_alpha, + stage.plane_count, + fmt, + runtime, + )?; + let (max_values, u8_scales, u16_scales) = j2k_pack_scale_arrays(stage.bit_depths); -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kForwardDwt53Params { - full_width: u32, - current_width: u32, - current_height: u32, - low_width: u32, - low_height: u32, -} + let params = J2kPackParams { + width: stage.dims.0, + height: stage.dims.1, + out_stride: j2k_u32_param(pitch_bytes, "J2K Metal output stride exceeds u32")?, + output_channels, + opaque_alpha, + max_values, + u8_scales, + u16_scales, + }; -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kMctStatus { - code: u32, - detail: u32, - _reserved0: u32, - _reserved1: u32, -} + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K decode hybrid plane pack"); + encoder.set_compute_pipeline_state(pipeline); + encoder.set_buffer( + 0, + stage.planes[0].as_ref().map(std::convert::AsRef::as_ref), + 0, + ); + encoder.set_buffer( + 1, + stage.planes[1].as_ref().map(std::convert::AsRef::as_ref), + 0, + ); + encoder.set_buffer( + 2, + stage.planes[2].as_ref().map(std::convert::AsRef::as_ref), + 0, + ); + encoder.set_buffer( + 3, + stage.planes[3].as_ref().map(std::convert::AsRef::as_ref), + 0, + ); + encoder.set_buffer(4, Some(&out_buffer), 0); + encoder.set_bytes( + 5, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_2d_pipeline(encoder, pipeline, stage.dims); + encoder.end_encoding(); -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kStoreParams { - input_width: u32, - source_x: u32, - source_y: u32, - copy_width: u32, - copy_height: u32, - output_width: u32, - output_x: u32, - output_y: u32, - addend: f32, + Ok(Surface::from_metal_buffer(out_buffer, stage.dims, fmt)) } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kRepeatedStoreParams { - input_width: u32, - input_height: u32, - source_x: u32, - source_y: u32, - copy_width: u32, - copy_height: u32, - output_width: u32, - output_height: u32, - output_x: u32, - output_y: u32, - addend: f32, - batch_count: u32, -} +fn encode_mct_rgb8_to_surface_in_command_buffer( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + planes: [&Buffer; 3], + dims: (u32, u32), + bit_depths: [u8; 3], + transform: J2kWaveletTransform, +) -> Result { + let pitch_bytes = dims.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let out_buffer = runtime.device.new_buffer( + (pitch_bytes * dims.1 as usize) as u64, + MTLResourceOptions::StorageModeShared, + ); + let (max_values, u8_scales, _) = j2k_pack_scale_arrays([ + u32::from(bit_depths[0]), + u32::from(bit_depths[1]), + u32::from(bit_depths[2]), + 0, + ]); + let params = J2kMctRgb8PackParams { + width: dims.0, + height: dims.1, + out_stride: j2k_u32_param(pitch_bytes, "J2K Metal output stride exceeds u32")?, + transform: mct_transform_code(transform), + addends: [ + signed_sample_bias(bit_depths[0]), + signed_sample_bias(bit_depths[1]), + signed_sample_bias(bit_depths[2]), + ], + max_values: [max_values[0], max_values[1], max_values[2]], + u8_scales: [u8_scales[0], u8_scales[1], u8_scales[2]], + }; -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kRepeatedGrayStoreParams { - input_width: u32, - input_height: u32, - source_x: u32, - source_y: u32, - copy_width: u32, - copy_height: u32, - output_width: u32, - output_height: u32, - output_x: u32, - output_y: u32, - addend: f32, - batch_count: u32, - max_value: f32, - u8_scale: f32, - u16_scale: f32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kGrayStoreParams { - input_width: u32, - source_x: u32, - source_y: u32, - copy_width: u32, - copy_height: u32, - output_width: u32, - output_x: u32, - output_y: u32, - addend: f32, - max_value: f32, - u8_scale: f32, - u16_scale: f32, -} - -const J2K_HT_STATUS_OK: u32 = 0; -#[cfg(target_os = "macos")] -const J2K_HT_STATUS_FAIL: u32 = 1; -#[cfg(target_os = "macos")] -const J2K_HT_STATUS_UNSUPPORTED: u32 = 2; - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kHtCleanupParams { - width: u32, - height: u32, - coded_len: u32, - cleanup_length: u32, - refinement_length: u32, - missing_msbs: u32, - num_bitplanes: u32, - number_of_coding_passes: u32, - output_stride: u32, - output_offset: u32, - dequantization_step: f32, - stripe_causal: u32, -} + let signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE); + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K decode hybrid MCT RGB8 pack"); + encoder.set_compute_pipeline_state(&runtime.pack_mct_rgb8); + encoder.set_buffer(0, Some(planes[0]), 0); + encoder.set_buffer(1, Some(planes[1]), 0); + encoder.set_buffer(2, Some(planes[2]), 0); + encoder.set_buffer(3, Some(&out_buffer), 0); + encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_2d_pipeline(encoder, &runtime.pack_mct_rgb8, dims); + encoder.end_encoding(); + drop(signpost); -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kHtCleanupBatchJob { - coded_offset: u32, - width: u32, - height: u32, - coded_len: u32, - cleanup_length: u32, - refinement_length: u32, - missing_msbs: u32, - num_bitplanes: u32, - number_of_coding_passes: u32, - output_stride: u32, - output_offset: u32, - dequantization_step: f32, - stripe_causal: u32, + Ok(Surface::from_metal_buffer( + out_buffer, + dims, + PixelFormat::Rgb8, + )) } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kHtRepeatedBatchParams { - job_count: u32, - output_plane_len: u32, - batch_count: u32, -} +fn encode_batched_mct_rgb8_to_surfaces_in_command_buffer( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + planes: [&Buffer; 3], + dims: (u32, u32), + count: usize, + bit_depths: [u8; 3], + transform: J2kWaveletTransform, +) -> Result, Error> { + let count_u32 = u32::try_from(count).map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect color batch count exceeds u32".to_string(), + })?; + let pitch_bytes = dims.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let surface_bytes = + pitch_bytes + .checked_mul(dims.1 as usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect color batch output size overflow".to_string(), + })?; + let total_bytes = surface_bytes + .checked_mul(count) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect color batch output size overflow".to_string(), + })?; + let out_buffer = runtime + .device + .new_buffer(total_bytes as u64, MTLResourceOptions::StorageModeShared); + let plane_stride = dims + .0 + .checked_mul(dims.1) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect color batch plane stride overflow".to_string(), + })?; + let (max_values, u8_scales, _) = j2k_pack_scale_arrays([ + u32::from(bit_depths[0]), + u32::from(bit_depths[1]), + u32::from(bit_depths[2]), + 0, + ]); + let params = J2kBatchedMctRgb8PackParams { + width: dims.0, + height: dims.1, + out_stride: j2k_u32_param(pitch_bytes, "J2K Metal output stride exceeds u32")?, + transform: mct_transform_code(transform), + batch_count: count_u32, + plane_stride, + output_stride: u32::try_from(surface_bytes).map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect color batch surface stride exceeds u32".to_string(), + })?, + addends: [ + signed_sample_bias(bit_depths[0]), + signed_sample_bias(bit_depths[1]), + signed_sample_bias(bit_depths[2]), + ], + max_values: [max_values[0], max_values[1], max_values[2]], + u8_scales: [u8_scales[0], u8_scales[1], u8_scales[2]], + }; -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kClassicRepeatedBatchParams { - job_count: u32, - output_plane_len: u32, - batch_count: u32, -} + let signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE); + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K decode hybrid batched MCT RGB8 pack"); + encoder.set_compute_pipeline_state(&runtime.pack_mct_rgb8_batched); + encoder.set_buffer(0, Some(planes[0]), 0); + encoder.set_buffer(1, Some(planes[1]), 0); + encoder.set_buffer(2, Some(planes[2]), 0); + encoder.set_buffer(3, Some(&out_buffer), 0); + encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_3d_pipeline( + encoder, + &runtime.pack_mct_rgb8_batched, + (dims.0, dims.1, count_u32), + ); + encoder.end_encoding(); + drop(signpost); -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kHtStatus { - code: u32, - detail: u32, - reserved0: u32, - reserved1: u32, + Ok((0..count) + .map(|index| { + Surface::from_metal_buffer_with_offset( + out_buffer.clone(), + dims, + PixelFormat::Rgb8, + index * surface_bytes, + ) + }) + .collect()) } #[cfg(target_os = "macos")] -const J2K_ENCODE_STATUS_OK: u32 = 0; -#[cfg(target_os = "macos")] -const J2K_ENCODE_STATUS_FAIL: u32 = 1; -#[cfg(target_os = "macos")] -const J2K_ENCODE_STATUS_UNSUPPORTED: u32 = 2; -#[cfg(target_os = "macos")] -const J2K_HT_ENCODE_MEL_SIZE: usize = 192; -#[cfg(target_os = "macos")] -const J2K_HT_ENCODE_VLC_SIZE: usize = 3072 - J2K_HT_ENCODE_MEL_SIZE; -#[cfg(target_os = "macos")] -const J2K_HT_ENCODE_MS_SIZE: usize = (16_384usize * 16).div_ceil(15); -#[cfg(target_os = "macos")] -const J2K_HT_ENCODE_BASE_OUTPUT_SIZE: usize = - J2K_HT_ENCODE_MS_SIZE + J2K_HT_ENCODE_MEL_SIZE + J2K_HT_ENCODE_VLC_SIZE; - -#[cfg(target_os = "macos")] -const HT_SIMD_PROTOTYPE_ENV: &str = "SIGNINUM_J2K_METAL_HT_SIMD_PROTOTYPE"; -#[cfg(target_os = "macos")] -const METAL_PROFILE_STAGES_ENV: &str = "SIGNINUM_J2K_METAL_PROFILE_STAGES"; +fn encode_repeated_mct_rgb8_to_surfaces_in_command_buffer( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + planes: [&Buffer; 3], + dims: (u32, u32), + count: usize, + bit_depths: [u8; 3], + transform: J2kWaveletTransform, +) -> Result, Error> { + let pitch_bytes = dims.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let surface_bytes = + pitch_bytes + .checked_mul(dims.1 as usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect repeated color batch output size overflow".to_string(), + })?; + let total_bytes = surface_bytes + .checked_mul(count) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect repeated color batch output size overflow".to_string(), + })?; + let output_len = u64::try_from(total_bytes.max(1)).map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect repeated output buffer exceeds u64".to_string(), + })?; + let out_buffer = runtime + .device + .new_buffer(output_len, MTLResourceOptions::StorageModeShared); + let plane_stride = dims + .0 + .checked_mul(dims.1) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect repeated color batch plane stride overflow".to_string(), + })?; + let (max_values, u8_scales, _) = j2k_pack_scale_arrays([ + u32::from(bit_depths[0]), + u32::from(bit_depths[1]), + u32::from(bit_depths[2]), + 0, + ]); + let params = J2kBatchedMctRgb8PackParams { + width: dims.0, + height: dims.1, + out_stride: j2k_u32_param(pitch_bytes, "J2K Metal output stride exceeds u32")?, + transform: mct_transform_code(transform), + batch_count: 1, + plane_stride, + output_stride: u32::try_from(surface_bytes).map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect repeated color batch surface stride exceeds u32".to_string(), + })?, + addends: [ + signed_sample_bias(bit_depths[0]), + signed_sample_bias(bit_depths[1]), + signed_sample_bias(bit_depths[2]), + ], + max_values: [max_values[0], max_values[1], max_values[2]], + u8_scales: [u8_scales[0], u8_scales[1], u8_scales[2]], + }; -#[cfg(target_os = "macos")] -fn env_flag_enabled(name: &str) -> bool { - matches!(std::env::var(name), Ok(value) if value == "1") -} + let signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE); + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K decode hybrid repeated MCT RGB8 pack"); + encoder.set_compute_pipeline_state(&runtime.pack_mct_rgb8_batched); + encoder.set_buffer(0, Some(planes[0]), 0); + encoder.set_buffer(1, Some(planes[1]), 0); + encoder.set_buffer(2, Some(planes[2]), 0); + encoder.set_buffer(3, Some(&out_buffer), 0); + encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_2d_pipeline(encoder, &runtime.pack_mct_rgb8_batched, dims); + encoder.end_encoding(); + drop(signpost); -#[cfg(target_os = "macos")] -fn ht_simd_prototype_env_requested() -> bool { - #[cfg(test)] - if let Some(enabled) = HT_SIMD_PROTOTYPE_ROUTE_OVERRIDE.with(Cell::get) { - return enabled; + if surface_bytes > 0 && count > 1 { + let blit = command_buffer.new_blit_command_encoder(); + if metal_profile_stages_enabled() { + blit.set_label("J2K decode hybrid repeated output blit"); + } + let mut copied = 1usize; + while copied < count { + let copy_count = copied.min(count - copied); + let dst_offset = + copied + .checked_mul(surface_bytes) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect repeated output destination offset overflow" + .to_string(), + })?; + let copy_bytes = + copy_count + .checked_mul(surface_bytes) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect repeated output copy size overflow".to_string(), + })?; + blit.copy_from_buffer( + &out_buffer, + 0, + &out_buffer, + u64::try_from(dst_offset).map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect repeated output destination offset exceeds u64" + .to_string(), + })?, + u64::try_from(copy_bytes).map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect repeated output copy size exceeds u64".to_string(), + })?, + ); + record_hybrid_repeated_output_blit(); + copied += copy_count; + } + blit.end_encoding(); } - env_flag_enabled(HT_SIMD_PROTOTYPE_ENV) -} - -#[cfg(target_os = "macos")] -fn metal_profile_stages_enabled() -> bool { - env_flag_enabled(METAL_PROFILE_STAGES_ENV) -} -#[cfg(target_os = "macos")] -fn label_command_buffer(command_buffer: &CommandBufferRef, label: &str) { - if metal_profile_stages_enabled() { - command_buffer.set_label(label); - } + Ok((0..count) + .map(|index| { + Surface::from_metal_buffer_with_offset( + out_buffer.clone(), + dims, + PixelFormat::Rgb8, + index * surface_bytes, + ) + }) + .collect()) } #[cfg(target_os = "macos")] -fn label_compute_encoder(encoder: &ComputeCommandEncoderRef, label: &str) { - if metal_profile_stages_enabled() { - encoder.set_label(label); - } +fn repeated_shared_direct_color_plan_count( + plans: &[Arc], +) -> Option { + let first = plans.first()?; + (plans.len() > 1 && plans.iter().all(|plan| Arc::ptr_eq(plan, first))).then_some(plans.len()) } #[cfg(target_os = "macos")] -fn compile_ht_simd_prototype_pipeline( - device: &Device, - options: &CompileOptions, -) -> Option { - let prototype_source = - format!("#define SIGNINUM_J2K_METAL_HT_SIMD_PROTOTYPE 1\n{SHADER_SOURCE}"); - let library = device - .new_library_with_source(&prototype_source, options) - .ok()?; - let function = library - .get_function("j2k_encode_ht_code_blocks_simd_prototype", None) - .ok()?; - let pipeline = device - .new_compute_pipeline_state_with_function(&function) - .ok()?; - if pipeline.thread_execution_width() == 32 && pipeline.max_total_threads_per_threadgroup() >= 32 - { - Some(pipeline) - } else { - None +fn mct_transform_code(transform: J2kWaveletTransform) -> u32 { + match transform { + J2kWaveletTransform::Reversible53 => 0, + J2kWaveletTransform::Irreversible97 => 1, } } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kClassicEncodeParams { - width: u32, - height: u32, - sub_band_type: u32, - total_bitplanes: u32, - style_flags: u32, - output_capacity: u32, - segment_capacity: u32, -} +fn prepare_classic_sub_band( + job: &j2k_native::J2kOwnedSubBandPlan, + tier1_prepare_mode: DirectTier1Mode, +) -> Result { + let mut jobs = Vec::with_capacity(job.jobs.len()); + let mut coded_data = Vec::new(); + let mut segments = Vec::new(); -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kClassicEncodeBatchJob { - coefficient_offset: u32, - output_offset: u32, - segment_offset: u32, - width: u32, - height: u32, - sub_band_type: u32, - total_bitplanes: u32, - style_flags: u32, - output_capacity: u32, - segment_capacity: u32, -} + for block in &job.jobs { + let coded_offset = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect coded payload exceeds u32".to_string(), + })?; + coded_data.extend_from_slice(&block.data); + let segment_offset = u32::try_from(segments.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect segment table exceeds u32".to_string(), + })?; + for segment in &block.segments { + let data_offset = coded_offset + .checked_add(segment.data_offset) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect segment offset overflow".to_string(), + })?; + segments.push(J2kClassicSegment { + data_offset, + data_length: segment.data_length, + start_coding_pass: u32::from(segment.start_coding_pass), + end_coding_pass: u32::from(segment.end_coding_pass), + use_arithmetic: u32::from(segment.use_arithmetic), + }); + } + jobs.push(J2kClassicCleanupBatchJob { + coded_offset, + coded_len: u32::try_from(block.data.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect coded payload exceeds u32".to_string(), + })?, + segment_offset, + segment_count: u32::try_from(block.segments.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect segment count exceeds u32".to_string(), + })?, + width: block.width, + height: block.height, + output_stride: job.width, + output_offset: block + .output_y + .checked_mul(job.width) + .and_then(|row| row.checked_add(block.output_x)) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect output offset overflow".to_string(), + })?, + missing_msbs: u32::from(block.missing_bit_planes), + total_bitplanes: u32::from(block.total_bitplanes), + roi_shift: u32::from(block.roi_shift), + number_of_coding_passes: u32::from(block.number_of_coding_passes), + sub_band_type: match block.sub_band_type { + j2k_native::J2kSubBandType::LowLow => 0, + j2k_native::J2kSubBandType::HighLow => 1, + j2k_native::J2kSubBandType::LowHigh => 2, + j2k_native::J2kSubBandType::HighHigh => 3, + }, + style_flags: classic_style_flags(block.style), + strict: u32::from(block.strict), + dequantization_step: block.dequantization_step, + }); + } -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kClassicEncodeStatus { - code: u32, - detail: u32, - data_len: u32, - number_of_coding_passes: u32, - missing_bit_planes: u32, - segment_count: u32, - reserved0: u32, - reserved1: u32, + with_runtime(|runtime| { + let coded_buffer = + prepare_direct_tier1_input_buffer(runtime, &coded_data, tier1_prepare_mode); + let jobs_buffer = prepare_direct_tier1_input_buffer(runtime, &jobs, tier1_prepare_mode); + let segments_buffer = + prepare_direct_tier1_input_buffer(runtime, &segments, tier1_prepare_mode); + Ok(PreparedClassicSubBand { + band_id: job.band_id, + width: job.width, + height: job.height, + zero_fill: false, + coded_data, + coded_buffer, + jobs, + jobs_buffer, + segments, + segments_buffer, + }) + }) } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kHtEncodeParams { - width: u32, - height: u32, - total_bitplanes: u32, - output_capacity: u32, +fn prepare_classic_sub_band_groups( + steps: &[PreparedDirectGrayscaleStep], + tier1_prepare_mode: DirectTier1Mode, +) -> Result, Error> { + let mut groups = Vec::new(); + let mut step_idx = 0; + while step_idx < steps.len() { + let start_step = step_idx; + let mut sub_bands = Vec::new(); + while let Some(PreparedDirectGrayscaleStep::ClassicSubBand(sub_band)) = steps.get(step_idx) + { + sub_bands.push(sub_band); + step_idx += 1; + } + if sub_bands.len() > 1 { + groups.push(prepare_classic_sub_band_group( + start_step, + step_idx, + &sub_bands, + tier1_prepare_mode, + )?); + } + if step_idx == start_step { + step_idx += 1; + } + } + Ok(groups) } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kHtEncodeBatchJob { - coefficient_offset: u32, - output_offset: u32, - width: u32, - height: u32, - total_bitplanes: u32, - output_capacity: u32, -} +fn prepare_classic_sub_band_group( + start_step: usize, + end_step: usize, + sub_bands: &[&PreparedClassicSubBand], + tier1_prepare_mode: DirectTier1Mode, +) -> Result { + let mut members = Vec::with_capacity(sub_bands.len()); + let mut jobs = Vec::new(); + let mut segments = Vec::new(); + let mut coded_data = Vec::new(); + let mut output_base = 0usize; -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kHtEncodeStatus { - code: u32, - detail: u32, - data_len: u32, - num_coding_passes: u32, - num_zero_bitplanes: u32, - reserved0: u32, - reserved1: u32, - reserved2: u32, -} + for sub_band in sub_bands { + members.push(PreparedClassicSubBandGroupMember { + band_id: sub_band.band_id, + offset_elements: output_base, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); -#[cfg(all(target_os = "macos", test))] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) struct HtCodeBlockSegmentLengthsForTest { - pub(crate) magnitude_sign: u32, - pub(crate) mel: u32, - pub(crate) vlc: u32, -} + let coded_base = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect grouped coded payload exceeds u32".to_string(), + })?; + let segment_base = u32::try_from(segments.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect grouped segment table exceeds u32".to_string(), + })?; + let output_base_u32 = u32::try_from(output_base).map_err(|_| Error::MetalKernel { + message: "classic J2K MetalDirect grouped coefficient arena exceeds u32".to_string(), + })?; -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kPacketEncodeParams { - resolution_count: u32, - num_layers: u32, - num_components: u32, - code_block_count: u32, - subband_count: u32, - descriptor_count: u32, - output_capacity: u32, - header_capacity: u32, - scratch_node_capacity: u32, -} + for segment in &sub_band.segments { + let mut grouped_segment = *segment; + grouped_segment.data_offset = + coded_base + .checked_add(segment.data_offset) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect grouped segment offset overflow" + .to_string(), + })?; + segments.push(grouped_segment); + } -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kBatchedPacketEncodeJob { - resolution_offset: u32, - subband_offset: u32, - block_offset: u32, - descriptor_offset: u32, - state_block_offset: u32, - output_offset: u32, - header_offset: u32, - scratch_offset: u32, - resolution_count: u32, - num_layers: u32, - num_components: u32, - code_block_count: u32, - subband_count: u32, - descriptor_count: u32, - output_capacity: u32, - header_capacity: u32, - scratch_node_capacity: u32, -} + for job in &sub_band.jobs { + let mut grouped_job = *job; + grouped_job.coded_offset = + coded_base + .checked_add(job.coded_offset) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect grouped job coded offset overflow" + .to_string(), + })?; + grouped_job.segment_offset = + segment_base + .checked_add(job.segment_offset) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect grouped job segment offset overflow" + .to_string(), + })?; + grouped_job.output_offset = + output_base_u32 + .checked_add(job.output_offset) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect grouped output offset overflow" + .to_string(), + })?; + jobs.push(grouped_job); + } -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kPacketDescriptor { - packet_index: u32, - state_index: u32, - layer: u32, - resolution: u32, - component: u32, - precinct_lo: u32, - precinct_hi: u32, - state_block_offset: u32, -} + coded_data.extend_from_slice(&sub_band.coded_data); + let sub_band_len = + sub_band + .width + .checked_mul(sub_band.height) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect grouped sub-band size overflow".to_string(), + })? as usize; + output_base = output_base + .checked_add(sub_band_len) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect grouped coefficient arena overflow".to_string(), + })?; + } -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kPacketResolution { - subband_offset: u32, - subband_count: u32, + with_runtime(|runtime| { + let coded_buffer = + prepare_direct_tier1_input_buffer(runtime, &coded_data, tier1_prepare_mode); + let jobs_buffer = prepare_direct_tier1_input_buffer(runtime, &jobs, tier1_prepare_mode); + let segments_buffer = + prepare_direct_tier1_input_buffer(runtime, &segments, tier1_prepare_mode); + Ok(PreparedClassicSubBandGroup { + start_step, + end_step, + total_coefficients: output_base, + zero_fill: sub_bands.iter().any(|sub_band| sub_band.zero_fill), + coded_data, + coded_buffer, + jobs, + jobs_buffer, + segments, + segments_buffer, + members, + }) + }) } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kPacketSubband { - block_offset: u32, - block_count: u32, - num_cbs_x: u32, - num_cbs_y: u32, -} +fn prepare_ht_sub_band( + job: &j2k_native::HtOwnedSubBandPlan, + _tier1_prepare_mode: DirectTier1Mode, +) -> Result { + let mut jobs = Vec::with_capacity(job.jobs.len()); + let mut coded_data = Vec::new(); + for block in &job.jobs { + let coded_offset = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { + message: "HTJ2K MetalDirect coded payload exceeds u32".to_string(), + })?; + coded_data.extend_from_slice(&block.data); + jobs.push(J2kHtCleanupBatchJob { + coded_offset, + width: block.width, + height: block.height, + coded_len: u32::try_from(block.data.len()).map_err(|_| Error::MetalKernel { + message: "HTJ2K MetalDirect coded payload exceeds u32".to_string(), + })?, + cleanup_length: block.cleanup_length, + refinement_length: block.refinement_length, + missing_msbs: u32::from(block.missing_bit_planes), + num_bitplanes: u32::from(block.num_bitplanes), + roi_shift: u32::from(block.roi_shift), + number_of_coding_passes: u32::from(block.number_of_coding_passes), + output_stride: job.width, + output_offset: block + .output_y + .checked_mul(job.width) + .and_then(|row| row.checked_add(block.output_x)) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect output offset overflow".to_string(), + })?, + dequantization_step: block.dequantization_step, + stripe_causal: u32::from(block.stripe_causal), + }); + } -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kPacketBlock { - data_offset: u32, - data_len: u32, - num_coding_passes: u32, - num_zero_bitplanes: u32, - previously_included: u32, - l_block: u32, - block_coding_mode: u32, - reserved0: u32, + Ok(PreparedHtSubBand { + band_id: job.band_id, + width: job.width, + height: job.height, + coded_data, + coded_buffer: None, + jobs, + jobs_buffer: None, + }) } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kResidentPacketBlock { - tier1_job_index: u32, - previously_included: u32, - l_block: u32, - block_coding_mode: u32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kResidentPacketBlockParams { - block_count: u32, - tier1_job_count: u32, +fn prepare_ht_sub_band_groups( + steps: &[PreparedDirectGrayscaleStep], + tier1_prepare_mode: DirectTier1Mode, +) -> Result, Error> { + let mut groups = Vec::new(); + let mut step_idx = 0; + while step_idx < steps.len() { + let start_step = step_idx; + let mut sub_bands = Vec::new(); + while let Some(PreparedDirectGrayscaleStep::HtSubBand(sub_band)) = steps.get(step_idx) { + sub_bands.push(sub_band); + step_idx += 1; + } + if sub_bands.len() > 1 { + groups.push(prepare_ht_sub_band_group( + start_step, + step_idx, + &sub_bands, + tier1_prepare_mode, + )?); + } + if step_idx == start_step { + step_idx += 1; + } + } + Ok(groups) } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kPacketStateBlock { - previously_included: u32, - l_block: u32, +fn prepare_ht_sub_band_group( + start_step: usize, + end_step: usize, + sub_bands: &[&PreparedHtSubBand], + tier1_prepare_mode: DirectTier1Mode, +) -> Result { + let mut members = Vec::with_capacity(sub_bands.len()); + let mut jobs = Vec::new(); + let mut coded_data = Vec::new(); + let mut output_base = 0usize; + + for sub_band in sub_bands { + members.push(PreparedHtSubBandGroupMember { + band_id: sub_band.band_id, + offset_elements: output_base, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); + + let coded_base = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped coded payload exceeds u32".to_string(), + })?; + let output_base_u32 = u32::try_from(output_base).map_err(|_| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped coefficient arena exceeds u32".to_string(), + })?; + for job in &sub_band.jobs { + let mut grouped_job = *job; + grouped_job.coded_offset = + coded_base + .checked_add(job.coded_offset) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped coded offset overflow".to_string(), + })?; + grouped_job.output_offset = + output_base_u32 + .checked_add(job.output_offset) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped output offset overflow".to_string(), + })?; + jobs.push(grouped_job); + } + coded_data.extend_from_slice(&sub_band.coded_data); + let sub_band_len = + sub_band + .width + .checked_mul(sub_band.height) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped sub-band size overflow".to_string(), + })? as usize; + output_base = output_base + .checked_add(sub_band_len) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect grouped coefficient arena overflow".to_string(), + })?; + } + + with_runtime(|runtime| { + let coded_buffer = + prepare_direct_tier1_input_buffer(runtime, &coded_data, tier1_prepare_mode); + let jobs_buffer = prepare_direct_tier1_input_buffer(runtime, &jobs, tier1_prepare_mode); + Ok(PreparedHtSubBandGroup { + start_step, + end_step, + total_coefficients: output_base, + coded_arena: HtCodedArena { + data: coded_data, + buffer: coded_buffer, + }, + jobs, + jobs_buffer, + members, + }) + }) } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kPacketEncodeStatus { - code: u32, - detail: u32, - data_len: u32, - reserved0: u32, +fn prepare_ungrouped_ht_sub_band_buffers( + steps: &mut [PreparedDirectGrayscaleStep], + groups: &[PreparedHtSubBandGroup], + tier1_prepare_mode: DirectTier1Mode, +) -> Result<(), Error> { + if tier1_prepare_mode != DirectTier1Mode::Metal { + return Ok(()); + } + + for (step_idx, step) in steps.iter_mut().enumerate() { + let PreparedDirectGrayscaleStep::HtSubBand(sub_band) = step else { + continue; + }; + if groups + .iter() + .any(|group| group.start_step <= step_idx && step_idx < group.end_step) + { + sub_band.coded_buffer = None; + sub_band.jobs_buffer = None; + continue; + } + with_runtime(|runtime| { + sub_band.coded_buffer = Some(prepare_direct_tier1_input_buffer( + runtime, + &sub_band.coded_data, + tier1_prepare_mode, + )); + sub_band.jobs_buffer = Some(prepare_direct_tier1_input_buffer( + runtime, + &sub_band.jobs, + tier1_prepare_mode, + )); + Ok(()) + })?; + } + + Ok(()) } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct J2kLosslessCodestreamAssemblyParams { - width: u32, - height: u32, - num_components: u32, - bit_depth: u32, - signed_samples: u32, - num_decomposition_levels: u32, - use_mct: u32, - guard_bits: u32, - progression_order: u32, - write_tlm: u32, - high_throughput: u32, - output_capacity: u32, +fn prepared_ht_buffer<'a>(buffer: Option<&'a Buffer>, label: &str) -> Result<&'a Buffer, Error> { + buffer.ok_or_else(|| Error::MetalKernel { + message: format!("HTJ2K MetalDirect ungrouped sub-band is missing prepared {label} buffer"), + }) } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kBatchedCodestreamAssemblyJob { - tile_data_offset: u32, - codestream_offset: u32, - width: u32, - height: u32, - num_components: u32, - bit_depth: u32, - signed_samples: u32, - num_decomposition_levels: u32, - use_mct: u32, - guard_bits: u32, - progression_order: u32, - write_tlm: u32, - high_throughput: u32, - output_capacity: u32, +pub(crate) fn prepare_direct_grayscale_plan( + plan: &J2kDirectGrayscalePlan, +) -> Result { + prepare_direct_grayscale_plan_with_tier1_mode(plan, DirectTier1Mode::Metal) } #[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct J2kCodestreamAssemblyStatus { - code: u32, - detail: u32, - data_len: u32, - reserved0: u32, +fn prepare_direct_grayscale_plan_for_cpu_upload( + plan: &J2kDirectGrayscalePlan, +) -> Result { + prepare_direct_grayscale_plan_with_tier1_mode(plan, DirectTier1Mode::CpuUpload) } #[cfg(target_os = "macos")] -static METAL_RUNTIME: OnceLock, String>> = OnceLock::new(); +fn prepare_direct_grayscale_plan_with_tier1_mode( + plan: &J2kDirectGrayscalePlan, + tier1_prepare_mode: DirectTier1Mode, +) -> Result { + let mut steps = Vec::with_capacity(plan.steps.len()); + for step in &plan.steps { + match step { + J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { + steps.push(PreparedDirectGrayscaleStep::ClassicSubBand( + prepare_classic_sub_band(sub_band, tier1_prepare_mode)?, + )); + } + J2kDirectGrayscaleStep::HtSubBand(sub_band) => { + steps.push(PreparedDirectGrayscaleStep::HtSubBand(prepare_ht_sub_band( + sub_band, + tier1_prepare_mode, + )?)); + } + J2kDirectGrayscaleStep::Idwt(idwt) => { + steps.push(PreparedDirectGrayscaleStep::Idwt(PreparedDirectIdwt { + step: *idwt, + output_window: BandRequiredRegion::full(idwt.rect.width(), idwt.rect.height()), + })); + } + J2kDirectGrayscaleStep::Store(store) => { + steps.push(PreparedDirectGrayscaleStep::Store(*store)); + } + } + } + let classic_groups = prepare_classic_sub_band_groups(&steps, tier1_prepare_mode)?; + let ht_groups = prepare_ht_sub_band_groups(&steps, tier1_prepare_mode)?; + prepare_ungrouped_ht_sub_band_buffers(&mut steps, &ht_groups, tier1_prepare_mode)?; + Ok(PreparedDirectGrayscalePlan { + dimensions: plan.dimensions, + bit_depth: plan.bit_depth, + tier1_prepare_mode, + steps, + classic_groups, + ht_groups, + cpu_tier1_cache: Arc::new(CpuTier1CoefficientCache::default()), + }) +} #[cfg(target_os = "macos")] -type MetalRuntimeCache = Mutex, String>>>; +pub(crate) fn crop_prepared_direct_grayscale_plan_to_output_region( + plan: &mut PreparedDirectGrayscalePlan, + region: Rect, +) -> Result<(), Error> { + if region.w == 0 || region.h == 0 { + return Err(Error::MetalKernel { + message: "J2K MetalDirect region-scaled grayscale plan has an empty output region" + .to_string(), + }); + } + if region.x == 0 + && region.y == 0 + && region.w == plan.dimensions.0 + && region.h == plan.dimensions.1 + { + return Ok(()); + } + + plan.clear_cpu_tier1_cache()?; + let mut store_count = 0; + for step in &mut plan.steps { + if let PreparedDirectGrayscaleStep::Store(store) = step { + crop_direct_store_step_to_output_region(store, region)?; + store_count += 1; + } + } + + if store_count == 0 { + return Err(Error::MetalKernel { + message: "J2K MetalDirect grayscale plan has no store step to crop".to_string(), + }); + } + + prune_prepared_direct_grayscale_plan_to_store_windows(plan)?; + plan.dimensions = (region.w, region.h); + Ok(()) +} #[cfg(target_os = "macos")] -static METAL_DEVICE_RUNTIMES: OnceLock = OnceLock::new(); +#[derive(Clone, Copy, Debug)] +struct BandRequiredRegion { + x0: u32, + y0: u32, + x1: u32, + y1: u32, +} #[cfg(target_os = "macos")] -thread_local! { - static METAL_RUNTIME_OVERRIDE: RefCell>> = const { RefCell::new(None) }; +impl BandRequiredRegion { + fn full(width: u32, height: u32) -> Self { + Self { + x0: 0, + y0: 0, + x1: width, + y1: height, + } + } + + fn new(x0: u32, y0: u32, x1: u32, y1: u32) -> Option { + (x0 < x1 && y0 < y1).then_some(Self { x0, y0, x1, y1 }) + } + + fn width(self) -> u32 { + self.x1 - self.x0 + } + + fn height(self) -> u32 { + self.y1 - self.y0 + } + + fn expanded(self, margin: u32, width: u32, height: u32) -> Self { + Self { + x0: self.x0.saturating_sub(margin), + y0: self.y0.saturating_sub(margin), + x1: self.x1.saturating_add(margin).min(width), + y1: self.y1.saturating_add(margin).min(height), + } + } + + fn union(self, other: Self) -> Self { + Self { + x0: self.x0.min(other.x0), + y0: self.y0.min(other.y0), + x1: self.x1.max(other.x1), + y1: self.y1.max(other.y1), + } + } + + fn intersects(self, x0: u32, y0: u32, width: u32, height: u32) -> bool { + let x1 = x0.saturating_add(width); + let y1 = y0.saturating_add(height); + self.x0 < x1 && x0 < self.x1 && self.y0 < y1 && y0 < self.y1 + } } #[cfg(target_os = "macos")] -struct MetalRuntime { - device: Device, - queue: CommandQueue, - zero_u32_buffer: ComputePipelineState, - validate_bytes_equal: ComputePipelineState, - copy_interleaved_padded: ComputePipelineState, - lossless_deinterleave_to_planes: ComputePipelineState, - lossless_extract_coefficients: ComputePipelineState, - pack_gray8: ComputePipelineState, - pack_rgb8: ComputePipelineState, - pack_mct_rgb8: ComputePipelineState, - pack_mct_rgb8_batched: ComputePipelineState, - pack_rgb_opaque_rgba8: ComputePipelineState, - pack_rgba8: ComputePipelineState, - pack_gray16: ComputePipelineState, - pack_rgb16: ComputePipelineState, - pack_u8_repeated_gray: ComputePipelineState, - pack_u16_repeated_gray: ComputePipelineState, - classic_cleanup_plain_batched: ComputePipelineState, - classic_cleanup_batched: ComputePipelineState, - classic_cleanup_plain_repeated_batched: ComputePipelineState, - classic_cleanup_plain_dev_repeated_batched: ComputePipelineState, - classic_cleanup_repeated_batched: ComputePipelineState, - classic_store_repeated_batched: ComputePipelineState, - idwt_interleave: ComputePipelineState, - idwt_reversible53_horizontal: ComputePipelineState, - idwt_reversible53_vertical: ComputePipelineState, - idwt_interleave_batched: ComputePipelineState, - idwt_reversible53_horizontal_batched: ComputePipelineState, - idwt_reversible53_vertical_batched: ComputePipelineState, - idwt_irreversible97_single_decomposition: ComputePipelineState, - fdwt53_horizontal: ComputePipelineState, - fdwt53_vertical: ComputePipelineState, - inverse_mct: ComputePipelineState, - forward_rct: ComputePipelineState, - store_component: ComputePipelineState, - store_component_repeated: ComputePipelineState, - store_component_repeated_gray_u8: ComputePipelineState, - store_component_repeated_gray_u16: ComputePipelineState, - store_component_repeated_gray_u8_contiguous: ComputePipelineState, - store_component_repeated_gray_u16_contiguous: ComputePipelineState, - store_component_gray_u8: ComputePipelineState, - store_component_gray_u16: ComputePipelineState, - ht_cleanup: ComputePipelineState, - ht_cleanup_batched: ComputePipelineState, - ht_cleanup_repeated_batched: ComputePipelineState, - classic_encode_code_block: ComputePipelineState, - classic_encode_code_blocks: ComputePipelineState, - ht_encode_code_block: ComputePipelineState, - ht_encode_code_blocks: ComputePipelineState, - ht_encode_code_blocks_simd_prototype: Option, - packet_block_prepare_resident_classic: ComputePipelineState, - packet_block_prepare_resident_ht: ComputePipelineState, - packet_encode: ComputePipelineState, - packet_encode_batched: ComputePipelineState, - lossless_codestream_assemble: ComputePipelineState, - lossless_codestream_assemble_batched: ComputePipelineState, - ht_vlc_table0: Buffer, - ht_vlc_table1: Buffer, - ht_uvlc_table0: Buffer, - ht_uvlc_table1: Buffer, - ht_vlc_encode_table0: Buffer, - ht_vlc_encode_table1: Buffer, - ht_uvlc_encode_table: Buffer, - private_buffer_pool: Mutex>>, -} - -#[cfg(target_os = "macos")] -impl MetalRuntime { - fn new() -> Result { - let device = Device::system_default() - .ok_or_else(|| "Metal is unavailable on this host".to_string())?; - Self::new_with_device(&device) +fn prune_prepared_direct_grayscale_plan_to_store_windows( + plan: &mut PreparedDirectGrayscalePlan, +) -> Result<(), Error> { + let mut required = HashMap::::new(); + for step in &plan.steps { + if let PreparedDirectGrayscaleStep::Store(store) = step { + let source_right = store + .source_x + .checked_add(store.copy_width) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect ROI source width overflows u32".to_string(), + })?; + let source_bottom = store + .source_y + .checked_add(store.copy_height) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect ROI source height overflows u32".to_string(), + })?; + if let Some(region) = + BandRequiredRegion::new(store.source_x, store.source_y, source_right, source_bottom) + { + add_required_region(&mut required, store.input_band_id, region); + } + } } - fn new_with_device(device: &Device) -> Result { - let options = CompileOptions::new(); - let library = device.new_library_with_source(SHADER_SOURCE, &options)?; - let pipeline = |name: &str| { - let function = library.get_function(name, None)?; - device.new_compute_pipeline_state_with_function(&function) - }; - let ht_encode_code_blocks_simd_prototype = - if cfg!(test) || ht_simd_prototype_env_requested() { - compile_ht_simd_prototype_pipeline(device, &options) - } else { - None + let mut idwt_output_windows = HashMap::::new(); + for step in plan.steps.iter().rev() { + if let PreparedDirectGrayscaleStep::Idwt(idwt) = step { + let Some(output_region) = required.get(&idwt.step.output_band_id).copied() else { + continue; }; - if ht_simd_prototype_env_requested() && ht_encode_code_blocks_simd_prototype.is_none() { - return Err("SIGNINUM_J2K_METAL_HT_SIMD_PROTOTYPE=1 requested, but the HTJ2K SIMD prototype pipeline is unavailable on this Metal device".to_string()); - } - let classic_cleanup_plain_batched_fn = - library.get_function("j2k_decode_classic_cleanup_plain_batched", None)?; - let classic_cleanup_batched_fn = - library.get_function("j2k_decode_classic_cleanup_batched", None)?; - let classic_cleanup_plain_repeated_batched_fn = - library.get_function("j2k_decode_classic_cleanup_plain_repeated_batched", None)?; - let classic_cleanup_plain_dev_repeated_batched_fn = library.get_function( - "j2k_decode_classic_cleanup_plain_dev_repeated_batched", - None, - )?; - let classic_cleanup_repeated_batched_fn = - library.get_function("j2k_decode_classic_cleanup_repeated_batched", None)?; - let classic_store_repeated_batched_fn = - library.get_function("j2k_store_classic_repeated_batched", None)?; - let idwt_interleave_fn = library.get_function("j2k_idwt_interleave", None)?; - let idwt_interleave_batched_fn = - library.get_function("j2k_idwt_interleave_batched", None)?; - let idwt_reversible53_horizontal_fn = - library.get_function("j2k_idwt_reversible53_horizontal_pass", None)?; - let idwt_reversible53_horizontal_batched_fn = - library.get_function("j2k_idwt_reversible53_horizontal_pass_batched", None)?; - let idwt_reversible53_vertical_fn = - library.get_function("j2k_idwt_reversible53_vertical_pass", None)?; - let idwt_reversible53_vertical_batched_fn = - library.get_function("j2k_idwt_reversible53_vertical_pass_batched", None)?; - let idwt_irreversible97_single_decomposition_fn = - library.get_function("j2k_idwt_irreversible97_single_decomposition", None)?; - let fdwt53_horizontal_fn = library.get_function("j2k_forward_dwt53_horizontal", None)?; - let fdwt53_vertical_fn = library.get_function("j2k_forward_dwt53_vertical", None)?; - let inverse_mct_fn = library.get_function("j2k_inverse_mct", None)?; - let forward_rct_fn = library.get_function("j2k_forward_rct", None)?; - let store_component_fn = library.get_function("j2k_store_component", None)?; - let store_component_repeated_fn = - library.get_function("j2k_store_component_repeated", None)?; - let store_component_repeated_gray_u8_fn = - library.get_function("j2k_store_component_repeated_gray_u8", None)?; - let store_component_repeated_gray_u16_fn = - library.get_function("j2k_store_component_repeated_gray_u16", None)?; - let store_component_repeated_gray_u8_contiguous_fn = - library.get_function("j2k_store_component_repeated_gray_u8_contiguous", None)?; - let store_component_repeated_gray_u16_contiguous_fn = - library.get_function("j2k_store_component_repeated_gray_u16_contiguous", None)?; - let store_component_gray_u8_fn = - library.get_function("j2k_store_component_gray_u8", None)?; - let store_component_gray_u16_fn = - library.get_function("j2k_store_component_gray_u16", None)?; - let ht_cleanup_fn = library.get_function("j2k_decode_ht_cleanup", None)?; - let ht_cleanup_batched_fn = library.get_function("j2k_decode_ht_cleanup_batched", None)?; - let ht_cleanup_repeated_batched_fn = - library.get_function("j2k_decode_ht_cleanup_repeated_batched", None)?; - let classic_cleanup_plain_batched = - device.new_compute_pipeline_state_with_function(&classic_cleanup_plain_batched_fn)?; - let classic_cleanup_batched = - device.new_compute_pipeline_state_with_function(&classic_cleanup_batched_fn)?; - let classic_cleanup_plain_repeated_batched = device - .new_compute_pipeline_state_with_function(&classic_cleanup_plain_repeated_batched_fn)?; - let classic_cleanup_plain_dev_repeated_batched = device - .new_compute_pipeline_state_with_function( - &classic_cleanup_plain_dev_repeated_batched_fn, - )?; - let classic_cleanup_repeated_batched = device - .new_compute_pipeline_state_with_function(&classic_cleanup_repeated_batched_fn)?; - let classic_store_repeated_batched = - device.new_compute_pipeline_state_with_function(&classic_store_repeated_batched_fn)?; - let idwt_interleave = - device.new_compute_pipeline_state_with_function(&idwt_interleave_fn)?; - let idwt_interleave_batched = - device.new_compute_pipeline_state_with_function(&idwt_interleave_batched_fn)?; - let idwt_reversible53_horizontal = - device.new_compute_pipeline_state_with_function(&idwt_reversible53_horizontal_fn)?; - let idwt_reversible53_horizontal_batched = device - .new_compute_pipeline_state_with_function(&idwt_reversible53_horizontal_batched_fn)?; - let idwt_reversible53_vertical = - device.new_compute_pipeline_state_with_function(&idwt_reversible53_vertical_fn)?; - let idwt_reversible53_vertical_batched = device - .new_compute_pipeline_state_with_function(&idwt_reversible53_vertical_batched_fn)?; - let idwt_irreversible97_single_decomposition = device - .new_compute_pipeline_state_with_function( - &idwt_irreversible97_single_decomposition_fn, - )?; - let fdwt53_horizontal = - device.new_compute_pipeline_state_with_function(&fdwt53_horizontal_fn)?; - let fdwt53_vertical = - device.new_compute_pipeline_state_with_function(&fdwt53_vertical_fn)?; - let inverse_mct = device.new_compute_pipeline_state_with_function(&inverse_mct_fn)?; - let forward_rct = device.new_compute_pipeline_state_with_function(&forward_rct_fn)?; - let store_component = - device.new_compute_pipeline_state_with_function(&store_component_fn)?; - let store_component_repeated = - device.new_compute_pipeline_state_with_function(&store_component_repeated_fn)?; - let store_component_repeated_gray_u8 = device - .new_compute_pipeline_state_with_function(&store_component_repeated_gray_u8_fn)?; - let store_component_repeated_gray_u16 = device - .new_compute_pipeline_state_with_function(&store_component_repeated_gray_u16_fn)?; - let store_component_repeated_gray_u8_contiguous = device - .new_compute_pipeline_state_with_function( - &store_component_repeated_gray_u8_contiguous_fn, - )?; - let store_component_repeated_gray_u16_contiguous = device - .new_compute_pipeline_state_with_function( - &store_component_repeated_gray_u16_contiguous_fn, - )?; - let store_component_gray_u8 = - device.new_compute_pipeline_state_with_function(&store_component_gray_u8_fn)?; - let store_component_gray_u16 = - device.new_compute_pipeline_state_with_function(&store_component_gray_u16_fn)?; - let ht_cleanup = device.new_compute_pipeline_state_with_function(&ht_cleanup_fn)?; - let ht_cleanup_batched = - device.new_compute_pipeline_state_with_function(&ht_cleanup_batched_fn)?; - let ht_cleanup_repeated_batched = - device.new_compute_pipeline_state_with_function(&ht_cleanup_repeated_batched_fn)?; - let queue = new_command_queue(device)?; - Ok(Self { - device: device.clone(), - queue, - zero_u32_buffer: pipeline("j2k_zero_u32_buffer")?, - validate_bytes_equal: pipeline("j2k_validate_bytes_equal")?, - copy_interleaved_padded: pipeline("j2k_copy_interleaved_padded")?, - lossless_deinterleave_to_planes: pipeline("j2k_lossless_deinterleave_to_planes")?, - lossless_extract_coefficients: pipeline("j2k_lossless_extract_coefficients")?, - pack_gray8: pipeline("j2k_pack_gray8")?, - pack_rgb8: pipeline("j2k_pack_rgb8")?, - pack_mct_rgb8: pipeline("j2k_pack_mct_rgb8")?, - pack_mct_rgb8_batched: pipeline("j2k_pack_mct_rgb8_batched")?, - pack_rgb_opaque_rgba8: pipeline("j2k_pack_rgb_opaque_rgba8")?, - pack_rgba8: pipeline("j2k_pack_rgba8")?, - pack_gray16: pipeline("j2k_pack_gray16")?, - pack_rgb16: pipeline("j2k_pack_rgb16")?, - pack_u8_repeated_gray: pipeline("j2k_pack_u8_repeated_gray")?, - pack_u16_repeated_gray: pipeline("j2k_pack_u16_repeated_gray")?, - classic_cleanup_plain_batched, - classic_cleanup_batched, - classic_cleanup_plain_repeated_batched, - classic_cleanup_plain_dev_repeated_batched, - classic_cleanup_repeated_batched, - classic_store_repeated_batched, - idwt_interleave, - idwt_reversible53_horizontal, - idwt_reversible53_vertical, - idwt_interleave_batched, - idwt_reversible53_horizontal_batched, - idwt_reversible53_vertical_batched, - idwt_irreversible97_single_decomposition, - fdwt53_horizontal, - fdwt53_vertical, - inverse_mct, - forward_rct, - store_component, - store_component_repeated, - store_component_repeated_gray_u8, - store_component_repeated_gray_u16, - store_component_repeated_gray_u8_contiguous, - store_component_repeated_gray_u16_contiguous, - store_component_gray_u8, - store_component_gray_u16, - ht_cleanup, - ht_cleanup_batched, - ht_cleanup_repeated_batched, - classic_encode_code_block: pipeline("j2k_encode_classic_code_block")?, - classic_encode_code_blocks: pipeline("j2k_encode_classic_code_blocks")?, - ht_encode_code_block: pipeline("j2k_encode_ht_code_block")?, - ht_encode_code_blocks: pipeline("j2k_encode_ht_code_blocks")?, - ht_encode_code_blocks_simd_prototype, - packet_block_prepare_resident_classic: pipeline( - "j2k_prepare_packet_blocks_from_classic_status", - )?, - packet_block_prepare_resident_ht: pipeline("j2k_prepare_packet_blocks_from_ht_status")?, - packet_encode: pipeline("j2k_encode_packetization")?, - packet_encode_batched: pipeline("j2k_encode_packetization_batched")?, - lossless_codestream_assemble: pipeline("j2k_assemble_lossless_classic_codestream")?, - lossless_codestream_assemble_batched: pipeline( - "j2k_assemble_lossless_codestream_batched", - )?, - ht_vlc_table0: device.new_buffer_with_data( - ht_vlc_table0().as_ptr().cast(), - size_of_val(ht_vlc_table0()) as u64, - MTLResourceOptions::StorageModeShared, - ), - ht_vlc_table1: device.new_buffer_with_data( - ht_vlc_table1().as_ptr().cast(), - size_of_val(ht_vlc_table1()) as u64, - MTLResourceOptions::StorageModeShared, - ), - ht_uvlc_table0: device.new_buffer_with_data( - ht_uvlc_table0().as_ptr().cast(), - size_of_val(ht_uvlc_table0()) as u64, - MTLResourceOptions::StorageModeShared, - ), - ht_uvlc_table1: device.new_buffer_with_data( - ht_uvlc_table1().as_ptr().cast(), - size_of_val(ht_uvlc_table1()) as u64, - MTLResourceOptions::StorageModeShared, - ), - ht_vlc_encode_table0: device.new_buffer_with_data( - ht_vlc_encode_table0().as_ptr().cast(), - size_of_val(ht_vlc_encode_table0()) as u64, - MTLResourceOptions::StorageModeShared, - ), - ht_vlc_encode_table1: device.new_buffer_with_data( - ht_vlc_encode_table1().as_ptr().cast(), - size_of_val(ht_vlc_encode_table1()) as u64, - MTLResourceOptions::StorageModeShared, - ), - ht_uvlc_encode_table: device.new_buffer_with_data( - ht_uvlc_encode_table().as_ptr().cast(), - size_of_val(ht_uvlc_encode_table()) as u64, - MTLResourceOptions::StorageModeShared, - ), - private_buffer_pool: Mutex::new(HashMap::new()), - }) + let expanded = output_region.expanded( + idwt_required_output_margin(idwt.step.transform), + idwt.step.rect.width(), + idwt.step.rect.height(), + ); + idwt_output_windows.insert(idwt.step.output_band_id, expanded); + add_idwt_input_required_regions(&mut required, &idwt.step, expanded); + } } - fn take_private_buffer(&self, bytes: usize) -> Buffer { - let bytes = bytes.max(1); - let mut pool = self - .private_buffer_pool - .lock() - .expect("private buffer pool lock not poisoned"); - if let Some(buffer) = pool.get_mut(&bytes).and_then(Vec::pop) { - buffer - } else { - #[cfg(test)] - PRIVATE_BUFFER_POOL_MISSES.with(|misses| misses.set(misses.get() + 1)); - self.device - .new_buffer(bytes as u64, MTLResourceOptions::StorageModePrivate) + for step in &mut plan.steps { + match step { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { + let before = sub_band.jobs.len(); + retain_classic_jobs_for_required_region( + &mut sub_band.jobs, + required.get(&sub_band.band_id).copied(), + ); + if sub_band.jobs.len() != before { + sub_band.zero_fill = true; + if plan.tier1_prepare_mode == DirectTier1Mode::Metal { + with_runtime(|runtime| { + sub_band.jobs_buffer = + borrow_slice_buffer(&runtime.device, &sub_band.jobs); + Ok(()) + })?; + } + } + } + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + let before = sub_band.jobs.len(); + retain_ht_jobs_for_required_region( + &mut sub_band.jobs, + required.get(&sub_band.band_id).copied(), + ); + if sub_band.jobs.len() != before { + compact_ht_sub_band_coded_data(sub_band, plan.tier1_prepare_mode)?; + } + } + PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => {} } } - fn recycle_private_buffer(&self, bytes: usize, buffer: Buffer) { - let bytes = bytes.max(1); - self.private_buffer_pool - .lock() - .expect("private buffer pool lock not poisoned") - .entry(bytes) - .or_default() - .push(buffer); - } + apply_prepared_direct_idwt_output_windows(plan, &idwt_output_windows)?; + plan.classic_groups = prepare_classic_sub_band_groups(&plan.steps, plan.tier1_prepare_mode)?; + plan.ht_groups = prepare_ht_sub_band_groups(&plan.steps, plan.tier1_prepare_mode)?; + prepare_ungrouped_ht_sub_band_buffers( + &mut plan.steps, + &plan.ht_groups, + plan.tier1_prepare_mode, + )?; + Ok(()) } #[cfg(target_os = "macos")] -fn new_command_queue(device: &Device) -> Result { - let queue: *mut MTLCommandQueue = unsafe { - device - .as_ref() - .send_message(Sel::register("newCommandQueue"), ()) - .map_err(|error| format!("Metal command queue creation failed: {error}"))? - }; - if queue.is_null() { - return Err("Metal command queue is unavailable on this host".to_string()); +fn apply_prepared_direct_idwt_output_windows( + plan: &mut PreparedDirectGrayscalePlan, + windows: &HashMap, +) -> Result<(), Error> { + for step in &mut plan.steps { + if let PreparedDirectGrayscaleStep::Idwt(idwt) = step { + idwt.output_window = windows + .get(&idwt.step.output_band_id) + .copied() + .unwrap_or_else(|| { + BandRequiredRegion::full(idwt.step.rect.width(), idwt.step.rect.height()) + }); + } } - Ok(unsafe { CommandQueue::from_ptr(queue) }) -} -#[cfg(target_os = "macos")] -fn with_runtime(f: impl FnOnce(&MetalRuntime) -> Result) -> Result { - let override_runtime = METAL_RUNTIME_OVERRIDE.with(|slot| slot.borrow().clone()); - if let Some(runtime) = override_runtime { - return f(&runtime); - } + for step in &mut plan.steps { + let PreparedDirectGrayscaleStep::Store(store) = step else { + continue; + }; + let Some(window) = windows.get(&store.input_band_id).copied() else { + continue; + }; - match METAL_RUNTIME.get_or_init(|| MetalRuntime::new().map(Arc::new)) { - Ok(runtime) => f(runtime), - Err(message) => Err(runtime_initialization_error(message)), + store.source_x = + store + .source_x + .checked_sub(window.x0) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect cropped IDWT store source x underflow".to_string(), + })?; + store.source_y = + store + .source_y + .checked_sub(window.y0) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect cropped IDWT store source y underflow".to_string(), + })?; + store.input_rect = j2k_native::J2kRect { + x0: store.input_rect.x0.saturating_add(window.x0), + y0: store.input_rect.y0.saturating_add(window.y0), + x1: store.input_rect.x0.saturating_add(window.x1), + y1: store.input_rect.y0.saturating_add(window.y1), + }; } -} -#[cfg(target_os = "macos")] -fn runtime_initialization_error(message: &str) -> Error { - match message { - "Metal is unavailable on this host" | "Metal command queue is unavailable on this host" => { - Error::MetalUnavailable - } - _ => Error::MetalKernel { - message: message.to_string(), - }, - } + Ok(()) } #[cfg(target_os = "macos")] -struct RuntimeOverrideGuard { - previous: Option>, -} +#[derive(Clone, Copy)] +struct PreparedIdwtInputWindows { + ll: BandRequiredRegion, + hl: BandRequiredRegion, + lh: BandRequiredRegion, + hh: BandRequiredRegion, +} -#[cfg(target_os = "macos")] -impl Drop for RuntimeOverrideGuard { - fn drop(&mut self) { - let previous = self.previous.take(); - METAL_RUNTIME_OVERRIDE.with(|slot| { - slot.replace(previous); - }); +fn idwt_input_windows_from_slices( + ll: &DirectBandSlice, + hl: &DirectBandSlice, + lh: &DirectBandSlice, + hh: &DirectBandSlice, +) -> PreparedIdwtInputWindows { + PreparedIdwtInputWindows { + ll: BandRequiredRegion::full(ll.window.width(), ll.window.height()), + hl: BandRequiredRegion::full(hl.window.width(), hl.window.height()), + lh: BandRequiredRegion::full(lh.window.width(), lh.window.height()), + hh: BandRequiredRegion::full(hh.window.width(), hh.window.height()), } } #[cfg(target_os = "macos")] -fn with_runtime_for_device( - device: &Device, - f: impl FnOnce(&MetalRuntime) -> Result, -) -> Result { - let override_runtime = METAL_RUNTIME_OVERRIDE.with(|slot| slot.borrow().clone()); - if let Some(runtime) = override_runtime { - if runtime.device.as_ptr() == device.as_ptr() { - return f(&runtime); - } - } - - let cache = METAL_DEVICE_RUNTIMES.get_or_init(|| Mutex::new(HashMap::new())); - let key = device.as_ptr() as usize; - let runtime = { - let mut cache = cache - .lock() - .expect("J2K Metal runtime cache lock not poisoned"); - cache - .entry(key) - .or_insert_with(|| MetalRuntime::new_with_device(device).map(Arc::new)) - .clone() +fn prepared_idwt_params( + idwt: &PreparedDirectIdwt, + inputs: PreparedIdwtInputWindows, +) -> J2kIdwtSingleDecompositionParams { + J2kIdwtSingleDecompositionParams { + x0: idwt.step.rect.x0, + y0: idwt.step.rect.y0, + output_x: idwt.output_window.x0, + output_y: idwt.output_window.y0, + width: idwt.output_window.width(), + height: idwt.output_window.height(), + ll_x: inputs.ll.x0, + ll_y: inputs.ll.y0, + ll_width: inputs.ll.width(), + ll_height: inputs.ll.height(), + hl_x: inputs.hl.x0, + hl_y: inputs.hl.y0, + hl_width: inputs.hl.width(), + hl_height: inputs.hl.height(), + lh_x: inputs.lh.x0, + lh_y: inputs.lh.y0, + lh_width: inputs.lh.width(), + lh_height: inputs.lh.height(), + hh_x: inputs.hh.x0, + hh_y: inputs.hh.y0, + hh_width: inputs.hh.width(), + hh_height: inputs.hh.height(), } - .map_err(|message| runtime_initialization_error(&message))?; - let previous = METAL_RUNTIME_OVERRIDE.with(|slot| slot.replace(Some(runtime.clone()))); - let _guard = RuntimeOverrideGuard { previous }; - f(&runtime) } -#[cfg(all(target_os = "macos", test))] -pub(crate) fn with_isolated_runtime_for_device_for_test( - device: &Device, - f: impl FnOnce() -> Result, -) -> Result { - let runtime = Arc::new( - MetalRuntime::new_with_device(device) - .map_err(|message| runtime_initialization_error(&message))?, - ); - let previous = METAL_RUNTIME_OVERRIDE.with(|slot| slot.replace(Some(runtime))); - let _guard = RuntimeOverrideGuard { previous }; - f() +#[cfg(target_os = "macos")] +fn repeated_idwt_params( + idwt: &PreparedDirectIdwt, + inputs: PreparedIdwtInputWindows, + strides: PreparedIdwtInputStrides, + batch_count: usize, + context: &'static str, +) -> Result { + Ok(J2kRepeatedIdwtSingleDecompositionParams { + x0: idwt.step.rect.x0, + y0: idwt.step.rect.y0, + output_x: idwt.output_window.x0, + output_y: idwt.output_window.y0, + width: idwt.output_window.width(), + height: idwt.output_window.height(), + ll_x: inputs.ll.x0, + ll_y: inputs.ll.y0, + ll_width: inputs.ll.width(), + ll_height: inputs.ll.height(), + hl_x: inputs.hl.x0, + hl_y: inputs.hl.y0, + hl_width: inputs.hl.width(), + hl_height: inputs.hl.height(), + lh_x: inputs.lh.x0, + lh_y: inputs.lh.y0, + lh_width: inputs.lh.width(), + lh_height: inputs.lh.height(), + hh_x: inputs.hh.x0, + hh_y: inputs.hh.y0, + hh_width: inputs.hh.width(), + hh_height: inputs.hh.height(), + ll_instance_stride: strides.ll, + hl_instance_stride: strides.hl, + lh_instance_stride: strides.lh, + hh_instance_stride: strides.hh, + batch_count: u32::try_from(batch_count).map_err(|_| Error::MetalKernel { + message: format!("J2K MetalDirect {context} IDWT batch count exceeds u32"), + })?, + }) } -#[cfg(all(target_os = "macos", test))] -pub(crate) fn ht_simd_prototype_available_for_test() -> Result { - with_runtime(|runtime| Ok(runtime.ht_encode_code_blocks_simd_prototype.is_some())) +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +struct PreparedIdwtInputStrides { + ll: u32, + hl: u32, + lh: u32, + hh: u32, } -#[cfg(all(target_os = "macos", test))] -pub(crate) fn ht_simd_prototype_available_for_device_for_test( - device: &Device, -) -> Result { - with_runtime_for_device(device, |runtime| { - Ok(runtime.ht_encode_code_blocks_simd_prototype.is_some()) - }) +#[cfg(target_os = "macos")] +fn prepared_idwt_output_len(idwt: &PreparedDirectIdwt) -> usize { + idwt.output_window.width() as usize * idwt.output_window.height() as usize } #[cfg(target_os = "macos")] -pub(crate) fn validate_metal_buffer_matches_bytes( - expected: &[u8], - actual_buffer: &Buffer, - actual_byte_offset: usize, - session: &crate::MetalBackendSession, -) -> Result<(), Error> { - if expected.is_empty() { - return Ok(()); - } - let byte_len = u32::try_from(expected.len()).map_err(|_| Error::MetalKernel { - message: "J2K Metal validation buffer exceeds u32 byte length".to_string(), - })?; - let actual_offset = u64::try_from(actual_byte_offset).map_err(|_| Error::MetalKernel { - message: "J2K Metal validation buffer offset exceeds u64".to_string(), - })?; - - with_runtime_for_device(&session.device, |runtime| { - let expected_buffer = runtime.device.new_buffer_with_data( - expected.as_ptr().cast(), - expected.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let status = J2kValidateBytesStatus::default(); - let status_buffer = runtime.device.new_buffer_with_data( - (&raw const status).cast(), - size_of::() as u64, - MTLResourceOptions::StorageModeShared, - ); - let params = J2kValidateBytesParams { byte_len }; - - let command_buffer = runtime.queue.new_command_buffer(); - label_command_buffer(command_buffer, "signinum-j2k lossless coefficient prep"); - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.validate_bytes_equal); - encoder.set_buffer(0, Some(actual_buffer), actual_offset); - encoder.set_buffer(1, Some(&expected_buffer), 0); - encoder.set_buffer(2, Some(&status_buffer), 0); - encoder.set_bytes( - 3, - size_of::() as u64, - (&raw const params).cast(), - ); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - ); - encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - - let status = unsafe { - status_buffer - .contents() - .cast::() - .read() - }; - if status.code == 0 { - return Ok(()); - } - - Err(Error::MetalKernel { - message: format!( - "J2K Metal validation mismatch at byte {}: expected {}, got {}", - status.index, status.expected, status.actual - ), - }) - }) +fn add_required_region( + required: &mut HashMap, + band_id: J2kDirectBandId, + region: BandRequiredRegion, +) { + required + .entry(band_id) + .and_modify(|existing| *existing = existing.union(region)) + .or_insert(region); } #[cfg(target_os = "macos")] -pub(crate) fn validate_metal_buffers_match( - expected_buffer: &Buffer, - expected_byte_offset: usize, - actual_buffer: &Buffer, - actual_byte_offset: usize, - byte_len: usize, - session: &crate::MetalBackendSession, -) -> Result<(), Error> { - if byte_len == 0 { - return Ok(()); +fn idwt_required_output_margin(transform: J2kWaveletTransform) -> u32 { + match transform { + J2kWaveletTransform::Reversible53 => 16, + J2kWaveletTransform::Irreversible97 => 40, } - let byte_len_u32 = u32::try_from(byte_len).map_err(|_| Error::MetalKernel { - message: "J2K Metal validation buffer exceeds u32 byte length".to_string(), - })?; - let expected_offset = u64::try_from(expected_byte_offset).map_err(|_| Error::MetalKernel { - message: "J2K Metal validation expected buffer offset exceeds u64".to_string(), - })?; - let actual_offset = u64::try_from(actual_byte_offset).map_err(|_| Error::MetalKernel { - message: "J2K Metal validation actual buffer offset exceeds u64".to_string(), - })?; - - with_runtime_for_device(&session.device, |runtime| { - let status = J2kValidateBytesStatus::default(); - let status_buffer = runtime.device.new_buffer_with_data( - (&raw const status).cast(), - size_of::() as u64, - MTLResourceOptions::StorageModeShared, - ); - let params = J2kValidateBytesParams { - byte_len: byte_len_u32, - }; - - let command_buffer = runtime.queue.new_command_buffer(); - label_command_buffer( - command_buffer, - "signinum-j2k lossless coefficient prep batch", - ); - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.validate_bytes_equal); - encoder.set_buffer(0, Some(actual_buffer), actual_offset); - encoder.set_buffer(1, Some(expected_buffer), expected_offset); - encoder.set_buffer(2, Some(&status_buffer), 0); - encoder.set_bytes( - 3, - size_of::() as u64, - (&raw const params).cast(), - ); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - ); - encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - - let status = unsafe { - status_buffer - .contents() - .cast::() - .read() - }; - if status.code == 0 { - return Ok(()); - } - - Err(Error::MetalKernel { - message: format!( - "J2K Metal validation mismatch at byte {}: expected {}, got {}", - status.index, status.expected, status.actual - ), - }) - }) } #[cfg(target_os = "macos")] -#[allow(clippy::too_many_arguments)] -pub(crate) fn copy_interleaved_padded_to_shared_buffer( - src_buffer: &Buffer, - src_byte_offset: usize, - src_width: u32, - src_height: u32, - src_pitch_bytes: usize, - dst_width: u32, - dst_height: u32, - bytes_per_pixel: usize, - session: &crate::MetalBackendSession, -) -> Result { - if src_width > dst_width || src_height > dst_height { - return Err(Error::MetalKernel { - message: "J2K Metal input tile cannot be larger than encoded tile".to_string(), - }); - } - let src_stride = u32::try_from(src_pitch_bytes).map_err(|_| Error::MetalKernel { - message: "J2K Metal input tile pitch exceeds u32".to_string(), - })?; - let dst_stride_usize = (dst_width as usize) - .checked_mul(bytes_per_pixel) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal padded tile stride overflow".to_string(), - })?; - let dst_stride = u32::try_from(dst_stride_usize).map_err(|_| Error::MetalKernel { - message: "J2K Metal padded tile stride exceeds u32".to_string(), - })?; - let dst_len = dst_stride_usize - .checked_mul(dst_height as usize) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal padded tile byte length overflow".to_string(), - })?; - let bytes_per_pixel = u32::try_from(bytes_per_pixel).map_err(|_| Error::MetalKernel { - message: "J2K Metal bytes-per-pixel exceeds u32".to_string(), - })?; - let src_offset = u64::try_from(src_byte_offset).map_err(|_| Error::MetalKernel { - message: "J2K Metal input tile offset exceeds u64".to_string(), - })?; - - with_runtime_for_device(&session.device, |runtime| { - let dst_buffer = runtime - .device - .new_buffer(dst_len as u64, MTLResourceOptions::StorageModeShared); - let params = J2kCopyInterleavedParams { - src_width, - src_height, - src_stride, - dst_width, - dst_height, - dst_stride, - bytes_per_pixel, - }; - let command_buffer = runtime.queue.new_command_buffer(); - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.copy_interleaved_padded); - encoder.set_buffer(0, Some(src_buffer), src_offset); - encoder.set_buffer(1, Some(&dst_buffer), 0); - encoder.set_bytes( - 2, - size_of::() as u64, - (&raw const params).cast(), - ); - let width = runtime - .copy_interleaved_padded - .thread_execution_width() - .max(1); - let max_threads = runtime - .copy_interleaved_padded - .max_total_threads_per_threadgroup() - .max(width); - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(dst_width), - height: u64::from(dst_height), - depth: 1, - }, - MTLSize { - width, - height, - depth: 1, - }, - ); - encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - Ok(dst_buffer) - }) +fn add_idwt_input_required_regions( + required: &mut HashMap, + idwt: &J2kDirectIdwtStep, + output_region: BandRequiredRegion, +) { + add_required_region( + required, + idwt.ll_band_id, + idwt_input_required_region( + output_region, + idwt.rect.x0, + idwt.rect.y0, + true, + true, + idwt.ll.width(), + idwt.ll.height(), + ), + ); + add_required_region( + required, + idwt.hl_band_id, + idwt_input_required_region( + output_region, + idwt.rect.x0, + idwt.rect.y0, + false, + true, + idwt.hl.width(), + idwt.hl.height(), + ), + ); + add_required_region( + required, + idwt.lh_band_id, + idwt_input_required_region( + output_region, + idwt.rect.x0, + idwt.rect.y0, + true, + false, + idwt.lh.width(), + idwt.lh.height(), + ), + ); + add_required_region( + required, + idwt.hh_band_id, + idwt_input_required_region( + output_region, + idwt.rect.x0, + idwt.rect.y0, + false, + false, + idwt.hh.width(), + idwt.hh.height(), + ), + ); } #[cfg(target_os = "macos")] -enum DirectStatusCheck { - Classic { buffer: Buffer, len: usize }, - Ht { buffer: Buffer, len: usize }, - Idwt(Buffer), - Mct(Buffer), +#[allow(clippy::fn_params_excessive_bools)] +fn idwt_input_required_region( + output_region: BandRequiredRegion, + output_origin_x: u32, + output_origin_y: u32, + low_x: bool, + low_y: bool, + band_width: u32, + band_height: u32, +) -> BandRequiredRegion { + let x0 = j2k_native::idwt_band_index(output_origin_x, output_region.x0, low_x); + let x1 = + j2k_native::idwt_band_index(output_origin_x, output_region.x1 - 1, low_x).saturating_add(1); + let y0 = j2k_native::idwt_band_index(output_origin_y, output_region.y0, low_y); + let y1 = + j2k_native::idwt_band_index(output_origin_y, output_region.y1 - 1, low_y).saturating_add(1); + BandRequiredRegion { + x0: x0.min(band_width), + y0: y0.min(band_height), + x1: x1.min(band_width), + y1: y1.min(band_height), + } } #[cfg(target_os = "macos")] -struct DirectScratchBuffer { - bytes: usize, - buffer: Buffer, +fn retain_classic_jobs_for_required_region( + jobs: &mut Vec, + required: Option, +) { + let Some(required) = required else { + jobs.clear(); + return; + }; + jobs.retain(|job| { + let output_x = job.output_offset % job.output_stride; + let output_y = job.output_offset / job.output_stride; + required.intersects(output_x, output_y, job.width, job.height) + }); } #[cfg(target_os = "macos")] -#[derive(Clone)] -pub(crate) struct PreparedDirectGrayscalePlan { - dimensions: (u32, u32), - bit_depth: u8, - steps: Vec, - classic_groups: Vec, - ht_groups: Vec, +fn retain_ht_jobs_for_required_region( + jobs: &mut Vec, + required: Option, +) { + let Some(required) = required else { + jobs.clear(); + return; + }; + jobs.retain(|job| { + let output_x = job.output_offset % job.output_stride; + let output_y = job.output_offset / job.output_stride; + required.intersects(output_x, output_y, job.width, job.height) + }); } #[cfg(target_os = "macos")] -#[derive(Clone)] -pub(crate) struct PreparedDirectColorPlan { - dimensions: (u32, u32), - bit_depths: [u8; 3], - mct: bool, - transform: J2kWaveletTransform, - component_plans: Vec, +fn compact_ht_sub_band_coded_data( + sub_band: &mut PreparedHtSubBand, + _tier1_prepare_mode: DirectTier1Mode, +) -> Result<(), Error> { + let previous = std::mem::take(&mut sub_band.coded_data); + let mut compacted = Vec::new(); + + for job in &mut sub_band.jobs { + let start = job.coded_offset as usize; + let len = job.coded_len as usize; + let end = start.checked_add(len).ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect cropped coded payload range overflow".to_string(), + })?; + if end > previous.len() { + return Err(Error::MetalKernel { + message: "HTJ2K MetalDirect cropped coded payload range out of bounds".to_string(), + }); + } + job.coded_offset = u32::try_from(compacted.len()).map_err(|_| Error::MetalKernel { + message: "HTJ2K MetalDirect cropped coded payload exceeds u32".to_string(), + })?; + compacted.extend_from_slice(&previous[start..end]); + } + + sub_band.coded_data = compacted; + sub_band.coded_buffer = None; + sub_band.jobs_buffer = None; + Ok(()) } #[cfg(target_os = "macos")] -#[derive(Clone)] -enum PreparedDirectGrayscaleStep { - ClassicSubBand(PreparedClassicSubBand), - HtSubBand(PreparedHtSubBand), - Idwt(PreparedDirectIdwt), - Store(J2kDirectStoreStep), +fn checked_rect_end(origin: u32, length: u32, label: &str) -> Result { + origin + .checked_add(length) + .ok_or_else(|| Error::MetalKernel { + message: format!("J2K MetalDirect region-scaled {label} overflows u32"), + }) } #[cfg(target_os = "macos")] -#[derive(Clone)] -struct PreparedDirectIdwt { - step: J2kDirectIdwtStep, - output_window: BandRequiredRegion, +fn crop_direct_store_step_to_output_region( + store: &mut J2kDirectStoreStep, + region: Rect, +) -> Result<(), Error> { + let store_bounds = ( + store.output_x, + store.output_y, + checked_rect_end(store.output_x, store.copy_width, "store width")?, + checked_rect_end(store.output_y, store.copy_height, "store height")?, + ); + let region_bounds = ( + region.x, + region.y, + checked_rect_end(region.x, region.w, "ROI width")?, + checked_rect_end(region.y, region.h, "ROI height")?, + ); + let intersection = ( + store_bounds.0.max(region_bounds.0), + store_bounds.1.max(region_bounds.1), + store_bounds.2.min(region_bounds.2), + store_bounds.3.min(region_bounds.3), + ); + if intersection.0 >= intersection.2 || intersection.1 >= intersection.3 { + return Err(Error::MetalKernel { + message: + "J2K MetalDirect region-scaled ROI does not intersect the decoded store window" + .to_string(), + }); + } + + let source_delta = ( + intersection.0 - store_bounds.0, + intersection.1 - store_bounds.1, + ); + store.source_x = + store + .source_x + .checked_add(source_delta.0) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect region-scaled source x overflows u32".to_string(), + })?; + store.source_y = + store + .source_y + .checked_add(source_delta.1) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect region-scaled source y overflows u32".to_string(), + })?; + store.copy_width = intersection.2 - intersection.0; + store.copy_height = intersection.3 - intersection.1; + store.output_width = region.w; + store.output_height = region.h; + store.output_x = intersection.0 - region_bounds.0; + store.output_y = intersection.1 - region_bounds.1; + Ok(()) } #[cfg(target_os = "macos")] -#[derive(Clone)] -struct PreparedClassicSubBand { - band_id: J2kDirectBandId, - width: u32, - height: u32, - zero_fill: bool, - coded_data: Vec, - coded_buffer: Buffer, - jobs: Vec, - jobs_buffer: Buffer, - segments: Vec, - segments_buffer: Buffer, +pub(crate) fn prepare_direct_color_plan( + plan: &J2kDirectColorPlan, +) -> Result { + prepare_direct_color_plan_with_tier1_mode(plan, DirectTier1Mode::Metal) } #[cfg(target_os = "macos")] -#[derive(Clone)] -struct PreparedClassicSubBandGroup { - start_step: usize, - end_step: usize, - total_coefficients: usize, - zero_fill: bool, - coded_data: Vec, - coded_buffer: Buffer, - jobs: Vec, - jobs_buffer: Buffer, - segments: Vec, - segments_buffer: Buffer, - members: Vec, +pub(crate) fn prepare_direct_color_plan_for_cpu_upload( + plan: &J2kDirectColorPlan, +) -> Result { + prepare_direct_color_plan_with_tier1_mode(plan, DirectTier1Mode::CpuUpload) } #[cfg(target_os = "macos")] -#[derive(Clone)] -struct PreparedClassicSubBandGroupMember { - band_id: J2kDirectBandId, - offset_elements: usize, - window: BandRequiredRegion, -} - -#[cfg(target_os = "macos")] -#[derive(Clone)] -struct PreparedHtSubBand { - band_id: J2kDirectBandId, - width: u32, - height: u32, - coded_data: Vec, - coded_buffer: Buffer, - jobs: Vec, - jobs_buffer: Buffer, -} - -#[cfg(target_os = "macos")] -#[derive(Clone)] -struct PreparedHtSubBandGroup { - start_step: usize, - end_step: usize, - total_coefficients: usize, - coded_data: Vec, - coded_buffer: Buffer, - jobs: Vec, - jobs_buffer: Buffer, - members: Vec, -} - -#[cfg(target_os = "macos")] -#[derive(Clone)] -struct PreparedHtSubBandGroupMember { - band_id: J2kDirectBandId, - offset_elements: usize, - window: BandRequiredRegion, -} - -#[cfg(target_os = "macos")] -struct PlaneStage { - dims: (u32, u32), - plane_count: usize, - color_space: NativeColorSpace, - has_alpha: bool, - bit_depths: [u32; 4], - planes: [Option; 4], +fn prepare_direct_color_plan_with_tier1_mode( + plan: &J2kDirectColorPlan, + tier1_prepare_mode: DirectTier1Mode, +) -> Result { + let component_plans = plan + .component_plans + .iter() + .map(|component| match tier1_prepare_mode { + DirectTier1Mode::Metal => prepare_direct_grayscale_plan(component), + DirectTier1Mode::CpuUpload => prepare_direct_grayscale_plan_for_cpu_upload(component), + }) + .collect::, _>>()?; + if component_plans.len() != 3 { + return Err(Error::MetalKernel { + message: format!( + "J2K MetalDirect color plan expected 3 component plans, got {}", + component_plans.len() + ), + }); + } + Ok(PreparedDirectColorPlan { + dimensions: plan.dimensions, + bit_depths: plan.bit_depths, + mct: plan.mct, + transform: plan.transform, + component_plans, + }) } #[cfg(target_os = "macos")] -impl PlaneStage { - fn from_planes( - device: &Device, - decoded: &NativeDecodedComponents<'_>, - roi: Option, - ) -> Result { - let full_dims = decoded.dimensions(); - let roi = roi.unwrap_or(Rect { - x: 0, - y: 0, - w: full_dims.0, - h: full_dims.1, +pub(crate) fn crop_prepared_direct_color_plan_to_output_region( + plan: &mut PreparedDirectColorPlan, + region: Rect, +) -> Result<(), Error> { + if region.w == 0 || region.h == 0 { + return Err(Error::MetalKernel { + message: "J2K MetalDirect region-scaled color plan has an empty output region" + .to_string(), }); - let dims = (roi.w, roi.h); - let plane_count = decoded.planes().len(); - if plane_count == 0 || plane_count > 4 { + } + + for component_plan in &mut plan.component_plans { + crop_prepared_direct_grayscale_plan_to_output_region(component_plan, region)?; + if component_plan.dimensions != (region.w, region.h) { return Err(Error::MetalKernel { - message: format!("unsupported J2K plane count {plane_count}"), + message: format!( + "J2K MetalDirect color component crop produced {:?}, expected {:?}", + component_plan.dimensions, + (region.w, region.h) + ), }); } - - let mut bit_depths = [0u32; 4]; - let mut planes: [Option; 4] = [None, None, None, None]; - for (index, plane) in decoded.planes().iter().enumerate() { - bit_depths[index] = u32::from(plane.bit_depth()); - let len = dims.0 as usize * dims.1 as usize; - let buffer = device.new_buffer( - (len * size_of::()) as u64, - MTLResourceOptions::StorageModeShared, - ); - copy_plane_samples(&buffer, plane.samples(), full_dims.0 as usize, roi); - planes[index] = Some(buffer); - } - - Ok(Self { - dims, - plane_count, - color_space: decoded.color_space().clone(), - has_alpha: decoded.has_alpha(), - bit_depths, - planes, - }) } - fn from_captured_planes( - decoded: &NativeDecodedComponents<'_>, - captured_planes: Vec, - ) -> Option { - let plane_count = decoded.planes().len(); - let supported_shape = matches!( - (decoded.color_space(), decoded.has_alpha(), plane_count), - (NativeColorSpace::Gray, false, 1) | (NativeColorSpace::RGB, false, 3) - ); - if !supported_shape { - return None; - } - if captured_planes.len() != plane_count || plane_count == 0 || plane_count > 4 { - return None; - } + plan.dimensions = (region.w, region.h); + Ok(()) +} - let mut bit_depths = [0u32; 4]; - let mut planes: [Option; 4] = [None, None, None, None]; - for (index, (plane, buffer)) in decoded.planes().iter().zip(captured_planes).enumerate() { - bit_depths[index] = u32::from(plane.bit_depth()); - planes[index] = Some(buffer); - } +#[cfg(target_os = "macos")] +impl PreparedDirectGrayscalePlan { + fn classic_group_starting_at(&self, step_idx: usize) -> Option<&PreparedClassicSubBandGroup> { + self.classic_groups + .iter() + .find(|group| group.start_step == step_idx) + } - Some(Self { - dims: decoded.dimensions(), - plane_count, - color_space: decoded.color_space().clone(), - has_alpha: decoded.has_alpha(), - bit_depths, - planes, - }) + fn ht_group_starting_at(&self, step_idx: usize) -> Option<&PreparedHtSubBandGroup> { + self.ht_groups + .iter() + .find(|group| group.start_step == step_idx) } +} - fn finish_with_runtime( - self, - runtime: &MetalRuntime, - fmt: PixelFormat, - ) -> Result { - let command_buffer = runtime.queue.new_command_buffer(); - let surface = - encode_plane_stage_to_surface_in_command_buffer(runtime, command_buffer, &self, fmt)?; - command_buffer.commit(); - command_buffer.wait_until_completed(); - Ok(surface) +#[cfg(all(test, target_os = "macos"))] +fn prepared_direct_grayscale_plan_compute_encoder_count( + plan: &PreparedDirectGrayscalePlan, + _fmt: PixelFormat, +) -> usize { + usize::from(!plan.steps.is_empty()) +} + +#[cfg(all(test, target_os = "macos"))] +fn prepared_repeated_direct_ht_cleanup_dispatch_count(plan: &PreparedDirectGrayscalePlan) -> usize { + let mut dispatches = 0; + let mut step_idx = 0; + while step_idx < plan.steps.len() { + if let Some(group) = plan.ht_group_starting_at(step_idx) { + dispatches += 1; + step_idx = group.end_step; + continue; + } + if matches!( + plan.steps[step_idx], + PreparedDirectGrayscaleStep::HtSubBand(_) + ) { + dispatches += 1; + } + step_idx += 1; } + dispatches } #[cfg(target_os = "macos")] -fn encode_plane_stage_to_surface_in_command_buffer( +fn encode_prepared_direct_grayscale_plan_in_command_buffer( runtime: &MetalRuntime, command_buffer: &CommandBufferRef, - stage: &PlaneStage, + plan: &PreparedDirectGrayscalePlan, fmt: PixelFormat, + retained_buffers: &mut Vec, + status_checks: &mut Vec, + scratch_buffers: &mut Vec, ) -> Result { - let pitch_bytes = stage.dims.0 as usize * fmt.bytes_per_pixel(); - let out_buffer = runtime.device.new_buffer( - (pitch_bytes * stage.dims.1 as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - let (output_channels, opaque_alpha, pipeline) = output_shape_for( - &stage.color_space, - stage.has_alpha, - stage.plane_count, - fmt, - runtime, - )?; - let (max_values, u8_scales, u16_scales) = j2k_pack_scale_arrays(stage.bit_depths); + let encoder = command_buffer.new_compute_command_encoder(); + let result = (|| { + let mut bands = Vec::::new(); + let mut final_surface = None; + let mut step_idx = 0; - let params = J2kPackParams { - width: stage.dims.0, - height: stage.dims.1, - out_stride: u32::try_from(pitch_bytes).expect("J2K Metal output stride fits in u32"), - output_channels, - opaque_alpha, - max_values, - u8_scales, - u16_scales, - }; + while step_idx < plan.steps.len() { + if let Some(group) = plan.classic_group_starting_at(step_idx) { + let output = take_f32_scratch_buffer(runtime, group.total_coefficients)?; + let (buffers, status_check) = + encode_prepared_classic_sub_band_group_to_buffer_in_encoder( + runtime, + encoder, + group, + &output.buffer, + scratch_buffers, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + for member in &group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: output.buffer.clone(), + offset_bytes: member.offset_elements * size_of::(), + window: member.window, + }); + } + scratch_buffers.push(output); + step_idx = group.end_step; + continue; + } - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(pipeline); - encoder.set_buffer( - 0, - stage.planes[0].as_ref().map(std::convert::AsRef::as_ref), - 0, - ); - encoder.set_buffer( - 1, - stage.planes[1].as_ref().map(std::convert::AsRef::as_ref), - 0, - ); - encoder.set_buffer( - 2, - stage.planes[2].as_ref().map(std::convert::AsRef::as_ref), - 0, - ); - encoder.set_buffer( - 3, - stage.planes[3].as_ref().map(std::convert::AsRef::as_ref), - 0, - ); - encoder.set_buffer(4, Some(&out_buffer), 0); - encoder.set_bytes( - 5, - size_of::() as u64, - (&raw const params).cast(), - ); + if let Some(group) = plan.ht_group_starting_at(step_idx) { + let output = take_f32_scratch_buffer(runtime, group.total_coefficients)?; + let (buffers, status_check) = + encode_prepared_ht_sub_band_group_to_buffer_in_encoder( + runtime, + encoder, + group, + &output.buffer, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + for member in &group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: output.buffer.clone(), + offset_bytes: member.offset_elements * size_of::(), + window: member.window, + }); + } + scratch_buffers.push(output); + step_idx = group.end_step; + continue; + } - let width = pipeline.thread_execution_width().max(1); - let max_threads = pipeline.max_total_threads_per_threadgroup().max(width); - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(stage.dims.0), - height: u64::from(stage.dims.1), - depth: 1, - }, - MTLSize { - width, - height, - depth: 1, - }, - ); + let step = &plan.steps[step_idx]; + match step { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { + let output = take_f32_scratch_buffer( + runtime, + sub_band.width as usize * sub_band.height as usize, + )?; + let (buffers, status_check) = + encode_prepared_classic_sub_band_to_buffer_in_encoder( + runtime, + encoder, + sub_band, + &output.buffer, + scratch_buffers, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + bands.push(DirectBandSlice { + band_id: sub_band.band_id, + buffer: output.buffer.clone(), + offset_bytes: 0, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); + scratch_buffers.push(output); + } + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + let output = take_f32_scratch_buffer( + runtime, + sub_band.width as usize * sub_band.height as usize, + )?; + let (buffers, status_check) = encode_prepared_ht_sub_band_to_buffer_in_encoder( + runtime, + encoder, + sub_band, + &output.buffer, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + bands.push(DirectBandSlice { + band_id: sub_band.band_id, + buffer: output.buffer.clone(), + offset_bytes: 0, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); + scratch_buffers.push(output); + } + PreparedDirectGrayscaleStep::Idwt(idwt) => { + let ll = + lookup_direct_band_slice_entry(&bands, idwt.step.ll_band_id, idwt.step.ll)?; + let hl = + lookup_direct_band_slice_entry(&bands, idwt.step.hl_band_id, idwt.step.hl)?; + let lh = + lookup_direct_band_slice_entry(&bands, idwt.step.lh_band_id, idwt.step.lh)?; + let hh = + lookup_direct_band_slice_entry(&bands, idwt.step.hh_band_id, idwt.step.hh)?; + let params = prepared_idwt_params( + idwt, + idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), + ); + let output = take_f32_scratch_buffer(runtime, prepared_idwt_output_len(idwt))?; + match idwt.step.transform { + J2kWaveletTransform::Reversible53 => { + dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets( + runtime, + encoder, + &ll.buffer, + ll.offset_bytes, + &hl.buffer, + hl.offset_bytes, + &lh.buffer, + lh.offset_bytes, + &hh.buffer, + hh.offset_bytes, + params, + &output.buffer, + 0, + ); + } + J2kWaveletTransform::Irreversible97 => { + let status_check = + dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets( + runtime, + encoder, + &ll.buffer, + ll.offset_bytes, + &hl.buffer, + hl.offset_bytes, + &lh.buffer, + lh.offset_bytes, + &hh.buffer, + hh.offset_bytes, + params, + &output.buffer, + 0, + ); + status_checks.push(status_check); + } + } + bands.push(DirectBandSlice { + band_id: idwt.step.output_band_id, + buffer: output.buffer.clone(), + offset_bytes: 0, + window: idwt.output_window, + }); + scratch_buffers.push(output); + } + PreparedDirectGrayscaleStep::Store(store) => { + let (input, input_offset) = + lookup_direct_band_slice(&bands, store.input_band_id, store.input_rect)?; + if matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { + let scale = j2k_scalar_pack_params(u32::from(plan.bit_depth)); + final_surface = Some(encode_gray_store_to_surface_in_encoder( + runtime, + encoder, + &input, + input_offset, + J2kGrayStoreParams { + input_width: store.input_rect.width(), + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + max_value: scale.max_value, + u8_scale: scale.u8_scale, + u16_scale: scale.u16_scale, + }, + plan.dimensions, + fmt, + )?); + } else { + let output = take_f32_scratch_buffer( + runtime, + store.output_width as usize * store.output_height as usize, + )?; + let params = J2kStoreParams { + input_width: store.input_rect.width(), + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + }; + dispatch_store_component_buffer_in_encoder_with_offsets( + runtime, + encoder, + &input, + input_offset, + &output.buffer, + 0, + params, + ); + retained_buffers.push(output.buffer.clone()); + final_surface = Some(encode_gray_plane_to_surface_in_encoder( + runtime, + encoder, + &output.buffer, + plan.dimensions, + plan.bit_depth, + fmt, + )?); + scratch_buffers.push(output); + } + } + } + step_idx += 1; + } + + final_surface.ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect prepared grayscale plan did not produce a final stored plane" + .to_string(), + }) + })(); encoder.end_encoding(); + result +} - Ok(Surface::from_metal_buffer(out_buffer, stage.dims, fmt)) +#[cfg(target_os = "macos")] +fn checked_coefficient_len(width: u32, height: u32, message: &str) -> Result { + (width as usize) + .checked_mul(height as usize) + .ok_or_else(|| Error::MetalKernel { + message: message.to_string(), + }) } #[cfg(target_os = "macos")] -fn encode_mct_rgb8_to_surface_in_command_buffer( +fn upload_cpu_decoded_coefficients( + runtime: &MetalRuntime, + mut coefficients: Vec, + retained_buffers: &mut Vec, + retained_cpu_coefficients: &mut Vec>, +) -> Buffer { + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_COEFFICIENT_UPLOAD); + let buffer = borrow_mut_slice_buffer(&runtime.device, &mut coefficients); + retained_buffers.push(buffer.clone()); + retained_cpu_coefficients.push(coefficients); + buffer +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn encode_prepared_direct_component_plane_in_command_buffer( runtime: &MetalRuntime, command_buffer: &CommandBufferRef, - planes: [&Buffer; 3], - dims: (u32, u32), - bit_depths: [u8; 3], - transform: J2kWaveletTransform, -) -> Surface { - let pitch_bytes = dims.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let out_buffer = runtime.device.new_buffer( - (pitch_bytes * dims.1 as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - let (max_values, u8_scales, _) = j2k_pack_scale_arrays([ - u32::from(bit_depths[0]), - u32::from(bit_depths[1]), - u32::from(bit_depths[2]), - 0, - ]); - let params = J2kMctRgb8PackParams { - width: dims.0, - height: dims.1, - out_stride: u32::try_from(pitch_bytes).expect("J2K Metal output stride fits in u32"), - transform: mct_transform_code(transform), - addends: [ - signed_sample_bias(bit_depths[0]), - signed_sample_bias(bit_depths[1]), - signed_sample_bias(bit_depths[2]), - ], - max_values: [max_values[0], max_values[1], max_values[2]], - u8_scales: [u8_scales[0], u8_scales[1], u8_scales[2]], - }; - + plan: &PreparedDirectGrayscalePlan, + tier1_mode: DirectTier1Mode, + stage_timings: &mut DirectHybridStageTimings, + retained_buffers: &mut Vec, + retained_cpu_coefficients: &mut Vec>, + status_checks: &mut Vec, + scratch_buffers: &mut Vec, +) -> Result { let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.pack_mct_rgb8); - encoder.set_buffer(0, Some(planes[0]), 0); - encoder.set_buffer(1, Some(planes[1]), 0); - encoder.set_buffer(2, Some(planes[2]), 0); - encoder.set_buffer(3, Some(&out_buffer), 0); - encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); + let result = (|| { + let mut bands = Vec::::new(); + let mut final_plane = None; + let mut step_idx = 0; + let profile_stages = metal_profile_stages_enabled(); - let width = runtime.pack_mct_rgb8.thread_execution_width().max(1); - let max_threads = runtime - .pack_mct_rgb8 - .max_total_threads_per_threadgroup() - .max(width); - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(dims.0), - height: u64::from(dims.1), - depth: 1, - }, - MTLSize { - width, - height, - depth: 1, - }, - ); - encoder.end_encoding(); + while step_idx < plan.steps.len() { + if let Some(group) = plan.classic_group_starting_at(step_idx) { + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(runtime, group.total_coefficients)?; + let (buffers, status_check) = + encode_prepared_classic_sub_band_group_to_buffer_in_encoder( + runtime, + encoder, + group, + &output.buffer, + scratch_buffers, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let decode_started = profile_stages.then(Instant::now); + let cpu_tier1_counters = + profile_stages.then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode_prepared_classic_sub_band_group_on_cpu_profile( + group, + cpu_tier1_counters.as_ref(), + )?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(stage_timings); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + }; + for member in &group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: buffer.clone(), + offset_bytes: member.offset_elements * size_of::(), + window: member.window, + }); + } + step_idx = group.end_step; + continue; + } + + if let Some(group) = plan.ht_group_starting_at(step_idx) { + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(runtime, group.total_coefficients)?; + let (buffers, status_check) = + encode_prepared_ht_sub_band_group_to_buffer_in_encoder( + runtime, + encoder, + group, + &output.buffer, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let decode_started = profile_stages.then(Instant::now); + let cpu_tier1_counters = + profile_stages.then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode_prepared_ht_sub_band_group_on_cpu_profile( + group, + cpu_tier1_counters.as_ref(), + )?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(stage_timings); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + }; + for member in &group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: buffer.clone(), + offset_bytes: member.offset_elements * size_of::(), + window: member.window, + }); + } + step_idx = group.end_step; + continue; + } + + match &plan.steps[step_idx] { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer( + runtime, + sub_band.width as usize * sub_band.height as usize, + )?; + let (buffers, status_check) = + encode_prepared_classic_sub_band_to_buffer_in_encoder( + runtime, + encoder, + sub_band, + &output.buffer, + scratch_buffers, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let decode_started = profile_stages.then(Instant::now); + let cpu_tier1_counters = + profile_stages.then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode_prepared_classic_sub_band_on_cpu_profile( + sub_band, + cpu_tier1_counters.as_ref(), + )?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(stage_timings); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + }; + bands.push(DirectBandSlice { + band_id: sub_band.band_id, + buffer, + offset_bytes: 0, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); + } + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer( + runtime, + sub_band.width as usize * sub_band.height as usize, + )?; + let (buffers, status_check) = + encode_prepared_ht_sub_band_to_buffer_in_encoder( + runtime, + encoder, + sub_band, + &output.buffer, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let decode_started = profile_stages.then(Instant::now); + let cpu_tier1_counters = + profile_stages.then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode_prepared_ht_sub_band_on_cpu_profile( + sub_band, + cpu_tier1_counters.as_ref(), + )?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(stage_timings); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + }; + bands.push(DirectBandSlice { + band_id: sub_band.band_id, + buffer, + offset_bytes: 0, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), + }); + } + PreparedDirectGrayscaleStep::Idwt(idwt) => { + let ll = + lookup_direct_band_slice_entry(&bands, idwt.step.ll_band_id, idwt.step.ll)?; + let hl = + lookup_direct_band_slice_entry(&bands, idwt.step.hl_band_id, idwt.step.hl)?; + let lh = + lookup_direct_band_slice_entry(&bands, idwt.step.lh_band_id, idwt.step.lh)?; + let hh = + lookup_direct_band_slice_entry(&bands, idwt.step.hh_band_id, idwt.step.hh)?; + let params = prepared_idwt_params( + idwt, + idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), + ); + let output = take_f32_scratch_buffer(runtime, prepared_idwt_output_len(idwt))?; + let encode_started = profile_stages.then(Instant::now); + match idwt.step.transform { + J2kWaveletTransform::Reversible53 => { + dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets( + runtime, + encoder, + &ll.buffer, + ll.offset_bytes, + &hl.buffer, + hl.offset_bytes, + &lh.buffer, + lh.offset_bytes, + &hh.buffer, + hh.offset_bytes, + params, + &output.buffer, + 0, + ); + } + J2kWaveletTransform::Irreversible97 => { + status_checks.push( + dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets( + runtime, + encoder, + &ll.buffer, + ll.offset_bytes, + &hl.buffer, + hl.offset_bytes, + &lh.buffer, + lh.offset_bytes, + &hh.buffer, + hh.offset_bytes, + params, + &output.buffer, + 0, + ), + ); + } + } + if let Some(started) = encode_started { + stage_timings.metal_idwt_encode += elapsed_us(started); + } + bands.push(DirectBandSlice { + band_id: idwt.step.output_band_id, + buffer: output.buffer.clone(), + offset_bytes: 0, + window: idwt.output_window, + }); + scratch_buffers.push(output); + } + PreparedDirectGrayscaleStep::Store(store) => { + let (input, input_offset) = + lookup_direct_band_slice(&bands, store.input_band_id, store.input_rect)?; + let output = take_f32_scratch_buffer( + runtime, + store.output_width as usize * store.output_height as usize, + )?; + let encode_started = profile_stages.then(Instant::now); + dispatch_store_component_buffer_in_encoder_with_offsets( + runtime, + encoder, + &input, + input_offset, + &output.buffer, + 0, + J2kStoreParams { + input_width: store.input_rect.width(), + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + }, + ); + if let Some(started) = encode_started { + stage_timings.metal_store_encode += elapsed_us(started); + } + final_plane = Some(output.buffer.clone()); + scratch_buffers.push(output); + } + } + step_idx += 1; + } - Surface::from_metal_buffer(out_buffer, dims, PixelFormat::Rgb8) + final_plane.ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect component plan did not produce a stored plane".to_string(), + }) + })(); + encoder.end_encoding(); + result } #[cfg(target_os = "macos")] -fn encode_batched_mct_rgb8_to_surfaces_in_command_buffer( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - planes: [&Buffer; 3], - dims: (u32, u32), +pub(crate) fn execute_repeated_prepared_direct_grayscale_plan( + plan: &PreparedDirectGrayscalePlan, + fmt: PixelFormat, count: usize, - bit_depths: [u8; 3], - transform: J2kWaveletTransform, ) -> Result, Error> { - let count_u32 = u32::try_from(count).map_err(|_| Error::MetalKernel { - message: "J2K MetalDirect color batch count exceeds u32".to_string(), - })?; - let pitch_bytes = dims.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let surface_bytes = - pitch_bytes - .checked_mul(dims.1 as usize) - .ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect color batch output size overflow".to_string(), - })?; - let total_bytes = surface_bytes - .checked_mul(count) - .ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect color batch output size overflow".to_string(), - })?; - let out_buffer = runtime - .device - .new_buffer(total_bytes as u64, MTLResourceOptions::StorageModeShared); - let plane_stride = dims - .0 - .checked_mul(dims.1) - .ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect color batch plane stride overflow".to_string(), - })?; - let (max_values, u8_scales, _) = j2k_pack_scale_arrays([ - u32::from(bit_depths[0]), - u32::from(bit_depths[1]), - u32::from(bit_depths[2]), - 0, - ]); - let params = J2kBatchedMctRgb8PackParams { - width: dims.0, - height: dims.1, - out_stride: u32::try_from(pitch_bytes).expect("J2K Metal output stride fits in u32"), - transform: mct_transform_code(transform), - batch_count: count_u32, - plane_stride, - output_stride: u32::try_from(surface_bytes).map_err(|_| Error::MetalKernel { - message: "J2K MetalDirect color batch surface stride exceeds u32".to_string(), - })?, - addends: [ - signed_sample_bias(bit_depths[0]), - signed_sample_bias(bit_depths[1]), - signed_sample_bias(bit_depths[2]), - ], - max_values: [max_values[0], max_values[1], max_values[2]], - u8_scales: [u8_scales[0], u8_scales[1], u8_scales[2]], - }; - - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.pack_mct_rgb8_batched); - encoder.set_buffer(0, Some(planes[0]), 0); - encoder.set_buffer(1, Some(planes[1]), 0); - encoder.set_buffer(2, Some(planes[2]), 0); - encoder.set_buffer(3, Some(&out_buffer), 0); - encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - - let width = runtime - .pack_mct_rgb8_batched - .thread_execution_width() - .max(1); - let max_threads = runtime - .pack_mct_rgb8_batched - .max_total_threads_per_threadgroup() - .max(width); - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(dims.0), - height: u64::from(dims.1), - depth: u64::from(count_u32), - }, - MTLSize { - width, - height, - depth: 1, - }, - ); - encoder.end_encoding(); - - Ok((0..count) - .map(|index| { - Surface::from_metal_buffer_with_offset( - out_buffer.clone(), - dims, - PixelFormat::Rgb8, - index * surface_bytes, - ) - }) - .collect()) + with_runtime(|runtime| { + let command_buffer = runtime.queue.new_command_buffer(); + let mut retained_buffers = Vec::new(); + let mut status_checks = Vec::new(); + let mut scratch_buffers = Vec::new(); + let surfaces = encode_repeated_direct_grayscale_plan_in_command_buffer( + runtime, + command_buffer, + plan, + fmt, + count, + &mut retained_buffers, + &mut status_checks, + &mut scratch_buffers, + )?; + command_buffer.commit(); + command_buffer.wait_until_completed(); + for status_check in status_checks { + validate_direct_status(status_check)?; + } + drop(retained_buffers); + recycle_scratch_buffers(runtime, scratch_buffers)?; + Ok(surfaces) + }) } #[cfg(target_os = "macos")] -fn mct_transform_code(transform: J2kWaveletTransform) -> u32 { - match transform { - J2kWaveletTransform::Reversible53 => 0, - J2kWaveletTransform::Irreversible97 => 1, - } +pub(crate) fn execute_prepared_direct_grayscale_plan( + plan: &PreparedDirectGrayscalePlan, + fmt: PixelFormat, +) -> Result { + with_runtime(|runtime| { + let command_buffer = runtime.queue.new_command_buffer(); + let mut retained_buffers = Vec::new(); + let mut status_checks = Vec::new(); + let mut scratch_buffers = Vec::new(); + let surface = encode_prepared_direct_grayscale_plan_in_command_buffer( + runtime, + command_buffer, + plan, + fmt, + &mut retained_buffers, + &mut status_checks, + &mut scratch_buffers, + )?; + command_buffer.commit(); + command_buffer.wait_until_completed(); + for status_check in status_checks { + validate_direct_status(status_check)?; + } + drop(retained_buffers); + recycle_scratch_buffers(runtime, scratch_buffers)?; + Ok(surface) + }) } #[cfg(target_os = "macos")] -fn prepare_classic_sub_band( - job: &signinum_j2k_native::J2kOwnedSubBandPlan, -) -> Result { - let mut jobs = Vec::with_capacity(job.jobs.len()); - let mut coded_data = Vec::new(); - let mut segments = Vec::new(); - - for block in &job.jobs { - let coded_offset = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect coded payload exceeds u32".to_string(), - })?; - coded_data.extend_from_slice(&block.data); - let segment_offset = u32::try_from(segments.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect segment table exceeds u32".to_string(), - })?; - for segment in &block.segments { - let data_offset = coded_offset - .checked_add(segment.data_offset) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect segment offset overflow".to_string(), - })?; - segments.push(J2kClassicSegment { - data_offset, - data_length: segment.data_length, - start_coding_pass: u32::from(segment.start_coding_pass), - end_coding_pass: u32::from(segment.end_coding_pass), - use_arithmetic: u32::from(segment.use_arithmetic), - }); - } - jobs.push(J2kClassicCleanupBatchJob { - coded_offset, - coded_len: u32::try_from(block.data.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect coded payload exceeds u32".to_string(), - })?, - segment_offset, - segment_count: u32::try_from(block.segments.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect segment count exceeds u32".to_string(), - })?, - width: block.width, - height: block.height, - output_stride: job.width, - output_offset: block - .output_y - .checked_mul(job.width) - .and_then(|row| row.checked_add(block.output_x)) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect output offset overflow".to_string(), - })?, - missing_msbs: u32::from(block.missing_bit_planes), - total_bitplanes: u32::from(block.total_bitplanes), - number_of_coding_passes: u32::from(block.number_of_coding_passes), - sub_band_type: match block.sub_band_type { - signinum_j2k_native::J2kSubBandType::LowLow => 0, - signinum_j2k_native::J2kSubBandType::HighLow => 1, - signinum_j2k_native::J2kSubBandType::LowHigh => 2, - signinum_j2k_native::J2kSubBandType::HighHigh => 3, - }, - style_flags: classic_style_flags(block.style), - strict: u32::from(block.strict), - dequantization_step: block.dequantization_step, - }); - } - - with_runtime(|runtime| { - let coded_buffer = borrow_slice_buffer(&runtime.device, &coded_data); - let jobs_buffer = borrow_slice_buffer(&runtime.device, &jobs); - let segments_buffer = borrow_slice_buffer(&runtime.device, &segments); - Ok(PreparedClassicSubBand { - band_id: job.band_id, - width: job.width, - height: job.height, - zero_fill: false, - coded_data, - coded_buffer, - jobs, - jobs_buffer, - segments, - segments_buffer, - }) +pub(crate) fn execute_prepared_direct_grayscale_plan_with_device( + plan: &PreparedDirectGrayscalePlan, + fmt: PixelFormat, + device: &Device, +) -> Result { + with_runtime_for_device(device, |_| { + execute_prepared_direct_grayscale_plan(plan, fmt) }) } #[cfg(target_os = "macos")] -fn prepare_classic_sub_band_groups( - steps: &[PreparedDirectGrayscaleStep], -) -> Result, Error> { - let mut groups = Vec::new(); - let mut step_idx = 0; - while step_idx < steps.len() { - let start_step = step_idx; - let mut sub_bands = Vec::new(); - while let Some(PreparedDirectGrayscaleStep::ClassicSubBand(sub_band)) = steps.get(step_idx) - { - sub_bands.push(sub_band); - step_idx += 1; - } - if sub_bands.len() > 1 { - groups.push(prepare_classic_sub_band_group( - start_step, step_idx, &sub_bands, - )?); - } - if step_idx == start_step { - step_idx += 1; - } +pub(crate) fn execute_prepared_direct_grayscale_plan_batch( + plans: &[Arc], + fmt: PixelFormat, +) -> Result, Error> { + if plans.is_empty() { + return Ok(Vec::new()); } - Ok(groups) -} -#[cfg(target_os = "macos")] -fn prepare_classic_sub_band_group( - start_step: usize, - end_step: usize, - sub_bands: &[&PreparedClassicSubBand], -) -> Result { - let mut members = Vec::with_capacity(sub_bands.len()); - let mut jobs = Vec::new(); - let mut segments = Vec::new(); - let mut coded_data = Vec::new(); - let mut output_base = 0usize; - - for sub_band in sub_bands { - members.push(PreparedClassicSubBandGroupMember { - band_id: sub_band.band_id, - offset_elements: output_base, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - - let coded_base = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect grouped coded payload exceeds u32".to_string(), - })?; - let segment_base = u32::try_from(segments.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect grouped segment table exceeds u32".to_string(), - })?; - let output_base_u32 = u32::try_from(output_base).map_err(|_| Error::MetalKernel { - message: "classic J2K MetalDirect grouped coefficient arena exceeds u32".to_string(), - })?; + with_runtime(|runtime| { + let command_buffer = runtime.queue.new_command_buffer(); + let mut retained_buffers = Vec::new(); + let mut retained_cpu_coefficients = Vec::>::new(); + let mut status_checks = Vec::new(); + let mut scratch_buffers = Vec::new(); + let mut stage_timings = DirectHybridStageTimings::default(); + let mut surfaces = Vec::with_capacity(plans.len()); - for segment in &sub_band.segments { - let mut grouped_segment = *segment; - grouped_segment.data_offset = - coded_base - .checked_add(segment.data_offset) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect grouped segment offset overflow" - .to_string(), - })?; - segments.push(grouped_segment); + let component_plan_refs = plans.iter().map(Arc::as_ref).collect::>(); + if plans.len() > 1 && supports_stacked_direct_component_plane_batch(&component_plan_refs) { + let stacked_plane = encode_stacked_direct_component_plane_batch( + runtime, + DirectColorBatchCommandBuffers::single(command_buffer), + &component_plan_refs, + 0, + None, + DirectTier1Mode::Metal, + &mut stage_timings, + &mut retained_buffers, + &mut retained_cpu_coefficients, + &mut status_checks, + &mut scratch_buffers, + )?; + let first = plans.first().expect("plans is not empty"); + if stacked_plane.dimensions == first.dimensions && stacked_plane.count == plans.len() { + surfaces = encode_repeated_gray_plane_to_surfaces_in_command_buffer( + runtime, + command_buffer, + &stacked_plane.buffer, + first.dimensions, + first.bit_depth, + fmt, + plans.len(), + )?; + } } - for job in &sub_band.jobs { - let mut grouped_job = *job; - grouped_job.coded_offset = - coded_base - .checked_add(job.coded_offset) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect grouped job coded offset overflow" - .to_string(), - })?; - grouped_job.segment_offset = - segment_base - .checked_add(job.segment_offset) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect grouped job segment offset overflow" - .to_string(), - })?; - grouped_job.output_offset = - output_base_u32 - .checked_add(job.output_offset) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect grouped output offset overflow" - .to_string(), - })?; - jobs.push(grouped_job); + for plan in plans { + if !surfaces.is_empty() { + break; + } + surfaces.push(encode_prepared_direct_grayscale_plan_in_command_buffer( + runtime, + command_buffer, + plan, + fmt, + &mut retained_buffers, + &mut status_checks, + &mut scratch_buffers, + )?); } - coded_data.extend_from_slice(&sub_band.coded_data); - let sub_band_len = - sub_band - .width - .checked_mul(sub_band.height) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect grouped sub-band size overflow".to_string(), - })? as usize; - output_base = output_base - .checked_add(sub_band_len) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect grouped coefficient arena overflow".to_string(), - })?; - } + command_buffer.commit(); + command_buffer.wait_until_completed(); + for status_check in status_checks { + validate_direct_status(status_check)?; + } + drop(retained_buffers); + drop(retained_cpu_coefficients); + recycle_scratch_buffers(runtime, scratch_buffers)?; + Ok(surfaces) + }) +} - with_runtime(|runtime| { - let coded_buffer = borrow_slice_buffer(&runtime.device, &coded_data); - let jobs_buffer = borrow_slice_buffer(&runtime.device, &jobs); - let segments_buffer = borrow_slice_buffer(&runtime.device, &segments); - Ok(PreparedClassicSubBandGroup { - start_step, - end_step, - total_coefficients: output_base, - zero_fill: sub_bands.iter().any(|sub_band| sub_band.zero_fill), - coded_data, - coded_buffer, - jobs, - jobs_buffer, - segments, - segments_buffer, - members, - }) +#[cfg(target_os = "macos")] +pub(crate) fn execute_prepared_direct_color_plan( + plan: &PreparedDirectColorPlan, + fmt: PixelFormat, +) -> Result { + let plans = [Arc::new(plan.clone())]; + let mut surfaces = execute_prepared_direct_color_plan_batch(&plans, fmt)?; + surfaces.pop().ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect color plan produced no surface".to_string(), }) } #[cfg(target_os = "macos")] -fn prepare_ht_sub_band( - job: &signinum_j2k_native::HtOwnedSubBandPlan, -) -> Result { - let mut jobs = Vec::with_capacity(job.jobs.len()); - let mut coded_data = Vec::new(); - for block in &job.jobs { - let coded_offset = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect coded payload exceeds u32".to_string(), - })?; - coded_data.extend_from_slice(&block.data); - jobs.push(J2kHtCleanupBatchJob { - coded_offset, - width: block.width, - height: block.height, - coded_len: u32::try_from(block.data.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect coded payload exceeds u32".to_string(), - })?, - cleanup_length: block.cleanup_length, - refinement_length: block.refinement_length, - missing_msbs: u32::from(block.missing_bit_planes), - num_bitplanes: u32::from(block.num_bitplanes), - number_of_coding_passes: u32::from(block.number_of_coding_passes), - output_stride: job.width, - output_offset: block - .output_y - .checked_mul(job.width) - .and_then(|row| row.checked_add(block.output_x)) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect output offset overflow".to_string(), - })?, - dequantization_step: block.dequantization_step, - stripe_causal: u32::from(block.stripe_causal), - }); - } +pub(crate) fn execute_prepared_direct_color_plan_with_device( + plan: &PreparedDirectColorPlan, + fmt: PixelFormat, + device: &Device, +) -> Result { + with_runtime_for_device(device, |_| execute_prepared_direct_color_plan(plan, fmt)) +} - with_runtime(|runtime| { - let coded_buffer = borrow_slice_buffer(&runtime.device, &coded_data); - let jobs_buffer = borrow_slice_buffer(&runtime.device, &jobs); - Ok(PreparedHtSubBand { - band_id: job.band_id, - width: job.width, - height: job.height, - coded_data, - coded_buffer, - jobs, - jobs_buffer, - }) +#[cfg(target_os = "macos")] +pub(crate) fn execute_prepared_direct_color_plan_batch( + plans: &[Arc], + fmt: PixelFormat, +) -> Result, Error> { + execute_direct_color_plan_batch_with_tier1(plans, fmt, DirectTier1Mode::Metal) +} + +#[cfg(target_os = "macos")] +pub(crate) fn execute_hybrid_cpu_tier1_direct_color_plan( + plan: &PreparedDirectColorPlan, + fmt: PixelFormat, +) -> Result { + let plans = [Arc::new(plan.clone())]; + let mut surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt)?; + surfaces.pop().ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect hybrid color plan produced no surface".to_string(), }) } #[cfg(target_os = "macos")] -fn prepare_ht_sub_band_groups( - steps: &[PreparedDirectGrayscaleStep], -) -> Result, Error> { - let mut groups = Vec::new(); - let mut step_idx = 0; - while step_idx < steps.len() { - let start_step = step_idx; - let mut sub_bands = Vec::new(); - while let Some(PreparedDirectGrayscaleStep::HtSubBand(sub_band)) = steps.get(step_idx) { - sub_bands.push(sub_band); - step_idx += 1; - } - if sub_bands.len() > 1 { - groups.push(prepare_ht_sub_band_group(start_step, step_idx, &sub_bands)?); - } - if step_idx == start_step { - step_idx += 1; - } - } - Ok(groups) +pub(crate) fn execute_hybrid_cpu_tier1_direct_color_plan_with_device( + plan: &PreparedDirectColorPlan, + fmt: PixelFormat, + device: &Device, +) -> Result { + with_runtime_for_device(device, |_| { + execute_hybrid_cpu_tier1_direct_color_plan(plan, fmt) + }) } #[cfg(target_os = "macos")] -fn prepare_ht_sub_band_group( - start_step: usize, - end_step: usize, - sub_bands: &[&PreparedHtSubBand], -) -> Result { - let mut members = Vec::with_capacity(sub_bands.len()); - let mut jobs = Vec::new(); - let mut coded_data = Vec::new(); - let mut output_base = 0usize; +pub(crate) fn execute_hybrid_cpu_tier1_direct_color_plan_batch( + plans: &[Arc], + fmt: PixelFormat, +) -> Result, Error> { + execute_direct_color_plan_batch_with_tier1(plans, fmt, DirectTier1Mode::CpuUpload) +} - for sub_band in sub_bands { - members.push(PreparedHtSubBandGroupMember { - band_id: sub_band.band_id, - offset_elements: output_base, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); +#[cfg(target_os = "macos")] +fn execute_direct_color_plan_batch_with_tier1( + plans: &[Arc], + fmt: PixelFormat, + tier1_mode: DirectTier1Mode, +) -> Result, Error> { + execute_direct_color_plan_batch_with_tier1_options(plans, fmt, tier1_mode, false) +} - let coded_base = u32::try_from(coded_data.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect grouped coded payload exceeds u32".to_string(), - })?; - let output_base_u32 = u32::try_from(output_base).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect grouped coefficient arena exceeds u32".to_string(), - })?; - for job in &sub_band.jobs { - let mut grouped_job = *job; - grouped_job.coded_offset = - coded_base - .checked_add(job.coded_offset) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect grouped coded offset overflow".to_string(), - })?; - grouped_job.output_offset = - output_base_u32 - .checked_add(job.output_offset) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect grouped output offset overflow".to_string(), - })?; - jobs.push(grouped_job); - } - coded_data.extend_from_slice(&sub_band.coded_data); - let sub_band_len = - sub_band - .width - .checked_mul(sub_band.height) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect grouped sub-band size overflow".to_string(), - })? as usize; - output_base = output_base - .checked_add(sub_band_len) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect grouped coefficient arena overflow".to_string(), - })?; +#[cfg(all(target_os = "macos", test))] +fn execute_flattened_hybrid_cpu_tier1_direct_color_plan_batch_for_test( + plans: &[Arc], + fmt: PixelFormat, +) -> Result, Error> { + execute_direct_color_plan_batch_with_tier1_options(plans, fmt, DirectTier1Mode::CpuUpload, true) +} + +#[cfg(target_os = "macos")] +fn execute_direct_color_plan_batch_with_tier1_options( + plans: &[Arc], + fmt: PixelFormat, + tier1_mode: DirectTier1Mode, + force_flattened_cpu_tier1: bool, +) -> Result, Error> { + if plans.is_empty() { + return Ok(Vec::new()); + } + if tier1_mode == DirectTier1Mode::Metal + && plans + .iter() + .any(|plan| !prepared_direct_color_plan_supports_runtime(plan, fmt)) + { + return Err(Error::MetalKernel { + message: "unsupported classic kernel input in direct component plan".to_string(), + }); } with_runtime(|runtime| { - let coded_buffer = borrow_slice_buffer(&runtime.device, &coded_data); - let jobs_buffer = borrow_slice_buffer(&runtime.device, &jobs); - Ok(PreparedHtSubBandGroup { - start_step, - end_step, - total_coefficients: output_base, - coded_data, - coded_buffer, - jobs, - jobs_buffer, - members, - }) - }) -} + let mut retained_buffers = Vec::new(); + let mut retained_cpu_coefficients = Vec::>::new(); + let mut status_checks = Vec::new(); + let mut scratch_buffers = Vec::new(); + let mut stage_timings = DirectHybridStageTimings::default(); + let profile_hybrid_stages = + tier1_mode == DirectTier1Mode::CpuUpload && metal_profile_stages_enabled(); -#[cfg(target_os = "macos")] -pub(crate) fn prepare_direct_grayscale_plan( - plan: &J2kDirectGrayscalePlan, -) -> Result { - let mut steps = Vec::with_capacity(plan.steps.len()); - for step in &plan.steps { - match step { - J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { - steps.push(PreparedDirectGrayscaleStep::ClassicSubBand( - prepare_classic_sub_band(sub_band)?, - )); - } - J2kDirectGrayscaleStep::HtSubBand(sub_band) => { - steps.push(PreparedDirectGrayscaleStep::HtSubBand(prepare_ht_sub_band( - sub_band, - )?)); + if fmt == PixelFormat::Rgb8 + && profile_hybrid_stages + && metal_profile_decode_split_commands_enabled() + { + let split_command_buffers = DecodeHybridSplitCommandBuffers::new(runtime); + if let Some(surfaces) = try_encode_stacked_mct_rgb8_direct_color_batch( + runtime, + split_command_buffers.refs(), + plans, + tier1_mode, + force_flattened_cpu_tier1, + &mut stage_timings, + &mut retained_buffers, + &mut retained_cpu_coefficients, + &mut status_checks, + &mut scratch_buffers, + )? { + split_command_buffers.commit_in_order(); + let wait_started = Instant::now(); + let _wait_signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_COMMAND_WAIT); + split_command_buffers.mct_pack.wait_until_completed(); + stage_timings.command_wait += elapsed_us(wait_started); + record_completed_decode_split_gpu_stages( + &mut stage_timings, + &split_command_buffers, + ); + for status_check in status_checks { + validate_direct_status(status_check)?; + } + emit_direct_hybrid_stage_timings(&stage_timings, fmt, plans.len()); + drop(retained_buffers); + drop(retained_cpu_coefficients); + recycle_scratch_buffers(runtime, scratch_buffers)?; + return Ok(surfaces); } - J2kDirectGrayscaleStep::Idwt(idwt) => { - steps.push(PreparedDirectGrayscaleStep::Idwt(PreparedDirectIdwt { - step: *idwt, - output_window: BandRequiredRegion::full(idwt.rect.width(), idwt.rect.height()), - })); + + drop(split_command_buffers); + retained_buffers.clear(); + retained_cpu_coefficients.clear(); + status_checks.clear(); + scratch_buffers.clear(); + stage_timings = DirectHybridStageTimings::default(); + } + + let command_buffer = runtime.queue.new_command_buffer(); + if profile_hybrid_stages { + label_command_buffer(command_buffer, "j2k decode hybrid direct color batch"); + } + + if fmt == PixelFormat::Rgb8 { + if let Some(surfaces) = try_encode_stacked_mct_rgb8_direct_color_batch( + runtime, + DirectColorBatchCommandBuffers::single(command_buffer), + plans, + tier1_mode, + force_flattened_cpu_tier1, + &mut stage_timings, + &mut retained_buffers, + &mut retained_cpu_coefficients, + &mut status_checks, + &mut scratch_buffers, + )? { + command_buffer.commit(); + let wait_started = profile_hybrid_stages.then(Instant::now); + let _wait_signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_COMMAND_WAIT); + command_buffer.wait_until_completed(); + if let Some(started) = wait_started { + stage_timings.command_wait += elapsed_us(started); + } + if profile_hybrid_stages { + if let Some(duration) = completed_command_buffer_gpu_duration(command_buffer) { + stage_timings.gpu_command += duration.as_micros(); + } + } + for status_check in status_checks { + validate_direct_status(status_check)?; + } + if tier1_mode == DirectTier1Mode::CpuUpload { + emit_direct_hybrid_stage_timings(&stage_timings, fmt, plans.len()); + } + drop(retained_buffers); + drop(retained_cpu_coefficients); + recycle_scratch_buffers(runtime, scratch_buffers)?; + return Ok(surfaces); } - J2kDirectGrayscaleStep::Store(store) => { - steps.push(PreparedDirectGrayscaleStep::Store(*store)); + } + + let mut surfaces = Vec::with_capacity(plans.len()); + + for plan in plans { + let surface = encode_prepared_direct_color_plan_in_command_buffer( + runtime, + command_buffer, + plan, + fmt, + tier1_mode, + &mut stage_timings, + &mut retained_buffers, + &mut retained_cpu_coefficients, + &mut status_checks, + &mut scratch_buffers, + )?; + surfaces.push(surface); + } + + command_buffer.commit(); + let wait_started = profile_hybrid_stages.then(Instant::now); + let _wait_signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_COMMAND_WAIT); + command_buffer.wait_until_completed(); + if let Some(started) = wait_started { + stage_timings.command_wait += elapsed_us(started); + } + if profile_hybrid_stages { + if let Some(duration) = completed_command_buffer_gpu_duration(command_buffer) { + stage_timings.gpu_command += duration.as_micros(); } } - } - let classic_groups = prepare_classic_sub_band_groups(&steps)?; - let ht_groups = prepare_ht_sub_band_groups(&steps)?; - Ok(PreparedDirectGrayscalePlan { - dimensions: plan.dimensions, - bit_depth: plan.bit_depth, - steps, - classic_groups, - ht_groups, + for status_check in status_checks { + validate_direct_status(status_check)?; + } + if tier1_mode == DirectTier1Mode::CpuUpload { + emit_direct_hybrid_stage_timings(&stage_timings, fmt, plans.len()); + } + drop(retained_buffers); + drop(retained_cpu_coefficients); + recycle_scratch_buffers(runtime, scratch_buffers)?; + Ok(surfaces) }) } #[cfg(target_os = "macos")] -pub(crate) fn crop_prepared_direct_grayscale_plan_to_output_region( - plan: &mut PreparedDirectGrayscalePlan, - region: Rect, -) -> Result<(), Error> { - if region.w == 0 || region.h == 0 { +fn signed_sample_bias(bit_depth: u8) -> f32 { + 2.0_f32.powi(i32::from(bit_depth) - 1) +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn encode_prepared_direct_color_plan_in_command_buffer( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + plan: &PreparedDirectColorPlan, + fmt: PixelFormat, + tier1_mode: DirectTier1Mode, + stage_timings: &mut DirectHybridStageTimings, + retained_buffers: &mut Vec, + retained_cpu_coefficients: &mut Vec>, + status_checks: &mut Vec, + scratch_buffers: &mut Vec, +) -> Result { + if plan.component_plans.len() != 3 { return Err(Error::MetalKernel { - message: "J2K MetalDirect region-scaled grayscale plan has an empty output region" - .to_string(), + message: format!( + "J2K MetalDirect color execution expected 3 component plans, got {}", + plan.component_plans.len() + ), }); } - if region.x == 0 - && region.y == 0 - && region.w == plan.dimensions.0 - && region.h == plan.dimensions.1 - { - return Ok(()); + + let mut planes = Vec::with_capacity(3); + for component_plan in &plan.component_plans { + planes.push(encode_prepared_direct_component_plane_in_command_buffer( + runtime, + command_buffer, + component_plan, + tier1_mode, + stage_timings, + retained_buffers, + retained_cpu_coefficients, + status_checks, + scratch_buffers, + )?); } - let mut store_count = 0; - for step in &mut plan.steps { - if let PreparedDirectGrayscaleStep::Store(store) = step { - crop_direct_store_step_to_output_region(store, region)?; - store_count += 1; + if plan.mct && fmt == PixelFormat::Rgb8 { + let encode_started = metal_profile_stages_enabled().then(Instant::now); + let surface = encode_mct_rgb8_to_surface_in_command_buffer( + runtime, + command_buffer, + [&planes[0], &planes[1], &planes[2]], + plan.dimensions, + plan.bit_depths, + plan.transform, + )?; + if let Some(started) = encode_started { + stage_timings.metal_mct_pack_encode += elapsed_us(started); } + return Ok(surface); } - if store_count == 0 { - return Err(Error::MetalKernel { - message: "J2K MetalDirect grayscale plan has no store step to crop".to_string(), - }); + if plan.mct { + let len = plan.dimensions.0 as usize * plan.dimensions.1 as usize; + let encode_started = metal_profile_stages_enabled().then(Instant::now); + status_checks.push(dispatch_inverse_mct_buffers_in_command_buffer( + runtime, + command_buffer, + [&planes[0], &planes[1], &planes[2]], + len, + plan.transform, + [ + signed_sample_bias(plan.bit_depths[0]), + signed_sample_bias(plan.bit_depths[1]), + signed_sample_bias(plan.bit_depths[2]), + ], + )?); + if let Some(started) = encode_started { + stage_timings.metal_mct_pack_encode += elapsed_us(started); + } } - prune_prepared_direct_grayscale_plan_to_store_windows(plan)?; - plan.dimensions = (region.w, region.h); - Ok(()) + let stage = PlaneStage { + dims: plan.dimensions, + plane_count: 3, + color_space: NativeColorSpace::RGB, + has_alpha: false, + bit_depths: [ + u32::from(plan.bit_depths[0]), + u32::from(plan.bit_depths[1]), + u32::from(plan.bit_depths[2]), + 0, + ], + planes: [ + Some(planes[0].clone()), + Some(planes[1].clone()), + Some(planes[2].clone()), + None, + ], + }; + let encode_started = metal_profile_stages_enabled().then(Instant::now); + let surface = + encode_plane_stage_to_surface_in_command_buffer(runtime, command_buffer, &stage, fmt); + if let Some(started) = encode_started { + stage_timings.metal_mct_pack_encode += elapsed_us(started); + } + surface } #[cfg(target_os = "macos")] -#[derive(Clone, Copy, Debug)] -struct BandRequiredRegion { - x0: u32, - y0: u32, - x1: u32, - y1: u32, +#[derive(Clone)] +struct DirectBandSlice { + band_id: J2kDirectBandId, + buffer: Buffer, + offset_bytes: usize, + window: BandRequiredRegion, } #[cfg(target_os = "macos")] -impl BandRequiredRegion { - fn full(width: u32, height: u32) -> Self { - Self { - x0: 0, - y0: 0, - x1: width, - y1: height, - } - } +fn lookup_direct_band_slice_entry( + bands: &[DirectBandSlice], + band_id: J2kDirectBandId, + rect: j2k_native::J2kRect, +) -> Result { + bands + .iter() + .find(|existing| existing.band_id == band_id) + .cloned() + .ok_or_else(|| Error::MetalKernel { + message: format!( + "missing J2K MetalDirect device band {} for rect ({}, {}, {}, {})", + band_id, rect.x0, rect.y0, rect.x1, rect.y1 + ), + }) +} - fn new(x0: u32, y0: u32, x1: u32, y1: u32) -> Option { - (x0 < x1 && y0 < y1).then_some(Self { x0, y0, x1, y1 }) - } +#[cfg(target_os = "macos")] +fn lookup_direct_band_slice( + bands: &[DirectBandSlice], + band_id: J2kDirectBandId, + rect: j2k_native::J2kRect, +) -> Result<(Buffer, usize), Error> { + let entry = lookup_direct_band_slice_entry(bands, band_id, rect)?; + Ok((entry.buffer, entry.offset_bytes)) +} - fn width(self) -> u32 { - self.x1 - self.x0 +#[cfg(target_os = "macos")] +fn lookup_repeated_direct_band_layout_entry( + band_sets: &[Vec], + band_id: J2kDirectBandId, + rect: j2k_native::J2kRect, +) -> Result<(DirectBandSlice, u32), Error> { + let first_bands = band_sets.first().ok_or_else(|| Error::MetalKernel { + message: "missing J2K MetalDirect repeated band set".to_string(), + })?; + let entry = lookup_direct_band_slice_entry(first_bands, band_id, rect)?; + let stride_bytes = if let Some(second_bands) = band_sets.get(1) { + let next = lookup_direct_band_slice_entry(second_bands, band_id, rect)?; + next.offset_bytes + .checked_sub(entry.offset_bytes) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect repeated band offsets are not monotonic".to_string(), + })? + } else { + entry.window.width() as usize * entry.window.height() as usize * size_of::() + }; + if stride_bytes % size_of::() != 0 { + return Err(Error::MetalKernel { + message: "J2K MetalDirect repeated band stride is not f32-aligned".to_string(), + }); } + let stride_elements = + u32::try_from(stride_bytes / size_of::()).map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect repeated band stride exceeds u32".to_string(), + })?; + Ok((entry, stride_elements)) +} - fn height(self) -> u32 { - self.y1 - self.y0 +#[cfg(target_os = "macos")] +struct StackedDirectComponentPlane { + buffer: Buffer, + dimensions: (u32, u32), + count: usize, +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn try_encode_stacked_mct_rgb8_direct_color_batch( + runtime: &MetalRuntime, + command_buffers: DirectColorBatchCommandBuffers<'_>, + plans: &[Arc], + tier1_mode: DirectTier1Mode, + force_flattened_cpu_tier1: bool, + stage_timings: &mut DirectHybridStageTimings, + retained_buffers: &mut Vec, + retained_cpu_coefficients: &mut Vec>, + status_checks: &mut Vec, + scratch_buffers: &mut Vec, +) -> Result>, Error> { + let Some(first) = plans.first() else { + return Ok(Some(Vec::new())); + }; + let repeated_count = repeated_shared_direct_color_plan_count(plans); + if plans.len() <= 1 + || !first.mct + || first.component_plans.len() != 3 + || !plans.iter().all(|plan| { + plan.mct + && plan.dimensions == first.dimensions + && plan.bit_depths == first.bit_depths + && plan.transform == first.transform + && plan.component_plans.len() == 3 + }) + { + return Ok(None); } + let execution_plans = if repeated_count.is_some() { + &plans[..1] + } else { + plans + }; - fn expanded(self, margin: u32, width: u32, height: u32) -> Self { - Self { - x0: self.x0.saturating_sub(margin), - y0: self.y0.saturating_sub(margin), - x1: self.x1.saturating_add(margin).min(width), - y1: self.y1.saturating_add(margin).min(height), + let flattened_cpu_tier1_cache = if tier1_mode == DirectTier1Mode::CpuUpload + && (force_flattened_cpu_tier1 + || flattened_hybrid_cpu_tier1_enabled() + || should_flatten_hybrid_cpu_tier1_color_batch(execution_plans)) + { + Some(build_flattened_cpu_tier1_cache( + runtime, + execution_plans, + stage_timings, + retained_buffers, + retained_cpu_coefficients, + )?) + } else { + None + }; + + let mut stacked_planes = Vec::with_capacity(3); + for component_idx in 0..3 { + let component_plan_refs = execution_plans + .iter() + .map(|plan| &plan.component_plans[component_idx]) + .collect::>(); + if !supports_stacked_direct_component_plane_batch(&component_plan_refs) { + return Ok(None); } + stacked_planes.push(encode_stacked_direct_component_plane_batch( + runtime, + command_buffers, + &component_plan_refs, + component_idx, + flattened_cpu_tier1_cache.as_ref(), + tier1_mode, + stage_timings, + retained_buffers, + retained_cpu_coefficients, + status_checks, + scratch_buffers, + )?); } - fn union(self, other: Self) -> Self { - Self { - x0: self.x0.min(other.x0), - y0: self.y0.min(other.y0), - x1: self.x1.max(other.x1), - y1: self.y1.max(other.y1), - } + if !stacked_planes + .iter() + .all(|plane| plane.dimensions == first.dimensions && plane.count == execution_plans.len()) + { + return Ok(None); } - fn intersects(self, x0: u32, y0: u32, width: u32, height: u32) -> bool { - let x1 = x0.saturating_add(width); - let y1 = y0.saturating_add(height); - self.x0 < x1 && x0 < self.x1 && self.y0 < y1 && y0 < self.y1 + let encode_started = metal_profile_stages_enabled().then(Instant::now); + let mct_plane_buffers = [ + &stacked_planes[0].buffer, + &stacked_planes[1].buffer, + &stacked_planes[2].buffer, + ]; + let surfaces = if let Some(count) = repeated_count { + encode_repeated_mct_rgb8_to_surfaces_in_command_buffer( + runtime, + command_buffers.mct_pack, + mct_plane_buffers, + first.dimensions, + count, + first.bit_depths, + first.transform, + )? + } else { + encode_batched_mct_rgb8_to_surfaces_in_command_buffer( + runtime, + command_buffers.mct_pack, + mct_plane_buffers, + first.dimensions, + execution_plans.len(), + first.bit_depths, + first.transform, + )? + }; + if let Some(started) = encode_started { + stage_timings.metal_mct_pack_encode += elapsed_us(started); } + Ok(Some(surfaces)) } #[cfg(target_os = "macos")] -fn prune_prepared_direct_grayscale_plan_to_store_windows( - plan: &mut PreparedDirectGrayscalePlan, -) -> Result<(), Error> { - let mut required = HashMap::::new(); - for step in &plan.steps { - if let PreparedDirectGrayscaleStep::Store(store) = step { - let source_right = store - .source_x - .checked_add(store.copy_width) - .ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect ROI source width overflows u32".to_string(), - })?; - let source_bottom = store - .source_y - .checked_add(store.copy_height) - .ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect ROI source height overflows u32".to_string(), - })?; - if let Some(region) = - BandRequiredRegion::new(store.source_x, store.source_y, source_right, source_bottom) +fn supports_stacked_direct_component_plane_batch(plans: &[&PreparedDirectGrayscalePlan]) -> bool { + let Some(first) = plans.first() else { + return false; + }; + if plans.iter().any(|plan| { + plan.dimensions != first.dimensions + || plan.bit_depth != first.bit_depth + || plan.steps.len() != first.steps.len() + }) { + return false; + } + + let mut step_idx = 0; + while step_idx < first.steps.len() { + if let Some(group) = first.classic_group_starting_at(step_idx) { + if group.end_step <= step_idx + || !plans.iter().all(|plan| { + plan.classic_group_starting_at(step_idx) + .is_some_and(|other| classic_group_shapes_match(group, other)) + }) { - add_required_region(&mut required, store.input_band_id, region); + return false; } + step_idx = group.end_step; + continue; } - } - - let mut idwt_output_windows = HashMap::::new(); - for step in plan.steps.iter().rev() { - if let PreparedDirectGrayscaleStep::Idwt(idwt) = step { - let Some(output_region) = required.get(&idwt.step.output_band_id).copied() else { - continue; - }; - let expanded = output_region.expanded( - idwt_required_output_margin(idwt.step.transform), - idwt.step.rect.width(), - idwt.step.rect.height(), - ); - idwt_output_windows.insert(idwt.step.output_band_id, expanded); - add_idwt_input_required_regions(&mut required, &idwt.step, expanded); + if let Some(group) = first.ht_group_starting_at(step_idx) { + if group.end_step <= step_idx + || !plans.iter().all(|plan| { + plan.ht_group_starting_at(step_idx) + .is_some_and(|other| ht_group_shapes_match(group, other)) + }) + { + return false; + } + step_idx = group.end_step; + continue; } - } - for step in &mut plan.steps { - match step { + match &first.steps[step_idx] { PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { - let before = sub_band.jobs.len(); - retain_classic_jobs_for_required_region( - &mut sub_band.jobs, - required.get(&sub_band.band_id).copied(), - ); - if sub_band.jobs.len() != before { - sub_band.zero_fill = true; - with_runtime(|runtime| { - sub_band.jobs_buffer = borrow_slice_buffer(&runtime.device, &sub_band.jobs); - Ok(()) - })?; + if !plans.iter().all(|plan| { + matches!( + &plan.steps[step_idx], + PreparedDirectGrayscaleStep::ClassicSubBand(other) + if classic_sub_band_shapes_match(sub_band, other) + ) + }) { + return false; } } PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { - let before = sub_band.jobs.len(); - retain_ht_jobs_for_required_region( - &mut sub_band.jobs, - required.get(&sub_band.band_id).copied(), - ); - if sub_band.jobs.len() != before { - compact_ht_sub_band_coded_data(sub_band)?; + if !plans.iter().all(|plan| { + matches!( + &plan.steps[step_idx], + PreparedDirectGrayscaleStep::HtSubBand(other) + if ht_sub_band_shapes_match(sub_band, other) + ) + }) { + return false; + } + } + PreparedDirectGrayscaleStep::Idwt(idwt) => { + if !plans.iter().all(|plan| { + matches!( + &plan.steps[step_idx], + PreparedDirectGrayscaleStep::Idwt(other) + if idwt_shapes_match(idwt, other) + ) + }) { + return false; + } + } + PreparedDirectGrayscaleStep::Store(store) => { + if !plans.iter().all(|plan| { + matches!( + &plan.steps[step_idx], + PreparedDirectGrayscaleStep::Store(other) + if store_shapes_match(store, other) + ) + }) { + return false; } } - PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => {} } + step_idx += 1; } - apply_prepared_direct_idwt_output_windows(plan, &idwt_output_windows)?; - plan.classic_groups = prepare_classic_sub_band_groups(&plan.steps)?; - plan.ht_groups = prepare_ht_sub_band_groups(&plan.steps)?; - Ok(()) + true } #[cfg(target_os = "macos")] -fn apply_prepared_direct_idwt_output_windows( - plan: &mut PreparedDirectGrayscalePlan, - windows: &HashMap, -) -> Result<(), Error> { - for step in &mut plan.steps { - if let PreparedDirectGrayscaleStep::Idwt(idwt) = step { - idwt.output_window = windows - .get(&idwt.step.output_band_id) - .copied() - .unwrap_or_else(|| { - BandRequiredRegion::full(idwt.step.rect.width(), idwt.step.rect.height()) - }); - } - } - - for step in &mut plan.steps { - let PreparedDirectGrayscaleStep::Store(store) = step else { - continue; - }; - let Some(window) = windows.get(&store.input_band_id).copied() else { - continue; - }; - - store.source_x = - store - .source_x - .checked_sub(window.x0) - .ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect cropped IDWT store source x underflow".to_string(), - })?; - store.source_y = - store - .source_y - .checked_sub(window.y0) - .ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect cropped IDWT store source y underflow".to_string(), - })?; - store.input_rect = signinum_j2k_native::J2kRect { - x0: store.input_rect.x0.saturating_add(window.x0), - y0: store.input_rect.y0.saturating_add(window.y0), - x1: store.input_rect.x0.saturating_add(window.x1), - y1: store.input_rect.y0.saturating_add(window.y1), - }; - } - - Ok(()) -} +#[allow(clippy::too_many_arguments)] +fn encode_stacked_direct_component_plane_batch( + runtime: &MetalRuntime, + command_buffers: DirectColorBatchCommandBuffers<'_>, + plans: &[&PreparedDirectGrayscalePlan], + component_idx: usize, + flattened_cpu_tier1_cache: Option<&FlattenedCpuTier1Cache>, + tier1_mode: DirectTier1Mode, + stage_timings: &mut DirectHybridStageTimings, + retained_buffers: &mut Vec, + retained_cpu_coefficients: &mut Vec>, + status_checks: &mut Vec, + scratch_buffers: &mut Vec, +) -> Result { + let Some(first) = plans.first() else { + return Err(Error::MetalKernel { + message: "J2K MetalDirect color batch has no component plans".to_string(), + }); + }; -#[cfg(target_os = "macos")] -#[derive(Clone, Copy)] -struct PreparedIdwtInputWindows { - ll: BandRequiredRegion, - hl: BandRequiredRegion, - lh: BandRequiredRegion, - hh: BandRequiredRegion, -} + let count = plans.len(); + let broadcast_tier1_inputs = tier1_mode == DirectTier1Mode::CpuUpload + && plans.iter().all(|plan| std::ptr::eq(*plan, *first)); + let mut band_sets = vec![Vec::::new(); count]; + let mut final_plane = None; + let mut step_idx = 0; + let profile_stages = tier1_mode == DirectTier1Mode::CpuUpload && metal_profile_stages_enabled(); -fn idwt_input_windows_from_slices( - ll: &DirectBandSlice, - hl: &DirectBandSlice, - lh: &DirectBandSlice, - hh: &DirectBandSlice, -) -> PreparedIdwtInputWindows { - PreparedIdwtInputWindows { - ll: ll.window, - hl: hl.window, - lh: lh.window, - hh: hh.window, - } -} + while step_idx < first.steps.len() { + if let Some(group) = first.classic_group_starting_at(step_idx) { + let groups = plans + .iter() + .map(|plan| { + plan.classic_group_starting_at(step_idx) + .expect("preflight validated classic group") + }) + .collect::>(); + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = + take_f32_scratch_buffer(runtime, group.total_coefficients * count)?; + let (buffers, status_check) = + encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer( + runtime, + command_buffers.default, + &groups, + &output.buffer, + scratch_buffers, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let input_groups = if broadcast_tier1_inputs { + &groups[..1] + } else { + &groups + }; + if let Some(cache) = flattened_cpu_tier1_cache { + cache.buffer_for( + component_idx, + step_idx, + group.total_coefficients, + input_groups.len(), + )? + } else { + let inputs = input_groups + .iter() + .map(|group| ClassicCpuDecodeInput { + coded_data: &group.coded_data, + segments: &group.segments, + jobs: &group.jobs, + output_len: group.total_coefficients, + }) + .collect::>(); + let decode_started = profile_stages.then(Instant::now); + let cpu_tier1_counters = + profile_stages.then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode_classic_inputs_on_cpu_with_plan_cache( + first, + step_idx, + &inputs, + cpu_tier1_counters.as_ref(), + )?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(stage_timings); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + } + }; + let stride_bytes = group.total_coefficients * size_of::(); + for (instance_idx, bands) in band_sets.iter_mut().enumerate() { + let source_group = if broadcast_tier1_inputs { + groups[0] + } else { + groups[instance_idx] + }; + let instance_offset = if broadcast_tier1_inputs { + 0 + } else { + instance_idx * stride_bytes + }; + for member in &source_group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: buffer.clone(), + offset_bytes: instance_offset + member.offset_elements * size_of::(), + window: member.window, + }); + } + } + step_idx = group.end_step; + continue; + } -#[cfg(target_os = "macos")] -fn prepared_idwt_params( - idwt: &PreparedDirectIdwt, - inputs: PreparedIdwtInputWindows, -) -> J2kIdwtSingleDecompositionParams { - J2kIdwtSingleDecompositionParams { - x0: idwt.step.rect.x0, - y0: idwt.step.rect.y0, - output_x: idwt.output_window.x0, - output_y: idwt.output_window.y0, - width: idwt.output_window.width(), - height: idwt.output_window.height(), - ll_x: inputs.ll.x0, - ll_y: inputs.ll.y0, - ll_width: inputs.ll.width(), - ll_height: inputs.ll.height(), - hl_x: inputs.hl.x0, - hl_y: inputs.hl.y0, - hl_width: inputs.hl.width(), - hl_height: inputs.hl.height(), - lh_x: inputs.lh.x0, - lh_y: inputs.lh.y0, - lh_width: inputs.lh.width(), - lh_height: inputs.lh.height(), - hh_x: inputs.hh.x0, - hh_y: inputs.hh.y0, - hh_width: inputs.hh.width(), - hh_height: inputs.hh.height(), - } -} + if let Some(group) = first.ht_group_starting_at(step_idx) { + let groups = plans + .iter() + .map(|plan| { + plan.ht_group_starting_at(step_idx) + .expect("preflight validated HT group") + }) + .collect::>(); + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = + take_f32_scratch_buffer(runtime, group.total_coefficients * count)?; + let (buffers, status_check) = + encode_distinct_ht_sub_band_groups_to_buffer_in_command_buffer( + runtime, + command_buffers.default, + &groups, + &output.buffer, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let input_groups = if broadcast_tier1_inputs { + &groups[..1] + } else { + &groups + }; + if let Some(cache) = flattened_cpu_tier1_cache { + cache.buffer_for( + component_idx, + step_idx, + group.total_coefficients, + input_groups.len(), + )? + } else { + let inputs = input_groups + .iter() + .map(|group| HtCpuDecodeInput { + coded_data: &group.coded_arena.data, + jobs: &group.jobs, + output_len: group.total_coefficients, + }) + .collect::>(); + let decode_started = profile_stages.then(Instant::now); + let cpu_tier1_counters = + profile_stages.then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode_ht_inputs_on_cpu_with_plan_cache( + first, + step_idx, + &inputs, + cpu_tier1_counters.as_ref(), + )?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(stage_timings); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + } + }; + let stride_bytes = group.total_coefficients * size_of::(); + for (instance_idx, bands) in band_sets.iter_mut().enumerate() { + let source_group = if broadcast_tier1_inputs { + groups[0] + } else { + groups[instance_idx] + }; + let instance_offset = if broadcast_tier1_inputs { + 0 + } else { + instance_idx * stride_bytes + }; + for member in &source_group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: buffer.clone(), + offset_bytes: instance_offset + member.offset_elements * size_of::(), + window: member.window, + }); + } + } + step_idx = group.end_step; + continue; + } -#[cfg(target_os = "macos")] -fn repeated_idwt_params( - idwt: &PreparedDirectIdwt, - inputs: PreparedIdwtInputWindows, - strides: PreparedIdwtInputStrides, - batch_count: usize, - context: &'static str, -) -> Result { - Ok(J2kRepeatedIdwtSingleDecompositionParams { - x0: idwt.step.rect.x0, - y0: idwt.step.rect.y0, - output_x: idwt.output_window.x0, - output_y: idwt.output_window.y0, - width: idwt.output_window.width(), - height: idwt.output_window.height(), - ll_x: inputs.ll.x0, - ll_y: inputs.ll.y0, - ll_width: inputs.ll.width(), - ll_height: inputs.ll.height(), - hl_x: inputs.hl.x0, - hl_y: inputs.hl.y0, - hl_width: inputs.hl.width(), - hl_height: inputs.hl.height(), - lh_x: inputs.lh.x0, - lh_y: inputs.lh.y0, - lh_width: inputs.lh.width(), - lh_height: inputs.lh.height(), - hh_x: inputs.hh.x0, - hh_y: inputs.hh.y0, - hh_width: inputs.hh.width(), - hh_height: inputs.hh.height(), - ll_instance_stride: strides.ll, - hl_instance_stride: strides.hl, - lh_instance_stride: strides.lh, - hh_instance_stride: strides.hh, - batch_count: u32::try_from(batch_count).map_err(|_| Error::MetalKernel { - message: format!("J2K MetalDirect {context} IDWT batch count exceeds u32"), - })?, - }) -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Copy)] -struct PreparedIdwtInputStrides { - ll: u32, - hl: u32, - lh: u32, - hh: u32, -} + match &first.steps[step_idx] { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { + let sub_bands = plans + .iter() + .map(|plan| match &plan.steps[step_idx] { + PreparedDirectGrayscaleStep::ClassicSubBand(other) => Ok(other), + _ => Err(direct_preflight_invariant( + "classic sub-band step mismatch in stacked component batch", + )), + }) + .collect::, Error>>()?; + let per_instance_len = sub_band.width as usize * sub_band.height as usize; + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(runtime, per_instance_len * count)?; + let (buffers, status_check) = + encode_distinct_classic_sub_bands_to_buffer_in_command_buffer( + runtime, + command_buffers.default, + &sub_bands, + &output.buffer, + scratch_buffers, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let input_sub_bands = if broadcast_tier1_inputs { + &sub_bands[..1] + } else { + &sub_bands + }; + if let Some(cache) = flattened_cpu_tier1_cache { + cache.buffer_for( + component_idx, + step_idx, + per_instance_len, + input_sub_bands.len(), + )? + } else { + let inputs = input_sub_bands + .iter() + .map(|sub_band| ClassicCpuDecodeInput { + coded_data: &sub_band.coded_data, + segments: &sub_band.segments, + jobs: &sub_band.jobs, + output_len: per_instance_len, + }) + .collect::>(); + let decode_started = profile_stages.then(Instant::now); + let cpu_tier1_counters = + profile_stages.then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode_classic_inputs_on_cpu_with_plan_cache( + first, + step_idx, + &inputs, + cpu_tier1_counters.as_ref(), + )?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(stage_timings); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + } + }; + let stride_bytes = per_instance_len * size_of::(); + for (instance_idx, bands) in band_sets.iter_mut().enumerate() { + let source_sub_band = if broadcast_tier1_inputs { + sub_bands[0] + } else { + sub_bands[instance_idx] + }; + let instance_offset = if broadcast_tier1_inputs { + 0 + } else { + instance_idx * stride_bytes + }; + bands.push(DirectBandSlice { + band_id: source_sub_band.band_id, + buffer: buffer.clone(), + offset_bytes: instance_offset, + window: BandRequiredRegion::full( + source_sub_band.width, + source_sub_band.height, + ), + }); + } + } + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + let sub_bands = plans + .iter() + .map(|plan| match &plan.steps[step_idx] { + PreparedDirectGrayscaleStep::HtSubBand(other) => Ok(other), + _ => Err(direct_preflight_invariant( + "HT sub-band step mismatch in stacked component batch", + )), + }) + .collect::, Error>>()?; + let per_instance_len = sub_band.width as usize * sub_band.height as usize; + let buffer = match tier1_mode { + DirectTier1Mode::Metal => { + let output = take_f32_scratch_buffer(runtime, per_instance_len * count)?; + let (buffers, status_check) = + encode_distinct_ht_sub_bands_to_buffer_in_command_buffer( + runtime, + command_buffers.default, + &sub_bands, + &output.buffer, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let buffer = output.buffer.clone(); + scratch_buffers.push(output); + buffer + } + DirectTier1Mode::CpuUpload => { + let input_sub_bands = if broadcast_tier1_inputs { + &sub_bands[..1] + } else { + &sub_bands + }; + if let Some(cache) = flattened_cpu_tier1_cache { + cache.buffer_for( + component_idx, + step_idx, + per_instance_len, + input_sub_bands.len(), + )? + } else { + let inputs = input_sub_bands + .iter() + .map(|sub_band| HtCpuDecodeInput { + coded_data: &sub_band.coded_data, + jobs: &sub_band.jobs, + output_len: per_instance_len, + }) + .collect::>(); + let decode_started = profile_stages.then(Instant::now); + let cpu_tier1_counters = + profile_stages.then(CpuTier1DecodeSubstageCounters::default); + let coefficients = decode_ht_inputs_on_cpu_with_plan_cache( + first, + step_idx, + &inputs, + cpu_tier1_counters.as_ref(), + )?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(stage_timings); + } + let upload_started = profile_stages.then(Instant::now); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + buffer + } + } + }; + let stride_bytes = per_instance_len * size_of::(); + for (instance_idx, bands) in band_sets.iter_mut().enumerate() { + let source_sub_band = if broadcast_tier1_inputs { + sub_bands[0] + } else { + sub_bands[instance_idx] + }; + let instance_offset = if broadcast_tier1_inputs { + 0 + } else { + instance_idx * stride_bytes + }; + bands.push(DirectBandSlice { + band_id: source_sub_band.band_id, + buffer: buffer.clone(), + offset_bytes: instance_offset, + window: BandRequiredRegion::full( + source_sub_band.width, + source_sub_band.height, + ), + }); + } + } + PreparedDirectGrayscaleStep::Idwt(idwt) => { + let per_instance_len = prepared_idwt_output_len(idwt); + let output = take_f32_scratch_buffer(runtime, per_instance_len * count)?; + let encode_started = profile_stages.then(Instant::now); + match idwt.step.transform { + J2kWaveletTransform::Reversible53 => { + let (ll, low_low_stride) = lookup_repeated_direct_band_layout_entry( + &band_sets, + idwt.step.ll_band_id, + idwt.step.ll, + )?; + let (hl, high_low_stride) = lookup_repeated_direct_band_layout_entry( + &band_sets, + idwt.step.hl_band_id, + idwt.step.hl, + )?; + let (lh, low_high_stride) = lookup_repeated_direct_band_layout_entry( + &band_sets, + idwt.step.lh_band_id, + idwt.step.lh, + )?; + let (hh, high_high_stride) = lookup_repeated_direct_band_layout_entry( + &band_sets, + idwt.step.hh_band_id, + idwt.step.hh, + )?; + let params = repeated_idwt_params( + idwt, + idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), + PreparedIdwtInputStrides { + ll: low_low_stride, + hl: high_low_stride, + lh: low_high_stride, + hh: high_high_stride, + }, + count, + "color", + )?; + dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets( + runtime, + command_buffers.idwt, + &ll.buffer, + ll.offset_bytes, + &hl.buffer, + hl.offset_bytes, + &lh.buffer, + lh.offset_bytes, + &hh.buffer, + hh.offset_bytes, + params, + &output.buffer, + ); + } + J2kWaveletTransform::Irreversible97 => { + for (instance_idx, bands) in band_sets.iter().enumerate() { + let PreparedDirectGrayscaleStep::Idwt(step) = + &plans[instance_idx].steps[step_idx] + else { + return Err(direct_preflight_invariant( + "IDWT step mismatch in stacked component batch", + )); + }; + let ll = lookup_direct_band_slice_entry( + bands, + step.step.ll_band_id, + step.step.ll, + )?; + let hl = lookup_direct_band_slice_entry( + bands, + step.step.hl_band_id, + step.step.hl, + )?; + let lh = lookup_direct_band_slice_entry( + bands, + step.step.lh_band_id, + step.step.lh, + )?; + let hh = lookup_direct_band_slice_entry( + bands, + step.step.hh_band_id, + step.step.hh, + )?; + let params = prepared_idwt_params( + step, + idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), + ); + status_checks.push( + dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets( + runtime, + command_buffers.idwt.interleave, + &ll.buffer, + ll.offset_bytes, + &hl.buffer, + hl.offset_bytes, + &lh.buffer, + lh.offset_bytes, + &hh.buffer, + hh.offset_bytes, + params, + &output.buffer, + instance_idx * per_instance_len * size_of::(), + ), + ); + } + } + } + if let Some(started) = encode_started { + stage_timings.metal_idwt_encode += elapsed_us(started); + } + let stride_bytes = per_instance_len * size_of::(); + for (instance_idx, bands) in band_sets.iter_mut().enumerate() { + let PreparedDirectGrayscaleStep::Idwt(step) = + &plans[instance_idx].steps[step_idx] + else { + return Err(direct_preflight_invariant( + "IDWT output step mismatch in stacked component batch", + )); + }; + bands.push(DirectBandSlice { + band_id: step.step.output_band_id, + buffer: output.buffer.clone(), + offset_bytes: instance_idx * stride_bytes, + window: step.output_window, + }); + } + scratch_buffers.push(output); + } + PreparedDirectGrayscaleStep::Store(store) => { + let (input, input_instance_stride) = lookup_repeated_direct_band_layout_entry( + &band_sets, + store.input_band_id, + store.input_rect, + )?; + let per_instance_len = store.output_width as usize * store.output_height as usize; + let output = take_f32_scratch_buffer(runtime, per_instance_len * count)?; + let encode_started = profile_stages.then(Instant::now); + dispatch_store_component_repeated_in_command_buffer( + runtime, + command_buffers.store, + &input.buffer, + input.offset_bytes, + &output.buffer, + J2kRepeatedStoreParams { + input_width: store.input_rect.width(), + input_height: store.input_rect.height(), + input_instance_stride, + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_height: store.output_height, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + batch_count: u32::try_from(count).map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect color store batch count exceeds u32" + .to_string(), + })?, + }, + ); + if let Some(started) = encode_started { + stage_timings.metal_store_encode += elapsed_us(started); + } + final_plane = Some(output.buffer.clone()); + scratch_buffers.push(output); + } + } + step_idx += 1; + } -#[cfg(target_os = "macos")] -fn prepared_idwt_output_len(idwt: &PreparedDirectIdwt) -> usize { - idwt.output_window.width() as usize * idwt.output_window.height() as usize + let buffer = final_plane.ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect color component batch did not produce a final plane".to_string(), + })?; + record_hybrid_stacked_component_batch(tier1_mode); + Ok(StackedDirectComponentPlane { + buffer, + dimensions: first.dimensions, + count, + }) } #[cfg(target_os = "macos")] -fn add_required_region( - required: &mut HashMap, - band_id: J2kDirectBandId, - region: BandRequiredRegion, -) { - required - .entry(band_id) - .and_modify(|existing| *existing = existing.union(region)) - .or_insert(region); -} - -#[cfg(target_os = "macos")] -fn idwt_required_output_margin(transform: J2kWaveletTransform) -> u32 { - match transform { - J2kWaveletTransform::Reversible53 => 4, - J2kWaveletTransform::Irreversible97 => 12, - } -} - -#[cfg(target_os = "macos")] -fn add_idwt_input_required_regions( - required: &mut HashMap, - idwt: &J2kDirectIdwtStep, - output_region: BandRequiredRegion, -) { - add_required_region( - required, - idwt.ll_band_id, - idwt_input_required_region( - output_region, - idwt.rect.x0, - idwt.rect.y0, - true, - true, - idwt.ll.width(), - idwt.ll.height(), - ), - ); - add_required_region( - required, - idwt.hl_band_id, - idwt_input_required_region( - output_region, - idwt.rect.x0, - idwt.rect.y0, - false, - true, - idwt.hl.width(), - idwt.hl.height(), - ), - ); - add_required_region( - required, - idwt.lh_band_id, - idwt_input_required_region( - output_region, - idwt.rect.x0, - idwt.rect.y0, - true, - false, - idwt.lh.width(), - idwt.lh.height(), - ), - ); - add_required_region( - required, - idwt.hh_band_id, - idwt_input_required_region( - output_region, - idwt.rect.x0, - idwt.rect.y0, - false, - false, - idwt.hh.width(), - idwt.hh.height(), - ), - ); -} - -#[cfg(target_os = "macos")] -#[allow(clippy::fn_params_excessive_bools)] -fn idwt_input_required_region( - output_region: BandRequiredRegion, - output_origin_x: u32, - output_origin_y: u32, - low_x: bool, - low_y: bool, - band_width: u32, - band_height: u32, -) -> BandRequiredRegion { - let x0 = signinum_j2k_native::idwt_band_index(output_origin_x, output_region.x0, low_x); - let x1 = signinum_j2k_native::idwt_band_index(output_origin_x, output_region.x1 - 1, low_x) - .saturating_add(1); - let y0 = signinum_j2k_native::idwt_band_index(output_origin_y, output_region.y0, low_y); - let y1 = signinum_j2k_native::idwt_band_index(output_origin_y, output_region.y1 - 1, low_y) - .saturating_add(1); - BandRequiredRegion { - x0: x0.min(band_width), - y0: y0.min(band_height), - x1: x1.min(band_width), - y1: y1.min(band_height), - } -} - -#[cfg(target_os = "macos")] -fn retain_classic_jobs_for_required_region( - jobs: &mut Vec, - required: Option, -) { - let Some(required) = required else { - jobs.clear(); - return; - }; - jobs.retain(|job| { - let output_x = job.output_offset % job.output_stride; - let output_y = job.output_offset / job.output_stride; - required.intersects(output_x, output_y, job.width, job.height) - }); -} - -#[cfg(target_os = "macos")] -fn retain_ht_jobs_for_required_region( - jobs: &mut Vec, - required: Option, -) { - let Some(required) = required else { - jobs.clear(); - return; - }; - jobs.retain(|job| { - let output_x = job.output_offset % job.output_stride; - let output_y = job.output_offset / job.output_stride; - required.intersects(output_x, output_y, job.width, job.height) - }); -} - -#[cfg(target_os = "macos")] -fn compact_ht_sub_band_coded_data(sub_band: &mut PreparedHtSubBand) -> Result<(), Error> { - let previous = std::mem::take(&mut sub_band.coded_data); - let mut compacted = Vec::new(); +#[allow(clippy::too_many_arguments)] +fn encode_repeated_direct_grayscale_plan_in_command_buffer( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + plan: &PreparedDirectGrayscalePlan, + fmt: PixelFormat, + count: usize, + retained_buffers: &mut Vec, + status_checks: &mut Vec, + scratch_buffers: &mut Vec, +) -> Result, Error> { + let mut band_sets = vec![Vec::::new(); count]; + let mut surfaces = Vec::with_capacity(count); + let mut stacked_outputs = true; + let mut step_idx = 0; - for job in &mut sub_band.jobs { - let start = job.coded_offset as usize; - let len = job.coded_len as usize; - let end = start.checked_add(len).ok_or_else(|| Error::MetalKernel { - message: "HTJ2K MetalDirect cropped coded payload range overflow".to_string(), - })?; - if end > previous.len() { - return Err(Error::MetalKernel { - message: "HTJ2K MetalDirect cropped coded payload range out of bounds".to_string(), - }); + while step_idx < plan.steps.len() { + if let Some(group) = plan.classic_group_starting_at(step_idx) { + let per_instance_len = group.total_coefficients; + let output = take_f32_scratch_buffer(runtime, per_instance_len * count)?; + let (buffers, status_check) = + encode_repeated_classic_sub_band_group_to_buffer_in_command_buffer( + runtime, + command_buffer, + group, + count, + &output.buffer, + scratch_buffers, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let stride_bytes = per_instance_len * size_of::(); + for (instance_idx, bands) in band_sets.iter_mut().enumerate() { + for member in &group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: output.buffer.clone(), + offset_bytes: instance_idx * stride_bytes + + member.offset_elements * size_of::(), + window: member.window, + }); + } + } + scratch_buffers.push(output); + step_idx = group.end_step; + continue; } - job.coded_offset = u32::try_from(compacted.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K MetalDirect cropped coded payload exceeds u32".to_string(), - })?; - compacted.extend_from_slice(&previous[start..end]); - } - - sub_band.coded_data = compacted; - with_runtime(|runtime| { - sub_band.coded_buffer = borrow_slice_buffer(&runtime.device, &sub_band.coded_data); - sub_band.jobs_buffer = borrow_slice_buffer(&runtime.device, &sub_band.jobs); - Ok(()) - }) -} -#[cfg(target_os = "macos")] -fn checked_rect_end(origin: u32, length: u32, label: &str) -> Result { - origin - .checked_add(length) - .ok_or_else(|| Error::MetalKernel { - message: format!("J2K MetalDirect region-scaled {label} overflows u32"), - }) -} + if let Some(group) = plan.ht_group_starting_at(step_idx) { + let per_instance_len = group.total_coefficients; + let output = take_f32_scratch_buffer(runtime, per_instance_len * count)?; + let (buffers, status_check) = + encode_repeated_ht_sub_band_group_to_buffer_in_command_buffer( + runtime, + command_buffer, + group, + count, + &output.buffer, + )?; + retained_buffers.extend(buffers); + status_checks.push(status_check); + let stride_bytes = per_instance_len * size_of::(); + for (instance_idx, bands) in band_sets.iter_mut().enumerate() { + for member in &group.members { + bands.push(DirectBandSlice { + band_id: member.band_id, + buffer: output.buffer.clone(), + offset_bytes: instance_idx * stride_bytes + + member.offset_elements * size_of::(), + window: member.window, + }); + } + } + scratch_buffers.push(output); + step_idx = group.end_step; + continue; + } -#[cfg(target_os = "macos")] -fn crop_direct_store_step_to_output_region( - store: &mut J2kDirectStoreStep, - region: Rect, -) -> Result<(), Error> { - let store_bounds = ( - store.output_x, - store.output_y, - checked_rect_end(store.output_x, store.copy_width, "store width")?, - checked_rect_end(store.output_y, store.copy_height, "store height")?, - ); - let region_bounds = ( - region.x, - region.y, - checked_rect_end(region.x, region.w, "ROI width")?, - checked_rect_end(region.y, region.h, "ROI height")?, - ); - let intersection = ( - store_bounds.0.max(region_bounds.0), - store_bounds.1.max(region_bounds.1), - store_bounds.2.min(region_bounds.2), - store_bounds.3.min(region_bounds.3), - ); - if intersection.0 >= intersection.2 || intersection.1 >= intersection.3 { - return Err(Error::MetalKernel { - message: - "J2K MetalDirect region-scaled ROI does not intersect the decoded store window" - .to_string(), - }); - } - - let source_delta = ( - intersection.0 - store_bounds.0, - intersection.1 - store_bounds.1, - ); - store.source_x = - store - .source_x - .checked_add(source_delta.0) - .ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect region-scaled source x overflows u32".to_string(), - })?; - store.source_y = - store - .source_y - .checked_add(source_delta.1) - .ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect region-scaled source y overflows u32".to_string(), - })?; - store.copy_width = intersection.2 - intersection.0; - store.copy_height = intersection.3 - intersection.1; - store.output_width = region.w; - store.output_height = region.h; - store.output_x = intersection.0 - region_bounds.0; - store.output_y = intersection.1 - region_bounds.1; - Ok(()) -} - -#[cfg(target_os = "macos")] -pub(crate) fn prepare_direct_color_plan( - plan: &J2kDirectColorPlan, -) -> Result { - let component_plans = plan - .component_plans - .iter() - .map(prepare_direct_grayscale_plan) - .collect::, _>>()?; - if component_plans.len() != 3 { - return Err(Error::MetalKernel { - message: format!( - "J2K MetalDirect color plan expected 3 component plans, got {}", - component_plans.len() - ), - }); - } - Ok(PreparedDirectColorPlan { - dimensions: plan.dimensions, - bit_depths: plan.bit_depths, - mct: plan.mct, - transform: plan.transform, - component_plans, - }) -} - -#[cfg(target_os = "macos")] -impl PreparedDirectGrayscalePlan { - fn classic_group_starting_at(&self, step_idx: usize) -> Option<&PreparedClassicSubBandGroup> { - self.classic_groups - .iter() - .find(|group| group.start_step == step_idx) - } - - fn ht_group_starting_at(&self, step_idx: usize) -> Option<&PreparedHtSubBandGroup> { - self.ht_groups - .iter() - .find(|group| group.start_step == step_idx) - } -} - -#[cfg(all(test, target_os = "macos"))] -fn prepared_direct_grayscale_plan_compute_encoder_count( - plan: &PreparedDirectGrayscalePlan, - _fmt: PixelFormat, -) -> usize { - usize::from(!plan.steps.is_empty()) -} - -#[cfg(all(test, target_os = "macos"))] -fn prepared_repeated_direct_ht_cleanup_dispatch_count(plan: &PreparedDirectGrayscalePlan) -> usize { - let mut dispatches = 0; - let mut step_idx = 0; - while step_idx < plan.steps.len() { - if let Some(group) = plan.ht_group_starting_at(step_idx) { - dispatches += 1; - step_idx = group.end_step; - continue; - } - if matches!( - plan.steps[step_idx], - PreparedDirectGrayscaleStep::HtSubBand(_) - ) { - dispatches += 1; - } - step_idx += 1; - } - dispatches -} - -#[cfg(target_os = "macos")] -fn encode_prepared_direct_grayscale_plan_in_command_buffer( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - plan: &PreparedDirectGrayscalePlan, - fmt: PixelFormat, - retained_buffers: &mut Vec, - status_checks: &mut Vec, - scratch_buffers: &mut Vec, -) -> Result { - let encoder = command_buffer.new_compute_command_encoder(); - let result = (|| { - let mut bands = Vec::::new(); - let mut final_surface = None; - let mut step_idx = 0; - - while step_idx < plan.steps.len() { - if let Some(group) = plan.classic_group_starting_at(step_idx) { - let output = take_f32_scratch_buffer(runtime, group.total_coefficients); + let step = &plan.steps[step_idx]; + match step { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { + let per_instance_len = sub_band.width as usize * sub_band.height as usize; + let output = take_f32_scratch_buffer(runtime, per_instance_len * count)?; let (buffers, status_check) = - encode_prepared_classic_sub_band_group_to_buffer_in_encoder( + encode_repeated_classic_sub_band_to_buffer_in_command_buffer( runtime, - encoder, - group, + command_buffer, + sub_band, + count, &output.buffer, scratch_buffers, )?; retained_buffers.extend(buffers); status_checks.push(status_check); - for member in &group.members { + let stride_bytes = per_instance_len * size_of::(); + for (instance_idx, bands) in band_sets.iter_mut().enumerate() { bands.push(DirectBandSlice { - band_id: member.band_id, + band_id: sub_band.band_id, buffer: output.buffer.clone(), - offset_bytes: member.offset_elements * size_of::(), - window: member.window, + offset_bytes: instance_idx * stride_bytes, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), }); } scratch_buffers.push(output); - step_idx = group.end_step; - continue; } - - if let Some(group) = plan.ht_group_starting_at(step_idx) { - let output = take_f32_scratch_buffer(runtime, group.total_coefficients); + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + let per_instance_len = sub_band.width as usize * sub_band.height as usize; + let output = take_f32_scratch_buffer(runtime, per_instance_len * count)?; let (buffers, status_check) = - encode_prepared_ht_sub_band_group_to_buffer_in_encoder( + encode_repeated_ht_sub_band_to_buffer_in_command_buffer( runtime, - encoder, - group, + command_buffer, + sub_band, + count, &output.buffer, )?; retained_buffers.extend(buffers); status_checks.push(status_check); - for member in &group.members { + let stride_bytes = per_instance_len * size_of::(); + for (instance_idx, bands) in band_sets.iter_mut().enumerate() { bands.push(DirectBandSlice { - band_id: member.band_id, + band_id: sub_band.band_id, buffer: output.buffer.clone(), - offset_bytes: member.offset_elements * size_of::(), - window: member.window, + offset_bytes: instance_idx * stride_bytes, + window: BandRequiredRegion::full(sub_band.width, sub_band.height), }); } scratch_buffers.push(output); - step_idx = group.end_step; - continue; } - - let step = &plan.steps[step_idx]; - match step { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { - let output = take_f32_scratch_buffer( + PreparedDirectGrayscaleStep::Idwt(idwt) => match idwt.step.transform { + J2kWaveletTransform::Reversible53 if stacked_outputs => { + let (ll, low_low_stride) = lookup_repeated_direct_band_layout_entry( + &band_sets, + idwt.step.ll_band_id, + idwt.step.ll, + )?; + let (hl, high_low_stride) = lookup_repeated_direct_band_layout_entry( + &band_sets, + idwt.step.hl_band_id, + idwt.step.hl, + )?; + let (lh, low_high_stride) = lookup_repeated_direct_band_layout_entry( + &band_sets, + idwt.step.lh_band_id, + idwt.step.lh, + )?; + let (hh, high_high_stride) = lookup_repeated_direct_band_layout_entry( + &band_sets, + idwt.step.hh_band_id, + idwt.step.hh, + )?; + let params = repeated_idwt_params( + idwt, + idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), + PreparedIdwtInputStrides { + ll: low_low_stride, + hl: high_low_stride, + lh: low_high_stride, + hh: high_high_stride, + }, + count, + "repeated", + )?; + let per_instance_len = prepared_idwt_output_len(idwt); + let output = take_f32_scratch_buffer(runtime, per_instance_len * count)?; + dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets( runtime, - sub_band.width as usize * sub_band.height as usize, + DirectIdwtCommandBuffers::single(command_buffer), + &ll.buffer, + ll.offset_bytes, + &hl.buffer, + hl.offset_bytes, + &lh.buffer, + lh.offset_bytes, + &hh.buffer, + hh.offset_bytes, + params, + &output.buffer, ); - let (buffers, status_check) = - encode_prepared_classic_sub_band_to_buffer_in_encoder( - runtime, - encoder, - sub_band, - &output.buffer, - scratch_buffers, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - bands.push(DirectBandSlice { - band_id: sub_band.band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); + let stride_bytes = per_instance_len * size_of::(); + for (instance_idx, bands) in band_sets.iter_mut().enumerate() { + bands.push(DirectBandSlice { + band_id: idwt.step.output_band_id, + buffer: output.buffer.clone(), + offset_bytes: instance_idx * stride_bytes, + window: idwt.output_window, + }); + } scratch_buffers.push(output); } - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { - let output = take_f32_scratch_buffer( - runtime, - sub_band.width as usize * sub_band.height as usize, - ); - let (buffers, status_check) = encode_prepared_ht_sub_band_to_buffer_in_encoder( - runtime, - encoder, - sub_band, - &output.buffer, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - bands.push(DirectBandSlice { - band_id: sub_band.band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - scratch_buffers.push(output); - } - PreparedDirectGrayscaleStep::Idwt(idwt) => { - let ll = - lookup_direct_band_slice_entry(&bands, idwt.step.ll_band_id, idwt.step.ll)?; - let hl = - lookup_direct_band_slice_entry(&bands, idwt.step.hl_band_id, idwt.step.hl)?; - let lh = - lookup_direct_band_slice_entry(&bands, idwt.step.lh_band_id, idwt.step.lh)?; - let hh = - lookup_direct_band_slice_entry(&bands, idwt.step.hh_band_id, idwt.step.hh)?; - let params = prepared_idwt_params( - idwt, - idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), - ); - let output = take_f32_scratch_buffer(runtime, prepared_idwt_output_len(idwt)); - match idwt.step.transform { - J2kWaveletTransform::Reversible53 => { - dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets( - runtime, - encoder, - &ll.buffer, - ll.offset_bytes, - &hl.buffer, - hl.offset_bytes, - &lh.buffer, - lh.offset_bytes, - &hh.buffer, - hh.offset_bytes, - params, - &output.buffer, - 0, - ); - } - J2kWaveletTransform::Irreversible97 => { - let status_check = - dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets( - runtime, - encoder, - &ll.buffer, - ll.offset_bytes, - &hl.buffer, - hl.offset_bytes, - &lh.buffer, - lh.offset_bytes, - &hh.buffer, - hh.offset_bytes, - params, - &output.buffer, - 0, - ); - status_checks.push(status_check); - } + _ => { + stacked_outputs = false; + for bands in &mut band_sets { + let ll = lookup_direct_band_slice_entry( + bands, + idwt.step.ll_band_id, + idwt.step.ll, + )?; + let hl = lookup_direct_band_slice_entry( + bands, + idwt.step.hl_band_id, + idwt.step.hl, + )?; + let lh = lookup_direct_band_slice_entry( + bands, + idwt.step.lh_band_id, + idwt.step.lh, + )?; + let hh = lookup_direct_band_slice_entry( + bands, + idwt.step.hh_band_id, + idwt.step.hh, + )?; + let params = prepared_idwt_params( + idwt, + idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), + ); + let output = + take_f32_scratch_buffer(runtime, prepared_idwt_output_len(idwt))?; + match idwt.step.transform { + J2kWaveletTransform::Reversible53 => { + dispatch_reversible53_single_decomposition_buffers_in_command_buffer_with_offsets( + runtime, + command_buffer, + &ll.buffer, + ll.offset_bytes, + &hl.buffer, + hl.offset_bytes, + &lh.buffer, + lh.offset_bytes, + &hh.buffer, + hh.offset_bytes, + params, + &output.buffer, + 0, + ); + } + J2kWaveletTransform::Irreversible97 => status_checks.push( + dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets( + runtime, + command_buffer, + &ll.buffer, + ll.offset_bytes, + &hl.buffer, + hl.offset_bytes, + &lh.buffer, + lh.offset_bytes, + &hh.buffer, + hh.offset_bytes, + params, + &output.buffer, + 0, + ), + ), + } + bands.push(DirectBandSlice { + band_id: idwt.step.output_band_id, + buffer: output.buffer.clone(), + offset_bytes: 0, + window: idwt.output_window, + }); + scratch_buffers.push(output); } - bands.push(DirectBandSlice { - band_id: idwt.step.output_band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: idwt.output_window, - }); - scratch_buffers.push(output); } - PreparedDirectGrayscaleStep::Store(store) => { - let (input, input_offset) = - lookup_direct_band_slice(&bands, store.input_band_id, store.input_rect)?; + }, + PreparedDirectGrayscaleStep::Store(store) => { + if stacked_outputs { + let (input, _) = lookup_direct_band_slice( + &band_sets[0], + store.input_band_id, + store.input_rect, + )?; + let batch_count = u32::try_from(count).map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect repeated store batch count exceeds u32" + .to_string(), + })?; if matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { let scale = j2k_scalar_pack_params(u32::from(plan.bit_depth)); - final_surface = Some(encode_gray_store_to_surface_in_encoder( + surfaces.extend(encode_repeated_gray_store_to_surfaces_in_command_buffer( runtime, - encoder, + command_buffer, &input, - input_offset, - J2kGrayStoreParams { + J2kRepeatedGrayStoreParams { input_width: store.input_rect.width(), + input_height: store.input_rect.height(), source_x: store.source_x, source_y: store.source_y, copy_width: store.copy_width, copy_height: store.copy_height, output_width: store.output_width, + output_height: store.output_height, output_x: store.output_x, output_y: store.output_y, addend: store.addend, + batch_count, max_value: scale.max_value, u8_scale: scale.u8_scale, u16_scale: scale.u16_scale, }, plan.dimensions, fmt, + count, )?); } else { + let per_instance_len = + store.output_width as usize * store.output_height as usize; + let output = take_f32_scratch_buffer(runtime, per_instance_len * count)?; + dispatch_store_component_repeated_in_command_buffer( + runtime, + command_buffer, + &input, + 0, + &output.buffer, + J2kRepeatedStoreParams { + input_width: store.input_rect.width(), + input_height: store.input_rect.height(), + input_instance_stride: store + .input_rect + .width() + .checked_mul(store.input_rect.height()) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect repeated store input stride overflows u32" + .to_string(), + })?, + source_x: store.source_x, + source_y: store.source_y, + copy_width: store.copy_width, + copy_height: store.copy_height, + output_width: store.output_width, + output_height: store.output_height, + output_x: store.output_x, + output_y: store.output_y, + addend: store.addend, + batch_count, + }, + ); + retained_buffers.push(output.buffer.clone()); + surfaces.extend(encode_repeated_gray_plane_to_surfaces_in_command_buffer( + runtime, + command_buffer, + &output.buffer, + plan.dimensions, + plan.bit_depth, + fmt, + count, + )?); + scratch_buffers.push(output); + } + } else { + for bands in &band_sets { + let (input, input_offset) = + lookup_direct_band_slice(bands, store.input_band_id, store.input_rect)?; let output = take_f32_scratch_buffer( runtime, store.output_width as usize * store.output_height as usize, - ); + )?; let params = J2kStoreParams { input_width: store.input_rect.width(), source_x: store.source_x, @@ -3942,9 +4797,9 @@ fn encode_prepared_direct_grayscale_plan_in_command_buffer( output_y: store.output_y, addend: store.addend, }; - dispatch_store_component_buffer_in_encoder_with_offsets( + dispatch_store_component_buffer_in_command_buffer_with_offsets( runtime, - encoder, + command_buffer, &input, input_offset, &output.buffer, @@ -3952,10 +4807,11 @@ fn encode_prepared_direct_grayscale_plan_in_command_buffer( params, ); retained_buffers.push(output.buffer.clone()); - final_surface = Some(encode_gray_plane_to_surface_in_encoder( + surfaces.push(encode_gray_plane_to_surface_in_command_buffer_with_offset( runtime, - encoder, + command_buffer, &output.buffer, + 0, plan.dimensions, plan.bit_depth, fmt, @@ -3964,2395 +4820,2344 @@ fn encode_prepared_direct_grayscale_plan_in_command_buffer( } } } - step_idx += 1; } + step_idx += 1; + } - final_surface.ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect prepared grayscale plan did not produce a final stored plane" - .to_string(), - }) - })(); - encoder.end_encoding(); - result + if surfaces.len() != count { + return Err(Error::MetalKernel { + message: format!( + "J2K MetalDirect repeated grayscale plan produced {} surfaces for count {}", + surfaces.len(), + count + ), + }); + } + + Ok(surfaces) } #[cfg(target_os = "macos")] -fn encode_prepared_direct_component_plane_in_command_buffer( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - plan: &PreparedDirectGrayscalePlan, - retained_buffers: &mut Vec, - status_checks: &mut Vec, - scratch_buffers: &mut Vec, -) -> Result { - let encoder = command_buffer.new_compute_command_encoder(); - let result = (|| { - let mut bands = Vec::::new(); - let mut final_plane = None; - let mut step_idx = 0; +fn copy_plane_samples(buffer: &Buffer, samples: &[f32], image_width: usize, roi: Rect) { + let row_width = roi.w as usize; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let dst = unsafe { + core::slice::from_raw_parts_mut(buffer.contents().cast::(), row_width * roi.h as usize) + }; - while step_idx < plan.steps.len() { - if let Some(group) = plan.classic_group_starting_at(step_idx) { - let output = take_f32_scratch_buffer(runtime, group.total_coefficients); - let (buffers, status_check) = - encode_prepared_classic_sub_band_group_to_buffer_in_encoder( - runtime, - encoder, - group, - &output.buffer, - scratch_buffers, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - for member in &group.members { - bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: member.offset_elements * size_of::(), - window: member.window, - }); - } - scratch_buffers.push(output); - step_idx = group.end_step; - continue; - } - - if let Some(group) = plan.ht_group_starting_at(step_idx) { - let output = take_f32_scratch_buffer(runtime, group.total_coefficients); - let (buffers, status_check) = - encode_prepared_ht_sub_band_group_to_buffer_in_encoder( - runtime, - encoder, - group, - &output.buffer, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - for member in &group.members { - bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: member.offset_elements * size_of::(), - window: member.window, - }); - } - scratch_buffers.push(output); - step_idx = group.end_step; - continue; - } - - match &plan.steps[step_idx] { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { - let output = take_f32_scratch_buffer( - runtime, - sub_band.width as usize * sub_band.height as usize, - ); - let (buffers, status_check) = - encode_prepared_classic_sub_band_to_buffer_in_encoder( - runtime, - encoder, - sub_band, - &output.buffer, - scratch_buffers, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - bands.push(DirectBandSlice { - band_id: sub_band.band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - scratch_buffers.push(output); - } - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { - let output = take_f32_scratch_buffer( - runtime, - sub_band.width as usize * sub_band.height as usize, - ); - let (buffers, status_check) = encode_prepared_ht_sub_band_to_buffer_in_encoder( - runtime, - encoder, - sub_band, - &output.buffer, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - bands.push(DirectBandSlice { - band_id: sub_band.band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - scratch_buffers.push(output); - } - PreparedDirectGrayscaleStep::Idwt(idwt) => { - let ll = - lookup_direct_band_slice_entry(&bands, idwt.step.ll_band_id, idwt.step.ll)?; - let hl = - lookup_direct_band_slice_entry(&bands, idwt.step.hl_band_id, idwt.step.hl)?; - let lh = - lookup_direct_band_slice_entry(&bands, idwt.step.lh_band_id, idwt.step.lh)?; - let hh = - lookup_direct_band_slice_entry(&bands, idwt.step.hh_band_id, idwt.step.hh)?; - let params = prepared_idwt_params( - idwt, - idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), - ); - let output = take_f32_scratch_buffer(runtime, prepared_idwt_output_len(idwt)); - match idwt.step.transform { - J2kWaveletTransform::Reversible53 => { - dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets( - runtime, - encoder, - &ll.buffer, - ll.offset_bytes, - &hl.buffer, - hl.offset_bytes, - &lh.buffer, - lh.offset_bytes, - &hh.buffer, - hh.offset_bytes, - params, - &output.buffer, - 0, - ); - } - J2kWaveletTransform::Irreversible97 => { - status_checks.push( - dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_offsets( - runtime, - encoder, - &ll.buffer, - ll.offset_bytes, - &hl.buffer, - hl.offset_bytes, - &lh.buffer, - lh.offset_bytes, - &hh.buffer, - hh.offset_bytes, - params, - &output.buffer, - 0, - ), - ); - } - } - bands.push(DirectBandSlice { - band_id: idwt.step.output_band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: idwt.output_window, - }); - scratch_buffers.push(output); - } - PreparedDirectGrayscaleStep::Store(store) => { - let (input, input_offset) = - lookup_direct_band_slice(&bands, store.input_band_id, store.input_rect)?; - let output = take_f32_scratch_buffer( - runtime, - store.output_width as usize * store.output_height as usize, - ); - dispatch_store_component_buffer_in_encoder_with_offsets( - runtime, - encoder, - &input, - input_offset, - &output.buffer, - 0, - J2kStoreParams { - input_width: store.input_rect.width(), - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - }, - ); - final_plane = Some(output.buffer.clone()); - scratch_buffers.push(output); - } - } - step_idx += 1; - } - - final_plane.ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect component plan did not produce a stored plane".to_string(), - }) - })(); - encoder.end_encoding(); - result + for row in 0..roi.h as usize { + let src_start = (roi.y as usize + row) * image_width + roi.x as usize; + let src_end = src_start + row_width; + let dst_start = row * row_width; + dst[dst_start..dst_start + row_width].copy_from_slice(&samples[src_start..src_end]); + } } #[cfg(target_os = "macos")] -pub(crate) fn execute_repeated_prepared_direct_grayscale_plan( - plan: &PreparedDirectGrayscalePlan, +fn encode_gray_plane_to_surface_in_encoder( + runtime: &MetalRuntime, + encoder: &ComputeCommandEncoderRef, + plane: &Buffer, + dims: (u32, u32), + bit_depth: u8, fmt: PixelFormat, - count: usize, -) -> Result, Error> { - with_runtime(|runtime| { - let command_buffer = runtime.queue.new_command_buffer(); - let mut retained_buffers = Vec::new(); - let mut status_checks = Vec::new(); - let mut scratch_buffers = Vec::new(); - let surfaces = encode_repeated_direct_grayscale_plan_in_command_buffer( - runtime, - command_buffer, - plan, - fmt, - count, - &mut retained_buffers, - &mut status_checks, - &mut scratch_buffers, - )?; - command_buffer.commit(); - command_buffer.wait_until_completed(); - for status_check in status_checks { - validate_direct_status(status_check)?; - } - drop(retained_buffers); - recycle_scratch_buffers(runtime, scratch_buffers); - Ok(surfaces) - }) +) -> Result { + encode_gray_plane_to_surface_in_encoder_with_offset( + runtime, encoder, plane, 0, dims, bit_depth, fmt, + ) } #[cfg(target_os = "macos")] -pub(crate) fn execute_prepared_direct_grayscale_plan( - plan: &PreparedDirectGrayscalePlan, +fn encode_gray_plane_to_surface_in_command_buffer_with_offset( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + plane: &Buffer, + plane_offset_bytes: usize, + dims: (u32, u32), + bit_depth: u8, fmt: PixelFormat, ) -> Result { - with_runtime(|runtime| { - let command_buffer = runtime.queue.new_command_buffer(); - let mut retained_buffers = Vec::new(); - let mut status_checks = Vec::new(); - let mut scratch_buffers = Vec::new(); - let surface = encode_prepared_direct_grayscale_plan_in_command_buffer( - runtime, - command_buffer, - plan, - fmt, - &mut retained_buffers, - &mut status_checks, - &mut scratch_buffers, - )?; - command_buffer.commit(); - command_buffer.wait_until_completed(); - for status_check in status_checks { - validate_direct_status(status_check)?; - } - drop(retained_buffers); - recycle_scratch_buffers(runtime, scratch_buffers); - Ok(surface) - }) + let encoder = command_buffer.new_compute_command_encoder(); + let result = encode_gray_plane_to_surface_in_encoder_with_offset( + runtime, + encoder, + plane, + plane_offset_bytes, + dims, + bit_depth, + fmt, + ); + encoder.end_encoding(); + result } #[cfg(target_os = "macos")] -pub(crate) fn execute_prepared_direct_grayscale_plan_with_device( - plan: &PreparedDirectGrayscalePlan, +fn encode_gray_plane_to_surface_in_encoder_with_offset( + runtime: &MetalRuntime, + encoder: &ComputeCommandEncoderRef, + plane: &Buffer, + plane_offset_bytes: usize, + dims: (u32, u32), + bit_depth: u8, fmt: PixelFormat, - device: &Device, ) -> Result { - with_runtime_for_device(device, |_| { - execute_prepared_direct_grayscale_plan(plan, fmt) - }) + let pitch_bytes = dims.0 as usize * fmt.bytes_per_pixel(); + let out_buffer = runtime.device.new_buffer( + (pitch_bytes * dims.1 as usize) as u64, + MTLResourceOptions::StorageModeShared, + ); + let (output_channels, opaque_alpha, pipeline) = + output_shape_for(&NativeColorSpace::Gray, false, 1, fmt, runtime)?; + let mut bit_depths = [0u32; 4]; + bit_depths[0] = u32::from(bit_depth); + let (max_values, u8_scales, u16_scales) = j2k_pack_scale_arrays(bit_depths); + let params = J2kPackParams { + width: dims.0, + height: dims.1, + out_stride: j2k_u32_param(pitch_bytes, "J2K Metal output stride exceeds u32")?, + output_channels, + opaque_alpha, + max_values, + u8_scales, + u16_scales, + }; + + encoder.set_compute_pipeline_state(pipeline); + encoder.set_buffer(0, Some(plane), plane_offset_bytes as u64); + encoder.set_buffer(1, None, 0); + encoder.set_buffer(2, None, 0); + encoder.set_buffer(3, None, 0); + encoder.set_buffer(4, Some(&out_buffer), 0); + encoder.set_bytes( + 5, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_2d_pipeline(encoder, pipeline, dims); + + Ok(Surface::from_metal_buffer(out_buffer, dims, fmt)) } #[cfg(target_os = "macos")] -pub(crate) fn execute_prepared_direct_grayscale_plan_batch( - plans: &[Arc], +fn encode_repeated_gray_plane_to_surfaces_in_command_buffer( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + plane: &Buffer, + dims: (u32, u32), + bit_depth: u8, fmt: PixelFormat, + count: usize, ) -> Result, Error> { - if plans.is_empty() { - return Ok(Vec::new()); - } - - with_runtime(|runtime| { - let command_buffer = runtime.queue.new_command_buffer(); - let mut retained_buffers = Vec::new(); - let mut status_checks = Vec::new(); - let mut scratch_buffers = Vec::new(); - let mut surfaces = Vec::with_capacity(plans.len()); - - let component_plan_refs = plans.iter().map(Arc::as_ref).collect::>(); - if plans.len() > 1 && supports_stacked_direct_component_plane_batch(&component_plan_refs) { - let stacked_plane = encode_stacked_direct_component_plane_batch( - runtime, - command_buffer, - &component_plan_refs, - &mut retained_buffers, - &mut status_checks, - &mut scratch_buffers, - )?; - let first = plans.first().expect("plans is not empty"); - if stacked_plane.dimensions == first.dimensions && stacked_plane.count == plans.len() { - surfaces = encode_repeated_gray_plane_to_surfaces_in_command_buffer( - runtime, - command_buffer, - &stacked_plane.buffer, - first.dimensions, - first.bit_depth, - fmt, - plans.len(), - )?; - } + let count_u32 = u32::try_from(count).map_err(|_| Error::MetalKernel { + message: "J2K Metal repeated grayscale surface count exceeds u32".to_string(), + })?; + let pitch_bytes = dims.0 as usize * fmt.bytes_per_pixel(); + let surface_bytes = + pitch_bytes + .checked_mul(dims.1 as usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal repeated grayscale surface size overflow".to_string(), + })?; + let total_bytes = surface_bytes + .checked_mul(count) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal repeated grayscale output size overflow".to_string(), + })?; + let out_buffer = runtime + .device + .new_buffer(total_bytes as u64, MTLResourceOptions::StorageModeShared); + let scale = j2k_scalar_pack_params(u32::from(bit_depth)); + let params = J2kRepeatedGrayPackParams { + width: dims.0, + height: dims.1, + out_stride: j2k_u32_param(pitch_bytes, "J2K Metal output stride exceeds u32")?, + batch_count: count_u32, + max_value: scale.max_value, + u8_scale: scale.u8_scale, + u16_scale: scale.u16_scale, + }; + let pipeline = match fmt { + PixelFormat::Gray8 => &runtime.pack_u8_repeated_gray, + PixelFormat::Gray16 => &runtime.pack_u16_repeated_gray, + _ => { + return Err(Error::MetalKernel { + message: format!("J2K Metal repeated grayscale pack does not support {fmt:?}"), + }) } + }; - for plan in plans { - if !surfaces.is_empty() { - break; - } - surfaces.push(encode_prepared_direct_grayscale_plan_in_command_buffer( - runtime, - command_buffer, - plan, - fmt, - &mut retained_buffers, - &mut status_checks, - &mut scratch_buffers, - )?); - } + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(pipeline); + encoder.set_buffer(0, Some(plane), 0); + encoder.set_buffer(1, Some(&out_buffer), 0); + encoder.set_bytes( + 2, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_3d_pipeline(encoder, pipeline, (dims.0, dims.1, count_u32)); + encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - for status_check in status_checks { - validate_direct_status(status_check)?; - } - drop(retained_buffers); - recycle_scratch_buffers(runtime, scratch_buffers); - Ok(surfaces) - }) + let mut surfaces = Vec::with_capacity(count); + for instance_idx in 0..count { + surfaces.push(Surface::from_metal_buffer_with_offset( + out_buffer.clone(), + dims, + fmt, + instance_idx * surface_bytes, + )); + } + Ok(surfaces) } #[cfg(target_os = "macos")] -pub(crate) fn execute_prepared_direct_color_plan( - plan: &PreparedDirectColorPlan, +fn j2k_pack_kernel_name_for( + color_space: &NativeColorSpace, + has_alpha: bool, + plane_count: usize, fmt: PixelFormat, -) -> Result { - let plans = [Arc::new(plan.clone())]; - let mut surfaces = execute_prepared_direct_color_plan_batch(&plans, fmt)?; - surfaces.pop().ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect color plan produced no surface".to_string(), - }) +) -> Option<&'static str> { + match (color_space, has_alpha, plane_count, fmt) { + (NativeColorSpace::Gray, false, 1, PixelFormat::Gray8) => Some("j2k_pack_gray8"), + (NativeColorSpace::RGB, false, 3, PixelFormat::Rgb8) + | (NativeColorSpace::RGB, true, 4, PixelFormat::Rgb8) => Some("j2k_pack_rgb8"), + (NativeColorSpace::RGB, false, 3, PixelFormat::Rgba8) => Some("j2k_pack_rgb_opaque_rgba8"), + (NativeColorSpace::RGB, true, 4, PixelFormat::Rgba8) => Some("j2k_pack_rgba8"), + (NativeColorSpace::Gray, false, 1, PixelFormat::Gray16) => Some("j2k_pack_gray16"), + (NativeColorSpace::RGB, false, 3, PixelFormat::Rgb16) => Some("j2k_pack_rgb16"), + _ => None, + } } #[cfg(target_os = "macos")] -pub(crate) fn execute_prepared_direct_color_plan_with_device( - plan: &PreparedDirectColorPlan, - fmt: PixelFormat, - device: &Device, -) -> Result { - with_runtime_for_device(device, |_| execute_prepared_direct_color_plan(plan, fmt)) +fn j2k_pack_pipeline_for<'a>( + runtime: &'a MetalRuntime, + kernel_name: &str, +) -> Result<&'a ComputePipelineState, Error> { + let pipeline = match kernel_name { + "j2k_pack_gray8" => &runtime.pack_gray8, + "j2k_pack_rgb8" => &runtime.pack_rgb8, + "j2k_pack_rgb_opaque_rgba8" => &runtime.pack_rgb_opaque_rgba8, + "j2k_pack_rgba8" => &runtime.pack_rgba8, + "j2k_pack_gray16" => &runtime.pack_gray16, + "j2k_pack_rgb16" => &runtime.pack_rgb16, + _ => { + return Err(Error::MetalKernel { + message: format!("unsupported validated J2K Metal pack kernel `{kernel_name}`"), + }); + } + }; + Ok(pipeline) } #[cfg(target_os = "macos")] -pub(crate) fn execute_prepared_direct_color_plan_batch( - plans: &[Arc], +fn output_shape_for<'a>( + color_space: &NativeColorSpace, + has_alpha: bool, + plane_count: usize, fmt: PixelFormat, -) -> Result, Error> { - if plans.is_empty() { - return Ok(Vec::new()); - } - if plans - .iter() - .any(|plan| !prepared_direct_color_plan_supports_runtime(plan, fmt)) - { + runtime: &'a MetalRuntime, +) -> Result<(u32, u32, &'a ComputePipelineState), Error> { + let Some(kernel_name) = j2k_pack_kernel_name_for(color_space, has_alpha, plane_count, fmt) + else { return Err(Error::MetalKernel { - message: "unsupported classic kernel input in direct component plan".to_string(), + message: format!( + "unsupported J2K Metal mapping for {color_space:?}, alpha={has_alpha}, planes={plane_count}, fmt={fmt:?}" + ), }); - } - - with_runtime(|runtime| { - let command_buffer = runtime.queue.new_command_buffer(); - let mut retained_buffers = Vec::new(); - let mut status_checks = Vec::new(); - let mut scratch_buffers = Vec::new(); - - if fmt == PixelFormat::Rgb8 { - if let Some(surfaces) = try_encode_stacked_mct_rgb8_direct_color_batch( - runtime, - command_buffer, - plans, - &mut retained_buffers, - &mut status_checks, - &mut scratch_buffers, - )? { - command_buffer.commit(); - command_buffer.wait_until_completed(); - for status_check in status_checks { - validate_direct_status(status_check)?; - } - drop(retained_buffers); - recycle_scratch_buffers(runtime, scratch_buffers); - return Ok(surfaces); - } + }; + let (output_channels, opaque_alpha) = match (color_space, has_alpha, plane_count, fmt) { + (NativeColorSpace::Gray, false, 1, PixelFormat::Gray8 | PixelFormat::Gray16) => (1, 0), + (NativeColorSpace::RGB, false, 3, PixelFormat::Rgb8 | PixelFormat::Rgb16) + | (NativeColorSpace::RGB, true, 4, PixelFormat::Rgb8) => (3, 0), + (NativeColorSpace::RGB, false, 3, PixelFormat::Rgba8) => (4, 1), + (NativeColorSpace::RGB, true, 4, PixelFormat::Rgba8) => (4, 0), + _ => { + return Err(Error::MetalKernel { + message: format!( + "unsupported validated J2K Metal pack shape for {color_space:?}, alpha={has_alpha}, planes={plane_count}, fmt={fmt:?}" + ), + }); } + }; + Ok(( + output_channels, + opaque_alpha, + j2k_pack_pipeline_for(runtime, kernel_name)?, + )) +} - let mut surfaces = Vec::with_capacity(plans.len()); - - for plan in plans { - let surface = encode_prepared_direct_color_plan_in_command_buffer( - runtime, - command_buffer, - plan, - fmt, - &mut retained_buffers, - &mut status_checks, - &mut scratch_buffers, - )?; - surfaces.push(surface); - } +#[cfg(target_os = "macos")] +fn required_classic_output_len(job: J2kCodeBlockDecodeJob<'_>) -> Result { + if job.height == 0 { + return Ok(0); + } - command_buffer.commit(); - command_buffer.wait_until_completed(); - for status_check in status_checks { - validate_direct_status(status_check)?; - } - drop(retained_buffers); - recycle_scratch_buffers(runtime, scratch_buffers); - Ok(surfaces) - }) + job.output_stride + .checked_mul(job.height as usize - 1) + .and_then(|prefix| prefix.checked_add(job.width as usize)) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal output size overflow".to_string(), + }) } #[cfg(target_os = "macos")] -fn signed_sample_bias(bit_depth: u8) -> f32 { - 2.0_f32.powi(i32::from(bit_depth) - 1) +fn classic_style_flags(style: j2k_native::J2kCodeBlockStyle) -> u32 { + let mut flags = 0u32; + if style.reset_context_probabilities { + flags |= J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES; + } + if style.termination_on_each_pass { + flags |= J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS; + } + if style.vertically_causal_context { + flags |= J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT; + } + if style.segmentation_symbols { + flags |= J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS; + } + if style.selective_arithmetic_coding_bypass { + flags |= J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS; + } + flags } #[cfg(target_os = "macos")] -fn encode_prepared_direct_color_plan_in_command_buffer( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - plan: &PreparedDirectColorPlan, - fmt: PixelFormat, - retained_buffers: &mut Vec, - status_checks: &mut Vec, - scratch_buffers: &mut Vec, -) -> Result { - if plan.component_plans.len() != 3 { +pub(crate) fn encode_forward_dwt53( + samples: &[f32], + width: u32, + height: u32, + num_levels: u8, +) -> Result { + if width == 0 || height == 0 { return Err(Error::MetalKernel { - message: format!( - "J2K MetalDirect color execution expected 3 component plans, got {}", - plan.component_plans.len() - ), + message: "J2K Metal forward DWT dimensions must be non-zero".to_string(), }); } - - let mut planes = Vec::with_capacity(3); - for component_plan in &plan.component_plans { - planes.push(encode_prepared_direct_component_plane_in_command_buffer( - runtime, - command_buffer, - component_plan, - retained_buffers, - status_checks, - scratch_buffers, - )?); + let expected_len = (width as usize) + .checked_mul(height as usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal forward DWT dimensions overflow".to_string(), + })?; + if samples.len() != expected_len { + return Err(Error::MetalKernel { + message: "J2K Metal forward DWT sample length mismatch".to_string(), + }); } - if plan.mct && fmt == PixelFormat::Rgb8 { - return Ok(encode_mct_rgb8_to_surface_in_command_buffer( - runtime, - command_buffer, - [&planes[0], &planes[1], &planes[2]], - plan.dimensions, - plan.bit_depths, - plan.transform, - )); - } + with_runtime(|runtime| { + let bytes = size_of_val(samples); + let buffer_a = copied_slice_buffer(&runtime.device, samples); + let buffer_b = runtime + .device + .new_buffer(bytes as u64, MTLResourceOptions::StorageModeShared); + let command_buffer = runtime.queue.new_command_buffer(); - if plan.mct { - let len = plan.dimensions.0 as usize * plan.dimensions.1 as usize; - status_checks.push(dispatch_inverse_mct_buffers_in_command_buffer( - runtime, - command_buffer, - [&planes[0], &planes[1], &planes[2]], - len, - plan.transform, - [ - signed_sample_bias(plan.bit_depths[0]), - signed_sample_bias(plan.bit_depths[1]), - signed_sample_bias(plan.bit_depths[2]), - ], - )?); - } + let mut current_width = width; + let mut current_height = height; + let mut shapes = Vec::new(); + let mut levels_run = 0u8; + let mut active_is_a = true; - let stage = PlaneStage { - dims: plan.dimensions, - plane_count: 3, - color_space: NativeColorSpace::RGB, - has_alpha: false, - bit_depths: [ - u32::from(plan.bit_depths[0]), - u32::from(plan.bit_depths[1]), - u32::from(plan.bit_depths[2]), - 0, - ], - planes: [ - Some(planes[0].clone()), - Some(planes[1].clone()), - Some(planes[2].clone()), - None, - ], - }; - encode_plane_stage_to_surface_in_command_buffer(runtime, command_buffer, &stage, fmt) + while levels_run < num_levels && (current_width >= 2 || current_height >= 2) { + let low_width = current_width.div_ceil(2); + let low_height = current_height.div_ceil(2); + let params = J2kForwardDwt53Params { + full_width: width, + current_width, + current_height, + low_width, + low_height, + }; + + if current_height >= 2 { + let (input, output) = + active_forward_dwt53_buffers(&buffer_a, &buffer_b, active_is_a); + dispatch_forward_dwt53_pass( + &runtime.fdwt53_vertical, + command_buffer, + input, + output, + params, + "J2K forward DWT 5/3 vertical", + ); + active_is_a = !active_is_a; + } + if current_width >= 2 { + let (input, output) = + active_forward_dwt53_buffers(&buffer_a, &buffer_b, active_is_a); + dispatch_forward_dwt53_pass( + &runtime.fdwt53_horizontal, + command_buffer, + input, + output, + params, + "J2K forward DWT 5/3 horizontal", + ); + active_is_a = !active_is_a; + } + + shapes.push(J2kForwardDwt53Level { + hl: Vec::new(), + lh: Vec::new(), + hh: Vec::new(), + width: current_width, + height: current_height, + low_width, + low_height, + high_width: current_width / 2, + high_height: current_height / 2, + }); + current_width = low_width; + current_height = low_height; + levels_run = levels_run.saturating_add(1); + } + + command_buffer.commit(); + command_buffer.wait_until_completed(); + + let active_buffer = if active_is_a { &buffer_a } else { &buffer_b }; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let transformed = unsafe { + core::slice::from_raw_parts(active_buffer.contents().cast::(), samples.len()) + }; + let output = extract_forward_dwt53_output( + transformed, + width, + current_width, + current_height, + shapes, + )?; + Ok(output) + }) } #[cfg(target_os = "macos")] -#[derive(Clone)] -struct DirectBandSlice { - band_id: J2kDirectBandId, - buffer: Buffer, - offset_bytes: usize, - window: BandRequiredRegion, -} +const FDWT97_ALPHA: f32 = -1.586_134_3; +#[cfg(target_os = "macos")] +const FDWT97_BETA: f32 = -0.052_980_117; +#[cfg(target_os = "macos")] +const FDWT97_GAMMA: f32 = 0.882_911_1; +#[cfg(target_os = "macos")] +const FDWT97_DELTA: f32 = 0.443_506_87; +#[cfg(target_os = "macos")] +const FDWT97_HIGH_PASS: u32 = 1; +#[cfg(target_os = "macos")] +const FDWT97_LOW_PASS: u32 = 0; #[cfg(target_os = "macos")] -fn lookup_direct_band_slice_entry( - bands: &[DirectBandSlice], - band_id: J2kDirectBandId, - rect: signinum_j2k_native::J2kRect, -) -> Result { - bands - .iter() - .find(|existing| existing.band_id == band_id) - .cloned() - .ok_or_else(|| Error::MetalKernel { - message: format!( - "missing J2K MetalDirect device band {} for rect ({}, {}, {}, {})", - band_id, rect.x0, rect.y0, rect.x1, rect.y1 - ), - }) +#[repr(C)] +#[derive(Clone, Copy)] +struct J2kForwardDwt97Params { + full_width: u32, + current_width: u32, + current_height: u32, + low_width: u32, + low_height: u32, + parity: u32, + coefficient: f32, + _reserved: u32, } #[cfg(target_os = "macos")] -fn lookup_direct_band_slice( - bands: &[DirectBandSlice], - band_id: J2kDirectBandId, - rect: signinum_j2k_native::J2kRect, -) -> Result<(Buffer, usize), Error> { - let entry = lookup_direct_band_slice_entry(bands, band_id, rect)?; - Ok((entry.buffer, entry.offset_bytes)) +pub(crate) fn encode_forward_dwt97( + samples: &[f32], + width: u32, + height: u32, + num_levels: u8, +) -> Result { + if width == 0 || height == 0 { + return Err(Error::MetalKernel { + message: "J2K Metal forward DWT dimensions must be non-zero".to_string(), + }); + } + let width_usize = usize::try_from(width).map_err(|_| Error::MetalKernel { + message: "J2K Metal forward DWT width does not fit usize".to_string(), + })?; + let height_usize = usize::try_from(height).map_err(|_| Error::MetalKernel { + message: "J2K Metal forward DWT height does not fit usize".to_string(), + })?; + let expected_len = width_usize + .checked_mul(height_usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal forward DWT dimensions overflow".to_string(), + })?; + if samples.len() != expected_len { + return Err(Error::MetalKernel { + message: "J2K Metal forward DWT sample length mismatch".to_string(), + }); + } + let bytes = size_of_val(samples); + let bytes_u64 = u64::try_from(bytes).map_err(|_| Error::MetalKernel { + message: "J2K Metal forward DWT buffer size exceeds u64".to_string(), + })?; + + with_runtime(|runtime| { + let buffer_a = copied_slice_buffer(&runtime.device, samples); + let buffer_b = runtime + .device + .new_buffer(bytes_u64, MTLResourceOptions::StorageModeShared); + let command_buffer = runtime.queue.new_command_buffer(); + + let mut current_width = width; + let mut current_height = height; + let mut shapes = Vec::new(); + let mut levels_run = 0u8; + let mut active_is_a = true; + + while levels_run < num_levels && (current_width >= 2 || current_height >= 2) { + let low_width = current_width.div_ceil(2); + let low_height = current_height.div_ceil(2); + let base_params = J2kForwardDwt97Params { + full_width: width, + current_width, + current_height, + low_width, + low_height, + parity: FDWT97_HIGH_PASS, + coefficient: 0.0, + _reserved: 0, + }; + + if current_height >= 2 { + dispatch_forward_dwt97_lift_steps( + &runtime.fdwt97_lift_vertical, + command_buffer, + &buffer_a, + &buffer_b, + active_is_a, + base_params, + "J2K forward DWT 9/7 vertical", + ); + let (input, output) = + active_forward_dwt53_buffers(&buffer_a, &buffer_b, active_is_a); + dispatch_forward_dwt97_pass( + &runtime.fdwt97_deinterleave_vertical, + command_buffer, + input, + output, + base_params, + "J2K forward DWT 9/7 vertical deinterleave", + ); + active_is_a = !active_is_a; + } + if current_width >= 2 { + dispatch_forward_dwt97_lift_steps( + &runtime.fdwt97_lift_horizontal, + command_buffer, + &buffer_a, + &buffer_b, + active_is_a, + base_params, + "J2K forward DWT 9/7 horizontal", + ); + let (input, output) = + active_forward_dwt53_buffers(&buffer_a, &buffer_b, active_is_a); + dispatch_forward_dwt97_pass( + &runtime.fdwt97_deinterleave_horizontal, + command_buffer, + input, + output, + base_params, + "J2K forward DWT 9/7 horizontal deinterleave", + ); + active_is_a = !active_is_a; + } + + shapes.push(J2kForwardDwt97Level { + hl: Vec::new(), + lh: Vec::new(), + hh: Vec::new(), + width: current_width, + height: current_height, + low_width, + low_height, + high_width: current_width / 2, + high_height: current_height / 2, + }); + current_width = low_width; + current_height = low_height; + levels_run = levels_run.saturating_add(1); + } + + command_buffer.commit(); + command_buffer.wait_until_completed(); + + let active_buffer = if active_is_a { &buffer_a } else { &buffer_b }; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let transformed = unsafe { + core::slice::from_raw_parts(active_buffer.contents().cast::(), samples.len()) + }; + extract_forward_dwt97_output(transformed, width, current_width, current_height, shapes) + }) } #[cfg(target_os = "macos")] -fn lookup_repeated_direct_band_layout_entry( - band_sets: &[Vec], - band_id: J2kDirectBandId, - rect: signinum_j2k_native::J2kRect, -) -> Result<(DirectBandSlice, u32), Error> { - let first_bands = band_sets.first().ok_or_else(|| Error::MetalKernel { - message: "missing J2K MetalDirect repeated band set".to_string(), +pub(crate) fn encode_deinterleave_to_f32( + job: J2kDeinterleaveToF32Job<'_>, +) -> Result>>, Error> { + validate_encode_deinterleave_to_f32_job(job)?; + let pixel_count = u32::try_from(job.num_pixels).map_err(|_| Error::MetalKernel { + message: "J2K Metal encode deinterleave pixel count exceeds u32".to_string(), })?; - let entry = lookup_direct_band_slice_entry(first_bands, band_id, rect)?; - let stride_bytes = if let Some(second_bands) = band_sets.get(1) { - let next = lookup_direct_band_slice_entry(second_bands, band_id, rect)?; - next.offset_bytes - .checked_sub(entry.offset_bytes) - .ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect repeated band offsets are not monotonic".to_string(), - })? - } else { - entry.window.width() as usize * entry.window.height() as usize * size_of::() - }; - if stride_bytes % size_of::() != 0 { + let bytes_per_sample = encode_deinterleave_bytes_per_sample(job.bit_depth); + let sample_count = job + .num_pixels + .checked_mul(usize::from(job.num_components)) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal encode deinterleave sample count overflow".to_string(), + })?; + let expected_len = job + .num_pixels + .checked_mul(usize::from(job.num_components)) + .and_then(|samples| samples.checked_mul(bytes_per_sample)) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal encode deinterleave input length overflow".to_string(), + })?; + if job.pixels.len() != expected_len { return Err(Error::MetalKernel { - message: "J2K MetalDirect repeated band stride is not f32-aligned".to_string(), + message: format!( + "J2K Metal encode deinterleave input length mismatch: expected {expected_len} bytes, got {}", + job.pixels.len() + ), }); } - let stride_elements = - u32::try_from(stride_bytes / size_of::()).map_err(|_| Error::MetalKernel { - message: "J2K MetalDirect repeated band stride exceeds u32".to_string(), + let src_stride = u32::try_from(expected_len).map_err(|_| Error::MetalKernel { + message: "J2K Metal encode deinterleave row stride exceeds u32".to_string(), + })?; + let plane_bytes = job + .num_pixels + .checked_mul(size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal encode deinterleave output length overflow".to_string(), })?; - Ok((entry, stride_elements)) -} -#[cfg(target_os = "macos")] -struct StackedDirectComponentPlane { - buffer: Buffer, - dimensions: (u32, u32), - count: usize, + with_runtime(|runtime| { + let input_buffer = copied_slice_buffer(&runtime.device, job.pixels); + let plane_buffers = (0..4) + .map(|_| { + runtime + .device + .new_buffer(plane_bytes as u64, MTLResourceOptions::StorageModeShared) + }) + .collect::>(); + let params = J2kLosslessDeinterleaveParams { + src_width: pixel_count, + src_height: 1, + src_stride, + dst_width: pixel_count, + dst_height: 1, + components: u32::from(job.num_components), + bytes_per_sample: bytes_per_sample as u32, + sample_offset: encode_deinterleave_sample_offset(job.bit_depth, job.signed), + signed_samples: u32::from(job.signed), + }; + + let command_buffer = runtime.queue.new_command_buffer(); + label_command_buffer(command_buffer, "j2k encode-stage deinterleave"); + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K encode-stage deinterleave"); + encoder.set_compute_pipeline_state(&runtime.lossless_deinterleave_to_planes); + encoder.set_buffer(0, Some(&input_buffer), 0); + encoder.set_buffer(1, Some(&plane_buffers[0]), 0); + encoder.set_buffer(2, Some(&plane_buffers[1]), 0); + encoder.set_buffer(3, Some(&plane_buffers[2]), 0); + encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const params).cast(), + ); + encoder.set_buffer(5, Some(&plane_buffers[3]), 0); + dispatch_2d_pipeline( + encoder, + &runtime.lossless_deinterleave_to_planes, + (pixel_count, 1), + ); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + let planes = plane_buffers + .iter() + .take(usize::from(job.num_components)) + .map(|buffer| { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let samples = unsafe { + core::slice::from_raw_parts(buffer.contents().cast::(), job.num_pixels) + }; + samples.to_vec() + }) + .collect(); + debug_assert_eq!( + sample_count.checked_mul(bytes_per_sample), + Some(expected_len) + ); + Ok(Some(planes)) + }) } #[cfg(target_os = "macos")] -fn try_encode_stacked_mct_rgb8_direct_color_batch( - runtime: &MetalRuntime, +fn dispatch_forward_dwt97_lift_steps( + pipeline: &ComputePipelineState, command_buffer: &CommandBufferRef, - plans: &[Arc], - retained_buffers: &mut Vec, - status_checks: &mut Vec, - scratch_buffers: &mut Vec, -) -> Result>, Error> { - let Some(first) = plans.first() else { - return Ok(Some(Vec::new())); - }; - if plans.len() <= 1 - || !first.mct - || first.component_plans.len() != 3 - || !plans.iter().all(|plan| { - plan.mct - && plan.dimensions == first.dimensions - && plan.bit_depths == first.bit_depths - && plan.transform == first.transform - && plan.component_plans.len() == 3 - }) - { - return Ok(None); - } - - let mut stacked_planes = Vec::with_capacity(3); - for component_idx in 0..3 { - let component_plan_refs = plans - .iter() - .map(|plan| &plan.component_plans[component_idx]) - .collect::>(); - if !supports_stacked_direct_component_plane_batch(&component_plan_refs) { - return Ok(None); - } - stacked_planes.push(encode_stacked_direct_component_plane_batch( - runtime, + buffer_a: &Buffer, + buffer_b: &Buffer, + active_is_a: bool, + base_params: J2kForwardDwt97Params, + label_prefix: &str, +) { + let active_buffer = if active_is_a { buffer_a } else { buffer_b }; + for (parity, coefficient) in [ + (FDWT97_HIGH_PASS, FDWT97_ALPHA), + (FDWT97_LOW_PASS, FDWT97_BETA), + (FDWT97_HIGH_PASS, FDWT97_GAMMA), + (FDWT97_LOW_PASS, FDWT97_DELTA), + ] { + let params = J2kForwardDwt97Params { + parity, + coefficient, + ..base_params + }; + dispatch_forward_dwt97_pass( + pipeline, command_buffer, - &component_plan_refs, - retained_buffers, - status_checks, - scratch_buffers, - )?); - } - - if !stacked_planes - .iter() - .all(|plane| plane.dimensions == first.dimensions && plane.count == plans.len()) - { - return Ok(None); + active_buffer, + active_buffer, + params, + label_prefix, + ); } +} - let surfaces = encode_batched_mct_rgb8_to_surfaces_in_command_buffer( - runtime, - command_buffer, - [ - &stacked_planes[0].buffer, - &stacked_planes[1].buffer, - &stacked_planes[2].buffer, - ], - first.dimensions, - plans.len(), - first.bit_depths, - first.transform, - )?; - Ok(Some(surfaces)) +#[cfg(target_os = "macos")] +fn dispatch_forward_dwt97_pass( + pipeline: &ComputePipelineState, + command_buffer: &CommandBufferRef, + input: &Buffer, + output: &Buffer, + params: J2kForwardDwt97Params, + label: &str, +) { + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, label); + encoder.set_compute_pipeline_state(pipeline); + encoder.set_buffer(0, Some(input), 0); + encoder.set_buffer(1, Some(output), 0); + encoder.set_bytes( + 2, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_2d_pipeline( + encoder, + pipeline, + (params.current_width, params.current_height), + ); + encoder.end_encoding(); } #[cfg(target_os = "macos")] -fn supports_stacked_direct_component_plane_batch(plans: &[&PreparedDirectGrayscalePlan]) -> bool { - let Some(first) = plans.first() else { - return false; - }; - if plans.iter().any(|plan| { - plan.dimensions != first.dimensions - || plan.bit_depth != first.bit_depth - || plan.steps.len() != first.steps.len() - }) { - return false; +fn validate_encode_deinterleave_to_f32_job(job: J2kDeinterleaveToF32Job<'_>) -> Result<(), Error> { + if job.num_pixels == 0 { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal encode deinterleave requires at least one pixel", + }); } - - let mut step_idx = 0; - while step_idx < first.steps.len() { - if let Some(group) = first.classic_group_starting_at(step_idx) { - if group.end_step <= step_idx - || !plans.iter().all(|plan| { - plan.classic_group_starting_at(step_idx) - .is_some_and(|other| classic_group_shapes_match(group, other)) - }) - { - return false; - } - step_idx = group.end_step; - continue; - } - if let Some(group) = first.ht_group_starting_at(step_idx) { - if group.end_step <= step_idx - || !plans.iter().all(|plan| { - plan.ht_group_starting_at(step_idx) - .is_some_and(|other| ht_group_shapes_match(group, other)) - }) - { - return false; - } - step_idx = group.end_step; - continue; - } - - match &first.steps[step_idx] { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { - if !plans.iter().all(|plan| { - matches!( - &plan.steps[step_idx], - PreparedDirectGrayscaleStep::ClassicSubBand(other) - if classic_sub_band_shapes_match(sub_band, other) - ) - }) { - return false; - } - } - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { - if !plans.iter().all(|plan| { - matches!( - &plan.steps[step_idx], - PreparedDirectGrayscaleStep::HtSubBand(other) - if ht_sub_band_shapes_match(sub_band, other) - ) - }) { - return false; - } - } - PreparedDirectGrayscaleStep::Idwt(idwt) => { - if !plans.iter().all(|plan| { - matches!( - &plan.steps[step_idx], - PreparedDirectGrayscaleStep::Idwt(other) - if idwt_shapes_match(idwt, other) - ) - }) { - return false; - } - } - PreparedDirectGrayscaleStep::Store(store) => { - if !plans.iter().all(|plan| { - matches!( - &plan.steps[step_idx], - PreparedDirectGrayscaleStep::Store(other) - if store_shapes_match(store, other) - ) - }) { - return false; - } - } - } - step_idx += 1; + if !(1..=4).contains(&job.num_components) { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal encode deinterleave supports 1-4 component samples", + }); } - - true -} - -#[cfg(target_os = "macos")] -fn prepared_direct_color_plan_supports_runtime( - plan: &PreparedDirectColorPlan, - fmt: PixelFormat, -) -> bool { - matches!( - fmt, - PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 - ) && plan.component_plans.len() == 3 - && plan - .component_plans - .iter() - .all(prepared_direct_component_plan_supports_runtime) + if job.bit_depth == 0 || job.bit_depth > 16 { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal encode deinterleave supports 1-16 bits per sample", + }); + } + Ok(()) } #[cfg(target_os = "macos")] -fn prepared_direct_component_plan_supports_runtime(plan: &PreparedDirectGrayscalePlan) -> bool { - plan.steps.iter().all(|step| match step { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => sub_band - .jobs - .iter() - .all(|job| classic_prepared_job_supports_runtime(job, &sub_band.segments)), - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { - sub_band.jobs.iter().all(ht_prepared_job_supports_runtime) - } - PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => true, - }) && plan.classic_groups.iter().all(|group| { - group - .jobs - .iter() - .all(|job| classic_prepared_job_supports_runtime(job, &group.segments)) - }) && plan - .ht_groups - .iter() - .all(|group| group.jobs.iter().all(ht_prepared_job_supports_runtime)) +fn encode_deinterleave_bytes_per_sample(bit_depth: u8) -> usize { + if bit_depth <= 8 { + 1 + } else { + 2 + } } #[cfg(target_os = "macos")] -fn classic_prepared_job_supports_runtime( - job: &J2kClassicCleanupBatchJob, - segments: &[J2kClassicSegment], -) -> bool { - if job.width == 0 || job.height == 0 { - return true; - } - if job.width > J2K_CLASSIC_MAX_WIDTH || job.height > J2K_CLASSIC_MAX_HEIGHT { - return false; - } - if job.output_stride < job.width { - return false; - } - if job.total_bitplanes == 0 || job.total_bitplanes > 31 || job.missing_msbs >= 31 { - return false; - } - let bitplanes = job.total_bitplanes.saturating_sub(job.missing_msbs); - if bitplanes == 0 { - return false; - } - let max_coding_passes = 1 + 3 * (bitplanes - 1); - if job.number_of_coding_passes == 0 || job.number_of_coding_passes > max_coding_passes { - return false; - } - - let start = job.segment_offset as usize; - let count = job.segment_count as usize; - let Some(end) = start.checked_add(count) else { - return false; - }; - if end > segments.len() || count == 0 { - return false; - } - - let uses_bypass = (job.style_flags & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) != 0; - let mut expected_start = 0u32; - let mut expected_offset = job.coded_offset; - for segment in &segments[start..end] { - if segment.start_coding_pass != expected_start - || segment.start_coding_pass > segment.end_coding_pass - { - return false; - } - if uses_bypass { - let expected_arithmetic = - segment.start_coding_pass <= 9 || segment.start_coding_pass % 3 == 0; - if (segment.use_arithmetic != 0) != expected_arithmetic { - return false; - } - if segment.use_arithmetic == 0 { - if segment.start_coding_pass % 3 != 1 { - return false; - } - if segment - .end_coding_pass - .saturating_sub(segment.start_coding_pass) - > 2 - { - return false; - } - if (segment.start_coding_pass..segment.end_coding_pass).any(|pass| pass % 3 == 0) { - return false; - } - } - } else if segment.use_arithmetic == 0 { - return false; - } - - let Some(data_end) = segment.data_offset.checked_add(segment.data_length) else { - return false; - }; - if segment.data_offset != expected_offset - || segment.data_offset < job.coded_offset - || data_end > job.coded_offset.saturating_add(job.coded_len) - { - return false; - } - expected_offset = data_end; - expected_start = segment.end_coding_pass; +fn encode_deinterleave_sample_offset(bit_depth: u8, signed: bool) -> u32 { + if signed { + 0 + } else { + 1u32 << (u32::from(bit_depth) - 1) } - - expected_start == job.number_of_coding_passes - && expected_offset == job.coded_offset.saturating_add(job.coded_len) } #[cfg(target_os = "macos")] -fn classic_group_shapes_match( - first: &PreparedClassicSubBandGroup, - other: &PreparedClassicSubBandGroup, -) -> bool { - first.end_step == other.end_step - && first.total_coefficients == other.total_coefficients - && first.members.len() == other.members.len() - && first - .members - .iter() - .zip(&other.members) - .all(|(left, right)| left.offset_elements == right.offset_elements) +fn active_forward_dwt53_buffers<'a>( + buffer_a: &'a Buffer, + buffer_b: &'a Buffer, + active_is_a: bool, +) -> (&'a Buffer, &'a Buffer) { + if active_is_a { + (buffer_a, buffer_b) + } else { + (buffer_b, buffer_a) + } } #[cfg(target_os = "macos")] -fn ht_group_shapes_match(first: &PreparedHtSubBandGroup, other: &PreparedHtSubBandGroup) -> bool { - first.end_step == other.end_step - && first.total_coefficients == other.total_coefficients - && first.members.len() == other.members.len() - && first - .members - .iter() - .zip(&other.members) - .all(|(left, right)| left.offset_elements == right.offset_elements) +fn dispatch_forward_dwt53_pass( + pipeline: &ComputePipelineState, + command_buffer: &CommandBufferRef, + input: &Buffer, + output: &Buffer, + params: J2kForwardDwt53Params, + label: &str, +) { + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, label); + encoder.set_compute_pipeline_state(pipeline); + encoder.set_buffer(0, Some(input), 0); + encoder.set_buffer(1, Some(output), 0); + encoder.set_bytes( + 2, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_2d_pipeline( + encoder, + pipeline, + (params.current_width, params.current_height), + ); + encoder.end_encoding(); } #[cfg(target_os = "macos")] -fn classic_sub_band_shapes_match( - first: &PreparedClassicSubBand, - other: &PreparedClassicSubBand, -) -> bool { - first.width == other.width && first.height == other.height -} +fn dispatch_forward_dwt53_batched_pass( + pipeline: &ComputePipelineState, + command_buffer: &CommandBufferRef, + inputs: &[Buffer], + outputs: &[Buffer], + params: J2kForwardDwt53BatchedParams, + label: &str, +) { + debug_assert!(!inputs.is_empty()); + debug_assert!(!outputs.is_empty()); + debug_assert!(params.component_count >= 1 && params.component_count <= 3); + let first_input_buffer = &inputs[0]; + let second_input_buffer = inputs.get(1).unwrap_or(first_input_buffer); + let third_input_buffer = inputs.get(2).unwrap_or(first_input_buffer); + let first_output_buffer = &outputs[0]; + let second_output_buffer = outputs.get(1).unwrap_or(first_output_buffer); + let third_output_buffer = outputs.get(2).unwrap_or(first_output_buffer); -#[cfg(target_os = "macos")] -fn ht_sub_band_shapes_match(first: &PreparedHtSubBand, other: &PreparedHtSubBand) -> bool { - first.width == other.width && first.height == other.height + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, label); + encoder.set_compute_pipeline_state(pipeline); + encoder.set_buffer(0, Some(first_input_buffer), 0); + encoder.set_buffer(1, Some(second_input_buffer), 0); + encoder.set_buffer(2, Some(third_input_buffer), 0); + encoder.set_buffer(3, Some(first_output_buffer), 0); + encoder.set_buffer(4, Some(second_output_buffer), 0); + encoder.set_buffer(5, Some(third_output_buffer), 0); + encoder.set_bytes( + 6, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_3d_pipeline( + encoder, + pipeline, + ( + params.current_width, + params.current_height, + params.component_count, + ), + ); + encoder.end_encoding(); } #[cfg(target_os = "macos")] -fn rect_shapes_match( - first: signinum_j2k_native::J2kRect, - other: signinum_j2k_native::J2kRect, -) -> bool { - first.x0 == other.x0 && first.y0 == other.y0 && first.x1 == other.x1 && first.y1 == other.y1 -} +fn extract_forward_dwt53_output( + transformed: &[f32], + full_width: u32, + ll_width: u32, + ll_height: u32, + mut shapes: Vec, +) -> Result { + let full_width_usize = full_width as usize; + let mut ll = Vec::with_capacity((ll_width as usize) * (ll_height as usize)); + for y in 0..ll_height as usize { + let row_start = y + .checked_mul(full_width_usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal forward DWT LL row offset overflow".to_string(), + })?; + ll.extend_from_slice(&transformed[row_start..row_start + ll_width as usize]); + } -#[cfg(target_os = "macos")] -fn idwt_shapes_match(first: &PreparedDirectIdwt, other: &PreparedDirectIdwt) -> bool { - first.step.transform == other.step.transform - && rect_shapes_match(first.step.rect, other.step.rect) - && first.output_window.x0 == other.output_window.x0 - && first.output_window.y0 == other.output_window.y0 - && first.output_window.x1 == other.output_window.x1 - && first.output_window.y1 == other.output_window.y1 - && rect_shapes_match(first.step.ll, other.step.ll) - && rect_shapes_match(first.step.hl, other.step.hl) - && rect_shapes_match(first.step.lh, other.step.lh) - && rect_shapes_match(first.step.hh, other.step.hh) -} + for shape in &mut shapes { + shape.hl = extract_subband( + transformed, + full_width_usize, + shape.low_width, + 0, + shape.high_width, + shape.low_height, + )?; + shape.lh = extract_subband( + transformed, + full_width_usize, + 0, + shape.low_height, + shape.low_width, + shape.high_height, + )?; + shape.hh = extract_subband( + transformed, + full_width_usize, + shape.low_width, + shape.low_height, + shape.high_width, + shape.high_height, + )?; + } + shapes.reverse(); -#[cfg(target_os = "macos")] -fn store_shapes_match(first: &J2kDirectStoreStep, other: &J2kDirectStoreStep) -> bool { - rect_shapes_match(first.input_rect, other.input_rect) - && first.source_x == other.source_x - && first.source_y == other.source_y - && first.copy_width == other.copy_width - && first.copy_height == other.copy_height - && first.output_width == other.output_width - && first.output_height == other.output_height - && first.output_x == other.output_x - && first.output_y == other.output_y - && first.addend.to_bits() == other.addend.to_bits() + Ok(J2kForwardDwt53Output { + ll, + ll_width, + ll_height, + levels: shapes, + }) } #[cfg(target_os = "macos")] -fn encode_stacked_direct_component_plane_batch( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - plans: &[&PreparedDirectGrayscalePlan], - retained_buffers: &mut Vec, - status_checks: &mut Vec, - scratch_buffers: &mut Vec, -) -> Result { - let Some(first) = plans.first() else { - return Err(Error::MetalKernel { - message: "J2K MetalDirect color batch has no component plans".to_string(), - }); - }; +fn extract_forward_dwt97_output( + transformed: &[f32], + full_width: u32, + ll_width: u32, + ll_height: u32, + mut shapes: Vec, +) -> Result { + let full_width_usize = usize::try_from(full_width).map_err(|_| Error::MetalKernel { + message: "J2K Metal forward DWT full width does not fit usize".to_string(), + })?; + let ll_width_usize = usize::try_from(ll_width).map_err(|_| Error::MetalKernel { + message: "J2K Metal forward DWT LL width does not fit usize".to_string(), + })?; + let ll_height_usize = usize::try_from(ll_height).map_err(|_| Error::MetalKernel { + message: "J2K Metal forward DWT LL height does not fit usize".to_string(), + })?; + let ll_len = ll_width_usize + .checked_mul(ll_height_usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal forward DWT LL dimensions overflow".to_string(), + })?; + let mut ll = Vec::with_capacity(ll_len); + for y in 0..ll_height_usize { + let row_start = y + .checked_mul(full_width_usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal forward DWT LL row offset overflow".to_string(), + })?; + let row_end = row_start + .checked_add(ll_width_usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal forward DWT LL row end overflow".to_string(), + })?; + let row = transformed + .get(row_start..row_end) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal forward DWT LL row out of bounds".to_string(), + })?; + ll.extend_from_slice(row); + } - let count = plans.len(); - let mut band_sets = vec![Vec::::new(); count]; - let mut final_plane = None; - let mut step_idx = 0; + for shape in &mut shapes { + shape.hl = extract_subband( + transformed, + full_width_usize, + shape.low_width, + 0, + shape.high_width, + shape.low_height, + )?; + shape.lh = extract_subband( + transformed, + full_width_usize, + 0, + shape.low_height, + shape.low_width, + shape.high_height, + )?; + shape.hh = extract_subband( + transformed, + full_width_usize, + shape.low_width, + shape.low_height, + shape.high_width, + shape.high_height, + )?; + } + shapes.reverse(); - while step_idx < first.steps.len() { - if let Some(group) = first.classic_group_starting_at(step_idx) { - let groups = plans - .iter() - .map(|plan| { - plan.classic_group_starting_at(step_idx) - .expect("preflight validated classic group") - }) - .collect::>(); - let output = take_f32_scratch_buffer(runtime, group.total_coefficients * count); - let (buffers, status_check) = - encode_distinct_classic_sub_band_groups_to_buffer_in_command_buffer( - runtime, - command_buffer, - &groups, - &output.buffer, - scratch_buffers, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - let stride_bytes = group.total_coefficients * size_of::(); - for (instance_idx, bands) in band_sets.iter_mut().enumerate() { - for member in &groups[instance_idx].members { - bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes - + member.offset_elements * size_of::(), - window: member.window, - }); - } - } - scratch_buffers.push(output); - step_idx = group.end_step; - continue; - } + Ok(J2kForwardDwt97Output { + ll, + ll_width, + ll_height, + levels: shapes, + }) +} - if let Some(group) = first.ht_group_starting_at(step_idx) { - let groups = plans - .iter() - .map(|plan| { - plan.ht_group_starting_at(step_idx) - .expect("preflight validated HT group") - }) - .collect::>(); - let output = take_f32_scratch_buffer(runtime, group.total_coefficients * count); - let (buffers, status_check) = - encode_distinct_ht_sub_band_groups_to_buffer_in_command_buffer( - runtime, - command_buffer, - &groups, - &output.buffer, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - let stride_bytes = group.total_coefficients * size_of::(); - for (instance_idx, bands) in band_sets.iter_mut().enumerate() { - for member in &groups[instance_idx].members { - bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes - + member.offset_elements * size_of::(), - window: member.window, - }); - } - } - scratch_buffers.push(output); - step_idx = group.end_step; - continue; - } +#[cfg(target_os = "macos")] +fn extract_subband( + transformed: &[f32], + full_width: usize, + x0: u32, + y0: u32, + width: u32, + height: u32, +) -> Result, Error> { + let mut out = Vec::with_capacity((width as usize) * (height as usize)); + for y in 0..height as usize { + let row_start = (y0 as usize) + .checked_add(y) + .and_then(|row| row.checked_mul(full_width)) + .and_then(|row| row.checked_add(x0 as usize)) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal forward DWT subband offset overflow".to_string(), + })?; + out.extend_from_slice(&transformed[row_start..row_start + width as usize]); + } + Ok(out) +} - match &first.steps[step_idx] { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { - let sub_bands = plans - .iter() - .map(|plan| match &plan.steps[step_idx] { - PreparedDirectGrayscaleStep::ClassicSubBand(other) => other, - _ => unreachable!("preflight validated classic sub-band"), - }) - .collect::>(); - let per_instance_len = sub_band.width as usize * sub_band.height as usize; - let output = take_f32_scratch_buffer(runtime, per_instance_len * count); - let (buffers, status_check) = - encode_distinct_classic_sub_bands_to_buffer_in_command_buffer( - runtime, - command_buffer, - &sub_bands, - &output.buffer, - scratch_buffers, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in band_sets.iter_mut().enumerate() { - bands.push(DirectBandSlice { - band_id: sub_bands[instance_idx].band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes, - window: BandRequiredRegion::full( - sub_bands[instance_idx].width, - sub_bands[instance_idx].height, - ), - }); - } - scratch_buffers.push(output); - } - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { - let sub_bands = plans - .iter() - .map(|plan| match &plan.steps[step_idx] { - PreparedDirectGrayscaleStep::HtSubBand(other) => other, - _ => unreachable!("preflight validated HT sub-band"), - }) - .collect::>(); - let per_instance_len = sub_band.width as usize * sub_band.height as usize; - let output = take_f32_scratch_buffer(runtime, per_instance_len * count); - let (buffers, status_check) = - encode_distinct_ht_sub_bands_to_buffer_in_command_buffer( - runtime, - command_buffer, - &sub_bands, - &output.buffer, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in band_sets.iter_mut().enumerate() { - bands.push(DirectBandSlice { - band_id: sub_bands[instance_idx].band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes, - window: BandRequiredRegion::full( - sub_bands[instance_idx].width, - sub_bands[instance_idx].height, - ), - }); - } - scratch_buffers.push(output); - } - PreparedDirectGrayscaleStep::Idwt(idwt) => { - let per_instance_len = prepared_idwt_output_len(idwt); - let output = take_f32_scratch_buffer(runtime, per_instance_len * count); - match idwt.step.transform { - J2kWaveletTransform::Reversible53 => { - let (ll, low_low_stride) = lookup_repeated_direct_band_layout_entry( - &band_sets, - idwt.step.ll_band_id, - idwt.step.ll, - )?; - let (hl, high_low_stride) = lookup_repeated_direct_band_layout_entry( - &band_sets, - idwt.step.hl_band_id, - idwt.step.hl, - )?; - let (lh, low_high_stride) = lookup_repeated_direct_band_layout_entry( - &band_sets, - idwt.step.lh_band_id, - idwt.step.lh, - )?; - let (hh, high_high_stride) = lookup_repeated_direct_band_layout_entry( - &band_sets, - idwt.step.hh_band_id, - idwt.step.hh, - )?; - let params = repeated_idwt_params( - idwt, - idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), - PreparedIdwtInputStrides { - ll: low_low_stride, - hl: high_low_stride, - lh: low_high_stride, - hh: high_high_stride, - }, - count, - "color", - )?; - dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets( - runtime, - command_buffer, - &ll.buffer, - ll.offset_bytes, - &hl.buffer, - hl.offset_bytes, - &lh.buffer, - lh.offset_bytes, - &hh.buffer, - hh.offset_bytes, - params, - &output.buffer, - ); - } - J2kWaveletTransform::Irreversible97 => { - for (instance_idx, bands) in band_sets.iter().enumerate() { - let PreparedDirectGrayscaleStep::Idwt(step) = - &plans[instance_idx].steps[step_idx] - else { - unreachable!("preflight validated IDWT") - }; - let ll = lookup_direct_band_slice_entry( - bands, - step.step.ll_band_id, - step.step.ll, - )?; - let hl = lookup_direct_band_slice_entry( - bands, - step.step.hl_band_id, - step.step.hl, - )?; - let lh = lookup_direct_band_slice_entry( - bands, - step.step.lh_band_id, - step.step.lh, - )?; - let hh = lookup_direct_band_slice_entry( - bands, - step.step.hh_band_id, - step.step.hh, - )?; - let params = prepared_idwt_params( - step, - idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), - ); - status_checks.push( - dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets( - runtime, - command_buffer, - &ll.buffer, - ll.offset_bytes, - &hl.buffer, - hl.offset_bytes, - &lh.buffer, - lh.offset_bytes, - &hh.buffer, - hh.offset_bytes, - params, - &output.buffer, - instance_idx * per_instance_len * size_of::(), - ), - ); - } - } - } - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in band_sets.iter_mut().enumerate() { - let PreparedDirectGrayscaleStep::Idwt(step) = - &plans[instance_idx].steps[step_idx] - else { - unreachable!("preflight validated IDWT") - }; - bands.push(DirectBandSlice { - band_id: step.step.output_band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes, - window: step.output_window, - }); - } - scratch_buffers.push(output); - } - PreparedDirectGrayscaleStep::Store(store) => { - let (input, _) = - lookup_direct_band_slice(&band_sets[0], store.input_band_id, store.input_rect)?; - let per_instance_len = store.output_width as usize * store.output_height as usize; - let output = take_f32_scratch_buffer(runtime, per_instance_len * count); - dispatch_store_component_repeated_in_command_buffer( - runtime, - command_buffer, - &input, - &output.buffer, - J2kRepeatedStoreParams { - input_width: store.input_rect.width(), - input_height: store.input_rect.height(), - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_height: store.output_height, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - batch_count: u32::try_from(count).map_err(|_| Error::MetalKernel { - message: "J2K MetalDirect color store batch count exceeds u32" - .to_string(), - })?, - }, - ); - final_plane = Some(output.buffer.clone()); - scratch_buffers.push(output); - } - } - step_idx += 1; - } +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug)] +pub(crate) struct J2kLosslessDeviceCodeBlock { + pub(crate) coefficient_offset: u32, + pub(crate) component: u32, + pub(crate) subband_x: u32, + pub(crate) subband_y: u32, + pub(crate) block_x: u32, + pub(crate) block_y: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) sub_band_type: j2k_native::J2kSubBandType, + pub(crate) total_bitplanes: u8, +} - let buffer = final_plane.ok_or_else(|| Error::MetalKernel { - message: "J2K MetalDirect color component batch did not produce a final plane".to_string(), - })?; - Ok(StackedDirectComponentPlane { - buffer, - dimensions: first.dimensions, - count, - }) +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug)] +pub(crate) struct J2kLosslessDevicePrepareJob<'a> { + pub(crate) input: &'a Buffer, + pub(crate) input_byte_offset: usize, + pub(crate) input_width: u32, + pub(crate) input_height: u32, + pub(crate) input_pitch_bytes: usize, + pub(crate) output_width: u32, + pub(crate) output_height: u32, + pub(crate) components: u8, + pub(crate) bytes_per_sample: u8, + pub(crate) bit_depth: u8, + pub(crate) num_decomposition_levels: u8, + pub(crate) coefficient_count: usize, } #[cfg(target_os = "macos")] -fn ht_prepared_job_supports_runtime(job: &J2kHtCleanupBatchJob) -> bool { - if job.width == 0 || job.height == 0 { - return true; - } - job.output_stride >= job.width && crate::ht::supports_metal_ht_geometry(job.width, job.height) +pub(crate) struct J2kLosslessDeviceBatchPrepareItem<'a> { + pub(crate) tile_index: usize, + pub(crate) job: J2kLosslessDevicePrepareJob<'a>, + pub(crate) code_blocks: Vec, } #[cfg(target_os = "macos")] -#[allow(clippy::too_many_arguments)] -fn encode_repeated_direct_grayscale_plan_in_command_buffer( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - plan: &PreparedDirectGrayscalePlan, - fmt: PixelFormat, - count: usize, - retained_buffers: &mut Vec, - status_checks: &mut Vec, - scratch_buffers: &mut Vec, -) -> Result, Error> { - let mut band_sets = vec![Vec::::new(); count]; - let mut surfaces = Vec::with_capacity(count); - let mut stacked_outputs = true; - let mut step_idx = 0; +pub(crate) struct J2kPreparedLosslessDeviceCodeBlocks { + coefficient_buffer: Buffer, + coefficient_byte_offset: usize, + coefficient_byte_len: usize, + coefficient_buffer_is_batch_shared: bool, + code_blocks: Vec, + recyclable_private_buffers: Vec<(usize, Buffer)>, + _prepare_command_buffer: CommandBuffer, + _prepare_deinterleave_rct_command_buffer: Option, + _prepare_dwt53_command_buffer: Option, + _prepare_dwt53_vertical_command_buffers: Vec, + _prepare_dwt53_horizontal_command_buffers: Vec, + _prepare_coefficient_extract_command_buffer: Option, + _deinterleave_status_buffer: Buffer, + _plane_buffers: Vec, + _scratch_buffers: Vec, + _coefficient_job_buffer: Buffer, +} - while step_idx < plan.steps.len() { - if let Some(group) = plan.classic_group_starting_at(step_idx) { - let per_instance_len = group.total_coefficients; - let output = take_f32_scratch_buffer(runtime, per_instance_len * count); - let (buffers, status_check) = - encode_repeated_classic_sub_band_group_to_buffer_in_command_buffer( - runtime, - command_buffer, - group, - count, - &output.buffer, - scratch_buffers, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in band_sets.iter_mut().enumerate() { - for member in &group.members { - bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes - + member.offset_elements * size_of::(), - window: member.window, - }); - } - } - scratch_buffers.push(output); - step_idx = group.end_step; - continue; - } +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug)] +pub(crate) struct J2kResidentPacketizationSubband { + pub(crate) code_block_start: u32, + pub(crate) code_block_count: u32, + pub(crate) num_cbs_x: u32, + pub(crate) num_cbs_y: u32, +} - if let Some(group) = plan.ht_group_starting_at(step_idx) { - let per_instance_len = group.total_coefficients; - let output = take_f32_scratch_buffer(runtime, per_instance_len * count); - let (buffers, status_check) = - encode_repeated_ht_sub_band_group_to_buffer_in_command_buffer( - runtime, - command_buffer, - group, - count, - &output.buffer, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in band_sets.iter_mut().enumerate() { - for member in &group.members { - bands.push(DirectBandSlice { - band_id: member.band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes - + member.offset_elements * size_of::(), - window: member.window, - }); - } - } - scratch_buffers.push(output); - step_idx = group.end_step; - continue; - } +#[cfg(target_os = "macos")] +#[derive(Clone, Debug)] +pub(crate) struct J2kResidentPacketizationResolution { + pub(crate) subbands: Vec, +} - let step = &plan.steps[step_idx]; - match step { - PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { - let per_instance_len = sub_band.width as usize * sub_band.height as usize; - let output = take_f32_scratch_buffer(runtime, per_instance_len * count); - let (buffers, status_check) = - encode_repeated_classic_sub_band_to_buffer_in_command_buffer( - runtime, - command_buffer, - sub_band, - count, - &output.buffer, - scratch_buffers, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in band_sets.iter_mut().enumerate() { - bands.push(DirectBandSlice { - band_id: sub_band.band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - } - scratch_buffers.push(output); - } - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { - let per_instance_len = sub_band.width as usize * sub_band.height as usize; - let output = take_f32_scratch_buffer(runtime, per_instance_len * count); - let (buffers, status_check) = - encode_repeated_ht_sub_band_to_buffer_in_command_buffer( - runtime, - command_buffer, - sub_band, - count, - &output.buffer, - )?; - retained_buffers.extend(buffers); - status_checks.push(status_check); - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in band_sets.iter_mut().enumerate() { - bands.push(DirectBandSlice { - band_id: sub_band.band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes, - window: BandRequiredRegion::full(sub_band.width, sub_band.height), - }); - } - scratch_buffers.push(output); - } - PreparedDirectGrayscaleStep::Idwt(idwt) => match idwt.step.transform { - J2kWaveletTransform::Reversible53 if stacked_outputs => { - let (ll, low_low_stride) = lookup_repeated_direct_band_layout_entry( - &band_sets, - idwt.step.ll_band_id, - idwt.step.ll, - )?; - let (hl, high_low_stride) = lookup_repeated_direct_band_layout_entry( - &band_sets, - idwt.step.hl_band_id, - idwt.step.hl, - )?; - let (lh, low_high_stride) = lookup_repeated_direct_band_layout_entry( - &band_sets, - idwt.step.lh_band_id, - idwt.step.lh, - )?; - let (hh, high_high_stride) = lookup_repeated_direct_band_layout_entry( - &band_sets, - idwt.step.hh_band_id, - idwt.step.hh, - )?; - let params = repeated_idwt_params( - idwt, - idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), - PreparedIdwtInputStrides { - ll: low_low_stride, - hl: high_low_stride, - lh: low_high_stride, - hh: high_high_stride, - }, - count, - "repeated", - )?; - let per_instance_len = prepared_idwt_output_len(idwt); - let output = take_f32_scratch_buffer(runtime, per_instance_len * count); - dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets( - runtime, - command_buffer, - &ll.buffer, - ll.offset_bytes, - &hl.buffer, - hl.offset_bytes, - &lh.buffer, - lh.offset_bytes, - &hh.buffer, - hh.offset_bytes, - params, - &output.buffer, - ); - let stride_bytes = per_instance_len * size_of::(); - for (instance_idx, bands) in band_sets.iter_mut().enumerate() { - bands.push(DirectBandSlice { - band_id: idwt.step.output_band_id, - buffer: output.buffer.clone(), - offset_bytes: instance_idx * stride_bytes, - window: idwt.output_window, - }); - } - scratch_buffers.push(output); - } - _ => { - stacked_outputs = false; - for bands in &mut band_sets { - let ll = lookup_direct_band_slice_entry( - bands, - idwt.step.ll_band_id, - idwt.step.ll, - )?; - let hl = lookup_direct_band_slice_entry( - bands, - idwt.step.hl_band_id, - idwt.step.hl, - )?; - let lh = lookup_direct_band_slice_entry( - bands, - idwt.step.lh_band_id, - idwt.step.lh, - )?; - let hh = lookup_direct_band_slice_entry( - bands, - idwt.step.hh_band_id, - idwt.step.hh, - )?; - let params = prepared_idwt_params( - idwt, - idwt_input_windows_from_slices(&ll, &hl, &lh, &hh), - ); - let output = - take_f32_scratch_buffer(runtime, prepared_idwt_output_len(idwt)); - match idwt.step.transform { - J2kWaveletTransform::Reversible53 => { - dispatch_reversible53_single_decomposition_buffers_in_command_buffer_with_offsets( - runtime, - command_buffer, - &ll.buffer, - ll.offset_bytes, - &hl.buffer, - hl.offset_bytes, - &lh.buffer, - lh.offset_bytes, - &hh.buffer, - hh.offset_bytes, - params, - &output.buffer, - 0, - ); - } - J2kWaveletTransform::Irreversible97 => status_checks.push( - dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_offsets( - runtime, - command_buffer, - &ll.buffer, - ll.offset_bytes, - &hl.buffer, - hl.offset_bytes, - &lh.buffer, - lh.offset_bytes, - &hh.buffer, - hh.offset_bytes, - params, - &output.buffer, - 0, - ), - ), - } - bands.push(DirectBandSlice { - band_id: idwt.step.output_band_id, - buffer: output.buffer.clone(), - offset_bytes: 0, - window: idwt.output_window, - }); - scratch_buffers.push(output); - } - } - }, - PreparedDirectGrayscaleStep::Store(store) => { - if stacked_outputs { - let (input, _) = lookup_direct_band_slice( - &band_sets[0], - store.input_band_id, - store.input_rect, - )?; - let batch_count = u32::try_from(count).map_err(|_| Error::MetalKernel { - message: "J2K MetalDirect repeated store batch count exceeds u32" - .to_string(), - })?; - if matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { - let scale = j2k_scalar_pack_params(u32::from(plan.bit_depth)); - surfaces.extend(encode_repeated_gray_store_to_surfaces_in_command_buffer( - runtime, - command_buffer, - &input, - J2kRepeatedGrayStoreParams { - input_width: store.input_rect.width(), - input_height: store.input_rect.height(), - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_height: store.output_height, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - batch_count, - max_value: scale.max_value, - u8_scale: scale.u8_scale, - u16_scale: scale.u16_scale, - }, - plan.dimensions, - fmt, - count, - )?); - } else { - let per_instance_len = - store.output_width as usize * store.output_height as usize; - let output = take_f32_scratch_buffer(runtime, per_instance_len * count); - dispatch_store_component_repeated_in_command_buffer( - runtime, - command_buffer, - &input, - &output.buffer, - J2kRepeatedStoreParams { - input_width: store.input_rect.width(), - input_height: store.input_rect.height(), - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_height: store.output_height, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - batch_count, - }, - ); - retained_buffers.push(output.buffer.clone()); - surfaces.extend(encode_repeated_gray_plane_to_surfaces_in_command_buffer( - runtime, - command_buffer, - &output.buffer, - plan.dimensions, - plan.bit_depth, - fmt, - count, - )?); - scratch_buffers.push(output); - } - } else { - for bands in &band_sets { - let (input, input_offset) = - lookup_direct_band_slice(bands, store.input_band_id, store.input_rect)?; - let output = take_f32_scratch_buffer( - runtime, - store.output_width as usize * store.output_height as usize, - ); - let params = J2kStoreParams { - input_width: store.input_rect.width(), - source_x: store.source_x, - source_y: store.source_y, - copy_width: store.copy_width, - copy_height: store.copy_height, - output_width: store.output_width, - output_x: store.output_x, - output_y: store.output_y, - addend: store.addend, - }; - dispatch_store_component_buffer_in_command_buffer_with_offsets( - runtime, - command_buffer, - &input, - input_offset, - &output.buffer, - 0, - params, - ); - retained_buffers.push(output.buffer.clone()); - surfaces.push(encode_gray_plane_to_surface_in_command_buffer_with_offset( - runtime, - command_buffer, - &output.buffer, - 0, - plan.dimensions, - plan.bit_depth, - fmt, - )?); - scratch_buffers.push(output); - } - } - } - } - step_idx += 1; +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +pub(crate) struct J2kResidentPacketizationEncodeJob<'a> { + pub(crate) resolution_count: u32, + pub(crate) num_layers: u8, + pub(crate) num_components: u8, + pub(crate) code_block_count: u32, + pub(crate) packet_descriptors: &'a [J2kPacketizationPacketDescriptor], + pub(crate) resolutions: &'a [J2kResidentPacketizationResolution], +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum J2kLosslessCodestreamBlockCodingMode { + Classic, + HighThroughput, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug)] +pub(crate) struct J2kLosslessCodestreamAssemblyJob { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) num_components: u8, + pub(crate) bit_depth: u8, + pub(crate) signed: bool, + pub(crate) num_decomposition_levels: u8, + pub(crate) use_mct: bool, + pub(crate) guard_bits: u8, + pub(crate) code_block_width_exp: u8, + pub(crate) code_block_height_exp: u8, + pub(crate) progression_order: EncodeProgressionOrder, + pub(crate) write_tlm: bool, + pub(crate) block_coding_mode: J2kLosslessCodestreamBlockCodingMode, +} + +#[cfg(target_os = "macos")] +pub(crate) struct J2kResidentLosslessTier1CodeBlocks { + output_buffer: Buffer, + status_buffer: Buffer, + job_buffer: Buffer, + batch_jobs: Vec, + code_blocks: Vec, + output_capacity_total: usize, + _segment_buffer: Buffer, + tier1_command_buffer: CommandBuffer, + _coefficient_buffer: Buffer, + prepare_command_buffer: CommandBuffer, + _deinterleave_status_buffer: Buffer, + _plane_buffers: Vec, + _scratch_buffers: Vec, + _coefficient_job_buffer: Buffer, +} + +#[cfg(target_os = "macos")] +pub(crate) struct J2kResidentLosslessHtCodeBlocks { + output_buffer: Buffer, + status_buffer: Buffer, + job_buffer: Buffer, + batch_jobs: Vec, + code_blocks: Vec, + output_capacity_total: usize, + tier1_command_buffer: CommandBuffer, + _coefficient_buffer: Buffer, + prepare_command_buffer: CommandBuffer, + _deinterleave_status_buffer: Buffer, + _plane_buffers: Vec, + _scratch_buffers: Vec, + _coefficient_job_buffer: Buffer, +} + +/// Family descriptor that lets the resident Tier-1 codestream drivers run +/// generically over the classic and HT code-block tables. Associated consts +/// keep diagnostics byte-identical to the pre-convergence per-family bodies. +#[cfg(target_os = "macos")] +pub(crate) trait ResidentLosslessTier1Metal { + /// Prefix for Tier-2 resident packet diagnostics ("" for classic). + const TIER2_PREFIX: &'static str; + /// Family name used in codestream assembly diagnostics. + const FAMILY_NAME: &'static str; + /// Value stored in `J2kResidentPacketBlock::block_coding_mode`. + const BLOCK_CODING_MODE: u32; + const CODESTREAM_STATUS_STAGE: &'static str; + const CODESTREAM_LENGTH_ERROR: &'static str; + const CODESTREAM_CAPACITY_ERROR: &'static str; + + fn batch_job_count(&self) -> usize; + fn code_block_count(&self) -> usize; + fn output_capacity_total(&self) -> usize; + fn output_buffer(&self) -> &Buffer; + fn status_buffer(&self) -> &Buffer; + fn job_buffer(&self) -> &Buffer; + fn prepare_command_buffer(&self) -> &CommandBuffer; + fn tier1_command_buffer(&self) -> &CommandBuffer; + fn packet_block_prepare_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState; +} + +#[cfg(target_os = "macos")] +impl ResidentLosslessTier1Metal for J2kResidentLosslessTier1CodeBlocks { + const TIER2_PREFIX: &'static str = ""; + const FAMILY_NAME: &'static str = "J2K"; + const BLOCK_CODING_MODE: u32 = 0; + const CODESTREAM_STATUS_STAGE: &'static str = "J2K codestream assembly"; + const CODESTREAM_LENGTH_ERROR: &'static str = + "J2K Metal codestream output length exceeds usize"; + const CODESTREAM_CAPACITY_ERROR: &'static str = + "J2K Metal codestream output length exceeds buffer"; + + fn batch_job_count(&self) -> usize { + self.batch_jobs.len() + } + fn code_block_count(&self) -> usize { + self.code_blocks.len() + } + fn output_capacity_total(&self) -> usize { + self.output_capacity_total + } + fn output_buffer(&self) -> &Buffer { + &self.output_buffer + } + fn status_buffer(&self) -> &Buffer { + &self.status_buffer + } + fn job_buffer(&self) -> &Buffer { + &self.job_buffer + } + fn prepare_command_buffer(&self) -> &CommandBuffer { + &self.prepare_command_buffer + } + fn tier1_command_buffer(&self) -> &CommandBuffer { + &self.tier1_command_buffer + } + fn packet_block_prepare_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.packet_block_prepare_resident_classic + } +} + +#[cfg(target_os = "macos")] +impl ResidentLosslessTier1Metal for J2kResidentLosslessHtCodeBlocks { + const TIER2_PREFIX: &'static str = "HTJ2K "; + const FAMILY_NAME: &'static str = "HTJ2K"; + const BLOCK_CODING_MODE: u32 = 1; + const CODESTREAM_STATUS_STAGE: &'static str = "HTJ2K codestream assembly"; + const CODESTREAM_LENGTH_ERROR: &'static str = + "HTJ2K Metal codestream output length exceeds usize"; + const CODESTREAM_CAPACITY_ERROR: &'static str = + "HTJ2K Metal codestream output length exceeds buffer"; + + fn batch_job_count(&self) -> usize { + self.batch_jobs.len() + } + fn code_block_count(&self) -> usize { + self.code_blocks.len() + } + fn output_capacity_total(&self) -> usize { + self.output_capacity_total } - - if surfaces.len() != count { - return Err(Error::MetalKernel { - message: format!( - "J2K MetalDirect repeated grayscale plan produced {} surfaces for count {}", - surfaces.len(), - count - ), - }); + fn output_buffer(&self) -> &Buffer { + &self.output_buffer + } + fn status_buffer(&self) -> &Buffer { + &self.status_buffer + } + fn job_buffer(&self) -> &Buffer { + &self.job_buffer + } + fn prepare_command_buffer(&self) -> &CommandBuffer { + &self.prepare_command_buffer } + fn tier1_command_buffer(&self) -> &CommandBuffer { + &self.tier1_command_buffer + } + fn packet_block_prepare_pipeline(runtime: &MetalRuntime) -> &ComputePipelineState { + &runtime.packet_block_prepare_resident_ht + } +} - Ok(surfaces) +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +enum J2kResidentTier1StatusKind { + Classic, + HighThroughput, } #[cfg(target_os = "macos")] -fn copy_plane_samples(buffer: &Buffer, samples: &[f32], image_width: usize, roi: Rect) { - let row_width = roi.w as usize; - let dst = unsafe { - core::slice::from_raw_parts_mut(buffer.contents().cast::(), row_width * roi.h as usize) - }; +struct J2kResidentTier1StatusReadback { + buffer: Buffer, + kind: J2kResidentTier1StatusKind, + classic_style_flags: u32, + classic_jobs: Option>, + count: usize, +} - for row in 0..roi.h as usize { - let src_start = (roi.y as usize + row) * image_width + roi.x as usize; - let src_end = src_start + row_width; - let dst_start = row * row_width; - dst[dst_start..dst_start + row_width].copy_from_slice(&samples[src_start..src_end]); - } +#[cfg(target_os = "macos")] +struct J2kResidentClassicTier1DensityReadback { + buffer: Buffer, + count: usize, } #[cfg(target_os = "macos")] -fn take_f32_scratch_buffer(runtime: &MetalRuntime, len: usize) -> DirectScratchBuffer { - let bytes = len.max(1).saturating_mul(size_of::()); - DirectScratchBuffer { - bytes, - buffer: runtime.take_private_buffer(bytes), - } +struct J2kResidentClassicTier1SymbolPlanReadback { + buffer: Buffer, + count: usize, } #[cfg(target_os = "macos")] -fn recycle_scratch_buffers(runtime: &MetalRuntime, scratch_buffers: Vec) { - for scratch in scratch_buffers { - runtime.recycle_private_buffer(scratch.bytes, scratch.buffer); - } +struct J2kResidentClassicTier1PassPlanReadback { + buffer: Buffer, + count: usize, } #[cfg(target_os = "macos")] -fn take_recyclable_private_buffer( - runtime: &MetalRuntime, - bytes: usize, - recyclable_private_buffers: &mut Vec<(usize, Buffer)>, -) -> Buffer { - let bytes = bytes.max(1); - let buffer = runtime.take_private_buffer(bytes); - recyclable_private_buffers.push((bytes, buffer.clone())); - buffer +struct J2kResidentClassicTier1TokenEmitReadback { + counter_buffer: Buffer, + token_buffer: Option, + segment_buffer: Option, + token_stride_bytes: usize, + token_segment_stride: usize, + count: usize, } #[cfg(target_os = "macos")] -fn recycle_private_buffers( - runtime: &MetalRuntime, +struct J2kResidentClassicTier1GpuTokenBuffers { + counter_buffer: Buffer, + token_buffer: Buffer, + segment_buffer: Buffer, + job_count: u32, + token_stride_bytes: u32, + token_segment_stride: u32, +} + +#[cfg(target_os = "macos")] +struct J2kResidentClassicTier1SplitTokenBuffers { + counter_buffer: Buffer, + mq_token_buffer: Buffer, + raw_token_buffer: Buffer, + segment_buffer: Buffer, + job_count: u32, + mq_token_stride_bytes: u32, + raw_token_stride_bytes: u32, + token_segment_stride: u32, +} + +#[cfg(target_os = "macos")] +pub(crate) struct J2kPendingResidentLosslessCodestreamBatch { + runtime: Arc, + buffer: Buffer, + byte_offsets: Vec, + capacities: Vec, + status_buffer: Buffer, + packet_status_buffer: Buffer, + tier1_status_readback: Option, + classic_tier1_density_readback: Option, + classic_tier1_symbol_plan_readback: Option, + classic_tier1_pass_plan_readback: Option, + classic_tier1_token_emit_readback: Option, + classic_tier1_split_token_emit_readback: Option, + classic_gpu_token_pack_used: bool, + command_buffer: CommandBuffer, + retained_command_buffers: Vec, + _retained_buffers: Vec, recyclable_private_buffers: Vec<(usize, Buffer)>, -) { - for (bytes, buffer) in recyclable_private_buffers { - runtime.recycle_private_buffer(bytes, buffer); - } + recyclable_shared_buffers: Vec<(usize, Buffer)>, + gpu_stage_command_buffers: Vec, + stage_stats: J2kResidentEncodeStageStats, + codestream_payload_copy_dispatched: bool, + status_stage: &'static str, + length_error: &'static str, + capacity_error: &'static str, } #[cfg(target_os = "macos")] -fn validate_direct_status(status_check: DirectStatusCheck) -> Result<(), Error> { - match status_check { - DirectStatusCheck::Classic { buffer, len } => { - let statuses = unsafe { - core::slice::from_raw_parts(buffer.contents().cast::(), len) - }; - if let Some(status) = statuses - .iter() - .copied() - .find(|status| status.code != J2K_CLASSIC_STATUS_OK) - { - return Err(decode_classic_status_error(status)); - } - } - DirectStatusCheck::Ht { buffer, len } => { - let statuses = unsafe { - core::slice::from_raw_parts(buffer.contents().cast::(), len) - }; - if let Some(status) = statuses - .iter() - .copied() - .find(|status| status.code != J2K_HT_STATUS_OK) - { - return Err(decode_ht_status_error(status)); - } - } - DirectStatusCheck::Idwt(buffer) => { - let status = unsafe { buffer.contents().cast::().read() }; - if status.code != J2K_IDWT_STATUS_OK { - return Err(decode_idwt_status_error(status)); - } - } - DirectStatusCheck::Mct(buffer) => { - let status = unsafe { buffer.contents().cast::().read() }; - if status.code != J2K_MCT_STATUS_OK { - return Err(decode_mct_status_error(status)); - } - } +#[derive(Clone, Copy)] +struct J2kBatchedPacketPayloadCopyDispatch<'a> { + payload_buffer: &'a Buffer, + packet_output_buffer: &'a Buffer, + packet_job_buffer: &'a Buffer, + packet_status_buffer: &'a Buffer, + packet_payload_copy_job_buffer: &'a Buffer, + tile_count: u64, + max_payload_copy_jobs_per_tile: u64, + label: &'a str, + signpost_name: HybridSignpostName, +} + +#[cfg(target_os = "macos")] +pub(crate) fn wait_resident_lossless_codestream( + pending: J2kPendingResidentLosslessCodestream, +) -> Result { + wait_resident_codestream_command_buffer(&pending.command_buffer); + let gpu_duration = completed_command_buffers_gpu_duration( + &pending.retained_command_buffers, + &pending.command_buffer, + ); + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let status = unsafe { + pending + .status_buffer + .contents() + .cast::() + .read() + }; + if status.code != J2K_ENCODE_STATUS_OK { + return Err(encode_status_error( + pending.status_stage, + status.code, + status.detail, + )); } + let data_len = usize::try_from(status.data_len).map_err(|_| Error::MetalKernel { + message: pending.length_error.to_string(), + })?; + if data_len > pending.capacity { + return Err(Error::MetalKernel { + message: pending.capacity_error.to_string(), + }); + } + Ok(J2kResidentLosslessCodestream { + buffer: pending.buffer, + byte_offset: 0, + byte_len: data_len, + capacity: pending.capacity, + gpu_duration, + }) +} - Ok(()) +#[cfg(target_os = "macos")] +pub(crate) fn wait_resident_lossless_codestream_batch( + pending: J2kPendingResidentLosslessCodestreamBatch, +) -> Result { + wait_resident_codestream_command_buffer(&pending.command_buffer); + finish_completed_resident_lossless_codestream_batch(pending) } #[cfg(target_os = "macos")] -fn encode_gray_plane_to_surface_in_encoder( +pub(crate) fn wait_resident_lossless_codestream_batches( + pending_batches: Vec, +) -> Result, Error> { + if let Some(last) = pending_batches.last() { + // These command buffers are submitted on the same Metal queue before + // harvest, so completing the final one implies earlier chunks are done. + wait_resident_codestream_command_buffer(&last.command_buffer); + } + pending_batches + .into_iter() + .map(finish_completed_resident_lossless_codestream_batch) + .collect() +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn schedule_resident_tier1_status_readback( runtime: &MetalRuntime, - encoder: &ComputeCommandEncoderRef, - plane: &Buffer, - dims: (u32, u32), - bit_depth: u8, - fmt: PixelFormat, -) -> Result { - encode_gray_plane_to_surface_in_encoder_with_offset( - runtime, encoder, plane, 0, dims, bit_depth, fmt, - ) + command_buffer: &CommandBufferRef, + status_buffer: &Buffer, + kind: J2kResidentTier1StatusKind, + classic_style_flags: u32, + classic_jobs: Option<&[J2kClassicEncodeBatchJob]>, + count: usize, + status_size: usize, + profile_stages: bool, +) -> Result, Error> { + if !profile_stages || count == 0 { + return Ok(None); + } + let byte_len = count + .checked_mul(status_size) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal resident Tier-1 status readback size overflow".to_string(), + })?; + let readback = runtime.device.new_buffer( + byte_len.max(1) as u64, + MTLResourceOptions::StorageModeShared, + ); + let blit = command_buffer.new_blit_command_encoder(); + blit.copy_from_buffer(status_buffer, 0, &readback, 0, byte_len as u64); + blit.end_encoding(); + Ok(Some(J2kResidentTier1StatusReadback { + buffer: readback, + kind, + classic_style_flags, + classic_jobs: classic_jobs.map(<[J2kClassicEncodeBatchJob]>::to_vec), + count, + })) } #[cfg(target_os = "macos")] -fn encode_gray_plane_to_surface_in_command_buffer_with_offset( +fn dispatch_classic_tier1_density_profile( runtime: &MetalRuntime, command_buffer: &CommandBufferRef, - plane: &Buffer, - plane_offset_bytes: usize, - dims: (u32, u32), - bit_depth: u8, - fmt: PixelFormat, -) -> Result { + coefficient_buffer: &Buffer, + tier1_job_buffer: &Buffer, + tier1_jobs: &[J2kClassicEncodeBatchJob], +) -> Result, Error> { + if !metal_profile_classic_tier1_density_enabled() || tier1_jobs.is_empty() { + return Ok(None); + } + if classic_encode_code_blocks_pipeline_kind(tier1_jobs) + != J2kClassicEncodePipelineKind::BypassU16_32 + { + return Err(Error::MetalKernel { + message: "J2K Metal classic Tier-1 density profiling currently supports only bypass_u16_32 resident jobs".to_string(), + }); + } + + let counter_buffer = runtime.device.new_buffer( + (tier1_jobs.len().max(1) * size_of::()) as u64, + MTLResourceOptions::StorageModeShared, + ); + let job_count = u32::try_from(tier1_jobs.len()).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 density job count exceeds u32".to_string(), + })?; let encoder = command_buffer.new_compute_command_encoder(); - let result = encode_gray_plane_to_surface_in_encoder_with_offset( - runtime, - encoder, - plane, - plane_offset_bytes, - dims, - bit_depth, - fmt, + label_compute_encoder(encoder, "J2K classic Tier-1 density profile"); + encoder.set_compute_pipeline_state(&runtime.classic_tier1_density_bypass_u16_32); + encoder.set_buffer(0, Some(coefficient_buffer), 0); + encoder.set_buffer(1, Some(tier1_job_buffer), 0); + encoder.set_buffer(2, Some(&counter_buffer), 0); + encoder.set_bytes(3, size_of::() as u64, (&raw const job_count).cast()); + encoder.dispatch_threads( + MTLSize { + width: u64::from(job_count), + height: 1, + depth: 1, + }, + MTLSize { + width: runtime + .classic_tier1_density_bypass_u16_32 + .thread_execution_width() + .max(1), + height: 1, + depth: 1, + }, ); encoder.end_encoding(); - result + Ok(Some(J2kResidentClassicTier1DensityReadback { + buffer: counter_buffer, + count: tier1_jobs.len(), + })) } #[cfg(target_os = "macos")] -fn encode_gray_plane_to_surface_in_encoder_with_offset( +fn dispatch_classic_tier1_raw_pack_profile( runtime: &MetalRuntime, - encoder: &ComputeCommandEncoderRef, - plane: &Buffer, - plane_offset_bytes: usize, - dims: (u32, u32), - bit_depth: u8, - fmt: PixelFormat, -) -> Result { - let pitch_bytes = dims.0 as usize * fmt.bytes_per_pixel(); - let out_buffer = runtime.device.new_buffer( - (pitch_bytes * dims.1 as usize) as u64, - MTLResourceOptions::StorageModeShared, + command_buffer: &CommandBufferRef, + coefficient_buffer: &Buffer, + tier1_job_buffer: &Buffer, + tier1_jobs: &[J2kClassicEncodeBatchJob], + tier1_output_capacity_total: usize, +) -> Result, Error> { + if !metal_profile_classic_tier1_raw_pack_enabled() || tier1_jobs.is_empty() { + return Ok(None); + } + if classic_encode_code_blocks_pipeline_kind(tier1_jobs) + != J2kClassicEncodePipelineKind::BypassU16_32 + { + return Err(Error::MetalKernel { + message: "J2K Metal classic Tier-1 raw-pack profiling currently supports only bypass_u16_32 resident jobs".to_string(), + }); + } + + let raw_output_buffer = runtime.device.new_buffer( + tier1_output_capacity_total.max(1) as u64, + MTLResourceOptions::StorageModePrivate, ); - let (output_channels, opaque_alpha, pipeline) = - output_shape_for(&NativeColorSpace::Gray, false, 1, fmt, runtime)?; - let mut bit_depths = [0u32; 4]; - bit_depths[0] = u32::from(bit_depth); - let (max_values, u8_scales, u16_scales) = j2k_pack_scale_arrays(bit_depths); - let params = J2kPackParams { - width: dims.0, - height: dims.1, - out_stride: u32::try_from(pitch_bytes).expect("J2K Metal output stride fits in u32"), - output_channels, - opaque_alpha, - max_values, - u8_scales, - u16_scales, - }; + let job_count = u32::try_from(tier1_jobs.len()).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 raw-pack job count exceeds u32".to_string(), + })?; + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K classic Tier-1 raw-pack profile"); + encoder.set_compute_pipeline_state(&runtime.classic_tier1_raw_pack_bypass_u16_32); + encoder.set_buffer(0, Some(coefficient_buffer), 0); + encoder.set_buffer(1, Some(tier1_job_buffer), 0); + encoder.set_buffer(2, Some(&raw_output_buffer), 0); + encoder.set_bytes(3, size_of::() as u64, (&raw const job_count).cast()); + encoder.dispatch_threads( + MTLSize { + width: u64::from(job_count), + height: 1, + depth: 1, + }, + MTLSize { + width: runtime + .classic_tier1_raw_pack_bypass_u16_32 + .thread_execution_width() + .max(1), + height: 1, + depth: 1, + }, + ); + encoder.end_encoding(); + Ok(Some(raw_output_buffer)) +} - encoder.set_compute_pipeline_state(pipeline); - encoder.set_buffer(0, Some(plane), plane_offset_bytes as u64); - encoder.set_buffer(1, None, 0); - encoder.set_buffer(2, None, 0); - encoder.set_buffer(3, None, 0); - encoder.set_buffer(4, Some(&out_buffer), 0); - encoder.set_bytes( - 5, - size_of::() as u64, - (&raw const params).cast(), +#[cfg(target_os = "macos")] +fn dispatch_classic_tier1_arithmetic_pack_profile( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + coefficient_buffer: &Buffer, + tier1_job_buffer: &Buffer, + tier1_jobs: &[J2kClassicEncodeBatchJob], + tier1_output_capacity_total: usize, +) -> Result, Error> { + if !metal_profile_classic_tier1_arithmetic_pack_enabled() || tier1_jobs.is_empty() { + return Ok(None); + } + if classic_encode_code_blocks_pipeline_kind(tier1_jobs) + != J2kClassicEncodePipelineKind::BypassU16_32 + { + return Err(Error::MetalKernel { + message: "J2K Metal classic Tier-1 arithmetic-pack profiling currently supports only bypass_u16_32 resident jobs".to_string(), + }); + } + + let arithmetic_output_buffer = runtime.device.new_buffer( + tier1_output_capacity_total.max(1) as u64, + MTLResourceOptions::StorageModePrivate, ); - let width = pipeline.thread_execution_width().max(1); - let max_threads = pipeline.max_total_threads_per_threadgroup().max(width); - let height = (max_threads / width).max(1); + let job_count = u32::try_from(tier1_jobs.len()).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 arithmetic-pack job count exceeds u32".to_string(), + })?; + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K classic Tier-1 arithmetic-pack profile"); + encoder.set_compute_pipeline_state(&runtime.classic_tier1_arithmetic_pack_bypass_u16_32); + encoder.set_buffer(0, Some(coefficient_buffer), 0); + encoder.set_buffer(1, Some(tier1_job_buffer), 0); + encoder.set_buffer(2, Some(&arithmetic_output_buffer), 0); + encoder.set_bytes(3, size_of::() as u64, (&raw const job_count).cast()); encoder.dispatch_threads( MTLSize { - width: u64::from(dims.0), - height: u64::from(dims.1), + width: u64::from(job_count), + height: 1, depth: 1, }, MTLSize { - width, - height, + width: runtime + .classic_tier1_arithmetic_pack_bypass_u16_32 + .thread_execution_width() + .max(1), + height: 1, depth: 1, }, ); - - Ok(Surface::from_metal_buffer(out_buffer, dims, fmt)) + encoder.end_encoding(); + Ok(Some(arithmetic_output_buffer)) } #[cfg(target_os = "macos")] -fn encode_repeated_gray_plane_to_surfaces_in_command_buffer( +fn dispatch_classic_tier1_symbol_plan_profile( runtime: &MetalRuntime, command_buffer: &CommandBufferRef, - plane: &Buffer, - dims: (u32, u32), - bit_depth: u8, - fmt: PixelFormat, - count: usize, -) -> Result, Error> { - let count_u32 = u32::try_from(count).map_err(|_| Error::MetalKernel { - message: "J2K Metal repeated grayscale surface count exceeds u32".to_string(), - })?; - let pitch_bytes = dims.0 as usize * fmt.bytes_per_pixel(); - let surface_bytes = - pitch_bytes - .checked_mul(dims.1 as usize) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal repeated grayscale surface size overflow".to_string(), - })?; - let total_bytes = surface_bytes - .checked_mul(count) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal repeated grayscale output size overflow".to_string(), - })?; - let out_buffer = runtime - .device - .new_buffer(total_bytes as u64, MTLResourceOptions::StorageModeShared); - let scale = j2k_scalar_pack_params(u32::from(bit_depth)); - let params = J2kRepeatedGrayPackParams { - width: dims.0, - height: dims.1, - out_stride: u32::try_from(pitch_bytes).expect("J2K Metal output stride fits in u32"), - batch_count: count_u32, - max_value: scale.max_value, - u8_scale: scale.u8_scale, - u16_scale: scale.u16_scale, - }; - let pipeline = match fmt { - PixelFormat::Gray8 => &runtime.pack_u8_repeated_gray, - PixelFormat::Gray16 => &runtime.pack_u16_repeated_gray, - _ => { - return Err(Error::MetalKernel { - message: format!("J2K Metal repeated grayscale pack does not support {fmt:?}"), - }) - } - }; + coefficient_buffer: &Buffer, + tier1_job_buffer: &Buffer, + tier1_jobs: &[J2kClassicEncodeBatchJob], +) -> Result, Error> { + if !metal_profile_classic_tier1_symbol_plan_enabled() || tier1_jobs.is_empty() { + return Ok(None); + } + if classic_encode_code_blocks_pipeline_kind(tier1_jobs) + != J2kClassicEncodePipelineKind::BypassU16_32 + { + return Err(Error::MetalKernel { + message: "J2K Metal classic Tier-1 symbol-plan profiling currently supports only bypass_u16_32 resident jobs".to_string(), + }); + } - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(pipeline); - encoder.set_buffer(0, Some(plane), 0); - encoder.set_buffer(1, Some(&out_buffer), 0); - encoder.set_bytes( - 2, - size_of::() as u64, - (&raw const params).cast(), + let counter_buffer = runtime.device.new_buffer( + (tier1_jobs.len().max(1) * size_of::()) as u64, + MTLResourceOptions::StorageModeShared, ); - let width = pipeline.thread_execution_width().max(1); - let max_threads = pipeline.max_total_threads_per_threadgroup().max(width); - let height = (max_threads / width).max(1); + let job_count = u32::try_from(tier1_jobs.len()).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 symbol-plan job count exceeds u32".to_string(), + })?; + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K classic Tier-1 symbol plan"); + encoder.set_compute_pipeline_state(&runtime.classic_tier1_symbol_plan_bypass_u16_32); + encoder.set_buffer(0, Some(coefficient_buffer), 0); + encoder.set_buffer(1, Some(tier1_job_buffer), 0); + encoder.set_buffer(2, Some(&counter_buffer), 0); + encoder.set_bytes(3, size_of::() as u64, (&raw const job_count).cast()); encoder.dispatch_threads( MTLSize { - width: u64::from(dims.0), - height: u64::from(dims.1), - depth: u64::from(count_u32), + width: u64::from(job_count), + height: 1, + depth: 1, }, MTLSize { - width, - height, + width: runtime + .classic_tier1_symbol_plan_bypass_u16_32 + .thread_execution_width() + .max(1), + height: 1, depth: 1, }, ); encoder.end_encoding(); - - let mut surfaces = Vec::with_capacity(count); - for instance_idx in 0..count { - surfaces.push(Surface::from_metal_buffer_with_offset( - out_buffer.clone(), - dims, - fmt, - instance_idx * surface_bytes, - )); - } - Ok(surfaces) -} - -#[cfg(target_os = "macos")] -fn owned_slice_buffer(device: &Device, data: &[T]) -> Buffer { - let size = size_of_val(data).max(1); - let buffer = device.new_buffer(size as u64, MTLResourceOptions::StorageModeShared); - if !data.is_empty() { - unsafe { - core::ptr::copy_nonoverlapping( - data.as_ptr().cast::(), - buffer.contents().cast::(), - size_of_val(data), - ); - } - } - buffer -} - -#[cfg(target_os = "macos")] -fn j2k_pack_kernel_name_for( - color_space: &NativeColorSpace, - has_alpha: bool, - plane_count: usize, - fmt: PixelFormat, -) -> Option<&'static str> { - match (color_space, has_alpha, plane_count, fmt) { - (NativeColorSpace::Gray, false, 1, PixelFormat::Gray8) => Some("j2k_pack_gray8"), - (NativeColorSpace::RGB, false, 3, PixelFormat::Rgb8) - | (NativeColorSpace::RGB, true, 4, PixelFormat::Rgb8) => Some("j2k_pack_rgb8"), - (NativeColorSpace::RGB, false, 3, PixelFormat::Rgba8) => Some("j2k_pack_rgb_opaque_rgba8"), - (NativeColorSpace::RGB, true, 4, PixelFormat::Rgba8) => Some("j2k_pack_rgba8"), - (NativeColorSpace::Gray, false, 1, PixelFormat::Gray16) => Some("j2k_pack_gray16"), - (NativeColorSpace::RGB, false, 3, PixelFormat::Rgb16) => Some("j2k_pack_rgb16"), - _ => None, - } + Ok(Some(J2kResidentClassicTier1SymbolPlanReadback { + buffer: counter_buffer, + count: tier1_jobs.len(), + })) } #[cfg(target_os = "macos")] -fn j2k_pack_pipeline_for<'a>( - runtime: &'a MetalRuntime, - kernel_name: &str, -) -> &'a ComputePipelineState { - match kernel_name { - "j2k_pack_gray8" => &runtime.pack_gray8, - "j2k_pack_rgb8" => &runtime.pack_rgb8, - "j2k_pack_rgb_opaque_rgba8" => &runtime.pack_rgb_opaque_rgba8, - "j2k_pack_rgba8" => &runtime.pack_rgba8, - "j2k_pack_gray16" => &runtime.pack_gray16, - "j2k_pack_rgb16" => &runtime.pack_rgb16, - _ => unreachable!("validated J2K pack kernel name"), +fn dispatch_classic_tier1_pass_plan_profile( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + coefficient_buffer: &Buffer, + tier1_job_buffer: &Buffer, + tier1_jobs: &[J2kClassicEncodeBatchJob], +) -> Result, Error> { + if !metal_profile_classic_tier1_pass_plan_enabled() || tier1_jobs.is_empty() { + return Ok(None); } -} - -#[cfg(target_os = "macos")] -fn output_shape_for<'a>( - color_space: &NativeColorSpace, - has_alpha: bool, - plane_count: usize, - fmt: PixelFormat, - runtime: &'a MetalRuntime, -) -> Result<(u32, u32, &'a ComputePipelineState), Error> { - let Some(kernel_name) = j2k_pack_kernel_name_for(color_space, has_alpha, plane_count, fmt) - else { + if classic_encode_code_blocks_pipeline_kind(tier1_jobs) + != J2kClassicEncodePipelineKind::BypassU16_32 + { return Err(Error::MetalKernel { - message: format!( - "unsupported J2K Metal mapping for {color_space:?}, alpha={has_alpha}, planes={plane_count}, fmt={fmt:?}" - ), + message: "J2K Metal classic Tier-1 pass-plan profiling currently supports only bypass_u16_32 resident jobs".to_string(), }); - }; - let (output_channels, opaque_alpha) = match (color_space, has_alpha, plane_count, fmt) { - (NativeColorSpace::Gray, false, 1, PixelFormat::Gray8 | PixelFormat::Gray16) => (1, 0), - (NativeColorSpace::RGB, false, 3, PixelFormat::Rgb8 | PixelFormat::Rgb16) - | (NativeColorSpace::RGB, true, 4, PixelFormat::Rgb8) => (3, 0), - (NativeColorSpace::RGB, false, 3, PixelFormat::Rgba8) => (4, 1), - (NativeColorSpace::RGB, true, 4, PixelFormat::Rgba8) => (4, 0), - _ => unreachable!("validated J2K pack shape"), - }; - Ok(( - output_channels, - opaque_alpha, - j2k_pack_pipeline_for(runtime, kernel_name), - )) -} - -#[cfg(target_os = "macos")] -fn required_classic_output_len(job: J2kCodeBlockDecodeJob<'_>) -> Result { - if job.height == 0 { - return Ok(0); } - job.output_stride - .checked_mul(job.height as usize - 1) - .and_then(|prefix| prefix.checked_add(job.width as usize)) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K Metal output size overflow".to_string(), - }) + let counter_buffer = runtime.device.new_buffer( + (tier1_jobs.len().max(1) * size_of::()) as u64, + MTLResourceOptions::StorageModeShared, + ); + let job_count = u32::try_from(tier1_jobs.len()).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 pass-plan job count exceeds u32".to_string(), + })?; + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K classic Tier-1 pass plan"); + encoder.set_compute_pipeline_state(&runtime.classic_tier1_pass_plan_bypass_u16_32); + encoder.set_buffer(0, Some(coefficient_buffer), 0); + encoder.set_buffer(1, Some(tier1_job_buffer), 0); + encoder.set_buffer(2, Some(&counter_buffer), 0); + encoder.set_bytes(3, size_of::() as u64, (&raw const job_count).cast()); + encoder.dispatch_threads( + MTLSize { + width: u64::from(job_count), + height: 1, + depth: 1, + }, + MTLSize { + width: runtime + .classic_tier1_pass_plan_bypass_u16_32 + .thread_execution_width() + .max(1), + height: 1, + depth: 1, + }, + ); + encoder.end_encoding(); + Ok(Some(J2kResidentClassicTier1PassPlanReadback { + buffer: counter_buffer, + count: tier1_jobs.len(), + })) } #[cfg(target_os = "macos")] -fn classic_style_flags(style: signinum_j2k_native::J2kCodeBlockStyle) -> u32 { - let mut flags = 0u32; - if style.reset_context_probabilities { - flags |= J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES; - } - if style.termination_on_each_pass { - flags |= J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS; - } - if style.vertically_causal_context { - flags |= J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT; - } - if style.segmentation_symbols { - flags |= J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS; +fn dispatch_classic_tier1_token_emit_profile( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + coefficient_buffer: &Buffer, + tier1_job_buffer: &Buffer, + tier1_jobs: &[J2kClassicEncodeBatchJob], +) -> Result, Error> { + if !metal_profile_classic_tier1_token_emit_enabled() || tier1_jobs.is_empty() { + return Ok(None); } - if style.selective_arithmetic_coding_bypass { - flags |= J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS; + if classic_encode_code_blocks_pipeline_kind(tier1_jobs) + != J2kClassicEncodePipelineKind::BypassU16_32 + { + return Err(Error::MetalKernel { + message: "J2K Metal classic Tier-1 token-emitter profiling currently supports only bypass_u16_32 resident jobs".to_string(), + }); } - flags -} -#[cfg(target_os = "macos")] -fn decode_classic_status_error(status: J2kClassicStatus) -> Error { - let kind = match status.code { - J2K_CLASSIC_STATUS_FAIL => "decode failure", - J2K_CLASSIC_STATUS_UNSUPPORTED => "unsupported classic kernel input", - _ => "unexpected classic kernel status", - }; - Error::MetalKernel { - message: format!("classic J2K Metal kernel {kind} (detail={})", status.detail), - } -} + let counter_buffer = runtime.device.new_buffer( + (tier1_jobs.len().max(1) * size_of::()) as u64, + MTLResourceOptions::StorageModeShared, + ); + let token_buffer_len = tier1_jobs + .len() + .max(1) + .checked_mul(CLASSIC_TIER1_TOKEN_ARENA_BYTES) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token buffer size overflow".to_string(), + })?; + let token_buffer = runtime.device.new_buffer( + token_buffer_len as u64, + MTLResourceOptions::StorageModeShared, + ); + let segment_buffer_len = tier1_jobs + .len() + .max(1) + .checked_mul(CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY) + .and_then(|count| count.checked_mul(size_of::())) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token segment buffer size overflow".to_string(), + })?; + let segment_buffer = runtime.device.new_buffer( + segment_buffer_len as u64, + MTLResourceOptions::StorageModeShared, + ); + let job_count = u32::try_from(tier1_jobs.len()).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token-emitter job count exceeds u32".to_string(), + })?; + let token_stride_bytes = + u32::try_from(CLASSIC_TIER1_TOKEN_ARENA_BYTES).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token arena stride exceeds u32".to_string(), + })?; + let token_segment_stride = + u32::try_from(CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token segment stride exceeds u32".to_string(), + })?; -#[cfg(target_os = "macos")] -fn decode_idwt_status_error(status: J2kIdwtStatus) -> Error { - let kind = match status.code { - J2K_IDWT_STATUS_FAIL => "decode failure", - _ => "unexpected IDWT kernel status", - }; - Error::MetalKernel { - message: format!("J2K Metal IDWT kernel {kind} (detail={})", status.detail), - } + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K classic Tier-1 token emit"); + encoder.set_compute_pipeline_state(&runtime.classic_tier1_token_emit_bypass_u16_32); + encoder.set_buffer(0, Some(coefficient_buffer), 0); + encoder.set_buffer(1, Some(tier1_job_buffer), 0); + encoder.set_buffer(2, Some(&counter_buffer), 0); + encoder.set_buffer(3, Some(&token_buffer), 0); + encoder.set_buffer(4, Some(&segment_buffer), 0); + encoder.set_bytes( + 5, + size_of::() as u64, + (&raw const token_stride_bytes).cast(), + ); + encoder.set_bytes( + 6, + size_of::() as u64, + (&raw const token_segment_stride).cast(), + ); + encoder.set_bytes(7, size_of::() as u64, (&raw const job_count).cast()); + encoder.dispatch_threads( + MTLSize { + width: u64::from(job_count), + height: 1, + depth: 1, + }, + MTLSize { + width: runtime + .classic_tier1_token_emit_bypass_u16_32 + .thread_execution_width() + .max(1), + height: 1, + depth: 1, + }, + ); + encoder.end_encoding(); + Ok(Some(J2kResidentClassicTier1TokenEmitReadback { + counter_buffer, + token_buffer: Some(token_buffer), + segment_buffer: Some(segment_buffer), + token_stride_bytes: CLASSIC_TIER1_TOKEN_ARENA_BYTES, + token_segment_stride: CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY, + count: tier1_jobs.len(), + })) } #[cfg(target_os = "macos")] -fn decode_mct_status_error(status: J2kMctStatus) -> Error { - let kind = match status.code { - J2K_MCT_STATUS_FAIL => "decode failure", - _ => "unexpected inverse MCT kernel status", - }; - Error::MetalKernel { - message: format!( - "J2K Metal inverse MCT kernel {kind} (detail={})", - status.detail - ), +fn dispatch_classic_tier1_split_token_emit_for_cpu_pack( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + coefficient_buffer: &Buffer, + tier1_job_buffer: &Buffer, + tier1_jobs: &[J2kClassicEncodeBatchJob], +) -> Result { + if !classic_tier1_gpu_token_pack_supported(tier1_jobs) { + return Err(Error::MetalKernel { + message: "J2K Metal classic split-token route currently supports only bypass_u16_32 resident jobs".to_string(), + }); } -} -fn wrap_f32_output_buffer(device: &Device, output: &mut [f32]) -> Buffer { - if output.is_empty() { - device.new_buffer( - size_of::() as u64, - MTLResourceOptions::StorageModeShared, - ) - } else { - device.new_buffer_with_bytes_no_copy( - output.as_mut_ptr().cast(), - size_of_val(output) as u64, - MTLResourceOptions::StorageModeShared, - None, - ) - } -} + let counter_buffer = runtime.device.new_buffer( + (tier1_jobs.len().max(1) * size_of::()) as u64, + MTLResourceOptions::StorageModeShared, + ); + let mq_token_buffer_len = tier1_jobs + .len() + .max(1) + .checked_mul(CLASSIC_TIER1_TOKEN_ARENA_BYTES) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal classic split-token MQ buffer size overflow".to_string(), + })?; + let mq_token_buffer = runtime.device.new_buffer( + mq_token_buffer_len as u64, + MTLResourceOptions::StorageModeShared, + ); + let raw_token_buffer_len = tier1_jobs + .len() + .max(1) + .checked_mul(CLASSIC_TIER1_TOKEN_ARENA_BYTES) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal classic split-token raw buffer size overflow".to_string(), + })?; + let raw_token_buffer = runtime.device.new_buffer( + raw_token_buffer_len as u64, + MTLResourceOptions::StorageModeShared, + ); + let segment_buffer_len = tier1_jobs + .len() + .max(1) + .checked_mul(CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY) + .and_then(|count| count.checked_mul(size_of::())) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal classic split-token segment buffer size overflow".to_string(), + })?; + let segment_buffer = runtime.device.new_buffer( + segment_buffer_len as u64, + MTLResourceOptions::StorageModeShared, + ); + let job_count = u32::try_from(tier1_jobs.len()).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic split-token job count exceeds u32".to_string(), + })?; + let mq_token_stride_bytes = + u32::try_from(CLASSIC_TIER1_TOKEN_ARENA_BYTES).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic split-token MQ arena stride exceeds u32".to_string(), + })?; + let raw_token_stride_bytes = + u32::try_from(CLASSIC_TIER1_TOKEN_ARENA_BYTES).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic split-token raw arena stride exceeds u32".to_string(), + })?; + let token_segment_stride = + u32::try_from(CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic split-token segment stride exceeds u32".to_string(), + })?; -#[cfg(target_os = "macos")] -fn borrow_slice_buffer(device: &Device, data: &[T]) -> Buffer { - if data.is_empty() { - device.new_buffer(1, MTLResourceOptions::StorageModeShared) - } else { - device.new_buffer_with_bytes_no_copy( - data.as_ptr().cast(), - size_of_val(data) as u64, - MTLResourceOptions::StorageModeShared, - None, - ) - } + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K classic Tier-1 split token emit"); + encoder.set_compute_pipeline_state(&runtime.classic_tier1_split_token_emit_bypass_u16_32); + encoder.set_buffer(0, Some(coefficient_buffer), 0); + encoder.set_buffer(1, Some(tier1_job_buffer), 0); + encoder.set_buffer(2, Some(&counter_buffer), 0); + encoder.set_buffer(3, Some(&mq_token_buffer), 0); + encoder.set_buffer(4, Some(&raw_token_buffer), 0); + encoder.set_buffer(5, Some(&segment_buffer), 0); + encoder.set_bytes( + 6, + size_of::() as u64, + (&raw const mq_token_stride_bytes).cast(), + ); + encoder.set_bytes( + 7, + size_of::() as u64, + (&raw const raw_token_stride_bytes).cast(), + ); + encoder.set_bytes( + 8, + size_of::() as u64, + (&raw const token_segment_stride).cast(), + ); + encoder.set_bytes(9, size_of::() as u64, (&raw const job_count).cast()); + encoder.dispatch_threads( + MTLSize { + width: u64::from(job_count), + height: 1, + depth: 1, + }, + MTLSize { + width: runtime + .classic_tier1_split_token_emit_bypass_u16_32 + .thread_execution_width() + .max(1), + height: 1, + depth: 1, + }, + ); + encoder.end_encoding(); + + Ok(J2kResidentClassicTier1SplitTokenBuffers { + counter_buffer, + mq_token_buffer, + raw_token_buffer, + segment_buffer, + job_count, + mq_token_stride_bytes, + raw_token_stride_bytes, + token_segment_stride, + }) } #[cfg(target_os = "macos")] -fn borrow_mut_slice_buffer(device: &Device, data: &mut [T]) -> Buffer { - if data.is_empty() { - device.new_buffer(1, MTLResourceOptions::StorageModeShared) - } else { - device.new_buffer_with_bytes_no_copy( - data.as_mut_ptr().cast(), - size_of_val(data) as u64, - MTLResourceOptions::StorageModeShared, - None, - ) +fn dispatch_classic_tier1_split_token_emit_profile( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + coefficient_buffer: &Buffer, + tier1_job_buffer: &Buffer, + tier1_jobs: &[J2kClassicEncodeBatchJob], +) -> Result, Error> { + if !metal_profile_classic_tier1_split_token_emit_enabled() || tier1_jobs.is_empty() { + return Ok(None); } + dispatch_classic_tier1_split_token_emit_for_cpu_pack( + runtime, + command_buffer, + coefficient_buffer, + tier1_job_buffer, + tier1_jobs, + ) + .map(Some) } #[cfg(target_os = "macos")] -fn copied_slice_buffer(device: &Device, data: &[T]) -> Buffer { - if data.is_empty() { - device.new_buffer(1, MTLResourceOptions::StorageModeShared) - } else { - device.new_buffer_with_data( - data.as_ptr().cast(), - size_of_val(data) as u64, - MTLResourceOptions::StorageModeShared, - ) +fn dispatch_classic_tier1_split_token_emit_for_gpu_pack( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + coefficient_buffer: &Buffer, + tier1_job_buffer: &Buffer, + tier1_jobs: &[J2kClassicEncodeBatchJob], + recyclable_private_buffers: &mut Vec<(usize, Buffer)>, + use_mq_byte_emit: bool, +) -> Result { + if !classic_tier1_gpu_token_pack_supported(tier1_jobs) { + return Err(Error::MetalKernel { + message: "J2K Metal classic split GPU token-pack route currently supports only bypass_u16_32 resident jobs".to_string(), + }); + } + #[cfg(test)] + if use_mq_byte_emit { + test_counters::record_classic_split_mq_byte_gpu_token_pack_dispatch(); } -} -#[cfg(target_os = "macos")] -fn classic_coefficients_scratch_bytes(job_count: usize) -> Result { - job_count + let counter_buffer = take_recyclable_private_buffer( + runtime, + tier1_jobs + .len() + .max(1) + .checked_mul(size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal classic split GPU token counter buffer size overflow" + .to_string(), + })?, + recyclable_private_buffers, + )?; + let mq_token_arena_bytes = if use_mq_byte_emit { + CLASSIC_TIER1_MQ_BYTE_TOKEN_ARENA_BYTES + } else { + CLASSIC_TIER1_TOKEN_ARENA_BYTES + }; + let mq_token_buffer_len = tier1_jobs + .len() .max(1) - .checked_mul(J2K_CLASSIC_MAX_COEFF_COUNT) - .and_then(|count| count.checked_mul(size_of::())) + .checked_mul(mq_token_arena_bytes) .ok_or_else(|| Error::MetalKernel { - message: "classic J2K coefficient scratch size overflow".to_string(), - }) -} - -#[cfg(target_os = "macos")] -fn take_classic_coefficients_scratch_buffer( - runtime: &MetalRuntime, - job_count: usize, -) -> Result { - let bytes = classic_coefficients_scratch_bytes(job_count)?; - Ok(DirectScratchBuffer { - bytes, - buffer: runtime.take_private_buffer(bytes), - }) -} - -#[cfg(target_os = "macos")] -fn classic_states_scratch_bytes(job_count: usize) -> Result { - job_count + message: "J2K Metal classic split GPU token MQ buffer size overflow".to_string(), + })?; + let mq_token_buffer = + take_recyclable_private_buffer(runtime, mq_token_buffer_len, recyclable_private_buffers)?; + let raw_token_buffer_len = tier1_jobs + .len() .max(1) - .checked_mul(J2K_CLASSIC_MAX_COEFF_COUNT) + .checked_mul(CLASSIC_TIER1_TOKEN_ARENA_BYTES) .ok_or_else(|| Error::MetalKernel { - message: "classic J2K MetalDirect states scratch overflow".to_string(), - }) -} + message: "J2K Metal classic split GPU token raw buffer size overflow".to_string(), + })?; + let raw_token_buffer = + take_recyclable_private_buffer(runtime, raw_token_buffer_len, recyclable_private_buffers)?; + let segment_buffer_len = tier1_jobs + .len() + .max(1) + .checked_mul(CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY) + .and_then(|count| count.checked_mul(size_of::())) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal classic split GPU token segment buffer size overflow".to_string(), + })?; + let segment_buffer = + take_recyclable_private_buffer(runtime, segment_buffer_len, recyclable_private_buffers)?; + let job_count = u32::try_from(tier1_jobs.len()).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic split GPU token job count exceeds u32".to_string(), + })?; + let mq_token_stride_bytes = + u32::try_from(mq_token_arena_bytes).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic split GPU token MQ arena stride exceeds u32".to_string(), + })?; + let raw_token_stride_bytes = + u32::try_from(CLASSIC_TIER1_TOKEN_ARENA_BYTES).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic split GPU token raw arena stride exceeds u32".to_string(), + })?; + let token_segment_stride = + u32::try_from(CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic split GPU token segment stride exceeds u32".to_string(), + })?; -#[cfg(target_os = "macos")] -fn take_classic_states_scratch_buffer( - runtime: &MetalRuntime, - job_count: usize, -) -> Result { - let bytes = classic_states_scratch_bytes(job_count)?; - Ok(DirectScratchBuffer { - bytes, - buffer: runtime.take_private_buffer(bytes), + let emit_pipeline = if use_mq_byte_emit { + &runtime.classic_tier1_split_mq_byte_token_emit_bypass_u16_32 + } else { + &runtime.classic_tier1_split_token_emit_bypass_u16_32 + }; + + let encoder = command_buffer.new_compute_command_encoder(); + if use_mq_byte_emit { + label_compute_encoder(encoder, "J2K classic Tier-1 split MQ-byte token emit"); + } else { + label_compute_encoder(encoder, "J2K classic Tier-1 split token emit"); + } + encoder.set_compute_pipeline_state(emit_pipeline); + encoder.set_buffer(0, Some(coefficient_buffer), 0); + encoder.set_buffer(1, Some(tier1_job_buffer), 0); + encoder.set_buffer(2, Some(&counter_buffer), 0); + encoder.set_buffer(3, Some(&mq_token_buffer), 0); + encoder.set_buffer(4, Some(&raw_token_buffer), 0); + encoder.set_buffer(5, Some(&segment_buffer), 0); + encoder.set_bytes( + 6, + size_of::() as u64, + (&raw const mq_token_stride_bytes).cast(), + ); + encoder.set_bytes( + 7, + size_of::() as u64, + (&raw const raw_token_stride_bytes).cast(), + ); + encoder.set_bytes( + 8, + size_of::() as u64, + (&raw const token_segment_stride).cast(), + ); + encoder.set_bytes(9, size_of::() as u64, (&raw const job_count).cast()); + dispatch_1d_pipeline(encoder, emit_pipeline, u64::from(job_count)); + encoder.end_encoding(); + + Ok(J2kResidentClassicTier1SplitTokenBuffers { + counter_buffer, + mq_token_buffer, + raw_token_buffer, + segment_buffer, + job_count, + mq_token_stride_bytes, + raw_token_stride_bytes, + token_segment_stride, }) } #[cfg(target_os = "macos")] -pub(crate) fn encode_forward_dwt53( - samples: &[f32], - width: u32, - height: u32, - num_levels: u8, -) -> Result { - if width == 0 || height == 0 { +fn dispatch_classic_tier1_token_emit_for_gpu_pack( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + coefficient_buffer: &Buffer, + tier1_job_buffer: &Buffer, + tier1_jobs: &[J2kClassicEncodeBatchJob], + recyclable_private_buffers: &mut Vec<(usize, Buffer)>, +) -> Result { + if !classic_tier1_gpu_token_pack_supported(tier1_jobs) { return Err(Error::MetalKernel { - message: "J2K Metal forward DWT dimensions must be non-zero".to_string(), + message: "J2K Metal classic GPU token-pack route currently supports only bypass_u16_32 resident jobs".to_string(), }); } - let expected_len = (width as usize) - .checked_mul(height as usize) + + let counter_buffer = take_recyclable_private_buffer( + runtime, + tier1_jobs + .len() + .max(1) + .checked_mul(size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token counter buffer size overflow".to_string(), + })?, + recyclable_private_buffers, + )?; + let token_buffer_len = tier1_jobs + .len() + .max(1) + .checked_mul(CLASSIC_TIER1_TOKEN_ARENA_BYTES) .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal forward DWT dimensions overflow".to_string(), + message: "J2K Metal classic Tier-1 token buffer size overflow".to_string(), + })?; + let token_buffer = + take_recyclable_private_buffer(runtime, token_buffer_len, recyclable_private_buffers)?; + let segment_buffer_len = tier1_jobs + .len() + .max(1) + .checked_mul(CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY) + .and_then(|count| count.checked_mul(size_of::())) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token segment buffer size overflow".to_string(), + })?; + let segment_buffer = + take_recyclable_private_buffer(runtime, segment_buffer_len, recyclable_private_buffers)?; + let job_count = u32::try_from(tier1_jobs.len()).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token-emitter job count exceeds u32".to_string(), + })?; + let token_stride_bytes = + u32::try_from(CLASSIC_TIER1_TOKEN_ARENA_BYTES).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token arena stride exceeds u32".to_string(), + })?; + let token_segment_stride = + u32::try_from(CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token segment stride exceeds u32".to_string(), })?; - if samples.len() != expected_len { - return Err(Error::MetalKernel { - message: "J2K Metal forward DWT sample length mismatch".to_string(), - }); - } - - with_runtime(|runtime| { - let bytes = size_of_val(samples); - let buffer_a = copied_slice_buffer(&runtime.device, samples); - let buffer_b = runtime - .device - .new_buffer(bytes as u64, MTLResourceOptions::StorageModeShared); - let command_buffer = runtime.queue.new_command_buffer(); - - let mut current_width = width; - let mut current_height = height; - let mut shapes = Vec::new(); - let mut levels_run = 0u8; - let mut active_is_a = true; - - while levels_run < num_levels && (current_width >= 2 || current_height >= 2) { - let low_width = current_width.div_ceil(2); - let low_height = current_height.div_ceil(2); - let params = J2kForwardDwt53Params { - full_width: width, - current_width, - current_height, - low_width, - low_height, - }; - - if current_height >= 2 { - let (input, output) = - active_forward_dwt53_buffers(&buffer_a, &buffer_b, active_is_a); - dispatch_forward_dwt53_pass( - &runtime.fdwt53_vertical, - command_buffer, - input, - output, - params, - ); - active_is_a = !active_is_a; - } - if current_width >= 2 { - let (input, output) = - active_forward_dwt53_buffers(&buffer_a, &buffer_b, active_is_a); - dispatch_forward_dwt53_pass( - &runtime.fdwt53_horizontal, - command_buffer, - input, - output, - params, - ); - active_is_a = !active_is_a; - } - - shapes.push(J2kForwardDwt53Level { - hl: Vec::new(), - lh: Vec::new(), - hh: Vec::new(), - width: current_width, - height: current_height, - low_width, - low_height, - high_width: current_width / 2, - high_height: current_height / 2, - }); - current_width = low_width; - current_height = low_height; - levels_run = levels_run.saturating_add(1); - } - command_buffer.commit(); - command_buffer.wait_until_completed(); + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K classic Tier-1 token emit"); + encoder.set_compute_pipeline_state(&runtime.classic_tier1_token_emit_bypass_u16_32); + encoder.set_buffer(0, Some(coefficient_buffer), 0); + encoder.set_buffer(1, Some(tier1_job_buffer), 0); + encoder.set_buffer(2, Some(&counter_buffer), 0); + encoder.set_buffer(3, Some(&token_buffer), 0); + encoder.set_buffer(4, Some(&segment_buffer), 0); + encoder.set_bytes( + 5, + size_of::() as u64, + (&raw const token_stride_bytes).cast(), + ); + encoder.set_bytes( + 6, + size_of::() as u64, + (&raw const token_segment_stride).cast(), + ); + encoder.set_bytes(7, size_of::() as u64, (&raw const job_count).cast()); + encoder.dispatch_threads( + MTLSize { + width: u64::from(job_count), + height: 1, + depth: 1, + }, + MTLSize { + width: runtime + .classic_tier1_token_emit_bypass_u16_32 + .thread_execution_width() + .max(1), + height: 1, + depth: 1, + }, + ); + encoder.end_encoding(); - let active_buffer = if active_is_a { &buffer_a } else { &buffer_b }; - let transformed = unsafe { - core::slice::from_raw_parts(active_buffer.contents().cast::(), samples.len()) - }; - let output = extract_forward_dwt53_output( - transformed, - width, - current_width, - current_height, - shapes, - )?; - Ok(output) + Ok(J2kResidentClassicTier1GpuTokenBuffers { + counter_buffer, + token_buffer, + segment_buffer, + job_count, + token_stride_bytes, + token_segment_stride, }) } #[cfg(target_os = "macos")] -fn active_forward_dwt53_buffers<'a>( - buffer_a: &'a Buffer, - buffer_b: &'a Buffer, - active_is_a: bool, -) -> (&'a Buffer, &'a Buffer) { - if active_is_a { - (buffer_a, buffer_b) - } else { - (buffer_b, buffer_a) - } +fn dispatch_classic_tier1_token_pack_from_gpu_tokens( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + tier1_job_buffer: &Buffer, + token_buffers: &J2kResidentClassicTier1GpuTokenBuffers, + tier1_output_buffer: &Buffer, + tier1_status_buffer: &Buffer, + tier1_segment_buffer: &Buffer, +) { + #[cfg(test)] + test_counters::record_classic_gpu_token_pack_dispatch(); + + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K classic Tier-1 token pack"); + encoder.set_compute_pipeline_state(&runtime.classic_tier1_token_pack_bypass_u16_32); + encoder.set_buffer(0, Some(tier1_job_buffer), 0); + encoder.set_buffer(1, Some(&token_buffers.counter_buffer), 0); + encoder.set_buffer(2, Some(&token_buffers.token_buffer), 0); + encoder.set_buffer(3, Some(&token_buffers.segment_buffer), 0); + encoder.set_buffer(4, Some(tier1_output_buffer), 0); + encoder.set_buffer(5, Some(tier1_status_buffer), 0); + encoder.set_buffer(6, Some(tier1_segment_buffer), 0); + encoder.set_bytes( + 7, + size_of::() as u64, + (&raw const token_buffers.token_stride_bytes).cast(), + ); + encoder.set_bytes( + 8, + size_of::() as u64, + (&raw const token_buffers.token_segment_stride).cast(), + ); + encoder.set_bytes( + 9, + size_of::() as u64, + (&raw const token_buffers.job_count).cast(), + ); + encoder.dispatch_threads( + MTLSize { + width: u64::from(token_buffers.job_count), + height: 1, + depth: 1, + }, + MTLSize { + width: runtime + .classic_tier1_token_pack_bypass_u16_32 + .thread_execution_width() + .max(1), + height: 1, + depth: 1, + }, + ); + encoder.end_encoding(); } #[cfg(target_os = "macos")] -fn dispatch_forward_dwt53_pass( - pipeline: &ComputePipelineState, +fn dispatch_classic_tier1_split_token_pack_from_gpu_tokens( + runtime: &MetalRuntime, command_buffer: &CommandBufferRef, - input: &Buffer, - output: &Buffer, - params: J2kForwardDwt53Params, + tier1_job_buffer: &Buffer, + token_buffers: &J2kResidentClassicTier1SplitTokenBuffers, + tier1_output_buffer: &Buffer, + tier1_status_buffer: &Buffer, + tier1_segment_buffer: &Buffer, ) { + #[cfg(test)] + test_counters::record_classic_gpu_token_pack_dispatch(); + let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(pipeline); - encoder.set_buffer(0, Some(input), 0); - encoder.set_buffer(1, Some(output), 0); + label_compute_encoder(encoder, "J2K classic Tier-1 split token pack"); + encoder.set_compute_pipeline_state(&runtime.classic_tier1_split_token_pack_bypass_u16_32); + encoder.set_buffer(0, Some(tier1_job_buffer), 0); + encoder.set_buffer(1, Some(&token_buffers.counter_buffer), 0); + encoder.set_buffer(2, Some(&token_buffers.mq_token_buffer), 0); + encoder.set_buffer(3, Some(&token_buffers.raw_token_buffer), 0); + encoder.set_buffer(4, Some(&token_buffers.segment_buffer), 0); + encoder.set_buffer(5, Some(tier1_output_buffer), 0); + encoder.set_buffer(6, Some(tier1_status_buffer), 0); + encoder.set_buffer(7, Some(tier1_segment_buffer), 0); encoder.set_bytes( - 2, - size_of::() as u64, - (&raw const params).cast(), + 8, + size_of::() as u64, + (&raw const token_buffers.mq_token_stride_bytes).cast(), + ); + encoder.set_bytes( + 9, + size_of::() as u64, + (&raw const token_buffers.raw_token_stride_bytes).cast(), + ); + encoder.set_bytes( + 10, + size_of::() as u64, + (&raw const token_buffers.token_segment_stride).cast(), + ); + encoder.set_bytes( + 11, + size_of::() as u64, + (&raw const token_buffers.job_count).cast(), ); - let width = pipeline.thread_execution_width().max(1); - let max_threads = pipeline.max_total_threads_per_threadgroup().max(width); - let height = (max_threads / width).max(1); encoder.dispatch_threads( MTLSize { - width: u64::from(params.current_width), - height: u64::from(params.current_height), + width: u64::from(token_buffers.job_count), + height: 1, depth: 1, }, MTLSize { - width, - height, + width: runtime + .classic_tier1_split_token_pack_bypass_u16_32 + .thread_execution_width() + .max(1), + height: 1, depth: 1, }, ); @@ -6360,361 +7165,1276 @@ fn dispatch_forward_dwt53_pass( } #[cfg(target_os = "macos")] -fn extract_forward_dwt53_output( - transformed: &[f32], - full_width: u32, - ll_width: u32, - ll_height: u32, - mut shapes: Vec, -) -> Result { - let full_width_usize = full_width as usize; - let mut ll = Vec::with_capacity((ll_width as usize) * (ll_height as usize)); - for y in 0..ll_height as usize { - let row_start = y - .checked_mul(full_width_usize) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal forward DWT LL row offset overflow".to_string(), - })?; - ll.extend_from_slice(&transformed[row_start..row_start + ll_width as usize]); - } - - for shape in &mut shapes { - shape.hl = extract_subband( - transformed, - full_width_usize, - shape.low_width, - 0, - shape.high_width, - shape.low_height, - )?; - shape.lh = extract_subband( - transformed, - full_width_usize, - 0, - shape.low_height, - shape.low_width, - shape.high_height, - )?; - shape.hh = extract_subband( - transformed, - full_width_usize, - shape.low_width, - shape.low_height, - shape.high_width, - shape.high_height, - )?; +fn schedule_classic_tier1_gpu_token_pack_readback( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + token_buffers: &J2kResidentClassicTier1GpuTokenBuffers, + profile_stages: bool, +) -> Result, Error> { + if !profile_stages || token_buffers.job_count == 0 { + return Ok(None); } - shapes.reverse(); - Ok(J2kForwardDwt53Output { - ll, - ll_width, - ll_height, - levels: shapes, - }) -} + let count = usize::try_from(token_buffers.job_count).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic GPU token-pack readback job count exceeds usize".to_string(), + })?; + let token_stride_bytes = + usize::try_from(token_buffers.token_stride_bytes).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic GPU token-pack token stride exceeds usize".to_string(), + })?; + let token_segment_stride = + usize::try_from(token_buffers.token_segment_stride).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic GPU token-pack segment stride exceeds usize".to_string(), + })?; + let counter_byte_len = count + .checked_mul(size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal classic GPU token-pack counter readback size overflow".to_string(), + })?; + let counter_readback = runtime.device.new_buffer( + counter_byte_len.max(1) as u64, + MTLResourceOptions::StorageModeShared, + ); -#[cfg(target_os = "macos")] -fn extract_subband( - transformed: &[f32], - full_width: usize, - x0: u32, - y0: u32, - width: u32, - height: u32, -) -> Result, Error> { - let mut out = Vec::with_capacity((width as usize) * (height as usize)); - for y in 0..height as usize { - let row_start = (y0 as usize) - .checked_add(y) - .and_then(|row| row.checked_mul(full_width)) - .and_then(|row| row.checked_add(x0 as usize)) + let copy_token_payloads = metal_profile_classic_tier1_token_pack_enabled(); + let (token_readback, token_byte_len) = if copy_token_payloads { + let byte_len = count + .checked_mul(token_stride_bytes) .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal forward DWT subband offset overflow".to_string(), + message: "J2K Metal classic GPU token-pack token readback size overflow" + .to_string(), })?; - out.extend_from_slice(&transformed[row_start..row_start + width as usize]); - } - Ok(out) -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Copy, Debug)] -pub(crate) struct J2kLosslessDeviceCodeBlock { - pub(crate) coefficient_offset: u32, - pub(crate) component: u32, - pub(crate) subband_x: u32, - pub(crate) subband_y: u32, - pub(crate) block_x: u32, - pub(crate) block_y: u32, - pub(crate) width: u32, - pub(crate) height: u32, - pub(crate) sub_band_type: signinum_j2k_native::J2kSubBandType, - pub(crate) total_bitplanes: u8, -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Copy, Debug)] -pub(crate) struct J2kLosslessDevicePrepareJob<'a> { - pub(crate) input: &'a Buffer, - pub(crate) input_byte_offset: usize, - pub(crate) input_width: u32, - pub(crate) input_height: u32, - pub(crate) input_pitch_bytes: usize, - pub(crate) output_width: u32, - pub(crate) output_height: u32, - pub(crate) components: u8, - pub(crate) bytes_per_sample: u8, - pub(crate) bit_depth: u8, - pub(crate) num_decomposition_levels: u8, - pub(crate) coefficient_count: usize, -} - -#[cfg(target_os = "macos")] -pub(crate) struct J2kLosslessDeviceBatchPrepareItem<'a> { - pub(crate) tile_index: usize, - pub(crate) job: J2kLosslessDevicePrepareJob<'a>, - pub(crate) code_blocks: Vec, -} - -#[cfg(target_os = "macos")] -pub(crate) struct J2kPreparedLosslessDeviceCodeBlocks { - coefficient_buffer: Buffer, - coefficient_byte_offset: usize, - coefficient_byte_len: usize, - coefficient_buffer_is_batch_shared: bool, - code_blocks: Vec, - recyclable_private_buffers: Vec<(usize, Buffer)>, - _prepare_command_buffer: CommandBuffer, - _deinterleave_status_buffer: Buffer, - _plane_buffers: Vec, - _scratch_buffers: Vec, - _coefficient_job_buffer: Buffer, -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Copy, Debug)] -pub(crate) struct J2kResidentPacketizationSubband { - pub(crate) code_block_start: u32, - pub(crate) code_block_count: u32, - pub(crate) num_cbs_x: u32, - pub(crate) num_cbs_y: u32, -} + ( + Some(runtime.device.new_buffer( + byte_len.max(1) as u64, + MTLResourceOptions::StorageModeShared, + )), + byte_len, + ) + } else { + (None, 0) + }; + let (segment_readback, segment_byte_len) = if copy_token_payloads { + let byte_len = count + .checked_mul(token_segment_stride) + .and_then(|segment_count| { + segment_count.checked_mul(size_of::()) + }) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal classic GPU token-pack segment readback size overflow" + .to_string(), + })?; + ( + Some(runtime.device.new_buffer( + byte_len.max(1) as u64, + MTLResourceOptions::StorageModeShared, + )), + byte_len, + ) + } else { + (None, 0) + }; -#[cfg(target_os = "macos")] -#[derive(Clone, Debug)] -pub(crate) struct J2kResidentPacketizationResolution { - pub(crate) subbands: Vec, + let blit = command_buffer.new_blit_command_encoder(); + blit.copy_from_buffer( + &token_buffers.counter_buffer, + 0, + &counter_readback, + 0, + counter_byte_len as u64, + ); + if let Some(token_readback) = token_readback.as_ref() { + blit.copy_from_buffer( + &token_buffers.token_buffer, + 0, + token_readback, + 0, + token_byte_len as u64, + ); + } + if let Some(segment_readback) = segment_readback.as_ref() { + blit.copy_from_buffer( + &token_buffers.segment_buffer, + 0, + segment_readback, + 0, + segment_byte_len as u64, + ); + } + blit.end_encoding(); + + Ok(Some(J2kResidentClassicTier1TokenEmitReadback { + counter_buffer: counter_readback, + token_buffer: token_readback, + segment_buffer: segment_readback, + token_stride_bytes, + token_segment_stride, + count, + })) } #[cfg(target_os = "macos")] -#[derive(Clone, Copy)] -pub(crate) struct J2kResidentPacketizationEncodeJob<'a> { - pub(crate) resolution_count: u32, - pub(crate) num_layers: u8, - pub(crate) num_components: u8, - pub(crate) code_block_count: u32, - pub(crate) packet_descriptors: &'a [J2kPacketizationPacketDescriptor], - pub(crate) resolutions: &'a [J2kResidentPacketizationResolution], +fn record_classic_tier1_density_counters( + stage_stats: &mut J2kResidentEncodeStageStats, + readback: &J2kResidentClassicTier1DensityReadback, +) -> Result<(), Error> { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let counters = unsafe { + core::slice::from_raw_parts( + readback + .buffer + .contents() + .cast::(), + readback.count, + ) + }; + for counter in counters { + stage_stats.tier1_sigprop_active_candidate_count_total = stage_stats + .tier1_sigprop_active_candidate_count_total + .saturating_add( + usize::try_from(counter.sigprop_active_candidates).map_err(|_| { + Error::MetalKernel { + message: "J2K Metal classic Tier-1 sigprop candidate count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_sigprop_new_significant_count_total = stage_stats + .tier1_sigprop_new_significant_count_total + .saturating_add( + usize::try_from(counter.sigprop_new_significant).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 sigprop significance count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_magref_active_candidate_count_total = stage_stats + .tier1_magref_active_candidate_count_total + .saturating_add( + usize::try_from(counter.magref_active_candidates).map_err(|_| { + Error::MetalKernel { + message: "J2K Metal classic Tier-1 magref candidate count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_arithmetic_sigprop_active_candidate_count_total = stage_stats + .tier1_arithmetic_sigprop_active_candidate_count_total + .saturating_add( + usize::try_from(counter.arithmetic_sigprop_active_candidates).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 arithmetic sigprop candidate count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_arithmetic_sigprop_new_significant_count_total = stage_stats + .tier1_arithmetic_sigprop_new_significant_count_total + .saturating_add( + usize::try_from(counter.arithmetic_sigprop_new_significant).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 arithmetic sigprop significance count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_raw_sigprop_active_candidate_count_total = stage_stats + .tier1_raw_sigprop_active_candidate_count_total + .saturating_add( + usize::try_from(counter.raw_sigprop_active_candidates).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 raw sigprop candidate count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_raw_sigprop_new_significant_count_total = stage_stats + .tier1_raw_sigprop_new_significant_count_total + .saturating_add( + usize::try_from(counter.raw_sigprop_new_significant).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 raw sigprop significance count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_arithmetic_magref_active_candidate_count_total = stage_stats + .tier1_arithmetic_magref_active_candidate_count_total + .saturating_add( + usize::try_from(counter.arithmetic_magref_active_candidates).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 arithmetic magref candidate count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_raw_magref_active_candidate_count_total = stage_stats + .tier1_raw_magref_active_candidate_count_total + .saturating_add( + usize::try_from(counter.raw_magref_active_candidates).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 raw magref candidate count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_cleanup_active_candidate_count_total = stage_stats + .tier1_cleanup_active_candidate_count_total + .saturating_add( + usize::try_from(counter.cleanup_active_candidates).map_err(|_| { + Error::MetalKernel { + message: "J2K Metal classic Tier-1 cleanup candidate count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_cleanup_new_significant_count_total = stage_stats + .tier1_cleanup_new_significant_count_total + .saturating_add( + usize::try_from(counter.cleanup_new_significant).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 cleanup significance count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_cleanup_rlc_stripe_count_total = stage_stats + .tier1_cleanup_rlc_stripe_count_total + .saturating_add(usize::try_from(counter.cleanup_rlc_stripes).map_err(|_| { + Error::MetalKernel { + message: "J2K Metal classic Tier-1 cleanup RLC stripe count exceeds usize" + .to_string(), + } + })?); + stage_stats.tier1_cleanup_rlc_zero_stripe_count_total = stage_stats + .tier1_cleanup_rlc_zero_stripe_count_total + .saturating_add( + usize::try_from(counter.cleanup_rlc_zero_stripes).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 cleanup zero-RLC stripe count exceeds usize" + .to_string(), + } + })?, + ); + } + Ok(()) } #[cfg(target_os = "macos")] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum J2kLosslessCodestreamBlockCodingMode { - Classic, - HighThroughput, +fn record_classic_tier1_symbol_plan_counters( + stage_stats: &mut J2kResidentEncodeStageStats, + readback: &J2kResidentClassicTier1SymbolPlanReadback, +) -> Result<(), Error> { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let counters = unsafe { + core::slice::from_raw_parts( + readback + .buffer + .contents() + .cast::(), + readback.count, + ) + }; + for counter in counters { + if counter.code != J2K_ENCODE_STATUS_OK { + return Err(encode_status_error( + "classic Tier-1 symbol plan", + counter.code, + counter.detail, + )); + } + stage_stats.tier1_symbol_plan_mq_symbol_count_total = stage_stats + .tier1_symbol_plan_mq_symbol_count_total + .saturating_add(usize::try_from(counter.mq_symbol_count).map_err(|_| { + Error::MetalKernel { + message: "J2K Metal classic Tier-1 symbol-plan MQ count exceeds usize" + .to_string(), + } + })?); + stage_stats.tier1_symbol_plan_raw_bit_count_total = stage_stats + .tier1_symbol_plan_raw_bit_count_total + .saturating_add(usize::try_from(counter.raw_bit_count).map_err(|_| { + Error::MetalKernel { + message: "J2K Metal classic Tier-1 symbol-plan raw bit count exceeds usize" + .to_string(), + } + })?); + let mq_symbol_count = + usize::try_from(counter.mq_symbol_count).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 symbol-plan MQ count exceeds usize".to_string(), + })?; + let raw_bit_count = + usize::try_from(counter.raw_bit_count).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 symbol-plan raw bit count exceeds usize" + .to_string(), + })?; + stage_stats.max_tier1_symbol_plan_mq_symbols_per_block = stage_stats + .max_tier1_symbol_plan_mq_symbols_per_block + .max(mq_symbol_count); + stage_stats.max_tier1_symbol_plan_raw_bits_per_block = stage_stats + .max_tier1_symbol_plan_raw_bits_per_block + .max(raw_bit_count); + let mq_packed_bytes = mq_symbol_count + .saturating_mul(6) + .saturating_add(7) + .checked_div(8) + .unwrap_or(usize::MAX); + let raw_packed_bytes = raw_bit_count + .saturating_add(7) + .checked_div(8) + .unwrap_or(usize::MAX); + let packed_token_bytes = mq_packed_bytes.saturating_add(raw_packed_bytes); + stage_stats.tier1_symbol_plan_packed_token_bytes_total = stage_stats + .tier1_symbol_plan_packed_token_bytes_total + .saturating_add(packed_token_bytes); + stage_stats.max_tier1_symbol_plan_packed_token_bytes_per_block = stage_stats + .max_tier1_symbol_plan_packed_token_bytes_per_block + .max(packed_token_bytes); + stage_stats.tier1_symbol_plan_cleanup_mq_symbol_count_total = stage_stats + .tier1_symbol_plan_cleanup_mq_symbol_count_total + .saturating_add( + usize::try_from(counter.cleanup_mq_symbol_count).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 symbol-plan cleanup MQ count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_symbol_plan_sigprop_mq_symbol_count_total = stage_stats + .tier1_symbol_plan_sigprop_mq_symbol_count_total + .saturating_add( + usize::try_from(counter.sigprop_mq_symbol_count).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 symbol-plan sigprop MQ count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_symbol_plan_magref_mq_symbol_count_total = stage_stats + .tier1_symbol_plan_magref_mq_symbol_count_total + .saturating_add( + usize::try_from(counter.magref_mq_symbol_count).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 symbol-plan magref MQ count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_symbol_plan_raw_sigprop_bit_count_total = stage_stats + .tier1_symbol_plan_raw_sigprop_bit_count_total + .saturating_add(usize::try_from(counter.raw_sigprop_bit_count).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 symbol-plan raw sigprop bit count exceeds usize" + .to_string(), + } + })?); + stage_stats.tier1_symbol_plan_raw_magref_bit_count_total = stage_stats + .tier1_symbol_plan_raw_magref_bit_count_total + .saturating_add(usize::try_from(counter.raw_magref_bit_count).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 symbol-plan raw magref bit count exceeds usize" + .to_string(), + } + })?); + stage_stats.tier1_symbol_plan_cleanup_sign_symbol_count_total = stage_stats + .tier1_symbol_plan_cleanup_sign_symbol_count_total + .saturating_add( + usize::try_from(counter.cleanup_sign_symbol_count).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 symbol-plan cleanup sign count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_symbol_plan_sigprop_sign_symbol_count_total = stage_stats + .tier1_symbol_plan_sigprop_sign_symbol_count_total + .saturating_add( + usize::try_from(counter.sigprop_sign_symbol_count).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 symbol-plan sigprop sign count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.tier1_symbol_plan_mq_symbol_hash_xor ^= usize::try_from(counter.mq_symbol_hash) + .map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 symbol-plan MQ hash exceeds usize".to_string(), + })?; + stage_stats.tier1_symbol_plan_raw_bit_hash_xor ^= usize::try_from(counter.raw_bit_hash) + .map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 symbol-plan raw hash exceeds usize".to_string(), + })?; + } + Ok(()) } #[cfg(target_os = "macos")] -#[derive(Clone, Copy, Debug)] -pub(crate) struct J2kLosslessCodestreamAssemblyJob { - pub(crate) width: u32, - pub(crate) height: u32, - pub(crate) num_components: u8, - pub(crate) bit_depth: u8, - pub(crate) signed: bool, - pub(crate) num_decomposition_levels: u8, - pub(crate) use_mct: bool, - pub(crate) guard_bits: u8, - pub(crate) progression_order: EncodeProgressionOrder, - pub(crate) write_tlm: bool, - pub(crate) block_coding_mode: J2kLosslessCodestreamBlockCodingMode, -} +fn record_classic_tier1_pass_plan_counters( + stage_stats: &mut J2kResidentEncodeStageStats, + readback: &J2kResidentClassicTier1PassPlanReadback, +) -> Result<(), Error> { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let counters = unsafe { + core::slice::from_raw_parts( + readback + .buffer + .contents() + .cast::(), + readback.count, + ) + }; + for counter in counters { + if counter.code != J2K_ENCODE_STATUS_OK { + return Err(encode_status_error( + "classic Tier-1 pass plan", + counter.code, + counter.detail, + )); + } + let mq_symbol_count = + usize::try_from(counter.mq_symbol_count).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 pass-plan MQ count exceeds usize".to_string(), + })?; + let raw_bit_count = + usize::try_from(counter.raw_bit_count).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 pass-plan raw bit count exceeds usize" + .to_string(), + })?; + stage_stats.tier1_pass_plan_mq_symbol_count_total = stage_stats + .tier1_pass_plan_mq_symbol_count_total + .saturating_add(mq_symbol_count); + stage_stats.tier1_pass_plan_raw_bit_count_total = stage_stats + .tier1_pass_plan_raw_bit_count_total + .saturating_add(raw_bit_count); + stage_stats.tier1_pass_plan_nonempty_mq_pass_count_total = stage_stats + .tier1_pass_plan_nonempty_mq_pass_count_total + .saturating_add(usize::try_from(counter.nonempty_mq_passes).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 pass-plan nonempty MQ pass count exceeds usize" + .to_string(), + } + })?); + stage_stats.tier1_pass_plan_nonempty_raw_pass_count_total = stage_stats + .tier1_pass_plan_nonempty_raw_pass_count_total + .saturating_add(usize::try_from(counter.nonempty_raw_passes).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 pass-plan nonempty raw pass count exceeds usize" + .to_string(), + } + })?); + stage_stats.max_tier1_pass_plan_mq_symbols_per_pass = + stage_stats.max_tier1_pass_plan_mq_symbols_per_pass.max( + usize::try_from(counter.max_mq_symbols_per_pass).map_err(|_| { + Error::MetalKernel { + message: + "J2K Metal classic Tier-1 pass-plan max MQ pass count exceeds usize" + .to_string(), + } + })?, + ); + stage_stats.max_tier1_pass_plan_raw_bits_per_pass = + stage_stats.max_tier1_pass_plan_raw_bits_per_pass.max( + usize::try_from(counter.max_raw_bits_per_pass).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 pass-plan max raw pass count exceeds usize" + .to_string(), + })?, + ); -#[cfg(target_os = "macos")] -pub(crate) struct J2kResidentLosslessTier1CodeBlocks { - output_buffer: Buffer, - status_buffer: Buffer, - job_buffer: Buffer, - batch_jobs: Vec, - code_blocks: Vec, - output_capacity_total: usize, - _segment_buffer: Buffer, - tier1_command_buffer: CommandBuffer, - _coefficient_buffer: Buffer, - prepare_command_buffer: CommandBuffer, - _deinterleave_status_buffer: Buffer, - _plane_buffers: Vec, - _scratch_buffers: Vec, - _coefficient_job_buffer: Buffer, + let pass_mq_total = counter.mq_symbols_by_pass.iter().try_fold( + 0usize, + |acc, &value| -> Result { + Ok(acc.saturating_add( + usize::try_from(value).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 pass-plan MQ pass count exceeds usize" + .to_string(), + })?, + )) + }, + )?; + let pass_raw_total = counter.raw_bits_by_pass.iter().try_fold( + 0usize, + |acc, &value| -> Result { + Ok(acc.saturating_add(usize::try_from(value).map_err(|_| { + Error::MetalKernel { + message: "J2K Metal classic Tier-1 pass-plan raw pass count exceeds usize" + .to_string(), + } + })?)) + }, + )?; + if pass_mq_total != mq_symbol_count || pass_raw_total != raw_bit_count { + return Err(Error::MetalKernel { + message: "J2K Metal classic Tier-1 pass-plan per-pass totals are inconsistent" + .to_string(), + }); + } + } + Ok(()) } #[cfg(target_os = "macos")] -pub(crate) struct J2kResidentLosslessHtCodeBlocks { - output_buffer: Buffer, - status_buffer: Buffer, - job_buffer: Buffer, - batch_jobs: Vec, - code_blocks: Vec, - output_capacity_total: usize, - tier1_command_buffer: CommandBuffer, - _coefficient_buffer: Buffer, - prepare_command_buffer: CommandBuffer, - _deinterleave_status_buffer: Buffer, - _plane_buffers: Vec, - _scratch_buffers: Vec, - _coefficient_job_buffer: Buffer, +fn compare_classic_tier1_symbol_plan_and_pass_plan_counters( + symbol_plan: &J2kResidentClassicTier1SymbolPlanReadback, + pass_plan: &J2kResidentClassicTier1PassPlanReadback, +) -> Result<(), Error> { + if symbol_plan.count != pass_plan.count { + return Err(Error::MetalKernel { + message: "J2K Metal classic Tier-1 pass-plan comparison count mismatch".to_string(), + }); + } + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let symbol_plan_counters = unsafe { + core::slice::from_raw_parts( + symbol_plan + .buffer + .contents() + .cast::(), + symbol_plan.count, + ) + }; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let pass_plan_counters = unsafe { + core::slice::from_raw_parts( + pass_plan + .buffer + .contents() + .cast::(), + pass_plan.count, + ) + }; + for (idx, (plan, pass)) in symbol_plan_counters + .iter() + .zip(pass_plan_counters) + .enumerate() + { + let plan_values = [ + plan.code, + plan.detail, + plan.coding_passes, + plan.missing_bit_planes, + plan.segment_count, + plan.mq_symbol_count, + plan.raw_bit_count, + ]; + let pass_values = [ + pass.code, + pass.detail, + pass.coding_passes, + pass.missing_bit_planes, + pass.segment_count, + pass.mq_symbol_count, + pass.raw_bit_count, + ]; + if plan_values != pass_values { + return Err(Error::MetalKernel { + message: format!( + "J2K Metal classic Tier-1 pass-plan diverged from symbol plan at block {idx}" + ), + }); + } + } + Ok(()) } #[cfg(target_os = "macos")] -pub(crate) struct J2kResidentLosslessCodestream { - pub(crate) buffer: Buffer, - pub(crate) byte_offset: usize, - pub(crate) byte_len: usize, - pub(crate) capacity: usize, - pub(crate) gpu_duration: Option, +fn record_classic_tier1_token_emit_counters( + stage_stats: &mut J2kResidentEncodeStageStats, + readback: &J2kResidentClassicTier1TokenEmitReadback, +) -> Result<(), Error> { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let counters = unsafe { + core::slice::from_raw_parts( + readback + .counter_buffer + .contents() + .cast::(), + readback.count, + ) + }; + for counter in counters { + if counter.code != J2K_ENCODE_STATUS_OK { + return Err(encode_status_error( + "classic Tier-1 token emit", + counter.code, + counter.detail, + )); + } + let mq_symbol_count = + usize::try_from(counter.mq_symbol_count).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token-emitter MQ count exceeds usize" + .to_string(), + })?; + let raw_bit_count = + usize::try_from(counter.raw_bit_count).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token-emitter raw bit count exceeds usize" + .to_string(), + })?; + let segment_count = + usize::try_from(counter.segment_count).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token-emitter segment count exceeds usize" + .to_string(), + })?; + let token_bytes = mq_symbol_count + .saturating_mul(6) + .saturating_add(raw_bit_count) + .saturating_add(7) + .checked_div(8) + .unwrap_or(usize::MAX); + stage_stats.tier1_token_emit_mq_symbol_count_total = stage_stats + .tier1_token_emit_mq_symbol_count_total + .saturating_add(mq_symbol_count); + stage_stats.tier1_token_emit_raw_bit_count_total = stage_stats + .tier1_token_emit_raw_bit_count_total + .saturating_add(raw_bit_count); + stage_stats.tier1_token_emit_token_bytes_total = stage_stats + .tier1_token_emit_token_bytes_total + .saturating_add(token_bytes); + stage_stats.max_tier1_token_emit_token_bytes_per_block = stage_stats + .max_tier1_token_emit_token_bytes_per_block + .max(token_bytes); + stage_stats.tier1_token_emit_segment_count_total = stage_stats + .tier1_token_emit_segment_count_total + .saturating_add(segment_count); + stage_stats.max_tier1_token_emit_segments_per_block = stage_stats + .max_tier1_token_emit_segments_per_block + .max(segment_count); + stage_stats.tier1_token_emit_mq_symbol_hash_xor ^= usize::try_from(counter.mq_symbol_hash) + .map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token-emitter MQ hash exceeds usize".to_string(), + })?; + stage_stats.tier1_token_emit_raw_bit_hash_xor ^= usize::try_from(counter.raw_bit_hash) + .map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 token-emitter raw hash exceeds usize" + .to_string(), + })?; + } + Ok(()) } #[cfg(target_os = "macos")] -pub(crate) struct J2kPendingResidentLosslessCodestream { - buffer: Buffer, - capacity: usize, - status_buffer: Buffer, - command_buffer: CommandBuffer, - retained_command_buffers: Vec, - _retained_buffers: Vec, - status_stage: &'static str, - length_error: &'static str, - capacity_error: &'static str, +fn compare_classic_tier1_symbol_plan_and_token_emit_counters( + symbol_plan: &J2kResidentClassicTier1SymbolPlanReadback, + token_emit: &J2kResidentClassicTier1TokenEmitReadback, +) -> Result<(), Error> { + if symbol_plan.count != token_emit.count { + return Err(Error::MetalKernel { + message: "J2K Metal classic Tier-1 token-emitter comparison count mismatch".to_string(), + }); + } + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let symbol_plan_counters = unsafe { + core::slice::from_raw_parts( + symbol_plan + .buffer + .contents() + .cast::(), + symbol_plan.count, + ) + }; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let token_emit_counters = unsafe { + core::slice::from_raw_parts( + token_emit + .counter_buffer + .contents() + .cast::(), + token_emit.count, + ) + }; + for (idx, (plan, emit)) in symbol_plan_counters + .iter() + .zip(token_emit_counters) + .enumerate() + { + let plan_values = [ + plan.code, + plan.detail, + plan.coding_passes, + plan.missing_bit_planes, + plan.segment_count, + plan.mq_symbol_count, + plan.raw_bit_count, + plan.cleanup_mq_symbol_count, + plan.sigprop_mq_symbol_count, + plan.magref_mq_symbol_count, + plan.raw_sigprop_bit_count, + plan.raw_magref_bit_count, + plan.cleanup_sign_symbol_count, + plan.sigprop_sign_symbol_count, + plan.mq_symbol_hash, + plan.raw_bit_hash, + ]; + let emit_values = [ + emit.code, + emit.detail, + emit.coding_passes, + emit.missing_bit_planes, + emit.segment_count, + emit.mq_symbol_count, + emit.raw_bit_count, + emit.cleanup_mq_symbol_count, + emit.sigprop_mq_symbol_count, + emit.magref_mq_symbol_count, + emit.raw_sigprop_bit_count, + emit.raw_magref_bit_count, + emit.cleanup_sign_symbol_count, + emit.sigprop_sign_symbol_count, + emit.mq_symbol_hash, + emit.raw_bit_hash, + ]; + if plan_values != emit_values { + return Err(Error::MetalKernel { + message: format!( + "J2K Metal classic Tier-1 token-emitter diverged from symbol plan at block {idx}" + ), + }); + } + } + Ok(()) } #[cfg(target_os = "macos")] -pub(crate) struct J2kResidentHtBatchEncodeItem { - pub(crate) prepared: J2kPreparedLosslessDeviceCodeBlocks, - pub(crate) resolution_count: u32, - pub(crate) num_layers: u8, - pub(crate) num_components: u8, - pub(crate) code_block_count: u32, - pub(crate) packet_descriptors: Vec, - pub(crate) resolutions: Vec, - pub(crate) codestream: J2kLosslessCodestreamAssemblyJob, +fn validate_classic_tier1_split_token_emit_counters( + readback: &J2kResidentClassicTier1SplitTokenBuffers, +) -> Result<(), Error> { + if readback.mq_token_stride_bytes == 0 + || readback.raw_token_stride_bytes == 0 + || readback.token_segment_stride == 0 + { + return Err(Error::MetalKernel { + message: "J2K Metal classic Tier-1 split-token readback has empty stride".to_string(), + }); + } + let count = usize::try_from(readback.job_count).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 split-token counter count exceeds usize".to_string(), + })?; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let counters = unsafe { + core::slice::from_raw_parts( + readback + .counter_buffer + .contents() + .cast::(), + count, + ) + }; + for counter in counters { + if counter.code != J2K_ENCODE_STATUS_OK { + return Err(encode_status_error( + "classic Tier-1 split-token emit", + counter.code, + counter.detail, + )); + } + } + Ok(()) } #[cfg(target_os = "macos")] -pub(crate) struct J2kResidentClassicBatchEncodeItem { - pub(crate) prepared: J2kPreparedLosslessDeviceCodeBlocks, - pub(crate) resolution_count: u32, - pub(crate) num_layers: u8, - pub(crate) num_components: u8, - pub(crate) code_block_count: u32, - pub(crate) packet_descriptors: Vec, - pub(crate) resolutions: Vec, - pub(crate) codestream: J2kLosslessCodestreamAssemblyJob, +fn compare_classic_tier1_symbol_plan_and_split_token_emit_counters( + symbol_plan: &J2kResidentClassicTier1SymbolPlanReadback, + split_emit: &J2kResidentClassicTier1SplitTokenBuffers, +) -> Result<(), Error> { + let split_count = usize::try_from(split_emit.job_count).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 split-token comparison count exceeds usize".to_string(), + })?; + if symbol_plan.count != split_count { + return Err(Error::MetalKernel { + message: "J2K Metal classic Tier-1 split-token comparison count mismatch".to_string(), + }); + } + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let symbol_plan_counters = unsafe { + core::slice::from_raw_parts( + symbol_plan + .buffer + .contents() + .cast::(), + symbol_plan.count, + ) + }; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let split_emit_counters = unsafe { + core::slice::from_raw_parts( + split_emit + .counter_buffer + .contents() + .cast::(), + split_count, + ) + }; + for (idx, (plan, emit)) in symbol_plan_counters + .iter() + .zip(split_emit_counters) + .enumerate() + { + let plan_values = [ + plan.code, + plan.detail, + plan.coding_passes, + plan.missing_bit_planes, + plan.segment_count, + plan.mq_symbol_count, + plan.raw_bit_count, + plan.cleanup_mq_symbol_count, + plan.sigprop_mq_symbol_count, + plan.magref_mq_symbol_count, + plan.raw_sigprop_bit_count, + plan.raw_magref_bit_count, + plan.cleanup_sign_symbol_count, + plan.sigprop_sign_symbol_count, + plan.mq_symbol_hash, + plan.raw_bit_hash, + ]; + let emit_values = [ + emit.code, + emit.detail, + emit.coding_passes, + emit.missing_bit_planes, + emit.segment_count, + emit.mq_symbol_count, + emit.raw_bit_count, + emit.cleanup_mq_symbol_count, + emit.sigprop_mq_symbol_count, + emit.magref_mq_symbol_count, + emit.raw_sigprop_bit_count, + emit.raw_magref_bit_count, + emit.cleanup_sign_symbol_count, + emit.sigprop_sign_symbol_count, + emit.mq_symbol_hash, + emit.raw_bit_hash, + ]; + if plan_values != emit_values { + return Err(Error::MetalKernel { + message: format!( + "J2K Metal classic Tier-1 split-token emitter diverged from symbol plan at block {idx}" + ), + }); + } + } + Ok(()) } #[cfg(target_os = "macos")] -pub(crate) struct J2kPendingResidentLosslessCodestreamBatch { - device: Device, - buffer: Buffer, - byte_offsets: Vec, - capacities: Vec, - status_buffer: Buffer, - command_buffer: CommandBuffer, - retained_command_buffers: Vec, - _retained_buffers: Vec, - recyclable_private_buffers: Vec<(usize, Buffer)>, - status_stage: &'static str, - length_error: &'static str, - capacity_error: &'static str, +fn profile_classic_tier1_token_pack( + stage_stats: &mut J2kResidentEncodeStageStats, + readback: &J2kResidentClassicTier1TokenEmitReadback, +) -> Result<(), Error> { + if !metal_profile_classic_tier1_token_pack_enabled() { + return Ok(()); + } + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let counters = unsafe { + core::slice::from_raw_parts( + readback + .counter_buffer + .contents() + .cast::(), + readback.count, + ) + }; + let token_buffer = readback + .token_buffer + .as_ref() + .ok_or_else(|| Error::MetalKernel { + message: + "J2K Metal classic Tier-1 token-pack profiling requires token payload readback" + .to_string(), + })?; + let segment_buffer = readback + .segment_buffer + .as_ref() + .ok_or_else(|| Error::MetalKernel { + message: + "J2K Metal classic Tier-1 token-pack profiling requires token segment readback" + .to_string(), + })?; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let token_bytes = unsafe { + core::slice::from_raw_parts( + token_buffer.contents().cast::(), + readback.count.saturating_mul(readback.token_stride_bytes), + ) + }; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let token_segments = unsafe { + core::slice::from_raw_parts( + segment_buffer + .contents() + .cast::(), + readback.count.saturating_mul(readback.token_segment_stride), + ) + }; + let token_stride_bytes = readback.token_stride_bytes; + let token_segment_stride = readback.token_segment_stride; + + let started = Instant::now(); + let packed_lengths = (0..readback.count) + .into_par_iter() + .map(|block_idx| -> Result { + let counter = &counters[block_idx]; + if counter.code != J2K_ENCODE_STATUS_OK { + return Err(format!( + "classic Tier-1 token pack input failed at block {block_idx}: code={} detail={}", + counter.code, counter.detail + )); + } + let segment_count = usize::try_from(counter.segment_count) + .map_err(|_| "J2K Metal classic Tier-1 token-pack segment count exceeds usize")?; + if segment_count > CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY { + return Err( + "J2K Metal classic Tier-1 token-pack segment count exceeds capacity" + .to_string(), + ); + } + let token_start = block_idx + .checked_mul(token_stride_bytes) + .ok_or("J2K Metal classic Tier-1 token-pack byte offset overflow")?; + let segment_start = block_idx + .checked_mul(token_segment_stride) + .ok_or("J2K Metal classic Tier-1 token-pack segment offset overflow")?; + let mut native_segments = Vec::with_capacity(segment_count); + for segment in &token_segments[segment_start..segment_start + segment_count] { + let start_coding_pass = u8::try_from(segment.pass_range & 0xFFFF) + .map_err(|_| "J2K Metal classic Tier-1 token-pack start pass exceeds u8")?; + let end_coding_pass = u8::try_from(segment.pass_range >> 16) + .map_err(|_| "J2K Metal classic Tier-1 token-pack end pass exceeds u8")?; + native_segments.push(J2kTier1TokenSegment { + token_bit_offset: segment.token_bit_offset, + token_bit_count: segment.token_bit_count, + start_coding_pass, + end_coding_pass, + use_arithmetic: (segment.flags & 1) != 0, + }); + } + let packed = pack_j2k_code_block_scalar_from_tier1_tokens( + &token_bytes[token_start..token_start + token_stride_bytes], + &native_segments, + u8::try_from(counter.coding_passes).map_err(|_| { + "J2K Metal classic Tier-1 token-pack coding-pass count exceeds u8" + })?, + u8::try_from(counter.missing_bit_planes).map_err(|_| { + "J2K Metal classic Tier-1 token-pack missing bitplanes exceed u8" + })?, + ) + .map_err(|message| format!("J2K Metal classic Tier-1 token-pack failed: {message}"))?; + Ok(packed.data.len()) + }) + .collect::, _>>() + .map_err(|message| Error::MetalKernel { message })?; + for output_len in packed_lengths { + stage_stats.tier1_token_pack_output_bytes_total = stage_stats + .tier1_token_pack_output_bytes_total + .saturating_add(output_len); + stage_stats.max_tier1_token_pack_output_bytes_per_block = stage_stats + .max_tier1_token_pack_output_bytes_per_block + .max(output_len); + } + stage_stats.classic_tier1_token_pack_duration = started.elapsed(); + Ok(()) } #[cfg(target_os = "macos")] -struct PreparedLosslessBatchTile { - coefficient_buffer: Buffer, - coefficient_byte_offset: usize, - coefficient_byte_len: usize, - coefficient_buffer_is_batch_shared: bool, - code_blocks: Vec, - recyclable_private_buffers: Vec<(usize, Buffer)>, - prepare_command_buffer: CommandBuffer, - deinterleave_status_buffer: Buffer, - plane_buffers: Vec, - scratch_buffers: Vec, - coefficient_job_buffer: Buffer, - resolution_count: u32, - num_layers: u8, - num_components: u8, - code_block_count: u32, - packet_descriptors: Vec, - resolutions: Vec, - codestream: J2kLosslessCodestreamAssemblyJob, +fn record_resident_tier1_output_usage( + stage_stats: &mut J2kResidentEncodeStageStats, + readback: &J2kResidentTier1StatusReadback, + classic_gpu_token_pack_used: bool, +) -> Result<(), Error> { + match readback.kind { + J2kResidentTier1StatusKind::Classic => { + let classic_jobs = + readback + .classic_jobs + .as_ref() + .ok_or_else(|| Error::MetalKernel { + message: + "J2K Metal classic Tier-1 profile readback is missing job metadata" + .to_string(), + })?; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let statuses = unsafe { + core::slice::from_raw_parts( + readback.buffer.contents().cast::(), + readback.count, + ) + }; + if classic_jobs.len() != statuses.len() { + return Err(Error::MetalKernel { + message: "J2K Metal classic Tier-1 profile readback job/status count mismatch" + .to_string(), + }); + } + for (status, job) in statuses.iter().zip(classic_jobs) { + if status.code != J2K_ENCODE_STATUS_OK { + return Err(encode_status_error( + "classic Tier-1", + status.code, + status.detail, + )); + } + let data_len = + usize::try_from(status.data_len).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 output length exceeds usize".to_string(), + })?; + stage_stats.tier1_output_used_bytes_total = stage_stats + .tier1_output_used_bytes_total + .saturating_add(data_len); + stage_stats.max_tier1_output_used_bytes = + stage_stats.max_tier1_output_used_bytes.max(data_len); + if classic_gpu_token_pack_used { + stage_stats.tier1_token_pack_output_bytes_total = stage_stats + .tier1_token_pack_output_bytes_total + .saturating_add(data_len); + stage_stats.max_tier1_token_pack_output_bytes_per_block = stage_stats + .max_tier1_token_pack_output_bytes_per_block + .max(data_len); + } + let coding_passes = + usize::try_from(status.number_of_coding_passes).map_err(|_| { + Error::MetalKernel { + message: "J2K Metal classic Tier-1 coding-pass count exceeds usize" + .to_string(), + } + })?; + stage_stats.tier1_coding_pass_count_total = stage_stats + .tier1_coding_pass_count_total + .saturating_add(coding_passes); + stage_stats.max_tier1_coding_passes_per_block = stage_stats + .max_tier1_coding_passes_per_block + .max(coding_passes); + let pass_counts = + classic_tier1_pass_class_counts(coding_passes, readback.classic_style_flags); + let coeff_count = usize::try_from(job.width) + .and_then(|width| { + usize::try_from(job.height).map(|height| width.saturating_mul(height)) + }) + .map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 code-block dimensions exceed usize" + .to_string(), + })?; + accumulate_classic_tier1_scan_estimates(stage_stats, pass_counts, coeff_count); + stage_stats.tier1_arithmetic_pass_count_total = stage_stats + .tier1_arithmetic_pass_count_total + .saturating_add(pass_counts.arithmetic); + stage_stats.tier1_raw_pass_count_total = stage_stats + .tier1_raw_pass_count_total + .saturating_add(pass_counts.raw); + stage_stats.tier1_cleanup_pass_count_total = stage_stats + .tier1_cleanup_pass_count_total + .saturating_add(pass_counts.cleanup); + stage_stats.tier1_sigprop_pass_count_total = stage_stats + .tier1_sigprop_pass_count_total + .saturating_add(pass_counts.sigprop); + stage_stats.tier1_magref_pass_count_total = stage_stats + .tier1_magref_pass_count_total + .saturating_add(pass_counts.magref); + stage_stats.tier1_arithmetic_cleanup_pass_count_total = stage_stats + .tier1_arithmetic_cleanup_pass_count_total + .saturating_add(pass_counts.arithmetic_cleanup); + stage_stats.tier1_arithmetic_sigprop_pass_count_total = stage_stats + .tier1_arithmetic_sigprop_pass_count_total + .saturating_add(pass_counts.arithmetic_sigprop); + stage_stats.tier1_arithmetic_magref_pass_count_total = stage_stats + .tier1_arithmetic_magref_pass_count_total + .saturating_add(pass_counts.arithmetic_magref); + stage_stats.tier1_raw_sigprop_pass_count_total = stage_stats + .tier1_raw_sigprop_pass_count_total + .saturating_add(pass_counts.raw_sigprop); + stage_stats.tier1_raw_magref_pass_count_total = stage_stats + .tier1_raw_magref_pass_count_total + .saturating_add(pass_counts.raw_magref); + if coding_passes == 0 { + stage_stats.tier1_zero_block_count_total = + stage_stats.tier1_zero_block_count_total.saturating_add(1); + } else { + stage_stats.tier1_nonzero_block_count_total = stage_stats + .tier1_nonzero_block_count_total + .saturating_add(1); + } + let missing_bitplanes = + usize::try_from(status.missing_bit_planes).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 missing-bitplane count exceeds usize" + .to_string(), + })?; + stage_stats.tier1_missing_bitplane_count_total = stage_stats + .tier1_missing_bitplane_count_total + .saturating_add(missing_bitplanes); + stage_stats.max_tier1_missing_bitplanes_per_block = stage_stats + .max_tier1_missing_bitplanes_per_block + .max(missing_bitplanes); + let segment_count = + usize::try_from(status.segment_count).map_err(|_| Error::MetalKernel { + message: "J2K Metal classic Tier-1 segment count exceeds usize".to_string(), + })?; + stage_stats.tier1_segment_count_total = stage_stats + .tier1_segment_count_total + .saturating_add(segment_count); + stage_stats.max_tier1_segments_per_block = + stage_stats.max_tier1_segments_per_block.max(segment_count); + } + } + J2kResidentTier1StatusKind::HighThroughput => { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let statuses = unsafe { + core::slice::from_raw_parts( + readback.buffer.contents().cast::(), + readback.count, + ) + }; + for status in statuses { + if status.code != J2K_ENCODE_STATUS_OK { + return Err(encode_status_error( + "HTJ2K Tier-1", + status.code, + status.detail, + )); + } + let data_len = + usize::try_from(status.data_len).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal Tier-1 output length exceeds usize".to_string(), + })?; + stage_stats.tier1_output_used_bytes_total = stage_stats + .tier1_output_used_bytes_total + .saturating_add(data_len); + stage_stats.max_tier1_output_used_bytes = + stage_stats.max_tier1_output_used_bytes.max(data_len); + let coding_passes = + usize::try_from(status.num_coding_passes).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal Tier-1 coding-pass count exceeds usize".to_string(), + })?; + stage_stats.tier1_coding_pass_count_total = stage_stats + .tier1_coding_pass_count_total + .saturating_add(coding_passes); + stage_stats.max_tier1_coding_passes_per_block = stage_stats + .max_tier1_coding_passes_per_block + .max(coding_passes); + if coding_passes == 0 { + stage_stats.tier1_zero_block_count_total = + stage_stats.tier1_zero_block_count_total.saturating_add(1); + } else { + stage_stats.tier1_nonzero_block_count_total = stage_stats + .tier1_nonzero_block_count_total + .saturating_add(1); + } + let missing_bitplanes = + usize::try_from(status.num_zero_bitplanes).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal Tier-1 missing-bitplane count exceeds usize" + .to_string(), + })?; + stage_stats.tier1_missing_bitplane_count_total = stage_stats + .tier1_missing_bitplane_count_total + .saturating_add(missing_bitplanes); + stage_stats.max_tier1_missing_bitplanes_per_block = stage_stats + .max_tier1_missing_bitplanes_per_block + .max(missing_bitplanes); + } + } + } + Ok(()) } #[cfg(target_os = "macos")] -pub(crate) fn wait_resident_lossless_codestream( - pending: J2kPendingResidentLosslessCodestream, -) -> Result { - pending.command_buffer.wait_until_completed(); - let gpu_duration = completed_command_buffers_gpu_duration( - &pending.retained_command_buffers, - &pending.command_buffer, - ); - let status = unsafe { - pending - .status_buffer - .contents() - .cast::() - .read() - }; - if status.code != J2K_ENCODE_STATUS_OK { - return Err(encode_status_error( - pending.status_stage, - status.code, - status.detail, - )); - } - let data_len = usize::try_from(status.data_len).map_err(|_| Error::MetalKernel { - message: pending.length_error.to_string(), - })?; - if data_len > pending.capacity { - return Err(Error::MetalKernel { - message: pending.capacity_error.to_string(), - }); - } - Ok(J2kResidentLosslessCodestream { - buffer: pending.buffer, - byte_offset: 0, - byte_len: data_len, - capacity: pending.capacity, - gpu_duration, - }) +fn wait_resident_codestream_command_buffer(command_buffer: &CommandBufferRef) { + #[cfg(test)] + test_counters::record_resident_codestream_command_buffer_wait(); + let _signpost = hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_COMMAND_WAIT); + command_buffer.wait_until_completed(); } #[cfg(target_os = "macos")] -pub(crate) fn wait_resident_lossless_codestream_batch( +fn finish_completed_resident_lossless_codestream_batch( pending: J2kPendingResidentLosslessCodestreamBatch, -) -> Result, Error> { - pending.command_buffer.wait_until_completed(); - let gpu_duration = completed_command_buffers_gpu_duration( +) -> Result { + let _signpost = hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_RESULT_HARVEST); + let profile_stages = metal_profile_stages_enabled(); + let result_harvest_started = profile_stages.then(Instant::now); + let gpu_timings = completed_command_buffers_gpu_duration_and_elapsed_window( &pending.retained_command_buffers, &pending.command_buffer, ); + let gpu_duration = gpu_timings.map(|timings| timings.0); + let gpu_elapsed_wall_duration = gpu_timings.map(|timings| timings.1); + let mut stage_stats = pending.stage_stats; + if let Some(duration) = gpu_elapsed_wall_duration { + stage_stats.gpu_elapsed_wall_duration = duration; + } + if profile_stages { + record_completed_resident_encode_gpu_stages( + &mut stage_stats, + &pending.gpu_stage_command_buffers, + ); + } + if let Some(readback) = pending.tier1_status_readback.as_ref() { + record_resident_tier1_output_usage( + &mut stage_stats, + readback, + pending.classic_gpu_token_pack_used, + )?; + } + if let Some(readback) = pending.classic_tier1_density_readback.as_ref() { + record_classic_tier1_density_counters(&mut stage_stats, readback)?; + } + if let Some(readback) = pending.classic_tier1_symbol_plan_readback.as_ref() { + record_classic_tier1_symbol_plan_counters(&mut stage_stats, readback)?; + } + if let (Some(symbol_plan), Some(pass_plan)) = ( + pending.classic_tier1_symbol_plan_readback.as_ref(), + pending.classic_tier1_pass_plan_readback.as_ref(), + ) { + compare_classic_tier1_symbol_plan_and_pass_plan_counters(symbol_plan, pass_plan)?; + } + if let Some(readback) = pending.classic_tier1_pass_plan_readback.as_ref() { + record_classic_tier1_pass_plan_counters(&mut stage_stats, readback)?; + } + if let (Some(symbol_plan), Some(token_emit)) = ( + pending.classic_tier1_symbol_plan_readback.as_ref(), + pending.classic_tier1_token_emit_readback.as_ref(), + ) { + compare_classic_tier1_symbol_plan_and_token_emit_counters(symbol_plan, token_emit)?; + } + if let Some(readback) = pending.classic_tier1_token_emit_readback.as_ref() { + record_classic_tier1_token_emit_counters(&mut stage_stats, readback)?; + profile_classic_tier1_token_pack(&mut stage_stats, readback)?; + } + if let Some(readback) = pending.classic_tier1_split_token_emit_readback.as_ref() { + validate_classic_tier1_split_token_emit_counters(readback)?; + } + if let (Some(symbol_plan), Some(split_emit)) = ( + pending.classic_tier1_symbol_plan_readback.as_ref(), + pending.classic_tier1_split_token_emit_readback.as_ref(), + ) { + compare_classic_tier1_symbol_plan_and_split_token_emit_counters(symbol_plan, split_emit)?; + } + let runtime = pending.runtime.clone(); let recyclable_private_buffers = pending.recyclable_private_buffers; - with_runtime_for_device(&pending.device, |runtime| { - recycle_private_buffers(runtime, recyclable_private_buffers); - Ok(()) - })?; + let private_recycle_started = profile_stages.then(Instant::now); + recycle_private_buffers(&runtime, recyclable_private_buffers)?; + if let Some(started) = private_recycle_started { + stage_stats.result_private_recycle_duration = started.elapsed(); + } let gpu_duration_share = gpu_duration.map(|duration| duration_share(duration, pending.capacities.len())); + let status_copy_started = profile_stages.then(Instant::now); + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. let statuses = unsafe { core::slice::from_raw_parts( pending @@ -6723,9 +8443,40 @@ pub(crate) fn wait_resident_lossless_codestream_batch( .cast::(), pending.capacities.len(), ) - }; + } + .to_vec(); + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let packet_statuses = unsafe { + core::slice::from_raw_parts( + pending + .packet_status_buffer + .contents() + .cast::(), + pending.capacities.len(), + ) + } + .to_vec(); + if let Some(started) = status_copy_started { + stage_stats.result_status_copy_duration = started.elapsed(); + } + let recyclable_shared_buffers = pending.recyclable_shared_buffers; + let shared_recycle_started = profile_stages.then(Instant::now); + recycle_shared_buffers(&runtime, recyclable_shared_buffers)?; + if let Some(started) = shared_recycle_started { + stage_stats.result_shared_recycle_duration = started.elapsed(); + } + let codestream_collect_started = profile_stages.then(Instant::now); let mut codestreams = Vec::with_capacity(pending.capacities.len()); - for (index, status) in statuses.iter().copied().enumerate() { + for (index, status) in statuses.into_iter().enumerate() { + let packet_status = packet_statuses + .get(index) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal packetization status missing for resident batch tile" + .to_string(), + })?; + if packet_status.code != J2K_ENCODE_STATUS_OK { + return Err(packet_encode_status_error(*packet_status)); + } if status.code != J2K_ENCODE_STATUS_OK { return Err(encode_status_error( pending.status_stage, @@ -6742,6 +8493,69 @@ pub(crate) fn wait_resident_lossless_codestream_batch( message: pending.capacity_error.to_string(), }); } + let packet_output_used = + usize::try_from(packet_status.data_len).map_err(|_| Error::MetalKernel { + message: "J2K Metal packet output length exceeds usize".to_string(), + })?; + let packet_payload_copy_jobs = + usize::try_from(packet_status.detail).map_err(|_| Error::MetalKernel { + message: "J2K Metal packet payload-copy count exceeds usize".to_string(), + })?; + let packet_payload_copy_bytes = + usize::try_from(packet_status.payload_copy_bytes).map_err(|_| Error::MetalKernel { + message: "J2K Metal packet payload-copy byte count exceeds usize".to_string(), + })?; + let packet_payload_copy_small_jobs = usize::try_from(packet_status.payload_copy_small_jobs) + .map_err(|_| Error::MetalKernel { + message: "J2K Metal small packet payload-copy count exceeds usize".to_string(), + })?; + let packet_payload_copy_medium_jobs = + usize::try_from(packet_status.payload_copy_medium_jobs).map_err(|_| { + Error::MetalKernel { + message: "J2K Metal medium packet payload-copy count exceeds usize".to_string(), + } + })?; + let packet_payload_copy_large_jobs = usize::try_from(packet_status.payload_copy_large_jobs) + .map_err(|_| Error::MetalKernel { + message: "J2K Metal large packet payload-copy count exceeds usize".to_string(), + })?; + let packet_payload_copy_active_stripes = + packet_payload_copy_jobs.saturating_mul(PACKET_PAYLOAD_COPY_STRIPES_PER_JOB as usize); + stage_stats.packet_output_used_bytes_total = stage_stats + .packet_output_used_bytes_total + .saturating_add(packet_output_used); + stage_stats.max_packet_output_used_bytes = stage_stats + .max_packet_output_used_bytes + .max(packet_output_used); + stage_stats.packet_payload_copy_job_count_total = stage_stats + .packet_payload_copy_job_count_total + .saturating_add(packet_payload_copy_jobs); + stage_stats.max_packet_payload_copy_jobs_used_per_tile = stage_stats + .max_packet_payload_copy_jobs_used_per_tile + .max(packet_payload_copy_jobs); + stage_stats.packet_payload_copy_bytes_total = stage_stats + .packet_payload_copy_bytes_total + .saturating_add(packet_payload_copy_bytes); + stage_stats.max_packet_payload_copy_bytes_per_tile = stage_stats + .max_packet_payload_copy_bytes_per_tile + .max(packet_payload_copy_bytes); + stage_stats.packet_payload_copy_small_job_count_total = stage_stats + .packet_payload_copy_small_job_count_total + .saturating_add(packet_payload_copy_small_jobs); + stage_stats.packet_payload_copy_medium_job_count_total = stage_stats + .packet_payload_copy_medium_job_count_total + .saturating_add(packet_payload_copy_medium_jobs); + stage_stats.packet_payload_copy_large_job_count_total = stage_stats + .packet_payload_copy_large_job_count_total + .saturating_add(packet_payload_copy_large_jobs); + stage_stats.packet_payload_copy_active_stripe_count_total = stage_stats + .packet_payload_copy_active_stripe_count_total + .saturating_add(packet_payload_copy_active_stripes); + if pending.codestream_payload_copy_dispatched { + stage_stats.codestream_payload_copy_bytes_total = stage_stats + .codestream_payload_copy_bytes_total + .saturating_add(packet_output_used); + } codestreams.push(J2kResidentLosslessCodestream { buffer: pending.buffer.clone(), byte_offset: pending.byte_offsets[index], @@ -6750,58 +8564,131 @@ pub(crate) fn wait_resident_lossless_codestream_batch( gpu_duration: gpu_duration_share, }); } - Ok(codestreams) -} - -#[cfg(target_os = "macos")] -fn duration_share(duration: Duration, count: usize) -> Duration { - if count == 0 { - return Duration::ZERO; + if let Some(started) = codestream_collect_started { + stage_stats.result_codestream_collect_duration = started.elapsed(); + } + if let Some(started) = result_harvest_started { + stage_stats.result_harvest_duration = started.elapsed(); } - let nanos = duration.as_nanos() / count as u128; - Duration::from_nanos(nanos.min(u128::from(u64::MAX)) as u64) + Ok(J2kResidentLosslessCodestreamBatchResult { + codestreams, + stage_stats, + }) } #[cfg(target_os = "macos")] -fn completed_command_buffers_gpu_duration( - retained: &[CommandBuffer], - final_buffer: &CommandBufferRef, -) -> Option { - let mut total = Duration::ZERO; - for command_buffer in retained { - total = total.saturating_add(completed_command_buffer_gpu_duration(command_buffer)?); +fn dispatch_batched_packet_payload_copy( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + dispatch: J2kBatchedPacketPayloadCopyDispatch<'_>, +) -> bool { + if dispatch.tile_count == 0 || dispatch.max_payload_copy_jobs_per_tile == 0 { + return false; } - total = total.saturating_add(completed_command_buffer_gpu_duration(final_buffer)?); - Some(total) + + let signpost = hybrid_stage_signpost(dispatch.signpost_name); + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, dispatch.label); + encoder.set_compute_pipeline_state(&runtime.packet_payload_copy_batched); + encoder.set_buffer(0, Some(dispatch.payload_buffer), 0); + encoder.set_buffer(1, Some(dispatch.packet_output_buffer), 0); + encoder.set_buffer(2, Some(dispatch.packet_job_buffer), 0); + encoder.set_buffer(3, Some(dispatch.packet_status_buffer), 0); + encoder.set_buffer(4, Some(dispatch.packet_payload_copy_job_buffer), 0); + let params = J2kPacketPayloadCopyParams { + bytes_per_thread: PACKET_PAYLOAD_COPY_BYTES_PER_STRIPE, + stripes_per_job: PACKET_PAYLOAD_COPY_STRIPES_PER_JOB, + }; + encoder.set_bytes( + 5, + size_of::() as u64, + (&raw const params).cast(), + ); + encoder.dispatch_threads( + MTLSize { + width: dispatch.max_payload_copy_jobs_per_tile, + height: dispatch.tile_count, + depth: u64::from(PACKET_PAYLOAD_COPY_STRIPES_PER_JOB), + }, + MTLSize { + width: runtime + .packet_payload_copy_batched + .thread_execution_width() + .max(1), + height: 1, + depth: 1, + }, + ); + encoder.end_encoding(); + drop(signpost); + true } #[cfg(target_os = "macos")] -fn completed_command_buffer_gpu_duration(command_buffer: &CommandBufferRef) -> Option { - let start: f64 = unsafe { - command_buffer - .send_message::<(), f64>(Sel::register("GPUStartTime"), ()) - .ok()? +fn dispatch_lossless_deinterleave( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + job: J2kLosslessDevicePrepareJob<'_>, + plane0: &Buffer, + plane1: &Buffer, + plane2: &Buffer, +) -> Result<(), Error> { + let input_byte_offset = + u64::try_from(job.input_byte_offset).map_err(|_| Error::MetalKernel { + message: "J2K Metal resident encode input offset exceeds u64".to_string(), + })?; + let src_stride = u32::try_from(job.input_pitch_bytes).map_err(|_| Error::MetalKernel { + message: "J2K Metal resident encode input pitch exceeds u32".to_string(), + })?; + let sample_offset = if job.bit_depth == 0 || job.bit_depth > 16 { + return Err(Error::MetalKernel { + message: "J2K Metal resident encode bit depth must be 1-16".to_string(), + }); + } else { + 1u32 << (u32::from(job.bit_depth) - 1) }; - let end: f64 = unsafe { - command_buffer - .send_message::<(), f64>(Sel::register("GPUEndTime"), ()) - .ok()? + let params = J2kLosslessDeinterleaveParams { + src_width: job.input_width, + src_height: job.input_height, + src_stride, + dst_width: job.output_width, + dst_height: job.output_height, + components: u32::from(job.components), + bytes_per_sample: u32::from(job.bytes_per_sample), + sample_offset, + signed_samples: 0, }; - if start.is_finite() && end.is_finite() && end > start { - Some(Duration::from_secs_f64(end - start)) - } else { - None - } + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K coefficient prep deinterleave"); + encoder.set_compute_pipeline_state(&runtime.lossless_deinterleave_to_planes); + encoder.set_buffer(0, Some(job.input), input_byte_offset); + encoder.set_buffer(1, Some(plane0), 0); + encoder.set_buffer(2, Some(plane1), 0); + encoder.set_buffer(3, Some(plane2), 0); + encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const params).cast(), + ); + encoder.set_buffer(5, Some(plane2), 0); + dispatch_2d_pipeline( + encoder, + &runtime.lossless_deinterleave_to_planes, + (job.output_width, job.output_height), + ); + encoder.end_encoding(); + Ok(()) } #[cfg(target_os = "macos")] -fn dispatch_lossless_deinterleave( +fn dispatch_lossless_deinterleave_rct_rgb8( runtime: &MetalRuntime, command_buffer: &CommandBufferRef, job: J2kLosslessDevicePrepareJob<'_>, plane0: &Buffer, plane1: &Buffer, plane2: &Buffer, + status_buffer: &Buffer, ) -> Result<(), Error> { let input_byte_offset = u64::try_from(job.input_byte_offset).map_err(|_| Error::MetalKernel { @@ -6826,10 +8713,11 @@ fn dispatch_lossless_deinterleave( components: u32::from(job.components), bytes_per_sample: u32::from(job.bytes_per_sample), sample_offset, + signed_samples: 0, }; let encoder = command_buffer.new_compute_command_encoder(); - label_compute_encoder(encoder, "J2K input deinterleave"); - encoder.set_compute_pipeline_state(&runtime.lossless_deinterleave_to_planes); + label_compute_encoder(encoder, "J2K coefficient prep deinterleave + RCT"); + encoder.set_compute_pipeline_state(&runtime.lossless_deinterleave_rct_rgb8_to_planes); encoder.set_buffer(0, Some(job.input), input_byte_offset); encoder.set_buffer(1, Some(plane0), 0); encoder.set_buffer(2, Some(plane1), 0); @@ -6839,31 +8727,23 @@ fn dispatch_lossless_deinterleave( size_of::() as u64, (&raw const params).cast(), ); - let width = runtime - .lossless_deinterleave_to_planes - .thread_execution_width() - .max(1); - let max_threads = runtime - .lossless_deinterleave_to_planes - .max_total_threads_per_threadgroup() - .max(width); - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(job.output_width), - height: u64::from(job.output_height), - depth: 1, - }, - MTLSize { - width, - height, - depth: 1, - }, + encoder.set_buffer(5, Some(status_buffer), 0); + dispatch_2d_pipeline( + encoder, + &runtime.lossless_deinterleave_rct_rgb8_to_planes, + (job.output_width, job.output_height), ); encoder.end_encoding(); + #[cfg(test)] + test_counters::record_lossless_deinterleave_rct_fused_dispatch(); Ok(()) } +#[cfg(target_os = "macos")] +fn lossless_deinterleave_rct_rgb8_supported(job: J2kLosslessDevicePrepareJob<'_>) -> bool { + job.components == 3 && job.bytes_per_sample == 1 +} + #[cfg(target_os = "macos")] fn dispatch_forward_rct_on_buffers( runtime: &MetalRuntime, @@ -6885,52 +8765,193 @@ fn dispatch_forward_rct_on_buffers( _reserved1: 0, _reserved2: 0, }; - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.forward_rct); - encoder.set_buffer(0, Some(plane0), 0); - encoder.set_buffer(1, Some(plane1), 0); - encoder.set_buffer(2, Some(plane2), 0); - encoder.set_bytes( - 3, - size_of::() as u64, - (&raw const params).cast(), - ); - encoder.set_buffer(4, Some(status_buffer), 0); - let width = runtime - .forward_rct - .thread_execution_width() - .max(1) - .min(len as u64); - encoder.dispatch_threads( - MTLSize { - width: len as u64, - height: 1, - depth: 1, - }, - MTLSize { - width, - height: 1, - depth: 1, - }, - ); - encoder.end_encoding(); - Ok(()) + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K coefficient prep RCT"); + encoder.set_compute_pipeline_state(&runtime.forward_rct); + encoder.set_buffer(0, Some(plane0), 0); + encoder.set_buffer(1, Some(plane1), 0); + encoder.set_buffer(2, Some(plane2), 0); + encoder.set_bytes( + 3, + size_of::() as u64, + (&raw const params).cast(), + ); + encoder.set_buffer(4, Some(status_buffer), 0); + let width = runtime + .forward_rct + .thread_execution_width() + .max(1) + .min(len as u64); + encoder.dispatch_threads( + MTLSize { + width: len as u64, + height: 1, + depth: 1, + }, + MTLSize { + width, + height: 1, + depth: 1, + }, + ); + encoder.end_encoding(); + Ok(()) +} + +#[cfg(target_os = "macos")] +fn dispatch_forward_dwt53_on_buffers( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + input: &Buffer, + scratch: &Buffer, + width: u32, + height: u32, + num_levels: u8, +) -> Buffer { + let mut current_width = width; + let mut current_height = height; + let mut levels_run = 0u8; + let mut active_is_input = true; + + while levels_run < num_levels && (current_width >= 2 || current_height >= 2) { + let low_width = current_width.div_ceil(2); + let low_height = current_height.div_ceil(2); + let params = J2kForwardDwt53Params { + full_width: width, + current_width, + current_height, + low_width, + low_height, + }; + + if current_height >= 2 { + let (src, dst) = active_forward_dwt53_buffers(input, scratch, active_is_input); + dispatch_forward_dwt53_pass( + &runtime.fdwt53_vertical, + command_buffer, + src, + dst, + params, + "J2K coefficient prep DWT 5/3 vertical", + ); + active_is_input = !active_is_input; + } + if current_width >= 2 { + let (src, dst) = active_forward_dwt53_buffers(input, scratch, active_is_input); + dispatch_forward_dwt53_pass( + &runtime.fdwt53_horizontal, + command_buffer, + src, + dst, + params, + "J2K coefficient prep DWT 5/3 horizontal", + ); + active_is_input = !active_is_input; + } + + current_width = low_width; + current_height = low_height; + levels_run = levels_run.saturating_add(1); + } + + if active_is_input { + input.to_owned() + } else { + scratch.to_owned() + } +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn dispatch_forward_dwt53_components_on_buffers( + runtime: &MetalRuntime, + command_buffer: &CommandBufferRef, + plane_buffers: &[Buffer], + scratch_buffers: &[Buffer], + width: u32, + height: u32, + num_levels: u8, + component_count: usize, +) -> Vec { + let mut current_width = width; + let mut current_height = height; + let mut levels_run = 0u8; + let mut active_is_input = true; + let component_count_u32 = component_count as u32; + + while levels_run < num_levels && (current_width >= 2 || current_height >= 2) { + let low_width = current_width.div_ceil(2); + let low_height = current_height.div_ceil(2); + let params = J2kForwardDwt53BatchedParams { + full_width: width, + current_width, + current_height, + low_width, + low_height, + component_count: component_count_u32, + }; + + if current_height >= 2 { + let (inputs, outputs) = if active_is_input { + (plane_buffers, scratch_buffers) + } else { + (scratch_buffers, plane_buffers) + }; + dispatch_forward_dwt53_batched_pass( + &runtime.fdwt53_vertical_batched, + command_buffer, + inputs, + outputs, + params, + "J2K coefficient prep DWT 5/3 vertical", + ); + active_is_input = !active_is_input; + } + if current_width >= 2 { + let (inputs, outputs) = if active_is_input { + (plane_buffers, scratch_buffers) + } else { + (scratch_buffers, plane_buffers) + }; + dispatch_forward_dwt53_batched_pass( + &runtime.fdwt53_horizontal_batched, + command_buffer, + inputs, + outputs, + params, + "J2K coefficient prep DWT 5/3 horizontal", + ); + active_is_input = !active_is_input; + } + + current_width = low_width; + current_height = low_height; + levels_run = levels_run.saturating_add(1); + } + + let active_buffers = if active_is_input { + plane_buffers + } else { + scratch_buffers + }; + active_buffers[..component_count].to_vec() } #[cfg(target_os = "macos")] -fn dispatch_forward_dwt53_on_buffers( +fn dispatch_forward_dwt53_on_buffers_split_profile( runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, input: &Buffer, scratch: &Buffer, width: u32, height: u32, num_levels: u8, -) -> Buffer { +) -> (Buffer, Vec, Vec) { let mut current_width = width; let mut current_height = height; let mut levels_run = 0u8; let mut active_is_input = true; + let mut vertical_command_buffers = Vec::new(); + let mut horizontal_command_buffers = Vec::new(); while levels_run < num_levels && (current_width >= 2 || current_height >= 2) { let low_width = current_width.div_ceil(2); @@ -6944,19 +8965,39 @@ fn dispatch_forward_dwt53_on_buffers( }; if current_height >= 2 { + let command_buffer = new_resident_encode_command_buffer( + runtime, + "j2k coefficient prep DWT 5/3 vertical", + ); let (src, dst) = active_forward_dwt53_buffers(input, scratch, active_is_input); - dispatch_forward_dwt53_pass(&runtime.fdwt53_vertical, command_buffer, src, dst, params); + dispatch_forward_dwt53_pass( + &runtime.fdwt53_vertical, + &command_buffer, + src, + dst, + params, + "J2K coefficient prep DWT 5/3 vertical", + ); + command_buffer.commit(); + vertical_command_buffers.push(command_buffer); active_is_input = !active_is_input; } if current_width >= 2 { + let command_buffer = new_resident_encode_command_buffer( + runtime, + "j2k coefficient prep DWT 5/3 horizontal", + ); let (src, dst) = active_forward_dwt53_buffers(input, scratch, active_is_input); dispatch_forward_dwt53_pass( &runtime.fdwt53_horizontal, - command_buffer, + &command_buffer, src, dst, params, + "J2K coefficient prep DWT 5/3 horizontal", ); + command_buffer.commit(); + horizontal_command_buffers.push(command_buffer); active_is_input = !active_is_input; } @@ -6965,11 +9006,104 @@ fn dispatch_forward_dwt53_on_buffers( levels_run = levels_run.saturating_add(1); } - if active_is_input { + let active = if active_is_input { input.to_owned() } else { scratch.to_owned() + }; + (active, vertical_command_buffers, horizontal_command_buffers) +} + +#[cfg(target_os = "macos")] +fn dispatch_forward_dwt53_components_split_profile( + runtime: &MetalRuntime, + plane_buffers: &[Buffer], + scratch_buffers: &[Buffer], + width: u32, + height: u32, + num_levels: u8, + component_count: usize, +) -> (Vec, Vec, Vec) { + let mut current_width = width; + let mut current_height = height; + let mut levels_run = 0u8; + let mut active_is_input = true; + let mut vertical_command_buffers = Vec::new(); + let mut horizontal_command_buffers = Vec::new(); + let component_count_u32 = component_count as u32; + + while levels_run < num_levels && (current_width >= 2 || current_height >= 2) { + let low_width = current_width.div_ceil(2); + let low_height = current_height.div_ceil(2); + let params = J2kForwardDwt53BatchedParams { + full_width: width, + current_width, + current_height, + low_width, + low_height, + component_count: component_count_u32, + }; + + if current_height >= 2 { + let command_buffer = new_resident_encode_command_buffer( + runtime, + "j2k coefficient prep DWT 5/3 vertical", + ); + let (inputs, outputs) = if active_is_input { + (plane_buffers, scratch_buffers) + } else { + (scratch_buffers, plane_buffers) + }; + dispatch_forward_dwt53_batched_pass( + &runtime.fdwt53_vertical_batched, + &command_buffer, + inputs, + outputs, + params, + "J2K coefficient prep DWT 5/3 vertical", + ); + command_buffer.commit(); + vertical_command_buffers.push(command_buffer); + active_is_input = !active_is_input; + } + if current_width >= 2 { + let command_buffer = new_resident_encode_command_buffer( + runtime, + "j2k coefficient prep DWT 5/3 horizontal", + ); + let (inputs, outputs) = if active_is_input { + (plane_buffers, scratch_buffers) + } else { + (scratch_buffers, plane_buffers) + }; + dispatch_forward_dwt53_batched_pass( + &runtime.fdwt53_horizontal_batched, + &command_buffer, + inputs, + outputs, + params, + "J2K coefficient prep DWT 5/3 horizontal", + ); + command_buffer.commit(); + horizontal_command_buffers.push(command_buffer); + active_is_input = !active_is_input; + } + + current_width = low_width; + current_height = low_height; + levels_run = levels_run.saturating_add(1); } + + let active_buffers = if active_is_input { + plane_buffers + } else { + scratch_buffers + }; + ( + active_buffers[..component_count].to_vec(), + vertical_command_buffers, + horizontal_command_buffers, + ) } #[cfg(target_os = "macos")] @@ -6996,7 +9130,7 @@ fn dispatch_lossless_extract_coefficients( .max() .unwrap_or(1); let encoder = command_buffer.new_compute_command_encoder(); - label_compute_encoder(encoder, "J2K coefficient prep"); + label_compute_encoder(encoder, "J2K coefficient prep extract"); encoder.set_compute_pipeline_state(&runtime.lossless_extract_coefficients); encoder.set_buffer(0, planes.first().map(|buffer| &**buffer), 0); encoder.set_buffer( @@ -7018,26 +9152,10 @@ fn dispatch_lossless_extract_coefficients( encoder.set_buffer(3, Some(coefficient_buffer), 0); encoder.set_buffer(4, Some(&coefficient_job_buffer), 0); encoder.set_bytes(5, size_of::() as u64, (&raw const job_count).cast()); - let width = runtime - .lossless_extract_coefficients - .thread_execution_width() - .max(1); - let max_threads = runtime - .lossless_extract_coefficients - .max_total_threads_per_threadgroup() - .max(width); - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(max_block_width), - height: u64::from(max_block_height), - depth: u64::from(job_count), - }, - MTLSize { - width, - height, - depth: 1, - }, + dispatch_3d_pipeline( + encoder, + &runtime.lossless_extract_coefficients, + (max_block_width, max_block_height, job_count), ); encoder.end_encoding(); let _ = output_width; @@ -7066,11 +9184,6 @@ fn lossless_prepare_sizes( reason: "J2K Metal resident encode supports 8-bit or 16-bit samples", }); } - if job.num_decomposition_levels > 1 { - return Err(Error::UnsupportedMetalRequest { - reason: "J2K Metal resident encode currently supports up to one DWT level", - }); - } let plane_len = (job.output_width as usize) .checked_mul(job.output_height as usize) .ok_or_else(|| Error::MetalKernel { @@ -7104,7 +9217,7 @@ pub(crate) fn prepare_lossless_device_code_blocks( ) -> Result { let sizes = lossless_prepare_sizes(job)?; - with_runtime_for_device(&session.device, |runtime| { + with_runtime_for_session(session, |runtime| { let mut plane_buffers = Vec::with_capacity(3); let mut scratch_buffers = Vec::with_capacity(usize::from(job.components)); for _ in 0..3 { @@ -7131,15 +9244,27 @@ pub(crate) fn prepare_lossless_device_code_blocks( ); let command_buffer = runtime.queue.new_command_buffer(); - dispatch_lossless_deinterleave( - runtime, - command_buffer, - job, - &plane_buffers[0], - &plane_buffers[1], - &plane_buffers[2], - )?; - if job.components == 3 { + if lossless_deinterleave_rct_rgb8_supported(job) { + dispatch_lossless_deinterleave_rct_rgb8( + runtime, + command_buffer, + job, + &plane_buffers[0], + &plane_buffers[1], + &plane_buffers[2], + &status_buffer, + )?; + } else { + dispatch_lossless_deinterleave( + runtime, + command_buffer, + job, + &plane_buffers[0], + &plane_buffers[1], + &plane_buffers[2], + )?; + } + if job.components == 3 && !lossless_deinterleave_rct_rgb8_supported(job) { dispatch_forward_rct_on_buffers( runtime, command_buffer, @@ -7203,6 +9328,11 @@ pub(crate) fn prepare_lossless_device_code_blocks( code_blocks, recyclable_private_buffers: Vec::new(), _prepare_command_buffer: command_buffer.to_owned(), + _prepare_deinterleave_rct_command_buffer: None, + _prepare_dwt53_command_buffer: None, + _prepare_dwt53_vertical_command_buffers: Vec::new(), + _prepare_dwt53_horizontal_command_buffers: Vec::new(), + _prepare_coefficient_extract_command_buffer: None, _deinterleave_status_buffer: status_buffer, _plane_buffers: plane_buffers, _scratch_buffers: scratch_buffers, @@ -7239,14 +9369,19 @@ pub(crate) fn prepare_lossless_device_code_blocks_batch( sizes.push(item_sizes); } - with_runtime_for_device(&session.device, |runtime| { + with_runtime_for_session(session, |runtime| { let mut shared_recyclable_private_buffers = Vec::new(); let coefficient_buffer = take_recyclable_private_buffer( runtime, total_coefficient_bytes.max(1), &mut shared_recyclable_private_buffers, - ); - let command_buffer = runtime.queue.new_command_buffer(); + )?; + let split_prepare_command_buffers = metal_profile_coefficient_prep_split_commands_enabled(); + let shared_command_buffer = if split_prepare_command_buffers { + None + } else { + Some(runtime.queue.new_command_buffer().to_owned()) + }; let mut prepared = Vec::with_capacity(items.len()); for ((item, item_sizes), coefficient_byte_offset) in @@ -7264,14 +9399,14 @@ pub(crate) fn prepare_lossless_device_code_blocks_batch( runtime, item_sizes.plane_bytes, &mut recyclable_private_buffers, - )); + )?); } for _ in 0..job.components { scratch_buffers.push(take_recyclable_private_buffer( runtime, item_sizes.plane_bytes, &mut recyclable_private_buffers, - )); + )?); } let deinterleave_status = J2kMctStatus::default(); @@ -7281,24 +9416,49 @@ pub(crate) fn prepare_lossless_device_code_blocks_batch( MTLResourceOptions::StorageModeShared, ); - dispatch_lossless_deinterleave( - runtime, - command_buffer, - job, - &plane_buffers[0], - &plane_buffers[1], - &plane_buffers[2], - ) + let mut prepare_deinterleave_rct_command_buffer = None; + let prepare_dwt53_command_buffer = None; + let mut prepare_dwt53_vertical_command_buffers = Vec::new(); + let mut prepare_dwt53_horizontal_command_buffers = Vec::new(); + let mut prepare_coefficient_extract_command_buffer = None; + let deinterleave_command_buffer = if split_prepare_command_buffers { + new_resident_encode_command_buffer(runtime, "j2k coefficient prep deinterleave rct") + } else { + shared_command_buffer + .as_ref() + .expect("shared coefficient prep command buffer exists") + .clone() + }; + if lossless_deinterleave_rct_rgb8_supported(job) { + dispatch_lossless_deinterleave_rct_rgb8( + runtime, + &deinterleave_command_buffer, + job, + &plane_buffers[0], + &plane_buffers[1], + &plane_buffers[2], + &status_buffer, + ) + } else { + dispatch_lossless_deinterleave( + runtime, + &deinterleave_command_buffer, + job, + &plane_buffers[0], + &plane_buffers[1], + &plane_buffers[2], + ) + } .map_err(|err| Error::MetalKernel { message: format!( "J2K Metal resident batch coefficient prep failed at tile {}: {err}", item.tile_index ), })?; - if job.components == 3 { + if job.components == 3 && !lossless_deinterleave_rct_rgb8_supported(job) { dispatch_forward_rct_on_buffers( runtime, - command_buffer, + &deinterleave_command_buffer, &plane_buffers[0], &plane_buffers[1], &plane_buffers[2], @@ -7312,21 +9472,88 @@ pub(crate) fn prepare_lossless_device_code_blocks_batch( ), })?; } + if split_prepare_command_buffers { + deinterleave_command_buffer.commit(); + prepare_deinterleave_rct_command_buffer = Some(deinterleave_command_buffer); + } let mut active_planes = Vec::with_capacity(usize::from(job.components)); - for component in 0..usize::from(job.components) { - if job.num_decomposition_levels == 0 { - active_planes.push(plane_buffers[component].clone()); + if job.num_decomposition_levels == 0 { + active_planes.extend( + plane_buffers + .iter() + .take(usize::from(job.components)) + .cloned(), + ); + } else if split_prepare_command_buffers { + let component_count = usize::from(job.components); + if component_count > 1 { + let ( + mut component_active_planes, + mut vertical_command_buffers, + mut horizontal_command_buffers, + ) = dispatch_forward_dwt53_components_split_profile( + runtime, + &plane_buffers, + &scratch_buffers, + job.output_width, + job.output_height, + job.num_decomposition_levels, + component_count, + ); + active_planes.append(&mut component_active_planes); + prepare_dwt53_vertical_command_buffers.append(&mut vertical_command_buffers); + prepare_dwt53_horizontal_command_buffers + .append(&mut horizontal_command_buffers); } else { - active_planes.push(dispatch_forward_dwt53_on_buffers( + for component in 0..component_count { + let ( + active_plane, + mut vertical_command_buffers, + mut horizontal_command_buffers, + ) = dispatch_forward_dwt53_on_buffers_split_profile( + runtime, + &plane_buffers[component], + &scratch_buffers[component], + job.output_width, + job.output_height, + job.num_decomposition_levels, + ); + active_planes.push(active_plane); + prepare_dwt53_vertical_command_buffers + .append(&mut vertical_command_buffers); + prepare_dwt53_horizontal_command_buffers + .append(&mut horizontal_command_buffers); + } + } + } else { + let dwt_command_buffer_ref = shared_command_buffer + .as_ref() + .expect("shared coefficient prep command buffer exists"); + let component_count = usize::from(job.components); + if component_count > 1 { + active_planes = dispatch_forward_dwt53_components_on_buffers( runtime, - command_buffer, - &plane_buffers[component], - &scratch_buffers[component], + dwt_command_buffer_ref, + &plane_buffers, + &scratch_buffers, job.output_width, job.output_height, job.num_decomposition_levels, - )); + component_count, + ); + } else { + for component in 0..component_count { + active_planes.push(dispatch_forward_dwt53_on_buffers( + runtime, + dwt_command_buffer_ref, + &plane_buffers[component], + &scratch_buffers[component], + job.output_width, + job.output_height, + job.num_decomposition_levels, + )); + } } } while active_planes.len() < 3 { @@ -7372,9 +9599,17 @@ pub(crate) fn prepare_lossless_device_code_blocks_batch( }) }) .collect::, Error>>()?; + let extract_command_buffer = if split_prepare_command_buffers { + new_resident_encode_command_buffer(runtime, "j2k coefficient prep extract") + } else { + shared_command_buffer + .as_ref() + .expect("shared coefficient prep command buffer exists") + .clone() + }; let coefficient_job_buffer = dispatch_lossless_extract_coefficients( runtime, - command_buffer, + &extract_command_buffer, &active_planes, &coefficient_buffer, &coefficient_jobs, @@ -7386,29 +9621,120 @@ pub(crate) fn prepare_lossless_device_code_blocks_batch( item.tile_index ), })?; + let prepare_command_buffer = extract_command_buffer.clone(); + if split_prepare_command_buffers { + extract_command_buffer.commit(); + prepare_coefficient_extract_command_buffer = Some(extract_command_buffer); + } + + prepared.push(J2kPreparedLosslessDeviceCodeBlocks { + coefficient_buffer: coefficient_buffer.clone(), + coefficient_byte_offset, + coefficient_byte_len: item_sizes.coefficient_bytes, + coefficient_buffer_is_batch_shared: true, + code_blocks: item.code_blocks, + recyclable_private_buffers, + _prepare_command_buffer: prepare_command_buffer, + _prepare_deinterleave_rct_command_buffer: prepare_deinterleave_rct_command_buffer, + _prepare_dwt53_command_buffer: prepare_dwt53_command_buffer, + _prepare_dwt53_vertical_command_buffers: prepare_dwt53_vertical_command_buffers, + _prepare_dwt53_horizontal_command_buffers: prepare_dwt53_horizontal_command_buffers, + _prepare_coefficient_extract_command_buffer: + prepare_coefficient_extract_command_buffer, + _deinterleave_status_buffer: status_buffer, + _plane_buffers: plane_buffers, + _scratch_buffers: scratch_buffers, + _coefficient_job_buffer: coefficient_job_buffer, + }); + } + + if let Some(command_buffer) = shared_command_buffer { + command_buffer.commit(); + } + Ok(prepared) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn encode_forward_rct( + plane0: &mut [f32], + plane1: &mut [f32], + plane2: &mut [f32], +) -> Result<(), Error> { + with_runtime(|runtime| { + let len = plane0.len(); + if len == 0 { + return Ok(()); + } + if plane1.len() != len || plane2.len() != len { + return Err(Error::MetalKernel { + message: "J2K Metal forward RCT plane lengths must match".to_string(), + }); + } + + let params = J2kForwardRctParams { + _len: u32::try_from(len).map_err(|_| Error::MetalKernel { + message: "J2K Metal forward RCT plane length exceeds u32".to_string(), + })?, + _reserved0: 0, + _reserved1: 0, + _reserved2: 0, + }; + let plane0_buffer = borrow_mut_slice_buffer(&runtime.device, plane0); + let plane1_buffer = borrow_mut_slice_buffer(&runtime.device, plane1); + let plane2_buffer = borrow_mut_slice_buffer(&runtime.device, plane2); + let status = J2kMctStatus::default(); + let status_buffer = runtime.device.new_buffer_with_data( + (&raw const status).cast(), + size_of::() as u64, + MTLResourceOptions::StorageModeShared, + ); + + let command_buffer = runtime.queue.new_command_buffer(); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.forward_rct); + encoder.set_buffer(0, Some(&plane0_buffer), 0); + encoder.set_buffer(1, Some(&plane1_buffer), 0); + encoder.set_buffer(2, Some(&plane2_buffer), 0); + encoder.set_bytes( + 3, + size_of::() as u64, + (&raw const params).cast(), + ); + encoder.set_buffer(4, Some(&status_buffer), 0); + let width = runtime + .forward_rct + .thread_execution_width() + .max(1) + .min(len as u64); + encoder.dispatch_threads( + MTLSize { + width: len as u64, + height: 1, + depth: 1, + }, + MTLSize { + width, + height: 1, + depth: 1, + }, + ); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); - prepared.push(J2kPreparedLosslessDeviceCodeBlocks { - coefficient_buffer: coefficient_buffer.clone(), - coefficient_byte_offset, - coefficient_byte_len: item_sizes.coefficient_bytes, - coefficient_buffer_is_batch_shared: true, - code_blocks: item.code_blocks, - recyclable_private_buffers, - _prepare_command_buffer: command_buffer.to_owned(), - _deinterleave_status_buffer: status_buffer, - _plane_buffers: plane_buffers, - _scratch_buffers: scratch_buffers, - _coefficient_job_buffer: coefficient_job_buffer, - }); + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let status = unsafe { status_buffer.contents().cast::().read() }; + if status.code != J2K_MCT_STATUS_OK { + return Err(decode_mct_status_error(status)); } - command_buffer.commit(); - Ok(prepared) + Ok(()) }) } #[cfg(target_os = "macos")] -pub(crate) fn encode_forward_rct( +pub(crate) fn encode_forward_ict( plane0: &mut [f32], plane1: &mut [f32], plane2: &mut [f32], @@ -7419,14 +9745,14 @@ pub(crate) fn encode_forward_rct( return Ok(()); } if plane1.len() != len || plane2.len() != len { - return Err(Error::MetalKernel { - message: "J2K Metal forward RCT plane lengths must match".to_string(), + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal forward ICT plane lengths must match", }); } - let params = J2kForwardRctParams { - _len: u32::try_from(len).map_err(|_| Error::MetalKernel { - message: "J2K Metal forward RCT plane length exceeds u32".to_string(), + let params = J2kForwardIctParams { + _len: u32::try_from(len).map_err(|_| Error::UnsupportedMetalRequest { + reason: "J2K Metal forward ICT plane length exceeds u32", })?, _reserved0: 0, _reserved1: 0, @@ -7444,18 +9770,18 @@ pub(crate) fn encode_forward_rct( let command_buffer = runtime.queue.new_command_buffer(); let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.forward_rct); + encoder.set_compute_pipeline_state(&runtime.forward_ict); encoder.set_buffer(0, Some(&plane0_buffer), 0); encoder.set_buffer(1, Some(&plane1_buffer), 0); encoder.set_buffer(2, Some(&plane2_buffer), 0); encoder.set_bytes( 3, - size_of::() as u64, + size_of::() as u64, (&raw const params).cast(), ); encoder.set_buffer(4, Some(&status_buffer), 0); let width = runtime - .forward_rct + .forward_ict .thread_execution_width() .max(1) .min(len as u64); @@ -7475,6 +9801,7 @@ pub(crate) fn encode_forward_rct( command_buffer.commit(); command_buffer.wait_until_completed(); + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. let status = unsafe { status_buffer.contents().cast::().read() }; if status.code != J2K_MCT_STATUS_OK { return Err(decode_mct_status_error(status)); @@ -7484,6 +9811,82 @@ pub(crate) fn encode_forward_rct( }) } +#[cfg(target_os = "macos")] +fn validate_encode_quantize_subband_job(job: J2kQuantizeSubbandJob<'_>) -> Result<(), Error> { + if job.step_exponent > 31 { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal encode quantize_subband supports step exponents <= 31", + }); + } + if job.step_mantissa > 2047 { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal encode quantize_subband supports step mantissas <= 2047", + }); + } + if job.range_bits == 0 || job.range_bits > 31 { + return Err(Error::UnsupportedMetalRequest { + reason: "J2K Metal encode quantize_subband supports range bits 1-31", + }); + } + Ok(()) +} + +#[cfg(target_os = "macos")] +pub(crate) fn encode_quantize_subband(job: J2kQuantizeSubbandJob<'_>) -> Result, Error> { + validate_encode_quantize_subband_job(job)?; + let len = job.coefficients.len(); + if len == 0 { + return Ok(Vec::new()); + } + let len_u32 = u32::try_from(len).map_err(|_| Error::UnsupportedMetalRequest { + reason: "J2K Metal encode quantize_subband coefficient count exceeds u32", + })?; + let output_bytes = len + .checked_mul(size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal encode quantize_subband output length overflow".to_string(), + })?; + + with_runtime(|runtime| { + let input_buffer = copied_slice_buffer(&runtime.device, job.coefficients); + let output_buffer = runtime + .device + .new_buffer(output_bytes as u64, MTLResourceOptions::StorageModeShared); + let params = J2kQuantizeSubbandParams { + _len: len_u32, + _step_exponent: u32::from(job.step_exponent), + _step_mantissa: u32::from(job.step_mantissa), + _range_bits: u32::from(job.range_bits), + _reversible: u32::from(job.reversible), + _reserved0: 0, + _reserved1: 0, + _reserved2: 0, + }; + + let command_buffer = runtime.queue.new_command_buffer(); + label_command_buffer(command_buffer, "j2k encode-stage quantize_subband"); + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K encode-stage quantize_subband"); + encoder.set_compute_pipeline_state(&runtime.quantize_subband); + encoder.set_buffer(0, Some(&input_buffer), 0); + encoder.set_buffer(1, Some(&output_buffer), 0); + encoder.set_bytes( + 2, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_1d_pipeline(encoder, &runtime.quantize_subband, u64::from(len_u32)); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let coefficients = + unsafe { core::slice::from_raw_parts(output_buffer.contents().cast::(), len) }; + Ok(coefficients.to_vec()) + }) +} + #[cfg(target_os = "macos")] pub(crate) fn decode_inverse_mct(job: J2kInverseMctJob<'_>) -> Result, Error> { let J2kInverseMctJob { @@ -7562,16 +9965,20 @@ pub(crate) fn decode_inverse_mct(job: J2kInverseMctJob<'_>) -> Result().read() }; if status.code != J2K_MCT_STATUS_OK { return Err(decode_mct_status_error(status)); } let plane0_host = + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. unsafe { core::slice::from_raw_parts(plane0_buffer.contents().cast::(), len) }; let plane1_host = + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. unsafe { core::slice::from_raw_parts(plane1_buffer.contents().cast::(), len) }; let plane2_host = + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. unsafe { core::slice::from_raw_parts(plane2_buffer.contents().cast::(), len) }; for (dst, sample) in plane0.iter_mut().zip(plane0_host.iter().copied()) { *dst = sample - addend0; @@ -7621,6 +10028,7 @@ fn dispatch_inverse_mct_buffers_in_command_buffer( MTLResourceOptions::StorageModeShared, ); + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE); let encoder = command_buffer.new_compute_command_encoder(); encoder.set_compute_pipeline_state(&runtime.inverse_mct); encoder.set_buffer(0, Some(planes[0]), 0); @@ -7738,24 +10146,7 @@ pub(crate) fn decode_store_component_and_capture( size_of::() as u64, (&raw const params).cast(), ); - let width = runtime.store_component.thread_execution_width().max(1); - let max_threads = runtime - .store_component - .max_total_threads_per_threadgroup() - .max(width); - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(copy_width), - height: u64::from(copy_height), - depth: 1, - }, - MTLSize { - width, - height, - depth: 1, - }, - ); + dispatch_2d_pipeline(encoder, &runtime.store_component, (copy_width, copy_height)); encoder.end_encoding(); command_buffer.commit(); command_buffer.wait_until_completed(); @@ -7773,7 +10164,9 @@ fn dispatch_store_component_buffer_in_command_buffer_with_offsets( output_offset_bytes: usize, params: J2kStoreParams, ) { + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_STORE_COMMAND_ENCODE); let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K decode hybrid component store"); dispatch_store_component_buffer_in_encoder_with_offsets( runtime, encoder, @@ -7804,23 +10197,10 @@ fn dispatch_store_component_buffer_in_encoder_with_offsets( size_of::() as u64, (&raw const params).cast(), ); - let width = runtime.store_component.thread_execution_width().max(1); - let max_threads = runtime - .store_component - .max_total_threads_per_threadgroup() - .max(width); - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(params.copy_width), - height: u64::from(params.copy_height), - depth: 1, - }, - MTLSize { - width, - height, - depth: 1, - }, + dispatch_2d_pipeline( + encoder, + &runtime.store_component, + (params.copy_width, params.copy_height), ); } @@ -7828,38 +10208,25 @@ fn dispatch_store_component_repeated_in_command_buffer( runtime: &MetalRuntime, command_buffer: &CommandBufferRef, input: &Buffer, + input_offset_bytes: usize, output: &Buffer, params: J2kRepeatedStoreParams, ) { + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_STORE_COMMAND_ENCODE); let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K decode hybrid repeated component store"); encoder.set_compute_pipeline_state(&runtime.store_component_repeated); - encoder.set_buffer(0, Some(input), 0); + encoder.set_buffer(0, Some(input), input_offset_bytes as u64); encoder.set_buffer(1, Some(output), 0); encoder.set_bytes( 2, size_of::() as u64, (&raw const params).cast(), ); - let width = runtime - .store_component_repeated - .thread_execution_width() - .max(1); - let max_threads = runtime - .store_component_repeated - .max_total_threads_per_threadgroup() - .max(width); - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(params.copy_width), - height: u64::from(params.copy_height), - depth: u64::from(params.batch_count), - }, - MTLSize { - width, - height, - depth: 1, - }, + dispatch_3d_pipeline( + encoder, + &runtime.store_component_repeated, + (params.copy_width, params.copy_height, params.batch_count), ); encoder.end_encoding(); } @@ -7945,18 +10312,10 @@ fn encode_repeated_gray_store_to_surfaces_in_command_buffer( }, ); } else { - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(params.copy_width), - height: u64::from(params.copy_height), - depth: u64::from(params.batch_count), - }, - MTLSize { - width, - height, - depth: 1, - }, + dispatch_3d_pipeline( + encoder, + pipeline, + (params.copy_width, params.copy_height, params.batch_count), ); } encoder.end_encoding(); @@ -8006,21 +10365,7 @@ fn encode_gray_store_to_surface_in_encoder( size_of::() as u64, (&raw const params).cast(), ); - let width = pipeline.thread_execution_width().max(1); - let max_threads = pipeline.max_total_threads_per_threadgroup().max(width); - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(params.copy_width), - height: u64::from(params.copy_height), - depth: 1, - }, - MTLSize { - width, - height, - depth: 1, - }, - ); + dispatch_2d_pipeline(encoder, pipeline, (params.copy_width, params.copy_height)); Ok(Surface::from_metal_buffer(out_buffer, dims, fmt)) } @@ -8083,24 +10428,10 @@ pub(crate) fn decode_reversible53_single_decomposition_idwt( size_of::() as u64, (&raw const params).cast(), ); - let interleave_width = runtime.idwt_interleave.thread_execution_width().max(1); - let interleave_height = (runtime - .idwt_interleave - .max_total_threads_per_threadgroup() - .max(interleave_width) - / interleave_width) - .max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(params.width), - height: u64::from(params.height), - depth: 1, - }, - MTLSize { - width: interleave_width, - height: interleave_height, - depth: 1, - }, + dispatch_2d_pipeline( + encoder, + &runtime.idwt_interleave, + (params.width, params.height), ); encoder.end_encoding(); @@ -8178,7 +10509,9 @@ fn dispatch_reversible53_single_decomposition_buffers_in_command_buffer_with_off decoded: &Buffer, decoded_offset: usize, ) { + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE); let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K decode hybrid reversible53 IDWT"); dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets( runtime, encoder, @@ -8225,24 +10558,10 @@ fn dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets( size_of::() as u64, (&raw const params).cast(), ); - let interleave_width = runtime.idwt_interleave.thread_execution_width().max(1); - let interleave_height = (runtime - .idwt_interleave - .max_total_threads_per_threadgroup() - .max(interleave_width) - / interleave_width) - .max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(params.width), - height: u64::from(params.height), - depth: 1, - }, - MTLSize { - width: interleave_width, - height: interleave_height, - depth: 1, - }, + dispatch_2d_pipeline( + encoder, + &runtime.idwt_interleave, + (params.width, params.height), ); encoder.set_compute_pipeline_state(&runtime.idwt_reversible53_horizontal); @@ -8298,7 +10617,7 @@ fn dispatch_reversible53_single_decomposition_buffers_in_encoder_with_offsets( #[allow(clippy::too_many_arguments)] fn dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets( runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, + command_buffers: DirectIdwtCommandBuffers<'_>, ll: &Buffer, ll_offset: usize, hl: &Buffer, @@ -8310,7 +10629,9 @@ fn dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets( params: J2kRepeatedIdwtSingleDecompositionParams, decoded: &Buffer, ) { - let encoder = command_buffer.new_compute_command_encoder(); + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE); + let encoder = command_buffers.interleave.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K decode hybrid repeated IDWT interleave"); encoder.set_compute_pipeline_state(&runtime.idwt_interleave_batched); encoder.set_buffer(0, Some(ll), ll_offset as u64); encoder.set_buffer(1, Some(hl), hl_offset as u64); @@ -8322,31 +10643,15 @@ fn dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets( size_of::() as u64, (&raw const params).cast(), ); - let interleave_width = runtime - .idwt_interleave_batched - .thread_execution_width() - .max(1); - let interleave_height = (runtime - .idwt_interleave_batched - .max_total_threads_per_threadgroup() - .max(interleave_width) - / interleave_width) - .max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(params.width), - height: u64::from(params.height), - depth: u64::from(params.batch_count), - }, - MTLSize { - width: interleave_width, - height: interleave_height, - depth: 1, - }, + dispatch_3d_pipeline( + encoder, + &runtime.idwt_interleave_batched, + (params.width, params.height, params.batch_count), ); encoder.end_encoding(); - let encoder = command_buffer.new_compute_command_encoder(); + let encoder = command_buffers.horizontal.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K decode hybrid repeated IDWT horizontal"); encoder.set_compute_pipeline_state(&runtime.idwt_reversible53_horizontal_batched); encoder.set_buffer(0, Some(decoded), 0); encoder.set_bytes( @@ -8372,7 +10677,8 @@ fn dispatch_reversible53_repeated_buffers_in_command_buffer_with_offsets( ); encoder.end_encoding(); - let encoder = command_buffer.new_compute_command_encoder(); + let encoder = command_buffers.vertical.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K decode hybrid repeated IDWT vertical"); encoder.set_compute_pipeline_state(&runtime.idwt_reversible53_vertical_batched); encoder.set_buffer(0, Some(decoded), 0); encoder.set_bytes( @@ -8461,22 +10767,12 @@ pub(crate) fn decode_irreversible97_single_decomposition_idwt( (&raw const params).cast(), ); encoder.set_buffer(6, Some(&status_buffer), 0); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - ); + dispatch_single_thread(encoder); encoder.end_encoding(); command_buffer.commit(); command_buffer.wait_until_completed(); + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. let status = unsafe { status_buffer.contents().cast::().read() }; if status.code != J2K_IDWT_STATUS_OK { return Err(decode_idwt_status_error(status)); @@ -8502,12 +10798,14 @@ fn dispatch_irreversible97_single_decomposition_buffers_in_command_buffer_with_o decoded: &Buffer, decoded_offset: usize, ) -> DirectStatusCheck { + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE); let status_buffer = runtime.device.new_buffer( size_of::() as u64, MTLResourceOptions::StorageModeShared, ); let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K decode hybrid irreversible97 IDWT"); dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_status( runtime, encoder, @@ -8600,18 +10898,7 @@ fn dispatch_irreversible97_single_decomposition_buffers_in_encoder_with_status( (&raw const params).cast(), ); encoder.set_buffer(6, Some(status_buffer), 0); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - ); + dispatch_single_thread(encoder); } #[cfg(target_os = "macos")] @@ -8734,6 +11021,7 @@ fn dispatch_classic_cleanup_batched( command_buffer.commit(); command_buffer.wait_until_completed(); + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. let statuses = unsafe { core::slice::from_raw_parts( status_buffer.contents().cast::(), @@ -8744,7 +11032,7 @@ fn dispatch_classic_cleanup_batched( .iter() .copied() .find(|status| status.code != J2K_CLASSIC_STATUS_OK); - runtime.recycle_private_buffer(coefficients_scratch.bytes, coefficients_scratch.buffer); + runtime.recycle_private_buffer(coefficients_scratch.bytes, coefficients_scratch.buffer)?; if let Some(status) = status { return Err(decode_classic_status_error(status)); } @@ -8906,7 +11194,7 @@ fn dispatch_classic_cleanup_repeated_batched_in_command_buffer( segments: &Buffer, decoded: &Buffer, coefficients_scratch: &Buffer, -) -> DirectStatusCheck { +) -> Result { let pipeline = if use_plain_fast_path { &runtime.classic_cleanup_plain_repeated_batched } else { @@ -8917,11 +11205,15 @@ fn dispatch_classic_cleanup_repeated_batched_in_command_buffer( MTLResourceOptions::StorageModeShared, ); let repeated = J2kClassicRepeatedBatchParams { - job_count: u32::try_from(job_count).expect("classic repeated base job count fits in u32"), - output_plane_len: u32::try_from(output_plane_len) - .expect("classic repeated output plane len fits in u32"), - batch_count: u32::try_from(total_job_count / job_count.max(1)) - .expect("classic repeated batch count fits in u32"), + job_count: j2k_u32_param(job_count, "classic repeated base job count exceeds u32")?, + output_plane_len: j2k_u32_param( + output_plane_len, + "classic repeated output plane len exceeds u32", + )?, + batch_count: j2k_u32_param( + total_job_count / job_count.max(1), + "classic repeated batch count exceeds u32", + )?, }; let encoder = command_buffer.new_compute_command_encoder(); @@ -8970,10 +11262,10 @@ fn dispatch_classic_cleanup_repeated_batched_in_command_buffer( } encoder.end_encoding(); - DirectStatusCheck::Classic { + Ok(DirectStatusCheck::Classic { buffer: status_buffer, len: total_job_count, - } + }) } #[cfg(target_os = "macos")] @@ -8990,17 +11282,21 @@ fn dispatch_classic_cleanup_plain_dev_repeated_batched_in_command_buffer( decoded: &Buffer, coefficients_scratch: &Buffer, states_scratch: &Buffer, -) -> DirectStatusCheck { +) -> Result { let status_buffer = runtime.device.new_buffer( (total_job_count.max(1) * size_of::()) as u64, MTLResourceOptions::StorageModeShared, ); let repeated = J2kClassicRepeatedBatchParams { - job_count: u32::try_from(job_count).expect("classic repeated base job count fits in u32"), - output_plane_len: u32::try_from(output_plane_len) - .expect("classic repeated output plane len fits in u32"), - batch_count: u32::try_from(total_job_count / job_count.max(1)) - .expect("classic repeated batch count fits in u32"), + job_count: j2k_u32_param(job_count, "classic repeated base job count exceeds u32")?, + output_plane_len: j2k_u32_param( + output_plane_len, + "classic repeated output plane len exceeds u32", + )?, + batch_count: j2k_u32_param( + total_job_count / job_count.max(1), + "classic repeated batch count exceeds u32", + )?, }; let encoder = command_buffer.new_compute_command_encoder(); @@ -9035,10 +11331,10 @@ fn dispatch_classic_cleanup_plain_dev_repeated_batched_in_command_buffer( ); encoder.end_encoding(); - DirectStatusCheck::Classic { + Ok(DirectStatusCheck::Classic { buffer: status_buffer, len: total_job_count, - } + }) } #[cfg(target_os = "macos")] @@ -9052,13 +11348,17 @@ fn dispatch_classic_store_repeated_batched_in_command_buffer( output_plane_len: usize, decoded: &Buffer, coefficients_scratch: &Buffer, -) { +) -> Result<(), Error> { let repeated = J2kClassicRepeatedBatchParams { - job_count: u32::try_from(job_count).expect("classic repeated base job count fits in u32"), - output_plane_len: u32::try_from(output_plane_len) - .expect("classic repeated output plane len fits in u32"), - batch_count: u32::try_from(total_job_count / job_count.max(1)) - .expect("classic repeated batch count fits in u32"), + job_count: j2k_u32_param(job_count, "classic repeated base job count exceeds u32")?, + output_plane_len: j2k_u32_param( + output_plane_len, + "classic repeated output plane len exceeds u32", + )?, + batch_count: j2k_u32_param( + total_job_count / job_count.max(1), + "classic repeated batch count exceeds u32", + )?, }; let encoder = command_buffer.new_compute_command_encoder(); @@ -9084,6 +11384,7 @@ fn dispatch_classic_store_repeated_batched_in_command_buffer( }, ); encoder.end_encoding(); + Ok(()) } #[cfg(target_os = "macos")] @@ -9339,7 +11640,7 @@ fn encode_distinct_ht_sub_band_groups_to_buffer_in_command_buffer( .iter() .enumerate() .map(|(index, group)| DistinctHtBatch { - coded_data: &group.coded_data, + coded_data: &group.coded_arena.data, jobs: &group.jobs, output_base: index * per_instance_len, }), @@ -9494,7 +11795,7 @@ fn encode_repeated_classic_sub_band_to_buffer_in_command_buffer( &segments_buffer, output, &coefficients_scratch.buffer, - ) + )? } else if let Some(states_scratch) = states_scratch.as_ref() { dispatch_classic_cleanup_plain_dev_repeated_batched_in_command_buffer( runtime, @@ -9508,7 +11809,7 @@ fn encode_repeated_classic_sub_band_to_buffer_in_command_buffer( output, &coefficients_scratch.buffer, &states_scratch.buffer, - ) + )? } else { dispatch_classic_cleanup_repeated_batched_in_command_buffer( runtime, @@ -9522,7 +11823,7 @@ fn encode_repeated_classic_sub_band_to_buffer_in_command_buffer( &segments_buffer, output, &coefficients_scratch.buffer, - ) + )? }; if !use_plain_fast_path { dispatch_classic_store_repeated_batched_in_command_buffer( @@ -9534,7 +11835,7 @@ fn encode_repeated_classic_sub_band_to_buffer_in_command_buffer( job.width as usize * job.height as usize, output, &coefficients_scratch.buffer, - ); + )?; } scratch_buffers.push(coefficients_scratch); if let Some(states_scratch) = states_scratch { @@ -9604,7 +11905,7 @@ fn encode_repeated_classic_sub_band_group_to_buffer_in_command_buffer( &segments_buffer, output, &coefficients_scratch.buffer, - ) + )? } else if let Some(states_scratch) = states_scratch.as_ref() { dispatch_classic_cleanup_plain_dev_repeated_batched_in_command_buffer( runtime, @@ -9618,7 +11919,7 @@ fn encode_repeated_classic_sub_band_group_to_buffer_in_command_buffer( output, &coefficients_scratch.buffer, &states_scratch.buffer, - ) + )? } else { dispatch_classic_cleanup_repeated_batched_in_command_buffer( runtime, @@ -9632,7 +11933,7 @@ fn encode_repeated_classic_sub_band_group_to_buffer_in_command_buffer( &segments_buffer, output, &coefficients_scratch.buffer, - ) + )? }; if !use_plain_fast_path { dispatch_classic_store_repeated_batched_in_command_buffer( @@ -9644,7 +11945,7 @@ fn encode_repeated_classic_sub_band_group_to_buffer_in_command_buffer( group.total_coefficients, output, &coefficients_scratch.buffer, - ); + )?; } scratch_buffers.push(coefficients_scratch); if let Some(states_scratch) = states_scratch { @@ -9812,8 +12113,8 @@ fn encode_repeated_ht_sub_band_to_buffer_in_command_buffer( .ok_or_else(|| Error::MetalKernel { message: "HTJ2K MetalDirect repeated job count overflow".to_string(), })?; - let coded_buffer = job.coded_buffer.clone(); - let jobs_buffer = job.jobs_buffer.clone(); + let coded_buffer = prepared_ht_buffer(job.coded_buffer.as_ref(), "coded")?.clone(); + let jobs_buffer = prepared_ht_buffer(job.jobs_buffer.as_ref(), "jobs")?.clone(); let status_check = dispatch_ht_cleanup_repeated_batched_in_command_buffer( runtime, command_buffer, @@ -9855,7 +12156,7 @@ fn encode_repeated_ht_sub_band_group_to_buffer_in_command_buffer( .ok_or_else(|| Error::MetalKernel { message: "HTJ2K MetalDirect repeated grouped job count overflow".to_string(), })?; - let coded_buffer = group.coded_buffer.clone(); + let coded_buffer = group.coded_arena.buffer.clone(); let jobs_buffer = group.jobs_buffer.clone(); let status_check = dispatch_ht_cleanup_repeated_batched_in_command_buffer( runtime, @@ -9896,8 +12197,8 @@ fn encode_prepared_ht_sub_band_to_buffer_in_encoder( )); } - let coded_buffer = job.coded_buffer.clone(); - let jobs_buffer = job.jobs_buffer.clone(); + let coded_buffer = prepared_ht_buffer(job.coded_buffer.as_ref(), "coded")?.clone(); + let jobs_buffer = prepared_ht_buffer(job.jobs_buffer.as_ref(), "jobs")?.clone(); let status_check = dispatch_ht_cleanup_batched_in_encoder( runtime, encoder, @@ -9931,7 +12232,7 @@ fn encode_prepared_ht_sub_band_group_to_buffer_in_encoder( )); } - let coded_buffer = group.coded_buffer.clone(); + let coded_buffer = group.coded_arena.buffer.clone(); let jobs_buffer = group.jobs_buffer.clone(); let status_check = dispatch_ht_cleanup_batched_in_encoder( runtime, @@ -9939,22 +12240,10 @@ fn encode_prepared_ht_sub_band_group_to_buffer_in_encoder( &coded_buffer, &jobs_buffer, group.jobs.len(), - output, - group.total_coefficients, - )?; - Ok((vec![coded_buffer, jobs_buffer], status_check)) -} - -#[cfg(target_os = "macos")] -fn decode_ht_status_error(status: J2kHtStatus) -> Error { - let kind = match status.code { - J2K_HT_STATUS_FAIL => "decode failure", - J2K_HT_STATUS_UNSUPPORTED => "unsupported HT kernel input", - _ => "unexpected HT kernel status", - }; - Error::MetalKernel { - message: format!("HTJ2K Metal kernel {kind} (detail={})", status.detail), - } + output, + group.total_coefficients, + )?; + Ok((vec![coded_buffer, jobs_buffer], status_check)) } #[cfg(target_os = "macos")] @@ -10007,19 +12296,7 @@ fn dispatch_zero_u32_buffer_in_encoder( encoder.set_compute_pipeline_state(&runtime.zero_u32_buffer); encoder.set_buffer(0, Some(buffer), 0); encoder.set_bytes(1, size_of::() as u64, (&raw const word_count).cast()); - let width = runtime.zero_u32_buffer.thread_execution_width().max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(word_count), - height: 1, - depth: 1, - }, - MTLSize { - width, - height: 1, - depth: 1, - }, - ); + dispatch_1d_pipeline(encoder, &runtime.zero_u32_buffer, u64::from(word_count)); Ok(()) } @@ -10036,34 +12313,24 @@ fn encode_status_error(stage: &str, code: u32, detail: u32) -> Error { } #[cfg(target_os = "macos")] -fn classic_encode_output_capacity( - width: u32, - height: u32, - total_bitplanes: u8, -) -> Result { - let samples = usize::try_from(width) - .ok() - .and_then(|w| usize::try_from(height).ok().and_then(|h| w.checked_mul(h))) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K Metal encode block size overflow".to_string(), - })?; - samples - .checked_mul(usize::from(total_bitplanes).max(1)) - .and_then(|bits| bits.checked_mul(8)) - .and_then(|bytes| bytes.checked_add(4096)) - .map(|bytes| bytes.max(4096) + 1) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K Metal encode output capacity overflow".to_string(), - }) +fn packet_encode_status_error(status: J2kPacketEncodeStatus) -> Error { + if status.code == J2K_ENCODE_STATUS_FAIL && status.detail == 7 { + return Error::MetalKernel { + message: format!( + "packetization Metal encode kernel failure (detail=7, tier1_detail={})", + status.data_len + ), + }; + } + encode_status_error("packetization", status.code, status.detail) } -#[cfg(target_os = "macos")] -fn classic_encode_sub_band_code(sub_band_type: signinum_j2k_native::J2kSubBandType) -> u32 { +fn classic_encode_sub_band_code(sub_band_type: j2k_native::J2kSubBandType) -> u32 { match sub_band_type { - signinum_j2k_native::J2kSubBandType::LowLow => 0, - signinum_j2k_native::J2kSubBandType::HighLow => 1, - signinum_j2k_native::J2kSubBandType::LowHigh => 2, - signinum_j2k_native::J2kSubBandType::HighHigh => 3, + j2k_native::J2kSubBandType::LowLow => 0, + j2k_native::J2kSubBandType::HighLow => 1, + j2k_native::J2kSubBandType::LowHigh => 2, + j2k_native::J2kSubBandType::HighHigh => 3, } } @@ -10087,19 +12354,9 @@ fn read_classic_encoded_code_block( let data_len = usize::try_from(status.data_len).map_err(|_| Error::MetalKernel { message: "classic J2K Metal encode length exceeds usize".to_string(), })?; - if data_len > output_capacity { - return Err(Error::MetalKernel { - message: "classic J2K Metal encode length exceeds output buffer".to_string(), - }); - } - let data = if data_len == 0 { - Vec::new() - } else { - unsafe { - core::slice::from_raw_parts(output.contents().cast::().add(output_offset), data_len) - } - .to_vec() - }; + let payload_skip = usize::try_from(status.reserved0).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal encode payload skip exceeds usize".to_string(), + })?; let number_of_coding_passes = u8::try_from(status.number_of_coding_passes).map_err(|_| Error::MetalKernel { message: "classic J2K Metal encode pass count exceeds u8".to_string(), @@ -10119,6 +12376,7 @@ fn read_classic_encoded_code_block( let raw_segments = if segment_count == 0 { &[][..] } else { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. unsafe { core::slice::from_raw_parts( segment_buffer @@ -10129,6 +12387,35 @@ fn read_classic_encoded_code_block( ) } }; + let data = if data_len == 0 { + Vec::new() + } else { + let payload_span = + data_len + .checked_add(payload_skip) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal encode payload span overflow".to_string(), + })?; + if payload_span > output_capacity { + return Err(Error::MetalKernel { + message: "classic J2K Metal encode length exceeds output buffer".to_string(), + }); + } + let payload_offset = + output_offset + .checked_add(payload_skip) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal encode payload offset overflow".to_string(), + })?; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + unsafe { + core::slice::from_raw_parts( + output.contents().cast::().add(payload_offset), + data_len, + ) + } + .to_vec() + }; let segments = raw_segments .iter() .map(|segment| { @@ -10171,7 +12458,6 @@ pub(crate) fn encode_classic_tier1_code_blocks( let mut batch_jobs = Vec::::with_capacity(jobs.len()); let mut output_capacity_total = 0usize; let mut segment_capacity_total = 0usize; - let segment_capacity_per_job = 256usize; for job in jobs { let expected_coefficients = usize::try_from(job.width) @@ -10204,6 +12490,9 @@ pub(crate) fn encode_classic_tier1_code_blocks( u32::try_from(segment_capacity_total).map_err(|_| Error::MetalKernel { message: "classic J2K Metal encode segment table exceeds u32".to_string(), })?; + let style_flags = classic_style_flags(job.style); + let segment_capacity = + classic_encode_segment_capacity(style_flags, job.total_bitplanes); batch_jobs.push(J2kClassicEncodeBatchJob { coefficient_offset, output_offset, @@ -10212,13 +12501,13 @@ pub(crate) fn encode_classic_tier1_code_blocks( height: job.height, sub_band_type: classic_encode_sub_band_code(job.sub_band_type), total_bitplanes: u32::from(job.total_bitplanes), - style_flags: classic_style_flags(job.style), + style_flags, output_capacity: u32::try_from(output_capacity).map_err(|_| { Error::MetalKernel { message: "classic J2K Metal encode output capacity exceeds u32".to_string(), } })?, - segment_capacity: u32::try_from(segment_capacity_per_job).map_err(|_| { + segment_capacity: u32::try_from(segment_capacity).map_err(|_| { Error::MetalKernel { message: "classic J2K Metal encode segment capacity exceeds u32" .to_string(), @@ -10231,7 +12520,7 @@ pub(crate) fn encode_classic_tier1_code_blocks( message: "classic J2K Metal encode output buffer overflow".to_string(), })?; segment_capacity_total = segment_capacity_total - .checked_add(segment_capacity_per_job) + .checked_add(segment_capacity) .ok_or_else(|| Error::MetalKernel { message: "classic J2K Metal encode segment buffer overflow".to_string(), })?; @@ -10257,32 +12546,20 @@ pub(crate) fn encode_classic_tier1_code_blocks( let command_buffer = runtime.queue.new_command_buffer(); let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.classic_encode_code_blocks); + let classic_encode_pipeline = classic_encode_code_blocks_pipeline(runtime, &batch_jobs); + encoder.set_compute_pipeline_state(classic_encode_pipeline); encoder.set_buffer(0, Some(&coefficient_buffer), 0); encoder.set_buffer(1, Some(&output), 0); encoder.set_buffer(2, Some(&job_buffer), 0); encoder.set_buffer(3, Some(&status_buffer), 0); encoder.set_buffer(4, Some(&segment_buffer), 0); encoder.set_bytes(5, size_of::() as u64, (&raw const job_count).cast()); - encoder.dispatch_threads( - MTLSize { - width: u64::from(job_count), - height: 1, - depth: 1, - }, - MTLSize { - width: runtime - .classic_encode_code_blocks - .thread_execution_width() - .max(1), - height: 1, - depth: 1, - }, - ); + dispatch_1d_pipeline(encoder, classic_encode_pipeline, u64::from(job_count)); encoder.end_encoding(); command_buffer.commit(); command_buffer.wait_until_completed(); + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. let statuses = unsafe { core::slice::from_raw_parts( status_buffer.contents().cast::(), @@ -10315,783 +12592,1285 @@ pub(crate) fn encode_classic_tier1_code_blocks( }) } -#[cfg(target_os = "macos")] -pub(crate) fn encode_classic_tier1_prepared_device_code_blocks_resident( - session: &crate::MetalBackendSession, - prepared: J2kPreparedLosslessDeviceCodeBlocks, -) -> Result { - let J2kPreparedLosslessDeviceCodeBlocks { - coefficient_buffer, - coefficient_byte_offset: _, - coefficient_byte_len: _, - coefficient_buffer_is_batch_shared: _, - code_blocks, - recyclable_private_buffers: _, - _prepare_command_buffer: prepare_command_buffer, - _deinterleave_status_buffer: deinterleave_status_buffer, - _plane_buffers: plane_buffers, - _scratch_buffers: scratch_buffers, - _coefficient_job_buffer: coefficient_job_buffer, - } = prepared; - with_runtime_for_device(&session.device, |runtime| { - if code_blocks.is_empty() { - let output = runtime - .device - .new_buffer(1, MTLResourceOptions::StorageModePrivate); - let status_buffer = runtime - .device - .new_buffer(1, MTLResourceOptions::StorageModePrivate); - let segment_buffer = runtime - .device - .new_buffer(1, MTLResourceOptions::StorageModePrivate); - let job_buffer = runtime - .device - .new_buffer(1, MTLResourceOptions::StorageModeShared); - let command_buffer = runtime.queue.new_command_buffer(); - command_buffer.commit(); - return Ok(J2kResidentLosslessTier1CodeBlocks { - output_buffer: output, - status_buffer, - job_buffer, - batch_jobs: Vec::new(), - code_blocks, - output_capacity_total: 0, - _segment_buffer: segment_buffer, - tier1_command_buffer: command_buffer.to_owned(), - _coefficient_buffer: coefficient_buffer, - prepare_command_buffer, - _deinterleave_status_buffer: deinterleave_status_buffer, - _plane_buffers: plane_buffers, - _scratch_buffers: scratch_buffers, - _coefficient_job_buffer: coefficient_job_buffer, +#[cfg(all(test, target_os = "macos"))] +pub(crate) fn encode_classic_tier1_code_blocks_via_gpu_token_pack_for_test( + jobs: &[J2kTier1CodeBlockEncodeJob<'_>], +) -> Result, Error> { + with_runtime(|runtime| { + if jobs.is_empty() { + return Ok(Vec::new()); + } + let mut coefficients = Vec::::new(); + let mut batch_jobs = Vec::::with_capacity(jobs.len()); + let mut output_capacity_total = 0usize; + let mut segment_capacity_total = 0usize; + + for job in jobs { + let expected_coefficients = usize::try_from(job.width) + .ok() + .and_then(|w| { + usize::try_from(job.height) + .ok() + .and_then(|h| w.checked_mul(h)) + }) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal token-pack coefficient count overflow".to_string(), + })?; + if job.coefficients.len() < expected_coefficients { + return Err(Error::MetalKernel { + message: "classic J2K Metal token-pack coefficient slice is too small" + .to_string(), + }); + } + let coefficient_offset = + u32::try_from(coefficients.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal token-pack coefficient table exceeds u32" + .to_string(), + })?; + coefficients.extend_from_slice(&job.coefficients[..expected_coefficients]); + let output_capacity = + classic_encode_output_capacity(job.width, job.height, job.total_bitplanes)?; + let output_offset = + u32::try_from(output_capacity_total).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal token-pack output table exceeds u32".to_string(), + })?; + let segment_offset = + u32::try_from(segment_capacity_total).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal token-pack segment table exceeds u32".to_string(), + })?; + let style_flags = classic_style_flags(job.style); + let segment_capacity = + classic_encode_segment_capacity(style_flags, job.total_bitplanes); + batch_jobs.push(J2kClassicEncodeBatchJob { + coefficient_offset, + output_offset, + segment_offset, + width: job.width, + height: job.height, + sub_band_type: classic_encode_sub_band_code(job.sub_band_type), + total_bitplanes: u32::from(job.total_bitplanes), + style_flags, + output_capacity: u32::try_from(output_capacity).map_err(|_| { + Error::MetalKernel { + message: "classic J2K Metal token-pack output capacity exceeds u32" + .to_string(), + } + })?, + segment_capacity: u32::try_from(segment_capacity).map_err(|_| { + Error::MetalKernel { + message: "classic J2K Metal token-pack segment capacity exceeds u32" + .to_string(), + } + })?, + }); + output_capacity_total = output_capacity_total + .checked_add(output_capacity) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal token-pack output buffer overflow".to_string(), + })?; + segment_capacity_total = segment_capacity_total + .checked_add(segment_capacity) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal token-pack segment buffer overflow".to_string(), + })?; + } + + if !classic_tier1_gpu_token_pack_supported(&batch_jobs) { + return Err(Error::MetalKernel { + message: + "classic J2K Metal token-pack parity helper supports only bypass_u16_32 jobs" + .to_string(), }); } - let mut batch_jobs = Vec::::with_capacity(code_blocks.len()); + + let coefficient_buffer = owned_slice_buffer(&runtime.device, &coefficients); + let job_buffer = owned_slice_buffer(&runtime.device, &batch_jobs); + let output = runtime.device.new_buffer( + output_capacity_total.max(1) as u64, + MTLResourceOptions::StorageModeShared, + ); + let status_buffer = runtime.device.new_buffer( + (jobs.len() * size_of::()) as u64, + MTLResourceOptions::StorageModeShared, + ); + let segment_buffer = runtime.device.new_buffer( + (segment_capacity_total * size_of::()) as u64, + MTLResourceOptions::StorageModeShared, + ); + let job_count = u32::try_from(batch_jobs.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal token-pack job count exceeds u32".to_string(), + })?; + let command_buffer = runtime.queue.new_command_buffer(); + let mut recyclable_private_buffers = Vec::<(usize, Buffer)>::new(); + let token_buffers = dispatch_classic_tier1_token_emit_for_gpu_pack( + runtime, + command_buffer, + &coefficient_buffer, + &job_buffer, + &batch_jobs, + &mut recyclable_private_buffers, + )?; + debug_assert_eq!(token_buffers.job_count, job_count); + dispatch_classic_tier1_token_pack_from_gpu_tokens( + runtime, + command_buffer, + &job_buffer, + &token_buffers, + &output, + &status_buffer, + &segment_buffer, + ); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let statuses = unsafe { + core::slice::from_raw_parts( + status_buffer.contents().cast::(), + jobs.len(), + ) + }; + let mut results = Vec::with_capacity(jobs.len()); + for (idx, status) in statuses.iter().copied().enumerate() { + let batch_job = batch_jobs[idx]; + results.push(read_classic_encoded_code_block( + status, + &output, + usize::try_from(batch_job.output_offset).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal token-pack output offset exceeds usize".to_string(), + })?, + usize::try_from(batch_job.output_capacity).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal token-pack output capacity exceeds usize" + .to_string(), + })?, + &segment_buffer, + usize::try_from(batch_job.segment_offset).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal token-pack segment offset exceeds usize" + .to_string(), + })?, + usize::try_from(batch_job.segment_capacity).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal token-pack segment capacity exceeds usize" + .to_string(), + })?, + )?); + } + + Ok(results) + }) +} + +#[cfg(all(test, target_os = "macos"))] +pub(crate) fn encode_classic_tier1_code_blocks_via_split_mq_raw_tokens_gpu_pack_for_test( + jobs: &[J2kTier1CodeBlockEncodeJob<'_>], +) -> Result, Error> { + encode_classic_tier1_code_blocks_via_split_mq_raw_tokens_gpu_pack_for_test_with_emit_route( + jobs, false, + ) +} + +#[cfg(all(test, target_os = "macos"))] +pub(crate) fn encode_classic_tier1_code_blocks_via_split_mq_byte_raw_tokens_gpu_pack_for_test( + jobs: &[J2kTier1CodeBlockEncodeJob<'_>], +) -> Result, Error> { + encode_classic_tier1_code_blocks_via_split_mq_raw_tokens_gpu_pack_for_test_with_emit_route( + jobs, true, + ) +} + +#[cfg(all(test, target_os = "macos"))] +fn encode_classic_tier1_code_blocks_via_split_mq_raw_tokens_gpu_pack_for_test_with_emit_route( + jobs: &[J2kTier1CodeBlockEncodeJob<'_>], + use_mq_byte_emit: bool, +) -> Result, Error> { + with_runtime(|runtime| { + if jobs.is_empty() { + return Ok(Vec::new()); + } + let mut coefficients = Vec::::new(); + let mut batch_jobs = Vec::::with_capacity(jobs.len()); let mut output_capacity_total = 0usize; let mut segment_capacity_total = 0usize; - let segment_capacity_per_job = 256usize; - for block in &code_blocks { + for job in jobs { + let expected_coefficients = usize::try_from(job.width) + .ok() + .and_then(|w| { + usize::try_from(job.height) + .ok() + .and_then(|h| w.checked_mul(h)) + }) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal split GPU token-pack coefficient count overflow" + .to_string(), + })?; + if job.coefficients.len() < expected_coefficients { + return Err(Error::MetalKernel { + message: + "classic J2K Metal split GPU token-pack coefficient slice is too small" + .to_string(), + }); + } + let coefficient_offset = + u32::try_from(coefficients.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split GPU token-pack coefficient table exceeds u32" + .to_string(), + })?; + coefficients.extend_from_slice(&job.coefficients[..expected_coefficients]); let output_capacity = - classic_encode_output_capacity(block.width, block.height, block.total_bitplanes)?; + classic_encode_output_capacity(job.width, job.height, job.total_bitplanes)?; let output_offset = u32::try_from(output_capacity_total).map_err(|_| Error::MetalKernel { - message: "classic J2K Metal resident encode output table exceeds u32" + message: "classic J2K Metal split GPU token-pack output table exceeds u32" .to_string(), })?; let segment_offset = u32::try_from(segment_capacity_total).map_err(|_| Error::MetalKernel { - message: "classic J2K Metal resident encode segment table exceeds u32" + message: "classic J2K Metal split GPU token-pack segment table exceeds u32" .to_string(), })?; + let style_flags = classic_style_flags(job.style); + let segment_capacity = + classic_encode_segment_capacity(style_flags, job.total_bitplanes); batch_jobs.push(J2kClassicEncodeBatchJob { - coefficient_offset: block.coefficient_offset, + coefficient_offset, output_offset, segment_offset, - width: block.width, - height: block.height, - sub_band_type: classic_encode_sub_band_code(block.sub_band_type), - total_bitplanes: u32::from(block.total_bitplanes), - style_flags: 0, + width: job.width, + height: job.height, + sub_band_type: classic_encode_sub_band_code(job.sub_band_type), + total_bitplanes: u32::from(job.total_bitplanes), + style_flags, output_capacity: u32::try_from(output_capacity).map_err(|_| { Error::MetalKernel { - message: "classic J2K Metal resident encode output capacity exceeds u32" - .to_string(), + message: + "classic J2K Metal split GPU token-pack output capacity exceeds u32" + .to_string(), } })?, - segment_capacity: u32::try_from(segment_capacity_per_job).map_err(|_| { + segment_capacity: u32::try_from(segment_capacity).map_err(|_| { Error::MetalKernel { - message: "classic J2K Metal resident encode segment capacity exceeds u32" - .to_string(), + message: + "classic J2K Metal split GPU token-pack segment capacity exceeds u32" + .to_string(), } })?, }); output_capacity_total = output_capacity_total .checked_add(output_capacity) .ok_or_else(|| Error::MetalKernel { - message: "classic J2K Metal resident encode output buffer overflow".to_string(), + message: "classic J2K Metal split GPU token-pack output buffer overflow" + .to_string(), })?; segment_capacity_total = segment_capacity_total - .checked_add(segment_capacity_per_job) + .checked_add(segment_capacity) .ok_or_else(|| Error::MetalKernel { - message: "classic J2K Metal resident encode segment buffer overflow" + message: "classic J2K Metal split GPU token-pack segment buffer overflow" .to_string(), })?; } + if !classic_tier1_gpu_token_pack_supported(&batch_jobs) { + return Err(Error::MetalKernel { + message: + "classic J2K Metal split GPU token-pack helper supports only bypass_u16_32 jobs" + .to_string(), + }); + } + + let coefficient_buffer = owned_slice_buffer(&runtime.device, &coefficients); let job_buffer = owned_slice_buffer(&runtime.device, &batch_jobs); let output = runtime.device.new_buffer( output_capacity_total.max(1) as u64, - MTLResourceOptions::StorageModePrivate, + MTLResourceOptions::StorageModeShared, ); let status_buffer = runtime.device.new_buffer( - (batch_jobs.len() * size_of::()) as u64, - MTLResourceOptions::StorageModePrivate, + (jobs.len() * size_of::()) as u64, + MTLResourceOptions::StorageModeShared, ); let segment_buffer = runtime.device.new_buffer( (segment_capacity_total * size_of::()) as u64, - MTLResourceOptions::StorageModePrivate, + MTLResourceOptions::StorageModeShared, ); - let job_count = u32::try_from(batch_jobs.len()).map_err(|_| Error::MetalKernel { - message: "classic J2K Metal resident encode job count exceeds u32".to_string(), - })?; - let command_buffer = runtime.queue.new_command_buffer(); - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.classic_encode_code_blocks); - encoder.set_buffer(0, Some(&coefficient_buffer), 0); - encoder.set_buffer(1, Some(&output), 0); - encoder.set_buffer(2, Some(&job_buffer), 0); - encoder.set_buffer(3, Some(&status_buffer), 0); - encoder.set_buffer(4, Some(&segment_buffer), 0); - encoder.set_bytes(5, size_of::() as u64, (&raw const job_count).cast()); - encoder.dispatch_threads( - MTLSize { - width: u64::from(job_count), - height: 1, - depth: 1, - }, - MTLSize { - width: runtime - .classic_encode_code_blocks - .thread_execution_width() - .max(1), - height: 1, - depth: 1, - }, + let mut recyclable_private_buffers = Vec::<(usize, Buffer)>::new(); + let split_buffers = dispatch_classic_tier1_split_token_emit_for_gpu_pack( + runtime, + command_buffer, + &coefficient_buffer, + &job_buffer, + &batch_jobs, + &mut recyclable_private_buffers, + use_mq_byte_emit, + )?; + dispatch_classic_tier1_split_token_pack_from_gpu_tokens( + runtime, + command_buffer, + &job_buffer, + &split_buffers, + &output, + &status_buffer, + &segment_buffer, ); - encoder.end_encoding(); command_buffer.commit(); + command_buffer.wait_until_completed(); - Ok(J2kResidentLosslessTier1CodeBlocks { - output_buffer: output, - status_buffer, - job_buffer, - batch_jobs, - code_blocks, - output_capacity_total, - _segment_buffer: segment_buffer, - tier1_command_buffer: command_buffer.to_owned(), - _coefficient_buffer: coefficient_buffer, - prepare_command_buffer, - _deinterleave_status_buffer: deinterleave_status_buffer, - _plane_buffers: plane_buffers, - _scratch_buffers: scratch_buffers, - _coefficient_job_buffer: coefficient_job_buffer, - }) + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let statuses = unsafe { + core::slice::from_raw_parts( + status_buffer.contents().cast::(), + jobs.len(), + ) + }; + let mut results = Vec::with_capacity(jobs.len()); + for (idx, status) in statuses.iter().copied().enumerate() { + let batch_job = batch_jobs[idx]; + results.push(read_classic_encoded_code_block( + status, + &output, + usize::try_from(batch_job.output_offset).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split GPU token-pack output offset exceeds usize" + .to_string(), + })?, + usize::try_from(batch_job.output_capacity).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split GPU token-pack output capacity exceeds usize" + .to_string(), + })?, + &segment_buffer, + usize::try_from(batch_job.segment_offset).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split GPU token-pack segment offset exceeds usize" + .to_string(), + })?, + usize::try_from(batch_job.segment_capacity).map_err(|_| Error::MetalKernel { + message: + "classic J2K Metal split GPU token-pack segment capacity exceeds usize" + .to_string(), + })?, + )?); + } + + Ok(results) }) } -#[cfg(target_os = "macos")] -pub(crate) fn encode_ht_prepared_device_code_blocks_resident( - session: &crate::MetalBackendSession, - prepared: J2kPreparedLosslessDeviceCodeBlocks, -) -> Result { - let J2kPreparedLosslessDeviceCodeBlocks { - coefficient_buffer, - coefficient_byte_offset: _, - coefficient_byte_len: _, - coefficient_buffer_is_batch_shared: _, - code_blocks, - recyclable_private_buffers: _, - _prepare_command_buffer: prepare_command_buffer, - _deinterleave_status_buffer: deinterleave_status_buffer, - _plane_buffers: plane_buffers, - _scratch_buffers: scratch_buffers, - _coefficient_job_buffer: coefficient_job_buffer, - } = prepared; - with_runtime_for_device(&session.device, |runtime| { - if code_blocks.is_empty() { - let output = runtime - .device - .new_buffer(1, MTLResourceOptions::StorageModePrivate); - let status_buffer = runtime - .device - .new_buffer(1, MTLResourceOptions::StorageModePrivate); - let job_buffer = runtime - .device - .new_buffer(1, MTLResourceOptions::StorageModeShared); - let command_buffer = runtime.queue.new_command_buffer(); - command_buffer.commit(); - return Ok(J2kResidentLosslessHtCodeBlocks { - output_buffer: output, - status_buffer, - job_buffer, - batch_jobs: Vec::new(), - code_blocks, - output_capacity_total: 0, - tier1_command_buffer: command_buffer.to_owned(), - _coefficient_buffer: coefficient_buffer, - prepare_command_buffer, - _deinterleave_status_buffer: deinterleave_status_buffer, - _plane_buffers: plane_buffers, - _scratch_buffers: scratch_buffers, - _coefficient_job_buffer: coefficient_job_buffer, - }); +#[cfg(all(test, target_os = "macos"))] +pub(crate) fn encode_classic_tier1_code_blocks_via_ordered_tokens_cpu_pack_for_test( + jobs: &[J2kTier1CodeBlockEncodeJob<'_>], +) -> Result, Error> { + with_runtime(|runtime| { + if jobs.is_empty() { + return Ok(Vec::new()); } - - let output_capacity = J2K_HT_ENCODE_BASE_OUTPUT_SIZE; - let output_capacity_u32 = - u32::try_from(output_capacity).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal resident encode output capacity exceeds u32".to_string(), - })?; - let mut batch_jobs = Vec::::with_capacity(code_blocks.len()); + let mut coefficients = Vec::::new(); + let mut batch_jobs = Vec::::with_capacity(jobs.len()); let mut output_capacity_total = 0usize; + let mut segment_capacity_total = 0usize; - for block in &code_blocks { + for job in jobs { + let expected_coefficients = usize::try_from(job.width) + .ok() + .and_then(|w| { + usize::try_from(job.height) + .ok() + .and_then(|h| w.checked_mul(h)) + }) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal ordered-token coefficient count overflow" + .to_string(), + })?; + if job.coefficients.len() < expected_coefficients { + return Err(Error::MetalKernel { + message: "classic J2K Metal ordered-token coefficient slice is too small" + .to_string(), + }); + } + let coefficient_offset = + u32::try_from(coefficients.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal ordered-token coefficient table exceeds u32" + .to_string(), + })?; + coefficients.extend_from_slice(&job.coefficients[..expected_coefficients]); + let output_capacity = + classic_encode_output_capacity(job.width, job.height, job.total_bitplanes)?; let output_offset = u32::try_from(output_capacity_total).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal resident encode output table exceeds u32".to_string(), + message: "classic J2K Metal ordered-token output table exceeds u32".to_string(), })?; - batch_jobs.push(J2kHtEncodeBatchJob { - coefficient_offset: block.coefficient_offset, + let segment_offset = + u32::try_from(segment_capacity_total).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal ordered-token segment table exceeds u32" + .to_string(), + })?; + let style_flags = classic_style_flags(job.style); + let segment_capacity = + classic_encode_segment_capacity(style_flags, job.total_bitplanes); + batch_jobs.push(J2kClassicEncodeBatchJob { + coefficient_offset, output_offset, - width: block.width, - height: block.height, - total_bitplanes: u32::from(block.total_bitplanes), - output_capacity: output_capacity_u32, + segment_offset, + width: job.width, + height: job.height, + sub_band_type: classic_encode_sub_band_code(job.sub_band_type), + total_bitplanes: u32::from(job.total_bitplanes), + style_flags, + output_capacity: u32::try_from(output_capacity).map_err(|_| { + Error::MetalKernel { + message: "classic J2K Metal ordered-token output capacity exceeds u32" + .to_string(), + } + })?, + segment_capacity: u32::try_from(segment_capacity).map_err(|_| { + Error::MetalKernel { + message: "classic J2K Metal ordered-token segment capacity exceeds u32" + .to_string(), + } + })?, }); output_capacity_total = output_capacity_total .checked_add(output_capacity) .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal resident encode output buffer overflow".to_string(), + message: "classic J2K Metal ordered-token output buffer overflow".to_string(), + })?; + segment_capacity_total = segment_capacity_total + .checked_add(segment_capacity) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal ordered-token segment buffer overflow".to_string(), })?; } - let job_buffer = owned_slice_buffer(&runtime.device, &batch_jobs); - let output = runtime.device.new_buffer( - output_capacity_total.max(1) as u64, - MTLResourceOptions::StorageModePrivate, - ); - let status_buffer = runtime.device.new_buffer( - (batch_jobs.len() * size_of::()) as u64, - MTLResourceOptions::StorageModePrivate, - ); - let job_count = u32::try_from(batch_jobs.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal resident encode job count exceeds u32".to_string(), - })?; - - let command_buffer = runtime.queue.new_command_buffer(); - label_command_buffer(command_buffer, "signinum-j2k htj2k resident tier1"); - let encoder = command_buffer.new_compute_command_encoder(); - label_compute_encoder(encoder, "HTJ2K Tier-1 encode"); - let kernel = HtEncodeCodeBlocksKernel::from_env(runtime); - let pipeline = kernel.pipeline(runtime)?; - encoder.set_compute_pipeline_state(pipeline); - encoder.set_buffer(0, Some(&coefficient_buffer), 0); - encoder.set_buffer(1, Some(&output), 0); - encoder.set_buffer(2, Some(&job_buffer), 0); - encoder.set_buffer(3, Some(&runtime.ht_vlc_encode_table0), 0); - encoder.set_buffer(4, Some(&runtime.ht_vlc_encode_table1), 0); - encoder.set_buffer(5, Some(&runtime.ht_uvlc_encode_table), 0); - encoder.set_buffer(6, Some(&status_buffer), 0); - encoder.set_bytes(7, size_of::() as u64, (&raw const job_count).cast()); - kernel.dispatch(encoder, pipeline, job_count); - encoder.end_encoding(); - command_buffer.commit(); - - Ok(J2kResidentLosslessHtCodeBlocks { - output_buffer: output, - status_buffer, - job_buffer, - batch_jobs, - code_blocks, - output_capacity_total, - tier1_command_buffer: command_buffer.to_owned(), - _coefficient_buffer: coefficient_buffer, - prepare_command_buffer, - _deinterleave_status_buffer: deinterleave_status_buffer, - _plane_buffers: plane_buffers, - _scratch_buffers: scratch_buffers, - _coefficient_job_buffer: coefficient_job_buffer, - }) - }) -} - -#[cfg(target_os = "macos")] -pub(crate) fn encode_classic_tier1_code_block( - job: J2kTier1CodeBlockEncodeJob<'_>, -) -> Result { - with_runtime(|runtime| { - let expected_coefficients = usize::try_from(job.width) - .ok() - .and_then(|w| { - usize::try_from(job.height) - .ok() - .and_then(|h| w.checked_mul(h)) - }) - .ok_or_else(|| Error::MetalKernel { - message: "classic J2K Metal encode coefficient count overflow".to_string(), - })?; - if job.coefficients.len() < expected_coefficients { + if !classic_tier1_gpu_token_pack_supported(&batch_jobs) { return Err(Error::MetalKernel { - message: "classic J2K Metal encode coefficient slice is too small".to_string(), + message: "classic J2K Metal ordered-token helper supports only bypass_u16_32 jobs" + .to_string(), }); } - let output_capacity = - classic_encode_output_capacity(job.width, job.height, job.total_bitplanes)?; - let output_capacity_u32 = - u32::try_from(output_capacity).map_err(|_| Error::MetalKernel { - message: "classic J2K Metal encode output capacity exceeds u32".to_string(), + let coefficient_buffer = owned_slice_buffer(&runtime.device, &coefficients); + let job_buffer = owned_slice_buffer(&runtime.device, &batch_jobs); + let command_buffer = runtime.queue.new_command_buffer(); + let mut recyclable_private_buffers = Vec::<(usize, Buffer)>::new(); + let token_buffers = dispatch_classic_tier1_token_emit_for_gpu_pack( + runtime, + command_buffer, + &coefficient_buffer, + &job_buffer, + &batch_jobs, + &mut recyclable_private_buffers, + )?; + let job_count = + usize::try_from(token_buffers.job_count).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal ordered-token job count exceeds usize".to_string(), })?; - let params = J2kClassicEncodeParams { - width: job.width, - height: job.height, - sub_band_type: classic_encode_sub_band_code(job.sub_band_type), - total_bitplanes: u32::from(job.total_bitplanes), - style_flags: classic_style_flags(job.style), - output_capacity: output_capacity_u32, - segment_capacity: 256, - }; - let coefficients = - borrow_slice_buffer(&runtime.device, &job.coefficients[..expected_coefficients]); - let output = runtime.device.new_buffer( - output_capacity as u64, + let token_stride_bytes = + usize::try_from(token_buffers.token_stride_bytes).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal ordered-token byte stride exceeds usize".to_string(), + })?; + let token_segment_stride = + usize::try_from(token_buffers.token_segment_stride).map_err(|_| { + Error::MetalKernel { + message: "classic J2K Metal ordered-token segment stride exceeds usize" + .to_string(), + } + })?; + let counter_byte_len = job_count + .checked_mul(size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal ordered-token counter readback overflow".to_string(), + })?; + let token_byte_len = + job_count + .checked_mul(token_stride_bytes) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal ordered-token byte readback overflow".to_string(), + })?; + let token_segment_byte_len = job_count + .checked_mul(token_segment_stride) + .and_then(|count| count.checked_mul(size_of::())) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal ordered-token segment readback overflow".to_string(), + })?; + let counter_readback = runtime.device.new_buffer( + counter_byte_len.max(1) as u64, MTLResourceOptions::StorageModeShared, ); - let status_buffer = runtime.device.new_buffer( - size_of::() as u64, + let token_readback = runtime.device.new_buffer( + token_byte_len.max(1) as u64, MTLResourceOptions::StorageModeShared, ); - let segment_buffer = runtime.device.new_buffer( - (usize::try_from(params.segment_capacity).map_err(|_| Error::MetalKernel { - message: "classic J2K Metal encode segment capacity exceeds usize".to_string(), - })? * size_of::()) as u64, + let token_segment_readback = runtime.device.new_buffer( + token_segment_byte_len.max(1) as u64, MTLResourceOptions::StorageModeShared, ); - let command_buffer = runtime.queue.new_command_buffer(); - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.classic_encode_code_block); - encoder.set_buffer(0, Some(&coefficients), 0); - encoder.set_buffer(1, Some(&output), 0); - encoder.set_bytes( - 2, - size_of::() as u64, - (&raw const params).cast(), + let blit = command_buffer.new_blit_command_encoder(); + blit.copy_from_buffer( + &token_buffers.counter_buffer, + 0, + &counter_readback, + 0, + counter_byte_len as u64, ); - encoder.set_buffer(3, Some(&status_buffer), 0); - encoder.set_buffer(4, Some(&segment_buffer), 0); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, + blit.copy_from_buffer( + &token_buffers.token_buffer, + 0, + &token_readback, + 0, + token_byte_len as u64, ); - encoder.end_encoding(); + blit.copy_from_buffer( + &token_buffers.segment_buffer, + 0, + &token_segment_readback, + 0, + token_segment_byte_len as u64, + ); + blit.end_encoding(); command_buffer.commit(); command_buffer.wait_until_completed(); - let status = unsafe { - status_buffer - .contents() - .cast::() - .read() + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let counters = unsafe { + core::slice::from_raw_parts( + counter_readback + .contents() + .cast::(), + job_count, + ) }; - if status.code != J2K_ENCODE_STATUS_OK { - return Err(encode_status_error( - "classic Tier-1", - status.code, - status.detail, - )); - } - let data_len = usize::try_from(status.data_len).map_err(|_| Error::MetalKernel { - message: "classic J2K Metal encode length exceeds usize".to_string(), - })?; - if data_len > output_capacity { - return Err(Error::MetalKernel { - message: "classic J2K Metal encode length exceeds output buffer".to_string(), - }); - } - let data = if data_len == 0 { - Vec::new() - } else { - unsafe { core::slice::from_raw_parts(output.contents().cast::(), data_len) } - .to_vec() + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let token_bytes = unsafe { + core::slice::from_raw_parts(token_readback.contents().cast::(), token_byte_len) }; - let number_of_coding_passes = - u8::try_from(status.number_of_coding_passes).map_err(|_| Error::MetalKernel { - message: "classic J2K Metal encode pass count exceeds u8".to_string(), - })?; - let missing_bit_planes = - u8::try_from(status.missing_bit_planes).map_err(|_| Error::MetalKernel { - message: "classic J2K Metal encode missing bitplanes exceeds u8".to_string(), - })?; - let segment_count = - usize::try_from(status.segment_count).map_err(|_| Error::MetalKernel { - message: "classic J2K Metal encode segment count exceeds usize".to_string(), - })?; - let segment_capacity = - usize::try_from(params.segment_capacity).map_err(|_| Error::MetalKernel { - message: "classic J2K Metal encode segment capacity exceeds usize".to_string(), - })?; - if segment_count > segment_capacity { - return Err(Error::MetalKernel { - message: "classic J2K Metal encode segment count exceeds buffer".to_string(), - }); - } - let raw_segments = if segment_count == 0 { - &[][..] - } else { - unsafe { - core::slice::from_raw_parts( - segment_buffer.contents().cast::(), - segment_count, - ) - } + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let token_segments = unsafe { + core::slice::from_raw_parts( + token_segment_readback + .contents() + .cast::(), + job_count.saturating_mul(token_segment_stride), + ) }; - let segments = raw_segments - .iter() - .map(|segment| { - Ok(J2kCodeBlockSegment { - data_offset: segment.data_offset, - data_length: segment.data_length, - start_coding_pass: u8::try_from(segment.start_coding_pass).map_err(|_| { - Error::MetalKernel { - message: "classic J2K Metal encode segment start pass exceeds u8" - .to_string(), - } - })?, - end_coding_pass: u8::try_from(segment.end_coding_pass).map_err(|_| { - Error::MetalKernel { - message: "classic J2K Metal encode segment end pass exceeds u8" - .to_string(), - } - })?, - use_arithmetic: segment.use_arithmetic != 0, - }) - }) - .collect::, Error>>()?; - Ok(EncodedJ2kCodeBlock { - data, - segments, - number_of_coding_passes, - missing_bit_planes, - }) + let mut results = Vec::with_capacity(job_count); + for (block_idx, counter) in counters.iter().enumerate() { + if counter.code != J2K_ENCODE_STATUS_OK { + return Err(encode_status_error( + "classic Tier-1 ordered-token emit", + counter.code, + counter.detail, + )); + } + let segment_count = + usize::try_from(counter.segment_count).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal ordered-token segment count exceeds usize" + .to_string(), + })?; + if segment_count > token_segment_stride { + return Err(Error::MetalKernel { + message: "classic J2K Metal ordered-token segment count exceeds capacity" + .to_string(), + }); + } + let token_start = + block_idx + .checked_mul(token_stride_bytes) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal ordered-token byte offset overflow".to_string(), + })?; + let segment_start = + block_idx + .checked_mul(token_segment_stride) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal ordered-token segment offset overflow" + .to_string(), + })?; + let mut native_segments = Vec::with_capacity(segment_count); + for segment in &token_segments[segment_start..segment_start + segment_count] { + let start_coding_pass = + u8::try_from(segment.pass_range & 0xFFFF).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal ordered-token start pass exceeds u8" + .to_string(), + })?; + let end_coding_pass = + u8::try_from(segment.pass_range >> 16).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal ordered-token end pass exceeds u8".to_string(), + })?; + native_segments.push(J2kTier1TokenSegment { + token_bit_offset: segment.token_bit_offset, + token_bit_count: segment.token_bit_count, + start_coding_pass, + end_coding_pass, + use_arithmetic: (segment.flags & 1) != 0, + }); + } + let packed = pack_j2k_code_block_scalar_from_tier1_tokens( + &token_bytes[token_start..token_start + token_stride_bytes], + &native_segments, + u8::try_from(counter.coding_passes).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal ordered-token coding-pass count exceeds u8" + .to_string(), + })?, + u8::try_from(counter.missing_bit_planes).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal ordered-token missing bitplanes exceed u8" + .to_string(), + })?, + ) + .map_err(|message| Error::MetalKernel { + message: format!("classic J2K Metal ordered-token CPU pack failed: {message}"), + })?; + results.push(packed); + } + + Ok(results) }) } -#[cfg(target_os = "macos")] -fn read_ht_encoded_code_block( - status: J2kHtEncodeStatus, - output: &Buffer, - output_offset: usize, - output_capacity: usize, -) -> Result { - if status.code != J2K_ENCODE_STATUS_OK { - return Err(encode_status_error( - "HTJ2K cleanup", - status.code, - status.detail, - )); +#[cfg(all(test, target_os = "macos"))] +#[derive(Default)] +struct ClassicTier1MsbBitWriter { + bytes: Vec, + current_byte: u8, + bits_in_current: u8, + bit_count: usize, +} + +#[cfg(all(test, target_os = "macos"))] +impl ClassicTier1MsbBitWriter { + fn write_bit(&mut self, bit: u8) { + self.current_byte = (self.current_byte << 1) | (bit & 1); + self.bits_in_current += 1; + self.bit_count += 1; + if self.bits_in_current == 8 { + self.bytes.push(self.current_byte); + self.current_byte = 0; + self.bits_in_current = 0; + } } - let data_len = usize::try_from(status.data_len).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal encode length exceeds usize".to_string(), - })?; - if data_len > output_capacity { + + fn bit_count_u32(&self) -> Result { + u32::try_from(self.bit_count).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token combined bit offset exceeds u32".to_string(), + }) + } + + fn finish(mut self) -> Vec { + if self.bits_in_current != 0 { + self.bytes + .push(self.current_byte << (8 - self.bits_in_current)); + } + self.bytes + } +} + +#[cfg(all(test, target_os = "macos"))] +fn classic_tier1_split_token_bit(source: &[u8], bit_offset: usize) -> Result { + if bit_offset >= source.len().saturating_mul(8) { return Err(Error::MetalKernel { - message: "HTJ2K Metal encode length exceeds output buffer".to_string(), + message: "classic J2K Metal split-token bit offset exceeds stream".to_string(), }); } - let data = if data_len == 0 { - Vec::new() - } else { - unsafe { - core::slice::from_raw_parts(output.contents().cast::().add(output_offset), data_len) + let byte = source[bit_offset / 8]; + let shift = 7 - (bit_offset % 8); + Ok((byte >> shift) & 1) +} + +#[cfg(all(test, target_os = "macos"))] +fn classic_tier1_append_split_token_bits( + writer: &mut ClassicTier1MsbBitWriter, + source: &[u8], + bit_offset: usize, + bit_count: usize, +) -> Result<(), Error> { + let end = bit_offset + .checked_add(bit_count) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal split-token bit range overflow".to_string(), + })?; + if end > source.len().saturating_mul(8) { + return Err(Error::MetalKernel { + message: "classic J2K Metal split-token bit range exceeds stream".to_string(), + }); + } + for bit_idx in 0..bit_count { + writer.write_bit(classic_tier1_split_token_bit(source, bit_offset + bit_idx)?); + } + Ok(()) +} + +#[cfg(all(test, target_os = "macos"))] +fn pack_classic_split_mq_raw_tokens_for_test( + mq_token_bytes: &[u8], + raw_token_bytes: &[u8], + split_segments: &[J2kClassicTier1TokenSegment], + counter: J2kClassicTier1SymbolPlanCounters, +) -> Result { + if counter.code != J2K_ENCODE_STATUS_OK { + return Err(encode_status_error( + "classic Tier-1 split-token emit", + counter.code, + counter.detail, + )); + } + + let mut combined = ClassicTier1MsbBitWriter::default(); + let mut native_segments = Vec::with_capacity(split_segments.len()); + for segment in split_segments { + if (segment.flags & !1) != 0 { + return Err(Error::MetalKernel { + message: "classic J2K Metal split-token segment has unsupported flags".to_string(), + }); } - .to_vec() - }; - Ok(EncodedHtJ2kCodeBlock { - data, - num_coding_passes: u8::try_from(status.num_coding_passes).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal encode pass count exceeds u8".to_string(), - } + let use_arithmetic = (segment.flags & 1) != 0; + let source = if use_arithmetic { + mq_token_bytes + } else { + raw_token_bytes + }; + let source_bit_offset = + usize::try_from(segment.token_bit_offset).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token bit offset exceeds usize".to_string(), + })?; + let source_bit_count = + usize::try_from(segment.token_bit_count).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token bit count exceeds usize".to_string(), + })?; + let combined_bit_offset = combined.bit_count_u32()?; + classic_tier1_append_split_token_bits( + &mut combined, + source, + source_bit_offset, + source_bit_count, + )?; + let start_coding_pass = + u8::try_from(segment.pass_range & 0xFFFF).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token start pass exceeds u8".to_string(), + })?; + let end_coding_pass = + u8::try_from(segment.pass_range >> 16).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token end pass exceeds u8".to_string(), + })?; + native_segments.push(J2kTier1TokenSegment { + token_bit_offset: combined_bit_offset, + token_bit_count: segment.token_bit_count, + start_coding_pass, + end_coding_pass, + use_arithmetic, + }); + } + + pack_j2k_code_block_scalar_from_tier1_tokens( + &combined.finish(), + &native_segments, + u8::try_from(counter.coding_passes).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token coding-pass count exceeds u8".to_string(), })?, - num_zero_bitplanes: u8::try_from(status.num_zero_bitplanes).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal encode zero bitplanes exceeds u8".to_string(), - } + u8::try_from(counter.missing_bit_planes).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token missing bitplanes exceed u8".to_string(), })?, + ) + .map_err(|message| Error::MetalKernel { + message: format!("classic J2K Metal split-token CPU pack failed: {message}"), }) } -#[cfg(target_os = "macos")] -#[derive(Clone, Copy)] -enum HtEncodeCodeBlocksKernel { - Scalar, - SimdPrototype, -} +#[cfg(all(test, target_os = "macos"))] +pub(crate) fn encode_classic_tier1_code_blocks_via_split_mq_raw_tokens_cpu_pack_for_test( + jobs: &[J2kTier1CodeBlockEncodeJob<'_>], +) -> Result, Error> { + with_runtime(|runtime| { + if jobs.is_empty() { + return Ok(Vec::new()); + } + let mut coefficients = Vec::::new(); + let mut batch_jobs = Vec::::with_capacity(jobs.len()); + let mut output_capacity_total = 0usize; + let mut segment_capacity_total = 0usize; -#[cfg(target_os = "macos")] -impl HtEncodeCodeBlocksKernel { - fn from_env(runtime: &MetalRuntime) -> Self { - if ht_simd_prototype_env_requested() - && runtime.ht_encode_code_blocks_simd_prototype.is_some() - { - Self::SimdPrototype - } else { - Self::Scalar + for job in jobs { + let expected_coefficients = usize::try_from(job.width) + .ok() + .and_then(|w| { + usize::try_from(job.height) + .ok() + .and_then(|h| w.checked_mul(h)) + }) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal split-token coefficient count overflow".to_string(), + })?; + if job.coefficients.len() < expected_coefficients { + return Err(Error::MetalKernel { + message: "classic J2K Metal split-token coefficient slice is too small" + .to_string(), + }); + } + let coefficient_offset = + u32::try_from(coefficients.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token coefficient table exceeds u32" + .to_string(), + })?; + coefficients.extend_from_slice(&job.coefficients[..expected_coefficients]); + let output_capacity = + classic_encode_output_capacity(job.width, job.height, job.total_bitplanes)?; + let output_offset = + u32::try_from(output_capacity_total).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token output table exceeds u32".to_string(), + })?; + let segment_offset = + u32::try_from(segment_capacity_total).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token segment table exceeds u32".to_string(), + })?; + let style_flags = classic_style_flags(job.style); + let segment_capacity = + classic_encode_segment_capacity(style_flags, job.total_bitplanes); + batch_jobs.push(J2kClassicEncodeBatchJob { + coefficient_offset, + output_offset, + segment_offset, + width: job.width, + height: job.height, + sub_band_type: classic_encode_sub_band_code(job.sub_band_type), + total_bitplanes: u32::from(job.total_bitplanes), + style_flags, + output_capacity: u32::try_from(output_capacity).map_err(|_| { + Error::MetalKernel { + message: "classic J2K Metal split-token output capacity exceeds u32" + .to_string(), + } + })?, + segment_capacity: u32::try_from(segment_capacity).map_err(|_| { + Error::MetalKernel { + message: "classic J2K Metal split-token segment capacity exceeds u32" + .to_string(), + } + })?, + }); + output_capacity_total = output_capacity_total + .checked_add(output_capacity) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal split-token output buffer overflow".to_string(), + })?; + segment_capacity_total = segment_capacity_total + .checked_add(segment_capacity) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal split-token segment buffer overflow".to_string(), + })?; } - } - fn pipeline(self, runtime: &MetalRuntime) -> Result<&ComputePipelineState, Error> { - match self { - Self::Scalar => Ok(&runtime.ht_encode_code_blocks), - Self::SimdPrototype => { - runtime - .ht_encode_code_blocks_simd_prototype - .as_ref() - .ok_or(Error::MetalKernel { - message: - "HTJ2K SIMD prototype pipeline is unavailable on this Metal device" - .to_string(), - }) + if !classic_tier1_gpu_token_pack_supported(&batch_jobs) { + return Err(Error::MetalKernel { + message: "classic J2K Metal split-token helper supports only bypass_u16_32 jobs" + .to_string(), + }); + } + + let coefficient_buffer = owned_slice_buffer(&runtime.device, &coefficients); + let job_buffer = owned_slice_buffer(&runtime.device, &batch_jobs); + let command_buffer = runtime.queue.new_command_buffer(); + let split_buffers = dispatch_classic_tier1_split_token_emit_for_cpu_pack( + runtime, + command_buffer, + &coefficient_buffer, + &job_buffer, + &batch_jobs, + )?; + command_buffer.commit(); + command_buffer.wait_until_completed(); + + let job_count = + usize::try_from(split_buffers.job_count).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token job count exceeds usize".to_string(), + })?; + let mq_token_stride_bytes = + usize::try_from(split_buffers.mq_token_stride_bytes).map_err(|_| { + Error::MetalKernel { + message: "classic J2K Metal split-token MQ byte stride exceeds usize" + .to_string(), + } + })?; + let raw_token_stride_bytes = usize::try_from(split_buffers.raw_token_stride_bytes) + .map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token raw byte stride exceeds usize".to_string(), + })?; + let token_segment_stride = + usize::try_from(split_buffers.token_segment_stride).map_err(|_| { + Error::MetalKernel { + message: "classic J2K Metal split-token segment stride exceeds usize" + .to_string(), + } + })?; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let counters = unsafe { + core::slice::from_raw_parts( + split_buffers + .counter_buffer + .contents() + .cast::(), + job_count, + ) + }; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let mq_token_bytes = unsafe { + core::slice::from_raw_parts( + split_buffers.mq_token_buffer.contents().cast::(), + job_count.saturating_mul(mq_token_stride_bytes), + ) + }; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let raw_token_bytes = unsafe { + core::slice::from_raw_parts( + split_buffers.raw_token_buffer.contents().cast::(), + job_count.saturating_mul(raw_token_stride_bytes), + ) + }; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let token_segments = unsafe { + core::slice::from_raw_parts( + split_buffers + .segment_buffer + .contents() + .cast::(), + job_count.saturating_mul(token_segment_stride), + ) + }; + + let mut results = Vec::with_capacity(job_count); + for (block_idx, counter) in counters.iter().copied().enumerate() { + let segment_count = + usize::try_from(counter.segment_count).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal split-token segment count exceeds usize" + .to_string(), + })?; + if segment_count > token_segment_stride { + return Err(Error::MetalKernel { + message: "classic J2K Metal split-token segment count exceeds capacity" + .to_string(), + }); } + let mq_token_start = block_idx + .checked_mul(mq_token_stride_bytes) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal split-token MQ byte offset overflow".to_string(), + })?; + let raw_token_start = + block_idx + .checked_mul(raw_token_stride_bytes) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal split-token raw byte offset overflow" + .to_string(), + })?; + let segment_start = + block_idx + .checked_mul(token_segment_stride) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal split-token segment offset overflow" + .to_string(), + })?; + results.push(pack_classic_split_mq_raw_tokens_for_test( + &mq_token_bytes[mq_token_start..mq_token_start + mq_token_stride_bytes], + &raw_token_bytes[raw_token_start..raw_token_start + raw_token_stride_bytes], + &token_segments[segment_start..segment_start + segment_count], + counter, + )?); + } + + Ok(results) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn encode_classic_tier1_prepared_device_code_blocks_resident( + session: &crate::MetalBackendSession, + prepared: J2kPreparedLosslessDeviceCodeBlocks, +) -> Result { + let J2kPreparedLosslessDeviceCodeBlocks { + coefficient_buffer, + coefficient_byte_offset: _, + coefficient_byte_len: _, + coefficient_buffer_is_batch_shared: _, + code_blocks, + recyclable_private_buffers: _, + _prepare_command_buffer: prepare_command_buffer, + _prepare_deinterleave_rct_command_buffer: _, + _prepare_dwt53_command_buffer: _, + _prepare_dwt53_vertical_command_buffers: _, + _prepare_dwt53_horizontal_command_buffers: _, + _prepare_coefficient_extract_command_buffer: _, + _deinterleave_status_buffer: deinterleave_status_buffer, + _plane_buffers: plane_buffers, + _scratch_buffers: scratch_buffers, + _coefficient_job_buffer: coefficient_job_buffer, + } = prepared; + with_runtime_for_session(session, |runtime| { + if code_blocks.is_empty() { + let output = runtime + .device + .new_buffer(1, MTLResourceOptions::StorageModePrivate); + let status_buffer = runtime + .device + .new_buffer(1, MTLResourceOptions::StorageModePrivate); + let segment_buffer = runtime + .device + .new_buffer(1, MTLResourceOptions::StorageModePrivate); + let job_buffer = runtime + .device + .new_buffer(1, MTLResourceOptions::StorageModeShared); + let command_buffer = runtime.queue.new_command_buffer(); + command_buffer.commit(); + return Ok(J2kResidentLosslessTier1CodeBlocks { + output_buffer: output, + status_buffer, + job_buffer, + batch_jobs: Vec::new(), + code_blocks, + output_capacity_total: 0, + _segment_buffer: segment_buffer, + tier1_command_buffer: command_buffer.to_owned(), + _coefficient_buffer: coefficient_buffer, + prepare_command_buffer, + _deinterleave_status_buffer: deinterleave_status_buffer, + _plane_buffers: plane_buffers, + _scratch_buffers: scratch_buffers, + _coefficient_job_buffer: coefficient_job_buffer, + }); } - } + let mut batch_jobs = Vec::::with_capacity(code_blocks.len()); + let mut output_capacity_total = 0usize; + let mut segment_capacity_total = 0usize; - fn dispatch( - self, - encoder: &ComputeCommandEncoderRef, - pipeline: &ComputePipelineState, - job_count: u32, - ) { - match self { - Self::Scalar => { - encoder.dispatch_threads( - MTLSize { - width: u64::from(job_count), - height: 1, - depth: 1, - }, - MTLSize { - width: pipeline.thread_execution_width().max(1), - height: 1, - depth: 1, - }, - ); - } - Self::SimdPrototype => { - #[cfg(test)] - HT_SIMD_PROTOTYPE_DISPATCHES - .with(|dispatches| dispatches.set(dispatches.get() + 1)); - encoder.dispatch_thread_groups( - MTLSize { - width: u64::from(job_count), - height: 1, - depth: 1, - }, - MTLSize { - width: 32, - height: 1, - depth: 1, - }, - ); - } + for block in &code_blocks { + let output_capacity = + classic_encode_output_capacity(block.width, block.height, block.total_bitplanes)?; + let output_offset = + u32::try_from(output_capacity_total).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal resident encode output table exceeds u32" + .to_string(), + })?; + let segment_offset = + u32::try_from(segment_capacity_total).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal resident encode segment table exceeds u32" + .to_string(), + })?; + let style_flags = 0; + let segment_capacity = + classic_encode_segment_capacity(style_flags, block.total_bitplanes); + batch_jobs.push(J2kClassicEncodeBatchJob { + coefficient_offset: block.coefficient_offset, + output_offset, + segment_offset, + width: block.width, + height: block.height, + sub_band_type: classic_encode_sub_band_code(block.sub_band_type), + total_bitplanes: u32::from(block.total_bitplanes), + style_flags, + output_capacity: u32::try_from(output_capacity).map_err(|_| { + Error::MetalKernel { + message: "classic J2K Metal resident encode output capacity exceeds u32" + .to_string(), + } + })?, + segment_capacity: u32::try_from(segment_capacity).map_err(|_| { + Error::MetalKernel { + message: "classic J2K Metal resident encode segment capacity exceeds u32" + .to_string(), + } + })?, + }); + output_capacity_total = output_capacity_total + .checked_add(output_capacity) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal resident encode output buffer overflow".to_string(), + })?; + segment_capacity_total = segment_capacity_total + .checked_add(segment_capacity) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal resident encode segment buffer overflow" + .to_string(), + })?; } - } -} -#[cfg(target_os = "macos")] -pub(crate) fn encode_ht_cleanup_code_blocks( - jobs: &[J2kHtCodeBlockEncodeJob<'_>], -) -> Result, Error> { - with_runtime(|runtime| { - encode_ht_cleanup_code_blocks_with_runtime( - runtime, - jobs, - HtEncodeCodeBlocksKernel::from_env(runtime), - ) - }) -} - -#[cfg(all(target_os = "macos", test))] -pub(crate) fn encode_ht_cleanup_code_blocks_simd_prototype_for_test( - jobs: &[J2kHtCodeBlockEncodeJob<'_>], -) -> Result, Error> { - with_runtime(|runtime| { - encode_ht_cleanup_code_blocks_with_runtime( - runtime, - jobs, - HtEncodeCodeBlocksKernel::SimdPrototype, - ) - }) -} + let job_buffer = owned_slice_buffer(&runtime.device, &batch_jobs); + let output = runtime.device.new_buffer( + output_capacity_total.max(1) as u64, + MTLResourceOptions::StorageModePrivate, + ); + let status_buffer = runtime.device.new_buffer( + (batch_jobs.len() * size_of::()) as u64, + MTLResourceOptions::StorageModePrivate, + ); + let segment_buffer = runtime.device.new_buffer( + (segment_capacity_total * size_of::()) as u64, + MTLResourceOptions::StorageModePrivate, + ); + let job_count = u32::try_from(batch_jobs.len()).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal resident encode job count exceeds u32".to_string(), + })?; -#[cfg(target_os = "macos")] -fn encode_ht_cleanup_code_blocks_with_runtime( - runtime: &MetalRuntime, - jobs: &[J2kHtCodeBlockEncodeJob<'_>], - kernel: HtEncodeCodeBlocksKernel, -) -> Result, Error> { - encode_ht_cleanup_code_blocks_with_runtime_and_statuses(runtime, jobs, kernel).map(|blocks| { - blocks - .into_iter() - .map(|(encoded, _status)| encoded) - .collect() - }) -} + let command_buffer = runtime.queue.new_command_buffer(); + let encoder = command_buffer.new_compute_command_encoder(); + let classic_encode_pipeline = classic_encode_code_blocks_pipeline(runtime, &batch_jobs); + encoder.set_compute_pipeline_state(classic_encode_pipeline); + encoder.set_buffer(0, Some(&coefficient_buffer), 0); + encoder.set_buffer(1, Some(&output), 0); + encoder.set_buffer(2, Some(&job_buffer), 0); + encoder.set_buffer(3, Some(&status_buffer), 0); + encoder.set_buffer(4, Some(&segment_buffer), 0); + encoder.set_bytes(5, size_of::() as u64, (&raw const job_count).cast()); + dispatch_1d_pipeline(encoder, classic_encode_pipeline, u64::from(job_count)); + encoder.end_encoding(); + command_buffer.commit(); -#[cfg(all(target_os = "macos", test))] -pub(crate) fn encode_ht_cleanup_code_blocks_with_segment_lengths_for_test( - jobs: &[J2kHtCodeBlockEncodeJob<'_>], - use_simd_prototype: bool, -) -> Result, Error> { - with_runtime(|runtime| { - let kernel = if use_simd_prototype { - HtEncodeCodeBlocksKernel::SimdPrototype - } else { - HtEncodeCodeBlocksKernel::Scalar - }; - encode_ht_cleanup_code_blocks_with_runtime_and_statuses(runtime, jobs, kernel).map( - |blocks| { - blocks - .into_iter() - .map(|(encoded, status)| { - ( - encoded, - HtCodeBlockSegmentLengthsForTest { - magnitude_sign: status.reserved0, - mel: status.reserved1, - vlc: status.reserved2, - }, - ) - }) - .collect() - }, - ) + Ok(J2kResidentLosslessTier1CodeBlocks { + output_buffer: output, + status_buffer, + job_buffer, + batch_jobs, + code_blocks, + output_capacity_total, + _segment_buffer: segment_buffer, + tier1_command_buffer: command_buffer.to_owned(), + _coefficient_buffer: coefficient_buffer, + prepare_command_buffer, + _deinterleave_status_buffer: deinterleave_status_buffer, + _plane_buffers: plane_buffers, + _scratch_buffers: scratch_buffers, + _coefficient_job_buffer: coefficient_job_buffer, + }) }) } #[cfg(target_os = "macos")] -fn encode_ht_cleanup_code_blocks_with_runtime_and_statuses( - runtime: &MetalRuntime, - jobs: &[J2kHtCodeBlockEncodeJob<'_>], - kernel: HtEncodeCodeBlocksKernel, -) -> Result, Error> { - if jobs.is_empty() { - return Ok(Vec::new()); - } - - let output_capacity = J2K_HT_ENCODE_BASE_OUTPUT_SIZE; - let output_capacity_u32 = u32::try_from(output_capacity).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal encode output capacity exceeds u32".to_string(), - })?; - let mut coefficients = Vec::::new(); - let mut batch_jobs = Vec::::with_capacity(jobs.len()); - let mut output_capacity_total = 0usize; - - for job in jobs { - let expected_coefficients = usize::try_from(job.width) - .ok() - .and_then(|w| { - usize::try_from(job.height) - .ok() - .and_then(|h| w.checked_mul(h)) - }) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal encode coefficient count overflow".to_string(), - })?; - if job.coefficients.len() < expected_coefficients { - return Err(Error::MetalKernel { - message: "HTJ2K Metal encode coefficient slice is too small".to_string(), +pub(crate) fn encode_ht_prepared_device_code_blocks_resident( + session: &crate::MetalBackendSession, + prepared: J2kPreparedLosslessDeviceCodeBlocks, +) -> Result { + let J2kPreparedLosslessDeviceCodeBlocks { + coefficient_buffer, + coefficient_byte_offset: _, + coefficient_byte_len: _, + coefficient_buffer_is_batch_shared: _, + code_blocks, + recyclable_private_buffers: _, + _prepare_command_buffer: prepare_command_buffer, + _prepare_deinterleave_rct_command_buffer: _, + _prepare_dwt53_command_buffer: _, + _prepare_dwt53_vertical_command_buffers: _, + _prepare_dwt53_horizontal_command_buffers: _, + _prepare_coefficient_extract_command_buffer: _, + _deinterleave_status_buffer: deinterleave_status_buffer, + _plane_buffers: plane_buffers, + _scratch_buffers: scratch_buffers, + _coefficient_job_buffer: coefficient_job_buffer, + } = prepared; + with_runtime_for_session(session, |runtime| { + if code_blocks.is_empty() { + let output = runtime + .device + .new_buffer(1, MTLResourceOptions::StorageModePrivate); + let status_buffer = runtime + .device + .new_buffer(1, MTLResourceOptions::StorageModePrivate); + let job_buffer = runtime + .device + .new_buffer(1, MTLResourceOptions::StorageModeShared); + let command_buffer = runtime.queue.new_command_buffer(); + command_buffer.commit(); + return Ok(J2kResidentLosslessHtCodeBlocks { + output_buffer: output, + status_buffer, + job_buffer, + batch_jobs: Vec::new(), + code_blocks, + output_capacity_total: 0, + tier1_command_buffer: command_buffer.to_owned(), + _coefficient_buffer: coefficient_buffer, + prepare_command_buffer, + _deinterleave_status_buffer: deinterleave_status_buffer, + _plane_buffers: plane_buffers, + _scratch_buffers: scratch_buffers, + _coefficient_job_buffer: coefficient_job_buffer, }); } - let coefficient_offset = - u32::try_from(coefficients.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal encode coefficient table exceeds u32".to_string(), - })?; - coefficients.extend_from_slice(&job.coefficients[..expected_coefficients]); - let output_offset = - u32::try_from(output_capacity_total).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal encode output table exceeds u32".to_string(), - })?; - batch_jobs.push(J2kHtEncodeBatchJob { - coefficient_offset, - output_offset, - width: job.width, - height: job.height, - total_bitplanes: u32::from(job.total_bitplanes), - output_capacity: output_capacity_u32, - }); - output_capacity_total = output_capacity_total - .checked_add(output_capacity) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal encode output buffer overflow".to_string(), - })?; - } - let coefficient_buffer = owned_slice_buffer(&runtime.device, &coefficients); - let job_buffer = owned_slice_buffer(&runtime.device, &batch_jobs); - let output = runtime.device.new_buffer( - output_capacity_total.max(1) as u64, - MTLResourceOptions::StorageModeShared, - ); - let status_buffer = runtime.device.new_buffer( - (jobs.len() * size_of::()) as u64, - MTLResourceOptions::StorageModeShared, - ); - let job_count = u32::try_from(batch_jobs.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal encode job count exceeds u32".to_string(), - })?; + let mut batch_jobs = Vec::::with_capacity(code_blocks.len()); + let mut output_capacity_total = 0usize; + + for block in &code_blocks { + let output_capacity = ht_encode_output_capacity(block.width, block.height)?; + let output_capacity_u32 = + u32::try_from(output_capacity).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal resident encode output capacity exceeds u32".to_string(), + })?; + let output_offset = + u32::try_from(output_capacity_total).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal resident encode output table exceeds u32".to_string(), + })?; + batch_jobs.push(J2kHtEncodeBatchJob { + coefficient_offset: block.coefficient_offset, + output_offset, + width: block.width, + height: block.height, + total_bitplanes: u32::from(block.total_bitplanes), + output_capacity: output_capacity_u32, + }); + output_capacity_total = output_capacity_total + .checked_add(output_capacity) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal resident encode output buffer overflow".to_string(), + })?; + } - let command_buffer = runtime.queue.new_command_buffer(); - label_command_buffer(command_buffer, "signinum-j2k htj2k tier1 batch"); - let encoder = command_buffer.new_compute_command_encoder(); - label_compute_encoder(encoder, "HTJ2K Tier-1 encode"); - let pipeline = kernel.pipeline(runtime)?; - encoder.set_compute_pipeline_state(pipeline); - encoder.set_buffer(0, Some(&coefficient_buffer), 0); - encoder.set_buffer(1, Some(&output), 0); - encoder.set_buffer(2, Some(&job_buffer), 0); - encoder.set_buffer(3, Some(&runtime.ht_vlc_encode_table0), 0); - encoder.set_buffer(4, Some(&runtime.ht_vlc_encode_table1), 0); - encoder.set_buffer(5, Some(&runtime.ht_uvlc_encode_table), 0); - encoder.set_buffer(6, Some(&status_buffer), 0); - encoder.set_bytes(7, size_of::() as u64, (&raw const job_count).cast()); - kernel.dispatch(encoder, pipeline, job_count); - encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); + let job_buffer = owned_slice_buffer(&runtime.device, &batch_jobs); + let output = runtime.device.new_buffer( + output_capacity_total.max(1) as u64, + MTLResourceOptions::StorageModePrivate, + ); + let status_buffer = runtime.device.new_buffer( + (batch_jobs.len() * size_of::()) as u64, + MTLResourceOptions::StorageModePrivate, + ); + let job_count = u32::try_from(batch_jobs.len()).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal resident encode job count exceeds u32".to_string(), + })?; - let statuses = unsafe { - core::slice::from_raw_parts( - status_buffer.contents().cast::(), - jobs.len(), - ) - }; - let mut results = Vec::with_capacity(jobs.len()); - for (idx, status) in statuses.iter().copied().enumerate() { - let batch_job = batch_jobs[idx]; - let encoded_block = read_ht_encoded_code_block( - status, - &output, - usize::try_from(batch_job.output_offset).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal encode output offset exceeds usize".to_string(), - })?, - usize::try_from(batch_job.output_capacity).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal encode output capacity exceeds usize".to_string(), - })?, - )?; - results.push((encoded_block, status)); - } + let command_buffer = runtime.queue.new_command_buffer(); + label_command_buffer(command_buffer, "j2k htj2k resident tier1"); + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "HTJ2K Tier-1 encode"); + let pipeline = &runtime.ht_encode_code_blocks; + encoder.set_compute_pipeline_state(pipeline); + encoder.set_buffer(0, Some(&coefficient_buffer), 0); + encoder.set_buffer(1, Some(&output), 0); + encoder.set_buffer(2, Some(&job_buffer), 0); + encoder.set_buffer(3, Some(&runtime.ht_vlc_encode_table0), 0); + encoder.set_buffer(4, Some(&runtime.ht_vlc_encode_table1), 0); + encoder.set_buffer(5, Some(&runtime.ht_uvlc_encode_table), 0); + encoder.set_buffer(6, Some(&status_buffer), 0); + encoder.set_bytes(7, size_of::() as u64, (&raw const job_count).cast()); + dispatch_1d_pipeline(encoder, pipeline, u64::from(job_count)); + encoder.end_encoding(); + command_buffer.commit(); - Ok(results) + Ok(J2kResidentLosslessHtCodeBlocks { + output_buffer: output, + status_buffer, + job_buffer, + batch_jobs, + code_blocks, + output_capacity_total, + tier1_command_buffer: command_buffer.to_owned(), + _coefficient_buffer: coefficient_buffer, + prepare_command_buffer, + _deinterleave_status_buffer: deinterleave_status_buffer, + _plane_buffers: plane_buffers, + _scratch_buffers: scratch_buffers, + _coefficient_job_buffer: coefficient_job_buffer, + }) + }) } #[cfg(target_os = "macos")] -pub(crate) fn encode_ht_cleanup_code_block( - job: J2kHtCodeBlockEncodeJob<'_>, -) -> Result { +pub(crate) fn encode_classic_tier1_code_block( + job: J2kTier1CodeBlockEncodeJob<'_>, +) -> Result { with_runtime(|runtime| { let expected_coefficients = usize::try_from(job.width) .ok() @@ -11101,23 +13880,32 @@ pub(crate) fn encode_ht_cleanup_code_block( .and_then(|h| w.checked_mul(h)) }) .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal encode coefficient count overflow".to_string(), + message: "classic J2K Metal encode coefficient count overflow".to_string(), })?; if job.coefficients.len() < expected_coefficients { return Err(Error::MetalKernel { - message: "HTJ2K Metal encode coefficient slice is too small".to_string(), + message: "classic J2K Metal encode coefficient slice is too small".to_string(), }); } - let output_capacity = J2K_HT_ENCODE_BASE_OUTPUT_SIZE; + + let output_capacity = + classic_encode_output_capacity(job.width, job.height, job.total_bitplanes)?; let output_capacity_u32 = u32::try_from(output_capacity).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal encode output capacity exceeds u32".to_string(), + message: "classic J2K Metal encode output capacity exceeds u32".to_string(), })?; - let params = J2kHtEncodeParams { + let style_flags = classic_style_flags(job.style); + let segment_capacity = classic_encode_segment_capacity(style_flags, job.total_bitplanes); + let params = J2kClassicEncodeParams { width: job.width, height: job.height, + sub_band_type: classic_encode_sub_band_code(job.sub_band_type), total_bitplanes: u32::from(job.total_bitplanes), + style_flags, output_capacity: output_capacity_u32, + segment_capacity: u32::try_from(segment_capacity).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal encode segment capacity exceeds u32".to_string(), + })?, }; let coefficients = borrow_slice_buffer(&runtime.device, &job.coefficients[..expected_coefficients]); @@ -11126,585 +13914,567 @@ pub(crate) fn encode_ht_cleanup_code_block( MTLResourceOptions::StorageModeShared, ); let status_buffer = runtime.device.new_buffer( - size_of::() as u64, + size_of::() as u64, + MTLResourceOptions::StorageModeShared, + ); + let segment_buffer = runtime.device.new_buffer( + (usize::try_from(params.segment_capacity).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal encode segment capacity exceeds usize".to_string(), + })? * size_of::()) as u64, MTLResourceOptions::StorageModeShared, ); let command_buffer = runtime.queue.new_command_buffer(); let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.ht_encode_code_block); + encoder.set_compute_pipeline_state(&runtime.classic_encode_code_block); encoder.set_buffer(0, Some(&coefficients), 0); encoder.set_buffer(1, Some(&output), 0); encoder.set_bytes( 2, - size_of::() as u64, + size_of::() as u64, (&raw const params).cast(), ); - encoder.set_buffer(3, Some(&runtime.ht_vlc_encode_table0), 0); - encoder.set_buffer(4, Some(&runtime.ht_vlc_encode_table1), 0); - encoder.set_buffer(5, Some(&runtime.ht_uvlc_encode_table), 0); - encoder.set_buffer(6, Some(&status_buffer), 0); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - ); + encoder.set_buffer(3, Some(&status_buffer), 0); + encoder.set_buffer(4, Some(&segment_buffer), 0); + dispatch_single_thread(encoder); encoder.end_encoding(); command_buffer.commit(); command_buffer.wait_until_completed(); - let status = unsafe { status_buffer.contents().cast::().read() }; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let status = unsafe { + status_buffer + .contents() + .cast::() + .read() + }; if status.code != J2K_ENCODE_STATUS_OK { return Err(encode_status_error( - "HTJ2K cleanup", + "classic Tier-1", status.code, status.detail, )); } let data_len = usize::try_from(status.data_len).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal encode length exceeds usize".to_string(), + message: "classic J2K Metal encode length exceeds usize".to_string(), })?; - if data_len > output_capacity { + let payload_skip = usize::try_from(status.reserved0).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal encode payload skip exceeds usize".to_string(), + })?; + let payload_span = + data_len + .checked_add(payload_skip) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal encode payload span overflow".to_string(), + })?; + if payload_span > output_capacity { return Err(Error::MetalKernel { - message: "HTJ2K Metal encode length exceeds output buffer".to_string(), + message: "classic J2K Metal encode length exceeds output buffer".to_string(), }); } - let data = if data_len == 0 { - Vec::new() - } else { - unsafe { core::slice::from_raw_parts(output.contents().cast::(), data_len) } - .to_vec() - }; - Ok(EncodedHtJ2kCodeBlock { - data, - num_coding_passes: u8::try_from(status.num_coding_passes).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal encode pass count exceeds u8".to_string(), - } - })?, - num_zero_bitplanes: u8::try_from(status.num_zero_bitplanes).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal encode zero bitplanes exceeds u8".to_string(), - } - })?, - }) - }) -} - -#[cfg(target_os = "macos")] -fn packet_tree_node_count(width: u32, height: u32) -> Result { - if width == 0 || height == 0 { - return Ok(0); - } - let mut total = 0usize; - let mut w = width; - let mut h = height; - loop { - total = total - .checked_add( - usize::try_from(w) - .ok() - .and_then(|wu| usize::try_from(h).ok().and_then(|hu| wu.checked_mul(hu))) - .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal packet tag-tree size overflow".to_string(), - })?, - ) - .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal packet tag-tree node count overflow".to_string(), - })?; - if w <= 1 && h <= 1 { - break; - } - w = w.div_ceil(2); - h = h.div_ceil(2); - } - Ok(total) -} - -#[cfg(target_os = "macos")] -fn lossless_codestream_assembly_capacity( - tile_capacity: usize, - job: J2kLosslessCodestreamAssemblyJob, -) -> Result { - let component_count = usize::from(job.num_components); - let qcd_steps = 1usize - .checked_add( - usize::from(job.num_decomposition_levels) - .checked_mul(3) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal codestream assembly QCD step count overflow".to_string(), - })?, - ) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal codestream assembly QCD step count overflow".to_string(), - })?; - let siz_total = 40usize - .checked_add( - component_count - .checked_mul(3) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal codestream assembly SIZ size overflow".to_string(), - })?, - ) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal codestream assembly SIZ size overflow".to_string(), - })?; - let qcd_total = 5usize - .checked_add(qcd_steps) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal codestream assembly QCD size overflow".to_string(), - })?; - 2usize - .checked_add(siz_total) - .and_then(|len| { - len.checked_add( - if job.block_coding_mode == J2kLosslessCodestreamBlockCodingMode::HighThroughput { - 10 - } else { - 0 - }, - ) - }) - .and_then(|len| len.checked_add(14)) - .and_then(|len| len.checked_add(qcd_total)) - .and_then(|len| len.checked_add(if job.write_tlm { 12 } else { 0 })) - .and_then(|len| len.checked_add(12)) - .and_then(|len| len.checked_add(2)) - .and_then(|len| len.checked_add(tile_capacity)) - .and_then(|len| len.checked_add(2)) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal codestream assembly capacity overflow".to_string(), - }) -} - -#[cfg(target_os = "macos")] -fn codestream_progression_order_code(order: EncodeProgressionOrder) -> u32 { - match order { - EncodeProgressionOrder::Lrcp => 0x00, - EncodeProgressionOrder::Rpcl => 0x02, - } -} - -#[cfg(target_os = "macos")] -pub(crate) fn encode_tier2_packetization( - job: J2kPacketizationEncodeJob<'_>, -) -> Result, Error> { - with_runtime(|runtime| { - let mut resolutions = Vec::::new(); - let mut subbands = Vec::::new(); - let mut blocks = Vec::::new(); - let mut payload = Vec::::new(); - let mut max_tree_nodes = 1usize; - - for resolution in job.resolutions { - let subband_offset = u32::try_from(subbands.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet subband table exceeds u32".to_string(), + let payload_offset = payload_skip; + let data = if data_len == 0 { + Vec::new() + } else { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + unsafe { + core::slice::from_raw_parts( + output.contents().cast::().add(payload_offset), + data_len, + ) + } + .to_vec() + }; + let number_of_coding_passes = + u8::try_from(status.number_of_coding_passes).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal encode pass count exceeds u8".to_string(), })?; - for subband in &resolution.subbands { - let block_offset = u32::try_from(blocks.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet block table exceeds u32".to_string(), - })?; - max_tree_nodes = max_tree_nodes.max(packet_tree_node_count( - subband.num_cbs_x, - subband.num_cbs_y, - )?); - for code_block in &subband.code_blocks { - let data_offset = - u32::try_from(payload.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet payload exceeds u32".to_string(), - })?; - let data_len = - u32::try_from(code_block.data.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet code-block payload exceeds u32" - .to_string(), - })?; - payload.extend_from_slice(code_block.data); - blocks.push(J2kPacketBlock { - data_offset, - data_len, - num_coding_passes: u32::from(code_block.num_coding_passes), - num_zero_bitplanes: u32::from(code_block.num_zero_bitplanes), - previously_included: u32::from(code_block.previously_included), - l_block: code_block.l_block, - block_coding_mode: match code_block.block_coding_mode { - J2kPacketizationBlockCodingMode::Classic => 0, - J2kPacketizationBlockCodingMode::HighThroughput => 1, - }, - reserved0: 0, - }); - } - subbands.push(J2kPacketSubband { - block_offset, - block_count: u32::try_from(subband.code_blocks.len()).map_err(|_| { + let missing_bit_planes = + u8::try_from(status.missing_bit_planes).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal encode missing bitplanes exceeds u8".to_string(), + })?; + let segment_count = + usize::try_from(status.segment_count).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal encode segment count exceeds usize".to_string(), + })?; + let segment_capacity = + usize::try_from(params.segment_capacity).map_err(|_| Error::MetalKernel { + message: "classic J2K Metal encode segment capacity exceeds usize".to_string(), + })?; + if segment_count > segment_capacity { + return Err(Error::MetalKernel { + message: "classic J2K Metal encode segment count exceeds buffer".to_string(), + }); + } + let raw_segments = if segment_count == 0 { + &[][..] + } else { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + unsafe { + core::slice::from_raw_parts( + segment_buffer.contents().cast::(), + segment_count, + ) + } + }; + let segments = raw_segments + .iter() + .map(|segment| { + Ok(J2kCodeBlockSegment { + data_offset: segment.data_offset, + data_length: segment.data_length, + start_coding_pass: u8::try_from(segment.start_coding_pass).map_err(|_| { Error::MetalKernel { - message: "Tier-2 Metal packet subband block count exceeds u32" + message: "classic J2K Metal encode segment start pass exceeds u8" .to_string(), } })?, - num_cbs_x: subband.num_cbs_x, - num_cbs_y: subband.num_cbs_y, - }); - } - resolutions.push(J2kPacketResolution { - subband_offset, - subband_count: u32::try_from(resolution.subbands.len()).map_err(|_| { - Error::MetalKernel { - message: "Tier-2 Metal packet resolution subband count exceeds u32" - .to_string(), - } - })?, - }); - } - - let mut state_block_offsets = HashMap::::new(); - let mut state_blocks = Vec::::new(); - let mut descriptors = - Vec::::with_capacity(job.packet_descriptors.len()); - for descriptor in job.packet_descriptors { - let packet_index = - usize::try_from(descriptor.packet_index).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet descriptor packet index exceeds usize" - .to_string(), - })?; - let resolution = resolutions - .get(packet_index) - .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal packet descriptor packet index out of range".to_string(), - })?; - let subband_start = - usize::try_from(resolution.subband_offset).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet descriptor subband offset exceeds usize" - .to_string(), - })?; - let subband_count = - usize::try_from(resolution.subband_count).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet descriptor subband count exceeds usize" - .to_string(), - })?; - let subband_end = - subband_start - .checked_add(subband_count) - .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal packet descriptor subband range overflow" - .to_string(), - })?; - if subband_end > subbands.len() { - return Err(Error::MetalKernel { - message: "Tier-2 Metal packet descriptor subband range out of bounds" - .to_string(), - }); - } - let mut packet_block_count = 0usize; - for subband in &subbands[subband_start..subband_end] { - packet_block_count = packet_block_count - .checked_add(usize::try_from(subband.block_count).map_err(|_| { + end_coding_pass: u8::try_from(segment.end_coding_pass).map_err(|_| { Error::MetalKernel { - message: "Tier-2 Metal packet descriptor block count exceeds usize" + message: "classic J2K Metal encode segment end pass exceeds u8" .to_string(), } - })?) - .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal packet descriptor block count overflow".to_string(), - })?; - } + })?, + use_arithmetic: segment.use_arithmetic != 0, + }) + }) + .collect::, Error>>()?; - let (state_block_offset, existing_count) = if let Some(&(offset, count)) = - state_block_offsets.get(&descriptor.state_index) - { - (offset, count) - } else { - let offset = u32::try_from(state_blocks.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet state block offset exceeds u32".to_string(), - })?; - for subband in &subbands[subband_start..subband_end] { - let block_start = - usize::try_from(subband.block_offset).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet state block offset exceeds usize" - .to_string(), - })?; - let block_count = - usize::try_from(subband.block_count).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet state block count exceeds usize" - .to_string(), - })?; - let block_end = - block_start - .checked_add(block_count) - .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal packet state block range overflow" - .to_string(), - })?; - if block_end > blocks.len() { - return Err(Error::MetalKernel { - message: "Tier-2 Metal packet state block range out of bounds" - .to_string(), - }); - } - for block in &blocks[block_start..block_end] { - state_blocks.push(J2kPacketStateBlock { - previously_included: block.previously_included, - l_block: block.l_block, - }); - } - } - state_block_offsets.insert(descriptor.state_index, (offset, packet_block_count)); - (offset, packet_block_count) - }; - if existing_count != packet_block_count { - return Err(Error::MetalKernel { - message: "Tier-2 Metal packet descriptor state layout mismatch".to_string(), - }); - } + Ok(EncodedJ2kCodeBlock { + data, + segments, + number_of_coding_passes, + missing_bit_planes, + }) + }) +} - descriptors.push(J2kPacketDescriptor { - packet_index: descriptor.packet_index, - state_index: descriptor.state_index, - layer: u32::from(descriptor.layer), - resolution: descriptor.resolution, - component: u32::from(descriptor.component), - precinct_lo: descriptor.precinct as u32, - precinct_hi: (descriptor.precinct >> 32) as u32, - state_block_offset, - }); +#[cfg(target_os = "macos")] +fn read_ht_encoded_code_block( + status: J2kHtEncodeStatus, + output: &Buffer, + output_offset: usize, + output_capacity: usize, +) -> Result { + if status.code != J2K_ENCODE_STATUS_OK { + return Err(encode_status_error( + "HTJ2K cleanup", + status.code, + status.detail, + )); + } + let data_len = usize::try_from(status.data_len).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal encode length exceeds usize".to_string(), + })?; + if data_len > output_capacity { + return Err(Error::MetalKernel { + message: "HTJ2K Metal encode length exceeds output buffer".to_string(), + }); + } + let data = if data_len == 0 { + Vec::new() + } else { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + unsafe { + core::slice::from_raw_parts(output.contents().cast::().add(output_offset), data_len) } + .to_vec() + }; + Ok(EncodedHtJ2kCodeBlock { + data, + cleanup_length: status.data_len, + refinement_length: 0, + num_coding_passes: u8::try_from(status.num_coding_passes).map_err(|_| { + Error::MetalKernel { + message: "HTJ2K Metal encode pass count exceeds u8".to_string(), + } + })?, + num_zero_bitplanes: u8::try_from(status.num_zero_bitplanes).map_err(|_| { + Error::MetalKernel { + message: "HTJ2K Metal encode zero bitplanes exceeds u8".to_string(), + } + })?, + }) +} - let header_capacity = blocks +#[cfg(target_os = "macos")] +pub(crate) fn read_resident_ht_tier1_code_blocks_for_cpu_packetization( + session: &crate::MetalBackendSession, + tier1: &J2kResidentLosslessHtCodeBlocks, +) -> Result, Error> { + with_runtime_for_session(session, |runtime| { + if tier1.batch_jobs.is_empty() { + return Ok(Vec::new()); + } + let output_bytes = tier1.output_capacity_total.max(1); + let status_bytes = tier1 + .batch_jobs .len() - .checked_mul(256) - .and_then(|bytes| bytes.checked_add(4096)) - .map(|bytes| bytes.max(4096)) + .checked_mul(size_of::()) .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal packet header capacity overflow".to_string(), + message: "HTJ2K Metal resident status readback size overflow".to_string(), })?; - let output_capacity = payload - .len() - .checked_add( - header_capacity - .checked_mul(descriptors.len().max(resolutions.len()).max(1)) - .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal packet output capacity overflow".to_string(), - })?, + let output = runtime + .device + .new_buffer(output_bytes as u64, MTLResourceOptions::StorageModeShared); + let status_buffer = runtime + .device + .new_buffer(status_bytes as u64, MTLResourceOptions::StorageModeShared); + + let command_buffer = runtime.queue.new_command_buffer(); + label_command_buffer(command_buffer, "j2k htj2k resident tier1 cpu readback"); + let blit = command_buffer.new_blit_command_encoder(); + blit.copy_from_buffer( + &tier1.output_buffer, + 0, + &output, + 0, + tier1.output_capacity_total as u64, + ); + blit.copy_from_buffer( + &tier1.status_buffer, + 0, + &status_buffer, + 0, + status_bytes as u64, + ); + blit.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let statuses = unsafe { + core::slice::from_raw_parts( + status_buffer.contents().cast::(), + tier1.batch_jobs.len(), ) - .and_then(|bytes| bytes.checked_add(1024)) + }; + tier1 + .batch_jobs + .iter() + .zip(statuses.iter().copied()) + .map(|(batch_job, status)| { + read_ht_encoded_code_block( + status, + &output, + usize::try_from(batch_job.output_offset).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal resident output offset exceeds usize".to_string(), + })?, + usize::try_from(batch_job.output_capacity).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal resident output capacity exceeds usize".to_string(), + })?, + ) + }) + .collect() + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn encode_ht_cleanup_code_blocks( + jobs: &[J2kHtCodeBlockEncodeJob<'_>], +) -> Result, Error> { + with_runtime(|runtime| encode_ht_cleanup_code_blocks_with_runtime(runtime, jobs)) +} + +#[cfg(target_os = "macos")] +fn encode_ht_cleanup_code_blocks_with_runtime( + runtime: &MetalRuntime, + jobs: &[J2kHtCodeBlockEncodeJob<'_>], +) -> Result, Error> { + encode_ht_cleanup_code_blocks_with_runtime_and_statuses(runtime, jobs).map(|blocks| { + blocks + .into_iter() + .map(|(encoded, _status)| encoded) + .collect() + }) +} + +#[cfg(target_os = "macos")] +fn encode_ht_cleanup_code_blocks_with_runtime_and_statuses( + runtime: &MetalRuntime, + jobs: &[J2kHtCodeBlockEncodeJob<'_>], +) -> Result, Error> { + if jobs.is_empty() { + return Ok(Vec::new()); + } + if jobs.iter().any(|job| job.target_coding_passes != 1) { + return Err(Error::MetalKernel { + message: "HTJ2K Metal cleanup encode supports one coding pass".to_string(), + }); + } + + let mut coefficients = Vec::::new(); + let mut batch_jobs = Vec::::with_capacity(jobs.len()); + let mut output_capacity_total = 0usize; + + for job in jobs { + let output_capacity = ht_encode_output_capacity(job.width, job.height)?; + let output_capacity_u32 = + u32::try_from(output_capacity).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal encode output capacity exceeds u32".to_string(), + })?; + let expected_coefficients = usize::try_from(job.width) + .ok() + .and_then(|w| { + usize::try_from(job.height) + .ok() + .and_then(|h| w.checked_mul(h)) + }) .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal packet output capacity overflow".to_string(), + message: "HTJ2K Metal encode coefficient count overflow".to_string(), + })?; + if job.coefficients.len() < expected_coefficients { + return Err(Error::MetalKernel { + message: "HTJ2K Metal encode coefficient slice is too small".to_string(), + }); + } + let coefficient_offset = + u32::try_from(coefficients.len()).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal encode coefficient table exceeds u32".to_string(), + })?; + coefficients.extend_from_slice(&job.coefficients[..expected_coefficients]); + let output_offset = + u32::try_from(output_capacity_total).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal encode output table exceeds u32".to_string(), + })?; + batch_jobs.push(J2kHtEncodeBatchJob { + coefficient_offset, + output_offset, + width: job.width, + height: job.height, + total_bitplanes: u32::from(job.total_bitplanes), + output_capacity: output_capacity_u32, + }); + output_capacity_total = output_capacity_total + .checked_add(output_capacity) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal encode output buffer overflow".to_string(), })?; + } - let params = J2kPacketEncodeParams { - resolution_count: u32::try_from(resolutions.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet resolution count exceeds u32".to_string(), - })?, - num_layers: u32::from(job.num_layers), - num_components: u32::from(job.num_components), - code_block_count: u32::try_from(blocks.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet code-block count exceeds u32".to_string(), - })?, - subband_count: u32::try_from(subbands.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet subband count exceeds u32".to_string(), - })?, - descriptor_count: u32::try_from(descriptors.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet descriptor count exceeds u32".to_string(), - })?, - output_capacity: u32::try_from(output_capacity).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet output capacity exceeds u32".to_string(), - })?, - header_capacity: u32::try_from(header_capacity).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet header capacity exceeds u32".to_string(), + let coefficient_buffer = owned_slice_buffer(&runtime.device, &coefficients); + let job_buffer = owned_slice_buffer(&runtime.device, &batch_jobs); + let output = runtime.device.new_buffer( + output_capacity_total.max(1) as u64, + MTLResourceOptions::StorageModeShared, + ); + let status_buffer = runtime.device.new_buffer( + (jobs.len() * size_of::()) as u64, + MTLResourceOptions::StorageModeShared, + ); + let job_count = u32::try_from(batch_jobs.len()).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal encode job count exceeds u32".to_string(), + })?; + + let command_buffer = runtime.queue.new_command_buffer(); + label_command_buffer(command_buffer, "j2k htj2k tier1 batch"); + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "HTJ2K Tier-1 encode"); + let pipeline = &runtime.ht_encode_code_blocks; + encoder.set_compute_pipeline_state(pipeline); + encoder.set_buffer(0, Some(&coefficient_buffer), 0); + encoder.set_buffer(1, Some(&output), 0); + encoder.set_buffer(2, Some(&job_buffer), 0); + encoder.set_buffer(3, Some(&runtime.ht_vlc_encode_table0), 0); + encoder.set_buffer(4, Some(&runtime.ht_vlc_encode_table1), 0); + encoder.set_buffer(5, Some(&runtime.ht_uvlc_encode_table), 0); + encoder.set_buffer(6, Some(&status_buffer), 0); + encoder.set_bytes(7, size_of::() as u64, (&raw const job_count).cast()); + dispatch_1d_pipeline(encoder, pipeline, u64::from(job_count)); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let statuses = unsafe { + core::slice::from_raw_parts( + status_buffer.contents().cast::(), + jobs.len(), + ) + }; + let mut results = Vec::with_capacity(jobs.len()); + for (idx, status) in statuses.iter().copied().enumerate() { + let batch_job = batch_jobs[idx]; + let encoded_block = read_ht_encoded_code_block( + status, + &output, + usize::try_from(batch_job.output_offset).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal encode output offset exceeds usize".to_string(), })?, - scratch_node_capacity: u32::try_from(max_tree_nodes).map_err(|_| { - Error::MetalKernel { - message: "Tier-2 Metal packet scratch node capacity exceeds u32".to_string(), - } + usize::try_from(batch_job.output_capacity).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal encode output capacity exceeds usize".to_string(), })?, - }; + )?; + results.push((encoded_block, status)); + } - let resolution_buffer = copied_slice_buffer(&runtime.device, &resolutions); - let subband_buffer = copied_slice_buffer(&runtime.device, &subbands); - let block_buffer = copied_slice_buffer(&runtime.device, &blocks); - let payload_buffer = copied_slice_buffer(&runtime.device, &payload); - let descriptor_buffer = copied_slice_buffer(&runtime.device, &descriptors); - let state_block_buffer = copied_slice_buffer(&runtime.device, &state_blocks); - let output_buffer = runtime.device.new_buffer( - output_capacity as u64, - MTLResourceOptions::StorageModeShared, - ); - let header_buffer = runtime.device.new_buffer( - header_capacity as u64, - MTLResourceOptions::StorageModePrivate, - ); - let scratch_words = max_tree_nodes - .checked_mul(6) + Ok(results) +} + +#[cfg(target_os = "macos")] +pub(crate) fn encode_ht_cleanup_code_block( + job: J2kHtCodeBlockEncodeJob<'_>, +) -> Result { + with_runtime(|runtime| { + if job.target_coding_passes != 1 { + return Err(Error::MetalKernel { + message: "HTJ2K Metal cleanup encode supports one coding pass".to_string(), + }); + } + let expected_coefficients = usize::try_from(job.width) + .ok() + .and_then(|w| { + usize::try_from(job.height) + .ok() + .and_then(|h| w.checked_mul(h)) + }) .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal packet scratch size overflow".to_string(), + message: "HTJ2K Metal encode coefficient count overflow".to_string(), })?; - let scratch_buffer = runtime.device.new_buffer( - (scratch_words * size_of::()) as u64, - MTLResourceOptions::StorageModePrivate, + if job.coefficients.len() < expected_coefficients { + return Err(Error::MetalKernel { + message: "HTJ2K Metal encode coefficient slice is too small".to_string(), + }); + } + let output_capacity = ht_encode_output_capacity(job.width, job.height)?; + let output_capacity_u32 = + u32::try_from(output_capacity).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal encode output capacity exceeds u32".to_string(), + })?; + let params = J2kHtEncodeParams { + width: job.width, + height: job.height, + total_bitplanes: u32::from(job.total_bitplanes), + output_capacity: output_capacity_u32, + }; + let coefficients = + borrow_slice_buffer(&runtime.device, &job.coefficients[..expected_coefficients]); + let output = runtime.device.new_buffer( + output_capacity as u64, + MTLResourceOptions::StorageModeShared, ); let status_buffer = runtime.device.new_buffer( - size_of::() as u64, + size_of::() as u64, MTLResourceOptions::StorageModeShared, ); let command_buffer = runtime.queue.new_command_buffer(); let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.packet_encode); - encoder.set_buffer(0, Some(&resolution_buffer), 0); - encoder.set_buffer(1, Some(&subband_buffer), 0); - encoder.set_buffer(2, Some(&block_buffer), 0); - encoder.set_buffer(3, Some(&payload_buffer), 0); - encoder.set_buffer(4, Some(&output_buffer), 0); - encoder.set_buffer(5, Some(&header_buffer), 0); - encoder.set_buffer(6, Some(&scratch_buffer), 0); + encoder.set_compute_pipeline_state(&runtime.ht_encode_code_block); + encoder.set_buffer(0, Some(&coefficients), 0); + encoder.set_buffer(1, Some(&output), 0); encoder.set_bytes( - 7, - size_of::() as u64, + 2, + size_of::() as u64, (&raw const params).cast(), ); - encoder.set_buffer(8, Some(&status_buffer), 0); - encoder.set_buffer(9, Some(&descriptor_buffer), 0); - encoder.set_buffer(10, Some(&state_block_buffer), 0); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - ); + encoder.set_buffer(3, Some(&runtime.ht_vlc_encode_table0), 0); + encoder.set_buffer(4, Some(&runtime.ht_vlc_encode_table1), 0); + encoder.set_buffer(5, Some(&runtime.ht_uvlc_encode_table), 0); + encoder.set_buffer(6, Some(&status_buffer), 0); + dispatch_single_thread(encoder); encoder.end_encoding(); command_buffer.commit(); command_buffer.wait_until_completed(); - let status = unsafe { - status_buffer - .contents() - .cast::() - .read() - }; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let status = unsafe { status_buffer.contents().cast::().read() }; if status.code != J2K_ENCODE_STATUS_OK { return Err(encode_status_error( - "Tier-2 packetization", + "HTJ2K cleanup", status.code, status.detail, )); } let data_len = usize::try_from(status.data_len).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal packet output length exceeds usize".to_string(), + message: "HTJ2K Metal encode length exceeds usize".to_string(), })?; if data_len > output_capacity { return Err(Error::MetalKernel { - message: "Tier-2 Metal packet output length exceeds buffer".to_string(), + message: "HTJ2K Metal encode length exceeds output buffer".to_string(), }); } - Ok(if data_len == 0 { + let data = if data_len == 0 { Vec::new() } else { - unsafe { core::slice::from_raw_parts(output_buffer.contents().cast::(), data_len) } + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + unsafe { core::slice::from_raw_parts(output.contents().cast::(), data_len) } .to_vec() + }; + Ok(EncodedHtJ2kCodeBlock { + data, + cleanup_length: status.data_len, + refinement_length: 0, + num_coding_passes: u8::try_from(status.num_coding_passes).map_err(|_| { + Error::MetalKernel { + message: "HTJ2K Metal encode pass count exceeds u8".to_string(), + } + })?, + num_zero_bitplanes: u8::try_from(status.num_zero_bitplanes).map_err(|_| { + Error::MetalKernel { + message: "HTJ2K Metal encode zero bitplanes exceeds u8".to_string(), + } + })?, }) }) } #[cfg(target_os = "macos")] -pub(crate) fn encode_lossless_codestream_buffer_from_resident_classic_tier1( - session: &crate::MetalBackendSession, - tier1: &J2kResidentLosslessTier1CodeBlocks, - job: J2kResidentPacketizationEncodeJob<'_>, - codestream_job: J2kLosslessCodestreamAssemblyJob, -) -> Result { - wait_resident_lossless_codestream( - submit_lossless_codestream_buffer_from_resident_classic_tier1( - session, - tier1, - job, - codestream_job, - )?, - ) -} - -#[cfg(target_os = "macos")] -pub(crate) fn submit_lossless_codestream_buffer_from_resident_classic_tier1( - session: &crate::MetalBackendSession, - tier1: &J2kResidentLosslessTier1CodeBlocks, - job: J2kResidentPacketizationEncodeJob<'_>, - codestream_job: J2kLosslessCodestreamAssemblyJob, -) -> Result { - with_runtime_for_device(&session.device, |runtime| { - if tier1.batch_jobs.len() != tier1.code_blocks.len() { - return Err(Error::MetalKernel { - message: "Tier-2 Metal resident packetization Tier-1 table mismatch".to_string(), - }); - } - +pub(crate) fn encode_tier2_packetization( + job: J2kPacketizationEncodeJob<'_>, +) -> Result, Error> { + with_runtime(|runtime| { let mut resolutions = Vec::::new(); let mut subbands = Vec::::new(); - let mut resident_blocks = Vec::::new(); + let mut blocks = Vec::::new(); + let mut payload = Vec::::new(); let mut max_tree_nodes = 1usize; for resolution in job.resolutions { let subband_offset = u32::try_from(subbands.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet subband table exceeds u32".to_string(), + message: "Tier-2 Metal packet subband table exceeds u32".to_string(), })?; for subband in &resolution.subbands { - let block_offset = - u32::try_from(resident_blocks.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet block table exceeds u32".to_string(), - })?; + let block_offset = u32::try_from(blocks.len()).map_err(|_| Error::MetalKernel { + message: "Tier-2 Metal packet block table exceeds u32".to_string(), + })?; max_tree_nodes = max_tree_nodes.max(packet_tree_node_count( subband.num_cbs_x, subband.num_cbs_y, )?); - let code_block_start = - usize::try_from(subband.code_block_start).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet code-block offset exceeds usize" - .to_string(), - })?; - let code_block_count = - usize::try_from(subband.code_block_count).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet code-block count exceeds usize" - .to_string(), - })?; - let code_block_end = - code_block_start - .checked_add(code_block_count) - .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal resident packet code-block range overflow" + for code_block in &subband.code_blocks { + let data_offset = + u32::try_from(payload.len()).map_err(|_| Error::MetalKernel { + message: "Tier-2 Metal packet payload exceeds u32".to_string(), + })?; + let data_len = + u32::try_from(code_block.data.len()).map_err(|_| Error::MetalKernel { + message: "Tier-2 Metal packet code-block payload exceeds u32" .to_string(), })?; - if code_block_end > tier1.batch_jobs.len() { - return Err(Error::MetalKernel { - message: "Tier-2 Metal resident packet code-block range out of bounds" - .to_string(), - }); - } - for tier1_job_index in code_block_start..code_block_end { - resident_blocks.push(J2kResidentPacketBlock { - tier1_job_index: u32::try_from(tier1_job_index).map_err(|_| { - Error::MetalKernel { - message: "Tier-2 Metal resident packet Tier-1 index exceeds u32" - .to_string(), - } - })?, - previously_included: 0, - l_block: 3, - block_coding_mode: 0, + payload.extend_from_slice(code_block.data); + blocks.push(J2kPacketBlock { + data_offset, + data_len, + num_coding_passes: u32::from(code_block.num_coding_passes), + num_zero_bitplanes: u32::from(code_block.num_zero_bitplanes), + previously_included: u32::from(code_block.previously_included), + l_block: code_block.l_block, + block_coding_mode: match code_block.block_coding_mode { + J2kPacketizationBlockCodingMode::Classic => 0, + J2kPacketizationBlockCodingMode::HighThroughput => 1, + }, + reserved0: 0, }); } subbands.push(J2kPacketSubband { block_offset, - block_count: subband.code_block_count, + block_count: u32::try_from(subband.code_blocks.len()).map_err(|_| { + Error::MetalKernel { + message: "Tier-2 Metal packet subband block count exceeds u32" + .to_string(), + } + })?, num_cbs_x: subband.num_cbs_x, num_cbs_y: subband.num_cbs_y, }); @@ -11713,33 +14483,13 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_classic_tier1( subband_offset, subband_count: u32::try_from(resolution.subbands.len()).map_err(|_| { Error::MetalKernel { - message: - "Tier-2 Metal resident packet resolution subband count exceeds u32" - .to_string(), + message: "Tier-2 Metal packet resolution subband count exceeds u32" + .to_string(), } })?, }); } - if resolutions.len() - != usize::try_from(job.resolution_count).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet resolution count exceeds usize".to_string(), - })? - { - return Err(Error::MetalKernel { - message: "Tier-2 Metal resident packet resolution count mismatch".to_string(), - }); - } - if resident_blocks.len() - != usize::try_from(job.code_block_count).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet code-block count exceeds usize".to_string(), - })? - { - return Err(Error::MetalKernel { - message: "Tier-2 Metal resident packet code-block count mismatch".to_string(), - }); - } - let mut state_block_offsets = HashMap::::new(); let mut state_blocks = Vec::::new(); let mut descriptors = @@ -11747,35 +14497,34 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_classic_tier1( for descriptor in job.packet_descriptors { let packet_index = usize::try_from(descriptor.packet_index).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet descriptor packet index exceeds usize" + message: "Tier-2 Metal packet descriptor packet index exceeds usize" .to_string(), })?; let resolution = resolutions .get(packet_index) .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal resident packet descriptor packet index out of range" - .to_string(), + message: "Tier-2 Metal packet descriptor packet index out of range".to_string(), })?; let subband_start = usize::try_from(resolution.subband_offset).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet descriptor subband offset exceeds usize" + message: "Tier-2 Metal packet descriptor subband offset exceeds usize" .to_string(), })?; let subband_count = usize::try_from(resolution.subband_count).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet descriptor subband count exceeds usize" + message: "Tier-2 Metal packet descriptor subband count exceeds usize" .to_string(), })?; let subband_end = subband_start .checked_add(subband_count) .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal resident packet descriptor subband range overflow" + message: "Tier-2 Metal packet descriptor subband range overflow" .to_string(), })?; if subband_end > subbands.len() { return Err(Error::MetalKernel { - message: "Tier-2 Metal resident packet descriptor subband range out of bounds" + message: "Tier-2 Metal packet descriptor subband range out of bounds" .to_string(), }); } @@ -11784,14 +14533,12 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_classic_tier1( packet_block_count = packet_block_count .checked_add(usize::try_from(subband.block_count).map_err(|_| { Error::MetalKernel { - message: - "Tier-2 Metal resident packet descriptor block count exceeds usize" - .to_string(), + message: "Tier-2 Metal packet descriptor block count exceeds usize" + .to_string(), } })?) .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal resident packet descriptor block count overflow" - .to_string(), + message: "Tier-2 Metal packet descriptor block count overflow".to_string(), })?; } @@ -11801,35 +14548,33 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_classic_tier1( (offset, count) } else { let offset = u32::try_from(state_blocks.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet state block offset exceeds u32" - .to_string(), + message: "Tier-2 Metal packet state block offset exceeds u32".to_string(), })?; for subband in &subbands[subband_start..subband_end] { let block_start = usize::try_from(subband.block_offset).map_err(|_| Error::MetalKernel { - message: - "Tier-2 Metal resident packet state block offset exceeds usize" - .to_string(), + message: "Tier-2 Metal packet state block offset exceeds usize" + .to_string(), })?; let block_count = usize::try_from(subband.block_count).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet state block count exceeds usize" + message: "Tier-2 Metal packet state block count exceeds usize" .to_string(), })?; let block_end = block_start .checked_add(block_count) .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal resident packet state block range overflow" + message: "Tier-2 Metal packet state block range overflow" .to_string(), })?; - if block_end > resident_blocks.len() { + if block_end > blocks.len() { return Err(Error::MetalKernel { - message: "Tier-2 Metal resident packet state block range out of bounds" + message: "Tier-2 Metal packet state block range out of bounds" .to_string(), }); } - for block in &resident_blocks[block_start..block_end] { + for block in &blocks[block_start..block_end] { state_blocks.push(J2kPacketStateBlock { previously_included: block.previously_included, l_block: block.l_block, @@ -11841,8 +14586,7 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_classic_tier1( }; if existing_count != packet_block_count { return Err(Error::MetalKernel { - message: "Tier-2 Metal resident packet descriptor state layout mismatch" - .to_string(), + message: "Tier-2 Metal packet descriptor state layout mismatch".to_string(), }); } @@ -11858,111 +14602,64 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_classic_tier1( }); } - let header_capacity = resident_blocks + let header_capacity = blocks .len() .checked_mul(256) .and_then(|bytes| bytes.checked_add(4096)) .map(|bytes| bytes.max(4096)) .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal resident packet header capacity overflow".to_string(), + message: "Tier-2 Metal packet header capacity overflow".to_string(), })?; - let output_capacity = tier1 - .output_capacity_total + let output_capacity = payload + .len() .checked_add( header_capacity .checked_mul(descriptors.len().max(resolutions.len()).max(1)) .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal resident packet output capacity overflow" - .to_string(), + message: "Tier-2 Metal packet output capacity overflow".to_string(), })?, ) .and_then(|bytes| bytes.checked_add(1024)) .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal resident packet output capacity overflow".to_string(), + message: "Tier-2 Metal packet output capacity overflow".to_string(), })?; - let codestream_capacity = - lossless_codestream_assembly_capacity(output_capacity, codestream_job)?; let params = J2kPacketEncodeParams { resolution_count: u32::try_from(resolutions.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet resolution count exceeds u32".to_string(), + message: "Tier-2 Metal packet resolution count exceeds u32".to_string(), })?, num_layers: u32::from(job.num_layers), num_components: u32::from(job.num_components), - code_block_count: u32::try_from(resident_blocks.len()).map_err(|_| { - Error::MetalKernel { - message: "Tier-2 Metal resident packet code-block count exceeds u32" - .to_string(), - } + code_block_count: u32::try_from(blocks.len()).map_err(|_| Error::MetalKernel { + message: "Tier-2 Metal packet code-block count exceeds u32".to_string(), })?, subband_count: u32::try_from(subbands.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet subband count exceeds u32".to_string(), + message: "Tier-2 Metal packet subband count exceeds u32".to_string(), })?, descriptor_count: u32::try_from(descriptors.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet descriptor count exceeds u32".to_string(), + message: "Tier-2 Metal packet descriptor count exceeds u32".to_string(), })?, output_capacity: u32::try_from(output_capacity).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet output capacity exceeds u32".to_string(), + message: "Tier-2 Metal packet output capacity exceeds u32".to_string(), })?, header_capacity: u32::try_from(header_capacity).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet header capacity exceeds u32".to_string(), + message: "Tier-2 Metal packet header capacity exceeds u32".to_string(), })?, scratch_node_capacity: u32::try_from(max_tree_nodes).map_err(|_| { Error::MetalKernel { - message: "Tier-2 Metal resident packet scratch node capacity exceeds u32" - .to_string(), - } - })?, - }; - let codestream_params = J2kLosslessCodestreamAssemblyParams { - width: codestream_job.width, - height: codestream_job.height, - num_components: u32::from(codestream_job.num_components), - bit_depth: u32::from(codestream_job.bit_depth), - signed_samples: u32::from(codestream_job.signed), - num_decomposition_levels: u32::from(codestream_job.num_decomposition_levels), - use_mct: u32::from(codestream_job.use_mct), - guard_bits: u32::from(codestream_job.guard_bits), - progression_order: codestream_progression_order_code(codestream_job.progression_order), - write_tlm: u32::from(codestream_job.write_tlm), - high_throughput: u32::from( - codestream_job.block_coding_mode - == J2kLosslessCodestreamBlockCodingMode::HighThroughput, - ), - output_capacity: u32::try_from(codestream_capacity).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal codestream assembly capacity exceeds u32".to_string(), - } - })?, - }; - - let resident_block_params = J2kResidentPacketBlockParams { - block_count: u32::try_from(resident_blocks.len()).map_err(|_| Error::MetalKernel { - message: "Tier-2 Metal resident packet block count exceeds u32".to_string(), - })?, - tier1_job_count: u32::try_from(tier1.batch_jobs.len()).map_err(|_| { - Error::MetalKernel { - message: "Tier-2 Metal resident packet Tier-1 job count exceeds u32" - .to_string(), + message: "Tier-2 Metal packet scratch node capacity exceeds u32".to_string(), } })?, }; let resolution_buffer = copied_slice_buffer(&runtime.device, &resolutions); - let subband_buffer = copied_slice_buffer(&runtime.device, &subbands); - let resident_block_buffer = copied_slice_buffer(&runtime.device, &resident_blocks); - let packet_block_buffer = runtime.device.new_buffer( - (resident_blocks.len().max(1) * size_of::()) as u64, - MTLResourceOptions::StorageModePrivate, - ); + let subband_buffer = copied_slice_buffer(&runtime.device, &subbands); + let block_buffer = copied_slice_buffer(&runtime.device, &blocks); + let payload_buffer = copied_slice_buffer(&runtime.device, &payload); let descriptor_buffer = copied_slice_buffer(&runtime.device, &descriptors); let state_block_buffer = copied_slice_buffer(&runtime.device, &state_blocks); let output_buffer = runtime.device.new_buffer( output_capacity as u64, - MTLResourceOptions::StorageModePrivate, - ); - let codestream_buffer = runtime.device.new_buffer( - codestream_capacity as u64, MTLResourceOptions::StorageModeShared, ); let header_buffer = runtime.device.new_buffer( @@ -11972,7 +14669,7 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_classic_tier1( let scratch_words = max_tree_nodes .checked_mul(6) .ok_or_else(|| Error::MetalKernel { - message: "Tier-2 Metal resident packet scratch size overflow".to_string(), + message: "Tier-2 Metal packet scratch size overflow".to_string(), })?; let scratch_buffer = runtime.device.new_buffer( (scratch_words * size_of::()) as u64, @@ -11980,50 +14677,16 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_classic_tier1( ); let status_buffer = runtime.device.new_buffer( size_of::() as u64, - MTLResourceOptions::StorageModePrivate, - ); - let codestream_status_buffer = runtime.device.new_buffer( - size_of::() as u64, MTLResourceOptions::StorageModeShared, ); let command_buffer = runtime.queue.new_command_buffer(); - if !resident_blocks.is_empty() { - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.packet_block_prepare_resident_classic); - encoder.set_buffer(0, Some(&resident_block_buffer), 0); - encoder.set_buffer(1, Some(&tier1.job_buffer), 0); - encoder.set_buffer(2, Some(&tier1.status_buffer), 0); - encoder.set_buffer(3, Some(&packet_block_buffer), 0); - encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const resident_block_params).cast(), - ); - encoder.dispatch_threads( - MTLSize { - width: resident_blocks.len() as u64, - height: 1, - depth: 1, - }, - MTLSize { - width: runtime - .packet_block_prepare_resident_classic - .thread_execution_width() - .max(1), - height: 1, - depth: 1, - }, - ); - encoder.end_encoding(); - } - let encoder = command_buffer.new_compute_command_encoder(); encoder.set_compute_pipeline_state(&runtime.packet_encode); encoder.set_buffer(0, Some(&resolution_buffer), 0); encoder.set_buffer(1, Some(&subband_buffer), 0); - encoder.set_buffer(2, Some(&packet_block_buffer), 0); - encoder.set_buffer(3, Some(&tier1.output_buffer), 0); + encoder.set_buffer(2, Some(&block_buffer), 0); + encoder.set_buffer(3, Some(&payload_buffer), 0); encoder.set_buffer(4, Some(&output_buffer), 0); encoder.set_buffer(5, Some(&header_buffer), 0); encoder.set_buffer(6, Some(&scratch_buffer), 0); @@ -12035,85 +14698,53 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_classic_tier1( encoder.set_buffer(8, Some(&status_buffer), 0); encoder.set_buffer(9, Some(&descriptor_buffer), 0); encoder.set_buffer(10, Some(&state_block_buffer), 0); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - ); - encoder.end_encoding(); - - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.lossless_codestream_assemble); - encoder.set_buffer(0, Some(&output_buffer), 0); - encoder.set_buffer(1, Some(&status_buffer), 0); - encoder.set_buffer(2, Some(&codestream_buffer), 0); - encoder.set_bytes( - 3, - size_of::() as u64, - (&raw const codestream_params).cast(), - ); - encoder.set_buffer(4, Some(&codestream_status_buffer), 0); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - ); + dispatch_single_thread(encoder); encoder.end_encoding(); command_buffer.commit(); + command_buffer.wait_until_completed(); - Ok(J2kPendingResidentLosslessCodestream { - buffer: codestream_buffer, - capacity: codestream_capacity, - status_buffer: codestream_status_buffer, - command_buffer: command_buffer.to_owned(), - retained_command_buffers: vec![ - tier1.prepare_command_buffer.clone(), - tier1.tier1_command_buffer.clone(), - ], - _retained_buffers: vec![ - resolution_buffer, - subband_buffer, - resident_block_buffer, - packet_block_buffer, - descriptor_buffer, - state_block_buffer, - output_buffer, - header_buffer, - scratch_buffer, - status_buffer, - tier1.output_buffer.clone(), - tier1.status_buffer.clone(), - tier1.job_buffer.clone(), - ], - status_stage: "J2K codestream assembly", - length_error: "J2K Metal codestream output length exceeds usize", - capacity_error: "J2K Metal codestream output length exceeds buffer", + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let status = unsafe { + status_buffer + .contents() + .cast::() + .read() + }; + if status.code != J2K_ENCODE_STATUS_OK { + return Err(encode_status_error( + "Tier-2 packetization", + status.code, + status.detail, + )); + } + let data_len = usize::try_from(status.data_len).map_err(|_| Error::MetalKernel { + message: "Tier-2 Metal packet output length exceeds usize".to_string(), + })?; + if data_len > output_capacity { + return Err(Error::MetalKernel { + message: "Tier-2 Metal packet output length exceeds buffer".to_string(), + }); + } + Ok(if data_len == 0 { + Vec::new() + } else { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + unsafe { core::slice::from_raw_parts(output_buffer.contents().cast::(), data_len) } + .to_vec() }) }) } #[cfg(target_os = "macos")] -pub(crate) fn encode_lossless_codestream_buffer_from_resident_ht_tier1( +pub(crate) fn encode_lossless_codestream_buffer_from_resident_tier1< + T: ResidentLosslessTier1Metal, +>( session: &crate::MetalBackendSession, - tier1: &J2kResidentLosslessHtCodeBlocks, + tier1: &T, job: J2kResidentPacketizationEncodeJob<'_>, codestream_job: J2kLosslessCodestreamAssemblyJob, ) -> Result { - wait_resident_lossless_codestream(submit_lossless_codestream_buffer_from_resident_ht_tier1( + wait_resident_lossless_codestream(submit_lossless_codestream_buffer_from_resident_tier1( session, tier1, job, @@ -12122,17 +14753,21 @@ pub(crate) fn encode_lossless_codestream_buffer_from_resident_ht_tier1( } #[cfg(target_os = "macos")] -pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( +pub(crate) fn submit_lossless_codestream_buffer_from_resident_tier1< + T: ResidentLosslessTier1Metal, +>( session: &crate::MetalBackendSession, - tier1: &J2kResidentLosslessHtCodeBlocks, + tier1: &T, job: J2kResidentPacketizationEncodeJob<'_>, codestream_job: J2kLosslessCodestreamAssemblyJob, ) -> Result { - with_runtime_for_device(&session.device, |runtime| { - if tier1.batch_jobs.len() != tier1.code_blocks.len() { + with_runtime_for_session(session, |runtime| { + if tier1.batch_job_count() != tier1.code_block_count() { return Err(Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packetization Tier-1 table mismatch" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packetization Tier-1 table mismatch", + T::TIER2_PREFIX + ), }); } @@ -12143,13 +14778,18 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( for resolution in job.resolutions { let subband_offset = u32::try_from(subbands.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet subband table exceeds u32".to_string(), + message: format!( + "{}Tier-2 Metal resident packet subband table exceeds u32", + T::TIER2_PREFIX + ), })?; for subband in &resolution.subbands { let block_offset = u32::try_from(resident_blocks.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet block table exceeds u32" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet block table exceeds u32", + T::TIER2_PREFIX + ), })?; max_tree_nodes = max_tree_nodes.max(packet_tree_node_count( subband.num_cbs_x, @@ -12157,42 +14797,48 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( )?); let code_block_start = usize::try_from(subband.code_block_start).map_err(|_| Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet code-block offset exceeds usize" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet code-block offset exceeds usize", + T::TIER2_PREFIX + ), })?; let code_block_count = usize::try_from(subband.code_block_count).map_err(|_| Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet code-block count exceeds usize" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet code-block count exceeds usize", + T::TIER2_PREFIX + ), })?; let code_block_end = code_block_start .checked_add(code_block_count) .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet code-block range overflow" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet code-block range overflow", + T::TIER2_PREFIX + ), })?; - if code_block_end > tier1.batch_jobs.len() { + if code_block_end > tier1.batch_job_count() { return Err(Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet code-block range out of bounds" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet code-block range out of bounds", + T::TIER2_PREFIX + ), }); } for tier1_job_index in code_block_start..code_block_end { resident_blocks.push(J2kResidentPacketBlock { tier1_job_index: u32::try_from(tier1_job_index).map_err(|_| { Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet Tier-1 index exceeds u32" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet Tier-1 index exceeds u32", + T::TIER2_PREFIX + ), } })?, previously_included: 0, l_block: 3, - block_coding_mode: 1, + block_coding_mode: T::BLOCK_CODING_MODE, }); } subbands.push(J2kPacketSubband { @@ -12206,8 +14852,10 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( subband_offset, subband_count: u32::try_from(resolution.subbands.len()).map_err(|_| { Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet resolution subband count exceeds u32" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet resolution subband count exceeds u32", + T::TIER2_PREFIX + ), } })?, }); @@ -12215,22 +14863,32 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( if resolutions.len() != usize::try_from(job.resolution_count).map_err(|_| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet resolution count exceeds usize" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet resolution count exceeds usize", + T::TIER2_PREFIX + ), })? { return Err(Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet resolution count mismatch".to_string(), + message: format!( + "{}Tier-2 Metal resident packet resolution count mismatch", + T::TIER2_PREFIX + ), }); } if resident_blocks.len() != usize::try_from(job.code_block_count).map_err(|_| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet code-block count exceeds usize" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet code-block count exceeds usize", + T::TIER2_PREFIX + ), })? { return Err(Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet code-block count mismatch".to_string(), + message: format!( + "{}Tier-2 Metal resident packet code-block count mismatch", + T::TIER2_PREFIX + ), }); } @@ -12241,42 +14899,48 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( for descriptor in job.packet_descriptors { let packet_index = usize::try_from(descriptor.packet_index).map_err(|_| Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet descriptor packet index exceeds usize" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet descriptor packet index exceeds usize", + T::TIER2_PREFIX + ), })?; let resolution = resolutions .get(packet_index) .ok_or_else(|| Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet descriptor packet index out of range" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet descriptor packet index out of range", + T::TIER2_PREFIX + ), })?; let subband_start = usize::try_from(resolution.subband_offset).map_err(|_| Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet descriptor subband offset exceeds usize" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet descriptor subband offset exceeds usize", + T::TIER2_PREFIX + ), })?; let subband_count = usize::try_from(resolution.subband_count).map_err(|_| Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet descriptor subband count exceeds usize" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet descriptor subband count exceeds usize", + T::TIER2_PREFIX + ), })?; let subband_end = subband_start .checked_add(subband_count) .ok_or_else(|| Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet descriptor subband range overflow" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet descriptor subband range overflow", + T::TIER2_PREFIX + ), })?; if subband_end > subbands.len() { return Err(Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet descriptor subband range out of bounds" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet descriptor subband range out of bounds", + T::TIER2_PREFIX + ), }); } let mut packet_block_count = 0usize; @@ -12284,13 +14948,11 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( packet_block_count = packet_block_count .checked_add(usize::try_from(subband.block_count).map_err(|_| { Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet descriptor block count exceeds usize" - .to_string(), + message: format!("{}Tier-2 Metal resident packet descriptor block count exceeds usize", T::TIER2_PREFIX), } })?) .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet descriptor block count overflow" - .to_string(), + message: format!("{}Tier-2 Metal resident packet descriptor block count overflow", T::TIER2_PREFIX), })?; } @@ -12300,34 +14962,41 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( (offset, count) } else { let offset = u32::try_from(state_blocks.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet state block offset exceeds u32" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet state block offset exceeds u32", + T::TIER2_PREFIX + ), })?; for subband in &subbands[subband_start..subband_end] { let block_start = usize::try_from(subband.block_offset).map_err(|_| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet state block offset exceeds usize" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet state block offset exceeds usize", + T::TIER2_PREFIX + ), })?; let block_count = usize::try_from(subband.block_count).map_err(|_| Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet state block count exceeds usize" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet state block count exceeds usize", + T::TIER2_PREFIX + ), })?; let block_end = block_start .checked_add(block_count) .ok_or_else(|| Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet state block range overflow" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet state block range overflow", + T::TIER2_PREFIX + ), })?; if block_end > resident_blocks.len() { return Err(Error::MetalKernel { - message: - "HTJ2K Tier-2 Metal resident packet state block range out of bounds" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet state block range out of bounds", + T::TIER2_PREFIX + ), }); } for block in &resident_blocks[block_start..block_end] { @@ -12342,8 +15011,10 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( }; if existing_count != packet_block_count { return Err(Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet descriptor state layout mismatch" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet descriptor state layout mismatch", + T::TIER2_PREFIX + ), }); } @@ -12365,57 +15036,80 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( .and_then(|bytes| bytes.checked_add(4096)) .map(|bytes| bytes.max(4096)) .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet header capacity overflow".to_string(), + message: format!( + "{}Tier-2 Metal resident packet header capacity overflow", + T::TIER2_PREFIX + ), })?; let output_capacity = tier1 - .output_capacity_total + .output_capacity_total() .checked_add( header_capacity .checked_mul(descriptors.len().max(resolutions.len()).max(1)) .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet output capacity overflow" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet output capacity overflow", + T::TIER2_PREFIX + ), })?, ) .and_then(|bytes| bytes.checked_add(1024)) .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet output capacity overflow".to_string(), + message: format!( + "{}Tier-2 Metal resident packet output capacity overflow", + T::TIER2_PREFIX + ), })?; let codestream_capacity = lossless_codestream_assembly_capacity(output_capacity, codestream_job)?; let params = J2kPacketEncodeParams { resolution_count: u32::try_from(resolutions.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet resolution count exceeds u32" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet resolution count exceeds u32", + T::TIER2_PREFIX + ), })?, num_layers: u32::from(job.num_layers), num_components: u32::from(job.num_components), code_block_count: u32::try_from(resident_blocks.len()).map_err(|_| { Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet code-block count exceeds u32" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet code-block count exceeds u32", + T::TIER2_PREFIX + ), } })?, subband_count: u32::try_from(subbands.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet subband count exceeds u32".to_string(), + message: format!( + "{}Tier-2 Metal resident packet subband count exceeds u32", + T::TIER2_PREFIX + ), })?, descriptor_count: u32::try_from(descriptors.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet descriptor count exceeds u32" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet descriptor count exceeds u32", + T::TIER2_PREFIX + ), })?, output_capacity: u32::try_from(output_capacity).map_err(|_| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet output capacity exceeds u32" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet output capacity exceeds u32", + T::TIER2_PREFIX + ), })?, header_capacity: u32::try_from(header_capacity).map_err(|_| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet header capacity exceeds u32" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet header capacity exceeds u32", + T::TIER2_PREFIX + ), })?, scratch_node_capacity: u32::try_from(max_tree_nodes).map_err(|_| { Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet scratch node capacity exceeds u32" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet scratch node capacity exceeds u32", + T::TIER2_PREFIX + ), } })?, }; @@ -12434,21 +15128,35 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( codestream_job.block_coding_mode == J2kLosslessCodestreamBlockCodingMode::HighThroughput, ), + code_block_style: match codestream_job.block_coding_mode { + J2kLosslessCodestreamBlockCodingMode::Classic => 0, + J2kLosslessCodestreamBlockCodingMode::HighThroughput => 0x40, + }, + code_block_width_exp: u32::from(codestream_job.code_block_width_exp), + code_block_height_exp: u32::from(codestream_job.code_block_height_exp), output_capacity: u32::try_from(codestream_capacity).map_err(|_| { Error::MetalKernel { - message: "HTJ2K Metal codestream assembly capacity exceeds u32".to_string(), + message: format!( + "{} Metal codestream assembly capacity exceeds u32", + T::FAMILY_NAME + ), } })?, }; let resident_block_params = J2kResidentPacketBlockParams { block_count: u32::try_from(resident_blocks.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet block count exceeds u32".to_string(), + message: format!( + "{}Tier-2 Metal resident packet block count exceeds u32", + T::TIER2_PREFIX + ), })?, - tier1_job_count: u32::try_from(tier1.batch_jobs.len()).map_err(|_| { + tier1_job_count: u32::try_from(tier1.batch_job_count()).map_err(|_| { Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet Tier-1 job count exceeds u32" - .to_string(), + message: format!( + "{}Tier-2 Metal resident packet Tier-1 job count exceeds u32", + T::TIER2_PREFIX + ), } })?, }; @@ -12477,7 +15185,10 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( let scratch_words = max_tree_nodes .checked_mul(6) .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Tier-2 Metal resident packet scratch size overflow".to_string(), + message: format!( + "{}Tier-2 Metal resident packet scratch size overflow", + T::TIER2_PREFIX + ), })?; let scratch_buffer = runtime.device.new_buffer( (scratch_words * size_of::()) as u64, @@ -12495,10 +15206,10 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( let command_buffer = runtime.queue.new_command_buffer(); if !resident_blocks.is_empty() { let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.packet_block_prepare_resident_ht); + encoder.set_compute_pipeline_state(T::packet_block_prepare_pipeline(runtime)); encoder.set_buffer(0, Some(&resident_block_buffer), 0); - encoder.set_buffer(1, Some(&tier1.job_buffer), 0); - encoder.set_buffer(2, Some(&tier1.status_buffer), 0); + encoder.set_buffer(1, Some(tier1.job_buffer()), 0); + encoder.set_buffer(2, Some(tier1.status_buffer()), 0); encoder.set_buffer(3, Some(&packet_block_buffer), 0); encoder.set_bytes( 4, @@ -12512,8 +15223,7 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( depth: 1, }, MTLSize { - width: runtime - .packet_block_prepare_resident_ht + width: T::packet_block_prepare_pipeline(runtime) .thread_execution_width() .max(1), height: 1, @@ -12528,7 +15238,7 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( encoder.set_buffer(0, Some(&resolution_buffer), 0); encoder.set_buffer(1, Some(&subband_buffer), 0); encoder.set_buffer(2, Some(&packet_block_buffer), 0); - encoder.set_buffer(3, Some(&tier1.output_buffer), 0); + encoder.set_buffer(3, Some(tier1.output_buffer()), 0); encoder.set_buffer(4, Some(&output_buffer), 0); encoder.set_buffer(5, Some(&header_buffer), 0); encoder.set_buffer(6, Some(&scratch_buffer), 0); @@ -12540,18 +15250,7 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( encoder.set_buffer(8, Some(&status_buffer), 0); encoder.set_buffer(9, Some(&descriptor_buffer), 0); encoder.set_buffer(10, Some(&state_block_buffer), 0); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - ); + dispatch_single_thread(encoder); encoder.end_encoding(); let encoder = command_buffer.new_compute_command_encoder(); @@ -12562,21 +15261,10 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( encoder.set_bytes( 3, size_of::() as u64, - (&raw const codestream_params).cast(), - ); - encoder.set_buffer(4, Some(&codestream_status_buffer), 0); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, + (&raw const codestream_params).cast(), ); + encoder.set_buffer(4, Some(&codestream_status_buffer), 0); + dispatch_single_thread(encoder); encoder.end_encoding(); command_buffer.commit(); @@ -12586,8 +15274,8 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( status_buffer: codestream_status_buffer, command_buffer: command_buffer.to_owned(), retained_command_buffers: vec![ - tier1.prepare_command_buffer.clone(), - tier1.tier1_command_buffer.clone(), + tier1.prepare_command_buffer().clone(), + tier1.tier1_command_buffer().clone(), ], _retained_buffers: vec![ resolution_buffer, @@ -12600,13 +15288,13 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( header_buffer, scratch_buffer, status_buffer, - tier1.output_buffer.clone(), - tier1.status_buffer.clone(), - tier1.job_buffer.clone(), + tier1.output_buffer().clone(), + tier1.status_buffer().clone(), + tier1.job_buffer().clone(), ], - status_stage: "HTJ2K codestream assembly", - length_error: "HTJ2K Metal codestream output length exceeds usize", - capacity_error: "HTJ2K Metal codestream output length exceeds buffer", + status_stage: T::CODESTREAM_STATUS_STAGE, + length_error: T::CODESTREAM_LENGTH_ERROR, + capacity_error: T::CODESTREAM_CAPACITY_ERROR, }) }) } @@ -12614,7 +15302,8 @@ pub(crate) fn submit_lossless_codestream_buffer_from_resident_ht_tier1( #[cfg(target_os = "macos")] pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( session: &crate::MetalBackendSession, - items: Vec, + items: Vec, + packet_capacity_mode: J2kHtPacketOutputCapacityMode, ) -> Result { if items.is_empty() { return Err(Error::MetalKernel { @@ -12622,49 +15311,24 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( }); } - let mut prepared_tiles = Vec::with_capacity(items.len()); - for item in items { - let J2kPreparedLosslessDeviceCodeBlocks { - coefficient_buffer, - coefficient_byte_offset, - coefficient_byte_len, - coefficient_buffer_is_batch_shared, - code_blocks, - recyclable_private_buffers, - _prepare_command_buffer: prepare_command_buffer, - _deinterleave_status_buffer: deinterleave_status_buffer, - _plane_buffers: plane_buffers, - _scratch_buffers: scratch_buffers, - _coefficient_job_buffer: coefficient_job_buffer, - } = item.prepared; - prepared_tiles.push(PreparedLosslessBatchTile { - coefficient_buffer, - coefficient_byte_offset, - coefficient_byte_len, - coefficient_buffer_is_batch_shared, - code_blocks, - recyclable_private_buffers, - prepare_command_buffer, - deinterleave_status_buffer, - plane_buffers, - scratch_buffers, - coefficient_job_buffer, - resolution_count: item.resolution_count, - num_layers: item.num_layers, - num_components: item.num_components, - code_block_count: item.code_block_count, - packet_descriptors: item.packet_descriptors, - resolutions: item.resolutions, - codestream: item.codestream, - }); - } - - with_runtime_for_device(&session.device, |runtime| { - let command_buffer = runtime.queue.new_command_buffer(); - label_command_buffer(command_buffer, "signinum-j2k htj2k resident encode batch"); + let prepared_tiles = prepared_lossless_batch_tiles(items); + + with_runtime_for_session(session, |runtime| { + let profile_stages = metal_profile_stages_enabled(); + let mut stage_stats = J2kResidentEncodeStageStats::default(); + let mut ht_table_build_duration = Duration::ZERO; + let mut ht_block_encode_duration = Duration::ZERO; + let mut packet_block_prep_duration = Duration::ZERO; + let mut packetization_duration = Duration::ZERO; + let mut codestream_assembly_duration = Duration::ZERO; + let mut ht_table_build_started = profile_stages.then(Instant::now); + let ht_tier1_setup_signpost = hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_HT_TIER1_SETUP); + let split_profile_commands = true; let mut retained_command_buffers = Vec::with_capacity(prepared_tiles.len()); + let mut gpu_stage_command_buffers = Vec::new(); let mut retained_buffers = Vec::::new(); let mut recyclable_private_buffers = Vec::<(usize, Buffer)>::new(); + let mut recyclable_shared_buffers = Vec::<(usize, Buffer)>::new(); let shared_coefficient_buffer = prepared_tiles.first().and_then(|first| { let ptr = first.coefficient_buffer.as_ptr(); prepared_tiles @@ -12675,6 +15339,16 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( }) .then(|| first.coefficient_buffer.clone()) }); + let needs_coefficient_copy = shared_coefficient_buffer.is_none(); + let initial_command_buffer_label = if split_profile_commands && needs_coefficient_copy { + "j2k htj2k resident coefficient copy" + } else if split_profile_commands { + "j2k htj2k resident tier1 encode" + } else { + "j2k htj2k resident encode batch" + }; + let mut command_buffer = + new_resident_encode_command_buffer(runtime, initial_command_buffer_label); let (coefficient_buffer, coefficient_offsets) = if let Some(coefficient_buffer) = shared_coefficient_buffer { @@ -12700,7 +15374,7 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( runtime, total_coefficient_bytes.max(1), &mut recyclable_private_buffers, - ); + )?; let blit = command_buffer.new_blit_command_encoder(); if metal_profile_stages_enabled() { blit.set_label("HTJ2K coefficient prep"); @@ -12708,7 +15382,7 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( for (tile, &dst_offset) in prepared_tiles.iter().zip(coefficient_offsets.iter()) { if tile.coefficient_byte_len > 0 { #[cfg(test)] - HT_BATCH_COEFFICIENT_COPY_BLITS.fetch_add(1, Ordering::Relaxed); + test_counters::record_ht_batch_coefficient_copy_blit(); blit.copy_from_buffer( &tile.coefficient_buffer, tile.coefficient_byte_offset as u64, @@ -12719,16 +15393,21 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( } } blit.end_encoding(); + if split_profile_commands { + command_buffer = finish_resident_encode_split_command_buffer( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::CoefficientCopy, + "j2k htj2k resident tier1 encode", + &mut gpu_stage_command_buffers, + ); + } (coefficient_buffer, coefficient_offsets) }; - let output_capacity_per_job = J2K_HT_ENCODE_BASE_OUTPUT_SIZE; - let output_capacity_per_job_u32 = - u32::try_from(output_capacity_per_job).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal batch output capacity exceeds u32".to_string(), - })?; let mut tier1_jobs = Vec::::new(); let mut tier1_output_capacity_total = 0usize; + let mut max_tier1_output_capacity = 0usize; let mut tile_tier1_job_bases = Vec::::with_capacity(prepared_tiles.len()); for (tile, &coefficient_byte_offset) in prepared_tiles.iter().zip(coefficient_offsets.iter()) @@ -12744,6 +15423,12 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( message: "HTJ2K Metal batch coefficient offset exceeds u32".to_string(), })?; for block in &tile.code_blocks { + let output_capacity_per_job = ht_encode_output_capacity(block.width, block.height)?; + max_tier1_output_capacity = max_tier1_output_capacity.max(output_capacity_per_job); + let output_capacity_per_job_u32 = + u32::try_from(output_capacity_per_job).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal batch output capacity exceeds u32".to_string(), + })?; let output_offset = u32::try_from(tier1_output_capacity_total).map_err(|_| Error::MetalKernel { message: "HTJ2K Metal batch Tier-1 output offset exceeds u32".to_string(), @@ -12775,20 +15460,25 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( runtime, tier1_output_capacity_total.max(1), &mut recyclable_private_buffers, - ); + )?; let tier1_status_buffer = take_recyclable_private_buffer( runtime, tier1_jobs.len().max(1) * size_of::(), &mut recyclable_private_buffers, - ); + )?; let tier1_job_count = u32::try_from(tier1_jobs.len()).map_err(|_| Error::MetalKernel { message: "HTJ2K Metal batch Tier-1 job count exceeds u32".to_string(), })?; - let kernel = HtEncodeCodeBlocksKernel::from_env(runtime); + drop(ht_tier1_setup_signpost); + if let Some(started) = ht_table_build_started.take() { + ht_table_build_duration = ht_table_build_duration.saturating_add(started.elapsed()); + } if tier1_job_count > 0 { + let command_encode_started = profile_stages.then(Instant::now); + let signpost = hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_HT_TIER1_COMMAND_ENCODE); let encoder = command_buffer.new_compute_command_encoder(); label_compute_encoder(encoder, "HTJ2K Tier-1 encode"); - let pipeline = kernel.pipeline(runtime)?; + let pipeline = &runtime.ht_encode_code_blocks; encoder.set_compute_pipeline_state(pipeline); encoder.set_buffer(0, Some(&coefficient_buffer), 0); encoder.set_buffer(1, Some(&tier1_output_buffer), 0); @@ -12802,424 +15492,150 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( size_of::() as u64, (&raw const tier1_job_count).cast(), ); - kernel.dispatch(encoder, pipeline, tier1_job_count); + dispatch_1d_pipeline(encoder, pipeline, u64::from(tier1_job_count)); encoder.end_encoding(); - } - - let mut packet_resolutions = Vec::::new(); - let mut packet_subbands = Vec::::new(); - let mut resident_blocks = Vec::::new(); - let mut packet_descriptors = Vec::::new(); - let mut state_blocks = Vec::::new(); - let mut packet_jobs = Vec::::with_capacity(prepared_tiles.len()); - let mut assembly_jobs = - Vec::::with_capacity(prepared_tiles.len()); - let mut packet_output_capacity_total = 0usize; - let mut header_capacity_total = 0usize; - let mut scratch_words_total = 0usize; - let mut codestream_capacity_total = 0usize; - let mut codestream_offsets = Vec::::with_capacity(prepared_tiles.len()); - let mut codestream_capacities = Vec::::with_capacity(prepared_tiles.len()); - - for (tile_index, tile) in prepared_tiles.iter().enumerate() { - let local_resolution_offset = packet_resolutions.len(); - let local_subband_offset = packet_subbands.len(); - let local_block_offset = resident_blocks.len(); - let local_descriptor_offset = packet_descriptors.len(); - let local_state_block_offset = state_blocks.len(); - let tier1_job_base = tile_tier1_job_bases[tile_index]; - let mut max_tree_nodes = 1usize; - let mut local_subband_count = 0usize; - let mut local_resident_block_count = 0usize; - - for resolution in &tile.resolutions { - let subband_offset = - u32::try_from(local_subband_count).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal batch packet subband offset exceeds u32".to_string(), - })?; - for subband in &resolution.subbands { - let block_offset = u32::try_from(local_resident_block_count).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch packet block offset exceeds u32" - .to_string(), - } - })?; - max_tree_nodes = max_tree_nodes.max(packet_tree_node_count( - subband.num_cbs_x, - subband.num_cbs_y, - )?); - let code_block_start = - usize::try_from(subband.code_block_start).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch packet code-block offset exceeds usize" - .to_string(), - } - })?; - let code_block_count = - usize::try_from(subband.code_block_count).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch packet code-block count exceeds usize" - .to_string(), - } - })?; - let code_block_end = code_block_start - .checked_add(code_block_count) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal batch packet code-block range overflow" - .to_string(), - })?; - if code_block_end > tile.code_blocks.len() { - return Err(Error::MetalKernel { - message: "HTJ2K Metal batch packet code-block range out of bounds" - .to_string(), - }); - } - for tier1_job_index in code_block_start..code_block_end { - resident_blocks.push(J2kResidentPacketBlock { - tier1_job_index: u32::try_from( - tier1_job_base.checked_add(tier1_job_index).ok_or_else(|| { - Error::MetalKernel { - message: "HTJ2K Metal batch Tier-1 index overflow" - .to_string(), - } - })?, - ) - .map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal batch Tier-1 index exceeds u32".to_string(), - })?, - previously_included: 0, - l_block: 3, - block_coding_mode: 1, - }); - } - packet_subbands.push(J2kPacketSubband { - block_offset, - block_count: subband.code_block_count, - num_cbs_x: subband.num_cbs_x, - num_cbs_y: subband.num_cbs_y, - }); - local_subband_count = - local_subband_count - .checked_add(1) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal batch subband count overflow".to_string(), - })?; - local_resident_block_count = local_resident_block_count - .checked_add(code_block_count) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal batch resident block count overflow".to_string(), - })?; - } - packet_resolutions.push(J2kPacketResolution { - subband_offset, - subband_count: u32::try_from(resolution.subbands.len()).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch resolution subband count exceeds u32" - .to_string(), - } - })?, - }); - } - - if tile.resolutions.len() - != usize::try_from(tile.resolution_count).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal batch resolution count exceeds usize".to_string(), - })? - { - return Err(Error::MetalKernel { - message: "HTJ2K Metal batch resolution count mismatch".to_string(), - }); - } - if local_resident_block_count - != usize::try_from(tile.code_block_count).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal batch code-block count exceeds usize".to_string(), - })? - { - return Err(Error::MetalKernel { - message: "HTJ2K Metal batch code-block count mismatch".to_string(), - }); + drop(signpost); + if let Some(started) = command_encode_started { + ht_block_encode_duration = + ht_block_encode_duration.saturating_add(started.elapsed()); } - - let mut state_block_offsets = HashMap::::new(); - for descriptor in &tile.packet_descriptors { - let packet_index = - usize::try_from(descriptor.packet_index).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal batch descriptor packet index exceeds usize" - .to_string(), - })?; - let resolution = packet_resolutions - .get(local_resolution_offset + packet_index) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal batch descriptor packet index out of range" - .to_string(), - })?; - let subband_start = - usize::try_from(resolution.subband_offset).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal batch descriptor subband offset exceeds usize" - .to_string(), - })?; - let subband_count = - usize::try_from(resolution.subband_count).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal batch descriptor subband count exceeds usize" - .to_string(), - })?; - let mut packet_block_count = 0usize; - for subband in &packet_subbands[local_subband_offset + subband_start - ..local_subband_offset + subband_start + subband_count] - { - packet_block_count = packet_block_count - .checked_add(usize::try_from(subband.block_count).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch descriptor block count exceeds usize" - .to_string(), - } - })?) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal batch descriptor block count overflow" - .to_string(), - })?; - } - let (state_block_offset, existing_count) = if let Some(&(offset, count)) = - state_block_offsets.get(&descriptor.state_index) - { - (offset, count) - } else { - let offset = u32::try_from(state_blocks.len() - local_state_block_offset) - .map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal batch state block offset exceeds u32".to_string(), - })?; - for subband in &packet_subbands[local_subband_offset + subband_start - ..local_subband_offset + subband_start + subband_count] - { - for idx in 0..subband.block_count { - let _ = idx; - state_blocks.push(J2kPacketStateBlock { - previously_included: 0, - l_block: 3, - }); - } - } - state_block_offsets - .insert(descriptor.state_index, (offset, packet_block_count)); - (offset, packet_block_count) - }; - if existing_count != packet_block_count { - return Err(Error::MetalKernel { - message: "HTJ2K Metal batch descriptor state layout mismatch".to_string(), - }); - } - packet_descriptors.push(J2kPacketDescriptor { - packet_index: descriptor.packet_index, - state_index: descriptor.state_index, - layer: u32::from(descriptor.layer), - resolution: descriptor.resolution, - component: u32::from(descriptor.component), - precinct_lo: descriptor.precinct as u32, - precinct_hi: (descriptor.precinct >> 32) as u32, - state_block_offset, - }); + if split_profile_commands { + command_buffer = finish_resident_encode_split_command_buffer( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::HtBlock, + "j2k htj2k resident packetization", + &mut gpu_stage_command_buffers, + ); } + } else if split_profile_commands { + label_command_buffer(&command_buffer, "j2k htj2k resident packetization"); + } - let header_capacity = local_resident_block_count - .checked_mul(256) - .and_then(|bytes| bytes.checked_add(4096)) - .map(|bytes| bytes.max(4096)) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal batch packet header capacity overflow".to_string(), - })?; - let packet_output_capacity = tile - .code_blocks - .len() - .checked_mul(output_capacity_per_job) - .and_then(|bytes| { - bytes.checked_add( - header_capacity.saturating_mul( - tile.packet_descriptors - .len() - .max(tile.resolutions.len()) - .max(1), - ), - ) - }) - .and_then(|bytes| bytes.checked_add(1024)) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal batch packet output capacity overflow".to_string(), - })?; - let codestream_capacity = - lossless_codestream_assembly_capacity(packet_output_capacity, tile.codestream)?; - let scratch_words = - max_tree_nodes - .checked_mul(6) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal batch scratch size overflow".to_string(), - })?; - - let packet_output_offset = packet_output_capacity_total; - let header_offset = header_capacity_total; - let scratch_offset = scratch_words_total; - let codestream_offset = codestream_capacity_total; - packet_jobs.push(J2kBatchedPacketEncodeJob { - resolution_offset: u32::try_from(local_resolution_offset).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch resolution offset exceeds u32".to_string(), - } - })?, - subband_offset: u32::try_from(local_subband_offset).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch subband offset exceeds u32".to_string(), - } - })?, - block_offset: u32::try_from(local_block_offset).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch block offset exceeds u32".to_string(), - } - })?, - descriptor_offset: u32::try_from(local_descriptor_offset).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch descriptor offset exceeds u32".to_string(), - } - })?, - state_block_offset: u32::try_from(local_state_block_offset).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch state block offset exceeds u32".to_string(), - } - })?, - output_offset: u32::try_from(packet_output_offset).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch packet output offset exceeds u32".to_string(), - } - })?, - header_offset: u32::try_from(header_offset).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal batch header offset exceeds u32".to_string(), - })?, - scratch_offset: u32::try_from(scratch_offset).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal batch scratch offset exceeds u32".to_string(), - })?, - resolution_count: tile.resolution_count, - num_layers: u32::from(tile.num_layers), - num_components: u32::from(tile.num_components), - code_block_count: tile.code_block_count, - subband_count: u32::try_from(local_subband_count).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch local subband count exceeds u32".to_string(), - } - })?, - descriptor_count: u32::try_from(tile.packet_descriptors.len()).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch descriptor count exceeds u32".to_string(), - } - })?, - output_capacity: u32::try_from(packet_output_capacity).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch packet output capacity exceeds u32".to_string(), - } - })?, - header_capacity: u32::try_from(header_capacity).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch header capacity exceeds u32".to_string(), - } - })?, - scratch_node_capacity: u32::try_from(max_tree_nodes).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch scratch node capacity exceeds u32".to_string(), - } - })?, - }); - assembly_jobs.push(J2kBatchedCodestreamAssemblyJob { - tile_data_offset: u32::try_from(packet_output_offset).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch assembly packet offset exceeds u32".to_string(), - } - })?, - codestream_offset: u32::try_from(codestream_offset).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch codestream offset exceeds u32".to_string(), - } - })?, - width: tile.codestream.width, - height: tile.codestream.height, - num_components: u32::from(tile.codestream.num_components), - bit_depth: u32::from(tile.codestream.bit_depth), - signed_samples: u32::from(tile.codestream.signed), - num_decomposition_levels: u32::from(tile.codestream.num_decomposition_levels), - use_mct: u32::from(tile.codestream.use_mct), - guard_bits: u32::from(tile.codestream.guard_bits), - progression_order: codestream_progression_order_code( - tile.codestream.progression_order, - ), - write_tlm: u32::from(tile.codestream.write_tlm), + ht_table_build_started = profile_stages.then(Instant::now); + let ht_packet_plan_signpost = hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_HT_PACKET_PLAN); + let ResidentBatchPacketPlan { + packet_resolutions, + packet_subbands, + resident_blocks, + packet_descriptors, + state_blocks, + packet_jobs, + assembly_jobs, + packet_output_capacity_total, + packet_payload_copy_job_capacity_total, + max_payload_copy_jobs_per_tile, + header_capacity_total, + scratch_words_total, + codestream_capacity_total, + codestream_offsets, + codestream_capacities, + } = build_resident_batch_packet_plan( + &prepared_tiles, + &tile_tier1_job_bases, + ResidentBatchPacketPlanParams { + family_name: "HTJ2K", + block_coding_mode: 1, high_throughput: 1, - output_capacity: u32::try_from(codestream_capacity).map_err(|_| { - Error::MetalKernel { - message: "HTJ2K Metal batch codestream capacity exceeds u32".to_string(), - } - })?, - }); - codestream_offsets.push(codestream_offset); - codestream_capacities.push(codestream_capacity); - packet_output_capacity_total = packet_output_capacity_total - .checked_add(packet_output_capacity) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal batch packet output total overflow".to_string(), - })?; - header_capacity_total = header_capacity_total - .checked_add(header_capacity) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal batch header total overflow".to_string(), - })?; - scratch_words_total = - scratch_words_total - .checked_add(scratch_words) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal batch scratch total overflow".to_string(), - })?; - codestream_capacity_total = codestream_capacity_total - .checked_add(codestream_capacity) - .ok_or_else(|| Error::MetalKernel { - message: "HTJ2K Metal batch codestream total overflow".to_string(), - })?; - } + code_block_style: 0x40, + }, + |_tile_index, tile, header_capacity| { + ht_packet_output_capacity_for_mode( + tile.code_blocks.len(), + header_capacity, + tile.packet_descriptors.len().max(tile.resolutions.len()), + tile.codestream, + packet_capacity_mode, + ) + }, + )?; - let packet_resolution_buffer = copied_slice_buffer(&runtime.device, &packet_resolutions); - let packet_subband_buffer = copied_slice_buffer(&runtime.device, &packet_subbands); - let resident_block_buffer = copied_slice_buffer(&runtime.device, &resident_blocks); + drop(ht_packet_plan_signpost); + if let Some(started) = ht_table_build_started.take() { + ht_table_build_duration = ht_table_build_duration.saturating_add(started.elapsed()); + } + let ht_buffer_allocation_started = profile_stages.then(Instant::now); + let ht_packet_buffer_setup_signpost = + hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_HT_PACKET_BUFFER_SETUP); + let packet_resolution_buffer = copied_recyclable_shared_slice_buffer( + runtime, + &packet_resolutions, + &mut recyclable_shared_buffers, + )?; + let packet_subband_buffer = copied_recyclable_shared_slice_buffer( + runtime, + &packet_subbands, + &mut recyclable_shared_buffers, + )?; + let resident_block_buffer = copied_recyclable_shared_slice_buffer( + runtime, + &resident_blocks, + &mut recyclable_shared_buffers, + )?; let packet_block_buffer = take_recyclable_private_buffer( runtime, resident_blocks.len().max(1) * size_of::(), &mut recyclable_private_buffers, - ); - let packet_descriptor_buffer = copied_slice_buffer(&runtime.device, &packet_descriptors); - let state_block_buffer = copied_slice_buffer(&runtime.device, &state_blocks); - let packet_output_buffer = take_recyclable_private_buffer( + )?; + let packet_descriptor_buffer = copied_recyclable_shared_slice_buffer( + runtime, + &packet_descriptors, + &mut recyclable_shared_buffers, + )?; + let state_block_buffer = copied_recyclable_shared_slice_buffer( + runtime, + &state_blocks, + &mut recyclable_shared_buffers, + )?; + let packet_payload_copy_job_buffer = take_recyclable_private_buffer( runtime, - packet_output_capacity_total.max(1), + packet_payload_copy_job_capacity_total + .max(1) + .checked_mul(size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal batch packet payload-copy buffer size overflow" + .to_string(), + })?, &mut recyclable_private_buffers, - ); + )?; let header_buffer = take_recyclable_private_buffer( runtime, header_capacity_total.max(1), &mut recyclable_private_buffers, - ); + )?; let scratch_buffer = take_recyclable_private_buffer( runtime, scratch_words_total.max(1) * size_of::(), &mut recyclable_private_buffers, - ); - let packet_job_buffer = copied_slice_buffer(&runtime.device, &packet_jobs); - let packet_status_buffer = take_recyclable_private_buffer( + )?; + let packet_job_buffer = copied_recyclable_shared_slice_buffer( + runtime, + &packet_jobs, + &mut recyclable_shared_buffers, + )?; + let packet_status_buffer = zeroed_recyclable_shared_buffer( runtime, packet_jobs.len().max(1) * size_of::(), - &mut recyclable_private_buffers, - ); - let codestream_job_buffer = copied_slice_buffer(&runtime.device, &assembly_jobs); + &mut recyclable_shared_buffers, + )?; + let codestream_job_buffer = copied_recyclable_shared_slice_buffer( + runtime, + &assembly_jobs, + &mut recyclable_shared_buffers, + )?; let codestream_buffer = runtime.device.new_buffer( codestream_capacity_total.max(1) as u64, MTLResourceOptions::StorageModeShared, ); - let codestream_status_buffer = runtime.device.new_buffer( - (assembly_jobs.len() * size_of::()) as u64, - MTLResourceOptions::StorageModeShared, - ); + let codestream_status_buffer = zeroed_recyclable_shared_buffer( + runtime, + assembly_jobs.len() * size_of::(), + &mut recyclable_shared_buffers, + )?; + drop(ht_packet_buffer_setup_signpost); + if let Some(started) = ht_buffer_allocation_started { + stage_stats.ht_buffer_allocation_duration = started.elapsed(); + } let resident_block_params = J2kResidentPacketBlockParams { block_count: u32::try_from(resident_blocks.len()).map_err(|_| Error::MetalKernel { @@ -13227,7 +15643,14 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( })?, tier1_job_count, }; + + let tile_count = u64::try_from(packet_jobs.len()).map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal batch tile count exceeds u64".to_string(), + })?; if !resident_blocks.is_empty() { + let command_encode_started = profile_stages.then(Instant::now); + let signpost = + hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_HT_PACKET_BLOCK_PREP_COMMAND_ENCODE); let encoder = command_buffer.new_compute_command_encoder(); label_compute_encoder(encoder, "HTJ2K packet block prep"); encoder.set_compute_pipeline_state(&runtime.packet_block_prepare_resident_ht); @@ -13256,11 +15679,26 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( }, ); encoder.end_encoding(); + drop(signpost); + if let Some(started) = command_encode_started { + packet_block_prep_duration = + packet_block_prep_duration.saturating_add(started.elapsed()); + } + if split_profile_commands { + command_buffer = finish_resident_encode_split_command_buffer( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::PacketBlockPrep, + "j2k htj2k resident packetization", + &mut gpu_stage_command_buffers, + ); + } + } else if split_profile_commands { + label_command_buffer(&command_buffer, "j2k htj2k resident packetization"); } - - let tile_count = u64::try_from(packet_jobs.len()).map_err(|_| Error::MetalKernel { - message: "HTJ2K Metal batch tile count exceeds u64".to_string(), - })?; + let command_encode_started = profile_stages.then(Instant::now); + let signpost = + hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_HT_PACKETIZATION_COMMAND_ENCODE); let encoder = command_buffer.new_compute_command_encoder(); label_compute_encoder(encoder, "HTJ2K packetization"); encoder.set_compute_pipeline_state(&runtime.packet_encode_batched); @@ -13268,13 +15706,14 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( encoder.set_buffer(1, Some(&packet_subband_buffer), 0); encoder.set_buffer(2, Some(&packet_block_buffer), 0); encoder.set_buffer(3, Some(&tier1_output_buffer), 0); - encoder.set_buffer(4, Some(&packet_output_buffer), 0); + encoder.set_buffer(4, Some(&codestream_buffer), 0); encoder.set_buffer(5, Some(&header_buffer), 0); encoder.set_buffer(6, Some(&scratch_buffer), 0); encoder.set_buffer(7, Some(&packet_job_buffer), 0); encoder.set_buffer(8, Some(&packet_status_buffer), 0); encoder.set_buffer(9, Some(&packet_descriptor_buffer), 0); encoder.set_buffer(10, Some(&state_block_buffer), 0); + encoder.set_buffer(11, Some(&packet_payload_copy_job_buffer), 0); encoder.dispatch_threads( MTLSize { width: tile_count, @@ -13291,11 +15730,55 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( }, ); encoder.end_encoding(); + drop(signpost); + if let Some(started) = command_encode_started { + packetization_duration = packetization_duration.saturating_add(started.elapsed()); + } + if split_profile_commands { + command_buffer = finish_resident_encode_split_command_buffer( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::Packetization, + "j2k htj2k resident packet payload copy", + &mut gpu_stage_command_buffers, + ); + } + let packet_payload_copy_dispatched = dispatch_batched_packet_payload_copy( + runtime, + &command_buffer, + J2kBatchedPacketPayloadCopyDispatch { + payload_buffer: &tier1_output_buffer, + packet_output_buffer: &codestream_buffer, + packet_job_buffer: &packet_job_buffer, + packet_status_buffer: &packet_status_buffer, + packet_payload_copy_job_buffer: &packet_payload_copy_job_buffer, + tile_count, + max_payload_copy_jobs_per_tile: max_payload_copy_jobs_per_tile as u64, + label: "HTJ2K packetization payload copy", + signpost_name: SIGNPOST_ENCODE_HYBRID_HT_PAYLOAD_COPY_COMMAND_ENCODE, + }, + ); + if split_profile_commands { + if packet_payload_copy_dispatched { + command_buffer = finish_resident_encode_split_command_buffer( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::PacketPayloadCopy, + "j2k htj2k resident codestream assembly", + &mut gpu_stage_command_buffers, + ); + } else { + label_command_buffer(&command_buffer, "j2k htj2k resident codestream assembly"); + } + } + let command_encode_started = profile_stages.then(Instant::now); + let signpost = + hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_HT_CODESTREAM_ASSEMBLY_COMMAND_ENCODE); let encoder = command_buffer.new_compute_command_encoder(); label_compute_encoder(encoder, "HTJ2K codestream assembly"); encoder.set_compute_pipeline_state(&runtime.lossless_codestream_assemble_batched); - encoder.set_buffer(0, Some(&packet_output_buffer), 0); + encoder.set_buffer(0, Some(&codestream_buffer), 0); encoder.set_buffer(1, Some(&packet_status_buffer), 0); encoder.set_buffer(2, Some(&codestream_buffer), 0); encoder.set_buffer(3, Some(&codestream_job_buffer), 0); @@ -13316,9 +15799,136 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( }, ); encoder.end_encoding(); + drop(signpost); + let max_packet_output_capacity = packet_jobs + .iter() + .map(|job| job.output_capacity) + .max() + .unwrap_or(0); + let max_packet_output_capacity_usize = usize::try_from(max_packet_output_capacity) + .map_err(|_| Error::MetalKernel { + message: "HTJ2K Metal batch max packet output capacity exceeds usize".to_string(), + })?; + if split_profile_commands { + command_buffer = finish_resident_encode_split_command_buffer( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::CodestreamAssembly, + "j2k htj2k resident result readback", + &mut gpu_stage_command_buffers, + ); + } + let codestream_payload_copy_dispatched = false; + if let Some(started) = command_encode_started { + codestream_assembly_duration = + codestream_assembly_duration.saturating_add(started.elapsed()); + } + let tier1_status_readback = schedule_resident_tier1_status_readback( + runtime, + &command_buffer, + &tier1_status_buffer, + J2kResidentTier1StatusKind::HighThroughput, + 0, + None, + tier1_jobs.len(), + size_of::(), + profile_stages, + )?; command_buffer.commit(); + if split_profile_commands && codestream_payload_copy_dispatched { + gpu_stage_command_buffers.push(J2kResidentEncodeGpuStageCommandBuffer { + stage: J2kResidentEncodeGpuStage::CodestreamPayloadCopy, + command_buffer: command_buffer.clone(), + }); + } + + if profile_stages { + let mut prepare_command_buffer_ptrs = Vec::new(); + for tile in &prepared_tiles { + let mut pushed_split_prepare = false; + for (stage, command_buffer) in [ + ( + J2kResidentEncodeGpuStage::CoefficientDeinterleaveRct, + tile.prepare_deinterleave_rct_command_buffer.as_ref(), + ), + ( + J2kResidentEncodeGpuStage::CoefficientDwt53, + tile.prepare_dwt53_command_buffer.as_ref(), + ), + ( + J2kResidentEncodeGpuStage::CoefficientExtract, + tile.prepare_coefficient_extract_command_buffer.as_ref(), + ), + ] { + if let Some(command_buffer) = command_buffer { + let ptr = command_buffer.as_ptr(); + if prepare_command_buffer_ptrs.contains(&ptr) { + continue; + } + prepare_command_buffer_ptrs.push(ptr); + pushed_split_prepare = true; + gpu_stage_command_buffers.push(J2kResidentEncodeGpuStageCommandBuffer { + stage, + command_buffer: command_buffer.clone(), + }); + } + } + for command_buffer in &tile.prepare_dwt53_vertical_command_buffers { + let ptr = command_buffer.as_ptr(); + if prepare_command_buffer_ptrs.contains(&ptr) { + continue; + } + prepare_command_buffer_ptrs.push(ptr); + pushed_split_prepare = true; + gpu_stage_command_buffers.push(J2kResidentEncodeGpuStageCommandBuffer { + stage: J2kResidentEncodeGpuStage::CoefficientDwt53Vertical, + command_buffer: command_buffer.clone(), + }); + } + for command_buffer in &tile.prepare_dwt53_horizontal_command_buffers { + let ptr = command_buffer.as_ptr(); + if prepare_command_buffer_ptrs.contains(&ptr) { + continue; + } + prepare_command_buffer_ptrs.push(ptr); + pushed_split_prepare = true; + gpu_stage_command_buffers.push(J2kResidentEncodeGpuStageCommandBuffer { + stage: J2kResidentEncodeGpuStage::CoefficientDwt53Horizontal, + command_buffer: command_buffer.clone(), + }); + } + if pushed_split_prepare { + continue; + } + let ptr = tile.prepare_command_buffer.as_ptr(); + if prepare_command_buffer_ptrs.contains(&ptr) { + continue; + } + prepare_command_buffer_ptrs.push(ptr); + gpu_stage_command_buffers.push(J2kResidentEncodeGpuStageCommandBuffer { + stage: J2kResidentEncodeGpuStage::CoefficientPrep, + command_buffer: tile.prepare_command_buffer.clone(), + }); + } + } + retained_command_buffers.extend( + gpu_stage_command_buffers + .iter() + .map(|stage_command_buffer| stage_command_buffer.command_buffer.clone()), + ); for tile in prepared_tiles { + if let Some(command_buffer) = tile.prepare_deinterleave_rct_command_buffer { + retained_command_buffers.push(command_buffer); + } + if let Some(command_buffer) = tile.prepare_dwt53_command_buffer { + retained_command_buffers.push(command_buffer); + } + retained_command_buffers.extend(tile.prepare_dwt53_vertical_command_buffers); + retained_command_buffers.extend(tile.prepare_dwt53_horizontal_command_buffers); + if let Some(command_buffer) = tile.prepare_coefficient_extract_command_buffer { + retained_command_buffers.push(command_buffer); + } retained_command_buffers.push(tile.prepare_command_buffer); retained_buffers.push(tile.coefficient_buffer); retained_buffers.push(tile.deinterleave_status_buffer); @@ -13337,23 +15947,57 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( retained_buffers.push(packet_block_buffer); retained_buffers.push(packet_descriptor_buffer); retained_buffers.push(state_block_buffer); - retained_buffers.push(packet_output_buffer); + retained_buffers.push(packet_payload_copy_job_buffer); retained_buffers.push(header_buffer); retained_buffers.push(scratch_buffer); retained_buffers.push(packet_job_buffer); - retained_buffers.push(packet_status_buffer); + retained_buffers.push(packet_status_buffer.clone()); retained_buffers.push(codestream_job_buffer); + stage_stats.ht_table_build_duration = ht_table_build_duration; + stage_stats.ht_block_encode_duration = ht_block_encode_duration; + stage_stats.packet_block_prep_duration = packet_block_prep_duration; + stage_stats.packetization_duration = packetization_duration; + stage_stats.codestream_assembly_duration = codestream_assembly_duration; + stage_stats.ht_command_encode_duration = ht_block_encode_duration + .saturating_add(packet_block_prep_duration) + .saturating_add(packetization_duration) + .saturating_add(codestream_assembly_duration); + stage_stats.packet_payload_copy_job_capacity_total = packet_payload_copy_job_capacity_total; + stage_stats.max_packet_payload_copy_jobs_per_tile = max_payload_copy_jobs_per_tile; + stage_stats.packet_payload_copy_launched_stripe_count_total = packet_jobs + .len() + .saturating_mul(max_payload_copy_jobs_per_tile) + .saturating_mul(PACKET_PAYLOAD_COPY_STRIPES_PER_JOB as usize); + stage_stats.tier1_output_capacity_total = tier1_output_capacity_total; + stage_stats.max_tier1_output_capacity = max_tier1_output_capacity; + stage_stats.packet_output_capacity_total = packet_output_capacity_total; + stage_stats.max_packet_output_capacity = max_packet_output_capacity_usize; + stage_stats.codestream_payload_copy_launched_thread_count_total = 0; + stage_stats.code_block_count = tier1_jobs.len(); + Ok(J2kPendingResidentLosslessCodestreamBatch { - device: runtime.device.clone(), + runtime: session.runtime()?, buffer: codestream_buffer, byte_offsets: codestream_offsets, capacities: codestream_capacities, status_buffer: codestream_status_buffer, - command_buffer: command_buffer.to_owned(), + packet_status_buffer, + tier1_status_readback, + classic_tier1_density_readback: None, + classic_tier1_symbol_plan_readback: None, + classic_tier1_pass_plan_readback: None, + classic_tier1_token_emit_readback: None, + classic_tier1_split_token_emit_readback: None, + classic_gpu_token_pack_used: false, + command_buffer, retained_command_buffers, _retained_buffers: retained_buffers, recyclable_private_buffers, + recyclable_shared_buffers, + gpu_stage_command_buffers, + stage_stats, + codestream_payload_copy_dispatched, status_stage: "HTJ2K batched codestream assembly", length_error: "HTJ2K Metal batched codestream output length exceeds usize", capacity_error: "HTJ2K Metal batched codestream output length exceeds buffer", @@ -13364,7 +16008,8 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_ht_batch( #[cfg(target_os = "macos")] pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( session: &crate::MetalBackendSession, - items: Vec, + items: Vec, + output_capacity_mode: J2kClassicEncodeOutputCapacityMode, ) -> Result { if items.is_empty() { return Err(Error::MetalKernel { @@ -13372,47 +16017,34 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( }); } - let mut prepared_tiles = Vec::with_capacity(items.len()); - for item in items { - let J2kPreparedLosslessDeviceCodeBlocks { - coefficient_buffer, - coefficient_byte_offset, - coefficient_byte_len, - coefficient_buffer_is_batch_shared, - code_blocks, - recyclable_private_buffers, - _prepare_command_buffer: prepare_command_buffer, - _deinterleave_status_buffer: deinterleave_status_buffer, - _plane_buffers: plane_buffers, - _scratch_buffers: scratch_buffers, - _coefficient_job_buffer: coefficient_job_buffer, - } = item.prepared; - prepared_tiles.push(PreparedLosslessBatchTile { - coefficient_buffer, - coefficient_byte_offset, - coefficient_byte_len, - coefficient_buffer_is_batch_shared, - code_blocks, - recyclable_private_buffers, - prepare_command_buffer, - deinterleave_status_buffer, - plane_buffers, - scratch_buffers, - coefficient_job_buffer, - resolution_count: item.resolution_count, - num_layers: item.num_layers, - num_components: item.num_components, - code_block_count: item.code_block_count, - packet_descriptors: item.packet_descriptors, - resolutions: item.resolutions, - codestream: item.codestream, - }); - } - - with_runtime_for_device(&session.device, |runtime| { - let command_buffer = runtime.queue.new_command_buffer(); + let prepared_tiles = prepared_lossless_batch_tiles(items); + + with_runtime_for_session(session, |runtime| { + let profile_stages = metal_profile_stages_enabled(); + // Commit classic stages independently so the long Tier-1 kernel can run + // while CPU packet metadata for the following stages is built. + let split_command_buffers = true; + let mut stage_stats = J2kResidentEncodeStageStats::default(); + let mut classic_tier1_setup_duration = Duration::ZERO; + let mut classic_block_encode_duration = Duration::ZERO; + let mut classic_packet_plan_duration = Duration::ZERO; + let mut classic_packet_buffer_setup_duration = Duration::ZERO; + let mut classic_command_buffer_commit_duration = Duration::ZERO; + let packet_block_prep_duration = Duration::ZERO; + let mut packetization_duration = Duration::ZERO; + let mut codestream_assembly_duration = Duration::ZERO; let mut retained_command_buffers = Vec::with_capacity(prepared_tiles.len()); + let mut gpu_stage_command_buffers = Vec::new(); let mut retained_buffers = Vec::::new(); + let profile_classic_tier1_density = metal_profile_classic_tier1_density_enabled(); + let profile_classic_tier1_raw_pack = metal_profile_classic_tier1_raw_pack_enabled(); + let profile_classic_tier1_arithmetic_pack = + metal_profile_classic_tier1_arithmetic_pack_enabled(); + let profile_classic_tier1_pass_plan = metal_profile_classic_tier1_pass_plan_enabled(); + let profile_classic_tier1_symbol_plan = metal_profile_classic_tier1_symbol_plan_enabled(); + let profile_classic_tier1_token_emit = metal_profile_classic_tier1_token_emit_enabled(); + let profile_classic_tier1_split_token_emit = + metal_profile_classic_tier1_split_token_emit_enabled(); let shared_coefficient_buffer = prepared_tiles.first().and_then(|first| { let ptr = first.coefficient_buffer.as_ptr(); prepared_tiles @@ -13423,6 +16055,16 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( }) .then(|| first.coefficient_buffer.clone()) }); + let needs_coefficient_copy = shared_coefficient_buffer.is_none(); + let initial_command_buffer_label = if split_command_buffers && needs_coefficient_copy { + "j2k classic resident coefficient copy" + } else if split_command_buffers { + "j2k classic resident Tier-1 encode" + } else { + "j2k classic resident encode batch" + }; + let mut command_buffer = + new_resident_encode_command_buffer(runtime, initial_command_buffer_label); let (coefficient_buffer, coefficient_offsets) = if let Some(coefficient_buffer) = shared_coefficient_buffer { ( @@ -13448,6 +16090,9 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( MTLResourceOptions::StorageModePrivate, ); let blit = command_buffer.new_blit_command_encoder(); + if profile_stages { + blit.set_label("J2K coefficient prep"); + } for (tile, &dst_offset) in prepared_tiles.iter().zip(coefficient_offsets.iter()) { if tile.coefficient_byte_len > 0 { blit.copy_from_buffer( @@ -13460,13 +16105,28 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( } } blit.end_encoding(); + if split_command_buffers { + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::CoefficientCopy, + "j2k classic resident Tier-1 encode", + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } (coefficient_buffer, coefficient_offsets) }; + let classic_tier1_setup_started = profile_stages.then(Instant::now); + let classic_tier1_setup_signpost = + hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_SETUP); + let classic_resident_style_flags = classic_resident_style_flags_from_env(); let mut tier1_jobs = Vec::::new(); let mut tier1_output_capacity_total = 0usize; + let mut max_tier1_output_capacity = 0usize; let mut tier1_segment_capacity_total = 0usize; - let segment_capacity_per_job = 256usize; let mut tile_tier1_job_bases = Vec::::with_capacity(prepared_tiles.len()); let mut tile_tier1_output_capacities = Vec::::with_capacity(prepared_tiles.len()); for (tile, &coefficient_byte_offset) in @@ -13484,11 +16144,13 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( message: "J2K Metal batch coefficient offset exceeds u32".to_string(), })?; for block in &tile.code_blocks { - let output_capacity = classic_encode_output_capacity( + let output_capacity = classic_encode_output_capacity_for_mode( block.width, block.height, block.total_bitplanes, + output_capacity_mode, )?; + max_tier1_output_capacity = max_tier1_output_capacity.max(output_capacity); let output_offset = u32::try_from(tier1_output_capacity_total).map_err(|_| Error::MetalKernel { message: "J2K Metal batch Tier-1 output offset exceeds u32".to_string(), @@ -13504,6 +16166,10 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( .ok_or_else(|| Error::MetalKernel { message: "J2K Metal batch coefficient offset overflow".to_string(), })?; + let segment_capacity = classic_encode_segment_capacity( + classic_resident_style_flags, + block.total_bitplanes, + ); tier1_jobs.push(J2kClassicEncodeBatchJob { coefficient_offset, output_offset, @@ -13512,14 +16178,14 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( height: block.height, sub_band_type: classic_encode_sub_band_code(block.sub_band_type), total_bitplanes: u32::from(block.total_bitplanes), - style_flags: 0, + style_flags: classic_resident_style_flags, output_capacity: u32::try_from(output_capacity).map_err(|_| { Error::MetalKernel { message: "J2K Metal batch Tier-1 output capacity exceeds u32" .to_string(), } })?, - segment_capacity: u32::try_from(segment_capacity_per_job).map_err(|_| { + segment_capacity: u32::try_from(segment_capacity).map_err(|_| { Error::MetalKernel { message: "J2K Metal batch Tier-1 segment capacity exceeds u32" .to_string(), @@ -13532,7 +16198,7 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( message: "J2K Metal batch Tier-1 output buffer overflow".to_string(), })?; tier1_segment_capacity_total = tier1_segment_capacity_total - .checked_add(segment_capacity_per_job) + .checked_add(segment_capacity) .ok_or_else(|| Error::MetalKernel { message: "J2K Metal batch Tier-1 segment buffer overflow".to_string(), })?; @@ -13550,439 +16216,558 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( message: "J2K Metal batch Tier-1 job count exceeds u32".to_string(), })?; let tier1_job_buffer = owned_slice_buffer(&runtime.device, &tier1_jobs); - let tier1_output_buffer = runtime.device.new_buffer( - tier1_output_capacity_total.max(1) as u64, - MTLResourceOptions::StorageModePrivate, - ); - let tier1_status_buffer = runtime.device.new_buffer( - (tier1_jobs.len().max(1) * size_of::()) as u64, - MTLResourceOptions::StorageModePrivate, - ); - let tier1_segment_buffer = runtime.device.new_buffer( - (tier1_segment_capacity_total.max(1) * size_of::()) as u64, - MTLResourceOptions::StorageModePrivate, - ); - if tier1_job_count > 0 { - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.classic_encode_code_blocks); - encoder.set_buffer(0, Some(&coefficient_buffer), 0); - encoder.set_buffer(1, Some(&tier1_output_buffer), 0); - encoder.set_buffer(2, Some(&tier1_job_buffer), 0); - encoder.set_buffer(3, Some(&tier1_status_buffer), 0); - encoder.set_buffer(4, Some(&tier1_segment_buffer), 0); - encoder.set_bytes( - 5, - size_of::() as u64, - (&raw const tier1_job_count).cast(), - ); - encoder.dispatch_threads( - MTLSize { - width: u64::from(tier1_job_count), - height: 1, - depth: 1, - }, - MTLSize { - width: runtime - .classic_encode_code_blocks - .thread_execution_width() - .max(1), - height: 1, - depth: 1, - }, - ); - encoder.end_encoding(); + let mut recyclable_private_buffers = Vec::<(usize, Buffer)>::new(); + let mut recyclable_shared_buffers = Vec::<(usize, Buffer)>::new(); + let tier1_output_buffer = take_recyclable_private_buffer( + runtime, + tier1_output_capacity_total.max(1), + &mut recyclable_private_buffers, + )?; + let tier1_status_buffer = take_recyclable_private_buffer( + runtime, + tier1_jobs.len().max(1) * size_of::(), + &mut recyclable_private_buffers, + )?; + let tier1_segment_buffer = take_recyclable_private_buffer( + runtime, + tier1_segment_capacity_total.max(1) * size_of::(), + &mut recyclable_private_buffers, + )?; + drop(classic_tier1_setup_signpost); + if let Some(started) = classic_tier1_setup_started { + classic_tier1_setup_duration = started.elapsed(); } - - let mut packet_resolutions = Vec::::new(); - let mut packet_subbands = Vec::::new(); - let mut resident_blocks = Vec::::new(); - let mut packet_descriptors = Vec::::new(); - let mut state_blocks = Vec::::new(); - let mut packet_jobs = Vec::::with_capacity(prepared_tiles.len()); - let mut assembly_jobs = - Vec::::with_capacity(prepared_tiles.len()); - let mut packet_output_capacity_total = 0usize; - let mut header_capacity_total = 0usize; - let mut scratch_words_total = 0usize; - let mut codestream_capacity_total = 0usize; - let mut codestream_offsets = Vec::::with_capacity(prepared_tiles.len()); - let mut codestream_capacities = Vec::::with_capacity(prepared_tiles.len()); - - for (tile_index, tile) in prepared_tiles.iter().enumerate() { - let local_resolution_offset = packet_resolutions.len(); - let local_subband_offset = packet_subbands.len(); - let local_block_offset = resident_blocks.len(); - let local_descriptor_offset = packet_descriptors.len(); - let local_state_block_offset = state_blocks.len(); - let tier1_job_base = tile_tier1_job_bases[tile_index]; - let mut max_tree_nodes = 1usize; - let mut local_subband_count = 0usize; - let mut local_resident_block_count = 0usize; - - for resolution in &tile.resolutions { - let subband_offset = - u32::try_from(local_subband_count).map_err(|_| Error::MetalKernel { - message: "J2K Metal batch packet subband offset exceeds u32".to_string(), - })?; - for subband in &resolution.subbands { - let block_offset = u32::try_from(local_resident_block_count).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch packet block offset exceeds u32".to_string(), - } - })?; - max_tree_nodes = max_tree_nodes.max(packet_tree_node_count( - subband.num_cbs_x, - subband.num_cbs_y, - )?); - let code_block_start = - usize::try_from(subband.code_block_start).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch packet code-block offset exceeds usize" - .to_string(), - } - })?; - let code_block_count = - usize::try_from(subband.code_block_count).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch packet code-block count exceeds usize" - .to_string(), - } - })?; - let code_block_end = code_block_start - .checked_add(code_block_count) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal batch packet code-block range overflow".to_string(), - })?; - if code_block_end > tile.code_blocks.len() { - return Err(Error::MetalKernel { - message: "J2K Metal batch packet code-block range out of bounds" - .to_string(), - }); - } - for tier1_job_index in code_block_start..code_block_end { - resident_blocks.push(J2kResidentPacketBlock { - tier1_job_index: u32::try_from( - tier1_job_base.checked_add(tier1_job_index).ok_or_else(|| { - Error::MetalKernel { - message: "J2K Metal batch Tier-1 index overflow" - .to_string(), - } - })?, - ) - .map_err(|_| Error::MetalKernel { - message: "J2K Metal batch Tier-1 index exceeds u32".to_string(), - })?, - previously_included: 0, - l_block: 3, - block_coding_mode: 0, - }); - } - packet_subbands.push(J2kPacketSubband { - block_offset, - block_count: subband.code_block_count, - num_cbs_x: subband.num_cbs_x, - num_cbs_y: subband.num_cbs_y, + let classic_split_mq_byte_gpu_token_pack_requested = + classic_tier1_split_mq_byte_gpu_token_pack_requested(); + let classic_split_mq_byte_gpu_token_pack_disabled = + classic_tier1_split_mq_byte_gpu_token_pack_disabled(); + let classic_split_gpu_token_pack_requested = classic_tier1_split_gpu_token_pack_requested(); + let classic_gpu_token_pack_requested = classic_tier1_gpu_token_pack_requested(); + let use_classic_split_mq_byte_gpu_token_pack = if tier1_job_count > 0 { + if classic_split_mq_byte_gpu_token_pack_requested { + if !classic_tier1_gpu_token_pack_supported(&tier1_jobs) { + return Err(Error::MetalKernel { + message: "J2K Metal classic split MQ-byte GPU token-pack route currently supports only bypass_u16_32 resident jobs".to_string(), }); - local_subband_count = - local_subband_count - .checked_add(1) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal batch subband count overflow".to_string(), - })?; - local_resident_block_count = local_resident_block_count - .checked_add(code_block_count) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal batch resident block count overflow".to_string(), - })?; } - packet_resolutions.push(J2kPacketResolution { - subband_offset, - subband_count: u32::try_from(resolution.subbands.len()).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch resolution subband count exceeds u32" - .to_string(), - } - })?, - }); + true + } else { + !classic_split_mq_byte_gpu_token_pack_disabled + && !classic_split_gpu_token_pack_requested + && !classic_gpu_token_pack_requested + && classic_tier1_gpu_token_pack_supported(&tier1_jobs) } - - if tile.resolutions.len() - != usize::try_from(tile.resolution_count).map_err(|_| Error::MetalKernel { - message: "J2K Metal batch resolution count exceeds usize".to_string(), - })? - { + } else { + false + }; + let use_classic_split_gpu_token_pack = if classic_split_gpu_token_pack_requested + && !use_classic_split_mq_byte_gpu_token_pack + && tier1_job_count > 0 + { + if !classic_tier1_gpu_token_pack_supported(&tier1_jobs) { return Err(Error::MetalKernel { - message: "J2K Metal batch resolution count mismatch".to_string(), + message: "J2K Metal classic split GPU token-pack route currently supports only bypass_u16_32 resident jobs".to_string(), }); } - if local_resident_block_count - != usize::try_from(tile.code_block_count).map_err(|_| Error::MetalKernel { - message: "J2K Metal batch code-block count exceeds usize".to_string(), - })? - { + true + } else { + false + }; + let use_classic_gpu_token_pack = if !use_classic_split_mq_byte_gpu_token_pack + && !use_classic_split_gpu_token_pack + && classic_gpu_token_pack_requested + && tier1_job_count > 0 + { + if !classic_tier1_gpu_token_pack_supported(&tier1_jobs) { return Err(Error::MetalKernel { - message: "J2K Metal batch code-block count mismatch".to_string(), + message: "J2K Metal classic GPU token-pack route currently supports only bypass_u16_32 resident jobs".to_string(), }); } - - let mut state_block_offsets = HashMap::::new(); - for descriptor in &tile.packet_descriptors { - let packet_index = - usize::try_from(descriptor.packet_index).map_err(|_| Error::MetalKernel { - message: "J2K Metal batch descriptor packet index exceeds usize" - .to_string(), - })?; - let resolution = packet_resolutions - .get(local_resolution_offset + packet_index) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal batch descriptor packet index out of range".to_string(), - })?; - let subband_start = - usize::try_from(resolution.subband_offset).map_err(|_| Error::MetalKernel { - message: "J2K Metal batch descriptor subband offset exceeds usize" - .to_string(), - })?; - let subband_count = - usize::try_from(resolution.subband_count).map_err(|_| Error::MetalKernel { - message: "J2K Metal batch descriptor subband count exceeds usize" - .to_string(), - })?; - let mut packet_block_count = 0usize; - for subband in &packet_subbands[local_subband_offset + subband_start - ..local_subband_offset + subband_start + subband_count] - { - packet_block_count = packet_block_count - .checked_add(usize::try_from(subband.block_count).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch descriptor block count exceeds usize" - .to_string(), - } - })?) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal batch descriptor block count overflow".to_string(), - })?; - } - let (state_block_offset, existing_count) = if let Some(&(offset, count)) = - state_block_offsets.get(&descriptor.state_index) - { - (offset, count) + true + } else { + false + }; + let mut classic_gpu_token_pack_readback = None; + if tier1_job_count > 0 { + let command_encode_started = profile_stages.then(Instant::now); + let signpost = + hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_COMMAND_ENCODE); + if use_classic_split_mq_byte_gpu_token_pack || use_classic_split_gpu_token_pack { + let token_buffers = dispatch_classic_tier1_split_token_emit_for_gpu_pack( + runtime, + &command_buffer, + &coefficient_buffer, + &tier1_job_buffer, + &tier1_jobs, + &mut recyclable_private_buffers, + use_classic_split_mq_byte_gpu_token_pack, + )?; + if split_command_buffers { + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::ClassicTier1SplitTokenEmit, + "j2k classic resident Tier-1 split token pack", + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + dispatch_classic_tier1_split_token_pack_from_gpu_tokens( + runtime, + &command_buffer, + &tier1_job_buffer, + &token_buffers, + &tier1_output_buffer, + &tier1_status_buffer, + &tier1_segment_buffer, + ); + drop(signpost); + if let Some(started) = command_encode_started { + classic_block_encode_duration = + classic_block_encode_duration.saturating_add(started.elapsed()); + } + if split_command_buffers { + let next_label = if profile_classic_tier1_density { + "j2k classic resident Tier-1 density profile" + } else if profile_classic_tier1_raw_pack { + "j2k classic resident Tier-1 raw-pack profile" + } else if profile_classic_tier1_arithmetic_pack { + "j2k classic resident Tier-1 arithmetic-pack profile" + } else if profile_classic_tier1_symbol_plan { + "j2k classic resident Tier-1 symbol plan" + } else if profile_classic_tier1_token_emit { + "j2k classic resident Tier-1 token emit" + } else if profile_classic_tier1_split_token_emit { + "j2k classic resident Tier-1 split token emit" + } else { + "j2k classic resident packetization" + }; + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::ClassicTier1TokenPack, + next_label, + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + } else if use_classic_gpu_token_pack { + let token_buffers = dispatch_classic_tier1_token_emit_for_gpu_pack( + runtime, + &command_buffer, + &coefficient_buffer, + &tier1_job_buffer, + &tier1_jobs, + &mut recyclable_private_buffers, + )?; + if split_command_buffers { + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::ClassicTier1TokenEmit, + "j2k classic resident Tier-1 token pack", + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + dispatch_classic_tier1_token_pack_from_gpu_tokens( + runtime, + &command_buffer, + &tier1_job_buffer, + &token_buffers, + &tier1_output_buffer, + &tier1_status_buffer, + &tier1_segment_buffer, + ); + classic_gpu_token_pack_readback = schedule_classic_tier1_gpu_token_pack_readback( + runtime, + &command_buffer, + &token_buffers, + profile_stages, + )?; + drop(signpost); + if let Some(started) = command_encode_started { + classic_block_encode_duration = + classic_block_encode_duration.saturating_add(started.elapsed()); + } + if split_command_buffers { + let next_label = if profile_classic_tier1_density { + "j2k classic resident Tier-1 density profile" + } else if profile_classic_tier1_raw_pack { + "j2k classic resident Tier-1 raw-pack profile" + } else if profile_classic_tier1_arithmetic_pack { + "j2k classic resident Tier-1 arithmetic-pack profile" + } else if profile_classic_tier1_symbol_plan { + "j2k classic resident Tier-1 symbol plan" + } else if profile_classic_tier1_token_emit { + "j2k classic resident Tier-1 token emit" + } else if profile_classic_tier1_split_token_emit { + "j2k classic resident Tier-1 split token emit" + } else { + "j2k classic resident packetization" + }; + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::ClassicTier1TokenPack, + next_label, + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + } else { + let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K Tier-1 encode"); + let classic_encode_pipeline = + classic_encode_code_blocks_pipeline(runtime, &tier1_jobs); + encoder.set_compute_pipeline_state(classic_encode_pipeline); + encoder.set_buffer(0, Some(&coefficient_buffer), 0); + encoder.set_buffer(1, Some(&tier1_output_buffer), 0); + encoder.set_buffer(2, Some(&tier1_job_buffer), 0); + encoder.set_buffer(3, Some(&tier1_status_buffer), 0); + encoder.set_buffer(4, Some(&tier1_segment_buffer), 0); + encoder.set_bytes( + 5, + size_of::() as u64, + (&raw const tier1_job_count).cast(), + ); + dispatch_1d_pipeline(encoder, classic_encode_pipeline, u64::from(tier1_job_count)); + encoder.end_encoding(); + drop(signpost); + if let Some(started) = command_encode_started { + classic_block_encode_duration = + classic_block_encode_duration.saturating_add(started.elapsed()); + } + if split_command_buffers { + let next_label = if profile_classic_tier1_density { + "j2k classic resident Tier-1 density profile" + } else if profile_classic_tier1_raw_pack { + "j2k classic resident Tier-1 raw-pack profile" + } else if profile_classic_tier1_arithmetic_pack { + "j2k classic resident Tier-1 arithmetic-pack profile" + } else if profile_classic_tier1_symbol_plan { + "j2k classic resident Tier-1 symbol plan" + } else if profile_classic_tier1_token_emit { + "j2k classic resident Tier-1 token emit" + } else if profile_classic_tier1_split_token_emit { + "j2k classic resident Tier-1 split token emit" + } else { + "j2k classic resident packetization" + }; + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::ClassicBlock, + next_label, + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + } + } else if split_command_buffers { + label_command_buffer(&command_buffer, "j2k classic resident packetization"); + } + let classic_tier1_density_readback = if tier1_job_count > 0 { + let readback = dispatch_classic_tier1_density_profile( + runtime, + &command_buffer, + &coefficient_buffer, + &tier1_job_buffer, + &tier1_jobs, + )?; + if readback.is_some() && split_command_buffers { + let next_label = if profile_classic_tier1_raw_pack { + "j2k classic resident Tier-1 raw-pack profile" + } else if profile_classic_tier1_arithmetic_pack { + "j2k classic resident Tier-1 arithmetic-pack profile" + } else if profile_classic_tier1_symbol_plan { + "j2k classic resident Tier-1 symbol plan" + } else if profile_classic_tier1_token_emit { + "j2k classic resident Tier-1 token emit" + } else if profile_classic_tier1_split_token_emit { + "j2k classic resident Tier-1 split token emit" + } else { + "j2k classic resident packetization" + }; + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::ClassicTier1Density, + next_label, + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + readback + } else { + None + }; + let classic_tier1_raw_pack_buffer = if tier1_job_count > 0 { + let buffer = dispatch_classic_tier1_raw_pack_profile( + runtime, + &command_buffer, + &coefficient_buffer, + &tier1_job_buffer, + &tier1_jobs, + tier1_output_capacity_total, + )?; + if buffer.is_some() && split_command_buffers { + let next_label = if profile_classic_tier1_arithmetic_pack { + "j2k classic resident Tier-1 arithmetic-pack profile" + } else if profile_classic_tier1_symbol_plan { + "j2k classic resident Tier-1 symbol plan" + } else if profile_classic_tier1_token_emit { + "j2k classic resident Tier-1 token emit" + } else if profile_classic_tier1_split_token_emit { + "j2k classic resident Tier-1 split token emit" + } else { + "j2k classic resident packetization" + }; + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::ClassicTier1RawPack, + next_label, + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + buffer + } else { + None + }; + let classic_tier1_arithmetic_pack_buffer = if tier1_job_count > 0 { + let buffer = dispatch_classic_tier1_arithmetic_pack_profile( + runtime, + &command_buffer, + &coefficient_buffer, + &tier1_job_buffer, + &tier1_jobs, + tier1_output_capacity_total, + )?; + if buffer.is_some() && split_command_buffers { + let next_label = if profile_classic_tier1_symbol_plan { + "j2k classic resident Tier-1 symbol plan" + } else if profile_classic_tier1_token_emit { + "j2k classic resident Tier-1 token emit" + } else if profile_classic_tier1_split_token_emit { + "j2k classic resident Tier-1 split token emit" + } else { + "j2k classic resident packetization" + }; + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::ClassicTier1ArithmeticPack, + next_label, + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + buffer + } else { + None + }; + let classic_tier1_symbol_plan_readback = if tier1_job_count > 0 { + let readback = dispatch_classic_tier1_symbol_plan_profile( + runtime, + &command_buffer, + &coefficient_buffer, + &tier1_job_buffer, + &tier1_jobs, + )?; + if readback.is_some() && split_command_buffers { + let next_label = if profile_classic_tier1_pass_plan { + "j2k classic resident Tier-1 pass plan" + } else if profile_classic_tier1_token_emit { + "j2k classic resident Tier-1 token emit" + } else if profile_classic_tier1_split_token_emit { + "j2k classic resident Tier-1 split token emit" + } else { + "j2k classic resident packetization" + }; + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::ClassicTier1SymbolPlan, + next_label, + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + readback + } else { + None + }; + let classic_tier1_pass_plan_readback = if tier1_job_count > 0 { + let readback = dispatch_classic_tier1_pass_plan_profile( + runtime, + &command_buffer, + &coefficient_buffer, + &tier1_job_buffer, + &tier1_jobs, + )?; + if readback.is_some() && split_command_buffers { + let next_label = if profile_classic_tier1_token_emit { + "j2k classic resident Tier-1 token emit" + } else if profile_classic_tier1_split_token_emit { + "j2k classic resident Tier-1 split token emit" + } else { + "j2k classic resident packetization" + }; + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::ClassicTier1PassPlan, + next_label, + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + readback + } else { + None + }; + let classic_tier1_token_emit_readback = if classic_gpu_token_pack_readback.is_some() { + classic_gpu_token_pack_readback + } else if tier1_job_count > 0 { + let readback = dispatch_classic_tier1_token_emit_profile( + runtime, + &command_buffer, + &coefficient_buffer, + &tier1_job_buffer, + &tier1_jobs, + )?; + if readback.is_some() && split_command_buffers { + let next_label = if profile_classic_tier1_split_token_emit { + "j2k classic resident Tier-1 split token emit" } else { - let offset = u32::try_from(state_blocks.len() - local_state_block_offset) - .map_err(|_| Error::MetalKernel { - message: "J2K Metal batch state block offset exceeds u32".to_string(), - })?; - for subband in &packet_subbands[local_subband_offset + subband_start - ..local_subband_offset + subband_start + subband_count] - { - for _ in 0..subband.block_count { - state_blocks.push(J2kPacketStateBlock { - previously_included: 0, - l_block: 3, - }); - } - } - state_block_offsets - .insert(descriptor.state_index, (offset, packet_block_count)); - (offset, packet_block_count) + "j2k classic resident packetization" }; - if existing_count != packet_block_count { - return Err(Error::MetalKernel { - message: "J2K Metal batch descriptor state layout mismatch".to_string(), - }); - } - packet_descriptors.push(J2kPacketDescriptor { - packet_index: descriptor.packet_index, - state_index: descriptor.state_index, - layer: u32::from(descriptor.layer), - resolution: descriptor.resolution, - component: u32::from(descriptor.component), - precinct_lo: descriptor.precinct as u32, - precinct_hi: (descriptor.precinct >> 32) as u32, - state_block_offset, - }); + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::ClassicTier1TokenEmit, + next_label, + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); } + readback + } else { + None + }; + let classic_tier1_split_token_emit_readback = if tier1_job_count > 0 { + let readback = dispatch_classic_tier1_split_token_emit_profile( + runtime, + &command_buffer, + &coefficient_buffer, + &tier1_job_buffer, + &tier1_jobs, + )?; + if readback.is_some() && split_command_buffers { + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::ClassicTier1SplitTokenEmit, + "j2k classic resident packetization", + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + readback + } else { + None + }; - let header_capacity = local_resident_block_count - .checked_mul(256) - .and_then(|bytes| bytes.checked_add(4096)) - .map(|bytes| bytes.max(4096)) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal batch packet header capacity overflow".to_string(), - })?; - let packet_output_capacity = tile_tier1_output_capacities[tile_index] - .checked_add( - header_capacity.saturating_mul( - tile.packet_descriptors - .len() - .max(tile.resolutions.len()) - .max(1), - ), - ) - .and_then(|bytes| bytes.checked_add(1024)) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal batch packet output capacity overflow".to_string(), - })?; - let codestream_capacity = - lossless_codestream_assembly_capacity(packet_output_capacity, tile.codestream)?; - let scratch_words = - max_tree_nodes - .checked_mul(6) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal batch scratch size overflow".to_string(), - })?; - - let packet_output_offset = packet_output_capacity_total; - let header_offset = header_capacity_total; - let scratch_offset = scratch_words_total; - let codestream_offset = codestream_capacity_total; - packet_jobs.push(J2kBatchedPacketEncodeJob { - resolution_offset: u32::try_from(local_resolution_offset).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch resolution offset exceeds u32".to_string(), - } - })?, - subband_offset: u32::try_from(local_subband_offset).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch subband offset exceeds u32".to_string(), - } - })?, - block_offset: u32::try_from(local_block_offset).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch block offset exceeds u32".to_string(), - } - })?, - descriptor_offset: u32::try_from(local_descriptor_offset).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch descriptor offset exceeds u32".to_string(), - } - })?, - state_block_offset: u32::try_from(local_state_block_offset).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch state block offset exceeds u32".to_string(), - } - })?, - output_offset: u32::try_from(packet_output_offset).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch packet output offset exceeds u32".to_string(), - } - })?, - header_offset: u32::try_from(header_offset).map_err(|_| Error::MetalKernel { - message: "J2K Metal batch header offset exceeds u32".to_string(), - })?, - scratch_offset: u32::try_from(scratch_offset).map_err(|_| Error::MetalKernel { - message: "J2K Metal batch scratch offset exceeds u32".to_string(), - })?, - resolution_count: tile.resolution_count, - num_layers: u32::from(tile.num_layers), - num_components: u32::from(tile.num_components), - code_block_count: tile.code_block_count, - subband_count: u32::try_from(local_subband_count).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch local subband count exceeds u32".to_string(), - } - })?, - descriptor_count: u32::try_from(tile.packet_descriptors.len()).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch descriptor count exceeds u32".to_string(), - } - })?, - output_capacity: u32::try_from(packet_output_capacity).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch packet output capacity exceeds u32".to_string(), - } - })?, - header_capacity: u32::try_from(header_capacity).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch header capacity exceeds u32".to_string(), - } - })?, - scratch_node_capacity: u32::try_from(max_tree_nodes).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch scratch node capacity exceeds u32".to_string(), - } - })?, - }); - assembly_jobs.push(J2kBatchedCodestreamAssemblyJob { - tile_data_offset: u32::try_from(packet_output_offset).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch assembly packet offset exceeds u32".to_string(), - } - })?, - codestream_offset: u32::try_from(codestream_offset).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch codestream offset exceeds u32".to_string(), - } - })?, - width: tile.codestream.width, - height: tile.codestream.height, - num_components: u32::from(tile.codestream.num_components), - bit_depth: u32::from(tile.codestream.bit_depth), - signed_samples: u32::from(tile.codestream.signed), - num_decomposition_levels: u32::from(tile.codestream.num_decomposition_levels), - use_mct: u32::from(tile.codestream.use_mct), - guard_bits: u32::from(tile.codestream.guard_bits), - progression_order: codestream_progression_order_code( - tile.codestream.progression_order, - ), - write_tlm: u32::from(tile.codestream.write_tlm), + let classic_packet_plan_started = profile_stages.then(Instant::now); + let classic_packet_plan_signpost = + hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_PLAN); + let ResidentBatchPacketPlan { + packet_resolutions, + packet_subbands, + resident_blocks, + packet_descriptors, + state_blocks, + packet_jobs, + assembly_jobs, + packet_output_capacity_total, + packet_payload_copy_job_capacity_total, + max_payload_copy_jobs_per_tile, + header_capacity_total, + scratch_words_total, + codestream_capacity_total, + codestream_offsets, + codestream_capacities, + } = build_resident_batch_packet_plan( + &prepared_tiles, + &tile_tier1_job_bases, + ResidentBatchPacketPlanParams { + family_name: "J2K", + block_coding_mode: 0, high_throughput: 0, - output_capacity: u32::try_from(codestream_capacity).map_err(|_| { - Error::MetalKernel { - message: "J2K Metal batch codestream capacity exceeds u32".to_string(), - } - })?, - }); - codestream_offsets.push(codestream_offset); - codestream_capacities.push(codestream_capacity); - packet_output_capacity_total = packet_output_capacity_total - .checked_add(packet_output_capacity) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal batch packet output total overflow".to_string(), - })?; - header_capacity_total = header_capacity_total - .checked_add(header_capacity) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal batch header total overflow".to_string(), - })?; - scratch_words_total = - scratch_words_total - .checked_add(scratch_words) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal batch scratch total overflow".to_string(), - })?; - codestream_capacity_total = codestream_capacity_total - .checked_add(codestream_capacity) - .ok_or_else(|| Error::MetalKernel { - message: "J2K Metal batch codestream total overflow".to_string(), - })?; + code_block_style: classic_cod_block_style_from_flags(classic_resident_style_flags), + }, + |tile_index, tile, header_capacity| { + classic_packet_output_capacity( + tile_tier1_output_capacities[tile_index], + header_capacity, + tile.packet_descriptors.len().max(tile.resolutions.len()), + tile.codestream, + ) + }, + )?; + drop(classic_packet_plan_signpost); + if let Some(started) = classic_packet_plan_started { + classic_packet_plan_duration = started.elapsed(); } + let classic_packet_buffer_setup_started = profile_stages.then(Instant::now); + let classic_packet_buffer_setup_signpost = + hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_BUFFER_SETUP); let packet_resolution_buffer = copied_slice_buffer(&runtime.device, &packet_resolutions); let packet_subband_buffer = copied_slice_buffer(&runtime.device, &packet_subbands); let resident_block_buffer = copied_slice_buffer(&runtime.device, &resident_blocks); - let packet_block_buffer = runtime.device.new_buffer( - (resident_blocks.len().max(1) * size_of::()) as u64, - MTLResourceOptions::StorageModePrivate, - ); let packet_descriptor_buffer = copied_slice_buffer(&runtime.device, &packet_descriptors); let state_block_buffer = copied_slice_buffer(&runtime.device, &state_blocks); - let packet_output_buffer = runtime.device.new_buffer( - packet_output_capacity_total.max(1) as u64, - MTLResourceOptions::StorageModePrivate, - ); - let header_buffer = runtime.device.new_buffer( - header_capacity_total.max(1) as u64, - MTLResourceOptions::StorageModePrivate, - ); - let scratch_buffer = runtime.device.new_buffer( - (scratch_words_total.max(1) * size_of::()) as u64, - MTLResourceOptions::StorageModePrivate, - ); + let packet_payload_copy_job_buffer = take_recyclable_private_buffer( + runtime, + packet_payload_copy_job_capacity_total + .max(1) + .checked_mul(size_of::()) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal batch packet payload-copy buffer size overflow".to_string(), + })?, + &mut recyclable_private_buffers, + )?; + let header_buffer = take_recyclable_private_buffer( + runtime, + header_capacity_total.max(1), + &mut recyclable_private_buffers, + )?; + let scratch_buffer = take_recyclable_private_buffer( + runtime, + scratch_words_total.max(1) * size_of::(), + &mut recyclable_private_buffers, + )?; let packet_job_buffer = copied_slice_buffer(&runtime.device, &packet_jobs); - let packet_status_buffer = runtime.device.new_buffer( - (packet_jobs.len() * size_of::()) as u64, - MTLResourceOptions::StorageModePrivate, - ); + let packet_status_buffer = zeroed_recyclable_shared_buffer( + runtime, + packet_jobs.len().max(1) * size_of::(), + &mut recyclable_shared_buffers, + )?; let codestream_job_buffer = copied_slice_buffer(&runtime.device, &assembly_jobs); let codestream_buffer = runtime.device.new_buffer( codestream_capacity_total.max(1) as u64, @@ -13992,6 +16777,10 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( (assembly_jobs.len() * size_of::()) as u64, MTLResourceOptions::StorageModeShared, ); + drop(classic_packet_buffer_setup_signpost); + if let Some(started) = classic_packet_buffer_setup_started { + classic_packet_buffer_setup_duration = started.elapsed(); + } let resident_block_params = J2kResidentPacketBlockParams { block_count: u32::try_from(resident_blocks.len()).map_err(|_| Error::MetalKernel { @@ -13999,52 +16788,36 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( })?, tier1_job_count, }; - if !resident_blocks.is_empty() { - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.packet_block_prepare_resident_classic); - encoder.set_buffer(0, Some(&resident_block_buffer), 0); - encoder.set_buffer(1, Some(&tier1_job_buffer), 0); - encoder.set_buffer(2, Some(&tier1_status_buffer), 0); - encoder.set_buffer(3, Some(&packet_block_buffer), 0); - encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const resident_block_params).cast(), - ); - encoder.dispatch_threads( - MTLSize { - width: resident_blocks.len() as u64, - height: 1, - depth: 1, - }, - MTLSize { - width: runtime - .packet_block_prepare_resident_classic - .thread_execution_width() - .max(1), - height: 1, - depth: 1, - }, - ); - encoder.end_encoding(); - } let tile_count = u64::try_from(packet_jobs.len()).map_err(|_| Error::MetalKernel { message: "J2K Metal batch tile count exceeds u64".to_string(), })?; + let command_encode_started = profile_stages.then(Instant::now); + let signpost = + hybrid_stage_signpost(SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKETIZATION_COMMAND_ENCODE); let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.packet_encode_batched); + label_compute_encoder(encoder, "J2K packetization"); + encoder.set_compute_pipeline_state(&runtime.packet_encode_resident_classic_batched); encoder.set_buffer(0, Some(&packet_resolution_buffer), 0); encoder.set_buffer(1, Some(&packet_subband_buffer), 0); - encoder.set_buffer(2, Some(&packet_block_buffer), 0); + encoder.set_buffer(2, Some(&resident_block_buffer), 0); encoder.set_buffer(3, Some(&tier1_output_buffer), 0); - encoder.set_buffer(4, Some(&packet_output_buffer), 0); + encoder.set_buffer(4, Some(&codestream_buffer), 0); encoder.set_buffer(5, Some(&header_buffer), 0); encoder.set_buffer(6, Some(&scratch_buffer), 0); encoder.set_buffer(7, Some(&packet_job_buffer), 0); encoder.set_buffer(8, Some(&packet_status_buffer), 0); encoder.set_buffer(9, Some(&packet_descriptor_buffer), 0); encoder.set_buffer(10, Some(&state_block_buffer), 0); + encoder.set_buffer(11, Some(&packet_payload_copy_job_buffer), 0); + encoder.set_buffer(12, Some(&tier1_job_buffer), 0); + encoder.set_buffer(13, Some(&tier1_status_buffer), 0); + encoder.set_buffer(14, Some(&tier1_segment_buffer), 0); + encoder.set_bytes( + 15, + size_of::() as u64, + (&raw const resident_block_params).cast(), + ); encoder.dispatch_threads( MTLSize { width: tile_count, @@ -14053,7 +16826,7 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( }, MTLSize { width: runtime - .packet_encode_batched + .packet_encode_resident_classic_batched .thread_execution_width() .max(1), height: 1, @@ -14061,10 +16834,69 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( }, ); encoder.end_encoding(); + drop(signpost); + if let Some(started) = command_encode_started { + packetization_duration = packetization_duration.saturating_add(started.elapsed()); + } + if split_command_buffers { + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::Packetization, + "j2k classic resident packet payload copy", + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + let packet_payload_copy_dispatched = dispatch_batched_packet_payload_copy( + runtime, + &command_buffer, + J2kBatchedPacketPayloadCopyDispatch { + payload_buffer: &tier1_output_buffer, + packet_output_buffer: &codestream_buffer, + packet_job_buffer: &packet_job_buffer, + packet_status_buffer: &packet_status_buffer, + packet_payload_copy_job_buffer: &packet_payload_copy_job_buffer, + tile_count, + max_payload_copy_jobs_per_tile: max_payload_copy_jobs_per_tile as u64, + label: "J2K packetization payload copy", + signpost_name: SIGNPOST_ENCODE_HYBRID_CLASSIC_PAYLOAD_COPY_COMMAND_ENCODE, + }, + ); + if split_command_buffers { + if packet_payload_copy_dispatched { + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::PacketPayloadCopy, + "j2k classic resident codestream assembly", + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } else { + label_command_buffer(&command_buffer, "j2k classic resident codestream assembly"); + } + } + let max_packet_output_capacity = packet_jobs + .iter() + .map(|job| job.output_capacity) + .max() + .unwrap_or(0); + let max_packet_output_capacity_usize = usize::try_from(max_packet_output_capacity) + .map_err(|_| Error::MetalKernel { + message: "J2K Metal batch max packet output capacity exceeds usize".to_string(), + })?; + let command_encode_started = profile_stages.then(Instant::now); + let signpost = hybrid_stage_signpost( + SIGNPOST_ENCODE_HYBRID_CLASSIC_CODESTREAM_ASSEMBLY_COMMAND_ENCODE, + ); let encoder = command_buffer.new_compute_command_encoder(); + label_compute_encoder(encoder, "J2K codestream assembly"); encoder.set_compute_pipeline_state(&runtime.lossless_codestream_assemble_batched); - encoder.set_buffer(0, Some(&packet_output_buffer), 0); + encoder.set_buffer(0, Some(&codestream_buffer), 0); encoder.set_buffer(1, Some(&packet_status_buffer), 0); encoder.set_buffer(2, Some(&codestream_buffer), 0); encoder.set_buffer(3, Some(&codestream_job_buffer), 0); @@ -14085,10 +16917,134 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( }, ); encoder.end_encoding(); + drop(signpost); + if split_command_buffers { + command_buffer = finish_resident_encode_split_command_buffer_timed( + command_buffer, + runtime, + J2kResidentEncodeGpuStage::CodestreamAssembly, + "j2k classic resident result readback", + &mut gpu_stage_command_buffers, + profile_stages, + &mut classic_command_buffer_commit_duration, + ); + } + let codestream_payload_copy_dispatched = false; + if let Some(started) = command_encode_started { + codestream_assembly_duration = + codestream_assembly_duration.saturating_add(started.elapsed()); + } + let tier1_status_readback = schedule_resident_tier1_status_readback( + runtime, + &command_buffer, + &tier1_status_buffer, + J2kResidentTier1StatusKind::Classic, + classic_resident_style_flags, + Some(&tier1_jobs), + tier1_jobs.len(), + size_of::(), + profile_stages, + )?; + let final_commit_started = profile_stages.then(Instant::now); command_buffer.commit(); + if let Some(started) = final_commit_started { + classic_command_buffer_commit_duration = + classic_command_buffer_commit_duration.saturating_add(started.elapsed()); + } + if split_command_buffers && codestream_payload_copy_dispatched { + gpu_stage_command_buffers.push(J2kResidentEncodeGpuStageCommandBuffer { + stage: J2kResidentEncodeGpuStage::CodestreamPayloadCopy, + command_buffer: command_buffer.clone(), + }); + } - let mut recyclable_private_buffers = Vec::<(usize, Buffer)>::new(); + if profile_stages { + let mut prepare_command_buffer_ptrs = Vec::new(); + for tile in &prepared_tiles { + let mut pushed_split_prepare = false; + for (stage, command_buffer) in [ + ( + J2kResidentEncodeGpuStage::CoefficientDeinterleaveRct, + tile.prepare_deinterleave_rct_command_buffer.as_ref(), + ), + ( + J2kResidentEncodeGpuStage::CoefficientDwt53, + tile.prepare_dwt53_command_buffer.as_ref(), + ), + ( + J2kResidentEncodeGpuStage::CoefficientExtract, + tile.prepare_coefficient_extract_command_buffer.as_ref(), + ), + ] { + if let Some(command_buffer) = command_buffer { + let ptr = command_buffer.as_ptr(); + if prepare_command_buffer_ptrs.contains(&ptr) { + continue; + } + prepare_command_buffer_ptrs.push(ptr); + pushed_split_prepare = true; + gpu_stage_command_buffers.push(J2kResidentEncodeGpuStageCommandBuffer { + stage, + command_buffer: command_buffer.clone(), + }); + } + } + for command_buffer in &tile.prepare_dwt53_vertical_command_buffers { + let ptr = command_buffer.as_ptr(); + if prepare_command_buffer_ptrs.contains(&ptr) { + continue; + } + prepare_command_buffer_ptrs.push(ptr); + pushed_split_prepare = true; + gpu_stage_command_buffers.push(J2kResidentEncodeGpuStageCommandBuffer { + stage: J2kResidentEncodeGpuStage::CoefficientDwt53Vertical, + command_buffer: command_buffer.clone(), + }); + } + for command_buffer in &tile.prepare_dwt53_horizontal_command_buffers { + let ptr = command_buffer.as_ptr(); + if prepare_command_buffer_ptrs.contains(&ptr) { + continue; + } + prepare_command_buffer_ptrs.push(ptr); + pushed_split_prepare = true; + gpu_stage_command_buffers.push(J2kResidentEncodeGpuStageCommandBuffer { + stage: J2kResidentEncodeGpuStage::CoefficientDwt53Horizontal, + command_buffer: command_buffer.clone(), + }); + } + if pushed_split_prepare { + continue; + } + let ptr = tile.prepare_command_buffer.as_ptr(); + if prepare_command_buffer_ptrs.contains(&ptr) { + continue; + } + prepare_command_buffer_ptrs.push(ptr); + gpu_stage_command_buffers.push(J2kResidentEncodeGpuStageCommandBuffer { + stage: J2kResidentEncodeGpuStage::CoefficientPrep, + command_buffer: tile.prepare_command_buffer.clone(), + }); + } + } + + retained_command_buffers.extend( + gpu_stage_command_buffers + .iter() + .map(|stage_command_buffer| stage_command_buffer.command_buffer.clone()), + ); for tile in prepared_tiles { + if let Some(command_buffer) = tile.prepare_deinterleave_rct_command_buffer { + retained_command_buffers.push(command_buffer); + } + if let Some(command_buffer) = tile.prepare_dwt53_command_buffer { + retained_command_buffers.push(command_buffer); + } + retained_command_buffers.extend(tile.prepare_dwt53_vertical_command_buffers); + retained_command_buffers.extend(tile.prepare_dwt53_horizontal_command_buffers); + if let Some(command_buffer) = tile.prepare_coefficient_extract_command_buffer { + retained_command_buffers.push(command_buffer); + } retained_command_buffers.push(tile.prepare_command_buffer); retained_buffers.push(tile.coefficient_buffer); retained_buffers.push(tile.deinterleave_status_buffer); @@ -14102,29 +17058,75 @@ pub(crate) fn submit_lossless_codestream_buffers_from_prepared_classic_batch( retained_buffers.push(tier1_output_buffer); retained_buffers.push(tier1_status_buffer); retained_buffers.push(tier1_segment_buffer); + if let Some(buffer) = classic_tier1_raw_pack_buffer { + retained_buffers.push(buffer); + } + if let Some(buffer) = classic_tier1_arithmetic_pack_buffer { + retained_buffers.push(buffer); + } retained_buffers.push(packet_resolution_buffer); retained_buffers.push(packet_subband_buffer); retained_buffers.push(resident_block_buffer); - retained_buffers.push(packet_block_buffer); retained_buffers.push(packet_descriptor_buffer); retained_buffers.push(state_block_buffer); - retained_buffers.push(packet_output_buffer); + retained_buffers.push(packet_payload_copy_job_buffer); retained_buffers.push(header_buffer); retained_buffers.push(scratch_buffer); retained_buffers.push(packet_job_buffer); - retained_buffers.push(packet_status_buffer); + retained_buffers.push(packet_status_buffer.clone()); retained_buffers.push(codestream_job_buffer); + stage_stats.classic_tier1_setup_duration = classic_tier1_setup_duration; + stage_stats.classic_block_encode_duration = classic_block_encode_duration; + stage_stats.classic_packet_plan_duration = classic_packet_plan_duration; + stage_stats.classic_packet_buffer_setup_duration = classic_packet_buffer_setup_duration; + stage_stats.classic_command_buffer_commit_duration = classic_command_buffer_commit_duration; + stage_stats.packet_block_prep_duration = packet_block_prep_duration; + stage_stats.packetization_duration = packetization_duration; + stage_stats.codestream_assembly_duration = codestream_assembly_duration; + stage_stats.packet_payload_copy_job_capacity_total = packet_payload_copy_job_capacity_total; + stage_stats.max_packet_payload_copy_jobs_per_tile = max_payload_copy_jobs_per_tile; + stage_stats.packet_payload_copy_launched_stripe_count_total = packet_jobs + .len() + .saturating_mul(max_payload_copy_jobs_per_tile) + .saturating_mul(PACKET_PAYLOAD_COPY_STRIPES_PER_JOB as usize); + stage_stats.tier1_output_capacity_total = tier1_output_capacity_total; + stage_stats.max_tier1_output_capacity = max_tier1_output_capacity; + stage_stats.tier1_segment_capacity_total = tier1_segment_capacity_total; + stage_stats.max_tier1_segment_capacity_per_block = tier1_jobs + .iter() + .map(|job| job.segment_capacity as usize) + .max() + .unwrap_or(0); + stage_stats.packet_output_capacity_total = packet_output_capacity_total; + stage_stats.max_packet_output_capacity = max_packet_output_capacity_usize; + stage_stats.codestream_payload_copy_launched_thread_count_total = 0; + stage_stats.code_block_count = tier1_jobs.len(); + Ok(J2kPendingResidentLosslessCodestreamBatch { - device: runtime.device.clone(), + runtime: session.runtime()?, buffer: codestream_buffer, byte_offsets: codestream_offsets, capacities: codestream_capacities, status_buffer: codestream_status_buffer, - command_buffer: command_buffer.to_owned(), + packet_status_buffer, + tier1_status_readback, + classic_tier1_density_readback, + classic_tier1_symbol_plan_readback, + classic_tier1_pass_plan_readback, + classic_tier1_token_emit_readback, + classic_tier1_split_token_emit_readback, + classic_gpu_token_pack_used: use_classic_gpu_token_pack + || use_classic_split_gpu_token_pack + || use_classic_split_mq_byte_gpu_token_pack, + command_buffer: command_buffer.clone(), retained_command_buffers, _retained_buffers: retained_buffers, recyclable_private_buffers, + recyclable_shared_buffers, + gpu_stage_command_buffers, + stage_stats, + codestream_payload_copy_dispatched, status_stage: "J2K batched codestream assembly", length_error: "J2K Metal batched codestream output length exceeds usize", capacity_error: "J2K Metal batched codestream output length exceeds buffer", @@ -14161,32 +17163,22 @@ fn dispatch_ht_cleanup( encoder.set_compute_pipeline_state(&runtime.ht_cleanup); encoder.set_buffer(0, Some(&input), 0); encoder.set_buffer(1, Some(decoded), 0); - encoder.set_bytes( - 2, - size_of::() as u64, - (&raw const params).cast(), - ); - encoder.set_buffer(3, Some(&runtime.ht_vlc_table0), 0); - encoder.set_buffer(4, Some(&runtime.ht_vlc_table1), 0); - encoder.set_buffer(5, Some(&runtime.ht_uvlc_table0), 0); - encoder.set_buffer(6, Some(&runtime.ht_uvlc_table1), 0); - encoder.set_buffer(7, Some(&status_buffer), 0); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, + encoder.set_bytes( + 2, + size_of::() as u64, + (&raw const params).cast(), ); + encoder.set_buffer(3, Some(&runtime.ht_vlc_table0), 0); + encoder.set_buffer(4, Some(&runtime.ht_vlc_table1), 0); + encoder.set_buffer(5, Some(&runtime.ht_uvlc_table0), 0); + encoder.set_buffer(6, Some(&runtime.ht_uvlc_table1), 0); + encoder.set_buffer(7, Some(&status_buffer), 0); + dispatch_single_thread(encoder); encoder.end_encoding(); command_buffer.commit(); command_buffer.wait_until_completed(); + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. let status = unsafe { status_buffer.contents().cast::().read() }; if status.code != J2K_HT_STATUS_OK { return Err(decode_ht_status_error(status)); @@ -14247,6 +17239,7 @@ fn dispatch_ht_cleanup_batched( command_buffer.commit(); command_buffer.wait_until_completed(); + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. let statuses = unsafe { core::slice::from_raw_parts(status_buffer.contents().cast::(), jobs.len()) }; @@ -14480,12 +17473,13 @@ pub(crate) fn decode_classic_cleanup_code_block( output_offset: 0, missing_msbs: u32::from(job.missing_bit_planes), total_bitplanes: u32::from(job.total_bitplanes), + roi_shift: u32::from(job.roi_shift), number_of_coding_passes: u32::from(job.number_of_coding_passes), sub_band_type: match job.sub_band_type { - signinum_j2k_native::J2kSubBandType::LowLow => 0, - signinum_j2k_native::J2kSubBandType::HighLow => 1, - signinum_j2k_native::J2kSubBandType::LowHigh => 2, - signinum_j2k_native::J2kSubBandType::HighHigh => 3, + j2k_native::J2kSubBandType::LowLow => 0, + j2k_native::J2kSubBandType::HighLow => 1, + j2k_native::J2kSubBandType::LowHigh => 2, + j2k_native::J2kSubBandType::HighHigh => 3, }, style_flags: classic_style_flags(job.style), strict: u32::from(job.strict), @@ -14599,12 +17593,13 @@ pub(crate) fn decode_classic_cleanup_sub_band( })?, missing_msbs: u32::from(block.code_block.missing_bit_planes), total_bitplanes: u32::from(block.code_block.total_bitplanes), + roi_shift: u32::from(block.code_block.roi_shift), number_of_coding_passes: u32::from(block.code_block.number_of_coding_passes), sub_band_type: match block.code_block.sub_band_type { - signinum_j2k_native::J2kSubBandType::LowLow => 0, - signinum_j2k_native::J2kSubBandType::HighLow => 1, - signinum_j2k_native::J2kSubBandType::LowHigh => 2, - signinum_j2k_native::J2kSubBandType::HighHigh => 3, + j2k_native::J2kSubBandType::LowLow => 0, + j2k_native::J2kSubBandType::HighLow => 1, + j2k_native::J2kSubBandType::LowHigh => 2, + j2k_native::J2kSubBandType::HighHigh => 3, }, style_flags: classic_style_flags(block.code_block.style), strict: u32::from(block.code_block.strict), @@ -14704,6 +17699,7 @@ pub(crate) fn decode_ht_cleanup_sub_band( refinement_length: block.code_block.refinement_length, missing_msbs: u32::from(block.code_block.missing_bit_planes), num_bitplanes: u32::from(block.code_block.num_bitplanes), + roi_shift: u32::from(block.code_block.roi_shift), number_of_coding_passes: u32::from(block.code_block.number_of_coding_passes), output_stride: job.width, output_offset: block @@ -14741,850 +17737,14 @@ pub(crate) fn decode_ht_cleanup_sub_band( }) } -#[cfg(all(test, target_os = "macos"))] -mod tests { - use super::{ - classic_batch_uses_plain_fast_path, classic_repeated_uses_plain_fast_path, - crop_prepared_direct_grayscale_plan_to_output_region, decode_scaled_to_surface, - j2k_pack_kernel_name_for, j2k_pack_scale_arrays, output_shape_for, - prepare_direct_grayscale_plan, prepared_direct_grayscale_plan_compute_encoder_count, - prepared_idwt_output_len, prepared_repeated_direct_ht_cleanup_dispatch_count, - repeated_gray_store_is_contiguous_full_surface, runtime_initialization_error, - supports_stacked_direct_component_plane_batch, with_runtime_for_device, - J2kClassicCleanupBatchJob, J2kClassicSegment, J2kRepeatedGrayStoreParams, MetalRuntime, - PreparedDirectGrayscaleStep, - }; - use metal::Device; - use signinum_core::PixelFormat; - use signinum_j2k_native::{ - encode, encode_htj2k, ColorSpace as NativeColorSpace, DecodeSettings, DecoderContext, - EncodeOptions, Image, - }; - - #[test] - fn rgb16_with_alpha_is_rejected() { - let runtime = MetalRuntime::new().expect("Metal runtime"); - let result = output_shape_for( - &NativeColorSpace::RGB, - true, - 4, - PixelFormat::Rgb16, - &runtime, - ); - assert!(result.is_err(), "RGBA input must not silently map to Rgb16"); - } - - #[test] - fn runtime_initialization_error_classifies_null_queue_as_unavailable() { - assert!(matches!( - runtime_initialization_error("Metal command queue is unavailable on this host"), - crate::Error::MetalUnavailable - )); - } - - #[test] - fn with_runtime_for_device_reuses_cached_runtime_for_device() { - let Some(device) = Device::system_default() else { - return; - }; - - let first = with_runtime_for_device(&device, |runtime| Ok(std::ptr::from_ref(runtime))) - .expect("first Metal runtime"); - let second = with_runtime_for_device(&device, |runtime| Ok(std::ptr::from_ref(runtime))) - .expect("second Metal runtime"); - - assert_eq!(first, second); - } - - #[test] - fn j2k_pack_selects_specialized_kernels_for_wsi_formats() { - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::Gray, false, 1, PixelFormat::Gray8), - Some("j2k_pack_gray8") - ); - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::RGB, false, 3, PixelFormat::Rgb8), - Some("j2k_pack_rgb8") - ); - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::RGB, false, 3, PixelFormat::Rgba8), - Some("j2k_pack_rgb_opaque_rgba8") - ); - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::RGB, true, 4, PixelFormat::Rgba8), - Some("j2k_pack_rgba8") - ); - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::Gray, false, 1, PixelFormat::Gray16), - Some("j2k_pack_gray16") - ); - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::RGB, false, 3, PixelFormat::Rgb16), - Some("j2k_pack_rgb16") - ); - assert_eq!( - j2k_pack_kernel_name_for(&NativeColorSpace::RGB, true, 4, PixelFormat::Rgb16), - None, - "RGBA input must not silently drop alpha when packing RGB16" - ); - } - - #[test] - fn j2k_pack_precomputes_scale_factors_on_cpu() { - let (max_values, u8_scales, u16_scales) = j2k_pack_scale_arrays([8, 12, 16, 0]); - - assert_f32_near(max_values[0], 255.0); - assert_f32_near(max_values[1], 4095.0); - assert_f32_near(max_values[2], 65_535.0); - assert_f32_near(max_values[3], 1.0); - assert_f32_near(u8_scales[0], 1.0); - assert_f32_near(u8_scales[1], 255.0 / 4095.0); - assert_f32_near(u16_scales[0], 257.0); - assert_f32_near(u16_scales[1], 1.0); - assert_f32_near(u16_scales[2], 1.0); - assert_f32_near(u16_scales[3], 65_535.0); - } - - fn assert_f32_near(actual: f32, expected: f32) { - assert!( - (actual - expected).abs() <= f32::EPSILON, - "expected {actual} to be within f32 epsilon of {expected}" - ); - } - - #[test] - fn scaled_htj2k_decode_runs_through_metal_compute_path() { - let pixels: Vec = (0..16).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 4, 4, 1, 8, false, &options).expect("encode ht gray8"); - - let image = Image::new( - &bytes, - &DecodeSettings { - target_resolution: Some((2, 2)), - ..DecodeSettings::default() - }, - ) - .expect("image"); - let host = image.decode().expect("host scaled decode"); - - let surface = decode_scaled_to_surface( - &bytes, - (4, 4), - PixelFormat::Gray8, - signinum_core::Downscale::Half, - ) - .expect("metal scaled decode"); - assert_eq!(surface.as_bytes(), host.as_slice()); - } - - #[test] - fn prepared_ht_direct_plan_groups_cleanup_subbands_before_idwt() { - let pixels: Vec = (0..64).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let ht_subband_steps = plan - .steps - .iter() - .filter(|step| { - matches!( - step, - signinum_j2k_native::J2kDirectGrayscaleStep::HtSubBand(_) - ) - }) - .count(); - assert!( - ht_subband_steps > 1, - "fixture must exercise multiple HT sub-band cleanup steps" - ); - - let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - assert_eq!( - prepared.ht_groups.len(), - 1, - "single-tile HTJ2K direct decode should group adjacent HT sub-bands into one cleanup dispatch" - ); - assert_eq!(prepared.ht_groups[0].members.len(), ht_subband_steps); - assert!(matches!( - prepared.steps[prepared.ht_groups[0].start_step], - PreparedDirectGrayscaleStep::HtSubBand(_) - )); - } - - #[test] - fn prepared_ht_direct_plan_encodes_full_decode_in_one_compute_encoder() { - let pixels: Vec = (0..64).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - - assert_eq!( - prepared_direct_grayscale_plan_compute_encoder_count(&prepared, PixelFormat::Gray8), - 1, - "prepared single-tile direct decode should keep cleanup, IDWT, and grayscale store in one compute encoder" - ); - } - - #[test] - fn repeated_prepared_ht_direct_plan_groups_cleanup_subbands_before_idwt() { - let pixels: Vec = (0..64).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let ht_subband_steps = plan - .steps - .iter() - .filter(|step| { - matches!( - step, - signinum_j2k_native::J2kDirectGrayscaleStep::HtSubBand(_) - ) - }) - .count(); - assert!( - ht_subband_steps > 1, - "fixture must exercise multiple HT sub-band cleanup steps" - ); - - let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - assert_eq!( - prepared_repeated_direct_ht_cleanup_dispatch_count(&prepared), - 1, - "repeated HTJ2K WSI tile batches should group adjacent sub-band cleanups like the single-tile path" - ); - } - - #[test] - fn distinct_prepared_ht_direct_plans_support_stacked_component_batch() { - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes_a = encode_htj2k(&(0..64).collect::>(), 8, 8, 1, 8, false, &options) - .expect("encode first ht gray8"); - let bytes_b = encode_htj2k( - &(0..64).rev().collect::>(), - 8, - 8, - 1, - 8, - false, - &options, - ) - .expect("encode second ht gray8"); - let image_a = Image::new(&bytes_a, &DecodeSettings::default()).expect("first image"); - let image_b = Image::new(&bytes_b, &DecodeSettings::default()).expect("second image"); - let mut context_a = DecoderContext::default(); - let mut context_b = DecoderContext::default(); - let plan_a = image_a - .build_direct_grayscale_plan_with_context(&mut context_a) - .expect("first direct plan"); - let plan_b = image_b - .build_direct_grayscale_plan_with_context(&mut context_b) - .expect("second direct plan"); - let prepared_a = prepare_direct_grayscale_plan(&plan_a).expect("first prepared plan"); - let prepared_b = prepare_direct_grayscale_plan(&plan_b).expect("second prepared plan"); - - assert!( - supports_stacked_direct_component_plane_batch(&[&prepared_a, &prepared_b]), - "distinct same-shape HTJ2K grayscale plans should be eligible for one stacked batch graph" - ); - } - - #[test] - fn cropped_region_scaled_ht_direct_plan_prunes_codeblocks_outside_output_roi() { - let mut pixels = Vec::with_capacity(128 * 128); - for y in 0..128u32 { - for x in 0..128u32 { - pixels.push(((x * 3 + y * 5) & 0xff) as u8); - } - } - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 3, - code_block_width_exp: 0, - code_block_height_exp: 0, - ..EncodeOptions::default() - }; - let bytes = - encode_htj2k(&pixels, 128, 128, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new( - &bytes, - &DecodeSettings { - target_resolution: Some((32, 32)), - ..DecodeSettings::default() - }, - ) - .expect("scaled image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - let full_jobs = prepared_direct_grayscale_ht_job_count(&prepared); - assert!( - full_jobs > 8, - "fixture should have multiple HT code-block jobs" - ); - - crop_prepared_direct_grayscale_plan_to_output_region( - &mut prepared, - signinum_core::Rect { - x: 10, - y: 10, - w: 4, - h: 4, - }, - ) - .expect("crop direct plan"); - let cropped_jobs = prepared_direct_grayscale_ht_job_count(&prepared); - - assert!( - cropped_jobs > 0 && cropped_jobs < full_jobs, - "cropped ROI should prune HT code-block jobs; full={full_jobs}, cropped={cropped_jobs}" - ); - } - - #[test] - fn cropped_region_scaled_ht_direct_plan_compacts_coded_payloads() { - let mut pixels = Vec::with_capacity(128 * 128); - for y in 0..128u32 { - for x in 0..128u32 { - pixels.push(((x * 3 + y * 5) & 0xff) as u8); - } - } - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 3, - code_block_width_exp: 0, - code_block_height_exp: 0, - ..EncodeOptions::default() - }; - let bytes = - encode_htj2k(&pixels, 128, 128, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new( - &bytes, - &DecodeSettings { - target_resolution: Some((32, 32)), - ..DecodeSettings::default() - }, - ) - .expect("scaled image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - let full_bytes = prepared_direct_grayscale_ht_coded_byte_count(&prepared); - assert!(full_bytes > 0, "fixture should carry HT coded payloads"); - - crop_prepared_direct_grayscale_plan_to_output_region( - &mut prepared, - signinum_core::Rect { - x: 10, - y: 10, - w: 4, - h: 4, - }, - ) - .expect("crop direct plan"); - let cropped_bytes = prepared_direct_grayscale_ht_coded_byte_count(&prepared); - - assert!( - cropped_bytes > 0 && cropped_bytes < full_bytes, - "cropped ROI should compact HT coded bytes; full={full_bytes}, cropped={cropped_bytes}" - ); - } - - #[test] - fn cropped_region_scaled_ht_direct_plan_reduces_idwt_output_work() { - let mut pixels = Vec::with_capacity(128 * 128); - for y in 0..128u32 { - for x in 0..128u32 { - pixels.push(((x * 3 + y * 5) & 0xff) as u8); - } - } - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 3, - code_block_width_exp: 0, - code_block_height_exp: 0, - ..EncodeOptions::default() - }; - let bytes = - encode_htj2k(&pixels, 128, 128, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new( - &bytes, - &DecodeSettings { - target_resolution: Some((32, 32)), - ..DecodeSettings::default() - }, - ) - .expect("scaled image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - let full_samples = prepared_direct_grayscale_idwt_output_sample_count(&prepared); - - crop_prepared_direct_grayscale_plan_to_output_region( - &mut prepared, - signinum_core::Rect { - x: 10, - y: 10, - w: 4, - h: 4, - }, - ) - .expect("crop direct plan"); - let cropped_samples = prepared_direct_grayscale_idwt_output_sample_count(&prepared); - - assert!( - cropped_samples > 0 && cropped_samples < full_samples, - "cropped ROI should reduce IDWT output work; full={full_samples}, cropped={cropped_samples}" - ); - } - - #[test] - fn cropped_region_ht_direct_plan_crops_all_idwt_levels() { - let mut pixels = Vec::with_capacity(128 * 128); - for y in 0..128u32 { - for x in 0..128u32 { - pixels.push(((x * 3 + y * 5) & 0xff) as u8); - } - } - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 3, - code_block_width_exp: 0, - code_block_height_exp: 0, - ..EncodeOptions::default() - }; - let bytes = - encode_htj2k(&pixels, 128, 128, 1, 8, false, &options).expect("encode ht gray8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - let idwt_levels = prepared_direct_grayscale_idwt_full_and_prepared_lens(&prepared); - assert!( - idwt_levels.len() >= 3, - "fixture should exercise a multi-level IDWT plan" - ); - - crop_prepared_direct_grayscale_plan_to_output_region( - &mut prepared, - signinum_core::Rect { - x: 56, - y: 56, - w: 16, - h: 16, - }, - ) - .expect("crop direct plan"); - let cropped_idwt_levels = prepared_direct_grayscale_idwt_full_and_prepared_lens(&prepared); - - assert_eq!(cropped_idwt_levels.len(), idwt_levels.len()); - for (level_idx, (full_len, cropped_len)) in cropped_idwt_levels.iter().copied().enumerate() - { - assert!( - cropped_len > 0 && cropped_len < full_len, - "cropped ROI should reduce IDWT level {level_idx} output work; full={full_len}, cropped={cropped_len}" - ); - } - } - - fn prepared_direct_grayscale_ht_job_count(plan: &super::PreparedDirectGrayscalePlan) -> usize { - plan.steps - .iter() - .map(|step| match step { - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => sub_band.jobs.len(), - _ => 0, - }) - .sum() - } - - fn prepared_direct_grayscale_ht_coded_byte_count( - plan: &super::PreparedDirectGrayscalePlan, - ) -> usize { - plan.steps - .iter() - .map(|step| match step { - PreparedDirectGrayscaleStep::HtSubBand(sub_band) => sub_band.coded_data.len(), - _ => 0, - }) - .sum() - } - - fn prepared_direct_grayscale_idwt_output_sample_count( - plan: &super::PreparedDirectGrayscalePlan, - ) -> usize { - plan.steps - .iter() - .map(|step| match step { - PreparedDirectGrayscaleStep::Idwt(idwt) => prepared_idwt_output_len(idwt), - _ => 0, - }) - .sum() - } - - fn prepared_direct_grayscale_idwt_full_and_prepared_lens( - plan: &super::PreparedDirectGrayscalePlan, - ) -> Vec<(usize, usize)> { - plan.steps - .iter() - .filter_map(|step| match step { - PreparedDirectGrayscaleStep::Idwt(idwt) => Some(( - idwt.step.rect.width() as usize * idwt.step.rect.height() as usize, - prepared_idwt_output_len(idwt), - )), - _ => None, - }) - .collect() - } - - #[test] - fn prepared_classic_direct_plan_groups_cleanup_subbands_before_idwt() { - let pixels: Vec = (0..64).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode(&pixels, 8, 8, 1, 8, false, &options).expect("encode j2k gray8"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("direct grayscale plan"); - let classic_subband_steps = plan - .steps - .iter() - .filter(|step| { - matches!( - step, - signinum_j2k_native::J2kDirectGrayscaleStep::ClassicSubBand(_) - ) - }) - .count(); - assert!( - classic_subband_steps > 1, - "fixture must exercise multiple classic sub-band cleanup steps" - ); - - let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); - assert_eq!( - prepared.classic_groups.len(), - 1, - "classic J2K direct decode should group adjacent sub-band cleanups before IDWT" - ); - assert_eq!( - prepared.classic_groups[0].members.len(), - classic_subband_steps - ); - assert!(matches!( - prepared.steps[prepared.classic_groups[0].start_step], - PreparedDirectGrayscaleStep::ClassicSubBand(_) - )); - } - - #[test] - fn classic_plain_fast_path_accepts_style_zero_arithmetic_jobs() { - let jobs = [J2kClassicCleanupBatchJob { - coded_offset: 0, - coded_len: 1, - segment_offset: 0, - segment_count: 1, - width: 64, - height: 64, - output_stride: 64, - output_offset: 0, - missing_msbs: 0, - total_bitplanes: 8, - number_of_coding_passes: 1, - sub_band_type: 0, - style_flags: 0, - strict: 1, - dequantization_step: 1.0, - }]; - let segments = [J2kClassicSegment { - data_offset: 0, - data_length: 1, - start_coding_pass: 0, - end_coding_pass: 1, - use_arithmetic: 1, - }]; - - assert!( - classic_batch_uses_plain_fast_path(&jobs, &segments), - "style-0 arithmetic-only classic J2K jobs should use the fused plain cleanup/store kernel" - ); - } - - #[test] - fn classic_repeated_plain_fast_path_stays_off_for_wsi_batch_size() { - let jobs = [J2kClassicCleanupBatchJob { - coded_offset: 0, - coded_len: 1, - segment_offset: 0, - segment_count: 1, - width: 64, - height: 64, - output_stride: 64, - output_offset: 0, - missing_msbs: 0, - total_bitplanes: 8, - number_of_coding_passes: 1, - sub_band_type: 0, - style_flags: 0, - strict: 1, - dequantization_step: 1.0, - }]; - let segments = [J2kClassicSegment { - data_offset: 0, - data_length: 1, - start_coding_pass: 0, - end_coding_pass: 1, - use_arithmetic: 1, - }]; - - assert!( - !classic_repeated_uses_plain_fast_path(16, &jobs, &segments), - "batch-16 WSI classic J2K should keep the device-state cleanup plus separate store path" - ); - } - - #[test] - fn repeated_gray_store_detects_contiguous_full_wsi_tiles() { - let full_tile = J2kRepeatedGrayStoreParams { - input_width: 1024, - input_height: 1024, - source_x: 0, - source_y: 0, - copy_width: 1024, - copy_height: 1024, - output_width: 1024, - output_height: 1024, - output_x: 0, - output_y: 0, - addend: 0.0, - batch_count: 16, - max_value: 255.0, - u8_scale: 1.0, - u16_scale: 257.0, - }; - assert!( - repeated_gray_store_is_contiguous_full_surface(full_tile), - "full repeated grayscale WSI stores should use the contiguous store kernel" - ); - - let windowed = J2kRepeatedGrayStoreParams { - source_x: 1, - copy_width: 1023, - ..full_tile - }; - assert!( - !repeated_gray_store_is_contiguous_full_surface(windowed), - "ROI/windowed repeated grayscale stores must stay on the generic store kernel" - ); - } -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_image_to_surface<'a>( - image: &NativeImage<'a>, - context: &mut NativeDecoderContext<'a>, - fmt: PixelFormat, -) -> Result { - with_runtime(|runtime| { - let mut code_block_decoder = MetalCodeBlockDecoder::default(); - let decoded = image - .decode_components_with_ht_decoder(context, &mut code_block_decoder) - .map_err(|error| Error::Decode(signinum_j2k::J2kError::Backend(error.to_string())))?; - let stage = select_plane_stage(runtime, image, &decoded, &mut code_block_decoder)?; - stage.finish_with_runtime(runtime, fmt) - }) -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_image_to_surface_with_device<'a>( - image: &NativeImage<'a>, - context: &mut NativeDecoderContext<'a>, - fmt: PixelFormat, - device: &Device, -) -> Result { - with_runtime_for_device(device, |_| decode_image_to_surface(image, context, fmt)) -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_image_region_to_surface<'a>( - image: &NativeImage<'a>, - context: &mut NativeDecoderContext<'a>, - fmt: PixelFormat, - roi: Rect, -) -> Result { - with_runtime(|runtime| { - let mut code_block_decoder = MetalCodeBlockDecoder::default(); - let decoded = image - .decode_region_components_with_ht_decoder( - context, - (roi.x, roi.y, roi.w, roi.h), - &mut code_block_decoder, - ) - .map_err(|error| Error::Decode(signinum_j2k::J2kError::Backend(error.to_string())))?; - let stage = select_plane_stage(runtime, image, &decoded, &mut code_block_decoder)?; - stage.finish_with_runtime(runtime, fmt) - }) -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_image_region_to_surface_with_device<'a>( - image: &NativeImage<'a>, - context: &mut NativeDecoderContext<'a>, - fmt: PixelFormat, - roi: Rect, - device: &Device, -) -> Result { - with_runtime_for_device(device, |_| { - decode_image_region_to_surface(image, context, fmt, roi) - }) -} - -#[cfg(target_os = "macos")] -fn select_plane_stage( - runtime: &MetalRuntime, - image: &NativeImage<'_>, - decoded: &NativeDecodedComponents<'_>, - code_block_decoder: &mut MetalCodeBlockDecoder, -) -> Result { - if image.supports_direct_device_plane_reuse() { - if matches!(decoded.color_space(), NativeColorSpace::RGB) - && !decoded.has_alpha() - && decoded.planes().len() == 3 - { - if let Some(stage) = PlaneStage::from_captured_planes( - decoded, - code_block_decoder.mct.take_captured_planes(), - ) { - return Ok(stage); - } - } - if matches!(decoded.color_space(), NativeColorSpace::Gray) - && !decoded.has_alpha() - && decoded.planes().len() == 1 - { - if let Some(stage) = PlaneStage::from_captured_planes( - decoded, - code_block_decoder.store.take_captured_planes(), - ) { - return Ok(stage); - } - } - } - - PlaneStage::from_planes(&runtime.device, decoded, None) -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_scaled_to_surface( - bytes: &[u8], - dims: (u32, u32), - fmt: PixelFormat, - scale: signinum_core::Downscale, -) -> Result { - let target_dims = ( - dims.0.div_ceil(scale.denominator()), - dims.1.div_ceil(scale.denominator()), - ); - let settings = NativeDecodeSettings { - target_resolution: Some(target_dims), - ..NativeDecodeSettings::default() - }; - let image = NativeImage::new(bytes, &settings) - .map_err(|error| Error::Decode(signinum_j2k::J2kError::Backend(error.to_string())))?; - let mut context = NativeDecoderContext::default(); - decode_image_to_surface(&image, &mut context, fmt) -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_region_scaled_to_surface( - bytes: &[u8], - dims: (u32, u32), - fmt: PixelFormat, - roi: signinum_core::Rect, - scale: signinum_core::Downscale, -) -> Result { - let target_dims = ( - dims.0.div_ceil(scale.denominator()), - dims.1.div_ceil(scale.denominator()), - ); - let settings = NativeDecodeSettings { - target_resolution: Some(target_dims), - ..NativeDecodeSettings::default() - }; - let image = NativeImage::new(bytes, &settings) - .map_err(|error| Error::Decode(signinum_j2k::J2kError::Backend(error.to_string())))?; - let mut context = NativeDecoderContext::default(); - decode_image_region_to_surface(&image, &mut context, fmt, roi.scaled_covering(scale)) -} - #[cfg(target_os = "macos")] -pub(crate) fn decode_scaled_to_surface_with_device( - bytes: &[u8], - dims: (u32, u32), - fmt: PixelFormat, - scale: signinum_core::Downscale, - device: &Device, -) -> Result { - with_runtime_for_device(device, |_| { - decode_scaled_to_surface(bytes, dims, fmt, scale) - }) -} - +mod surface_decode; +#[cfg(all(test, target_os = "macos"))] +mod tests; #[cfg(target_os = "macos")] -pub(crate) fn decode_region_scaled_to_surface_with_device( - bytes: &[u8], - dims: (u32, u32), - fmt: PixelFormat, - roi: signinum_core::Rect, - scale: signinum_core::Downscale, - device: &Device, -) -> Result { - with_runtime_for_device(device, |_| { - decode_region_scaled_to_surface(bytes, dims, fmt, roi, scale) - }) -} +pub(crate) use self::surface_decode::{ + decode_image_region_to_surface, decode_image_region_to_surface_with_device, + decode_image_to_surface, decode_image_to_surface_with_device, decode_region_scaled_to_surface, + decode_region_scaled_to_surface_with_device, decode_scaled_to_surface, + decode_scaled_to_surface_with_device, +}; diff --git a/crates/j2k-metal/src/compute/abi.rs b/crates/j2k-metal/src/compute/abi.rs new file mode 100644 index 00000000..acb10043 --- /dev/null +++ b/crates/j2k-metal/src/compute/abi.rs @@ -0,0 +1,904 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kValidateBytesParams { + pub(crate) byte_len: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kValidateBytesStatus { + pub(crate) code: u32, + pub(crate) index: u32, + pub(crate) expected: u32, + pub(crate) actual: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kCopyInterleavedParams { + pub(crate) src_width: u32, + pub(crate) src_height: u32, + pub(crate) src_stride: u32, + pub(crate) dst_width: u32, + pub(crate) dst_height: u32, + pub(crate) dst_stride: u32, + pub(crate) bytes_per_pixel: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kLosslessDeinterleaveParams { + pub(crate) src_width: u32, + pub(crate) src_height: u32, + pub(crate) src_stride: u32, + pub(crate) dst_width: u32, + pub(crate) dst_height: u32, + pub(crate) components: u32, + pub(crate) bytes_per_sample: u32, + pub(crate) sample_offset: u32, + pub(crate) signed_samples: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kLosslessCoefficientJob { + pub(crate) coefficient_offset: u32, + pub(crate) component: u32, + pub(crate) subband_x: u32, + pub(crate) subband_y: u32, + pub(crate) block_x: u32, + pub(crate) block_y: u32, + pub(crate) block_width: u32, + pub(crate) block_height: u32, + pub(crate) full_width: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kPackParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) out_stride: u32, + pub(crate) output_channels: u32, + pub(crate) opaque_alpha: u32, + pub(crate) max_values: [f32; 4], + pub(crate) u8_scales: [f32; 4], + pub(crate) u16_scales: [f32; 4], +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kMctRgb8PackParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) out_stride: u32, + pub(crate) transform: u32, + pub(crate) addends: [f32; 3], + pub(crate) max_values: [f32; 3], + pub(crate) u8_scales: [f32; 3], +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kBatchedMctRgb8PackParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) out_stride: u32, + pub(crate) transform: u32, + pub(crate) batch_count: u32, + pub(crate) plane_stride: u32, + pub(crate) output_stride: u32, + pub(crate) addends: [f32; 3], + pub(crate) max_values: [f32; 3], + pub(crate) u8_scales: [f32; 3], +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kRepeatedGrayPackParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) out_stride: u32, + pub(crate) batch_count: u32, + pub(crate) max_value: f32, + pub(crate) u8_scale: f32, + pub(crate) u16_scale: f32, +} +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_STATUS_OK: u32 = 0; +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_STATUS_FAIL: u32 = 1; +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_STATUS_UNSUPPORTED: u32 = 2; +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES: u32 = 1 << 0; +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS: u32 = 1 << 1; +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT: u32 = 1 << 2; +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS: u32 = 1 << 3; +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS: u32 = 1 << 4; +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_MAX_WIDTH: u32 = 64; +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_MAX_HEIGHT: u32 = 64; +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_MAX_COEFF_COUNT: usize = + (J2K_CLASSIC_MAX_WIDTH as usize + 2) * (J2K_CLASSIC_MAX_HEIGHT as usize + 2); +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_ENCODE_32_MAX_WIDTH: u32 = 32; +#[cfg(target_os = "macos")] +pub(crate) const J2K_CLASSIC_ENCODE_32_MAX_HEIGHT: u32 = 32; + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kClassicCleanupBatchJob { + pub(crate) coded_offset: u32, + pub(crate) coded_len: u32, + pub(crate) segment_offset: u32, + pub(crate) segment_count: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) output_stride: u32, + pub(crate) output_offset: u32, + pub(crate) missing_msbs: u32, + pub(crate) total_bitplanes: u32, + pub(crate) roi_shift: u32, + pub(crate) number_of_coding_passes: u32, + pub(crate) sub_band_type: u32, + pub(crate) style_flags: u32, + pub(crate) strict: u32, + pub(crate) dequantization_step: f32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kClassicSegment { + pub(crate) data_offset: u32, + pub(crate) data_length: u32, + pub(crate) start_coding_pass: u32, + pub(crate) end_coding_pass: u32, + pub(crate) use_arithmetic: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kClassicStatus { + pub(crate) code: u32, + pub(crate) detail: u32, + pub(crate) reserved0: u32, + pub(crate) reserved1: u32, +} + +#[cfg(target_os = "macos")] +pub(crate) const J2K_IDWT_STATUS_OK: u32 = 0; +#[cfg(target_os = "macos")] +pub(crate) const J2K_IDWT_STATUS_FAIL: u32 = 1; + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kIdwtSingleDecompositionParams { + pub(crate) x0: u32, + pub(crate) y0: u32, + pub(crate) output_x: u32, + pub(crate) output_y: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) ll_x: u32, + pub(crate) ll_y: u32, + pub(crate) ll_width: u32, + pub(crate) ll_height: u32, + pub(crate) hl_x: u32, + pub(crate) hl_y: u32, + pub(crate) hl_width: u32, + pub(crate) hl_height: u32, + pub(crate) lh_x: u32, + pub(crate) lh_y: u32, + pub(crate) lh_width: u32, + pub(crate) lh_height: u32, + pub(crate) hh_x: u32, + pub(crate) hh_y: u32, + pub(crate) hh_width: u32, + pub(crate) hh_height: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kRepeatedIdwtSingleDecompositionParams { + pub(crate) x0: u32, + pub(crate) y0: u32, + pub(crate) output_x: u32, + pub(crate) output_y: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) ll_x: u32, + pub(crate) ll_y: u32, + pub(crate) ll_width: u32, + pub(crate) ll_height: u32, + pub(crate) hl_x: u32, + pub(crate) hl_y: u32, + pub(crate) hl_width: u32, + pub(crate) hl_height: u32, + pub(crate) lh_x: u32, + pub(crate) lh_y: u32, + pub(crate) lh_width: u32, + pub(crate) lh_height: u32, + pub(crate) hh_x: u32, + pub(crate) hh_y: u32, + pub(crate) hh_width: u32, + pub(crate) hh_height: u32, + pub(crate) ll_instance_stride: u32, + pub(crate) hl_instance_stride: u32, + pub(crate) lh_instance_stride: u32, + pub(crate) hh_instance_stride: u32, + pub(crate) batch_count: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kIdwtStatus { + pub(crate) code: u32, + pub(crate) detail: u32, + pub(crate) reserved0: u32, + pub(crate) reserved1: u32, +} + +#[cfg(target_os = "macos")] +pub(crate) const J2K_MCT_STATUS_OK: u32 = 0; +#[cfg(target_os = "macos")] +pub(crate) const J2K_MCT_STATUS_FAIL: u32 = 1; + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kInverseMctParams { + pub(crate) _len: u32, + pub(crate) _transform: u32, + pub(crate) _addend0: f32, + pub(crate) _addend1: f32, + pub(crate) _addend2: f32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kForwardRctParams { + pub(crate) _len: u32, + pub(crate) _reserved0: u32, + pub(crate) _reserved1: u32, + pub(crate) _reserved2: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kForwardIctParams { + pub(crate) _len: u32, + pub(crate) _reserved0: u32, + pub(crate) _reserved1: u32, + pub(crate) _reserved2: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kQuantizeSubbandParams { + pub(crate) _len: u32, + pub(crate) _step_exponent: u32, + pub(crate) _step_mantissa: u32, + pub(crate) _range_bits: u32, + pub(crate) _reversible: u32, + pub(crate) _reserved0: u32, + pub(crate) _reserved1: u32, + pub(crate) _reserved2: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kForwardDwt53Params { + pub(crate) full_width: u32, + pub(crate) current_width: u32, + pub(crate) current_height: u32, + pub(crate) low_width: u32, + pub(crate) low_height: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kForwardDwt53BatchedParams { + pub(crate) full_width: u32, + pub(crate) current_width: u32, + pub(crate) current_height: u32, + pub(crate) low_width: u32, + pub(crate) low_height: u32, + pub(crate) component_count: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kMctStatus { + pub(crate) code: u32, + pub(crate) detail: u32, + pub(crate) _reserved0: u32, + pub(crate) _reserved1: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kStoreParams { + pub(crate) input_width: u32, + pub(crate) source_x: u32, + pub(crate) source_y: u32, + pub(crate) copy_width: u32, + pub(crate) copy_height: u32, + pub(crate) output_width: u32, + pub(crate) output_x: u32, + pub(crate) output_y: u32, + pub(crate) addend: f32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kRepeatedStoreParams { + pub(crate) input_width: u32, + pub(crate) input_height: u32, + pub(crate) input_instance_stride: u32, + pub(crate) source_x: u32, + pub(crate) source_y: u32, + pub(crate) copy_width: u32, + pub(crate) copy_height: u32, + pub(crate) output_width: u32, + pub(crate) output_height: u32, + pub(crate) output_x: u32, + pub(crate) output_y: u32, + pub(crate) addend: f32, + pub(crate) batch_count: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kRepeatedGrayStoreParams { + pub(crate) input_width: u32, + pub(crate) input_height: u32, + pub(crate) source_x: u32, + pub(crate) source_y: u32, + pub(crate) copy_width: u32, + pub(crate) copy_height: u32, + pub(crate) output_width: u32, + pub(crate) output_height: u32, + pub(crate) output_x: u32, + pub(crate) output_y: u32, + pub(crate) addend: f32, + pub(crate) batch_count: u32, + pub(crate) max_value: f32, + pub(crate) u8_scale: f32, + pub(crate) u16_scale: f32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kGrayStoreParams { + pub(crate) input_width: u32, + pub(crate) source_x: u32, + pub(crate) source_y: u32, + pub(crate) copy_width: u32, + pub(crate) copy_height: u32, + pub(crate) output_width: u32, + pub(crate) output_x: u32, + pub(crate) output_y: u32, + pub(crate) addend: f32, + pub(crate) max_value: f32, + pub(crate) u8_scale: f32, + pub(crate) u16_scale: f32, +} + +pub(crate) const J2K_HT_STATUS_OK: u32 = 0; +#[cfg(target_os = "macos")] +pub(crate) const J2K_HT_STATUS_FAIL: u32 = 1; +#[cfg(target_os = "macos")] +pub(crate) const J2K_HT_STATUS_UNSUPPORTED: u32 = 2; + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kHtCleanupParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) coded_len: u32, + pub(crate) cleanup_length: u32, + pub(crate) refinement_length: u32, + pub(crate) missing_msbs: u32, + pub(crate) num_bitplanes: u32, + pub(crate) number_of_coding_passes: u32, + pub(crate) output_stride: u32, + pub(crate) output_offset: u32, + pub(crate) dequantization_step: f32, + pub(crate) stripe_causal: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kHtCleanupBatchJob { + pub(crate) coded_offset: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) coded_len: u32, + pub(crate) cleanup_length: u32, + pub(crate) refinement_length: u32, + pub(crate) missing_msbs: u32, + pub(crate) num_bitplanes: u32, + pub(crate) roi_shift: u32, + pub(crate) number_of_coding_passes: u32, + pub(crate) output_stride: u32, + pub(crate) output_offset: u32, + pub(crate) dequantization_step: f32, + pub(crate) stripe_causal: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kHtRepeatedBatchParams { + pub(crate) job_count: u32, + pub(crate) output_plane_len: u32, + pub(crate) batch_count: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kClassicRepeatedBatchParams { + pub(crate) job_count: u32, + pub(crate) output_plane_len: u32, + pub(crate) batch_count: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kHtStatus { + pub(crate) code: u32, + pub(crate) detail: u32, + pub(crate) reserved0: u32, + pub(crate) reserved1: u32, +} + +#[cfg(target_os = "macos")] +pub(crate) const J2K_ENCODE_STATUS_OK: u32 = 0; +#[cfg(target_os = "macos")] +pub(crate) const J2K_ENCODE_STATUS_FAIL: u32 = 1; +#[cfg(target_os = "macos")] +pub(crate) const J2K_ENCODE_STATUS_UNSUPPORTED: u32 = 2; +#[cfg(target_os = "macos")] +pub(crate) const J2K_HT_ENCODE_MEL_SIZE: usize = 192; +#[cfg(target_os = "macos")] +pub(crate) const J2K_HT_ENCODE_VLC_SIZE: usize = 3072 - J2K_HT_ENCODE_MEL_SIZE; +#[cfg(target_os = "macos")] +pub(crate) const J2K_HT_ENCODE_MS_SIZE: usize = (16_384usize * 16).div_ceil(15); +#[cfg(target_os = "macos")] +pub(crate) const J2K_HT_ENCODE_BASE_OUTPUT_SIZE: usize = + J2K_HT_ENCODE_MS_SIZE + J2K_HT_ENCODE_MEL_SIZE + J2K_HT_ENCODE_VLC_SIZE; +#[cfg(target_os = "macos")] +pub(crate) const J2K_HT_ENCODE_MAX_SAMPLES: usize = 16_384; +#[cfg(target_os = "macos")] +pub(crate) const J2K_HT_ENCODE_MS_BYTES_PER_SAMPLE_FLOOR: usize = 5; +#[cfg(target_os = "macos")] +pub(crate) const PACKET_PAYLOAD_COPY_BYTES_PER_STRIPE: u32 = 256; +#[cfg(target_os = "macos")] +pub(crate) const PACKET_PAYLOAD_COPY_STRIPES_PER_JOB: u32 = 4; + +#[cfg(target_os = "macos")] +pub(crate) const HT_PACKET_CAPACITY_ENV: &str = "J2K_METAL_HT_PACKET_CAPACITY"; +#[cfg(target_os = "macos")] +pub(crate) const CLASSIC_TIER1_TOKEN_ARENA_BYTES: usize = 4096; +#[cfg(target_os = "macos")] +pub(crate) const CLASSIC_TIER1_MQ_BYTE_TOKEN_ARENA_BYTES: usize = 8192; +#[cfg(target_os = "macos")] +pub(crate) const CLASSIC_TIER1_TOKEN_SEGMENT_CAPACITY: usize = 48; +#[cfg(target_os = "macos")] +pub(crate) const CLASSIC_TIER1_PASS_PLAN_CAPACITY: usize = 48; +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kClassicEncodeParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) sub_band_type: u32, + pub(crate) total_bitplanes: u32, + pub(crate) style_flags: u32, + pub(crate) output_capacity: u32, + pub(crate) segment_capacity: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kClassicEncodeBatchJob { + pub(crate) coefficient_offset: u32, + pub(crate) output_offset: u32, + pub(crate) segment_offset: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) sub_band_type: u32, + pub(crate) total_bitplanes: u32, + pub(crate) style_flags: u32, + pub(crate) output_capacity: u32, + pub(crate) segment_capacity: u32, +} +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kClassicEncodeStatus { + pub(crate) code: u32, + pub(crate) detail: u32, + pub(crate) data_len: u32, + pub(crate) number_of_coding_passes: u32, + pub(crate) missing_bit_planes: u32, + pub(crate) segment_count: u32, + pub(crate) reserved0: u32, + pub(crate) reserved1: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kClassicTier1DensityCounters { + pub(crate) sigprop_active_candidates: u32, + pub(crate) sigprop_new_significant: u32, + pub(crate) magref_active_candidates: u32, + pub(crate) cleanup_active_candidates: u32, + pub(crate) cleanup_new_significant: u32, + pub(crate) cleanup_rlc_stripes: u32, + pub(crate) cleanup_rlc_zero_stripes: u32, + pub(crate) arithmetic_sigprop_active_candidates: u32, + pub(crate) arithmetic_sigprop_new_significant: u32, + pub(crate) raw_sigprop_active_candidates: u32, + pub(crate) raw_sigprop_new_significant: u32, + pub(crate) arithmetic_magref_active_candidates: u32, + pub(crate) raw_magref_active_candidates: u32, + pub(crate) reserved0: u32, + pub(crate) reserved1: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kClassicTier1SymbolPlanCounters { + pub(crate) code: u32, + pub(crate) detail: u32, + pub(crate) coding_passes: u32, + pub(crate) missing_bit_planes: u32, + pub(crate) segment_count: u32, + pub(crate) mq_symbol_count: u32, + pub(crate) raw_bit_count: u32, + pub(crate) cleanup_mq_symbol_count: u32, + pub(crate) sigprop_mq_symbol_count: u32, + pub(crate) magref_mq_symbol_count: u32, + pub(crate) raw_sigprop_bit_count: u32, + pub(crate) raw_magref_bit_count: u32, + pub(crate) cleanup_sign_symbol_count: u32, + pub(crate) sigprop_sign_symbol_count: u32, + pub(crate) mq_symbol_hash: u32, + pub(crate) raw_bit_hash: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kClassicTier1PassPlanCounters { + pub(crate) code: u32, + pub(crate) detail: u32, + pub(crate) coding_passes: u32, + pub(crate) missing_bit_planes: u32, + pub(crate) segment_count: u32, + pub(crate) mq_symbol_count: u32, + pub(crate) raw_bit_count: u32, + pub(crate) nonempty_mq_passes: u32, + pub(crate) nonempty_raw_passes: u32, + pub(crate) max_mq_symbols_per_pass: u32, + pub(crate) max_raw_bits_per_pass: u32, + pub(crate) reserved0: u32, + pub(crate) reserved1: u32, + pub(crate) reserved2: u32, + pub(crate) reserved3: u32, + pub(crate) reserved4: u32, + pub(crate) mq_symbols_by_pass: [u32; CLASSIC_TIER1_PASS_PLAN_CAPACITY], + pub(crate) raw_bits_by_pass: [u32; CLASSIC_TIER1_PASS_PLAN_CAPACITY], +} + +#[cfg(target_os = "macos")] +impl Default for J2kClassicTier1PassPlanCounters { + fn default() -> Self { + Self { + code: 0, + detail: 0, + coding_passes: 0, + missing_bit_planes: 0, + segment_count: 0, + mq_symbol_count: 0, + raw_bit_count: 0, + nonempty_mq_passes: 0, + nonempty_raw_passes: 0, + max_mq_symbols_per_pass: 0, + max_raw_bits_per_pass: 0, + reserved0: 0, + reserved1: 0, + reserved2: 0, + reserved3: 0, + reserved4: 0, + mq_symbols_by_pass: [0; CLASSIC_TIER1_PASS_PLAN_CAPACITY], + raw_bits_by_pass: [0; CLASSIC_TIER1_PASS_PLAN_CAPACITY], + } + } +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kClassicTier1TokenSegment { + pub(crate) token_bit_offset: u32, + pub(crate) token_bit_count: u32, + pub(crate) pass_range: u32, + pub(crate) flags: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kHtEncodeParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) total_bitplanes: u32, + pub(crate) output_capacity: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kHtEncodeBatchJob { + pub(crate) coefficient_offset: u32, + pub(crate) output_offset: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) total_bitplanes: u32, + pub(crate) output_capacity: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kHtEncodeStatus { + pub(crate) code: u32, + pub(crate) detail: u32, + pub(crate) data_len: u32, + pub(crate) num_coding_passes: u32, + pub(crate) num_zero_bitplanes: u32, + pub(crate) reserved0: u32, + pub(crate) reserved1: u32, + pub(crate) reserved2: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kPacketEncodeParams { + pub(crate) resolution_count: u32, + pub(crate) num_layers: u32, + pub(crate) num_components: u32, + pub(crate) code_block_count: u32, + pub(crate) subband_count: u32, + pub(crate) descriptor_count: u32, + pub(crate) output_capacity: u32, + pub(crate) header_capacity: u32, + pub(crate) scratch_node_capacity: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kBatchedPacketEncodeJob { + pub(crate) resolution_offset: u32, + pub(crate) subband_offset: u32, + pub(crate) block_offset: u32, + pub(crate) descriptor_offset: u32, + pub(crate) state_block_offset: u32, + pub(crate) output_offset: u32, + pub(crate) header_offset: u32, + pub(crate) scratch_offset: u32, + pub(crate) payload_copy_offset: u32, + pub(crate) payload_copy_capacity: u32, + pub(crate) resolution_count: u32, + pub(crate) num_layers: u32, + pub(crate) num_components: u32, + pub(crate) code_block_count: u32, + pub(crate) subband_count: u32, + pub(crate) descriptor_count: u32, + pub(crate) output_capacity: u32, + pub(crate) header_capacity: u32, + pub(crate) scratch_node_capacity: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kPacketPayloadCopyJob { + pub(crate) src_offset: u32, + pub(crate) dst_offset: u32, + pub(crate) byte_len: u32, + pub(crate) reserved0: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kPacketPayloadCopyParams { + pub(crate) bytes_per_thread: u32, + pub(crate) stripes_per_job: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kPacketDescriptor { + pub(crate) packet_index: u32, + pub(crate) state_index: u32, + pub(crate) layer: u32, + pub(crate) resolution: u32, + pub(crate) component: u32, + pub(crate) precinct_lo: u32, + pub(crate) precinct_hi: u32, + pub(crate) state_block_offset: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kPacketResolution { + pub(crate) subband_offset: u32, + pub(crate) subband_count: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kPacketSubband { + pub(crate) block_offset: u32, + pub(crate) block_count: u32, + pub(crate) num_cbs_x: u32, + pub(crate) num_cbs_y: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kPacketBlock { + pub(crate) data_offset: u32, + pub(crate) data_len: u32, + pub(crate) num_coding_passes: u32, + pub(crate) num_zero_bitplanes: u32, + pub(crate) previously_included: u32, + pub(crate) l_block: u32, + pub(crate) block_coding_mode: u32, + pub(crate) reserved0: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kResidentPacketBlock { + pub(crate) tier1_job_index: u32, + pub(crate) previously_included: u32, + pub(crate) l_block: u32, + pub(crate) block_coding_mode: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kResidentPacketBlockParams { + pub(crate) block_count: u32, + pub(crate) tier1_job_count: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kPacketStateBlock { + pub(crate) previously_included: u32, + pub(crate) l_block: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kPacketEncodeStatus { + pub(crate) code: u32, + pub(crate) detail: u32, + pub(crate) data_len: u32, + pub(crate) reserved0: u32, + pub(crate) payload_copy_bytes: u32, + pub(crate) payload_copy_small_jobs: u32, + pub(crate) payload_copy_medium_jobs: u32, + pub(crate) payload_copy_large_jobs: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy)] +pub(crate) struct J2kLosslessCodestreamAssemblyParams { + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) num_components: u32, + pub(crate) bit_depth: u32, + pub(crate) signed_samples: u32, + pub(crate) num_decomposition_levels: u32, + pub(crate) use_mct: u32, + pub(crate) guard_bits: u32, + pub(crate) progression_order: u32, + pub(crate) write_tlm: u32, + pub(crate) high_throughput: u32, + pub(crate) code_block_style: u32, + pub(crate) code_block_width_exp: u32, + pub(crate) code_block_height_exp: u32, + pub(crate) output_capacity: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kBatchedCodestreamAssemblyJob { + pub(crate) tile_data_offset: u32, + pub(crate) codestream_offset: u32, + pub(crate) width: u32, + pub(crate) height: u32, + pub(crate) num_components: u32, + pub(crate) bit_depth: u32, + pub(crate) signed_samples: u32, + pub(crate) num_decomposition_levels: u32, + pub(crate) use_mct: u32, + pub(crate) guard_bits: u32, + pub(crate) progression_order: u32, + pub(crate) write_tlm: u32, + pub(crate) high_throughput: u32, + pub(crate) code_block_style: u32, + pub(crate) code_block_width_exp: u32, + pub(crate) code_block_height_exp: u32, + pub(crate) output_capacity: u32, +} + +#[cfg(target_os = "macos")] +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub(crate) struct J2kCodestreamAssemblyStatus { + pub(crate) code: u32, + pub(crate) detail: u32, + pub(crate) data_len: u32, + pub(crate) reserved0: u32, +} diff --git a/crates/j2k-metal/src/compute/buffer_validation.rs b/crates/j2k-metal/src/compute/buffer_validation.rs new file mode 100644 index 00000000..f58b3cf0 --- /dev/null +++ b/crates/j2k-metal/src/compute/buffer_validation.rs @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::mem::size_of; + +use j2k_metal_support::{dispatch_2d_pipeline, dispatch_single_thread}; +use metal::{Buffer, MTLResourceOptions}; + +use crate::{profile_env::label_command_buffer, Error}; + +use super::{ + with_runtime_for_session, J2kCopyInterleavedParams, J2kValidateBytesParams, + J2kValidateBytesStatus, +}; + +pub(crate) fn validate_metal_buffer_matches_bytes( + expected: &[u8], + actual_buffer: &Buffer, + actual_byte_offset: usize, + session: &crate::MetalBackendSession, +) -> Result<(), Error> { + if expected.is_empty() { + return Ok(()); + } + let byte_len = u32::try_from(expected.len()).map_err(|_| Error::MetalKernel { + message: "J2K Metal validation buffer exceeds u32 byte length".to_string(), + })?; + let actual_offset = u64::try_from(actual_byte_offset).map_err(|_| Error::MetalKernel { + message: "J2K Metal validation buffer offset exceeds u64".to_string(), + })?; + + with_runtime_for_session(session, |runtime| { + let expected_buffer = runtime.device.new_buffer_with_data( + expected.as_ptr().cast(), + expected.len() as u64, + MTLResourceOptions::StorageModeShared, + ); + let status = J2kValidateBytesStatus::default(); + let status_buffer = runtime.device.new_buffer_with_data( + (&raw const status).cast(), + size_of::() as u64, + MTLResourceOptions::StorageModeShared, + ); + let params = J2kValidateBytesParams { byte_len }; + + let command_buffer = runtime.queue.new_command_buffer(); + label_command_buffer(command_buffer, "j2k lossless coefficient prep"); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.validate_bytes_equal); + encoder.set_buffer(0, Some(actual_buffer), actual_offset); + encoder.set_buffer(1, Some(&expected_buffer), 0); + encoder.set_buffer(2, Some(&status_buffer), 0); + encoder.set_bytes( + 3, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_single_thread(encoder); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let status = unsafe { + status_buffer + .contents() + .cast::() + .read() + }; + if status.code == 0 { + return Ok(()); + } + + Err(Error::MetalKernel { + message: format!( + "J2K Metal validation mismatch at byte {}: expected {}, got {}", + status.index, status.expected, status.actual + ), + }) + }) +} + +pub(crate) fn validate_metal_buffers_match( + expected_buffer: &Buffer, + expected_byte_offset: usize, + actual_buffer: &Buffer, + actual_byte_offset: usize, + byte_len: usize, + session: &crate::MetalBackendSession, +) -> Result<(), Error> { + if byte_len == 0 { + return Ok(()); + } + let byte_len_u32 = u32::try_from(byte_len).map_err(|_| Error::MetalKernel { + message: "J2K Metal validation buffer exceeds u32 byte length".to_string(), + })?; + let expected_offset = u64::try_from(expected_byte_offset).map_err(|_| Error::MetalKernel { + message: "J2K Metal validation expected buffer offset exceeds u64".to_string(), + })?; + let actual_offset = u64::try_from(actual_byte_offset).map_err(|_| Error::MetalKernel { + message: "J2K Metal validation actual buffer offset exceeds u64".to_string(), + })?; + + with_runtime_for_session(session, |runtime| { + let status = J2kValidateBytesStatus::default(); + let status_buffer = runtime.device.new_buffer_with_data( + (&raw const status).cast(), + size_of::() as u64, + MTLResourceOptions::StorageModeShared, + ); + let params = J2kValidateBytesParams { + byte_len: byte_len_u32, + }; + + let command_buffer = runtime.queue.new_command_buffer(); + label_command_buffer(command_buffer, "j2k lossless coefficient prep batch"); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.validate_bytes_equal); + encoder.set_buffer(0, Some(actual_buffer), actual_offset); + encoder.set_buffer(1, Some(expected_buffer), expected_offset); + encoder.set_buffer(2, Some(&status_buffer), 0); + encoder.set_bytes( + 3, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_single_thread(encoder); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let status = unsafe { + status_buffer + .contents() + .cast::() + .read() + }; + if status.code == 0 { + return Ok(()); + } + + Err(Error::MetalKernel { + message: format!( + "J2K Metal validation mismatch at byte {}: expected {}, got {}", + status.index, status.expected, status.actual + ), + }) + }) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn copy_interleaved_padded_to_shared_buffer( + src_buffer: &Buffer, + src_byte_offset: usize, + src_width: u32, + src_height: u32, + src_pitch_bytes: usize, + dst_width: u32, + dst_height: u32, + bytes_per_pixel: usize, + session: &crate::MetalBackendSession, +) -> Result { + if src_width > dst_width || src_height > dst_height { + return Err(Error::MetalKernel { + message: "J2K Metal input tile cannot be larger than encoded tile".to_string(), + }); + } + let src_stride = u32::try_from(src_pitch_bytes).map_err(|_| Error::MetalKernel { + message: "J2K Metal input tile pitch exceeds u32".to_string(), + })?; + let dst_stride_usize = (dst_width as usize) + .checked_mul(bytes_per_pixel) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal padded tile stride overflow".to_string(), + })?; + let dst_stride = u32::try_from(dst_stride_usize).map_err(|_| Error::MetalKernel { + message: "J2K Metal padded tile stride exceeds u32".to_string(), + })?; + let dst_len = dst_stride_usize + .checked_mul(dst_height as usize) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal padded tile byte length overflow".to_string(), + })?; + let bytes_per_pixel = u32::try_from(bytes_per_pixel).map_err(|_| Error::MetalKernel { + message: "J2K Metal bytes-per-pixel exceeds u32".to_string(), + })?; + let src_offset = u64::try_from(src_byte_offset).map_err(|_| Error::MetalKernel { + message: "J2K Metal input tile offset exceeds u64".to_string(), + })?; + + with_runtime_for_session(session, |runtime| { + let dst_buffer = runtime + .device + .new_buffer(dst_len as u64, MTLResourceOptions::StorageModeShared); + let params = J2kCopyInterleavedParams { + src_width, + src_height, + src_stride, + dst_width, + dst_height, + dst_stride, + bytes_per_pixel, + }; + let command_buffer = runtime.queue.new_command_buffer(); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.copy_interleaved_padded); + encoder.set_buffer(0, Some(src_buffer), src_offset); + encoder.set_buffer(1, Some(&dst_buffer), 0); + encoder.set_bytes( + 2, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_2d_pipeline( + encoder, + &runtime.copy_interleaved_padded, + (dst_width, dst_height), + ); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + Ok(dst_buffer) + }) +} diff --git a/crates/j2k-metal/src/compute/classic_encode_pipeline.rs b/crates/j2k-metal/src/compute/classic_encode_pipeline.rs new file mode 100644 index 00000000..e39311bb --- /dev/null +++ b/crates/j2k-metal/src/compute/classic_encode_pipeline.rs @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 + +use metal::ComputePipelineState; + +use crate::profile_env::classic_selective_bypass_disabled; + +use super::{ + J2kClassicEncodeBatchJob, MetalRuntime, J2K_CLASSIC_ENCODE_32_MAX_HEIGHT, + J2K_CLASSIC_ENCODE_32_MAX_WIDTH, J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES, + J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, + J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS, J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT, +}; + +pub(super) fn classic_resident_style_flags_from_env() -> u32 { + if classic_selective_bypass_disabled() { + 0 + } else { + J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS + } +} + +pub(super) fn classic_cod_block_style_from_flags(flags: u32) -> u32 { + let mut style = 0u32; + if (flags & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) != 0 { + style |= 0x01; + } + if (flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0 { + style |= 0x02; + } + if (flags & J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS) != 0 { + style |= 0x04; + } + if (flags & J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT) != 0 { + style |= 0x08; + } + if (flags & J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS) != 0 { + style |= 0x20; + } + style +} + +pub(super) fn classic_tier1_gpu_token_pack_supported(jobs: &[J2kClassicEncodeBatchJob]) -> bool { + !jobs.is_empty() + && classic_encode_code_blocks_pipeline_kind(jobs) + == J2kClassicEncodePipelineKind::BypassU16_32 +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum J2kClassicEncodePipelineKind { + Generic, + Generic32, + Bypass32, + BypassU16_32, + Style0, + Style0_32, +} + +pub(super) fn classic_encode_code_blocks_pipeline_kind( + jobs: &[J2kClassicEncodeBatchJob], +) -> J2kClassicEncodePipelineKind { + let all_32 = jobs.iter().all(|job| { + job.width <= J2K_CLASSIC_ENCODE_32_MAX_WIDTH + && job.height <= J2K_CLASSIC_ENCODE_32_MAX_HEIGHT + }); + let all_style0 = jobs.iter().all(|job| job.style_flags == 0); + let all_bypass = jobs + .iter() + .all(|job| job.style_flags == J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS); + let all_u16_bitplanes = jobs.iter().all(|job| job.total_bitplanes <= 16); + match (all_style0, all_bypass, all_32, all_u16_bitplanes) { + (true, _, true, _) => J2kClassicEncodePipelineKind::Style0_32, + (true, _, false, _) => J2kClassicEncodePipelineKind::Style0, + (false, true, true, true) => J2kClassicEncodePipelineKind::BypassU16_32, + (false, true, true, false) => J2kClassicEncodePipelineKind::Bypass32, + (false, _, true, _) => J2kClassicEncodePipelineKind::Generic32, + (false, _, false, _) => J2kClassicEncodePipelineKind::Generic, + } +} + +pub(super) fn classic_encode_code_blocks_pipeline<'a>( + runtime: &'a MetalRuntime, + jobs: &[J2kClassicEncodeBatchJob], +) -> &'a ComputePipelineState { + match classic_encode_code_blocks_pipeline_kind(jobs) { + J2kClassicEncodePipelineKind::Generic => &runtime.classic_encode_code_blocks, + J2kClassicEncodePipelineKind::Generic32 => &runtime.classic_encode_code_blocks_32, + J2kClassicEncodePipelineKind::Bypass32 => &runtime.classic_encode_code_blocks_bypass_32, + J2kClassicEncodePipelineKind::BypassU16_32 => { + &runtime.classic_encode_code_blocks_bypass_u16_32 + } + J2kClassicEncodePipelineKind::Style0 => &runtime.classic_encode_code_blocks_style0, + J2kClassicEncodePipelineKind::Style0_32 => &runtime.classic_encode_code_blocks_style0_32, + } +} diff --git a/crates/j2k-metal/src/compute/classic_tier1_stats.rs b/crates/j2k-metal/src/compute/classic_tier1_stats.rs new file mode 100644 index 00000000..cb3a63c4 --- /dev/null +++ b/crates/j2k-metal/src/compute/classic_tier1_stats.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: Apache-2.0 + +use super::{J2kResidentEncodeStageStats, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) struct J2kClassicTier1PassClassCounts { + pub(super) arithmetic: usize, + pub(super) raw: usize, + pub(super) cleanup: usize, + pub(super) sigprop: usize, + pub(super) magref: usize, + pub(super) arithmetic_cleanup: usize, + pub(super) arithmetic_sigprop: usize, + pub(super) arithmetic_magref: usize, + pub(super) raw_sigprop: usize, + pub(super) raw_magref: usize, +} + +pub(super) fn classic_tier1_pass_class_counts( + coding_passes: usize, + style_flags: u32, +) -> J2kClassicTier1PassClassCounts { + let selective_bypass = + (style_flags & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) != 0; + let mut counts = J2kClassicTier1PassClassCounts::default(); + for coding_pass in 0..coding_passes { + let pass_type = coding_pass % 3; + let arithmetic = !selective_bypass || coding_pass <= 9 || pass_type == 0; + match pass_type { + 0 => { + counts.cleanup = counts.cleanup.saturating_add(1); + counts.arithmetic_cleanup = counts.arithmetic_cleanup.saturating_add(1); + } + 1 => { + counts.sigprop = counts.sigprop.saturating_add(1); + if arithmetic { + counts.arithmetic_sigprop = counts.arithmetic_sigprop.saturating_add(1); + } else { + counts.raw_sigprop = counts.raw_sigprop.saturating_add(1); + } + } + _ => { + counts.magref = counts.magref.saturating_add(1); + if arithmetic { + counts.arithmetic_magref = counts.arithmetic_magref.saturating_add(1); + } else { + counts.raw_magref = counts.raw_magref.saturating_add(1); + } + } + } + if arithmetic { + counts.arithmetic = counts.arithmetic.saturating_add(1); + } else { + counts.raw = counts.raw.saturating_add(1); + } + } + counts +} + +pub(super) fn accumulate_classic_tier1_scan_estimates( + stage_stats: &mut J2kResidentEncodeStageStats, + pass_counts: J2kClassicTier1PassClassCounts, + coeff_count: usize, +) { + let full_scan_visits = pass_counts + .cleanup + .saturating_add(pass_counts.sigprop) + .saturating_add(pass_counts.magref) + .saturating_mul(coeff_count); + stage_stats.tier1_full_scan_coeff_visit_count_total = stage_stats + .tier1_full_scan_coeff_visit_count_total + .saturating_add(full_scan_visits); + stage_stats.max_tier1_full_scan_coeff_visits_per_block = stage_stats + .max_tier1_full_scan_coeff_visits_per_block + .max(full_scan_visits); + stage_stats.tier1_arithmetic_scan_coeff_visit_count_total = stage_stats + .tier1_arithmetic_scan_coeff_visit_count_total + .saturating_add(pass_counts.arithmetic.saturating_mul(coeff_count)); + stage_stats.tier1_raw_scan_coeff_visit_count_total = stage_stats + .tier1_raw_scan_coeff_visit_count_total + .saturating_add(pass_counts.raw.saturating_mul(coeff_count)); + stage_stats.tier1_cleanup_scan_coeff_visit_count_total = stage_stats + .tier1_cleanup_scan_coeff_visit_count_total + .saturating_add(pass_counts.cleanup.saturating_mul(coeff_count)); + stage_stats.tier1_sigprop_scan_coeff_visit_count_total = stage_stats + .tier1_sigprop_scan_coeff_visit_count_total + .saturating_add(pass_counts.sigprop.saturating_mul(coeff_count)); + stage_stats.tier1_magref_scan_coeff_visit_count_total = stage_stats + .tier1_magref_scan_coeff_visit_count_total + .saturating_add(pass_counts.magref.saturating_mul(coeff_count)); +} diff --git a/crates/j2k-metal/src/compute/code_block_decoder.rs b/crates/j2k-metal/src/compute/code_block_decoder.rs new file mode 100644 index 00000000..0ffbbfde --- /dev/null +++ b/crates/j2k-metal/src/compute/code_block_decoder.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_native::{ + HtCodeBlockDecodeJob, HtCodeBlockDecoder, HtSubBandDecodeJob, J2kCodeBlockDecodeJob, + J2kInverseMctJob, J2kSingleDecompositionIdwtJob, J2kStoreComponentJob, J2kSubBandDecodeJob, +}; + +use crate::{ + classic::MetalClassicBlockDecoder, ht::MetalHtBlockDecoder, idwt::MetalIdwtDecoder, + mct::MetalMctDecoder, store::MetalStoreDecoder, +}; + +#[derive(Default)] +pub(super) struct MetalCodeBlockDecoder { + classic: MetalClassicBlockDecoder, + ht: MetalHtBlockDecoder, + idwt: MetalIdwtDecoder, + pub(super) mct: MetalMctDecoder, + pub(super) store: MetalStoreDecoder, +} + +impl HtCodeBlockDecoder for MetalCodeBlockDecoder { + fn decode_j2k_sub_band( + &mut self, + job: J2kSubBandDecodeJob<'_>, + output: &mut [f32], + ) -> j2k_native::Result { + self.classic.decode_j2k_sub_band(job, output) + } + + fn decode_j2k_code_block( + &mut self, + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], + ) -> j2k_native::Result { + self.classic.decode_j2k_code_block(job, output) + } + + fn decode_sub_band( + &mut self, + job: HtSubBandDecodeJob<'_>, + output: &mut [f32], + ) -> j2k_native::Result { + self.ht.decode_sub_band(job, output) + } + + fn decode_code_block( + &mut self, + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + ) -> j2k_native::Result<()> { + self.ht.decode_code_block(job, output) + } + + fn decode_single_decomposition_idwt( + &mut self, + job: J2kSingleDecompositionIdwtJob<'_>, + output: &mut [f32], + ) -> j2k_native::Result { + self.idwt.decode_single_decomposition_idwt(job, output) + } + + fn decode_inverse_mct(&mut self, job: J2kInverseMctJob<'_>) -> j2k_native::Result { + self.mct.decode_inverse_mct(job) + } + + fn decode_store_component( + &mut self, + job: J2kStoreComponentJob<'_>, + ) -> j2k_native::Result { + self.store.decode_store_component(job) + } +} diff --git a/crates/j2k-metal/src/compute/direct_buffers.rs b/crates/j2k-metal/src/compute/direct_buffers.rs new file mode 100644 index 00000000..55b06205 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_buffers.rs @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::mem::{size_of, size_of_val}; + +use metal::{Buffer, Device, MTLResourceOptions}; + +use crate::Error; + +use super::{ + direct_scratch::{take_recyclable_shared_buffer, DirectScratchBuffer}, + MetalRuntime, J2K_CLASSIC_MAX_COEFF_COUNT, +}; + +pub(super) fn owned_slice_buffer(device: &Device, data: &[T]) -> Buffer { + let size = size_of_val(data).max(1); + let buffer = device.new_buffer(size as u64, MTLResourceOptions::StorageModeShared); + if !data.is_empty() { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + unsafe { + core::ptr::copy_nonoverlapping( + data.as_ptr().cast::(), + buffer.contents().cast::(), + size_of_val(data), + ); + } + } + buffer +} + +pub(super) fn wrap_f32_output_buffer(device: &Device, output: &mut [f32]) -> Buffer { + if output.is_empty() { + device.new_buffer( + size_of::() as u64, + MTLResourceOptions::StorageModeShared, + ) + } else { + device.new_buffer_with_bytes_no_copy( + output.as_mut_ptr().cast(), + size_of_val(output) as u64, + MTLResourceOptions::StorageModeShared, + None, + ) + } +} + +pub(super) fn borrow_slice_buffer(device: &Device, data: &[T]) -> Buffer { + if data.is_empty() { + device.new_buffer(1, MTLResourceOptions::StorageModeShared) + } else { + device.new_buffer_with_bytes_no_copy( + data.as_ptr().cast(), + size_of_val(data) as u64, + MTLResourceOptions::StorageModeShared, + None, + ) + } +} + +pub(super) fn borrow_mut_slice_buffer(device: &Device, data: &mut [T]) -> Buffer { + if data.is_empty() { + device.new_buffer(1, MTLResourceOptions::StorageModeShared) + } else { + device.new_buffer_with_bytes_no_copy( + data.as_mut_ptr().cast(), + size_of_val(data) as u64, + MTLResourceOptions::StorageModeShared, + None, + ) + } +} + +pub(super) fn copied_slice_buffer(device: &Device, data: &[T]) -> Buffer { + if data.is_empty() { + device.new_buffer(1, MTLResourceOptions::StorageModeShared) + } else { + device.new_buffer_with_data( + data.as_ptr().cast(), + size_of_val(data) as u64, + MTLResourceOptions::StorageModeShared, + ) + } +} + +pub(super) fn copied_recyclable_shared_slice_buffer( + runtime: &MetalRuntime, + data: &[T], + recyclable_shared_buffers: &mut Vec<(usize, Buffer)>, +) -> Result { + let size = size_of_val(data).max(1); + let buffer = take_recyclable_shared_buffer(runtime, size, recyclable_shared_buffers)?; + if !data.is_empty() { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + unsafe { + core::ptr::copy_nonoverlapping( + data.as_ptr().cast::(), + buffer.contents().cast::(), + size_of_val(data), + ); + } + } + Ok(buffer) +} + +pub(super) fn zeroed_recyclable_shared_buffer( + runtime: &MetalRuntime, + bytes: usize, + recyclable_shared_buffers: &mut Vec<(usize, Buffer)>, +) -> Result { + let bytes = bytes.max(1); + let buffer = take_recyclable_shared_buffer(runtime, bytes, recyclable_shared_buffers)?; + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + unsafe { + core::ptr::write_bytes(buffer.contents().cast::(), 0, bytes); + } + Ok(buffer) +} + +fn classic_coefficients_scratch_bytes(job_count: usize) -> Result { + job_count + .max(1) + .checked_mul(J2K_CLASSIC_MAX_COEFF_COUNT) + .and_then(|count| count.checked_mul(size_of::())) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K coefficient scratch size overflow".to_string(), + }) +} + +pub(super) fn take_classic_coefficients_scratch_buffer( + runtime: &MetalRuntime, + job_count: usize, +) -> Result { + let bytes = classic_coefficients_scratch_bytes(job_count)?; + Ok(DirectScratchBuffer { + bytes, + buffer: runtime.take_private_buffer(bytes)?, + }) +} + +fn classic_states_scratch_bytes(job_count: usize) -> Result { + job_count + .max(1) + .checked_mul(J2K_CLASSIC_MAX_COEFF_COUNT) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect states scratch overflow".to_string(), + }) +} + +pub(super) fn take_classic_states_scratch_buffer( + runtime: &MetalRuntime, + job_count: usize, +) -> Result { + let bytes = classic_states_scratch_bytes(job_count)?; + Ok(DirectScratchBuffer { + bytes, + buffer: runtime.take_private_buffer(bytes)?, + }) +} diff --git a/crates/j2k-metal/src/compute/direct_cache.rs b/crates/j2k-metal/src/compute/direct_cache.rs new file mode 100644 index 00000000..6fda9ab5 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_cache.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use super::PreparedDirectGrayscalePlan; +use crate::Error; + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct CpuTier1CoefficientCacheKey { + step_idx: usize, + output_len: usize, +} + +#[derive(Default)] +pub(super) struct CpuTier1CoefficientCache { + entries: Mutex>>, +} + +impl PreparedDirectGrayscalePlan { + pub(super) fn cached_cpu_tier1_coefficients( + &self, + step_idx: usize, + output_len: usize, + ) -> Result>, Error> { + let key = CpuTier1CoefficientCacheKey { + step_idx, + output_len, + }; + let entries = self + .cpu_tier1_cache + .entries + .lock() + .map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect hybrid CPU Tier-1 cache lock is poisoned".to_string(), + })?; + Ok(entries.get(&key).map(|coefficients| coefficients.to_vec())) + } + + pub(super) fn store_cpu_tier1_coefficients( + &self, + step_idx: usize, + output_len: usize, + coefficients: Vec, + ) -> Result, Error> { + let key = CpuTier1CoefficientCacheKey { + step_idx, + output_len, + }; + let cached = Arc::<[f32]>::from(coefficients.clone()); + let mut entries = self + .cpu_tier1_cache + .entries + .lock() + .map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect hybrid CPU Tier-1 cache lock is poisoned".to_string(), + })?; + entries.insert(key, cached); + Ok(coefficients) + } + + pub(super) fn clear_cpu_tier1_cache(&self) -> Result<(), Error> { + let mut entries = self + .cpu_tier1_cache + .entries + .lock() + .map_err(|_| Error::MetalKernel { + message: "J2K MetalDirect hybrid CPU Tier-1 cache lock is poisoned".to_string(), + })?; + entries.clear(); + Ok(()) + } +} diff --git a/crates/j2k-metal/src/compute/direct_commands.rs b/crates/j2k-metal/src/compute/direct_commands.rs new file mode 100644 index 00000000..aeec1377 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_commands.rs @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 + +use metal::{CommandBuffer, CommandBufferRef}; + +use crate::profile_env::label_command_buffer; + +use super::MetalRuntime; + +#[derive(Clone, Copy)] +pub(super) struct DirectIdwtCommandBuffers<'a> { + pub(super) interleave: &'a CommandBufferRef, + pub(super) horizontal: &'a CommandBufferRef, + pub(super) vertical: &'a CommandBufferRef, +} + +impl<'a> DirectIdwtCommandBuffers<'a> { + pub(super) fn single(command_buffer: &'a CommandBufferRef) -> Self { + Self { + interleave: command_buffer, + horizontal: command_buffer, + vertical: command_buffer, + } + } +} + +#[derive(Clone, Copy)] +pub(super) struct DirectColorBatchCommandBuffers<'a> { + pub(super) default: &'a CommandBufferRef, + pub(super) idwt: DirectIdwtCommandBuffers<'a>, + pub(super) store: &'a CommandBufferRef, + pub(super) mct_pack: &'a CommandBufferRef, +} + +impl<'a> DirectColorBatchCommandBuffers<'a> { + pub(super) fn single(command_buffer: &'a CommandBufferRef) -> Self { + Self { + default: command_buffer, + idwt: DirectIdwtCommandBuffers::single(command_buffer), + store: command_buffer, + mct_pack: command_buffer, + } + } +} + +pub(super) struct DecodeHybridSplitCommandBuffers { + pub(super) idwt_interleave: CommandBuffer, + pub(super) idwt_horizontal: CommandBuffer, + pub(super) idwt_vertical: CommandBuffer, + pub(super) store: CommandBuffer, + pub(super) mct_pack: CommandBuffer, +} + +impl DecodeHybridSplitCommandBuffers { + pub(super) fn new(runtime: &MetalRuntime) -> Self { + let idwt_interleave = runtime.queue.new_command_buffer().to_owned(); + label_command_buffer(&idwt_interleave, "j2k decode hybrid IDWT interleave stage"); + let idwt_horizontal = runtime.queue.new_command_buffer().to_owned(); + label_command_buffer(&idwt_horizontal, "j2k decode hybrid IDWT horizontal stage"); + let idwt_vertical = runtime.queue.new_command_buffer().to_owned(); + label_command_buffer(&idwt_vertical, "j2k decode hybrid IDWT vertical stage"); + let store = runtime.queue.new_command_buffer().to_owned(); + label_command_buffer(&store, "j2k decode hybrid store stage"); + let mct_pack = runtime.queue.new_command_buffer().to_owned(); + label_command_buffer(&mct_pack, "j2k decode hybrid MCT pack stage"); + Self { + idwt_interleave, + idwt_horizontal, + idwt_vertical, + store, + mct_pack, + } + } + + pub(super) fn refs(&self) -> DirectColorBatchCommandBuffers<'_> { + DirectColorBatchCommandBuffers { + default: &self.idwt_interleave, + idwt: DirectIdwtCommandBuffers { + interleave: &self.idwt_interleave, + horizontal: &self.idwt_horizontal, + vertical: &self.idwt_vertical, + }, + store: &self.store, + mct_pack: &self.mct_pack, + } + } + + pub(super) fn commit_in_order(&self) { + self.idwt_interleave.commit(); + self.idwt_horizontal.commit(); + self.idwt_vertical.commit(); + self.store.commit(); + self.mct_pack.commit(); + } +} diff --git a/crates/j2k-metal/src/compute/direct_cpu.rs b/crates/j2k-metal/src/compute/direct_cpu.rs new file mode 100644 index 00000000..7c5eaa40 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_cpu.rs @@ -0,0 +1,589 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::time::Instant; + +use j2k_native::{ + decode_ht_code_block_scalar_with_workspace, + decode_ht_code_block_scalar_with_workspace_profiled, + decode_j2k_code_block_scalar_with_workspace, + decode_j2k_code_block_scalar_with_workspace_profiled, HtCodeBlockDecodeJob, + HtCodeBlockDecodeProfile, HtCodeBlockDecodeWorkspace, J2kCodeBlockDecodeJob, + J2kCodeBlockDecodeProfile, J2kCodeBlockDecodeWorkspace, J2kCodeBlockSegment, J2kCodeBlockStyle, + J2kSubBandType, +}; +use rayon::prelude::*; + +use crate::Error; + +use super::{ + checked_coefficient_len, hybrid_stage_signpost, packed_cpu_decode_coefficients, + packed_cpu_decode_output_len, record_hybrid_cpu_decode_inputs, + record_hybrid_cpu_decode_worker_init, required_classic_output_len, required_ht_output_len, + CpuTier1DecodeSubstageCounters, J2kClassicCleanupBatchJob, J2kClassicSegment, + J2kHtCleanupBatchJob, PreparedClassicSubBand, PreparedClassicSubBandGroup, + PreparedDirectGrayscalePlan, PreparedHtSubBand, PreparedHtSubBandGroup, + HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK, J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES, + J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, + J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS, J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT, + SIGNPOST_DECODE_HYBRID_CPU_TIER1, +}; + +#[cfg(test)] +pub(super) fn decode_prepared_classic_sub_band_on_cpu( + sub_band: &PreparedClassicSubBand, +) -> Result, Error> { + decode_prepared_classic_sub_band_on_cpu_profile(sub_band, None) +} + +pub(super) fn decode_prepared_classic_sub_band_on_cpu_profile( + sub_band: &PreparedClassicSubBand, + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result, Error> { + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_CPU_TIER1); + let len = checked_coefficient_len( + sub_band.width, + sub_band.height, + "classic J2K MetalDirect hybrid sub-band size overflow", + )?; + let mut output = vec![0.0_f32; len]; + if let Some(counters) = profile_counters { + let mut scratch = ClassicCpuDecodeScratch::default(); + decode_prepared_classic_jobs_on_cpu_with_scratch_profiled( + &sub_band.coded_data, + &sub_band.segments, + &sub_band.jobs, + &mut output, + &mut scratch, + counters, + )?; + } else { + decode_prepared_classic_jobs_on_cpu( + &sub_band.coded_data, + &sub_band.segments, + &sub_band.jobs, + &mut output, + )?; + } + Ok(output) +} + +pub(super) fn decode_prepared_classic_sub_band_group_on_cpu_profile( + group: &PreparedClassicSubBandGroup, + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result, Error> { + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_CPU_TIER1); + let mut output = vec![0.0_f32; group.total_coefficients]; + if let Some(counters) = profile_counters { + let mut scratch = ClassicCpuDecodeScratch::default(); + decode_prepared_classic_jobs_on_cpu_with_scratch_profiled( + &group.coded_data, + &group.segments, + &group.jobs, + &mut output, + &mut scratch, + counters, + )?; + } else { + decode_prepared_classic_jobs_on_cpu( + &group.coded_data, + &group.segments, + &group.jobs, + &mut output, + )?; + } + Ok(output) +} + +#[derive(Default)] +pub(super) struct ClassicCpuDecodeScratch { + segments: Vec, + decode: J2kCodeBlockDecodeWorkspace, +} + +fn decode_prepared_classic_jobs_on_cpu( + coded_data: &[u8], + segments: &[J2kClassicSegment], + jobs: &[J2kClassicCleanupBatchJob], + output: &mut [f32], +) -> Result<(), Error> { + let mut scratch = ClassicCpuDecodeScratch::default(); + decode_prepared_classic_jobs_on_cpu_with_scratch( + coded_data, + segments, + jobs, + output, + &mut scratch, + ) +} + +pub(super) fn decode_prepared_classic_jobs_on_cpu_with_scratch( + coded_data: &[u8], + segments: &[J2kClassicSegment], + jobs: &[J2kClassicCleanupBatchJob], + output: &mut [f32], + scratch: &mut ClassicCpuDecodeScratch, +) -> Result<(), Error> { + decode_prepared_classic_jobs_on_cpu_with_scratch_impl::( + coded_data, segments, jobs, output, scratch, None, + ) +} + +pub(super) fn decode_prepared_classic_jobs_on_cpu_with_scratch_profiled( + coded_data: &[u8], + segments: &[J2kClassicSegment], + jobs: &[J2kClassicCleanupBatchJob], + output: &mut [f32], + scratch: &mut ClassicCpuDecodeScratch, + profile_counters: &CpuTier1DecodeSubstageCounters, +) -> Result<(), Error> { + decode_prepared_classic_jobs_on_cpu_with_scratch_impl::( + coded_data, + segments, + jobs, + output, + scratch, + Some(profile_counters), + ) +} + +fn decode_prepared_classic_jobs_on_cpu_with_scratch_impl( + coded_data: &[u8], + segments: &[J2kClassicSegment], + jobs: &[J2kClassicCleanupBatchJob], + output: &mut [f32], + scratch: &mut ClassicCpuDecodeScratch, + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result<(), Error> { + for job in jobs { + let prep_started = PROFILE.then(Instant::now); + let start = job.output_offset as usize; + let segment_window = prepared_classic_segment_window(segments, job)?; + scratch.segments.clear(); + scratch.segments.reserve(segment_window.len()); + for segment in segment_window { + scratch.segments.push(prepared_classic_segment(segment)?); + } + let decode_job = prepared_classic_decode_job(coded_data, &scratch.segments, job)?; + let required_len = required_classic_output_len(decode_job)?; + let end = start + .checked_add(required_len) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect hybrid output offset overflow".to_string(), + })?; + let Some(output_window) = output.get_mut(start..end) else { + return Err(Error::MetalKernel { + message: "classic J2K MetalDirect hybrid output slice is too small".to_string(), + }); + }; + if let Some(started) = prep_started { + profile_counters + .expect("profile counters required for profiled classic decode") + .record_classic_segment_prep(started); + } + if PROFILE { + let decode_started = Instant::now(); + let mut profile = J2kCodeBlockDecodeProfile::default(); + decode_j2k_code_block_scalar_with_workspace_profiled( + decode_job, + output_window, + &mut scratch.decode, + &mut profile, + ) + .map_err(native_decode_error)?; + profile_counters + .expect("profile counters required for profiled classic decode") + .record_classic_block_decode(decode_started, &profile); + } else { + decode_j2k_code_block_scalar_with_workspace( + decode_job, + output_window, + &mut scratch.decode, + ) + .map_err(native_decode_error)?; + } + } + Ok(()) +} + +fn prepared_classic_segment_window<'a>( + segments: &'a [J2kClassicSegment], + job: &J2kClassicCleanupBatchJob, +) -> Result<&'a [J2kClassicSegment], Error> { + let segment_start = job.segment_offset as usize; + let segment_end = segment_start + .checked_add(job.segment_count as usize) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect hybrid segment span overflow".to_string(), + })?; + segments + .get(segment_start..segment_end) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K MetalDirect hybrid segment span is invalid".to_string(), + }) +} + +fn prepared_classic_decode_job<'a>( + coded_data: &'a [u8], + segments: &'a [J2kCodeBlockSegment], + job: &J2kClassicCleanupBatchJob, +) -> Result, Error> { + Ok(J2kCodeBlockDecodeJob { + data: coded_data, + segments, + width: job.width, + height: job.height, + output_stride: job.output_stride as usize, + missing_bit_planes: checked_u8(job.missing_msbs, "classic missing bit planes")?, + number_of_coding_passes: checked_u8(job.number_of_coding_passes, "classic coding passes")?, + total_bitplanes: checked_u8(job.total_bitplanes, "classic total bitplanes")?, + roi_shift: checked_u8(job.roi_shift, "classic ROI shift")?, + sub_band_type: prepared_classic_sub_band_type(job.sub_band_type)?, + style: prepared_classic_style(job.style_flags), + strict: job.strict != 0, + dequantization_step: job.dequantization_step, + }) +} + +fn prepared_classic_segment(segment: &J2kClassicSegment) -> Result { + Ok(J2kCodeBlockSegment { + data_offset: segment.data_offset, + data_length: segment.data_length, + start_coding_pass: checked_u8(segment.start_coding_pass, "classic segment start pass")?, + end_coding_pass: checked_u8(segment.end_coding_pass, "classic segment end pass")?, + use_arithmetic: segment.use_arithmetic != 0, + }) +} + +fn prepared_classic_sub_band_type(value: u32) -> Result { + match value { + 0 => Ok(J2kSubBandType::LowLow), + 1 => Ok(J2kSubBandType::HighLow), + 2 => Ok(J2kSubBandType::LowHigh), + 3 => Ok(J2kSubBandType::HighHigh), + _ => Err(Error::MetalKernel { + message: format!("classic J2K MetalDirect hybrid sub-band type {value} is invalid"), + }), + } +} + +fn prepared_classic_style(flags: u32) -> J2kCodeBlockStyle { + J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: (flags + & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) + != 0, + reset_context_probabilities: (flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0, + termination_on_each_pass: (flags & J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS) != 0, + vertically_causal_context: (flags & J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT) != 0, + segmentation_symbols: (flags & J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS) != 0, + } +} + +pub(super) fn decode_prepared_ht_sub_band_on_cpu_profile( + sub_band: &PreparedHtSubBand, + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result, Error> { + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_CPU_TIER1); + let len = checked_coefficient_len( + sub_band.width, + sub_band.height, + "HTJ2K MetalDirect hybrid sub-band size overflow", + )?; + let mut output = vec![0.0_f32; len]; + if let Some(counters) = profile_counters { + let mut workspace = HtCodeBlockDecodeWorkspace::default(); + decode_prepared_ht_jobs_on_cpu_with_workspace_profiled( + &sub_band.coded_data, + &sub_band.jobs, + &mut output, + &mut workspace, + counters, + )?; + } else { + decode_prepared_ht_jobs_on_cpu(&sub_band.coded_data, &sub_band.jobs, &mut output)?; + } + Ok(output) +} + +pub(super) fn decode_prepared_ht_sub_band_group_on_cpu_profile( + group: &PreparedHtSubBandGroup, + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result, Error> { + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_CPU_TIER1); + let mut output = vec![0.0_f32; group.total_coefficients]; + if let Some(counters) = profile_counters { + let mut workspace = HtCodeBlockDecodeWorkspace::default(); + decode_prepared_ht_jobs_on_cpu_with_workspace_profiled( + &group.coded_arena.data, + &group.jobs, + &mut output, + &mut workspace, + counters, + )?; + } else { + decode_prepared_ht_jobs_on_cpu(&group.coded_arena.data, &group.jobs, &mut output)?; + } + Ok(output) +} + +fn decode_prepared_ht_jobs_on_cpu( + coded_data: &[u8], + jobs: &[J2kHtCleanupBatchJob], + output: &mut [f32], +) -> Result<(), Error> { + let mut workspace = HtCodeBlockDecodeWorkspace::default(); + decode_prepared_ht_jobs_on_cpu_with_workspace(coded_data, jobs, output, &mut workspace) +} + +pub(super) fn decode_prepared_ht_jobs_on_cpu_with_workspace( + coded_data: &[u8], + jobs: &[J2kHtCleanupBatchJob], + output: &mut [f32], + workspace: &mut HtCodeBlockDecodeWorkspace, +) -> Result<(), Error> { + decode_prepared_ht_jobs_on_cpu_with_workspace_impl::( + coded_data, jobs, output, workspace, None, + ) +} + +pub(super) fn decode_prepared_ht_jobs_on_cpu_with_workspace_profiled( + coded_data: &[u8], + jobs: &[J2kHtCleanupBatchJob], + output: &mut [f32], + workspace: &mut HtCodeBlockDecodeWorkspace, + profile_counters: &CpuTier1DecodeSubstageCounters, +) -> Result<(), Error> { + decode_prepared_ht_jobs_on_cpu_with_workspace_impl::( + coded_data, + jobs, + output, + workspace, + Some(profile_counters), + ) +} + +fn decode_prepared_ht_jobs_on_cpu_with_workspace_impl( + coded_data: &[u8], + jobs: &[J2kHtCleanupBatchJob], + output: &mut [f32], + workspace: &mut HtCodeBlockDecodeWorkspace, + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result<(), Error> { + for job in jobs { + let start = job.output_offset as usize; + let decode_job = prepared_ht_decode_job(coded_data, job)?; + let required_len = required_ht_output_len(decode_job)?; + let end = start + .checked_add(required_len) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect hybrid output offset overflow".to_string(), + })?; + let Some(output_window) = output.get_mut(start..end) else { + return Err(Error::MetalKernel { + message: "HTJ2K MetalDirect hybrid output slice is too small".to_string(), + }); + }; + if PROFILE { + let decode_started = Instant::now(); + let mut profile = HtCodeBlockDecodeProfile::default(); + decode_ht_code_block_scalar_with_workspace_profiled( + decode_job, + output_window, + workspace, + &mut profile, + ) + .map_err(native_decode_error)?; + profile_counters + .expect("profile counters required for profiled HT decode") + .record_ht_block_decode(decode_started, &profile); + } else { + decode_ht_code_block_scalar_with_workspace(decode_job, output_window, workspace) + .map_err(native_decode_error)?; + } + } + Ok(()) +} + +fn prepared_ht_decode_job<'a>( + coded_data: &'a [u8], + job: &J2kHtCleanupBatchJob, +) -> Result, Error> { + let start = job.coded_offset as usize; + let len = job.coded_len as usize; + let end = start.checked_add(len).ok_or_else(|| Error::MetalKernel { + message: "HTJ2K MetalDirect hybrid coded span overflow".to_string(), + })?; + let Some(data) = coded_data.get(start..end) else { + return Err(Error::MetalKernel { + message: "HTJ2K MetalDirect hybrid coded span is invalid".to_string(), + }); + }; + + Ok(HtCodeBlockDecodeJob { + data, + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + width: job.width, + height: job.height, + output_stride: job.output_stride as usize, + missing_bit_planes: checked_u8(job.missing_msbs, "HTJ2K missing bit planes")?, + number_of_coding_passes: checked_u8(job.number_of_coding_passes, "HTJ2K coding passes")?, + num_bitplanes: checked_u8(job.num_bitplanes, "HTJ2K total bitplanes")?, + roi_shift: checked_u8(job.roi_shift, "HTJ2K ROI shift")?, + stripe_causal: job.stripe_causal != 0, + strict: true, + dequantization_step: job.dequantization_step, + }) +} + +fn checked_u8(value: u32, label: &str) -> Result { + u8::try_from(value).map_err(|_| Error::MetalKernel { + message: format!("J2K MetalDirect hybrid {label} exceeds u8"), + }) +} + +fn native_decode_error(error: j2k_native::DecodeError) -> Error { + Error::Decode(j2k::J2kError::Backend(error.to_string())) +} + +pub(super) struct ClassicCpuDecodeInput<'a> { + pub(super) coded_data: &'a [u8], + pub(super) segments: &'a [J2kClassicSegment], + pub(super) jobs: &'a [J2kClassicCleanupBatchJob], + pub(super) output_len: usize, +} + +pub(super) struct HtCpuDecodeInput<'a> { + pub(super) coded_data: &'a [u8], + pub(super) jobs: &'a [J2kHtCleanupBatchJob], + pub(super) output_len: usize, +} + +fn decode_classic_inputs_on_cpu_parallel( + inputs: &[ClassicCpuDecodeInput<'_>], + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result, Error> { + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_CPU_TIER1); + record_hybrid_cpu_decode_inputs(inputs.len()); + let Some(output_len) = packed_cpu_decode_output_len( + inputs.iter().map(|input| input.output_len), + "classic J2K MetalDirect hybrid batch", + )? + else { + return Ok(Vec::new()); + }; + let mut coefficients = packed_cpu_decode_coefficients(inputs.len(), output_len)?; + coefficients + .par_chunks_mut(output_len) + .zip(inputs.par_iter()) + .with_min_len(HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK) + .try_for_each_init( + || { + record_hybrid_cpu_decode_worker_init(); + ClassicCpuDecodeScratch::default() + }, + |scratch, (output, input)| { + if let Some(counters) = profile_counters { + decode_prepared_classic_jobs_on_cpu_with_scratch_profiled( + input.coded_data, + input.segments, + input.jobs, + output, + scratch, + counters, + ) + } else { + decode_prepared_classic_jobs_on_cpu_with_scratch( + input.coded_data, + input.segments, + input.jobs, + output, + scratch, + ) + } + }, + )?; + Ok(coefficients) +} + +fn decode_ht_inputs_on_cpu_parallel( + inputs: &[HtCpuDecodeInput<'_>], + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result, Error> { + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_CPU_TIER1); + record_hybrid_cpu_decode_inputs(inputs.len()); + let Some(output_len) = packed_cpu_decode_output_len( + inputs.iter().map(|input| input.output_len), + "HTJ2K MetalDirect hybrid batch", + )? + else { + return Ok(Vec::new()); + }; + let mut coefficients = packed_cpu_decode_coefficients(inputs.len(), output_len)?; + coefficients + .par_chunks_mut(output_len) + .zip(inputs.par_iter()) + .with_min_len(HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK) + .try_for_each_init( + || { + record_hybrid_cpu_decode_worker_init(); + HtCodeBlockDecodeWorkspace::default() + }, + |workspace, (output, input)| { + if let Some(counters) = profile_counters { + decode_prepared_ht_jobs_on_cpu_with_workspace_profiled( + input.coded_data, + input.jobs, + output, + workspace, + counters, + ) + } else { + decode_prepared_ht_jobs_on_cpu_with_workspace( + input.coded_data, + input.jobs, + output, + workspace, + ) + } + }, + )?; + Ok(coefficients) +} + +pub(super) fn decode_classic_inputs_on_cpu_with_plan_cache( + plan: &PreparedDirectGrayscalePlan, + step_idx: usize, + inputs: &[ClassicCpuDecodeInput<'_>], + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result, Error> { + if inputs.len() != 1 { + return decode_classic_inputs_on_cpu_parallel(inputs, profile_counters); + } + + let output_len = inputs[0].output_len; + if let Some(coefficients) = plan.cached_cpu_tier1_coefficients(step_idx, output_len)? { + return Ok(coefficients); + } + + let coefficients = decode_classic_inputs_on_cpu_parallel(inputs, profile_counters)?; + plan.store_cpu_tier1_coefficients(step_idx, output_len, coefficients) +} + +pub(super) fn decode_ht_inputs_on_cpu_with_plan_cache( + plan: &PreparedDirectGrayscalePlan, + step_idx: usize, + inputs: &[HtCpuDecodeInput<'_>], + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result, Error> { + if inputs.len() != 1 { + return decode_ht_inputs_on_cpu_parallel(inputs, profile_counters); + } + + let output_len = inputs[0].output_len; + if let Some(coefficients) = plan.cached_cpu_tier1_coefficients(step_idx, output_len)? { + return Ok(coefficients); + } + + let coefficients = decode_ht_inputs_on_cpu_parallel(inputs, profile_counters)?; + plan.store_cpu_tier1_coefficients(step_idx, output_len, coefficients) +} diff --git a/crates/j2k-metal/src/compute/direct_flattened.rs b/crates/j2k-metal/src/compute/direct_flattened.rs new file mode 100644 index 00000000..6cb4720c --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_flattened.rs @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::{collections::HashMap, sync::Arc, time::Instant}; + +use j2k_native::HtCodeBlockDecodeWorkspace; +use metal::Buffer; + +use crate::Error; + +use super::{ + checked_coefficient_len, decode_prepared_classic_jobs_on_cpu_with_scratch, + decode_prepared_classic_jobs_on_cpu_with_scratch_profiled, + decode_prepared_ht_jobs_on_cpu_with_workspace, + decode_prepared_ht_jobs_on_cpu_with_workspace_profiled, elapsed_us, hybrid_stage_signpost, + metal_profile_stages_enabled, record_flattened_hybrid_cpu_decode_batch, + record_hybrid_cpu_decode_inputs, record_hybrid_cpu_decode_worker_init, + upload_cpu_decoded_coefficients, ClassicCpuDecodeScratch, CpuTier1DecodeSubstageCounters, + DirectHybridStageTimings, J2kClassicCleanupBatchJob, J2kClassicSegment, J2kHtCleanupBatchJob, + MetalRuntime, PreparedDirectColorPlan, PreparedDirectGrayscaleStep, + HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK, SIGNPOST_DECODE_HYBRID_CPU_TIER1, +}; + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +struct FlattenedCpuTier1Key { + component_idx: usize, + step_idx: usize, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +enum FlattenedCpuTier1Source<'a> { + Classic { + coded_data: &'a [u8], + segments: &'a [J2kClassicSegment], + jobs: &'a [J2kClassicCleanupBatchJob], + }, + Ht { + coded_data: &'a [u8], + jobs: &'a [J2kHtCleanupBatchJob], + }, +} + +#[cfg(target_os = "macos")] +struct FlattenedCpuTier1BucketSpec<'a> { + key: FlattenedCpuTier1Key, + output_len: usize, + inputs: Vec>, +} + +#[cfg(target_os = "macos")] +#[derive(Clone)] +struct FlattenedCpuTier1Bucket { + buffer: Buffer, + output_len: usize, + input_count: usize, +} + +#[cfg(target_os = "macos")] +pub(super) struct FlattenedCpuTier1Cache { + buckets: HashMap, +} + +#[cfg(target_os = "macos")] +impl FlattenedCpuTier1Cache { + pub(super) fn buffer_for( + &self, + component_idx: usize, + step_idx: usize, + output_len: usize, + input_count: usize, + ) -> Result { + let key = FlattenedCpuTier1Key { + component_idx, + step_idx, + }; + let bucket = self.buckets.get(&key).ok_or_else(|| Error::MetalKernel { + message: format!( + "J2K MetalDirect flattened hybrid cache is missing component {component_idx} step {step_idx}" + ), + })?; + if bucket.output_len != output_len || bucket.input_count != input_count { + return Err(Error::MetalKernel { + message: format!( + "J2K MetalDirect flattened hybrid cache shape mismatch at component {component_idx} step {step_idx}" + ), + }); + } + Ok(bucket.buffer.clone()) + } +} + +#[cfg(target_os = "macos")] +struct FlattenedCpuTier1WorkItem<'a> { + output_len: usize, + output: FlattenedCpuTier1Output, + source: FlattenedCpuTier1Source<'a>, +} + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +struct FlattenedCpuTier1Output(*mut f32); + +// SAFETY: Work items are constructed from non-overlapping ranges in preallocated +// packed coefficient slabs. Each pointer is written exactly once before the +// owning Vec is moved or exposed again. +#[cfg(target_os = "macos")] +// SAFETY: Metal buffer access follows validated sizes and synchronized command completion. +unsafe impl Send for FlattenedCpuTier1Output {} + +#[cfg(target_os = "macos")] +// SAFETY: Metal buffer access follows validated sizes and synchronized command completion. +unsafe impl Sync for FlattenedCpuTier1Output {} + +#[cfg(target_os = "macos")] +impl FlattenedCpuTier1Output { + unsafe fn as_slice_mut<'a>(self, len: usize) -> &'a mut [f32] { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + unsafe { std::slice::from_raw_parts_mut(self.0, len) } + } +} + +#[cfg(target_os = "macos")] +#[derive(Default)] +struct FlattenedCpuTier1DecodeScratch { + classic: ClassicCpuDecodeScratch, + ht: HtCodeBlockDecodeWorkspace, +} + +#[cfg(target_os = "macos")] +pub(super) fn build_flattened_cpu_tier1_cache( + runtime: &MetalRuntime, + plans: &[Arc], + stage_timings: &mut DirectHybridStageTimings, + retained_buffers: &mut Vec, + retained_cpu_coefficients: &mut Vec>, +) -> Result { + let specs = collect_flattened_cpu_tier1_bucket_specs(plans)?; + stage_timings.cpu_tier1_flattened_batches = + stage_timings.cpu_tier1_flattened_batches.saturating_add(1); + let decode_started = metal_profile_stages_enabled().then(Instant::now); + let cpu_tier1_counters = + metal_profile_stages_enabled().then(CpuTier1DecodeSubstageCounters::default); + let decoded_buckets = decode_flattened_cpu_tier1_buckets(&specs, cpu_tier1_counters.as_ref())?; + if let Some(started) = decode_started { + stage_timings.cpu_tier1 += elapsed_us(started); + } + if let Some(counters) = &cpu_tier1_counters { + counters.add_to_stage_timings(stage_timings); + } + + let upload_started = metal_profile_stages_enabled().then(Instant::now); + let mut buckets = HashMap::with_capacity(specs.len()); + for (spec, coefficients) in specs.iter().zip(decoded_buckets) { + let input_count = spec.inputs.len(); + let buffer = upload_cpu_decoded_coefficients( + runtime, + coefficients, + retained_buffers, + retained_cpu_coefficients, + ); + buckets.insert( + spec.key, + FlattenedCpuTier1Bucket { + buffer, + output_len: spec.output_len, + input_count, + }, + ); + } + if let Some(started) = upload_started { + stage_timings.coefficient_upload += elapsed_us(started); + } + + Ok(FlattenedCpuTier1Cache { buckets }) +} + +#[cfg(target_os = "macos")] +fn collect_flattened_cpu_tier1_bucket_specs( + plans: &[Arc], +) -> Result>, Error> { + let Some(first) = plans.first() else { + return Ok(Vec::new()); + }; + let mut specs = Vec::new(); + + for component_idx in 0..3 { + let component_plans = plans + .iter() + .map(|plan| &plan.component_plans[component_idx]) + .collect::>(); + let Some(first_component) = component_plans.first().copied() else { + continue; + }; + let broadcast_tier1_inputs = component_plans + .iter() + .all(|plan| std::ptr::eq(*plan, first_component)); + let mut step_idx = 0; + while step_idx < first.component_plans[component_idx].steps.len() { + if let Some(group) = first_component.classic_group_starting_at(step_idx) { + let inputs = component_plans + .iter() + .take(if broadcast_tier1_inputs { + 1 + } else { + component_plans.len() + }) + .map(|plan| { + let group = plan.classic_group_starting_at(step_idx).ok_or_else(|| { + Error::MetalKernel { + message: "J2K MetalDirect flattened hybrid missing classic group" + .to_string(), + } + })?; + Ok(FlattenedCpuTier1Source::Classic { + coded_data: &group.coded_data, + segments: &group.segments, + jobs: &group.jobs, + }) + }) + .collect::, Error>>()?; + specs.push(FlattenedCpuTier1BucketSpec { + key: FlattenedCpuTier1Key { + component_idx, + step_idx, + }, + output_len: group.total_coefficients, + inputs, + }); + step_idx = group.end_step; + continue; + } + + if let Some(group) = first_component.ht_group_starting_at(step_idx) { + let inputs = component_plans + .iter() + .take(if broadcast_tier1_inputs { + 1 + } else { + component_plans.len() + }) + .map(|plan| { + let group = plan.ht_group_starting_at(step_idx).ok_or_else(|| { + Error::MetalKernel { + message: "J2K MetalDirect flattened hybrid missing HT group" + .to_string(), + } + })?; + Ok(FlattenedCpuTier1Source::Ht { + coded_data: &group.coded_arena.data, + jobs: &group.jobs, + }) + }) + .collect::, Error>>()?; + specs.push(FlattenedCpuTier1BucketSpec { + key: FlattenedCpuTier1Key { + component_idx, + step_idx, + }, + output_len: group.total_coefficients, + inputs, + }); + step_idx = group.end_step; + continue; + } + + match &first_component.steps[step_idx] { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => { + let output_len = checked_coefficient_len( + sub_band.width, + sub_band.height, + "classic J2K MetalDirect flattened hybrid sub-band size overflow", + )?; + let inputs = component_plans + .iter() + .take(if broadcast_tier1_inputs { + 1 + } else { + component_plans.len() + }) + .map(|plan| match &plan.steps[step_idx] { + PreparedDirectGrayscaleStep::ClassicSubBand(other) => { + Ok(FlattenedCpuTier1Source::Classic { + coded_data: &other.coded_data, + segments: &other.segments, + jobs: &other.jobs, + }) + } + _ => Err(Error::MetalKernel { + message: + "J2K MetalDirect flattened hybrid missing classic sub-band" + .to_string(), + }), + }) + .collect::, Error>>()?; + specs.push(FlattenedCpuTier1BucketSpec { + key: FlattenedCpuTier1Key { + component_idx, + step_idx, + }, + output_len, + inputs, + }); + } + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + let output_len = checked_coefficient_len( + sub_band.width, + sub_band.height, + "HTJ2K MetalDirect flattened hybrid sub-band size overflow", + )?; + let inputs = component_plans + .iter() + .take(if broadcast_tier1_inputs { + 1 + } else { + component_plans.len() + }) + .map(|plan| match &plan.steps[step_idx] { + PreparedDirectGrayscaleStep::HtSubBand(other) => { + Ok(FlattenedCpuTier1Source::Ht { + coded_data: &other.coded_data, + jobs: &other.jobs, + }) + } + _ => Err(Error::MetalKernel { + message: "J2K MetalDirect flattened hybrid missing HT sub-band" + .to_string(), + }), + }) + .collect::, Error>>()?; + specs.push(FlattenedCpuTier1BucketSpec { + key: FlattenedCpuTier1Key { + component_idx, + step_idx, + }, + output_len, + inputs, + }); + } + PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => {} + } + step_idx += 1; + } + } + + Ok(specs) +} + +#[cfg(target_os = "macos")] +fn decode_flattened_cpu_tier1_buckets( + specs: &[FlattenedCpuTier1BucketSpec<'_>], + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result>, Error> { + let _signpost = hybrid_stage_signpost(SIGNPOST_DECODE_HYBRID_CPU_TIER1); + let mut buckets = specs + .iter() + .map(|spec| packed_cpu_decode_coefficients(spec.inputs.len(), spec.output_len)) + .collect::, Error>>()?; + let mut work_items = Vec::new(); + for (bucket_idx, spec) in specs.iter().enumerate() { + for (input_idx, source) in spec.inputs.iter().copied().enumerate() { + let start = + input_idx + .checked_mul(spec.output_len) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect flattened hybrid bucket offset overflow" + .to_string(), + })?; + let end = start + .checked_add(spec.output_len) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect flattened hybrid bucket end overflow".to_string(), + })?; + if end > buckets[bucket_idx].len() { + return Err(Error::MetalKernel { + message: "J2K MetalDirect flattened hybrid bucket slice is too small" + .to_string(), + }); + } + let output = + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + FlattenedCpuTier1Output(unsafe { buckets[bucket_idx].as_mut_ptr().add(start) }); + work_items.push(FlattenedCpuTier1WorkItem { + output_len: spec.output_len, + output, + source, + }); + } + } + + record_flattened_hybrid_cpu_decode_batch(); + record_hybrid_cpu_decode_inputs(work_items.len()); + + decode_flattened_cpu_tier1_work_items_chunked(&work_items, profile_counters)?; + + Ok(buckets) +} + +#[cfg(target_os = "macos")] +fn decode_flattened_cpu_tier1_work_items_chunked( + work_items: &[FlattenedCpuTier1WorkItem<'_>], + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result<(), Error> { + if work_items.is_empty() { + return Ok(()); + } + + let worker_count = hybrid_cpu_decode_worker_count(work_items.len()); + let chunk_size = work_items.len().div_ceil(worker_count); + std::thread::scope(|scope| { + let mut handles = Vec::with_capacity(worker_count); + for chunk in work_items.chunks(chunk_size) { + handles.push(scope.spawn(move || { + record_hybrid_cpu_decode_worker_init(); + let mut scratch = FlattenedCpuTier1DecodeScratch::default(); + for item in chunk { + decode_flattened_cpu_tier1_work_item(item, &mut scratch, profile_counters)?; + } + Ok(()) + })); + } + + for handle in handles { + match handle.join() { + Ok(Ok(())) => {} + Ok(Err(error)) => return Err(error), + Err(payload) => std::panic::resume_unwind(payload), + } + } + Ok(()) + }) +} + +#[cfg(target_os = "macos")] +pub(super) fn hybrid_cpu_decode_worker_count(item_count: usize) -> usize { + let available = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get); + let useful = item_count + .div_ceil(HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK.max(1)) + .max(1); + available.min(useful).max(1) +} + +#[cfg(target_os = "macos")] +fn decode_flattened_cpu_tier1_work_item( + item: &FlattenedCpuTier1WorkItem<'_>, + scratch: &mut FlattenedCpuTier1DecodeScratch, + profile_counters: Option<&CpuTier1DecodeSubstageCounters>, +) -> Result<(), Error> { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let output = unsafe { item.output.as_slice_mut(item.output_len) }; + match item.source { + FlattenedCpuTier1Source::Classic { + coded_data, + segments, + jobs, + } => { + if let Some(counters) = profile_counters { + decode_prepared_classic_jobs_on_cpu_with_scratch_profiled( + coded_data, + segments, + jobs, + output, + &mut scratch.classic, + counters, + ) + } else { + decode_prepared_classic_jobs_on_cpu_with_scratch( + coded_data, + segments, + jobs, + output, + &mut scratch.classic, + ) + } + } + FlattenedCpuTier1Source::Ht { coded_data, jobs } => { + if let Some(counters) = profile_counters { + decode_prepared_ht_jobs_on_cpu_with_workspace_profiled( + coded_data, + jobs, + output, + &mut scratch.ht, + counters, + ) + } else { + decode_prepared_ht_jobs_on_cpu_with_workspace( + coded_data, + jobs, + output, + &mut scratch.ht, + ) + } + } + } +} + +#[cfg(target_os = "macos")] +pub(super) fn packed_cpu_decode_output_len( + output_lens: impl IntoIterator, + label: &str, +) -> Result, Error> { + let mut output_lens = output_lens.into_iter(); + let Some(output_len) = output_lens.next() else { + return Ok(None); + }; + if output_len == 0 { + return Ok(None); + } + if output_lens.any(|other| other != output_len) { + return Err(Error::MetalKernel { + message: format!("{label} has mixed coefficient lengths"), + }); + } + Ok(Some(output_len)) +} + +#[cfg(target_os = "macos")] +pub(super) fn packed_cpu_decode_coefficients( + count: usize, + output_len: usize, +) -> Result, Error> { + let total_len = count + .checked_mul(output_len) + .ok_or_else(|| Error::MetalKernel { + message: "J2K MetalDirect hybrid packed coefficient length overflows usize".to_string(), + })?; + Ok(vec![0.0_f32; total_len]) +} diff --git a/crates/j2k-metal/src/compute/direct_plan_support.rs b/crates/j2k-metal/src/compute/direct_plan_support.rs new file mode 100644 index 00000000..dba1f4da --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_plan_support.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_core::PixelFormat; +use j2k_native::J2kRect; + +use super::{ + DirectTier1Mode, J2kClassicCleanupBatchJob, J2kClassicSegment, J2kDirectStoreStep, + J2kHtCleanupBatchJob, PreparedClassicSubBand, PreparedClassicSubBandGroup, + PreparedDirectColorPlan, PreparedDirectGrayscalePlan, PreparedDirectGrayscaleStep, + PreparedDirectIdwt, PreparedHtSubBand, PreparedHtSubBandGroup, J2K_CLASSIC_MAX_HEIGHT, + J2K_CLASSIC_MAX_WIDTH, J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, +}; +use crate::Error; + +#[cfg(test)] +pub(super) fn prepared_direct_color_tier1_input_count(plan: &PreparedDirectColorPlan) -> usize { + plan.component_plans + .iter() + .map(prepared_direct_component_tier1_input_count) + .sum() +} + +#[cfg(test)] +fn prepared_direct_component_tier1_input_count(plan: &PreparedDirectGrayscalePlan) -> usize { + let mut count = 0; + let mut step_idx = 0; + while step_idx < plan.steps.len() { + if let Some(group) = plan.classic_group_starting_at(step_idx) { + count += 1; + step_idx = group.end_step; + continue; + } + if let Some(group) = plan.ht_group_starting_at(step_idx) { + count += 1; + step_idx = group.end_step; + continue; + } + if matches!( + &plan.steps[step_idx], + PreparedDirectGrayscaleStep::ClassicSubBand(_) + | PreparedDirectGrayscaleStep::HtSubBand(_) + ) { + count += 1; + } + step_idx += 1; + } + count +} + +pub(super) fn prepared_direct_color_plan_supports_runtime( + plan: &PreparedDirectColorPlan, + fmt: PixelFormat, +) -> bool { + matches!( + fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 + ) && plan.component_plans.len() == 3 + && plan + .component_plans + .iter() + .all(prepared_direct_component_plan_supports_runtime) +} + +fn prepared_direct_component_plan_supports_runtime(plan: &PreparedDirectGrayscalePlan) -> bool { + plan.tier1_prepare_mode == DirectTier1Mode::Metal + && plan.steps.iter().all(|step| match step { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => sub_band + .jobs + .iter() + .all(|job| classic_prepared_job_supports_runtime(job, &sub_band.segments)), + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => { + sub_band.jobs.iter().all(ht_prepared_job_supports_runtime) + } + PreparedDirectGrayscaleStep::Idwt(_) | PreparedDirectGrayscaleStep::Store(_) => true, + }) + && plan.classic_groups.iter().all(|group| { + group + .jobs + .iter() + .all(|job| classic_prepared_job_supports_runtime(job, &group.segments)) + }) + && plan + .ht_groups + .iter() + .all(|group| group.jobs.iter().all(ht_prepared_job_supports_runtime)) +} + +fn classic_prepared_job_supports_runtime( + job: &J2kClassicCleanupBatchJob, + segments: &[J2kClassicSegment], +) -> bool { + if job.width == 0 || job.height == 0 { + return true; + } + if job.width > J2K_CLASSIC_MAX_WIDTH || job.height > J2K_CLASSIC_MAX_HEIGHT { + return false; + } + if job.output_stride < job.width { + return false; + } + if job.roi_shift != 0 { + return false; + } + if job.total_bitplanes == 0 || job.total_bitplanes > 31 || job.missing_msbs >= 31 { + return false; + } + let bitplanes = job.total_bitplanes.saturating_sub(job.missing_msbs); + if bitplanes == 0 { + return false; + } + let max_coding_passes = 1 + 3 * (bitplanes - 1); + if job.number_of_coding_passes == 0 || job.number_of_coding_passes > max_coding_passes { + return false; + } + + let start = job.segment_offset as usize; + let count = job.segment_count as usize; + let Some(end) = start.checked_add(count) else { + return false; + }; + if end > segments.len() || count == 0 { + return false; + } + + let uses_bypass = (job.style_flags & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) != 0; + let mut expected_start = 0u32; + let mut expected_offset = job.coded_offset; + for segment in &segments[start..end] { + if segment.start_coding_pass != expected_start + || segment.start_coding_pass > segment.end_coding_pass + { + return false; + } + if uses_bypass { + let expected_arithmetic = + segment.start_coding_pass <= 9 || segment.start_coding_pass % 3 == 0; + if (segment.use_arithmetic != 0) != expected_arithmetic { + return false; + } + if segment.use_arithmetic == 0 { + if segment.start_coding_pass % 3 != 1 { + return false; + } + if segment + .end_coding_pass + .saturating_sub(segment.start_coding_pass) + > 2 + { + return false; + } + if (segment.start_coding_pass..segment.end_coding_pass).any(|pass| pass % 3 == 0) { + return false; + } + } + } else if segment.use_arithmetic == 0 { + return false; + } + + let Some(data_end) = segment.data_offset.checked_add(segment.data_length) else { + return false; + }; + if segment.data_offset != expected_offset + || segment.data_offset < job.coded_offset + || data_end > job.coded_offset.saturating_add(job.coded_len) + { + return false; + } + expected_offset = data_end; + expected_start = segment.end_coding_pass; + } + + expected_start == job.number_of_coding_passes + && expected_offset == job.coded_offset.saturating_add(job.coded_len) +} + +pub(super) fn classic_group_shapes_match( + first: &PreparedClassicSubBandGroup, + other: &PreparedClassicSubBandGroup, +) -> bool { + first.end_step == other.end_step + && first.total_coefficients == other.total_coefficients + && first.members.len() == other.members.len() + && first + .members + .iter() + .zip(&other.members) + .all(|(left, right)| left.offset_elements == right.offset_elements) +} + +pub(super) fn ht_group_shapes_match( + first: &PreparedHtSubBandGroup, + other: &PreparedHtSubBandGroup, +) -> bool { + first.end_step == other.end_step + && first.total_coefficients == other.total_coefficients + && first.members.len() == other.members.len() + && first + .members + .iter() + .zip(&other.members) + .all(|(left, right)| left.offset_elements == right.offset_elements) +} + +pub(super) fn classic_sub_band_shapes_match( + first: &PreparedClassicSubBand, + other: &PreparedClassicSubBand, +) -> bool { + first.width == other.width && first.height == other.height +} + +pub(super) fn ht_sub_band_shapes_match( + first: &PreparedHtSubBand, + other: &PreparedHtSubBand, +) -> bool { + first.width == other.width && first.height == other.height +} + +fn rect_shapes_match(first: J2kRect, other: J2kRect) -> bool { + first.x0 == other.x0 && first.y0 == other.y0 && first.x1 == other.x1 && first.y1 == other.y1 +} + +pub(super) fn idwt_shapes_match(first: &PreparedDirectIdwt, other: &PreparedDirectIdwt) -> bool { + first.step.transform == other.step.transform + && rect_shapes_match(first.step.rect, other.step.rect) + && first.output_window.x0 == other.output_window.x0 + && first.output_window.y0 == other.output_window.y0 + && first.output_window.x1 == other.output_window.x1 + && first.output_window.y1 == other.output_window.y1 + && rect_shapes_match(first.step.ll, other.step.ll) + && rect_shapes_match(first.step.hl, other.step.hl) + && rect_shapes_match(first.step.lh, other.step.lh) + && rect_shapes_match(first.step.hh, other.step.hh) +} + +pub(super) fn store_shapes_match(first: &J2kDirectStoreStep, other: &J2kDirectStoreStep) -> bool { + rect_shapes_match(first.input_rect, other.input_rect) + && first.source_x == other.source_x + && first.source_y == other.source_y + && first.copy_width == other.copy_width + && first.copy_height == other.copy_height + && first.output_width == other.output_width + && first.output_height == other.output_height + && first.output_x == other.output_x + && first.output_y == other.output_y + && first.addend.to_bits() == other.addend.to_bits() +} + +pub(super) fn direct_preflight_invariant(message: &'static str) -> Error { + Error::MetalKernel { + message: format!("internal J2K Metal direct preflight error: {message}"), + } +} + +fn ht_prepared_job_supports_runtime(job: &J2kHtCleanupBatchJob) -> bool { + if job.width == 0 || job.height == 0 { + return true; + } + job.roi_shift == 0 + && job.output_stride >= job.width + && crate::ht::supports_metal_ht_geometry(job.width, job.height) +} diff --git a/crates/j2k-metal/src/compute/direct_profile.rs b/crates/j2k-metal/src/compute/direct_profile.rs new file mode 100644 index 00000000..dd17a607 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_profile.rs @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + sync::atomic::{AtomicU64, Ordering as AtomicOrdering}, + time::{Duration, Instant}, +}; + +use j2k_core::PixelFormat; +use j2k_native::{HtCodeBlockDecodeProfile, J2kCodeBlockDecodeProfile}; + +use crate::profile_env::{decode_profile_label, metal_profile_stages_enabled}; + +#[cfg(target_os = "macos")] +use super::{completed_command_buffer_gpu_duration, DecodeHybridSplitCommandBuffers}; + +#[derive(Default)] +pub(super) struct DirectHybridStageTimings { + pub(super) cpu_tier1: u128, + pub(super) cpu_tier1_flattened_batches: u128, + pub(super) cpu_tier1_classic_segment_prep: u128, + pub(super) cpu_tier1_classic_block_decode: u128, + pub(super) cpu_tier1_classic_sigprop: u128, + pub(super) cpu_tier1_classic_magref: u128, + pub(super) cpu_tier1_classic_cleanup: u128, + pub(super) cpu_tier1_classic_bypass: u128, + pub(super) cpu_tier1_classic_output_convert: u128, + pub(super) cpu_tier1_ht_block_decode: u128, + pub(super) cpu_tier1_ht_cleanup: u128, + pub(super) cpu_tier1_ht_mag_sgn: u128, + pub(super) cpu_tier1_ht_sigma: u128, + pub(super) cpu_tier1_ht_sigprop: u128, + pub(super) cpu_tier1_ht_magref: u128, + pub(super) coefficient_upload: u128, + pub(super) metal_idwt_encode: u128, + pub(super) metal_store_encode: u128, + pub(super) metal_mct_pack_encode: u128, + pub(super) command_wait: u128, + pub(super) gpu_command: u128, + pub(super) metal_idwt_gpu: u128, + pub(super) metal_idwt_interleave_gpu: u128, + pub(super) metal_idwt_horizontal_gpu: u128, + pub(super) metal_idwt_vertical_gpu: u128, + pub(super) metal_store_gpu: u128, + pub(super) metal_mct_pack_gpu: u128, +} + +#[derive(Default)] +pub(super) struct CpuTier1DecodeSubstageCounters { + classic_segment_prep: AtomicU64, + classic_block_decode: AtomicU64, + classic_sigprop: AtomicU64, + classic_magref: AtomicU64, + classic_cleanup: AtomicU64, + classic_bypass: AtomicU64, + classic_output_convert: AtomicU64, + ht_block_decode: AtomicU64, + ht_cleanup: AtomicU64, + ht_mag_sgn: AtomicU64, + ht_sigma: AtomicU64, + ht_sigprop: AtomicU64, + ht_magref: AtomicU64, +} + +impl CpuTier1DecodeSubstageCounters { + fn add_counter(counter: &AtomicU64, elapsed_us: u128) { + counter.fetch_add( + elapsed_us.min(u128::from(u64::MAX)) as u64, + AtomicOrdering::Relaxed, + ); + } + + pub(super) fn record_classic_segment_prep(&self, started: Instant) { + self.classic_segment_prep + .fetch_add(elapsed_us_u64(started), AtomicOrdering::Relaxed); + } + + pub(super) fn record_classic_block_decode( + &self, + started: Instant, + profile: &J2kCodeBlockDecodeProfile, + ) { + self.classic_block_decode + .fetch_add(elapsed_us_u64(started), AtomicOrdering::Relaxed); + Self::add_counter(&self.classic_sigprop, profile.sigprop_us); + Self::add_counter(&self.classic_magref, profile.magref_us); + Self::add_counter(&self.classic_cleanup, profile.cleanup_us); + Self::add_counter(&self.classic_bypass, profile.bypass_us); + Self::add_counter(&self.classic_output_convert, profile.output_convert_us); + } + + pub(super) fn record_ht_block_decode( + &self, + started: Instant, + profile: &HtCodeBlockDecodeProfile, + ) { + self.ht_block_decode + .fetch_add(elapsed_us_u64(started), AtomicOrdering::Relaxed); + Self::add_counter(&self.ht_cleanup, profile.cleanup_us); + Self::add_counter(&self.ht_mag_sgn, profile.mag_sgn_us); + Self::add_counter(&self.ht_sigma, profile.sigma_us); + Self::add_counter(&self.ht_sigprop, profile.sigprop_us); + Self::add_counter(&self.ht_magref, profile.magref_us); + } + + fn load_counter(counter: &AtomicU64) -> u128 { + u128::from(counter.load(AtomicOrdering::Relaxed)) + } + + pub(super) fn add_to_stage_timings(&self, timings: &mut DirectHybridStageTimings) { + timings.cpu_tier1_classic_segment_prep = timings + .cpu_tier1_classic_segment_prep + .saturating_add(Self::load_counter(&self.classic_segment_prep)); + timings.cpu_tier1_classic_block_decode = timings + .cpu_tier1_classic_block_decode + .saturating_add(Self::load_counter(&self.classic_block_decode)); + timings.cpu_tier1_classic_sigprop = timings + .cpu_tier1_classic_sigprop + .saturating_add(Self::load_counter(&self.classic_sigprop)); + timings.cpu_tier1_classic_magref = timings + .cpu_tier1_classic_magref + .saturating_add(Self::load_counter(&self.classic_magref)); + timings.cpu_tier1_classic_cleanup = timings + .cpu_tier1_classic_cleanup + .saturating_add(Self::load_counter(&self.classic_cleanup)); + timings.cpu_tier1_classic_bypass = timings + .cpu_tier1_classic_bypass + .saturating_add(Self::load_counter(&self.classic_bypass)); + timings.cpu_tier1_classic_output_convert = timings + .cpu_tier1_classic_output_convert + .saturating_add(Self::load_counter(&self.classic_output_convert)); + timings.cpu_tier1_ht_block_decode = timings + .cpu_tier1_ht_block_decode + .saturating_add(Self::load_counter(&self.ht_block_decode)); + timings.cpu_tier1_ht_cleanup = timings + .cpu_tier1_ht_cleanup + .saturating_add(Self::load_counter(&self.ht_cleanup)); + timings.cpu_tier1_ht_mag_sgn = timings + .cpu_tier1_ht_mag_sgn + .saturating_add(Self::load_counter(&self.ht_mag_sgn)); + timings.cpu_tier1_ht_sigma = timings + .cpu_tier1_ht_sigma + .saturating_add(Self::load_counter(&self.ht_sigma)); + timings.cpu_tier1_ht_sigprop = timings + .cpu_tier1_ht_sigprop + .saturating_add(Self::load_counter(&self.ht_sigprop)); + timings.cpu_tier1_ht_magref = timings + .cpu_tier1_ht_magref + .saturating_add(Self::load_counter(&self.ht_magref)); + } +} + +pub(super) fn elapsed_us(started: Instant) -> u128 { + started.elapsed().as_micros() +} + +#[cfg(target_os = "macos")] +pub(super) fn record_completed_decode_split_gpu_stages( + timings: &mut DirectHybridStageTimings, + command_buffers: &DecodeHybridSplitCommandBuffers, +) { + let mut gpu_command = Duration::ZERO; + let mut idwt_gpu = Duration::ZERO; + if let Some(duration) = completed_command_buffer_gpu_duration(&command_buffers.idwt_interleave) + { + timings.metal_idwt_interleave_gpu = timings + .metal_idwt_interleave_gpu + .saturating_add(duration.as_micros()); + idwt_gpu = idwt_gpu.saturating_add(duration); + gpu_command = gpu_command.saturating_add(duration); + } + if let Some(duration) = completed_command_buffer_gpu_duration(&command_buffers.idwt_horizontal) + { + timings.metal_idwt_horizontal_gpu = timings + .metal_idwt_horizontal_gpu + .saturating_add(duration.as_micros()); + idwt_gpu = idwt_gpu.saturating_add(duration); + gpu_command = gpu_command.saturating_add(duration); + } + if let Some(duration) = completed_command_buffer_gpu_duration(&command_buffers.idwt_vertical) { + timings.metal_idwt_vertical_gpu = timings + .metal_idwt_vertical_gpu + .saturating_add(duration.as_micros()); + idwt_gpu = idwt_gpu.saturating_add(duration); + gpu_command = gpu_command.saturating_add(duration); + } + timings.metal_idwt_gpu = timings.metal_idwt_gpu.saturating_add(idwt_gpu.as_micros()); + if let Some(duration) = completed_command_buffer_gpu_duration(&command_buffers.store) { + timings.metal_store_gpu = timings.metal_store_gpu.saturating_add(duration.as_micros()); + gpu_command = gpu_command.saturating_add(duration); + } + if let Some(duration) = completed_command_buffer_gpu_duration(&command_buffers.mct_pack) { + timings.metal_mct_pack_gpu = timings + .metal_mct_pack_gpu + .saturating_add(duration.as_micros()); + gpu_command = gpu_command.saturating_add(duration); + } + timings.gpu_command = timings.gpu_command.saturating_add(gpu_command.as_micros()); +} + +fn elapsed_us_u64(started: Instant) -> u64 { + elapsed_us(started).min(u128::from(u64::MAX)) as u64 +} + +pub(super) fn emit_direct_hybrid_stage_timings( + timings: &DirectHybridStageTimings, + fmt: PixelFormat, + batch_count: usize, +) { + if !metal_profile_stages_enabled() { + return; + } + + let fmt_s = format!("{fmt:?}"); + let batch_count_s = batch_count.to_string(); + let label = decode_profile_label(); + for (stage, elapsed_us) in [ + ("cpu_tier1", timings.cpu_tier1), + ( + "cpu_tier1_flattened_batches", + timings.cpu_tier1_flattened_batches, + ), + ( + "cpu_tier1_classic_segment_prep", + timings.cpu_tier1_classic_segment_prep, + ), + ( + "cpu_tier1_classic_block_decode", + timings.cpu_tier1_classic_block_decode, + ), + ( + "cpu_tier1_classic_sigprop", + timings.cpu_tier1_classic_sigprop, + ), + ("cpu_tier1_classic_magref", timings.cpu_tier1_classic_magref), + ( + "cpu_tier1_classic_cleanup", + timings.cpu_tier1_classic_cleanup, + ), + ("cpu_tier1_classic_bypass", timings.cpu_tier1_classic_bypass), + ( + "cpu_tier1_classic_output_convert", + timings.cpu_tier1_classic_output_convert, + ), + ( + "cpu_tier1_ht_block_decode", + timings.cpu_tier1_ht_block_decode, + ), + ("cpu_tier1_ht_cleanup", timings.cpu_tier1_ht_cleanup), + ("cpu_tier1_ht_mag_sgn", timings.cpu_tier1_ht_mag_sgn), + ("cpu_tier1_ht_sigma", timings.cpu_tier1_ht_sigma), + ("cpu_tier1_ht_sigprop", timings.cpu_tier1_ht_sigprop), + ("cpu_tier1_ht_magref", timings.cpu_tier1_ht_magref), + ("coefficient_upload", timings.coefficient_upload), + ("metal_idwt_encode", timings.metal_idwt_encode), + ("metal_store_encode", timings.metal_store_encode), + ("metal_mct_pack_encode", timings.metal_mct_pack_encode), + ("command_wait", timings.command_wait), + ("gpu_command", timings.gpu_command), + ("metal_idwt_gpu", timings.metal_idwt_gpu), + ( + "metal_idwt_interleave_gpu", + timings.metal_idwt_interleave_gpu, + ), + ( + "metal_idwt_horizontal_gpu", + timings.metal_idwt_horizontal_gpu, + ), + ("metal_idwt_vertical_gpu", timings.metal_idwt_vertical_gpu), + ("metal_store_gpu", timings.metal_store_gpu), + ("metal_mct_pack_gpu", timings.metal_mct_pack_gpu), + ] { + let elapsed_us_s = elapsed_us.to_string(); + let processor = stage_processor(stage); + let metric = stage_metric(stage); + let metric_kind = stage_metric_kind(stage); + let aggregation = stage_aggregation(stage); + j2k_profile::emit_profile_row_now( + "j2k", + "decode", + "metal_cpu_hybrid", + &[ + ("pipeline", "decode_hybrid".to_string()), + ("label", label.clone()), + ("stage", stage.to_string()), + ("processor", processor.to_string()), + ("metric", metric.to_string()), + ("metric_kind", metric_kind.to_string()), + ("aggregation", aggregation.to_string()), + ("fmt", fmt_s.clone()), + ("batch_count", batch_count_s.clone()), + ("elapsed_us", elapsed_us_s), + ], + ); + } +} + +fn stage_processor(stage: &str) -> &'static str { + match stage { + "cpu_tier1_flattened_batches" => "scheduler", + "cpu_tier1" + | "cpu_tier1_classic_segment_prep" + | "cpu_tier1_classic_block_decode" + | "cpu_tier1_classic_sigprop" + | "cpu_tier1_classic_magref" + | "cpu_tier1_classic_cleanup" + | "cpu_tier1_classic_bypass" + | "cpu_tier1_classic_output_convert" + | "cpu_tier1_ht_block_decode" + | "cpu_tier1_ht_cleanup" + | "cpu_tier1_ht_mag_sgn" + | "cpu_tier1_ht_sigma" + | "cpu_tier1_ht_sigprop" + | "cpu_tier1_ht_magref" => "cpu", + "coefficient_upload" => "transfer", + "metal_idwt_encode" + | "metal_store_encode" + | "metal_mct_pack_encode" + | "gpu_command" + | "metal_idwt_gpu" + | "metal_idwt_interleave_gpu" + | "metal_idwt_horizontal_gpu" + | "metal_idwt_vertical_gpu" + | "metal_store_gpu" + | "metal_mct_pack_gpu" => "metal", + "command_wait" => "wait", + _ => "hybrid", + } +} + +fn stage_metric(stage: &str) -> &'static str { + match stage { + "cpu_tier1_flattened_batches" => "count", + "cpu_tier1_classic_segment_prep" + | "cpu_tier1_classic_block_decode" + | "cpu_tier1_classic_sigprop" + | "cpu_tier1_classic_magref" + | "cpu_tier1_classic_cleanup" + | "cpu_tier1_classic_bypass" + | "cpu_tier1_classic_output_convert" + | "cpu_tier1_ht_block_decode" + | "cpu_tier1_ht_cleanup" + | "cpu_tier1_ht_mag_sgn" + | "cpu_tier1_ht_sigma" + | "cpu_tier1_ht_sigprop" + | "cpu_tier1_ht_magref" => "cpu_worker_us", + "gpu_command" + | "metal_idwt_gpu" + | "metal_idwt_interleave_gpu" + | "metal_idwt_horizontal_gpu" + | "metal_idwt_vertical_gpu" + | "metal_store_gpu" + | "metal_mct_pack_gpu" => "gpu_elapsed_us", + _ => "wall_us", + } +} + +fn stage_metric_kind(stage: &str) -> &'static str { + match stage_metric(stage) { + "count" => "counter", + "cpu_worker_us" => "cpu_worker_sum", + "gpu_elapsed_us" => "gpu_busy_sum", + _ => "wall_elapsed", + } +} + +fn stage_aggregation(stage: &str) -> &'static str { + match stage_metric(stage) { + "count" | "cpu_worker_us" | "gpu_elapsed_us" => "sum", + _ => "exclusive", + } +} diff --git a/crates/j2k-metal/src/compute/direct_scratch.rs b/crates/j2k-metal/src/compute/direct_scratch.rs new file mode 100644 index 00000000..72d1b7ab --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_scratch.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::mem::size_of; + +use metal::Buffer; + +use super::MetalRuntime; +use crate::Error; + +pub(super) struct DirectScratchBuffer { + pub(super) bytes: usize, + pub(super) buffer: Buffer, +} + +pub(super) fn take_f32_scratch_buffer( + runtime: &MetalRuntime, + len: usize, +) -> Result { + let bytes = len.max(1).saturating_mul(size_of::()); + Ok(DirectScratchBuffer { + bytes, + buffer: runtime.take_private_buffer(bytes)?, + }) +} + +pub(super) fn recycle_scratch_buffers( + runtime: &MetalRuntime, + scratch_buffers: Vec, +) -> Result<(), Error> { + for scratch in scratch_buffers { + runtime.recycle_private_buffer(scratch.bytes, scratch.buffer)?; + } + Ok(()) +} + +pub(super) fn take_recyclable_private_buffer( + runtime: &MetalRuntime, + bytes: usize, + recyclable_private_buffers: &mut Vec<(usize, Buffer)>, +) -> Result { + let bytes = bytes.max(1); + let buffer = runtime.take_private_buffer(bytes)?; + recyclable_private_buffers.push((bytes, buffer.clone())); + Ok(buffer) +} + +pub(super) fn recycle_private_buffers( + runtime: &MetalRuntime, + recyclable_private_buffers: Vec<(usize, Buffer)>, +) -> Result<(), Error> { + for (bytes, buffer) in recyclable_private_buffers { + runtime.recycle_private_buffer(bytes, buffer)?; + } + Ok(()) +} + +pub(super) fn take_recyclable_shared_buffer( + runtime: &MetalRuntime, + bytes: usize, + recyclable_shared_buffers: &mut Vec<(usize, Buffer)>, +) -> Result { + let bytes = bytes.max(1); + let buffer = runtime.take_shared_buffer(bytes)?; + recyclable_shared_buffers.push((bytes, buffer.clone())); + Ok(buffer) +} + +pub(super) fn recycle_shared_buffers( + runtime: &MetalRuntime, + recyclable_shared_buffers: Vec<(usize, Buffer)>, +) -> Result<(), Error> { + for (bytes, buffer) in recyclable_shared_buffers { + runtime.recycle_shared_buffer(bytes, buffer)?; + } + Ok(()) +} diff --git a/crates/j2k-metal/src/compute/direct_status.rs b/crates/j2k-metal/src/compute/direct_status.rs new file mode 100644 index 00000000..d1fe75c3 --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_status.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: Apache-2.0 + +use metal::Buffer; + +use crate::Error; + +use super::{ + J2kClassicStatus, J2kHtStatus, J2kIdwtStatus, J2kMctStatus, J2K_CLASSIC_STATUS_FAIL, + J2K_CLASSIC_STATUS_OK, J2K_CLASSIC_STATUS_UNSUPPORTED, J2K_HT_STATUS_FAIL, J2K_HT_STATUS_OK, + J2K_HT_STATUS_UNSUPPORTED, J2K_IDWT_STATUS_FAIL, J2K_IDWT_STATUS_OK, J2K_MCT_STATUS_FAIL, + J2K_MCT_STATUS_OK, +}; + +pub(super) enum DirectStatusCheck { + Classic { buffer: Buffer, len: usize }, + Ht { buffer: Buffer, len: usize }, + Idwt(Buffer), + Mct(Buffer), +} + +pub(super) fn validate_direct_status(status_check: DirectStatusCheck) -> Result<(), Error> { + match status_check { + DirectStatusCheck::Classic { buffer, len } => { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let statuses = unsafe { + core::slice::from_raw_parts(buffer.contents().cast::(), len) + }; + if let Some(status) = statuses + .iter() + .copied() + .find(|status| status.code != J2K_CLASSIC_STATUS_OK) + { + return Err(decode_classic_status_error(status)); + } + } + DirectStatusCheck::Ht { buffer, len } => { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let statuses = unsafe { + core::slice::from_raw_parts(buffer.contents().cast::(), len) + }; + if let Some(status) = statuses + .iter() + .copied() + .find(|status| status.code != J2K_HT_STATUS_OK) + { + return Err(decode_ht_status_error(status)); + } + } + DirectStatusCheck::Idwt(buffer) => { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let status = unsafe { buffer.contents().cast::().read() }; + if status.code != J2K_IDWT_STATUS_OK { + return Err(decode_idwt_status_error(status)); + } + } + DirectStatusCheck::Mct(buffer) => { + // SAFETY: Metal buffer access follows validated sizes and synchronized command completion. + let status = unsafe { buffer.contents().cast::().read() }; + if status.code != J2K_MCT_STATUS_OK { + return Err(decode_mct_status_error(status)); + } + } + } + + Ok(()) +} + +pub(super) fn decode_classic_status_error(status: J2kClassicStatus) -> Error { + let kind = match status.code { + J2K_CLASSIC_STATUS_FAIL => "decode failure", + J2K_CLASSIC_STATUS_UNSUPPORTED => "unsupported classic kernel input", + _ => "unexpected classic kernel status", + }; + Error::MetalKernel { + message: format!("classic J2K Metal kernel {kind} (detail={})", status.detail), + } +} + +pub(super) fn decode_idwt_status_error(status: J2kIdwtStatus) -> Error { + let kind = match status.code { + J2K_IDWT_STATUS_FAIL => "decode failure", + _ => "unexpected IDWT kernel status", + }; + Error::MetalKernel { + message: format!("J2K Metal IDWT kernel {kind} (detail={})", status.detail), + } +} + +pub(super) fn decode_mct_status_error(status: J2kMctStatus) -> Error { + let kind = match status.code { + J2K_MCT_STATUS_FAIL => "decode failure", + _ => "unexpected inverse MCT kernel status", + }; + Error::MetalKernel { + message: format!( + "J2K Metal inverse MCT kernel {kind} (detail={})", + status.detail + ), + } +} + +pub(super) fn decode_ht_status_error(status: J2kHtStatus) -> Error { + let kind = match status.code { + J2K_HT_STATUS_FAIL => "decode failure", + J2K_HT_STATUS_UNSUPPORTED => "unsupported HT kernel input", + _ => "unexpected HT kernel status", + }; + Error::MetalKernel { + message: format!("HTJ2K Metal kernel {kind} (detail={})", status.detail), + } +} diff --git a/crates/j2k-metal/src/compute/direct_tier1.rs b/crates/j2k-metal/src/compute/direct_tier1.rs new file mode 100644 index 00000000..0f89303f --- /dev/null +++ b/crates/j2k-metal/src/compute/direct_tier1.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::Arc; + +use metal::Buffer; + +use super::{borrow_slice_buffer, MetalRuntime, PreparedDirectColorPlan}; + +pub(super) const HYBRID_CPU_DECODE_MIN_INPUTS_PER_TASK: usize = 1; + +const HYBRID_FLAT_CPU_TIER1_MIN_DIM: u32 = 1024; +const HYBRID_FLAT_CPU_TIER1_MIN_COUNT: usize = 16; +const HYBRID_FLAT_CPU_TIER1_ENV: &str = "J2K_HYBRID_FLAT_CPU_TIER1"; + +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum DirectTier1Mode { + Metal, + CpuUpload, +} + +#[cfg(test)] +fn record_direct_tier1_input_buffer_prepare() { + super::test_counters::record_direct_tier1_input_buffer_prepare(); +} + +#[cfg(not(test))] +fn record_direct_tier1_input_buffer_prepare() {} + +pub(super) fn prepare_direct_tier1_input_buffer( + runtime: &MetalRuntime, + data: &[T], + mode: DirectTier1Mode, +) -> Buffer { + match mode { + DirectTier1Mode::Metal => { + record_direct_tier1_input_buffer_prepare(); + borrow_slice_buffer(&runtime.device, data) + } + DirectTier1Mode::CpuUpload => runtime.tier1_dummy_buffer.clone(), + } +} + +pub(super) fn flattened_hybrid_cpu_tier1_enabled() -> bool { + std::env::var_os(HYBRID_FLAT_CPU_TIER1_ENV).is_some_and(|value| { + let value = value.to_string_lossy(); + !value.is_empty() && value != "0" && value != "false" + }) +} + +pub(super) fn should_flatten_hybrid_cpu_tier1_color_batch( + plans: &[Arc], +) -> bool { + let Some(first) = plans.first() else { + return false; + }; + plans.len() >= HYBRID_FLAT_CPU_TIER1_MIN_COUNT + && first.dimensions.0.max(first.dimensions.1) >= HYBRID_FLAT_CPU_TIER1_MIN_DIM + && !plans.iter().all(|plan| Arc::ptr_eq(plan, first)) +} + +#[cfg(test)] +pub(super) fn record_hybrid_stacked_component_batch(tier1_mode: DirectTier1Mode) { + if tier1_mode == DirectTier1Mode::CpuUpload { + super::test_counters::record_hybrid_stacked_component_batch(); + } +} + +#[cfg(not(test))] +pub(super) fn record_hybrid_stacked_component_batch(_tier1_mode: DirectTier1Mode) {} + +#[cfg(test)] +pub(super) fn record_hybrid_repeated_output_blit() { + super::test_counters::record_hybrid_repeated_output_blit(); +} + +#[cfg(not(test))] +pub(super) fn record_hybrid_repeated_output_blit() {} + +#[cfg(test)] +pub(super) fn record_hybrid_cpu_decode_worker_init() { + super::test_counters::record_hybrid_cpu_decode_worker_init(); +} + +#[cfg(not(test))] +pub(super) fn record_hybrid_cpu_decode_worker_init() {} + +#[cfg(test)] +pub(super) fn record_hybrid_cpu_decode_inputs(count: usize) { + super::test_counters::record_hybrid_cpu_decode_inputs(count); +} + +#[cfg(not(test))] +pub(super) fn record_hybrid_cpu_decode_inputs(_count: usize) {} + +#[cfg(test)] +pub(super) fn record_flattened_hybrid_cpu_decode_batch() { + super::test_counters::record_flattened_hybrid_cpu_decode_batch(); +} + +#[cfg(not(test))] +pub(super) fn record_flattened_hybrid_cpu_decode_batch() {} diff --git a/crates/j2k-metal/src/compute/encode_capacity.rs b/crates/j2k-metal/src/compute/encode_capacity.rs new file mode 100644 index 00000000..c28436c5 --- /dev/null +++ b/crates/j2k-metal/src/compute/encode_capacity.rs @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_native::EncodeProgressionOrder; + +use super::{ + J2kLosslessCodestreamAssemblyJob, J2kLosslessCodestreamBlockCodingMode, HT_PACKET_CAPACITY_ENV, + J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, + J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS, J2K_HT_ENCODE_BASE_OUTPUT_SIZE, + J2K_HT_ENCODE_MAX_SAMPLES, J2K_HT_ENCODE_MEL_SIZE, J2K_HT_ENCODE_MS_BYTES_PER_SAMPLE_FLOOR, + J2K_HT_ENCODE_MS_SIZE, J2K_HT_ENCODE_VLC_SIZE, +}; +use crate::Error; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum J2kClassicEncodeOutputCapacityMode { + Conservative, + Tight, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum J2kHtPacketOutputCapacityMode { + Conservative, + Tight, +} + +pub(crate) fn ht_packet_output_capacity_mode_from_env() -> J2kHtPacketOutputCapacityMode { + match std::env::var(HT_PACKET_CAPACITY_ENV) { + Ok(value) if value.eq_ignore_ascii_case("conservative") => { + J2kHtPacketOutputCapacityMode::Conservative + } + _ => J2kHtPacketOutputCapacityMode::Tight, + } +} + +pub(super) fn classic_encode_output_capacity_for_mode( + width: u32, + height: u32, + total_bitplanes: u8, + mode: J2kClassicEncodeOutputCapacityMode, +) -> Result { + let samples = usize::try_from(width) + .ok() + .and_then(|w| usize::try_from(height).ok().and_then(|h| w.checked_mul(h))) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal encode block size overflow".to_string(), + })?; + let bitplane_bytes = samples + .checked_mul(usize::from(total_bitplanes).max(1)) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal encode output capacity overflow".to_string(), + })?; + let payload_bytes = match mode { + J2kClassicEncodeOutputCapacityMode::Conservative => bitplane_bytes.checked_mul(8), + J2kClassicEncodeOutputCapacityMode::Tight => Some(bitplane_bytes), + } + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal encode output capacity overflow".to_string(), + })?; + payload_bytes + .checked_add(4096) + .map(|bytes| bytes.max(4096) + 1) + .ok_or_else(|| Error::MetalKernel { + message: "classic J2K Metal encode output capacity overflow".to_string(), + }) +} + +pub(super) fn classic_encode_output_capacity( + width: u32, + height: u32, + total_bitplanes: u8, +) -> Result { + classic_encode_output_capacity_for_mode( + width, + height, + total_bitplanes, + J2kClassicEncodeOutputCapacityMode::Conservative, + ) +} + +fn classic_encode_total_coding_passes(total_bitplanes: u8) -> usize { + if total_bitplanes == 0 { + 0 + } else { + 1 + 3 * (usize::from(total_bitplanes) - 1) + } +} + +fn classic_bypass_segment_index(pass_idx: usize) -> usize { + if pass_idx < 10 { + 0 + } else { + 1 + 2 * ((pass_idx - 10) / 3) + usize::from(((pass_idx - 10) % 3) == 2) + } +} + +pub(super) fn classic_encode_segment_capacity(style_flags: u32, total_bitplanes: u8) -> usize { + let total_passes = classic_encode_total_coding_passes(total_bitplanes); + if total_passes == 0 { + return 1; + } + if (style_flags & J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS) != 0 { + total_passes + } else if (style_flags & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) != 0 { + classic_bypass_segment_index(total_passes - 1) + 1 + } else { + 1 + } +} + +fn ht_scaled_scratch_size(max_size: usize, sample_count: usize) -> Result { + if sample_count > J2K_HT_ENCODE_MAX_SAMPLES { + return Err(Error::MetalKernel { + message: "HTJ2K Metal encode code-block exceeds maximum sample count".to_string(), + }); + } + + max_size + .checked_mul(sample_count) + .map(|bytes| bytes.div_ceil(J2K_HT_ENCODE_MAX_SAMPLES).max(1)) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal encode output capacity overflow".to_string(), + }) +} + +pub(super) fn ht_encode_output_capacity(width: u32, height: u32) -> Result { + let sample_count = usize::try_from(width) + .ok() + .and_then(|w| usize::try_from(height).ok().and_then(|h| w.checked_mul(h))) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal encode sample count overflow".to_string(), + })?; + let scaled_ms_size = ht_scaled_scratch_size(J2K_HT_ENCODE_MS_SIZE, sample_count)?; + let ms_floor = sample_count + .checked_mul(J2K_HT_ENCODE_MS_BYTES_PER_SAMPLE_FLOOR) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal encode output capacity overflow".to_string(), + })?; + let ms_size = scaled_ms_size.max(ms_floor).min(J2K_HT_ENCODE_MS_SIZE); + let mel_size = J2K_HT_ENCODE_MEL_SIZE; + let vlc_size = J2K_HT_ENCODE_VLC_SIZE; + ms_size + .checked_add(mel_size) + .and_then(|bytes| bytes.checked_add(vlc_size)) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal encode output capacity overflow".to_string(), + }) +} + +pub(super) fn packet_tree_node_count(width: u32, height: u32) -> Result { + if width == 0 || height == 0 { + return Ok(0); + } + let mut total = 0usize; + let mut w = width; + let mut h = height; + loop { + total = total + .checked_add( + usize::try_from(w) + .ok() + .and_then(|wu| usize::try_from(h).ok().and_then(|hu| wu.checked_mul(hu))) + .ok_or_else(|| Error::MetalKernel { + message: "Tier-2 Metal packet tag-tree size overflow".to_string(), + })?, + ) + .ok_or_else(|| Error::MetalKernel { + message: "Tier-2 Metal packet tag-tree node count overflow".to_string(), + })?; + if w <= 1 && h <= 1 { + break; + } + w = w.div_ceil(2); + h = h.div_ceil(2); + } + Ok(total) +} + +pub(super) fn lossless_codestream_payload_offset( + job: J2kLosslessCodestreamAssemblyJob, +) -> Result { + let component_count = usize::from(job.num_components); + let qcd_steps = 1usize + .checked_add( + usize::from(job.num_decomposition_levels) + .checked_mul(3) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal codestream assembly QCD step count overflow".to_string(), + })?, + ) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal codestream assembly QCD step count overflow".to_string(), + })?; + let siz_total = 40usize + .checked_add( + component_count + .checked_mul(3) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal codestream assembly SIZ size overflow".to_string(), + })?, + ) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal codestream assembly SIZ size overflow".to_string(), + })?; + let qcd_total = 5usize + .checked_add(qcd_steps) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal codestream assembly QCD size overflow".to_string(), + })?; + 2usize + .checked_add(siz_total) + .and_then(|len| { + len.checked_add( + if job.block_coding_mode == J2kLosslessCodestreamBlockCodingMode::HighThroughput { + 10 + } else { + 0 + }, + ) + }) + .and_then(|len| len.checked_add(14)) + .and_then(|len| len.checked_add(qcd_total)) + .and_then(|len| len.checked_add(if job.write_tlm { 12 } else { 0 })) + .and_then(|len| len.checked_add(12)) + .and_then(|len| len.checked_add(2)) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal codestream payload offset overflow".to_string(), + }) +} + +pub(super) fn lossless_codestream_assembly_capacity( + tile_capacity: usize, + job: J2kLosslessCodestreamAssemblyJob, +) -> Result { + lossless_codestream_payload_offset(job)? + .checked_add(tile_capacity) + .and_then(|len| len.checked_add(2)) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal codestream assembly capacity overflow".to_string(), + }) +} + +fn lossless_raw_sample_bytes( + job: J2kLosslessCodestreamAssemblyJob, + overflow_message: &'static str, +) -> Result { + let pixels = usize::try_from(job.width) + .ok() + .and_then(|width| { + usize::try_from(job.height) + .ok() + .and_then(|height| width.checked_mul(height)) + }) + .ok_or_else(|| Error::MetalKernel { + message: overflow_message.to_string(), + })?; + let component_count = usize::from(job.num_components); + let bytes_per_sample = usize::from(job.bit_depth).div_ceil(8).max(1); + pixels + .checked_mul(component_count) + .and_then(|bytes| bytes.checked_mul(bytes_per_sample)) + .ok_or_else(|| Error::MetalKernel { + message: overflow_message.to_string(), + }) +} + +fn ht_lossless_raw_sample_bytes(job: J2kLosslessCodestreamAssemblyJob) -> Result { + lossless_raw_sample_bytes(job, "HTJ2K Metal batch raw sample byte count overflow") +} + +pub(super) fn classic_packet_output_capacity( + tier1_output_capacity: usize, + header_capacity: usize, + packet_descriptor_count: usize, + codestream: J2kLosslessCodestreamAssemblyJob, +) -> Result { + let descriptor_count = packet_descriptor_count.max(1); + let conservative_capacity = tier1_output_capacity + .checked_add(header_capacity.saturating_mul(descriptor_count)) + .and_then(|bytes| bytes.checked_add(1024)) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal batch packet output capacity overflow".to_string(), + })?; + let raw_bytes = + lossless_raw_sample_bytes(codestream, "J2K Metal batch raw sample byte count overflow")?; + let descriptor_header_slack = + descriptor_count + .checked_mul(256) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal batch packet descriptor slack overflow".to_string(), + })?; + let tight_capacity = raw_bytes + .checked_add(header_capacity) + .and_then(|bytes| bytes.checked_add(descriptor_header_slack)) + .and_then(|bytes| bytes.checked_add(64 * 1024)) + .map(|bytes| bytes.max(4096)) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal batch packet output capacity overflow".to_string(), + })?; + + Ok(tight_capacity.min(conservative_capacity)) +} + +pub(super) fn ht_packet_output_capacity_for_mode( + code_block_count: usize, + header_capacity: usize, + packet_descriptor_count: usize, + codestream: J2kLosslessCodestreamAssemblyJob, + mode: J2kHtPacketOutputCapacityMode, +) -> Result { + let descriptor_count = packet_descriptor_count.max(1); + match mode { + J2kHtPacketOutputCapacityMode::Conservative => code_block_count + .checked_mul(J2K_HT_ENCODE_BASE_OUTPUT_SIZE) + .and_then(|bytes| bytes.checked_add(header_capacity.saturating_mul(descriptor_count))) + .and_then(|bytes| bytes.checked_add(1024)) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal batch packet output capacity overflow".to_string(), + }), + J2kHtPacketOutputCapacityMode::Tight => { + let raw_bytes = ht_lossless_raw_sample_bytes(codestream)?; + let descriptor_header_slack = + descriptor_count + .checked_mul(256) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal batch packet descriptor slack overflow".to_string(), + })?; + raw_bytes + .checked_add(header_capacity) + .and_then(|bytes| bytes.checked_add(descriptor_header_slack)) + .and_then(|bytes| bytes.checked_add(64 * 1024)) + .map(|bytes| bytes.max(4096)) + .ok_or_else(|| Error::MetalKernel { + message: "HTJ2K Metal batch packet output capacity overflow".to_string(), + }) + } + } +} + +pub(super) fn codestream_progression_order_code(order: EncodeProgressionOrder) -> u32 { + match order { + EncodeProgressionOrder::Lrcp => 0x00, + EncodeProgressionOrder::Rlcp => 0x01, + EncodeProgressionOrder::Rpcl => 0x02, + EncodeProgressionOrder::Pcrl => 0x03, + EncodeProgressionOrder::Cprl => 0x04, + } +} diff --git a/crates/j2k-metal/src/compute/gpu_timing.rs b/crates/j2k-metal/src/compute/gpu_timing.rs new file mode 100644 index 00000000..e1f95af2 --- /dev/null +++ b/crates/j2k-metal/src/compute/gpu_timing.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::time::Duration; + +use metal::{ + foreign_types::{ForeignType, ForeignTypeRef}, + objc::{runtime::Sel, Message}, + CommandBuffer, CommandBufferRef, +}; + +pub(super) fn completed_command_buffers_gpu_duration( + retained: &[CommandBuffer], + final_buffer: &CommandBufferRef, +) -> Option { + completed_command_buffers_gpu_duration_and_elapsed_window(retained, final_buffer) + .map(|(duration, _window)| duration) +} + +pub(super) fn completed_command_buffers_gpu_duration_and_elapsed_window( + retained: &[CommandBuffer], + final_buffer: &CommandBufferRef, +) -> Option<(Duration, Duration)> { + let mut total = Duration::ZERO; + let mut min_start = f64::INFINITY; + let mut max_end = f64::NEG_INFINITY; + let mut seen = Vec::with_capacity(retained.len().saturating_add(1)); + for command_buffer in retained { + let ptr = command_buffer.as_ptr(); + if seen.contains(&ptr) { + continue; + } + seen.push(ptr); + let (start, end) = completed_command_buffer_gpu_times(command_buffer)?; + total = total.saturating_add(Duration::from_secs_f64(end - start)); + min_start = min_start.min(start); + max_end = max_end.max(end); + } + let final_ptr = final_buffer.as_ptr(); + if !seen.contains(&final_ptr) { + let (start, end) = completed_command_buffer_gpu_times(final_buffer)?; + total = total.saturating_add(Duration::from_secs_f64(end - start)); + min_start = min_start.min(start); + max_end = max_end.max(end); + } + if min_start.is_finite() && max_end.is_finite() && max_end > min_start { + Some((total, Duration::from_secs_f64(max_end - min_start))) + } else { + None + } +} + +pub(super) fn completed_command_buffer_gpu_duration( + command_buffer: &CommandBufferRef, +) -> Option { + let (start, end) = completed_command_buffer_gpu_times(command_buffer)?; + Some(Duration::from_secs_f64(end - start)) +} + +fn completed_command_buffer_gpu_times(command_buffer: &CommandBufferRef) -> Option<(f64, f64)> { + #[cfg(test)] + super::test_counters::record_resident_gpu_timestamp_query(); + + // SAFETY: Objective-C timestamp access is queried after command-buffer completion. + let start: f64 = unsafe { + command_buffer + .send_message::<(), f64>(Sel::register("GPUStartTime"), ()) + .ok()? + }; + // SAFETY: Objective-C timestamp access is queried after command-buffer completion. + let end: f64 = unsafe { + command_buffer + .send_message::<(), f64>(Sel::register("GPUEndTime"), ()) + .ok()? + }; + if start.is_finite() && end.is_finite() && end > start { + Some((start, end)) + } else { + None + } +} diff --git a/crates/j2k-metal/src/compute/pack_params.rs b/crates/j2k-metal/src/compute/pack_params.rs new file mode 100644 index 00000000..cb77dea5 --- /dev/null +++ b/crates/j2k-metal/src/compute/pack_params.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 + +use crate::Error; + +#[cfg(target_os = "macos")] +#[derive(Clone, Copy)] +pub(super) struct J2kScalarPackParams { + pub(super) max_value: f32, + pub(super) u8_scale: f32, + pub(super) u16_scale: f32, +} + +#[cfg(target_os = "macos")] +pub(super) fn j2k_scalar_pack_params(bit_depth: u32) -> J2kScalarPackParams { + let clamped = bit_depth.min(16); + let max_value_u16 = ((1u32 << clamped) - 1).max(1) as u16; + let max_value = f32::from(max_value_u16); + let u8_scale = 255.0 / max_value; + let u16_scale = if bit_depth <= 8 { + 65_535.0 / max_value + } else { + 1.0 + }; + J2kScalarPackParams { + max_value, + u8_scale, + u16_scale, + } +} + +#[cfg(target_os = "macos")] +pub(super) fn j2k_u32_param(value: usize, message: &'static str) -> Result { + u32::try_from(value).map_err(|_| Error::MetalKernel { + message: message.to_string(), + }) +} + +#[cfg(target_os = "macos")] +pub(super) fn j2k_pack_scale_arrays(bit_depths: [u32; 4]) -> ([f32; 4], [f32; 4], [f32; 4]) { + let mut max_values = [1.0f32; 4]; + let mut u8_scales = [255.0f32; 4]; + let mut u16_scales = [65_535.0f32; 4]; + for (index, bit_depth) in bit_depths.into_iter().enumerate() { + let params = j2k_scalar_pack_params(bit_depth); + max_values[index] = params.max_value; + u8_scales[index] = params.u8_scale; + u16_scales[index] = params.u16_scale; + } + (max_values, u8_scales, u16_scales) +} diff --git a/crates/j2k-metal/src/compute/resident_packet_plan.rs b/crates/j2k-metal/src/compute/resident_packet_plan.rs new file mode 100644 index 00000000..a536c149 --- /dev/null +++ b/crates/j2k-metal/src/compute/resident_packet_plan.rs @@ -0,0 +1,420 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; + +use j2k_native::J2kPacketizationPacketDescriptor; +use metal::{Buffer, CommandBuffer}; + +use super::{ + encode_capacity::{ + codestream_progression_order_code, lossless_codestream_assembly_capacity, + lossless_codestream_payload_offset, packet_tree_node_count, + }, + J2kBatchedCodestreamAssemblyJob, J2kBatchedPacketEncodeJob, J2kLosslessCodestreamAssemblyJob, + J2kLosslessDeviceCodeBlock, J2kPacketDescriptor, J2kPacketResolution, J2kPacketStateBlock, + J2kPacketSubband, J2kPreparedLosslessDeviceCodeBlocks, J2kResidentBatchEncodeItem, + J2kResidentPacketBlock, J2kResidentPacketizationResolution, +}; +use crate::Error; + +pub(super) struct PreparedLosslessBatchTile { + pub(super) coefficient_buffer: Buffer, + pub(super) coefficient_byte_offset: usize, + pub(super) coefficient_byte_len: usize, + pub(super) coefficient_buffer_is_batch_shared: bool, + pub(super) code_blocks: Vec, + pub(super) recyclable_private_buffers: Vec<(usize, Buffer)>, + pub(super) prepare_command_buffer: CommandBuffer, + pub(super) prepare_deinterleave_rct_command_buffer: Option, + pub(super) prepare_dwt53_command_buffer: Option, + pub(super) prepare_dwt53_vertical_command_buffers: Vec, + pub(super) prepare_dwt53_horizontal_command_buffers: Vec, + pub(super) prepare_coefficient_extract_command_buffer: Option, + pub(super) deinterleave_status_buffer: Buffer, + pub(super) plane_buffers: Vec, + pub(super) scratch_buffers: Vec, + pub(super) coefficient_job_buffer: Buffer, + pub(super) resolution_count: u32, + pub(super) num_layers: u8, + pub(super) num_components: u8, + pub(super) code_block_count: u32, + pub(super) packet_descriptors: Vec, + pub(super) resolutions: Vec, + pub(super) codestream: J2kLosslessCodestreamAssemblyJob, +} + +/// Moves resident batch encode items into the family-neutral per-tile form +/// shared by the classic and HT batch drivers. +pub(super) fn prepared_lossless_batch_tiles( + items: Vec, +) -> Vec { + let mut prepared_tiles = Vec::with_capacity(items.len()); + for item in items { + let J2kPreparedLosslessDeviceCodeBlocks { + coefficient_buffer, + coefficient_byte_offset, + coefficient_byte_len, + coefficient_buffer_is_batch_shared, + code_blocks, + recyclable_private_buffers, + _prepare_command_buffer: prepare_command_buffer, + _prepare_deinterleave_rct_command_buffer: prepare_deinterleave_rct_command_buffer, + _prepare_dwt53_command_buffer: prepare_dwt53_command_buffer, + _prepare_dwt53_vertical_command_buffers: prepare_dwt53_vertical_command_buffers, + _prepare_dwt53_horizontal_command_buffers: prepare_dwt53_horizontal_command_buffers, + _prepare_coefficient_extract_command_buffer: prepare_coefficient_extract_command_buffer, + _deinterleave_status_buffer: deinterleave_status_buffer, + _plane_buffers: plane_buffers, + _scratch_buffers: scratch_buffers, + _coefficient_job_buffer: coefficient_job_buffer, + } = item.prepared; + prepared_tiles.push(PreparedLosslessBatchTile { + coefficient_buffer, + coefficient_byte_offset, + coefficient_byte_len, + coefficient_buffer_is_batch_shared, + code_blocks, + recyclable_private_buffers, + prepare_command_buffer, + prepare_deinterleave_rct_command_buffer, + prepare_dwt53_command_buffer, + prepare_dwt53_vertical_command_buffers, + prepare_dwt53_horizontal_command_buffers, + prepare_coefficient_extract_command_buffer, + deinterleave_status_buffer, + plane_buffers, + scratch_buffers, + coefficient_job_buffer, + resolution_count: item.resolution_count, + num_layers: item.num_layers, + num_components: item.num_components, + code_block_count: item.code_block_count, + packet_descriptors: item.packet_descriptors, + resolutions: item.resolutions, + codestream: item.codestream, + }); + } + prepared_tiles +} + +/// Per-family constants for the shared resident batch packet planner; values +/// reproduce each family's original literals so diagnostics and GPU job +/// fields stay byte-identical. +#[derive(Clone, Copy)] +pub(super) struct ResidentBatchPacketPlanParams { + pub(super) family_name: &'static str, + pub(super) block_coding_mode: u32, + pub(super) high_throughput: u32, + pub(super) code_block_style: u32, +} + +pub(super) struct ResidentBatchPacketPlan { + pub(super) packet_resolutions: Vec, + pub(super) packet_subbands: Vec, + pub(super) resident_blocks: Vec, + pub(super) packet_descriptors: Vec, + pub(super) state_blocks: Vec, + pub(super) packet_jobs: Vec, + pub(super) assembly_jobs: Vec, + pub(super) packet_output_capacity_total: usize, + pub(super) packet_payload_copy_job_capacity_total: usize, + pub(super) max_payload_copy_jobs_per_tile: usize, + pub(super) header_capacity_total: usize, + pub(super) scratch_words_total: usize, + pub(super) codestream_capacity_total: usize, + pub(super) codestream_offsets: Vec, + pub(super) codestream_capacities: Vec, +} + +/// Builds the per-tile packet/assembly plan shared by the classic and HT +/// resident batch encode drivers (the packet-plan stage of both was +/// token-identical apart from the values now carried in `params` and the +/// per-family packet output capacity rule). +pub(super) fn build_resident_batch_packet_plan( + prepared_tiles: &[PreparedLosslessBatchTile], + tile_tier1_job_bases: &[usize], + params: ResidentBatchPacketPlanParams, + tile_packet_output_capacity: impl Fn( + usize, + &PreparedLosslessBatchTile, + usize, + ) -> Result, +) -> Result { + let batch_err = |suffix: &str| Error::MetalKernel { + message: format!("{} Metal batch {}", params.family_name, suffix), + }; + let mut packet_resolutions = Vec::::new(); + let mut packet_subbands = Vec::::new(); + let mut resident_blocks = Vec::::new(); + let mut packet_descriptors = Vec::::new(); + let mut state_blocks = Vec::::new(); + let mut packet_jobs = Vec::::with_capacity(prepared_tiles.len()); + let mut assembly_jobs = + Vec::::with_capacity(prepared_tiles.len()); + let mut packet_output_capacity_total = 0usize; + let mut packet_payload_copy_job_capacity_total = 0usize; + let mut max_payload_copy_jobs_per_tile = 0usize; + let mut header_capacity_total = 0usize; + let mut scratch_words_total = 0usize; + let mut codestream_capacity_total = 0usize; + let mut codestream_offsets = Vec::::with_capacity(prepared_tiles.len()); + let mut codestream_capacities = Vec::::with_capacity(prepared_tiles.len()); + + for (tile_index, tile) in prepared_tiles.iter().enumerate() { + let local_resolution_offset = packet_resolutions.len(); + let local_subband_offset = packet_subbands.len(); + let local_block_offset = resident_blocks.len(); + let local_descriptor_offset = packet_descriptors.len(); + let local_state_block_offset = state_blocks.len(); + let tier1_job_base = tile_tier1_job_bases[tile_index]; + let mut max_tree_nodes = 1usize; + let mut local_subband_count = 0usize; + let mut local_resident_block_count = 0usize; + let mut local_payload_copy_job_capacity = 0usize; + + for resolution in &tile.resolutions { + let subband_offset = u32::try_from(local_subband_count) + .map_err(|_| batch_err("packet subband offset exceeds u32"))?; + for subband in &resolution.subbands { + let block_offset = u32::try_from(local_resident_block_count) + .map_err(|_| batch_err("packet block offset exceeds u32"))?; + max_tree_nodes = max_tree_nodes.max(packet_tree_node_count( + subband.num_cbs_x, + subband.num_cbs_y, + )?); + let code_block_start = usize::try_from(subband.code_block_start) + .map_err(|_| batch_err("packet code-block offset exceeds usize"))?; + let code_block_count = usize::try_from(subband.code_block_count) + .map_err(|_| batch_err("packet code-block count exceeds usize"))?; + let code_block_end = code_block_start + .checked_add(code_block_count) + .ok_or_else(|| batch_err("packet code-block range overflow"))?; + if code_block_end > tile.code_blocks.len() { + return Err(batch_err("packet code-block range out of bounds")); + } + for tier1_job_index in code_block_start..code_block_end { + resident_blocks.push(J2kResidentPacketBlock { + tier1_job_index: u32::try_from( + tier1_job_base + .checked_add(tier1_job_index) + .ok_or_else(|| batch_err("Tier-1 index overflow"))?, + ) + .map_err(|_| batch_err("Tier-1 index exceeds u32"))?, + previously_included: 0, + l_block: 3, + block_coding_mode: params.block_coding_mode, + }); + } + packet_subbands.push(J2kPacketSubband { + block_offset, + block_count: subband.code_block_count, + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + }); + local_subband_count = local_subband_count + .checked_add(1) + .ok_or_else(|| batch_err("subband count overflow"))?; + local_resident_block_count = local_resident_block_count + .checked_add(code_block_count) + .ok_or_else(|| batch_err("resident block count overflow"))?; + } + packet_resolutions.push(J2kPacketResolution { + subband_offset, + subband_count: u32::try_from(resolution.subbands.len()) + .map_err(|_| batch_err("resolution subband count exceeds u32"))?, + }); + } + + if tile.resolutions.len() + != usize::try_from(tile.resolution_count) + .map_err(|_| batch_err("resolution count exceeds usize"))? + { + return Err(batch_err("resolution count mismatch")); + } + if local_resident_block_count + != usize::try_from(tile.code_block_count) + .map_err(|_| batch_err("code-block count exceeds usize"))? + { + return Err(batch_err("code-block count mismatch")); + } + + let mut state_block_offsets = HashMap::::new(); + for descriptor in &tile.packet_descriptors { + let packet_index = usize::try_from(descriptor.packet_index) + .map_err(|_| batch_err("descriptor packet index exceeds usize"))?; + let resolution = packet_resolutions + .get(local_resolution_offset + packet_index) + .ok_or_else(|| batch_err("descriptor packet index out of range"))?; + let subband_start = usize::try_from(resolution.subband_offset) + .map_err(|_| batch_err("descriptor subband offset exceeds usize"))?; + let subband_count = usize::try_from(resolution.subband_count) + .map_err(|_| batch_err("descriptor subband count exceeds usize"))?; + let mut packet_block_count = 0usize; + for subband in &packet_subbands[local_subband_offset + subband_start + ..local_subband_offset + subband_start + subband_count] + { + let subband_block_count = usize::try_from(subband.block_count) + .map_err(|_| batch_err("descriptor block count exceeds usize"))?; + packet_block_count = packet_block_count + .checked_add(subband_block_count) + .ok_or_else(|| batch_err("descriptor block count overflow"))?; + } + let (state_block_offset, existing_count) = if let Some(&(offset, count)) = + state_block_offsets.get(&descriptor.state_index) + { + (offset, count) + } else { + let offset = u32::try_from(state_blocks.len() - local_state_block_offset) + .map_err(|_| batch_err("state block offset exceeds u32"))?; + for subband in &packet_subbands[local_subband_offset + subband_start + ..local_subband_offset + subband_start + subband_count] + { + for _ in 0..subband.block_count { + state_blocks.push(J2kPacketStateBlock { + previously_included: 0, + l_block: 3, + }); + } + } + state_block_offsets.insert(descriptor.state_index, (offset, packet_block_count)); + (offset, packet_block_count) + }; + if existing_count != packet_block_count { + return Err(batch_err("descriptor state layout mismatch")); + } + local_payload_copy_job_capacity = local_payload_copy_job_capacity + .checked_add(packet_block_count) + .ok_or_else(|| batch_err("packet payload-copy job count overflow"))?; + packet_descriptors.push(J2kPacketDescriptor { + packet_index: descriptor.packet_index, + state_index: descriptor.state_index, + layer: u32::from(descriptor.layer), + resolution: descriptor.resolution, + component: u32::from(descriptor.component), + precinct_lo: descriptor.precinct as u32, + precinct_hi: (descriptor.precinct >> 32) as u32, + state_block_offset, + }); + } + + let header_capacity = local_resident_block_count + .checked_mul(256) + .and_then(|bytes| bytes.checked_add(4096)) + .map(|bytes| bytes.max(4096)) + .ok_or_else(|| batch_err("packet header capacity overflow"))?; + let packet_output_capacity = + tile_packet_output_capacity(tile_index, tile, header_capacity)?; + let codestream_capacity = + lossless_codestream_assembly_capacity(packet_output_capacity, tile.codestream)?; + let codestream_payload_offset = lossless_codestream_payload_offset(tile.codestream)?; + let scratch_words = max_tree_nodes + .checked_mul(6) + .ok_or_else(|| batch_err("scratch size overflow"))?; + + let header_offset = header_capacity_total; + let scratch_offset = scratch_words_total; + if tile.packet_descriptors.is_empty() { + local_payload_copy_job_capacity = local_resident_block_count; + } + let payload_copy_offset = packet_payload_copy_job_capacity_total; + let codestream_offset = codestream_capacity_total; + let packet_output_offset = codestream_offset + .checked_add(codestream_payload_offset) + .ok_or_else(|| batch_err("direct packet output offset overflow"))?; + packet_jobs.push(J2kBatchedPacketEncodeJob { + resolution_offset: u32::try_from(local_resolution_offset) + .map_err(|_| batch_err("resolution offset exceeds u32"))?, + subband_offset: u32::try_from(local_subband_offset) + .map_err(|_| batch_err("subband offset exceeds u32"))?, + block_offset: u32::try_from(local_block_offset) + .map_err(|_| batch_err("block offset exceeds u32"))?, + descriptor_offset: u32::try_from(local_descriptor_offset) + .map_err(|_| batch_err("descriptor offset exceeds u32"))?, + state_block_offset: u32::try_from(local_state_block_offset) + .map_err(|_| batch_err("state block offset exceeds u32"))?, + output_offset: u32::try_from(packet_output_offset) + .map_err(|_| batch_err("packet output offset exceeds u32"))?, + header_offset: u32::try_from(header_offset) + .map_err(|_| batch_err("header offset exceeds u32"))?, + scratch_offset: u32::try_from(scratch_offset) + .map_err(|_| batch_err("scratch offset exceeds u32"))?, + payload_copy_offset: u32::try_from(payload_copy_offset) + .map_err(|_| batch_err("packet payload-copy offset exceeds u32"))?, + payload_copy_capacity: u32::try_from(local_payload_copy_job_capacity) + .map_err(|_| batch_err("packet payload-copy capacity exceeds u32"))?, + resolution_count: tile.resolution_count, + num_layers: u32::from(tile.num_layers), + num_components: u32::from(tile.num_components), + code_block_count: tile.code_block_count, + subband_count: u32::try_from(local_subband_count) + .map_err(|_| batch_err("local subband count exceeds u32"))?, + descriptor_count: u32::try_from(tile.packet_descriptors.len()) + .map_err(|_| batch_err("descriptor count exceeds u32"))?, + output_capacity: u32::try_from(packet_output_capacity) + .map_err(|_| batch_err("packet output capacity exceeds u32"))?, + header_capacity: u32::try_from(header_capacity) + .map_err(|_| batch_err("header capacity exceeds u32"))?, + scratch_node_capacity: u32::try_from(max_tree_nodes) + .map_err(|_| batch_err("scratch node capacity exceeds u32"))?, + }); + assembly_jobs.push(J2kBatchedCodestreamAssemblyJob { + tile_data_offset: u32::try_from(packet_output_offset) + .map_err(|_| batch_err("assembly packet offset exceeds u32"))?, + codestream_offset: u32::try_from(codestream_offset) + .map_err(|_| batch_err("codestream offset exceeds u32"))?, + width: tile.codestream.width, + height: tile.codestream.height, + num_components: u32::from(tile.codestream.num_components), + bit_depth: u32::from(tile.codestream.bit_depth), + signed_samples: u32::from(tile.codestream.signed), + num_decomposition_levels: u32::from(tile.codestream.num_decomposition_levels), + use_mct: u32::from(tile.codestream.use_mct), + guard_bits: u32::from(tile.codestream.guard_bits), + progression_order: codestream_progression_order_code(tile.codestream.progression_order), + write_tlm: u32::from(tile.codestream.write_tlm), + high_throughput: params.high_throughput, + code_block_style: params.code_block_style, + code_block_width_exp: u32::from(tile.codestream.code_block_width_exp), + code_block_height_exp: u32::from(tile.codestream.code_block_height_exp), + output_capacity: u32::try_from(codestream_capacity) + .map_err(|_| batch_err("codestream capacity exceeds u32"))?, + }); + codestream_offsets.push(codestream_offset); + codestream_capacities.push(codestream_capacity); + packet_output_capacity_total = packet_output_capacity_total + .checked_add(packet_output_capacity) + .ok_or_else(|| batch_err("packet output total overflow"))?; + packet_payload_copy_job_capacity_total = packet_payload_copy_job_capacity_total + .checked_add(local_payload_copy_job_capacity) + .ok_or_else(|| batch_err("packet payload-copy job total overflow"))?; + max_payload_copy_jobs_per_tile = + max_payload_copy_jobs_per_tile.max(local_payload_copy_job_capacity); + header_capacity_total = header_capacity_total + .checked_add(header_capacity) + .ok_or_else(|| batch_err("header total overflow"))?; + scratch_words_total = scratch_words_total + .checked_add(scratch_words) + .ok_or_else(|| batch_err("scratch total overflow"))?; + codestream_capacity_total = codestream_capacity_total + .checked_add(codestream_capacity) + .ok_or_else(|| batch_err("codestream total overflow"))?; + } + + Ok(ResidentBatchPacketPlan { + packet_resolutions, + packet_subbands, + resident_blocks, + packet_descriptors, + state_blocks, + packet_jobs, + assembly_jobs, + packet_output_capacity_total, + packet_payload_copy_job_capacity_total, + max_payload_copy_jobs_per_tile, + header_capacity_total, + scratch_words_total, + codestream_capacity_total, + codestream_offsets, + codestream_capacities, + }) +} diff --git a/crates/j2k-metal/src/compute/resident_stage_timing.rs b/crates/j2k-metal/src/compute/resident_stage_timing.rs new file mode 100644 index 00000000..6d5d7edc --- /dev/null +++ b/crates/j2k-metal/src/compute/resident_stage_timing.rs @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::time::{Duration, Instant}; + +use metal::CommandBuffer; + +use crate::profile_env::label_command_buffer; + +use super::{completed_command_buffer_gpu_duration, J2kResidentEncodeStageStats, MetalRuntime}; + +pub(super) struct J2kResidentEncodeGpuStageCommandBuffer { + pub(super) stage: J2kResidentEncodeGpuStage, + pub(super) command_buffer: CommandBuffer, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum J2kResidentEncodeGpuStage { + CoefficientPrep, + CoefficientDeinterleaveRct, + CoefficientDwt53, + CoefficientDwt53Vertical, + CoefficientDwt53Horizontal, + CoefficientExtract, + CoefficientCopy, + ClassicBlock, + ClassicTier1Density, + ClassicTier1RawPack, + ClassicTier1ArithmeticPack, + ClassicTier1SymbolPlan, + ClassicTier1PassPlan, + ClassicTier1TokenEmit, + ClassicTier1SplitTokenEmit, + ClassicTier1TokenPack, + HtBlock, + PacketBlockPrep, + Packetization, + PacketPayloadCopy, + CodestreamAssembly, + CodestreamPayloadCopy, +} + +pub(super) fn duration_share(duration: Duration, count: usize) -> Duration { + if count == 0 { + return Duration::ZERO; + } + let nanos = duration.as_nanos() / count as u128; + Duration::from_nanos(nanos.min(u128::from(u64::MAX)) as u64) +} + +pub(super) fn record_completed_resident_encode_gpu_stages( + stats: &mut J2kResidentEncodeStageStats, + command_buffers: &[J2kResidentEncodeGpuStageCommandBuffer], +) { + for stage_command_buffer in command_buffers { + let Some(duration) = + completed_command_buffer_gpu_duration(&stage_command_buffer.command_buffer) + else { + continue; + }; + match stage_command_buffer.stage { + J2kResidentEncodeGpuStage::CoefficientPrep => { + stats.coefficient_prep_gpu_duration = + stats.coefficient_prep_gpu_duration.saturating_add(duration); + } + J2kResidentEncodeGpuStage::CoefficientDeinterleaveRct => { + stats.coefficient_deinterleave_rct_gpu_duration = stats + .coefficient_deinterleave_rct_gpu_duration + .saturating_add(duration); + stats.coefficient_prep_gpu_duration = + stats.coefficient_prep_gpu_duration.saturating_add(duration); + } + J2kResidentEncodeGpuStage::CoefficientDwt53 => { + stats.coefficient_dwt53_gpu_duration = stats + .coefficient_dwt53_gpu_duration + .saturating_add(duration); + stats.coefficient_prep_gpu_duration = + stats.coefficient_prep_gpu_duration.saturating_add(duration); + } + J2kResidentEncodeGpuStage::CoefficientDwt53Vertical => { + stats.coefficient_dwt53_vertical_gpu_duration = stats + .coefficient_dwt53_vertical_gpu_duration + .saturating_add(duration); + stats.coefficient_dwt53_gpu_duration = stats + .coefficient_dwt53_gpu_duration + .saturating_add(duration); + stats.coefficient_prep_gpu_duration = + stats.coefficient_prep_gpu_duration.saturating_add(duration); + } + J2kResidentEncodeGpuStage::CoefficientDwt53Horizontal => { + stats.coefficient_dwt53_horizontal_gpu_duration = stats + .coefficient_dwt53_horizontal_gpu_duration + .saturating_add(duration); + stats.coefficient_dwt53_gpu_duration = stats + .coefficient_dwt53_gpu_duration + .saturating_add(duration); + stats.coefficient_prep_gpu_duration = + stats.coefficient_prep_gpu_duration.saturating_add(duration); + } + J2kResidentEncodeGpuStage::CoefficientExtract => { + stats.coefficient_extract_gpu_duration = stats + .coefficient_extract_gpu_duration + .saturating_add(duration); + stats.coefficient_prep_gpu_duration = + stats.coefficient_prep_gpu_duration.saturating_add(duration); + } + J2kResidentEncodeGpuStage::CoefficientCopy => { + stats.coefficient_copy_gpu_duration = + stats.coefficient_copy_gpu_duration.saturating_add(duration); + } + J2kResidentEncodeGpuStage::ClassicBlock => { + stats.classic_block_gpu_duration = + stats.classic_block_gpu_duration.saturating_add(duration); + } + J2kResidentEncodeGpuStage::ClassicTier1Density => { + stats.classic_tier1_density_gpu_duration = stats + .classic_tier1_density_gpu_duration + .saturating_add(duration); + } + J2kResidentEncodeGpuStage::ClassicTier1RawPack => { + stats.classic_tier1_raw_pack_gpu_duration = stats + .classic_tier1_raw_pack_gpu_duration + .saturating_add(duration); + } + J2kResidentEncodeGpuStage::ClassicTier1ArithmeticPack => { + stats.classic_tier1_arithmetic_pack_gpu_duration = stats + .classic_tier1_arithmetic_pack_gpu_duration + .saturating_add(duration); + } + J2kResidentEncodeGpuStage::ClassicTier1SymbolPlan => { + stats.classic_tier1_symbol_plan_gpu_duration = stats + .classic_tier1_symbol_plan_gpu_duration + .saturating_add(duration); + } + J2kResidentEncodeGpuStage::ClassicTier1PassPlan => { + stats.classic_tier1_pass_plan_gpu_duration = stats + .classic_tier1_pass_plan_gpu_duration + .saturating_add(duration); + } + J2kResidentEncodeGpuStage::ClassicTier1TokenEmit => { + stats.classic_tier1_token_emit_gpu_duration = stats + .classic_tier1_token_emit_gpu_duration + .saturating_add(duration); + } + J2kResidentEncodeGpuStage::ClassicTier1SplitTokenEmit => { + stats.classic_tier1_split_token_emit_gpu_duration = stats + .classic_tier1_split_token_emit_gpu_duration + .saturating_add(duration); + } + J2kResidentEncodeGpuStage::ClassicTier1TokenPack => { + stats.classic_tier1_token_pack_gpu_duration = stats + .classic_tier1_token_pack_gpu_duration + .saturating_add(duration); + } + J2kResidentEncodeGpuStage::HtBlock => { + stats.ht_block_gpu_duration = stats.ht_block_gpu_duration.saturating_add(duration); + } + J2kResidentEncodeGpuStage::PacketBlockPrep => { + stats.packet_block_prep_gpu_duration = stats + .packet_block_prep_gpu_duration + .saturating_add(duration); + } + J2kResidentEncodeGpuStage::Packetization => { + stats.packetization_gpu_duration = + stats.packetization_gpu_duration.saturating_add(duration); + } + J2kResidentEncodeGpuStage::PacketPayloadCopy => { + stats.packet_payload_copy_gpu_duration = stats + .packet_payload_copy_gpu_duration + .saturating_add(duration); + } + J2kResidentEncodeGpuStage::CodestreamAssembly => { + stats.codestream_assembly_gpu_duration = stats + .codestream_assembly_gpu_duration + .saturating_add(duration); + } + J2kResidentEncodeGpuStage::CodestreamPayloadCopy => { + stats.codestream_payload_copy_gpu_duration = stats + .codestream_payload_copy_gpu_duration + .saturating_add(duration); + } + } + } +} + +pub(super) fn new_resident_encode_command_buffer( + runtime: &MetalRuntime, + label: &str, +) -> CommandBuffer { + let command_buffer = runtime.queue.new_command_buffer().to_owned(); + label_command_buffer(&command_buffer, label); + command_buffer +} + +pub(super) fn finish_resident_encode_split_command_buffer( + command_buffer: CommandBuffer, + runtime: &MetalRuntime, + stage: J2kResidentEncodeGpuStage, + next_label: &str, + command_buffers: &mut Vec, +) -> CommandBuffer { + command_buffer.commit(); + command_buffers.push(J2kResidentEncodeGpuStageCommandBuffer { + stage, + command_buffer, + }); + new_resident_encode_command_buffer(runtime, next_label) +} + +pub(super) fn finish_resident_encode_split_command_buffer_timed( + command_buffer: CommandBuffer, + runtime: &MetalRuntime, + stage: J2kResidentEncodeGpuStage, + next_label: &str, + command_buffers: &mut Vec, + profile_stages: bool, + accumulated: &mut Duration, +) -> CommandBuffer { + let started = profile_stages.then(Instant::now); + let next = finish_resident_encode_split_command_buffer( + command_buffer, + runtime, + stage, + next_label, + command_buffers, + ); + if let Some(started) = started { + *accumulated = accumulated.saturating_add(started.elapsed()); + } + next +} diff --git a/crates/j2k-metal/src/compute/resident_types.rs b/crates/j2k-metal/src/compute/resident_types.rs new file mode 100644 index 00000000..c3cc764e --- /dev/null +++ b/crates/j2k-metal/src/compute/resident_types.rs @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::time::Duration; + +use j2k_native::J2kPacketizationPacketDescriptor; +use metal::{Buffer, CommandBuffer}; + +use super::{ + J2kLosslessCodestreamAssemblyJob, J2kPreparedLosslessDeviceCodeBlocks, + J2kResidentPacketizationResolution, +}; + +pub(crate) struct J2kResidentLosslessCodestream { + pub(crate) buffer: Buffer, + pub(crate) byte_offset: usize, + pub(crate) byte_len: usize, + pub(crate) capacity: usize, + pub(crate) gpu_duration: Option, +} + +pub(crate) struct J2kPendingResidentLosslessCodestream { + pub(super) buffer: Buffer, + pub(super) capacity: usize, + pub(super) status_buffer: Buffer, + pub(super) command_buffer: CommandBuffer, + pub(super) retained_command_buffers: Vec, + pub(super) _retained_buffers: Vec, + pub(super) status_stage: &'static str, + pub(super) length_error: &'static str, + pub(super) capacity_error: &'static str, +} + +pub(crate) struct J2kResidentBatchEncodeItem { + pub(crate) prepared: J2kPreparedLosslessDeviceCodeBlocks, + pub(crate) resolution_count: u32, + pub(crate) num_layers: u8, + pub(crate) num_components: u8, + pub(crate) code_block_count: u32, + pub(crate) packet_descriptors: Vec, + pub(crate) resolutions: Vec, + pub(crate) codestream: J2kLosslessCodestreamAssemblyJob, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct J2kResidentEncodeStageStats { + /// Host-side wall time spent preparing resident encode coefficients. + pub(crate) coefficient_prep_duration: Duration, + /// Reserved for future finer-grained profiling within coefficient prep. + pub(crate) deinterleave_rct_duration: Duration, + /// Reserved for future finer-grained profiling within coefficient prep. + pub(crate) dwt53_duration: Duration, + /// Reserved for future finer-grained profiling within coefficient prep. + pub(crate) coefficient_extract_duration: Duration, + pub(crate) ht_table_build_duration: Duration, + pub(crate) ht_buffer_allocation_duration: Duration, + pub(crate) ht_command_encode_duration: Duration, + pub(crate) ht_block_encode_duration: Duration, + pub(crate) classic_tier1_setup_duration: Duration, + pub(crate) classic_block_encode_duration: Duration, + pub(crate) classic_tier1_token_pack_duration: Duration, + pub(crate) classic_packet_plan_duration: Duration, + pub(crate) classic_packet_buffer_setup_duration: Duration, + pub(crate) classic_command_buffer_commit_duration: Duration, + pub(crate) result_harvest_duration: Duration, + pub(crate) result_status_copy_duration: Duration, + pub(crate) result_private_recycle_duration: Duration, + pub(crate) result_shared_recycle_duration: Duration, + pub(crate) result_codestream_collect_duration: Duration, + pub(crate) packet_block_prep_duration: Duration, + pub(crate) packetization_duration: Duration, + pub(crate) codestream_assembly_duration: Duration, + pub(crate) coefficient_prep_gpu_duration: Duration, + pub(crate) coefficient_deinterleave_rct_gpu_duration: Duration, + pub(crate) coefficient_dwt53_gpu_duration: Duration, + pub(crate) coefficient_dwt53_vertical_gpu_duration: Duration, + pub(crate) coefficient_dwt53_horizontal_gpu_duration: Duration, + pub(crate) coefficient_extract_gpu_duration: Duration, + pub(crate) coefficient_copy_gpu_duration: Duration, + pub(crate) gpu_elapsed_wall_duration: Duration, + pub(crate) classic_block_gpu_duration: Duration, + pub(crate) classic_tier1_density_gpu_duration: Duration, + pub(crate) classic_tier1_raw_pack_gpu_duration: Duration, + pub(crate) classic_tier1_arithmetic_pack_gpu_duration: Duration, + pub(crate) classic_tier1_symbol_plan_gpu_duration: Duration, + pub(crate) classic_tier1_pass_plan_gpu_duration: Duration, + pub(crate) classic_tier1_token_emit_gpu_duration: Duration, + pub(crate) classic_tier1_split_token_emit_gpu_duration: Duration, + pub(crate) classic_tier1_token_pack_gpu_duration: Duration, + pub(crate) ht_block_gpu_duration: Duration, + pub(crate) packet_block_prep_gpu_duration: Duration, + pub(crate) packetization_gpu_duration: Duration, + pub(crate) packet_payload_copy_gpu_duration: Duration, + pub(crate) codestream_assembly_gpu_duration: Duration, + pub(crate) codestream_payload_copy_gpu_duration: Duration, + pub(crate) tier1_output_capacity_total: usize, + pub(crate) max_tier1_output_capacity: usize, + pub(crate) tier1_output_used_bytes_total: usize, + pub(crate) max_tier1_output_used_bytes: usize, + pub(crate) tier1_segment_capacity_total: usize, + pub(crate) max_tier1_segment_capacity_per_block: usize, + pub(crate) tier1_coding_pass_count_total: usize, + pub(crate) max_tier1_coding_passes_per_block: usize, + pub(crate) tier1_arithmetic_pass_count_total: usize, + pub(crate) tier1_raw_pass_count_total: usize, + pub(crate) tier1_cleanup_pass_count_total: usize, + pub(crate) tier1_sigprop_pass_count_total: usize, + pub(crate) tier1_magref_pass_count_total: usize, + pub(crate) tier1_arithmetic_cleanup_pass_count_total: usize, + pub(crate) tier1_arithmetic_sigprop_pass_count_total: usize, + pub(crate) tier1_arithmetic_magref_pass_count_total: usize, + pub(crate) tier1_raw_sigprop_pass_count_total: usize, + pub(crate) tier1_raw_magref_pass_count_total: usize, + pub(crate) tier1_full_scan_coeff_visit_count_total: usize, + pub(crate) tier1_arithmetic_scan_coeff_visit_count_total: usize, + pub(crate) tier1_raw_scan_coeff_visit_count_total: usize, + pub(crate) tier1_cleanup_scan_coeff_visit_count_total: usize, + pub(crate) tier1_sigprop_scan_coeff_visit_count_total: usize, + pub(crate) tier1_magref_scan_coeff_visit_count_total: usize, + pub(crate) max_tier1_full_scan_coeff_visits_per_block: usize, + pub(crate) tier1_sigprop_active_candidate_count_total: usize, + pub(crate) tier1_sigprop_new_significant_count_total: usize, + pub(crate) tier1_magref_active_candidate_count_total: usize, + pub(crate) tier1_arithmetic_sigprop_active_candidate_count_total: usize, + pub(crate) tier1_arithmetic_sigprop_new_significant_count_total: usize, + pub(crate) tier1_raw_sigprop_active_candidate_count_total: usize, + pub(crate) tier1_raw_sigprop_new_significant_count_total: usize, + pub(crate) tier1_arithmetic_magref_active_candidate_count_total: usize, + pub(crate) tier1_raw_magref_active_candidate_count_total: usize, + pub(crate) tier1_cleanup_active_candidate_count_total: usize, + pub(crate) tier1_cleanup_new_significant_count_total: usize, + pub(crate) tier1_cleanup_rlc_stripe_count_total: usize, + pub(crate) tier1_cleanup_rlc_zero_stripe_count_total: usize, + pub(crate) tier1_symbol_plan_mq_symbol_count_total: usize, + pub(crate) tier1_symbol_plan_raw_bit_count_total: usize, + pub(crate) max_tier1_symbol_plan_mq_symbols_per_block: usize, + pub(crate) max_tier1_symbol_plan_raw_bits_per_block: usize, + pub(crate) tier1_symbol_plan_packed_token_bytes_total: usize, + pub(crate) max_tier1_symbol_plan_packed_token_bytes_per_block: usize, + pub(crate) tier1_symbol_plan_cleanup_mq_symbol_count_total: usize, + pub(crate) tier1_symbol_plan_sigprop_mq_symbol_count_total: usize, + pub(crate) tier1_symbol_plan_magref_mq_symbol_count_total: usize, + pub(crate) tier1_symbol_plan_raw_sigprop_bit_count_total: usize, + pub(crate) tier1_symbol_plan_raw_magref_bit_count_total: usize, + pub(crate) tier1_symbol_plan_cleanup_sign_symbol_count_total: usize, + pub(crate) tier1_symbol_plan_sigprop_sign_symbol_count_total: usize, + pub(crate) tier1_symbol_plan_mq_symbol_hash_xor: usize, + pub(crate) tier1_symbol_plan_raw_bit_hash_xor: usize, + pub(crate) tier1_pass_plan_mq_symbol_count_total: usize, + pub(crate) tier1_pass_plan_raw_bit_count_total: usize, + pub(crate) tier1_pass_plan_nonempty_mq_pass_count_total: usize, + pub(crate) tier1_pass_plan_nonempty_raw_pass_count_total: usize, + pub(crate) max_tier1_pass_plan_mq_symbols_per_pass: usize, + pub(crate) max_tier1_pass_plan_raw_bits_per_pass: usize, + pub(crate) tier1_token_emit_mq_symbol_count_total: usize, + pub(crate) tier1_token_emit_raw_bit_count_total: usize, + pub(crate) tier1_token_emit_token_bytes_total: usize, + pub(crate) max_tier1_token_emit_token_bytes_per_block: usize, + pub(crate) tier1_token_emit_segment_count_total: usize, + pub(crate) max_tier1_token_emit_segments_per_block: usize, + pub(crate) tier1_token_emit_mq_symbol_hash_xor: usize, + pub(crate) tier1_token_emit_raw_bit_hash_xor: usize, + pub(crate) tier1_token_pack_output_bytes_total: usize, + pub(crate) max_tier1_token_pack_output_bytes_per_block: usize, + pub(crate) tier1_nonzero_block_count_total: usize, + pub(crate) tier1_zero_block_count_total: usize, + pub(crate) tier1_missing_bitplane_count_total: usize, + pub(crate) max_tier1_missing_bitplanes_per_block: usize, + pub(crate) tier1_segment_count_total: usize, + pub(crate) max_tier1_segments_per_block: usize, + pub(crate) packet_payload_copy_job_capacity_total: usize, + pub(crate) max_packet_payload_copy_jobs_per_tile: usize, + pub(crate) packet_payload_copy_job_count_total: usize, + pub(crate) max_packet_payload_copy_jobs_used_per_tile: usize, + pub(crate) packet_payload_copy_bytes_total: usize, + pub(crate) max_packet_payload_copy_bytes_per_tile: usize, + pub(crate) packet_payload_copy_small_job_count_total: usize, + pub(crate) packet_payload_copy_medium_job_count_total: usize, + pub(crate) packet_payload_copy_large_job_count_total: usize, + pub(crate) packet_payload_copy_launched_stripe_count_total: usize, + pub(crate) packet_payload_copy_active_stripe_count_total: usize, + pub(crate) packet_output_capacity_total: usize, + pub(crate) max_packet_output_capacity: usize, + pub(crate) packet_output_used_bytes_total: usize, + pub(crate) max_packet_output_used_bytes: usize, + pub(crate) codestream_payload_copy_bytes_total: usize, + pub(crate) codestream_payload_copy_launched_thread_count_total: usize, + pub(crate) codestream_payload_copy_active_thread_count_total: usize, + pub(crate) code_block_count: usize, +} + +pub(crate) struct J2kResidentLosslessCodestreamBatchResult { + pub(crate) codestreams: Vec, + pub(crate) stage_stats: J2kResidentEncodeStageStats, +} diff --git a/crates/j2k-metal/src/compute/shader_source.rs b/crates/j2k-metal/src/compute/shader_source.rs new file mode 100644 index 00000000..97cec87e --- /dev/null +++ b/crates/j2k-metal/src/compute/shader_source.rs @@ -0,0 +1,533 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(target_os = "macos")] +pub(super) const SHADER_SOURCE: &str = concat!( + r#" +#include +using namespace metal; + +kernel void j2k_zero_u32_buffer( + device uint *buffer [[buffer(0)]], + constant uint &word_count [[buffer(1)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= word_count) { + return; + } + + buffer[gid] = 0u; +} + +struct J2kValidateBytesParams { + uint byte_len; +}; + +struct J2kValidateBytesStatus { + uint code; + uint index; + uint expected; + uint actual; +}; + +kernel void j2k_validate_bytes_equal( + device const uchar *actual [[buffer(0)]], + device const uchar *expected [[buffer(1)]], + device J2kValidateBytesStatus *status [[buffer(2)]], + constant J2kValidateBytesParams ¶ms [[buffer(3)]], + uint gid [[thread_position_in_grid]] +) { + if (gid != 0u) { + return; + } + + status[0].code = 0u; + status[0].index = 0u; + status[0].expected = 0u; + status[0].actual = 0u; + + for (uint i = 0u; i < params.byte_len; ++i) { + const uchar actual_byte = actual[i]; + const uchar expected_byte = expected[i]; + if (actual_byte != expected_byte) { + status[0].code = 1u; + status[0].index = i; + status[0].expected = uint(expected_byte); + status[0].actual = uint(actual_byte); + return; + } + } +} + +struct J2kCopyInterleavedParams { + uint src_width; + uint src_height; + uint src_stride; + uint dst_width; + uint dst_height; + uint dst_stride; + uint bytes_per_pixel; +}; + +kernel void j2k_copy_interleaved_padded( + device const uchar *src [[buffer(0)]], + device uchar *dst [[buffer(1)]], + constant J2kCopyInterleavedParams ¶ms [[buffer(2)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.dst_width || gid.y >= params.dst_height) { + return; + } + + const uint dst_idx = gid.y * params.dst_stride + gid.x * params.bytes_per_pixel; + const bool inside_src = gid.x < params.src_width && gid.y < params.src_height; + const uint src_idx = gid.y * params.src_stride + gid.x * params.bytes_per_pixel; + for (uint byte_idx = 0u; byte_idx < params.bytes_per_pixel; ++byte_idx) { + dst[dst_idx + byte_idx] = inside_src ? src[src_idx + byte_idx] : uchar(0); + } +} + +struct J2kLosslessDeinterleaveParams { + uint src_width; + uint src_height; + uint src_stride; + uint dst_width; + uint dst_height; + uint components; + uint bytes_per_sample; + uint sample_offset; + uint signed_samples; +}; + +inline float j2k_lossless_load_sample( + device const uchar *src, + uint base, + uint component, + uint components, + uint bytes_per_sample, + uint sample_offset, + uint signed_samples, + bool inside_src +) { + if (!inside_src) { + return signed_samples == 0u ? -float(int(sample_offset)) : 0.0f; + } + if (bytes_per_sample == 1u) { + const uint raw = uint(src[base + component]); + if (signed_samples != 0u) { + return float(raw >= 128u ? int(raw) - 256 : int(raw)); + } + return float(int(raw) - int(sample_offset)); + } + const uint byte_offset = base + component * 2u; + const uint raw = uint(src[byte_offset]) | (uint(src[byte_offset + 1u]) << 8u); + if (signed_samples != 0u) { + return float(raw >= 32768u ? int(raw) - 65536 : int(raw)); + } + return float(int(raw) - int(sample_offset)); +} + +kernel void j2k_lossless_deinterleave_to_planes( + device const uchar *src [[buffer(0)]], + device float *plane0 [[buffer(1)]], + device float *plane1 [[buffer(2)]], + device float *plane2 [[buffer(3)]], + constant J2kLosslessDeinterleaveParams ¶ms [[buffer(4)]], + device float *plane3 [[buffer(5)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.dst_width || gid.y >= params.dst_height) { + return; + } + + const bool inside_src = gid.x < params.src_width && gid.y < params.src_height; + const uint src_base = gid.y * params.src_stride + + gid.x * params.components * params.bytes_per_sample; + const uint dst_idx = gid.y * params.dst_width + gid.x; + plane0[dst_idx] = j2k_lossless_load_sample( + src, + src_base, + 0u, + params.components, + params.bytes_per_sample, + params.sample_offset, + params.signed_samples, + inside_src + ); + if (params.components >= 2u) { + plane1[dst_idx] = j2k_lossless_load_sample( + src, + src_base, + 1u, + params.components, + params.bytes_per_sample, + params.sample_offset, + params.signed_samples, + inside_src + ); + } + if (params.components >= 3u) { + plane2[dst_idx] = j2k_lossless_load_sample( + src, + src_base, + 2u, + params.components, + params.bytes_per_sample, + params.sample_offset, + params.signed_samples, + inside_src + ); + } + if (params.components >= 4u) { + plane3[dst_idx] = j2k_lossless_load_sample( + src, + src_base, + 3u, + params.components, + params.bytes_per_sample, + params.sample_offset, + params.signed_samples, + inside_src + ); + } +} + +struct J2kLosslessCoefficientJob { + uint coefficient_offset; + uint component; + uint subband_x; + uint subband_y; + uint block_x; + uint block_y; + uint block_width; + uint block_height; + uint full_width; +}; + +kernel void j2k_lossless_extract_coefficients( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device int *coefficients [[buffer(3)]], + constant J2kLosslessCoefficientJob *jobs [[buffer(4)]], + constant uint &job_count [[buffer(5)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.z >= job_count) { + return; + } + constant J2kLosslessCoefficientJob &job = jobs[gid.z]; + if (gid.x >= job.block_width || gid.y >= job.block_height) { + return; + } + + device const float *plane = plane0; + if (job.component == 1u) { + plane = plane1; + } else if (job.component == 2u) { + plane = plane2; + } + const uint src_x = job.subband_x + job.block_x + gid.x; + const uint src_y = job.subband_y + job.block_y + gid.y; + const uint src_idx = src_y * job.full_width + src_x; + const uint dst_idx = job.coefficient_offset + gid.y * job.block_width + gid.x; + coefficients[dst_idx] = int(round(plane[src_idx])); +} + +struct J2kPackParams { + uint width; + uint height; + uint out_stride; + uint output_channels; + uint opaque_alpha; + float max_values[4]; + float u8_scales[4]; + float u16_scales[4]; +}; + +struct J2kMctRgb8PackParams { + uint width; + uint height; + uint out_stride; + uint transform; + float addends[3]; + float max_values[3]; + float u8_scales[3]; +}; + +struct J2kBatchedMctRgb8PackParams { + uint width; + uint height; + uint out_stride; + uint transform; + uint batch_count; + uint plane_stride; + uint output_stride; + float addends[3]; + float max_values[3]; + float u8_scales[3]; +}; + +inline uchar scale_to_u8(float sample, float max_value, float scale) { + const float clamped = clamp(sample, 0.0f, max_value); + return uchar(min(floor(clamped * scale + 0.5f), 255.0f)); +} + +inline ushort pack_to_u16(float sample, float max_value, float scale) { + const float clamped = clamp(sample, 0.0f, max_value); + return ushort(min(floor(clamped * scale + 0.5f), 65535.0f)); +} + +kernel void j2k_pack_gray8( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device const float *plane3 [[buffer(3)]], + device uchar *out [[buffer(4)]], + constant J2kPackParams ¶ms [[buffer(5)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height) { + return; + } + + const uint idx = gid.y * params.width + gid.x; + const uint out_idx = gid.y * params.out_stride + gid.x; + out[out_idx] = scale_to_u8(plane0[idx], params.max_values[0], params.u8_scales[0]); +} + +kernel void j2k_pack_rgb8( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device const float *plane3 [[buffer(3)]], + device uchar *out [[buffer(4)]], + constant J2kPackParams ¶ms [[buffer(5)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height) { + return; + } + + const uint idx = gid.y * params.width + gid.x; + const uint out_idx = gid.y * params.out_stride + gid.x * 3u; + out[out_idx] = scale_to_u8(plane0[idx], params.max_values[0], params.u8_scales[0]); + out[out_idx + 1] = scale_to_u8(plane1[idx], params.max_values[1], params.u8_scales[1]); + out[out_idx + 2] = scale_to_u8(plane2[idx], params.max_values[2], params.u8_scales[2]); +} + +kernel void j2k_pack_mct_rgb8( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device uchar *out [[buffer(3)]], + constant J2kMctRgb8PackParams ¶ms [[buffer(4)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height) { + return; + } + + const uint idx = gid.y * params.width + gid.x; + const float y0 = plane0[idx]; + const float y1 = plane1[idx]; + const float y2 = plane2[idx]; + float rgb0; + float rgb1; + float rgb2; + + if (params.transform == 0u) { + const float i1 = y0 - floor((y2 + y1) * 0.25f); + rgb0 = y2 + i1 + params.addends[0]; + rgb1 = i1 + params.addends[1]; + rgb2 = y1 + i1 + params.addends[2]; + } else { + rgb0 = y2 * 1.402f + y0 + params.addends[0]; + rgb1 = y2 * -0.71414f + y1 * -0.34413f + y0 + params.addends[1]; + rgb2 = y1 * 1.772f + y0 + params.addends[2]; + } + + const uint out_idx = gid.y * params.out_stride + gid.x * 3u; + out[out_idx] = scale_to_u8(rgb0, params.max_values[0], params.u8_scales[0]); + out[out_idx + 1] = scale_to_u8(rgb1, params.max_values[1], params.u8_scales[1]); + out[out_idx + 2] = scale_to_u8(rgb2, params.max_values[2], params.u8_scales[2]); +} + +kernel void j2k_pack_mct_rgb8_batched( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device uchar *out [[buffer(3)]], + constant J2kBatchedMctRgb8PackParams ¶ms [[buffer(4)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height || gid.z >= params.batch_count) { + return; + } + + const uint plane_base = gid.z * params.plane_stride; + const uint idx = plane_base + gid.y * params.width + gid.x; + const float y0 = plane0[idx]; + const float y1 = plane1[idx]; + const float y2 = plane2[idx]; + float rgb0; + float rgb1; + float rgb2; + + if (params.transform == 0u) { + const float i1 = y0 - floor((y2 + y1) * 0.25f); + rgb0 = y2 + i1 + params.addends[0]; + rgb1 = i1 + params.addends[1]; + rgb2 = y1 + i1 + params.addends[2]; + } else { + rgb0 = y2 * 1.402f + y0 + params.addends[0]; + rgb1 = y2 * -0.71414f + y1 * -0.34413f + y0 + params.addends[1]; + rgb2 = y1 * 1.772f + y0 + params.addends[2]; + } + + const uint out_idx = gid.z * params.output_stride + gid.y * params.out_stride + gid.x * 3u; + out[out_idx] = scale_to_u8(rgb0, params.max_values[0], params.u8_scales[0]); + out[out_idx + 1] = scale_to_u8(rgb1, params.max_values[1], params.u8_scales[1]); + out[out_idx + 2] = scale_to_u8(rgb2, params.max_values[2], params.u8_scales[2]); +} + +kernel void j2k_pack_rgb_opaque_rgba8( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device const float *plane3 [[buffer(3)]], + device uchar *out [[buffer(4)]], + constant J2kPackParams ¶ms [[buffer(5)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height) { + return; + } + + const uint idx = gid.y * params.width + gid.x; + const uint out_idx = gid.y * params.out_stride + gid.x * 4u; + out[out_idx] = scale_to_u8(plane0[idx], params.max_values[0], params.u8_scales[0]); + out[out_idx + 1] = scale_to_u8(plane1[idx], params.max_values[1], params.u8_scales[1]); + out[out_idx + 2] = scale_to_u8(plane2[idx], params.max_values[2], params.u8_scales[2]); + out[out_idx + 3] = uchar(255); +} + +kernel void j2k_pack_rgba8( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device const float *plane3 [[buffer(3)]], + device uchar *out [[buffer(4)]], + constant J2kPackParams ¶ms [[buffer(5)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height) { + return; + } + + const uint idx = gid.y * params.width + gid.x; + const uint out_idx = gid.y * params.out_stride + gid.x * 4u; + out[out_idx] = scale_to_u8(plane0[idx], params.max_values[0], params.u8_scales[0]); + out[out_idx + 1] = scale_to_u8(plane1[idx], params.max_values[1], params.u8_scales[1]); + out[out_idx + 2] = scale_to_u8(plane2[idx], params.max_values[2], params.u8_scales[2]); + out[out_idx + 3] = scale_to_u8(plane3[idx], params.max_values[3], params.u8_scales[3]); +} + +kernel void j2k_pack_gray16( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device const float *plane3 [[buffer(3)]], + device ushort *out [[buffer(4)]], + constant J2kPackParams ¶ms [[buffer(5)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height) { + return; + } + + const uint idx = gid.y * params.width + gid.x; + const uint out_idx = (gid.y * params.out_stride) / 2u + gid.x; + out[out_idx] = pack_to_u16(plane0[idx], params.max_values[0], params.u16_scales[0]); +} + +kernel void j2k_pack_rgb16( + device const float *plane0 [[buffer(0)]], + device const float *plane1 [[buffer(1)]], + device const float *plane2 [[buffer(2)]], + device const float *plane3 [[buffer(3)]], + device ushort *out [[buffer(4)]], + constant J2kPackParams ¶ms [[buffer(5)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height) { + return; + } + + const uint idx = gid.y * params.width + gid.x; + const uint out_idx = (gid.y * params.out_stride) / 2u + gid.x * 3u; + out[out_idx] = pack_to_u16(plane0[idx], params.max_values[0], params.u16_scales[0]); + out[out_idx + 1] = pack_to_u16(plane1[idx], params.max_values[1], params.u16_scales[1]); + out[out_idx + 2] = pack_to_u16(plane2[idx], params.max_values[2], params.u16_scales[2]); +} + +struct J2kRepeatedGrayPackParams { + uint width; + uint height; + uint out_stride; + uint batch_count; + float max_value; + float u8_scale; + float u16_scale; +}; + +kernel void j2k_pack_u8_repeated_gray( + device const float *plane0 [[buffer(0)]], + device uchar *out [[buffer(1)]], + constant J2kRepeatedGrayPackParams ¶ms [[buffer(2)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height || gid.z >= params.batch_count) { + return; + } + + const uint plane_base = gid.z * params.width * params.height; + const uint out_base = gid.z * params.out_stride * params.height; + const uint plane_idx = plane_base + gid.y * params.width + gid.x; + const uint out_idx = out_base + gid.y * params.out_stride + gid.x; + out[out_idx] = scale_to_u8(plane0[plane_idx], params.max_value, params.u8_scale); +} + +kernel void j2k_pack_u16_repeated_gray( + device const float *plane0 [[buffer(0)]], + device ushort *out [[buffer(1)]], + constant J2kRepeatedGrayPackParams ¶ms [[buffer(2)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.width || gid.y >= params.height || gid.z >= params.batch_count) { + return; + } + + const uint plane_base = gid.z * params.width * params.height; + const uint out_base = (gid.z * params.out_stride * params.height) / 2u; + const uint plane_idx = plane_base + gid.y * params.width + gid.x; + const uint out_idx = out_base + gid.y * (params.out_stride / 2u) + gid.x; + out[out_idx] = pack_to_u16(plane0[plane_idx], params.max_value, params.u16_scale); +} +"#, + "\n", + include_str!("../classic.metal"), + "\n", + include_str!("../encode_bitstream.metal"), + "\n", + include_str!("../idwt.metal"), + "\n", + include_str!("../fdwt.metal"), + "\n", + include_str!("../mct.metal"), + "\n", + include_str!("../quantize.metal"), + "\n", + include_str!("../store.metal"), + "\n", + include_str!("../ht_cleanup.metal"), +); diff --git a/crates/j2k-metal/src/compute/surface_decode.rs b/crates/j2k-metal/src/compute/surface_decode.rs new file mode 100644 index 00000000..d08c8cf5 --- /dev/null +++ b/crates/j2k-metal/src/compute/surface_decode.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_core::{PixelFormat, Rect}; +use j2k_native::{ + ColorSpace as NativeColorSpace, DecodeSettings as NativeDecodeSettings, + DecodedComponents as NativeDecodedComponents, DecoderContext as NativeDecoderContext, + Image as NativeImage, +}; +use metal::Device; + +use super::{ + with_runtime, with_runtime_for_device, MetalCodeBlockDecoder, MetalRuntime, PlaneStage, +}; +use crate::{Error, Surface}; + +#[cfg(target_os = "macos")] +pub(crate) fn decode_image_to_surface<'a>( + image: &NativeImage<'a>, + context: &mut NativeDecoderContext<'a>, + fmt: PixelFormat, +) -> Result { + with_runtime(|runtime| { + let mut code_block_decoder = MetalCodeBlockDecoder::default(); + let decoded = image + .decode_components_with_ht_decoder(context, &mut code_block_decoder) + .map_err(|error| Error::Decode(j2k::J2kError::Backend(error.to_string())))?; + let stage = select_plane_stage(runtime, image, &decoded, &mut code_block_decoder)?; + stage.finish_with_runtime(runtime, fmt) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_image_to_surface_with_device<'a>( + image: &NativeImage<'a>, + context: &mut NativeDecoderContext<'a>, + fmt: PixelFormat, + device: &Device, +) -> Result { + with_runtime_for_device(device, |_| decode_image_to_surface(image, context, fmt)) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_image_region_to_surface<'a>( + image: &NativeImage<'a>, + context: &mut NativeDecoderContext<'a>, + fmt: PixelFormat, + roi: Rect, +) -> Result { + with_runtime(|runtime| { + let mut code_block_decoder = MetalCodeBlockDecoder::default(); + let decoded = image + .decode_region_components_with_ht_decoder( + context, + (roi.x, roi.y, roi.w, roi.h), + &mut code_block_decoder, + ) + .map_err(|error| Error::Decode(j2k::J2kError::Backend(error.to_string())))?; + let stage = select_plane_stage(runtime, image, &decoded, &mut code_block_decoder)?; + stage.finish_with_runtime(runtime, fmt) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_image_region_to_surface_with_device<'a>( + image: &NativeImage<'a>, + context: &mut NativeDecoderContext<'a>, + fmt: PixelFormat, + roi: Rect, + device: &Device, +) -> Result { + with_runtime_for_device(device, |_| { + decode_image_region_to_surface(image, context, fmt, roi) + }) +} + +#[cfg(target_os = "macos")] +fn select_plane_stage( + runtime: &MetalRuntime, + image: &NativeImage<'_>, + decoded: &NativeDecodedComponents<'_>, + code_block_decoder: &mut MetalCodeBlockDecoder, +) -> Result { + if image.supports_direct_device_plane_reuse() { + if matches!(decoded.color_space(), NativeColorSpace::RGB) + && !decoded.has_alpha() + && decoded.planes().len() == 3 + { + if let Some(stage) = PlaneStage::from_captured_planes( + decoded, + code_block_decoder.mct.take_captured_planes(), + ) { + return Ok(stage); + } + } + if matches!(decoded.color_space(), NativeColorSpace::Gray) + && !decoded.has_alpha() + && decoded.planes().len() == 1 + { + if let Some(stage) = PlaneStage::from_captured_planes( + decoded, + code_block_decoder.store.take_captured_planes(), + ) { + return Ok(stage); + } + } + } + + PlaneStage::from_planes(&runtime.device, decoded, None) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_scaled_to_surface( + bytes: &[u8], + dims: (u32, u32), + fmt: PixelFormat, + scale: j2k_core::Downscale, +) -> Result { + let target_dims = ( + dims.0.div_ceil(scale.denominator()), + dims.1.div_ceil(scale.denominator()), + ); + let settings = NativeDecodeSettings { + target_resolution: Some(target_dims), + ..NativeDecodeSettings::default() + }; + let image = NativeImage::new(bytes, &settings) + .map_err(|error| Error::Decode(j2k::J2kError::Backend(error.to_string())))?; + let mut context = NativeDecoderContext::default(); + decode_image_to_surface(&image, &mut context, fmt) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_region_scaled_to_surface( + bytes: &[u8], + dims: (u32, u32), + fmt: PixelFormat, + roi: j2k_core::Rect, + scale: j2k_core::Downscale, +) -> Result { + let target_dims = ( + dims.0.div_ceil(scale.denominator()), + dims.1.div_ceil(scale.denominator()), + ); + let settings = NativeDecodeSettings { + target_resolution: Some(target_dims), + ..NativeDecodeSettings::default() + }; + let image = NativeImage::new(bytes, &settings) + .map_err(|error| Error::Decode(j2k::J2kError::Backend(error.to_string())))?; + let mut context = NativeDecoderContext::default(); + decode_image_region_to_surface(&image, &mut context, fmt, roi.scaled_covering(scale)) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_scaled_to_surface_with_device( + bytes: &[u8], + dims: (u32, u32), + fmt: PixelFormat, + scale: j2k_core::Downscale, + device: &Device, +) -> Result { + with_runtime_for_device(device, |_| { + decode_scaled_to_surface(bytes, dims, fmt, scale) + }) +} + +#[cfg(target_os = "macos")] +pub(crate) fn decode_region_scaled_to_surface_with_device( + bytes: &[u8], + dims: (u32, u32), + fmt: PixelFormat, + roi: j2k_core::Rect, + scale: j2k_core::Downscale, + device: &Device, +) -> Result { + with_runtime_for_device(device, |_| { + decode_region_scaled_to_surface(bytes, dims, fmt, roi, scale) + }) +} diff --git a/crates/j2k-metal/src/compute/test_counters.rs b/crates/j2k-metal/src/compute/test_counters.rs new file mode 100644 index 00000000..bfba0eb2 --- /dev/null +++ b/crates/j2k-metal/src/compute/test_counters.rs @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::cell::Cell; +use std::sync::atomic::{AtomicUsize, Ordering}; + +macro_rules! test_atomic_counter { + ($counter:ident, $reset:ident, $load:ident) => { + static $counter: AtomicUsize = AtomicUsize::new(0); + + pub(crate) fn $reset() { + $counter.store(0, Ordering::Relaxed); + } + + pub(crate) fn $load() -> usize { + $counter.load(Ordering::Relaxed) + } + }; +} + +test_atomic_counter!( + HT_BATCH_COEFFICIENT_COPY_BLITS, + reset_ht_batch_coefficient_copy_blits_for_test, + ht_batch_coefficient_copy_blits_for_test +); +test_atomic_counter!( + HYBRID_STACKED_COMPONENT_BATCHES, + reset_hybrid_stacked_component_batches_for_test, + hybrid_stacked_component_batches_for_test +); +test_atomic_counter!( + HYBRID_REPEATED_OUTPUT_BLITS, + reset_hybrid_repeated_output_blits_for_test, + hybrid_repeated_output_blits_for_test +); +test_atomic_counter!( + HYBRID_CPU_DECODE_WORKER_INITS, + reset_hybrid_cpu_decode_worker_inits_for_test, + hybrid_cpu_decode_worker_inits_for_test +); +test_atomic_counter!( + HYBRID_CPU_DECODE_INPUTS, + reset_hybrid_cpu_decode_inputs_for_test, + hybrid_cpu_decode_inputs_for_test +); +test_atomic_counter!( + FLATTENED_HYBRID_CPU_DECODE_BATCHES, + reset_flattened_hybrid_cpu_decode_batches_for_test, + flattened_hybrid_cpu_decode_batches_for_test +); + +std::thread_local! { + static RESIDENT_GPU_TIMESTAMP_QUERIES: Cell = const { Cell::new(0) }; + static RESIDENT_CODESTREAM_COMMAND_BUFFER_WAITS: Cell = const { Cell::new(0) }; + static DIRECT_TIER1_INPUT_BUFFER_PREPARES: Cell = const { Cell::new(0) }; + static LOSSLESS_DEINTERLEAVE_RCT_FUSED_DISPATCHES: Cell = const { Cell::new(0) }; + static CLASSIC_GPU_TOKEN_PACK_DISPATCHES: Cell = const { Cell::new(0) }; + static CLASSIC_SPLIT_MQ_BYTE_GPU_TOKEN_PACK_DISPATCHES: Cell = const { Cell::new(0) }; +} + +pub(crate) fn reset_resident_gpu_timestamp_queries_for_test() { + RESIDENT_GPU_TIMESTAMP_QUERIES.with(|queries| queries.set(0)); +} + +pub(crate) fn resident_gpu_timestamp_queries_for_test() -> usize { + RESIDENT_GPU_TIMESTAMP_QUERIES.with(Cell::get) +} + +pub(crate) fn record_resident_gpu_timestamp_query() { + RESIDENT_GPU_TIMESTAMP_QUERIES.with(|queries| queries.set(queries.get() + 1)); +} + +pub(crate) fn reset_resident_codestream_command_buffer_waits_for_test() { + RESIDENT_CODESTREAM_COMMAND_BUFFER_WAITS.with(|waits| waits.set(0)); +} + +pub(crate) fn resident_codestream_command_buffer_waits_for_test() -> usize { + RESIDENT_CODESTREAM_COMMAND_BUFFER_WAITS.with(Cell::get) +} + +pub(crate) fn record_resident_codestream_command_buffer_wait() { + RESIDENT_CODESTREAM_COMMAND_BUFFER_WAITS.with(|waits| waits.set(waits.get() + 1)); +} + +pub(crate) fn reset_direct_tier1_input_buffer_prepares_for_test() { + DIRECT_TIER1_INPUT_BUFFER_PREPARES.with(|counter| counter.set(0)); +} + +pub(crate) fn direct_tier1_input_buffer_prepares_for_test() -> usize { + DIRECT_TIER1_INPUT_BUFFER_PREPARES.with(Cell::get) +} + +pub(crate) fn record_direct_tier1_input_buffer_prepare() { + DIRECT_TIER1_INPUT_BUFFER_PREPARES.with(|counter| counter.set(counter.get() + 1)); +} + +pub(crate) fn reset_lossless_deinterleave_rct_fused_dispatches_for_test() { + LOSSLESS_DEINTERLEAVE_RCT_FUSED_DISPATCHES.with(|dispatches| dispatches.set(0)); +} + +pub(crate) fn lossless_deinterleave_rct_fused_dispatches_for_test() -> usize { + LOSSLESS_DEINTERLEAVE_RCT_FUSED_DISPATCHES.with(Cell::get) +} + +pub(crate) fn record_lossless_deinterleave_rct_fused_dispatch() { + LOSSLESS_DEINTERLEAVE_RCT_FUSED_DISPATCHES + .with(|dispatches| dispatches.set(dispatches.get().saturating_add(1))); +} + +pub(crate) fn reset_classic_gpu_token_pack_dispatches_for_test() { + CLASSIC_GPU_TOKEN_PACK_DISPATCHES.with(|dispatches| dispatches.set(0)); +} + +pub(crate) fn classic_gpu_token_pack_dispatches_for_test() -> usize { + CLASSIC_GPU_TOKEN_PACK_DISPATCHES.with(Cell::get) +} + +pub(crate) fn record_classic_gpu_token_pack_dispatch() { + CLASSIC_GPU_TOKEN_PACK_DISPATCHES + .with(|dispatches| dispatches.set(dispatches.get().saturating_add(1))); +} + +pub(crate) fn reset_classic_split_mq_byte_gpu_token_pack_dispatches_for_test() { + CLASSIC_SPLIT_MQ_BYTE_GPU_TOKEN_PACK_DISPATCHES.with(|dispatches| dispatches.set(0)); +} + +pub(crate) fn classic_split_mq_byte_gpu_token_pack_dispatches_for_test() -> usize { + CLASSIC_SPLIT_MQ_BYTE_GPU_TOKEN_PACK_DISPATCHES.with(Cell::get) +} + +pub(crate) fn record_classic_split_mq_byte_gpu_token_pack_dispatch() { + CLASSIC_SPLIT_MQ_BYTE_GPU_TOKEN_PACK_DISPATCHES + .with(|dispatches| dispatches.set(dispatches.get().saturating_add(1))); +} + +pub(crate) fn record_ht_batch_coefficient_copy_blit() { + HT_BATCH_COEFFICIENT_COPY_BLITS.fetch_add(1, Ordering::Relaxed); +} + +pub(crate) fn record_hybrid_stacked_component_batch() { + HYBRID_STACKED_COMPONENT_BATCHES.fetch_add(1, Ordering::Relaxed); +} + +pub(crate) fn record_hybrid_repeated_output_blit() { + HYBRID_REPEATED_OUTPUT_BLITS.fetch_add(1, Ordering::Relaxed); +} + +pub(crate) fn record_hybrid_cpu_decode_worker_init() { + HYBRID_CPU_DECODE_WORKER_INITS.fetch_add(1, Ordering::Relaxed); +} + +pub(crate) fn record_hybrid_cpu_decode_inputs(count: usize) { + HYBRID_CPU_DECODE_INPUTS.fetch_add(count, Ordering::Relaxed); +} + +pub(crate) fn record_flattened_hybrid_cpu_decode_batch() { + FLATTENED_HYBRID_CPU_DECODE_BATCHES.fetch_add(1, Ordering::Relaxed); +} diff --git a/crates/j2k-metal/src/compute/tests.rs b/crates/j2k-metal/src/compute/tests.rs new file mode 100644 index 00000000..58692bcc --- /dev/null +++ b/crates/j2k-metal/src/compute/tests.rs @@ -0,0 +1,1435 @@ +// SPDX-License-Identifier: Apache-2.0 + +use super::{ + classic_batch_uses_plain_fast_path, classic_repeated_uses_plain_fast_path, + crop_prepared_direct_grayscale_plan_to_output_region, decode_prepared_classic_sub_band_on_cpu, + decode_scaled_to_surface, direct_tier1_input_buffer_prepares_for_test, + execute_flattened_hybrid_cpu_tier1_direct_color_plan_batch_for_test, + execute_hybrid_cpu_tier1_direct_color_plan_batch, flattened_hybrid_cpu_decode_batches_for_test, + hybrid_cpu_decode_inputs_for_test, hybrid_cpu_decode_worker_count, + hybrid_cpu_decode_worker_inits_for_test, hybrid_repeated_output_blits_for_test, + hybrid_stacked_component_batches_for_test, j2k_pack_kernel_name_for, j2k_pack_scale_arrays, + output_shape_for, prepare_direct_color_plan, prepare_direct_color_plan_for_cpu_upload, + prepare_direct_grayscale_plan, prepared_direct_color_tier1_input_count, + prepared_direct_grayscale_plan_compute_encoder_count, prepared_idwt_output_len, + prepared_repeated_direct_ht_cleanup_dispatch_count, + repeated_gray_store_is_contiguous_full_surface, + reset_direct_tier1_input_buffer_prepares_for_test, + reset_flattened_hybrid_cpu_decode_batches_for_test, reset_hybrid_cpu_decode_inputs_for_test, + reset_hybrid_cpu_decode_worker_inits_for_test, reset_hybrid_repeated_output_blits_for_test, + reset_hybrid_stacked_component_batches_for_test, reset_shared_buffer_pool_misses_for_test, + runtime_initialization_error, shared_buffer_pool_misses_for_test, + should_flatten_hybrid_cpu_tier1_color_batch, supports_stacked_direct_component_plane_batch, + with_runtime_for_device, J2kClassicCleanupBatchJob, J2kClassicSegment, + J2kRepeatedGrayStoreParams, MetalRuntime, MetalSupportError, PreparedClassicSubBand, + PreparedDirectColorPlan, PreparedDirectGrayscaleStep, +}; +use j2k_core::PixelFormat; +use j2k_native::{ + decode_j2k_sub_band_scalar, encode, encode_htj2k, ColorSpace as NativeColorSpace, + DecodeSettings, DecoderContext, EncodeOptions, Image, J2kCodeBlockBatchJob, + J2kCodeBlockDecodeJob, J2kDirectGrayscaleStep as NativeDirectGrayscaleStep, + J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, J2kSubBandDecodeJob, J2kWaveletTransform, +}; +use metal::foreign_types::ForeignType; +use metal::Device; +use std::sync::{Arc, Mutex}; + +static HYBRID_COUNTER_TEST_LOCK: Mutex<()> = Mutex::new(()); + +#[test] +fn rgb16_with_alpha_is_rejected() { + let runtime = MetalRuntime::new().expect("Metal runtime"); + let result = output_shape_for( + &NativeColorSpace::RGB, + true, + 4, + PixelFormat::Rgb16, + &runtime, + ); + assert!(result.is_err(), "RGBA input must not silently map to Rgb16"); +} + +#[test] +fn runtime_initialization_error_classifies_null_queue_as_unavailable() { + assert!(matches!( + runtime_initialization_error(&MetalSupportError::CommandQueueUnavailable), + crate::Error::MetalUnavailable + )); +} + +#[test] +fn classic_encode_output_capacity_keeps_conservative_default() { + let capacity = + super::classic_encode_output_capacity(64, 64, 11).expect("classic output capacity"); + + assert_eq!(capacity, 64 * 64 * 11 * 8 + 4097); +} + +#[test] +fn classic_encode_segment_capacity_uses_coding_style_bound() { + assert_eq!(super::classic_encode_segment_capacity(0, 16), 1); + assert_eq!( + super::classic_encode_segment_capacity( + super::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, + 9, + ), + 11 + ); + assert_eq!( + super::classic_encode_segment_capacity( + super::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, + 16, + ), + 25 + ); + assert_eq!( + super::classic_encode_segment_capacity( + super::J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS, + 16, + ), + 46 + ); +} + +#[test] +fn two_d_threads_per_group_clamps_empty_pipeline_limits() { + let threads = j2k_metal_support::two_d_threads_per_group(0, 0); + + assert_eq!((threads.width, threads.height, threads.depth), (1, 1, 1)); +} + +#[test] +fn one_d_threads_per_group_clamps_empty_pipeline_width() { + let threads = j2k_metal_support::one_d_threads_per_group(0); + + assert_eq!((threads.width, threads.height, threads.depth), (1, 1, 1)); +} + +#[test] +fn two_d_threads_per_group_preserves_simd_width_and_derives_height() { + let threads = j2k_metal_support::two_d_threads_per_group(32, 1024); + + assert_eq!((threads.width, threads.height, threads.depth), (32, 32, 1)); +} + +#[test] +fn classic_tier1_pass_class_counts_split_bypass_pass_types() { + let counts = super::classic_tier1_pass_class_counts( + 23, + super::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, + ); + + assert_eq!(counts.arithmetic, 14); + assert_eq!(counts.raw, 9); + assert_eq!(counts.cleanup, 8); + assert_eq!(counts.sigprop, 8); + assert_eq!(counts.magref, 7); + assert_eq!(counts.arithmetic_cleanup, 8); + assert_eq!(counts.arithmetic_sigprop, 3); + assert_eq!(counts.arithmetic_magref, 3); + assert_eq!(counts.raw_sigprop, 5); + assert_eq!(counts.raw_magref, 4); +} + +#[test] +fn classic_tier1_pass_class_counts_style0_stays_arithmetic() { + let counts = super::classic_tier1_pass_class_counts(5, 0); + + assert_eq!(counts.arithmetic, 5); + assert_eq!(counts.raw, 0); + assert_eq!(counts.cleanup, 2); + assert_eq!(counts.sigprop, 2); + assert_eq!(counts.magref, 1); + assert_eq!(counts.arithmetic_cleanup, 2); + assert_eq!(counts.arithmetic_sigprop, 2); + assert_eq!(counts.arithmetic_magref, 1); + assert_eq!(counts.raw_sigprop, 0); + assert_eq!(counts.raw_magref, 0); +} + +#[test] +fn classic_tier1_scan_estimates_multiply_passes_by_block_area() { + let pass_counts = super::classic_tier1_pass_class_counts( + 23, + super::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, + ); + let mut stats = super::J2kResidentEncodeStageStats::default(); + + super::accumulate_classic_tier1_scan_estimates(&mut stats, pass_counts, 32 * 32); + + assert_eq!(stats.tier1_full_scan_coeff_visit_count_total, 23 * 1024); + assert_eq!( + stats.tier1_arithmetic_scan_coeff_visit_count_total, + 14 * 1024 + ); + assert_eq!(stats.tier1_raw_scan_coeff_visit_count_total, 9 * 1024); + assert_eq!(stats.tier1_cleanup_scan_coeff_visit_count_total, 8 * 1024); + assert_eq!(stats.tier1_sigprop_scan_coeff_visit_count_total, 8 * 1024); + assert_eq!(stats.tier1_magref_scan_coeff_visit_count_total, 7 * 1024); + assert_eq!(stats.max_tier1_full_scan_coeff_visits_per_block, 23 * 1024); +} + +#[test] +fn classic_packet_output_capacity_uses_raw_sample_bound_when_smaller() { + let codestream = super::J2kLosslessCodestreamAssemblyJob { + width: 512, + height: 512, + num_components: 3, + bit_depth: 8, + signed: false, + num_decomposition_levels: 3, + use_mct: true, + guard_bits: 2, + code_block_width_exp: 4, + code_block_height_exp: 4, + progression_order: j2k_native::EncodeProgressionOrder::Lrcp, + write_tlm: false, + block_coding_mode: super::J2kLosslessCodestreamBlockCodingMode::Classic, + }; + let header_capacity = 1024 * 256 + 4096; + let conservative_capacity = 12 * 1024 * 1024; + let packet_descriptor_count = 3; + + let capacity = super::classic_packet_output_capacity( + conservative_capacity, + header_capacity, + packet_descriptor_count, + codestream, + ) + .expect("classic packet capacity"); + + let raw_bytes = 512 * 512 * 3; + let descriptor_slack = packet_descriptor_count * 256; + assert_eq!( + capacity, + raw_bytes + header_capacity + descriptor_slack + 64 * 1024 + ); + + let tiny_tier1_capacity = 4096; + let clamped = super::classic_packet_output_capacity( + tiny_tier1_capacity, + header_capacity, + packet_descriptor_count, + codestream, + ) + .expect("classic packet capacity"); + let conservative_packet_capacity = + tiny_tier1_capacity + header_capacity * packet_descriptor_count + 1024; + assert_eq!(clamped, conservative_packet_capacity); +} + +#[test] +fn ht_encode_output_capacity_scales_with_code_block_area() { + let max_block = super::ht_encode_output_capacity(128, 128).expect("max HT output capacity"); + assert_eq!(max_block, super::J2K_HT_ENCODE_BASE_OUTPUT_SIZE); + + let smaller_block = + super::ht_encode_output_capacity(32, 32).expect("scaled HT output capacity"); + assert!(smaller_block < max_block / 2); + assert!(smaller_block >= 8192); +} + +#[test] +fn classic_encode_pipeline_kind_prefers_style0_32_for_resident_jobs() { + let jobs = [super::J2kClassicEncodeBatchJob { + width: 32, + height: 32, + style_flags: 0, + ..super::J2kClassicEncodeBatchJob::default() + }]; + + assert_eq!( + super::classic_encode_code_blocks_pipeline_kind(&jobs), + super::J2kClassicEncodePipelineKind::Style0_32 + ); +} + +#[test] +fn classic_encode_pipeline_kind_prefers_bypass_32_for_resident_jobs() { + let jobs = [super::J2kClassicEncodeBatchJob { + width: 32, + height: 32, + style_flags: super::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, + total_bitplanes: 31, + ..super::J2kClassicEncodeBatchJob::default() + }]; + + assert_eq!( + super::classic_encode_code_blocks_pipeline_kind(&jobs), + super::J2kClassicEncodePipelineKind::Bypass32 + ); +} + +#[test] +fn classic_encode_pipeline_kind_prefers_bypass_u16_32_for_low_bitplane_resident_jobs() { + let jobs = [super::J2kClassicEncodeBatchJob { + width: 32, + height: 32, + style_flags: super::J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS, + total_bitplanes: 16, + ..super::J2kClassicEncodeBatchJob::default() + }]; + + assert_eq!( + super::classic_encode_code_blocks_pipeline_kind(&jobs), + super::J2kClassicEncodePipelineKind::BypassU16_32 + ); +} + +#[test] +fn with_runtime_for_device_scopes_runtime_to_requested_device() { + let Some(device) = Device::system_default() else { + return; + }; + + let runtime_device = + with_runtime_for_device(&device, |runtime| Ok(runtime.device.as_ptr() as usize)) + .expect("Metal runtime"); + + assert_eq!(runtime_device, device.as_ptr() as usize); +} + +#[test] +fn runtime_reuses_recycled_shared_buffers() -> Result<(), crate::Error> { + let Some(device) = Device::system_default() else { + return Ok(()); + }; + let runtime = MetalRuntime::new_with_device(&device).expect("Metal runtime"); + + reset_shared_buffer_pool_misses_for_test(); + let first = runtime.take_shared_buffer(64)?; + runtime.recycle_shared_buffer(64, first)?; + let _second = runtime.take_shared_buffer(64)?; + + assert_eq!( + shared_buffer_pool_misses_for_test(), + 1, + "recycled shared metadata buffers should be reused instead of allocating again" + ); + Ok(()) +} + +#[test] +fn j2k_pack_selects_specialized_kernels_for_wsi_formats() { + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::Gray, false, 1, PixelFormat::Gray8), + Some("j2k_pack_gray8") + ); + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::RGB, false, 3, PixelFormat::Rgb8), + Some("j2k_pack_rgb8") + ); + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::RGB, false, 3, PixelFormat::Rgba8), + Some("j2k_pack_rgb_opaque_rgba8") + ); + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::RGB, true, 4, PixelFormat::Rgba8), + Some("j2k_pack_rgba8") + ); + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::Gray, false, 1, PixelFormat::Gray16), + Some("j2k_pack_gray16") + ); + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::RGB, false, 3, PixelFormat::Rgb16), + Some("j2k_pack_rgb16") + ); + assert_eq!( + j2k_pack_kernel_name_for(&NativeColorSpace::RGB, true, 4, PixelFormat::Rgb16), + None, + "RGBA input must not silently drop alpha when packing RGB16" + ); +} + +#[test] +fn j2k_pack_precomputes_scale_factors_on_cpu() { + let (max_values, u8_scales, u16_scales) = j2k_pack_scale_arrays([8, 12, 16, 0]); + + assert_f32_near(max_values[0], 255.0); + assert_f32_near(max_values[1], 4095.0); + assert_f32_near(max_values[2], 65_535.0); + assert_f32_near(max_values[3], 1.0); + assert_f32_near(u8_scales[0], 1.0); + assert_f32_near(u8_scales[1], 255.0 / 4095.0); + assert_f32_near(u16_scales[0], 257.0); + assert_f32_near(u16_scales[1], 1.0); + assert_f32_near(u16_scales[2], 1.0); + assert_f32_near(u16_scales[3], 65_535.0); +} + +fn assert_f32_near(actual: f32, expected: f32) { + assert!( + (actual - expected).abs() <= f32::EPSILON, + "expected {actual} to be within f32 epsilon of {expected}" + ); +} + +#[test] +fn scaled_htj2k_decode_runs_through_metal_compute_path() { + let pixels: Vec = (0..16).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 4, 4, 1, 8, false, &options).expect("encode ht gray8"); + + let image = Image::new( + &bytes, + &DecodeSettings { + target_resolution: Some((2, 2)), + ..DecodeSettings::default() + }, + ) + .expect("image"); + let host = image.decode().expect("host scaled decode"); + + let surface = decode_scaled_to_surface( + &bytes, + (4, 4), + PixelFormat::Gray8, + j2k_core::Downscale::Half, + ) + .expect("metal scaled decode"); + assert_eq!(surface.as_bytes(), host.as_slice()); +} + +#[test] +fn prepared_ht_direct_plan_groups_cleanup_subbands_before_idwt() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let ht_subband_steps = plan + .steps + .iter() + .filter(|step| matches!(step, j2k_native::J2kDirectGrayscaleStep::HtSubBand(_))) + .count(); + assert!( + ht_subband_steps > 1, + "fixture must exercise multiple HT sub-band cleanup steps" + ); + + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + assert_eq!( + prepared.ht_groups.len(), + 1, + "single-tile HTJ2K direct decode should group adjacent HT sub-bands into one cleanup dispatch" + ); + assert_eq!(prepared.ht_groups[0].members.len(), ht_subband_steps); + assert!(matches!( + prepared.steps[prepared.ht_groups[0].start_step], + PreparedDirectGrayscaleStep::HtSubBand(_) + )); +} + +#[test] +fn grouped_ht_direct_plan_uses_one_group_coded_arena() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + + reset_direct_tier1_input_buffer_prepares_for_test(); + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + assert_eq!( + prepared.ht_groups.len(), + 1, + "fixture must exercise one grouped HT dispatch" + ); + let group = &prepared.ht_groups[0]; + assert!(!group.coded_arena.data.is_empty()); + assert_eq!( + direct_tier1_input_buffer_prepares_for_test(), + 2, + "grouped HT dispatch should prepare one coded arena buffer and one job buffer" + ); + + for step in &prepared.steps[group.start_step..group.end_step] { + let PreparedDirectGrayscaleStep::HtSubBand(sub_band) = step else { + panic!("HT group should only span HT sub-band steps"); + }; + assert!(sub_band.coded_buffer.is_none()); + assert!(sub_band.jobs_buffer.is_none()); + } +} + +#[test] +fn prepared_classic_sub_band_decodes_on_cpu_for_hybrid_upload() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 8, 8, 1, 8, false, &options).expect("encode classic gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + let native_sub_band = first_native_classic_sub_band(&plan); + let prepared_sub_band = first_prepared_classic_sub_band(&prepared); + + let expected = decode_native_classic_sub_band(native_sub_band); + let actual = + decode_prepared_classic_sub_band_on_cpu(prepared_sub_band).expect("prepared CPU decode"); + + assert_eq!(actual, expected); +} + +#[test] +fn cpu_upload_color_prepare_skips_tier1_metal_input_buffers() { + if Device::system_default().is_none() { + eprintln!("skipping CPUUpload prepare test: no Metal device"); + return; + } + + let pixels = j2k_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + + reset_direct_tier1_input_buffer_prepares_for_test(); + let metal_prepared = prepare_direct_color_plan(&plan).expect("Metal prepared color plan"); + assert_eq!(metal_prepared.component_plans.len(), 3); + assert!( + direct_tier1_input_buffer_prepares_for_test() > 0, + "normal Metal preparation should build Tier-1 input buffers" + ); + + reset_direct_tier1_input_buffer_prepares_for_test(); + let cpu_upload_prepared = + prepare_direct_color_plan_for_cpu_upload(&plan).expect("CPUUpload prepared color plan"); + assert_eq!(cpu_upload_prepared.component_plans.len(), 3); + assert_eq!( + direct_tier1_input_buffer_prepares_for_test(), + 0, + "CPUUpload preparation should keep coded Tier-1 payloads on CPU and skip Metal input buffers" + ); +} + +fn first_native_classic_sub_band( + plan: &j2k_native::J2kDirectGrayscalePlan, +) -> &J2kOwnedSubBandPlan { + plan.steps + .iter() + .find_map(|step| match step { + NativeDirectGrayscaleStep::ClassicSubBand(sub_band) => Some(sub_band), + _ => None, + }) + .expect("classic sub-band step") +} + +fn first_prepared_classic_sub_band( + plan: &super::PreparedDirectGrayscalePlan, +) -> &PreparedClassicSubBand { + plan.steps + .iter() + .find_map(|step| match step { + PreparedDirectGrayscaleStep::ClassicSubBand(sub_band) => Some(sub_band), + _ => None, + }) + .expect("prepared classic sub-band step") +} + +fn decode_native_classic_sub_band(plan: &J2kOwnedSubBandPlan) -> Vec { + let mut output = vec![0.0_f32; plan.width as usize * plan.height as usize]; + let jobs = plan + .jobs + .iter() + .map(|job| J2kCodeBlockBatchJob { + output_x: job.output_x, + output_y: job.output_y, + code_block: native_classic_job(job), + }) + .collect::>(); + decode_j2k_sub_band_scalar( + J2kSubBandDecodeJob { + width: plan.width, + height: plan.height, + jobs: &jobs, + }, + &mut output, + ) + .expect("native scalar classic sub-band decode"); + output +} + +fn native_classic_job(job: &J2kOwnedCodeBlockBatchJob) -> J2kCodeBlockDecodeJob<'_> { + J2kCodeBlockDecodeJob { + data: &job.data, + segments: &job.segments, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + total_bitplanes: job.total_bitplanes, + roi_shift: job.roi_shift, + sub_band_type: job.sub_band_type, + style: job.style, + strict: job.strict, + dequantization_step: job.dequantization_step, + } +} + +#[test] +fn prepared_ht_direct_plan_encodes_full_decode_in_one_compute_encoder() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + + assert_eq!( + prepared_direct_grayscale_plan_compute_encoder_count(&prepared, PixelFormat::Gray8), + 1, + "prepared single-tile direct decode should keep cleanup, IDWT, and grayscale store in one compute encoder" + ); +} + +#[test] +fn repeated_prepared_ht_direct_plan_groups_cleanup_subbands_before_idwt() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let ht_subband_steps = plan + .steps + .iter() + .filter(|step| matches!(step, j2k_native::J2kDirectGrayscaleStep::HtSubBand(_))) + .count(); + assert!( + ht_subband_steps > 1, + "fixture must exercise multiple HT sub-band cleanup steps" + ); + + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + assert_eq!( + prepared_repeated_direct_ht_cleanup_dispatch_count(&prepared), + 1, + "repeated HTJ2K WSI tile batches should group adjacent sub-band cleanups like the single-tile path" + ); +} + +#[test] +fn distinct_prepared_ht_direct_plans_support_stacked_component_batch() { + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes_a = encode_htj2k(&(0..64).collect::>(), 8, 8, 1, 8, false, &options) + .expect("encode first ht gray8"); + let bytes_b = encode_htj2k( + &(0..64).rev().collect::>(), + 8, + 8, + 1, + 8, + false, + &options, + ) + .expect("encode second ht gray8"); + let image_a = Image::new(&bytes_a, &DecodeSettings::default()).expect("first image"); + let image_b = Image::new(&bytes_b, &DecodeSettings::default()).expect("second image"); + let mut context_a = DecoderContext::default(); + let mut context_b = DecoderContext::default(); + let plan_a = image_a + .build_direct_grayscale_plan_with_context(&mut context_a) + .expect("first direct plan"); + let plan_b = image_b + .build_direct_grayscale_plan_with_context(&mut context_b) + .expect("second direct plan"); + let prepared_a = prepare_direct_grayscale_plan(&plan_a).expect("first prepared plan"); + let prepared_b = prepare_direct_grayscale_plan(&plan_b).expect("second prepared plan"); + + assert!( + supports_stacked_direct_component_plane_batch(&[&prepared_a, &prepared_b]), + "distinct same-shape HTJ2K grayscale plans should be eligible for one stacked batch graph" + ); +} + +#[test] +fn hybrid_rgb8_batch_uses_stacked_component_graph() { + let pixels = j2k_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_stacked_component_batches_for_test(); + reset_hybrid_cpu_decode_worker_inits_for_test(); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[prepared.clone(), prepared], + PixelFormat::Rgb8, + ) + .expect("hybrid RGB8 batch"); + + assert_eq!(surfaces.len(), 2); + assert!( + hybrid_stacked_component_batches_for_test() >= 3, + "hybrid RGB batch should stack each component plane instead of encoding each tile/component serially" + ); + assert!( + hybrid_cpu_decode_worker_inits_for_test() > 0, + "hybrid RGB batch should use worker-local CPU decode scratch instead of per-input decode/flatten" + ); +} + +#[test] +fn hybrid_rgb8_repeated_batch_decodes_shared_tier1_inputs_once() { + let pixels = j2k_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); + let unique_tier1_inputs = prepared_direct_color_tier1_input_count(&prepared); + assert!( + unique_tier1_inputs > 0, + "fixture should have Tier-1 inputs to decode" + ); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_cpu_decode_inputs_for_test(); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[prepared.clone(), prepared.clone(), prepared], + PixelFormat::Rgb8, + ) + .expect("hybrid repeated RGB8 batch"); + + assert_eq!(surfaces.len(), 3); + assert_eq!( + hybrid_cpu_decode_inputs_for_test(), + unique_tier1_inputs, + "repeated RGB hybrid batches should decode each shared coefficient input once, not once per output surface" + ); +} + +#[test] +fn hybrid_rgb8_reused_plan_caches_cpu_tier1_inputs_across_calls() { + let pixels = j2k_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); + let unique_tier1_inputs = prepared_direct_color_tier1_input_count(&prepared); + assert!( + unique_tier1_inputs > 0, + "fixture should have Tier-1 inputs to decode" + ); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_cpu_decode_inputs_for_test(); + + for _ in 0..2 { + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[prepared.clone(), prepared.clone()], + PixelFormat::Rgb8, + ) + .expect("hybrid repeated RGB8 batch"); + assert_eq!(surfaces.len(), 2); + } + + assert_eq!( + hybrid_cpu_decode_inputs_for_test(), + unique_tier1_inputs, + "reusing the same RGB hybrid plan across calls should reuse decoded CPU Tier-1 coefficients" + ); +} + +#[test] +fn hybrid_rgb8_repeated_batch_decodes_once_and_blits_distinct_outputs() { + let pixels = j2k_test_support::gradient_u8(32, 32, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 32, 32, 3, 8, false, &options).expect("encode rgb8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("direct color plan"); + let prepared = Arc::new(prepare_direct_color_plan(&plan).expect("prepared color plan")); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_repeated_output_blits_for_test(); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[ + prepared.clone(), + prepared.clone(), + prepared.clone(), + prepared, + ], + PixelFormat::Rgb8, + ) + .expect("hybrid repeated RGB8 batch"); + + assert_eq!(surfaces.len(), 4); + let surface_bytes = surfaces[0].as_bytes().len(); + let offsets = surfaces + .iter() + .map(|surface| surface.metal_buffer().expect("resident Metal surface").1) + .collect::>(); + assert_eq!( + offsets, + (0..surfaces.len()) + .map(|index| index * surface_bytes) + .collect::>(), + "repeated outputs must retain distinct Metal buffer offsets" + ); + for surface in &surfaces[1..] { + assert_eq!( + surface.as_bytes(), + surfaces[0].as_bytes(), + "repeated outputs should remain byte-identical" + ); + } + assert_eq!( + hybrid_repeated_output_blits_for_test(), + 2, + "repeated RGB hybrid batches should duplicate packed output surfaces with logarithmic Metal blit ranges" + ); +} + +#[test] +fn hybrid_rgb8_distinct_batch_keeps_tier1_inputs_separate() { + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes_a = encode( + &j2k_test_support::gradient_variant_u8(32, 32, 3, 0), + 32, + 32, + 3, + 8, + false, + &options, + ) + .expect("encode first rgb8"); + let bytes_b = encode( + &j2k_test_support::gradient_variant_u8(32, 32, 3, 7), + 32, + 32, + 3, + 8, + false, + &options, + ) + .expect("encode second rgb8"); + let image_a = Image::new(&bytes_a, &DecodeSettings::default()).expect("first image"); + let image_b = Image::new(&bytes_b, &DecodeSettings::default()).expect("second image"); + let mut context_a = DecoderContext::default(); + let mut context_b = DecoderContext::default(); + let plan_a = image_a + .build_direct_color_plan_with_context(&mut context_a) + .expect("first direct color plan"); + let plan_b = image_b + .build_direct_color_plan_with_context(&mut context_b) + .expect("second direct color plan"); + let prepared_a = Arc::new(prepare_direct_color_plan(&plan_a).expect("first prepared")); + let prepared_b = Arc::new(prepare_direct_color_plan(&plan_b).expect("second prepared")); + let expected_inputs = prepared_direct_color_tier1_input_count(&prepared_a) + + prepared_direct_color_tier1_input_count(&prepared_b); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_cpu_decode_inputs_for_test(); + + let surfaces = execute_hybrid_cpu_tier1_direct_color_plan_batch( + &[prepared_a, prepared_b], + PixelFormat::Rgb8, + ) + .expect("hybrid distinct RGB8 batch"); + + assert_eq!(surfaces.len(), 2); + assert_ne!( + surfaces[0].as_bytes(), + surfaces[1].as_bytes(), + "distinct RGB inputs must not reuse the first tile's decoded coefficients" + ); + assert_eq!( + hybrid_cpu_decode_inputs_for_test(), + expected_inputs, + "distinct RGB hybrid batches should decode each tile's own Tier-1 inputs" + ); +} + +#[test] +fn hybrid_rgb8_flattened_cpu_tier1_batch_uses_one_decode_queue() { + let pixels_a = j2k_test_support::gradient_variant_u8(32, 32, 3, 0); + let pixels_b = j2k_test_support::gradient_variant_u8(32, 32, 3, 11); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes_a = encode(&pixels_a, 32, 32, 3, 8, false, &options).expect("encode first rgb8"); + let bytes_b = encode(&pixels_b, 32, 32, 3, 8, false, &options).expect("encode second rgb8"); + let image_a = Image::new(&bytes_a, &DecodeSettings::default()).expect("first image"); + let image_b = Image::new(&bytes_b, &DecodeSettings::default()).expect("second image"); + let mut context_a = DecoderContext::default(); + let mut context_b = DecoderContext::default(); + let plan_a = image_a + .build_direct_color_plan_with_context(&mut context_a) + .expect("first direct color plan"); + let plan_b = image_b + .build_direct_color_plan_with_context(&mut context_b) + .expect("second direct color plan"); + let prepared_a = Arc::new(prepare_direct_color_plan(&plan_a).expect("first prepared")); + let prepared_b = Arc::new(prepare_direct_color_plan(&plan_b).expect("second prepared")); + let expected_inputs = prepared_direct_color_tier1_input_count(&prepared_a) + + prepared_direct_color_tier1_input_count(&prepared_b); + let _guard = HYBRID_COUNTER_TEST_LOCK + .lock() + .expect("hybrid counter lock"); + reset_hybrid_cpu_decode_inputs_for_test(); + reset_flattened_hybrid_cpu_decode_batches_for_test(); + + let surfaces = execute_flattened_hybrid_cpu_tier1_direct_color_plan_batch_for_test( + &[prepared_a, prepared_b], + PixelFormat::Rgb8, + ) + .expect("flattened hybrid distinct RGB8 batch"); + + assert_eq!(surfaces.len(), 2); + assert_ne!( + surfaces[0].as_bytes(), + surfaces[1].as_bytes(), + "flattened distinct RGB hybrid batches must keep each tile's coefficients separate" + ); + assert_eq!( + hybrid_cpu_decode_inputs_for_test(), + expected_inputs, + "flattened RGB hybrid batches should still decode every distinct Tier-1 input" + ); + assert_eq!( + flattened_hybrid_cpu_decode_batches_for_test(), + 1, + "flattened RGB hybrid should collect Tier-1 work into one CPU decode queue" + ); +} + +#[test] +fn flattened_cpu_tier1_default_gate_targets_large_distinct_batches_only() { + fn color_plan(width: u32, height: u32) -> Arc { + Arc::new(PreparedDirectColorPlan { + dimensions: (width, height), + bit_depths: [8, 8, 8], + mct: true, + transform: J2kWaveletTransform::Reversible53, + component_plans: Vec::new(), + }) + } + + let repeated = vec![color_plan(1024, 1024); 16]; + assert!( + !should_flatten_hybrid_cpu_tier1_color_batch(&repeated), + "repeated RGB batches already win through shared Tier-1 decode and should not use the flattened distinct scheduler" + ); + + let small_distinct = (0..16).map(|_| color_plan(256, 256)).collect::>(); + assert!( + !should_flatten_hybrid_cpu_tier1_color_batch(&small_distinct), + "small RGB batches measured slower with flattened Tier-1 and should stay on the grouped path" + ); + + let large_distinct = (0..16).map(|_| color_plan(1024, 1024)).collect::>(); + assert!( + should_flatten_hybrid_cpu_tier1_color_batch(&large_distinct), + "large distinct RGB explicit hybrid batches measured faster with flattened Tier-1" + ); +} + +#[test] +fn hybrid_cpu_decode_worker_count_allows_two_way_small_batch_parallelism() { + let available = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get); + if available < 2 { + return; + } + + assert_eq!( + hybrid_cpu_decode_worker_count(2), + 2, + "two independent hybrid CPU Tier-1 inputs should be able to use two workers" + ); +} + +#[test] +fn cropped_region_scaled_ht_direct_plan_prunes_codeblocks_outside_output_roi() { + let mut pixels = Vec::with_capacity(256 * 256); + for y in 0..256u32 { + for x in 0..256u32 { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 256, 256, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new( + &bytes, + &DecodeSettings { + target_resolution: Some((64, 64)), + ..DecodeSettings::default() + }, + ) + .expect("scaled image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + let full_jobs = prepared_direct_grayscale_ht_job_count(&prepared); + assert!( + full_jobs > 8, + "fixture should have multiple HT code-block jobs" + ); + + crop_prepared_direct_grayscale_plan_to_output_region( + &mut prepared, + j2k_core::Rect { + x: 24, + y: 24, + w: 8, + h: 8, + }, + ) + .expect("crop direct plan"); + let cropped_jobs = prepared_direct_grayscale_ht_job_count(&prepared); + + assert!( + cropped_jobs > 0 && cropped_jobs < full_jobs, + "cropped ROI should prune HT code-block jobs; full={full_jobs}, cropped={cropped_jobs}" + ); +} + +#[test] +fn cropped_region_scaled_ht_direct_plan_compacts_coded_payloads() { + let mut pixels = Vec::with_capacity(256 * 256); + for y in 0..256u32 { + for x in 0..256u32 { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 256, 256, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new( + &bytes, + &DecodeSettings { + target_resolution: Some((64, 64)), + ..DecodeSettings::default() + }, + ) + .expect("scaled image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + let full_bytes = prepared_direct_grayscale_ht_coded_byte_count(&prepared); + assert!(full_bytes > 0, "fixture should carry HT coded payloads"); + + crop_prepared_direct_grayscale_plan_to_output_region( + &mut prepared, + j2k_core::Rect { + x: 24, + y: 24, + w: 8, + h: 8, + }, + ) + .expect("crop direct plan"); + let cropped_bytes = prepared_direct_grayscale_ht_coded_byte_count(&prepared); + + assert!( + cropped_bytes > 0 && cropped_bytes < full_bytes, + "cropped ROI should compact HT coded bytes; full={full_bytes}, cropped={cropped_bytes}" + ); +} + +#[test] +fn cropped_region_scaled_ht_direct_plan_reduces_idwt_output_work() { + let mut pixels = Vec::with_capacity(128 * 128); + for y in 0..128u32 { + for x in 0..128u32 { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 128, 128, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new( + &bytes, + &DecodeSettings { + target_resolution: Some((32, 32)), + ..DecodeSettings::default() + }, + ) + .expect("scaled image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + let full_samples = prepared_direct_grayscale_idwt_output_sample_count(&prepared); + + crop_prepared_direct_grayscale_plan_to_output_region( + &mut prepared, + j2k_core::Rect { + x: 10, + y: 10, + w: 4, + h: 4, + }, + ) + .expect("crop direct plan"); + let cropped_samples = prepared_direct_grayscale_idwt_output_sample_count(&prepared); + + assert!( + cropped_samples > 0 && cropped_samples < full_samples, + "cropped ROI should reduce IDWT output work; full={full_samples}, cropped={cropped_samples}" + ); +} + +#[test] +fn cropped_region_ht_direct_plan_keeps_idwt_windows_bounded() { + let mut pixels = Vec::with_capacity(256 * 256); + for y in 0..256u32 { + for x in 0..256u32 { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 256, 256, 1, 8, false, &options).expect("encode ht gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let mut prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + let idwt_levels = prepared_direct_grayscale_idwt_full_and_prepared_lens(&prepared); + assert!( + idwt_levels.len() >= 3, + "fixture should exercise a multi-level IDWT plan" + ); + + crop_prepared_direct_grayscale_plan_to_output_region( + &mut prepared, + j2k_core::Rect { + x: 112, + y: 112, + w: 32, + h: 32, + }, + ) + .expect("crop direct plan"); + let cropped_idwt_levels = prepared_direct_grayscale_idwt_full_and_prepared_lens(&prepared); + + assert_eq!(cropped_idwt_levels.len(), idwt_levels.len()); + for (level_idx, (full_len, cropped_len)) in cropped_idwt_levels.iter().copied().enumerate() { + assert!( + cropped_len > 0 && cropped_len <= full_len, + "cropped ROI should keep IDWT level {level_idx} bounded; full={full_len}, cropped={cropped_len}" + ); + } + assert!( + cropped_idwt_levels + .iter() + .any(|(full_len, cropped_len)| cropped_len < full_len), + "cropped ROI should reduce at least one IDWT level" + ); +} + +fn prepared_direct_grayscale_ht_job_count(plan: &super::PreparedDirectGrayscalePlan) -> usize { + plan.steps + .iter() + .map(|step| match step { + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => sub_band.jobs.len(), + _ => 0, + }) + .sum() +} + +fn prepared_direct_grayscale_ht_coded_byte_count( + plan: &super::PreparedDirectGrayscalePlan, +) -> usize { + plan.steps + .iter() + .map(|step| match step { + PreparedDirectGrayscaleStep::HtSubBand(sub_band) => sub_band.coded_data.len(), + _ => 0, + }) + .sum() +} + +fn prepared_direct_grayscale_idwt_output_sample_count( + plan: &super::PreparedDirectGrayscalePlan, +) -> usize { + plan.steps + .iter() + .map(|step| match step { + PreparedDirectGrayscaleStep::Idwt(idwt) => prepared_idwt_output_len(idwt), + _ => 0, + }) + .sum() +} + +fn prepared_direct_grayscale_idwt_full_and_prepared_lens( + plan: &super::PreparedDirectGrayscalePlan, +) -> Vec<(usize, usize)> { + plan.steps + .iter() + .filter_map(|step| match step { + PreparedDirectGrayscaleStep::Idwt(idwt) => Some(( + idwt.step.rect.width() as usize * idwt.step.rect.height() as usize, + prepared_idwt_output_len(idwt), + )), + _ => None, + }) + .collect() +} + +#[test] +fn prepared_classic_direct_plan_groups_cleanup_subbands_before_idwt() { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode(&pixels, 8, 8, 1, 8, false, &options).expect("encode j2k gray8"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("direct grayscale plan"); + let classic_subband_steps = plan + .steps + .iter() + .filter(|step| matches!(step, j2k_native::J2kDirectGrayscaleStep::ClassicSubBand(_))) + .count(); + assert!( + classic_subband_steps > 1, + "fixture must exercise multiple classic sub-band cleanup steps" + ); + + let prepared = prepare_direct_grayscale_plan(&plan).expect("prepared direct plan"); + assert_eq!( + prepared.classic_groups.len(), + 1, + "classic J2K direct decode should group adjacent sub-band cleanups before IDWT" + ); + assert_eq!( + prepared.classic_groups[0].members.len(), + classic_subband_steps + ); + assert!(matches!( + prepared.steps[prepared.classic_groups[0].start_step], + PreparedDirectGrayscaleStep::ClassicSubBand(_) + )); +} + +#[test] +fn classic_plain_fast_path_accepts_style_zero_arithmetic_jobs() { + let jobs = [J2kClassicCleanupBatchJob { + coded_offset: 0, + coded_len: 1, + segment_offset: 0, + segment_count: 1, + width: 64, + height: 64, + output_stride: 64, + output_offset: 0, + missing_msbs: 0, + total_bitplanes: 8, + roi_shift: 0, + number_of_coding_passes: 1, + sub_band_type: 0, + style_flags: 0, + strict: 1, + dequantization_step: 1.0, + }]; + let segments = [J2kClassicSegment { + data_offset: 0, + data_length: 1, + start_coding_pass: 0, + end_coding_pass: 1, + use_arithmetic: 1, + }]; + + assert!( + classic_batch_uses_plain_fast_path(&jobs, &segments), + "style-0 arithmetic-only classic J2K jobs should use the fused plain cleanup/store kernel" + ); +} + +#[test] +fn classic_repeated_plain_fast_path_stays_off_for_wsi_batch_size() { + let jobs = [J2kClassicCleanupBatchJob { + coded_offset: 0, + coded_len: 1, + segment_offset: 0, + segment_count: 1, + width: 64, + height: 64, + output_stride: 64, + output_offset: 0, + missing_msbs: 0, + total_bitplanes: 8, + roi_shift: 0, + number_of_coding_passes: 1, + sub_band_type: 0, + style_flags: 0, + strict: 1, + dequantization_step: 1.0, + }]; + let segments = [J2kClassicSegment { + data_offset: 0, + data_length: 1, + start_coding_pass: 0, + end_coding_pass: 1, + use_arithmetic: 1, + }]; + + assert!( + !classic_repeated_uses_plain_fast_path(16, &jobs, &segments), + "batch-16 WSI classic J2K should keep the device-state cleanup plus separate store path" + ); +} + +#[test] +fn repeated_gray_store_detects_contiguous_full_wsi_tiles() { + let full_tile = J2kRepeatedGrayStoreParams { + input_width: 1024, + input_height: 1024, + source_x: 0, + source_y: 0, + copy_width: 1024, + copy_height: 1024, + output_width: 1024, + output_height: 1024, + output_x: 0, + output_y: 0, + addend: 0.0, + batch_count: 16, + max_value: 255.0, + u8_scale: 1.0, + u16_scale: 257.0, + }; + assert!( + repeated_gray_store_is_contiguous_full_surface(full_tile), + "full repeated grayscale WSI stores should use the contiguous store kernel" + ); + + let windowed = J2kRepeatedGrayStoreParams { + source_x: 1, + copy_width: 1023, + ..full_tile + }; + assert!( + !repeated_gray_store_is_contiguous_full_surface(windowed), + "ROI/windowed repeated grayscale stores must stay on the generic store kernel" + ); +} diff --git a/crates/signinum-j2k-metal/src/direct.rs b/crates/j2k-metal/src/direct.rs similarity index 92% rename from crates/signinum-j2k-metal/src/direct.rs rename to crates/j2k-metal/src/direct.rs index a83ed791..66214c1d 100644 --- a/crates/signinum-j2k-metal/src/direct.rs +++ b/crates/j2k-metal/src/direct.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 #[cfg(all(target_os = "macos", test))] -use signinum_j2k_native::{ +use j2k_native::{ HtCodeBlockBatchJob, HtCodeBlockDecodeJob, HtOwnedCodeBlockBatchJob, HtOwnedSubBandPlan, J2kCodeBlockBatchJob, J2kCodeBlockDecodeJob, J2kDirectBandId, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kIdwtBand, J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, J2kRect, @@ -96,10 +96,10 @@ fn execute_grayscale_plan_to_plane( }, }; match idwt.transform { - signinum_j2k_native::J2kWaveletTransform::Reversible53 => { + j2k_native::J2kWaveletTransform::Reversible53 => { compute::decode_reversible53_single_decomposition_idwt(job, &mut output)?; } - signinum_j2k_native::J2kWaveletTransform::Irreversible97 => { + j2k_native::J2kWaveletTransform::Irreversible97 => { compute::decode_irreversible97_single_decomposition_idwt(job, &mut output)?; } } @@ -109,7 +109,7 @@ fn execute_grayscale_plan_to_plane( let input = state.band(store.input_band_id, store.input_rect)?; let mut output = vec![0.0f32; store.output_width as usize * store.output_height as usize]; - let job = signinum_j2k_native::J2kStoreComponentJob { + let job = j2k_native::J2kStoreComponentJob { input, input_width: store.input_rect.width(), source_x: store.source_x, @@ -157,7 +157,7 @@ fn decode_classic_sub_band(plan: &J2kOwnedSubBandPlan, output: &mut [f32]) -> Re }) .collect(); compute::decode_classic_cleanup_sub_band( - signinum_j2k_native::J2kSubBandDecodeJob { + j2k_native::J2kSubBandDecodeJob { width: plan.width, height: plan.height, jobs: &jobs, @@ -184,7 +184,7 @@ fn decode_ht_sub_band(plan: &HtOwnedSubBandPlan, output: &mut [f32]) -> Result<( }) .collect(); compute::decode_ht_cleanup_sub_band( - signinum_j2k_native::HtSubBandDecodeJob { + j2k_native::HtSubBandDecodeJob { width: plan.width, height: plan.height, jobs: &jobs, @@ -205,6 +205,7 @@ fn classic_job(owned: &J2kOwnedCodeBlockBatchJob) -> J2kCodeBlockDecodeJob<'_> { missing_bit_planes: owned.missing_bit_planes, number_of_coding_passes: owned.number_of_coding_passes, total_bitplanes: owned.total_bitplanes, + roi_shift: owned.roi_shift, sub_band_type: owned.sub_band_type, style: owned.style, strict: owned.strict, @@ -225,6 +226,7 @@ fn ht_job(owned: &HtOwnedCodeBlockBatchJob) -> HtCodeBlockDecodeJob<'_> { missing_bit_planes: owned.missing_bit_planes, number_of_coding_passes: owned.number_of_coding_passes, num_bitplanes: owned.num_bitplanes, + roi_shift: owned.roi_shift, stripe_causal: owned.stripe_causal, strict: owned.strict, dequantization_step: owned.dequantization_step, @@ -246,12 +248,12 @@ mod tests { decode_classic_sub_band, decode_ht_sub_band, execute_grayscale_plan_to_plane, DirectExecutionState, }; - use signinum_j2k_native::{ + use j2k_native::{ encode, encode_htj2k, DecodeSettings, DecoderContext, EncodeOptions, HtCodeBlockDecoder, Image, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kSingleDecompositionIdwtJob, }; - fn classic_plan() -> signinum_j2k_native::J2kDirectGrayscalePlan { + fn classic_plan() -> j2k_native::J2kDirectGrayscalePlan { let pixels: Vec = (0..16).collect(); let options = EncodeOptions { reversible: true, @@ -266,7 +268,7 @@ mod tests { .expect("direct plan") } - fn ht_plan() -> signinum_j2k_native::J2kDirectGrayscalePlan { + fn ht_plan() -> j2k_native::J2kDirectGrayscalePlan { let pixels: Vec = (0..16).collect(); let options = EncodeOptions { reversible: true, @@ -305,35 +307,35 @@ mod tests { let hh = state.band(idwt.hh_band_id, idwt.hh).expect("hh"); let mut output = vec![0.0f32; idwt.rect.width() as usize * idwt.rect.height() as usize]; - let job = signinum_j2k_native::J2kSingleDecompositionIdwtJob { + let job = j2k_native::J2kSingleDecompositionIdwtJob { rect: idwt.rect, transform: idwt.transform, - ll: signinum_j2k_native::J2kIdwtBand { + ll: j2k_native::J2kIdwtBand { rect: idwt.ll, coefficients: ll, }, - hl: signinum_j2k_native::J2kIdwtBand { + hl: j2k_native::J2kIdwtBand { rect: idwt.hl, coefficients: hl, }, - lh: signinum_j2k_native::J2kIdwtBand { + lh: j2k_native::J2kIdwtBand { rect: idwt.lh, coefficients: lh, }, - hh: signinum_j2k_native::J2kIdwtBand { + hh: j2k_native::J2kIdwtBand { rect: idwt.hh, coefficients: hh, }, }; match idwt.transform { - signinum_j2k_native::J2kWaveletTransform::Reversible53 => { + j2k_native::J2kWaveletTransform::Reversible53 => { crate::compute::decode_reversible53_single_decomposition_idwt( job, &mut output, ) .expect("53 idwt"); } - signinum_j2k_native::J2kWaveletTransform::Irreversible97 => { + j2k_native::J2kWaveletTransform::Irreversible97 => { crate::compute::decode_irreversible97_single_decomposition_idwt( job, &mut output, @@ -356,7 +358,7 @@ mod tests { fn execute_to_first_idwt_inputs( plan: &J2kDirectGrayscalePlan, ) -> ( - signinum_j2k_native::J2kDirectIdwtStep, + j2k_native::J2kDirectIdwtStep, Vec, Vec, Vec, @@ -394,11 +396,11 @@ mod tests { #[derive(Default)] struct CaptureIdwtJob { - rect: Option, - ll_rect: Option, - hl_rect: Option, - lh_rect: Option, - hh_rect: Option, + rect: Option, + ll_rect: Option, + hl_rect: Option, + lh_rect: Option, + hh_rect: Option, ll: Vec, hl: Vec, lh: Vec, @@ -408,17 +410,17 @@ mod tests { impl HtCodeBlockDecoder for CaptureIdwtJob { fn decode_code_block( &mut self, - job: signinum_j2k_native::HtCodeBlockDecodeJob<'_>, + job: j2k_native::HtCodeBlockDecodeJob<'_>, output: &mut [f32], - ) -> signinum_j2k_native::Result<()> { - signinum_j2k_native::decode_ht_code_block_scalar(job, output) + ) -> j2k_native::Result<()> { + j2k_native::decode_ht_code_block_scalar(job, output) } fn decode_single_decomposition_idwt( &mut self, job: J2kSingleDecompositionIdwtJob<'_>, _output: &mut [f32], - ) -> signinum_j2k_native::Result { + ) -> j2k_native::Result { if self.rect.is_none() { self.rect = Some(job.rect); self.ll_rect = Some(job.ll.rect); diff --git a/crates/j2k-metal/src/encode.rs b/crates/j2k-metal/src/encode.rs new file mode 100644 index 00000000..6c83cf57 --- /dev/null +++ b/crates/j2k-metal/src/encode.rs @@ -0,0 +1,1874 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(any(test, target_os = "macos"))] +mod config; +mod encoded; +#[cfg(target_os = "macos")] +mod packet_plan; +#[cfg(target_os = "macos")] +mod plan; +#[cfg(target_os = "macos")] +mod resident_estimate; +#[cfg(target_os = "macos")] +mod resident_types; +mod roundtrip_validation; +mod stage_accelerator; +mod stats; +mod submitted; +#[cfg(all(test, target_os = "macos"))] +mod test_helpers; +mod types; +#[cfg(target_os = "macos")] +mod validation; + +#[cfg(target_os = "macos")] +use crate::compute; +use j2k::J2kLosslessEncodeOptions; +#[cfg(target_os = "macos")] +use j2k::{EncodeBackendPreference, J2kBlockCodingMode, J2kEncodeValidation, ReversibleTransform}; +#[cfg(target_os = "macos")] +use j2k::{EncodedJ2k, J2kLosslessSamples}; +#[cfg(target_os = "macos")] +use j2k_core::{BackendKind, DeviceSurface, PixelFormat}; +#[cfg(target_os = "macos")] +use j2k_native::{EncodeOptions, EncodeProgressionOrder}; +#[cfg(target_os = "macos")] +use j2k_native::{J2kEncodeStageAccelerator, J2kHtj2kTileEncodeJob, J2kPacketizationEncodeJob}; +#[cfg(target_os = "macos")] +use metal::Buffer; +#[cfg(target_os = "macos")] +use std::time::Duration; +#[cfg(target_os = "macos")] +use std::time::Instant; + +#[cfg(test)] +use self::config::{ + default_gpu_encode_memory_budget_bytes_for_hw_mem, resident_lossless_chunk_ranges_for_test, + resolve_lossless_encode_config_for_test, +}; +#[cfg(target_os = "macos")] +use self::config::{ + resident_lossless_chunk_ranges_from_code_blocks, resolve_lossless_encode_config, +}; +#[cfg(any(test, target_os = "macos"))] +use self::config::{ + resident_lossless_code_block_chunk_cap, resident_lossless_encode_config_for_mode, +}; +pub use self::encoded::MetalEncodedJ2k; +#[cfg(target_os = "macos")] +use self::packet_plan::{ + cpu_packetization_resolutions_from_lossless_device_plan, + lossless_options_for_resident_htj2k_tile_job, packet_descriptors_for_lossless_device_order, + packetization_progression_order, resident_packetization_resolutions_from_lossless_device_plan, + should_use_resident_htj2k_host_tile_for_auto, +}; +#[cfg(target_os = "macos")] +use self::plan::{ + lossless_device_encode_plan, LosslessDeviceEncodePlan, RESIDENT_CLASSIC_CODE_BLOCK_EDGE, +}; +#[cfg(all(test, target_os = "macos"))] +use self::resident_estimate::estimated_tier1_output_bytes; +#[cfg(target_os = "macos")] +use self::resident_estimate::{ + checked_mul_bytes, estimate_resident_lossless_encode_peak_bytes, + resident_classic_batch_encode_should_retry_conservative, + resident_codestream_assembly_job_for_metadata, + resident_ht_batch_encode_should_retry_conservative, +}; +#[cfg(target_os = "macos")] +use self::resident_types::{ + FinishedResidentLosslessBufferEncode, PlannedResidentLosslessBufferEncode, + PreparedResidentLosslessBufferEncode, ResidentLosslessBufferEncodeMetadata, + SubmittedResidentLosslessMetalBufferEncodeBatch, + SubmittedResidentLosslessMetalBufferEncodeBatchKind, + SubmittedResidentLosslessMetalBufferEncodeChunk, +}; +pub use self::roundtrip_validation::{ + validate_lossless_roundtrip_on_metal, validate_lossless_roundtrip_on_metal_with_session, +}; +#[cfg(all(test, target_os = "macos"))] +use self::stage_accelerator::metal_dispatch_option; +pub use self::stage_accelerator::MetalEncodeStageAccelerator; +#[cfg(test)] +use self::stats::add_resident_prep_duration; +#[cfg(any(test, target_os = "macos"))] +use self::stats::add_resident_prep_wall_duration; +pub use self::stats::{ + MetalLosslessBufferEncodeBatchOutcome, MetalLosslessEncodeBatchStats, + MetalLosslessEncodeStageStats, +}; +#[cfg(target_os = "macos")] +use self::submitted::{ + OwnedMetalLosslessEncodeTile, SubmittedJ2kLosslessMetalBufferEncodeBatchState, + SubmittedJ2kLosslessMetalEncodeBatchState, +}; +pub use self::submitted::{ + SubmittedJ2kLosslessMetalBufferEncodeBatch, SubmittedJ2kLosslessMetalEncodeBatch, +}; +#[cfg(all(test, target_os = "macos"))] +use self::test_helpers::{ + collect_inflight_limited_ordered, encode_lossless_from_metal_buffer, + encode_lossless_from_metal_buffer_to_metal_with_report, + encode_lossless_from_metal_buffers_to_metal_with_report, + encode_lossless_from_padded_metal_buffer_to_metal_with_report, + encode_lossless_from_padded_metal_buffer_with_report, + encode_lossless_from_padded_metal_buffers_to_metal_batch, + encode_lossless_from_padded_metal_buffers_to_metal_with_report, + encode_lossless_from_padded_metal_buffers_with_report, set_test_resident_encode_failure_index, + submit_lossless_from_metal_buffer, submit_lossless_from_padded_metal_buffer, + test_resident_encode_failure_index, +}; +pub use self::types::{ + MetalEncodeInputStaging, MetalLosslessBufferEncodeOutcome, MetalLosslessEncodeBatchRequest, + MetalLosslessEncodeConfig, MetalLosslessEncodeOutcome, MetalLosslessEncodeResidency, + MetalLosslessEncodeTile, +}; +#[cfg(target_os = "macos")] +use self::validation::{ + lossless_sample_shape, validate_metal_encode_tile, validate_padded_contiguous_metal_encode_tile, +}; + +#[cfg(target_os = "macos")] +/// Submit a lossless tile batch that resolves to host codestream bytes. +pub fn submit_lossless_batch( + request: MetalLosslessEncodeBatchRequest<'_, '_>, + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result { + submit_lossless_tiles( + request.tiles, + *options, + session, + request.staging, + request.config, + ) +} + +#[cfg(target_os = "macos")] +/// Submit a lossless tile batch that resolves to Metal-backed codestreams. +pub fn submit_lossless_batch_to_metal( + request: MetalLosslessEncodeBatchRequest<'_, '_>, + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result { + submit_lossless_tiles_to_metal_buffer_batch( + request.tiles, + *options, + session, + request.staging, + request.config, + ) +} + +#[cfg(target_os = "macos")] +/// Encode a lossless tile batch and return host-byte timing reports. +pub fn encode_lossless_batch_with_report( + request: MetalLosslessEncodeBatchRequest<'_, '_>, + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result, crate::Error> { + encode_lossless_tiles_with_report( + request.tiles, + *options, + session, + request.staging, + request.config, + ) +} + +#[cfg(target_os = "macos")] +fn host_outcome_from_buffer_outcome( + outcome: MetalLosslessBufferEncodeOutcome, +) -> Result { + let (encoded, host_readback_duration) = + outcome.encoded.to_encoded_j2k_with_readback_duration()?; + Ok(MetalLosslessEncodeOutcome { + encoded, + input_copy_used: outcome.input_copy_used, + resident: outcome.resident, + input_copy_duration: outcome.input_copy_duration, + encode_duration: outcome.encode_duration, + gpu_duration: outcome.gpu_duration, + validation_duration: outcome.validation_duration, + host_readback_duration, + }) +} + +#[cfg(target_os = "macos")] +fn encode_lossless_tiles_with_report( + tiles: &[MetalLosslessEncodeTile<'_>], + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, + config: MetalLosslessEncodeConfig, +) -> Result, crate::Error> { + if should_try_resident_lossless_host_encode(options) { + let batch = try_encode_resident_lossless_tiles_to_metal_buffer_batch( + tiles, options, session, staging, config, + )?; + if let Some(outcomes) = batch { + return outcomes + .outcomes + .into_iter() + .map(host_outcome_from_buffer_outcome) + .collect(); + } + } + + let mut accelerator = MetalEncodeStageAccelerator::for_host_output(options); + tiles + .iter() + .map(|&tile| { + encode_lossless_tile_with_report(tile, options, session, staging, &mut accelerator) + }) + .collect() +} + +#[cfg(target_os = "macos")] +fn encode_lossless_owned_tiles_with_report( + tiles: &[OwnedMetalLosslessEncodeTile], + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, + config: MetalLosslessEncodeConfig, +) -> Result, crate::Error> { + let borrowed = tiles + .iter() + .map(OwnedMetalLosslessEncodeTile::as_tile) + .collect::>(); + if should_try_resident_lossless_host_encode(options) { + let batch = try_encode_resident_lossless_tiles_to_metal_buffer_batch( + &borrowed, options, session, staging, config, + )?; + if let Some(outcomes) = batch { + return outcomes + .outcomes + .into_iter() + .map(host_outcome_from_buffer_outcome) + .collect(); + } + } + + let mut accelerator = MetalEncodeStageAccelerator::for_host_output(options); + borrowed + .iter() + .map(|&tile| { + encode_lossless_tile_with_report(tile, options, session, staging, &mut accelerator) + }) + .collect() +} + +#[cfg(target_os = "macos")] +fn submit_lossless_tiles_to_metal_buffer_batch( + tiles: &[MetalLosslessEncodeTile<'_>], + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, + config: MetalLosslessEncodeConfig, +) -> Result { + if options.backend != EncodeBackendPreference::CpuOnly { + if let Some(submitted) = try_submit_resident_lossless_tiles_to_metal_buffer_batch( + tiles, options, session, staging, config, + )? { + return Ok(SubmittedJ2kLosslessMetalBufferEncodeBatch { + state: SubmittedJ2kLosslessMetalBufferEncodeBatchState::Resident(Box::new( + submitted, + )), + }); + } + } + + let mut owned = Vec::with_capacity(tiles.len()); + for &tile in tiles { + validate_metal_encode_tile(tile)?; + if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { + lossless_sample_shape(tile.format)?; + validate_padded_contiguous_metal_encode_tile(tile, tile.format.bytes_per_pixel())?; + } + owned.push(OwnedMetalLosslessEncodeTile::from_tile(tile)); + } + Ok(SubmittedJ2kLosslessMetalBufferEncodeBatch { + state: SubmittedJ2kLosslessMetalBufferEncodeBatchState::Deferred { + tiles: owned, + options, + session: session.clone(), + staging, + }, + }) +} + +#[cfg(target_os = "macos")] +fn encode_owned_lossless_tiles_to_metal_buffer_fallback_batch( + tiles: &[OwnedMetalLosslessEncodeTile], + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, +) -> Result { + let mut outcomes = Vec::with_capacity(tiles.len()); + for tile in tiles { + outcomes.push(encode_lossless_tile_to_metal_buffer_with_report( + tile.as_tile(), + options, + session, + staging, + )?); + } + Ok(MetalLosslessBufferEncodeBatchOutcome { + outcomes, + stats: MetalLosslessEncodeBatchStats::default(), + }) +} + +#[cfg(target_os = "macos")] +fn try_submit_resident_lossless_tiles_to_metal_buffer_batch( + tiles: &[MetalLosslessEncodeTile<'_>], + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, + config: MetalLosslessEncodeConfig, +) -> Result, crate::Error> { + let profile_stages = compute::metal_profile_stages_enabled(); + if tiles.is_empty() { + return Ok(Some(SubmittedResidentLosslessMetalBufferEncodeBatch { + options, + session: session.clone(), + stats: resolve_lossless_encode_config(0, 1, config)?, + encode_started: Instant::now(), + tiles: Vec::new(), + staging, + kind: SubmittedResidentLosslessMetalBufferEncodeBatchKind::Empty, + })); + } + + let plan_started = profile_stages.then(Instant::now); + let mut planned = Vec::with_capacity(tiles.len()); + for (index, &tile) in tiles.iter().enumerate() { + let Some(item) = plan_resident_lossless_buffer_encode(index, tile, options, staging)? + else { + return Ok(None); + }; + planned.push(item); + } + let estimated_peak_bytes_per_tile = planned + .iter() + .map(PlannedResidentLosslessBufferEncode::estimated_peak_bytes) + .max() + .unwrap_or(1); + let classic_resident_mode = planned + .iter() + .all(|planned| planned.metadata.plan.block_coding_mode == J2kBlockCodingMode::Classic); + let ht_resident_mode = planned.iter().all(|planned| { + planned.metadata.plan.block_coding_mode == J2kBlockCodingMode::HighThroughput + }); + if !(classic_resident_mode || ht_resident_mode) { + return Ok(None); + } + let resolved_config = + resident_lossless_encode_config_for_mode(config, classic_resident_mode, tiles.len()); + let mut stats = resolve_lossless_encode_config( + tiles.len(), + estimated_peak_bytes_per_tile, + resolved_config, + )?; + if let Some(started) = plan_started { + stats.stage_stats.plan_duration = started.elapsed(); + } + let encode_started = Instant::now(); + let kind = submit_planned_resident_lossless_tiles( + planned, + session, + stats.effective_inflight_tiles, + &mut stats, + )?; + let tiles = tiles + .iter() + .map(|&tile| OwnedMetalLosslessEncodeTile::from_tile(tile)) + .collect(); + Ok(Some(SubmittedResidentLosslessMetalBufferEncodeBatch { + options, + session: session.clone(), + stats, + encode_started, + tiles, + staging, + kind, + })) +} + +#[cfg(target_os = "macos")] +fn try_encode_resident_lossless_tiles_to_metal_buffer_batch( + tiles: &[MetalLosslessEncodeTile<'_>], + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, + config: MetalLosslessEncodeConfig, +) -> Result, crate::Error> { + let Some(submitted) = try_submit_resident_lossless_tiles_to_metal_buffer_batch( + tiles, options, session, staging, config, + )? + else { + return Ok(None); + }; + wait_submitted_resident_lossless_buffer_encode_batch(submitted).map(Some) +} + +#[cfg(target_os = "macos")] +fn submit_lossless_tiles( + tiles: &[MetalLosslessEncodeTile<'_>], + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, + config: MetalLosslessEncodeConfig, +) -> Result { + if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) + && should_try_resident_lossless_host_encode(options) + { + let mut ready = Vec::with_capacity(tiles.len()); + let mut all_ready = true; + for &tile in tiles { + validate_metal_encode_tile(tile)?; + lossless_sample_shape(tile.format)?; + validate_padded_contiguous_metal_encode_tile(tile, tile.format.bytes_per_pixel())?; + if let Some(outcome) = try_encode_lossless_tile_device_resident_with_report( + tile, options, session, staging, + )? { + ready.push(outcome.encoded); + } else { + all_ready = false; + break; + } + } + if all_ready { + return Ok(SubmittedJ2kLosslessMetalEncodeBatch { + state: SubmittedJ2kLosslessMetalEncodeBatchState::Ready(ready), + }); + } + if options.backend == EncodeBackendPreference::RequireDevice { + return Err(crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal resident encode requires classic padded contiguous Gray/RGB lossless input with at most one DWT level", + }); + } + } + + let mut owned = Vec::with_capacity(tiles.len()); + for &tile in tiles { + validate_metal_encode_tile(tile)?; + if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { + lossless_sample_shape(tile.format)?; + validate_padded_contiguous_metal_encode_tile(tile, tile.format.bytes_per_pixel())?; + } + owned.push(OwnedMetalLosslessEncodeTile::from_tile(tile)); + } + Ok(SubmittedJ2kLosslessMetalEncodeBatch { + state: SubmittedJ2kLosslessMetalEncodeBatchState::Deferred { + tiles: owned, + options, + session: session.clone(), + staging, + config, + }, + }) +} + +#[cfg(target_os = "macos")] +fn should_try_resident_lossless_host_encode(options: J2kLosslessEncodeOptions) -> bool { + options.backend == EncodeBackendPreference::RequireDevice +} + +#[cfg(target_os = "macos")] +fn host_output_encode_options(mut options: J2kLosslessEncodeOptions) -> J2kLosslessEncodeOptions { + options.validation = J2kEncodeValidation::External; + options +} + +#[cfg(target_os = "macos")] +fn borrow_padded_metal_buffer_from_bytes( + session: &crate::MetalBackendSession, + bytes: &[u8], +) -> Result { + if bytes.is_empty() { + return Err(crate::Error::MetalKernel { + message: "J2K Metal hybrid encode input is empty".to_string(), + }); + } + Ok(session.device().new_buffer_with_bytes_no_copy( + bytes.as_ptr().cast(), + bytes.len() as u64, + metal::MTLResourceOptions::StorageModeShared, + None, + )) +} + +#[cfg(target_os = "macos")] +struct ResidentHybridHtTileBody { + tile_data: Vec, + components: u8, + bit_depth: u8, + bytes_per_pixel: usize, + code_block_count: usize, + code_block_width_exp: u8, + code_block_height_exp: u8, + num_decomposition_levels: u8, + used_fused_rct: bool, + guard_bits: u8, + progression_order: EncodeProgressionOrder, + write_tlm: bool, + forward_dwt53_dispatches: usize, + ht_code_block_dispatches: usize, +} + +#[cfg(target_os = "macos")] +fn encode_resident_ht_tile_body_with_cpu_packetization( + tile: MetalLosslessEncodeTile<'_>, + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, + code_block_width: u32, + code_block_height: u32, +) -> Result, crate::Error> { + if !should_try_resident_lossless_ht_cpu_packetization(tile, options, staging) { + return Ok(None); + } + validate_metal_encode_tile(tile)?; + let (components, bit_depth) = lossless_sample_shape(tile.format)?; + let bytes_per_pixel = tile.format.bytes_per_pixel(); + let bytes_per_sample = + u8::try_from(tile.format.bytes_per_sample()).map_err(|_| crate::Error::MetalKernel { + message: "J2K Metal resident hybrid bytes per sample exceeds u8".to_string(), + })?; + validate_padded_contiguous_metal_encode_tile(tile, bytes_per_pixel)?; + let Some(plan) = lossless_device_encode_plan( + tile.output_width, + tile.output_height, + components, + bit_depth, + options, + code_block_width, + code_block_height, + )? + else { + return Ok(None); + }; + if plan.block_coding_mode != J2kBlockCodingMode::HighThroughput { + return Ok(None); + } + + let coefficient_count = lossless_device_coefficient_count(&plan.code_blocks)?; + let prepared = compute::prepare_lossless_device_code_blocks( + session, + compute::J2kLosslessDevicePrepareJob { + input: tile.buffer, + input_byte_offset: tile.byte_offset, + input_width: tile.width, + input_height: tile.height, + input_pitch_bytes: tile.pitch_bytes, + output_width: tile.output_width, + output_height: tile.output_height, + components, + bytes_per_sample, + bit_depth, + num_decomposition_levels: plan.num_decomposition_levels, + coefficient_count, + }, + plan.code_blocks.clone(), + )?; + let resident_tier1 = + compute::encode_ht_prepared_device_code_blocks_resident(session, prepared)?; + let encoded_blocks = compute::read_resident_ht_tier1_code_blocks_for_cpu_packetization( + session, + &resident_tier1, + )?; + let packetization_resolutions = + cpu_packetization_resolutions_from_lossless_device_plan(&plan, &encoded_blocks)?; + let packet_descriptors = packet_descriptors_for_lossless_device_order( + plan.resolutions.len(), + plan.components, + plan.progression_order, + )?; + let packetization_job = J2kPacketizationEncodeJob { + resolution_count: u32::try_from(plan.resolutions.len()).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal resident hybrid resolution count exceeds u32".to_string(), + } + })?, + num_layers: 1, + num_components: plan.components, + code_block_count: u32::try_from(plan.code_blocks.len()).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal resident hybrid code-block count exceeds u32".to_string(), + } + })?, + progression_order: packetization_progression_order(plan.progression_order), + packet_descriptors: &packet_descriptors, + resolutions: &packetization_resolutions, + }; + let tile_data = + j2k_native::encode_j2k_packetization_scalar(packetization_job).map_err(|reason| { + crate::Error::MetalKernel { + message: format!("J2K Metal resident hybrid CPU packetization failed: {reason}"), + } + })?; + + Ok(Some(ResidentHybridHtTileBody { + tile_data, + components, + bit_depth, + bytes_per_pixel, + code_block_count: plan.code_blocks.len(), + code_block_width_exp: plan.code_block_width_exp, + code_block_height_exp: plan.code_block_height_exp, + num_decomposition_levels: plan.num_decomposition_levels, + used_fused_rct: plan.use_mct && tile.format == PixelFormat::Rgb8, + guard_bits: plan.guard_bits, + progression_order: plan.progression_order, + write_tlm: plan.write_tlm, + forward_dwt53_dispatches: if plan.num_decomposition_levels > 0 { + usize::from(plan.components) + } else { + 0 + }, + ht_code_block_dispatches: usize::from(!plan.code_blocks.is_empty()), + })) +} + +#[cfg(target_os = "macos")] +#[derive(Debug, Default)] +struct PrepacketizedHtj2kTileAccelerator { + tile_data: Option>, +} + +#[cfg(target_os = "macos")] +impl J2kEncodeStageAccelerator for PrepacketizedHtj2kTileAccelerator { + fn encode_htj2k_tile( + &mut self, + _job: J2kHtj2kTileEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + Ok(self.tile_data.take()) + } +} + +#[cfg(target_os = "macos")] +fn lossless_device_coefficient_count( + code_blocks: &[compute::J2kLosslessDeviceCodeBlock], +) -> Result { + let mut count = 0usize; + for block in code_blocks { + let offset = + usize::try_from(block.coefficient_offset).map_err(|_| crate::Error::MetalKernel { + message: "J2K Metal resident encode coefficient offset exceeds usize".to_string(), + })?; + let block_count = (block.width as usize) + .checked_mul(block.height as usize) + .ok_or_else(|| crate::Error::MetalKernel { + message: "J2K Metal resident encode coefficient count overflow".to_string(), + })?; + count = count.max(offset.checked_add(block_count).ok_or_else(|| { + crate::Error::MetalKernel { + message: "J2K Metal resident encode coefficient count overflow".to_string(), + } + })?); + } + Ok(count) +} + +#[cfg(target_os = "macos")] +fn plan_resident_lossless_buffer_encode( + index: usize, + tile: MetalLosslessEncodeTile<'_>, + options: J2kLosslessEncodeOptions, + staging: MetalEncodeInputStaging, +) -> Result, crate::Error> { + validate_metal_encode_tile(tile)?; + if options.backend == EncodeBackendPreference::CpuOnly { + return Ok(None); + } + let (components, bit_depth) = lossless_sample_shape(tile.format)?; + let bytes_per_pixel = tile.format.bytes_per_pixel(); + let bytes_per_sample = + u8::try_from(tile.format.bytes_per_sample()).map_err(|_| crate::Error::MetalKernel { + message: "J2K Metal resident encode bytes per sample exceeds u8".to_string(), + })?; + if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { + validate_padded_contiguous_metal_encode_tile(tile, bytes_per_pixel)?; + } + let Some(plan) = lossless_device_encode_plan( + tile.output_width, + tile.output_height, + components, + bit_depth, + options, + RESIDENT_CLASSIC_CODE_BLOCK_EDGE, + RESIDENT_CLASSIC_CODE_BLOCK_EDGE, + )? + else { + return Ok(None); + }; + let coefficient_count = lossless_device_coefficient_count(&plan.code_blocks)?; + let packetization_resolutions = + resident_packetization_resolutions_from_lossless_device_plan(&plan)?; + let packet_descriptors = packet_descriptors_for_lossless_device_order( + plan.resolutions.len(), + plan.components, + plan.progression_order, + )?; + let metadata = ResidentLosslessBufferEncodeMetadata { + tile: OwnedMetalLosslessEncodeTile::from_tile(tile), + components, + bit_depth, + bytes_per_pixel, + plan, + packet_descriptors, + packetization_resolutions, + }; + let estimated_peak_bytes = + estimate_resident_lossless_encode_peak_bytes(&metadata, coefficient_count, staging); + Ok(Some(PlannedResidentLosslessBufferEncode { + index, + metadata, + coefficient_count, + bytes_per_sample, + estimated_peak_bytes, + #[cfg(test)] + failure_injection_index: test_resident_encode_failure_index(), + })) +} + +#[cfg(target_os = "macos")] +fn wait_submitted_resident_lossless_buffer_encode_batch( + mut submitted: SubmittedResidentLosslessMetalBufferEncodeBatch, +) -> Result { + let result = wait_submitted_resident_lossless_buffer_encode_batch_once(&mut submitted); + match result { + Ok(outcome) => Ok(outcome), + Err(err) => { + if submitted.options.block_coding_mode == J2kBlockCodingMode::Classic + && !submitted.tiles.is_empty() + && resident_classic_batch_encode_should_retry_conservative(&err) + { + return encode_owned_lossless_tiles_to_metal_buffer_fallback_batch( + &submitted.tiles, + submitted.options, + &submitted.session, + submitted.staging, + ) + .map_err(|retry_err| crate::Error::MetalKernel { + message: format!( + "J2K Metal resident classic batch conservative retry failed after tight resident capacity failure ({err}); retry error: {retry_err}" + ), + }); + } + if submitted.options.block_coding_mode == J2kBlockCodingMode::HighThroughput + && !submitted.tiles.is_empty() + && resident_ht_batch_encode_should_retry_conservative(&err) + { + return encode_owned_lossless_tiles_to_metal_buffer_fallback_batch( + &submitted.tiles, + submitted.options, + &submitted.session, + submitted.staging, + ) + .map_err(|retry_err| crate::Error::MetalKernel { + message: format!( + "J2K Metal resident HT batch conservative retry failed after tight packet capacity failure ({err}); retry error: {retry_err}" + ), + }); + } + Err(err) + } + } +} + +#[cfg(target_os = "macos")] +fn wait_submitted_resident_lossless_buffer_encode_batch_once( + submitted: &mut SubmittedResidentLosslessMetalBufferEncodeBatch, +) -> Result { + let mut outcomes = Vec::new(); + match std::mem::replace( + &mut submitted.kind, + SubmittedResidentLosslessMetalBufferEncodeBatchKind::Empty, + ) { + SubmittedResidentLosslessMetalBufferEncodeBatchKind::Empty => {} + SubmittedResidentLosslessMetalBufferEncodeBatchKind::Chunks(chunks) => { + outcomes.reserve(chunks.iter().map(|chunk| chunk.metadatas.len()).sum()); + if submitted.options.validation == J2kEncodeValidation::External + && submitted.options.block_coding_mode == J2kBlockCodingMode::HighThroughput + && chunks.len() > 1 + { + let wait_started = compute::metal_profile_stages_enabled().then(Instant::now); + let mut chunk_metadatas = Vec::with_capacity(chunks.len()); + let mut pending_batches = Vec::with_capacity(chunks.len()); + for chunk in chunks { + chunk_metadatas.push(( + chunk.metadatas, + chunk.prepare_durations, + chunk.batch_started, + )); + pending_batches.push(chunk.pending); + } + let batches = compute::wait_resident_lossless_codestream_batches(pending_batches)?; + if let Some(started) = wait_started { + let elapsed = started.elapsed(); + submitted.stats.stage_stats.codestream_wait_duration = submitted + .stats + .stage_stats + .codestream_wait_duration + .saturating_add(elapsed); + submitted.stats.stage_stats.sync_wait_duration = submitted + .stats + .stage_stats + .sync_wait_duration + .saturating_add(elapsed); + } + for ((metadatas, prepare_durations, batch_started), batch) in + chunk_metadatas.into_iter().zip(batches) + { + if compute::metal_profile_stages_enabled() { + submitted + .stats + .stage_stats + .add_assign(MetalLosslessEncodeStageStats::from(batch.stage_stats)); + } + let codestreams = batch.codestreams; + let batch_duration = duration_share(batch_started.elapsed(), codestreams.len()); + for ((metadata, prepare_duration), codestream) in metadatas + .into_iter() + .zip(prepare_durations) + .zip(codestreams) + { + let finished = finished_resident_lossless_buffer_encode( + metadata, + codestream, + prepare_duration.saturating_add(batch_duration), + ); + outcomes.push(validate_finished_resident_lossless_buffer_encode( + finished, + submitted.options, + &submitted.session, + )?); + } + } + } else { + for chunk in chunks { + let wait_started = compute::metal_profile_stages_enabled().then(Instant::now); + let batch = compute::wait_resident_lossless_codestream_batch(chunk.pending)?; + if let Some(started) = wait_started { + let elapsed = started.elapsed(); + submitted.stats.stage_stats.codestream_wait_duration = submitted + .stats + .stage_stats + .codestream_wait_duration + .saturating_add(elapsed); + submitted.stats.stage_stats.sync_wait_duration = submitted + .stats + .stage_stats + .sync_wait_duration + .saturating_add(elapsed); + submitted + .stats + .stage_stats + .add_assign(MetalLosslessEncodeStageStats::from(batch.stage_stats)); + } + let codestreams = batch.codestreams; + let batch_duration = + duration_share(chunk.batch_started.elapsed(), codestreams.len()); + for ((metadata, prepare_duration), codestream) in chunk + .metadatas + .into_iter() + .zip(chunk.prepare_durations) + .zip(codestreams) + { + let finished = finished_resident_lossless_buffer_encode( + metadata, + codestream, + prepare_duration.saturating_add(batch_duration), + ); + outcomes.push(validate_finished_resident_lossless_buffer_encode( + finished, + submitted.options, + &submitted.session, + )?); + } + } + } + } + } + submitted.stats.encode_wall_duration = submitted.encode_started.elapsed(); + Ok(MetalLosslessBufferEncodeBatchOutcome { + outcomes, + stats: submitted.stats, + }) +} + +#[cfg(target_os = "macos")] +fn finished_resident_lossless_buffer_encode( + metadata: ResidentLosslessBufferEncodeMetadata, + codestream: compute::J2kResidentLosslessCodestream, + encode_duration: Duration, +) -> FinishedResidentLosslessBufferEncode { + let encoded = MetalEncodedJ2k { + codestream_buffer: codestream.buffer, + byte_offset: codestream.byte_offset, + byte_len: codestream.byte_len, + capacity: codestream.capacity, + width: metadata.tile.output_width, + height: metadata.tile.output_height, + components: metadata.components, + bit_depth: metadata.bit_depth, + signed: false, + }; + + FinishedResidentLosslessBufferEncode { + metadata, + encoded, + encode_duration, + gpu_duration: codestream.gpu_duration, + } +} + +#[cfg(target_os = "macos")] +fn validate_finished_resident_lossless_buffer_encode( + finished: FinishedResidentLosslessBufferEncode, + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result { + let FinishedResidentLosslessBufferEncode { + metadata, + encoded, + encode_duration, + gpu_duration, + } = finished; + + let validation_duration = if options.validation == J2kEncodeValidation::CpuRoundTrip { + let validation_started = Instant::now(); + let tile = metadata.tile.as_tile(); + if tile.width == tile.output_width + && tile.height == tile.output_height + && tile.pitch_bytes == tile.output_width as usize * metadata.bytes_per_pixel + { + validate_lossless_roundtrip_on_metal_tile_with_session( + tile, + encoded.codestream_bytes()?, + session, + )?; + } else { + validate_lossless_roundtrip_on_metal_region_with_session( + tile, + tile.output_width, + tile.output_height, + metadata.bytes_per_pixel, + encoded.codestream_bytes()?, + session, + )?; + } + validation_started.elapsed() + } else { + Duration::ZERO + }; + + Ok(MetalLosslessBufferEncodeOutcome { + encoded, + input_copy_used: false, + resident: MetalLosslessEncodeResidency { + coefficient_prep_used: true, + packetization_used: true, + codestream_assembly_used: true, + }, + input_copy_duration: Duration::ZERO, + encode_duration, + gpu_duration, + validation_duration, + }) +} + +#[cfg(target_os = "macos")] +fn submit_planned_resident_lossless_tiles( + planned: Vec, + session: &crate::MetalBackendSession, + inflight_tiles: usize, + stats: &mut MetalLosslessEncodeBatchStats, +) -> Result { + if planned.is_empty() { + return Ok(SubmittedResidentLosslessMetalBufferEncodeBatchKind::Empty); + } + if planned.iter().all(|planned| { + planned.metadata.plan.block_coding_mode == J2kBlockCodingMode::HighThroughput + }) { + return submit_planned_resident_ht_lossless_tiles_batch( + planned, + session, + inflight_tiles, + stats, + ); + } + if planned + .iter() + .all(|planned| planned.metadata.plan.block_coding_mode == J2kBlockCodingMode::Classic) + { + return submit_planned_resident_classic_lossless_tiles_batch( + planned, + session, + inflight_tiles, + stats, + ); + } + Ok(SubmittedResidentLosslessMetalBufferEncodeBatchKind::Empty) +} + +#[cfg(target_os = "macos")] +struct PreparedResidentLosslessBatchItem { + prepared: PreparedResidentLosslessBufferEncode, + prepare_duration: Duration, +} + +#[cfg(target_os = "macos")] +fn prepare_planned_resident_lossless_tiles_batch( + planned: Vec, + session: &crate::MetalBackendSession, +) -> Result, crate::Error> { + struct BatchPlanInfo { + index: usize, + coefficient_count: usize, + bytes_per_sample: u8, + code_blocks: Vec, + } + + let started = Instant::now(); + let mut metadatas = Vec::with_capacity(planned.len()); + let mut plan_infos = Vec::with_capacity(planned.len()); + for planned in planned { + #[cfg(test)] + if planned.failure_injection_index == Some(planned.index) { + return Err(crate::Error::MetalKernel { + message: format!( + "injected J2K Metal resident encode failure at tile {}", + planned.index + ), + }); + } + + plan_infos.push(BatchPlanInfo { + index: planned.index, + coefficient_count: planned.coefficient_count, + bytes_per_sample: planned.bytes_per_sample, + code_blocks: planned.metadata.plan.code_blocks.clone(), + }); + metadatas.push(planned.metadata); + } + + let mut batch_items = Vec::with_capacity(metadatas.len()); + for (metadata, plan_info) in metadatas.iter().zip(plan_infos) { + let tile = metadata.tile.as_tile(); + batch_items.push(compute::J2kLosslessDeviceBatchPrepareItem { + tile_index: plan_info.index, + job: compute::J2kLosslessDevicePrepareJob { + input: tile.buffer, + input_byte_offset: tile.byte_offset, + input_width: tile.width, + input_height: tile.height, + input_pitch_bytes: tile.pitch_bytes, + output_width: tile.output_width, + output_height: tile.output_height, + components: metadata.components, + bytes_per_sample: plan_info.bytes_per_sample, + bit_depth: metadata.bit_depth, + num_decomposition_levels: metadata.plan.num_decomposition_levels, + coefficient_count: plan_info.coefficient_count, + }, + code_blocks: plan_info.code_blocks, + }); + } + + let prepared = compute::prepare_lossless_device_code_blocks_batch(session, batch_items)?; + let prepare_duration = duration_share(started.elapsed(), prepared.len()); + Ok(metadatas + .into_iter() + .zip(prepared) + .map(|(metadata, prepared)| PreparedResidentLosslessBatchItem { + prepared: PreparedResidentLosslessBufferEncode { metadata, prepared }, + prepare_duration, + }) + .collect()) +} + +#[cfg(target_os = "macos")] +fn submit_planned_resident_ht_lossless_tiles_batch( + planned: Vec, + session: &crate::MetalBackendSession, + inflight_tiles: usize, + stats: &mut MetalLosslessEncodeBatchStats, +) -> Result { + let code_block_counts = planned + .iter() + .map(|planned| planned.metadata.plan.code_blocks.len()) + .collect::>(); + let chunk_ranges = resident_lossless_chunk_ranges_from_code_blocks( + &code_block_counts, + inflight_tiles, + resident_lossless_code_block_chunk_cap(&code_block_counts), + ); + submit_planned_resident_lossless_tiles_chunked( + planned, + session, + stats, + "HT", + chunk_ranges, + true, + |session, batch_items| { + compute::submit_lossless_codestream_buffers_from_prepared_ht_batch( + session, + batch_items, + compute::ht_packet_output_capacity_mode_from_env(), + ) + }, + ) +} + +#[cfg(target_os = "macos")] +fn submit_planned_resident_classic_lossless_tiles_batch( + planned: Vec, + session: &crate::MetalBackendSession, + inflight_tiles: usize, + stats: &mut MetalLosslessEncodeBatchStats, +) -> Result { + let batch_limit = inflight_tiles.max(1); + let chunk_ranges = (0..planned.len()) + .step_by(batch_limit) + .map(|start| start..(start + batch_limit).min(planned.len())) + .collect::>(); + submit_planned_resident_lossless_tiles_chunked( + planned, + session, + stats, + "classic", + chunk_ranges, + false, + |session, batch_items| { + compute::submit_lossless_codestream_buffers_from_prepared_classic_batch( + session, + batch_items, + compute::J2kClassicEncodeOutputCapacityMode::Tight, + ) + }, + ) +} + +/// Shared chunked submit driver for the per-family resident lossless batch +/// paths. `time_prepare_in_submit` preserves each family's historical +/// prepare_submit_duration semantics: HT (true) measures prepare + item +/// build + submit, classic (false) measures only the submit call. +#[cfg(target_os = "macos")] +fn submit_planned_resident_lossless_tiles_chunked( + mut planned: Vec, + session: &crate::MetalBackendSession, + stats: &mut MetalLosslessEncodeBatchStats, + family_name: &str, + chunk_ranges: Vec>, + time_prepare_in_submit: bool, + submit_chunk: impl Fn( + &crate::MetalBackendSession, + Vec, + ) + -> Result, +) -> Result { + let planned_len = planned.len(); + let profile_stages = compute::metal_profile_stages_enabled(); + if profile_stages { + stats.stage_stats.chunk_count = stats + .stage_stats + .chunk_count + .saturating_add(chunk_ranges.len()); + stats.stage_stats.tile_count = stats.stage_stats.tile_count.saturating_add(planned_len); + } + stats.max_observed_inflight_tiles = stats.max_observed_inflight_tiles.max( + chunk_ranges + .iter() + .map(std::ops::Range::len) + .max() + .unwrap_or(0), + ); + + let mut chunks = Vec::with_capacity(chunk_ranges.len()); + for range in chunk_ranges { + let take = range.len(); + let chunk_planned = planned.drain(..take).collect::>(); + let early_prepare_submit_started = + (profile_stages && time_prepare_in_submit).then(Instant::now); + let prep_wall_started = profile_stages.then(Instant::now); + let prepared = prepare_planned_resident_lossless_tiles_batch(chunk_planned, session) + .map_err(|err| crate::Error::MetalKernel { + message: format!("J2K Metal resident {family_name} batch encode failed: {err}"), + })?; + if let Some(started) = prep_wall_started { + add_resident_prep_wall_duration(stats, started.elapsed(), profile_stages); + } + + let mut metadatas = Vec::with_capacity(prepared.len()); + let mut prepare_durations = Vec::with_capacity(prepared.len()); + let mut batch_items = Vec::with_capacity(prepared.len()); + for item in prepared { + let PreparedResidentLosslessBatchItem { + prepared, + prepare_duration, + } = item; + let metadata = prepared.metadata; + let codestream = resident_codestream_assembly_job_for_metadata(&metadata); + batch_items.push(compute::J2kResidentBatchEncodeItem { + prepared: prepared.prepared, + resolution_count: u32::try_from(metadata.plan.resolutions.len()).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal resident encode resolution count exceeds u32" + .to_string(), + } + })?, + num_layers: 1, + num_components: metadata.plan.components, + code_block_count: u32::try_from(metadata.plan.code_blocks.len()).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal resident encode code-block count exceeds u32" + .to_string(), + } + })?, + packet_descriptors: metadata.packet_descriptors.clone(), + resolutions: metadata.packetization_resolutions.clone(), + codestream, + }); + prepare_durations.push(prepare_duration); + metadatas.push(metadata); + } + + let batch_started = Instant::now(); + let prepare_submit_started = if time_prepare_in_submit { + early_prepare_submit_started + } else { + profile_stages.then(Instant::now) + }; + let pending = submit_chunk(session, batch_items)?; + if let Some(started) = prepare_submit_started { + stats.stage_stats.prepare_submit_duration = stats + .stage_stats + .prepare_submit_duration + .saturating_add(started.elapsed()); + } + chunks.push(SubmittedResidentLosslessMetalBufferEncodeChunk { + metadatas, + prepare_durations, + pending, + batch_started, + }); + } + + if !planned.is_empty() { + return Err(crate::Error::MetalKernel { + message: format!( + "J2K Metal resident {family_name} batch chunking left unsubmitted tiles" + ), + }); + } + + if chunks.is_empty() && planned_len > 0 { + return Err(crate::Error::MetalKernel { + message: format!("J2K Metal resident {family_name} batch chunking produced no chunks"), + }); + } + + Ok(SubmittedResidentLosslessMetalBufferEncodeBatchKind::Chunks( + chunks, + )) +} + +#[cfg(target_os = "macos")] +fn duration_share(duration: Duration, count: usize) -> Duration { + if count == 0 { + return Duration::ZERO; + } + let nanos = duration.as_nanos() / count as u128; + Duration::from_nanos(nanos.min(u128::from(u64::MAX)) as u64) +} + +#[cfg(target_os = "macos")] +fn validate_lossless_roundtrip_on_metal_tile_with_session( + tile: MetalLosslessEncodeTile<'_>, + codestream: &[u8], + session: &crate::MetalBackendSession, +) -> Result<(), crate::Error> { + let mut decoder = crate::J2kDecoder::new(codestream)?; + let surface = decoder.decode_to_device_with_session(tile.format, session)?; + if surface.dimensions() != (tile.output_width, tile.output_height) { + return Err(crate::Error::MetalKernel { + message: format!( + "J2K Metal resident validation geometry mismatch: expected {}x{}, got {}x{}", + tile.output_width, + tile.output_height, + surface.dimensions().0, + surface.dimensions().1 + ), + }); + } + if surface.pixel_format() != tile.format { + return Err(crate::Error::MetalKernel { + message: format!( + "J2K Metal resident validation format mismatch: expected {:?}, got {:?}", + tile.format, + surface.pixel_format() + ), + }); + } + let expected_pitch = tile.output_width as usize * tile.format.bytes_per_pixel(); + if surface.pitch_bytes() != expected_pitch || tile.pitch_bytes != expected_pitch { + return Err(crate::Error::MetalKernel { + message: "J2K Metal resident validation requires contiguous source and decoded rows" + .to_string(), + }); + } + let byte_len = expected_pitch + .checked_mul(tile.output_height as usize) + .ok_or_else(|| crate::Error::MetalKernel { + message: "J2K Metal resident validation byte length overflow".to_string(), + })?; + let (decoded_buffer, decoded_offset) = + surface + .metal_buffer() + .ok_or(crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal resident validation decode did not return a Metal buffer", + })?; + compute::validate_metal_buffers_match( + tile.buffer, + tile.byte_offset, + decoded_buffer, + decoded_offset, + byte_len, + session, + ) +} + +#[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] +fn validate_lossless_roundtrip_on_metal_region_with_session( + source: MetalLosslessEncodeTile<'_>, + output_width: u32, + output_height: u32, + bytes_per_pixel: usize, + codestream: &[u8], + session: &crate::MetalBackendSession, +) -> Result<(), crate::Error> { + let staged_buffer = compute::copy_interleaved_padded_to_shared_buffer( + source.buffer, + source.byte_offset, + source.width, + source.height, + source.pitch_bytes, + output_width, + output_height, + bytes_per_pixel, + session, + )?; + let staged_tile = MetalLosslessEncodeTile { + buffer: &staged_buffer, + byte_offset: 0, + width: output_width, + height: output_height, + pitch_bytes: output_width as usize * bytes_per_pixel, + output_width, + output_height, + format: source.format, + }; + validate_lossless_roundtrip_on_metal_tile_with_session(staged_tile, codestream, session) +} + +#[cfg(target_os = "macos")] +fn should_try_resident_lossless_ht_cpu_packetization( + tile: MetalLosslessEncodeTile<'_>, + options: J2kLosslessEncodeOptions, + staging: MetalEncodeInputStaging, +) -> bool { + options.backend == EncodeBackendPreference::Auto + && options.block_coding_mode == J2kBlockCodingMode::HighThroughput + && options.reversible_transform == ReversibleTransform::Rct53 + && matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) + && tile.format == PixelFormat::Rgb8 +} + +#[cfg(target_os = "macos")] +fn encode_cpu_codestream_from_prepacketized_ht_tile( + tile_body: ResidentHybridHtTileBody, + tile: MetalLosslessEncodeTile<'_>, +) -> Result { + let dummy_len = checked_mul_bytes( + checked_mul_bytes(tile.output_width as usize, tile.output_height as usize), + tile_body.bytes_per_pixel, + ); + let dummy = vec![0u8; dummy_len]; + let samples = J2kLosslessSamples::new( + &dummy, + tile.output_width, + tile.output_height, + tile_body.components, + tile_body.bit_depth, + false, + ) + .map_err(crate::Error::Decode)?; + let mut wrapper = PrepacketizedHtj2kTileAccelerator { + tile_data: Some(tile_body.tile_data), + }; + let native_options = EncodeOptions { + reversible: true, + num_decomposition_levels: tile_body.num_decomposition_levels, + code_block_width_exp: tile_body.code_block_width_exp, + code_block_height_exp: tile_body.code_block_height_exp, + guard_bits: tile_body.guard_bits, + use_ht_block_coding: true, + progression_order: tile_body.progression_order, + write_tlm: tile_body.write_tlm, + use_mct: tile_body.used_fused_rct, + validate_high_throughput_codestream: false, + ..EncodeOptions::default() + }; + let codestream = j2k_native::encode_with_accelerator( + samples.data, + samples.width, + samples.height, + samples.components, + samples.bit_depth, + samples.signed, + &native_options, + &mut wrapper, + ) + .map_err(|err| { + crate::Error::Decode(j2k::J2kError::Backend(format!( + "JPEG 2000 lossless encode failed: {err}" + ))) + })?; + Ok(EncodedJ2k { + codestream, + backend: BackendKind::Cpu, + dispatch_report: j2k::adapter::encode_stage::J2kEncodeDispatchReport::default(), + width: samples.width, + height: samples.height, + components: samples.components, + bit_depth: samples.bit_depth, + signed: samples.signed, + }) +} + +#[cfg(target_os = "macos")] +fn try_encode_lossless_tile_resident_ht_cpu_packetization_with_report( + tile: MetalLosslessEncodeTile<'_>, + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, +) -> Result, crate::Error> { + let encode_started = Instant::now(); + let Some(tile_body) = encode_resident_ht_tile_body_with_cpu_packetization( + tile, + options, + session, + staging, + RESIDENT_CLASSIC_CODE_BLOCK_EDGE, + RESIDENT_CLASSIC_CODE_BLOCK_EDGE, + )? + else { + return Ok(None); + }; + let encoded = encode_cpu_codestream_from_prepacketized_ht_tile(tile_body, tile)?; + let encode_duration = encode_started.elapsed(); + let validation_duration = if options.validation == J2kEncodeValidation::CpuRoundTrip { + let validation_started = Instant::now(); + validate_lossless_roundtrip_on_metal_tile_with_session( + tile, + encoded.codestream.as_slice(), + session, + )?; + validation_started.elapsed() + } else { + Duration::ZERO + }; + + Ok(Some(MetalLosslessEncodeOutcome { + encoded, + input_copy_used: false, + resident: MetalLosslessEncodeResidency { + coefficient_prep_used: true, + packetization_used: false, + codestream_assembly_used: false, + }, + input_copy_duration: Duration::ZERO, + encode_duration, + gpu_duration: None, + validation_duration, + host_readback_duration: Duration::ZERO, + })) +} + +#[cfg(target_os = "macos")] +fn try_encode_lossless_tile_device_resident_with_report( + tile: MetalLosslessEncodeTile<'_>, + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, +) -> Result, crate::Error> { + let Some(outcome) = try_encode_lossless_tile_device_resident_to_metal_buffer_with_report( + tile, options, session, staging, + )? + else { + return Ok(None); + }; + host_outcome_from_buffer_outcome(outcome).map(Some) +} + +#[cfg(target_os = "macos")] +fn try_encode_lossless_tile_device_resident_to_metal_buffer_with_report( + tile: MetalLosslessEncodeTile<'_>, + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, +) -> Result, crate::Error> { + if options.backend == EncodeBackendPreference::CpuOnly { + return Ok(None); + } + let (components, bit_depth) = lossless_sample_shape(tile.format)?; + let bytes_per_pixel = tile.format.bytes_per_pixel(); + let bytes_per_sample = + u8::try_from(tile.format.bytes_per_sample()).map_err(|_| crate::Error::MetalKernel { + message: "J2K Metal resident encode bytes per sample exceeds u8".to_string(), + })?; + if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { + validate_padded_contiguous_metal_encode_tile(tile, bytes_per_pixel)?; + } + let Some(plan) = lossless_device_encode_plan( + tile.output_width, + tile.output_height, + components, + bit_depth, + options, + RESIDENT_CLASSIC_CODE_BLOCK_EDGE, + RESIDENT_CLASSIC_CODE_BLOCK_EDGE, + )? + else { + return Ok(None); + }; + + let encode_started = Instant::now(); + let coefficient_count = lossless_device_coefficient_count(&plan.code_blocks)?; + let prepared = compute::prepare_lossless_device_code_blocks( + session, + compute::J2kLosslessDevicePrepareJob { + input: tile.buffer, + input_byte_offset: tile.byte_offset, + input_width: tile.width, + input_height: tile.height, + input_pitch_bytes: tile.pitch_bytes, + output_width: tile.output_width, + output_height: tile.output_height, + components, + bytes_per_sample, + bit_depth, + num_decomposition_levels: plan.num_decomposition_levels, + coefficient_count, + }, + plan.code_blocks.clone(), + )?; + let packetization_resolutions = + resident_packetization_resolutions_from_lossless_device_plan(&plan)?; + let packet_descriptors = packet_descriptors_for_lossless_device_order( + plan.resolutions.len(), + plan.components, + plan.progression_order, + )?; + let packetization_job = compute::J2kResidentPacketizationEncodeJob { + resolution_count: u32::try_from(plan.resolutions.len()).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal resident encode resolution count exceeds u32".to_string(), + } + })?, + num_layers: 1, + num_components: plan.components, + code_block_count: u32::try_from(plan.code_blocks.len()).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal resident encode code-block count exceeds u32".to_string(), + } + })?, + packet_descriptors: &packet_descriptors, + resolutions: &packetization_resolutions, + }; + let assembly_job = compute::J2kLosslessCodestreamAssemblyJob { + width: tile.output_width, + height: tile.output_height, + num_components: plan.components, + bit_depth: plan.bit_depth, + signed: false, + num_decomposition_levels: plan.num_decomposition_levels, + use_mct: plan.use_mct, + guard_bits: plan.guard_bits, + code_block_width_exp: plan.code_block_width_exp, + code_block_height_exp: plan.code_block_height_exp, + progression_order: plan.progression_order, + write_tlm: plan.write_tlm, + block_coding_mode: match plan.block_coding_mode { + J2kBlockCodingMode::Classic => compute::J2kLosslessCodestreamBlockCodingMode::Classic, + J2kBlockCodingMode::HighThroughput => { + compute::J2kLosslessCodestreamBlockCodingMode::HighThroughput + } + }, + }; + let codestream = match plan.block_coding_mode { + J2kBlockCodingMode::Classic => { + let resident_tier1 = + compute::encode_classic_tier1_prepared_device_code_blocks_resident( + session, prepared, + )?; + compute::encode_lossless_codestream_buffer_from_resident_tier1( + session, + &resident_tier1, + packetization_job, + assembly_job, + )? + } + J2kBlockCodingMode::HighThroughput => { + let resident_tier1 = + compute::encode_ht_prepared_device_code_blocks_resident(session, prepared)?; + compute::encode_lossless_codestream_buffer_from_resident_tier1( + session, + &resident_tier1, + packetization_job, + assembly_job, + )? + } + }; + let encode_duration = encode_started.elapsed(); + + let encoded = MetalEncodedJ2k { + codestream_buffer: codestream.buffer, + byte_offset: codestream.byte_offset, + byte_len: codestream.byte_len, + capacity: codestream.capacity, + width: tile.output_width, + height: tile.output_height, + components, + bit_depth, + signed: false, + }; + + let validation_duration = if options.validation == J2kEncodeValidation::CpuRoundTrip { + let validation_started = Instant::now(); + if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { + validate_lossless_roundtrip_on_metal_tile_with_session( + tile, + encoded.codestream_bytes()?, + session, + )?; + } else { + validate_lossless_roundtrip_on_metal_region_with_session( + tile, + tile.output_width, + tile.output_height, + bytes_per_pixel, + encoded.codestream_bytes()?, + session, + )?; + } + validation_started.elapsed() + } else { + Duration::ZERO + }; + + Ok(Some(MetalLosslessBufferEncodeOutcome { + encoded, + input_copy_used: false, + resident: MetalLosslessEncodeResidency { + coefficient_prep_used: true, + packetization_used: true, + codestream_assembly_used: true, + }, + input_copy_duration: Duration::ZERO, + encode_duration, + gpu_duration: codestream.gpu_duration, + validation_duration, + })) +} + +#[cfg(target_os = "macos")] +fn encode_lossless_tile_to_metal_buffer_with_report( + tile: MetalLosslessEncodeTile<'_>, + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, +) -> Result { + validate_metal_encode_tile(tile)?; + lossless_sample_shape(tile.format)?; + if options.backend == EncodeBackendPreference::CpuOnly { + return Err(crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal buffer output encode requires a device backend", + }); + } + let bytes_per_pixel = tile.format.bytes_per_pixel(); + if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { + validate_padded_contiguous_metal_encode_tile(tile, bytes_per_pixel)?; + } + if let Some(outcome) = try_encode_lossless_tile_device_resident_to_metal_buffer_with_report( + tile, options, session, staging, + )? { + return Ok(outcome); + } + Err(crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal buffer output encode requires classic padded contiguous Gray/RGB lossless input with at most one DWT level", + }) +} + +#[cfg(target_os = "macos")] +fn encode_lossless_tile_with_report( + tile: MetalLosslessEncodeTile<'_>, + options: J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + staging: MetalEncodeInputStaging, + accelerator: &mut MetalEncodeStageAccelerator, +) -> Result { + validate_metal_encode_tile(tile)?; + let (components, bit_depth) = lossless_sample_shape(tile.format)?; + let bytes_per_pixel = tile.format.bytes_per_pixel(); + if let Some(outcome) = try_encode_lossless_tile_resident_ht_cpu_packetization_with_report( + tile, options, session, staging, + )? { + return Ok(outcome); + } + if should_try_resident_lossless_host_encode(options) { + if let Some(outcome) = + try_encode_lossless_tile_device_resident_with_report(tile, options, session, staging)? + { + return Ok(outcome); + } + } + if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) + && options.backend == EncodeBackendPreference::RequireDevice + { + return Err(crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal resident encode requires classic padded contiguous Gray/RGB lossless input with at most one DWT level", + }); + } + let mut input_copy_used = false; + let mut input_copy_duration = Duration::ZERO; + let mut staged_buffer = None; + let mut source_byte_offset = tile.byte_offset; + if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { + validate_padded_contiguous_metal_encode_tile(tile, bytes_per_pixel)?; + if tile.buffer.contents().is_null() { + let copy_started = Instant::now(); + staged_buffer = Some(compute::copy_interleaved_padded_to_shared_buffer( + tile.buffer, + tile.byte_offset, + tile.width, + tile.height, + tile.pitch_bytes, + tile.output_width, + tile.output_height, + bytes_per_pixel, + session, + )?); + input_copy_duration = copy_started.elapsed(); + input_copy_used = true; + source_byte_offset = 0; + } + } else { + let copy_started = Instant::now(); + staged_buffer = Some(compute::copy_interleaved_padded_to_shared_buffer( + tile.buffer, + tile.byte_offset, + tile.width, + tile.height, + tile.pitch_bytes, + tile.output_width, + tile.output_height, + bytes_per_pixel, + session, + )?); + input_copy_duration = copy_started.elapsed(); + input_copy_used = true; + source_byte_offset = 0; + } + let buffer = staged_buffer.as_ref().unwrap_or(tile.buffer); + let len = tile.output_width as usize * tile.output_height as usize * bytes_per_pixel; + let ptr = buffer.contents().cast::(); + if ptr.is_null() { + return Err(crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal encode input buffer is not host-visible", + }); + } + // SAFETY: Encoded Metal buffer views are bounds-checked before slice construction. + let data = unsafe { core::slice::from_raw_parts(ptr.add(source_byte_offset), len) }; + let samples = J2kLosslessSamples::new( + data, + tile.output_width, + tile.output_height, + components, + bit_depth, + false, + ) + .map_err(crate::Error::Decode)?; + + let encode_options = host_output_encode_options(options); + let encode_started = Instant::now(); + let encoded = j2k::encode_j2k_lossless_with_accelerator( + samples, + &encode_options, + BackendKind::Metal, + accelerator, + ) + .map_err(crate::Error::Decode)?; + let encode_duration = encode_started.elapsed(); + let validation_duration = if options.validation == J2kEncodeValidation::CpuRoundTrip { + let validation_started = Instant::now(); + validate_lossless_roundtrip_on_metal_with_session(samples, &encoded.codestream, session)?; + validation_started.elapsed() + } else { + Duration::ZERO + }; + Ok(MetalLosslessEncodeOutcome { + encoded, + input_copy_used, + resident: MetalLosslessEncodeResidency { + coefficient_prep_used: false, + packetization_used: false, + codestream_assembly_used: false, + }, + input_copy_duration, + encode_duration, + gpu_duration: None, + validation_duration, + host_readback_duration: Duration::ZERO, + }) +} + +#[cfg(not(target_os = "macos"))] +/// Return `Error::MetalUnavailable` for submitted host-byte batch encode on non-macOS. +pub fn submit_lossless_batch( + request: MetalLosslessEncodeBatchRequest<'_, '_>, + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result { + let _ = (request, options, session); + Err(crate::Error::MetalUnavailable) +} + +#[cfg(not(target_os = "macos"))] +/// Return `Error::MetalUnavailable` for submitted Metal-buffer batch encode on non-macOS. +pub fn submit_lossless_batch_to_metal( + request: MetalLosslessEncodeBatchRequest<'_, '_>, + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result { + let _ = (request, options, session); + Err(crate::Error::MetalUnavailable) +} + +#[cfg(not(target_os = "macos"))] +/// Return `Error::MetalUnavailable` for reported batch encode on non-macOS. +pub fn encode_lossless_batch_with_report( + request: MetalLosslessEncodeBatchRequest<'_, '_>, + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result, crate::Error> { + let _ = (request, options, session); + Err(crate::Error::MetalUnavailable) +} + +#[cfg(test)] +mod tests; diff --git a/crates/j2k-metal/src/encode/config.rs b/crates/j2k-metal/src/encode/config.rs new file mode 100644 index 00000000..9eb34fea --- /dev/null +++ b/crates/j2k-metal/src/encode/config.rs @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::time::Duration; + +use super::{ + MetalLosslessEncodeBatchStats, MetalLosslessEncodeConfig, MetalLosslessEncodeStageStats, +}; + +const GPU_ENCODE_DEFAULT_INFLIGHT_TILES: usize = 512; +const CLASSIC_GPU_ENCODE_SMALL_BATCH_INFLIGHT_TILES: usize = 16; +const CLASSIC_GPU_ENCODE_LARGE_BATCH_INFLIGHT_TILES: usize = 64; +const CLASSIC_GPU_ENCODE_VERY_LARGE_BATCH_MIN_TILES: usize = 64; +const CLASSIC_GPU_ENCODE_VERY_LARGE_BATCH_INFLIGHT_TILES: usize = 128; +const HTJ2K_GPU_ENCODE_MEDIUM_BATCH_TILES: usize = 64; +const HTJ2K_GPU_ENCODE_MEDIUM_BATCH_INFLIGHT_TILES: usize = 32; +const HTJ2K_GPU_ENCODE_LARGE_BATCH_MIN_TILES: usize = 64; +const HTJ2K_GPU_ENCODE_LARGE_BATCH_INFLIGHT_TILES: usize = 64; +const GPU_ENCODE_FALLBACK_HW_MEM_BYTES: usize = 8 * 1024 * 1024 * 1024; +const GPU_ENCODE_MAX_DEFAULT_MEMORY_BUDGET_BYTES: usize = 10 * 1024 * 1024 * 1024; +const GPU_ENCODE_MEMORY_BUDGET_PERCENT: usize = 40; +const RESIDENT_HT_DEFAULT_CHUNK_CODE_BLOCKS: usize = 131_072; + +#[cfg(test)] +pub(super) fn default_gpu_encode_memory_budget_bytes_for_hw_mem(hw_memsize: usize) -> usize { + default_gpu_encode_memory_budget_bytes_for_hw_mem_inner(hw_memsize) +} + +fn default_gpu_encode_memory_budget_bytes_for_hw_mem_inner(hw_memsize: usize) -> usize { + hw_memsize + .saturating_mul(GPU_ENCODE_MEMORY_BUDGET_PERCENT) + .checked_div(100) + .unwrap_or(0) + .clamp(1, GPU_ENCODE_MAX_DEFAULT_MEMORY_BUDGET_BYTES) +} + +fn default_gpu_encode_memory_budget_bytes() -> usize { + let hw_memsize = host_memory_bytes().unwrap_or(GPU_ENCODE_FALLBACK_HW_MEM_BYTES); + default_gpu_encode_memory_budget_bytes_for_hw_mem_inner(hw_memsize) +} + +pub(super) fn resident_lossless_encode_config_for_mode( + config: MetalLosslessEncodeConfig, + classic_resident_mode: bool, + tile_count: usize, +) -> MetalLosslessEncodeConfig { + if config.gpu_encode_inflight_tiles.is_some() { + return config; + } + if classic_resident_mode { + let classic_inflight_tiles = if tile_count <= CLASSIC_GPU_ENCODE_SMALL_BATCH_INFLIGHT_TILES + { + CLASSIC_GPU_ENCODE_SMALL_BATCH_INFLIGHT_TILES + } else if tile_count <= CLASSIC_GPU_ENCODE_VERY_LARGE_BATCH_MIN_TILES { + CLASSIC_GPU_ENCODE_LARGE_BATCH_INFLIGHT_TILES + } else { + CLASSIC_GPU_ENCODE_VERY_LARGE_BATCH_INFLIGHT_TILES + }; + MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(classic_inflight_tiles), + ..config + } + } else if tile_count == HTJ2K_GPU_ENCODE_MEDIUM_BATCH_TILES { + MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(HTJ2K_GPU_ENCODE_MEDIUM_BATCH_INFLIGHT_TILES), + ..config + } + } else if tile_count > HTJ2K_GPU_ENCODE_LARGE_BATCH_MIN_TILES { + MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(HTJ2K_GPU_ENCODE_LARGE_BATCH_INFLIGHT_TILES), + ..config + } + } else { + config + } +} + +#[cfg(target_os = "macos")] +fn host_memory_bytes() -> Option { + let mut value = 0u64; + let mut len = core::mem::size_of::(); + let name = b"hw.memsize\0"; + // SAFETY: `sysctlbyname` writes a u64 into the provided buffer when the name is supported. + let rc = unsafe { + libc::sysctlbyname( + name.as_ptr().cast(), + (&raw mut value).cast(), + &raw mut len, + core::ptr::null_mut(), + 0, + ) + }; + (rc == 0 && len == core::mem::size_of::()) + .then(|| usize::try_from(value).ok()) + .flatten() +} + +#[cfg(all(test, not(target_os = "macos")))] +fn host_memory_bytes() -> Option { + None +} + +pub(super) fn resolve_lossless_encode_config( + tile_count: usize, + estimated_peak_bytes_per_tile: usize, + config: MetalLosslessEncodeConfig, +) -> Result { + if config.gpu_encode_inflight_tiles == Some(0) { + return Err(crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal encode in-flight tile cap must be greater than zero", + }); + } + if config.gpu_encode_memory_budget_bytes == Some(0) { + return Err(crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal encode memory budget must be greater than zero", + }); + } + + let effective_memory_budget_bytes = config + .gpu_encode_memory_budget_bytes + .unwrap_or_else(default_gpu_encode_memory_budget_bytes) + .max(1); + let estimated_peak_bytes_per_tile = estimated_peak_bytes_per_tile.max(1); + let memory_limited_tiles = + (effective_memory_budget_bytes / estimated_peak_bytes_per_tile).max(1); + let configured_or_default = config + .gpu_encode_inflight_tiles + .unwrap_or(GPU_ENCODE_DEFAULT_INFLIGHT_TILES); + let effective_inflight_tiles = configured_or_default + .min(memory_limited_tiles) + .min(tile_count.max(1)) + .max(1); + + Ok(MetalLosslessEncodeBatchStats { + configured_inflight_tiles: config.gpu_encode_inflight_tiles, + effective_inflight_tiles, + configured_memory_budget_bytes: config.gpu_encode_memory_budget_bytes, + effective_memory_budget_bytes, + estimated_peak_bytes_per_tile, + max_observed_inflight_tiles: 0, + encode_wall_duration: Duration::ZERO, + stage_stats: MetalLosslessEncodeStageStats::default(), + }) +} + +#[cfg(test)] +pub(super) fn resolve_lossless_encode_config_for_test( + tile_count: usize, + estimated_peak_bytes_per_tile: usize, + config: MetalLosslessEncodeConfig, +) -> Result { + resolve_lossless_encode_config(tile_count, estimated_peak_bytes_per_tile, config) +} + +pub(super) fn resident_lossless_code_block_chunk_cap(code_block_counts: &[usize]) -> usize { + code_block_counts + .iter() + .copied() + .max() + .unwrap_or(1) + .max(RESIDENT_HT_DEFAULT_CHUNK_CODE_BLOCKS) +} + +pub(super) fn resident_lossless_chunk_ranges_from_code_blocks( + code_block_counts: &[usize], + max_tiles: usize, + max_code_blocks: usize, +) -> Vec> { + if code_block_counts.is_empty() { + return Vec::new(); + } + let max_tiles = max_tiles.max(1); + let max_code_blocks = max_code_blocks.max(1); + let mut ranges = Vec::new(); + let mut start = 0usize; + while start < code_block_counts.len() { + let mut end = start; + let mut chunk_code_blocks = 0usize; + while end < code_block_counts.len() && end - start < max_tiles { + let next_code_blocks = code_block_counts[end].max(1); + let would_exceed_code_blocks = + end > start && chunk_code_blocks.saturating_add(next_code_blocks) > max_code_blocks; + if would_exceed_code_blocks { + break; + } + chunk_code_blocks = chunk_code_blocks.saturating_add(next_code_blocks); + end += 1; + } + if end == start { + end += 1; + } + ranges.push(start..end); + start = end; + } + ranges +} + +#[cfg(test)] +pub(super) fn resident_lossless_chunk_ranges_for_test( + code_block_counts: &[usize], + max_tiles: usize, + max_code_blocks: usize, +) -> Vec> { + resident_lossless_chunk_ranges_from_code_blocks(code_block_counts, max_tiles, max_code_blocks) +} diff --git a/crates/j2k-metal/src/encode/encoded.rs b/crates/j2k-metal/src/encode/encoded.rs new file mode 100644 index 00000000..a8491e36 --- /dev/null +++ b/crates/j2k-metal/src/encode/encoded.rs @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(target_os = "macos")] +use j2k::EncodedJ2k; +#[cfg(target_os = "macos")] +use j2k_core::BackendKind; +#[cfg(target_os = "macos")] +use metal::Buffer; +#[cfg(target_os = "macos")] +use std::time::Duration; +#[cfg(target_os = "macos")] +use std::time::Instant; + +#[cfg(target_os = "macos")] +/// JPEG 2000 codestream bytes owned by a Metal buffer. +/// +/// The buffer is CPU-readable for the current padded resident encode API, so +/// callers can stream `codestream_bytes()` into file or network writers without +/// first materializing an owned `Vec`. +pub struct MetalEncodedJ2k { + /// Metal buffer containing the codestream bytes. + pub codestream_buffer: Buffer, + /// Byte offset of the first codestream byte in `codestream_buffer`. + pub byte_offset: usize, + /// Number of valid codestream bytes. + pub byte_len: usize, + /// Allocated codestream capacity in bytes. + pub capacity: usize, + /// Encoded image width in pixels. + pub width: u32, + /// Encoded image height in pixels. + pub height: u32, + /// Number of encoded components. + pub components: u8, + /// Component bit depth. + pub bit_depth: u8, + /// Whether components are signed. + pub signed: bool, +} + +#[cfg(target_os = "macos")] +impl MetalEncodedJ2k { + /// Borrow the finished codestream bytes from the backing Metal buffer. + pub fn codestream_bytes(&self) -> Result<&[u8], crate::Error> { + let end = self.byte_offset.checked_add(self.byte_len).ok_or_else(|| { + crate::Error::MetalKernel { + message: "J2K Metal codestream byte range overflow".to_string(), + } + })?; + let buffer_len = usize::try_from(self.codestream_buffer.length()).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal codestream buffer length exceeds usize".to_string(), + } + })?; + if end > buffer_len { + return Err(crate::Error::MetalKernel { + message: "J2K Metal codestream byte range exceeds buffer length".to_string(), + }); + } + let ptr = self.codestream_buffer.contents().cast::(); + if ptr.is_null() { + return Err(crate::Error::MetalKernel { + message: "J2K Metal codestream buffer is not CPU-readable".to_string(), + }); + } + // SAFETY: Encoded Metal buffer views are bounds-checked before slice construction. + Ok(unsafe { core::slice::from_raw_parts(ptr.add(self.byte_offset), self.byte_len) }) + } + + /// Materialize the buffer-backed codestream into the compatibility `Vec` API shape. + pub fn to_encoded_j2k(&self) -> Result { + let (encoded, _host_readback_duration) = self.to_encoded_j2k_with_readback_duration()?; + Ok(encoded) + } + + pub(super) fn to_encoded_j2k_with_readback_duration( + &self, + ) -> Result<(EncodedJ2k, Duration), crate::Error> { + let readback_started = Instant::now(); + let codestream = self.codestream_bytes()?.to_vec(); + let host_readback_duration = readback_started.elapsed(); + Ok(( + EncodedJ2k { + codestream, + backend: BackendKind::Metal, + dispatch_report: j2k::adapter::encode_stage::J2kEncodeDispatchReport::default(), + width: self.width, + height: self.height, + components: self.components, + bit_depth: self.bit_depth, + signed: self.signed, + }, + host_readback_duration, + )) + } +} + +#[cfg(not(target_os = "macos"))] +/// Placeholder Metal codestream type for non-macOS builds. +pub struct MetalEncodedJ2k { + _private: (), +} diff --git a/crates/j2k-metal/src/encode/packet_plan.rs b/crates/j2k-metal/src/encode/packet_plan.rs new file mode 100644 index 00000000..20dc36f9 --- /dev/null +++ b/crates/j2k-metal/src/encode/packet_plan.rs @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::{ + EncodeBackendPreference, J2kBlockCodingMode, J2kEncodeValidation, J2kLosslessEncodeOptions, + J2kProgressionOrder, ReversibleTransform, +}; +use j2k_native::{ + EncodeProgressionOrder, EncodedHtJ2kCodeBlock, J2kHtj2kTileEncodeJob, + J2kPacketizationBlockCodingMode, J2kPacketizationCodeBlock, J2kPacketizationPacketDescriptor, + J2kPacketizationProgressionOrder, J2kPacketizationResolution, J2kPacketizationSubband, +}; + +use super::plan::LosslessDeviceEncodePlan; +use crate::compute; + +const AUTO_HTJ2K_HOST_RESIDENT_MIN_PIXELS: usize = 1024 * 1024; + +fn lossless_progression_from_packetization_order( + order: J2kPacketizationProgressionOrder, +) -> J2kProgressionOrder { + match order { + J2kPacketizationProgressionOrder::Lrcp => J2kProgressionOrder::Lrcp, + J2kPacketizationProgressionOrder::Rlcp => J2kProgressionOrder::Rlcp, + J2kPacketizationProgressionOrder::Rpcl => J2kProgressionOrder::Rpcl, + J2kPacketizationProgressionOrder::Pcrl => J2kProgressionOrder::Pcrl, + J2kPacketizationProgressionOrder::Cprl => J2kProgressionOrder::Cprl, + } +} + +pub(super) fn lossless_options_for_resident_htj2k_tile_job( + job: J2kHtj2kTileEncodeJob<'_>, +) -> Option { + if job.num_components != 3 + || job.bit_depth != 8 + || job.signed + || !job.reversible + || !job.use_mct + || job.guard_bits != 2 + || job.code_block_width != 64 + || job.code_block_height != 64 + { + return None; + } + if job.component_sampling.len() != usize::from(job.num_components) + || job + .component_sampling + .iter() + .any(|&(x_sampling, y_sampling)| x_sampling != 1 || y_sampling != 1) + { + return None; + } + let expected_len = (job.width as usize) + .checked_mul(job.height as usize)? + .checked_mul(usize::from(job.num_components))?; + if expected_len != job.pixels.len() { + return None; + } + Some(J2kLosslessEncodeOptions::new( + EncodeBackendPreference::Auto, + J2kBlockCodingMode::HighThroughput, + lossless_progression_from_packetization_order(job.progression_order), + Some(job.num_decomposition_levels), + ReversibleTransform::Rct53, + J2kEncodeValidation::External, + )) +} + +pub(super) fn should_use_resident_htj2k_host_tile_for_auto(job: J2kHtj2kTileEncodeJob<'_>) -> bool { + (job.width as usize).saturating_mul(job.height as usize) >= AUTO_HTJ2K_HOST_RESIDENT_MIN_PIXELS +} + +pub(super) fn packet_descriptors_for_lossless_device_order( + packet_count: usize, + num_components: u8, + progression_order: EncodeProgressionOrder, +) -> Result, crate::Error> { + let component_count = usize::from(num_components).max(1); + let mut descriptors = (0..packet_count) + .map(|packet_index| { + Ok(J2kPacketizationPacketDescriptor { + packet_index: u32::try_from(packet_index).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal resident encode packet index exceeds u32".to_string(), + } + })?, + state_index: u32::try_from(packet_index).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal resident encode packet state index exceeds u32" + .to_string(), + } + })?, + layer: 0, + resolution: u32::try_from(packet_index / component_count).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal resident encode packet resolution exceeds u32" + .to_string(), + } + })?, + component: u8::try_from(packet_index % component_count).map_err(|_| { + crate::Error::MetalKernel { + message: "J2K Metal resident encode packet component exceeds u8" + .to_string(), + } + })?, + precinct: 0, + }) + }) + .collect::, crate::Error>>()?; + sort_lossless_device_packet_descriptors(&mut descriptors, progression_order); + Ok(descriptors) +} + +fn sort_lossless_device_packet_descriptors( + descriptors: &mut [J2kPacketizationPacketDescriptor], + progression_order: EncodeProgressionOrder, +) { + match progression_order { + EncodeProgressionOrder::Lrcp => descriptors.sort_by_key(|descriptor| { + ( + descriptor.layer, + descriptor.resolution, + descriptor.component, + descriptor.precinct, + ) + }), + EncodeProgressionOrder::Rlcp => descriptors.sort_by_key(|descriptor| { + ( + descriptor.resolution, + descriptor.layer, + descriptor.component, + descriptor.precinct, + ) + }), + EncodeProgressionOrder::Rpcl => descriptors.sort_by_key(|descriptor| { + ( + descriptor.resolution, + descriptor.precinct, + descriptor.component, + descriptor.layer, + ) + }), + EncodeProgressionOrder::Pcrl => descriptors.sort_by_key(|descriptor| { + ( + descriptor.precinct, + descriptor.component, + descriptor.resolution, + descriptor.layer, + ) + }), + EncodeProgressionOrder::Cprl => descriptors.sort_by_key(|descriptor| { + ( + descriptor.component, + descriptor.precinct, + descriptor.resolution, + descriptor.layer, + ) + }), + } +} + +pub(super) fn resident_packetization_resolutions_from_lossless_device_plan( + plan: &LosslessDeviceEncodePlan, +) -> Result, crate::Error> { + plan.resolutions + .iter() + .map(|resolution| { + let subbands = resolution + .subbands + .iter() + .map(|subband| { + let code_block_end = subband + .code_block_start + .checked_add(subband.code_block_count) + .ok_or_else(|| crate::Error::MetalKernel { + message: "J2K Metal resident encode code-block range overflow" + .to_string(), + })?; + if code_block_end > plan.code_blocks.len() { + return Err(crate::Error::MetalKernel { + message: "J2K Metal resident encode code-block range out of bounds" + .to_string(), + }); + } + Ok(compute::J2kResidentPacketizationSubband { + code_block_start: u32::try_from(subband.code_block_start).map_err( + |_| crate::Error::MetalKernel { + message: "J2K Metal resident encode code-block offset exceeds u32" + .to_string(), + }, + )?, + code_block_count: u32::try_from(subband.code_block_count).map_err( + |_| crate::Error::MetalKernel { + message: "J2K Metal resident encode code-block count exceeds u32" + .to_string(), + }, + )?, + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + }) + }) + .collect::, crate::Error>>()?; + Ok(compute::J2kResidentPacketizationResolution { subbands }) + }) + .collect() +} + +pub(super) fn packetization_progression_order( + order: EncodeProgressionOrder, +) -> J2kPacketizationProgressionOrder { + match order { + EncodeProgressionOrder::Lrcp => J2kPacketizationProgressionOrder::Lrcp, + EncodeProgressionOrder::Rlcp => J2kPacketizationProgressionOrder::Rlcp, + EncodeProgressionOrder::Rpcl => J2kPacketizationProgressionOrder::Rpcl, + EncodeProgressionOrder::Pcrl => J2kPacketizationProgressionOrder::Pcrl, + EncodeProgressionOrder::Cprl => J2kPacketizationProgressionOrder::Cprl, + } +} + +pub(super) fn cpu_packetization_resolutions_from_lossless_device_plan<'a>( + plan: &LosslessDeviceEncodePlan, + encoded_blocks: &'a [EncodedHtJ2kCodeBlock], +) -> Result>, crate::Error> { + if encoded_blocks.len() != plan.code_blocks.len() { + return Err(crate::Error::MetalKernel { + message: "J2K Metal resident hybrid HT block count mismatch".to_string(), + }); + } + plan.resolutions + .iter() + .map(|resolution| { + let subbands = resolution + .subbands + .iter() + .map(|subband| { + let code_block_end = subband + .code_block_start + .checked_add(subband.code_block_count) + .ok_or_else(|| crate::Error::MetalKernel { + message: "J2K Metal resident hybrid code-block range overflow" + .to_string(), + })?; + let encoded = encoded_blocks + .get(subband.code_block_start..code_block_end) + .ok_or_else(|| crate::Error::MetalKernel { + message: "J2K Metal resident hybrid code-block range out of bounds" + .to_string(), + })?; + let code_blocks = encoded + .iter() + .map(|block| J2kPacketizationCodeBlock { + data: block.data.as_slice(), + ht_cleanup_length: block.cleanup_length, + ht_refinement_length: block.refinement_length, + num_coding_passes: block.num_coding_passes, + num_zero_bitplanes: block.num_zero_bitplanes, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }) + .collect(); + Ok(J2kPacketizationSubband { + code_blocks, + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + }) + }) + .collect::, crate::Error>>()?; + Ok(J2kPacketizationResolution { subbands }) + }) + .collect() +} diff --git a/crates/j2k-metal/src/encode/plan.rs b/crates/j2k-metal/src/encode/plan.rs new file mode 100644 index 00000000..9e2bf1eb --- /dev/null +++ b/crates/j2k-metal/src/encode/plan.rs @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::{J2kBlockCodingMode, J2kLosslessEncodeOptions, J2kProgressionOrder}; +use j2k_native::{EncodeProgressionOrder, J2kSubBandType}; + +use crate::compute; + +#[derive(Clone, Copy)] +pub(super) struct LosslessSubbandPlan { + pub(super) num_cbs_x: u32, + pub(super) num_cbs_y: u32, + pub(super) code_block_start: usize, + pub(super) code_block_count: usize, +} + +#[derive(Clone)] +pub(super) struct LosslessResolutionPlan { + pub(super) subbands: Vec, +} + +pub(super) struct LosslessDeviceEncodePlan { + pub(super) components: u8, + pub(super) bit_depth: u8, + pub(super) block_coding_mode: J2kBlockCodingMode, + pub(super) num_decomposition_levels: u8, + pub(super) use_mct: bool, + pub(super) guard_bits: u8, + pub(super) code_block_width_exp: u8, + pub(super) code_block_height_exp: u8, + pub(super) code_blocks: Vec, + pub(super) resolutions: Vec, + pub(super) progression_order: EncodeProgressionOrder, + pub(super) write_tlm: bool, +} + +pub(super) const RESIDENT_CLASSIC_CODE_BLOCK_EDGE: u32 = 32; + +fn lossless_device_encode_levels(width: u32, height: u32, options: J2kLosslessEncodeOptions) -> u8 { + const MIN_LOSSLESS_DWT_DIMENSION: u32 = 64; + let levels = if matches!( + options.progression, + J2kProgressionOrder::Rpcl | J2kProgressionOrder::Pcrl | J2kProgressionOrder::Cprl + ) { + let mut levels = 0u8; + let mut w = width; + let mut h = height; + let max_levels = if width.min(height) <= 1 { + 0 + } else { + width.min(height).ilog2() as u8 + }; + while w.min(h) > MIN_LOSSLESS_DWT_DIMENSION && levels < max_levels { + w = w.div_ceil(2); + h = h.div_ceil(2); + levels = levels.saturating_add(1); + } + levels + } else { + u8::from(width.min(height) >= MIN_LOSSLESS_DWT_DIMENSION) + }; + + options + .max_decomposition_levels + .map_or(levels, |requested| { + let max_levels = if width.min(height) <= 1 { + 0 + } else { + width.min(height).ilog2() as u8 + }; + requested.min(max_levels) + }) +} + +#[derive(Clone, Copy)] +struct LosslessDwtLevelPlan { + low_width: u32, + low_height: u32, + high_width: u32, + high_height: u32, +} + +#[derive(Clone, Copy)] +struct LosslessSubbandInput { + component: u32, + subband_x: u32, + subband_y: u32, + width: u32, + height: u32, + sub_band_type: J2kSubBandType, + total_bitplanes: u8, +} + +fn lossless_code_block_exp(edge: u32, axis: &str) -> Result { + if edge < 4 || !edge.is_power_of_two() { + return Err(crate::Error::MetalKernel { + message: format!( + "J2K Metal resident encode {axis} code-block edge must be a power of two >= 4" + ), + }); + } + let exp = edge + .trailing_zeros() + .checked_sub(2) + .ok_or_else(|| crate::Error::MetalKernel { + message: format!("J2K Metal resident encode {axis} code-block exponent underflow"), + })?; + if exp > 8 { + return Err(crate::Error::MetalKernel { + message: format!( + "J2K Metal resident encode {axis} code-block edge exceeds JPEG 2000 COD range" + ), + }); + } + u8::try_from(exp).map_err(|_| crate::Error::MetalKernel { + message: format!("J2K Metal resident encode {axis} code-block exponent exceeds u8"), + }) +} + +fn push_lossless_subband_plan( + resolution: &mut LosslessResolutionPlan, + code_blocks: &mut Vec, + coefficient_offset: &mut u32, + code_block_width: u32, + code_block_height: u32, + subband: LosslessSubbandInput, +) -> Result<(), crate::Error> { + if subband.width == 0 || subband.height == 0 { + resolution.subbands.push(LosslessSubbandPlan { + num_cbs_x: 0, + num_cbs_y: 0, + code_block_start: code_blocks.len(), + code_block_count: 0, + }); + return Ok(()); + } + let cb_width = code_block_width; + let cb_height = code_block_height; + let num_cbs_x = subband.width.div_ceil(cb_width); + let num_cbs_y = subband.height.div_ceil(cb_height); + let code_block_start = code_blocks.len(); + for cby in 0..num_cbs_y { + for cbx in 0..num_cbs_x { + let block_x = cbx * cb_width; + let block_y = cby * cb_height; + let block_width = (block_x + cb_width).min(subband.width) - block_x; + let block_height = (block_y + cb_height).min(subband.height) - block_y; + let coeff_count = + block_width + .checked_mul(block_height) + .ok_or_else(|| crate::Error::MetalKernel { + message: "J2K Metal resident encode code-block size overflow".to_string(), + })?; + code_blocks.push(compute::J2kLosslessDeviceCodeBlock { + coefficient_offset: *coefficient_offset, + component: subband.component, + subband_x: subband.subband_x, + subband_y: subband.subband_y, + block_x, + block_y, + width: block_width, + height: block_height, + sub_band_type: subband.sub_band_type, + total_bitplanes: subband.total_bitplanes, + }); + *coefficient_offset = coefficient_offset.checked_add(coeff_count).ok_or_else(|| { + crate::Error::MetalKernel { + message: "J2K Metal resident encode coefficient offset overflow".to_string(), + } + })?; + } + } + resolution.subbands.push(LosslessSubbandPlan { + num_cbs_x, + num_cbs_y, + code_block_start, + code_block_count: code_blocks.len() - code_block_start, + }); + Ok(()) +} + +fn lossless_dwt_level_plans( + width: u32, + height: u32, + num_decomposition_levels: u8, +) -> Vec { + let mut levels = Vec::with_capacity(usize::from(num_decomposition_levels)); + let mut current_width = width; + let mut current_height = height; + for _ in 0..num_decomposition_levels { + let low_width = current_width.div_ceil(2); + let low_height = current_height.div_ceil(2); + let high_width = current_width / 2; + let high_height = current_height / 2; + levels.push(LosslessDwtLevelPlan { + low_width, + low_height, + high_width, + high_height, + }); + current_width = low_width; + current_height = low_height; + } + levels +} + +pub(super) fn lossless_device_encode_plan( + width: u32, + height: u32, + components: u8, + bit_depth: u8, + options: J2kLosslessEncodeOptions, + code_block_width: u32, + code_block_height: u32, +) -> Result, crate::Error> { + if !matches!( + options.block_coding_mode, + J2kBlockCodingMode::Classic | J2kBlockCodingMode::HighThroughput + ) { + return Ok(None); + } + if code_block_width == 0 || code_block_height == 0 { + return Err(crate::Error::MetalKernel { + message: "J2K Metal resident encode code-block dimensions must be non-zero".to_string(), + }); + } + let code_block_width_exp = lossless_code_block_exp(code_block_width, "width")?; + let code_block_height_exp = lossless_code_block_exp(code_block_height, "height")?; + let num_decomposition_levels = lossless_device_encode_levels(width, height, options); + let progression_order = match options.progression { + J2kProgressionOrder::Lrcp => EncodeProgressionOrder::Lrcp, + J2kProgressionOrder::Rlcp => EncodeProgressionOrder::Rlcp, + J2kProgressionOrder::Rpcl => EncodeProgressionOrder::Rpcl, + J2kProgressionOrder::Pcrl => EncodeProgressionOrder::Pcrl, + J2kProgressionOrder::Cprl => EncodeProgressionOrder::Cprl, + }; + let use_mct = components >= 3; + let guard_bits: u8 = if use_mct { 2 } else { 1 }; + let mut code_blocks = Vec::new(); + let mut coefficient_offset = 0u32; + let mut component_resolutions = Vec::>::new(); + for component in 0..components { + let mut component_packets = Vec::new(); + let dwt_levels = lossless_dwt_level_plans(width, height, num_decomposition_levels); + let mut base_packet = LosslessResolutionPlan { + subbands: Vec::new(), + }; + if num_decomposition_levels == 0 { + push_lossless_subband_plan( + &mut base_packet, + &mut code_blocks, + &mut coefficient_offset, + code_block_width, + code_block_height, + LosslessSubbandInput { + component: u32::from(component), + subband_x: 0, + subband_y: 0, + width, + height, + sub_band_type: J2kSubBandType::LowLow, + total_bitplanes: guard_bits.saturating_add(bit_depth).saturating_sub(1), + }, + )?; + component_packets.push(base_packet); + } else { + let final_ll = dwt_levels + .last() + .expect("nonzero DWT level count has a final LL level"); + push_lossless_subband_plan( + &mut base_packet, + &mut code_blocks, + &mut coefficient_offset, + code_block_width, + code_block_height, + LosslessSubbandInput { + component: u32::from(component), + subband_x: 0, + subband_y: 0, + width: final_ll.low_width, + height: final_ll.low_height, + sub_band_type: J2kSubBandType::LowLow, + total_bitplanes: guard_bits.saturating_add(bit_depth).saturating_sub(1), + }, + )?; + component_packets.push(base_packet); + + for level in dwt_levels.iter().rev().copied() { + let mut detail_packet = LosslessResolutionPlan { + subbands: Vec::new(), + }; + push_lossless_subband_plan( + &mut detail_packet, + &mut code_blocks, + &mut coefficient_offset, + code_block_width, + code_block_height, + LosslessSubbandInput { + component: u32::from(component), + subband_x: level.low_width, + subband_y: 0, + width: level.high_width, + height: level.low_height, + sub_band_type: J2kSubBandType::HighLow, + total_bitplanes: guard_bits.saturating_add(bit_depth), + }, + )?; + push_lossless_subband_plan( + &mut detail_packet, + &mut code_blocks, + &mut coefficient_offset, + code_block_width, + code_block_height, + LosslessSubbandInput { + component: u32::from(component), + subband_x: 0, + subband_y: level.low_height, + width: level.low_width, + height: level.high_height, + sub_band_type: J2kSubBandType::LowHigh, + total_bitplanes: guard_bits.saturating_add(bit_depth), + }, + )?; + push_lossless_subband_plan( + &mut detail_packet, + &mut code_blocks, + &mut coefficient_offset, + code_block_width, + code_block_height, + LosslessSubbandInput { + component: u32::from(component), + subband_x: level.low_width, + subband_y: level.low_height, + width: level.high_width, + height: level.high_height, + sub_band_type: J2kSubBandType::HighHigh, + total_bitplanes: guard_bits.saturating_add(bit_depth).saturating_add(1), + }, + )?; + component_packets.push(detail_packet); + } + } + component_resolutions.push(component_packets); + } + + let resolution_count = component_resolutions.first().map_or(0usize, Vec::len); + let mut resolutions = + Vec::with_capacity(resolution_count.saturating_mul(usize::from(components))); + for resolution in 0..resolution_count { + for component in &component_resolutions { + resolutions.push(component[resolution].clone()); + } + } + + Ok(Some(LosslessDeviceEncodePlan { + components, + bit_depth, + block_coding_mode: options.block_coding_mode, + num_decomposition_levels, + use_mct, + guard_bits, + code_block_width_exp, + code_block_height_exp, + code_blocks, + resolutions, + progression_order, + write_tlm: options.progression == J2kProgressionOrder::Rpcl, + })) +} diff --git a/crates/j2k-metal/src/encode/resident_estimate.rs b/crates/j2k-metal/src/encode/resident_estimate.rs new file mode 100644 index 00000000..53e4bea0 --- /dev/null +++ b/crates/j2k-metal/src/encode/resident_estimate.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::J2kBlockCodingMode; + +use super::{ + LosslessDeviceEncodePlan, MetalEncodeInputStaging, ResidentLosslessBufferEncodeMetadata, +}; +use crate::compute; + +pub(super) fn checked_add_bytes(lhs: usize, rhs: usize) -> usize { + lhs.saturating_add(rhs) +} + +pub(super) fn checked_mul_bytes(lhs: usize, rhs: usize) -> usize { + lhs.saturating_mul(rhs) +} + +pub(super) fn estimate_resident_lossless_encode_peak_bytes( + metadata: &ResidentLosslessBufferEncodeMetadata, + coefficient_count: usize, + staging: MetalEncodeInputStaging, +) -> usize { + let pixels = checked_mul_bytes( + metadata.tile.output_width as usize, + metadata.tile.output_height as usize, + ) + .max(1); + let plane_bytes = checked_mul_bytes(pixels, core::mem::size_of::()); + let code_block_count = metadata.plan.code_blocks.len().max(1); + let packet_count = metadata + .packet_descriptors + .len() + .max(metadata.plan.resolutions.len()) + .max(1); + let input_bytes = checked_mul_bytes( + checked_mul_bytes(metadata.tile.width as usize, metadata.tile.height as usize), + metadata.bytes_per_pixel, + ); + let staged_input_bytes = if matches!(staging, MetalEncodeInputStaging::CopyAndPad) { + checked_mul_bytes(pixels, metadata.bytes_per_pixel) + } else { + 0 + }; + let coefficient_bytes = + checked_mul_bytes(coefficient_count.max(1), core::mem::size_of::()); + let plane_buffers = checked_mul_bytes(3, plane_bytes); + let scratch_buffers = checked_mul_bytes(usize::from(metadata.components), plane_bytes); + let code_block_tables = checked_mul_bytes(code_block_count, 256); + let tier1_output = estimated_tier1_output_bytes(&metadata.plan); + let packet_header = checked_add_bytes(checked_mul_bytes(code_block_count, 256), 4096); + let packet_output = checked_add_bytes( + checked_add_bytes(tier1_output, checked_mul_bytes(packet_header, packet_count)), + 1024, + ); + let codestream_capacity = checked_add_bytes( + packet_output, + checked_add_bytes(4096, checked_mul_bytes(pixels, metadata.bytes_per_pixel)), + ); + let validation_bytes = checked_mul_bytes(pixels, metadata.bytes_per_pixel).saturating_mul( + usize::from(metadata.plan.write_tlm || metadata.plan.use_mct || metadata.components > 0), + ); + + [ + input_bytes / 4, + staged_input_bytes, + plane_buffers, + scratch_buffers, + coefficient_bytes, + code_block_tables, + tier1_output, + packet_output, + codestream_capacity, + validation_bytes, + 4 * 1024 * 1024, + ] + .into_iter() + .fold(0usize, checked_add_bytes) +} + +pub(super) fn estimated_tier1_output_bytes(plan: &LosslessDeviceEncodePlan) -> usize { + fn estimated_ht_output_capacity(width: usize, height: usize) -> usize { + const HT_MAX_SAMPLES: usize = 16_384; + const HT_MEL_SIZE: usize = 192; + const HT_VLC_SIZE: usize = 3072 - HT_MEL_SIZE; + const HT_MS_SIZE: usize = (HT_MAX_SAMPLES * 16).div_ceil(15); + const HT_MS_BYTES_PER_SAMPLE_FLOOR: usize = 5; + + let samples = checked_mul_bytes(width, height).min(HT_MAX_SAMPLES); + let scaled_ms = checked_mul_bytes(HT_MS_SIZE, samples) + .div_ceil(HT_MAX_SAMPLES) + .max(1); + let ms_floor = checked_mul_bytes(samples, HT_MS_BYTES_PER_SAMPLE_FLOOR); + let ms_size = scaled_ms.max(ms_floor).min(HT_MS_SIZE); + let fixed_entropy = checked_add_bytes(HT_MEL_SIZE, HT_VLC_SIZE); + checked_add_bytes(ms_size, fixed_entropy) + } + + plan.code_blocks + .iter() + .map(|block| match plan.block_coding_mode { + J2kBlockCodingMode::HighThroughput => { + estimated_ht_output_capacity(block.width as usize, block.height as usize) + } + J2kBlockCodingMode::Classic => { + let samples = checked_mul_bytes(block.width as usize, block.height as usize); + checked_add_bytes( + checked_mul_bytes(samples, usize::from(block.total_bitplanes).max(1)), + 4097, + ) + .max(4097) + } + }) + .fold(0usize, checked_add_bytes) + .max(1) +} + +pub(super) fn resident_codestream_assembly_job_for_metadata( + metadata: &ResidentLosslessBufferEncodeMetadata, +) -> compute::J2kLosslessCodestreamAssemblyJob { + compute::J2kLosslessCodestreamAssemblyJob { + width: metadata.tile.output_width, + height: metadata.tile.output_height, + num_components: metadata.plan.components, + bit_depth: metadata.plan.bit_depth, + signed: false, + num_decomposition_levels: metadata.plan.num_decomposition_levels, + use_mct: metadata.plan.use_mct, + guard_bits: metadata.plan.guard_bits, + code_block_width_exp: metadata.plan.code_block_width_exp, + code_block_height_exp: metadata.plan.code_block_height_exp, + progression_order: metadata.plan.progression_order, + write_tlm: metadata.plan.write_tlm, + block_coding_mode: match metadata.plan.block_coding_mode { + J2kBlockCodingMode::Classic => compute::J2kLosslessCodestreamBlockCodingMode::Classic, + J2kBlockCodingMode::HighThroughput => { + compute::J2kLosslessCodestreamBlockCodingMode::HighThroughput + } + }, + } +} + +pub(super) fn resident_classic_batch_encode_should_retry_conservative( + error: &crate::Error, +) -> bool { + let crate::Error::MetalKernel { message } = error else { + return false; + }; + + message.contains("classic Tier-1 Metal encode kernel failure (detail=4)") + || message.contains("classic Tier-1 Metal encode kernel failure (detail=5)") + || message.contains("packetization Metal encode kernel failure (detail=3)") + || message.contains("packetization Metal encode kernel failure (detail=4)") + || message.contains("packetization Metal encode kernel failure (detail=5)") + || message.contains("packetization Metal encode kernel failure (detail=7, tier1_detail=4)") + || message.contains("packetization Metal encode kernel failure (detail=7, tier1_detail=5)") + || message + .contains("J2K batched codestream assembly Metal encode kernel failure (detail=2)") + || message + .contains("J2K batched codestream assembly Metal encode kernel failure (detail=3)") +} + +pub(super) fn resident_ht_batch_encode_should_retry_conservative(error: &crate::Error) -> bool { + let crate::Error::MetalKernel { message } = error else { + return false; + }; + + message.contains("packetization Metal encode kernel failure (detail=3)") + || message.contains("packetization Metal encode kernel failure (detail=4)") + || message.contains("packetization Metal encode kernel failure (detail=5)") + || message + .contains("HTJ2K batched codestream assembly Metal encode kernel failure (detail=2)") + || message + .contains("HTJ2K batched codestream assembly Metal encode kernel failure (detail=3)") +} diff --git a/crates/j2k-metal/src/encode/resident_types.rs b/crates/j2k-metal/src/encode/resident_types.rs new file mode 100644 index 00000000..58e5ba18 --- /dev/null +++ b/crates/j2k-metal/src/encode/resident_types.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::time::{Duration, Instant}; + +use j2k::J2kLosslessEncodeOptions; +use j2k_native::J2kPacketizationPacketDescriptor; + +use crate::compute; + +use super::{ + submitted::OwnedMetalLosslessEncodeTile, LosslessDeviceEncodePlan, MetalEncodeInputStaging, + MetalEncodedJ2k, MetalLosslessEncodeBatchStats, +}; + +pub(super) struct ResidentLosslessBufferEncodeMetadata { + pub(super) tile: OwnedMetalLosslessEncodeTile, + pub(super) components: u8, + pub(super) bit_depth: u8, + pub(super) bytes_per_pixel: usize, + pub(super) plan: LosslessDeviceEncodePlan, + pub(super) packet_descriptors: Vec, + pub(super) packetization_resolutions: Vec, +} + +pub(super) struct PreparedResidentLosslessBufferEncode { + pub(super) metadata: ResidentLosslessBufferEncodeMetadata, + pub(super) prepared: compute::J2kPreparedLosslessDeviceCodeBlocks, +} + +pub(super) struct PlannedResidentLosslessBufferEncode { + pub(super) index: usize, + pub(super) metadata: ResidentLosslessBufferEncodeMetadata, + pub(super) coefficient_count: usize, + pub(super) bytes_per_sample: u8, + pub(super) estimated_peak_bytes: usize, + #[cfg(test)] + pub(super) failure_injection_index: Option, +} + +impl PlannedResidentLosslessBufferEncode { + pub(super) fn estimated_peak_bytes(&self) -> usize { + self.estimated_peak_bytes + } +} + +pub(super) struct SubmittedResidentLosslessMetalBufferEncodeBatch { + pub(super) options: J2kLosslessEncodeOptions, + pub(super) session: crate::MetalBackendSession, + pub(super) stats: MetalLosslessEncodeBatchStats, + pub(super) encode_started: Instant, + pub(super) tiles: Vec, + pub(super) staging: MetalEncodeInputStaging, + pub(super) kind: SubmittedResidentLosslessMetalBufferEncodeBatchKind, +} + +pub(super) enum SubmittedResidentLosslessMetalBufferEncodeBatchKind { + Empty, + Chunks(Vec), +} + +pub(super) struct SubmittedResidentLosslessMetalBufferEncodeChunk { + pub(super) metadatas: Vec, + pub(super) prepare_durations: Vec, + pub(super) pending: compute::J2kPendingResidentLosslessCodestreamBatch, + pub(super) batch_started: Instant, +} + +pub(super) struct FinishedResidentLosslessBufferEncode { + pub(super) metadata: ResidentLosslessBufferEncodeMetadata, + pub(super) encoded: MetalEncodedJ2k, + pub(super) encode_duration: Duration, + pub(super) gpu_duration: Option, +} diff --git a/crates/j2k-metal/src/encode/roundtrip_validation.rs b/crates/j2k-metal/src/encode/roundtrip_validation.rs new file mode 100644 index 00000000..6bcead42 --- /dev/null +++ b/crates/j2k-metal/src/encode/roundtrip_validation.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::J2kLosslessSamples; + +#[cfg(target_os = "macos")] +use super::validation::validation_pixel_format; +#[cfg(target_os = "macos")] +use crate::compute; +#[cfg(target_os = "macos")] +use j2k_core::DeviceSurface; + +#[cfg(target_os = "macos")] +/// Validate a lossless codestream by decoding it through the default Metal session. +pub fn validate_lossless_roundtrip_on_metal( + samples: J2kLosslessSamples<'_>, + codestream: &[u8], +) -> Result<(), crate::Error> { + let session = crate::MetalBackendSession::system_default()?; + validate_lossless_roundtrip_on_metal_with_session(samples, codestream, &session) +} + +#[cfg(not(target_os = "macos"))] +/// Return `Error::MetalUnavailable` for Metal roundtrip validation on non-macOS. +pub fn validate_lossless_roundtrip_on_metal( + samples: J2kLosslessSamples<'_>, + codestream: &[u8], +) -> Result<(), crate::Error> { + let _ = (samples, codestream); + Err(crate::Error::MetalUnavailable) +} + +#[cfg(target_os = "macos")] +/// Validate a lossless codestream by decoding it through a provided Metal session. +pub fn validate_lossless_roundtrip_on_metal_with_session( + samples: J2kLosslessSamples<'_>, + codestream: &[u8], + session: &crate::MetalBackendSession, +) -> Result<(), crate::Error> { + let fmt = validation_pixel_format(samples)?; + let mut decoder = crate::J2kDecoder::new(codestream)?; + let surface = decoder.decode_to_device_with_session(fmt, session)?; + + if surface.dimensions() != (samples.width, samples.height) { + return Err(crate::Error::MetalKernel { + message: format!( + "J2K Metal validation geometry mismatch: expected {}x{}, got {}x{}", + samples.width, + samples.height, + surface.dimensions().0, + surface.dimensions().1 + ), + }); + } + if surface.pixel_format() != fmt { + return Err(crate::Error::MetalKernel { + message: format!( + "J2K Metal validation format mismatch: expected {:?}, got {:?}", + fmt, + surface.pixel_format() + ), + }); + } + let expected_pitch = samples.width as usize * fmt.bytes_per_pixel(); + if surface.pitch_bytes() != expected_pitch { + return Err(crate::Error::MetalKernel { + message: format!( + "J2K Metal validation pitch mismatch: expected {expected_pitch}, got {}", + surface.pitch_bytes() + ), + }); + } + if surface.byte_len() != samples.data.len() { + return Err(crate::Error::MetalKernel { + message: format!( + "J2K Metal validation length mismatch: expected {} bytes, got {} bytes", + samples.data.len(), + surface.byte_len() + ), + }); + } + + let (buffer, byte_offset) = + surface + .metal_buffer() + .ok_or(crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal validation decode did not return a Metal buffer", + })?; + compute::validate_metal_buffer_matches_bytes(samples.data, buffer, byte_offset, session) +} + +#[cfg(not(target_os = "macos"))] +/// Return `Error::MetalUnavailable` for session validation on non-macOS. +pub fn validate_lossless_roundtrip_on_metal_with_session( + samples: J2kLosslessSamples<'_>, + codestream: &[u8], + session: &crate::MetalBackendSession, +) -> Result<(), crate::Error> { + let _ = (samples, codestream, session); + Err(crate::Error::MetalUnavailable) +} diff --git a/crates/j2k-metal/src/encode/stage_accelerator.rs b/crates/j2k-metal/src/encode/stage_accelerator.rs new file mode 100644 index 00000000..02e63fe7 --- /dev/null +++ b/crates/j2k-metal/src/encode/stage_accelerator.rs @@ -0,0 +1,772 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(target_os = "macos")] +use crate::compute; +use j2k::adapter::encode_stage; +#[cfg(target_os = "macos")] +use j2k::{EncodeBackendPreference, J2kLosslessEncodeOptions}; +#[cfg(target_os = "macos")] +use j2k_core::PixelFormat; + +#[cfg(target_os = "macos")] +use super::{ + borrow_padded_metal_buffer_from_bytes, encode_resident_ht_tile_body_with_cpu_packetization, + lossless_options_for_resident_htj2k_tile_job, should_use_resident_htj2k_host_tile_for_auto, + MetalEncodeInputStaging, MetalLosslessEncodeTile, +}; + +const AUTO_HOST_OUTPUT_STAGE_MIN_PIXELS: usize = 512 * 512; + +/// Encode-stage accelerator for JPEG 2000 Metal work. +/// +/// The type is wired into the public J2K encode-stage interface and reports +/// dispatches for each required encode stage. +#[derive(Debug, Clone)] +pub struct MetalEncodeStageAccelerator { + dispatch_stages: MetalEncodeDispatchStages, + route_profile: MetalEncodeRouteProfile, + parallel_cpu_code_block_fallback: bool, + auto_host_output_force_cpu_fallback: bool, + deinterleave_attempts: usize, + forward_rct_attempts: usize, + forward_ict_attempts: usize, + forward_dwt53_attempts: usize, + forward_dwt97_attempts: usize, + quantize_subband_attempts: usize, + tier1_code_block_attempts: usize, + ht_code_block_attempts: usize, + packetization_attempts: usize, + deinterleave_dispatches: usize, + forward_rct_dispatches: usize, + forward_ict_dispatches: usize, + forward_dwt53_dispatches: usize, + forward_dwt97_dispatches: usize, + quantize_subband_dispatches: usize, + tier1_code_block_dispatches: usize, + ht_code_block_dispatches: usize, + packetization_dispatches: usize, +} + +impl Default for MetalEncodeStageAccelerator { + fn default() -> Self { + Self { + dispatch_stages: MetalEncodeDispatchStages::ALL, + route_profile: MetalEncodeRouteProfile::Explicit, + parallel_cpu_code_block_fallback: false, + auto_host_output_force_cpu_fallback: false, + deinterleave_attempts: 0, + forward_rct_attempts: 0, + forward_ict_attempts: 0, + forward_dwt53_attempts: 0, + forward_dwt97_attempts: 0, + quantize_subband_attempts: 0, + tier1_code_block_attempts: 0, + ht_code_block_attempts: 0, + packetization_attempts: 0, + deinterleave_dispatches: 0, + forward_rct_dispatches: 0, + forward_ict_dispatches: 0, + forward_dwt53_dispatches: 0, + forward_dwt97_dispatches: 0, + quantize_subband_dispatches: 0, + tier1_code_block_dispatches: 0, + ht_code_block_dispatches: 0, + packetization_dispatches: 0, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MetalEncodeRouteProfile { + Explicit, + AutoHostOutput, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct MetalEncodeDispatchStages(u16); + +impl MetalEncodeDispatchStages { + const DEINTERLEAVE: Self = Self(1 << 0); + const FORWARD_RCT: Self = Self(1 << 1); + const FORWARD_ICT: Self = Self(1 << 2); + const FORWARD_DWT53: Self = Self(1 << 3); + const FORWARD_DWT97: Self = Self(1 << 4); + const QUANTIZE_SUBBAND: Self = Self(1 << 5); + const TIER1_CODE_BLOCK: Self = Self(1 << 6); + const HT_CODE_BLOCK: Self = Self(1 << 7); + const PACKETIZATION: Self = Self(1 << 8); + const AUTO_HOST_OUTPUT_STAGE_DISPATCHES: Self = Self( + Self::DEINTERLEAVE.0 + | Self::FORWARD_RCT.0 + | Self::FORWARD_ICT.0 + | Self::FORWARD_DWT53.0 + | Self::FORWARD_DWT97.0 + | Self::QUANTIZE_SUBBAND.0, + ); + const ALL: Self = Self( + Self::DEINTERLEAVE.0 + | Self::FORWARD_RCT.0 + | Self::FORWARD_ICT.0 + | Self::FORWARD_DWT53.0 + | Self::FORWARD_DWT97.0 + | Self::QUANTIZE_SUBBAND.0 + | Self::TIER1_CODE_BLOCK.0 + | Self::HT_CODE_BLOCK.0 + | Self::PACKETIZATION.0, + ); + + fn contains(self, stage: Self) -> bool { + self.0 & stage.0 != 0 + } + + fn without(self, stage: Self) -> Self { + Self(self.0 & !stage.0) + } +} + +impl MetalEncodeStageAccelerator { + /// Create an accelerator that leaves forward RCT on the CPU path. + pub fn with_cpu_forward_rct() -> Self { + Self { + dispatch_stages: MetalEncodeDispatchStages::ALL + .without(MetalEncodeDispatchStages::FORWARD_RCT), + ..Self::default() + } + } + + /// Create the conservative automatic accelerator for host codestream output. + pub fn for_auto_host_output() -> Self { + Self { + dispatch_stages: MetalEncodeDispatchStages::AUTO_HOST_OUTPUT_STAGE_DISPATCHES, + route_profile: MetalEncodeRouteProfile::AutoHostOutput, + parallel_cpu_code_block_fallback: true, + ..Self::default() + } + } + + /// Create an accelerator that only attempts the HT code-block stage on Metal. + pub fn for_ht_code_block_encode() -> Self { + Self { + dispatch_stages: MetalEncodeDispatchStages::HT_CODE_BLOCK, + parallel_cpu_code_block_fallback: true, + ..Self::default() + } + } + + #[cfg(all(test, target_os = "macos"))] + pub(super) fn for_forward_dwt97_encode() -> Self { + Self { + dispatch_stages: MetalEncodeDispatchStages::FORWARD_DWT97, + ..Self::default() + } + } + + #[cfg(target_os = "macos")] + pub(super) fn for_host_output(options: J2kLosslessEncodeOptions) -> Self { + if options.backend == EncodeBackendPreference::Auto { + Self::for_auto_host_output() + } else { + Self::with_cpu_forward_rct() + } + } + + /// Number of deinterleave stage attempts. + pub fn deinterleave_attempts(&self) -> usize { + self.deinterleave_attempts + } + + /// Number of forward RCT stage attempts. + pub fn forward_rct_attempts(&self) -> usize { + self.forward_rct_attempts + } + + /// Number of forward ICT stage attempts. + pub fn forward_ict_attempts(&self) -> usize { + self.forward_ict_attempts + } + + /// Number of forward 5/3 DWT stage attempts. + pub fn forward_dwt53_attempts(&self) -> usize { + self.forward_dwt53_attempts + } + + /// Number of forward 9/7 DWT stage attempts. + pub fn forward_dwt97_attempts(&self) -> usize { + self.forward_dwt97_attempts + } + + /// Number of sub-band quantization stage attempts. + pub fn quantize_subband_attempts(&self) -> usize { + self.quantize_subband_attempts + } + + /// Number of classic Tier-1 code-block encode attempts. + pub fn tier1_code_block_attempts(&self) -> usize { + self.tier1_code_block_attempts + } + + /// Number of HT code-block encode attempts. + pub fn ht_code_block_attempts(&self) -> usize { + self.ht_code_block_attempts + } + + /// Number of packetization stage attempts. + pub fn packetization_attempts(&self) -> usize { + self.packetization_attempts + } + + /// Number of deinterleave Metal dispatches. + pub fn deinterleave_dispatches(&self) -> usize { + self.deinterleave_dispatches + } + + /// Number of forward RCT Metal dispatches. + pub fn forward_rct_dispatches(&self) -> usize { + self.forward_rct_dispatches + } + + /// Number of forward ICT Metal dispatches. + pub fn forward_ict_dispatches(&self) -> usize { + self.forward_ict_dispatches + } + + /// Number of forward 5/3 DWT Metal dispatches. + pub fn forward_dwt53_dispatches(&self) -> usize { + self.forward_dwt53_dispatches + } + + /// Number of forward 9/7 DWT Metal dispatches. + pub fn forward_dwt97_dispatches(&self) -> usize { + self.forward_dwt97_dispatches + } + + /// Number of sub-band quantization Metal dispatches. + pub fn quantize_subband_dispatches(&self) -> usize { + self.quantize_subband_dispatches + } + + /// Number of classic Tier-1 Metal dispatches. + pub fn tier1_code_block_dispatches(&self) -> usize { + self.tier1_code_block_dispatches + } + + /// Number of HT code-block Metal dispatches. + pub fn ht_code_block_dispatches(&self) -> usize { + self.ht_code_block_dispatches + } + + /// Number of packetization Metal dispatches. + pub fn packetization_dispatches(&self) -> usize { + self.packetization_dispatches + } + + fn auto_host_stage_supported_for_len(&self, len: usize) -> bool { + self.route_profile != MetalEncodeRouteProfile::AutoHostOutput + || len >= AUTO_HOST_OUTPUT_STAGE_MIN_PIXELS + } +} + +#[cfg(target_os = "macos")] +fn metal_dispatch_result( + result: &Result<(), crate::Error>, + message: &'static str, +) -> Result { + match result { + Ok(()) => Ok(true), + Err(crate::Error::MetalUnavailable) => Ok(false), + Err(_) => Err(message), + } +} + +#[cfg(target_os = "macos")] +pub(super) fn metal_dispatch_option( + result: Result, + message: &'static str, +) -> Result, &'static str> { + match result { + Ok(value) => Ok(Some(value)), + Err(crate::Error::MetalUnavailable) => Ok(None), + Err(_) => Err(message), + } +} + +impl encode_stage::J2kEncodeStageAccelerator for MetalEncodeStageAccelerator { + fn dispatch_report(&self) -> encode_stage::J2kEncodeDispatchReport { + encode_stage::J2kEncodeDispatchReport { + deinterleave: self.deinterleave_dispatches, + forward_rct: self.forward_rct_dispatches, + forward_ict: self.forward_ict_dispatches, + forward_dwt53: self.forward_dwt53_dispatches, + forward_dwt97: self.forward_dwt97_dispatches, + quantize_subband: self.quantize_subband_dispatches, + tier1_code_block: self.tier1_code_block_dispatches, + ht_code_block: self.ht_code_block_dispatches, + packetization: self.packetization_dispatches, + } + } + + fn prefer_parallel_cpu_code_block_fallback(&self) -> bool { + self.parallel_cpu_code_block_fallback + } + + fn encode_deinterleave( + &mut self, + job: encode_stage::J2kDeinterleaveToF32Job<'_>, + ) -> core::result::Result>>, &'static str> { + self.deinterleave_attempts = self.deinterleave_attempts.saturating_add(1); + if !self + .dispatch_stages + .contains(MetalEncodeDispatchStages::DEINTERLEAVE) + || self.auto_host_output_force_cpu_fallback + || !self.auto_host_stage_supported_for_len(job.num_pixels) + { + let _ = job; + return Ok(None); + } + #[cfg(target_os = "macos")] + { + match compute::encode_deinterleave_to_f32(job) { + Ok(Some(components)) => { + self.deinterleave_dispatches = self.deinterleave_dispatches.saturating_add(1); + Ok(Some(components)) + } + Ok(None) | Err(crate::Error::MetalUnavailable) => Ok(None), + Err(crate::Error::UnsupportedMetalRequest { .. }) => { + Err("Metal deinterleave encode shape is unsupported") + } + Err(_) => Err("Metal deinterleave encode kernel failed"), + } + } + #[cfg(not(target_os = "macos"))] + { + let _ = job; + Ok(None) + } + } + + fn encode_forward_rct( + &mut self, + job: encode_stage::J2kForwardRctJob<'_>, + ) -> core::result::Result { + self.forward_rct_attempts = self.forward_rct_attempts.saturating_add(1); + if !self + .dispatch_stages + .contains(MetalEncodeDispatchStages::FORWARD_RCT) + || self.auto_host_output_force_cpu_fallback + || !self.auto_host_stage_supported_for_len(job.plane0.len()) + { + let _ = job; + return Ok(false); + } + #[cfg(target_os = "macos")] + { + let result = compute::encode_forward_rct(job.plane0, job.plane1, job.plane2); + let dispatched = + metal_dispatch_result(&result, "Metal forward RCT encode kernel failed")?; + if dispatched { + self.forward_rct_dispatches = self.forward_rct_dispatches.saturating_add(1); + } + Ok(dispatched) + } + #[cfg(not(target_os = "macos"))] + { + let _ = job; + Ok(false) + } + } + + fn encode_forward_ict( + &mut self, + job: encode_stage::J2kForwardIctJob<'_>, + ) -> core::result::Result { + self.forward_ict_attempts = self.forward_ict_attempts.saturating_add(1); + if !self + .dispatch_stages + .contains(MetalEncodeDispatchStages::FORWARD_ICT) + || self.auto_host_output_force_cpu_fallback + || !self.auto_host_stage_supported_for_len(job.plane0.len()) + { + let _ = job; + return Ok(false); + } + #[cfg(target_os = "macos")] + { + match compute::encode_forward_ict(job.plane0, job.plane1, job.plane2) { + Ok(()) => { + self.forward_ict_dispatches = self.forward_ict_dispatches.saturating_add(1); + Ok(true) + } + Err(crate::Error::MetalUnavailable) => Ok(false), + Err(crate::Error::UnsupportedMetalRequest { .. }) => { + Err("Metal forward ICT encode shape is unsupported") + } + Err(_) => Err("Metal forward ICT encode kernel failed"), + } + } + #[cfg(not(target_os = "macos"))] + { + let _ = job; + Ok(false) + } + } + + fn encode_forward_dwt53( + &mut self, + job: encode_stage::J2kForwardDwt53Job<'_>, + ) -> core::result::Result, &'static str> { + self.forward_dwt53_attempts = self.forward_dwt53_attempts.saturating_add(1); + if job.num_levels == 0 { + return Ok(None); + } + if self.auto_host_output_force_cpu_fallback { + let _ = job; + return Ok(None); + } + let sample_count = (job.width as usize).saturating_mul(job.height as usize); + if !self.auto_host_stage_supported_for_len(sample_count) { + let _ = job; + return Ok(None); + } + if !self + .dispatch_stages + .contains(MetalEncodeDispatchStages::FORWARD_DWT53) + { + let _ = job; + return Ok(None); + } + #[cfg(target_os = "macos")] + { + let output = metal_dispatch_option( + compute::encode_forward_dwt53(job.samples, job.width, job.height, job.num_levels), + "Metal forward 5/3 DWT encode kernel failed", + )?; + if output.is_some() { + self.forward_dwt53_dispatches = self.forward_dwt53_dispatches.saturating_add(1); + } + Ok(output) + } + #[cfg(not(target_os = "macos"))] + { + let _ = job; + Ok(None) + } + } + + fn encode_forward_dwt97( + &mut self, + job: encode_stage::J2kForwardDwt97Job<'_>, + ) -> core::result::Result, &'static str> { + self.forward_dwt97_attempts = self.forward_dwt97_attempts.saturating_add(1); + if job.num_levels == 0 || (job.width < 2 && job.height < 2) { + return Ok(None); + } + if self.auto_host_output_force_cpu_fallback { + let _ = job; + return Ok(None); + } + let sample_count = (job.width as usize).saturating_mul(job.height as usize); + if !self.auto_host_stage_supported_for_len(sample_count) { + let _ = job; + return Ok(None); + } + if !self + .dispatch_stages + .contains(MetalEncodeDispatchStages::FORWARD_DWT97) + { + let _ = job; + return Ok(None); + } + #[cfg(target_os = "macos")] + { + let output = metal_dispatch_option( + compute::encode_forward_dwt97(job.samples, job.width, job.height, job.num_levels), + "Metal forward 9/7 DWT encode kernel failed", + )?; + if output.is_some() { + self.forward_dwt97_dispatches = self.forward_dwt97_dispatches.saturating_add(1); + } + Ok(output) + } + #[cfg(not(target_os = "macos"))] + { + let _ = job; + Ok(None) + } + } + + fn encode_quantize_subband( + &mut self, + job: encode_stage::J2kQuantizeSubbandJob<'_>, + ) -> core::result::Result>, &'static str> { + self.quantize_subband_attempts = self.quantize_subband_attempts.saturating_add(1); + if !self + .dispatch_stages + .contains(MetalEncodeDispatchStages::QUANTIZE_SUBBAND) + || self.auto_host_output_force_cpu_fallback + || !self.auto_host_stage_supported_for_len(job.coefficients.len()) + { + let _ = job; + return Ok(None); + } + #[cfg(target_os = "macos")] + { + match compute::encode_quantize_subband(job) { + Ok(coefficients) => { + if !coefficients.is_empty() { + self.quantize_subband_dispatches = + self.quantize_subband_dispatches.saturating_add(1); + } + Ok(Some(coefficients)) + } + Err(crate::Error::MetalUnavailable) => Ok(None), + Err(crate::Error::UnsupportedMetalRequest { .. }) => { + Err("Metal quantize_subband encode shape is unsupported") + } + Err(_) => Err("Metal quantize_subband encode kernel failed"), + } + } + #[cfg(not(target_os = "macos"))] + { + let _ = job; + Ok(None) + } + } + + fn encode_tier1_code_block( + &mut self, + job: encode_stage::J2kTier1CodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + self.tier1_code_block_attempts = self.tier1_code_block_attempts.saturating_add(1); + if !self + .dispatch_stages + .contains(MetalEncodeDispatchStages::TIER1_CODE_BLOCK) + { + let _ = job; + return Ok(None); + } + #[cfg(target_os = "macos")] + { + let encoded = metal_dispatch_option( + compute::encode_classic_tier1_code_block(job), + "Metal classic Tier-1 encode kernel failed", + )?; + if encoded.is_some() { + self.tier1_code_block_dispatches = + self.tier1_code_block_dispatches.saturating_add(1); + } + Ok(encoded) + } + #[cfg(not(target_os = "macos"))] + { + let _ = job; + Ok(None) + } + } + + fn encode_tier1_code_blocks( + &mut self, + jobs: &[encode_stage::J2kTier1CodeBlockEncodeJob<'_>], + ) -> core::result::Result>, &'static str> { + self.tier1_code_block_attempts = self.tier1_code_block_attempts.saturating_add(jobs.len()); + if !self + .dispatch_stages + .contains(MetalEncodeDispatchStages::TIER1_CODE_BLOCK) + { + let _ = jobs; + return Ok(None); + } + #[cfg(target_os = "macos")] + { + let encoded = metal_dispatch_option( + compute::encode_classic_tier1_code_blocks(jobs), + "Metal classic Tier-1 encode batch kernel failed", + )?; + if encoded.is_some() && !jobs.is_empty() { + self.tier1_code_block_dispatches = + self.tier1_code_block_dispatches.saturating_add(1); + } + Ok(encoded) + } + #[cfg(not(target_os = "macos"))] + { + let _ = jobs; + Ok(None) + } + } + + fn encode_ht_code_block( + &mut self, + job: encode_stage::J2kHtCodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + self.ht_code_block_attempts = self.ht_code_block_attempts.saturating_add(1); + if !self + .dispatch_stages + .contains(MetalEncodeDispatchStages::HT_CODE_BLOCK) + || self.auto_host_output_force_cpu_fallback + { + let _ = job; + return Ok(None); + } + #[cfg(target_os = "macos")] + { + let encoded = metal_dispatch_option( + compute::encode_ht_cleanup_code_block(job), + "Metal HTJ2K code-block encode kernel failed", + )?; + if encoded.is_some() { + self.ht_code_block_dispatches = self.ht_code_block_dispatches.saturating_add(1); + } + Ok(encoded) + } + #[cfg(not(target_os = "macos"))] + { + let _ = job; + Ok(None) + } + } + + fn encode_ht_code_blocks( + &mut self, + jobs: &[encode_stage::J2kHtCodeBlockEncodeJob<'_>], + ) -> core::result::Result>, &'static str> { + self.ht_code_block_attempts = self.ht_code_block_attempts.saturating_add(jobs.len()); + if !self + .dispatch_stages + .contains(MetalEncodeDispatchStages::HT_CODE_BLOCK) + || self.auto_host_output_force_cpu_fallback + { + let _ = jobs; + return Ok(None); + } + #[cfg(target_os = "macos")] + { + let encoded = metal_dispatch_option( + compute::encode_ht_cleanup_code_blocks(jobs), + "Metal HTJ2K code-block encode batch kernel failed", + )?; + if encoded.is_some() && !jobs.is_empty() { + self.ht_code_block_dispatches = self.ht_code_block_dispatches.saturating_add(1); + } + Ok(encoded) + } + #[cfg(not(target_os = "macos"))] + { + let _ = jobs; + Ok(None) + } + } + + fn encode_htj2k_tile( + &mut self, + job: encode_stage::J2kHtj2kTileEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + #[cfg(target_os = "macos")] + { + if self.route_profile != MetalEncodeRouteProfile::AutoHostOutput { + let _ = job; + return Ok(None); + } + let native_job = job; + self.auto_host_output_force_cpu_fallback = false; + let Some(options) = lossless_options_for_resident_htj2k_tile_job(native_job) else { + return Ok(None); + }; + if !should_use_resident_htj2k_host_tile_for_auto(native_job) { + self.auto_host_output_force_cpu_fallback = true; + return Ok(None); + } + let session = match crate::MetalBackendSession::system_default() { + Ok(session) => session, + Err(crate::Error::MetalUnavailable) => return Ok(None), + Err(_) => return Err("Metal HTJ2K hybrid tile encode session creation failed"), + }; + let source_buffer = match borrow_padded_metal_buffer_from_bytes(&session, job.pixels) { + Ok(buffer) => buffer, + Err(crate::Error::MetalUnavailable) => return Ok(None), + Err(_) => return Err("Metal HTJ2K hybrid tile input buffer creation failed"), + }; + let pitch_bytes = (job.width as usize) + .checked_mul(usize::from(job.num_components)) + .ok_or("Metal HTJ2K hybrid tile pitch overflow")?; + let tile = MetalLosslessEncodeTile { + buffer: &source_buffer, + byte_offset: 0, + width: job.width, + height: job.height, + pitch_bytes, + output_width: job.width, + output_height: job.height, + format: PixelFormat::Rgb8, + }; + let encoded = match encode_resident_ht_tile_body_with_cpu_packetization( + tile, + options, + &session, + MetalEncodeInputStaging::AlreadyPaddedContiguous, + job.code_block_width, + job.code_block_height, + ) { + Ok(Some(encoded)) => encoded, + Ok(None) | Err(crate::Error::MetalUnavailable) => return Ok(None), + Err(_) => return Err("Metal resident HTJ2K hybrid tile encode failed"), + }; + + self.forward_rct_attempts = self.forward_rct_attempts.saturating_add(1); + if encoded.used_fused_rct { + self.forward_rct_dispatches = self.forward_rct_dispatches.saturating_add(1); + } + if encoded.num_decomposition_levels > 0 { + let component_count = usize::from(job.num_components); + self.forward_dwt53_attempts = + self.forward_dwt53_attempts.saturating_add(component_count); + self.forward_dwt53_dispatches = self + .forward_dwt53_dispatches + .saturating_add(encoded.forward_dwt53_dispatches); + } + self.ht_code_block_attempts = self + .ht_code_block_attempts + .saturating_add(encoded.code_block_count); + self.ht_code_block_dispatches = self + .ht_code_block_dispatches + .saturating_add(encoded.ht_code_block_dispatches); + Ok(Some(encoded.tile_data)) + } + #[cfg(not(target_os = "macos"))] + { + let _ = job; + Ok(None) + } + } + + fn encode_packetization( + &mut self, + job: encode_stage::J2kPacketizationEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + self.packetization_attempts = self.packetization_attempts.saturating_add(1); + self.auto_host_output_force_cpu_fallback = false; + if !self + .dispatch_stages + .contains(MetalEncodeDispatchStages::PACKETIZATION) + { + let _ = job; + return Ok(None); + } + #[cfg(target_os = "macos")] + { + let native_job = job; + let encoded = metal_dispatch_option( + compute::encode_tier2_packetization(native_job), + "Metal Tier-2 packetization encode kernel failed", + )?; + if encoded.is_some() { + self.packetization_dispatches = self.packetization_dispatches.saturating_add(1); + } + Ok(encoded) + } + #[cfg(not(target_os = "macos"))] + { + let _ = job; + Ok(None) + } + } +} diff --git a/crates/j2k-metal/src/encode/stats.rs b/crates/j2k-metal/src/encode/stats.rs new file mode 100644 index 00000000..381cb1c3 --- /dev/null +++ b/crates/j2k-metal/src/encode/stats.rs @@ -0,0 +1,710 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::time::Duration; + +#[cfg(target_os = "macos")] +use crate::compute; + +use super::MetalLosslessBufferEncodeOutcome; + +/// Optional resident Metal encode stage timings. +/// +/// API note: this diagnostic report is constructed by this crate. It is not +/// `#[non_exhaustive]`, but adapter releases may add diagnostic fields as the +/// resident encode path gains more profiling detail. +/// +/// Unless a field explicitly says otherwise, timing fields are host-side +/// `Instant` buckets for RCA and should not be read as exact GPU execution +/// elapsed time. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct MetalLosslessEncodeStageStats { + /// Time spent planning the resident encode batch. + pub plan_duration: Duration, + /// Time spent preparing and submitting Metal work. + pub prepare_submit_duration: Duration, + /// Host-side wall time spent preparing resident encode coefficients. + pub coefficient_prep_duration: Duration, + /// Reserved for future finer-grained deinterleave plus RCT profiling. + /// + /// Current resident prep timing is reported in `coefficient_prep_duration`. + pub deinterleave_rct_duration: Duration, + /// Reserved for future finer-grained forward 5/3 DWT profiling. + /// + /// Current resident prep timing is reported in `coefficient_prep_duration`. + pub dwt53_duration: Duration, + /// Reserved for future finer-grained coefficient extraction profiling. + /// + /// Current resident prep timing is reported in `coefficient_prep_duration`. + pub coefficient_extract_duration: Duration, + /// Time spent building HT lookup tables. + pub ht_table_build_duration: Duration, + /// Time spent allocating HT output buffers. + pub ht_buffer_allocation_duration: Duration, + /// Host-side Metal command encoding time for HT resident command buffers. + /// + /// This is the sum of the split command-encode buckets below and is not GPU + /// kernel execution elapsed time. + pub ht_command_encode_duration: Duration, + /// Host-side Metal command encoding time for HT code-block dispatch setup. + pub ht_block_encode_duration: Duration, + /// CPU-side setup time for classic Tier-1 batch jobs and buffers. + pub classic_tier1_setup_duration: Duration, + /// Host-side Metal command encoding time for classic code-block dispatch setup. + pub classic_block_encode_duration: Duration, + /// Host-side CPU time spent packing compact classic Tier-1 tokens. + /// + /// This is populated only when + /// `J2K_METAL_PROFILE_CLASSIC_TIER1_TOKEN_PACK=1` is enabled. + pub classic_tier1_token_pack_duration: Duration, + /// CPU-side packet metadata planning time for classic resident batches. + pub classic_packet_plan_duration: Duration, + /// CPU-side packet/codestream buffer setup time for classic resident batches. + pub classic_packet_buffer_setup_duration: Duration, + /// Host-side time spent committing split classic resident command buffers. + pub classic_command_buffer_commit_duration: Duration, + /// Host-side wall time spent harvesting completed resident batch results. + pub result_harvest_duration: Duration, + /// Host-side time spent copying shared status buffers into CPU-owned status arrays. + pub result_status_copy_duration: Duration, + /// Host-side time spent returning private buffers to the resident buffer pool. + pub result_private_recycle_duration: Duration, + /// Host-side time spent returning shared buffers to the resident buffer pool. + pub result_shared_recycle_duration: Duration, + /// Host-side time spent validating per-tile status and building codestream handles. + pub result_codestream_collect_duration: Duration, + /// Host-side Metal command encoding time for packet block metadata dispatch setup. + pub packet_block_prep_duration: Duration, + /// Host-side Metal command encoding time for packet body dispatch setup. + pub packetization_duration: Duration, + /// Host-side Metal command encoding time for codestream assembly dispatch setup. + pub codestream_assembly_duration: Duration, + /// GPU time spent preparing resident coefficient buffers. + /// + /// This includes the resident input deinterleave/RCT, DWT, and coefficient + /// extraction command buffer when stage profiling is enabled. + pub coefficient_prep_gpu_duration: Duration, + /// GPU time spent deinterleaving resident input planes and applying RCT. + /// + /// This is populated only when resident coefficient-prep split profiling is enabled. + pub coefficient_deinterleave_rct_gpu_duration: Duration, + /// GPU time spent running resident forward DWT 5/3 coefficient prep. + /// + /// This is populated only when resident coefficient-prep split profiling is enabled. + pub coefficient_dwt53_gpu_duration: Duration, + /// GPU time spent in resident forward DWT 5/3 vertical passes. + /// + /// This is populated only when resident coefficient-prep split profiling is enabled. + pub coefficient_dwt53_vertical_gpu_duration: Duration, + /// GPU time spent in resident forward DWT 5/3 horizontal passes. + /// + /// This is populated only when resident coefficient-prep split profiling is enabled. + pub coefficient_dwt53_horizontal_gpu_duration: Duration, + /// GPU time spent extracting resident code-block coefficients. + /// + /// This is populated only when resident coefficient-prep split profiling is enabled. + pub coefficient_extract_gpu_duration: Duration, + /// GPU time spent copying per-tile coefficient buffers into a batch buffer. + /// + /// This is populated only when resident split-command profiling is enabled. + pub coefficient_copy_gpu_duration: Duration, + /// Elapsed GPU timestamp window across the resident encode command buffers. + /// + /// This is `max(GPUEndTime) - min(GPUStartTime)` for the command buffers + /// retained by the batch. It is a wall-window companion to summed GPU busy + /// rows and should not be added to per-stage GPU durations. + pub gpu_elapsed_wall_duration: Duration, + /// GPU time spent in the classic Tier-1 code-block encode command. + /// + /// This is populated only when classic split-command profiling is enabled. + pub classic_block_gpu_duration: Duration, + /// GPU time spent in the profile-only classic Tier-1 density probe. + /// + /// This is populated only when classic split-command profiling and + /// `J2K_METAL_PROFILE_CLASSIC_TIER1_DENSITY=1` are enabled. + pub classic_tier1_density_gpu_duration: Duration, + /// GPU time spent in the profile-only classic Tier-1 raw bypass packing probe. + /// + /// This is populated only when classic split-command profiling and + /// `J2K_METAL_PROFILE_CLASSIC_TIER1_RAW_PACK=1` are enabled. + pub classic_tier1_raw_pack_gpu_duration: Duration, + /// GPU time spent in the profile-only classic Tier-1 MQ arithmetic packing probe. + /// + /// This is populated only when classic split-command profiling and + /// `J2K_METAL_PROFILE_CLASSIC_TIER1_ARITHMETIC_PACK=1` are enabled. + pub classic_tier1_arithmetic_pack_gpu_duration: Duration, + /// GPU time spent in the profile-only classic Tier-1 ordered symbol-plan probe. + /// + /// This is populated only when classic split-command profiling and + /// `J2K_METAL_PROFILE_CLASSIC_TIER1_SYMBOL_PLAN=1` are enabled. + pub classic_tier1_symbol_plan_gpu_duration: Duration, + /// GPU time spent in the profile-only classic Tier-1 pass-plan probe. + /// + /// This is populated only when classic split-command profiling and + /// `J2K_METAL_PROFILE_CLASSIC_TIER1_PASS_PLAN=1` are enabled. + pub classic_tier1_pass_plan_gpu_duration: Duration, + /// GPU time spent in the profile-only classic Tier-1 compact token-emitter probe. + /// + /// This is populated only when classic split-command profiling and + /// `J2K_METAL_PROFILE_CLASSIC_TIER1_TOKEN_EMIT=1` are enabled. + pub classic_tier1_token_emit_gpu_duration: Duration, + /// GPU time spent in the profile-only classic Tier-1 split MQ/raw token-emitter probe. + /// + /// This is populated only when classic split-command profiling and + /// `J2K_METAL_PROFILE_CLASSIC_TIER1_SPLIT_TOKEN_EMIT=1` are enabled. + pub classic_tier1_split_token_emit_gpu_duration: Duration, + /// GPU time spent packing compact classic Tier-1 tokens into resident payloads. + /// + /// This is populated when the gated classic GPU token-pack route is enabled. + pub classic_tier1_token_pack_gpu_duration: Duration, + /// GPU time spent in the HT Tier-1 code-block encode command. + /// + /// This is populated only when HT split-command profiling is enabled. + pub ht_block_gpu_duration: Duration, + /// GPU time spent preparing packet-block metadata from HT Tier-1 status. + /// + /// This is populated only when HT split-command profiling is enabled. + pub packet_block_prep_gpu_duration: Duration, + /// GPU time spent in HTJ2K packetization. + /// + /// This is populated only when HT split-command profiling is enabled. + pub packetization_gpu_duration: Duration, + /// GPU time spent copying packet payload bytes after header packetization. + /// + /// This is populated only when HT split-command profiling is enabled. + pub packet_payload_copy_gpu_duration: Duration, + /// GPU time spent assembling the HTJ2K codestream buffer. + /// + /// This is populated only when HT split-command profiling is enabled. + pub codestream_assembly_gpu_duration: Duration, + /// GPU time spent copying packet payload bytes into final codestream buffers. + /// + /// This is populated only when HT split-command profiling is enabled. + pub codestream_payload_copy_gpu_duration: Duration, + /// Total Tier-1 output capacity, in bytes, across resident code blocks. + pub tier1_output_capacity_total: usize, + /// Maximum Tier-1 output capacity, in bytes, for any resident code block. + pub max_tier1_output_capacity: usize, + /// Actual Tier-1 output bytes written across resident code blocks. + pub tier1_output_used_bytes_total: usize, + /// Maximum actual Tier-1 output bytes written by any resident code block. + pub max_tier1_output_used_bytes: usize, + /// Total Tier-1 segment metadata capacity across resident code blocks. + pub tier1_segment_capacity_total: usize, + /// Maximum Tier-1 segment metadata capacity for any resident code block. + pub max_tier1_segment_capacity_per_block: usize, + /// Actual Tier-1 coding passes emitted across resident code blocks. + pub tier1_coding_pass_count_total: usize, + /// Maximum actual Tier-1 coding passes emitted by any resident code block. + pub max_tier1_coding_passes_per_block: usize, + /// Estimated classic MQ/arithmetic coding passes across resident code blocks. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_arithmetic_pass_count_total: usize, + /// Estimated classic raw bypass coding passes across resident code blocks. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_raw_pass_count_total: usize, + /// Estimated classic cleanup passes across resident code blocks. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_cleanup_pass_count_total: usize, + /// Estimated classic significance propagation passes across resident code blocks. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_sigprop_pass_count_total: usize, + /// Estimated classic magnitude refinement passes across resident code blocks. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_magref_pass_count_total: usize, + /// Estimated classic MQ/arithmetic cleanup passes across resident code blocks. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_arithmetic_cleanup_pass_count_total: usize, + /// Estimated classic MQ/arithmetic significance propagation passes. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_arithmetic_sigprop_pass_count_total: usize, + /// Estimated classic MQ/arithmetic magnitude refinement passes. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_arithmetic_magref_pass_count_total: usize, + /// Estimated classic raw bypass significance propagation passes. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_raw_sigprop_pass_count_total: usize, + /// Estimated classic raw bypass magnitude refinement passes. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_raw_magref_pass_count_total: usize, + /// Estimated full coefficient visits made by classic Tier-1 pass scans. + /// + /// This is derived from actual emitted pass counts and code-block areas. + /// For HTJ2K Tier-1 this remains zero. + pub tier1_full_scan_coeff_visit_count_total: usize, + /// Estimated full coefficient visits made by MQ/arithmetic pass scans. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_arithmetic_scan_coeff_visit_count_total: usize, + /// Estimated full coefficient visits made by raw bypass pass scans. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_raw_scan_coeff_visit_count_total: usize, + /// Estimated full coefficient visits made by cleanup pass scans. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_cleanup_scan_coeff_visit_count_total: usize, + /// Estimated full coefficient visits made by significance propagation scans. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_sigprop_scan_coeff_visit_count_total: usize, + /// Estimated full coefficient visits made by magnitude refinement scans. + /// + /// For HTJ2K Tier-1 this remains zero. + pub tier1_magref_scan_coeff_visit_count_total: usize, + /// Maximum estimated full coefficient scan visits for any classic block. + /// + /// For HTJ2K Tier-1 this remains zero. + pub max_tier1_full_scan_coeff_visits_per_block: usize, + /// Profile-only count of classic significance propagation candidates. + /// + /// This is populated only when classic Tier-1 density profiling is enabled. + pub tier1_sigprop_active_candidate_count_total: usize, + /// Profile-only count of coefficients that become significant in sigprop. + /// + /// This is populated only when classic Tier-1 density profiling is enabled. + pub tier1_sigprop_new_significant_count_total: usize, + /// Profile-only count of classic magnitude refinement candidates. + /// + /// This is populated only when classic Tier-1 density profiling is enabled. + pub tier1_magref_active_candidate_count_total: usize, + /// Profile-only count of arithmetic-coded significance propagation candidates. + pub tier1_arithmetic_sigprop_active_candidate_count_total: usize, + /// Profile-only count of coefficients that become significant in arithmetic sigprop. + pub tier1_arithmetic_sigprop_new_significant_count_total: usize, + /// Profile-only count of raw bypass significance propagation candidates. + pub tier1_raw_sigprop_active_candidate_count_total: usize, + /// Profile-only count of coefficients that become significant in raw sigprop. + pub tier1_raw_sigprop_new_significant_count_total: usize, + /// Profile-only count of arithmetic-coded magnitude refinement candidates. + pub tier1_arithmetic_magref_active_candidate_count_total: usize, + /// Profile-only count of raw bypass magnitude refinement candidates. + pub tier1_raw_magref_active_candidate_count_total: usize, + /// Profile-only count of cleanup-pass coefficient candidates. + /// + /// This excludes coefficients represented only by cleanup RLC stripes. + pub tier1_cleanup_active_candidate_count_total: usize, + /// Profile-only count of coefficients that become significant in cleanup. + /// + /// This includes significance discovered through cleanup RLC. + pub tier1_cleanup_new_significant_count_total: usize, + /// Profile-only count of cleanup stripes encoded by the RLC path. + pub tier1_cleanup_rlc_stripe_count_total: usize, + /// Profile-only count of cleanup RLC stripes with no significant coefficient. + pub tier1_cleanup_rlc_zero_stripe_count_total: usize, + /// Profile-only exact MQ symbol count from the ordered symbol-plan probe. + pub tier1_symbol_plan_mq_symbol_count_total: usize, + /// Profile-only exact raw bypass bit count from the ordered symbol-plan probe. + pub tier1_symbol_plan_raw_bit_count_total: usize, + /// Maximum MQ symbols emitted by any block in the ordered symbol-plan probe. + pub max_tier1_symbol_plan_mq_symbols_per_block: usize, + /// Maximum raw bypass bits emitted by any block in the ordered symbol-plan probe. + pub max_tier1_symbol_plan_raw_bits_per_block: usize, + /// Estimated compact token bytes needed for all blocks in the symbol-plan probe. + pub tier1_symbol_plan_packed_token_bytes_total: usize, + /// Maximum estimated compact token bytes needed by any one block. + pub max_tier1_symbol_plan_packed_token_bytes_per_block: usize, + /// Profile-only exact cleanup MQ symbol count from the ordered symbol-plan probe. + pub tier1_symbol_plan_cleanup_mq_symbol_count_total: usize, + /// Profile-only exact sigprop MQ symbol count from the ordered symbol-plan probe. + pub tier1_symbol_plan_sigprop_mq_symbol_count_total: usize, + /// Profile-only exact magref MQ symbol count from the ordered symbol-plan probe. + pub tier1_symbol_plan_magref_mq_symbol_count_total: usize, + /// Profile-only exact raw sigprop bit count from the ordered symbol-plan probe. + pub tier1_symbol_plan_raw_sigprop_bit_count_total: usize, + /// Profile-only exact raw magref bit count from the ordered symbol-plan probe. + pub tier1_symbol_plan_raw_magref_bit_count_total: usize, + /// Profile-only cleanup sign-symbol count from the ordered symbol-plan probe. + pub tier1_symbol_plan_cleanup_sign_symbol_count_total: usize, + /// Profile-only sigprop sign-symbol count from the ordered symbol-plan probe. + pub tier1_symbol_plan_sigprop_sign_symbol_count_total: usize, + /// XOR of per-block order-sensitive MQ symbol hashes from the symbol-plan probe. + pub tier1_symbol_plan_mq_symbol_hash_xor: usize, + /// XOR of per-block order-sensitive raw bit hashes from the symbol-plan probe. + pub tier1_symbol_plan_raw_bit_hash_xor: usize, + /// Profile-only MQ symbols counted by coding-pass index. + pub tier1_pass_plan_mq_symbol_count_total: usize, + /// Profile-only raw bypass bits counted by coding-pass index. + pub tier1_pass_plan_raw_bit_count_total: usize, + /// Count of block-local coding passes that emit at least one MQ symbol. + pub tier1_pass_plan_nonempty_mq_pass_count_total: usize, + /// Count of block-local coding passes that emit at least one raw bypass bit. + pub tier1_pass_plan_nonempty_raw_pass_count_total: usize, + /// Maximum MQ symbols emitted by any single block-local coding pass. + pub max_tier1_pass_plan_mq_symbols_per_pass: usize, + /// Maximum raw bypass bits emitted by any single block-local coding pass. + pub max_tier1_pass_plan_raw_bits_per_pass: usize, + /// Exact MQ symbol count from the compact token-emitter probe or gated GPU token-pack route. + pub tier1_token_emit_mq_symbol_count_total: usize, + /// Exact raw bypass bit count from the compact token-emitter probe or gated GPU token-pack route. + pub tier1_token_emit_raw_bit_count_total: usize, + /// Compact token bytes emitted by the token-emitter probe or gated GPU token-pack route. + pub tier1_token_emit_token_bytes_total: usize, + /// Maximum compact token bytes emitted by any one block. + pub max_tier1_token_emit_token_bytes_per_block: usize, + /// Segment records emitted by the token-emitter probe or gated GPU token-pack route. + pub tier1_token_emit_segment_count_total: usize, + /// Maximum token-emitter segment records for any one block. + pub max_tier1_token_emit_segments_per_block: usize, + /// XOR of per-block order-sensitive MQ symbol hashes from token emission. + pub tier1_token_emit_mq_symbol_hash_xor: usize, + /// XOR of per-block order-sensitive raw bit hashes from token emission. + pub tier1_token_emit_raw_bit_hash_xor: usize, + /// Total bytes produced by packing emitted Tier-1 tokens. + pub tier1_token_pack_output_bytes_total: usize, + /// Maximum token-pack output bytes for any one block. + pub max_tier1_token_pack_output_bytes_per_block: usize, + /// Resident Tier-1 code blocks that emitted at least one coding pass. + pub tier1_nonzero_block_count_total: usize, + /// Resident Tier-1 code blocks that emitted no coding passes. + pub tier1_zero_block_count_total: usize, + /// Missing most-significant bitplanes across resident Tier-1 code blocks. + pub tier1_missing_bitplane_count_total: usize, + /// Maximum missing most-significant bitplanes for any resident code block. + pub max_tier1_missing_bitplanes_per_block: usize, + /// Classic Tier-1 segment records emitted across resident code blocks. + /// + /// This remains zero for HTJ2K Tier-1, which does not use classic segment + /// records. + pub tier1_segment_count_total: usize, + /// Maximum classic Tier-1 segment records emitted by any resident code block. + pub max_tier1_segments_per_block: usize, + /// Total host-planned packet payload-copy job slots across resident chunks. + pub packet_payload_copy_job_capacity_total: usize, + /// Maximum packet payload-copy job slots needed by any tile in the batch. + pub max_packet_payload_copy_jobs_per_tile: usize, + /// Actual packet payload-copy jobs emitted by packetization across resident chunks. + pub packet_payload_copy_job_count_total: usize, + /// Maximum actual packet payload-copy jobs emitted by any tile in the batch. + pub max_packet_payload_copy_jobs_used_per_tile: usize, + /// Actual packet payload-copy bytes emitted by packetization across resident chunks. + pub packet_payload_copy_bytes_total: usize, + /// Maximum actual packet payload-copy bytes emitted by any tile in the batch. + pub max_packet_payload_copy_bytes_per_tile: usize, + /// Packet payload-copy jobs at or below one copy-kernel stripe. + pub packet_payload_copy_small_job_count_total: usize, + /// Packet payload-copy jobs above one stripe and at or below 512 bytes. + pub packet_payload_copy_medium_job_count_total: usize, + /// Packet payload-copy jobs above 512 bytes. + pub packet_payload_copy_large_job_count_total: usize, + /// Packet payload-copy stripes launched by the copy kernel. + pub packet_payload_copy_launched_stripe_count_total: usize, + /// Packet payload-copy stripes that correspond to emitted copy jobs. + pub packet_payload_copy_active_stripe_count_total: usize, + /// Total packet output capacity, in bytes, across resident chunks. + pub packet_output_capacity_total: usize, + /// Maximum packet output capacity, in bytes, for any tile in the batch. + pub max_packet_output_capacity: usize, + /// Actual packet output bytes written by packetization across resident chunks. + pub packet_output_used_bytes_total: usize, + /// Maximum actual packet output bytes written by any tile in the batch. + pub max_packet_output_used_bytes: usize, + /// Codestream payload-copy bytes, in bytes, across resident chunks. + pub codestream_payload_copy_bytes_total: usize, + /// Codestream payload-copy threads launched by the copy kernel. + pub codestream_payload_copy_launched_thread_count_total: usize, + /// Estimated codestream payload-copy threads with in-range bytes to copy. + pub codestream_payload_copy_active_thread_count_total: usize, + /// Time spent waiting for codestream buffers. + pub codestream_wait_duration: Duration, + /// Alias of `codestream_wait_duration` using RCA naming. + /// + /// Do not sum this with `codestream_wait_duration` as an independent bucket. + pub sync_wait_duration: Duration, + /// Time spent materializing buffer-backed codestream bytes into host bytes. + /// + /// Current batch stats paths may leave this at zero. Host byte + /// materialization timing is surfaced on `MetalLosslessEncodeOutcome` where + /// applicable; this stage-stats bucket is reserved for stats-bearing + /// host-output paths. + pub host_readback_duration: Duration, + /// Number of resident encode chunks. + pub chunk_count: usize, + /// Number of encoded tiles. + pub tile_count: usize, + /// Number of encoded code blocks. + pub code_block_count: usize, +} + +/// Combine rule for one stage-stat field in +/// [`MetalLosslessEncodeStageStats::add_assign`]: `dur` and `count` add with +/// saturation, `max` keeps the per-batch maximum, `xor` folds hashes. +macro_rules! stage_stat_combine { + (dur, $self:ident, $other:ident, $field:ident) => { + $self.$field = $self.$field.saturating_add($other.$field); + }; + (count, $self:ident, $other:ident, $field:ident) => { + $self.$field = $self.$field.saturating_add($other.$field); + }; + (max, $self:ident, $other:ident, $field:ident) => { + $self.$field = $self.$field.max($other.$field); + }; + (xor, $self:ident, $other:ident, $field:ident) => { + $self.$field ^= $other.$field; + }; +} + +/// Contribution of one stage-stat field to +/// [`MetalLosslessEncodeStageStats::has_timings`]: only `dur` fields count. +macro_rules! stage_stat_timing_flag { + (dur, $any:ident, $self:ident, $field:ident) => { + $any = $any || $self.$field > Duration::ZERO; + }; + ($class:ident, $any:ident, $self:ident, $field:ident) => {}; +} + +/// `From` rule for one stage-stat +/// field: `resident` fields copy from the compute-layer stats, `local` +/// fields are facade-side only and keep their default. +#[cfg(target_os = "macos")] +macro_rules! stage_stat_from_resident { + (resident, $out:ident, $stats:ident, $field:ident) => { + $out.$field = $stats.$field; + }; + (local, $out:ident, $stats:ident, $field:ident) => {}; +} + +/// Generate the per-field `MetalLosslessEncodeStageStats` impls from the +/// field table. The destructuring check at the end makes the table +/// exhaustive: adding a struct field without a table entry fails to compile. +macro_rules! j2k_metal_stage_stats_impls { + ($(($field:ident, $class:ident, $source:ident)),* $(,)?) => { + impl MetalLosslessEncodeStageStats { + /// Return whether any non-zero timing was recorded. + pub fn has_timings(&self) -> bool { + let mut any = false; + $(stage_stat_timing_flag!($class, any, self, $field);)* + any + } + + /// Accumulate another stage-stats value using saturating duration and counter additions. + pub fn add_assign(&mut self, other: Self) { + $(stage_stat_combine!($class, self, other, $field);)* + } + } + + #[cfg(target_os = "macos")] + impl From for MetalLosslessEncodeStageStats { + fn from(stats: compute::J2kResidentEncodeStageStats) -> Self { + let mut out = Self::default(); + $(stage_stat_from_resident!($source, out, stats, $field);)* + out + } + } + + const _: fn(MetalLosslessEncodeStageStats) = |stats| { + let MetalLosslessEncodeStageStats { $($field: _),* } = stats; + }; + }; +} + +j2k_metal_stage_stats_impls! { + (plan_duration, dur, local), + (prepare_submit_duration, dur, local), + (coefficient_prep_duration, dur, resident), + (deinterleave_rct_duration, dur, resident), + (dwt53_duration, dur, resident), + (coefficient_extract_duration, dur, resident), + (ht_table_build_duration, dur, resident), + (ht_buffer_allocation_duration, dur, resident), + (ht_command_encode_duration, dur, resident), + (ht_block_encode_duration, dur, resident), + (classic_tier1_setup_duration, dur, resident), + (classic_block_encode_duration, dur, resident), + (classic_tier1_token_pack_duration, dur, resident), + (classic_packet_plan_duration, dur, resident), + (classic_packet_buffer_setup_duration, dur, resident), + (classic_command_buffer_commit_duration, dur, resident), + (result_harvest_duration, dur, resident), + (result_status_copy_duration, dur, resident), + (result_private_recycle_duration, dur, resident), + (result_shared_recycle_duration, dur, resident), + (result_codestream_collect_duration, dur, resident), + (packet_block_prep_duration, dur, resident), + (packetization_duration, dur, resident), + (codestream_assembly_duration, dur, resident), + (coefficient_prep_gpu_duration, dur, resident), + (coefficient_deinterleave_rct_gpu_duration, dur, resident), + (coefficient_dwt53_gpu_duration, dur, resident), + (coefficient_dwt53_vertical_gpu_duration, dur, resident), + (coefficient_dwt53_horizontal_gpu_duration, dur, resident), + (coefficient_extract_gpu_duration, dur, resident), + (coefficient_copy_gpu_duration, dur, resident), + (gpu_elapsed_wall_duration, dur, resident), + (classic_block_gpu_duration, dur, resident), + (classic_tier1_density_gpu_duration, dur, resident), + (classic_tier1_raw_pack_gpu_duration, dur, resident), + (classic_tier1_arithmetic_pack_gpu_duration, dur, resident), + (classic_tier1_symbol_plan_gpu_duration, dur, resident), + (classic_tier1_pass_plan_gpu_duration, dur, resident), + (classic_tier1_token_emit_gpu_duration, dur, resident), + (classic_tier1_split_token_emit_gpu_duration, dur, resident), + (classic_tier1_token_pack_gpu_duration, dur, resident), + (ht_block_gpu_duration, dur, resident), + (packet_block_prep_gpu_duration, dur, resident), + (packetization_gpu_duration, dur, resident), + (packet_payload_copy_gpu_duration, dur, resident), + (codestream_assembly_gpu_duration, dur, resident), + (codestream_payload_copy_gpu_duration, dur, resident), + (tier1_output_capacity_total, count, resident), + (max_tier1_output_capacity, max, resident), + (tier1_output_used_bytes_total, count, resident), + (max_tier1_output_used_bytes, max, resident), + (tier1_segment_capacity_total, count, resident), + (max_tier1_segment_capacity_per_block, max, resident), + (tier1_coding_pass_count_total, count, resident), + (max_tier1_coding_passes_per_block, max, resident), + (tier1_arithmetic_pass_count_total, count, resident), + (tier1_raw_pass_count_total, count, resident), + (tier1_cleanup_pass_count_total, count, resident), + (tier1_sigprop_pass_count_total, count, resident), + (tier1_magref_pass_count_total, count, resident), + (tier1_arithmetic_cleanup_pass_count_total, count, resident), + (tier1_arithmetic_sigprop_pass_count_total, count, resident), + (tier1_arithmetic_magref_pass_count_total, count, resident), + (tier1_raw_sigprop_pass_count_total, count, resident), + (tier1_raw_magref_pass_count_total, count, resident), + (tier1_full_scan_coeff_visit_count_total, count, resident), + (tier1_arithmetic_scan_coeff_visit_count_total, count, resident), + (tier1_raw_scan_coeff_visit_count_total, count, resident), + (tier1_cleanup_scan_coeff_visit_count_total, count, resident), + (tier1_sigprop_scan_coeff_visit_count_total, count, resident), + (tier1_magref_scan_coeff_visit_count_total, count, resident), + (max_tier1_full_scan_coeff_visits_per_block, max, resident), + (tier1_sigprop_active_candidate_count_total, count, resident), + (tier1_sigprop_new_significant_count_total, count, resident), + (tier1_magref_active_candidate_count_total, count, resident), + (tier1_arithmetic_sigprop_active_candidate_count_total, count, resident), + (tier1_arithmetic_sigprop_new_significant_count_total, count, resident), + (tier1_raw_sigprop_active_candidate_count_total, count, resident), + (tier1_raw_sigprop_new_significant_count_total, count, resident), + (tier1_arithmetic_magref_active_candidate_count_total, count, resident), + (tier1_raw_magref_active_candidate_count_total, count, resident), + (tier1_cleanup_active_candidate_count_total, count, resident), + (tier1_cleanup_new_significant_count_total, count, resident), + (tier1_cleanup_rlc_stripe_count_total, count, resident), + (tier1_cleanup_rlc_zero_stripe_count_total, count, resident), + (tier1_symbol_plan_mq_symbol_count_total, count, resident), + (tier1_symbol_plan_raw_bit_count_total, count, resident), + (max_tier1_symbol_plan_mq_symbols_per_block, max, resident), + (max_tier1_symbol_plan_raw_bits_per_block, max, resident), + (tier1_symbol_plan_packed_token_bytes_total, count, resident), + (max_tier1_symbol_plan_packed_token_bytes_per_block, max, resident), + (tier1_symbol_plan_cleanup_mq_symbol_count_total, count, resident), + (tier1_symbol_plan_sigprop_mq_symbol_count_total, count, resident), + (tier1_symbol_plan_magref_mq_symbol_count_total, count, resident), + (tier1_symbol_plan_raw_sigprop_bit_count_total, count, resident), + (tier1_symbol_plan_raw_magref_bit_count_total, count, resident), + (tier1_symbol_plan_cleanup_sign_symbol_count_total, count, resident), + (tier1_symbol_plan_sigprop_sign_symbol_count_total, count, resident), + (tier1_symbol_plan_mq_symbol_hash_xor, xor, resident), + (tier1_symbol_plan_raw_bit_hash_xor, xor, resident), + (tier1_pass_plan_mq_symbol_count_total, count, resident), + (tier1_pass_plan_raw_bit_count_total, count, resident), + (tier1_pass_plan_nonempty_mq_pass_count_total, count, resident), + (tier1_pass_plan_nonempty_raw_pass_count_total, count, resident), + (max_tier1_pass_plan_mq_symbols_per_pass, max, resident), + (max_tier1_pass_plan_raw_bits_per_pass, max, resident), + (tier1_token_emit_mq_symbol_count_total, count, resident), + (tier1_token_emit_raw_bit_count_total, count, resident), + (tier1_token_emit_token_bytes_total, count, resident), + (max_tier1_token_emit_token_bytes_per_block, max, resident), + (tier1_token_emit_segment_count_total, count, resident), + (max_tier1_token_emit_segments_per_block, max, resident), + (tier1_token_emit_mq_symbol_hash_xor, xor, resident), + (tier1_token_emit_raw_bit_hash_xor, xor, resident), + (tier1_token_pack_output_bytes_total, count, resident), + (max_tier1_token_pack_output_bytes_per_block, max, resident), + (tier1_nonzero_block_count_total, count, resident), + (tier1_zero_block_count_total, count, resident), + (tier1_missing_bitplane_count_total, count, resident), + (max_tier1_missing_bitplanes_per_block, max, resident), + (tier1_segment_count_total, count, resident), + (max_tier1_segments_per_block, max, resident), + (packet_payload_copy_job_capacity_total, count, resident), + (max_packet_payload_copy_jobs_per_tile, max, resident), + (packet_payload_copy_job_count_total, count, resident), + (max_packet_payload_copy_jobs_used_per_tile, max, resident), + (packet_payload_copy_bytes_total, count, resident), + (max_packet_payload_copy_bytes_per_tile, max, resident), + (packet_payload_copy_small_job_count_total, count, resident), + (packet_payload_copy_medium_job_count_total, count, resident), + (packet_payload_copy_large_job_count_total, count, resident), + (packet_payload_copy_launched_stripe_count_total, count, resident), + (packet_payload_copy_active_stripe_count_total, count, resident), + (packet_output_capacity_total, count, resident), + (max_packet_output_capacity, max, resident), + (packet_output_used_bytes_total, count, resident), + (max_packet_output_used_bytes, max, resident), + (codestream_payload_copy_bytes_total, count, resident), + (codestream_payload_copy_launched_thread_count_total, count, resident), + (codestream_payload_copy_active_thread_count_total, count, resident), + (codestream_wait_duration, dur, local), + (sync_wait_duration, dur, local), + (host_readback_duration, dur, local), + (chunk_count, count, local), + (tile_count, count, local), + (code_block_count, count, resident), +} + +#[cfg(any(target_os = "macos", test))] +pub(super) fn add_resident_prep_duration( + stats: &mut MetalLosslessEncodeBatchStats, + duration: Duration, + profile_stages: bool, +) { + if !profile_stages { + return; + } + stats.stage_stats.coefficient_prep_duration = stats + .stage_stats + .coefficient_prep_duration + .saturating_add(duration); +} + +#[cfg(any(target_os = "macos", test))] +pub(super) fn add_resident_prep_wall_duration( + stats: &mut MetalLosslessEncodeBatchStats, + wall_duration: Duration, + profile_stages: bool, +) { + add_resident_prep_duration(stats, wall_duration, profile_stages); +} + +/// Resolved resident Metal lossless J2K/HTJ2K tile batch encode metrics. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct MetalLosslessEncodeBatchStats { + /// Caller-requested maximum number of in-flight tiles. + pub configured_inflight_tiles: Option, + /// Effective maximum number of in-flight tiles after clamping. + pub effective_inflight_tiles: usize, + /// Caller-requested resident encode memory budget in bytes. + pub configured_memory_budget_bytes: Option, + /// Effective resident encode memory budget in bytes. + pub effective_memory_budget_bytes: usize, + /// Estimated peak resident memory required per tile. + pub estimated_peak_bytes_per_tile: usize, + /// Maximum observed in-flight tiles during the batch. + pub max_observed_inflight_tiles: usize, + /// End-to-end wall time for the batch encode. + pub encode_wall_duration: Duration, + /// Resident encode stage timing summary. + pub stage_stats: MetalLosslessEncodeStageStats, +} + +/// Resident Metal lossless J2K/HTJ2K tile batch output and batch-level metrics. +pub struct MetalLosslessBufferEncodeBatchOutcome { + /// Per-tile buffer-backed encode outcomes. + pub outcomes: Vec, + /// Batch-level resident encode metrics. + pub stats: MetalLosslessEncodeBatchStats, +} diff --git a/crates/j2k-metal/src/encode/submitted.rs b/crates/j2k-metal/src/encode/submitted.rs new file mode 100644 index 00000000..1ebd1094 --- /dev/null +++ b/crates/j2k-metal/src/encode/submitted.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::EncodedJ2k; +#[cfg(target_os = "macos")] +use j2k::J2kLosslessEncodeOptions; +use j2k_core::DeviceSubmission; +#[cfg(target_os = "macos")] +use j2k_core::PixelFormat; +#[cfg(target_os = "macos")] +use metal::Buffer; + +use super::MetalLosslessBufferEncodeBatchOutcome; +#[cfg(target_os = "macos")] +use super::{ + encode_lossless_owned_tiles_with_report, + encode_owned_lossless_tiles_to_metal_buffer_fallback_batch, + wait_submitted_resident_lossless_buffer_encode_batch, MetalEncodeInputStaging, + MetalLosslessEncodeConfig, MetalLosslessEncodeTile, + SubmittedResidentLosslessMetalBufferEncodeBatch, +}; + +#[cfg(target_os = "macos")] +/// Submitted multi-tile Metal encode that resolves to host codestream bytes. +pub struct SubmittedJ2kLosslessMetalEncodeBatch { + pub(super) state: SubmittedJ2kLosslessMetalEncodeBatchState, +} + +#[cfg(target_os = "macos")] +/// Submitted multi-tile Metal encode that resolves to Metal-backed codestreams. +pub struct SubmittedJ2kLosslessMetalBufferEncodeBatch { + pub(super) state: SubmittedJ2kLosslessMetalBufferEncodeBatchState, +} + +#[cfg(target_os = "macos")] +pub(super) enum SubmittedJ2kLosslessMetalEncodeBatchState { + Ready(Vec), + Deferred { + tiles: Vec, + options: J2kLosslessEncodeOptions, + session: crate::MetalBackendSession, + staging: MetalEncodeInputStaging, + config: MetalLosslessEncodeConfig, + }, +} + +#[cfg(target_os = "macos")] +pub(super) enum SubmittedJ2kLosslessMetalBufferEncodeBatchState { + Resident(Box), + Deferred { + tiles: Vec, + options: J2kLosslessEncodeOptions, + session: crate::MetalBackendSession, + staging: MetalEncodeInputStaging, + }, +} + +#[cfg(target_os = "macos")] +pub(super) struct OwnedMetalLosslessEncodeTile { + pub(super) buffer: Buffer, + pub(super) byte_offset: usize, + pub(super) width: u32, + pub(super) height: u32, + pub(super) pitch_bytes: usize, + pub(super) output_width: u32, + pub(super) output_height: u32, + pub(super) format: PixelFormat, +} + +#[cfg(target_os = "macos")] +impl OwnedMetalLosslessEncodeTile { + pub(super) fn from_tile(tile: MetalLosslessEncodeTile<'_>) -> Self { + Self { + buffer: tile.buffer.to_owned(), + byte_offset: tile.byte_offset, + width: tile.width, + height: tile.height, + pitch_bytes: tile.pitch_bytes, + output_width: tile.output_width, + output_height: tile.output_height, + format: tile.format, + } + } + + pub(super) fn as_tile(&self) -> MetalLosslessEncodeTile<'_> { + MetalLosslessEncodeTile { + buffer: &self.buffer, + byte_offset: self.byte_offset, + width: self.width, + height: self.height, + pitch_bytes: self.pitch_bytes, + output_width: self.output_width, + output_height: self.output_height, + format: self.format, + } + } +} + +#[cfg(not(target_os = "macos"))] +/// Placeholder submitted multi-tile encode for non-macOS builds. +pub struct SubmittedJ2kLosslessMetalEncodeBatch { + _private: (), +} + +#[cfg(not(target_os = "macos"))] +/// Placeholder submitted Metal-buffer encode for non-macOS builds. +pub struct SubmittedJ2kLosslessMetalBufferEncodeBatch { + _private: (), +} + +#[cfg(target_os = "macos")] +impl DeviceSubmission for SubmittedJ2kLosslessMetalEncodeBatch { + type Output = Vec; + type Error = crate::Error; + + fn wait(self) -> Result { + match self.state { + SubmittedJ2kLosslessMetalEncodeBatchState::Ready(encoded) => Ok(encoded), + SubmittedJ2kLosslessMetalEncodeBatchState::Deferred { + tiles, + options, + session, + staging, + config, + } => { + encode_lossless_owned_tiles_with_report(&tiles, options, &session, staging, config) + .map(|outcomes| { + outcomes + .into_iter() + .map(|outcome| outcome.encoded) + .collect() + }) + } + } + } +} + +#[cfg(target_os = "macos")] +impl DeviceSubmission for SubmittedJ2kLosslessMetalBufferEncodeBatch { + type Output = MetalLosslessBufferEncodeBatchOutcome; + type Error = crate::Error; + + fn wait(self) -> Result { + match self.state { + SubmittedJ2kLosslessMetalBufferEncodeBatchState::Resident(submitted) => { + wait_submitted_resident_lossless_buffer_encode_batch(*submitted) + } + SubmittedJ2kLosslessMetalBufferEncodeBatchState::Deferred { + tiles, + options, + session, + staging, + } => encode_owned_lossless_tiles_to_metal_buffer_fallback_batch( + &tiles, options, &session, staging, + ), + } + } +} + +#[cfg(not(target_os = "macos"))] +impl DeviceSubmission for SubmittedJ2kLosslessMetalEncodeBatch { + type Output = Vec; + type Error = crate::Error; + + fn wait(self) -> Result { + let _ = self; + Err(crate::Error::MetalUnavailable) + } +} + +#[cfg(not(target_os = "macos"))] +impl DeviceSubmission for SubmittedJ2kLosslessMetalBufferEncodeBatch { + type Output = MetalLosslessBufferEncodeBatchOutcome; + type Error = crate::Error; + + fn wait(self) -> Result { + let _ = self; + Err(crate::Error::MetalUnavailable) + } +} diff --git a/crates/j2k-metal/src/encode/test_helpers.rs b/crates/j2k-metal/src/encode/test_helpers.rs new file mode 100644 index 00000000..ec439646 --- /dev/null +++ b/crates/j2k-metal/src/encode/test_helpers.rs @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::{EncodedJ2k, J2kLosslessEncodeOptions}; +use j2k_core::DeviceSubmission; +use rayon::prelude::*; +use std::{ + cell::Cell, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, +}; + +use super::{ + encode_lossless_batch_with_report, encode_lossless_tile_with_report, submit_lossless_batch, + submit_lossless_batch_to_metal, MetalEncodeInputStaging, MetalEncodeStageAccelerator, + MetalLosslessBufferEncodeBatchOutcome, MetalLosslessBufferEncodeOutcome, + MetalLosslessEncodeBatchRequest, MetalLosslessEncodeConfig, MetalLosslessEncodeOutcome, + MetalLosslessEncodeTile, SubmittedJ2kLosslessMetalEncodeBatch, +}; + +pub(super) struct InflightLimitedOrderedItems { + pub(super) items: Vec, + pub(super) max_observed_inflight_items: usize, +} + +pub(super) fn collect_inflight_limited_ordered( + items: Vec, + inflight_items: usize, + f: F, +) -> Result, crate::Error> +where + T: Send, + O: Send, + F: Fn(usize, T) -> Result + Sync, +{ + if items.is_empty() { + return Ok(InflightLimitedOrderedItems { + items: Vec::new(), + max_observed_inflight_items: 0, + }); + } + + let active = Arc::new(AtomicUsize::new(0)); + let observed = Arc::new(AtomicUsize::new(0)); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(inflight_items.max(1)) + .build() + .map_err(|err| crate::Error::MetalKernel { + message: format!("J2K Metal encode worker pool initialization failed: {err}"), + })?; + + let active_for_tasks = Arc::clone(&active); + let observed_for_tasks = Arc::clone(&observed); + let results = pool.install(|| { + items + .into_par_iter() + .enumerate() + .map(|(index, item)| { + let _guard = ActiveTileGuard::new(&active_for_tasks, &observed_for_tasks); + f(index, item) + }) + .collect::>() + }); + + let max_observed_inflight_items = observed.load(Ordering::Relaxed); + let mut ordered = Vec::with_capacity(results.len()); + let mut first_error = None; + for result in results { + match result { + Ok(item) if first_error.is_none() => ordered.push(item), + Ok(_) => {} + Err(err) => { + if first_error.is_none() { + first_error = Some(err); + } + } + } + } + + if let Some(err) = first_error { + return Err(err); + } + + Ok(InflightLimitedOrderedItems { + items: ordered, + max_observed_inflight_items, + }) +} + +struct ActiveTileGuard<'a> { + active: &'a AtomicUsize, +} + +impl<'a> ActiveTileGuard<'a> { + fn new(active: &'a AtomicUsize, observed: &AtomicUsize) -> Self { + let now = active.fetch_add(1, Ordering::AcqRel).saturating_add(1); + let mut current = observed.load(Ordering::Relaxed); + while now > current { + match observed.compare_exchange(current, now, Ordering::AcqRel, Ordering::Relaxed) { + Ok(_) => break, + Err(next) => current = next, + } + } + Self { active } + } +} + +impl Drop for ActiveTileGuard<'_> { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::AcqRel); + } +} + +thread_local! { + static TEST_RESIDENT_ENCODE_FAILURE_INDEX: Cell> = const { Cell::new(None) }; +} + +pub(super) fn set_test_resident_encode_failure_index(index: Option) { + TEST_RESIDENT_ENCODE_FAILURE_INDEX.set(index); +} + +pub(super) fn test_resident_encode_failure_index() -> Option { + TEST_RESIDENT_ENCODE_FAILURE_INDEX.get() +} + +// Pre-8c permutation shims: the public API collapsed to the request-based +// entries, but the device tests keep their original entry-point coverage +// (and the single-tile report tests their exact staged routing) through +// these test-only equivalents of the removed wrappers. +pub(super) struct TestSubmittedSingleLosslessEncode { + inner: SubmittedJ2kLosslessMetalEncodeBatch, +} + +impl TestSubmittedSingleLosslessEncode { + pub(super) fn wait(self) -> Result { + let mut encoded = self.inner.wait()?; + if encoded.len() != 1 { + return Err(crate::Error::MetalKernel { + message: "submitted J2K Metal single encode produced an unexpected batch length" + .to_string(), + }); + } + Ok(encoded.remove(0)) + } +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // shims keep the removed wrappers' exact signatures +pub(super) fn submit_lossless_from_metal_buffer( + tile: MetalLosslessEncodeTile<'_>, + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result { + Ok(TestSubmittedSingleLosslessEncode { + inner: submit_lossless_batch( + MetalLosslessEncodeBatchRequest { + tiles: &[tile], + staging: MetalEncodeInputStaging::CopyAndPad, + config: MetalLosslessEncodeConfig::default(), + }, + options, + session, + )?, + }) +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // shims keep the removed wrappers' exact signatures +pub(super) fn submit_lossless_from_padded_metal_buffer( + tile: MetalLosslessEncodeTile<'_>, + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result { + Ok(TestSubmittedSingleLosslessEncode { + inner: submit_lossless_batch( + MetalLosslessEncodeBatchRequest { + tiles: &[tile], + staging: MetalEncodeInputStaging::AlreadyPaddedContiguous, + config: MetalLosslessEncodeConfig::default(), + }, + options, + session, + )?, + }) +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // shims keep the removed wrappers' exact signatures +pub(super) fn encode_lossless_from_metal_buffer( + tile: MetalLosslessEncodeTile<'_>, + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result { + submit_lossless_from_metal_buffer(tile, options, session)?.wait() +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // shims keep the removed wrappers' exact signatures +pub(super) fn encode_lossless_from_padded_metal_buffer_with_report( + tile: MetalLosslessEncodeTile<'_>, + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result { + let mut accelerator = MetalEncodeStageAccelerator::for_host_output(*options); + encode_lossless_tile_with_report( + tile, + *options, + session, + MetalEncodeInputStaging::AlreadyPaddedContiguous, + &mut accelerator, + ) +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // shims keep the removed wrappers' exact signatures +pub(super) fn encode_lossless_from_metal_buffers_to_metal_with_report( + tiles: &[MetalLosslessEncodeTile<'_>], + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result, crate::Error> { + Ok(submit_lossless_batch_to_metal( + MetalLosslessEncodeBatchRequest { + tiles, + staging: MetalEncodeInputStaging::CopyAndPad, + config: MetalLosslessEncodeConfig::default(), + }, + options, + session, + )? + .wait()? + .outcomes) +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // shims keep the removed wrappers' exact signatures +pub(super) fn encode_lossless_from_padded_metal_buffers_to_metal_with_report( + tiles: &[MetalLosslessEncodeTile<'_>], + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result, crate::Error> { + Ok(submit_lossless_batch_to_metal( + MetalLosslessEncodeBatchRequest { + tiles, + staging: MetalEncodeInputStaging::AlreadyPaddedContiguous, + config: MetalLosslessEncodeConfig::default(), + }, + options, + session, + )? + .wait()? + .outcomes) +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // shims keep the removed wrappers' exact signatures +pub(super) fn encode_lossless_from_metal_buffer_to_metal_with_report( + tile: MetalLosslessEncodeTile<'_>, + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result { + let mut outcomes = + encode_lossless_from_metal_buffers_to_metal_with_report(&[tile], options, session)?; + if outcomes.len() != 1 { + return Err(crate::Error::MetalKernel { + message: "J2K Metal single buffer encode produced an unexpected batch length" + .to_string(), + }); + } + Ok(outcomes.remove(0)) +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // shims keep the removed wrappers' exact signatures +pub(super) fn encode_lossless_from_padded_metal_buffer_to_metal_with_report( + tile: MetalLosslessEncodeTile<'_>, + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result { + let mut outcomes = + encode_lossless_from_padded_metal_buffers_to_metal_with_report(&[tile], options, session)?; + if outcomes.len() != 1 { + return Err(crate::Error::MetalKernel { + message: "J2K Metal single buffer encode produced an unexpected batch length" + .to_string(), + }); + } + Ok(outcomes.remove(0)) +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // shims keep the removed wrappers' exact signatures +pub(super) fn encode_lossless_from_padded_metal_buffers_with_report( + tiles: &[MetalLosslessEncodeTile<'_>], + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, +) -> Result, crate::Error> { + encode_lossless_batch_with_report( + MetalLosslessEncodeBatchRequest { + tiles, + staging: MetalEncodeInputStaging::AlreadyPaddedContiguous, + config: MetalLosslessEncodeConfig::default(), + }, + options, + session, + ) +} + +#[allow(clippy::trivially_copy_pass_by_ref)] // shims keep the removed wrappers' exact signatures +pub(super) fn encode_lossless_from_padded_metal_buffers_to_metal_batch( + tiles: &[MetalLosslessEncodeTile<'_>], + options: &J2kLosslessEncodeOptions, + session: &crate::MetalBackendSession, + config: MetalLosslessEncodeConfig, +) -> Result { + submit_lossless_batch_to_metal( + MetalLosslessEncodeBatchRequest { + tiles, + staging: MetalEncodeInputStaging::AlreadyPaddedContiguous, + config, + }, + options, + session, + )? + .wait() +} diff --git a/crates/j2k-metal/src/encode/tests.rs b/crates/j2k-metal/src/encode/tests.rs new file mode 100644 index 00000000..1959a31d --- /dev/null +++ b/crates/j2k-metal/src/encode/tests.rs @@ -0,0 +1,4488 @@ +// SPDX-License-Identifier: Apache-2.0 + +use super::MetalEncodeStageAccelerator; +#[cfg(target_os = "macos")] +use crate::compute; +#[cfg(target_os = "macos")] +use j2k::adapter::encode_stage::{ + J2kDeinterleaveToF32Job, J2kForwardDwt53Job, J2kForwardIctJob, J2kQuantizeSubbandJob, + NativeEncodeStageAdapter, +}; +use j2k::adapter::encode_stage::{ + J2kEncodeDispatchReport, J2kEncodeStageAccelerator, J2kForwardRctJob, +}; +use j2k::{ + encode_j2k_lossless_with_accelerator, EncodeBackendPreference, EncodedJ2k, + J2kLosslessEncodeOptions, J2kLosslessSamples, +}; +#[cfg(target_os = "macos")] +use j2k::{ + encode_j2k_lossy_with_accelerator, J2kBlockCodingMode, J2kEncodeValidation, + J2kLossyEncodeOptions, J2kLossySamples, J2kMarkerSegment, J2kProgressionOrder, +}; +#[cfg(target_os = "macos")] +use j2k_core::CodecError; +use j2k_core::DeviceSubmission; +#[cfg(target_os = "macos")] +use j2k_core::{BackendKind, PixelFormat}; +#[cfg(target_os = "macos")] +use j2k_native::{ + deinterleave_reference, forward_dwt53_reference, + quantize_reversible_reference as quantize_reference, EncodeOptions, J2kCodeBlockStyle, +}; +use j2k_native::{DecodeSettings, Image}; +#[cfg(target_os = "macos")] +use metal::foreign_types::ForeignType; +#[cfg(target_os = "macos")] +use metal::Buffer; +use std::time::Duration; + +#[cfg(target_os = "macos")] +macro_rules! lossless_options { + ($($field:ident: $value:expr),+ $(,)?) => {{ + let mut options = J2kLosslessEncodeOptions::default(); + $(options.$field = $value;)+ + options + }}; +} + +#[cfg(target_os = "macos")] +fn private_buffer_with_bytes(session: &crate::MetalBackendSession, bytes: &[u8]) -> Buffer { + let upload = session.device().new_buffer_with_data( + bytes.as_ptr().cast(), + bytes.len() as u64, + metal::MTLResourceOptions::StorageModeShared, + ); + let private = session.device().new_buffer( + bytes.len() as u64, + metal::MTLResourceOptions::StorageModePrivate, + ); + let queue = session.device().new_command_queue(); + let command_buffer = queue.new_command_buffer(); + let blit = command_buffer.new_blit_command_encoder(); + blit.copy_from_buffer(&upload, 0, &private, 0, bytes.len() as u64); + blit.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + private +} + +#[cfg(target_os = "macos")] +fn overwrite_private_buffer_with_bytes( + session: &crate::MetalBackendSession, + dst: &Buffer, + bytes: &[u8], +) { + let upload = session.device().new_buffer_with_data( + bytes.as_ptr().cast(), + bytes.len() as u64, + metal::MTLResourceOptions::StorageModeShared, + ); + let queue = session.device().new_command_queue(); + let command_buffer = queue.new_command_buffer(); + let blit = command_buffer.new_blit_command_encoder(); + blit.copy_from_buffer(&upload, 0, dst, 0, bytes.len() as u64); + blit.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); +} + +#[cfg(target_os = "macos")] +fn assert_decoded_bytes_match(actual: &[u8], expected: &[u8]) { + if actual == expected { + return; + } + let mismatch = actual + .iter() + .zip(expected.iter()) + .position(|(actual, expected)| actual != expected) + .unwrap_or_else(|| actual.len().min(expected.len())); + let actual_value = actual.get(mismatch).copied(); + let expected_value = expected.get(mismatch).copied(); + panic!( + "decoded bytes mismatch at byte {mismatch}: actual={actual_value:?} expected={expected_value:?} actual_len={} expected_len={}", + actual.len(), + expected.len() + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_encode_deinterleave_public_layouts_match_native_reference() { + #[derive(Clone, Copy)] + struct Case { + name: &'static str, + num_components: u8, + bit_depth: u8, + signed: bool, + } + + fn case_pixels(case: Case, num_pixels: usize) -> Vec { + let sample_count = num_pixels * usize::from(case.num_components); + if case.bit_depth <= 8 { + return (0..sample_count) + .map(|idx| ((idx * 37 + 11) & 0xff) as u8) + .collect(); + } + + let mut pixels = Vec::with_capacity(sample_count * 2); + for idx in 0..sample_count { + let sample = ((idx * 1031 + 0x7123) & 0xffff) as u16; + pixels.extend_from_slice(&sample.to_le_bytes()); + } + pixels + } + + let cases = [ + Case { + name: "Gray8 unsigned", + num_components: 1, + bit_depth: 8, + signed: false, + }, + Case { + name: "two-component 8-bit unsigned", + num_components: 2, + bit_depth: 8, + signed: false, + }, + Case { + name: "RGB8 unsigned", + num_components: 3, + bit_depth: 8, + signed: false, + }, + Case { + name: "RGBA8 unsigned", + num_components: 4, + bit_depth: 8, + signed: false, + }, + Case { + name: "RGB8 signed", + num_components: 3, + bit_depth: 8, + signed: true, + }, + Case { + name: "Gray16 signed", + num_components: 1, + bit_depth: 16, + signed: true, + }, + Case { + name: "RGB16 unsigned", + num_components: 3, + bit_depth: 16, + signed: false, + }, + Case { + name: "RGBA12 unsigned", + num_components: 4, + bit_depth: 12, + signed: false, + }, + ]; + + for case in cases { + let pixels = case_pixels(case, 5); + J2kLosslessSamples::new( + &pixels, + 5, + 1, + case.num_components, + case.bit_depth, + case.signed, + ) + .unwrap_or_else(|err| panic!("lossless public layout rejected for {}: {err}", case.name)); + J2kLossySamples::new( + &pixels, + 5, + 1, + case.num_components, + case.bit_depth, + case.signed, + ) + .unwrap_or_else(|err| panic!("lossy public layout rejected for {}: {err}", case.name)); + + let job = J2kDeinterleaveToF32Job { + pixels: &pixels, + num_pixels: 5, + num_components: case.num_components, + bit_depth: case.bit_depth, + signed: case.signed, + }; + let expected = deinterleave_reference( + job.pixels, + job.num_pixels, + job.num_components, + job.bit_depth, + job.signed, + ); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let actual = accelerator + .encode_deinterleave(job) + .unwrap_or_else(|err| { + panic!("Metal deinterleave stage failed for {}: {err}", case.name) + }) + .unwrap_or_else(|| panic!("Metal deinterleave did not dispatch for {}", case.name)); + + assert_eq!(actual, expected, "{}", case.name); + assert_eq!(accelerator.deinterleave_attempts(), 1, "{}", case.name); + assert_eq!(accelerator.deinterleave_dispatches(), 1, "{}", case.name); + assert_eq!( + accelerator.dispatch_report().deinterleave, + 1, + "{}", + case.name + ); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_encode_deinterleave_invalid_component_count_errors_without_dispatch() { + let pixels = [0_u8; 10]; + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let err = accelerator + .encode_deinterleave(J2kDeinterleaveToF32Job { + pixels: &pixels, + num_pixels: 2, + num_components: 5, + bit_depth: 8, + signed: false, + }) + .unwrap_err(); + + assert_eq!(err, "Metal deinterleave encode shape is unsupported"); + assert_eq!(accelerator.deinterleave_attempts(), 1); + assert_eq!(accelerator.deinterleave_dispatches(), 0); + assert_eq!(accelerator.dispatch_report().deinterleave, 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_encode_deinterleave_compute_rejects_invalid_shape_structured() { + let pixels = [0_u8; 10]; + + let err = compute::encode_deinterleave_to_f32(J2kDeinterleaveToF32Job { + pixels: &pixels, + num_pixels: 2, + num_components: 5, + bit_depth: 8, + signed: false, + }) + .unwrap_err(); + + assert!(matches!( + err, + crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal encode deinterleave supports 1-4 component samples" + } + )); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_quantize_subband_kernel_matches_cpu_reference() { + #[derive(Clone, Copy)] + struct Case { + name: &'static str, + step_exponent: u16, + step_mantissa: u16, + range_bits: u8, + reversible: bool, + } + + let coefficients = (0_u16..257) + .map(|idx| { + let centered = f32::from(idx) - 128.0; + centered * 0.375 + f32::from(idx % 7) * 0.125 - if idx % 5 == 0 { 0.5 } else { 0.0 } + }) + .collect::>(); + + for case in [ + Case { + name: "reversible", + step_exponent: 12, + step_mantissa: 0, + range_bits: 8, + reversible: true, + }, + Case { + name: "irreversible_delta_1", + step_exponent: 8, + step_mantissa: 0, + range_bits: 8, + reversible: false, + }, + Case { + name: "irreversible_fractional_delta", + step_exponent: 9, + step_mantissa: 512, + range_bits: 8, + reversible: false, + }, + Case { + name: "irreversible_large_delta", + step_exponent: 6, + step_mantissa: 1024, + range_bits: 10, + reversible: false, + }, + ] { + let expected = quantize_reference( + &coefficients, + case.step_exponent, + case.step_mantissa, + case.range_bits, + case.reversible, + ); + let actual = compute::encode_quantize_subband(J2kQuantizeSubbandJob { + coefficients: &coefficients, + step_exponent: case.step_exponent, + step_mantissa: case.step_mantissa, + range_bits: case.range_bits, + reversible: case.reversible, + }) + .unwrap_or_else(|err| panic!("Metal quantize_subband failed for {}: {err}", case.name)); + + assert_eq!(actual, expected, "{}", case.name); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_quantize_subband_stage_reports_dispatch() { + let coefficients = [-4.5, -1.25, -0.5, 0.0, 0.5, 1.25, 4.5]; + let expected = quantize_reference(&coefficients, 8, 0, 8, true); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let actual = accelerator + .encode_quantize_subband(J2kQuantizeSubbandJob { + coefficients: &coefficients, + step_exponent: 8, + step_mantissa: 0, + range_bits: 8, + reversible: true, + }) + .expect("Metal quantize_subband stage should not error") + .expect("Metal quantize_subband should dispatch"); + + assert_eq!(actual, expected); + assert_eq!(accelerator.quantize_subband_attempts(), 1); + assert_eq!(accelerator.quantize_subband_dispatches(), 1); + assert_eq!(accelerator.dispatch_report().quantize_subband, 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_quantize_subband_invalid_shape_errors_without_dispatch() { + let coefficients = [1.0, -2.0, 3.0]; + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let err = accelerator + .encode_quantize_subband(J2kQuantizeSubbandJob { + coefficients: &coefficients, + step_exponent: 8, + step_mantissa: 2048, + range_bits: 8, + reversible: false, + }) + .unwrap_err(); + + assert_eq!(err, "Metal quantize_subband encode shape is unsupported"); + assert_eq!(accelerator.quantize_subband_attempts(), 1); + assert_eq!(accelerator.quantize_subband_dispatches(), 0); + assert_eq!(accelerator.dispatch_report().quantize_subband, 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_quantize_subband_compute_rejects_invalid_shape_structured() { + let coefficients = [1.0, -2.0, 3.0]; + let err = compute::encode_quantize_subband(J2kQuantizeSubbandJob { + coefficients: &coefficients, + step_exponent: 8, + step_mantissa: 2048, + range_bits: 8, + reversible: false, + }) + .unwrap_err(); + + assert!(matches!( + err, + crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal encode quantize_subband supports step mantissas <= 2047" + } + )); +} + +#[test] +fn auto_encode_deinterleave_disabled_fallback_still_encodes() { + let mut pixels = Vec::with_capacity(8 * 8 * 2); + for idx in 0..8 * 8 { + let sample = u16::try_from((idx * 257 + 19) & 0xffff).expect("masked sample fits u16"); + pixels.extend_from_slice(&sample.to_le_bytes()); + } + let samples = + J2kLosslessSamples::new(&pixels, 8, 8, 1, 16, false).expect("valid Gray16 samples"); + let options = J2kLosslessEncodeOptions::default().with_max_decomposition_levels(Some(0)); + let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &options, + j2k_core::BackendKind::Metal, + &mut accelerator, + ) + .expect("Gray16 encode should succeed with CPU deinterleave fallback"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert_eq!(encoded.dispatch_report.deinterleave, 0); + assert_eq!(accelerator.deinterleave_attempts(), 1); + assert_eq!(accelerator.deinterleave_dispatches(), 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn inflight_limited_runner_starts_next_item_before_slow_peer_finishes() { + use std::sync::{Arc, Condvar, Mutex}; + use std::time::Duration; + + #[derive(Default)] + struct Probe { + third_item_started: bool, + } + + let probe = Arc::new((Mutex::new(Probe::default()), Condvar::new())); + let task_probe = Arc::clone(&probe); + + let outcomes = super::collect_inflight_limited_ordered(vec![0usize, 1, 2], 2, move |_, item| { + match item { + 0 => Ok(item), + 1 => { + let (lock, cvar) = &*task_probe; + let state = lock.lock().expect("probe mutex"); + let (state, _timeout) = cvar + .wait_timeout_while(state, Duration::from_millis(250), |state| { + !state.third_item_started + }) + .expect("probe wait"); + if !state.third_item_started { + return Err(crate::Error::MetalKernel { + message: + "runner waited for the whole in-flight chunk before scheduling more work" + .to_string(), + }); + } + Ok(item) + } + 2 => { + let (lock, cvar) = &*task_probe; + let mut state = lock.lock().expect("probe mutex"); + state.third_item_started = true; + cvar.notify_all(); + Ok(item) + } + _ => unreachable!("unexpected test item"), + } + }) + .expect("in-flight runner should slide past a slow peer"); + + assert_eq!(outcomes.items, vec![0, 1, 2]); + assert!(outcomes.max_observed_inflight_items <= 2); + assert!(outcomes.max_observed_inflight_items > 0); +} + +#[test] +fn submitted_lossless_metal_encode_public_api_is_available() { + fn assert_batch_submission< + S: DeviceSubmission, Error = crate::Error>, + >() { + } + fn assert_submit_batch_fn( + _submit: for<'tiles, 'tile, 'options, 'session> fn( + super::MetalLosslessEncodeBatchRequest<'tiles, 'tile>, + &'options J2kLosslessEncodeOptions, + &'session crate::MetalBackendSession, + ) -> Result< + crate::SubmittedJ2kLosslessMetalEncodeBatch, + crate::Error, + >, + ) { + } + + assert_batch_submission::(); + assert_submit_batch_fn(crate::submit_lossless_batch); +} + +#[test] +fn submitted_lossless_metal_buffer_encode_public_api_is_available() { + fn assert_buffer_batch_submission< + S: DeviceSubmission< + Output = super::MetalLosslessBufferEncodeBatchOutcome, + Error = crate::Error, + >, + >() { + } + fn assert_submit_buffer_batch_fn( + _submit: for<'tiles, 'tile, 'options, 'session> fn( + super::MetalLosslessEncodeBatchRequest<'tiles, 'tile>, + &'options J2kLosslessEncodeOptions, + &'session crate::MetalBackendSession, + ) -> Result< + crate::SubmittedJ2kLosslessMetalBufferEncodeBatch, + crate::Error, + >, + ) { + } + + assert_buffer_batch_submission::(); + assert_submit_buffer_batch_fn(crate::submit_lossless_batch_to_metal); +} + +#[test] +fn resident_lossless_stage_stats_default_to_zero() { + let stats = super::MetalLosslessEncodeBatchStats::default(); + + assert_eq!( + stats.stage_stats, + super::MetalLosslessEncodeStageStats::default() + ); + assert_eq!(stats.stage_stats.coefficient_prep_duration, Duration::ZERO); + assert_eq!(stats.stage_stats.deinterleave_rct_duration, Duration::ZERO); + assert_eq!(stats.stage_stats.dwt53_duration, Duration::ZERO); + assert_eq!( + stats.stage_stats.coefficient_extract_duration, + Duration::ZERO + ); + assert_eq!(stats.stage_stats.ht_block_encode_duration, Duration::ZERO); + assert_eq!( + stats.stage_stats.classic_tier1_setup_duration, + Duration::ZERO + ); + assert_eq!( + stats.stage_stats.classic_block_encode_duration, + Duration::ZERO + ); + assert_eq!( + stats.stage_stats.classic_packet_plan_duration, + Duration::ZERO + ); + assert_eq!( + stats.stage_stats.classic_packet_buffer_setup_duration, + Duration::ZERO + ); + assert_eq!( + stats.stage_stats.classic_command_buffer_commit_duration, + Duration::ZERO + ); + assert_eq!(stats.stage_stats.packet_block_prep_duration, Duration::ZERO); + assert_eq!(stats.stage_stats.packetization_duration, Duration::ZERO); + assert_eq!( + stats.stage_stats.packet_payload_copy_gpu_duration, + Duration::ZERO + ); + assert_eq!(stats.stage_stats.gpu_elapsed_wall_duration, Duration::ZERO); + assert_eq!(stats.stage_stats.classic_block_gpu_duration, Duration::ZERO); + assert_eq!( + stats.stage_stats.classic_tier1_density_gpu_duration, + Duration::ZERO + ); + assert_eq!( + stats.stage_stats.codestream_assembly_duration, + Duration::ZERO + ); + assert_eq!( + stats.stage_stats.codestream_payload_copy_gpu_duration, + Duration::ZERO + ); + assert_eq!(stats.stage_stats.tier1_output_capacity_total, 0); + assert_eq!(stats.stage_stats.max_tier1_output_capacity, 0); + assert_eq!(stats.stage_stats.tier1_output_used_bytes_total, 0); + assert_eq!(stats.stage_stats.max_tier1_output_used_bytes, 0); + assert_eq!(stats.stage_stats.tier1_coding_pass_count_total, 0); + assert_eq!(stats.stage_stats.max_tier1_coding_passes_per_block, 0); + assert_eq!(stats.stage_stats.tier1_arithmetic_pass_count_total, 0); + assert_eq!(stats.stage_stats.tier1_raw_pass_count_total, 0); + assert_eq!(stats.stage_stats.tier1_cleanup_pass_count_total, 0); + assert_eq!(stats.stage_stats.tier1_sigprop_pass_count_total, 0); + assert_eq!(stats.stage_stats.tier1_magref_pass_count_total, 0); + assert_eq!( + stats.stage_stats.tier1_arithmetic_cleanup_pass_count_total, + 0 + ); + assert_eq!( + stats.stage_stats.tier1_arithmetic_sigprop_pass_count_total, + 0 + ); + assert_eq!( + stats.stage_stats.tier1_arithmetic_magref_pass_count_total, + 0 + ); + assert_eq!(stats.stage_stats.tier1_raw_sigprop_pass_count_total, 0); + assert_eq!(stats.stage_stats.tier1_raw_magref_pass_count_total, 0); + assert_eq!(stats.stage_stats.tier1_full_scan_coeff_visit_count_total, 0); + assert_eq!( + stats + .stage_stats + .tier1_arithmetic_scan_coeff_visit_count_total, + 0 + ); + assert_eq!(stats.stage_stats.tier1_raw_scan_coeff_visit_count_total, 0); + assert_eq!( + stats.stage_stats.tier1_cleanup_scan_coeff_visit_count_total, + 0 + ); + assert_eq!( + stats.stage_stats.tier1_sigprop_scan_coeff_visit_count_total, + 0 + ); + assert_eq!( + stats.stage_stats.tier1_magref_scan_coeff_visit_count_total, + 0 + ); + assert_eq!( + stats.stage_stats.max_tier1_full_scan_coeff_visits_per_block, + 0 + ); + assert_eq!( + stats.stage_stats.tier1_sigprop_active_candidate_count_total, + 0 + ); + assert_eq!( + stats.stage_stats.tier1_sigprop_new_significant_count_total, + 0 + ); + assert_eq!( + stats.stage_stats.tier1_magref_active_candidate_count_total, + 0 + ); + assert_eq!( + stats + .stage_stats + .tier1_arithmetic_sigprop_active_candidate_count_total, + 0 + ); + assert_eq!( + stats + .stage_stats + .tier1_arithmetic_sigprop_new_significant_count_total, + 0 + ); + assert_eq!( + stats + .stage_stats + .tier1_raw_sigprop_active_candidate_count_total, + 0 + ); + assert_eq!( + stats + .stage_stats + .tier1_raw_sigprop_new_significant_count_total, + 0 + ); + assert_eq!( + stats + .stage_stats + .tier1_arithmetic_magref_active_candidate_count_total, + 0 + ); + assert_eq!( + stats + .stage_stats + .tier1_raw_magref_active_candidate_count_total, + 0 + ); + assert_eq!( + stats.stage_stats.tier1_cleanup_active_candidate_count_total, + 0 + ); + assert_eq!( + stats.stage_stats.tier1_cleanup_new_significant_count_total, + 0 + ); + assert_eq!(stats.stage_stats.tier1_cleanup_rlc_stripe_count_total, 0); + assert_eq!( + stats.stage_stats.tier1_cleanup_rlc_zero_stripe_count_total, + 0 + ); + assert_eq!(stats.stage_stats.tier1_nonzero_block_count_total, 0); + assert_eq!(stats.stage_stats.tier1_zero_block_count_total, 0); + assert_eq!(stats.stage_stats.tier1_missing_bitplane_count_total, 0); + assert_eq!(stats.stage_stats.max_tier1_missing_bitplanes_per_block, 0); + assert_eq!(stats.stage_stats.tier1_segment_count_total, 0); + assert_eq!(stats.stage_stats.max_tier1_segments_per_block, 0); + assert_eq!(stats.stage_stats.packet_payload_copy_job_capacity_total, 0); + assert_eq!(stats.stage_stats.max_packet_payload_copy_jobs_per_tile, 0); + assert_eq!(stats.stage_stats.packet_payload_copy_job_count_total, 0); + assert_eq!( + stats.stage_stats.max_packet_payload_copy_jobs_used_per_tile, + 0 + ); + assert_eq!(stats.stage_stats.packet_payload_copy_bytes_total, 0); + assert_eq!(stats.stage_stats.max_packet_payload_copy_bytes_per_tile, 0); + assert_eq!( + stats.stage_stats.packet_payload_copy_small_job_count_total, + 0 + ); + assert_eq!( + stats.stage_stats.packet_payload_copy_medium_job_count_total, + 0 + ); + assert_eq!( + stats.stage_stats.packet_payload_copy_large_job_count_total, + 0 + ); + assert_eq!(stats.stage_stats.packet_output_capacity_total, 0); + assert_eq!(stats.stage_stats.max_packet_output_capacity, 0); + assert_eq!(stats.stage_stats.packet_output_used_bytes_total, 0); + assert_eq!(stats.stage_stats.max_packet_output_used_bytes, 0); + assert_eq!(stats.stage_stats.sync_wait_duration, Duration::ZERO); + assert_eq!(stats.stage_stats.host_readback_duration, Duration::ZERO); + assert!(!stats.stage_stats.has_timings()); +} + +#[test] +fn resident_lossless_stage_stats_add_assign_saturates() { + let mut stats = super::MetalLosslessEncodeStageStats { + plan_duration: Duration::MAX, + tile_count: usize::MAX, + ..super::MetalLosslessEncodeStageStats::default() + }; + + stats.add_assign(super::MetalLosslessEncodeStageStats { + plan_duration: Duration::from_micros(1), + prepare_submit_duration: Duration::from_micros(2), + classic_tier1_setup_duration: Duration::from_micros(4), + classic_block_encode_duration: Duration::from_micros(5), + classic_tier1_token_pack_duration: Duration::from_micros(9), + classic_packet_plan_duration: Duration::from_micros(6), + classic_packet_buffer_setup_duration: Duration::from_micros(7), + classic_command_buffer_commit_duration: Duration::from_micros(8), + packet_payload_copy_job_capacity_total: 11, + max_packet_payload_copy_jobs_per_tile: 5, + packet_payload_copy_job_count_total: 13, + max_packet_payload_copy_jobs_used_per_tile: 6, + packet_payload_copy_bytes_total: 23, + max_packet_payload_copy_bytes_per_tile: 12, + packet_payload_copy_small_job_count_total: 2, + packet_payload_copy_medium_job_count_total: 3, + packet_payload_copy_large_job_count_total: 4, + tier1_output_capacity_total: 17, + max_tier1_output_capacity: 9, + tier1_output_used_bytes_total: 19, + max_tier1_output_used_bytes: 10, + tier1_segment_capacity_total: 25, + max_tier1_segment_capacity_per_block: 11, + tier1_coding_pass_count_total: 31, + max_tier1_coding_passes_per_block: 8, + tier1_arithmetic_pass_count_total: 21, + tier1_raw_pass_count_total: 10, + tier1_cleanup_pass_count_total: 11, + tier1_sigprop_pass_count_total: 10, + tier1_magref_pass_count_total: 10, + tier1_arithmetic_cleanup_pass_count_total: 11, + tier1_arithmetic_sigprop_pass_count_total: 6, + tier1_arithmetic_magref_pass_count_total: 4, + tier1_raw_sigprop_pass_count_total: 4, + tier1_raw_magref_pass_count_total: 6, + tier1_full_scan_coeff_visit_count_total: 31_744, + tier1_arithmetic_scan_coeff_visit_count_total: 21_504, + tier1_raw_scan_coeff_visit_count_total: 10_240, + tier1_cleanup_scan_coeff_visit_count_total: 11_264, + tier1_sigprop_scan_coeff_visit_count_total: 10_240, + tier1_magref_scan_coeff_visit_count_total: 10_240, + max_tier1_full_scan_coeff_visits_per_block: 8_192, + tier1_sigprop_active_candidate_count_total: 101, + tier1_sigprop_new_significant_count_total: 37, + tier1_magref_active_candidate_count_total: 203, + tier1_arithmetic_sigprop_active_candidate_count_total: 61, + tier1_arithmetic_sigprop_new_significant_count_total: 23, + tier1_raw_sigprop_active_candidate_count_total: 40, + tier1_raw_sigprop_new_significant_count_total: 14, + tier1_arithmetic_magref_active_candidate_count_total: 123, + tier1_raw_magref_active_candidate_count_total: 80, + tier1_cleanup_active_candidate_count_total: 307, + tier1_cleanup_new_significant_count_total: 41, + tier1_cleanup_rlc_stripe_count_total: 53, + tier1_cleanup_rlc_zero_stripe_count_total: 47, + tier1_token_pack_output_bytes_total: 29, + max_tier1_token_pack_output_bytes_per_block: 15, + tier1_nonzero_block_count_total: 2, + tier1_zero_block_count_total: 1, + tier1_missing_bitplane_count_total: 5, + max_tier1_missing_bitplanes_per_block: 4, + tier1_segment_count_total: 7, + max_tier1_segments_per_block: 3, + packet_output_capacity_total: 17, + max_packet_output_capacity: 9, + packet_output_used_bytes_total: 19, + max_packet_output_used_bytes: 10, + tile_count: 1, + code_block_count: 3, + ..super::MetalLosslessEncodeStageStats::default() + }); + + assert_eq!(stats.plan_duration, Duration::MAX); + assert_eq!(stats.prepare_submit_duration, Duration::from_micros(2)); + assert_eq!(stats.classic_tier1_setup_duration, Duration::from_micros(4)); + assert_eq!( + stats.classic_block_encode_duration, + Duration::from_micros(5) + ); + assert_eq!( + stats.classic_tier1_token_pack_duration, + Duration::from_micros(9) + ); + assert_eq!(stats.classic_packet_plan_duration, Duration::from_micros(6)); + assert_eq!( + stats.classic_packet_buffer_setup_duration, + Duration::from_micros(7) + ); + assert_eq!( + stats.classic_command_buffer_commit_duration, + Duration::from_micros(8) + ); + assert_eq!(stats.tile_count, usize::MAX); + assert_eq!(stats.code_block_count, 3); + assert_eq!(stats.packet_payload_copy_job_capacity_total, 11); + assert_eq!(stats.max_packet_payload_copy_jobs_per_tile, 5); + assert_eq!(stats.packet_payload_copy_job_count_total, 13); + assert_eq!(stats.max_packet_payload_copy_jobs_used_per_tile, 6); + assert_eq!(stats.packet_payload_copy_bytes_total, 23); + assert_eq!(stats.max_packet_payload_copy_bytes_per_tile, 12); + assert_eq!(stats.packet_payload_copy_small_job_count_total, 2); + assert_eq!(stats.packet_payload_copy_medium_job_count_total, 3); + assert_eq!(stats.packet_payload_copy_large_job_count_total, 4); + assert_eq!(stats.tier1_output_capacity_total, 17); + assert_eq!(stats.max_tier1_output_capacity, 9); + assert_eq!(stats.tier1_output_used_bytes_total, 19); + assert_eq!(stats.max_tier1_output_used_bytes, 10); + assert_eq!(stats.tier1_segment_capacity_total, 25); + assert_eq!(stats.max_tier1_segment_capacity_per_block, 11); + assert_eq!(stats.tier1_coding_pass_count_total, 31); + assert_eq!(stats.max_tier1_coding_passes_per_block, 8); + assert_eq!(stats.tier1_arithmetic_pass_count_total, 21); + assert_eq!(stats.tier1_raw_pass_count_total, 10); + assert_eq!(stats.tier1_cleanup_pass_count_total, 11); + assert_eq!(stats.tier1_sigprop_pass_count_total, 10); + assert_eq!(stats.tier1_magref_pass_count_total, 10); + assert_eq!(stats.tier1_arithmetic_cleanup_pass_count_total, 11); + assert_eq!(stats.tier1_arithmetic_sigprop_pass_count_total, 6); + assert_eq!(stats.tier1_arithmetic_magref_pass_count_total, 4); + assert_eq!(stats.tier1_raw_sigprop_pass_count_total, 4); + assert_eq!(stats.tier1_raw_magref_pass_count_total, 6); + assert_eq!(stats.tier1_full_scan_coeff_visit_count_total, 31_744); + assert_eq!(stats.tier1_arithmetic_scan_coeff_visit_count_total, 21_504); + assert_eq!(stats.tier1_raw_scan_coeff_visit_count_total, 10_240); + assert_eq!(stats.tier1_cleanup_scan_coeff_visit_count_total, 11_264); + assert_eq!(stats.tier1_sigprop_scan_coeff_visit_count_total, 10_240); + assert_eq!(stats.tier1_magref_scan_coeff_visit_count_total, 10_240); + assert_eq!(stats.max_tier1_full_scan_coeff_visits_per_block, 8_192); + assert_eq!(stats.tier1_sigprop_active_candidate_count_total, 101); + assert_eq!(stats.tier1_sigprop_new_significant_count_total, 37); + assert_eq!(stats.tier1_magref_active_candidate_count_total, 203); + assert_eq!( + stats.tier1_arithmetic_sigprop_active_candidate_count_total, + 61 + ); + assert_eq!( + stats.tier1_arithmetic_sigprop_new_significant_count_total, + 23 + ); + assert_eq!(stats.tier1_raw_sigprop_active_candidate_count_total, 40); + assert_eq!(stats.tier1_raw_sigprop_new_significant_count_total, 14); + assert_eq!( + stats.tier1_arithmetic_magref_active_candidate_count_total, + 123 + ); + assert_eq!(stats.tier1_raw_magref_active_candidate_count_total, 80); + assert_eq!(stats.tier1_cleanup_active_candidate_count_total, 307); + assert_eq!(stats.tier1_cleanup_new_significant_count_total, 41); + assert_eq!(stats.tier1_cleanup_rlc_stripe_count_total, 53); + assert_eq!(stats.tier1_cleanup_rlc_zero_stripe_count_total, 47); + assert_eq!(stats.tier1_token_pack_output_bytes_total, 29); + assert_eq!(stats.max_tier1_token_pack_output_bytes_per_block, 15); + assert_eq!(stats.tier1_nonzero_block_count_total, 2); + assert_eq!(stats.tier1_zero_block_count_total, 1); + assert_eq!(stats.tier1_missing_bitplane_count_total, 5); + assert_eq!(stats.max_tier1_missing_bitplanes_per_block, 4); + assert_eq!(stats.tier1_segment_count_total, 7); + assert_eq!(stats.max_tier1_segments_per_block, 3); + assert_eq!(stats.packet_output_capacity_total, 17); + assert_eq!(stats.max_packet_output_capacity, 9); + assert_eq!(stats.packet_output_used_bytes_total, 19); + assert_eq!(stats.max_packet_output_used_bytes, 10); +} + +#[test] +fn resident_lossless_stage_stats_accumulates_split_gpu_durations() { + let mut stats = super::MetalLosslessEncodeStageStats { + ht_block_gpu_duration: Duration::from_micros(2), + ..super::MetalLosslessEncodeStageStats::default() + }; + + stats.add_assign(super::MetalLosslessEncodeStageStats { + coefficient_prep_gpu_duration: Duration::from_micros(23), + coefficient_deinterleave_rct_gpu_duration: Duration::from_micros(4), + coefficient_dwt53_gpu_duration: Duration::from_micros(6), + coefficient_dwt53_vertical_gpu_duration: Duration::from_micros(2), + coefficient_dwt53_horizontal_gpu_duration: Duration::from_micros(4), + coefficient_extract_gpu_duration: Duration::from_micros(8), + coefficient_copy_gpu_duration: Duration::from_micros(1), + gpu_elapsed_wall_duration: Duration::from_micros(29), + classic_block_gpu_duration: Duration::from_micros(19), + classic_tier1_density_gpu_duration: Duration::from_micros(31), + classic_tier1_raw_pack_gpu_duration: Duration::from_micros(37), + classic_tier1_arithmetic_pack_gpu_duration: Duration::from_micros(39), + classic_tier1_symbol_plan_gpu_duration: Duration::from_micros(41), + classic_tier1_token_emit_gpu_duration: Duration::from_micros(43), + classic_tier1_split_token_emit_gpu_duration: Duration::from_micros(45), + classic_tier1_token_pack_gpu_duration: Duration::from_micros(47), + ht_block_gpu_duration: Duration::from_micros(3), + packet_block_prep_gpu_duration: Duration::from_micros(5), + packetization_gpu_duration: Duration::from_micros(7), + packet_payload_copy_gpu_duration: Duration::from_micros(11), + codestream_assembly_gpu_duration: Duration::from_micros(13), + codestream_payload_copy_gpu_duration: Duration::from_micros(17), + ..super::MetalLosslessEncodeStageStats::default() + }); + + assert_eq!( + stats.coefficient_prep_gpu_duration, + Duration::from_micros(23) + ); + assert_eq!( + stats.coefficient_deinterleave_rct_gpu_duration, + Duration::from_micros(4) + ); + assert_eq!( + stats.coefficient_dwt53_gpu_duration, + Duration::from_micros(6) + ); + assert_eq!( + stats.coefficient_dwt53_vertical_gpu_duration, + Duration::from_micros(2) + ); + assert_eq!( + stats.coefficient_dwt53_horizontal_gpu_duration, + Duration::from_micros(4) + ); + assert_eq!( + stats.coefficient_extract_gpu_duration, + Duration::from_micros(8) + ); + assert_eq!( + stats.coefficient_copy_gpu_duration, + Duration::from_micros(1) + ); + assert_eq!(stats.gpu_elapsed_wall_duration, Duration::from_micros(29)); + assert_eq!(stats.classic_block_gpu_duration, Duration::from_micros(19)); + assert_eq!( + stats.classic_tier1_density_gpu_duration, + Duration::from_micros(31) + ); + assert_eq!( + stats.classic_tier1_raw_pack_gpu_duration, + Duration::from_micros(37) + ); + assert_eq!( + stats.classic_tier1_arithmetic_pack_gpu_duration, + Duration::from_micros(39) + ); + assert_eq!( + stats.classic_tier1_symbol_plan_gpu_duration, + Duration::from_micros(41) + ); + assert_eq!( + stats.classic_tier1_token_emit_gpu_duration, + Duration::from_micros(43) + ); + assert_eq!( + stats.classic_tier1_split_token_emit_gpu_duration, + Duration::from_micros(45) + ); + assert_eq!( + stats.classic_tier1_token_pack_gpu_duration, + Duration::from_micros(47) + ); + assert_eq!(stats.ht_block_gpu_duration, Duration::from_micros(5)); + assert_eq!( + stats.packet_block_prep_gpu_duration, + Duration::from_micros(5) + ); + assert_eq!(stats.packetization_gpu_duration, Duration::from_micros(7)); + assert_eq!( + stats.packet_payload_copy_gpu_duration, + Duration::from_micros(11) + ); + assert_eq!( + stats.codestream_assembly_gpu_duration, + Duration::from_micros(13) + ); + assert_eq!( + stats.codestream_payload_copy_gpu_duration, + Duration::from_micros(17) + ); + assert!(stats.has_timings()); +} + +#[test] +fn resident_lossless_prep_duration_only_records_when_profiled() { + let mut stats = super::MetalLosslessEncodeBatchStats::default(); + + super::add_resident_prep_duration(&mut stats, Duration::from_micros(7), false); + assert_eq!(stats.stage_stats.coefficient_prep_duration, Duration::ZERO); + assert!(!stats.stage_stats.has_timings()); + + super::add_resident_prep_duration(&mut stats, Duration::from_micros(7), true); + assert_eq!( + stats.stage_stats.coefficient_prep_duration, + Duration::from_micros(7) + ); + assert!(stats.stage_stats.has_timings()); +} + +#[test] +fn resident_lossless_prep_duration_uses_wall_time_not_per_tile_sum() { + let mut stats = super::MetalLosslessEncodeBatchStats::default(); + let wall_duration = Duration::from_micros(11); + let per_tile_sum = Duration::from_micros(9).saturating_add(Duration::from_micros(10)); + assert_ne!(wall_duration, per_tile_sum); + + super::add_resident_prep_wall_duration(&mut stats, wall_duration, true); + + assert_eq!(stats.stage_stats.coefficient_prep_duration, wall_duration); +} + +#[cfg(target_os = "macos")] +#[test] +fn resident_classic_peak_estimate_matches_tight_batch_capacity() { + let plan = super::LosslessDeviceEncodePlan { + components: 1, + bit_depth: 8, + block_coding_mode: J2kBlockCodingMode::Classic, + num_decomposition_levels: 0, + use_mct: false, + guard_bits: 2, + code_block_width_exp: 4, + code_block_height_exp: 4, + code_blocks: vec![compute::J2kLosslessDeviceCodeBlock { + coefficient_offset: 0, + component: 0, + subband_x: 0, + subband_y: 0, + block_x: 0, + block_y: 0, + width: 64, + height: 64, + sub_band_type: j2k_native::J2kSubBandType::LowLow, + total_bitplanes: 11, + }], + resolutions: Vec::new(), + progression_order: j2k_native::EncodeProgressionOrder::Lrcp, + write_tlm: false, + }; + + assert_eq!( + super::estimated_tier1_output_bytes(&plan), + 64 * 64 * 11 + 4097 + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn resident_classic_batch_retry_covers_tight_capacity_failures() { + let tight_tier1_error = crate::Error::MetalKernel { + message: "packetization Metal encode kernel failure (detail=7, tier1_detail=4)".to_string(), + }; + assert!(super::resident_classic_batch_encode_should_retry_conservative(&tight_tier1_error)); + + let tight_tier1_finish_error = crate::Error::MetalKernel { + message: "classic Tier-1 Metal encode kernel failure (detail=5)".to_string(), + }; + assert!( + super::resident_classic_batch_encode_should_retry_conservative(&tight_tier1_finish_error) + ); + + let packet_error = crate::Error::MetalKernel { + message: "packetization Metal encode kernel failure (detail=5)".to_string(), + }; + assert!(super::resident_classic_batch_encode_should_retry_conservative(&packet_error)); + + let codestream_error = crate::Error::MetalKernel { + message: "J2K batched codestream assembly Metal encode kernel failure (detail=2)" + .to_string(), + }; + assert!(super::resident_classic_batch_encode_should_retry_conservative(&codestream_error)); + + let unrelated_error = crate::Error::MetalKernel { + message: "packetization Metal encode kernel failure (detail=8)".to_string(), + }; + assert!(!super::resident_classic_batch_encode_should_retry_conservative(&unrelated_error)); +} + +#[test] +fn resident_lossless_ht_command_duration_matches_split_buckets() { + let stats = super::MetalLosslessEncodeStageStats { + ht_command_encode_duration: Duration::from_micros(2) + .saturating_add(Duration::from_micros(3)) + .saturating_add(Duration::from_micros(5)) + .saturating_add(Duration::from_micros(7)), + ht_block_encode_duration: Duration::from_micros(2), + packet_block_prep_duration: Duration::from_micros(3), + packetization_duration: Duration::from_micros(5), + codestream_assembly_duration: Duration::from_micros(7), + ..super::MetalLosslessEncodeStageStats::default() + }; + + assert_eq!( + stats.ht_command_encode_duration, + stats + .ht_block_encode_duration + .saturating_add(stats.packet_block_prep_duration) + .saturating_add(stats.packetization_duration) + .saturating_add(stats.codestream_assembly_duration) + ); +} + +#[test] +fn lossless_encode_outcome_exposes_host_readback_duration() { + let outcome = super::MetalLosslessEncodeOutcome { + encoded: EncodedJ2k { + codestream: Vec::new(), + backend: j2k_core::BackendKind::Metal, + dispatch_report: J2kEncodeDispatchReport::default(), + width: 0, + height: 0, + components: 1, + bit_depth: 8, + signed: false, + }, + input_copy_used: false, + resident: super::MetalLosslessEncodeResidency { + coefficient_prep_used: false, + packetization_used: false, + codestream_assembly_used: false, + }, + input_copy_duration: Duration::ZERO, + encode_duration: Duration::ZERO, + gpu_duration: None, + validation_duration: Duration::ZERO, + host_readback_duration: Duration::from_micros(3), + }; + + assert_eq!(outcome.host_readback_duration, Duration::from_micros(3)); +} + +#[test] +fn resident_lossless_chunk_ranges_respect_inflight_and_code_block_caps() { + assert_eq!( + super::resident_lossless_chunk_ranges_for_test(&[32, 32, 32, 32, 32], 3, 96), + vec![0..3, 3..5] + ); + assert_eq!( + super::resident_lossless_chunk_ranges_for_test(&[80, 80, 10], 8, 96), + vec![0..1, 1..3] + ); +} + +#[test] +fn resident_lossless_default_code_block_cap_allows_large_wsi_chunks() { + let code_blocks = vec![192usize; 600]; + let cap = super::resident_lossless_code_block_chunk_cap(&code_blocks); + + assert_eq!( + super::resident_lossless_chunk_ranges_for_test(&code_blocks, 512, cap), + vec![0..512, 512..600] + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_dispatch_option_treats_unavailable_as_no_dispatch() { + let result: Result, &'static str> = + super::metal_dispatch_option(Err(crate::Error::MetalUnavailable), "kernel failed"); + + assert_eq!(result, Ok(None)); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_dispatch_option_preserves_kernel_errors() { + let result: Result, &'static str> = super::metal_dispatch_option( + Err(crate::Error::MetalKernel { + message: "bad status".to_string(), + }), + "kernel failed", + ); + + assert_eq!(result, Err("kernel failed")); +} + +#[test] +fn metal_encode_stage_accelerator_preserves_cpu_codestream_validity() { + let pixels: Vec = (0..8 * 8 * 3).map(|i| (i & 0xFF) as u8).collect(); + let samples = J2kLosslessSamples::new(&pixels, 8, 8, 3, 8, false).expect("valid RGB samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::Auto) + .with_max_decomposition_levels(Some(1)); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &options, + j2k_core::BackendKind::Metal, + &mut accelerator, + ) + .expect("encode with metal stage accelerator"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.width, 8); + assert_eq!(decoded.height, 8); + assert_eq!(decoded.num_components, 3); + assert_eq!(decoded.bit_depth, 8); + assert_eq!(accelerator.forward_rct_attempts(), 1); + assert_eq!(accelerator.forward_dwt53_attempts(), 3); + assert!(accelerator.tier1_code_block_attempts() > 0); + assert_eq!(accelerator.packetization_attempts(), 1); +} + +#[test] +fn metal_encode_stage_accelerator_can_leave_forward_rct_on_cpu() { + let mut plane0 = vec![0.0, 64.0, 128.0, 255.0]; + let mut plane1 = vec![3.0, 67.0, 131.0, 252.0]; + let mut plane2 = vec![7.0, 71.0, 135.0, 248.0]; + let original = (plane0.clone(), plane1.clone(), plane2.clone()); + let mut accelerator = MetalEncodeStageAccelerator::with_cpu_forward_rct(); + + let dispatched = accelerator + .encode_forward_rct(J2kForwardRctJob { + plane0: &mut plane0, + plane1: &mut plane1, + plane2: &mut plane2, + }) + .expect("CPU RCT fallback should be selectable"); + + assert!(!dispatched); + assert_eq!(accelerator.forward_rct_attempts(), 1); + assert_eq!(accelerator.forward_rct_dispatches(), 0); + assert_eq!((plane0, plane1, plane2), original); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_forward_ict_dispatch_matches_cpu_reference() { + fn forward_ict_reference( + plane0: &[f32], + plane1: &[f32], + plane2: &[f32], + ) -> (Vec, Vec, Vec) { + let mut out0 = Vec::with_capacity(plane0.len()); + let mut out1 = Vec::with_capacity(plane1.len()); + let mut out2 = Vec::with_capacity(plane2.len()); + for ((&r, &g), &b) in plane0.iter().zip(plane1).zip(plane2) { + out0.push(0.299 * r + 0.587 * g + 0.114 * b); + out1.push(-0.16875 * r - 0.33126 * g + 0.5 * b); + out2.push(0.5 * r - 0.41869 * g - 0.08131 * b); + } + (out0, out1, out2) + } + + fn assert_near(actual: &[f32], expected: &[f32], label: &str) { + assert_eq!(actual.len(), expected.len(), "{label} length mismatch"); + for (index, (&actual, &expected)) in actual.iter().zip(expected).enumerate() { + assert!( + (actual - expected).abs() <= 0.0001, + "{label}[{index}] mismatch: actual={actual}, expected={expected}" + ); + } + } + + let mut plane0 = vec![0.0, 64.0, 128.0, 255.0, -12.5, 42.25]; + let mut plane1 = vec![3.0, 67.0, 131.0, 252.0, 19.75, -8.5]; + let mut plane2 = vec![7.0, 71.0, 135.0, 248.0, 33.5, 128.0]; + let expected = forward_ict_reference(&plane0, &plane1, &plane2); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let dispatched = accelerator + .encode_forward_ict(J2kForwardIctJob { + plane0: &mut plane0, + plane1: &mut plane1, + plane2: &mut plane2, + }) + .expect("Metal ICT dispatch"); + + assert!(dispatched); + assert_near(&plane0, &expected.0, "Y"); + assert_near(&plane1, &expected.1, "Cb"); + assert_near(&plane2, &expected.2, "Cr"); + assert_eq!(accelerator.forward_ict_attempts(), 1); + assert_eq!(accelerator.forward_ict_dispatches(), 1); + let report = accelerator.dispatch_report(); + assert_eq!(report.forward_ict, 1); + assert_eq!(report.forward_rct, 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_forward_ict_invalid_plane_lengths_error_without_dispatch() { + let mut plane0 = vec![0.0, 64.0]; + let mut plane1 = vec![3.0]; + let mut plane2 = vec![7.0, 71.0]; + let original = (plane0.clone(), plane1.clone(), plane2.clone()); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let err = accelerator + .encode_forward_ict(J2kForwardIctJob { + plane0: &mut plane0, + plane1: &mut plane1, + plane2: &mut plane2, + }) + .unwrap_err(); + + assert_eq!(err, "Metal forward ICT encode shape is unsupported"); + assert_eq!(accelerator.forward_ict_attempts(), 1); + assert_eq!(accelerator.forward_ict_dispatches(), 0); + assert_eq!(accelerator.dispatch_report().forward_ict, 0); + assert_eq!((plane0, plane1, plane2), original); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_forward_ict_compute_rejects_invalid_shape_structured() { + let mut plane0 = vec![0.0, 64.0]; + let mut plane1 = vec![3.0]; + let mut plane2 = vec![7.0, 71.0]; + + let err = compute::encode_forward_ict(&mut plane0, &mut plane1, &mut plane2).unwrap_err(); + + assert!(matches!( + err, + crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal forward ICT plane lengths must match" + } + )); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_forward_rct_dispatch_round_trips_rgb8_lossless_tile() { + let pixels: Vec = (0..7 * 5 * 3).map(|i| ((i * 17) & 0xFF) as u8).collect(); + let samples = J2kLosslessSamples::new(&pixels, 7, 5, 3, 8, false).expect("valid RGB samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_max_decomposition_levels(Some(0)); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &options, + BackendKind::Metal, + &mut accelerator, + ) + .expect("encode with metal forward RCT"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert_eq!(encoded.dispatch_report.deinterleave, 1); + assert_eq!(accelerator.deinterleave_attempts(), 1); + assert_eq!(accelerator.deinterleave_dispatches(), 1); + assert_eq!(accelerator.forward_rct_attempts(), 1); + assert_eq!(accelerator.forward_rct_dispatches(), 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_deinterleave_gray16_lossless_facade_dispatches_and_round_trips() { + let mut pixels = Vec::with_capacity(8 * 8 * 2); + for idx in 0..8 * 8 { + let sample = ((idx * 1021 + 0x2345) & 0xffff) as u16; + pixels.extend_from_slice(&sample.to_le_bytes()); + } + let samples = + J2kLosslessSamples::new(&pixels, 8, 8, 1, 16, false).expect("valid Gray16 samples"); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + max_decomposition_levels: Some(0), + }, + BackendKind::Metal, + &mut accelerator, + ) + .expect("Metal-accelerated Gray16 lossless encode"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert_eq!(encoded.dispatch_report.deinterleave, 1); + assert_eq!(accelerator.deinterleave_attempts(), 1); + assert_eq!(accelerator.deinterleave_dispatches(), 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_validation_decodes_and_compares_lossless_codestream_on_device() { + let pixels: Vec = (0..16 * 16 * 3).map(|i| ((i * 29) & 0xFF) as u8).collect(); + let samples = J2kLosslessSamples::new(&pixels, 16, 16, 3, 8, false).unwrap(); + let encoded = j2k::encode_j2k_lossless( + samples, + &lossless_options! { + backend: EncodeBackendPreference::CpuOnly, + }, + ) + .expect("lossless encode"); + + super::validate_lossless_roundtrip_on_metal(samples, &encoded.codestream) + .expect("Metal lossless validation"); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_buffer_lossless_encode_pads_edge_tile_on_device() { + let pixels: Vec = (0..7 * 5 * 3).map(|i| ((i * 19) & 0xFF) as u8).collect(); + let device = metal::Device::system_default().expect("Metal device"); + let session = crate::MetalBackendSession::new(device); + let buffer = session.device().new_buffer_with_data( + pixels.as_ptr().cast(), + pixels.len() as u64, + metal::MTLResourceOptions::StorageModeShared, + ); + + let encoded = super::encode_lossless_from_metal_buffer( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 7, + height: 5, + pitch_bytes: 7 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("Metal buffer lossless encode"); + + assert_eq!(encoded.backend, BackendKind::Metal); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.width, 8); + assert_eq!(decoded.height, 8); + for y in 0..8usize { + for x in 0..8usize { + let dst = (y * 8 + x) * 3; + if x < 7 && y < 5 { + let src = (y * 7 + x) * 3; + assert_eq!(&decoded.data[dst..dst + 3], &pixels[src..src + 3]); + } else { + assert_eq!(&decoded.data[dst..dst + 3], &[0, 0, 0]); + } + } + } +} + +#[cfg(target_os = "macos")] +#[test] +fn submitted_metal_buffer_lossless_encode_wait_round_trips() { + let pixels: Vec = (0..7 * 5 * 3).map(|i| ((i * 19) & 0xFF) as u8).collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = session.device().new_buffer_with_data( + pixels.as_ptr().cast(), + pixels.len() as u64, + metal::MTLResourceOptions::StorageModeShared, + ); + + let submitted = super::submit_lossless_from_metal_buffer( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 7, + height: 5, + pitch_bytes: 7 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("submit Metal buffer lossless encode"); + let encoded = submitted.wait().expect("wait Metal buffer lossless encode"); + + assert_eq!(encoded.backend, BackendKind::Metal); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.width, 8); + assert_eq!(decoded.height, 8); + for y in 0..8usize { + for x in 0..8usize { + let dst = (y * 8 + x) * 3; + if x < 7 && y < 5 { + let src = (y * 7 + x) * 3; + assert_eq!(&decoded.data[dst..dst + 3], &pixels[src..src + 3]); + } else { + assert_eq!(&decoded.data[dst..dst + 3], &[0, 0, 0]); + } + } + } +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_buffer_lossless_encode_accepts_padded_contiguous_input_without_copy() { + let pixels: Vec = (0..8 * 8 * 3).map(|i| ((i * 31) & 0xFF) as u8).collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = session.device().new_buffer_with_data( + pixels.as_ptr().cast(), + pixels.len() as u64, + metal::MTLResourceOptions::StorageModeShared, + ); + + let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("Metal padded buffer lossless encode"); + + assert_eq!(encoded.encoded.backend, BackendKind::Metal); + assert!(!encoded.input_copy_used); + assert_eq!(encoded.input_copy_duration, std::time::Duration::ZERO); + let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.width, 8); + assert_eq!(decoded.height, 8); + assert_eq!(decoded.data, pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_rgb8_encode_uses_resident_coefficient_prep() { + let pixels: Vec = (0..8 * 8 * 3).map(|i| ((i * 31) & 0xFF) as u8).collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("Metal private padded buffer lossless encode"); + + assert_eq!(encoded.encoded.backend, BackendKind::Metal); + assert!(!encoded.input_copy_used); + assert!(encoded.resident.coefficient_prep_used); + assert!(encoded.resident.packetization_used); + assert!(encoded.resident.codestream_assembly_used); + let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_decoded_bytes_match(&decoded.data, &pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn auto_host_output_encode_options_preserve_auto_for_hybrid_path() { + let routed = super::host_output_encode_options(lossless_options! { + backend: EncodeBackendPreference::Auto, + validation: J2kEncodeValidation::CpuRoundTrip, + }); + + assert_eq!(routed.backend, EncodeBackendPreference::Auto); + assert_eq!(routed.validation, J2kEncodeValidation::External); +} + +#[cfg(target_os = "macos")] +#[test] +fn auto_classic_host_output_stays_cpu_without_metal_dispatches() { + let pixels: Vec = (0..64 * 64).map(|i| ((i * 17) & 0xff) as u8).collect(); + let samples = + J2kLosslessSamples::new(&pixels, 64, 64, 1, 8, false).expect("valid gray samples"); + let options = lossless_options! { + backend: EncodeBackendPreference::Auto, + validation: J2kEncodeValidation::External, + }; + let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &options, + BackendKind::Metal, + &mut accelerator, + ) + .expect("hybrid host-output encode"); + + assert_eq!(encoded.backend, BackendKind::Cpu); + assert_eq!(accelerator.forward_dwt53_dispatches(), 0); + assert_eq!(accelerator.tier1_code_block_dispatches(), 0); + assert_eq!(accelerator.packetization_dispatches(), 0); + assert_eq!(encoded.dispatch_report, J2kEncodeDispatchReport::default()); + assert!(accelerator.prefer_parallel_cpu_code_block_fallback()); +} + +#[cfg(target_os = "macos")] +#[test] +fn auto_classic_large_host_output_dispatches_benchmark_gated_prep_stages_only() { + let width = 1024u32; + let height = 1024u32; + let pixels: Vec = (0..width * height) + .map(|idx| ((idx * 19 + idx / 13) & 0xff) as u8) + .collect(); + let samples = + J2kLosslessSamples::new(&pixels, width, height, 1, 8, false).expect("valid gray samples"); + let options = lossless_options! { + backend: EncodeBackendPreference::Auto, + max_decomposition_levels: Some(1), + validation: J2kEncodeValidation::External, + }; + let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &options, + BackendKind::Metal, + &mut accelerator, + ) + .expect("benchmark-gated Auto host-output encode"); + + assert_eq!(encoded.backend, BackendKind::Cpu); + assert_eq!(accelerator.deinterleave_dispatches(), 1); + assert_eq!(accelerator.forward_dwt53_dispatches(), 1); + assert!(accelerator.quantize_subband_dispatches() > 0); + assert_eq!(accelerator.tier1_code_block_dispatches(), 0); + assert_eq!(accelerator.packetization_dispatches(), 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn auto_lossy_host_output_stays_cpu_without_metal_dispatches() { + let pixels: Vec = (0..64 * 64) + .map(|idx| ((idx * 29 + idx / 7) & 0xff) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 64, 64, 1, 8, false).expect("valid gray samples"); + let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); + + let encoded = encode_j2k_lossy_with_accelerator( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::Auto) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::External), + BackendKind::Metal, + &mut accelerator, + ) + .expect("Auto lossy host-output encode should fall back to CPU"); + + assert_eq!(encoded.backend, BackendKind::Cpu); + assert_eq!(accelerator.ht_code_block_dispatches(), 0); + assert_eq!(accelerator.packetization_dispatches(), 0); + assert_eq!(encoded.dispatch_report, J2kEncodeDispatchReport::default()); +} + +#[cfg(target_os = "macos")] +#[test] +fn auto_lossy_packet_marker_shape_stays_cpu_without_packetization_dispatch() { + let pixels: Vec = (0..64 * 64) + .map(|idx| ((idx * 31 + idx / 5) & 0xff) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 64, 64, 1, 8, false).expect("valid gray samples"); + let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); + + let encoded = encode_j2k_lossy_with_accelerator( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::Auto) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(0)) + .with_marker_segments(vec![J2kMarkerSegment::Sop]) + .with_validation(J2kEncodeValidation::External), + BackendKind::Metal, + &mut accelerator, + ) + .expect("Auto lossy marker shape should fall back to CPU"); + + assert_eq!(encoded.backend, BackendKind::Cpu); + assert_eq!(accelerator.packetization_attempts(), 0); + assert_eq!(accelerator.packetization_dispatches(), 0); + assert_eq!(encoded.dispatch_report, J2kEncodeDispatchReport::default()); +} + +#[cfg(target_os = "macos")] +#[test] +fn strict_metal_lossy_packet_marker_shape_requires_packetization_dispatch() { + let pixels: Vec = (0..64 * 64) + .map(|idx| ((idx * 37 + idx / 9) & 0xff) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 64, 64, 1, 8, false).expect("valid gray samples"); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let err = encode_j2k_lossy_with_accelerator( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(0)) + .with_marker_segments(vec![J2kMarkerSegment::Sop]) + .with_validation(J2kEncodeValidation::External), + BackendKind::Metal, + &mut accelerator, + ) + .unwrap_err(); + + assert!(err.is_unsupported()); + assert!(err.to_string().contains("packetization")); + assert!(accelerator.ht_code_block_dispatches() > 0); + assert_eq!(accelerator.packetization_attempts(), 0); + assert_eq!(accelerator.packetization_dispatches(), 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn auto_htj2k_small_host_output_stays_cpu_below_resident_gate() { + let mut pixels = Vec::with_capacity(64 * 64 * 3); + for y in 0..64u32 { + for x in 0..64u32 { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + pixels.push(((x * 7 + y * 11) & 0xff) as u8); + pixels.push(((x * 13 + y * 17) & 0xff) as u8); + } + } + let samples = J2kLosslessSamples::new(&pixels, 64, 64, 3, 8, false).expect("valid RGB samples"); + let options = lossless_options! { + backend: EncodeBackendPreference::Auto, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + validation: J2kEncodeValidation::External, + }; + let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &options, + BackendKind::Metal, + &mut accelerator, + ) + .expect("hybrid HTJ2K host-output encode"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert_eq!(encoded.backend, BackendKind::Cpu); + assert_eq!(accelerator.forward_rct_dispatches(), 0); + assert_eq!(accelerator.forward_dwt53_dispatches(), 0); + assert_eq!(accelerator.ht_code_block_dispatches(), 0); + assert_eq!(accelerator.packetization_dispatches(), 0); + assert_eq!(encoded.dispatch_report, J2kEncodeDispatchReport::default()); +} + +#[cfg(target_os = "macos")] +#[test] +fn auto_htj2k_large_host_output_uses_resident_metal_rct_dwt_and_ht_with_cpu_packetization() { + let width = 1024u32; + let height = 1024u32; + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + pixels.push(((x * 7 + y * 11) & 0xff) as u8); + pixels.push(((x * 13 + y * 17) & 0xff) as u8); + } + } + let samples = + J2kLosslessSamples::new(&pixels, width, height, 3, 8, false).expect("valid RGB samples"); + let options = lossless_options! { + backend: EncodeBackendPreference::Auto, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + validation: J2kEncodeValidation::External, + }; + let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &options, + BackendKind::Metal, + &mut accelerator, + ) + .expect("hybrid HTJ2K host-output encode"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert_eq!(encoded.backend, BackendKind::Cpu); + assert!(accelerator.forward_rct_dispatches() > 0); + assert_eq!(accelerator.forward_dwt53_dispatches(), 3); + assert!(accelerator.ht_code_block_dispatches() > 0); + assert_eq!(accelerator.packetization_dispatches(), 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn auto_htj2k_padded_rgb8_uses_fused_metal_rct_with_cpu_packetization() { + let mut pixels = Vec::with_capacity(64 * 64 * 3); + for y in 0..64u32 { + for x in 0..64u32 { + pixels.push(((x * 19 + y * 3) & 0xff) as u8); + pixels.push(((x * 5 + y * 23) & 0xff) as u8); + pixels.push(((x * 11 + y * 13) & 0xff) as u8); + } + } + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + compute::reset_lossless_deinterleave_rct_fused_dispatches_for_test(); + + let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 64, + height: 64, + pitch_bytes: 64 * 3, + output_width: 64, + output_height: 64, + format: PixelFormat::Rgb8, + }, + &lossless_options! { + backend: EncodeBackendPreference::Auto, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + validation: J2kEncodeValidation::External, + }, + &session, + ) + .expect("Auto HTJ2K resident hybrid encode"); + let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert_eq!(encoded.encoded.backend, BackendKind::Cpu); + assert!(encoded.resident.coefficient_prep_used); + assert!(!encoded.resident.packetization_used); + assert!(!encoded.resident.codestream_assembly_used); + assert!( + compute::lossless_deinterleave_rct_fused_dispatches_for_test() > 0, + "Auto HTJ2K resident hybrid should use fused deinterleave + RCT" + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_rgb8_auto_host_encode_routes_away_from_resident_prep() { + let pixels: Vec = (0..8 * 8 * 3).map(|i| ((i * 43) & 0xFF) as u8).collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + &lossless_options! { + backend: EncodeBackendPreference::Auto, + validation: J2kEncodeValidation::External, + }, + &session, + ) + .expect("Auto host-output encode should avoid resident prep and still succeed"); + + assert_eq!(encoded.encoded.backend, BackendKind::Cpu); + assert!(!encoded.resident.coefficient_prep_used); + assert!(!encoded.resident.packetization_used); + assert!(!encoded.resident.codestream_assembly_used); + let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_decoded_bytes_match(&decoded.data, &pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_rgb8_encode_to_metal_buffer_exposes_finished_bytes() { + let pixels: Vec = (0..8 * 8 * 3).map(|i| ((i * 37) & 0xFF) as u8).collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_padded_metal_buffer_to_metal_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("Metal private padded buffer lossless encode to Metal buffer"); + + assert!(!encoded.input_copy_used); + assert!(encoded.resident.coefficient_prep_used); + assert!(encoded.resident.packetization_used); + assert!(encoded.resident.codestream_assembly_used); + if let Some(duration) = encoded.gpu_duration { + assert!(duration > Duration::ZERO); + } + assert_eq!(encoded.encoded.byte_offset, 0); + assert!(encoded.encoded.byte_len > 0); + assert!(encoded.encoded.capacity >= encoded.encoded.byte_len); + let codestream = encoded + .encoded + .codestream_bytes() + .expect("Metal codestream bytes are CPU-readable"); + assert!(codestream.starts_with(&[0xFF, 0x4F])); + let decoded = Image::new(codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.data, pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_edge_private_rgb8_encode_to_metal_buffer_pads_and_stays_resident() { + let pixels: Vec = (0..7 * 5 * 3).map(|i| ((i * 41) & 0xFF) as u8).collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_metal_buffer_to_metal_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 7, + height: 5, + pitch_bytes: 7 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("Metal private edge buffer lossless encode to Metal buffer"); + + assert!(!encoded.input_copy_used); + assert!(encoded.resident.coefficient_prep_used); + assert!(encoded.resident.packetization_used); + assert!(encoded.resident.codestream_assembly_used); + let codestream = encoded + .encoded + .codestream_bytes() + .expect("Metal codestream bytes are CPU-readable"); + assert!(codestream.starts_with(&[0xFF, 0x4F])); + let decoded = Image::new(codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.width, 8); + assert_eq!(decoded.height, 8); + for y in 0..8usize { + for x in 0..8usize { + let dst = (y * 8 + x) * 3; + if x < 7 && y < 5 { + let src = (y * 7 + x) * 3; + assert_eq!(&decoded.data[dst..dst + 3], &pixels[src..src + 3]); + } else { + assert_eq!(&decoded.data[dst..dst + 3], &[0, 0, 0]); + } + } + } +} + +#[cfg(target_os = "macos")] +#[test] +fn submitted_private_padded_rgb8_encode_snapshots_before_wait() { + let pixels: Vec = (0..8 * 8 * 3).map(|i| ((i * 31) & 0xFF) as u8).collect(); + let replacement = vec![0u8; pixels.len()]; + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let submitted = super::submit_lossless_from_padded_metal_buffer( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("submit Metal private padded RGB8 encode"); + overwrite_private_buffer_with_bytes(&session, &buffer, &replacement); + + let encoded = submitted.wait().expect("wait submitted encode"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.data, pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_gray8_dwt_encode_uses_resident_coefficient_prep() { + let mut pixels = Vec::with_capacity(128 * 128); + for y in 0..128u32 { + for x in 0..128u32 { + pixels.push(((x * 7 + y * 11 + (x ^ y)) & 0xFF) as u8); + } + } + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 128, + height: 128, + pitch_bytes: 128, + output_width: 128, + output_height: 128, + format: PixelFormat::Gray8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("Metal private padded DWT buffer lossless encode"); + + assert_eq!(encoded.encoded.backend, BackendKind::Metal); + assert!(!encoded.input_copy_used); + assert!(encoded.resident.coefficient_prep_used); + assert!(encoded.resident.packetization_used); + assert!(encoded.resident.codestream_assembly_used); + let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_decoded_bytes_match(&decoded.data, &pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_rgb8_dwt_encode_uses_resident_coefficient_prep() { + let mut pixels = Vec::with_capacity(128 * 128 * 3); + for y in 0..128u32 { + for x in 0..128u32 { + pixels.push(((x * 3 + y * 5) & 0xFF) as u8); + pixels.push(((x * 7 + y * 11) & 0xFF) as u8); + pixels.push(((x * 13 + y * 17) & 0xFF) as u8); + } + } + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 128, + height: 128, + pitch_bytes: 128 * 3, + output_width: 128, + output_height: 128, + format: PixelFormat::Rgb8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("Metal private padded RGB8 DWT buffer lossless encode"); + + assert_eq!(encoded.encoded.backend, BackendKind::Metal); + assert!(encoded.resident.coefficient_prep_used); + assert!(encoded.resident.packetization_used); + assert!(encoded.resident.codestream_assembly_used); + let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_decoded_bytes_match(&decoded.data, &pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_gray8_dwt_resident_codestream_decodes_natively() { + let mut pixels = Vec::with_capacity(128 * 128); + for y in 0..128u32 { + for x in 0..128u32 { + pixels.push(((x * 7 + y * 11 + (x ^ y)) & 0xFF) as u8); + } + } + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 128, + height: 128, + pitch_bytes: 128, + output_width: 128, + output_height: 128, + format: PixelFormat::Gray8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + validation: J2kEncodeValidation::External, + }, + &session, + ) + .expect("Metal private padded DWT buffer lossless encode"); + + let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_decoded_bytes_match(&decoded.data, &pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_rgb8_dwt_resident_codestream_decodes_natively() { + let mut pixels = Vec::with_capacity(128 * 128 * 3); + for y in 0..128u32 { + for x in 0..128u32 { + pixels.push(((x * 3 + y * 5) & 0xFF) as u8); + pixels.push(((x * 7 + y * 11) & 0xFF) as u8); + pixels.push(((x * 13 + y * 17) & 0xFF) as u8); + } + } + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 128, + height: 128, + pitch_bytes: 128 * 3, + output_width: 128, + output_height: 128, + format: PixelFormat::Rgb8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + validation: J2kEncodeValidation::External, + }, + &session, + ) + .expect("Metal private padded RGB8 DWT buffer lossless encode"); + + let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_decoded_bytes_match(&decoded.data, &pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_gray8_rpcl_encode_uses_resident_coefficient_prep() { + let mut pixels = Vec::with_capacity(128 * 128); + for y in 0..128u32 { + for x in 0..128u32 { + pixels.push(((x * 5 + y * 9 + (x ^ y)) & 0xFF) as u8); + } + } + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 128, + height: 128, + pitch_bytes: 128, + output_width: 128, + output_height: 128, + format: PixelFormat::Gray8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + progression: J2kProgressionOrder::Rpcl, + }, + &session, + ) + .expect("Metal private padded RPCL buffer lossless encode"); + + assert_eq!(encoded.encoded.backend, BackendKind::Metal); + assert!(encoded.resident.coefficient_prep_used); + assert!(encoded.resident.packetization_used); + assert!(encoded.resident.codestream_assembly_used); + let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.data, pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_gray16_encode_uses_resident_coefficient_prep() { + let mut pixels = Vec::with_capacity(8 * 8 * 2); + for idx in 0..64u16 { + let value = idx.wrapping_mul(997).wrapping_add(123); + pixels.extend_from_slice(&value.to_le_bytes()); + } + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8 * 2, + output_width: 8, + output_height: 8, + format: PixelFormat::Gray16, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("Metal private padded Gray16 buffer lossless encode"); + + assert_eq!(encoded.encoded.backend, BackendKind::Metal); + assert!(!encoded.input_copy_used); + assert!(encoded.resident.coefficient_prep_used); + assert!(encoded.resident.packetization_used); + assert!(encoded.resident.codestream_assembly_used); + let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.data, pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_ht_encode_to_metal_buffer_stays_resident() { + let pixels: Vec = (0..8 * 8).map(|i| ((i * 31) & 0xFF) as u8).collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_padded_metal_buffer_to_metal_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8, + output_width: 8, + output_height: 8, + format: PixelFormat::Gray8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + }, + &session, + ) + .expect("Metal private padded HTJ2K buffer lossless encode"); + + assert!(!encoded.input_copy_used); + assert!(encoded.resident.coefficient_prep_used); + assert!(encoded.resident.packetization_used); + assert!(encoded.resident.codestream_assembly_used); + let codestream = encoded + .encoded + .codestream_bytes() + .expect("Metal codestream bytes are CPU-readable"); + assert!(codestream.windows(2).any(|window| window == [0xFF, 0x50])); + let cod_marker = codestream + .windows(2) + .position(|window| window == [0xFF, 0x52]) + .expect("COD marker"); + assert_eq!(codestream[cod_marker + 12], 0x40); + let decoded = Image::new(codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.data, pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_rgb8_ht_rpcl_512_encode_preserves_three_dwt_levels_and_stays_resident() { + let pixels: Vec = (0..512 * 512 * 3) + .map(|idx| ((idx * 47 + idx / 17) & 0xFF) as u8) + .collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffer = private_buffer_with_bytes(&session, &pixels); + + let encoded = super::encode_lossless_from_padded_metal_buffer_to_metal_with_report( + super::MetalLosslessEncodeTile { + buffer: &buffer, + byte_offset: 0, + width: 512, + height: 512, + pitch_bytes: 512 * 3, + output_width: 512, + output_height: 512, + format: PixelFormat::Rgb8, + }, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + progression: J2kProgressionOrder::Rpcl, + }, + &session, + ) + .expect("Metal private padded HTJ2K RPCL 512 buffer lossless encode"); + + assert!(!encoded.input_copy_used); + assert!(encoded.resident.coefficient_prep_used); + assert!(encoded.resident.packetization_used); + assert!(encoded.resident.codestream_assembly_used); + let codestream = encoded + .encoded + .codestream_bytes() + .expect("Metal codestream bytes are CPU-readable"); + let cod_marker = codestream + .windows(2) + .position(|window| window == [0xFF, 0x52]) + .expect("COD marker"); + assert_eq!(codestream[cod_marker + 5], 0x02); + assert_eq!(codestream[cod_marker + 9], 3); + assert_eq!(codestream[cod_marker + 12], 0x40); + let decoded = Image::new(codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.data, pixels); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_rgb8_ht_batch_uses_fused_deinterleave_rct_kernel() { + const WIDTH: usize = 32; + const HEIGHT: usize = 32; + let first: Vec = (0..WIDTH * HEIGHT * 3) + .map(|idx| ((idx * 29 + idx / 7) & 0xFF) as u8) + .collect(); + let second: Vec = (0..WIDTH * HEIGHT * 3) + .map(|idx| 255u8.wrapping_sub(((idx * 13 + idx / 5) & 0xFF) as u8)) + .collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let first_buffer = private_buffer_with_bytes(&session, &first); + let second_buffer = private_buffer_with_bytes(&session, &second); + let tiles = [ + super::MetalLosslessEncodeTile { + buffer: &first_buffer, + byte_offset: 0, + width: WIDTH as u32, + height: HEIGHT as u32, + pitch_bytes: WIDTH * 3, + output_width: WIDTH as u32, + output_height: HEIGHT as u32, + format: PixelFormat::Rgb8, + }, + super::MetalLosslessEncodeTile { + buffer: &second_buffer, + byte_offset: 0, + width: WIDTH as u32, + height: HEIGHT as u32, + pitch_bytes: WIDTH * 3, + output_width: WIDTH as u32, + output_height: HEIGHT as u32, + format: PixelFormat::Rgb8, + }, + ]; + let options = lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + validation: J2kEncodeValidation::External, + }; + + compute::reset_lossless_deinterleave_rct_fused_dispatches_for_test(); + let encoded = super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( + &tiles, &options, &session, + ) + .expect("Metal RGB8 HTJ2K batch encode"); + + assert_eq!(encoded.len(), 2); + assert!( + compute::lossless_deinterleave_rct_fused_dispatches_for_test() > 0, + "RGB8 resident lossless encode should fuse deinterleave and RCT" + ); + for (frame, expected) in encoded.iter().zip([first, second]) { + let codestream = frame + .encoded + .codestream_bytes() + .expect("Metal codestream bytes are CPU-readable"); + let decoded = Image::new(codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.data, expected); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_buffer_lossless_batch_encodes_padded_contiguous_inputs() { + let first: Vec = (0..8 * 8 * 3).map(|i| ((i * 7) & 0xFF) as u8).collect(); + let second: Vec = (0..8 * 8 * 3) + .map(|i| ((i * 13 + 5) & 0xFF) as u8) + .collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let first_buffer = session.device().new_buffer_with_data( + first.as_ptr().cast(), + first.len() as u64, + metal::MTLResourceOptions::StorageModeShared, + ); + let second_buffer = session.device().new_buffer_with_data( + second.as_ptr().cast(), + second.len() as u64, + metal::MTLResourceOptions::StorageModeShared, + ); + let tiles = [ + super::MetalLosslessEncodeTile { + buffer: &first_buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + super::MetalLosslessEncodeTile { + buffer: &second_buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + ]; + + let encoded = super::encode_lossless_from_padded_metal_buffers_with_report( + &tiles, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("Metal padded buffer batch lossless encode"); + + assert_eq!(encoded.len(), 2); + for (frame, expected) in encoded.iter().zip([first, second]) { + assert_eq!(frame.encoded.backend, BackendKind::Metal); + assert!(!frame.input_copy_used); + let decoded = Image::new(&frame.encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.data, expected); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_batch_encode_to_metal_buffers_exposes_per_frame_bytes() { + let first: Vec = (0..8 * 8 * 3).map(|i| ((i * 17) & 0xFF) as u8).collect(); + let second: Vec = (0..8 * 8 * 3) + .map(|i| 255u8.wrapping_sub(((i * 23) & 0xFF) as u8)) + .collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let first_buffer = private_buffer_with_bytes(&session, &first); + let second_buffer = private_buffer_with_bytes(&session, &second); + let tiles = [ + super::MetalLosslessEncodeTile { + buffer: &first_buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + super::MetalLosslessEncodeTile { + buffer: &second_buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + ]; + + let encoded = super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( + &tiles, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("Metal padded buffer batch lossless encode to Metal buffers"); + + assert_eq!(encoded.len(), 2); + assert_eq!( + encoded[0].encoded.codestream_buffer.as_ptr(), + encoded[1].encoded.codestream_buffer.as_ptr(), + "classic J2K resident batch encode should assemble codestreams into one shared batch buffer" + ); + assert_eq!(encoded[0].encoded.byte_offset, 0); + assert!( + encoded[1].encoded.byte_offset > 0, + "second classic J2K batch codestream should be a nonzero slice into the shared batch buffer" + ); + for (frame, expected) in encoded.iter().zip([first, second]) { + assert!(!frame.input_copy_used); + assert!(frame.resident.coefficient_prep_used); + assert!(frame.resident.packetization_used); + assert!(frame.resident.codestream_assembly_used); + let codestream = frame + .encoded + .codestream_bytes() + .expect("Metal codestream bytes are CPU-readable"); + let decoded = Image::new(codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.data, expected); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_padded_private_batch_dwt_encode_to_metal_buffers_round_trips() { + let first: Vec = (0..128 * 128 * 3) + .map(|i| ((i * 17 + i / 3) & 0xFF) as u8) + .collect(); + let second: Vec = (0..128 * 128 * 3) + .map(|i| 255u8.wrapping_sub(((i * 23 + i / 5) & 0xFF) as u8)) + .collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let first_buffer = private_buffer_with_bytes(&session, &first); + let second_buffer = private_buffer_with_bytes(&session, &second); + let tiles = [ + super::MetalLosslessEncodeTile { + buffer: &first_buffer, + byte_offset: 0, + width: 128, + height: 128, + pitch_bytes: 128 * 3, + output_width: 128, + output_height: 128, + format: PixelFormat::Rgb8, + }, + super::MetalLosslessEncodeTile { + buffer: &second_buffer, + byte_offset: 0, + width: 128, + height: 128, + pitch_bytes: 128 * 3, + output_width: 128, + output_height: 128, + format: PixelFormat::Rgb8, + }, + ]; + + let encoded = super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( + &tiles, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + validation: J2kEncodeValidation::External, + }, + &session, + ) + .expect("Metal padded DWT buffer batch lossless encode to Metal buffers"); + + assert_eq!(encoded.len(), 2); + for (frame, expected) in encoded.iter().zip([first, second]) { + assert!(!frame.input_copy_used); + assert!(frame.resident.coefficient_prep_used); + assert!(frame.resident.packetization_used); + assert!(frame.resident.codestream_assembly_used); + let codestream = frame + .encoded + .codestream_bytes() + .expect("Metal codestream bytes are CPU-readable"); + let decoded = Image::new(codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_decoded_bytes_match(&decoded.data, &expected); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_edge_private_batch_encode_to_metal_buffers_stays_resident() { + let first: Vec = (0..7 * 5 * 3).map(|i| ((i * 17) & 0xFF) as u8).collect(); + let second: Vec = (0..6 * 8 * 3) + .map(|i| 255u8.wrapping_sub(((i * 19) & 0xFF) as u8)) + .collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let first_buffer = private_buffer_with_bytes(&session, &first); + let second_buffer = private_buffer_with_bytes(&session, &second); + compute::reset_ht_batch_coefficient_copy_blits_for_test(); + let tiles = [ + super::MetalLosslessEncodeTile { + buffer: &first_buffer, + byte_offset: 0, + width: 7, + height: 5, + pitch_bytes: 7 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + super::MetalLosslessEncodeTile { + buffer: &second_buffer, + byte_offset: 0, + width: 6, + height: 8, + pitch_bytes: 6 * 3, + output_width: 8, + output_height: 8, + format: PixelFormat::Rgb8, + }, + ]; + + let encoded = super::encode_lossless_from_metal_buffers_to_metal_with_report( + &tiles, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + }, + &session, + ) + .expect("Metal edge buffer batch lossless encode to Metal buffers"); + + assert_eq!(encoded.len(), 2); + for frame in &encoded { + assert!(!frame.input_copy_used); + assert!(frame.resident.coefficient_prep_used); + assert!(frame.resident.packetization_used); + assert!(frame.resident.codestream_assembly_used); + } + + for (frame, (expected, width, height)) in encoded + .iter() + .zip([(first, 7usize, 5usize), (second, 6usize, 8usize)]) + { + let codestream = frame + .encoded + .codestream_bytes() + .expect("Metal codestream bytes are CPU-readable"); + let decoded = Image::new(codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + for y in 0..8usize { + for x in 0..8usize { + let dst = (y * 8 + x) * 3; + if x < width && y < height { + let src = (y * width + x) * 3; + assert_eq!(&decoded.data[dst..dst + 3], &expected[src..src + 3]); + } else { + assert_eq!(&decoded.data[dst..dst + 3], &[0, 0, 0]); + } + } + } + } +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_ht_private_batch_encode_to_metal_buffers_stays_resident() { + let first: Vec = (0..8 * 8).map(|i| ((i * 11) & 0xFF) as u8).collect(); + let second: Vec = (0..8 * 8) + .map(|i| 255u8.wrapping_sub(((i * 13) & 0xFF) as u8)) + .collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let first_buffer = private_buffer_with_bytes(&session, &first); + let second_buffer = private_buffer_with_bytes(&session, &second); + let tiles = [ + super::MetalLosslessEncodeTile { + buffer: &first_buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8, + output_width: 8, + output_height: 8, + format: PixelFormat::Gray8, + }, + super::MetalLosslessEncodeTile { + buffer: &second_buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8, + output_width: 8, + output_height: 8, + format: PixelFormat::Gray8, + }, + ]; + + compute::reset_resident_gpu_timestamp_queries_for_test(); + let encoded = super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( + &tiles, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + }, + &session, + ) + .expect("Metal HTJ2K batch lossless encode to Metal buffers"); + + assert_eq!(encoded.len(), 2); + assert_eq!( + compute::ht_batch_coefficient_copy_blits_for_test(), + 0, + "HTJ2K resident batch prep should write directly into the batch coefficient buffer" + ); + assert_eq!( + compute::resident_gpu_timestamp_queries_for_test(), + 7, + "HTJ2K resident batch should query each unique retained command buffer timestamp once" + ); + assert_eq!( + encoded[0].encoded.codestream_buffer.as_ptr(), + encoded[1].encoded.codestream_buffer.as_ptr(), + "HTJ2K resident batch encode should assemble codestreams into one shared batch buffer" + ); + assert_eq!(encoded[0].encoded.byte_offset, 0); + assert!( + encoded[1].encoded.byte_offset > 0, + "second HTJ2K batch codestream should be a nonzero slice into the shared batch buffer" + ); + for (frame, expected) in encoded.iter().zip([first, second]) { + assert!(!frame.input_copy_used); + assert!(frame.resident.coefficient_prep_used); + assert!(frame.resident.packetization_used); + assert!(frame.resident.codestream_assembly_used); + let codestream = frame + .encoded + .codestream_bytes() + .expect("Metal codestream bytes are CPU-readable"); + assert!(codestream.windows(2).any(|window| window == [0xFF, 0x50])); + let decoded = Image::new(codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(decoded.data, expected); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_ht_private_batch_encode_reuses_private_arenas_between_batches() { + const WIDTH: usize = 37; + const HEIGHT: usize = 41; + let first: Vec = (0..WIDTH * HEIGHT) + .map(|i| ((i * 7 + 3) & 0xFF) as u8) + .collect(); + let second: Vec = (0..WIDTH * HEIGHT) + .map(|i| 255u8.wrapping_sub(((i * 5 + 11) & 0xFF) as u8)) + .collect(); + let device = metal::Device::system_default().expect("Metal device"); + let session = crate::MetalBackendSession::new(device.clone()); + let first_buffer = private_buffer_with_bytes(&session, &first); + let second_buffer = private_buffer_with_bytes(&session, &second); + let tiles = [ + super::MetalLosslessEncodeTile { + buffer: &first_buffer, + byte_offset: 0, + width: WIDTH as u32, + height: HEIGHT as u32, + pitch_bytes: WIDTH, + output_width: WIDTH as u32, + output_height: HEIGHT as u32, + format: PixelFormat::Gray8, + }, + super::MetalLosslessEncodeTile { + buffer: &second_buffer, + byte_offset: 0, + width: WIDTH as u32, + height: HEIGHT as u32, + pitch_bytes: WIDTH, + output_width: WIDTH as u32, + output_height: HEIGHT as u32, + format: PixelFormat::Gray8, + }, + ]; + let options = lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + validation: J2kEncodeValidation::External, + }; + + compute::with_isolated_runtime_for_device_for_test(&device, || { + compute::reset_private_buffer_pool_misses_for_test(); + super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( + &tiles, &options, &session, + )?; + let first_misses = compute::private_buffer_pool_misses_for_test(); + assert!( + first_misses > 0, + "first unique HTJ2K batch should populate reusable private arenas" + ); + + compute::reset_private_buffer_pool_misses_for_test(); + let encoded = super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( + &tiles, &options, &session, + )?; + + assert_eq!( + compute::private_buffer_pool_misses_for_test(), + 0, + "second same-shape HTJ2K batch should reuse private arenas" + ); + assert_eq!(encoded.len(), 2); + Ok(()) + }) + .expect("isolated HTJ2K Metal runtime"); +} + +#[test] +fn default_gpu_encode_memory_budget_uses_forty_percent_capped_at_ten_gib() { + const GIB: usize = 1024 * 1024 * 1024; + + assert_eq!( + super::default_gpu_encode_memory_budget_bytes_for_hw_mem(8 * GIB), + 8 * GIB * 40 / 100 + ); + assert_eq!( + super::default_gpu_encode_memory_budget_bytes_for_hw_mem(16 * GIB), + 16 * GIB * 40 / 100 + ); + assert_eq!( + super::default_gpu_encode_memory_budget_bytes_for_hw_mem(24 * GIB), + 24 * GIB * 40 / 100 + ); + assert_eq!( + super::default_gpu_encode_memory_budget_bytes_for_hw_mem(64 * GIB), + 10 * GIB + ); +} + +#[test] +fn gpu_encode_inflight_resolution_clamps_requested_tiles_by_memory_budget() { + let stats = super::resolve_lossless_encode_config_for_test( + 100, + 1_000, + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(32), + gpu_encode_memory_budget_bytes: Some(4_500), + }, + ) + .expect("resolved config"); + + assert_eq!(stats.configured_inflight_tiles, Some(32)); + assert_eq!(stats.effective_inflight_tiles, 4); + assert_eq!(stats.configured_memory_budget_bytes, Some(4_500)); + assert_eq!(stats.effective_memory_budget_bytes, 4_500); + assert_eq!(stats.estimated_peak_bytes_per_tile, 1_000); +} + +#[test] +fn gpu_encode_default_inflight_uses_large_wsi_batch_when_memory_allows() { + let stats = super::resolve_lossless_encode_config_for_test( + 600, + 1_000, + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: None, + gpu_encode_memory_budget_bytes: Some(1_000_000), + }, + ) + .expect("resolved config"); + + assert_eq!(stats.configured_inflight_tiles, None); + assert_eq!(stats.effective_inflight_tiles, 512); +} + +#[test] +fn resident_classic_encode_default_inflight_uses_profiled_cap() { + let config = super::resident_lossless_encode_config_for_mode( + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: None, + gpu_encode_memory_budget_bytes: Some(1_000_000), + }, + true, + 16, + ); + + assert_eq!(config.gpu_encode_inflight_tiles, Some(16)); + assert_eq!(config.gpu_encode_memory_budget_bytes, Some(1_000_000)); +} + +#[test] +fn resident_classic_encode_default_inflight_uses_large_batch_cap() { + let config = super::resident_lossless_encode_config_for_mode( + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: None, + gpu_encode_memory_budget_bytes: Some(1_000_000), + }, + true, + 64, + ); + + assert_eq!(config.gpu_encode_inflight_tiles, Some(64)); + assert_eq!(config.gpu_encode_memory_budget_bytes, Some(1_000_000)); +} + +#[test] +fn resident_classic_encode_default_inflight_uses_very_large_batch_cap() { + let config = super::resident_lossless_encode_config_for_mode( + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: None, + gpu_encode_memory_budget_bytes: Some(1_000_000), + }, + true, + 128, + ); + + assert_eq!(config.gpu_encode_inflight_tiles, Some(128)); + assert_eq!(config.gpu_encode_memory_budget_bytes, Some(1_000_000)); +} + +#[test] +fn resident_htj2k_encode_medium_batch_default_inflight_uses_profiled_cap() { + let config = super::resident_lossless_encode_config_for_mode( + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: None, + gpu_encode_memory_budget_bytes: Some(1_000_000), + }, + false, + 64, + ); + + assert_eq!(config.gpu_encode_inflight_tiles, Some(32)); + assert_eq!(config.gpu_encode_memory_budget_bytes, Some(1_000_000)); +} + +#[test] +fn resident_htj2k_encode_large_batch_default_inflight_uses_profiled_cap() { + let config = super::resident_lossless_encode_config_for_mode( + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: None, + gpu_encode_memory_budget_bytes: Some(1_000_000), + }, + false, + 128, + ); + + assert_eq!(config.gpu_encode_inflight_tiles, Some(64)); + assert_eq!(config.gpu_encode_memory_budget_bytes, Some(1_000_000)); +} + +#[test] +fn gpu_encode_inflight_resolution_rejects_zero_overrides() { + let err = super::resolve_lossless_encode_config_for_test( + 4, + 1_000, + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(0), + gpu_encode_memory_budget_bytes: Some(4_000), + }, + ) + .unwrap_err(); + assert!( + err.to_string().contains("in-flight"), + "unexpected error: {err}" + ); + + let err = super::resolve_lossless_encode_config_for_test( + 4, + 1_000, + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(2), + gpu_encode_memory_budget_bytes: Some(0), + }, + ) + .unwrap_err(); + assert!( + err.to_string().contains("memory budget"), + "unexpected error: {err}" + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_ht_batch_encode_preserves_order_and_matches_inflight_one() { + let inputs = [ + (0..8 * 8) + .map(|i| ((i * 11 + 3) & 0xFF) as u8) + .collect::>(), + (0..8 * 8) + .map(|i| ((i * 13 + 5) & 0xFF) as u8) + .collect::>(), + (0..8 * 8) + .map(|i| ((i * 17 + 7) & 0xFF) as u8) + .collect::>(), + (0..8 * 8) + .map(|i| ((i * 19 + 9) & 0xFF) as u8) + .collect::>(), + ]; + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let buffers = inputs + .iter() + .map(|bytes| private_buffer_with_bytes(&session, bytes)) + .collect::>(); + let tiles = buffers + .iter() + .map(|buffer| super::MetalLosslessEncodeTile { + buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8, + output_width: 8, + output_height: 8, + format: PixelFormat::Gray8, + }) + .collect::>(); + let options = lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + validation: J2kEncodeValidation::External, + }; + + compute::reset_resident_codestream_command_buffer_waits_for_test(); + let serial = super::encode_lossless_from_padded_metal_buffers_to_metal_batch( + &tiles, + &options, + &session, + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(1), + gpu_encode_memory_budget_bytes: Some(1024 * 1024 * 1024), + }, + ) + .expect("serial Metal HTJ2K batch"); + assert_eq!( + compute::resident_codestream_command_buffer_waits_for_test(), + 1, + "multi-chunk HT batch should wait once before harvesting completed chunks" + ); + + let cpu_validated_options = lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + validation: J2kEncodeValidation::CpuRoundTrip, + }; + compute::reset_resident_codestream_command_buffer_waits_for_test(); + let cpu_validated = super::encode_lossless_from_padded_metal_buffers_to_metal_batch( + &tiles, + &cpu_validated_options, + &session, + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(1), + gpu_encode_memory_budget_bytes: Some(1024 * 1024 * 1024), + }, + ) + .expect("CPU-validated Metal HTJ2K batch"); + assert_eq!(cpu_validated.outcomes.len(), inputs.len()); + assert_eq!( + compute::resident_codestream_command_buffer_waits_for_test(), + inputs.len(), + "CPU roundtrip validation should keep per-chunk waits to preserve overlap" + ); + + let parallel = super::encode_lossless_from_padded_metal_buffers_to_metal_batch( + &tiles, + &options, + &session, + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(2), + gpu_encode_memory_budget_bytes: Some(1024 * 1024 * 1024), + }, + ) + .expect("parallel Metal HTJ2K batch"); + let repeated_parallel = super::encode_lossless_from_padded_metal_buffers_to_metal_batch( + &tiles, + &options, + &session, + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(2), + gpu_encode_memory_budget_bytes: Some(1024 * 1024 * 1024), + }, + ) + .expect("repeated parallel Metal HTJ2K batch"); + + assert_eq!(serial.outcomes.len(), inputs.len()); + assert_eq!(parallel.outcomes.len(), inputs.len()); + assert_eq!(parallel.stats.effective_inflight_tiles, 2); + assert!(parallel.stats.max_observed_inflight_tiles <= 2); + assert!(parallel.stats.max_observed_inflight_tiles > 0); + for (((serial_outcome, parallel_outcome), repeated_outcome), expected) in serial + .outcomes + .iter() + .zip(parallel.outcomes.iter()) + .zip(repeated_parallel.outcomes.iter()) + .zip(inputs.iter()) + { + let serial_bytes = serial_outcome + .encoded + .codestream_bytes() + .expect("serial codestream"); + let parallel_bytes = parallel_outcome + .encoded + .codestream_bytes() + .expect("parallel codestream"); + let repeated_bytes = repeated_outcome + .encoded + .codestream_bytes() + .expect("repeated parallel codestream"); + assert_eq!(parallel_bytes, serial_bytes); + assert_eq!(repeated_bytes, serial_bytes); + + let decoded = Image::new(parallel_bytes, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + assert_eq!(&decoded.data, expected); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_parallel_batch_returns_indexed_injected_failure() { + let first: Vec = (0..8 * 8).map(|i| ((i * 3) & 0xFF) as u8).collect(); + let second: Vec = (0..8 * 8).map(|i| ((i * 5) & 0xFF) as u8).collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let first_buffer = private_buffer_with_bytes(&session, &first); + let second_buffer = private_buffer_with_bytes(&session, &second); + let tiles = [ + super::MetalLosslessEncodeTile { + buffer: &first_buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8, + output_width: 8, + output_height: 8, + format: PixelFormat::Gray8, + }, + super::MetalLosslessEncodeTile { + buffer: &second_buffer, + byte_offset: 0, + width: 8, + height: 8, + pitch_bytes: 8, + output_width: 8, + output_height: 8, + format: PixelFormat::Gray8, + }, + ]; + let options = lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + validation: J2kEncodeValidation::External, + }; + + super::set_test_resident_encode_failure_index(Some(1)); + let Err(err) = super::encode_lossless_from_padded_metal_buffers_to_metal_batch( + &tiles, + &options, + &session, + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(2), + gpu_encode_memory_budget_bytes: Some(1024 * 1024 * 1024), + }, + ) else { + panic!("injected failure should fail the batch"); + }; + super::set_test_resident_encode_failure_index(None); + + assert!( + err.to_string().contains("tile 1"), + "unexpected error: {err}" + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_forward_dwt53_dispatch_round_trips_gray8_lossless_tile() { + let pixels: Vec = (0..64 * 64).map(|i| ((i * 5) & 0xFF) as u8).collect(); + let samples = + J2kLosslessSamples::new(&pixels, 64, 64, 1, 8, false).expect("valid gray samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_max_decomposition_levels(Some(1)); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &options, + BackendKind::Metal, + &mut accelerator, + ) + .expect("encode with metal forward DWT 5/3"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert_eq!(accelerator.forward_dwt53_attempts(), 1); + assert_eq!(accelerator.forward_dwt53_dispatches(), 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_lossless_facade_dispatches_rct_and_dwt_for_wsi_sized_rgb_tile() { + let mut pixels = Vec::with_capacity(128 * 128 * 3); + for y in 0..128u32 { + for x in 0..128u32 { + pixels.push(((x * 3 + y * 5) & 0xFF) as u8); + pixels.push(((x * 7 + y * 11) & 0xFF) as u8); + pixels.push(((x * 13 + y * 17) & 0xFF) as u8); + } + } + let samples = + J2kLosslessSamples::new(&pixels, 128, 128, 3, 8, false).expect("valid RGB samples"); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &lossless_options! { + backend: EncodeBackendPreference::Auto, + }, + BackendKind::Metal, + &mut accelerator, + ) + .expect("Metal-accelerated lossless encode"); + + assert_eq!(encoded.backend, BackendKind::Metal); + assert_eq!(accelerator.forward_rct_dispatches(), 1); + assert_eq!(accelerator.forward_dwt53_dispatches(), 3); + assert!(accelerator.tier1_code_block_attempts() > 0); + assert_eq!(accelerator.packetization_attempts(), 1); + assert!(accelerator.tier1_code_block_dispatches() > 0); + assert_eq!(accelerator.packetization_dispatches(), 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_classic_tier1_uses_one_batched_dispatch_for_multiple_code_blocks() { + let pixels: Vec = (0..256 * 256) + .map(|idx| ((idx * 17 + 3) & 0xFF) as u8) + .collect(); + let samples = + J2kLosslessSamples::new(&pixels, 256, 256, 1, 8, false).expect("valid gray samples"); + let options = J2kLosslessEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_max_decomposition_levels(Some(0)); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &options, + BackendKind::Metal, + &mut accelerator, + ) + .expect("encode with batched Metal classic Tier-1"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(decoded.data, pixels); + assert!(accelerator.tier1_code_block_attempts() > 1); + assert_eq!(accelerator.tier1_code_block_dispatches(), 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_classic_resident_uses_mq_byte_split_gpu_token_pack_by_default() { + let _profile_guard = compute::force_metal_profile_stages_for_test(true); + compute::reset_classic_gpu_token_pack_dispatches_for_test(); + compute::reset_classic_split_mq_byte_gpu_token_pack_dispatches_for_test(); + let first: Vec = (0..256 * 256) + .map(|idx| { + let x = idx % 256; + let y = idx / 256; + ((x + y * 5) & 0xFF) as u8 + }) + .collect(); + let second: Vec = (0..256 * 256) + .map(|idx| { + let x = idx % 256; + let y = idx / 256; + ((x * 3 + y * 7) & 0xFF) as u8 + }) + .collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let first_buffer = private_buffer_with_bytes(&session, &first); + let second_buffer = private_buffer_with_bytes(&session, &second); + let tiles = [ + super::MetalLosslessEncodeTile { + buffer: &first_buffer, + byte_offset: 0, + width: 256, + height: 256, + pitch_bytes: 256, + output_width: 256, + output_height: 256, + format: PixelFormat::Gray8, + }, + super::MetalLosslessEncodeTile { + buffer: &second_buffer, + byte_offset: 0, + width: 256, + height: 256, + pitch_bytes: 256, + output_width: 256, + output_height: 256, + format: PixelFormat::Gray8, + }, + ]; + + let encoded = super::encode_lossless_from_padded_metal_buffers_to_metal_batch( + &tiles, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::Classic, + validation: J2kEncodeValidation::External, + }, + &session, + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(2), + gpu_encode_memory_budget_bytes: Some(1024 * 1024 * 1024), + }, + ) + .expect("resident batch encode with default MQ-byte GPU token-pack Classic Tier-1"); + assert_eq!(encoded.outcomes.len(), 2); + for (outcome, expected) in encoded.outcomes.iter().zip([&first, &second]) { + let codestream = outcome + .encoded + .to_encoded_j2k() + .expect("codestream readback"); + let decoded = Image::new(&codestream.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(codestream.backend, BackendKind::Metal); + assert_eq!(&decoded.data, expected); + } + assert!( + compute::classic_gpu_token_pack_dispatches_for_test() > 0, + "default Classic GPU token-pack route was not dispatched" + ); + assert!( + compute::classic_split_mq_byte_gpu_token_pack_dispatches_for_test() > 0, + "default Classic GPU token-pack route did not use MQ-byte split token emit" + ); + assert_eq!( + encoded + .stats + .stage_stats + .tier1_token_pack_output_bytes_total, + encoded.stats.stage_stats.tier1_output_used_bytes_total, + "default Classic GPU token-pack route should attribute Tier-1 output bytes to token pack" + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_classic_resident_gpu_token_pack_route_round_trips() { + let _guard = compute::force_classic_gpu_token_pack_route_for_test(true); + let _profile_guard = compute::force_metal_profile_stages_for_test(true); + compute::reset_classic_gpu_token_pack_dispatches_for_test(); + let first: Vec = (0..256 * 256) + .map(|idx| { + let x = idx % 256; + let y = idx / 256; + ((x + y * 3) & 0xFF) as u8 + }) + .collect(); + let second: Vec = (0..256 * 256) + .map(|idx| { + let x = idx % 256; + let y = idx / 256; + ((x * 2 + y) & 0xFF) as u8 + }) + .collect(); + let session = crate::MetalBackendSession::system_default().expect("Metal session"); + let first_buffer = private_buffer_with_bytes(&session, &first); + let second_buffer = private_buffer_with_bytes(&session, &second); + let tiles = [ + super::MetalLosslessEncodeTile { + buffer: &first_buffer, + byte_offset: 0, + width: 256, + height: 256, + pitch_bytes: 256, + output_width: 256, + output_height: 256, + format: PixelFormat::Gray8, + }, + super::MetalLosslessEncodeTile { + buffer: &second_buffer, + byte_offset: 0, + width: 256, + height: 256, + pitch_bytes: 256, + output_width: 256, + output_height: 256, + format: PixelFormat::Gray8, + }, + ]; + + let encoded = super::encode_lossless_from_padded_metal_buffers_to_metal_batch( + &tiles, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::Classic, + validation: J2kEncodeValidation::External, + }, + &session, + super::MetalLosslessEncodeConfig { + gpu_encode_inflight_tiles: Some(2), + gpu_encode_memory_budget_bytes: Some(1024 * 1024 * 1024), + }, + ) + .expect("resident batch encode with gated GPU token-pack Classic Tier-1"); + assert_eq!(encoded.outcomes.len(), 2); + for (outcome, expected) in encoded.outcomes.iter().zip([&first, &second]) { + let codestream = outcome + .encoded + .to_encoded_j2k() + .expect("codestream readback"); + let decoded = Image::new(&codestream.codestream, &DecodeSettings::default()) + .expect("codestream parses") + .decode_native() + .expect("codestream decodes"); + + assert_eq!(codestream.backend, BackendKind::Metal); + assert_eq!(&decoded.data, expected); + } + assert!( + compute::classic_gpu_token_pack_dispatches_for_test() > 0, + "gated Classic GPU token-pack route was not dispatched" + ); + assert!( + encoded.stats.stage_stats.tier1_token_emit_token_bytes_total > 0, + "gated Classic GPU token-pack route did not expose token-emitter byte counters" + ); + assert!( + encoded + .stats + .stage_stats + .tier1_token_emit_segment_count_total + > 0, + "gated Classic GPU token-pack route did not expose token segment counters" + ); + assert_eq!( + encoded + .stats + .stage_stats + .tier1_token_pack_output_bytes_total, + encoded.stats.stage_stats.tier1_output_used_bytes_total, + "gated Classic GPU token-pack route should attribute Tier-1 output bytes to token pack" + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_htj2k_uses_one_batched_dispatch_for_multiple_code_blocks() { + let pixels: Vec = (0..256 * 256) + .map(|idx| ((idx * 23 + 9) & 0xFF) as u8) + .collect(); + let samples = + J2kLosslessSamples::new(&pixels, 256, 256, 1, 8, false).expect("valid gray samples"); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + }, + BackendKind::Metal, + &mut accelerator, + ) + .expect("Metal-accelerated HTJ2K lossless encode"); + + assert_eq!(encoded.backend, BackendKind::Metal); + assert!(accelerator.ht_code_block_attempts() > 1); + assert_eq!(accelerator.ht_code_block_dispatches(), 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_htj2k_lossless_facade_dispatches_ht_code_blocks_and_packetization() { + let pixels: Vec = (0..64).map(|value| ((value * 13) & 0xFF) as u8).collect(); + let samples = J2kLosslessSamples::new(&pixels, 8, 8, 1, 8, false).expect("valid gray samples"); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &lossless_options! { + backend: EncodeBackendPreference::RequireDevice, + block_coding_mode: J2kBlockCodingMode::HighThroughput, + }, + BackendKind::Metal, + &mut accelerator, + ) + .expect("Metal-accelerated HTJ2K lossless encode"); + + assert_eq!(encoded.backend, BackendKind::Metal); + assert_eq!(encoded.dispatch_report.deinterleave, 1); + assert_eq!(accelerator.deinterleave_attempts(), 1); + assert_eq!(accelerator.deinterleave_dispatches(), 1); + assert!(encoded.dispatch_report.quantize_subband > 0); + assert!(accelerator.quantize_subband_attempts() > 0); + assert!(accelerator.quantize_subband_dispatches() > 0); + assert!(accelerator.ht_code_block_attempts() > 0); + assert!(accelerator.ht_code_block_dispatches() > 0); + assert_eq!(accelerator.packetization_attempts(), 1); + assert_eq!(accelerator.packetization_dispatches(), 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_htj2k_lossy_facade_require_device_dispatches_supported_stages() { + let pixels: Vec = (0..16 * 16) + .map(|idx| ((idx * 17 + idx / 3) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 16, 16, 1, 8, false).expect("valid gray samples"); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let encoded = encode_j2k_lossy_with_accelerator( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::CpuRoundTrip), + BackendKind::Metal, + &mut accelerator, + ) + .expect("Metal-accelerated HTJ2K lossy encode"); + + assert_eq!(encoded.backend, BackendKind::Metal); + assert_eq!(encoded.dispatch_report.deinterleave, 1); + assert_eq!(accelerator.deinterleave_attempts(), 1); + assert_eq!(accelerator.deinterleave_dispatches(), 1); + assert!(encoded.dispatch_report.quantize_subband > 0); + assert!(accelerator.quantize_subband_attempts() > 0); + assert!(accelerator.quantize_subband_dispatches() > 0); + assert!(accelerator.ht_code_block_attempts() > 0); + assert!(accelerator.ht_code_block_dispatches() > 0); + assert_eq!(accelerator.packetization_attempts(), 1); + assert_eq!(accelerator.packetization_dispatches(), 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_htj2k_lossy_rgb_facade_reports_forward_ict_dispatch() { + let width = 16; + let height = 16; + let pixels: Vec = (0..width * height * 3) + .map(|idx| ((idx * 19 + idx / 5 + 7) & 0xFF) as u8) + .collect(); + let samples = + J2kLossySamples::new(&pixels, width, height, 3, 8, false).expect("valid RGB samples"); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let encoded = encode_j2k_lossy_with_accelerator( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(0)) + .with_validation(J2kEncodeValidation::CpuRoundTrip), + BackendKind::Metal, + &mut accelerator, + ) + .expect("Metal-accelerated RGB HTJ2K lossy encode"); + + assert_eq!(encoded.backend, BackendKind::Metal); + assert_eq!(encoded.dispatch_report.deinterleave, 1); + assert_eq!(encoded.dispatch_report.forward_ict, 1); + assert_eq!(encoded.dispatch_report.forward_rct, 0); + assert_eq!(encoded.dispatch_report.forward_dwt97, 0); + assert!(encoded.dispatch_report.quantize_subband > 0); + assert_eq!(accelerator.forward_ict_attempts(), 1); + assert_eq!(accelerator.forward_ict_dispatches(), 1); + assert!(accelerator.quantize_subband_attempts() > 0); + assert!(accelerator.quantize_subband_dispatches() > 0); + assert!(accelerator.ht_code_block_dispatches() > 0); + assert_eq!(accelerator.packetization_dispatches(), 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_htj2k_lossy_facade_reports_forward_dwt97_dispatch() { + let width = 64; + let height = 64; + let pixels: Vec = (0..width * height) + .map(|idx| ((idx * 23 + idx / 11 + 31) & 0xFF) as u8) + .collect(); + let samples = + J2kLossySamples::new(&pixels, width, height, 1, 8, false).expect("valid gray samples"); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let encoded = encode_j2k_lossy_with_accelerator( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(2)) + .with_validation(J2kEncodeValidation::CpuRoundTrip), + BackendKind::Metal, + &mut accelerator, + ) + .expect("Metal-accelerated HTJ2K lossy encode with DWT 9/7"); + + assert_eq!(encoded.backend, BackendKind::Metal); + assert!(accelerator.forward_dwt97_attempts() > 0); + assert!(accelerator.forward_dwt97_dispatches() > 0); + assert!(encoded.dispatch_report.forward_dwt97 > 0); + assert!(encoded.dispatch_report.quantize_subband > 0); + assert!(accelerator.quantize_subband_dispatches() > 0); + assert!(accelerator.ht_code_block_dispatches() > 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_classic_tier1_kernel_matches_scalar_oracle() { + let coeffs: Vec = (0..64) + .map(|idx| { + let value = ((idx * 37 + 11) & 0x1ff) - 255; + if idx % 5 == 0 { + 0 + } else { + value + } + }) + .collect(); + let style = J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: false, + reset_context_probabilities: false, + termination_on_each_pass: false, + vertically_causal_context: false, + segmentation_symbols: false, + }; + let job = j2k_native::J2kTier1CodeBlockEncodeJob { + coefficients: &coeffs, + width: 8, + height: 8, + sub_band_type: j2k_native::J2kSubBandType::HighHigh, + total_bitplanes: 9, + style, + }; + + let gpu = compute::encode_classic_tier1_code_block(job).expect("Metal classic encode"); + let cpu = j2k_native::encode_j2k_code_block_scalar_with_style( + &coeffs, + 8, + 8, + j2k_native::J2kSubBandType::HighHigh, + 9, + style, + ) + .expect("scalar classic encode"); + + assert_eq!(gpu.data, cpu.data); + assert_eq!(gpu.segments.len(), cpu.segments.len()); + for (gpu_segment, cpu_segment) in gpu.segments.iter().zip(cpu.segments.iter()) { + assert_eq!(gpu_segment.data_offset, cpu_segment.data_offset); + assert_eq!(gpu_segment.data_length, cpu_segment.data_length); + assert_eq!(gpu_segment.start_coding_pass, cpu_segment.start_coding_pass); + assert_eq!(gpu_segment.end_coding_pass, cpu_segment.end_coding_pass); + assert_eq!(gpu_segment.use_arithmetic, cpu_segment.use_arithmetic); + } + assert_eq!(gpu.number_of_coding_passes, cpu.number_of_coding_passes); + assert_eq!(gpu.missing_bit_planes, cpu.missing_bit_planes); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_classic_tier1_kernel_matches_scalar_for_terminated_passes() { + let coeffs: Vec = (0..64) + .map(|idx| { + let value = ((idx * 43 + 5) & 0x3ff) - 511; + if idx % 6 == 0 { + 0 + } else { + value + } + }) + .collect(); + let style = J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: false, + reset_context_probabilities: true, + termination_on_each_pass: true, + vertically_causal_context: false, + segmentation_symbols: true, + }; + let job = j2k_native::J2kTier1CodeBlockEncodeJob { + coefficients: &coeffs, + width: 8, + height: 8, + sub_band_type: j2k_native::J2kSubBandType::LowHigh, + total_bitplanes: 10, + style, + }; + + let gpu = + compute::encode_classic_tier1_code_block(job).expect("Metal classic terminated encode"); + let cpu = j2k_native::encode_j2k_code_block_scalar_with_style( + &coeffs, + 8, + 8, + j2k_native::J2kSubBandType::LowHigh, + 10, + style, + ) + .expect("scalar classic terminated encode"); + + assert_eq!(gpu.data, cpu.data); + assert_eq!(gpu.segments.len(), cpu.segments.len()); + for (gpu_segment, cpu_segment) in gpu.segments.iter().zip(cpu.segments.iter()) { + assert_eq!(gpu_segment.data_offset, cpu_segment.data_offset); + assert_eq!(gpu_segment.data_length, cpu_segment.data_length); + assert_eq!(gpu_segment.start_coding_pass, cpu_segment.start_coding_pass); + assert_eq!(gpu_segment.end_coding_pass, cpu_segment.end_coding_pass); + assert_eq!(gpu_segment.use_arithmetic, cpu_segment.use_arithmetic); + } + assert_eq!(gpu.number_of_coding_passes, cpu.number_of_coding_passes); + assert_eq!(gpu.missing_bit_planes, cpu.missing_bit_planes); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_classic_tier1_kernel_matches_scalar_for_selective_bypass() { + let coeffs: Vec = (0..64) + .map(|idx| { + let value = ((idx * 61 + 29) & 0x7ff) - 1023; + if idx % 4 == 0 { + 0 + } else { + value + } + }) + .collect(); + let style = J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: true, + reset_context_probabilities: false, + termination_on_each_pass: false, + vertically_causal_context: false, + segmentation_symbols: false, + }; + let job = j2k_native::J2kTier1CodeBlockEncodeJob { + coefficients: &coeffs, + width: 8, + height: 8, + sub_band_type: j2k_native::J2kSubBandType::HighLow, + total_bitplanes: 11, + style, + }; + + let gpu = compute::encode_classic_tier1_code_block(job).expect("Metal classic bypass encode"); + let cpu = j2k_native::encode_j2k_code_block_scalar_with_style( + &coeffs, + 8, + 8, + j2k_native::J2kSubBandType::HighLow, + 11, + style, + ) + .expect("scalar classic bypass encode"); + + assert_eq!(gpu.data, cpu.data); + assert_eq!(gpu.segments.len(), cpu.segments.len()); + for (gpu_segment, cpu_segment) in gpu.segments.iter().zip(cpu.segments.iter()) { + assert_eq!(gpu_segment.data_offset, cpu_segment.data_offset); + assert_eq!(gpu_segment.data_length, cpu_segment.data_length); + assert_eq!(gpu_segment.start_coding_pass, cpu_segment.start_coding_pass); + assert_eq!(gpu_segment.end_coding_pass, cpu_segment.end_coding_pass); + assert_eq!(gpu_segment.use_arithmetic, cpu_segment.use_arithmetic); + } + assert_eq!(gpu.number_of_coding_passes, cpu.number_of_coding_passes); + assert_eq!(gpu.missing_bit_planes, cpu.missing_bit_planes); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_classic_tier1_batched_bypass_u16_32_matches_scalar() { + let coeffs: Vec = (0..32 * 32) + .map(|idx| { + let value = ((idx * 97 + idx / 3 + 19) & 0x7ff) - 1023; + if idx % 11 == 0 || idx % 17 == 0 { + 0 + } else { + value + } + }) + .collect(); + let style = J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: true, + reset_context_probabilities: false, + termination_on_each_pass: false, + vertically_causal_context: false, + segmentation_symbols: false, + }; + let job = j2k_native::J2kTier1CodeBlockEncodeJob { + coefficients: &coeffs, + width: 32, + height: 32, + sub_band_type: j2k_native::J2kSubBandType::HighHigh, + total_bitplanes: 11, + style, + }; + + let gpu = compute::encode_classic_tier1_code_blocks(&[job]) + .expect("batched Metal classic bypass_u16_32 encode") + .pop() + .expect("one encoded codeblock"); + let cpu = j2k_native::encode_j2k_code_block_scalar_with_style( + &coeffs, + 32, + 32, + j2k_native::J2kSubBandType::HighHigh, + 11, + style, + ) + .expect("scalar classic bypass encode"); + + assert_eq!(gpu.data, cpu.data); + assert_eq!(gpu.segments.len(), cpu.segments.len()); + for (gpu_segment, cpu_segment) in gpu.segments.iter().zip(cpu.segments.iter()) { + assert_eq!(gpu_segment.data_offset, cpu_segment.data_offset); + assert_eq!(gpu_segment.data_length, cpu_segment.data_length); + assert_eq!(gpu_segment.start_coding_pass, cpu_segment.start_coding_pass); + assert_eq!(gpu_segment.end_coding_pass, cpu_segment.end_coding_pass); + assert_eq!(gpu_segment.use_arithmetic, cpu_segment.use_arithmetic); + } + assert_eq!(gpu.number_of_coding_passes, cpu.number_of_coding_passes); + assert_eq!(gpu.missing_bit_planes, cpu.missing_bit_planes); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_classic_tier1_token_routes_match_scalar_bytes() { + let first_coeffs: Vec = (0..32 * 32) + .map(|idx| { + let value = ((idx * 37 + idx / 5 + 31) & 0xff) - 127; + if idx % 5 == 0 || idx % 11 == 0 { + 0 + } else { + value + } + }) + .collect(); + let second_coeffs: Vec = (0..17 * 29) + .map(|idx| { + let value = ((idx * 73 + idx / 7 + 11) & 0xff) - 127; + if idx % 7 == 0 || idx % 23 == 0 { + 0 + } else { + value + } + }) + .collect(); + let style = J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: true, + reset_context_probabilities: false, + termination_on_each_pass: false, + vertically_causal_context: false, + segmentation_symbols: false, + }; + let jobs = [ + j2k_native::J2kTier1CodeBlockEncodeJob { + coefficients: &first_coeffs, + width: 32, + height: 32, + sub_band_type: j2k_native::J2kSubBandType::HighHigh, + total_bitplanes: 8, + style, + }, + j2k_native::J2kTier1CodeBlockEncodeJob { + coefficients: &second_coeffs, + width: 17, + height: 29, + sub_band_type: j2k_native::J2kSubBandType::LowLow, + total_bitplanes: 8, + style, + }, + ]; + + let gpu_packed = compute::encode_classic_tier1_code_blocks_via_gpu_token_pack_for_test(&jobs) + .expect("Metal classic GPU token-pack encode"); + let cpu_packed = + compute::encode_classic_tier1_code_blocks_via_ordered_tokens_cpu_pack_for_test(&jobs) + .expect("Metal classic ordered-token CPU-pack encode"); + let split_packed = + compute::encode_classic_tier1_code_blocks_via_split_mq_raw_tokens_cpu_pack_for_test(&jobs) + .expect("Metal classic split MQ/raw token CPU-pack encode"); + let split_gpu_packed = + compute::encode_classic_tier1_code_blocks_via_split_mq_raw_tokens_gpu_pack_for_test(&jobs) + .expect("Metal classic split MQ/raw token GPU-pack encode"); + let mq_byte_split_gpu_packed = + compute::encode_classic_tier1_code_blocks_via_split_mq_byte_raw_tokens_gpu_pack_for_test( + &jobs, + ) + .expect("Metal classic split MQ-byte/raw-bit token GPU-pack encode"); + + assert_eq!(gpu_packed.len(), jobs.len()); + assert_eq!(cpu_packed.len(), jobs.len()); + assert_eq!(split_packed.len(), jobs.len()); + assert_eq!(split_gpu_packed.len(), jobs.len()); + assert_eq!(mq_byte_split_gpu_packed.len(), jobs.len()); + for ( + ( + (((gpu_block, cpu_packed_block), split_packed_block), split_gpu_packed_block), + mq_byte_split_gpu_packed_block, + ), + job, + coeffs, + ) in gpu_packed + .iter() + .zip(cpu_packed.iter()) + .zip(split_packed.iter()) + .zip(split_gpu_packed.iter()) + .zip(mq_byte_split_gpu_packed.iter()) + .zip(jobs.iter()) + .zip([&first_coeffs, &second_coeffs]) + .map(|((blocks, job), coeffs)| (blocks, job, coeffs)) + { + let cpu = j2k_native::encode_j2k_code_block_scalar_with_style( + coeffs, + job.width, + job.height, + job.sub_band_type, + job.total_bitplanes, + style, + ) + .expect("scalar classic bypass encode"); + + assert_eq!(gpu_block.data, cpu.data); + assert_eq!(gpu_block.segments, cpu.segments); + assert_eq!( + gpu_block.number_of_coding_passes, + cpu.number_of_coding_passes + ); + assert_eq!(gpu_block.missing_bit_planes, cpu.missing_bit_planes); + assert_eq!(cpu_packed_block.data, cpu.data); + assert_eq!(cpu_packed_block.segments, cpu.segments); + assert_eq!( + cpu_packed_block.number_of_coding_passes, + cpu.number_of_coding_passes + ); + assert_eq!(cpu_packed_block.missing_bit_planes, cpu.missing_bit_planes); + assert_eq!(split_packed_block.data, cpu.data); + assert_eq!(split_packed_block.segments, cpu.segments); + assert_eq!( + split_packed_block.number_of_coding_passes, + cpu.number_of_coding_passes + ); + assert_eq!( + split_packed_block.missing_bit_planes, + cpu.missing_bit_planes + ); + assert_eq!(split_gpu_packed_block.data, cpu.data); + assert_eq!(split_gpu_packed_block.segments, cpu.segments); + assert_eq!( + split_gpu_packed_block.number_of_coding_passes, + cpu.number_of_coding_passes + ); + assert_eq!( + split_gpu_packed_block.missing_bit_planes, + cpu.missing_bit_planes + ); + assert_eq!(mq_byte_split_gpu_packed_block.data, cpu.data); + assert_eq!(mq_byte_split_gpu_packed_block.segments, cpu.segments); + assert_eq!( + mq_byte_split_gpu_packed_block.number_of_coding_passes, + cpu.number_of_coding_passes + ); + assert_eq!( + mq_byte_split_gpu_packed_block.missing_bit_planes, + cpu.missing_bit_planes + ); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_htj2k_cleanup_kernel_matches_scalar_oracle() { + let coeffs: Vec = (0..64) + .map(|idx| { + let value = ((idx * 19 + 7) & 0xff) - 127; + if idx % 7 == 0 { + 0 + } else { + value + } + }) + .collect(); + let job = j2k_native::J2kHtCodeBlockEncodeJob { + coefficients: &coeffs, + width: 8, + height: 8, + total_bitplanes: 8, + target_coding_passes: 1, + }; + + let gpu = compute::encode_ht_cleanup_code_block(job).expect("Metal HT encode"); + let cpu = j2k_native::encode_ht_code_block_scalar(&coeffs, 8, 8, 8).expect("scalar HT encode"); + + assert_eq!(gpu.data, cpu.data); + assert_eq!(gpu.num_coding_passes, cpu.num_coding_passes); + assert_eq!(gpu.num_zero_bitplanes, cpu.num_zero_bitplanes); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_tier2_packetization_kernel_matches_scalar_oracle() { + let block0 = [0x12, 0x34, 0x56, 0x78]; + let block1 = [0x9a, 0xbc]; + let code_blocks = vec![ + j2k_native::J2kPacketizationCodeBlock { + data: &block0, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: j2k_native::J2kPacketizationBlockCodingMode::Classic, + }, + j2k_native::J2kPacketizationCodeBlock { + data: &block1, + ht_cleanup_length: u32::try_from(block1.len()).expect("test payload fits u32"), + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 1, + previously_included: false, + l_block: 3, + block_coding_mode: j2k_native::J2kPacketizationBlockCodingMode::HighThroughput, + }, + ]; + let subband = j2k_native::J2kPacketizationSubband { + code_blocks, + num_cbs_x: 2, + num_cbs_y: 1, + }; + let resolution = j2k_native::J2kPacketizationResolution { + subbands: vec![subband], + }; + let resolutions = [resolution]; + let packet_descriptors = [j2k_native::J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }]; + let job = j2k_native::J2kPacketizationEncodeJob { + resolution_count: 1, + num_layers: 1, + num_components: 1, + code_block_count: 2, + progression_order: j2k_native::J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &packet_descriptors, + resolutions: &resolutions, + }; + + let gpu = compute::encode_tier2_packetization(job).expect("Metal packet encode"); + let cpu = j2k_native::encode_j2k_packetization_scalar(job).expect("scalar packet encode"); + + assert_eq!(gpu, cpu); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_tier2_packetization_reuses_descriptor_state_across_layers() { + let block0 = vec![0x11]; + let block1 = vec![0x22]; + let first = j2k_native::J2kPacketizationResolution { + subbands: vec![j2k_native::J2kPacketizationSubband { + code_blocks: vec![j2k_native::J2kPacketizationCodeBlock { + data: &block0, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 0, + previously_included: false, + l_block: 3, + block_coding_mode: j2k_native::J2kPacketizationBlockCodingMode::Classic, + }], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }; + let second = j2k_native::J2kPacketizationResolution { + subbands: vec![j2k_native::J2kPacketizationSubband { + code_blocks: vec![j2k_native::J2kPacketizationCodeBlock { + data: &block1, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 0, + previously_included: false, + l_block: 3, + block_coding_mode: j2k_native::J2kPacketizationBlockCodingMode::Classic, + }], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }; + let resolutions = [first, second]; + let packet_descriptors = [ + j2k_native::J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }, + j2k_native::J2kPacketizationPacketDescriptor { + packet_index: 1, + state_index: 0, + layer: 1, + resolution: 0, + component: 0, + precinct: 0, + }, + ]; + let job = j2k_native::J2kPacketizationEncodeJob { + resolution_count: 2, + num_layers: 2, + num_components: 1, + code_block_count: 2, + progression_order: j2k_native::J2kPacketizationProgressionOrder::Rpcl, + packet_descriptors: &packet_descriptors, + resolutions: &resolutions, + }; + + let gpu = compute::encode_tier2_packetization(job).expect("Metal packet encode"); + let cpu = j2k_native::encode_j2k_packetization_scalar(job).expect("scalar packet encode"); + + assert_eq!(gpu, cpu); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_tier2_packetization_honors_explicit_descriptor_order() { + let block0 = vec![0xA0]; + let block1 = vec![0xB0]; + let first = j2k_native::J2kPacketizationResolution { + subbands: vec![j2k_native::J2kPacketizationSubband { + code_blocks: vec![j2k_native::J2kPacketizationCodeBlock { + data: &block0, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 0, + previously_included: false, + l_block: 3, + block_coding_mode: j2k_native::J2kPacketizationBlockCodingMode::Classic, + }], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }; + let second = j2k_native::J2kPacketizationResolution { + subbands: vec![j2k_native::J2kPacketizationSubband { + code_blocks: vec![j2k_native::J2kPacketizationCodeBlock { + data: &block1, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 0, + previously_included: false, + l_block: 3, + block_coding_mode: j2k_native::J2kPacketizationBlockCodingMode::Classic, + }], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }; + let resolutions = [first, second]; + let packet_descriptors = [ + j2k_native::J2kPacketizationPacketDescriptor { + packet_index: 1, + state_index: 1, + layer: 0, + resolution: 1, + component: 0, + precinct: 0, + }, + j2k_native::J2kPacketizationPacketDescriptor { + packet_index: 0, + state_index: 0, + layer: 0, + resolution: 0, + component: 0, + precinct: 0, + }, + ]; + let job = j2k_native::J2kPacketizationEncodeJob { + resolution_count: 2, + num_layers: 1, + num_components: 1, + code_block_count: 2, + progression_order: j2k_native::J2kPacketizationProgressionOrder::Rpcl, + packet_descriptors: &packet_descriptors, + resolutions: &resolutions, + }; + + let gpu = compute::encode_tier2_packetization(job).expect("Metal packet encode"); + let cpu = j2k_native::encode_j2k_packetization_scalar(job).expect("scalar packet encode"); + + assert_eq!(gpu, cpu); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_forward_dwt53_handles_single_sample_edge_dimensions() { + for (width, height) in [(1, 8), (8, 1)] { + let samples: Vec = (0..width * height) + .map(|i| { + f32::from( + u8::try_from((i * 11 + width * 3 + height * 5) & 0xFF) + .expect("masked sample fits in u8"), + ) - 128.0 + }) + .collect(); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let output = accelerator + .encode_forward_dwt53(J2kForwardDwt53Job { + samples: &samples, + width, + height, + num_levels: 1, + }) + .expect("metal DWT 5/3 stage") + .expect("metal DWT 5/3 dispatch"); + + assert_eq!(output.ll_width, width.div_ceil(2)); + assert_eq!(output.ll_height, height.div_ceil(2)); + assert_eq!(output.levels.len(), 1); + assert_eq!(accelerator.forward_dwt53_attempts(), 1); + assert_eq!(accelerator.forward_dwt53_dispatches(), 1); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_forward_dwt53_matches_reference_for_fractional_stage_samples() { + fn assert_slice_near(actual: &[f32], expected: &[f32], label: &str) { + assert_eq!(actual.len(), expected.len(), "{label} length mismatch"); + for (index, (&actual, &expected)) in actual.iter().zip(expected).enumerate() { + assert!( + (actual - expected).abs() <= 0.0001, + "{label}[{index}] mismatch: actual={actual}, expected={expected}" + ); + } + } + + let width = 8; + let height = 8; + let samples = (0..width * height) + .map(|idx| f32::from(u16::try_from(idx).expect("test index fits u16")) * 0.5 - 15.25) + .collect::>(); + let expected = forward_dwt53_reference(&samples, width, height, 1); + let mut accelerator = MetalEncodeStageAccelerator::default(); + + let actual = accelerator + .encode_forward_dwt53(J2kForwardDwt53Job { + samples: &samples, + width, + height, + num_levels: 1, + }) + .expect("metal DWT 5/3 stage") + .expect("metal DWT 5/3 dispatch"); + + assert_eq!(actual.ll_width, expected.ll_width); + assert_eq!(actual.ll_height, expected.ll_height); + assert_slice_near(&actual.ll, &expected.ll, "LL"); + assert_eq!(actual.levels.len(), expected.levels.len()); + for (index, (actual, expected)) in actual.levels.iter().zip(&expected.levels).enumerate() { + assert_eq!(actual.width, expected.width, "level {index} width"); + assert_eq!(actual.height, expected.height, "level {index} height"); + assert_eq!( + actual.low_width, expected.low_width, + "level {index} low width" + ); + assert_eq!( + actual.low_height, expected.low_height, + "level {index} low height" + ); + assert_eq!( + actual.high_width, expected.high_width, + "level {index} high width" + ); + assert_eq!( + actual.high_height, expected.high_height, + "level {index} high height" + ); + assert_slice_near(&actual.hl, &expected.hl, "HL"); + assert_slice_near(&actual.lh, &expected.lh, "LH"); + assert_slice_near(&actual.hh, &expected.hh, "HH"); + } +} + +#[cfg(target_os = "macos")] +fn native_lossy_dwt97_options(num_decomposition_levels: u8) -> EncodeOptions { + EncodeOptions { + num_decomposition_levels, + reversible: false, + guard_bits: 2, + use_ht_block_coding: true, + ..Default::default() + } +} + +#[cfg(target_os = "macos")] +fn assert_metal_dwt97_matches_native_encode( + pixels: &[u8], + width: u32, + height: u32, + num_decomposition_levels: u8, +) { + let options = native_lossy_dwt97_options(num_decomposition_levels); + let expected = j2k_native::encode(pixels, width, height, 1, 8, false, &options) + .expect("native lossy DWT 9/7 encode"); + let mut accelerator = MetalEncodeStageAccelerator::for_forward_dwt97_encode(); + let actual = { + let mut adapter = NativeEncodeStageAdapter::new(&mut accelerator); + j2k_native::encode_with_accelerator( + pixels, + width, + height, + 1, + 8, + false, + &options, + &mut adapter, + ) + .expect("Metal DWT 9/7 lossy encode") + }; + + assert_eq!(actual, expected); + assert_eq!(accelerator.forward_dwt97_attempts(), 1); + assert_eq!(accelerator.forward_dwt97_dispatches(), 1); + let report = accelerator.dispatch_report(); + assert_eq!(report.forward_dwt97, 1); + assert_eq!(report.forward_dwt53, 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_forward_dwt97_single_level_matches_native_encode_output() { + let width = 17; + let height = 15; + let pixels = (0..width * height) + .map(|idx| ((idx * 29 + idx / 5 + 17) & 0xFF) as u8) + .collect::>(); + + assert_metal_dwt97_matches_native_encode(&pixels, width, height, 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn metal_forward_dwt97_multi_level_matches_native_encode_output() { + let width = 32; + let height = 16; + let pixels = (0..width * height) + .map(|idx| ((idx * 43 + idx / 7 + 91) & 0xFF) as u8) + .collect::>(); + + assert_metal_dwt97_matches_native_encode(&pixels, width, height, 3); +} diff --git a/crates/j2k-metal/src/encode/types.rs b/crates/j2k-metal/src/encode/types.rs new file mode 100644 index 00000000..4989c98e --- /dev/null +++ b/crates/j2k-metal/src/encode/types.rs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::EncodedJ2k; +#[cfg(target_os = "macos")] +use j2k_core::PixelFormat; +#[cfg(target_os = "macos")] +use metal::Buffer; +use std::time::Duration; + +use super::MetalEncodedJ2k; + +#[cfg(target_os = "macos")] +#[derive(Debug, Clone, Copy)] +/// Metal buffer and layout metadata for one lossless J2K encode tile. +pub struct MetalLosslessEncodeTile<'a> { + /// Source Metal buffer containing Gray or RGB pixels. + pub buffer: &'a Buffer, + /// Byte offset of the first source pixel in `buffer`. + pub byte_offset: usize, + /// Width of the valid input region in pixels. + pub width: u32, + /// Height of the valid input region in pixels. + pub height: u32, + /// Number of bytes between consecutive input rows. + pub pitch_bytes: usize, + /// Encoded image width in pixels. + pub output_width: u32, + /// Encoded image height in pixels. + pub output_height: u32, + /// Pixel format of the source buffer. + pub format: PixelFormat, +} + +#[cfg(not(target_os = "macos"))] +#[derive(Debug, Clone, Copy)] +/// Placeholder lossless encode tile type for non-macOS builds. +pub struct MetalLosslessEncodeTile<'a> { + _private: core::marker::PhantomData<&'a ()>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Residency decisions used by a lossless Metal encode. +pub struct MetalLosslessEncodeResidency { + /// Whether coefficient preparation ran on Metal. + pub coefficient_prep_used: bool, + /// Whether packetization ran on Metal. + pub packetization_used: bool, + /// Whether codestream assembly stayed resident on Metal. + pub codestream_assembly_used: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Lossless Metal encode output with host codestream bytes and timings. +/// +/// API note: this diagnostic report is constructed by this crate. It is not +/// `#[non_exhaustive]`, but adapter releases may add diagnostic fields as the +/// resident encode path gains more profiling detail. +pub struct MetalLosslessEncodeOutcome { + /// Encoded J2K codestream. + pub encoded: EncodedJ2k, + /// Whether the input buffer had to be copied or padded. + pub input_copy_used: bool, + /// Residency decisions for the encode stages. + pub resident: MetalLosslessEncodeResidency, + /// Time spent copying or padding the input. + pub input_copy_duration: Duration, + /// End-to-end encode duration for this tile. + pub encode_duration: Duration, + /// GPU-only duration when timestamp data is available. + pub gpu_duration: Option, + /// Time spent validating the encoded output. + pub validation_duration: Duration, + /// Time spent materializing buffer-backed codestream bytes into host bytes. + pub host_readback_duration: Duration, +} + +/// Metal lossless encode report for buffer-backed codestream output. +pub struct MetalLosslessBufferEncodeOutcome { + /// Encoded codestream stored in a Metal buffer. + pub encoded: MetalEncodedJ2k, + /// Whether the input buffer had to be copied or padded. + pub input_copy_used: bool, + /// Residency decisions for the encode stages. + pub resident: MetalLosslessEncodeResidency, + /// Time spent copying or padding the input. + pub input_copy_duration: Duration, + /// End-to-end encode duration for this tile. + pub encode_duration: Duration, + /// GPU-only duration when timestamp data is available. + pub gpu_duration: Option, + /// Time spent validating the encoded output. + pub validation_duration: Duration, +} + +/// Tuning knobs for resident Metal lossless J2K/HTJ2K tile batch encode. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct MetalLosslessEncodeConfig { + /// Requested maximum number of tiles submitted concurrently. + /// + /// `None` uses the crate default and still clamps by the memory budget. + pub gpu_encode_inflight_tiles: Option, + /// Resident encode memory budget in bytes. + /// + /// `None` uses `min(10 GiB, hw_memsize * 0.40)` when host memory can be + /// discovered. + pub gpu_encode_memory_budget_bytes: Option, +} + +/// Batched lossless encode request over Metal-resident tiles. +/// +/// Collapses the former per-permutation entry points: pick the input +/// staging mode and batch tuning here, then submit through +/// [`crate::submit_lossless_batch`], [`crate::submit_lossless_batch_to_metal`], +/// or [`crate::encode_lossless_batch_with_report`]. +#[derive(Clone, Copy)] +pub struct MetalLosslessEncodeBatchRequest<'a, 'b> { + /// Metal-resident tiles to encode. + pub tiles: &'a [MetalLosslessEncodeTile<'b>], + /// How tile samples reach the encoder's padded staging layout. + pub staging: MetalEncodeInputStaging, + /// Batch tuning knobs (inflight tiles, memory budget). + pub config: MetalLosslessEncodeConfig, +} + +/// How tile samples reach the encoder's padded staging layout. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MetalEncodeInputStaging { + /// Copy the tile into freshly padded staging storage. + CopyAndPad, + /// The tile is already padded and contiguous; encode it in place. + AlreadyPaddedContiguous, +} diff --git a/crates/j2k-metal/src/encode/validation.rs b/crates/j2k-metal/src/encode/validation.rs new file mode 100644 index 00000000..a1c50beb --- /dev/null +++ b/crates/j2k-metal/src/encode/validation.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::J2kLosslessSamples; +use j2k_core::PixelFormat; + +use super::MetalLosslessEncodeTile; + +pub(super) fn validation_pixel_format( + samples: J2kLosslessSamples<'_>, +) -> Result { + match (samples.components, samples.bit_depth) { + (1, 1..=8) => Ok(PixelFormat::Gray8), + (3, 1..=8) => Ok(PixelFormat::Rgb8), + (1, 9..=16) => Ok(PixelFormat::Gray16), + (3, 9..=16) => Ok(PixelFormat::Rgb16), + _ => Err(crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal validation supports only grayscale or RGB samples up to 16 bits", + }), + } +} + +pub(super) fn lossless_sample_shape(format: PixelFormat) -> Result<(u8, u8), crate::Error> { + match format { + PixelFormat::Gray8 => Ok((1, 8)), + PixelFormat::Rgb8 => Ok((3, 8)), + PixelFormat::Gray16 => Ok((1, 16)), + PixelFormat::Rgb16 => Ok((3, 16)), + PixelFormat::Rgba8 | PixelFormat::Rgba16 => Err(crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal encode from RGBA tiles requires explicit alpha handling", + }), + _ => Err(crate::Error::UnsupportedMetalRequest { + reason: "J2K Metal encode received an unknown pixel format", + }), + } +} + +pub(super) fn validate_metal_encode_tile( + tile: MetalLosslessEncodeTile<'_>, +) -> Result<(), crate::Error> { + if tile.width == 0 || tile.height == 0 || tile.output_width == 0 || tile.output_height == 0 { + return Err(crate::Error::MetalKernel { + message: "J2K Metal encode tile dimensions must be nonzero".to_string(), + }); + } + if tile.width > tile.output_width || tile.height > tile.output_height { + return Err(crate::Error::MetalKernel { + message: "J2K Metal encode input tile exceeds output tile dimensions".to_string(), + }); + } + let row_bytes = tile + .width + .checked_mul(tile.format.bytes_per_pixel() as u32) + .ok_or_else(|| crate::Error::MetalKernel { + message: "J2K Metal encode row byte count overflow".to_string(), + })? as usize; + if tile.pitch_bytes < row_bytes { + return Err(crate::Error::MetalKernel { + message: "J2K Metal encode tile pitch is shorter than one row".to_string(), + }); + } + let required_end = tile + .byte_offset + .checked_add( + tile.pitch_bytes + .checked_mul(tile.height.saturating_sub(1) as usize) + .and_then(|prefix| prefix.checked_add(row_bytes)) + .ok_or_else(|| crate::Error::MetalKernel { + message: "J2K Metal encode input byte range overflow".to_string(), + })?, + ) + .ok_or_else(|| crate::Error::MetalKernel { + message: "J2K Metal encode input byte range overflow".to_string(), + })?; + let buffer_len = + usize::try_from(tile.buffer.length()).map_err(|_| crate::Error::MetalKernel { + message: "J2K Metal encode buffer length exceeds usize".to_string(), + })?; + if required_end > buffer_len { + return Err(crate::Error::MetalKernel { + message: format!( + "J2K Metal encode input byte range exceeds buffer length: need {required_end}, buffer has {buffer_len}" + ), + }); + } + Ok(()) +} + +pub(super) fn validate_padded_contiguous_metal_encode_tile( + tile: MetalLosslessEncodeTile<'_>, + bytes_per_pixel: usize, +) -> Result<(), crate::Error> { + if tile.width != tile.output_width || tile.height != tile.output_height { + return Err(crate::Error::MetalKernel { + message: + "J2K Metal no-copy encode requires input dimensions to match output dimensions" + .to_string(), + }); + } + let expected_pitch = (tile.output_width as usize) + .checked_mul(bytes_per_pixel) + .ok_or_else(|| crate::Error::MetalKernel { + message: "J2K Metal no-copy encode pitch overflow".to_string(), + })?; + if tile.pitch_bytes != expected_pitch { + return Err(crate::Error::MetalKernel { + message: format!( + "J2K Metal no-copy encode requires contiguous rows: expected pitch {expected_pitch}, got {}", + tile.pitch_bytes + ), + }); + } + Ok(()) +} diff --git a/crates/j2k-metal/src/encode_bitstream.metal b/crates/j2k-metal/src/encode_bitstream.metal new file mode 100644 index 00000000..e5c365d8 --- /dev/null +++ b/crates/j2k-metal/src/encode_bitstream.metal @@ -0,0 +1,8326 @@ +#include +using namespace metal; + +constant uint J2K_ENCODE_STATUS_OK = 0u; +constant uint J2K_ENCODE_STATUS_FAIL = 1u; +constant uint J2K_ENCODE_STATUS_UNSUPPORTED = 2u; +constant uint J2K_PACKET_PAYLOAD_COPY_SMALL_JOB_BYTES = 64u; +constant uint J2K_PACKET_PAYLOAD_COPY_MEDIUM_JOB_BYTES = 512u; + +constant uchar J2K_ENCODE_SIGNIFICANT = uchar(1u << 7u); +constant uchar J2K_ENCODE_MAGNITUDE_REFINED = uchar(1u << 6u); +constant uchar J2K_ENCODE_SIGN = uchar(1u << 5u); +constant uchar J2K_ENCODE_CODED_MARKER_MASK = uchar(0x1Fu); +constant uint J2K_CLASSIC_ENCODE_32_MAX_WIDTH = 32u; +constant uint J2K_CLASSIC_ENCODE_32_MAX_HEIGHT = 32u; +constant uint J2K_CLASSIC_ENCODE_32_PADDED_WIDTH = J2K_CLASSIC_ENCODE_32_MAX_WIDTH + 2u; +constant uint J2K_CLASSIC_ENCODE_32_PADDED_HEIGHT = J2K_CLASSIC_ENCODE_32_MAX_HEIGHT + 2u; +constant uint J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT = + J2K_CLASSIC_ENCODE_32_PADDED_WIDTH * J2K_CLASSIC_ENCODE_32_PADDED_HEIGHT; +constant uint J2K_CLASSIC_TIER1_PASS_PLAN_CAPACITY = 48u; + +struct J2kClassicEncodeParams { + uint width; + uint height; + uint sub_band_type; + uint total_bitplanes; + uint style_flags; + uint output_capacity; + uint segment_capacity; +}; + +struct J2kClassicEncodeStatus { + uint code; + uint detail; + uint data_len; + uint number_of_coding_passes; + uint missing_bit_planes; + uint segment_count; + uint reserved0; + uint reserved1; +}; + +struct J2kMqEncoder { + device uchar *data; + uint max_len; + uint len; + uint a; + uint c; + uint ct; + uint failed; +}; + +struct J2kRawBitWriter { + device uchar *data; + uint max_len; + uint len; + uint buffer; + uint bits_in_buffer; + uint last_byte_was_ff; + uint failed; +}; + +inline void j2k_set_encode_status( + device J2kClassicEncodeStatus *status, + uint code, + uint detail, + uint data_len, + uint passes, + uint missing, + uint segments +) { + status->code = code; + status->detail = detail; + status->data_len = data_len; + status->number_of_coding_passes = passes; + status->missing_bit_planes = missing; + status->segment_count = segments; + status->reserved0 = 0u; + status->reserved1 = 0u; +} + +inline void j2k_set_encode_status_with_payload_skip( + device J2kClassicEncodeStatus *status, + uint code, + uint detail, + uint data_len, + uint passes, + uint missing, + uint segments, + uint payload_skip +) { + j2k_set_encode_status(status, code, detail, data_len, passes, missing, segments); + status->reserved0 = payload_skip; +} + +inline void j2k_mq_init(thread J2kMqEncoder &encoder, device uchar *out, uint capacity) { + encoder.data = out; + encoder.max_len = capacity; + encoder.len = 0u; + encoder.a = 0x8000u; + encoder.c = 0u; + encoder.ct = 12u; + encoder.failed = 0u; + if (capacity == 0u) { + encoder.failed = 1u; + return; + } + encoder.data[0] = uchar(0); + encoder.len = 1u; +} + +inline void j2k_mq_push(thread J2kMqEncoder &encoder, uchar value) { + if (encoder.len >= encoder.max_len) { + encoder.failed = 1u; + return; + } + encoder.data[encoder.len] = value; + encoder.len += 1u; +} + +inline void j2k_mq_byte_out(thread J2kMqEncoder &encoder) { + if (encoder.failed != 0u || encoder.len == 0u) { + encoder.failed = 1u; + return; + } + + uchar last_byte = encoder.data[encoder.len - 1u]; + if (last_byte == uchar(0xFFu)) { + const uchar b = uchar(encoder.c >> 20u); + j2k_mq_push(encoder, b); + encoder.c &= 0xFFFFFu; + encoder.ct = 7u; + } else if ((encoder.c & 0x8000000u) == 0u) { + const uchar b = uchar(encoder.c >> 19u); + j2k_mq_push(encoder, b); + encoder.c &= 0x7FFFFu; + encoder.ct = 8u; + } else { + encoder.data[encoder.len - 1u] = uchar(encoder.data[encoder.len - 1u] + uchar(1u)); + encoder.c &= 0x7FFFFFFu; + if (encoder.data[encoder.len - 1u] == uchar(0xFFu)) { + const uchar b = uchar(encoder.c >> 20u); + j2k_mq_push(encoder, b); + encoder.c &= 0xFFFFFu; + encoder.ct = 7u; + } else { + const uchar b = uchar(encoder.c >> 19u); + j2k_mq_push(encoder, b); + encoder.c &= 0x7FFFFu; + encoder.ct = 8u; + } + } +} + +inline void j2k_mq_renormalize(thread J2kMqEncoder &encoder) { + do { + encoder.a <<= 1u; + encoder.c <<= 1u; + encoder.ct -= 1u; + if (encoder.ct == 0u) { + j2k_mq_byte_out(encoder); + } + } while ((encoder.a & 0x8000u) == 0u && encoder.failed == 0u); +} + +inline void j2k_mq_encode( + thread J2kMqEncoder &encoder, + thread uchar *contexts, + uint ctx_label, + uint bit +) { + uchar ctx = contexts[ctx_label]; + const J2kQeData qe = J2K_QE_TABLE[ctx & uchar(0x7Fu)]; + const uint mps = uint(ctx >> 7u); + encoder.a -= qe.qe; + + if (bit == mps) { + if ((encoder.a & 0x8000u) != 0u) { + encoder.c += qe.qe; + return; + } + if (encoder.a < qe.qe) { + encoder.a = qe.qe; + } else { + encoder.c += qe.qe; + } + ctx = uchar((ctx & uchar(0x80u)) | qe.nmps); + } else { + if (encoder.a < qe.qe) { + encoder.c += qe.qe; + } else { + encoder.a = qe.qe; + } + if (qe.switch_mps != 0u) { + ctx ^= uchar(0x80u); + } + ctx = uchar((ctx & uchar(0x80u)) | qe.nlps); + } + + contexts[ctx_label] = ctx; + j2k_mq_renormalize(encoder); +} + +inline void j2k_mq_set_bits(thread J2kMqEncoder &encoder) { + const uint temp = encoder.c + encoder.a; + encoder.c |= 0xFFFFu; + if (encoder.c >= temp) { + encoder.c -= 0x8000u; + } +} + +inline void j2k_mq_finish(thread J2kMqEncoder &encoder) { + j2k_mq_set_bits(encoder); + encoder.c <<= encoder.ct; + j2k_mq_byte_out(encoder); + encoder.c <<= encoder.ct; + j2k_mq_byte_out(encoder); +} + +inline void j2k_raw_writer_init(thread J2kRawBitWriter &writer, device uchar *out, uint capacity) { + writer.data = out; + writer.max_len = capacity; + writer.len = 0u; + writer.buffer = 0u; + writer.bits_in_buffer = 0u; + writer.last_byte_was_ff = 0u; + writer.failed = 0u; +} + +inline void j2k_raw_writer_push(thread J2kRawBitWriter &writer, uchar value) { + if (writer.len >= writer.max_len) { + writer.failed = 1u; + return; + } + writer.data[writer.len] = value; + writer.len += 1u; + writer.last_byte_was_ff = value == uchar(0xFFu) ? 1u : 0u; +} + +inline void j2k_raw_writer_flush_byte(thread J2kRawBitWriter &writer) { + const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; + const uchar byte = uchar(writer.buffer >> (writer.bits_in_buffer - limit)); + j2k_raw_writer_push(writer, byte); + writer.bits_in_buffer -= limit; + writer.buffer &= writer.bits_in_buffer == 0u ? 0u : ((1u << writer.bits_in_buffer) - 1u); +} + +inline void j2k_raw_writer_write_bit(thread J2kRawBitWriter &writer, uint bit) { + writer.buffer = (writer.buffer << 1u) | (bit & 1u); + writer.bits_in_buffer += 1u; + const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; + if (writer.bits_in_buffer >= limit) { + j2k_raw_writer_flush_byte(writer); + } +} + +inline void j2k_raw_writer_finish(thread J2kRawBitWriter &writer) { + if (writer.bits_in_buffer == 0u) { + return; + } + const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; + const uint shift = limit - writer.bits_in_buffer; + j2k_raw_writer_push(writer, uchar(writer.buffer << shift)); + writer.buffer = 0u; + writer.bits_in_buffer = 0u; +} + +inline uint j2k_classic_magnitude(int value) { + return value < 0 ? uint(-value) : uint(value); +} + +inline void j2k_classic_clear_state_border( + thread uchar *states, + uint padded_width, + uint width, + uint height +) { + const uint bottom_row = height + 1u; + for (uint x = 0u; x < padded_width; ++x) { + states[coeff_index(padded_width, x, 0u)] = uchar(0u); + states[coeff_index(padded_width, x, bottom_row)] = uchar(0u); + } + for (uint y = 1u; y <= height; ++y) { + states[coeff_index(padded_width, 0u, y)] = uchar(0u); + states[coeff_index(padded_width, width + 1u, y)] = uchar(0u); + } +} + +inline uchar j2k_classic_effective_neighbors( + thread const uchar *states, + uint padded_width, + uint index_x, + uint index_y, + uint height, + uint style_flags +) { + return effective_neighborhood_states(states, padded_width, index_x, index_y, height, style_flags); +} + +inline bool j2k_classic_coded_marker_matches( + thread const uchar *states, + uint idx, + uchar coded_marker +) { + return (states[idx] & J2K_ENCODE_CODED_MARKER_MASK) == + (coded_marker & J2K_ENCODE_CODED_MARKER_MASK); +} + +inline void j2k_classic_set_coded_marker( + thread uchar *states, + uint idx, + uchar coded_marker +) { + states[idx] = + (states[idx] & uchar(~J2K_ENCODE_CODED_MARKER_MASK)) | + (coded_marker & J2K_ENCODE_CODED_MARKER_MASK); +} + +inline void j2k_classic_encode_sign_from_neighbors( + uint idx, + thread const uchar *states, + thread J2kMqEncoder &encoder, + thread uchar *contexts, + uint padded_width, + uint index_x, + uint index_y, + uint height, + uint style_flags, + uchar neighbor_sig +) { + const uchar significances = neighbor_sig & uchar(0b01010101); + const uint left_sign = coeff_sign(states, coeff_index(padded_width, index_x - 1u, index_y)); + const uint right_sign = coeff_sign(states, coeff_index(padded_width, index_x + 1u, index_y)); + const uint top_sign = coeff_sign(states, coeff_index(padded_width, index_x, index_y - 1u)); + const uint bottom_sign = + ((style_flags & J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT) != 0u && + neighbor_in_next_stripe(index_y, height)) + ? 0u + : coeff_sign(states, coeff_index(padded_width, index_x, index_y + 1u)); + const uchar signs = uchar((top_sign << 6u) | (left_sign << 4u) | (right_sign << 2u) | bottom_sign); + const uchar negative = significances & signs; + const uchar positive = significances & uchar(~signs); + const uchar2 sign_ctx = SIGN_CONTEXT_LOOKUP[uchar((negative << 1u) | positive)]; + const uint sign_bit = (uint(states[idx]) >> 5u) & 1u; + j2k_mq_encode(encoder, contexts, uint(sign_ctx.x), sign_bit ^ uint(sign_ctx.y)); +} + +template +inline void j2k_classic_significance_pass( + thread const Magnitude *magnitudes, + thread uchar *states, + uchar coded_marker, + thread J2kMqEncoder &encoder, + thread uchar *contexts, + uint width, + uint height, + uint padded_width, + uint bit_mask, + uint sub_band_type, + uint style_flags +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + const uchar neighbor_sig = + j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && neighbor_sig != 0u) { + const uint ctx_label = uint(zero_context_label(neighbor_sig, sub_band_type)); + const uint bit = (magnitudes[idx] & bit_mask) != 0u ? 1u : 0u; + j2k_mq_encode(encoder, contexts, ctx_label, bit); + j2k_classic_set_coded_marker(states, idx, coded_marker); + if (bit != 0u) { + j2k_classic_encode_sign_from_neighbors( + idx, + states, + encoder, + contexts, + padded_width, + ix, + iy, + height, + style_flags, + neighbor_sig + ); + set_significant(states, padded_width, ix, iy); + } + } + } + } + } +} + +template +inline void j2k_classic_magnitude_refinement_pass( + thread const Magnitude *magnitudes, + thread uchar *states, + uchar coded_marker, + thread J2kMqEncoder &encoder, + thread uchar *contexts, + uint width, + uint height, + uint padded_width, + uint bit_mask, + uint style_flags +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) != 0u && + !j2k_classic_coded_marker_matches(states, idx, coded_marker)) { + const uint ctx_label = + uint(magnitude_refinement_context(states, padded_width, ix, iy, height, style_flags)); + const uint bit = (magnitudes[idx] & bit_mask) != 0u ? 1u : 0u; + j2k_mq_encode(encoder, contexts, ctx_label, bit); + states[idx] |= J2K_ENCODE_MAGNITUDE_REFINED; + } + } + } + } +} + +template +inline void j2k_classic_significance_pass_raw( + thread const Magnitude *magnitudes, + thread uchar *states, + uchar coded_marker, + thread J2kRawBitWriter &writer, + uint width, + uint height, + uint padded_width, + uint bit_mask, + uint style_flags +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + const uchar neighbor_sig = + j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && neighbor_sig != 0u) { + const uint bit = (magnitudes[idx] & bit_mask) != 0u ? 1u : 0u; + j2k_raw_writer_write_bit(writer, bit); + j2k_classic_set_coded_marker(states, idx, coded_marker); + if (bit != 0u) { + j2k_raw_writer_write_bit(writer, (uint(states[idx]) >> 5u) & 1u); + set_significant(states, padded_width, ix, iy); + } + } + } + } + } +} + +template +inline void j2k_classic_magnitude_refinement_pass_raw( + thread const Magnitude *magnitudes, + thread uchar *states, + uchar coded_marker, + thread J2kRawBitWriter &writer, + uint width, + uint height, + uint padded_width, + uint bit_mask +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) != 0u && + !j2k_classic_coded_marker_matches(states, idx, coded_marker)) { + const uint bit = (magnitudes[idx] & bit_mask) != 0u ? 1u : 0u; + j2k_raw_writer_write_bit(writer, bit); + states[idx] |= J2K_ENCODE_MAGNITUDE_REFINED; + } + } + } + } +} + +template +inline void j2k_classic_cleanup_pass( + thread const Magnitude *magnitudes, + thread uchar *states, + uchar coded_marker, + thread J2kMqEncoder &encoder, + thread uchar *contexts, + uint width, + uint height, + uint padded_width, + uint bit_mask, + uint sub_band_type, + uint style_flags +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + const uint stripe_height = y_end - y_base; + + if (stripe_height == 4u) { + bool all_zero_uncoded = true; + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + const uchar neighbor_sig = + j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) != 0u || + j2k_classic_coded_marker_matches(states, idx, coded_marker) || + neighbor_sig != 0u) { + all_zero_uncoded = false; + break; + } + } + + if (all_zero_uncoded) { + uint first_sig = 4u; + for (uint pos = 0u; pos < 4u; ++pos) { + const uint idx = coeff_index(padded_width, x + 1u, y_base + pos + 1u); + if ((magnitudes[idx] & bit_mask) != 0u) { + first_sig = pos; + break; + } + } + + if (first_sig < 4u) { + j2k_mq_encode(encoder, contexts, 17u, 1u); + j2k_mq_encode(encoder, contexts, 18u, (first_sig >> 1u) & 1u); + j2k_mq_encode(encoder, contexts, 18u, first_sig & 1u); + + const uint sig_y = y_base + first_sig; + const uint sig_idx = coeff_index(padded_width, x + 1u, sig_y + 1u); + j2k_classic_encode_sign_from_neighbors( + sig_idx, + states, + encoder, + contexts, + padded_width, + x + 1u, + sig_y + 1u, + height, + style_flags, + uchar(0u) + ); + set_significant(states, padded_width, x + 1u, sig_y + 1u); + + for (uint y = sig_y + 1u; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && + !j2k_classic_coded_marker_matches(states, idx, coded_marker)) { + const uchar neighbor_sig = + j2k_classic_effective_neighbors( + states, + padded_width, + ix, + iy, + height, + style_flags + ); + const uint ctx_label = uint(zero_context_label(neighbor_sig, sub_band_type)); + const uint bit = (magnitudes[idx] & bit_mask) != 0u ? 1u : 0u; + j2k_mq_encode(encoder, contexts, ctx_label, bit); + if (bit != 0u) { + j2k_classic_encode_sign_from_neighbors( + idx, + states, + encoder, + contexts, + padded_width, + ix, + iy, + height, + style_flags, + neighbor_sig + ); + set_significant(states, padded_width, ix, iy); + } + } + } + continue; + } + + j2k_mq_encode(encoder, contexts, 17u, 0u); + continue; + } + } + + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && + !j2k_classic_coded_marker_matches(states, idx, coded_marker)) { + const uchar neighbor_sig = + j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); + const uint ctx_label = uint(zero_context_label(neighbor_sig, sub_band_type)); + const uint bit = (magnitudes[idx] & bit_mask) != 0u ? 1u : 0u; + j2k_mq_encode(encoder, contexts, ctx_label, bit); + if (bit != 0u) { + j2k_classic_encode_sign_from_neighbors( + idx, + states, + encoder, + contexts, + padded_width, + ix, + iy, + height, + style_flags, + neighbor_sig + ); + set_significant(states, padded_width, ix, iy); + } + } + } + } + } +} + +inline void j2k_classic_encode_segmentation_symbols( + thread J2kMqEncoder &encoder, + thread uchar *contexts +) { + j2k_mq_encode(encoder, contexts, 18u, 1u); + j2k_mq_encode(encoder, contexts, 18u, 0u); + j2k_mq_encode(encoder, contexts, 18u, 1u); + j2k_mq_encode(encoder, contexts, 18u, 0u); +} + +inline uint j2k_classic_bypass_segment_idx(uint pass_idx) { + if (pass_idx < 10u) { + return 0u; + } + return 1u + (2u * ((pass_idx - 10u) / 3u)) + (((pass_idx - 10u) % 3u) == 2u ? 1u : 0u); +} + +inline bool j2k_classic_push_segment( + device J2kClassicSegment *segments, + uint segment_capacity, + thread uint &segment_count, + uint data_offset, + uint data_length, + uint start_pass, + uint end_pass, + bool use_arithmetic +) { + if (segment_count >= segment_capacity) { + return false; + } + segments[segment_count].data_offset = data_offset; + segments[segment_count].data_length = data_length; + segments[segment_count].start_coding_pass = start_pass; + segments[segment_count].end_coding_pass = end_pass; + segments[segment_count].use_arithmetic = use_arithmetic ? 1u : 0u; + segment_count += 1u; + return true; +} + +inline uint j2k_classic_finish_arithmetic_segment(thread J2kMqEncoder &encoder) { + j2k_mq_finish(encoder); + if (encoder.failed != 0u || encoder.len == 0u) { + encoder.failed = 1u; + return 0u; + } + const uint data_len = encoder.len - 1u; + for (uint idx = 0u; idx < data_len; ++idx) { + encoder.data[idx] = encoder.data[idx + 1u]; + } + return data_len; +} + +inline uint j2k_classic_finish_arithmetic_segment_unshifted(thread J2kMqEncoder &encoder) { + j2k_mq_finish(encoder); + if (encoder.failed != 0u || encoder.len == 0u) { + encoder.failed = 1u; + return 0u; + } + return encoder.len - 1u; +} + +inline void j2k_encode_classic_code_block_impl_with_scratch( + device const int *coefficients, + device uchar *out, + J2kClassicEncodeParams params, + device J2kClassicEncodeStatus *status, + device J2kClassicSegment *segments, + thread uint *magnitudes, + thread uchar *states, + uint max_width, + uint max_height +) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u, 0u, 0u, 0u); + + if (params.width == 0u || params.height == 0u || + params.width > max_width || + params.height > max_height || + params.total_bitplanes > 31u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 1u, 0u, 0u, 0u, 0u); + return; + } + + const uint padded_width = params.width + 2u; + + j2k_classic_clear_state_border(states, padded_width, params.width, params.height); + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint src_idx = y * params.width + x; + const int value = coefficients[src_idx]; + const uint dst_idx = coeff_index(padded_width, x + 1u, y + 1u); + const uint magnitude = j2k_classic_magnitude(value); + magnitudes[dst_idx] = magnitude; + states[dst_idx] = value < 0 ? J2K_ENCODE_SIGN : uchar(0u); + max_magnitude = max(max_magnitude, magnitude); + } + } + + if (max_magnitude == 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_OK, + 0u, + 0u, + 0u, + params.total_bitplanes, + 0u + ); + return; + } + + const uint num_bitplanes = 32u - clz(max_magnitude); + if (num_bitplanes > params.total_bitplanes) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 3u, 0u, 0u, 0u, 0u); + return; + } + const uint missing_bit_planes = params.total_bitplanes - num_bitplanes; + + thread uchar contexts[19]; + reset_contexts(contexts); + uchar coded_marker = uchar(1u); + + if ((params.style_flags & (J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS | + J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS)) != 0u) { + const uint total_passes = 1u + 3u * (num_bitplanes - 1u); + uint data_cursor = 0u; + uint segment_count = 0u; + uint current_segment_idx = 0xFFFFFFFFu; + uint current_segment_start_pass = 0u; + bool current_use_arithmetic = true; + bool have_segment = false; + thread J2kMqEncoder arithmetic_encoder; + thread J2kRawBitWriter raw_writer; + + for (uint coding_pass = 0u; coding_pass < total_passes; ++coding_pass) { + const uint segment_idx = + (params.style_flags & J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS) != 0u + ? coding_pass + : j2k_classic_bypass_segment_idx(coding_pass); + const bool use_arithmetic = + (params.style_flags & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) == 0u || + coding_pass <= 9u || + (coding_pass % 3u) == 0u; + + if (!have_segment || current_segment_idx != segment_idx) { + if (have_segment) { + uint segment_len = 0u; + if (current_use_arithmetic) { + segment_len = j2k_classic_finish_arithmetic_segment(arithmetic_encoder); + if (arithmetic_encoder.failed != 0u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 6u, 0u, 0u, 0u, 0u); + return; + } + } else { + j2k_raw_writer_finish(raw_writer); + if (raw_writer.failed != 0u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 7u, 0u, 0u, 0u, 0u); + return; + } + segment_len = raw_writer.len; + } + if (!j2k_classic_push_segment( + segments, + params.segment_capacity, + segment_count, + data_cursor, + segment_len, + current_segment_start_pass, + coding_pass, + current_use_arithmetic)) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 8u, 0u, 0u, 0u, 0u); + return; + } + data_cursor += segment_len; + } + + current_segment_idx = segment_idx; + current_segment_start_pass = coding_pass; + current_use_arithmetic = use_arithmetic; + have_segment = true; + if (data_cursor > params.output_capacity) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 9u, 0u, 0u, 0u, 0u); + return; + } + const uint remaining_capacity = params.output_capacity - data_cursor; + if (use_arithmetic) { + j2k_mq_init(arithmetic_encoder, out + data_cursor, remaining_capacity); + } else { + j2k_raw_writer_init(raw_writer, out + data_cursor, remaining_capacity); + } + } + + const uint current_bitplane = (coding_pass + 2u) / 3u; + const uint bit_mask = 1u << (num_bitplanes - 1u - current_bitplane); + switch (coding_pass % 3u) { + case 0u: + j2k_classic_cleanup_pass( + magnitudes, + states, + coded_marker, + arithmetic_encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + params.style_flags + ); + if ((params.style_flags & J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS) != 0u) { + j2k_classic_encode_segmentation_symbols(arithmetic_encoder, contexts); + } + coded_marker += uchar(1u); + break; + case 1u: + if (use_arithmetic) { + j2k_classic_significance_pass( + magnitudes, + states, + coded_marker, + arithmetic_encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + params.style_flags + ); + } else { + j2k_classic_significance_pass_raw( + magnitudes, + states, + coded_marker, + raw_writer, + params.width, + params.height, + padded_width, + bit_mask, + params.style_flags + ); + } + break; + default: + if (use_arithmetic) { + j2k_classic_magnitude_refinement_pass( + magnitudes, + states, + coded_marker, + arithmetic_encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.style_flags + ); + } else { + j2k_classic_magnitude_refinement_pass_raw( + magnitudes, + states, + coded_marker, + raw_writer, + params.width, + params.height, + padded_width, + bit_mask + ); + } + break; + } + + if ((params.style_flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0u) { + reset_contexts(contexts); + } + const bool current_failed = use_arithmetic + ? arithmetic_encoder.failed != 0u + : raw_writer.failed != 0u; + if (current_failed) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 10u, 0u, 0u, 0u, 0u); + return; + } +} + + if (have_segment) { + uint segment_len = 0u; + if (current_use_arithmetic) { + segment_len = j2k_classic_finish_arithmetic_segment(arithmetic_encoder); + if (arithmetic_encoder.failed != 0u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 11u, 0u, 0u, 0u, 0u); + return; + } + } else { + j2k_raw_writer_finish(raw_writer); + if (raw_writer.failed != 0u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 12u, 0u, 0u, 0u, 0u); + return; + } + segment_len = raw_writer.len; + } + if (!j2k_classic_push_segment( + segments, + params.segment_capacity, + segment_count, + data_cursor, + segment_len, + current_segment_start_pass, + total_passes, + current_use_arithmetic)) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 13u, 0u, 0u, 0u, 0u); + return; + } + data_cursor += segment_len; + } + + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_OK, + 0u, + data_cursor, + total_passes, + missing_bit_planes, + segment_count + ); + return; + } + + thread J2kMqEncoder encoder; + j2k_mq_init(encoder, out, params.output_capacity); + + uint pass_count = 0u; + for (int bp = int(num_bitplanes) - 1; bp >= 0; --bp) { + const uint bit_mask = 1u << uint(bp); + const bool first_bitplane = uint(bp) == num_bitplanes - 1u; + + if (first_bitplane) { + j2k_classic_cleanup_pass( + magnitudes, + states, + coded_marker, + encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + params.style_flags + ); + if ((params.style_flags & J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS) != 0u) { + j2k_classic_encode_segmentation_symbols(encoder, contexts); + } + pass_count += 1u; + if ((params.style_flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0u) { + reset_contexts(contexts); + } + } else { + j2k_classic_significance_pass( + magnitudes, + states, + coded_marker, + encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + params.style_flags + ); + pass_count += 1u; + if ((params.style_flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0u) { + reset_contexts(contexts); + } + + j2k_classic_magnitude_refinement_pass( + magnitudes, + states, + coded_marker, + encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.style_flags + ); + pass_count += 1u; + if ((params.style_flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0u) { + reset_contexts(contexts); + } + + j2k_classic_cleanup_pass( + magnitudes, + states, + coded_marker, + encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + params.style_flags + ); + if ((params.style_flags & J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS) != 0u) { + j2k_classic_encode_segmentation_symbols(encoder, contexts); + } + pass_count += 1u; + if ((params.style_flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0u) { + reset_contexts(contexts); + } + } + + coded_marker += uchar(1u); + + if (encoder.failed != 0u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 4u, 0u, 0u, 0u, 0u); + return; + } + } + + const uint payload_skip = 1u; + const uint data_len = j2k_classic_finish_arithmetic_segment_unshifted(encoder); + if (encoder.failed != 0u || encoder.len == 0u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 5u, 0u, 0u, 0u, 0u); + return; + } + uint segment_count = 0u; + if (!j2k_classic_push_segment( + segments, + params.segment_capacity, + segment_count, + 0u, + data_len, + 0u, + pass_count, + true)) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 14u, 0u, 0u, 0u, 0u); + return; + } + + j2k_set_encode_status_with_payload_skip( + status, + J2K_ENCODE_STATUS_OK, + 0u, + data_len, + pass_count, + missing_bit_planes, + segment_count, + payload_skip + ); +} + +inline void j2k_encode_classic_code_block_impl( + device const int *coefficients, + device uchar *out, + J2kClassicEncodeParams params, + device J2kClassicEncodeStatus *status, + device J2kClassicSegment *segments +) { + thread uint magnitudes[J2K_CLASSIC_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_MAX_COEFF_COUNT]; + j2k_encode_classic_code_block_impl_with_scratch( + coefficients, + out, + params, + status, + segments, + magnitudes, + states, + J2K_CLASSIC_MAX_WIDTH, + J2K_CLASSIC_MAX_HEIGHT + ); +} + +inline void j2k_encode_classic_code_block_impl_style0_with_scratch( + device const int *coefficients, + device uchar *out, + J2kClassicEncodeParams params, + device J2kClassicEncodeStatus *status, + device J2kClassicSegment *segments, + thread uint *magnitudes, + thread uchar *states, + uint max_width, + uint max_height +) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u, 0u, 0u, 0u); + + if (params.width == 0u || params.height == 0u || + params.width > max_width || + params.height > max_height || + params.total_bitplanes > 31u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 1u, 0u, 0u, 0u, 0u); + return; + } + + const uint padded_width = params.width + 2u; + + j2k_classic_clear_state_border(states, padded_width, params.width, params.height); + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint src_idx = y * params.width + x; + const int value = coefficients[src_idx]; + const uint dst_idx = coeff_index(padded_width, x + 1u, y + 1u); + const uint magnitude = j2k_classic_magnitude(value); + magnitudes[dst_idx] = magnitude; + states[dst_idx] = value < 0 ? J2K_ENCODE_SIGN : uchar(0u); + max_magnitude = max(max_magnitude, magnitude); + } + } + + if (max_magnitude == 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_OK, + 0u, + 0u, + 0u, + params.total_bitplanes, + 0u + ); + return; + } + + const uint num_bitplanes = 32u - clz(max_magnitude); + if (num_bitplanes > params.total_bitplanes) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 3u, 0u, 0u, 0u, 0u); + return; + } + const uint missing_bit_planes = params.total_bitplanes - num_bitplanes; + + thread uchar contexts[19]; + reset_contexts(contexts); + uchar coded_marker = uchar(1u); + thread J2kMqEncoder encoder; + j2k_mq_init(encoder, out, params.output_capacity); + + uint pass_count = 0u; + for (int bp = int(num_bitplanes) - 1; bp >= 0; --bp) { + const uint bit_mask = 1u << uint(bp); + const bool first_bitplane = uint(bp) == num_bitplanes - 1u; + + if (first_bitplane) { + j2k_classic_cleanup_pass( + magnitudes, + states, + coded_marker, + encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u + ); + pass_count += 1u; + } else { + j2k_classic_significance_pass( + magnitudes, + states, + coded_marker, + encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u + ); + pass_count += 1u; + + j2k_classic_magnitude_refinement_pass( + magnitudes, + states, + coded_marker, + encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + 0u + ); + pass_count += 1u; + + j2k_classic_cleanup_pass( + magnitudes, + states, + coded_marker, + encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u + ); + pass_count += 1u; + } + + coded_marker += uchar(1u); + + if (encoder.failed != 0u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 4u, 0u, 0u, 0u, 0u); + return; + } + } + + const uint payload_skip = 1u; + const uint data_len = j2k_classic_finish_arithmetic_segment_unshifted(encoder); + if (encoder.failed != 0u || encoder.len == 0u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 5u, 0u, 0u, 0u, 0u); + return; + } + uint segment_count = 0u; + if (!j2k_classic_push_segment( + segments, + params.segment_capacity, + segment_count, + 0u, + data_len, + 0u, + pass_count, + true)) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 14u, 0u, 0u, 0u, 0u); + return; + } + + j2k_set_encode_status_with_payload_skip( + status, + J2K_ENCODE_STATUS_OK, + 0u, + data_len, + pass_count, + missing_bit_planes, + segment_count, + payload_skip + ); +} + +inline void j2k_encode_classic_code_block_impl_style0( + device const int *coefficients, + device uchar *out, + J2kClassicEncodeParams params, + device J2kClassicEncodeStatus *status, + device J2kClassicSegment *segments +) { + thread uint magnitudes[J2K_CLASSIC_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_MAX_COEFF_COUNT]; + j2k_encode_classic_code_block_impl_style0_with_scratch( + coefficients, + out, + params, + status, + segments, + magnitudes, + states, + J2K_CLASSIC_MAX_WIDTH, + J2K_CLASSIC_MAX_HEIGHT + ); +} + +inline void j2k_encode_classic_code_block_impl_32( + device const int *coefficients, + device uchar *out, + J2kClassicEncodeParams params, + device J2kClassicEncodeStatus *status, + device J2kClassicSegment *segments +) { + thread uint magnitudes[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + j2k_encode_classic_code_block_impl_with_scratch( + coefficients, + out, + params, + status, + segments, + magnitudes, + states, + J2K_CLASSIC_ENCODE_32_MAX_WIDTH, + J2K_CLASSIC_ENCODE_32_MAX_HEIGHT + ); +} + +template +inline void j2k_encode_classic_code_block_impl_bypass_32_with_scratch( + device const int *coefficients, + device uchar *out, + J2kClassicEncodeParams params, + device J2kClassicEncodeStatus *status, + device J2kClassicSegment *segments, + thread Magnitude *magnitudes, + thread uchar *states, + uint max_bitplanes +) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u, 0u, 0u, 0u); + + if (params.width == 0u || params.height == 0u || + params.width > J2K_CLASSIC_ENCODE_32_MAX_WIDTH || + params.height > J2K_CLASSIC_ENCODE_32_MAX_HEIGHT || + params.total_bitplanes > max_bitplanes) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 1u, 0u, 0u, 0u, 0u); + return; + } + + const uint padded_width = params.width + 2u; + + j2k_classic_clear_state_border(states, padded_width, params.width, params.height); + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint src_idx = y * params.width + x; + const int value = coefficients[src_idx]; + const uint dst_idx = coeff_index(padded_width, x + 1u, y + 1u); + const uint magnitude = j2k_classic_magnitude(value); + magnitudes[dst_idx] = Magnitude(magnitude); + states[dst_idx] = value < 0 ? J2K_ENCODE_SIGN : uchar(0u); + max_magnitude = max(max_magnitude, magnitude); + } + } + + if (max_magnitude == 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_OK, + 0u, + 0u, + 0u, + params.total_bitplanes, + 0u + ); + return; + } + + const uint num_bitplanes = 32u - clz(max_magnitude); + if (num_bitplanes > params.total_bitplanes) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 3u, 0u, 0u, 0u, 0u); + return; + } + const uint missing_bit_planes = params.total_bitplanes - num_bitplanes; + + thread uchar contexts[19]; + reset_contexts(contexts); + uchar coded_marker = uchar(1u); + const uint total_passes = 1u + 3u * (num_bitplanes - 1u); + uint data_cursor = 0u; + uint physical_cursor = 0u; + uint segment_count = 0u; + uint current_segment_idx = 0xFFFFFFFFu; + uint current_segment_start_pass = 0u; + bool current_use_arithmetic = true; + bool have_segment = false; + thread J2kMqEncoder arithmetic_encoder; + thread J2kRawBitWriter raw_writer; + + for (uint coding_pass = 0u; coding_pass < total_passes; ++coding_pass) { + const uint segment_idx = j2k_classic_bypass_segment_idx(coding_pass); + const bool use_arithmetic = coding_pass <= 9u || (coding_pass % 3u) == 0u; + + if (!have_segment || current_segment_idx != segment_idx) { + if (have_segment) { + uint segment_len = 0u; + if (current_use_arithmetic) { + segment_len = j2k_classic_finish_arithmetic_segment(arithmetic_encoder); + if (arithmetic_encoder.failed != 0u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 6u, 0u, 0u, 0u, 0u); + return; + } + physical_cursor += segment_len; + } else { + j2k_raw_writer_finish(raw_writer); + if (raw_writer.failed != 0u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 7u, 0u, 0u, 0u, 0u); + return; + } + segment_len = raw_writer.len; + physical_cursor += segment_len; + } + if (!j2k_classic_push_segment( + segments, + params.segment_capacity, + segment_count, + data_cursor, + segment_len, + current_segment_start_pass, + coding_pass, + current_use_arithmetic)) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 8u, 0u, 0u, 0u, 0u); + return; + } + data_cursor += segment_len; + } + + current_segment_idx = segment_idx; + current_segment_start_pass = coding_pass; + current_use_arithmetic = use_arithmetic; + have_segment = true; + if (physical_cursor > params.output_capacity) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 9u, 0u, 0u, 0u, 0u); + return; + } + const uint remaining_capacity = params.output_capacity - physical_cursor; + if (use_arithmetic) { + j2k_mq_init(arithmetic_encoder, out + physical_cursor, remaining_capacity); + } else { + j2k_raw_writer_init(raw_writer, out + physical_cursor, remaining_capacity); + } + } + + const uint current_bitplane = (coding_pass + 2u) / 3u; + const uint bit_mask = 1u << (num_bitplanes - 1u - current_bitplane); + switch (coding_pass % 3u) { + case 0u: + j2k_classic_cleanup_pass( + magnitudes, + states, + coded_marker, + arithmetic_encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u + ); + coded_marker += uchar(1u); + break; + case 1u: + if (use_arithmetic) { + j2k_classic_significance_pass( + magnitudes, + states, + coded_marker, + arithmetic_encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u + ); + } else { + j2k_classic_significance_pass_raw( + magnitudes, + states, + coded_marker, + raw_writer, + params.width, + params.height, + padded_width, + bit_mask, + 0u + ); + } + break; + default: + if (use_arithmetic) { + j2k_classic_magnitude_refinement_pass( + magnitudes, + states, + coded_marker, + arithmetic_encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + 0u + ); + } else { + j2k_classic_magnitude_refinement_pass_raw( + magnitudes, + states, + coded_marker, + raw_writer, + params.width, + params.height, + padded_width, + bit_mask + ); + } + break; + } + + const bool current_failed = use_arithmetic + ? arithmetic_encoder.failed != 0u + : raw_writer.failed != 0u; + if (current_failed) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 10u, 0u, 0u, 0u, 0u); + return; + } + } + + if (have_segment) { + uint segment_len = 0u; + if (current_use_arithmetic) { + segment_len = j2k_classic_finish_arithmetic_segment(arithmetic_encoder); + if (arithmetic_encoder.failed != 0u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 11u, 0u, 0u, 0u, 0u); + return; + } + physical_cursor += segment_len; + } else { + j2k_raw_writer_finish(raw_writer); + if (raw_writer.failed != 0u) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 12u, 0u, 0u, 0u, 0u); + return; + } + segment_len = raw_writer.len; + physical_cursor += segment_len; + } + if (!j2k_classic_push_segment( + segments, + params.segment_capacity, + segment_count, + data_cursor, + segment_len, + current_segment_start_pass, + total_passes, + current_use_arithmetic)) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 13u, 0u, 0u, 0u, 0u); + return; + } + data_cursor += segment_len; + } + + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_OK, + 0u, + data_cursor, + total_passes, + missing_bit_planes, + segment_count + ); +} + +inline void j2k_encode_classic_code_block_impl_bypass_32( + device const int *coefficients, + device uchar *out, + J2kClassicEncodeParams params, + device J2kClassicEncodeStatus *status, + device J2kClassicSegment *segments +) { + thread uint magnitudes[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + j2k_encode_classic_code_block_impl_bypass_32_with_scratch( + coefficients, + out, + params, + status, + segments, + magnitudes, + states, + 31u + ); +} + +inline void j2k_encode_classic_code_block_impl_bypass_u16_32( + device const int *coefficients, + device uchar *out, + J2kClassicEncodeParams params, + device J2kClassicEncodeStatus *status, + device J2kClassicSegment *segments +) { + thread ushort magnitudes[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + j2k_encode_classic_code_block_impl_bypass_32_with_scratch( + coefficients, + out, + params, + status, + segments, + magnitudes, + states, + 16u + ); +} + +inline void j2k_encode_classic_code_block_impl_style0_32( + device const int *coefficients, + device uchar *out, + J2kClassicEncodeParams params, + device J2kClassicEncodeStatus *status, + device J2kClassicSegment *segments +) { + thread uint magnitudes[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + j2k_encode_classic_code_block_impl_style0_with_scratch( + coefficients, + out, + params, + status, + segments, + magnitudes, + states, + J2K_CLASSIC_ENCODE_32_MAX_WIDTH, + J2K_CLASSIC_ENCODE_32_MAX_HEIGHT + ); +} + +struct J2kClassicEncodeBatchJob { + uint coefficient_offset; + uint output_offset; + uint segment_offset; + uint width; + uint height; + uint sub_band_type; + uint total_bitplanes; + uint style_flags; + uint output_capacity; + uint segment_capacity; +}; + +struct J2kClassicTier1DensityCounters { + uint sigprop_active_candidates; + uint sigprop_new_significant; + uint magref_active_candidates; + uint cleanup_active_candidates; + uint cleanup_new_significant; + uint cleanup_rlc_stripes; + uint cleanup_rlc_zero_stripes; + uint arithmetic_sigprop_active_candidates; + uint arithmetic_sigprop_new_significant; + uint raw_sigprop_active_candidates; + uint raw_sigprop_new_significant; + uint arithmetic_magref_active_candidates; + uint raw_magref_active_candidates; + uint reserved0; + uint reserved1; +}; + +struct J2kClassicTier1SymbolPlanCounters { + uint code; + uint detail; + uint coding_passes; + uint missing_bit_planes; + uint segment_count; + uint mq_symbol_count; + uint raw_bit_count; + uint cleanup_mq_symbol_count; + uint sigprop_mq_symbol_count; + uint magref_mq_symbol_count; + uint raw_sigprop_bit_count; + uint raw_magref_bit_count; + uint cleanup_sign_symbol_count; + uint sigprop_sign_symbol_count; + uint mq_symbol_hash; + uint raw_bit_hash; +}; + +struct J2kClassicTier1PassPlanCounters { + uint code; + uint detail; + uint coding_passes; + uint missing_bit_planes; + uint segment_count; + uint mq_symbol_count; + uint raw_bit_count; + uint nonempty_mq_passes; + uint nonempty_raw_passes; + uint max_mq_symbols_per_pass; + uint max_raw_bits_per_pass; + uint reserved0; + uint reserved1; + uint reserved2; + uint reserved3; + uint reserved4; + uint mq_symbols_by_pass[J2K_CLASSIC_TIER1_PASS_PLAN_CAPACITY]; + uint raw_bits_by_pass[J2K_CLASSIC_TIER1_PASS_PLAN_CAPACITY]; +}; + +struct J2kClassicTier1TokenSegment { + uint token_bit_offset; + uint token_bit_count; + uint pass_range; + uint flags; +}; + +struct J2kClassicTier1TokenBitWriter { + device uchar *data; + uint capacity; + uint len; + uint bit_count; + uint current_byte; + uint bits_in_current; + uint failed; +}; + +struct J2kClassicTier1TokenByteWriter { + device uchar *data; + uint capacity; + uint len; + uint failed; +}; + +struct J2kClassicTier1TokenEmitter { + J2kClassicTier1SymbolPlanCounters counters; + J2kClassicTier1TokenBitWriter writer; + device J2kClassicTier1TokenSegment *segments; + uint segment_capacity; + uint current_segment_start_bit; + uint segment_failed; +}; + +struct J2kClassicTier1SplitTokenEmitter { + J2kClassicTier1SymbolPlanCounters counters; + J2kClassicTier1TokenBitWriter mq_writer; + J2kClassicTier1TokenBitWriter raw_writer; + device J2kClassicTier1TokenSegment *segments; + uint segment_capacity; + uint current_segment_start_bit; + uint segment_failed; +}; + +struct J2kClassicTier1SplitMqByteRawTokenEmitter { + J2kClassicTier1SymbolPlanCounters counters; + J2kClassicTier1TokenByteWriter mq_writer; + J2kClassicTier1TokenBitWriter raw_writer; + device J2kClassicTier1TokenSegment *segments; + uint segment_capacity; + uint current_segment_start_bit; + uint segment_failed; +}; + +struct J2kClassicTier1PassPlanEmitter { + J2kClassicTier1PassPlanCounters counters; + uint current_pass; +}; + +struct J2kClassicTier1TokenBitReader { + device const uchar *data; + uint bit_pos; + uint bit_capacity; + uint failed; +}; + +inline void j2k_classic_tier1_density_zero(thread J2kClassicTier1DensityCounters &counters) { + counters.sigprop_active_candidates = 0u; + counters.sigprop_new_significant = 0u; + counters.magref_active_candidates = 0u; + counters.cleanup_active_candidates = 0u; + counters.cleanup_new_significant = 0u; + counters.cleanup_rlc_stripes = 0u; + counters.cleanup_rlc_zero_stripes = 0u; + counters.arithmetic_sigprop_active_candidates = 0u; + counters.arithmetic_sigprop_new_significant = 0u; + counters.raw_sigprop_active_candidates = 0u; + counters.raw_sigprop_new_significant = 0u; + counters.arithmetic_magref_active_candidates = 0u; + counters.raw_magref_active_candidates = 0u; + counters.reserved0 = 0u; + counters.reserved1 = 0u; +} + +inline void j2k_classic_tier1_density_store( + device J2kClassicTier1DensityCounters *out, + thread const J2kClassicTier1DensityCounters &counters +) { + out->sigprop_active_candidates = counters.sigprop_active_candidates; + out->sigprop_new_significant = counters.sigprop_new_significant; + out->magref_active_candidates = counters.magref_active_candidates; + out->cleanup_active_candidates = counters.cleanup_active_candidates; + out->cleanup_new_significant = counters.cleanup_new_significant; + out->cleanup_rlc_stripes = counters.cleanup_rlc_stripes; + out->cleanup_rlc_zero_stripes = counters.cleanup_rlc_zero_stripes; + out->arithmetic_sigprop_active_candidates = counters.arithmetic_sigprop_active_candidates; + out->arithmetic_sigprop_new_significant = counters.arithmetic_sigprop_new_significant; + out->raw_sigprop_active_candidates = counters.raw_sigprop_active_candidates; + out->raw_sigprop_new_significant = counters.raw_sigprop_new_significant; + out->arithmetic_magref_active_candidates = counters.arithmetic_magref_active_candidates; + out->raw_magref_active_candidates = counters.raw_magref_active_candidates; + out->reserved0 = counters.reserved0; + out->reserved1 = counters.reserved1; +} + +inline void j2k_classic_symbol_plan_zero(thread J2kClassicTier1SymbolPlanCounters &counters) { + counters.code = J2K_ENCODE_STATUS_OK; + counters.detail = 0u; + counters.coding_passes = 0u; + counters.missing_bit_planes = 0u; + counters.segment_count = 0u; + counters.mq_symbol_count = 0u; + counters.raw_bit_count = 0u; + counters.cleanup_mq_symbol_count = 0u; + counters.sigprop_mq_symbol_count = 0u; + counters.magref_mq_symbol_count = 0u; + counters.raw_sigprop_bit_count = 0u; + counters.raw_magref_bit_count = 0u; + counters.cleanup_sign_symbol_count = 0u; + counters.sigprop_sign_symbol_count = 0u; + counters.mq_symbol_hash = 2166136261u; + counters.raw_bit_hash = 2166136261u; +} + +inline void j2k_classic_symbol_plan_store( + device J2kClassicTier1SymbolPlanCounters *out, + thread const J2kClassicTier1SymbolPlanCounters &counters +) { + out->code = counters.code; + out->detail = counters.detail; + out->coding_passes = counters.coding_passes; + out->missing_bit_planes = counters.missing_bit_planes; + out->segment_count = counters.segment_count; + out->mq_symbol_count = counters.mq_symbol_count; + out->raw_bit_count = counters.raw_bit_count; + out->cleanup_mq_symbol_count = counters.cleanup_mq_symbol_count; + out->sigprop_mq_symbol_count = counters.sigprop_mq_symbol_count; + out->magref_mq_symbol_count = counters.magref_mq_symbol_count; + out->raw_sigprop_bit_count = counters.raw_sigprop_bit_count; + out->raw_magref_bit_count = counters.raw_magref_bit_count; + out->cleanup_sign_symbol_count = counters.cleanup_sign_symbol_count; + out->sigprop_sign_symbol_count = counters.sigprop_sign_symbol_count; + out->mq_symbol_hash = counters.mq_symbol_hash; + out->raw_bit_hash = counters.raw_bit_hash; +} + +inline void j2k_classic_pass_plan_zero(thread J2kClassicTier1PassPlanCounters &counters) { + counters.code = J2K_ENCODE_STATUS_OK; + counters.detail = 0u; + counters.coding_passes = 0u; + counters.missing_bit_planes = 0u; + counters.segment_count = 0u; + counters.mq_symbol_count = 0u; + counters.raw_bit_count = 0u; + counters.nonempty_mq_passes = 0u; + counters.nonempty_raw_passes = 0u; + counters.max_mq_symbols_per_pass = 0u; + counters.max_raw_bits_per_pass = 0u; + counters.reserved0 = 0u; + counters.reserved1 = 0u; + counters.reserved2 = 0u; + counters.reserved3 = 0u; + counters.reserved4 = 0u; + for (uint pass = 0u; pass < J2K_CLASSIC_TIER1_PASS_PLAN_CAPACITY; ++pass) { + counters.mq_symbols_by_pass[pass] = 0u; + counters.raw_bits_by_pass[pass] = 0u; + } +} + +inline void j2k_classic_pass_plan_store( + device J2kClassicTier1PassPlanCounters *out, + thread const J2kClassicTier1PassPlanCounters &counters +) { + out->code = counters.code; + out->detail = counters.detail; + out->coding_passes = counters.coding_passes; + out->missing_bit_planes = counters.missing_bit_planes; + out->segment_count = counters.segment_count; + out->mq_symbol_count = counters.mq_symbol_count; + out->raw_bit_count = counters.raw_bit_count; + out->nonempty_mq_passes = counters.nonempty_mq_passes; + out->nonempty_raw_passes = counters.nonempty_raw_passes; + out->max_mq_symbols_per_pass = counters.max_mq_symbols_per_pass; + out->max_raw_bits_per_pass = counters.max_raw_bits_per_pass; + out->reserved0 = counters.reserved0; + out->reserved1 = counters.reserved1; + out->reserved2 = counters.reserved2; + out->reserved3 = counters.reserved3; + out->reserved4 = counters.reserved4; + for (uint pass = 0u; pass < J2K_CLASSIC_TIER1_PASS_PLAN_CAPACITY; ++pass) { + out->mq_symbols_by_pass[pass] = counters.mq_symbols_by_pass[pass]; + out->raw_bits_by_pass[pass] = counters.raw_bits_by_pass[pass]; + } +} + +inline void j2k_classic_tier1_token_writer_init( + thread J2kClassicTier1TokenBitWriter &writer, + device uchar *data, + uint capacity +) { + writer.data = data; + writer.capacity = capacity; + writer.len = 0u; + writer.bit_count = 0u; + writer.current_byte = 0u; + writer.bits_in_current = 0u; + writer.failed = 0u; +} + +inline void j2k_classic_tier1_token_writer_push_byte( + thread J2kClassicTier1TokenBitWriter &writer, + uint value +) { + if (writer.len >= writer.capacity) { + writer.failed = 1u; + return; + } + writer.data[writer.len] = uchar(value & 0xFFu); + writer.len += 1u; +} + +inline void j2k_classic_tier1_token_writer_write_bit( + thread J2kClassicTier1TokenBitWriter &writer, + uint bit +) { + writer.current_byte = (writer.current_byte << 1u) | (bit & 1u); + writer.bits_in_current += 1u; + writer.bit_count += 1u; + if (writer.bits_in_current == 8u) { + j2k_classic_tier1_token_writer_push_byte(writer, writer.current_byte); + writer.current_byte = 0u; + writer.bits_in_current = 0u; + } +} + +inline void j2k_classic_tier1_token_writer_write_bits( + thread J2kClassicTier1TokenBitWriter &writer, + uint value, + uint width +) { + for (uint bit_idx = 0u; bit_idx < width; ++bit_idx) { + const uint shift = width - 1u - bit_idx; + j2k_classic_tier1_token_writer_write_bit(writer, (value >> shift) & 1u); + } +} + +inline void j2k_classic_tier1_token_writer_finish( + thread J2kClassicTier1TokenBitWriter &writer +) { + if (writer.bits_in_current == 0u) { + return; + } + j2k_classic_tier1_token_writer_push_byte( + writer, + writer.current_byte << (8u - writer.bits_in_current) + ); + writer.current_byte = 0u; + writer.bits_in_current = 0u; +} + +inline void j2k_classic_tier1_token_byte_writer_init( + thread J2kClassicTier1TokenByteWriter &writer, + device uchar *data, + uint capacity +) { + writer.data = data; + writer.capacity = capacity; + writer.len = 0u; + writer.failed = 0u; +} + +inline void j2k_classic_tier1_token_byte_writer_push( + thread J2kClassicTier1TokenByteWriter &writer, + uint value +) { + if (writer.len >= writer.capacity) { + writer.failed = 1u; + return; + } + writer.data[writer.len] = uchar(value & 0xFFu); + writer.len += 1u; +} + +inline void j2k_classic_tier1_token_emit_init( + thread J2kClassicTier1TokenEmitter &emitter, + device uchar *token_data, + uint token_capacity, + device J2kClassicTier1TokenSegment *segments, + uint segment_capacity +) { + j2k_classic_symbol_plan_zero(emitter.counters); + j2k_classic_tier1_token_writer_init(emitter.writer, token_data, token_capacity); + emitter.segments = segments; + emitter.segment_capacity = segment_capacity; + emitter.current_segment_start_bit = 0u; + emitter.segment_failed = 0u; +} + +inline void j2k_classic_tier1_token_emit_push_segment( + thread J2kClassicTier1TokenEmitter &emitter, + uint start_pass, + uint end_pass, + bool use_arithmetic +) { + const uint segment_idx = emitter.counters.segment_count; + emitter.counters.segment_count += 1u; + if (segment_idx >= emitter.segment_capacity) { + emitter.segment_failed = 1u; + return; + } + const uint token_bit_count = emitter.writer.bit_count - emitter.current_segment_start_bit; + emitter.segments[segment_idx].token_bit_offset = emitter.current_segment_start_bit; + emitter.segments[segment_idx].token_bit_count = token_bit_count; + emitter.segments[segment_idx].pass_range = + (start_pass & 0xFFFFu) | ((end_pass & 0xFFFFu) << 16u); + emitter.segments[segment_idx].flags = use_arithmetic ? 1u : 0u; + emitter.current_segment_start_bit = emitter.writer.bit_count; +} + +inline void j2k_classic_tier1_token_emit_store( + device J2kClassicTier1SymbolPlanCounters *out, + thread J2kClassicTier1TokenEmitter &emitter +) { + if (emitter.writer.failed != 0u) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 21u; + } else if (emitter.segment_failed != 0u) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 22u; + } + j2k_classic_symbol_plan_store(out, emitter.counters); +} + +inline void j2k_classic_tier1_split_token_emit_init( + thread J2kClassicTier1SplitTokenEmitter &emitter, + device uchar *mq_token_data, + uint mq_token_capacity, + device uchar *raw_token_data, + uint raw_token_capacity, + device J2kClassicTier1TokenSegment *segments, + uint segment_capacity +) { + j2k_classic_symbol_plan_zero(emitter.counters); + j2k_classic_tier1_token_writer_init(emitter.mq_writer, mq_token_data, mq_token_capacity); + j2k_classic_tier1_token_writer_init(emitter.raw_writer, raw_token_data, raw_token_capacity); + emitter.segments = segments; + emitter.segment_capacity = segment_capacity; + emitter.current_segment_start_bit = 0u; + emitter.segment_failed = 0u; +} + +inline uint j2k_classic_tier1_split_token_emit_stream_bit_count( + thread J2kClassicTier1SplitTokenEmitter &emitter, + bool use_arithmetic +) { + return use_arithmetic ? emitter.mq_writer.bit_count : emitter.raw_writer.bit_count; +} + +inline void j2k_classic_tier1_split_token_emit_push_segment( + thread J2kClassicTier1SplitTokenEmitter &emitter, + uint start_pass, + uint end_pass, + bool use_arithmetic +) { + const uint segment_idx = emitter.counters.segment_count; + emitter.counters.segment_count += 1u; + if (segment_idx >= emitter.segment_capacity) { + emitter.segment_failed = 1u; + return; + } + const uint stream_bit_count = + j2k_classic_tier1_split_token_emit_stream_bit_count(emitter, use_arithmetic); + const uint token_bit_count = stream_bit_count - emitter.current_segment_start_bit; + emitter.segments[segment_idx].token_bit_offset = emitter.current_segment_start_bit; + emitter.segments[segment_idx].token_bit_count = token_bit_count; + emitter.segments[segment_idx].pass_range = + (start_pass & 0xFFFFu) | ((end_pass & 0xFFFFu) << 16u); + emitter.segments[segment_idx].flags = use_arithmetic ? 1u : 0u; +} + +inline void j2k_classic_tier1_split_token_emit_store( + device J2kClassicTier1SymbolPlanCounters *out, + thread J2kClassicTier1SplitTokenEmitter &emitter +) { + if (emitter.mq_writer.failed != 0u) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 23u; + } else if (emitter.raw_writer.failed != 0u) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 24u; + } else if (emitter.segment_failed != 0u) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 22u; + } + j2k_classic_symbol_plan_store(out, emitter.counters); +} + +inline void j2k_classic_tier1_split_mq_byte_token_emit_init( + thread J2kClassicTier1SplitMqByteRawTokenEmitter &emitter, + device uchar *mq_token_data, + uint mq_token_capacity, + device uchar *raw_token_data, + uint raw_token_capacity, + device J2kClassicTier1TokenSegment *segments, + uint segment_capacity +) { + j2k_classic_symbol_plan_zero(emitter.counters); + j2k_classic_tier1_token_byte_writer_init( + emitter.mq_writer, + mq_token_data, + mq_token_capacity + ); + j2k_classic_tier1_token_writer_init(emitter.raw_writer, raw_token_data, raw_token_capacity); + emitter.segments = segments; + emitter.segment_capacity = segment_capacity; + emitter.current_segment_start_bit = 0u; + emitter.segment_failed = 0u; +} + +inline uint j2k_classic_tier1_split_mq_byte_token_emit_stream_bit_count( + thread J2kClassicTier1SplitMqByteRawTokenEmitter &emitter, + bool use_arithmetic +) { + return use_arithmetic ? emitter.mq_writer.len * 8u : emitter.raw_writer.bit_count; +} + +inline void j2k_classic_tier1_split_mq_byte_token_emit_push_segment( + thread J2kClassicTier1SplitMqByteRawTokenEmitter &emitter, + uint start_pass, + uint end_pass, + bool use_arithmetic +) { + const uint segment_idx = emitter.counters.segment_count; + emitter.counters.segment_count += 1u; + if (segment_idx >= emitter.segment_capacity) { + emitter.segment_failed = 1u; + return; + } + const uint stream_bit_count = + j2k_classic_tier1_split_mq_byte_token_emit_stream_bit_count(emitter, use_arithmetic); + const uint token_bit_count = stream_bit_count - emitter.current_segment_start_bit; + emitter.segments[segment_idx].token_bit_offset = emitter.current_segment_start_bit; + emitter.segments[segment_idx].token_bit_count = token_bit_count; + emitter.segments[segment_idx].pass_range = + (start_pass & 0xFFFFu) | ((end_pass & 0xFFFFu) << 16u); + emitter.segments[segment_idx].flags = use_arithmetic ? 3u : 0u; +} + +inline void j2k_classic_tier1_split_mq_byte_token_emit_store( + device J2kClassicTier1SymbolPlanCounters *out, + thread J2kClassicTier1SplitMqByteRawTokenEmitter &emitter +) { + if (emitter.mq_writer.failed != 0u) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 25u; + } else if (emitter.raw_writer.failed != 0u) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 24u; + } else if (emitter.segment_failed != 0u) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 22u; + } + j2k_classic_symbol_plan_store(out, emitter.counters); +} + +inline void j2k_classic_tier1_token_reader_init( + thread J2kClassicTier1TokenBitReader &reader, + device const uchar *data, + uint capacity_bits +) { + reader.data = data; + reader.bit_pos = 0u; + reader.bit_capacity = capacity_bits; + reader.failed = 0u; +} + +inline void j2k_classic_tier1_token_reader_seek( + thread J2kClassicTier1TokenBitReader &reader, + uint bit_pos +) { + if (bit_pos > reader.bit_capacity) { + reader.failed = 1u; + return; + } + reader.bit_pos = bit_pos; +} + +inline uint j2k_classic_tier1_token_reader_read_bits( + thread J2kClassicTier1TokenBitReader &reader, + uint count +) { + if (count > 32u || reader.bit_pos > reader.bit_capacity || + count > (reader.bit_capacity - reader.bit_pos)) { + reader.failed = 1u; + return 0u; + } + uint value = 0u; + for (uint idx = 0u; idx < count; ++idx) { + const uint byte_idx = reader.bit_pos >> 3u; + const uint shift = 7u - (reader.bit_pos & 7u); + value = (value << 1u) | ((uint(reader.data[byte_idx]) >> shift) & 1u); + reader.bit_pos += 1u; + } + return value; +} + +inline void j2k_pack_classic_tier1_tokens_bypass_u16_32_impl( + J2kClassicEncodeParams params, + J2kClassicTier1SymbolPlanCounters counters, + device const uchar *token_data, + device const J2kClassicTier1TokenSegment *token_segments, + uint token_capacity_bytes, + uint token_segment_capacity, + device uchar *out, + device J2kClassicEncodeStatus *status, + device J2kClassicSegment *segments +) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 30u, 0u, 0u, 0u, 0u); + + if (params.style_flags != J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_UNSUPPORTED, + 2u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + if (counters.code != J2K_ENCODE_STATUS_OK) { + j2k_set_encode_status( + status, + counters.code, + counters.detail, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + if (counters.segment_count > token_segment_capacity || + counters.segment_count > params.segment_capacity) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 31u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + + const uint token_capacity_bits = token_capacity_bytes * 8u; + thread J2kClassicTier1TokenBitReader reader; + j2k_classic_tier1_token_reader_init(reader, token_data, token_capacity_bits); + thread uchar contexts[19]; + reset_contexts(contexts); + + uint data_cursor = 0u; + uint segment_count = 0u; + for (uint segment_idx = 0u; segment_idx < counters.segment_count; ++segment_idx) { + const J2kClassicTier1TokenSegment token_segment = token_segments[segment_idx]; + const uint start_pass = token_segment.pass_range & 0xFFFFu; + const uint end_pass = token_segment.pass_range >> 16u; + const bool use_arithmetic = (token_segment.flags & 1u) != 0u; + if ((token_segment.flags & ~1u) != 0u || + start_pass > end_pass || + end_pass > counters.coding_passes) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 32u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + if (token_segment.token_bit_offset > token_capacity_bits || + token_segment.token_bit_count > (token_capacity_bits - token_segment.token_bit_offset)) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 33u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + if (data_cursor > params.output_capacity) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 34u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + j2k_classic_tier1_token_reader_seek(reader, token_segment.token_bit_offset); + if (reader.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 35u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + + uint segment_len = 0u; + if (use_arithmetic) { + if ((token_segment.token_bit_count % 6u) != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 36u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + thread J2kMqEncoder encoder; + j2k_mq_init(encoder, out + data_cursor, params.output_capacity - data_cursor); + const uint symbol_count = token_segment.token_bit_count / 6u; + for (uint symbol_idx = 0u; symbol_idx < symbol_count; ++symbol_idx) { + const uint token = j2k_classic_tier1_token_reader_read_bits(reader, 6u); + if (reader.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 37u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + const uint ctx_label = token & 0x1Fu; + if (ctx_label >= 19u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 38u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + j2k_mq_encode(encoder, contexts, ctx_label, (token >> 5u) & 1u); + if (encoder.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 39u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + } + segment_len = j2k_classic_finish_arithmetic_segment(encoder); + if (encoder.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 40u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + } else { + thread J2kRawBitWriter writer; + j2k_raw_writer_init(writer, out + data_cursor, params.output_capacity - data_cursor); + for (uint bit_idx = 0u; bit_idx < token_segment.token_bit_count; ++bit_idx) { + const uint bit = j2k_classic_tier1_token_reader_read_bits(reader, 1u); + if (reader.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 41u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + j2k_raw_writer_write_bit(writer, bit); + if (writer.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 42u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + } + j2k_raw_writer_finish(writer); + if (writer.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 43u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + segment_len = writer.len; + } + + if (segment_len > (params.output_capacity - data_cursor)) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 44u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + if (!j2k_classic_push_segment( + segments, + params.segment_capacity, + segment_count, + data_cursor, + segment_len, + start_pass, + end_pass, + use_arithmetic)) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 45u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + data_cursor += segment_len; + } + + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_OK, + 0u, + data_cursor, + counters.coding_passes, + counters.missing_bit_planes, + segment_count + ); +} + +inline void j2k_pack_classic_tier1_split_tokens_bypass_u16_32_impl( + J2kClassicEncodeParams params, + J2kClassicTier1SymbolPlanCounters counters, + device const uchar *mq_token_data, + device const uchar *raw_token_data, + device const J2kClassicTier1TokenSegment *token_segments, + uint mq_token_capacity_bytes, + uint raw_token_capacity_bytes, + uint token_segment_capacity, + device uchar *out, + device J2kClassicEncodeStatus *status, + device J2kClassicSegment *segments +) { + j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 70u, 0u, 0u, 0u, 0u); + + if (params.style_flags != J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_UNSUPPORTED, + 2u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + if (counters.code != J2K_ENCODE_STATUS_OK) { + j2k_set_encode_status( + status, + counters.code, + counters.detail, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + if (counters.segment_count > token_segment_capacity || + counters.segment_count > params.segment_capacity) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 71u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + + const uint mq_token_capacity_bits = mq_token_capacity_bytes * 8u; + const uint raw_token_capacity_bits = raw_token_capacity_bytes * 8u; + thread J2kClassicTier1TokenBitReader mq_reader; + thread J2kClassicTier1TokenBitReader raw_reader; + j2k_classic_tier1_token_reader_init(mq_reader, mq_token_data, mq_token_capacity_bits); + j2k_classic_tier1_token_reader_init(raw_reader, raw_token_data, raw_token_capacity_bits); + thread uchar contexts[19]; + reset_contexts(contexts); + + uint data_cursor = 0u; + uint segment_count = 0u; + for (uint segment_idx = 0u; segment_idx < counters.segment_count; ++segment_idx) { + const J2kClassicTier1TokenSegment token_segment = token_segments[segment_idx]; + const uint start_pass = token_segment.pass_range & 0xFFFFu; + const uint end_pass = token_segment.pass_range >> 16u; + const bool use_arithmetic = (token_segment.flags & 1u) != 0u; + const bool mq_tokens_are_bytes = (token_segment.flags & 2u) != 0u; + const uint token_capacity_bits = + use_arithmetic ? mq_token_capacity_bits : raw_token_capacity_bits; + if ((token_segment.flags & ~3u) != 0u || + (!use_arithmetic && mq_tokens_are_bytes) || + start_pass > end_pass || + end_pass > counters.coding_passes) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 72u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + if (token_segment.token_bit_offset > token_capacity_bits || + token_segment.token_bit_count > (token_capacity_bits - token_segment.token_bit_offset)) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 73u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + if (data_cursor > params.output_capacity) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 74u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + + uint segment_len = 0u; + if (use_arithmetic) { + if (!mq_tokens_are_bytes && (token_segment.token_bit_count % 6u) != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 75u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + if (mq_tokens_are_bytes && + ((token_segment.token_bit_offset | token_segment.token_bit_count) & 7u) != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 75u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + if (!mq_tokens_are_bytes) { + j2k_classic_tier1_token_reader_seek(mq_reader, token_segment.token_bit_offset); + } + if (!mq_tokens_are_bytes && mq_reader.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 76u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + thread J2kMqEncoder encoder; + j2k_mq_init(encoder, out + data_cursor, params.output_capacity - data_cursor); + const uint symbol_count = + mq_tokens_are_bytes ? (token_segment.token_bit_count >> 3u) : + (token_segment.token_bit_count / 6u); + const uint mq_byte_offset = token_segment.token_bit_offset >> 3u; + for (uint symbol_idx = 0u; symbol_idx < symbol_count; ++symbol_idx) { + const uint token = mq_tokens_are_bytes + ? uint(mq_token_data[mq_byte_offset + symbol_idx]) + : j2k_classic_tier1_token_reader_read_bits(mq_reader, 6u); + if (!mq_tokens_are_bytes && mq_reader.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 77u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + const uint ctx_label = token & 0x1Fu; + if (ctx_label >= 19u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 78u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + j2k_mq_encode(encoder, contexts, ctx_label, (token >> 5u) & 1u); + if (encoder.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 79u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + } + segment_len = j2k_classic_finish_arithmetic_segment(encoder); + if (encoder.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 80u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + } else { + j2k_classic_tier1_token_reader_seek(raw_reader, token_segment.token_bit_offset); + if (raw_reader.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 81u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + thread J2kRawBitWriter writer; + j2k_raw_writer_init(writer, out + data_cursor, params.output_capacity - data_cursor); + for (uint bit_idx = 0u; bit_idx < token_segment.token_bit_count; ++bit_idx) { + const uint bit = j2k_classic_tier1_token_reader_read_bits(raw_reader, 1u); + if (raw_reader.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 82u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + j2k_raw_writer_write_bit(writer, bit); + if (writer.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 83u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + } + j2k_raw_writer_finish(writer); + if (writer.failed != 0u) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 84u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + segment_len = writer.len; + } + + if (segment_len > (params.output_capacity - data_cursor)) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 85u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + if (!j2k_classic_push_segment( + segments, + params.segment_capacity, + segment_count, + data_cursor, + segment_len, + start_pass, + end_pass, + use_arithmetic)) { + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_FAIL, + 86u, + 0u, + counters.coding_passes, + counters.missing_bit_planes, + 0u + ); + return; + } + data_cursor += segment_len; + } + + j2k_set_encode_status( + status, + J2K_ENCODE_STATUS_OK, + 0u, + data_cursor, + counters.coding_passes, + counters.missing_bit_planes, + segment_count + ); +} + +inline void j2k_classic_symbol_plan_record_mq( + thread J2kClassicTier1SymbolPlanCounters &counters, + uint ctx_label, + uint bit, + uint pass_type +) { + counters.mq_symbol_count += 1u; + switch (pass_type) { + case 0u: + counters.cleanup_mq_symbol_count += 1u; + break; + case 1u: + counters.sigprop_mq_symbol_count += 1u; + break; + default: + counters.magref_mq_symbol_count += 1u; + break; + } + const uint packed = (ctx_label & 0x1Fu) | ((bit & 1u) << 5u) | ((pass_type & 3u) << 6u); + counters.mq_symbol_hash = (counters.mq_symbol_hash ^ packed) * 16777619u; +} + +inline void j2k_classic_symbol_plan_record_mq( + thread J2kClassicTier1TokenEmitter &emitter, + uint ctx_label, + uint bit, + uint pass_type +) { + emitter.counters.mq_symbol_count += 1u; + switch (pass_type) { + case 0u: + emitter.counters.cleanup_mq_symbol_count += 1u; + break; + case 1u: + emitter.counters.sigprop_mq_symbol_count += 1u; + break; + default: + emitter.counters.magref_mq_symbol_count += 1u; + break; + } + const uint packed_hash = + (ctx_label & 0x1Fu) | ((bit & 1u) << 5u) | ((pass_type & 3u) << 6u); + emitter.counters.mq_symbol_hash = + (emitter.counters.mq_symbol_hash ^ packed_hash) * 16777619u; + const uint packed_token = (ctx_label & 0x1Fu) | ((bit & 1u) << 5u); + j2k_classic_tier1_token_writer_write_bits(emitter.writer, packed_token, 6u); +} + +inline void j2k_classic_symbol_plan_record_mq( + thread J2kClassicTier1SplitTokenEmitter &emitter, + uint ctx_label, + uint bit, + uint pass_type +) { + emitter.counters.mq_symbol_count += 1u; + switch (pass_type) { + case 0u: + emitter.counters.cleanup_mq_symbol_count += 1u; + break; + case 1u: + emitter.counters.sigprop_mq_symbol_count += 1u; + break; + default: + emitter.counters.magref_mq_symbol_count += 1u; + break; + } + const uint packed_hash = + (ctx_label & 0x1Fu) | ((bit & 1u) << 5u) | ((pass_type & 3u) << 6u); + emitter.counters.mq_symbol_hash = + (emitter.counters.mq_symbol_hash ^ packed_hash) * 16777619u; + const uint packed_token = (ctx_label & 0x1Fu) | ((bit & 1u) << 5u); + j2k_classic_tier1_token_writer_write_bits(emitter.mq_writer, packed_token, 6u); +} + +inline void j2k_classic_symbol_plan_record_mq( + thread J2kClassicTier1SplitMqByteRawTokenEmitter &emitter, + uint ctx_label, + uint bit, + uint pass_type +) { + emitter.counters.mq_symbol_count += 1u; + switch (pass_type) { + case 0u: + emitter.counters.cleanup_mq_symbol_count += 1u; + break; + case 1u: + emitter.counters.sigprop_mq_symbol_count += 1u; + break; + default: + emitter.counters.magref_mq_symbol_count += 1u; + break; + } + const uint packed_hash = + (ctx_label & 0x1Fu) | ((bit & 1u) << 5u) | ((pass_type & 3u) << 6u); + emitter.counters.mq_symbol_hash = + (emitter.counters.mq_symbol_hash ^ packed_hash) * 16777619u; + const uint packed_token = (ctx_label & 0x1Fu) | ((bit & 1u) << 5u); + j2k_classic_tier1_token_byte_writer_push(emitter.mq_writer, packed_token); +} + +inline void j2k_classic_pass_plan_record_mq_symbol( + thread J2kClassicTier1PassPlanEmitter &emitter +) { + emitter.counters.mq_symbol_count += 1u; + if (emitter.current_pass >= J2K_CLASSIC_TIER1_PASS_PLAN_CAPACITY) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 60u; + return; + } + thread uint &pass_count = emitter.counters.mq_symbols_by_pass[emitter.current_pass]; + if (pass_count == 0u) { + emitter.counters.nonempty_mq_passes += 1u; + } + pass_count += 1u; + emitter.counters.max_mq_symbols_per_pass = + max(emitter.counters.max_mq_symbols_per_pass, pass_count); +} + +inline void j2k_classic_symbol_plan_record_mq( + thread J2kClassicTier1PassPlanEmitter &emitter, + uint ctx_label, + uint bit, + uint pass_type +) { + (void)ctx_label; + (void)bit; + (void)pass_type; + j2k_classic_pass_plan_record_mq_symbol(emitter); +} + +inline void j2k_classic_symbol_plan_record_raw( + thread J2kClassicTier1SymbolPlanCounters &counters, + uint bit, + uint pass_type +) { + counters.raw_bit_count += 1u; + if (pass_type == 1u) { + counters.raw_sigprop_bit_count += 1u; + } else { + counters.raw_magref_bit_count += 1u; + } + const uint packed = (bit & 1u) | ((pass_type & 3u) << 1u); + counters.raw_bit_hash = (counters.raw_bit_hash ^ packed) * 16777619u; +} + +inline void j2k_classic_symbol_plan_record_raw( + thread J2kClassicTier1TokenEmitter &emitter, + uint bit, + uint pass_type +) { + emitter.counters.raw_bit_count += 1u; + if (pass_type == 1u) { + emitter.counters.raw_sigprop_bit_count += 1u; + } else { + emitter.counters.raw_magref_bit_count += 1u; + } + const uint packed = (bit & 1u) | ((pass_type & 3u) << 1u); + emitter.counters.raw_bit_hash = (emitter.counters.raw_bit_hash ^ packed) * 16777619u; + j2k_classic_tier1_token_writer_write_bit(emitter.writer, bit); +} + +inline void j2k_classic_symbol_plan_record_raw( + thread J2kClassicTier1SplitTokenEmitter &emitter, + uint bit, + uint pass_type +) { + emitter.counters.raw_bit_count += 1u; + if (pass_type == 1u) { + emitter.counters.raw_sigprop_bit_count += 1u; + } else { + emitter.counters.raw_magref_bit_count += 1u; + } + const uint packed = (bit & 1u) | ((pass_type & 3u) << 1u); + emitter.counters.raw_bit_hash = (emitter.counters.raw_bit_hash ^ packed) * 16777619u; + j2k_classic_tier1_token_writer_write_bit(emitter.raw_writer, bit); +} + +inline void j2k_classic_symbol_plan_record_raw( + thread J2kClassicTier1SplitMqByteRawTokenEmitter &emitter, + uint bit, + uint pass_type +) { + emitter.counters.raw_bit_count += 1u; + if (pass_type == 1u) { + emitter.counters.raw_sigprop_bit_count += 1u; + } else { + emitter.counters.raw_magref_bit_count += 1u; + } + const uint packed = (bit & 1u) | ((pass_type & 3u) << 1u); + emitter.counters.raw_bit_hash = (emitter.counters.raw_bit_hash ^ packed) * 16777619u; + j2k_classic_tier1_token_writer_write_bit(emitter.raw_writer, bit); +} + +inline void j2k_classic_symbol_plan_record_raw( + thread J2kClassicTier1PassPlanEmitter &emitter, + uint bit, + uint pass_type +) { + (void)bit; + (void)pass_type; + emitter.counters.raw_bit_count += 1u; + if (emitter.current_pass >= J2K_CLASSIC_TIER1_PASS_PLAN_CAPACITY) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 60u; + return; + } + thread uint &pass_count = emitter.counters.raw_bits_by_pass[emitter.current_pass]; + if (pass_count == 0u) { + emitter.counters.nonempty_raw_passes += 1u; + } + pass_count += 1u; + emitter.counters.max_raw_bits_per_pass = + max(emitter.counters.max_raw_bits_per_pass, pass_count); +} + +inline void j2k_classic_symbol_plan_record_sign_count( + thread J2kClassicTier1SymbolPlanCounters &counters, + uint pass_type +) { + if (pass_type == 0u) { + counters.cleanup_sign_symbol_count += 1u; + } else { + counters.sigprop_sign_symbol_count += 1u; + } +} + +inline void j2k_classic_symbol_plan_record_sign_count( + thread J2kClassicTier1TokenEmitter &emitter, + uint pass_type +) { + if (pass_type == 0u) { + emitter.counters.cleanup_sign_symbol_count += 1u; + } else { + emitter.counters.sigprop_sign_symbol_count += 1u; + } +} + +inline void j2k_classic_symbol_plan_record_sign_count( + thread J2kClassicTier1SplitTokenEmitter &emitter, + uint pass_type +) { + if (pass_type == 0u) { + emitter.counters.cleanup_sign_symbol_count += 1u; + } else { + emitter.counters.sigprop_sign_symbol_count += 1u; + } +} + +inline void j2k_classic_symbol_plan_record_sign_count( + thread J2kClassicTier1SplitMqByteRawTokenEmitter &emitter, + uint pass_type +) { + if (pass_type == 0u) { + emitter.counters.cleanup_sign_symbol_count += 1u; + } else { + emitter.counters.sigprop_sign_symbol_count += 1u; + } +} + +inline void j2k_classic_symbol_plan_record_sign_count( + thread J2kClassicTier1PassPlanEmitter &emitter, + uint pass_type +) { + (void)emitter; + (void)pass_type; +} + +template +inline void j2k_classic_symbol_plan_record_sign( + uint idx, + thread const uchar *states, + thread Recorder &counters, + uint padded_width, + uint index_x, + uint index_y, + uint height, + uint style_flags, + uchar neighbor_sig, + uint pass_type +) { + const uchar significances = neighbor_sig & uchar(0b01010101); + const uint left_sign = coeff_sign(states, coeff_index(padded_width, index_x - 1u, index_y)); + const uint right_sign = coeff_sign(states, coeff_index(padded_width, index_x + 1u, index_y)); + const uint top_sign = coeff_sign(states, coeff_index(padded_width, index_x, index_y - 1u)); + const uint bottom_sign = + ((style_flags & J2K_CLASSIC_STYLE_VERTICALLY_CAUSAL_CONTEXT) != 0u && + neighbor_in_next_stripe(index_y, height)) + ? 0u + : coeff_sign(states, coeff_index(padded_width, index_x, index_y + 1u)); + const uchar signs = uchar((top_sign << 6u) | (left_sign << 4u) | (right_sign << 2u) | bottom_sign); + const uchar negative = significances & signs; + const uchar positive = significances & uchar(~signs); + const uchar2 sign_ctx = SIGN_CONTEXT_LOOKUP[uchar((negative << 1u) | positive)]; + const uint sign_bit = (uint(states[idx]) >> 5u) & 1u; + j2k_classic_symbol_plan_record_sign_count(counters, pass_type); + j2k_classic_symbol_plan_record_mq( + counters, + uint(sign_ctx.x), + sign_bit ^ uint(sign_ctx.y), + pass_type + ); +} + +inline void j2k_classic_profile_significance_density( + thread const ushort *magnitudes, + thread uchar *states, + uchar coded_marker, + uint width, + uint height, + uint padded_width, + uint bit_mask, + uint style_flags, + bool use_arithmetic, + thread J2kClassicTier1DensityCounters &counters +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + const uchar neighbor_sig = + j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && neighbor_sig != 0u) { + counters.sigprop_active_candidates += 1u; + if (use_arithmetic) { + counters.arithmetic_sigprop_active_candidates += 1u; + } else { + counters.raw_sigprop_active_candidates += 1u; + } + j2k_classic_set_coded_marker(states, idx, coded_marker); + if ((magnitudes[idx] & ushort(bit_mask)) != 0u) { + counters.sigprop_new_significant += 1u; + if (use_arithmetic) { + counters.arithmetic_sigprop_new_significant += 1u; + } else { + counters.raw_sigprop_new_significant += 1u; + } + set_significant(states, padded_width, ix, iy); + } + } + } + } + } +} + +inline void j2k_classic_profile_magnitude_density( + thread uchar *states, + uchar coded_marker, + uint width, + uint height, + uint padded_width, + bool use_arithmetic, + thread J2kClassicTier1DensityCounters &counters +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + for (uint y = y_base; y < y_end; ++y) { + const uint idx = coeff_index(padded_width, x + 1u, y + 1u); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) != 0u && + !j2k_classic_coded_marker_matches(states, idx, coded_marker)) { + counters.magref_active_candidates += 1u; + if (use_arithmetic) { + counters.arithmetic_magref_active_candidates += 1u; + } else { + counters.raw_magref_active_candidates += 1u; + } + states[idx] |= J2K_ENCODE_MAGNITUDE_REFINED; + } + } + } + } +} + +inline void j2k_classic_profile_cleanup_density( + thread const ushort *magnitudes, + thread uchar *states, + uchar coded_marker, + uint width, + uint height, + uint padded_width, + uint bit_mask, + uint style_flags, + thread J2kClassicTier1DensityCounters &counters +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + const uint stripe_height = y_end - y_base; + + if (stripe_height == 4u) { + bool all_zero_uncoded = true; + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + const uchar neighbor_sig = + j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) != 0u || + j2k_classic_coded_marker_matches(states, idx, coded_marker) || + neighbor_sig != 0u) { + all_zero_uncoded = false; + break; + } + } + + if (all_zero_uncoded) { + counters.cleanup_rlc_stripes += 1u; + uint first_sig = 4u; + for (uint pos = 0u; pos < 4u; ++pos) { + const uint idx = coeff_index(padded_width, x + 1u, y_base + pos + 1u); + if ((magnitudes[idx] & ushort(bit_mask)) != 0u) { + first_sig = pos; + break; + } + } + + if (first_sig < 4u) { + counters.cleanup_new_significant += 1u; + const uint sig_y = y_base + first_sig; + set_significant(states, padded_width, x + 1u, sig_y + 1u); + + for (uint y = sig_y + 1u; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && + !j2k_classic_coded_marker_matches(states, idx, coded_marker)) { + counters.cleanup_active_candidates += 1u; + if ((magnitudes[idx] & ushort(bit_mask)) != 0u) { + counters.cleanup_new_significant += 1u; + set_significant(states, padded_width, ix, iy); + } + } + } + continue; + } + + counters.cleanup_rlc_zero_stripes += 1u; + continue; + } + } + + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && + !j2k_classic_coded_marker_matches(states, idx, coded_marker)) { + counters.cleanup_active_candidates += 1u; + if ((magnitudes[idx] & ushort(bit_mask)) != 0u) { + counters.cleanup_new_significant += 1u; + set_significant(states, padded_width, ix, iy); + } + } + } + } + } +} + +template +inline void j2k_classic_symbol_plan_significance_pass( + thread const ushort *magnitudes, + thread uchar *states, + uchar coded_marker, + uint width, + uint height, + uint padded_width, + uint bit_mask, + uint sub_band_type, + uint style_flags, + thread Recorder &counters +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + const uchar neighbor_sig = + j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && neighbor_sig != 0u) { + const uint ctx_label = uint(zero_context_label(neighbor_sig, sub_band_type)); + const uint bit = (magnitudes[idx] & ushort(bit_mask)) != 0u ? 1u : 0u; + j2k_classic_symbol_plan_record_mq(counters, ctx_label, bit, 1u); + j2k_classic_set_coded_marker(states, idx, coded_marker); + if (bit != 0u) { + j2k_classic_symbol_plan_record_sign( + idx, + states, + counters, + padded_width, + ix, + iy, + height, + style_flags, + neighbor_sig, + 1u + ); + set_significant(states, padded_width, ix, iy); + } + } + } + } + } +} + +template +inline void j2k_classic_symbol_plan_magnitude_pass( + thread const ushort *magnitudes, + thread uchar *states, + uchar coded_marker, + uint width, + uint height, + uint padded_width, + uint bit_mask, + uint style_flags, + thread Recorder &counters +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) != 0u && + !j2k_classic_coded_marker_matches(states, idx, coded_marker)) { + const uint ctx_label = + uint(magnitude_refinement_context(states, padded_width, ix, iy, height, style_flags)); + const uint bit = (magnitudes[idx] & ushort(bit_mask)) != 0u ? 1u : 0u; + j2k_classic_symbol_plan_record_mq(counters, ctx_label, bit, 2u); + states[idx] |= J2K_ENCODE_MAGNITUDE_REFINED; + } + } + } + } +} + +template +inline void j2k_classic_symbol_plan_significance_pass_raw( + thread const ushort *magnitudes, + thread uchar *states, + uchar coded_marker, + uint width, + uint height, + uint padded_width, + uint bit_mask, + uint style_flags, + thread Recorder &counters +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + const uchar neighbor_sig = + j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && neighbor_sig != 0u) { + const uint bit = (magnitudes[idx] & ushort(bit_mask)) != 0u ? 1u : 0u; + j2k_classic_symbol_plan_record_raw(counters, bit, 1u); + j2k_classic_set_coded_marker(states, idx, coded_marker); + if (bit != 0u) { + j2k_classic_symbol_plan_record_raw(counters, (uint(states[idx]) >> 5u) & 1u, 1u); + set_significant(states, padded_width, ix, iy); + } + } + } + } + } +} + +template +inline void j2k_classic_symbol_plan_magnitude_pass_raw( + thread const ushort *magnitudes, + thread uchar *states, + uchar coded_marker, + uint width, + uint height, + uint padded_width, + uint bit_mask, + thread Recorder &counters +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) != 0u && + !j2k_classic_coded_marker_matches(states, idx, coded_marker)) { + const uint bit = (magnitudes[idx] & ushort(bit_mask)) != 0u ? 1u : 0u; + j2k_classic_symbol_plan_record_raw(counters, bit, 2u); + states[idx] |= J2K_ENCODE_MAGNITUDE_REFINED; + } + } + } + } +} + +template +inline void j2k_classic_symbol_plan_cleanup_pass( + thread const ushort *magnitudes, + thread uchar *states, + uchar coded_marker, + uint width, + uint height, + uint padded_width, + uint bit_mask, + uint sub_band_type, + uint style_flags, + thread Recorder &counters +) { + for (uint y_base = 0u; y_base < height; y_base += 4u) { + for (uint x = 0u; x < width; ++x) { + const uint y_end = min(y_base + 4u, height); + const uint stripe_height = y_end - y_base; + + if (stripe_height == 4u) { + bool all_zero_uncoded = true; + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + const uchar neighbor_sig = + j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) != 0u || + j2k_classic_coded_marker_matches(states, idx, coded_marker) || + neighbor_sig != 0u) { + all_zero_uncoded = false; + break; + } + } + + if (all_zero_uncoded) { + uint first_sig = 4u; + for (uint pos = 0u; pos < 4u; ++pos) { + const uint idx = coeff_index(padded_width, x + 1u, y_base + pos + 1u); + if ((magnitudes[idx] & ushort(bit_mask)) != 0u) { + first_sig = pos; + break; + } + } + + if (first_sig < 4u) { + j2k_classic_symbol_plan_record_mq(counters, 17u, 1u, 0u); + j2k_classic_symbol_plan_record_mq(counters, 18u, (first_sig >> 1u) & 1u, 0u); + j2k_classic_symbol_plan_record_mq(counters, 18u, first_sig & 1u, 0u); + + const uint sig_y = y_base + first_sig; + const uint sig_idx = coeff_index(padded_width, x + 1u, sig_y + 1u); + j2k_classic_symbol_plan_record_sign( + sig_idx, + states, + counters, + padded_width, + x + 1u, + sig_y + 1u, + height, + style_flags, + uchar(0u), + 0u + ); + set_significant(states, padded_width, x + 1u, sig_y + 1u); + + for (uint y = sig_y + 1u; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && + !j2k_classic_coded_marker_matches(states, idx, coded_marker)) { + const uchar neighbor_sig = + j2k_classic_effective_neighbors( + states, + padded_width, + ix, + iy, + height, + style_flags + ); + const uint ctx_label = uint(zero_context_label(neighbor_sig, sub_band_type)); + const uint bit = (magnitudes[idx] & ushort(bit_mask)) != 0u ? 1u : 0u; + j2k_classic_symbol_plan_record_mq(counters, ctx_label, bit, 0u); + if (bit != 0u) { + j2k_classic_symbol_plan_record_sign( + idx, + states, + counters, + padded_width, + ix, + iy, + height, + style_flags, + neighbor_sig, + 0u + ); + set_significant(states, padded_width, ix, iy); + } + } + } + continue; + } + + j2k_classic_symbol_plan_record_mq(counters, 17u, 0u, 0u); + continue; + } + } + + for (uint y = y_base; y < y_end; ++y) { + const uint ix = x + 1u; + const uint iy = y + 1u; + const uint idx = coeff_index(padded_width, ix, iy); + if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && + !j2k_classic_coded_marker_matches(states, idx, coded_marker)) { + const uchar neighbor_sig = + j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); + const uint ctx_label = uint(zero_context_label(neighbor_sig, sub_band_type)); + const uint bit = (magnitudes[idx] & ushort(bit_mask)) != 0u ? 1u : 0u; + j2k_classic_symbol_plan_record_mq(counters, ctx_label, bit, 0u); + if (bit != 0u) { + j2k_classic_symbol_plan_record_sign( + idx, + states, + counters, + padded_width, + ix, + iy, + height, + style_flags, + neighbor_sig, + 0u + ); + set_significant(states, padded_width, ix, iy); + } + } + } + } + } +} + +inline void j2k_profile_classic_tier1_density_bypass_u16_32_impl( + device const int *coefficients, + J2kClassicEncodeParams params, + device J2kClassicTier1DensityCounters *out +) { + thread J2kClassicTier1DensityCounters counters; + j2k_classic_tier1_density_zero(counters); + + if (params.width == 0u || params.height == 0u || + params.width > J2K_CLASSIC_ENCODE_32_MAX_WIDTH || + params.height > J2K_CLASSIC_ENCODE_32_MAX_HEIGHT || + params.total_bitplanes > 16u) { + j2k_classic_tier1_density_store(out, counters); + return; + } + + thread ushort magnitudes[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + const uint padded_width = params.width + 2u; + j2k_classic_clear_state_border(states, padded_width, params.width, params.height); + + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint src_idx = y * params.width + x; + const int value = coefficients[src_idx]; + const uint dst_idx = coeff_index(padded_width, x + 1u, y + 1u); + const uint magnitude = j2k_classic_magnitude(value); + magnitudes[dst_idx] = ushort(magnitude); + states[dst_idx] = value < 0 ? J2K_ENCODE_SIGN : uchar(0u); + max_magnitude = max(max_magnitude, magnitude); + } + } + + if (max_magnitude == 0u) { + j2k_classic_tier1_density_store(out, counters); + return; + } + + const uint num_bitplanes = 32u - clz(max_magnitude); + if (num_bitplanes > params.total_bitplanes) { + j2k_classic_tier1_density_store(out, counters); + return; + } + + uchar coded_marker = uchar(1u); + const uint total_passes = 1u + 3u * (num_bitplanes - 1u); + for (uint coding_pass = 0u; coding_pass < total_passes; ++coding_pass) { + const uint current_bitplane = (coding_pass + 2u) / 3u; + const uint bit_mask = 1u << (num_bitplanes - 1u - current_bitplane); + const bool use_arithmetic = + (params.style_flags & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) == 0u || + coding_pass <= 9u || + (coding_pass % 3u) == 0u; + switch (coding_pass % 3u) { + case 0u: + j2k_classic_profile_cleanup_density( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.style_flags, + counters + ); + coded_marker += uchar(1u); + break; + case 1u: + j2k_classic_profile_significance_density( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.style_flags, + use_arithmetic, + counters + ); + break; + default: + j2k_classic_profile_magnitude_density( + states, + coded_marker, + params.width, + params.height, + padded_width, + use_arithmetic, + counters + ); + break; + } + } + + j2k_classic_tier1_density_store(out, counters); +} + +inline void j2k_plan_classic_tier1_symbols_bypass_u16_32_impl( + device const int *coefficients, + J2kClassicEncodeParams params, + device J2kClassicTier1SymbolPlanCounters *out +) { + thread J2kClassicTier1SymbolPlanCounters counters; + j2k_classic_symbol_plan_zero(counters); + + if (params.width == 0u || params.height == 0u || + params.width > J2K_CLASSIC_ENCODE_32_MAX_WIDTH || + params.height > J2K_CLASSIC_ENCODE_32_MAX_HEIGHT || + params.total_bitplanes > 16u) { + counters.code = J2K_ENCODE_STATUS_UNSUPPORTED; + counters.detail = 1u; + j2k_classic_symbol_plan_store(out, counters); + return; + } + if (params.style_flags != J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) { + counters.code = J2K_ENCODE_STATUS_UNSUPPORTED; + counters.detail = 2u; + j2k_classic_symbol_plan_store(out, counters); + return; + } + + thread ushort magnitudes[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + const uint padded_width = params.width + 2u; + j2k_classic_clear_state_border(states, padded_width, params.width, params.height); + + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint src_idx = y * params.width + x; + const int value = coefficients[src_idx]; + const uint dst_idx = coeff_index(padded_width, x + 1u, y + 1u); + const uint magnitude = j2k_classic_magnitude(value); + magnitudes[dst_idx] = ushort(magnitude); + states[dst_idx] = value < 0 ? J2K_ENCODE_SIGN : uchar(0u); + max_magnitude = max(max_magnitude, magnitude); + } + } + + if (max_magnitude == 0u) { + counters.missing_bit_planes = params.total_bitplanes; + j2k_classic_symbol_plan_store(out, counters); + return; + } + + const uint num_bitplanes = 32u - clz(max_magnitude); + if (num_bitplanes > params.total_bitplanes) { + counters.code = J2K_ENCODE_STATUS_FAIL; + counters.detail = 3u; + j2k_classic_symbol_plan_store(out, counters); + return; + } + counters.missing_bit_planes = params.total_bitplanes - num_bitplanes; + + uchar coded_marker = uchar(1u); + const uint total_passes = 1u + 3u * (num_bitplanes - 1u); + counters.coding_passes = total_passes; + uint current_segment_idx = 0xFFFFFFFFu; + for (uint coding_pass = 0u; coding_pass < total_passes; ++coding_pass) { + const uint segment_idx = j2k_classic_bypass_segment_idx(coding_pass); + if (current_segment_idx != segment_idx) { + current_segment_idx = segment_idx; + counters.segment_count += 1u; + } + + const uint pass_type = coding_pass % 3u; + const bool use_arithmetic = coding_pass <= 9u || pass_type == 0u; + const uint current_bitplane = (coding_pass + 2u) / 3u; + const uint bit_mask = 1u << (num_bitplanes - 1u - current_bitplane); + switch (pass_type) { + case 0u: + j2k_classic_symbol_plan_cleanup_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u, + counters + ); + coded_marker += uchar(1u); + break; + case 1u: + if (use_arithmetic) { + j2k_classic_symbol_plan_significance_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u, + counters + ); + } else { + j2k_classic_symbol_plan_significance_pass_raw( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + 0u, + counters + ); + } + break; + default: + if (use_arithmetic) { + j2k_classic_symbol_plan_magnitude_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + 0u, + counters + ); + } else { + j2k_classic_symbol_plan_magnitude_pass_raw( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + counters + ); + } + break; + } + } + + j2k_classic_symbol_plan_store(out, counters); +} + +inline void j2k_plan_classic_tier1_passes_bypass_u16_32_impl( + device const int *coefficients, + J2kClassicEncodeParams params, + device J2kClassicTier1PassPlanCounters *out +) { + thread J2kClassicTier1PassPlanEmitter emitter; + j2k_classic_pass_plan_zero(emitter.counters); + emitter.current_pass = 0u; + + if (params.width == 0u || params.height == 0u || + params.width > J2K_CLASSIC_ENCODE_32_MAX_WIDTH || + params.height > J2K_CLASSIC_ENCODE_32_MAX_HEIGHT || + params.total_bitplanes > 16u) { + emitter.counters.code = J2K_ENCODE_STATUS_UNSUPPORTED; + emitter.counters.detail = 1u; + j2k_classic_pass_plan_store(out, emitter.counters); + return; + } + if (params.style_flags != J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) { + emitter.counters.code = J2K_ENCODE_STATUS_UNSUPPORTED; + emitter.counters.detail = 2u; + j2k_classic_pass_plan_store(out, emitter.counters); + return; + } + + thread ushort magnitudes[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + const uint padded_width = params.width + 2u; + j2k_classic_clear_state_border(states, padded_width, params.width, params.height); + + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint src_idx = y * params.width + x; + const int value = coefficients[src_idx]; + const uint dst_idx = coeff_index(padded_width, x + 1u, y + 1u); + const uint magnitude = j2k_classic_magnitude(value); + magnitudes[dst_idx] = ushort(magnitude); + states[dst_idx] = value < 0 ? J2K_ENCODE_SIGN : uchar(0u); + max_magnitude = max(max_magnitude, magnitude); + } + } + + if (max_magnitude == 0u) { + emitter.counters.missing_bit_planes = params.total_bitplanes; + j2k_classic_pass_plan_store(out, emitter.counters); + return; + } + + const uint num_bitplanes = 32u - clz(max_magnitude); + if (num_bitplanes > params.total_bitplanes) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 3u; + j2k_classic_pass_plan_store(out, emitter.counters); + return; + } + emitter.counters.missing_bit_planes = params.total_bitplanes - num_bitplanes; + + uchar coded_marker = uchar(1u); + const uint total_passes = 1u + 3u * (num_bitplanes - 1u); + if (total_passes > J2K_CLASSIC_TIER1_PASS_PLAN_CAPACITY) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 61u; + j2k_classic_pass_plan_store(out, emitter.counters); + return; + } + emitter.counters.coding_passes = total_passes; + uint current_segment_idx = 0xFFFFFFFFu; + for (uint coding_pass = 0u; coding_pass < total_passes; ++coding_pass) { + emitter.current_pass = coding_pass; + const uint segment_idx = j2k_classic_bypass_segment_idx(coding_pass); + if (current_segment_idx != segment_idx) { + current_segment_idx = segment_idx; + emitter.counters.segment_count += 1u; + } + + const uint pass_type = coding_pass % 3u; + const bool use_arithmetic = coding_pass <= 9u || pass_type == 0u; + const uint current_bitplane = (coding_pass + 2u) / 3u; + const uint bit_mask = 1u << (num_bitplanes - 1u - current_bitplane); + switch (pass_type) { + case 0u: + j2k_classic_symbol_plan_cleanup_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u, + emitter + ); + coded_marker += uchar(1u); + break; + case 1u: + if (use_arithmetic) { + j2k_classic_symbol_plan_significance_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u, + emitter + ); + } else { + j2k_classic_symbol_plan_significance_pass_raw( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + 0u, + emitter + ); + } + break; + default: + if (use_arithmetic) { + j2k_classic_symbol_plan_magnitude_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + 0u, + emitter + ); + } else { + j2k_classic_symbol_plan_magnitude_pass_raw( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + emitter + ); + } + break; + } + } + + j2k_classic_pass_plan_store(out, emitter.counters); +} + +inline void j2k_emit_classic_tier1_tokens_bypass_u16_32_impl( + device const int *coefficients, + J2kClassicEncodeParams params, + device J2kClassicTier1SymbolPlanCounters *out, + device uchar *token_data, + device J2kClassicTier1TokenSegment *token_segments, + uint token_capacity, + uint token_segment_capacity +) { + thread J2kClassicTier1TokenEmitter emitter; + j2k_classic_tier1_token_emit_init( + emitter, + token_data, + token_capacity, + token_segments, + token_segment_capacity + ); + + if (params.width == 0u || params.height == 0u || + params.width > J2K_CLASSIC_ENCODE_32_MAX_WIDTH || + params.height > J2K_CLASSIC_ENCODE_32_MAX_HEIGHT || + params.total_bitplanes > 16u) { + emitter.counters.code = J2K_ENCODE_STATUS_UNSUPPORTED; + emitter.counters.detail = 1u; + j2k_classic_tier1_token_emit_store(out, emitter); + return; + } + if (params.style_flags != J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) { + emitter.counters.code = J2K_ENCODE_STATUS_UNSUPPORTED; + emitter.counters.detail = 2u; + j2k_classic_tier1_token_emit_store(out, emitter); + return; + } + + thread ushort magnitudes[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + const uint padded_width = params.width + 2u; + j2k_classic_clear_state_border(states, padded_width, params.width, params.height); + + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint src_idx = y * params.width + x; + const int value = coefficients[src_idx]; + const uint dst_idx = coeff_index(padded_width, x + 1u, y + 1u); + const uint magnitude = j2k_classic_magnitude(value); + magnitudes[dst_idx] = ushort(magnitude); + states[dst_idx] = value < 0 ? J2K_ENCODE_SIGN : uchar(0u); + max_magnitude = max(max_magnitude, magnitude); + } + } + + if (max_magnitude == 0u) { + emitter.counters.missing_bit_planes = params.total_bitplanes; + j2k_classic_tier1_token_emit_store(out, emitter); + return; + } + + const uint num_bitplanes = 32u - clz(max_magnitude); + if (num_bitplanes > params.total_bitplanes) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 3u; + j2k_classic_tier1_token_emit_store(out, emitter); + return; + } + emitter.counters.missing_bit_planes = params.total_bitplanes - num_bitplanes; + + uchar coded_marker = uchar(1u); + const uint total_passes = 1u + 3u * (num_bitplanes - 1u); + emitter.counters.coding_passes = total_passes; + uint current_segment_idx = 0xFFFFFFFFu; + uint current_segment_start_pass = 0u; + bool current_use_arithmetic = true; + bool have_segment = false; + for (uint coding_pass = 0u; coding_pass < total_passes; ++coding_pass) { + const uint pass_type = coding_pass % 3u; + const uint segment_idx = j2k_classic_bypass_segment_idx(coding_pass); + const bool use_arithmetic = coding_pass <= 9u || pass_type == 0u; + if (!have_segment || current_segment_idx != segment_idx) { + if (have_segment) { + j2k_classic_tier1_token_emit_push_segment( + emitter, + current_segment_start_pass, + coding_pass, + current_use_arithmetic + ); + } + current_segment_idx = segment_idx; + current_segment_start_pass = coding_pass; + current_use_arithmetic = use_arithmetic; + have_segment = true; + } + + const uint current_bitplane = (coding_pass + 2u) / 3u; + const uint bit_mask = 1u << (num_bitplanes - 1u - current_bitplane); + switch (pass_type) { + case 0u: + j2k_classic_symbol_plan_cleanup_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u, + emitter + ); + coded_marker += uchar(1u); + break; + case 1u: + if (use_arithmetic) { + j2k_classic_symbol_plan_significance_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u, + emitter + ); + } else { + j2k_classic_symbol_plan_significance_pass_raw( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + 0u, + emitter + ); + } + break; + default: + if (use_arithmetic) { + j2k_classic_symbol_plan_magnitude_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + 0u, + emitter + ); + } else { + j2k_classic_symbol_plan_magnitude_pass_raw( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + emitter + ); + } + break; + } + } + + if (have_segment) { + j2k_classic_tier1_token_emit_push_segment( + emitter, + current_segment_start_pass, + total_passes, + current_use_arithmetic + ); + } + j2k_classic_tier1_token_writer_finish(emitter.writer); + j2k_classic_tier1_token_emit_store(out, emitter); +} + +inline void j2k_emit_classic_tier1_split_tokens_bypass_u16_32_impl( + device const int *coefficients, + J2kClassicEncodeParams params, + device J2kClassicTier1SymbolPlanCounters *out, + device uchar *mq_token_data, + device uchar *raw_token_data, + device J2kClassicTier1TokenSegment *token_segments, + uint mq_token_capacity, + uint raw_token_capacity, + uint token_segment_capacity +) { + thread J2kClassicTier1SplitTokenEmitter emitter; + j2k_classic_tier1_split_token_emit_init( + emitter, + mq_token_data, + mq_token_capacity, + raw_token_data, + raw_token_capacity, + token_segments, + token_segment_capacity + ); + + if (params.width == 0u || params.height == 0u || + params.width > J2K_CLASSIC_ENCODE_32_MAX_WIDTH || + params.height > J2K_CLASSIC_ENCODE_32_MAX_HEIGHT || + params.total_bitplanes > 16u) { + emitter.counters.code = J2K_ENCODE_STATUS_UNSUPPORTED; + emitter.counters.detail = 1u; + j2k_classic_tier1_split_token_emit_store(out, emitter); + return; + } + if (params.style_flags != J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) { + emitter.counters.code = J2K_ENCODE_STATUS_UNSUPPORTED; + emitter.counters.detail = 2u; + j2k_classic_tier1_split_token_emit_store(out, emitter); + return; + } + + thread ushort magnitudes[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + const uint padded_width = params.width + 2u; + j2k_classic_clear_state_border(states, padded_width, params.width, params.height); + + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint src_idx = y * params.width + x; + const int value = coefficients[src_idx]; + const uint dst_idx = coeff_index(padded_width, x + 1u, y + 1u); + const uint magnitude = j2k_classic_magnitude(value); + magnitudes[dst_idx] = ushort(magnitude); + states[dst_idx] = value < 0 ? J2K_ENCODE_SIGN : uchar(0u); + max_magnitude = max(max_magnitude, magnitude); + } + } + + if (max_magnitude == 0u) { + emitter.counters.missing_bit_planes = params.total_bitplanes; + j2k_classic_tier1_split_token_emit_store(out, emitter); + return; + } + + const uint num_bitplanes = 32u - clz(max_magnitude); + if (num_bitplanes > params.total_bitplanes) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 3u; + j2k_classic_tier1_split_token_emit_store(out, emitter); + return; + } + emitter.counters.missing_bit_planes = params.total_bitplanes - num_bitplanes; + + uchar coded_marker = uchar(1u); + const uint total_passes = 1u + 3u * (num_bitplanes - 1u); + emitter.counters.coding_passes = total_passes; + uint current_segment_idx = 0xFFFFFFFFu; + uint current_segment_start_pass = 0u; + bool current_use_arithmetic = true; + bool have_segment = false; + for (uint coding_pass = 0u; coding_pass < total_passes; ++coding_pass) { + const uint pass_type = coding_pass % 3u; + const uint segment_idx = j2k_classic_bypass_segment_idx(coding_pass); + const bool use_arithmetic = coding_pass <= 9u || pass_type == 0u; + if (!have_segment || current_segment_idx != segment_idx) { + if (have_segment) { + j2k_classic_tier1_split_token_emit_push_segment( + emitter, + current_segment_start_pass, + coding_pass, + current_use_arithmetic + ); + } + current_segment_idx = segment_idx; + current_segment_start_pass = coding_pass; + current_use_arithmetic = use_arithmetic; + emitter.current_segment_start_bit = + j2k_classic_tier1_split_token_emit_stream_bit_count(emitter, use_arithmetic); + have_segment = true; + } + + const uint current_bitplane = (coding_pass + 2u) / 3u; + const uint bit_mask = 1u << (num_bitplanes - 1u - current_bitplane); + switch (pass_type) { + case 0u: + j2k_classic_symbol_plan_cleanup_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u, + emitter + ); + coded_marker += uchar(1u); + break; + case 1u: + if (use_arithmetic) { + j2k_classic_symbol_plan_significance_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u, + emitter + ); + } else { + j2k_classic_symbol_plan_significance_pass_raw( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + 0u, + emitter + ); + } + break; + default: + if (use_arithmetic) { + j2k_classic_symbol_plan_magnitude_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + 0u, + emitter + ); + } else { + j2k_classic_symbol_plan_magnitude_pass_raw( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + emitter + ); + } + break; + } + } + + if (have_segment) { + j2k_classic_tier1_split_token_emit_push_segment( + emitter, + current_segment_start_pass, + total_passes, + current_use_arithmetic + ); + } + j2k_classic_tier1_token_writer_finish(emitter.mq_writer); + j2k_classic_tier1_token_writer_finish(emitter.raw_writer); + j2k_classic_tier1_split_token_emit_store(out, emitter); +} + +inline void j2k_emit_classic_tier1_split_mq_byte_raw_tokens_bypass_u16_32_impl( + device const int *coefficients, + J2kClassicEncodeParams params, + device J2kClassicTier1SymbolPlanCounters *out, + device uchar *mq_token_data, + device uchar *raw_token_data, + device J2kClassicTier1TokenSegment *token_segments, + uint mq_token_capacity, + uint raw_token_capacity, + uint token_segment_capacity +) { + thread J2kClassicTier1SplitMqByteRawTokenEmitter emitter; + j2k_classic_tier1_split_mq_byte_token_emit_init( + emitter, + mq_token_data, + mq_token_capacity, + raw_token_data, + raw_token_capacity, + token_segments, + token_segment_capacity + ); + + if (params.width == 0u || params.height == 0u || + params.width > J2K_CLASSIC_ENCODE_32_MAX_WIDTH || + params.height > J2K_CLASSIC_ENCODE_32_MAX_HEIGHT || + params.total_bitplanes > 16u) { + emitter.counters.code = J2K_ENCODE_STATUS_UNSUPPORTED; + emitter.counters.detail = 1u; + j2k_classic_tier1_split_mq_byte_token_emit_store(out, emitter); + return; + } + if (params.style_flags != J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) { + emitter.counters.code = J2K_ENCODE_STATUS_UNSUPPORTED; + emitter.counters.detail = 2u; + j2k_classic_tier1_split_mq_byte_token_emit_store(out, emitter); + return; + } + + thread ushort magnitudes[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + const uint padded_width = params.width + 2u; + j2k_classic_clear_state_border(states, padded_width, params.width, params.height); + + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint src_idx = y * params.width + x; + const int value = coefficients[src_idx]; + const uint dst_idx = coeff_index(padded_width, x + 1u, y + 1u); + const uint magnitude = j2k_classic_magnitude(value); + magnitudes[dst_idx] = ushort(magnitude); + states[dst_idx] = value < 0 ? J2K_ENCODE_SIGN : uchar(0u); + max_magnitude = max(max_magnitude, magnitude); + } + } + + if (max_magnitude == 0u) { + emitter.counters.missing_bit_planes = params.total_bitplanes; + j2k_classic_tier1_split_mq_byte_token_emit_store(out, emitter); + return; + } + + const uint num_bitplanes = 32u - clz(max_magnitude); + if (num_bitplanes > params.total_bitplanes) { + emitter.counters.code = J2K_ENCODE_STATUS_FAIL; + emitter.counters.detail = 3u; + j2k_classic_tier1_split_mq_byte_token_emit_store(out, emitter); + return; + } + emitter.counters.missing_bit_planes = params.total_bitplanes - num_bitplanes; + + uchar coded_marker = uchar(1u); + const uint total_passes = 1u + 3u * (num_bitplanes - 1u); + emitter.counters.coding_passes = total_passes; + uint current_segment_idx = 0xFFFFFFFFu; + uint current_segment_start_pass = 0u; + bool current_use_arithmetic = true; + bool have_segment = false; + for (uint coding_pass = 0u; coding_pass < total_passes; ++coding_pass) { + const uint pass_type = coding_pass % 3u; + const uint segment_idx = j2k_classic_bypass_segment_idx(coding_pass); + const bool use_arithmetic = coding_pass <= 9u || pass_type == 0u; + if (!have_segment || current_segment_idx != segment_idx) { + if (have_segment) { + j2k_classic_tier1_split_mq_byte_token_emit_push_segment( + emitter, + current_segment_start_pass, + coding_pass, + current_use_arithmetic + ); + } + current_segment_idx = segment_idx; + current_segment_start_pass = coding_pass; + current_use_arithmetic = use_arithmetic; + emitter.current_segment_start_bit = + j2k_classic_tier1_split_mq_byte_token_emit_stream_bit_count( + emitter, + use_arithmetic + ); + have_segment = true; + } + + const uint current_bitplane = (coding_pass + 2u) / 3u; + const uint bit_mask = 1u << (num_bitplanes - 1u - current_bitplane); + switch (pass_type) { + case 0u: + j2k_classic_symbol_plan_cleanup_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u, + emitter + ); + coded_marker += uchar(1u); + break; + case 1u: + if (use_arithmetic) { + j2k_classic_symbol_plan_significance_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u, + emitter + ); + } else { + j2k_classic_symbol_plan_significance_pass_raw( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + 0u, + emitter + ); + } + break; + default: + if (use_arithmetic) { + j2k_classic_symbol_plan_magnitude_pass( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + 0u, + emitter + ); + } else { + j2k_classic_symbol_plan_magnitude_pass_raw( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + emitter + ); + } + break; + } + } + + if (have_segment) { + j2k_classic_tier1_split_mq_byte_token_emit_push_segment( + emitter, + current_segment_start_pass, + total_passes, + current_use_arithmetic + ); + } + j2k_classic_tier1_token_writer_finish(emitter.raw_writer); + j2k_classic_tier1_split_mq_byte_token_emit_store(out, emitter); +} + +inline void j2k_profile_classic_tier1_raw_pack_bypass_u16_32_impl( + device const int *coefficients, + device uchar *out, + J2kClassicEncodeParams params +) { + if (params.width == 0u || params.height == 0u || + params.width > J2K_CLASSIC_ENCODE_32_MAX_WIDTH || + params.height > J2K_CLASSIC_ENCODE_32_MAX_HEIGHT || + params.total_bitplanes > 16u) { + return; + } + + thread ushort magnitudes[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + const uint padded_width = params.width + 2u; + j2k_classic_clear_state_border(states, padded_width, params.width, params.height); + + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint src_idx = y * params.width + x; + const int value = coefficients[src_idx]; + const uint dst_idx = coeff_index(padded_width, x + 1u, y + 1u); + const uint magnitude = j2k_classic_magnitude(value); + magnitudes[dst_idx] = ushort(magnitude); + states[dst_idx] = value < 0 ? J2K_ENCODE_SIGN : uchar(0u); + max_magnitude = max(max_magnitude, magnitude); + } + } + + if (max_magnitude == 0u) { + return; + } + + const uint num_bitplanes = 32u - clz(max_magnitude); + if (num_bitplanes > params.total_bitplanes) { + return; + } + + thread J2kClassicTier1DensityCounters counters; + j2k_classic_tier1_density_zero(counters); + + uchar coded_marker = uchar(1u); + const uint total_passes = 1u + 3u * (num_bitplanes - 1u); + uint data_cursor = 0u; + uint current_raw_segment_idx = 0xFFFFFFFFu; + bool have_raw_segment = false; + thread J2kRawBitWriter raw_writer; + + for (uint coding_pass = 0u; coding_pass < total_passes; ++coding_pass) { + const uint current_bitplane = (coding_pass + 2u) / 3u; + const uint bit_mask = 1u << (num_bitplanes - 1u - current_bitplane); + const uint pass_type = coding_pass % 3u; + const bool use_arithmetic = + (params.style_flags & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) == 0u || + coding_pass <= 9u || + pass_type == 0u; + + if (!use_arithmetic) { + const uint segment_idx = + (params.style_flags & J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS) != 0u + ? coding_pass + : j2k_classic_bypass_segment_idx(coding_pass); + if (!have_raw_segment || current_raw_segment_idx != segment_idx) { + if (have_raw_segment) { + j2k_raw_writer_finish(raw_writer); + if (raw_writer.failed != 0u) { + return; + } + data_cursor += raw_writer.len; + } + if (data_cursor > params.output_capacity) { + return; + } + current_raw_segment_idx = segment_idx; + have_raw_segment = true; + j2k_raw_writer_init(raw_writer, out + data_cursor, params.output_capacity - data_cursor); + } + } + + switch (pass_type) { + case 0u: + j2k_classic_profile_cleanup_density( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.style_flags, + counters + ); + coded_marker += uchar(1u); + break; + case 1u: + if (use_arithmetic) { + j2k_classic_profile_significance_density( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + params.style_flags, + true, + counters + ); + } else { + j2k_classic_significance_pass_raw( + magnitudes, + states, + coded_marker, + raw_writer, + params.width, + params.height, + padded_width, + bit_mask, + params.style_flags + ); + } + break; + default: + if (use_arithmetic) { + j2k_classic_profile_magnitude_density( + states, + coded_marker, + params.width, + params.height, + padded_width, + true, + counters + ); + } else { + j2k_classic_magnitude_refinement_pass_raw( + magnitudes, + states, + coded_marker, + raw_writer, + params.width, + params.height, + padded_width, + bit_mask + ); + } + break; + } + } + + if (have_raw_segment) { + j2k_raw_writer_finish(raw_writer); + } +} + +inline void j2k_profile_classic_tier1_arithmetic_pack_bypass_u16_32_impl( + device const int *coefficients, + device uchar *out, + J2kClassicEncodeParams params +) { + if (params.width == 0u || params.height == 0u || + params.width > J2K_CLASSIC_ENCODE_32_MAX_WIDTH || + params.height > J2K_CLASSIC_ENCODE_32_MAX_HEIGHT || + params.total_bitplanes > 16u || + params.style_flags != J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) { + return; + } + + thread ushort magnitudes[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + thread uchar states[J2K_CLASSIC_ENCODE_32_MAX_COEFF_COUNT]; + const uint padded_width = params.width + 2u; + j2k_classic_clear_state_border(states, padded_width, params.width, params.height); + + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + const uint src_idx = y * params.width + x; + const int value = coefficients[src_idx]; + const uint dst_idx = coeff_index(padded_width, x + 1u, y + 1u); + const uint magnitude = j2k_classic_magnitude(value); + magnitudes[dst_idx] = ushort(magnitude); + states[dst_idx] = value < 0 ? J2K_ENCODE_SIGN : uchar(0u); + max_magnitude = max(max_magnitude, magnitude); + } + } + + if (max_magnitude == 0u) { + return; + } + + const uint num_bitplanes = 32u - clz(max_magnitude); + if (num_bitplanes > params.total_bitplanes) { + return; + } + + thread uchar contexts[19]; + reset_contexts(contexts); + thread J2kMqEncoder arithmetic_encoder; + thread J2kClassicTier1DensityCounters counters; + j2k_classic_tier1_density_zero(counters); + + uchar coded_marker = uchar(1u); + const uint total_passes = 1u + 3u * (num_bitplanes - 1u); + uint data_cursor = 0u; + uint current_arithmetic_segment_idx = 0xFFFFFFFFu; + bool have_arithmetic_segment = false; + + for (uint coding_pass = 0u; coding_pass < total_passes; ++coding_pass) { + const uint current_bitplane = (coding_pass + 2u) / 3u; + const uint bit_mask = 1u << (num_bitplanes - 1u - current_bitplane); + const uint pass_type = coding_pass % 3u; + const bool use_arithmetic = coding_pass <= 9u || pass_type == 0u; + + if (use_arithmetic) { + const uint segment_idx = j2k_classic_bypass_segment_idx(coding_pass); + if (!have_arithmetic_segment || current_arithmetic_segment_idx != segment_idx) { + if (have_arithmetic_segment) { + const uint segment_len = j2k_classic_finish_arithmetic_segment(arithmetic_encoder); + if (arithmetic_encoder.failed != 0u) { + return; + } + data_cursor += segment_len; + } + if (data_cursor > params.output_capacity) { + return; + } + current_arithmetic_segment_idx = segment_idx; + have_arithmetic_segment = true; + j2k_mq_init(arithmetic_encoder, out + data_cursor, params.output_capacity - data_cursor); + } + } + + switch (pass_type) { + case 0u: + j2k_classic_cleanup_pass( + magnitudes, + states, + coded_marker, + arithmetic_encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u + ); + coded_marker += uchar(1u); + break; + case 1u: + if (use_arithmetic) { + j2k_classic_significance_pass( + magnitudes, + states, + coded_marker, + arithmetic_encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + params.sub_band_type, + 0u + ); + } else { + j2k_classic_profile_significance_density( + magnitudes, + states, + coded_marker, + params.width, + params.height, + padded_width, + bit_mask, + 0u, + false, + counters + ); + } + break; + default: + if (use_arithmetic) { + j2k_classic_magnitude_refinement_pass( + magnitudes, + states, + coded_marker, + arithmetic_encoder, + contexts, + params.width, + params.height, + padded_width, + bit_mask, + 0u + ); + } else { + j2k_classic_profile_magnitude_density( + states, + coded_marker, + params.width, + params.height, + padded_width, + false, + counters + ); + } + break; + } + + if (use_arithmetic && arithmetic_encoder.failed != 0u) { + return; + } + } + + if (have_arithmetic_segment) { + j2k_classic_finish_arithmetic_segment(arithmetic_encoder); + } +} + +kernel void j2k_encode_classic_code_block( + device const int *coefficients [[buffer(0)]], + device uchar *out [[buffer(1)]], + constant J2kClassicEncodeParams ¶ms [[buffer(2)]], + device J2kClassicEncodeStatus *status [[buffer(3)]], + device J2kClassicSegment *segments [[buffer(4)]], + uint gid [[thread_position_in_grid]] +) { + if (gid != 0u) { + return; + } + j2k_encode_classic_code_block_impl(coefficients, out, params, status, segments); +} + +kernel void j2k_encode_classic_code_blocks( + device const int *coefficients [[buffer(0)]], + device uchar *out [[buffer(1)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(2)]], + device J2kClassicEncodeStatus *statuses [[buffer(3)]], + device J2kClassicSegment *segments [[buffer(4)]], + constant uint &job_count [[buffer(5)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = job.style_flags; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_encode_classic_code_block_impl( + coefficients + job.coefficient_offset, + out + job.output_offset, + params, + statuses + gid, + segments + job.segment_offset + ); +} + +kernel void j2k_encode_classic_code_blocks_style0( + device const int *coefficients [[buffer(0)]], + device uchar *out [[buffer(1)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(2)]], + device J2kClassicEncodeStatus *statuses [[buffer(3)]], + device J2kClassicSegment *segments [[buffer(4)]], + constant uint &job_count [[buffer(5)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = 0u; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_encode_classic_code_block_impl_style0( + coefficients + job.coefficient_offset, + out + job.output_offset, + params, + statuses + gid, + segments + job.segment_offset + ); +} + +kernel void j2k_encode_classic_code_blocks_32( + device const int *coefficients [[buffer(0)]], + device uchar *out [[buffer(1)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(2)]], + device J2kClassicEncodeStatus *statuses [[buffer(3)]], + device J2kClassicSegment *segments [[buffer(4)]], + constant uint &job_count [[buffer(5)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = job.style_flags; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_encode_classic_code_block_impl_32( + coefficients + job.coefficient_offset, + out + job.output_offset, + params, + statuses + gid, + segments + job.segment_offset + ); +} + +kernel void j2k_encode_classic_code_blocks_bypass_32( + device const int *coefficients [[buffer(0)]], + device uchar *out [[buffer(1)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(2)]], + device J2kClassicEncodeStatus *statuses [[buffer(3)]], + device J2kClassicSegment *segments [[buffer(4)]], + constant uint &job_count [[buffer(5)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_encode_classic_code_block_impl_bypass_32( + coefficients + job.coefficient_offset, + out + job.output_offset, + params, + statuses + gid, + segments + job.segment_offset + ); +} + +kernel void j2k_encode_classic_code_blocks_bypass_u16_32( + device const int *coefficients [[buffer(0)]], + device uchar *out [[buffer(1)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(2)]], + device J2kClassicEncodeStatus *statuses [[buffer(3)]], + device J2kClassicSegment *segments [[buffer(4)]], + constant uint &job_count [[buffer(5)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_encode_classic_code_block_impl_bypass_u16_32( + coefficients + job.coefficient_offset, + out + job.output_offset, + params, + statuses + gid, + segments + job.segment_offset + ); +} + +kernel void j2k_profile_classic_tier1_density_bypass_u16_32( + device const int *coefficients [[buffer(0)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(1)]], + device J2kClassicTier1DensityCounters *counters [[buffer(2)]], + constant uint &job_count [[buffer(3)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = job.style_flags; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_profile_classic_tier1_density_bypass_u16_32_impl( + coefficients + job.coefficient_offset, + params, + counters + gid + ); +} + +kernel void j2k_plan_classic_tier1_symbols_bypass_u16_32( + device const int *coefficients [[buffer(0)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(1)]], + device J2kClassicTier1SymbolPlanCounters *counters [[buffer(2)]], + constant uint &job_count [[buffer(3)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = job.style_flags; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_plan_classic_tier1_symbols_bypass_u16_32_impl( + coefficients + job.coefficient_offset, + params, + counters + gid + ); +} + +kernel void j2k_plan_classic_tier1_passes_bypass_u16_32( + device const int *coefficients [[buffer(0)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(1)]], + device J2kClassicTier1PassPlanCounters *counters [[buffer(2)]], + constant uint &job_count [[buffer(3)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = job.style_flags; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_plan_classic_tier1_passes_bypass_u16_32_impl( + coefficients + job.coefficient_offset, + params, + counters + gid + ); +} + +kernel void j2k_emit_classic_tier1_tokens_bypass_u16_32( + device const int *coefficients [[buffer(0)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(1)]], + device J2kClassicTier1SymbolPlanCounters *counters [[buffer(2)]], + device uchar *token_data [[buffer(3)]], + device J2kClassicTier1TokenSegment *token_segments [[buffer(4)]], + constant uint &token_stride_bytes [[buffer(5)]], + constant uint &token_segment_stride [[buffer(6)]], + constant uint &job_count [[buffer(7)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = job.style_flags; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_emit_classic_tier1_tokens_bypass_u16_32_impl( + coefficients + job.coefficient_offset, + params, + counters + gid, + token_data + (gid * token_stride_bytes), + token_segments + (gid * token_segment_stride), + token_stride_bytes, + token_segment_stride + ); +} + +kernel void j2k_emit_classic_tier1_split_tokens_bypass_u16_32( + device const int *coefficients [[buffer(0)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(1)]], + device J2kClassicTier1SymbolPlanCounters *counters [[buffer(2)]], + device uchar *mq_token_data [[buffer(3)]], + device uchar *raw_token_data [[buffer(4)]], + device J2kClassicTier1TokenSegment *token_segments [[buffer(5)]], + constant uint &mq_token_stride_bytes [[buffer(6)]], + constant uint &raw_token_stride_bytes [[buffer(7)]], + constant uint &token_segment_stride [[buffer(8)]], + constant uint &job_count [[buffer(9)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = job.style_flags; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_emit_classic_tier1_split_tokens_bypass_u16_32_impl( + coefficients + job.coefficient_offset, + params, + counters + gid, + mq_token_data + (gid * mq_token_stride_bytes), + raw_token_data + (gid * raw_token_stride_bytes), + token_segments + (gid * token_segment_stride), + mq_token_stride_bytes, + raw_token_stride_bytes, + token_segment_stride + ); +} + +kernel void j2k_emit_classic_tier1_split_mq_byte_raw_tokens_bypass_u16_32( + device const int *coefficients [[buffer(0)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(1)]], + device J2kClassicTier1SymbolPlanCounters *counters [[buffer(2)]], + device uchar *mq_token_data [[buffer(3)]], + device uchar *raw_token_data [[buffer(4)]], + device J2kClassicTier1TokenSegment *token_segments [[buffer(5)]], + constant uint &mq_token_stride_bytes [[buffer(6)]], + constant uint &raw_token_stride_bytes [[buffer(7)]], + constant uint &token_segment_stride [[buffer(8)]], + constant uint &job_count [[buffer(9)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = job.style_flags; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_emit_classic_tier1_split_mq_byte_raw_tokens_bypass_u16_32_impl( + coefficients + job.coefficient_offset, + params, + counters + gid, + mq_token_data + (gid * mq_token_stride_bytes), + raw_token_data + (gid * raw_token_stride_bytes), + token_segments + (gid * token_segment_stride), + mq_token_stride_bytes, + raw_token_stride_bytes, + token_segment_stride + ); +} + +kernel void j2k_pack_classic_tier1_tokens_bypass_u16_32( + device const J2kClassicEncodeBatchJob *jobs [[buffer(0)]], + device const J2kClassicTier1SymbolPlanCounters *counters [[buffer(1)]], + device const uchar *token_data [[buffer(2)]], + device const J2kClassicTier1TokenSegment *token_segments [[buffer(3)]], + device uchar *out [[buffer(4)]], + device J2kClassicEncodeStatus *statuses [[buffer(5)]], + device J2kClassicSegment *segments [[buffer(6)]], + constant uint &token_stride_bytes [[buffer(7)]], + constant uint &token_segment_stride [[buffer(8)]], + constant uint &job_count [[buffer(9)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = job.style_flags; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_pack_classic_tier1_tokens_bypass_u16_32_impl( + params, + counters[gid], + token_data + (gid * token_stride_bytes), + token_segments + (gid * token_segment_stride), + token_stride_bytes, + token_segment_stride, + out + job.output_offset, + statuses + gid, + segments + job.segment_offset + ); +} + +kernel void j2k_pack_classic_tier1_split_tokens_bypass_u16_32( + device const J2kClassicEncodeBatchJob *jobs [[buffer(0)]], + device const J2kClassicTier1SymbolPlanCounters *counters [[buffer(1)]], + device const uchar *mq_token_data [[buffer(2)]], + device const uchar *raw_token_data [[buffer(3)]], + device const J2kClassicTier1TokenSegment *token_segments [[buffer(4)]], + device uchar *out [[buffer(5)]], + device J2kClassicEncodeStatus *statuses [[buffer(6)]], + device J2kClassicSegment *segments [[buffer(7)]], + constant uint &mq_token_stride_bytes [[buffer(8)]], + constant uint &raw_token_stride_bytes [[buffer(9)]], + constant uint &token_segment_stride [[buffer(10)]], + constant uint &job_count [[buffer(11)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = job.style_flags; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_pack_classic_tier1_split_tokens_bypass_u16_32_impl( + params, + counters[gid], + mq_token_data + gid * mq_token_stride_bytes, + raw_token_data + gid * raw_token_stride_bytes, + token_segments + gid * token_segment_stride, + mq_token_stride_bytes, + raw_token_stride_bytes, + token_segment_stride, + out + job.output_offset, + statuses + gid, + segments + job.segment_offset + ); +} + +kernel void j2k_profile_classic_tier1_raw_pack_bypass_u16_32( + device const int *coefficients [[buffer(0)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(1)]], + device uchar *out [[buffer(2)]], + constant uint &job_count [[buffer(3)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = job.style_flags; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_profile_classic_tier1_raw_pack_bypass_u16_32_impl( + coefficients + job.coefficient_offset, + out + job.output_offset, + params + ); +} + +kernel void j2k_profile_classic_tier1_arithmetic_pack_bypass_u16_32( + device const int *coefficients [[buffer(0)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(1)]], + device uchar *out [[buffer(2)]], + constant uint &job_count [[buffer(3)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = job.style_flags; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_profile_classic_tier1_arithmetic_pack_bypass_u16_32_impl( + coefficients + job.coefficient_offset, + out + job.output_offset, + params + ); +} + +kernel void j2k_encode_classic_code_blocks_style0_32( + device const int *coefficients [[buffer(0)]], + device uchar *out [[buffer(1)]], + device const J2kClassicEncodeBatchJob *jobs [[buffer(2)]], + device J2kClassicEncodeStatus *statuses [[buffer(3)]], + device J2kClassicSegment *segments [[buffer(4)]], + constant uint &job_count [[buffer(5)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kClassicEncodeBatchJob job = jobs[gid]; + J2kClassicEncodeParams params; + params.width = job.width; + params.height = job.height; + params.sub_band_type = job.sub_band_type; + params.total_bitplanes = job.total_bitplanes; + params.style_flags = 0u; + params.output_capacity = job.output_capacity; + params.segment_capacity = job.segment_capacity; + j2k_encode_classic_code_block_impl_style0_32( + coefficients + job.coefficient_offset, + out + job.output_offset, + params, + statuses + gid, + segments + job.segment_offset + ); +} + +constant uint J2K_HT_MAX_BITPLANES = 30u; +constant uint J2K_HT_MAX_SAMPLES = 16384u; +constant uint J2K_HT_MS_BYTES_PER_SAMPLE_FLOOR = 5u; +constant uint J2K_HT_MEL_SIZE = 192u; +constant uint J2K_HT_VLC_SIZE = 3072u - J2K_HT_MEL_SIZE; +constant uint J2K_HT_MS_SIZE = ((16384u * 16u) + 14u) / 15u; +constant uint J2K_HT_MEL_OFFSET = J2K_HT_MS_SIZE; +constant uint J2K_HT_VLC_OFFSET = J2K_HT_MS_SIZE + J2K_HT_MEL_SIZE; + +struct J2kHtEncodeParams { + uint width; + uint height; + uint total_bitplanes; + uint output_capacity; +}; + +struct J2kHtEncodeStatus { + uint code; + uint detail; + uint data_len; + uint num_coding_passes; + uint num_zero_bitplanes; + uint reserved0; + uint reserved1; + uint reserved2; +}; + +struct J2kHtMelEncoder { + uint pos; + uint remaining_bits; + uchar tmp; + uint run; + uint k; + uint threshold; + uint failed; + uint offset; + uint capacity; +}; + +struct J2kHtVlcEncoder { + uint pos; + uint used_bits; + uchar tmp; + uint last_greater_than_8f; + uint failed; + uint offset; + uint capacity; +}; + +struct J2kHtMagSgnEncoder { + uint pos; + uint max_bits; + uint used_bits; + uint tmp; + uint failed; + uint capacity; +}; + +constant uint J2K_HT_MEL_EXP[13] = { + 0u, 0u, 0u, 1u, 1u, 1u, 2u, 2u, 2u, 3u, 3u, 4u, 5u +}; + +inline uint j2k_ht_scaled_scratch_size(uint max_size, uint sample_count) { + const ulong scaled = + (ulong(max_size) * ulong(sample_count) + ulong(J2K_HT_MAX_SAMPLES - 1u)) / + ulong(J2K_HT_MAX_SAMPLES); + return uint(max(ulong(1u), scaled)); +} + +inline uint j2k_ht_sample_count(J2kHtEncodeParams params) { + return params.width * params.height; +} + +inline uint j2k_ht_ms_size(J2kHtEncodeParams params) { + const uint sample_count = j2k_ht_sample_count(params); + const uint scaled = j2k_ht_scaled_scratch_size(J2K_HT_MS_SIZE, sample_count); + const uint floor = sample_count * J2K_HT_MS_BYTES_PER_SAMPLE_FLOOR; + return min(J2K_HT_MS_SIZE, max(scaled, floor)); +} + +inline uint j2k_ht_mel_size(J2kHtEncodeParams params) { + return J2K_HT_MEL_SIZE; +} + +inline uint j2k_ht_vlc_size(J2kHtEncodeParams params) { + return J2K_HT_VLC_SIZE; +} + +inline uint j2k_ht_mel_offset(J2kHtEncodeParams params) { + return j2k_ht_ms_size(params); +} + +inline uint j2k_ht_vlc_offset(J2kHtEncodeParams params) { + return j2k_ht_ms_size(params) + j2k_ht_mel_size(params); +} + +inline uint j2k_ht_output_size(J2kHtEncodeParams params) { + return j2k_ht_ms_size(params) + j2k_ht_mel_size(params) + j2k_ht_vlc_size(params); +} + +inline void j2k_set_ht_encode_status( + device J2kHtEncodeStatus *status, + uint code, + uint detail, + uint data_len, + uint passes, + uint zbp +) { + status->code = code; + status->detail = detail; + status->data_len = data_len; + status->num_coding_passes = passes; + status->num_zero_bitplanes = zbp; + status->reserved0 = 0u; + status->reserved1 = 0u; + status->reserved2 = 0u; +} + +inline void j2k_set_ht_encode_status_with_segments( + device J2kHtEncodeStatus *status, + uint code, + uint detail, + uint data_len, + uint passes, + uint zbp, + uint ms_len, + uint mel_len, + uint vlc_len +) { + status->code = code; + status->detail = detail; + status->data_len = data_len; + status->num_coding_passes = passes; + status->num_zero_bitplanes = zbp; + status->reserved0 = ms_len; + status->reserved1 = mel_len; + status->reserved2 = vlc_len; +} + +inline uint j2k_ht_aligned_sign_magnitude(int coefficient, uint total_bitplanes) { + if (coefficient == 0) { + return 0u; + } + const uint sign = coefficient < 0 ? 0x80000000u : 0u; + const uint magnitude = (coefficient < 0 ? uint(-coefficient) : uint(coefficient)) + << (31u - total_bitplanes); + return sign | magnitude; +} + +inline void j2k_ht_mel_init(thread J2kHtMelEncoder &mel, J2kHtEncodeParams params) { + mel.pos = 0u; + mel.remaining_bits = 8u; + mel.tmp = uchar(0u); + mel.run = 0u; + mel.k = 0u; + mel.threshold = 1u; + mel.failed = 0u; + mel.offset = j2k_ht_mel_offset(params); + mel.capacity = j2k_ht_mel_size(params); +} + +inline void j2k_ht_vlc_init(thread J2kHtVlcEncoder &vlc, device uchar *out, J2kHtEncodeParams params) { + vlc.pos = 1u; + vlc.used_bits = 4u; + vlc.tmp = uchar(0x0Fu); + vlc.last_greater_than_8f = 1u; + vlc.failed = 0u; + vlc.offset = j2k_ht_vlc_offset(params); + vlc.capacity = j2k_ht_vlc_size(params); + out[vlc.offset + vlc.capacity - 1u] = uchar(0xFFu); +} + +inline void j2k_ht_ms_init(thread J2kHtMagSgnEncoder &ms, J2kHtEncodeParams params) { + ms.pos = 0u; + ms.max_bits = 8u; + ms.used_bits = 0u; + ms.tmp = 0u; + ms.failed = 0u; + ms.capacity = j2k_ht_ms_size(params); +} + +inline void j2k_ht_mel_emit_bit(thread J2kHtMelEncoder &mel, device uchar *out, bool bit) { + mel.tmp = uchar((uint(mel.tmp) << 1u) | (bit ? 1u : 0u)); + mel.remaining_bits -= 1u; + if (mel.remaining_bits == 0u) { + if (mel.pos >= mel.capacity) { + mel.failed = 1u; + return; + } + out[mel.offset + mel.pos] = mel.tmp; + mel.pos += 1u; + mel.remaining_bits = mel.tmp == uchar(0xFFu) ? 7u : 8u; + mel.tmp = uchar(0u); + } +} + +inline void j2k_ht_mel_encode(thread J2kHtMelEncoder &mel, device uchar *out, bool bit) { + if (!bit) { + mel.run += 1u; + if (mel.run >= mel.threshold) { + j2k_ht_mel_emit_bit(mel, out, true); + mel.run = 0u; + mel.k = min(mel.k + 1u, 12u); + mel.threshold = 1u << J2K_HT_MEL_EXP[mel.k]; + } + } else { + j2k_ht_mel_emit_bit(mel, out, false); + uint t = J2K_HT_MEL_EXP[mel.k]; + while (t > 0u) { + t -= 1u; + j2k_ht_mel_emit_bit(mel, out, ((mel.run >> t) & 1u) != 0u); + } + mel.run = 0u; + mel.k = mel.k == 0u ? 0u : mel.k - 1u; + mel.threshold = 1u << J2K_HT_MEL_EXP[mel.k]; + } +} + +inline void j2k_ht_vlc_encode( + thread J2kHtVlcEncoder &vlc, + device uchar *out, + uint codeword, + uint codeword_len +) { + while (codeword_len > 0u) { + if (vlc.pos >= vlc.capacity) { + vlc.failed = 1u; + return; + } + + uint available_bits = 8u - vlc.last_greater_than_8f - vlc.used_bits; + const uint take = min(available_bits, codeword_len); + const uint mask = take == 32u ? 0xFFFFFFFFu : ((1u << take) - 1u); + vlc.tmp = uchar(uint(vlc.tmp) | ((codeword & mask) << vlc.used_bits)); + vlc.used_bits += take; + available_bits -= take; + codeword_len -= take; + codeword >>= take; + + if (available_bits == 0u) { + if (vlc.last_greater_than_8f != 0u && vlc.tmp != uchar(0x7Fu)) { + vlc.last_greater_than_8f = 0u; + continue; + } + + const uint write_index = vlc.capacity - 1u - vlc.pos; + out[vlc.offset + write_index] = vlc.tmp; + vlc.pos += 1u; + vlc.last_greater_than_8f = vlc.tmp > uchar(0x8Fu) ? 1u : 0u; + vlc.tmp = uchar(0u); + vlc.used_bits = 0u; + } + } +} + +inline void j2k_ht_ms_encode( + thread J2kHtMagSgnEncoder &ms, + device uchar *out, + uint codeword, + uint codeword_len +) { + while (codeword_len > 0u) { + if (ms.pos >= ms.capacity) { + ms.failed = 1u; + return; + } + + const uint take = min(ms.max_bits - ms.used_bits, codeword_len); + const uint mask = take == 32u ? 0xFFFFFFFFu : ((1u << take) - 1u); + ms.tmp |= (codeword & mask) << ms.used_bits; + ms.used_bits += take; + codeword >>= take; + codeword_len -= take; + + if (ms.used_bits >= ms.max_bits) { + out[ms.pos] = uchar(ms.tmp); + ms.pos += 1u; + ms.max_bits = ms.tmp == 0xFFu ? 7u : 8u; + ms.tmp = 0u; + ms.used_bits = 0u; + } + } +} + +inline void j2k_ht_ms_terminate(thread J2kHtMagSgnEncoder &ms, device uchar *out) { + if (ms.used_bits > 0u) { + const uint unused = ms.max_bits - ms.used_bits; + ms.tmp |= (0xFFu & ((1u << unused) - 1u)) << ms.used_bits; + ms.used_bits += unused; + if (ms.tmp != 0xFFu) { + if (ms.pos >= ms.capacity) { + ms.failed = 1u; + return; + } + out[ms.pos] = uchar(ms.tmp); + ms.pos += 1u; + } + } else if (ms.max_bits == 7u) { + ms.pos = ms.pos == 0u ? 0u : ms.pos - 1u; + } +} + +inline void j2k_ht_process_sample( + uint slot, + uint value, + uint p, + thread int *rho_acc, + thread int *e_q, + thread int &e_qmax, + thread uint *s +) { + uint val = value + value; + val >>= p; + val &= ~1u; + if (val != 0u) { + rho_acc[0] |= int(1u << (slot & 0x3u)); + val -= 1u; + e_q[slot] = int(32u - clz(val)); + e_qmax = max(e_qmax, e_q[slot]); + val -= 1u; + s[slot] = val + (value >> 31u); + } +} + +inline uchar j2k_ht_uvlc_byte(device const uchar *table, uint index, uint field) { + return table[index * 6u + field]; +} + +inline void j2k_ht_encode_uvlc_pair( + thread J2kHtVlcEncoder &vlc, + device uchar *out, + device const uchar *uvlc_table, + uint first_index, + uint second_index +) { + const uchar first_pre = j2k_ht_uvlc_byte(uvlc_table, first_index, 0u); + const uchar first_pre_len = j2k_ht_uvlc_byte(uvlc_table, first_index, 1u); + const uchar first_suf = j2k_ht_uvlc_byte(uvlc_table, first_index, 2u); + const uchar first_suf_len = j2k_ht_uvlc_byte(uvlc_table, first_index, 3u); + const uchar second_pre = j2k_ht_uvlc_byte(uvlc_table, second_index, 0u); + const uchar second_pre_len = j2k_ht_uvlc_byte(uvlc_table, second_index, 1u); + const uchar second_suf = j2k_ht_uvlc_byte(uvlc_table, second_index, 2u); + const uchar second_suf_len = j2k_ht_uvlc_byte(uvlc_table, second_index, 3u); + j2k_ht_vlc_encode(vlc, out, uint(first_pre), uint(first_pre_len)); + j2k_ht_vlc_encode(vlc, out, uint(second_pre), uint(second_pre_len)); + j2k_ht_vlc_encode(vlc, out, uint(first_suf), uint(first_suf_len)); + j2k_ht_vlc_encode(vlc, out, uint(second_suf), uint(second_suf_len)); +} + +inline void j2k_ht_encode_uvlc( + int u_q0, + int u_q1, + thread J2kHtVlcEncoder &vlc, + device uchar *out, + device const uchar *uvlc_table +) { + if (u_q0 > 2 && u_q1 > 2) { + j2k_ht_encode_uvlc_pair(vlc, out, uvlc_table, uint(u_q0 - 2), uint(u_q1 - 2)); + } else if (u_q0 > 2 && u_q1 > 0) { + const uint first_index = uint(u_q0); + const uchar first_pre = j2k_ht_uvlc_byte(uvlc_table, first_index, 0u); + const uchar first_pre_len = j2k_ht_uvlc_byte(uvlc_table, first_index, 1u); + const uchar first_suf = j2k_ht_uvlc_byte(uvlc_table, first_index, 2u); + const uchar first_suf_len = j2k_ht_uvlc_byte(uvlc_table, first_index, 3u); + j2k_ht_vlc_encode(vlc, out, uint(first_pre), uint(first_pre_len)); + j2k_ht_vlc_encode(vlc, out, uint(u_q1 - 1), 1u); + j2k_ht_vlc_encode(vlc, out, uint(first_suf), uint(first_suf_len)); + } else { + j2k_ht_encode_uvlc_pair( + vlc, + out, + uvlc_table, + uint(max(u_q0, 0)), + uint(max(u_q1, 0)) + ); + } +} + +inline void j2k_ht_encode_uvlc_non_initial( + int u_q0, + int u_q1, + thread J2kHtVlcEncoder &vlc, + device uchar *out, + device const uchar *uvlc_table +) { + j2k_ht_encode_uvlc_pair( + vlc, + out, + uvlc_table, + uint(max(u_q0, 0)), + uint(max(u_q1, 0)) + ); +} + +inline void j2k_ht_encode_mag_signs( + int rho, + int u_q, + ushort tuple, + thread const uint *s, + uint offset, + thread J2kHtMagSgnEncoder &ms, + device uchar *out +) { + const uint e_k = uint(tuple & ushort(0xFu)); + for (uint bit = 0u; bit < 4u; ++bit) { + const int sample_mask = int(1u << bit); + if ((rho & sample_mask) == 0) { + continue; + } + const int reduction = int((e_k >> bit) & 1u); + const uint magnitude_bits = uint(u_q - reduction); + const uint payload = magnitude_bits == 0u + ? 0u + : (s[offset + bit] & ((1u << magnitude_bits) - 1u)); + j2k_ht_ms_encode(ms, out, payload, magnitude_bits); + } +} + +inline int j2k_ht_encode_quad_initial_row( + uint offset, + uint c_q, + int rho, + int e_qmax, + thread const int *e_q, + thread const uint *s, + uint lep, + uint lcxp, + thread uchar *e_val, + thread uchar *cx_val, + thread J2kHtMelEncoder &mel, + thread J2kHtVlcEncoder &vlc, + thread J2kHtMagSgnEncoder &ms, + device uchar *out, + device const ushort *vlc_table0 +) { + const int u_q = max(e_qmax, 1) - 1; + uint eps = 0u; + if (u_q > 0) { + eps |= uint(e_q[offset] == e_qmax); + eps |= uint(e_q[offset + 1u] == e_qmax) << 1u; + eps |= uint(e_q[offset + 2u] == e_qmax) << 2u; + eps |= uint(e_q[offset + 3u] == e_qmax) << 3u; + } + + e_val[lep] = max(e_val[lep], uchar(e_q[offset + 1u])); + e_val[lep + 1u] = uchar(e_q[offset + 3u]); + cx_val[lcxp] = uchar(uint(cx_val[lcxp]) | uint((rho & 2) >> 1)); + cx_val[lcxp + 1u] = uchar((rho & 8) >> 3); + + const ushort tuple = vlc_table0[(c_q << 8u) | (uint(rho) << 4u) | eps]; + j2k_ht_vlc_encode(vlc, out, uint(tuple >> 8u), uint((tuple >> 4u) & ushort(0x7u))); + if (c_q == 0u) { + j2k_ht_mel_encode(mel, out, rho != 0); + } + j2k_ht_encode_mag_signs(rho, max(e_qmax, 1), tuple, s, offset, ms, out); + return u_q; +} + +inline int j2k_ht_encode_quad_non_initial_row( + uint offset, + uint c_q, + int rho, + int e_qmax, + int max_e, + thread const int *e_q, + thread const uint *s, + thread J2kHtMelEncoder &mel, + thread J2kHtVlcEncoder &vlc, + thread J2kHtMagSgnEncoder &ms, + device uchar *out, + device const ushort *vlc_table1 +) { + const int kappa = (rho & (rho - 1)) != 0 ? max(max_e, 1) : 1; + const int u_q = max(e_qmax, kappa) - kappa; + uint eps = 0u; + if (u_q > 0) { + eps |= uint(e_q[offset] == e_qmax); + eps |= uint(e_q[offset + 1u] == e_qmax) << 1u; + eps |= uint(e_q[offset + 2u] == e_qmax) << 2u; + eps |= uint(e_q[offset + 3u] == e_qmax) << 3u; + } + + const ushort tuple = vlc_table1[(c_q << 8u) | (uint(rho) << 4u) | eps]; + j2k_ht_vlc_encode(vlc, out, uint(tuple >> 8u), uint((tuple >> 4u) & ushort(0x7u))); + if (c_q == 0u) { + j2k_ht_mel_encode(mel, out, rho != 0); + } + j2k_ht_encode_mag_signs(rho, max(e_qmax, kappa), tuple, s, offset, ms, out); + return u_q; +} + +inline void j2k_ht_clear_quad_state(thread int *rho, thread int *e_q, thread int *e_qmax, thread uint *s) { + rho[0] = 0; + rho[1] = 0; + for (uint idx = 0u; idx < 8u; ++idx) { + e_q[idx] = 0; + s[idx] = 0u; + } + e_qmax[0] = 0; + e_qmax[1] = 0; +} + +inline int j2k_ht_encode_first_quad_pair( + device const int *coefficients, + uint stride, + uint height, + uint total_bitplanes, + uint p, + thread uint &sp, + uint x, + thread uchar *e_val, + thread uchar *cx_val, + thread uint &c_q0, + thread int *rho, + thread int *e_q, + thread int *e_qmax, + thread uint *s, + thread J2kHtMelEncoder &mel, + thread J2kHtVlcEncoder &vlc, + thread J2kHtMagSgnEncoder &ms, + device uchar *out, + device const ushort *vlc_table0, + device const uchar *uvlc_table +) { + const uint lep = x / 2u; + const uint lcxp = x / 2u; + + j2k_ht_process_sample(0u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[0], e_q, e_qmax[0], s); + j2k_ht_process_sample( + 1u, + height > 1u ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, + p, + &rho[0], + e_q, + e_qmax[0], + s + ); + sp += 1u; + + if (x + 1u < stride) { + j2k_ht_process_sample(2u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[0], e_q, e_qmax[0], s); + j2k_ht_process_sample( + 3u, + height > 1u ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, + p, + &rho[0], + e_q, + e_qmax[0], + s + ); + sp += 1u; + } + + const int u_q0 = j2k_ht_encode_quad_initial_row( + 0u, c_q0, rho[0], e_qmax[0], e_q, s, lep, lcxp, e_val, cx_val, mel, vlc, ms, out, vlc_table0 + ); + + if (x + 2u < stride) { + j2k_ht_process_sample(4u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[1], e_q, e_qmax[1], s); + j2k_ht_process_sample( + 5u, + height > 1u ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, + p, + &rho[1], + e_q, + e_qmax[1], + s + ); + sp += 1u; + + if (x + 3u < stride) { + j2k_ht_process_sample(6u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[1], e_q, e_qmax[1], s); + j2k_ht_process_sample( + 7u, + height > 1u ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, + p, + &rho[1], + e_q, + e_qmax[1], + s + ); + sp += 1u; + } + + const uint c_q1 = uint((rho[0] >> 1) | (rho[0] & 1)); + const int u_q1 = j2k_ht_encode_quad_initial_row( + 4u, c_q1, rho[1], e_qmax[1], e_q, s, lep + 1u, lcxp + 1u, e_val, cx_val, mel, vlc, ms, out, vlc_table0 + ); + + if (u_q0 > 0 && u_q1 > 0) { + j2k_ht_mel_encode(mel, out, min(u_q0, u_q1) > 2); + } + j2k_ht_encode_uvlc(u_q0, u_q1, vlc, out, uvlc_table); + c_q0 = uint((rho[1] >> 1) | (rho[1] & 1)); + } else { + j2k_ht_encode_uvlc(u_q0, 0, vlc, out, uvlc_table); + c_q0 = 0u; + } + + j2k_ht_clear_quad_state(rho, e_q, e_qmax, s); + return 0; +} + +inline int j2k_ht_encode_non_initial_quad_pair( + device const int *coefficients, + uint stride, + uint width, + uint height, + uint y, + uint total_bitplanes, + uint p, + thread uint &sp, + uint x, + thread uchar *e_val, + thread uchar *cx_val, + thread uint &lep, + thread uint &lcxp, + thread int &max_e, + thread uint &c_q0, + thread int *rho, + thread int *e_q, + thread int *e_qmax, + thread uint *s, + thread J2kHtMelEncoder &mel, + thread J2kHtVlcEncoder &vlc, + thread J2kHtMagSgnEncoder &ms, + device uchar *out, + device const ushort *vlc_table1, + device const uchar *uvlc_table +) { + j2k_ht_process_sample(0u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[0], e_q, e_qmax[0], s); + j2k_ht_process_sample( + 1u, + y + 1u < height ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, + p, + &rho[0], + e_q, + e_qmax[0], + s + ); + sp += 1u; + + if (x + 1u < width) { + j2k_ht_process_sample(2u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[0], e_q, e_qmax[0], s); + j2k_ht_process_sample( + 3u, + y + 1u < height ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, + p, + &rho[0], + e_q, + e_qmax[0], + s + ); + sp += 1u; + } + + const int prev_max = max_e; + const int u_q0 = j2k_ht_encode_quad_non_initial_row( + 0u, c_q0, rho[0], e_qmax[0], prev_max, e_q, s, mel, vlc, ms, out, vlc_table1 + ); + + e_val[lep] = max(e_val[lep], uchar(e_q[1])); + lep += 1u; + max_e = int(max(e_val[lep], e_val[lep + 1u])) - 1; + e_val[lep] = uchar(e_q[3]); + cx_val[lcxp] = uchar(uint(cx_val[lcxp]) | uint((rho[0] & 2) >> 1)); + lcxp += 1u; + uint c_q1 = uint(cx_val[lcxp]) + (uint(cx_val[lcxp + 1u]) << 2u); + cx_val[lcxp] = uchar((rho[0] & 8) >> 3); + + int u_q1 = 0; + if (x + 2u < width) { + j2k_ht_process_sample(4u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[1], e_q, e_qmax[1], s); + j2k_ht_process_sample( + 5u, + y + 1u < height ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, + p, + &rho[1], + e_q, + e_qmax[1], + s + ); + sp += 1u; + + if (x + 3u < width) { + j2k_ht_process_sample(6u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[1], e_q, e_qmax[1], s); + j2k_ht_process_sample( + 7u, + y + 1u < height ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, + p, + &rho[1], + e_q, + e_qmax[1], + s + ); + sp += 1u; + } + + c_q1 |= uint((rho[0] & 4) >> 1); + c_q1 |= uint((rho[0] & 8) >> 2); + u_q1 = j2k_ht_encode_quad_non_initial_row( + 4u, c_q1, rho[1], e_qmax[1], max_e, e_q, s, mel, vlc, ms, out, vlc_table1 + ); + + e_val[lep] = max(e_val[lep], uchar(e_q[5])); + lep += 1u; + max_e = int(max(e_val[lep], e_val[lep + 1u])) - 1; + e_val[lep] = uchar(e_q[7]); + cx_val[lcxp] = uchar(uint(cx_val[lcxp]) | uint((rho[1] & 2) >> 1)); + lcxp += 1u; + c_q0 = uint(cx_val[lcxp]) + (uint(cx_val[lcxp + 1u]) << 2u); + cx_val[lcxp] = uchar((rho[1] & 8) >> 3); + c_q0 |= uint((rho[1] & 4) >> 1); + c_q0 |= uint((rho[1] & 8) >> 2); + } else { + c_q0 = 0u; + } + + j2k_ht_encode_uvlc_non_initial(u_q0, u_q1, vlc, out, uvlc_table); + j2k_ht_clear_quad_state(rho, e_q, e_qmax, s); + return 0; +} + +inline void j2k_ht_terminate_mel_vlc( + thread J2kHtMelEncoder &mel, + thread J2kHtVlcEncoder &vlc, + device uchar *out +) { + if (mel.run > 0u) { + j2k_ht_mel_emit_bit(mel, out, true); + } + + mel.tmp = uchar(uint(mel.tmp) << mel.remaining_bits); + const uchar mel_mask = uchar((0xFFu << mel.remaining_bits) & 0xFFu); + const uchar vlc_mask = vlc.used_bits == 0u + ? uchar(0u) + : uchar((1u << vlc.used_bits) - 1u); + + if ((mel_mask | vlc_mask) == uchar(0u)) { + return; + } + + const uchar fused = mel.tmp | vlc.tmp; + const bool fused_ok = + ((((fused ^ mel.tmp) & mel_mask) | ((fused ^ vlc.tmp) & vlc_mask)) == uchar(0u)) && + fused != uchar(0xFFu); + + if (fused_ok && vlc.pos > 1u) { + if (mel.pos >= mel.capacity) { + mel.failed = 1u; + return; + } + out[mel.offset + mel.pos] = fused; + mel.pos += 1u; + } else { + if (mel.pos >= mel.capacity || vlc.pos >= vlc.capacity) { + mel.failed = 1u; + vlc.failed = 1u; + return; + } + out[mel.offset + mel.pos] = mel.tmp; + mel.pos += 1u; + const uint write_index = vlc.capacity - 1u - vlc.pos; + out[vlc.offset + write_index] = vlc.tmp; + vlc.pos += 1u; + } +} + +inline void j2k_encode_ht_code_block_impl_with_max_and_assembly( + device const int *coefficients, + device uchar *out, + J2kHtEncodeParams params, + device const ushort *vlc_table0, + device const ushort *vlc_table1, + device const uchar *uvlc_table, + device J2kHtEncodeStatus *status, + uint max_magnitude, + bool assemble_final +) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u, 0u, 0u); + + if (params.width == 0u || params.height == 0u || + params.total_bitplanes == 0u || params.total_bitplanes > J2K_HT_MAX_BITPLANES || + params.width * params.height > J2K_HT_MAX_SAMPLES || + params.output_capacity < j2k_ht_output_size(params)) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 1u, 0u, 0u, 0u); + return; + } + + if (max_magnitude == 0u) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_OK, 0u, 0u, 0u, params.total_bitplanes); + return; + } + + const uint block_bitplanes = 32u - clz(max_magnitude); + if (block_bitplanes > params.total_bitplanes) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_FAIL, 2u, 0u, 0u, 0u); + return; + } + + const uint missing_msbs = params.total_bitplanes - 1u; + const uint p = 30u - missing_msbs; + + thread J2kHtMelEncoder mel; + thread J2kHtVlcEncoder vlc; + thread J2kHtMagSgnEncoder ms; + j2k_ht_mel_init(mel, params); + j2k_ht_vlc_init(vlc, out, params); + j2k_ht_ms_init(ms, params); + + thread uchar e_val[513]; + thread uchar cx_val[513]; + for (uint idx = 0u; idx < 513u; ++idx) { + e_val[idx] = uchar(0u); + cx_val[idx] = uchar(0u); + } + + thread int e_qmax[2]; + thread int e_q[8]; + thread int rho[2]; + thread uint s[8]; + j2k_ht_clear_quad_state(rho, e_q, e_qmax, s); + + uint c_q0 = 0u; + uint sp = 0u; + uint x = 0u; + while (x < params.width) { + j2k_ht_encode_first_quad_pair( + coefficients, + params.width, + params.height, + params.total_bitplanes, + p, + sp, + x, + e_val, + cx_val, + c_q0, + rho, + e_q, + e_qmax, + s, + mel, + vlc, + ms, + out, + vlc_table0, + uvlc_table + ); + x += 4u; + } + + const uint e_val_sentinel = (params.width + 1u) / 2u + 1u; + if (e_val_sentinel < 513u) { + e_val[e_val_sentinel] = uchar(0u); + } + + uint y = 2u; + while (y < params.height) { + uint lep = 0u; + int max_e = int(max(e_val[lep], e_val[lep + 1u])) - 1; + e_val[lep] = uchar(0u); + + uint lcxp = 0u; + c_q0 = uint(cx_val[lcxp]) + (uint(cx_val[lcxp + 1u]) << 2u); + cx_val[lcxp] = uchar(0u); + + sp = y * params.width; + x = 0u; + while (x < params.width) { + j2k_ht_encode_non_initial_quad_pair( + coefficients, + params.width, + params.width, + params.height, + y, + params.total_bitplanes, + p, + sp, + x, + e_val, + cx_val, + lep, + lcxp, + max_e, + c_q0, + rho, + e_q, + e_qmax, + s, + mel, + vlc, + ms, + out, + vlc_table1, + uvlc_table + ); + x += 4u; + } + + y += 2u; + } + + j2k_ht_terminate_mel_vlc(mel, vlc, out); + j2k_ht_ms_terminate(ms, out); + + if (mel.failed != 0u || vlc.failed != 0u || ms.failed != 0u) { + const uint fail_detail = ms.failed != 0u ? 32u : (vlc.failed != 0u ? 31u : 30u); + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_FAIL, fail_detail, 0u, 0u, 0u); + return; + } + + const uint ms_len = ms.pos; + const uint mel_len = mel.pos; + const uint vlc_len = vlc.pos; + const uint total_len = ms_len + mel_len + vlc_len; + if (total_len < 2u || total_len > params.output_capacity) { + j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_FAIL, 4u, 0u, 0u, 0u); + return; + } + + if (assemble_final) { + for (uint idx = 0u; idx < mel_len; ++idx) { + out[ms_len + idx] = out[mel.offset + idx]; + } + const uint vlc_start = vlc.capacity - vlc_len; + for (uint idx = 0u; idx < vlc_len; ++idx) { + out[ms_len + mel_len + idx] = out[vlc.offset + vlc_start + idx]; + } + + const uint last = total_len - 1u; + const uint prev = total_len - 2u; + const uint locator_bytes = mel_len + vlc_len; + out[last] = uchar(locator_bytes >> 4u); + out[prev] = uchar((out[prev] & uchar(0xF0u)) | uchar(locator_bytes & 0x0Fu)); + } + + j2k_set_ht_encode_status_with_segments( + status, + J2K_ENCODE_STATUS_OK, + 0u, + total_len, + 1u, + missing_msbs, + ms_len, + mel_len, + vlc_len + ); +} + +inline void j2k_encode_ht_code_block_impl_with_max( + device const int *coefficients, + device uchar *out, + J2kHtEncodeParams params, + device const ushort *vlc_table0, + device const ushort *vlc_table1, + device const uchar *uvlc_table, + device J2kHtEncodeStatus *status, + uint max_magnitude +) { + j2k_encode_ht_code_block_impl_with_max_and_assembly( + coefficients, + out, + params, + vlc_table0, + vlc_table1, + uvlc_table, + status, + max_magnitude, + true + ); +} + +inline void j2k_encode_ht_code_block_impl( + device const int *coefficients, + device uchar *out, + J2kHtEncodeParams params, + device const ushort *vlc_table0, + device const ushort *vlc_table1, + device const uchar *uvlc_table, + device J2kHtEncodeStatus *status +) { + uint max_magnitude = 0u; + for (uint y = 0u; y < params.height; ++y) { + for (uint x = 0u; x < params.width; ++x) { + max_magnitude = max(max_magnitude, j2k_classic_magnitude(coefficients[y * params.width + x])); + } + } + j2k_encode_ht_code_block_impl_with_max( + coefficients, + out, + params, + vlc_table0, + vlc_table1, + uvlc_table, + status, + max_magnitude + ); +} + +struct J2kHtEncodeBatchJob { + uint coefficient_offset; + uint output_offset; + uint width; + uint height; + uint total_bitplanes; + uint output_capacity; +}; + +kernel void j2k_encode_ht_code_block( + device const int *coefficients [[buffer(0)]], + device uchar *out [[buffer(1)]], + constant J2kHtEncodeParams ¶ms [[buffer(2)]], + device const ushort *vlc_table0 [[buffer(3)]], + device const ushort *vlc_table1 [[buffer(4)]], + device const uchar *uvlc_table [[buffer(5)]], + device J2kHtEncodeStatus *status [[buffer(6)]], + uint gid [[thread_position_in_grid]] +) { + if (gid != 0u) { + return; + } + j2k_encode_ht_code_block_impl( + coefficients, + out, + params, + vlc_table0, + vlc_table1, + uvlc_table, + status + ); +} + +kernel void j2k_encode_ht_code_blocks( + device const int *coefficients [[buffer(0)]], + device uchar *out [[buffer(1)]], + device const J2kHtEncodeBatchJob *jobs [[buffer(2)]], + device const ushort *vlc_table0 [[buffer(3)]], + device const ushort *vlc_table1 [[buffer(4)]], + device const uchar *uvlc_table [[buffer(5)]], + device J2kHtEncodeStatus *statuses [[buffer(6)]], + constant uint &job_count [[buffer(7)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= job_count) { + return; + } + const J2kHtEncodeBatchJob job = jobs[gid]; + J2kHtEncodeParams params; + params.width = job.width; + params.height = job.height; + params.total_bitplanes = job.total_bitplanes; + params.output_capacity = job.output_capacity; + j2k_encode_ht_code_block_impl( + coefficients + job.coefficient_offset, + out + job.output_offset, + params, + vlc_table0, + vlc_table1, + uvlc_table, + statuses + gid + ); +} + +struct J2kPacketEncodeParams { + uint resolution_count; + uint num_layers; + uint num_components; + uint code_block_count; + uint subband_count; + uint descriptor_count; + uint output_capacity; + uint header_capacity; + uint scratch_node_capacity; +}; + +struct J2kPacketDescriptor { + uint packet_index; + uint state_index; + uint layer; + uint resolution; + uint component; + uint precinct_lo; + uint precinct_hi; + uint state_block_offset; +}; + +struct J2kPacketResolution { + uint subband_offset; + uint subband_count; +}; + +struct J2kPacketSubband { + uint block_offset; + uint block_count; + uint num_cbs_x; + uint num_cbs_y; +}; + +struct J2kPacketBlock { + uint data_offset; + uint data_len; + uint num_coding_passes; + uint num_zero_bitplanes; + uint previously_included; + uint l_block; + uint block_coding_mode; + uint reserved0; +}; + +struct J2kResidentPacketBlock { + uint tier1_job_index; + uint previously_included; + uint l_block; + uint block_coding_mode; +}; + +struct J2kResidentPacketBlockParams { + uint block_count; + uint tier1_job_count; +}; + +struct J2kPacketStateBlock { + uint previously_included; + uint l_block; +}; + +struct J2kPacketEncodeStatus { + uint code; + uint detail; + uint data_len; + uint reserved0; + uint payload_copy_bytes; + uint payload_copy_small_jobs; + uint payload_copy_medium_jobs; + uint payload_copy_large_jobs; +}; + +struct J2kLosslessCodestreamAssemblyParams { + uint width; + uint height; + uint num_components; + uint bit_depth; + uint signed_samples; + uint num_decomposition_levels; + uint use_mct; + uint guard_bits; + uint progression_order; + uint write_tlm; + uint high_throughput; + uint code_block_style; + uint code_block_width_exp; + uint code_block_height_exp; + uint output_capacity; +}; + +struct J2kCodestreamAssemblyStatus { + uint code; + uint detail; + uint data_len; + uint reserved0; +}; + +struct J2kPacketBitWriter { + device uchar *data; + uint capacity; + uint len; + uint buffer; + uint bits_in_buffer; + uint last_byte_was_ff; + uint failed; +}; + +inline void j2k_set_packet_status(device J2kPacketEncodeStatus *status, uint code, uint detail, uint len) { + status->code = code; + status->detail = detail; + status->data_len = len; + status->reserved0 = 0u; + status->payload_copy_bytes = 0u; + status->payload_copy_small_jobs = 0u; + status->payload_copy_medium_jobs = 0u; + status->payload_copy_large_jobs = 0u; +} + +inline void j2k_set_packet_status_payload_copy( + device J2kPacketEncodeStatus *status, + uint code, + uint detail, + uint len, + uint payload_copy_bytes, + uint payload_copy_small_jobs, + uint payload_copy_medium_jobs, + uint payload_copy_large_jobs +) { + status->code = code; + status->detail = detail; + status->data_len = len; + status->reserved0 = 0u; + status->payload_copy_bytes = payload_copy_bytes; + status->payload_copy_small_jobs = payload_copy_small_jobs; + status->payload_copy_medium_jobs = payload_copy_medium_jobs; + status->payload_copy_large_jobs = payload_copy_large_jobs; +} + +inline void j2k_set_codestream_status( + device J2kCodestreamAssemblyStatus *status, + uint code, + uint detail, + uint len +) { + status->code = code; + status->detail = detail; + status->data_len = len; + status->reserved0 = 0u; +} + +inline bool j2k_codestream_write_u8( + device uchar *out, + uint capacity, + thread uint &cursor, + uint value +) { + if (cursor >= capacity) { + return false; + } + out[cursor] = uchar(value & 0xFFu); + cursor += 1u; + return true; +} + +inline bool j2k_codestream_write_u16( + device uchar *out, + uint capacity, + thread uint &cursor, + uint value +) { + return j2k_codestream_write_u8(out, capacity, cursor, value >> 8u) && + j2k_codestream_write_u8(out, capacity, cursor, value); +} + +inline bool j2k_codestream_write_u32( + device uchar *out, + uint capacity, + thread uint &cursor, + uint value +) { + return j2k_codestream_write_u8(out, capacity, cursor, value >> 24u) && + j2k_codestream_write_u8(out, capacity, cursor, value >> 16u) && + j2k_codestream_write_u8(out, capacity, cursor, value >> 8u) && + j2k_codestream_write_u8(out, capacity, cursor, value); +} + +inline bool j2k_codestream_write_marker( + device uchar *out, + uint capacity, + thread uint &cursor, + uint marker +) { + return j2k_codestream_write_u8(out, capacity, cursor, 0xFFu) && + j2k_codestream_write_u8(out, capacity, cursor, marker); +} + +kernel void j2k_assemble_lossless_classic_codestream( + device const uchar *tile_data [[buffer(0)]], + device const J2kPacketEncodeStatus *tile_status [[buffer(1)]], + device uchar *out [[buffer(2)]], + constant J2kLosslessCodestreamAssemblyParams ¶ms [[buffer(3)]], + device J2kCodestreamAssemblyStatus *status [[buffer(4)]], + uint gid [[thread_position_in_grid]] +) { + if (gid != 0u) { + return; + } + + j2k_set_codestream_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u); + const J2kPacketEncodeStatus packet_status = tile_status[0]; + if (packet_status.code != J2K_ENCODE_STATUS_OK) { + j2k_set_codestream_status(status, J2K_ENCODE_STATUS_FAIL, packet_status.detail, 0u); + return; + } + if (params.num_components == 0u || params.num_components > 255u || + params.bit_depth == 0u || params.bit_depth > 16u || + params.num_decomposition_levels > 31u) { + j2k_set_codestream_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 1u, 0u); + return; + } + + const uint tile_len = packet_status.data_len; + const uint tile_part_len = 14u + tile_len; + uint cursor = 0u; + bool ok = true; + + ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x4Fu); + + ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x51u); + const uint siz_len = 38u + 3u * params.num_components; + ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, siz_len); + ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, params.width); + ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, params.height); + ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, params.width); + ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, params.height); + ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, params.num_components); + const uint ssiz = (params.bit_depth - 1u) | (params.signed_samples != 0u ? 0x80u : 0u); + for (uint comp = 0u; comp < params.num_components && ok; ++comp) { + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, ssiz); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 1u); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 1u); + } + + if (params.high_throughput != 0u) { + const uint magnitude_bits = params.bit_depth - 1u; + const uint bp = magnitude_bits <= 8u ? 0u : + (magnitude_bits < 28u ? magnitude_bits - 8u : 13u + (magnitude_bits >> 2u)); + ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x50u); + ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 8u); + ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, 0x00020000u); + ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, bp); + } + + ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x52u); + ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 12u); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.progression_order); + ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 1u); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.use_mct != 0u ? 1u : 0u); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.num_decomposition_levels); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.code_block_width_exp); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.code_block_height_exp); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.code_block_style); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 1u); + + ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x5Cu); + const uint qcd_steps = 1u + 3u * params.num_decomposition_levels; + ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 3u + qcd_steps); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.guard_bits << 5u); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.bit_depth << 3u); + for (uint level = 0u; level < params.num_decomposition_levels && ok; ++level) { + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, (params.bit_depth + 1u) << 3u); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, (params.bit_depth + 1u) << 3u); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, (params.bit_depth + 2u) << 3u); + } + + if (params.write_tlm != 0u) { + ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x55u); + ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 10u); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 0x22u); + ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, tile_part_len); + } + + ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x90u); + ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 10u); + ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, tile_part_len); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 1u); + ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x93u); + + if (!ok || cursor + tile_len + 2u > params.output_capacity) { + j2k_set_codestream_status(status, J2K_ENCODE_STATUS_FAIL, 2u, cursor); + return; + } + for (uint idx = 0u; idx < tile_len; ++idx) { + out[cursor + idx] = tile_data[idx]; + } + cursor += tile_len; + ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0xD9u); + if (!ok) { + j2k_set_codestream_status(status, J2K_ENCODE_STATUS_FAIL, 3u, cursor); + return; + } + + j2k_set_codestream_status(status, J2K_ENCODE_STATUS_OK, 0u, cursor); +} + +struct J2kBatchedCodestreamAssemblyJob { + uint tile_data_offset; + uint codestream_offset; + uint width; + uint height; + uint num_components; + uint bit_depth; + uint signed_samples; + uint num_decomposition_levels; + uint use_mct; + uint guard_bits; + uint progression_order; + uint write_tlm; + uint high_throughput; + uint code_block_style; + uint code_block_width_exp; + uint code_block_height_exp; + uint output_capacity; +}; + +kernel void j2k_assemble_lossless_codestream_batched( + device const uchar *tile_data [[buffer(0)]], + device const J2kPacketEncodeStatus *tile_status [[buffer(1)]], + device uchar *out [[buffer(2)]], + device const J2kBatchedCodestreamAssemblyJob *jobs [[buffer(3)]], + device J2kCodestreamAssemblyStatus *status [[buffer(4)]], + uint gid [[thread_position_in_grid]] +) { + const J2kBatchedCodestreamAssemblyJob job = jobs[gid]; + device J2kCodestreamAssemblyStatus *tile_status_out = status + gid; + device uchar *tile_out = out + job.codestream_offset; + const J2kPacketEncodeStatus packet_status = tile_status[gid]; + + j2k_set_codestream_status(tile_status_out, J2K_ENCODE_STATUS_FAIL, 0u, 0u); + if (packet_status.code != J2K_ENCODE_STATUS_OK) { + j2k_set_codestream_status(tile_status_out, J2K_ENCODE_STATUS_FAIL, packet_status.detail, 0u); + return; + } + if (job.num_components == 0u || job.num_components > 255u || + job.bit_depth == 0u || job.bit_depth > 16u || + job.num_decomposition_levels > 31u) { + j2k_set_codestream_status(tile_status_out, J2K_ENCODE_STATUS_UNSUPPORTED, 1u, 0u); + return; + } + + const uint tile_len = packet_status.data_len; + const uint tile_part_len = 14u + tile_len; + uint cursor = 0u; + bool ok = true; + + ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x4Fu); + + ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x51u); + const uint siz_len = 38u + 3u * job.num_components; + ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, siz_len); + ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, job.width); + ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, job.height); + ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, job.width); + ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, job.height); + ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, job.num_components); + const uint ssiz = (job.bit_depth - 1u) | (job.signed_samples != 0u ? 0x80u : 0u); + for (uint comp = 0u; comp < job.num_components && ok; ++comp) { + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, ssiz); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 1u); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 1u); + } + + if (job.high_throughput != 0u) { + const uint magnitude_bits = job.bit_depth - 1u; + const uint bp = magnitude_bits <= 8u ? 0u : + (magnitude_bits < 28u ? magnitude_bits - 8u : 13u + (magnitude_bits >> 2u)); + ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x50u); + ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 8u); + ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, 0x00020000u); + ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, bp); + } + + ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x52u); + ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 12u); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.progression_order); + ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 1u); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.use_mct != 0u ? 1u : 0u); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.num_decomposition_levels); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.code_block_width_exp); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.code_block_height_exp); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.code_block_style); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 1u); + + ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x5Cu); + const uint qcd_steps = 1u + 3u * job.num_decomposition_levels; + ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 3u + qcd_steps); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.guard_bits << 5u); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.bit_depth << 3u); + for (uint level = 0u; level < job.num_decomposition_levels && ok; ++level) { + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, (job.bit_depth + 1u) << 3u); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, (job.bit_depth + 1u) << 3u); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, (job.bit_depth + 2u) << 3u); + } + + if (job.write_tlm != 0u) { + ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x55u); + ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 10u); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 0x22u); + ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, tile_part_len); + } + + ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x90u); + ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 10u); + ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, tile_part_len); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 0u); + ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 1u); + ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x93u); + const uint payload_offset = cursor; + + if (!ok || cursor + tile_len + 2u > job.output_capacity) { + j2k_set_codestream_status(tile_status_out, J2K_ENCODE_STATUS_FAIL, 2u, cursor); + return; + } + cursor += tile_len; + ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0xD9u); + if (!ok) { + j2k_set_codestream_status(tile_status_out, J2K_ENCODE_STATUS_FAIL, 3u, cursor); + return; + } + + j2k_set_codestream_status(tile_status_out, J2K_ENCODE_STATUS_OK, payload_offset, cursor); +} + +inline J2kPacketBlock j2k_classic_packet_block_from_resident( + device const J2kResidentPacketBlock *resident_blocks, + device const J2kClassicEncodeBatchJob *tier1_jobs, + device const J2kClassicEncodeStatus *tier1_statuses, + uint tier1_job_count, + uint block_index +) { + const J2kResidentPacketBlock resident = resident_blocks[block_index]; + J2kPacketBlock packet; + packet.data_offset = 0u; + packet.data_len = 0u; + packet.num_coding_passes = 1u; + packet.num_zero_bitplanes = 0u; + packet.previously_included = resident.previously_included; + packet.l_block = resident.l_block; + packet.block_coding_mode = 0xFFFFFFFFu; + packet.reserved0 = 0u; + + if (resident.tier1_job_index < tier1_job_count) { + const J2kClassicEncodeBatchJob job = tier1_jobs[resident.tier1_job_index]; + const J2kClassicEncodeStatus tier1_status = tier1_statuses[resident.tier1_job_index]; + if (tier1_status.code == J2K_ENCODE_STATUS_OK && + tier1_status.data_len <= job.output_capacity && + tier1_status.reserved0 <= 1u && + tier1_status.reserved1 == 0u && + tier1_status.data_len + tier1_status.reserved0 >= tier1_status.data_len && + tier1_status.data_len + tier1_status.reserved0 <= job.output_capacity && + tier1_status.segment_count <= job.segment_capacity) { + packet.data_offset = job.output_offset + tier1_status.reserved0; + packet.data_len = tier1_status.data_len; + packet.num_coding_passes = tier1_status.number_of_coding_passes; + packet.num_zero_bitplanes = tier1_status.missing_bit_planes; + packet.block_coding_mode = resident.block_coding_mode; + } else { + packet.reserved0 = tier1_status.detail; + } + } + + return packet; +} + +inline J2kPacketBlock j2k_ht_packet_block_from_resident( + device const J2kResidentPacketBlock *resident_blocks, + device const J2kHtEncodeBatchJob *tier1_jobs, + device const J2kHtEncodeStatus *tier1_statuses, + uint tier1_job_count, + uint block_index +) { + const J2kResidentPacketBlock resident = resident_blocks[block_index]; + J2kPacketBlock packet; + packet.data_offset = 0u; + packet.data_len = 0u; + packet.num_coding_passes = 1u; + packet.num_zero_bitplanes = 0u; + packet.previously_included = resident.previously_included; + packet.l_block = resident.l_block; + packet.block_coding_mode = 0xFFFFFFFFu; + packet.reserved0 = 0u; + + if (resident.tier1_job_index < tier1_job_count) { + const J2kHtEncodeBatchJob job = tier1_jobs[resident.tier1_job_index]; + const J2kHtEncodeStatus tier1_status = tier1_statuses[resident.tier1_job_index]; + if (tier1_status.code == J2K_ENCODE_STATUS_OK && + tier1_status.data_len <= job.output_capacity) { + packet.data_offset = job.output_offset; + packet.data_len = tier1_status.data_len; + packet.num_coding_passes = tier1_status.num_coding_passes; + packet.num_zero_bitplanes = tier1_status.num_zero_bitplanes; + packet.block_coding_mode = resident.block_coding_mode; + } else { + packet.reserved0 = tier1_status.detail; + } + } + + return packet; +} + +kernel void j2k_prepare_packet_blocks_from_classic_status( + device const J2kResidentPacketBlock *resident_blocks [[buffer(0)]], + device const J2kClassicEncodeBatchJob *tier1_jobs [[buffer(1)]], + device const J2kClassicEncodeStatus *tier1_statuses [[buffer(2)]], + device J2kPacketBlock *packet_blocks [[buffer(3)]], + constant J2kResidentPacketBlockParams ¶ms [[buffer(4)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= params.block_count) { + return; + } + + packet_blocks[gid] = j2k_classic_packet_block_from_resident( + resident_blocks, + tier1_jobs, + tier1_statuses, + params.tier1_job_count, + gid + ); +} + +kernel void j2k_prepare_packet_blocks_from_ht_status( + device const J2kResidentPacketBlock *resident_blocks [[buffer(0)]], + device const J2kHtEncodeBatchJob *tier1_jobs [[buffer(1)]], + device const J2kHtEncodeStatus *tier1_statuses [[buffer(2)]], + device J2kPacketBlock *packet_blocks [[buffer(3)]], + constant J2kResidentPacketBlockParams ¶ms [[buffer(4)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= params.block_count) { + return; + } + + packet_blocks[gid] = j2k_ht_packet_block_from_resident( + resident_blocks, + tier1_jobs, + tier1_statuses, + params.tier1_job_count, + gid + ); +} + +inline void j2k_packet_writer_init(thread J2kPacketBitWriter &writer, device uchar *data, uint capacity) { + writer.data = data; + writer.capacity = capacity; + writer.len = 0u; + writer.buffer = 0u; + writer.bits_in_buffer = 0u; + writer.last_byte_was_ff = 0u; + writer.failed = 0u; +} + +inline void j2k_packet_flush_byte(thread J2kPacketBitWriter &writer) { + const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; + const uchar byte = uchar(writer.buffer >> (writer.bits_in_buffer - limit)); + if (writer.len >= writer.capacity) { + writer.failed = 1u; + return; + } + writer.data[writer.len] = byte; + writer.len += 1u; + writer.last_byte_was_ff = byte == uchar(0xFFu) ? 1u : 0u; + writer.bits_in_buffer -= limit; + writer.buffer &= writer.bits_in_buffer == 0u ? 0u : ((1u << writer.bits_in_buffer) - 1u); +} + +inline void j2k_packet_write_bit(thread J2kPacketBitWriter &writer, uint bit) { + writer.buffer = (writer.buffer << 1u) | (bit & 1u); + writer.bits_in_buffer += 1u; + const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; + if (writer.bits_in_buffer >= limit) { + j2k_packet_flush_byte(writer); + } +} + +inline void j2k_packet_write_bits(thread J2kPacketBitWriter &writer, uint value, uint count) { + for (int bit = int(count) - 1; bit >= 0; --bit) { + j2k_packet_write_bit(writer, (value >> uint(bit)) & 1u); + } +} + +inline void j2k_packet_writer_finish(thread J2kPacketBitWriter &writer) { + if (writer.bits_in_buffer > 0u) { + const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; + const uint shift = limit - writer.bits_in_buffer; + const uchar byte = uchar(writer.buffer << shift); + if (writer.len >= writer.capacity) { + writer.failed = 1u; + return; + } + writer.data[writer.len] = byte; + writer.len += 1u; + writer.last_byte_was_ff = byte == uchar(0xFFu) ? 1u : 0u; + writer.buffer = 0u; + writer.bits_in_buffer = 0u; + } +} + +inline uint j2k_packet_ilog2(uint value) { + return value == 0u ? 0u : 31u - clz(value); +} + +inline bool j2k_packet_value_fits(uint value, uint bits) { + return bits >= 32u || value < (1u << bits); +} + +inline uint j2k_packet_bits_for_length(uint l_block, uint passes) { + const uint log2_passes = passes <= 1u ? 0u : j2k_packet_ilog2(passes); + return l_block + log2_passes; +} + +inline uint j2k_packet_bits_for_ht_length(uint l_block, uint passes) { + const uint placeholder_groups = (passes > 0u ? passes - 1u : 0u) / 3u; + const uint placeholder_passes = placeholder_groups * 3u; + return l_block + j2k_packet_ilog2(placeholder_passes + 1u); +} + +inline void j2k_packet_encode_num_passes(uint passes, thread J2kPacketBitWriter &writer) { + if (passes == 1u) { + j2k_packet_write_bit(writer, 0u); + } else if (passes == 2u) { + j2k_packet_write_bits(writer, 0b10u, 2u); + } else if (passes == 3u) { + j2k_packet_write_bits(writer, 0b1100u, 4u); + } else if (passes == 4u) { + j2k_packet_write_bits(writer, 0b1101u, 4u); + } else if (passes == 5u) { + j2k_packet_write_bits(writer, 0b1110u, 4u); + } else if (passes <= 36u) { + j2k_packet_write_bits(writer, 0b1111u, 4u); + j2k_packet_write_bits(writer, passes - 6u, 5u); + } else { + j2k_packet_write_bits(writer, 0x1FFu, 9u); + j2k_packet_write_bits(writer, passes - 37u, 7u); + } +} + +inline void j2k_packet_encode_num_ht_passes(uint passes, thread J2kPacketBitWriter &writer) { + if (passes == 1u) { + j2k_packet_write_bit(writer, 0u); + } else if (passes == 2u) { + j2k_packet_write_bits(writer, 0b10u, 2u); + } else if (passes <= 5u) { + j2k_packet_write_bits(writer, 0b11u, 2u); + j2k_packet_write_bits(writer, passes - 3u, 2u); + } else if (passes <= 36u) { + j2k_packet_write_bits(writer, 0b11u, 2u); + j2k_packet_write_bits(writer, 0b11u, 2u); + j2k_packet_write_bits(writer, passes - 6u, 5u); + } else { + j2k_packet_write_bits(writer, 0b11u, 2u); + j2k_packet_write_bits(writer, 0b11u, 2u); + j2k_packet_write_bits(writer, 31u, 5u); + j2k_packet_write_bits(writer, passes - 37u, 7u); + } +} + +inline void j2k_packet_encode_length( + uint length, + thread uint &l_block, + uint num_bits, + thread J2kPacketBitWriter &writer +) { + while (!j2k_packet_value_fits(length, num_bits)) { + j2k_packet_write_bit(writer, 1u); + l_block += 1u; + num_bits += 1u; + } + j2k_packet_write_bit(writer, 0u); + j2k_packet_write_bits(writer, length, num_bits); +} + +inline bool j2k_packet_encode_classic_segment_lengths_resident( + const J2kPacketBlock block, + const J2kClassicEncodeBatchJob tier1_job, + const J2kClassicEncodeStatus tier1_status, + device const J2kClassicSegment *tier1_segments, + thread uint &l_block, + thread J2kPacketBitWriter &writer +) { + if (tier1_status.segment_count <= 1u) { + const uint num_bits = j2k_packet_bits_for_length(l_block, block.num_coding_passes); + j2k_packet_encode_length(block.data_len, l_block, num_bits, writer); + return true; + } + + uint segment_data_sum = 0u; + uint segment_pass_sum = 0u; + uint required_l_block = l_block; + for (uint segment_idx = 0u; segment_idx < tier1_status.segment_count; ++segment_idx) { + const J2kClassicSegment segment = + tier1_segments[tier1_job.segment_offset + segment_idx]; + if (segment.start_coding_pass >= segment.end_coding_pass || + segment.data_offset != segment_data_sum || + segment_data_sum > 0xffffffffu - segment.data_length) { + return false; + } + const uint segment_passes = segment.end_coding_pass - segment.start_coding_pass; + segment_data_sum += segment.data_length; + segment_pass_sum += segment_passes; + while (!j2k_packet_value_fits( + segment.data_length, + j2k_packet_bits_for_length(required_l_block, segment_passes))) { + required_l_block += 1u; + } + } + if (segment_data_sum != block.data_len || segment_pass_sum != block.num_coding_passes) { + return false; + } + + while (l_block < required_l_block) { + j2k_packet_write_bit(writer, 1u); + l_block += 1u; + } + j2k_packet_write_bit(writer, 0u); + + for (uint segment_idx = 0u; segment_idx < tier1_status.segment_count; ++segment_idx) { + const J2kClassicSegment segment = + tier1_segments[tier1_job.segment_offset + segment_idx]; + const uint segment_passes = segment.end_coding_pass - segment.start_coding_pass; + const uint length_bits = j2k_packet_bits_for_length(l_block, segment_passes); + j2k_packet_write_bits(writer, segment.data_length, length_bits); + } + return writer.failed == 0u; +} + +inline uint j2k_packet_tree_offsets( + uint width, + uint height, + thread uint *level_offsets, + thread uint *level_widths, + thread uint *level_heights, + thread uint &levels +) { + uint total = 0u; + uint w = width; + uint h = height; + levels = 0u; + while (true) { + level_offsets[levels] = total; + level_widths[levels] = w; + level_heights[levels] = h; + total += w * h; + levels += 1u; + if (w <= 1u && h <= 1u) { + break; + } + w = (w + 1u) / 2u; + h = (h + 1u) / 2u; + } + return total; +} + +inline bool j2k_packet_prepare_tree( + device const J2kPacketBlock *blocks, + uint block_offset, + uint block_count, + uint num_cbs_x, + uint num_cbs_y, + bool zero_bitplanes, + uint inclusion_layer, + device uint *value, + device uint *current, + device uint *known, + uint node_capacity, + thread uint *level_offsets, + thread uint *level_widths, + thread uint *level_heights, + thread uint &levels +) { + if (num_cbs_x == 0u || num_cbs_y == 0u || num_cbs_x * num_cbs_y != block_count) { + return false; + } + const uint node_count = + j2k_packet_tree_offsets(num_cbs_x, num_cbs_y, level_offsets, level_widths, level_heights, levels); + if (node_count > node_capacity || levels > 16u) { + return false; + } + for (uint idx = 0u; idx < node_count; ++idx) { + value[idx] = 0u; + current[idx] = 0u; + known[idx] = 0u; + } + for (uint idx = 0u; idx < block_count; ++idx) { + const J2kPacketBlock block = blocks[block_offset + idx]; + value[idx] = zero_bitplanes + ? block.num_zero_bitplanes + : (block.num_coding_passes > 0u ? inclusion_layer : 0x7FFFFFFFu); + } + for (uint level = 1u; level < levels; ++level) { + const uint prev_w = level_widths[level - 1u]; + const uint prev_h = level_heights[level - 1u]; + const uint cur_w = level_widths[level]; + const uint cur_h = level_heights[level]; + for (uint py = 0u; py < cur_h; ++py) { + for (uint px = 0u; px < cur_w; ++px) { + uint min_value = 0xFFFFFFFFu; + for (uint dy = 0u; dy < 2u; ++dy) { + const uint cy = py * 2u + dy; + if (cy >= prev_h) { + continue; + } + for (uint dx = 0u; dx < 2u; ++dx) { + const uint cx = px * 2u + dx; + if (cx >= prev_w) { + continue; + } + const uint child = level_offsets[level - 1u] + cy * prev_w + cx; + min_value = min(min_value, value[child]); + } + } + value[level_offsets[level] + py * cur_w + px] = min_value; + } + } + } + return true; +} + +inline bool j2k_packet_prepare_tree_resident_classic( + device const J2kResidentPacketBlock *resident_blocks, + device const J2kClassicEncodeBatchJob *tier1_jobs, + device const J2kClassicEncodeStatus *tier1_statuses, + uint tier1_job_count, + uint block_offset, + uint block_count, + uint num_cbs_x, + uint num_cbs_y, + bool zero_bitplanes, + uint inclusion_layer, + device uint *value, + device uint *current, + device uint *known, + uint node_capacity, + thread uint *level_offsets, + thread uint *level_widths, + thread uint *level_heights, + thread uint &levels +) { + if (num_cbs_x == 0u || num_cbs_y == 0u || num_cbs_x * num_cbs_y != block_count) { + return false; + } + const uint node_count = + j2k_packet_tree_offsets(num_cbs_x, num_cbs_y, level_offsets, level_widths, level_heights, levels); + if (node_count > node_capacity || levels > 16u) { + return false; + } + for (uint idx = 0u; idx < node_count; ++idx) { + value[idx] = 0u; + current[idx] = 0u; + known[idx] = 0u; + } + for (uint idx = 0u; idx < block_count; ++idx) { + const J2kPacketBlock block = j2k_classic_packet_block_from_resident( + resident_blocks, + tier1_jobs, + tier1_statuses, + tier1_job_count, + block_offset + idx + ); + value[idx] = zero_bitplanes + ? block.num_zero_bitplanes + : (block.num_coding_passes > 0u ? inclusion_layer : 0x7FFFFFFFu); + } + for (uint level = 1u; level < levels; ++level) { + const uint prev_w = level_widths[level - 1u]; + const uint prev_h = level_heights[level - 1u]; + const uint cur_w = level_widths[level]; + const uint cur_h = level_heights[level]; + for (uint py = 0u; py < cur_h; ++py) { + for (uint px = 0u; px < cur_w; ++px) { + uint min_value = 0xFFFFFFFFu; + for (uint dy = 0u; dy < 2u; ++dy) { + const uint cy = py * 2u + dy; + if (cy >= prev_h) { + continue; + } + for (uint dx = 0u; dx < 2u; ++dx) { + const uint cx = px * 2u + dx; + if (cx >= prev_w) { + continue; + } + const uint child = level_offsets[level - 1u] + cy * prev_w + cx; + min_value = min(min_value, value[child]); + } + } + value[level_offsets[level] + py * cur_w + px] = min_value; + } + } + } + return true; +} + +inline void j2k_packet_tree_encode( + uint x, + uint y, + uint threshold, + device uint *value, + device uint *current, + device uint *known, + thread uint *level_offsets, + thread uint *level_widths, + uint levels, + thread J2kPacketBitWriter &writer +) { + thread uint path[16]; + uint cx = x; + uint cy = y; + for (uint level = 0u; level < levels; ++level) { + path[level] = level_offsets[level] + cy * level_widths[level] + cx; + cx /= 2u; + cy /= 2u; + } + + uint parent_val = 0u; + for (int level = int(levels) - 1; level >= 0; --level) { + const uint node = path[uint(level)]; + const uint start = max(current[node], parent_val); + if (known[node] == 0u) { + const uint target = min(value[node], threshold); + for (uint v = start; v < target; ++v) { + j2k_packet_write_bit(writer, 0u); + } + if (value[node] < threshold) { + j2k_packet_write_bit(writer, 1u); + known[node] = 1u; + } + current[node] = target; + } + parent_val = current[node]; + } +} + +struct J2kPacketTreeScratch { + device uint *inc_value; + device uint *inc_current; + device uint *inc_known; + device uint *zbp_value; + device uint *zbp_current; + device uint *zbp_known; +}; + +inline J2kPacketTreeScratch j2k_packet_tree_scratch( + device uint *tree_scratch, + uint node_capacity +) { + return { + tree_scratch, + tree_scratch + node_capacity, + tree_scratch + node_capacity * 2u, + tree_scratch + node_capacity * 3u, + tree_scratch + node_capacity * 4u, + tree_scratch + node_capacity * 5u, + }; +} + +inline J2kPacketDescriptor j2k_packet_descriptor_for_order( + device const J2kPacketDescriptor *descriptors, + uint descriptor_count, + uint packet_order_idx +) { + return descriptor_count > 0u + ? descriptors[packet_order_idx] + : J2kPacketDescriptor{ + packet_order_idx, + packet_order_idx, + 0u, + packet_order_idx, + 0u, + 0u, + 0u, + 0u, + }; +} + +inline bool j2k_packet_append_header( + device uchar *out, + device const uchar *header, + thread J2kPacketBitWriter &writer, + uint output_capacity, + thread uint &out_len, + device J2kPacketEncodeStatus *status +) { + if (writer.failed != 0u || out_len + writer.len > output_capacity) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 3u, 0u); + return false; + } + for (uint idx = 0u; idx < writer.len; ++idx) { + out[out_len + idx] = header[idx]; + } + out_len += writer.len; + if (writer.len > 0u && header[writer.len - 1u] == uchar(0xFFu)) { + if (out_len >= output_capacity) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 4u, 0u); + return false; + } + out[out_len] = uchar(0u); + out_len += 1u; + } + return true; +} + +kernel void j2k_encode_packetization( + device const J2kPacketResolution *resolutions [[buffer(0)]], + device const J2kPacketSubband *subbands [[buffer(1)]], + device const J2kPacketBlock *blocks [[buffer(2)]], + device const uchar *payload [[buffer(3)]], + device uchar *out [[buffer(4)]], + device uchar *header [[buffer(5)]], + device uint *tree_scratch [[buffer(6)]], + constant J2kPacketEncodeParams ¶ms [[buffer(7)]], + device J2kPacketEncodeStatus *status [[buffer(8)]], + device const J2kPacketDescriptor *descriptors [[buffer(9)]], + device J2kPacketStateBlock *state_blocks [[buffer(10)]], + uint gid [[thread_position_in_grid]] +) { + if (gid != 0u) { + return; + } + + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u); + + const uint node_capacity = params.scratch_node_capacity; + const J2kPacketTreeScratch scratch = j2k_packet_tree_scratch(tree_scratch, node_capacity); + + uint out_len = 0u; + const uint packet_count = + params.descriptor_count > 0u ? params.descriptor_count : params.resolution_count; + for (uint packet_order_idx = 0u; packet_order_idx < packet_count; ++packet_order_idx) { + const bool has_descriptor = params.descriptor_count > 0u; + const J2kPacketDescriptor descriptor = j2k_packet_descriptor_for_order( + descriptors, + params.descriptor_count, + packet_order_idx + ); + const uint packet_index = descriptor.packet_index; + if (packet_index >= params.resolution_count) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 6u, 0u); + return; + } + const J2kPacketResolution resolution = resolutions[packet_index]; + uint state_block_cursor = descriptor.state_block_offset; + bool any_data = false; + for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { + const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; + for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { + if (blocks[subband.block_offset + block_idx].num_coding_passes > 0u) { + any_data = true; + break; + } + } + } + + thread J2kPacketBitWriter writer; + j2k_packet_writer_init(writer, header, params.header_capacity); + if (!any_data) { + j2k_packet_write_bit(writer, 0u); + j2k_packet_writer_finish(writer); + } else { + j2k_packet_write_bit(writer, 1u); + for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { + const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; + const uint subband_state_block_offset = state_block_cursor; + state_block_cursor += subband.block_count; + thread uint level_offsets[16]; + thread uint level_widths[16]; + thread uint level_heights[16]; + uint levels = 0u; + if (!j2k_packet_prepare_tree( + blocks, + subband.block_offset, + subband.block_count, + subband.num_cbs_x, + subband.num_cbs_y, + false, + descriptor.layer, + scratch.inc_value, + scratch.inc_current, + scratch.inc_known, + node_capacity, + level_offsets, + level_widths, + level_heights, + levels)) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 1u, 0u); + return; + } + thread uint z_level_offsets[16]; + thread uint z_level_widths[16]; + thread uint z_level_heights[16]; + uint z_levels = 0u; + if (!j2k_packet_prepare_tree( + blocks, + subband.block_offset, + subband.block_count, + subband.num_cbs_x, + subband.num_cbs_y, + true, + descriptor.layer, + scratch.zbp_value, + scratch.zbp_current, + scratch.zbp_known, + node_capacity, + z_level_offsets, + z_level_widths, + z_level_heights, + z_levels)) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 2u, 0u); + return; + } + + for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { + const uint x = block_idx % subband.num_cbs_x; + const uint y = block_idx / subband.num_cbs_x; + const J2kPacketBlock block = blocks[subband.block_offset + block_idx]; + const uint state_block_index = subband_state_block_offset + block_idx; + uint previously_included = block.previously_included; + uint local_l_block = block.l_block; + if (has_descriptor) { + previously_included = state_blocks[state_block_index].previously_included; + local_l_block = state_blocks[state_block_index].l_block; + } + if (previously_included == 0u) { + j2k_packet_tree_encode( + x, + y, + descriptor.layer + 1u, + scratch.inc_value, + scratch.inc_current, + scratch.inc_known, + level_offsets, + level_widths, + levels, + writer + ); + if (block.num_coding_passes == 0u) { + continue; + } + j2k_packet_tree_encode( + x, + y, + block.num_zero_bitplanes + 1u, + scratch.zbp_value, + scratch.zbp_current, + scratch.zbp_known, + z_level_offsets, + z_level_widths, + z_levels, + writer + ); + } else if (block.num_coding_passes > 0u) { + j2k_packet_write_bit(writer, 1u); + } else { + j2k_packet_write_bit(writer, 0u); + continue; + } + + if (block.block_coding_mode == 0u) { + const uint num_bits = + j2k_packet_bits_for_length(local_l_block, block.num_coding_passes); + j2k_packet_encode_num_passes(block.num_coding_passes, writer); + j2k_packet_encode_length(block.data_len, local_l_block, num_bits, writer); + } else if (block.block_coding_mode == 1u) { + const uint num_bits = + j2k_packet_bits_for_ht_length(local_l_block, block.num_coding_passes); + j2k_packet_encode_num_ht_passes(block.num_coding_passes, writer); + j2k_packet_encode_length(block.data_len, local_l_block, num_bits, writer); + } else { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 7u, block.reserved0); + return; + } + if (has_descriptor) { + state_blocks[state_block_index].previously_included = 1u; + state_blocks[state_block_index].l_block = local_l_block; + } + } + } + j2k_packet_writer_finish(writer); + } + + if (!j2k_packet_append_header(out, header, writer, params.output_capacity, out_len, status)) { + return; + } + + if (any_data) { + for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { + const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; + for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { + const J2kPacketBlock block = blocks[subband.block_offset + block_idx]; + if (block.num_coding_passes == 0u) { + continue; + } + if (out_len + block.data_len > params.output_capacity) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 5u, 0u); + return; + } + for (uint byte_idx = 0u; byte_idx < block.data_len; ++byte_idx) { + out[out_len + byte_idx] = payload[block.data_offset + byte_idx]; + } + out_len += block.data_len; + } + } + } + } + + j2k_set_packet_status(status, J2K_ENCODE_STATUS_OK, 0u, out_len); +} + +struct J2kPacketPayloadCopyJob { + uint src_offset; + uint dst_offset; + uint byte_len; + uint reserved0; +}; + +struct J2kPacketPayloadCopyParams { + uint bytes_per_thread; + uint stripes_per_job; +}; + +inline void j2k_packet_record_payload_copy_job( + uint byte_len, + thread uint &payload_copy_bytes, + thread uint &payload_copy_small_jobs, + thread uint &payload_copy_medium_jobs, + thread uint &payload_copy_large_jobs +) { + payload_copy_bytes = payload_copy_bytes > 0xffffffffu - byte_len + ? 0xffffffffu + : payload_copy_bytes + byte_len; + if (byte_len <= J2K_PACKET_PAYLOAD_COPY_SMALL_JOB_BYTES) { + payload_copy_small_jobs += 1u; + } else if (byte_len <= J2K_PACKET_PAYLOAD_COPY_MEDIUM_JOB_BYTES) { + payload_copy_medium_jobs += 1u; + } else { + payload_copy_large_jobs += 1u; + } +} + +inline bool j2k_packet_push_payload_copy_job( + device J2kPacketPayloadCopyJob *payload_copy_jobs, + uint payload_copy_capacity, + uint src_offset, + uint dst_offset, + uint byte_len, + thread uint &payload_copy_count, + thread uint &payload_copy_bytes, + thread uint &payload_copy_small_jobs, + thread uint &payload_copy_medium_jobs, + thread uint &payload_copy_large_jobs +) { + if (byte_len == 0u) { + return true; + } + if (payload_copy_count >= payload_copy_capacity) { + return false; + } + payload_copy_jobs[payload_copy_count] = + J2kPacketPayloadCopyJob{src_offset, dst_offset, byte_len, 0u}; + payload_copy_count += 1u; + j2k_packet_record_payload_copy_job( + byte_len, + payload_copy_bytes, + payload_copy_small_jobs, + payload_copy_medium_jobs, + payload_copy_large_jobs + ); + return true; +} + +struct J2kBatchedPacketEncodeJob { + uint resolution_offset; + uint subband_offset; + uint block_offset; + uint descriptor_offset; + uint state_block_offset; + uint output_offset; + uint header_offset; + uint scratch_offset; + uint payload_copy_offset; + uint payload_copy_capacity; + uint resolution_count; + uint num_layers; + uint num_components; + uint code_block_count; + uint subband_count; + uint descriptor_count; + uint output_capacity; + uint header_capacity; + uint scratch_node_capacity; +}; + +kernel void j2k_encode_packetization_batched( + device const J2kPacketResolution *all_resolutions [[buffer(0)]], + device const J2kPacketSubband *all_subbands [[buffer(1)]], + device const J2kPacketBlock *all_blocks [[buffer(2)]], + device const uchar *payload [[buffer(3)]], + device uchar *all_out [[buffer(4)]], + device uchar *all_header [[buffer(5)]], + device uint *all_tree_scratch [[buffer(6)]], + device const J2kBatchedPacketEncodeJob *jobs [[buffer(7)]], + device J2kPacketEncodeStatus *all_status [[buffer(8)]], + device const J2kPacketDescriptor *all_descriptors [[buffer(9)]], + device J2kPacketStateBlock *all_state_blocks [[buffer(10)]], + device J2kPacketPayloadCopyJob *all_payload_copy_jobs [[buffer(11)]], + uint gid [[thread_position_in_grid]] +) { + const J2kBatchedPacketEncodeJob job = jobs[gid]; + device const J2kPacketResolution *resolutions = all_resolutions + job.resolution_offset; + device const J2kPacketSubband *subbands = all_subbands + job.subband_offset; + device const J2kPacketBlock *blocks = all_blocks + job.block_offset; + device uchar *out = all_out + job.output_offset; + device uchar *header = all_header + job.header_offset; + device uint *tree_scratch = all_tree_scratch + job.scratch_offset; + device J2kPacketEncodeStatus *status = all_status + gid; + device const J2kPacketDescriptor *descriptors = all_descriptors + job.descriptor_offset; + device J2kPacketStateBlock *state_blocks = all_state_blocks + job.state_block_offset; + device J2kPacketPayloadCopyJob *payload_copy_jobs = + all_payload_copy_jobs + job.payload_copy_offset; + + J2kPacketEncodeParams params; + params.resolution_count = job.resolution_count; + params.num_layers = job.num_layers; + params.num_components = job.num_components; + params.code_block_count = job.code_block_count; + params.subband_count = job.subband_count; + params.descriptor_count = job.descriptor_count; + params.output_capacity = job.output_capacity; + params.header_capacity = job.header_capacity; + params.scratch_node_capacity = job.scratch_node_capacity; + + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u); + + const uint node_capacity = params.scratch_node_capacity; + const J2kPacketTreeScratch scratch = j2k_packet_tree_scratch(tree_scratch, node_capacity); + + uint out_len = 0u; + uint payload_copy_count = 0u; + uint payload_copy_bytes = 0u; + uint payload_copy_small_jobs = 0u; + uint payload_copy_medium_jobs = 0u; + uint payload_copy_large_jobs = 0u; + const uint packet_count = + params.descriptor_count > 0u ? params.descriptor_count : params.resolution_count; + for (uint packet_order_idx = 0u; packet_order_idx < packet_count; ++packet_order_idx) { + const bool has_descriptor = params.descriptor_count > 0u; + const J2kPacketDescriptor descriptor = j2k_packet_descriptor_for_order( + descriptors, + params.descriptor_count, + packet_order_idx + ); + const uint packet_index = descriptor.packet_index; + if (packet_index >= params.resolution_count) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 6u, 0u); + return; + } + const J2kPacketResolution resolution = resolutions[packet_index]; + uint state_block_cursor = descriptor.state_block_offset; + bool any_data = false; + for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { + const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; + for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { + if (blocks[subband.block_offset + block_idx].num_coding_passes > 0u) { + any_data = true; + break; + } + } + } + + thread J2kPacketBitWriter writer; + j2k_packet_writer_init(writer, header, params.header_capacity); + if (!any_data) { + j2k_packet_write_bit(writer, 0u); + j2k_packet_writer_finish(writer); + } else { + j2k_packet_write_bit(writer, 1u); + for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { + const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; + const uint subband_state_block_offset = state_block_cursor; + state_block_cursor += subband.block_count; + thread uint level_offsets[16]; + thread uint level_widths[16]; + thread uint level_heights[16]; + uint levels = 0u; + if (!j2k_packet_prepare_tree( + blocks, + subband.block_offset, + subband.block_count, + subband.num_cbs_x, + subband.num_cbs_y, + false, + descriptor.layer, + scratch.inc_value, + scratch.inc_current, + scratch.inc_known, + node_capacity, + level_offsets, + level_widths, + level_heights, + levels)) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 1u, 0u); + return; + } + thread uint z_level_offsets[16]; + thread uint z_level_widths[16]; + thread uint z_level_heights[16]; + uint z_levels = 0u; + if (!j2k_packet_prepare_tree( + blocks, + subband.block_offset, + subband.block_count, + subband.num_cbs_x, + subband.num_cbs_y, + true, + descriptor.layer, + scratch.zbp_value, + scratch.zbp_current, + scratch.zbp_known, + node_capacity, + z_level_offsets, + z_level_widths, + z_level_heights, + z_levels)) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 2u, 0u); + return; + } + + for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { + const uint x = block_idx % subband.num_cbs_x; + const uint y = block_idx / subband.num_cbs_x; + const J2kPacketBlock block = blocks[subband.block_offset + block_idx]; + const uint state_block_index = subband_state_block_offset + block_idx; + uint previously_included = block.previously_included; + uint local_l_block = block.l_block; + if (has_descriptor) { + previously_included = state_blocks[state_block_index].previously_included; + local_l_block = state_blocks[state_block_index].l_block; + } + if (previously_included == 0u) { + j2k_packet_tree_encode( + x, + y, + descriptor.layer + 1u, + scratch.inc_value, + scratch.inc_current, + scratch.inc_known, + level_offsets, + level_widths, + levels, + writer + ); + if (block.num_coding_passes == 0u) { + continue; + } + j2k_packet_tree_encode( + x, + y, + block.num_zero_bitplanes + 1u, + scratch.zbp_value, + scratch.zbp_current, + scratch.zbp_known, + z_level_offsets, + z_level_widths, + z_levels, + writer + ); + } else if (block.num_coding_passes > 0u) { + j2k_packet_write_bit(writer, 1u); + } else { + j2k_packet_write_bit(writer, 0u); + continue; + } + + if (block.block_coding_mode == 0u) { + const uint num_bits = + j2k_packet_bits_for_length(local_l_block, block.num_coding_passes); + j2k_packet_encode_num_passes(block.num_coding_passes, writer); + j2k_packet_encode_length(block.data_len, local_l_block, num_bits, writer); + } else if (block.block_coding_mode == 1u) { + const uint num_bits = + j2k_packet_bits_for_ht_length(local_l_block, block.num_coding_passes); + j2k_packet_encode_num_ht_passes(block.num_coding_passes, writer); + j2k_packet_encode_length(block.data_len, local_l_block, num_bits, writer); + } else { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 7u, block.reserved0); + return; + } + if (has_descriptor) { + state_blocks[state_block_index].previously_included = 1u; + state_blocks[state_block_index].l_block = local_l_block; + } + } + } + j2k_packet_writer_finish(writer); + } + + if (!j2k_packet_append_header(out, header, writer, params.output_capacity, out_len, status)) { + return; + } + + if (any_data) { + for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { + const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; + for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { + const J2kPacketBlock block = blocks[subband.block_offset + block_idx]; + if (block.num_coding_passes == 0u) { + continue; + } + if (out_len + block.data_len > params.output_capacity) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 5u, 0u); + return; + } + if (!j2k_packet_push_payload_copy_job( + payload_copy_jobs, + job.payload_copy_capacity, + block.data_offset, + out_len, + block.data_len, + payload_copy_count, + payload_copy_bytes, + payload_copy_small_jobs, + payload_copy_medium_jobs, + payload_copy_large_jobs)) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 8u, payload_copy_count); + return; + } + out_len += block.data_len; + } + } + } + } + + j2k_set_packet_status_payload_copy( + status, + J2K_ENCODE_STATUS_OK, + payload_copy_count, + out_len, + payload_copy_bytes, + payload_copy_small_jobs, + payload_copy_medium_jobs, + payload_copy_large_jobs + ); +} + +kernel void j2k_encode_packetization_resident_classic_batched( + device const J2kPacketResolution *all_resolutions [[buffer(0)]], + device const J2kPacketSubband *all_subbands [[buffer(1)]], + device const J2kResidentPacketBlock *all_resident_blocks [[buffer(2)]], + device const uchar *payload [[buffer(3)]], + device uchar *all_out [[buffer(4)]], + device uchar *all_header [[buffer(5)]], + device uint *all_tree_scratch [[buffer(6)]], + device const J2kBatchedPacketEncodeJob *jobs [[buffer(7)]], + device J2kPacketEncodeStatus *all_status [[buffer(8)]], + device const J2kPacketDescriptor *all_descriptors [[buffer(9)]], + device J2kPacketStateBlock *all_state_blocks [[buffer(10)]], + device J2kPacketPayloadCopyJob *all_payload_copy_jobs [[buffer(11)]], + device const J2kClassicEncodeBatchJob *tier1_jobs [[buffer(12)]], + device const J2kClassicEncodeStatus *tier1_statuses [[buffer(13)]], + device const J2kClassicSegment *tier1_segments [[buffer(14)]], + constant J2kResidentPacketBlockParams &resident_params [[buffer(15)]], + uint gid [[thread_position_in_grid]] +) { + const J2kBatchedPacketEncodeJob job = jobs[gid]; + device const J2kPacketResolution *resolutions = all_resolutions + job.resolution_offset; + device const J2kPacketSubband *subbands = all_subbands + job.subband_offset; + device const J2kResidentPacketBlock *resident_blocks = all_resident_blocks + job.block_offset; + device uchar *out = all_out + job.output_offset; + device uchar *header = all_header + job.header_offset; + device uint *tree_scratch = all_tree_scratch + job.scratch_offset; + device J2kPacketEncodeStatus *status = all_status + gid; + device const J2kPacketDescriptor *descriptors = all_descriptors + job.descriptor_offset; + device J2kPacketStateBlock *state_blocks = all_state_blocks + job.state_block_offset; + device J2kPacketPayloadCopyJob *payload_copy_jobs = + all_payload_copy_jobs + job.payload_copy_offset; + + J2kPacketEncodeParams params; + params.resolution_count = job.resolution_count; + params.num_layers = job.num_layers; + params.num_components = job.num_components; + params.code_block_count = job.code_block_count; + params.subband_count = job.subband_count; + params.descriptor_count = job.descriptor_count; + params.output_capacity = job.output_capacity; + params.header_capacity = job.header_capacity; + params.scratch_node_capacity = job.scratch_node_capacity; + + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u); + + const uint node_capacity = params.scratch_node_capacity; + const J2kPacketTreeScratch scratch = j2k_packet_tree_scratch(tree_scratch, node_capacity); + + uint out_len = 0u; + uint payload_copy_count = 0u; + uint payload_copy_bytes = 0u; + uint payload_copy_small_jobs = 0u; + uint payload_copy_medium_jobs = 0u; + uint payload_copy_large_jobs = 0u; + const uint packet_count = + params.descriptor_count > 0u ? params.descriptor_count : params.resolution_count; + for (uint packet_order_idx = 0u; packet_order_idx < packet_count; ++packet_order_idx) { + const bool has_descriptor = params.descriptor_count > 0u; + const J2kPacketDescriptor descriptor = j2k_packet_descriptor_for_order( + descriptors, + params.descriptor_count, + packet_order_idx + ); + const uint packet_index = descriptor.packet_index; + if (packet_index >= params.resolution_count) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 6u, 0u); + return; + } + const J2kPacketResolution resolution = resolutions[packet_index]; + uint state_block_cursor = descriptor.state_block_offset; + bool any_data = false; + for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { + const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; + for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { + const J2kPacketBlock block = j2k_classic_packet_block_from_resident( + resident_blocks, + tier1_jobs, + tier1_statuses, + resident_params.tier1_job_count, + subband.block_offset + block_idx + ); + if (block.num_coding_passes > 0u) { + any_data = true; + break; + } + } + } + + thread J2kPacketBitWriter writer; + j2k_packet_writer_init(writer, header, params.header_capacity); + if (!any_data) { + j2k_packet_write_bit(writer, 0u); + j2k_packet_writer_finish(writer); + } else { + j2k_packet_write_bit(writer, 1u); + for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { + const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; + const uint subband_state_block_offset = state_block_cursor; + state_block_cursor += subband.block_count; + thread uint level_offsets[16]; + thread uint level_widths[16]; + thread uint level_heights[16]; + uint levels = 0u; + if (!j2k_packet_prepare_tree_resident_classic( + resident_blocks, + tier1_jobs, + tier1_statuses, + resident_params.tier1_job_count, + subband.block_offset, + subband.block_count, + subband.num_cbs_x, + subband.num_cbs_y, + false, + descriptor.layer, + scratch.inc_value, + scratch.inc_current, + scratch.inc_known, + node_capacity, + level_offsets, + level_widths, + level_heights, + levels)) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 1u, 0u); + return; + } + thread uint z_level_offsets[16]; + thread uint z_level_widths[16]; + thread uint z_level_heights[16]; + uint z_levels = 0u; + if (!j2k_packet_prepare_tree_resident_classic( + resident_blocks, + tier1_jobs, + tier1_statuses, + resident_params.tier1_job_count, + subband.block_offset, + subband.block_count, + subband.num_cbs_x, + subband.num_cbs_y, + true, + descriptor.layer, + scratch.zbp_value, + scratch.zbp_current, + scratch.zbp_known, + node_capacity, + z_level_offsets, + z_level_widths, + z_level_heights, + z_levels)) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 2u, 0u); + return; + } + + for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { + const uint x = block_idx % subband.num_cbs_x; + const uint y = block_idx / subband.num_cbs_x; + const J2kPacketBlock block = j2k_classic_packet_block_from_resident( + resident_blocks, + tier1_jobs, + tier1_statuses, + resident_params.tier1_job_count, + subband.block_offset + block_idx + ); + const uint state_block_index = subband_state_block_offset + block_idx; + uint previously_included = block.previously_included; + uint local_l_block = block.l_block; + if (has_descriptor) { + previously_included = state_blocks[state_block_index].previously_included; + local_l_block = state_blocks[state_block_index].l_block; + } + if (previously_included == 0u) { + j2k_packet_tree_encode( + x, + y, + descriptor.layer + 1u, + scratch.inc_value, + scratch.inc_current, + scratch.inc_known, + level_offsets, + level_widths, + levels, + writer + ); + if (block.num_coding_passes == 0u) { + continue; + } + j2k_packet_tree_encode( + x, + y, + block.num_zero_bitplanes + 1u, + scratch.zbp_value, + scratch.zbp_current, + scratch.zbp_known, + z_level_offsets, + z_level_widths, + z_levels, + writer + ); + } else if (block.num_coding_passes > 0u) { + j2k_packet_write_bit(writer, 1u); + } else { + j2k_packet_write_bit(writer, 0u); + continue; + } + + if (block.block_coding_mode == 0u) { + j2k_packet_encode_num_passes(block.num_coding_passes, writer); + const J2kResidentPacketBlock resident = + resident_blocks[subband.block_offset + block_idx]; + if (resident.tier1_job_index >= resident_params.tier1_job_count) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 7u, block.reserved0); + return; + } + const J2kClassicEncodeBatchJob tier1_job = + tier1_jobs[resident.tier1_job_index]; + const J2kClassicEncodeStatus tier1_status = + tier1_statuses[resident.tier1_job_index]; + if (!j2k_packet_encode_classic_segment_lengths_resident( + block, + tier1_job, + tier1_status, + tier1_segments, + local_l_block, + writer)) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 9u, block.reserved0); + return; + } + } else { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 7u, block.reserved0); + return; + } + if (has_descriptor) { + state_blocks[state_block_index].previously_included = 1u; + state_blocks[state_block_index].l_block = local_l_block; + } + } + } + j2k_packet_writer_finish(writer); + } + + if (!j2k_packet_append_header(out, header, writer, params.output_capacity, out_len, status)) { + return; + } + + if (any_data) { + for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { + const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; + for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { + const J2kPacketBlock block = j2k_classic_packet_block_from_resident( + resident_blocks, + tier1_jobs, + tier1_statuses, + resident_params.tier1_job_count, + subband.block_offset + block_idx + ); + if (block.num_coding_passes == 0u) { + continue; + } + if (out_len + block.data_len > params.output_capacity) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 5u, 0u); + return; + } + if (!j2k_packet_push_payload_copy_job( + payload_copy_jobs, + job.payload_copy_capacity, + block.data_offset, + out_len, + block.data_len, + payload_copy_count, + payload_copy_bytes, + payload_copy_small_jobs, + payload_copy_medium_jobs, + payload_copy_large_jobs)) { + j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 8u, payload_copy_count); + return; + } + out_len += block.data_len; + } + } + } + } + + j2k_set_packet_status_payload_copy( + status, + J2K_ENCODE_STATUS_OK, + payload_copy_count, + out_len, + payload_copy_bytes, + payload_copy_small_jobs, + payload_copy_medium_jobs, + payload_copy_large_jobs + ); +} + +kernel void j2k_copy_packet_payload_batched( + device const uchar *payload [[buffer(0)]], + device uchar *all_out [[buffer(1)]], + device const J2kBatchedPacketEncodeJob *jobs [[buffer(2)]], + device const J2kPacketEncodeStatus *all_status [[buffer(3)]], + device const J2kPacketPayloadCopyJob *all_payload_copy_jobs [[buffer(4)]], + constant J2kPacketPayloadCopyParams ¶ms [[buffer(5)]], + uint3 gid [[thread_position_in_grid]] +) { + const uint copy_index = gid.x; + const uint tile = gid.y; + const uint stripe = gid.z; + if (stripe >= params.stripes_per_job || params.bytes_per_thread == 0u) { + return; + } + + const J2kPacketEncodeStatus status = all_status[tile]; + if (status.code != J2K_ENCODE_STATUS_OK || copy_index >= status.detail) { + return; + } + + const J2kBatchedPacketEncodeJob job = jobs[tile]; + const J2kPacketPayloadCopyJob copy_job = + all_payload_copy_jobs[job.payload_copy_offset + copy_index]; + device uchar *out = all_out + job.output_offset + copy_job.dst_offset; + device const uchar *src = payload + copy_job.src_offset; + const uint stride = params.stripes_per_job * params.bytes_per_thread; + for (uint byte_base = stripe * params.bytes_per_thread; + byte_base < copy_job.byte_len; + byte_base += stride) { + const uint byte_end = min(copy_job.byte_len, byte_base + params.bytes_per_thread); + for (uint byte_idx = byte_base; byte_idx < byte_end; ++byte_idx) { + out[byte_idx] = src[byte_idx]; + } + } +} diff --git a/crates/j2k-metal/src/fdwt.metal b/crates/j2k-metal/src/fdwt.metal new file mode 100644 index 00000000..96a2e01d --- /dev/null +++ b/crates/j2k-metal/src/fdwt.metal @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: Apache-2.0 + +#include +using namespace metal; + +struct J2kForwardDwt53Params { + uint full_width; + uint current_width; + uint current_height; + uint low_width; + uint low_height; +}; + +struct J2kForwardDwt53BatchedParams { + uint full_width; + uint current_width; + uint current_height; + uint low_width; + uint low_height; + uint component_count; +}; + +struct J2kForwardDwt97Params { + uint full_width; + uint current_width; + uint current_height; + uint low_width; + uint low_height; + uint parity; + float coefficient; + uint _reserved; +}; + +constant float J2K_FDWT97_KAPPA = 1.2301741f; +constant float J2K_FDWT97_INV_KAPPA = 1.0f / 1.2301741f; + +inline float j2k_fdwt53_predict_row( + device const float *src, + uint row_base, + uint width, + uint high_index +) { + const uint odd = high_index * 2u + 1u; + const uint last_even = (width % 2u == 0u) ? width - 2u : width - 1u; + const float left = src[row_base + odd - 1u]; + const float right = (odd + 1u < width) ? src[row_base + odd + 1u] : src[row_base + last_even]; + return src[row_base + odd] - floor((left + right) * 0.5f); +} + +inline float j2k_fdwt53_predict_col( + device const float *src, + uint x, + uint full_width, + uint height, + uint high_index +) { + const uint odd = high_index * 2u + 1u; + const uint last_even = (height % 2u == 0u) ? height - 2u : height - 1u; + const float top = src[(odd - 1u) * full_width + x]; + const float bottom = (odd + 1u < height) + ? src[(odd + 1u) * full_width + x] + : src[last_even * full_width + x]; + return src[odd * full_width + x] - floor((top + bottom) * 0.5f); +} + +inline void j2k_fdwt53_horizontal_step( + device const float *src, + device float *dst, + uint full_width, + uint current_width, + uint low_width, + uint2 gid +) { + const uint row_base = gid.y * full_width; + if (gid.x < low_width) { + const uint even = gid.x * 2u; + const float left = gid.x > 0u + ? j2k_fdwt53_predict_row(src, row_base, current_width, gid.x - 1u) + : j2k_fdwt53_predict_row(src, row_base, current_width, 0u); + const float right = even + 1u < current_width + ? j2k_fdwt53_predict_row(src, row_base, current_width, gid.x) + : left; + dst[row_base + gid.x] = + src[row_base + even] + floor((left + right) * 0.25f + 0.5f); + return; + } + + const uint high_index = gid.x - low_width; + dst[row_base + gid.x] = j2k_fdwt53_predict_row( + src, + row_base, + current_width, + high_index + ); +} + +inline void j2k_fdwt53_vertical_step( + device const float *src, + device float *dst, + uint full_width, + uint current_height, + uint low_height, + uint2 gid +) { + if (gid.y < low_height) { + const uint even = gid.y * 2u; + const float top = gid.y > 0u + ? j2k_fdwt53_predict_col(src, gid.x, full_width, current_height, gid.y - 1u) + : j2k_fdwt53_predict_col(src, gid.x, full_width, current_height, 0u); + const float bottom = even + 1u < current_height + ? j2k_fdwt53_predict_col(src, gid.x, full_width, current_height, gid.y) + : top; + dst[gid.y * full_width + gid.x] = + src[even * full_width + gid.x] + floor((top + bottom) * 0.25f + 0.5f); + return; + } + + const uint high_index = gid.y - low_height; + dst[gid.y * full_width + gid.x] = j2k_fdwt53_predict_col( + src, + gid.x, + full_width, + current_height, + high_index + ); +} + +kernel void j2k_forward_dwt53_horizontal( + device const float *src [[buffer(0)]], + device float *dst [[buffer(1)]], + constant J2kForwardDwt53Params ¶ms [[buffer(2)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.current_width || gid.y >= params.current_height) { + return; + } + + j2k_fdwt53_horizontal_step( + src, + dst, + params.full_width, + params.current_width, + params.low_width, + gid + ); +} + +kernel void j2k_forward_dwt53_vertical( + device const float *src [[buffer(0)]], + device float *dst [[buffer(1)]], + constant J2kForwardDwt53Params ¶ms [[buffer(2)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.current_width || gid.y >= params.current_height) { + return; + } + + j2k_fdwt53_vertical_step( + src, + dst, + params.full_width, + params.current_height, + params.low_height, + gid + ); +} + +kernel void j2k_forward_dwt53_horizontal_batched( + device const float *src0 [[buffer(0)]], + device const float *src1 [[buffer(1)]], + device const float *src2 [[buffer(2)]], + device float *dst0 [[buffer(3)]], + device float *dst1 [[buffer(4)]], + device float *dst2 [[buffer(5)]], + constant J2kForwardDwt53BatchedParams ¶ms [[buffer(6)]], + uint3 gid [[thread_position_in_grid]] +) { + if ( + gid.x >= params.current_width || + gid.y >= params.current_height || + gid.z >= params.component_count + ) { + return; + } + + device const float *src = gid.z == 0u ? src0 : (gid.z == 1u ? src1 : src2); + device float *dst = gid.z == 0u ? dst0 : (gid.z == 1u ? dst1 : dst2); + j2k_fdwt53_horizontal_step( + src, + dst, + params.full_width, + params.current_width, + params.low_width, + gid.xy + ); +} + +kernel void j2k_forward_dwt53_vertical_batched( + device const float *src0 [[buffer(0)]], + device const float *src1 [[buffer(1)]], + device const float *src2 [[buffer(2)]], + device float *dst0 [[buffer(3)]], + device float *dst1 [[buffer(4)]], + device float *dst2 [[buffer(5)]], + constant J2kForwardDwt53BatchedParams ¶ms [[buffer(6)]], + uint3 gid [[thread_position_in_grid]] +) { + if ( + gid.x >= params.current_width || + gid.y >= params.current_height || + gid.z >= params.component_count + ) { + return; + } + + device const float *src = gid.z == 0u ? src0 : (gid.z == 1u ? src1 : src2); + device float *dst = gid.z == 0u ? dst0 : (gid.z == 1u ? dst1 : dst2); + j2k_fdwt53_vertical_step( + src, + dst, + params.full_width, + params.current_height, + params.low_height, + gid.xy + ); +} + +inline bool j2k_fdwt97_is_active_parity(uint index, uint parity) { + return (index & 1u) == parity; +} + +inline float j2k_fdwt97_horizontal_neighbor( + device const float *data, + uint row_base, + uint width, + uint x, + bool update_high, + bool left_neighbor +) { + if (update_high) { + if (left_neighbor) { + return data[row_base + x - 1u]; + } + const uint last_even = (width & 1u) == 0u ? width - 2u : width - 1u; + return (x + 1u < width) ? data[row_base + x + 1u] : data[row_base + last_even]; + } + + if (left_neighbor) { + return x > 0u ? data[row_base + x - 1u] : data[row_base + 1u]; + } + return (x + 1u < width) ? data[row_base + x + 1u] : data[row_base + x - 1u]; +} + +inline float j2k_fdwt97_vertical_neighbor( + device const float *data, + uint full_width, + uint height, + uint x, + uint y, + bool update_high, + bool top_neighbor +) { + if (update_high) { + if (top_neighbor) { + return data[(y - 1u) * full_width + x]; + } + const uint last_even = (height & 1u) == 0u ? height - 2u : height - 1u; + const uint neighbor_y = (y + 1u < height) ? y + 1u : last_even; + return data[neighbor_y * full_width + x]; + } + + if (top_neighbor) { + const uint neighbor_y = y > 0u ? y - 1u : 1u; + return data[neighbor_y * full_width + x]; + } + const uint neighbor_y = (y + 1u < height) ? y + 1u : y - 1u; + return data[neighbor_y * full_width + x]; +} + +kernel void j2k_forward_dwt97_lift_horizontal( + device float *data [[buffer(0)]], + device float *unused [[buffer(1)]], + constant J2kForwardDwt97Params ¶ms [[buffer(2)]], + uint2 gid [[thread_position_in_grid]] +) { + (void)unused; + if ( + gid.x >= params.current_width || + gid.y >= params.current_height || + !j2k_fdwt97_is_active_parity(gid.x, params.parity) + ) { + return; + } + + const bool update_high = params.parity == 1u; + const uint row_base = gid.y * params.full_width; + const float left = j2k_fdwt97_horizontal_neighbor( + data, + row_base, + params.current_width, + gid.x, + update_high, + true + ); + const float right = j2k_fdwt97_horizontal_neighbor( + data, + row_base, + params.current_width, + gid.x, + update_high, + false + ); + data[row_base + gid.x] += params.coefficient * (left + right); +} + +kernel void j2k_forward_dwt97_lift_vertical( + device float *data [[buffer(0)]], + device float *unused [[buffer(1)]], + constant J2kForwardDwt97Params ¶ms [[buffer(2)]], + uint2 gid [[thread_position_in_grid]] +) { + (void)unused; + if ( + gid.x >= params.current_width || + gid.y >= params.current_height || + !j2k_fdwt97_is_active_parity(gid.y, params.parity) + ) { + return; + } + + const bool update_high = params.parity == 1u; + const float top = j2k_fdwt97_vertical_neighbor( + data, + params.full_width, + params.current_height, + gid.x, + gid.y, + update_high, + true + ); + const float bottom = j2k_fdwt97_vertical_neighbor( + data, + params.full_width, + params.current_height, + gid.x, + gid.y, + update_high, + false + ); + data[gid.y * params.full_width + gid.x] += params.coefficient * (top + bottom); +} + +kernel void j2k_forward_dwt97_deinterleave_horizontal( + device const float *src [[buffer(0)]], + device float *dst [[buffer(1)]], + constant J2kForwardDwt97Params ¶ms [[buffer(2)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.current_width || gid.y >= params.current_height) { + return; + } + + const uint row_base = gid.y * params.full_width; + if (gid.x < params.low_width) { + dst[row_base + gid.x] = src[row_base + gid.x * 2u] * J2K_FDWT97_INV_KAPPA; + return; + } + + const uint high_index = gid.x - params.low_width; + dst[row_base + gid.x] = src[row_base + high_index * 2u + 1u] * J2K_FDWT97_KAPPA; +} + +kernel void j2k_forward_dwt97_deinterleave_vertical( + device const float *src [[buffer(0)]], + device float *dst [[buffer(1)]], + constant J2kForwardDwt97Params ¶ms [[buffer(2)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.current_width || gid.y >= params.current_height) { + return; + } + + if (gid.y < params.low_height) { + dst[gid.y * params.full_width + gid.x] = + src[(gid.y * 2u) * params.full_width + gid.x] * J2K_FDWT97_INV_KAPPA; + return; + } + + const uint high_index = gid.y - params.low_height; + dst[gid.y * params.full_width + gid.x] = + src[(high_index * 2u + 1u) * params.full_width + gid.x] * J2K_FDWT97_KAPPA; +} diff --git a/crates/signinum-j2k-metal/src/ht.rs b/crates/j2k-metal/src/ht.rs similarity index 98% rename from crates/signinum-j2k-metal/src/ht.rs rename to crates/j2k-metal/src/ht.rs index 3fcc49a5..1b43a5ae 100644 --- a/crates/signinum-j2k-metal/src/ht.rs +++ b/crates/j2k-metal/src/ht.rs @@ -3,8 +3,8 @@ #[cfg(target_os = "macos")] use crate::compute; #[cfg(target_os = "macos")] -use signinum_j2k_native::DecodingError; -use signinum_j2k_native::{ +use j2k_native::DecodingError; +use j2k_native::{ decode_ht_code_block_scalar, HtCodeBlockDecodeJob, HtCodeBlockDecoder, HtSubBandDecodeJob, Result, }; @@ -88,6 +88,9 @@ fn supports_metal_ht_kernel(job: &HtCodeBlockDecodeJob<'_>) -> bool { if !supports_metal_ht_geometry(job.width, job.height) { return false; } + if job.roi_shift != 0 { + return false; + } if job.num_bitplanes == 0 || job.num_bitplanes > 31 || job.missing_bit_planes >= 30 { return false; } @@ -171,7 +174,7 @@ mod tests { use super::MetalHtBlockDecoder; #[cfg(target_os = "macos")] use crate::compute; - use signinum_j2k_native::{ + use j2k_native::{ decode_ht_code_block_scalar, encode_htj2k, ColorSpace, DecodeSettings, DecoderContext, EncodeOptions, HtCodeBlockDecodeJob, HtCodeBlockDecoder, Image, }; @@ -187,6 +190,7 @@ mod tests { missing_bit_planes: u8, number_of_coding_passes: u8, num_bitplanes: u8, + roi_shift: u8, stripe_causal: bool, strict: bool, dequantization_step: f32, @@ -204,6 +208,7 @@ mod tests { missing_bit_planes: self.missing_bit_planes, number_of_coding_passes: self.number_of_coding_passes, num_bitplanes: self.num_bitplanes, + roi_shift: self.roi_shift, stripe_causal: self.stripe_causal, strict: self.strict, dequantization_step: self.dequantization_step, @@ -225,7 +230,7 @@ mod tests { &mut self, job: HtCodeBlockDecodeJob<'_>, output: &mut [f32], - ) -> signinum_j2k_native::Result<()> { + ) -> j2k_native::Result<()> { if self.first.is_none() { self.first = Some(OwnedHtJob { data: job.data.to_vec(), @@ -237,6 +242,7 @@ mod tests { missing_bit_planes: job.missing_bit_planes, number_of_coding_passes: job.number_of_coding_passes, num_bitplanes: job.num_bitplanes, + roi_shift: job.roi_shift, stripe_causal: job.stripe_causal, strict: job.strict, dequantization_step: job.dequantization_step, diff --git a/crates/signinum-j2k-metal/src/ht_cleanup.metal b/crates/j2k-metal/src/ht_cleanup.metal similarity index 99% rename from crates/signinum-j2k-metal/src/ht_cleanup.metal rename to crates/j2k-metal/src/ht_cleanup.metal index 13d83fb1..d9d6f099 100644 --- a/crates/signinum-j2k-metal/src/ht_cleanup.metal +++ b/crates/j2k-metal/src/ht_cleanup.metal @@ -25,6 +25,7 @@ struct J2kHtCleanupBatchJob { uint refinement_length; uint missing_msbs; uint num_bitplanes; + uint roi_shift; uint number_of_coding_passes; uint output_stride; uint output_offset; diff --git a/crates/j2k-metal/src/hybrid.rs b/crates/j2k-metal/src/hybrid.rs new file mode 100644 index 00000000..ba391c91 --- /dev/null +++ b/crates/j2k-metal/src/hybrid.rs @@ -0,0 +1,765 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(test)] +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::{ + collections::{hash_map::DefaultHasher, HashMap}, + hash::{Hash, Hasher}, + sync::{Arc, Mutex, OnceLock}, + time::Instant, +}; + +use j2k::J2kError; +use j2k_core::{Downscale, PixelFormat, Rect}; +use j2k_native::{ + DecodeSettings as NativeDecodeSettings, DecoderContext as NativeDecoderContext, + Image as NativeImage, +}; +use metal::Device; +use rayon::prelude::*; + +use crate::{direct, Error, J2kDecoder, Surface}; + +pub(crate) const RGB_REGION_SCALED_METAL_DIRECT_UNSUPPORTED: &str = + "J2K Metal ROI+scaled hybrid decode currently supports single-tile RGB direct plans for Rgb8/Rgba8/Rgb16"; +const METAL_PROFILE_DECODE_LABEL_ENV: &str = "J2K_METAL_PROFILE_DECODE_LABEL"; +const REGION_SCALED_COLOR_PLAN_CACHE_CAP: usize = 128; + +static REGION_SCALED_COLOR_PLAN_CACHE: OnceLock< + Mutex>>, +> = OnceLock::new(); + +#[cfg(test)] +macro_rules! test_atomic_counter { + ($counter:ident, $reset:ident, $load:ident) => { + static $counter: AtomicUsize = AtomicUsize::new(0); + + pub(crate) fn $reset() { + $counter.store(0, Ordering::Relaxed); + } + + pub(crate) fn $load() -> usize { + $counter.load(Ordering::Relaxed) + } + }; +} + +#[cfg(test)] +test_atomic_counter!( + REGION_SCALED_COLOR_PLAN_BUILDS, + reset_region_scaled_color_plan_builds_for_test, + region_scaled_color_plan_builds_for_test +); +#[cfg(test)] +static REGION_SCALED_COLOR_PLAN_TEST_LOCK: Mutex<()> = Mutex::new(()); + +#[cfg(test)] +pub(crate) fn region_scaled_color_plan_test_lock_for_test() -> std::sync::MutexGuard<'static, ()> { + REGION_SCALED_COLOR_PLAN_TEST_LOCK + .lock() + .expect("region scaled color plan test lock") +} + +#[cfg(test)] +pub(crate) fn reset_region_scaled_color_plan_cache_for_test() { + if let Some(cache) = REGION_SCALED_COLOR_PLAN_CACHE.get() { + if let Ok(mut guard) = cache.lock() { + guard.clear(); + } + } +} + +enum PreparedRegionScaledDirectPlan { + Gray(crate::compute::PreparedDirectGrayscalePlan), + Color(crate::compute::PreparedDirectColorPlan), +} + +pub(crate) fn decode_region_scaled_direct_to_surface( + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, +) -> Result, Error> { + let Some(prepared) = build_region_scaled_direct_plan(input, fmt, roi, scale)? else { + return Ok(None); + }; + execute_region_scaled_direct_plan(prepared, fmt) +} +pub(crate) fn decode_region_scaled_direct_to_surface_with_session( + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + session: &crate::MetalBackendSession, +) -> Result, Error> { + let Some(prepared) = + build_region_scaled_direct_plan_with_session(input, fmt, roi, scale, session)? + else { + return Ok(None); + }; + execute_region_scaled_direct_plan_with_device(prepared, fmt, &session.device) +} + +fn build_region_scaled_direct_plan( + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, +) -> Result, Error> { + match fmt { + PixelFormat::Gray8 | PixelFormat::Gray16 => { + match build_region_scaled_direct_gray_plan(input, roi, scale) { + Ok(plan) => Ok(Some(PreparedRegionScaledDirectPlan::Gray(plan))), + Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => Ok(None), + Err(error) => Err(error), + } + } + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 => { + Ok(Some(PreparedRegionScaledDirectPlan::Color( + build_region_scaled_direct_color_plan(input, roi, scale)?, + ))) + } + _ => Ok(None), + } +} + +fn build_region_scaled_direct_plan_with_session( + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, + session: &crate::MetalBackendSession, +) -> Result, Error> { + match fmt { + PixelFormat::Gray8 | PixelFormat::Gray16 => { + match build_region_scaled_direct_gray_plan(input, roi, scale) { + Ok(plan) => Ok(Some(PreparedRegionScaledDirectPlan::Gray(plan))), + Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => Ok(None), + Err(error) => Err(error), + } + } + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 => { + Ok(Some(PreparedRegionScaledDirectPlan::Color( + (*build_region_scaled_direct_color_plan_cached_with_session( + input, roi, scale, session, + )?) + .clone(), + ))) + } + _ => Ok(None), + } +} + +#[doc(hidden)] +pub(crate) fn benchmark_region_scaled_direct_plan_prepare( + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, +) -> Result<(), Error> { + if build_region_scaled_direct_plan(input, fmt, roi, scale)?.is_some() { + Ok(()) + } else { + Err(Error::UnsupportedMetalRequest { + reason: "J2K MetalDirect ROI+scaled plan preparation is unsupported for this benchmark input", + }) + } +} + +fn execute_region_scaled_direct_plan( + plan: PreparedRegionScaledDirectPlan, + fmt: PixelFormat, +) -> Result, Error> { + match plan { + PreparedRegionScaledDirectPlan::Gray(plan) => { + match crate::compute::execute_prepared_direct_grayscale_plan(&plan, fmt) { + Ok(surface) => Ok(Some(surface)), + Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => Ok(None), + Err(error) => Err(error), + } + } + PreparedRegionScaledDirectPlan::Color(plan) => { + match crate::compute::execute_hybrid_cpu_tier1_direct_color_plan(&plan, fmt) { + Ok(surface) => Ok(Some(surface)), + Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => { + Err(Error::UnsupportedMetalRequest { + reason: RGB_REGION_SCALED_METAL_DIRECT_UNSUPPORTED, + }) + } + Err(error) => Err(error), + } + } + } +} + +fn execute_region_scaled_direct_plan_with_device( + plan: PreparedRegionScaledDirectPlan, + fmt: PixelFormat, + device: &Device, +) -> Result, Error> { + match plan { + PreparedRegionScaledDirectPlan::Gray(plan) => { + match crate::compute::execute_prepared_direct_grayscale_plan_with_device( + &plan, fmt, device, + ) { + Ok(surface) => Ok(Some(surface)), + Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => Ok(None), + Err(error) => Err(error), + } + } + PreparedRegionScaledDirectPlan::Color(plan) => { + match crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_with_device( + &plan, fmt, device, + ) { + Ok(surface) => Ok(Some(surface)), + Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => { + Err(Error::UnsupportedMetalRequest { + reason: RGB_REGION_SCALED_METAL_DIRECT_UNSUPPORTED, + }) + } + Err(error) => Err(error), + } + } + } +} + +pub(crate) fn decode_region_scaled_grayscale_batch_direct_to_device( + requests: &[(Arc<[u8]>, Rect, Downscale)], + fmt: PixelFormat, +) -> Result, Error> { + if requests.is_empty() { + return Ok(Vec::new()); + } + if !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { + return Err(Error::MetalKernel { + message: format!( + "J2K MetalDirect region-scaled grayscale batch does not support {fmt:?}" + ), + }); + } + + let mut plans = Vec::with_capacity(requests.len()); + for (input, roi, scale) in requests { + let plan = build_region_scaled_direct_gray_plan(input.as_ref(), *roi, *scale)?; + plans.push(Arc::new(plan)); + } + crate::compute::execute_prepared_direct_grayscale_plan_batch(&plans, fmt) +} + +pub(crate) fn decode_region_scaled_color_batch_direct_to_device( + requests: &[(Arc<[u8]>, Rect, Downscale)], + fmt: PixelFormat, +) -> Result, Error> { + if requests.is_empty() { + return Ok(Vec::new()); + } + if !matches!( + fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 + ) { + return Err(Error::MetalKernel { + message: format!("J2K MetalDirect region-scaled color batch does not support {fmt:?}"), + }); + } + + if let Some((input, roi, scale)) = repeated_region_scaled_request(requests) { + let plan = build_region_scaled_direct_color_plan_cached(input.as_ref(), roi, scale)?; + let plans = vec![plan; requests.len()]; + return crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt); + } + + let plans = requests + .par_iter() + .map(|(input, roi, scale)| { + build_region_scaled_direct_color_plan_cached(input.as_ref(), *roi, *scale) + }) + .collect::, _>>()?; + crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt) +} + +pub(crate) fn decode_repeated_region_scaled_color_batch_direct_to_device( + input: &[u8], + roi: Rect, + scale: Downscale, + fmt: PixelFormat, + count: usize, +) -> Result, Error> { + if count == 0 { + return Err(Error::MetalKernel { + message: "J2K MetalDirect repeated region-scaled color batch requires count > 0" + .to_string(), + }); + } + if !matches!( + fmt, + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 + ) { + return Err(Error::MetalKernel { + message: format!("J2K MetalDirect region-scaled color batch does not support {fmt:?}"), + }); + } + + let plan = build_region_scaled_direct_color_plan_cached(input, roi, scale)?; + let plans = vec![plan; count]; + crate::compute::execute_hybrid_cpu_tier1_direct_color_plan_batch(&plans, fmt) +} + +fn repeated_region_scaled_request( + requests: &[(Arc<[u8]>, Rect, Downscale)], +) -> Option<(&Arc<[u8]>, Rect, Downscale)> { + let (first_input, first_roi, first_scale) = requests.first()?; + requests + .iter() + .all(|(input, roi, scale)| { + *roi == *first_roi + && *scale == *first_scale + && (Arc::ptr_eq(input, first_input) || input.as_ref() == first_input.as_ref()) + }) + .then_some((first_input, *first_roi, *first_scale)) +} + +fn build_region_scaled_direct_gray_plan( + input: &[u8], + roi: Rect, + scale: Downscale, +) -> Result { + let image = build_region_scaled_native_image(input, scale)?; + let mut context = NativeDecoderContext::default(); + let output_region = roi.scaled_covering(scale); + let plan = match image.build_direct_grayscale_plan_region_with_context( + &mut context, + ( + output_region.x, + output_region.y, + output_region.w, + output_region.h, + ), + ) { + Ok(plan) => plan, + Err(error) if direct::is_unsupported_direct_plan_error(&error.to_string()) => { + return Err(Error::MetalKernel { + message: format!( + "explicit J2K MetalDirect region-scaled batch currently supports grayscale direct plans only: {error}" + ), + }); + } + Err(error) => { + return Err(Error::Decode(J2kError::Backend(format!( + "failed to build J2K MetalDirect region-scaled grayscale plan: {error}" + )))); + } + }; + let mut prepared = crate::compute::prepare_direct_grayscale_plan(&plan)?; + crate::compute::crop_prepared_direct_grayscale_plan_to_output_region( + &mut prepared, + output_region, + )?; + Ok(prepared) +} + +fn build_region_scaled_direct_color_plan( + input: &[u8], + roi: Rect, + scale: Downscale, +) -> Result { + #[cfg(test)] + REGION_SCALED_COLOR_PLAN_BUILDS.fetch_add(1, Ordering::Relaxed); + + let profile_stages = crate::compute::metal_profile_stages_enabled(); + let total_started = profile_stages.then(Instant::now); + let native_image_started = profile_stages.then(Instant::now); + let image = build_region_scaled_native_image(input, scale)?; + let native_image_us = native_image_started.map(elapsed_us).unwrap_or_default(); + let direct_plan_started = profile_stages.then(Instant::now); + let mut context = NativeDecoderContext::default(); + let output_region = roi.scaled_covering(scale); + let plan = match image.build_direct_color_plan_region_with_context( + &mut context, + ( + output_region.x, + output_region.y, + output_region.w, + output_region.h, + ), + ) { + Ok(plan) => plan, + Err(error) if direct::is_unsupported_direct_plan_error(&error.to_string()) => { + return Err(Error::UnsupportedMetalRequest { + reason: RGB_REGION_SCALED_METAL_DIRECT_UNSUPPORTED, + }); + } + Err(error) => { + return Err(Error::Decode(J2kError::Backend(format!( + "failed to build J2K MetalDirect region-scaled color plan: {error}" + )))); + } + }; + let direct_plan_us = direct_plan_started.map(elapsed_us).unwrap_or_default(); + let prepare_started = profile_stages.then(Instant::now); + let mut prepared = crate::compute::prepare_direct_color_plan_for_cpu_upload(&plan)?; + let prepare_us = prepare_started.map(elapsed_us).unwrap_or_default(); + let crop_started = profile_stages.then(Instant::now); + crate::compute::crop_prepared_direct_color_plan_to_output_region(&mut prepared, output_region)?; + let crop_us = crop_started.map(elapsed_us).unwrap_or_default(); + if let Some(started) = total_started { + emit_region_scaled_color_plan_build_timings( + native_image_us, + direct_plan_us, + prepare_us, + crop_us, + elapsed_us(started), + ); + } + Ok(prepared) +} + +fn build_region_scaled_direct_color_plan_cached( + input: &[u8], + roi: Rect, + scale: Downscale, +) -> Result, Error> { + let cache_key = region_scaled_color_plan_cache_key(input, roi, scale); + if let Some(plan) = cached_region_scaled_direct_color_plan(cache_key) { + return Ok(plan); + } + + let plan = Arc::new(build_region_scaled_direct_color_plan(input, roi, scale)?); + store_region_scaled_direct_color_plan(cache_key, plan.clone()); + Ok(plan) +} + +fn build_region_scaled_direct_color_plan_cached_with_session( + input: &[u8], + roi: Rect, + scale: Downscale, + session: &crate::MetalBackendSession, +) -> Result, Error> { + let cache_key = region_scaled_color_plan_cache_key(input, roi, scale); + if let Some(plan) = cached_region_scaled_direct_color_plan_with_session(session, cache_key) { + return Ok(plan); + } + + let plan = Arc::new(build_region_scaled_direct_color_plan(input, roi, scale)?); + store_region_scaled_direct_color_plan_with_session(session, cache_key, plan.clone()); + Ok(plan) +} + +fn cached_region_scaled_direct_color_plan_with_session( + session: &crate::MetalBackendSession, + key: u64, +) -> Option> { + let guard = session.region_scaled_color_plan_cache.lock().ok()?; + guard.get(&key).cloned() +} + +fn store_region_scaled_direct_color_plan_with_session( + session: &crate::MetalBackendSession, + key: u64, + plan: Arc, +) { + if let Ok(mut guard) = session.region_scaled_color_plan_cache.lock() { + evict_one_region_scaled_color_plan_if_needed(&mut guard); + guard.insert(key, plan); + } +} + +fn region_scaled_color_plan_cache_key(input: &[u8], roi: Rect, scale: Downscale) -> u64 { + let mut hasher = DefaultHasher::new(); + input.len().hash(&mut hasher); + input.hash(&mut hasher); + roi.hash(&mut hasher); + scale.hash(&mut hasher); + hasher.finish() +} + +fn cached_region_scaled_direct_color_plan( + key: u64, +) -> Option> { + let cache = REGION_SCALED_COLOR_PLAN_CACHE.get_or_init(|| Mutex::new(HashMap::new())); + let guard = cache.lock().ok()?; + guard.get(&key).cloned() +} + +fn store_region_scaled_direct_color_plan( + key: u64, + plan: Arc, +) { + let cache = REGION_SCALED_COLOR_PLAN_CACHE.get_or_init(|| Mutex::new(HashMap::new())); + if let Ok(mut guard) = cache.lock() { + evict_one_region_scaled_color_plan_if_needed(&mut guard); + guard.insert(key, plan); + } +} + +fn evict_one_region_scaled_color_plan_if_needed(cache: &mut HashMap) { + if cache.len() < REGION_SCALED_COLOR_PLAN_CACHE_CAP { + return; + } + if let Some(key) = cache.keys().next().copied() { + cache.remove(&key); + } +} + +fn elapsed_us(started: Instant) -> u128 { + started.elapsed().as_micros() +} + +fn emit_region_scaled_color_plan_build_timings( + native_image_us: u128, + direct_plan_us: u128, + prepare_us: u128, + crop_us: u128, + total_us: u128, +) { + if !crate::compute::metal_profile_stages_enabled() { + return; + } + + let label = decode_profile_label(); + for (stage, elapsed_us) in [ + ("native_image", native_image_us), + ("direct_color_plan", direct_plan_us), + ("prepare_cpu_upload", prepare_us), + ("crop_prepared_plan", crop_us), + ("plan_total", total_us), + ] { + let processor = plan_stage_processor(stage); + let metric = plan_stage_metric(stage); + let metric_kind = plan_stage_metric_kind(stage); + let aggregation = plan_stage_aggregation(stage); + j2k_profile::emit_profile_row_now( + "j2k", + "decode", + "metal_cpu_hybrid_plan", + &[ + ("pipeline", "decode_hybrid".to_string()), + ("label", label.clone()), + ("stage", stage.to_string()), + ("processor", processor.to_string()), + ("metric", metric.to_string()), + ("metric_kind", metric_kind.to_string()), + ("aggregation", aggregation.to_string()), + ("fmt", "Rgb".to_string()), + ("batch_count", "1".to_string()), + ("elapsed_us", elapsed_us.to_string()), + ], + ); + } +} + +fn decode_profile_label() -> String { + std::env::var(METAL_PROFILE_DECODE_LABEL_ENV) + .ok() + .filter(|label| !label.is_empty()) + .map_or_else( + || "unlabeled".to_string(), + |label| sanitize_profile_label(&label), + ) +} + +fn sanitize_profile_label(label: &str) -> String { + label + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.') { + ch + } else { + '_' + } + }) + .collect() +} + +fn plan_stage_processor(stage: &str) -> &'static str { + match stage { + "native_image" | "direct_color_plan" | "prepare_cpu_upload" | "crop_prepared_plan" => "cpu", + _ => "hybrid", + } +} + +fn plan_stage_metric(stage: &str) -> &'static str { + match stage { + "native_image" => "native_image_us", + "direct_color_plan" => "direct_color_plan_us", + "prepare_cpu_upload" => "prepare_cpu_upload_us", + "crop_prepared_plan" => "crop_prepared_plan_us", + "plan_total" => "plan_total_us", + _ => "wall_us", + } +} + +fn plan_stage_metric_kind(_stage: &str) -> &'static str { + "wall_elapsed" +} + +fn plan_stage_aggregation(stage: &str) -> &'static str { + match stage { + "plan_total" => "inclusive", + _ => "exclusive", + } +} + +fn build_region_scaled_native_image( + input: &[u8], + scale: Downscale, +) -> Result, Error> { + let decoder = J2kDecoder::new(input)?; + let dims = decoder.inner.info().dimensions; + let target_dims = ( + dims.0.div_ceil(scale.denominator()), + dims.1.div_ceil(scale.denominator()), + ); + let settings = NativeDecodeSettings { + target_resolution: Some(target_dims), + ..NativeDecodeSettings::default() + }; + let image = + NativeImage::new(input, &settings).map_err(|error| J2kError::Backend(error.to_string()))?; + Ok(image) +} + +fn is_direct_region_scaled_runtime_fallback_error(error: &Error) -> bool { + crate::is_direct_runtime_fallback_error(error) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn encoded_rgb8_tile_for_region_scaled_plan_cache(seed: u8) -> Arc<[u8]> { + let mut pixels = j2k_test_support::gradient_u8(64, 64, 3); + for pixel in pixels.chunks_exact_mut(3) { + pixel[0] = pixel[0].wrapping_add(seed); + pixel[1] = pixel[1].wrapping_add(seed.wrapping_mul(3)); + pixel[2] = pixel[2].wrapping_add(seed.wrapping_mul(5)); + } + let options = j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..j2k_native::EncodeOptions::default() + }; + Arc::<[u8]>::from( + j2k_native::encode(&pixels, 64, 64, 3, 8, false, &options).expect("encode rgb8"), + ) + } + + fn region_scaled_plan_cache_roi() -> Rect { + Rect { + x: 8, + y: 8, + w: 32, + h: 32, + } + } + + #[test] + fn known_repeated_region_scaled_color_batch_builds_one_plan() { + let _guard = region_scaled_color_plan_test_lock_for_test(); + reset_region_scaled_color_plan_builds_for_test(); + let input = Arc::<[u8]>::from([1_u8, 2, 3, 4]); + let roi = Rect { + x: 0, + y: 0, + w: 64, + h: 64, + }; + + let result = decode_repeated_region_scaled_color_batch_direct_to_device( + input.as_ref(), + roi, + Downscale::Half, + PixelFormat::Rgb8, + 4, + ); + + assert!(result.is_err()); + assert_eq!(region_scaled_color_plan_builds_for_test(), 1); + } + + #[test] + fn known_repeated_region_scaled_color_batch_rejects_zero_count() { + let _guard = region_scaled_color_plan_test_lock_for_test(); + reset_region_scaled_color_plan_builds_for_test(); + let result = decode_repeated_region_scaled_color_batch_direct_to_device( + &[1_u8, 2, 3, 4], + Rect { + x: 0, + y: 0, + w: 64, + h: 64, + }, + Downscale::Half, + PixelFormat::Rgb8, + 0, + ); + + assert!(matches!(result, Err(Error::MetalKernel { .. }))); + assert_eq!(region_scaled_color_plan_builds_for_test(), 0); + } + + #[cfg(target_os = "macos")] + #[test] + fn known_repeated_region_scaled_color_batch_reuses_cached_plan_across_calls() { + if Device::system_default().is_none() { + eprintln!("skipping repeated color plan cache test: no Metal device"); + return; + } + + let _guard = region_scaled_color_plan_test_lock_for_test(); + let input = encoded_rgb8_tile_for_region_scaled_plan_cache(17); + let roi = region_scaled_plan_cache_roi(); + reset_region_scaled_color_plan_cache_for_test(); + reset_region_scaled_color_plan_builds_for_test(); + + for _ in 0..2 { + let surfaces = decode_repeated_region_scaled_color_batch_direct_to_device( + input.as_ref(), + roi, + Downscale::Quarter, + PixelFormat::Rgb8, + 4, + ) + .expect("repeated RGB region-scaled batch"); + assert_eq!(surfaces.len(), 4); + } + + assert_eq!( + region_scaled_color_plan_builds_for_test(), + 1, + "same RGB ROI+scaled batch should reuse the prepared direct color plan across calls" + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn known_distinct_region_scaled_color_batch_reuses_cached_plans_across_calls() { + if Device::system_default().is_none() { + eprintln!("skipping distinct color plan cache test: no Metal device"); + return; + } + + let _guard = region_scaled_color_plan_test_lock_for_test(); + let first = encoded_rgb8_tile_for_region_scaled_plan_cache(29); + let second = encoded_rgb8_tile_for_region_scaled_plan_cache(43); + let roi = region_scaled_plan_cache_roi(); + let requests = vec![ + (first, roi, Downscale::Quarter), + (second, roi, Downscale::Quarter), + ]; + reset_region_scaled_color_plan_cache_for_test(); + reset_region_scaled_color_plan_builds_for_test(); + + for _ in 0..2 { + let surfaces = + decode_region_scaled_color_batch_direct_to_device(&requests, PixelFormat::Rgb8) + .expect("distinct RGB region-scaled batch"); + assert_eq!(surfaces.len(), requests.len()); + } + + assert_eq!( + region_scaled_color_plan_builds_for_test(), + 2, + "same distinct RGB ROI+scaled inputs should reuse prepared direct color plans across calls" + ); + } +} diff --git a/crates/signinum-j2k-metal/src/idwt.metal b/crates/j2k-metal/src/idwt.metal similarity index 91% rename from crates/signinum-j2k-metal/src/idwt.metal rename to crates/j2k-metal/src/idwt.metal index f5b431b1..5cb0fe91 100644 --- a/crates/signinum-j2k-metal/src/idwt.metal +++ b/crates/j2k-metal/src/idwt.metal @@ -149,19 +149,27 @@ kernel void j2k_idwt_interleave( if (low_y && low_x) { const uint band_x = full_band_x - params.ll_x; const uint band_y = full_band_y - params.ll_y; - out[out_idx] = ll[band_y * params.ll_width + band_x]; + out[out_idx] = (band_x < params.ll_width && band_y < params.ll_height) + ? ll[band_y * params.ll_width + band_x] + : 0.0f; } else if (low_y) { const uint band_x = full_band_x - params.hl_x; const uint band_y = full_band_y - params.hl_y; - out[out_idx] = hl[band_y * params.hl_width + band_x]; + out[out_idx] = (band_x < params.hl_width && band_y < params.hl_height) + ? hl[band_y * params.hl_width + band_x] + : 0.0f; } else if (low_x) { const uint band_x = full_band_x - params.lh_x; const uint band_y = full_band_y - params.lh_y; - out[out_idx] = lh[band_y * params.lh_width + band_x]; + out[out_idx] = (band_x < params.lh_width && band_y < params.lh_height) + ? lh[band_y * params.lh_width + band_x] + : 0.0f; } else { const uint band_x = full_band_x - params.hh_x; const uint band_y = full_band_y - params.hh_y; - out[out_idx] = hh[band_y * params.hh_width + band_x]; + out[out_idx] = (band_x < params.hh_width && band_y < params.hh_height) + ? hh[band_y * params.hh_width + band_x] + : 0.0f; } } @@ -192,19 +200,27 @@ kernel void j2k_idwt_interleave_batched( if (low_y && low_x) { const uint band_x = full_band_x - params.ll_x; const uint band_y = full_band_y - params.ll_y; - out[out_idx] = ll[gid.z * params.ll_instance_stride + band_y * params.ll_width + band_x]; + out[out_idx] = (band_x < params.ll_width && band_y < params.ll_height) + ? ll[gid.z * params.ll_instance_stride + band_y * params.ll_width + band_x] + : 0.0f; } else if (low_y) { const uint band_x = full_band_x - params.hl_x; const uint band_y = full_band_y - params.hl_y; - out[out_idx] = hl[gid.z * params.hl_instance_stride + band_y * params.hl_width + band_x]; + out[out_idx] = (band_x < params.hl_width && band_y < params.hl_height) + ? hl[gid.z * params.hl_instance_stride + band_y * params.hl_width + band_x] + : 0.0f; } else if (low_x) { const uint band_x = full_band_x - params.lh_x; const uint band_y = full_band_y - params.lh_y; - out[out_idx] = lh[gid.z * params.lh_instance_stride + band_y * params.lh_width + band_x]; + out[out_idx] = (band_x < params.lh_width && band_y < params.lh_height) + ? lh[gid.z * params.lh_instance_stride + band_y * params.lh_width + band_x] + : 0.0f; } else { const uint band_x = full_band_x - params.hh_x; const uint band_y = full_band_y - params.hh_y; - out[out_idx] = hh[gid.z * params.hh_instance_stride + band_y * params.hh_width + band_x]; + out[out_idx] = (band_x < params.hh_width && band_y < params.hh_height) + ? hh[gid.z * params.hh_instance_stride + band_y * params.hh_width + band_x] + : 0.0f; } } @@ -461,39 +477,27 @@ kernel void j2k_idwt_irreversible97_single_decomposition( if (low_y && low_x) { const uint band_x = full_band_x - params.ll_x; const uint band_y = full_band_y - params.ll_y; - if (band_x >= params.ll_width || band_y >= params.ll_height) { - status->code = J2K_IDWT_STATUS_FAIL; - status->detail = 2u; - return; - } - out[out_idx] = ll[band_y * params.ll_width + band_x]; + out[out_idx] = (band_x < params.ll_width && band_y < params.ll_height) + ? ll[band_y * params.ll_width + band_x] + : 0.0f; } else if (low_y) { const uint band_x = full_band_x - params.hl_x; const uint band_y = full_band_y - params.hl_y; - if (band_x >= params.hl_width || band_y >= params.hl_height) { - status->code = J2K_IDWT_STATUS_FAIL; - status->detail = 3u; - return; - } - out[out_idx] = hl[band_y * params.hl_width + band_x]; + out[out_idx] = (band_x < params.hl_width && band_y < params.hl_height) + ? hl[band_y * params.hl_width + band_x] + : 0.0f; } else if (low_x) { const uint band_x = full_band_x - params.lh_x; const uint band_y = full_band_y - params.lh_y; - if (band_x >= params.lh_width || band_y >= params.lh_height) { - status->code = J2K_IDWT_STATUS_FAIL; - status->detail = 4u; - return; - } - out[out_idx] = lh[band_y * params.lh_width + band_x]; + out[out_idx] = (band_x < params.lh_width && band_y < params.lh_height) + ? lh[band_y * params.lh_width + band_x] + : 0.0f; } else { const uint band_x = full_band_x - params.hh_x; const uint band_y = full_band_y - params.hh_y; - if (band_x >= params.hh_width || band_y >= params.hh_height) { - status->code = J2K_IDWT_STATUS_FAIL; - status->detail = 5u; - return; - } - out[out_idx] = hh[band_y * params.hh_width + band_x]; + out[out_idx] = (band_x < params.hh_width && band_y < params.hh_height) + ? hh[band_y * params.hh_width + band_x] + : 0.0f; } } } diff --git a/crates/signinum-j2k-metal/src/idwt.rs b/crates/j2k-metal/src/idwt.rs similarity index 96% rename from crates/signinum-j2k-metal/src/idwt.rs rename to crates/j2k-metal/src/idwt.rs index 146e4f27..d15dff8b 100644 --- a/crates/signinum-j2k-metal/src/idwt.rs +++ b/crates/j2k-metal/src/idwt.rs @@ -3,8 +3,8 @@ #[cfg(target_os = "macos")] use crate::compute; #[cfg(target_os = "macos")] -use signinum_j2k_native::J2kWaveletTransform; -use signinum_j2k_native::{ +use j2k_native::J2kWaveletTransform; +use j2k_native::{ decode_ht_code_block_scalar, HtCodeBlockDecodeJob, HtCodeBlockDecoder, J2kSingleDecompositionIdwtJob, Result, }; @@ -38,7 +38,7 @@ impl HtCodeBlockDecoder for MetalIdwtDecoder { compute::decode_irreversible97_single_decomposition_idwt(job, output) } } - .map_err(|_| signinum_j2k_native::DecodingError::CodeBlockDecodeFailure)?; + .map_err(|_| j2k_native::DecodingError::CodeBlockDecodeFailure)?; self.kernel_dispatches = self.kernel_dispatches.saturating_add(1); return Ok(true); } @@ -89,7 +89,7 @@ fn supports_metal_idwt(job: &J2kSingleDecompositionIdwtJob<'_>) -> bool { #[cfg(test)] mod tests { use super::MetalIdwtDecoder; - use signinum_j2k_native::{ + use j2k_native::{ encode, DecodeSettings, DecoderContext, EncodeOptions, HtCodeBlockDecodeJob, HtCodeBlockDecoder, Image, }; @@ -216,8 +216,8 @@ mod tests { &mut self, job: HtCodeBlockDecodeJob<'_>, output: &mut [f32], - ) -> signinum_j2k_native::Result<()> { - signinum_j2k_native::decode_ht_code_block_scalar(job, output) + ) -> j2k_native::Result<()> { + j2k_native::decode_ht_code_block_scalar(job, output) } } diff --git a/crates/signinum-j2k-metal/src/lib.rs b/crates/j2k-metal/src/lib.rs similarity index 74% rename from crates/signinum-j2k-metal/src/lib.rs rename to crates/j2k-metal/src/lib.rs index 0e8c58ed..3770aaa5 100644 --- a/crates/signinum-j2k-metal/src/lib.rs +++ b/crates/j2k-metal/src/lib.rs @@ -1,100 +1,150 @@ // SPDX-License-Identifier: Apache-2.0 +//! Metal-backed JPEG 2000 and HTJ2K decode and encode adapters. +//! +//! This crate wraps the CPU/native J2K implementation with optional +//! Metal-resident decode surfaces, batch decode sessions, and lossless encode +//! helpers on macOS. Non-macOS builds keep the same API surface and return +//! `Error::MetalUnavailable` for explicit Metal-only requests. + #![deny(unsafe_op_in_unsafe_fn)] #![warn(unreachable_pub)] mod batch; +#[cfg(target_os = "macos")] +mod buffer_pool; #[cfg(any(test, target_os = "macos"))] mod classic; #[cfg(target_os = "macos")] mod compute; -mod dicom; #[cfg(target_os = "macos")] mod direct; mod encode; #[cfg(any(test, target_os = "macos"))] mod ht; +#[cfg(target_os = "macos")] +mod hybrid; #[cfg(any(test, target_os = "macos"))] mod idwt; #[cfg(any(test, target_os = "macos"))] mod mct; mod profile; +#[cfg(target_os = "macos")] +mod profile_env; mod routing; #[cfg(any(test, target_os = "macos"))] mod store; use core::convert::Infallible; -use std::sync::Arc; #[cfg(target_os = "macos")] use std::{ collections::{hash_map::DefaultHasher, HashMap}, hash::{Hash, Hasher}, sync::{Mutex, OnceLock}, }; +use std::{ops::Range, sync::Arc}; -use signinum_core::{ - copy_tight_pixels_to_strided_output, BackendKind, BackendRequest, BufferError, CodecError, - DecodeOutcome, DeviceSubmission, DeviceSurface, Downscale, ImageCodec, ImageDecode, - ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, ReadySubmission, Rect, - TileBatchDecodeDevice, TileBatchDecodeManyDevice, TileBatchDecodeSubmit, -}; -use signinum_j2k::{ +use j2k::{ adapter::device_plan::{DeviceDecodePlan, DeviceDecodeRequest}, J2kContext as CpuJ2kContext, J2kDecoder as CpuDecoder, J2kError, J2kScratchPool as CpuJ2kScratchPool, J2kView, }; +use j2k_core::{ + copy_tight_pixels_to_strided_output, BackendKind, BackendRequest, BufferError, CodecError, + DecodeOutcome, DeviceMemoryRange, DeviceSubmission, DeviceSurface, Downscale, ImageCodec, + ImageDecode, ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, ReadySubmission, Rect, + TileBatchDecodeDevice, TileBatchDecodeManyDevice, TileBatchDecodeSubmit, +}; #[cfg(target_os = "macos")] -use signinum_j2k_native::{ +use j2k_native::{ DecodeSettings as NativeDecodeSettings, DecoderContext as NativeDecoderContext, Image as NativeImage, J2kDirectColorPlan, J2kDirectGrayscalePlan, }; +#[cfg(target_os = "macos")] +use j2k_metal_support::{system_default_device, MetalSupportError}; +#[cfg(target_os = "macos")] +use metal::foreign_types::ForeignType; #[cfg(target_os = "macos")] use metal::{Buffer, Device, MTLResourceOptions}; #[doc(hidden)] -pub use dicom::{ - extract_dicom_encapsulated_frames, extract_dicom_encapsulated_frames_with_limit, - DicomFrameExtractError, -}; +pub use batch::{benchmark_group_region_scaled_requests, BenchmarkGroupedRequests}; pub use encode::{ - encode_lossless_from_metal_buffer, encode_lossless_from_metal_buffer_to_metal, - encode_lossless_from_metal_buffer_to_metal_with_report, - encode_lossless_from_metal_buffer_with_report, encode_lossless_from_metal_buffers, - encode_lossless_from_metal_buffers_to_metal, encode_lossless_from_metal_buffers_to_metal_batch, - encode_lossless_from_metal_buffers_to_metal_with_report, - encode_lossless_from_metal_buffers_with_report, encode_lossless_from_padded_metal_buffer, - encode_lossless_from_padded_metal_buffer_to_metal, - encode_lossless_from_padded_metal_buffer_to_metal_with_report, - encode_lossless_from_padded_metal_buffer_with_report, - encode_lossless_from_padded_metal_buffers, encode_lossless_from_padded_metal_buffers_to_metal, - encode_lossless_from_padded_metal_buffers_to_metal_batch, - encode_lossless_from_padded_metal_buffers_to_metal_with_report, - encode_lossless_from_padded_metal_buffers_with_report, submit_lossless_from_metal_buffer, - submit_lossless_from_metal_buffers, submit_lossless_from_metal_buffers_with_config, - submit_lossless_from_padded_metal_buffer, submit_lossless_from_padded_metal_buffers, - submit_lossless_from_padded_metal_buffers_with_config, validate_lossless_roundtrip_on_metal, - validate_lossless_roundtrip_on_metal_with_session, MetalEncodeStageAccelerator, - MetalEncodedJ2k, MetalLosslessBufferEncodeBatchOutcome, MetalLosslessBufferEncodeOutcome, - MetalLosslessEncodeBatchStats, MetalLosslessEncodeConfig, MetalLosslessEncodeOutcome, - MetalLosslessEncodeResidency, MetalLosslessEncodeTile, SubmittedJ2kLosslessMetalEncode, + encode_lossless_batch_with_report, submit_lossless_batch, submit_lossless_batch_to_metal, + validate_lossless_roundtrip_on_metal, validate_lossless_roundtrip_on_metal_with_session, + MetalEncodeInputStaging, MetalEncodeStageAccelerator, MetalEncodedJ2k, + MetalLosslessBufferEncodeBatchOutcome, MetalLosslessBufferEncodeOutcome, + MetalLosslessEncodeBatchRequest, MetalLosslessEncodeBatchStats, MetalLosslessEncodeConfig, + MetalLosslessEncodeOutcome, MetalLosslessEncodeResidency, MetalLosslessEncodeStageStats, + MetalLosslessEncodeTile, SubmittedJ2kLosslessMetalBufferEncodeBatch, SubmittedJ2kLosslessMetalEncodeBatch, }; +#[cfg(target_os = "macos")] +#[doc(hidden)] +pub fn benchmark_region_scaled_direct_plan_prepare( + input: &[u8], + fmt: PixelFormat, + roi: Rect, + scale: Downscale, +) -> Result<(), Error> { + hybrid::benchmark_region_scaled_direct_plan_prepare(input, fmt, roi, scale) +} + +#[cfg(not(target_os = "macos"))] +#[doc(hidden)] +pub fn benchmark_region_scaled_direct_plan_prepare( + _input: &[u8], + _fmt: PixelFormat, + _roi: Rect, + _scale: Downscale, +) -> Result<(), Error> { + Err(Error::MetalUnavailable) +} + #[derive(Debug, thiserror::Error)] +/// Errors returned by the Metal J2K backend. pub enum Error { + /// Error returned by the CPU or native J2K decoder. #[error(transparent)] Decode(#[from] J2kError), + /// Output buffer validation failed. #[error(transparent)] Buffer(#[from] BufferError), - #[error("backend request {request:?} is not supported by signinum-j2k-metal")] - UnsupportedBackend { request: BackendRequest }, + /// The requested backend is unsupported by this crate. + #[error("backend request {request:?} is not supported by j2k-metal")] + UnsupportedBackend { + /// Backend requested by the caller. + request: BackendRequest, + }, + /// A Metal-specific request is structurally unsupported. #[error("unsupported J2K Metal request: {reason}")] - UnsupportedMetalRequest { reason: &'static str }, + UnsupportedMetalRequest { + /// Static reason describing the rejected request. + reason: &'static str, + }, + /// Metal is not available on the current host. #[error("Metal is unavailable on this host")] MetalUnavailable, + /// Metal runtime creation or device setup failed. + #[error("Metal runtime error: {message}")] + MetalRuntime { + /// Runtime error message. + message: String, + }, + /// Metal kernel launch, validation, or completion failed. #[error("Metal kernel error: {message}")] - MetalKernel { message: String }, + MetalKernel { + /// Kernel error message. + message: String, + }, + /// Shared Metal backend state was poisoned by a prior panic. + #[error("Metal state `{state}` is poisoned")] + MetalStatePoisoned { + /// Name of the poisoned state. + state: &'static str, + }, } impl CodecError for Error { @@ -112,7 +162,6 @@ impl CodecError for Error { Self::UnsupportedBackend { .. } | Self::UnsupportedMetalRequest { .. } | Self::MetalUnavailable - | Self::MetalKernel { .. } ) || matches!(self, Self::Decode(inner) if inner.is_unsupported()) } @@ -143,12 +192,6 @@ struct DirectColorPlanCacheEntry { prepared: Arc, } -#[cfg(target_os = "macos")] -static DIRECT_GRAY_PLAN_CACHE: OnceLock>> = - OnceLock::new(); -#[cfg(target_os = "macos")] -static DIRECT_COLOR_PLAN_CACHE: OnceLock>> = - OnceLock::new(); #[cfg(target_os = "macos")] const DIRECT_PLAN_CACHE_CAP: usize = 128; #[cfg(target_os = "macos")] @@ -157,6 +200,7 @@ const AUTO_REPEATED_GRAYSCALE_MIN_DIM: u32 = 512; const AUTO_REPEATED_GRAYSCALE_MIN_COUNT: usize = 16; #[derive(Clone)] +/// Decoded J2K image surface returned by the Metal backend. pub struct Surface { backend: BackendKind, residency: SurfaceResidency, @@ -168,50 +212,103 @@ pub struct Surface { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Where a decoded J2K surface is currently resident. pub enum SurfaceResidency { + /// Pixel bytes are resident in host memory. Host, + /// Pixel bytes were produced directly by a Metal decode kernel. MetalResidentDecode, + /// Pixel bytes were decoded on CPU and uploaded into a Metal buffer. CpuStagedMetalUpload, } #[cfg(target_os = "macos")] const CPU_STAGED_METAL_REQUIRES_EXPLICIT_API: &str = "CPU-staged Metal upload requires the explicit CPU-staged API; BackendRequest::Metal only accepts resident Metal decode"; - impl Surface { + /// Current residency for the surface bytes. pub fn residency(&self) -> SurfaceResidency { self.residency } + /// Number of bytes between consecutive rows. pub fn pitch_bytes(&self) -> usize { self.pitch_bytes } - pub fn as_bytes(&self) -> &[u8] { + fn checked_storage_range(&self, storage_len: usize) -> Result, Error> { + let len = self.byte_len(); + let end = self + .byte_offset + .checked_add(len) + .ok_or_else(|| Error::MetalKernel { + message: "J2K Metal surface byte range overflows usize".to_string(), + })?; + if end > storage_len { + return Err(Error::MetalKernel { + message: format!( + "J2K Metal surface byte range {start}..{end} exceeds storage length {storage_len}", + start = self.byte_offset + ), + }); + } + Ok(self.byte_offset..end) + } + + fn storage_bytes(&self) -> Result<&[u8], Error> { match &self.storage { Storage::Host(bytes) => { - let len = self.byte_len(); - &bytes[self.byte_offset..self.byte_offset + len] + let range = self.checked_storage_range(bytes.len())?; + Ok(&bytes[range]) } #[cfg(target_os = "macos")] Storage::Metal(buffer) => { - let len = self.byte_len(); + let storage_len = + usize::try_from(buffer.length()).map_err(|_| Error::MetalKernel { + message: "J2K Metal buffer length does not fit usize".to_string(), + })?; + let range = self.checked_storage_range(storage_len)?; + let contents = buffer.contents(); + if contents.is_null() { + return Err(Error::MetalKernel { + message: "J2K Metal surface buffer is not host-addressable".to_string(), + }); + } + // SAFETY: Metal surface byte views are bounded by validated dimensions and formats. unsafe { - core::slice::from_raw_parts( - buffer.contents().cast::().add(self.byte_offset), - len, - ) + Ok(core::slice::from_raw_parts( + contents.cast::().add(range.start), + range.len(), + )) } } } } + /// Return the tightly packed surface bytes. + /// + /// Metal-backed surfaces are expected to use host-addressable buffers. This + /// method panics only if the surface metadata is internally inconsistent; + /// fallible operations such as [`Self::download_into`] return those errors. + pub fn as_bytes(&self) -> &[u8] { + self.storage_bytes() + .expect("validated J2K Metal surface byte range") + } + + /// Copy the tightly packed surface into a caller-provided strided buffer. pub fn download_into(&self, out: &mut [u8], stride: usize) -> Result<(), Error> { - copy_tight_pixels_to_strided_output(self.as_bytes(), self.dimensions, self.fmt, out, stride) - .map_err(Error::from) + copy_tight_pixels_to_strided_output( + self.storage_bytes()?, + self.dimensions, + self.fmt, + out, + stride, + ) + .map_err(Error::from) } #[cfg(target_os = "macos")] + /// Return the Metal buffer and byte offset when the surface is Metal-backed. pub fn metal_buffer(&self) -> Option<(&Buffer, usize)> { match &self.storage { Storage::Metal(buffer) => Some((buffer, self.byte_offset)), @@ -260,6 +357,18 @@ impl DeviceSurface for Surface { self.backend } + fn residency(&self) -> j2k_core::SurfaceResidency { + match self.residency { + SurfaceResidency::Host => j2k_core::SurfaceResidency::Host, + SurfaceResidency::MetalResidentDecode => { + j2k_core::SurfaceResidency::MetalResidentDecode + } + SurfaceResidency::CpuStagedMetalUpload => { + j2k_core::SurfaceResidency::CpuStagedMetalUpload + } + } + } + fn dimensions(&self) -> (u32, u32) { self.dimensions } @@ -271,29 +380,82 @@ impl DeviceSurface for Surface { fn byte_len(&self) -> usize { self.pitch_bytes * self.dimensions.1 as usize } + + fn memory_range(&self) -> Option { + match &self.storage { + Storage::Host(_) => None, + #[cfg(target_os = "macos")] + Storage::Metal(buffer) => Some(DeviceMemoryRange::new( + BackendKind::Metal, + u64::try_from(buffer.as_ptr() as usize).ok()?, + self.byte_offset, + self.byte_len(), + )), + } + } } #[cfg(target_os = "macos")] #[derive(Clone)] +/// Reusable Metal device session for J2K decode and encode submissions. pub struct MetalBackendSession { device: Device, + runtime: Arc, MetalSupportError>>>, + direct_gray_plan_cache: Arc>>, + direct_color_plan_cache: Arc>>, + region_scaled_color_plan_cache: + Arc>>>, } #[cfg(target_os = "macos")] impl MetalBackendSession { + /// Create a session bound to an existing Metal device. pub fn new(device: Device) -> Self { - Self { device } + Self { + device, + runtime: Arc::new(OnceLock::new()), + direct_gray_plan_cache: Arc::new(Mutex::new(HashMap::new())), + direct_color_plan_cache: Arc::new(Mutex::new(HashMap::new())), + region_scaled_color_plan_cache: Arc::new(Mutex::new(HashMap::new())), + } } + /// Create a session from the system default Metal device. pub fn system_default() -> Result { - Device::system_default() + system_default_device() .map(Self::new) - .ok_or(Error::MetalUnavailable) + .map_err(|error| crate::compute::runtime_initialization_error(&error)) } + /// Metal device used by this session. pub fn device(&self) -> &metal::DeviceRef { self.device.as_ref() } + + pub(crate) fn runtime(&self) -> Result, Error> { + match self.runtime.get_or_init(|| { + crate::compute::MetalRuntime::new_with_device(&self.device).map(Arc::new) + }) { + Ok(runtime) => Ok(runtime.clone()), + Err(error) => Err(crate::compute::runtime_initialization_error(error)), + } + } + + #[cfg(test)] + pub(crate) fn direct_cache_ids_for_test(&self) -> (usize, usize, usize) { + ( + Arc::as_ptr(&self.direct_gray_plan_cache) as usize, + Arc::as_ptr(&self.direct_color_plan_cache) as usize, + Arc::as_ptr(&self.region_scaled_color_plan_cache) as usize, + ) + } +} + +#[cfg(target_os = "macos")] +impl j2k_core::AcceleratorSession for MetalBackendSession { + fn backend_kind(&self) -> BackendKind { + BackendKind::Metal + } } #[cfg(target_os = "macos")] @@ -301,36 +463,58 @@ impl core::fmt::Debug for MetalBackendSession { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("MetalBackendSession") .field("device", &self.device.name()) - .finish() + .finish_non_exhaustive() } } #[cfg(not(target_os = "macos"))] #[derive(Clone, Copy, Debug, Default)] +/// Placeholder Metal session for non-macOS builds. pub struct MetalBackendSession { _private: (), } #[cfg(not(target_os = "macos"))] impl MetalBackendSession { + /// Return `Error::MetalUnavailable` on hosts without Metal support. pub fn system_default() -> Result { Err(Error::MetalUnavailable) } } #[derive(Clone, Default)] +/// Shared batching session used by J2K Metal submit APIs. pub struct MetalSession { shared: batch::SharedSession, + #[cfg(target_os = "macos")] + backend: Option, } impl MetalSession { - pub fn submissions(&self) -> u64 { - self.shared.0.lock().expect("J2K Metal session").submissions + /// Create a batching session backed by an explicit Metal backend session. + #[cfg(target_os = "macos")] + pub fn with_backend_session(backend: MetalBackendSession) -> Self { + Self { + shared: batch::SharedSession::default(), + backend: Some(backend), + } + } + + /// Metal backend session owned by this batching session, if any. + #[cfg(target_os = "macos")] + pub fn backend_session(&self) -> Option<&MetalBackendSession> { + self.backend.as_ref() } - fn record_submit(&mut self) { - let mut session = self.shared.0.lock().expect("J2K Metal session"); + /// Number of Metal or emulated submissions flushed through this session. + pub fn submissions(&self) -> Result { + Ok(self.shared.lock()?.submissions) + } + + fn record_submit(&mut self) -> Result<(), Error> { + let mut session = self.shared.lock()?; session.submissions = session.submissions.saturating_add(1); + Ok(()) } } @@ -382,7 +566,7 @@ impl MetalTileBatch { /// /// Queued requests normally do not increment this until `decode_all` waits /// on the first result. - pub fn submissions(&self) -> u64 { + pub fn submissions(&self) -> Result { self.session.submissions() } @@ -411,7 +595,7 @@ impl MetalTileBatch { fmt, backend, batch::BatchOp::Full, - ); + )?; self.submissions.push(submission); Ok(slot) } @@ -443,7 +627,7 @@ impl MetalTileBatch { fmt, backend, batch::BatchOp::Region(roi), - ); + )?; self.submissions.push(submission); Ok(slot) } @@ -475,7 +659,7 @@ impl MetalTileBatch { fmt, backend, batch::BatchOp::Scaled(scale), - ); + )?; self.submissions.push(submission); Ok(slot) } @@ -508,7 +692,7 @@ impl MetalTileBatch { fmt, backend, batch::BatchOp::RegionScaled { roi, scale }, - ); + )?; self.submissions.push(submission); Ok(slot) } @@ -523,6 +707,7 @@ impl MetalTileBatch { } } +/// JPEG 2000 decoder that can return host or Metal-resident surfaces. pub struct J2kDecoder<'a> { inner: CpuDecoder<'a>, pool: CpuJ2kScratchPool, @@ -541,6 +726,7 @@ pub struct J2kDecoder<'a> { } impl<'a> J2kDecoder<'a> { + /// Parse a J2K or HTJ2K codestream into a decoder. pub fn new(input: &'a [u8]) -> Result { Ok(Self { inner: CpuDecoder::new(input)?, @@ -560,6 +746,7 @@ impl<'a> J2kDecoder<'a> { }) } + /// Create a decoder from an already parsed J2K view. pub fn from_view(view: J2kView<'a>) -> Result { Ok(Self { inner: CpuDecoder::from_view(view)?, @@ -579,10 +766,12 @@ impl<'a> J2kDecoder<'a> { }) } + /// Borrow the underlying CPU J2K decoder. pub fn inner(&self) -> &CpuDecoder<'a> { &self.inner } + /// Decode a full image into a Metal-resident device surface. pub fn decode_to_device_with_session( &mut self, fmt: PixelFormat, @@ -596,13 +785,13 @@ impl<'a> J2kDecoder<'a> { #[cfg(target_os = "macos")] { - if let Some(surface) = - self.decode_direct_to_surface_with_device(fmt, &session.device)? - { - Ok(surface) - } else { - self.decode_full_to_metal_surface_with_device(fmt, &session.device) - } + crate::compute::with_runtime_for_session(session, |_| { + if let Some(surface) = self.decode_direct_to_surface_with_session(fmt, session)? { + Ok(surface) + } else { + self.decode_full_to_metal_surface_with_device(fmt, &session.device) + } + }) } #[cfg(not(target_os = "macos"))] { @@ -611,10 +800,12 @@ impl<'a> J2kDecoder<'a> { } } + /// Decode a full image into a host-backed surface. pub fn decode_to_host_surface(&mut self, fmt: PixelFormat) -> Result { self.decode_to_cpu_surface(fmt) } + /// Decode a source region into a host-backed surface. pub fn decode_region_to_host_surface( &mut self, fmt: PixelFormat, @@ -627,6 +818,7 @@ impl<'a> J2kDecoder<'a> { self.decode_region_to_cpu_surface(fmt, plan) } + /// Decode a full image at reduced resolution into a host-backed surface. pub fn decode_scaled_to_host_surface( &mut self, fmt: PixelFormat, @@ -639,6 +831,7 @@ impl<'a> J2kDecoder<'a> { self.decode_scaled_to_cpu_surface(fmt, scale, plan) } + /// Decode a source region at reduced resolution into a host-backed surface. pub fn decode_region_scaled_to_host_surface( &mut self, fmt: PixelFormat, @@ -652,6 +845,7 @@ impl<'a> J2kDecoder<'a> { self.decode_region_scaled_to_cpu_surface(fmt, roi, scale, plan) } + /// Decode a full image on CPU and upload the result into a Metal surface. pub fn decode_to_cpu_staged_metal_surface_with_session( &mut self, fmt: PixelFormat, @@ -678,6 +872,7 @@ impl<'a> J2kDecoder<'a> { } } + /// Decode a source region on CPU and upload the result into a Metal surface. pub fn decode_region_to_cpu_staged_metal_surface_with_session( &mut self, fmt: PixelFormat, @@ -714,6 +909,7 @@ impl<'a> J2kDecoder<'a> { } } + /// Decode a full image at reduced resolution on CPU and upload it into Metal. pub fn decode_scaled_to_cpu_staged_metal_surface_with_session( &mut self, fmt: PixelFormat, @@ -745,6 +941,7 @@ impl<'a> J2kDecoder<'a> { } } + /// Decode a scaled source region on CPU and upload it into a Metal surface. pub fn decode_region_scaled_to_cpu_staged_metal_surface_with_session( &mut self, fmt: PixelFormat, @@ -795,12 +992,13 @@ impl<'a> J2kDecoder<'a> { } #[cfg(target_os = "macos")] - fn ensure_prepared_direct_gray_plan( + fn ensure_prepared_direct_gray_plan_with_session( &mut self, + session: &MetalBackendSession, ) -> Result>, Error> { let cache_key = direct_gray_plan_cache_key(self.inner.bytes()); if self.native_prepared_direct_gray_plan.is_none() { - if let Some((plan, prepared)) = cached_global_direct_gray_plan(cache_key) { + if let Some((plan, prepared)) = cached_session_direct_gray_plan(session, cache_key) { self.native_direct_gray_plan = Some(plan); self.native_prepared_direct_gray_plan = Some(prepared); } @@ -827,7 +1025,7 @@ impl<'a> J2kDecoder<'a> { } }; let prepared = Arc::new(crate::compute::prepare_direct_grayscale_plan(&plan)?); - store_global_direct_gray_plan(cache_key, &plan, prepared.clone()); + store_session_direct_gray_plan(session, cache_key, &plan, prepared.clone()); self.native_direct_gray_plan = Some(plan); self.native_prepared_direct_gray_plan = Some(prepared); } @@ -836,12 +1034,13 @@ impl<'a> J2kDecoder<'a> { } #[cfg(target_os = "macos")] - fn ensure_prepared_direct_color_plan( + fn ensure_prepared_direct_color_plan_with_session( &mut self, + session: &MetalBackendSession, ) -> Result>, Error> { let cache_key = direct_plan_cache_key(self.inner.bytes()); if self.native_prepared_direct_color_plan.is_none() { - if let Some((plan, prepared)) = cached_global_direct_color_plan(cache_key) { + if let Some((plan, prepared)) = cached_session_direct_color_plan(session, cache_key) { self.native_direct_color_plan = Some(plan); self.native_prepared_direct_color_plan = Some(prepared); } @@ -868,7 +1067,7 @@ impl<'a> J2kDecoder<'a> { } }; let prepared = Arc::new(crate::compute::prepare_direct_color_plan(&plan)?); - store_global_direct_color_plan(cache_key, &plan, prepared.clone()); + store_session_direct_color_plan(session, cache_key, &plan, prepared.clone()); self.native_direct_color_plan = Some(plan); self.native_prepared_direct_color_plan = Some(prepared); } @@ -876,6 +1075,68 @@ impl<'a> J2kDecoder<'a> { Ok(self.native_prepared_direct_color_plan.clone()) } + #[cfg(target_os = "macos")] + fn ensure_prepared_direct_gray_plan( + &mut self, + ) -> Result>, Error> { + if self.native_prepared_direct_gray_plan.is_none() { + self.ensure_native_image()?; + let (Some(image), native_context) = + (self.native_image.as_ref(), &mut self.native_context) + else { + return Err(Error::Decode(J2kError::Backend( + "native image cache missing".to_string(), + ))); + }; + let plan = match image.build_direct_grayscale_plan_with_context(native_context) { + Ok(plan) => plan, + Err(error) if direct::is_unsupported_direct_plan_error(&error.to_string()) => { + return Ok(None); + } + Err(error) => { + return Err(Error::Decode(J2kError::Backend(format!( + "failed to build J2K MetalDirect grayscale plan: {error}" + )))); + } + }; + let prepared = Arc::new(crate::compute::prepare_direct_grayscale_plan(&plan)?); + self.native_direct_gray_plan = Some(plan); + self.native_prepared_direct_gray_plan = Some(prepared); + } + Ok(self.native_prepared_direct_gray_plan.clone()) + } + + #[cfg(target_os = "macos")] + fn ensure_prepared_direct_color_plan( + &mut self, + ) -> Result>, Error> { + if self.native_prepared_direct_color_plan.is_none() { + self.ensure_native_image()?; + let (Some(image), native_context) = + (self.native_image.as_ref(), &mut self.native_context) + else { + return Err(Error::Decode(J2kError::Backend( + "native image cache missing".to_string(), + ))); + }; + let plan = match image.build_direct_color_plan_with_context(native_context) { + Ok(plan) => plan, + Err(error) if direct::is_unsupported_direct_plan_error(&error.to_string()) => { + return Ok(None); + } + Err(error) => { + return Err(Error::Decode(J2kError::Backend(format!( + "failed to build J2K MetalDirect color plan: {error}" + )))); + } + }; + let prepared = Arc::new(crate::compute::prepare_direct_color_plan(&plan)?); + self.native_direct_color_plan = Some(plan); + self.native_prepared_direct_color_plan = Some(prepared); + } + Ok(self.native_prepared_direct_color_plan.clone()) + } + #[cfg(target_os = "macos")] fn decode_direct_to_surface(&mut self, fmt: PixelFormat) -> Result, Error> { if matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { @@ -905,18 +1166,20 @@ impl<'a> J2kDecoder<'a> { } #[cfg(target_os = "macos")] - fn decode_direct_to_surface_with_device( + fn decode_direct_to_surface_with_session( &mut self, fmt: PixelFormat, - device: &Device, + session: &MetalBackendSession, ) -> Result, Error> { if matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { - let Some(plan) = self.ensure_prepared_direct_gray_plan()? else { + let Some(plan) = self.ensure_prepared_direct_gray_plan_with_session(session)? else { return Ok(None); }; return Ok(Some( crate::compute::execute_prepared_direct_grayscale_plan_with_device( - &plan, fmt, device, + &plan, + fmt, + &session.device, )?, )); } @@ -925,11 +1188,13 @@ impl<'a> J2kDecoder<'a> { fmt, PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 ) { - let Some(plan) = self.ensure_prepared_direct_color_plan()? else { + let Some(plan) = self.ensure_prepared_direct_color_plan_with_session(session)? else { return Ok(None); }; return match crate::compute::execute_prepared_direct_color_plan_with_device( - &plan, fmt, device, + &plan, + fmt, + &session.device, ) { Ok(surface) => Ok(Some(surface)), Err(error) if is_direct_color_runtime_fallback_error(&error) => Ok(None), @@ -947,52 +1212,25 @@ impl<'a> J2kDecoder<'a> { roi: Rect, scale: Downscale, ) -> Result, Error> { - if !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { - return Ok(None); - } - let prepared = match build_region_scaled_direct_gray_plan(self.inner.bytes(), roi, scale) { - Ok(prepared) => prepared, - Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => { - return Ok(None); - } - Err(error) => return Err(error), - }; - match crate::compute::execute_prepared_direct_grayscale_plan(&Arc::new(prepared), fmt) { - Ok(surface) => Ok(Some(surface)), - Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => Ok(None), - Err(error) => Err(error), - } + crate::hybrid::decode_region_scaled_direct_to_surface(self.inner.bytes(), fmt, roi, scale) } #[cfg(target_os = "macos")] - fn decode_region_scaled_direct_to_surface_with_device( + fn decode_region_scaled_direct_to_surface_with_session( &mut self, fmt: PixelFormat, roi: Rect, scale: Downscale, - device: &Device, + session: &MetalBackendSession, ) -> Result, Error> { - if !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { - return Ok(None); - } - let prepared = match build_region_scaled_direct_gray_plan(self.inner.bytes(), roi, scale) { - Ok(prepared) => prepared, - Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => { - return Ok(None); - } - Err(error) => return Err(error), - }; - match crate::compute::execute_prepared_direct_grayscale_plan_with_device( - &Arc::new(prepared), + crate::hybrid::decode_region_scaled_direct_to_surface_with_session( + self.inner.bytes(), fmt, - device, - ) { - Ok(surface) => Ok(Some(surface)), - Err(error) if is_direct_region_scaled_runtime_fallback_error(&error) => Ok(None), - Err(error) => Err(error), - } + roi, + scale, + session, + ) } - #[cfg(target_os = "macos")] fn decode_full_to_metal_surface(&mut self, fmt: PixelFormat) -> Result { self.ensure_native_image()?; @@ -1058,13 +1296,6 @@ impl<'a> J2kDecoder<'a> { if count == 0 { return Ok(Vec::new()); } - if self.native_direct_gray_plan.is_none() { - let cache_key = direct_gray_plan_cache_key(self.inner.bytes()); - if let Some((plan, prepared)) = cached_global_direct_gray_plan(cache_key) { - self.native_direct_gray_plan = Some(plan); - self.native_prepared_direct_gray_plan = Some(prepared); - } - } if self.native_direct_gray_plan.is_none() { self.ensure_native_image()?; let (Some(image), native_context) = @@ -1074,12 +1305,10 @@ impl<'a> J2kDecoder<'a> { "native image cache missing".to_string(), ))); }; - let cache_key = direct_gray_plan_cache_key(self.inner.bytes()); let plan = image .build_direct_grayscale_plan_with_context(native_context) .map_err(|error| J2kError::Backend(error.to_string()))?; let prepared = Arc::new(crate::compute::prepare_direct_grayscale_plan(&plan)?); - store_global_direct_gray_plan(cache_key, &plan, prepared.clone()); self.native_direct_gray_plan = Some(plan); self.native_prepared_direct_gray_plan = Some(prepared); } @@ -1122,13 +1351,6 @@ impl<'a> J2kDecoder<'a> { { return self.decode_repeated_grayscale_cpu_to_surfaces(fmt, count); } - if self.native_direct_gray_plan.is_none() { - let cache_key = direct_gray_plan_cache_key(self.inner.bytes()); - if let Some((plan, prepared)) = cached_global_direct_gray_plan(cache_key) { - self.native_direct_gray_plan = Some(plan); - self.native_prepared_direct_gray_plan = Some(prepared); - } - } if self.native_direct_gray_plan.is_none() { self.ensure_native_image()?; let (Some(image), native_context) = @@ -1138,12 +1360,10 @@ impl<'a> J2kDecoder<'a> { "native image cache missing".to_string(), ))); }; - let cache_key = direct_gray_plan_cache_key(self.inner.bytes()); let Ok(plan) = image.build_direct_grayscale_plan_with_context(native_context) else { return self.decode_repeated_grayscale_cpu_to_surfaces(fmt, count); }; let prepared = Arc::new(crate::compute::prepare_direct_grayscale_plan(&plan)?); - store_global_direct_gray_plan(cache_key, &plan, prepared.clone()); self.native_direct_gray_plan = Some(plan); self.native_prepared_direct_gray_plan = Some(prepared); } @@ -1301,29 +1521,30 @@ impl<'a> J2kDecoder<'a> { device, ) } - #[cfg(target_os = "macos")] - fn decode_region_scaled_to_metal_surface_with_device( + fn decode_region_scaled_to_metal_surface_with_session( &mut self, fmt: PixelFormat, roi: Rect, scale: Downscale, plan: DeviceDecodePlan, - device: &Device, + session: &MetalBackendSession, ) -> Result { if let Some(surface) = - self.decode_region_scaled_direct_to_surface_with_device(fmt, roi, scale, device)? + self.decode_region_scaled_direct_to_surface_with_session(fmt, roi, scale, session)? { return Ok(surface); } - crate::compute::decode_region_scaled_to_surface_with_device( - self.inner.bytes(), - plan.source_dims(), - fmt, - roi, - scale, - device, - ) + crate::compute::with_runtime_for_session(session, |_| { + crate::compute::decode_region_scaled_to_surface_with_device( + self.inner.bytes(), + plan.source_dims(), + fmt, + roi, + scale, + &session.device, + ) + }) } pub(crate) fn decode_to_surface_impl( @@ -1383,6 +1604,7 @@ impl<'a> J2kDecoder<'a> { } } + /// Decode a source region into a Metal-resident device surface. pub fn decode_region_to_device_with_session( &mut self, fmt: PixelFormat, @@ -1401,7 +1623,9 @@ impl<'a> J2kDecoder<'a> { self.inner.info().dimensions, DeviceDecodeRequest::Region { roi }, )?; - self.decode_region_to_metal_surface_with_device(fmt, plan, &session.device) + crate::compute::with_runtime_for_session(session, |_| { + self.decode_region_to_metal_surface_with_device(fmt, plan, &session.device) + }) } #[cfg(not(target_os = "macos"))] { @@ -1451,18 +1675,6 @@ impl<'a> J2kDecoder<'a> { if let Some(error) = routing::decision_error(route) { return Err(error); } - #[cfg(target_os = "macos")] - { - if matches!(route, routing::RouteDecision::MetalKernel) - && !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) - { - return Err(Error::UnsupportedMetalRequest { - reason: - "J2K Metal ROI+scaled direct decode currently supports Gray8/Gray16 only", - }); - } - } - let plan = DeviceDecodePlan::for_image( self.inner.info().dimensions, DeviceDecodeRequest::RegionScaled { roi, scale }, @@ -1484,6 +1696,7 @@ impl<'a> J2kDecoder<'a> { } } + /// Decode a full image at reduced resolution into a Metal-resident surface. pub fn decode_scaled_to_device_with_session( &mut self, fmt: PixelFormat, @@ -1497,7 +1710,7 @@ impl<'a> J2kDecoder<'a> { } if !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { return Err(Error::UnsupportedMetalRequest { - reason: "J2K Metal ROI+scaled direct decode currently supports Gray8/Gray16 only", + reason: "J2K Metal session scaled decode currently supports Gray8/Gray16 only", }); } @@ -1507,7 +1720,9 @@ impl<'a> J2kDecoder<'a> { self.inner.info().dimensions, DeviceDecodeRequest::Scaled { scale }, )?; - self.decode_scaled_to_metal_surface_with_device(fmt, scale, plan, &session.device) + crate::compute::with_runtime_for_session(session, |_| { + self.decode_scaled_to_metal_surface_with_device(fmt, scale, plan, &session.device) + }) } #[cfg(not(target_os = "macos"))] { @@ -1516,6 +1731,7 @@ impl<'a> J2kDecoder<'a> { } } + /// Decode a scaled source region into a Metal-resident device surface. pub fn decode_region_scaled_to_device_with_session( &mut self, fmt: PixelFormat, @@ -1535,13 +1751,7 @@ impl<'a> J2kDecoder<'a> { self.inner.info().dimensions, DeviceDecodeRequest::RegionScaled { roi, scale }, )?; - self.decode_region_scaled_to_metal_surface_with_device( - fmt, - roi, - scale, - plan, - &session.device, - ) + self.decode_region_scaled_to_metal_surface_with_session(fmt, roi, scale, plan, session) } #[cfg(not(target_os = "macos"))] { @@ -1564,27 +1774,27 @@ fn direct_gray_plan_cache_key(bytes: &[u8]) -> u64 { } #[cfg(target_os = "macos")] -fn cached_global_direct_gray_plan( +fn cached_session_direct_gray_plan( + session: &MetalBackendSession, key: u64, ) -> Option<( J2kDirectGrayscalePlan, Arc, )> { - let cache = DIRECT_GRAY_PLAN_CACHE.get_or_init(|| Mutex::new(HashMap::new())); - let guard = cache.lock().ok()?; + let guard = session.direct_gray_plan_cache.lock().ok()?; guard .get(&key) .map(|entry| (entry.plan.clone(), entry.prepared.clone())) } #[cfg(target_os = "macos")] -fn store_global_direct_gray_plan( +fn store_session_direct_gray_plan( + session: &MetalBackendSession, key: u64, plan: &J2kDirectGrayscalePlan, prepared: Arc, ) { - let cache = DIRECT_GRAY_PLAN_CACHE.get_or_init(|| Mutex::new(HashMap::new())); - if let Ok(mut guard) = cache.lock() { + if let Ok(mut guard) = session.direct_gray_plan_cache.lock() { evict_one_direct_plan_if_needed(&mut guard); guard.insert( key, @@ -1597,27 +1807,27 @@ fn store_global_direct_gray_plan( } #[cfg(target_os = "macos")] -fn cached_global_direct_color_plan( +fn cached_session_direct_color_plan( + session: &MetalBackendSession, key: u64, ) -> Option<( J2kDirectColorPlan, Arc, )> { - let cache = DIRECT_COLOR_PLAN_CACHE.get_or_init(|| Mutex::new(HashMap::new())); - let guard = cache.lock().ok()?; + let guard = session.direct_color_plan_cache.lock().ok()?; guard .get(&key) .map(|entry| (entry.plan.clone(), entry.prepared.clone())) } #[cfg(target_os = "macos")] -fn store_global_direct_color_plan( +fn store_session_direct_color_plan( + session: &MetalBackendSession, key: u64, plan: &J2kDirectColorPlan, prepared: Arc, ) { - let cache = DIRECT_COLOR_PLAN_CACHE.get_or_init(|| Mutex::new(HashMap::new())); - if let Ok(mut guard) = cache.lock() { + if let Ok(mut guard) = session.direct_color_plan_cache.lock() { evict_one_direct_plan_if_needed(&mut guard); guard.insert( key, @@ -1653,14 +1863,10 @@ fn is_direct_runtime_fallback_error(error: &Error) -> bool { || message.contains("unsupported HT kernel input") || message.contains("direct component plan") || message.contains("currently supports grayscale direct plans only") + || message.contains("currently supports color direct plans only") ) } -#[cfg(target_os = "macos")] -fn is_direct_region_scaled_runtime_fallback_error(error: &Error) -> bool { - is_direct_runtime_fallback_error(error) -} - #[cfg(target_os = "macos")] pub(crate) fn decode_full_grayscale_batch_direct_to_device( inputs: &[Arc<[u8]>], @@ -1690,72 +1896,6 @@ pub(crate) fn decode_full_grayscale_batch_direct_to_device( crate::compute::execute_prepared_direct_grayscale_plan_batch(&plans, fmt) } -#[cfg(target_os = "macos")] -pub(crate) fn decode_region_scaled_grayscale_batch_direct_to_device( - requests: &[(Arc<[u8]>, Rect, Downscale)], - fmt: PixelFormat, -) -> Result, Error> { - if requests.is_empty() { - return Ok(Vec::new()); - } - if !matches!(fmt, PixelFormat::Gray8 | PixelFormat::Gray16) { - return Err(Error::MetalKernel { - message: format!( - "J2K MetalDirect region-scaled grayscale batch does not support {fmt:?}" - ), - }); - } - - let mut plans = Vec::with_capacity(requests.len()); - for (input, roi, scale) in requests { - let plan = build_region_scaled_direct_gray_plan(input.as_ref(), *roi, *scale)?; - plans.push(Arc::new(plan)); - } - crate::compute::execute_prepared_direct_grayscale_plan_batch(&plans, fmt) -} - -#[cfg(target_os = "macos")] -fn build_region_scaled_direct_gray_plan( - input: &[u8], - roi: Rect, - scale: Downscale, -) -> Result { - let decoder = J2kDecoder::new(input)?; - let dims = decoder.inner.info().dimensions; - let target_dims = ( - dims.0.div_ceil(scale.denominator()), - dims.1.div_ceil(scale.denominator()), - ); - let settings = NativeDecodeSettings { - target_resolution: Some(target_dims), - ..NativeDecodeSettings::default() - }; - let image = - NativeImage::new(input, &settings).map_err(|error| J2kError::Backend(error.to_string()))?; - let mut context = NativeDecoderContext::default(); - let plan = match image.build_direct_grayscale_plan_with_context(&mut context) { - Ok(plan) => plan, - Err(error) if direct::is_unsupported_direct_plan_error(&error.to_string()) => { - return Err(Error::MetalKernel { - message: format!( - "explicit J2K MetalDirect region-scaled batch currently supports grayscale direct plans only: {error}" - ), - }); - } - Err(error) => { - return Err(Error::Decode(J2kError::Backend(format!( - "failed to build J2K MetalDirect region-scaled grayscale plan: {error}" - )))); - } - }; - let mut prepared = crate::compute::prepare_direct_grayscale_plan(&plan)?; - crate::compute::crop_prepared_direct_grayscale_plan_to_output_region( - &mut prepared, - roi.scaled_covering(scale), - )?; - Ok(prepared) -} - #[cfg(target_os = "macos")] pub(crate) fn decode_full_color_batch_direct_to_device( inputs: &[Arc<[u8]>], @@ -1805,7 +1945,7 @@ impl ImageCodec for J2kDecoder<'_> { impl<'a> ImageDecode<'a> for J2kDecoder<'a> { type View = J2kView<'a>; - fn inspect(input: &'a [u8]) -> Result { + fn inspect(input: &'a [u8]) -> Result { Ok(CpuDecoder::inspect(input)?) } @@ -1882,6 +2022,7 @@ impl<'a> ImageDecodeDevice<'a> for J2kDecoder<'a> { } #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +/// J2K codec marker used by J2K's generic decode traits. pub struct Codec; impl ImageCodec for Codec { @@ -1901,7 +2042,7 @@ impl<'a> ImageDecodeSubmit<'a> for J2kDecoder<'a> { fmt: PixelFormat, backend: BackendRequest, ) -> Result { - session.record_submit(); + session.record_submit()?; Ok(ReadySubmission::from_result( self.decode_to_surface_impl(fmt, backend), )) @@ -1914,7 +2055,7 @@ impl<'a> ImageDecodeSubmit<'a> for J2kDecoder<'a> { roi: Rect, backend: BackendRequest, ) -> Result { - session.record_submit(); + session.record_submit()?; Ok(ReadySubmission::from_result( self.decode_region_to_surface_impl(fmt, roi, backend), )) @@ -1927,7 +2068,7 @@ impl<'a> ImageDecodeSubmit<'a> for J2kDecoder<'a> { scale: Downscale, backend: BackendRequest, ) -> Result { - session.record_submit(); + session.record_submit()?; Ok(ReadySubmission::from_result( self.decode_scaled_to_surface_impl(fmt, scale, backend), )) @@ -1941,7 +2082,7 @@ impl<'a> ImageDecodeSubmit<'a> for J2kDecoder<'a> { scale: Downscale, backend: BackendRequest, ) -> Result { - session.record_submit(); + session.record_submit()?; Ok(ReadySubmission::from_result( self.decode_region_scaled_to_surface_impl(fmt, roi, scale, backend), )) @@ -1955,7 +2096,7 @@ impl TileBatchDecodeSubmit for Codec { type SubmittedSurface = batch::MetalSubmission; fn submit_tile_to_device( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, session: &mut Self::Session, pool: &mut Self::Pool, input: &[u8], @@ -1963,17 +2104,11 @@ impl TileBatchDecodeSubmit for Codec { backend: BackendRequest, ) -> Result { let _ = (ctx, pool); - Ok(batch::queue_tile_request( - session, - input, - fmt, - backend, - batch::BatchOp::Full, - )) + batch::queue_tile_request(session, input, fmt, backend, batch::BatchOp::Full) } fn submit_tile_region_to_device( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, session: &mut Self::Session, pool: &mut Self::Pool, input: &[u8], @@ -1982,17 +2117,11 @@ impl TileBatchDecodeSubmit for Codec { backend: BackendRequest, ) -> Result { let _ = (ctx, pool); - Ok(batch::queue_tile_request( - session, - input, - fmt, - backend, - batch::BatchOp::Region(roi), - )) + batch::queue_tile_request(session, input, fmt, backend, batch::BatchOp::Region(roi)) } fn submit_tile_scaled_to_device( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, session: &mut Self::Session, pool: &mut Self::Pool, input: &[u8], @@ -2001,17 +2130,11 @@ impl TileBatchDecodeSubmit for Codec { backend: BackendRequest, ) -> Result { let _ = (ctx, pool); - Ok(batch::queue_tile_request( - session, - input, - fmt, - backend, - batch::BatchOp::Scaled(scale), - )) + batch::queue_tile_request(session, input, fmt, backend, batch::BatchOp::Scaled(scale)) } fn submit_tile_region_scaled_to_device( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, session: &mut Self::Session, pool: &mut Self::Pool, input: &[u8], @@ -2021,13 +2144,13 @@ impl TileBatchDecodeSubmit for Codec { backend: BackendRequest, ) -> Result { let _ = (ctx, pool); - Ok(batch::queue_tile_request( + batch::queue_tile_request( session, input, fmt, backend, batch::BatchOp::RegionScaled { roi, scale }, - )) + ) } } @@ -2036,7 +2159,7 @@ impl TileBatchDecodeManyDevice for Codec { type DeviceSurface = Surface; fn decode_tiles_to_device( - ctx: &mut signinum_core::DecoderContext, + ctx: &mut j2k_core::DecoderContext, pool: &mut Self::Pool, inputs: &[&[u8]], fmt: PixelFormat, @@ -2063,7 +2186,7 @@ impl TileBatchDecodeManyDevice for Codec { submissions .into_iter() - .map(signinum_core::DeviceSubmission::wait) + .map(j2k_core::DeviceSubmission::wait) .collect() } } @@ -2132,12 +2255,29 @@ fn upload_surface_to_metal_with_device( } } -pub use signinum_j2k::{J2kContext, J2kScratchPool}; +pub use j2k::{J2kContext, J2kScratchPool}; #[cfg(test)] mod tests { use super::*; + #[test] + fn metal_runtime_failures_are_not_unsupported_errors() { + for err in [ + Error::MetalRuntime { + message: "runtime".to_string(), + }, + Error::MetalKernel { + message: "kernel".to_string(), + }, + Error::MetalStatePoisoned { + state: "J2K Metal session", + }, + ] { + assert!(!err.is_unsupported(), "{err:?}"); + } + } + #[test] fn cpu_uploaded_surface_reports_host_residency() { let surface = upload_surface( @@ -2154,6 +2294,47 @@ mod tests { assert!(surface.metal_buffer().is_none()); } + #[test] + fn download_into_reports_inconsistent_surface_storage_range() { + let surface = Surface { + backend: BackendKind::Cpu, + residency: SurfaceResidency::Host, + dimensions: (2, 1), + fmt: PixelFormat::Gray8, + pitch_bytes: 2, + byte_offset: 0, + storage: Storage::Host(vec![7]), + }; + let mut out = [0_u8; 2]; + + let err = surface + .download_into(&mut out, 2) + .expect_err("inconsistent surface storage should be reported"); + + assert!(matches!( + err, + Error::MetalKernel { message } + if message == "J2K Metal surface byte range 0..2 exceeds storage length 1" + )); + } + + #[cfg(target_os = "macos")] + #[test] + fn metal_backend_sessions_own_distinct_direct_plan_caches() { + let Some(device) = Device::system_default() else { + eprintln!("skipping session cache ownership test: no Metal device"); + return; + }; + + let first = MetalBackendSession::new(device.clone()); + let second = MetalBackendSession::new(device); + + assert_ne!( + first.direct_cache_ids_for_test(), + second.direct_cache_ids_for_test() + ); + } + #[cfg(target_os = "macos")] #[test] fn explicit_metal_request_does_not_stage_cpu_pixels() { @@ -2177,4 +2358,44 @@ mod tests { && reason.contains("Metal") )); } + + #[cfg(target_os = "macos")] + #[test] + fn repeated_region_scaled_color_batch_reuses_prepared_plan() { + if Device::system_default().is_none() { + eprintln!("skipping repeated color plan reuse test: no Metal device"); + return; + } + + let pixels = j2k_test_support::gradient_u8(64, 64, 3); + let options = j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..j2k_native::EncodeOptions::default() + }; + let input = Arc::<[u8]>::from( + j2k_native::encode(&pixels, 64, 64, 3, 8, false, &options).expect("encode rgb8"), + ); + let roi = Rect { + x: 8, + y: 8, + w: 32, + h: 32, + }; + let scale = Downscale::Quarter; + let requests = vec![(input.clone(), roi, scale); 4]; + let _guard = hybrid::region_scaled_color_plan_test_lock_for_test(); + hybrid::reset_region_scaled_color_plan_builds_for_test(); + + let surfaces = + hybrid::decode_region_scaled_color_batch_direct_to_device(&requests, PixelFormat::Rgb8) + .expect("repeated RGB region-scaled batch"); + + assert_eq!(surfaces.len(), requests.len()); + assert_eq!( + hybrid::region_scaled_color_plan_builds_for_test(), + 1, + "repeated RGB ROI+scaled batches should build and crop one prepared direct color plan" + ); + } } diff --git a/crates/signinum-j2k-metal/src/mct.metal b/crates/j2k-metal/src/mct.metal similarity index 50% rename from crates/signinum-j2k-metal/src/mct.metal rename to crates/j2k-metal/src/mct.metal index e256dda1..a30149ab 100644 --- a/crates/signinum-j2k-metal/src/mct.metal +++ b/crates/j2k-metal/src/mct.metal @@ -25,6 +25,13 @@ struct J2kForwardRctParams { uint reserved2; }; +struct J2kForwardIctParams { + uint len; + uint reserved0; + uint reserved1; + uint reserved2; +}; + constant uint J2K_MCT_TRANSFORM_REVERSIBLE53 = 0; constant uint J2K_MCT_TRANSFORM_IRREVERSIBLE97 = 1; constant uint J2K_MCT_STATUS_OK = 0; @@ -92,3 +99,86 @@ kernel void j2k_forward_rct( status->detail = 0; } } + +kernel void j2k_forward_ict( + device float *plane0 [[buffer(0)]], + device float *plane1 [[buffer(1)]], + device float *plane2 [[buffer(2)]], + constant J2kForwardIctParams ¶ms [[buffer(3)]], + device J2kMctStatus *status [[buffer(4)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= params.len) { + return; + } + + const float r = plane0[gid]; + const float g = plane1[gid]; + const float b = plane2[gid]; + + plane0[gid] = 0.299f * r + 0.587f * g + 0.114f * b; + plane1[gid] = -0.16875f * r - 0.33126f * g + 0.5f * b; + plane2[gid] = 0.5f * r - 0.41869f * g - 0.08131f * b; + + if (gid == 0) { + status->code = J2K_MCT_STATUS_OK; + status->detail = 0; + } +} + +kernel void j2k_lossless_deinterleave_rct_rgb8_to_planes( + device const uchar *src [[buffer(0)]], + device float *plane0 [[buffer(1)]], + device float *plane1 [[buffer(2)]], + device float *plane2 [[buffer(3)]], + constant J2kLosslessDeinterleaveParams ¶ms [[buffer(4)]], + device J2kMctStatus *status [[buffer(5)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.dst_width || gid.y >= params.dst_height) { + return; + } + + const bool inside_src = gid.x < params.src_width && gid.y < params.src_height; + const uint src_base = gid.y * params.src_stride + gid.x * 3u; + const uint dst_idx = gid.y * params.dst_width + gid.x; + const float r = j2k_lossless_load_sample( + src, + src_base, + 0u, + 3u, + 1u, + params.sample_offset, + 0u, + inside_src + ); + const float g = j2k_lossless_load_sample( + src, + src_base, + 1u, + 3u, + 1u, + params.sample_offset, + 0u, + inside_src + ); + const float b = j2k_lossless_load_sample( + src, + src_base, + 2u, + 3u, + 1u, + params.sample_offset, + 0u, + inside_src + ); + + plane0[dst_idx] = floor((r + 2.0f * g + b) * 0.25f); + plane1[dst_idx] = b - g; + plane2[dst_idx] = r - g; + + if (gid.x == 0u && gid.y == 0u) { + status->code = J2K_MCT_STATUS_OK; + status->detail = 0; + } +} diff --git a/crates/signinum-j2k-metal/src/mct.rs b/crates/j2k-metal/src/mct.rs similarity index 95% rename from crates/signinum-j2k-metal/src/mct.rs rename to crates/j2k-metal/src/mct.rs index 740ccfaf..7eabc8fc 100644 --- a/crates/signinum-j2k-metal/src/mct.rs +++ b/crates/j2k-metal/src/mct.rs @@ -2,11 +2,11 @@ #[cfg(target_os = "macos")] use crate::compute; -#[cfg(target_os = "macos")] -use metal::Buffer; -use signinum_j2k_native::{ +use j2k_native::{ decode_ht_code_block_scalar, HtCodeBlockDecodeJob, HtCodeBlockDecoder, J2kInverseMctJob, Result, }; +#[cfg(target_os = "macos")] +use metal::Buffer; #[derive(Default)] pub(crate) struct MetalMctDecoder { @@ -33,7 +33,7 @@ impl HtCodeBlockDecoder for MetalMctDecoder { #[cfg(target_os = "macos")] if supports_metal_inverse_mct(&job) { self.captured_planes = compute::decode_inverse_mct(job) - .map_err(|_| signinum_j2k_native::DecodingError::CodeBlockDecodeFailure)?; + .map_err(|_| j2k_native::DecodingError::CodeBlockDecodeFailure)?; self.kernel_dispatches = self.kernel_dispatches.saturating_add(1); return Ok(true); } @@ -47,7 +47,7 @@ impl HtCodeBlockDecoder for MetalMctDecoder { &mut self, job: HtCodeBlockDecodeJob<'_>, output: &mut [f32], - ) -> signinum_j2k_native::Result<()> { + ) -> j2k_native::Result<()> { decode_ht_code_block_scalar(job, output) } } @@ -61,7 +61,7 @@ fn supports_metal_inverse_mct(job: &J2kInverseMctJob<'_>) -> bool { #[cfg(test)] mod tests { use super::MetalMctDecoder; - use signinum_j2k_native::{ + use j2k_native::{ encode, DecodeSettings, DecoderContext, EncodeOptions, HtCodeBlockDecodeJob, HtCodeBlockDecoder, Image, }; @@ -124,8 +124,8 @@ mod tests { &mut self, job: HtCodeBlockDecodeJob<'_>, output: &mut [f32], - ) -> signinum_j2k_native::Result<()> { - signinum_j2k_native::decode_ht_code_block_scalar(job, output) + ) -> j2k_native::Result<()> { + j2k_native::decode_ht_code_block_scalar(job, output) } } @@ -188,6 +188,7 @@ mod tests { let captured = decoder.take_captured_planes(); assert_eq!(captured.len(), components.planes().len()); for (plane, buffer) in components.planes().iter().zip(captured.iter()) { + // SAFETY: SIMD lane captures operate on fixed-width stack buffers. let captured = unsafe { core::slice::from_raw_parts( buffer.contents().cast::(), diff --git a/crates/j2k-metal/src/profile.rs b/crates/j2k-metal/src/profile.rs new file mode 100644 index 00000000..22b02dde --- /dev/null +++ b/crates/j2k-metal/src/profile.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::cell::RefCell; +use std::sync::OnceLock; +use std::time::Instant; + +use j2k_profile::{profile_stage_mode_from_env, same_summary_labels, ProfileStageMode}; + +const METAL_PROFILE_STAGES_ENV: &str = "J2K_METAL_PROFILE_STAGES"; + +thread_local! { + static METAL_BATCH_PROFILE_SUMMARY: RefCell = + RefCell::new(j2k_profile::ProfileSummary::new(same_summary_labels(&[ + "slice", + "stage", + "pipeline", + "processor", + "metric_kind", + "aggregation", + "route", + "backend", + "fmt", + "outcome", + ])).emit_on_drop()); +} + +pub(crate) type ProfileInstant = Instant; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct MetalBatchProfileRow<'a> { + pub(crate) slice: &'a str, + pub(crate) stage: &'a str, + pub(crate) pipeline: &'a str, + pub(crate) processor: &'a str, + pub(crate) route: &'a str, + pub(crate) backend: &'a str, + pub(crate) fmt: &'a str, + pub(crate) request_count: usize, + pub(crate) output_count: usize, + pub(crate) elapsed_us: u128, + pub(crate) outcome: &'a str, +} + +pub(crate) fn metal_profile_stage_mode() -> ProfileStageMode { + static MODE: OnceLock = OnceLock::new(); + *MODE.get_or_init(|| profile_stage_mode_from_env(METAL_PROFILE_STAGES_ENV)) +} + +pub(crate) fn metal_profile_stages_enabled() -> bool { + metal_profile_stage_mode() != ProfileStageMode::Disabled +} + +pub(crate) fn profile_now(enabled: bool) -> Option { + enabled.then(Instant::now) +} + +pub(crate) fn elapsed_us(start: Option) -> u128 { + start.map_or(0, |start| start.elapsed().as_micros()) +} + +pub(crate) fn emit_metal_batch_profile_row(path: &str, row: &MetalBatchProfileRow<'_>) { + let fields = format_metal_batch_profile_fields(row); + j2k_profile::emit_profile_row( + metal_profile_stage_mode(), + &METAL_BATCH_PROFILE_SUMMARY, + "j2k", + "metal_batch", + path, + fields.as_slice(), + ); +} + +pub(crate) fn format_metal_batch_profile_fields( + row: &MetalBatchProfileRow<'_>, +) -> Vec<(String, String)> { + vec![ + ("slice".to_string(), row.slice.to_string()), + ("stage".to_string(), row.stage.to_string()), + ("pipeline".to_string(), row.pipeline.to_string()), + ("processor".to_string(), row.processor.to_string()), + ("metric".to_string(), "wall_us".to_string()), + ("metric_kind".to_string(), "wall_elapsed".to_string()), + ("aggregation".to_string(), "exclusive".to_string()), + ("route".to_string(), row.route.to_string()), + ("backend".to_string(), row.backend.to_string()), + ("fmt".to_string(), row.fmt.to_string()), + ("request_count".to_string(), row.request_count.to_string()), + ("output_count".to_string(), row.output_count.to_string()), + ("elapsed_us".to_string(), row.elapsed_us.to_string()), + ("outcome".to_string(), row.outcome.to_string()), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn metal_batch_profile_fields_include_processor_and_timing_context() { + let fields = format_metal_batch_profile_fields(&MetalBatchProfileRow { + slice: "decode_batch", + stage: "execute", + pipeline: "metal_cpu_hybrid", + processor: "hybrid", + route: "auto_repeated_region_scaled_direct_metal", + backend: "Auto", + fmt: "Rgb8", + request_count: 16, + output_count: 16, + elapsed_us: 42, + outcome: "metal_surface", + }); + + assert!( + fields + .iter() + .any(|(name, value)| name == "pipeline" && value == "metal_cpu_hybrid"), + "hybrid batch profile rows must identify the Metal/CPU hybrid pipeline" + ); + assert!( + fields + .iter() + .any(|(name, value)| name == "processor" && value == "hybrid"), + "hybrid batch profile rows must identify whether time is CPU, Metal, transfer, wait, or scheduler work" + ); + assert!( + fields + .iter() + .any(|(name, value)| name == "metric_kind" && value == "wall_elapsed"), + "hybrid batch profile rows must identify wall-time semantics" + ); + assert!( + fields + .iter() + .any(|(name, value)| name == "aggregation" && value == "exclusive"), + "hybrid batch profile rows must identify whether elapsed time is exclusive or aggregated" + ); + assert_eq!( + fields, + vec![ + ("slice".to_string(), "decode_batch".to_string()), + ("stage".to_string(), "execute".to_string()), + ("pipeline".to_string(), "metal_cpu_hybrid".to_string()), + ("processor".to_string(), "hybrid".to_string()), + ("metric".to_string(), "wall_us".to_string()), + ("metric_kind".to_string(), "wall_elapsed".to_string()), + ("aggregation".to_string(), "exclusive".to_string()), + ( + "route".to_string(), + "auto_repeated_region_scaled_direct_metal".to_string() + ), + ("backend".to_string(), "Auto".to_string()), + ("fmt".to_string(), "Rgb8".to_string()), + ("request_count".to_string(), "16".to_string()), + ("output_count".to_string(), "16".to_string()), + ("elapsed_us".to_string(), "42".to_string()), + ("outcome".to_string(), "metal_surface".to_string()), + ] + ); + } +} diff --git a/crates/j2k-metal/src/profile_env.rs b/crates/j2k-metal/src/profile_env.rs new file mode 100644 index 00000000..540d594c --- /dev/null +++ b/crates/j2k-metal/src/profile_env.rs @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(test)] +use std::cell::Cell; +use std::sync::OnceLock; + +use metal::{CommandBufferRef, ComputeCommandEncoderRef}; + +const CLASSIC_SELECTIVE_BYPASS_ENV: &str = "J2K_METAL_CLASSIC_SELECTIVE_BYPASS"; +const METAL_PROFILE_STAGES_ENV: &str = "J2K_METAL_PROFILE_STAGES"; +const METAL_PROFILE_SIGNPOSTS_ENV: &str = "J2K_METAL_PROFILE_SIGNPOSTS"; +const METAL_PROFILE_DECODE_LABEL_ENV: &str = "J2K_METAL_PROFILE_DECODE_LABEL"; +const METAL_PROFILE_DECODE_SPLIT_COMMANDS_ENV: &str = "J2K_METAL_PROFILE_DECODE_SPLIT_COMMANDS"; +const METAL_PROFILE_COEFFICIENT_PREP_SPLIT_COMMANDS_ENV: &str = + "J2K_METAL_PROFILE_COEFFICIENT_PREP_SPLIT_COMMANDS"; +const METAL_PROFILE_CLASSIC_TIER1_DENSITY_ENV: &str = "J2K_METAL_PROFILE_CLASSIC_TIER1_DENSITY"; +const METAL_PROFILE_CLASSIC_TIER1_RAW_PACK_ENV: &str = "J2K_METAL_PROFILE_CLASSIC_TIER1_RAW_PACK"; +const METAL_PROFILE_CLASSIC_TIER1_ARITHMETIC_PACK_ENV: &str = + "J2K_METAL_PROFILE_CLASSIC_TIER1_ARITHMETIC_PACK"; +const METAL_PROFILE_CLASSIC_TIER1_SYMBOL_PLAN_ENV: &str = + "J2K_METAL_PROFILE_CLASSIC_TIER1_SYMBOL_PLAN"; +const METAL_PROFILE_CLASSIC_TIER1_PASS_PLAN_ENV: &str = "J2K_METAL_PROFILE_CLASSIC_TIER1_PASS_PLAN"; +const METAL_PROFILE_CLASSIC_TIER1_TOKEN_EMIT_ENV: &str = + "J2K_METAL_PROFILE_CLASSIC_TIER1_TOKEN_EMIT"; +const METAL_PROFILE_CLASSIC_TIER1_SPLIT_TOKEN_EMIT_ENV: &str = + "J2K_METAL_PROFILE_CLASSIC_TIER1_SPLIT_TOKEN_EMIT"; +const METAL_PROFILE_CLASSIC_TIER1_TOKEN_PACK_ENV: &str = + "J2K_METAL_PROFILE_CLASSIC_TIER1_TOKEN_PACK"; +const CLASSIC_TIER1_GPU_TOKEN_PACK_ENV: &str = "J2K_METAL_CLASSIC_TIER1_GPU_TOKEN_PACK"; +const CLASSIC_TIER1_SPLIT_GPU_TOKEN_PACK_ENV: &str = "J2K_METAL_CLASSIC_TIER1_SPLIT_GPU_TOKEN_PACK"; +const CLASSIC_TIER1_SPLIT_MQ_BYTE_GPU_TOKEN_PACK_ENV: &str = + "J2K_METAL_CLASSIC_TIER1_SPLIT_MQ_BYTE_GPU_TOKEN_PACK"; + +pub(crate) type HybridSignpostName = u32; + +pub(crate) const SIGNPOST_DECODE_HYBRID_CPU_TIER1: HybridSignpostName = 1; +pub(crate) const SIGNPOST_DECODE_HYBRID_COEFFICIENT_UPLOAD: HybridSignpostName = 2; +pub(crate) const SIGNPOST_DECODE_HYBRID_COMMAND_WAIT: HybridSignpostName = 3; +pub(crate) const SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE: HybridSignpostName = 4; +pub(crate) const SIGNPOST_DECODE_HYBRID_STORE_COMMAND_ENCODE: HybridSignpostName = 5; +pub(crate) const SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE: HybridSignpostName = 6; +pub(crate) const SIGNPOST_ENCODE_HYBRID_COMMAND_WAIT: HybridSignpostName = 7; +pub(crate) const SIGNPOST_ENCODE_HYBRID_RESULT_HARVEST: HybridSignpostName = 8; +pub(crate) const SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_SETUP: HybridSignpostName = 9; +pub(crate) const SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_COMMAND_ENCODE: HybridSignpostName = 10; +pub(crate) const SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_PLAN: HybridSignpostName = 11; +pub(crate) const SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_BUFFER_SETUP: HybridSignpostName = 12; +pub(crate) const SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKETIZATION_COMMAND_ENCODE: HybridSignpostName = + 13; +pub(crate) const SIGNPOST_ENCODE_HYBRID_CLASSIC_PAYLOAD_COPY_COMMAND_ENCODE: HybridSignpostName = + 14; +pub(crate) const SIGNPOST_ENCODE_HYBRID_CLASSIC_CODESTREAM_ASSEMBLY_COMMAND_ENCODE: + HybridSignpostName = 15; +pub(crate) const SIGNPOST_ENCODE_HYBRID_HT_TIER1_SETUP: HybridSignpostName = 16; +pub(crate) const SIGNPOST_ENCODE_HYBRID_HT_TIER1_COMMAND_ENCODE: HybridSignpostName = 17; +pub(crate) const SIGNPOST_ENCODE_HYBRID_HT_PACKET_PLAN: HybridSignpostName = 18; +pub(crate) const SIGNPOST_ENCODE_HYBRID_HT_PACKET_BUFFER_SETUP: HybridSignpostName = 19; +pub(crate) const SIGNPOST_ENCODE_HYBRID_HT_PACKET_BLOCK_PREP_COMMAND_ENCODE: HybridSignpostName = + 20; +pub(crate) const SIGNPOST_ENCODE_HYBRID_HT_PACKETIZATION_COMMAND_ENCODE: HybridSignpostName = 21; +pub(crate) const SIGNPOST_ENCODE_HYBRID_HT_PAYLOAD_COPY_COMMAND_ENCODE: HybridSignpostName = 22; +pub(crate) const SIGNPOST_ENCODE_HYBRID_HT_CODESTREAM_ASSEMBLY_COMMAND_ENCODE: HybridSignpostName = + 23; + +#[cfg(test)] +std::thread_local! { + static CLASSIC_GPU_TOKEN_PACK_ROUTE_OVERRIDE: Cell> = const { Cell::new(None) }; + static METAL_PROFILE_STAGES_OVERRIDE: Cell> = const { Cell::new(None) }; +} + +fn env_flag_enabled(name: &str) -> bool { + matches!(std::env::var(name), Ok(value) if value == "1") +} + +pub(crate) fn classic_selective_bypass_disabled() -> bool { + matches!(std::env::var(CLASSIC_SELECTIVE_BYPASS_ENV), Ok(value) if value == "0") +} + +#[cfg(test)] +pub(crate) struct ClassicGpuTokenPackRouteOverrideGuard { + previous: Option, +} + +#[cfg(test)] +impl Drop for ClassicGpuTokenPackRouteOverrideGuard { + fn drop(&mut self) { + CLASSIC_GPU_TOKEN_PACK_ROUTE_OVERRIDE.with(|slot| slot.set(self.previous)); + } +} + +#[cfg(test)] +pub(crate) fn force_classic_gpu_token_pack_route_for_test( + enabled: bool, +) -> ClassicGpuTokenPackRouteOverrideGuard { + let previous = CLASSIC_GPU_TOKEN_PACK_ROUTE_OVERRIDE.with(|slot| slot.replace(Some(enabled))); + ClassicGpuTokenPackRouteOverrideGuard { previous } +} + +#[cfg(test)] +pub(crate) struct MetalProfileStagesOverrideGuard { + previous: Option, +} + +#[cfg(test)] +impl Drop for MetalProfileStagesOverrideGuard { + fn drop(&mut self) { + METAL_PROFILE_STAGES_OVERRIDE.with(|slot| slot.set(self.previous)); + } +} + +#[cfg(test)] +pub(crate) fn force_metal_profile_stages_for_test( + enabled: bool, +) -> MetalProfileStagesOverrideGuard { + let previous = METAL_PROFILE_STAGES_OVERRIDE.with(|slot| slot.replace(Some(enabled))); + MetalProfileStagesOverrideGuard { previous } +} + +pub(crate) fn classic_tier1_gpu_token_pack_requested() -> bool { + #[cfg(test)] + if let Some(enabled) = CLASSIC_GPU_TOKEN_PACK_ROUTE_OVERRIDE.with(Cell::get) { + return enabled; + } + env_flag_enabled(CLASSIC_TIER1_GPU_TOKEN_PACK_ENV) +} + +pub(crate) fn classic_tier1_split_gpu_token_pack_requested() -> bool { + env_flag_enabled(CLASSIC_TIER1_SPLIT_GPU_TOKEN_PACK_ENV) +} + +fn classic_tier1_split_mq_byte_gpu_token_pack_setting() -> Option { + match std::env::var(CLASSIC_TIER1_SPLIT_MQ_BYTE_GPU_TOKEN_PACK_ENV) { + Ok(value) if value == "1" => Some(true), + Ok(value) if value == "0" => Some(false), + _ => None, + } +} + +pub(crate) fn classic_tier1_split_mq_byte_gpu_token_pack_requested() -> bool { + classic_tier1_split_mq_byte_gpu_token_pack_setting() == Some(true) +} + +pub(crate) fn classic_tier1_split_mq_byte_gpu_token_pack_disabled() -> bool { + classic_tier1_split_mq_byte_gpu_token_pack_setting() == Some(false) +} + +pub(crate) fn metal_profile_stages_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + #[cfg(test)] + if let Some(enabled) = METAL_PROFILE_STAGES_OVERRIDE.with(Cell::get) { + return enabled; + } + *ENABLED.get_or_init(|| env_flag_enabled(METAL_PROFILE_STAGES_ENV)) +} + +fn metal_profile_signposts_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| env_flag_enabled(METAL_PROFILE_SIGNPOSTS_ENV)) +} + +pub(crate) fn metal_profile_decode_split_commands_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + metal_profile_stages_enabled() + && *ENABLED.get_or_init(|| env_flag_enabled(METAL_PROFILE_DECODE_SPLIT_COMMANDS_ENV)) +} + +pub(crate) fn metal_profile_coefficient_prep_split_commands_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + metal_profile_stages_enabled() + && *ENABLED + .get_or_init(|| env_flag_enabled(METAL_PROFILE_COEFFICIENT_PREP_SPLIT_COMMANDS_ENV)) +} + +pub(crate) fn metal_profile_classic_tier1_density_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + metal_profile_stages_enabled() + && *ENABLED.get_or_init(|| env_flag_enabled(METAL_PROFILE_CLASSIC_TIER1_DENSITY_ENV)) +} + +pub(crate) fn metal_profile_classic_tier1_raw_pack_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + metal_profile_stages_enabled() + && *ENABLED.get_or_init(|| env_flag_enabled(METAL_PROFILE_CLASSIC_TIER1_RAW_PACK_ENV)) +} + +pub(crate) fn metal_profile_classic_tier1_arithmetic_pack_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + metal_profile_stages_enabled() + && *ENABLED + .get_or_init(|| env_flag_enabled(METAL_PROFILE_CLASSIC_TIER1_ARITHMETIC_PACK_ENV)) +} + +pub(crate) fn metal_profile_classic_tier1_symbol_plan_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + metal_profile_stages_enabled() + && *ENABLED.get_or_init(|| { + env_flag_enabled(METAL_PROFILE_CLASSIC_TIER1_SYMBOL_PLAN_ENV) + || env_flag_enabled(METAL_PROFILE_CLASSIC_TIER1_PASS_PLAN_ENV) + }) +} + +pub(crate) fn metal_profile_classic_tier1_pass_plan_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + metal_profile_stages_enabled() + && *ENABLED.get_or_init(|| env_flag_enabled(METAL_PROFILE_CLASSIC_TIER1_PASS_PLAN_ENV)) +} + +pub(crate) fn metal_profile_classic_tier1_token_emit_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + metal_profile_stages_enabled() + && *ENABLED.get_or_init(|| { + env_flag_enabled(METAL_PROFILE_CLASSIC_TIER1_TOKEN_EMIT_ENV) + || env_flag_enabled(METAL_PROFILE_CLASSIC_TIER1_TOKEN_PACK_ENV) + }) +} + +pub(crate) fn metal_profile_classic_tier1_split_token_emit_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + metal_profile_stages_enabled() + && *ENABLED + .get_or_init(|| env_flag_enabled(METAL_PROFILE_CLASSIC_TIER1_SPLIT_TOKEN_EMIT_ENV)) +} + +pub(crate) fn metal_profile_classic_tier1_token_pack_enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + metal_profile_stages_enabled() + && *ENABLED.get_or_init(|| env_flag_enabled(METAL_PROFILE_CLASSIC_TIER1_TOKEN_PACK_ENV)) +} + +pub(crate) fn decode_profile_label() -> String { + std::env::var(METAL_PROFILE_DECODE_LABEL_ENV) + .ok() + .filter(|label| !label.is_empty()) + .map_or_else( + || "unlabeled".to_string(), + |label| sanitize_profile_label(&label), + ) +} + +fn sanitize_profile_label(label: &str) -> String { + label + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.') { + ch + } else { + '_' + } + }) + .collect() +} + +pub(crate) fn label_command_buffer(command_buffer: &CommandBufferRef, label: &str) { + if metal_profile_stages_enabled() { + command_buffer.set_label(label); + } +} + +type OsSignpostId = u64; + +const OS_SIGNPOST_ID_NULL: OsSignpostId = 0; +const OS_SIGNPOST_ID_INVALID: OsSignpostId = OsSignpostId::MAX; + +unsafe extern "C" { + fn j2k_metal_signpost_begin(name: HybridSignpostName) -> OsSignpostId; + fn j2k_metal_signpost_end(name: HybridSignpostName, id: OsSignpostId); +} + +pub(crate) struct HybridStageSignpost { + id: OsSignpostId, + name: HybridSignpostName, +} + +impl Drop for HybridStageSignpost { + fn drop(&mut self) { + // SAFETY: The shim accepts the signpost id returned by the matching begin call. + unsafe { + j2k_metal_signpost_end(self.name, self.id); + } + } +} + +pub(crate) fn hybrid_stage_signpost(name: HybridSignpostName) -> Option { + if !metal_profile_signposts_enabled() { + return None; + } + // SAFETY: The signpost shim is provided by this crate's build and has no Rust aliasing contract. + let id = unsafe { j2k_metal_signpost_begin(name) }; + if id == OS_SIGNPOST_ID_NULL || id == OS_SIGNPOST_ID_INVALID { + return None; + } + Some(HybridStageSignpost { id, name }) +} + +pub(crate) fn label_compute_encoder(encoder: &ComputeCommandEncoderRef, label: &str) { + if metal_profile_stages_enabled() { + encoder.set_label(label); + } +} diff --git a/crates/j2k-metal/src/quantize.metal b/crates/j2k-metal/src/quantize.metal new file mode 100644 index 00000000..e9c13cd3 --- /dev/null +++ b/crates/j2k-metal/src/quantize.metal @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 + +#include +using namespace metal; + +struct J2kQuantizeSubbandParams { + uint len; + uint step_exponent; + uint step_mantissa; + uint range_bits; + uint reversible; + uint reserved0; + uint reserved1; + uint reserved2; +}; + +inline int j2k_quantize_round_to_i32(float sample) { + const float rounded = sample >= 0.0f + ? floor(sample + 0.5f) + : -floor(-sample + 0.5f); + return int(rounded); +} + +inline int j2k_quantize_sample(float sample, constant J2kQuantizeSubbandParams ¶ms) { + if (params.reversible != 0u) { + return j2k_quantize_round_to_i32(sample); + } + + const int exponent = int(params.range_bits) - int(params.step_exponent); + const float base = exp2(float(exponent)); + const float delta = base * (1.0f + float(params.step_mantissa) / 2048.0f); + if (delta <= 0.0f) { + return 0; + } + + const int sign = sample < 0.0f ? -1 : 1; + const int magnitude = int(floor(fabs(sample) / delta)); + return sign * magnitude; +} + +kernel void j2k_quantize_subband( + device const float *samples [[buffer(0)]], + device int *coefficients [[buffer(1)]], + constant J2kQuantizeSubbandParams ¶ms [[buffer(2)]], + uint gid [[thread_position_in_grid]] +) { + if (gid >= params.len) { + return; + } + + coefficients[gid] = j2k_quantize_sample(samples[gid], params); +} diff --git a/crates/signinum-j2k-metal/src/routing.rs b/crates/j2k-metal/src/routing.rs similarity index 61% rename from crates/signinum-j2k-metal/src/routing.rs rename to crates/j2k-metal/src/routing.rs index 68fb5e8a..08e32c81 100644 --- a/crates/signinum-j2k-metal/src/routing.rs +++ b/crates/j2k-metal/src/routing.rs @@ -1,8 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_core::{BackendRequest, PixelFormat}; +use j2k_core::{BackendRequest, PixelFormat}; +#[cfg(target_os = "macos")] +use j2k_metal_support::metal_kernel_route; +use j2k_metal_support::{ + cpu_host_route, reject_explicit_metal_route, reject_unsupported_backend_route, + MetalRouteProfileLabels, +}; -use crate::{profile, Error}; +use crate::Error; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum RouteDecision { @@ -10,7 +16,7 @@ pub(crate) enum RouteDecision { #[cfg(target_os = "macos")] MetalKernel, RejectExplicitMetal { - reason: &'static str, + reason: ExplicitMetalRejection, }, RejectUnsupportedBackend { request: BackendRequest, @@ -19,6 +25,30 @@ pub(crate) enum RouteDecision { MetalUnavailable, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ExplicitMetalRejection { + UnsupportedFormat { fmt: PixelFormat }, +} + +impl ExplicitMetalRejection { + fn error_reason(self) -> &'static str { + match self { + Self::UnsupportedFormat { + fmt: PixelFormat::Rgba16, + } => "J2K Metal does not support PixelFormat::Rgba16", + Self::UnsupportedFormat { .. } => { + "J2K Metal does not support the requested PixelFormat" + } + } + } + + fn profile_reason(self) -> &'static str { + match self { + Self::UnsupportedFormat { .. } => "unsupported_format", + } + } +} + pub(crate) fn supports_metal_format(fmt: PixelFormat) -> bool { matches!( fmt, @@ -53,19 +83,18 @@ pub(crate) fn decide_route(backend: BackendRequest, fmt: PixelFormat) -> RouteDe request: BackendRequest::Cuda, }, }; - if profile::gpu_route_profile_enabled() { + if j2k_profile::gpu_route_profile_enabled() { let request_s = format!("{backend:?}"); let fmt_s = format!("{fmt:?}"); - let (decision_s, reason_s) = j2k_route_decision_profile(decision); - profile::emit_gpu_route_profile( + let labels = j2k_route_decision_profile(decision); + j2k_profile::emit_gpu_route_profile( "j2k", - "gpu_route", "metal", &[ ("request", request_s.as_str()), ("fmt", fmt_s.as_str()), - ("decision", decision_s), - ("reason", reason_s), + ("decision", labels.decision), + ("reason", labels.reason), ], ); } @@ -74,9 +103,9 @@ pub(crate) fn decide_route(backend: BackendRequest, fmt: PixelFormat) -> RouteDe pub(crate) fn decision_error(decision: RouteDecision) -> Option { match decision { - RouteDecision::RejectExplicitMetal { reason } => { - Some(Error::UnsupportedMetalRequest { reason }) - } + RouteDecision::RejectExplicitMetal { reason } => Some(Error::UnsupportedMetalRequest { + reason: reason.error_reason(), + }), RouteDecision::RejectUnsupportedBackend { request } => { Some(Error::UnsupportedBackend { request }) } @@ -89,26 +118,21 @@ pub(crate) fn decision_error(decision: RouteDecision) -> Option { } } -fn unsupported_metal_format_reason(fmt: PixelFormat) -> &'static str { - match fmt { - PixelFormat::Rgba16 => "J2K Metal does not support PixelFormat::Rgba16", - _ => "J2K Metal does not support the requested PixelFormat", - } +fn unsupported_metal_format_reason(fmt: PixelFormat) -> ExplicitMetalRejection { + ExplicitMetalRejection::UnsupportedFormat { fmt } } -fn j2k_route_decision_profile(decision: RouteDecision) -> (&'static str, &'static str) { +fn j2k_route_decision_profile(decision: RouteDecision) -> MetalRouteProfileLabels { match decision { - RouteDecision::CpuHost => ("cpu_host", "none"), + RouteDecision::CpuHost => cpu_host_route(), #[cfg(target_os = "macos")] - RouteDecision::MetalKernel => ("metal_kernel", "none"), - RouteDecision::RejectExplicitMetal { .. } => { - ("reject_explicit_metal", "unsupported_format") - } - RouteDecision::RejectUnsupportedBackend { .. } => { - ("reject_unsupported_backend", "unsupported_backend") + RouteDecision::MetalKernel => metal_kernel_route(), + RouteDecision::RejectExplicitMetal { reason } => { + reject_explicit_metal_route(reason.profile_reason()) } + RouteDecision::RejectUnsupportedBackend { .. } => reject_unsupported_backend_route(), #[cfg(not(target_os = "macos"))] - RouteDecision::MetalUnavailable => ("metal_unavailable", "metal_unavailable"), + RouteDecision::MetalUnavailable => j2k_metal_support::metal_unavailable_route(), } } @@ -136,7 +160,11 @@ mod tests { fn explicit_metal_unsupported_format_is_rejected_before_launch() { assert!(matches!( decide_route(BackendRequest::Metal, PixelFormat::Rgba16), - RouteDecision::RejectExplicitMetal { reason } if reason.contains("Rgba16") + RouteDecision::RejectExplicitMetal { + reason: ExplicitMetalRejection::UnsupportedFormat { + fmt: PixelFormat::Rgba16 + } + } )); } @@ -145,7 +173,11 @@ mod tests { fn explicit_metal_unsupported_format_is_rejected_before_host_unavailability() { assert!(matches!( decide_route(BackendRequest::Metal, PixelFormat::Rgba16), - RouteDecision::RejectExplicitMetal { reason } if reason.contains("Rgba16") + RouteDecision::RejectExplicitMetal { + reason: ExplicitMetalRejection::UnsupportedFormat { + fmt: PixelFormat::Rgba16 + } + } )); assert!(matches!( decide_route(BackendRequest::Metal, PixelFormat::Rgb8), diff --git a/crates/j2k-metal/src/signpost.c b/crates/j2k-metal/src/signpost.c new file mode 100644 index 00000000..8c20d52c --- /dev/null +++ b/crates/j2k-metal/src/signpost.c @@ -0,0 +1,188 @@ +#include + +#include +#include +#include + +enum { + J2K_SIGNPOST_DECODE_HYBRID_CPU_TIER1 = 1, + J2K_SIGNPOST_DECODE_HYBRID_COEFFICIENT_UPLOAD = 2, + J2K_SIGNPOST_DECODE_HYBRID_COMMAND_WAIT = 3, + J2K_SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE = 4, + J2K_SIGNPOST_DECODE_HYBRID_STORE_COMMAND_ENCODE = 5, + J2K_SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE = 6, + J2K_SIGNPOST_ENCODE_HYBRID_COMMAND_WAIT = 7, + J2K_SIGNPOST_ENCODE_HYBRID_RESULT_HARVEST = 8, + J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_SETUP = 9, + J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_COMMAND_ENCODE = 10, + J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_PLAN = 11, + J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_BUFFER_SETUP = 12, + J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKETIZATION_COMMAND_ENCODE = 13, + J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_PAYLOAD_COPY_COMMAND_ENCODE = 14, + J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_CODESTREAM_ASSEMBLY_COMMAND_ENCODE = 15, + J2K_SIGNPOST_ENCODE_HYBRID_HT_TIER1_SETUP = 16, + J2K_SIGNPOST_ENCODE_HYBRID_HT_TIER1_COMMAND_ENCODE = 17, + J2K_SIGNPOST_ENCODE_HYBRID_HT_PACKET_PLAN = 18, + J2K_SIGNPOST_ENCODE_HYBRID_HT_PACKET_BUFFER_SETUP = 19, + J2K_SIGNPOST_ENCODE_HYBRID_HT_PACKET_BLOCK_PREP_COMMAND_ENCODE = 20, + J2K_SIGNPOST_ENCODE_HYBRID_HT_PACKETIZATION_COMMAND_ENCODE = 21, + J2K_SIGNPOST_ENCODE_HYBRID_HT_PAYLOAD_COPY_COMMAND_ENCODE = 22, + J2K_SIGNPOST_ENCODE_HYBRID_HT_CODESTREAM_ASSEMBLY_COMMAND_ENCODE = 23, +}; + +static os_log_t j2k_metal_log_handle; +static dispatch_once_t j2k_metal_log_once; + +static void j2k_metal_log_init(void *context) { + (void)context; + j2k_metal_log_handle = os_log_create( + "com.frames. j2k.j2k-metal", + OS_LOG_CATEGORY_POINTS_OF_INTEREST); +} + +static os_log_t j2k_metal_log(void) { + dispatch_once_f(&j2k_metal_log_once, NULL, j2k_metal_log_init); + return j2k_metal_log_handle; +} + +#define J2K_SIGNPOST_BEGIN_CASE(id, name_literal) \ + case id: \ + os_signpost_interval_begin(log, signpost_id, name_literal); \ + return (uint64_t)signpost_id + +#define J2K_SIGNPOST_END_CASE(id, name_literal) \ + case id: \ + os_signpost_interval_end(log, signpost_id, name_literal); \ + return + +uint64_t j2k_metal_signpost_begin(uint32_t name_id) { + os_log_t log = j2k_metal_log(); + if (log == NULL || !os_signpost_enabled(log)) { + return 0; + } + + os_signpost_id_t signpost_id = os_signpost_id_generate(log); + if (signpost_id == OS_SIGNPOST_ID_NULL || signpost_id == OS_SIGNPOST_ID_INVALID) { + return 0; + } + + switch (name_id) { + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_DECODE_HYBRID_CPU_TIER1, + "j2k decode hybrid cpu tier1"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_DECODE_HYBRID_COEFFICIENT_UPLOAD, + "j2k decode hybrid coefficient upload"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_DECODE_HYBRID_COMMAND_WAIT, + "j2k decode hybrid command wait"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE, + "j2k decode hybrid idwt command encode"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_DECODE_HYBRID_STORE_COMMAND_ENCODE, + "j2k decode hybrid store command encode"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE, + "j2k decode hybrid mct pack command encode"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_COMMAND_WAIT, + "j2k encode hybrid command wait"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_RESULT_HARVEST, + "j2k encode hybrid result harvest"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_SETUP, + "j2k encode hybrid classic tier1 setup"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_COMMAND_ENCODE, + "j2k encode hybrid classic tier1 command encode"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_PLAN, + "j2k encode hybrid classic packet plan"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_BUFFER_SETUP, + "j2k encode hybrid classic packet buffer setup"); + J2K_SIGNPOST_BEGIN_CASE( + J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKETIZATION_COMMAND_ENCODE, + "j2k encode hybrid classic packetization command encode"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_PAYLOAD_COPY_COMMAND_ENCODE, + "j2k encode hybrid classic payload copy command encode"); + J2K_SIGNPOST_BEGIN_CASE( + J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_CODESTREAM_ASSEMBLY_COMMAND_ENCODE, + "j2k encode hybrid classic codestream assembly command encode"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_TIER1_SETUP, + "j2k encode hybrid ht tier1 setup"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_TIER1_COMMAND_ENCODE, + "j2k encode hybrid ht tier1 command encode"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_PACKET_PLAN, + "j2k encode hybrid ht packet plan"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_PACKET_BUFFER_SETUP, + "j2k encode hybrid ht packet buffer setup"); + J2K_SIGNPOST_BEGIN_CASE( + J2K_SIGNPOST_ENCODE_HYBRID_HT_PACKET_BLOCK_PREP_COMMAND_ENCODE, + "j2k encode hybrid ht packet block prep command encode"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_PACKETIZATION_COMMAND_ENCODE, + "j2k encode hybrid ht packetization command encode"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_PAYLOAD_COPY_COMMAND_ENCODE, + "j2k encode hybrid ht payload copy command encode"); + J2K_SIGNPOST_BEGIN_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_CODESTREAM_ASSEMBLY_COMMAND_ENCODE, + "j2k encode hybrid ht codestream assembly command encode"); + default: + return 0; + } +} + +void j2k_metal_signpost_end(uint32_t name_id, uint64_t raw_signpost_id) { + if (raw_signpost_id == OS_SIGNPOST_ID_NULL || raw_signpost_id == OS_SIGNPOST_ID_INVALID) { + return; + } + + os_log_t log = j2k_metal_log(); + if (log == NULL) { + return; + } + + os_signpost_id_t signpost_id = (os_signpost_id_t)raw_signpost_id; + switch (name_id) { + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_DECODE_HYBRID_CPU_TIER1, + "j2k decode hybrid cpu tier1"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_DECODE_HYBRID_COEFFICIENT_UPLOAD, + "j2k decode hybrid coefficient upload"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_DECODE_HYBRID_COMMAND_WAIT, + "j2k decode hybrid command wait"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_DECODE_HYBRID_IDWT_COMMAND_ENCODE, + "j2k decode hybrid idwt command encode"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_DECODE_HYBRID_STORE_COMMAND_ENCODE, + "j2k decode hybrid store command encode"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_DECODE_HYBRID_MCT_PACK_COMMAND_ENCODE, + "j2k decode hybrid mct pack command encode"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_COMMAND_WAIT, + "j2k encode hybrid command wait"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_RESULT_HARVEST, + "j2k encode hybrid result harvest"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_SETUP, + "j2k encode hybrid classic tier1 setup"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_TIER1_COMMAND_ENCODE, + "j2k encode hybrid classic tier1 command encode"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_PLAN, + "j2k encode hybrid classic packet plan"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKET_BUFFER_SETUP, + "j2k encode hybrid classic packet buffer setup"); + J2K_SIGNPOST_END_CASE( + J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_PACKETIZATION_COMMAND_ENCODE, + "j2k encode hybrid classic packetization command encode"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_PAYLOAD_COPY_COMMAND_ENCODE, + "j2k encode hybrid classic payload copy command encode"); + J2K_SIGNPOST_END_CASE( + J2K_SIGNPOST_ENCODE_HYBRID_CLASSIC_CODESTREAM_ASSEMBLY_COMMAND_ENCODE, + "j2k encode hybrid classic codestream assembly command encode"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_TIER1_SETUP, + "j2k encode hybrid ht tier1 setup"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_TIER1_COMMAND_ENCODE, + "j2k encode hybrid ht tier1 command encode"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_PACKET_PLAN, + "j2k encode hybrid ht packet plan"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_PACKET_BUFFER_SETUP, + "j2k encode hybrid ht packet buffer setup"); + J2K_SIGNPOST_END_CASE( + J2K_SIGNPOST_ENCODE_HYBRID_HT_PACKET_BLOCK_PREP_COMMAND_ENCODE, + "j2k encode hybrid ht packet block prep command encode"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_PACKETIZATION_COMMAND_ENCODE, + "j2k encode hybrid ht packetization command encode"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_PAYLOAD_COPY_COMMAND_ENCODE, + "j2k encode hybrid ht payload copy command encode"); + J2K_SIGNPOST_END_CASE(J2K_SIGNPOST_ENCODE_HYBRID_HT_CODESTREAM_ASSEMBLY_COMMAND_ENCODE, + "j2k encode hybrid ht codestream assembly command encode"); + default: + return; + } +} diff --git a/crates/signinum-j2k-metal/src/store.metal b/crates/j2k-metal/src/store.metal similarity index 61% rename from crates/signinum-j2k-metal/src/store.metal rename to crates/j2k-metal/src/store.metal index 316a9875..c754aad2 100644 --- a/crates/signinum-j2k-metal/src/store.metal +++ b/crates/j2k-metal/src/store.metal @@ -18,6 +18,7 @@ struct J2kStoreParams { struct J2kRepeatedStoreParams { uint input_width; uint input_height; + uint input_instance_stride; uint source_x; uint source_y; uint copy_width; @@ -63,6 +64,33 @@ struct J2kGrayStoreParams { float u16_scale; }; +struct J2kStoreWindowIndices { + uint src_idx; + uint dst_idx; +}; + +inline J2kStoreWindowIndices j2k_store_window_indices( + uint input_width, + uint output_width, + uint source_x, + uint source_y, + uint output_x, + uint output_y, + uint2 gid, + uint input_offset, + uint output_offset +) { + const uint src_x = source_x + gid.x; + const uint src_y = source_y + gid.y; + const uint dst_x = output_x + gid.x; + const uint dst_y = output_y + gid.y; + + return { + input_offset + src_y * input_width + src_x, + output_offset + dst_y * output_width + dst_x, + }; +} + kernel void j2k_store_component( device const float *input [[buffer(0)]], device float *output [[buffer(1)]], @@ -73,14 +101,18 @@ kernel void j2k_store_component( return; } - const uint src_x = params.source_x + gid.x; - const uint src_y = params.source_y + gid.y; - const uint dst_x = params.output_x + gid.x; - const uint dst_y = params.output_y + gid.y; - - const uint src_idx = src_y * params.input_width + src_x; - const uint dst_idx = dst_y * params.output_width + dst_x; - output[dst_idx] = input[src_idx] + params.addend; + const J2kStoreWindowIndices indices = j2k_store_window_indices( + params.input_width, + params.output_width, + params.source_x, + params.source_y, + params.output_x, + params.output_y, + gid, + 0u, + 0u + ); + output[indices.dst_idx] = input[indices.src_idx] + params.addend; } kernel void j2k_store_component_repeated( @@ -93,16 +125,19 @@ kernel void j2k_store_component_repeated( return; } - const uint input_plane_len = params.input_width * params.input_height; const uint output_plane_len = params.output_width * params.output_height; - const uint src_x = params.source_x + gid.x; - const uint src_y = params.source_y + gid.y; - const uint dst_x = params.output_x + gid.x; - const uint dst_y = params.output_y + gid.y; - - const uint src_idx = gid.z * input_plane_len + src_y * params.input_width + src_x; - const uint dst_idx = gid.z * output_plane_len + dst_y * params.output_width + dst_x; - output[dst_idx] = input[src_idx] + params.addend; + const J2kStoreWindowIndices indices = j2k_store_window_indices( + params.input_width, + params.output_width, + params.source_x, + params.source_y, + params.output_x, + params.output_y, + gid.xy, + gid.z * params.input_instance_stride, + gid.z * output_plane_len + ); + output[indices.dst_idx] = input[indices.src_idx] + params.addend; } kernel void j2k_store_component_repeated_gray_u8( @@ -117,14 +152,18 @@ kernel void j2k_store_component_repeated_gray_u8( const uint input_plane_len = params.input_width * params.input_height; const uint output_plane_len = params.output_width * params.output_height; - const uint src_x = params.source_x + gid.x; - const uint src_y = params.source_y + gid.y; - const uint dst_x = params.output_x + gid.x; - const uint dst_y = params.output_y + gid.y; - - const uint src_idx = gid.z * input_plane_len + src_y * params.input_width + src_x; - const uint dst_idx = gid.z * output_plane_len + dst_y * params.output_width + dst_x; - output[dst_idx] = scale_to_u8(input[src_idx] + params.addend, params.max_value, params.u8_scale); + const J2kStoreWindowIndices indices = j2k_store_window_indices( + params.input_width, + params.output_width, + params.source_x, + params.source_y, + params.output_x, + params.output_y, + gid.xy, + gid.z * input_plane_len, + gid.z * output_plane_len + ); + output[indices.dst_idx] = scale_to_u8(input[indices.src_idx] + params.addend, params.max_value, params.u8_scale); } kernel void j2k_store_component_repeated_gray_u16( @@ -139,14 +178,18 @@ kernel void j2k_store_component_repeated_gray_u16( const uint input_plane_len = params.input_width * params.input_height; const uint output_plane_len = params.output_width * params.output_height; - const uint src_x = params.source_x + gid.x; - const uint src_y = params.source_y + gid.y; - const uint dst_x = params.output_x + gid.x; - const uint dst_y = params.output_y + gid.y; - - const uint src_idx = gid.z * input_plane_len + src_y * params.input_width + src_x; - const uint dst_idx = gid.z * output_plane_len + dst_y * params.output_width + dst_x; - output[dst_idx] = pack_to_u16(input[src_idx] + params.addend, params.max_value, params.u16_scale); + const J2kStoreWindowIndices indices = j2k_store_window_indices( + params.input_width, + params.output_width, + params.source_x, + params.source_y, + params.output_x, + params.output_y, + gid.xy, + gid.z * input_plane_len, + gid.z * output_plane_len + ); + output[indices.dst_idx] = pack_to_u16(input[indices.src_idx] + params.addend, params.max_value, params.u16_scale); } kernel void j2k_store_component_repeated_gray_u8_contiguous( @@ -189,14 +232,18 @@ kernel void j2k_store_component_gray_u8( return; } - const uint src_x = params.source_x + gid.x; - const uint src_y = params.source_y + gid.y; - const uint dst_x = params.output_x + gid.x; - const uint dst_y = params.output_y + gid.y; - - const uint src_idx = src_y * params.input_width + src_x; - const uint dst_idx = dst_y * params.output_width + dst_x; - output[dst_idx] = scale_to_u8(input[src_idx] + params.addend, params.max_value, params.u8_scale); + const J2kStoreWindowIndices indices = j2k_store_window_indices( + params.input_width, + params.output_width, + params.source_x, + params.source_y, + params.output_x, + params.output_y, + gid, + 0u, + 0u + ); + output[indices.dst_idx] = scale_to_u8(input[indices.src_idx] + params.addend, params.max_value, params.u8_scale); } kernel void j2k_store_component_gray_u16( @@ -209,12 +256,16 @@ kernel void j2k_store_component_gray_u16( return; } - const uint src_x = params.source_x + gid.x; - const uint src_y = params.source_y + gid.y; - const uint dst_x = params.output_x + gid.x; - const uint dst_y = params.output_y + gid.y; - - const uint src_idx = src_y * params.input_width + src_x; - const uint dst_idx = dst_y * params.output_width + dst_x; - output[dst_idx] = pack_to_u16(input[src_idx] + params.addend, params.max_value, params.u16_scale); + const J2kStoreWindowIndices indices = j2k_store_window_indices( + params.input_width, + params.output_width, + params.source_x, + params.source_y, + params.output_x, + params.output_y, + gid, + 0u, + 0u + ); + output[indices.dst_idx] = pack_to_u16(input[indices.src_idx] + params.addend, params.max_value, params.u16_scale); } diff --git a/crates/signinum-j2k-metal/src/store.rs b/crates/j2k-metal/src/store.rs similarity index 95% rename from crates/signinum-j2k-metal/src/store.rs rename to crates/j2k-metal/src/store.rs index 69a9fa03..5fd73b60 100644 --- a/crates/signinum-j2k-metal/src/store.rs +++ b/crates/j2k-metal/src/store.rs @@ -2,12 +2,12 @@ #[cfg(target_os = "macos")] use crate::compute; -#[cfg(target_os = "macos")] -use metal::Buffer; -use signinum_j2k_native::{ +use j2k_native::{ decode_ht_code_block_scalar, HtCodeBlockDecodeJob, HtCodeBlockDecoder, J2kStoreComponentJob, Result, }; +#[cfg(target_os = "macos")] +use metal::Buffer; #[derive(Default)] pub(crate) struct MetalStoreDecoder { @@ -39,7 +39,7 @@ impl HtCodeBlockDecoder for MetalStoreDecoder { #[cfg(target_os = "macos")] if supports_metal_store(&job) { let captured = compute::decode_store_component_and_capture(job) - .map_err(|_| signinum_j2k_native::DecodingError::CodeBlockDecodeFailure)?; + .map_err(|_| j2k_native::DecodingError::CodeBlockDecodeFailure)?; self.captured_planes.push(captured); self.kernel_dispatches = self.kernel_dispatches.saturating_add(1); return Ok(true); @@ -54,7 +54,7 @@ impl HtCodeBlockDecoder for MetalStoreDecoder { &mut self, job: HtCodeBlockDecodeJob<'_>, output: &mut [f32], - ) -> signinum_j2k_native::Result<()> { + ) -> j2k_native::Result<()> { decode_ht_code_block_scalar(job, output) } } @@ -72,7 +72,7 @@ fn supports_metal_store(job: &J2kStoreComponentJob<'_>) -> bool { #[cfg(test)] mod tests { use super::MetalStoreDecoder; - use signinum_j2k_native::{ + use j2k_native::{ encode, DecodeSettings, DecoderContext, EncodeOptions, HtCodeBlockDecodeJob, HtCodeBlockDecoder, Image, }; @@ -122,8 +122,8 @@ mod tests { &mut self, job: HtCodeBlockDecodeJob<'_>, output: &mut [f32], - ) -> signinum_j2k_native::Result<()> { - signinum_j2k_native::decode_ht_code_block_scalar(job, output) + ) -> j2k_native::Result<()> { + j2k_native::decode_ht_code_block_scalar(job, output) } } diff --git a/crates/j2k-metal/tests/bench_harness.rs b/crates/j2k-metal/tests/bench_harness.rs new file mode 100644 index 00000000..6ea81345 --- /dev/null +++ b/crates/j2k-metal/tests/bench_harness.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::path::{Path, PathBuf}; + +fn manifest_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) +} + +fn cargo_toml() -> String { + let path = manifest_dir().join("Cargo.toml"); + std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("must read {}: {error}", path.display())) +} + +fn bench_sources_under(path: &Path) -> Vec { + if !path.exists() { + return Vec::new(); + } + + let mut sources = Vec::new(); + let mut pending = vec![path.to_path_buf()]; + while let Some(dir) = pending.pop() { + let entries = std::fs::read_dir(&dir) + .unwrap_or_else(|error| panic!("must read {}: {error}", dir.display())); + for entry in entries { + let entry = entry.unwrap_or_else(|error| { + panic!("must enumerate {}: {error}", dir.display()); + }); + let path = entry.path(); + if path.is_dir() { + pending.push(path); + } else if path.extension().is_some_and(|extension| extension == "rs") { + sources.push(path); + } + } + } + sources.sort(); + sources +} + +#[test] +fn j2k_metal_has_no_legacy_criterion_bench_targets() { + let cargo = cargo_toml(); + + assert!( + !cargo.contains("[[bench]]"), + "j2k-metal bench targets were reset for a clean profiling redesign" + ); + + for target in ["device_upload", "compare", "encode_stages", "decode_stages"] { + assert!( + !cargo.contains(&format!("name = \"{target}\"")), + "legacy j2k-metal bench target must stay removed: {target}" + ); + } +} + +#[test] +fn j2k_metal_has_no_legacy_bench_only_dev_dependencies() { + let cargo = cargo_toml(); + + for dependency in ["criterion", "j2k-compare"] { + assert!( + !cargo.contains(&format!("{dependency} =")), + "legacy bench-only dev dependency must stay removed: {dependency}" + ); + } +} + +#[test] +fn j2k_metal_benches_directory_is_clean_for_redesign() { + let sources = bench_sources_under(&manifest_dir().join("benches")); + + assert!( + sources.is_empty(), + "remove stale j2k-metal bench sources before adding new profiling benches: {sources:?}" + ); +} diff --git a/crates/signinum-j2k-metal/tests/device.rs b/crates/j2k-metal/tests/device.rs similarity index 81% rename from crates/signinum-j2k-metal/tests/device.rs rename to crates/j2k-metal/tests/device.rs index 17bb9808..004c0f59 100644 --- a/crates/signinum-j2k-metal/tests/device.rs +++ b/crates/j2k-metal/tests/device.rs @@ -2,17 +2,19 @@ use std::sync::Arc; -use signinum_core::{ +use j2k::J2kContext; +use j2k_core::{ BackendKind, BackendRequest, CodecError, DeviceSubmission, DeviceSurface, Downscale, ImageDecode, ImageDecodeDevice, PixelFormat, Rect, TileBatchDecodeDevice, TileBatchDecodeManyDevice, TileBatchDecodeSubmit, }; -use signinum_j2k::J2kContext; -use signinum_j2k_metal::{ +use j2k_metal::{ Codec, Error, J2kDecoder, J2kScratchPool, MetalBackendSession, MetalSession, MetalTileBatch, SurfaceResidency, }; -use signinum_j2k_native::{encode, encode_htj2k, EncodeOptions}; +use j2k_native::{encode, encode_htj2k, EncodeOptions}; + +const UNSUPPORTED_RGBA16_REASON: &str = "J2K Metal does not support PixelFormat::Rgba16"; fn fixture_rgb8() -> Vec { let pixels = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]; @@ -66,6 +68,24 @@ fn fixture_ht_gray8_sized(width: u32, height: u32) -> Vec { encode_htj2k(&pixels, width, height, 1, 8, false, &options).expect("encode sized ht gray8") } +fn fixture_rgb8_sized(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 3 + y * 5) & 0xFF) as u8); + pixels.push(((x * 7 + y * 11 + 13) & 0xFF) as u8); + pixels.push(((x * 17 + y * 19 + 29) & 0xFF) as u8); + } + } + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + guard_bits: 2, + ..EncodeOptions::default() + }; + encode(&pixels, width, height, 3, 8, false, &options).expect("encode sized rgb8") +} + fn fixture_ht_gray8_unsupported_direct_width() -> Vec { let width = 512u32; let height = 8u32; @@ -332,7 +352,7 @@ fn auto_repeated_grayscale_uses_metal_for_512_batch() { #[test] fn tile_full_grayscale_device_path_uses_metal_direct() { let bytes = fixture_gray8(); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut pool = J2kScratchPool::new(); let surface = Codec::decode_tile_to_device( &mut ctx, @@ -421,7 +441,7 @@ fn decode_to_device_with_session_unsupported_rgba16_is_rejected() { match result { Err(Error::UnsupportedMetalRequest { reason }) => { - assert!(reason.contains("Rgba16")); + assert_eq!(reason, UNSUPPORTED_RGBA16_REASON); } Err(other) => panic!("unexpected explicit Metal session error: {other:?}"), Ok(surface) => panic!( @@ -434,7 +454,7 @@ fn decode_to_device_with_session_unsupported_rgba16_is_rejected() { #[test] fn submitted_full_grayscale_tiles_flush_as_one_device_batch() { let bytes = fixture_gray8(); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut session = MetalSession::default(); let mut pool = J2kScratchPool::new(); @@ -453,7 +473,7 @@ fn submitted_full_grayscale_tiles_flush_as_one_device_batch() { .collect::>(); assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 0, "submitted tile surfaces should stay queued until a wait flushes the session" ); @@ -471,7 +491,7 @@ fn submitted_full_grayscale_tiles_flush_as_one_device_batch() { assert_eq!(surface.as_bytes(), host.as_slice()); } assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 1, "compatible queued grayscale tiles should flush through one repeated Metal batch" ); @@ -480,7 +500,7 @@ fn submitted_full_grayscale_tiles_flush_as_one_device_batch() { #[test] fn submitted_auto_512_grayscale_tiles_flush_as_one_metal_batch() { let bytes = fixture_gray8_sized(512, 512); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut session = MetalSession::default(); let mut pool = J2kScratchPool::new(); @@ -499,7 +519,7 @@ fn submitted_auto_512_grayscale_tiles_flush_as_one_metal_batch() { .collect::>(); assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 0, "auto submitted tile surfaces should stay queued until a wait flushes the session" ); @@ -510,7 +530,7 @@ fn submitted_auto_512_grayscale_tiles_flush_as_one_metal_batch() { assert_eq!(surface.dimensions(), (512, 512)); } assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 1, "compatible auto grayscale tiles should flush through one repeated Metal batch" ); @@ -520,7 +540,7 @@ fn submitted_auto_512_grayscale_tiles_flush_as_one_metal_batch() { fn submitted_distinct_full_grayscale_tiles_flush_as_one_device_batch() { let classic_bytes = fixture_gray8(); let reversed_bytes = fixture_gray8_reversed(); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut session = MetalSession::default(); let mut pool = J2kScratchPool::new(); @@ -544,7 +564,7 @@ fn submitted_distinct_full_grayscale_tiles_flush_as_one_device_batch() { .expect("submit reversed tile"); assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 0, "distinct submitted tile surfaces should stay queued until wait" ); @@ -569,7 +589,7 @@ fn submitted_distinct_full_grayscale_tiles_flush_as_one_device_batch() { assert_eq!(classic_surface.as_bytes(), classic_host.as_slice()); assert_eq!(reversed_surface.as_bytes(), reversed_host.as_slice()); assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 1, "distinct queued grayscale tiles should flush through one Metal command buffer" ); @@ -578,7 +598,7 @@ fn submitted_distinct_full_grayscale_tiles_flush_as_one_device_batch() { #[test] fn submitted_full_rgb_tiles_flush_as_one_device_batch() { let bytes = fixture_direct_rgb8(); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut session = MetalSession::default(); let mut pool = J2kScratchPool::new(); @@ -597,7 +617,7 @@ fn submitted_full_rgb_tiles_flush_as_one_device_batch() { .collect::>(); assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 0, "submitted RGB tile surfaces should stay queued until a wait flushes the session" ); @@ -614,7 +634,7 @@ fn submitted_full_rgb_tiles_flush_as_one_device_batch() { assert_eq!(surface.as_bytes(), host.as_slice()); } assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 1, "compatible queued RGB tiles should flush through one Metal batch" ); @@ -629,7 +649,7 @@ fn submitted_distinct_full_rgb_tiles_stay_resident_when_batch_route_falls_back() ]; assert_ne!(rgb_tiles[0], rgb_tiles[1], "RGB batch fixtures must differ"); assert_ne!(rgb_tiles[1], rgb_tiles[2], "RGB batch fixtures must differ"); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut session = MetalSession::default(); let mut pool = J2kScratchPool::new(); @@ -649,7 +669,7 @@ fn submitted_distinct_full_rgb_tiles_stay_resident_when_batch_route_falls_back() .collect::>(); assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 0, "distinct RGB tile surfaces should stay queued until a wait flushes the session" ); @@ -676,7 +696,7 @@ fn submitted_distinct_full_rgb_tiles_stay_resident_when_batch_route_falls_back() surfaces.push(surface); } assert!( - session.submissions() >= 1, + session.submissions().expect("session submissions") >= 1, "queued RGB tiles should submit at least one resident Metal decode" ); for surface in surfaces { @@ -709,7 +729,7 @@ fn metal_tile_batch_decodes_submitted_tiles_in_order() { 1 ); assert_eq!(batch.len(), 2); - assert_eq!(batch.submissions(), 0); + assert_eq!(batch.submissions().expect("batch submissions"), 0); let surfaces = batch.decode_all().expect("batch decode"); assert_eq!(surfaces.len(), 2); @@ -737,7 +757,7 @@ fn metal_tile_batch_decodes_submitted_tiles_in_order() { fn tile_batch_decode_many_device_preserves_full_tile_order() { let classic_bytes = fixture_gray8(); let reversed_bytes = fixture_gray8_reversed(); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut pool = J2kScratchPool::new(); let inputs = [classic_bytes.as_slice(), reversed_bytes.as_slice()]; @@ -852,7 +872,7 @@ fn submitted_distinct_region_scaled_htj2k_grayscale_tiles_flush_as_one_device_ba }; let scale = Downscale::Half; let scaled = roi.scaled_covering(scale); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut session = MetalSession::default(); let mut pool = J2kScratchPool::new(); @@ -880,7 +900,7 @@ fn submitted_distinct_region_scaled_htj2k_grayscale_tiles_flush_as_one_device_ba .expect("submit reversed ht region-scaled tile"); assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 0, "region-scaled submitted tile surfaces should stay queued until wait" ); @@ -916,7 +936,7 @@ fn submitted_distinct_region_scaled_htj2k_grayscale_tiles_flush_as_one_device_ba assert_eq!(ht_surface.as_bytes(), expected[0].as_slice()); assert_eq!(reversed_surface.as_bytes(), expected[1].as_slice()); assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 1, "distinct queued HTJ2K region-scaled grayscale tiles should flush through one Metal command buffer" ); @@ -938,7 +958,7 @@ fn submitted_distinct_region_scaled_htj2k_gray16_tiles_flush_as_one_device_batch }; let scale = Downscale::Half; let scaled = roi.scaled_covering(scale); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut session = MetalSession::default(); let mut pool = J2kScratchPool::new(); @@ -994,7 +1014,7 @@ fn submitted_distinct_region_scaled_htj2k_gray16_tiles_flush_as_one_device_batch assert_eq!(first_surface.as_bytes(), expected[0].as_slice()); assert_eq!(second_surface.as_bytes(), expected[1].as_slice()); assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 1, "distinct queued HTJ2K region-scaled Gray16 tiles should flush through one Metal command buffer" ); @@ -1010,7 +1030,7 @@ fn submitted_auto_region_scaled_grayscale_keeps_short_batch_on_cpu() { h: 256, }; let scale = Downscale::Quarter; - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut session = MetalSession::default(); let mut pool = J2kScratchPool::new(); let submissions = (0..16) @@ -1034,7 +1054,7 @@ fn submitted_auto_region_scaled_grayscale_keeps_short_batch_on_cpu() { assert_eq!(surface.backend_kind(), BackendKind::Cpu); } assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 1, "short auto ROI+scaled grayscale tile batches should use one CPU batch fallback" ); @@ -1050,7 +1070,7 @@ fn submitted_auto_region_scaled_rgb_tiles_flush_as_one_cpu_batch() { h: 1, }; let scale = Downscale::None; - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut session = MetalSession::default(); let mut pool = J2kScratchPool::new(); let submissions = (0..3) @@ -1089,7 +1109,7 @@ fn submitted_auto_region_scaled_rgb_tiles_flush_as_one_cpu_batch() { assert_eq!(surface.as_bytes(), host.as_slice()); } assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 1, "auto RGB ROI+scaled tile batches should flush through one CPU batch fallback" ); @@ -1106,7 +1126,7 @@ fn submitted_auto_region_scaled_grayscale_batch64_uses_one_metal_batch() { }; let scale = Downscale::Quarter; let scaled = roi.scaled_covering(scale); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut session = MetalSession::default(); let mut pool = J2kScratchPool::new(); let submissions = (0..64) @@ -1146,7 +1166,7 @@ fn submitted_auto_region_scaled_grayscale_batch64_uses_one_metal_batch() { assert_eq!(surface.as_bytes(), host.as_slice()); } assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 1, "large auto ROI+scaled grayscale tile batches should use one Metal batch" ); @@ -1163,7 +1183,7 @@ fn submitted_auto_region_scaled_ht_grayscale_1024_batch16_uses_one_metal_batch() }; let scale = Downscale::Quarter; let scaled = roi.scaled_covering(scale); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut session = MetalSession::default(); let mut pool = J2kScratchPool::new(); let submissions = (0..16) @@ -1203,12 +1223,70 @@ fn submitted_auto_region_scaled_ht_grayscale_1024_batch16_uses_one_metal_batch() assert_eq!(surface.as_bytes(), host.as_slice()); } assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 1, "1024-class auto HT ROI+scaled grayscale tile batches should use one Metal batch" ); } +#[test] +fn submitted_auto_region_scaled_rgb_1024_batch16_uses_hybrid_metal() { + let bytes = fixture_rgb8_sized(1024, 1024); + let roi = Rect { + x: 128, + y: 128, + w: 512, + h: 256, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let mut ctx = j2k_core::DecoderContext::::new(); + let mut session = MetalSession::default(); + let mut pool = J2kScratchPool::new(); + let submissions = (0..16) + .map(|_| { + Codec::submit_tile_region_scaled_to_device( + &mut ctx, + &mut session, + &mut pool, + &bytes, + PixelFormat::Rgb8, + roi, + scale, + BackendRequest::Auto, + ) + .expect("submit auto region-scaled RGB tile") + }) + .collect::>(); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("host region-scaled RGB decode"); + + for submission in submissions { + let surface = submission.wait().expect("auto region-scaled RGB surface"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.as_bytes(), host.as_slice()); + } + assert_eq!( + session.submissions().expect("session submissions"), + 1, + "1024-class auto ROI+scaled RGB tile batches should use one resident hybrid Metal batch" + ); +} + #[test] fn submitted_auto_region_scaled_ht_grayscale_batch16_is_not_order_dependent() { let small_bytes = fixture_ht_gray8_sized(64, 64); @@ -1227,7 +1305,7 @@ fn submitted_auto_region_scaled_ht_grayscale_batch16_is_not_order_dependent() { }; let scale = Downscale::Quarter; let large_scaled = large_roi.scaled_covering(scale); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut session = MetalSession::default(); let mut pool = J2kScratchPool::new(); @@ -1287,7 +1365,7 @@ fn submitted_auto_region_scaled_ht_grayscale_batch16_is_not_order_dependent() { assert_eq!(surfaces[1].dimensions(), (large_scaled.w, large_scaled.h)); assert_eq!(surfaces[1].as_bytes(), host.as_slice()); assert_eq!( - session.submissions(), + session.submissions().expect("session submissions"), 2, "auto ROI+scaled should use one Metal batch for the sixteen qualifying 1024-class tiles and leave the leading small tile on CPU" ); @@ -1411,7 +1489,7 @@ fn explicit_metal_unsupported_rgba16_full_decode_is_rejected() { match result { Err(Error::UnsupportedMetalRequest { reason }) => { - assert!(reason.contains("Rgba16")); + assert_eq!(reason, UNSUPPORTED_RGBA16_REASON); } Err(other) => panic!("unexpected explicit Metal error: {other:?}"), Ok(surface) => panic!( @@ -1572,6 +1650,53 @@ fn explicit_metal_region_scaled_grayscale_matches_host_decode() { assert_eq!(surface.as_bytes(), host.as_slice()); } +#[test] +fn explicit_metal_region_scaled_grayscale_large_cropped_matches_host_decode() { + let bytes = fixture_gray8_sized(1024, 1024); + let roi = Rect { + x: 128, + y: 128, + w: 512, + h: 512, + }; + + for scale in [Downscale::Half, Downscale::None] { + let scaled = roi.scaled_covering(scale); + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let mut host = vec![0u8; scaled.w as usize * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + scaled.w as usize, + PixelFormat::Gray8, + roi, + scale, + ) + .expect("host region scaled decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + if surface.as_bytes() != host.as_slice() { + let mismatch = surface + .as_bytes() + .iter() + .zip(&host) + .position(|(actual, expected)| actual != expected) + .expect("mismatched buffers should have a differing byte"); + panic!( + "scale={scale:?} first mismatch at byte {mismatch}: metal={} host={}", + surface.as_bytes()[mismatch], + host[mismatch] + ); + } + } +} + #[test] fn explicit_metal_region_scaled_htj2k_grayscale_matches_host_decode() { let bytes = fixture_ht_gray8(); @@ -1641,28 +1766,145 @@ fn explicit_metal_region_scaled_htj2k_falls_back_when_direct_width_is_unsupporte } #[test] -fn explicit_metal_region_scaled_rgb_is_rejected_instead_of_cpu_staging() { - let bytes = fixture_rgb8(); +fn explicit_metal_region_scaled_rgb_matches_host_decode() { + let bytes = fixture_direct_rgb8_variant(3); + let roi = Rect { + x: 1, + y: 2, + w: 5, + h: 4, + }; + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("host region scaled RGB decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Rgb8, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled RGB decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.as_bytes(), host.as_slice()); + + let mut host_decoder = J2kDecoder::new(&bytes).expect("rgba8 host decoder"); + let stride = scaled.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgba8, + roi, + scale, + ) + .expect("host region scaled RGBA decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("rgba8 decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Rgba8, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled RGBA decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.as_bytes(), host.as_slice()); + + let bytes = fixture_rgb12(); let roi = Rect { x: 0, y: 0, - w: 1, + w: 2, h: 1, }; - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let Err(error) = decoder.decode_region_scaled_to_device( - PixelFormat::Rgb8, - roi, - Downscale::None, - BackendRequest::Metal, - ) else { - panic!("explicit Metal RGB ROI+scaled should not hide a CPU-staged path") + let scale = Downscale::Half; + let scaled = roi.scaled_covering(scale); + let mut host_decoder = J2kDecoder::new(&bytes).expect("rgb16 host decoder"); + let stride = scaled.w as usize * PixelFormat::Rgb16.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + PixelFormat::Rgb16, + roi, + scale, + ) + .expect("host region scaled RGB16 decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("rgb16 decoder"); + let surface = decoder + .decode_region_scaled_to_device(PixelFormat::Rgb16, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled RGB16 decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + assert_eq!(surface.as_bytes(), host.as_slice()); +} + +#[test] +fn explicit_metal_region_scaled_rgb_large_cropped_matches_host_decode() { + let bytes = fixture_rgb8_sized(1024, 1024); + let roi = Rect { + x: 128, + y: 128, + w: 512, + h: 512, }; - assert!(matches!( - error, - Error::UnsupportedMetalRequest { reason } - if reason.contains("ROI+scaled") && reason.contains("Gray8/Gray16") - )); + + for scale in [Downscale::Half, Downscale::None] { + let scaled = roi.scaled_covering(scale); + for fmt in [PixelFormat::Rgb8, PixelFormat::Rgba8] { + let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); + let stride = scaled.w as usize * fmt.bytes_per_pixel(); + let mut host = vec![0u8; stride * scaled.h as usize]; + host_decoder + .decode_region_scaled_into( + &mut J2kScratchPool::new(), + &mut host, + stride, + fmt, + roi, + scale, + ) + .expect("host region scaled RGB decode"); + + let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); + let surface = decoder + .decode_region_scaled_to_device(fmt, roi, scale, BackendRequest::Metal) + .expect("explicit Metal region scaled RGB decode"); + assert_eq!(surface.backend_kind(), BackendKind::Metal); + assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); + assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); + if surface.as_bytes() != host.as_slice() { + let mismatch = surface + .as_bytes() + .iter() + .zip(&host) + .position(|(actual, expected)| actual != expected) + .expect("mismatched buffers should have a differing byte"); + panic!( + "fmt={fmt:?} scale={scale:?} first mismatch at byte {mismatch}: metal={} host={}", + surface.as_bytes()[mismatch], + host[mismatch] + ); + } + } + } } #[test] @@ -1725,7 +1967,7 @@ fn invalid_region_reports_error_instead_of_panicking() { h: 2, }; match decoder.decode_region_to_device(PixelFormat::Rgb8, roi, BackendRequest::Auto) { - Err(Error::Decode(signinum_j2k::J2kError::InvalidRegion { .. })) => {} + Err(Error::Decode(j2k::J2kError::InvalidRegion { .. })) => {} Err(other) => panic!("unexpected error for invalid ROI: {other:?}"), Ok(_) => panic!("invalid ROI should fail"), } @@ -1734,7 +1976,7 @@ fn invalid_region_reports_error_instead_of_panicking() { #[test] fn explicit_metal_tile_unsupported_rgba16_is_rejected() { let bytes = fixture_rgb12(); - let mut ctx = signinum_core::DecoderContext::::new(); + let mut ctx = j2k_core::DecoderContext::::new(); let mut pool = J2kScratchPool::new(); let result = Codec::decode_tile_to_device( @@ -1747,7 +1989,7 @@ fn explicit_metal_tile_unsupported_rgba16_is_rejected() { match result { Err(Error::UnsupportedMetalRequest { reason }) => { - assert!(reason.contains("Rgba16")); + assert_eq!(reason, UNSUPPORTED_RGBA16_REASON); } Err(other) => panic!("unexpected explicit Metal tile error: {other:?}"), Ok(surface) => panic!( @@ -1756,3 +1998,21 @@ fn explicit_metal_tile_unsupported_rgba16_is_rejected() { ), } } + +#[test] +fn hybrid_ht_cpuupload_uses_worker_local_decode_workspace() { + let source = include_str!("../src/compute/direct_cpu.rs"); + + assert!( + source.contains("decode_prepared_ht_jobs_on_cpu_with_workspace"), + "HT CPUUpload decode must expose a workspace-aware helper" + ); + assert!( + source.contains("HtCodeBlockDecodeWorkspace::default()"), + "parallel HT CPUUpload decode must initialize worker-local HT decode workspaces" + ); + assert!( + source.contains("decode_ht_code_block_scalar_with_workspace"), + "HT CPUUpload decode must call the scratch-reusing scalar helper" + ); +} diff --git a/crates/j2k-metal/tests/encode_auto_routing_benchmark.rs b/crates/j2k-metal/tests/encode_auto_routing_benchmark.rs new file mode 100644 index 00000000..e6ad014f --- /dev/null +++ b/crates/j2k-metal/tests/encode_auto_routing_benchmark.rs @@ -0,0 +1,670 @@ +// SPDX-License-Identifier: Apache-2.0 + +#![allow(clippy::similar_names)] + +use std::time::{Duration, Instant}; + +use j2k::adapter::encode_stage::{ + J2kDeinterleaveToF32Job, J2kEncodeDispatchReport, J2kEncodeStageAccelerator, + J2kForwardDwt53Job, J2kForwardDwt97Job, J2kForwardIctJob, J2kForwardRctJob, + J2kQuantizeSubbandJob, +}; +use j2k::{ + encode_j2k_lossless, encode_j2k_lossless_with_accelerator, encode_j2k_lossy, + encode_j2k_lossy_with_accelerator, EncodeBackendPreference, EncodedJ2k, EncodedLossyJ2k, + J2kBlockCodingMode, J2kEncodeValidation, J2kLosslessEncodeOptions, J2kLosslessSamples, + J2kLossyEncodeOptions, J2kLossySamples, J2kRateTarget, +}; +use j2k_core::BackendKind; +use j2k_metal::MetalEncodeStageAccelerator; +use j2k_native::{ + deinterleave_reference, forward_dwt53_reference, forward_dwt97_reference, + forward_ict_reference, forward_rct_reference, quantize_subband_reference, +}; +use j2k_test_support::{patterned_gray8, patterned_rgb8}; + +const DIMS: &[u32] = &[128, 512, 1024]; +const ITERS: usize = 5; +const AUTO_STAGE_MIN_PIXELS: u64 = 512 * 512; +const AUTO_HTJ2K_HOST_RESIDENT_MIN_PIXELS: u64 = 1024 * 1024; + +#[test] +#[ignore = "benchmark harness; run explicitly with --ignored --nocapture"] +fn encode_auto_routing_benchmark() { + run_stage_benchmarks(); + for &dim in DIMS { + run_lossless_case(Codec::Classic, Components::Gray8, dim); + run_lossless_case(Codec::Classic, Components::Rgb8, dim); + run_lossless_case(Codec::Htj2k, Components::Rgb8, dim); + run_lossy_case(Codec::Classic, Components::Gray8, dim); + run_lossy_case(Codec::Htj2k, Components::Gray8, dim); + run_lossy_case(Codec::Htj2k, Components::Rgb8, dim); + } +} + +fn run_stage_benchmarks() { + for &dim in DIMS { + run_deinterleave_stage_benchmark(dim); + run_forward_rct_stage_benchmark(dim); + run_forward_ict_stage_benchmark(dim); + run_forward_dwt53_stage_benchmark(dim); + run_forward_dwt97_stage_benchmark(dim); + run_quantize_subband_stage_benchmark(dim); + } +} + +#[derive(Clone, Copy)] +enum Codec { + Classic, + Htj2k, +} + +impl Codec { + const fn block_coding_mode(self) -> J2kBlockCodingMode { + match self { + Self::Classic => J2kBlockCodingMode::Classic, + Self::Htj2k => J2kBlockCodingMode::HighThroughput, + } + } + + const fn label(self) -> &'static str { + match self { + Self::Classic => "classic", + Self::Htj2k => "htj2k", + } + } +} + +#[derive(Clone, Copy)] +enum Components { + Gray8, + Rgb8, +} + +impl Components { + const fn count(self) -> u8 { + match self { + Self::Gray8 => 1, + Self::Rgb8 => 3, + } + } + + const fn label(self) -> &'static str { + match self { + Self::Gray8 => "gray8", + Self::Rgb8 => "rgb8", + } + } + + fn pixels(self, width: u32, height: u32) -> Vec { + match self { + Self::Gray8 => patterned_gray8(width, height), + Self::Rgb8 => patterned_rgb8(width, height), + } + } +} + +fn run_lossless_case(codec: Codec, components: Components, dim: u32) { + let pixels = components.pixels(dim, dim); + let auto_probe = probe_lossless_auto(&pixels, dim, codec, components); + emit_probe("lossless", codec, components, dim, &auto_probe); + let cpu = measure(|| { + let samples = lossless_samples(std::hint::black_box(pixels.as_slice()), dim, components); + let options = lossless_options(codec, EncodeBackendPreference::CpuOnly); + let encoded = encode_j2k_lossless(samples, &options).expect("CPU lossless encode"); + assert_eq!(encoded.backend, BackendKind::Cpu); + encoded.codestream.len() + }); + let expected_dispatch = expected_lossless_auto_dispatch(codec, components, dim); + let auto = should_bench_auto(&auto_probe, expected_dispatch).then(|| { + measure(|| { + let samples = + lossless_samples(std::hint::black_box(pixels.as_slice()), dim, components); + let options = lossless_options(codec, EncodeBackendPreference::Auto); + let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); + let encoded = encode_j2k_lossless_with_accelerator( + samples, + &options, + BackendKind::Metal, + &mut accelerator, + ) + .expect("Auto Metal lossless encode"); + encoded.codestream.len() + }) + }); + emit_timing("lossless", codec, components, dim, cpu, auto); +} + +fn run_lossy_case(codec: Codec, components: Components, dim: u32) { + let pixels = components.pixels(dim, dim); + let auto_probe = probe_lossy_auto(&pixels, dim, codec, components); + emit_probe("lossy", codec, components, dim, &auto_probe); + let cpu = measure(|| { + let samples = lossy_samples(std::hint::black_box(pixels.as_slice()), dim, components); + let options = lossy_options(codec, components, EncodeBackendPreference::CpuOnly); + let encoded = encode_j2k_lossy(samples, &options).expect("CPU lossy encode"); + assert_eq!(encoded.backend, BackendKind::Cpu); + encoded.codestream.len() + }); + let auto = should_bench_auto(&auto_probe, false).then(|| { + measure(|| { + let samples = lossy_samples(std::hint::black_box(pixels.as_slice()), dim, components); + let options = lossy_options(codec, components, EncodeBackendPreference::Auto); + let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); + let encoded = encode_j2k_lossy_with_accelerator( + samples, + &options, + BackendKind::Metal, + &mut accelerator, + ) + .expect("Auto Metal lossy encode"); + encoded.codestream.len() + }) + }); + emit_timing("lossy", codec, components, dim, cpu, auto); +} + +fn run_deinterleave_stage_benchmark(dim: u32) { + let pixels = Components::Rgb8.pixels(dim, dim); + let num_pixels = usize::try_from(u64::from(dim) * u64::from(dim)).expect("dim fits usize"); + let cpu = measure(|| { + let planes = deinterleave_reference( + std::hint::black_box(pixels.as_slice()), + num_pixels, + 3, + 8, + false, + ); + plane_len(&planes) + }); + let metal = probe_deinterleave_stage(&pixels, num_pixels).map(|dispatch| { + let timing = measure(|| { + let mut accelerator = MetalEncodeStageAccelerator::default(); + let planes = accelerator + .encode_deinterleave(J2kDeinterleaveToF32Job { + pixels: std::hint::black_box(pixels.as_slice()), + num_pixels, + num_components: 3, + bit_depth: 8, + signed: false, + }) + .expect("Metal deinterleave stage") + .expect("Metal deinterleave dispatch"); + plane_len(&planes) + }); + (timing, dispatch) + }); + emit_stage_timing("deinterleave", dim, cpu, metal); +} + +fn run_forward_rct_stage_benchmark(dim: u32) { + let planes = stage_planes(dim); + let cpu = measure(|| { + let transformed = forward_rct_reference(std::hint::black_box(planes.clone())); + plane_len(&transformed) + }); + let metal = probe_forward_rct_stage(&planes).map(|dispatch| { + let timing = measure(|| { + let mut stage_planes = planes.clone(); + let mut accelerator = MetalEncodeStageAccelerator::default(); + let (plane0, plane1, plane2) = split_three_planes(&mut stage_planes); + let dispatched = accelerator + .encode_forward_rct(J2kForwardRctJob { + plane0, + plane1, + plane2, + }) + .expect("Metal forward RCT stage"); + assert!(dispatched); + plane_len(&stage_planes) + }); + (timing, dispatch) + }); + emit_stage_timing("forward_rct", dim, cpu, metal); +} + +fn run_forward_ict_stage_benchmark(dim: u32) { + let planes = stage_planes(dim); + let cpu = measure(|| { + let transformed = forward_ict_reference(std::hint::black_box(planes.clone())); + plane_len(&transformed) + }); + let metal = probe_forward_ict_stage(&planes).map(|dispatch| { + let timing = measure(|| { + let mut stage_planes = planes.clone(); + let mut accelerator = MetalEncodeStageAccelerator::default(); + let (plane0, plane1, plane2) = split_three_planes(&mut stage_planes); + let dispatched = accelerator + .encode_forward_ict(J2kForwardIctJob { + plane0, + plane1, + plane2, + }) + .expect("Metal forward ICT stage"); + assert!(dispatched); + plane_len(&stage_planes) + }); + (timing, dispatch) + }); + emit_stage_timing("forward_ict", dim, cpu, metal); +} + +fn run_forward_dwt53_stage_benchmark(dim: u32) { + let samples = stage_samples(dim); + let cpu = measure(|| { + let output = forward_dwt53_reference(std::hint::black_box(samples.as_slice()), dim, dim, 1); + dwt53_len(&output) + }); + let metal = probe_forward_dwt53_stage(&samples, dim).map(|dispatch| { + let timing = measure(|| { + let mut accelerator = MetalEncodeStageAccelerator::default(); + let output = accelerator + .encode_forward_dwt53(J2kForwardDwt53Job { + samples: std::hint::black_box(samples.as_slice()), + width: dim, + height: dim, + num_levels: 1, + }) + .expect("Metal forward DWT 5/3 stage") + .expect("Metal forward DWT 5/3 dispatch"); + dwt53_len(&output) + }); + (timing, dispatch) + }); + emit_stage_timing("forward_dwt53", dim, cpu, metal); +} + +fn run_forward_dwt97_stage_benchmark(dim: u32) { + let samples = stage_samples(dim); + let cpu = measure(|| { + let output = forward_dwt97_reference(std::hint::black_box(samples.as_slice()), dim, dim, 1); + dwt97_len(&output) + }); + let metal = probe_forward_dwt97_stage(&samples, dim).map(|dispatch| { + let timing = measure(|| { + let mut accelerator = MetalEncodeStageAccelerator::default(); + let output = accelerator + .encode_forward_dwt97(J2kForwardDwt97Job { + samples: std::hint::black_box(samples.as_slice()), + width: dim, + height: dim, + num_levels: 1, + }) + .expect("Metal forward DWT 9/7 stage") + .expect("Metal forward DWT 9/7 dispatch"); + dwt97_len(&output) + }); + (timing, dispatch) + }); + emit_stage_timing("forward_dwt97", dim, cpu, metal); +} + +fn run_quantize_subband_stage_benchmark(dim: u32) { + let coefficients = stage_samples(dim); + let cpu = measure(|| { + let quantized = quantize_subband_reference( + std::hint::black_box(coefficients.as_slice()), + 8, + 256, + 8, + false, + ); + quantized.len() + }); + let metal = probe_quantize_subband_stage(&coefficients).map(|dispatch| { + let timing = measure(|| { + let mut accelerator = MetalEncodeStageAccelerator::default(); + let quantized = accelerator + .encode_quantize_subband(J2kQuantizeSubbandJob { + coefficients: std::hint::black_box(coefficients.as_slice()), + step_exponent: 8, + step_mantissa: 256, + range_bits: 8, + reversible: false, + }) + .expect("Metal quantize_subband stage") + .expect("Metal quantize_subband dispatch"); + quantized.len() + }); + (timing, dispatch) + }); + emit_stage_timing("quantize_subband", dim, cpu, metal); +} + +fn measure(mut run: impl FnMut() -> usize) -> Duration { + std::hint::black_box(run()); + let mut durations = Vec::with_capacity(ITERS); + for _ in 0..ITERS { + let started = Instant::now(); + std::hint::black_box(run()); + durations.push(started.elapsed()); + } + durations.sort_unstable(); + durations[durations.len() / 2] +} + +fn lossless_samples(pixels: &[u8], dim: u32, components: Components) -> J2kLosslessSamples<'_> { + J2kLosslessSamples::new(pixels, dim, dim, components.count(), 8, false) + .expect("valid lossless samples") +} + +fn lossy_samples(pixels: &[u8], dim: u32, components: Components) -> J2kLossySamples<'_> { + J2kLossySamples::new(pixels, dim, dim, components.count(), 8, false) + .expect("valid lossy samples") +} + +fn lossless_options(codec: Codec, backend: EncodeBackendPreference) -> J2kLosslessEncodeOptions { + J2kLosslessEncodeOptions::default() + .with_backend(backend) + .with_block_coding_mode(codec.block_coding_mode()) + .with_max_decomposition_levels(Some(1)) + .with_validation(J2kEncodeValidation::External) +} + +fn lossy_options( + codec: Codec, + components: Components, + backend: EncodeBackendPreference, +) -> J2kLossyEncodeOptions { + let mut options = J2kLossyEncodeOptions::default() + .with_backend(backend) + .with_block_coding_mode(codec.block_coding_mode()) + .with_max_decomposition_levels(Some(1)) + .with_rate_target(Some(J2kRateTarget::BitsPerPixel( + 8.0 * f64::from(components.count()), + ))) + .with_validation(J2kEncodeValidation::External); + options.psnr_iteration_budget = 1; + options +} + +fn stage_samples(dim: u32) -> Vec { + let len = usize::try_from(u64::from(dim) * u64::from(dim)).expect("dim fits usize"); + (0..len) + .map(|idx| stage_sample_value(idx * 37 + idx / 11 + 17)) + .collect() +} + +fn stage_planes(dim: u32) -> Vec> { + let len = usize::try_from(u64::from(dim) * u64::from(dim)).expect("dim fits usize"); + (0..3) + .map(|component| { + (0..len) + .map(|idx| { + stage_sample_value(idx * (31 + component * 6) + idx / 7 + component * 19) + }) + .collect() + }) + .collect() +} + +fn stage_sample_value(value: usize) -> f32 { + f32::from(u8::try_from(value & 0xff).expect("masked stage sample fits u8")) - 128.0 +} + +fn split_three_planes(planes: &mut [Vec]) -> (&mut [f32], &mut [f32], &mut [f32]) { + assert!(planes.len() >= 3); + let (plane0, rest) = planes.split_at_mut(1); + let (plane1, plane2) = rest.split_at_mut(1); + (&mut plane0[0], &mut plane1[0], &mut plane2[0]) +} + +fn plane_len(planes: &[Vec]) -> usize { + planes.iter().map(Vec::len).sum() +} + +fn dwt53_len(output: &j2k::adapter::encode_stage::J2kForwardDwt53Output) -> usize { + output.ll.len() + + output + .levels + .iter() + .map(|level| level.hl.len() + level.lh.len() + level.hh.len()) + .sum::() +} + +fn dwt97_len(output: &j2k::adapter::encode_stage::J2kForwardDwt97Output) -> usize { + output.ll.len() + + output + .levels + .iter() + .map(|level| level.hl.len() + level.lh.len() + level.hh.len()) + .sum::() +} + +fn expected_lossless_auto_dispatch(codec: Codec, components: Components, dim: u32) -> bool { + let pixels = u64::from(dim).saturating_mul(u64::from(dim)); + let resident_htj2k_rgb = matches!(codec, Codec::Htj2k) + && matches!(components, Components::Rgb8) + && pixels >= AUTO_HTJ2K_HOST_RESIDENT_MIN_PIXELS; + let stage_gated_classic = matches!(codec, Codec::Classic) && pixels >= AUTO_STAGE_MIN_PIXELS; + resident_htj2k_rgb || stage_gated_classic +} + +fn probe_deinterleave_stage( + pixels: &[u8], + num_pixels: usize, +) -> Result { + let mut accelerator = MetalEncodeStageAccelerator::default(); + let components = accelerator + .encode_deinterleave(J2kDeinterleaveToF32Job { + pixels, + num_pixels, + num_components: 3, + bit_depth: 8, + signed: false, + }) + .map_err(str::to_string)?; + if components.is_none() { + return Err("Metal deinterleave stage did not dispatch".to_string()); + } + Ok(accelerator.dispatch_report()) +} + +fn probe_forward_rct_stage(planes: &[Vec]) -> Result { + let mut stage_planes = planes.to_vec(); + let mut accelerator = MetalEncodeStageAccelerator::default(); + let (plane0, plane1, plane2) = split_three_planes(&mut stage_planes); + let dispatched = accelerator + .encode_forward_rct(J2kForwardRctJob { + plane0, + plane1, + plane2, + }) + .map_err(str::to_string)?; + if !dispatched { + return Err("Metal forward RCT stage did not dispatch".to_string()); + } + Ok(accelerator.dispatch_report()) +} + +fn probe_forward_ict_stage(planes: &[Vec]) -> Result { + let mut stage_planes = planes.to_vec(); + let mut accelerator = MetalEncodeStageAccelerator::default(); + let (plane0, plane1, plane2) = split_three_planes(&mut stage_planes); + let dispatched = accelerator + .encode_forward_ict(J2kForwardIctJob { + plane0, + plane1, + plane2, + }) + .map_err(str::to_string)?; + if !dispatched { + return Err("Metal forward ICT stage did not dispatch".to_string()); + } + Ok(accelerator.dispatch_report()) +} + +fn probe_forward_dwt53_stage(samples: &[f32], dim: u32) -> Result { + let mut accelerator = MetalEncodeStageAccelerator::default(); + let output = accelerator + .encode_forward_dwt53(J2kForwardDwt53Job { + samples, + width: dim, + height: dim, + num_levels: 1, + }) + .map_err(str::to_string)?; + if output.is_none() { + return Err("Metal forward DWT 5/3 stage did not dispatch".to_string()); + } + Ok(accelerator.dispatch_report()) +} + +fn probe_forward_dwt97_stage(samples: &[f32], dim: u32) -> Result { + let mut accelerator = MetalEncodeStageAccelerator::default(); + let output = accelerator + .encode_forward_dwt97(J2kForwardDwt97Job { + samples, + width: dim, + height: dim, + num_levels: 1, + }) + .map_err(str::to_string)?; + if output.is_none() { + return Err("Metal forward DWT 9/7 stage did not dispatch".to_string()); + } + Ok(accelerator.dispatch_report()) +} + +fn probe_quantize_subband_stage(coefficients: &[f32]) -> Result { + let mut accelerator = MetalEncodeStageAccelerator::default(); + let quantized = accelerator + .encode_quantize_subband(J2kQuantizeSubbandJob { + coefficients, + step_exponent: 8, + step_mantissa: 256, + range_bits: 8, + reversible: false, + }) + .map_err(str::to_string)?; + if quantized.is_none() { + return Err("Metal quantize_subband stage did not dispatch".to_string()); + } + Ok(accelerator.dispatch_report()) +} + +fn probe_lossless_auto( + pixels: &[u8], + dim: u32, + codec: Codec, + components: Components, +) -> Result { + let samples = lossless_samples(pixels, dim, components); + let options = lossless_options(codec, EncodeBackendPreference::Auto); + let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); + encode_j2k_lossless_with_accelerator(samples, &options, BackendKind::Metal, &mut accelerator) + .map(|encoded: EncodedJ2k| encoded.dispatch_report) + .map_err(|error| error.to_string()) +} + +fn probe_lossy_auto( + pixels: &[u8], + dim: u32, + codec: Codec, + components: Components, +) -> Result { + let samples = lossy_samples(pixels, dim, components); + let options = lossy_options(codec, components, EncodeBackendPreference::Auto); + let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); + encode_j2k_lossy_with_accelerator(samples, &options, BackendKind::Metal, &mut accelerator) + .map(|encoded: EncodedLossyJ2k| encoded.dispatch_report) + .map_err(|error| error.to_string()) +} + +fn should_bench_auto( + probe: &Result, + expected_dispatch: bool, +) -> bool { + match probe { + Ok(dispatch) if !expected_dispatch || *dispatch != J2kEncodeDispatchReport::default() => { + true + } + Ok(_) if std::env::var_os("J2K_REQUIRE_METAL_BENCH").is_some() => { + panic!("J2K_REQUIRE_METAL_BENCH is set but Auto Metal encode did not dispatch") + } + Ok(_) => { + eprintln!("skipping Auto Metal encode bench: route did not dispatch"); + false + } + Err(error) if std::env::var_os("J2K_REQUIRE_METAL_BENCH").is_some() => { + panic!("J2K_REQUIRE_METAL_BENCH is set but Auto Metal encode failed: {error}") + } + Err(error) => { + eprintln!("skipping Auto Metal encode bench: {error}"); + false + } + } +} + +fn emit_stage_timing( + stage: &str, + dim: u32, + cpu: Duration, + metal: Result<(Duration, J2kEncodeDispatchReport), String>, +) { + match metal { + Ok((metal, dispatch)) => println!( + "j2k_metal_encode_stage_bench stage={stage} size={}x{} cpu_ms={:.3} metal_ms={:.3} dispatch={dispatch:?}", + dim, + dim, + cpu.as_secs_f64() * 1_000.0, + metal.as_secs_f64() * 1_000.0, + ), + Err(error) if std::env::var_os("J2K_REQUIRE_METAL_BENCH").is_some() => { + panic!("J2K_REQUIRE_METAL_BENCH is set but Metal stage bench failed: {error}"); + } + Err(error) => println!( + "j2k_metal_encode_stage_bench stage={stage} size={}x{} cpu_ms={:.3} metal_ms=skipped error={error}", + dim, + dim, + cpu.as_secs_f64() * 1_000.0, + ), + } +} + +fn emit_probe( + mode: &str, + codec: Codec, + components: Components, + dim: u32, + probe: &Result, +) { + match probe { + Ok(dispatch) => println!( + "j2k_metal_encode_auto_probe mode={mode} codec={} components={} size={}x{} dispatch={dispatch:?}", + codec.label(), + components.label(), + dim, + dim + ), + Err(error) => println!( + "j2k_metal_encode_auto_probe mode={mode} codec={} components={} size={}x{} error={error}", + codec.label(), + components.label(), + dim, + dim + ), + } +} + +fn emit_timing( + mode: &str, + codec: Codec, + components: Components, + dim: u32, + cpu: Duration, + auto: Option, +) { + let auto_ms = auto.map_or_else( + || "skipped".to_string(), + |duration| format!("{:.3}", duration.as_secs_f64() * 1_000.0), + ); + println!( + "j2k_metal_encode_auto_bench mode={mode} codec={} components={} size={}x{} cpu_ms={:.3} auto_ms={auto_ms}", + codec.label(), + components.label(), + dim, + dim, + cpu.as_secs_f64() * 1_000.0 + ); +} diff --git a/crates/j2k-metal/tests/shader_integrity.rs b/crates/j2k-metal/tests/shader_integrity.rs new file mode 100644 index 00000000..e8378b97 --- /dev/null +++ b/crates/j2k-metal/tests/shader_integrity.rs @@ -0,0 +1,24 @@ +use j2k_test_support::unwired_metal_kernels; + +const COMPUTE_SOURCE: &str = include_str!("../src/compute.rs"); +const SHADER_SOURCES: &[&str] = &[ + COMPUTE_SOURCE, + include_str!("../src/classic.metal"), + include_str!("../src/encode_bitstream.metal"), + include_str!("../src/fdwt.metal"), + include_str!("../src/ht_cleanup.metal"), + include_str!("../src/idwt.metal"), + include_str!("../src/mct.metal"), + include_str!("../src/quantize.metal"), + include_str!("../src/store.metal"), +]; + +#[test] +fn metal_kernels_are_wired_to_host_pipelines() { + let unused = unwired_metal_kernels(SHADER_SOURCES.iter().copied(), COMPUTE_SOURCE); + + assert!( + unused.is_empty(), + "Metal kernels must be compiled by host pipeline setup or removed: {unused:?}" + ); +} diff --git a/crates/signinum-j2k-native/Cargo.toml b/crates/j2k-native/Cargo.toml similarity index 50% rename from crates/signinum-j2k-native/Cargo.toml rename to crates/j2k-native/Cargo.toml index d4609f2b..71875205 100644 --- a/crates/signinum-j2k-native/Cargo.toml +++ b/crates/j2k-native/Cargo.toml @@ -1,30 +1,35 @@ [package] -name = "signinum-j2k-native" -description = "Pure-Rust JPEG 2000 codec engine for signinum" -version = "0.4.2" +name = "j2k-native" +description = "Pure-Rust JPEG 2000 and HTJ2K codec engine for j2k" +version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true readme = "README.md" +[package.metadata.docs.rs] +all-features = true +targets = [] + [features] default = ["std", "simd", "parallel"] logging = ["dep:log"] parallel = ["dep:rayon", "std"] simd = ["fearless_simd", "std"] -std = ["signinum-profile/std"] +std = ["j2k-profile/std"] [lib] -name = "signinum_j2k_native" +name = "j2k_native" path = "src/lib.rs" [dependencies] -fearless_simd = { version = "0.3.0", optional = true } +fearless_simd = { version = "0.4", optional = true } libm = { version = "0.2", default-features = false } -log = { version = "0.4", optional = true } -rayon = { version = "1", optional = true } -signinum-profile = { path = "../signinum-profile", version = "=0.4.2", default-features = false } +log = { workspace = true, optional = true } +rayon = { workspace = true, optional = true } +j2k-profile = { path = "../j2k-profile", version = "=0.6.0", default-features = false } +j2k-types = { path = "../j2k-types", version = "=0.6.0" } [dev-dependencies] criterion = { workspace = true } @@ -33,6 +38,14 @@ criterion = { workspace = true } name = "tier1_bitplane" harness = false +[[bench]] +name = "htj2k_sigprop_phase" +harness = false + +[[bench]] +name = "direct_cpu" +harness = false + [lints.rust] unsafe_code = "forbid" unreachable_pub = "warn" diff --git a/crates/signinum-j2k-native/LICENSE-APACHE b/crates/j2k-native/LICENSE-APACHE similarity index 100% rename from crates/signinum-j2k-native/LICENSE-APACHE rename to crates/j2k-native/LICENSE-APACHE diff --git a/crates/signinum-j2k-native/LICENSE-MIT b/crates/j2k-native/LICENSE-MIT similarity index 100% rename from crates/signinum-j2k-native/LICENSE-MIT rename to crates/j2k-native/LICENSE-MIT diff --git a/crates/j2k-native/README.md b/crates/j2k-native/README.md new file mode 100644 index 00000000..3550a6b9 --- /dev/null +++ b/crates/j2k-native/README.md @@ -0,0 +1,9 @@ +# j2k-native + +Native JPEG 2000 / HTJ2K engine used by J2K APIs and adapters. + +This crate owns codestream parsing, native encode/decode helpers, packetization +support, HTJ2K table helpers, and header inspection helpers used by higher-level +crates. + +Most application code should use `j2k` instead. diff --git a/crates/signinum-j2k-native/assets/CGATS001Compat-v2-micro.icc b/crates/j2k-native/assets/CGATS001Compat-v2-micro.icc similarity index 100% rename from crates/signinum-j2k-native/assets/CGATS001Compat-v2-micro.icc rename to crates/j2k-native/assets/CGATS001Compat-v2-micro.icc diff --git a/crates/signinum-j2k-native/assets/LAB.icc b/crates/j2k-native/assets/LAB.icc similarity index 100% rename from crates/signinum-j2k-native/assets/LAB.icc rename to crates/j2k-native/assets/LAB.icc diff --git a/crates/signinum-j2k-native/assets/LICENSE.txt b/crates/j2k-native/assets/LICENSE.txt similarity index 100% rename from crates/signinum-j2k-native/assets/LICENSE.txt rename to crates/j2k-native/assets/LICENSE.txt diff --git a/crates/signinum-j2k-native/assets/ProPhoto-v2-micro.icc b/crates/j2k-native/assets/ProPhoto-v2-micro.icc similarity index 100% rename from crates/signinum-j2k-native/assets/ProPhoto-v2-micro.icc rename to crates/j2k-native/assets/ProPhoto-v2-micro.icc diff --git a/crates/signinum-j2k-native/assets/README.md b/crates/j2k-native/assets/README.md similarity index 100% rename from crates/signinum-j2k-native/assets/README.md rename to crates/j2k-native/assets/README.md diff --git a/crates/j2k-native/benches/direct_cpu.rs b/crates/j2k-native/benches/direct_cpu.rs new file mode 100644 index 00000000..d9eaa796 --- /dev/null +++ b/crates/j2k-native/benches/direct_cpu.rs @@ -0,0 +1,222 @@ +use criterion::{criterion_group, criterion_main, Criterion}; +use j2k_native::{ + encode_htj2k, execute_direct_color_plan_rgb8_into, execute_direct_color_plan_rgba8_into, + CpuDecodeParallelism, DecodeSettings, DecoderContext, EncodeOptions, Image, + J2kDirectCpuScratch, J2kRect, +}; + +const TILE_SIDE: u32 = 512; + +fn patterned_rgb8(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + pixels.push(((x * 7 + y * 11 + 17) & 0xff) as u8); + pixels.push(((x * 13 + y * 19 + 31) & 0xff) as u8); + } + } + pixels +} + +fn patterned_gray8(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 17 + y * 31) & 0xff) as u8); + } + } + pixels +} + +fn htj2k_gray_codestream(width: u32, height: u32, reversible: bool) -> Vec { + let pixels = patterned_gray8(width, height); + let options = EncodeOptions { + reversible, + guard_bits: if reversible { 1 } else { 2 }, + num_decomposition_levels: 5, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, width, height, 1, 8, false, &options).expect("encode HTJ2K gray") +} + +fn htj2k_rgb_codestream(width: u32, height: u32) -> Vec { + let pixels = patterned_rgb8(width, height); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, width, height, 3, 8, false, &options).expect("encode HTJ2K RGB") +} + +fn htj2k_rgb97_codestream(width: u32, height: u32) -> Vec { + let pixels = patterned_rgb8(width, height); + let options = EncodeOptions { + reversible: false, + guard_bits: 2, + num_decomposition_levels: 5, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, width, height, 3, 8, false, &options).expect("encode HTJ2K 9/7 RGB") +} + +fn direct_roi_plan(bytes: &[u8]) -> (j2k_native::J2kDirectColorPlan, J2kRect) { + let image = Image::new( + bytes, + &DecodeSettings { + target_resolution: Some((TILE_SIDE / 4, TILE_SIDE / 4)), + ..DecodeSettings::default() + }, + ) + .expect("scaled HTJ2K image"); + let output_region = J2kRect { + x0: 32, + y0: 32, + x1: 96, + y1: 96, + }; + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_region_with_context( + &mut context, + ( + output_region.x0, + output_region.y0, + output_region.width(), + output_region.height(), + ), + ) + .expect("direct RGB region plan"); + (plan, output_region) +} + +fn bench_direct_color_plan_97(c: &mut Criterion) { + let codestream = htj2k_rgb97_codestream(TILE_SIDE, TILE_SIDE); + let (plan, output_region) = direct_roi_plan(&codestream); + let rgb_stride = output_region.width() as usize * 3; + let rgb_len = rgb_stride * output_region.height() as usize; + + let mut group = c.benchmark_group("j2k_native_direct_cpu_color_plan"); + group.bench_function("htj2k_rgb8_97_roi256_q4_reuse_scratch", |b| { + let mut scratch = J2kDirectCpuScratch::new(); + let mut out = vec![0_u8; rgb_len]; + b.iter(|| { + execute_direct_color_plan_rgb8_into( + std::hint::black_box(&plan), + output_region, + &mut scratch, + &mut out, + rgb_stride, + ) + .expect("execute RGB direct 9/7 plan"); + std::hint::black_box(&out); + }); + }); + group.finish(); +} + +fn bench_full_decode(c: &mut Criterion) { + let gray_codestream = htj2k_gray_codestream(TILE_SIDE, TILE_SIDE, false); + let gray_image = + Image::new(&gray_codestream, &DecodeSettings::default()).expect("HTJ2K 9/7 gray image"); + let rgb_codestream = htj2k_rgb97_codestream(TILE_SIDE, TILE_SIDE); + let rgb_image = + Image::new(&rgb_codestream, &DecodeSettings::default()).expect("HTJ2K 9/7 RGB image"); + + let mut group = c.benchmark_group("j2k_native_full_decode"); + group.bench_function("htj2k_gray8_512x512_97_reuse_context", |b| { + let mut context = DecoderContext::default(); + b.iter(|| { + let decoded = gray_image + .decode_with_context(&mut context) + .expect("decode HTJ2K 9/7 gray"); + std::hint::black_box(decoded.data.len()); + }); + }); + group.bench_function("htj2k_rgb8_512x512_97_reuse_context", |b| { + let mut context = DecoderContext::default(); + b.iter(|| { + let decoded = rgb_image + .decode_with_context(&mut context) + .expect("decode HTJ2K 9/7 RGB"); + std::hint::black_box(decoded.data.len()); + }); + }); + group.bench_function("htj2k_rgb8_512x512_97_serial_context", |b| { + let mut context = DecoderContext::default(); + context.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + b.iter(|| { + let decoded = rgb_image + .decode_with_context(&mut context) + .expect("decode HTJ2K 9/7 RGB"); + std::hint::black_box(decoded.data.len()); + }); + }); + group.finish(); +} + +fn bench_direct_color_plan(c: &mut Criterion) { + let codestream = htj2k_rgb_codestream(TILE_SIDE, TILE_SIDE); + let (plan, output_region) = direct_roi_plan(&codestream); + let rgb_stride = output_region.width() as usize * 3; + let rgba_stride = output_region.width() as usize * 4; + let rgb_len = rgb_stride * output_region.height() as usize; + let rgba_len = rgba_stride * output_region.height() as usize; + + let mut group = c.benchmark_group("j2k_native_direct_cpu_color_plan"); + group.bench_function("htj2k_rgb8_roi256_q4_fresh_scratch", |b| { + b.iter(|| { + let mut scratch = J2kDirectCpuScratch::new(); + let mut out = vec![0_u8; rgb_len]; + execute_direct_color_plan_rgb8_into( + std::hint::black_box(&plan), + output_region, + &mut scratch, + &mut out, + rgb_stride, + ) + .expect("execute RGB direct plan"); + std::hint::black_box(out); + }); + }); + group.bench_function("htj2k_rgb8_roi256_q4_reuse_scratch", |b| { + let mut scratch = J2kDirectCpuScratch::new(); + let mut out = vec![0_u8; rgb_len]; + b.iter(|| { + execute_direct_color_plan_rgb8_into( + std::hint::black_box(&plan), + output_region, + &mut scratch, + &mut out, + rgb_stride, + ) + .expect("execute RGB direct plan"); + std::hint::black_box(&out); + }); + }); + group.bench_function("htj2k_rgba8_roi256_q4_reuse_scratch", |b| { + let mut scratch = J2kDirectCpuScratch::new(); + let mut out = vec![0_u8; rgba_len]; + b.iter(|| { + execute_direct_color_plan_rgba8_into( + std::hint::black_box(&plan), + output_region, + &mut scratch, + &mut out, + rgba_stride, + ) + .expect("execute RGBA direct plan"); + std::hint::black_box(&out); + }); + }); + group.finish(); +} + +criterion_group!( + benches, + bench_full_decode, + bench_direct_color_plan, + bench_direct_color_plan_97 +); +criterion_main!(benches); diff --git a/crates/j2k-native/benches/htj2k_sigprop_phase.rs b/crates/j2k-native/benches/htj2k_sigprop_phase.rs new file mode 100644 index 00000000..5e9a2031 --- /dev/null +++ b/crates/j2k-native/benches/htj2k_sigprop_phase.rs @@ -0,0 +1,178 @@ +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use j2k_native::{ + decode_ht_code_block_scalar, decode_ht_sigprop_benchmark_state, + prepare_ht_sigprop_benchmark_state, DecodeSettings, DecoderContext, HtCodeBlockDecodeJob, + HtCodeBlockDecoder, Image, Result, +}; + +const HTJ2K_REFINEMENT_FIXTURE: &[u8] = + include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k"); + +#[derive(Clone)] +struct OwnedHtCodeBlockDecodeJob { + data: Vec, + cleanup_length: u32, + refinement_length: u32, + width: u32, + height: u32, + missing_bit_planes: u8, + number_of_coding_passes: u8, + num_bitplanes: u8, + roi_shift: u8, + stripe_causal: bool, + strict: bool, + dequantization_step: f32, +} + +impl OwnedHtCodeBlockDecodeJob { + fn from_job(job: HtCodeBlockDecodeJob<'_>) -> Self { + Self { + data: job.data.to_vec(), + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + width: job.width, + height: job.height, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + num_bitplanes: job.num_bitplanes, + roi_shift: job.roi_shift, + stripe_causal: job.stripe_causal, + strict: job.strict, + dequantization_step: job.dequantization_step, + } + } + + fn borrowed_with_passes( + &self, + number_of_coding_passes: u8, + refinement_length: u32, + ) -> HtCodeBlockDecodeJob<'_> { + HtCodeBlockDecodeJob { + data: &self.data, + cleanup_length: self.cleanup_length, + refinement_length, + width: self.width, + height: self.height, + output_stride: self.width as usize, + missing_bit_planes: self.missing_bit_planes, + number_of_coding_passes, + num_bitplanes: self.num_bitplanes, + roi_shift: self.roi_shift, + stripe_causal: self.stripe_causal, + strict: self.strict, + dequantization_step: self.dequantization_step, + } + } +} + +fn decode_scalar_ht_batch( + jobs: &[OwnedHtCodeBlockDecodeJob], + outputs: &mut [Vec], +) -> Result<()> { + for (job, output) in jobs.iter().zip(outputs.iter_mut()) { + decode_ht_code_block_scalar( + job.borrowed_with_passes(job.number_of_coding_passes, job.refinement_length), + output, + )?; + } + Ok(()) +} + +#[derive(Default)] +struct CollectingHtDecoder { + jobs: Vec, +} + +impl HtCodeBlockDecoder for CollectingHtDecoder { + fn decode_code_block( + &mut self, + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + ) -> Result<()> { + if job.refinement_length > 0 { + self.jobs.push(OwnedHtCodeBlockDecodeJob::from_job(job)); + } + + decode_ht_code_block_scalar(job, output) + } +} + +fn collect_refinement_fixture_jobs() -> Vec { + let image = + Image::new(HTJ2K_REFINEMENT_FIXTURE, &DecodeSettings::default()).expect("fixture image"); + let mut context = DecoderContext::default(); + let mut decoder = CollectingHtDecoder::default(); + image + .decode_components_with_ht_decoder(&mut context, &mut decoder) + .expect("decode fixture while collecting HTJ2K jobs"); + assert_eq!( + decoder.jobs.len(), + 14, + "expected every ds0_ht_09_b11 HT block to carry refinement data" + ); + decoder.jobs +} + +fn bench_htj2k_refinement_sigprop_phase(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_refinement_sigprop_phase"); + let jobs = collect_refinement_fixture_jobs(); + let job = jobs + .first() + .expect("fixture must contain a refinement HTJ2K block"); + let sigprop_job = job.borrowed_with_passes(2, job.refinement_length); + let mut state = + prepare_ht_sigprop_benchmark_state(sigprop_job).expect("prepare SigProp benchmark state"); + let mut output = vec![0u32; state.output_len()]; + + group.bench_function("ds0_ht_09_b11_sigprop_only", |b| { + b.iter(|| { + decode_ht_sigprop_benchmark_state(std::hint::black_box(&mut state), &mut output) + .expect("decode HTJ2K SigProp phase"); + std::hint::black_box(&output); + }); + }); + + group.finish(); +} + +fn bench_htj2k_cpuupload_decode_batch(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_cpuupload_decode_batch"); + let fixture_jobs = collect_refinement_fixture_jobs(); + + for count in [1_usize, 2, 4, fixture_jobs.len()] { + let jobs = fixture_jobs + .iter() + .cloned() + .cycle() + .take(count) + .collect::>(); + let mut outputs = jobs + .iter() + .map(|job| vec![0.0_f32; job.width as usize * job.height as usize]) + .collect::>(); + + group.bench_with_input( + BenchmarkId::new("ds0_ht_09_b11_scalar_batch", count), + &count, + |b, _| { + b.iter(|| { + decode_scalar_ht_batch( + std::hint::black_box(&jobs), + std::hint::black_box(&mut outputs), + ) + .expect("decode HTJ2K scalar batch"); + std::hint::black_box(&outputs); + }); + }, + ); + } + + group.finish(); +} + +criterion_group!( + benches, + bench_htj2k_refinement_sigprop_phase, + bench_htj2k_cpuupload_decode_batch +); +criterion_main!(benches); diff --git a/crates/j2k-native/benches/tier1_bitplane.rs b/crates/j2k-native/benches/tier1_bitplane.rs new file mode 100644 index 00000000..b71c2286 --- /dev/null +++ b/crates/j2k-native/benches/tier1_bitplane.rs @@ -0,0 +1,719 @@ +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; +use j2k_native::{ + collect_ht_cleanup_encode_distribution, decode_ht_code_block_scalar, + decode_j2k_code_block_scalar, encode_ht_code_block_scalar, + encode_j2k_code_block_scalar_with_style, DecodeSettings, DecoderContext, HtCodeBlockDecodeJob, + HtCodeBlockDecoder, Image, J2kCodeBlockDecodeJob, J2kCodeBlockStyle, J2kSubBandType, Result, +}; +use rayon::prelude::*; +use rayon::ThreadPoolBuilder; + +const HTJ2K_REFINEMENT_FIXTURE: &[u8] = + include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k"); + +#[derive(Clone)] +struct OwnedHtCodeBlockDecodeJob { + data: Vec, + cleanup_length: u32, + refinement_length: u32, + width: u32, + height: u32, + missing_bit_planes: u8, + number_of_coding_passes: u8, + num_bitplanes: u8, + roi_shift: u8, + stripe_causal: bool, + strict: bool, + dequantization_step: f32, +} + +impl OwnedHtCodeBlockDecodeJob { + fn from_job(job: HtCodeBlockDecodeJob<'_>) -> Self { + Self { + data: job.data.to_vec(), + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + width: job.width, + height: job.height, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + num_bitplanes: job.num_bitplanes, + roi_shift: job.roi_shift, + stripe_causal: job.stripe_causal, + strict: job.strict, + dequantization_step: job.dequantization_step, + } + } + + fn borrowed_with_passes( + &self, + number_of_coding_passes: u8, + refinement_length: u32, + ) -> HtCodeBlockDecodeJob<'_> { + HtCodeBlockDecodeJob { + data: &self.data, + cleanup_length: self.cleanup_length, + refinement_length, + width: self.width, + height: self.height, + output_stride: self.width as usize, + missing_bit_planes: self.missing_bit_planes, + number_of_coding_passes, + num_bitplanes: self.num_bitplanes, + roi_shift: self.roi_shift, + stripe_causal: self.stripe_causal, + strict: self.strict, + dequantization_step: self.dequantization_step, + } + } + + fn output_len(&self) -> usize { + self.width as usize * self.height as usize + } +} + +#[derive(Default)] +struct CollectingHtDecoder { + jobs: Vec, +} + +impl HtCodeBlockDecoder for CollectingHtDecoder { + fn decode_code_block( + &mut self, + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + ) -> Result<()> { + if job.refinement_length > 0 { + self.jobs.push(OwnedHtCodeBlockDecodeJob::from_job(job)); + } + + decode_ht_code_block_scalar(job, output) + } +} + +fn collect_refinement_fixture_jobs() -> Vec { + let image = + Image::new(HTJ2K_REFINEMENT_FIXTURE, &DecodeSettings::default()).expect("fixture image"); + let mut context = DecoderContext::default(); + let mut decoder = CollectingHtDecoder::default(); + image + .decode_components_with_ht_decoder(&mut context, &mut decoder) + .expect("decode fixture while collecting HTJ2K jobs"); + assert_eq!( + decoder.jobs.len(), + 14, + "expected every ds0_ht_09_b11 HT block to carry refinement data" + ); + decoder.jobs +} + +fn generated_coefficients(width: u32, height: u32, seed: u32) -> Vec { + let mut state = seed ^ 0xa24b_aed4; + let mut coefficients = Vec::with_capacity(width as usize * height as usize); + for idx in 0..width * height { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + let value = ((state >> 16) & 0x01ff) as i32 - 255; + coefficients.push(if (idx + seed).is_multiple_of(13) { + 0 + } else { + value + }); + } + coefficients +} + +fn default_style() -> J2kCodeBlockStyle { + J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: false, + reset_context_probabilities: false, + termination_on_each_pass: false, + vertically_causal_context: false, + segmentation_symbols: false, + } +} + +fn generated_htj2k_encode_batch(blocks: usize, width: u32, height: u32) -> Vec> { + (0..blocks) + .map(|index| generated_coefficients(width, height, 0xA5A5_0000 ^ index as u32)) + .collect() +} + +fn encoded_ht_block_len( + coefficients: &[i32], + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + let encoded = encode_ht_code_block_scalar(coefficients, width, height, total_bitplanes) + .expect("encode HTJ2K code block"); + encoded.data.len() + + usize::from(encoded.num_coding_passes) + + usize::from(encoded.num_zero_bitplanes) +} + +fn encode_ht_batch_serial( + blocks: &[Vec], + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + blocks + .iter() + .map(|coefficients| encoded_ht_block_len(coefficients, width, height, total_bitplanes)) + .sum() +} + +fn encode_ht_batch_rayon( + blocks: &[Vec], + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + blocks + .par_iter() + .map(|coefficients| encoded_ht_block_len(coefficients, width, height, total_bitplanes)) + .sum() +} + +fn encode_ht_batch_rayon_chunks( + blocks: &[Vec], + chunk_size: usize, + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + blocks + .par_chunks(chunk_size) + .map(|chunk| encode_ht_batch_serial(chunk, width, height, total_bitplanes)) + .sum() +} + +fn encoded_j2k_block_len( + coefficients: &[i32], + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + let encoded = encode_j2k_code_block_scalar_with_style( + coefficients, + width, + height, + J2kSubBandType::LowLow, + total_bitplanes, + default_style(), + ) + .expect("encode J2K code block"); + encoded.data.len() + + encoded.segments.len() + + usize::from(encoded.number_of_coding_passes) + + usize::from(encoded.missing_bit_planes) +} + +fn encode_j2k_batch_serial( + blocks: &[Vec], + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + blocks + .iter() + .map(|coefficients| encoded_j2k_block_len(coefficients, width, height, total_bitplanes)) + .sum() +} + +fn encode_j2k_batch_rayon( + blocks: &[Vec], + width: u32, + height: u32, + total_bitplanes: u8, +) -> usize { + blocks + .par_iter() + .map(|coefficients| encoded_j2k_block_len(coefficients, width, height, total_bitplanes)) + .sum() +} + +fn benchmark_thread_counts() -> Vec { + let available = std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(1); + let mut counts = vec![1, available.min(2), available.min(4), available]; + counts.sort_unstable(); + counts.dedup(); + counts +} + +fn bench_tier1_decode(c: &mut Criterion) { + let mut group = c.benchmark_group("tier1_bitplane_decode"); + let cases = [ + ("default", default_style(), J2kSubBandType::LowLow), + ( + "bypass", + J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: true, + ..default_style() + }, + J2kSubBandType::HighLow, + ), + ( + "segmented", + J2kCodeBlockStyle { + termination_on_each_pass: true, + reset_context_probabilities: true, + segmentation_symbols: true, + ..default_style() + }, + J2kSubBandType::HighHigh, + ), + ( + "vertically_causal", + J2kCodeBlockStyle { + vertically_causal_context: true, + ..default_style() + }, + J2kSubBandType::LowHigh, + ), + ]; + + for (name, style, sub_band_type) in cases { + let width = 64; + let height = 64; + let total_bitplanes = 10; + let coefficients = generated_coefficients(width, height, 0x5151_0000); + let encoded = encode_j2k_code_block_scalar_with_style( + &coefficients, + width, + height, + sub_band_type, + total_bitplanes, + style, + ) + .expect("encode code block"); + let job = J2kCodeBlockDecodeJob { + data: &encoded.data, + segments: &encoded.segments, + width, + height, + output_stride: width as usize, + missing_bit_planes: encoded.missing_bit_planes, + number_of_coding_passes: encoded.number_of_coding_passes, + total_bitplanes, + roi_shift: 0, + sub_band_type, + style, + strict: true, + dequantization_step: 1.0, + }; + let mut output = vec![0.0; width as usize * height as usize]; + + group.bench_with_input(BenchmarkId::new("decode_64x64", name), &job, |b, job| { + b.iter(|| { + decode_j2k_code_block_scalar(*job, &mut output).expect("decode code block"); + std::hint::black_box(&output); + }); + }); + } + + group.finish(); +} + +fn bench_tier1_encode(c: &mut Criterion) { + let mut group = c.benchmark_group("tier1_bitplane_encode"); + let cases = [ + ("default", default_style(), J2kSubBandType::LowLow), + ( + "bypass", + J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: true, + ..default_style() + }, + J2kSubBandType::HighLow, + ), + ( + "segmented", + J2kCodeBlockStyle { + termination_on_each_pass: true, + reset_context_probabilities: true, + segmentation_symbols: true, + ..default_style() + }, + J2kSubBandType::HighHigh, + ), + ( + "vertically_causal", + J2kCodeBlockStyle { + vertically_causal_context: true, + ..default_style() + }, + J2kSubBandType::LowHigh, + ), + ]; + + for (name, style, sub_band_type) in cases { + let width = 64; + let height = 64; + let total_bitplanes = 10; + let coefficients = generated_coefficients(width, height, 0x7171_0000); + + group.bench_with_input( + BenchmarkId::new("encode_64x64", name), + &coefficients, + |b, coefficients| { + b.iter(|| { + let encoded = encode_j2k_code_block_scalar_with_style( + std::hint::black_box(coefficients), + width, + height, + sub_band_type, + total_bitplanes, + style, + ) + .expect("encode code block"); + std::hint::black_box(encoded); + }); + }, + ); + } + + group.finish(); +} + +fn bench_htj2k_cleanup_decode(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_cleanup_decode"); + for &seed in &[0x9191_0000, 0x9191_0001] { + let width = 64; + let height = 64; + let total_bitplanes = 10; + let coefficients = generated_coefficients(width, height, seed); + let encoded = encode_ht_code_block_scalar(&coefficients, width, height, total_bitplanes) + .expect("encode HTJ2K code block"); + let job = HtCodeBlockDecodeJob { + data: &encoded.data, + cleanup_length: encoded.data.len() as u32, + refinement_length: 0, + width, + height, + output_stride: width as usize, + missing_bit_planes: encoded.num_zero_bitplanes, + number_of_coding_passes: encoded.num_coding_passes, + num_bitplanes: total_bitplanes, + roi_shift: 0, + stripe_causal: false, + strict: true, + dequantization_step: 1.0, + }; + let mut output = vec![0.0; width as usize * height as usize]; + + group.bench_with_input(BenchmarkId::new("decode_64x64", seed), &job, |b, job| { + b.iter(|| { + decode_ht_code_block_scalar(*job, &mut output).expect("decode HTJ2K code block"); + std::hint::black_box(&output); + }); + }); + } + group.finish(); +} + +fn bench_htj2k_refinement_fixture_decode(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_refinement_fixture_decode"); + let image = + Image::new(HTJ2K_REFINEMENT_FIXTURE, &DecodeSettings::default()).expect("fixture image"); + + group.bench_function("ds0_ht_09_b11_full", |b| { + b.iter(|| { + let mut context = DecoderContext::default(); + let components = image + .decode_components_with_context(&mut context) + .expect("decode HTJ2K refinement fixture"); + std::hint::black_box(components.planes()[0].samples()); + }); + }); + + group.finish(); +} + +fn bench_htj2k_refinement_block_decode(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_refinement_block_decode"); + let jobs = collect_refinement_fixture_jobs(); + let job = jobs + .first() + .expect("fixture must contain a refinement HTJ2K block"); + let cleanup_job = job.borrowed_with_passes(1, 0); + let sigprop_job = job.borrowed_with_passes(2, job.refinement_length); + let magref_job = job.borrowed_with_passes(job.number_of_coding_passes, job.refinement_length); + + let mut cleanup_output = vec![0.0; job.output_len()]; + group.bench_function("ds0_ht_09_b11_cleanup", |b| { + b.iter(|| { + cleanup_output.fill(0.0); + decode_ht_code_block_scalar(std::hint::black_box(cleanup_job), &mut cleanup_output) + .expect("cleanup-limited HTJ2K decode"); + std::hint::black_box(&cleanup_output); + }); + }); + + let mut sigprop_output = vec![0.0; job.output_len()]; + group.bench_function("ds0_ht_09_b11_sigprop", |b| { + b.iter(|| { + sigprop_output.fill(0.0); + decode_ht_code_block_scalar(std::hint::black_box(sigprop_job), &mut sigprop_output) + .expect("significance-propagation-limited HTJ2K decode"); + std::hint::black_box(&sigprop_output); + }); + }); + + let mut magref_output = vec![0.0; job.output_len()]; + group.bench_function("ds0_ht_09_b11_magref_full", |b| { + b.iter(|| { + magref_output.fill(0.0); + decode_ht_code_block_scalar(std::hint::black_box(magref_job), &mut magref_output) + .expect("full HTJ2K decode through magnitude refinement"); + std::hint::black_box(&magref_output); + }); + }); + + group.finish(); +} + +fn bench_htj2k_cleanup_encode(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_cleanup_encode"); + for &seed in &[0x9292_0000, 0x9292_0001] { + let width = 64; + let height = 64; + let total_bitplanes = 10; + let coefficients = generated_coefficients(width, height, seed); + + group.bench_with_input( + BenchmarkId::new("encode_64x64", seed), + &coefficients, + |b, coefficients| { + b.iter(|| { + let encoded = encode_ht_code_block_scalar( + std::hint::black_box(coefficients), + width, + height, + total_bitplanes, + ) + .expect("encode HTJ2K code block"); + std::hint::black_box(encoded); + }); + }, + ); + } + group.finish(); +} + +fn bench_htj2k_cleanup_encode_distribution(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_cleanup_encode_distribution"); + for &seed in &[0x9292_0000, 0x9292_0001] { + let width = 64; + let height = 64; + let total_bitplanes = 10; + let coefficients = generated_coefficients(width, height, seed); + + group.bench_with_input( + BenchmarkId::new("rho_eq_uq_64x64", seed), + &coefficients, + |b, coefficients| { + b.iter(|| { + let distribution = collect_ht_cleanup_encode_distribution( + std::hint::black_box(coefficients), + width, + height, + total_bitplanes, + ) + .expect("collect HTJ2K cleanup encode distribution"); + std::hint::black_box(distribution); + }); + }, + ); + } + group.finish(); +} + +fn bench_htj2k_cleanup_encode_parallel_granularity(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_cleanup_encode_parallel_granularity"); + let width = 64; + let height = 64; + let total_bitplanes = 10; + let blocks = generated_htj2k_encode_batch(128, width, height); + + group.bench_function("serial_128_blocks", |b| { + b.iter(|| { + let total_len = encode_ht_batch_serial( + std::hint::black_box(&blocks), + width, + height, + total_bitplanes, + ); + std::hint::black_box(total_len); + }); + }); + + group.bench_function("rayon_par_iter_global_128_blocks", |b| { + b.iter(|| { + let total_len = encode_ht_batch_rayon( + std::hint::black_box(&blocks), + width, + height, + total_bitplanes, + ); + std::hint::black_box(total_len); + }); + }); + + for threads in benchmark_thread_counts() { + let pool = ThreadPoolBuilder::new() + .num_threads(threads) + .build() + .expect("build benchmark Rayon pool"); + group.bench_with_input( + BenchmarkId::new("rayon_par_iter_threads", threads), + &threads, + |b, _| { + b.iter(|| { + let total_len = pool.install(|| { + encode_ht_batch_rayon( + std::hint::black_box(&blocks), + width, + height, + total_bitplanes, + ) + }); + std::hint::black_box(total_len); + }); + }, + ); + } + + let available = std::thread::available_parallelism() + .map(std::num::NonZeroUsize::get) + .unwrap_or(1); + let chunk_pool = ThreadPoolBuilder::new() + .num_threads(available) + .build() + .expect("build benchmark Rayon chunk pool"); + for chunk_size in [4usize, 8, 16, 32, 64] { + group.bench_with_input( + BenchmarkId::new("rayon_par_chunks_128_blocks", chunk_size), + &chunk_size, + |b, &chunk_size| { + b.iter(|| { + let total_len = chunk_pool.install(|| { + encode_ht_batch_rayon_chunks( + std::hint::black_box(&blocks), + chunk_size, + width, + height, + total_bitplanes, + ) + }); + std::hint::black_box(total_len); + }); + }, + ); + } + + group.finish(); +} + +fn bench_htj2k_cleanup_encode_parallel_batch_size(c: &mut Criterion) { + let mut group = c.benchmark_group("htj2k_cleanup_encode_parallel_batch_size"); + let width = 64; + let height = 64; + let total_bitplanes = 10; + + for block_count in [1usize, 2, 4, 8, 16, 32, 128] { + let blocks = generated_htj2k_encode_batch(block_count, width, height); + group.bench_with_input( + BenchmarkId::new("serial_blocks", block_count), + &blocks, + |b, blocks| { + b.iter(|| { + let total_len = encode_ht_batch_serial( + std::hint::black_box(blocks), + width, + height, + total_bitplanes, + ); + std::hint::black_box(total_len); + }); + }, + ); + group.bench_with_input( + BenchmarkId::new("rayon_par_iter_global_blocks", block_count), + &blocks, + |b, blocks| { + b.iter(|| { + let total_len = encode_ht_batch_rayon( + std::hint::black_box(blocks), + width, + height, + total_bitplanes, + ); + std::hint::black_box(total_len); + }); + }, + ); + } + + group.finish(); +} + +fn bench_j2k_tier1_encode_parallel_batch_size(c: &mut Criterion) { + let mut group = c.benchmark_group("j2k_tier1_encode_parallel_batch_size"); + let width = 64; + let height = 64; + let total_bitplanes = 10; + + for block_count in [1usize, 2, 4, 8, 16, 32, 128] { + let blocks = generated_htj2k_encode_batch(block_count, width, height); + group.bench_with_input( + BenchmarkId::new("serial_blocks", block_count), + &blocks, + |b, blocks| { + b.iter(|| { + let total_len = encode_j2k_batch_serial( + std::hint::black_box(blocks), + width, + height, + total_bitplanes, + ); + std::hint::black_box(total_len); + }); + }, + ); + group.bench_with_input( + BenchmarkId::new("rayon_par_iter_global_blocks", block_count), + &blocks, + |b, blocks| { + b.iter(|| { + let total_len = encode_j2k_batch_rayon( + std::hint::black_box(blocks), + width, + height, + total_bitplanes, + ); + std::hint::black_box(total_len); + }); + }, + ); + } + + group.finish(); +} + +criterion_group!( + benches, + bench_tier1_decode, + bench_tier1_encode, + bench_htj2k_cleanup_decode, + bench_htj2k_refinement_fixture_decode, + bench_htj2k_refinement_block_decode, + bench_htj2k_cleanup_encode, + bench_htj2k_cleanup_encode_distribution, + bench_htj2k_cleanup_encode_parallel_granularity, + bench_htj2k_cleanup_encode_parallel_batch_size, + bench_j2k_tier1_encode_parallel_batch_size +); +criterion_main!(benches); diff --git a/crates/j2k-native/fixtures/htj2k/LICENSE.OpenHTJ2K b/crates/j2k-native/fixtures/htj2k/LICENSE.OpenHTJ2K new file mode 100644 index 00000000..d896ab36 --- /dev/null +++ b/crates/j2k-native/fixtures/htj2k/LICENSE.OpenHTJ2K @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2019 - 2021, Osamu Watanabe +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/crates/j2k-native/fixtures/htj2k/README.md b/crates/j2k-native/fixtures/htj2k/README.md new file mode 100644 index 00000000..1c5d7e6f --- /dev/null +++ b/crates/j2k-native/fixtures/htj2k/README.md @@ -0,0 +1,22 @@ +HTJ2K fixtures for native decoder coverage. + +These fixtures are from the OpenHTJ2K conformance data, copied from OpenHTJ2K +commit `ffe5acf9f1eedb87c36c3fd2134fdc1ddea5e75f`. They are tiny HTONLY +codestreams derived from the JPEG 2000 Part 4 / ITU-T T.803 HTJ2K conformance +set. + +`openhtj2k_ds0_ht_12_b11.j2k` is copied from `ds0_ht_12_b11.j2k`, blob +`cf3fb0bc7e55898b4e6977f38ba0d38d91c359bf`. The native decoder sees 8 HT code +blocks, 2 non-empty refinement jobs, and up to 3 HT coding passes. + +`openhtj2k_ds0_ht_09_b11.j2k` is copied from `ds0_ht_09_b11.j2k`, blob +`d4f2031359c32eb24825d00dde05a92cf3ae451e`. The native decoder sees 14 HT +code blocks, all with non-empty refinement jobs, and up to 3 HT coding passes. + +It is intentionally checked-in test data: tests must not invoke an external +encoder or decoder at runtime. + +The paired `.gray` file contains the expected 8-bit grayscale samples in +row-major order from decoding the checked-in codestream with OpenJPH. + +The OpenHTJ2K source license is retained in `LICENSE.OpenHTJ2K`. diff --git a/crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray b/crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray new file mode 100644 index 00000000..115938f1 Binary files /dev/null and b/crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray differ diff --git a/crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k b/crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k new file mode 100644 index 00000000..d4f20313 Binary files /dev/null and b/crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k differ diff --git a/crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_12_b11.gray b/crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_12_b11.gray new file mode 100644 index 00000000..63feb637 --- /dev/null +++ b/crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_12_b11.gray @@ -0,0 +1 @@ +7b2Y_-m.Q2o} \ No newline at end of file diff --git a/crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_12_b11.j2k b/crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_12_b11.j2k new file mode 100644 index 00000000..cf3fb0bc Binary files /dev/null and b/crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_12_b11.j2k differ diff --git a/crates/j2k-native/src/direct_cpu.rs b/crates/j2k-native/src/direct_cpu.rs new file mode 100644 index 00000000..3ed8a993 --- /dev/null +++ b/crates/j2k-native/src/direct_cpu.rs @@ -0,0 +1,719 @@ +use alloc::vec::Vec; + +use crate::error::{bail, DecodingError, Result}; +use crate::j2c::idwt; +use crate::math::{floor_f32, round_f32}; +use crate::{ + decode_ht_code_block_scalar, decode_j2k_code_block_scalar, HtCodeBlockDecodeJob, + HtOwnedSubBandPlan, J2kCodeBlockDecodeJob, J2kDirectBandId, J2kDirectColorPlan, + J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kDirectIdwtStep, J2kDirectStoreStep, + J2kIdwtBand, J2kOwnedSubBandPlan, J2kRect, J2kSingleDecompositionIdwtJob, J2kWaveletTransform, +}; + +/// Adapter reusable scratch for executing direct J2K RGB plans on the CPU. +#[derive(Debug, Default)] +pub struct J2kDirectCpuScratch { + component_band_sets: Vec, + component_planes: Vec, +} + +impl J2kDirectCpuScratch { + /// Create empty direct-plan CPU scratch. + #[must_use] + pub const fn new() -> Self { + Self { + component_band_sets: Vec::new(), + component_planes: Vec::new(), + } + } + + /// Release retained scratch allocations. + pub fn clear(&mut self) { + self.component_band_sets.clear(); + self.component_planes.clear(); + } + + fn prepare_component_scratch(&mut self, component_count: usize) { + while self.component_band_sets.len() < component_count { + self.component_band_sets + .push(DirectComponentBandScratch::default()); + } + while self.component_planes.len() < component_count { + self.component_planes.push(DirectComponentPlane::default()); + } + } + + #[cfg(test)] + fn allocation_profile_for_tests(&self) -> DirectScratchAllocationProfile { + let band_buffers = self + .component_band_sets + .iter() + .map(|component| component.bands.len()) + .sum(); + let band_sample_len = self + .component_band_sets + .iter() + .flat_map(|component| component.bands.iter()) + .map(|band| band.coefficients.len()) + .sum(); + let band_sample_capacity = self + .component_band_sets + .iter() + .flat_map(|component| component.bands.iter()) + .map(|band| band.coefficients.capacity()) + .sum(); + let component_sample_len = self + .component_planes + .iter() + .map(|plane| plane.samples.len()) + .sum(); + let component_sample_capacity = self + .component_planes + .iter() + .map(|plane| plane.samples.capacity()) + .sum(); + DirectScratchAllocationProfile { + component_band_sets: self.component_band_sets.len(), + component_planes: self.component_planes.len(), + band_buffers, + band_sample_len, + band_sample_capacity, + component_sample_len, + component_sample_capacity, + } + } +} + +#[cfg(test)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct DirectScratchAllocationProfile { + component_band_sets: usize, + component_planes: usize, + band_buffers: usize, + band_sample_len: usize, + band_sample_capacity: usize, + component_sample_len: usize, + component_sample_capacity: usize, +} + +#[derive(Debug, Default)] +struct DirectComponentBandScratch { + bands: Vec, + active_len: usize, +} + +impl DirectComponentBandScratch { + fn reset(&mut self) { + self.active_len = 0; + } + + fn active(&self) -> &[DirectCpuBand] { + &self.bands[..self.active_len] + } + + fn prepare_band(&mut self, band_id: J2kDirectBandId, rect: J2kRect, len: usize) -> usize { + let index = self.active_len; + if index == self.bands.len() { + self.bands.push(DirectCpuBand::empty()); + } + let band = &mut self.bands[index]; + band.band_id = band_id; + band.rect = rect; + resize_and_zero(&mut band.coefficients, len); + self.active_len += 1; + index + } +} + +#[derive(Debug)] +struct DirectCpuBand { + band_id: J2kDirectBandId, + rect: J2kRect, + coefficients: Vec, +} + +impl DirectCpuBand { + const fn empty() -> Self { + Self { + band_id: 0, + rect: J2kRect { + x0: 0, + y0: 0, + x1: 0, + y1: 0, + }, + coefficients: Vec::new(), + } + } +} + +#[derive(Debug, Default)] +struct DirectComponentPlane { + width: u32, + height: u32, + samples: Vec, +} + +/// Execute a adapter direct RGB plan on the CPU and write an RGB8 output region. +pub fn execute_direct_color_plan_rgb8_into( + plan: &J2kDirectColorPlan, + output_region: J2kRect, + scratch: &mut J2kDirectCpuScratch, + out: &mut [u8], + stride: usize, +) -> Result<()> { + execute_direct_color_plan_u8_into( + plan, + output_region, + scratch, + out, + stride, + DirectColorU8Output::Rgb8, + ) +} + +/// Execute a adapter direct RGB plan on the CPU and write an RGBA8 output region. +pub fn execute_direct_color_plan_rgba8_into( + plan: &J2kDirectColorPlan, + output_region: J2kRect, + scratch: &mut J2kDirectCpuScratch, + out: &mut [u8], + stride: usize, +) -> Result<()> { + execute_direct_color_plan_u8_into( + plan, + output_region, + scratch, + out, + stride, + DirectColorU8Output::Rgba8, + ) +} + +#[derive(Clone, Copy)] +enum DirectColorU8Output { + Rgb8, + Rgba8, +} + +impl DirectColorU8Output { + const fn bytes_per_pixel(self) -> usize { + match self { + Self::Rgb8 => 3, + Self::Rgba8 => 4, + } + } +} + +fn execute_direct_color_plan_u8_into( + plan: &J2kDirectColorPlan, + output_region: J2kRect, + scratch: &mut J2kDirectCpuScratch, + out: &mut [u8], + stride: usize, + output: DirectColorU8Output, +) -> Result<()> { + if plan.component_plans.len() != 3 { + bail!(DecodingError::UnsupportedFeature( + "direct CPU color plan requires three components" + )); + } + validate_output_region(plan, output_region, out.len(), stride, output)?; + + scratch.prepare_component_scratch(plan.component_plans.len()); + for (component_index, component_plan) in plan.component_plans.iter().enumerate() { + let band_scratch = &mut scratch.component_band_sets[component_index]; + let plane = &mut scratch.component_planes[component_index]; + execute_component_plan(component_plan, band_scratch, plane)?; + } + + let [plane0, plane1, plane2, ..] = scratch.component_planes.as_mut_slice() else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + if plan.mct { + apply_inverse_mct(plan.transform, plan.bit_depths, plane0, plane1, plane2)?; + } + write_rgb8_region( + [plane0, plane1, plane2], + plan.bit_depths, + output_region, + out, + stride, + output, + ) +} + +fn execute_component_plan( + plan: &J2kDirectGrayscalePlan, + bands: &mut DirectComponentBandScratch, + output: &mut DirectComponentPlane, +) -> Result<()> { + bands.reset(); + let mut output_written = false; + + for step in &plan.steps { + match step { + J2kDirectGrayscaleStep::ClassicSubBand(sub_band) => { + execute_classic_sub_band(sub_band, bands)?; + } + J2kDirectGrayscaleStep::HtSubBand(sub_band) => { + execute_ht_sub_band(sub_band, bands)?; + } + J2kDirectGrayscaleStep::Idwt(step) => { + execute_idwt_step(step, bands)?; + } + J2kDirectGrayscaleStep::Store(store) => { + store_component(store, bands.active(), output, &mut output_written)?; + } + } + } + + if output_written { + Ok(()) + } else { + Err(DecodingError::CodeBlockDecodeFailure.into()) + } +} + +fn execute_classic_sub_band( + plan: &J2kOwnedSubBandPlan, + bands: &mut DirectComponentBandScratch, +) -> Result<()> { + let required_len = checked_area(plan.width, plan.height)?; + let band_index = bands.prepare_band(plan.band_id, plan.rect, required_len); + let output = &mut bands.bands[band_index].coefficients; + let sub_band_width = + usize::try_from(plan.width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + + for job in &plan.jobs { + let base_idx = checked_block_base(job.output_x, job.output_y, sub_band_width)?; + let block_len = checked_block_output_len(job.output_stride, job.width, job.height)?; + let end_idx = base_idx + .checked_add(block_len) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if end_idx > output.len() + || job + .output_x + .checked_add(job.width) + .is_none_or(|x| x > plan.width) + || job + .output_y + .checked_add(job.height) + .is_none_or(|y| y > plan.height) + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + let code_block = J2kCodeBlockDecodeJob { + data: &job.data, + segments: &job.segments, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + total_bitplanes: job.total_bitplanes, + roi_shift: job.roi_shift, + sub_band_type: job.sub_band_type, + style: job.style, + strict: job.strict, + dequantization_step: job.dequantization_step, + }; + decode_j2k_code_block_scalar(code_block, &mut output[base_idx..end_idx])?; + } + Ok(()) +} + +fn execute_ht_sub_band( + plan: &HtOwnedSubBandPlan, + bands: &mut DirectComponentBandScratch, +) -> Result<()> { + let required_len = checked_area(plan.width, plan.height)?; + let band_index = bands.prepare_band(plan.band_id, plan.rect, required_len); + let output = &mut bands.bands[band_index].coefficients; + let sub_band_width = + usize::try_from(plan.width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + + for job in &plan.jobs { + let base_idx = checked_block_base(job.output_x, job.output_y, sub_band_width)?; + let block_len = checked_block_output_len(job.output_stride, job.width, job.height)?; + let end_idx = base_idx + .checked_add(block_len) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if end_idx > output.len() + || job + .output_x + .checked_add(job.width) + .is_none_or(|x| x > plan.width) + || job + .output_y + .checked_add(job.height) + .is_none_or(|y| y > plan.height) + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + let code_block = HtCodeBlockDecodeJob { + data: &job.data, + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + num_bitplanes: job.num_bitplanes, + roi_shift: job.roi_shift, + stripe_causal: job.stripe_causal, + strict: job.strict, + dequantization_step: job.dequantization_step, + }; + decode_ht_code_block_scalar(code_block, &mut output[base_idx..end_idx])?; + } + Ok(()) +} + +fn execute_idwt_step( + step: &J2kDirectIdwtStep, + bands: &mut DirectComponentBandScratch, +) -> Result<()> { + let output_index = bands.prepare_band(step.output_band_id, step.rect, 0); + let (input_bands, output_bands) = bands.bands.split_at_mut(output_index); + let output = &mut output_bands[0].coefficients; + let ll = find_idwt_band(input_bands, step.ll_band_id)?; + let hl = find_idwt_band(input_bands, step.hl_band_id)?; + let lh = find_idwt_band(input_bands, step.lh_band_id)?; + let hh = find_idwt_band(input_bands, step.hh_band_id)?; + let job = J2kSingleDecompositionIdwtJob { + rect: step.rect, + transform: step.transform, + ll, + hl, + lh, + hh, + }; + idwt::apply_single_decomposition_idwt_job(job, output) +} + +fn find_idwt_band(bands: &[DirectCpuBand], band_id: J2kDirectBandId) -> Result> { + let band = find_band(bands, band_id)?; + Ok(J2kIdwtBand { + rect: band.rect, + coefficients: &band.coefficients, + }) +} + +fn store_component( + store: &J2kDirectStoreStep, + bands: &[DirectCpuBand], + plane: &mut DirectComponentPlane, + output_written: &mut bool, +) -> Result<()> { + let input = find_band(bands, store.input_band_id)?; + if !*output_written { + plane.width = store.output_width; + plane.height = store.output_height; + let required_len = checked_area(store.output_width, store.output_height)?; + resize_and_zero(&mut plane.samples, required_len); + *output_written = true; + } + if plane.width != store.output_width + || plane.height != store.output_height + || plane.samples.len() != checked_area(store.output_width, store.output_height)? + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + validate_store_bounds(store, input, plane)?; + let input_width = input.rect.width() as usize; + let output_width = plane.width as usize; + let copy_width = store.copy_width as usize; + for row in 0..store.copy_height as usize { + let src_start = (store.source_y as usize + row) + .checked_mul(input_width) + .and_then(|base| base.checked_add(store.source_x as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let dst_start = (store.output_y as usize + row) + .checked_mul(output_width) + .and_then(|base| base.checked_add(store.output_x as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let src = &input.coefficients[src_start..src_start + copy_width]; + let dst = &mut plane.samples[dst_start..dst_start + copy_width]; + for (src, dst) in src.iter().zip(dst.iter_mut()) { + *dst = *src + store.addend; + } + } + Ok(()) +} + +fn find_band(bands: &[DirectCpuBand], band_id: J2kDirectBandId) -> Result<&DirectCpuBand> { + bands + .iter() + .find(|band| band.band_id == band_id) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) +} + +fn validate_store_bounds( + store: &J2kDirectStoreStep, + input: &DirectCpuBand, + output: &DirectComponentPlane, +) -> Result<()> { + if store + .source_x + .checked_add(store.copy_width) + .is_none_or(|x| x > input.rect.width()) + || store + .source_y + .checked_add(store.copy_height) + .is_none_or(|y| y > input.rect.height()) + || store + .output_x + .checked_add(store.copy_width) + .is_none_or(|x| x > output.width) + || store + .output_y + .checked_add(store.copy_height) + .is_none_or(|y| y > output.height) + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(()) +} + +fn apply_inverse_mct( + transform: J2kWaveletTransform, + bit_depths: [u8; 3], + plane0: &mut DirectComponentPlane, + plane1: &mut DirectComponentPlane, + plane2: &mut DirectComponentPlane, +) -> Result<()> { + if plane0.width != plane1.width + || plane1.width != plane2.width + || plane0.height != plane1.height + || plane1.height != plane2.height + || plane0.samples.len() != plane1.samples.len() + || plane1.samples.len() != plane2.samples.len() + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + let addend0 = sign_addend(bit_depths[0]); + let addend1 = sign_addend(bit_depths[1]); + let addend2 = sign_addend(bit_depths[2]); + for ((y0, y1), y2) in plane0 + .samples + .iter_mut() + .zip(plane1.samples.iter_mut()) + .zip(plane2.samples.iter_mut()) + { + let src0 = *y0; + let src1 = *y1; + let src2 = *y2; + let (out0, out1, out2) = match transform { + J2kWaveletTransform::Irreversible97 => ( + src0 + 1.402 * src2, + src0 - 0.34413 * src1 - 0.71414 * src2, + src0 + 1.772 * src1, + ), + J2kWaveletTransform::Reversible53 => { + let i1 = src0 - floor_f32((src2 + src1) * 0.25); + (src2 + i1, i1, src1 + i1) + } + }; + *y0 = out0 + addend0; + *y1 = out1 + addend1; + *y2 = out2 + addend2; + } + Ok(()) +} + +fn write_rgb8_region( + planes: [&DirectComponentPlane; 3], + bit_depths: [u8; 3], + output_region: J2kRect, + out: &mut [u8], + stride: usize, + output: DirectColorU8Output, +) -> Result<()> { + let width = output_region.width() as usize; + let height = output_region.height() as usize; + let bytes_per_pixel = output.bytes_per_pixel(); + let row_bytes = width + .checked_mul(bytes_per_pixel) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + for plane in planes { + if output_region.x1 > plane.width || output_region.y1 > plane.height { + bail!(DecodingError::CodeBlockDecodeFailure); + } + } + + for y in 0..height { + let src_y = output_region.y0 as usize + y; + let dst = &mut out[y * stride..y * stride + row_bytes]; + for x in 0..width { + let src_x = output_region.x0 as usize + x; + let dst = &mut dst[x * bytes_per_pixel..x * bytes_per_pixel + bytes_per_pixel]; + for channel in 0..3 { + let plane = planes[channel]; + let sample = plane.samples[src_y * plane.width as usize + src_x]; + dst[channel] = sample_as_u8(sample, bit_depths[channel]); + } + if matches!(output, DirectColorU8Output::Rgba8) { + dst[3] = u8::MAX; + } + } + } + Ok(()) +} + +fn validate_output_region( + plan: &J2kDirectColorPlan, + output_region: J2kRect, + out_len: usize, + stride: usize, + output: DirectColorU8Output, +) -> Result<()> { + if output_region.x1 > plan.dimensions.0 + || output_region.y1 > plan.dimensions.1 + || output_region.x0 > output_region.x1 + || output_region.y0 > output_region.y1 + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let row_bytes = output_region + .width() + .checked_mul(output.bytes_per_pixel() as u32) + .and_then(|len| usize::try_from(len).ok()) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if stride < row_bytes { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let height = usize::try_from(output_region.height()) + .map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + let required = if height == 0 { + 0 + } else { + stride + .checked_mul(height - 1) + .and_then(|prefix| prefix.checked_add(row_bytes)) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + }; + if out_len < required { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(()) +} + +fn checked_area(width: u32, height: u32) -> Result { + usize::try_from(width) + .ok() + .and_then(|width| width.checked_mul(height as usize)) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) +} + +fn checked_block_base(output_x: u32, output_y: u32, stride: usize) -> Result { + usize::try_from(output_y) + .ok() + .and_then(|y| y.checked_mul(stride)) + .and_then(|base| base.checked_add(output_x as usize)) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) +} + +fn checked_block_output_len(stride: usize, width: u32, height: u32) -> Result { + if height == 0 { + return Ok(0); + } + stride + .checked_mul(height as usize - 1) + .and_then(|prefix| prefix.checked_add(width as usize)) + .ok_or_else(|| DecodingError::CodeBlockDecodeFailure.into()) +} + +fn resize_and_zero(buffer: &mut Vec, len: usize) { + buffer.resize(len, 0.0); + buffer.fill(0.0); +} + +fn sign_addend(bit_depth: u8) -> f32 { + (1_u32 << (bit_depth - 1)) as f32 +} + +fn sample_as_u8(sample: f32, bit_depth: u8) -> u8 { + let rounded = round_f32(sample); + if bit_depth == 8 { + return rounded.clamp(0.0, f32::from(u8::MAX)) as u8; + } + let max_value = if bit_depth >= 16 { + f32::from(u16::MAX) + } else { + f32::from(((1_u16 << bit_depth) - 1).max(1)) + }; + round_f32((rounded.clamp(0.0, max_value) / max_value) * f32::from(u8::MAX)) as u8 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{encode_htj2k, DecodeSettings, DecoderContext, EncodeOptions, Image}; + use alloc::vec; + + fn direct_htj2k_rgb_plan() -> (J2kDirectColorPlan, J2kRect) { + let pixels = (0..16 * 16 * 3) + .map(|idx| ((idx * 13 + idx / 3) & 0xff) as u8) + .collect::>(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 16, 16, 3, 8, false, &options).expect("encode HTJ2K RGB"); + let image = Image::new( + &bytes, + &DecodeSettings { + target_resolution: Some((4, 4)), + ..DecodeSettings::default() + }, + ) + .expect("scaled image"); + let output_region = J2kRect { + x0: 1, + y0: 1, + x1: 3, + y1: 3, + }; + let mut context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_region_with_context(&mut context, (1, 1, 2, 2)) + .expect("direct color plan"); + (plan, output_region) + } + + #[test] + fn direct_cpu_scratch_retains_component_buffers_between_executions() { + let (plan, output_region) = direct_htj2k_rgb_plan(); + let stride = output_region.width() as usize * 3; + let mut out = vec![0_u8; stride * output_region.height() as usize]; + let mut scratch = J2kDirectCpuScratch::new(); + + execute_direct_color_plan_rgb8_into(&plan, output_region, &mut scratch, &mut out, stride) + .expect("first direct execute"); + let first = scratch.allocation_profile_for_tests(); + + execute_direct_color_plan_rgb8_into(&plan, output_region, &mut scratch, &mut out, stride) + .expect("second direct execute"); + let second = scratch.allocation_profile_for_tests(); + + assert_eq!(first.component_band_sets, 3); + assert_eq!(first.component_planes, 3); + assert_eq!(second.component_band_sets, first.component_band_sets); + assert_eq!(second.component_planes, first.component_planes); + assert_eq!(second.band_buffers, first.band_buffers); + assert_eq!( + second.component_sample_capacity, + first.component_sample_capacity + ); + assert_eq!(second.band_sample_capacity, first.band_sample_capacity); + assert!(second.band_sample_capacity >= second.band_sample_len); + assert!(second.component_sample_capacity >= second.component_sample_len); + } +} diff --git a/crates/signinum-j2k-native/src/direct_plan.rs b/crates/j2k-native/src/direct_plan.rs similarity index 89% rename from crates/signinum-j2k-native/src/direct_plan.rs rename to crates/j2k-native/src/direct_plan.rs index 08f0993e..e5e9ab55 100644 --- a/crates/signinum-j2k-native/src/direct_plan.rs +++ b/crates/j2k-native/src/direct_plan.rs @@ -2,12 +2,10 @@ use alloc::vec::Vec; use crate::{J2kRect, J2kWaveletTransform}; -/// Hidden identifier for one device-owned grayscale coefficient band. -#[doc(hidden)] +/// Adapter identifier for one device-owned grayscale coefficient band. pub type J2kDirectBandId = u32; -/// Hidden grayscale-only direct device-plan step for backend experimentation. -#[doc(hidden)] +/// Adapter grayscale-only direct device-plan step for backend experimentation. #[derive(Debug, Clone)] pub enum J2kDirectGrayscaleStep { /// Decode one classic J2K sub-band into a device-owned coefficient buffer. @@ -20,8 +18,7 @@ pub enum J2kDirectGrayscaleStep { Store(J2kDirectStoreStep), } -/// Hidden grayscale-only direct device plan for backend experimentation. -#[doc(hidden)] +/// Adapter grayscale-only direct device plan for backend experimentation. #[derive(Debug, Clone)] pub struct J2kDirectGrayscalePlan { /// Final output dimensions. @@ -32,8 +29,7 @@ pub struct J2kDirectGrayscalePlan { pub steps: Vec, } -/// Hidden RGB direct device plan for backend experimentation. -#[doc(hidden)] +/// Adapter RGB direct device plan for backend experimentation. #[derive(Debug, Clone)] pub struct J2kDirectColorPlan { /// Final output dimensions. @@ -48,8 +44,7 @@ pub struct J2kDirectColorPlan { pub component_plans: Vec, } -/// Hidden owned classic J2K sub-band decode job. -#[doc(hidden)] +/// Adapter owned classic J2K sub-band decode job. #[derive(Debug, Clone)] pub struct J2kOwnedSubBandPlan { /// Stable identifier for the decoded coefficient band produced by this step. @@ -64,8 +59,7 @@ pub struct J2kOwnedSubBandPlan { pub jobs: Vec, } -/// Hidden owned HTJ2K sub-band decode job. -#[doc(hidden)] +/// Adapter owned HTJ2K sub-band decode job. #[derive(Debug, Clone)] pub struct HtOwnedSubBandPlan { /// Stable identifier for the decoded coefficient band produced by this step. @@ -80,8 +74,7 @@ pub struct HtOwnedSubBandPlan { pub jobs: Vec, } -/// Hidden owned classic J2K batched code-block decode job. -#[doc(hidden)] +/// Adapter owned classic J2K batched code-block decode job. #[derive(Debug, Clone)] pub struct J2kOwnedCodeBlockBatchJob { /// X offset within the target sub-band coefficient buffer. @@ -104,6 +97,8 @@ pub struct J2kOwnedCodeBlockBatchJob { pub number_of_coding_passes: u8, /// Total coded bitplanes for the parent sub-band. pub total_bitplanes: u8, + /// Region-of-interest maxshift value from RGN marker metadata. + pub roi_shift: u8, /// The sub-band type containing this code block. pub sub_band_type: crate::J2kSubBandType, /// The code-block style flags. @@ -114,8 +109,7 @@ pub struct J2kOwnedCodeBlockBatchJob { pub dequantization_step: f32, } -/// Hidden owned HTJ2K batched code-block decode job. -#[doc(hidden)] +/// Adapter owned HTJ2K batched code-block decode job. #[derive(Debug, Clone)] pub struct HtOwnedCodeBlockBatchJob { /// X offset within the target sub-band coefficient buffer. @@ -140,6 +134,8 @@ pub struct HtOwnedCodeBlockBatchJob { pub number_of_coding_passes: u8, /// Total coded bitplanes for the parent sub-band. pub num_bitplanes: u8, + /// Region-of-interest maxshift value from RGN marker metadata. + pub roi_shift: u8, /// Whether vertically causal context was enabled. pub stripe_causal: bool, /// Whether strict decode validation is enabled for the parent image. @@ -148,8 +144,7 @@ pub struct HtOwnedCodeBlockBatchJob { pub dequantization_step: f32, } -/// Hidden single grayscale IDWT step for a direct device plan. -#[doc(hidden)] +/// Adapter single grayscale IDWT step for a direct device plan. #[derive(Debug, Clone, Copy)] pub struct J2kDirectIdwtStep { /// Stable identifier of the output coefficient band produced by this step. @@ -176,8 +171,7 @@ pub struct J2kDirectIdwtStep { pub hh: J2kRect, } -/// Hidden grayscale store step for a direct device plan. -#[doc(hidden)] +/// Adapter grayscale store step for a direct device plan. #[derive(Debug, Clone, Copy)] pub struct J2kDirectStoreStep { /// Stable identifier of the input coefficient band. diff --git a/crates/signinum-j2k-native/src/error.rs b/crates/j2k-native/src/error.rs similarity index 93% rename from crates/signinum-j2k-native/src/error.rs rename to crates/j2k-native/src/error.rs index 87d89018..bf0b084f 100644 --- a/crates/signinum-j2k-native/src/error.rs +++ b/crates/j2k-native/src/error.rs @@ -4,6 +4,7 @@ use core::fmt; /// The main error type for JPEG 2000 decoding operations. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum DecodeError { /// Errors related to JP2 file format and box parsing. Format(FormatError), @@ -21,6 +22,7 @@ pub enum DecodeError { /// Errors related to JP2 file format and box parsing. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum FormatError { /// Invalid JP2 signature. InvalidSignature, @@ -36,6 +38,7 @@ pub enum FormatError { /// Errors related to codestream markers. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum MarkerError { /// Invalid marker encountered. Invalid, @@ -51,6 +54,7 @@ pub enum MarkerError { /// Errors related to tile processing. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum TileError { /// Invalid image tile was encountered. Invalid, @@ -64,6 +68,7 @@ pub enum TileError { /// Errors related to image dimensions and validation. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum ValidationError { /// Invalid image dimensions. InvalidDimensions, @@ -71,8 +76,12 @@ pub enum ValidationError { ImageTooLarge, /// Image has too many channels. TooManyChannels, + /// The SIZ tile grid implies more tiles than any conforming codestream can address. + TooManyTiles, /// Invalid component metadata. InvalidComponentMetadata, + /// Invalid JP2 channel definition metadata. + InvalidChannelDefinition, /// Invalid progression order. InvalidProgressionOrder, /// Invalid transformation type. @@ -91,6 +100,7 @@ pub enum ValidationError { /// Errors related to decoding operations. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum DecodingError { /// An error occurred while decoding a code-block. CodeBlockDecodeFailure, @@ -108,10 +118,13 @@ pub enum DecodingError { InvalidProgressionIterator, /// Unexpected end of data. UnexpectedEof, + /// Caller-provided output buffer is too small for the decoded image. + OutputBufferTooSmall, } /// Errors related to color space and component handling. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum ColorError { /// Multi-component transform failed. Mct, @@ -182,7 +195,9 @@ impl fmt::Display for ValidationError { Self::InvalidDimensions => write!(f, "invalid image dimensions"), Self::ImageTooLarge => write!(f, "image is too large"), Self::TooManyChannels => write!(f, "image has too many channels"), + Self::TooManyTiles => write!(f, "image has too many tiles"), Self::InvalidComponentMetadata => write!(f, "invalid component metadata"), + Self::InvalidChannelDefinition => write!(f, "invalid channel definition"), Self::InvalidProgressionOrder => write!(f, "invalid progression order"), Self::InvalidTransformation => write!(f, "invalid transformation type"), Self::InvalidQuantizationStyle => write!(f, "invalid quantization style"), @@ -215,6 +230,7 @@ impl fmt::Display for DecodingError { write!(f, "a progression iterator was invalid") } Self::UnexpectedEof => write!(f, "unexpected end of data"), + Self::OutputBufferTooSmall => write!(f, "output buffer is too small"), } } } diff --git a/crates/j2k-native/src/inspect.rs b/crates/j2k-native/src/inspect.rs new file mode 100644 index 00000000..afba486d --- /dev/null +++ b/crates/j2k-native/src/inspect.rs @@ -0,0 +1,567 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Lightweight JPEG 2000 codestream header inspection. + +extern crate alloc; + +use alloc::vec::Vec; +use core::fmt; + +use crate::{MAX_J2K_IMAGE_DIMENSION, MAX_J2K_SPEC_COMPONENTS, MAX_J2K_TILE_COUNT}; + +const MARKER_SOC: u8 = 0x4F; +const MARKER_CAP: u8 = 0x50; +const MARKER_SIZ: u8 = 0x51; +const MARKER_COD: u8 = 0x52; +const MARKER_SOT: u8 = 0x90; +const MARKER_SOD: u8 = 0x93; +const MARKER_EOC: u8 = 0xD9; + +/// Parsed JPEG 2000 codestream metadata from the main header. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct J2kCodestreamHeaderMetadata { + /// Reference-grid image dimensions derived from SIZ. + pub dimensions: (u32, u32), + /// Number of codestream components. + pub components: u16, + /// Maximum component precision in bits. + pub bit_depth: u8, + /// Reference tile width and height. + pub tile_size: (u32, u32), + /// Number of reference tiles horizontally and vertically. + pub tile_count: (u32, u32), + /// Per-component SIZ precision and sampling metadata. + pub component_info: Vec, + /// Number of resolution levels from COD. + pub resolution_levels: u8, + /// Whether COD enables a multi-component transform. + pub has_mct: bool, + /// Whether COD selects the reversible 5/3 transform. + pub reversible: bool, + /// Whether the codestream advertises high-throughput block coding. + pub high_throughput: bool, +} + +/// Parsed SIZ component metadata. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct J2kCodestreamComponentHeader { + /// Component precision in bits. + pub bit_depth: u8, + /// Whether component samples are signed. + pub signed: bool, + /// Horizontal SIZ sampling factor (`XRsiz`). + pub x_rsiz: u8, + /// Vertical SIZ sampling factor (`YRsiz`). + pub y_rsiz: u8, +} + +/// Error returned by [`inspect_j2k_codestream_header`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum J2kCodestreamHeaderError { + /// Input was shorter than the required prefix. + TooShort { + /// Required byte count. + need: usize, + /// Available byte count. + have: usize, + }, + /// Input ended while reading a marker or marker segment. + TruncatedAt { + /// Byte offset where the truncated segment begins. + offset: usize, + /// Segment being read. + segment: &'static str, + }, + /// A codestream marker did not start with `0xFF`. + InvalidMarker { + /// Byte offset of the invalid marker. + offset: usize, + /// Byte found where the marker code was expected. + marker: u8, + }, + /// A required codestream marker was absent. + MissingRequiredMarker { + /// Missing marker name. + marker: &'static str, + }, + /// A generic marker segment was malformed. + InvalidSegment { + /// Byte offset of the segment length. + offset: usize, + /// Description of the invalid segment. + what: &'static str, + }, + /// The SIZ marker segment was malformed or unsupported. + InvalidSiz { + /// Description of the invalid SIZ segment. + what: &'static str, + }, + /// The COD marker segment was malformed or unsupported. + InvalidCod { + /// Description of the invalid COD segment. + what: &'static str, + }, + /// The header is valid, but outside the public inspection contract. + Unsupported { + /// Description of the unsupported feature. + what: &'static str, + }, +} + +impl fmt::Display for J2kCodestreamHeaderError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::TooShort { need, have } => { + write!(f, "input too short: need {need} bytes, have {have}") + } + Self::TruncatedAt { offset, segment } => { + write!(f, "truncated {segment} at offset {offset}") + } + Self::InvalidMarker { offset, marker } => { + write!( + f, + "invalid codestream marker FF{marker:02X} at offset {offset}" + ) + } + Self::MissingRequiredMarker { marker } => { + write!(f, "missing required codestream marker {marker}") + } + Self::InvalidSegment { what, .. } => write!(f, "invalid marker segment: {what}"), + Self::InvalidSiz { what } => write!(f, "invalid SIZ segment: {what}"), + Self::InvalidCod { what } => write!(f, "invalid COD segment: {what}"), + Self::Unsupported { what } => write!(f, "unsupported codestream header: {what}"), + } + } +} + +/// Inspect a raw JPEG 2000 codestream main header without decoding tile data. +/// +/// This helper reads SIZ/COD metadata and stops at SOT/SOD/EOC. It intentionally +/// does not require full decode headers such as QCD, so callers can inspect the +/// same lightweight codestreams that later decode construction may reject. +pub fn inspect_j2k_codestream_header( + input: &[u8], +) -> Result { + if input.len() < 2 { + return Err(J2kCodestreamHeaderError::TooShort { + need: 2, + have: input.len(), + }); + } + if !looks_like_j2k_codestream(input) { + return Err(J2kCodestreamHeaderError::InvalidMarker { + offset: 0, + marker: input[1], + }); + } + + let mut offset = 2usize; + let mut siz = None; + let mut cod = None; + let mut high_throughput_cap = false; + let mut terminated = false; + + while offset < input.len() { + let marker = read_marker(input, &mut offset)?; + match marker { + MARKER_SOT | MARKER_SOD | MARKER_EOC => { + terminated = true; + break; + } + MARKER_SIZ => { + let payload = read_segment_payload(input, &mut offset, "SIZ")?; + siz = Some(parse_siz(payload)?); + } + MARKER_COD => { + let payload = read_segment_payload(input, &mut offset, "COD")?; + cod = Some(parse_cod(payload)?); + } + MARKER_CAP => { + let _ = read_segment_payload(input, &mut offset, "CAP")?; + high_throughput_cap = true; + } + _ => { + let _ = read_segment_payload(input, &mut offset, "segment")?; + } + } + } + + if !terminated { + return Err(J2kCodestreamHeaderError::TruncatedAt { + offset, + segment: "main header terminator", + }); + } + + let siz = siz.ok_or(J2kCodestreamHeaderError::MissingRequiredMarker { marker: "SIZ" })?; + let cod = cod + .ok_or(J2kCodestreamHeaderError::MissingRequiredMarker { marker: "COD" })? + .with_high_throughput_cap(high_throughput_cap); + + Ok(J2kCodestreamHeaderMetadata { + dimensions: siz.dimensions, + components: siz.components, + bit_depth: siz.bit_depth, + tile_size: siz.tile_size, + tile_count: siz.tile_count, + component_info: siz.component_info, + resolution_levels: cod.resolution_levels, + has_mct: cod.has_mct, + reversible: cod.reversible, + high_throughput: cod.high_throughput, + }) +} + +/// Return whether bytes start with the raw JPEG 2000 SOC marker. +#[must_use] +pub fn looks_like_j2k_codestream(input: &[u8]) -> bool { + input.len() >= 2 && input[0] == 0xFF && input[1] == MARKER_SOC +} + +#[derive(Debug, Clone)] +struct ParsedSiz { + dimensions: (u32, u32), + components: u16, + bit_depth: u8, + tile_size: (u32, u32), + tile_count: (u32, u32), + component_info: Vec, +} + +#[derive(Debug, Clone, Copy)] +struct ParsedCod { + resolution_levels: u8, + has_mct: bool, + reversible: bool, + high_throughput: bool, +} + +impl ParsedCod { + const fn with_high_throughput_cap(mut self, high_throughput_cap: bool) -> Self { + self.high_throughput |= high_throughput_cap; + self + } +} + +fn read_marker(input: &[u8], offset: &mut usize) -> Result { + if *offset + 2 > input.len() { + return Err(J2kCodestreamHeaderError::TruncatedAt { + offset: *offset, + segment: "marker", + }); + } + if input[*offset] != 0xFF { + return Err(J2kCodestreamHeaderError::InvalidMarker { + offset: *offset, + marker: input[*offset], + }); + } + let marker = input[*offset + 1]; + *offset += 2; + Ok(marker) +} + +fn read_segment_payload<'a>( + input: &'a [u8], + offset: &mut usize, + segment: &'static str, +) -> Result<&'a [u8], J2kCodestreamHeaderError> { + if *offset + 2 > input.len() { + return Err(J2kCodestreamHeaderError::TruncatedAt { + offset: *offset, + segment, + }); + } + let length = u16::from_be_bytes([input[*offset], input[*offset + 1]]) as usize; + if length < 2 { + return Err(J2kCodestreamHeaderError::InvalidSegment { + offset: *offset, + what: "segment length smaller than header", + }); + } + let start = *offset + 2; + let end = *offset + length; + if end > input.len() { + return Err(J2kCodestreamHeaderError::TruncatedAt { + offset: *offset, + segment, + }); + } + *offset = end; + Ok(&input[start..end]) +} + +#[allow(clippy::similar_names)] +fn parse_siz(payload: &[u8]) -> Result { + if payload.len() < 36 { + return Err(J2kCodestreamHeaderError::InvalidSiz { + what: "payload shorter than fixed SIZ header", + }); + } + let x_size = read_u32(payload, 2); + let y_size = read_u32(payload, 6); + let x_origin = read_u32(payload, 10); + let y_origin = read_u32(payload, 14); + let tile_width = read_u32(payload, 18); + let tile_height = read_u32(payload, 22); + let tile_x_origin = read_u32(payload, 26); + let tile_y_origin = read_u32(payload, 30); + let component_count = read_u16(payload, 34); + + let component_bytes = usize::from(component_count) * 3; + if payload.len() < 36 + component_bytes { + return Err(J2kCodestreamHeaderError::InvalidSiz { + what: "component descriptors truncated", + }); + } + if component_count == 0 { + return Err(J2kCodestreamHeaderError::InvalidSiz { + what: "component count must be non-zero", + }); + } + if component_count > MAX_J2K_SPEC_COMPONENTS { + return Err(J2kCodestreamHeaderError::InvalidSiz { + what: "component count exceeds JPEG 2000 limit", + }); + } + if x_size <= x_origin || y_size <= y_origin { + return Err(J2kCodestreamHeaderError::InvalidSiz { + what: "image origin must be smaller than image size", + }); + } + if tile_width == 0 || tile_height == 0 { + return Err(J2kCodestreamHeaderError::InvalidSiz { + what: "tile size must be non-zero", + }); + } + if tile_x_origin >= x_size || tile_y_origin >= y_size { + return Err(J2kCodestreamHeaderError::InvalidSiz { + what: "tile origin must be within image bounds", + }); + } + if tile_x_origin > x_origin || tile_y_origin > y_origin { + return Err(J2kCodestreamHeaderError::InvalidSiz { + what: "tile origin must not exceed image origin", + }); + } + if tile_x_origin + .checked_add(tile_width) + .ok_or(J2kCodestreamHeaderError::InvalidSiz { + what: "tile extent overflows", + })? + <= x_origin + || tile_y_origin + .checked_add(tile_height) + .ok_or(J2kCodestreamHeaderError::InvalidSiz { + what: "tile extent overflows", + })? + <= y_origin + { + return Err(J2kCodestreamHeaderError::InvalidSiz { + what: "first tile must overlap image area", + }); + } + + let width = x_size - x_origin; + let height = y_size - y_origin; + if width > MAX_J2K_IMAGE_DIMENSION || height > MAX_J2K_IMAGE_DIMENSION { + return Err(J2kCodestreamHeaderError::InvalidSiz { + what: "image dimensions exceed JPEG 2000 inspect limit", + }); + } + let tiles_x = (x_size - tile_x_origin).div_ceil(tile_width); + let tiles_y = (y_size - tile_y_origin).div_ceil(tile_height); + let tile_count = u64::from(tiles_x) * u64::from(tiles_y); + if tile_count > MAX_J2K_TILE_COUNT { + return Err(J2kCodestreamHeaderError::InvalidSiz { + what: "image has too many tiles", + }); + } + let mut bit_depth = 0u8; + let mut component_info = Vec::with_capacity(usize::from(component_count)); + for idx in 0..usize::from(component_count) { + let ssiz = payload[36 + idx * 3]; + let precision = (ssiz & 0x7F) + 1; + let x_rsiz = payload[36 + idx * 3 + 1]; + let y_rsiz = payload[36 + idx * 3 + 2]; + if x_rsiz == 0 || y_rsiz == 0 { + return Err(J2kCodestreamHeaderError::InvalidSiz { + what: "component sampling factors must be non-zero", + }); + } + bit_depth = bit_depth.max(precision); + component_info.push(J2kCodestreamComponentHeader { + bit_depth: precision, + signed: ssiz & 0x80 != 0, + x_rsiz, + y_rsiz, + }); + } + + Ok(ParsedSiz { + dimensions: (width, height), + components: component_count, + bit_depth, + tile_size: (tile_width, tile_height), + tile_count: (tiles_x, tiles_y), + component_info, + }) +} + +fn parse_cod(payload: &[u8]) -> Result { + if payload.len() < 10 { + return Err(J2kCodestreamHeaderError::InvalidCod { + what: "payload shorter than fixed COD header", + }); + } + Ok(ParsedCod { + resolution_levels: payload[5].saturating_add(1), + has_mct: payload[4] != 0, + reversible: payload[9] == 1, + high_throughput: payload[8] & 0x40 != 0, + }) +} + +fn read_u16(bytes: &[u8], offset: usize) -> u16 { + u16::from_be_bytes([bytes[offset], bytes[offset + 1]]) +} + +fn read_u32(bytes: &[u8], offset: usize) -> u32 { + u32::from_be_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + ]) +} + +#[cfg(test)] +mod tests { + use super::{inspect_j2k_codestream_header, J2kCodestreamHeaderError}; + use alloc::{vec, vec::Vec}; + + #[test] + fn inspect_j2k_codestream_header_accepts_minimal_main_header() { + let header = inspect_j2k_codestream_header(&minimal_codestream()).expect("header"); + + assert_eq!(header.dimensions, (128, 64)); + assert_eq!(header.components, 3); + assert_eq!(header.bit_depth, 8); + assert_eq!(header.tile_size, (64, 64)); + assert_eq!(header.tile_count, (2, 1)); + assert_eq!(header.resolution_levels, 6); + assert!(header.reversible); + } + + #[test] + fn inspect_rejects_zero_component_sampling() { + let mut bytes = minimal_codestream(); + rewrite_component_sampling(&mut bytes, 0, 0, 1); + + let err = inspect_j2k_codestream_header(&bytes).expect_err("zero sampling must reject"); + + assert!(matches!(err, J2kCodestreamHeaderError::InvalidSiz { .. })); + } + + #[test] + fn inspect_rejects_oversized_dimensions() { + let mut bytes = minimal_codestream(); + rewrite_siz_u32(&mut bytes, 2, 60_001); + + let err = inspect_j2k_codestream_header(&bytes).expect_err("oversized width must reject"); + + assert!(matches!(err, J2kCodestreamHeaderError::InvalidSiz { .. })); + } + + #[test] + fn inspect_rejects_tile_origin_after_image_origin() { + let mut bytes = minimal_codestream(); + rewrite_siz_u32(&mut bytes, 26, 1); + + let err = inspect_j2k_codestream_header(&bytes).expect_err("bad tile origin must reject"); + + assert!(matches!(err, J2kCodestreamHeaderError::InvalidSiz { .. })); + } + + #[test] + fn inspect_rejects_tile_extent_overflow() { + let mut bytes = minimal_codestream(); + rewrite_siz_u32(&mut bytes, 2, u32::MAX); + rewrite_siz_u32(&mut bytes, 10, u32::MAX - 1); + rewrite_siz_u32(&mut bytes, 18, 10); + rewrite_siz_u32(&mut bytes, 26, u32::MAX - 2); + + let err = inspect_j2k_codestream_header(&bytes).expect_err("overflow must reject"); + + assert!(matches!(err, J2kCodestreamHeaderError::InvalidSiz { .. })); + } + + #[test] + fn inspect_rejects_excessive_tile_count() { + let mut bytes = minimal_codestream(); + rewrite_siz_u32(&mut bytes, 2, 257); + rewrite_siz_u32(&mut bytes, 6, 257); + rewrite_siz_u32(&mut bytes, 18, 1); + rewrite_siz_u32(&mut bytes, 22, 1); + + let err = inspect_j2k_codestream_header(&bytes).expect_err("tile count must reject"); + + assert!(matches!(err, J2kCodestreamHeaderError::InvalidSiz { .. })); + } + + fn minimal_codestream() -> Vec { + let mut bytes = vec![0xFF, 0x4F]; + let mut siz = Vec::new(); + push_u16(&mut siz, 0); + push_u32(&mut siz, 128); + push_u32(&mut siz, 64); + push_u32(&mut siz, 0); + push_u32(&mut siz, 0); + push_u32(&mut siz, 64); + push_u32(&mut siz, 64); + push_u32(&mut siz, 0); + push_u32(&mut siz, 0); + push_u16(&mut siz, 3); + for _ in 0..3 { + siz.extend_from_slice(&[0x07, 0x01, 0x01]); + } + bytes.extend_from_slice(&[0xFF, 0x51]); + push_u16(&mut bytes, (siz.len() + 2) as u16); + bytes.extend_from_slice(&siz); + + let cod = [0x00, 0x00, 0x00, 0x01, 0x01, 0x05, 0x04, 0x04, 0x00, 0x01]; + bytes.extend_from_slice(&[0xFF, 0x52]); + push_u16(&mut bytes, (cod.len() + 2) as u16); + bytes.extend_from_slice(&cod); + bytes.extend_from_slice(&[0xFF, 0x90, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); + bytes + } + + fn push_u16(out: &mut Vec, value: u16) { + out.extend_from_slice(&value.to_be_bytes()); + } + + fn push_u32(out: &mut Vec, value: u32) { + out.extend_from_slice(&value.to_be_bytes()); + } + + fn rewrite_siz_u32(bytes: &mut [u8], payload_offset: usize, value: u32) { + let siz = bytes + .windows(2) + .position(|marker| marker == [0xFF, 0x51]) + .expect("SIZ marker"); + let offset = siz + 4 + payload_offset; + bytes[offset..offset + 4].copy_from_slice(&value.to_be_bytes()); + } + + fn rewrite_component_sampling(bytes: &mut [u8], component: usize, x_rsiz: u8, y_rsiz: u8) { + let siz = bytes + .windows(2) + .position(|marker| marker == [0xFF, 0x51]) + .expect("SIZ marker"); + let component_offset = siz + 40 + component * 3; + bytes[component_offset + 1] = x_rsiz; + bytes[component_offset + 2] = y_rsiz; + } +} diff --git a/crates/signinum-j2k-native/src/j2c/arithmetic_decoder.rs b/crates/j2k-native/src/j2c/arithmetic_decoder.rs similarity index 77% rename from crates/signinum-j2k-native/src/j2c/arithmetic_decoder.rs rename to crates/j2k-native/src/j2c/arithmetic_decoder.rs index 204cf163..74bca624 100644 --- a/crates/signinum-j2k-native/src/j2c/arithmetic_decoder.rs +++ b/crates/j2k-native/src/j2c/arithmetic_decoder.rs @@ -110,80 +110,6 @@ impl<'a> ArithmeticDecoder<'a> { } } - /// The `LPS_EXCHANGE` procedure from C.3.2. - #[inline(always)] - #[allow(unused, reason = "for reference")] - fn exchange_lps(&mut self, context: &mut ArithmeticDecoderContext, qe_entry: &QeData) -> u32 { - // Original code: - // let d; - // - // if self.a < qe_entry.qe { - // self.a = qe_entry.qe; - // d = context.mps; - // context.index = qe_entry.nmps; - // } else { - // self.a = qe_entry.qe; - // d = 1 - context.mps; - // - // if qe_entry.switch { - // context.mps = 1 - context.mps; - // } - // - // context.index = qe_entry.nlps; - // } - - // Branchless version, shows better performance. - - let cond = (self.a < qe_entry.qe) as u32; - let inv_cond = 1 - cond; - - self.a = qe_entry.qe; - // d = if cond { mps } else { 1 - mps } - let d = context.mps() ^ inv_cond; - // flip mps only when !cond && switch - context.xor_mps(inv_cond & (qe_entry.switch as u32)); - // index = if cond { nmps } else { nlps } - let cond_u8 = cond as u8; - let inv_cond_u8 = inv_cond as u8; - context.set_index(cond_u8 * qe_entry.nmps + inv_cond_u8 * qe_entry.nlps); - - d - } - - /// The `MPS_EXCHANGE` procedure from C.3.2. - #[inline(always)] - #[allow(unused, reason = "for reference")] - fn exchange_mps(&mut self, context: &mut ArithmeticDecoderContext, qe_entry: &QeData) -> u32 { - // Original code: - // let d; - // - // if self.a < qe_entry.qe { - // d = 1 - context.mps; - // - // if qe_entry.switch { - // context.mps = 1 - context.mps; - // } - // - // context.index = qe_entry.nlps; - // } else { - // d = context.mps; - // context.index = qe_entry.nmps; - // } - - // Branchless version, shows better performance. - let cond = (self.a < qe_entry.qe) as u32; - let inv_cond = 1 - cond; - // d = if cond { 1 - mps } else { mps } - let d = context.mps() ^ cond; - // flip mps only when cond && switch - context.xor_mps(cond & (qe_entry.switch as u32)); - // index = if cond { nlps } else { nmps } - let cond_u8 = cond as u8; - let inv_cond_u8 = inv_cond as u8; - context.set_index(cond_u8 * qe_entry.nlps + inv_cond_u8 * qe_entry.nmps); - d - } - /// The DECODE procedure from C.3.2. /// /// We use the version from Annex G from . @@ -203,10 +129,8 @@ impl<'a> ArithmeticDecoder<'a> { return context.mps(); } - // Unified branchless MPS_EXCHANGE / LPS_EXCHANGE. - // - // Compare with exchange_mps and exchange_lps above — the only - // difference between them is that LPS flips the role of cond: + // Unified branchless MPS_EXCHANGE / LPS_EXCHANGE. In the Annex C.3.2 + // procedures, the only difference is that LPS flips the role of cond: // exchange_mps: d = mps ^ cond, flip when cond, index = cond*nlps + inv*nmps // exchange_lps: d = mps ^ inv_cond, flip when inv_cond, index = cond*nmps + inv*nlps // @@ -279,6 +203,7 @@ impl<'a> ArithmeticDecoder<'a> { pub(crate) struct ArithmeticDecoderContext(u8); impl ArithmeticDecoderContext { + #[cfg_attr(not(test), allow(dead_code))] #[inline(always)] pub(crate) fn index(self) -> u32 { (self.0 & 0x7F) as u32 diff --git a/crates/signinum-j2k-native/src/j2c/arithmetic_encoder.rs b/crates/j2k-native/src/j2c/arithmetic_encoder.rs similarity index 92% rename from crates/signinum-j2k-native/src/j2c/arithmetic_encoder.rs rename to crates/j2k-native/src/j2c/arithmetic_encoder.rs index 75957ed4..20759bd8 100644 --- a/crates/signinum-j2k-native/src/j2c/arithmetic_encoder.rs +++ b/crates/j2k-native/src/j2c/arithmetic_encoder.rs @@ -3,7 +3,6 @@ //! This is the encoding counterpart of `arithmetic_decoder.rs`. //! It uses the same QE probability table and context state machine. -use alloc::vec; use alloc::vec::Vec; /// MQ arithmetic encoder context (identical layout to decoder context). @@ -123,8 +122,15 @@ pub(crate) struct ArithmeticEncoder { impl ArithmeticEncoder { pub(crate) fn new() -> Self { + Self::with_capacity(1) + } + + pub(crate) fn with_capacity(capacity: usize) -> Self { + let mut data = Vec::with_capacity(capacity.max(1)); + data.push(0x00); + Self { - data: vec![0x00], // Sentinel byte at index 0 + data, // Sentinel byte at index 0 a: 0x8000, c: 0, ct: 12, @@ -132,7 +138,7 @@ impl ArithmeticEncoder { } /// Encode a single symbol (0 or 1) with the given context (C.2.6 ENCODE). - #[inline] + #[inline(always)] pub(crate) fn encode(&mut self, bit: u32, context: &mut ArithmeticEncoderContext) { let qe_entry = &QE_TABLE[context.index() as usize]; self.a -= qe_entry.qe; @@ -255,6 +261,7 @@ impl ArithmeticEncoder { mod tests { use super::*; use crate::j2c::arithmetic_decoder::{ArithmeticDecoder, ArithmeticDecoderContext}; + use alloc::{vec, vec::Vec}; #[test] fn test_encode_decode_round_trip() { @@ -295,6 +302,24 @@ mod tests { } } + #[test] + fn with_capacity_preserves_round_trip_encoding() { + let mut encoder = ArithmeticEncoder::with_capacity(128); + let mut enc_ctx = ArithmeticEncoderContext::default(); + let symbols = [0u32, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1]; + + for &symbol in &symbols { + encoder.encode(symbol, &mut enc_ctx); + } + let encoded = encoder.finish(); + + let mut decoder = ArithmeticDecoder::new(&encoded); + let mut dec_ctx = ArithmeticDecoderContext::default(); + for &symbol in &symbols { + assert_eq!(decoder.decode(&mut dec_ctx), symbol); + } + } + #[test] fn test_encode_all_lps() { let mut encoder = ArithmeticEncoder::new(); diff --git a/crates/signinum-j2k-native/src/j2c/bitplane.rs b/crates/j2k-native/src/j2c/bitplane.rs similarity index 59% rename from crates/signinum-j2k-native/src/j2c/bitplane.rs rename to crates/j2k-native/src/j2c/bitplane.rs index 4baa7b88..e8f4bfd3 100644 --- a/crates/signinum-j2k-native/src/j2c/bitplane.rs +++ b/crates/j2k-native/src/j2c/bitplane.rs @@ -17,9 +17,85 @@ use super::build::{CodeBlock, SubBandType}; use super::codestream::CodeBlockStyle; use super::decode::{DecompositionStorage, TileDecodeContext}; use crate::error::{bail, DecodingError, Result}; +use crate::profile; use crate::reader::BitReader; use crate::J2kCodeBlockSegment; +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct J2kBlockDecodeStats { + pub(crate) sigprop_us: u128, + pub(crate) magref_us: u128, + pub(crate) cleanup_us: u128, + pub(crate) bypass_us: u128, +} + +trait J2kDecodeObserver { + #[inline(always)] + fn phase_start(&self) -> Option { + None + } + + #[inline(always)] + fn add_sigprop_us(&mut self, _start: Option) {} + + #[inline(always)] + fn add_magref_us(&mut self, _start: Option) {} + + #[inline(always)] + fn add_cleanup_us(&mut self, _start: Option) {} + + #[inline(always)] + fn add_bypass_us(&mut self, _start: Option) {} +} + +struct NoJ2kDecodeStats; + +impl J2kDecodeObserver for NoJ2kDecodeStats {} + +struct RecordingJ2kDecodeStats<'a> { + stats: &'a mut J2kBlockDecodeStats, + profile_enabled: bool, +} + +impl J2kDecodeObserver for RecordingJ2kDecodeStats<'_> { + #[inline(always)] + fn phase_start(&self) -> Option { + if self.profile_enabled { + profile::profile_now(true) + } else { + None + } + } + + #[inline(always)] + fn add_sigprop_us(&mut self, start: Option) { + if self.profile_enabled { + self.stats.sigprop_us += profile::elapsed_us(start); + } + } + + #[inline(always)] + fn add_magref_us(&mut self, start: Option) { + if self.profile_enabled { + self.stats.magref_us += profile::elapsed_us(start); + } + } + + #[inline(always)] + fn add_cleanup_us(&mut self, start: Option) { + if self.profile_enabled { + self.stats.cleanup_us += profile::elapsed_us(start); + } + } + + #[inline(always)] + fn add_bypass_us(&mut self, start: Option) { + if self.profile_enabled { + self.stats.bypass_us += profile::elapsed_us(start); + } + } +} + /// Decode the layers of the given code block into coefficients. /// /// The result will be stored in the form of a vector of signs and magnitudes @@ -65,6 +141,72 @@ pub(crate) fn decode_code_block_segments_validated( code_block_style: &CodeBlockStyle, strict: bool, ctx: &mut BitPlaneDecodeContext, +) -> Result<()> { + let mut observer = NoJ2kDecodeStats; + decode_code_block_segments_validated_with_observer( + data, + segments, + width, + height, + missing_bit_planes, + number_of_coding_passes, + total_bitplanes, + sub_band_type, + code_block_style, + strict, + ctx, + &mut observer, + ) +} + +pub(crate) fn decode_code_block_segments_validated_profiled( + data: &[u8], + segments: &[J2kCodeBlockSegment], + width: u32, + height: u32, + missing_bit_planes: u8, + number_of_coding_passes: u8, + total_bitplanes: u8, + sub_band_type: SubBandType, + code_block_style: &CodeBlockStyle, + strict: bool, + ctx: &mut BitPlaneDecodeContext, + stats: &mut J2kBlockDecodeStats, + profile_enabled: bool, +) -> Result<()> { + let mut observer = RecordingJ2kDecodeStats { + stats, + profile_enabled, + }; + decode_code_block_segments_validated_with_observer( + data, + segments, + width, + height, + missing_bit_planes, + number_of_coding_passes, + total_bitplanes, + sub_band_type, + code_block_style, + strict, + ctx, + &mut observer, + ) +} + +fn decode_code_block_segments_validated_with_observer( + data: &[u8], + segments: &[J2kCodeBlockSegment], + width: u32, + height: u32, + missing_bit_planes: u8, + number_of_coding_passes: u8, + total_bitplanes: u8, + sub_band_type: SubBandType, + code_block_style: &CodeBlockStyle, + strict: bool, + ctx: &mut BitPlaneDecodeContext, + observer: &mut O, ) -> Result<()> { ctx.reset_for_job( width, @@ -81,7 +223,7 @@ pub(crate) fn decode_code_block_segments_validated( return Ok(()); } - decode_code_block_segments_inner(data, segments, number_of_coding_passes, ctx) + decode_code_block_segments_inner(data, segments, number_of_coding_passes, ctx, observer) .ok_or(DecodingError::CodeBlockDecodeFailure)?; Ok(()) @@ -92,6 +234,17 @@ fn decode_inner( storage: &DecompositionStorage<'_>, ctx: &mut BitPlaneDecodeContext, bp_buffers: &mut BitPlaneDecodeBuffers, +) -> Option<()> { + let mut observer = NoJ2kDecodeStats; + decode_inner_with_observer(code_block, storage, ctx, bp_buffers, &mut observer) +} + +fn decode_inner_with_observer( + code_block: &CodeBlock, + storage: &DecompositionStorage<'_>, + ctx: &mut BitPlaneDecodeContext, + bp_buffers: &mut BitPlaneDecodeBuffers, + observer: &mut O, ) -> Option<()> { bp_buffers.reset(); @@ -134,14 +287,14 @@ fn decode_inner( // Only one termination per code block, so we can just decode the // whole range in one single go, processing all coding passes at once. let mut decoder = ArithmeticDecoder::new(&bp_buffers.combined_layers); - handle_arithmetic_coding_passes( - 0, - code_block - .number_of_coding_passes - .min(ctx.max_coding_passes), - ctx, - &mut decoder, - )?; + let end = code_block + .number_of_coding_passes + .min(ctx.max_coding_passes); + if ctx.uses_normal_arithmetic_neighbor_path() { + handle_normal_arithmetic_coding_passes(0, end, ctx, &mut decoder, observer)?; + } else { + handle_arithmetic_coding_passes(0, end, ctx, &mut decoder, observer)?; + } } else { // Otherwise, each segment introduces a termination. For "termination on // each pass", each segment only covers one coding pass @@ -174,10 +327,17 @@ fn decode_inner( end_coding_pass, ctx, &mut decoder, + observer, )?; } else { let mut decoder = BypassDecoder::new(data, ctx.strict); - handle_bypass_coding_passes(start_coding_pass, end_coding_pass, ctx, &mut decoder)?; + handle_bypass_coding_passes( + start_coding_pass, + end_coding_pass, + ctx, + &mut decoder, + observer, + )?; } } } @@ -190,6 +350,27 @@ fn handle_arithmetic_coding_passes( end: u8, ctx: &mut BitPlaneDecodeContext, decoder: &mut ArithmeticDecoder<'_>, + observer: &mut impl J2kDecodeObserver, +) -> Option<()> { + handle_arithmetic_coding_passes_with_neighbors::(start, end, ctx, decoder, observer) +} + +fn handle_normal_arithmetic_coding_passes( + start: u8, + end: u8, + ctx: &mut BitPlaneDecodeContext, + decoder: &mut ArithmeticDecoder<'_>, + observer: &mut impl J2kDecodeObserver, +) -> Option<()> { + handle_arithmetic_coding_passes_with_neighbors::(start, end, ctx, decoder, observer) +} + +fn handle_arithmetic_coding_passes_with_neighbors( + start: u8, + end: u8, + ctx: &mut BitPlaneDecodeContext, + decoder: &mut ArithmeticDecoder<'_>, + observer: &mut impl J2kDecodeObserver, ) -> Option<()> { for coding_pass in start..end { let current_bitplane = coding_pass.div_ceil(3); @@ -199,7 +380,8 @@ fn handle_arithmetic_coding_passes( // are in the order SPP -> MRR -> C. match coding_pass % 3 { 0 => { - SafeScalarTier1::cleanup_pass_arithmetic(ctx, decoder); + let phase_start = observer.phase_start(); + cleanup_pass_arithmetic_with_neighbors::(ctx, decoder); if ctx.style.segmentation_symbols { let b0 = decoder.read_bit(ctx.arithmetic_decoder_context(18)); @@ -213,12 +395,21 @@ fn handle_arithmetic_coding_passes( } ctx.reset_for_next_bitplane(); + observer.add_cleanup_us(phase_start); } 1 => { - SafeScalarTier1::significance_propagation_pass_arithmetic(ctx, decoder); + let phase_start = observer.phase_start(); + significance_propagation_pass_arithmetic_with_neighbors::( + ctx, decoder, + ); + observer.add_sigprop_us(phase_start); } 2 => { - SafeScalarTier1::magnitude_refinement_pass_arithmetic(ctx, decoder); + let phase_start = observer.phase_start(); + magnitude_refinement_pass_arithmetic_with_neighbors::( + ctx, decoder, + ); + observer.add_magref_us(phase_start); } _ => unreachable!(), } @@ -236,8 +427,10 @@ fn handle_bypass_coding_passes( end: u8, ctx: &mut BitPlaneDecodeContext, decoder: &mut BypassDecoder<'_>, + observer: &mut impl J2kDecodeObserver, ) -> Option<()> { for coding_pass in start..end { + let phase_start = observer.phase_start(); let current_bitplane = coding_pass.div_ceil(3); ctx.current_bit_position = ctx.bitplanes - 1 - current_bitplane; @@ -270,6 +463,7 @@ fn handle_bypass_coding_passes( if ctx.style.reset_context_probabilities { ctx.reset_contexts(); } + observer.add_bypass_us(phase_start); } Some(()) @@ -278,10 +472,9 @@ fn handle_bypass_coding_passes( // We only allow 31 bit planes because we need one bit for the sign. pub(crate) const BITPLANE_BIT_SIZE: u32 = size_of::() as u32 * 8 - 1; -const SIGNIFICANCE_SHIFT: u8 = 7; const HAS_MAGNITUDE_REFINEMENT_SHIFT: u8 = 6; const HAS_ZERO_CODING_SHIFT: u8 = 5; -const SIGNIFICANCE_MASK: u8 = 1 << SIGNIFICANCE_SHIFT; +const SIGNIFICANCE_MASK: u8 = 1 << 7; const HAS_MAGNITUDE_REFINEMENT_MASK: u8 = 1 << HAS_MAGNITUDE_REFINEMENT_SHIFT; const HAS_ZERO_CODING_MASK: u8 = 1 << HAS_ZERO_CODING_SHIFT; @@ -293,27 +486,14 @@ const HAS_ZERO_CODING_MASK: u8 = 1 << HAS_ZERO_CODING_SHIFT; pub(crate) struct CoefficientState(u8); impl CoefficientState { - #[inline(always)] - fn set_bit(&mut self, shift: u8, value: u8) { - debug_assert!(value < 2); - - self.0 &= !(1_u8 << shift); - self.0 |= value << shift; - } - #[inline(always)] fn set_significant(&mut self) { - self.set_bit(SIGNIFICANCE_SHIFT, 1); + self.0 |= SIGNIFICANCE_MASK; } #[inline(always)] fn is_significant(&self) -> bool { - self.significance() == 1 - } - - #[inline(always)] - fn significance(&self) -> u8 { - (self.0 >> SIGNIFICANCE_SHIFT) & 1 + self.0 & SIGNIFICANCE_MASK != 0 } } @@ -420,6 +600,10 @@ impl BitPlaneDecodeBuffers { pub(crate) struct BitPlaneDecodeContext { /// A vector of bit-packed fields for each coefficient in the code-block. coefficient_states: Vec, + /// One 4-bit mask per scan stripe column for coefficients that are significant. + significant_scan_masks: Vec, + /// One 4-bit mask per scan stripe column for zero-coded coefficients in this bitplane. + zero_coding_scan_masks: Vec, /// The neighbor significances for each coefficient. neighbor_significances: Vec, /// The magnitude and signs of each coefficient that is successively built @@ -451,6 +635,8 @@ impl Default for BitPlaneDecodeContext { fn default() -> Self { Self { coefficient_states: vec![], + significant_scan_masks: vec![], + zero_coding_scan_masks: vec![], coefficients: vec![], neighbor_significances: vec![], width: 0, @@ -495,6 +681,12 @@ impl BitPlaneDecodeContext { self.coefficient_states .resize(num_coefficients, CoefficientState::default()); + let scan_units = width as usize * height.div_ceil(4) as usize; + self.significant_scan_masks.clear(); + self.significant_scan_masks.resize(scan_units, 0); + self.zero_coding_scan_masks.clear(); + self.zero_coding_scan_masks.resize(scan_units, 0); + self.width = width; self.padded_width = padded_width; self.height = height; @@ -586,6 +778,7 @@ impl BitPlaneDecodeContext { state.0 &= !HAS_ZERO_CODING_MASK; } } + self.zero_coding_scan_masks.fill(0); } #[inline(always)] @@ -599,6 +792,7 @@ impl BitPlaneDecodeContext { if !is_significant { self.coefficient_states[idx].set_significant(); + self.set_significant_scan_mask(idx, padded_width); // Update all neighbors so they know this coefficient is significant // now. @@ -613,11 +807,81 @@ impl BitPlaneDecodeContext { } } + #[inline(always)] + fn set_significant_index_for_path( + &mut self, + idx: usize, + padded_width: usize, + ) { + if NORMAL_NEIGHBORS { + self.set_significant_index_normal(idx, padded_width); + } else { + self.set_significant_index(idx, padded_width); + } + } + + #[inline(always)] + fn set_significant_index_normal(&mut self, idx: usize, padded_width: usize) { + if self.coefficient_states[idx].is_significant() { + return; + } + + self.coefficient_states[idx].set_significant(); + self.set_significant_scan_mask(idx, padded_width); + + let top_start = idx - padded_width - 1; + let top = &mut self.neighbor_significances[top_start..top_start + 3]; + top[0].set_bottom_right(); + top[1].set_bottom(); + top[2].set_bottom_left(); + + let middle_start = idx - 1; + let middle = &mut self.neighbor_significances[middle_start..middle_start + 3]; + middle[0].set_right(); + middle[2].set_left(); + + let bottom_start = idx + padded_width - 1; + let bottom = &mut self.neighbor_significances[bottom_start..bottom_start + 3]; + bottom[0].set_top_right(); + bottom[1].set_top(); + bottom[2].set_top_left(); + } + #[inline(always)] fn push_magnitude_bit_index(&mut self, idx: usize, bit: u32) { self.coefficients[idx].push_bit_at(bit, self.current_bit_position); } + #[inline(always)] + fn set_zero_coding_index(&mut self, idx: usize, padded_width: usize) { + self.coefficient_states[idx].0 |= HAS_ZERO_CODING_MASK; + let (scan_unit, bit) = self.scan_unit_mask_index(idx, padded_width); + self.zero_coding_scan_masks[scan_unit] |= bit; + } + + #[inline(always)] + fn set_significant_scan_mask(&mut self, idx: usize, padded_width: usize) { + let (scan_unit, bit) = self.scan_unit_mask_index(idx, padded_width); + self.significant_scan_masks[scan_unit] |= bit; + } + + #[inline(always)] + fn scan_unit_mask_index(&self, idx: usize, padded_width: usize) -> (usize, u8) { + let row = idx / padded_width; + let col = idx - row * padded_width; + let pad = COEFFICIENTS_PADDING as usize; + debug_assert!(row >= pad); + debug_assert!(col >= pad); + + let y = row - pad; + let x = col - pad; + debug_assert!(y < self.height as usize); + debug_assert!(x < self.width as usize); + + let scan_unit = (y >> 2) * self.width as usize + x; + (scan_unit, 1u8 << (y & 3)) + } + #[inline(always)] fn sign_index(&self, idx: usize) -> u8 { self.coefficients[idx].sign() as u8 @@ -639,6 +903,18 @@ impl BitPlaneDecodeContext { neighbors.all() } } + + #[inline(always)] + fn normal_neighborhood_significance_states_index(&self, idx: usize) -> u8 { + self.neighbor_significances[idx].all() + } + + #[inline(always)] + fn uses_normal_arithmetic_neighbor_path(&self) -> bool { + !self.style.selective_arithmetic_coding_bypass + && !self.style.termination_on_each_pass + && !self.style.vertically_causal_context + } } fn decode_code_block_segments_inner( @@ -646,6 +922,7 @@ fn decode_code_block_segments_inner( segments: &[J2kCodeBlockSegment], number_of_coding_passes: u8, ctx: &mut BitPlaneDecodeContext, + observer: &mut impl J2kDecodeObserver, ) -> Option<()> { let mut expected_start = 0u8; @@ -666,10 +943,32 @@ fn decode_code_block_segments_inner( if segment.use_arithmetic { let mut decoder = ArithmeticDecoder::new(segment_data); - handle_arithmetic_coding_passes(start_coding_pass, end_coding_pass, ctx, &mut decoder)?; + if ctx.uses_normal_arithmetic_neighbor_path() { + handle_normal_arithmetic_coding_passes( + start_coding_pass, + end_coding_pass, + ctx, + &mut decoder, + observer, + )?; + } else { + handle_arithmetic_coding_passes( + start_coding_pass, + end_coding_pass, + ctx, + &mut decoder, + observer, + )?; + } } else { let mut decoder = BypassDecoder::new(segment_data, ctx.strict); - handle_bypass_coding_passes(start_coding_pass, end_coding_pass, ctx, &mut decoder)?; + handle_bypass_coding_passes( + start_coding_pass, + end_coding_pass, + ctx, + &mut decoder, + observer, + )?; } } @@ -680,48 +979,13 @@ fn decode_code_block_segments_inner( Some(()) } -trait Tier1PassDecoder { - fn cleanup_pass_arithmetic( - ctx: &mut BitPlaneDecodeContext, - decoder: &mut ArithmeticDecoder<'_>, - ); - - fn significance_propagation_pass_arithmetic( - ctx: &mut BitPlaneDecodeContext, - decoder: &mut ArithmeticDecoder<'_>, - ); - - fn magnitude_refinement_pass_arithmetic( - ctx: &mut BitPlaneDecodeContext, - decoder: &mut ArithmeticDecoder<'_>, - ); +struct SafeScalarTier1; +impl SafeScalarTier1 { fn cleanup_pass_bypass( ctx: &mut BitPlaneDecodeContext, decoder: &mut BypassDecoder<'_>, - ) -> Option<()>; - - fn significance_propagation_pass_bypass( - ctx: &mut BitPlaneDecodeContext, - decoder: &mut BypassDecoder<'_>, - ) -> Option<()>; - - fn magnitude_refinement_pass_bypass( - ctx: &mut BitPlaneDecodeContext, - decoder: &mut BypassDecoder<'_>, - ) -> Option<()>; -} - -struct SafeScalarTier1; - -impl Tier1PassDecoder for SafeScalarTier1 { - /// Perform the cleanup pass, specified in D.3.4. - /// - /// See also the flow chart in Figure 7.3 in the JPEG2000 book. - fn cleanup_pass_arithmetic( - ctx: &mut BitPlaneDecodeContext, - decoder: &mut ArithmeticDecoder<'_>, - ) { + ) -> Option<()> { let width = ctx.width as usize; let height = ctx.height as usize; let padded_width = ctx.padded_width as usize; @@ -738,26 +1002,24 @@ impl Tier1PassDecoder for SafeScalarTier1 { if stripe_height == 4 && cleanup_run_length_candidate(ctx, top_idx, padded_width, base_y) { - // The four contiguous samples are all cleanup candidates - // with zero context, so Annex D permits the RLC context. - let bit = decoder.read_bit(ctx.arithmetic_decoder_context(17)); + let bit = decoder.read_bit(ctx.arithmetic_decoder_context(17))?; if bit == 0 { continue; } - let first_significant = (decoder.read_bit(ctx.arithmetic_decoder_context(18)) - << 1) - | decoder.read_bit(ctx.arithmetic_decoder_context(18)); + let first_significant = + (decoder.read_bit(ctx.arithmetic_decoder_context(18))? << 1) + | decoder.read_bit(ctx.arithmetic_decoder_context(18))?; let first_significant = first_significant as usize; let significant_y = base_y + first_significant; let significant_idx = top_idx + first_significant * padded_width; ctx.push_magnitude_bit_index(significant_idx, 1); - decode_sign_bit_arithmetic(significant_idx, significant_y, ctx, decoder); + decode_sign_bit_bypass(significant_idx, significant_y, ctx, decoder)?; ctx.set_significant_index(significant_idx, padded_width); let mut idx = significant_idx + padded_width; for y in significant_y + 1..y_end { - cleanup_coefficient_arithmetic(ctx, decoder, idx, y, padded_width); + cleanup_coefficient_bypass(ctx, decoder, idx, y, padded_width)?; idx += padded_width; } continue; @@ -765,20 +1027,19 @@ impl Tier1PassDecoder for SafeScalarTier1 { let mut idx = top_idx; for y in base_y..y_end { - cleanup_coefficient_arithmetic(ctx, decoder, idx, y, padded_width); + cleanup_coefficient_bypass(ctx, decoder, idx, y, padded_width)?; idx += padded_width; } } } + + Some(()) } - /// Perform the significance propagation pass (Section D.3.1). - /// - /// See also the flow chart in Figure 7.4 in the JPEG2000 book. - fn significance_propagation_pass_arithmetic( + fn significance_propagation_pass_bypass( ctx: &mut BitPlaneDecodeContext, - decoder: &mut ArithmeticDecoder<'_>, - ) { + decoder: &mut BypassDecoder<'_>, + ) -> Option<()> { let width = ctx.width as usize; let height = ctx.height as usize; let padded_width = ctx.padded_width as usize; @@ -794,22 +1055,15 @@ impl Tier1PassDecoder for SafeScalarTier1 { let state = ctx.coefficient_states[idx].0; let neighbors = ctx.neighborhood_significance_states_index(idx, y); - // "The significance propagation pass only includes bits of coefficients - // that were insignificant (the significance state has yet to be set) - // and have a non-zero context." if state & SIGNIFICANCE_MASK == 0 && neighbors != 0 { let ctx_label = context_label_zero_coding_from_neighbors(neighbors, ctx.sub_band_type); - let bit = decoder.read_bit(ctx.arithmetic_decoder_context(ctx_label)); + let bit = decoder.read_bit(ctx.arithmetic_decoder_context(ctx_label))?; ctx.push_magnitude_bit_index(idx, bit); - ctx.coefficient_states[idx].0 |= HAS_ZERO_CODING_MASK; + ctx.set_zero_coding_index(idx, padded_width); - // "If the value of this bit is 1 then the significance - // state is set to 1 and the immediate next bit to be decoded is - // the sign bit for the coefficient. Otherwise, the significance - // state remains 0." if bit == 1 { - decode_sign_bit_arithmetic(idx, y, ctx, decoder); + decode_sign_bit_bypass(idx, y, ctx, decoder)?; ctx.set_significant_index(idx, padded_width); } } @@ -818,45 +1072,11 @@ impl Tier1PassDecoder for SafeScalarTier1 { } } } - } - - /// Perform the magnitude refinement pass, specified in Section D.3.3. - /// - /// See also the flow chart in Figure 7.5 in the JPEG2000 book. - fn magnitude_refinement_pass_arithmetic( - ctx: &mut BitPlaneDecodeContext, - decoder: &mut ArithmeticDecoder<'_>, - ) { - let width = ctx.width as usize; - let height = ctx.height as usize; - let padded_width = ctx.padded_width as usize; - - for base_y in (0..height).step_by(4) { - let y_end = (base_y + 4).min(height); - for x in 0..width { - let mut idx = (base_y + COEFFICIENTS_PADDING as usize) * padded_width - + x - + COEFFICIENTS_PADDING as usize; - - for y in base_y..y_end { - let state = ctx.coefficient_states[idx].0; - if state & SIGNIFICANCE_MASK != 0 && state & HAS_ZERO_CODING_MASK == 0 { - let neighbors = ctx.neighborhood_significance_states_index(idx, y); - let ctx_label = - context_label_magnitude_refinement_coding_from_state(state, neighbors); - let bit = decoder.read_bit(ctx.arithmetic_decoder_context(ctx_label)); - ctx.push_magnitude_bit_index(idx, bit); - ctx.coefficient_states[idx].0 |= HAS_MAGNITUDE_REFINEMENT_MASK; - } - - idx += padded_width; - } - } - } + Some(()) } - fn cleanup_pass_bypass( + fn magnitude_refinement_pass_bypass( ctx: &mut BitPlaneDecodeContext, decoder: &mut BypassDecoder<'_>, ) -> Option<()> { @@ -864,130 +1084,266 @@ impl Tier1PassDecoder for SafeScalarTier1 { let height = ctx.height as usize; let padded_width = ctx.padded_width as usize; - for base_y in (0..height).step_by(4) { - let y_end = (base_y + 4).min(height); - let stripe_height = y_end - base_y; + for (stripe, base_y) in (0..height).step_by(4).enumerate() { + let stripe_height = (base_y + 4).min(height) - base_y; + let scan_unit_row = stripe * width; for x in 0..width { + let mut mask = ctx.significant_scan_masks[scan_unit_row + x] + & !ctx.zero_coding_scan_masks[scan_unit_row + x]; + if mask == 0 { + continue; + } + let top_idx = (base_y + COEFFICIENTS_PADDING as usize) * padded_width + x + COEFFICIENTS_PADDING as usize; - if stripe_height == 4 - && cleanup_run_length_candidate(ctx, top_idx, padded_width, base_y) - { - let bit = decoder.read_bit(ctx.arithmetic_decoder_context(17))?; - if bit == 0 { + while mask != 0 { + let bit_y = mask.trailing_zeros() as usize; + mask &= mask - 1; + if bit_y >= stripe_height { continue; } - let first_significant = - (decoder.read_bit(ctx.arithmetic_decoder_context(18))? << 1) - | decoder.read_bit(ctx.arithmetic_decoder_context(18))?; - let first_significant = first_significant as usize; - let significant_y = base_y + first_significant; - let significant_idx = top_idx + first_significant * padded_width; - ctx.push_magnitude_bit_index(significant_idx, 1); - decode_sign_bit_bypass(significant_idx, significant_y, ctx, decoder)?; - ctx.set_significant_index(significant_idx, padded_width); + let y = base_y + bit_y; + let idx = top_idx + bit_y * padded_width; + let state = ctx.coefficient_states[idx].0; - let mut idx = significant_idx + padded_width; - for y in significant_y + 1..y_end { - cleanup_coefficient_bypass(ctx, decoder, idx, y, padded_width)?; - idx += padded_width; - } - continue; - } + debug_assert!(state & SIGNIFICANCE_MASK != 0); + debug_assert!(state & HAS_ZERO_CODING_MASK == 0); - let mut idx = top_idx; - for y in base_y..y_end { - cleanup_coefficient_bypass(ctx, decoder, idx, y, padded_width)?; - idx += padded_width; + let neighbors = ctx.neighborhood_significance_states_index(idx, y); + let ctx_label = + context_label_magnitude_refinement_coding_from_state(state, neighbors); + let bit = decoder.read_bit(ctx.arithmetic_decoder_context(ctx_label))?; + ctx.push_magnitude_bit_index(idx, bit); + ctx.coefficient_states[idx].0 |= HAS_MAGNITUDE_REFINEMENT_MASK; } } } Some(()) } +} - fn significance_propagation_pass_bypass( - ctx: &mut BitPlaneDecodeContext, - decoder: &mut BypassDecoder<'_>, - ) -> Option<()> { - let width = ctx.width as usize; - let height = ctx.height as usize; - let padded_width = ctx.padded_width as usize; +fn cleanup_pass_arithmetic_with_neighbors( + ctx: &mut BitPlaneDecodeContext, + decoder: &mut ArithmeticDecoder<'_>, +) { + let width = ctx.width as usize; + let height = ctx.height as usize; + let padded_width = ctx.padded_width as usize; - for base_y in (0..height).step_by(4) { - let y_end = (base_y + 4).min(height); - for x in 0..width { - let mut idx = (base_y + COEFFICIENTS_PADDING as usize) * padded_width - + x - + COEFFICIENTS_PADDING as usize; + for (stripe, base_y) in (0..height).step_by(4).enumerate() { + let y_end = (base_y + 4).min(height); + let stripe_height = y_end - base_y; + let valid_mask = scan_unit_valid_mask(stripe_height); + let scan_unit_row = stripe * width; + + for x in 0..width { + let scan_unit = scan_unit_row + x; + let candidate_mask = cleanup_candidate_scan_mask(ctx, scan_unit, stripe_height); + if candidate_mask == 0 { + continue; + } - for y in base_y..y_end { - let state = ctx.coefficient_states[idx].0; - let neighbors = ctx.neighborhood_significance_states_index(idx, y); + let top_idx = (base_y + COEFFICIENTS_PADDING as usize) * padded_width + + x + + COEFFICIENTS_PADDING as usize; - if state & SIGNIFICANCE_MASK == 0 && neighbors != 0 { - let ctx_label = - context_label_zero_coding_from_neighbors(neighbors, ctx.sub_band_type); - let bit = decoder.read_bit(ctx.arithmetic_decoder_context(ctx_label))?; - ctx.push_magnitude_bit_index(idx, bit); - ctx.coefficient_states[idx].0 |= HAS_ZERO_CODING_MASK; + if candidate_mask == valid_mask + && stripe_height == 4 + && cleanup_run_length_candidate_with_neighbors::( + ctx, + top_idx, + padded_width, + base_y, + ) + { + // The four contiguous samples are all cleanup candidates + // with zero context, so Annex D permits the RLC context. + let bit = decoder.read_bit(ctx.arithmetic_decoder_context(17)); + if bit == 0 { + continue; + } - if bit == 1 { - decode_sign_bit_bypass(idx, y, ctx, decoder)?; - ctx.set_significant_index(idx, padded_width); - } - } + let first_significant = (decoder.read_bit(ctx.arithmetic_decoder_context(18)) << 1) + | decoder.read_bit(ctx.arithmetic_decoder_context(18)); + let first_significant = first_significant as usize; + let significant_y = base_y + first_significant; + let significant_idx = top_idx + first_significant * padded_width; + ctx.push_magnitude_bit_index(significant_idx, 1); + decode_sign_bit_arithmetic_with_neighbors::( + significant_idx, + significant_y, + ctx, + decoder, + ); + ctx.set_significant_index_for_path::( + significant_idx, + padded_width, + ); + let mut idx = significant_idx + padded_width; + for y in significant_y + 1..y_end { + cleanup_coefficient_arithmetic_with_neighbors::( + ctx, + decoder, + idx, + y, + padded_width, + ); idx += padded_width; } + continue; } - } - Some(()) + let mut mask = candidate_mask; + while mask != 0 { + let bit_y = mask.trailing_zeros() as usize; + mask &= mask - 1; + let y = base_y + bit_y; + let idx = top_idx + bit_y * padded_width; + cleanup_coefficient_arithmetic_with_neighbors::( + ctx, + decoder, + idx, + y, + padded_width, + ); + } + } } +} - fn magnitude_refinement_pass_bypass( - ctx: &mut BitPlaneDecodeContext, - decoder: &mut BypassDecoder<'_>, - ) -> Option<()> { - let width = ctx.width as usize; - let height = ctx.height as usize; - let padded_width = ctx.padded_width as usize; +#[inline(always)] +fn cleanup_candidate_scan_mask( + ctx: &BitPlaneDecodeContext, + scan_unit: usize, + stripe_height: usize, +) -> u8 { + scan_unit_valid_mask(stripe_height) + & !(ctx.significant_scan_masks[scan_unit] | ctx.zero_coding_scan_masks[scan_unit]) +} - for base_y in (0..height).step_by(4) { - let y_end = (base_y + 4).min(height); - for x in 0..width { - let mut idx = (base_y + COEFFICIENTS_PADDING as usize) * padded_width - + x - + COEFFICIENTS_PADDING as usize; +#[inline(always)] +fn scan_unit_valid_mask(stripe_height: usize) -> u8 { + (1u8 << stripe_height) - 1 +} - for y in base_y..y_end { - let state = ctx.coefficient_states[idx].0; +fn significance_propagation_pass_arithmetic_with_neighbors( + ctx: &mut BitPlaneDecodeContext, + decoder: &mut ArithmeticDecoder<'_>, +) { + let width = ctx.width as usize; + let height = ctx.height as usize; + let padded_width = ctx.padded_width as usize; - if state & SIGNIFICANCE_MASK != 0 && state & HAS_ZERO_CODING_MASK == 0 { - let neighbors = ctx.neighborhood_significance_states_index(idx, y); - let ctx_label = - context_label_magnitude_refinement_coding_from_state(state, neighbors); - let bit = decoder.read_bit(ctx.arithmetic_decoder_context(ctx_label))?; - ctx.push_magnitude_bit_index(idx, bit); - ctx.coefficient_states[idx].0 |= HAS_MAGNITUDE_REFINEMENT_MASK; + for base_y in (0..height).step_by(4) { + let y_end = (base_y + 4).min(height); + for x in 0..width { + let mut idx = (base_y + COEFFICIENTS_PADDING as usize) * padded_width + + x + + COEFFICIENTS_PADDING as usize; + + for y in base_y..y_end { + let state = ctx.coefficient_states[idx].0; + let neighbors = + neighborhood_significance_states_for_path::(ctx, idx, y); + + // "The significance propagation pass only includes bits of coefficients + // that were insignificant (the significance state has yet to be set) + // and have a non-zero context." + if state & SIGNIFICANCE_MASK == 0 && neighbors != 0 { + let ctx_label = + context_label_zero_coding_from_neighbors(neighbors, ctx.sub_band_type); + let bit = decoder.read_bit(ctx.arithmetic_decoder_context(ctx_label)); + ctx.push_magnitude_bit_index(idx, bit); + ctx.set_zero_coding_index(idx, padded_width); + + // "If the value of this bit is 1 then the significance + // state is set to 1 and the immediate next bit to be decoded is + // the sign bit for the coefficient. Otherwise, the significance + // state remains 0." + if bit == 1 { + decode_sign_bit_arithmetic_with_neighbors::( + idx, y, ctx, decoder, + ); + ctx.set_significant_index_for_path::(idx, padded_width); } + } - idx += padded_width; + idx += padded_width; + } + } + } +} + +fn magnitude_refinement_pass_arithmetic_with_neighbors( + ctx: &mut BitPlaneDecodeContext, + decoder: &mut ArithmeticDecoder<'_>, +) { + let width = ctx.width as usize; + let height = ctx.height as usize; + let padded_width = ctx.padded_width as usize; + + for (stripe, base_y) in (0..height).step_by(4).enumerate() { + let stripe_height = (base_y + 4).min(height) - base_y; + let scan_unit_row = stripe * width; + + for x in 0..width { + let mut mask = ctx.significant_scan_masks[scan_unit_row + x] + & !ctx.zero_coding_scan_masks[scan_unit_row + x]; + if mask == 0 { + continue; + } + + let top_idx = (base_y + COEFFICIENTS_PADDING as usize) * padded_width + + x + + COEFFICIENTS_PADDING as usize; + + while mask != 0 { + let bit_y = mask.trailing_zeros() as usize; + mask &= mask - 1; + if bit_y >= stripe_height { + continue; } + + let y = base_y + bit_y; + let idx = top_idx + bit_y * padded_width; + let state = ctx.coefficient_states[idx].0; + + debug_assert!(state & SIGNIFICANCE_MASK != 0); + debug_assert!(state & HAS_ZERO_CODING_MASK == 0); + + let ctx_label = + context_label_magnitude_refinement_coding_from_state_lazy(state, || { + neighborhood_significance_states_for_path::(ctx, idx, y) + }); + let bit = decoder.read_bit(ctx.arithmetic_decoder_context(ctx_label)); + ctx.push_magnitude_bit_index(idx, bit); + ctx.coefficient_states[idx].0 |= HAS_MAGNITUDE_REFINEMENT_MASK; } } + } +} - Some(()) +#[inline(always)] +fn neighborhood_significance_states_for_path( + ctx: &BitPlaneDecodeContext, + idx: usize, + y: usize, +) -> u8 { + if NORMAL_NEIGHBORS { + ctx.normal_neighborhood_significance_states_index(idx) + } else { + ctx.neighborhood_significance_states_index(idx, y) } } #[inline(always)] -fn cleanup_run_length_candidate( +fn cleanup_run_length_candidate_with_neighbors( ctx: &BitPlaneDecodeContext, top_idx: usize, padded_width: usize, @@ -996,7 +1352,7 @@ fn cleanup_run_length_candidate( let mut idx = top_idx; for y in base_y..base_y + 4 { if ctx.coefficient_states[idx].0 & (SIGNIFICANCE_MASK | HAS_ZERO_CODING_MASK) != 0 - || ctx.neighborhood_significance_states_index(idx, y) != 0 + || neighborhood_significance_states_for_path::(ctx, idx, y) != 0 { return false; } @@ -1007,7 +1363,7 @@ fn cleanup_run_length_candidate( } #[inline(always)] -fn cleanup_coefficient_arithmetic( +fn cleanup_coefficient_arithmetic_with_neighbors( ctx: &mut BitPlaneDecodeContext, decoder: &mut ArithmeticDecoder<'_>, idx: usize, @@ -1015,18 +1371,28 @@ fn cleanup_coefficient_arithmetic( padded_width: usize, ) { if ctx.coefficient_states[idx].0 & (SIGNIFICANCE_MASK | HAS_ZERO_CODING_MASK) == 0 { - let neighbors = ctx.neighborhood_significance_states_index(idx, y); + let neighbors = neighborhood_significance_states_for_path::(ctx, idx, y); let ctx_label = context_label_zero_coding_from_neighbors(neighbors, ctx.sub_band_type); let bit = decoder.read_bit(ctx.arithmetic_decoder_context(ctx_label)); ctx.push_magnitude_bit_index(idx, bit); if bit == 1 { - decode_sign_bit_arithmetic(idx, y, ctx, decoder); - ctx.set_significant_index(idx, padded_width); + decode_sign_bit_arithmetic_with_neighbors::(idx, y, ctx, decoder); + ctx.set_significant_index_for_path::(idx, padded_width); } } } +#[inline(always)] +fn cleanup_run_length_candidate( + ctx: &BitPlaneDecodeContext, + top_idx: usize, + padded_width: usize, + base_y: usize, +) -> bool { + cleanup_run_length_candidate_with_neighbors::(ctx, top_idx, padded_width, base_y) +} + #[inline(always)] fn cleanup_coefficient_bypass( ctx: &mut BitPlaneDecodeContext, @@ -1127,15 +1493,15 @@ const ZERO_CTX_HH_LOOKUP: [u8; 256] = [ 8, 8, 8, 8, 8, 8, ]; -/// Decode a sign bit (Section D.3.2). #[inline(always)] -fn decode_sign_bit_arithmetic( +fn decode_sign_bit_arithmetic_with_neighbors( idx: usize, y: usize, ctx: &mut BitPlaneDecodeContext, decoder: &mut ArithmeticDecoder<'_>, ) { - let (ctx_label, xor_bit) = context_label_sign_coding_index(idx, y, ctx); + let (ctx_label, xor_bit) = + context_label_sign_coding_index_with_neighbors::(idx, y, ctx); let sign_bit = decoder.read_bit(ctx.arithmetic_decoder_context(ctx_label)) ^ xor_bit as u32; ctx.set_sign_index(idx, sign_bit as u8); } @@ -1188,6 +1554,37 @@ fn context_label_sign_coding_index(idx: usize, y: usize, ctx: &BitPlaneDecodeCon SIGN_CONTEXT_LOOKUP[merged_significances as usize] } +#[inline(always)] +fn context_label_sign_coding_index_with_neighbors( + idx: usize, + y: usize, + ctx: &BitPlaneDecodeContext, +) -> (u8, u8) { + if NORMAL_NEIGHBORS { + context_label_sign_coding_index_normal(idx, ctx) + } else { + context_label_sign_coding_index(idx, y, ctx) + } +} + +#[inline(always)] +fn context_label_sign_coding_index_normal(idx: usize, ctx: &BitPlaneDecodeContext) -> (u8, u8) { + let significances = ctx.normal_neighborhood_significance_states_index(idx) & 0b0101_0101; + let padded_width = ctx.padded_width as usize; + + let top_sign = ctx.sign_index(idx - padded_width); + let left_sign = ctx.sign_index(idx - 1); + let right_sign = ctx.sign_index(idx + 1); + let bottom_sign = ctx.sign_index(idx + padded_width); + + let signs = (top_sign << 6) | (left_sign << 4) | (right_sign << 2) | bottom_sign; + let negative_significances = significances & signs; + let positive_significances = significances & !signs; + let merged_significances = (negative_significances << 1) | positive_significances; + + SIGN_CONTEXT_LOOKUP[merged_significances as usize] +} + /// Return the context label for zero coding (Section D.3.1). #[inline(always)] fn context_label_zero_coding_from_neighbors(neighbors: u8, sub_band_type: SubBandType) -> u8 { @@ -1204,12 +1601,21 @@ fn context_label_zero_coding_from_neighbors(neighbors: u8, sub_band_type: SubBan /// Return the context label for magnitude refinement coding (Table D.4). #[inline(always)] fn context_label_magnitude_refinement_coding_from_state(state: u8, neighbors: u8) -> u8 { - // If magnitude refined, then 16. - let m1 = ((state & HAS_MAGNITUDE_REFINEMENT_MASK) >> HAS_MAGNITUDE_REFINEMENT_SHIFT) * 16; - // Else: If at least one neighbor is significant then 15, else 14. - let m2 = 14 + neighbors.min(1); + context_label_magnitude_refinement_coding_from_state_lazy(state, || neighbors) +} - u8::max(m1, m2) +#[inline(always)] +fn context_label_magnitude_refinement_coding_from_state_lazy( + state: u8, + neighbors: impl FnOnce() -> u8, +) -> u8 { + // If magnitude refined, then 16. + if state & HAS_MAGNITUDE_REFINEMENT_MASK != 0 { + 16 + } else { + // Else: If at least one neighbor is significant then 15, else 14. + 14 + neighbors().min(1) + } } // Bypass bit reads can fail in strict mode when the raw segment runs short. @@ -1398,6 +1804,198 @@ mod tests { } } + #[test] + fn normal_neighborhood_significance_fast_path_returns_unmasked_neighbors() { + let mut ctx = BitPlaneDecodeContext { + width: 1, + height: 8, + padded_width: 3, + style: CodeBlockStyle { + vertically_causal_context: true, + ..CodeBlockStyle::default() + }, + ..BitPlaneDecodeContext::default() + }; + ctx.neighbor_significances.resize( + ctx.padded_width as usize * 10, + NeighborSignificances::default(), + ); + + let y = 3; + let idx = (y + COEFFICIENTS_PADDING as usize) * ctx.padded_width as usize + + COEFFICIENTS_PADDING as usize; + ctx.neighbor_significances[idx].set_top(); + ctx.neighbor_significances[idx].set_bottom(); + + assert_eq!(ctx.neighborhood_significance_states_index(idx, y), 1 << 6); + assert_eq!( + ctx.normal_neighborhood_significance_states_index(idx), + (1 << 6) | 1 + ); + } + + #[test] + fn normal_sign_context_matches_generic_non_vertical_context() { + let mut ctx = BitPlaneDecodeContext { + width: 3, + height: 3, + padded_width: 5, + style: CodeBlockStyle::default(), + ..BitPlaneDecodeContext::default() + }; + let len = ctx.padded_width as usize * (ctx.height as usize + 2); + ctx.coefficients.resize(len, Coefficient::default()); + ctx.neighbor_significances + .resize(len, NeighborSignificances::default()); + + let y = 1; + let idx = (y + COEFFICIENTS_PADDING as usize) * ctx.padded_width as usize + + COEFFICIENTS_PADDING as usize + + 1; + let padded_width = ctx.padded_width as usize; + + ctx.neighbor_significances[idx].set_top(); + ctx.neighbor_significances[idx].set_left(); + ctx.neighbor_significances[idx].set_right(); + ctx.neighbor_significances[idx].set_bottom(); + ctx.set_sign_index(idx - padded_width, 1); + ctx.set_sign_index(idx - 1, 0); + ctx.set_sign_index(idx + 1, 1); + ctx.set_sign_index(idx + padded_width, 0); + + assert_eq!( + context_label_sign_coding_index_normal(idx, &ctx), + context_label_sign_coding_index(idx, y, &ctx) + ); + } + + #[test] + fn normal_set_significant_index_matches_generic_neighbor_updates() { + let mut generic = BitPlaneDecodeContext { + width: 3, + height: 3, + padded_width: 5, + significant_scan_masks: vec![0; 3], + zero_coding_scan_masks: vec![0; 3], + style: CodeBlockStyle::default(), + ..BitPlaneDecodeContext::default() + }; + let len = generic.padded_width as usize * (generic.height as usize + 2); + generic + .coefficient_states + .resize(len, CoefficientState::default()); + generic + .neighbor_significances + .resize(len, NeighborSignificances::default()); + let mut normal = BitPlaneDecodeContext { + width: generic.width, + height: generic.height, + padded_width: generic.padded_width, + style: generic.style, + coefficient_states: generic.coefficient_states.clone(), + significant_scan_masks: vec![0; 3], + zero_coding_scan_masks: vec![0; 3], + neighbor_significances: generic.neighbor_significances.clone(), + ..BitPlaneDecodeContext::default() + }; + + let padded_width = generic.padded_width as usize; + let idx = + (1 + COEFFICIENTS_PADDING as usize) * padded_width + COEFFICIENTS_PADDING as usize + 1; + + generic.set_significant_index(idx, padded_width); + normal.set_significant_index_normal(idx, padded_width); + + assert_eq!( + normal.coefficient_states[idx].0, + generic.coefficient_states[idx].0 + ); + assert_eq!( + normal + .neighbor_significances + .iter() + .map(|neighbors| neighbors.0) + .collect::>(), + generic + .neighbor_significances + .iter() + .map(|neighbors| neighbors.0) + .collect::>() + ); + } + + #[test] + fn refined_magnitude_context_does_not_require_neighbor_state() { + let state = SIGNIFICANCE_MASK | HAS_MAGNITUDE_REFINEMENT_MASK; + + assert_eq!( + context_label_magnitude_refinement_coding_from_state_lazy(state, || { + panic!("refined magnitude context should not inspect neighbors") + }), + 16 + ); + } + + #[test] + fn first_magnitude_context_uses_neighbor_presence() { + assert_eq!( + context_label_magnitude_refinement_coding_from_state_lazy(SIGNIFICANCE_MASK, || 0), + 14 + ); + assert_eq!( + context_label_magnitude_refinement_coding_from_state_lazy(SIGNIFICANCE_MASK, || 1), + 15 + ); + } + + #[test] + fn scan_unit_masks_track_significance_and_current_bitplane_zero_coding() { + let mut ctx = BitPlaneDecodeContext::default(); + let style = CodeBlockStyle::default(); + ctx.reset_for_job(5, 6, 0, 4, SubBandType::LowLow, &style, 8, true) + .expect("reset context"); + + let padded_width = ctx.padded_width as usize; + let y = 4usize; + let x = 3usize; + let idx = + (y + COEFFICIENTS_PADDING as usize) * padded_width + x + COEFFICIENTS_PADDING as usize; + let scan_unit = (y >> 2) * ctx.width as usize + x; + let bit = 1u8 << (y & 3); + + ctx.set_significant_index(idx, padded_width); + ctx.set_zero_coding_index(idx, padded_width); + + assert_eq!(ctx.significant_scan_masks[scan_unit], bit); + assert_eq!(ctx.zero_coding_scan_masks[scan_unit], bit); + + ctx.reset_for_next_bitplane(); + + assert_eq!(ctx.significant_scan_masks[scan_unit], bit); + assert_eq!(ctx.zero_coding_scan_masks[scan_unit], 0); + } + + #[test] + fn cleanup_candidate_mask_excludes_significant_and_zero_coded_coefficients() { + let mut ctx = BitPlaneDecodeContext::default(); + let style = CodeBlockStyle::default(); + ctx.reset_for_job(5, 6, 0, 4, SubBandType::LowLow, &style, 8, true) + .expect("reset context"); + + let padded_width = ctx.padded_width as usize; + let significant_idx = + (1 + COEFFICIENTS_PADDING as usize) * padded_width + COEFFICIENTS_PADDING as usize + 2; + let zero_coded_idx = + (3 + COEFFICIENTS_PADDING as usize) * padded_width + COEFFICIENTS_PADDING as usize + 2; + let scan_unit = 2; + + ctx.set_significant_index(significant_idx, padded_width); + ctx.set_zero_coding_index(zero_coded_idx, padded_width); + + assert_eq!(cleanup_candidate_scan_mask(&ctx, scan_unit, 4), 0b0101); + assert_eq!(cleanup_candidate_scan_mask(&ctx, scan_unit, 2), 0b0001); + } + #[test] fn classic_bitplane_round_trips_subband_and_style_matrix() { let styles = [ diff --git a/crates/signinum-j2k-native/src/j2c/bitplane_encode.rs b/crates/j2k-native/src/j2c/bitplane_encode.rs similarity index 69% rename from crates/signinum-j2k-native/src/j2c/bitplane_encode.rs rename to crates/j2k-native/src/j2c/bitplane_encode.rs index 019bf422..841849f4 100644 --- a/crates/signinum-j2k-native/src/j2c/bitplane_encode.rs +++ b/crates/j2k-native/src/j2c/bitplane_encode.rs @@ -29,6 +29,10 @@ pub(crate) struct EncodedCodeBlock { pub(crate) num_coding_passes: u8, /// Number of leading zero bitplanes (missing MSBs). pub(crate) num_zero_bitplanes: u8, + /// HTJ2K cleanup segment length in bytes when this block uses HT coding. + pub(crate) ht_cleanup_length: u32, + /// HTJ2K refinement segment length in bytes when this block uses HT coding. + pub(crate) ht_refinement_length: u32, } #[derive(Debug, Clone, Copy)] @@ -37,6 +41,7 @@ pub(crate) struct EncodedCodeBlockSegment { pub(crate) data_length: u32, pub(crate) start_coding_pass: u8, pub(crate) end_coding_pass: u8, + pub(crate) distortion_delta: f64, pub(crate) use_arithmetic: bool, } @@ -48,6 +53,126 @@ pub(crate) struct EncodedCodeBlockWithSegments { pub(crate) num_zero_bitplanes: u8, } +#[derive(Debug, Clone, Copy)] +pub(crate) struct ClassicTier1TokenSegment { + pub(crate) token_bit_offset: u32, + pub(crate) token_bit_count: u32, + pub(crate) start_coding_pass: u8, + pub(crate) end_coding_pass: u8, + pub(crate) use_arithmetic: bool, +} + +pub(crate) fn pack_classic_selective_bypass_tier1_tokens( + token_bytes: &[u8], + token_segments: &[ClassicTier1TokenSegment], + number_of_coding_passes: u8, + missing_bit_planes: u8, +) -> Result { + let mut reader = ClassicTier1TokenReader::new(token_bytes); + let mut contexts = [ArithmeticEncoderContext::default(); 19]; + reset_contexts(&mut contexts); + let mut data = Vec::new(); + let mut segments = Vec::with_capacity(token_segments.len()); + + for segment in token_segments { + if segment.start_coding_pass > segment.end_coding_pass { + return Err("classic Tier-1 token segment pass range is invalid"); + } + if segment.end_coding_pass > number_of_coding_passes { + return Err("classic Tier-1 token segment exceeds coding passes"); + } + let token_bit_offset = usize::try_from(segment.token_bit_offset) + .map_err(|_| "classic Tier-1 token bit offset exceeds usize")?; + let token_bit_count = usize::try_from(segment.token_bit_count) + .map_err(|_| "classic Tier-1 token bit count exceeds usize")?; + reader.seek(token_bit_offset)?; + if segment.use_arithmetic { + if token_bit_count % 6 != 0 { + return Err("classic Tier-1 MQ token segment is not aligned to 6-bit symbols"); + } + let symbol_count = token_bit_count / 6; + let mut encoder = + ArithmeticEncoder::with_capacity(symbol_count.saturating_div(16) + 32); + for _ in 0..symbol_count { + let token = reader.read_bits(6)?; + let ctx = (token & 0x1F) as usize; + if ctx >= contexts.len() { + return Err("classic Tier-1 MQ token context is out of range"); + } + let bit = (token >> 5) & 1; + encoder.encode(bit, &mut contexts[ctx]); + } + push_segment( + &mut data, + &mut segments, + segment.start_coding_pass, + segment.end_coding_pass, + encoder.finish(), + f64::EPSILON, + true, + ); + } else { + let mut writer = BitWriter::new(); + for _ in 0..token_bit_count { + writer.write_bit(reader.read_bits(1)?); + } + push_segment( + &mut data, + &mut segments, + segment.start_coding_pass, + segment.end_coding_pass, + writer.finish(), + f64::EPSILON, + false, + ); + } + } + + Ok(EncodedCodeBlockWithSegments { + data, + segments, + num_coding_passes: number_of_coding_passes, + num_zero_bitplanes: missing_bit_planes, + }) +} + +struct ClassicTier1TokenReader<'a> { + bytes: &'a [u8], + bit_pos: usize, +} + +impl<'a> ClassicTier1TokenReader<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { bytes, bit_pos: 0 } + } + + fn seek(&mut self, bit_pos: usize) -> Result<(), &'static str> { + if bit_pos > self.bytes.len().saturating_mul(8) { + return Err("classic Tier-1 token offset exceeds token buffer"); + } + self.bit_pos = bit_pos; + Ok(()) + } + + fn read_bits(&mut self, count: u8) -> Result { + let end = self + .bit_pos + .checked_add(usize::from(count)) + .ok_or("classic Tier-1 token bit range overflows")?; + if end > self.bytes.len().saturating_mul(8) { + return Err("classic Tier-1 token read exceeds token buffer"); + } + let mut value = 0u32; + for _ in 0..count { + let byte = self.bytes[self.bit_pos / 8]; + let shift = 7 - (self.bit_pos % 8); + value = (value << 1) | u32::from((byte >> shift) & 1); + self.bit_pos += 1; + } + Ok(value) + } +} + /// Context labels for zero coding (Table D.1). /// Index into 256-entry lookup tables by neighbor significance pattern. #[rustfmt::skip] @@ -204,6 +329,8 @@ pub(crate) fn encode_code_block_with_style( data: Vec::new(), num_coding_passes: 0, num_zero_bitplanes: total_bitplanes, + ht_cleanup_length: 0, + ht_refinement_length: 0, }; } @@ -216,7 +343,8 @@ pub(crate) fn encode_code_block_with_style( let (magnitudes, mut states) = prepare_padded_coefficients(coefficients, w, h, pw); let mut neighbors = vec![0u8; magnitudes.len()]; // Packed neighbor significances - let mut encoder = ArithmeticEncoder::new(); + let mut encoder = + ArithmeticEncoder::with_capacity(arithmetic_encoder_capacity(w, h, num_bitplanes as usize)); let mut contexts = [ArithmeticEncoderContext::default(); 19]; reset_contexts(&mut contexts); @@ -319,6 +447,8 @@ pub(crate) fn encode_code_block_with_style( data, num_coding_passes, num_zero_bitplanes, + ht_cleanup_length: 0, + ht_refinement_length: 0, } } @@ -348,6 +478,12 @@ pub(crate) fn encode_code_block_segments_with_style( .expect("classic code-block payload length fits in u32"), start_coding_pass: 0, end_coding_pass: encoded.num_coding_passes, + distortion_delta: segment_distortion_delta( + coefficients, + 0, + encoded.num_coding_passes, + total_bitplanes, + ), use_arithmetic: true, }] }; @@ -422,6 +558,12 @@ pub(crate) fn encode_code_block_segments_with_style( .take() .expect("arithmetic segment encoder exists") .finish(), + segment_distortion_delta( + coefficients, + current_segment_start_pass, + coding_pass, + num_bitplanes as u8, + ), true, ); } else { @@ -434,6 +576,12 @@ pub(crate) fn encode_code_block_segments_with_style( .take() .expect("bypass segment writer exists") .finish(), + segment_distortion_delta( + coefficients, + current_segment_start_pass, + coding_pass, + num_bitplanes as u8, + ), false, ); } @@ -563,6 +711,12 @@ pub(crate) fn encode_code_block_segments_with_style( .take() .expect("final arithmetic segment encoder exists") .finish(), + segment_distortion_delta( + coefficients, + current_segment_start_pass, + total_passes, + num_bitplanes as u8, + ), true, ); } else { @@ -575,6 +729,12 @@ pub(crate) fn encode_code_block_segments_with_style( .take() .expect("final bypass segment writer exists") .finish(), + segment_distortion_delta( + coefficients, + current_segment_start_pass, + total_passes, + num_bitplanes as u8, + ), false, ); } @@ -595,6 +755,15 @@ fn reset_contexts(contexts: &mut [ArithmeticEncoderContext; 19]) { contexts[18].reset_with_index(46); } +fn arithmetic_encoder_capacity(width: usize, height: usize, bitplanes: usize) -> usize { + 1 + width + .saturating_mul(height) + .saturating_mul(bitplanes) + .checked_div(16) + .unwrap_or(usize::MAX) + .max(32) +} + fn encode_segmentation_symbols( encoder: &mut ArithmeticEncoder, contexts: &mut [ArithmeticEncoderContext; 19], @@ -620,6 +789,7 @@ fn push_segment( start_coding_pass: u8, end_coding_pass: u8, segment_data: Vec, + distortion_delta: f64, use_arithmetic: bool, ) { let data_offset = @@ -632,10 +802,63 @@ fn push_segment( data_length, start_coding_pass, end_coding_pass, + distortion_delta, use_arithmetic, }); } +fn segment_distortion_delta( + coefficients: &[i32], + start_coding_pass: u8, + end_coding_pass: u8, + num_bitplanes: u8, +) -> f64 { + let before = + coefficient_distortion_after_passes(coefficients, start_coding_pass, num_bitplanes); + let after = coefficient_distortion_after_passes(coefficients, end_coding_pass, num_bitplanes); + (before - after).max(f64::EPSILON) +} + +fn coefficient_distortion_after_passes( + coefficients: &[i32], + completed_passes: u8, + num_bitplanes: u8, +) -> f64 { + coefficients + .iter() + .map(|coefficient| { + let magnitude = coefficient.unsigned_abs(); + let reconstructed = + reconstructed_magnitude_after_passes(magnitude, completed_passes, num_bitplanes); + let error = f64::from(magnitude.saturating_sub(reconstructed)); + error * error + }) + .sum() +} + +fn reconstructed_magnitude_after_passes( + magnitude: u32, + completed_passes: u8, + num_bitplanes: u8, +) -> u32 { + if magnitude == 0 || completed_passes == 0 || num_bitplanes == 0 { + return 0; + } + + let deepest_coded_bitplane = completed_passes + .saturating_sub(1) + .div_ceil(3) + .min(num_bitplanes.saturating_sub(1)); + let retained_bitplanes = deepest_coded_bitplane.saturating_add(1); + if retained_bitplanes >= num_bitplanes { + return magnitude; + } + + let lower_bits = u32::from(num_bitplanes - retained_bitplanes); + let mask = !((1u32 << lower_bits) - 1); + magnitude & mask +} + fn mark_coded_in_current_pass(idx: usize, states: &mut [u8], coded_indices: &mut Vec) { if states[idx] & CODED_IN_CURRENT_PASS == 0 { states[idx] |= CODED_IN_CURRENT_PASS; @@ -663,6 +886,50 @@ fn significance_propagation_pass( bit_mask: u32, sub_band_type: SubBandType, style: &CodeBlockStyle, +) { + if style.vertically_causal_context { + significance_propagation_pass_impl::( + magnitudes, + states, + neighbors, + coded_indices, + encoder, + contexts, + w, + h, + pw, + bit_mask, + sub_band_type, + ); + } else { + significance_propagation_pass_impl::( + magnitudes, + states, + neighbors, + coded_indices, + encoder, + contexts, + w, + h, + pw, + bit_mask, + sub_band_type, + ); + } +} + +fn significance_propagation_pass_impl( + magnitudes: &[u32], + states: &mut [u8], + neighbors: &mut [u8], + coded_indices: &mut Vec, + encoder: &mut ArithmeticEncoder, + contexts: &mut [ArithmeticEncoderContext; 19], + w: usize, + h: usize, + pw: usize, + bit_mask: u32, + sub_band_type: SubBandType, ) { for y_base in (0..h).step_by(4) { for x in 0..w { @@ -670,7 +937,7 @@ fn significance_propagation_pass( for y in y_base..y_end { let idx = (y + 1) * pw + (x + 1); let is_significant = states[idx] & SIGNIFICANT != 0; - let neighbor_sig = effective_neighbor_sig(neighbors[idx], y, h, style); + let neighbor_sig = effective_neighbor_sig::(neighbors[idx], y, h); let has_sig_neighbors = neighbor_sig != 0; if !is_significant && has_sig_neighbors { @@ -680,7 +947,9 @@ fn significance_propagation_pass( mark_coded_in_current_pass(idx, states, coded_indices); if bit == 1 { - encode_sign(idx, neighbors, states, encoder, contexts, pw, y, h, style); + encode_sign::( + idx, neighbors, states, encoder, contexts, pw, y, h, + ); set_significant(idx, states, neighbors, pw); } } @@ -700,6 +969,44 @@ fn significance_propagation_pass_raw( pw: usize, bit_mask: u32, style: &CodeBlockStyle, +) { + if style.vertically_causal_context { + significance_propagation_pass_raw_impl::( + magnitudes, + states, + neighbors, + coded_indices, + writer, + w, + h, + pw, + bit_mask, + ); + } else { + significance_propagation_pass_raw_impl::( + magnitudes, + states, + neighbors, + coded_indices, + writer, + w, + h, + pw, + bit_mask, + ); + } +} + +fn significance_propagation_pass_raw_impl( + magnitudes: &[u32], + states: &mut [u8], + neighbors: &mut [u8], + coded_indices: &mut Vec, + writer: &mut BitWriter, + w: usize, + h: usize, + pw: usize, + bit_mask: u32, ) { for y_base in (0..h).step_by(4) { for x in 0..w { @@ -707,7 +1014,7 @@ fn significance_propagation_pass_raw( for y in y_base..y_end { let idx = (y + 1) * pw + (x + 1); let is_significant = states[idx] & SIGNIFICANT != 0; - let neighbor_sig = effective_neighbor_sig(neighbors[idx], y, h, style); + let neighbor_sig = effective_neighbor_sig::(neighbors[idx], y, h); if !is_significant && neighbor_sig != 0 { let bit = (magnitudes[idx] & bit_mask != 0) as u32; writer.write_bit(bit); @@ -734,6 +1041,28 @@ fn magnitude_refinement_pass( pw: usize, bit_mask: u32, style: &CodeBlockStyle, +) { + if style.vertically_causal_context { + magnitude_refinement_pass_impl::( + magnitudes, states, neighbors, encoder, contexts, w, h, pw, bit_mask, + ); + } else { + magnitude_refinement_pass_impl::( + magnitudes, states, neighbors, encoder, contexts, w, h, pw, bit_mask, + ); + } +} + +fn magnitude_refinement_pass_impl( + magnitudes: &[u32], + states: &mut [u8], + neighbors: &mut [u8], + encoder: &mut ArithmeticEncoder, + contexts: &mut [ArithmeticEncoderContext; 19], + w: usize, + h: usize, + pw: usize, + bit_mask: u32, ) { for y_base in (0..h).step_by(4) { for x in 0..w { @@ -746,7 +1075,7 @@ fn magnitude_refinement_pass( if is_significant && !coded_this_pass { let ctx_label = magnitude_refinement_ctx( states[idx], - effective_neighbor_sig(neighbors[idx], y, h, style), + effective_neighbor_sig::(neighbors[idx], y, h), ); let bit = (magnitudes[idx] & bit_mask != 0) as u32; encoder.encode(bit, &mut contexts[ctx_label as usize]); @@ -767,6 +1096,27 @@ fn magnitude_refinement_pass_raw( pw: usize, bit_mask: u32, style: &CodeBlockStyle, +) { + if style.vertically_causal_context { + magnitude_refinement_pass_raw_impl::( + magnitudes, states, neighbors, writer, w, h, pw, bit_mask, + ); + } else { + magnitude_refinement_pass_raw_impl::( + magnitudes, states, neighbors, writer, w, h, pw, bit_mask, + ); + } +} + +fn magnitude_refinement_pass_raw_impl( + magnitudes: &[u32], + states: &mut [u8], + neighbors: &mut [u8], + writer: &mut BitWriter, + w: usize, + h: usize, + pw: usize, + bit_mask: u32, ) { for y_base in (0..h).step_by(4) { for x in 0..w { @@ -775,7 +1125,7 @@ fn magnitude_refinement_pass_raw( let idx = (y + 1) * pw + (x + 1); let is_significant = states[idx] & SIGNIFICANT != 0; let coded_this_pass = states[idx] & CODED_IN_CURRENT_PASS != 0; - let _neighbor_sig = effective_neighbor_sig(neighbors[idx], y, h, style); + let _neighbor_sig = effective_neighbor_sig::(neighbors[idx], y, h); if is_significant && !coded_this_pass { let bit = (magnitudes[idx] & bit_mask != 0) as u32; writer.write_bit(bit); @@ -799,6 +1149,47 @@ fn cleanup_pass( bit_mask: u32, sub_band_type: SubBandType, style: &CodeBlockStyle, +) { + if style.vertically_causal_context { + cleanup_pass_impl::( + magnitudes, + states, + neighbors, + encoder, + contexts, + w, + h, + pw, + bit_mask, + sub_band_type, + ); + } else { + cleanup_pass_impl::( + magnitudes, + states, + neighbors, + encoder, + contexts, + w, + h, + pw, + bit_mask, + sub_band_type, + ); + } +} + +fn cleanup_pass_impl( + magnitudes: &[u32], + states: &mut [u8], + neighbors: &mut [u8], + encoder: &mut ArithmeticEncoder, + contexts: &mut [ArithmeticEncoderContext; 19], + w: usize, + h: usize, + pw: usize, + bit_mask: u32, + sub_band_type: SubBandType, ) { for y_base in (0..h).step_by(4) { for x in 0..w { @@ -811,7 +1202,7 @@ fn cleanup_pass( for y in y_base..y_end { let idx = (y + 1) * pw + (x + 1); if states[idx] & (SIGNIFICANT | CODED_IN_CURRENT_PASS) != 0 - || effective_neighbor_sig(neighbors[idx], y, h, style) != 0 + || effective_neighbor_sig::(neighbors[idx], y, h) != 0 { all_zero_uncoded = false; break; @@ -838,7 +1229,9 @@ fn cleanup_pass( // Encode sign for the first significant let y = y_base + pos; let idx = (y + 1) * pw + (x + 1); - encode_sign(idx, neighbors, states, encoder, contexts, pw, y, h, style); + encode_sign::( + idx, neighbors, states, encoder, contexts, pw, y, h, + ); set_significant(idx, states, neighbors, pw); // Continue cleanup for remaining samples in stripe @@ -846,14 +1239,14 @@ fn cleanup_pass( let idx = (y + 1) * pw + (x + 1); if states[idx] & (SIGNIFICANT | CODED_IN_CURRENT_PASS) == 0 { let ctx_label = zero_coding_ctx( - effective_neighbor_sig(neighbors[idx], y, h, style), + effective_neighbor_sig::(neighbors[idx], y, h), sub_band_type, ); let bit = (magnitudes[idx] & bit_mask != 0) as u32; encoder.encode(bit, &mut contexts[ctx_label as usize]); if bit == 1 { - encode_sign( - idx, neighbors, states, encoder, contexts, pw, y, h, style, + encode_sign::( + idx, neighbors, states, encoder, contexts, pw, y, h, ); set_significant(idx, states, neighbors, pw); } @@ -873,13 +1266,15 @@ fn cleanup_pass( let idx = (y + 1) * pw + (x + 1); if states[idx] & (SIGNIFICANT | CODED_IN_CURRENT_PASS) == 0 { let ctx_label = zero_coding_ctx( - effective_neighbor_sig(neighbors[idx], y, h, style), + effective_neighbor_sig::(neighbors[idx], y, h), sub_band_type, ); let bit = (magnitudes[idx] & bit_mask != 0) as u32; encoder.encode(bit, &mut contexts[ctx_label as usize]); if bit == 1 { - encode_sign(idx, neighbors, states, encoder, contexts, pw, y, h, style); + encode_sign::( + idx, neighbors, states, encoder, contexts, pw, y, h, + ); set_significant(idx, states, neighbors, pw); } } @@ -893,7 +1288,7 @@ fn cleanup_pass( /// The sign context is computed exactly as the decoder does it: /// combine significance and sign of the 4 cardinal neighbors into a /// merged byte and look up SIGN_CONTEXT_LOOKUP. -fn encode_sign( +fn encode_sign( idx: usize, neighbors: &[u8], states: &[u8], @@ -902,10 +1297,10 @@ fn encode_sign( pw: usize, y: usize, h: usize, - style: &CodeBlockStyle, ) { // Get cardinal-neighbor significances: T(6), L(4), R(2), B(0) - let significances = effective_neighbor_sig(neighbors[idx], y, h, style) & 0b0101_0101; + let significances = + effective_neighbor_sig::(neighbors[idx], y, h) & 0b0101_0101; // Get sign of each cardinal neighbor (0=positive, 1=negative). // Only meaningful for significant neighbors; insignificant neighbors get 0. @@ -924,7 +1319,7 @@ fn encode_sign( } else { 0 }; - let bottom_sign = if style.vertically_causal_context && neighbor_in_next_stripe(y, h) { + let bottom_sign = if VERTICAL_CAUSAL && neighbor_in_next_stripe(y, h) { 0 } else if states[idx + pw] & SIGNIFICANT != 0 { ((states[idx + pw] & NEGATIVE) != 0) as u8 @@ -957,9 +1352,13 @@ fn neighbor_in_next_stripe(y: usize, height: usize) -> bool { y + 1 < height && ((y + 1) >> 2) > (y >> 2) } -#[inline] -fn effective_neighbor_sig(neighbor_sig: u8, y: usize, height: usize, style: &CodeBlockStyle) -> u8 { - if style.vertically_causal_context && neighbor_in_next_stripe(y, height) { +#[inline(always)] +fn effective_neighbor_sig( + neighbor_sig: u8, + y: usize, + height: usize, +) -> u8 { + if VERTICAL_CAUSAL && neighbor_in_next_stripe(y, height) { neighbor_sig & 0b1111_0100 } else { neighbor_sig @@ -1031,6 +1430,81 @@ mod tests { assert_eq!(result.num_zero_bitplanes, 0); } + #[test] + fn pack_classic_selective_bypass_tokens_matches_scalar_single_cleanup_block() { + let style = CodeBlockStyle { + selective_arithmetic_coding_bypass: true, + reset_context_probabilities: false, + termination_on_each_pass: false, + vertically_causal_context: false, + segmentation_symbols: false, + high_throughput_block_coding: false, + }; + let coefficients = [1i32]; + let scalar = encode_code_block_segments_with_style( + &coefficients, + 1, + 1, + SubBandType::LowLow, + 1, + &style, + ); + let token_bytes = pack_mq_test_tokens(&[(0, 1), (9, 0)]); + let packed = pack_classic_selective_bypass_tier1_tokens( + &token_bytes, + &[ClassicTier1TokenSegment { + token_bit_offset: 0, + token_bit_count: 12, + start_coding_pass: 0, + end_coding_pass: 1, + use_arithmetic: true, + }], + scalar.num_coding_passes, + scalar.num_zero_bitplanes, + ) + .expect("tokens pack"); + + assert_eq!(packed.data, scalar.data); + assert_eq!(packed.num_coding_passes, scalar.num_coding_passes); + assert_eq!(packed.num_zero_bitplanes, scalar.num_zero_bitplanes); + assert_eq!(packed.segments.len(), scalar.segments.len()); + for (packed_segment, scalar_segment) in packed.segments.iter().zip(&scalar.segments) { + assert_eq!(packed_segment.data_offset, scalar_segment.data_offset); + assert_eq!(packed_segment.data_length, scalar_segment.data_length); + assert_eq!( + packed_segment.start_coding_pass, + scalar_segment.start_coding_pass + ); + assert_eq!( + packed_segment.end_coding_pass, + scalar_segment.end_coding_pass + ); + assert_eq!(packed_segment.use_arithmetic, scalar_segment.use_arithmetic); + } + } + + fn pack_mq_test_tokens(tokens: &[(u8, u8)]) -> Vec { + let mut bytes = Vec::new(); + let mut current = 0u8; + let mut bits = 0u8; + for &(ctx, bit) in tokens { + let value = (ctx & 0x1F) | ((bit & 1) << 5); + for shift in (0..6).rev() { + current = (current << 1) | ((value >> shift) & 1); + bits += 1; + if bits == 8 { + bytes.push(current); + current = 0; + bits = 0; + } + } + } + if bits != 0 { + bytes.push(current << (8 - bits)); + } + bytes + } + #[test] fn test_encode_various_magnitudes() { let coeffs: Vec = (0..64) @@ -1080,4 +1554,15 @@ mod tests { assert_eq!(states[6], SIGNIFICANT); assert!(coded_indices.is_empty()); } + + #[test] + fn pcrd_distortion_delta_reflects_residual_error_reduction() { + let sparse_delta = segment_distortion_delta(&[8], 0, 1, 4); + let dense_delta = segment_distortion_delta(&[15], 0, 1, 4); + + assert!( + dense_delta > sparse_delta, + "coefficients with the same MSB but larger residual error should have larger PCRD distortion reduction" + ); + } } diff --git a/crates/signinum-j2k-native/src/j2c/build.rs b/crates/j2k-native/src/j2c/build.rs similarity index 100% rename from crates/signinum-j2k-native/src/j2c/build.rs rename to crates/j2k-native/src/j2c/build.rs diff --git a/crates/signinum-j2k-native/src/j2c/codestream.rs b/crates/j2k-native/src/j2c/codestream.rs similarity index 77% rename from crates/signinum-j2k-native/src/j2c/codestream.rs rename to crates/j2k-native/src/j2c/codestream.rs index 52ff1f43..7eb70019 100644 --- a/crates/signinum-j2k-native/src/j2c/codestream.rs +++ b/crates/j2k-native/src/j2c/codestream.rs @@ -6,8 +6,9 @@ use alloc::vec::Vec; use super::bitplane::BITPLANE_BIT_SIZE; use super::build::SubBandType; use super::DecodeSettings; -use crate::error::{bail, err, MarkerError, Result, ValidationError}; +use crate::error::{bail, err, DecodingError, MarkerError, Result, ValidationError}; use crate::reader::BitReader; +use crate::{MAX_J2K_SPEC_COMPONENTS, MAX_NATIVE_DECODE_COMPONENTS}; const MAX_LAYER_COUNT: u8 = 32; const MAX_RESOLUTION_COUNT: u8 = 32; @@ -18,6 +19,8 @@ pub(crate) struct Header<'a> { pub(crate) size_data: SizeData, pub(crate) global_coding_style: CodingStyleDefault, pub(crate) component_infos: Vec, + pub(crate) progression_changes: Vec, + pub(crate) plm_packet_lengths: Vec, pub(crate) ppm_packets: Vec>, pub(crate) skipped_resolution_levels: u8, /// Whether strict mode is enabled for decoding. @@ -35,6 +38,12 @@ pub(crate) struct PpmPacket<'a> { pub(crate) data: &'a [u8], } +#[derive(Debug, Clone)] +pub(crate) struct PacketLengthMarker { + pub(crate) sequence_idx: u8, + pub(crate) packet_lengths: Vec, +} + pub(crate) fn read_header<'a>( reader: &mut BitReader<'a>, settings: &DecodeSettings, @@ -51,6 +60,9 @@ pub(crate) fn read_header<'a>( let num_components = size_data.component_sizes.len() as u16; let mut cod_components = vec![None; num_components as usize]; let mut qcd_components = vec![None; num_components as usize]; + let mut rgn_components = vec![None; num_components as usize]; + let mut progression_changes = vec![]; + let mut plm_markers = vec![]; let mut ppm_markers = vec![]; loop { @@ -84,14 +96,36 @@ pub(crate) fn read_header<'a>( .get_mut(component_index as usize) .ok_or(MarkerError::ParseFailure("QCC"))? = Some(qcc); } + markers::POC => { + reader.read_marker()?; + let num_layers = cod + .as_ref() + .ok_or(MarkerError::ParseFailure("POC"))? + .num_layers; + progression_changes.extend( + poc_marker(reader, num_components, num_layers) + .ok_or(MarkerError::ParseFailure("POC"))?, + ); + } markers::RGN => { reader.read_marker()?; - rgn_marker(reader).ok_or(MarkerError::ParseFailure("RGN"))?; + let rgn = + rgn_marker(reader, num_components).ok_or(MarkerError::ParseFailure("RGN"))?; + if rgn.style != 0 { + bail!(DecodingError::UnsupportedFeature("explicit ROI coding")); + } + *rgn_components + .get_mut(rgn.component_index as usize) + .ok_or(MarkerError::ParseFailure("RGN"))? = Some(rgn.shift); } markers::TLM => { reader.read_marker()?; tlm_marker(reader).ok_or(MarkerError::ParseFailure("TLM"))?; } + markers::PLM => { + reader.read_marker()?; + plm_markers.push(plm_marker(reader).ok_or(MarkerError::ParseFailure("PLM"))?); + } markers::COM => { reader.read_marker()?; com_marker(reader).ok_or(MarkerError::ParseFailure("COM"))?; @@ -135,6 +169,7 @@ pub(crate) fn read_header<'a>( }) .unwrap_or(cod.component_parameters.clone()), quantization_info: qcd_components[idx].clone().unwrap_or(qcd.clone()), + roi_shift: rgn_components[idx].unwrap_or(0), }) .collect(); @@ -148,6 +183,9 @@ pub(crate) fn read_header<'a>( .ok_or(ValidationError::InvalidComponentMetadata)?; let skipped_resolution_levels = if let Some((target_width, target_height)) = settings.target_resolution { + if target_width == 0 || target_height == 0 { + bail!(ValidationError::InvalidDimensions); + } let width_log = (size_data.image_width() / target_width) .checked_ilog2() .unwrap_or(0); @@ -167,11 +205,17 @@ pub(crate) fn read_header<'a>( size_data.y_resolution_shrink_factor *= 1 << skipped_resolution_levels; ppm_markers.sort_by_key(|ppm_marker| ppm_marker.sequence_idx); + plm_markers.sort_by_key(|plm_marker| plm_marker.sequence_idx); let header = Header { size_data, global_coding_style: cod.clone(), component_infos, + progression_changes, + plm_packet_lengths: plm_markers + .into_iter() + .flat_map(|marker| marker.packet_lengths) + .collect(), ppm_packets: ppm_markers .into_iter() .flat_map(|i| i.packets) @@ -219,6 +263,7 @@ pub(crate) struct ComponentInfo { pub(crate) size_info: ComponentSizeInfo, pub(crate) coding_style: CodingStyleComponent, pub(crate) quantization_info: QuantizationInfo, + pub(crate) roi_shift: u8, } impl ComponentInfo { @@ -230,7 +275,8 @@ impl ComponentInfo { let n_ll = self.coding_style.parameters.num_decomposition_levels; let sb_index = match sub_band_type { - // TODO: Shouldn't be reached. + // LL only has a quantization entry at resolution 0; non-zero LL + // lookups fall through to the missing-step error below. SubBandType::LowLow => u16::MAX, SubBandType::HighLow => 0, SubBandType::LowHigh => 1, @@ -316,6 +362,16 @@ pub(crate) enum ProgressionOrder { ComponentPositionResolutionLayer, } +#[derive(Debug, Clone)] +pub(crate) struct ProgressionChange { + pub(crate) resolution_start: u8, + pub(crate) component_start: u8, + pub(crate) layer_end: u8, + pub(crate) resolution_end: u8, + pub(crate) component_end: u8, + pub(crate) progression_order: ProgressionOrder, +} + impl ProgressionOrder { fn from_u8(value: u8) -> Result { match value { @@ -536,8 +592,12 @@ impl SizeData { } /// The total number of tiles. + /// + /// Saturating: `size_marker` rejects grids beyond `MAX_TILES`, so any + /// validated header stays far below the saturation point; saturation only + /// keeps unvalidated values panic-free. pub(crate) fn num_tiles(&self) -> u32 { - self.num_x_tiles() * self.num_y_tiles() + self.num_x_tiles().saturating_mul(self.num_y_tiles()) } /// Return the overall width of the image. @@ -555,7 +615,7 @@ impl SizeData { /// SIZ marker (A.5.1). fn size_marker(reader: &mut BitReader<'_>) -> Result { - let size_data = size_marker_inner(reader).ok_or(MarkerError::ParseFailure("SIZ"))?; + let size_data = size_marker_inner(reader)?; if size_data.tile_width == 0 || size_data.tile_height == 0 @@ -602,46 +662,75 @@ fn size_marker(reader: &mut BitReader<'_>) -> Result { } } - const MAX_DIMENSIONS: usize = 60000; - - if size_data.image_width() as usize > MAX_DIMENSIONS - || size_data.image_height() as usize > MAX_DIMENSIONS + if size_data.image_width() > crate::MAX_J2K_IMAGE_DIMENSION + || size_data.image_height() > crate::MAX_J2K_IMAGE_DIMENSION { bail!(ValidationError::ImageTooLarge); } + // Isot is a u16, so no conforming codestream addresses more than 65,536 + // tiles (the encoder enforces the same ceiling). Rejecting here also stops + // crafted SIZ values from overflowing num_tiles() or driving the eager + // per-tile allocation in tile parsing. + let num_tiles = u64::from(size_data.num_x_tiles()) * u64::from(size_data.num_y_tiles()); + if num_tiles > crate::MAX_J2K_TILE_COUNT { + bail!(ValidationError::TooManyTiles); + } + Ok(size_data) } -fn size_marker_inner(reader: &mut BitReader<'_>) -> Option { +fn read_siz_byte(reader: &mut BitReader<'_>) -> Result { + reader + .read_byte() + .ok_or(MarkerError::ParseFailure("SIZ").into()) +} + +fn read_siz_u16(reader: &mut BitReader<'_>) -> Result { + reader + .read_u16() + .ok_or(MarkerError::ParseFailure("SIZ").into()) +} + +fn read_siz_u32(reader: &mut BitReader<'_>) -> Result { + reader + .read_u32() + .ok_or(MarkerError::ParseFailure("SIZ").into()) +} + +fn size_marker_inner(reader: &mut BitReader<'_>) -> Result { // Length. - let _ = reader.read_u16()?; + let _ = read_siz_u16(reader)?; // Decoder capabilities. - let _ = reader.read_u16()?; - - let xsiz = reader.read_u32()?; - let ysiz = reader.read_u32()?; - let x_osiz = reader.read_u32()?; - let y_osiz = reader.read_u32()?; - let xt_siz = reader.read_u32()?; - let yt_siz = reader.read_u32()?; - let xto_siz = reader.read_u32()?; - let yto_siz = reader.read_u32()?; - let csiz = reader.read_u16()?; + let _ = read_siz_u16(reader)?; + + let xsiz = read_siz_u32(reader)?; + let ysiz = read_siz_u32(reader)?; + let x_osiz = read_siz_u32(reader)?; + let y_osiz = read_siz_u32(reader)?; + let xt_siz = read_siz_u32(reader)?; + let yt_siz = read_siz_u32(reader)?; + let xto_siz = read_siz_u32(reader)?; + let yto_siz = read_siz_u32(reader)?; + let csiz = read_siz_u16(reader)?; if x_osiz >= xsiz || y_osiz >= ysiz { - return None; + bail!(ValidationError::InvalidDimensions); } if csiz == 0 { - return None; + bail!(ValidationError::InvalidComponentMetadata); + } + + if csiz > MAX_J2K_SPEC_COMPONENTS || csiz > MAX_NATIVE_DECODE_COMPONENTS { + bail!(ValidationError::TooManyChannels); } let mut components = Vec::with_capacity(csiz as usize); for _ in 0..csiz { - let ssiz = reader.read_byte()?; - let x_rsiz = reader.read_byte()?; - let y_rsiz = reader.read_byte()?; + let ssiz = read_siz_byte(reader)?; + let x_rsiz = read_siz_byte(reader)?; + let y_rsiz = read_siz_byte(reader)?; let precision = (ssiz & 0x7F) + 1; // No idea how to process signed images, but as far as I can tell @@ -650,7 +739,7 @@ fn size_marker_inner(reader: &mut BitReader<'_>) -> Option { // In theory up to 38 is allowed, but we don't support more than that. if precision as u32 > BITPLANE_BIT_SIZE { - return None; + bail!(ValidationError::InvalidComponentMetadata); } components.push(ComponentSizeInfo { @@ -682,7 +771,7 @@ fn size_marker_inner(reader: &mut BitReader<'_>) -> Option { y_shrink_factor = vr as u32; } - Some(SizeData { + Ok(SizeData { reference_grid_width: xsiz, reference_grid_height: ysiz, image_area_x_offset: x_osiz, @@ -758,6 +847,65 @@ fn tlm_marker(reader: &mut BitReader<'_>) -> Option<()> { skip_marker_segment(reader) } +/// PLM marker (A.7.2). +fn plm_marker(reader: &mut BitReader<'_>) -> Option { + let segment_len = reader.read_u16()?.checked_sub(2)? as usize; + let segment = reader.read_bytes(segment_len)?; + let mut reader = BitReader::new(segment); + + let sequence_idx = reader.read_byte()?; + let mut packet_lengths = vec![]; + + while !reader.at_end() { + let length_data_len = reader.read_u32()? as usize; + let length_data = reader.read_bytes(length_data_len)?; + packet_lengths.extend(decode_packet_lengths(length_data)?); + } + + Some(PacketLengthMarker { + sequence_idx, + packet_lengths, + }) +} + +/// PLT marker (A.7.3). +pub(crate) fn plt_marker(reader: &mut BitReader<'_>) -> Option { + let segment_len = reader.read_u16()?.checked_sub(2)? as usize; + let segment = reader.read_bytes(segment_len)?; + let mut reader = BitReader::new(segment); + + let sequence_idx = reader.read_byte()?; + let packet_lengths = decode_packet_lengths(reader.tail()?)?; + + Some(PacketLengthMarker { + sequence_idx, + packet_lengths, + }) +} + +pub(crate) fn decode_packet_lengths(data: &[u8]) -> Option> { + let mut packet_lengths = vec![]; + let mut value = 0_u32; + let mut in_progress = false; + + for byte in data { + value = value.checked_shl(7)?.checked_add(u32::from(byte & 0x7F))?; + in_progress = true; + + if byte & 0x80 == 0 { + packet_lengths.push(value); + value = 0; + in_progress = false; + } + } + + if in_progress { + return None; + } + + Some(packet_lengths) +} + /// PPM marker (A.7.4). fn ppm_marker<'a>(reader: &mut BitReader<'a>) -> Option> { let segment_len = reader.read_u16()?.checked_sub(2)? as usize; @@ -767,8 +915,9 @@ fn ppm_marker<'a>(reader: &mut BitReader<'a>) -> Option> { let mut reader = BitReader::new(ppm_data); let sequence_idx = reader.read_byte()?; - // TODO: Handle case where next packet doesn't have nppm parameter. - + // This parser handles complete packet payloads carried by the current PPM + // marker. Continuations across multiple PPM markers are rejected by normal + // length parsing until a multi-marker accumulator is added. while !reader.at_end() { let packet_len = reader.read_u16()? as usize; let data = reader.read_bytes(packet_len)?; @@ -783,8 +932,33 @@ fn ppm_marker<'a>(reader: &mut BitReader<'a>) -> Option> { } /// RGN marker (A.6.3). -fn rgn_marker(reader: &mut BitReader<'_>) -> Option<()> { - skip_marker_segment(reader) +pub(crate) fn rgn_marker(reader: &mut BitReader<'_>, csiz: u16) -> Option { + let length = reader.read_u16()?; + let component_index_bytes = if csiz < 257 { 1 } else { 2 }; + if length != 4 + component_index_bytes { + return None; + } + + let component_index = read_component_index(reader, csiz)?; + if component_index >= csiz { + return None; + } + + let style = reader.read_byte()?; + let shift = reader.read_byte()?; + + Some(RgnMarkerData { + component_index, + style, + shift, + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct RgnMarkerData { + pub(crate) component_index: u16, + pub(crate) style: u8, + pub(crate) shift: u8, } pub(crate) fn skip_marker_segment(reader: &mut BitReader<'_>) -> Option<()> { @@ -892,6 +1066,61 @@ pub(crate) fn qcc_marker(reader: &mut BitReader<'_>, csiz: u16) -> Option<(u16, Some((component_index, parameters)) } +/// POC marker (A.6.6). +pub(crate) fn poc_marker( + reader: &mut BitReader<'_>, + csiz: u16, + _num_layers: u8, +) -> Option> { + let length = reader.read_u16()?; + let remaining_bytes = length.checked_sub(2)?; + let component_index_size = if csiz < 257 { 1u16 } else { 2u16 }; + let change_size = 1 + component_index_size + 2 + 1 + component_index_size + 1; + if remaining_bytes == 0 || remaining_bytes % change_size != 0 { + return None; + } + + let change_count = remaining_bytes / change_size; + let mut changes = Vec::with_capacity(change_count as usize); + for _ in 0..change_count { + let resolution_start = reader.read_byte()?; + let component_start = read_component_index(reader, csiz)?; + let layer_end = reader.read_u16()?; + let resolution_end = reader.read_byte()?; + let component_end = read_component_index(reader, csiz)?; + let progression_order = ProgressionOrder::from_u8(reader.read_byte()?).ok()?; + + if resolution_start >= resolution_end + || component_start >= component_end + || component_start >= csiz + || layer_end == 0 + || layer_end > u16::from(u8::MAX) + || component_end > u16::from(u8::MAX) + { + return None; + } + + changes.push(ProgressionChange { + resolution_start, + component_start: component_start as u8, + layer_end: layer_end as u8, + resolution_end, + component_end: component_end as u8, + progression_order, + }); + } + + Some(changes) +} + +fn read_component_index(reader: &mut BitReader<'_>, csiz: u16) -> Option { + if csiz < 257 { + Some(u16::from(reader.read_byte()?)) + } else { + reader.read_u16() + } +} + fn quantization_parameters( reader: &mut BitReader<'_>, quantization_style: QuantizationStyle, diff --git a/crates/signinum-j2k-native/src/j2c/codestream_write.rs b/crates/j2k-native/src/j2c/codestream_write.rs similarity index 65% rename from crates/signinum-j2k-native/src/j2c/codestream_write.rs rename to crates/j2k-native/src/j2c/codestream_write.rs index eb6601ce..efa01d0d 100644 --- a/crates/signinum-j2k-native/src/j2c/codestream_write.rs +++ b/crates/j2k-native/src/j2c/codestream_write.rs @@ -22,6 +22,8 @@ pub(crate) enum BlockCodingMode { pub(crate) struct EncodeParams { pub(crate) width: u32, pub(crate) height: u32, + pub(crate) tile_width: u32, + pub(crate) tile_height: u32, pub(crate) num_components: u8, pub(crate) bit_depth: u8, pub(crate) signed: bool, @@ -35,6 +37,13 @@ pub(crate) struct EncodeParams { pub(crate) block_coding_mode: BlockCodingMode, pub(crate) progression_order: EncodeProgressionOrder, pub(crate) write_tlm: bool, + pub(crate) write_plt: bool, + pub(crate) write_plm: bool, + pub(crate) write_sop: bool, + pub(crate) write_eph: bool, + pub(crate) terminate_coding_passes: bool, + pub(crate) component_sampling: Vec<(u8, u8)>, + pub(crate) precinct_exponents: Vec<(u8, u8)>, } impl Default for EncodeParams { @@ -42,6 +51,8 @@ impl Default for EncodeParams { Self { width: 0, height: 0, + tile_width: 0, + tile_height: 0, num_components: 1, bit_depth: 8, signed: false, @@ -55,17 +66,85 @@ impl Default for EncodeParams { block_coding_mode: BlockCodingMode::Classic, progression_order: EncodeProgressionOrder::Lrcp, write_tlm: false, + write_plt: false, + write_plm: false, + write_sop: false, + write_eph: false, + terminate_coding_passes: false, + component_sampling: Vec::new(), + precinct_exponents: Vec::new(), } } } +pub(crate) struct TilePartData<'a> { + pub(crate) tile_index: u16, + pub(crate) data: &'a [u8], + pub(crate) packet_lengths: &'a [u32], +} + /// Write the complete JPEG 2000 codestream. pub(crate) fn write_codestream( params: &EncodeParams, tile_data: &[u8], quantization_step_sizes: &[(u16, u16)], // (exponent, mantissa) ) -> Vec { - let mut out = Vec::with_capacity(tile_data.len() + 256); + write_codestream_with_packet_lengths(params, tile_data, quantization_step_sizes, &[]) +} + +pub(crate) fn write_codestream_with_packet_lengths( + params: &EncodeParams, + tile_data: &[u8], + quantization_step_sizes: &[(u16, u16)], // (exponent, mantissa) + packet_lengths: &[u32], +) -> Vec { + let tile = TilePartData { + tile_index: 0, + data: tile_data, + packet_lengths, + }; + write_codestream_tiles(params, &[tile], quantization_step_sizes) +} + +pub(crate) fn write_codestream_tiles( + params: &EncodeParams, + tiles: &[TilePartData<'_>], + quantization_step_sizes: &[(u16, u16)], // (exponent, mantissa) +) -> Vec { + struct PreparedTilePart<'a> { + tile_index: u16, + data: &'a [u8], + markers: Vec, + tile_part_len: u32, + } + + let mut prepared_tiles = Vec::with_capacity(tiles.len()); + let mut main_header_packet_lengths = Vec::new(); + let mut total_tile_bytes = 0usize; + for tile in tiles { + let mut markers = Vec::new(); + if params.write_plt && !tile.packet_lengths.is_empty() { + write_plt_markers(&mut markers, tile.packet_lengths); + } + if params.write_plm { + main_header_packet_lengths.extend_from_slice(tile.packet_lengths); + } + let tile_part_len = 14 + + u32::try_from(markers.len()).unwrap_or(u32::MAX) + + u32::try_from(tile.data.len()).unwrap_or(u32::MAX); + total_tile_bytes = total_tile_bytes + .saturating_add(markers.len()) + .saturating_add(tile.data.len()) + .saturating_add(14); + prepared_tiles.push(PreparedTilePart { + tile_index: tile.tile_index, + data: tile.data, + markers, + tile_part_len, + }); + } + + let mut out = Vec::with_capacity(total_tile_bytes + 256); // SOC (Start of codestream) write_marker(&mut out, markers::SOC); @@ -83,20 +162,22 @@ pub(crate) fn write_codestream( // QCD (Quantization defaults) write_qcd_marker(&mut out, params, quantization_step_sizes); - // TLM (Tile-part lengths), required by DICOM HTJ2K RPCL. - let tile_part_len = 14 + tile_data.len() as u32; // SOT marker+segment(12) + SOD(2) + tile data - if params.write_tlm { - write_tlm_marker(&mut out, 0, tile_part_len); + if params.write_plm && !main_header_packet_lengths.is_empty() { + write_plm_markers(&mut out, &main_header_packet_lengths); } - // SOT (Start of tile-part) — single tile covering entire image - write_sot_marker(&mut out, 0, tile_part_len - 2); - - // SOD (Start of data) - write_marker(&mut out, markers::SOD); + if params.write_tlm { + for tile in &prepared_tiles { + write_tlm_marker(&mut out, tile.tile_index, tile.tile_part_len); + } + } - // Tile bitstream data - out.extend_from_slice(tile_data); + for tile in prepared_tiles { + write_sot_marker(&mut out, tile.tile_index, tile.tile_part_len - 2); + out.extend_from_slice(&tile.markers); + write_marker(&mut out, markers::SOD); + out.extend_from_slice(tile.data); + } // EOC (End of codestream) write_marker(&mut out, markers::EOC); @@ -128,10 +209,20 @@ fn write_siz_marker(out: &mut Vec, params: &EncodeParams) { out.extend_from_slice(&0u32.to_be_bytes()); // YOsiz (image area y offset) out.extend_from_slice(&0u32.to_be_bytes()); - // XTsiz (tile width = image width, single tile) - out.extend_from_slice(¶ms.width.to_be_bytes()); - // YTsiz (tile height = image height, single tile) - out.extend_from_slice(¶ms.height.to_be_bytes()); + let tile_width = if params.tile_width == 0 { + params.width + } else { + params.tile_width + }; + let tile_height = if params.tile_height == 0 { + params.height + } else { + params.tile_height + }; + // XTsiz (tile width) + out.extend_from_slice(&tile_width.to_be_bytes()); + // YTsiz (tile height) + out.extend_from_slice(&tile_height.to_be_bytes()); // XTOsiz (tile x offset) out.extend_from_slice(&0u32.to_be_bytes()); // YTOsiz (tile y offset) @@ -140,7 +231,7 @@ fn write_siz_marker(out: &mut Vec, params: &EncodeParams) { out.extend_from_slice(&num_comp.to_be_bytes()); // Per-component info - for _ in 0..params.num_components { + for component_index in 0..params.num_components as usize { // Ssiz: bit depth - 1 (unsigned) or bit depth - 1 + 0x80 (signed) let ssiz = if params.signed { (params.bit_depth - 1) | 0x80 @@ -148,10 +239,15 @@ fn write_siz_marker(out: &mut Vec, params: &EncodeParams) { params.bit_depth - 1 }; out.push(ssiz); + let (x_rsiz, y_rsiz) = params + .component_sampling + .get(component_index) + .copied() + .unwrap_or((1, 1)); // XRsiz (horizontal sampling factor) - out.push(1); + out.push(x_rsiz); // YRsiz (vertical sampling factor) - out.push(1); + out.push(y_rsiz); } } @@ -181,11 +277,23 @@ fn ht_capability_word(params: &EncodeParams) -> u16 { fn write_cod_marker(out: &mut Vec, params: &EncodeParams) { write_marker(out, markers::COD); - let marker_len = 12u16; // Fixed length for no precincts + let marker_len = 12u16 + + u16::try_from(params.precinct_exponents.len()) + .expect("precinct exponent count fits in COD marker length"); out.extend_from_slice(&marker_len.to_be_bytes()); - // Scod (coding style flags) — no precincts, no SOP, no EPH - out.push(0x00); + // Scod (coding style flags) + let mut scod = 0u8; + if !params.precinct_exponents.is_empty() { + scod |= 0x01; + } + if params.write_sop { + scod |= 0x02; + } + if params.write_eph { + scod |= 0x04; + } + out.push(scod); // SGcod: Progression order out.push(progression_order_byte(params.progression_order)); @@ -201,18 +309,27 @@ fn write_cod_marker(out: &mut Vec, params: &EncodeParams) { // Code-block height exponent - 2 out.push(params.code_block_height_exp); // Code-block style - out.push(match params.block_coding_mode { - BlockCodingMode::Classic => 0x00, - BlockCodingMode::HighThroughput => 0x40, - }); + let code_block_style = u8::from(params.terminate_coding_passes) << 2 + | match params.block_coding_mode { + BlockCodingMode::Classic => 0x00, + BlockCodingMode::HighThroughput => 0x40, + }; + out.push(code_block_style); // Wavelet transform: 0 = irreversible 9-7, 1 = reversible 5-3 out.push(if params.reversible { 1 } else { 0 }); + + for &(ppx, ppy) in ¶ms.precinct_exponents { + out.push((ppy << 4) | ppx); + } } fn progression_order_byte(progression_order: EncodeProgressionOrder) -> u8 { match progression_order { EncodeProgressionOrder::Lrcp => 0x00, + EncodeProgressionOrder::Rlcp => 0x01, EncodeProgressionOrder::Rpcl => 0x02, + EncodeProgressionOrder::Pcrl => 0x03, + EncodeProgressionOrder::Cprl => 0x04, } } @@ -226,6 +343,52 @@ fn write_tlm_marker(out: &mut Vec, tile_index: u16, tile_part_length: u32) { out.extend_from_slice(&tile_part_length.to_be_bytes()); } +fn write_plt_markers(out: &mut Vec, packet_lengths: &[u32]) { + let data = packet_length_bytes(packet_lengths); + for (sequence_idx, chunk) in data.chunks(usize::from(u16::MAX) - 3).enumerate() { + write_marker(out, markers::PLT); + let marker_len = u16::try_from(3 + chunk.len()).expect("PLT marker chunk length fits"); + out.extend_from_slice(&marker_len.to_be_bytes()); + out.push(sequence_idx as u8); + out.extend_from_slice(chunk); + } +} + +fn write_plm_markers(out: &mut Vec, packet_lengths: &[u32]) { + let data = packet_length_bytes(packet_lengths); + for (sequence_idx, chunk) in data.chunks(usize::from(u16::MAX) - 7).enumerate() { + write_marker(out, markers::PLM); + let marker_len = u16::try_from(7 + chunk.len()).expect("PLM marker chunk length fits"); + out.extend_from_slice(&marker_len.to_be_bytes()); + out.push(sequence_idx as u8); + out.extend_from_slice(&u32::try_from(chunk.len()).unwrap_or(u32::MAX).to_be_bytes()); + out.extend_from_slice(chunk); + } +} + +fn packet_length_bytes(packet_lengths: &[u32]) -> Vec { + let mut out = Vec::new(); + + for &packet_length in packet_lengths { + let mut value = packet_length; + let mut groups = Vec::new(); + groups.push((value & 0x7F) as u8); + value >>= 7; + + while value > 0 { + groups.push((value & 0x7F) as u8); + value >>= 7; + } + + for (idx, group) in groups.iter().rev().enumerate() { + let continuation = idx + 1 != groups.len(); + out.push(if continuation { *group | 0x80 } else { *group }); + } + } + + out +} + /// Write QCD marker segment (A.6.4). fn write_qcd_marker(out: &mut Vec, params: &EncodeParams, step_sizes: &[(u16, u16)]) { write_marker(out, markers::QCD); @@ -277,6 +440,7 @@ fn write_sot_marker(out: &mut Vec, tile_index: u16, tile_part_length: u32) { #[cfg(test)] mod tests { use super::*; + use alloc::{vec, vec::Vec}; fn find_marker_offset(codestream: &[u8], marker: u8) -> Option { codestream diff --git a/crates/signinum-j2k-native/src/j2c/decode.rs b/crates/j2k-native/src/j2c/decode.rs similarity index 82% rename from crates/signinum-j2k-native/src/j2c/decode.rs rename to crates/j2k-native/src/j2c/decode.rs index 730ba13e..4b04b5a7 100644 --- a/crates/signinum-j2k-native/src/j2c/decode.rs +++ b/crates/j2k-native/src/j2c/decode.rs @@ -10,16 +10,10 @@ use alloc::vec::Vec; use super::bitplane::{BitPlaneDecodeBuffers, BitPlaneDecodeContext}; use super::build::{CodeBlock, Decomposition, Layer, Precinct, Segment, SubBand, SubBandType}; -use super::codestream::{ComponentInfo, Header, ProgressionOrder, QuantizationStyle}; +use super::codestream::{ComponentInfo, Header, QuantizationStyle}; use super::ht_block_decode::{self, HtBlockDecodeContext}; use super::idwt::IDWTOutput; -use super::progression::{ - component_position_resolution_layer_progression, - layer_resolution_component_position_progression, - position_component_resolution_layer_progression, - resolution_layer_component_position_progression, - resolution_position_component_layer_progression, IteratorInput, ProgressionData, -}; +use super::progression::{progression_iterator, ProgressionData}; use super::roi::RoiPlan; use super::tag_tree::TagNode; use super::tile::{ComponentTile, ResolutionTile, Tile}; @@ -30,13 +24,18 @@ use crate::math::SimdBuffer; use crate::profile; use crate::reader::BitReader; use crate::{ - decode_j2k_code_block_scalar, HtCodeBlockBatchJob, HtCodeBlockDecodeJob, HtCodeBlockDecoder, - HtOwnedCodeBlockBatchJob, HtOwnedSubBandPlan, HtSubBandDecodeJob, J2kCodeBlockBatchJob, - J2kCodeBlockDecodeJob, J2kCodeBlockSegment, J2kCodeBlockStyle, J2kDirectBandId, - J2kDirectColorPlan, J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kDirectIdwtStep, - J2kDirectStoreStep, J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, J2kRect, - J2kStoreComponentJob, J2kSubBandDecodeJob, J2kSubBandType, J2kWaveletTransform, + add_roi_shift_to_bitplanes, apply_roi_maxshift_inverse_i32, checked_decode_byte_len3, + checked_decode_sample_count, decode_j2k_code_block_scalar, HtCodeBlockBatchJob, + HtCodeBlockDecodeJob, HtCodeBlockDecoder, HtOwnedCodeBlockBatchJob, HtOwnedSubBandPlan, + HtSubBandDecodeJob, J2kCodeBlockBatchJob, J2kCodeBlockDecodeJob, J2kCodeBlockSegment, + J2kCodeBlockStyle, J2kDirectBandId, J2kDirectColorPlan, J2kDirectGrayscalePlan, + J2kDirectGrayscaleStep, J2kDirectIdwtStep, J2kDirectStoreStep, J2kOwnedCodeBlockBatchJob, + J2kOwnedSubBandPlan, J2kRect, J2kStoreComponentJob, J2kSubBandDecodeJob, J2kSubBandType, + J2kWaveletTransform, }; +#[cfg(feature = "parallel")] +use crate::{decode_ht_code_block_scalar_with_workspace, HtCodeBlockDecodeWorkspace}; +use core::mem::size_of; use core::ops::{DerefMut, Range}; pub(crate) fn decode<'a>( @@ -57,7 +56,7 @@ pub(crate) fn decode<'a>( bail!(TileError::Invalid); } - ctx.reset(header, &tiles[0]); + ctx.reset(header, &tiles[0])?; let cpu_decode_parallelism = ctx.cpu_decode_parallelism; let (tile_ctx, storage) = (&mut ctx.tile_decode_context, &mut ctx.storage); @@ -71,34 +70,10 @@ pub(crate) fn decode<'a>( tile.rect.height(), ); - let iter_input = IteratorInput::new(tile); - - let progression_iterator: Box> = - match tile.progression_order { - ProgressionOrder::LayerResolutionComponentPosition => { - Box::new(layer_resolution_component_position_progression(iter_input)) - } - ProgressionOrder::ResolutionLayerComponentPosition => { - Box::new(resolution_layer_component_position_progression(iter_input)) - } - ProgressionOrder::ResolutionPositionComponentLayer => Box::new( - resolution_position_component_layer_progression(iter_input) - .ok_or(DecodingError::InvalidProgressionIterator)?, - ), - ProgressionOrder::PositionComponentResolutionLayer => Box::new( - position_component_resolution_layer_progression(iter_input) - .ok_or(DecodingError::InvalidProgressionIterator)?, - ), - ProgressionOrder::ComponentPositionResolutionLayer => Box::new( - component_position_resolution_layer_progression(iter_input) - .ok_or(DecodingError::InvalidProgressionIterator)?, - ), - }; - decode_tile( tile, header, - progression_iterator, + progression_iterator(tile)?, tile_ctx, storage, ht_decoder, @@ -119,20 +94,7 @@ pub(crate) fn decode<'a>( } if profile_enabled { - profile::emit_profile_row( - "decode", - "cpu", - &[ - ("parse_tiles_us", profile_timings.parse_tiles_us), - ("build_us", profile_timings.build_us), - ("segment_us", profile_timings.segment_us), - ("codeblock_us", profile_timings.codeblock_us), - ("idwt_us", profile_timings.idwt_us), - ("store_us", profile_timings.store_us), - ("mct_us", profile_timings.mct_us), - ("total_us", profile::elapsed_us(total_start)), - ], - ); + emit_decode_profile_row(tile_ctx, &profile_timings, total_start); } Ok(()) @@ -159,34 +121,14 @@ pub(crate) fn build_direct_grayscale_plan<'a>( )); } ctx.tile_decode_context.channel_data.clear(); - ctx.tile_decode_context.output_region = None; ctx.storage.reset(); build::build(tile, &mut ctx.storage)?; + if let Some(output_region) = ctx.tile_decode_context.output_region { + ctx.storage.roi_plan = RoiPlan::build(tile, header, &ctx.storage, output_region); + } - let iter_input = IteratorInput::new(tile); - let progression_iterator: Box> = - match tile.progression_order { - ProgressionOrder::LayerResolutionComponentPosition => { - Box::new(layer_resolution_component_position_progression(iter_input)) - } - ProgressionOrder::ResolutionLayerComponentPosition => { - Box::new(resolution_layer_component_position_progression(iter_input)) - } - ProgressionOrder::ResolutionPositionComponentLayer => Box::new( - resolution_position_component_layer_progression(iter_input) - .ok_or(DecodingError::InvalidProgressionIterator)?, - ), - ProgressionOrder::PositionComponentResolutionLayer => Box::new( - position_component_resolution_layer_progression(iter_input) - .ok_or(DecodingError::InvalidProgressionIterator)?, - ), - ProgressionOrder::ComponentPositionResolutionLayer => Box::new( - component_position_resolution_layer_progression(iter_input) - .ok_or(DecodingError::InvalidProgressionIterator)?, - ), - }; - segment::parse(tile, progression_iterator, header, &mut ctx.storage)?; + segment::parse(tile, progression_iterator(tile)?, header, &mut ctx.storage)?; let component_info = &tile.component_infos[0]; build_component_plan_from_storage( @@ -218,12 +160,6 @@ pub(crate) fn build_direct_color_plan<'a>( "direct color plan only supports three-component RGB codestreams" )); } - if header.skipped_resolution_levels != 0 { - bail!(DecodingError::UnsupportedFeature( - "direct color plan only supports full-resolution decode" - )); - } - let transform = tile.component_infos[0].wavelet_transform(); if tile.mct && (transform != tile.component_infos[1].wavelet_transform() @@ -233,34 +169,14 @@ pub(crate) fn build_direct_color_plan<'a>( } ctx.tile_decode_context.channel_data.clear(); - ctx.tile_decode_context.output_region = None; ctx.storage.reset(); build::build(tile, &mut ctx.storage)?; + if let Some(output_region) = ctx.tile_decode_context.output_region { + ctx.storage.roi_plan = RoiPlan::build(tile, header, &ctx.storage, output_region); + } - let iter_input = IteratorInput::new(tile); - let progression_iterator: Box> = - match tile.progression_order { - ProgressionOrder::LayerResolutionComponentPosition => { - Box::new(layer_resolution_component_position_progression(iter_input)) - } - ProgressionOrder::ResolutionLayerComponentPosition => { - Box::new(resolution_layer_component_position_progression(iter_input)) - } - ProgressionOrder::ResolutionPositionComponentLayer => Box::new( - resolution_position_component_layer_progression(iter_input) - .ok_or(DecodingError::InvalidProgressionIterator)?, - ), - ProgressionOrder::PositionComponentResolutionLayer => Box::new( - position_component_resolution_layer_progression(iter_input) - .ok_or(DecodingError::InvalidProgressionIterator)?, - ), - ProgressionOrder::ComponentPositionResolutionLayer => Box::new( - component_position_resolution_layer_progression(iter_input) - .ok_or(DecodingError::InvalidProgressionIterator)?, - ), - }; - segment::parse(tile, progression_iterator, header, &mut ctx.storage)?; + segment::parse(tile, progression_iterator(tile)?, header, &mut ctx.storage)?; let mut bit_depths = [0_u8; 3]; let mut component_plans = Vec::with_capacity(3); @@ -321,7 +237,20 @@ fn build_component_plan_from_storage( .ok_or(DecodingError::UnsupportedFeature( "direct component decomposition index is out of range", ))?; - let mut steps = Vec::new(); + let decompositions = &storage.decompositions[tile_decompositions.decompositions.clone()]; + let active_decomposition_count = decompositions + .len() + .saturating_sub(header.skipped_resolution_levels as usize); + let sub_band_step_count = (0..component_info.num_resolution_levels() + - header.skipped_resolution_levels) + .map(|resolution| { + tile_decompositions + .sub_band_iter(resolution, &storage.decompositions) + .count() + }) + .sum::(); + let mut steps = + Vec::with_capacity(sub_band_step_count + active_decomposition_count.saturating_add(1)); let mut next_band_id: J2kDirectBandId = 0; let mut sub_band_ids = vec![None; storage.sub_bands.len()]; @@ -330,6 +259,7 @@ fn build_component_plan_from_storage( for sub_band_idx in sub_band_iter { if let Some(step) = build_grayscale_sub_band_step( &storage.sub_bands[sub_band_idx], + sub_band_idx, next_band_id, resolution, component_info, @@ -348,10 +278,7 @@ fn build_component_plan_from_storage( let mut current_ll_rect = storage.sub_bands[tile_decompositions.first_ll_sub_band].rect; let mut current_ll_band_id = sub_band_ids[tile_decompositions.first_ll_sub_band] .ok_or(DecodingError::CodeBlockDecodeFailure)?; - let decompositions = &storage.decompositions[tile_decompositions.decompositions.clone()]; - let decompositions = &decompositions[..decompositions - .len() - .saturating_sub(header.skipped_resolution_levels as usize)]; + let decompositions = &decompositions[..active_decomposition_count]; for decomposition in decompositions { let hl = &storage.sub_bands[decomposition.sub_bands[0]]; let lh = &storage.sub_bands[decomposition.sub_bands[1]]; @@ -425,6 +352,7 @@ fn build_component_plan_from_storage( fn build_grayscale_sub_band_step( sub_band: &SubBand, + sub_band_idx: usize, band_id: J2kDirectBandId, resolution: u8, component_info: &ComponentInfo, @@ -474,12 +402,14 @@ fn build_grayscale_sub_band_step( .code_block_style .uses_high_throughput_block_coding() { + let coded_bitplanes = + add_roi_shift_to_bitplanes(num_bitplanes, component_info.roi_shift, 31)?; let stripe_causal = component_info .coding_style .parameters .code_block_style .vertically_causal_context; - let mut jobs = Vec::new(); + let mut jobs = Vec::with_capacity(direct_sub_band_job_capacity(sub_band, storage)); for precinct in sub_band .precincts .clone() @@ -490,12 +420,15 @@ fn build_grayscale_sub_band_step( .clone() .map(|idx| &storage.code_blocks[idx]) { + if !code_block_required_by_index(storage, sub_band_idx, code_block) { + continue; + } let actual_bitplanes = if header.strict { - num_bitplanes + coded_bitplanes .checked_sub(code_block.missing_bit_planes) .ok_or(DecodingError::InvalidBitplaneCount)? } else { - num_bitplanes.saturating_sub(code_block.missing_bit_planes) + coded_bitplanes.saturating_sub(code_block.missing_bit_planes) }; let max_coding_passes = if actual_bitplanes == 0 { 0 @@ -522,6 +455,7 @@ fn build_grayscale_sub_band_step( missing_bit_planes: code_block.missing_bit_planes, number_of_coding_passes: code_block.number_of_coding_passes, num_bitplanes, + roi_shift: component_info.roi_shift, stripe_causal, strict: header.strict, dequantization_step, @@ -574,7 +508,7 @@ fn build_grayscale_sub_band_step( .segmentation_symbols, }; - let mut jobs = Vec::new(); + let mut jobs = Vec::with_capacity(direct_sub_band_job_capacity(sub_band, storage)); for precinct in sub_band .precincts .clone() @@ -585,6 +519,9 @@ fn build_grayscale_sub_band_step( .clone() .map(|idx| &storage.code_blocks[idx]) { + if !code_block_required_by_index(storage, sub_band_idx, code_block) { + continue; + } let (combined_data, segments) = collect_classic_code_block_data( code_block, &component_info.coding_style.parameters.code_block_style, @@ -601,6 +538,7 @@ fn build_grayscale_sub_band_step( missing_bit_planes: code_block.missing_bit_planes, number_of_coding_passes: code_block.number_of_coding_passes, total_bitplanes: num_bitplanes, + roi_shift: component_info.roi_shift, sub_band_type: classic_job_sub_band_type, style: classic_job_style, strict: header.strict, @@ -620,6 +558,14 @@ fn build_grayscale_sub_band_step( ))) } +fn direct_sub_band_job_capacity(sub_band: &SubBand, storage: &DecompositionStorage<'_>) -> usize { + sub_band + .precincts + .clone() + .map(|idx| storage.precincts[idx].code_blocks.len()) + .sum() +} + fn collect_classic_code_block_data( code_block: &CodeBlock, style: &super::codestream::CodeBlockStyle, @@ -749,6 +695,7 @@ pub(crate) struct DecodeDebugCounters { pub(crate) decoded_code_blocks: usize, pub(crate) skipped_code_blocks: usize, pub(crate) idwt_output_samples: usize, + pub(crate) ht_phase_stats: ht_block_decode::HtBlockDecodeStats, } /// CPU parallelism policy for native JPEG 2000 decode. @@ -764,7 +711,7 @@ pub enum CpuDecodeParallelism { /// A decoder context for decoding JPEG2000 images. pub struct DecoderContext<'a> { pub(crate) tile_decode_context: TileDecodeContext, - storage: DecompositionStorage<'a>, + pub(crate) storage: DecompositionStorage<'a>, cpu_decode_parallelism: CpuDecodeParallelism, } @@ -779,9 +726,10 @@ impl Default for DecoderContext<'_> { } impl DecoderContext<'_> { - fn reset(&mut self, header: &Header<'_>, initial_tile: &Tile<'_>) { - self.tile_decode_context.reset(header, initial_tile); + fn reset(&mut self, header: &Header<'_>, initial_tile: &Tile<'_>) -> Result<()> { + self.tile_decode_context.reset(header, initial_tile)?; self.storage.reset(); + Ok(()) } pub(crate) fn set_output_region(&mut self, output_region: Option<(u32, u32, u32, u32)>) { @@ -839,6 +787,7 @@ fn decode_tile<'a, 'b>( header, ht_decoder, cpu_decode_parallelism, + profile_enabled, )?; profile_timings.codeblock_us += profile::elapsed_us(stage_start); @@ -885,6 +834,62 @@ struct DecodeProfileTimings { mct_us: u128, } +#[cold] +#[inline(never)] +fn emit_decode_profile_row( + tile_ctx: &TileDecodeContext, + profile_timings: &DecodeProfileTimings, + total_start: Option, +) { + profile::emit_profile_row( + "decode", + "cpu", + &[ + ("parse_tiles_us", profile_timings.parse_tiles_us), + ("build_us", profile_timings.build_us), + ("segment_us", profile_timings.segment_us), + ("codeblock_us", profile_timings.codeblock_us), + ("ht_blocks", tile_ctx.debug_counters.ht_phase_stats.blocks), + ( + "ht_refinement_blocks", + tile_ctx.debug_counters.ht_phase_stats.refinement_blocks, + ), + ( + "ht_cleanup_bytes", + tile_ctx.debug_counters.ht_phase_stats.cleanup_bytes, + ), + ( + "ht_refinement_bytes", + tile_ctx.debug_counters.ht_phase_stats.refinement_bytes, + ), + ( + "ht_cleanup_us", + tile_ctx.debug_counters.ht_phase_stats.ht_cleanup_us, + ), + ( + "ht_mag_sgn_us", + tile_ctx.debug_counters.ht_phase_stats.ht_mag_sgn_us, + ), + ( + "ht_sigma_us", + tile_ctx.debug_counters.ht_phase_stats.ht_sigma_us, + ), + ( + "ht_sigprop_us", + tile_ctx.debug_counters.ht_phase_stats.ht_sigprop_us, + ), + ( + "ht_magref_us", + tile_ctx.debug_counters.ht_phase_stats.ht_magref_us, + ), + ("idwt_us", profile_timings.idwt_us), + ("store_us", profile_timings.store_us), + ("mct_us", profile_timings.mct_us), + ("total_us", profile::elapsed_us(total_start)), + ], + ); +} + /// All decompositions for a single tile. #[derive(Clone)] pub(crate) struct TileDecompositions { @@ -962,7 +967,7 @@ pub(crate) struct DecompositionStorage<'a> { } impl DecompositionStorage<'_> { - fn reset(&mut self) { + pub(crate) fn reset(&mut self) { self.segments.clear(); self.layers.clear(); self.code_blocks.clear(); @@ -1004,7 +1009,7 @@ pub(crate) struct TileDecodeContext { impl TileDecodeContext { /// Reset the context for processing a new image. - fn reset(&mut self, header: &Header<'_>, initial_tile: &Tile<'_>) { + fn reset(&mut self, header: &Header<'_>, initial_tile: &Tile<'_>) -> Result<()> { // Bitplane decode context and buffers will be reset in the // corresponding methods. IDWT output and scratch buffer will be // overridden on demand, so those don't need to be reset either. @@ -1017,23 +1022,33 @@ impl TileDecodeContext { header.size_data.image_height(), )); - // TODO: SIMD Buffers should be reused across runs! + let sample_count = checked_decode_sample_count(output_width, output_height)?; + checked_decode_byte_len3( + sample_count, + initial_tile.component_infos.len(), + size_of::(), + )?; + + // Allocate per component here; the surrounding context reuses the + // higher-level vectors while `SimdBuffer` owns its initialized storage. for info in &initial_tile.component_infos { self.channel_data.push(ComponentData { - container: SimdBuffer::zeros(output_width as usize * output_height as usize), + container: SimdBuffer::zeros(sample_count), bit_depth: info.size_info.precision, }); } + Ok(()) } } -fn decode_component_tile_bit_planes<'a>( +pub(crate) fn decode_component_tile_bit_planes<'a>( tile: &Tile<'a>, tile_ctx: &mut TileDecodeContext, storage: &mut DecompositionStorage<'a>, header: &Header<'_>, ht_decoder: &mut Option<&mut dyn HtCodeBlockDecoder>, cpu_decode_parallelism: CpuDecodeParallelism, + profile_enabled: bool, ) -> Result<()> { for (tile_decompositions_idx, component_info) in tile.component_infos.iter().enumerate() { // Only decode the resolution levels we actually care about. @@ -1053,6 +1068,7 @@ fn decode_component_tile_bit_planes<'a>( header, ht_decoder, cpu_decode_parallelism, + profile_enabled, )?; } } @@ -1070,6 +1086,7 @@ fn decode_sub_band_bitplanes( header: &Header<'_>, ht_decoder: &mut Option<&mut dyn HtCodeBlockDecoder>, cpu_decode_parallelism: CpuDecodeParallelism, + profile_enabled: bool, ) -> Result<()> { let sub_band = storage.sub_bands[sub_band_idx].clone(); @@ -1125,12 +1142,17 @@ fn decode_sub_band_bitplanes( storage, header, ht_decoder, + cpu_decode_parallelism, num_bitplanes, dequantization_step, + profile_enabled, )?; return Ok(()); } + let coded_bitplanes = + add_roi_shift_to_bitplanes(num_bitplanes, component_info.roi_shift, MAX_BITPLANE_COUNT)?; + let classic_job_sub_band_type = match sub_band.sub_band_type { SubBandType::LowLow => J2kSubBandType::LowLow, SubBandType::HighLow => J2kSubBandType::HighLow, @@ -1183,6 +1205,7 @@ fn decode_sub_band_bitplanes( missing_bit_planes: pending.missing_bit_planes, number_of_coding_passes: pending.number_of_coding_passes, total_bitplanes: num_bitplanes, + roi_shift: component_info.roi_shift, sub_band_type: classic_job_sub_band_type, style: classic_job_style, strict: header.strict, @@ -1238,6 +1261,7 @@ fn decode_sub_band_bitplanes( classic_job_style, header.strict, num_bitplanes, + component_info.roi_shift, dequantization_step, )?; tile_ctx.debug_counters.decoded_code_blocks += decoded_blocks.len(); @@ -1269,7 +1293,7 @@ fn decode_sub_band_bitplanes( bitplane::decode( code_block, sub_band.sub_band_type, - num_bitplanes, + coded_bitplanes, &component_info.coding_style.parameters.code_block_style, tile_ctx, storage, @@ -1283,7 +1307,9 @@ fn decode_sub_band_bitplanes( let out_row = &mut base_store[base_idx..]; for (output, coefficient) in out_row.iter_mut().zip(coefficients.iter().copied()) { - *output = coefficient.get() as f32; + let coefficient = + apply_roi_maxshift_inverse_i32(coefficient.get(), component_info.roi_shift); + *output = coefficient as f32; *output *= dequantization_step; } @@ -1325,6 +1351,15 @@ struct DecodedClassicBlock { coefficients: Vec, } +#[cfg(feature = "parallel")] +struct DecodedHtBlock { + output_x: u32, + output_y: u32, + width: u32, + height: u32, + coefficients: Vec, +} + fn count_classic_code_blocks( sub_band_idx: usize, sub_band: &SubBand, @@ -1399,6 +1434,87 @@ fn collect_pending_classic_blocks( Ok(pending_blocks) } +fn count_ht_code_blocks( + sub_band_idx: usize, + sub_band: &SubBand, + storage: &DecompositionStorage<'_>, +) -> usize { + sub_band + .precincts + .clone() + .map(|idx| &storage.precincts[idx]) + .map(|precinct| { + precinct + .code_blocks + .clone() + .filter(|idx| { + let code_block = &storage.code_blocks[*idx]; + code_block_required_by_index(storage, sub_band_idx, code_block) + && code_block.number_of_coding_passes > 0 + }) + .count() + }) + .sum() +} + +#[cfg(feature = "parallel")] +fn collect_pending_ht_blocks( + sub_band_idx: usize, + sub_band: &SubBand, + storage: &DecompositionStorage<'_>, + header: &Header<'_>, + num_bitplanes: u8, + roi_shift: u8, +) -> Result> { + let coded_bitplanes = add_roi_shift_to_bitplanes(num_bitplanes, roi_shift, 31)?; + let mut pending_blocks = + Vec::with_capacity(count_ht_code_blocks(sub_band_idx, sub_band, storage)); + for precinct in sub_band + .precincts + .clone() + .map(|idx| &storage.precincts[idx]) + { + for code_block in precinct + .code_blocks + .clone() + .map(|idx| &storage.code_blocks[idx]) + { + if !code_block_required_by_index(storage, sub_band_idx, code_block) { + continue; + } + let actual_bitplanes = if header.strict { + coded_bitplanes + .checked_sub(code_block.missing_bit_planes) + .ok_or(DecodingError::InvalidBitplaneCount)? + } else { + coded_bitplanes.saturating_sub(code_block.missing_bit_planes) + }; + let max_coding_passes = if actual_bitplanes == 0 { + 0 + } else { + 1 + 3 * (actual_bitplanes - 1) + }; + if code_block.number_of_coding_passes > max_coding_passes && header.strict { + bail!(DecodingError::TooManyCodingPasses); + } + if code_block.number_of_coding_passes == 0 || actual_bitplanes == 0 { + continue; + } + + pending_blocks.push(PendingHtBlock { + combined: ht_block_decode::collect_code_block_data(code_block, storage)?, + output_x: code_block.rect.x0 - sub_band.rect.x0, + output_y: code_block.rect.y0 - sub_band.rect.y0, + width: code_block.rect.width(), + height: code_block.rect.height(), + missing_bit_planes: code_block.missing_bit_planes, + number_of_coding_passes: code_block.number_of_coding_passes, + }); + } + } + Ok(pending_blocks) +} + pub(crate) fn should_decode_classic_sub_band_in_parallel( parallelism: CpuDecodeParallelism, code_block_count: usize, @@ -1406,6 +1522,13 @@ pub(crate) fn should_decode_classic_sub_band_in_parallel( cfg!(feature = "parallel") && parallelism == CpuDecodeParallelism::Auto && code_block_count >= 4 } +pub(crate) fn should_decode_ht_sub_band_in_parallel( + parallelism: CpuDecodeParallelism, + code_block_count: usize, +) -> bool { + cfg!(feature = "parallel") && parallelism == CpuDecodeParallelism::Auto && code_block_count >= 4 +} + #[cfg(feature = "parallel")] fn decode_classic_sub_band_blocks_parallel( pending_blocks: &[PendingClassicBlock], @@ -1413,6 +1536,7 @@ fn decode_classic_sub_band_blocks_parallel( style: J2kCodeBlockStyle, strict: bool, total_bitplanes: u8, + roi_shift: u8, dequantization_step: f32, ) -> Result> { use rayon::prelude::*; @@ -1435,6 +1559,7 @@ fn decode_classic_sub_band_blocks_parallel( missing_bit_planes: pending.missing_bit_planes, number_of_coding_passes: pending.number_of_coding_passes, total_bitplanes, + roi_shift, sub_band_type, style, strict, @@ -1496,6 +1621,99 @@ fn copy_decoded_classic_blocks_to_sub_band( Ok(()) } +#[cfg(feature = "parallel")] +fn decode_ht_sub_band_blocks_parallel( + pending_blocks: &[PendingHtBlock], + strict: bool, + num_bitplanes: u8, + roi_shift: u8, + stripe_causal: bool, + dequantization_step: f32, +) -> Result> { + use rayon::prelude::*; + + pending_blocks + .par_iter() + .map(|pending| { + let output_stride = pending.width as usize; + let output_len = output_stride + .checked_mul(pending.height as usize) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let mut coefficients = vec![0.0; output_len]; + let mut workspace = HtCodeBlockDecodeWorkspace::default(); + decode_ht_code_block_scalar_with_workspace( + HtCodeBlockDecodeJob { + data: &pending.combined.data, + cleanup_length: pending.combined.cleanup_length, + refinement_length: pending.combined.refinement_length, + width: pending.width, + height: pending.height, + output_stride, + missing_bit_planes: pending.missing_bit_planes, + number_of_coding_passes: pending.number_of_coding_passes, + num_bitplanes, + roi_shift, + stripe_causal, + strict, + dequantization_step, + }, + &mut coefficients, + &mut workspace, + )?; + Ok(DecodedHtBlock { + output_x: pending.output_x, + output_y: pending.output_y, + width: pending.width, + height: pending.height, + coefficients, + }) + }) + .collect::>() + .into_iter() + .collect() +} + +#[cfg(feature = "parallel")] +fn copy_decoded_ht_blocks_to_sub_band( + decoded_blocks: &[DecodedHtBlock], + sub_band: &SubBand, + storage: &mut DecompositionStorage<'_>, +) -> Result<()> { + let sub_band_width = sub_band.rect.width() as usize; + let base_store = &mut storage.coefficients[sub_band.coefficients.clone()]; + for block in decoded_blocks { + if block + .output_x + .checked_add(block.width) + .is_none_or(|x1| x1 > sub_band.rect.width()) + || block + .output_y + .checked_add(block.height) + .is_none_or(|y1| y1 > sub_band.rect.height()) + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let block_width = block.width as usize; + for row in 0..block.height as usize { + let dst_start = (block.output_y as usize + row) + .checked_mul(sub_band_width) + .and_then(|offset| offset.checked_add(block.output_x as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let dst_end = dst_start + .checked_add(block_width) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let src_start = row + .checked_mul(block_width) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let src_end = src_start + .checked_add(block_width) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + base_store[dst_start..dst_end].copy_from_slice(&block.coefficients[src_start..src_end]); + } + } + Ok(()) +} + fn decode_sub_band_ht_blocks( sub_band_idx: usize, sub_band: &SubBand, @@ -1504,9 +1722,12 @@ fn decode_sub_band_ht_blocks( storage: &mut DecompositionStorage<'_>, header: &Header<'_>, ht_decoder: &mut Option<&mut dyn HtCodeBlockDecoder>, + cpu_decode_parallelism: CpuDecodeParallelism, num_bitplanes: u8, dequantization_step: f32, + profile_enabled: bool, ) -> Result<()> { + let coded_bitplanes = add_roi_shift_to_bitplanes(num_bitplanes, component_info.roi_shift, 31)?; let stripe_causal = component_info .coding_style .parameters @@ -1529,11 +1750,11 @@ fn decode_sub_band_ht_blocks( continue; } let actual_bitplanes = if header.strict { - num_bitplanes + coded_bitplanes .checked_sub(code_block.missing_bit_planes) .ok_or(DecodingError::InvalidBitplaneCount)? } else { - num_bitplanes.saturating_sub(code_block.missing_bit_planes) + coded_bitplanes.saturating_sub(code_block.missing_bit_planes) }; let max_coding_passes = if actual_bitplanes == 0 { 0 @@ -1574,6 +1795,7 @@ fn decode_sub_band_ht_blocks( missing_bit_planes: pending.missing_bit_planes, number_of_coding_passes: pending.number_of_coding_passes, num_bitplanes, + roi_shift: component_info.roi_shift, stripe_causal, strict: header.strict, dequantization_step, @@ -1612,6 +1834,34 @@ fn decode_sub_band_ht_blocks( return Ok(()); } + let code_block_count = count_ht_code_blocks(sub_band_idx, sub_band, storage); + if !profile_enabled + && should_decode_ht_sub_band_in_parallel(cpu_decode_parallelism, code_block_count) + { + #[cfg(feature = "parallel")] + { + let pending_blocks = collect_pending_ht_blocks( + sub_band_idx, + sub_band, + storage, + header, + num_bitplanes, + component_info.roi_shift, + )?; + let decoded_blocks = decode_ht_sub_band_blocks_parallel( + &pending_blocks, + header.strict, + num_bitplanes, + component_info.roi_shift, + stripe_causal, + dequantization_step, + )?; + tile_ctx.debug_counters.decoded_code_blocks += decoded_blocks.len(); + copy_decoded_ht_blocks_to_sub_band(&decoded_blocks, sub_band, storage)?; + return Ok(()); + } + } + for precinct in sub_band .precincts .clone() @@ -1627,13 +1877,15 @@ fn decode_sub_band_ht_blocks( continue; } tile_ctx.debug_counters.decoded_code_blocks += 1; - ht_block_decode::decode( + ht_block_decode::decode_with_stats( code_block, - num_bitplanes, + coded_bitplanes, stripe_causal, &mut tile_ctx.ht_block_decode_context, storage, header.strict, + Some(&mut tile_ctx.debug_counters.ht_phase_stats), + profile_enabled, )?; let x_offset = code_block.rect.x0 - sub_band.rect.x0; @@ -1646,8 +1898,11 @@ fn decode_sub_band_ht_blocks( let out_row = &mut base_store[base_idx..]; for (output, coefficient) in out_row.iter_mut().zip(coefficients.iter().copied()) { - *output = - ht_block_decode::coefficient_to_i32(coefficient, num_bitplanes) as f32; + let coefficient = + ht_block_decode::coefficient_to_i32(coefficient, coded_bitplanes); + let coefficient = + apply_roi_maxshift_inverse_i32(coefficient, component_info.roi_shift); + *output = coefficient as f32; *output *= dequantization_step; } @@ -2004,6 +2259,7 @@ mod tests { use crate::error::DecodingError; use crate::j2c::codestream::CodeBlockStyle; use crate::j2c::rect::IntRect; + use alloc::vec; fn classic_test_style() -> CodeBlockStyle { CodeBlockStyle { @@ -2106,4 +2362,20 @@ mod tests { assert_eq!(error, DecodingError::CodeBlockDecodeFailure.into()); } + + #[test] + fn auto_cpu_parallelism_enables_ht_sub_band_parallel_branch() { + assert!(super::should_decode_ht_sub_band_in_parallel( + super::CpuDecodeParallelism::Auto, + 16 + )); + } + + #[test] + fn serial_cpu_parallelism_disables_ht_sub_band_parallel_branch() { + assert!(!super::should_decode_ht_sub_band_in_parallel( + super::CpuDecodeParallelism::Serial, + 16 + )); + } } diff --git a/crates/j2k-native/src/j2c/encode.rs b/crates/j2k-native/src/j2c/encode.rs new file mode 100644 index 00000000..8668f058 --- /dev/null +++ b/crates/j2k-native/src/j2c/encode.rs @@ -0,0 +1,7356 @@ +//! Top-level JPEG 2000 encode orchestration. +//! +//! Coordinates the full encoding pipeline: +//! pixels → MCT → DWT → quantize → EBCOT T1 → T2 → codestream +//! +//! Supports both lossless (5-3 reversible) and lossy (9-7 irreversible) encoding. + +use alloc::vec; +use alloc::vec::Vec; +use core::cmp::Ordering; +use core::ops::Range; + +#[cfg(feature = "parallel")] +use rayon::prelude::*; + +use super::bitplane_encode; +use super::build::SubBandType; +use super::codestream::CodeBlockStyle; +use super::codestream_write::{self, BlockCodingMode, EncodeParams}; +use super::fdwt::{self, DwtDecomposition}; +use super::forward_mct; +use super::ht_block_encode; +use super::packet_encode::{self, CodeBlockPacketData, ResolutionPacket, SubbandPrecinct}; +pub use super::quantize::irreversible_quantization_step_for_subband; +use super::quantize::{self, QuantStepSize}; +use crate::math::{floor_f32, log2_f32}; +use crate::profile; +use crate::{ + CpuOnlyJ2kEncodeStageAccelerator, EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, + IrreversibleQuantizationSubbandScales, J2kDeinterleaveToF32Job, J2kEncodeStageAccelerator, + J2kForwardDwt53Job, J2kForwardDwt53Level, J2kForwardDwt53Output, J2kForwardDwt97Job, + J2kForwardDwt97Level, J2kForwardDwt97Output, J2kForwardIctJob, J2kForwardRctJob, + J2kHtSubbandEncodeJob, J2kHtj2kTileEncodeJob, J2kPacketizationBlockCodingMode, + J2kPacketizationCodeBlock, J2kPacketizationEncodeJob, J2kPacketizationPacketDescriptor, + J2kPacketizationResolution, J2kPacketizationSubband, J2kQuantizeSubbandJob, J2kSubBandType, + J2kTier1CodeBlockEncodeJob, PrecomputedHtj2k53Component, PrecomputedHtj2k53Image, + PrecomputedHtj2k97Component, PrecomputedHtj2k97Image, PreencodedHtj2k97CodeBlock, + PreencodedHtj2k97CompactCodeBlock, PreencodedHtj2k97CompactComponent, + PreencodedHtj2k97CompactImage, PreencodedHtj2k97CompactResolution, + PreencodedHtj2k97CompactSubband, PreencodedHtj2k97Component, PreencodedHtj2k97Image, + PreencodedHtj2k97Resolution, PreencodedHtj2k97Subband, PrequantizedHtj2k97Component, + PrequantizedHtj2k97Image, PrequantizedHtj2k97Resolution, PrequantizedHtj2k97Subband, +}; +use crate::{DecodeSettings, Image}; + +const HT_CPU_PARALLEL_FALLBACK_MIN_JOBS: usize = 4; + +/// Encoding options for JPEG 2000. +#[derive(Debug, Clone)] +pub struct EncodeOptions { + /// Number of decomposition levels (default: 5). + pub num_decomposition_levels: u8, + /// Use reversible (lossless) transform (default: true). + pub reversible: bool, + /// Code-block width exponent minus 2 (default: 4, meaning 2^6=64). + pub code_block_width_exp: u8, + /// Code-block height exponent minus 2 (default: 4, meaning 2^6=64). + pub code_block_height_exp: u8, + /// Number of guard bits (default: 1 for reversible, 2 for irreversible). + pub guard_bits: u8, + /// Encode using HT block coding (HTJ2K / Part 15) instead of classic EBCOT. + pub use_ht_block_coding: bool, + /// Packet progression order to write in COD and use for packetization. + pub progression_order: EncodeProgressionOrder, + /// Write a TLM marker segment for the single tile-part. + pub write_tlm: bool, + /// Write PLT packet-length marker segments in the tile-part header. + pub write_plt: bool, + /// Write PLM packet-length marker segments in the main header. + pub write_plm: bool, + /// Write SOP marker segments before packets. + pub write_sop: bool, + /// Write EPH markers after packet headers. + pub write_eph: bool, + /// Apply the JPEG 2000 multi-component color transform for 3+ component inputs. + pub use_mct: bool, + /// Number of cumulative quality layers to emit. + pub num_layers: u8, + /// Optional cumulative packet-body byte targets for each quality layer. + pub quality_layer_byte_targets: Vec, + /// Decode and verify HTJ2K codestreams inside the native encoder. + pub validate_high_throughput_codestream: bool, + /// Multiplier applied to irreversible 9/7 scalar quantization step sizes. + /// + /// `1.0` preserves the near-lossless default step sizes. Larger values + /// produce smaller codestreams by coarsening quantization. + pub irreversible_quantization_scale: f32, + /// Per-subband multipliers applied on top of + /// `irreversible_quantization_scale`. + pub irreversible_quantization_subband_scales: IrreversibleQuantizationSubbandScales, + /// Optional per-component SIZ sampling factors (`XRsiz`, `YRsiz`). + /// + /// `None` means every component is stored at the reference-grid + /// resolution. This is experimental and primarily intended for precomputed + /// coefficient encoders that preserve JPEG-native chroma subsampling. + pub component_sampling: Option>, + /// Optional tile width and height for multi-tile codestream output. + pub tile_size: Option<(u32, u32)>, + /// Optional precinct exponents in COD order, one per resolution level. + pub precinct_exponents: Vec<(u8, u8)>, +} + +impl Default for EncodeOptions { + fn default() -> Self { + Self { + num_decomposition_levels: 5, + reversible: true, + code_block_width_exp: 4, + code_block_height_exp: 4, + guard_bits: 1, + use_ht_block_coding: false, + progression_order: EncodeProgressionOrder::Lrcp, + write_tlm: false, + write_plt: false, + write_plm: false, + write_sop: false, + write_eph: false, + use_mct: true, + num_layers: 1, + quality_layer_byte_targets: Vec::new(), + validate_high_throughput_codestream: true, + irreversible_quantization_scale: 1.0, + irreversible_quantization_subband_scales: + IrreversibleQuantizationSubbandScales::default(), + component_sampling: None, + tile_size: None, + precinct_exponents: Vec::new(), + } + } +} + +/// JPEG 2000 packet progression orders supported by the encoder. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum EncodeProgressionOrder { + /// Layer-resolution-component-position progression. + #[default] + Lrcp, + /// Resolution-layer-component-position progression. + Rlcp, + /// Resolution-position-component-layer progression. + Rpcl, + /// Position-component-resolution-layer progression. + Pcrl, + /// Component-position-resolution-layer progression. + Cprl, +} + +/// Encode pixel data into a JPEG 2000 codestream. +/// +/// # Arguments +/// * `pixels` — Raw pixel data. For 8-bit: one byte per sample. For >8-bit: two bytes per sample (little-endian u16). +/// * `width` — Image width in pixels. +/// * `height` — Image height in pixels. +/// * `num_components` — Number of components (1 for grayscale, 3 for RGB). +/// * `bit_depth` — Bits per sample (e.g., 8, 12, 16). +/// * `signed` — Whether samples are signed. +/// * `options` — Encoding parameters. +/// +/// # Returns +/// The encoded JPEG 2000 codestream bytes (`.j2c` format). +pub fn encode( + pixels: &[u8], + width: u32, + height: u32, + num_components: u8, + bit_depth: u8, + signed: bool, + options: &EncodeOptions, +) -> Result, &'static str> { + let mut accelerator = CpuOnlyJ2kEncodeStageAccelerator; + encode_with_accelerator( + pixels, + width, + height, + num_components, + bit_depth, + signed, + options, + &mut accelerator, + ) +} + +/// Encode pixel data into a JPEG 2000 codestream using optional encode-stage hooks. +/// +/// Stage hooks may accelerate forward RCT, forward 5/3 DWT, Tier-1 code-block +/// encode, and packetization. Returning fallback from a hook preserves the CPU +/// baseline for that stage. +pub fn encode_with_accelerator( + pixels: &[u8], + width: u32, + height: u32, + num_components: u8, + bit_depth: u8, + signed: bool, + options: &EncodeOptions, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + let block_coding_mode = block_coding_mode(options); + let codestream = encode_impl( + pixels, + width, + height, + num_components, + bit_depth, + signed, + options, + block_coding_mode, + accelerator, + )?; + + if block_coding_mode == BlockCodingMode::HighThroughput + && options.validate_high_throughput_codestream + { + validate_htj2k_codestream( + &codestream, + pixels, + width, + height, + num_components, + bit_depth, + signed, + options.reversible, + )?; + } + + Ok(codestream) +} + +/// Encode pixel data into an HTJ2K codestream. +/// +/// Lossless HTJ2K output is self-validated before it is returned. +pub fn encode_htj2k( + pixels: &[u8], + width: u32, + height: u32, + num_components: u8, + bit_depth: u8, + signed: bool, + options: &EncodeOptions, +) -> Result, &'static str> { + let mut options = options.clone(); + options.use_ht_block_coding = true; + encode( + pixels, + width, + height, + num_components, + bit_depth, + signed, + &options, + ) +} + +/// Encode precomputed reversible 5/3 wavelet coefficients into an HTJ2K +/// codestream. +/// +/// This experimental entry point reuses the existing quantization, HT block +/// coding, packetization, and codestream writer stages. It bypasses the +/// encoder's forward DWT stage by supplying precomputed DWT output through the +/// internal stage hook. Coefficients are expected in the same sample domain as +/// the native encoder's FDWT input: unsigned components are already level +/// shifted by subtracting `2^(bit_depth - 1)`. +pub fn encode_precomputed_htj2k_53( + image: &PrecomputedHtj2k53Image, + options: &EncodeOptions, +) -> Result, &'static str> { + let mut accelerator = CpuOnlyJ2kEncodeStageAccelerator; + encode_precomputed_htj2k_53_with_mct_and_accelerator(image, options, false, &mut accelerator) +} + +/// Encode precomputed reversible 5/3 wavelet coefficients into an HTJ2K +/// codestream using optional block encode and packetization hooks. +pub fn encode_precomputed_htj2k_53_with_accelerator( + image: &PrecomputedHtj2k53Image, + options: &EncodeOptions, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + encode_precomputed_htj2k_53_with_mct_and_accelerator(image, options, false, accelerator) +} + +/// Encode precomputed reversible 5/3 wavelet coefficients into an HTJ2K +/// codestream while controlling the output COD multi-component transform flag. +/// +/// This is intended for coefficient-domain JPEG 2000 family recoding, where +/// source codestream components may already be reversible-color-transformed. +pub fn encode_precomputed_htj2k_53_with_mct( + image: &PrecomputedHtj2k53Image, + options: &EncodeOptions, + use_mct: bool, +) -> Result, &'static str> { + let mut accelerator = CpuOnlyJ2kEncodeStageAccelerator; + encode_precomputed_htj2k_53_with_mct_and_accelerator(image, options, use_mct, &mut accelerator) +} + +/// Encode precomputed reversible 5/3 wavelet coefficients while controlling +/// the output COD multi-component transform flag and using optional encode +/// stage hooks. +pub fn encode_precomputed_htj2k_53_with_mct_and_accelerator( + image: &PrecomputedHtj2k53Image, + options: &EncodeOptions, + use_mct: bool, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + if image.width == 0 || image.height == 0 { + return Err("invalid dimensions"); + } + if image.components.is_empty() || image.components.len() > 4 { + return Err("unsupported component count"); + } + if image.bit_depth == 0 || image.bit_depth > 16 { + return Err("unsupported bit depth"); + } + if image + .components + .iter() + .any(|component| component.x_rsiz == 0 || component.y_rsiz == 0) + { + return Err("component sampling factors must be non-zero"); + } + validate_precomputed_dwt_geometry(image)?; + + let num_components = + u8::try_from(image.components.len()).map_err(|_| "unsupported component count")?; + let num_levels = precomputed_level_count(&image.components)?; + let mut precomputed_options = options.clone(); + precomputed_options.num_decomposition_levels = num_levels; + precomputed_options.reversible = true; + precomputed_options.use_ht_block_coding = true; + precomputed_options.use_mct = use_mct; + precomputed_options.validate_high_throughput_codestream = false; + precomputed_options.component_sampling = Some( + image + .components + .iter() + .map(|component| (component.x_rsiz, component.y_rsiz)) + .collect(), + ); + + let dummy_pixels = + zero_pixel_buffer(image.width, image.height, num_components, image.bit_depth)?; + let mut precomputed_accelerator = PrecomputedDwtAccelerator { + outputs: image + .components + .iter() + .map(|component| component.dwt.clone()) + .collect(), + encode_accelerator: accelerator, + }; + + encode_with_accelerator( + &dummy_pixels, + image.width, + image.height, + num_components, + image.bit_depth, + image.signed, + &precomputed_options, + &mut precomputed_accelerator, + ) +} + +/// Encode precomputed irreversible 9/7 wavelet coefficients into an HTJ2K +/// codestream. +/// +/// This experimental entry point is the lossy counterpart of +/// [`encode_precomputed_htj2k_53`]. It bypasses the encoder's forward 9/7 DWT +/// stage by supplying precomputed floating-point DWT output through the +/// internal stage hook. Coefficients are expected in the same sample domain as +/// the native irreversible FDWT input: unsigned components are already level +/// shifted by subtracting `2^(bit_depth - 1)`. +pub fn encode_precomputed_htj2k_97( + image: &PrecomputedHtj2k97Image, + options: &EncodeOptions, +) -> Result, &'static str> { + let mut accelerator = CpuOnlyJ2kEncodeStageAccelerator; + encode_precomputed_htj2k_97_with_accelerator(image, options, &mut accelerator) +} + +/// Encode precomputed irreversible 9/7 wavelet coefficients into an HTJ2K +/// codestream using optional block encode and packetization hooks. +pub fn encode_precomputed_htj2k_97_with_accelerator( + image: &PrecomputedHtj2k97Image, + options: &EncodeOptions, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + if image.width == 0 || image.height == 0 { + return Err("invalid dimensions"); + } + if image.components.is_empty() || image.components.len() > 4 { + return Err("unsupported component count"); + } + if image.bit_depth == 0 || image.bit_depth > 16 { + return Err("unsupported bit depth"); + } + validate_irreversible_quantization_profile(options)?; + if image + .components + .iter() + .any(|component| component.x_rsiz == 0 || component.y_rsiz == 0) + { + return Err("component sampling factors must be non-zero"); + } + validate_precomputed_dwt97_geometry(image)?; + + let num_components = + u8::try_from(image.components.len()).map_err(|_| "unsupported component count")?; + let num_levels = precomputed_97_level_count(&image.components)?; + let mut precomputed_options = options.clone(); + precomputed_options.num_decomposition_levels = num_levels; + precomputed_options.reversible = false; + precomputed_options.use_ht_block_coding = true; + precomputed_options.use_mct = false; + precomputed_options.validate_high_throughput_codestream = false; + precomputed_options.component_sampling = Some( + image + .components + .iter() + .map(|component| (component.x_rsiz, component.y_rsiz)) + .collect(), + ); + + let dummy_pixels = + zero_pixel_buffer(image.width, image.height, num_components, image.bit_depth)?; + let mut precomputed_accelerator = PrecomputedDwt97Accelerator { + outputs: image + .components + .iter() + .map(|component| component.dwt.clone()) + .collect(), + encode_accelerator: accelerator, + }; + + encode_with_accelerator( + &dummy_pixels, + image.width, + image.height, + num_components, + image.bit_depth, + image.signed, + &precomputed_options, + &mut precomputed_accelerator, + ) +} + +/// Encode multiple precomputed irreversible 9/7 wavelet images while sharing +/// one HT code-block batch across all prepared tiles. +pub fn encode_precomputed_htj2k_97_batch_with_accelerator( + images: &[PrecomputedHtj2k97Image], + options: &EncodeOptions, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result>, &'static str> { + if images.is_empty() { + return Ok(Vec::new()); + } + if options.num_layers != 1 { + return Err("batch precomputed 9/7 encode currently supports one quality layer"); + } + + let mut prepared_images = prepare_precomputed_htj2k97_images_for_batch(images, options)?; + let mut all_packets = Vec::new(); + for prepared in &mut prepared_images { + prepared.packet_count = prepared.prepared_packets.len(); + all_packets.append(&mut prepared.prepared_packets); + } + + let mut encoded_packets = + encode_prepared_resolution_packets(all_packets, accelerator)?.into_iter(); + let mut codestreams = Vec::with_capacity(prepared_images.len()); + for prepared in prepared_images { + let mut resolution_packets = Vec::with_capacity(prepared.packet_count); + for _ in 0..prepared.packet_count { + resolution_packets.push( + encoded_packets + .next() + .ok_or("encoded packet count mismatch")?, + ); + } + let scalar_packet_descriptors = scalar_packet_descriptors(&prepared.packet_descriptors); + let packetized_tile = + packet_encode::form_tile_bitstream_with_descriptors_lengths_and_markers( + &mut resolution_packets, + &scalar_packet_descriptors, + packet_encode::PacketMarkerOptions { + write_sop: prepared.params.write_sop, + write_eph: prepared.params.write_eph, + }, + )?; + codestreams.push(codestream_write::write_codestream_with_packet_lengths( + &prepared.params, + &packetized_tile.data, + &prepared.quant_params, + &packetized_tile.packet_lengths, + )); + } + if encoded_packets.next().is_some() { + return Err("encoded packet count mismatch"); + } + + Ok(codestreams) +} + +#[cfg(feature = "parallel")] +fn prepare_precomputed_htj2k97_images_for_batch( + images: &[PrecomputedHtj2k97Image], + options: &EncodeOptions, +) -> Result, &'static str> { + images + .par_iter() + .map(|image| prepare_precomputed_htj2k97_image_for_batch(image, options)) + .collect() +} + +#[cfg(not(feature = "parallel"))] +fn prepare_precomputed_htj2k97_images_for_batch( + images: &[PrecomputedHtj2k97Image], + options: &EncodeOptions, +) -> Result, &'static str> { + images + .iter() + .map(|image| prepare_precomputed_htj2k97_image_for_batch(image, options)) + .collect() +} + +/// Encode prequantized irreversible 9/7 code-block coefficients into an HTJ2K +/// codestream. +pub fn encode_prequantized_htj2k_97( + image: &PrequantizedHtj2k97Image, + options: &EncodeOptions, +) -> Result, &'static str> { + let mut accelerator = CpuOnlyJ2kEncodeStageAccelerator; + encode_prequantized_htj2k_97_with_accelerator(image, options, &mut accelerator) +} + +/// Encode prequantized irreversible 9/7 code-block coefficients into an HTJ2K +/// codestream using optional block encode and packetization hooks. +pub fn encode_prequantized_htj2k_97_with_accelerator( + image: &PrequantizedHtj2k97Image, + options: &EncodeOptions, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + if image.width == 0 || image.height == 0 { + return Err("invalid dimensions"); + } + if image.components.is_empty() || image.components.len() > 4 { + return Err("unsupported component count"); + } + if image.bit_depth == 0 || image.bit_depth > 16 { + return Err("unsupported bit depth"); + } + validate_irreversible_quantization_profile(options)?; + if image + .components + .iter() + .any(|component| component.x_rsiz == 0 || component.y_rsiz == 0) + { + return Err("component sampling factors must be non-zero"); + } + + let num_components = + u8::try_from(image.components.len()).map_err(|_| "unsupported component count")?; + let num_levels = prequantized_97_level_count(&image.components)?; + let guard_bits = options.guard_bits.max(2); + let step_sizes = quantize::compute_step_sizes_with_irreversible_profile( + image.bit_depth, + num_levels, + false, + guard_bits, + options.irreversible_quantization_scale, + options.irreversible_quantization_subband_scales, + ); + validate_prequantized_htj2k97_image(image, guard_bits, &step_sizes)?; + + let mut prequantized_options = options.clone(); + prequantized_options.num_decomposition_levels = num_levels; + prequantized_options.reversible = false; + prequantized_options.use_ht_block_coding = true; + prequantized_options.use_mct = false; + prequantized_options.validate_high_throughput_codestream = false; + prequantized_options.component_sampling = Some( + image + .components + .iter() + .map(|component| (component.x_rsiz, component.y_rsiz)) + .collect(), + ); + + let component_resolution_packets = image + .components + .iter() + .enumerate() + .map(|(component_idx, component)| { + prepared_resolution_packets_from_prequantized_component(component_idx, component) + }) + .collect::, &'static str>>()?; + let prepared_resolution_packets = + ordered_prepared_resolution_packets(component_resolution_packets, &prequantized_options)?; + let packet_descriptors = packet_descriptors_for_order( + &prepared_resolution_packets, + 1, + prequantized_options.progression_order, + )?; + let mut resolution_packets = + encode_prepared_resolution_packets(prepared_resolution_packets, accelerator)?; + let packetization_resolutions = public_packetization_resolutions(&resolution_packets); + let packetization_job = J2kPacketizationEncodeJob { + resolution_count: resolution_packets.len() as u32, + num_layers: 1, + num_components, + code_block_count: count_code_blocks(&resolution_packets)?, + progression_order: public_packetization_progression_order( + prequantized_options.progression_order, + ), + packet_descriptors: &packet_descriptors, + resolutions: &packetization_resolutions, + }; + let tile_data = accelerator + .encode_packetization(packetization_job)? + .unwrap_or_else(|| { + packet_encode::form_tile_bitstream(&mut resolution_packets, 1, num_components) + }); + + let quant_params: Vec<(u16, u16)> = step_sizes + .iter() + .map(|s| (s.exponent, s.mantissa)) + .collect(); + let params = EncodeParams { + width: image.width, + height: image.height, + tile_width: image.width, + tile_height: image.height, + num_components, + bit_depth: image.bit_depth, + signed: image.signed, + num_decomposition_levels: num_levels, + reversible: false, + code_block_width_exp: prequantized_options.code_block_width_exp, + code_block_height_exp: prequantized_options.code_block_height_exp, + num_layers: 1, + use_mct: false, + guard_bits, + block_coding_mode: BlockCodingMode::HighThroughput, + progression_order: prequantized_options.progression_order, + write_tlm: prequantized_options.write_tlm, + write_plt: prequantized_options.write_plt, + write_plm: prequantized_options.write_plm, + write_sop: prequantized_options.write_sop, + write_eph: prequantized_options.write_eph, + terminate_coding_passes: false, + component_sampling: prequantized_options + .component_sampling + .clone() + .ok_or("component sampling missing")?, + precinct_exponents: precinct_exponents_for_options(&prequantized_options, num_levels)?, + }; + + Ok(codestream_write::write_codestream( + ¶ms, + &tile_data, + &quant_params, + )) +} + +/// Encode preencoded irreversible 9/7 HTJ2K code-block payloads into a +/// codestream. +pub fn encode_preencoded_htj2k_97( + image: &PreencodedHtj2k97Image, + options: &EncodeOptions, +) -> Result, &'static str> { + let mut accelerator = CpuOnlyJ2kEncodeStageAccelerator; + encode_preencoded_htj2k_97_with_accelerator(image, options, &mut accelerator) +} + +/// Encode preencoded irreversible 9/7 HTJ2K code-block payloads into a +/// codestream using optional packetization hooks. +pub fn encode_preencoded_htj2k_97_with_accelerator( + image: &PreencodedHtj2k97Image, + options: &EncodeOptions, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + if image.width == 0 || image.height == 0 { + return Err("invalid dimensions"); + } + if image.components.is_empty() || image.components.len() > 4 { + return Err("unsupported component count"); + } + if image.bit_depth == 0 || image.bit_depth > 16 { + return Err("unsupported bit depth"); + } + validate_irreversible_quantization_profile(options)?; + if image + .components + .iter() + .any(|component| component.x_rsiz == 0 || component.y_rsiz == 0) + { + return Err("component sampling factors must be non-zero"); + } + + let num_components = + u8::try_from(image.components.len()).map_err(|_| "unsupported component count")?; + let num_levels = preencoded_97_level_count(&image.components)?; + let guard_bits = options.guard_bits.max(2); + let step_sizes = quantize::compute_step_sizes_with_irreversible_profile( + image.bit_depth, + num_levels, + false, + guard_bits, + options.irreversible_quantization_scale, + options.irreversible_quantization_subband_scales, + ); + validate_preencoded_htj2k97_image(image, guard_bits, &step_sizes)?; + + let mut preencoded_options = options.clone(); + preencoded_options.num_decomposition_levels = num_levels; + preencoded_options.reversible = false; + preencoded_options.use_ht_block_coding = true; + preencoded_options.use_mct = false; + preencoded_options.validate_high_throughput_codestream = false; + preencoded_options.component_sampling = Some( + image + .components + .iter() + .map(|component| (component.x_rsiz, component.y_rsiz)) + .collect(), + ); + + let component_resolution_packets = image + .components + .iter() + .enumerate() + .map(|(component_idx, component)| { + prepared_resolution_packets_from_preencoded_component(component_idx, component) + }) + .collect::, &'static str>>()?; + let prepared_resolution_packets = + ordered_prepared_resolution_packets(component_resolution_packets, &preencoded_options)?; + let packet_descriptors = packet_descriptors_for_order( + &prepared_resolution_packets, + 1, + preencoded_options.progression_order, + )?; + let mut resolution_packets = + encode_prepared_resolution_packets(prepared_resolution_packets, accelerator)?; + let packetization_resolutions = public_packetization_resolutions(&resolution_packets); + let packetization_job = J2kPacketizationEncodeJob { + resolution_count: resolution_packets.len() as u32, + num_layers: 1, + num_components, + code_block_count: count_code_blocks(&resolution_packets)?, + progression_order: public_packetization_progression_order( + preencoded_options.progression_order, + ), + packet_descriptors: &packet_descriptors, + resolutions: &packetization_resolutions, + }; + let tile_data = accelerator + .encode_packetization(packetization_job)? + .unwrap_or_else(|| { + packet_encode::form_tile_bitstream(&mut resolution_packets, 1, num_components) + }); + + let quant_params: Vec<(u16, u16)> = step_sizes + .iter() + .map(|s| (s.exponent, s.mantissa)) + .collect(); + let params = EncodeParams { + width: image.width, + height: image.height, + tile_width: image.width, + tile_height: image.height, + num_components, + bit_depth: image.bit_depth, + signed: image.signed, + num_decomposition_levels: num_levels, + reversible: false, + code_block_width_exp: preencoded_options.code_block_width_exp, + code_block_height_exp: preencoded_options.code_block_height_exp, + num_layers: 1, + use_mct: false, + guard_bits, + block_coding_mode: BlockCodingMode::HighThroughput, + progression_order: preencoded_options.progression_order, + write_tlm: preencoded_options.write_tlm, + write_plt: preencoded_options.write_plt, + write_plm: preencoded_options.write_plm, + write_sop: preencoded_options.write_sop, + write_eph: preencoded_options.write_eph, + terminate_coding_passes: false, + component_sampling: preencoded_options + .component_sampling + .clone() + .ok_or("component sampling missing")?, + precinct_exponents: precinct_exponents_for_options(&preencoded_options, num_levels)?, + }; + + Ok(codestream_write::write_codestream( + ¶ms, + &tile_data, + &quant_params, + )) +} + +/// Encode preencoded irreversible 9/7 HTJ2K code-block payloads into a +/// codestream, consuming the image so code-block payloads can move into packet +/// preparation without cloning. +pub fn encode_preencoded_htj2k_97_owned_with_accelerator( + image: PreencodedHtj2k97Image, + options: &EncodeOptions, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + if image.width == 0 || image.height == 0 { + return Err("invalid dimensions"); + } + if image.components.is_empty() || image.components.len() > 4 { + return Err("unsupported component count"); + } + if image.bit_depth == 0 || image.bit_depth > 16 { + return Err("unsupported bit depth"); + } + validate_irreversible_quantization_profile(options)?; + if image + .components + .iter() + .any(|component| component.x_rsiz == 0 || component.y_rsiz == 0) + { + return Err("component sampling factors must be non-zero"); + } + + let width = image.width; + let height = image.height; + let bit_depth = image.bit_depth; + let signed = image.signed; + let num_components = + u8::try_from(image.components.len()).map_err(|_| "unsupported component count")?; + let num_levels = preencoded_97_level_count(&image.components)?; + let guard_bits = options.guard_bits.max(2); + let step_sizes = quantize::compute_step_sizes_with_irreversible_profile( + bit_depth, + num_levels, + false, + guard_bits, + options.irreversible_quantization_scale, + options.irreversible_quantization_subband_scales, + ); + validate_preencoded_htj2k97_image(&image, guard_bits, &step_sizes)?; + + let component_sampling = image + .components + .iter() + .map(|component| (component.x_rsiz, component.y_rsiz)) + .collect::>(); + let mut preencoded_options = options.clone(); + preencoded_options.num_decomposition_levels = num_levels; + preencoded_options.reversible = false; + preencoded_options.use_ht_block_coding = true; + preencoded_options.use_mct = false; + preencoded_options.validate_high_throughput_codestream = false; + preencoded_options.component_sampling = Some(component_sampling); + + let component_resolution_packets = image + .components + .into_iter() + .enumerate() + .map(|(component_idx, component)| { + prepared_resolution_packets_from_preencoded_component_owned(component_idx, component) + }) + .collect::, &'static str>>()?; + let prepared_resolution_packets = + ordered_prepared_resolution_packets(component_resolution_packets, &preencoded_options)?; + let packet_descriptors = packet_descriptors_for_order( + &prepared_resolution_packets, + 1, + preencoded_options.progression_order, + )?; + let mut resolution_packets = + encode_prepared_resolution_packets(prepared_resolution_packets, accelerator)?; + let packetization_resolutions = public_packetization_resolutions(&resolution_packets); + let packetization_job = J2kPacketizationEncodeJob { + resolution_count: resolution_packets.len() as u32, + num_layers: 1, + num_components, + code_block_count: count_code_blocks(&resolution_packets)?, + progression_order: public_packetization_progression_order( + preencoded_options.progression_order, + ), + packet_descriptors: &packet_descriptors, + resolutions: &packetization_resolutions, + }; + let tile_data = accelerator + .encode_packetization(packetization_job)? + .unwrap_or_else(|| { + packet_encode::form_tile_bitstream(&mut resolution_packets, 1, num_components) + }); + + let quant_params: Vec<(u16, u16)> = step_sizes + .iter() + .map(|s| (s.exponent, s.mantissa)) + .collect(); + let params = EncodeParams { + width, + height, + tile_width: width, + tile_height: height, + num_components, + bit_depth, + signed, + num_decomposition_levels: num_levels, + reversible: false, + code_block_width_exp: preencoded_options.code_block_width_exp, + code_block_height_exp: preencoded_options.code_block_height_exp, + num_layers: 1, + use_mct: false, + guard_bits, + block_coding_mode: BlockCodingMode::HighThroughput, + progression_order: preencoded_options.progression_order, + write_tlm: preencoded_options.write_tlm, + write_plt: preencoded_options.write_plt, + write_plm: preencoded_options.write_plm, + write_sop: preencoded_options.write_sop, + write_eph: preencoded_options.write_eph, + terminate_coding_passes: false, + component_sampling: preencoded_options + .component_sampling + .clone() + .ok_or("component sampling missing")?, + precinct_exponents: precinct_exponents_for_options(&preencoded_options, num_levels)?, + }; + + Ok(codestream_write::write_codestream( + ¶ms, + &tile_data, + &quant_params, + )) +} + +/// Encode compact preencoded irreversible 9/7 HTJ2K code-block payloads into a +/// codestream, borrowing code-block ranges from one image-level payload buffer +/// during packetization. +pub fn encode_preencoded_htj2k_97_compact_owned_with_accelerator( + image: PreencodedHtj2k97CompactImage, + options: &EncodeOptions, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + if image.width == 0 || image.height == 0 { + return Err("invalid dimensions"); + } + if image.components.is_empty() || image.components.len() > 4 { + return Err("unsupported component count"); + } + if image.bit_depth == 0 || image.bit_depth > 16 { + return Err("unsupported bit depth"); + } + validate_irreversible_quantization_profile(options)?; + if image + .components + .iter() + .any(|component| component.x_rsiz == 0 || component.y_rsiz == 0) + { + return Err("component sampling factors must be non-zero"); + } + + let width = image.width; + let height = image.height; + let bit_depth = image.bit_depth; + let signed = image.signed; + let num_components = + u8::try_from(image.components.len()).map_err(|_| "unsupported component count")?; + let num_levels = preencoded_compact_97_level_count(&image.components)?; + let guard_bits = options.guard_bits.max(2); + let step_sizes = quantize::compute_step_sizes_with_irreversible_profile( + bit_depth, + num_levels, + false, + guard_bits, + options.irreversible_quantization_scale, + options.irreversible_quantization_subband_scales, + ); + validate_preencoded_compact_htj2k97_image(&image, guard_bits, &step_sizes)?; + + let component_sampling = image + .components + .iter() + .map(|component| (component.x_rsiz, component.y_rsiz)) + .collect::>(); + let mut preencoded_options = options.clone(); + preencoded_options.num_decomposition_levels = num_levels; + preencoded_options.reversible = false; + preencoded_options.use_ht_block_coding = true; + preencoded_options.use_mct = false; + preencoded_options.validate_high_throughput_codestream = false; + preencoded_options.component_sampling = Some(component_sampling); + + let PreencodedHtj2k97CompactImage { + payload, + components, + .. + } = image; + let component_resolution_packets = components + .iter() + .enumerate() + .map(|(component_idx, component)| { + prepared_resolution_packets_from_preencoded_compact_component( + component_idx, + component, + &payload, + ) + }) + .collect::, &'static str>>()?; + let prepared_resolution_packets = ordered_prepared_compact_resolution_packets( + component_resolution_packets, + &preencoded_options, + )?; + let packet_descriptors = packet_descriptors_for_compact_order( + &prepared_resolution_packets, + 1, + preencoded_options.progression_order, + )?; + let packetization_resolutions = + public_packetization_resolutions_from_compact(&prepared_resolution_packets); + let packetization_job = J2kPacketizationEncodeJob { + resolution_count: packetization_resolutions.len() as u32, + num_layers: 1, + num_components, + code_block_count: count_compact_code_blocks(&prepared_resolution_packets)?, + progression_order: public_packetization_progression_order( + preencoded_options.progression_order, + ), + packet_descriptors: &packet_descriptors, + resolutions: &packetization_resolutions, + }; + let tile_data = accelerator + .encode_packetization(packetization_job)? + .map_or_else( + || crate::encode_j2k_packetization_scalar(packetization_job), + Ok, + )?; + + let quant_params: Vec<(u16, u16)> = step_sizes + .iter() + .map(|s| (s.exponent, s.mantissa)) + .collect(); + let params = EncodeParams { + width, + height, + tile_width: width, + tile_height: height, + num_components, + bit_depth, + signed, + num_decomposition_levels: num_levels, + reversible: false, + code_block_width_exp: preencoded_options.code_block_width_exp, + code_block_height_exp: preencoded_options.code_block_height_exp, + num_layers: 1, + use_mct: false, + guard_bits, + block_coding_mode: BlockCodingMode::HighThroughput, + progression_order: preencoded_options.progression_order, + write_tlm: preencoded_options.write_tlm, + write_plt: preencoded_options.write_plt, + write_plm: preencoded_options.write_plm, + write_sop: preencoded_options.write_sop, + write_eph: preencoded_options.write_eph, + terminate_coding_passes: false, + component_sampling: preencoded_options + .component_sampling + .clone() + .ok_or("component sampling missing")?, + precinct_exponents: precinct_exponents_for_options(&preencoded_options, num_levels)?, + }; + + Ok(codestream_write::write_codestream( + ¶ms, + &tile_data, + &quant_params, + )) +} + +fn validate_precomputed_dwt_geometry(image: &PrecomputedHtj2k53Image) -> Result<(), &'static str> { + for component in &image.components { + let component_width = image.width.div_ceil(u32::from(component.x_rsiz)); + let component_height = image.height.div_ceil(u32::from(component.y_rsiz)); + validate_precomputed_component_dwt_geometry( + &component.dwt, + component_width, + component_height, + )?; + } + + Ok(()) +} + +fn validate_precomputed_dwt97_geometry( + image: &PrecomputedHtj2k97Image, +) -> Result<(), &'static str> { + for component in &image.components { + let component_width = image.width.div_ceil(u32::from(component.x_rsiz)); + let component_height = image.height.div_ceil(u32::from(component.y_rsiz)); + validate_precomputed_component_dwt_geometry( + &component.dwt, + component_width, + component_height, + )?; + } + + Ok(()) +} + +fn validate_precomputed_component_dwt_geometry( + dwt: &impl PrecomputedDwtGeometryView, + component_width: u32, + component_height: u32, +) -> Result<(), &'static str> { + if dwt.level_count() == 0 { + return Err("precomputed DWT must contain at least one decomposition level"); + } + if let Some(highest_level) = dwt.last_level_geometry() { + if highest_level.width != component_width || highest_level.height != component_height { + return Err("precomputed DWT component dimensions mismatch"); + } + } + + let mut expected_width = component_width; + let mut expected_height = component_height; + for level_index in (0..dwt.level_count()).rev() { + let level = dwt.level_geometry(level_index); + let low_width = expected_width.div_ceil(2); + let low_height = expected_height.div_ceil(2); + let high_width = expected_width / 2; + let high_height = expected_height / 2; + + if level.width != expected_width + || level.height != expected_height + || level.low_width != low_width + || level.low_height != low_height + || level.high_width != high_width + || level.high_height != high_height + { + return Err("precomputed DWT recursive geometry mismatch"); + } + validate_band_len(level.hl_len, high_width, low_height)?; + validate_band_len(level.lh_len, low_width, high_height)?; + validate_band_len(level.hh_len, high_width, high_height)?; + + expected_width = low_width; + expected_height = low_height; + } + + if dwt.ll_width() != expected_width || dwt.ll_height() != expected_height { + return Err("precomputed DWT component dimensions mismatch"); + } + validate_band_len(dwt.ll_len(), expected_width, expected_height) +} + +#[derive(Debug, Clone, Copy)] +struct PrecomputedDwtLevelGeometry { + width: u32, + height: u32, + low_width: u32, + low_height: u32, + high_width: u32, + high_height: u32, + hl_len: usize, + lh_len: usize, + hh_len: usize, +} + +trait PrecomputedDwtGeometryView { + fn ll_len(&self) -> usize; + fn ll_width(&self) -> u32; + fn ll_height(&self) -> u32; + fn level_count(&self) -> usize; + fn level_geometry(&self, index: usize) -> PrecomputedDwtLevelGeometry; + + fn last_level_geometry(&self) -> Option { + self.level_count() + .checked_sub(1) + .map(|index| self.level_geometry(index)) + } +} + +impl PrecomputedDwtGeometryView for J2kForwardDwt53Output { + fn ll_len(&self) -> usize { + self.ll.len() + } + + fn ll_width(&self) -> u32 { + self.ll_width + } + + fn ll_height(&self) -> u32 { + self.ll_height + } + + fn level_count(&self) -> usize { + self.levels.len() + } + + fn level_geometry(&self, index: usize) -> PrecomputedDwtLevelGeometry { + let level = &self.levels[index]; + PrecomputedDwtLevelGeometry { + width: level.width, + height: level.height, + low_width: level.low_width, + low_height: level.low_height, + high_width: level.high_width, + high_height: level.high_height, + hl_len: level.hl.len(), + lh_len: level.lh.len(), + hh_len: level.hh.len(), + } + } +} + +impl PrecomputedDwtGeometryView for J2kForwardDwt97Output { + fn ll_len(&self) -> usize { + self.ll.len() + } + + fn ll_width(&self) -> u32 { + self.ll_width + } + + fn ll_height(&self) -> u32 { + self.ll_height + } + + fn level_count(&self) -> usize { + self.levels.len() + } + + fn level_geometry(&self, index: usize) -> PrecomputedDwtLevelGeometry { + let level = &self.levels[index]; + PrecomputedDwtLevelGeometry { + width: level.width, + height: level.height, + low_width: level.low_width, + low_height: level.low_height, + high_width: level.high_width, + high_height: level.high_height, + hl_len: level.hl.len(), + lh_len: level.lh.len(), + hh_len: level.hh.len(), + } + } +} + +fn precomputed_level_count(components: &[PrecomputedHtj2k53Component]) -> Result { + let first = components + .first() + .ok_or("unsupported component count")? + .dwt + .levels + .len(); + if components + .iter() + .any(|component| component.dwt.levels.len() != first) + { + return Err("precomputed components must use the same decomposition level count"); + } + u8::try_from(first).map_err(|_| "decomposition level count exceeds u8") +} + +fn precomputed_97_level_count( + components: &[PrecomputedHtj2k97Component], +) -> Result { + let first = components + .first() + .ok_or("unsupported component count")? + .dwt + .levels + .len(); + if components + .iter() + .any(|component| component.dwt.levels.len() != first) + { + return Err("precomputed components must use the same decomposition level count"); + } + u8::try_from(first).map_err(|_| "decomposition level count exceeds u8") +} + +fn prequantized_97_level_count( + components: &[PrequantizedHtj2k97Component], +) -> Result { + let first = components + .first() + .ok_or("unsupported component count")? + .resolutions + .len() + .checked_sub(1) + .ok_or("prequantized components must contain at least one decomposition level")?; + if components + .iter() + .any(|component| component.resolutions.len() != first + 1) + { + return Err("prequantized components must use the same decomposition level count"); + } + u8::try_from(first).map_err(|_| "decomposition level count exceeds u8") +} + +fn preencoded_97_level_count( + components: &[PreencodedHtj2k97Component], +) -> Result { + let first = components + .first() + .ok_or("unsupported component count")? + .resolutions + .len() + .checked_sub(1) + .ok_or("preencoded components must contain at least one decomposition level")?; + if components + .iter() + .any(|component| component.resolutions.len() != first + 1) + { + return Err("preencoded components must use the same decomposition level count"); + } + u8::try_from(first).map_err(|_| "decomposition level count exceeds u8") +} + +fn preencoded_compact_97_level_count( + components: &[PreencodedHtj2k97CompactComponent], +) -> Result { + let first = components + .first() + .ok_or("unsupported component count")? + .resolutions + .len() + .checked_sub(1) + .ok_or("preencoded components must contain at least one decomposition level")?; + if components + .iter() + .any(|component| component.resolutions.len() != first + 1) + { + return Err("preencoded components must use the same decomposition level count"); + } + u8::try_from(first).map_err(|_| "decomposition level count exceeds u8") +} + +fn validate_prequantized_htj2k97_image( + image: &PrequantizedHtj2k97Image, + guard_bits: u8, + step_sizes: &[QuantStepSize], +) -> Result<(), &'static str> { + for component in &image.components { + if component.resolutions.is_empty() { + return Err("prequantized components must contain at least one resolution"); + } + validate_prequantized_resolution( + &component.resolutions[0], + &[J2kSubBandType::LowLow], + guard_bits, + &step_sizes[0..1], + )?; + for (level_index, resolution) in component.resolutions.iter().enumerate().skip(1) { + let step_base = 1 + (level_index - 1) * 3; + validate_prequantized_resolution( + resolution, + &[ + J2kSubBandType::HighLow, + J2kSubBandType::LowHigh, + J2kSubBandType::HighHigh, + ], + guard_bits, + &step_sizes[step_base..step_base + 3], + )?; + } + } + + Ok(()) +} + +fn validate_preencoded_htj2k97_image( + image: &PreencodedHtj2k97Image, + guard_bits: u8, + step_sizes: &[QuantStepSize], +) -> Result<(), &'static str> { + for component in &image.components { + if component.resolutions.is_empty() { + return Err("preencoded components must contain at least one resolution"); + } + validate_preencoded_resolution( + &component.resolutions[0], + &[J2kSubBandType::LowLow], + guard_bits, + &step_sizes[0..1], + )?; + for (level_index, resolution) in component.resolutions.iter().enumerate().skip(1) { + let step_base = 1 + (level_index - 1) * 3; + validate_preencoded_resolution( + resolution, + &[ + J2kSubBandType::HighLow, + J2kSubBandType::LowHigh, + J2kSubBandType::HighHigh, + ], + guard_bits, + &step_sizes[step_base..step_base + 3], + )?; + } + } + + Ok(()) +} + +fn validate_preencoded_compact_htj2k97_image( + image: &PreencodedHtj2k97CompactImage, + guard_bits: u8, + step_sizes: &[QuantStepSize], +) -> Result<(), &'static str> { + for component in &image.components { + if component.resolutions.is_empty() { + return Err("preencoded components must contain at least one resolution"); + } + validate_preencoded_compact_resolution( + &component.resolutions[0], + &[J2kSubBandType::LowLow], + guard_bits, + &step_sizes[0..1], + image.payload.len(), + )?; + for (level_index, resolution) in component.resolutions.iter().enumerate().skip(1) { + let step_base = 1 + (level_index - 1) * 3; + validate_preencoded_compact_resolution( + resolution, + &[ + J2kSubBandType::HighLow, + J2kSubBandType::LowHigh, + J2kSubBandType::HighHigh, + ], + guard_bits, + &step_sizes[step_base..step_base + 3], + image.payload.len(), + )?; + } + } + + Ok(()) +} + +fn validate_prequantized_resolution( + resolution: &PrequantizedHtj2k97Resolution, + expected_subbands: &[J2kSubBandType], + guard_bits: u8, + step_sizes: &[QuantStepSize], +) -> Result<(), &'static str> { + if resolution.subbands.len() != expected_subbands.len() { + return Err("prequantized resolution subband count mismatch"); + } + for ((subband, expected_subband), step_size) in resolution + .subbands + .iter() + .zip(expected_subbands) + .zip(step_sizes) + { + if subband.sub_band_type != *expected_subband { + return Err("prequantized resolution subband order mismatch"); + } + let expected_blocks = subband + .num_cbs_x + .checked_mul(subband.num_cbs_y) + .ok_or("prequantized code-block count overflow")?; + if expected_blocks == 0 { + if subband.total_bitplanes != 0 || !subband.code_blocks.is_empty() { + return Err("empty prequantized subbands must not contain code-block data"); + } + continue; + } + debug_assert!(step_size.exponent <= u16::from(u8::MAX)); + let expected_total_bitplanes = guard_bits + .saturating_add(step_size.exponent as u8) + .saturating_sub(1); + if subband.total_bitplanes != expected_total_bitplanes { + return Err("prequantized subband bitplane count mismatch"); + } + if usize::try_from(expected_blocks).map_err(|_| "prequantized code-block count overflow")? + != subband.code_blocks.len() + { + return Err("prequantized code-block count mismatch"); + } + for block in &subband.code_blocks { + if block.width == 0 || block.height == 0 { + return Err("prequantized code-block dimensions must be non-zero"); + } + validate_band_len(block.coefficients.len(), block.width, block.height)?; + } + } + + Ok(()) +} + +fn validate_preencoded_resolution( + resolution: &PreencodedHtj2k97Resolution, + expected_subbands: &[J2kSubBandType], + guard_bits: u8, + step_sizes: &[QuantStepSize], +) -> Result<(), &'static str> { + if resolution.subbands.len() != expected_subbands.len() { + return Err("preencoded resolution subband count mismatch"); + } + for ((subband, expected_subband), step_size) in resolution + .subbands + .iter() + .zip(expected_subbands) + .zip(step_sizes) + { + if subband.sub_band_type != *expected_subband { + return Err("preencoded resolution subband order mismatch"); + } + let expected_blocks = subband + .num_cbs_x + .checked_mul(subband.num_cbs_y) + .ok_or("preencoded code-block count overflow")?; + if expected_blocks == 0 { + if subband.total_bitplanes != 0 || !subband.code_blocks.is_empty() { + return Err("empty preencoded subbands must not contain code-block data"); + } + continue; + } + debug_assert!(step_size.exponent <= u16::from(u8::MAX)); + let expected_total_bitplanes = guard_bits + .saturating_add(step_size.exponent as u8) + .saturating_sub(1); + if subband.total_bitplanes != expected_total_bitplanes { + return Err("preencoded subband bitplane count mismatch"); + } + if usize::try_from(expected_blocks).map_err(|_| "preencoded code-block count overflow")? + != subband.code_blocks.len() + { + return Err("preencoded code-block count mismatch"); + } + for block in &subband.code_blocks { + if block.width == 0 || block.height == 0 { + return Err("preencoded code-block dimensions must be non-zero"); + } + validate_preencoded_code_block_payload(&block.encoded, subband.total_bitplanes)?; + } + } + + Ok(()) +} + +fn validate_preencoded_compact_resolution( + resolution: &PreencodedHtj2k97CompactResolution, + expected_subbands: &[J2kSubBandType], + guard_bits: u8, + step_sizes: &[QuantStepSize], + payload_len: usize, +) -> Result<(), &'static str> { + if resolution.subbands.len() != expected_subbands.len() { + return Err("preencoded resolution subband count mismatch"); + } + for ((subband, expected_subband), step_size) in resolution + .subbands + .iter() + .zip(expected_subbands) + .zip(step_sizes) + { + if subband.sub_band_type != *expected_subband { + return Err("preencoded resolution subband order mismatch"); + } + let expected_blocks = subband + .num_cbs_x + .checked_mul(subband.num_cbs_y) + .ok_or("preencoded code-block count overflow")?; + if expected_blocks == 0 { + if subband.total_bitplanes != 0 || !subband.code_blocks.is_empty() { + return Err("empty preencoded subbands must not contain code-block data"); + } + continue; + } + debug_assert!(step_size.exponent <= u16::from(u8::MAX)); + let expected_total_bitplanes = guard_bits + .saturating_add(step_size.exponent as u8) + .saturating_sub(1); + if subband.total_bitplanes != expected_total_bitplanes { + return Err("preencoded subband bitplane count mismatch"); + } + if usize::try_from(expected_blocks).map_err(|_| "preencoded code-block count overflow")? + != subband.code_blocks.len() + { + return Err("preencoded code-block count mismatch"); + } + for block in &subband.code_blocks { + if block.width == 0 || block.height == 0 { + return Err("preencoded code-block dimensions must be non-zero"); + } + validate_preencoded_compact_code_block_payload( + block, + payload_len, + subband.total_bitplanes, + )?; + } + } + + Ok(()) +} + +fn validate_preencoded_code_block_payload( + block: &EncodedHtJ2kCodeBlock, + total_bitplanes: u8, +) -> Result<(), &'static str> { + let data_len = u32::try_from(block.data.len()).map_err(|_| "HTJ2K payload too large")?; + if block.num_coding_passes == 0 { + if data_len != 0 || block.cleanup_length != 0 || block.refinement_length != 0 { + return Err("empty HTJ2K code-block payload metadata mismatch"); + } + if block.num_zero_bitplanes != total_bitplanes { + return Err("empty HTJ2K code-block zero-bitplane count mismatch"); + } + return Ok(()); + } + if block.num_coding_passes > 164 { + return Err("HTJ2K code-block coding pass count out of range"); + } + if block.num_zero_bitplanes >= total_bitplanes { + return Err("HTJ2K code-block zero-bitplane count out of range"); + } + let segment_len = block + .cleanup_length + .checked_add(block.refinement_length) + .ok_or("HTJ2K payload segment length overflow")?; + if segment_len != data_len { + return Err("HTJ2K payload segment length mismatch"); + } + Ok(()) +} + +fn validate_preencoded_compact_code_block_payload( + block: &PreencodedHtj2k97CompactCodeBlock, + payload_len: usize, + total_bitplanes: u8, +) -> Result<(), &'static str> { + if block.payload_range.start > block.payload_range.end || block.payload_range.end > payload_len + { + return Err("HTJ2K payload range out of bounds"); + } + let data_len = u32::try_from(block.payload_range.end - block.payload_range.start) + .map_err(|_| "HTJ2K payload too large")?; + if block.num_coding_passes == 0 { + if data_len != 0 || block.cleanup_length != 0 || block.refinement_length != 0 { + return Err("empty HTJ2K code-block payload metadata mismatch"); + } + if block.num_zero_bitplanes != total_bitplanes { + return Err("empty HTJ2K code-block zero-bitplane count mismatch"); + } + return Ok(()); + } + if block.num_coding_passes > 164 { + return Err("HTJ2K code-block coding pass count out of range"); + } + if block.num_zero_bitplanes >= total_bitplanes { + return Err("HTJ2K code-block zero-bitplane count out of range"); + } + let segment_len = block + .cleanup_length + .checked_add(block.refinement_length) + .ok_or("HTJ2K payload segment length overflow")?; + if segment_len != data_len { + return Err("HTJ2K payload segment length mismatch"); + } + Ok(()) +} + +fn prepared_resolution_packets_from_prequantized_component( + component_idx: usize, + component: &PrequantizedHtj2k97Component, +) -> Result, &'static str> { + let component_idx = u8::try_from(component_idx).map_err(|_| "component index exceeds u8")?; + component + .resolutions + .iter() + .enumerate() + .map(|(resolution_idx, resolution)| { + Ok(PreparedResolutionPacket { + component: component_idx, + resolution: u32::try_from(resolution_idx) + .map_err(|_| "resolution index exceeds u32")?, + precinct: 0, + subbands: resolution + .subbands + .iter() + .map(prepared_subband_from_prequantized) + .collect::, &'static str>>()?, + }) + }) + .collect() +} + +fn prepared_subband_from_prequantized( + subband: &PrequantizedHtj2k97Subband, +) -> Result { + Ok(PreparedEncodeSubband { + code_blocks: subband + .code_blocks + .iter() + .map(|block| PreparedEncodeCodeBlock { + coefficients: block.coefficients.clone(), + width: block.width, + height: block.height, + }) + .collect(), + preencoded_ht_code_blocks: None, + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + code_block_width: subband + .code_blocks + .iter() + .map(|block| block.width) + .max() + .unwrap_or(0), + code_block_height: subband + .code_blocks + .iter() + .map(|block| block.height) + .max() + .unwrap_or(0), + width: precomputed_subband_width( + subband.num_cbs_x, + subband.code_blocks.iter().map(|block| block.width), + ), + height: precomputed_subband_height( + subband.num_cbs_x, + subband.num_cbs_y, + subband.code_blocks.iter().map(|block| block.height), + ), + sub_band_type: internal_sub_band_type(subband.sub_band_type), + total_bitplanes: subband.total_bitplanes, + block_coding_mode: BlockCodingMode::HighThroughput, + }) +} + +fn precomputed_subband_width(width_in_blocks: u32, widths: impl Iterator) -> u32 { + if width_in_blocks == 0 { + return 0; + } + + widths.take(width_in_blocks as usize).sum() +} + +fn precomputed_subband_height( + width_in_blocks: u32, + height_in_blocks: u32, + heights: impl Iterator, +) -> u32 { + if width_in_blocks == 0 || height_in_blocks == 0 { + return 0; + } + + heights + .step_by(width_in_blocks as usize) + .take(height_in_blocks as usize) + .sum() +} + +fn prepared_resolution_packets_from_preencoded_component( + component_idx: usize, + component: &PreencodedHtj2k97Component, +) -> Result, &'static str> { + let component_idx = u8::try_from(component_idx).map_err(|_| "component index exceeds u8")?; + component + .resolutions + .iter() + .enumerate() + .map(|(resolution_idx, resolution)| { + Ok(PreparedResolutionPacket { + component: component_idx, + resolution: u32::try_from(resolution_idx) + .map_err(|_| "resolution index exceeds u32")?, + precinct: 0, + subbands: resolution + .subbands + .iter() + .map(prepared_subband_from_preencoded) + .collect::, &'static str>>()?, + }) + }) + .collect() +} + +fn prepared_resolution_packets_from_preencoded_component_owned( + component_idx: usize, + component: PreencodedHtj2k97Component, +) -> Result, &'static str> { + let component_idx = u8::try_from(component_idx).map_err(|_| "component index exceeds u8")?; + component + .resolutions + .into_iter() + .enumerate() + .map(|(resolution_idx, resolution)| { + Ok(PreparedResolutionPacket { + component: component_idx, + resolution: u32::try_from(resolution_idx) + .map_err(|_| "resolution index exceeds u32")?, + precinct: 0, + subbands: resolution + .subbands + .into_iter() + .map(prepared_subband_from_preencoded_owned) + .collect::, &'static str>>()?, + }) + }) + .collect() +} + +fn prepared_resolution_packets_from_preencoded_compact_component<'a>( + component_idx: usize, + component: &'a PreencodedHtj2k97CompactComponent, + payload: &'a [u8], +) -> Result>, &'static str> { + let component_idx = u8::try_from(component_idx).map_err(|_| "component index exceeds u8")?; + component + .resolutions + .iter() + .enumerate() + .map(|(resolution_idx, resolution)| { + Ok(PreparedCompactResolutionPacket { + component: component_idx, + resolution: u32::try_from(resolution_idx) + .map_err(|_| "resolution index exceeds u32")?, + precinct: 0, + subbands: resolution + .subbands + .iter() + .map(|subband| prepared_subband_from_preencoded_compact(subband, payload)) + .collect::, &'static str>>()?, + }) + }) + .collect() +} + +fn prepared_subband_from_preencoded( + subband: &PreencodedHtj2k97Subband, +) -> Result { + Ok(PreparedEncodeSubband { + code_blocks: subband + .code_blocks + .iter() + .map(|block| PreparedEncodeCodeBlock { + coefficients: Vec::new(), + width: block.width, + height: block.height, + }) + .collect(), + preencoded_ht_code_blocks: Some( + subband + .code_blocks + .iter() + .map(|block| block.encoded.clone()) + .collect(), + ), + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + code_block_width: subband + .code_blocks + .iter() + .map(|block| block.width) + .max() + .unwrap_or(0), + code_block_height: subband + .code_blocks + .iter() + .map(|block| block.height) + .max() + .unwrap_or(0), + width: precomputed_subband_width( + subband.num_cbs_x, + subband.code_blocks.iter().map(|block| block.width), + ), + height: precomputed_subband_height( + subband.num_cbs_x, + subband.num_cbs_y, + subband.code_blocks.iter().map(|block| block.height), + ), + sub_band_type: internal_sub_band_type(subband.sub_band_type), + total_bitplanes: subband.total_bitplanes, + block_coding_mode: BlockCodingMode::HighThroughput, + }) +} + +fn prepared_subband_from_preencoded_owned( + subband: PreencodedHtj2k97Subband, +) -> Result { + let code_block_width = subband + .code_blocks + .iter() + .map(|block| block.width) + .max() + .unwrap_or(0); + let code_block_height = subband + .code_blocks + .iter() + .map(|block| block.height) + .max() + .unwrap_or(0); + let width = precomputed_subband_width( + subband.num_cbs_x, + subband.code_blocks.iter().map(|block| block.width), + ); + let height = precomputed_subband_height( + subband.num_cbs_x, + subband.num_cbs_y, + subband.code_blocks.iter().map(|block| block.height), + ); + let code_blocks = subband + .code_blocks + .into_iter() + .map(|block| { + let PreencodedHtj2k97CodeBlock { + width, + height, + encoded, + } = block; + ( + PreparedEncodeCodeBlock { + coefficients: Vec::new(), + width, + height, + }, + encoded, + ) + }) + .collect::>(); + let (code_blocks, preencoded_ht_code_blocks): (Vec<_>, Vec<_>) = + code_blocks.into_iter().unzip(); + + Ok(PreparedEncodeSubband { + code_blocks, + preencoded_ht_code_blocks: Some(preencoded_ht_code_blocks), + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + code_block_width, + code_block_height, + width, + height, + sub_band_type: internal_sub_band_type(subband.sub_band_type), + total_bitplanes: subband.total_bitplanes, + block_coding_mode: BlockCodingMode::HighThroughput, + }) +} + +fn prepared_subband_from_preencoded_compact<'a>( + subband: &'a PreencodedHtj2k97CompactSubband, + payload: &'a [u8], +) -> Result, &'static str> { + let code_blocks = subband + .code_blocks + .iter() + .map(|block| { + Ok(PreparedCompactCodeBlock { + data: compact_payload_slice(payload, &block.payload_range)?, + cleanup_length: block.cleanup_length, + refinement_length: block.refinement_length, + num_coding_passes: block.num_coding_passes, + num_zero_bitplanes: block.num_zero_bitplanes, + }) + }) + .collect::, &'static str>>()?; + + Ok(PreparedCompactSubband { + code_blocks, + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + }) +} + +fn compact_payload_slice<'a>( + payload: &'a [u8], + range: &Range, +) -> Result<&'a [u8], &'static str> { + if range.start > range.end || range.end > payload.len() { + return Err("HTJ2K payload range out of bounds"); + } + Ok(&payload[range.clone()]) +} + +fn zero_pixel_buffer( + width: u32, + height: u32, + num_components: u8, + bit_depth: u8, +) -> Result, &'static str> { + let bytes_per_sample = if bit_depth <= 8 { 1usize } else { 2usize }; + let len = width as usize; + let len = len + .checked_mul(height as usize) + .and_then(|value| value.checked_mul(usize::from(num_components))) + .and_then(|value| value.checked_mul(bytes_per_sample)) + .ok_or("pixel buffer dimensions overflow")?; + Ok(vec![0; len]) +} + +struct PrecomputedDwtAccelerator<'a, A: J2kEncodeStageAccelerator> { + outputs: Vec, + encode_accelerator: &'a mut A, +} + +struct PrecomputedDwt97Accelerator<'a, A: J2kEncodeStageAccelerator> { + outputs: Vec, + encode_accelerator: &'a mut A, +} + +impl J2kEncodeStageAccelerator for PrecomputedDwtAccelerator<'_, A> { + fn dispatch_report(&self) -> crate::J2kEncodeDispatchReport { + self.encode_accelerator.dispatch_report() + } + + fn encode_forward_dwt53( + &mut self, + _job: J2kForwardDwt53Job<'_>, + ) -> Result, &'static str> { + if self.outputs.is_empty() { + return Err("precomputed DWT output exhausted"); + } + + Ok(Some(self.outputs.remove(0))) + } + + fn encode_quantize_subband( + &mut self, + job: J2kQuantizeSubbandJob<'_>, + ) -> Result>, &'static str> { + self.encode_accelerator.encode_quantize_subband(job) + } + + fn encode_tier1_code_block( + &mut self, + job: J2kTier1CodeBlockEncodeJob<'_>, + ) -> Result, &'static str> { + self.encode_accelerator.encode_tier1_code_block(job) + } + + fn encode_tier1_code_blocks( + &mut self, + jobs: &[J2kTier1CodeBlockEncodeJob<'_>], + ) -> Result>, &'static str> { + self.encode_accelerator.encode_tier1_code_blocks(jobs) + } + + fn encode_ht_code_block( + &mut self, + job: crate::J2kHtCodeBlockEncodeJob<'_>, + ) -> Result, &'static str> { + self.encode_accelerator.encode_ht_code_block(job) + } + + fn encode_ht_code_blocks( + &mut self, + jobs: &[crate::J2kHtCodeBlockEncodeJob<'_>], + ) -> Result>, &'static str> { + self.encode_accelerator.encode_ht_code_blocks(jobs) + } + + fn prefer_parallel_cpu_code_block_fallback(&self) -> bool { + self.encode_accelerator + .prefer_parallel_cpu_code_block_fallback() + } + + fn prefer_parallel_cpu_tile_encode(&self) -> bool { + self.encode_accelerator.prefer_parallel_cpu_tile_encode() + } + + fn encode_packetization( + &mut self, + job: J2kPacketizationEncodeJob<'_>, + ) -> Result>, &'static str> { + self.encode_accelerator.encode_packetization(job) + } +} + +impl J2kEncodeStageAccelerator + for PrecomputedDwt97Accelerator<'_, A> +{ + fn dispatch_report(&self) -> crate::J2kEncodeDispatchReport { + self.encode_accelerator.dispatch_report() + } + + fn encode_forward_dwt97( + &mut self, + _job: J2kForwardDwt97Job<'_>, + ) -> Result, &'static str> { + if self.outputs.is_empty() { + return Err("precomputed DWT output exhausted"); + } + + Ok(Some(self.outputs.remove(0))) + } + + fn encode_quantize_subband( + &mut self, + job: J2kQuantizeSubbandJob<'_>, + ) -> Result>, &'static str> { + self.encode_accelerator.encode_quantize_subband(job) + } + + fn encode_tier1_code_block( + &mut self, + job: J2kTier1CodeBlockEncodeJob<'_>, + ) -> Result, &'static str> { + self.encode_accelerator.encode_tier1_code_block(job) + } + + fn encode_tier1_code_blocks( + &mut self, + jobs: &[J2kTier1CodeBlockEncodeJob<'_>], + ) -> Result>, &'static str> { + self.encode_accelerator.encode_tier1_code_blocks(jobs) + } + + fn encode_ht_code_block( + &mut self, + job: crate::J2kHtCodeBlockEncodeJob<'_>, + ) -> Result, &'static str> { + self.encode_accelerator.encode_ht_code_block(job) + } + + fn encode_ht_code_blocks( + &mut self, + jobs: &[crate::J2kHtCodeBlockEncodeJob<'_>], + ) -> Result>, &'static str> { + self.encode_accelerator.encode_ht_code_blocks(jobs) + } + + fn prefer_parallel_cpu_code_block_fallback(&self) -> bool { + self.encode_accelerator + .prefer_parallel_cpu_code_block_fallback() + } + + fn prefer_parallel_cpu_tile_encode(&self) -> bool { + self.encode_accelerator.prefer_parallel_cpu_tile_encode() + } + + fn encode_packetization( + &mut self, + job: J2kPacketizationEncodeJob<'_>, + ) -> Result>, &'static str> { + self.encode_accelerator.encode_packetization(job) + } +} + +fn block_coding_mode(options: &EncodeOptions) -> BlockCodingMode { + if options.use_ht_block_coding { + BlockCodingMode::HighThroughput + } else { + BlockCodingMode::Classic + } +} + +fn validate_irreversible_quantization_scale(scale: f32) -> Result<(), &'static str> { + if scale.is_finite() && scale > 0.0 { + Ok(()) + } else { + Err("irreversible quantization scale must be finite and greater than zero") + } +} + +fn validate_irreversible_quantization_profile(options: &EncodeOptions) -> Result<(), &'static str> { + validate_irreversible_quantization_scale(options.irreversible_quantization_scale)?; + if quantize::subband_scales_all_valid(options.irreversible_quantization_subband_scales) { + Ok(()) + } else { + Err("irreversible quantization subband scales must be finite and greater than zero") + } +} + +fn validate_htj2k_codestream( + codestream: &[u8], + pixels: &[u8], + width: u32, + height: u32, + num_components: u8, + bit_depth: u8, + signed: bool, + reversible: bool, +) -> Result<(), &'static str> { + let image = Image::new(codestream, &DecodeSettings::default()) + .map_err(|_| "generated HTJ2K codestream failed self-validation")?; + let decoded = image + .decode_native() + .map_err(|_| "generated HTJ2K codestream failed self-validation")?; + + if decoded.width != width + || decoded.height != height + || decoded.bit_depth != bit_depth + || decoded.num_components != num_components + { + return Err("generated HTJ2K codestream failed self-validation"); + } + + if reversible && !native_samples_equal(pixels, &decoded.data, bit_depth, signed) { + return Err("generated HTJ2K codestream did not roundtrip"); + } + + Ok(()) +} + +fn native_samples_equal(expected: &[u8], actual: &[u8], bit_depth: u8, signed: bool) -> bool { + if expected.len() != actual.len() { + return false; + } + + let bytes_per_sample = if bit_depth <= 8 { 1 } else { 2 }; + let sample_count = expected.len() / bytes_per_sample; + (0..sample_count).all(|sample_index| { + decode_native_sample(expected, sample_index, bit_depth, signed) + == decode_native_sample(actual, sample_index, bit_depth, signed) + }) +} + +fn decode_native_sample(bytes: &[u8], sample_index: usize, bit_depth: u8, signed: bool) -> i32 { + let byte_offset = sample_index * if bit_depth <= 8 { 1 } else { 2 }; + let mask = (1u32 << u32::from(bit_depth)) - 1; + let raw = if bit_depth <= 8 { + u32::from(bytes[byte_offset]) + } else { + u32::from(u16::from_le_bytes([ + bytes[byte_offset], + bytes[byte_offset + 1], + ])) + } & mask; + + if signed { + let shift = 32 - u32::from(bit_depth); + ((raw << shift) as i32) >> shift + } else { + raw as i32 + } +} + +fn encode_impl( + pixels: &[u8], + width: u32, + height: u32, + num_components: u8, + bit_depth: u8, + signed: bool, + options: &EncodeOptions, + block_coding_mode: BlockCodingMode, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + if width == 0 || height == 0 { + return Err("invalid dimensions"); + } + if num_components == 0 || num_components > 4 { + return Err("unsupported component count"); + } + if bit_depth == 0 || bit_depth > 16 { + return Err("unsupported bit depth"); + } + if options.num_layers == 0 || options.num_layers > 32 { + return Err("unsupported quality layer count"); + } + if !options.quality_layer_byte_targets.is_empty() + && options.quality_layer_byte_targets.len() != usize::from(options.num_layers) + { + return Err("quality layer byte target count must match quality layer count"); + } + if !options.reversible { + validate_irreversible_quantization_profile(options)?; + } + + let num_pixels = (width as usize) + .checked_mul(height as usize) + .ok_or("image dimensions overflow")?; + let bytes_per_sample = if bit_depth <= 8 { 1 } else { 2 }; + let expected_len = num_pixels + .checked_mul(num_components as usize) + .and_then(|len| len.checked_mul(bytes_per_sample)) + .ok_or("image dimensions overflow")?; + if pixels.len() < expected_len { + return Err("pixel data too short"); + } + let component_sampling = component_sampling_for_options(options, num_components)?; + if let Some((tile_width, tile_height)) = options.tile_size { + if tile_width == 0 || tile_height == 0 { + return Err("invalid tile dimensions"); + } + if component_sampling + .iter() + .any(|sampling| *sampling != (1, 1)) + { + return Err("multi-tile encode with component sampling is not implemented"); + } + if tile_width < width || tile_height < height { + return encode_multitile_impl( + pixels, + width, + height, + num_components, + bit_depth, + signed, + options, + block_coding_mode, + accelerator, + tile_width, + tile_height, + ); + } + } + + let profile_enabled = profile::profile_stages_enabled(); + let total_start = profile::profile_now(profile_enabled); + + let use_mct = options.use_mct && num_components >= 3; + let num_levels = options.num_decomposition_levels.min( + // Don't decompose more than the image supports + max_decomposition_levels(width, height), + ); + let guard_bits = if options.reversible { + if use_mct { + options.guard_bits.max(2) + } else { + options.guard_bits + } + } else { + options.guard_bits.max(2) + }; + let step_sizes = quantize::compute_step_sizes_with_irreversible_profile( + bit_depth, + num_levels, + options.reversible, + guard_bits, + options.irreversible_quantization_scale, + options.irreversible_quantization_subband_scales, + ); + let quant_params: Vec<(u16, u16)> = step_sizes + .iter() + .map(|s| (s.exponent, s.mantissa)) + .collect(); + let cb_width = 1u32 << (options.code_block_width_exp + 2); + let cb_height = 1u32 << (options.code_block_height_exp + 2); + let precinct_exponents = precinct_exponents_for_options(options, num_levels)?; + let params = EncodeParams { + width, + height, + tile_width: options + .tile_size + .map_or(width, |(tile_width, _)| tile_width), + tile_height: options + .tile_size + .map_or(height, |(_, tile_height)| tile_height), + num_components, + bit_depth, + signed, + num_decomposition_levels: num_levels, + reversible: options.reversible, + code_block_width_exp: options.code_block_width_exp, + code_block_height_exp: options.code_block_height_exp, + num_layers: options.num_layers, + use_mct, + guard_bits, + block_coding_mode, + progression_order: options.progression_order, + write_tlm: options.write_tlm, + write_plt: options.write_plt, + write_plm: options.write_plm, + write_sop: options.write_sop, + write_eph: options.write_eph, + terminate_coding_passes: block_coding_mode == BlockCodingMode::Classic + && options.num_layers > 1, + component_sampling, + precinct_exponents, + }; + + let stage_start = profile::profile_now(profile_enabled); + if block_coding_mode == BlockCodingMode::HighThroughput + && !(params.write_plt || params.write_plm || params.write_sop || params.write_eph) + { + if let Some(tile_data) = accelerator.encode_htj2k_tile(J2kHtj2kTileEncodeJob { + pixels, + width, + height, + num_components, + bit_depth, + signed, + num_decomposition_levels: num_levels, + reversible: options.reversible, + use_mct, + guard_bits, + code_block_width: cb_width, + code_block_height: cb_height, + progression_order: public_packetization_progression_order(options.progression_order), + component_sampling: ¶ms.component_sampling, + quantization_steps: &quant_params, + })? { + let tile_body_us = profile::elapsed_us(stage_start); + let stage_start = profile::profile_now(profile_enabled); + let codestream = codestream_write::write_codestream(¶ms, &tile_data, &quant_params); + let codestream_us = profile::elapsed_us(stage_start); + if profile_enabled { + profile::emit_profile_row( + "encode", + "accelerated", + &[ + ("tile_body_us", tile_body_us), + ("codestream_us", codestream_us), + ("total_us", profile::elapsed_us(total_start)), + ], + ); + } + return Ok(codestream); + } + } + + // Step 1: Convert pixel bytes to f32 component arrays + let stage_start = profile::profile_now(profile_enabled); + let mut components = match accelerator.encode_deinterleave(J2kDeinterleaveToF32Job { + pixels, + num_pixels, + num_components, + bit_depth, + signed, + })? { + Some(components) => { + validate_deinterleaved_components(components, num_components, num_pixels)? + } + None => deinterleave_to_f32(pixels, num_pixels, num_components, bit_depth, signed), + }; + let deinterleave_us = profile::elapsed_us(stage_start); + + // Step 2: Apply forward MCT if RGB with 3+ components + let stage_start = profile::profile_now(profile_enabled); + if use_mct { + if options.reversible { + if !try_encode_forward_rct(&mut components, accelerator)? { + forward_mct::forward_rct(&mut components); + } + } else if !try_encode_forward_ict(&mut components, accelerator)? { + forward_mct::forward_ict(&mut components); + } + } + let mct_us = profile::elapsed_us(stage_start); + + // Step 3: Apply forward DWT to each component + let stage_start = profile::profile_now(profile_enabled); + let decompositions: Vec = components + .iter() + .map(|comp| { + encode_forward_dwt( + comp, + width, + height, + num_levels, + options.reversible, + accelerator, + ) + }) + .collect::, &'static str>>()?; + let dwt_us = profile::elapsed_us(stage_start); + + // Step 5: Quantize and encode code-blocks for each component + let mut component_resolution_packets: Vec> = + Vec::with_capacity(num_components as usize); + + let stage_start = profile::profile_now(profile_enabled); + for (component_idx, decomp) in decompositions + .iter() + .take(num_components as usize) + .enumerate() + { + let component = u8::try_from(component_idx).map_err(|_| "component index exceeds u8")?; + let mut packets = Vec::with_capacity(num_levels as usize + 1); + + // LL subband (resolution 0) + let ll_subband = prepare_subband( + &decomp.ll, + decomp.ll_width, + decomp.ll_height, + &step_sizes[0], + bit_depth, + guard_bits, + options.reversible, + block_coding_mode, + cb_width, + cb_height, + SubBandType::LowLow, + accelerator, + )?; + packets.push(PreparedResolutionPacket { + component, + resolution: 0, + precinct: 0, + subbands: vec![ll_subband], + }); + + // Higher resolution levels + for (level_idx, level) in decomp.levels.iter().enumerate() { + let step_base = 1 + level_idx * 3; + + // HL subband + let hl_subband = prepare_subband( + &level.hl, + level.high_width, + level.low_height, + &step_sizes[step_base], + bit_depth, + guard_bits, + options.reversible, + block_coding_mode, + cb_width, + cb_height, + SubBandType::HighLow, + accelerator, + )?; + + // LH subband + let lh_subband = prepare_subband( + &level.lh, + level.low_width, + level.high_height, + &step_sizes[step_base + 1], + bit_depth, + guard_bits, + options.reversible, + block_coding_mode, + cb_width, + cb_height, + SubBandType::LowHigh, + accelerator, + )?; + + // HH subband + let hh_subband = prepare_subband( + &level.hh, + level.high_width, + level.high_height, + &step_sizes[step_base + 2], + bit_depth, + guard_bits, + options.reversible, + block_coding_mode, + cb_width, + cb_height, + SubBandType::HighHigh, + accelerator, + )?; + + packets.push(PreparedResolutionPacket { + component, + resolution: u32::try_from(level_idx + 1) + .map_err(|_| "resolution index exceeds u32")?, + precinct: 0, + subbands: vec![hl_subband, lh_subband, hh_subband], + }); + } + + component_resolution_packets.push(packets); + } + let subband_prepare_us = profile::elapsed_us(stage_start); + + let component_resolution_packets = split_component_resolution_packets_by_precinct( + component_resolution_packets, + width, + height, + num_levels, + ¶ms.precinct_exponents, + )?; + let prepared_resolution_packets = + ordered_prepared_resolution_packets(component_resolution_packets, options)?; + let stage_start = profile::profile_now(profile_enabled); + let (resolution_packets, packet_descriptors, allow_packetization_accelerator) = + if options.num_layers > 1 { + let (resolution_packets, packet_descriptors) = + encode_prepared_resolution_packets_layered( + prepared_resolution_packets, + options.num_layers, + options.progression_order, + &options.quality_layer_byte_targets, + accelerator, + )?; + (resolution_packets, packet_descriptors, false) + } else { + let packet_descriptors = packet_descriptors_for_order( + &prepared_resolution_packets, + 1, + options.progression_order, + )?; + let resolution_packets = + encode_prepared_resolution_packets(prepared_resolution_packets, accelerator)?; + (resolution_packets, packet_descriptors, true) + }; + let block_encode_us = profile::elapsed_us(stage_start); + + // Step 6: Form tile bitstream (T2) + let stage_start = profile::profile_now(profile_enabled); + let mut resolution_packets = resolution_packets; + let packetization_resolutions = public_packetization_resolutions(&resolution_packets); + let scalar_packet_descriptors = scalar_packet_descriptors(&packet_descriptors); + let packetization_job = J2kPacketizationEncodeJob { + resolution_count: resolution_packets.len() as u32, + num_layers: options.num_layers, + num_components, + code_block_count: count_code_blocks(&resolution_packets)?, + progression_order: public_packetization_progression_order(options.progression_order), + packet_descriptors: &packet_descriptors, + resolutions: &packetization_resolutions, + }; + let needs_scalar_packetization = + params.write_plt || params.write_plm || params.write_sop || params.write_eph; + let accelerated_tile_data = if allow_packetization_accelerator && !needs_scalar_packetization { + accelerator.encode_packetization(packetization_job)? + } else { + None + }; + let packetized_tile = if let Some(data) = accelerated_tile_data { + packet_encode::PacketizedTileData { + data, + packet_lengths: Vec::new(), + } + } else { + packet_encode::form_tile_bitstream_with_descriptors_lengths_and_markers( + &mut resolution_packets, + &scalar_packet_descriptors, + packet_encode::PacketMarkerOptions { + write_sop: params.write_sop, + write_eph: params.write_eph, + }, + )? + }; + let packetize_us = profile::elapsed_us(stage_start); + + // Step 7: Write codestream + let stage_start = profile::profile_now(profile_enabled); + let codestream = codestream_write::write_codestream_with_packet_lengths( + ¶ms, + &packetized_tile.data, + &quant_params, + &packetized_tile.packet_lengths, + ); + let codestream_us = profile::elapsed_us(stage_start); + + if profile_enabled { + profile::emit_profile_row( + "encode", + "cpu", + &[ + ("deinterleave_us", deinterleave_us), + ("mct_us", mct_us), + ("dwt_us", dwt_us), + ("subband_prepare_us", subband_prepare_us), + ("block_encode_us", block_encode_us), + ("packetize_us", packetize_us), + ("codestream_us", codestream_us), + ("total_us", profile::elapsed_us(total_start)), + ], + ); + } + + Ok(codestream) +} + +fn encode_multitile_impl( + pixels: &[u8], + width: u32, + height: u32, + num_components: u8, + bit_depth: u8, + signed: bool, + options: &EncodeOptions, + block_coding_mode: BlockCodingMode, + accelerator: &mut impl J2kEncodeStageAccelerator, + tile_width: u32, + tile_height: u32, +) -> Result, &'static str> { + let num_x_tiles = width.div_ceil(tile_width); + let num_y_tiles = height.div_ceil(tile_height); + let num_tiles = num_x_tiles + .checked_mul(num_y_tiles) + .ok_or("tile count overflow")?; + if num_tiles > u32::from(u16::MAX) + 1 { + return Err("multi-tile encode supports at most 65536 tiles"); + } + + let min_tile_width = if width.is_multiple_of(tile_width) { + tile_width + } else { + width % tile_width + }; + let min_tile_height = if height.is_multiple_of(tile_height) { + tile_height + } else { + height % tile_height + }; + let num_levels = options + .num_decomposition_levels + .min(max_decomposition_levels(min_tile_width, min_tile_height)); + let use_mct = options.use_mct && num_components >= 3; + let guard_bits = if options.reversible { + if use_mct { + options.guard_bits.max(2) + } else { + options.guard_bits + } + } else { + options.guard_bits.max(2) + }; + let step_sizes = quantize::compute_step_sizes_with_irreversible_profile( + bit_depth, + num_levels, + options.reversible, + guard_bits, + options.irreversible_quantization_scale, + options.irreversible_quantization_subband_scales, + ); + let quant_params: Vec<(u16, u16)> = step_sizes + .iter() + .map(|s| (s.exponent, s.mantissa)) + .collect(); + + let mut child_options = options.clone(); + child_options.num_decomposition_levels = num_levels; + child_options.tile_size = None; + child_options.write_tlm = false; + child_options.write_plt = options.write_plt || options.write_plm; + child_options.write_plm = false; + + struct EncodedTilePart { + data: Vec, + packet_lengths: Vec, + } + + let mut tile_bodies = Vec::with_capacity(num_tiles as usize); + for tile_y in 0..num_y_tiles { + for tile_x in 0..num_x_tiles { + let x0 = tile_x * tile_width; + let y0 = tile_y * tile_height; + let actual_width = (width - x0).min(tile_width); + let actual_height = (height - y0).min(tile_height); + let tile_pixels = extract_interleaved_tile( + pixels, + width, + x0, + y0, + actual_width, + actual_height, + num_components, + bit_depth, + )?; + let tile_codestream = encode_impl( + &tile_pixels, + actual_width, + actual_height, + num_components, + bit_depth, + signed, + &child_options, + block_coding_mode, + accelerator, + )?; + let packet_lengths = if options.write_plt || options.write_plm { + extract_single_tile_plt_packet_lengths(&tile_codestream)? + } else { + Vec::new() + }; + tile_bodies.push(EncodedTilePart { + data: extract_single_tile_body(&tile_codestream)?.to_vec(), + packet_lengths, + }); + } + } + + let component_sampling = component_sampling_for_options(options, num_components)?; + let precinct_exponents = precinct_exponents_for_options(options, num_levels)?; + let params = EncodeParams { + width, + height, + tile_width, + tile_height, + num_components, + bit_depth, + signed, + num_decomposition_levels: num_levels, + reversible: options.reversible, + code_block_width_exp: options.code_block_width_exp, + code_block_height_exp: options.code_block_height_exp, + num_layers: options.num_layers, + use_mct, + guard_bits, + block_coding_mode, + progression_order: options.progression_order, + write_tlm: options.write_tlm, + write_plt: options.write_plt, + write_plm: options.write_plm, + write_sop: options.write_sop, + write_eph: options.write_eph, + terminate_coding_passes: block_coding_mode == BlockCodingMode::Classic + && options.num_layers > 1, + component_sampling, + precinct_exponents, + }; + let tile_parts = tile_bodies + .iter() + .enumerate() + .map(|(tile_index, tile)| { + Ok(codestream_write::TilePartData { + tile_index: u16::try_from(tile_index).map_err(|_| "tile index exceeds u16")?, + data: &tile.data, + packet_lengths: &tile.packet_lengths, + }) + }) + .collect::, &'static str>>()?; + + Ok(codestream_write::write_codestream_tiles( + ¶ms, + &tile_parts, + &quant_params, + )) +} + +fn extract_interleaved_tile( + pixels: &[u8], + image_width: u32, + x0: u32, + y0: u32, + tile_width: u32, + tile_height: u32, + num_components: u8, + bit_depth: u8, +) -> Result, &'static str> { + let bytes_per_sample = if bit_depth <= 8 { 1usize } else { 2usize }; + let bytes_per_pixel = usize::from(num_components) + .checked_mul(bytes_per_sample) + .ok_or("pixel stride overflow")?; + let row_bytes = usize::try_from(tile_width) + .map_err(|_| "tile width exceeds usize")? + .checked_mul(bytes_per_pixel) + .ok_or("tile row byte count overflow")?; + let out_len = row_bytes + .checked_mul(usize::try_from(tile_height).map_err(|_| "tile height exceeds usize")?) + .ok_or("tile byte count overflow")?; + let mut tile = Vec::with_capacity(out_len); + let image_row_bytes = usize::try_from(image_width) + .map_err(|_| "image width exceeds usize")? + .checked_mul(bytes_per_pixel) + .ok_or("image row byte count overflow")?; + let x_byte_offset = usize::try_from(x0) + .map_err(|_| "tile x offset exceeds usize")? + .checked_mul(bytes_per_pixel) + .ok_or("tile x byte offset overflow")?; + + for y in y0..y0 + tile_height { + let row_start = usize::try_from(y) + .map_err(|_| "tile y offset exceeds usize")? + .checked_mul(image_row_bytes) + .and_then(|offset| offset.checked_add(x_byte_offset)) + .ok_or("tile row offset overflow")?; + let row_end = row_start + .checked_add(row_bytes) + .ok_or("tile row range overflow")?; + tile.extend_from_slice( + pixels + .get(row_start..row_end) + .ok_or("tile row range outside source pixels")?, + ); + } + + Ok(tile) +} + +fn extract_single_tile_body(codestream: &[u8]) -> Result<&[u8], &'static str> { + let sod = codestream + .windows(2) + .position(|marker| marker == [0xFF, super::codestream::markers::SOD]) + .ok_or("encoded tile codestream missing SOD")?; + let eoc = codestream + .windows(2) + .rposition(|marker| marker == [0xFF, super::codestream::markers::EOC]) + .ok_or("encoded tile codestream missing EOC")?; + if eoc < sod + 2 { + return Err("encoded tile codestream marker order invalid"); + } + Ok(&codestream[sod + 2..eoc]) +} + +fn extract_single_tile_plt_packet_lengths(codestream: &[u8]) -> Result, &'static str> { + let sod = codestream + .windows(2) + .position(|marker| marker == [0xFF, super::codestream::markers::SOD]) + .ok_or("encoded tile codestream missing SOD")?; + let mut packet_lengths = Vec::new(); + let mut offset = 0usize; + + while offset + 4 <= sod { + if codestream[offset] == 0xFF && codestream[offset + 1] == super::codestream::markers::PLT { + let marker_len = + u16::from_be_bytes([codestream[offset + 2], codestream[offset + 3]]) as usize; + if marker_len < 3 { + return Err("encoded tile codestream has invalid PLT length"); + } + let marker_end = offset + .checked_add(2) + .and_then(|value| value.checked_add(marker_len)) + .ok_or("encoded tile codestream PLT length overflow")?; + if marker_end > sod { + return Err("encoded tile codestream PLT extends past SOD"); + } + let length_bytes = codestream + .get(offset + 5..marker_end) + .ok_or("encoded tile codestream PLT payload out of range")?; + packet_lengths.extend( + super::codestream::decode_packet_lengths(length_bytes) + .ok_or("encoded tile codestream has invalid PLT packet lengths")?, + ); + offset = marker_end; + } else { + offset += 1; + } + } + + if packet_lengths.is_empty() { + return Err("encoded tile codestream missing PLT packet lengths"); + } + + Ok(packet_lengths) +} + +fn try_encode_forward_rct( + components: &mut [Vec], + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result { + debug_assert!(components.len() >= 3); + let (plane0, rest) = components.split_at_mut(1); + let (plane1, plane2) = rest.split_at_mut(1); + accelerator.encode_forward_rct(J2kForwardRctJob { + plane0: &mut plane0[0], + plane1: &mut plane1[0], + plane2: &mut plane2[0], + }) +} + +fn try_encode_forward_ict( + components: &mut [Vec], + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result { + debug_assert!(components.len() >= 3); + let (plane0, rest) = components.split_at_mut(1); + let (plane1, plane2) = rest.split_at_mut(1); + accelerator.encode_forward_ict(J2kForwardIctJob { + plane0: &mut plane0[0], + plane1: &mut plane1[0], + plane2: &mut plane2[0], + }) +} + +fn encode_forward_dwt( + component: &[f32], + width: u32, + height: u32, + num_levels: u8, + reversible: bool, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result { + if reversible { + if let Some(output) = accelerator.encode_forward_dwt53(J2kForwardDwt53Job { + samples: component, + width, + height, + num_levels, + })? { + return convert_forward_dwt53_output(output); + } + } else if let Some(output) = accelerator.encode_forward_dwt97(J2kForwardDwt97Job { + samples: component, + width, + height, + num_levels, + })? { + return convert_forward_dwt97_output(output); + } + + Ok(fdwt::forward_dwt( + component, width, height, num_levels, reversible, + )) +} + +fn convert_forward_dwt53_output( + output: J2kForwardDwt53Output, +) -> Result { + validate_band_len(output.ll.len(), output.ll_width, output.ll_height)?; + let mut levels = Vec::with_capacity(output.levels.len()); + for level in output.levels { + validate_dwt53_level(&level)?; + levels.push(fdwt::DwtLevel { + hl: level.hl, + lh: level.lh, + hh: level.hh, + low_width: level.low_width, + low_height: level.low_height, + high_width: level.high_width, + high_height: level.high_height, + }); + } + Ok(DwtDecomposition { + ll: output.ll, + ll_width: output.ll_width, + ll_height: output.ll_height, + levels, + }) +} + +fn convert_forward_dwt97_output( + output: J2kForwardDwt97Output, +) -> Result { + validate_band_len(output.ll.len(), output.ll_width, output.ll_height)?; + let mut levels = Vec::with_capacity(output.levels.len()); + for level in output.levels { + validate_dwt97_level(&level)?; + levels.push(fdwt::DwtLevel { + hl: level.hl, + lh: level.lh, + hh: level.hh, + low_width: level.low_width, + low_height: level.low_height, + high_width: level.high_width, + high_height: level.high_height, + }); + } + Ok(DwtDecomposition { + ll: output.ll, + ll_width: output.ll_width, + ll_height: output.ll_height, + levels, + }) +} + +fn validate_dwt53_level(level: &J2kForwardDwt53Level) -> Result<(), &'static str> { + validate_band_len(level.hl.len(), level.high_width, level.low_height)?; + validate_band_len(level.lh.len(), level.low_width, level.high_height)?; + validate_band_len(level.hh.len(), level.high_width, level.high_height)?; + Ok(()) +} + +fn validate_dwt97_level(level: &J2kForwardDwt97Level) -> Result<(), &'static str> { + validate_band_len(level.hl.len(), level.high_width, level.low_height)?; + validate_band_len(level.lh.len(), level.low_width, level.high_height)?; + validate_band_len(level.hh.len(), level.high_width, level.high_height)?; + Ok(()) +} + +fn validate_band_len(actual: usize, width: u32, height: u32) -> Result<(), &'static str> { + let expected = (width as usize) + .checked_mul(height as usize) + .ok_or("accelerated DWT output dimensions overflow")?; + if actual != expected { + return Err("accelerated DWT output length mismatch"); + } + Ok(()) +} + +fn validate_deinterleaved_components( + components: Vec>, + num_components: u8, + num_pixels: usize, +) -> Result>, &'static str> { + if components.len() != usize::from(num_components) { + return Err("accelerated deinterleave component count mismatch"); + } + if components + .iter() + .any(|component| component.len() != num_pixels) + { + return Err("accelerated deinterleave component length mismatch"); + } + Ok(components) +} + +fn component_sampling_for_options( + options: &EncodeOptions, + num_components: u8, +) -> Result, &'static str> { + match &options.component_sampling { + Some(component_sampling) => { + if component_sampling.len() != usize::from(num_components) { + return Err("component sampling count does not match component count"); + } + if component_sampling + .iter() + .any(|&(x_rsiz, y_rsiz)| x_rsiz == 0 || y_rsiz == 0) + { + return Err("component sampling factors must be non-zero"); + } + Ok(component_sampling.clone()) + } + None => Ok(vec![(1, 1); usize::from(num_components)]), + } +} + +fn precinct_exponents_for_options( + options: &EncodeOptions, + num_decomposition_levels: u8, +) -> Result, &'static str> { + if options.precinct_exponents.is_empty() { + return Ok(Vec::new()); + } + + let expected = usize::from(num_decomposition_levels) + 1; + if options.precinct_exponents.len() != expected { + return Err("precinct exponent count must match resolution level count"); + } + if options + .precinct_exponents + .iter() + .any(|&(ppx, ppy)| ppx > 15 || ppy > 15) + { + return Err("precinct exponents must fit in COD marker nybbles"); + } + let code_block_width_exp = options.code_block_width_exp + 2; + let code_block_height_exp = options.code_block_height_exp + 2; + for (resolution, &(ppx, ppy)) in options.precinct_exponents.iter().enumerate() { + let min_ppx = if resolution == 0 { + code_block_width_exp + } else { + code_block_width_exp + 1 + }; + let min_ppy = if resolution == 0 { + code_block_height_exp + } else { + code_block_height_exp + 1 + }; + if ppx < min_ppx || ppy < min_ppy { + return Err("precinct exponents must not reduce encoder code-block dimensions"); + } + } + Ok(options.precinct_exponents.clone()) +} + +fn count_code_blocks(resolution_packets: &[ResolutionPacket]) -> Result { + let count = resolution_packets + .iter() + .flat_map(|resolution| resolution.subbands.iter()) + .try_fold(0usize, |acc, subband| { + acc.checked_add(subband.code_blocks.len()) + .ok_or("packetization code-block count overflow") + })?; + u32::try_from(count).map_err(|_| "packetization code-block count exceeds u32") +} + +fn count_compact_code_blocks( + resolution_packets: &[PreparedCompactResolutionPacket<'_>], +) -> Result { + let count = resolution_packets + .iter() + .flat_map(|resolution| resolution.subbands.iter()) + .try_fold(0usize, |acc, subband| { + acc.checked_add(subband.code_blocks.len()) + .ok_or("packetization code-block count overflow") + })?; + u32::try_from(count).map_err(|_| "packetization code-block count exceeds u32") +} + +fn split_component_resolution_packets_by_precinct( + component_resolution_packets: Vec>, + width: u32, + height: u32, + num_decomposition_levels: u8, + precinct_exponents: &[(u8, u8)], +) -> Result>, &'static str> { + if precinct_exponents.is_empty() { + return Ok(component_resolution_packets); + } + + component_resolution_packets + .into_iter() + .map(|component_packets| { + let mut split_packets = Vec::new(); + for packet in component_packets { + split_packets.extend(split_prepared_resolution_packet_by_precinct( + packet, + width, + height, + num_decomposition_levels, + precinct_exponents, + )?); + } + Ok(split_packets) + }) + .collect() +} + +fn split_prepared_resolution_packet_by_precinct( + packet: PreparedResolutionPacket, + width: u32, + height: u32, + num_decomposition_levels: u8, + precinct_exponents: &[(u8, u8)], +) -> Result, &'static str> { + let resolution = + usize::try_from(packet.resolution).map_err(|_| "resolution index exceeds usize")?; + let &(ppx, ppy) = precinct_exponents + .get(resolution) + .ok_or("missing precinct exponents for resolution")?; + let (precincts_x, precincts_y) = resolution_precinct_grid( + width, + height, + num_decomposition_levels, + packet.resolution, + ppx, + ppy, + )?; + let packet_count = (precincts_x as usize) + .checked_mul(precincts_y as usize) + .ok_or("precinct packet count overflow")?; + let component = packet.component; + let resolution = packet.resolution; + let subbands = packet.subbands; + let mut packets = Vec::with_capacity(packet_count); + + for precinct_y in 0..precincts_y { + for precinct_x in 0..precincts_x { + let precinct = u64::from(precinct_y) + .checked_mul(u64::from(precincts_x)) + .and_then(|value| value.checked_add(u64::from(precinct_x))) + .ok_or("precinct index overflow")?; + let split_subbands = subbands + .iter() + .map(|subband| { + split_prepared_subband_by_precinct( + subband, resolution, ppx, ppy, precinct_x, precinct_y, + ) + }) + .collect::, &'static str>>()?; + packets.push(PreparedResolutionPacket { + component, + resolution, + precinct, + subbands: split_subbands, + }); + } + } + + Ok(packets) +} + +fn resolution_precinct_grid( + width: u32, + height: u32, + num_decomposition_levels: u8, + resolution: u32, + ppx: u8, + ppy: u8, +) -> Result<(u32, u32), &'static str> { + let resolution_shift = u32::from(num_decomposition_levels) + .checked_sub(resolution) + .ok_or("resolution exceeds decomposition level count")?; + let resolution_scale = pow2_u32(resolution_shift)?; + let resolution_width = width.div_ceil(resolution_scale); + let resolution_height = height.div_ceil(resolution_scale); + let precinct_width = pow2_u32(u32::from(ppx))?; + let precinct_height = pow2_u32(u32::from(ppy))?; + + Ok(( + if resolution_width == 0 { + 0 + } else { + resolution_width.div_ceil(precinct_width) + }, + if resolution_height == 0 { + 0 + } else { + resolution_height.div_ceil(precinct_height) + }, + )) +} + +fn split_prepared_subband_by_precinct( + subband: &PreparedEncodeSubband, + resolution: u32, + ppx: u8, + ppy: u8, + precinct_x: u32, + precinct_y: u32, +) -> Result { + if subband.code_blocks.is_empty() || subband.width == 0 || subband.height == 0 { + return Ok(empty_prepared_subband_precinct(subband)); + } + + let subband_ppx = if resolution > 0 { + ppx.checked_sub(1) + .ok_or("nonzero resolution precinct exponent underflow")? + } else { + ppx + }; + let subband_ppy = if resolution > 0 { + ppy.checked_sub(1) + .ok_or("nonzero resolution precinct exponent underflow")? + } else { + ppy + }; + let precinct_width = pow2_u32(u32::from(subband_ppx))?; + let precinct_height = pow2_u32(u32::from(subband_ppy))?; + let precinct_x0 = precinct_x + .checked_mul(precinct_width) + .ok_or("precinct x coordinate overflow")?; + let precinct_y0 = precinct_y + .checked_mul(precinct_height) + .ok_or("precinct y coordinate overflow")?; + let x0 = precinct_x0.min(subband.width); + let y0 = precinct_y0.min(subband.height); + let x1 = precinct_x0 + .checked_add(precinct_width) + .ok_or("precinct x extent overflow")? + .min(subband.width); + let y1 = precinct_y0 + .checked_add(precinct_height) + .ok_or("precinct y extent overflow")? + .min(subband.height); + + if x0 >= x1 || y0 >= y1 { + return Ok(empty_prepared_subband_precinct(subband)); + } + + let cb_width = subband.code_block_width; + let cb_height = subband.code_block_height; + if cb_width == 0 || cb_height == 0 { + return Ok(empty_prepared_subband_precinct(subband)); + } + + let cb_x0 = (x0 / cb_width) * cb_width; + let cb_y0 = (y0 / cb_height) * cb_height; + let cb_x1 = x1.div_ceil(cb_width) * cb_width; + let cb_y1 = y1.div_ceil(cb_height) * cb_height; + let cbx_start = cb_x0 / cb_width; + let cby_start = cb_y0 / cb_height; + let cbx_end = cb_x1 / cb_width; + let cby_end = cb_y1 / cb_height; + let num_cbs_x = cbx_end.saturating_sub(cbx_start); + let num_cbs_y = cby_end.saturating_sub(cby_start); + let mut indices = Vec::with_capacity((num_cbs_x as usize).saturating_mul(num_cbs_y as usize)); + + for cby in cby_start..cby_end { + for cbx in cbx_start..cbx_end { + let index = cby + .checked_mul(subband.num_cbs_x) + .and_then(|value| value.checked_add(cbx)) + .ok_or("precinct code-block index overflow")?; + indices.push(usize::try_from(index).map_err(|_| "code-block index exceeds usize")?); + } + } + + let code_blocks = indices + .iter() + .map(|&idx| { + subband + .code_blocks + .get(idx) + .cloned() + .ok_or("precinct code-block index out of range") + }) + .collect::, &'static str>>()?; + let preencoded_ht_code_blocks = subband + .preencoded_ht_code_blocks + .as_ref() + .map(|blocks| { + indices + .iter() + .map(|&idx| { + blocks + .get(idx) + .cloned() + .ok_or("precinct preencoded code-block index out of range") + }) + .collect::, &'static str>>() + }) + .transpose()?; + + Ok(PreparedEncodeSubband { + code_blocks, + preencoded_ht_code_blocks, + num_cbs_x, + num_cbs_y, + code_block_width: cb_width, + code_block_height: cb_height, + width: x1 - x0, + height: y1 - y0, + sub_band_type: subband.sub_band_type, + total_bitplanes: subband.total_bitplanes, + block_coding_mode: subband.block_coding_mode, + }) +} + +fn empty_prepared_subband_precinct(subband: &PreparedEncodeSubband) -> PreparedEncodeSubband { + PreparedEncodeSubband { + code_blocks: Vec::new(), + preencoded_ht_code_blocks: subband + .preencoded_ht_code_blocks + .as_ref() + .map(|_| Vec::new()), + num_cbs_x: 0, + num_cbs_y: 0, + code_block_width: subband.code_block_width, + code_block_height: subband.code_block_height, + width: 0, + height: 0, + sub_band_type: subband.sub_band_type, + total_bitplanes: subband.total_bitplanes, + block_coding_mode: subband.block_coding_mode, + } +} + +fn pow2_u32(exponent: u32) -> Result { + 1_u32 + .checked_shl(exponent) + .ok_or("precinct exponent exceeds u32 shift width") +} + +fn packet_descriptors_for_order( + packets: &[PreparedResolutionPacket], + num_layers: u8, + progression_order: EncodeProgressionOrder, +) -> Result, &'static str> { + if num_layers != 1 { + return Err("encode currently prepares one packet contribution layer"); + } + let mut descriptors = packets + .iter() + .enumerate() + .map(|(packet_index, packet)| { + Ok(J2kPacketizationPacketDescriptor { + packet_index: u32::try_from(packet_index) + .map_err(|_| "packet descriptor index exceeds u32")?, + state_index: u32::try_from(packet_index) + .map_err(|_| "packet descriptor state index exceeds u32")?, + layer: 0, + resolution: packet.resolution, + component: packet.component, + precinct: packet.precinct, + }) + }) + .collect::, &'static str>>()?; + sort_packet_descriptors_for_progression(&mut descriptors, progression_order); + Ok(descriptors) +} + +fn packet_descriptors_for_compact_order( + packets: &[PreparedCompactResolutionPacket<'_>], + num_layers: u8, + progression_order: EncodeProgressionOrder, +) -> Result, &'static str> { + if num_layers != 1 { + return Err("encode currently prepares one packet contribution layer"); + } + let mut descriptors = packets + .iter() + .enumerate() + .map(|(packet_index, packet)| { + Ok(J2kPacketizationPacketDescriptor { + packet_index: u32::try_from(packet_index) + .map_err(|_| "packet descriptor index exceeds u32")?, + state_index: u32::try_from(packet_index) + .map_err(|_| "packet descriptor state index exceeds u32")?, + layer: 0, + resolution: packet.resolution, + component: packet.component, + precinct: packet.precinct, + }) + }) + .collect::, &'static str>>()?; + sort_packet_descriptors_for_progression(&mut descriptors, progression_order); + Ok(descriptors) +} + +fn ordered_prepared_resolution_packets( + component_resolution_packets: Vec>, + options: &EncodeOptions, +) -> Result, &'static str> { + match options.progression_order { + EncodeProgressionOrder::Lrcp + | EncodeProgressionOrder::Rlcp + | EncodeProgressionOrder::Rpcl => { + lrcp_ordered_prepared_resolution_packets(component_resolution_packets) + } + EncodeProgressionOrder::Pcrl | EncodeProgressionOrder::Cprl => { + component_ordered_prepared_resolution_packets(component_resolution_packets) + } + } +} + +fn ordered_prepared_compact_resolution_packets<'a>( + component_resolution_packets: Vec>>, + options: &EncodeOptions, +) -> Result>, &'static str> { + match options.progression_order { + EncodeProgressionOrder::Lrcp + | EncodeProgressionOrder::Rlcp + | EncodeProgressionOrder::Rpcl => { + lrcp_ordered_prepared_compact_resolution_packets(component_resolution_packets) + } + EncodeProgressionOrder::Pcrl | EncodeProgressionOrder::Cprl => { + component_ordered_prepared_compact_resolution_packets(component_resolution_packets) + } + } +} + +fn lrcp_ordered_prepared_resolution_packets( + component_resolution_packets: Vec>, +) -> Result, &'static str> { + let resolution_count = component_resolution_packets + .first() + .map_or(0usize, alloc::vec::Vec::len); + let mut component_iters: Vec<_> = component_resolution_packets + .into_iter() + .map(alloc::vec::Vec::into_iter) + .collect(); + let mut resolution_packets = + Vec::with_capacity(resolution_count.saturating_mul(component_iters.len())); + + for _resolution in 0..resolution_count { + for component in &mut component_iters { + resolution_packets.push( + component + .next() + .ok_or("component packet resolution count mismatch")?, + ); + } + } + + if component_iters + .iter_mut() + .any(|component| component.next().is_some()) + { + return Err("component packet resolution count mismatch"); + } + + Ok(resolution_packets) +} + +fn lrcp_ordered_prepared_compact_resolution_packets<'a>( + component_resolution_packets: Vec>>, +) -> Result>, &'static str> { + let resolution_count = component_resolution_packets + .first() + .map_or(0usize, alloc::vec::Vec::len); + let mut component_iters: Vec<_> = component_resolution_packets + .into_iter() + .map(alloc::vec::Vec::into_iter) + .collect(); + let mut resolution_packets = + Vec::with_capacity(resolution_count.saturating_mul(component_iters.len())); + + for _resolution in 0..resolution_count { + for component in &mut component_iters { + resolution_packets.push( + component + .next() + .ok_or("component packet resolution count mismatch")?, + ); + } + } + + if component_iters + .iter_mut() + .any(|component| component.next().is_some()) + { + return Err("component packet resolution count mismatch"); + } + + Ok(resolution_packets) +} + +fn component_ordered_prepared_resolution_packets( + component_resolution_packets: Vec>, +) -> Result, &'static str> { + let resolution_count = component_resolution_packets + .first() + .map_or(0usize, alloc::vec::Vec::len); + let mut resolution_packets = + Vec::with_capacity(resolution_count.saturating_mul(component_resolution_packets.len())); + + for component in component_resolution_packets { + if component.len() != resolution_count { + return Err("component packet resolution count mismatch"); + } + resolution_packets.extend(component); + } + + Ok(resolution_packets) +} + +fn component_ordered_prepared_compact_resolution_packets<'a>( + component_resolution_packets: Vec>>, +) -> Result>, &'static str> { + let resolution_count = component_resolution_packets + .first() + .map_or(0usize, alloc::vec::Vec::len); + let mut resolution_packets = + Vec::with_capacity(resolution_count.saturating_mul(component_resolution_packets.len())); + + for component in component_resolution_packets { + if component.len() != resolution_count { + return Err("component packet resolution count mismatch"); + } + resolution_packets.extend(component); + } + + Ok(resolution_packets) +} + +fn public_packetization_progression_order( + progression_order: EncodeProgressionOrder, +) -> crate::J2kPacketizationProgressionOrder { + match progression_order { + EncodeProgressionOrder::Lrcp => crate::J2kPacketizationProgressionOrder::Lrcp, + EncodeProgressionOrder::Rlcp => crate::J2kPacketizationProgressionOrder::Rlcp, + EncodeProgressionOrder::Rpcl => crate::J2kPacketizationProgressionOrder::Rpcl, + EncodeProgressionOrder::Pcrl => crate::J2kPacketizationProgressionOrder::Pcrl, + EncodeProgressionOrder::Cprl => crate::J2kPacketizationProgressionOrder::Cprl, + } +} + +fn scalar_packet_descriptors( + descriptors: &[J2kPacketizationPacketDescriptor], +) -> Vec { + descriptors + .iter() + .map(|descriptor| packet_encode::PacketDescriptor { + packet_index: descriptor.packet_index, + state_index: descriptor.state_index, + layer: descriptor.layer, + resolution: descriptor.resolution, + component: descriptor.component, + precinct: descriptor.precinct, + }) + .collect() +} + +fn public_packetization_resolutions( + resolution_packets: &[ResolutionPacket], +) -> Vec> { + resolution_packets + .iter() + .map(|resolution| J2kPacketizationResolution { + subbands: resolution + .subbands + .iter() + .map(|subband| J2kPacketizationSubband { + code_blocks: subband + .code_blocks + .iter() + .map(|code_block| J2kPacketizationCodeBlock { + data: &code_block.data, + ht_cleanup_length: code_block.ht_cleanup_length, + ht_refinement_length: code_block.ht_refinement_length, + num_coding_passes: code_block.num_coding_passes, + num_zero_bitplanes: code_block.num_zero_bitplanes, + previously_included: code_block.previously_included, + l_block: code_block.l_block, + block_coding_mode: public_packetization_block_coding_mode( + code_block.block_coding_mode, + ), + }) + .collect(), + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + }) + .collect(), + }) + .collect() +} + +fn public_packetization_resolutions_from_compact<'a>( + resolution_packets: &'a [PreparedCompactResolutionPacket<'a>], +) -> Vec> { + resolution_packets + .iter() + .map(|resolution| J2kPacketizationResolution { + subbands: resolution + .subbands + .iter() + .map(|subband| J2kPacketizationSubband { + code_blocks: subband + .code_blocks + .iter() + .map(|code_block| J2kPacketizationCodeBlock { + data: code_block.data, + ht_cleanup_length: code_block.cleanup_length, + ht_refinement_length: code_block.refinement_length, + num_coding_passes: code_block.num_coding_passes, + num_zero_bitplanes: code_block.num_zero_bitplanes, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }) + .collect(), + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + }) + .collect(), + }) + .collect() +} + +fn public_packetization_block_coding_mode( + block_coding_mode: BlockCodingMode, +) -> J2kPacketizationBlockCodingMode { + match block_coding_mode { + BlockCodingMode::Classic => J2kPacketizationBlockCodingMode::Classic, + BlockCodingMode::HighThroughput => J2kPacketizationBlockCodingMode::HighThroughput, + } +} + +#[derive(Clone)] +struct PreparedEncodeCodeBlock { + coefficients: Vec, + width: u32, + height: u32, +} + +#[derive(Clone)] +struct PreparedEncodeSubband { + code_blocks: Vec, + preencoded_ht_code_blocks: Option>, + num_cbs_x: u32, + num_cbs_y: u32, + code_block_width: u32, + code_block_height: u32, + width: u32, + height: u32, + sub_band_type: SubBandType, + total_bitplanes: u8, + block_coding_mode: BlockCodingMode, +} + +struct PreparedResolutionPacket { + component: u8, + resolution: u32, + precinct: u64, + subbands: Vec, +} + +struct PreparedCompactCodeBlock<'a> { + data: &'a [u8], + cleanup_length: u32, + refinement_length: u32, + num_coding_passes: u8, + num_zero_bitplanes: u8, +} + +struct PreparedCompactSubband<'a> { + code_blocks: Vec>, + num_cbs_x: u32, + num_cbs_y: u32, +} + +struct PreparedCompactResolutionPacket<'a> { + component: u8, + resolution: u32, + precinct: u64, + subbands: Vec>, +} + +struct PreparedPrecomputedHtj2k97Image { + params: EncodeParams, + quant_params: Vec<(u16, u16)>, + packet_descriptors: Vec, + packet_count: usize, + prepared_packets: Vec, +} + +fn prepare_precomputed_htj2k97_image_for_batch( + image: &PrecomputedHtj2k97Image, + options: &EncodeOptions, +) -> Result { + if image.width == 0 || image.height == 0 { + return Err("invalid dimensions"); + } + if image.components.is_empty() || image.components.len() > 4 { + return Err("unsupported component count"); + } + if image.bit_depth == 0 || image.bit_depth > 16 { + return Err("unsupported bit depth"); + } + validate_irreversible_quantization_profile(options)?; + if image + .components + .iter() + .any(|component| component.x_rsiz == 0 || component.y_rsiz == 0) + { + return Err("component sampling factors must be non-zero"); + } + validate_precomputed_dwt97_geometry(image)?; + + let num_components = + u8::try_from(image.components.len()).map_err(|_| "unsupported component count")?; + let num_levels = precomputed_97_level_count(&image.components)?; + let guard_bits = options.guard_bits.max(2); + let step_sizes = quantize::compute_step_sizes_with_irreversible_profile( + image.bit_depth, + num_levels, + false, + guard_bits, + options.irreversible_quantization_scale, + options.irreversible_quantization_subband_scales, + ); + let quant_params: Vec<(u16, u16)> = step_sizes + .iter() + .map(|s| (s.exponent, s.mantissa)) + .collect(); + let cb_width = 1u32 << (options.code_block_width_exp + 2); + let cb_height = 1u32 << (options.code_block_height_exp + 2); + let component_sampling = image + .components + .iter() + .map(|component| (component.x_rsiz, component.y_rsiz)) + .collect::>(); + let mut precomputed_options = options.clone(); + precomputed_options.num_decomposition_levels = num_levels; + precomputed_options.reversible = false; + precomputed_options.use_ht_block_coding = true; + precomputed_options.use_mct = false; + precomputed_options.validate_high_throughput_codestream = false; + precomputed_options.component_sampling = Some(component_sampling.clone()); + let precinct_exponents = precinct_exponents_for_options(&precomputed_options, num_levels)?; + let params = EncodeParams { + width: image.width, + height: image.height, + tile_width: image.width, + tile_height: image.height, + num_components, + bit_depth: image.bit_depth, + signed: image.signed, + num_decomposition_levels: num_levels, + reversible: false, + code_block_width_exp: precomputed_options.code_block_width_exp, + code_block_height_exp: precomputed_options.code_block_height_exp, + num_layers: 1, + use_mct: false, + guard_bits, + block_coding_mode: BlockCodingMode::HighThroughput, + progression_order: precomputed_options.progression_order, + write_tlm: precomputed_options.write_tlm, + write_plt: precomputed_options.write_plt, + write_plm: precomputed_options.write_plm, + write_sop: precomputed_options.write_sop, + write_eph: precomputed_options.write_eph, + terminate_coding_passes: false, + component_sampling, + precinct_exponents, + }; + + let component_resolution_packets = image + .components + .iter() + .enumerate() + .map(|(component_idx, component)| { + prepared_resolution_packets_from_precomputed_97_component( + component_idx, + component, + &step_sizes, + image.bit_depth, + guard_bits, + cb_width, + cb_height, + ) + }) + .collect::, _>>()?; + let component_resolution_packets = split_component_resolution_packets_by_precinct( + component_resolution_packets, + image.width, + image.height, + num_levels, + ¶ms.precinct_exponents, + )?; + let prepared_packets = + ordered_prepared_resolution_packets(component_resolution_packets, &precomputed_options)?; + let packet_descriptors = + packet_descriptors_for_order(&prepared_packets, 1, precomputed_options.progression_order)?; + + Ok(PreparedPrecomputedHtj2k97Image { + params, + quant_params, + packet_descriptors, + packet_count: 0, + prepared_packets, + }) +} + +fn prepared_resolution_packets_from_precomputed_97_component( + component_idx: usize, + component: &PrecomputedHtj2k97Component, + step_sizes: &[QuantStepSize], + bit_depth: u8, + guard_bits: u8, + cb_width: u32, + cb_height: u32, +) -> Result, &'static str> { + let component_idx = u8::try_from(component_idx).map_err(|_| "component index exceeds u8")?; + let mut packets = Vec::with_capacity(component.dwt.levels.len() + 1); + packets.push(PreparedResolutionPacket { + component: component_idx, + resolution: 0, + precinct: 0, + subbands: vec![prepare_subband_cpu_quantized( + &component.dwt.ll, + component.dwt.ll_width, + component.dwt.ll_height, + step_sizes + .first() + .ok_or("irreversible quantization step missing")?, + bit_depth, + guard_bits, + false, + BlockCodingMode::HighThroughput, + cb_width, + cb_height, + SubBandType::LowLow, + )?], + }); + + for (level_idx, level) in component.dwt.levels.iter().enumerate() { + let step_base = 1 + level_idx * 3; + packets.push(PreparedResolutionPacket { + component: component_idx, + resolution: u32::try_from(level_idx + 1).map_err(|_| "resolution index exceeds u32")?, + precinct: 0, + subbands: vec![ + prepare_subband_cpu_quantized( + &level.hl, + level.high_width, + level.low_height, + step_sizes + .get(step_base) + .ok_or("irreversible quantization step missing")?, + bit_depth, + guard_bits, + false, + BlockCodingMode::HighThroughput, + cb_width, + cb_height, + SubBandType::HighLow, + )?, + prepare_subband_cpu_quantized( + &level.lh, + level.low_width, + level.high_height, + step_sizes + .get(step_base + 1) + .ok_or("irreversible quantization step missing")?, + bit_depth, + guard_bits, + false, + BlockCodingMode::HighThroughput, + cb_width, + cb_height, + SubBandType::LowHigh, + )?, + prepare_subband_cpu_quantized( + &level.hh, + level.high_width, + level.high_height, + step_sizes + .get(step_base + 2) + .ok_or("irreversible quantization step missing")?, + bit_depth, + guard_bits, + false, + BlockCodingMode::HighThroughput, + cb_width, + cb_height, + SubBandType::HighHigh, + )?, + ], + }); + } + + Ok(packets) +} + +fn copy_code_block_coefficients( + quantized: &[i32], + width: usize, + x0: usize, + y0: usize, + cbw: usize, + cbh: usize, +) -> Vec { + let len = cbw * cbh; + let start = y0 * width + x0; + if cbw == width { + return quantized[start..start + len].to_vec(); + } + + let mut coefficients = Vec::with_capacity(len); + for y in 0..cbh { + let row_start = (y0 + y) * width + x0; + coefficients.extend_from_slice(&quantized[row_start..row_start + cbw]); + } + coefficients +} + +fn prepare_subband( + coefficients: &[f32], + width: u32, + height: u32, + step_size: &QuantStepSize, + bit_depth: u8, + guard_bits: u8, + reversible: bool, + block_coding_mode: BlockCodingMode, + cb_width: u32, + cb_height: u32, + sub_band_type: SubBandType, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result { + if width == 0 || height == 0 { + return Ok(PreparedEncodeSubband { + code_blocks: Vec::new(), + preencoded_ht_code_blocks: None, + num_cbs_x: 0, + num_cbs_y: 0, + code_block_width: cb_width, + code_block_height: cb_height, + width, + height, + sub_band_type, + total_bitplanes: 0, + block_coding_mode, + }); + } + + let range_bits = subband_range_bits(bit_depth, sub_band_type); + debug_assert!(step_size.exponent <= u16::from(u8::MAX)); + let total_bitplanes = guard_bits + .saturating_add(step_size.exponent as u8) + .saturating_sub(1); + let num_cbs_x = width.div_ceil(cb_width); + let num_cbs_y = height.div_ceil(cb_height); + + if block_coding_mode == BlockCodingMode::HighThroughput { + if let Some(encoded) = accelerator.encode_ht_subband(J2kHtSubbandEncodeJob { + coefficients, + width, + height, + step_exponent: step_size.exponent, + step_mantissa: step_size.mantissa, + range_bits, + reversible, + code_block_width: cb_width, + code_block_height: cb_height, + total_bitplanes, + })? { + let expected_code_blocks = (num_cbs_x as usize) + .checked_mul(num_cbs_y as usize) + .ok_or("code-block count overflow")?; + if encoded.len() != expected_code_blocks { + return Err("accelerated HT subband code-block count mismatch"); + } + return Ok(PreparedEncodeSubband { + code_blocks: code_block_shapes(width, height, cb_width, cb_height)?, + preencoded_ht_code_blocks: Some(encoded), + num_cbs_x, + num_cbs_y, + code_block_width: cb_width, + code_block_height: cb_height, + width, + height, + sub_band_type, + total_bitplanes, + block_coding_mode, + }); + } + } + + let quantized = match accelerator.encode_quantize_subband(J2kQuantizeSubbandJob { + coefficients, + step_exponent: step_size.exponent, + step_mantissa: step_size.mantissa, + range_bits, + reversible, + })? { + Some(quantized) => { + if quantized.len() != coefficients.len() { + return Err("accelerated quantized subband length mismatch"); + } + quantized + } + None => quantize::quantize_subband(coefficients, step_size, range_bits, reversible), + }; + + // Split into code-blocks + let mut code_blocks = Vec::with_capacity((num_cbs_x * num_cbs_y) as usize); + + for cby in 0..num_cbs_y { + for cbx in 0..num_cbs_x { + let x0 = cbx * cb_width; + let y0 = cby * cb_height; + let x1 = (x0 + cb_width).min(width); + let y1 = (y0 + cb_height).min(height); + let cbw = x1 - x0; + let cbh = y1 - y0; + + let cb_coeffs = copy_code_block_coefficients( + &quantized, + width as usize, + x0 as usize, + y0 as usize, + cbw as usize, + cbh as usize, + ); + + code_blocks.push(PreparedEncodeCodeBlock { + coefficients: cb_coeffs, + width: cbw, + height: cbh, + }); + } + } + + Ok(PreparedEncodeSubband { + code_blocks, + preencoded_ht_code_blocks: None, + num_cbs_x, + num_cbs_y, + code_block_width: cb_width, + code_block_height: cb_height, + width, + height, + sub_band_type, + total_bitplanes, + block_coding_mode, + }) +} + +fn prepare_subband_cpu_quantized( + coefficients: &[f32], + width: u32, + height: u32, + step_size: &QuantStepSize, + bit_depth: u8, + guard_bits: u8, + reversible: bool, + block_coding_mode: BlockCodingMode, + cb_width: u32, + cb_height: u32, + sub_band_type: SubBandType, +) -> Result { + if width == 0 || height == 0 { + return Ok(PreparedEncodeSubband { + code_blocks: Vec::new(), + preencoded_ht_code_blocks: None, + num_cbs_x: 0, + num_cbs_y: 0, + code_block_width: cb_width, + code_block_height: cb_height, + width, + height, + sub_band_type, + total_bitplanes: 0, + block_coding_mode, + }); + } + + let range_bits = subband_range_bits(bit_depth, sub_band_type); + debug_assert!(step_size.exponent <= u16::from(u8::MAX)); + let total_bitplanes = guard_bits + .saturating_add(step_size.exponent as u8) + .saturating_sub(1); + let num_cbs_x = width.div_ceil(cb_width); + let num_cbs_y = height.div_ceil(cb_height); + let quantized = quantize::quantize_subband(coefficients, step_size, range_bits, reversible); + let mut code_blocks = Vec::with_capacity((num_cbs_x * num_cbs_y) as usize); + + for cby in 0..num_cbs_y { + for cbx in 0..num_cbs_x { + let x0 = cbx * cb_width; + let y0 = cby * cb_height; + let x1 = (x0 + cb_width).min(width); + let y1 = (y0 + cb_height).min(height); + let cbw = x1 - x0; + let cbh = y1 - y0; + let cb_coeffs = copy_code_block_coefficients( + &quantized, + width as usize, + x0 as usize, + y0 as usize, + cbw as usize, + cbh as usize, + ); + + code_blocks.push(PreparedEncodeCodeBlock { + coefficients: cb_coeffs, + width: cbw, + height: cbh, + }); + } + } + + Ok(PreparedEncodeSubband { + code_blocks, + preencoded_ht_code_blocks: None, + num_cbs_x, + num_cbs_y, + code_block_width: cb_width, + code_block_height: cb_height, + width, + height, + sub_band_type, + total_bitplanes, + block_coding_mode, + }) +} + +fn code_block_shapes( + width: u32, + height: u32, + cb_width: u32, + cb_height: u32, +) -> Result, &'static str> { + let num_cbs_x = width.div_ceil(cb_width); + let num_cbs_y = height.div_ceil(cb_height); + let count = (num_cbs_x as usize) + .checked_mul(num_cbs_y as usize) + .ok_or("code-block count overflow")?; + let mut code_blocks = Vec::with_capacity(count); + for cby in 0..num_cbs_y { + for cbx in 0..num_cbs_x { + let x0 = cbx * cb_width; + let y0 = cby * cb_height; + let x1 = (x0 + cb_width).min(width); + let y1 = (y0 + cb_height).min(height); + code_blocks.push(PreparedEncodeCodeBlock { + coefficients: Vec::new(), + width: x1 - x0, + height: y1 - y0, + }); + } + } + Ok(code_blocks) +} + +fn subband_range_bits(bit_depth: u8, sub_band_type: SubBandType) -> u8 { + let log_gain = match sub_band_type { + SubBandType::LowLow => 0, + SubBandType::LowHigh | SubBandType::HighLow => 1, + SubBandType::HighHigh => 2, + }; + + bit_depth.saturating_add(log_gain) +} + +fn encode_prepared_resolution_packets( + prepared_packets: Vec, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + let subband_counts: Vec<_> = prepared_packets + .iter() + .map(|packet| packet.subbands.len()) + .collect(); + let prepared_subbands: Vec<_> = prepared_packets + .into_iter() + .flat_map(|packet| packet.subbands) + .collect(); + let mut encoded_subbands = + encode_prepared_subbands(prepared_subbands, accelerator)?.into_iter(); + + subband_counts + .into_iter() + .map(|subband_count| { + let mut subbands = Vec::with_capacity(subband_count); + for _ in 0..subband_count { + subbands.push( + encoded_subbands + .next() + .ok_or("encoded subband count mismatch")?, + ); + } + Ok(ResolutionPacket { subbands }) + }) + .collect() +} + +fn encode_prepared_resolution_packets_layered( + prepared_packets: Vec, + num_layers: u8, + progression_order: EncodeProgressionOrder, + quality_layer_byte_targets: &[u64], + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result<(Vec, Vec), &'static str> { + let layer_count = usize::from(num_layers); + let mut layered_packets = Vec::with_capacity(prepared_packets.len()); + let mut classic_candidates = Vec::new(); + let mut classic_locations = Vec::new(); + let mut classic_block_index = 0usize; + let mut ht_candidates = Vec::new(); + let mut ht_locations = Vec::new(); + let mut ht_block_index = 0usize; + + for prepared_packet in prepared_packets { + let packet_idx = layered_packets.len(); + let mut layered_packet = LayeredPreparedPacket { + component: prepared_packet.component, + resolution: prepared_packet.resolution, + precinct: prepared_packet.precinct, + subbands: Vec::with_capacity(prepared_packet.subbands.len()), + }; + + for subband in prepared_packet.subbands { + let subband_idx = layered_packet.subbands.len(); + let mut layered_subband = LayeredPreparedSubband { + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + blocks: Vec::with_capacity(subband.code_blocks.len()), + }; + + match subband.block_coding_mode { + BlockCodingMode::Classic => { + for block in subband.code_blocks { + let block_idx = layered_subband.blocks.len(); + let encoded = bitplane_encode::encode_code_block_segments_with_style( + &block.coefficients, + block.width, + block.height, + subband.sub_band_type, + subband.total_bitplanes, + &classic_multilayer_code_block_style(), + ); + let segment_layers = if quality_layer_byte_targets.is_empty() { + classic_unbudgeted_segment_layers(&encoded, num_layers)? + } else { + for (segment_idx, segment) in encoded.segments.iter().enumerate() { + classic_candidates.push(ClassicSegmentAssignmentCandidate { + block_index: classic_block_index, + segment_index: segment_idx, + rate: u64::from(segment.data_length), + distortion_delta: segment.distortion_delta, + }); + classic_locations.push(ClassicSegmentLocation { + packet_idx, + subband_idx, + block_idx, + segment_idx, + }); + } + vec![layer_count.saturating_sub(1); encoded.segments.len()] + }; + layered_subband.blocks.push(LayeredPreparedBlock::Classic { + encoded, + segment_layers, + }); + classic_block_index = classic_block_index + .checked_add(1) + .ok_or("classic PCRD block index overflow")?; + } + } + BlockCodingMode::HighThroughput => { + let encoded_blocks = + encode_all_ht_code_blocks(core::slice::from_ref(&subband), accelerator)?; + let block_count = encoded_blocks.len(); + for (block_idx, encoded) in encoded_blocks.into_iter().enumerate() { + let target_layer = if quality_layer_byte_targets.is_empty() { + ht_target_layer(block_idx, block_count, layer_count)? + } else { + ht_candidates.push(HtSegmentAssignmentCandidate { + block_index: ht_block_index, + rate: u64::try_from(encoded.data.len()) + .map_err(|_| "HTJ2K packet contribution length exceeds u64")?, + }); + ht_locations.push(HtSegmentLocation { + packet_idx, + subband_idx, + block_idx: layered_subband.blocks.len(), + }); + layer_count.saturating_sub(1) + }; + layered_subband + .blocks + .push(LayeredPreparedBlock::HighThroughput { + encoded, + target_layer, + }); + ht_block_index = ht_block_index + .checked_add(1) + .ok_or("HTJ2K segment block index overflow")?; + } + } + } + + layered_packet.subbands.push(layered_subband); + } + + layered_packets.push(layered_packet); + } + + if !quality_layer_byte_targets.is_empty() { + let assignments = assign_classic_segment_layers_by_slope( + &classic_candidates, + layer_count, + quality_layer_byte_targets, + )?; + for (assignment_idx, layer) in assignments.into_iter().enumerate() { + let location = classic_locations + .get(assignment_idx) + .ok_or("classic PCRD assignment location mismatch")?; + let block = layered_packets + .get_mut(location.packet_idx) + .ok_or("classic PCRD packet index mismatch")? + .subbands + .get_mut(location.subband_idx) + .ok_or("classic PCRD subband index mismatch")? + .blocks + .get_mut(location.block_idx) + .ok_or("classic PCRD block index mismatch")?; + let LayeredPreparedBlock::Classic { segment_layers, .. } = block else { + return Err("classic PCRD assignment referenced HT block"); + }; + let segment_layer = segment_layers + .get_mut(location.segment_idx) + .ok_or("classic PCRD segment index mismatch")?; + *segment_layer = layer; + } + enforce_classic_segment_layer_monotonicity(&mut layered_packets); + } + if !quality_layer_byte_targets.is_empty() { + let assignments = assign_ht_segment_layers_by_budget( + &ht_candidates, + layer_count, + quality_layer_byte_targets, + )?; + for (assignment_idx, layer) in assignments.into_iter().enumerate() { + let location = ht_locations + .get(assignment_idx) + .ok_or("HTJ2K segment assignment location mismatch")?; + let block = layered_packets + .get_mut(location.packet_idx) + .ok_or("HTJ2K packet index mismatch")? + .subbands + .get_mut(location.subband_idx) + .ok_or("HTJ2K subband index mismatch")? + .blocks + .get_mut(location.block_idx) + .ok_or("HTJ2K block index mismatch")?; + let LayeredPreparedBlock::HighThroughput { target_layer, .. } = block else { + return Err("HTJ2K segment assignment referenced classic block"); + }; + *target_layer = layer; + } + } + + let mut resolution_packets = Vec::with_capacity(layered_packets.len() * layer_count); + let mut descriptors = Vec::with_capacity(layered_packets.len() * layer_count); + for (state_index, layered_packet) in layered_packets.into_iter().enumerate() { + let mut layer_packets: Vec<_> = (0..layer_count) + .map(|_| ResolutionPacket { + subbands: Vec::with_capacity(layered_packet.subbands.len()), + }) + .collect(); + + for subband in layered_packet.subbands { + let mut layer_subbands: Vec<_> = (0..layer_count) + .map(|_| SubbandPrecinct { + code_blocks: Vec::with_capacity(subband.blocks.len()), + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + }) + .collect(); + + for block in subband.blocks { + let contributions = match block { + LayeredPreparedBlock::Classic { + encoded, + segment_layers, + } => classic_layer_contributions(encoded, num_layers, &segment_layers)?, + LayeredPreparedBlock::HighThroughput { + encoded, + target_layer, + } => ht_layer_contributions(encoded, num_layers, target_layer)?, + }; + for (layer_idx, contribution) in contributions.into_iter().enumerate() { + layer_subbands[layer_idx].code_blocks.push(contribution); + } + } + + for (layer_packet, layer_subband) in layer_packets.iter_mut().zip(layer_subbands) { + layer_packet.subbands.push(layer_subband); + } + } + + let state_index = + u32::try_from(state_index).map_err(|_| "packet descriptor state index exceeds u32")?; + for (layer_idx, layer_packet) in layer_packets.into_iter().enumerate() { + let packet_index = u32::try_from(resolution_packets.len()) + .map_err(|_| "packet descriptor index exceeds u32")?; + resolution_packets.push(layer_packet); + descriptors.push(J2kPacketizationPacketDescriptor { + packet_index, + state_index, + layer: u8::try_from(layer_idx).map_err(|_| "quality layer index exceeds u8")?, + resolution: layered_packet.resolution, + component: layered_packet.component, + precinct: layered_packet.precinct, + }); + } + } + + sort_packet_descriptors_for_progression(&mut descriptors, progression_order); + + Ok((resolution_packets, descriptors)) +} + +fn sort_packet_descriptors_for_progression( + descriptors: &mut [J2kPacketizationPacketDescriptor], + progression_order: EncodeProgressionOrder, +) { + match progression_order { + EncodeProgressionOrder::Lrcp => descriptors.sort_by_key(|descriptor| { + ( + descriptor.layer, + descriptor.resolution, + descriptor.component, + descriptor.precinct, + ) + }), + EncodeProgressionOrder::Rlcp => descriptors.sort_by_key(|descriptor| { + ( + descriptor.resolution, + descriptor.layer, + descriptor.component, + descriptor.precinct, + ) + }), + EncodeProgressionOrder::Rpcl => descriptors.sort_by_key(|descriptor| { + ( + descriptor.resolution, + descriptor.precinct, + descriptor.component, + descriptor.layer, + ) + }), + EncodeProgressionOrder::Pcrl => descriptors.sort_by_key(|descriptor| { + ( + descriptor.precinct, + descriptor.component, + descriptor.resolution, + descriptor.layer, + ) + }), + EncodeProgressionOrder::Cprl => descriptors.sort_by_key(|descriptor| { + ( + descriptor.component, + descriptor.precinct, + descriptor.resolution, + descriptor.layer, + ) + }), + } +} + +fn classic_multilayer_code_block_style() -> CodeBlockStyle { + CodeBlockStyle { + termination_on_each_pass: true, + ..CodeBlockStyle::default() + } +} + +struct LayeredPreparedPacket { + component: u8, + resolution: u32, + precinct: u64, + subbands: Vec, +} + +struct LayeredPreparedSubband { + num_cbs_x: u32, + num_cbs_y: u32, + blocks: Vec, +} + +enum LayeredPreparedBlock { + Classic { + encoded: bitplane_encode::EncodedCodeBlockWithSegments, + segment_layers: Vec, + }, + HighThroughput { + encoded: bitplane_encode::EncodedCodeBlock, + target_layer: usize, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +struct ClassicSegmentAssignmentCandidate { + block_index: usize, + segment_index: usize, + rate: u64, + distortion_delta: f64, +} + +#[derive(Debug, Clone, Copy)] +struct ClassicSegmentLocation { + packet_idx: usize, + subband_idx: usize, + block_idx: usize, + segment_idx: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct HtSegmentAssignmentCandidate { + block_index: usize, + rate: u64, +} + +#[derive(Debug, Clone, Copy)] +struct HtSegmentLocation { + packet_idx: usize, + subband_idx: usize, + block_idx: usize, +} + +struct ClassicLayerBudgetAllocator { + cumulative_targets: Vec, + cumulative_used: Vec, +} + +impl ClassicLayerBudgetAllocator { + fn new(cumulative_targets: &[u64], layer_count: usize) -> Result { + if cumulative_targets.is_empty() { + return Ok(Self { + cumulative_targets: Vec::new(), + cumulative_used: Vec::new(), + }); + } + if cumulative_targets.len() != layer_count { + return Err("quality layer byte target count must match quality layer count"); + } + if cumulative_targets.windows(2).any(|pair| pair[0] > pair[1]) { + return Err("quality layer byte targets must be cumulative and monotonic"); + } + Ok(Self { + cumulative_targets: cumulative_targets + .iter() + .map(|&target| target.saturating_add(classic_rate_target_tolerance(target))) + .collect(), + cumulative_used: vec![0; layer_count], + }) + } + + fn is_budgeted(&self) -> bool { + !self.cumulative_targets.is_empty() + } + + fn assign_segment( + &mut self, + min_layer: usize, + data_length: u64, + ) -> Result { + if !self.is_budgeted() { + return Ok(min_layer); + } + + let rate = data_length; + let last_layer = self + .cumulative_targets + .len() + .checked_sub(1) + .ok_or("quality layer target count underflow")?; + for layer_idx in min_layer..last_layer { + if self.layer_can_accept(layer_idx, rate)? { + self.record_segment(layer_idx, rate)?; + return Ok(layer_idx); + } + } + self.record_segment(last_layer, rate)?; + Ok(last_layer) + } + + fn layer_can_accept(&self, layer_idx: usize, rate: u64) -> Result { + for cumulative_idx in layer_idx..self.cumulative_targets.len() { + let used = self.cumulative_used[cumulative_idx] + .checked_add(rate) + .ok_or("quality layer byte budget overflow")?; + if used > self.cumulative_targets[cumulative_idx] { + return Ok(false); + } + } + Ok(true) + } + + fn record_segment(&mut self, layer_idx: usize, rate: u64) -> Result<(), &'static str> { + for used in &mut self.cumulative_used[layer_idx..] { + *used = used + .checked_add(rate) + .ok_or("quality layer byte budget overflow")?; + } + Ok(()) + } +} + +fn classic_rate_target_tolerance(target: u64) -> u64 { + (target / 100).max(512) +} + +fn assign_classic_segment_layers_by_slope( + candidates: &[ClassicSegmentAssignmentCandidate], + layer_count: usize, + cumulative_targets: &[u64], +) -> Result, &'static str> { + let mut allocator = ClassicLayerBudgetAllocator::new(cumulative_targets, layer_count)?; + if candidates.is_empty() { + return Ok(Vec::new()); + } + + let block_count = candidates + .iter() + .map(|candidate| candidate.block_index) + .max() + .and_then(|max| max.checked_add(1)) + .ok_or("classic PCRD block count overflow")?; + let mut block_candidates = vec![Vec::new(); block_count]; + for (candidate_idx, candidate) in candidates.iter().enumerate() { + block_candidates + .get_mut(candidate.block_index) + .ok_or("classic PCRD block index mismatch")? + .push(candidate_idx); + } + for block in &mut block_candidates { + block.sort_by_key(|&idx| candidates[idx].segment_index); + } + + let mut block_min_layers = vec![0usize; block_count]; + let mut assignments = vec![layer_count.saturating_sub(1); candidates.len()]; + let mut next_block_segment = vec![0usize; block_count]; + let mut remaining = candidates.len(); + while remaining > 0 { + let candidate_idx = block_candidates + .iter() + .enumerate() + .filter_map(|(block_idx, block)| block.get(next_block_segment[block_idx]).copied()) + .min_by(|&left, &right| compare_classic_segment_candidates(candidates, left, right)) + .ok_or("classic PCRD candidate queue underflow")?; + let candidate = candidates[candidate_idx]; + let min_layer = *block_min_layers + .get(candidate.block_index) + .ok_or("classic PCRD block index mismatch")?; + let layer = allocator.assign_segment(min_layer, candidate.rate)?; + assignments[candidate_idx] = layer; + if let Some(block_layer) = block_min_layers.get_mut(candidate.block_index) { + *block_layer = layer; + } + if let Some(next) = next_block_segment.get_mut(candidate.block_index) { + *next = next + .checked_add(1) + .ok_or("classic PCRD segment index overflow")?; + } + remaining -= 1; + } + + enforce_classic_assignment_monotonicity(candidates, &mut assignments); + Ok(assignments) +} + +fn compare_classic_segment_candidates( + candidates: &[ClassicSegmentAssignmentCandidate], + left: usize, + right: usize, +) -> Ordering { + let left_candidate = candidates[left]; + let right_candidate = candidates[right]; + pcrd_slope(right_candidate) + .partial_cmp(&pcrd_slope(left_candidate)) + .unwrap_or(Ordering::Equal) + .then_with(|| left_candidate.block_index.cmp(&right_candidate.block_index)) + .then_with(|| { + left_candidate + .segment_index + .cmp(&right_candidate.segment_index) + }) +} + +fn pcrd_slope(candidate: ClassicSegmentAssignmentCandidate) -> f64 { + if candidate.rate == 0 { + return f64::INFINITY; + } + candidate.distortion_delta / candidate.rate as f64 +} + +fn enforce_classic_assignment_monotonicity( + candidates: &[ClassicSegmentAssignmentCandidate], + assignments: &mut [usize], +) { + let mut order: Vec<_> = (0..candidates.len()).collect(); + order.sort_by_key(|&idx| (candidates[idx].block_index, candidates[idx].segment_index)); + let mut current_block = None; + let mut min_layer = 0usize; + for idx in order { + if current_block != Some(candidates[idx].block_index) { + current_block = Some(candidates[idx].block_index); + min_layer = 0; + } + if assignments[idx] < min_layer { + assignments[idx] = min_layer; + } + min_layer = assignments[idx]; + } +} + +fn enforce_classic_segment_layer_monotonicity(layered_packets: &mut [LayeredPreparedPacket]) { + for packet in layered_packets { + for subband in &mut packet.subbands { + for block in &mut subband.blocks { + if let LayeredPreparedBlock::Classic { segment_layers, .. } = block { + let mut min_layer = 0usize; + for layer in segment_layers { + if *layer < min_layer { + *layer = min_layer; + } + min_layer = *layer; + } + } + } + } + } +} + +fn assign_ht_segment_layers_by_budget( + candidates: &[HtSegmentAssignmentCandidate], + layer_count: usize, + cumulative_targets: &[u64], +) -> Result, &'static str> { + let mut allocator = ClassicLayerBudgetAllocator::new(cumulative_targets, layer_count)?; + let mut assignments = vec![layer_count.saturating_sub(1); candidates.len()]; + let mut candidate_order: Vec<_> = (0..candidates.len()).collect(); + candidate_order.sort_by_key(|&idx| candidates[idx].block_index); + + for candidate_idx in candidate_order { + let candidate = candidates + .get(candidate_idx) + .ok_or("HTJ2K segment candidate index mismatch")?; + assignments[candidate_idx] = allocator.assign_segment(0, candidate.rate)?; + } + + Ok(assignments) +} + +fn classic_unbudgeted_segment_layers( + encoded: &bitplane_encode::EncodedCodeBlockWithSegments, + num_layers: u8, +) -> Result, &'static str> { + let mut segment_layers = Vec::with_capacity(encoded.segments.len()); + for segment in &encoded.segments { + let mut assigned = None; + for layer_idx in 0..usize::from(num_layers) { + let previous_pass = + previous_layer_pass_count(encoded.num_coding_passes, layer_idx, num_layers)?; + let cumulative_passes = if layer_idx + 1 == usize::from(num_layers) { + encoded.num_coding_passes + } else { + layer_pass_count(encoded.num_coding_passes, layer_idx + 1, num_layers)? + }; + if segment.start_coding_pass >= previous_pass + && segment.end_coding_pass <= cumulative_passes + { + assigned = Some(layer_idx); + break; + } + } + segment_layers.push( + assigned.ok_or("classic quality layer split must align to terminated coding passes")?, + ); + } + Ok(segment_layers) +} + +fn classic_layer_contributions( + encoded: bitplane_encode::EncodedCodeBlockWithSegments, + num_layers: u8, + segment_layers: &[usize], +) -> Result, &'static str> { + let layer_count = usize::from(num_layers); + if segment_layers.len() != encoded.segments.len() { + return Err("classic PCRD segment assignment count mismatch"); + } + if segment_layers.iter().any(|&layer| layer >= layer_count) { + return Err("classic PCRD segment layer exceeds layer count"); + } + let mut contributions = Vec::with_capacity(layer_count); + + for layer_idx in 0..layer_count { + let mut data = Vec::new(); + let mut classic_segment_lengths = Vec::new(); + let mut contribution_passes = 0u8; + + for (segment_idx, segment) in encoded.segments.iter().enumerate() { + if segment_layers[segment_idx] != layer_idx { + continue; + } + let start = usize::try_from(segment.data_offset) + .map_err(|_| "classic code-block segment offset overflow")?; + let len = usize::try_from(segment.data_length) + .map_err(|_| "classic code-block segment length overflow")?; + let end = start + .checked_add(len) + .ok_or("classic code-block segment range overflow")?; + data.extend_from_slice( + encoded + .data + .get(start..end) + .ok_or("classic code-block segment range invalid")?, + ); + classic_segment_lengths.push(segment.data_length); + contribution_passes = contribution_passes + .checked_add(segment.end_coding_pass - segment.start_coding_pass) + .ok_or("classic code-block contribution pass count overflow")?; + } + + contributions.push(CodeBlockPacketData { + data, + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: contribution_passes, + classic_segment_lengths, + num_zero_bitplanes: encoded.num_zero_bitplanes, + previously_included: false, + l_block: 3, + block_coding_mode: BlockCodingMode::Classic, + }); + } + + Ok(contributions) +} + +fn layer_pass_count( + num_coding_passes: u8, + layer_count: usize, + num_layers: u8, +) -> Result { + let numerator = u32::from(num_coding_passes) + .checked_mul(u32::try_from(layer_count).map_err(|_| "layer index overflow")?) + .ok_or("quality layer pass allocation overflow")?; + numerator + .div_ceil(u32::from(num_layers)) + .try_into() + .map_err(|_| "quality layer pass allocation overflow") +} + +fn previous_layer_pass_count( + num_coding_passes: u8, + layer_idx: usize, + num_layers: u8, +) -> Result { + if layer_idx == 0 { + Ok(0) + } else { + layer_pass_count(num_coding_passes, layer_idx, num_layers) + } +} + +fn ht_target_layer( + block_idx: usize, + block_count: usize, + layer_count: usize, +) -> Result { + if block_count == 0 || layer_count == 0 { + return Err("HTJ2K layer allocation requires non-empty inputs"); + } + Ok(block_idx + .checked_mul(layer_count) + .ok_or("HTJ2K layer allocation overflow")? + / block_count) +} + +fn ht_layer_contributions( + encoded: bitplane_encode::EncodedCodeBlock, + num_layers: u8, + target_layer: usize, +) -> Result, &'static str> { + let layer_count = usize::from(num_layers); + if target_layer >= layer_count { + return Err("HTJ2K target layer out of range"); + } + + let mut data = Some(encoded.data); + let mut contributions = Vec::with_capacity(layer_count); + for layer_idx in 0..layer_count { + let include = layer_idx == target_layer && encoded.num_coding_passes > 0; + let layer_data = if include { + data.take() + .ok_or("HTJ2K layer contribution data was already consumed")? + } else { + Vec::new() + }; + contributions.push(CodeBlockPacketData { + data: layer_data, + ht_cleanup_length: if include { + encoded.ht_cleanup_length + } else { + 0 + }, + ht_refinement_length: if include { + encoded.ht_refinement_length + } else { + 0 + }, + num_coding_passes: if include { + encoded.num_coding_passes + } else { + 0 + }, + classic_segment_lengths: Vec::new(), + num_zero_bitplanes: encoded.num_zero_bitplanes, + previously_included: false, + l_block: 3, + block_coding_mode: BlockCodingMode::HighThroughput, + }); + } + + Ok(contributions) +} + +fn encode_prepared_subbands( + prepared_subbands: Vec, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + let block_coding_mode = prepared_subbands + .iter() + .find(|subband| !subband.code_blocks.is_empty()) + .map(|subband| subband.block_coding_mode); + let encoded_blocks = match block_coding_mode { + Some(BlockCodingMode::HighThroughput) => { + encode_all_ht_code_blocks(&prepared_subbands, accelerator)? + } + Some(BlockCodingMode::Classic) => { + encode_all_tier1_code_blocks(&prepared_subbands, accelerator)? + } + None => Vec::new(), + }; + + let mut encoded_iter = encoded_blocks.into_iter(); + let mut precincts = Vec::with_capacity(prepared_subbands.len()); + for subband in prepared_subbands { + let mut code_blocks = Vec::with_capacity(subband.code_blocks.len()); + for _ in 0..subband.code_blocks.len() { + let encoded = encoded_iter + .next() + .ok_or("encoded code-block count mismatch")?; + code_blocks.push(CodeBlockPacketData { + data: encoded.data, + ht_cleanup_length: if subband.block_coding_mode == BlockCodingMode::HighThroughput { + encoded.ht_cleanup_length + } else { + 0 + }, + ht_refinement_length: if subband.block_coding_mode + == BlockCodingMode::HighThroughput + { + encoded.ht_refinement_length + } else { + 0 + }, + num_coding_passes: encoded.num_coding_passes, + classic_segment_lengths: Vec::new(), + num_zero_bitplanes: encoded.num_zero_bitplanes, + previously_included: false, + l_block: 3, + block_coding_mode: subband.block_coding_mode, + }); + } + precincts.push(SubbandPrecinct { + code_blocks, + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + }); + } + if encoded_iter.next().is_some() { + return Err("encoded code-block count mismatch"); + } + + Ok(precincts) +} + +fn encode_all_ht_code_blocks( + prepared_subbands: &[PreparedEncodeSubband], + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + if prepared_subbands.iter().all(|subband| { + subband.code_blocks.is_empty() || subband.preencoded_ht_code_blocks.is_some() + }) { + let total_blocks = prepared_subbands + .iter() + .map(|subband| subband.code_blocks.len()) + .sum(); + let mut encoded = Vec::with_capacity(total_blocks); + for subband in prepared_subbands { + if let Some(blocks) = &subband.preencoded_ht_code_blocks { + if blocks.len() != subband.code_blocks.len() { + return Err("preencoded HT subband code-block count mismatch"); + } + encoded.extend( + blocks + .iter() + .cloned() + .map(ht_encoded_code_block_from_accelerator), + ); + } + } + return Ok(encoded); + } + if prepared_subbands + .iter() + .any(|subband| subband.preencoded_ht_code_blocks.is_some()) + { + return Err("mixed preencoded and quantized HT subbands are unsupported"); + } + + let jobs: Vec<_> = prepared_subbands + .iter() + .flat_map(|subband| { + subband + .code_blocks + .iter() + .map(move |block| crate::J2kHtCodeBlockEncodeJob { + coefficients: &block.coefficients, + width: block.width, + height: block.height, + total_bitplanes: subband.total_bitplanes, + target_coding_passes: 1, + }) + }) + .collect(); + + if let Some(encoded) = accelerator.encode_ht_code_blocks(&jobs)? { + if encoded.len() != jobs.len() { + return Err("accelerated HT code-block batch length mismatch"); + } + return Ok(encoded + .into_iter() + .map(ht_encoded_code_block_from_accelerator) + .collect()); + } + + if accelerator.prefer_parallel_cpu_code_block_fallback() { + if jobs.len() < HT_CPU_PARALLEL_FALLBACK_MIN_JOBS { + return encode_all_ht_code_blocks_serial_cpu(&jobs); + } + return encode_all_ht_code_blocks_parallel(&jobs); + } + + jobs.iter() + .map(|job| { + encode_ht_code_block( + job.coefficients, + job.width, + job.height, + job.total_bitplanes, + accelerator, + ) + }) + .collect() +} + +fn encode_all_tier1_code_blocks( + prepared_subbands: &[PreparedEncodeSubband], + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, &'static str> { + let style = default_public_code_block_style(); + let jobs: Vec<_> = prepared_subbands + .iter() + .flat_map(|subband| { + let public_sub_band_type = public_sub_band_type(subband.sub_band_type); + subband + .code_blocks + .iter() + .map(move |block| J2kTier1CodeBlockEncodeJob { + coefficients: &block.coefficients, + width: block.width, + height: block.height, + sub_band_type: public_sub_band_type, + total_bitplanes: subband.total_bitplanes, + style, + }) + }) + .collect(); + + if let Some(encoded) = accelerator.encode_tier1_code_blocks(&jobs)? { + if encoded.len() != jobs.len() { + return Err("accelerated classic code-block batch length mismatch"); + } + return Ok(encoded + .into_iter() + .map(encoded_code_block_from_accelerator) + .collect()); + } + + if accelerator.prefer_parallel_cpu_code_block_fallback() { + return encode_all_tier1_code_blocks_parallel(&jobs); + } + + let mut encoded = Vec::with_capacity(jobs.len()); + for subband in prepared_subbands { + for block in &subband.code_blocks { + encoded.push(encode_tier1_code_block( + &block.coefficients, + block.width, + block.height, + subband.sub_band_type, + subband.total_bitplanes, + accelerator, + )?); + } + } + Ok(encoded) +} + +fn encode_all_ht_code_blocks_serial_cpu( + jobs: &[crate::J2kHtCodeBlockEncodeJob<'_>], +) -> Result, &'static str> { + if jobs.iter().any(|job| job.target_coding_passes != 1) { + return Err("CPU HTJ2K code-block fallback supports cleanup-only encode"); + } + jobs.iter() + .map(|job| { + ht_block_encode::encode_code_block( + job.coefficients, + job.width, + job.height, + job.total_bitplanes, + ) + }) + .collect() +} + +#[cfg(feature = "parallel")] +fn encode_all_ht_code_blocks_parallel( + jobs: &[crate::J2kHtCodeBlockEncodeJob<'_>], +) -> Result, &'static str> { + if jobs.iter().any(|job| job.target_coding_passes != 1) { + return Err("CPU HTJ2K code-block fallback supports cleanup-only encode"); + } + jobs.par_iter() + .map(|job| { + ht_block_encode::encode_code_block( + job.coefficients, + job.width, + job.height, + job.total_bitplanes, + ) + }) + .collect() +} + +#[cfg(not(feature = "parallel"))] +fn encode_all_ht_code_blocks_parallel( + jobs: &[crate::J2kHtCodeBlockEncodeJob<'_>], +) -> Result, &'static str> { + if jobs.iter().any(|job| job.target_coding_passes != 1) { + return Err("CPU HTJ2K code-block fallback supports cleanup-only encode"); + } + jobs.iter() + .map(|job| { + ht_block_encode::encode_code_block( + job.coefficients, + job.width, + job.height, + job.total_bitplanes, + ) + }) + .collect() +} + +#[cfg(feature = "parallel")] +fn encode_all_tier1_code_blocks_parallel( + jobs: &[J2kTier1CodeBlockEncodeJob<'_>], +) -> Result, &'static str> { + jobs.par_iter() + .map(|job| { + Ok(bitplane_encode::encode_code_block( + job.coefficients, + job.width, + job.height, + internal_sub_band_type(job.sub_band_type), + job.total_bitplanes, + )) + }) + .collect() +} + +#[cfg(not(feature = "parallel"))] +fn encode_all_tier1_code_blocks_parallel( + jobs: &[J2kTier1CodeBlockEncodeJob<'_>], +) -> Result, &'static str> { + jobs.iter() + .map(|job| { + Ok(bitplane_encode::encode_code_block( + job.coefficients, + job.width, + job.height, + internal_sub_band_type(job.sub_band_type), + job.total_bitplanes, + )) + }) + .collect() +} + +fn encode_ht_code_block( + coefficients: &[i32], + width: u32, + height: u32, + total_bitplanes: u8, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result { + if let Some(encoded) = accelerator.encode_ht_code_block(crate::J2kHtCodeBlockEncodeJob { + coefficients, + width, + height, + total_bitplanes, + target_coding_passes: 1, + })? { + return Ok(ht_encoded_code_block_from_accelerator(encoded)); + } + + ht_block_encode::encode_code_block(coefficients, width, height, total_bitplanes) +} + +fn ht_encoded_code_block_from_accelerator( + encoded: crate::EncodedHtJ2kCodeBlock, +) -> bitplane_encode::EncodedCodeBlock { + bitplane_encode::EncodedCodeBlock { + data: encoded.data, + num_coding_passes: encoded.num_coding_passes, + num_zero_bitplanes: encoded.num_zero_bitplanes, + ht_cleanup_length: encoded.cleanup_length, + ht_refinement_length: encoded.refinement_length, + } +} + +fn encode_tier1_code_block( + coefficients: &[i32], + width: u32, + height: u32, + sub_band_type: SubBandType, + total_bitplanes: u8, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result { + if let Some(encoded) = accelerator.encode_tier1_code_block(J2kTier1CodeBlockEncodeJob { + coefficients, + width, + height, + sub_band_type: public_sub_band_type(sub_band_type), + total_bitplanes, + style: default_public_code_block_style(), + })? { + return Ok(encoded_code_block_from_accelerator(encoded)); + } + + Ok(bitplane_encode::encode_code_block( + coefficients, + width, + height, + sub_band_type, + total_bitplanes, + )) +} + +fn encoded_code_block_from_accelerator( + encoded: EncodedJ2kCodeBlock, +) -> bitplane_encode::EncodedCodeBlock { + bitplane_encode::EncodedCodeBlock { + data: encoded.data, + num_coding_passes: encoded.number_of_coding_passes, + num_zero_bitplanes: encoded.missing_bit_planes, + ht_cleanup_length: 0, + ht_refinement_length: 0, + } +} + +fn public_sub_band_type(sub_band_type: SubBandType) -> J2kSubBandType { + match sub_band_type { + SubBandType::LowLow => J2kSubBandType::LowLow, + SubBandType::HighLow => J2kSubBandType::HighLow, + SubBandType::LowHigh => J2kSubBandType::LowHigh, + SubBandType::HighHigh => J2kSubBandType::HighHigh, + } +} + +fn internal_sub_band_type(sub_band_type: J2kSubBandType) -> SubBandType { + match sub_band_type { + J2kSubBandType::LowLow => SubBandType::LowLow, + J2kSubBandType::HighLow => SubBandType::HighLow, + J2kSubBandType::LowHigh => SubBandType::LowHigh, + J2kSubBandType::HighHigh => SubBandType::HighHigh, + } +} + +fn default_public_code_block_style() -> crate::J2kCodeBlockStyle { + crate::J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: false, + reset_context_probabilities: false, + termination_on_each_pass: false, + vertically_causal_context: false, + segmentation_symbols: false, + } +} + +/// Convert interleaved pixel bytes to per-component f32 arrays. +pub(crate) fn deinterleave_to_f32( + pixels: &[u8], + num_pixels: usize, + num_components: u8, + bit_depth: u8, + signed: bool, +) -> Vec> { + if num_components == 3 && bit_depth == 8 && !signed { + return deinterleave_rgb8_unsigned_to_f32(pixels, num_pixels); + } + + let nc = num_components as usize; + let mut components = vec![vec![0.0f32; num_pixels]; nc]; + let unsigned_offset = if signed { + 0.0 + } else { + (1u32 << (bit_depth as u32 - 1)) as f32 + }; + + if bit_depth <= 8 { + for (i, pixel) in pixels.chunks_exact(nc).take(num_pixels).enumerate() { + for (c, component) in components.iter_mut().enumerate().take(nc) { + let val = pixel[c]; + component[i] = if signed { + (val as i8) as f32 + } else { + val as f32 - unsigned_offset + }; + } + } + } else { + // 16-bit samples (little-endian) + for (i, pixel) in pixels.chunks_exact(nc * 2).take(num_pixels).enumerate() { + for (c, component) in components.iter_mut().enumerate().take(nc) { + let offset = c * 2; + let val = u16::from_le_bytes([pixel[offset], pixel[offset + 1]]); + component[i] = if signed { + (val as i16) as f32 + } else { + val as f32 - unsigned_offset + }; + } + } + } + + components +} + +fn deinterleave_rgb8_unsigned_to_f32(pixels: &[u8], num_pixels: usize) -> Vec> { + let mut r = Vec::with_capacity(num_pixels); + let mut g = Vec::with_capacity(num_pixels); + let mut b = Vec::with_capacity(num_pixels); + + for pixel in pixels.chunks_exact(3).take(num_pixels) { + r.push(f32::from(pixel[0]) - 128.0); + g.push(f32::from(pixel[1]) - 128.0); + b.push(f32::from(pixel[2]) - 128.0); + } + + vec![r, g, b] +} + +/// Calculate the maximum number of decomposition levels for given dimensions. +fn max_decomposition_levels(width: u32, height: u32) -> u8 { + let min_dim = width.min(height); + if min_dim <= 1 { + return 0; + } + floor_f32(log2_f32(min_dim as f32)) as u8 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::PrequantizedHtj2k97CodeBlock; + + fn test_preencoded_subband_payload(marker: u8) -> PreencodedHtj2k97Subband { + PreencodedHtj2k97Subband { + sub_band_type: J2kSubBandType::LowLow, + num_cbs_x: 1, + num_cbs_y: 1, + total_bitplanes: 8, + code_blocks: vec![PreencodedHtj2k97CodeBlock { + width: 1, + height: 1, + encoded: crate::EncodedHtJ2kCodeBlock { + data: vec![marker; 8], + cleanup_length: 8, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 0, + }, + }], + } + } + + #[test] + fn prepared_subband_from_owned_preencoded_moves_payload_without_clone() { + let subband = test_preencoded_subband_payload(7); + let original_ptr = subband.code_blocks[0].encoded.data.as_ptr() as usize; + + let prepared = + prepared_subband_from_preencoded_owned(subband).expect("owned preencoded subband"); + let prepared_blocks = prepared + .preencoded_ht_code_blocks + .expect("preencoded payloads"); + + assert_eq!(prepared_blocks[0].data.as_ptr() as usize, original_ptr); + assert!(prepared.code_blocks[0].coefficients.is_empty()); + } + + #[test] + fn compact_preencoded_packetization_borrows_payload_ranges() { + #[derive(Default)] + struct RecordingPacketizationAccelerator { + payload_base: usize, + observed_offsets: Vec, + observed_lengths: Vec, + } + + impl crate::J2kEncodeStageAccelerator for RecordingPacketizationAccelerator { + fn encode_packetization( + &mut self, + job: crate::J2kPacketizationEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + for code_block in job + .resolutions + .iter() + .flat_map(|resolution| resolution.subbands.iter()) + .flat_map(|subband| subband.code_blocks.iter()) + .filter(|code_block| !code_block.data.is_empty()) + { + self.observed_offsets + .push((code_block.data.as_ptr() as usize) - self.payload_base); + self.observed_lengths.push(code_block.data.len()); + } + Ok(Some(crate::encode_j2k_packetization_scalar(job)?)) + } + } + + let (preencoded, options) = sample_preencoded_htj2k97_for_test(); + let expected = encode_preencoded_htj2k_97(&preencoded, &options).expect("owned preencoded"); + let mut payload = Vec::new(); + let mut expected_offsets = Vec::new(); + let mut expected_lengths = Vec::new(); + let components = preencoded + .components + .iter() + .map(|component| PreencodedHtj2k97CompactComponent { + x_rsiz: component.x_rsiz, + y_rsiz: component.y_rsiz, + resolutions: component + .resolutions + .iter() + .map(|resolution| PreencodedHtj2k97CompactResolution { + subbands: resolution + .subbands + .iter() + .map(|subband| PreencodedHtj2k97CompactSubband { + sub_band_type: subband.sub_band_type, + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + total_bitplanes: subband.total_bitplanes, + code_blocks: subband + .code_blocks + .iter() + .map(|block| { + let start = payload.len(); + payload.extend_from_slice(&block.encoded.data); + let end = payload.len(); + if start != end { + expected_offsets.push(start); + expected_lengths.push(end - start); + } + PreencodedHtj2k97CompactCodeBlock { + width: block.width, + height: block.height, + payload_range: start..end, + cleanup_length: block.encoded.cleanup_length, + refinement_length: block.encoded.refinement_length, + num_coding_passes: block.encoded.num_coding_passes, + num_zero_bitplanes: block.encoded.num_zero_bitplanes, + } + }) + .collect(), + }) + .collect(), + }) + .collect(), + }) + .collect(); + let compact = PreencodedHtj2k97CompactImage { + width: preencoded.width, + height: preencoded.height, + bit_depth: preencoded.bit_depth, + signed: preencoded.signed, + payload, + components, + }; + let mut accelerator = RecordingPacketizationAccelerator { + payload_base: compact.payload.as_ptr() as usize, + ..Default::default() + }; + + let actual = encode_preencoded_htj2k_97_compact_owned_with_accelerator( + compact, + &options, + &mut accelerator, + ) + .expect("compact preencoded"); + + assert_eq!(actual, expected); + assert_eq!(accelerator.observed_offsets, expected_offsets); + assert_eq!(accelerator.observed_lengths, expected_lengths); + } + + #[test] + fn test_encode_8bit_gray() { + let width = 8u32; + let height = 8u32; + let pixels: Vec = (0..64).collect(); + + let result = encode( + &pixels, + width, + height, + 1, + 8, + false, + &EncodeOptions { + num_decomposition_levels: 2, + ..Default::default() + }, + ); + + assert!(result.is_ok()); + let codestream = result.unwrap(); + // Verify SOC marker + assert_eq!(codestream[0], 0xFF); + assert_eq!(codestream[1], 0x4F); + // Verify EOC marker + let len = codestream.len(); + assert_eq!(codestream[len - 2], 0xFF); + assert_eq!(codestream[len - 1], 0xD9); + } + + #[test] + fn test_encode_16bit_gray() { + let width = 8u32; + let height = 8u32; + let mut pixels = Vec::with_capacity(128); + for i in 0..64u16 { + let val = i * 100; + pixels.extend_from_slice(&val.to_le_bytes()); + } + + let result = encode( + &pixels, + width, + height, + 1, + 16, + false, + &EncodeOptions { + num_decomposition_levels: 2, + ..Default::default() + }, + ); + + assert!(result.is_ok()); + } + + #[test] + fn test_encode_rgb() { + let width = 16u32; + let height = 16u32; + let pixels: Vec = (0..width * height * 3).map(|i| (i & 0xFF) as u8).collect(); + + let result = encode( + &pixels, + width, + height, + 3, + 8, + false, + &EncodeOptions { + num_decomposition_levels: 3, + ..Default::default() + }, + ); + + assert!(result.is_ok(), "RGB encode failed: {:?}", result.err()); + } + + #[test] + fn encode_with_accelerator_calls_lossless_stage_hooks() { + #[derive(Default)] + struct CountingAccelerator { + forward_rct: usize, + forward_dwt53: usize, + tier1_code_blocks: usize, + tier1_code_block_batches: usize, + tier1_batched_jobs: usize, + packetization: usize, + packetization_resolution_count: u32, + packetization_code_block_count: u32, + packetization_saw_payload: bool, + } + + impl crate::J2kEncodeStageAccelerator for CountingAccelerator { + fn encode_forward_rct( + &mut self, + _job: crate::J2kForwardRctJob<'_>, + ) -> core::result::Result { + self.forward_rct += 1; + Ok(false) + } + + fn encode_forward_dwt53( + &mut self, + _job: crate::J2kForwardDwt53Job<'_>, + ) -> core::result::Result, &'static str> + { + self.forward_dwt53 += 1; + Ok(None) + } + + fn encode_tier1_code_block( + &mut self, + _job: crate::J2kTier1CodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> + { + self.tier1_code_blocks += 1; + Ok(None) + } + + fn encode_tier1_code_blocks( + &mut self, + jobs: &[crate::J2kTier1CodeBlockEncodeJob<'_>], + ) -> core::result::Result>, &'static str> + { + self.tier1_code_block_batches += 1; + self.tier1_batched_jobs += jobs.len(); + Ok(None) + } + + fn encode_packetization( + &mut self, + job: crate::J2kPacketizationEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + self.packetization += 1; + self.packetization_resolution_count = job.resolution_count; + self.packetization_code_block_count = job.code_block_count; + self.packetization_saw_payload = job + .resolutions + .iter() + .flat_map(|resolution| resolution.subbands.iter()) + .flat_map(|subband| subband.code_blocks.iter()) + .any(|code_block| !code_block.data.is_empty()); + Ok(None) + } + } + + let pixels: Vec = (0..8 * 8 * 3).map(|i| (i & 0xFF) as u8).collect(); + let options = EncodeOptions { + num_decomposition_levels: 1, + reversible: true, + ..EncodeOptions::default() + }; + let mut accelerator = CountingAccelerator::default(); + + let codestream = + encode_with_accelerator(&pixels, 8, 8, 3, 8, false, &options, &mut accelerator) + .expect("encode with accelerator hooks"); + + assert!(codestream.starts_with(&[0xFF, 0x4F])); + assert_eq!(accelerator.forward_rct, 1); + assert_eq!(accelerator.forward_dwt53, 3); + assert!(accelerator.tier1_code_block_batches > 0); + assert_eq!( + accelerator.tier1_code_blocks, + accelerator.tier1_batched_jobs + ); + assert_eq!(accelerator.packetization, 1); + assert_eq!(accelerator.packetization_resolution_count, 6); + assert_eq!( + accelerator.packetization_code_block_count, + u32::try_from(accelerator.tier1_code_blocks).expect("test code-block count fits u32") + ); + assert!(accelerator.packetization_saw_payload); + } + + #[test] + fn cpu_only_accelerator_opts_into_parallel_block_fallback_only_for_native_cpu() { + #[derive(Default)] + struct ExternalAccelerator; + + impl crate::J2kEncodeStageAccelerator for ExternalAccelerator {} + + let cpu = crate::CpuOnlyJ2kEncodeStageAccelerator; + let external = ExternalAccelerator; + + assert!(cpu.prefer_parallel_cpu_code_block_fallback()); + assert!(!external.prefer_parallel_cpu_code_block_fallback()); + } + + #[test] + fn cpu_parallel_block_fallback_matches_serial_classic_and_htj2k_output() { + #[derive(Default)] + struct SerialCpuFallbackAccelerator; + + impl crate::J2kEncodeStageAccelerator for SerialCpuFallbackAccelerator {} + + let pixels = gradient_u8(96, 80); + for use_ht_block_coding in [false, true] { + let options = EncodeOptions { + num_decomposition_levels: 1, + code_block_width_exp: 2, + code_block_height_exp: 2, + use_ht_block_coding, + ..EncodeOptions::default() + }; + let parallel = encode(&pixels, 96, 80, 1, 8, false, &options) + .expect("parallel CPU fallback encode"); + let mut serial_accelerator = SerialCpuFallbackAccelerator; + let serial = encode_with_accelerator( + &pixels, + 96, + 80, + 1, + 8, + false, + &options, + &mut serial_accelerator, + ) + .expect("serial CPU fallback encode"); + + assert_eq!(parallel, serial); + } + } + + #[test] + fn precomputed_htj2k53_offers_ht_code_blocks_to_encode_accelerator() { + let image = sample_precomputed_htj2k53_image(); + let options = EncodeOptions { + num_decomposition_levels: 1, + reversible: true, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let mut accelerator = CountingHtEncodeAccelerator::default(); + + let encoded = + encode_precomputed_htj2k_53_with_accelerator(&image, &options, &mut accelerator) + .expect("precomputed 5/3 encode accepts encode accelerator"); + + assert!(encoded.starts_with(&[0xff, 0x4f])); + assert_eq!(accelerator.forward_dwt53, 0); + assert_eq!(accelerator.forward_dwt97, 0); + assert_eq!(accelerator.ht_batches, 1); + assert!(accelerator.ht_jobs > 0); + assert_eq!(accelerator.ht_single_blocks, accelerator.ht_jobs); + } + + #[test] + fn precomputed_htj2k97_offers_ht_code_blocks_to_encode_accelerator() { + let image = sample_precomputed_htj2k97_image(); + let options = EncodeOptions { + num_decomposition_levels: 1, + reversible: false, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let mut accelerator = CountingHtEncodeAccelerator::default(); + + let encoded = + encode_precomputed_htj2k_97_with_accelerator(&image, &options, &mut accelerator) + .expect("precomputed 9/7 encode accepts encode accelerator"); + + assert!(encoded.starts_with(&[0xff, 0x4f])); + assert_eq!(accelerator.forward_dwt53, 0); + assert_eq!(accelerator.forward_dwt97, 0); + assert_eq!(accelerator.ht_batches, 1); + assert!(accelerator.ht_jobs > 0); + assert_eq!(accelerator.ht_single_blocks, accelerator.ht_jobs); + } + + #[test] + fn precomputed_dwt_geometry_validation_rejects_recursive_mismatch_for_both_filters() { + let mut dwt53 = sample_precomputed_htj2k53_image(); + dwt53.components[0].dwt.levels[0].low_width += 1; + assert_eq!( + validate_precomputed_dwt_geometry(&dwt53), + Err("precomputed DWT recursive geometry mismatch") + ); + + let mut dwt97 = sample_precomputed_htj2k97_image(); + dwt97.components[0].dwt.levels[0].low_width += 1; + assert_eq!( + validate_precomputed_dwt97_geometry(&dwt97), + Err("precomputed DWT recursive geometry mismatch") + ); + } + + #[test] + fn prequantized_htj2k97_offers_ht_code_blocks_to_encode_accelerator() { + let image = sample_precomputed_htj2k97_image(); + let options = EncodeOptions { + num_decomposition_levels: 1, + reversible: false, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let prequantized = prequantized_htj2k97_image_from_precomputed_for_test(&image, &options) + .expect("test prequantized image"); + let mut accelerator = CountingHtEncodeAccelerator::default(); + + let encoded = encode_prequantized_htj2k_97_with_accelerator( + &prequantized, + &options, + &mut accelerator, + ) + .expect("prequantized 9/7 encode accepts encode accelerator"); + + assert!(encoded.starts_with(&[0xff, 0x4f])); + assert_eq!(accelerator.forward_dwt53, 0); + assert_eq!(accelerator.forward_dwt97, 0); + assert_eq!(accelerator.ht_batches, 1); + assert!(accelerator.ht_jobs > 0); + assert_eq!(accelerator.ht_single_blocks, accelerator.ht_jobs); + } + + #[test] + fn precomputed_htj2k97_batch_offers_all_ht_code_blocks_in_one_accelerator_call() { + let image = sample_precomputed_htj2k97_image(); + let options = EncodeOptions { + num_decomposition_levels: 1, + reversible: false, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let mut accelerator = CountingHtEncodeAccelerator::default(); + + let encoded = encode_precomputed_htj2k_97_batch_with_accelerator( + &[image.clone(), image], + &options, + &mut accelerator, + ) + .expect("batch precomputed 9/7 encode accepts encode accelerator"); + + assert_eq!(encoded.len(), 2); + assert!(encoded + .iter() + .all(|codestream| codestream.starts_with(&[0xff, 0x4f]))); + assert_eq!(accelerator.forward_dwt53, 0); + assert_eq!(accelerator.forward_dwt97, 0); + assert_eq!(accelerator.ht_batches, 1); + assert!(accelerator.ht_jobs > 0); + assert_eq!(accelerator.ht_single_blocks, accelerator.ht_jobs); + } + + #[derive(Default)] + struct CountingHtEncodeAccelerator { + forward_dwt53: usize, + forward_dwt97: usize, + ht_batches: usize, + ht_jobs: usize, + ht_single_blocks: usize, + } + + impl crate::J2kEncodeStageAccelerator for CountingHtEncodeAccelerator { + fn encode_forward_dwt53( + &mut self, + _job: crate::J2kForwardDwt53Job<'_>, + ) -> core::result::Result, &'static str> { + self.forward_dwt53 += 1; + Ok(None) + } + + fn encode_forward_dwt97( + &mut self, + _job: crate::J2kForwardDwt97Job<'_>, + ) -> core::result::Result, &'static str> { + self.forward_dwt97 += 1; + Ok(None) + } + + fn encode_ht_code_blocks( + &mut self, + jobs: &[crate::J2kHtCodeBlockEncodeJob<'_>], + ) -> core::result::Result>, &'static str> { + self.ht_batches += 1; + self.ht_jobs += jobs.len(); + Ok(None) + } + + fn encode_ht_code_block( + &mut self, + _job: crate::J2kHtCodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + self.ht_single_blocks += 1; + Ok(None) + } + } + + #[test] + fn prepare_subband_uses_fused_ht_subband_without_host_quantized_codeblocks() { + #[derive(Default)] + struct FusedHtSubbandAccelerator { + subband_calls: usize, + quantize_calls: usize, + ht_batch_calls: usize, + } + + impl crate::J2kEncodeStageAccelerator for FusedHtSubbandAccelerator { + fn encode_ht_subband( + &mut self, + job: crate::J2kHtSubbandEncodeJob<'_>, + ) -> core::result::Result>, &'static str> + { + self.subband_calls += 1; + let count = (job.width.div_ceil(job.code_block_width) as usize) + .checked_mul(job.height.div_ceil(job.code_block_height) as usize) + .ok_or("test code-block count overflow")?; + Ok(Some( + (0..count) + .map(|idx| crate::EncodedHtJ2kCodeBlock { + data: vec![u8::try_from(idx).expect("test block index fits"), 0], + cleanup_length: 2, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 0, + }) + .collect(), + )) + } + + fn encode_quantize_subband( + &mut self, + _job: crate::J2kQuantizeSubbandJob<'_>, + ) -> core::result::Result>, &'static str> { + self.quantize_calls += 1; + Ok(None) + } + + fn encode_ht_code_blocks( + &mut self, + _jobs: &[crate::J2kHtCodeBlockEncodeJob<'_>], + ) -> core::result::Result>, &'static str> + { + self.ht_batch_calls += 1; + Ok(None) + } + } + + let coefficients = vec![0.0; 16]; + let mut accelerator = FusedHtSubbandAccelerator::default(); + let prepared = prepare_subband( + &coefficients, + 4, + 4, + &QuantStepSize { + exponent: 8, + mantissa: 0, + }, + 8, + 2, + true, + BlockCodingMode::HighThroughput, + 2, + 2, + SubBandType::LowLow, + &mut accelerator, + ) + .expect("fused HT subband prepare"); + + assert_eq!(accelerator.subband_calls, 1); + assert_eq!(accelerator.quantize_calls, 0); + assert!(prepared.preencoded_ht_code_blocks.is_some()); + assert!(prepared + .code_blocks + .iter() + .all(|block| block.coefficients.is_empty())); + + let precincts = encode_prepared_subbands(vec![prepared], &mut accelerator) + .expect("preencoded HT subband packet data"); + + assert_eq!(accelerator.ht_batch_calls, 0); + assert_eq!(precincts[0].code_blocks.len(), 4); + assert_eq!(precincts[0].code_blocks[2].data, vec![2, 0]); + } + + #[test] + fn ht_cpu_parallel_fallback_threshold_matches_parallel_output() { + assert_eq!(HT_CPU_PARALLEL_FALLBACK_MIN_JOBS, 4); + + let blocks: Vec> = (0..HT_CPU_PARALLEL_FALLBACK_MIN_JOBS) + .map(|seed| { + (0usize..64 * 64) + .map(|index| { + let value = (((index * 31) ^ (seed * 17)) & 0x01ff) as i32 - 255; + if (index + seed).is_multiple_of(11) { + 0 + } else { + value + } + }) + .collect() + }) + .collect(); + let jobs: Vec<_> = blocks + .iter() + .map(|coefficients| crate::J2kHtCodeBlockEncodeJob { + coefficients, + width: 64, + height: 64, + total_bitplanes: 10, + target_coding_passes: 1, + }) + .collect(); + + let serial = + encode_all_ht_code_blocks_serial_cpu(&jobs[..HT_CPU_PARALLEL_FALLBACK_MIN_JOBS - 1]) + .expect("serial tiny HT encode"); + let parallel = + encode_all_ht_code_blocks_parallel(&jobs[..HT_CPU_PARALLEL_FALLBACK_MIN_JOBS]) + .expect("parallel HT encode"); + let serial_threshold = + encode_all_ht_code_blocks_serial_cpu(&jobs[..HT_CPU_PARALLEL_FALLBACK_MIN_JOBS]) + .expect("serial threshold HT encode"); + + assert_eq!(serial.len(), HT_CPU_PARALLEL_FALLBACK_MIN_JOBS - 1); + assert_eq!(parallel.len(), HT_CPU_PARALLEL_FALLBACK_MIN_JOBS); + assert_eq!(serial_threshold.len(), parallel.len()); + for (serial, parallel) in serial_threshold.iter().zip(¶llel) { + assert_eq!(serial.data, parallel.data); + assert_eq!(serial.num_coding_passes, parallel.num_coding_passes); + assert_eq!(serial.num_zero_bitplanes, parallel.num_zero_bitplanes); + } + } + + #[test] + fn code_block_extraction_copies_partial_edge_blocks_rowwise() { + let quantized: Vec = (0..20).collect(); + + let block = copy_code_block_coefficients(&quantized, 5, 3, 1, 2, 3); + + assert_eq!(block, vec![8, 9, 13, 14, 18, 19]); + } + + #[test] + fn test_encode_lossy() { + let pixels: Vec = (0..64).collect(); + + let result = encode( + &pixels, + 8, + 8, + 1, + 8, + false, + &EncodeOptions { + num_decomposition_levels: 2, + reversible: false, + guard_bits: 2, + ..Default::default() + }, + ); + + assert!(result.is_ok()); + } + + #[test] + fn prequantized_htj2k97_matches_precomputed_dwt97_codestream() { + let image = sample_precomputed_htj2k97_image(); + let options = EncodeOptions { + num_decomposition_levels: 1, + reversible: false, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + + let precomputed = + encode_precomputed_htj2k_97(&image, &options).expect("precomputed DWT encode"); + let prequantized = prequantized_htj2k97_image_from_precomputed_for_test(&image, &options) + .expect("test prequantized image"); + let direct = + encode_prequantized_htj2k_97(&prequantized, &options).expect("prequantized encode"); + + assert_eq!(direct, precomputed); + } + + #[test] + fn preencoded_htj2k97_matches_prequantized_codestream() { + let image = sample_precomputed_htj2k97_image(); + let options = EncodeOptions { + num_decomposition_levels: 1, + reversible: false, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let prequantized = prequantized_htj2k97_image_from_precomputed_for_test(&image, &options) + .expect("test prequantized image"); + let expected = + encode_prequantized_htj2k_97(&prequantized, &options).expect("prequantized encode"); + let preencoded = preencoded_htj2k97_image_from_prequantized_for_test(&prequantized) + .expect("test preencoded image"); + let actual = encode_preencoded_htj2k_97(&preencoded, &options).expect("preencoded encode"); + + assert_eq!(actual, expected); + } + + #[test] + fn preencoded_htj2k97_rejects_empty_block_with_wrong_zero_bitplanes() { + let (mut image, options) = sample_preencoded_htj2k97_for_test(); + let block = &mut image.components[0].resolutions[0].subbands[0].code_blocks[0]; + block.encoded = EncodedHtJ2kCodeBlock { + data: Vec::new(), + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 0, + num_zero_bitplanes: 0, + }; + + let error = encode_preencoded_htj2k_97(&image, &options) + .expect_err("invalid all-zero block metadata must be rejected"); + + assert_eq!(error, "empty HTJ2K code-block zero-bitplane count mismatch"); + } + + #[test] + fn preencoded_htj2k97_rejects_coded_block_with_too_many_zero_bitplanes() { + let (mut image, options) = sample_preencoded_htj2k97_for_test(); + let subband = &mut image.components[0].resolutions[0].subbands[0]; + subband.code_blocks[0].encoded.num_zero_bitplanes = subband.total_bitplanes; + + let error = encode_preencoded_htj2k_97(&image, &options) + .expect_err("coded block with no coded bitplanes must be rejected"); + + assert_eq!(error, "HTJ2K code-block zero-bitplane count out of range"); + } + + #[cfg(feature = "std")] + #[test] + fn preencoded_htj2k97_rejects_too_many_coding_passes_without_panic() { + let (mut image, options) = sample_preencoded_htj2k97_for_test(); + image.components[0].resolutions[0].subbands[0].code_blocks[0] + .encoded + .num_coding_passes = 165; + + let result = std::panic::catch_unwind(|| encode_preencoded_htj2k_97(&image, &options)); + + assert!(result.is_ok(), "invalid coding pass count must not panic"); + assert_eq!( + result.expect("catch_unwind returned checked result"), + Err("HTJ2K code-block coding pass count out of range") + ); + } + + #[test] + fn prequantized_htj2k97_accepts_empty_high_subbands() { + let options = EncodeOptions { + num_decomposition_levels: 1, + reversible: false, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let image = PrequantizedHtj2k97Image { + width: 1, + height: 1, + bit_depth: 8, + signed: false, + components: vec![PrequantizedHtj2k97Component { + x_rsiz: 1, + y_rsiz: 1, + resolutions: vec![ + PrequantizedHtj2k97Resolution { + subbands: vec![PrequantizedHtj2k97Subband { + sub_band_type: J2kSubBandType::LowLow, + num_cbs_x: 1, + num_cbs_y: 1, + total_bitplanes: 11, + code_blocks: vec![PrequantizedHtj2k97CodeBlock { + coefficients: vec![0], + width: 1, + height: 1, + }], + }], + }, + PrequantizedHtj2k97Resolution { + subbands: vec![ + empty_prequantized_subband(J2kSubBandType::HighLow), + empty_prequantized_subband(J2kSubBandType::LowHigh), + empty_prequantized_subband(J2kSubBandType::HighHigh), + ], + }, + ], + }], + }; + + let encoded = + encode_prequantized_htj2k_97(&image, &options).expect("empty high subbands encode"); + + assert!(encoded.starts_with(&[0xff, 0x4f])); + } + + fn empty_prequantized_subband(sub_band_type: J2kSubBandType) -> PrequantizedHtj2k97Subband { + PrequantizedHtj2k97Subband { + sub_band_type, + num_cbs_x: 0, + num_cbs_y: 0, + total_bitplanes: 0, + code_blocks: Vec::new(), + } + } + + fn sample_precomputed_htj2k97_image() -> PrecomputedHtj2k97Image { + let width = 17u32; + let height = 13u32; + let low_width = width.div_ceil(2); + let low_height = height.div_ceil(2); + let high_width = width / 2; + let high_height = height / 2; + + PrecomputedHtj2k97Image { + width, + height, + bit_depth: 8, + signed: false, + components: vec![PrecomputedHtj2k97Component { + x_rsiz: 1, + y_rsiz: 1, + dwt: J2kForwardDwt97Output { + ll: sample_f32_coefficients(low_width * low_height, 0.25), + ll_width: low_width, + ll_height: low_height, + levels: vec![J2kForwardDwt97Level { + hl: sample_f32_coefficients(high_width * low_height, -0.75), + lh: sample_f32_coefficients(low_width * high_height, 1.25), + hh: sample_f32_coefficients(high_width * high_height, -1.5), + width, + height, + low_width, + low_height, + high_width, + high_height, + }], + }, + }], + } + } + + fn sample_precomputed_htj2k53_image() -> PrecomputedHtj2k53Image { + let width = 17u32; + let height = 13u32; + let low_width = width.div_ceil(2); + let low_height = height.div_ceil(2); + let high_width = width / 2; + let high_height = height / 2; + + PrecomputedHtj2k53Image { + width, + height, + bit_depth: 8, + signed: false, + components: vec![PrecomputedHtj2k53Component { + x_rsiz: 1, + y_rsiz: 1, + dwt: J2kForwardDwt53Output { + ll: sample_f32_coefficients(low_width * low_height, 0.0), + ll_width: low_width, + ll_height: low_height, + levels: vec![J2kForwardDwt53Level { + hl: sample_f32_coefficients(high_width * low_height, -2.0), + lh: sample_f32_coefficients(low_width * high_height, 2.0), + hh: sample_f32_coefficients(high_width * high_height, -4.0), + width, + height, + low_width, + low_height, + high_width, + high_height, + }], + }, + }], + } + } + + fn sample_f32_coefficients(len: u32, offset: f32) -> Vec { + (0..len) + .map(|idx| ((idx % 17) as f32 - 8.0) * 0.5 + offset) + .collect() + } + + fn prequantized_htj2k97_image_from_precomputed_for_test( + image: &PrecomputedHtj2k97Image, + options: &EncodeOptions, + ) -> Result { + let guard_bits = options.guard_bits.max(2); + let step_sizes = quantize::compute_step_sizes_with_irreversible_profile( + image.bit_depth, + 1, + false, + guard_bits, + options.irreversible_quantization_scale, + options.irreversible_quantization_subband_scales, + ); + let cb_width = 1u32 << (options.code_block_width_exp + 2); + let cb_height = 1u32 << (options.code_block_height_exp + 2); + + let components = image + .components + .iter() + .map(|component| { + let mut resolutions = Vec::with_capacity(component.dwt.levels.len() + 1); + resolutions.push(PrequantizedHtj2k97Resolution { + subbands: vec![prequantized_subband_for_test( + &component.dwt.ll, + component.dwt.ll_width, + component.dwt.ll_height, + SubBandType::LowLow, + &step_sizes[0], + image.bit_depth, + guard_bits, + cb_width, + cb_height, + )?], + }); + + for (level_index, level) in component.dwt.levels.iter().enumerate() { + let step_base = 1 + level_index * 3; + resolutions.push(PrequantizedHtj2k97Resolution { + subbands: vec![ + prequantized_subband_for_test( + &level.hl, + level.high_width, + level.low_height, + SubBandType::HighLow, + &step_sizes[step_base], + image.bit_depth, + guard_bits, + cb_width, + cb_height, + )?, + prequantized_subband_for_test( + &level.lh, + level.low_width, + level.high_height, + SubBandType::LowHigh, + &step_sizes[step_base + 1], + image.bit_depth, + guard_bits, + cb_width, + cb_height, + )?, + prequantized_subband_for_test( + &level.hh, + level.high_width, + level.high_height, + SubBandType::HighHigh, + &step_sizes[step_base + 2], + image.bit_depth, + guard_bits, + cb_width, + cb_height, + )?, + ], + }); + } + + Ok(PrequantizedHtj2k97Component { + x_rsiz: component.x_rsiz, + y_rsiz: component.y_rsiz, + resolutions, + }) + }) + .collect::, &'static str>>()?; + + Ok(PrequantizedHtj2k97Image { + width: image.width, + height: image.height, + bit_depth: image.bit_depth, + signed: image.signed, + components, + }) + } + + #[allow(clippy::too_many_arguments)] + fn prequantized_subband_for_test( + coefficients: &[f32], + width: u32, + height: u32, + sub_band_type: SubBandType, + step_size: &QuantStepSize, + bit_depth: u8, + guard_bits: u8, + cb_width: u32, + cb_height: u32, + ) -> Result { + let mut accelerator = CpuOnlyJ2kEncodeStageAccelerator; + let prepared = prepare_subband( + coefficients, + width, + height, + step_size, + bit_depth, + guard_bits, + false, + BlockCodingMode::HighThroughput, + cb_width, + cb_height, + sub_band_type, + &mut accelerator, + )?; + + Ok(PrequantizedHtj2k97Subband { + sub_band_type: public_sub_band_type(sub_band_type), + num_cbs_x: prepared.num_cbs_x, + num_cbs_y: prepared.num_cbs_y, + total_bitplanes: prepared.total_bitplanes, + code_blocks: prepared + .code_blocks + .into_iter() + .map(|block| PrequantizedHtj2k97CodeBlock { + coefficients: block.coefficients, + width: block.width, + height: block.height, + }) + .collect(), + }) + } + + fn preencoded_htj2k97_image_from_prequantized_for_test( + image: &PrequantizedHtj2k97Image, + ) -> Result { + let components = image + .components + .iter() + .map(|component| { + Ok(PreencodedHtj2k97Component { + x_rsiz: component.x_rsiz, + y_rsiz: component.y_rsiz, + resolutions: component + .resolutions + .iter() + .map(|resolution| { + Ok(PreencodedHtj2k97Resolution { + subbands: resolution + .subbands + .iter() + .map(preencoded_subband_from_prequantized_for_test) + .collect::, &'static str>>()?, + }) + }) + .collect::, &'static str>>()?, + }) + }) + .collect::, &'static str>>()?; + + Ok(PreencodedHtj2k97Image { + width: image.width, + height: image.height, + bit_depth: image.bit_depth, + signed: image.signed, + components, + }) + } + + fn sample_preencoded_htj2k97_for_test() -> (PreencodedHtj2k97Image, EncodeOptions) { + let image = sample_precomputed_htj2k97_image(); + let options = EncodeOptions { + num_decomposition_levels: 1, + reversible: false, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + let prequantized = prequantized_htj2k97_image_from_precomputed_for_test(&image, &options) + .expect("test prequantized image"); + let preencoded = preencoded_htj2k97_image_from_prequantized_for_test(&prequantized) + .expect("test preencoded image"); + (preencoded, options) + } + + fn preencoded_subband_from_prequantized_for_test( + subband: &PrequantizedHtj2k97Subband, + ) -> Result { + let code_blocks = subband + .code_blocks + .iter() + .map(|block| { + let encoded = ht_block_encode::encode_code_block( + &block.coefficients, + block.width, + block.height, + subband.total_bitplanes, + )?; + Ok(PreencodedHtj2k97CodeBlock { + width: block.width, + height: block.height, + encoded: EncodedHtJ2kCodeBlock { + data: encoded.data, + cleanup_length: encoded.ht_cleanup_length, + refinement_length: encoded.ht_refinement_length, + num_coding_passes: encoded.num_coding_passes, + num_zero_bitplanes: encoded.num_zero_bitplanes, + }, + }) + }) + .collect::, &'static str>>()?; + + Ok(PreencodedHtj2k97Subband { + sub_band_type: subband.sub_band_type, + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + total_bitplanes: subband.total_bitplanes, + code_blocks, + }) + } + + fn assert_htj2k_lossless_roundtrip( + pixels: &[u8], + width: u32, + height: u32, + bit_depth: u8, + num_decomposition_levels: u8, + ) { + let codestream = encode_htj2k( + pixels, + width, + height, + 1, + bit_depth, + false, + &EncodeOptions { + num_decomposition_levels, + ..Default::default() + }, + ) + .expect("HTJ2K encode"); + + assert!(codestream.windows(2).any(|window| window == [0xFF, 0x50])); + let cod_offset = codestream + .windows(2) + .position(|window| window == [0xFF, 0x52]) + .expect("COD marker"); + assert_eq!(codestream[cod_offset + 12], 0x40); + + let image = Image::new( + &codestream, + &DecodeSettings { + resolve_palette_indices: true, + strict: true, + target_resolution: None, + }, + ) + .expect("parse HT codestream"); + let decoded = image.decode_native().expect("decode HT codestream"); + + assert_eq!(decoded.width, width); + assert_eq!(decoded.height, height); + assert_eq!(decoded.bit_depth, bit_depth); + assert_eq!(decoded.data, pixels); + } + + fn gradient_u8(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity((width * height) as usize); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 17 + y * 31) % 256) as u8); + } + } + pixels + } + + fn lossy_htj2k_roundtrip_u8( + pixels: &[u8], + width: u32, + height: u32, + num_decomposition_levels: u8, + ) -> (Vec, usize) { + let codestream = encode_htj2k( + pixels, + width, + height, + 1, + 8, + false, + &EncodeOptions { + num_decomposition_levels, + reversible: false, + guard_bits: 2, + ..Default::default() + }, + ) + .expect("lossy HT encode"); + + assert!(codestream.windows(2).any(|window| window == [0xFF, 0x50])); + + let image = Image::new( + &codestream, + &DecodeSettings { + resolve_palette_indices: true, + strict: true, + target_resolution: None, + }, + ) + .expect("parse lossy HT codestream"); + let decoded = image.decode_native().expect("decode lossy HT codestream"); + + assert_eq!(decoded.width, width); + assert_eq!(decoded.height, height); + assert_eq!(decoded.bit_depth, 8); + + (decoded.data, codestream.len()) + } + + fn max_abs_error(expected: &[u8], actual: &[u8]) -> u8 { + expected + .iter() + .zip(actual) + .map(|(&expected, &actual)| expected.abs_diff(actual)) + .max() + .unwrap_or(0) + } + + fn psnr_db(expected: &[u8], actual: &[u8]) -> f64 { + let mse = expected + .iter() + .zip(actual) + .map(|(&expected, &actual)| { + let diff = f64::from(expected) - f64::from(actual); + diff * diff + }) + .sum::() + / expected.len() as f64; + + if mse == 0.0 { + f64::INFINITY + } else { + 20.0 * 255.0f64.log10() - 10.0 * mse.log10() + } + } + + fn assert_not_flat_128(decoded: &[u8]) { + assert!( + decoded.iter().any(|&sample| sample != 128), + "lossy decode collapsed to flat 128" + ); + } + + #[test] + fn test_encode_high_throughput_zero_image_roundtrip() { + let width = 4u32; + let height = 4u32; + let sample = 2048u16.to_le_bytes(); + let mut pixels = Vec::with_capacity((width * height * 2) as usize); + for _ in 0..(width * height) { + pixels.extend_from_slice(&sample); + } + + let codestream = encode( + &pixels, + width, + height, + 1, + 12, + false, + &EncodeOptions { + num_decomposition_levels: 2, + use_ht_block_coding: true, + ..Default::default() + }, + ) + .expect("HT all-zero encode"); + + assert!(codestream.windows(2).any(|window| window == [0xFF, 0x50])); + let cod_offset = codestream + .windows(2) + .position(|window| window == [0xFF, 0x52]) + .expect("COD marker"); + assert_eq!(codestream[cod_offset + 12], 0x40); + + let image = + Image::new(&codestream, &DecodeSettings::default()).expect("parse HT codestream"); + let decoded = image.decode_native().expect("decode HT codestream"); + + assert_eq!(decoded.width, width); + assert_eq!(decoded.height, height); + assert_eq!(decoded.bit_depth, 12); + assert_eq!(decoded.data, pixels); + } + + #[test] + fn test_encode_high_throughput_nonzero_roundtrip() { + let width = 1u32; + let height = 1u32; + let pixels = 2049u16.to_le_bytes().to_vec(); + + let codestream = encode_htj2k( + &pixels, + width, + height, + 1, + 12, + false, + &EncodeOptions { + num_decomposition_levels: 0, + ..Default::default() + }, + ) + .expect("HT non-zero encode"); + + assert!(codestream.windows(2).any(|window| window == [0xFF, 0x50])); + let image = + Image::new(&codestream, &DecodeSettings::default()).expect("parse HT codestream"); + let decoded = image.decode_native().expect("decode HT codestream"); + + assert_eq!(decoded.width, width); + assert_eq!(decoded.height, height); + assert_eq!(decoded.bit_depth, 12); + assert_eq!(decoded.data, pixels); + } + + #[test] + fn test_encode_high_throughput_varied_12bit_roundtrip() { + let mut pixels = Vec::with_capacity(32); + for i in 0u16..16 { + pixels.extend_from_slice(&((i * 257) & 0x0FFF).to_le_bytes()); + } + + let codestream = encode_htj2k( + &pixels, + 4, + 4, + 1, + 12, + false, + &EncodeOptions { + num_decomposition_levels: 1, + ..Default::default() + }, + ) + .expect("HT varied encode"); + + let image = + Image::new(&codestream, &DecodeSettings::default()).expect("parse HT codestream"); + let decoded = image.decode_native().expect("decode HT codestream"); + + assert_eq!(decoded.width, 4); + assert_eq!(decoded.height, 4); + assert_eq!(decoded.bit_depth, 12); + assert_eq!(decoded.data, pixels); + } + + #[test] + fn test_encode_high_throughput_gradient_8bit_roundtrip() { + let pixels: Vec = (0..64).collect(); + + let codestream = encode_htj2k( + &pixels, + 8, + 8, + 1, + 8, + false, + &EncodeOptions { + num_decomposition_levels: 3, + ..Default::default() + }, + ) + .expect("HT gradient encode"); + + let image = + Image::new(&codestream, &DecodeSettings::default()).expect("parse HT codestream"); + let decoded = image.decode_native().expect("decode HT codestream"); + + assert_eq!(decoded.width, 8); + assert_eq!(decoded.height, 8); + assert_eq!(decoded.bit_depth, 8); + assert_eq!(decoded.data, pixels); + } + + #[test] + fn test_encode_high_throughput_varied_12bit_large_roundtrip() { + let width = 16u32; + let height = 8u32; + let mut pixels = Vec::with_capacity((width * height * 2) as usize); + for y in 0u16..height as u16 { + for x in 0u16..width as u16 { + let value = (x * 257 + y * 17) & 0x0FFF; + pixels.extend_from_slice(&value.to_le_bytes()); + } + } + + assert_htj2k_lossless_roundtrip(&pixels, width, height, 12, 4); + } + + #[test] + fn test_encode_high_throughput_ramp_16bit_roundtrip() { + let width = 48u32; + let height = 24u32; + let mut pixels = Vec::with_capacity((width * height * 2) as usize); + for y in 0u16..height as u16 { + for x in 0u16..width as u16 { + let value = x * 521 + y * 997; + pixels.extend_from_slice(&value.to_le_bytes()); + } + } + + assert_htj2k_lossless_roundtrip(&pixels, width, height, 16, 4); + } + + #[test] + fn test_encode_high_throughput_lossy_large_gradient_is_parseable() { + let pixels = gradient_u8(128, 128); + + let (decoded, codestream_len) = lossy_htj2k_roundtrip_u8(&pixels, 128, 128, 5); + + assert!(codestream_len > 110); + assert_not_flat_128(&decoded); + assert!( + psnr_db(&pixels, &decoded) >= 30.0, + "psnr={} max_abs={}", + psnr_db(&pixels, &decoded), + max_abs_error(&pixels, &decoded) + ); + } + + #[test] + fn test_encode_high_throughput_lossy_constant_extremes_are_not_midgray() { + for sample in [0u8, 255] { + let pixels = vec![sample; 64 * 64]; + let (decoded, codestream_len) = lossy_htj2k_roundtrip_u8(&pixels, 64, 64, 4); + + assert!(codestream_len > 110); + assert_not_flat_128(&decoded); + assert!( + max_abs_error(&pixels, &decoded) <= 2, + "sample={sample} max_abs={} decoded_min={} decoded_max={}", + max_abs_error(&pixels, &decoded), + decoded.iter().min().unwrap(), + decoded.iter().max().unwrap() + ); + } + } + + #[test] + fn test_encode_invalid_dimensions() { + let result = encode(&[], 0, 0, 1, 8, false, &EncodeOptions::default()); + assert!(result.is_err()); + } + + #[test] + fn test_encode_too_short() { + let pixels = vec![0u8; 10]; // Way too short for 8x8 + let result = encode(&pixels, 8, 8, 1, 8, false, &EncodeOptions::default()); + assert!(result.is_err()); + } + + #[test] + fn test_deinterleave_rgb() { + let pixels = vec![ + 10u8, 20, 30, // pixel 0: R=10, G=20, B=30 + 40, 50, 60, // pixel 1: R=40, G=50, B=60 + ]; + let comps = deinterleave_to_f32(&pixels, 2, 3, 8, false); + assert_eq!(comps[0], vec![-118.0, -88.0]); // R + assert_eq!(comps[1], vec![-108.0, -78.0]); // G + assert_eq!(comps[2], vec![-98.0, -68.0]); // B + } + + #[test] + fn deinterleave_rgb8_unsigned_fast_path_matches_generic_output() { + let pixels = (0..96) + .map(|value| ((value * 19 + value / 3) & 0xff) as u8) + .collect::>(); + + let expected = deinterleave_to_f32(&pixels, 32, 3, 8, false); + let actual = deinterleave_rgb8_unsigned_to_f32(&pixels, 32); + + assert_eq!(actual, expected); + } + + #[test] + fn test_encode_decode_roundtrip_gray_8bit() { + use crate::{DecodeSettings, Image}; + + // Constant image: all pixels = 42 — simplest possible test + let original: Vec = vec![42u8; 64]; // 8x8, all same value + let encoded = encode( + &original, + 8, + 8, + 1, + 8, + false, + &EncodeOptions { + num_decomposition_levels: 0, + reversible: true, + ..Default::default() + }, + ) + .expect("encode failed"); + + let settings = DecodeSettings { + resolve_palette_indices: false, + strict: false, + target_resolution: None, + }; + let image = Image::new(&encoded, &settings).expect("parse failed"); + let decoded = image.decode_native().expect("decode failed"); + + assert_eq!(decoded.width, 8); + assert_eq!(decoded.height, 8); + assert_eq!(decoded.data, original, "round-trip mismatch"); + } + + #[test] + fn test_encode_decode_roundtrip_gray_8bit_single_dwt_level() { + use crate::{DecodeSettings, Image}; + + let original: Vec = (0..64 * 64) + .map(|value| ((value * 37 + value / 7) & 0xFF) as u8) + .collect(); + let encoded = encode( + &original, + 64, + 64, + 1, + 8, + false, + &EncodeOptions { + num_decomposition_levels: 1, + reversible: true, + ..Default::default() + }, + ) + .expect("encode failed"); + + let image = Image::new(&encoded, &DecodeSettings::default()).expect("parse failed"); + let decoded = image.decode_native().expect("decode failed"); + + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.data, original, "round-trip mismatch"); + } + + /// Precondition gate: native encode_htj2k must produce byte-identical output + /// across repeated invocations with the same input before CUDA parity can be + /// asserted. 96x80 with 3 components and 5 decomposition levels exercises + /// multi-codeblock subbands. + #[cfg(feature = "std")] + #[test] + fn encode_htj2k_is_byte_deterministic() { + const WIDTH: u32 = 96; + const HEIGHT: u32 = 80; + const NUM_COMPONENTS: u8 = 3; + const BIT_DEPTH: u8 = 8; + const REPETITIONS: usize = 8; + + // Deterministic pseudo-random pixel data: simple LCG-like sequence. + let pixel_count = (WIDTH * HEIGHT) as usize * usize::from(NUM_COMPONENTS); + let pixels: Vec = (0..pixel_count) + .map(|i| { + let v = i + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (v >> 56) as u8 + }) + .collect(); + + let options = EncodeOptions { + use_ht_block_coding: true, + reversible: true, + num_decomposition_levels: 5, + validate_high_throughput_codestream: true, + ..EncodeOptions::default() + }; + + let baseline = encode_htj2k( + &pixels, + WIDTH, + HEIGHT, + NUM_COMPONENTS, + BIT_DEPTH, + false, + &options, + ) + .expect("encode_htj2k baseline failed"); + + assert!( + !baseline.is_empty(), + "baseline codestream must not be empty" + ); + + for i in 0..REPETITIONS { + let result = encode_htj2k( + &pixels, + WIDTH, + HEIGHT, + NUM_COMPONENTS, + BIT_DEPTH, + false, + &options, + ) + .unwrap_or_else(|e| panic!("encode_htj2k repetition {i} failed: {e}")); + assert_eq!( + result, + baseline, + "encode_htj2k repetition {i} produced different bytes \ + (len baseline={}, len result={})", + baseline.len(), + result.len() + ); + } + + println!( + "encode_htj2k_is_byte_deterministic: {} bytes, {} repetitions all identical", + baseline.len(), + REPETITIONS + ); + } + + /// Precondition gate: prove native encode_htj2k round-trips 2-component + /// 8-bit lossless images exactly with independent component channels. + #[cfg(feature = "std")] + #[test] + fn native_htj2k_roundtrips_two_component_lossless() { + const WIDTH: u32 = 32; + const HEIGHT: u32 = 24; + const NUM_COMPONENTS: u8 = 2; + const BIT_DEPTH: u8 = 8; + + // Deterministic per-pixel pattern: each sample is a function of its + // flat index so the two planes carry different, non-trivial data. + let pixel_count = WIDTH as usize * HEIGHT as usize * usize::from(NUM_COMPONENTS); + let pixels: Vec = (0..pixel_count) + .map(|i| ((i.wrapping_mul(251).wrapping_add(i / 7)) & 0xFF) as u8) + .collect(); + + let codestream = encode_htj2k( + &pixels, + WIDTH, + HEIGHT, + NUM_COMPONENTS, + BIT_DEPTH, + false, + &EncodeOptions::default(), + ) + .expect("native 2-component HTJ2K encode failed"); + + let image = Image::new( + &codestream, + &DecodeSettings { + resolve_palette_indices: true, + strict: true, + target_resolution: None, + }, + ) + .expect("native 2-component HTJ2K parse failed"); + let decoded = image + .decode_native() + .expect("native 2-component HTJ2K decode failed"); + + assert_eq!(decoded.width, WIDTH, "width mismatch"); + assert_eq!(decoded.height, HEIGHT, "height mismatch"); + assert_eq!(decoded.bit_depth, BIT_DEPTH, "bit_depth mismatch"); + assert_eq!( + decoded.num_components, NUM_COMPONENTS, + "component count mismatch" + ); + assert_eq!( + decoded.data, pixels, + "2-component HTJ2K lossless round-trip mismatch" + ); + + println!( + "native_htj2k_roundtrips_two_component_lossless: {} bytes codestream, {} pixel bytes", + codestream.len(), + pixels.len() + ); + } + + /// Precondition gate: prove native encode_htj2k round-trips 4-component + /// (e.g. RGBA) 8-bit lossless images exactly. + /// Required before a CUDA parity oracle can be established for this component count. + #[cfg(feature = "std")] + #[test] + fn native_htj2k_roundtrips_four_component_lossless() { + const WIDTH: u32 = 32; + const HEIGHT: u32 = 24; + const NUM_COMPONENTS: u8 = 4; + const BIT_DEPTH: u8 = 8; + + // Deterministic per-sample pattern across all four planes. + let pixel_count = WIDTH as usize * HEIGHT as usize * usize::from(NUM_COMPONENTS); + let pixels: Vec = (0..pixel_count) + .map(|i| ((i.wrapping_mul(197).wrapping_add(i / 13)) & 0xFF) as u8) + .collect(); + + let codestream = encode_htj2k( + &pixels, + WIDTH, + HEIGHT, + NUM_COMPONENTS, + BIT_DEPTH, + false, + &EncodeOptions::default(), + ) + .expect("native 4-component HTJ2K encode failed"); + + let image = Image::new( + &codestream, + &DecodeSettings { + resolve_palette_indices: true, + strict: true, + target_resolution: None, + }, + ) + .expect("native 4-component HTJ2K parse failed"); + let decoded = image + .decode_native() + .expect("native 4-component HTJ2K decode failed"); + + assert_eq!(decoded.width, WIDTH, "width mismatch"); + assert_eq!(decoded.height, HEIGHT, "height mismatch"); + assert_eq!(decoded.bit_depth, BIT_DEPTH, "bit_depth mismatch"); + assert_eq!( + decoded.num_components, NUM_COMPONENTS, + "component count mismatch" + ); + assert_eq!( + decoded.data, pixels, + "4-component HTJ2K lossless round-trip mismatch" + ); + + println!( + "native_htj2k_roundtrips_four_component_lossless: {} bytes codestream, {} pixel bytes", + codestream.len(), + pixels.len() + ); + } + + #[test] + fn classic_pcrd_assigns_limited_budget_by_distortion_slope() { + let candidates = vec![ + ClassicSegmentAssignmentCandidate { + block_index: 0, + segment_index: 0, + rate: 500, + distortion_delta: 500.0, + }, + ClassicSegmentAssignmentCandidate { + block_index: 1, + segment_index: 0, + rate: 700, + distortion_delta: 7_000.0, + }, + ClassicSegmentAssignmentCandidate { + block_index: 2, + segment_index: 0, + rate: 600, + distortion_delta: 3_000.0, + }, + ]; + + let assignments = assign_classic_segment_layers_by_slope(&candidates, 2, &[256, 3_000]) + .expect("PCRD assignment"); + + assert_eq!( + assignments, + vec![1, 0, 1], + "the highest slope contribution should consume the constrained first-layer budget" + ); + } + + #[test] + fn classic_pcrd_allows_byte_target_tolerance_for_first_legal_truncation() { + let candidates = vec![ClassicSegmentAssignmentCandidate { + block_index: 0, + segment_index: 0, + rate: 300, + distortion_delta: 1_000.0, + }]; + + let assignments = assign_classic_segment_layers_by_slope(&candidates, 2, &[256, 1_000]) + .expect("PCRD assignment"); + + assert_eq!(assignments, vec![0]); + } + + #[test] + fn classic_pcrd_does_not_spend_budget_on_non_prefix_segments() { + let candidates = vec![ + ClassicSegmentAssignmentCandidate { + block_index: 0, + segment_index: 0, + rate: 1_000, + distortion_delta: 1_000.0, + }, + ClassicSegmentAssignmentCandidate { + block_index: 0, + segment_index: 1, + rate: 500, + distortion_delta: 10_000.0, + }, + ClassicSegmentAssignmentCandidate { + block_index: 1, + segment_index: 0, + rate: 300, + distortion_delta: 600.0, + }, + ]; + + let assignments = assign_classic_segment_layers_by_slope(&candidates, 2, &[256, 2_000]) + .expect("PCRD assignment"); + + assert_eq!( + assignments, + vec![1, 1, 0], + "first-layer budget must go to the best legal prefix contribution" + ); + } + + #[test] + fn ht_layer_assignment_uses_segment_budget_before_block_index() { + let candidates = vec![ + HtSegmentAssignmentCandidate { + block_index: 0, + rate: 900, + }, + HtSegmentAssignmentCandidate { + block_index: 1, + rate: 200, + }, + HtSegmentAssignmentCandidate { + block_index: 2, + rate: 200, + }, + ]; + + let assignments = assign_ht_segment_layers_by_budget(&candidates, 2, &[256, 2_000]) + .expect("HTJ2K segment assignment"); + + assert_eq!( + assignments, + vec![1, 0, 0], + "HTJ2K early layers should be filled by segment byte budget, not block index" + ); + } +} diff --git a/crates/signinum-j2k-native/src/j2c/fdwt.rs b/crates/j2k-native/src/j2c/fdwt.rs similarity index 77% rename from crates/signinum-j2k-native/src/j2c/fdwt.rs rename to crates/j2k-native/src/j2c/fdwt.rs index ac72feba..6744178e 100644 --- a/crates/signinum-j2k-native/src/j2c/fdwt.rs +++ b/crates/j2k-native/src/j2c/fdwt.rs @@ -193,6 +193,11 @@ fn forward_lift_53(data: &mut [f32]) { return; } + if n.is_multiple_of(2) { + forward_lift_53_even(data); + return; + } + // Step 1: Predict (high-pass) — update odd samples // d(i) = x(2i+1) - floor((x(2i) + x(2i+2)) / 2) let last_even = if n.is_multiple_of(2) { n - 2 } else { n - 1 }; @@ -215,58 +220,84 @@ fn forward_lift_53(data: &mut [f32]) { } } +fn forward_lift_53_even(data: &mut [f32]) { + let n = data.len(); + debug_assert!(n >= 2); + debug_assert!(n.is_multiple_of(2)); + + for i in (1..n - 1).step_by(2) { + data[i] -= floor_f32((data[i - 1] + data[i + 1]) * 0.5); + } + data[n - 1] -= floor_f32(data[n - 2]); + + data[0] += floor_f32(data[1] * 0.5 + 0.5); + for i in (2..n).step_by(2) { + data[i] += floor_f32((data[i - 1] + data[i + 1]) * 0.25 + 0.5); + } +} + /// Forward 9-7 irreversible lifting (floating-point). /// /// The forward transform applies the lifting steps in the order that is /// the reverse of the inverse DWT in idwt.rs. /// /// Forward lifting steps: -/// 1. s(n) += α * (d(n-1) + d(n)) (predict high from low neighbors) -/// 2. d(n) += β * (s(n) + s(n+1)) (update low from high neighbors) -/// 3. s(n) += γ * (d(n-1) + d(n)) (second predict) -/// 4. d(n) += δ * (s(n) + s(n+1)) (second update) -/// 5. s(n) *= κ (scale low-pass) -/// 6. d(n) *= 1/κ (scale high-pass) +/// 1. d(n) += α * (s(n) + s(n+1)) (predict high from low neighbors) +/// 2. s(n) += β * (d(n-1) + d(n)) (update low from high neighbors) +/// 3. d(n) += γ * (s(n) + s(n+1)) (second predict) +/// 4. s(n) += δ * (d(n-1) + d(n)) (second update) +/// 5. s(n) *= 1/κ (scale low-pass) +/// 6. d(n) *= κ (scale high-pass) fn forward_lift_97(data: &mut [f32]) { let n = data.len(); if n < 2 { return; } - // Step 1: α lifting on even (low-pass) samples - for i in (0..n).step_by(2) { - let left = if i > 0 { data[i - 1] } else { data[1] }; - let right = if i + 1 < n { data[i + 1] } else { left }; - data[i] += ALPHA * (left + right); - } + let last_even = if n.is_multiple_of(2) { n - 2 } else { n - 1 }; - // Step 2: β lifting on odd (high-pass) samples + // Step 1: α predict on odd (high-pass) samples for i in (1..n).step_by(2) { let left = data[i - 1]; - let right = if i + 1 < n { data[i + 1] } else { data[i - 1] }; - data[i] += BETA * (left + right); + let right = if i + 1 < n { + data[i + 1] + } else { + data[last_even] + }; + data[i] += ALPHA * (left + right); } - // Step 3: γ lifting on even samples + // Step 2: β update on even (low-pass) samples for i in (0..n).step_by(2) { let left = if i > 0 { data[i - 1] } else { data[1] }; let right = if i + 1 < n { data[i + 1] } else { left }; - data[i] += GAMMA * (left + right); + data[i] += BETA * (left + right); } - // Step 4: δ lifting on odd samples + // Step 3: γ predict on odd samples for i in (1..n).step_by(2) { let left = data[i - 1]; - let right = if i + 1 < n { data[i + 1] } else { data[i - 1] }; + let right = if i + 1 < n { + data[i + 1] + } else { + data[last_even] + }; + data[i] += GAMMA * (left + right); + } + + // Step 4: δ update on even samples + for i in (0..n).step_by(2) { + let left = if i > 0 { data[i - 1] } else { data[1] }; + let right = if i + 1 < n { data[i + 1] } else { left }; data[i] += DELTA * (left + right); } // Step 5 & 6: Scale for i in (0..n).step_by(2) { - data[i] *= KAPPA; + data[i] *= INV_KAPPA; } for i in (1..n).step_by(2) { - data[i] *= INV_KAPPA; + data[i] *= KAPPA; } } @@ -289,13 +320,53 @@ mod tests { assert!(approx_eq_slice(&data, &[10.0, 20.0, 30.0, 40.0], 0.001)); } + #[test] + fn forward_53_even_fast_path_matches_reference_for_common_tile_widths() { + for len in [2usize, 4, 8, 64, 512] { + let mut expected = (0..len) + .map(|idx| ((idx * 37 + idx / 3) & 0xff) as f32 - 128.0) + .collect::>(); + let mut actual = expected.clone(); + + forward_lift_53_reference(&mut expected); + forward_lift_53_even(&mut actual); + + assert_eq!(actual, expected, "len={len}"); + } + } + #[test] fn test_forward_97_round_trip() { - let original = vec![10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0]; - let mut data = original.clone(); - forward_lift_97(&mut data); - inverse_lift_97(&mut data); - assert!(approx_eq_slice(&data, &original, 0.01)); + for len in [2usize, 3, 8, 9, 64, 65] { + let original: Vec = (0..len) + .map(|idx| ((idx * 37 + idx / 3) & 0xff) as f32 - 128.0) + .collect(); + let mut data = original.clone(); + + forward_lift_97(&mut data); + crate::j2c::idwt::test_irreversible_filter_97i(&mut data, len, 0); + + assert!( + approx_eq_slice(&data, &original, 0.01), + "len={len} data={data:?} original={original:?}" + ); + } + } + + #[test] + fn forward_lift_97_places_constant_signal_in_low_pass() { + for len in [2usize, 3, 8, 9, 64, 65] { + let mut data = vec![50.0; len]; + + forward_lift_97(&mut data); + + for &low in data.iter().step_by(2) { + assert!((low - 50.0).abs() < 0.001, "len={len} data={data:?}"); + } + for &high in data.iter().skip(1).step_by(2) { + assert!(high.abs() < 0.001, "len={len} data={data:?}"); + } + } } #[test] @@ -353,38 +424,27 @@ mod tests { } } - fn inverse_lift_97(data: &mut [f32]) { + fn forward_lift_53_reference(data: &mut [f32]) { let n = data.len(); if n < 2 { return; } - // Undo scale - for i in (0..n).step_by(2) { - data[i] *= 1.0 / KAPPA; - } - for i in (1..n).step_by(2) { - data[i] *= KAPPA; - } - // Undo δ, γ, β, α in reverse order - for i in (1..n).step_by(2) { - let left = data[i - 1]; - let right = if i + 1 < n { data[i + 1] } else { data[i - 1] }; - data[i] -= DELTA * (left + right); - } - for i in (0..n).step_by(2) { - let left = if i > 0 { data[i - 1] } else { data[1] }; - let right = if i + 1 < n { data[i + 1] } else { left }; - data[i] -= GAMMA * (left + right); - } + + let last_even = if n.is_multiple_of(2) { n - 2 } else { n - 1 }; for i in (1..n).step_by(2) { let left = data[i - 1]; - let right = if i + 1 < n { data[i + 1] } else { data[i - 1] }; - data[i] -= BETA * (left + right); + let right = if i + 1 < n { + data[i + 1] + } else { + data[last_even] + }; + data[i] -= ((left + right) * 0.5).floor(); } + for i in (0..n).step_by(2) { let left = if i > 0 { data[i - 1] } else { data[1] }; let right = if i + 1 < n { data[i + 1] } else { left }; - data[i] -= ALPHA * (left + right); + data[i] += ((left + right) * 0.25 + 0.5).floor(); } } } diff --git a/crates/signinum-j2k-native/src/j2c/forward_mct.rs b/crates/j2k-native/src/j2c/forward_mct.rs similarity index 99% rename from crates/signinum-j2k-native/src/j2c/forward_mct.rs rename to crates/j2k-native/src/j2c/forward_mct.rs index 7e0853e3..38d451f8 100644 --- a/crates/signinum-j2k-native/src/j2c/forward_mct.rs +++ b/crates/j2k-native/src/j2c/forward_mct.rs @@ -81,6 +81,7 @@ pub(crate) fn forward_ict(components: &mut [Vec]) { #[cfg(test)] mod tests { use super::*; + use alloc::vec; fn approx_eq(a: f32, b: f32, eps: f32) -> bool { (a - b).abs() < eps diff --git a/crates/j2k-native/src/j2c/ht_block_decode.rs b/crates/j2k-native/src/j2c/ht_block_decode.rs new file mode 100644 index 00000000..48e80b46 --- /dev/null +++ b/crates/j2k-native/src/j2c/ht_block_decode.rs @@ -0,0 +1,2259 @@ +//! Scalar HTJ2K block decoding. + +use alloc::{vec, vec::Vec}; + +use super::build::CodeBlock; +use super::decode::DecompositionStorage; +use super::ht_tables::{UVLC_TABLE0, UVLC_TABLE1, VLC_TABLE0, VLC_TABLE1}; +use crate::error::{bail, DecodingError, Result}; +use crate::profile; + +#[derive(Default)] +pub(crate) struct HtBlockDecodeContext { + coefficients: Vec, + scratch: HtBlockDecodeScratch, + width: u32, + height: u32, +} + +impl HtBlockDecodeContext { + fn reset(&mut self, code_block: &CodeBlock) { + self.width = code_block.rect.width(); + self.height = code_block.rect.height(); + self.coefficients.clear(); + self.coefficients + .resize((self.width * self.height) as usize, 0); + } + + pub(crate) fn coefficient_rows(&self) -> impl Iterator { + self.coefficients.chunks_exact(self.width as usize) + } +} + +#[derive(Default)] +pub(crate) struct HtBlockDecodeScratch { + cleanup: Vec, + v_n: Vec, + sigma: Vec, + prev_row_sig: Vec, +} + +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct HtBlockDecodeStats { + pub(crate) blocks: u128, + pub(crate) refinement_blocks: u128, + pub(crate) cleanup_bytes: u128, + pub(crate) refinement_bytes: u128, + pub(crate) ht_cleanup_us: u128, + pub(crate) ht_mag_sgn_us: u128, + pub(crate) ht_sigma_us: u128, + pub(crate) ht_sigprop_us: u128, + pub(crate) ht_magref_us: u128, +} + +impl HtBlockDecodeStats { + fn record_block(&mut self, cleanup_bytes: usize, refinement_bytes: usize) { + self.blocks += 1; + self.cleanup_bytes += cleanup_bytes as u128; + if refinement_bytes > 0 { + self.refinement_blocks += 1; + self.refinement_bytes += refinement_bytes as u128; + } + } +} + +pub(crate) const PHASE_LIMIT_CLEANUP: u8 = 0; +pub(crate) const PHASE_LIMIT_SIGPROP: u8 = 1; +pub(crate) const PHASE_LIMIT_MAGREF: u8 = 2; + +const SIGPROP_SPREAD_MASKS: [u32; 16] = [ + 0x33, 0x76, 0xEC, 0xC8, 0x330, 0x760, 0xEC0, 0xC80, 0x3300, 0x7600, 0xEC00, 0xC800, 0x33000, + 0x76000, 0xEC000, 0xC8000, +]; + +trait HtDecodeObserver { + #[inline(always)] + fn record_block(&mut self, _cleanup_bytes: usize, _refinement_bytes: usize) {} + + #[inline(always)] + fn phase_start(&self) -> Option { + None + } + + #[inline(always)] + fn add_cleanup_us(&mut self, _start: Option) {} + + #[inline(always)] + fn add_mag_sgn_us(&mut self, _start: Option) {} + + #[inline(always)] + fn add_sigma_us(&mut self, _start: Option) {} + + #[inline(always)] + fn add_sigprop_us(&mut self, _start: Option) {} + + #[inline(always)] + fn add_magref_us(&mut self, _start: Option) {} +} + +struct NoHtDecodeStats; + +impl HtDecodeObserver for NoHtDecodeStats {} + +struct RecordingHtDecodeStats<'a> { + stats: &'a mut HtBlockDecodeStats, + profile_enabled: bool, +} + +impl HtDecodeObserver for RecordingHtDecodeStats<'_> { + #[inline(always)] + fn record_block(&mut self, cleanup_bytes: usize, refinement_bytes: usize) { + self.stats.record_block(cleanup_bytes, refinement_bytes); + } + + #[inline(always)] + fn phase_start(&self) -> Option { + if self.profile_enabled { + profile::profile_now(true) + } else { + None + } + } + + #[inline(always)] + fn add_cleanup_us(&mut self, start: Option) { + if self.profile_enabled { + self.stats.ht_cleanup_us += profile::elapsed_us(start); + } + } + + #[inline(always)] + fn add_mag_sgn_us(&mut self, start: Option) { + if self.profile_enabled { + self.stats.ht_mag_sgn_us += profile::elapsed_us(start); + } + } + + #[inline(always)] + fn add_sigma_us(&mut self, start: Option) { + if self.profile_enabled { + self.stats.ht_sigma_us += profile::elapsed_us(start); + } + } + + #[inline(always)] + fn add_sigprop_us(&mut self, start: Option) { + if self.profile_enabled { + self.stats.ht_sigprop_us += profile::elapsed_us(start); + } + } + + #[inline(always)] + fn add_magref_us(&mut self, start: Option) { + if self.profile_enabled { + self.stats.ht_magref_us += profile::elapsed_us(start); + } + } +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct HtBlockDecodeScratchCapacities { + cleanup: usize, + v_n: usize, + sigma: usize, + prev_row_sig: usize, +} + +#[cfg(test)] +impl HtBlockDecodeScratch { + fn capacities_for_test(&self) -> HtBlockDecodeScratchCapacities { + HtBlockDecodeScratchCapacities { + cleanup: self.cleanup.capacity(), + v_n: self.v_n.capacity(), + sigma: self.sigma.capacity(), + prev_row_sig: self.prev_row_sig.capacity(), + } + } + + fn poison_for_test(&mut self) { + self.cleanup.fill(u16::MAX); + self.v_n.fill(u32::MAX); + self.sigma.fill(u16::MAX); + self.prev_row_sig.fill(u16::MAX); + } +} + +#[inline(always)] +fn zeroed_u16_scratch(buffer: &mut Vec, len: usize) -> &mut [u16] { + if buffer.len() < len { + buffer.resize(len, 0); + } + buffer[..len].fill(0); + + &mut buffer[..len] +} + +#[cfg(test)] +fn zeroed_u32_scratch(buffer: &mut Vec, len: usize) -> &mut [u32] { + if buffer.len() < len { + buffer.resize(len, 0); + } + buffer[..len].fill(0); + + &mut buffer[..len] +} + +#[inline(always)] +fn resized_u16_scratch(buffer: &mut Vec, len: usize) -> &mut [u16] { + if buffer.len() < len { + buffer.resize(len, 0); + } + + &mut buffer[..len] +} + +#[inline(always)] +fn resized_u32_scratch(buffer: &mut Vec, len: usize) -> &mut [u32] { + if buffer.len() < len { + buffer.resize(len, 0); + } + + &mut buffer[..len] +} + +pub(crate) fn coefficient_to_i32(value: u32, k_max: u8) -> i32 { + let shift = 31_u32.saturating_sub(k_max as u32); + let magnitude = ((value & 0x7FFF_FFFF) >> shift) as i32; + + if (value & 0x8000_0000) != 0 { + -magnitude + } else { + magnitude + } +} + +pub(crate) fn decode_with_stats( + code_block: &CodeBlock, + total_bitplanes: u8, + stripe_causal: bool, + ctx: &mut HtBlockDecodeContext, + storage: &DecompositionStorage<'_>, + strict: bool, + stats: Option<&mut HtBlockDecodeStats>, + profile_enabled: bool, +) -> Result<()> { + ctx.reset(code_block); + + if total_bitplanes == 0 { + return Ok(()); + } + + if total_bitplanes > 31 { + bail!(DecodingError::TooManyBitplanes); + } + + let actual_bitplanes = if strict { + total_bitplanes + .checked_sub(code_block.missing_bit_planes) + .ok_or(DecodingError::InvalidBitplaneCount)? + } else { + total_bitplanes.saturating_sub(code_block.missing_bit_planes) + }; + + let max_coding_passes = if actual_bitplanes == 0 { + 0 + } else { + 1 + 3 * (actual_bitplanes - 1) + }; + + if code_block.number_of_coding_passes > max_coding_passes && strict { + bail!(DecodingError::TooManyCodingPasses); + } + + if code_block.number_of_coding_passes == 0 || actual_bitplanes == 0 { + return Ok(()); + } + + let segments = collect_code_block_segments(code_block, storage)?; + decode_segments_validated_with_scratch_for_phase::( + &segments, + code_block.missing_bit_planes, + total_bitplanes, + code_block.number_of_coding_passes, + stripe_causal, + strict, + &mut ctx.coefficients, + code_block.rect.width(), + code_block.rect.height(), + code_block.rect.width(), + &mut ctx.scratch, + stats, + profile_enabled, + ) +} + +pub(crate) struct CombinedCodeBlockData { + pub(crate) data: Vec, + pub(crate) cleanup_length: u32, + pub(crate) refinement_length: u32, +} + +pub(crate) struct HtCodeBlockSegments<'a> { + pub(crate) cleanup: &'a [u8], + pub(crate) refinement: &'a [u8], +} + +impl<'a> HtCodeBlockSegments<'a> { + pub(crate) fn from_combined_payload( + data: &'a [u8], + cleanup_length: u32, + refinement_length: u32, + ) -> Result { + let cleanup_len = cleanup_length as usize; + let refinement_len = refinement_length as usize; + let total_len = cleanup_len + .checked_add(refinement_len) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if data.len() < total_len { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + Ok(Self { + cleanup: &data[..cleanup_len], + refinement: &data[cleanup_len..total_len], + }) + } +} + +pub(crate) struct HtSigPropBenchmarkState { + refinement_data: Vec, + sigma: Vec, + prev_row_sig: Vec, + width: u32, + height: u32, + stride: u32, + mstr: usize, + stripe_causal: bool, + p: u32, +} + +impl HtSigPropBenchmarkState { + pub(crate) fn output_len(&self) -> usize { + if self.height == 0 { + 0 + } else { + (self.stride as usize * (self.height as usize - 1)) + self.width as usize + } + } +} + +pub(crate) fn prepare_sigprop_benchmark_state( + segments: &HtCodeBlockSegments<'_>, + missing_bit_planes: u8, + total_bitplanes: u8, + number_of_coding_passes: u8, + stripe_causal: bool, + strict: bool, + width: u32, + height: u32, + stride: u32, +) -> Result { + if !validate_combined_decode( + missing_bit_planes, + total_bitplanes, + number_of_coding_passes, + strict, + )? { + bail!(DecodingError::CodeBlockDecodeFailure); + } + if number_of_coding_passes < 2 || segments.refinement.is_empty() || missing_bit_planes > 28 { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + let lcup = segments.cleanup.len(); + let scup = cleanup_segment_suffix_length(segments.cleanup, lcup) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let sstr = cleanup_symbol_stride(width); + let quad_rows = height.div_ceil(2) as usize; + let mut cleanup = vec![0u16; sstr * (quad_rows + 1)]; + decode_cleanup_symbols( + segments.cleanup, + lcup, + scup, + width, + height, + sstr, + &mut cleanup, + ) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + + let mstr = sigma_stride(width); + let sigma_rows = height.div_ceil(4) as usize + 1; + let mut sigma = vec![0u16; sigma_rows * mstr]; + build_sigma_from_cleanup_phase(&cleanup, &mut sigma, width, height, sstr, mstr) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + + Ok(HtSigPropBenchmarkState { + refinement_data: segments.refinement.to_vec(), + sigma, + prev_row_sig: vec![0u16; width.div_ceil(4) as usize + 8], + width, + height, + stride, + mstr, + stripe_causal, + p: 30 - u32::from(missing_bit_planes), + }) +} + +pub(crate) fn decode_sigprop_benchmark_state( + state: &mut HtSigPropBenchmarkState, + decoded_data: &mut [u32], +) -> Result<()> { + if decoded_data.len() < state.output_len() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + apply_significance_propagation_phase( + &state.refinement_data, + &state.sigma, + decoded_data, + state.width, + state.height, + state.stride, + state.mstr, + state.stripe_causal, + state.p, + &mut state.prev_row_sig, + ) + .ok_or(DecodingError::CodeBlockDecodeFailure.into()) +} + +#[cfg(test)] +impl CombinedCodeBlockData { + pub(crate) fn segments(&self) -> Result> { + HtCodeBlockSegments::from_combined_payload( + &self.data, + self.cleanup_length, + self.refinement_length, + ) + } +} + +pub(crate) fn collect_code_block_segments<'a>( + code_block: &CodeBlock, + storage: &'a DecompositionStorage<'a>, +) -> Result> { + let mut cleanup = None; + let mut refinement = None; + + for layer in &storage.layers[code_block.layers.start..code_block.layers.end] { + let Some(range) = layer.segments.clone() else { + continue; + }; + + for segment in &storage.segments[range] { + match segment.idx { + 0 if cleanup.is_none() => { + cleanup = Some(segment.data); + } + 1 if refinement.is_none() => { + refinement = Some(segment.data); + } + _ => bail!(DecodingError::UnsupportedFeature( + "unexpected HTJ2K segment layout" + )), + } + } + } + + let Some(cleanup) = cleanup else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + + Ok(HtCodeBlockSegments { + cleanup, + refinement: refinement.unwrap_or(&[]), + }) +} + +pub(crate) fn collect_code_block_data<'a>( + code_block: &CodeBlock, + storage: &'a DecompositionStorage<'a>, +) -> Result { + let segments = collect_code_block_segments(code_block, storage)?; + let cleanup_length = + u32::try_from(segments.cleanup.len()).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + let refinement_length = u32::try_from(segments.refinement.len()) + .map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + let mut data = Vec::with_capacity(segments.cleanup.len() + segments.refinement.len()); + data.extend_from_slice(segments.cleanup); + data.extend_from_slice(segments.refinement); + + Ok(CombinedCodeBlockData { + data, + cleanup_length, + refinement_length, + }) +} + +#[inline(always)] +fn decode_segments_with_scratch_for_phase( + segments: &HtCodeBlockSegments<'_>, + missing_bit_planes: u8, + number_of_coding_passes: u8, + width: u32, + height: u32, + stride: u32, + stripe_causal: bool, + decoded_data: &mut [u32], + scratch: &mut HtBlockDecodeScratch, + stats: Option<&mut HtBlockDecodeStats>, + profile_enabled: bool, +) -> Result<()> { + let decoded = if let Some(stats) = stats { + let mut observer = RecordingHtDecodeStats { + stats, + profile_enabled, + }; + decode_impl::( + segments.cleanup, + segments.refinement, + decoded_data, + missing_bit_planes as u32, + number_of_coding_passes as u32, + width, + height, + stride, + stripe_causal, + scratch, + &mut observer, + ) + } else { + let mut observer = NoHtDecodeStats; + decode_impl::( + segments.cleanup, + segments.refinement, + decoded_data, + missing_bit_planes as u32, + number_of_coding_passes as u32, + width, + height, + stride, + stripe_causal, + scratch, + &mut observer, + ) + }; + + decoded.ok_or(DecodingError::CodeBlockDecodeFailure.into()) +} + +#[cfg(test)] +pub(crate) fn decode_segments_validated( + segments: &HtCodeBlockSegments<'_>, + missing_bit_planes: u8, + total_bitplanes: u8, + number_of_coding_passes: u8, + stripe_causal: bool, + strict: bool, + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, +) -> Result<()> { + decode_segments_validated_for_phase::( + segments, + missing_bit_planes, + total_bitplanes, + number_of_coding_passes, + stripe_causal, + strict, + decoded_data, + width, + height, + stride, + ) +} + +#[inline(always)] +#[cfg(test)] +pub(crate) fn decode_segments_validated_for_phase( + segments: &HtCodeBlockSegments<'_>, + missing_bit_planes: u8, + total_bitplanes: u8, + number_of_coding_passes: u8, + stripe_causal: bool, + strict: bool, + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, +) -> Result<()> { + if !validate_combined_decode( + missing_bit_planes, + total_bitplanes, + number_of_coding_passes, + strict, + )? { + return Ok(()); + } + + let mut scratch = HtBlockDecodeScratch::default(); + decode_segments_with_scratch_for_phase::( + segments, + missing_bit_planes, + number_of_coding_passes, + width, + height, + stride, + stripe_causal, + decoded_data, + &mut scratch, + None, + false, + ) +} + +#[cfg(test)] +fn decode_segments_validated_with_scratch( + segments: &HtCodeBlockSegments<'_>, + missing_bit_planes: u8, + total_bitplanes: u8, + number_of_coding_passes: u8, + stripe_causal: bool, + strict: bool, + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, + scratch: &mut HtBlockDecodeScratch, +) -> Result<()> { + decode_segments_validated_with_scratch_for_phase::( + segments, + missing_bit_planes, + total_bitplanes, + number_of_coding_passes, + stripe_causal, + strict, + decoded_data, + width, + height, + stride, + scratch, + None, + false, + ) +} + +#[inline(always)] +pub(crate) fn decode_segments_validated_with_scratch_for_phase( + segments: &HtCodeBlockSegments<'_>, + missing_bit_planes: u8, + total_bitplanes: u8, + number_of_coding_passes: u8, + stripe_causal: bool, + strict: bool, + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, + scratch: &mut HtBlockDecodeScratch, + stats: Option<&mut HtBlockDecodeStats>, + profile_enabled: bool, +) -> Result<()> { + if !validate_combined_decode( + missing_bit_planes, + total_bitplanes, + number_of_coding_passes, + strict, + )? { + return Ok(()); + } + + decode_segments_with_scratch_for_phase::( + segments, + missing_bit_planes, + number_of_coding_passes, + width, + height, + stride, + stripe_causal, + decoded_data, + scratch, + stats, + profile_enabled, + ) +} + +fn validate_combined_decode( + missing_bit_planes: u8, + total_bitplanes: u8, + number_of_coding_passes: u8, + strict: bool, +) -> Result { + if total_bitplanes == 0 { + return Ok(false); + } + + if total_bitplanes > 31 { + bail!(DecodingError::TooManyBitplanes); + } + + let actual_bitplanes = if strict { + total_bitplanes + .checked_sub(missing_bit_planes) + .ok_or(DecodingError::InvalidBitplaneCount)? + } else { + total_bitplanes.saturating_sub(missing_bit_planes) + }; + + let max_coding_passes = if actual_bitplanes == 0 { + 0 + } else { + 1 + 3 * (actual_bitplanes - 1) + }; + + if number_of_coding_passes > max_coding_passes && strict { + bail!(DecodingError::TooManyCodingPasses); + } + + Ok(number_of_coding_passes != 0 && actual_bitplanes != 0) +} + +#[cfg(test)] +pub(crate) fn decode_combined_validated( + combined: &CombinedCodeBlockData, + missing_bit_planes: u8, + total_bitplanes: u8, + number_of_coding_passes: u8, + stripe_causal: bool, + strict: bool, + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, +) -> Result<()> { + let segments = combined.segments()?; + decode_segments_validated( + &segments, + missing_bit_planes, + total_bitplanes, + number_of_coding_passes, + stripe_causal, + strict, + decoded_data, + width, + height, + stride, + ) +} + +#[cfg(test)] +fn decode_combined_validated_with_scratch( + combined: &CombinedCodeBlockData, + missing_bit_planes: u8, + total_bitplanes: u8, + number_of_coding_passes: u8, + stripe_causal: bool, + strict: bool, + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, + scratch: &mut HtBlockDecodeScratch, +) -> Result<()> { + let segments = combined.segments()?; + decode_segments_validated_with_scratch( + &segments, + missing_bit_planes, + total_bitplanes, + number_of_coding_passes, + stripe_causal, + strict, + decoded_data, + width, + height, + stride, + scratch, + ) +} + +struct MelDecoder<'a> { + data: &'a [u8], + pos: usize, + remaining: usize, + unstuff: bool, + current_byte: u8, + bits_left: u8, + k: usize, + num_runs: usize, + runs: u64, +} + +impl<'a> MelDecoder<'a> { + fn new(data: &'a [u8], lcup: usize, scup: usize) -> Self { + Self { + data, + pos: lcup - scup, + remaining: scup - 1, + unstuff: false, + current_byte: 0, + bits_left: 0, + k: 0, + num_runs: 0, + runs: 0, + } + } + + fn read_bit(&mut self) -> Option { + if self.bits_left == 0 { + let mut byte = if self.remaining > 0 { + let byte = self.data.get(self.pos).copied()?; + self.pos += 1; + self.remaining -= 1; + byte + } else { + 0xFF + }; + + if self.remaining == 0 { + byte |= 0x0F; + } + + self.current_byte = byte; + self.bits_left = 8 - u8::from(self.unstuff); + self.unstuff = byte == 0xFF; + } + + self.bits_left -= 1; + Some(((self.current_byte >> self.bits_left) & 1) as u32) + } + + fn read_bits(&mut self, count: usize) -> Option { + let mut value = 0; + + for _ in 0..count { + value = (value << 1) | self.read_bit()?; + } + + Some(value) + } + + fn decode_more_runs(&mut self) -> Option<()> { + const MEL_EXP: [usize; 13] = [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5]; + + while self.num_runs < 8 { + let eval = MEL_EXP[self.k]; + let first = self.read_bit()?; + let run = if first == 1 { + self.k = (self.k + 1).min(12); + ((1usize << eval) - 1) << 1 + } else { + self.k = self.k.saturating_sub(1); + (self.read_bits(eval)? as usize) << 1 | 1 + }; + + self.runs |= (run as u64) << (self.num_runs * 7); + self.num_runs += 1; + + if eval == 5 && first == 0 && self.num_runs >= 8 { + break; + } + } + + Some(()) + } + + fn get_run(&mut self) -> Option { + if self.num_runs == 0 { + self.decode_more_runs()?; + } + + let run = (self.runs & 0x7F) as i32; + self.runs >>= 7; + self.num_runs -= 1; + Some(run) + } +} + +struct ForwardBitReader<'a, const PAD: u8> { + data: &'a [u8], + pos: usize, + tmp: u64, + bits: u32, + unstuff: bool, +} + +impl<'a, const PAD: u8> ForwardBitReader<'a, PAD> { + fn new(data: &'a [u8]) -> Self { + Self { + data, + pos: 0, + tmp: 0, + bits: 0, + unstuff: false, + } + } + + fn fill(&mut self) { + while self.bits <= 32 { + let byte = if self.pos < self.data.len() { + let byte = self.data[self.pos]; + self.pos += 1; + byte + } else { + PAD + }; + + self.tmp |= (byte as u64) << self.bits; + self.bits += 8 - u32::from(self.unstuff); + self.unstuff = byte == 0xFF; + } + } + + fn fetch(&mut self) -> u32 { + if self.bits < 32 { + self.fill(); + } + + self.tmp as u32 + } + + fn advance(&mut self, count: u32) { + debug_assert!(count <= self.bits); + self.tmp >>= count; + self.bits -= count; + } +} + +struct ReverseBitReader<'a> { + data: &'a [u8], + pos: isize, + remaining: usize, + tmp: u64, + bits: u32, + unstuff: bool, +} + +impl<'a> ReverseBitReader<'a> { + fn new_vlc(data: &'a [u8], lcup: usize, scup: usize) -> Self { + let d = data[lcup - 2]; + let tmp = u64::from(d >> 4); + let bits = 4 - u32::from((tmp & 0x7) == 0x7); + + Self { + data, + pos: lcup as isize - 3, + remaining: scup - 2, + tmp, + bits, + unstuff: (d | 0x0F) > 0x8F, + } + } + + fn new_mrp(data: &'a [u8]) -> Self { + Self { + data, + pos: data.len() as isize - 1, + remaining: data.len(), + tmp: 0, + bits: 0, + unstuff: true, + } + } + + fn fill(&mut self) { + while self.bits <= 32 { + let byte = if self.remaining > 0 { + let byte = self.data[self.pos as usize]; + self.pos -= 1; + self.remaining -= 1; + byte + } else { + 0 + }; + + let d_bits = 8 - u32::from(self.unstuff && (byte & 0x7F) == 0x7F); + self.tmp |= (byte as u64) << self.bits; + self.bits += d_bits; + self.unstuff = byte > 0x8F; + } + } + + fn fetch(&mut self) -> u32 { + if self.bits < 32 { + self.fill(); + } + + self.tmp as u32 + } + + fn advance(&mut self, count: u32) -> u32 { + debug_assert!(count <= self.bits); + self.tmp >>= count; + self.bits -= count; + self.tmp as u32 + } +} + +#[inline(always)] +fn read_u32_pair(values: &[u16], index: usize) -> u32 { + u32::from(values[index]) | (u32::from(values[index + 1]) << 16) +} + +fn sample_mask(bit: u32) -> u32 { + 1 << (4 + bit) +} + +fn decode_mag_sgn_sample_with_vn( + magsgn: &mut ForwardBitReader<0xFF>, + inf: u32, + bit: u32, + uq: u32, + p: u32, +) -> (u32, u32) { + if (inf & sample_mask(bit)) == 0 { + return (0, 0); + } + + let ms_val = magsgn.fetch(); + let m_n = uq - ((inf >> (12 + bit)) & 1); + magsgn.advance(m_n); + + let mut value = ms_val << 31; + let mask = if m_n == 0 { 0 } else { (1_u32 << m_n) - 1 }; + let mut v_n = ms_val & mask; + v_n |= ((inf >> (8 + bit)) & 1) << m_n; + v_n |= 1; + value |= (v_n + 2) << (p - 1); + (value, v_n) +} + +fn cleanup_symbol_stride(width: u32) -> usize { + ((width + 2 + 7) & !7) as usize +} + +fn sigma_stride(width: u32) -> usize { + ((width.div_ceil(4) + 2 + 7) & !7) as usize +} + +fn cleanup_segment_suffix_length(coded_data: &[u8], lcup: usize) -> Option { + if lcup < 2 || coded_data.len() < lcup { + return None; + } + + let scup = ((coded_data[lcup - 1] as usize) << 4) + usize::from(coded_data[lcup - 2] & 0x0F); + if !(2..=lcup).contains(&scup) || scup > 4079 { + return None; + } + + Some(scup) +} + +/// Decodes the first (initial) cleanup quad row into `scratch`, advancing the +/// shared MEL/VLC state; the trailing sentinel pair is written after the row. +#[inline(always)] +fn decode_cleanup_symbols_first_row( + mel: &mut MelDecoder, + vlc: &mut ReverseBitReader, + run: &mut i32, + scratch: &mut [u16], + width: u32, +) -> Option<()> { + let mut c_q = 0u32; + let mut row_offset = 0usize; + let mut x = 0u32; + + while x < width { + let mut vlc_val = vlc.fetch(); + let mut t0 = u32::from(VLC_TABLE0[(c_q + (vlc_val & 0x7F)) as usize]); + if c_q == 0 { + *run -= 2; + t0 = if *run == -1 { t0 } else { 0 }; + if *run < 0 { + *run = mel.get_run()?; + } + } + scratch[row_offset] = t0 as u16; + x += 2; + c_q = ((t0 & 0x10) << 3) | ((t0 & 0xE0) << 2); + vlc_val = vlc.advance(t0 & 0x7); + + let mut t1 = u32::from(VLC_TABLE0[(c_q + (vlc_val & 0x7F)) as usize]); + if c_q == 0 && x < width { + *run -= 2; + t1 = if *run == -1 { t1 } else { 0 }; + if *run < 0 { + *run = mel.get_run()?; + } + } + if x >= width { + t1 = 0; + } + scratch[row_offset + 2] = t1 as u16; + x += 2; + c_q = ((t1 & 0x10) << 3) | ((t1 & 0xE0) << 2); + vlc_val = vlc.advance(t1 & 0x7); + + let mut uvlc_mode = ((t0 & 0x8) << 3) | ((t1 & 0x8) << 4); + if uvlc_mode == 0xC0 { + *run -= 2; + if *run == -1 { + uvlc_mode += 0x40; + } + if *run < 0 { + *run = mel.get_run()?; + } + } + + let mut uvlc_entry = u32::from(UVLC_TABLE0[(uvlc_mode + (vlc_val & 0x3F)) as usize]); + vlc_val = vlc.advance(uvlc_entry & 0x7); + uvlc_entry >>= 3; + let mut len = uvlc_entry & 0xF; + let tmp = vlc_val & ((1_u32 << len) - 1); + let _ = vlc.advance(len); + uvlc_entry >>= 4; + len = uvlc_entry & 0x7; + uvlc_entry >>= 3; + scratch[row_offset + 1] = (1 + (uvlc_entry & 0x7) + (tmp & !(0xFF_u32 << len))) as u16; + scratch[row_offset + 3] = (1 + (uvlc_entry >> 3) + (tmp >> len)) as u16; + + row_offset += 4; + } + scratch[row_offset] = 0; + scratch[row_offset + 1] = 0; + + Some(()) +} + +/// Decodes one non-initial cleanup quad row (`y >= 2`, even) into `scratch`, +/// reading the previous quad row's context and advancing the shared MEL/VLC +/// state; the trailing sentinel pair is written after the row. +#[inline(always)] +fn decode_cleanup_symbols_row( + mel: &mut MelDecoder, + vlc: &mut ReverseBitReader, + run: &mut i32, + scratch: &mut [u16], + width: u32, + y: u32, + sstr: usize, +) -> Option<()> { + let row_base = (y >> 1) as usize * sstr; + let prev_base = row_base - sstr; + let mut x = 0u32; + let mut c_q = 0u32; + let mut row_offset = row_base; + + while x < width { + c_q |= (u32::from(scratch[prev_base + (row_offset - row_base)]) & 0xA0) << 2; + c_q |= (u32::from(scratch[prev_base + (row_offset - row_base) + 2]) & 0x20) << 4; + + let mut vlc_val = vlc.fetch(); + let mut t0 = u32::from(VLC_TABLE1[(c_q + (vlc_val & 0x7F)) as usize]); + if c_q == 0 { + *run -= 2; + t0 = if *run == -1 { t0 } else { 0 }; + if *run < 0 { + *run = mel.get_run()?; + } + } + scratch[row_offset] = t0 as u16; + x += 2; + + c_q = ((t0 & 0x40) << 2) | ((t0 & 0x80) << 1); + c_q |= u32::from(scratch[prev_base + (row_offset - row_base)]) & 0x80; + c_q |= (u32::from(scratch[prev_base + (row_offset - row_base) + 2]) & 0xA0) << 2; + c_q |= (u32::from(scratch[prev_base + (row_offset - row_base) + 4]) & 0x20) << 4; + vlc_val = vlc.advance(t0 & 0x7); + + let mut t1 = u32::from(VLC_TABLE1[(c_q + (vlc_val & 0x7F)) as usize]); + if c_q == 0 && x < width { + *run -= 2; + t1 = if *run == -1 { t1 } else { 0 }; + if *run < 0 { + *run = mel.get_run()?; + } + } + if x >= width { + t1 = 0; + } + scratch[row_offset + 2] = t1 as u16; + x += 2; + + c_q = ((t1 & 0x40) << 2) | ((t1 & 0x80) << 1); + c_q |= u32::from(scratch[prev_base + (row_offset - row_base) + 2]) & 0x80; + vlc_val = vlc.advance(t1 & 0x7); + + let uvlc_mode = ((t0 & 0x8) << 3) | ((t1 & 0x8) << 4); + let mut uvlc_entry = u32::from(UVLC_TABLE1[(uvlc_mode + (vlc_val & 0x3F)) as usize]); + vlc_val = vlc.advance(uvlc_entry & 0x7); + uvlc_entry >>= 3; + let mut len = uvlc_entry & 0xF; + let tmp = vlc_val & ((1_u32 << len) - 1); + let _ = vlc.advance(len); + uvlc_entry >>= 4; + len = uvlc_entry & 0x7; + uvlc_entry >>= 3; + scratch[row_offset + 1] = ((uvlc_entry & 0x7) + (tmp & !(0xFF_u32 << len))) as u16; + scratch[row_offset + 3] = ((uvlc_entry >> 3) + (tmp >> len)) as u16; + + row_offset += 4; + } + + scratch[row_offset] = 0; + scratch[row_offset + 1] = 0; + + Some(()) +} + +#[inline(never)] +fn decode_cleanup_symbols( + coded_data: &[u8], + lcup: usize, + scup: usize, + width: u32, + height: u32, + sstr: usize, + scratch: &mut [u16], +) -> Option<()> { + let quad_rows = height.div_ceil(2) as usize; + if scratch.len() < sstr * (quad_rows + 1) { + return None; + } + + let mut mel = MelDecoder::new(coded_data, lcup, scup); + let mut vlc = ReverseBitReader::new_vlc(coded_data, lcup, scup); + let mut run = mel.get_run()?; + + decode_cleanup_symbols_first_row(&mut mel, &mut vlc, &mut run, scratch, width)?; + + for y in (2..height).step_by(2) { + decode_cleanup_symbols_row(&mut mel, &mut vlc, &mut run, scratch, width, y, sstr)?; + } + + Some(()) +} + +#[inline(always)] +fn build_sigma_from_cleanup_phase( + cleanup: &[u16], + sigma: &mut [u16], + width: u32, + height: u32, + sstr: usize, + mstr: usize, +) -> Option<()> { + let sigma_rows = height.div_ceil(4) as usize + 1; + if sigma.len() < sigma_rows * mstr { + return None; + } + + let mut y = 0u32; + while y < height { + let sp_base = (y >> 1) as usize * sstr; + let dp_base = (y >> 2) as usize * mstr; + let mut x = 0u32; + let mut sp = sp_base; + let mut dp = dp_base; + while x < width { + let mut t0 = + ((u32::from(cleanup[sp]) & 0x30) >> 4) | ((u32::from(cleanup[sp]) & 0xC0) >> 2); + t0 |= ((u32::from(cleanup[sp + 2]) & 0x30) << 4) + | ((u32::from(cleanup[sp + 2]) & 0xC0) << 6); + let mut t1 = ((u32::from(cleanup[sp + sstr]) & 0x30) >> 2) + | (u32::from(cleanup[sp + sstr]) & 0xC0); + t1 |= ((u32::from(cleanup[sp + sstr + 2]) & 0x30) << 6) + | ((u32::from(cleanup[sp + sstr + 2]) & 0xC0) << 8); + sigma[dp] = (t0 | t1) as u16; + x += 4; + sp += 4; + dp += 1; + } + sigma[dp] = 0; + y += 4; + } + + let dp_base = (height.div_ceil(4) as usize) * mstr; + for x in 0..=width.div_ceil(4) as usize { + sigma[dp_base + x] = 0; + } + + Some(()) +} + +#[inline(always)] +fn apply_significance_propagation_phase( + refinement_data: &[u8], + sigma: &[u16], + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, + mstr: usize, + stripe_causal: bool, + p: u32, + prev_row_sig: &mut [u16], +) -> Option<()> { + if prev_row_sig.len() < width.div_ceil(4) as usize + 8 { + return None; + } + + prev_row_sig.fill(0); + let mut sigprop = ForwardBitReader::<0>::new(refinement_data); + let stride_us = stride as usize; + + for y in (0..height).step_by(4) { + let mut pattern = 0xFFFFu32; + if height - y < 4 { + pattern = 0x7777; + if height - y < 3 { + pattern = 0x3333; + if height - y < 2 { + pattern = 0x1111; + } + } + } + + let mut prev = 0u32; + let cur_row = (y >> 2) as usize * mstr; + let next_row = cur_row + mstr; + let dpp = (y * stride) as usize; + + for x in (0..width).step_by(4) { + let mut col_pattern = pattern; + let mut s = x as i32 + 4 - width as i32; + s = s.max(0); + col_pattern >>= (s * 4) as u32; + + let idx = (x >> 2) as usize; + let ps = u32::from(prev_row_sig[idx]) | (u32::from(prev_row_sig[idx + 1]) << 16); + let ns = read_u32_pair(sigma, next_row + idx); + let mut u = (ps & 0x8888_8888) >> 3; + if !stripe_causal { + u |= (ns & 0x1111_1111) << 3; + } + + let cs = read_u32_pair(sigma, cur_row + idx); + let mut mbr = cs; + mbr |= (cs & 0x7777_7777) << 1; + mbr |= (cs & 0xEEEE_EEEE) >> 1; + mbr |= u; + let t = mbr; + mbr |= t << 4; + mbr |= t >> 4; + mbr |= prev >> 12; + mbr &= col_pattern; + mbr &= !cs; + + let mut new_sig = 0u32; + if mbr != 0 { + let mut cwd = sigprop.fetch(); + let mut cnt = 0u32; + let inv_sig = !cs & col_pattern; + let mut candidates = mbr; + let mut processed = 0u32; + + while candidates != 0 { + let bit = candidates.trailing_zeros(); + let sample_mask = 1u32 << bit; + candidates &= !sample_mask; + processed |= sample_mask; + + if (cwd & 1) != 0 { + new_sig |= sample_mask; + candidates |= SIGPROP_SPREAD_MASKS[bit as usize] & inv_sig & !processed; + } + cwd >>= 1; + cnt += 1; + } + + if new_sig != 0 { + let value = 3u32 << (p - 2); + let block_base = dpp + x as usize; + let mut sign_bits = new_sig; + + while sign_bits != 0 { + let bit = sign_bits.trailing_zeros(); + let sample_mask = 1u32 << bit; + sign_bits &= !sample_mask; + + let offset = (bit >> 2) as usize + ((bit & 3) as usize * stride_us); + decoded_data[block_base + offset] = ((cwd & 1) << 31) | value; + cwd >>= 1; + cnt += 1; + } + } + + sigprop.advance(cnt); + } + + let combined_sig = new_sig | cs; + prev_row_sig[idx] = combined_sig as u16; + if idx + 1 < prev_row_sig.len() { + prev_row_sig[idx + 1] = (combined_sig >> 16) as u16; + } + + let t = combined_sig; + let mut next_prev = combined_sig; + next_prev |= (t & 0x7777) << 1; + next_prev |= (t & 0xEEEE) >> 1; + prev = (next_prev | u) & 0xF000; + } + } + + Some(()) +} + +#[inline(always)] +fn apply_magnitude_refinement_phase( + refinement_data: &[u8], + sigma: &[u16], + decoded_data: &mut [u32], + width: u32, + height: u32, + stride: u32, + mstr: usize, + p: u32, +) -> Option<()> { + if p < 2 { + return None; + } + + let mut magref = ReverseBitReader::new_mrp(refinement_data); + let half = 1u32 << (p - 2); + + for y in (0..height).step_by(4) { + let mut cur_sig_idx = (y >> 2) as usize * mstr; + let dpp = (y * stride) as usize; + + for i in (0..width).step_by(8) { + let cwd = magref.fetch(); + let sig = read_u32_pair(sigma, cur_sig_idx); + cur_sig_idx += 2; + let mut col_mask = 0xFu32; + let mut cwd_mut = cwd; + + if sig != 0 { + for j in 0..8 { + if (sig & col_mask) != 0 { + let mut dp = dpp + i as usize + j; + let mut sample_mask = 0x1111_1111u32 & col_mask; + + for _ in 0..4 { + if (sig & sample_mask) != 0 { + let mut sym = cwd_mut & 1; + sym = (1 - sym) << (p - 1); + sym |= half; + decoded_data[dp] ^= sym; + cwd_mut >>= 1; + } + sample_mask <<= 1; + dp += stride as usize; + } + } + col_mask <<= 4; + } + } + + magref.advance(sig.count_ones()); + } + } + + Some(()) +} + +#[inline(never)] +fn decode_magnitude_sign_phase( + coded_data: &[u8], + lcup: usize, + scup: usize, + scratch: &[u16], + decoded_data: &mut [u32], + missing_msbs: u32, + width: u32, + height: u32, + stride: u32, + sstr: usize, + v_n_scratch: &mut [u32], +) -> Option<()> { + let v_n_width = width.div_ceil(2) as usize + 2; + if v_n_scratch.len() < v_n_width { + return None; + } + v_n_scratch[..v_n_width].fill(0); + + let mut magsgn = ForwardBitReader::<0xFF>::new(&coded_data[..lcup - scup]); + + decode_magnitude_sign_first_row_from_cleanup( + &mut magsgn, + scratch, + decoded_data, + v_n_scratch, + missing_msbs, + width, + height, + stride, + )?; + + for y in (2..height).step_by(2) { + decode_magnitude_sign_row_from_cleanup( + &mut magsgn, + scratch, + decoded_data, + v_n_scratch, + missing_msbs, + width, + height, + y, + stride, + sstr, + )?; + } + + Some(()) +} + +#[inline(always)] +fn decode_magnitude_sign_pair_from_cleanup( + magsgn: &mut ForwardBitReader<0xFF>, + decoded_data: &mut [u32], + v_n_scratch: &mut [u32], + inf: u32, + uq: u32, + mmsbp2: u32, + p: u32, + width: u32, + stride: usize, + second_row_present: bool, + x: &mut u32, + dp: &mut usize, + vp: &mut usize, + prev_v_n: &mut u32, +) -> Option<()> { + if uq > mmsbp2 { + return None; + } + + let (val0, _) = decode_mag_sgn_sample_with_vn(magsgn, inf, 0, uq, p); + decoded_data[*dp] = val0; + + let (val1, v_n1) = decode_mag_sgn_sample_with_vn(magsgn, inf, 1, uq, p); + if second_row_present { + decoded_data[*dp + stride] = val1; + } + v_n_scratch[*vp] = *prev_v_n | v_n1; + *prev_v_n = 0; + *dp += 1; + *x += 1; + + if *x >= width { + *vp += 1; + return Some(()); + } + + let (val2, _) = decode_mag_sgn_sample_with_vn(magsgn, inf, 2, uq, p); + decoded_data[*dp] = val2; + + let (val3, v_n3) = decode_mag_sgn_sample_with_vn(magsgn, inf, 3, uq, p); + if second_row_present { + decoded_data[*dp + stride] = val3; + } + *prev_v_n = v_n3; + *dp += 1; + *x += 1; + *vp += 1; + + Some(()) +} + +#[inline(always)] +fn decode_magnitude_sign_first_row_from_cleanup( + magsgn: &mut ForwardBitReader<0xFF>, + cleanup: &[u16], + decoded_data: &mut [u32], + v_n_scratch: &mut [u32], + missing_msbs: u32, + width: u32, + height: u32, + stride: u32, +) -> Option<()> { + let p = 30 - missing_msbs; + let mmsbp2 = missing_msbs + 2; + let stride = stride as usize; + let second_row_present = height > 1; + let mut prev_v_n = 0u32; + let mut x = 0u32; + let mut sp = 0usize; + let mut vp = 0usize; + let mut dp = 0usize; + + while x < width { + let inf = u32::from(cleanup[sp]); + let uq = u32::from(cleanup[sp + 1]); + decode_magnitude_sign_pair_from_cleanup( + magsgn, + decoded_data, + v_n_scratch, + inf, + uq, + mmsbp2, + p, + width, + stride, + second_row_present, + &mut x, + &mut dp, + &mut vp, + &mut prev_v_n, + )?; + sp += 2; + } + v_n_scratch[vp] = prev_v_n; + + Some(()) +} + +#[inline(always)] +fn decode_magnitude_sign_row_from_cleanup( + magsgn: &mut ForwardBitReader<0xFF>, + cleanup: &[u16], + decoded_data: &mut [u32], + v_n_scratch: &mut [u32], + missing_msbs: u32, + width: u32, + height: u32, + y: u32, + stride: u32, + sstr: usize, +) -> Option<()> { + let p = 30 - missing_msbs; + let mmsbp2 = missing_msbs + 2; + let row_base = (y >> 1) as usize * sstr; + let stride = stride as usize; + let mut sp = row_base; + let mut vp = 0usize; + let mut dp = y as usize * stride; + let mut prev_v_n = 0u32; + let mut x = 0u32; + let second_row_present = y + 1 < height; + + while x < width { + let inf = u32::from(cleanup[sp]); + let u_q = u32::from(cleanup[sp + 1]); + let mut gamma = inf & 0xF0; + gamma &= gamma.wrapping_sub(0x10); + let mut emax = v_n_scratch[vp] | v_n_scratch[vp + 1]; + emax = 31 - (emax | 2).leading_zeros(); + let kappa = if gamma != 0 { emax } else { 1 }; + let uq = u_q + kappa; + + decode_magnitude_sign_pair_from_cleanup( + magsgn, + decoded_data, + v_n_scratch, + inf, + uq, + mmsbp2, + p, + width, + stride, + second_row_present, + &mut x, + &mut dp, + &mut vp, + &mut prev_v_n, + )?; + sp += 2; + } + + v_n_scratch[vp] = prev_v_n; + + Some(()) +} + +#[inline(never)] +fn decode_cleanup_and_magnitude_sign_phase( + coded_data: &[u8], + lcup: usize, + scup: usize, + decoded_data: &mut [u32], + missing_msbs: u32, + width: u32, + height: u32, + stride: u32, + sstr: usize, + scratch: &mut [u16], + v_n_scratch: &mut [u32], + observer: &mut impl HtDecodeObserver, +) -> Option<()> { + let quad_rows = height.div_ceil(2) as usize; + if scratch.len() < sstr * (quad_rows + 1) { + return None; + } + let v_n_width = width.div_ceil(2) as usize + 2; + if v_n_scratch.len() < v_n_width { + return None; + } + v_n_scratch[..v_n_width].fill(0); + + let mut mel = MelDecoder::new(coded_data, lcup, scup); + let mut vlc = ReverseBitReader::new_vlc(coded_data, lcup, scup); + let mut magsgn = ForwardBitReader::<0xFF>::new(&coded_data[..lcup - scup]); + let mut run = mel.get_run()?; + + let phase_start = observer.phase_start(); + decode_cleanup_symbols_first_row(&mut mel, &mut vlc, &mut run, scratch, width)?; + observer.add_cleanup_us(phase_start); + + let phase_start = observer.phase_start(); + decode_magnitude_sign_first_row_from_cleanup( + &mut magsgn, + scratch, + decoded_data, + v_n_scratch, + missing_msbs, + width, + height, + stride, + )?; + observer.add_mag_sgn_us(phase_start); + + for y in (2..height).step_by(2) { + let phase_start = observer.phase_start(); + decode_cleanup_symbols_row(&mut mel, &mut vlc, &mut run, scratch, width, y, sstr)?; + observer.add_cleanup_us(phase_start); + + let phase_start = observer.phase_start(); + decode_magnitude_sign_row_from_cleanup( + &mut magsgn, + scratch, + decoded_data, + v_n_scratch, + missing_msbs, + width, + height, + y, + stride, + sstr, + )?; + observer.add_mag_sgn_us(phase_start); + } + + Some(()) +} + +#[inline(always)] +fn decode_impl( + cleanup_data: &[u8], + refinement_data: &[u8], + decoded_data: &mut [u32], + missing_msbs: u32, + mut num_passes: u32, + width: u32, + height: u32, + stride: u32, + stripe_causal: bool, + scratch_buffers: &mut HtBlockDecodeScratch, + observer: &mut O, +) -> Option<()> { + observer.record_block(cleanup_data.len(), refinement_data.len()); + + if num_passes > 1 && refinement_data.is_empty() { + num_passes = 1; + } + + if num_passes > 3 || missing_msbs > 30 { + return None; + } + + if missing_msbs == 30 { + return None; + } + + if missing_msbs == 29 && num_passes > 1 { + num_passes = 1; + } + + let p = 30 - missing_msbs; + let lcup = cleanup_data.len(); + + if lcup < 2 { + return None; + } + + let scup = cleanup_segment_suffix_length(cleanup_data, lcup)?; + + let quad_rows = height.div_ceil(2) as usize; + let sstr = cleanup_symbol_stride(width); + let v_n_width = width.div_ceil(2) as usize + 2; + let v_n_scratch = resized_u32_scratch(&mut scratch_buffers.v_n, v_n_width); + let cleanup_only = PHASE_LIMIT == PHASE_LIMIT_CLEANUP || num_passes == 1; + let scratch = if cleanup_only { + resized_u16_scratch(&mut scratch_buffers.cleanup, sstr * (quad_rows + 1)) + } else { + zeroed_u16_scratch(&mut scratch_buffers.cleanup, sstr * (quad_rows + 1)) + }; + + if cleanup_only { + decode_cleanup_and_magnitude_sign_phase( + cleanup_data, + lcup, + scup, + decoded_data, + missing_msbs, + width, + height, + stride, + sstr, + scratch, + v_n_scratch, + observer, + )?; + } else { + let phase_start = observer.phase_start(); + decode_cleanup_symbols(cleanup_data, lcup, scup, width, height, sstr, scratch)?; + observer.add_cleanup_us(phase_start); + + let phase_start = observer.phase_start(); + decode_magnitude_sign_phase( + cleanup_data, + lcup, + scup, + scratch, + decoded_data, + missing_msbs, + width, + height, + stride, + sstr, + v_n_scratch, + )?; + observer.add_mag_sgn_us(phase_start); + } + + if PHASE_LIMIT == PHASE_LIMIT_CLEANUP { + return Some(()); + } + + if num_passes > 1 { + let sigma_rows = height.div_ceil(4) as usize + 1; + let mstr = sigma_stride(width); + let sigma = zeroed_u16_scratch(&mut scratch_buffers.sigma, sigma_rows * mstr); + let phase_start = observer.phase_start(); + build_sigma_from_cleanup_phase(scratch, sigma, width, height, sstr, mstr)?; + observer.add_sigma_us(phase_start); + + let prev_row_sig = resized_u16_scratch( + &mut scratch_buffers.prev_row_sig, + width.div_ceil(4) as usize + 8, + ); + let phase_start = observer.phase_start(); + apply_significance_propagation_phase( + refinement_data, + sigma, + decoded_data, + width, + height, + stride, + mstr, + stripe_causal, + p, + prev_row_sig, + )?; + observer.add_sigprop_us(phase_start); + + if PHASE_LIMIT == PHASE_LIMIT_SIGPROP { + return Some(()); + } + + if num_passes > 2 { + let phase_start = observer.phase_start(); + apply_magnitude_refinement_phase( + refinement_data, + sigma, + decoded_data, + width, + height, + stride, + mstr, + p, + )?; + observer.add_magref_us(phase_start); + } + } + + Some(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::j2c::ht_block_encode::encode_code_block; + + #[test] + fn test_coefficient_to_i32_shifted_alignment() { + let aligned = 3u32 << (31 - 5); + assert_eq!(coefficient_to_i32(aligned, 5), 3); + assert_eq!(coefficient_to_i32(0x8000_0000 | aligned, 5), -3); + } + + #[test] + fn test_direct_ht_block_roundtrip_varied_4x4() { + let original: Vec = (0..16).map(|i| (i * 3) - 20).collect(); + let total_bitplanes = 6u8; + let encoded = encode_code_block(&original, 4, 4, total_bitplanes).expect("encode HT block"); + assert_eq!(encoded.num_coding_passes, 1); + + let mut decoded = vec![0u32; original.len()]; + let mut scratch = HtBlockDecodeScratch::default(); + let mut observer = NoHtDecodeStats; + let decoded_ok = decode_impl::( + &encoded.data, + &[], + &mut decoded, + u32::from(encoded.num_zero_bitplanes), + u32::from(encoded.num_coding_passes), + 4, + 4, + 4, + false, + &mut scratch, + &mut observer, + ); + assert!(decoded_ok.is_some(), "encoded={:02x?}", encoded.data); + + let decoded_i32: Vec = decoded + .into_iter() + .map(|value| coefficient_to_i32(value, total_bitplanes)) + .collect(); + assert_eq!(decoded_i32, original, "encoded={:02x?}", encoded.data); + } + + #[test] + fn test_direct_ht_block_roundtrip_positive_varied_4x4() { + let original: Vec = (0..16).map(|i| i * 3).collect(); + let total_bitplanes = 6u8; + let encoded = encode_code_block(&original, 4, 4, total_bitplanes).expect("encode HT block"); + assert_eq!(encoded.num_coding_passes, 1); + + let mut decoded = vec![0u32; original.len()]; + let mut scratch = HtBlockDecodeScratch::default(); + let mut observer = NoHtDecodeStats; + let decoded_ok = decode_impl::( + &encoded.data, + &[], + &mut decoded, + u32::from(encoded.num_zero_bitplanes), + u32::from(encoded.num_coding_passes), + 4, + 4, + 4, + false, + &mut scratch, + &mut observer, + ); + assert!(decoded_ok.is_some(), "encoded={:02x?}", encoded.data); + + let decoded_i32: Vec = decoded + .into_iter() + .map(|value| coefficient_to_i32(value, total_bitplanes)) + .collect(); + assert_eq!(decoded_i32, original, "encoded={:02x?}", encoded.data); + } + + #[test] + fn cleanup_and_magnitude_sign_phases_decode_odd_sized_block() { + let width = 15u32; + let height = 13u32; + let original: Vec = (0..(width * height)) + .map(|i| { + let value = (i as i32 % 61) - 30; + if i % 7 == 0 { + 0 + } else { + value + } + }) + .collect(); + let total_bitplanes = 6u8; + let encoded = + encode_code_block(&original, width, height, total_bitplanes).expect("encode HT block"); + assert_eq!(encoded.num_coding_passes, 1); + + let lcup = encoded.data.len(); + let scup = cleanup_segment_suffix_length(&encoded.data, lcup).expect("valid cleanup info"); + let sstr = cleanup_symbol_stride(width); + let quad_rows = height.div_ceil(2) as usize; + let mut cleanup = vec![0u16; sstr * (quad_rows + 1)]; + decode_cleanup_symbols(&encoded.data, lcup, scup, width, height, sstr, &mut cleanup) + .expect("decode cleanup symbols"); + + let mut decoded = vec![0u32; original.len()]; + let mut v_n_scratch = vec![0u32; width.div_ceil(2) as usize + 2]; + decode_magnitude_sign_phase( + &encoded.data, + lcup, + scup, + &cleanup, + &mut decoded, + u32::from(encoded.num_zero_bitplanes), + width, + height, + width, + sstr, + &mut v_n_scratch, + ) + .expect("decode magnitude/sign phase"); + + let decoded_i32: Vec = decoded + .into_iter() + .map(|value| coefficient_to_i32(value, total_bitplanes)) + .collect(); + assert_eq!(decoded_i32, original, "encoded={:02x?}", encoded.data); + } + + #[test] + fn sigma_phase_builds_masks_and_zeroes_edge_sentinels() { + let width = 7u32; + let height = 5u32; + let sstr = cleanup_symbol_stride(width); + let mstr = sigma_stride(width); + let sigma_rows = height.div_ceil(4) as usize + 1; + let mut cleanup = vec![0u16; sstr * (height.div_ceil(2) as usize + 1)]; + cleanup[0] = 0x30; + cleanup[2] = 0xC0; + cleanup[sstr] = 0xF0; + cleanup[sstr + 2] = 0x30; + cleanup[2 * sstr] = 0xC0; + cleanup[2 * sstr + 2] = 0xF0; + let mut sigma = vec![0u16; sigma_rows * mstr]; + + build_sigma_from_cleanup_phase(&cleanup, &mut sigma, width, height, sstr, mstr) + .expect("build sigma"); + + let expected_first = (((u32::from(cleanup[0]) & 0x30) >> 4) + | ((u32::from(cleanup[0]) & 0xC0) >> 2) + | ((u32::from(cleanup[2]) & 0x30) << 4) + | ((u32::from(cleanup[2]) & 0xC0) << 6) + | ((u32::from(cleanup[sstr]) & 0x30) >> 2) + | (u32::from(cleanup[sstr]) & 0xC0) + | ((u32::from(cleanup[sstr + 2]) & 0x30) << 6) + | ((u32::from(cleanup[sstr + 2]) & 0xC0) << 8)) as u16; + let expected_second = (((u32::from(cleanup[4]) & 0x30) >> 4) + | ((u32::from(cleanup[4]) & 0xC0) >> 2) + | ((u32::from(cleanup[6]) & 0x30) << 4) + | ((u32::from(cleanup[6]) & 0xC0) << 6) + | ((u32::from(cleanup[sstr + 4]) & 0x30) >> 2) + | (u32::from(cleanup[sstr + 4]) & 0xC0) + | ((u32::from(cleanup[sstr + 6]) & 0x30) << 6) + | ((u32::from(cleanup[sstr + 6]) & 0xC0) << 8)) as u16; + assert_eq!(sigma[0], expected_first); + assert_eq!(sigma[1], expected_second); + assert_eq!(sigma[2], 0); + + let bottom = height.div_ceil(4) as usize * mstr; + for x in 0..=width.div_ceil(4) as usize { + assert_eq!(sigma[bottom + x], 0); + } + } + + #[test] + fn refinement_phases_leave_output_unchanged_for_empty_sigma() { + let width = 7u32; + let height = 5u32; + let stride = width; + let mstr = sigma_stride(width); + let sigma = vec![0u16; (height.div_ceil(4) as usize + 1) * mstr]; + let mut prev_row_sig = vec![0u16; width.div_ceil(4) as usize + 8]; + let mut decoded = vec![0x1234_5678u32; (stride * height) as usize]; + let expected = decoded.clone(); + + apply_significance_propagation_phase( + &[], + &sigma, + &mut decoded, + width, + height, + stride, + mstr, + false, + 5, + &mut prev_row_sig, + ) + .expect("empty sigma sigprop"); + apply_magnitude_refinement_phase(&[], &sigma, &mut decoded, width, height, stride, mstr, 5) + .expect("empty sigma magref"); + + assert_eq!(decoded, expected); + } + + #[test] + fn sigprop_spread_masks_follow_column_major_scan_order() { + let row_patterns = [0x33u32, 0x76, 0xEC, 0xC8]; + + for bit in 0..16 { + let expected = row_patterns[bit & 3] << (bit & !3); + assert_eq!(SIGPROP_SPREAD_MASKS[bit], expected, "bit={bit}"); + assert_eq!(SIGPROP_SPREAD_MASKS[bit] & ((1u32 << bit) - 1), 0); + } + } + + #[test] + fn combined_data_exposes_borrowed_segment_slices() { + let combined = CombinedCodeBlockData { + data: vec![0x11, 0x22, 0x33, 0x44, 0x55], + cleanup_length: 3, + refinement_length: 2, + }; + + let segments = combined.segments().expect("split combined data"); + + assert_eq!(segments.cleanup, &[0x11, 0x22, 0x33]); + assert_eq!(segments.refinement, &[0x44, 0x55]); + } + + #[test] + fn borrowed_segments_decode_matches_owned_combined_decode() { + let width = 16u32; + let height = 16u32; + let original: Vec = (0..(width * height)) + .map(|i| { + let value = (i as i32 % 47) - 23; + if i % 5 == 0 { + 0 + } else { + value + } + }) + .collect(); + let total_bitplanes = 6u8; + let encoded = + encode_code_block(&original, width, height, total_bitplanes).expect("encode HT block"); + let combined = CombinedCodeBlockData { + data: encoded.data.clone(), + cleanup_length: encoded.data.len() as u32, + refinement_length: 0, + }; + let segments = HtCodeBlockSegments { + cleanup: &encoded.data, + refinement: &[], + }; + let mut owned_decoded = vec![0u32; original.len()]; + let mut borrowed_decoded = vec![0u32; original.len()]; + let mut scratch = HtBlockDecodeScratch::default(); + + decode_combined_validated( + &combined, + encoded.num_zero_bitplanes, + total_bitplanes, + encoded.num_coding_passes, + false, + true, + &mut owned_decoded, + width, + height, + width, + ) + .expect("decode owned combined payload"); + decode_segments_validated_with_scratch( + &segments, + encoded.num_zero_bitplanes, + total_bitplanes, + encoded.num_coding_passes, + false, + true, + &mut borrowed_decoded, + width, + height, + width, + &mut scratch, + ) + .expect("decode borrowed payload segments"); + + assert_eq!(borrowed_decoded, owned_decoded); + } + + #[test] + fn scratch_resize_zeroes_existing_values_when_growing() { + let mut scratch = HtBlockDecodeScratch::default(); + + zeroed_u16_scratch(&mut scratch.cleanup, 4).fill(7); + assert_eq!(zeroed_u16_scratch(&mut scratch.cleanup, 8), &[0; 8]); + + zeroed_u32_scratch(&mut scratch.v_n, 4).fill(9); + assert_eq!(zeroed_u32_scratch(&mut scratch.v_n, 8), &[0; 8]); + } + + #[test] + fn decode_combined_validated_with_scratch_reuses_zeroed_buffers() { + let width = 16u32; + let height = 16u32; + let original: Vec = (0..(width * height)) + .map(|i| { + let value = (i as i32 % 47) - 23; + if i % 5 == 0 { + 0 + } else { + value + } + }) + .collect(); + let total_bitplanes = 6u8; + let encoded = + encode_code_block(&original, width, height, total_bitplanes).expect("encode HT block"); + let combined = CombinedCodeBlockData { + data: encoded.data.clone(), + cleanup_length: encoded.data.len() as u32, + refinement_length: 0, + }; + let mut scratch = HtBlockDecodeScratch::default(); + let mut decoded = vec![0u32; original.len()]; + + decode_combined_validated_with_scratch( + &combined, + encoded.num_zero_bitplanes, + total_bitplanes, + encoded.num_coding_passes, + false, + true, + &mut decoded, + width, + height, + width, + &mut scratch, + ) + .expect("decode HT block"); + + let first_capacities = scratch.capacities_for_test(); + assert!(first_capacities.cleanup > 0); + assert!(first_capacities.v_n > 0); + + scratch.poison_for_test(); + decoded.fill(0); + + decode_combined_validated_with_scratch( + &combined, + encoded.num_zero_bitplanes, + total_bitplanes, + encoded.num_coding_passes, + false, + true, + &mut decoded, + width, + height, + width, + &mut scratch, + ) + .expect("decode HT block after scratch poison"); + + assert_eq!(scratch.capacities_for_test(), first_capacities); + let decoded_i32: Vec = decoded + .into_iter() + .map(|value| coefficient_to_i32(value, total_bitplanes)) + .collect(); + assert_eq!(decoded_i32, original, "encoded={:02x?}", encoded.data); + } +} diff --git a/crates/j2k-native/src/j2c/ht_block_encode.rs b/crates/j2k-native/src/j2c/ht_block_encode.rs new file mode 100644 index 00000000..21f1202f --- /dev/null +++ b/crates/j2k-native/src/j2c/ht_block_encode.rs @@ -0,0 +1,1764 @@ +//! Scalar HTJ2K cleanup-only block encoding. + +use alloc::{vec, vec::Vec}; +use core::convert::Infallible; + +use super::bitplane_encode::EncodedCodeBlock; +use super::ht_encode_tables::{ + HtUvlcTableEntry, HT_UVLC_ENCODE_TABLE, HT_VLC_ENCODE_TABLE0, HT_VLC_ENCODE_TABLE1, +}; +use crate::HtCleanupEncodeDistribution; + +const MEL_EXP: [usize; 13] = [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5]; +const MAX_HT_BITPLANES: u8 = 30; +const MEL_SIZE: usize = 192; +const VLC_SIZE: usize = 3072 - MEL_SIZE; +const MS_SIZE: usize = (16384usize * 16).div_ceil(15); + +#[inline(always)] +fn increment_limited_count(counts: &mut [u64; 32], value: i32) { + let index = value.clamp(0, 31) as usize; + counts[index] += 1; +} + +fn record_distribution_initial_quad( + distribution: &mut HtCleanupEncodeDistribution, + rho: i32, + _e_qmax: i32, + _u_q: i32, +) { + let rho_index = (rho & 0xF) as usize; + distribution.total_quads += 1; + distribution.initial_quads += 1; + distribution.rho_counts[rho_index] += 1; + distribution.initial_rho_counts[rho_index] += 1; +} + +fn record_distribution_non_initial_quad( + distribution: &mut HtCleanupEncodeDistribution, + rho: i32, + e_qmax: i32, + kappa: i32, + u_q: i32, +) { + let rho_index = (rho & 0xF) as usize; + let u_q_index = u_q.clamp(0, 31) as usize; + distribution.total_quads += 1; + distribution.non_initial_quads += 1; + distribution.rho_counts[rho_index] += 1; + distribution.non_initial_rho_counts[rho_index] += 1; + increment_limited_count(&mut distribution.non_initial_u_q_counts, u_q); + increment_limited_count(&mut distribution.non_initial_e_qmax_counts, e_qmax); + increment_limited_count(&mut distribution.non_initial_kappa_counts, kappa); + distribution.non_initial_rho_u_q_counts[rho_index][u_q_index] += 1; +} + +fn record_distribution_mag_signs( + distribution: &mut HtCleanupEncodeDistribution, + rho: i32, + u_q: i32, + tuple: u16, +) { + let rho_index = (rho & 0xF) as usize; + let rho_bits = (rho as u32) & 0xF; + if rho_bits == 0 { + return; + } + + let e_k = u32::from(tuple & 0xF); + let u_q = u_q.max(0) as u32; + + distribution.mag_sign_calls += 1; + distribution.mag_sign_rho_counts[rho_index] += 1; + + for bit in 0..4 { + if (rho_bits & (1 << bit)) == 0 { + continue; + } + let reduction = (e_k >> bit) & 1; + let magnitude_bits = u_q.saturating_sub(reduction).min(31) as usize; + distribution.mag_sign_sample_bit_counts[magnitude_bits] += 1; + distribution.mag_sign_encoded_samples += 1; + } +} + +struct MelEncoder { + buffer: Vec, + pos: usize, + remaining_bits: u8, + tmp: u8, + run: usize, + k: usize, + threshold: usize, +} + +impl MelEncoder { + fn new() -> Self { + Self { + buffer: vec![0; MEL_SIZE], + pos: 0, + remaining_bits: 8, + tmp: 0, + run: 0, + k: 0, + threshold: 1, + } + } + + fn emit_bit(&mut self, bit: bool) -> Result<(), &'static str> { + self.tmp = (self.tmp << 1) | u8::from(bit); + self.remaining_bits -= 1; + + if self.remaining_bits == 0 { + if self.pos >= self.buffer.len() { + return Err("HTJ2K MEL encoder buffer is full"); + } + + self.buffer[self.pos] = self.tmp; + self.pos += 1; + self.remaining_bits = if self.tmp == 0xFF { 7 } else { 8 }; + self.tmp = 0; + } + + Ok(()) + } + + fn encode(&mut self, bit: bool) -> Result<(), &'static str> { + if !bit { + self.run += 1; + if self.run >= self.threshold { + self.emit_bit(true)?; + self.run = 0; + self.k = (self.k + 1).min(MEL_EXP.len() - 1); + self.threshold = 1 << MEL_EXP[self.k]; + } + } else { + self.emit_bit(false)?; + let mut t = MEL_EXP[self.k]; + while t > 0 { + t -= 1; + self.emit_bit(((self.run >> t) & 1) != 0)?; + } + self.run = 0; + self.k = self.k.saturating_sub(1); + self.threshold = 1 << MEL_EXP[self.k]; + } + + Ok(()) + } +} + +struct VlcEncoder { + buffer: Vec, + pos: usize, + used_bits: u8, + tmp: u8, + last_greater_than_8f: bool, +} + +impl VlcEncoder { + fn new() -> Self { + let mut buffer = vec![0; VLC_SIZE]; + let last = buffer.len() - 1; + buffer[last] = 0xFF; + + Self { + buffer, + pos: 1, + used_bits: 4, + tmp: 0x0F, + last_greater_than_8f: true, + } + } + + fn encode(&mut self, mut codeword: u32, mut codeword_len: u8) -> Result<(), &'static str> { + while codeword_len > 0 { + if self.pos >= self.buffer.len() { + return Err("HTJ2K VLC encoder buffer is full"); + } + + let mut available_bits = 8 - u8::from(self.last_greater_than_8f) - self.used_bits; + let take = available_bits.min(codeword_len); + let mask = if take == 32 { + u32::MAX + } else { + (1u32 << take) - 1 + }; + self.tmp |= ((codeword & mask) as u8) << self.used_bits; + self.used_bits += take; + available_bits -= take; + codeword_len -= take; + codeword >>= take; + + if available_bits == 0 { + if self.last_greater_than_8f && self.tmp != 0x7F { + self.last_greater_than_8f = false; + continue; + } + + let write_index = self.buffer.len() - 1 - self.pos; + self.buffer[write_index] = self.tmp; + self.pos += 1; + self.last_greater_than_8f = self.tmp > 0x8F; + self.tmp = 0; + self.used_bits = 0; + } + } + + Ok(()) + } +} + +struct MagSgnEncoder { + buffer: Vec, + pos: usize, + max_bits: u8, + used_bits: u8, + tmp: u32, +} + +impl MagSgnEncoder { + fn new() -> Self { + Self { + buffer: vec![0; MS_SIZE], + pos: 0, + max_bits: 8, + used_bits: 0, + tmp: 0, + } + } + + #[inline(always)] + fn encode(&mut self, mut codeword: u32, mut codeword_len: u32) -> Result<(), &'static str> { + while codeword_len > 0 { + if self.pos >= self.buffer.len() { + return Err("HTJ2K magnitude/sign encoder buffer is full"); + } + + let take = u32::from(self.max_bits - self.used_bits).min(codeword_len); + let mask = if take == 32 { + u32::MAX + } else { + (1u32 << take) - 1 + }; + self.tmp |= (codeword & mask) << self.used_bits; + self.used_bits += take as u8; + codeword >>= take; + codeword_len -= take; + + if self.used_bits >= self.max_bits { + self.buffer[self.pos] = self.tmp as u8; + self.pos += 1; + self.max_bits = if self.tmp == 0xFF { 7 } else { 8 }; + self.tmp = 0; + self.used_bits = 0; + } + } + + Ok(()) + } + + fn terminate(&mut self) -> Result<(), &'static str> { + if self.used_bits > 0 { + let unused = self.max_bits - self.used_bits; + self.tmp |= (0xFF & ((1u32 << unused) - 1)) << self.used_bits; + self.used_bits += unused; + + if self.tmp != 0xFF { + if self.pos >= self.buffer.len() { + return Err("HTJ2K magnitude/sign encoder buffer is full"); + } + + self.buffer[self.pos] = self.tmp as u8; + self.pos += 1; + } + } else if self.max_bits == 7 { + self.pos = self.pos.saturating_sub(1); + } + + Ok(()) + } +} + +pub(crate) fn encode_code_block( + coefficients: &[i32], + width: u32, + height: u32, + total_bitplanes: u8, +) -> Result { + if total_bitplanes == 0 || total_bitplanes > MAX_HT_BITPLANES { + return Err("HTJ2K scalar encoder currently supports 1..=30 bitplanes"); + } + + let Some(max_magnitude) = max_nonzero_magnitude(coefficients) else { + return Ok(EncodedCodeBlock { + data: Vec::new(), + num_coding_passes: 0, + num_zero_bitplanes: total_bitplanes, + ht_cleanup_length: 0, + ht_refinement_length: 0, + }); + }; + + let block_bitplanes = (u32::BITS - max_magnitude.leading_zeros()) as u8; + if block_bitplanes > total_bitplanes { + return Err("HTJ2K block magnitude exceeds configured bitplane count"); + } + + let missing_msbs = total_bitplanes.saturating_sub(1); + let data = encode_cleanup_segment_from_coefficients( + coefficients, + missing_msbs, + width as usize, + height as usize, + total_bitplanes, + )?; + let ht_cleanup_length = + u32::try_from(data.len()).map_err(|_| "HTJ2K cleanup segment exceeds u32 length")?; + + Ok(EncodedCodeBlock { + data, + num_coding_passes: 1, + num_zero_bitplanes: missing_msbs, + ht_cleanup_length, + ht_refinement_length: 0, + }) +} + +pub(crate) fn collect_encode_distribution( + coefficients: &[i32], + width: u32, + height: u32, + total_bitplanes: u8, +) -> Result { + if total_bitplanes == 0 || total_bitplanes > MAX_HT_BITPLANES { + return Err("HTJ2K scalar encoder currently supports 1..=30 bitplanes"); + } + + let Some(max_magnitude) = max_nonzero_magnitude(coefficients) else { + return Ok(HtCleanupEncodeDistribution::default()); + }; + + let block_bitplanes = (u32::BITS - max_magnitude.leading_zeros()) as u8; + if block_bitplanes > total_bitplanes { + return Err("HTJ2K block magnitude exceeds configured bitplane count"); + } + + let source = I32CleanupCoefficients { + coefficients, + shift: u32::from(31_u8.saturating_sub(total_bitplanes)), + }; + let mut distribution = HtCleanupEncodeDistribution::default(); + let missing_msbs = total_bitplanes.saturating_sub(1); + collect_encode_distribution_from_source( + &source, + missing_msbs, + width as usize, + height as usize, + &mut distribution, + )?; + Ok(distribution) +} + +#[cfg(test)] +fn convert_nonzero_to_aligned_sign_magnitude_and_max( + coefficients: &[i32], + k_max: u8, +) -> Option<(Vec, u32)> { + let first_nonzero = coefficients + .iter() + .position(|&coefficient| coefficient != 0)?; + let shift = u32::from(31_u8.saturating_sub(k_max)); + let mut aligned = Vec::with_capacity(coefficients.len()); + aligned.resize(first_nonzero, 0); + let mut max_magnitude = 0u32; + + for &coefficient in &coefficients[first_nonzero..] { + let magnitude = coefficient.unsigned_abs(); + max_magnitude = max_magnitude.max(magnitude); + + if magnitude == 0 { + aligned.push(0); + } else { + let sign = if coefficient < 0 { 0x8000_0000 } else { 0 }; + aligned.push(sign | (magnitude << shift)); + } + } + + Some((aligned, max_magnitude)) +} + +fn max_nonzero_magnitude(coefficients: &[i32]) -> Option { + let mut max_magnitude = 0u32; + for &coefficient in coefficients { + max_magnitude = max_magnitude.max(coefficient.unsigned_abs()); + } + (max_magnitude != 0).then_some(max_magnitude) +} + +trait CleanupCoefficientSource { + fn aligned_value(&self, index: usize) -> u32; +} + +impl CleanupCoefficientSource for [u32] { + #[inline(always)] + fn aligned_value(&self, index: usize) -> u32 { + self[index] + } +} + +struct I32CleanupCoefficients<'a> { + coefficients: &'a [i32], + shift: u32, +} + +impl CleanupCoefficientSource for I32CleanupCoefficients<'_> { + #[inline(always)] + fn aligned_value(&self, index: usize) -> u32 { + aligned_sign_magnitude(self.coefficients[index], self.shift) + } +} + +#[inline(always)] +fn aligned_sign_magnitude(coefficient: i32, shift: u32) -> u32 { + let magnitude = coefficient.unsigned_abs(); + if magnitude == 0 { + 0 + } else { + let sign = if coefficient < 0 { 0x8000_0000 } else { 0 }; + sign | (magnitude << shift) + } +} + +fn encode_cleanup_segment_from_coefficients( + coefficients: &[i32], + missing_msbs: u8, + width: usize, + height: usize, + total_bitplanes: u8, +) -> Result, &'static str> { + let source = I32CleanupCoefficients { + coefficients, + shift: u32::from(31_u8.saturating_sub(total_bitplanes)), + }; + encode_cleanup_segment_from_source(&source, missing_msbs, width, height) +} + +#[cfg(test)] +fn encode_cleanup_segment( + coefficients: &[u32], + missing_msbs: u8, + width: usize, + height: usize, +) -> Result, &'static str> { + encode_cleanup_segment_from_source(coefficients, missing_msbs, width, height) +} + +fn encode_cleanup_segment_from_source( + coefficients: &S, + missing_msbs: u8, + width: usize, + height: usize, +) -> Result, &'static str> { + let mut mel = MelEncoder::new(); + let mut vlc = VlcEncoder::new(); + let mut ms = MagSgnEncoder::new(); + + let p = 30_u32.saturating_sub(u32::from(missing_msbs)); + let stride = width; + + let mut e_val = [0u8; 513]; + let mut cx_val = [0u8; 513]; + + let mut e_qmax = [0i32; 2]; + let mut e_q = [0i32; 8]; + let mut rho = [0i32; 2]; + let mut c_q0 = 0usize; + let mut s = [0u32; 8]; + let mut sp = 0usize; + let mut x = 0usize; + + while x < width { + encode_first_quad_pair( + coefficients, + stride, + height, + p, + &mut sp, + x, + &mut e_val, + &mut cx_val, + &mut c_q0, + &mut rho, + &mut e_q, + &mut e_qmax, + &mut s, + &mut mel, + &mut vlc, + &mut ms, + )?; + x += 4; + } + + let e_val_sentinel = width.div_ceil(2) + 1; + e_val[e_val_sentinel] = 0; + + let mut y = 2usize; + while y < height { + let mut lep = 0usize; + let mut max_e = i32::from(e_val[lep].max(e_val[lep + 1])) - 1; + e_val[lep] = 0; + + let mut lcxp = 0usize; + c_q0 = usize::from(cx_val[lcxp]) + (usize::from(cx_val[lcxp + 1]) << 2); + cx_val[lcxp] = 0; + + sp = y * stride; + x = 0; + while x < width { + encode_non_initial_quad_pair( + coefficients, + stride, + width, + height, + y, + p, + &mut sp, + x, + &mut e_val, + &mut cx_val, + &mut lep, + &mut lcxp, + &mut max_e, + &mut c_q0, + &mut rho, + &mut e_q, + &mut e_qmax, + &mut s, + &mut mel, + &mut vlc, + &mut ms, + )?; + x += 4; + } + + y += 2; + } + + terminate_mel_vlc(&mut mel, &mut vlc)?; + ms.terminate()?; + + let total_len = ms.pos + mel.pos + vlc.pos; + if total_len < 2 { + return Err("HTJ2K cleanup segment is too short"); + } + + let mut data = Vec::with_capacity(total_len); + data.extend_from_slice(&ms.buffer[..ms.pos]); + data.extend_from_slice(&mel.buffer[..mel.pos]); + let vlc_start = vlc.buffer.len() - vlc.pos; + data.extend_from_slice(&vlc.buffer[vlc_start..]); + + let locator_bytes = mel.pos + vlc.pos; + let last = data.len() - 1; + let prev = data.len() - 2; + data[last] = (locator_bytes >> 4) as u8; + data[prev] = (data[prev] & 0xF0) | ((locator_bytes as u8) & 0x0F); + + Ok(data) +} + +fn collect_encode_distribution_from_source( + coefficients: &S, + missing_msbs: u8, + width: usize, + height: usize, + distribution: &mut HtCleanupEncodeDistribution, +) -> Result<(), &'static str> { + let p = 30_u32.saturating_sub(u32::from(missing_msbs)); + let stride = width; + + let mut e_val = [0u8; 513]; + let mut cx_val = [0u8; 513]; + + let mut e_qmax = [0i32; 2]; + let mut e_q = [0i32; 8]; + let mut rho = [0i32; 2]; + let mut c_q0 = 0usize; + let mut s = [0u32; 8]; + let mut sp = 0usize; + let mut x = 0usize; + + while x < width { + collect_first_quad_pair( + coefficients, + stride, + height, + p, + &mut sp, + x, + &mut e_val, + &mut cx_val, + &mut c_q0, + &mut rho, + &mut e_q, + &mut e_qmax, + &mut s, + distribution, + ); + x += 4; + } + + let e_val_sentinel = width.div_ceil(2) + 1; + e_val[e_val_sentinel] = 0; + + let mut y = 2usize; + while y < height { + let mut lep = 0usize; + let mut max_e = i32::from(e_val[lep].max(e_val[lep + 1])) - 1; + e_val[lep] = 0; + + let mut lcxp = 0usize; + c_q0 = usize::from(cx_val[lcxp]) + (usize::from(cx_val[lcxp + 1]) << 2); + cx_val[lcxp] = 0; + + sp = y * stride; + x = 0; + while x < width { + collect_non_initial_quad_pair( + coefficients, + stride, + width, + height, + y, + p, + &mut sp, + x, + &mut e_val, + &mut cx_val, + &mut lep, + &mut lcxp, + &mut max_e, + &mut c_q0, + &mut rho, + &mut e_q, + &mut e_qmax, + &mut s, + distribution, + ); + x += 4; + } + + y += 2; + } + + Ok(()) +} + +fn process_sample( + slot: usize, + value: u32, + p: u32, + rho_acc: &mut i32, + e_q: &mut [i32; 8], + e_qmax: &mut i32, + s: &mut [u32; 8], +) { + let mut val = value.wrapping_add(value); + val >>= p; + val &= !1u32; + if val != 0 { + *rho_acc |= 1 << (slot & 0x3); + val -= 1; + e_q[slot] = (u32::BITS - val.leading_zeros()) as i32; + *e_qmax = (*e_qmax).max(e_q[slot]); + val -= 1; + s[slot] = val + (value >> 31); + } +} + +/// Per-quad operations that differ between the byte-emitting encoder and the +/// distribution collector; the quad-pair walking logic itself is shared by +/// `first_quad_pair` / `non_initial_quad_pair`. +trait QuadSink { + type Error; + + #[allow(clippy::too_many_arguments)] + fn quad_initial( + &mut self, + offset: usize, + c_q: usize, + rho: i32, + e_qmax: i32, + e_q: &[i32; 8], + s: &[u32; 8], + lep: usize, + lcxp: usize, + e_val: &mut [u8; 513], + cx_val: &mut [u8; 513], + ) -> Result; + + #[allow(clippy::too_many_arguments)] + fn quad_non_initial( + &mut self, + offset: usize, + c_q: usize, + rho: i32, + e_qmax: i32, + max_e: i32, + e_q: &[i32; 8], + s: &[u32; 8], + ) -> Result; + + fn initial_uvlc_pair(&mut self, u_q0: i32, u_q1: i32) -> Result<(), Self::Error>; + + fn initial_uvlc_lone(&mut self, u_q0: i32) -> Result<(), Self::Error>; + + fn non_initial_uvlc(&mut self, u_q0: i32, u_q1: i32) -> Result<(), Self::Error>; +} + +struct EncodeQuadSink<'a> { + mel: &'a mut MelEncoder, + vlc: &'a mut VlcEncoder, + ms: &'a mut MagSgnEncoder, +} + +impl QuadSink for EncodeQuadSink<'_> { + type Error = &'static str; + + #[inline] + fn quad_initial( + &mut self, + offset: usize, + c_q: usize, + rho: i32, + e_qmax: i32, + e_q: &[i32; 8], + s: &[u32; 8], + lep: usize, + lcxp: usize, + e_val: &mut [u8; 513], + cx_val: &mut [u8; 513], + ) -> Result { + encode_quad_initial_row( + offset, c_q, rho, e_qmax, e_q, s, lep, lcxp, e_val, cx_val, self.mel, self.vlc, self.ms, + ) + } + + #[inline] + fn quad_non_initial( + &mut self, + offset: usize, + c_q: usize, + rho: i32, + e_qmax: i32, + max_e: i32, + e_q: &[i32; 8], + s: &[u32; 8], + ) -> Result { + encode_quad_non_initial_row( + offset, c_q, rho, e_qmax, max_e, e_q, s, self.mel, self.vlc, self.ms, + ) + } + + #[inline] + fn initial_uvlc_pair(&mut self, u_q0: i32, u_q1: i32) -> Result<(), Self::Error> { + if u_q0 > 0 && u_q1 > 0 { + self.mel.encode(u_q0.min(u_q1) > 2)?; + } + encode_uvlc(u_q0, u_q1, self.vlc) + } + + #[inline] + fn initial_uvlc_lone(&mut self, u_q0: i32) -> Result<(), Self::Error> { + encode_uvlc(u_q0, 0, self.vlc) + } + + #[inline] + fn non_initial_uvlc(&mut self, u_q0: i32, u_q1: i32) -> Result<(), Self::Error> { + encode_uvlc_non_initial(u_q0, u_q1, self.vlc) + } +} + +struct CollectQuadSink<'a> { + distribution: &'a mut HtCleanupEncodeDistribution, +} + +impl QuadSink for CollectQuadSink<'_> { + type Error = Infallible; + + #[inline] + fn quad_initial( + &mut self, + offset: usize, + c_q: usize, + rho: i32, + e_qmax: i32, + e_q: &[i32; 8], + _s: &[u32; 8], + lep: usize, + lcxp: usize, + e_val: &mut [u8; 513], + cx_val: &mut [u8; 513], + ) -> Result { + Ok(collect_quad_initial_row( + offset, + c_q, + rho, + e_qmax, + e_q, + lep, + lcxp, + e_val, + cx_val, + self.distribution, + )) + } + + #[inline] + fn quad_non_initial( + &mut self, + offset: usize, + c_q: usize, + rho: i32, + e_qmax: i32, + max_e: i32, + e_q: &[i32; 8], + _s: &[u32; 8], + ) -> Result { + Ok(collect_quad_non_initial_row( + offset, + c_q, + rho, + e_qmax, + max_e, + e_q, + self.distribution, + )) + } + + #[inline] + fn initial_uvlc_pair(&mut self, _u_q0: i32, _u_q1: i32) -> Result<(), Self::Error> { + Ok(()) + } + + #[inline] + fn initial_uvlc_lone(&mut self, _u_q0: i32) -> Result<(), Self::Error> { + Ok(()) + } + + #[inline] + fn non_initial_uvlc(&mut self, _u_q0: i32, _u_q1: i32) -> Result<(), Self::Error> { + Ok(()) + } +} + +#[allow(clippy::too_many_arguments)] +#[allow(clippy::inline_always)] // per-quad-pair hot path: keep each sink monomorphization fused +#[inline(always)] +fn first_quad_pair( + coefficients: &(impl CleanupCoefficientSource + ?Sized), + stride: usize, + height: usize, + p: u32, + sp: &mut usize, + x: usize, + e_val: &mut [u8; 513], + cx_val: &mut [u8; 513], + c_q0: &mut usize, + rho: &mut [i32; 2], + e_q: &mut [i32; 8], + e_qmax: &mut [i32; 2], + s: &mut [u32; 8], + sink: &mut S, +) -> Result<(), S::Error> { + let lep = x / 2; + let lcxp = x / 2; + + process_sample( + 0, + coefficients.aligned_value(*sp), + p, + &mut rho[0], + e_q, + &mut e_qmax[0], + s, + ); + process_sample( + 1, + if height > 1 { + coefficients.aligned_value(*sp + stride) + } else { + 0 + }, + p, + &mut rho[0], + e_q, + &mut e_qmax[0], + s, + ); + *sp += 1; + + if x + 1 < stride { + process_sample( + 2, + coefficients.aligned_value(*sp), + p, + &mut rho[0], + e_q, + &mut e_qmax[0], + s, + ); + process_sample( + 3, + if height > 1 { + coefficients.aligned_value(*sp + stride) + } else { + 0 + }, + p, + &mut rho[0], + e_q, + &mut e_qmax[0], + s, + ); + *sp += 1; + } + + let u_q0 = sink.quad_initial( + 0, *c_q0, rho[0], e_qmax[0], e_q, s, lep, lcxp, e_val, cx_val, + )?; + + if x + 2 < stride { + process_sample( + 4, + coefficients.aligned_value(*sp), + p, + &mut rho[1], + e_q, + &mut e_qmax[1], + s, + ); + process_sample( + 5, + if height > 1 { + coefficients.aligned_value(*sp + stride) + } else { + 0 + }, + p, + &mut rho[1], + e_q, + &mut e_qmax[1], + s, + ); + *sp += 1; + + if x + 3 < stride { + process_sample( + 6, + coefficients.aligned_value(*sp), + p, + &mut rho[1], + e_q, + &mut e_qmax[1], + s, + ); + process_sample( + 7, + if height > 1 { + coefficients.aligned_value(*sp + stride) + } else { + 0 + }, + p, + &mut rho[1], + e_q, + &mut e_qmax[1], + s, + ); + *sp += 1; + } + + let c_q1 = ((rho[0] >> 1) | (rho[0] & 1)) as usize; + let u_q1 = sink.quad_initial( + 4, + c_q1, + rho[1], + e_qmax[1], + e_q, + s, + lep + 1, + lcxp + 1, + e_val, + cx_val, + )?; + + sink.initial_uvlc_pair(u_q0, u_q1)?; + *c_q0 = ((rho[1] >> 1) | (rho[1] & 1)) as usize; + } else { + sink.initial_uvlc_lone(u_q0)?; + *c_q0 = 0; + } + + *rho = [0; 2]; + *e_q = [0; 8]; + *e_qmax = [0; 2]; + *s = [0; 8]; + + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +#[allow(clippy::inline_always)] // per-quad-pair hot path: keep each sink monomorphization fused +#[inline(always)] +fn non_initial_quad_pair( + coefficients: &(impl CleanupCoefficientSource + ?Sized), + stride: usize, + width: usize, + height: usize, + y: usize, + p: u32, + sp: &mut usize, + x: usize, + e_val: &mut [u8; 513], + cx_val: &mut [u8; 513], + lep: &mut usize, + lcxp: &mut usize, + max_e: &mut i32, + c_q0: &mut usize, + rho: &mut [i32; 2], + e_q: &mut [i32; 8], + e_qmax: &mut [i32; 2], + s: &mut [u32; 8], + sink: &mut S, +) -> Result<(), S::Error> { + process_sample( + 0, + coefficients.aligned_value(*sp), + p, + &mut rho[0], + e_q, + &mut e_qmax[0], + s, + ); + process_sample( + 1, + if y + 1 < height { + coefficients.aligned_value(*sp + stride) + } else { + 0 + }, + p, + &mut rho[0], + e_q, + &mut e_qmax[0], + s, + ); + *sp += 1; + + if x + 1 < width { + process_sample( + 2, + coefficients.aligned_value(*sp), + p, + &mut rho[0], + e_q, + &mut e_qmax[0], + s, + ); + process_sample( + 3, + if y + 1 < height { + coefficients.aligned_value(*sp + stride) + } else { + 0 + }, + p, + &mut rho[0], + e_q, + &mut e_qmax[0], + s, + ); + *sp += 1; + } + + let prev_max = *max_e; + let u_q0 = sink.quad_non_initial(0, *c_q0, rho[0], e_qmax[0], prev_max, e_q, s)?; + + e_val[*lep] = e_val[*lep].max(e_q[1] as u8); + *lep += 1; + *max_e = i32::from(e_val[*lep].max(e_val[*lep + 1])) - 1; + e_val[*lep] = e_q[3] as u8; + cx_val[*lcxp] |= ((rho[0] & 2) >> 1) as u8; + *lcxp += 1; + let c_q1 = usize::from(cx_val[*lcxp]) + (usize::from(cx_val[*lcxp + 1]) << 2); + cx_val[*lcxp] = ((rho[0] & 8) >> 3) as u8; + + let mut u_q1 = 0; + if x + 2 < width { + process_sample( + 4, + coefficients.aligned_value(*sp), + p, + &mut rho[1], + e_q, + &mut e_qmax[1], + s, + ); + process_sample( + 5, + if y + 1 < height { + coefficients.aligned_value(*sp + stride) + } else { + 0 + }, + p, + &mut rho[1], + e_q, + &mut e_qmax[1], + s, + ); + *sp += 1; + + if x + 3 < width { + process_sample( + 6, + coefficients.aligned_value(*sp), + p, + &mut rho[1], + e_q, + &mut e_qmax[1], + s, + ); + process_sample( + 7, + if y + 1 < height { + coefficients.aligned_value(*sp + stride) + } else { + 0 + }, + p, + &mut rho[1], + e_q, + &mut e_qmax[1], + s, + ); + *sp += 1; + } + + let mut c_q1_local = c_q1; + c_q1_local |= ((rho[0] & 4) >> 1) as usize; + c_q1_local |= ((rho[0] & 8) >> 2) as usize; + + u_q1 = sink.quad_non_initial(4, c_q1_local, rho[1], e_qmax[1], *max_e, e_q, s)?; + + e_val[*lep] = e_val[*lep].max(e_q[5] as u8); + *lep += 1; + *max_e = i32::from(e_val[*lep].max(e_val[*lep + 1])) - 1; + e_val[*lep] = e_q[7] as u8; + cx_val[*lcxp] |= ((rho[1] & 2) >> 1) as u8; + *lcxp += 1; + *c_q0 = usize::from(cx_val[*lcxp]) + (usize::from(cx_val[*lcxp + 1]) << 2); + cx_val[*lcxp] = ((rho[1] & 8) >> 3) as u8; + + *c_q0 |= ((rho[1] & 4) >> 1) as usize; + *c_q0 |= ((rho[1] & 8) >> 2) as usize; + } else { + *c_q0 = 0; + } + + sink.non_initial_uvlc(u_q0, u_q1)?; + + *rho = [0; 2]; + *e_q = [0; 8]; + *e_qmax = [0; 2]; + *s = [0; 8]; + + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn encode_first_quad_pair( + coefficients: &(impl CleanupCoefficientSource + ?Sized), + stride: usize, + height: usize, + p: u32, + sp: &mut usize, + x: usize, + e_val: &mut [u8; 513], + cx_val: &mut [u8; 513], + c_q0: &mut usize, + rho: &mut [i32; 2], + e_q: &mut [i32; 8], + e_qmax: &mut [i32; 2], + s: &mut [u32; 8], + mel: &mut MelEncoder, + vlc: &mut VlcEncoder, + ms: &mut MagSgnEncoder, +) -> Result<(), &'static str> { + first_quad_pair( + coefficients, + stride, + height, + p, + sp, + x, + e_val, + cx_val, + c_q0, + rho, + e_q, + e_qmax, + s, + &mut EncodeQuadSink { mel, vlc, ms }, + ) +} + +#[allow(clippy::too_many_arguments)] +fn encode_non_initial_quad_pair( + coefficients: &(impl CleanupCoefficientSource + ?Sized), + stride: usize, + width: usize, + height: usize, + y: usize, + p: u32, + sp: &mut usize, + x: usize, + e_val: &mut [u8; 513], + cx_val: &mut [u8; 513], + lep: &mut usize, + lcxp: &mut usize, + max_e: &mut i32, + c_q0: &mut usize, + rho: &mut [i32; 2], + e_q: &mut [i32; 8], + e_qmax: &mut [i32; 2], + s: &mut [u32; 8], + mel: &mut MelEncoder, + vlc: &mut VlcEncoder, + ms: &mut MagSgnEncoder, +) -> Result<(), &'static str> { + non_initial_quad_pair( + coefficients, + stride, + width, + height, + y, + p, + sp, + x, + e_val, + cx_val, + lep, + lcxp, + max_e, + c_q0, + rho, + e_q, + e_qmax, + s, + &mut EncodeQuadSink { mel, vlc, ms }, + ) +} + +#[allow(clippy::too_many_arguments)] +fn collect_first_quad_pair( + coefficients: &(impl CleanupCoefficientSource + ?Sized), + stride: usize, + height: usize, + p: u32, + sp: &mut usize, + x: usize, + e_val: &mut [u8; 513], + cx_val: &mut [u8; 513], + c_q0: &mut usize, + rho: &mut [i32; 2], + e_q: &mut [i32; 8], + e_qmax: &mut [i32; 2], + s: &mut [u32; 8], + distribution: &mut HtCleanupEncodeDistribution, +) { + match first_quad_pair( + coefficients, + stride, + height, + p, + sp, + x, + e_val, + cx_val, + c_q0, + rho, + e_q, + e_qmax, + s, + &mut CollectQuadSink { distribution }, + ) { + Ok(()) => {} + Err(err) => match err {}, + } +} + +#[allow(clippy::too_many_arguments)] +fn collect_non_initial_quad_pair( + coefficients: &(impl CleanupCoefficientSource + ?Sized), + stride: usize, + width: usize, + height: usize, + y: usize, + p: u32, + sp: &mut usize, + x: usize, + e_val: &mut [u8; 513], + cx_val: &mut [u8; 513], + lep: &mut usize, + lcxp: &mut usize, + max_e: &mut i32, + c_q0: &mut usize, + rho: &mut [i32; 2], + e_q: &mut [i32; 8], + e_qmax: &mut [i32; 2], + s: &mut [u32; 8], + distribution: &mut HtCleanupEncodeDistribution, +) { + match non_initial_quad_pair( + coefficients, + stride, + width, + height, + y, + p, + sp, + x, + e_val, + cx_val, + lep, + lcxp, + max_e, + c_q0, + rho, + e_q, + e_qmax, + s, + &mut CollectQuadSink { distribution }, + ) { + Ok(()) => {} + Err(err) => match err {}, + } +} + +#[allow(clippy::too_many_arguments)] +fn collect_quad_initial_row( + offset: usize, + c_q: usize, + rho: i32, + e_qmax: i32, + e_q: &[i32; 8], + lep: usize, + lcxp: usize, + e_val: &mut [u8; 513], + cx_val: &mut [u8; 513], + distribution: &mut HtCleanupEncodeDistribution, +) -> i32 { + let u_q = e_qmax.max(1) - 1; + let mut eps = 0u16; + + if u_q > 0 { + eps |= u16::from((e_q[offset] == e_qmax) as u8); + eps |= u16::from((e_q[offset + 1] == e_qmax) as u8) << 1; + eps |= u16::from((e_q[offset + 2] == e_qmax) as u8) << 2; + eps |= u16::from((e_q[offset + 3] == e_qmax) as u8) << 3; + } + + e_val[lep] = e_val[lep].max(e_q[offset + 1] as u8); + e_val[lep + 1] = e_q[offset + 3] as u8; + cx_val[lcxp] |= ((rho & 2) >> 1) as u8; + cx_val[lcxp + 1] = ((rho & 8) >> 3) as u8; + + let tuple = HT_VLC_ENCODE_TABLE0[(c_q << 8) | ((rho as usize) << 4) | eps as usize]; + record_distribution_initial_quad(distribution, rho, e_qmax, u_q); + record_distribution_mag_signs(distribution, rho, e_qmax.max(1), tuple); + u_q +} + +fn collect_quad_non_initial_row( + offset: usize, + c_q: usize, + rho: i32, + e_qmax: i32, + max_e: i32, + e_q: &[i32; 8], + distribution: &mut HtCleanupEncodeDistribution, +) -> i32 { + let kappa = if (rho & (rho - 1)) != 0 { + max_e.max(1) + } else { + 1 + }; + let u_q = e_qmax.max(kappa) - kappa; + let mut eps = 0u16; + + if u_q > 0 { + eps |= u16::from((e_q[offset] == e_qmax) as u8); + eps |= u16::from((e_q[offset + 1] == e_qmax) as u8) << 1; + eps |= u16::from((e_q[offset + 2] == e_qmax) as u8) << 2; + eps |= u16::from((e_q[offset + 3] == e_qmax) as u8) << 3; + } + + let tuple = HT_VLC_ENCODE_TABLE1[(c_q << 8) | ((rho as usize) << 4) | eps as usize]; + record_distribution_non_initial_quad(distribution, rho, e_qmax, kappa, u_q); + record_distribution_mag_signs(distribution, rho, e_qmax.max(kappa), tuple); + u_q +} + +#[allow(clippy::too_many_arguments)] +fn encode_quad_initial_row( + offset: usize, + c_q: usize, + rho: i32, + e_qmax: i32, + e_q: &[i32; 8], + s: &[u32; 8], + lep: usize, + lcxp: usize, + e_val: &mut [u8; 513], + cx_val: &mut [u8; 513], + mel: &mut MelEncoder, + vlc: &mut VlcEncoder, + ms: &mut MagSgnEncoder, +) -> Result { + let u_q = e_qmax.max(1) - 1; + let mut eps = 0u16; + + if u_q > 0 { + eps |= u16::from((e_q[offset] == e_qmax) as u8); + eps |= u16::from((e_q[offset + 1] == e_qmax) as u8) << 1; + eps |= u16::from((e_q[offset + 2] == e_qmax) as u8) << 2; + eps |= u16::from((e_q[offset + 3] == e_qmax) as u8) << 3; + } + + e_val[lep] = e_val[lep].max(e_q[offset + 1] as u8); + e_val[lep + 1] = e_q[offset + 3] as u8; + cx_val[lcxp] |= ((rho & 2) >> 1) as u8; + cx_val[lcxp + 1] = ((rho & 8) >> 3) as u8; + + let tuple = HT_VLC_ENCODE_TABLE0[(c_q << 8) | ((rho as usize) << 4) | eps as usize]; + vlc.encode(u32::from(tuple >> 8), ((tuple >> 4) & 0x7) as u8)?; + + if c_q == 0 { + mel.encode(rho != 0)?; + } + + encode_mag_signs(rho, e_qmax.max(1), tuple, s, offset, ms)?; + Ok(u_q) +} + +#[allow(clippy::too_many_arguments)] +fn encode_quad_non_initial_row( + offset: usize, + c_q: usize, + rho: i32, + e_qmax: i32, + max_e: i32, + e_q: &[i32; 8], + s: &[u32; 8], + mel: &mut MelEncoder, + vlc: &mut VlcEncoder, + ms: &mut MagSgnEncoder, +) -> Result { + let kappa = if (rho & (rho - 1)) != 0 { + max_e.max(1) + } else { + 1 + }; + let u_q = e_qmax.max(kappa) - kappa; + let mut eps = 0u16; + + if u_q > 0 { + eps |= u16::from((e_q[offset] == e_qmax) as u8); + eps |= u16::from((e_q[offset + 1] == e_qmax) as u8) << 1; + eps |= u16::from((e_q[offset + 2] == e_qmax) as u8) << 2; + eps |= u16::from((e_q[offset + 3] == e_qmax) as u8) << 3; + } + + let tuple = HT_VLC_ENCODE_TABLE1[(c_q << 8) | ((rho as usize) << 4) | eps as usize]; + vlc.encode(u32::from(tuple >> 8), ((tuple >> 4) & 0x7) as u8)?; + + if c_q == 0 { + mel.encode(rho != 0)?; + } + + encode_mag_signs(rho, e_qmax.max(kappa), tuple, s, offset, ms)?; + Ok(u_q) +} + +#[inline(always)] +fn encode_mag_signs( + rho: i32, + u_q: i32, + tuple: u16, + s: &[u32; 8], + offset: usize, + ms: &mut MagSgnEncoder, +) -> Result<(), &'static str> { + let e_k = tuple & 0xF; + let mut encode = |bit: i32, shift: u32, sample_offset: usize| -> Result<(), &'static str> { + let sample_mask = 1 << bit; + if (rho & sample_mask) == 0 { + return Ok(()); + } + + let reduction = ((u32::from(e_k) >> shift) & 1) as i32; + let magnitude_bits = (u_q - reduction) as u32; + let payload = if magnitude_bits == 0 { + 0 + } else { + s[offset + sample_offset] & ((1u32 << magnitude_bits) - 1) + }; + ms.encode(payload, magnitude_bits) + }; + + encode(0, 0, 0)?; + encode(1, 1, 1)?; + encode(2, 2, 2)?; + encode(3, 3, 3)?; + + Ok(()) +} + +fn encode_uvlc(u_q0: i32, u_q1: i32, vlc: &mut VlcEncoder) -> Result<(), &'static str> { + if u_q0 > 2 && u_q1 > 2 { + let first = HT_UVLC_ENCODE_TABLE[(u_q0 - 2) as usize]; + let second = HT_UVLC_ENCODE_TABLE[(u_q1 - 2) as usize]; + encode_uvlc_pair(vlc, first, second) + } else if u_q0 > 2 && u_q1 > 0 { + let first = HT_UVLC_ENCODE_TABLE[u_q0 as usize]; + vlc.encode(u32::from(first.pre), first.pre_len)?; + vlc.encode((u_q1 - 1) as u32, 1)?; + vlc.encode(u32::from(first.suf), first.suf_len) + } else { + let first = HT_UVLC_ENCODE_TABLE[u_q0.max(0) as usize]; + let second = HT_UVLC_ENCODE_TABLE[u_q1.max(0) as usize]; + encode_uvlc_pair(vlc, first, second) + } +} + +fn encode_uvlc_non_initial(u_q0: i32, u_q1: i32, vlc: &mut VlcEncoder) -> Result<(), &'static str> { + let first = HT_UVLC_ENCODE_TABLE[u_q0.max(0) as usize]; + let second = HT_UVLC_ENCODE_TABLE[u_q1.max(0) as usize]; + encode_uvlc_pair(vlc, first, second) +} + +fn encode_uvlc_pair( + vlc: &mut VlcEncoder, + first: HtUvlcTableEntry, + second: HtUvlcTableEntry, +) -> Result<(), &'static str> { + vlc.encode(u32::from(first.pre), first.pre_len)?; + vlc.encode(u32::from(second.pre), second.pre_len)?; + vlc.encode(u32::from(first.suf), first.suf_len)?; + vlc.encode(u32::from(second.suf), second.suf_len) +} + +fn terminate_mel_vlc(mel: &mut MelEncoder, vlc: &mut VlcEncoder) -> Result<(), &'static str> { + if mel.run > 0 { + mel.emit_bit(true)?; + } + + mel.tmp = (u16::from(mel.tmp) << mel.remaining_bits) as u8; + let mel_mask = ((0xFFu16 << mel.remaining_bits) & 0xFF) as u8; + let vlc_mask = if vlc.used_bits == 0 { + 0 + } else { + ((1u16 << vlc.used_bits) - 1) as u8 + }; + + if (mel_mask | vlc_mask) == 0 { + return Ok(()); + } + + let fused = mel.tmp | vlc.tmp; + let fused_ok = + (((fused ^ mel.tmp) & mel_mask) | ((fused ^ vlc.tmp) & vlc_mask)) == 0 && fused != 0xFF; + + if fused_ok && vlc.pos > 1 { + if mel.pos >= mel.buffer.len() { + return Err("HTJ2K MEL encoder buffer is full"); + } + + mel.buffer[mel.pos] = fused; + mel.pos += 1; + } else { + if mel.pos >= mel.buffer.len() { + return Err("HTJ2K MEL encoder buffer is full"); + } + if vlc.pos >= vlc.buffer.len() { + return Err("HTJ2K VLC encoder buffer is full"); + } + + mel.buffer[mel.pos] = mel.tmp; + mel.pos += 1; + let write_index = vlc.buffer.len() - 1 - vlc.pos; + vlc.buffer[write_index] = vlc.tmp; + vlc.pos += 1; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_convert_to_aligned_sign_magnitude() { + let (aligned, _) = convert_nonzero_to_aligned_sign_magnitude_and_max(&[0, 1, -2, 3], 2) + .expect("non-zero block"); + assert_eq!(aligned, vec![0, 0x2000_0000, 0xC000_0000, 0x6000_0000]); + } + + #[test] + fn aligned_sign_magnitude_conversion_reports_max_and_skips_all_zero_blocks() { + assert!(convert_nonzero_to_aligned_sign_magnitude_and_max(&[0, 0, 0], 5).is_none()); + + let (aligned, max_magnitude) = + convert_nonzero_to_aligned_sign_magnitude_and_max(&[0, 1, -2, 3], 2) + .expect("non-zero block"); + assert_eq!(max_magnitude, 3); + assert_eq!(aligned, vec![0, 0x2000_0000, 0xC000_0000, 0x6000_0000]); + } + + #[test] + fn cleanup_segment_from_i32_coefficients_matches_preconverted_path() { + let coefficients: Vec = (0..64) + .map(|index| match index % 5 { + 0 => 0, + 1 => index * 3, + 2 => -(index * 2), + 3 => 7 - index, + _ => index / 2, + }) + .collect(); + let total_bitplanes = 10; + let missing_msbs = total_bitplanes - 1; + let (aligned, _) = + convert_nonzero_to_aligned_sign_magnitude_and_max(&coefficients, total_bitplanes) + .expect("non-zero block"); + + let expected = + encode_cleanup_segment(&aligned, missing_msbs, 8, 8).expect("preconverted encode"); + let actual = encode_cleanup_segment_from_coefficients( + &coefficients, + missing_msbs, + 8, + 8, + total_bitplanes, + ) + .expect("i32 encode"); + + assert_eq!(actual, expected); + } + + #[test] + fn cleanup_encode_distribution_counts_quads_and_mag_sign_payloads() { + let coefficients: Vec = (0..8 * 6) + .map(|index| { + if index % 7 == 0 { + 0 + } else { + let value = ((index * 29) & 0x1ff) - 255; + if index % 3 == 0 { + -value + } else { + value + } + } + }) + .collect(); + + let distribution = + collect_encode_distribution(&coefficients, 8, 6, 10).expect("collect distribution"); + + assert_eq!(distribution.total_quads, 12); + assert_eq!(distribution.initial_quads, 4); + assert_eq!(distribution.non_initial_quads, 8); + assert_eq!(distribution.rho_counts.iter().sum::(), 12); + assert_eq!(distribution.initial_rho_counts.iter().sum::(), 4); + assert_eq!(distribution.non_initial_rho_counts.iter().sum::(), 8); + assert_eq!(distribution.non_initial_u_q_counts.iter().sum::(), 8); + assert!(distribution.mag_sign_calls > 0); + assert!(distribution.mag_sign_encoded_samples > 0); + } + + #[cfg(feature = "std")] + #[test] + #[ignore = "prints HT cleanup encode rho/e_q/u_q distribution for manual tuning"] + fn ht_cleanup_encode_distribution_report() { + fn nonzero_histogram(counts: &[u64; N]) -> Vec<(usize, u64)> { + counts + .iter() + .copied() + .enumerate() + .filter(|&(_, count)| count != 0) + .collect() + } + + let coefficients: Vec = (0usize..64 * 64) + .map(|index| { + let value = (((index * 73) ^ (index >> 2)) & 0x01ff) as i32 - 255; + if index % 13 == 0 { + 0 + } else { + value + } + }) + .collect(); + let distribution = + collect_encode_distribution(&coefficients, 64, 64, 10).expect("collect distribution"); + + let mut rho_u_q = Vec::new(); + for (rho, counts) in distribution.non_initial_rho_u_q_counts.iter().enumerate() { + for (u_q, count) in counts.iter().copied().enumerate() { + if count != 0 { + rho_u_q.push((rho, u_q, count)); + } + } + } + rho_u_q.sort_by_key(|&(_, _, count)| core::cmp::Reverse(count)); + + println!( + "quads total={} initial={} non_initial={}", + distribution.total_quads, distribution.initial_quads, distribution.non_initial_quads + ); + println!("rho={:?}", nonzero_histogram(&distribution.rho_counts)); + println!( + "non_initial_u_q={:?}", + nonzero_histogram(&distribution.non_initial_u_q_counts) + ); + println!( + "non_initial_e_qmax={:?}", + nonzero_histogram(&distribution.non_initial_e_qmax_counts) + ); + println!( + "non_initial_kappa={:?}", + nonzero_histogram(&distribution.non_initial_kappa_counts) + ); + println!( + "mag_sign_sample_bits={:?}", + nonzero_histogram(&distribution.mag_sign_sample_bit_counts) + ); + println!( + "top_non_initial_rho_u_q={:?}", + &rho_u_q[..rho_u_q.len().min(8)] + ); + } + + #[test] + fn test_encode_cleanup_only_nonzero_block() { + let encoded = encode_code_block(&[1], 1, 1, 5).expect("encode HT block"); + assert_eq!(encoded.num_coding_passes, 1); + assert_eq!(encoded.num_zero_bitplanes, 4); + assert!(encoded.data.len() >= 2); + } +} diff --git a/crates/signinum-j2k-native/src/j2c/ht_encode_tables.rs b/crates/j2k-native/src/j2c/ht_encode_tables.rs similarity index 98% rename from crates/signinum-j2k-native/src/j2c/ht_encode_tables.rs rename to crates/j2k-native/src/j2c/ht_encode_tables.rs index 72fdd9ea..272af8e3 100644 --- a/crates/signinum-j2k-native/src/j2c/ht_encode_tables.rs +++ b/crates/j2k-native/src/j2c/ht_encode_tables.rs @@ -1,8 +1,7 @@ //! HTJ2K encoder lookup tables generated from OpenJPH source rows. -#![allow(dead_code)] #[repr(C)] -/// Hidden HTJ2K UVLC encoder table row for backend experimentation. +/// Adapter HTJ2K UVLC encoder table row for backend experimentation. #[derive(Clone, Copy, Debug)] pub struct HtUvlcTableEntry { /// Prefix code bits. @@ -969,3 +968,23 @@ pub(crate) const HT_UVLC_ENCODE_TABLE: [HtUvlcTableEntry; 75] = [ ext_len: 4, }, ]; + +const fn pack_ht_uvlc_encode_table(table: &[HtUvlcTableEntry; 75]) -> [u8; 75 * 6] { + let mut bytes = [0u8; 75 * 6]; + let mut index = 0; + while index < 75 { + let offset = index * 6; + let entry = table[index]; + bytes[offset] = entry.pre; + bytes[offset + 1] = entry.pre_len; + bytes[offset + 2] = entry.suf; + bytes[offset + 3] = entry.suf_len; + bytes[offset + 4] = entry.ext; + bytes[offset + 5] = entry.ext_len; + index += 1; + } + bytes +} + +pub(crate) const HT_UVLC_ENCODE_TABLE_BYTES: [u8; 75 * 6] = + pack_ht_uvlc_encode_table(&HT_UVLC_ENCODE_TABLE); diff --git a/crates/signinum-j2k-native/src/j2c/ht_tables.rs b/crates/j2k-native/src/j2c/ht_tables.rs similarity index 100% rename from crates/signinum-j2k-native/src/j2c/ht_tables.rs rename to crates/j2k-native/src/j2c/ht_tables.rs diff --git a/crates/signinum-j2k-native/src/j2c/idwt.rs b/crates/j2k-native/src/j2c/idwt.rs similarity index 93% rename from crates/signinum-j2k-native/src/j2c/idwt.rs rename to crates/j2k-native/src/j2c/idwt.rs index 77fa6c44..e343b9ce 100644 --- a/crates/signinum-j2k-native/src/j2c/idwt.rs +++ b/crates/j2k-native/src/j2c/idwt.rs @@ -8,7 +8,7 @@ use super::codestream::WaveletTransform; use super::decode::{DecompositionStorage, TileDecodeContext}; use super::rect::IntRect; use super::roi; -use crate::error::DecodingError; +use crate::error::{bail, DecodingError}; use crate::j2c::Header; use crate::math::{self, dispatch, f32x8, Level, Simd, SIMD_WIDTH}; use crate::{ @@ -417,6 +417,70 @@ fn apply_level( } } +pub(crate) fn apply_single_decomposition_idwt_job( + job: J2kSingleDecompositionIdwtJob<'_>, + target: &mut Vec, +) -> Result<()> { + let rect = int_rect_from_public(job.rect); + validate_direct_band(job.ll)?; + validate_direct_band(job.hl)?; + validate_direct_band(job.lh)?; + validate_direct_band(job.hh)?; + + target.clear(); + let required_len = rect + .width() + .checked_mul(rect.height()) + .and_then(|len| usize::try_from(len).ok()) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + target.resize(required_len, 0.0); + + interleave_samples_roi( + direct_coefficient_source(job.ll), + direct_coefficient_source(job.hl), + direct_coefficient_source(job.lh), + direct_coefficient_source(job.hh), + target, + rect, + rect, + ); + if rect.width() > 0 && rect.height() > 0 { + let transform = wavelet_transform_from_public(job.transform); + filter_horizontal(target, rect, transform); + filter_vertical(target, rect, transform); + } + Ok(()) +} + +fn validate_direct_band(band: J2kIdwtBand<'_>) -> Result<()> { + let rect = int_rect_from_public(band.rect); + let required_len = rect + .width() + .checked_mul(rect.height()) + .and_then(|len| usize::try_from(len).ok()) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if band.coefficients.len() < required_len { + bail!(DecodingError::CodeBlockDecodeFailure); + } + Ok(()) +} + +fn direct_coefficient_source(band: J2kIdwtBand<'_>) -> CoefficientSource<'_> { + let rect = int_rect_from_public(band.rect); + CoefficientSource::new(band.coefficients, rect, rect.width()) +} + +fn int_rect_from_public(rect: J2kRect) -> IntRect { + IntRect::from_ltrb(rect.x0, rect.y0, rect.x1, rect.y1) +} + +fn wavelet_transform_from_public(transform: J2kWaveletTransform) -> WaveletTransform { + match transform { + J2kWaveletTransform::Reversible53 => WaveletTransform::Reversible53, + J2kWaveletTransform::Irreversible97 => WaveletTransform::Irreversible97, + } +} + #[derive(Clone, Copy)] struct IDWTInput<'a> { rect: IntRect, @@ -770,6 +834,11 @@ fn irreversible_filter_97i(scanline: &mut [f32], width: usize, x0: usize) { ); } +#[cfg(test)] +pub(crate) fn test_irreversible_filter_97i(scanline: &mut [f32], width: usize, x0: usize) { + irreversible_filter_97i(scanline, width, x0); +} + #[inline(always)] fn filter_step_horizontal( scanline: &mut [f32], diff --git a/crates/signinum-j2k-native/src/j2c/mct.rs b/crates/j2k-native/src/j2c/mct.rs similarity index 100% rename from crates/signinum-j2k-native/src/j2c/mct.rs rename to crates/j2k-native/src/j2c/mct.rs diff --git a/crates/signinum-j2k-native/src/j2c/mod.rs b/crates/j2k-native/src/j2c/mod.rs similarity index 81% rename from crates/signinum-j2k-native/src/j2c/mod.rs rename to crates/j2k-native/src/j2c/mod.rs index 027de8bc..97c38b1c 100644 --- a/crates/signinum-j2k-native/src/j2c/mod.rs +++ b/crates/j2k-native/src/j2c/mod.rs @@ -13,11 +13,12 @@ pub(crate) mod ht_block_decode; pub(crate) mod ht_block_encode; pub(crate) mod ht_encode_tables; pub(crate) mod ht_tables; -mod idwt; +pub(crate) mod idwt; mod mct; pub(crate) mod packet_encode; mod progression; pub(crate) mod quantize; +pub(crate) mod recode; mod rect; mod roi; mod segment; @@ -38,6 +39,7 @@ pub(crate) use codestream::Header; pub(crate) use decode::should_decode_classic_sub_band_in_parallel; pub(crate) use decode::{build_direct_color_plan, build_direct_grayscale_plan, decode}; pub use decode::{CpuDecodeParallelism, DecoderContext}; +pub use recode::Reversible53CoefficientImage; pub(crate) struct ParsedCodestream<'a> { pub(crate) header: Header<'a>, @@ -55,12 +57,14 @@ pub(crate) fn parse<'a>(stream: &'a [u8], settings: &DecodeSettings) -> Result ColorSpace::Enumerated(EnumeratedColorspace::Greyscale), + 2 => ColorSpace::Unknown, + _ => ColorSpace::Enumerated(EnumeratedColorspace::Srgb), }; boxes.color_specification = Some(ColorSpecificationBox { color_space: cs }); diff --git a/crates/signinum-j2k-native/src/j2c/packet_encode.rs b/crates/j2k-native/src/j2c/packet_encode.rs similarity index 70% rename from crates/signinum-j2k-native/src/j2c/packet_encode.rs rename to crates/j2k-native/src/j2c/packet_encode.rs index 0462fcd2..4909e1d3 100644 --- a/crates/signinum-j2k-native/src/j2c/packet_encode.rs +++ b/crates/j2k-native/src/j2c/packet_encode.rs @@ -13,8 +13,10 @@ use alloc::vec; use alloc::vec::Vec; +use super::codestream::markers; use super::codestream_write::BlockCodingMode; use super::tag_tree_encode::TagTreeEncoder; +use crate::packet_math::{self, bits_for_ht_cleanup_length, bits_for_length, value_fits_in_bits}; use crate::writer::BitWriter; use crate::J2kPacketizationProgressionOrder; @@ -23,8 +25,14 @@ use crate::J2kPacketizationProgressionOrder; pub(crate) struct CodeBlockPacketData { /// Encoded bitstream data. pub(crate) data: Vec, + /// HTJ2K cleanup segment length in bytes. + pub(crate) ht_cleanup_length: u32, + /// HTJ2K refinement segment length in bytes. + pub(crate) ht_refinement_length: u32, /// Number of coding passes in this contribution. pub(crate) num_coding_passes: u8, + /// Per-pass classic segment lengths when code-block pass termination is enabled. + pub(crate) classic_segment_lengths: Vec, /// Number of zero bitplanes (only relevant for first inclusion). pub(crate) num_zero_bitplanes: u8, /// Whether this code-block has been included in a previous packet. @@ -84,6 +92,17 @@ struct PacketState { subbands: Vec, } +pub(crate) struct PacketizedTileData { + pub(crate) data: Vec, + pub(crate) packet_lengths: Vec, +} + +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct PacketMarkerOptions { + pub(crate) write_sop: bool, + pub(crate) write_eph: bool, +} + /// Form a packet from a resolution-level packet (possibly multiple subbands). /// /// Returns the packet bytes (header + body). @@ -100,7 +119,7 @@ pub(crate) fn form_packet(resolution: &mut ResolutionPacket) -> Vec { if !any_data { // Empty packet: just write 0 bit header_writer.write_bit(0); - return header_writer.finish(); + return finish_packet(header_writer, Vec::new(), PacketMarkerOptions::default(), 0); } // Non-empty packet indicator @@ -155,18 +174,14 @@ pub(crate) fn form_packet(resolution: &mut ResolutionPacket) -> Vec { let data_len = cb.data.len() as u32; match cb.block_coding_mode { BlockCodingMode::Classic => { - let num_bits = bits_for_length(cb.l_block, cb.num_coding_passes); encode_num_coding_passes(cb.num_coding_passes, &mut header_writer); - encode_length(data_len, &mut cb.l_block, num_bits, &mut header_writer); + encode_classic_segment_lengths(cb, data_len, &mut header_writer) + .expect("encoder prepared valid classic segment lengths"); } BlockCodingMode::HighThroughput => { - debug_assert!( - cb.num_coding_passes <= 1, - "current HT packet writer only supports cleanup-only contributions" - ); - let num_bits = bits_for_ht_cleanup_length(cb.l_block, cb.num_coding_passes); encode_num_ht_coding_passes(cb.num_coding_passes, &mut header_writer); - encode_length(data_len, &mut cb.l_block, num_bits, &mut header_writer); + encode_ht_segment_lengths(cb, &mut header_writer) + .expect("encoder prepared valid HTJ2K segment lengths"); } } @@ -176,15 +191,7 @@ pub(crate) fn form_packet(resolution: &mut ResolutionPacket) -> Vec { } } - // Assemble: header (byte-aligned) + body. Packet headers use JPEG 2000 - // bit stuffing; if the final header byte is 0xff, the following byte must - // carry the stuffed zero bit before any packet body bytes. - let mut packet = header_writer.finish(); - if packet.last().copied() == Some(0xff) { - packet.push(0x00); - } - packet.extend_from_slice(&body); - packet + finish_packet(header_writer, body, PacketMarkerOptions::default(), 0) } fn packet_state_seed(packet: &ResolutionPacket) -> Result { @@ -331,10 +338,12 @@ fn build_packet_states( .collect() } -fn form_packet_with_state( +fn form_packet_with_state_and_options( packet_data: &ResolutionPacket, state: &mut PacketState, layer: u8, + marker_options: PacketMarkerOptions, + packet_sequence: u16, ) -> Result, &'static str> { if state.subbands.len() != packet_data.subbands.len() { return Err("packet descriptor state layout mismatch"); @@ -349,7 +358,12 @@ fn form_packet_with_state( if !any_data { header_writer.write_bit(0); - return Ok(header_writer.finish()); + return Ok(finish_packet( + header_writer, + Vec::new(), + marker_options, + packet_sequence, + )); } header_writer.write_bit(1); @@ -393,32 +407,21 @@ fn form_packet_with_state( let data_len = packet_block.data.len() as u32; match packet_block.block_coding_mode { BlockCodingMode::Classic => { - let num_bits = - bits_for_length(state_block.l_block, packet_block.num_coding_passes); encode_num_coding_passes(packet_block.num_coding_passes, &mut header_writer); - encode_length( + encode_classic_segment_lengths_with_lblock( + packet_block, data_len, &mut state_block.l_block, - num_bits, &mut header_writer, - ); + )?; } BlockCodingMode::HighThroughput => { - debug_assert!( - packet_block.num_coding_passes <= 1, - "current HT packet writer only supports cleanup-only contributions" - ); - let num_bits = bits_for_ht_cleanup_length( - state_block.l_block, - packet_block.num_coding_passes, - ); encode_num_ht_coding_passes(packet_block.num_coding_passes, &mut header_writer); - encode_length( - data_len, + encode_ht_segment_lengths_with_lblock( + packet_block, &mut state_block.l_block, - num_bits, &mut header_writer, - ); + )?; } } body.extend_from_slice(&packet_block.data); @@ -426,12 +429,41 @@ fn form_packet_with_state( } } - let mut packet = header_writer.finish(); - if packet.last().copied() == Some(0xff) { - packet.push(0x00); + Ok(finish_packet( + header_writer, + body, + marker_options, + packet_sequence, + )) +} + +fn finish_packet( + header_writer: BitWriter, + body: Vec, + marker_options: PacketMarkerOptions, + packet_sequence: u16, +) -> Vec { + let mut packet = Vec::new(); + if marker_options.write_sop { + packet.push(0xFF); + packet.push(markers::SOP); + packet.extend_from_slice(&4u16.to_be_bytes()); + packet.extend_from_slice(&packet_sequence.to_be_bytes()); + } + + let mut header = header_writer.finish(); + if header.last().copied() == Some(0xff) { + header.push(0x00); } + packet.extend_from_slice(&header); + + if marker_options.write_eph { + packet.push(0xFF); + packet.push(markers::EPH); + } + packet.extend_from_slice(&body); - Ok(packet) + packet } /// Encode the number of coding passes using the variable-length code from Table B.4. @@ -487,24 +519,105 @@ fn encode_length(length: u32, l_block: &mut u32, mut num_bits: u32, writer: &mut writer.write_bits(length, num_bits as u8); } -fn value_fits_in_bits(value: u32, bits: u32) -> bool { - bits >= u32::BITS || value < (1u32 << bits) +fn encode_classic_segment_lengths( + code_block: &mut CodeBlockPacketData, + data_len: u32, + writer: &mut BitWriter, +) -> Result<(), &'static str> { + let mut l_block = code_block.l_block; + encode_classic_segment_lengths_with_lblock(code_block, data_len, &mut l_block, writer)?; + code_block.l_block = l_block; + Ok(()) } -/// Calculate number of bits needed to encode a segment length. -fn bits_for_length(l_block: u32, num_coding_passes: u8) -> u32 { - let log2_passes = if num_coding_passes <= 1 { - 0 - } else { - (num_coding_passes as u32).ilog2() - }; - l_block + log2_passes +fn encode_classic_segment_lengths_with_lblock( + code_block: &CodeBlockPacketData, + data_len: u32, + l_block: &mut u32, + writer: &mut BitWriter, +) -> Result<(), &'static str> { + if code_block.classic_segment_lengths.is_empty() { + let num_bits = bits_for_length(*l_block, code_block.num_coding_passes); + encode_length(data_len, l_block, num_bits, writer); + return Ok(()); + } + + if code_block.classic_segment_lengths.len() != usize::from(code_block.num_coding_passes) { + return Err("classic pass-terminated contribution segment count mismatch"); + } + let segment_sum = code_block + .classic_segment_lengths + .iter() + .try_fold(0u32, |acc, segment_len| acc.checked_add(*segment_len)) + .ok_or("classic packet contribution segment length overflow")?; + if segment_sum != data_len { + return Err("classic packet contribution segment length mismatch"); + } + + let mut required_l_block = *l_block; + while code_block + .classic_segment_lengths + .iter() + .any(|&segment_len| !value_fits_in_bits(segment_len, bits_for_length(required_l_block, 1))) + { + writer.write_bit(1); + required_l_block += 1; + } + writer.write_bit(0); + *l_block = required_l_block; + + let length_bits = bits_for_length(*l_block, 1); + for &segment_len in &code_block.classic_segment_lengths { + writer.write_bits(segment_len, length_bits as u8); + } + + Ok(()) +} + +fn encode_ht_segment_lengths( + code_block: &mut CodeBlockPacketData, + writer: &mut BitWriter, +) -> Result<(), &'static str> { + let mut l_block = code_block.l_block; + encode_ht_segment_lengths_with_lblock(code_block, &mut l_block, writer)?; + code_block.l_block = l_block; + Ok(()) +} + +fn encode_ht_segment_lengths_with_lblock( + code_block: &CodeBlockPacketData, + l_block: &mut u32, + writer: &mut BitWriter, +) -> Result<(), &'static str> { + let (cleanup_length, refinement_length) = ht_segment_lengths(code_block)?; + let mut cleanup_bits = bits_for_ht_cleanup_length(*l_block, code_block.num_coding_passes); + let refinement_extra_bits = u32::from(code_block.num_coding_passes > 2); + + while !value_fits_in_bits(cleanup_length, cleanup_bits) + || (code_block.num_coding_passes > 1 + && !value_fits_in_bits(refinement_length, *l_block + refinement_extra_bits)) + { + writer.write_bit(1); + *l_block += 1; + cleanup_bits += 1; + } + writer.write_bit(0); + writer.write_bits(cleanup_length, cleanup_bits as u8); + + if code_block.num_coding_passes > 1 { + writer.write_bits(refinement_length, (*l_block + refinement_extra_bits) as u8); + } + + Ok(()) } -fn bits_for_ht_cleanup_length(l_block: u32, raw_num_passes: u8) -> u32 { - let placeholder_groups = u32::from(raw_num_passes.saturating_sub(1)) / 3; - let placeholder_passes = placeholder_groups * 3; - l_block + (placeholder_passes + 1).ilog2() +fn ht_segment_lengths(code_block: &CodeBlockPacketData) -> Result<(u32, u32), &'static str> { + packet_math::ht_segment_lengths( + code_block.num_coding_passes, + code_block.data.len(), + code_block.ht_cleanup_length, + code_block.ht_refinement_length, + ) } /// Form tile bitstream from resolution packets in LRCP order. @@ -533,22 +646,57 @@ pub(crate) fn form_tile_bitstream_with_descriptors( resolution_packets: &mut [ResolutionPacket], descriptors: &[PacketDescriptor], ) -> Result, &'static str> { + Ok(form_tile_bitstream_with_descriptors_and_lengths(resolution_packets, descriptors)?.data) +} + +pub(crate) fn form_tile_bitstream_with_descriptors_and_lengths( + resolution_packets: &mut [ResolutionPacket], + descriptors: &[PacketDescriptor], +) -> Result { + form_tile_bitstream_with_descriptors_lengths_and_markers( + resolution_packets, + descriptors, + PacketMarkerOptions::default(), + ) +} + +pub(crate) fn form_tile_bitstream_with_descriptors_lengths_and_markers( + resolution_packets: &mut [ResolutionPacket], + descriptors: &[PacketDescriptor], + marker_options: PacketMarkerOptions, +) -> Result { if descriptors.is_empty() { - return Ok(Vec::new()); + return Ok(PacketizedTileData { + data: Vec::new(), + packet_lengths: Vec::new(), + }); } let mut states = build_packet_states(resolution_packets, descriptors)?; let mut tile_data = Vec::new(); - for descriptor in descriptors { + let mut packet_lengths = Vec::with_capacity(descriptors.len()); + for (packet_sequence, descriptor) in descriptors.iter().enumerate() { let packet = resolution_packets .get(descriptor.packet_index as usize) .ok_or("packet descriptor packet index out of range")?; let state = states .get_mut(descriptor.state_index as usize) .ok_or("packet descriptor state index out of range")?; - tile_data.extend_from_slice(&form_packet_with_state(packet, state, descriptor.layer)?); + let packet = form_packet_with_state_and_options( + packet, + state, + descriptor.layer, + marker_options, + u16::try_from(packet_sequence % (usize::from(u16::MAX) + 1)) + .expect("SOP packet sequence modulo 65536 fits u16"), + )?; + packet_lengths.push(u32::try_from(packet.len()).map_err(|_| "packet length exceeds u32")?); + tile_data.extend_from_slice(&packet); } - Ok(tile_data) + Ok(PacketizedTileData { + data: tile_data, + packet_lengths, + }) } pub(crate) fn form_tile_bitstream_for_progression( @@ -558,12 +706,31 @@ pub(crate) fn form_tile_bitstream_for_progression( progression_order: J2kPacketizationProgressionOrder, ) -> Vec { match progression_order { - J2kPacketizationProgressionOrder::Lrcp | J2kPacketizationProgressionOrder::Rpcl => { + J2kPacketizationProgressionOrder::Lrcp + | J2kPacketizationProgressionOrder::Rlcp + | J2kPacketizationProgressionOrder::Rpcl + | J2kPacketizationProgressionOrder::Pcrl + | J2kPacketizationProgressionOrder::Cprl => { form_tile_bitstream(resolution_packets, num_layers, num_components) } } } +pub(crate) fn validate_ht_segment_lengths( + resolution_packets: &[ResolutionPacket], +) -> Result<(), &'static str> { + for resolution in resolution_packets { + for subband in &resolution.subbands { + for code_block in &subband.code_blocks { + if code_block.block_coding_mode == BlockCodingMode::HighThroughput { + ht_segment_lengths(code_block)?; + } + } + } + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -572,6 +739,10 @@ mod tests { fn decode_num_ht_coding_passes_for_test(data: &[u8]) -> Option { let mut reader = BitReader::new(data); + decode_num_ht_coding_passes_from_reader_for_test(&mut reader) + } + + fn decode_num_ht_coding_passes_from_reader_for_test(reader: &mut BitReader<'_>) -> Option { let mut num_passes = 1u32; if reader.read_bits_with_stuffing(1)? == 1 { @@ -634,7 +805,10 @@ mod tests { subbands: vec![SubbandPrecinct { code_blocks: vec![CodeBlockPacketData { data: Vec::new(), + ht_cleanup_length: 0, + ht_refinement_length: 0, num_coding_passes: 0, + classic_segment_lengths: Vec::new(), num_zero_bitplanes: 31, previously_included: false, l_block: 3, @@ -655,7 +829,10 @@ mod tests { subbands: vec![SubbandPrecinct { code_blocks: vec![CodeBlockPacketData { data: vec![0x12, 0x34, 0x56], + ht_cleanup_length: 0, + ht_refinement_length: 0, num_coding_passes: 1, + classic_segment_lengths: Vec::new(), num_zero_bitplanes: 20, previously_included: false, l_block: 3, @@ -692,7 +869,10 @@ mod tests { .zip(lengths.iter().copied()) .map(|(num_zero_bitplanes, len)| CodeBlockPacketData { data: vec![0; len], + ht_cleanup_length: 0, + ht_refinement_length: 0, num_coding_passes: 1 + 3 * (8 - num_zero_bitplanes) - 2, + classic_segment_lengths: Vec::new(), num_zero_bitplanes, previously_included: false, l_block: 3, @@ -752,7 +932,10 @@ mod tests { subbands: vec![SubbandPrecinct { code_blocks: vec![CodeBlockPacketData { data: vec![0x80; len], + ht_cleanup_length: 0, + ht_refinement_length: 0, num_coding_passes: 1, + classic_segment_lengths: Vec::new(), num_zero_bitplanes: 0, previously_included: false, l_block: 3, @@ -817,6 +1000,49 @@ mod tests { panic!("did not find a packet header ending in 0xff"); } + #[test] + fn classic_pass_terminated_lengths_share_one_lblock_increment() { + let lengths = [1u32, 9, 17]; + let mut code_block = CodeBlockPacketData { + data: vec![0; lengths.iter().sum::() as usize], + ht_cleanup_length: 0, + ht_refinement_length: 0, + num_coding_passes: lengths.len() as u8, + classic_segment_lengths: lengths.to_vec(), + num_zero_bitplanes: 0, + previously_included: false, + l_block: 3, + block_coding_mode: BlockCodingMode::Classic, + }; + let mut writer = BitWriter::new(); + let data_len = u32::try_from(code_block.data.len()).expect("test payload length fits u32"); + + encode_num_coding_passes(code_block.num_coding_passes, &mut writer); + encode_classic_segment_lengths(&mut code_block, data_len, &mut writer) + .expect("classic segment lengths encode"); + + let bytes = writer.finish(); + let mut reader = BitReader::new(&bytes); + let passes = decode_num_coding_passes_from_reader_for_test(&mut reader) + .expect("number of coding passes"); + assert_eq!(passes, lengths.len() as u8); + + let mut l_block = 3u32; + while reader.read_bits_with_stuffing(1).expect("lblock increment") == 1 { + l_block += 1; + } + + let decoded_lengths: Vec<_> = lengths + .iter() + .map(|_| { + reader + .read_bits_with_stuffing(l_block as u8) + .expect("terminated pass segment length") + }) + .collect(); + assert_eq!(decoded_lengths, lengths); + } + #[test] fn test_multi_subband_packet() { let mut resolution = ResolutionPacket { @@ -824,7 +1050,10 @@ mod tests { SubbandPrecinct { code_blocks: vec![CodeBlockPacketData { data: vec![0x10, 0x20], + ht_cleanup_length: 0, + ht_refinement_length: 0, num_coding_passes: 1, + classic_segment_lengths: Vec::new(), num_zero_bitplanes: 20, previously_included: false, l_block: 3, @@ -836,7 +1065,10 @@ mod tests { SubbandPrecinct { code_blocks: vec![CodeBlockPacketData { data: vec![0x30, 0x40], + ht_cleanup_length: 0, + ht_refinement_length: 0, num_coding_passes: 1, + classic_segment_lengths: Vec::new(), num_zero_bitplanes: 22, previously_included: false, l_block: 3, @@ -848,7 +1080,10 @@ mod tests { SubbandPrecinct { code_blocks: vec![CodeBlockPacketData { data: vec![0x50], + ht_cleanup_length: 0, + ht_refinement_length: 0, num_coding_passes: 1, + classic_segment_lengths: Vec::new(), num_zero_bitplanes: 24, previously_included: false, l_block: 3, @@ -902,7 +1137,10 @@ mod tests { subbands: vec![SubbandPrecinct { code_blocks: vec![CodeBlockPacketData { data: vec![0x12, 0x34, 0x56], + ht_cleanup_length: 3, + ht_refinement_length: 0, num_coding_passes: 1, + classic_segment_lengths: Vec::new(), num_zero_bitplanes: 20, previously_included: false, l_block: 3, @@ -917,12 +1155,93 @@ mod tests { assert!(packet.len() >= 3); } + #[test] + fn ht_packet_header_round_trips_refinement_pass_count_and_length() { + let payload = vec![0x12, 0x34, 0x56, 0x78, 0x9a]; + let mut resolution = ResolutionPacket { + subbands: vec![SubbandPrecinct { + code_blocks: vec![CodeBlockPacketData { + data: payload.clone(), + ht_cleanup_length: 3, + ht_refinement_length: 2, + num_coding_passes: 3, + classic_segment_lengths: Vec::new(), + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: BlockCodingMode::HighThroughput, + }], + num_cbs_x: 1, + num_cbs_y: 1, + }], + }; + + let packet = form_packet(&mut resolution); + let header_len = packet.len() - payload.len(); + let mut reader = BitReader::new(&packet[..header_len]); + assert_eq!(reader.read_bits_with_stuffing(1), Some(1)); + + let mut inclusion_nodes = Vec::::new(); + let mut inclusion_tree = TagTree::new(1, 1, &mut inclusion_nodes); + assert_eq!( + inclusion_tree.read(0, 0, &mut reader, 1, &mut inclusion_nodes), + Some(0) + ); + + let mut zbp_nodes = Vec::::new(); + let mut zbp_tree = TagTree::new(1, 1, &mut zbp_nodes); + assert_eq!( + zbp_tree.read(0, 0, &mut reader, u32::MAX, &mut zbp_nodes), + Some(2) + ); + + let passes = decode_num_ht_coding_passes_from_reader_for_test(&mut reader) + .expect("HT coding pass count"); + assert_eq!(passes, 3); + + let mut l_block = 3u32; + let mut length_bits = bits_for_ht_cleanup_length(l_block, passes); + while reader.read_bits_with_stuffing(1).expect("lblock increment") == 1 { + l_block += 1; + length_bits += 1; + } + assert_eq!(reader.read_bits_with_stuffing(length_bits as u8), Some(3)); + let refinement_bits = l_block + 1; + assert_eq!( + reader.read_bits_with_stuffing(refinement_bits as u8), + Some(2) + ); + assert_eq!(&packet[header_len..], payload.as_slice()); + } + + #[test] + fn ht_packet_segment_lengths_reject_overflowing_refinement_sum() { + let code_block = CodeBlockPacketData { + data: vec![0x12], + ht_cleanup_length: u32::MAX, + ht_refinement_length: 1, + num_coding_passes: 3, + classic_segment_lengths: Vec::new(), + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: BlockCodingMode::HighThroughput, + }; + + let err = ht_segment_lengths(&code_block).expect_err("overflowing HT lengths rejected"); + + assert_eq!(err, "multi-pass HTJ2K packet contribution length overflow"); + } + fn single_block_packet(data: Vec, previously_included: bool) -> ResolutionPacket { ResolutionPacket { subbands: vec![SubbandPrecinct { code_blocks: vec![CodeBlockPacketData { data, + ht_cleanup_length: 0, + ht_refinement_length: 0, num_coding_passes: 1, + classic_segment_lengths: Vec::new(), num_zero_bitplanes: 0, previously_included, l_block: 3, diff --git a/crates/signinum-j2k-native/src/j2c/progression.rs b/crates/j2k-native/src/j2c/progression.rs similarity index 77% rename from crates/signinum-j2k-native/src/j2c/progression.rs rename to crates/j2k-native/src/j2c/progression.rs index 11f9f913..ae98395f 100644 --- a/crates/signinum-j2k-native/src/j2c/progression.rs +++ b/crates/j2k-native/src/j2c/progression.rs @@ -8,6 +8,8 @@ use alloc::vec; use alloc::vec::Vec; use super::tile::{ComponentTile, ResolutionTile, Tile}; +use crate::error::{DecodingError, Result}; +use alloc::boxed::Box; use core::cmp::Ordering; use core::iter; @@ -38,11 +40,21 @@ impl<'a> IteratorInput<'a> { } pub(crate) fn new_with_custom_bounds( + tile: &'a Tile<'a>, + resolutions: (u8, u8), + layers: (u8, u8), + components: (u8, u8), + ) -> Self { + Self::try_new_with_custom_bounds(tile, resolutions, layers, components) + .expect("valid progression iterator bounds") + } + + pub(crate) fn try_new_with_custom_bounds( tile: &'a Tile<'a>, mut resolutions: (u8, u8), mut layers: (u8, u8), mut components: (u8, u8), - ) -> Self { + ) -> Option { let max_resolution = tile .component_infos .iter() @@ -57,16 +69,16 @@ impl<'a> IteratorInput<'a> { layers.1 = layers.1.min(max_layer); components.1 = components.1.min(max_component); - assert!(resolutions.1 > resolutions.0); - assert!(layers.1 > layers.0); - assert!(components.1 > components.0); + if resolutions.1 <= resolutions.0 || layers.1 <= layers.0 || components.1 <= components.0 { + return None; + } - Self { + Some(Self { layers, tile, resolutions, components, - } + }) } fn min_layer(&self) -> u8 { @@ -110,6 +122,58 @@ impl<'a> IteratorInput<'a> { } } +pub(crate) fn progression_iterator<'a>( + tile: &'a Tile<'a>, +) -> Result + 'a>> { + if tile.progression_changes.is_empty() { + return progression_iterator_for_order(tile.progression_order, IteratorInput::new(tile)); + } + + let mut iterators = Vec::with_capacity(tile.progression_changes.len()); + for change in &tile.progression_changes { + let iter_input = IteratorInput::try_new_with_custom_bounds( + tile, + (change.resolution_start, change.resolution_end), + (0, change.layer_end), + (change.component_start, change.component_end), + ) + .ok_or(DecodingError::InvalidProgressionIterator)?; + iterators.push(progression_iterator_for_order( + change.progression_order, + iter_input, + )?); + } + + Ok(Box::new(iterators.into_iter().flatten())) +} + +fn progression_iterator_for_order<'a>( + progression_order: super::codestream::ProgressionOrder, + iter_input: IteratorInput<'a>, +) -> Result + 'a>> { + let iterator: Box> = match progression_order { + super::codestream::ProgressionOrder::LayerResolutionComponentPosition => { + Box::new(layer_resolution_component_position_progression(iter_input)) + } + super::codestream::ProgressionOrder::ResolutionLayerComponentPosition => { + Box::new(resolution_layer_component_position_progression(iter_input)) + } + super::codestream::ProgressionOrder::ResolutionPositionComponentLayer => Box::new( + resolution_position_component_layer_progression(iter_input) + .ok_or(DecodingError::InvalidProgressionIterator)?, + ), + super::codestream::ProgressionOrder::PositionComponentResolutionLayer => Box::new( + position_component_resolution_layer_progression(iter_input) + .ok_or(DecodingError::InvalidProgressionIterator)?, + ), + super::codestream::ProgressionOrder::ComponentPositionResolutionLayer => Box::new( + component_position_resolution_layer_progression(iter_input) + .ok_or(DecodingError::InvalidProgressionIterator)?, + ), + }; + Ok(iterator) +} + /// B.12.1.1 Layer-resolution level-component-position progression. pub(crate) fn layer_resolution_component_position_progression<'a>( input: IteratorInput<'a>, @@ -178,9 +242,9 @@ pub(crate) fn resolution_layer_component_position_progression<'a>( ) -> impl Iterator + 'a { let component_tiles = input.component_tiles(); - let mut layer = 0; - let mut resolution = 0; - let mut component_idx = 0; + let mut layer = input.min_layer(); + let mut resolution = input.min_resolution(); + let mut component_idx = input.min_comp(); let mut resolution_tile = ResolutionTile::new(component_tiles[component_idx as usize], resolution); let mut precinct = 0; @@ -196,11 +260,11 @@ pub(crate) fn resolution_layer_component_position_progression<'a>( component_idx += 1; if component_idx == input.max_comp() { - component_idx = 0; + component_idx = input.min_comp(); layer += 1; if layer == input.max_layer() { - layer = 0; + layer = input.min_layer(); resolution += 1; if resolution == input.total_max_resolution() { diff --git a/crates/j2k-native/src/j2c/quantize.rs b/crates/j2k-native/src/j2c/quantize.rs new file mode 100644 index 00000000..c8f882dc --- /dev/null +++ b/crates/j2k-native/src/j2c/quantize.rs @@ -0,0 +1,429 @@ +//! Forward quantization for JPEG 2000 encoding. +//! +//! - Lossless (reversible 5-3): No quantization, just sign/magnitude conversion +//! - Lossy (irreversible 9-7): Scalar deadzone quantization with step sizes +//! derived from the DWT subband gain norms. + +use alloc::vec; +use alloc::vec::Vec; + +use crate::math::{floor_f32, log2_f32, pow2i, round_f32}; +use crate::{IrreversibleQuantizationStep, IrreversibleQuantizationSubbandScales, J2kSubBandType}; + +/// Quantization parameters for a single subband. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct QuantStepSize { + pub(crate) exponent: u16, + pub(crate) mantissa: u16, +} + +pub(crate) fn subband_scales_all_valid(scales: IrreversibleQuantizationSubbandScales) -> bool { + [ + scales.low_low, + scales.high_low, + scales.low_high, + scales.high_high, + ] + .iter() + .all(|scale| scale.is_finite() && *scale > 0.0) +} + +fn subband_scale_for_step_index( + scales: IrreversibleQuantizationSubbandScales, + index: usize, +) -> f32 { + if index == 0 { + return scales.low_low; + } + match (index - 1) % 3 { + 0 => scales.high_low, + 1 => scales.low_high, + _ => scales.high_high, + } +} + +fn subband_scale_for_subband( + scales: IrreversibleQuantizationSubbandScales, + subband: J2kSubBandType, +) -> f32 { + match subband { + J2kSubBandType::LowLow => scales.low_low, + J2kSubBandType::HighLow => scales.high_low, + J2kSubBandType::LowHigh => scales.low_high, + J2kSubBandType::HighHigh => scales.high_high, + } +} + +impl QuantStepSize { + /// Compute the JPEG 2000 irreversible step size: + /// Δ = 2^(R_b - exponent) × (1 + mantissa/2048). + fn delta(&self, range_bits: u8) -> f32 { + let rb = range_bits as i32 - self.exponent as i32; + let base = pow2i(rb); + base * (1.0 + self.mantissa as f32 / 2048.0) + } + + fn from_delta(range_bits: u8, delta: f32) -> Self { + debug_assert!(delta.is_finite() && delta > 0.0); + + let floor_log2 = floor_f32(log2_f32(delta)) as i32; + let mut exponent = i32::from(range_bits) - floor_log2; + let normalized = delta / pow2i(floor_log2); + let mut mantissa = round_f32((normalized - 1.0) * 2048.0) as i32; + + if mantissa >= 2048 { + exponent -= 1; + mantissa = 0; + } + + Self { + exponent: u16::try_from(exponent.clamp(0, 31)).expect("clamped exponent fits u16"), + mantissa: u16::try_from(mantissa.clamp(0, 2047)).expect("clamped mantissa fits u16"), + } + } +} + +/// Compute the exact irreversible 9/7 quantization step tuple the native encoder +/// writes for one subband under a global plus per-subband profile. +/// +/// # Panics +/// +/// Panics if the internal quantization step exponent is not clamped to the +/// JPEG 2000 exponent range before conversion. +#[must_use] +pub fn irreversible_quantization_step_for_subband( + bit_depth: u8, + guard_bits: u8, + irreversible_quantization_scale: f32, + irreversible_quantization_subband_scales: IrreversibleQuantizationSubbandScales, + subband: J2kSubBandType, +) -> IrreversibleQuantizationStep { + let base_step = QuantStepSize { + exponent: bit_depth as u16 + guard_bits as u16, + mantissa: 0, + }; + let scale = + if irreversible_quantization_scale.is_finite() && irreversible_quantization_scale > 0.0 { + irreversible_quantization_scale + } else { + 1.0 + }; + let subband_scales = if subband_scales_all_valid(irreversible_quantization_subband_scales) { + irreversible_quantization_subband_scales + } else { + IrreversibleQuantizationSubbandScales::default() + }; + let step_size = QuantStepSize::from_delta( + bit_depth, + base_step.delta(bit_depth) * scale * subband_scale_for_subband(subband_scales, subband), + ); + IrreversibleQuantizationStep { + exponent: u8::try_from(step_size.exponent).expect("step exponent is clamped to u8 range"), + mantissa: step_size.mantissa, + } +} + +/// Compute default quantization step sizes for the irreversible 9-7 transform. +/// +/// The step sizes are derived from the DWT 9-7 subband gain norms (Table E.1 in T.800). +/// For lossless mode, step sizes are not used (exponents store bit depth info only). +#[cfg(test)] +pub(crate) fn compute_step_sizes( + bit_depth: u8, + num_decompositions: u8, + reversible: bool, + guard_bits: u8, +) -> Vec { + compute_step_sizes_with_irreversible_scale( + bit_depth, + num_decompositions, + reversible, + guard_bits, + 1.0, + ) +} + +/// Compute quantization step sizes with an irreversible 9-7 scale multiplier. +/// +/// A scale of 1.0 preserves the quality-first default. Larger scales coarsen +/// the irreversible quantizer while keeping the same subband gain relationship. +#[cfg(test)] +pub(crate) fn compute_step_sizes_with_irreversible_scale( + bit_depth: u8, + num_decompositions: u8, + reversible: bool, + guard_bits: u8, + irreversible_quantization_scale: f32, +) -> Vec { + compute_step_sizes_with_irreversible_profile( + bit_depth, + num_decompositions, + reversible, + guard_bits, + irreversible_quantization_scale, + IrreversibleQuantizationSubbandScales::default(), + ) +} + +/// Compute quantization step sizes with global and per-subband irreversible +/// 9/7 scale multipliers. +pub(crate) fn compute_step_sizes_with_irreversible_profile( + bit_depth: u8, + num_decompositions: u8, + reversible: bool, + guard_bits: u8, + irreversible_quantization_scale: f32, + irreversible_quantization_subband_scales: IrreversibleQuantizationSubbandScales, +) -> Vec { + let mut step_sizes = Vec::new(); + + if reversible { + // For reversible 5-3, QCD stores the subband exponent only. + // The decoder reconstructs the number of bitplanes as: + // Mb = guard_bits + exponent - 1 + // For lossless coding we therefore need exponents that reproduce the + // reversible subband dynamic range: + // LL => bit_depth + 0 + // HL/LH => bit_depth + 1 + // HH => bit_depth + 2 + // This gain depends on subband orientation, not decomposition level. + step_sizes.push(QuantStepSize { + exponent: bit_depth as u16, + mantissa: 0, + }); + + for _ in 0..num_decompositions { + step_sizes.push(QuantStepSize { + exponent: bit_depth as u16 + 1, + mantissa: 0, + }); + step_sizes.push(QuantStepSize { + exponent: bit_depth as u16 + 1, + mantissa: 0, + }); + step_sizes.push(QuantStepSize { + exponent: bit_depth as u16 + 2, + mantissa: 0, + }); + } + } else { + // Quality-first irreversible 9-7 default. Use one exponent/mantissa for all + // subbands and let R_b = bit_depth + log_gain make LL finest and HH + // coarsest under the decoder's QCD formula. + let base_step = QuantStepSize { + exponent: bit_depth as u16 + guard_bits as u16, + mantissa: 0, + }; + let scale = if irreversible_quantization_scale.is_finite() + && irreversible_quantization_scale > 0.0 + { + irreversible_quantization_scale + } else { + 1.0 + }; + let subband_scales = if subband_scales_all_valid(irreversible_quantization_subband_scales) { + irreversible_quantization_subband_scales + } else { + IrreversibleQuantizationSubbandScales::default() + }; + let step_count = 1usize + usize::from(num_decompositions) * 3; + + for index in 0..step_count { + let subband_scale = subband_scale_for_step_index(subband_scales, index); + step_sizes.push(QuantStepSize::from_delta( + bit_depth, + base_step.delta(bit_depth) * scale * subband_scale, + )); + } + } + + step_sizes +} + +/// Quantize wavelet coefficients for a single subband. +/// +/// For lossless: converts f32 to i32 (round to nearest integer). +/// For lossy: applies scalar deadzone quantization. +/// +/// Returns (magnitude, sign) pairs packed as i32 values. +pub(crate) fn quantize_subband( + coefficients: &[f32], + step_size: &QuantStepSize, + range_bits: u8, + reversible: bool, +) -> Vec { + if reversible { + // No quantization: round to nearest integer + coefficients.iter().map(|&c| round_f32(c) as i32).collect() + } else { + let delta = step_size.delta(range_bits); + if delta <= 0.0 { + return vec![0i32; coefficients.len()]; + } + let inv_delta = 1.0 / delta; + + coefficients + .iter() + .map(|&c| { + // Deadzone quantization: q = sign(c) * floor(|c| / Δ) + let sign = if c < 0.0 { -1 } else { 1 }; + let magnitude = floor_f32(c.abs() * inv_delta) as i32; + sign * magnitude + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_lossless_quantize() { + let coeffs = vec![10.0, -5.0, 3.7, -8.2, 0.0]; + let step = QuantStepSize { + exponent: 12, + mantissa: 0, + }; + let result = quantize_subband(&coeffs, &step, 1, true); + assert_eq!(result, vec![10, -5, 4, -8, 0]); + } + + #[test] + fn test_lossy_quantize() { + let coeffs = vec![10.0, -5.0, 0.3, -0.1]; + let step = QuantStepSize { + exponent: 8, + mantissa: 0, + }; + let delta = step.delta(8); + assert!((delta - 1.0).abs() < 0.01); + + let result = quantize_subband(&coeffs, &step, 8, false); + assert_eq!(result[0], 10); + assert_eq!(result[1], -5); + assert_eq!(result[2], 0); // Below deadzone + assert_eq!(result[3], 0); // Below deadzone + } + + #[test] + fn test_compute_step_sizes_reversible() { + let steps = compute_step_sizes(8, 3, true, 1); + // 1 LL + 3 levels × 3 subbands = 10 + assert_eq!(steps.len(), 10); + // All mantissas should be 0 for reversible + assert!(steps.iter().all(|s| s.mantissa == 0)); + let exponents: Vec = steps.iter().map(|s| s.exponent).collect(); + assert_eq!(exponents, vec![8, 9, 9, 10, 9, 9, 10, 9, 9, 10]); + } + + #[test] + fn test_compute_step_sizes_irreversible() { + let steps = compute_step_sizes(8, 3, false, 1); + assert_eq!(steps.len(), 10); + } + + #[test] + fn irreversible_steps_match_decoder_qcd_contract() { + let steps = compute_step_sizes(8, 1, false, 2); + let exponents: Vec = steps.iter().map(|step| step.exponent).collect(); + let mantissas: Vec = steps.iter().map(|step| step.mantissa).collect(); + assert_eq!(exponents, vec![10, 10, 10, 10]); + assert_eq!(mantissas, vec![0, 0, 0, 0]); + + let deltas: Vec = [8u8, 9, 9, 10] + .iter() + .zip(&steps) + .map(|(&range_bits, step)| step.delta(range_bits)) + .collect(); + assert!((deltas[0] - 0.25).abs() < 0.001); + assert!((deltas[1] - 0.5).abs() < 0.001); + assert!((deltas[2] - 0.5).abs() < 0.001); + assert!((deltas[3] - 1.0).abs() < 0.001); + } + + #[test] + fn irreversible_quantization_scale_coarsens_qcd_deltas() { + let steps = compute_step_sizes_with_irreversible_scale(8, 1, false, 2, 4.0); + let exponents: Vec = steps.iter().map(|step| step.exponent).collect(); + let mantissas: Vec = steps.iter().map(|step| step.mantissa).collect(); + assert_eq!(exponents, vec![8, 8, 8, 8]); + assert_eq!(mantissas, vec![0, 0, 0, 0]); + + let deltas: Vec = [8u8, 9, 9, 10] + .iter() + .zip(&steps) + .map(|(&range_bits, step)| step.delta(range_bits)) + .collect(); + assert!((deltas[0] - 1.0).abs() < 0.001); + assert!((deltas[1] - 2.0).abs() < 0.001); + assert!((deltas[2] - 2.0).abs() < 0.001); + assert!((deltas[3] - 4.0).abs() < 0.001); + } + + #[test] + fn irreversible_quantization_scale_uses_mantissa_for_fractional_steps() { + let steps = compute_step_sizes_with_irreversible_scale(8, 1, false, 2, 5.0); + let exponents: Vec = steps.iter().map(|step| step.exponent).collect(); + let mantissas: Vec = steps.iter().map(|step| step.mantissa).collect(); + assert_eq!(exponents, vec![8, 8, 8, 8]); + assert_eq!(mantissas, vec![512, 512, 512, 512]); + + let deltas: Vec = [8u8, 9, 9, 10] + .iter() + .zip(&steps) + .map(|(&range_bits, step)| step.delta(range_bits)) + .collect(); + assert!((deltas[0] - 1.25).abs() < 0.001); + assert!((deltas[1] - 2.5).abs() < 0.001); + assert!((deltas[2] - 2.5).abs() < 0.001); + assert!((deltas[3] - 5.0).abs() < 0.001); + } + + #[test] + fn irreversible_subband_scales_change_only_selected_97_steps() { + let subband_scales = IrreversibleQuantizationSubbandScales { + low_low: 1.0, + high_low: 1.0, + low_high: 1.0, + high_high: 1.5, + }; + + let default_steps = compute_step_sizes_with_irreversible_profile( + 8, + 1, + false, + 2, + 1.9, + IrreversibleQuantizationSubbandScales::default(), + ); + let shaped_steps = + compute_step_sizes_with_irreversible_profile(8, 1, false, 2, 1.9, subband_scales); + + assert_eq!(shaped_steps[0], default_steps[0]); + assert_eq!(shaped_steps[1], default_steps[1]); + assert_eq!(shaped_steps[2], default_steps[2]); + assert!(shaped_steps[3].delta(10) > default_steps[3].delta(10)); + } + + #[test] + fn saturated_irreversible_coefficients_fit_declared_bitplanes() { + let guard_bits = 2; + let steps = compute_step_sizes(8, 1, false, guard_bits); + let range_bits = [8u8, 9, 9, 10]; + + for (&range_bits, step) in range_bits.iter().zip(&steps) { + let quantized = quantize_subband(&[-128.0, 127.0], step, range_bits, false); + let total_bitplanes = guard_bits as u16 + step.exponent - 1; + let max_abs = quantized + .iter() + .map(|coefficient| coefficient.unsigned_abs()) + .max() + .unwrap(); + assert!( + max_abs < (1u32 << total_bitplanes), + "range_bits={range_bits} step={step:?} quantized={quantized:?} total_bitplanes={total_bitplanes}" + ); + } + } +} diff --git a/crates/j2k-native/src/j2c/recode.rs b/crates/j2k-native/src/j2c/recode.rs new file mode 100644 index 00000000..8528cabd --- /dev/null +++ b/crates/j2k-native/src/j2c/recode.rs @@ -0,0 +1,249 @@ +//! Coefficient-domain JPEG 2000 family recode helpers. + +use alloc::vec::Vec; + +use super::build::{self, Decomposition, SubBand}; +use super::codestream::{ComponentInfo, Header, QuantizationStyle, WaveletTransform}; +use super::decode::{decode_component_tile_bit_planes, DecoderContext, DecompositionStorage}; +use super::progression::progression_iterator; +use super::segment; +use super::tile::{self, Tile}; +use crate::error::{bail, DecodingError, Result, TileError}; +use crate::reader::BitReader; +use crate::{ + J2kForwardDwt53Level, J2kForwardDwt53Output, PrecomputedHtj2k53Component, + PrecomputedHtj2k53Image, +}; + +/// Reversible 5/3 source coefficients ready for HTJ2K code-block recoding. +#[derive(Debug, Clone)] +pub struct Reversible53CoefficientImage { + /// Precomputed wavelet coefficients in the native HTJ2K encoder shape. + pub image: PrecomputedHtj2k53Image, + /// Source COD multi-component transform flag to preserve in output. + pub use_mct: bool, + /// Source code-block width exponent minus two. + pub code_block_width_exp: u8, + /// Source code-block height exponent minus two. + pub code_block_height_exp: u8, + /// Source quantization guard-bit count. + pub guard_bits: u8, +} + +pub(crate) fn extract_reversible_53_coefficients<'a>( + data: &'a [u8], + header: &Header<'a>, + ctx: &mut DecoderContext<'a>, +) -> Result { + validate_header_for_reversible_53_recode(header)?; + + let mut reader = BitReader::new(data); + let tiles = tile::parse(&mut reader, header)?; + if tiles.len() != 1 { + bail!(DecodingError::UnsupportedFeature( + "coefficient-domain 5/3 recode currently supports single-tile codestreams" + )); + } + + let tile = &tiles[0]; + validate_tile_for_reversible_53_recode(tile)?; + + ctx.tile_decode_context.channel_data.clear(); + ctx.storage.reset(); + + build::build(tile, &mut ctx.storage)?; + segment::parse(tile, progression_iterator(tile)?, header, &mut ctx.storage)?; + + let mut no_ht_decoder = None; + let cpu_decode_parallelism = ctx.cpu_decode_parallelism(); + decode_component_tile_bit_planes( + tile, + &mut ctx.tile_decode_context, + &mut ctx.storage, + header, + &mut no_ht_decoder, + cpu_decode_parallelism, + false, + )?; + + let image = precomputed_image_from_storage(header, tile, &ctx.storage)?; + let first = tile.component_infos.first().ok_or(TileError::Invalid)?; + let params = &first.coding_style.parameters; + Ok(Reversible53CoefficientImage { + image, + use_mct: tile.mct, + code_block_width_exp: params.code_block_width.saturating_sub(2), + code_block_height_exp: params.code_block_height.saturating_sub(2), + guard_bits: first.quantization_info.guard_bits, + }) +} + +fn validate_header_for_reversible_53_recode(header: &Header<'_>) -> Result<()> { + if header.skipped_resolution_levels != 0 { + bail!(DecodingError::UnsupportedFeature( + "coefficient-domain 5/3 recode requires full-resolution decode settings" + )); + } + if header.size_data.num_tiles() != 1 { + bail!(DecodingError::UnsupportedFeature( + "coefficient-domain 5/3 recode currently supports single-tile codestreams" + )); + } + if header.size_data.image_area_x_offset != 0 + || header.size_data.image_area_y_offset != 0 + || header.size_data.tile_x_offset != 0 + || header.size_data.tile_y_offset != 0 + { + bail!(DecodingError::UnsupportedFeature( + "coefficient-domain 5/3 recode currently requires zero image and tile origins" + )); + } + Ok(()) +} + +fn validate_tile_for_reversible_53_recode(tile: &Tile<'_>) -> Result<()> { + if !matches!(tile.component_infos.len(), 1 | 3) { + bail!(DecodingError::UnsupportedFeature( + "coefficient-domain 5/3 recode supports only grayscale or RGB codestreams" + )); + } + if tile.mct && tile.component_infos.len() != 3 { + bail!(DecodingError::UnsupportedFeature( + "reversible color transform requires three components" + )); + } + + let first = tile.component_infos.first().ok_or(TileError::Invalid)?; + let first_params = &first.coding_style.parameters; + let first_bit_depth = first.size_info.precision; + let first_guard_bits = first.quantization_info.guard_bits; + + for component in &tile.component_infos { + validate_component_for_reversible_53_recode(component)?; + if component.size_info.precision != first_bit_depth { + bail!(DecodingError::UnsupportedFeature( + "coefficient-domain 5/3 recode requires equal component bit depths" + )); + } + if component.quantization_info.guard_bits != first_guard_bits { + bail!(DecodingError::UnsupportedFeature( + "coefficient-domain 5/3 recode requires equal component guard bits" + )); + } + let params = &component.coding_style.parameters; + if params.num_decomposition_levels != first_params.num_decomposition_levels + || params.code_block_width != first_params.code_block_width + || params.code_block_height != first_params.code_block_height + { + bail!(DecodingError::UnsupportedFeature( + "coefficient-domain 5/3 recode requires matching component coding geometry" + )); + } + } + + Ok(()) +} + +fn validate_component_for_reversible_53_recode(component: &ComponentInfo) -> Result<()> { + if component.wavelet_transform() != WaveletTransform::Reversible53 { + bail!(DecodingError::UnsupportedFeature( + "coefficient-domain lossless recode currently supports only reversible 5/3 sources" + )); + } + if component.num_decomposition_levels() == 0 { + bail!(DecodingError::UnsupportedFeature( + "coefficient-domain 5/3 recode requires at least one decomposition level" + )); + } + if component.quantization_info.quantization_style != QuantizationStyle::NoQuantization { + bail!(DecodingError::UnsupportedFeature( + "coefficient-domain 5/3 recode requires no-quantization QCD/QCC" + )); + } + if component + .coding_style + .parameters + .code_block_style + .uses_high_throughput_block_coding() + { + bail!(DecodingError::UnsupportedFeature( + "source already uses HT block coding" + )); + } + Ok(()) +} + +fn precomputed_image_from_storage( + header: &Header<'_>, + tile: &Tile<'_>, + storage: &DecompositionStorage<'_>, +) -> Result { + let mut components = Vec::with_capacity(tile.component_infos.len()); + for (component_index, component_info) in tile.component_infos.iter().enumerate() { + let tile_decomposition = storage + .tile_decompositions + .get(component_index) + .ok_or(TileError::Invalid)?; + components.push(PrecomputedHtj2k53Component { + x_rsiz: component_info.size_info.horizontal_resolution, + y_rsiz: component_info.size_info.vertical_resolution, + dwt: component_dwt_from_storage(tile_decomposition, storage)?, + }); + } + + let first = tile.component_infos.first().ok_or(TileError::Invalid)?; + Ok(PrecomputedHtj2k53Image { + width: header.size_data.image_width(), + height: header.size_data.image_height(), + bit_depth: first.size_info.precision, + signed: false, + components, + }) +} + +fn component_dwt_from_storage( + tile_decomposition: &super::decode::TileDecompositions, + storage: &DecompositionStorage<'_>, +) -> Result { + let ll = storage + .sub_bands + .get(tile_decomposition.first_ll_sub_band) + .ok_or(TileError::Invalid)?; + + let mut levels = Vec::with_capacity(tile_decomposition.decompositions.len()); + for idx in tile_decomposition.decompositions.clone() { + let decomposition = storage.decompositions.get(idx).ok_or(TileError::Invalid)?; + levels.push(level_from_decomposition(decomposition, storage)); + } + + Ok(J2kForwardDwt53Output { + ll: subband_coefficients(ll, storage), + ll_width: ll.rect.width(), + ll_height: ll.rect.height(), + levels, + }) +} + +fn level_from_decomposition( + decomposition: &Decomposition, + storage: &DecompositionStorage<'_>, +) -> J2kForwardDwt53Level { + let hl = &storage.sub_bands[decomposition.sub_bands[0]]; + let lh = &storage.sub_bands[decomposition.sub_bands[1]]; + let hh = &storage.sub_bands[decomposition.sub_bands[2]]; + J2kForwardDwt53Level { + hl: subband_coefficients(hl, storage), + lh: subband_coefficients(lh, storage), + hh: subband_coefficients(hh, storage), + width: decomposition.rect.width(), + height: decomposition.rect.height(), + low_width: lh.rect.width(), + low_height: hl.rect.height(), + high_width: hl.rect.width(), + high_height: lh.rect.height(), + } +} + +fn subband_coefficients(subband: &SubBand, storage: &DecompositionStorage<'_>) -> Vec { + storage.coefficients[subband.coefficients.clone()].to_vec() +} diff --git a/crates/signinum-j2k-native/src/j2c/rect.rs b/crates/j2k-native/src/j2c/rect.rs similarity index 100% rename from crates/signinum-j2k-native/src/j2c/rect.rs rename to crates/j2k-native/src/j2c/rect.rs diff --git a/crates/signinum-j2k-native/src/j2c/roi.rs b/crates/j2k-native/src/j2c/roi.rs similarity index 100% rename from crates/signinum-j2k-native/src/j2c/roi.rs rename to crates/j2k-native/src/j2c/roi.rs diff --git a/crates/signinum-j2k-native/src/j2c/segment.rs b/crates/j2k-native/src/j2c/segment.rs similarity index 98% rename from crates/signinum-j2k-native/src/j2c/segment.rs rename to crates/j2k-native/src/j2c/segment.rs index ba057064..a74167d1 100644 --- a/crates/signinum-j2k-native/src/j2c/segment.rs +++ b/crates/j2k-native/src/j2c/segment.rs @@ -9,7 +9,7 @@ use super::decode::DecompositionStorage; use super::progression::ProgressionData; use super::tag_tree::TagNode; use super::tile::{Tile, TilePart}; -use crate::error::{bail, DecodingError, Result, TileError}; +use crate::error::{bail, Result, TileError}; use crate::reader::BitReader; pub(crate) const MAX_BITPLANE_COUNT: u8 = 32; @@ -20,20 +20,6 @@ pub(crate) fn parse<'a, 'b>( header: &Header<'_>, storage: &mut DecompositionStorage<'a>, ) -> Result<()> { - if tile.num_layers > 1 - && tile.component_infos.iter().any(|component_info| { - component_info - .coding_style - .parameters - .code_block_style - .uses_high_throughput_block_coding() - }) - { - bail!(DecodingError::UnsupportedFeature( - "multi-layer HTJ2K packet assembly" - )); - } - for tile_part in &tile.tile_parts { if parse_inner( tile_part.clone(), @@ -65,6 +51,7 @@ fn parse_inner<'a>( &mut storage.tile_decompositions[progression_data.component as usize]; let sub_band_iter = tile_decompositions.sub_band_iter(resolution, &storage.decompositions); + let packet_start = tile_part.packet_start_offset(); let body_reader = tile_part.body(); if component_info.coding_style.flags.may_use_sop_markers() @@ -135,8 +122,12 @@ fn parse_inner<'a>( } } } + + tile_part.validate_packet_length(packet_start)?; } + tile_part.validate_all_packet_lengths_consumed()?; + Some(()) } @@ -595,6 +586,7 @@ mod tests { use super::*; use crate::j2c::codestream::CodeBlockStyle; use crate::writer::BitWriter; + use alloc::vec::Vec; fn encode_ht_num_passes(num_passes: u8) -> Vec { let mut writer = BitWriter::new(); diff --git a/crates/signinum-j2k-native/src/j2c/tag_tree.rs b/crates/j2k-native/src/j2c/tag_tree.rs similarity index 100% rename from crates/signinum-j2k-native/src/j2c/tag_tree.rs rename to crates/j2k-native/src/j2c/tag_tree.rs diff --git a/crates/signinum-j2k-native/src/j2c/tag_tree_encode.rs b/crates/j2k-native/src/j2c/tag_tree_encode.rs similarity index 91% rename from crates/signinum-j2k-native/src/j2c/tag_tree_encode.rs rename to crates/j2k-native/src/j2c/tag_tree_encode.rs index e38442ea..6c16b9e6 100644 --- a/crates/signinum-j2k-native/src/j2c/tag_tree_encode.rs +++ b/crates/j2k-native/src/j2c/tag_tree_encode.rs @@ -93,14 +93,18 @@ impl TagTreeEncoder { // Parent's value is the minimum of all its children let child_x_start = cx * 2; let child_y_start = cy * 2; - let child_x_end = ((cx + 1) * 2).min(if level == 1 { self.width } else { cw * 2 }); - let child_y_end = ((cy + 1) * 2).min(if level == 1 { self.height } else { ch * 2 }); - let prev_w = if level == 1 { self.width } else { (self.width + (1 << (level - 1)) - 1) >> (level - 1) }; + let prev_h = if level == 1 { + self.height + } else { + (self.height + (1 << (level - 1)) - 1) >> (level - 1) + }; + let child_x_end = ((cx + 1) * 2).min(prev_w); + let child_y_end = ((cy + 1) * 2).min(prev_h); let mut min_val = u32::MAX; for ccy in child_y_start..child_y_end { @@ -198,6 +202,22 @@ mod tests { assert_eq!(tree.nodes.len(), 21); } + #[test] + fn one_row_odd_width_tree_sets_all_leaf_values() { + let mut tree = TagTreeEncoder::new(31, 1); + for x in 0..31 { + tree.set_value(x, 0, x); + } + } + + #[test] + fn one_column_odd_height_tree_sets_all_leaf_values() { + let mut tree = TagTreeEncoder::new(1, 31); + for y in 0..31 { + tree.set_value(0, y, y); + } + } + #[test] fn varied_8x8_values_round_trip_against_decoder_tag_tree() { let values = [ diff --git a/crates/signinum-j2k-native/src/j2c/tile.rs b/crates/j2k-native/src/j2c/tile.rs similarity index 81% rename from crates/signinum-j2k-native/src/j2c/tile.rs rename to crates/j2k-native/src/j2c/tile.rs index d07f418c..f65278cc 100644 --- a/crates/signinum-j2k-native/src/j2c/tile.rs +++ b/crates/j2k-native/src/j2c/tile.rs @@ -2,13 +2,17 @@ use alloc::vec; use alloc::vec::Vec; +use core::mem::size_of; use super::build::{PrecinctData, SubBandType}; -use super::codestream::{markers, skip_marker_segment, ComponentInfo, Header, ProgressionOrder}; +use super::codestream::{ + markers, skip_marker_segment, ComponentInfo, Header, ProgressionChange, ProgressionOrder, +}; use super::rect::IntRect; -use crate::error::{bail, err, MarkerError, Result, TileError, ValidationError}; +use crate::error::{bail, err, DecodingError, MarkerError, Result, TileError, ValidationError}; use crate::j2c::codestream; use crate::reader::BitReader; +use crate::DEFAULT_MAX_DECODE_BYTES; /// A single tile in the image. #[derive(Clone, Debug)] @@ -26,6 +30,7 @@ pub(crate) struct Tile<'a> { /// exclusive. pub(crate) rect: IntRect, pub(crate) progression_order: ProgressionOrder, + pub(crate) progression_changes: Vec, pub(crate) num_layers: u8, pub(crate) mct: bool, } @@ -34,6 +39,7 @@ pub(crate) struct Tile<'a> { #[derive(Clone, Debug)] pub(crate) struct MergedTilePart<'a> { pub(crate) data: BitReader<'a>, + packet_lengths: PacketLengthMetadata, } /// A tile part where packet headers and packet data are separated. @@ -42,6 +48,7 @@ pub(crate) struct SeparatedTilePart<'a> { pub(crate) headers: Vec>, pub(crate) active_header_reader: usize, pub(crate) body: BitReader<'a>, + packet_lengths: PacketLengthMetadata, } #[derive(Clone, Debug)] @@ -50,6 +57,47 @@ pub(crate) enum TilePart<'a> { Separated(SeparatedTilePart<'a>), } +#[derive(Clone, Debug, Default)] +struct PacketLengthMetadata { + present: bool, + lengths: Vec, + next: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PacketLengthExpectation { + NotTracked, + Length(u32), +} + +impl PacketLengthMetadata { + fn new(present: bool, lengths: Vec) -> Self { + Self { + present, + lengths, + next: 0, + } + } + + fn is_present(&self) -> bool { + self.present + } + + fn next(&mut self) -> Option { + if !self.present { + return Some(PacketLengthExpectation::NotTracked); + } + + let packet_length = self.lengths.get(self.next).copied()?; + self.next += 1; + Some(PacketLengthExpectation::Length(packet_length)) + } + + fn fully_consumed(&self) -> bool { + !self.present || self.next == self.lengths.len() + } +} + impl<'a> TilePart<'a> { pub(crate) fn header(&mut self) -> &mut BitReader<'a> { match self { @@ -72,6 +120,44 @@ impl<'a> TilePart<'a> { TilePart::Separated(s) => &mut s.body, } } + + pub(crate) fn packet_start_offset(&self) -> Option { + match self { + TilePart::Merged(m) if m.packet_lengths.is_present() => Some(m.data.offset()), + TilePart::Separated(_) => None, + TilePart::Merged(_) => None, + } + } + + pub(crate) fn validate_packet_length(&mut self, packet_start: Option) -> Option<()> { + let expected = match self { + TilePart::Merged(m) => m.packet_lengths.next()?, + TilePart::Separated(s) => s.packet_lengths.next()?, + }; + + let expected = match expected { + PacketLengthExpectation::NotTracked => return Some(()), + PacketLengthExpectation::Length(expected) => expected, + }; + + let packet_start = packet_start?; + let actual = match self { + TilePart::Merged(m) => m.data.offset().checked_sub(packet_start)?, + TilePart::Separated(_) => return Some(()), + }; + + if actual != expected as usize { + return None; + } + Some(()) + } + + pub(crate) fn validate_all_packet_lengths_consumed(&self) -> Option<()> { + match self { + TilePart::Merged(m) => m.packet_lengths.fully_consumed().then_some(()), + TilePart::Separated(s) => s.packet_lengths.fully_consumed().then_some(()), + } + } } impl<'a> Tile<'a> { @@ -82,23 +168,35 @@ impl<'a> Tile<'a> { let x_coord = size_data.tile_x_coord(idx); let y_coord = size_data.tile_y_coord(idx); - // See B-7, B-8, B-9 and B-10. + // See B-7, B-8, B-9 and B-10. Saturating arithmetic: the results + // are clamped against the reference grid anyway, and a saturated + // intermediate already exceeds every in-grid bound, so the clamp + // still produces the mathematically correct edge for crafted + // headers whose products overflow u32. let x0 = u32::max( - size_data.tile_x_offset + x_coord * size_data.tile_width, + size_data + .tile_x_offset + .saturating_add(x_coord.saturating_mul(size_data.tile_width)), size_data.image_area_x_offset, ); let y0 = u32::max( - size_data.tile_y_offset + y_coord * size_data.tile_height, + size_data + .tile_y_offset + .saturating_add(y_coord.saturating_mul(size_data.tile_height)), size_data.image_area_y_offset, ); // Note that `x1` and `y1` are exclusive. let x1 = u32::min( - size_data.tile_x_offset + (x_coord + 1) * size_data.tile_width, + size_data + .tile_x_offset + .saturating_add((x_coord + 1).saturating_mul(size_data.tile_width)), size_data.reference_grid_width, ); let y1 = u32::min( - size_data.tile_y_offset + (y_coord + 1) * size_data.tile_height, + size_data + .tile_y_offset + .saturating_add((y_coord + 1).saturating_mul(size_data.tile_height)), size_data.reference_grid_height, ); @@ -115,6 +213,7 @@ impl<'a> Tile<'a> { // might be overridden. component_infos: header.component_infos.clone(), progression_order: header.global_coding_style.progression_order, + progression_changes: header.progression_changes.clone(), mct: header.global_coding_style.mct, num_layers: header.global_coding_style.num_layers, } @@ -132,6 +231,8 @@ pub(crate) fn parse<'a>( reader: &mut BitReader<'a>, main_header: &Header<'a>, ) -> Result>> { + validate_tile_structural_budget(main_header)?; + let mut tiles = (0..main_header.size_data.num_tiles() as usize) .map(|idx| Tile::new(idx as u32, main_header)) .collect::>(); @@ -153,6 +254,32 @@ pub(crate) fn parse<'a>( Ok(tiles) } +fn validate_tile_structural_budget(main_header: &Header<'_>) -> Result<()> { + let num_tiles = usize::try_from(main_header.size_data.num_tiles()) + .map_err(|_| ValidationError::ImageTooLarge)?; + let component_count = main_header.component_infos.len(); + let progression_change_count = main_header.progression_changes.len(); + + let per_tile_components = size_of::() + .checked_mul(component_count) + .ok_or(ValidationError::ImageTooLarge)?; + let per_tile_progression_changes = size_of::() + .checked_mul(progression_change_count) + .ok_or(ValidationError::ImageTooLarge)?; + let per_tile_bytes = size_of::>() + .checked_add(per_tile_components) + .and_then(|bytes| bytes.checked_add(per_tile_progression_changes)) + .ok_or(ValidationError::ImageTooLarge)?; + let total_bytes = per_tile_bytes + .checked_mul(num_tiles) + .ok_or(ValidationError::ImageTooLarge)?; + + if total_bytes > DEFAULT_MAX_DECODE_BYTES { + bail!(ValidationError::ImageTooLarge); + } + Ok(()) +} + fn parse_tile_part<'a>( reader: &mut BitReader<'a>, main_header: &Header<'a>, @@ -184,6 +311,8 @@ fn parse_tile_part<'a>( let tile = &mut tiles[tile_part_header.tile_index as usize]; let num_components = tile.component_infos.len(); + let mut packet_length_markers = vec![]; + let mut packet_lengths_present = false; let mut ppt_headers = vec![]; loop { @@ -247,6 +376,25 @@ fn parse_tile_part<'a>( .ok_or(ValidationError::InvalidComponentMetadata)? .quantization_info = qcc.clone(); } + markers::POC => { + reader.read_marker()?; + tile.progression_changes.extend( + codestream::poc_marker(reader, num_components as u16, tile.num_layers) + .ok_or(MarkerError::ParseFailure("POC"))?, + ); + } + markers::RGN => { + reader.read_marker()?; + let rgn = codestream::rgn_marker(reader, num_components as u16) + .ok_or(MarkerError::ParseFailure("RGN"))?; + if rgn.style != 0 { + bail!(DecodingError::UnsupportedFeature("explicit ROI coding")); + } + tile.component_infos + .get_mut(rgn.component_index as usize) + .ok_or(ValidationError::InvalidComponentMetadata)? + .roi_shift = rgn.shift; + } markers::EOC => break, markers::PPT => { if !main_header.ppm_packets.is_empty() { @@ -257,9 +405,10 @@ fn parse_tile_part<'a>( ppt_headers.push(ppt_marker(reader).ok_or(MarkerError::ParseFailure("PPT"))?); } markers::PLT => { - // Can be inferred ourselves. reader.read_marker()?; - skip_marker_segment(reader).ok_or(MarkerError::ParseFailure("PLT"))?; + packet_lengths_present = true; + packet_length_markers + .push(codestream::plt_marker(reader).ok_or(MarkerError::ParseFailure("PLT"))?); } markers::COM => { reader.read_marker()?; @@ -290,6 +439,23 @@ fn parse_tile_part<'a>( ppt_headers.sort_by_key(|ppt_header| ppt_header.sequence_idx); let mut headers: Vec<_> = ppt_headers.iter().map(|i| BitReader::new(i.data)).collect(); + packet_length_markers.sort_by_key(|marker| marker.sequence_idx); + let use_main_header_packet_lengths = !packet_lengths_present + && !main_header.plm_packet_lengths.is_empty() + && main_header.size_data.num_tiles() == 1 + && tile_part_header.tile_part_index == 0 + && tile_part_header.num_tile_parts == 1; + let packet_lengths = if use_main_header_packet_lengths { + PacketLengthMetadata::new(true, main_header.plm_packet_lengths.clone()) + } else { + PacketLengthMetadata::new( + packet_lengths_present, + packet_length_markers + .into_iter() + .flat_map(|marker| marker.packet_lengths) + .collect(), + ) + }; if let Some(ppm_marker) = main_header.ppm_packets.get(tile_part_idx) { headers.push(BitReader::new(ppm_marker.data)); @@ -304,10 +470,12 @@ fn parse_tile_part<'a>( headers, active_header_reader: 0, body: BitReader::new(data), + packet_lengths, }) } else { TilePart::Merged(MergedTilePart { data: BitReader::new(data), + packet_lengths, }) }; @@ -687,6 +855,8 @@ impl<'a> ResolutionTile<'a> { struct TilePartHeader { tile_index: u16, tile_part_length: u32, + tile_part_index: u8, + num_tile_parts: u8, } struct PptMarkerData<'a> { @@ -714,12 +884,14 @@ fn sot_marker(reader: &mut BitReader<'_>) -> Option { let tile_part_length = reader.read_u32()?; // We infer those ourselves. - let _tile_part_index = reader.read_byte()? as u16; - let _num_tile_parts = reader.read_byte()?; + let tile_part_index = reader.read_byte()?; + let num_tile_parts = reader.read_byte()?; Some(TilePartHeader { tile_index, tile_part_length, + tile_part_index, + num_tile_parts, }) } @@ -764,6 +936,7 @@ mod tests { size_info: component_size_info_0, coding_style: dummy_component_coding_style.clone(), quantization_info: dummy_quantization_info.clone(), + roi_shift: 0, }; let component_size_info_1 = ComponentSizeInfo { @@ -776,6 +949,7 @@ mod tests { size_info: component_size_info_1, coding_style: dummy_component_coding_style.clone(), quantization_info: dummy_quantization_info.clone(), + roi_shift: 0, }; let size_data = SizeData { @@ -822,6 +996,8 @@ mod tests { }, }, component_infos: vec![], + progression_changes: vec![], + plm_packet_lengths: vec![], ppm_packets: vec![], skipped_resolution_levels: 0, strict: false, diff --git a/crates/signinum-j2k-native/src/jp2/box.rs b/crates/j2k-native/src/jp2/box.rs similarity index 100% rename from crates/signinum-j2k-native/src/jp2/box.rs rename to crates/j2k-native/src/jp2/box.rs diff --git a/crates/signinum-j2k-native/src/jp2/cdef.rs b/crates/j2k-native/src/jp2/cdef.rs similarity index 100% rename from crates/signinum-j2k-native/src/jp2/cdef.rs rename to crates/j2k-native/src/jp2/cdef.rs diff --git a/crates/signinum-j2k-native/src/jp2/cmap.rs b/crates/j2k-native/src/jp2/cmap.rs similarity index 93% rename from crates/signinum-j2k-native/src/jp2/cmap.rs rename to crates/j2k-native/src/jp2/cmap.rs index 01c33a00..c51646d4 100644 --- a/crates/signinum-j2k-native/src/jp2/cmap.rs +++ b/crates/j2k-native/src/jp2/cmap.rs @@ -7,6 +7,10 @@ use crate::jp2::ImageBoxes; use crate::reader::BitReader; pub(crate) fn parse(boxes: &mut ImageBoxes, data: &[u8]) -> Result<()> { + if data.is_empty() || !data.len().is_multiple_of(4) { + bail!(FormatError::InvalidBox); + } + let mut reader = BitReader::new(data); let mut entries = Vec::with_capacity(data.len() / 4); diff --git a/crates/signinum-j2k-native/src/jp2/colr.rs b/crates/j2k-native/src/jp2/colr.rs similarity index 100% rename from crates/signinum-j2k-native/src/jp2/colr.rs rename to crates/j2k-native/src/jp2/colr.rs diff --git a/crates/signinum-j2k-native/src/jp2/icc.rs b/crates/j2k-native/src/jp2/icc.rs similarity index 100% rename from crates/signinum-j2k-native/src/jp2/icc.rs rename to crates/j2k-native/src/jp2/icc.rs diff --git a/crates/signinum-j2k-native/src/jp2/mod.rs b/crates/j2k-native/src/jp2/mod.rs similarity index 100% rename from crates/signinum-j2k-native/src/jp2/mod.rs rename to crates/j2k-native/src/jp2/mod.rs diff --git a/crates/signinum-j2k-native/src/jp2/pclr.rs b/crates/j2k-native/src/jp2/pclr.rs similarity index 100% rename from crates/signinum-j2k-native/src/jp2/pclr.rs rename to crates/j2k-native/src/jp2/pclr.rs diff --git a/crates/j2k-native/src/lib.rs b/crates/j2k-native/src/lib.rs new file mode 100644 index 00000000..d88a8010 --- /dev/null +++ b/crates/j2k-native/src/lib.rs @@ -0,0 +1,4727 @@ +/*! +Internal pure-Rust JPEG 2000 codec engine for `j2k`. + +This module tree was imported from the `dicom-toolkit-jpeg2000` 0.5.0 crate +and adapted in-repo so `j2k` no longer depends on an external +production decoder crate. + +`dicom-toolkit-jpeg2000` is the JPEG 2000 engine used by `dicom-toolkit-rs`. +It is a maintained fork of the original `hayro-jpeg2000` project with +DICOM-focused extensions, including native-bit-depth decode for 8/12/16-bit +images and pure-Rust JPEG 2000 encoding. + +The crate can decode both raw JPEG 2000 codestreams (`.j2c`) and images wrapped +inside the JP2 container format. The decoder supports the vast majority of features +defined in the JPEG 2000 core coding system (ISO/IEC 15444-1) as well as some color +spaces from the extensions (ISO/IEC 15444-2). There are still some missing pieces +for some "obscure" features (for example support for progression order +changes in tile-parts), but the features that commonly appear in real-world +images are supported. + +The crate offers both a high-level 8-bit decode path for general image use and +a native-bit-depth decode path for integrations such as DICOM, plus encoder APIs +for emitting raw JPEG 2000 and HTJ2K codestreams. + +# Example +```rust,no_run +use j2k_native::{DecodeSettings, Image}; + +let data = std::fs::read("image.jp2").unwrap(); +let image = Image::new(&data, &DecodeSettings::default()).unwrap(); + +println!( + "{}x{} image in {:?} with alpha={}", + image.width(), + image.height(), + image.color_space(), + image.has_alpha(), +); + +let bitmap = image.decode().unwrap(); +``` + +If you want to see a more comprehensive example, please take a look +at the example in [GitHub](https://github.com/knopkem/dicom-toolkit-rs/blob/main/crates/dicom-toolkit-jpeg2000/examples/png.rs), +which shows the main steps needed to convert a JPEG 2000 image into PNG. + +# Testing +The decoder has been tested against 20.000+ images scraped from random PDFs +on the internet and also passes a large part of the `OpenJPEG` test suite. So you +can expect the crate to perform decently in terms of decoding correctness. + +# Performance +A decent amount of effort has already been put into optimizing this crate +(both in terms of raw performance but also memory allocations). However, there +are some more important optimizations that have not been implemented yet, so +there is definitely still room for improvement (and I am planning on implementing +them eventually). + +Overall, you should expect this crate to have worse performance than `OpenJPEG`, +but the difference gap should not be too large. + +# Safety +By default, the crate has the `simd` feature enabled, which uses the +[`fearless_simd`](https://github.com/linebender/fearless_simd) crate to accelerate +important parts of the pipeline. If you want to eliminate any usage of unsafe +in this crate as well as its dependencies, you can simply disable this +feature, at the cost of worse decoding performance. Unsafe code is forbidden +via a crate-level attribute. + +The crate is `no_std` compatible but requires an allocator to be available. +*/ + +#![cfg_attr(not(feature = "std"), no_std)] +#![forbid(unsafe_code)] +#![forbid(missing_docs)] +#![allow(clippy::too_many_arguments)] + +extern crate alloc; + +use alloc::vec; +use alloc::vec::Vec; + +use crate::error::{bail, err}; +use crate::j2c::{ComponentData, Header}; +use crate::jp2::cdef::{ChannelAssociation, ChannelType}; +use crate::jp2::cmap::ComponentMappingType; +use crate::jp2::colr::{CieLab, EnumeratedColorspace}; +use crate::jp2::icc::ICCMetadata; +use crate::jp2::{DecodedImage, ImageBoxes}; + +pub mod error; +mod inspect; +#[macro_use] +pub(crate) mod log; +mod direct_cpu; +mod direct_plan; +pub(crate) mod math; +#[doc(hidden)] +pub mod packet_math; +pub(crate) mod profile; +pub(crate) mod writer; + +use crate::math::{dispatch, f32x8, Level, Simd, SIMD_WIDTH}; +pub use direct_cpu::{ + execute_direct_color_plan_rgb8_into, execute_direct_color_plan_rgba8_into, J2kDirectCpuScratch, +}; +pub use direct_plan::{ + HtOwnedCodeBlockBatchJob, HtOwnedSubBandPlan, J2kDirectBandId, J2kDirectColorPlan, + J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kDirectIdwtStep, J2kDirectStoreStep, + J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, +}; +pub use inspect::{ + inspect_j2k_codestream_header, looks_like_j2k_codestream, J2kCodestreamComponentHeader, + J2kCodestreamHeaderError, J2kCodestreamHeaderMetadata, +}; + +/// Maps an output coordinate within an IDWT step to the source sub-band index. +/// +/// `origin` is the global coordinate of the IDWT output rectangle, +/// `local_coord` is the coordinate within that output rectangle, and +/// `low_pass` selects the low-pass (`LL`/`LH`) or high-pass (`HL`/`HH`) band +/// along one axis. This helper is exposed so backend adapters can compute +/// required input windows with the same odd-origin rounding as the native IDWT. +#[must_use] +pub fn idwt_band_index(origin: u32, local_coord: u32, low_pass: bool) -> u32 { + let global = u64::from(origin) + u64::from(local_coord); + let origin = u64::from(origin); + let index = if low_pass { + global.div_ceil(2).saturating_sub(origin.div_ceil(2)) + } else { + (global / 2).saturating_sub(origin / 2) + }; + u32::try_from(index).unwrap_or(u32::MAX) +} + +pub use error::{ + ColorError, DecodeError, DecodingError, FormatError, MarkerError, Result, TileError, + ValidationError, +}; +pub use j2c::encode::{ + encode, encode_htj2k, encode_precomputed_htj2k_53, + encode_precomputed_htj2k_53_with_accelerator, encode_precomputed_htj2k_53_with_mct, + encode_precomputed_htj2k_53_with_mct_and_accelerator, encode_precomputed_htj2k_97, + encode_precomputed_htj2k_97_batch_with_accelerator, + encode_precomputed_htj2k_97_with_accelerator, encode_preencoded_htj2k_97, + encode_preencoded_htj2k_97_compact_owned_with_accelerator, + encode_preencoded_htj2k_97_owned_with_accelerator, encode_preencoded_htj2k_97_with_accelerator, + encode_prequantized_htj2k_97, encode_prequantized_htj2k_97_with_accelerator, + encode_with_accelerator, irreversible_quantization_step_for_subband, EncodeOptions, + EncodeProgressionOrder, +}; +pub use j2c::{CpuDecodeParallelism, DecoderContext, Reversible53CoefficientImage}; +pub use j2k_types::{ + CpuOnlyJ2kEncodeStageAccelerator, EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, + IrreversibleQuantizationStep, IrreversibleQuantizationSubbandScales, J2kCodeBlockSegment, + J2kCodeBlockStyle, J2kDeinterleaveToF32Job, J2kEncodeDispatchReport, J2kForwardDwt53Job, + J2kForwardDwt53Level, J2kForwardDwt53Output, J2kForwardDwt97Job, J2kForwardDwt97Level, + J2kForwardDwt97Output, J2kForwardIctJob, J2kForwardRctJob, J2kHtCodeBlockEncodeJob, + J2kHtSubbandEncodeJob, J2kHtj2kTileEncodeJob, J2kPacketizationBlockCodingMode, + J2kPacketizationCodeBlock, J2kPacketizationEncodeJob, J2kPacketizationPacketDescriptor, + J2kPacketizationProgressionOrder, J2kPacketizationResolution, J2kPacketizationSubband, + J2kQuantizeSubbandJob, J2kSubBandType, J2kTier1CodeBlockEncodeJob, PrecomputedHtj2k53Component, + PrecomputedHtj2k53Image, PrecomputedHtj2k97Component, PrecomputedHtj2k97Image, + PreencodedHtj2k97CodeBlock, PreencodedHtj2k97CompactCodeBlock, + PreencodedHtj2k97CompactComponent, PreencodedHtj2k97CompactImage, + PreencodedHtj2k97CompactResolution, PreencodedHtj2k97CompactSubband, + PreencodedHtj2k97Component, PreencodedHtj2k97Image, PreencodedHtj2k97Resolution, + PreencodedHtj2k97Subband, PrequantizedHtj2k97CodeBlock, PrequantizedHtj2k97Component, + PrequantizedHtj2k97Image, PrequantizedHtj2k97Resolution, PrequantizedHtj2k97Subband, +}; + +mod j2c; +mod jp2; +pub(crate) mod reader; +pub use j2c::ht_encode_tables::HtUvlcTableEntry; + +const MAX_CLASSIC_DECODE_BITPLANES: u8 = 32; +pub(crate) const MAX_J2K_SPEC_COMPONENTS: u16 = 16_384; +pub(crate) const MAX_NATIVE_DECODE_COMPONENTS: u16 = u8::MAX as u16; +pub(crate) const MAX_J2K_IMAGE_DIMENSION: u32 = 60_000; +pub(crate) const MAX_J2K_TILE_COUNT: u64 = u16::MAX as u64 + 1; +pub(crate) const DEFAULT_MAX_DECODE_BYTES: usize = 512 * 1024 * 1024; + +#[inline] +pub(crate) fn checked_decode_usize_product2(left: usize, right: usize) -> Result { + left.checked_mul(right) + .ok_or(ValidationError::ImageTooLarge.into()) +} + +#[inline] +fn checked_decode_byte_cap(len: usize) -> Result { + if len > DEFAULT_MAX_DECODE_BYTES { + bail!(ValidationError::ImageTooLarge); + } + Ok(len) +} + +#[inline] +pub(crate) fn checked_decode_byte_len2(left: usize, right: usize) -> Result { + checked_decode_byte_cap(checked_decode_usize_product2(left, right)?) +} + +#[inline] +pub(crate) fn checked_decode_byte_len3(first: usize, second: usize, third: usize) -> Result { + let partial = checked_decode_usize_product2(first, second)?; + checked_decode_byte_cap(checked_decode_usize_product2(partial, third)?) +} + +#[inline] +pub(crate) fn checked_decode_byte_len4( + first: usize, + second: usize, + third: usize, + fourth: usize, +) -> Result { + let partial = checked_decode_usize_product2(first, second)?; + let partial = checked_decode_usize_product2(partial, third)?; + checked_decode_byte_cap(checked_decode_usize_product2(partial, fourth)?) +} + +#[inline] +pub(crate) fn checked_decode_sample_count(width: u32, height: u32) -> Result { + #[cfg(target_pointer_width = "64")] + { + Ok((u64::from(width) * u64::from(height)) as usize) + } + + #[cfg(not(target_pointer_width = "64"))] + { + checked_decode_usize_product2(width as usize, height as usize) + } +} + +/// Adapter HTJ2K code-block job description for backend experimentation. +#[derive(Debug, Clone, Copy)] +pub struct HtCodeBlockDecodeJob<'a> { + /// Combined cleanup/refinement bytes for the code block. + pub data: &'a [u8], + /// Cleanup segment length in bytes. + pub cleanup_length: u32, + /// Refinement segment length in bytes. + pub refinement_length: u32, + /// Code-block width in samples. + pub width: u32, + /// Code-block height in samples. + pub height: u32, + /// Output row stride, in samples, for the target sub-band storage. + pub output_stride: usize, + /// Missing most-significant bit planes for this code block. + pub missing_bit_planes: u8, + /// Number of coding passes present for this code block. + pub number_of_coding_passes: u8, + /// Total coded bitplanes for the parent sub-band. + pub num_bitplanes: u8, + /// Region-of-interest maxshift value from RGN marker metadata. + pub roi_shift: u8, + /// Whether vertically causal context was enabled. + pub stripe_causal: bool, + /// Whether strict decode validation is enabled for the parent image. + pub strict: bool, + /// Dequantization step to apply to decoded coefficients. + pub dequantization_step: f32, +} + +/// Adapter HTJ2K scalar decode phase limit for backend experimentation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HtCodeBlockDecodePhaseLimit { + /// Stop after the cleanup pass has produced coefficient magnitudes/signs. + Cleanup, + /// Stop after the significance propagation refinement pass. + SignificancePropagation, + /// Decode through the magnitude refinement pass when present. + MagnitudeRefinement, +} + +/// Adapter HTJ2K batched code-block decode job for one sub-band. +#[derive(Debug, Clone, Copy)] +pub struct HtCodeBlockBatchJob<'a> { + /// X offset within the target sub-band coefficient buffer. + pub output_x: u32, + /// Y offset within the target sub-band coefficient buffer. + pub output_y: u32, + /// The actual code-block decode parameters. + pub code_block: HtCodeBlockDecodeJob<'a>, +} + +/// Adapter HTJ2K batched sub-band decode request for backend experimentation. +#[derive(Debug, Clone, Copy)] +pub struct HtSubBandDecodeJob<'a> { + /// Sub-band width in samples. + pub width: u32, + /// Sub-band height in samples. + pub height: u32, + /// Code blocks to decode into this sub-band. + pub jobs: &'a [HtCodeBlockBatchJob<'a>], +} + +/// Adapter Classic Tier-1 compact token segment for backend experimentation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct J2kTier1TokenSegment { + /// Bit offset of this segment within the compact token buffer. + pub token_bit_offset: u32, + /// Number of token bits in this segment. + /// + /// Arithmetic segments contain 6-bit MQ tokens. Raw bypass segments contain + /// one bit per raw bypass event. + pub token_bit_count: u32, + /// First coding pass covered by this segment. + pub start_coding_pass: u8, + /// One-past-last coding pass covered by this segment. + pub end_coding_pass: u8, + /// Whether this segment should be packed through the MQ arithmetic path. + pub use_arithmetic: bool, +} + +/// Adapter classic J2K code-block job description for backend experimentation. +#[derive(Debug, Clone, Copy)] +pub struct J2kCodeBlockDecodeJob<'a> { + /// Combined payload bytes for all coded segments in this code block. + pub data: &'a [u8], + /// Coded segments for the code block. + pub segments: &'a [J2kCodeBlockSegment], + /// Code-block width in samples. + pub width: u32, + /// Code-block height in samples. + pub height: u32, + /// Output row stride, in samples, for the target sub-band storage. + pub output_stride: usize, + /// Missing most-significant bit planes for this code block. + pub missing_bit_planes: u8, + /// Number of coding passes present for this code block. + pub number_of_coding_passes: u8, + /// Total coded bitplanes for the parent sub-band. + pub total_bitplanes: u8, + /// Region-of-interest maxshift value from RGN marker metadata. + pub roi_shift: u8, + /// The sub-band type containing this code block. + pub sub_band_type: J2kSubBandType, + /// The code-block style flags. + pub style: J2kCodeBlockStyle, + /// Whether strict decode validation is enabled for the parent image. + pub strict: bool, + /// Dequantization step to apply to decoded coefficients. + pub dequantization_step: f32, +} + +/// Adapter HTJ2K cleanup-encode shape counters for backend benchmarking. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct HtCleanupEncodeDistribution { + /// Total 2x2 cleanup quads visited. + pub total_quads: u64, + /// Quads encoded in the first cleanup row pair. + pub initial_quads: u64, + /// Quads encoded after the first cleanup row pair. + pub non_initial_quads: u64, + /// All-quad `rho` histogram, indexed by the low four `rho` bits. + pub rho_counts: [u64; 16], + /// First-row-pair `rho` histogram, indexed by the low four `rho` bits. + pub initial_rho_counts: [u64; 16], + /// Non-initial-row `rho` histogram, indexed by the low four `rho` bits. + pub non_initial_rho_counts: [u64; 16], + /// Non-initial-row `u_q` histogram. + pub non_initial_u_q_counts: [u64; 32], + /// Non-initial-row `e_qmax` histogram. + pub non_initial_e_qmax_counts: [u64; 32], + /// Non-initial-row `kappa` histogram. + pub non_initial_kappa_counts: [u64; 32], + /// Non-initial-row joint `rho`/`u_q` histogram. + pub non_initial_rho_u_q_counts: [[u64; 32]; 16], + /// Calls that emitted at least one magnitude/sign sample. + pub mag_sign_calls: u64, + /// Magnitude/sign call histogram, indexed by the low four `rho` bits. + pub mag_sign_rho_counts: [u64; 16], + /// Encoded magnitude/sign sample payload bit-count histogram. + pub mag_sign_sample_bit_counts: [u64; 32], + /// Number of individual magnitude/sign samples emitted. + pub mag_sign_encoded_samples: u64, +} + +/// Adapter JPEG 2000 encode-stage accelerator for backend experimentation. +pub trait J2kEncodeStageAccelerator { + /// Report cumulative backend dispatches completed by this accelerator. + fn dispatch_report(&self) -> J2kEncodeDispatchReport { + J2kEncodeDispatchReport::default() + } + + /// Optionally deinterleave interleaved pixel bytes into f32 component planes. + /// + /// Return `Ok(Some(components))` with one plane per component. Return + /// `Ok(None)` to use the CPU fallback. + fn encode_deinterleave( + &mut self, + _job: J2kDeinterleaveToF32Job<'_>, + ) -> core::result::Result>>, &'static str> { + Ok(None) + } + + /// Optionally apply forward RCT in place. + /// + /// Return `Ok(true)` after writing transformed planes. Return `Ok(false)` + /// to use the CPU fallback. + fn encode_forward_rct( + &mut self, + _job: J2kForwardRctJob<'_>, + ) -> core::result::Result { + Ok(false) + } + + /// Optionally apply forward ICT in place. + /// + /// Return `Ok(true)` after writing transformed planes. Return `Ok(false)` + /// to use the CPU fallback. + fn encode_forward_ict( + &mut self, + _job: J2kForwardIctJob<'_>, + ) -> core::result::Result { + Ok(false) + } + + /// Optionally run a forward reversible 5/3 DWT. + /// + /// Return `Ok(Some(output))` with all subbands populated. Return + /// `Ok(None)` to use the CPU fallback. + fn encode_forward_dwt53( + &mut self, + _job: J2kForwardDwt53Job<'_>, + ) -> core::result::Result, &'static str> { + Ok(None) + } + + /// Optionally run a forward irreversible 9/7 DWT. + /// + /// Return `Ok(Some(output))` with all subbands populated. Return + /// `Ok(None)` to use the CPU fallback. + fn encode_forward_dwt97( + &mut self, + _job: J2kForwardDwt97Job<'_>, + ) -> core::result::Result, &'static str> { + Ok(None) + } + + /// Optionally quantize one sub-band. + /// + /// Return `Ok(Some(coefficients))` with one quantized coefficient for each + /// input coefficient. Return `Ok(None)` to use the CPU fallback. + fn encode_quantize_subband( + &mut self, + _job: J2kQuantizeSubbandJob<'_>, + ) -> core::result::Result>, &'static str> { + Ok(None) + } + + /// Optionally encode one classic Tier-1 code-block. + /// + /// Return `Ok(Some(output))` with encoded bytes and pass metadata. Return + /// `Ok(None)` to use the CPU fallback. + fn encode_tier1_code_block( + &mut self, + _job: J2kTier1CodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + Ok(None) + } + + /// Optionally encode multiple classic Tier-1 code-blocks in one backend dispatch. + /// + /// Return `Ok(Some(outputs))` with one encoded output per input job. Return + /// `Ok(None)` to use the per-block hook or CPU fallback. + fn encode_tier1_code_blocks( + &mut self, + _jobs: &[J2kTier1CodeBlockEncodeJob<'_>], + ) -> core::result::Result>, &'static str> { + Ok(None) + } + + /// Optionally encode one HTJ2K code-block. + /// + /// Return `Ok(Some(output))` with encoded bytes and pass metadata. Return + /// `Ok(None)` to use the CPU fallback. + fn encode_ht_code_block( + &mut self, + _job: J2kHtCodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + Ok(None) + } + + /// Optionally encode multiple HTJ2K code-blocks in one backend dispatch. + /// + /// Return `Ok(Some(outputs))` with one encoded output per input job. Return + /// `Ok(None)` to use the per-block hook or CPU fallback. + fn encode_ht_code_blocks( + &mut self, + _jobs: &[J2kHtCodeBlockEncodeJob<'_>], + ) -> core::result::Result>, &'static str> { + Ok(None) + } + + /// Optionally quantize and encode one HTJ2K cleanup-only sub-band. + /// + /// Return `Ok(Some(outputs))` with one encoded output per code block in + /// raster code-block order. Return `Ok(None)` to use the separate + /// quantization and code-block hooks or CPU fallback. + fn encode_ht_subband( + &mut self, + _job: J2kHtSubbandEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + Ok(None) + } + + /// Optionally encode the complete HTJ2K tile packet body. + /// + /// Return `Ok(Some(bytes))` with the complete tile bitstream body. CPU + /// marker/header writing remains outside this hook. Return `Ok(None)` to + /// use the normal staged encode pipeline. + fn encode_htj2k_tile( + &mut self, + _job: J2kHtj2kTileEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + Ok(None) + } + + /// Return whether native CPU code-block fallback should use internal rayon parallelism. + /// + /// External accelerators keep serial per-block fallback so their hooks still + /// observe every fallback block after a declined batch hook. + fn prefer_parallel_cpu_code_block_fallback(&self) -> bool { + false + } + + /// Return whether whole-tile CPU-only batch encode may be parallelized by callers. + /// + /// This is narrower than [`Self::prefer_parallel_cpu_code_block_fallback`]: + /// callers must only bypass the supplied accelerator when it is known to + /// have no observable hooks. + fn prefer_parallel_cpu_tile_encode(&self) -> bool { + false + } + + /// Optionally packetize prepared packet contributions. + /// + /// Return `Ok(Some(bytes))` with the complete tile bitstream. Return + /// `Ok(None)` to use the CPU fallback. + fn encode_packetization( + &mut self, + _job: J2kPacketizationEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + Ok(None) + } +} + +impl J2kEncodeStageAccelerator for CpuOnlyJ2kEncodeStageAccelerator { + fn prefer_parallel_cpu_code_block_fallback(&self) -> bool { + true + } + + fn prefer_parallel_cpu_tile_encode(&self) -> bool { + true + } +} + +/// Adapter classic J2K batched code-block decode job for one sub-band. +#[derive(Debug, Clone, Copy)] +pub struct J2kCodeBlockBatchJob<'a> { + /// X offset within the target sub-band coefficient buffer. + pub output_x: u32, + /// Y offset within the target sub-band coefficient buffer. + pub output_y: u32, + /// The actual code-block decode parameters. + pub code_block: J2kCodeBlockDecodeJob<'a>, +} + +/// Adapter classic J2K batched sub-band decode request for backend experimentation. +#[derive(Debug, Clone, Copy)] +pub struct J2kSubBandDecodeJob<'a> { + /// Sub-band width in samples. + pub width: u32, + /// Sub-band height in samples. + pub height: u32, + /// Code blocks to decode into this sub-band. + pub jobs: &'a [J2kCodeBlockBatchJob<'a>], +} + +/// Adapter integer rectangle for backend experimentation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct J2kRect { + /// Inclusive minimum x coordinate. + pub x0: u32, + /// Inclusive minimum y coordinate. + pub y0: u32, + /// Exclusive maximum x coordinate. + pub x1: u32, + /// Exclusive maximum y coordinate. + pub y1: u32, +} + +impl J2kRect { + /// Rectangle width in samples. + pub fn width(self) -> u32 { + self.x1.saturating_sub(self.x0) + } + + /// Rectangle height in samples. + pub fn height(self) -> u32 { + self.y1.saturating_sub(self.y0) + } +} + +/// Adapter wavelet transform selector for backend experimentation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum J2kWaveletTransform { + /// Reversible 5/3 transform. + Reversible53, + /// Irreversible 9/7 transform. + Irreversible97, +} + +/// Adapter single sub-band payload for backend experimentation. +#[derive(Debug, Clone, Copy)] +pub struct J2kIdwtBand<'a> { + /// Rect covered by this band. + pub rect: J2kRect, + /// Band coefficients in row-major order. + pub coefficients: &'a [f32], +} + +/// Adapter single-decomposition IDWT job for backend experimentation. +#[derive(Debug, Clone, Copy)] +pub struct J2kSingleDecompositionIdwtJob<'a> { + /// Output rect of the decomposition level. + pub rect: J2kRect, + /// Transform to apply. + pub transform: J2kWaveletTransform, + /// LL band input. + pub ll: J2kIdwtBand<'a>, + /// HL band input. + pub hl: J2kIdwtBand<'a>, + /// LH band input. + pub lh: J2kIdwtBand<'a>, + /// HH band input. + pub hh: J2kIdwtBand<'a>, +} + +/// Adapter inverse MCT job for backend experimentation. +#[derive(Debug)] +pub struct J2kInverseMctJob<'a> { + /// Transform to apply. + pub transform: J2kWaveletTransform, + /// First component plane, updated in place. + pub plane0: &'a mut [f32], + /// Second component plane, updated in place. + pub plane1: &'a mut [f32], + /// Third component plane, updated in place. + pub plane2: &'a mut [f32], + /// Constant sign-shift addend applied to the first plane after inverse MCT. + pub addend0: f32, + /// Constant sign-shift addend applied to the second plane after inverse MCT. + pub addend1: f32, + /// Constant sign-shift addend applied to the third plane after inverse MCT. + pub addend2: f32, +} + +/// Adapter component-store job for backend experimentation. +#[derive(Debug)] +pub struct J2kStoreComponentJob<'a> { + /// Source IDWT coefficients in row-major order. + pub input: &'a [f32], + /// Source row width. + pub input_width: u32, + /// Source x offset to begin copying from. + pub source_x: u32, + /// Source y offset to begin copying from. + pub source_y: u32, + /// Number of samples to copy per row. + pub copy_width: u32, + /// Number of rows to copy. + pub copy_height: u32, + /// Destination component plane in row-major order. + pub output: &'a mut [f32], + /// Destination row width. + pub output_width: u32, + /// Destination x offset to begin writing at. + pub output_x: u32, + /// Destination y offset to begin writing at. + pub output_y: u32, + /// Constant value added to every copied sample. + pub addend: f32, +} + +/// Adapter HTJ2K code-block decode hook for backend experimentation. +pub trait HtCodeBlockDecoder { + /// Optionally decode a full classic J2K sub-band in one batch. + /// + /// Implementations should return `Ok(true)` if they handled the request and + /// wrote the decoded coefficients into `output`. Returning `Ok(false)` + /// falls back to per-code-block decode via `decode_j2k_code_block`. + fn decode_j2k_sub_band( + &mut self, + _job: J2kSubBandDecodeJob<'_>, + _output: &mut [f32], + ) -> Result { + Ok(false) + } + + /// Optionally decode one classic J2K code block. + /// + /// Implementations should return `Ok(true)` if they handled the request + /// and wrote the decoded coefficients into `output`. Returning `Ok(false)` + /// falls back to the scalar bitplane decoder. + fn decode_j2k_code_block( + &mut self, + _job: J2kCodeBlockDecodeJob<'_>, + _output: &mut [f32], + ) -> Result { + Ok(false) + } + + /// Optionally decode a full HTJ2K sub-band in one batch. + /// + /// Implementations should return `Ok(true)` if they handled the request and + /// wrote the decoded coefficients into `output`. Returning `Ok(false)` + /// falls back to per-code-block decode via `decode_code_block`. + fn decode_sub_band( + &mut self, + _job: HtSubBandDecodeJob<'_>, + _output: &mut [f32], + ) -> Result { + Ok(false) + } + + /// Optionally decode one single-decomposition IDWT level on a backend. + /// + /// Implementations should return `Ok(true)` if they handled the request + /// and wrote the transformed coefficients into `output`. Returning + /// `Ok(false)` falls back to the scalar/SIMD CPU IDWT path. + fn decode_single_decomposition_idwt( + &mut self, + _job: J2kSingleDecompositionIdwtJob<'_>, + _output: &mut [f32], + ) -> Result { + Ok(false) + } + + /// Optionally apply inverse MCT on a backend. + /// + /// Implementations should return `Ok(true)` if they handled the request + /// and updated the component planes in place. Returning `Ok(false)` falls + /// back to the scalar/SIMD CPU MCT path. + fn decode_inverse_mct(&mut self, _job: J2kInverseMctJob<'_>) -> Result { + Ok(false) + } + + /// Optionally store one component plane on a backend. + /// + /// Implementations should return `Ok(true)` if they handled the request + /// and updated the destination plane in place. Returning `Ok(false)` falls + /// back to the CPU store path. + fn decode_store_component(&mut self, _job: J2kStoreComponentJob<'_>) -> Result { + Ok(false) + } + + /// Decode one HTJ2K code block into `output`, writing `job.width` samples per row. + fn decode_code_block( + &mut self, + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + ) -> Result<()>; +} + +fn internal_j2k_sub_band_type(sub_band_type: J2kSubBandType) -> j2c::build::SubBandType { + match sub_band_type { + J2kSubBandType::LowLow => j2c::build::SubBandType::LowLow, + J2kSubBandType::HighLow => j2c::build::SubBandType::HighLow, + J2kSubBandType::LowHigh => j2c::build::SubBandType::LowHigh, + J2kSubBandType::HighHigh => j2c::build::SubBandType::HighHigh, + } +} + +fn internal_j2k_code_block_style(style: J2kCodeBlockStyle) -> j2c::codestream::CodeBlockStyle { + j2c::codestream::CodeBlockStyle { + selective_arithmetic_coding_bypass: style.selective_arithmetic_coding_bypass, + reset_context_probabilities: style.reset_context_probabilities, + termination_on_each_pass: style.termination_on_each_pass, + vertically_causal_context: style.vertically_causal_context, + segmentation_symbols: style.segmentation_symbols, + high_throughput_block_coding: false, + } +} + +pub(crate) fn add_roi_shift_to_bitplanes( + bitplanes: u8, + roi_shift: u8, + max_bitplanes: u8, +) -> Result { + let Some(coded_bitplanes) = bitplanes.checked_add(roi_shift) else { + bail!(DecodingError::TooManyBitplanes); + }; + if coded_bitplanes > max_bitplanes { + bail!(DecodingError::TooManyBitplanes); + } + Ok(coded_bitplanes) +} + +pub(crate) fn apply_roi_maxshift_inverse_i32(coefficient: i32, roi_shift: u8) -> i32 { + if roi_shift == 0 || coefficient == 0 { + return coefficient; + } + + let magnitude = i64::from(coefficient).abs(); + let threshold = 1_i64.checked_shl(roi_shift as u32).unwrap_or(i64::MAX); + if magnitude < threshold { + return coefficient; + } + + let shifted = magnitude >> roi_shift; + let shifted = shifted.min(i64::from(i32::MAX)) as i32; + if coefficient < 0 { + -shifted + } else { + shifted + } +} + +/// Adapter scalar classic J2K encoder helper for backend experimentation. +pub fn encode_j2k_code_block_scalar_with_style( + coefficients: &[i32], + width: u32, + height: u32, + sub_band_type: J2kSubBandType, + total_bitplanes: u8, + style: J2kCodeBlockStyle, +) -> core::result::Result { + let encoded = j2c::bitplane_encode::encode_code_block_segments_with_style( + coefficients, + width, + height, + internal_j2k_sub_band_type(sub_band_type), + total_bitplanes, + &internal_j2k_code_block_style(style), + ); + let segments = encoded + .segments + .into_iter() + .map(|segment| J2kCodeBlockSegment { + data_offset: segment.data_offset, + data_length: segment.data_length, + start_coding_pass: segment.start_coding_pass, + end_coding_pass: segment.end_coding_pass, + use_arithmetic: segment.use_arithmetic, + }) + .collect(); + + Ok(EncodedJ2kCodeBlock { + data: encoded.data, + segments, + number_of_coding_passes: encoded.num_coding_passes, + missing_bit_planes: encoded.num_zero_bitplanes, + }) +} + +/// Adapter scalar Classic Tier-1 compact token packer for backend experimentation. +/// +/// The token format matches the Metal Classic Tier-1 token-emitter contract: +/// arithmetic segments are 6-bit `(context_label, bit)` MQ tokens, while raw +/// bypass segments are one bit per raw bypass event. +pub fn pack_j2k_code_block_scalar_from_tier1_tokens( + token_bytes: &[u8], + token_segments: &[J2kTier1TokenSegment], + number_of_coding_passes: u8, + missing_bit_planes: u8, +) -> core::result::Result { + let internal_segments = token_segments + .iter() + .map(|segment| j2c::bitplane_encode::ClassicTier1TokenSegment { + token_bit_offset: segment.token_bit_offset, + token_bit_count: segment.token_bit_count, + start_coding_pass: segment.start_coding_pass, + end_coding_pass: segment.end_coding_pass, + use_arithmetic: segment.use_arithmetic, + }) + .collect::>(); + let encoded = j2c::bitplane_encode::pack_classic_selective_bypass_tier1_tokens( + token_bytes, + &internal_segments, + number_of_coding_passes, + missing_bit_planes, + )?; + let segments = encoded + .segments + .into_iter() + .map(|segment| J2kCodeBlockSegment { + data_offset: segment.data_offset, + data_length: segment.data_length, + start_coding_pass: segment.start_coding_pass, + end_coding_pass: segment.end_coding_pass, + use_arithmetic: segment.use_arithmetic, + }) + .collect(); + + Ok(EncodedJ2kCodeBlock { + data: encoded.data, + segments, + number_of_coding_passes: encoded.num_coding_passes, + missing_bit_planes: encoded.num_zero_bitplanes, + }) +} + +/// Adapter scalar HTJ2K cleanup-only encoder helper for backend experimentation. +pub fn encode_ht_code_block_scalar( + coefficients: &[i32], + width: u32, + height: u32, + total_bitplanes: u8, +) -> core::result::Result { + let encoded = + j2c::ht_block_encode::encode_code_block(coefficients, width, height, total_bitplanes)?; + Ok(EncodedHtJ2kCodeBlock { + data: encoded.data, + cleanup_length: encoded.ht_cleanup_length, + refinement_length: encoded.ht_refinement_length, + num_coding_passes: encoded.num_coding_passes, + num_zero_bitplanes: encoded.num_zero_bitplanes, + }) +} + +/// Adapter HTJ2K cleanup-encode distribution helper for benchmark tuning. +pub fn collect_ht_cleanup_encode_distribution( + coefficients: &[i32], + width: u32, + height: u32, + total_bitplanes: u8, +) -> core::result::Result { + j2c::ht_block_encode::collect_encode_distribution(coefficients, width, height, total_bitplanes) +} + +/// Adapter scalar forward 5/3 DWT reference for CUDA stage parity. +/// +/// Runs the native CPU reversible 5/3 forward DWT on `samples` and returns +/// the decomposed subbands packed into the public `J2kForwardDwt53Output` +/// type. The returned layout matches what the encoder feeds to Tier-1. +pub fn forward_dwt53_reference( + samples: &[f32], + width: u32, + height: u32, + num_levels: u8, +) -> J2kForwardDwt53Output { + let decomp = j2c::fdwt::forward_dwt(samples, width, height, num_levels, true); + let levels = decomp + .levels + .into_iter() + .map(|lvl| J2kForwardDwt53Level { + hl: lvl.hl, + lh: lvl.lh, + hh: lvl.hh, + width: lvl.low_width + lvl.high_width, + height: lvl.low_height + lvl.high_height, + low_width: lvl.low_width, + low_height: lvl.low_height, + high_width: lvl.high_width, + high_height: lvl.high_height, + }) + .collect(); + J2kForwardDwt53Output { + ll: decomp.ll, + ll_width: decomp.ll_width, + ll_height: decomp.ll_height, + levels, + } +} + +/// Adapter scalar forward 9/7 DWT reference for Metal/CUDA stage parity. +/// +/// Runs the native CPU irreversible 9/7 forward DWT on `samples` and returns +/// the decomposed subbands packed into the public `J2kForwardDwt97Output` +/// type. The returned layout matches what the encoder feeds to Tier-1. +pub fn forward_dwt97_reference( + samples: &[f32], + width: u32, + height: u32, + num_levels: u8, +) -> J2kForwardDwt97Output { + let decomp = j2c::fdwt::forward_dwt(samples, width, height, num_levels, false); + let levels = decomp + .levels + .into_iter() + .map(|lvl| J2kForwardDwt97Level { + hl: lvl.hl, + lh: lvl.lh, + hh: lvl.hh, + width: lvl.low_width + lvl.high_width, + height: lvl.low_height + lvl.high_height, + low_width: lvl.low_width, + low_height: lvl.low_height, + high_width: lvl.high_width, + high_height: lvl.high_height, + }) + .collect(); + J2kForwardDwt97Output { + ll: decomp.ll, + ll_width: decomp.ll_width, + ll_height: decomp.ll_height, + levels, + } +} + +/// Adapter scalar forward RCT reference for CUDA stage parity. +/// +/// Applies the native CPU forward Reversible Color Transform to three +/// component planes supplied as owned `Vec` arrays. The transform is +/// applied in place and the mutated planes are returned, so callers do not +/// need to pass a mutable slice. +pub fn forward_rct_reference(mut planes: Vec>) -> Vec> { + j2c::forward_mct::forward_rct(&mut planes); + planes +} + +/// Adapter scalar forward ICT reference for Metal/CUDA stage parity. +/// +/// Applies the native CPU forward Irreversible Color Transform to three +/// component planes supplied as owned `Vec` arrays. The transform is +/// applied in place and the mutated planes are returned. +pub fn forward_ict_reference(mut planes: Vec>) -> Vec> { + j2c::forward_mct::forward_ict(&mut planes); + planes +} + +/// Adapter scalar sub-band quantization reference for backend stage parity. +pub fn quantize_subband_reference( + coefficients: &[f32], + step_exponent: u16, + step_mantissa: u16, + range_bits: u8, + reversible: bool, +) -> Vec { + let step = j2c::quantize::QuantStepSize { + exponent: step_exponent, + mantissa: step_mantissa, + }; + j2c::quantize::quantize_subband(coefficients, &step, range_bits, reversible) +} + +/// Adapter scalar reversible sub-band quantization reference for CUDA stage parity. +/// +/// Quantizes `coefficients` using the reversible (lossless) integer path of +/// the native CPU quantizer. `step_exponent` and `step_mantissa` encode the +/// JPEG 2000 `QuantStepSize` for the sub-band; `range_bits` is the nominal +/// bit depth for the sub-band. When `reversible` is `true` the step-size +/// parameters are ignored and each coefficient is rounded to the nearest +/// integer. +pub fn quantize_reversible_reference( + coefficients: &[f32], + step_exponent: u16, + step_mantissa: u16, + range_bits: u8, + reversible: bool, +) -> Vec { + quantize_subband_reference( + coefficients, + step_exponent, + step_mantissa, + range_bits, + reversible, + ) +} + +/// Adapter scalar pixel deinterleave/level-shift reference for CUDA stage parity. +/// +/// Converts interleaved pixel bytes to per-component f32 planes with the +/// same level-shift logic as the native CPU encode path. The result is one +/// `Vec` per component, each of length `num_pixels`. +pub fn deinterleave_reference( + pixels: &[u8], + num_pixels: usize, + num_components: u8, + bit_depth: u8, + signed: bool, +) -> Vec> { + j2c::encode::deinterleave_to_f32(pixels, num_pixels, num_components, bit_depth, signed) +} + +/// Adapter scalar Tier-2 packetization helper for backend experimentation. +pub fn encode_j2k_packetization_scalar( + job: J2kPacketizationEncodeJob<'_>, +) -> core::result::Result, &'static str> { + let mut resolutions = job + .resolutions + .iter() + .map(|resolution| j2c::packet_encode::ResolutionPacket { + subbands: resolution + .subbands + .iter() + .map(|subband| j2c::packet_encode::SubbandPrecinct { + code_blocks: subband + .code_blocks + .iter() + .map(|code_block| j2c::packet_encode::CodeBlockPacketData { + data: code_block.data.to_vec(), + ht_cleanup_length: code_block.ht_cleanup_length, + ht_refinement_length: code_block.ht_refinement_length, + num_coding_passes: code_block.num_coding_passes, + classic_segment_lengths: Vec::new(), + num_zero_bitplanes: code_block.num_zero_bitplanes, + previously_included: code_block.previously_included, + l_block: code_block.l_block, + block_coding_mode: match code_block.block_coding_mode { + J2kPacketizationBlockCodingMode::Classic => { + j2c::codestream_write::BlockCodingMode::Classic + } + J2kPacketizationBlockCodingMode::HighThroughput => { + j2c::codestream_write::BlockCodingMode::HighThroughput + } + }, + }) + .collect(), + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + }) + .collect(), + }) + .collect::>(); + + let descriptors = job + .packet_descriptors + .iter() + .map(|descriptor| j2c::packet_encode::PacketDescriptor { + packet_index: descriptor.packet_index, + state_index: descriptor.state_index, + layer: descriptor.layer, + resolution: descriptor.resolution, + component: descriptor.component, + precinct: descriptor.precinct, + }) + .collect::>(); + + j2c::packet_encode::validate_ht_segment_lengths(&resolutions)?; + + if descriptors.is_empty() { + Ok(j2c::packet_encode::form_tile_bitstream_for_progression( + &mut resolutions, + job.num_layers, + job.num_components, + job.progression_order, + )) + } else { + j2c::packet_encode::form_tile_bitstream_with_descriptors(&mut resolutions, &descriptors) + } +} + +/// Adapter scalar classic J2K decoder helper for backend experimentation. +pub fn decode_j2k_code_block_scalar( + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], +) -> Result<()> { + let mut workspace = J2kCodeBlockDecodeWorkspace::default(); + decode_j2k_code_block_scalar_with_workspace(job, output, &mut workspace) +} + +/// Reusable scratch for scalar classic J2K code-block decoding. +#[derive(Default)] +pub struct J2kCodeBlockDecodeWorkspace { + bit_plane_decode_context: j2c::bitplane::BitPlaneDecodeContext, +} + +/// Adapter scalar classic J2K decoder helper that reuses caller-provided scratch. +pub fn decode_j2k_code_block_scalar_with_workspace( + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], + workspace: &mut J2kCodeBlockDecodeWorkspace, +) -> Result<()> { + let required_len = if job.height == 0 { + 0 + } else { + job.output_stride + .checked_mul(job.height as usize - 1) + .and_then(|prefix| prefix.checked_add(job.width as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + }; + if output.len() < required_len { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + let style = internal_j2k_code_block_style(job.style); + let sub_band_type = internal_j2k_sub_band_type(job.sub_band_type); + let code_block_stride = + usize::try_from(job.width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + let coded_bitplanes = add_roi_shift_to_bitplanes( + job.total_bitplanes, + job.roi_shift, + MAX_CLASSIC_DECODE_BITPLANES, + )?; + + j2c::bitplane::decode_code_block_segments_validated( + job.data, + job.segments, + job.width, + job.height, + job.missing_bit_planes, + job.number_of_coding_passes, + coded_bitplanes, + sub_band_type, + &style, + job.strict, + &mut workspace.bit_plane_decode_context, + )?; + + for (row_idx, coeff_row) in workspace + .bit_plane_decode_context + .coefficient_rows() + .enumerate() + .take(job.height as usize) + { + let row_start = row_idx * job.output_stride; + let output_row = &mut output[row_start..row_start + code_block_stride]; + for (coefficient, sample) in coeff_row.iter().zip(output_row.iter_mut()) { + let coefficient = apply_roi_maxshift_inverse_i32(coefficient.get(), job.roi_shift); + *sample = coefficient as f32 * job.dequantization_step; + } + } + + Ok(()) +} + +/// Adapter scalar classic J2K pass timings for backend experimentation. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub struct J2kCodeBlockDecodeProfile { + /// Significance propagation pass elapsed time in microseconds. + pub sigprop_us: u128, + /// Magnitude refinement pass elapsed time in microseconds. + pub magref_us: u128, + /// Cleanup pass elapsed time in microseconds. + pub cleanup_us: u128, + /// Raw bypass pass elapsed time in microseconds. + pub bypass_us: u128, + /// Coefficient output conversion elapsed time in microseconds. + pub output_convert_us: u128, +} + +impl J2kCodeBlockDecodeProfile { + fn add_native_stats(&mut self, stats: j2c::bitplane::J2kBlockDecodeStats) { + self.sigprop_us += stats.sigprop_us; + self.magref_us += stats.magref_us; + self.cleanup_us += stats.cleanup_us; + self.bypass_us += stats.bypass_us; + } +} + +/// Adapter scalar classic J2K decoder helper that records pass timings. +pub fn decode_j2k_code_block_scalar_profiled( + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], + profile: &mut J2kCodeBlockDecodeProfile, +) -> Result<()> { + let mut workspace = J2kCodeBlockDecodeWorkspace::default(); + decode_j2k_code_block_scalar_with_workspace_profiled(job, output, &mut workspace, profile) +} + +/// Adapter scalar classic J2K decoder helper that records pass timings and reuses scratch. +pub fn decode_j2k_code_block_scalar_with_workspace_profiled( + job: J2kCodeBlockDecodeJob<'_>, + output: &mut [f32], + workspace: &mut J2kCodeBlockDecodeWorkspace, + profile: &mut J2kCodeBlockDecodeProfile, +) -> Result<()> { + let required_len = if job.height == 0 { + 0 + } else { + job.output_stride + .checked_mul(job.height as usize - 1) + .and_then(|prefix| prefix.checked_add(job.width as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + }; + if output.len() < required_len { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + let style = internal_j2k_code_block_style(job.style); + let sub_band_type = internal_j2k_sub_band_type(job.sub_band_type); + let code_block_stride = + usize::try_from(job.width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + let coded_bitplanes = add_roi_shift_to_bitplanes( + job.total_bitplanes, + job.roi_shift, + MAX_CLASSIC_DECODE_BITPLANES, + )?; + let mut stats = j2c::bitplane::J2kBlockDecodeStats::default(); + + j2c::bitplane::decode_code_block_segments_validated_profiled( + job.data, + job.segments, + job.width, + job.height, + job.missing_bit_planes, + job.number_of_coding_passes, + coded_bitplanes, + sub_band_type, + &style, + job.strict, + &mut workspace.bit_plane_decode_context, + &mut stats, + true, + )?; + profile.add_native_stats(stats); + + let output_convert_started = profile::profile_now(true); + for (row_idx, coeff_row) in workspace + .bit_plane_decode_context + .coefficient_rows() + .enumerate() + .take(job.height as usize) + { + let row_start = row_idx * job.output_stride; + let output_row = &mut output[row_start..row_start + code_block_stride]; + for (coefficient, sample) in coeff_row.iter().zip(output_row.iter_mut()) { + let coefficient = apply_roi_maxshift_inverse_i32(coefficient.get(), job.roi_shift); + *sample = coefficient as f32 * job.dequantization_step; + } + } + profile.output_convert_us += profile::elapsed_us(output_convert_started); + + Ok(()) +} + +/// Adapter scalar classic J2K batched decoder helper for backend experimentation. +pub fn decode_j2k_sub_band_scalar(job: J2kSubBandDecodeJob<'_>, output: &mut [f32]) -> Result<()> { + let required_len = if job.height == 0 { + 0 + } else { + usize::try_from(job.width) + .ok() + .and_then(|width| width.checked_mul(job.height as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + }; + if output.len() < required_len { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + let sub_band_width = + usize::try_from(job.width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + + for batch_job in job.jobs { + let code_block = batch_job.code_block; + if code_block.output_stride != sub_band_width { + bail!(DecodingError::CodeBlockDecodeFailure); + } + if batch_job + .output_x + .checked_add(code_block.width) + .is_none_or(|x1| x1 > job.width) + || batch_job + .output_y + .checked_add(code_block.height) + .is_none_or(|y1| y1 > job.height) + { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + let base_idx = usize::try_from(batch_job.output_y) + .ok() + .and_then(|y| y.checked_mul(sub_band_width)) + .and_then(|row| row.checked_add(batch_job.output_x as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + let block_output_len = if code_block.height == 0 { + 0 + } else { + code_block + .output_stride + .checked_mul(code_block.height as usize - 1) + .and_then(|prefix| prefix.checked_add(code_block.width as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + }; + let end_idx = base_idx + .checked_add(block_output_len) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + if end_idx > output.len() { + bail!(DecodingError::CodeBlockDecodeFailure); + } + + decode_j2k_code_block_scalar(code_block, &mut output[base_idx..end_idx])?; + } + + Ok(()) +} + +/// Adapter scalar HTJ2K decoder helper for backend experimentation. +pub fn decode_ht_code_block_scalar( + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], +) -> Result<()> { + decode_ht_code_block_scalar_for_phase::<{ j2c::ht_block_decode::PHASE_LIMIT_MAGREF }>( + job, output, + ) +} + +/// Adapter scalar HTJ2K decoder helper that stops after the selected phase. +pub fn decode_ht_code_block_scalar_until_phase( + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + phase_limit: HtCodeBlockDecodePhaseLimit, +) -> Result<()> { + match phase_limit { + HtCodeBlockDecodePhaseLimit::Cleanup => decode_ht_code_block_scalar_for_phase::< + { j2c::ht_block_decode::PHASE_LIMIT_CLEANUP }, + >(job, output), + HtCodeBlockDecodePhaseLimit::SignificancePropagation => { + decode_ht_code_block_scalar_for_phase::<{ j2c::ht_block_decode::PHASE_LIMIT_SIGPROP }>( + job, output, + ) + } + HtCodeBlockDecodePhaseLimit::MagnitudeRefinement => { + decode_ht_code_block_scalar_for_phase::<{ j2c::ht_block_decode::PHASE_LIMIT_MAGREF }>( + job, output, + ) + } + } +} + +/// Adapter reusable scalar HTJ2K decode workspace for backend experimentation. +#[derive(Default)] +pub struct HtCodeBlockDecodeWorkspace { + coefficients: Vec, + scratch: j2c::ht_block_decode::HtBlockDecodeScratch, +} + +impl HtCodeBlockDecodeWorkspace { + /// Current coefficient buffer capacity retained by this workspace. + pub fn coefficient_capacity(&self) -> usize { + self.coefficients.capacity() + } +} + +/// Adapter scalar HTJ2K phase timings for backend experimentation. +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] +pub struct HtCodeBlockDecodeProfile { + /// Number of decoded HT code blocks. + pub blocks: u128, + /// Number of decoded HT code blocks with refinement data. + pub refinement_blocks: u128, + /// Total cleanup segment bytes consumed by decoded HT code blocks. + pub cleanup_bytes: u128, + /// Total refinement segment bytes consumed by decoded HT code blocks. + pub refinement_bytes: u128, + /// Cleanup phase elapsed time in microseconds. + pub cleanup_us: u128, + /// Magnitude/sign phase elapsed time in microseconds. + pub mag_sgn_us: u128, + /// Sigma build phase elapsed time in microseconds. + pub sigma_us: u128, + /// Significance propagation phase elapsed time in microseconds. + pub sigprop_us: u128, + /// Magnitude refinement phase elapsed time in microseconds. + pub magref_us: u128, +} + +impl HtCodeBlockDecodeProfile { + fn add_native_stats(&mut self, stats: j2c::ht_block_decode::HtBlockDecodeStats) { + self.blocks += stats.blocks; + self.refinement_blocks += stats.refinement_blocks; + self.cleanup_bytes += stats.cleanup_bytes; + self.refinement_bytes += stats.refinement_bytes; + self.cleanup_us += stats.ht_cleanup_us; + self.mag_sgn_us += stats.ht_mag_sgn_us; + self.sigma_us += stats.ht_sigma_us; + self.sigprop_us += stats.ht_sigprop_us; + self.magref_us += stats.ht_magref_us; + } +} + +/// Adapter scalar HTJ2K decoder helper that reuses caller-owned scratch buffers. +pub fn decode_ht_code_block_scalar_with_workspace( + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + workspace: &mut HtCodeBlockDecodeWorkspace, +) -> Result<()> { + decode_ht_code_block_scalar_for_phase_with_workspace::< + { j2c::ht_block_decode::PHASE_LIMIT_MAGREF }, + >(job, output, workspace) +} + +/// Adapter scalar HTJ2K decoder helper that reuses scratch and records phase timings. +pub fn decode_ht_code_block_scalar_with_workspace_profiled( + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + workspace: &mut HtCodeBlockDecodeWorkspace, + profile: &mut HtCodeBlockDecodeProfile, +) -> Result<()> { + decode_ht_code_block_scalar_for_phase_with_workspace_profiled::< + { j2c::ht_block_decode::PHASE_LIMIT_MAGREF }, + >(job, output, workspace, profile) +} + +fn decode_ht_code_block_scalar_for_phase( + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], +) -> Result<()> { + let mut workspace = HtCodeBlockDecodeWorkspace::default(); + decode_ht_code_block_scalar_for_phase_with_workspace::(job, output, &mut workspace) +} + +fn decode_ht_code_block_scalar_for_phase_with_workspace( + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + workspace: &mut HtCodeBlockDecodeWorkspace, +) -> Result<()> { + let required_len = if job.height == 0 { + 0 + } else { + job.output_stride + .checked_mul(job.height as usize - 1) + .and_then(|prefix| prefix.checked_add(job.width as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + }; + if output.len() < required_len { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let code_block_stride = + usize::try_from(job.width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + let code_block_len = code_block_stride + .checked_mul(job.height as usize) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + + let segments = j2c::ht_block_decode::HtCodeBlockSegments::from_combined_payload( + job.data, + job.cleanup_length, + job.refinement_length, + )?; + let coded_bitplanes = add_roi_shift_to_bitplanes(job.num_bitplanes, job.roi_shift, 31)?; + workspace.coefficients.clear(); + workspace.coefficients.resize(code_block_len, 0); + j2c::ht_block_decode::decode_segments_validated_with_scratch_for_phase::( + &segments, + job.missing_bit_planes, + coded_bitplanes, + job.number_of_coding_passes, + job.stripe_causal, + job.strict, + &mut workspace.coefficients, + job.width, + job.height, + job.width, + &mut workspace.scratch, + None, + false, + )?; + + for (row_idx, coeff_row) in workspace + .coefficients + .chunks_exact(code_block_stride) + .enumerate() + .take(job.height as usize) + { + let row_start = row_idx * job.output_stride; + let output_row = &mut output[row_start..row_start + code_block_stride]; + for (coefficient, sample) in coeff_row.iter().copied().zip(output_row.iter_mut()) { + let coefficient = + j2c::ht_block_decode::coefficient_to_i32(coefficient, coded_bitplanes); + let coefficient = apply_roi_maxshift_inverse_i32(coefficient, job.roi_shift); + *sample = coefficient as f32 * job.dequantization_step; + } + } + + Ok(()) +} + +fn decode_ht_code_block_scalar_for_phase_with_workspace_profiled( + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + workspace: &mut HtCodeBlockDecodeWorkspace, + profile: &mut HtCodeBlockDecodeProfile, +) -> Result<()> { + let required_len = if job.height == 0 { + 0 + } else { + job.output_stride + .checked_mul(job.height as usize - 1) + .and_then(|prefix| prefix.checked_add(job.width as usize)) + .ok_or(DecodingError::CodeBlockDecodeFailure)? + }; + if output.len() < required_len { + bail!(DecodingError::CodeBlockDecodeFailure); + } + let code_block_stride = + usize::try_from(job.width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; + let code_block_len = code_block_stride + .checked_mul(job.height as usize) + .ok_or(DecodingError::CodeBlockDecodeFailure)?; + + let segments = j2c::ht_block_decode::HtCodeBlockSegments::from_combined_payload( + job.data, + job.cleanup_length, + job.refinement_length, + )?; + let coded_bitplanes = add_roi_shift_to_bitplanes(job.num_bitplanes, job.roi_shift, 31)?; + workspace.coefficients.clear(); + workspace.coefficients.resize(code_block_len, 0); + let mut stats = j2c::ht_block_decode::HtBlockDecodeStats::default(); + j2c::ht_block_decode::decode_segments_validated_with_scratch_for_phase::( + &segments, + job.missing_bit_planes, + coded_bitplanes, + job.number_of_coding_passes, + job.stripe_causal, + job.strict, + &mut workspace.coefficients, + job.width, + job.height, + job.width, + &mut workspace.scratch, + Some(&mut stats), + true, + )?; + profile.add_native_stats(stats); + + for (row_idx, coeff_row) in workspace + .coefficients + .chunks_exact(code_block_stride) + .enumerate() + .take(job.height as usize) + { + let row_start = row_idx * job.output_stride; + let output_row = &mut output[row_start..row_start + code_block_stride]; + for (coefficient, sample) in coeff_row.iter().copied().zip(output_row.iter_mut()) { + let coefficient = + j2c::ht_block_decode::coefficient_to_i32(coefficient, coded_bitplanes); + let coefficient = apply_roi_maxshift_inverse_i32(coefficient, job.roi_shift); + *sample = coefficient as f32 * job.dequantization_step; + } + } + + Ok(()) +} + +/// Adapter HTJ2K SigProp benchmark state for backend experimentation. +pub struct HtSigPropBenchmarkState(j2c::ht_block_decode::HtSigPropBenchmarkState); + +impl HtSigPropBenchmarkState { + /// Coefficient buffer length required by `decode_ht_sigprop_benchmark_state`. + pub fn output_len(&self) -> usize { + self.0.output_len() + } +} + +/// Adapter helper that precomputes cleanup-derived SigProp inputs for benchmarks. +pub fn prepare_ht_sigprop_benchmark_state( + job: HtCodeBlockDecodeJob<'_>, +) -> Result { + let segments = j2c::ht_block_decode::HtCodeBlockSegments::from_combined_payload( + job.data, + job.cleanup_length, + job.refinement_length, + )?; + let state = j2c::ht_block_decode::prepare_sigprop_benchmark_state( + &segments, + job.missing_bit_planes, + job.num_bitplanes, + job.number_of_coding_passes, + job.stripe_causal, + job.strict, + job.width, + job.height, + job.width, + )?; + Ok(HtSigPropBenchmarkState(state)) +} + +/// Adapter helper that runs only the HTJ2K significance-propagation phase. +pub fn decode_ht_sigprop_benchmark_state( + state: &mut HtSigPropBenchmarkState, + output: &mut [u32], +) -> Result<()> { + j2c::ht_block_decode::decode_sigprop_benchmark_state(&mut state.0, output) +} + +/// Adapter HTJ2K VLC table 0 for backend experimentation. +pub fn ht_vlc_table0() -> &'static [u16; 1024] { + &j2c::ht_tables::VLC_TABLE0 +} + +/// Adapter HTJ2K VLC table 1 for backend experimentation. +pub fn ht_vlc_table1() -> &'static [u16; 1024] { + &j2c::ht_tables::VLC_TABLE1 +} + +/// Adapter HTJ2K UVLC table 0 for backend experimentation. +pub fn ht_uvlc_table0() -> &'static [u16; 320] { + &j2c::ht_tables::UVLC_TABLE0 +} + +/// Adapter HTJ2K UVLC table 1 for backend experimentation. +pub fn ht_uvlc_table1() -> &'static [u16; 256] { + &j2c::ht_tables::UVLC_TABLE1 +} + +/// Adapter HTJ2K cleanup encoder VLC table 0 for backend experimentation. +pub fn ht_vlc_encode_table0() -> &'static [u16; 2048] { + &j2c::ht_encode_tables::HT_VLC_ENCODE_TABLE0 +} + +/// Adapter HTJ2K cleanup encoder VLC table 1 for backend experimentation. +pub fn ht_vlc_encode_table1() -> &'static [u16; 2048] { + &j2c::ht_encode_tables::HT_VLC_ENCODE_TABLE1 +} + +/// Adapter HTJ2K cleanup encoder UVLC table for backend experimentation. +pub fn ht_uvlc_encode_table() -> &'static [HtUvlcTableEntry; 75] { + &j2c::ht_encode_tables::HT_UVLC_ENCODE_TABLE +} + +/// Adapter HTJ2K cleanup encoder UVLC table packed for byte-addressed backends. +pub fn ht_uvlc_encode_table_bytes() -> &'static [u8] { + &j2c::ht_encode_tables::HT_UVLC_ENCODE_TABLE_BYTES +} + +/// JP2 signature box: 00 00 00 0C 6A 50 20 20 +pub(crate) const JP2_MAGIC: &[u8] = b"\x00\x00\x00\x0C\x6A\x50\x20\x20"; +/// Codestream signature: FF 4F FF 51 (SOC + SIZ markers) +pub(crate) const CODESTREAM_MAGIC: &[u8] = b"\xFF\x4F\xFF\x51"; + +/// Settings to apply during decoding. +#[derive(Debug, Copy, Clone)] +pub struct DecodeSettings { + /// Whether palette indices should be resolved. + /// + /// JPEG2000 images can be stored in two different ways. First, by storing + /// RGB values (depending on the color space) for each pixel. Secondly, by + /// only storing a single index for each channel, and then resolving the + /// actual color using the index. + /// + /// If you disable this option, in case you have an image with palette + /// indices, they will not be resolved, but instead a grayscale image + /// will be returned, with each pixel value corresponding to the palette + /// index of the location. + pub resolve_palette_indices: bool, + /// Whether strict mode should be enabled when decoding. + /// + /// It is recommended to leave this flag disabled, unless you have a + /// specific reason not to. + pub strict: bool, + /// A hint for the target resolution that the image should be decoded at. + pub target_resolution: Option<(u32, u32)>, +} + +impl Default for DecodeSettings { + fn default() -> Self { + Self { + resolve_palette_indices: true, + strict: false, + target_resolution: None, + } + } +} + +/// A JPEG2000 image or codestream. +pub struct Image<'a> { + /// The tile-part payload used by the legacy JPEG 2000 decoder. + pub(crate) codestream: &'a [u8], + /// The header of the J2C codestream. + pub(crate) header: Header<'a>, + /// The JP2 boxes of the image. In the case of a raw codestream, we + /// will synthesize the necessary boxes. + pub(crate) boxes: ImageBoxes, + /// Settings that should be applied during decoding. + pub(crate) settings: DecodeSettings, + /// Whether the image has an alpha channel. + pub(crate) has_alpha: bool, + /// The color space of the image. + pub(crate) color_space: ColorSpace, +} + +impl<'a> Image<'a> { + /// Try to create a new JPEG2000 image from the given data. + pub fn new(data: &'a [u8], settings: &DecodeSettings) -> Result { + if data.starts_with(JP2_MAGIC) { + jp2::parse(data, *settings) + } else if data.starts_with(CODESTREAM_MAGIC) { + j2c::parse(data, settings) + } else { + err!(FormatError::InvalidSignature) + } + } + + /// Whether the image has an alpha channel. + pub fn has_alpha(&self) -> bool { + self.has_alpha + } + + /// The color space of the image. + pub fn color_space(&self) -> &ColorSpace { + &self.color_space + } + + /// The width of the image. + pub fn width(&self) -> u32 { + self.header.size_data.image_width() + } + + /// The height of the image. + pub fn height(&self) -> u32 { + self.header.size_data.image_height() + } + + /// The original bit depth of the image. You usually don't need to do anything + /// with this parameter, it just exists for informational purposes. + pub fn original_bit_depth(&self) -> u8 { + // Note that this only works if all components have the same precision. + self.header.component_infos[0].size_info.precision + } + + /// Whether decode finishes with additional host-side component mutation or reordering. + pub fn supports_direct_device_plane_reuse(&self) -> bool { + if self.settings.resolve_palette_indices && self.boxes.palette.is_some() { + return false; + } + if self.boxes.channel_definition.is_some() { + return false; + } + !matches!( + self.boxes + .color_specification + .as_ref() + .map(|spec| &spec.color_space), + Some(jp2::colr::ColorSpace::Enumerated( + EnumeratedColorspace::Sycc | EnumeratedColorspace::CieLab(_) + )) + ) + } + + /// Decode the image and return its decoded result as a `Vec`, with each + /// channel interleaved. + pub fn decode(&self) -> Result> { + let bitmap = self.decode_with_context(&mut DecoderContext::default())?; + Ok(bitmap.data) + } + + /// Decode the image and return its decoded result using a caller-provided + /// decoder context so allocations can be reused across repeated decodes. + pub fn decode_with_context(&self, decoder_context: &mut DecoderContext<'a>) -> Result { + let buffer_size = checked_decode_byte_len3( + self.width() as usize, + self.height() as usize, + self.color_space.num_channels() as usize + if self.has_alpha { 1 } else { 0 }, + )?; + let mut buf = vec![0; buffer_size]; + self.decode_into(&mut buf, decoder_context)?; + + Ok(Bitmap { + color_space: self.color_space.clone(), + data: buf, + has_alpha: self.has_alpha, + width: self.width(), + height: self.height(), + original_bit_depth: self.original_bit_depth(), + }) + } + + /// Decode the image into borrowed component planes using a caller-provided + /// decoder context so allocations can be reused across repeated decodes. + pub fn decode_components_with_context<'ctx>( + &self, + decoder_context: &'ctx mut DecoderContext<'a>, + ) -> Result> { + let decoded_image = self.prepare_decoded_image(decoder_context)?; + let planes = decoded_image + .decoded_components + .iter() + .map(|component| ComponentPlane { + samples: component.container.truncated(), + bit_depth: component.bit_depth, + }) + .collect(); + + Ok(DecodedComponents { + dimensions: (self.width(), self.height()), + color_space: self.color_space.clone(), + has_alpha: self.has_alpha, + planes, + }) + } + + /// Build a adapter grayscale direct device plan without materializing host component planes. + pub fn build_direct_grayscale_plan_with_context( + &self, + decoder_context: &mut DecoderContext<'a>, + ) -> Result { + if !matches!(self.color_space, ColorSpace::Gray) || self.has_alpha { + bail!(DecodingError::UnsupportedFeature( + "direct grayscale plan only supports grayscale images without alpha" + )); + } + + j2c::build_direct_grayscale_plan(self.codestream, &self.header, decoder_context) + } + + /// Build a adapter grayscale direct device plan for an output-space region. + pub fn build_direct_grayscale_plan_region_with_context( + &self, + decoder_context: &mut DecoderContext<'a>, + output_region: (u32, u32, u32, u32), + ) -> Result { + if !matches!(self.color_space, ColorSpace::Gray) || self.has_alpha { + bail!(DecodingError::UnsupportedFeature( + "direct grayscale plan only supports grayscale images without alpha" + )); + } + + decoder_context.set_output_region(Some(output_region)); + let result = + j2c::build_direct_grayscale_plan(self.codestream, &self.header, decoder_context); + decoder_context.set_output_region(None); + result + } + + /// Build a adapter RGB direct device plan without materializing host component planes. + pub fn build_direct_color_plan_with_context( + &self, + decoder_context: &mut DecoderContext<'a>, + ) -> Result { + if !matches!(self.color_space, ColorSpace::RGB) || self.has_alpha { + bail!(DecodingError::UnsupportedFeature( + "direct color plan only supports RGB images without alpha" + )); + } + + j2c::build_direct_color_plan(self.codestream, &self.header, decoder_context) + } + + /// Build a adapter RGB direct device plan for an output-space region. + pub fn build_direct_color_plan_region_with_context( + &self, + decoder_context: &mut DecoderContext<'a>, + output_region: (u32, u32, u32, u32), + ) -> Result { + if !matches!(self.color_space, ColorSpace::RGB) || self.has_alpha { + bail!(DecodingError::UnsupportedFeature( + "direct color plan only supports RGB images without alpha" + )); + } + + decoder_context.set_output_region(Some(output_region)); + let result = j2c::build_direct_color_plan(self.codestream, &self.header, decoder_context); + decoder_context.set_output_region(None); + result + } + + /// Decode borrowed component planes while delegating HTJ2K code-block decode. + pub fn decode_components_with_ht_decoder<'ctx>( + &self, + decoder_context: &'ctx mut DecoderContext<'a>, + ht_decoder: &mut dyn HtCodeBlockDecoder, + ) -> Result> { + let decoded_image = + self.prepare_decoded_image_with_ht_decoder(decoder_context, ht_decoder)?; + let planes = decoded_image + .decoded_components + .iter() + .map(|component| ComponentPlane { + samples: component.container.truncated(), + bit_depth: component.bit_depth, + }) + .collect(); + + Ok(DecodedComponents { + dimensions: (self.width(), self.height()), + color_space: self.color_space.clone(), + has_alpha: self.has_alpha, + planes, + }) + } + + /// Decode borrowed component planes for a requested region using a + /// caller-provided decoder context. + pub fn decode_region_components_with_context<'ctx>( + &self, + roi: (u32, u32, u32, u32), + decoder_context: &'ctx mut DecoderContext<'a>, + ) -> Result> { + validate_roi((self.width(), self.height()), roi)?; + let (_x, _y, width, height) = roi; + let decoded_image = self.prepare_decoded_image_with_region(decoder_context, Some(roi))?; + let planes = decoded_image + .decoded_components + .iter() + .map(|component| ComponentPlane { + samples: component.container.truncated(), + bit_depth: component.bit_depth, + }) + .collect(); + + Ok(DecodedComponents { + dimensions: (width, height), + color_space: self.color_space.clone(), + has_alpha: self.has_alpha, + planes, + }) + } + + /// Decode borrowed component planes for a requested region while + /// delegating code-block/transform stages through the adapter backend hook. + pub fn decode_region_components_with_ht_decoder<'ctx>( + &self, + decoder_context: &'ctx mut DecoderContext<'a>, + roi: (u32, u32, u32, u32), + ht_decoder: &mut dyn HtCodeBlockDecoder, + ) -> Result> { + validate_roi((self.width(), self.height()), roi)?; + let (_x, _y, width, height) = roi; + let decoded_image = self.prepare_decoded_image_with_region_and_ht_decoder( + decoder_context, + Some(roi), + Some(ht_decoder), + )?; + let planes = decoded_image + .decoded_components + .iter() + .map(|component| ComponentPlane { + samples: component.container.truncated(), + bit_depth: component.bit_depth, + }) + .collect(); + + Ok(DecodedComponents { + dimensions: (width, height), + color_space: self.color_space.clone(), + has_alpha: self.has_alpha, + planes, + }) + } + + /// Decode a region of the image and return it as an 8-bit interleaved bitmap. + pub fn decode_region(&self, roi: (u32, u32, u32, u32)) -> Result { + self.decode_region_with_context(roi, &mut DecoderContext::default()) + } + + /// Decode a region of the image and return it as an 8-bit interleaved bitmap + /// using a caller-provided decoder context. + pub fn decode_region_with_context( + &self, + roi: (u32, u32, u32, u32), + decoder_context: &mut DecoderContext<'a>, + ) -> Result { + validate_roi((self.width(), self.height()), roi)?; + let mut decoded_image = + self.prepare_decoded_image_with_region(decoder_context, Some(roi))?; + let (_x, _y, width, height) = roi; + let channels = + self.color_space.num_channels() as usize + if self.has_alpha { 1 } else { 0 }; + let data_len = checked_decode_byte_len3(width as usize, height as usize, channels)?; + let mut data = vec![0; data_len]; + interleave_and_convert_region( + &mut decoded_image, + width as usize, + (0, 0, width, height), + &mut data, + ); + Ok(Bitmap { + color_space: self.color_space.clone(), + data, + has_alpha: self.has_alpha, + width, + height, + original_bit_depth: self.original_bit_depth(), + }) + } + + /// Decode the image at native bit depth without scaling to 8-bit. + /// + /// For images with bit depth ≤ 8, returns pixel data as `Vec`. + /// For images with bit depth > 8 (e.g., 12-bit or 16-bit), returns + /// pixel data as little-endian `u16` values packed into `Vec`. + /// + /// This is essential for medical imaging (DICOM) where 12-bit and 16-bit + /// images must preserve their full dynamic range. + pub fn decode_native(&self) -> Result { + let mut decoder_context = DecoderContext::default(); + self.decode_native_with_context(&mut decoder_context) + } + + /// Extract reversible 5/3 wavelet coefficients for coefficient-domain + /// classic JPEG 2000 to HTJ2K recoding. + /// + /// This decodes classic Tier-1 code-blocks into dequantized reversible + /// wavelet coefficients, but does not run inverse DWT or color conversion. + pub fn decode_reversible_53_coefficients(&self) -> Result { + let mut decoder_context = DecoderContext::default(); + self.decode_reversible_53_coefficients_with_context(&mut decoder_context) + } + + /// Extract reversible 5/3 wavelet coefficients using a caller-provided + /// decoder context. + pub fn decode_reversible_53_coefficients_with_context( + &self, + decoder_context: &mut DecoderContext<'a>, + ) -> Result { + j2c::recode::extract_reversible_53_coefficients( + self.codestream, + &self.header, + decoder_context, + ) + } + + /// Decode a region of the image at native bit depth. + pub fn decode_native_region(&self, roi: (u32, u32, u32, u32)) -> Result { + self.decode_native_region_with_context(roi, &mut DecoderContext::default()) + } + + /// Decode the image at native bit depth using a caller-provided decoder + /// context so allocations can be reused across repeated decodes. + pub fn decode_native_with_context( + &self, + decoder_context: &mut DecoderContext<'a>, + ) -> Result { + self.decode_with_output_region(decoder_context, None)?; + + let components = &decoder_context.tile_decode_context.channel_data; + let bit_depth = self.original_bit_depth(); + let num_components = + u8::try_from(components.len()).map_err(|_| ValidationError::TooManyChannels)?; + let width = self.width(); + let height = self.height(); + let pixel_count = checked_decode_sample_count(width, height)?; + + if bit_depth <= 8 { + let max_val = ((1u32 << bit_depth) - 1) as f32; + let capacity = checked_decode_byte_len2(pixel_count, num_components as usize)?; + let mut data = Vec::with_capacity(capacity); + for i in 0..pixel_count { + for component in components.iter() { + let v = math::round_f32(component.container.truncated()[i]); + let clamped = if v < 0.0 { + 0.0 + } else if v > max_val { + max_val + } else { + v + }; + data.push(clamped as u8); + } + } + Ok(RawBitmap { + data, + width, + height, + bit_depth, + num_components, + bytes_per_sample: 1, + }) + } else { + let max_val = ((1u32 << bit_depth) - 1) as f32; + let capacity = checked_decode_byte_len3(pixel_count, num_components as usize, 2)?; + let mut data = Vec::with_capacity(capacity); + for i in 0..pixel_count { + for component in components.iter() { + let v = math::round_f32(component.container.truncated()[i]); + let clamped = if v < 0.0 { + 0.0 + } else if v > max_val { + max_val + } else { + v + }; + let val = clamped as u16; + data.extend_from_slice(&val.to_le_bytes()); + } + } + Ok(RawBitmap { + data, + width, + height, + bit_depth, + num_components, + bytes_per_sample: 2, + }) + } + } + + /// Decode a region of the image at native bit depth using a caller-provided + /// decoder context. + pub fn decode_native_region_with_context( + &self, + roi: (u32, u32, u32, u32), + decoder_context: &mut DecoderContext<'a>, + ) -> Result { + validate_roi((self.width(), self.height()), roi)?; + self.decode_with_output_region(decoder_context, Some(roi))?; + + let components = &decoder_context.tile_decode_context.channel_data; + let bit_depth = self.original_bit_depth(); + let num_components = + u8::try_from(components.len()).map_err(|_| ValidationError::TooManyChannels)?; + let bytes_per_sample = if bit_depth <= 8 { 1 } else { 2 }; + let (_x, _y, width, height) = roi; + let capacity = checked_decode_byte_len4( + width as usize, + height as usize, + num_components as usize, + bytes_per_sample, + )?; + let mut data = Vec::with_capacity(capacity); + let max_val = ((1u32 << bit_depth) - 1) as f32; + + for row in 0..height as usize { + for col in 0..width as usize { + let idx = row * width as usize + col; + for component in components { + let v = math::round_f32(component.container.truncated()[idx]); + let clamped = if v < 0.0 { + 0.0 + } else if v > max_val { + max_val + } else { + v + }; + if bit_depth <= 8 { + data.push(clamped as u8); + } else { + data.extend_from_slice(&(clamped as u16).to_le_bytes()); + } + } + } + } + + Ok(RawBitmap { + data, + width, + height, + bit_depth, + num_components, + bytes_per_sample: bytes_per_sample as u8, + }) + } + + /// Decode the image into the given buffer. + /// + /// This method does the same as [`Image::decode`], but you can provide + /// a custom buffer for the output, as well as a decoder context. Doing + /// so allows the internal decode engine to reuse memory allocations, so + /// this is especially recommended if you plan on converting multiple + /// images in the same session. + /// + /// The buffer must have the correct size. + pub fn decode_into( + &self, + buf: &mut [u8], + decoder_context: &mut DecoderContext<'a>, + ) -> Result<()> { + let mut decoded_image = self.prepare_decoded_image(decoder_context)?; + validate_interleaved_output_buffer(&decoded_image, buf)?; + interleave_and_convert(&mut decoded_image, buf)?; + + Ok(()) + } + + fn prepare_decoded_image<'ctx>( + &self, + decoder_context: &'ctx mut DecoderContext<'a>, + ) -> Result> { + self.prepare_decoded_image_with_region(decoder_context, None) + } + + fn prepare_decoded_image_with_ht_decoder<'ctx>( + &self, + decoder_context: &'ctx mut DecoderContext<'a>, + ht_decoder: &mut dyn HtCodeBlockDecoder, + ) -> Result> { + self.prepare_decoded_image_with_region_and_ht_decoder( + decoder_context, + None, + Some(ht_decoder), + ) + } + + fn prepare_decoded_image_with_region<'ctx>( + &self, + decoder_context: &'ctx mut DecoderContext<'a>, + output_region: Option<(u32, u32, u32, u32)>, + ) -> Result> { + self.prepare_decoded_image_with_region_and_ht_decoder(decoder_context, output_region, None) + } + + fn prepare_decoded_image_with_region_and_ht_decoder<'ctx>( + &self, + decoder_context: &'ctx mut DecoderContext<'a>, + output_region: Option<(u32, u32, u32, u32)>, + ht_decoder: Option<&mut dyn HtCodeBlockDecoder>, + ) -> Result> { + let settings = &self.settings; + self.decode_with_output_region_and_ht_decoder(decoder_context, output_region, ht_decoder)?; + let mut decoded_image = DecodedImage { + decoded_components: &mut decoder_context.tile_decode_context.channel_data, + boxes: self.boxes.clone(), + }; + + if settings.resolve_palette_indices { + let components = core::mem::take(decoded_image.decoded_components); + *decoded_image.decoded_components = + resolve_palette_indices(components, &decoded_image.boxes)?; + } + + if let Some(cdef) = &decoded_image.boxes.channel_definition { + validate_channel_definition(cdef, decoded_image.decoded_components.len())?; + let mut components = decoded_image + .decoded_components + .iter() + .cloned() + .zip( + cdef.channel_definitions + .iter() + .map(|c| match c._association { + ChannelAssociation::WholeImage => u16::MAX, + ChannelAssociation::Colour(c) => c, + }), + ) + .collect::>(); + components.sort_by_key(|component| component.1); + *decoded_image.decoded_components = components.into_iter().map(|c| c.0).collect(); + } + + let bit_depth = decoded_image.decoded_components[0].bit_depth; + convert_color_space(&mut decoded_image, bit_depth)?; + Ok(decoded_image) + } + + fn decode_with_output_region( + &self, + decoder_context: &mut DecoderContext<'a>, + output_region: Option<(u32, u32, u32, u32)>, + ) -> Result<()> { + self.decode_with_output_region_and_ht_decoder(decoder_context, output_region, None) + } + + fn decode_with_output_region_and_ht_decoder( + &self, + decoder_context: &mut DecoderContext<'a>, + output_region: Option<(u32, u32, u32, u32)>, + mut ht_decoder: Option<&mut dyn HtCodeBlockDecoder>, + ) -> Result<()> { + decoder_context.set_output_region(output_region); + let decode_result = j2c::decode( + self.codestream, + &self.header, + decoder_context, + &mut ht_decoder, + ); + decoder_context.set_output_region(None); + decode_result + } +} + +fn validate_channel_definition( + cdef: &jp2::cdef::ChannelDefinitionBox, + component_count: usize, +) -> Result<()> { + if cdef.channel_definitions.len() != component_count { + bail!(ValidationError::InvalidChannelDefinition); + } + + let mut seen_color_associations = vec![false; component_count]; + for definition in &cdef.channel_definitions { + if let ChannelAssociation::Colour(association) = definition._association { + let Some(index) = association.checked_sub(1).map(usize::from) else { + bail!(ValidationError::InvalidChannelDefinition); + }; + if index >= component_count || seen_color_associations[index] { + bail!(ValidationError::InvalidChannelDefinition); + } + seen_color_associations[index] = true; + } + } + + Ok(()) +} + +pub(crate) fn resolve_alpha_and_color_space( + boxes: &ImageBoxes, + header: &Header<'_>, + settings: &DecodeSettings, +) -> Result<(ColorSpace, bool)> { + let mut num_components = header.component_infos.len(); + + // Override number of components with what is actually in the palette box + // in case we resolve them. + if settings.resolve_palette_indices { + if let Some(palette_box) = &boxes.palette { + num_components = palette_box.columns.len(); + } + } + + let mut has_alpha = false; + + if let Some(cdef) = &boxes.channel_definition { + let last = cdef.channel_definitions.last().unwrap(); + has_alpha = last.channel_type == ChannelType::Opacity; + } + + let mut color_space = get_color_space(boxes, num_components)?; + + // If we didn't resolve palette indices, we need to assume grayscale image. + if !settings.resolve_palette_indices && boxes.palette.is_some() { + has_alpha = false; + color_space = ColorSpace::Gray; + } + + let actual_num_components = header.component_infos.len(); + + // Validate the number of channels. + if boxes.palette.is_none() + && actual_num_components + != (color_space.num_channels() + if has_alpha { 1 } else { 0 }) as usize + { + if !settings.strict + && actual_num_components == color_space.num_channels() as usize + 1 + && !has_alpha + { + // See OPENJPEG test case orb-blue10-lin-j2k. Assume that we have an + // alpha channel in this case. + has_alpha = true; + } else { + // Color space is invalid, attempt to repair. + if actual_num_components == 1 || (actual_num_components == 2 && has_alpha) { + color_space = ColorSpace::Gray; + } else if actual_num_components == 3 { + color_space = ColorSpace::RGB; + } else if actual_num_components == 4 { + if has_alpha { + color_space = ColorSpace::RGB; + } else { + color_space = ColorSpace::CMYK; + } + } else { + bail!(ValidationError::TooManyChannels); + } + } + } + + Ok((color_space, has_alpha)) +} + +/// The color space of the image. +#[derive(Debug, Clone)] +pub enum ColorSpace { + /// A grayscale image. + Gray, + /// An RGB image. + RGB, + /// A CMYK image. + CMYK, + /// An unknown color space. + Unknown { + /// The number of channels of the color space. + num_channels: u8, + }, + /// An image based on an ICC profile. + Icc { + /// The raw data of the ICC profile. + profile: Vec, + /// The number of channels used by the ICC profile. + num_channels: u8, + }, +} + +impl ColorSpace { + /// Return the number of expected channels for the color space. + pub fn num_channels(&self) -> u8 { + match self { + Self::Gray => 1, + Self::RGB => 3, + Self::CMYK => 4, + Self::Unknown { num_channels } => *num_channels, + Self::Icc { + num_channels: num_components, + .. + } => *num_components, + } + } +} + +/// A bitmap storing the decoded result of the image. +pub struct Bitmap { + /// The color space of the image. + pub color_space: ColorSpace, + /// The raw pixel data of the image. The result will always be in + /// 8-bit (in case the original image had a different bit-depth, this + /// decode path scales it to 8-bit). + /// + /// The size is guaranteed to equal + /// `width * height * (num_channels + (if has_alpha { 1 } else { 0 })`. + /// Pixels are interleaved on a per-channel basis, the alpha channel always + /// appearing as the last channel, if available. + pub data: Vec, + /// Whether the image has an alpha channel. + pub has_alpha: bool, + /// The width of the image. + pub width: u32, + /// The height of the image. + pub height: u32, + /// The original bit depth of the image. You usually don't need to do anything + /// with this parameter, it just exists for informational purposes. + pub original_bit_depth: u8, +} + +/// Raw decoded pixel data at native bit depth (no 8-bit scaling). +/// +/// For bit depths ≤ 8, `data` contains one byte per sample. +/// For bit depths > 8 (e.g., 12 or 16), `data` contains two bytes per sample +/// in little-endian byte order (`u16` LE). +/// +/// Samples are interleaved: for a 3-component image, the layout is +/// `[R0, G0, B0, R1, G1, B1, ...]`. +pub struct RawBitmap { + /// The raw pixel data at native bit depth. + pub data: Vec, + /// The width of the image in pixels. + pub width: u32, + /// The height of the image in pixels. + pub height: u32, + /// The original bit depth per sample (e.g., 8, 12, 16). + pub bit_depth: u8, + /// The number of components (e.g., 1 for grayscale, 3 for RGB). + pub num_components: u8, + /// Bytes per sample: 1 for bit_depth ≤ 8, 2 for bit_depth > 8. + pub bytes_per_sample: u8, +} + +/// A borrowed decoded component plane. +pub struct ComponentPlane<'a> { + samples: &'a [f32], + bit_depth: u8, +} + +impl<'a> ComponentPlane<'a> { + /// Component samples in row-major order. + pub fn samples(&self) -> &'a [f32] { + self.samples + } + + /// Bit depth of this component plane. + pub fn bit_depth(&self) -> u8 { + self.bit_depth + } +} + +/// Borrowed decoded component planes for an image. +pub struct DecodedComponents<'a> { + dimensions: (u32, u32), + color_space: ColorSpace, + has_alpha: bool, + planes: Vec>, +} + +impl<'a> DecodedComponents<'a> { + /// Dimensions of the decoded image represented by these planes. + pub fn dimensions(&self) -> (u32, u32) { + self.dimensions + } + + /// Color space after JPEG 2000 color conversion has been applied. + pub fn color_space(&self) -> &ColorSpace { + &self.color_space + } + + /// Whether the decoded image has an alpha channel. + pub fn has_alpha(&self) -> bool { + self.has_alpha + } + + /// Borrowed decoded component planes in display order. + pub fn planes(&self) -> &[ComponentPlane<'a>] { + &self.planes + } +} + +fn validate_interleaved_output_buffer(image: &DecodedImage<'_>, buf: &[u8]) -> Result<()> { + let required_len = interleaved_output_len(image)?; + if buf.len() < required_len { + bail!(DecodingError::OutputBufferTooSmall); + } + Ok(()) +} + +fn interleaved_output_len(image: &DecodedImage<'_>) -> Result { + let Some(first) = image.decoded_components.first() else { + bail!(DecodingError::CodeBlockDecodeFailure); + }; + first + .container + .truncated() + .len() + .checked_mul(image.decoded_components.len()) + .ok_or(ValidationError::ImageTooLarge.into()) +} + +fn interleave_and_convert(image: &mut DecodedImage<'_>, buf: &mut [u8]) -> Result<()> { + let components = &mut *image.decoded_components; + let num_components = components.len(); + + let mut all_same_bit_depth = Some(components[0].bit_depth); + + for component in components.iter().skip(1) { + if Some(component.bit_depth) != all_same_bit_depth { + all_same_bit_depth = None; + } + } + + let max_len = components[0].container.truncated().len(); + + let mut output_iter = buf.iter_mut(); + + if all_same_bit_depth == Some(8) && num_components <= 4 { + // Fast path for the common case. + match num_components { + // Gray-scale. + 1 => { + for (output, input) in output_iter.zip( + components[0] + .container + .iter() + .map(|v| math::round_f32(*v) as u8), + ) { + *output = input; + } + } + // Gray-scale with alpha. + 2 => { + let c0 = &components[0]; + let c1 = &components[1]; + + let c0 = &c0.container[..max_len]; + let c1 = &c1.container[..max_len]; + + for i in 0..max_len { + *output_iter.next().unwrap() = math::round_f32(c0[i]) as u8; + *output_iter.next().unwrap() = math::round_f32(c1[i]) as u8; + } + } + // RGB + 3 => { + let c0 = &components[0]; + let c1 = &components[1]; + let c2 = &components[2]; + + let c0 = &c0.container[..max_len]; + let c1 = &c1.container[..max_len]; + let c2 = &c2.container[..max_len]; + + for i in 0..max_len { + *output_iter.next().unwrap() = math::round_f32(c0[i]) as u8; + *output_iter.next().unwrap() = math::round_f32(c1[i]) as u8; + *output_iter.next().unwrap() = math::round_f32(c2[i]) as u8; + } + } + // RGBA or CMYK. + 4 => { + let c0 = &components[0]; + let c1 = &components[1]; + let c2 = &components[2]; + let c3 = &components[3]; + + let c0 = &c0.container[..max_len]; + let c1 = &c1.container[..max_len]; + let c2 = &c2.container[..max_len]; + let c3 = &c3.container[..max_len]; + + for i in 0..max_len { + *output_iter.next().unwrap() = math::round_f32(c0[i]) as u8; + *output_iter.next().unwrap() = math::round_f32(c1[i]) as u8; + *output_iter.next().unwrap() = math::round_f32(c2[i]) as u8; + *output_iter.next().unwrap() = math::round_f32(c3[i]) as u8; + } + } + _ => bail!(ValidationError::TooManyChannels), + } + } else { + // Slow path that also requires us to scale to 8 bit. + let mul_factor = ((1 << 8) - 1) as f32; + + for sample in 0..max_len { + for channel in components.iter() { + *output_iter.next().unwrap() = math::round_f32( + (channel.container[sample] / ((1_u32 << channel.bit_depth) - 1) as f32) + * mul_factor, + ) as u8; + } + } + } + + Ok(()) +} + +fn interleave_and_convert_region( + image: &mut DecodedImage<'_>, + image_width: usize, + roi: (u32, u32, u32, u32), + buf: &mut [u8], +) { + let components = &mut *image.decoded_components; + let num_components = components.len(); + let (x, y, width, height) = roi; + let mut output_iter = buf.iter_mut(); + + let mut all_same_bit_depth = Some(components[0].bit_depth); + for component in components.iter().skip(1) { + if Some(component.bit_depth) != all_same_bit_depth { + all_same_bit_depth = None; + } + } + + if all_same_bit_depth == Some(8) && num_components <= 4 { + for row in y as usize..(y + height) as usize { + let row_base = row * image_width; + for col in x as usize..(x + width) as usize { + let idx = row_base + col; + for component in components.iter() { + *output_iter.next().unwrap() = math::round_f32(component.container[idx]) as u8; + } + } + } + } else { + let mul_factor = ((1 << 8) - 1) as f32; + for row in y as usize..(y + height) as usize { + let row_base = row * image_width; + for col in x as usize..(x + width) as usize { + let idx = row_base + col; + for component in components.iter() { + *output_iter.next().unwrap() = math::round_f32( + (component.container[idx] / ((1_u32 << component.bit_depth) - 1) as f32) + * mul_factor, + ) as u8; + } + } + } + } +} + +fn validate_roi(dims: (u32, u32), roi: (u32, u32, u32, u32)) -> Result<()> { + let (image_width, image_height) = dims; + let (x, y, width, height) = roi; + let x_end = x + .checked_add(width) + .ok_or(ValidationError::InvalidDimensions)?; + let y_end = y + .checked_add(height) + .ok_or(ValidationError::InvalidDimensions)?; + if x_end > image_width || y_end > image_height { + return Err(ValidationError::InvalidDimensions.into()); + } + Ok(()) +} + +fn convert_color_space(image: &mut DecodedImage<'_>, bit_depth: u8) -> Result<()> { + if let Some(jp2::colr::ColorSpace::Enumerated(e)) = &image + .boxes + .color_specification + .as_ref() + .map(|i| &i.color_space) + { + match e { + EnumeratedColorspace::Sycc => { + dispatch!(Level::new(), simd => { + sycc_to_rgb(simd, image.decoded_components, bit_depth) + })?; + } + EnumeratedColorspace::CieLab(cielab) => { + dispatch!(Level::new(), simd => { + cielab_to_rgb(simd, image.decoded_components, bit_depth, cielab) + })?; + } + _ => {} + } + } + + Ok(()) +} + +fn get_color_space(boxes: &ImageBoxes, num_components: usize) -> Result { + let cs = match boxes + .color_specification + .as_ref() + .map(|c| &c.color_space) + .unwrap_or(&jp2::colr::ColorSpace::Unknown) + { + jp2::colr::ColorSpace::Enumerated(e) => { + match e { + EnumeratedColorspace::Cmyk => ColorSpace::CMYK, + EnumeratedColorspace::Srgb => ColorSpace::RGB, + EnumeratedColorspace::RommRgb => { + // Use an ICC profile to process the RommRGB color space. + ColorSpace::Icc { + profile: include_bytes!("../assets/ProPhoto-v2-micro.icc").to_vec(), + num_channels: 3, + } + } + EnumeratedColorspace::EsRgb => ColorSpace::RGB, + EnumeratedColorspace::Greyscale => ColorSpace::Gray, + EnumeratedColorspace::Sycc => ColorSpace::RGB, + EnumeratedColorspace::CieLab(_) => ColorSpace::Icc { + profile: include_bytes!("../assets/LAB.icc").to_vec(), + num_channels: 3, + }, + _ => bail!(FormatError::Unsupported), + } + } + jp2::colr::ColorSpace::Icc(icc) => { + if let Some(metadata) = ICCMetadata::from_data(icc) { + ColorSpace::Icc { + profile: icc.clone(), + num_channels: metadata.color_space.num_components(), + } + } else { + // See OPENJPEG test orb-blue10-lin-jp2.jp2. They seem to + // assume RGB in this case (even though the image has 4 + // components with no opacity channel, they assume RGBA instead + // of CMYK). + ColorSpace::RGB + } + } + jp2::colr::ColorSpace::Unknown => match num_components { + 1 => ColorSpace::Gray, + 3 => ColorSpace::RGB, + 4 => ColorSpace::CMYK, + _ => ColorSpace::Unknown { + num_channels: num_components as u8, + }, + }, + }; + + Ok(cs) +} + +fn resolve_palette_indices( + components: Vec, + boxes: &ImageBoxes, +) -> Result> { + let Some(palette) = boxes.palette.as_ref() else { + // Nothing to resolve. + return Ok(components); + }; + + let Some(mapping) = boxes.component_mapping.as_ref() else { + bail!(ColorError::PaletteResolutionFailed); + }; + if mapping.entries.is_empty() { + bail!(ColorError::PaletteResolutionFailed); + } + + let mut resolved = Vec::with_capacity(mapping.entries.len()); + + for entry in &mapping.entries { + let component_idx = entry.component_index as usize; + let component = components + .get(component_idx) + .ok_or(ColorError::PaletteResolutionFailed)?; + + match entry.mapping_type { + ComponentMappingType::Direct => resolved.push(component.clone()), + ComponentMappingType::Palette { column } => { + let column_idx = column as usize; + let column_info = palette + .columns + .get(column_idx) + .ok_or(ColorError::PaletteResolutionFailed)?; + + let mut mapped = + Vec::with_capacity(component.container.truncated().len() + SIMD_WIDTH); + + for &sample in component.container.truncated() { + let index = math::round_f32(sample) as i64; + let value = palette + .map(index as usize, column_idx) + .ok_or(ColorError::PaletteResolutionFailed)?; + mapped.push(value as f32); + } + + resolved.push(ComponentData { + container: math::SimdBuffer::new(mapped), + bit_depth: column_info.bit_depth, + }); + } + } + } + + Ok(resolved) +} + +#[inline(always)] +fn cielab_to_rgb( + simd: S, + components: &mut [ComponentData], + bit_depth: u8, + lab: &CieLab, +) -> Result<()> { + let (head, _) = components + .split_at_mut_checked(3) + .ok_or(ColorError::LabConversionFailed)?; + + let [l, a, b] = head else { + bail!(ColorError::LabConversionFailed); + }; + + let prec0 = l.bit_depth; + let prec1 = a.bit_depth; + let prec2 = b.bit_depth; + + // Prevent underflows/divisions by zero further below. + if prec0 < 4 || prec1 < 4 || prec2 < 4 { + bail!(ColorError::LabConversionFailed); + } + + let rl = lab.rl.unwrap_or(100); + let ra = lab.ra.unwrap_or(170); + let rb = lab.ra.unwrap_or(200); + let ol = lab.ol.unwrap_or(0); + let oa = lab.oa.unwrap_or(1 << (bit_depth - 1)); + let ob = lab + .ob + .unwrap_or((1 << (bit_depth - 2)) + (1 << (bit_depth - 3))); + + // Copied from OpenJPEG. + let min_l = -(rl as f32 * ol as f32) / ((1 << prec0) - 1) as f32; + let max_l = min_l + rl as f32; + let min_a = -(ra as f32 * oa as f32) / ((1 << prec1) - 1) as f32; + let max_a = min_a + ra as f32; + let min_b = -(rb as f32 * ob as f32) / ((1 << prec2) - 1) as f32; + let max_b = min_b + rb as f32; + + let bit_max = (1_u32 << bit_depth) - 1; + + // Note that we are not doing the actual conversion with the ICC profile yet, + // just decoding the raw LAB values. + // We leave applying the ICC profile to the user. + let divisor_l = ((1 << prec0) - 1) as f32; + let divisor_a = ((1 << prec1) - 1) as f32; + let divisor_b = ((1 << prec2) - 1) as f32; + + let scale_l_final = bit_max as f32 / 100.0; + let scale_ab_final = bit_max as f32 / 255.0; + + let l_offset = min_l * scale_l_final; + let l_scale = (max_l - min_l) / divisor_l * scale_l_final; + let a_offset = (min_a + 128.0) * scale_ab_final; + let a_scale = (max_a - min_a) / divisor_a * scale_ab_final; + let b_offset = (min_b + 128.0) * scale_ab_final; + let b_scale = (max_b - min_b) / divisor_b * scale_ab_final; + + let l_offset_v = f32x8::splat(simd, l_offset); + let l_scale_v = f32x8::splat(simd, l_scale); + let a_offset_v = f32x8::splat(simd, a_offset); + let a_scale_v = f32x8::splat(simd, a_scale); + let b_offset_v = f32x8::splat(simd, b_offset); + let b_scale_v = f32x8::splat(simd, b_scale); + + // Note that we are not doing the actual conversion with the ICC profile yet, + // just decoding the raw LAB values. + // We leave applying the ICC profile to the user. + for ((l_chunk, a_chunk), b_chunk) in l + .container + .chunks_exact_mut(SIMD_WIDTH) + .zip(a.container.chunks_exact_mut(SIMD_WIDTH)) + .zip(b.container.chunks_exact_mut(SIMD_WIDTH)) + { + let l_v = f32x8::from_slice(simd, l_chunk); + let a_v = f32x8::from_slice(simd, a_chunk); + let b_v = f32x8::from_slice(simd, b_chunk); + + l_v.mul_add(l_scale_v, l_offset_v).store(l_chunk); + a_v.mul_add(a_scale_v, a_offset_v).store(a_chunk); + b_v.mul_add(b_scale_v, b_offset_v).store(b_chunk); + } + + Ok(()) +} + +#[inline(always)] +fn sycc_to_rgb(simd: S, components: &mut [ComponentData], bit_depth: u8) -> Result<()> { + let offset = (1_u32 << (bit_depth as u32 - 1)) as f32; + let max_value = ((1_u32 << bit_depth as u32) - 1) as f32; + + let (head, _) = components + .split_at_mut_checked(3) + .ok_or(ColorError::SyccConversionFailed)?; + + let [y, cb, cr] = head else { + bail!(ColorError::SyccConversionFailed); + }; + + let offset_v = f32x8::splat(simd, offset); + let max_v = f32x8::splat(simd, max_value); + let zero_v = f32x8::splat(simd, 0.0); + let cr_to_r = f32x8::splat(simd, 1.402); + let cb_to_g = f32x8::splat(simd, -0.344136); + let cr_to_g = f32x8::splat(simd, -0.714136); + let cb_to_b = f32x8::splat(simd, 1.772); + + for ((y_chunk, cb_chunk), cr_chunk) in y + .container + .chunks_exact_mut(SIMD_WIDTH) + .zip(cb.container.chunks_exact_mut(SIMD_WIDTH)) + .zip(cr.container.chunks_exact_mut(SIMD_WIDTH)) + { + let y_v = f32x8::from_slice(simd, y_chunk); + let cb_v = f32x8::from_slice(simd, cb_chunk) - offset_v; + let cr_v = f32x8::from_slice(simd, cr_chunk) - offset_v; + + // r = y + 1.402 * cr + let r = cr_v.mul_add(cr_to_r, y_v); + // g = y - 0.344136 * cb - 0.714136 * cr + let g = cr_v.mul_add(cr_to_g, cb_v.mul_add(cb_to_g, y_v)); + // b = y + 1.772 * cb + let b = cb_v.mul_add(cb_to_b, y_v); + + r.min(max_v).max(zero_v).store(y_chunk); + g.min(max_v).max(zero_v).store(cb_chunk); + b.min(max_v).max(zero_v).store(cr_chunk); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ht_uvlc_encode_table_bytes_match_entry_packing_order() { + let entries = ht_uvlc_encode_table(); + let bytes = ht_uvlc_encode_table_bytes(); + + assert_eq!(bytes.len(), entries.len() * 6); + for (index, entry) in entries.iter().enumerate() { + let offset = index * 6; + assert_eq!( + &bytes[offset..offset + 6], + &[ + entry.pre, + entry.pre_len, + entry.suf, + entry.suf_len, + entry.ext, + entry.ext_len + ], + ); + } + } + + #[test] + fn roi_maxshift_inverse_preserves_background_and_unshifts_roi_coefficients() { + assert_eq!(apply_roi_maxshift_inverse_i32(127, 7), 127); + assert_eq!(apply_roi_maxshift_inverse_i32(-127, 7), -127); + assert_eq!(apply_roi_maxshift_inverse_i32(128, 7), 1); + assert_eq!(apply_roi_maxshift_inverse_i32(-128, 7), -1); + assert_eq!(apply_roi_maxshift_inverse_i32(255, 7), 1); + assert_eq!(apply_roi_maxshift_inverse_i32(-255, 7), -1); + assert_eq!(apply_roi_maxshift_inverse_i32(256, 7), 2); + assert_eq!(apply_roi_maxshift_inverse_i32(-256, 7), -2); + assert_eq!(apply_roi_maxshift_inverse_i32(42, 0), 42); + } + + #[test] + fn classic_scalar_decode_applies_nonzero_roi_maxshift() { + let roi_shift = 3; + let total_bitplanes = 3; + let style = J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: false, + reset_context_probabilities: false, + termination_on_each_pass: false, + vertically_causal_context: false, + segmentation_symbols: false, + }; + let coded_coefficients = [0, 5, 1 << roi_shift, -(2 << roi_shift)]; + let encoded = encode_j2k_code_block_scalar_with_style( + &coded_coefficients, + 2, + 2, + J2kSubBandType::LowLow, + total_bitplanes + roi_shift, + style, + ) + .expect("encode ROI-shifted code block"); + let job = J2kCodeBlockDecodeJob { + data: &encoded.data, + segments: &encoded.segments, + width: 2, + height: 2, + output_stride: 2, + missing_bit_planes: encoded.missing_bit_planes, + number_of_coding_passes: encoded.number_of_coding_passes, + total_bitplanes, + roi_shift, + sub_band_type: J2kSubBandType::LowLow, + style, + strict: true, + dequantization_step: 1.0, + }; + let mut output = [0.0; 4]; + + decode_j2k_code_block_scalar(job, &mut output).expect("decode ROI-shifted code block"); + + assert_eq!(output, [0.0, 5.0, 1.0, -2.0]); + } + + #[test] + fn classic_scalar_token_pack_matches_scalar_single_cleanup_block() { + let style = J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: true, + reset_context_probabilities: false, + termination_on_each_pass: false, + vertically_causal_context: false, + segmentation_symbols: false, + }; + let scalar = + encode_j2k_code_block_scalar_with_style(&[1], 1, 1, J2kSubBandType::LowLow, 1, style) + .expect("encode scalar"); + let token_bytes = pack_mq_test_tokens(&[(0, 1), (9, 0)]); + let packed = pack_j2k_code_block_scalar_from_tier1_tokens( + &token_bytes, + &[J2kTier1TokenSegment { + token_bit_offset: 0, + token_bit_count: 12, + start_coding_pass: 0, + end_coding_pass: 1, + use_arithmetic: true, + }], + scalar.number_of_coding_passes, + scalar.missing_bit_planes, + ) + .expect("pack tokens"); + + assert_eq!(packed.data, scalar.data); + assert_eq!(packed.segments, scalar.segments); + assert_eq!( + packed.number_of_coding_passes, + scalar.number_of_coding_passes + ); + assert_eq!(packed.missing_bit_planes, scalar.missing_bit_planes); + } + + fn pack_mq_test_tokens(tokens: &[(u8, u8)]) -> Vec { + let mut bytes = Vec::new(); + let mut current = 0u8; + let mut bits = 0u8; + for &(ctx, bit) in tokens { + let value = (ctx & 0x1F) | ((bit & 1) << 5); + for shift in (0..6).rev() { + current = (current << 1) | ((value >> shift) & 1); + bits += 1; + if bits == 8 { + bytes.push(current); + current = 0; + bits = 0; + } + } + } + if bits != 0 { + bytes.push(current << (8 - bits)); + } + bytes + } + + #[test] + fn classic_scalar_profiled_decode_matches_unprofiled_decode() { + let width = 64u32; + let height = 64u32; + let sample_count = width as usize * height as usize; + let total_bitplanes = 12; + let style = J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: false, + reset_context_probabilities: false, + termination_on_each_pass: false, + vertically_causal_context: false, + segmentation_symbols: false, + }; + let coefficients = (0..sample_count) + .map(|idx| { + let value = i32::try_from((idx * 37) % 4095).expect("sample value fits i32") - 2048; + if idx % 17 == 0 { + 0 + } else { + value + } + }) + .collect::>(); + let encoded = encode_j2k_code_block_scalar_with_style( + &coefficients, + width, + height, + J2kSubBandType::LowLow, + total_bitplanes, + style, + ) + .expect("encode classic block"); + let job = J2kCodeBlockDecodeJob { + data: &encoded.data, + segments: &encoded.segments, + width, + height, + output_stride: width as usize, + missing_bit_planes: encoded.missing_bit_planes, + number_of_coding_passes: encoded.number_of_coding_passes, + total_bitplanes, + roi_shift: 0, + sub_band_type: J2kSubBandType::LowLow, + style, + strict: true, + dequantization_step: 1.0, + }; + let mut expected = vec![0.0_f32; sample_count]; + let mut actual = vec![0.0_f32; sample_count]; + let mut profile = J2kCodeBlockDecodeProfile::default(); + + decode_j2k_code_block_scalar(job, &mut expected).expect("unprofiled classic decode"); + decode_j2k_code_block_scalar_profiled(job, &mut actual, &mut profile) + .expect("profiled classic decode"); + + assert_eq!(actual, expected); + assert!(profile.cleanup_us > 0); + } + + #[test] + fn classic_scalar_workspace_reuse_matches_fresh_decode() { + let total_bitplanes = 6; + let style = J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: false, + reset_context_probabilities: false, + termination_on_each_pass: false, + vertically_causal_context: false, + segmentation_symbols: false, + }; + let mut workspace = J2kCodeBlockDecodeWorkspace::default(); + + for (width, height, seed) in [(8, 8, 0x31), (4, 16, 0x47)] { + let coefficients = (0..width * height) + .map(|idx| { + let value = ((idx as i32 * seed) % 23) - 11; + if idx % 7 == 0 { + 0 + } else { + value + } + }) + .collect::>(); + let encoded = encode_j2k_code_block_scalar_with_style( + &coefficients, + width, + height, + J2kSubBandType::LowLow, + total_bitplanes, + style, + ) + .expect("encode classic block"); + let job = J2kCodeBlockDecodeJob { + data: &encoded.data, + segments: &encoded.segments, + width, + height, + output_stride: width as usize, + missing_bit_planes: encoded.missing_bit_planes, + number_of_coding_passes: encoded.number_of_coding_passes, + total_bitplanes, + roi_shift: 0, + sub_band_type: J2kSubBandType::LowLow, + style, + strict: true, + dequantization_step: 1.0, + }; + let mut fresh = vec![0.0_f32; width as usize * height as usize]; + let mut reused = vec![0.0_f32; width as usize * height as usize]; + + decode_j2k_code_block_scalar(job, &mut fresh).expect("fresh classic decode"); + decode_j2k_code_block_scalar_with_workspace(job, &mut reused, &mut workspace) + .expect("workspace classic decode"); + + assert_eq!(reused, fresh); + } + } + + #[test] + fn scalar_packetization_rejects_overflowing_ht_refinement_lengths_without_panic() { + let payload = [0x12]; + let block = J2kPacketizationCodeBlock { + data: &payload, + ht_cleanup_length: u32::MAX, + ht_refinement_length: 1, + num_coding_passes: 3, + num_zero_bitplanes: 2, + previously_included: false, + l_block: 3, + block_coding_mode: J2kPacketizationBlockCodingMode::HighThroughput, + }; + let subband = J2kPacketizationSubband { + code_blocks: vec![block], + num_cbs_x: 1, + num_cbs_y: 1, + }; + let resolution = J2kPacketizationResolution { + subbands: vec![subband], + }; + let resolutions = [resolution]; + let job = J2kPacketizationEncodeJob { + resolution_count: 1, + num_layers: 1, + num_components: 1, + code_block_count: 1, + progression_order: J2kPacketizationProgressionOrder::Lrcp, + packet_descriptors: &[], + resolutions: &resolutions, + }; + + let err = encode_j2k_packetization_scalar(job) + .expect_err("overflowing HT packetization segment lengths rejected"); + + assert_eq!(err, "multi-pass HTJ2K packet contribution length overflow"); + } + + #[derive(Default)] + struct DecodeWorkCounter { + classic_code_blocks: usize, + ht_code_blocks: usize, + idwt_output_samples: usize, + } + + impl DecodeWorkCounter { + fn code_blocks(&self) -> usize { + self.classic_code_blocks + self.ht_code_blocks + } + } + + struct FailingHtDecoder { + called: bool, + } + + impl HtCodeBlockDecoder for FailingHtDecoder { + fn decode_code_block( + &mut self, + _job: HtCodeBlockDecodeJob<'_>, + _output: &mut [f32], + ) -> Result<()> { + self.called = true; + Err(DecodingError::CodeBlockDecodeFailure.into()) + } + } + + struct FailingClassicDecoder { + called: bool, + } + + impl HtCodeBlockDecoder for FailingClassicDecoder { + fn decode_code_block( + &mut self, + _job: HtCodeBlockDecodeJob<'_>, + _output: &mut [f32], + ) -> Result<()> { + panic!("HT hook must not be used for classic J2K test") + } + + fn decode_j2k_code_block( + &mut self, + _job: J2kCodeBlockDecodeJob<'_>, + _output: &mut [f32], + ) -> Result { + self.called = true; + Err(DecodingError::CodeBlockDecodeFailure.into()) + } + } + + struct FailingClassicBatchDecoder { + called: bool, + } + + #[derive(Default)] + struct CapturingHtDecoder { + called: bool, + blocks: usize, + refinement_jobs: usize, + max_coding_passes: u8, + } + + impl HtCodeBlockDecoder for CapturingHtDecoder { + fn decode_code_block( + &mut self, + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + ) -> Result<()> { + self.called = true; + self.blocks += 1; + self.max_coding_passes = self.max_coding_passes.max(job.number_of_coding_passes); + if job.refinement_length > 0 { + self.refinement_jobs += 1; + assert!( + job.number_of_coding_passes > 1, + "refinement bytes must correspond to refinement coding passes" + ); + } + + decode_ht_code_block_scalar(job, output) + } + } + + #[derive(Clone)] + struct CapturedHtDecodeJob { + data: Vec, + cleanup_length: u32, + refinement_length: u32, + width: u32, + height: u32, + output_stride: usize, + missing_bit_planes: u8, + number_of_coding_passes: u8, + num_bitplanes: u8, + roi_shift: u8, + stripe_causal: bool, + strict: bool, + dequantization_step: f32, + } + + impl CapturedHtDecodeJob { + fn from_job(job: HtCodeBlockDecodeJob<'_>) -> Self { + Self { + data: job.data.to_vec(), + cleanup_length: job.cleanup_length, + refinement_length: job.refinement_length, + width: job.width, + height: job.height, + output_stride: job.output_stride, + missing_bit_planes: job.missing_bit_planes, + number_of_coding_passes: job.number_of_coding_passes, + num_bitplanes: job.num_bitplanes, + roi_shift: job.roi_shift, + stripe_causal: job.stripe_causal, + strict: job.strict, + dequantization_step: job.dequantization_step, + } + } + + fn borrowed(&self) -> HtCodeBlockDecodeJob<'_> { + HtCodeBlockDecodeJob { + data: &self.data, + cleanup_length: self.cleanup_length, + refinement_length: self.refinement_length, + width: self.width, + height: self.height, + output_stride: self.output_stride, + missing_bit_planes: self.missing_bit_planes, + number_of_coding_passes: self.number_of_coding_passes, + num_bitplanes: self.num_bitplanes, + roi_shift: self.roi_shift, + stripe_causal: self.stripe_causal, + strict: self.strict, + dequantization_step: self.dequantization_step, + } + } + } + + #[derive(Default)] + struct FirstHtJobDecoder { + job: Option, + } + + impl HtCodeBlockDecoder for FirstHtJobDecoder { + fn decode_code_block( + &mut self, + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + ) -> Result<()> { + if self.job.is_none() { + self.job = Some(CapturedHtDecodeJob::from_job(job)); + } + decode_ht_code_block_scalar(job, output) + } + } + + struct ZeroRefinementHtDecoder; + + impl HtCodeBlockDecoder for ZeroRefinementHtDecoder { + fn decode_code_block( + &mut self, + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + ) -> Result<()> { + let mut data = job.data.to_vec(); + let cleanup_len = job.cleanup_length as usize; + let refinement_len = job.refinement_length as usize; + data[cleanup_len..cleanup_len + refinement_len].fill(0); + let zeroed = HtCodeBlockDecodeJob { data: &data, ..job }; + + decode_ht_code_block_scalar(zeroed, output) + } + } + + #[derive(Default)] + struct CleanupLimitedHtDecoder { + blocks: usize, + refinement_blocks: usize, + cleanup_bytes: usize, + refinement_bytes: usize, + } + + impl HtCodeBlockDecoder for CleanupLimitedHtDecoder { + fn decode_code_block( + &mut self, + job: HtCodeBlockDecodeJob<'_>, + output: &mut [f32], + ) -> Result<()> { + self.blocks += 1; + self.cleanup_bytes += job.cleanup_length as usize; + if job.refinement_length > 0 { + self.refinement_blocks += 1; + self.refinement_bytes += job.refinement_length as usize; + } + + decode_ht_code_block_scalar_until_phase( + job, + output, + HtCodeBlockDecodePhaseLimit::Cleanup, + ) + } + } + + impl HtCodeBlockDecoder for FailingClassicBatchDecoder { + fn decode_code_block( + &mut self, + _job: HtCodeBlockDecodeJob<'_>, + _output: &mut [f32], + ) -> Result<()> { + panic!("HT hook must not be used for classic J2K batch test") + } + + fn decode_j2k_code_block( + &mut self, + _job: J2kCodeBlockDecodeJob<'_>, + _output: &mut [f32], + ) -> Result { + panic!( + "per-block classic hook must not be used when the batch hook handles the sub-band" + ) + } + + fn decode_j2k_sub_band( + &mut self, + _job: J2kSubBandDecodeJob<'_>, + _output: &mut [f32], + ) -> Result { + self.called = true; + Err(DecodingError::CodeBlockDecodeFailure.into()) + } + } + + fn fixture() -> Vec { + let pixels = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]; + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + encode(&pixels, 2, 2, 3, 8, false, &options).expect("encode") + } + + #[test] + fn decode_into_rejects_short_output_buffer() { + let bytes = fixture(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let mut output = vec![0; 11]; + + let err = image + .decode_into(&mut output, &mut context) + .expect_err("short output buffer must be rejected"); + + assert_eq!( + err, + DecodeError::Decoding(DecodingError::OutputBufferTooSmall) + ); + } + + fn fixture_multi_block() -> Vec { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 0, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + encode(&pixels, 8, 8, 1, 8, false, &options).expect("encode multi-block classic") + } + + fn fixture_gray() -> Vec { + let pixels: Vec = (0..16).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + encode(&pixels, 4, 4, 1, 8, false, &options).expect("encode classic gray8") + } + + fn rewrite_siz_to_single_large_tile(codestream: &mut [u8], dimensions: u32) { + let siz = codestream + .windows(2) + .position(|w| w == [0xFF, 0x51]) + .expect("SIZ marker"); + codestream[siz + 6..siz + 10].copy_from_slice(&dimensions.to_be_bytes()); + codestream[siz + 10..siz + 14].copy_from_slice(&dimensions.to_be_bytes()); + codestream[siz + 22..siz + 26].copy_from_slice(&dimensions.to_be_bytes()); + codestream[siz + 26..siz + 30].copy_from_slice(&dimensions.to_be_bytes()); + } + + fn rewrite_siz_tile_grid(codestream: &mut [u8], dimensions: (u32, u32), tile_size: (u32, u32)) { + let siz = codestream + .windows(2) + .position(|w| w == [0xFF, 0x51]) + .expect("SIZ marker"); + codestream[siz + 6..siz + 10].copy_from_slice(&dimensions.0.to_be_bytes()); + codestream[siz + 10..siz + 14].copy_from_slice(&dimensions.1.to_be_bytes()); + codestream[siz + 22..siz + 26].copy_from_slice(&tile_size.0.to_be_bytes()); + codestream[siz + 26..siz + 30].copy_from_slice(&tile_size.1.to_be_bytes()); + } + + fn rewrite_siz_component_count(codestream: &mut Vec, component_count: u16) { + let siz = codestream + .windows(2) + .position(|w| w == [0xFF, 0x51]) + .expect("SIZ marker"); + let old_component_count = + u16::from_be_bytes([codestream[siz + 38], codestream[siz + 39]]) as usize; + let component_start = siz + 40; + let component_end = component_start + old_component_count * 3; + let descriptor = codestream[component_start..component_start + 3].to_vec(); + let mut descriptors = Vec::with_capacity(usize::from(component_count) * 3); + for _ in 0..component_count { + descriptors.extend_from_slice(&descriptor); + } + + let siz_len = 38_u16 + .checked_add( + component_count + .checked_mul(3) + .expect("SIZ component bytes fit"), + ) + .expect("SIZ length fits"); + codestream[siz + 2..siz + 4].copy_from_slice(&siz_len.to_be_bytes()); + codestream[siz + 38..siz + 40].copy_from_slice(&component_count.to_be_bytes()); + codestream.splice(component_start..component_end, descriptors); + } + + #[test] + fn inspect_rejects_component_count_above_j2k_spec_cap() { + let mut bytes = fixture_gray(); + rewrite_siz_component_count(&mut bytes, MAX_J2K_SPEC_COMPONENTS + 1); + + let err = inspect_j2k_codestream_header(&bytes) + .expect_err("SIZ component count above spec cap must be rejected"); + + assert_eq!( + err, + J2kCodestreamHeaderError::InvalidSiz { + what: "component count exceeds JPEG 2000 limit" + } + ); + } + + #[test] + fn native_decode_rejects_component_count_above_u8_before_bitmap_truncation() { + let mut bytes = fixture_gray(); + rewrite_siz_component_count(&mut bytes, MAX_NATIVE_DECODE_COMPONENTS + 1); + + let err = match Image::new(&bytes, &DecodeSettings::default()) { + Err(err) => err, + Ok(_) => { + panic!("native decode must reject component counts that cannot fit RawBitmap") + } + }; + + assert_eq!( + err, + DecodeError::Validation(ValidationError::TooManyChannels) + ); + } + + #[test] + fn tile_parse_rejects_component_tile_structural_bomb_before_allocation() { + let mut bytes = fixture_gray(); + rewrite_siz_component_count(&mut bytes, MAX_NATIVE_DECODE_COMPONENTS); + rewrite_siz_tile_grid(&mut bytes, (256, 256), (1, 1)); + let parsed = j2c::parse_raw(&bytes, &DecodeSettings::default()).expect("raw header parses"); + let mut context = j2c::DecoderContext::default(); + let mut ht_decoder: Option<&mut dyn HtCodeBlockDecoder> = None; + + let err = j2c::decode(parsed.data, &parsed.header, &mut context, &mut ht_decoder) + .expect_err("tile structural budget must reject before tile allocation"); + + assert_eq!(err, DecodeError::Validation(ValidationError::ImageTooLarge)); + } + + #[test] + fn owned_decode_rejects_large_siz_before_allocating_output() { + let mut bytes = fixture_gray(); + rewrite_siz_to_single_large_tile(&mut bytes, 60_000); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("large SIZ parses"); + + let err = match image.decode() { + Err(err) => err, + Ok(_) => panic!("large owned decode must be capped"), + }; + + assert_eq!(err, DecodeError::Validation(ValidationError::ImageTooLarge)); + } + + #[test] + fn decode_into_rejects_large_siz_before_allocating_component_storage() { + let mut bytes = fixture_gray(); + rewrite_siz_to_single_large_tile(&mut bytes, 60_000); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("large SIZ parses"); + let mut context = DecoderContext::default(); + let mut out = []; + + let err = match image.decode_into(&mut out, &mut context) { + Err(err) => err, + Ok(_) => panic!("component storage must be capped before allocation"), + }; + + assert_eq!(err, DecodeError::Validation(ValidationError::ImageTooLarge)); + } + + fn fixture_ht_gray() -> Vec { + let pixels: Vec = (0..16).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, 4, 4, 1, 8, false, &options).expect("encode ht gray8") + } + + fn fixture_ht_multi_block() -> Vec { + let pixels: Vec = (0..64).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 0, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, 8, 8, 1, 8, false, &options).expect("encode multi-block HT gray8") + } + + fn fixture_ht_rgb_multi_block() -> Vec { + let pixels = gradient_pixels(8, 8, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 0, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, 8, 8, 3, 8, false, &options).expect("encode multi-block HT RGB8") + } + + fn direct_ht_job_count(plan: &J2kDirectGrayscalePlan) -> usize { + plan.steps + .iter() + .map(|step| match step { + J2kDirectGrayscaleStep::HtSubBand(sub_band) => sub_band.jobs.len(), + _ => 0, + }) + .sum() + } + + fn direct_color_ht_job_count(plan: &J2kDirectColorPlan) -> usize { + plan.component_plans.iter().map(direct_ht_job_count).sum() + } + + fn fixture_openhtj2k_ht_refinement() -> &'static [u8] { + include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_12_b11.j2k") + } + + fn fixture_openhtj2k_ht_refinement_pixels() -> &'static [u8] { + include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_12_b11.gray") + } + + fn fixture_openhtj2k_ht_refinement_odd() -> &'static [u8] { + include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k") + } + + fn fixture_openhtj2k_ht_refinement_odd_pixels() -> &'static [u8] { + include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_09_b11.gray") + } + + fn gradient_pixels(width: u32, height: u32, components: u8) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * components as usize); + for y in 0..height { + for x in 0..width { + for component in 0..components { + pixels.push(((x * 3 + y * 5 + u32::from(component) * 41) & 0xff) as u8); + } + } + } + pixels + } + + fn roi_fixture(classic: bool, components: u8) -> Vec { + let width = 64; + let height = 64; + let pixels = gradient_pixels(width, height, components); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + if classic { + encode(&pixels, width, height, components, 8, false, &options) + .expect("encode ROI classic fixture") + } else { + encode_htj2k(&pixels, width, height, components, 8, false, &options) + .expect("encode ROI HT fixture") + } + } + + fn crop_interleaved( + full: &[u8], + full_width: u32, + channels: usize, + roi: (u32, u32, u32, u32), + ) -> Vec { + let (x, y, width, height) = roi; + let mut out = Vec::with_capacity(width as usize * height as usize * channels); + let row_bytes = full_width as usize * channels; + let roi_row_bytes = width as usize * channels; + for row in y as usize..(y + height) as usize { + let start = row * row_bytes + x as usize * channels; + out.extend_from_slice(&full[start..start + roi_row_bytes]); + } + out + } + + fn count_decode_work(bytes: &[u8], roi: Option<(u32, u32, u32, u32)>) -> DecodeWorkCounter { + let image = Image::new(bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + match roi { + Some(roi) => { + image + .decode_region_with_context(roi, &mut context) + .expect("region decode with counter"); + } + None => { + image + .decode_with_context(&mut context) + .expect("full decode with counter"); + } + } + let counters = context.tile_decode_context.debug_counters; + DecodeWorkCounter { + classic_code_blocks: counters.decoded_code_blocks, + ht_code_blocks: 0, + idwt_output_samples: counters.idwt_output_samples, + } + } + + #[test] + fn roi_decode_matches_full_crop_for_classic_and_htj2k_gray_and_rgb() { + let cases = [ + (true, 1_u8, true, false), + (true, 3_u8, false, false), + (false, 1_u8, true, false), + (false, 3_u8, false, false), + ]; + let rois = [ + (20, 18, 17, 19), + (0, 0, 9, 11), + (63, 63, 1, 1), + (7, 5, 13, 9), + (0, 0, 64, 64), + ]; + + for (classic, components, expect_gray, has_alpha) in cases { + let bytes = roi_fixture(classic, components); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let full = image.decode().expect("full decode"); + let channels = components as usize; + for roi in rois { + let region = image.decode_region(roi).expect("region decode"); + assert_eq!(matches!(region.color_space, ColorSpace::Gray), expect_gray); + assert_eq!(region.has_alpha, has_alpha); + assert_eq!( + region.data, + crop_interleaved(&full, 64, channels, roi), + "classic={classic} components={components} roi={roi:?}" + ); + } + } + } + + #[test] + fn roi_decode_prunes_code_blocks_and_idwt_work_for_classic_and_htj2k() { + let roi = (48, 48, 16, 16); + for classic in [true, false] { + let bytes = { + let pixels = gradient_pixels(128, 128, 1); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..EncodeOptions::default() + }; + if classic { + encode(&pixels, 128, 128, 1, 8, false, &options) + .expect("encode classic work fixture") + } else { + encode_htj2k(&pixels, 128, 128, 1, 8, false, &options) + .expect("encode ht work fixture") + } + }; + let full = count_decode_work(&bytes, None); + let region = count_decode_work(&bytes, Some(roi)); + + assert!( + region.code_blocks() > 0 && region.code_blocks() < full.code_blocks(), + "ROI should decode fewer code-blocks for classic={classic}; full={}, region={}", + full.code_blocks(), + region.code_blocks() + ); + assert!( + region.idwt_output_samples > 0 + && region.idwt_output_samples < full.idwt_output_samples, + "ROI should produce fewer IDWT output samples for classic={classic}; full={}, region={}", + full.idwt_output_samples, + region.idwt_output_samples + ); + } + } + + #[test] + fn region_decode_reuses_region_sized_component_storage() { + let bytes = fixture(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + + let bitmap = image + .decode_region_with_context((1, 0, 1, 2), &mut context) + .expect("region decode"); + + assert_eq!((bitmap.width, bitmap.height), (1, 2)); + assert!(context + .tile_decode_context + .channel_data + .iter() + .all(|component| component.container.truncated().len() == 2)); + } + + #[test] + fn native_region_decode_reuses_region_sized_component_storage() { + let bytes = fixture(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + + let bitmap = image + .decode_native_region_with_context((1, 0, 1, 2), &mut context) + .expect("native region decode"); + + assert_eq!((bitmap.width, bitmap.height), (1, 2)); + assert!(context + .tile_decode_context + .channel_data + .iter() + .all(|component| component.container.truncated().len() == 2)); + } + + #[test] + fn decoder_context_defaults_to_auto_cpu_parallelism() { + let context = DecoderContext::default(); + + assert_eq!(context.cpu_decode_parallelism(), CpuDecodeParallelism::Auto); + } + + #[test] + fn classic_j2k_auto_and_serial_cpu_parallelism_match_pixels() { + let bytes = fixture_multi_block(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut auto_context = DecoderContext::default(); + let mut serial_context = DecoderContext::default(); + serial_context.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + + let auto = image + .decode_with_context(&mut auto_context) + .expect("auto decode"); + let serial = image + .decode_with_context(&mut serial_context) + .expect("serial decode"); + + assert_eq!(auto.data, serial.data); + } + + #[test] + fn htj2k_97_auto_and_serial_cpu_parallelism_match_pixels() { + let width = 128_u32; + let height = 128_u32; + let pixels = (0..width * height) + .map(|idx| ((idx * 17 + idx / width * 31) & 0xff) as u8) + .collect::>(); + let bytes = encode_htj2k( + &pixels, + width, + height, + 1, + 8, + false, + &EncodeOptions { + reversible: false, + guard_bits: 2, + num_decomposition_levels: 5, + ..EncodeOptions::default() + }, + ) + .expect("encode HTJ2K 9/7"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut auto_context = DecoderContext::default(); + let mut serial_context = DecoderContext::default(); + serial_context.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + + let auto = image + .decode_with_context(&mut auto_context) + .expect("auto decode"); + let serial = image + .decode_with_context(&mut serial_context) + .expect("serial decode"); + + assert_eq!(auto.data, serial.data); + } + + #[test] + fn serial_cpu_parallelism_disables_classic_sub_band_parallel_branch() { + assert!(!j2c::should_decode_classic_sub_band_in_parallel( + CpuDecodeParallelism::Serial, + 16 + )); + } + + #[test] + fn grayscale_direct_plan_is_built_without_materializing_channel_data() { + let bytes = fixture_gray(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("build direct plan"); + + assert_eq!(plan.dimensions, (4, 4)); + assert_eq!(plan.bit_depth, 8); + assert!( + !plan.steps.is_empty(), + "direct plan must contain executable steps" + ); + assert!( + plan.steps.iter().any(|step| matches!( + step, + J2kDirectGrayscaleStep::ClassicSubBand(plan) if !plan.jobs.is_empty() + )), + "classic J2K direct plan must contain at least one non-empty classic sub-band job" + ); + assert!( + context.tile_decode_context.channel_data.is_empty(), + "building a direct plan must not materialize host component planes" + ); + } + + #[test] + fn grayscale_direct_plan_honors_target_resolution() { + let bytes = fixture_ht_gray(); + let image = Image::new( + &bytes, + &DecodeSettings { + target_resolution: Some((2, 2)), + ..DecodeSettings::default() + }, + ) + .expect("scaled image"); + let mut context = DecoderContext::default(); + + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("build scaled direct plan"); + + assert_eq!(plan.dimensions, (2, 2)); + assert!(plan.steps.iter().any(|step| matches!( + step, + J2kDirectGrayscaleStep::HtSubBand(plan) if !plan.jobs.is_empty() + ))); + assert!(plan.steps.iter().any(|step| matches!( + step, + J2kDirectGrayscaleStep::Store(store) + if store.output_width == 2 && store.output_height == 2 + ))); + assert!( + context.tile_decode_context.channel_data.is_empty(), + "building a scaled direct plan must not materialize host component planes" + ); + } + + #[test] + fn grayscale_direct_plan_region_prunes_unneeded_ht_code_blocks() { + let bytes = fixture_ht_multi_block(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut full_context = DecoderContext::default(); + let mut roi_context = DecoderContext::default(); + + let full = image + .build_direct_grayscale_plan_with_context(&mut full_context) + .expect("build full direct plan"); + let roi = image + .build_direct_grayscale_plan_region_with_context(&mut roi_context, (0, 0, 2, 2)) + .expect("build ROI direct plan"); + + let full_jobs = direct_ht_job_count(&full); + let roi_jobs = direct_ht_job_count(&roi); + assert!(full_jobs > 1, "fixture must expose multiple HT jobs"); + assert!( + roi_jobs < full_jobs, + "ROI direct plan must prune HT jobs before device preparation" + ); + } + + #[test] + fn color_direct_plan_region_prunes_unneeded_ht_code_blocks() { + let bytes = fixture_ht_rgb_multi_block(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut full_context = DecoderContext::default(); + let mut roi_context = DecoderContext::default(); + + let full = image + .build_direct_color_plan_with_context(&mut full_context) + .expect("build full RGB direct plan"); + let roi = image + .build_direct_color_plan_region_with_context(&mut roi_context, (0, 0, 2, 2)) + .expect("build ROI RGB direct plan"); + + let full_jobs = direct_color_ht_job_count(&full); + let roi_jobs = direct_color_ht_job_count(&roi); + assert!(full_jobs > 3, "fixture must expose multiple RGB HT jobs"); + assert!( + roi_jobs < full_jobs, + "RGB ROI direct plan must prune HT jobs before device preparation" + ); + } + + #[test] + fn color_direct_plan_honors_target_resolution() { + for (name, bytes) in [ + ("classic", { + let pixels = gradient_pixels(8, 8, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + encode(&pixels, 8, 8, 3, 8, false, &options).expect("encode classic rgb8") + }), + ("htj2k", { + let pixels = gradient_pixels(8, 8, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, 8, 8, 3, 8, false, &options).expect("encode ht rgb8") + }), + ] { + let image = Image::new( + &bytes, + &DecodeSettings { + target_resolution: Some((4, 4)), + ..DecodeSettings::default() + }, + ) + .expect("scaled RGB image"); + let mut context = DecoderContext::default(); + + let plan = image + .build_direct_color_plan_with_context(&mut context) + .expect("build scaled direct color plan"); + + assert_eq!(plan.dimensions, (4, 4), "{name}: output dimensions"); + assert_eq!(plan.component_plans.len(), 3, "{name}: component count"); + for component_plan in &plan.component_plans { + assert_eq!(component_plan.dimensions, (4, 4), "{name}: component dims"); + assert!(component_plan.steps.iter().any(|step| matches!( + step, + J2kDirectGrayscaleStep::Store(store) + if store.output_width == 4 && store.output_height == 4 + ))); + } + assert!( + context.tile_decode_context.channel_data.is_empty(), + "{name}: building a scaled color direct plan must not materialize host component planes" + ); + } + } + + #[test] + fn direct_color_cpu_rgb8_executor_matches_scaled_region_decode() { + for (name, bytes) in [ + ("classic", { + let pixels = gradient_pixels(16, 16, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + encode(&pixels, 16, 16, 3, 8, false, &options).expect("encode classic rgb8") + }), + ("htj2k", { + let pixels = gradient_pixels(16, 16, 3); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + encode_htj2k(&pixels, 16, 16, 3, 8, false, &options).expect("encode ht rgb8") + }), + ] { + let image = Image::new( + &bytes, + &DecodeSettings { + target_resolution: Some((4, 4)), + ..DecodeSettings::default() + }, + ) + .expect("scaled RGB image"); + let mut expected_context = DecoderContext::default(); + let expected_full = image + .decode_with_context(&mut expected_context) + .expect("decode scaled reference"); + let output_region = J2kRect { + x0: 1, + y0: 1, + x1: 3, + y1: 3, + }; + let mut direct_context = DecoderContext::default(); + let plan = image + .build_direct_color_plan_region_with_context( + &mut direct_context, + ( + output_region.x0, + output_region.y0, + output_region.width(), + output_region.height(), + ), + ) + .expect("build direct RGB region plan"); + + let stride = output_region.width() as usize * 3; + let mut direct = vec![0_u8; stride * output_region.height() as usize]; + let mut scratch = J2kDirectCpuScratch::new(); + execute_direct_color_plan_rgb8_into( + &plan, + output_region, + &mut scratch, + &mut direct, + stride, + ) + .expect("execute direct RGB plan"); + + let mut expected = Vec::with_capacity(direct.len()); + let full_stride = image.width() as usize * 3; + for y in output_region.y0..output_region.y1 { + let start = y as usize * full_stride + output_region.x0 as usize * 3; + expected.extend_from_slice(&expected_full.data[start..start + stride]); + } + + assert_eq!(direct, expected, "{name}: direct RGB output"); + + let rgba_stride = output_region.width() as usize * 4; + let mut direct_rgba = vec![0_u8; rgba_stride * output_region.height() as usize]; + execute_direct_color_plan_rgba8_into( + &plan, + output_region, + &mut scratch, + &mut direct_rgba, + rgba_stride, + ) + .expect("execute direct RGBA plan"); + + let mut expected_rgba = Vec::with_capacity(direct_rgba.len()); + for rgb in expected.chunks_exact(3) { + expected_rgba.extend_from_slice(rgb); + expected_rgba.push(255); + } + assert_eq!(direct_rgba, expected_rgba, "{name}: direct RGBA output"); + } + } + + #[test] + fn htj2k_grayscale_direct_plan_contains_ht_sub_band_steps() { + let bytes = fixture_ht_gray(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + + let plan = image + .build_direct_grayscale_plan_with_context(&mut context) + .expect("build direct plan"); + + assert!( + plan.steps.iter().any(|step| matches!( + step, + J2kDirectGrayscaleStep::HtSubBand(plan) if !plan.jobs.is_empty() + )), + "HTJ2K direct plan must contain at least one non-empty HT sub-band decode step" + ); + } + + #[test] + fn ht_decoder_hook_is_used_for_htj2k_codeblocks() { + let pixels: Vec = (0..16).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k(&pixels, 4, 4, 1, 8, false, &options).expect("encode ht"); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut hooked_context = DecoderContext::default(); + let mut hook = FailingHtDecoder { called: false }; + let error = match image.decode_components_with_ht_decoder(&mut hooked_context, &mut hook) { + Ok(_) => panic!("hooked decode must use external HT decoder"), + Err(error) => error, + }; + + assert!(hook.called, "HT decoder hook must be invoked"); + assert_eq!( + error, + DecodeError::Decoding(DecodingError::CodeBlockDecodeFailure) + ); + } + + #[test] + fn openhtj2k_conformance_fixture_exercises_refinement_passes() { + for fixture in [ + ( + "ds0_ht_12_b11", + fixture_openhtj2k_ht_refinement(), + fixture_openhtj2k_ht_refinement_pixels(), + (3, 5), + 8, + 2, + 4, + ), + ( + "ds0_ht_09_b11", + fixture_openhtj2k_ht_refinement_odd(), + fixture_openhtj2k_ht_refinement_odd_pixels(), + (17, 37), + 14, + 14, + 629, + ), + ] { + let ( + name, + codestream, + expected_pixels, + dimensions, + blocks, + refinement_jobs, + zero_diffs, + ) = fixture; + let image = Image::new(codestream, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let mut hook = CapturingHtDecoder::default(); + + let components = image + .decode_components_with_ht_decoder(&mut context, &mut hook) + .expect("decode OpenHTJ2K HTJ2K fixture"); + + assert!( + hook.called, + "{name}: HTJ2K fixture must use HT code-block decode" + ); + assert!( + hook.refinement_jobs > 0, + "{name}: OpenHTJ2K fixture must contain non-empty refinement segments" + ); + assert!( + hook.max_coding_passes > 1, + "{name}: OpenHTJ2K fixture must exercise more than the cleanup pass" + ); + assert_eq!(hook.blocks, blocks, "{name}: HT code-block count"); + assert_eq!( + hook.refinement_jobs, refinement_jobs, + "{name}: refinement job count" + ); + assert_eq!(hook.max_coding_passes, 3, "{name}: max HT coding passes"); + assert_eq!(components.dimensions(), dimensions, "{name}: dimensions"); + assert_eq!(components.planes().len(), 1, "{name}: component planes"); + + let decoded: Vec = components.planes()[0] + .samples() + .iter() + .map(|sample| sample.round().clamp(0.0, 255.0) as u8) + .collect(); + assert_eq!(decoded, expected_pixels, "{name}: decoded pixels"); + + let mut zero_context = DecoderContext::default(); + let mut zero_hook = ZeroRefinementHtDecoder; + let zeroed_components = image + .decode_components_with_ht_decoder(&mut zero_context, &mut zero_hook) + .expect("decode OpenHTJ2K fixture with zeroed refinement bytes"); + let actual_zero_diffs = components.planes()[0] + .samples() + .iter() + .zip(zeroed_components.planes()[0].samples()) + .filter(|(actual, zeroed)| (*actual - *zeroed).abs() > f32::EPSILON) + .count(); + assert_eq!( + actual_zero_diffs, zero_diffs, + "{name}: zeroing refinement bytes must change decoded samples" + ); + } + } + + #[test] + fn openhtj2k_refinement_phase_limited_decode_differs_and_records_ht_stats() { + let image = Image::new( + fixture_openhtj2k_ht_refinement_odd(), + &DecodeSettings::default(), + ) + .expect("image"); + let mut full_context = DecoderContext::default(); + + let (full_samples, full_decoded) = { + let full_components = image + .decode_components_with_context(&mut full_context) + .expect("full native decode of OpenHTJ2K refinement fixture"); + let full_samples = full_components.planes()[0].samples().to_vec(); + let full_decoded: Vec = full_samples + .iter() + .map(|sample| sample.round().clamp(0.0, 255.0) as u8) + .collect(); + (full_samples, full_decoded) + }; + assert_eq!( + full_decoded, + fixture_openhtj2k_ht_refinement_odd_pixels(), + "full decode must match the checked-in OpenHTJ2K oracle" + ); + + let stats = full_context + .tile_decode_context + .debug_counters + .ht_phase_stats; + assert_eq!(stats.blocks, 14, "HT block count"); + assert_eq!(stats.refinement_blocks, 14, "HT refinement block count"); + assert!(stats.cleanup_bytes > 0, "cleanup byte total"); + assert!(stats.refinement_bytes > 0, "refinement byte total"); + + let mut cleanup_context = DecoderContext::default(); + let mut cleanup_hook = CleanupLimitedHtDecoder::default(); + let cleanup_components = image + .decode_components_with_ht_decoder(&mut cleanup_context, &mut cleanup_hook) + .expect("cleanup-limited decode of OpenHTJ2K refinement fixture"); + let cleanup_decoded: Vec = cleanup_components.planes()[0] + .samples() + .iter() + .map(|sample| sample.round().clamp(0.0, 255.0) as u8) + .collect(); + let cleanup_sample_diffs = full_samples + .iter() + .zip(cleanup_components.planes()[0].samples()) + .filter(|(full, cleanup)| (*full - *cleanup).abs() > f32::EPSILON) + .count(); + + assert!( + cleanup_sample_diffs > 0, + "cleanup-limited decode must omit refinement effects" + ); + assert_eq!( + cleanup_decoded, full_decoded, + "fixture refinement differences are below final u8 clamping" + ); + assert_eq!(cleanup_hook.blocks, 14, "hook HT block count"); + assert_eq!( + cleanup_hook.refinement_blocks, 14, + "hook HT refinement block count" + ); + assert!(cleanup_hook.cleanup_bytes > 0, "hook cleanup byte total"); + assert!( + cleanup_hook.refinement_bytes > 0, + "hook refinement byte total" + ); + } + + #[test] + fn scalar_htj2k_encoder_contract_is_cleanup_only() { + let coefficients = (0..64) + .map(|index| { + let magnitude = (index % 7) + 1; + if index % 2 == 0 { + magnitude + } else { + -magnitude + } + }) + .collect::>(); + + let encoded = + encode_ht_code_block_scalar(&coefficients, 8, 8, 8).expect("encode HT code block"); + + assert_eq!( + encoded.num_coding_passes, 1, + "current scalar HTJ2K encoder emits only the cleanup pass" + ); + assert_eq!( + encoded.num_zero_bitplanes, 7, + "current cleanup-only HTJ2K encoder includes one bitplane" + ); + assert!( + !encoded.data.is_empty(), + "non-zero cleanup-only block must still produce payload bytes" + ); + } + + #[test] + fn scalar_htj2k_decode_workspace_matches_fresh_decode_and_reuses_capacity() { + let image = Image::new( + fixture_openhtj2k_ht_refinement_odd(), + &DecodeSettings::default(), + ) + .expect("image"); + let mut context = DecoderContext::default(); + let mut hook = FirstHtJobDecoder::default(); + image + .decode_components_with_ht_decoder(&mut context, &mut hook) + .expect("decode fixture while collecting HT jobs"); + let job = hook + .job + .as_ref() + .expect("fixture must expose an HT decode job") + .borrowed(); + let mut fresh = vec![0.0_f32; job.width as usize * job.height as usize]; + let mut reused = vec![0.0_f32; fresh.len()]; + let mut profiled = vec![0.0_f32; fresh.len()]; + let mut workspace = HtCodeBlockDecodeWorkspace::default(); + let mut profile = HtCodeBlockDecodeProfile::default(); + + decode_ht_code_block_scalar(job, &mut fresh).expect("fresh HT decode"); + decode_ht_code_block_scalar_with_workspace(job, &mut reused, &mut workspace) + .expect("workspace HT decode"); + let first_capacity = workspace.coefficient_capacity(); + decode_ht_code_block_scalar_with_workspace(job, &mut reused, &mut workspace) + .expect("second workspace HT decode"); + decode_ht_code_block_scalar_with_workspace_profiled( + job, + &mut profiled, + &mut workspace, + &mut profile, + ) + .expect("profiled workspace HT decode"); + + assert_eq!(reused, fresh); + assert_eq!(profiled, fresh); + assert!(first_capacity >= fresh.len()); + assert_eq!(workspace.coefficient_capacity(), first_capacity); + assert_eq!(profile.blocks, 1); + assert!(profile.cleanup_bytes > 0); + } + + #[test] + fn classic_decoder_hook_is_used_for_j2k_codeblocks() { + let bytes = fixture(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut hooked_context = DecoderContext::default(); + let mut hook = FailingClassicDecoder { called: false }; + let error = match image.decode_components_with_ht_decoder(&mut hooked_context, &mut hook) { + Ok(_) => panic!("hooked decode must use external classic decoder"), + Err(error) => error, + }; + + assert!(hook.called, "classic decoder hook must be invoked"); + assert_eq!( + error, + DecodeError::Decoding(DecodingError::CodeBlockDecodeFailure) + ); + } + + #[test] + fn classic_sub_band_decoder_hook_is_used_for_j2k_codeblocks() { + let bytes = fixture_multi_block(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut hooked_context = DecoderContext::default(); + let mut hook = FailingClassicBatchDecoder { called: false }; + let error = match image.decode_components_with_ht_decoder(&mut hooked_context, &mut hook) { + Ok(_) => panic!("hooked decode must use external classic batch decoder"), + Err(error) => error, + }; + + assert!(hook.called, "classic sub-band decoder hook must be invoked"); + assert_eq!( + error, + DecodeError::Decoding(DecodingError::CodeBlockDecodeFailure) + ); + } + + // ----------------------------------------------------------------------- + // Sanity tests for the four scalar-reference exports + // ----------------------------------------------------------------------- + + #[test] + fn forward_dwt53_reference_matches_internal_path() { + // 4×4 constant-ramp input; 1 decomposition level. + let samples: Vec = (0..16).map(|i| i as f32).collect(); + let out = forward_dwt53_reference(&samples, 4, 4, 1); + + // Internal path + let internal = j2c::fdwt::forward_dwt(&samples, 4, 4, 1, true); + + assert_eq!(out.ll, internal.ll, "LL subband mismatch"); + assert_eq!(out.ll_width, internal.ll_width, "LL width mismatch"); + assert_eq!(out.ll_height, internal.ll_height, "LL height mismatch"); + assert_eq!(out.levels.len(), internal.levels.len(), "level count"); + for (pub_lvl, int_lvl) in out.levels.iter().zip(internal.levels.iter()) { + assert_eq!(pub_lvl.hl, int_lvl.hl, "HL mismatch"); + assert_eq!(pub_lvl.lh, int_lvl.lh, "LH mismatch"); + assert_eq!(pub_lvl.hh, int_lvl.hh, "HH mismatch"); + } + } + + #[test] + fn forward_rct_reference_matches_internal_path() { + // Single pixel: R=100, G=150, B=200 + let planes = vec![vec![100.0f32], vec![150.0f32], vec![200.0f32]]; + let result = forward_rct_reference(planes.clone()); + + // Internal path + let mut internal = planes; + j2c::forward_mct::forward_rct(&mut internal); + + assert_eq!(result, internal, "RCT output mismatch"); + // Y = floor((100 + 300 + 200) / 4) = 150 + assert_eq!(result[0][0], 150.0, "Y component"); + assert_eq!(result[1][0], 50.0, "Cb component"); + assert_eq!(result[2][0], -50.0, "Cr component"); + } + + #[test] + fn forward_ict_reference_matches_internal_path() { + let planes = vec![vec![100.0f32], vec![150.0f32], vec![200.0f32]]; + let result = forward_ict_reference(planes.clone()); + + let mut internal = planes; + j2c::forward_mct::forward_ict(&mut internal); + + assert_eq!(result, internal, "ICT output mismatch"); + } + + #[test] + fn forward_dwt97_reference_matches_internal_path() { + let samples = (0..64) + .map(|idx| { + f32::from(u8::try_from((idx * 19 + idx / 3) & 0xff).expect("masked sample fits u8")) + - 128.0 + }) + .collect::>(); + let result = forward_dwt97_reference(&samples, 8, 8, 2); + let internal = j2c::fdwt::forward_dwt(&samples, 8, 8, 2, false); + + assert_eq!(result.ll, internal.ll, "DWT 9/7 LL mismatch"); + assert_eq!(result.ll_width, internal.ll_width); + assert_eq!(result.ll_height, internal.ll_height); + assert_eq!(result.levels.len(), internal.levels.len()); + for (actual, expected) in result.levels.iter().zip(internal.levels.iter()) { + assert_eq!(actual.hl, expected.hl, "DWT 9/7 HL mismatch"); + assert_eq!(actual.lh, expected.lh, "DWT 9/7 LH mismatch"); + assert_eq!(actual.hh, expected.hh, "DWT 9/7 HH mismatch"); + } + } + + #[test] + fn quantize_reversible_reference_matches_internal_path() { + let coefficients = vec![3.7f32, -8.2, 0.5, -0.5, 10.0]; + let exponent = 8u16; + let mantissa = 0u16; + let range_bits = 8u8; + + let result = + quantize_reversible_reference(&coefficients, exponent, mantissa, range_bits, true); + + // Internal path + let step = j2c::quantize::QuantStepSize { exponent, mantissa }; + let internal = j2c::quantize::quantize_subband(&coefficients, &step, range_bits, true); + + assert_eq!(result, internal, "quantize output mismatch"); + // reversible: round to nearest + assert_eq!(result[0], 4, "3.7 rounds to 4"); + assert_eq!(result[1], -8, "-8.2 rounds to -8"); + } + + #[test] + fn quantize_subband_reference_matches_irreversible_internal_path() { + let coefficients = vec![3.7f32, -8.2, 0.5, -0.5, 10.0]; + let exponent = 8u16; + let mantissa = 256u16; + let range_bits = 8u8; + + let result = + quantize_subband_reference(&coefficients, exponent, mantissa, range_bits, false); + + let step = j2c::quantize::QuantStepSize { exponent, mantissa }; + let internal = j2c::quantize::quantize_subband(&coefficients, &step, range_bits, false); + + assert_eq!(result, internal, "irreversible quantize output mismatch"); + } + + #[test] + fn deinterleave_reference_matches_internal_path() { + // 2-pixel RGB8 unsigned: [R0,G0,B0, R1,G1,B1] + let pixels: Vec = vec![128, 64, 200, 10, 20, 30]; + let result = deinterleave_reference(&pixels, 2, 3, 8, false); + + let internal = j2c::encode::deinterleave_to_f32(&pixels, 2, 3, 8, false); + + assert_eq!(result, internal, "deinterleave output mismatch"); + assert_eq!(result.len(), 3, "three component planes"); + assert_eq!(result[0].len(), 2, "two pixels per plane"); + // unsigned 8-bit with level shift: val - 128 + assert!((result[0][0] - 0.0f32).abs() < 1e-6, "R0 level-shifted"); + assert!((result[1][0] - (-64.0f32)).abs() < 1e-6, "G0 level-shifted"); + assert!((result[2][0] - 72.0f32).abs() < 1e-6, "B0 level-shifted"); + } +} diff --git a/crates/signinum-j2k-native/src/log.rs b/crates/j2k-native/src/log.rs similarity index 100% rename from crates/signinum-j2k-native/src/log.rs rename to crates/j2k-native/src/log.rs diff --git a/crates/signinum-j2k-native/src/math.rs b/crates/j2k-native/src/math.rs similarity index 94% rename from crates/signinum-j2k-native/src/math.rs rename to crates/j2k-native/src/math.rs index 63e18afe..9f946f8f 100644 --- a/crates/signinum-j2k-native/src/math.rs +++ b/crates/j2k-native/src/math.rs @@ -36,7 +36,7 @@ mod inner { #[inline(always)] pub(crate) fn mul_add(self, mul: Self, addend: Self) -> Self { Self { - inner: self.inner.madd(mul.inner, addend.inner), + inner: self.inner.mul_add(mul.inner, addend.inner), } } @@ -49,7 +49,7 @@ mod inner { #[inline(always)] pub(crate) fn store(self, slice: &mut [f32]) { - slice[..SIMD_WIDTH].copy_from_slice(&self.inner.val); + self.inner.store_slice(&mut slice[..SIMD_WIDTH]); } #[inline(always)] @@ -498,54 +498,6 @@ pub(crate) fn log2_f32(x: f32) -> f32 { } } -#[inline(always)] -pub(crate) fn floor_f64(x: f64) -> f64 { - #[cfg(feature = "std")] - { - x.floor() - } - #[cfg(not(feature = "std"))] - { - libm::floor(x) - } -} - -#[inline(always)] -pub(crate) fn round_f64(x: f64) -> f64 { - #[cfg(feature = "std")] - { - x.round() - } - #[cfg(not(feature = "std"))] - { - libm::round(x) - } -} - -#[inline(always)] -pub(crate) fn log2_f64(x: f64) -> f64 { - #[cfg(feature = "std")] - { - x.log2() - } - #[cfg(not(feature = "std"))] - { - libm::log2(x) - } -} - -#[inline(always)] -pub(crate) fn powi_f64(x: f64, n: i32) -> f64 { - #[cfg(feature = "std")] - { - x.powi(n) - } - #[cfg(not(feature = "std"))] - { - libm::pow(x, f64::from(n)) - } -} - #[inline(always)] pub(crate) fn pow2i(exp: i32) -> f32 { if exp >= 0 { diff --git a/crates/j2k-native/src/packet_math.rs b/crates/j2k-native/src/packet_math.rs new file mode 100644 index 00000000..8503f6c7 --- /dev/null +++ b/crates/j2k-native/src/packet_math.rs @@ -0,0 +1,85 @@ +//! Packet-header segment-length math shared between the CPU packetizer and +//! GPU packetization planners. +//! +//! Hidden from the rendered public API: GPU planner crates call these helpers +//! internally so their packet headers stay bit-identical with the CPU +//! encoder's signaling decisions. + +/// Returns whether `value` can be signalled in `bits` bits. +#[inline] +pub fn value_fits_in_bits(value: u32, bits: u32) -> bool { + bits >= u32::BITS || value < (1u32 << bits) +} + +/// Calculate number of bits needed to encode a segment length. +#[inline] +pub fn bits_for_length(l_block: u32, num_coding_passes: u8) -> u32 { + let log2_passes = if num_coding_passes <= 1 { + 0 + } else { + (num_coding_passes as u32).ilog2() + }; + l_block + log2_passes +} + +/// Number of bits needed to encode an HT cleanup segment length, folding +/// placeholder passes into the pass count. +#[inline] +pub fn bits_for_ht_cleanup_length(l_block: u32, raw_num_passes: u8) -> u32 { + let placeholder_groups = u32::from(raw_num_passes.saturating_sub(1)) / 3; + let placeholder_passes = placeholder_groups * 3; + l_block + (placeholder_passes + 1).ilog2() +} + +/// Splits an HT code-block packet contribution into its +/// `(cleanup, refinement)` segment lengths, validating them against the +/// contribution payload length. +pub fn ht_segment_lengths( + num_coding_passes: u8, + data_len: usize, + ht_cleanup_length: u32, + ht_refinement_length: u32, +) -> Result<(u32, u32), &'static str> { + if num_coding_passes == 0 { + if data_len == 0 && ht_cleanup_length == 0 && ht_refinement_length == 0 { + return Ok((0, 0)); + } + return Err("empty HTJ2K packet contribution must not carry segment bytes"); + } + + let data_len = + u32::try_from(data_len).map_err(|_| "HTJ2K packet contribution exceeds u32 length")?; + if num_coding_passes == 1 { + if ht_refinement_length != 0 { + return Err("single-pass HTJ2K packet contribution must not carry refinement bytes"); + } + let cleanup_length = if ht_cleanup_length == 0 { + data_len + } else { + ht_cleanup_length + }; + if cleanup_length != data_len { + return Err("single-pass HTJ2K packet contribution length mismatch"); + } + return Ok((cleanup_length, 0)); + } + + if ht_cleanup_length == 0 || ht_refinement_length == 0 { + return Err("multi-pass HTJ2K packet contribution requires cleanup/refinement lengths"); + } + if ht_cleanup_length + .checked_add(ht_refinement_length) + .ok_or("multi-pass HTJ2K packet contribution length overflow")? + != data_len + { + return Err("multi-pass HTJ2K packet contribution length mismatch"); + } + if !(2..65535).contains(&ht_cleanup_length) { + return Err("HTJ2K cleanup segment length is out of range"); + } + if ht_refinement_length >= 2047 { + return Err("HTJ2K refinement segment length is out of range"); + } + + Ok((ht_cleanup_length, ht_refinement_length)) +} diff --git a/crates/signinum-j2k-native/src/profile.rs b/crates/j2k-native/src/profile.rs similarity index 81% rename from crates/signinum-j2k-native/src/profile.rs rename to crates/j2k-native/src/profile.rs index af165ac1..7f8597f5 100644 --- a/crates/signinum-j2k-native/src/profile.rs +++ b/crates/j2k-native/src/profile.rs @@ -2,7 +2,7 @@ use core::cell::RefCell; #[cfg(feature = "std")] -use signinum_profile::{profile_stage_mode_from_env, ProfileStageMode}; +use j2k_profile::{profile_stage_mode_from_env, ProfileStageMode}; #[cfg(feature = "std")] use std::sync::OnceLock; @@ -10,15 +10,15 @@ use std::sync::OnceLock; use std::time::Instant; #[cfg(all(test, feature = "std"))] -pub(crate) use signinum_profile::ProfileSummary; +pub(crate) use j2k_profile::ProfileSummary; #[cfg(feature = "std")] -const PROFILE_ENV_VAR: &str = "SIGNINUM_J2K_PROFILE_STAGES"; +const PROFILE_ENV_VAR: &str = "J2K_PROFILE_STAGES"; #[cfg(feature = "std")] #[cfg(test)] pub(crate) fn parse_profile_env_flag(value: Option<&str>) -> bool { - signinum_profile::profile_stage_mode_from_value(value) != ProfileStageMode::Disabled + j2k_profile::profile_stage_mode_from_value(value) != ProfileStageMode::Disabled } #[cfg(feature = "std")] @@ -65,7 +65,7 @@ pub(crate) fn elapsed_us(_start: Option) -> u128 { #[cfg(feature = "std")] pub(crate) fn emit_profile_row(op: &str, path: &str, fields: &[(&str, u128)]) { - signinum_profile::emit_profile_row_u128( + j2k_profile::emit_profile_row_u128( profile_stage_mode(), &PROFILE_SUMMARY, "j2k", @@ -80,8 +80,8 @@ pub(crate) fn emit_profile_row(_op: &str, _path: &str, _fields: &[(&str, u128)]) #[cfg(feature = "std")] thread_local! { - static PROFILE_SUMMARY: RefCell = - RefCell::new(signinum_profile::ProfileSummary::default()); + static PROFILE_SUMMARY: RefCell = + RefCell::new(j2k_profile::ProfileSummary::default().emit_on_drop()); } #[cfg(all(test, feature = "std"))] @@ -128,7 +128,7 @@ mod tests { assert_eq!( summary.format_rows(), vec![ - "signinum_profile_summary codec=j2k op=encode path=cpu count=2 block_encode_us_sum=132 block_encode_us_avg=66 deinterleave_us_sum=24 deinterleave_us_avg=12 dwt_us_sum=72 dwt_us_avg=36 total_us_sum=228 total_us_avg=114" + "j2k_profile_summary codec=j2k op=encode path=cpu count=2 block_encode_us_sum=132 block_encode_us_avg=66 deinterleave_us_sum=24 deinterleave_us_avg=12 dwt_us_sum=72 dwt_us_avg=36 total_us_sum=228 total_us_avg=114" ] ); } diff --git a/crates/signinum-j2k-native/src/reader.rs b/crates/j2k-native/src/reader.rs similarity index 92% rename from crates/signinum-j2k-native/src/reader.rs rename to crates/j2k-native/src/reader.rs index f283140a..80068e2b 100644 --- a/crates/signinum-j2k-native/src/reader.rs +++ b/crates/j2k-native/src/reader.rs @@ -145,7 +145,7 @@ impl<'a> BitReader<'a> { pub(crate) fn needs_to_read_stuff_bit(&mut self) -> bool { // B.10.1: "If the value of the byte is 0xFF, the next byte includes an extra zero bit // stuffed into the MSB." - self.bit_pos() == 7 && self.data[self.byte_pos()] == 0xff + self.bit_pos() == 7 && self.data.get(self.byte_pos()) == Some(&0xff) } #[inline] @@ -179,3 +179,18 @@ impl<'a> BitReader<'a> { self.clone().read_marker().ok() } } + +#[cfg(test)] +mod tests { + use super::BitReader; + + #[test] + fn stuffing_check_at_eof_returns_none_instead_of_panicking() { + let mut reader = BitReader { + data: &[], + cur_pos: 7, + }; + + assert_eq!(reader.read_bits_with_stuffing(1), None); + } +} diff --git a/crates/signinum-j2k-native/src/writer.rs b/crates/j2k-native/src/writer.rs similarity index 99% rename from crates/signinum-j2k-native/src/writer.rs rename to crates/j2k-native/src/writer.rs index 2bc2eeb6..90b8693e 100644 --- a/crates/signinum-j2k-native/src/writer.rs +++ b/crates/j2k-native/src/writer.rs @@ -98,6 +98,7 @@ impl BitWriter { #[cfg(test)] mod tests { use super::*; + use alloc::vec; #[test] fn test_write_bits_basic() { diff --git a/crates/j2k-native/tests/bench_harness.rs b/crates/j2k-native/tests/bench_harness.rs new file mode 100644 index 00000000..dd852e65 --- /dev/null +++ b/crates/j2k-native/tests/bench_harness.rs @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[test] +fn native_bench_exposes_classic_and_htj2k_code_block_surface() { + let tier1_bench = include_str!("../benches/tier1_bitplane.rs"); + let sigprop_bench = include_str!("../benches/htj2k_sigprop_phase.rs"); + let direct_cpu_bench = include_str!("../benches/direct_cpu.rs"); + + for expected in [ + "tier1_bitplane_decode", + "tier1_bitplane_encode", + "htj2k_cleanup_decode", + "htj2k_cleanup_encode", + "htj2k_cleanup_encode_distribution", + "rho_eq_uq_64x64", + "htj2k_cleanup_encode_parallel_granularity", + "serial_128_blocks", + "rayon_par_iter_global_128_blocks", + "rayon_par_iter_threads", + "rayon_par_chunks_128_blocks", + "htj2k_cleanup_encode_parallel_batch_size", + "j2k_tier1_encode_parallel_batch_size", + "serial_blocks", + "rayon_par_iter_global_blocks", + "htj2k_refinement_fixture_decode", + "htj2k_refinement_block_decode", + "ds0_ht_09_b11_full", + "ds0_ht_09_b11_cleanup", + "ds0_ht_09_b11_sigprop", + "ds0_ht_09_b11_magref_full", + "decode_64x64", + "encode_64x64", + ] { + assert!( + tier1_bench.contains(expected), + "native benchmark is missing `{expected}`" + ); + } + + for expected in [ + "htj2k_refinement_sigprop_phase", + "ds0_ht_09_b11_sigprop_only", + "htj2k_cpuupload_decode_batch", + "ds0_ht_09_b11_scalar_batch", + ] { + assert!( + sigprop_bench.contains(expected), + "native SigProp benchmark is missing `{expected}`" + ); + } + + for expected in [ + "j2k_native_direct_cpu_color_plan", + "htj2k_rgb8_roi256_q4_fresh_scratch", + "htj2k_rgb8_roi256_q4_reuse_scratch", + "htj2k_rgba8_roi256_q4_reuse_scratch", + ] { + assert!( + direct_cpu_bench.contains(expected), + "native direct CPU benchmark is missing `{expected}`" + ); + } +} diff --git a/crates/signinum-j2k-native/tests/component_planes.rs b/crates/j2k-native/tests/component_planes.rs similarity index 54% rename from crates/signinum-j2k-native/tests/component_planes.rs rename to crates/j2k-native/tests/component_planes.rs index 6d1f3568..5c2f7228 100644 --- a/crates/signinum-j2k-native/tests/component_planes.rs +++ b/crates/j2k-native/tests/component_planes.rs @@ -1,4 +1,4 @@ -use signinum_j2k_native::{encode, DecodeSettings, DecoderContext, EncodeOptions, Image}; +use j2k_native::{encode, DecodeSettings, DecoderContext, EncodeOptions, Image}; fn fixture() -> Vec { let pixels = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]; @@ -38,3 +38,31 @@ fn decoded_components_expose_component_planes() { } assert_eq!(interleaved, bitmap.data); } + +#[test] +fn decoded_region_components_expose_cropped_component_planes() { + let bytes = fixture(); + let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); + let mut context = DecoderContext::default(); + let bitmap = image + .decode_region_with_context((1, 0, 1, 2), &mut DecoderContext::default()) + .expect("bitmap"); + let planes = image + .decode_region_components_with_context((1, 0, 1, 2), &mut context) + .expect("component region decode"); + + assert_eq!(planes.dimensions(), (1, 2)); + assert_eq!(planes.planes().len(), 3); + assert!(planes + .planes() + .iter() + .all(|plane| plane.samples().len() == 2)); + + let mut interleaved = Vec::with_capacity(6); + for idx in 0..2 { + for plane in planes.planes() { + interleaved.push(plane.samples()[idx].round() as u8); + } + } + assert_eq!(interleaved, bitmap.data); +} diff --git a/crates/j2k-native/tests/empty_cmap.rs b/crates/j2k-native/tests/empty_cmap.rs new file mode 100644 index 00000000..367b2183 --- /dev/null +++ b/crates/j2k-native/tests/empty_cmap.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Regression scaffold for JP2 palette/component-map validation. + +use j2k_native::{ + encode, ColorError, DecodeError, DecodeSettings, EncodeOptions, FormatError, Image, +}; + +fn jp2_box(box_type: &[u8; 4], payload: &[u8]) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(&(8u32 + payload.len() as u32).to_be_bytes()); + out.extend_from_slice(box_type); + out.extend_from_slice(payload); + out +} + +#[test] +fn empty_cmap_with_palette_returns_error() { + let pixels: Vec = (0..16).collect(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let codestream = encode(&pixels, 4, 4, 1, 8, false, &options).expect("encode fixture"); + let jp2 = jp2_with_empty_cmap(&codestream); + + let result = Image::new(&jp2, &DecodeSettings::default()).and_then(|image| image.decode()); + + let err = result.expect_err("empty cmap must be rejected explicitly"); + assert!( + matches!( + err, + DecodeError::Format(FormatError::InvalidBox) + | DecodeError::Color(ColorError::PaletteResolutionFailed) + ), + "unexpected empty cmap error: {err:?}" + ); +} + +fn jp2_with_empty_cmap(codestream: &[u8]) -> Vec { + let ihdr = { + let mut payload = Vec::new(); + payload.extend_from_slice(&4_u32.to_be_bytes()); + payload.extend_from_slice(&4_u32.to_be_bytes()); + payload.extend_from_slice(&1_u16.to_be_bytes()); + payload.extend_from_slice(&[7, 7, 0, 0]); + jp2_box(b"ihdr", &payload) + }; + let colr = jp2_box(b"colr", &[1, 0, 0, 0, 0, 0, 17]); + let pclr = { + let mut payload = Vec::new(); + payload.extend_from_slice(&1_u16.to_be_bytes()); + payload.push(1); + payload.push(7); + payload.push(0); + jp2_box(b"pclr", &payload) + }; + let cmap = jp2_box(b"cmap", &[]); + + let mut jp2h_payload = Vec::new(); + jp2h_payload.extend_from_slice(&ihdr); + jp2h_payload.extend_from_slice(&colr); + jp2h_payload.extend_from_slice(&pclr); + jp2h_payload.extend_from_slice(&cmap); + + let mut out = Vec::new(); + out.extend_from_slice(&jp2_box(b"jP ", &[0x0d, 0x0a, 0x87, 0x0a])); + out.extend_from_slice(&jp2_box(b"ftyp", b"jp2 \0\0\0\0jp2 ")); + out.extend_from_slice(&jp2_box(b"jp2h", &jp2h_payload)); + out.extend_from_slice(&jp2_box(b"jp2c", codestream)); + out +} diff --git a/crates/j2k-native/tests/encode_coefficients.rs b/crates/j2k-native/tests/encode_coefficients.rs new file mode 100644 index 00000000..baea1737 --- /dev/null +++ b/crates/j2k-native/tests/encode_coefficients.rs @@ -0,0 +1,338 @@ +use j2k_native::{ + encode, encode_precomputed_htj2k_53, encode_precomputed_htj2k_97, DecodeSettings, + EncodeOptions, Image, J2kForwardDwt53Level, J2kForwardDwt53Output, J2kForwardDwt97Level, + J2kForwardDwt97Output, PrecomputedHtj2k53Component, PrecomputedHtj2k53Image, + PrecomputedHtj2k97Component, PrecomputedHtj2k97Image, +}; + +#[test] +fn native_encode_rejects_dimension_overflow_before_length_check() { + let err = encode( + &[0; 16], + u32::MAX, + u32::MAX, + 4, + 16, + false, + &EncodeOptions::default(), + ) + .expect_err("dimension overflow should be rejected"); + + assert_eq!(err, "image dimensions overflow"); +} + +#[test] +fn native_decode_rejects_zero_target_resolution() { + let bytes = encode( + &[10, 20, 30, 40], + 2, + 2, + 1, + 8, + false, + &EncodeOptions::default(), + ) + .expect("encode fixture"); + let settings = DecodeSettings { + target_resolution: Some((0, 1)), + ..DecodeSettings::default() + }; + + assert!(Image::new(&bytes, &settings).is_err()); +} + +#[test] +fn jp2_decode_rejects_short_channel_definition_box() { + let codestream = encode( + &[10, 20, 30, 40, 50, 60], + 2, + 1, + 3, + 8, + false, + &EncodeOptions::default(), + ) + .expect("encode RGB fixture"); + let jp2 = wrap_codestream_jp2_with_short_cdef(&codestream, 2, 1, 3, 8); + + let image = Image::new(&jp2, &DecodeSettings::default()).expect("parse JP2 fixture"); + let err = image + .decode() + .expect_err("short cdef must not silently drop a component"); + + assert_eq!(err.to_string(), "invalid channel definition"); +} + +#[test] +fn precomputed_zero_grayscale_53_coefficients_decode_with_native_decoder() { + let image = PrecomputedHtj2k53Image { + width: 8, + height: 8, + bit_depth: 8, + signed: false, + components: vec![PrecomputedHtj2k53Component { + x_rsiz: 1, + y_rsiz: 1, + dwt: zero_dwt53(8, 8), + }], + }; + let bytes = encode_precomputed_htj2k_53(&image, &precomputed_options()) + .expect("encode precomputed HTJ2K coefficients"); + let decoded = Image::new(&bytes, &DecodeSettings::default()) + .expect("native parser accepts codestream") + .decode_native() + .expect("native decoder accepts codestream"); + + assert_eq!((decoded.width, decoded.height), (8, 8)); + assert_eq!(decoded.num_components, 1); + assert!(decoded.data.iter().all(|&sample| sample == 128)); +} + +#[test] +fn precomputed_encode_writes_component_sampling_in_siz() { + let image = PrecomputedHtj2k53Image { + width: 16, + height: 16, + bit_depth: 8, + signed: false, + components: vec![ + PrecomputedHtj2k53Component { + x_rsiz: 1, + y_rsiz: 1, + dwt: zero_dwt53(16, 16), + }, + PrecomputedHtj2k53Component { + x_rsiz: 2, + y_rsiz: 2, + dwt: zero_dwt53(8, 8), + }, + PrecomputedHtj2k53Component { + x_rsiz: 2, + y_rsiz: 2, + dwt: zero_dwt53(8, 8), + }, + ], + }; + let bytes = encode_precomputed_htj2k_53(&image, &precomputed_options()) + .expect("encode precomputed subsampled HTJ2K coefficients"); + let siz = find_marker(&bytes, 0x51).expect("SIZ marker"); + let component_info = siz + 40; + + assert_eq!(bytes[component_info + 1], 1); + assert_eq!(bytes[component_info + 2], 1); + assert_eq!(bytes[component_info + 4], 2); + assert_eq!(bytes[component_info + 5], 2); + assert_eq!(bytes[component_info + 7], 2); + assert_eq!(bytes[component_info + 8], 2); +} + +#[test] +fn precomputed_encode_rejects_component_sampling_geometry_mismatch() { + let image = PrecomputedHtj2k53Image { + width: 16, + height: 16, + bit_depth: 8, + signed: false, + components: vec![PrecomputedHtj2k53Component { + x_rsiz: 2, + y_rsiz: 2, + dwt: zero_dwt53(16, 16), + }], + }; + + let err = encode_precomputed_htj2k_53(&image, &precomputed_options()) + .expect_err("component DWT geometry must match SIZ sampling"); + + assert_eq!(err, "precomputed DWT component dimensions mismatch"); +} + +#[test] +fn precomputed_encode_rejects_recursive_level_geometry_mismatch() { + let mut dwt = zero_dwt53(8, 8); + dwt.levels[0].low_width = 3; + let image = PrecomputedHtj2k53Image { + width: 8, + height: 8, + bit_depth: 8, + signed: false, + components: vec![PrecomputedHtj2k53Component { + x_rsiz: 1, + y_rsiz: 1, + dwt, + }], + }; + + let err = encode_precomputed_htj2k_53(&image, &precomputed_options()) + .expect_err("level geometry must match recursive 5/3 expectations"); + + assert_eq!(err, "precomputed DWT recursive geometry mismatch"); +} + +#[test] +fn precomputed_zero_grayscale_97_coefficients_decode_with_native_decoder() { + let image = PrecomputedHtj2k97Image { + width: 8, + height: 8, + bit_depth: 8, + signed: false, + components: vec![PrecomputedHtj2k97Component { + x_rsiz: 1, + y_rsiz: 1, + dwt: zero_dwt97(8, 8), + }], + }; + let bytes = encode_precomputed_htj2k_97(&image, &precomputed_lossy_options()) + .expect("encode precomputed 9/7 HTJ2K coefficients"); + let decoded = Image::new(&bytes, &DecodeSettings::default()) + .expect("native parser accepts 9/7 codestream") + .decode_native() + .expect("native decoder accepts 9/7 codestream"); + + assert_eq!((decoded.width, decoded.height), (8, 8)); + assert_eq!(decoded.num_components, 1); + assert!(decoded.data.iter().all(|&sample| sample == 128)); +} + +#[test] +fn precomputed_97_encode_rejects_component_sampling_geometry_mismatch() { + let image = PrecomputedHtj2k97Image { + width: 16, + height: 16, + bit_depth: 8, + signed: false, + components: vec![PrecomputedHtj2k97Component { + x_rsiz: 2, + y_rsiz: 2, + dwt: zero_dwt97(16, 16), + }], + }; + + let err = encode_precomputed_htj2k_97(&image, &precomputed_lossy_options()) + .expect_err("component DWT geometry must match SIZ sampling"); + + assert_eq!(err, "precomputed DWT component dimensions mismatch"); +} + +fn precomputed_options() -> EncodeOptions { + EncodeOptions { + num_decomposition_levels: 1, + reversible: true, + use_ht_block_coding: true, + use_mct: false, + validate_high_throughput_codestream: false, + ..EncodeOptions::default() + } +} + +fn precomputed_lossy_options() -> EncodeOptions { + EncodeOptions { + num_decomposition_levels: 1, + reversible: false, + use_ht_block_coding: true, + use_mct: false, + validate_high_throughput_codestream: false, + ..EncodeOptions::default() + } +} + +fn zero_dwt53(width: u32, height: u32) -> J2kForwardDwt53Output { + let low_width = width.div_ceil(2); + let low_height = height.div_ceil(2); + let high_width = width / 2; + let high_height = height / 2; + + J2kForwardDwt53Output { + ll: vec![0.0; (low_width * low_height) as usize], + ll_width: low_width, + ll_height: low_height, + levels: vec![J2kForwardDwt53Level { + hl: vec![0.0; (high_width * low_height) as usize], + lh: vec![0.0; (low_width * high_height) as usize], + hh: vec![0.0; (high_width * high_height) as usize], + width, + height, + low_width, + low_height, + high_width, + high_height, + }], + } +} + +fn zero_dwt97(width: u32, height: u32) -> J2kForwardDwt97Output { + let low_width = width.div_ceil(2); + let low_height = height.div_ceil(2); + let high_width = width / 2; + let high_height = height / 2; + + J2kForwardDwt97Output { + ll: vec![0.0; (low_width * low_height) as usize], + ll_width: low_width, + ll_height: low_height, + levels: vec![J2kForwardDwt97Level { + hl: vec![0.0; (high_width * low_height) as usize], + lh: vec![0.0; (low_width * high_height) as usize], + hh: vec![0.0; (high_width * high_height) as usize], + width, + height, + low_width, + low_height, + high_width, + high_height, + }], + } +} + +fn find_marker(codestream: &[u8], marker: u8) -> Option { + codestream + .windows(2) + .position(|window| window == [0xff, marker]) +} + +fn wrap_codestream_jp2_with_short_cdef( + codestream: &[u8], + width: u32, + height: u32, + components: u16, + bit_depth: u8, +) -> Vec { + let mut bytes = Vec::new(); + push_box(&mut bytes, b"jP ", &[0x0D, 0x0A, 0x87, 0x0A]); + push_box( + &mut bytes, + b"ftyp", + &[b'j', b'p', b'2', b' ', 0, 0, 0, 0, b'j', b'p', b'2', b' '], + ); + + let mut jp2h = Vec::new(); + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&height.to_be_bytes()); + ihdr.extend_from_slice(&width.to_be_bytes()); + ihdr.extend_from_slice(&components.to_be_bytes()); + ihdr.extend_from_slice(&[bit_depth.saturating_sub(1), 7, 0, 0]); + push_box(&mut jp2h, b"ihdr", &ihdr); + + let mut colr = vec![1, 0, 0]; + colr.extend_from_slice(&16_u32.to_be_bytes()); + push_box(&mut jp2h, b"colr", &colr); + + let mut cdef = Vec::new(); + cdef.extend_from_slice(&2_u16.to_be_bytes()); + for (channel_index, association) in [(0_u16, 1_u16), (1, 2)] { + cdef.extend_from_slice(&channel_index.to_be_bytes()); + cdef.extend_from_slice(&0_u16.to_be_bytes()); + cdef.extend_from_slice(&association.to_be_bytes()); + } + push_box(&mut jp2h, b"cdef", &cdef); + + push_box(&mut bytes, b"jp2h", &jp2h); + push_box(&mut bytes, b"jp2c", codestream); + bytes +} + +fn push_box(bytes: &mut Vec, box_type: &[u8; 4], payload: &[u8]) { + let len = (8 + payload.len()) as u32; + bytes.extend_from_slice(&len.to_be_bytes()); + bytes.extend_from_slice(box_type); + bytes.extend_from_slice(payload); +} diff --git a/crates/signinum-j2k-native/tests/idwt_helpers.rs b/crates/j2k-native/tests/idwt_helpers.rs similarity index 93% rename from crates/signinum-j2k-native/tests/idwt_helpers.rs rename to crates/j2k-native/tests/idwt_helpers.rs index 79a201c6..412655d8 100644 --- a/crates/signinum-j2k-native/tests/idwt_helpers.rs +++ b/crates/j2k-native/tests/idwt_helpers.rs @@ -1,4 +1,4 @@ -use signinum_j2k_native::idwt_band_index; +use j2k_native::idwt_band_index; #[test] fn idwt_band_index_matches_low_and_high_band_coordinate_mapping() { diff --git a/crates/j2k-native/tests/plt_decode.rs b/crates/j2k-native/tests/plt_decode.rs new file mode 100644 index 00000000..2cfbdca5 --- /dev/null +++ b/crates/j2k-native/tests/plt_decode.rs @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_native::{encode, DecodeSettings, EncodeOptions, Image}; + +fn marker_offset(codestream: &[u8], marker: u8) -> usize { + codestream + .windows(2) + .position(|window| window == [0xFF, marker]) + .unwrap_or_else(|| panic!("missing marker FF{marker:02X}")) +} + +fn packet_length_bytes(mut length: usize) -> Vec { + let mut groups = vec![(length & 0x7F) as u8]; + length >>= 7; + + while length > 0 { + groups.push((length & 0x7F) as u8); + length >>= 7; + } + + groups + .iter() + .rev() + .enumerate() + .map(|(idx, group)| { + if idx + 1 == groups.len() { + *group + } else { + *group | 0x80 + } + }) + .collect() +} + +fn insert_plt(mut codestream: Vec, packet_length: usize) -> Vec { + let sod_offset = marker_offset(&codestream, 0x93); + let mut plt = vec![0xFF, 0x58]; + let packet_length_bytes = packet_length_bytes(packet_length); + let marker_len = u16::try_from(3 + packet_length_bytes.len()).expect("PLT marker length"); + plt.extend_from_slice(&marker_len.to_be_bytes()); + plt.push(0); + plt.extend_from_slice(&packet_length_bytes); + + codestream.splice(sod_offset..sod_offset, plt.iter().copied()); + + let sot_offset = marker_offset(&codestream, 0x90); + let psot = u32::from_be_bytes([ + codestream[sot_offset + 6], + codestream[sot_offset + 7], + codestream[sot_offset + 8], + codestream[sot_offset + 9], + ]); + codestream[sot_offset + 6..sot_offset + 10] + .copy_from_slice(&(psot + u32::try_from(plt.len()).unwrap()).to_be_bytes()); + + codestream +} + +fn insert_plm(mut codestream: Vec, packet_length: usize) -> Vec { + let sot_offset = marker_offset(&codestream, 0x90); + let mut plm = vec![0xFF, 0x57]; + let packet_length_bytes = packet_length_bytes(packet_length); + let marker_len = u16::try_from(7 + packet_length_bytes.len()).expect("PLM marker length"); + plm.extend_from_slice(&marker_len.to_be_bytes()); + plm.push(0); + plm.extend_from_slice( + &u32::try_from(packet_length_bytes.len()) + .unwrap() + .to_be_bytes(), + ); + plm.extend_from_slice(&packet_length_bytes); + + codestream.splice(sot_offset..sot_offset, plm); + codestream +} + +fn fixture_pixels() -> Vec { + (0..64u8) + .flat_map(|y| (0..64u8).map(move |x| x.wrapping_mul(3).wrapping_add(y))) + .collect::>() +} + +fn one_packet_fixture() -> Vec { + let pixels = fixture_pixels(); + encode( + &pixels, + 64, + 64, + 1, + 8, + false, + &EncodeOptions { + num_decomposition_levels: 0, + ..EncodeOptions::default() + }, + ) + .expect("one-packet fixture encode") +} + +fn tile_body_len(codestream: &[u8]) -> usize { + let sod_offset = marker_offset(codestream, 0x93); + let eoc_offset = marker_offset(codestream, 0xD9); + eoc_offset - (sod_offset + 2) +} + +fn strict_decode(codestream: &[u8]) -> j2k_native::Result { + Image::new( + codestream, + &DecodeSettings { + strict: true, + ..DecodeSettings::default() + }, + )? + .decode_native() +} + +#[test] +fn strict_decode_accepts_matching_plt_packet_length() { + let codestream = one_packet_fixture(); + let packet_length = tile_body_len(&codestream); + let with_plt = insert_plt(codestream, packet_length); + + let decoded = strict_decode(&with_plt).expect("matching PLT packet length decodes"); + + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.num_components, 1); + assert_eq!(decoded.data, fixture_pixels()); +} + +#[test] +fn strict_decode_rejects_wrong_plt_packet_length() { + let codestream = one_packet_fixture(); + let wrong_packet_length = tile_body_len(&codestream) + 1; + let with_plt = insert_plt(codestream, wrong_packet_length); + + assert!( + strict_decode(&with_plt).is_err(), + "strict decode must reject PLT packet lengths that do not match consumed packet bytes" + ); +} + +#[test] +fn strict_decode_accepts_matching_plm_packet_length() { + let codestream = one_packet_fixture(); + let packet_length = tile_body_len(&codestream); + let with_plm = insert_plm(codestream, packet_length); + + let decoded = strict_decode(&with_plm).expect("matching PLM packet length decodes"); + + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.num_components, 1); + assert_eq!(decoded.data, fixture_pixels()); +} + +#[test] +fn strict_decode_rejects_wrong_plm_packet_length() { + let codestream = one_packet_fixture(); + let wrong_packet_length = tile_body_len(&codestream) + 1; + let with_plm = insert_plm(codestream, wrong_packet_length); + + assert!( + strict_decode(&with_plm).is_err(), + "strict decode must reject PLM packet lengths that do not match consumed packet bytes" + ); +} diff --git a/crates/j2k-native/tests/poc_decode.rs b/crates/j2k-native/tests/poc_decode.rs new file mode 100644 index 00000000..caf96c4e --- /dev/null +++ b/crates/j2k-native/tests/poc_decode.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_native::{encode, DecodeSettings, EncodeOptions, EncodeProgressionOrder, Image}; + +fn marker_offset(codestream: &[u8], marker: u8) -> usize { + codestream + .windows(2) + .position(|window| window == [0xFF, marker]) + .unwrap_or_else(|| panic!("missing marker FF{marker:02X}")) +} + +fn cod_marker_end(codestream: &[u8]) -> usize { + let cod_offset = marker_offset(codestream, 0x52); + let lcod = u16::from_be_bytes([codestream[cod_offset + 2], codestream[cod_offset + 3]]); + cod_offset + 2 + usize::from(lcod) +} + +fn force_cod_lrcp_and_insert_main_header_cprl_poc(mut codestream: Vec) -> Vec { + let cod_offset = marker_offset(&codestream, 0x52); + assert_eq!( + codestream[cod_offset + 5], + 0x04, + "fixture must start as CPRL" + ); + codestream[cod_offset + 5] = 0x00; + + let poc = [ + 0xFF, 0x5F, // POC + 0x00, 0x09, // Lpoc + 0x00, // RSpoc + 0x00, // CSpoc + 0x00, 0x01, // LYEpoc + 0x02, // REpoc: two resolutions + 0x03, // CEpoc: three components + 0x04, // Ppoc: CPRL + ]; + codestream.splice( + cod_marker_end(&codestream)..cod_marker_end(&codestream), + poc, + ); + codestream +} + +fn force_cod_lrcp_and_insert_tile_header_cprl_poc(mut codestream: Vec) -> Vec { + let cod_offset = marker_offset(&codestream, 0x52); + assert_eq!( + codestream[cod_offset + 5], + 0x04, + "fixture must start as CPRL" + ); + codestream[cod_offset + 5] = 0x00; + + let sod_offset = marker_offset(&codestream, 0x93); + let poc = [ + 0xFF, 0x5F, // POC + 0x00, 0x09, // Lpoc + 0x00, // RSpoc + 0x00, // CSpoc + 0x00, 0x01, // LYEpoc + 0x02, // REpoc: two resolutions + 0x03, // CEpoc: three components + 0x04, // Ppoc: CPRL + ]; + codestream.splice(sod_offset..sod_offset, poc); + + let sot_offset = marker_offset(&codestream, 0x90); + let psot = u32::from_be_bytes([ + codestream[sot_offset + 6], + codestream[sot_offset + 7], + codestream[sot_offset + 8], + codestream[sot_offset + 9], + ]); + codestream[sot_offset + 6..sot_offset + 10] + .copy_from_slice(&(psot + u32::try_from(poc.len()).unwrap()).to_be_bytes()); + + codestream +} + +fn force_cod_lrcp_and_insert_main_header_cprl_poc_with_sentinel_ends( + mut codestream: Vec, +) -> Vec { + let cod_offset = marker_offset(&codestream, 0x52); + assert_eq!( + codestream[cod_offset + 5], + 0x04, + "fixture must start as CPRL" + ); + codestream[cod_offset + 5] = 0x00; + + let poc = [ + 0xFF, 0x5F, // POC + 0x00, 0x09, // Lpoc + 0x00, // RSpoc + 0x00, // CSpoc + 0x00, 0x02, // LYEpoc: clamp to actual layer count + 0x21, // REpoc: official vectors use 33 as an all-resolutions bound + 0xFF, // CEpoc: official profile 0 vectors use 255 as an all-components bound + 0x04, // Ppoc: CPRL + ]; + codestream.splice( + cod_marker_end(&codestream)..cod_marker_end(&codestream), + poc, + ); + codestream +} + +fn cprl_fixture(pixels: &[u8]) -> Vec { + encode( + pixels, + 64, + 64, + 3, + 8, + false, + &EncodeOptions { + num_decomposition_levels: 1, + progression_order: EncodeProgressionOrder::Cprl, + use_mct: false, + ..EncodeOptions::default() + }, + ) + .expect("CPRL fixture encode") +} + +fn fixture_pixels() -> Vec { + let mut pixels = Vec::with_capacity(64 * 64 * 3); + for y in 0..64u8 { + for x in 0..64u8 { + pixels.push(x.wrapping_mul(3).wrapping_add(y)); + pixels.push(y.wrapping_mul(5).wrapping_add(x / 2)); + pixels.push(x.wrapping_mul(7).wrapping_sub(y.wrapping_mul(2))); + } + } + pixels +} + +#[test] +fn main_header_poc_sentinel_end_bounds_are_clamped_to_tile_shape() { + let pixels = fixture_pixels(); + let cprl = cprl_fixture(&pixels); + let with_poc = force_cod_lrcp_and_insert_main_header_cprl_poc_with_sentinel_ends(cprl); + + let decoded = Image::new(&with_poc, &DecodeSettings::default()) + .expect("POC codestream with sentinel end bounds parses") + .decode_native() + .expect("POC codestream with sentinel end bounds decodes"); + + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.num_components, 3); + assert_eq!(decoded.data, pixels); +} + +#[test] +fn main_header_poc_changes_packet_iteration_order() { + let pixels = fixture_pixels(); + let cprl = cprl_fixture(&pixels); + let with_poc = force_cod_lrcp_and_insert_main_header_cprl_poc(cprl); + + let decoded = Image::new(&with_poc, &DecodeSettings::default()) + .expect("POC codestream parses") + .decode_native() + .expect("POC codestream decodes"); + + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.num_components, 3); + assert_eq!(decoded.data, pixels); +} + +#[test] +fn tile_header_poc_changes_packet_iteration_order() { + let pixels = fixture_pixels(); + let cprl = cprl_fixture(&pixels); + let with_poc = force_cod_lrcp_and_insert_tile_header_cprl_poc(cprl); + + let decoded = Image::new(&with_poc, &DecodeSettings::default()) + .expect("tile-header POC codestream parses") + .decode_native() + .expect("tile-header POC codestream decodes"); + + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.num_components, 3); + assert_eq!(decoded.data, pixels); +} diff --git a/crates/j2k-native/tests/tile_header_markers.rs b/crates/j2k-native/tests/tile_header_markers.rs new file mode 100644 index 00000000..b204d454 --- /dev/null +++ b/crates/j2k-native/tests/tile_header_markers.rs @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::path::Path; + +use j2k_native::{encode, DecodeSettings, EncodeOptions, Image}; + +fn marker_offset(codestream: &[u8], marker: u8) -> usize { + codestream + .windows(2) + .position(|window| window == [0xFF, marker]) + .unwrap_or_else(|| panic!("missing marker FF{marker:02X}")) +} + +fn cod_marker_end(codestream: &[u8]) -> usize { + let cod_offset = marker_offset(codestream, 0x52); + let lcod = u16::from_be_bytes([codestream[cod_offset + 2], codestream[cod_offset + 3]]); + cod_offset + 2 + usize::from(lcod) +} + +fn insert_main_header_rgn(mut codestream: Vec, roi_shift: u8, style: u8) -> Vec { + let rgn = [ + 0xFF, 0x5E, // RGN + 0x00, 0x05, // Lrgn + 0x00, // Crgn + style, // Srgn + roi_shift, // SPrgn + ]; + codestream.splice( + cod_marker_end(&codestream)..cod_marker_end(&codestream), + rgn, + ); + codestream +} + +fn insert_tile_header_rgn(mut codestream: Vec, roi_shift: u8, style: u8) -> Vec { + let sod_offset = marker_offset(&codestream, 0x93); + let rgn = [ + 0xFF, 0x5E, // RGN + 0x00, 0x05, // Lrgn + 0x00, // Crgn + style, // Srgn + roi_shift, // SPrgn + ]; + codestream.splice(sod_offset..sod_offset, rgn); + + let sot_offset = marker_offset(&codestream, 0x90); + let psot = u32::from_be_bytes([ + codestream[sot_offset + 6], + codestream[sot_offset + 7], + codestream[sot_offset + 8], + codestream[sot_offset + 9], + ]); + codestream[sot_offset + 6..sot_offset + 10] + .copy_from_slice(&(psot + u32::try_from(rgn.len()).unwrap()).to_be_bytes()); + + codestream +} + +#[test] +fn tile_header_rgn_marker_with_zero_shift_is_a_noop() { + let pixels: Vec<_> = (0..64 * 64).map(|idx| (idx % 251) as u8).collect(); + let codestream = encode(&pixels, 64, 64, 1, 8, false, &EncodeOptions::default()) + .expect("lossless fixture encode"); + let with_rgn = insert_tile_header_rgn(codestream, 0, 0); + + let decoded = Image::new(&with_rgn, &DecodeSettings::default()) + .expect("tile-header RGN codestream parses") + .decode_native() + .expect("tile-header RGN codestream decodes"); + + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.num_components, 1); + assert_eq!(decoded.data, pixels); +} + +#[test] +fn tile_header_rgn_with_explicit_style_is_rejected() { + let pixels: Vec<_> = (0..64 * 64).map(|idx| (idx % 251) as u8).collect(); + let codestream = encode(&pixels, 64, 64, 1, 8, false, &EncodeOptions::default()) + .expect("lossless fixture encode"); + let with_rgn = insert_tile_header_rgn(codestream, 7, 1); + + let err = Image::new(&with_rgn, &DecodeSettings::default()) + .expect("tile-header RGN codestream parses") + .decode_native() + .err() + .expect("explicit ROI coding style is unsupported"); + + assert_eq!( + err.to_string(), + "unsupported decoding feature: explicit ROI coding" + ); +} + +#[test] +fn main_header_rgn_with_explicit_style_is_rejected() { + let pixels: Vec<_> = (0..64 * 64).map(|idx| (idx % 251) as u8).collect(); + let codestream = encode(&pixels, 64, 64, 1, 8, false, &EncodeOptions::default()) + .expect("lossless fixture encode"); + let with_rgn = insert_main_header_rgn(codestream, 7, 1); + + let err = Image::new(&with_rgn, &DecodeSettings::default()) + .err() + .expect("explicit ROI coding style is unsupported"); + + assert_eq!( + err.to_string(), + "unsupported decoding feature: explicit ROI coding" + ); +} + +#[test] +fn crafted_siz_with_absurd_tile_grid_is_rejected_without_allocating() { + let pixels: Vec<_> = (0..16 * 16).map(|idx| (idx % 251) as u8).collect(); + let mut codestream = encode(&pixels, 16, 16, 1, 8, false, &EncodeOptions::default()) + .expect("lossless fixture encode"); + + // Rewrite SIZ to a 15,300,000² reference grid of 1×1 tiles. The component + // resolutions go to 255 so image_width stays at the 60,000 dimension cap, + // but the tile grid implies ~2.3e14 tiles — overflowing num_tiles() and + // driving the eager per-tile allocation in tile parsing. + let siz = codestream + .windows(2) + .position(|w| w == [0xFF, 0x51]) + .expect("SIZ marker"); + codestream[siz + 6..siz + 10].copy_from_slice(&15_300_000u32.to_be_bytes()); // Xsiz + codestream[siz + 10..siz + 14].copy_from_slice(&15_300_000u32.to_be_bytes()); // Ysiz + codestream[siz + 22..siz + 26].copy_from_slice(&1u32.to_be_bytes()); // XTsiz + codestream[siz + 26..siz + 30].copy_from_slice(&1u32.to_be_bytes()); // YTsiz + codestream[siz + 41] = 255; // XRsiz (component 0) + codestream[siz + 42] = 255; // YRsiz (component 0) + + let result = + Image::new(&codestream, &DecodeSettings::default()).and_then(|image| image.decode_native()); + let err = result.err().expect("absurd tile grid must be rejected"); + assert_eq!(err.to_string(), "image has too many tiles"); +} + +#[test] +fn iso_p0_03_tile_header_roi_maxshift_matches_reference_when_available() { + let Some(root) = std::env::var_os("J2K_ISO_CONFORMANCE_DIR") else { + return; + }; + let root = Path::new(&root); + let codestream = + std::fs::read(root.join("codestreams_profile0/p0_03.j2k")).expect("read p0_03 codestream"); + let reference = read_pgx( + root.join("reference_class1_profile0/c1p0_03-0.pgx") + .as_path(), + ); + + let decoded = Image::new(&codestream, &DecodeSettings::default()) + .expect("p0_03 parses") + .decode_native() + .expect("p0_03 decodes"); + + assert_eq!(decoded.width, reference.width); + assert_eq!(decoded.height, reference.height); + assert_eq!(decoded.bit_depth, reference.bit_depth); + assert_eq!(decoded.num_components, 1); + assert_eq!(decoded.bytes_per_sample, 1); + + let sign_shift = if reference.signed { + 1_i32 << (reference.bit_depth - 1) + } else { + 0 + }; + let expected = reference + .samples + .iter() + .map(|sample| u8::try_from(sample + sign_shift).expect("4-bit reference sample fits u8")) + .collect::>(); + assert_eq!(decoded.data, expected); +} + +struct PgxImage { + signed: bool, + bit_depth: u8, + width: u32, + height: u32, + samples: Vec, +} + +fn read_pgx(path: &Path) -> PgxImage { + let bytes = std::fs::read(path).expect("read PGX reference"); + let header_end = bytes + .iter() + .position(|byte| *byte == b'\n') + .expect("PGX header terminator"); + let header = std::str::from_utf8(&bytes[..header_end]).expect("PGX header is UTF-8"); + let parts = header.split_whitespace().collect::>(); + assert_eq!(parts.len(), 5); + assert_eq!(parts[0], "PG"); + + let big_endian = match parts[1] { + "ML" => true, + "LM" => false, + endian => panic!("unsupported PGX byte order {endian}"), + }; + let signed = parts[2].starts_with('-'); + let bit_depth = parts[2] + .trim_start_matches(['+', '-']) + .parse::() + .expect("PGX bit depth"); + let width = parts[3].parse::().expect("PGX width"); + let height = parts[4].parse::().expect("PGX height"); + let bytes_per_sample = if bit_depth <= 8 { 1 } else { 2 }; + let payload = &bytes[header_end + 1..]; + assert_eq!( + payload.len(), + width as usize * height as usize * bytes_per_sample + ); + + let samples = if bytes_per_sample == 1 { + payload + .iter() + .map(|byte| { + if signed { + i32::from(*byte as i8) + } else { + i32::from(*byte) + } + }) + .collect() + } else { + payload + .chunks_exact(2) + .map(|chunk| { + let raw = if big_endian { + u16::from_be_bytes([chunk[0], chunk[1]]) + } else { + u16::from_le_bytes([chunk[0], chunk[1]]) + }; + if signed { + i32::from(raw as i16) + } else { + i32::from(raw) + } + }) + .collect() + }; + + PgxImage { + signed, + bit_depth, + width, + height, + samples, + } +} diff --git a/crates/j2k-profile/Cargo.toml b/crates/j2k-profile/Cargo.toml new file mode 100644 index 00000000..e7daaddd --- /dev/null +++ b/crates/j2k-profile/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "j2k-profile" +description = "Internal profiling helpers for j2k crates" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +readme = "README.md" + +[package.metadata.docs.rs] +all-features = true +targets = [] + +[lib] +name = "j2k_profile" +path = "src/lib.rs" + +[features] +default = ["std"] +std = [] + +[lints] +workspace = true diff --git a/crates/j2k-profile/README.md b/crates/j2k-profile/README.md new file mode 100644 index 00000000..a74c44c7 --- /dev/null +++ b/crates/j2k-profile/README.md @@ -0,0 +1,6 @@ +# j2k-profile + +Small profiling and route-summary helper crate for J2K. + +Codec and adapter crates use it to emit stable profile rows and route decisions +without each crate reimplementing environment parsing or summary formatting. diff --git a/crates/signinum-profile/src/lib.rs b/crates/j2k-profile/src/lib.rs similarity index 86% rename from crates/signinum-profile/src/lib.rs rename to crates/j2k-profile/src/lib.rs index a8de97bb..c62b969c 100644 --- a/crates/signinum-profile/src/lib.rs +++ b/crates/j2k-profile/src/lib.rs @@ -1,4 +1,4 @@ -//! Internal profiling helpers shared by the `signinum` workspace crates. +//! Internal profiling helpers shared by the `j2k` workspace crates. #![doc(hidden)] #![cfg_attr(not(feature = "std"), no_std)] @@ -94,7 +94,7 @@ pub fn profile_stage_mode_from_env(key: &str) -> ProfileStageMode { } /// Environment variable that controls GPU route profiling rows or summaries. -pub const GPU_ROUTE_PROFILE_ENV: &str = "SIGNINUM_GPU_ROUTE_PROFILE"; +pub const GPU_ROUTE_PROFILE_ENV: &str = "J2K_GPU_ROUTE_PROFILE"; /// Parses the GPU route profiling mode from an optional environment value. pub fn gpu_route_profile_stage_mode_from_value(value: Option<&str>) -> ProfileStageMode { @@ -141,7 +141,7 @@ pub fn gpu_route_profile_summary() -> ProfileSummary { #[cfg(feature = "std")] thread_local! { static GPU_ROUTE_PROFILE_SUMMARY: std::cell::RefCell = - std::cell::RefCell::new(gpu_route_profile_summary()); + std::cell::RefCell::new(gpu_route_profile_summary().emit_on_drop()); } #[cfg(feature = "std")] @@ -204,19 +204,65 @@ where row } +#[cfg(feature = "std")] +/// Emits a preformatted profile row to stderr. +pub fn emit_profile_line(row: impl AsRef) { + std::eprintln!("{}", row.as_ref()); +} + +#[cfg(feature = "std")] +/// Formats and emits a string-valued profile row to stderr. +pub fn emit_profile_row_now( + codec: impl AsRef, + op: impl AsRef, + path: impl AsRef, + fields: &[(K, V)], +) where + K: AsRef, + V: AsRef, +{ + emit_profile_line(format_profile_row(codec, op, path, fields)); +} + +#[cfg(feature = "std")] +/// Formats and emits an integer-valued profile row to stderr. +pub fn emit_profile_row_u128_now( + codec: impl AsRef, + op: impl AsRef, + path: impl AsRef, + fields: &[(K, u128)], +) where + K: AsRef, +{ + emit_profile_line(format_profile_row_u128(codec, op, path, fields)); +} + fn format_profile_prefix(codec: &str, op: &str, path: &str) -> String { let mut row = String::new(); - write!(row, "signinum_profile codec={codec} op={op} path={path}") - .expect("writing to String failed"); + write!(row, "j2k_profile codec={codec} op={op} path={path}").expect("writing to String failed"); row } /// Aggregates profiling rows by codec, operation, path, and configured labels. -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct ProfileSummary { labels: Vec, numeric_mode: SummaryNumericMode, rows: BTreeMap, + #[cfg(feature = "std")] + emit_on_drop: bool, +} + +impl Clone for ProfileSummary { + fn clone(&self) -> Self { + Self { + labels: self.labels.clone(), + numeric_mode: self.numeric_mode, + rows: self.rows.clone(), + #[cfg(feature = "std")] + emit_on_drop: false, + } + } } impl ProfileSummary { @@ -238,7 +284,26 @@ impl ProfileSummary { labels: labels.into_iter().collect(), numeric_mode, rows: BTreeMap::new(), + #[cfg(feature = "std")] + emit_on_drop: false, + } + } + + /// Emits accumulated rows to stderr when the summary is dropped. + #[cfg(feature = "std")] + #[must_use] + pub fn emit_on_drop(mut self) -> Self { + self.emit_on_drop = true; + self + } + + /// Flushes accumulated rows to stderr and clears them. + #[cfg(feature = "std")] + pub fn flush_to_stderr(&mut self) { + for row in self.format_rows() { + std::eprintln!("{row}"); } + self.rows.clear(); } /// Records a profiling row with string field values. @@ -392,8 +457,8 @@ impl Default for ProfileSummary { #[cfg(feature = "std")] impl Drop for ProfileSummary { fn drop(&mut self) { - for row in self.format_rows() { - std::eprintln!("{row}"); + if self.emit_on_drop { + self.flush_to_stderr(); } } } @@ -453,11 +518,8 @@ impl SummaryRow { fn format_profile_summary_prefix(codec: &str, op: &str, path: &str) -> String { let mut row = String::new(); - write!( - row, - "signinum_profile_summary codec={codec} op={op} path={path}" - ) - .expect("writing to String failed"); + write!(row, "j2k_profile_summary codec={codec} op={op} path={path}") + .expect("writing to String failed"); row } @@ -649,7 +711,7 @@ mod tests { ); assert_eq!( - "signinum_profile codec=jpeg op=decode path=tile/0 rows=4 elapsed_us=12", + "j2k_profile codec=jpeg op=decode path=tile/0 rows=4 elapsed_us=12", row ); } @@ -664,7 +726,7 @@ mod tests { ); assert_eq!( - "signinum_profile codec=j2k op=decode path=tile/1 elapsed_us=34 bytes=99", + "j2k_profile codec=j2k op=decode path=tile/1 elapsed_us=34 bytes=99", row ); } @@ -695,7 +757,7 @@ mod tests { assert_eq!( vec![ - "signinum_profile_summary codec=jpeg op=decode path=tile/0 stage=idct backend=cpu count=2" + "j2k_profile_summary codec=jpeg op=decode path=tile/0 stage=idct backend=cpu count=2" ], summary.format_rows() ); @@ -720,7 +782,7 @@ mod tests { assert_eq!( vec![ - "signinum_profile_summary codec=jpeg op=decode path=tile/0 stage=entropy count=2 bytes_sum=150 elapsed_us_sum=30 elapsed_us_avg=15" + "j2k_profile_summary codec=jpeg op=decode path=tile/0 stage=entropy count=2 bytes_sum=150 elapsed_us_sum=30 elapsed_us_avg=15" ], summary.format_rows() ); @@ -754,7 +816,7 @@ mod tests { ); assert_eq!( - vec!["signinum_profile_summary codec=j2k-cuda op=decode path=tile/2 route=1 count=2"], + vec!["j2k_profile_summary codec=j2k-cuda op=decode path=tile/2 route=1 count=2"], summary.format_rows() ); } @@ -775,7 +837,7 @@ mod tests { assert_eq!( vec![ - "signinum_profile_summary codec=jpeg op=decode path=tile/0 backend=cpu count=1 elapsed_us_sum=8 elapsed_us_avg=8" + "j2k_profile_summary codec=jpeg op=decode path=tile/0 backend=cpu count=1 elapsed_us_sum=8 elapsed_us_avg=8" ], summary.format_rows() ); @@ -800,7 +862,7 @@ mod tests { assert_eq!( vec![ - "signinum_profile_summary codec=j2k op=decode path=tile/1 backend=7 count=2 elapsed_ns_sum=12 elapsed_ns_avg=6" + "j2k_profile_summary codec=j2k op=decode path=tile/1 backend=7 count=2 elapsed_ns_sum=12 elapsed_ns_avg=6" ], summary.format_rows() ); @@ -808,16 +870,21 @@ mod tests { #[cfg(feature = "std")] #[test] - fn profile_summary_drop_formats_rows_without_panic() { - { - let mut summary = ProfileSummary::new([SummaryLabel::same("stage")]); - summary.record_str( - "jpeg", - "decode", - "tile/0", - &[("stage", "emit"), ("elapsed_ms", "4")], - ); - } + fn profile_summary_emit_on_drop_is_explicit_and_not_cloned() { + let mut summary = ProfileSummary::new([SummaryLabel::same("stage")]).emit_on_drop(); + summary.record_str( + "jpeg", + "decode", + "tile/0", + &[("stage", "emit"), ("elapsed_ms", "4")], + ); + + let cloned = summary.clone(); + assert!(summary.emit_on_drop); + assert!(!cloned.emit_on_drop); + + summary.flush_to_stderr(); + assert!(summary.format_rows().is_empty()); } #[cfg(feature = "std")] @@ -858,8 +925,8 @@ mod tests { TEST_SUMMARY.with(|summary| { assert_eq!( vec![ - "signinum_profile_summary codec=jpeg op=decode path=tile/0 stage=1 count=1 elapsed_us_sum=5 elapsed_us_avg=5", - "signinum_profile_summary codec=jpeg op=decode path=tile/0 stage=on count=1 elapsed_us_sum=10 elapsed_us_avg=10", + "j2k_profile_summary codec=jpeg op=decode path=tile/0 stage=1 count=1 elapsed_us_sum=5 elapsed_us_avg=5", + "j2k_profile_summary codec=jpeg op=decode path=tile/0 stage=on count=1 elapsed_us_sum=10 elapsed_us_avg=10", ], summary.borrow().format_rows() ); @@ -869,7 +936,7 @@ mod tests { #[cfg(feature = "std")] #[test] fn parses_stage_mode_from_named_env_var() { - const ENV_KEY: &str = "SIGNINUM_PROFILE_TEST_STAGE_MODE"; + const ENV_KEY: &str = "J2K_PROFILE_TEST_STAGE_MODE"; std::env::set_var(ENV_KEY, "summary"); assert_eq!( @@ -897,7 +964,7 @@ mod tests { assert_eq!( vec![ - "signinum_profile_summary codec=jpeg op=decode path=cpu mode=full fmt=Rgb8 count=1 total_us_sum=5 total_us_avg=5" + "j2k_profile_summary codec=jpeg op=decode path=cpu mode=full fmt=Rgb8 count=1 total_us_sum=5 total_us_avg=5" ], summary.format_rows() ); @@ -958,7 +1025,7 @@ mod tests { assert_eq!( summary.format_rows(), vec![ - "signinum_profile_summary codec=jpeg op=gpu_route path=metal route_op=full request=Metal fmt=Rgb8 decision=metal_kernel reason=none count=2" + "j2k_profile_summary codec=jpeg op=gpu_route path=metal route_op=full request=Metal fmt=Rgb8 decision=metal_kernel reason=none count=2" ] ); } @@ -999,7 +1066,7 @@ mod tests { assert_eq!( vec![ - "signinum_profile_summary codec=jpeg op=decode path=cpu mode=full fmt=Rgb8 count=2 decode_us_sum=12 decode_us_avg=6 total_us_sum=16 total_us_avg=8" + "j2k_profile_summary codec=jpeg op=decode path=cpu mode=full fmt=Rgb8 count=2 decode_us_sum=12 decode_us_avg=6 total_us_sum=16 total_us_avg=8" ], summary.format_rows() ); @@ -1026,7 +1093,7 @@ mod tests { TEST_SUMMARY.with(|summary| { assert_eq!( vec![ - "signinum_profile_summary codec=jpeg op=decode path=cpu stage=entropy count=1 elapsed_us_sum=9 elapsed_us_avg=9" + "j2k_profile_summary codec=jpeg op=decode path=cpu stage=entropy count=1 elapsed_us_sum=9 elapsed_us_avg=9" ], summary.borrow().format_rows() ); diff --git a/crates/signinum-test-support/Cargo.toml b/crates/j2k-test-support/Cargo.toml similarity index 61% rename from crates/signinum-test-support/Cargo.toml rename to crates/j2k-test-support/Cargo.toml index 9ec22ae8..fa1fa9a0 100644 --- a/crates/signinum-test-support/Cargo.toml +++ b/crates/j2k-test-support/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "signinum-test-support" -description = "Dev-only fixtures and generators for signinum workspace tests" +name = "j2k-test-support" +description = "Dev-only fixtures and generators for j2k workspace tests" version.workspace = true edition.workspace = true rust-version.workspace = true @@ -9,9 +9,15 @@ repository.workspace = true publish = false [lib] -name = "signinum_test_support" +name = "j2k_test_support" path = "src/lib.rs" +[features] +j2k-native-fixtures = ["dep:j2k-native"] + +[dependencies] +j2k-native = { path = "../j2k-native", version = "=0.6.0", optional = true } + [lints.rust] unsafe_code = "forbid" unreachable_pub = "warn" diff --git a/crates/signinum-jpeg/fixtures/conformance/baseline_420_16x16.jpg b/crates/j2k-test-support/fixtures/conformance/baseline_420_16x16.jpg similarity index 100% rename from crates/signinum-jpeg/fixtures/conformance/baseline_420_16x16.jpg rename to crates/j2k-test-support/fixtures/conformance/baseline_420_16x16.jpg diff --git a/crates/signinum-jpeg/fixtures/conformance/baseline_420_16x16.rgb b/crates/j2k-test-support/fixtures/conformance/baseline_420_16x16.rgb similarity index 100% rename from crates/signinum-jpeg/fixtures/conformance/baseline_420_16x16.rgb rename to crates/j2k-test-support/fixtures/conformance/baseline_420_16x16.rgb diff --git a/crates/signinum-jpeg/fixtures/conformance/baseline_420_restart_32x16.jpg b/crates/j2k-test-support/fixtures/conformance/baseline_420_restart_32x16.jpg similarity index 100% rename from crates/signinum-jpeg/fixtures/conformance/baseline_420_restart_32x16.jpg rename to crates/j2k-test-support/fixtures/conformance/baseline_420_restart_32x16.jpg diff --git a/crates/signinum-jpeg/fixtures/conformance/baseline_420_restart_32x16.rgb b/crates/j2k-test-support/fixtures/conformance/baseline_420_restart_32x16.rgb similarity index 100% rename from crates/signinum-jpeg/fixtures/conformance/baseline_420_restart_32x16.rgb rename to crates/j2k-test-support/fixtures/conformance/baseline_420_restart_32x16.rgb diff --git a/crates/j2k-test-support/fixtures/conformance/baseline_422_16x8.jpg b/crates/j2k-test-support/fixtures/conformance/baseline_422_16x8.jpg new file mode 100644 index 00000000..60b479c5 Binary files /dev/null and b/crates/j2k-test-support/fixtures/conformance/baseline_422_16x8.jpg differ diff --git a/crates/signinum-jpeg/fixtures/conformance/baseline_422_16x8.rgb b/crates/j2k-test-support/fixtures/conformance/baseline_422_16x8.rgb similarity index 100% rename from crates/signinum-jpeg/fixtures/conformance/baseline_422_16x8.rgb rename to crates/j2k-test-support/fixtures/conformance/baseline_422_16x8.rgb diff --git a/crates/j2k-test-support/fixtures/conformance/baseline_444_8x8.jpg b/crates/j2k-test-support/fixtures/conformance/baseline_444_8x8.jpg new file mode 100644 index 00000000..0f4b494a Binary files /dev/null and b/crates/j2k-test-support/fixtures/conformance/baseline_444_8x8.jpg differ diff --git a/crates/signinum-jpeg/fixtures/conformance/baseline_444_8x8.rgb b/crates/j2k-test-support/fixtures/conformance/baseline_444_8x8.rgb similarity index 100% rename from crates/signinum-jpeg/fixtures/conformance/baseline_444_8x8.rgb rename to crates/j2k-test-support/fixtures/conformance/baseline_444_8x8.rgb diff --git a/crates/signinum-jpeg/fixtures/conformance/grayscale_8x8.gray b/crates/j2k-test-support/fixtures/conformance/grayscale_8x8.gray similarity index 100% rename from crates/signinum-jpeg/fixtures/conformance/grayscale_8x8.gray rename to crates/j2k-test-support/fixtures/conformance/grayscale_8x8.gray diff --git a/crates/signinum-jpeg/fixtures/conformance/grayscale_8x8.jpg b/crates/j2k-test-support/fixtures/conformance/grayscale_8x8.jpg similarity index 100% rename from crates/signinum-jpeg/fixtures/conformance/grayscale_8x8.jpg rename to crates/j2k-test-support/fixtures/conformance/grayscale_8x8.jpg diff --git a/crates/j2k-test-support/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k b/crates/j2k-test-support/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k new file mode 100644 index 00000000..d4f20313 Binary files /dev/null and b/crates/j2k-test-support/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k differ diff --git a/crates/j2k-test-support/src/corpus.rs b/crates/j2k-test-support/src/corpus.rs new file mode 100644 index 00000000..715059cd --- /dev/null +++ b/crates/j2k-test-support/src/corpus.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Filesystem helpers for benchmark corpora. + +use std::path::{Path, PathBuf}; + +/// Expands an environment variable containing platform-separated paths. +pub fn paths_from_env(env_var: &str) -> Vec { + std::env::var_os(env_var) + .map(|paths| std::env::split_paths(&paths).collect()) + .unwrap_or_default() +} + +/// Returns true when `path` has a JPEG file extension. +pub fn is_jpeg_path(path: &Path) -> bool { + path.extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| matches!(ext.to_ascii_lowercase().as_str(), "jpg" | "jpeg")) +} + +/// Collects JPEG file paths from a file or directory tree. +pub fn collect_jpeg_paths(path: &Path) -> Vec { + if path.is_file() { + return is_jpeg_path(path) + .then(|| path.to_path_buf()) + .into_iter() + .collect(); + } + if !path.is_dir() { + return Vec::new(); + } + + let mut out = Vec::new(); + let mut stack = vec![path.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let child = entry.path(); + if child.is_dir() { + stack.push(child); + } else if is_jpeg_path(&child) { + out.push(child); + } + } + } + out.sort(); + out +} diff --git a/crates/j2k-test-support/src/cuda.rs b/crates/j2k-test-support/src/cuda.rs new file mode 100644 index 00000000..cfaf5bc1 --- /dev/null +++ b/crates/j2k-test-support/src/cuda.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Environment gates used by CUDA tests and benches. + +/// Returns true when tests should require a working CUDA runtime. +pub fn cuda_runtime_required() -> bool { + std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_some() +} + +/// Returns true when HTJ2K CUDA tests should require strict GPU execution. +pub fn cuda_htj2k_strict_required() -> bool { + std::env::var_os("J2K_REQUIRE_CUDA_HTJ2K_STRICT").is_some() +} + +/// Returns true when JPEG CUDA tests should require hardware JPEG decode. +pub fn cuda_jpeg_hardware_decode_required() -> bool { + std::env::var_os("J2K_REQUIRE_CUDA_JPEG_HARDWARE_DECODE").is_some() +} + +/// Returns true when CUDA benches should require the runtime instead of skipping. +pub fn cuda_bench_required() -> bool { + std::env::var_os("J2K_REQUIRE_CUDA_BENCH").is_some() + || std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_some() +} diff --git a/crates/j2k-test-support/src/fixtures.rs b/crates/j2k-test-support/src/fixtures.rs new file mode 100644 index 00000000..74e050cd --- /dev/null +++ b/crates/j2k-test-support/src/fixtures.rs @@ -0,0 +1,455 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Shared byte fixtures and container builders for integration tests. + +pub const JPEG_BASELINE_420_16X16: &[u8] = + include_bytes!("../fixtures/conformance/baseline_420_16x16.jpg"); +pub const JPEG_BASELINE_420_16X16_RGB: &[u8] = + include_bytes!("../fixtures/conformance/baseline_420_16x16.rgb"); +pub const JPEG_GRAYSCALE_8X8: &[u8] = include_bytes!("../fixtures/conformance/grayscale_8x8.jpg"); +pub const JPEG_GRAYSCALE_8X8_GRAY: &[u8] = + include_bytes!("../fixtures/conformance/grayscale_8x8.gray"); +pub const JPEG_BASELINE_444_8X8: &[u8] = + include_bytes!("../fixtures/conformance/baseline_444_8x8.jpg"); +pub const JPEG_BASELINE_444_8X8_RGB: &[u8] = + include_bytes!("../fixtures/conformance/baseline_444_8x8.rgb"); +pub const JPEG_BASELINE_422_16X8: &[u8] = + include_bytes!("../fixtures/conformance/baseline_422_16x8.jpg"); +pub const JPEG_BASELINE_422_16X8_RGB: &[u8] = + include_bytes!("../fixtures/conformance/baseline_422_16x8.rgb"); +pub const JPEG_BASELINE_420_RESTART_32X16: &[u8] = + include_bytes!("../fixtures/conformance/baseline_420_restart_32x16.jpg"); +pub const JPEG_BASELINE_420_RESTART_32X16_RGB: &[u8] = + include_bytes!("../fixtures/conformance/baseline_420_restart_32x16.rgb"); + +/// A 16x16 baseline JPEG with 4:2:0 sampling. +pub fn jpeg_baseline_420_16x16() -> Vec { + JPEG_BASELINE_420_16X16.to_vec() +} + +/// Reference RGB pixels for [`jpeg_baseline_420_16x16`]. +pub fn jpeg_baseline_420_16x16_rgb() -> Vec { + JPEG_BASELINE_420_16X16_RGB.to_vec() +} + +/// An 8x8 grayscale baseline JPEG. +pub fn jpeg_grayscale_8x8() -> Vec { + JPEG_GRAYSCALE_8X8.to_vec() +} + +/// Reference grayscale pixels for [`jpeg_grayscale_8x8`]. +pub fn jpeg_grayscale_8x8_gray() -> Vec { + JPEG_GRAYSCALE_8X8_GRAY.to_vec() +} + +/// An 8x8 baseline JPEG with 4:4:4 sampling. +pub fn jpeg_baseline_444_8x8() -> Vec { + JPEG_BASELINE_444_8X8.to_vec() +} + +/// Reference RGB pixels for [`jpeg_baseline_444_8x8`]. +pub fn jpeg_baseline_444_8x8_rgb() -> Vec { + JPEG_BASELINE_444_8X8_RGB.to_vec() +} + +/// A 16x8 baseline JPEG with 4:2:2 sampling. +pub fn jpeg_baseline_422_16x8() -> Vec { + JPEG_BASELINE_422_16X8.to_vec() +} + +/// Reference RGB pixels for [`jpeg_baseline_422_16x8`]. +pub fn jpeg_baseline_422_16x8_rgb() -> Vec { + JPEG_BASELINE_422_16X8_RGB.to_vec() +} + +/// A 32x16 baseline JPEG with 4:2:0 sampling and restart coding. +pub fn jpeg_baseline_420_restart_32x16() -> Vec { + JPEG_BASELINE_420_RESTART_32X16.to_vec() +} + +/// Reference RGB pixels for [`jpeg_baseline_420_restart_32x16`]. +pub fn jpeg_baseline_420_restart_32x16_rgb() -> Vec { + JPEG_BASELINE_420_RESTART_32X16_RGB.to_vec() +} + +/// Minimal grayscale JPEG used by CLI tests. +pub fn minimal_gray8_jpeg() -> Vec { + jpeg_grayscale_8x8() +} + +/// Minimal grayscale JPEG with caller-provided dimensions and one entropy byte. +/// +/// This is intentionally not a complete image for large dimensions; tests use +/// it to exercise header validation and row-streaming paths without allocating +/// full-image entropy payloads. +pub fn minimal_grayscale_jpeg_with_dimensions(width: u16, height: u16) -> Vec { + let mut bytes = grayscale_jpeg_header(width, height); + bytes.extend_from_slice(&[0x00, 0xff, 0xd9]); + bytes +} + +/// Baseline grayscale JPEG with one zero-DC entropy byte per MCU. +pub fn baseline_grayscale_jpeg(width: u16, height: u16) -> Vec { + let mut bytes = grayscale_jpeg_header(width, height); + let mcu_cols = u32::from(width).div_ceil(8); + let mcu_rows = u32::from(height).div_ceil(8); + let mcu_count = (mcu_cols * mcu_rows) as usize; + bytes.extend(core::iter::repeat_n(0x00, mcu_count)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// Minimal 16x16 baseline JPEG with 4:2:0 sampling. +pub fn minimal_baseline_jpeg() -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(&[0xff, 0xd8]); + out.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + out.extend(core::iter::repeat_n(1u8, 64)); + out.extend_from_slice(&[ + 0xff, 0xc0, 0x00, 17, 8, 0, 16, 0, 16, 3, 1, 0x22, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + out.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xaa, + ]); + out.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xbb, + ]); + out.extend_from_slice(&[0xff, 0xda, 0x00, 12, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0]); + out.extend_from_slice(&[0x00, 0xff, 0xd9]); + out +} + +/// Minimal baseline JPEG with a DRI marker inserted before SOS. +/// +/// # Panics +/// +/// Panics if the generated minimal fixture no longer contains an SOS marker. +pub fn minimal_baseline_jpeg_with_restart_interval(interval: u16) -> Vec { + let mut bytes = minimal_baseline_jpeg(); + let sos_pos = bytes + .windows(2) + .position(|window| window == [0xff, 0xda]) + .expect("minimal fixture includes SOS marker"); + bytes.splice( + sos_pos..sos_pos, + [ + 0xff, + 0xdd, + 0x00, + 0x04, + (interval >> 8) as u8, + interval as u8, + ], + ); + bytes +} + +/// Restart-coded grayscale JPEG with one zero-DC block per MCU. +pub fn restart_coded_grayscale_jpeg(width: u16, height: u16) -> Vec { + let mut bytes = grayscale_jpeg_prefix(width, height); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); + append_grayscale_huffman_and_scan_header(&mut bytes); + + let mcu_cols = u32::from(width).div_ceil(8); + let mcu_rows = u32::from(height).div_ceil(8); + let mcu_count = (mcu_cols * mcu_rows) as usize; + for mcu in 0..mcu_count { + bytes.push(0x00); + if mcu + 1 != mcu_count { + bytes.extend_from_slice(&[0xff, 0xd0 | ((mcu as u8) & 0x07)]); + } + } + + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn grayscale_jpeg_header(width: u16, height: u16) -> Vec { + let mut bytes = grayscale_jpeg_prefix(width, height); + append_grayscale_huffman_and_scan_header(&mut bytes); + bytes +} + +fn grayscale_jpeg_prefix(width: u16, height: u16) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(core::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, + 0xc0, + 0x00, + 11, + 8, + (height >> 8) as u8, + height as u8, + (width >> 8) as u8, + width as u8, + 1, + 1, + 0x11, + 0, + ]); + bytes +} + +fn append_grayscale_huffman_and_scan_header(bytes: &mut Vec) { + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); +} + +/// Minimal raw J2K codestream containing SIZ, COD, and SOT markers. +pub fn minimal_j2k_codestream() -> Vec { + let mut bytes = vec![0xff, 0x4f]; + let mut siz = Vec::new(); + push_u16(&mut siz, 0); + push_u32(&mut siz, 128); + push_u32(&mut siz, 64); + push_u32(&mut siz, 0); + push_u32(&mut siz, 0); + push_u32(&mut siz, 64); + push_u32(&mut siz, 64); + push_u32(&mut siz, 0); + push_u32(&mut siz, 0); + push_u16(&mut siz, 3); + for _ in 0..3 { + siz.extend_from_slice(&[0x07, 0x01, 0x01]); + } + bytes.extend_from_slice(&[0xff, 0x51]); + push_u16(&mut bytes, (siz.len() + 2) as u16); + bytes.extend_from_slice(&siz); + + let cod = [0x00, 0x00, 0x00, 0x01, 0x01, 0x05, 0x04, 0x04, 0x00, 0x01]; + bytes.extend_from_slice(&[0xff, 0x52]); + push_u16(&mut bytes, (cod.len() + 2) as u16); + bytes.extend_from_slice(&cod); + bytes.extend_from_slice(&[0xff, 0x90, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); + bytes +} + +/// Minimal JP2 wrapper around [`minimal_j2k_codestream`]. +pub fn minimal_jp2() -> Vec { + wrap_jp2_codestream(&minimal_j2k_codestream(), 128, 64, 3, 8, 16) +} + +/// Wraps a codestream in a JP2 container with an enumerated colorspace. +pub fn wrap_jp2_codestream( + codestream: &[u8], + width: u32, + height: u32, + components: u16, + bit_depth: u8, + colorspace_enum: u32, +) -> Vec { + let mut bytes = jp2_prefix(); + let bpc = bit_depth.saturating_sub(1); + bytes.extend_from_slice(&[ + 0, 0, 0, 45, b'j', b'p', b'2', b'h', 0, 0, 0, 22, b'i', b'h', b'd', b'r', + ]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&components.to_be_bytes()); + bytes.extend_from_slice(&[bpc, 7, 0, 0]); + bytes.extend_from_slice(&[0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0]); + bytes.extend_from_slice(&colorspace_enum.to_be_bytes()); + append_jp2c(&mut bytes, codestream); + bytes +} + +/// Wraps a four-component codestream in a JP2 container with an alpha channel. +pub fn wrap_jp2_rgba_codestream( + codestream: &[u8], + width: u32, + height: u32, + bit_depth: u8, +) -> Vec { + let mut bytes = jp2_prefix(); + let bpc = bit_depth.saturating_sub(1); + let jp2h_len = 8 + 22 + 15 + 34; + bytes.extend_from_slice(&(jp2h_len as u32).to_be_bytes()); + bytes.extend_from_slice(b"jp2h"); + bytes.extend_from_slice(&[0, 0, 0, 22, b'i', b'h', b'd', b'r']); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&4_u16.to_be_bytes()); + bytes.extend_from_slice(&[bpc, 7, 0, 0]); + bytes.extend_from_slice(&[0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0]); + bytes.extend_from_slice(&16_u32.to_be_bytes()); + bytes.extend_from_slice(&[0, 0, 0, 34, b'c', b'd', b'e', b'f']); + bytes.extend_from_slice(&4_u16.to_be_bytes()); + for (channel, channel_type, association) in [ + (0_u16, 0_u16, 1_u16), + (1_u16, 0_u16, 2_u16), + (2_u16, 0_u16, 3_u16), + (3_u16, 1_u16, 0_u16), + ] { + bytes.extend_from_slice(&channel.to_be_bytes()); + bytes.extend_from_slice(&channel_type.to_be_bytes()); + bytes.extend_from_slice(&association.to_be_bytes()); + } + append_jp2c(&mut bytes, codestream); + bytes +} + +fn jp2_prefix() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0d, 0x0a, 0x87, 0x0a]); + bytes.extend_from_slice(&[ + 0, 0, 0, 20, b'f', b't', b'y', b'p', b'j', b'p', b'2', b' ', 0, 0, 0, 0, b'j', b'p', b'2', + b' ', + ]); + bytes +} + +fn append_jp2c(bytes: &mut Vec, codestream: &[u8]) { + let len = (8 + codestream.len()) as u32; + bytes.extend_from_slice(&len.to_be_bytes()); + bytes.extend_from_slice(b"jp2c"); + bytes.extend_from_slice(codestream); +} + +fn push_u16(out: &mut Vec, value: u16) { + out.extend_from_slice(&value.to_be_bytes()); +} + +fn push_u32(out: &mut Vec, value: u32) { + out.extend_from_slice(&value.to_be_bytes()); +} + +#[cfg(feature = "j2k-native-fixtures")] +fn htj2k_options(reversible: bool) -> j2k_native::EncodeOptions { + j2k_native::EncodeOptions { + reversible, + num_decomposition_levels: 1, + ..j2k_native::EncodeOptions::default() + } +} + +#[cfg(feature = "j2k-native-fixtures")] +fn encode_htj2k_fixture( + pixels: &[u8], + width: u32, + height: u32, + components: u8, + reversible: bool, +) -> Vec { + j2k_native::encode_htj2k( + pixels, + width, + height, + components, + 8, + false, + &htj2k_options(reversible), + ) + .expect("encode HTJ2K fixture") +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Deterministic reversible HTJ2K grayscale fixture. +pub fn htj2k_gray8_fixture(width: u32, height: u32) -> Vec { + let pixels = (0..width * height) + .map(|idx| (idx & 0xff) as u8) + .collect::>(); + encode_htj2k_fixture(&pixels, width, height, 1, true) +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Deterministic irreversible 9/7 HTJ2K grayscale fixture. +pub fn htj2k_gray8_97_fixture(width: u32, height: u32) -> Vec { + let pixels = (0..width * height) + .map(|idx| ((idx * 11) & 0xff) as u8) + .collect::>(); + encode_htj2k_fixture(&pixels, width, height, 1, false) +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Larger reversible grayscale HTJ2K fixture with 64x64 code blocks. +/// +/// # Panics +/// +/// Panics if the native encoder rejects the deterministic fixture input. +pub fn htj2k_gray8_large_fixture(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 3 + y * 5) & 0xff) as u8); + } + } + let options = j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 3, + code_block_width_exp: 0, + code_block_height_exp: 0, + ..j2k_native::EncodeOptions::default() + }; + j2k_native::encode_htj2k(&pixels, width, height, 1, 8, false, &options) + .expect("encode large HTJ2K grayscale fixture") +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Deterministic reversible HTJ2K RGB fixture. +pub fn htj2k_rgb8_fixture(width: u32, height: u32) -> Vec { + htj2k_rgb8_fixture_with_pixels(width, height).0 +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Deterministic reversible HTJ2K RGB fixture and its source pixels. +pub fn htj2k_rgb8_fixture_with_pixels(width: u32, height: u32) -> (Vec, Vec) { + let pixels = (0u32..width * height * 3) + .map(|idx| ((idx * 13 + idx / 3) & 0xff) as u8) + .collect::>(); + let codestream = encode_htj2k_fixture(&pixels, width, height, 3, true); + (codestream, pixels) +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Seeded reversible HTJ2K RGB fixture. +pub fn htj2k_rgb8_pattern_fixture(width: u32, height: u32, seed: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for idx in 0..width * height { + pixels.push(((idx * seed + idx / 3) & 0xff) as u8); + pixels.push(((idx * (seed + 11) + 7) & 0xff) as u8); + pixels.push(((idx * (seed + 23) + 19) & 0xff) as u8); + } + encode_htj2k_fixture(&pixels, width, height, 3, true) +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Deterministic irreversible 9/7 HTJ2K RGB fixture. +pub fn htj2k_rgb8_97_fixture(width: u32, height: u32) -> Vec { + let pixels = (0u32..width * height * 3) + .map(|idx| ((idx * 17 + idx / 5) & 0xff) as u8) + .collect::>(); + encode_htj2k_fixture(&pixels, width, height, 3, false) +} + +#[cfg(feature = "j2k-native-fixtures")] +/// Deterministic classic J2K grayscale fixture. +/// +/// # Panics +/// +/// Panics if the native encoder rejects the deterministic fixture input. +pub fn classic_j2k_gray8_fixture(width: u32, height: u32) -> Vec { + let pixels = (0..width * height) + .map(|idx| (idx & 0xff) as u8) + .collect::>(); + let options = j2k_native::EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..j2k_native::EncodeOptions::default() + }; + j2k_native::encode(&pixels, width, height, 1, 8, false, &options) + .expect("encode classic J2K grayscale fixture") +} + +#[cfg(feature = "j2k-native-fixtures")] +/// `OpenHTJ2K` odd refinement fixture used by CUDA plan tests. +pub fn openhtj2k_refinement_odd_fixture() -> &'static [u8] { + include_bytes!("../fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k") +} diff --git a/crates/j2k-test-support/src/jpeg_fixtures.rs b/crates/j2k-test-support/src/jpeg_fixtures.rs new file mode 100644 index 00000000..4495244c --- /dev/null +++ b/crates/j2k-test-support/src/jpeg_fixtures.rs @@ -0,0 +1,3930 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Fixture JPEGs for decode integration tests. Shared conformance inputs are +//! committed under `j2k-test-support/fixtures/conformance/` and embedded +//! via `include_bytes!` so tests remain hermetic. + +/// A 16×16 baseline JPEG with 4:2:0 sampling. +pub fn minimal_baseline_420_jpeg() -> Vec { + include_bytes!("../fixtures/conformance/baseline_420_16x16.jpg").to_vec() +} + +/// An 8×8 grayscale (single-component) baseline JPEG. +pub fn grayscale_8x8_jpeg() -> Vec { + include_bytes!("../fixtures/conformance/grayscale_8x8.jpg").to_vec() +} + +/// An 8x8 12-bit extended sequential grayscale JPEG with all-zero DCT blocks. +pub fn extended_12bit_grayscale_8x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[0xff, 0xc1, 0x00, 11, 12, 0, 8, 0, 8, 1, 1, 0x11, 0]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); + bytes.push(0x00); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// A 16x8 12-bit extended sequential grayscale JPEG with DRI=1 and RST0. +pub fn extended_12bit_grayscale_restart_16x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[0xff, 0xc1, 0x00, 11, 12, 0, 8, 0, 16, 1, 1, 0x11, 0]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); + bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); + bytes.extend_from_slice(&[0x00, 0xff, 0xd0, 0x00]); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// An 8x8 12-bit extended sequential APP14 RGB JPEG. +pub fn extended_12bit_rgb_8x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc1, 0x00, 17, 12, 0, 8, 0, 8, 3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0, + ]); + bytes.extend(dc_category4_rgb_entropy()); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// A 16x8 12-bit extended sequential APP14 RGB JPEG with DRI=1 and RST0. +pub fn extended_12bit_rgb_restart_16x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc1, 0x00, 17, 12, 0, 8, 0, 16, 3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0, + ]); + bytes.extend(restart_segmented_entropy( + 2, + &dc_category4_rgb_mcu_bits(false), + )); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// A 32x8 12-bit extended sequential APP14 RGB JPEG with 4:2:2 sampling. +pub fn extended_12bit_rgb_422_32x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc1, 0x00, 17, 12, 0, 8, 0, 32, 3, 1, 0x21, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 21, 0x00, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0, + ]); + bytes.extend(dc_rgb422_uniform_entropy(false, 2)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// A 32x32 12-bit extended sequential APP14 RGB JPEG with 4:2:0 sampling. +pub fn extended_12bit_rgb_420_32x32_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc1, 0x00, 17, 12, 0, 32, 0, 32, 3, 1, 0x22, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 21, 0x00, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0, + ]); + bytes.extend(dc_rgb420_uniform_entropy(false, 4)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// Reference Rgb16 pixels for [`extended_12bit_rgb_8x8_jpeg`]. +pub fn extended_12bit_rgb_8x8_rgb16() -> Vec { + repeat_rgb16_pixels(8, 8, [2064, 2072, 2032]) +} + +/// Reference Rgb16 pixels for [`extended_12bit_rgb_restart_16x8_jpeg`]. +pub fn extended_12bit_rgb_restart_16x8_rgb16() -> Vec { + repeat_rgb16_pixels(16, 8, [2064, 2072, 2032]) +} + +/// Reference Rgb16 pixels for 32x8 12-bit APP14 RGB fixtures. +pub fn extended_12bit_rgb_32x8_rgb16() -> Vec { + repeat_rgb16_pixels(32, 8, [2064, 2072, 2032]) +} + +/// Reference Rgb16 pixels for 32x32 12-bit APP14 RGB fixtures. +pub fn extended_12bit_rgb_32x32_rgb16() -> Vec { + repeat_rgb16_pixels(32, 32, [2064, 2072, 2032]) +} + +fn repeat_rgb16_pixels(width: usize, height: usize, rgb: [u16; 3]) -> Vec { + let mut out = Vec::with_capacity(width * height * 6); + for _ in 0..width * height { + for sample in rgb { + out.extend_from_slice(&sample.to_le_bytes()); + } + } + out +} + +/// An 8x8 12-bit extended sequential YCbCr 4:4:4 JPEG. +pub fn extended_12bit_ycbcr_8x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc1, 0x00, 17, 12, 0, 8, 0, 8, 3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0, + ]); + bytes.extend(dc_category4_rgb_entropy()); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// A 16x8 12-bit extended sequential YCbCr 4:4:4 JPEG with DRI=1 and RST0. +pub fn extended_12bit_ycbcr_restart_16x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc1, 0x00, 17, 12, 0, 8, 0, 16, 3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0, + ]); + bytes.extend(restart_segmented_entropy( + 2, + &dc_category4_rgb_mcu_bits(false), + )); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// Reference Rgb16 pixels for [`extended_12bit_ycbcr_8x8_jpeg`]. +pub fn extended_12bit_ycbcr_8x8_rgb16() -> Vec { + repeat_rgb16_pixels(8, 8, [2042, 2067, 2107]) +} + +/// Reference Rgb16 pixels for [`extended_12bit_ycbcr_restart_16x8_jpeg`]. +pub fn extended_12bit_ycbcr_restart_16x8_rgb16() -> Vec { + repeat_rgb16_pixels(16, 8, [2042, 2067, 2107]) +} + +/// A 32x8 12-bit extended sequential YCbCr 4:2:2 JPEG. +pub fn extended_12bit_ycbcr_422_32x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc1, 0x00, 17, 12, 0, 8, 0, 32, 3, 1, 0x21, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 21, 0x00, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0, + ]); + bytes.extend(dc_ycbcr422_entropy(false)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// A 32x8 12-bit extended sequential YCbCr 4:2:2 JPEG with DRI=1. +pub fn extended_12bit_ycbcr_422_restart_32x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc1, 0x00, 17, 12, 0, 8, 0, 32, 3, 1, 0x21, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 21, 0x00, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0, + ]); + bytes.extend(restart_segmented_entropy( + 2, + &dc_ycbcr422_uniform_mcu_bits(false), + )); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// Reference Rgb16 pixels for [`extended_12bit_ycbcr_422_32x8_jpeg`]. +pub fn extended_12bit_ycbcr_422_32x8_rgb16() -> Vec { + let mut out = Vec::with_capacity(32 * 8 * 6); + for _ in 0..8 { + append_rgb16_run(&mut out, 15, [2042, 2067, 2107]); + append_rgb16_run(&mut out, 1, [2047, 2066, 2099]); + append_rgb16_run(&mut out, 1, [2058, 2063, 2085]); + append_rgb16_run(&mut out, 15, [2064, 2061, 2078]); + } + out +} + +/// Reference Rgb16 pixels for [`extended_12bit_ycbcr_422_restart_32x8_jpeg`]. +pub fn extended_12bit_ycbcr_422_restart_32x8_rgb16() -> Vec { + repeat_rgb16_pixels(32, 8, [2042, 2067, 2107]) +} + +/// A 32x32 12-bit extended sequential YCbCr 4:2:0 JPEG. +pub fn extended_12bit_ycbcr_420_32x32_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc1, 0x00, 17, 12, 0, 32, 0, 32, 3, 1, 0x22, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 21, 0x00, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0, + ]); + bytes.extend(dc_ycbcr420_entropy(false)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// A 32x32 12-bit extended sequential YCbCr 4:2:0 JPEG with DRI=1. +pub fn extended_12bit_ycbcr_420_restart_32x32_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc1, 0x00, 17, 12, 0, 32, 0, 32, 3, 1, 0x22, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 21, 0x00, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0, + ]); + bytes.extend(restart_segmented_entropy( + 4, + &dc_ycbcr420_uniform_mcu_bits(false), + )); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// Reference Rgb16 pixels for [`extended_12bit_ycbcr_420_32x32_jpeg`]. +pub fn extended_12bit_ycbcr_420_32x32_rgb16() -> Vec { + let mut out = Vec::with_capacity(32 * 32 * 6); + let blue_difference_plane = ycbcr420_chroma_plane_for_fixture(2072, 2056, 2040, 2064); + let red_difference_plane = ycbcr420_chroma_plane_for_fixture(2032, 2048, 2072, 2056); + for y in 0..32 { + for x in 0..32 { + let cb = upsample_h2v2_12bit_for_fixture(&blue_difference_plane, x, y); + let cr = upsample_h2v2_12bit_for_fixture(&red_difference_plane, x, y); + let (r, g, b) = ycbcr12_to_rgb16_for_fixture(2064, cb, cr); + append_rgb16_pixel(&mut out, [r, g, b]); + } + } + out +} + +/// Reference Rgb16 pixels for [`extended_12bit_ycbcr_420_restart_32x32_jpeg`]. +pub fn extended_12bit_ycbcr_420_restart_32x32_rgb16() -> Vec { + repeat_rgb16_pixels(32, 32, [2042, 2067, 2107]) +} + +/// An 8x8 12-bit progressive grayscale JPEG with one DC-only scan. +pub fn progressive_12bit_grayscale_8x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[0xff, 0xc2, 0x00, 11, 12, 0, 8, 0, 8, 1, 1, 0x11, 0]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 0, 0]); + bytes.push(0x00); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// An 8x8 12-bit progressive APP14 RGB JPEG with one DC-only scan. +pub fn progressive_12bit_rgb_8x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc2, 0x00, 17, 12, 0, 8, 0, 8, 3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 0, 0, + ]); + bytes.extend(dc_category4_rgb_progressive_entropy()); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// A 32x8 12-bit progressive APP14 RGB JPEG with 4:2:2 sampling and one DC-only scan. +pub fn progressive_12bit_rgb_422_32x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc2, 0x00, 17, 12, 0, 8, 0, 32, 3, 1, 0x21, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 21, 0x00, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 0, 0, + ]); + bytes.extend(dc_rgb422_uniform_entropy(true, 2)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// A 32x32 12-bit progressive APP14 RGB JPEG with 4:2:0 sampling and one DC-only scan. +pub fn progressive_12bit_rgb_420_32x32_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc2, 0x00, 17, 12, 0, 32, 0, 32, 3, 1, 0x22, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 21, 0x00, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 0, 0, + ]); + bytes.extend(dc_rgb420_uniform_entropy(true, 4)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// An 8x8 12-bit progressive YCbCr 4:4:4 JPEG with one DC-only scan. +pub fn progressive_12bit_ycbcr_8x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc2, 0x00, 17, 12, 0, 8, 0, 8, 3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 0, 0, + ]); + bytes.extend(dc_category4_rgb_progressive_entropy()); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// A 32x8 12-bit progressive YCbCr 4:2:2 JPEG with one DC-only scan. +pub fn progressive_12bit_ycbcr_422_32x8_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc2, 0x00, 17, 12, 0, 8, 0, 32, 3, 1, 0x21, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 21, 0x00, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 0, 0, + ]); + bytes.extend(dc_ycbcr422_entropy(true)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +/// A 32x32 12-bit progressive YCbCr 4:2:0 JPEG with one DC-only scan. +pub fn progressive_12bit_ycbcr_420_32x32_jpeg() -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, 0xc2, 0x00, 17, 12, 0, 32, 0, 32, 3, 1, 0x22, 0, 2, 0x11, 0, 3, 0x11, 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 21, 0x00, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 0, 0, + ]); + bytes.extend(dc_ycbcr420_entropy(true)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn dc_category4_rgb_entropy() -> Vec { + pack_entropy_bits(dc_category4_rgb_mcu_bits(false)) +} + +fn dc_category4_rgb_mcu_bits(progressive: bool) -> Vec { + let mut bits = Vec::new(); + for magnitude in [0b1000, 0b1100, 0b0111] { + push_bits(&mut bits, 0, 1); + push_bits(&mut bits, magnitude, 4); + if !progressive { + push_bits(&mut bits, 0, 1); + } + } + bits +} + +fn dc_category4_rgb_progressive_entropy() -> Vec { + pack_entropy_bits(dc_category4_rgb_mcu_bits(true)) +} + +fn dc_category4_cmyk_mcu_bits(progressive: bool) -> Vec { + let mut bits = Vec::new(); + for magnitude in [0b1000, 0b1100, 0b1010, 0b0111] { + push_bits(&mut bits, 0, 1); + push_bits(&mut bits, magnitude, 4); + if !progressive { + push_bits(&mut bits, 0, 1); + } + } + bits +} + +fn dc_ycbcr422_entropy(progressive: bool) -> Vec { + let mut bits = Vec::new(); + for magnitude in [ + Some(0b1000), + None, + Some(0b1100), + Some(0b0111), + None, + None, + Some(0b0111), + Some(0b1000), + ] { + match magnitude { + Some(value) => { + push_bits(&mut bits, 1, 1); + push_bits(&mut bits, value, 4); + } + None => push_bits(&mut bits, 0, 1), + } + if !progressive { + push_bits(&mut bits, 0, 1); + } + } + pack_entropy_bits(bits) +} + +fn dc_rgb422_uniform_entropy(progressive: bool, mcus: usize) -> Vec { + let mut bits = Vec::new(); + for mcu in 0..mcus { + let magnitudes = if mcu == 0 { + [Some(0b1000), None, Some(0b1100), Some(0b0111)] + } else { + [None, None, None, None] + }; + for magnitude in magnitudes { + match magnitude { + Some(value) => { + push_bits(&mut bits, 1, 1); + push_bits(&mut bits, value, 4); + } + None => push_bits(&mut bits, 0, 1), + } + if !progressive { + push_bits(&mut bits, 0, 1); + } + } + } + pack_entropy_bits(bits) +} + +fn dc_ycbcr422_uniform_mcu_bits(progressive: bool) -> Vec { + let mut bits = Vec::new(); + for magnitude in [Some(0b1000), None, Some(0b1100), Some(0b0111)] { + match magnitude { + Some(value) => { + push_bits(&mut bits, 1, 1); + push_bits(&mut bits, value, 4); + } + None => push_bits(&mut bits, 0, 1), + } + if !progressive { + push_bits(&mut bits, 0, 1); + } + } + bits +} + +fn dc_ycbcr420_entropy(progressive: bool) -> Vec { + let mut bits = Vec::new(); + let mcu_blocks = [ + [Some(0b1000), None, None, None, Some(0b1100), Some(0b0111)], + [None, None, None, None, Some(0b0111), Some(0b1000)], + [None, None, None, None, Some(0b0111), Some(0b1100)], + [None, None, None, None, Some(0b1100), Some(0b0111)], + ]; + for mcu in mcu_blocks { + for magnitude in mcu { + match magnitude { + Some(value) => { + push_bits(&mut bits, 1, 1); + push_bits(&mut bits, value, 4); + } + None => push_bits(&mut bits, 0, 1), + } + if !progressive { + push_bits(&mut bits, 0, 1); + } + } + } + pack_entropy_bits(bits) +} + +fn dc_rgb420_uniform_entropy(progressive: bool, mcus: usize) -> Vec { + let mut bits = Vec::new(); + for mcu in 0..mcus { + let magnitudes = if mcu == 0 { + [Some(0b1000), None, None, None, Some(0b1100), Some(0b0111)] + } else { + [None, None, None, None, None, None] + }; + for magnitude in magnitudes { + match magnitude { + Some(value) => { + push_bits(&mut bits, 1, 1); + push_bits(&mut bits, value, 4); + } + None => push_bits(&mut bits, 0, 1), + } + if !progressive { + push_bits(&mut bits, 0, 1); + } + } + } + pack_entropy_bits(bits) +} + +fn dc_ycbcr420_uniform_mcu_bits(progressive: bool) -> Vec { + let mut bits = Vec::new(); + for magnitude in [Some(0b1000), None, None, None, Some(0b1100), Some(0b0111)] { + match magnitude { + Some(value) => { + push_bits(&mut bits, 1, 1); + push_bits(&mut bits, value, 4); + } + None => push_bits(&mut bits, 0, 1), + } + if !progressive { + push_bits(&mut bits, 0, 1); + } + } + bits +} + +fn restart_segmented_entropy(mcus: usize, mcu_bits: &[bool]) -> Vec { + let mut entropy = Vec::new(); + let packed_mcu = pack_entropy_bits(mcu_bits.to_vec()); + for mcu in 0..mcus { + entropy.extend_from_slice(&packed_mcu); + if mcu + 1 != mcus { + entropy.extend_from_slice(&[0xff, 0xd0 | ((mcu as u8) & 0x07)]); + } + } + entropy +} + +fn append_rgb16_run(out: &mut Vec, len: usize, rgb: [u16; 3]) { + for _ in 0..len { + append_rgb16_pixel(out, rgb); + } +} + +fn append_rgb16_pixel(out: &mut Vec, rgb: [u16; 3]) { + for sample in rgb { + out.extend_from_slice(&sample.to_le_bytes()); + } +} + +fn ycbcr420_chroma_row_for_fixture(left: u16, right: u16) -> [u16; 16] { + let mut row = [0u16; 16]; + row[..8].fill(left); + row[8..].fill(right); + row +} + +fn ycbcr420_chroma_plane_for_fixture( + top_left: u16, + top_right: u16, + bottom_left: u16, + bottom_right: u16, +) -> [[u16; 16]; 16] { + let top = ycbcr420_chroma_row_for_fixture(top_left, top_right); + let bottom = ycbcr420_chroma_row_for_fixture(bottom_left, bottom_right); + core::array::from_fn(|y| if y < 8 { top } else { bottom }) +} + +fn upsample_h2v2_12bit_for_fixture( + plane: &[[u16; 16]; 16], + output_x: usize, + output_y: usize, +) -> u16 { + let chroma_y = output_y / 2; + let current = &plane[chroma_y]; + let near_y = if output_y.is_multiple_of(2) { + chroma_y.saturating_sub(1) + } else { + (chroma_y + 1).min(15) + }; + let near = &plane[near_y]; + let sample = output_x / 2; + let colsum = + |row: &[u16; 16], index: usize| 3 * u32::from(current[index]) + u32::from(row[index]); + let this = colsum(near, sample); + match output_x { + 0 => ((this * 4 + 8) >> 4) as u16, + 31 => ((this * 4 + 7) >> 4) as u16, + _ if output_x.is_multiple_of(2) => { + let last = colsum(near, sample - 1); + ((this * 3 + last + 8) >> 4) as u16 + } + _ => { + let next = colsum(near, sample + 1); + ((this * 3 + next + 7) >> 4) as u16 + } + } +} + +fn ycbcr12_to_rgb16_for_fixture(y: u16, cb: u16, cr: u16) -> (u16, u16, u16) { + const FIX_1_40200: i32 = 91_881; + const FIX_0_34414: i32 = 22_554; + const FIX_0_71414: i32 = 46_802; + const FIX_1_77200: i32 = 116_130; + const ROUND: i32 = 1 << 15; + + let y = i32::from(y); + let blue_delta = i32::from(cb) - 2048; + let red_delta = i32::from(cr) - 2048; + let r = y + ((FIX_1_40200 * red_delta + ROUND) >> 16); + let g = y - ((FIX_0_34414 * blue_delta + FIX_0_71414 * red_delta + ROUND) >> 16); + let b = y + ((FIX_1_77200 * blue_delta + ROUND) >> 16); + + ( + r.clamp(0, 4095) as u16, + g.clamp(0, 4095) as u16, + b.clamp(0, 4095) as u16, + ) +} + +pub const LOSSLESS_GRAYSCALE_3X3_PIXELS: [u8; 9] = [130, 132, 136, 128, 135, 142, 125, 137, 150]; + +pub const LOSSLESS_GRAYSCALE_16BIT_3X3_PIXELS: [u16; 9] = [ + 33000, 33012, 33025, 32990, 33020, 33044, 32970, 33030, 33080, +]; + +pub const LOSSLESS_RGB_3X3_PIXELS: [u8; 27] = [ + 130, 50, 200, 132, 53, 198, 136, 55, 195, 128, 54, 202, 135, 56, 199, 142, 59, 196, 125, 57, + 204, 137, 60, 201, 150, 64, 198, +]; + +pub const LOSSLESS_RGB_16BIT_3X3_PIXELS: [u16; 27] = [ + 33000, 16000, 50000, 33012, 16040, 49960, 33025, 16075, 49910, 32990, 16055, 50025, 33020, + 16090, 49980, 33044, 16120, 49930, 32970, 16080, 50050, 33030, 16130, 50000, 33080, 16190, + 49950, +]; + +pub const LOSSLESS_YCBCR_3X3_PIXELS: [u8; 27] = [ + 100, 150, 200, 104, 145, 196, 110, 140, 190, 96, 155, 202, 102, 150, 198, 108, 144, 193, 92, + 160, 205, 101, 153, 199, 116, 148, 194, +]; + +pub const LOSSLESS_YCBCR_16BIT_3X3_PIXELS: [u16; 27] = [ + 33000, 35000, 40000, 33120, 34800, 39700, 33280, 34600, 39250, 32940, 35200, 40250, 33080, + 35040, 39880, 33300, 34720, 39440, 32780, 35480, 40500, 33160, 35120, 39960, 33600, 34920, + 39680, +]; + +const LOSSLESS_RGB_8BIT_422_4X2_C0: [u8; 8] = [130, 132, 136, 140, 128, 135, 142, 150]; +const LOSSLESS_RGB_8BIT_422_4X2_C1: [u8; 4] = [50, 55, 54, 60]; +const LOSSLESS_RGB_8BIT_422_4X2_C2: [u8; 4] = [200, 195, 202, 198]; + +const LOSSLESS_YCBCR_8BIT_422_4X2_C0: [u8; 8] = [100, 104, 110, 116, 96, 102, 108, 114]; +const LOSSLESS_YCBCR_8BIT_422_4X2_C1: [u8; 4] = [150, 140, 155, 144]; +const LOSSLESS_YCBCR_8BIT_422_4X2_C2: [u8; 4] = [200, 190, 202, 193]; + +const LOSSLESS_RGB_8BIT_420_4X4_C0: [u8; 16] = [ + 130, 132, 136, 140, 128, 135, 142, 150, 126, 133, 139, 146, 124, 131, 137, 144, +]; +const LOSSLESS_RGB_8BIT_420_4X4_C1: [u8; 4] = [50, 55, 54, 60]; +const LOSSLESS_RGB_8BIT_420_4X4_C2: [u8; 4] = [200, 195, 202, 198]; + +const LOSSLESS_YCBCR_8BIT_420_4X4_C0: [u8; 16] = [ + 100, 104, 110, 116, 96, 102, 108, 114, 92, 101, 109, 117, 90, 99, 107, 115, +]; +const LOSSLESS_YCBCR_8BIT_420_4X4_C1: [u8; 4] = [150, 140, 155, 144]; +const LOSSLESS_YCBCR_8BIT_420_4X4_C2: [u8; 4] = [200, 190, 202, 193]; + +const LOSSLESS_RGB_16BIT_422_4X2_C0: [u16; 8] = + [33000, 33012, 33025, 33045, 32990, 33020, 33044, 33070]; +const LOSSLESS_RGB_16BIT_422_4X2_C1: [u16; 4] = [16000, 16100, 16055, 16140]; +const LOSSLESS_RGB_16BIT_422_4X2_C2: [u16; 4] = [50000, 49880, 50025, 49920]; + +const LOSSLESS_YCBCR_16BIT_422_4X2_C0: [u16; 8] = + [33000, 33120, 33280, 33420, 32940, 33080, 33300, 33460]; +const LOSSLESS_YCBCR_16BIT_422_4X2_C1: [u16; 4] = [35000, 34600, 35200, 34720]; +const LOSSLESS_YCBCR_16BIT_422_4X2_C2: [u16; 4] = [40000, 39250, 40250, 39440]; + +const LOSSLESS_RGB_16BIT_420_4X4_C0: [u16; 16] = [ + 33000, 33012, 33025, 33045, 32990, 33020, 33044, 33070, 33010, 33034, 33058, 33082, 32980, + 33016, 33052, 33088, +]; +const LOSSLESS_RGB_16BIT_420_4X4_C1: [u16; 4] = [16000, 16100, 16055, 16140]; +const LOSSLESS_RGB_16BIT_420_4X4_C2: [u16; 4] = [50000, 49880, 50025, 49920]; + +const LOSSLESS_YCBCR_16BIT_420_4X4_C0: [u16; 16] = [ + 33000, 33120, 33280, 33420, 32940, 33080, 33300, 33460, 33030, 33160, 33310, 33470, 32970, + 33110, 33340, 33490, +]; +const LOSSLESS_YCBCR_16BIT_420_4X4_C1: [u16; 4] = [35000, 34600, 35200, 34720]; +const LOSSLESS_YCBCR_16BIT_420_4X4_C2: [u16; 4] = [40000, 39250, 40250, 39440]; + +#[derive(Clone, Copy)] +struct Lossless422Planes<'a> { + c0: &'a [u16], + c1: &'a [u16], + c2: &'a [u16], +} + +#[derive(Clone, Copy)] +struct Lossless420Planes<'a> { + c0: &'a [u16], + c1: &'a [u16], + c2: &'a [u16], +} + +#[derive(Clone, Copy)] +struct Lossless422Planes8<'a> { + c0: &'a [u8], + c1: &'a [u8], + c2: &'a [u8], +} + +#[derive(Clone, Copy)] +struct Lossless420Planes8<'a> { + c0: &'a [u8], + c1: &'a [u8], + c2: &'a [u8], +} + +/// A 3x3 SOF3 lossless grayscale JPEG using predictor 1..=7. +pub fn lossless_predictor_grayscale_3x3_jpeg(predictor: u8) -> Vec { + lossless_grayscale_jpeg(3, 3, predictor, &LOSSLESS_GRAYSCALE_3X3_PIXELS) +} + +/// A 3x3 SOF3 lossless APP14 RGB JPEG using predictor 1..=7. +pub fn lossless_predictor_rgb_3x3_jpeg(predictor: u8) -> Vec { + lossless_rgb_jpeg(3, 3, predictor, &LOSSLESS_RGB_3X3_PIXELS) +} + +/// A 3x3 SOF3 lossless APP14 RGB JPEG with row-boundary restart markers. +pub fn lossless_restart_predictor_rgb_3x3_jpeg(predictor: u8) -> Vec { + lossless_rgb_restart_jpeg(3, 3, predictor, 3, &LOSSLESS_RGB_3X3_PIXELS) +} + +/// A 3x3 SOF3 lossless grayscale JPEG with row-boundary restart markers. +pub fn lossless_restart_predictor_grayscale_3x3_jpeg(predictor: u8) -> Vec { + lossless_grayscale_restart_jpeg(3, 3, predictor, 3, &LOSSLESS_GRAYSCALE_3X3_PIXELS) +} + +/// A 3x3 16-bit SOF3 lossless grayscale JPEG using predictor 1..=7. +pub fn lossless_predictor_grayscale_16bit_3x3_jpeg(predictor: u8) -> Vec { + lossless_grayscale_16bit_jpeg(3, 3, predictor, &LOSSLESS_GRAYSCALE_16BIT_3X3_PIXELS) +} + +/// A 3x3 16-bit SOF3 lossless grayscale JPEG with row-boundary restart markers. +pub fn lossless_restart_predictor_grayscale_16bit_3x3_jpeg(predictor: u8) -> Vec { + lossless_grayscale_16bit_restart_jpeg(3, 3, predictor, 3, &LOSSLESS_GRAYSCALE_16BIT_3X3_PIXELS) +} + +/// A 3x3 16-bit SOF3 lossless APP14 RGB JPEG using predictor 1..=7. +pub fn lossless_predictor_rgb_16bit_3x3_jpeg(predictor: u8) -> Vec { + lossless_rgb_16bit_jpeg(3, 3, predictor, &LOSSLESS_RGB_16BIT_3X3_PIXELS) +} + +/// A 3x3 16-bit SOF3 lossless APP14 RGB JPEG with unsupported 4:2:2 sampling. +pub fn lossless_rgb_16bit_422_3x3_jpeg() -> Vec { + let mut bytes = lossless_predictor_rgb_16bit_3x3_jpeg(1); + set_first_sof3_component_sampling(&mut bytes, 0x21); + bytes +} + +/// A 4x2 8-bit SOF3 lossless APP14 RGB JPEG with valid 4:2:2 sampling. +pub fn lossless_rgb_8bit_422_4x2_jpeg(predictor: u8) -> Vec { + lossless_color_8bit_422_jpeg( + Some(0), + 4, + 2, + predictor, + Lossless422Planes8 { + c0: &LOSSLESS_RGB_8BIT_422_4X2_C0, + c1: &LOSSLESS_RGB_8BIT_422_4X2_C1, + c2: &LOSSLESS_RGB_8BIT_422_4X2_C2, + }, + ) +} + +/// A 4x2 8-bit SOF3 lossless APP14 RGB JPEG with valid 4:2:2 sampling and restart markers. +pub fn lossless_rgb_8bit_422_restart_4x2_jpeg(predictor: u8) -> Vec { + lossless_color_8bit_422_restart_jpeg( + Some(0), + 4, + 2, + predictor, + Lossless422Planes8 { + c0: &LOSSLESS_RGB_8BIT_422_4X2_C0, + c1: &LOSSLESS_RGB_8BIT_422_4X2_C1, + c2: &LOSSLESS_RGB_8BIT_422_4X2_C2, + }, + 2, + ) +} + +pub fn lossless_rgb_8bit_422_4x2_rgb8() -> Vec { + lossless_422_planes_to_rgb8( + ColorSpaceFixture::Rgb, + 4, + 2, + &LOSSLESS_RGB_8BIT_422_4X2_C0, + &LOSSLESS_RGB_8BIT_422_4X2_C1, + &LOSSLESS_RGB_8BIT_422_4X2_C2, + ) +} + +/// A 4x2 16-bit SOF3 lossless APP14 RGB JPEG with valid 4:2:2 sampling. +pub fn lossless_rgb_16bit_422_4x2_jpeg(predictor: u8) -> Vec { + lossless_color_16bit_422_jpeg( + Some(0), + 4, + 2, + predictor, + Lossless422Planes { + c0: &LOSSLESS_RGB_16BIT_422_4X2_C0, + c1: &LOSSLESS_RGB_16BIT_422_4X2_C1, + c2: &LOSSLESS_RGB_16BIT_422_4X2_C2, + }, + ) +} + +/// A 4x2 16-bit SOF3 lossless APP14 RGB JPEG with valid 4:2:2 sampling and restart markers. +pub fn lossless_rgb_16bit_422_restart_4x2_jpeg(predictor: u8) -> Vec { + lossless_color_16bit_422_restart_jpeg( + Some(0), + 4, + 2, + predictor, + Lossless422Planes { + c0: &LOSSLESS_RGB_16BIT_422_4X2_C0, + c1: &LOSSLESS_RGB_16BIT_422_4X2_C1, + c2: &LOSSLESS_RGB_16BIT_422_4X2_C2, + }, + 2, + ) +} + +pub fn lossless_rgb_16bit_422_4x2_rgb16() -> Vec { + lossless_422_planes_to_rgb16( + ColorSpaceFixture::Rgb, + 4, + 2, + &LOSSLESS_RGB_16BIT_422_4X2_C0, + &LOSSLESS_RGB_16BIT_422_4X2_C1, + &LOSSLESS_RGB_16BIT_422_4X2_C2, + ) +} + +/// A 4x4 8-bit SOF3 lossless APP14 RGB JPEG with valid 4:2:0 sampling. +pub fn lossless_rgb_8bit_420_4x4_jpeg(predictor: u8) -> Vec { + lossless_color_8bit_420_jpeg( + Some(0), + 4, + 4, + predictor, + Lossless420Planes8 { + c0: &LOSSLESS_RGB_8BIT_420_4X4_C0, + c1: &LOSSLESS_RGB_8BIT_420_4X4_C1, + c2: &LOSSLESS_RGB_8BIT_420_4X4_C2, + }, + ) +} + +/// A 4x4 8-bit SOF3 lossless APP14 RGB JPEG with valid 4:2:0 sampling and restart markers. +pub fn lossless_rgb_8bit_420_restart_4x4_jpeg(predictor: u8) -> Vec { + lossless_color_8bit_420_restart_jpeg( + Some(0), + 4, + 4, + predictor, + Lossless420Planes8 { + c0: &LOSSLESS_RGB_8BIT_420_4X4_C0, + c1: &LOSSLESS_RGB_8BIT_420_4X4_C1, + c2: &LOSSLESS_RGB_8BIT_420_4X4_C2, + }, + 2, + ) +} + +pub fn lossless_rgb_8bit_420_4x4_rgb8() -> Vec { + lossless_420_planes_to_rgb8( + ColorSpaceFixture::Rgb, + 4, + 4, + &LOSSLESS_RGB_8BIT_420_4X4_C0, + &LOSSLESS_RGB_8BIT_420_4X4_C1, + &LOSSLESS_RGB_8BIT_420_4X4_C2, + ) +} + +/// A 4x4 16-bit SOF3 lossless APP14 RGB JPEG with valid 4:2:0 sampling. +pub fn lossless_rgb_16bit_420_4x4_jpeg(predictor: u8) -> Vec { + lossless_color_16bit_420_jpeg( + Some(0), + 4, + 4, + predictor, + Lossless420Planes { + c0: &LOSSLESS_RGB_16BIT_420_4X4_C0, + c1: &LOSSLESS_RGB_16BIT_420_4X4_C1, + c2: &LOSSLESS_RGB_16BIT_420_4X4_C2, + }, + ) +} + +/// A 4x4 16-bit SOF3 lossless APP14 RGB JPEG with valid 4:2:0 sampling and restart markers. +pub fn lossless_rgb_16bit_420_restart_4x4_jpeg(predictor: u8) -> Vec { + lossless_color_16bit_420_restart_jpeg( + Some(0), + 4, + 4, + predictor, + Lossless420Planes { + c0: &LOSSLESS_RGB_16BIT_420_4X4_C0, + c1: &LOSSLESS_RGB_16BIT_420_4X4_C1, + c2: &LOSSLESS_RGB_16BIT_420_4X4_C2, + }, + 2, + ) +} + +pub fn lossless_rgb_16bit_420_4x4_rgb16() -> Vec { + lossless_420_planes_to_rgb16( + ColorSpaceFixture::Rgb, + 4, + 4, + &LOSSLESS_RGB_16BIT_420_4X4_C0, + &LOSSLESS_RGB_16BIT_420_4X4_C1, + &LOSSLESS_RGB_16BIT_420_4X4_C2, + ) +} + +/// A 3x3 16-bit SOF3 lossless APP14 RGB JPEG with row-boundary restart markers. +pub fn lossless_restart_predictor_rgb_16bit_3x3_jpeg(predictor: u8) -> Vec { + lossless_rgb_16bit_restart_jpeg(3, 3, predictor, 3, &LOSSLESS_RGB_16BIT_3X3_PIXELS) +} + +/// A 3x3 8-bit SOF3 lossless YCbCr JPEG using predictor 1..=7. +pub fn lossless_predictor_ycbcr_3x3_jpeg(predictor: u8) -> Vec { + lossless_ycbcr_jpeg(3, 3, predictor, &LOSSLESS_YCBCR_3X3_PIXELS) +} + +/// A 3x3 8-bit SOF3 lossless YCbCr JPEG with row-boundary restart markers. +pub fn lossless_restart_predictor_ycbcr_3x3_jpeg(predictor: u8) -> Vec { + lossless_ycbcr_restart_jpeg(3, 3, predictor, 3, &LOSSLESS_YCBCR_3X3_PIXELS) +} + +pub fn lossless_ycbcr_3x3_rgb8() -> Vec { + ycbcr8_pixels_to_rgb8(&LOSSLESS_YCBCR_3X3_PIXELS) +} + +/// A 3x3 16-bit SOF3 lossless YCbCr JPEG using predictor 1..=7. +pub fn lossless_predictor_ycbcr_16bit_3x3_jpeg(predictor: u8) -> Vec { + lossless_ycbcr_16bit_jpeg(3, 3, predictor, &LOSSLESS_YCBCR_16BIT_3X3_PIXELS) +} + +/// A 4x2 8-bit SOF3 lossless YCbCr JPEG with valid 4:2:2 sampling. +pub fn lossless_ycbcr_8bit_422_4x2_jpeg(predictor: u8) -> Vec { + lossless_color_8bit_422_jpeg( + None, + 4, + 2, + predictor, + Lossless422Planes8 { + c0: &LOSSLESS_YCBCR_8BIT_422_4X2_C0, + c1: &LOSSLESS_YCBCR_8BIT_422_4X2_C1, + c2: &LOSSLESS_YCBCR_8BIT_422_4X2_C2, + }, + ) +} + +/// A 4x2 8-bit SOF3 lossless YCbCr JPEG with valid 4:2:2 sampling and restart markers. +pub fn lossless_ycbcr_8bit_422_restart_4x2_jpeg(predictor: u8) -> Vec { + lossless_color_8bit_422_restart_jpeg( + None, + 4, + 2, + predictor, + Lossless422Planes8 { + c0: &LOSSLESS_YCBCR_8BIT_422_4X2_C0, + c1: &LOSSLESS_YCBCR_8BIT_422_4X2_C1, + c2: &LOSSLESS_YCBCR_8BIT_422_4X2_C2, + }, + 2, + ) +} + +pub fn lossless_ycbcr_8bit_422_4x2_rgb8() -> Vec { + lossless_422_planes_to_rgb8( + ColorSpaceFixture::YCbCr, + 4, + 2, + &LOSSLESS_YCBCR_8BIT_422_4X2_C0, + &LOSSLESS_YCBCR_8BIT_422_4X2_C1, + &LOSSLESS_YCBCR_8BIT_422_4X2_C2, + ) +} + +/// A 3x3 16-bit SOF3 lossless YCbCr JPEG with unsupported 4:2:2 sampling. +pub fn lossless_ycbcr_16bit_422_3x3_jpeg() -> Vec { + let mut bytes = lossless_predictor_ycbcr_16bit_3x3_jpeg(1); + set_first_sof3_component_sampling(&mut bytes, 0x21); + bytes +} + +/// A 4x2 16-bit SOF3 lossless YCbCr JPEG with valid 4:2:2 sampling. +pub fn lossless_ycbcr_16bit_422_4x2_jpeg(predictor: u8) -> Vec { + lossless_color_16bit_422_jpeg( + None, + 4, + 2, + predictor, + Lossless422Planes { + c0: &LOSSLESS_YCBCR_16BIT_422_4X2_C0, + c1: &LOSSLESS_YCBCR_16BIT_422_4X2_C1, + c2: &LOSSLESS_YCBCR_16BIT_422_4X2_C2, + }, + ) +} + +/// A 4x2 16-bit SOF3 lossless YCbCr JPEG with valid 4:2:2 sampling and restart markers. +pub fn lossless_ycbcr_16bit_422_restart_4x2_jpeg(predictor: u8) -> Vec { + lossless_color_16bit_422_restart_jpeg( + None, + 4, + 2, + predictor, + Lossless422Planes { + c0: &LOSSLESS_YCBCR_16BIT_422_4X2_C0, + c1: &LOSSLESS_YCBCR_16BIT_422_4X2_C1, + c2: &LOSSLESS_YCBCR_16BIT_422_4X2_C2, + }, + 2, + ) +} + +pub fn lossless_ycbcr_16bit_422_4x2_rgb16() -> Vec { + lossless_422_planes_to_rgb16( + ColorSpaceFixture::YCbCr, + 4, + 2, + &LOSSLESS_YCBCR_16BIT_422_4X2_C0, + &LOSSLESS_YCBCR_16BIT_422_4X2_C1, + &LOSSLESS_YCBCR_16BIT_422_4X2_C2, + ) +} + +/// A 4x4 8-bit SOF3 lossless YCbCr JPEG with valid 4:2:0 sampling. +pub fn lossless_ycbcr_8bit_420_4x4_jpeg(predictor: u8) -> Vec { + lossless_color_8bit_420_jpeg( + None, + 4, + 4, + predictor, + Lossless420Planes8 { + c0: &LOSSLESS_YCBCR_8BIT_420_4X4_C0, + c1: &LOSSLESS_YCBCR_8BIT_420_4X4_C1, + c2: &LOSSLESS_YCBCR_8BIT_420_4X4_C2, + }, + ) +} + +/// A 4x4 8-bit SOF3 lossless YCbCr JPEG with valid 4:2:0 sampling and restart markers. +pub fn lossless_ycbcr_8bit_420_restart_4x4_jpeg(predictor: u8) -> Vec { + lossless_color_8bit_420_restart_jpeg( + None, + 4, + 4, + predictor, + Lossless420Planes8 { + c0: &LOSSLESS_YCBCR_8BIT_420_4X4_C0, + c1: &LOSSLESS_YCBCR_8BIT_420_4X4_C1, + c2: &LOSSLESS_YCBCR_8BIT_420_4X4_C2, + }, + 2, + ) +} + +pub fn lossless_ycbcr_8bit_420_4x4_rgb8() -> Vec { + lossless_420_planes_to_rgb8( + ColorSpaceFixture::YCbCr, + 4, + 4, + &LOSSLESS_YCBCR_8BIT_420_4X4_C0, + &LOSSLESS_YCBCR_8BIT_420_4X4_C1, + &LOSSLESS_YCBCR_8BIT_420_4X4_C2, + ) +} + +/// A 4x4 16-bit SOF3 lossless YCbCr JPEG with valid 4:2:0 sampling. +pub fn lossless_ycbcr_16bit_420_4x4_jpeg(predictor: u8) -> Vec { + lossless_color_16bit_420_jpeg( + None, + 4, + 4, + predictor, + Lossless420Planes { + c0: &LOSSLESS_YCBCR_16BIT_420_4X4_C0, + c1: &LOSSLESS_YCBCR_16BIT_420_4X4_C1, + c2: &LOSSLESS_YCBCR_16BIT_420_4X4_C2, + }, + ) +} + +/// A 4x4 16-bit SOF3 lossless YCbCr JPEG with valid 4:2:0 sampling and restart markers. +pub fn lossless_ycbcr_16bit_420_restart_4x4_jpeg(predictor: u8) -> Vec { + lossless_color_16bit_420_restart_jpeg( + None, + 4, + 4, + predictor, + Lossless420Planes { + c0: &LOSSLESS_YCBCR_16BIT_420_4X4_C0, + c1: &LOSSLESS_YCBCR_16BIT_420_4X4_C1, + c2: &LOSSLESS_YCBCR_16BIT_420_4X4_C2, + }, + 2, + ) +} + +pub fn lossless_ycbcr_16bit_420_4x4_rgb16() -> Vec { + lossless_420_planes_to_rgb16( + ColorSpaceFixture::YCbCr, + 4, + 4, + &LOSSLESS_YCBCR_16BIT_420_4X4_C0, + &LOSSLESS_YCBCR_16BIT_420_4X4_C1, + &LOSSLESS_YCBCR_16BIT_420_4X4_C2, + ) +} + +/// A 3x3 16-bit SOF3 lossless YCbCr JPEG with row-boundary restart markers. +pub fn lossless_restart_predictor_ycbcr_16bit_3x3_jpeg(predictor: u8) -> Vec { + lossless_ycbcr_16bit_restart_jpeg(3, 3, predictor, 3, &LOSSLESS_YCBCR_16BIT_3X3_PIXELS) +} + +pub fn lossless_ycbcr_16bit_3x3_rgb16() -> Vec { + ycbcr16_pixels_to_rgb16(&LOSSLESS_YCBCR_16BIT_3X3_PIXELS) +} + +fn set_first_sof3_component_sampling(bytes: &mut [u8], sampling: u8) { + let sof = bytes + .windows(2) + .position(|window| window == [0xff, 0xc3]) + .expect("fixture has SOF3 marker"); + bytes[sof + 11] = sampling; +} + +fn lossless_grayscale_jpeg(width: u16, height: u16, predictor: u8, samples: &[u8]) -> Vec { + assert_eq!(samples.len(), usize::from(width) * usize::from(height)); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 11, 8]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[1, 1, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=8); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, predictor, 0, 0]); + bytes.extend(lossless_entropy(width, predictor, samples)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_rgb_jpeg(width: u16, height: u16, predictor: u8, samples: &[u8]) -> Vec { + assert_eq!(samples.len(), usize::from(width) * usize::from(height) * 3); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]); + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 17, 8]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=8); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, predictor, 0, 0, + ]); + bytes.extend(lossless_rgb_entropy(width, predictor, samples)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_rgb_restart_jpeg( + width: u16, + height: u16, + predictor: u8, + restart_interval: u16, + samples: &[u8], +) -> Vec { + assert_eq!(samples.len(), usize::from(width) * usize::from(height) * 3); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]); + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 17, 8]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=8); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04]); + bytes.extend_from_slice(&restart_interval.to_be_bytes()); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, predictor, 0, 0, + ]); + bytes.extend(lossless_rgb_entropy_with_restarts( + width, + predictor, + samples, + restart_interval, + )); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_ycbcr_jpeg(width: u16, height: u16, predictor: u8, samples: &[u8]) -> Vec { + assert_eq!(samples.len(), usize::from(width) * usize::from(height) * 3); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 17, 8]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=8); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, predictor, 0, 0, + ]); + bytes.extend(lossless_rgb_entropy(width, predictor, samples)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_ycbcr_restart_jpeg( + width: u16, + height: u16, + predictor: u8, + restart_interval: u16, + samples: &[u8], +) -> Vec { + assert_eq!(samples.len(), usize::from(width) * usize::from(height) * 3); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 17, 8]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=8); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04]); + bytes.extend_from_slice(&restart_interval.to_be_bytes()); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, predictor, 0, 0, + ]); + bytes.extend(lossless_rgb_entropy_with_restarts( + width, + predictor, + samples, + restart_interval, + )); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_grayscale_restart_jpeg( + width: u16, + height: u16, + predictor: u8, + restart_interval: u16, + samples: &[u8], +) -> Vec { + assert_eq!(samples.len(), usize::from(width) * usize::from(height)); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 11, 8]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[1, 1, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=8); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04]); + bytes.extend_from_slice(&restart_interval.to_be_bytes()); + bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, predictor, 0, 0]); + bytes.extend(lossless_entropy_with_restarts( + width, + predictor, + samples, + restart_interval, + )); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_grayscale_16bit_jpeg( + width: u16, + height: u16, + predictor: u8, + samples: &[u16], +) -> Vec { + assert_eq!(samples.len(), usize::from(width) * usize::from(height)); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 11, 16]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[1, 1, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=15); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, predictor, 0, 0]); + bytes.extend(lossless_entropy_16bit(width, predictor, samples)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_grayscale_16bit_restart_jpeg( + width: u16, + height: u16, + predictor: u8, + restart_interval: u16, + samples: &[u16], +) -> Vec { + assert_eq!(samples.len(), usize::from(width) * usize::from(height)); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 11, 16]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[1, 1, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=15); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04]); + bytes.extend_from_slice(&restart_interval.to_be_bytes()); + bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, predictor, 0, 0]); + bytes.extend(lossless_entropy_16bit_with_restarts( + width, + predictor, + samples, + restart_interval, + )); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_rgb_16bit_jpeg(width: u16, height: u16, predictor: u8, samples: &[u16]) -> Vec { + assert_eq!(samples.len(), usize::from(width) * usize::from(height) * 3); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]); + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 17, 16]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=15); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, predictor, 0, 0, + ]); + bytes.extend(lossless_rgb_entropy_16bit(width, predictor, samples)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_rgb_16bit_restart_jpeg( + width: u16, + height: u16, + predictor: u8, + restart_interval: u16, + samples: &[u16], +) -> Vec { + assert_eq!(samples.len(), usize::from(width) * usize::from(height) * 3); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, + 0x00, + ]); + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 17, 16]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=15); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04]); + bytes.extend_from_slice(&restart_interval.to_be_bytes()); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, predictor, 0, 0, + ]); + bytes.extend(lossless_rgb_entropy_16bit_with_restarts( + width, + predictor, + samples, + restart_interval, + )); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_ycbcr_16bit_jpeg(width: u16, height: u16, predictor: u8, samples: &[u16]) -> Vec { + assert_eq!(samples.len(), usize::from(width) * usize::from(height) * 3); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 17, 16]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=15); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, predictor, 0, 0, + ]); + bytes.extend(lossless_rgb_entropy_16bit(width, predictor, samples)); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_ycbcr_16bit_restart_jpeg( + width: u16, + height: u16, + predictor: u8, + restart_interval: u16, + samples: &[u16], +) -> Vec { + assert_eq!(samples.len(), usize::from(width) * usize::from(height) * 3); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 17, 16]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[3, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=15); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04]); + bytes.extend_from_slice(&restart_interval.to_be_bytes()); + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, predictor, 0, 0, + ]); + bytes.extend(lossless_rgb_entropy_16bit_with_restarts( + width, + predictor, + samples, + restart_interval, + )); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_color_8bit_422_jpeg( + adobe_transform: Option, + width: u16, + height: u16, + predictor: u8, + planes: Lossless422Planes8<'_>, +) -> Vec { + lossless_color_8bit_422_jpeg_impl(adobe_transform, width, height, predictor, planes, None) +} + +fn lossless_color_8bit_422_restart_jpeg( + adobe_transform: Option, + width: u16, + height: u16, + predictor: u8, + planes: Lossless422Planes8<'_>, + restart_interval: u16, +) -> Vec { + lossless_color_8bit_422_jpeg_impl( + adobe_transform, + width, + height, + predictor, + planes, + Some(restart_interval), + ) +} + +fn lossless_color_8bit_422_jpeg_impl( + adobe_transform: Option, + width: u16, + height: u16, + predictor: u8, + planes: Lossless422Planes8<'_>, + restart_interval: Option, +) -> Vec { + let width_usize = usize::from(width); + let height_usize = usize::from(height); + let chroma_width = width_usize.div_ceil(2); + assert_eq!(planes.c0.len(), width_usize * height_usize); + assert_eq!(planes.c1.len(), chroma_width * height_usize); + assert_eq!(planes.c2.len(), chroma_width * height_usize); + + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + if let Some(transform) = adobe_transform { + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, + 0x00, transform, + ]); + } + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 17, 8]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[3, 1, 0x21, 0, 2, 0x11, 0, 3, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=8); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + if let Some(restart_interval) = restart_interval { + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04]); + bytes.extend_from_slice(&restart_interval.to_be_bytes()); + } + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, predictor, 0, 0, + ]); + let entropy = if let Some(restart_interval) = restart_interval { + lossless_422_entropy_8bit_with_restarts( + width, + height, + predictor, + planes.c0, + planes.c1, + planes.c2, + restart_interval, + ) + } else { + lossless_422_entropy_8bit(width, height, predictor, planes.c0, planes.c1, planes.c2) + }; + bytes.extend(entropy); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_color_8bit_420_jpeg( + adobe_transform: Option, + width: u16, + height: u16, + predictor: u8, + planes: Lossless420Planes8<'_>, +) -> Vec { + lossless_color_8bit_420_jpeg_impl(adobe_transform, width, height, predictor, planes, None) +} + +fn lossless_color_8bit_420_restart_jpeg( + adobe_transform: Option, + width: u16, + height: u16, + predictor: u8, + planes: Lossless420Planes8<'_>, + restart_interval: u16, +) -> Vec { + lossless_color_8bit_420_jpeg_impl( + adobe_transform, + width, + height, + predictor, + planes, + Some(restart_interval), + ) +} + +fn lossless_color_8bit_420_jpeg_impl( + adobe_transform: Option, + width: u16, + height: u16, + predictor: u8, + planes: Lossless420Planes8<'_>, + restart_interval: Option, +) -> Vec { + let width_usize = usize::from(width); + let height_usize = usize::from(height); + let chroma_width = width_usize.div_ceil(2); + let chroma_height = height_usize.div_ceil(2); + assert_eq!(planes.c0.len(), width_usize * height_usize); + assert_eq!(planes.c1.len(), chroma_width * chroma_height); + assert_eq!(planes.c2.len(), chroma_width * chroma_height); + + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + if let Some(transform) = adobe_transform { + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, + 0x00, transform, + ]); + } + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 17, 8]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[3, 1, 0x22, 0, 2, 0x11, 0, 3, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=8); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + if let Some(restart_interval) = restart_interval { + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04]); + bytes.extend_from_slice(&restart_interval.to_be_bytes()); + } + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, predictor, 0, 0, + ]); + let entropy = if let Some(restart_interval) = restart_interval { + lossless_420_entropy_8bit_with_restarts(width, height, predictor, planes, restart_interval) + } else { + lossless_420_entropy_8bit(width, height, predictor, planes) + }; + bytes.extend(entropy); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_color_16bit_422_jpeg( + adobe_transform: Option, + width: u16, + height: u16, + predictor: u8, + planes: Lossless422Planes<'_>, +) -> Vec { + lossless_color_16bit_422_jpeg_impl(adobe_transform, width, height, predictor, planes, None) +} + +fn lossless_color_16bit_422_restart_jpeg( + adobe_transform: Option, + width: u16, + height: u16, + predictor: u8, + planes: Lossless422Planes<'_>, + restart_interval: u16, +) -> Vec { + lossless_color_16bit_422_jpeg_impl( + adobe_transform, + width, + height, + predictor, + planes, + Some(restart_interval), + ) +} + +fn lossless_color_16bit_422_jpeg_impl( + adobe_transform: Option, + width: u16, + height: u16, + predictor: u8, + planes: Lossless422Planes<'_>, + restart_interval: Option, +) -> Vec { + let width_usize = usize::from(width); + let height_usize = usize::from(height); + let chroma_width = width_usize.div_ceil(2); + assert_eq!(planes.c0.len(), width_usize * height_usize); + assert_eq!(planes.c1.len(), chroma_width * height_usize); + assert_eq!(planes.c2.len(), chroma_width * height_usize); + + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + if let Some(transform) = adobe_transform { + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, + 0x00, transform, + ]); + } + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 17, 16]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[3, 1, 0x21, 0, 2, 0x11, 0, 3, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=15); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + if let Some(restart_interval) = restart_interval { + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04]); + bytes.extend_from_slice(&restart_interval.to_be_bytes()); + } + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, predictor, 0, 0, + ]); + let entropy = if let Some(restart_interval) = restart_interval { + lossless_422_entropy_16bit_with_restarts( + width, + height, + predictor, + planes.c0, + planes.c1, + planes.c2, + restart_interval, + ) + } else { + lossless_422_entropy_16bit(width, height, predictor, planes.c0, planes.c1, planes.c2) + }; + bytes.extend(entropy); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_color_16bit_420_jpeg( + adobe_transform: Option, + width: u16, + height: u16, + predictor: u8, + planes: Lossless420Planes<'_>, +) -> Vec { + lossless_color_16bit_420_jpeg_impl(adobe_transform, width, height, predictor, planes, None) +} + +fn lossless_color_16bit_420_restart_jpeg( + adobe_transform: Option, + width: u16, + height: u16, + predictor: u8, + planes: Lossless420Planes<'_>, + restart_interval: u16, +) -> Vec { + lossless_color_16bit_420_jpeg_impl( + adobe_transform, + width, + height, + predictor, + planes, + Some(restart_interval), + ) +} + +fn lossless_color_16bit_420_jpeg_impl( + adobe_transform: Option, + width: u16, + height: u16, + predictor: u8, + planes: Lossless420Planes<'_>, + restart_interval: Option, +) -> Vec { + let width_usize = usize::from(width); + let height_usize = usize::from(height); + let chroma_width = width_usize.div_ceil(2); + let chroma_height = height_usize.div_ceil(2); + assert_eq!(planes.c0.len(), width_usize * height_usize); + assert_eq!(planes.c1.len(), chroma_width * chroma_height); + assert_eq!(planes.c2.len(), chroma_width * chroma_height); + + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + if let Some(transform) = adobe_transform { + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, + 0x00, transform, + ]); + } + bytes.extend_from_slice(&[0xff, 0xc3, 0x00, 17, 16]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.extend_from_slice(&[3, 1, 0x22, 0, 2, 0x11, 0, 3, 0x11, 0]); + let mut dht = Vec::new(); + dht.push(0x00); + dht.extend_from_slice(&[0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + dht.extend(0..=15); + bytes.extend_from_slice(&[0xff, 0xc4]); + bytes.extend_from_slice(&(dht.len() as u16 + 2).to_be_bytes()); + bytes.extend(dht); + if let Some(restart_interval) = restart_interval { + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04]); + bytes.extend_from_slice(&restart_interval.to_be_bytes()); + } + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0c, 3, 1, 0x00, 2, 0x00, 3, 0x00, predictor, 0, 0, + ]); + let entropy = if let Some(restart_interval) = restart_interval { + lossless_420_entropy_16bit_with_restarts(width, height, predictor, planes, restart_interval) + } else { + lossless_420_entropy_16bit(width, height, predictor, planes) + }; + bytes.extend(entropy); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn lossless_entropy(width: u16, predictor: u8, samples: &[u8]) -> Vec { + let width = usize::from(width); + let mut bits = Vec::new(); + for (idx, &sample) in samples.iter().enumerate() { + let x = idx % width; + let y = idx / width; + let predicted = lossless_predicted_value(samples, width, x, y, predictor); + let diff = i32::from(sample) - predicted; + let category = lossless_diff_category(diff); + push_bits(&mut bits, u32::from(category), 4); + if category != 0 { + push_bits(&mut bits, lossless_magnitude_bits(diff, category), category); + } + } + pack_entropy_bits(bits) +} + +fn lossless_rgb_entropy(width: u16, predictor: u8, samples: &[u8]) -> Vec { + let width = usize::from(width); + let mut bits = Vec::new(); + for pixel in 0..samples.len() / 3 { + let x = pixel % width; + let y = pixel / width; + for component in 0..3 { + let sample = samples[pixel * 3 + component]; + let predicted = + lossless_predicted_rgb_value(samples, width, x, y, component, predictor); + let diff = i32::from(sample) - predicted; + let category = lossless_diff_category(diff); + push_bits(&mut bits, u32::from(category), 4); + if category != 0 { + push_bits(&mut bits, lossless_magnitude_bits(diff, category), category); + } + } + } + pack_entropy_bits(bits) +} + +fn lossless_rgb_entropy_with_restarts( + width: u16, + predictor: u8, + samples: &[u8], + restart_interval: u16, +) -> Vec { + assert!(restart_interval > 0); + let width = usize::from(width); + let restart_interval = usize::from(restart_interval); + let pixel_count = samples.len() / 3; + let mut out = Vec::new(); + let mut expected_rst = 0u8; + for segment_start in (0..pixel_count).step_by(restart_interval) { + let segment_end = (segment_start + restart_interval).min(pixel_count); + let mut bits = Vec::new(); + for (segment_offset, pixel) in (segment_start..segment_end).enumerate() { + let x = pixel % width; + let y = pixel / width; + for component in 0..3 { + let sample = samples[pixel * 3 + component]; + let predicted = if segment_offset == 0 { + 128 + } else { + lossless_predicted_rgb_value(samples, width, x, y, component, predictor) + }; + let diff = i32::from(sample) - predicted; + let category = lossless_diff_category(diff); + push_bits(&mut bits, u32::from(category), 4); + if category != 0 { + push_bits(&mut bits, lossless_magnitude_bits(diff, category), category); + } + } + } + out.extend(pack_entropy_bits(bits)); + if segment_end < pixel_count { + out.extend_from_slice(&[0xff, 0xd0 + expected_rst]); + expected_rst = (expected_rst + 1) & 0x07; + } + } + out +} + +fn lossless_entropy_with_restarts( + width: u16, + predictor: u8, + samples: &[u8], + restart_interval: u16, +) -> Vec { + assert!(restart_interval > 0); + let width = usize::from(width); + let restart_interval = usize::from(restart_interval); + let mut out = Vec::new(); + let mut expected_rst = 0u8; + for segment_start in (0..samples.len()).step_by(restart_interval) { + let segment_end = (segment_start + restart_interval).min(samples.len()); + let mut bits = Vec::new(); + for (segment_offset, idx) in (segment_start..segment_end).enumerate() { + let sample = samples[idx]; + let x = idx % width; + let y = idx / width; + let predicted = if segment_offset == 0 { + 128 + } else { + lossless_predicted_value(samples, width, x, y, predictor) + }; + let diff = i32::from(sample) - predicted; + let category = lossless_diff_category(diff); + push_bits(&mut bits, u32::from(category), 4); + if category != 0 { + push_bits(&mut bits, lossless_magnitude_bits(diff, category), category); + } + } + out.extend(pack_entropy_bits(bits)); + if segment_end < samples.len() { + out.extend_from_slice(&[0xff, 0xd0 + expected_rst]); + expected_rst = (expected_rst + 1) & 0x07; + } + } + out +} + +fn lossless_422_entropy_8bit( + width: u16, + height: u16, + predictor: u8, + c0: &[u8], + c1: &[u8], + c2: &[u8], +) -> Vec { + let width = usize::from(width); + let height = usize::from(height); + let chroma_width = width.div_ceil(2); + let mut bits = Vec::new(); + for y in 0..height { + for mcu_x in 0..chroma_width { + let x0 = mcu_x * 2; + encode_lossless_component_sample_8bit(&mut bits, c0, width, x0, y, predictor); + if x0 + 1 < width { + encode_lossless_component_sample_8bit(&mut bits, c0, width, x0 + 1, y, predictor); + } + encode_lossless_component_sample_8bit(&mut bits, c1, chroma_width, mcu_x, y, predictor); + encode_lossless_component_sample_8bit(&mut bits, c2, chroma_width, mcu_x, y, predictor); + } + } + pack_entropy_bits(bits) +} + +fn lossless_422_entropy_8bit_with_restarts( + width: u16, + height: u16, + predictor: u8, + c0: &[u8], + c1: &[u8], + c2: &[u8], + restart_interval: u16, +) -> Vec { + assert!(restart_interval > 0); + let width = usize::from(width); + let height = usize::from(height); + let chroma_width = width.div_ceil(2); + let total_mcus = chroma_width * height; + let restart_interval = usize::from(restart_interval); + let mut out = Vec::new(); + let mut expected_rst = 0u8; + for segment_start in (0..total_mcus).step_by(restart_interval) { + let segment_end = (segment_start + restart_interval).min(total_mcus); + let mut bits = Vec::new(); + for (segment_offset, mcu) in (segment_start..segment_end).enumerate() { + let y = mcu / chroma_width; + let mcu_x = mcu % chroma_width; + let x0 = mcu_x * 2; + encode_lossless_component_sample_8bit_with_restart( + &mut bits, + c0, + width, + x0, + y, + predictor, + segment_offset == 0, + ); + if x0 + 1 < width { + encode_lossless_component_sample_8bit(&mut bits, c0, width, x0 + 1, y, predictor); + } + encode_lossless_component_sample_8bit_with_restart( + &mut bits, + c1, + chroma_width, + mcu_x, + y, + predictor, + segment_offset == 0, + ); + encode_lossless_component_sample_8bit_with_restart( + &mut bits, + c2, + chroma_width, + mcu_x, + y, + predictor, + segment_offset == 0, + ); + } + out.extend(pack_entropy_bits(bits)); + if segment_end < total_mcus { + out.extend_from_slice(&[0xff, 0xd0 + expected_rst]); + expected_rst = (expected_rst + 1) & 0x07; + } + } + out +} + +fn lossless_420_entropy_8bit( + width: u16, + height: u16, + predictor: u8, + planes: Lossless420Planes8<'_>, +) -> Vec { + let width = usize::from(width); + let height = usize::from(height); + let chroma_width = width.div_ceil(2); + let chroma_height = height.div_ceil(2); + let mut bits = Vec::new(); + for mcu_y in 0..chroma_height { + for mcu_x in 0..chroma_width { + encode_lossless_420_mcu_8bit( + &mut bits, + predictor, + planes, + Lossless420Mcu { + image_width: width, + image_height: height, + chroma_width, + mcu_x, + mcu_y, + restart_first_mcu: false, + }, + ); + } + } + pack_entropy_bits(bits) +} + +fn lossless_420_entropy_8bit_with_restarts( + width: u16, + height: u16, + predictor: u8, + planes: Lossless420Planes8<'_>, + restart_interval: u16, +) -> Vec { + assert!(restart_interval > 0); + let width = usize::from(width); + let height = usize::from(height); + let chroma_width = width.div_ceil(2); + let chroma_height = height.div_ceil(2); + let total_mcus = chroma_width * chroma_height; + let restart_interval = usize::from(restart_interval); + let mut out = Vec::new(); + let mut expected_rst = 0u8; + for segment_start in (0..total_mcus).step_by(restart_interval) { + let segment_end = (segment_start + restart_interval).min(total_mcus); + let mut bits = Vec::new(); + for (segment_offset, mcu) in (segment_start..segment_end).enumerate() { + let mcu_y = mcu / chroma_width; + let mcu_x = mcu % chroma_width; + encode_lossless_420_mcu_8bit( + &mut bits, + predictor, + planes, + Lossless420Mcu { + image_width: width, + image_height: height, + chroma_width, + mcu_x, + mcu_y, + restart_first_mcu: segment_offset == 0, + }, + ); + } + out.extend(pack_entropy_bits(bits)); + if segment_end < total_mcus { + out.extend_from_slice(&[0xff, 0xd0 + expected_rst]); + expected_rst = (expected_rst + 1) & 0x07; + } + } + out +} + +fn lossless_entropy_16bit(width: u16, predictor: u8, samples: &[u16]) -> Vec { + let width = usize::from(width); + let mut bits = Vec::new(); + for (idx, &sample) in samples.iter().enumerate() { + let x = idx % width; + let y = idx / width; + let predicted = lossless_predicted_value_16bit(samples, width, x, y, predictor); + let diff = i32::from(sample) - predicted; + let category = lossless_diff_category(diff); + push_bits(&mut bits, u32::from(category), 4); + if category != 0 { + push_bits(&mut bits, lossless_magnitude_bits(diff, category), category); + } + } + pack_entropy_bits(bits) +} + +fn lossless_422_entropy_16bit( + width: u16, + height: u16, + predictor: u8, + c0: &[u16], + c1: &[u16], + c2: &[u16], +) -> Vec { + let width = usize::from(width); + let height = usize::from(height); + let chroma_width = width.div_ceil(2); + let mut bits = Vec::new(); + for y in 0..height { + for mcu_x in 0..chroma_width { + let x0 = mcu_x * 2; + encode_lossless_component_sample_16bit(&mut bits, c0, width, x0, y, predictor); + if x0 + 1 < width { + encode_lossless_component_sample_16bit(&mut bits, c0, width, x0 + 1, y, predictor); + } + encode_lossless_component_sample_16bit( + &mut bits, + c1, + chroma_width, + mcu_x, + y, + predictor, + ); + encode_lossless_component_sample_16bit( + &mut bits, + c2, + chroma_width, + mcu_x, + y, + predictor, + ); + } + } + pack_entropy_bits(bits) +} + +fn lossless_422_entropy_16bit_with_restarts( + width: u16, + height: u16, + predictor: u8, + c0: &[u16], + c1: &[u16], + c2: &[u16], + restart_interval: u16, +) -> Vec { + assert!(restart_interval > 0); + let width = usize::from(width); + let height = usize::from(height); + let chroma_width = width.div_ceil(2); + let total_mcus = chroma_width * height; + let restart_interval = usize::from(restart_interval); + let mut out = Vec::new(); + let mut expected_rst = 0u8; + for segment_start in (0..total_mcus).step_by(restart_interval) { + let segment_end = (segment_start + restart_interval).min(total_mcus); + let mut bits = Vec::new(); + for (segment_offset, mcu) in (segment_start..segment_end).enumerate() { + let y = mcu / chroma_width; + let mcu_x = mcu % chroma_width; + let x0 = mcu_x * 2; + encode_lossless_component_sample_16bit_with_restart( + &mut bits, + c0, + width, + x0, + y, + predictor, + segment_offset == 0, + ); + if x0 + 1 < width { + encode_lossless_component_sample_16bit(&mut bits, c0, width, x0 + 1, y, predictor); + } + encode_lossless_component_sample_16bit_with_restart( + &mut bits, + c1, + chroma_width, + mcu_x, + y, + predictor, + segment_offset == 0, + ); + encode_lossless_component_sample_16bit_with_restart( + &mut bits, + c2, + chroma_width, + mcu_x, + y, + predictor, + segment_offset == 0, + ); + } + out.extend(pack_entropy_bits(bits)); + if segment_end < total_mcus { + out.extend_from_slice(&[0xff, 0xd0 + expected_rst]); + expected_rst = (expected_rst + 1) & 0x07; + } + } + out +} + +fn lossless_420_entropy_16bit( + width: u16, + height: u16, + predictor: u8, + planes: Lossless420Planes<'_>, +) -> Vec { + let width = usize::from(width); + let height = usize::from(height); + let chroma_width = width.div_ceil(2); + let chroma_height = height.div_ceil(2); + let mut bits = Vec::new(); + for mcu_y in 0..chroma_height { + for mcu_x in 0..chroma_width { + encode_lossless_420_mcu_16bit( + &mut bits, + predictor, + planes, + Lossless420Mcu { + image_width: width, + image_height: height, + chroma_width, + mcu_x, + mcu_y, + restart_first_mcu: false, + }, + ); + } + } + pack_entropy_bits(bits) +} + +fn lossless_420_entropy_16bit_with_restarts( + width: u16, + height: u16, + predictor: u8, + planes: Lossless420Planes<'_>, + restart_interval: u16, +) -> Vec { + assert!(restart_interval > 0); + let width = usize::from(width); + let height = usize::from(height); + let chroma_width = width.div_ceil(2); + let chroma_height = height.div_ceil(2); + let total_mcus = chroma_width * chroma_height; + let restart_interval = usize::from(restart_interval); + let mut out = Vec::new(); + let mut expected_rst = 0u8; + for segment_start in (0..total_mcus).step_by(restart_interval) { + let segment_end = (segment_start + restart_interval).min(total_mcus); + let mut bits = Vec::new(); + for (segment_offset, mcu) in (segment_start..segment_end).enumerate() { + let mcu_y = mcu / chroma_width; + let mcu_x = mcu % chroma_width; + encode_lossless_420_mcu_16bit( + &mut bits, + predictor, + planes, + Lossless420Mcu { + image_width: width, + image_height: height, + chroma_width, + mcu_x, + mcu_y, + restart_first_mcu: segment_offset == 0, + }, + ); + } + out.extend(pack_entropy_bits(bits)); + if segment_end < total_mcus { + out.extend_from_slice(&[0xff, 0xd0 + expected_rst]); + expected_rst = (expected_rst + 1) & 0x07; + } + } + out +} + +#[derive(Clone, Copy)] +struct Lossless420Mcu { + image_width: usize, + image_height: usize, + chroma_width: usize, + mcu_x: usize, + mcu_y: usize, + restart_first_mcu: bool, +} + +fn encode_lossless_420_mcu_8bit( + bits: &mut Vec, + predictor: u8, + planes: Lossless420Planes8<'_>, + mcu: Lossless420Mcu, +) { + let x0 = mcu.mcu_x * 2; + let y0 = mcu.mcu_y * 2; + for local_y in 0..2 { + for local_x in 0..2 { + let x = x0 + local_x; + let y = y0 + local_y; + if x < mcu.image_width && y < mcu.image_height { + encode_lossless_component_sample_8bit_with_restart( + bits, + planes.c0, + mcu.image_width, + x, + y, + predictor, + mcu.restart_first_mcu && local_x == 0 && local_y == 0, + ); + } + } + } + encode_lossless_component_sample_8bit_with_restart( + bits, + planes.c1, + mcu.chroma_width, + mcu.mcu_x, + mcu.mcu_y, + predictor, + mcu.restart_first_mcu, + ); + encode_lossless_component_sample_8bit_with_restart( + bits, + planes.c2, + mcu.chroma_width, + mcu.mcu_x, + mcu.mcu_y, + predictor, + mcu.restart_first_mcu, + ); +} + +fn encode_lossless_component_sample_8bit( + bits: &mut Vec, + samples: &[u8], + width: usize, + x: usize, + y: usize, + predictor: u8, +) { + encode_lossless_component_sample_8bit_with_restart( + bits, samples, width, x, y, predictor, false, + ); +} + +fn encode_lossless_component_sample_8bit_with_restart( + bits: &mut Vec, + samples: &[u8], + width: usize, + x: usize, + y: usize, + predictor: u8, + restart_first_sample: bool, +) { + let sample = samples[y * width + x]; + let predicted = if restart_first_sample { + 128 + } else { + lossless_predicted_value(samples, width, x, y, predictor) + }; + let diff = i32::from(sample) - predicted; + let category = lossless_diff_category(diff); + push_bits(bits, u32::from(category), 4); + if category != 0 { + push_bits(bits, lossless_magnitude_bits(diff, category), category); + } +} + +fn encode_lossless_420_mcu_16bit( + bits: &mut Vec, + predictor: u8, + planes: Lossless420Planes<'_>, + mcu: Lossless420Mcu, +) { + let x0 = mcu.mcu_x * 2; + let y0 = mcu.mcu_y * 2; + for local_y in 0..2 { + for local_x in 0..2 { + let x = x0 + local_x; + let y = y0 + local_y; + if x < mcu.image_width && y < mcu.image_height { + encode_lossless_component_sample_16bit_with_restart( + bits, + planes.c0, + mcu.image_width, + x, + y, + predictor, + mcu.restart_first_mcu && local_x == 0 && local_y == 0, + ); + } + } + } + encode_lossless_component_sample_16bit_with_restart( + bits, + planes.c1, + mcu.chroma_width, + mcu.mcu_x, + mcu.mcu_y, + predictor, + mcu.restart_first_mcu, + ); + encode_lossless_component_sample_16bit_with_restart( + bits, + planes.c2, + mcu.chroma_width, + mcu.mcu_x, + mcu.mcu_y, + predictor, + mcu.restart_first_mcu, + ); +} + +fn encode_lossless_component_sample_16bit( + bits: &mut Vec, + samples: &[u16], + width: usize, + x: usize, + y: usize, + predictor: u8, +) { + encode_lossless_component_sample_16bit_with_restart( + bits, samples, width, x, y, predictor, false, + ); +} + +fn encode_lossless_component_sample_16bit_with_restart( + bits: &mut Vec, + samples: &[u16], + width: usize, + x: usize, + y: usize, + predictor: u8, + restart_first_sample: bool, +) { + let sample = samples[y * width + x]; + let predicted = if restart_first_sample { + 32768 + } else { + lossless_predicted_value_16bit(samples, width, x, y, predictor) + }; + let diff = i32::from(sample) - predicted; + let category = lossless_diff_category(diff); + push_bits(bits, u32::from(category), 4); + if category != 0 { + push_bits(bits, lossless_magnitude_bits(diff, category), category); + } +} + +fn lossless_entropy_16bit_with_restarts( + width: u16, + predictor: u8, + samples: &[u16], + restart_interval: u16, +) -> Vec { + assert!(restart_interval > 0); + let width = usize::from(width); + let restart_interval = usize::from(restart_interval); + let mut out = Vec::new(); + let mut expected_rst = 0u8; + for segment_start in (0..samples.len()).step_by(restart_interval) { + let segment_end = (segment_start + restart_interval).min(samples.len()); + let mut bits = Vec::new(); + for (segment_offset, idx) in (segment_start..segment_end).enumerate() { + let sample = samples[idx]; + let x = idx % width; + let y = idx / width; + let predicted = if segment_offset == 0 { + 32768 + } else { + lossless_predicted_value_16bit(samples, width, x, y, predictor) + }; + let diff = i32::from(sample) - predicted; + let category = lossless_diff_category(diff); + push_bits(&mut bits, u32::from(category), 4); + if category != 0 { + push_bits(&mut bits, lossless_magnitude_bits(diff, category), category); + } + } + out.extend(pack_entropy_bits(bits)); + if segment_end < samples.len() { + out.extend_from_slice(&[0xff, 0xd0 + expected_rst]); + expected_rst = (expected_rst + 1) & 0x07; + } + } + out +} + +fn lossless_rgb_entropy_16bit(width: u16, predictor: u8, samples: &[u16]) -> Vec { + let width = usize::from(width); + let mut bits = Vec::new(); + for pixel in 0..samples.len() / 3 { + let x = pixel % width; + let y = pixel / width; + for component in 0..3 { + let sample = samples[pixel * 3 + component]; + let predicted = + lossless_predicted_rgb_value_16bit(samples, width, x, y, component, predictor); + let diff = i32::from(sample) - predicted; + let category = lossless_diff_category(diff); + push_bits(&mut bits, u32::from(category), 4); + if category != 0 { + push_bits(&mut bits, lossless_magnitude_bits(diff, category), category); + } + } + } + pack_entropy_bits(bits) +} + +fn lossless_rgb_entropy_16bit_with_restarts( + width: u16, + predictor: u8, + samples: &[u16], + restart_interval: u16, +) -> Vec { + assert!(restart_interval > 0); + let width = usize::from(width); + let restart_interval = usize::from(restart_interval); + let pixel_count = samples.len() / 3; + let mut out = Vec::new(); + let mut expected_rst = 0u8; + for segment_start in (0..pixel_count).step_by(restart_interval) { + let segment_end = (segment_start + restart_interval).min(pixel_count); + let mut bits = Vec::new(); + for (segment_offset, pixel) in (segment_start..segment_end).enumerate() { + let x = pixel % width; + let y = pixel / width; + for component in 0..3 { + let sample = samples[pixel * 3 + component]; + let predicted = if segment_offset == 0 { + 32768 + } else { + lossless_predicted_rgb_value_16bit(samples, width, x, y, component, predictor) + }; + let diff = i32::from(sample) - predicted; + let category = lossless_diff_category(diff); + push_bits(&mut bits, u32::from(category), 4); + if category != 0 { + push_bits(&mut bits, lossless_magnitude_bits(diff, category), category); + } + } + } + out.extend(pack_entropy_bits(bits)); + if segment_end < pixel_count { + out.extend_from_slice(&[0xff, 0xd0 + expected_rst]); + expected_rst = (expected_rst + 1) & 0x07; + } + } + out +} + +fn lossless_predicted_value( + samples: &[u8], + width: usize, + x: usize, + y: usize, + predictor: u8, +) -> i32 { + let idx = y * width + x; + if x == 0 && y == 0 { + return 128; + } + if y == 0 { + return i32::from(samples[idx - 1]); + } + if x == 0 { + return i32::from(samples[idx - width]); + } + + let ra = i32::from(samples[idx - 1]); + let rb = i32::from(samples[idx - width]); + let rc = i32::from(samples[idx - width - 1]); + match predictor { + 1 => ra, + 2 => rb, + 3 => rc, + 4 => ra + rb - rc, + 5 => ra + ((rb - rc) >> 1), + 6 => rb + ((ra - rc) >> 1), + 7 => (ra + rb) >> 1, + _ => 128, + } +} + +fn lossless_predicted_rgb_value_16bit( + samples: &[u16], + width: usize, + x: usize, + y: usize, + component: usize, + predictor: u8, +) -> i32 { + let idx = (y * width + x) * 3 + component; + if x == 0 && y == 0 { + return 32768; + } + if y == 0 { + return i32::from(samples[idx - 3]); + } + if x == 0 { + return i32::from(samples[idx - width * 3]); + } + + let ra = i32::from(samples[idx - 3]); + let rb = i32::from(samples[idx - width * 3]); + let rc = i32::from(samples[idx - width * 3 - 3]); + match predictor { + 1 => ra, + 2 => rb, + 3 => rc, + 4 => ra + rb - rc, + 5 => ra + ((rb - rc) >> 1), + 6 => rb + ((ra - rc) >> 1), + 7 => (ra + rb) >> 1, + _ => 32768, + } +} + +fn lossless_predicted_value_16bit( + samples: &[u16], + width: usize, + x: usize, + y: usize, + predictor: u8, +) -> i32 { + let idx = y * width + x; + if x == 0 && y == 0 { + return 32768; + } + if y == 0 { + return i32::from(samples[idx - 1]); + } + if x == 0 { + return i32::from(samples[idx - width]); + } + + let ra = i32::from(samples[idx - 1]); + let rb = i32::from(samples[idx - width]); + let rc = i32::from(samples[idx - width - 1]); + match predictor { + 1 => ra, + 2 => rb, + 3 => rc, + 4 => ra + rb - rc, + 5 => ra + ((rb - rc) >> 1), + 6 => rb + ((ra - rc) >> 1), + 7 => (ra + rb) >> 1, + _ => 32768, + } +} + +fn lossless_predicted_rgb_value( + samples: &[u8], + width: usize, + x: usize, + y: usize, + component: usize, + predictor: u8, +) -> i32 { + let idx = (y * width + x) * 3 + component; + if x == 0 && y == 0 { + return 128; + } + if y == 0 { + return i32::from(samples[idx - 3]); + } + if x == 0 { + return i32::from(samples[idx - width * 3]); + } + + let ra = i32::from(samples[idx - 3]); + let rb = i32::from(samples[idx - width * 3]); + let rc = i32::from(samples[idx - (width + 1) * 3]); + match predictor { + 1 => ra, + 2 => rb, + 3 => rc, + 4 => ra + rb - rc, + 5 => ra + ((rb - rc) >> 1), + 6 => rb + ((ra - rc) >> 1), + 7 => (ra + rb) >> 1, + _ => 128, + } +} + +fn ycbcr8_pixels_to_rgb8(samples: &[u8]) -> Vec { + let mut out = Vec::with_capacity(samples.len()); + for pixel in samples.chunks_exact(3) { + let (r, g, b) = ycbcr8_to_rgb8_for_fixture(pixel[0], pixel[1], pixel[2]); + out.extend_from_slice(&[r, g, b]); + } + out +} + +fn ycbcr8_to_rgb8_for_fixture(y: u8, cb: u8, cr: u8) -> (u8, u8, u8) { + const FIX_1_40200: i32 = 91_881; + const FIX_0_34414: i32 = 22_554; + const FIX_0_71414: i32 = 46_802; + const FIX_1_77200: i32 = 116_130; + const ROUND: i32 = 1 << 15; + + let y = i32::from(y); + let blue_delta = i32::from(cb) - 128; + let red_delta = i32::from(cr) - 128; + let r = y + ((FIX_1_40200 * red_delta + ROUND) >> 16); + let g = y - ((FIX_0_34414 * blue_delta + FIX_0_71414 * red_delta + ROUND) >> 16); + let b = y + ((FIX_1_77200 * blue_delta + ROUND) >> 16); + + ( + r.clamp(0, 255) as u8, + g.clamp(0, 255) as u8, + b.clamp(0, 255) as u8, + ) +} + +fn ycbcr16_pixels_to_rgb16(samples: &[u16]) -> Vec { + let mut out = Vec::with_capacity(samples.len() * 2); + for pixel in samples.chunks_exact(3) { + let (r, g, b) = ycbcr16_to_rgb16_for_fixture(pixel[0], pixel[1], pixel[2]); + out.extend_from_slice(&r.to_le_bytes()); + out.extend_from_slice(&g.to_le_bytes()); + out.extend_from_slice(&b.to_le_bytes()); + } + out +} + +fn ycbcr16_to_rgb16_for_fixture(y: u16, cb: u16, cr: u16) -> (u16, u16, u16) { + const FIX_1_40200: i64 = 91_881; + const FIX_0_34414: i64 = 22_554; + const FIX_0_71414: i64 = 46_802; + const FIX_1_77200: i64 = 116_130; + const ROUND: i64 = 1 << 15; + + let y = i64::from(y); + let blue_delta = i64::from(cb) - 32768; + let red_delta = i64::from(cr) - 32768; + let r = y + ((FIX_1_40200 * red_delta + ROUND) >> 16); + let g = y - ((FIX_0_34414 * blue_delta + FIX_0_71414 * red_delta + ROUND) >> 16); + let b = y + ((FIX_1_77200 * blue_delta + ROUND) >> 16); + + ( + r.clamp(0, i64::from(u16::MAX)) as u16, + g.clamp(0, i64::from(u16::MAX)) as u16, + b.clamp(0, i64::from(u16::MAX)) as u16, + ) +} + +#[derive(Clone, Copy)] +enum ColorSpaceFixture { + Rgb, + YCbCr, +} + +fn lossless_422_planes_to_rgb8( + color_space: ColorSpaceFixture, + width: usize, + height: usize, + c0: &[u8], + c1: &[u8], + c2: &[u8], +) -> Vec { + let chroma_width = width.div_ceil(2); + let mut out = Vec::with_capacity(width * height * 3); + for y in 0..height { + let c1_row = &c1[y * chroma_width..(y + 1) * chroma_width]; + let c2_row = &c2[y * chroma_width..(y + 1) * chroma_width]; + for x in 0..width { + let c0_sample = c0[y * width + x]; + let c1_sample = upsample_h2v1_8bit_for_fixture(c1_row, x); + let c2_sample = upsample_h2v1_8bit_for_fixture(c2_row, x); + let (r, g, b) = match color_space { + ColorSpaceFixture::Rgb => (c0_sample, c1_sample, c2_sample), + ColorSpaceFixture::YCbCr => { + ycbcr8_to_rgb8_for_fixture(c0_sample, c1_sample, c2_sample) + } + }; + out.extend_from_slice(&[r, g, b]); + } + } + out +} + +fn lossless_420_planes_to_rgb8( + color_space: ColorSpaceFixture, + width: usize, + height: usize, + c0: &[u8], + c1: &[u8], + c2: &[u8], +) -> Vec { + let chroma_width = width.div_ceil(2); + let chroma_height = height.div_ceil(2); + let mut out = Vec::with_capacity(width * height * 3); + for y in 0..height { + for x in 0..width { + let c0_sample = c0[y * width + x]; + let c1_sample = + upsample_h2v2_8bit_for_fixture(c1, chroma_width, chroma_height, width, x, y); + let c2_sample = + upsample_h2v2_8bit_for_fixture(c2, chroma_width, chroma_height, width, x, y); + let (r, g, b) = match color_space { + ColorSpaceFixture::Rgb => (c0_sample, c1_sample, c2_sample), + ColorSpaceFixture::YCbCr => { + ycbcr8_to_rgb8_for_fixture(c0_sample, c1_sample, c2_sample) + } + }; + out.extend_from_slice(&[r, g, b]); + } + } + out +} + +fn lossless_422_planes_to_rgb16( + color_space: ColorSpaceFixture, + width: usize, + height: usize, + c0: &[u16], + c1: &[u16], + c2: &[u16], +) -> Vec { + let chroma_width = width.div_ceil(2); + let mut out = Vec::with_capacity(width * height * 6); + for y in 0..height { + let c1_row = &c1[y * chroma_width..(y + 1) * chroma_width]; + let c2_row = &c2[y * chroma_width..(y + 1) * chroma_width]; + for x in 0..width { + let c0_sample = c0[y * width + x]; + let c1_sample = upsample_h2v1_16bit_for_fixture(c1_row, x); + let c2_sample = upsample_h2v1_16bit_for_fixture(c2_row, x); + let (r, g, b) = match color_space { + ColorSpaceFixture::Rgb => (c0_sample, c1_sample, c2_sample), + ColorSpaceFixture::YCbCr => { + ycbcr16_to_rgb16_for_fixture(c0_sample, c1_sample, c2_sample) + } + }; + append_rgb16_pixel(&mut out, [r, g, b]); + } + } + out +} + +fn lossless_420_planes_to_rgb16( + color_space: ColorSpaceFixture, + width: usize, + height: usize, + c0: &[u16], + c1: &[u16], + c2: &[u16], +) -> Vec { + let chroma_width = width.div_ceil(2); + let chroma_height = height.div_ceil(2); + let mut out = Vec::with_capacity(width * height * 6); + for y in 0..height { + for x in 0..width { + let c0_sample = c0[y * width + x]; + let c1_sample = + upsample_h2v2_16bit_for_fixture(c1, chroma_width, chroma_height, width, x, y); + let c2_sample = + upsample_h2v2_16bit_for_fixture(c2, chroma_width, chroma_height, width, x, y); + let (r, g, b) = match color_space { + ColorSpaceFixture::Rgb => (c0_sample, c1_sample, c2_sample), + ColorSpaceFixture::YCbCr => { + ycbcr16_to_rgb16_for_fixture(c0_sample, c1_sample, c2_sample) + } + }; + append_rgb16_pixel(&mut out, [r, g, b]); + } + } + out +} + +fn upsample_h2v1_8bit_for_fixture(row: &[u8], output_x: usize) -> u8 { + if row.len() == 1 { + return row[0]; + } + let sample = output_x / 2; + if output_x == 0 { + row[0] + } else if output_x == row.len() * 2 - 1 { + row[row.len() - 1] + } else if output_x.is_multiple_of(2) { + ((3 * u32::from(row[sample]) + u32::from(row[sample - 1]) + 2) / 4) as u8 + } else { + ((3 * u32::from(row[sample]) + u32::from(row[sample + 1]) + 2) / 4) as u8 + } +} + +fn upsample_h2v2_8bit_for_fixture( + plane: &[u8], + chroma_width: usize, + chroma_height: usize, + output_width: usize, + output_x: usize, + output_y: usize, +) -> u8 { + let chroma_y = output_y / 2; + let current = &plane[chroma_y * chroma_width..(chroma_y + 1) * chroma_width]; + let near_y = if output_y.is_multiple_of(2) { + chroma_y.saturating_sub(1) + } else { + (chroma_y + 1).min(chroma_height - 1) + }; + let near = &plane[near_y * chroma_width..(near_y + 1) * chroma_width]; + let colsum = |index: usize| 3 * u32::from(current[index]) + u32::from(near[index]); + if chroma_width == 1 { + return ((4 * colsum(0) + 8) >> 4) as u8; + } + + let sample = output_x / 2; + let this = colsum(sample); + match output_x { + 0 => ((this * 4 + 8) >> 4) as u8, + _ if output_x == output_width - 1 => ((this * 4 + 7) >> 4) as u8, + _ if output_x.is_multiple_of(2) => { + let last = colsum(sample - 1); + ((this * 3 + last + 8) >> 4) as u8 + } + _ => { + let next = colsum(sample + 1); + ((this * 3 + next + 7) >> 4) as u8 + } + } +} + +fn upsample_h2v1_16bit_for_fixture(row: &[u16], output_x: usize) -> u16 { + if row.len() == 1 { + return row[0]; + } + let sample = output_x / 2; + if output_x == 0 { + row[0] + } else if output_x == row.len() * 2 - 1 { + row[row.len() - 1] + } else if output_x.is_multiple_of(2) { + ((3 * u32::from(row[sample]) + u32::from(row[sample - 1]) + 2) / 4) as u16 + } else { + ((3 * u32::from(row[sample]) + u32::from(row[sample + 1]) + 2) / 4) as u16 + } +} + +fn upsample_h2v2_16bit_for_fixture( + plane: &[u16], + chroma_width: usize, + chroma_height: usize, + output_width: usize, + output_x: usize, + output_y: usize, +) -> u16 { + let chroma_y = output_y / 2; + let current = &plane[chroma_y * chroma_width..(chroma_y + 1) * chroma_width]; + let near_y = if output_y.is_multiple_of(2) { + chroma_y.saturating_sub(1) + } else { + (chroma_y + 1).min(chroma_height - 1) + }; + let near = &plane[near_y * chroma_width..(near_y + 1) * chroma_width]; + let colsum = |index: usize| 3 * u32::from(current[index]) + u32::from(near[index]); + if chroma_width == 1 { + return ((4 * colsum(0) + 8) >> 4) as u16; + } + + let sample = output_x / 2; + let this = colsum(sample); + match output_x { + 0 => ((this * 4 + 8) >> 4) as u16, + _ if output_x == output_width - 1 => ((this * 4 + 7) >> 4) as u16, + _ if output_x.is_multiple_of(2) => { + let last = colsum(sample - 1); + ((this * 3 + last + 8) >> 4) as u16 + } + _ => { + let next = colsum(sample + 1); + ((this * 3 + next + 7) >> 4) as u16 + } + } +} + +fn lossless_diff_category(diff: i32) -> u8 { + if diff == 0 { + return 0; + } + let magnitude = diff.unsigned_abs(); + (32 - magnitude.leading_zeros()) as u8 +} + +fn lossless_magnitude_bits(diff: i32, category: u8) -> u32 { + if diff >= 0 { + return diff as u32; + } + (diff + ((1i32 << category) - 1)) as u32 +} + +fn push_bits(bits: &mut Vec, value: u32, count: u8) { + for bit in (0..count).rev() { + bits.push(((value >> bit) & 1) != 0); + } +} + +fn pack_entropy_bits(mut bits: Vec) -> Vec { + while !bits.len().is_multiple_of(8) { + bits.push(true); + } + let mut out = Vec::new(); + for chunk in bits.chunks_exact(8) { + let mut byte = 0u8; + for &bit in chunk { + byte = (byte << 1) | u8::from(bit); + } + out.push(byte); + if byte == 0xff { + out.push(0x00); + } + } + out +} + +/// An 8x8 Adobe APP14 CMYK JPEG whose four decoded channels are all 128. +pub fn cmyk_8x8_jpeg() -> Vec { + four_component_8x8_jpeg(Some(0)) +} + +/// An 8x8 12-bit extended sequential Adobe APP14 CMYK JPEG. +pub fn extended_12bit_cmyk_8x8_jpeg() -> Vec { + four_component_12bit_8x8_jpeg(Some(0)) +} + +/// An 8x8 12-bit extended sequential Adobe APP14 CMYK JPEG with non-neutral DC coefficients. +pub fn extended_12bit_cmyk_nonconstant_8x8_jpeg() -> Vec { + four_component_12bit_nonconstant_8x8_jpeg(Some(0), false) +} + +/// A 16x8 12-bit extended sequential Adobe APP14 CMYK JPEG with restart coding. +pub fn extended_12bit_cmyk_restart_16x8_jpeg() -> Vec { + four_component_12bit_constant_restart_jpeg(Some(0), 16, 8, [(1, 1), (1, 1), (1, 1), (1, 1)], 1) +} + +/// A 16x8 12-bit extended sequential Adobe APP14 CMYK JPEG with 4:2:2 sampling. +pub fn extended_12bit_cmyk_16x8_422_jpeg() -> Vec { + four_component_12bit_constant_jpeg(Some(0), 16, 8, [(2, 1), (1, 1), (1, 1), (1, 1)]) +} + +/// A 32x8 12-bit extended sequential Adobe APP14 CMYK JPEG with 4:2:2 sampling and restart coding. +pub fn extended_12bit_cmyk_422_restart_32x8_jpeg() -> Vec { + four_component_12bit_constant_restart_jpeg(Some(0), 32, 8, [(2, 1), (1, 1), (1, 1), (1, 1)], 1) +} + +/// A 16x16 12-bit extended sequential Adobe APP14 CMYK JPEG with 4:2:0 sampling. +pub fn extended_12bit_cmyk_16x16_420_jpeg() -> Vec { + four_component_12bit_constant_jpeg(Some(0), 16, 16, [(2, 2), (1, 1), (1, 1), (1, 1)]) +} + +/// A 32x16 12-bit extended sequential Adobe APP14 CMYK JPEG with 4:2:0 sampling and restart coding. +pub fn extended_12bit_cmyk_420_restart_32x16_jpeg() -> Vec { + four_component_12bit_constant_restart_jpeg(Some(0), 32, 16, [(2, 2), (1, 1), (1, 1), (1, 1)], 1) +} + +/// An 8x8 12-bit progressive Adobe APP14 CMYK JPEG with 4:4:4 sampling. +pub fn progressive_12bit_cmyk_8x8_jpeg() -> Vec { + progressive_12bit_four_component_constant_jpeg(Some(0), 8, 8, [(1, 1), (1, 1), (1, 1), (1, 1)]) +} + +/// An 8x8 12-bit progressive Adobe APP14 CMYK JPEG with non-neutral DC coefficients. +pub fn progressive_12bit_cmyk_nonconstant_8x8_jpeg() -> Vec { + four_component_12bit_nonconstant_8x8_jpeg(Some(0), true) +} + +/// A 16x8 12-bit progressive Adobe APP14 CMYK JPEG with restart coding. +pub fn progressive_12bit_cmyk_restart_16x8_jpeg() -> Vec { + progressive_12bit_four_component_constant_restart_jpeg( + Some(0), + 16, + 8, + [(1, 1), (1, 1), (1, 1), (1, 1)], + 1, + ) +} + +/// A 16x8 12-bit progressive Adobe APP14 CMYK JPEG with 4:2:2 sampling. +pub fn progressive_12bit_cmyk_16x8_422_jpeg() -> Vec { + progressive_12bit_four_component_constant_jpeg(Some(0), 16, 8, [(2, 1), (1, 1), (1, 1), (1, 1)]) +} + +/// A 32x8 12-bit progressive Adobe APP14 CMYK JPEG with 4:2:2 sampling and restart coding. +pub fn progressive_12bit_cmyk_422_restart_32x8_jpeg() -> Vec { + progressive_12bit_four_component_constant_restart_jpeg( + Some(0), + 32, + 8, + [(2, 1), (1, 1), (1, 1), (1, 1)], + 1, + ) +} + +/// A 16x16 12-bit progressive Adobe APP14 CMYK JPEG with 4:2:0 sampling. +pub fn progressive_12bit_cmyk_16x16_420_jpeg() -> Vec { + progressive_12bit_four_component_constant_jpeg( + Some(0), + 16, + 16, + [(2, 2), (1, 1), (1, 1), (1, 1)], + ) +} + +/// A 32x16 12-bit progressive Adobe APP14 CMYK JPEG with 4:2:0 sampling and restart coding. +pub fn progressive_12bit_cmyk_420_restart_32x16_jpeg() -> Vec { + progressive_12bit_four_component_constant_restart_jpeg( + Some(0), + 32, + 16, + [(2, 2), (1, 1), (1, 1), (1, 1)], + 1, + ) +} + +/// A 16x8 Adobe APP14 CMYK JPEG with 4:2:2 sampling and all decoded channels 128. +pub fn cmyk_16x8_422_jpeg() -> Vec { + four_component_constant_jpeg(Some(0), 16, 8, [(2, 1), (1, 1), (1, 1), (1, 1)]) +} + +/// A 16x16 Adobe APP14 CMYK JPEG with 4:2:0 sampling and all decoded channels 128. +pub fn cmyk_16x16_420_jpeg() -> Vec { + four_component_constant_jpeg(Some(0), 16, 16, [(2, 2), (1, 1), (1, 1), (1, 1)]) +} + +/// A legal but unusual 16x8 CMYK JPEG where component 0 is not the maximally sampled plane. +pub fn cmyk_16x8_nonleading_max_422_jpeg() -> Vec { + four_component_constant_jpeg(Some(0), 16, 8, [(1, 1), (2, 1), (1, 1), (1, 1)]) +} + +/// A malformed CMYK JPEG whose component sampling factors require a non-integer upsample ratio. +pub fn malformed_cmyk_nondivisible_sampling_jpeg() -> Vec { + four_component_constant_jpeg(Some(0), 24, 8, [(3, 1), (2, 1), (1, 1), (1, 1)]) +} + +/// An 8x8 Adobe APP14 YCCK JPEG whose four decoded channels are all 128. +pub fn ycck_8x8_jpeg() -> Vec { + four_component_8x8_jpeg(Some(2)) +} + +/// An 8x8 12-bit extended sequential Adobe APP14 YCCK JPEG. +pub fn extended_12bit_ycck_8x8_jpeg() -> Vec { + four_component_12bit_8x8_jpeg(Some(2)) +} + +/// An 8x8 12-bit extended sequential Adobe APP14 YCCK JPEG with non-neutral DC coefficients. +pub fn extended_12bit_ycck_nonconstant_8x8_jpeg() -> Vec { + four_component_12bit_nonconstant_8x8_jpeg(Some(2), false) +} + +/// A 16x8 12-bit extended sequential Adobe APP14 YCCK JPEG with restart coding. +pub fn extended_12bit_ycck_restart_16x8_jpeg() -> Vec { + four_component_12bit_constant_restart_jpeg(Some(2), 16, 8, [(1, 1), (1, 1), (1, 1), (1, 1)], 1) +} + +/// A 16x8 12-bit extended sequential Adobe APP14 YCCK JPEG with 4:2:2 sampling. +pub fn extended_12bit_ycck_16x8_422_jpeg() -> Vec { + four_component_12bit_constant_jpeg(Some(2), 16, 8, [(2, 1), (1, 1), (1, 1), (1, 1)]) +} + +/// A 32x8 12-bit extended sequential Adobe APP14 YCCK JPEG with 4:2:2 sampling and restart coding. +pub fn extended_12bit_ycck_422_restart_32x8_jpeg() -> Vec { + four_component_12bit_constant_restart_jpeg(Some(2), 32, 8, [(2, 1), (1, 1), (1, 1), (1, 1)], 1) +} + +/// A 16x16 12-bit extended sequential Adobe APP14 YCCK JPEG with 4:2:0 sampling. +pub fn extended_12bit_ycck_16x16_420_jpeg() -> Vec { + four_component_12bit_constant_jpeg(Some(2), 16, 16, [(2, 2), (1, 1), (1, 1), (1, 1)]) +} + +/// A 32x16 12-bit extended sequential Adobe APP14 YCCK JPEG with 4:2:0 sampling and restart coding. +pub fn extended_12bit_ycck_420_restart_32x16_jpeg() -> Vec { + four_component_12bit_constant_restart_jpeg(Some(2), 32, 16, [(2, 2), (1, 1), (1, 1), (1, 1)], 1) +} + +/// An 8x8 12-bit progressive Adobe APP14 YCCK JPEG with 4:4:4 sampling. +pub fn progressive_12bit_ycck_8x8_jpeg() -> Vec { + progressive_12bit_four_component_constant_jpeg(Some(2), 8, 8, [(1, 1), (1, 1), (1, 1), (1, 1)]) +} + +/// An 8x8 12-bit progressive Adobe APP14 YCCK JPEG with non-neutral DC coefficients. +pub fn progressive_12bit_ycck_nonconstant_8x8_jpeg() -> Vec { + four_component_12bit_nonconstant_8x8_jpeg(Some(2), true) +} + +/// A 16x8 12-bit progressive Adobe APP14 YCCK JPEG with restart coding. +pub fn progressive_12bit_ycck_restart_16x8_jpeg() -> Vec { + progressive_12bit_four_component_constant_restart_jpeg( + Some(2), + 16, + 8, + [(1, 1), (1, 1), (1, 1), (1, 1)], + 1, + ) +} + +/// A 16x8 12-bit progressive Adobe APP14 YCCK JPEG with 4:2:2 sampling. +pub fn progressive_12bit_ycck_16x8_422_jpeg() -> Vec { + progressive_12bit_four_component_constant_jpeg(Some(2), 16, 8, [(2, 1), (1, 1), (1, 1), (1, 1)]) +} + +/// A 32x8 12-bit progressive Adobe APP14 YCCK JPEG with 4:2:2 sampling and restart coding. +pub fn progressive_12bit_ycck_422_restart_32x8_jpeg() -> Vec { + progressive_12bit_four_component_constant_restart_jpeg( + Some(2), + 32, + 8, + [(2, 1), (1, 1), (1, 1), (1, 1)], + 1, + ) +} + +/// A 16x16 12-bit progressive Adobe APP14 YCCK JPEG with 4:2:0 sampling. +pub fn progressive_12bit_ycck_16x16_420_jpeg() -> Vec { + progressive_12bit_four_component_constant_jpeg( + Some(2), + 16, + 16, + [(2, 2), (1, 1), (1, 1), (1, 1)], + ) +} + +/// A 32x16 12-bit progressive Adobe APP14 YCCK JPEG with 4:2:0 sampling and restart coding. +pub fn progressive_12bit_ycck_420_restart_32x16_jpeg() -> Vec { + progressive_12bit_four_component_constant_restart_jpeg( + Some(2), + 32, + 16, + [(2, 2), (1, 1), (1, 1), (1, 1)], + 1, + ) +} + +/// A 16x8 Adobe APP14 YCCK JPEG with 4:2:2 sampling and all decoded channels 128. +pub fn ycck_16x8_422_jpeg() -> Vec { + four_component_constant_jpeg(Some(2), 16, 8, [(2, 1), (1, 1), (1, 1), (1, 1)]) +} + +/// A legal but unusual 16x8 YCCK JPEG where component 0 is not the maximally sampled plane. +pub fn ycck_16x8_nonleading_max_422_jpeg() -> Vec { + four_component_constant_jpeg(Some(2), 16, 8, [(1, 1), (2, 1), (1, 1), (1, 1)]) +} + +/// A 16x16 Adobe APP14 YCCK JPEG with 4:2:0 sampling and all decoded channels 128. +pub fn ycck_16x16_420_jpeg() -> Vec { + four_component_constant_jpeg(Some(2), 16, 16, [(2, 2), (1, 1), (1, 1), (1, 1)]) +} + +/// Reference RGB pixels for [`cmyk_8x8_jpeg`] and [`ycck_8x8_jpeg`]. +pub fn four_component_8x8_rgb() -> Vec { + vec![64; 8 * 8 * 3] +} + +/// Reference native-range RGB16 pixels for 12-bit CMYK/YCCK 4:4:4 fixtures. +pub fn four_component_12bit_8x8_rgb16() -> Vec { + repeat_rgb16_pixels(8, 8, [1024, 1024, 1024]) +} + +/// Reference native-range RGB16 pixels for non-constant 12-bit CMYK fixtures. +pub fn four_component_12bit_8x8_cmyk_nonconstant_rgb16() -> Vec { + repeat_rgb16_pixels(8, 8, [1024, 1028, 1026]) +} + +/// Reference native-range RGB16 pixels for non-constant 12-bit YCCK fixtures. +pub fn four_component_12bit_8x8_ycck_nonconstant_rgb16() -> Vec { + repeat_rgb16_pixels(8, 8, [1038, 1013, 1046]) +} + +/// Reference native-range RGB16 pixels for 16x8 12-bit CMYK/YCCK fixtures. +pub fn four_component_12bit_16x8_rgb16() -> Vec { + repeat_rgb16_pixels(16, 8, [1024, 1024, 1024]) +} + +/// Reference native-range RGB16 pixels for 32x8 12-bit CMYK/YCCK fixtures. +pub fn four_component_12bit_32x8_rgb16() -> Vec { + repeat_rgb16_pixels(32, 8, [1024, 1024, 1024]) +} + +/// Reference native-range RGB16 pixels for 16x16 12-bit CMYK/YCCK fixtures. +pub fn four_component_12bit_16x16_rgb16() -> Vec { + repeat_rgb16_pixels(16, 16, [1024, 1024, 1024]) +} + +/// Reference native-range RGB16 pixels for 32x16 12-bit CMYK/YCCK fixtures. +pub fn four_component_12bit_32x16_rgb16() -> Vec { + repeat_rgb16_pixels(32, 16, [1024, 1024, 1024]) +} + +/// Reference RGB pixels for 16x8 constant CMYK/YCCK fixtures. +pub fn four_component_16x8_rgb() -> Vec { + vec![64; 16 * 8 * 3] +} + +/// Reference RGB pixels for 16x16 constant CMYK/YCCK fixtures. +pub fn four_component_16x16_rgb() -> Vec { + vec![64; 16 * 16 * 3] +} + +/// An 8×8 baseline JPEG with 4:4:4 sampling. +pub fn baseline_444_8x8_jpeg() -> Vec { + include_bytes!("../fixtures/conformance/baseline_444_8x8.jpg").to_vec() +} + +/// Reference pixels for [`baseline_444_8x8_jpeg`]. +pub fn baseline_444_8x8_rgb() -> Vec { + include_bytes!("../fixtures/conformance/baseline_444_8x8.rgb").to_vec() +} + +/// A 16×8 baseline JPEG with 4:2:2 sampling. +pub fn baseline_422_16x8_jpeg() -> Vec { + include_bytes!("../fixtures/conformance/baseline_422_16x8.jpg").to_vec() +} + +/// Reference pixels for [`baseline_422_16x8_jpeg`]. +pub fn baseline_422_16x8_rgb() -> Vec { + include_bytes!("../fixtures/conformance/baseline_422_16x8.rgb").to_vec() +} + +/// A 32×16 baseline JPEG with 4:2:0 sampling and restart coding. +pub fn baseline_420_restart_32x16_jpeg() -> Vec { + include_bytes!("../fixtures/conformance/baseline_420_restart_32x16.jpg").to_vec() +} + +/// Reference pixels for [`baseline_420_restart_32x16_jpeg`]. +pub fn baseline_420_restart_32x16_rgb() -> Vec { + include_bytes!("../fixtures/conformance/baseline_420_restart_32x16.rgb").to_vec() +} + +/// An 8×8 APP14 RGB JPEG with constant pixel value `(200, 20, 10)`. +pub fn rgb_app14_8x8_jpeg() -> Vec { + vec![ + 0xff, 0xd8, 0xff, 0xee, 0x00, 0x0e, 0x41, 0x64, 0x6f, 0x62, 0x65, 0x00, 0x64, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xdb, 0x00, 0x43, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0xff, 0xc0, 0x00, + 0x11, 0x08, 0x00, 0x08, 0x00, 0x08, 0x03, 0x52, 0x11, 0x00, 0x47, 0x11, 0x00, 0x42, 0x11, + 0x00, 0xff, 0xc4, 0x00, 0x1f, 0x00, 0x00, 0x01, 0x05, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xc4, 0x00, 0xb5, 0x10, 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, + 0x04, 0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7d, 0x01, 0x02, 0x03, 0x00, 0x04, + 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, + 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, 0x62, + 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, + 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, + 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, + 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, + 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, + 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, + 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, + 0xd9, 0xda, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, + 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xff, 0xda, 0x00, 0x0c, 0x03, 0x52, 0x00, 0x47, + 0x00, 0x42, 0x00, 0x00, 0x3f, 0x00, 0xfe, 0x90, 0x2b, 0xf8, 0x9f, 0xaf, 0xe1, 0x3e, 0xbf, + 0xff, 0xd9, + ] +} + +/// Reference pixels for [`rgb_app14_8x8_jpeg`]. +pub fn rgb_app14_8x8_rgb() -> Vec { + let mut out = Vec::with_capacity(8 * 8 * 3); + for _ in 0..64 { + out.extend_from_slice(&[200, 20, 10]); + } + out +} + +/// A progressive 8×8 JPEG with 10 SOS markers. +pub fn progressive_8x8_jpeg() -> Vec { + vec![ + 0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01, 0x01, 0x00, 0x00, + 0x01, 0x00, 0x01, 0x00, 0x00, 0xff, 0xdb, 0x00, 0x43, 0x00, 0x03, 0x02, 0x02, 0x03, 0x02, + 0x02, 0x03, 0x03, 0x03, 0x03, 0x04, 0x03, 0x03, 0x04, 0x05, 0x08, 0x05, 0x05, 0x04, 0x04, + 0x05, 0x0a, 0x07, 0x07, 0x06, 0x08, 0x0c, 0x0a, 0x0c, 0x0c, 0x0b, 0x0a, 0x0b, 0x0b, 0x0d, + 0x0e, 0x12, 0x10, 0x0d, 0x0e, 0x11, 0x0e, 0x0b, 0x0b, 0x10, 0x16, 0x10, 0x11, 0x13, 0x14, + 0x15, 0x15, 0x15, 0x0c, 0x0f, 0x17, 0x18, 0x16, 0x14, 0x18, 0x12, 0x14, 0x15, 0x14, 0xff, + 0xdb, 0x00, 0x43, 0x01, 0x03, 0x04, 0x04, 0x05, 0x04, 0x05, 0x09, 0x05, 0x05, 0x09, 0x14, + 0x0d, 0x0b, 0x0d, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, + 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, + 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, + 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0x14, 0xff, 0xc2, 0x00, 0x11, 0x08, 0x00, 0x08, + 0x00, 0x08, 0x03, 0x01, 0x22, 0x00, 0x02, 0x11, 0x01, 0x03, 0x11, 0x01, 0xff, 0xc4, 0x00, + 0x15, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x06, 0xff, 0xc4, 0x00, 0x15, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x06, 0xff, 0xda, + 0x00, 0x0c, 0x03, 0x01, 0x00, 0x02, 0x10, 0x03, 0x10, 0x00, 0x00, 0x01, 0x88, 0x13, 0x6f, + 0x7f, 0xff, 0xc4, 0x00, 0x14, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xda, 0x00, 0x08, 0x01, 0x01, + 0x00, 0x01, 0x05, 0x02, 0x7f, 0xff, 0xc4, 0x00, 0x14, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xda, 0x00, + 0x08, 0x01, 0x03, 0x01, 0x01, 0x3f, 0x01, 0x7f, 0xff, 0xc4, 0x00, 0x14, 0x11, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xda, 0x00, 0x08, 0x01, 0x02, 0x01, 0x01, 0x3f, 0x01, 0x7f, 0xff, 0xc4, 0x00, 0x14, + 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0xff, 0xda, 0x00, 0x08, 0x01, 0x01, 0x00, 0x06, 0x3f, 0x02, 0x7f, 0xff, + 0xc4, 0x00, 0x14, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xda, 0x00, 0x08, 0x01, 0x01, 0x00, 0x01, 0x3f, + 0x21, 0x7f, 0xff, 0xda, 0x00, 0x0c, 0x03, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x00, 0x00, + 0x10, 0xf7, 0xff, 0xc4, 0x00, 0x14, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xda, 0x00, 0x08, 0x01, 0x03, + 0x01, 0x01, 0x3f, 0x10, 0x7f, 0xff, 0xc4, 0x00, 0x14, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xda, 0x00, + 0x08, 0x01, 0x02, 0x01, 0x01, 0x3f, 0x10, 0x7f, 0xff, 0xc4, 0x00, 0x14, 0x10, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xda, 0x00, 0x08, 0x01, 0x01, 0x00, 0x01, 0x3f, 0x10, 0x7f, 0xff, 0xd9, + ] +} + +fn four_component_8x8_jpeg(app14_transform: Option) -> Vec { + four_component_constant_jpeg(app14_transform, 8, 8, [(1, 1), (1, 1), (1, 1), (1, 1)]) +} + +fn four_component_12bit_8x8_jpeg(app14_transform: Option) -> Vec { + four_component_12bit_constant_jpeg(app14_transform, 8, 8, [(1, 1), (1, 1), (1, 1), (1, 1)]) +} + +fn four_component_12bit_constant_jpeg( + app14_transform: Option, + width: u16, + height: u16, + sampling: [(u8, u8); 4], +) -> Vec { + four_component_12bit_constant_jpeg_with_restart(app14_transform, width, height, sampling, None) +} + +fn four_component_12bit_constant_restart_jpeg( + app14_transform: Option, + width: u16, + height: u16, + sampling: [(u8, u8); 4], + restart_interval: u16, +) -> Vec { + four_component_12bit_constant_jpeg_with_restart( + app14_transform, + width, + height, + sampling, + Some(restart_interval), + ) +} + +fn four_component_12bit_constant_jpeg_with_restart( + app14_transform: Option, + width: u16, + height: u16, + sampling: [(u8, u8); 4], + restart_interval: Option, +) -> Vec { + let mut bytes = four_component_constant_jpeg_with_restart( + app14_transform, + width, + height, + sampling, + restart_interval, + ); + let sof = bytes + .windows(2) + .position(|window| window == [0xff, 0xc0]) + .expect("four-component fixture has SOF0 marker"); + bytes[sof + 1] = 0xc1; + bytes[sof + 4] = 12; + bytes +} + +fn progressive_12bit_four_component_constant_jpeg( + app14_transform: Option, + width: u16, + height: u16, + sampling: [(u8, u8); 4], +) -> Vec { + progressive_12bit_four_component_constant_jpeg_with_restart( + app14_transform, + width, + height, + sampling, + None, + ) +} + +fn progressive_12bit_four_component_constant_restart_jpeg( + app14_transform: Option, + width: u16, + height: u16, + sampling: [(u8, u8); 4], + restart_interval: u16, +) -> Vec { + progressive_12bit_four_component_constant_jpeg_with_restart( + app14_transform, + width, + height, + sampling, + Some(restart_interval), + ) +} + +fn progressive_12bit_four_component_constant_jpeg_with_restart( + app14_transform: Option, + width: u16, + height: u16, + sampling: [(u8, u8); 4], + restart_interval: Option, +) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + if let Some(transform) = app14_transform { + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, + 0x00, transform, + ]); + } + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(1u8, 64)); + bytes.extend_from_slice(&[0xff, 0xc2, 0x00, 20, 12]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.push(4); + for (idx, &(h, v)) in sampling.iter().enumerate() { + bytes.push((idx + 1) as u8); + bytes.push((h << 4) | v); + bytes.push(0); + } + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x00, + ]); + if let Some(interval) = restart_interval { + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04]); + bytes.extend_from_slice(&interval.to_be_bytes()); + } + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0e, 4, 1, 0x00, 2, 0x00, 3, 0x00, 4, 0x00, 0, 0, 0, + ]); + let max_h = sampling.iter().map(|&(h, _)| h).max().unwrap_or(1); + let max_v = sampling.iter().map(|&(_, v)| v).max().unwrap_or(1); + let mcu_cols = u32::from(width).div_ceil(u32::from(max_h) * 8); + let mcu_rows = u32::from(height).div_ceil(u32::from(max_v) * 8); + let blocks_per_mcu = sampling + .iter() + .map(|&(h, v)| u32::from(h) * u32::from(v)) + .sum::(); + let mcu_count = (mcu_cols * mcu_rows) as usize; + if let Some(interval) = restart_interval { + assert_eq!(interval, 1, "fixture helper restart segments every MCU"); + let mcu_bits = vec![false; blocks_per_mcu as usize]; + bytes.extend(restart_segmented_entropy(mcu_count, &mcu_bits)); + } else { + let bits = vec![false; mcu_count * blocks_per_mcu as usize]; + bytes.extend(pack_entropy_bits(bits)); + } + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn four_component_12bit_nonconstant_8x8_jpeg( + app14_transform: Option, + progressive: bool, +) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + if let Some(transform) = app14_transform { + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, + 0x00, transform, + ]); + } + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(16u8, 64)); + bytes.extend_from_slice(&[ + 0xff, + if progressive { 0xc2 } else { 0xc1 }, + 0x00, + 20, + 12, + 0, + 8, + 0, + 8, + 4, + 1, + 0x11, + 0, + 2, + 0x11, + 0, + 3, + 0x11, + 0, + 4, + 0x11, + 0, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + ]); + if !progressive { + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]); + } + bytes.extend_from_slice(&[ + 0xff, + 0xda, + 0x00, + 0x0e, + 4, + 1, + 0x00, + 2, + 0x00, + 3, + 0x00, + 4, + 0x00, + 0, + if progressive { 0 } else { 63 }, + 0, + ]); + bytes.extend(pack_entropy_bits(dc_category4_cmyk_mcu_bits(progressive))); + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} + +fn four_component_constant_jpeg( + app14_transform: Option, + width: u16, + height: u16, + sampling: [(u8, u8); 4], +) -> Vec { + four_component_constant_jpeg_with_restart(app14_transform, width, height, sampling, None) +} + +fn four_component_constant_jpeg_with_restart( + app14_transform: Option, + width: u16, + height: u16, + sampling: [(u8, u8); 4], + restart_interval: Option, +) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[0xff, 0xd8]); + if let Some(transform) = app14_transform { + bytes.extend_from_slice(&[ + 0xff, 0xee, 0x00, 0x0e, b'A', b'd', b'o', b'b', b'e', 0x00, 0x64, 0x00, 0x00, 0x00, + 0x00, transform, + ]); + } + bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); + bytes.extend(std::iter::repeat_n(1u8, 64)); + bytes.extend_from_slice(&[0xff, 0xc0, 0x00, 20, 8]); + bytes.extend_from_slice(&height.to_be_bytes()); + bytes.extend_from_slice(&width.to_be_bytes()); + bytes.push(4); + for (idx, &(h, v)) in sampling.iter().enumerate() { + bytes.push((idx + 1) as u8); + bytes.push((h << 4) | v); + bytes.push(0); + } + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x00, + ]); + bytes.extend_from_slice(&[ + 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x00, + ]); + if let Some(interval) = restart_interval { + bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04]); + bytes.extend_from_slice(&interval.to_be_bytes()); + } + bytes.extend_from_slice(&[ + 0xff, 0xda, 0x00, 0x0e, 4, 1, 0x00, 2, 0x00, 3, 0x00, 4, 0x00, 0, 63, 0, + ]); + let max_h = sampling.iter().map(|&(h, _)| h).max().unwrap_or(1); + let max_v = sampling.iter().map(|&(_, v)| v).max().unwrap_or(1); + let mcu_cols = u32::from(width).div_ceil(u32::from(max_h) * 8); + let mcu_rows = u32::from(height).div_ceil(u32::from(max_v) * 8); + let blocks_per_mcu = sampling + .iter() + .map(|&(h, v)| u32::from(h) * u32::from(v)) + .sum::(); + let mcu_count = (mcu_cols * mcu_rows) as usize; + if let Some(interval) = restart_interval { + assert_eq!(interval, 1, "fixture helper restart segments every MCU"); + let mut mcu_bits = Vec::with_capacity(blocks_per_mcu as usize * 2); + for _ in 0..blocks_per_mcu { + mcu_bits.push(false); + mcu_bits.push(false); + } + bytes.extend(restart_segmented_entropy(mcu_count, &mcu_bits)); + } else { + let mut bits = Vec::new(); + for _ in 0..(mcu_cols * mcu_rows * blocks_per_mcu) { + bits.push(false); + bits.push(false); + } + bytes.extend(pack_entropy_bits(bits)); + } + bytes.extend_from_slice(&[0xff, 0xd9]); + bytes +} diff --git a/crates/j2k-test-support/src/lib.rs b/crates/j2k-test-support/src/lib.rs new file mode 100644 index 00000000..e78d3e20 --- /dev/null +++ b/crates/j2k-test-support/src/lib.rs @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: Apache-2.0 + +#![forbid(unsafe_code)] + +use std::{ + fs, + io::{self, ErrorKind}, + path::Path, +}; + +mod corpus; +mod cuda; +mod fixtures; +mod jpeg_fixtures; +mod metal_shader; +mod pixels; + +pub use corpus::{collect_jpeg_paths, is_jpeg_path, paths_from_env}; +pub use cuda::{ + cuda_bench_required, cuda_htj2k_strict_required, cuda_jpeg_hardware_decode_required, + cuda_runtime_required, +}; +pub use fixtures::{ + baseline_grayscale_jpeg, jpeg_baseline_420_16x16, jpeg_baseline_420_16x16_rgb, + jpeg_baseline_420_restart_32x16, jpeg_baseline_420_restart_32x16_rgb, jpeg_baseline_422_16x8, + jpeg_baseline_422_16x8_rgb, jpeg_baseline_444_8x8, jpeg_baseline_444_8x8_rgb, + jpeg_grayscale_8x8, jpeg_grayscale_8x8_gray, minimal_baseline_jpeg, + minimal_baseline_jpeg_with_restart_interval, minimal_gray8_jpeg, + minimal_grayscale_jpeg_with_dimensions, minimal_j2k_codestream, minimal_jp2, + restart_coded_grayscale_jpeg, wrap_jp2_codestream, wrap_jp2_rgba_codestream, + JPEG_BASELINE_420_16X16, JPEG_BASELINE_420_16X16_RGB, JPEG_BASELINE_420_RESTART_32X16, + JPEG_BASELINE_420_RESTART_32X16_RGB, JPEG_BASELINE_422_16X8, JPEG_BASELINE_422_16X8_RGB, + JPEG_BASELINE_444_8X8, JPEG_BASELINE_444_8X8_RGB, JPEG_GRAYSCALE_8X8, JPEG_GRAYSCALE_8X8_GRAY, +}; +#[cfg(feature = "j2k-native-fixtures")] +pub use fixtures::{ + classic_j2k_gray8_fixture, htj2k_gray8_97_fixture, htj2k_gray8_fixture, + htj2k_gray8_large_fixture, htj2k_rgb8_97_fixture, htj2k_rgb8_fixture, + htj2k_rgb8_fixture_with_pixels, htj2k_rgb8_pattern_fixture, openhtj2k_refinement_odd_fixture, +}; +pub use jpeg_fixtures::*; +pub use metal_shader::{host_compiles_metal_pipeline, metal_kernel_names, unwired_metal_kernels}; +pub use pixels::{ + centered_rect, crop_interleaved_bytes, crop_interleaved_u16, crop_interleaved_u8, + project_scaled_interleaved_u16, project_scaled_interleaved_u8, rgb16le_to_rgba16le, + rgb16ne_to_opaque_rgba16ne, rgb8_to_rgba8, scaled_rect_covering, u16_samples_to_le_bytes, + PixelRect, +}; + +/// Generates deterministic RGB8 pixels for tests and benches. +pub fn patterned_rgb8(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 17 + y * 3) & 0xFF) as u8); + pixels.push(((x * 5 + y * 11 + 40) & 0xFF) as u8); + pixels.push(((x * 13 + y * 7 + 90) & 0xFF) as u8); + } + } + pixels +} + +/// Generates deterministic grayscale pixels for tests and benches. +pub fn patterned_gray8(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize); + for y in 0..height { + for x in 0..width { + pixels.push(((x * 19 + y * 23) & 0xFF) as u8); + } + } + pixels +} + +/// Generates a simple deterministic gradient for reference codec parity cases. +pub fn gradient_u8(width: u32, height: u32, channels: usize) -> Vec { + gradient_variant_u8(width, height, channels, 0) +} + +/// Generates a deterministic gradient variant keyed by `seed`. +pub fn gradient_variant_u8(width: u32, height: u32, channels: usize, seed: u32) -> Vec { + let mut out = Vec::with_capacity(width as usize * height as usize * channels); + for y in 0..height { + for x in 0..width { + for c in 0..channels { + out.push(((x + y + seed * 13 + (c as u32 * 17)) & 0xFF) as u8); + } + } + } + out +} + +/// Generates deterministic RGB8 pixels used by GPU upload/decode benches. +pub fn gpu_bench_rgb8(width: u32, height: u32) -> Vec { + let mut rgb = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + rgb.push(((x * 13 + y * 3) & 0xff) as u8); + rgb.push(((x * 5 + y * 11 + (x ^ y)) & 0xff) as u8); + rgb.push(((x * 7 + y * 17 + (x.wrapping_mul(y) >> 5)) & 0xff) as u8); + } + } + rgb +} + +/// Generates contiguous RGB8 tiles for JPEG baseline encode benches. +pub fn patterned_rgb8_tiles(width: u32, height: u32, tile_count: usize) -> Vec { + let mut rgb = Vec::with_capacity(width as usize * height as usize * 3 * tile_count); + for tile in 0..tile_count as u32 { + for y in 0..height { + for x in 0..width { + rgb.push(((x * 13 + y * 3 + tile * 29) & 0xff) as u8); + rgb.push(((x * 5 + y * 11 + (x ^ y) + tile * 17) & 0xff) as u8); + rgb.push(((x * 7 + y * 17 + (x.wrapping_mul(y) >> 5) + tile * 23) & 0xff) as u8); + } + } + } + rgb +} + +/// Compatibility alias for the old JP2 fixture helper name. +#[must_use] +pub fn wrap_codestream_jp2( + codestream: &[u8], + width: u32, + height: u32, + components: u16, + bit_depth: u8, + colorspace_enum: u32, +) -> Vec { + fixtures::wrap_jp2_codestream( + codestream, + width, + height, + components, + bit_depth, + colorspace_enum, + ) +} + +/// Builds a binary PGM (`P5`) or PPM (`P6`) fixture from raw 8-bit pixels. +/// +/// # Errors +/// +/// Returns an error when `channels` is not `1` or `3`, when the dimensions +/// overflow, or when `pixels.len()` does not match the requested image shape. +pub fn pnm_bytes(pixels: &[u8], width: u32, height: u32, channels: usize) -> io::Result> { + let magic = match channels { + 1 => "P5", + 3 => "P6", + _ => { + return Err(io::Error::new( + ErrorKind::InvalidInput, + "PNM fixtures support only 1 or 3 channels", + )); + } + }; + let expected_len = (width as usize) + .checked_mul(height as usize) + .and_then(|pixels| pixels.checked_mul(channels)) + .ok_or_else(|| io::Error::new(ErrorKind::InvalidInput, "PNM dimensions overflow"))?; + if pixels.len() != expected_len { + return Err(io::Error::new( + ErrorKind::InvalidInput, + format!( + "PNM pixel length {} does not match expected {expected_len}", + pixels.len() + ), + )); + } + + let mut bytes = format!("{magic}\n{width} {height}\n255\n").into_bytes(); + bytes.extend_from_slice(pixels); + Ok(bytes) +} + +/// Writes a binary PGM (`P5`) or PPM (`P6`) fixture. +/// +/// # Errors +/// +/// Returns an error when [`pnm_bytes`] rejects the image shape or when the file +/// cannot be written. +pub fn write_pnm( + path: impl AsRef, + pixels: &[u8], + width: u32, + height: u32, + channels: usize, +) -> io::Result<()> { + fs::write(path, pnm_bytes(pixels, width, height, channels)?) +} + +/// Reads pixel payload bytes from a binary PGM (`P5`) or PPM (`P6`) fixture. +/// +/// # Errors +/// +/// Returns an error when the file cannot be read or the PNM header is missing +/// the expected `P5`/`P6` magic, dimensions, or max-value fields. +pub fn read_pnm_pixels(path: impl AsRef) -> io::Result> { + let bytes = fs::read(path)?; + let mut cursor = 0; + + let magic = read_pnm_token(&bytes, &mut cursor) + .ok_or_else(|| io::Error::new(ErrorKind::InvalidData, "PNM missing magic"))?; + if magic != b"P5" && magic != b"P6" { + return Err(io::Error::new( + ErrorKind::InvalidData, + "PNM magic must be P5 or P6", + )); + } + + read_pnm_token(&bytes, &mut cursor) + .ok_or_else(|| io::Error::new(ErrorKind::InvalidData, "PNM missing width"))?; + read_pnm_token(&bytes, &mut cursor) + .ok_or_else(|| io::Error::new(ErrorKind::InvalidData, "PNM missing height"))?; + read_pnm_token(&bytes, &mut cursor) + .ok_or_else(|| io::Error::new(ErrorKind::InvalidData, "PNM missing max value"))?; + + while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() { + cursor += 1; + } + + Ok(bytes[cursor..].to_vec()) +} + +fn read_pnm_token<'a>(bytes: &'a [u8], cursor: &mut usize) -> Option<&'a [u8]> { + skip_pnm_separators(bytes, cursor); + if *cursor >= bytes.len() { + return None; + } + + let start = *cursor; + while *cursor < bytes.len() { + let byte = bytes[*cursor]; + if byte.is_ascii_whitespace() || byte == b'#' { + break; + } + *cursor += 1; + } + + (start != *cursor).then_some(&bytes[start..*cursor]) +} + +fn skip_pnm_separators(bytes: &[u8], cursor: &mut usize) { + while *cursor < bytes.len() { + let byte = bytes[*cursor]; + if byte.is_ascii_whitespace() { + *cursor += 1; + continue; + } + if byte == b'#' { + *cursor += 1; + while *cursor < bytes.len() && bytes[*cursor] != b'\n' { + *cursor += 1; + } + continue; + } + break; + } +} + +#[cfg(test)] +mod tests { + use super::{ + minimal_j2k_codestream, minimal_jp2, pnm_bytes, read_pnm_pixels, wrap_codestream_jp2, + write_pnm, + }; + + #[test] + fn minimal_j2k_codestream_has_j2k_magic_and_siz_marker() { + let codestream = minimal_j2k_codestream(); + + assert!(codestream.starts_with(&[0xFF, 0x4F])); + assert!(codestream.windows(2).any(|marker| marker == [0xFF, 0x51])); + } + + #[test] + fn minimal_jp2_wraps_the_minimal_codestream() { + let jp2 = minimal_jp2(); + + assert!(jp2.starts_with(&[0, 0, 0, 12, b'j', b'P', b' ', b' '])); + assert!(jp2.windows(4).any(|box_type| box_type == b"jp2c")); + assert!(jp2.windows(2).any(|marker| marker == [0xFF, 0x4F])); + } + + #[test] + fn jp2_wrapper_writes_image_header_dimensions_and_colorspace() { + let jp2 = wrap_codestream_jp2(&[0xFF, 0x4F], 320, 240, 3, 8, 16); + + assert!(jp2.windows(4).any(|box_type| box_type == b"jp2h")); + assert!(jp2.windows(4).any(|value| value == 240u32.to_be_bytes())); + assert!(jp2.windows(4).any(|value| value == 320u32.to_be_bytes())); + assert!(jp2.windows(4).any(|value| value == 16u32.to_be_bytes())); + } + + #[test] + fn pnm_bytes_writes_p5_and_p6_headers() { + assert_eq!( + pnm_bytes(&[1, 2], 2, 1, 1).unwrap(), + b"P5\n2 1\n255\n\x01\x02" + ); + assert_eq!( + pnm_bytes(&[1, 2, 3, 4, 5, 6], 1, 2, 3).unwrap(), + b"P6\n1 2\n255\n\x01\x02\x03\x04\x05\x06" + ); + } + + #[test] + fn pnm_bytes_rejects_unsupported_shape() { + assert!(pnm_bytes(&[1, 2], 1, 1, 2).is_err()); + assert!(pnm_bytes(&[1, 2], 2, 2, 1).is_err()); + } + + #[test] + fn write_and_read_pnm_round_trips_pixels_with_comments() { + let path = + std::env::temp_dir().join(format!("j2k-test-support-pnm-{}.ppm", std::process::id())); + let bytes = b"P6\n# generated by test\n2 1\n255\n\x01\x02\x03\x04\x05\x06"; + std::fs::write(&path, bytes).expect("write commented pnm fixture"); + + assert_eq!( + read_pnm_pixels(&path).expect("read pnm pixels"), + vec![1, 2, 3, 4, 5, 6] + ); + + write_pnm(&path, &[7, 8], 2, 1, 1).expect("write pnm"); + assert_eq!(read_pnm_pixels(&path).expect("read pnm pixels"), vec![7, 8]); + let _ = std::fs::remove_file(path); + } +} diff --git a/crates/j2k-test-support/src/metal_shader.rs b/crates/j2k-test-support/src/metal_shader.rs new file mode 100644 index 00000000..8150b9ef --- /dev/null +++ b/crates/j2k-test-support/src/metal_shader.rs @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 + +/// Return Metal kernel entry-point names declared as `kernel void`. +pub fn metal_kernel_names(source: &str) -> Vec { + source.lines().filter_map(metal_kernel_name).collect() +} + +/// Return true when `host_source` compiles a named Metal kernel into a pipeline. +pub fn host_compiles_metal_pipeline(host_source: &str, kernel_name: &str) -> bool { + let quoted = format!("\"{kernel_name}\""); + host_source.match_indices("ed).any(|(index, _)| { + let context = &host_source[index.saturating_sub(96)..index]; + context.contains("get_function(") || context.contains("pipeline(") + }) +} + +/// Return Metal kernels declared by shader sources but not compiled by host setup. +pub fn unwired_metal_kernels<'a>( + shader_sources: impl IntoIterator, + host_source: &str, +) -> Vec { + let mut names = Vec::new(); + for source in shader_sources { + for name in metal_kernel_names(source) { + if names.iter().all(|existing| existing != &name) { + names.push(name); + } + } + } + + names + .into_iter() + .filter(|name| !host_compiles_metal_pipeline(host_source, name)) + .collect() +} + +fn metal_kernel_name(line: &str) -> Option { + let rest = line.trim_start().strip_prefix("kernel void ")?; + rest.split(|ch: char| !(ch.is_ascii_alphanumeric() || ch == '_')) + .next() + .map(ToOwned::to_owned) +} diff --git a/crates/j2k-test-support/src/pixels.rs b/crates/j2k-test-support/src/pixels.rs new file mode 100644 index 00000000..77a7a9bf --- /dev/null +++ b/crates/j2k-test-support/src/pixels.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Pixel conversion and region projection helpers for tests. + +/// Rectangle type used by test helper APIs without depending on `j2k-core`. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct PixelRect { + pub x: u32, + pub y: u32, + pub w: u32, + pub h: u32, +} + +impl PixelRect { + /// Creates a pixel rectangle. + pub const fn new(x: u32, y: u32, w: u32, h: u32) -> Self { + Self { x, y, w, h } + } +} + +/// Returns a centered rectangle capped to the provided dimensions. +pub fn centered_rect((width, height): (u32, u32), side: u32) -> PixelRect { + let w = side.min(width); + let h = side.min(height); + PixelRect { + x: (width - w) / 2, + y: (height - h) / 2, + w, + h, + } +} + +/// Returns the downscaled rectangle that fully covers `rect`. +pub fn scaled_rect_covering(rect: PixelRect, denom: u32) -> PixelRect { + let x1 = (rect.x + rect.w).div_ceil(denom); + let y1 = (rect.y + rect.h).div_ceil(denom); + PixelRect { + x: rect.x / denom, + y: rect.y / denom, + w: x1 - rect.x / denom, + h: y1 - rect.y / denom, + } +} + +/// Crops interleaved `u8` samples from a full image. +pub fn crop_interleaved_u8( + full: &[u8], + full_width: usize, + channels: usize, + roi: PixelRect, +) -> Vec { + crop_interleaved_bytes(full, full_width, channels, roi) +} + +/// Crops interleaved bytes from a full image. +pub fn crop_interleaved_bytes( + full: &[u8], + full_width: usize, + bytes_per_pixel: usize, + roi: PixelRect, +) -> Vec { + let mut out = Vec::with_capacity(roi.w as usize * roi.h as usize * bytes_per_pixel); + let row_bytes = full_width * bytes_per_pixel; + let roi_row_bytes = roi.w as usize * bytes_per_pixel; + for y in roi.y as usize..(roi.y + roi.h) as usize { + let start = y * row_bytes + roi.x as usize * bytes_per_pixel; + out.extend_from_slice(&full[start..start + roi_row_bytes]); + } + out +} + +/// Crops interleaved `u16` samples from a full image. +pub fn crop_interleaved_u16( + full: &[u16], + full_width: usize, + channels: usize, + roi: PixelRect, +) -> Vec { + let mut out = Vec::with_capacity(roi.w as usize * roi.h as usize * channels); + let row_samples = full_width * channels; + let roi_row_samples = roi.w as usize * channels; + for y in roi.y as usize..(roi.y + roi.h) as usize { + let start = y * row_samples + roi.x as usize * channels; + out.extend_from_slice(&full[start..start + roi_row_samples]); + } + out +} + +/// Projects a downscaled interleaved `u8` output rectangle from a full image. +pub fn project_scaled_interleaved_u8( + full: &[u8], + width: u32, + height: u32, + channels: usize, + output_rect: PixelRect, + denom: u32, +) -> Vec { + let mut out = Vec::with_capacity(output_rect.w as usize * output_rect.h as usize * channels); + for sy in output_rect.y..output_rect.y + output_rect.h { + let src_y = sy.saturating_mul(denom).min(height - 1); + for sx in output_rect.x..output_rect.x + output_rect.w { + let src_x = sx.saturating_mul(denom).min(width - 1); + let offset = (src_y as usize * width as usize + src_x as usize) * channels; + out.extend_from_slice(&full[offset..offset + channels]); + } + } + out +} + +/// Projects a downscaled interleaved `u16` output rectangle from a full image. +pub fn project_scaled_interleaved_u16( + full: &[u16], + width: u32, + height: u32, + channels: usize, + output_rect: PixelRect, + denom: u32, +) -> Vec { + let mut out = Vec::with_capacity(output_rect.w as usize * output_rect.h as usize * channels); + for sy in output_rect.y..output_rect.y + output_rect.h { + let src_y = sy.saturating_mul(denom).min(height - 1); + for sx in output_rect.x..output_rect.x + output_rect.w { + let src_x = sx.saturating_mul(denom).min(width - 1); + let offset = (src_y as usize * width as usize + src_x as usize) * channels; + out.extend_from_slice(&full[offset..offset + channels]); + } + } + out +} + +/// Converts RGB8 bytes to RGBA8 bytes with constant alpha. +pub fn rgb8_to_rgba8(rgb: &[u8], alpha: u8) -> Vec { + let mut out = Vec::with_capacity(rgb.len() / 3 * 4); + for pixel in rgb.chunks_exact(3) { + out.extend_from_slice(&[pixel[0], pixel[1], pixel[2], alpha]); + } + out +} + +/// Converts `u16` samples to little-endian bytes. +pub fn u16_samples_to_le_bytes(samples: &[u16]) -> Vec { + let mut out = Vec::with_capacity(samples.len() * 2); + for sample in samples { + out.extend_from_slice(&sample.to_le_bytes()); + } + out +} + +/// Converts little-endian RGB16 bytes to RGBA16 bytes with constant alpha. +pub fn rgb16le_to_rgba16le(rgb: &[u8], alpha: u16) -> Vec { + let mut out = Vec::with_capacity(rgb.len() / 6 * 8); + let alpha = alpha.to_le_bytes(); + for pixel in rgb.chunks_exact(6) { + out.extend_from_slice(pixel); + out.extend_from_slice(&alpha); + } + out +} + +/// Converts native-endian RGB16 bytes to opaque RGBA16 bytes. +pub fn rgb16ne_to_opaque_rgba16ne(rgb: &[u8]) -> Vec { + let mut out = Vec::with_capacity(rgb.len() / 6 * 8); + for pixel in rgb.chunks_exact(6) { + out.extend_from_slice(pixel); + out.extend_from_slice(&u16::MAX.to_ne_bytes()); + } + out +} diff --git a/crates/j2k-tilecodec/Cargo.toml b/crates/j2k-tilecodec/Cargo.toml new file mode 100644 index 00000000..9b7b48a3 --- /dev/null +++ b/crates/j2k-tilecodec/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "j2k-tilecodec" +description = "Tile-oriented decompression codecs for image containers" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +readme = "README.md" + +[package.metadata.docs.rs] +all-features = true +targets = [] + +[lib] +name = "j2k_tilecodec" +path = "src/lib.rs" + +[dependencies] +j2k-core = { path = "../j2k-core", version = "=0.6.0" } +thiserror = { workspace = true } +flate2 = { workspace = true } +zstd = { workspace = true } +weezl = { workspace = true } + +[dev-dependencies] +criterion = { workspace = true } + +[[bench]] +name = "compare" +harness = false + +[lints] +workspace = true diff --git a/crates/j2k-tilecodec/README.md b/crates/j2k-tilecodec/README.md new file mode 100644 index 00000000..72c2c47f --- /dev/null +++ b/crates/j2k-tilecodec/README.md @@ -0,0 +1,7 @@ +# j2k-tilecodec + +Tile decompression helpers for J2K. + +Supported codecs include Deflate, Zstd, LZW, and uncompressed copy paths. +Shared bounded-read and scratch-pool helpers keep errors explicit and avoid +unbounded temporary allocation. diff --git a/crates/signinum-tilecodec/benches/compare.rs b/crates/j2k-tilecodec/benches/compare.rs similarity index 82% rename from crates/signinum-tilecodec/benches/compare.rs rename to crates/j2k-tilecodec/benches/compare.rs index ede35b6e..c8b82068 100644 --- a/crates/signinum-tilecodec/benches/compare.rs +++ b/crates/j2k-tilecodec/benches/compare.rs @@ -2,8 +2,8 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use flate2::{write::ZlibEncoder, Compression}; -use signinum_core::TileDecompress; -use signinum_tilecodec::{ +use j2k_core::TileDecompress; +use j2k_tilecodec::{ DeflateCodec, DeflatePool, LzwCodec, LzwPool, NoPool, UncompressedCodec, ZstdCodec, ZstdPool, }; use std::io::{Read, Write}; @@ -32,11 +32,11 @@ fn bench_compare(c: &mut Criterion) { let mut lzw_pool = LzwPool::new(); let mut no_pool = NoPool; - group.bench_function(BenchmarkId::new("signinum", "deflate"), |b| { + group.bench_function(BenchmarkId::new("j2k", "deflate"), |b| { let mut out = vec![0_u8; source.len()]; b.iter(|| { DeflateCodec::decompress_into(&mut deflate_pool, &deflate, &mut out) - .expect("signinum deflate") + .expect("j2k deflate") }); }); @@ -50,11 +50,9 @@ fn bench_compare(c: &mut Criterion) { }); }); - group.bench_function(BenchmarkId::new("signinum", "zstd"), |b| { + group.bench_function(BenchmarkId::new("j2k", "zstd"), |b| { let mut out = vec![0_u8; source.len()]; - b.iter(|| { - ZstdCodec::decompress_into(&mut zstd_pool, &zstd, &mut out).expect("signinum zstd") - }); + b.iter(|| ZstdCodec::decompress_into(&mut zstd_pool, &zstd, &mut out).expect("j2k zstd")); }); group.bench_function(BenchmarkId::new("reference", "zstd"), |b| { @@ -68,9 +66,9 @@ fn bench_compare(c: &mut Criterion) { }); }); - group.bench_function(BenchmarkId::new("signinum", "lzw"), |b| { + group.bench_function(BenchmarkId::new("j2k", "lzw"), |b| { let mut out = vec![0_u8; source.len()]; - b.iter(|| LzwCodec::decompress_into(&mut lzw_pool, &lzw, &mut out).expect("signinum lzw")); + b.iter(|| LzwCodec::decompress_into(&mut lzw_pool, &lzw, &mut out).expect("j2k lzw")); }); group.bench_function(BenchmarkId::new("reference", "lzw"), |b| { @@ -78,11 +76,10 @@ fn bench_compare(c: &mut Criterion) { b.iter(|| decoder.decode(&lzw).expect("reference lzw")); }); - group.bench_function(BenchmarkId::new("signinum", "uncompressed"), |b| { + group.bench_function(BenchmarkId::new("j2k", "uncompressed"), |b| { let mut out = vec![0_u8; source.len()]; b.iter(|| { - UncompressedCodec::decompress_into(&mut no_pool, &source, &mut out) - .expect("signinum raw") + UncompressedCodec::decompress_into(&mut no_pool, &source, &mut out).expect("j2k raw") }); }); diff --git a/crates/signinum-tilecodec/examples/decompress.rs b/crates/j2k-tilecodec/examples/decompress.rs similarity index 85% rename from crates/signinum-tilecodec/examples/decompress.rs rename to crates/j2k-tilecodec/examples/decompress.rs index 63d45066..5966ab1e 100644 --- a/crates/signinum-tilecodec/examples/decompress.rs +++ b/crates/j2k-tilecodec/examples/decompress.rs @@ -3,10 +3,10 @@ //! Decompress a zlib-wrapped Deflate tile payload into caller-owned output. //! //! Run with: -//! `cargo run -p signinum-tilecodec --example decompress` +//! `cargo run -p j2k-tilecodec --example decompress` use flate2::{write::ZlibEncoder, Compression}; -use signinum_tilecodec::{DeflateCodec, DeflatePool, TileDecompress}; +use j2k_tilecodec::{DeflateCodec, DeflatePool, TileDecompress}; use std::io::Write; fn main() -> Result<(), Box> { diff --git a/crates/signinum-j2k/fuzz/.gitignore b/crates/j2k-tilecodec/fuzz/.gitignore similarity index 72% rename from crates/signinum-j2k/fuzz/.gitignore rename to crates/j2k-tilecodec/fuzz/.gitignore index a0925114..1a45eee7 100644 --- a/crates/signinum-j2k/fuzz/.gitignore +++ b/crates/j2k-tilecodec/fuzz/.gitignore @@ -1,3 +1,4 @@ target corpus artifacts +coverage diff --git a/crates/signinum-tilecodec/fuzz/Cargo.lock b/crates/j2k-tilecodec/fuzz/Cargo.lock similarity index 86% rename from crates/signinum-tilecodec/fuzz/Cargo.lock rename to crates/j2k-tilecodec/fuzz/Cargo.lock index 9e02cd08..c870a530 100644 --- a/crates/signinum-tilecodec/fuzz/Cargo.lock +++ b/crates/j2k-tilecodec/fuzz/Cargo.lock @@ -16,9 +16,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.60" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", "jobserver", @@ -70,6 +70,32 @@ dependencies = [ "wasip2", ] +[[package]] +name = "j2k-core" +version = "0.6.0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "j2k-tilecodec" +version = "0.6.0" +dependencies = [ + "flate2", + "j2k-core", + "thiserror", + "weezl", + "zstd", +] + +[[package]] +name = "j2k-tilecodec-fuzz" +version = "0.1.0" +dependencies = [ + "j2k-tilecodec", + "libfuzzer-sys", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -82,15 +108,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libfuzzer-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12a681b7dd8ce12bff52488013ba614b869148d54dd79836ab85aafdd53f08d" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" dependencies = [ "arbitrary", "cc", @@ -98,9 +124,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.28" +version = "1.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc3a226e576f50782b3305c5ccf458698f92798987f551c6a02efe8276721e22" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" dependencies = [ "cc", "pkg-config", @@ -149,35 +175,9 @@ checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signinum-core" -version = "1.0.0" -dependencies = [ - "thiserror", -] - -[[package]] -name = "signinum-tilecodec" -version = "1.0.0" -dependencies = [ - "flate2", - "signinum-core", - "thiserror", - "weezl", - "zstd", -] - -[[package]] -name = "signinum-tilecodec-fuzz" -version = "0.1.0" -dependencies = [ - "libfuzzer-sys", - "signinum-tilecodec", -] +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "simd-adler32" @@ -187,9 +187,9 @@ checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -230,18 +230,18 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen", ] [[package]] name = "weezl" -version = "0.1.12" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2" [[package]] name = "wit-bindgen" diff --git a/crates/signinum-tilecodec/fuzz/Cargo.toml b/crates/j2k-tilecodec/fuzz/Cargo.toml similarity index 80% rename from crates/signinum-tilecodec/fuzz/Cargo.toml rename to crates/j2k-tilecodec/fuzz/Cargo.toml index 9220b9b0..83e93044 100644 --- a/crates/signinum-tilecodec/fuzz/Cargo.toml +++ b/crates/j2k-tilecodec/fuzz/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "signinum-tilecodec-fuzz" +name = "j2k-tilecodec-fuzz" version = "0.1.0" edition = "2021" publish = false @@ -9,7 +9,7 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" -signinum-tilecodec = { path = ".." } +j2k-tilecodec = { path = ".." } [[bin]] name = "decompress_fuzz" diff --git a/crates/signinum-tilecodec/fuzz/fuzz_targets/decompress_fuzz.rs b/crates/j2k-tilecodec/fuzz/fuzz_targets/decompress_fuzz.rs similarity index 91% rename from crates/signinum-tilecodec/fuzz/fuzz_targets/decompress_fuzz.rs rename to crates/j2k-tilecodec/fuzz/fuzz_targets/decompress_fuzz.rs index e5d8009c..81a3b927 100644 --- a/crates/signinum-tilecodec/fuzz/fuzz_targets/decompress_fuzz.rs +++ b/crates/j2k-tilecodec/fuzz/fuzz_targets/decompress_fuzz.rs @@ -1,10 +1,10 @@ #![no_main] -use libfuzzer_sys::fuzz_target; -use signinum_tilecodec::TileDecompress; -use signinum_tilecodec::{ +use j2k_tilecodec::TileDecompress; +use j2k_tilecodec::{ DeflateCodec, DeflatePool, LzwCodec, LzwPool, NoPool, UncompressedCodec, ZstdCodec, ZstdPool, }; +use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { let mut out = vec![0_u8; data.len().saturating_mul(8).saturating_add(1024)]; diff --git a/crates/signinum-tilecodec/src/bounded.rs b/crates/j2k-tilecodec/src/bounded.rs similarity index 77% rename from crates/signinum-tilecodec/src/bounded.rs rename to crates/j2k-tilecodec/src/bounded.rs index 2100478a..973f5dd5 100644 --- a/crates/signinum-tilecodec/src/bounded.rs +++ b/crates/j2k-tilecodec/src/bounded.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_core::BufferError; +use j2k_core::BufferError; use std::{fmt, io::Read}; #[derive(Debug)] @@ -22,7 +22,9 @@ impl From for crate::TileCodecError { fn from(error: BoundedReadError) -> Self { match error { BoundedReadError::OutputTooSmall(error) => Self::Buffer(error), - BoundedReadError::Io(error) => Self::Backend(error.to_string()), + BoundedReadError::Io(error) => { + crate::error::input_or_backend_io_error(&error, "bounded decode") + } } } } @@ -50,6 +52,15 @@ pub(crate) fn read_to_scratch_bounded( Ok(scratch.len()) } +pub(crate) fn read_to_output_bounded( + reader: R, + scratch: &mut Vec, + out: &mut [u8], +) -> Result { + read_to_scratch_bounded(reader, scratch, out.len())?; + Ok(copy_scratch_to_output(scratch, out)) +} + pub(crate) fn observed_too_small(required: usize, have: usize) -> BufferError { BufferError::OutputTooSmall { required, have } } diff --git a/crates/j2k-tilecodec/src/deflate.rs b/crates/j2k-tilecodec/src/deflate.rs new file mode 100644 index 00000000..6e435618 --- /dev/null +++ b/crates/j2k-tilecodec/src/deflate.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + bounded::{read_to_output_bounded, BoundedReadError}, + pool::DeflatePool, + TileCodecError, +}; +use flate2::read::{DeflateDecoder, ZlibDecoder}; +use j2k_core::TileDecompress; + +/// Decoder for zlib-wrapped or raw DEFLATE tile payloads. +pub struct DeflateCodec; + +impl TileDecompress for DeflateCodec { + type Error = TileCodecError; + type Pool = DeflatePool; + + fn expected_size(_input: &[u8]) -> Result, Self::Error> { + Ok(None) + } + + fn decompress_into( + pool: &mut Self::Pool, + input: &[u8], + out: &mut [u8], + ) -> Result { + match read_to_output_bounded(ZlibDecoder::new(input), &mut pool.scratch, out) { + Ok(written) => Ok(written), + Err(BoundedReadError::OutputTooSmall(error)) => Err(error.into()), + Err(BoundedReadError::Io(_zlib_error)) => { + pool.scratch.clear(); + match read_to_output_bounded(DeflateDecoder::new(input), &mut pool.scratch, out) { + Ok(written) => Ok(written), + Err(BoundedReadError::OutputTooSmall(error)) => Err(error.into()), + Err(BoundedReadError::Io(raw_error)) => { + Err(crate::error::input_or_backend_io_error( + &raw_error, + "deflate decode failed", + )) + } + } + } + } + } +} diff --git a/crates/j2k-tilecodec/src/error.rs b/crates/j2k-tilecodec/src/error.rs new file mode 100644 index 00000000..d403d6f7 --- /dev/null +++ b/crates/j2k-tilecodec/src/error.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_core::{BufferError, CodecError, InputError, Unsupported}; +use std::io::ErrorKind; + +#[derive(Debug, thiserror::Error)] +/// Error returned by tile decompression codecs. +pub enum TileCodecError { + #[error(transparent)] + /// Output buffer or allocation limit error. + Buffer(#[from] BufferError), + #[error(transparent)] + /// Input payload is truncated. + Input(#[from] InputError), + #[error("{context}: malformed input: {message}")] + /// Input payload is present but invalid for the selected codec. + Malformed { + /// Codec operation that rejected the payload. + context: &'static str, + /// Backend error or decoder status. + message: String, + }, + #[error(transparent)] + /// Compression type or feature is unsupported. + Unsupported(#[from] Unsupported), + #[error("{0}")] + /// Backend library reported a decode failure. + Backend(String), +} + +pub(crate) fn truncated_input(segment: &'static str) -> TileCodecError { + TileCodecError::Input(InputError::TruncatedAt { offset: 0, segment }) +} + +pub(crate) fn malformed_input(context: &'static str, message: impl Into) -> TileCodecError { + TileCodecError::Malformed { + context, + message: message.into(), + } +} + +pub(crate) fn malformed_io_error(error: &std::io::Error, context: &'static str) -> TileCodecError { + if error.kind() == ErrorKind::UnexpectedEof { + return truncated_input(context); + } + malformed_input(context, error.to_string()) +} + +pub(crate) fn input_or_backend_io_error( + error: &std::io::Error, + context: &'static str, +) -> TileCodecError { + match error.kind() { + ErrorKind::UnexpectedEof => truncated_input(context), + ErrorKind::InvalidData | ErrorKind::InvalidInput => { + malformed_input(context, error.to_string()) + } + _ => TileCodecError::Backend(format!("{context}: {error}")), + } +} + +impl CodecError for TileCodecError { + fn is_truncated(&self) -> bool { + matches!(self, Self::Input(_)) + } + + fn is_not_implemented(&self) -> bool { + false + } + + fn is_unsupported(&self) -> bool { + matches!(self, Self::Unsupported(_)) + } + + fn is_buffer_error(&self) -> bool { + matches!(self, Self::Buffer(_)) + } +} diff --git a/crates/signinum-tilecodec/src/lib.rs b/crates/j2k-tilecodec/src/lib.rs similarity index 75% rename from crates/signinum-tilecodec/src/lib.rs rename to crates/j2k-tilecodec/src/lib.rs index 3e5e8830..689006ac 100644 --- a/crates/signinum-tilecodec/src/lib.rs +++ b/crates/j2k-tilecodec/src/lib.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +//! TIFF-style tile decompression codecs used by image container adapters. + mod bounded; mod deflate; mod error; @@ -10,8 +12,8 @@ mod zstd_codec; pub use deflate::DeflateCodec; pub use error::TileCodecError; +pub use j2k_core::TileDecompress; pub use lzw::LzwCodec; pub use pool::{DeflatePool, LzwPool, NoPool, ZstdPool}; -pub use signinum_core::TileDecompress; pub use uncompressed::UncompressedCodec; pub use zstd_codec::ZstdCodec; diff --git a/crates/signinum-tilecodec/src/lzw.rs b/crates/j2k-tilecodec/src/lzw.rs similarity index 79% rename from crates/signinum-tilecodec/src/lzw.rs rename to crates/j2k-tilecodec/src/lzw.rs index e7c3fe52..2723f2bf 100644 --- a/crates/signinum-tilecodec/src/lzw.rs +++ b/crates/j2k-tilecodec/src/lzw.rs @@ -5,9 +5,10 @@ use crate::{ pool::LzwPool, TileCodecError, }; -use signinum_core::TileDecompress; +use j2k_core::TileDecompress; use weezl::{decode::Decoder, BitOrder}; +/// Decoder for TIFF-style LZW tile payloads. pub struct LzwCodec; impl TileDecompress for LzwCodec { @@ -59,21 +60,15 @@ impl TileDecompress for LzwCodec { } Ok(weezl::LzwStatus::Ok) => {} Ok(weezl::LzwStatus::NoProgress) => { - return Err(TileCodecError::Backend( - "lzw decode failed: no progress before end marker".to_string(), - )); + return Err(crate::error::malformed_input("lzw decode", "no progress")); } - Err(error) => { - return Err(TileCodecError::Backend(format!( - "lzw decode failed: {error:?}" - ))); + Err(_error) => { + return Err(crate::error::malformed_input("lzw decode", "decoder error")); } } if input_offset == input.len() { - return Err(TileCodecError::Backend( - "lzw decode failed: missing end marker".to_string(), - )); + return Err(crate::error::truncated_input("lzw decode")); } } } diff --git a/crates/j2k-tilecodec/src/pool.rs b/crates/j2k-tilecodec/src/pool.rs new file mode 100644 index 00000000..99d74a04 --- /dev/null +++ b/crates/j2k-tilecodec/src/pool.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_core::ScratchPool; + +macro_rules! vec_scratch_pool { + ($(#[$meta:meta])* $name:ident, $new_doc:literal) => { + $(#[$meta])* + #[derive(Debug, Default)] + pub struct $name { + pub(crate) scratch: Vec, + } + + impl $name { + #[must_use] + #[doc = $new_doc] + pub fn new() -> Self { + Self::default() + } + } + + impl ScratchPool for $name { + fn bytes_allocated(&self) -> usize { + self.scratch.capacity() + } + + fn reset(&mut self) { + self.scratch.clear(); + } + } + }; +} + +vec_scratch_pool!( + /// Scratch storage reused by DEFLATE decoding. + DeflatePool, + "Create an empty DEFLATE scratch pool." +); + +vec_scratch_pool!( + /// Scratch storage reused by Zstandard decoding. + ZstdPool, + "Create an empty Zstandard scratch pool." +); + +vec_scratch_pool!( + /// Scratch storage reused by LZW decoding. + LzwPool, + "Create an empty LZW scratch pool." +); + +#[derive(Debug, Default, Clone, Copy)] +/// Zero-sized scratch pool for codecs that do not allocate. +pub struct NoPool; + +impl ScratchPool for NoPool { + fn bytes_allocated(&self) -> usize { + 0 + } + + fn reset(&mut self) {} +} diff --git a/crates/signinum-tilecodec/src/uncompressed.rs b/crates/j2k-tilecodec/src/uncompressed.rs similarity index 88% rename from crates/signinum-tilecodec/src/uncompressed.rs rename to crates/j2k-tilecodec/src/uncompressed.rs index 4d023e1d..d494789a 100644 --- a/crates/signinum-tilecodec/src/uncompressed.rs +++ b/crates/j2k-tilecodec/src/uncompressed.rs @@ -1,8 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 use crate::{pool::NoPool, TileCodecError}; -use signinum_core::{BufferError, TileDecompress}; +use j2k_core::{BufferError, TileDecompress}; +/// Pass-through codec for uncompressed tile payloads. pub struct UncompressedCodec; impl TileDecompress for UncompressedCodec { diff --git a/crates/j2k-tilecodec/src/zstd_codec.rs b/crates/j2k-tilecodec/src/zstd_codec.rs new file mode 100644 index 00000000..6467053b --- /dev/null +++ b/crates/j2k-tilecodec/src/zstd_codec.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + bounded::{read_to_output_bounded, BoundedReadError}, + pool::ZstdPool, + TileCodecError, +}; +use j2k_core::TileDecompress; + +/// Decoder for Zstandard-compressed tile payloads. +pub struct ZstdCodec; + +impl TileDecompress for ZstdCodec { + type Error = TileCodecError; + type Pool = ZstdPool; + + fn expected_size(_input: &[u8]) -> Result, Self::Error> { + Ok(None) + } + + fn decompress_into( + pool: &mut Self::Pool, + input: &[u8], + out: &mut [u8], + ) -> Result { + pool.scratch.clear(); + let mut decoder = zstd::stream::read::Decoder::new(input) + .map_err(|error| crate::error::malformed_io_error(&error, "zstd decoder init"))?; + read_to_output_bounded(&mut decoder, &mut pool.scratch, out).map_err(|error| match error { + BoundedReadError::OutputTooSmall(error) => TileCodecError::Buffer(error), + BoundedReadError::Io(error) => { + crate::error::malformed_io_error(&error, "zstd decode failed") + } + }) + } +} diff --git a/crates/signinum-tilecodec/tests/decompress.rs b/crates/j2k-tilecodec/tests/decompress.rs similarity index 75% rename from crates/signinum-tilecodec/tests/decompress.rs rename to crates/j2k-tilecodec/tests/decompress.rs index 2d24dae5..d2eca3af 100644 --- a/crates/signinum-tilecodec/tests/decompress.rs +++ b/crates/j2k-tilecodec/tests/decompress.rs @@ -4,8 +4,8 @@ use flate2::{ write::{DeflateEncoder, ZlibEncoder}, Compression, }; -use signinum_core::{ScratchPool, TileDecompress}; -use signinum_tilecodec::{ +use j2k_core::{CodecError, ScratchPool, TileDecompress}; +use j2k_tilecodec::{ DeflateCodec, DeflatePool, LzwCodec, LzwPool, NoPool, TileCodecError, UncompressedCodec, ZstdCodec, ZstdPool, }; @@ -17,6 +17,7 @@ fn sample_bytes() -> Vec { } #[test] +#[cfg_attr(miri, ignore = "flate2 zlib backend calls foreign functions")] fn deflate_codec_decodes_zlib_wrapped_payload() { let source = sample_bytes(); let mut compressor = ZlibEncoder::new(Vec::new(), Compression::default()); @@ -34,6 +35,7 @@ fn deflate_codec_decodes_zlib_wrapped_payload() { } #[test] +#[cfg_attr(miri, ignore = "flate2 zlib backend calls foreign functions")] fn deflate_codec_decodes_raw_deflate_payload() { let source = sample_bytes(); let mut compressor = DeflateEncoder::new(Vec::new(), Compression::default()); @@ -50,6 +52,7 @@ fn deflate_codec_decodes_raw_deflate_payload() { } #[test] +#[cfg_attr(miri, ignore = "zstd backend calls foreign functions")] fn zstd_codec_roundtrips_payload() { let source = sample_bytes(); let encoded = zstd::stream::encode_all(std::io::Cursor::new(&source), 1).expect("zstd encode"); @@ -76,6 +79,46 @@ fn lzw_codec_roundtrips_payload() { assert_eq!(&out[..written], source.as_slice()); } +#[test] +#[cfg_attr(miri, ignore = "deflate and zstd backends call foreign functions")] +fn malformed_payloads_are_not_truncated_for_codec_error_classifier() { + let source = sample_bytes(); + + // Stored Deflate block with an invalid LEN/NLEN pair; this cannot be + // accepted by either the zlib-wrapped or raw-Deflate fallback path. + let deflate = [0x01_u8, 0x01, 0x00, 0x00, 0x00]; + + let mut deflate_pool = DeflatePool::new(); + let mut zstd_pool = ZstdPool::new(); + let mut out = vec![0_u8; source.len()]; + + for err in [ + DeflateCodec::decompress_into(&mut deflate_pool, &deflate, &mut out) + .expect_err("malformed deflate must fail"), + ZstdCodec::decompress_into(&mut zstd_pool, &[0, 1, 2, 3], &mut out) + .expect_err("malformed zstd must fail"), + ] { + assert!(matches!(err, TileCodecError::Malformed { .. }), "{err:?}"); + assert!(!CodecError::is_truncated(&err), "{err:?}"); + } +} + +#[test] +fn truncated_payloads_are_input_errors_for_codec_error_classifier() { + let source = sample_bytes(); + let mut lzw_encoder = Encoder::new(BitOrder::Msb, 8); + let mut lzw = lzw_encoder.encode(&source).expect("lzw encode"); + lzw.pop(); + + let mut lzw_pool = LzwPool::new(); + let mut out = vec![0_u8; source.len()]; + let err = LzwCodec::decompress_into(&mut lzw_pool, &lzw, &mut out) + .expect_err("truncated lzw must fail"); + + assert!(matches!(err, TileCodecError::Input(_)), "{err:?}"); + assert!(CodecError::is_truncated(&err), "{err:?}"); +} + #[test] fn uncompressed_codec_copies_input_verbatim() { let source = sample_bytes(); @@ -92,6 +135,7 @@ fn uncompressed_codec_copies_input_verbatim() { } #[test] +#[cfg_attr(miri, ignore = "deflate and zstd setup calls foreign functions")] fn codecs_reject_undersized_output() { let source = sample_bytes(); @@ -128,6 +172,7 @@ fn codecs_reject_undersized_output() { } #[test] +#[cfg_attr(miri, ignore = "flate2 zlib backend calls foreign functions")] fn deflate_codec_rejects_oversized_zlib_without_full_scratch_allocation() { let source = vec![0xA5; 1 << 20]; let mut compressor = ZlibEncoder::new(Vec::new(), Compression::best()); @@ -140,7 +185,7 @@ fn deflate_codec_rejects_oversized_zlib_without_full_scratch_allocation() { assert!(matches!( err, - TileCodecError::Buffer(signinum_core::BufferError::OutputTooSmall { + TileCodecError::Buffer(j2k_core::BufferError::OutputTooSmall { required, have: 128 }) if required == 129 @@ -154,6 +199,7 @@ fn deflate_codec_rejects_oversized_zlib_without_full_scratch_allocation() { } #[test] +#[cfg_attr(miri, ignore = "zstd backend calls foreign functions")] fn zstd_codec_rejects_oversized_payload_without_full_scratch_allocation() { let source = vec![0x5A; 1 << 20]; let encoded = zstd::stream::encode_all(std::io::Cursor::new(&source), 19).expect("zstd encode"); @@ -164,7 +210,7 @@ fn zstd_codec_rejects_oversized_payload_without_full_scratch_allocation() { assert!(matches!( err, - TileCodecError::Buffer(signinum_core::BufferError::OutputTooSmall { + TileCodecError::Buffer(j2k_core::BufferError::OutputTooSmall { required, have: 128 }) if required == 129 @@ -178,6 +224,7 @@ fn zstd_codec_rejects_oversized_payload_without_full_scratch_allocation() { } #[test] +#[cfg_attr(miri, ignore = "large LZW stress case is too slow under Miri")] fn lzw_codec_rejects_oversized_payload_without_full_scratch_allocation() { let source = vec![0x33; 1 << 20]; let mut compressor = Encoder::new(BitOrder::Msb, 8); @@ -189,7 +236,7 @@ fn lzw_codec_rejects_oversized_payload_without_full_scratch_allocation() { assert!(matches!( err, - TileCodecError::Buffer(signinum_core::BufferError::OutputTooSmall { + TileCodecError::Buffer(j2k_core::BufferError::OutputTooSmall { required, have: 128 }) if required == 129 @@ -203,6 +250,7 @@ fn lzw_codec_rejects_oversized_payload_without_full_scratch_allocation() { } #[test] +#[cfg_attr(miri, ignore = "zstd backend calls foreign functions")] fn pools_can_be_reused_across_calls() { let source = sample_bytes(); let encoded = zstd::stream::encode_all(std::io::Cursor::new(&source), 1).expect("zstd encode"); diff --git a/crates/j2k-transcode-cuda/Cargo.toml b/crates/j2k-transcode-cuda/Cargo.toml new file mode 100644 index 00000000..ac9dd453 --- /dev/null +++ b/crates/j2k-transcode-cuda/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "j2k-transcode-cuda" +description = "CUDA acceleration for JPEG to J2K and HTJ2K transcode paths" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +readme = "README.md" + +[package.metadata.docs.rs] +all-features = true +targets = [] + +[lib] +name = "j2k_transcode_cuda" +path = "src/lib.rs" + +[features] +default = [] +cuda-runtime = ["dep:j2k-cuda-runtime"] +cuda-profiling = ["cuda-runtime", "j2k-cuda-runtime/cuda-profiling"] + +[dependencies] +j2k-transcode = { path = "../j2k-transcode", version = "=0.6.0" } +j2k-cuda-runtime = { path = "../j2k-cuda-runtime", version = "=0.6.0", optional = true } +j2k-native = { path = "../j2k-native", version = "=0.6.0" } + +[dev-dependencies] +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.6.0" } +j2k-test-support = { path = "../j2k-test-support" } + +[lints.rust] +unsafe_code = "allow" +unsafe_op_in_unsafe_fn = "deny" +unreachable_pub = "warn" + +[lints.clippy] +pedantic = { level = "warn", priority = -1 } +undocumented_unsafe_blocks = "warn" +module_name_repetitions = "allow" +must_use_candidate = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +cast_possible_truncation = "allow" +cast_precision_loss = "allow" diff --git a/crates/j2k-transcode-cuda/README.md b/crates/j2k-transcode-cuda/README.md new file mode 100644 index 00000000..d18c6fde --- /dev/null +++ b/crates/j2k-transcode-cuda/README.md @@ -0,0 +1,6 @@ +# j2k-transcode-cuda + +CUDA acceleration adapter for J2K JPEG-to-J2K/HTJ2K transcode stages. + +This crate accelerates supported transform and code-block preparation stages. +It does not replace the transcode API in `j2k-transcode`. diff --git a/crates/j2k-transcode-cuda/src/cuda.rs b/crates/j2k-transcode-cuda/src/cuda.rs new file mode 100644 index 00000000..8cae8357 --- /dev/null +++ b/crates/j2k-transcode-cuda/src/cuda.rs @@ -0,0 +1,1836 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! CUDA dispatch boundary for the transcode accelerator. +//! +//! Each function uploads a DCT-grid job to the device, runs the ported kernel +//! in `j2k-cuda-runtime`, and returns wavelet bands / prequantized +//! components matching the `j2k-transcode` scalar oracle. Kernels are +//! wired incrementally; until a path is wired its dispatch returns a typed +//! [`CudaTranscodeError::UnsupportedJob`], which Auto mode treats as a scalar +//! fallback and Explicit mode surfaces as an error. + +use j2k_transcode::accelerator::{ + DctGridI16ToHtj2k97CodeBlockBatch, DctGridI16ToHtj2k97CodeBlockJob, DctGridToDwt53Job, + DctGridToDwt97Job, DctGridToHtj2k97CodeBlockJob, DctGridToReversibleDwt53Job, + Dwt97BatchStageTimings, EncodedHtJ2kCodeBlock, Htj2k97CodeBlockOptions, J2kSubBandType, + PreencodedHtj2k97CodeBlock, PreencodedHtj2k97CompactBatch, PreencodedHtj2k97CompactBatchGroups, + PreencodedHtj2k97CompactCodeBlock, PreencodedHtj2k97CompactComponent, + PreencodedHtj2k97CompactResolution, PreencodedHtj2k97CompactSubband, + PreencodedHtj2k97Component, PreencodedHtj2k97Resolution, PreencodedHtj2k97Subband, + PrequantizedHtj2k97CodeBlock, PrequantizedHtj2k97Component, PrequantizedHtj2k97Resolution, + PrequantizedHtj2k97Subband, ReversibleDwt53FirstLevel, +}; +use j2k_transcode::dct53_2d::Dwt53TwoDimensional; +use j2k_transcode::dct97_2d::Dwt97TwoDimensional; +use j2k_transcode::htj2k97_codeblock_oracle::{ + htj2k97_subband_delta, htj2k97_subband_total_bitplanes, +}; + +use std::sync::Arc; + +use j2k_cuda_runtime::{ + transcode_kernels_built, CudaBufferPool, CudaContext, CudaDwt97BatchStageTimings, + CudaHtj2k97CodeblockBands, CudaHtj2k97DeviceCodeblockBands, CudaHtj2k97QuantizeParams, + CudaHtj2kCompactEncodedCodeBlock, CudaHtj2kEncodeCodeBlockJob, CudaHtj2kEncodeResidentTarget, + CudaHtj2kEncodeResources, CudaHtj2kEncodeStageTimings, CudaHtj2kEncodeTables, + CudaHtj2kEncodedCodeBlock, CudaPooledDeviceBuffer, CudaTranscodeDwt97Bands, + CudaTranscodeReversible53Bands, +}; + +use crate::CudaTranscodeError; + +/// Returned until a given kernel path is wired to `j2k-cuda-runtime`. +const NOT_WIRED: CudaTranscodeError = + CudaTranscodeError::UnsupportedJob("j2k-transcode-cuda kernel not yet wired"); + +type GroupedPreencodedComponents = Vec<(usize, Vec)>; +type GroupedCompactPreencodedComponents = Vec<(usize, Vec)>; +type ResidentPreencodedGroups = ( + GroupedPreencodedComponents, + CudaHtj2kEncodeStageTimings, + usize, +); +type ResidentCompactPreencodedGroups = ( + Vec, + GroupedCompactPreencodedComponents, + CudaHtj2kEncodeStageTimings, + usize, +); + +/// Caller-owned CUDA runtime state reused across transcode dispatches. +#[derive(Clone, Debug, Default)] +pub(crate) struct CudaTranscodeSession { + context: Option, + buffer_pool: Option, + encode_resources: Option>, +} + +impl CudaTranscodeSession { + fn context(&mut self) -> Result { + if self.context.is_none() { + self.context = Some( + CudaContext::system_default().map_err(|_| CudaTranscodeError::CudaUnavailable)?, + ); + } + self.context + .clone() + .ok_or(CudaTranscodeError::CudaUnavailable) + } + + fn buffer_pool(&mut self, context: &CudaContext) -> CudaBufferPool { + if let Some(pool) = &self.buffer_pool { + return pool.clone(); + } + let pool = context.buffer_pool(); + self.buffer_pool = Some(pool.clone()); + pool + } + + fn encode_resources( + &mut self, + context: &CudaContext, + ) -> Result, CudaTranscodeError> { + if let Some(resources) = &self.encode_resources { + return Ok(Arc::clone(resources)); + } + let resources = Arc::new( + context + .upload_htj2k_encode_resources(cuda_htj2k_encode_tables()) + .map_err(|_| { + CudaTranscodeError::Kernel("CUDA HTJ2K encode resource upload failed") + })?, + ); + self.encode_resources = Some(Arc::clone(&resources)); + Ok(resources) + } +} + +/// Flatten `&[[i16; 64]]` into the contiguous `&[i16]` the runtime job expects. +fn flatten_blocks(blocks: &[[i16; 64]]) -> &[i16] { + blocks.as_flattened() +} + +fn bands_to_first_level(bands: CudaTranscodeReversible53Bands) -> ReversibleDwt53FirstLevel { + ReversibleDwt53FirstLevel { + ll: bands.ll, + hl: bands.hl, + lh: bands.lh, + hh: bands.hh, + low_width: bands.low_width, + low_height: bands.low_height, + high_width: bands.high_width, + high_height: bands.high_height, + } +} + +fn run_reversible( + context: &CudaContext, + job: DctGridToReversibleDwt53Job<'_>, +) -> Result { + let bands = context + .j2k_transcode_reversible_dwt53( + flatten_blocks(job.dequantized_blocks), + job.block_cols, + job.block_rows, + job.width, + job.height, + ) + .map_err(|_| CudaTranscodeError::Kernel("CUDA reversible 5/3 transcode dispatch failed"))?; + Ok(bands_to_first_level(bands)) +} + +pub(crate) fn dispatch_reversible_dwt53( + session: &mut CudaTranscodeSession, + job: DctGridToReversibleDwt53Job<'_>, +) -> Result { + if !transcode_kernels_built() { + return Err(CudaTranscodeError::CudaUnavailable); + } + let context = session.context()?; + run_reversible(&context, job) +} + +pub(crate) fn dispatch_reversible_dwt53_batch( + session: &mut CudaTranscodeSession, + jobs: &[DctGridToReversibleDwt53Job<'_>], +) -> Result, CudaTranscodeError> { + if !transcode_kernels_built() { + return Err(CudaTranscodeError::CudaUnavailable); + } + let context = session.context()?; + let mut outputs = Vec::with_capacity(jobs.len()); + for job in jobs { + outputs.push(run_reversible(&context, *job)?); + } + Ok(outputs) +} + +pub(crate) fn dispatch_dwt53( + _job: DctGridToDwt53Job<'_>, +) -> Result, CudaTranscodeError> { + Err(NOT_WIRED) +} + +/// Append the job's `[[f64; 8]; 8]` natural-order DCT blocks to a contiguous +/// `f32` coefficient buffer (row-major within block) the runtime kernels consume. +fn append_f64_blocks_to_f32(blocks: &[[[f64; 8]; 8]], out: &mut Vec) { + for block in blocks { + for row in block { + for &coeff in row { + out.push(coeff as f32); + } + } + } +} + +/// Append natural-order dequantized i16 DCT blocks directly to the contiguous +/// i16 coefficient buffer the runtime kernels consume. +fn append_i16_blocks(blocks: &[[i16; 64]], out: &mut Vec) { + out.extend_from_slice(flatten_blocks(blocks)); +} + +/// Flatten one job's DCT blocks into a fresh contiguous `f32` buffer. +fn flatten_f64_blocks_to_f32(blocks: &[[[f64; 8]; 8]]) -> Vec { + let mut out = Vec::with_capacity(blocks.len() * 64); + append_f64_blocks_to_f32(blocks, &mut out); + out +} + +/// Map the runtime's local batch timings onto the transcode accelerator type. +fn map_batch_timings(timings: CudaDwt97BatchStageTimings) -> Dwt97BatchStageTimings { + Dwt97BatchStageTimings { + pack_upload_us: timings.pack_upload_us, + idct_row_lift_us: timings.idct_row_lift_us, + column_lift_us: timings.column_lift_us, + quantize_codeblock_us: timings.quantize_codeblock_us, + ht_encode_us: timings.ht_encode_us, + ht_kernel_us: 0, + ht_status_readback_us: 0, + ht_compact_us: 0, + ht_output_readback_us: 0, + ht_codeblock_dispatches: timings.ht_codeblock_dispatches, + readback_us: timings.readback_us, + } +} + +fn set_ht_encode_timings( + timings: &mut Dwt97BatchStageTimings, + ht_timings: CudaHtj2kEncodeStageTimings, +) { + timings.ht_encode_us = ht_timings.ht_encode_us; + timings.ht_kernel_us = ht_timings.ht_kernel_us; + timings.ht_status_readback_us = ht_timings.ht_status_readback_us; + timings.ht_compact_us = ht_timings.ht_compact_us; + timings.ht_output_readback_us = ht_timings.ht_output_readback_us; +} + +fn add_ht_encode_timings( + timings: &mut Dwt97BatchStageTimings, + ht_timings: CudaHtj2kEncodeStageTimings, +) { + timings.ht_encode_us = timings.ht_encode_us.saturating_add(ht_timings.ht_encode_us); + timings.ht_kernel_us = timings.ht_kernel_us.saturating_add(ht_timings.ht_kernel_us); + timings.ht_status_readback_us = timings + .ht_status_readback_us + .saturating_add(ht_timings.ht_status_readback_us); + timings.ht_compact_us = timings + .ht_compact_us + .saturating_add(ht_timings.ht_compact_us); + timings.ht_output_readback_us = timings + .ht_output_readback_us + .saturating_add(ht_timings.ht_output_readback_us); +} + +fn accumulate_batch_timings(total: &mut Dwt97BatchStageTimings, next: Dwt97BatchStageTimings) { + total.pack_upload_us = total.pack_upload_us.saturating_add(next.pack_upload_us); + total.idct_row_lift_us = total.idct_row_lift_us.saturating_add(next.idct_row_lift_us); + total.column_lift_us = total.column_lift_us.saturating_add(next.column_lift_us); + total.quantize_codeblock_us = total + .quantize_codeblock_us + .saturating_add(next.quantize_codeblock_us); + total.ht_encode_us = total.ht_encode_us.saturating_add(next.ht_encode_us); + total.ht_kernel_us = total.ht_kernel_us.saturating_add(next.ht_kernel_us); + total.ht_status_readback_us = total + .ht_status_readback_us + .saturating_add(next.ht_status_readback_us); + total.ht_compact_us = total.ht_compact_us.saturating_add(next.ht_compact_us); + total.ht_output_readback_us = total + .ht_output_readback_us + .saturating_add(next.ht_output_readback_us); + total.ht_codeblock_dispatches = total + .ht_codeblock_dispatches + .saturating_add(next.ht_codeblock_dispatches); + total.readback_us = total.readback_us.saturating_add(next.readback_us); +} + +fn dwt97_bands_to_f64(bands: CudaTranscodeDwt97Bands) -> Dwt97TwoDimensional { + let widen = |band: Vec| -> Vec { band.into_iter().map(f64::from).collect() }; + Dwt97TwoDimensional { + ll: widen(bands.ll), + hl: widen(bands.hl), + lh: widen(bands.lh), + hh: widen(bands.hh), + low_width: bands.low_width, + low_height: bands.low_height, + high_width: bands.high_width, + high_height: bands.high_height, + } +} + +fn run_dwt97( + context: &CudaContext, + job: DctGridToDwt97Job<'_>, +) -> Result, CudaTranscodeError> { + let coeffs = flatten_f64_blocks_to_f32(job.blocks); + let bands = context + .j2k_transcode_dwt97( + &coeffs, + job.block_cols, + job.block_rows, + job.width, + job.height, + ) + .map_err(|_| CudaTranscodeError::Kernel("CUDA 9/7 transcode dispatch failed"))?; + Ok(dwt97_bands_to_f64(bands)) +} + +pub(crate) fn dispatch_dwt97( + session: &mut CudaTranscodeSession, + job: DctGridToDwt97Job<'_>, +) -> Result, CudaTranscodeError> { + if !transcode_kernels_built() { + return Err(CudaTranscodeError::CudaUnavailable); + } + let context = session.context()?; + run_dwt97(&context, job) +} + +pub(crate) fn dispatch_dwt97_batch( + session: &mut CudaTranscodeSession, + jobs: &[DctGridToDwt97Job<'_>], +) -> Result<(Vec>, Dwt97BatchStageTimings), CudaTranscodeError> { + if !transcode_kernels_built() { + return Err(CudaTranscodeError::CudaUnavailable); + } + let context = session.context()?; + + let Some(first) = jobs.first() else { + return Ok((Vec::new(), Dwt97BatchStageTimings::default())); + }; + + // Non-uniform geometry falls back to the per-job path (still correct, but no + // staged batch timings), matching Metal's same-geometry batch gating. + let uniform = jobs.iter().all(|job| { + job.block_cols == first.block_cols + && job.block_rows == first.block_rows + && job.width == first.width + && job.height == first.height + }); + if !uniform { + let mut outputs = Vec::with_capacity(jobs.len()); + for job in jobs { + outputs.push(run_dwt97(&context, *job)?); + } + return Ok((outputs, Dwt97BatchStageTimings::default())); + } + + let mut blocks = Vec::with_capacity(jobs.len() * first.block_cols * first.block_rows * 64); + for job in jobs { + append_f64_blocks_to_f32(job.blocks, &mut blocks); + } + let (bands, timings) = context + .j2k_transcode_dwt97_batch_with_pool( + &blocks, + jobs.len(), + first.block_cols, + first.block_rows, + first.width, + first.height, + &session.buffer_pool(&context), + ) + .map_err(|_| CudaTranscodeError::Kernel("CUDA 9/7 batch transcode dispatch failed"))?; + let outputs = bands.into_iter().map(dwt97_bands_to_f64).collect(); + Ok((outputs, map_batch_timings(timings))) +} + +/// Reslice one subband's code-block-major `i32` buffer (one item) into a +/// prequantized HTJ2K subband, mirroring the shared code-block oracle layout +/// (outer code-block row, inner code-block column, each block row-major). +fn subband_from_codeblock_slice( + data: &[i32], + width: usize, + height: usize, + sub_band_type: J2kSubBandType, + options: Htj2k97CodeBlockOptions, +) -> Result { + let cb_width = htj2k97_code_block_dim(options.code_block_width_exp)?; + let cb_height = htj2k97_code_block_dim(options.code_block_height_exp)?; + let num_cbs_x = width.div_ceil(cb_width); + let num_cbs_y = height.div_ceil(cb_height); + let mut code_blocks = Vec::with_capacity(num_cbs_x * num_cbs_y); + let mut offset = 0usize; + for cby in 0..num_cbs_y { + for cbx in 0..num_cbs_x { + let block_width = (width - cbx * cb_width).min(cb_width); + let block_height = (height - cby * cb_height).min(cb_height); + let len = block_width * block_height; + let end = offset.checked_add(len).ok_or(CudaTranscodeError::Kernel( + "CUDA 9/7 code-block band length overflow", + ))?; + if end > data.len() { + return Err(CudaTranscodeError::Kernel( + "CUDA 9/7 code-block band output is shorter than expected", + )); + } + code_blocks.push(PrequantizedHtj2k97CodeBlock { + coefficients: data[offset..end].to_vec(), + width: block_width as u32, + height: block_height as u32, + }); + offset = end; + } + } + if offset != data.len() { + return Err(CudaTranscodeError::Kernel( + "CUDA 9/7 code-block band output has trailing data", + )); + } + Ok(PrequantizedHtj2k97Subband { + sub_band_type, + num_cbs_x: num_cbs_x as u32, + num_cbs_y: num_cbs_y as u32, + total_bitplanes: htj2k97_subband_total_bitplanes(options, sub_band_type), + code_blocks, + }) +} + +/// Reslice the per-item code-block bands into prequantized HTJ2K components, +/// one per job (resolution nesting `[[LL], [HL, LH, HH]]`). +#[allow(clippy::similar_names)] +fn codeblock_bands_to_components( + bands: &CudaHtj2k97CodeblockBands, + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, +) -> Result, CudaTranscodeError> { + if bands.item_count != jobs.len() { + return Err(CudaTranscodeError::Kernel( + "CUDA 9/7 code-block band item count mismatch", + )); + } + let ll_size = bands.low_width * bands.low_height; + let hl_size = bands.high_width * bands.low_height; + let lh_size = bands.low_width * bands.high_height; + let hh_size = bands.high_width * bands.high_height; + validate_band_len(&bands.ll, bands.item_count, ll_size)?; + validate_band_len(&bands.hl, bands.item_count, hl_size)?; + validate_band_len(&bands.lh, bands.item_count, lh_size)?; + validate_band_len(&bands.hh, bands.item_count, hh_size)?; + jobs.iter() + .enumerate() + .map(|(item, job)| { + let ll = &bands.ll[item * ll_size..(item + 1) * ll_size]; + let hl = &bands.hl[item * hl_size..(item + 1) * hl_size]; + let lh = &bands.lh[item * lh_size..(item + 1) * lh_size]; + let hh = &bands.hh[item * hh_size..(item + 1) * hh_size]; + Ok(PrequantizedHtj2k97Component { + x_rsiz: job.x_rsiz, + y_rsiz: job.y_rsiz, + resolutions: vec![ + PrequantizedHtj2k97Resolution { + subbands: vec![subband_from_codeblock_slice( + ll, + bands.low_width, + bands.low_height, + J2kSubBandType::LowLow, + options, + )?], + }, + PrequantizedHtj2k97Resolution { + subbands: vec![ + subband_from_codeblock_slice( + hl, + bands.high_width, + bands.low_height, + J2kSubBandType::HighLow, + options, + )?, + subband_from_codeblock_slice( + lh, + bands.low_width, + bands.high_height, + J2kSubBandType::LowHigh, + options, + )?, + subband_from_codeblock_slice( + hh, + bands.high_width, + bands.high_height, + J2kSubBandType::HighHigh, + options, + )?, + ], + }, + ], + }) + }) + .collect() +} + +pub(crate) fn dispatch_htj2k97_codeblock_batch( + session: &mut CudaTranscodeSession, + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, +) -> Result<(Vec, Dwt97BatchStageTimings), CudaTranscodeError> { + if !transcode_kernels_built() { + return Err(CudaTranscodeError::CudaUnavailable); + } + let (cb_width, cb_height) = validate_htj2k97_codeblock_options(options)?; + let context = session.context()?; + + let Some(first) = jobs.first() else { + return Ok((Vec::new(), Dwt97BatchStageTimings::default())); + }; + + // The fused staged kernels require uniform block geometry across the batch. + let uniform = jobs.iter().all(|job| { + job.block_cols == first.block_cols + && job.block_rows == first.block_rows + && job.width == first.width + && job.height == first.height + }); + if !uniform { + return Err(CudaTranscodeError::UnsupportedJob( + "CUDA 9/7 code-block batch requires uniform job geometry", + )); + } + + // Per-subband inverse step sizes from the shared oracle (same numbers the + // CPU oracle and Metal use), plus code-block geometry from the options. + let inv_delta = + |sub: J2kSubBandType| -> f32 { (1.0 / htj2k97_subband_delta(options, sub)) as f32 }; + let params = CudaHtj2k97QuantizeParams { + inv_delta_ll: inv_delta(J2kSubBandType::LowLow), + inv_delta_hl: inv_delta(J2kSubBandType::HighLow), + inv_delta_lh: inv_delta(J2kSubBandType::LowHigh), + inv_delta_hh: inv_delta(J2kSubBandType::HighHigh), + cb_width, + cb_height, + }; + + let mut blocks = Vec::with_capacity(jobs.len() * first.block_cols * first.block_rows * 64); + for job in jobs { + append_f64_blocks_to_f32(job.blocks, &mut blocks); + } + let (codeblock_bands, timings) = context + .j2k_transcode_htj2k97_codeblock_batch_with_pool( + &blocks, + jobs.len(), + first.block_cols, + first.block_rows, + first.width, + first.height, + params, + &session.buffer_pool(&context), + ) + .map_err(|_| CudaTranscodeError::Kernel("CUDA 9/7 code-block batch dispatch failed"))?; + + let components = codeblock_bands_to_components(&codeblock_bands, jobs, options)?; + Ok((components, map_batch_timings(timings))) +} + +pub(crate) fn dispatch_htj2k97_preencoded_batch( + session: &mut CudaTranscodeSession, + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, +) -> Result<(Vec, Dwt97BatchStageTimings), CudaTranscodeError> { + if !transcode_kernels_built() { + return Err(CudaTranscodeError::CudaUnavailable); + } + validate_htj2k97_codeblock_options(options)?; + let context = session.context()?; + + let Some(first) = jobs.first() else { + return Ok((Vec::new(), Dwt97BatchStageTimings::default())); + }; + + let uniform = jobs.iter().all(|job| { + job.block_cols == first.block_cols + && job.block_rows == first.block_rows + && job.width == first.width + && job.height == first.height + }); + if !uniform { + return Err(CudaTranscodeError::UnsupportedJob( + "CUDA 9/7 resident HT batch requires uniform job geometry", + )); + } + + let params = htj2k97_quantize_params(options)?; + let mut blocks = Vec::with_capacity(jobs.len() * first.block_cols * first.block_rows * 64); + for job in jobs { + append_f64_blocks_to_f32(job.blocks, &mut blocks); + } + let pool = session.buffer_pool(&context); + let (device_bands, cuda_timings) = context + .j2k_transcode_htj2k97_codeblock_batch_resident_with_pool( + &blocks, + jobs.len(), + first.block_cols, + first.block_rows, + first.width, + first.height, + params, + &pool, + ) + .map_err(|_| CudaTranscodeError::Kernel("CUDA 9/7 resident batch dispatch failed"))?; + let mut timings = map_batch_timings(cuda_timings); + + let resources = session.encode_resources(&context)?; + let (components, ht_timings, ht_dispatches) = device_bands_to_preencoded_components( + &context, + resources.as_ref(), + &pool, + &device_bands, + jobs, + options, + )?; + set_ht_encode_timings(&mut timings, ht_timings); + timings.ht_codeblock_dispatches = ht_dispatches; + Ok((components, timings)) +} + +fn dispatch_htj2k97_preencoded_i16_batch_with_sink<'a, 'j, R>( + session: &mut CudaTranscodeSession, + jobs: &'j [DctGridI16ToHtj2k97CodeBlockJob<'a>], + options: Htj2k97CodeBlockOptions, + empty: impl FnOnce() -> R, + sink: impl FnOnce( + &CudaContext, + &CudaHtj2kEncodeResources, + &CudaBufferPool, + &CudaHtj2k97DeviceCodeblockBands, + &'j [DctGridI16ToHtj2k97CodeBlockJob<'a>], + Htj2k97CodeBlockOptions, + ) -> Result<(R, CudaHtj2kEncodeStageTimings, usize), CudaTranscodeError>, +) -> Result<(R, Dwt97BatchStageTimings), CudaTranscodeError> { + if !transcode_kernels_built() { + return Err(CudaTranscodeError::CudaUnavailable); + } + validate_htj2k97_codeblock_options(options)?; + let context = session.context()?; + + let Some(first) = jobs.first() else { + return Ok((empty(), Dwt97BatchStageTimings::default())); + }; + + let uniform = jobs.iter().all(|job| { + job.block_cols == first.block_cols + && job.block_rows == first.block_rows + && job.width == first.width + && job.height == first.height + }); + if !uniform { + return Err(CudaTranscodeError::UnsupportedJob( + "CUDA 9/7 resident HT i16 batch requires uniform job geometry", + )); + } + + let params = htj2k97_quantize_params(options)?; + let mut blocks = Vec::with_capacity(jobs.len() * first.block_cols * first.block_rows * 64); + for job in jobs { + append_i16_blocks(job.dequantized_blocks, &mut blocks); + } + let pool = session.buffer_pool(&context); + let (device_bands, cuda_timings) = context + .j2k_transcode_htj2k97_codeblock_i16_batch_resident_with_pool( + &blocks, + jobs.len(), + first.block_cols, + first.block_rows, + first.width, + first.height, + params, + &pool, + ) + .map_err(|_| CudaTranscodeError::Kernel("CUDA 9/7 resident i16 batch dispatch failed"))?; + let mut timings = map_batch_timings(cuda_timings); + + let resources = session.encode_resources(&context)?; + let (output, ht_timings, ht_dispatches) = sink( + &context, + resources.as_ref(), + &pool, + &device_bands, + jobs, + options, + )?; + set_ht_encode_timings(&mut timings, ht_timings); + timings.ht_codeblock_dispatches = ht_dispatches; + Ok((output, timings)) +} + +pub(crate) fn dispatch_htj2k97_preencoded_i16_batch( + session: &mut CudaTranscodeSession, + jobs: &[DctGridI16ToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, +) -> Result<(Vec, Dwt97BatchStageTimings), CudaTranscodeError> { + dispatch_htj2k97_preencoded_i16_batch_with_sink( + session, + jobs, + options, + Vec::new, + |context, resources, pool, bands, jobs, options| { + device_bands_to_preencoded_components(context, resources, pool, bands, jobs, options) + }, + ) +} + +pub(crate) fn dispatch_htj2k97_compact_preencoded_i16_batch( + session: &mut CudaTranscodeSession, + jobs: &[DctGridI16ToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, +) -> Result<(PreencodedHtj2k97CompactBatch, Dwt97BatchStageTimings), CudaTranscodeError> { + dispatch_htj2k97_preencoded_i16_batch_with_sink( + session, + jobs, + options, + || PreencodedHtj2k97CompactBatch { + payload: Vec::new(), + components: Vec::new(), + }, + |context, resources, pool, bands, jobs, options| { + device_bands_to_compact_preencoded_batch(context, resources, pool, bands, jobs, options) + }, + ) +} + +#[allow(clippy::type_complexity)] +fn dispatch_htj2k97_preencoded_i16_batch_groups_with_sink<'a, 'g, 'j, C, X: Default>( + session: &mut CudaTranscodeSession, + groups: &'g [DctGridI16ToHtj2k97CodeBlockBatch<'a, 'j>], + options: Htj2k97CodeBlockOptions, + missing_group_error: &'static str, + sink: impl FnOnce( + &CudaContext, + &CudaHtj2kEncodeResources, + &CudaBufferPool, + &[ResidentDeviceGroup<'j, DctGridI16ToHtj2k97CodeBlockJob<'a>>], + Htj2k97CodeBlockOptions, + ) -> Result< + (X, Vec<(usize, Vec)>, CudaHtj2kEncodeStageTimings, usize), + CudaTranscodeError, + >, +) -> Result<(X, Vec>, Dwt97BatchStageTimings), CudaTranscodeError> { + if !transcode_kernels_built() { + return Err(CudaTranscodeError::CudaUnavailable); + } + validate_htj2k97_codeblock_options(options)?; + let context = session.context()?; + let params = htj2k97_quantize_params(options)?; + let pool = session.buffer_pool(&context); + let mut timings = Dwt97BatchStageTimings::default(); + let mut outputs = std::iter::repeat_with(|| None) + .take(groups.len()) + .collect::>>>(); + let mut device_groups = Vec::new(); + + for (group_index, group) in groups.iter().enumerate() { + let Some(first) = group.jobs.first() else { + outputs[group_index] = Some(Vec::new()); + continue; + }; + let uniform = group.jobs.iter().all(|job| { + job.block_cols == first.block_cols + && job.block_rows == first.block_rows + && job.width == first.width + && job.height == first.height + }); + if !uniform { + return Err(CudaTranscodeError::UnsupportedJob( + "CUDA grouped 9/7 resident HT i16 batches require uniform geometry inside each group", + )); + } + + let mut blocks = + Vec::with_capacity(group.jobs.len() * first.block_cols * first.block_rows * 64); + for job in group.jobs { + append_i16_blocks(job.dequantized_blocks, &mut blocks); + } + let (bands, group_timings) = context + .j2k_transcode_htj2k97_codeblock_i16_batch_resident_with_pool( + &blocks, + group.jobs.len(), + first.block_cols, + first.block_rows, + first.width, + first.height, + params, + &pool, + ) + .map_err(|_| { + CudaTranscodeError::Kernel("CUDA grouped 9/7 resident i16 batch dispatch failed") + })?; + accumulate_batch_timings(&mut timings, map_batch_timings(group_timings)); + device_groups.push(ResidentDeviceGroup { + group_index, + bands, + jobs: group.jobs, + }); + } + + let mut extra = X::default(); + if !device_groups.is_empty() { + let resources = session.encode_resources(&context)?; + let (sink_extra, encoded_groups, ht_timings, ht_dispatches) = + sink(&context, resources.as_ref(), &pool, &device_groups, options)?; + extra = sink_extra; + add_ht_encode_timings(&mut timings, ht_timings); + timings.ht_codeblock_dispatches = timings + .ht_codeblock_dispatches + .saturating_add(ht_dispatches); + for (group_index, components) in encoded_groups { + outputs[group_index] = Some(components); + } + } + + let outputs = outputs + .into_iter() + .map(|components| components.ok_or(CudaTranscodeError::Kernel(missing_group_error))) + .collect::, _>>()?; + + Ok((extra, outputs, timings)) +} + +pub(crate) fn dispatch_htj2k97_preencoded_i16_batch_groups( + session: &mut CudaTranscodeSession, + groups: &[DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], + options: Htj2k97CodeBlockOptions, +) -> Result<(Vec>, Dwt97BatchStageTimings), CudaTranscodeError> { + let ((), outputs, timings) = dispatch_htj2k97_preencoded_i16_batch_groups_with_sink( + session, + groups, + options, + "CUDA grouped 9/7 resident HT output group missing", + |context, resources, pool, device_groups, options| { + let (encoded_groups, ht_timings, ht_dispatches) = + device_band_groups_to_preencoded_components( + context, + resources, + pool, + device_groups, + options, + )?; + Ok(((), encoded_groups, ht_timings, ht_dispatches)) + }, + )?; + Ok((outputs, timings)) +} + +pub(crate) fn dispatch_htj2k97_compact_preencoded_i16_batch_groups( + session: &mut CudaTranscodeSession, + groups: &[DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], + options: Htj2k97CodeBlockOptions, +) -> Result<(PreencodedHtj2k97CompactBatchGroups, Dwt97BatchStageTimings), CudaTranscodeError> { + let (payload, groups, timings) = dispatch_htj2k97_preencoded_i16_batch_groups_with_sink( + session, + groups, + options, + "CUDA grouped 9/7 resident compact HT output group missing", + |context, resources, pool, device_groups, options| { + device_band_groups_to_compact_preencoded_components( + context, + resources, + pool, + device_groups, + options, + ) + }, + )?; + Ok(( + PreencodedHtj2k97CompactBatchGroups { payload, groups }, + timings, + )) +} + +fn htj2k97_quantize_params( + options: Htj2k97CodeBlockOptions, +) -> Result { + let (cb_width, cb_height) = validate_htj2k97_codeblock_options(options)?; + let inv_delta = + |sub: J2kSubBandType| -> f32 { (1.0 / htj2k97_subband_delta(options, sub)) as f32 }; + Ok(CudaHtj2k97QuantizeParams { + inv_delta_ll: inv_delta(J2kSubBandType::LowLow), + inv_delta_hl: inv_delta(J2kSubBandType::HighLow), + inv_delta_lh: inv_delta(J2kSubBandType::LowHigh), + inv_delta_hh: inv_delta(J2kSubBandType::HighHigh), + cb_width, + cb_height, + }) +} + +#[allow(clippy::similar_names, clippy::too_many_lines)] +fn device_bands_to_preencoded_components( + context: &CudaContext, + resources: &CudaHtj2kEncodeResources, + pool: &CudaBufferPool, + bands: &CudaHtj2k97DeviceCodeblockBands, + jobs: &[J], + options: Htj2k97CodeBlockOptions, +) -> Result< + ( + Vec, + CudaHtj2kEncodeStageTimings, + usize, + ), + CudaTranscodeError, +> { + if bands.item_count != jobs.len() { + return Err(CudaTranscodeError::Kernel( + "CUDA resident 9/7 band item count mismatch", + )); + } + + let (ll_subbands, hl_subbands, lh_subbands, hh_subbands, ht_timings, dispatches) = + encode_resident_subbands(context, resources, pool, bands, bands.item_count, options)?; + + let components = + assemble_preencoded_components(jobs, ll_subbands, hl_subbands, lh_subbands, hh_subbands)?; + + Ok((components, ht_timings, dispatches)) +} + +#[allow(clippy::similar_names, clippy::too_many_lines)] +fn device_bands_to_compact_preencoded_batch( + context: &CudaContext, + resources: &CudaHtj2kEncodeResources, + pool: &CudaBufferPool, + bands: &CudaHtj2k97DeviceCodeblockBands, + jobs: &[J], + options: Htj2k97CodeBlockOptions, +) -> Result< + ( + PreencodedHtj2k97CompactBatch, + CudaHtj2kEncodeStageTimings, + usize, + ), + CudaTranscodeError, +> { + if bands.item_count != jobs.len() { + return Err(CudaTranscodeError::Kernel( + "CUDA resident 9/7 band item count mismatch", + )); + } + + let (payload, ll_subbands, hl_subbands, lh_subbands, hh_subbands, ht_timings, dispatches) = + encode_resident_compact_subbands( + context, + resources, + pool, + bands, + bands.item_count, + options, + )?; + + let components = assemble_compact_preencoded_components( + jobs, + ll_subbands, + hl_subbands, + lh_subbands, + hh_subbands, + )?; + + Ok(( + PreencodedHtj2k97CompactBatch { + payload, + components, + }, + ht_timings, + dispatches, + )) +} + +type ResidentSubbands = ( + Vec, + Vec, + Vec, + Vec, + CudaHtj2kEncodeStageTimings, + usize, +); + +type CompactResidentSubbands = ( + Vec, + Vec, + Vec, + Vec, + Vec, + CudaHtj2kEncodeStageTimings, + usize, +); + +struct ResidentDeviceGroup<'a, J> { + group_index: usize, + bands: CudaHtj2k97DeviceCodeblockBands, + jobs: &'a [J], +} + +struct ResidentSubbandEncodePlan<'a> { + coefficients: &'a j2k_cuda_runtime::CudaDeviceBuffer, + coefficient_count: usize, + jobs: Vec, + shapes: Vec<(u32, u32)>, + sub_band_type: J2kSubBandType, + num_cbs_x: usize, + num_cbs_y: usize, + total_bitplanes: u8, +} + +struct ResidentSubbandGroupPlans<'a, J> { + group_index: usize, + jobs: &'a [J], + ll: ResidentSubbandEncodePlan<'a>, + hl: ResidentSubbandEncodePlan<'a>, + lh: ResidentSubbandEncodePlan<'a>, + hh: ResidentSubbandEncodePlan<'a>, +} + +impl<'a, J> ResidentSubbandGroupPlans<'a, J> { + fn plans(&self) -> [&ResidentSubbandEncodePlan<'a>; 4] { + [&self.ll, &self.hl, &self.lh, &self.hh] + } +} + +trait Htj2k97ComponentJob { + fn x_rsiz(&self) -> u8; + fn y_rsiz(&self) -> u8; +} + +impl Htj2k97ComponentJob for DctGridToHtj2k97CodeBlockJob<'_> { + fn x_rsiz(&self) -> u8 { + self.x_rsiz + } + + fn y_rsiz(&self) -> u8 { + self.y_rsiz + } +} + +impl Htj2k97ComponentJob for DctGridI16ToHtj2k97CodeBlockJob<'_> { + fn x_rsiz(&self) -> u8 { + self.x_rsiz + } + + fn y_rsiz(&self) -> u8 { + self.y_rsiz + } +} + +#[allow(clippy::similar_names)] +fn assemble_preencoded_components( + jobs: &[J], + ll_subbands: Vec, + hl_subbands: Vec, + lh_subbands: Vec, + hh_subbands: Vec, +) -> Result, CudaTranscodeError> { + if ll_subbands.len() != jobs.len() + || hl_subbands.len() != jobs.len() + || lh_subbands.len() != jobs.len() + || hh_subbands.len() != jobs.len() + { + return Err(CudaTranscodeError::Kernel( + "CUDA resident HTJ2K component assembly count mismatch", + )); + } + + let components = jobs + .iter() + .zip(ll_subbands) + .zip(hl_subbands) + .zip(lh_subbands) + .zip(hh_subbands) + .map(|((((job, ll), hl), lh), hh)| PreencodedHtj2k97Component { + x_rsiz: job.x_rsiz(), + y_rsiz: job.y_rsiz(), + resolutions: vec![ + PreencodedHtj2k97Resolution { subbands: vec![ll] }, + PreencodedHtj2k97Resolution { + subbands: vec![hl, lh, hh], + }, + ], + }) + .collect(); + + Ok(components) +} + +#[allow(clippy::similar_names)] +fn assemble_compact_preencoded_components( + jobs: &[J], + ll_subbands: Vec, + hl_subbands: Vec, + lh_subbands: Vec, + hh_subbands: Vec, +) -> Result, CudaTranscodeError> { + if ll_subbands.len() != jobs.len() + || hl_subbands.len() != jobs.len() + || lh_subbands.len() != jobs.len() + || hh_subbands.len() != jobs.len() + { + return Err(CudaTranscodeError::Kernel( + "CUDA resident HTJ2K compact component assembly count mismatch", + )); + } + + let components = jobs + .iter() + .zip(ll_subbands) + .zip(hl_subbands) + .zip(lh_subbands) + .zip(hh_subbands) + .map( + |((((job, ll), hl), lh), hh)| PreencodedHtj2k97CompactComponent { + x_rsiz: job.x_rsiz(), + y_rsiz: job.y_rsiz(), + resolutions: vec![ + PreencodedHtj2k97CompactResolution { subbands: vec![ll] }, + PreencodedHtj2k97CompactResolution { + subbands: vec![hl, lh, hh], + }, + ], + }, + ) + .collect(); + + Ok(components) +} + +fn encode_resident_subbands( + context: &CudaContext, + resources: &CudaHtj2kEncodeResources, + pool: &CudaBufferPool, + bands: &CudaHtj2k97DeviceCodeblockBands, + item_count: usize, + options: Htj2k97CodeBlockOptions, +) -> Result { + let plans = [ + resident_subband_encode_plan( + &bands.ll, + item_count, + bands.low_width, + bands.low_height, + J2kSubBandType::LowLow, + options, + )?, + resident_subband_encode_plan( + &bands.hl, + item_count, + bands.high_width, + bands.low_height, + J2kSubBandType::HighLow, + options, + )?, + resident_subband_encode_plan( + &bands.lh, + item_count, + bands.low_width, + bands.high_height, + J2kSubBandType::LowHigh, + options, + )?, + resident_subband_encode_plan( + &bands.hh, + item_count, + bands.high_width, + bands.high_height, + J2kSubBandType::HighHigh, + options, + )?, + ]; + let targets: Vec<_> = plans + .iter() + .filter(|plan| !plan.jobs.is_empty()) + .map(|plan| CudaHtj2kEncodeResidentTarget { + coefficients: plan.coefficients, + coefficient_count: plan.coefficient_count, + jobs: &plan.jobs, + }) + .collect(); + let encoded = context + .encode_htj2k_codeblocks_multi_resident_with_resources_and_pool(&targets, resources, pool) + .map_err(|_| CudaTranscodeError::Kernel("CUDA resident multi-input HTJ2K encode failed"))?; + let expected_blocks = plans.iter().map(|plan| plan.jobs.len()).sum::(); + if encoded.code_blocks().len() != expected_blocks { + return Err(CudaTranscodeError::Kernel( + "CUDA resident multi-input HTJ2K encode returned wrong block count", + )); + } + let ht_timings = encoded.stage_timings(); + let dispatches = encoded.execution().kernel_dispatches(); + let mut encoded_blocks = encoded.into_code_blocks().into_iter(); + + let ll = split_resident_subband_blocks(&plans[0], item_count, &mut encoded_blocks)?; + let hl = split_resident_subband_blocks(&plans[1], item_count, &mut encoded_blocks)?; + let lh = split_resident_subband_blocks(&plans[2], item_count, &mut encoded_blocks)?; + let hh = split_resident_subband_blocks(&plans[3], item_count, &mut encoded_blocks)?; + if encoded_blocks.next().is_some() { + return Err(CudaTranscodeError::Kernel( + "CUDA resident multi-input HTJ2K output count mismatch", + )); + } + + Ok((ll, hl, lh, hh, ht_timings, dispatches)) +} + +fn encode_resident_compact_subbands( + context: &CudaContext, + resources: &CudaHtj2kEncodeResources, + pool: &CudaBufferPool, + bands: &CudaHtj2k97DeviceCodeblockBands, + item_count: usize, + options: Htj2k97CodeBlockOptions, +) -> Result { + let plans = [ + resident_subband_encode_plan( + &bands.ll, + item_count, + bands.low_width, + bands.low_height, + J2kSubBandType::LowLow, + options, + )?, + resident_subband_encode_plan( + &bands.hl, + item_count, + bands.high_width, + bands.low_height, + J2kSubBandType::HighLow, + options, + )?, + resident_subband_encode_plan( + &bands.lh, + item_count, + bands.low_width, + bands.high_height, + J2kSubBandType::LowHigh, + options, + )?, + resident_subband_encode_plan( + &bands.hh, + item_count, + bands.high_width, + bands.high_height, + J2kSubBandType::HighHigh, + options, + )?, + ]; + let targets: Vec<_> = plans + .iter() + .filter(|plan| !plan.jobs.is_empty()) + .map(|plan| CudaHtj2kEncodeResidentTarget { + coefficients: plan.coefficients, + coefficient_count: plan.coefficient_count, + jobs: &plan.jobs, + }) + .collect(); + let encoded = context + .encode_htj2k_codeblocks_multi_resident_compact_with_resources_and_pool( + &targets, resources, pool, + ) + .map_err(|_| { + CudaTranscodeError::Kernel("CUDA resident compact multi-input HTJ2K encode failed") + })?; + let expected_blocks = plans.iter().map(|plan| plan.jobs.len()).sum::(); + if encoded.code_blocks().len() != expected_blocks { + return Err(CudaTranscodeError::Kernel( + "CUDA resident compact multi-input HTJ2K encode returned wrong block count", + )); + } + let ht_timings = encoded.stage_timings(); + let dispatches = encoded.execution().kernel_dispatches(); + let (payload, encoded_blocks) = encoded.into_payload_and_code_blocks(); + let mut encoded_blocks = encoded_blocks.into_iter(); + + let ll = split_resident_compact_subband_blocks(&plans[0], item_count, &mut encoded_blocks)?; + let hl = split_resident_compact_subband_blocks(&plans[1], item_count, &mut encoded_blocks)?; + let lh = split_resident_compact_subband_blocks(&plans[2], item_count, &mut encoded_blocks)?; + let hh = split_resident_compact_subband_blocks(&plans[3], item_count, &mut encoded_blocks)?; + if encoded_blocks.next().is_some() { + return Err(CudaTranscodeError::Kernel( + "CUDA resident compact multi-input HTJ2K output count mismatch", + )); + } + + Ok((payload, ll, hl, lh, hh, ht_timings, dispatches)) +} + +#[allow(clippy::similar_names)] +fn device_band_groups_to_preencoded_components( + context: &CudaContext, + resources: &CudaHtj2kEncodeResources, + pool: &CudaBufferPool, + groups: &[ResidentDeviceGroup<'_, J>], + options: Htj2k97CodeBlockOptions, +) -> Result { + let group_plans = groups + .iter() + .map(|group| { + if group.bands.item_count != group.jobs.len() { + return Err(CudaTranscodeError::Kernel( + "CUDA grouped resident 9/7 band item count mismatch", + )); + } + Ok(ResidentSubbandGroupPlans { + group_index: group.group_index, + jobs: group.jobs, + ll: resident_subband_encode_plan( + &group.bands.ll, + group.bands.item_count, + group.bands.low_width, + group.bands.low_height, + J2kSubBandType::LowLow, + options, + )?, + hl: resident_subband_encode_plan( + &group.bands.hl, + group.bands.item_count, + group.bands.high_width, + group.bands.low_height, + J2kSubBandType::HighLow, + options, + )?, + lh: resident_subband_encode_plan( + &group.bands.lh, + group.bands.item_count, + group.bands.low_width, + group.bands.high_height, + J2kSubBandType::LowHigh, + options, + )?, + hh: resident_subband_encode_plan( + &group.bands.hh, + group.bands.item_count, + group.bands.high_width, + group.bands.high_height, + J2kSubBandType::HighHigh, + options, + )?, + }) + }) + .collect::, CudaTranscodeError>>()?; + + let targets = group_plans + .iter() + .flat_map(ResidentSubbandGroupPlans::plans) + .filter(|plan| !plan.jobs.is_empty()) + .map(|plan| CudaHtj2kEncodeResidentTarget { + coefficients: plan.coefficients, + coefficient_count: plan.coefficient_count, + jobs: &plan.jobs, + }) + .collect::>(); + let encoded = context + .encode_htj2k_codeblocks_multi_resident_with_resources_and_pool(&targets, resources, pool) + .map_err(|_| { + CudaTranscodeError::Kernel("CUDA grouped resident multi-input HTJ2K encode failed") + })?; + let expected_blocks = group_plans + .iter() + .flat_map(ResidentSubbandGroupPlans::plans) + .map(|plan| plan.jobs.len()) + .sum::(); + if encoded.code_blocks().len() != expected_blocks { + return Err(CudaTranscodeError::Kernel( + "CUDA grouped resident multi-input HTJ2K encode returned wrong block count", + )); + } + let ht_timings = encoded.stage_timings(); + let dispatches = encoded.execution().kernel_dispatches(); + let mut encoded_blocks = encoded.into_code_blocks().into_iter(); + let mut outputs = Vec::with_capacity(group_plans.len()); + + for group in &group_plans { + let item_count = group.jobs.len(); + let ll = split_resident_subband_blocks(&group.ll, item_count, &mut encoded_blocks)?; + let hl = split_resident_subband_blocks(&group.hl, item_count, &mut encoded_blocks)?; + let lh = split_resident_subband_blocks(&group.lh, item_count, &mut encoded_blocks)?; + let hh = split_resident_subband_blocks(&group.hh, item_count, &mut encoded_blocks)?; + let components = assemble_preencoded_components(group.jobs, ll, hl, lh, hh)?; + outputs.push((group.group_index, components)); + } + if encoded_blocks.next().is_some() { + return Err(CudaTranscodeError::Kernel( + "CUDA grouped resident multi-input HTJ2K output count mismatch", + )); + } + + Ok((outputs, ht_timings, dispatches)) +} + +#[allow(clippy::similar_names)] +fn device_band_groups_to_compact_preencoded_components( + context: &CudaContext, + resources: &CudaHtj2kEncodeResources, + pool: &CudaBufferPool, + groups: &[ResidentDeviceGroup<'_, J>], + options: Htj2k97CodeBlockOptions, +) -> Result { + let group_plans = groups + .iter() + .map(|group| { + if group.bands.item_count != group.jobs.len() { + return Err(CudaTranscodeError::Kernel( + "CUDA grouped resident 9/7 band item count mismatch", + )); + } + Ok(ResidentSubbandGroupPlans { + group_index: group.group_index, + jobs: group.jobs, + ll: resident_subband_encode_plan( + &group.bands.ll, + group.bands.item_count, + group.bands.low_width, + group.bands.low_height, + J2kSubBandType::LowLow, + options, + )?, + hl: resident_subband_encode_plan( + &group.bands.hl, + group.bands.item_count, + group.bands.high_width, + group.bands.low_height, + J2kSubBandType::HighLow, + options, + )?, + lh: resident_subband_encode_plan( + &group.bands.lh, + group.bands.item_count, + group.bands.low_width, + group.bands.high_height, + J2kSubBandType::LowHigh, + options, + )?, + hh: resident_subband_encode_plan( + &group.bands.hh, + group.bands.item_count, + group.bands.high_width, + group.bands.high_height, + J2kSubBandType::HighHigh, + options, + )?, + }) + }) + .collect::, CudaTranscodeError>>()?; + + let targets = group_plans + .iter() + .flat_map(ResidentSubbandGroupPlans::plans) + .filter(|plan| !plan.jobs.is_empty()) + .map(|plan| CudaHtj2kEncodeResidentTarget { + coefficients: plan.coefficients, + coefficient_count: plan.coefficient_count, + jobs: &plan.jobs, + }) + .collect::>(); + let encoded = context + .encode_htj2k_codeblocks_multi_resident_compact_with_resources_and_pool( + &targets, resources, pool, + ) + .map_err(|_| { + CudaTranscodeError::Kernel( + "CUDA grouped resident compact multi-input HTJ2K encode failed", + ) + })?; + let expected_blocks = group_plans + .iter() + .flat_map(ResidentSubbandGroupPlans::plans) + .map(|plan| plan.jobs.len()) + .sum::(); + if encoded.code_blocks().len() != expected_blocks { + return Err(CudaTranscodeError::Kernel( + "CUDA grouped resident compact multi-input HTJ2K encode returned wrong block count", + )); + } + let ht_timings = encoded.stage_timings(); + let dispatches = encoded.execution().kernel_dispatches(); + let (payload, encoded_blocks) = encoded.into_payload_and_code_blocks(); + let mut encoded_blocks = encoded_blocks.into_iter(); + let mut outputs = Vec::with_capacity(group_plans.len()); + + for group in &group_plans { + let item_count = group.jobs.len(); + let ll = split_resident_compact_subband_blocks(&group.ll, item_count, &mut encoded_blocks)?; + let hl = split_resident_compact_subband_blocks(&group.hl, item_count, &mut encoded_blocks)?; + let lh = split_resident_compact_subband_blocks(&group.lh, item_count, &mut encoded_blocks)?; + let hh = split_resident_compact_subband_blocks(&group.hh, item_count, &mut encoded_blocks)?; + let components = assemble_compact_preencoded_components(group.jobs, ll, hl, lh, hh)?; + outputs.push((group.group_index, components)); + } + if encoded_blocks.next().is_some() { + return Err(CudaTranscodeError::Kernel( + "CUDA grouped resident compact multi-input HTJ2K output count mismatch", + )); + } + + Ok((payload, outputs, ht_timings, dispatches)) +} + +#[allow(clippy::too_many_arguments)] +fn resident_subband_encode_plan( + coefficients: &CudaPooledDeviceBuffer, + item_count: usize, + width: usize, + height: usize, + sub_band_type: J2kSubBandType, + options: Htj2k97CodeBlockOptions, +) -> Result, CudaTranscodeError> { + let coefficient_buffer = coefficients + .as_device_buffer() + .ok_or(CudaTranscodeError::Kernel( + "CUDA resident 9/7 pooled band checkout missing", + ))?; + let cb_width = htj2k97_code_block_dim(options.code_block_width_exp)?; + let cb_height = htj2k97_code_block_dim(options.code_block_height_exp)?; + let num_cbs_x = if width == 0 { + 0 + } else { + width.div_ceil(cb_width) + }; + let num_cbs_y = if height == 0 { + 0 + } else { + height.div_ceil(cb_height) + }; + let total_bitplanes = htj2k97_subband_total_bitplanes(options, sub_band_type); + if width == 0 || height == 0 { + return Ok(ResidentSubbandEncodePlan { + coefficients: coefficient_buffer, + coefficient_count: 0, + jobs: Vec::new(), + shapes: Vec::new(), + sub_band_type, + num_cbs_x: 0, + num_cbs_y: 0, + total_bitplanes, + }); + } + + let item_stride = width.checked_mul(height).ok_or(CudaTranscodeError::Kernel( + "CUDA resident 9/7 band dimensions overflow", + ))?; + let coefficient_count = + item_stride + .checked_mul(item_count) + .ok_or(CudaTranscodeError::Kernel( + "CUDA resident 9/7 band item count overflow", + ))?; + let mut encode_jobs = Vec::with_capacity(item_count * num_cbs_x * num_cbs_y); + let mut shapes = Vec::with_capacity(item_count * num_cbs_x * num_cbs_y); + for item in 0..item_count { + let item_offset = item + .checked_mul(item_stride) + .ok_or(CudaTranscodeError::Kernel( + "CUDA resident 9/7 band item offset overflow", + ))?; + for cby in 0..num_cbs_y { + for cbx in 0..num_cbs_x { + let block_width = (width - cbx * cb_width).min(cb_width); + let block_height = (height - cby * cb_height).min(cb_height); + let block_offset = cby + .checked_mul(cb_height) + .and_then(|value| value.checked_mul(width)) + .and_then(|value| { + value.checked_add(cbx.checked_mul(cb_width)?.checked_mul(block_height)?) + }) + .and_then(|value| value.checked_add(item_offset)) + .ok_or(CudaTranscodeError::Kernel( + "CUDA resident 9/7 code-block offset overflow", + ))?; + encode_jobs.push(CudaHtj2kEncodeCodeBlockJob { + coefficient_offset: to_u32(block_offset)?, + width: to_u32(block_width)?, + height: to_u32(block_height)?, + total_bitplanes, + target_coding_passes: 1, + }); + shapes.push((to_u32(block_width)?, to_u32(block_height)?)); + } + } + } + + Ok(ResidentSubbandEncodePlan { + coefficients: coefficient_buffer, + coefficient_count, + jobs: encode_jobs, + shapes, + sub_band_type, + num_cbs_x, + num_cbs_y, + total_bitplanes, + }) +} + +fn split_resident_subband_blocks( + plan: &ResidentSubbandEncodePlan<'_>, + item_count: usize, + encoded_blocks: &mut impl Iterator, +) -> Result, CudaTranscodeError> { + let blocks_per_item = + plan.num_cbs_x + .checked_mul(plan.num_cbs_y) + .ok_or(CudaTranscodeError::Kernel( + "CUDA resident HTJ2K code-block count overflow", + ))?; + let mut shape_index = 0usize; + let mut subbands = Vec::with_capacity(item_count); + for _ in 0..item_count { + let mut code_blocks = Vec::with_capacity(blocks_per_item); + for _ in 0..blocks_per_item { + let (width, height) = + *plan + .shapes + .get(shape_index) + .ok_or(CudaTranscodeError::Kernel( + "CUDA resident HTJ2K shape count mismatch", + ))?; + shape_index = shape_index.saturating_add(1); + let encoded = encoded_blocks.next().ok_or(CudaTranscodeError::Kernel( + "CUDA resident HTJ2K output count mismatch", + ))?; + let (data, cleanup_length, refinement_length, num_coding_passes, num_zero_bitplanes) = + encoded.into_parts(); + code_blocks.push(PreencodedHtj2k97CodeBlock { + width, + height, + encoded: EncodedHtJ2kCodeBlock { + data, + cleanup_length, + refinement_length, + num_coding_passes, + num_zero_bitplanes, + }, + }); + } + subbands.push(PreencodedHtj2k97Subband { + sub_band_type: plan.sub_band_type, + num_cbs_x: to_u32(plan.num_cbs_x)?, + num_cbs_y: to_u32(plan.num_cbs_y)?, + total_bitplanes: plan.total_bitplanes, + code_blocks, + }); + } + if shape_index != plan.shapes.len() { + return Err(CudaTranscodeError::Kernel( + "CUDA resident HTJ2K shape count mismatch", + )); + } + Ok(subbands) +} + +fn split_resident_compact_subband_blocks( + plan: &ResidentSubbandEncodePlan<'_>, + item_count: usize, + encoded_blocks: &mut impl Iterator, +) -> Result, CudaTranscodeError> { + let blocks_per_item = + plan.num_cbs_x + .checked_mul(plan.num_cbs_y) + .ok_or(CudaTranscodeError::Kernel( + "CUDA resident HTJ2K compact code-block count overflow", + ))?; + let mut shape_index = 0usize; + let mut subbands = Vec::with_capacity(item_count); + for _ in 0..item_count { + let mut code_blocks = Vec::with_capacity(blocks_per_item); + for _ in 0..blocks_per_item { + let (width, height) = + *plan + .shapes + .get(shape_index) + .ok_or(CudaTranscodeError::Kernel( + "CUDA resident HTJ2K compact shape count mismatch", + ))?; + shape_index = shape_index.saturating_add(1); + let encoded = encoded_blocks.next().ok_or(CudaTranscodeError::Kernel( + "CUDA resident HTJ2K compact output count mismatch", + ))?; + let ( + payload_range, + cleanup_length, + refinement_length, + num_coding_passes, + num_zero_bitplanes, + ) = encoded.into_parts(); + code_blocks.push(PreencodedHtj2k97CompactCodeBlock { + width, + height, + payload_range, + cleanup_length, + refinement_length, + num_coding_passes, + num_zero_bitplanes, + }); + } + subbands.push(PreencodedHtj2k97CompactSubband { + sub_band_type: plan.sub_band_type, + num_cbs_x: to_u32(plan.num_cbs_x)?, + num_cbs_y: to_u32(plan.num_cbs_y)?, + total_bitplanes: plan.total_bitplanes, + code_blocks, + }); + } + if shape_index != plan.shapes.len() { + return Err(CudaTranscodeError::Kernel( + "CUDA resident HTJ2K compact shape count mismatch", + )); + } + Ok(subbands) +} + +fn to_u32(value: usize) -> Result { + u32::try_from(value).map_err(|_| CudaTranscodeError::Kernel("CUDA value exceeds u32")) +} + +fn cuda_htj2k_encode_tables() -> CudaHtj2kEncodeTables<'static> { + CudaHtj2kEncodeTables { + vlc_table0: j2k_native::ht_vlc_encode_table0(), + vlc_table1: j2k_native::ht_vlc_encode_table1(), + uvlc_table: j2k_native::ht_uvlc_encode_table_bytes(), + } +} + +fn validate_band_len( + band: &[i32], + item_count: usize, + item_size: usize, +) -> Result<(), CudaTranscodeError> { + let expected = item_count + .checked_mul(item_size) + .ok_or(CudaTranscodeError::Kernel( + "CUDA 9/7 code-block band length overflow", + ))?; + if band.len() != expected { + return Err(CudaTranscodeError::Kernel( + "CUDA 9/7 code-block band output length mismatch", + )); + } + Ok(()) +} + +fn validate_htj2k97_codeblock_options( + options: Htj2k97CodeBlockOptions, +) -> Result<(usize, usize), CudaTranscodeError> { + j2k_transcode::htj2k97_codeblock_oracle::validate_htj2k97_codeblock_options(options) + .map_err(CudaTranscodeError::UnsupportedJob) +} + +fn htj2k97_code_block_dim(exp_minus_two: u8) -> Result { + 1usize + .checked_shl(u32::from(exp_minus_two) + 2) + .ok_or(CudaTranscodeError::UnsupportedJob( + "CUDA 9/7 code-block exponent is too large", + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + struct TestComponentJob { + x_rsiz: u8, + y_rsiz: u8, + } + + impl Htj2k97ComponentJob for TestComponentJob { + fn x_rsiz(&self) -> u8 { + self.x_rsiz + } + + fn y_rsiz(&self) -> u8 { + self.y_rsiz + } + } + + fn test_subband(sub_band_type: J2kSubBandType, marker: u8) -> PreencodedHtj2k97Subband { + PreencodedHtj2k97Subband { + sub_band_type, + num_cbs_x: 1, + num_cbs_y: 1, + total_bitplanes: 8, + code_blocks: vec![PreencodedHtj2k97CodeBlock { + width: 1, + height: 1, + encoded: EncodedHtJ2kCodeBlock { + data: vec![marker; 8], + cleanup_length: 8, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 0, + }, + }], + } + } + + fn payload_ptr(subband: &PreencodedHtj2k97Subband) -> usize { + subband.code_blocks[0].encoded.data.as_ptr() as usize + } + + #[test] + #[allow(clippy::similar_names)] + fn assemble_preencoded_components_moves_subband_payloads_without_clone() { + let jobs = [TestComponentJob { + x_rsiz: 1, + y_rsiz: 2, + }]; + let ll = vec![test_subband(J2kSubBandType::LowLow, 1)]; + let hl = vec![test_subband(J2kSubBandType::HighLow, 2)]; + let lh = vec![test_subband(J2kSubBandType::LowHigh, 3)]; + let hh = vec![test_subband(J2kSubBandType::HighHigh, 4)]; + let ll_ptr = payload_ptr(&ll[0]); + let hl_ptr = payload_ptr(&hl[0]); + let lh_ptr = payload_ptr(&lh[0]); + let hh_ptr = payload_ptr(&hh[0]); + + let components = assemble_preencoded_components(&jobs, ll, hl, lh, hh).expect("components"); + + assert_eq!(components.len(), 1); + assert_eq!(components[0].x_rsiz, 1); + assert_eq!(components[0].y_rsiz, 2); + assert_eq!( + payload_ptr(&components[0].resolutions[0].subbands[0]), + ll_ptr + ); + assert_eq!( + payload_ptr(&components[0].resolutions[1].subbands[0]), + hl_ptr + ); + assert_eq!( + payload_ptr(&components[0].resolutions[1].subbands[1]), + lh_ptr + ); + assert_eq!( + payload_ptr(&components[0].resolutions[1].subbands[2]), + hh_ptr + ); + } + + #[test] + fn append_i16_blocks_preserves_prefix_and_flattens_blocks() { + let mut first = [0i16; 64]; + first[0] = -7; + first[63] = 42; + let mut second = [0i16; 64]; + second[1] = 9; + second[62] = -11; + let mut out = vec![123]; + + append_i16_blocks(&[first, second], &mut out); + + assert_eq!(out[0], 123); + assert_eq!(out.len(), 1 + 128); + assert_eq!(out[1], -7); + assert_eq!(out[64], 42); + assert_eq!(out[66], 9); + assert_eq!(out[127], -11); + } +} diff --git a/crates/j2k-transcode-cuda/src/lib.rs b/crates/j2k-transcode-cuda/src/lib.rs new file mode 100644 index 00000000..2609b895 --- /dev/null +++ b/crates/j2k-transcode-cuda/src/lib.rs @@ -0,0 +1,981 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! CUDA acceleration for coefficient-domain JPEG to HTJ2K transcode stages. +//! +//! Mirrors `j2k-transcode-metal`: it implements +//! [`DctToWaveletStageAccelerator`] for direct DCT-grid to one-level 5/3 and 9/7 +//! wavelet projections (and the fused 9/7 HTJ2K code-block path), so JPEG can be +//! transcoded to HTJ2K without an IDCT->pixels->DWT spatial round-trip. The CPU +//! scalar code in `j2k-transcode` remains the oracle and fallback; this +//! crate never reimplements it. +//! +//! The actual GPU kernels live in `j2k-cuda-runtime` (the repo keeps all +//! `.cu` + `build.rs` PTX there). The GPU path is gated behind the +//! `cuda-runtime` feature; without it this accelerator behaves like Metal's +//! non-macOS path (Explicit -> typed `Err`, Auto -> `Ok(None)` scalar fallback). + +#[cfg(feature = "cuda-runtime")] +mod cuda; + +use core::fmt; + +use j2k_transcode::accelerator::{ + DctGridI16ToHtj2k97CodeBlockBatch, DctGridI16ToHtj2k97CodeBlockJob, DctGridToDwt53Job, + DctGridToDwt97Job, DctGridToHtj2k97CodeBlockJob, DctGridToReversibleDwt53Job, + DctToWaveletStageAccelerator, Dwt97BatchStageTimings, Htj2k97CodeBlockOptions, + PreencodedHtj2k97CompactBatch, PreencodedHtj2k97CompactBatchGroups, PreencodedHtj2k97Component, + PrequantizedHtj2k97Component, ReversibleDwt53FirstLevel, TranscodeStageError, +}; +use j2k_transcode::dct53_2d::Dwt53TwoDimensional; +use j2k_transcode::dct97_2d::Dwt97TwoDimensional; + +/// Stable message returned when the CUDA runtime is unavailable (feature not +/// compiled, no device, or the transcode kernels were not built). +pub const CUDA_UNAVAILABLE: &str = "CUDA is unavailable on this host"; + +/// Default minimum component sample count before Auto mode offers a job to CUDA. +const DEFAULT_AUTO_MIN_SAMPLES: usize = 224 * 224; +const DEFAULT_AUTO_REVERSIBLE_BATCH_MIN_JOBS: usize = 32; +const DEFAULT_AUTO_REVERSIBLE_BATCH_MIN_SAMPLES: usize = 224 * 224 * 32; +const DEFAULT_AUTO_DWT97_BATCH_MIN_JOBS: usize = 32; +const DEFAULT_AUTO_DWT97_BATCH_MIN_SAMPLES: usize = 224 * 224 * 32; +const DISABLE_COMPACT_PREENCODED_ENV: &str = "J2K_CUDA_DISABLE_COMPACT_PREENCODED"; + +/// Error returned by the CUDA transcode accelerator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CudaTranscodeError { + /// CUDA is unavailable on this host or the kernels were not built. + CudaUnavailable, + /// The request is outside the current CUDA implementation. + UnsupportedJob(&'static str), + /// CUDA runtime or kernel execution failed. + Kernel(&'static str), +} + +impl CudaTranscodeError { + /// Whether Auto mode may recover from this error by using the scalar + /// fallback (`Ok(None)`). Hard kernel failures propagate as `Err`. + #[cfg(feature = "cuda-runtime")] + const fn is_recoverable(self) -> bool { + matches!(self, Self::CudaUnavailable | Self::UnsupportedJob(_)) + } +} + +impl fmt::Display for CudaTranscodeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::CudaUnavailable => f.write_str(CUDA_UNAVAILABLE), + Self::UnsupportedJob(reason) | Self::Kernel(reason) => f.write_str(reason), + } + } +} + +impl From for TranscodeStageError { + fn from(error: CudaTranscodeError) -> Self { + match error { + CudaTranscodeError::CudaUnavailable => Self::DeviceUnavailable, + CudaTranscodeError::UnsupportedJob(reason) => Self::Unsupported(reason), + CudaTranscodeError::Kernel(reason) => Self::Backend(reason.to_string()), + } + } +} + +impl std::error::Error for CudaTranscodeError {} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum CudaDispatchMode { + /// Treat an unavailable/unsupported CUDA dispatch as an error. + Explicit, + /// Fall back to the scalar oracle (`Ok(None)`) for small or unsupported + /// jobs. + Auto, +} + +/// Optional CUDA accelerator for `j2k-transcode` transform stages. +#[derive(Debug, Clone)] +pub struct CudaDctToWaveletStageAccelerator { + mode: CudaDispatchMode, + min_auto_samples: usize, + min_auto_reversible_batch_jobs: usize, + min_auto_reversible_batch_samples: usize, + min_auto_dwt97_batch_jobs: usize, + min_auto_dwt97_batch_samples: usize, + reversible_dwt53_attempts: usize, + reversible_dwt53_dispatches: usize, + reversible_dwt53_batch_attempts: usize, + reversible_dwt53_batch_dispatches: usize, + dwt53_attempts: usize, + dwt53_dispatches: usize, + dwt97_attempts: usize, + dwt97_dispatches: usize, + dwt97_batch_attempts: usize, + dwt97_batch_dispatches: usize, + htj2k97_codeblock_batch_attempts: usize, + htj2k97_codeblock_batch_dispatches: usize, + last_dwt97_batch_stage_timings: Option, + resident_ht_encode: bool, + #[cfg(feature = "cuda-runtime")] + session: Option, +} + +impl CudaDctToWaveletStageAccelerator { + /// Create an accelerator that treats unavailable/unsupported CUDA dispatch + /// as an error (no silent scalar fallback). + #[must_use] + pub const fn new_explicit() -> Self { + Self::with_mode(CudaDispatchMode::Explicit, 0) + } + + /// Create an explicit accelerator that keeps 9/7 code-block coefficients + /// resident and HT-encodes them on the same CUDA context before CPU + /// packetization. + #[must_use] + pub const fn new_explicit_resident_ht_encode() -> Self { + let mut accelerator = Self::with_mode(CudaDispatchMode::Explicit, 0); + accelerator.resident_ht_encode = true; + accelerator + } + + /// Create an accelerator that falls back to the scalar oracle for small or + /// unsupported jobs. + #[must_use] + pub const fn for_auto() -> Self { + let mut accelerator = Self::with_mode(CudaDispatchMode::Auto, DEFAULT_AUTO_MIN_SAMPLES); + accelerator.min_auto_reversible_batch_jobs = DEFAULT_AUTO_REVERSIBLE_BATCH_MIN_JOBS; + accelerator.min_auto_reversible_batch_samples = DEFAULT_AUTO_REVERSIBLE_BATCH_MIN_SAMPLES; + accelerator.min_auto_dwt97_batch_jobs = DEFAULT_AUTO_DWT97_BATCH_MIN_JOBS; + accelerator.min_auto_dwt97_batch_samples = DEFAULT_AUTO_DWT97_BATCH_MIN_SAMPLES; + accelerator + } + + const fn with_mode(mode: CudaDispatchMode, min_auto_samples: usize) -> Self { + Self { + mode, + min_auto_samples, + min_auto_reversible_batch_jobs: 0, + min_auto_reversible_batch_samples: 0, + min_auto_dwt97_batch_jobs: 0, + min_auto_dwt97_batch_samples: 0, + reversible_dwt53_attempts: 0, + reversible_dwt53_dispatches: 0, + reversible_dwt53_batch_attempts: 0, + reversible_dwt53_batch_dispatches: 0, + dwt53_attempts: 0, + dwt53_dispatches: 0, + dwt97_attempts: 0, + dwt97_dispatches: 0, + dwt97_batch_attempts: 0, + dwt97_batch_dispatches: 0, + htj2k97_codeblock_batch_attempts: 0, + htj2k97_codeblock_batch_dispatches: 0, + last_dwt97_batch_stage_timings: None, + resident_ht_encode: false, + #[cfg(feature = "cuda-runtime")] + session: None, + } + } + + #[cfg(feature = "cuda-runtime")] + fn cuda_session(&mut self) -> &mut cuda::CudaTranscodeSession { + self.session + .get_or_insert_with(cuda::CudaTranscodeSession::default) + } + + /// Override the reversible 5/3 batch thresholds used before Auto mode + /// dispatches a batch to CUDA. + #[must_use] + pub const fn with_auto_reversible_batch_thresholds( + mut self, + min_jobs: usize, + min_samples: usize, + ) -> Self { + self.min_auto_reversible_batch_jobs = min_jobs; + self.min_auto_reversible_batch_samples = min_samples; + self + } + + /// Override the 9/7 batch thresholds used before Auto mode dispatches a + /// same-geometry batch to CUDA. + #[must_use] + pub const fn with_auto_dwt97_batch_thresholds( + mut self, + min_jobs: usize, + min_samples: usize, + ) -> Self { + self.min_auto_dwt97_batch_jobs = min_jobs; + self.min_auto_dwt97_batch_samples = min_samples; + self + } + + /// Number of reversible 5/3 jobs offered to this accelerator. + #[must_use] + pub const fn reversible_dwt53_attempts(&self) -> usize { + self.reversible_dwt53_attempts + } + + /// Number of reversible 5/3 jobs handled on the GPU. + #[must_use] + pub const fn reversible_dwt53_dispatches(&self) -> usize { + self.reversible_dwt53_dispatches + } + + /// Number of reversible 5/3 batches offered to this accelerator. + #[must_use] + pub const fn reversible_dwt53_batch_attempts(&self) -> usize { + self.reversible_dwt53_batch_attempts + } + + /// Number of reversible 5/3 batches handled on the GPU. + #[must_use] + pub const fn reversible_dwt53_batch_dispatches(&self) -> usize { + self.reversible_dwt53_batch_dispatches + } + + /// Number of float 5/3 jobs offered to this accelerator. + #[must_use] + pub const fn dwt53_attempts(&self) -> usize { + self.dwt53_attempts + } + + /// Number of float 5/3 jobs handled on the GPU. + #[must_use] + pub const fn dwt53_dispatches(&self) -> usize { + self.dwt53_dispatches + } + + /// Number of 9/7 jobs offered to this accelerator. + #[must_use] + pub const fn dwt97_attempts(&self) -> usize { + self.dwt97_attempts + } + + /// Number of 9/7 jobs handled on the GPU. + #[must_use] + pub const fn dwt97_dispatches(&self) -> usize { + self.dwt97_dispatches + } + + /// Number of 9/7 batches offered to this accelerator. + #[must_use] + pub const fn dwt97_batch_attempts(&self) -> usize { + self.dwt97_batch_attempts + } + + /// Number of 9/7 batches handled on the GPU. + #[must_use] + pub const fn dwt97_batch_dispatches(&self) -> usize { + self.dwt97_batch_dispatches + } + + /// Number of prequantized 9/7 HTJ2K code-block batches offered. + #[must_use] + pub const fn htj2k97_codeblock_batch_attempts(&self) -> usize { + self.htj2k97_codeblock_batch_attempts + } + + /// Number of prequantized 9/7 HTJ2K code-block batches handled on the GPU. + #[must_use] + pub const fn htj2k97_codeblock_batch_dispatches(&self) -> usize { + self.htj2k97_codeblock_batch_dispatches + } + + /// Outcome for a job that CUDA cannot serve, resolved by dispatch mode. + #[cfg(not(feature = "cuda-runtime"))] + fn unavailable(&self) -> Result, TranscodeStageError> { + match self.mode { + CudaDispatchMode::Explicit => Err(TranscodeStageError::DeviceUnavailable), + CudaDispatchMode::Auto => Ok(None), + } + } + + /// Map a CUDA dispatch error to the trait outcome for the current mode: + /// Auto recovers from recoverable errors with `Ok(None)`; Explicit and hard + /// kernel failures propagate as `Err`. + #[cfg(feature = "cuda-runtime")] + fn recover(&self, error: CudaTranscodeError) -> Result, TranscodeStageError> { + if self.mode == CudaDispatchMode::Auto && error.is_recoverable() { + Ok(None) + } else { + Err(error.into()) + } + } +} + +fn reversible_batch_total_samples(jobs: &[DctGridToReversibleDwt53Job<'_>]) -> usize { + jobs.iter().fold(0usize, |total, job| { + total.saturating_add(job.width.saturating_mul(job.height)) + }) +} + +fn dwt97_batch_total_samples(jobs: &[DctGridToDwt97Job<'_>]) -> usize { + jobs.iter().fold(0usize, |total, job| { + total.saturating_add(job.width.saturating_mul(job.height)) + }) +} + +fn htj2k97_codeblock_batch_total_samples(jobs: &[DctGridToHtj2k97CodeBlockJob<'_>]) -> usize { + jobs.iter().fold(0usize, |total, job| { + total.saturating_add(job.width.saturating_mul(job.height)) + }) +} + +fn htj2k97_i16_codeblock_batch_total_samples( + jobs: &[DctGridI16ToHtj2k97CodeBlockJob<'_>], +) -> usize { + jobs.iter().fold(0usize, |total, job| { + total.saturating_add(job.width.saturating_mul(job.height)) + }) +} + +fn htj2k97_i16_codeblock_batch_group_total_samples( + groups: &[DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], +) -> usize { + groups.iter().fold(0usize, |total, group| { + total.saturating_add(htj2k97_i16_codeblock_batch_total_samples(group.jobs)) + }) +} + +impl Default for CudaDctToWaveletStageAccelerator { + fn default() -> Self { + Self::for_auto() + } +} + +impl DctToWaveletStageAccelerator for CudaDctToWaveletStageAccelerator { + fn supports_dwt97_batch(&self) -> bool { + true + } + + // The fused DCT->9/7->prequantized-codeblock path runs the staged 9/7 + // kernels followed by per-subband deadzone quantization into code-block-major + // layout, mirroring the local Metal backend. + fn supports_htj2k97_codeblock_batch(&self) -> bool { + true + } + + fn supports_htj2k97_i16_preencoded_batch(&self) -> bool { + self.resident_ht_encode + } + + fn supports_htj2k97_compact_preencoded_batch(&self) -> bool { + self.resident_ht_encode && std::env::var_os(DISABLE_COMPACT_PREENCODED_ENV).is_none() + } + + fn dct_grid_to_reversible_dwt53( + &mut self, + job: DctGridToReversibleDwt53Job<'_>, + ) -> Result, TranscodeStageError> { + self.reversible_dwt53_attempts = self.reversible_dwt53_attempts.saturating_add(1); + + if self.mode == CudaDispatchMode::Auto + && job.width.saturating_mul(job.height) < self.min_auto_samples + { + return Ok(None); + } + + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = job; + self.unavailable() + } + + #[cfg(feature = "cuda-runtime")] + { + match cuda::dispatch_reversible_dwt53(self.cuda_session(), job) { + Ok(output) => { + self.reversible_dwt53_dispatches = + self.reversible_dwt53_dispatches.saturating_add(1); + Ok(Some(output)) + } + Err(error) => self.recover(error), + } + } + } + + fn dct_grid_to_reversible_dwt53_batch( + &mut self, + jobs: &[DctGridToReversibleDwt53Job<'_>], + ) -> Result>, TranscodeStageError> { + self.reversible_dwt53_batch_attempts = + self.reversible_dwt53_batch_attempts.saturating_add(1); + + if jobs.is_empty() { + return Ok(Some(Vec::new())); + } + if self.mode == CudaDispatchMode::Auto + && (jobs.len() < self.min_auto_reversible_batch_jobs + || reversible_batch_total_samples(jobs) < self.min_auto_reversible_batch_samples) + { + return Ok(None); + } + + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = jobs; + self.unavailable() + } + + #[cfg(feature = "cuda-runtime")] + { + match cuda::dispatch_reversible_dwt53_batch(self.cuda_session(), jobs) { + Ok(output) => { + self.reversible_dwt53_batch_dispatches = + self.reversible_dwt53_batch_dispatches.saturating_add(1); + Ok(Some(output)) + } + Err(error) => self.recover(error), + } + } + } + + fn dct_grid_to_dwt53( + &mut self, + job: DctGridToDwt53Job<'_>, + ) -> Result>, TranscodeStageError> { + self.dwt53_attempts = self.dwt53_attempts.saturating_add(1); + + if self.mode == CudaDispatchMode::Auto + && job.width.saturating_mul(job.height) < self.min_auto_samples + { + return Ok(None); + } + + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = job; + self.unavailable() + } + + #[cfg(feature = "cuda-runtime")] + { + match cuda::dispatch_dwt53(job) { + Ok(output) => { + self.dwt53_dispatches = self.dwt53_dispatches.saturating_add(1); + Ok(Some(output)) + } + Err(error) => self.recover(error), + } + } + } + + fn dct_grid_to_dwt97( + &mut self, + job: DctGridToDwt97Job<'_>, + ) -> Result>, TranscodeStageError> { + self.dwt97_attempts = self.dwt97_attempts.saturating_add(1); + + if self.mode == CudaDispatchMode::Auto + && job.width.saturating_mul(job.height) < self.min_auto_samples + { + return Ok(None); + } + + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = job; + self.unavailable() + } + + #[cfg(feature = "cuda-runtime")] + { + match cuda::dispatch_dwt97(self.cuda_session(), job) { + Ok(output) => { + self.dwt97_dispatches = self.dwt97_dispatches.saturating_add(1); + Ok(Some(output)) + } + Err(error) => self.recover(error), + } + } + } + + fn dct_grid_to_dwt97_batch( + &mut self, + jobs: &[DctGridToDwt97Job<'_>], + ) -> Result>>, TranscodeStageError> { + self.dwt97_batch_attempts = self.dwt97_batch_attempts.saturating_add(1); + self.last_dwt97_batch_stage_timings = None; + + if jobs.is_empty() { + return Ok(Some(Vec::new())); + } + if self.mode == CudaDispatchMode::Auto + && (jobs.len() < self.min_auto_dwt97_batch_jobs + || dwt97_batch_total_samples(jobs) < self.min_auto_dwt97_batch_samples) + { + return Ok(None); + } + + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = jobs; + self.unavailable() + } + + #[cfg(feature = "cuda-runtime")] + { + match cuda::dispatch_dwt97_batch(self.cuda_session(), jobs) { + Ok((output, timings)) => { + self.dwt97_batch_dispatches = self.dwt97_batch_dispatches.saturating_add(1); + self.last_dwt97_batch_stage_timings = Some(timings); + Ok(Some(output)) + } + Err(error) => self.recover(error), + } + } + } + + fn dct_grid_to_htj2k97_codeblock_batch( + &mut self, + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, + ) -> Result>, TranscodeStageError> { + // The code-block path is a staged 9/7 batch plus quantization, so it + // counts as both a 9/7 batch and a code-block batch (matching Metal). + self.dwt97_batch_attempts = self.dwt97_batch_attempts.saturating_add(1); + self.htj2k97_codeblock_batch_attempts = + self.htj2k97_codeblock_batch_attempts.saturating_add(1); + self.last_dwt97_batch_stage_timings = None; + + if jobs.is_empty() { + return Ok(Some(Vec::new())); + } + if self.mode == CudaDispatchMode::Auto + && (jobs.len() < self.min_auto_dwt97_batch_jobs + || htj2k97_codeblock_batch_total_samples(jobs) < self.min_auto_dwt97_batch_samples) + { + return Ok(None); + } + + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = (jobs, options); + self.unavailable() + } + + #[cfg(feature = "cuda-runtime")] + { + match cuda::dispatch_htj2k97_codeblock_batch(self.cuda_session(), jobs, options) { + Ok((output, timings)) => { + self.dwt97_batch_dispatches = self.dwt97_batch_dispatches.saturating_add(1); + self.htj2k97_codeblock_batch_dispatches = + self.htj2k97_codeblock_batch_dispatches.saturating_add(1); + self.last_dwt97_batch_stage_timings = Some(timings); + Ok(Some(output)) + } + Err(error) => self.recover(error), + } + } + } + + fn dct_grid_to_htj2k97_preencoded_batch( + &mut self, + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, + ) -> Result>, TranscodeStageError> { + if !self.resident_ht_encode { + return Ok(None); + } + + self.dwt97_batch_attempts = self.dwt97_batch_attempts.saturating_add(1); + self.htj2k97_codeblock_batch_attempts = + self.htj2k97_codeblock_batch_attempts.saturating_add(1); + self.last_dwt97_batch_stage_timings = None; + + if jobs.is_empty() { + return Ok(Some(Vec::new())); + } + if self.mode == CudaDispatchMode::Auto + && (jobs.len() < self.min_auto_dwt97_batch_jobs + || htj2k97_codeblock_batch_total_samples(jobs) < self.min_auto_dwt97_batch_samples) + { + return Ok(None); + } + + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = (jobs, options); + self.unavailable() + } + + #[cfg(feature = "cuda-runtime")] + { + match cuda::dispatch_htj2k97_preencoded_batch(self.cuda_session(), jobs, options) { + Ok((output, timings)) => { + self.dwt97_batch_dispatches = self.dwt97_batch_dispatches.saturating_add(1); + self.htj2k97_codeblock_batch_dispatches = + self.htj2k97_codeblock_batch_dispatches.saturating_add(1); + self.last_dwt97_batch_stage_timings = Some(timings); + Ok(Some(output)) + } + Err(error) => self.recover(error), + } + } + } + + fn dct_grid_i16_to_htj2k97_preencoded_batch( + &mut self, + jobs: &[DctGridI16ToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, + ) -> Result>, TranscodeStageError> { + if !self.resident_ht_encode { + return Ok(None); + } + + self.dwt97_batch_attempts = self.dwt97_batch_attempts.saturating_add(1); + self.htj2k97_codeblock_batch_attempts = + self.htj2k97_codeblock_batch_attempts.saturating_add(1); + self.last_dwt97_batch_stage_timings = None; + + if jobs.is_empty() { + return Ok(Some(Vec::new())); + } + if self.mode == CudaDispatchMode::Auto + && (jobs.len() < self.min_auto_dwt97_batch_jobs + || htj2k97_i16_codeblock_batch_total_samples(jobs) + < self.min_auto_dwt97_batch_samples) + { + return Ok(None); + } + + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = (jobs, options); + self.unavailable() + } + + #[cfg(feature = "cuda-runtime")] + { + match cuda::dispatch_htj2k97_preencoded_i16_batch(self.cuda_session(), jobs, options) { + Ok((output, timings)) => { + self.dwt97_batch_dispatches = self.dwt97_batch_dispatches.saturating_add(1); + self.htj2k97_codeblock_batch_dispatches = + self.htj2k97_codeblock_batch_dispatches.saturating_add(1); + self.last_dwt97_batch_stage_timings = Some(timings); + Ok(Some(output)) + } + Err(error) => self.recover(error), + } + } + } + + fn dct_grid_i16_to_htj2k97_compact_preencoded_batch( + &mut self, + jobs: &[DctGridI16ToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, + ) -> Result, TranscodeStageError> { + if !self.resident_ht_encode { + return Ok(None); + } + + self.dwt97_batch_attempts = self.dwt97_batch_attempts.saturating_add(1); + self.htj2k97_codeblock_batch_attempts = + self.htj2k97_codeblock_batch_attempts.saturating_add(1); + self.last_dwt97_batch_stage_timings = None; + + if jobs.is_empty() { + return Ok(Some(PreencodedHtj2k97CompactBatch { + payload: Vec::new(), + components: Vec::new(), + })); + } + if self.mode == CudaDispatchMode::Auto + && (jobs.len() < self.min_auto_dwt97_batch_jobs + || htj2k97_i16_codeblock_batch_total_samples(jobs) + < self.min_auto_dwt97_batch_samples) + { + return Ok(None); + } + + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = (jobs, options); + self.unavailable() + } + + #[cfg(feature = "cuda-runtime")] + { + match cuda::dispatch_htj2k97_compact_preencoded_i16_batch( + self.cuda_session(), + jobs, + options, + ) { + Ok((output, timings)) => { + self.dwt97_batch_dispatches = self.dwt97_batch_dispatches.saturating_add(1); + self.htj2k97_codeblock_batch_dispatches = + self.htj2k97_codeblock_batch_dispatches.saturating_add(1); + self.last_dwt97_batch_stage_timings = Some(timings); + Ok(Some(output)) + } + Err(error) => self.recover(error), + } + } + } + + fn dct_grid_i16_to_htj2k97_preencoded_batch_groups( + &mut self, + groups: &[DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], + options: Htj2k97CodeBlockOptions, + ) -> Result>>, TranscodeStageError> { + if !self.resident_ht_encode { + return Ok(None); + } + + self.dwt97_batch_attempts = self.dwt97_batch_attempts.saturating_add(groups.len()); + self.htj2k97_codeblock_batch_attempts = self + .htj2k97_codeblock_batch_attempts + .saturating_add(groups.len()); + self.last_dwt97_batch_stage_timings = None; + + if groups.is_empty() { + return Ok(Some(Vec::new())); + } + let total_jobs = groups.iter().map(|group| group.jobs.len()).sum::(); + if self.mode == CudaDispatchMode::Auto + && (total_jobs < self.min_auto_dwt97_batch_jobs + || htj2k97_i16_codeblock_batch_group_total_samples(groups) + < self.min_auto_dwt97_batch_samples) + { + return Ok(None); + } + + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = (groups, options); + self.unavailable() + } + + #[cfg(feature = "cuda-runtime")] + { + match cuda::dispatch_htj2k97_preencoded_i16_batch_groups( + self.cuda_session(), + groups, + options, + ) { + Ok((output, timings)) => { + self.dwt97_batch_dispatches = + self.dwt97_batch_dispatches.saturating_add(groups.len()); + self.htj2k97_codeblock_batch_dispatches = self + .htj2k97_codeblock_batch_dispatches + .saturating_add(timings.ht_codeblock_dispatches); + self.last_dwt97_batch_stage_timings = Some(timings); + Ok(Some(output)) + } + Err(error) => self.recover(error), + } + } + } + + fn dct_grid_i16_to_htj2k97_compact_preencoded_batch_groups( + &mut self, + groups: &[DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], + options: Htj2k97CodeBlockOptions, + ) -> Result, TranscodeStageError> { + if !self.resident_ht_encode { + return Ok(None); + } + + self.dwt97_batch_attempts = self.dwt97_batch_attempts.saturating_add(groups.len()); + self.htj2k97_codeblock_batch_attempts = self + .htj2k97_codeblock_batch_attempts + .saturating_add(groups.len()); + self.last_dwt97_batch_stage_timings = None; + + if groups.is_empty() { + return Ok(Some(PreencodedHtj2k97CompactBatchGroups { + payload: Vec::new(), + groups: Vec::new(), + })); + } + let total_jobs = groups.iter().map(|group| group.jobs.len()).sum::(); + if self.mode == CudaDispatchMode::Auto + && (total_jobs < self.min_auto_dwt97_batch_jobs + || htj2k97_i16_codeblock_batch_group_total_samples(groups) + < self.min_auto_dwt97_batch_samples) + { + return Ok(None); + } + + #[cfg(not(feature = "cuda-runtime"))] + { + let _ = (groups, options); + self.unavailable() + } + + #[cfg(feature = "cuda-runtime")] + { + match cuda::dispatch_htj2k97_compact_preencoded_i16_batch_groups( + self.cuda_session(), + groups, + options, + ) { + Ok((output, timings)) => { + self.dwt97_batch_dispatches = + self.dwt97_batch_dispatches.saturating_add(groups.len()); + self.htj2k97_codeblock_batch_dispatches = self + .htj2k97_codeblock_batch_dispatches + .saturating_add(timings.ht_codeblock_dispatches); + self.last_dwt97_batch_stage_timings = Some(timings); + Ok(Some(output)) + } + Err(error) => self.recover(error), + } + } + } + + fn last_dwt97_batch_stage_timings(&self) -> Option { + self.last_dwt97_batch_stage_timings + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + static ENV_LOCK: Mutex<()> = Mutex::new(()); + + fn test_htj2k97_codeblock_options() -> Htj2k97CodeBlockOptions { + Htj2k97CodeBlockOptions { + bit_depth: 8, + guard_bits: 2, + code_block_width_exp: 4, + code_block_height_exp: 4, + irreversible_quantization_scale: 1.0, + irreversible_quantization_subband_scales: + j2k_transcode::accelerator::IrreversibleQuantizationSubbandScales::default(), + } + } + + #[test] + fn explicit_mode_without_cuda_runtime_errors_on_reversible_job() { + // Without the cuda-runtime feature, Explicit mode must surface a typed + // error rather than silently using the scalar fallback. + let mut accelerator = CudaDctToWaveletStageAccelerator::new_explicit(); + let blocks: Vec<[i16; 64]> = vec![[0i16; 64]]; + let job = DctGridToReversibleDwt53Job { + dequantized_blocks: &blocks, + block_cols: 1, + block_rows: 1, + width: 8, + height: 8, + }; + let result = accelerator.dct_grid_to_reversible_dwt53(job); + #[cfg(not(feature = "cuda-runtime"))] + assert_eq!(result, Err(TranscodeStageError::DeviceUnavailable)); + let _ = result; + assert_eq!(accelerator.reversible_dwt53_attempts(), 1); + } + + #[test] + fn auto_mode_falls_back_to_scalar_for_small_jobs() { + // Auto mode returns Ok(None) for sub-threshold jobs so the transcode + // pipeline uses its scalar oracle. + let mut accelerator = CudaDctToWaveletStageAccelerator::for_auto(); + let blocks: Vec<[i16; 64]> = vec![[0i16; 64]]; + let job = DctGridToReversibleDwt53Job { + dequantized_blocks: &blocks, + block_cols: 1, + block_rows: 1, + width: 8, + height: 8, + }; + assert_eq!(accelerator.dct_grid_to_reversible_dwt53(job), Ok(None)); + } + + #[test] + fn empty_batches_return_empty_without_dispatch() { + let mut accelerator = CudaDctToWaveletStageAccelerator::new_explicit(); + assert_eq!( + accelerator.dct_grid_to_reversible_dwt53_batch(&[]), + Ok(Some(Vec::new())) + ); + assert_eq!( + accelerator.dct_grid_to_dwt97_batch(&[]), + Ok(Some(Vec::new())) + ); + } + + #[test] + fn compact_preencoded_support_obeys_cuda_env_gate() { + let _guard = ENV_LOCK.lock().expect("env lock"); + let previous = std::env::var_os(DISABLE_COMPACT_PREENCODED_ENV); + std::env::remove_var(DISABLE_COMPACT_PREENCODED_ENV); + let accelerator = CudaDctToWaveletStageAccelerator::new_explicit_resident_ht_encode(); + assert!(accelerator.supports_htj2k97_i16_preencoded_batch()); + assert!(accelerator.supports_htj2k97_compact_preencoded_batch()); + + std::env::set_var(DISABLE_COMPACT_PREENCODED_ENV, "1"); + let accelerator = CudaDctToWaveletStageAccelerator::new_explicit_resident_ht_encode(); + assert!(accelerator.supports_htj2k97_i16_preencoded_batch()); + assert!(!accelerator.supports_htj2k97_compact_preencoded_batch()); + + if let Some(previous) = previous { + std::env::set_var(DISABLE_COMPACT_PREENCODED_ENV, previous); + } else { + std::env::remove_var(DISABLE_COMPACT_PREENCODED_ENV); + } + } + + #[test] + fn auto_mode_declines_under_amortized_reversible_batches() { + let mut accelerator = CudaDctToWaveletStageAccelerator::for_auto() + .with_auto_reversible_batch_thresholds(2, 224 * 224 * 2); + let blocks = vec![[0i16; 64]; 256 * 256 / 64]; + let job = DctGridToReversibleDwt53Job { + dequantized_blocks: &blocks, + block_cols: 32, + block_rows: 32, + width: 256, + height: 256, + }; + + assert_eq!( + accelerator.dct_grid_to_reversible_dwt53_batch(&[job]), + Ok(None) + ); + assert_eq!(accelerator.reversible_dwt53_batch_attempts(), 1); + assert_eq!(accelerator.reversible_dwt53_batch_dispatches(), 0); + } + + #[test] + fn auto_mode_declines_under_amortized_dwt97_batches() { + let mut accelerator = CudaDctToWaveletStageAccelerator::for_auto() + .with_auto_dwt97_batch_thresholds(2, 224 * 224 * 2); + let blocks = vec![[[0.0f64; 8]; 8]; 256 * 256 / 64]; + let job = DctGridToDwt97Job { + blocks: &blocks, + block_cols: 32, + block_rows: 32, + width: 256, + height: 256, + }; + + assert_eq!(accelerator.dct_grid_to_dwt97_batch(&[job]), Ok(None)); + assert_eq!(accelerator.dwt97_batch_attempts(), 1); + assert_eq!(accelerator.dwt97_batch_dispatches(), 0); + } + + #[test] + fn auto_mode_declines_under_amortized_htj2k97_codeblock_batches() { + let mut accelerator = CudaDctToWaveletStageAccelerator::for_auto() + .with_auto_dwt97_batch_thresholds(2, 224 * 224 * 2); + let blocks = vec![[[0.0f64; 8]; 8]; 256 * 256 / 64]; + let job = DctGridToHtj2k97CodeBlockJob { + blocks: &blocks, + block_cols: 32, + block_rows: 32, + width: 256, + height: 256, + x_rsiz: 1, + y_rsiz: 1, + }; + + let result = accelerator + .dct_grid_to_htj2k97_codeblock_batch(&[job], test_htj2k97_codeblock_options()); + assert!(matches!(result, Ok(None))); + assert_eq!(accelerator.dwt97_batch_attempts(), 1); + assert_eq!(accelerator.dwt97_batch_dispatches(), 0); + assert_eq!(accelerator.htj2k97_codeblock_batch_attempts(), 1); + assert_eq!(accelerator.htj2k97_codeblock_batch_dispatches(), 0); + } +} diff --git a/crates/j2k-transcode-cuda/tests/dwt97_batch_parity.rs b/crates/j2k-transcode-cuda/tests/dwt97_batch_parity.rs new file mode 100644 index 00000000..3abeba68 --- /dev/null +++ b/crates/j2k-transcode-cuda/tests/dwt97_batch_parity.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Batch parity (Gap B): the CUDA same-geometry 9/7 batch must produce the exact +// same bands as the per-job CUDA 9/7 path (identical f32 kernels with per-item +// offsets, --fmad=false), and must report real (non-zero) staged batch timings. +// +// Compiled only with `cuda-runtime`; asserts only on the CUDA runner. +#![cfg(feature = "cuda-runtime")] + +use j2k_test_support::cuda_runtime_required; +use j2k_transcode::accelerator::{DctGridToDwt97Job, DctToWaveletStageAccelerator}; +use j2k_transcode_cuda::CudaDctToWaveletStageAccelerator; + +/// Deterministic small f64 DCT coefficients, varied per job by `salt`. +fn make_blocks(block_cols: usize, block_rows: usize, salt: usize) -> Vec<[[f64; 8]; 8]> { + let mut blocks = vec![[[0.0f64; 8]; 8]; block_cols * block_rows]; + for (bi, block) in blocks.iter_mut().enumerate() { + for (fy, row) in block.iter_mut().enumerate() { + for (fx, coeff) in row.iter_mut().enumerate() { + *coeff = (((bi * 7 + fy * 8 + fx * 3 + salt) % 23) as f64) - 11.0; + } + } + } + blocks +} + +#[test] +fn cuda_dwt97_batch_matches_per_job_and_reports_stage_timings_when_required() { + if !cuda_runtime_required() { + return; + } + + // Non-multiple-of-8 dimensions to exercise partial edge blocks/bands. + let (block_cols, block_rows, width, height) = (4usize, 4usize, 29usize, 31usize); + let first = make_blocks(block_cols, block_rows, 0); + let second = make_blocks(block_cols, block_rows, 37); + let jobs = [ + DctGridToDwt97Job { + blocks: &first, + block_cols, + block_rows, + width, + height, + }, + DctGridToDwt97Job { + blocks: &second, + block_cols, + block_rows, + width, + height, + }, + ]; + + let mut accelerator = CudaDctToWaveletStageAccelerator::new_explicit(); + let batch = accelerator + .dct_grid_to_dwt97_batch(&jobs) + .expect("CUDA 9/7 batch dispatch should succeed on the runner") + .expect("CUDA should handle the 9/7 batch (explicit mode)"); + + assert_eq!(batch.len(), jobs.len()); + + // The batched staged kernels must match the single-job path bit-for-bit. + for (job, batched) in jobs.iter().zip(batch.iter()) { + let per_job = CudaDctToWaveletStageAccelerator::new_explicit() + .dct_grid_to_dwt97(*job) + .expect("CUDA 9/7 dispatch should succeed on the runner") + .expect("CUDA should handle the 9/7 job (explicit mode)"); + assert_eq!( + batched, &per_job, + "batch item diverged from the per-job 9/7 transcode for {width}x{height}" + ); + } + + let timings = accelerator + .last_dwt97_batch_stage_timings() + .expect("CUDA 9/7 batch records backend stage timings"); + assert!(timings.pack_upload_us > 0, "pack/upload stage not timed"); + assert!( + timings.idct_row_lift_us > 0, + "idct+row-lift stage not timed" + ); + assert!(timings.column_lift_us > 0, "column-lift stage not timed"); + assert_eq!( + timings.quantize_codeblock_us, 0, + "band path must not run the quantize stage" + ); + assert!(timings.readback_us > 0, "readback stage not timed"); +} + +#[test] +fn cuda_dwt97_batch_non_uniform_geometry_falls_back_to_per_job_when_required() { + if !cuda_runtime_required() { + return; + } + + // Two jobs share a 4x4 block grid (covers 32x32) but differ in logical + // dimensions, so the batch cannot use the same-geometry staged path and must + // fall back to the per-job loop — still bit-exact, just without batch timings. + let block_cols = 4usize; + let block_rows = 4usize; + let first = make_blocks(block_cols, block_rows, 0); + let second = make_blocks(block_cols, block_rows, 37); + let jobs = [ + DctGridToDwt97Job { + blocks: &first, + block_cols, + block_rows, + width: 29, + height: 31, + }, + DctGridToDwt97Job { + blocks: &second, + block_cols, + block_rows, + width: 24, + height: 26, + }, + ]; + + let mut accelerator = CudaDctToWaveletStageAccelerator::new_explicit(); + let batch = accelerator + .dct_grid_to_dwt97_batch(&jobs) + .expect("CUDA 9/7 batch dispatch should succeed on the runner") + .expect("CUDA should handle the mixed-geometry 9/7 batch via per-job fallback"); + + assert_eq!(batch.len(), jobs.len()); + for (job, batched) in jobs.iter().zip(batch.iter()) { + let per_job = CudaDctToWaveletStageAccelerator::new_explicit() + .dct_grid_to_dwt97(*job) + .expect("CUDA 9/7 dispatch should succeed on the runner") + .expect("CUDA should handle the 9/7 job (explicit mode)"); + assert_eq!( + batched, &per_job, + "non-uniform batch item diverged from per-job 9/7 for {}x{}", + job.width, job.height + ); + } +} diff --git a/crates/j2k-transcode-cuda/tests/dwt97_parity.rs b/crates/j2k-transcode-cuda/tests/dwt97_parity.rs new file mode 100644 index 00000000..62d88521 --- /dev/null +++ b/crates/j2k-transcode-cuda/tests/dwt97_parity.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Tolerance parity: CUDA irreversible 9/7 DCT->wavelet transcode vs the +//j2k-transcode scalar float oracle. Mirrors j2k-transcode-metal's +// dct97.rs (band max-abs-diff <= 2.0e-2; device math is f32). +// +// Compiled only with `cuda-runtime`; asserts only on the CUDA runner. +#![cfg(feature = "cuda-runtime")] + +use j2k_test_support::cuda_runtime_required; +use j2k_transcode::accelerator::{DctGridToDwt97Job, DctToWaveletStageAccelerator}; +use j2k_transcode::dct97_2d::{dct8x8_blocks_then_dwt97_float, Dwt97TwoDimensional}; +use j2k_transcode_cuda::CudaDctToWaveletStageAccelerator; + +const TOLERANCE: f64 = 2.0e-2; + +/// Deterministic small f64 DCT coefficients. +fn make_blocks(block_cols: usize, block_rows: usize) -> Vec<[[f64; 8]; 8]> { + let mut blocks = vec![[[0.0f64; 8]; 8]; block_cols * block_rows]; + for (bi, block) in blocks.iter_mut().enumerate() { + for (fy, row) in block.iter_mut().enumerate() { + for (fx, coeff) in row.iter_mut().enumerate() { + *coeff = (((bi * 7 + fy * 8 + fx * 3) % 23) as f64) - 11.0; + } + } + } + blocks +} + +fn band_max_diff(actual: &[f64], expected: &[f64]) -> f64 { + actual + .iter() + .zip(expected.iter()) + .map(|(a, b)| (a - b).abs()) + .fold(0.0_f64, f64::max) +} + +fn max_abs_diff(actual: &Dwt97TwoDimensional, expected: &Dwt97TwoDimensional) -> f64 { + band_max_diff(&actual.ll, &expected.ll) + .max(band_max_diff(&actual.hl, &expected.hl)) + .max(band_max_diff(&actual.lh, &expected.lh)) + .max(band_max_diff(&actual.hh, &expected.hh)) +} + +#[test] +fn cuda_dwt97_matches_scalar_oracle_within_tolerance_when_required() { + if !cuda_runtime_required() { + return; + } + + let cases = [ + (1usize, 1usize, 8usize, 8usize), + (2, 2, 16, 16), + (3, 2, 24, 16), + (2, 2, 15, 13), + (2, 3, 16, 23), + ]; + + for (block_cols, block_rows, width, height) in cases { + let blocks = make_blocks(block_cols, block_rows); + let job = DctGridToDwt97Job { + blocks: &blocks, + block_cols, + block_rows, + width, + height, + }; + + let actual = CudaDctToWaveletStageAccelerator::new_explicit() + .dct_grid_to_dwt97(job) + .expect("CUDA 9/7 dispatch should succeed on the runner") + .expect("CUDA should handle the 9/7 job (explicit mode)"); + + let expected = + dct8x8_blocks_then_dwt97_float(&blocks, block_cols, block_rows, width, height) + .expect("scalar 9/7 oracle accepts the job"); + + let diff = max_abs_diff(&actual, &expected); + assert!( + diff <= TOLERANCE, + "9/7 transcode diverged for {width}x{height} ({block_cols}x{block_rows} blocks): {diff}" + ); + } +} diff --git a/crates/j2k-transcode-cuda/tests/fixtures/conformance/baseline_420_16x16.jpg b/crates/j2k-transcode-cuda/tests/fixtures/conformance/baseline_420_16x16.jpg new file mode 100644 index 00000000..af176169 Binary files /dev/null and b/crates/j2k-transcode-cuda/tests/fixtures/conformance/baseline_420_16x16.jpg differ diff --git a/crates/j2k-transcode-cuda/tests/htj2k97_codeblock_parity.rs b/crates/j2k-transcode-cuda/tests/htj2k97_codeblock_parity.rs new file mode 100644 index 00000000..379d49e8 --- /dev/null +++ b/crates/j2k-transcode-cuda/tests/htj2k97_codeblock_parity.rs @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Fused code-block parity (Gap C): the CUDA staged 9/7 + deadzone quantization +// batch must match the shared j2k-transcode code-block oracle in layout +// exactly and in quantized coefficients within +/-1 (device math is f32 vs the +// oracle's f64 at deadzone boundaries), and must report all five stage timings. +// +// Compiled only with `cuda-runtime`; asserts only on the CUDA runner. +#![cfg(feature = "cuda-runtime")] + +use j2k_transcode::accelerator::{ + DctGridToHtj2k97CodeBlockJob, DctToWaveletStageAccelerator, Htj2k97CodeBlockOptions, + IrreversibleQuantizationSubbandScales, PrequantizedHtj2k97Component, +}; +use j2k_transcode::dct97_2d::dct8x8_blocks_then_dwt97_float; +use j2k_transcode::htj2k97_codeblock_oracle::prequantized_component_from_dwt97; +use j2k_transcode::{JpegTileBatchInput, JpegToHtj2kOptions, JpegToHtj2kTranscoder}; +use j2k_transcode_cuda::CudaDctToWaveletStageAccelerator; + +use j2k_test_support::{cuda_runtime_required, jpeg_baseline_420_16x16}; + +/// Deterministic small f64 DCT coefficients, varied per job by `salt`. +fn make_blocks(block_cols: usize, block_rows: usize, salt: usize) -> Vec<[[f64; 8]; 8]> { + let mut blocks = vec![[[0.0f64; 8]; 8]; block_cols * block_rows]; + for (bi, block) in blocks.iter_mut().enumerate() { + for (fy, row) in block.iter_mut().enumerate() { + for (fx, coeff) in row.iter_mut().enumerate() { + *coeff = (((bi * 7 + fy * 8 + fx * 3 + salt) % 23) as f64) - 11.0; + } + } + } + blocks +} + +/// Code-block geometry must match exactly: same component/resolution/subband +/// nesting, code-block counts, declared bitplanes, and code-block dimensions. +fn assert_layout_eq( + actual: &PrequantizedHtj2k97Component, + expected: &PrequantizedHtj2k97Component, +) { + assert_eq!(actual.x_rsiz, expected.x_rsiz); + assert_eq!(actual.y_rsiz, expected.y_rsiz); + assert_eq!(actual.resolutions.len(), expected.resolutions.len()); + for (actual_res, expected_res) in actual.resolutions.iter().zip(expected.resolutions.iter()) { + assert_eq!(actual_res.subbands.len(), expected_res.subbands.len()); + for (actual_sub, expected_sub) in + actual_res.subbands.iter().zip(expected_res.subbands.iter()) + { + assert_eq!(actual_sub.sub_band_type, expected_sub.sub_band_type); + assert_eq!(actual_sub.num_cbs_x, expected_sub.num_cbs_x); + assert_eq!(actual_sub.num_cbs_y, expected_sub.num_cbs_y); + assert_eq!(actual_sub.total_bitplanes, expected_sub.total_bitplanes); + assert_eq!(actual_sub.code_blocks.len(), expected_sub.code_blocks.len()); + for (actual_block, expected_block) in actual_sub + .code_blocks + .iter() + .zip(expected_sub.code_blocks.iter()) + { + assert_eq!(actual_block.width, expected_block.width); + assert_eq!(actual_block.height, expected_block.height); + assert_eq!( + actual_block.coefficients.len(), + expected_block.coefficients.len() + ); + } + } + } +} + +/// Quantized coefficients must agree within `max_abs_error` (f32 GPU vs f64 oracle). +fn assert_coefficients_close( + actual: &PrequantizedHtj2k97Component, + expected: &PrequantizedHtj2k97Component, + max_abs_error: i32, +) { + for (actual_res, expected_res) in actual.resolutions.iter().zip(expected.resolutions.iter()) { + for (actual_sub, expected_sub) in + actual_res.subbands.iter().zip(expected_res.subbands.iter()) + { + for (actual_block, expected_block) in actual_sub + .code_blocks + .iter() + .zip(expected_sub.code_blocks.iter()) + { + for (&actual_coeff, &expected_coeff) in actual_block + .coefficients + .iter() + .zip(expected_block.coefficients.iter()) + { + assert!( + (actual_coeff - expected_coeff).abs() <= max_abs_error, + "quantized coefficient diverged: actual {actual_coeff}, expected {expected_coeff}" + ); + } + } + } + } +} + +#[test] +fn cuda_htj2k97_codeblock_batch_matches_oracle_when_required() { + if !cuda_runtime_required() { + return; + } + + // Non-multiple-of-8 dimensions to exercise partial code-blocks and bands. + let (block_cols, block_rows, width, height) = (4usize, 4usize, 29usize, 31usize); + let first = make_blocks(block_cols, block_rows, 0); + let second = make_blocks(block_cols, block_rows, 37); + let jobs = [ + DctGridToHtj2k97CodeBlockJob { + blocks: &first, + block_cols, + block_rows, + width, + height, + x_rsiz: 1, + y_rsiz: 1, + }, + DctGridToHtj2k97CodeBlockJob { + blocks: &second, + block_cols, + block_rows, + width, + height, + x_rsiz: 1, + y_rsiz: 1, + }, + ]; + let options = Htj2k97CodeBlockOptions { + bit_depth: 8, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + irreversible_quantization_scale: 2.5, + irreversible_quantization_subband_scales: IrreversibleQuantizationSubbandScales { + low_low: 0.9, + high_low: 1.1, + low_high: 1.2, + high_high: 1.5, + }, + }; + + let mut accelerator = CudaDctToWaveletStageAccelerator::new_explicit(); + let actual = accelerator + .dct_grid_to_htj2k97_codeblock_batch(&jobs, options) + .expect("CUDA code-block batch dispatch should succeed on the runner") + .expect("CUDA should handle the code-block batch (explicit mode)"); + + assert_eq!(actual.len(), jobs.len()); + for (job, component) in jobs.iter().zip(actual.iter()) { + let dwt = dct8x8_blocks_then_dwt97_float( + job.blocks, + job.block_cols, + job.block_rows, + job.width, + job.height, + ) + .expect("scalar 9/7 oracle accepts the job"); + let expected = prequantized_component_from_dwt97(&dwt, options, job.x_rsiz, job.y_rsiz); + assert_layout_eq(component, &expected); + assert_coefficients_close(component, &expected, 1); + } + + let timings = accelerator + .last_dwt97_batch_stage_timings() + .expect("CUDA code-block batch records backend stage timings"); + assert!(timings.pack_upload_us > 0, "pack/upload stage not timed"); + assert!( + timings.idct_row_lift_us > 0, + "idct+row-lift stage not timed" + ); + assert!(timings.column_lift_us > 0, "column-lift stage not timed"); + assert!( + timings.quantize_codeblock_us > 0, + "quantize stage not timed" + ); + assert!(timings.readback_us > 0, "readback stage not timed"); +} + +#[test] +fn cuda_htj2k97_codeblock_batch_rejects_non_uniform_geometry_when_required() { + if !cuda_runtime_required() { + return; + } + + // The fused code-block kernels require uniform geometry; a mixed-geometry + // batch must surface a typed error in Explicit mode (Auto would fall back to + // the scalar oracle). There is no single-job GPU code-block entry point, so + // the whole batch is rejected rather than handled per job. + let block_cols = 4usize; + let block_rows = 4usize; + let first = make_blocks(block_cols, block_rows, 0); + let second = make_blocks(block_cols, block_rows, 37); + let jobs = [ + DctGridToHtj2k97CodeBlockJob { + blocks: &first, + block_cols, + block_rows, + width: 29, + height: 31, + x_rsiz: 1, + y_rsiz: 1, + }, + DctGridToHtj2k97CodeBlockJob { + blocks: &second, + block_cols, + block_rows, + width: 24, + height: 26, + x_rsiz: 1, + y_rsiz: 1, + }, + ]; + let options = Htj2k97CodeBlockOptions { + bit_depth: 8, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + irreversible_quantization_scale: 2.5, + irreversible_quantization_subband_scales: IrreversibleQuantizationSubbandScales::default(), + }; + + let result = CudaDctToWaveletStageAccelerator::new_explicit() + .dct_grid_to_htj2k97_codeblock_batch(&jobs, options); + assert!( + result.is_err(), + "explicit CUDA code-block batch must reject non-uniform geometry, got {result:?}" + ); +} + +#[test] +fn cuda_resident_htj2k97_batch_matches_host_bounce_codestream_when_required() { + if !cuda_runtime_required() { + return; + } + + let jpeg = jpeg_baseline_420_16x16(); + let inputs = vec![ + JpegTileBatchInput { + bytes: jpeg.as_slice(), + }; + 2 + ]; + let options = JpegToHtj2kOptions::lossy_97(); + + let mut host_transcoder = JpegToHtj2kTranscoder::default(); + let mut host_accelerator = CudaDctToWaveletStageAccelerator::new_explicit(); + let host_batch = host_transcoder + .transcode_batch_with_accelerator(&inputs, &options, &mut host_accelerator) + .expect("host-bounce CUDA 9/7 batch succeeds"); + + let mut resident_transcoder = JpegToHtj2kTranscoder::default(); + let mut resident_accelerator = + CudaDctToWaveletStageAccelerator::new_explicit_resident_ht_encode(); + let resident_batch = resident_transcoder + .transcode_batch_with_accelerator(&inputs, &options, &mut resident_accelerator) + .expect("resident CUDA 9/7 + HT batch succeeds"); + + assert_eq!(resident_batch.report.failed_tiles, 0); + assert_eq!(resident_batch.report.timings.cpu_fallback_jobs, 0); + assert_eq!( + resident_batch + .report + .timings + .dwt97_batch_ht_codeblock_dispatches, + 1, + "resident path should share one CUDA HT code-block encode across compatible batch groups" + ); + assert_eq!( + resident_batch.report.timings.dwt97_batch_readback_us, 0, + "resident path must avoid quantized coefficient readback" + ); + + let host_codestreams = host_batch + .tiles + .into_iter() + .map(|tile| tile.expect("host tile succeeds").codestream) + .collect::>(); + let resident_codestreams = resident_batch + .tiles + .into_iter() + .map(|tile| tile.expect("resident tile succeeds").codestream) + .collect::>(); + assert_eq!(resident_codestreams, host_codestreams); +} diff --git a/crates/j2k-transcode-cuda/tests/jpeg_to_htj2k.rs b/crates/j2k-transcode-cuda/tests/jpeg_to_htj2k.rs new file mode 100644 index 00000000..0b46aa56 --- /dev/null +++ b/crates/j2k-transcode-cuda/tests/jpeg_to_htj2k.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// End-to-end pipeline parity: drive the full JPEG -> HTJ2K transcode through the +// CUDA accelerator with the fused 9/7 code-block path active (supports_* +// = true), and assert the produced codestream decodes correctly. This exercises +// the CUDA-specific glue the isolated hook parity tests do not: DCT repack, +// multi-component (YCbCr 4:2:0) job construction, sampling factors, and the +// prequantized encode integration. Mirrors j2k-transcode-metal's +// ycbcr_420_batch_transcodes_with_explicit_metal_97_codeblock_path. +// +// Compiled only with `cuda-runtime`; asserts only on the CUDA runner. +#![cfg(feature = "cuda-runtime")] + +use j2k_native::{DecodeSettings, Image}; +use j2k_test_support::{cuda_runtime_required, jpeg_baseline_420_16x16}; +use j2k_transcode::{ + JpegTileBatchInput, JpegToHtj2kCoefficientPath, JpegToHtj2kOptions, JpegToHtj2kTranscoder, +}; +use j2k_transcode_cuda::CudaDctToWaveletStageAccelerator; + +#[test] +fn ycbcr_420_batch_transcodes_to_htj2k_with_explicit_cuda_97_codeblock_path() { + if !cuda_runtime_required() { + return; + } + + let jpeg = jpeg_baseline_420_16x16(); + let inputs = vec![ + JpegTileBatchInput { + bytes: jpeg.as_slice(), + }; + 4 + ]; + let options = JpegToHtj2kOptions::lossy_97(); + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = CudaDctToWaveletStageAccelerator::new_explicit(); + + let batch = transcoder + .transcode_batch_with_accelerator(&inputs, &options, &mut accelerator) + .expect("explicit CUDA 9/7 code-block batch transcode should succeed on the runner"); + + // Pipeline-level accounting: 4 tiles x 3 components = 12 jobs, grouped into + // luma plus the compatible chroma pair, no scalar fallback. + assert_eq!(batch.report.tile_count, inputs.len()); + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!(batch.report.failed_tiles, 0); + assert_eq!(batch.report.timings.batch_jobs, 12); + assert_eq!(batch.report.timings.accelerator_dispatches, 2); + assert_eq!(batch.report.timings.accelerator_dispatched_jobs, 12); + assert_eq!(batch.report.timings.cpu_fallback_jobs, 0); + + // Real staged timings flowed through from the GPU code-block path. + assert!(batch.report.timings.dwt97_batch_pack_upload_us > 0); + assert!(batch.report.timings.dwt97_batch_idct_row_lift_us > 0); + assert!(batch.report.timings.dwt97_batch_column_lift_us > 0); + assert!(batch.report.timings.dwt97_batch_quantize_codeblock_us > 0); + assert!(batch.report.timings.dwt97_batch_readback_us > 0); + + // The code-block path is a staged 9/7 batch plus quantize, so it registers as + // both (single-job 9/7 never used). + assert_eq!(accelerator.dwt97_attempts(), 0); + assert_eq!(accelerator.dwt97_batch_attempts(), 2); + assert_eq!(accelerator.dwt97_batch_dispatches(), 2); + assert_eq!(accelerator.htj2k97_codeblock_batch_attempts(), 2); + assert_eq!(accelerator.htj2k97_codeblock_batch_dispatches(), 2); + + for tile in batch.tiles { + let tile = tile.expect("valid 9/7 code-block tile transcodes"); + let decoded = Image::new(&tile.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated CUDA code-block 9/7 HTJ2K") + .decode_native() + .expect("native decoder accepts generated CUDA code-block 9/7 HTJ2K"); + assert_eq!((decoded.width, decoded.height), (16, 16)); + assert_eq!(decoded.num_components, 3); + assert_eq!( + tile.report.coefficient_path, + JpegToHtj2kCoefficientPath::FloatDirectLinear97 + ); + assert_eq!( + tile.report.path, + "native_component_sampling_float_direct_97" + ); + assert!(tile.report.float_reference_metrics.is_none()); + assert_component_sampling(&tile.codestream, &[(1, 1), (2, 2), (2, 2)]); + } +} + +/// Assert the SIZ marker's per-component sub-sampling factors (`XRsiz`, `YRsiz`). +fn assert_component_sampling(codestream: &[u8], expected: &[(u8, u8)]) { + let siz = find_marker(codestream, 0x51).expect("SIZ marker"); + let component_info = siz + 40; + for (component_index, &(x_rsiz, y_rsiz)) in expected.iter().enumerate() { + let offset = component_info + component_index * 3; + assert_eq!(codestream[offset + 1], x_rsiz); + assert_eq!(codestream[offset + 2], y_rsiz); + } +} + +fn find_marker(codestream: &[u8], marker: u8) -> Option { + codestream + .windows(2) + .position(|window| window == [0xff, marker]) +} diff --git a/crates/j2k-transcode-cuda/tests/reversible_dwt53_parity.rs b/crates/j2k-transcode-cuda/tests/reversible_dwt53_parity.rs new file mode 100644 index 00000000..5faba4e0 --- /dev/null +++ b/crates/j2k-transcode-cuda/tests/reversible_dwt53_parity.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Bit-exact parity: CUDA reversible 5/3 DCT->wavelet transcode vs the +//j2k-transcode scalar oracle. Mirrors j2k-transcode-metal's dct53.rs. +// +// Compiled only with the `cuda-runtime` feature; asserts only on the CUDA runner +// (J2K_REQUIRE_CUDA_RUNTIME set), matching the HTJ2K encode parity gate. +#![cfg(feature = "cuda-runtime")] + +use j2k_test_support::cuda_runtime_required; +use j2k_transcode::accelerator::{ + DctGridToReversibleDwt53Job, DctToWaveletStageAccelerator, RayonReversibleDwt53Accelerator, +}; +use j2k_transcode_cuda::CudaDctToWaveletStageAccelerator; + +/// Deterministic small signed "DCT" coefficients (the transcode does an exact +/// integer IDCT, so any integer input exercises the bit-exact path). +fn make_blocks(block_cols: usize, block_rows: usize) -> Vec<[i16; 64]> { + let mut blocks = vec![[0i16; 64]; block_cols * block_rows]; + for (bi, block) in blocks.iter_mut().enumerate() { + for (i, coeff) in block.iter_mut().enumerate() { + *coeff = i16::try_from((bi * 31 + i * 7) % 193).unwrap_or(0) - 96; + } + } + blocks +} + +#[test] +fn cuda_reversible_dwt53_matches_scalar_oracle_when_required() { + if !cuda_runtime_required() { + return; + } + + // (block_cols, block_rows, width, height) including non-multiple-of-8 and + // odd dimensions to exercise the 5/3 boundary cases. + let cases = [ + (1usize, 1usize, 8usize, 8usize), + (2, 2, 16, 16), + (3, 2, 24, 16), + (2, 2, 15, 13), + (2, 3, 16, 23), + ]; + + for (block_cols, block_rows, width, height) in cases { + let blocks = make_blocks(block_cols, block_rows); + let job = DctGridToReversibleDwt53Job { + dequantized_blocks: &blocks, + block_cols, + block_rows, + width, + height, + }; + + let actual = CudaDctToWaveletStageAccelerator::new_explicit() + .dct_grid_to_reversible_dwt53(job) + .expect("CUDA reversible 5/3 dispatch should succeed on the runner") + .expect("CUDA should handle the reversible 5/3 job (explicit mode)"); + + let expected = RayonReversibleDwt53Accelerator::default() + .dct_grid_to_reversible_dwt53(job) + .expect("scalar reversible 5/3 oracle accepts the job") + .expect("scalar oracle handles the job"); + + assert_eq!( + actual, expected, + "reversible 5/3 mismatch for {width}x{height} ({block_cols}x{block_rows} blocks)" + ); + } +} diff --git a/crates/j2k-transcode-metal/Cargo.toml b/crates/j2k-transcode-metal/Cargo.toml new file mode 100644 index 00000000..31de9c4c --- /dev/null +++ b/crates/j2k-transcode-metal/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "j2k-transcode-metal" +description = "Metal acceleration for JPEG to J2K and HTJ2K transcode paths" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +readme = "README.md" + +[package.metadata.docs.rs] +all-features = true +targets = [] + +[lib] +name = "j2k_transcode_metal" +path = "src/lib.rs" + +[dependencies] +j2k-transcode = { path = "../j2k-transcode", version = "=0.6.0" } + +[target.'cfg(target_os = "macos")'.dependencies] +metal = { workspace = true } +rayon = { workspace = true } +j2k-core = { path = "../j2k-core", version = "=0.6.0" } +j2k-metal-support = { path = "../j2k-metal-support", version = "=0.6.0" } + +[dev-dependencies] +criterion = { workspace = true } +j2k-native = { path = "../j2k-native", version = "=0.6.0" } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.6.0" } +j2k-test-support = { path = "../j2k-test-support" } + +[[bench]] +name = "dct97" +harness = false + +[lints.rust] +unsafe_code = "allow" +unsafe_op_in_unsafe_fn = "deny" +unreachable_pub = "warn" + +[lints.clippy] +pedantic = { level = "warn", priority = -1 } +undocumented_unsafe_blocks = "warn" +module_name_repetitions = "allow" +must_use_candidate = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +cast_possible_truncation = "allow" +cast_precision_loss = "allow" diff --git a/crates/j2k-transcode-metal/README.md b/crates/j2k-transcode-metal/README.md new file mode 100644 index 00000000..9b01631d --- /dev/null +++ b/crates/j2k-transcode-metal/README.md @@ -0,0 +1,7 @@ +# j2k-transcode-metal + +Metal acceleration adapter for J2K JPEG-to-J2K/HTJ2K transcode stages on +macOS. + +This crate accelerates supported transform stages and delegates runtime setup to +`j2k-metal-support`. diff --git a/crates/j2k-transcode-metal/benches/dct97.rs b/crates/j2k-transcode-metal/benches/dct97.rs new file mode 100644 index 00000000..2c5bd6b9 --- /dev/null +++ b/crates/j2k-transcode-metal/benches/dct97.rs @@ -0,0 +1,1014 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::time::Duration; + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use j2k_jpeg::{ + encode_jpeg_baseline, JpegBackend, JpegEncodeOptions, JpegSamples, JpegSubsampling, +}; +use j2k_transcode::accelerator::{ + DctGridToDwt53Job, DctGridToDwt97Job, DctGridToReversibleDwt53Job, + DctToWaveletStageAccelerator, RayonReversibleDwt53Accelerator, +}; +use j2k_transcode::dct53_2d::{dct8x8_blocks_to_dwt53_float_linear_with_scratch, Dct53GridScratch}; +use j2k_transcode::dct97_2d::{dct8x8_blocks_then_dwt97_float_with_scratch, Dct97GridScratch}; +use j2k_transcode::{ + EncodedTranscodeBatch, JpegTileBatchInput, JpegToHtj2kCoefficientPath, JpegToHtj2kOptions, + JpegToHtj2kTranscoder, +}; +use j2k_transcode_metal::{MetalDctToWaveletStageAccelerator, METAL_UNAVAILABLE}; + +const WSI_DIMS: [usize; 4] = [224, 512, 1024, 2048]; +const REVERSIBLE_BATCH_SIZES: [usize; 5] = [1, 8, 32, 128, 512]; +const MAX_REVERSIBLE_BATCH_SAMPLES: usize = 512 * 512 * 512; +const WSI_TILE_BATCH_SIZES: [usize; 3] = [128, 256, 512]; +const TRANSCODE_PROFILE_STAGES_ENV: &str = "J2K_TRANSCODE_METAL_PROFILE_STAGES"; + +const DIRECT_BENCH_MARKERS: [&str; 8] = [ + "cpu_idct_dwt_224x224", + "metal_explicit_224x224", + "cpu_idct_dwt_512x512", + "metal_explicit_512x512", + "cpu_idct_dwt_1024x1024", + "metal_explicit_1024x1024", + "cpu_idct_dwt_2048x2048", + "metal_explicit_2048x2048", +]; + +const REVERSIBLE_BENCH_MARKERS: [&str; 8] = [ + "rayon_224x224", + "metal_explicit_224x224", + "rayon_512x512", + "metal_explicit_512x512", + "rayon_1024x1024", + "metal_explicit_1024x1024", + "rayon_2048x2048", + "metal_explicit_2048x2048", +]; + +const REVERSIBLE_BATCH_BENCH_MARKERS: [&str; 11] = [ + "reversible_dct53_batch_metal_projection", + "batch_1", + "batch_8", + "batch_32", + "batch_128", + "batch_512", + "rayon_224x224_batch_1", + "metal_explicit_224x224_batch_1", + "rayon_512x512_batch_512", + "rayon_1024x1024_batch_128", + "rayon_2048x2048_batch_32", +]; + +const WSI_TILE_BATCH_BENCH_MARKERS: &[&str] = &[ + "jpeg_to_htj2k_wsi_integer_53_tile_batch", + "rayon_batch", + "metal_auto_batch", + "metal_explicit_batch", + "batch_128", + "batch_256", + "batch_512", + "p3_like_ybr444_224_batch_128", + "p3_like_ybr444_224_batch_256", + "p3_like_ybr444_224_batch_512", + "p3_like_ybr444_512_batch_128", + "p3_like_ybr444_512_batch_256", + "p3_like_ybr444_512_batch_512", + "p3_like_ybr444_1024_batch_128", + "p3_like_ybr444_1024_batch_256", + "p3_like_ybr444_1024_batch_512", + "p3_like_ybr444_2048_batch_128", + "p3_like_ybr444_2048_batch_256", + "p3_like_ybr444_2048_batch_512", + "srgb_ybr420_224_batch_128", + "srgb_ybr420_224_batch_256", + "srgb_ybr420_224_batch_512", + "srgb_ybr420_512_batch_128", + "srgb_ybr420_512_batch_256", + "srgb_ybr420_512_batch_512", + "srgb_ybr420_1024_batch_128", + "srgb_ybr420_1024_batch_256", + "srgb_ybr420_1024_batch_512", + "srgb_ybr420_2048_batch_128", + "srgb_ybr420_2048_batch_256", + "srgb_ybr420_2048_batch_512", + "ycbcr_like_ybr420_224_batch_128", + "ycbcr_like_ybr420_224_batch_256", + "ycbcr_like_ybr420_224_batch_512", + "ycbcr_like_ybr420_512_batch_128", + "ycbcr_like_ybr420_512_batch_256", + "ycbcr_like_ybr420_512_batch_512", + "ycbcr_like_ybr420_1024_batch_128", + "ycbcr_like_ybr420_1024_batch_256", + "ycbcr_like_ybr420_1024_batch_512", + "ycbcr_like_ybr420_2048_batch_128", + "ycbcr_like_ybr420_2048_batch_256", + "ycbcr_like_ybr420_2048_batch_512", +]; + +const WSI_FIXTURES: [WsiFixtureSpec; 12] = [ + WsiFixtureSpec { + name: "p3_like_ybr444_224", + dim: 224, + subsampling: JpegSubsampling::Ybr444, + generator: rgb_p3_like_pattern, + }, + WsiFixtureSpec { + name: "p3_like_ybr444_512", + dim: 512, + subsampling: JpegSubsampling::Ybr444, + generator: rgb_p3_like_pattern, + }, + WsiFixtureSpec { + name: "p3_like_ybr444_1024", + dim: 1024, + subsampling: JpegSubsampling::Ybr444, + generator: rgb_p3_like_pattern, + }, + WsiFixtureSpec { + name: "p3_like_ybr444_2048", + dim: 2048, + subsampling: JpegSubsampling::Ybr444, + generator: rgb_p3_like_pattern, + }, + WsiFixtureSpec { + name: "srgb_ybr420_224", + dim: 224, + subsampling: JpegSubsampling::Ybr420, + generator: rgb_srgb_pattern, + }, + WsiFixtureSpec { + name: "srgb_ybr420_512", + dim: 512, + subsampling: JpegSubsampling::Ybr420, + generator: rgb_srgb_pattern, + }, + WsiFixtureSpec { + name: "srgb_ybr420_1024", + dim: 1024, + subsampling: JpegSubsampling::Ybr420, + generator: rgb_srgb_pattern, + }, + WsiFixtureSpec { + name: "srgb_ybr420_2048", + dim: 2048, + subsampling: JpegSubsampling::Ybr420, + generator: rgb_srgb_pattern, + }, + WsiFixtureSpec { + name: "ycbcr_like_ybr420_224", + dim: 224, + subsampling: JpegSubsampling::Ybr420, + generator: rgb_ycbcr_like_pattern, + }, + WsiFixtureSpec { + name: "ycbcr_like_ybr420_512", + dim: 512, + subsampling: JpegSubsampling::Ybr420, + generator: rgb_ycbcr_like_pattern, + }, + WsiFixtureSpec { + name: "ycbcr_like_ybr420_1024", + dim: 1024, + subsampling: JpegSubsampling::Ybr420, + generator: rgb_ycbcr_like_pattern, + }, + WsiFixtureSpec { + name: "ycbcr_like_ybr420_2048", + dim: 2048, + subsampling: JpegSubsampling::Ybr420, + generator: rgb_ycbcr_like_pattern, + }, +]; + +fn bench_dct97_idct_dwt(c: &mut Criterion) { + std::hint::black_box(DIRECT_BENCH_MARKERS); + let mut group = c.benchmark_group("dct97_metal_idct_dwt"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(2)); + + for dim in WSI_DIMS { + let block_cols = dim / 8; + let block_rows = dim / 8; + let blocks = structured_blocks(block_cols, block_rows); + let job = DctGridToDwt97Job { + blocks: &blocks, + block_cols, + block_rows, + width: dim, + height: dim, + }; + group.throughput(Throughput::Elements((dim * dim) as u64)); + + group.bench_with_input( + BenchmarkId::from_parameter(format!("cpu_idct_dwt_{dim}x{dim}")), + &job, + |b, job| { + let mut scratch = Dct97GridScratch::default(); + b.iter(|| { + std::hint::black_box( + dct8x8_blocks_then_dwt97_float_with_scratch( + std::hint::black_box(job.blocks), + job.block_cols, + job.block_rows, + job.width, + job.height, + &mut scratch, + ) + .expect("CPU 9/7 IDCT-DWT accepts fixture grid"), + ); + }); + }, + ); + + if explicit_metal_accepts(job) { + group.bench_with_input( + BenchmarkId::from_parameter(format!("metal_explicit_{dim}x{dim}")), + &job, + |b, job| { + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + b.iter(|| { + std::hint::black_box( + accelerator + .dct_grid_to_dwt97(std::hint::black_box(*job)) + .expect("explicit Metal 9/7 transform succeeds") + .expect("explicit Metal handles benchmark job"), + ); + }); + }, + ); + } else { + eprintln!("skipping metal_explicit_{dim}x{dim} benchmark: {METAL_UNAVAILABLE}"); + } + } + + group.finish(); +} + +fn bench_dct53_projection(c: &mut Criterion) { + std::hint::black_box(DIRECT_BENCH_MARKERS); + let mut group = c.benchmark_group("dct53_metal_projection"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(2)); + + for dim in WSI_DIMS { + let block_cols = dim / 8; + let block_rows = dim / 8; + let blocks = structured_blocks(block_cols, block_rows); + let job = DctGridToDwt53Job { + blocks: &blocks, + block_cols, + block_rows, + width: dim, + height: dim, + }; + group.throughput(Throughput::Elements((dim * dim) as u64)); + + group.bench_with_input( + BenchmarkId::from_parameter(format!("scalar_{dim}x{dim}")), + &job, + |b, job| { + let mut scratch = Dct53GridScratch::default(); + b.iter(|| { + std::hint::black_box( + dct8x8_blocks_to_dwt53_float_linear_with_scratch( + std::hint::black_box(job.blocks), + job.block_cols, + job.block_rows, + job.width, + job.height, + &mut scratch, + ) + .expect("scalar 5/3 projection accepts fixture grid"), + ); + }); + }, + ); + + if explicit_metal_accepts_53(job) { + group.bench_with_input( + BenchmarkId::from_parameter(format!("metal_explicit_{dim}x{dim}")), + &job, + |b, job| { + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + b.iter(|| { + std::hint::black_box( + accelerator + .dct_grid_to_dwt53(std::hint::black_box(*job)) + .expect("explicit Metal 5/3 projection succeeds") + .expect("explicit Metal handles benchmark job"), + ); + }); + }, + ); + } else { + eprintln!("skipping metal_explicit_{dim}x{dim} benchmark: {METAL_UNAVAILABLE}"); + } + } + + group.finish(); +} + +fn bench_reversible_dct53_projection(c: &mut Criterion) { + std::hint::black_box(REVERSIBLE_BENCH_MARKERS); + let mut group = c.benchmark_group("reversible_dct53_metal_projection"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(2)); + + for dim in WSI_DIMS { + let block_cols = dim / 8; + let block_rows = dim / 8; + let blocks = structured_i16_blocks(block_cols, block_rows); + let job = DctGridToReversibleDwt53Job { + dequantized_blocks: &blocks, + block_cols, + block_rows, + width: dim, + height: dim, + }; + group.throughput(Throughput::Elements((dim * dim) as u64)); + + group.bench_with_input( + BenchmarkId::from_parameter(format!("rayon_{dim}x{dim}")), + &job, + |b, job| { + let mut accelerator = RayonReversibleDwt53Accelerator::default(); + b.iter(|| { + std::hint::black_box( + accelerator + .dct_grid_to_reversible_dwt53(std::hint::black_box(*job)) + .expect("rayon reversible 5/3 projection succeeds") + .expect("rayon handles benchmark job"), + ); + }); + }, + ); + + if explicit_metal_accepts_reversible_53(job) { + group.bench_with_input( + BenchmarkId::from_parameter(format!("metal_explicit_{dim}x{dim}")), + &job, + |b, job| { + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + b.iter(|| { + std::hint::black_box( + accelerator + .dct_grid_to_reversible_dwt53(std::hint::black_box(*job)) + .expect("explicit Metal reversible 5/3 projection succeeds") + .expect("explicit Metal handles benchmark job"), + ); + }); + }, + ); + } else { + eprintln!("skipping metal_explicit_{dim}x{dim} benchmark: {METAL_UNAVAILABLE}"); + } + } + + group.finish(); +} + +fn bench_reversible_dct53_batch_projection(c: &mut Criterion) { + std::hint::black_box(REVERSIBLE_BATCH_BENCH_MARKERS); + let mut group = c.benchmark_group("reversible_dct53_batch_metal_projection"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(2)); + + for dim in WSI_DIMS { + for batch_size in REVERSIBLE_BATCH_SIZES { + let total_samples = dim.saturating_mul(dim).saturating_mul(batch_size); + if total_samples > MAX_REVERSIBLE_BATCH_SAMPLES { + continue; + } + + let block_cols = dim / 8; + let block_rows = dim / 8; + let batch_blocks: Vec<_> = (0..batch_size) + .map(|idx| { + let offset = + i16::try_from(idx.saturating_mul(3)).expect("benchmark offset fits i16"); + structured_i16_blocks_with_offset(block_cols, block_rows, offset) + }) + .collect(); + let jobs: Vec<_> = batch_blocks + .iter() + .map(|blocks| DctGridToReversibleDwt53Job { + dequantized_blocks: blocks, + block_cols, + block_rows, + width: dim, + height: dim, + }) + .collect(); + + group.throughput(Throughput::Elements(total_samples as u64)); + + group.bench_with_input( + BenchmarkId::from_parameter(format!("rayon_{dim}x{dim}_batch_{batch_size}")), + &jobs, + |b, jobs| { + let mut accelerator = RayonReversibleDwt53Accelerator::default(); + b.iter(|| { + let mut outputs = Vec::with_capacity(jobs.len()); + for job in jobs { + outputs.push( + accelerator + .dct_grid_to_reversible_dwt53(std::hint::black_box(*job)) + .expect("rayon reversible 5/3 batch item succeeds") + .expect("rayon handles reversible 5/3 batch item"), + ); + } + std::hint::black_box(outputs); + }); + }, + ); + + if explicit_metal_accepts_reversible_53_batch(&jobs) { + group.bench_with_input( + BenchmarkId::from_parameter(format!( + "metal_explicit_{dim}x{dim}_batch_{batch_size}" + )), + &jobs, + |b, jobs| { + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + b.iter(|| { + std::hint::black_box( + accelerator + .dct_grid_to_reversible_dwt53_batch(std::hint::black_box(jobs)) + .expect("explicit Metal reversible 5/3 batch succeeds") + .expect("explicit Metal handles benchmark batch"), + ); + }); + }, + ); + } else { + eprintln!( + "skipping metal_explicit_{dim}x{dim}_batch_{batch_size} benchmark: \ + {METAL_UNAVAILABLE}" + ); + } + } + } + + group.finish(); +} + +fn bench_jpeg_to_htj2k_wsi(c: &mut Criterion) { + let mut group = c.benchmark_group("jpeg_to_htj2k_wsi_97"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(2)); + + for spec in WSI_FIXTURES { + let jpeg = encoded_fixture(spec); + group.throughput(Throughput::Bytes(jpeg.len() as u64)); + + group.bench_with_input(BenchmarkId::new(spec.name, "scalar"), &jpeg, |b, jpeg| { + let mut transcoder = JpegToHtj2kTranscoder::default(); + let options = JpegToHtj2kOptions::lossy_97(); + b.iter(|| { + std::hint::black_box( + transcoder + .transcode(std::hint::black_box(jpeg), &options) + .expect("scalar JPEG to HTJ2K 9/7 transcode succeeds"), + ); + }); + }); + + if metal_available() { + group.bench_with_input( + BenchmarkId::new(spec.name, "metal_explicit"), + &jpeg, + |b, jpeg| { + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + let options = JpegToHtj2kOptions::lossy_97(); + b.iter(|| { + std::hint::black_box( + transcoder + .transcode_with_accelerator( + std::hint::black_box(jpeg), + &options, + &mut accelerator, + ) + .expect("Metal JPEG to HTJ2K 9/7 transcode succeeds"), + ); + }); + }, + ); + } else { + eprintln!( + "skipping {}/metal_explicit benchmark: {METAL_UNAVAILABLE}", + spec.name + ); + } + } + + group.finish(); +} + +fn bench_jpeg_to_htj2k_wsi_53(c: &mut Criterion) { + let mut group = c.benchmark_group("jpeg_to_htj2k_wsi_53"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(2)); + + for spec in WSI_FIXTURES { + let jpeg = encoded_fixture(spec); + group.throughput(Throughput::Bytes(jpeg.len() as u64)); + + group.bench_with_input(BenchmarkId::new(spec.name, "scalar"), &jpeg, |b, jpeg| { + let mut transcoder = JpegToHtj2kTranscoder::default(); + let options = JpegToHtj2kOptions { + coefficient_path: JpegToHtj2kCoefficientPath::FloatDirectLinear53, + ..JpegToHtj2kOptions::lossless_53() + }; + b.iter(|| { + std::hint::black_box( + transcoder + .transcode(std::hint::black_box(jpeg), &options) + .expect("scalar JPEG to HTJ2K 5/3 transcode succeeds"), + ); + }); + }); + + if metal_available() { + group.bench_with_input( + BenchmarkId::new(spec.name, "metal_explicit"), + &jpeg, + |b, jpeg| { + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + let options = JpegToHtj2kOptions { + coefficient_path: JpegToHtj2kCoefficientPath::FloatDirectLinear53, + ..JpegToHtj2kOptions::lossless_53() + }; + b.iter(|| { + std::hint::black_box( + transcoder + .transcode_with_accelerator( + std::hint::black_box(jpeg), + &options, + &mut accelerator, + ) + .expect("Metal JPEG to HTJ2K 5/3 transcode succeeds"), + ); + }); + }, + ); + } else { + eprintln!( + "skipping {}/metal_explicit benchmark: {METAL_UNAVAILABLE}", + spec.name + ); + } + } + + group.finish(); +} + +fn bench_jpeg_to_htj2k_wsi_integer_53(c: &mut Criterion) { + let mut group = c.benchmark_group("jpeg_to_htj2k_wsi_integer_53"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(2)); + + for spec in WSI_FIXTURES { + let jpeg = encoded_fixture(spec); + group.throughput(Throughput::Bytes(jpeg.len() as u64)); + + group.bench_with_input(BenchmarkId::new(spec.name, "scalar"), &jpeg, |b, jpeg| { + let mut transcoder = JpegToHtj2kTranscoder::default(); + let options = JpegToHtj2kOptions::lossless_53(); + b.iter(|| { + std::hint::black_box( + transcoder + .transcode(std::hint::black_box(jpeg), &options) + .expect("scalar JPEG to HTJ2K IntegerDirect53 transcode succeeds"), + ); + }); + }); + + group.bench_with_input(BenchmarkId::new(spec.name, "rayon"), &jpeg, |b, jpeg| { + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = RayonReversibleDwt53Accelerator::default(); + let options = JpegToHtj2kOptions::lossless_53(); + b.iter(|| { + std::hint::black_box( + transcoder + .transcode_with_accelerator( + std::hint::black_box(jpeg), + &options, + &mut accelerator, + ) + .expect("rayon JPEG to HTJ2K IntegerDirect53 transcode succeeds"), + ); + }); + }); + + group.bench_with_input( + BenchmarkId::new(spec.name, "metal_auto"), + &jpeg, + |b, jpeg| { + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::for_auto(); + let options = JpegToHtj2kOptions::lossless_53(); + b.iter(|| { + std::hint::black_box( + transcoder + .transcode_with_accelerator( + std::hint::black_box(jpeg), + &options, + &mut accelerator, + ) + .expect("auto Metal JPEG to HTJ2K IntegerDirect53 transcode succeeds"), + ); + }); + }, + ); + + if metal_available() { + group.bench_with_input( + BenchmarkId::new(spec.name, "metal_explicit"), + &jpeg, + |b, jpeg| { + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + let options = JpegToHtj2kOptions::lossless_53(); + b.iter(|| { + std::hint::black_box( + transcoder + .transcode_with_accelerator( + std::hint::black_box(jpeg), + &options, + &mut accelerator, + ) + .expect("Metal JPEG to HTJ2K IntegerDirect53 transcode succeeds"), + ); + }); + }, + ); + } else { + eprintln!( + "skipping {}/metal_explicit benchmark: {METAL_UNAVAILABLE}", + spec.name + ); + } + } + + group.finish(); +} + +fn bench_jpeg_to_htj2k_wsi_integer_53_tile_batch(c: &mut Criterion) { + std::hint::black_box(WSI_TILE_BATCH_BENCH_MARKERS); + let mut group = c.benchmark_group("jpeg_to_htj2k_wsi_integer_53_tile_batch"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(2)); + + for spec in WSI_FIXTURES { + let jpeg = encoded_fixture(spec); + let options = JpegToHtj2kOptions::lossless_53(); + + for batch_size in WSI_TILE_BATCH_SIZES { + let inputs = tile_batch_inputs(&jpeg, batch_size); + let benchmark_name = format!("{}_batch_{}", spec.name, batch_size); + group.throughput(Throughput::Bytes( + u64::try_from(jpeg.len().saturating_mul(batch_size)) + .expect("benchmark input byte count fits u64"), + )); + + group.bench_with_input( + BenchmarkId::new(benchmark_name.as_str(), "rayon_batch"), + &inputs, + |b, inputs| { + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = RayonReversibleDwt53Accelerator::default(); + b.iter(|| { + std::hint::black_box(expect_successful_batch( + transcoder + .transcode_batch_with_accelerator( + std::hint::black_box(inputs), + &options, + &mut accelerator, + ) + .expect("rayon JPEG tile batch transcode succeeds"), + "rayon JPEG tile batch transcode", + )); + }); + }, + ); + + group.bench_with_input( + BenchmarkId::new(benchmark_name.as_str(), "metal_auto_batch"), + &inputs, + |b, inputs| { + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::for_auto(); + b.iter(|| { + std::hint::black_box(expect_successful_batch( + transcoder + .transcode_batch_with_accelerator( + std::hint::black_box(inputs), + &options, + &mut accelerator, + ) + .expect("auto Metal JPEG tile batch transcode succeeds"), + "auto Metal JPEG tile batch transcode", + )); + }); + }, + ); + + if metal_available() { + group.bench_with_input( + BenchmarkId::new(benchmark_name.as_str(), "metal_explicit_batch"), + &inputs, + |b, inputs| { + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + b.iter(|| { + std::hint::black_box(expect_successful_batch( + transcoder + .transcode_batch_with_accelerator( + std::hint::black_box(inputs), + &options, + &mut accelerator, + ) + .expect("explicit Metal JPEG tile batch transcode succeeds"), + "explicit Metal JPEG tile batch transcode", + )); + }); + }, + ); + } else { + eprintln!( + "skipping {benchmark_name}/metal_explicit_batch benchmark: \ + {METAL_UNAVAILABLE}" + ); + } + } + } + + group.finish(); +} + +#[derive(Clone, Copy)] +struct WsiFixtureSpec { + name: &'static str, + dim: usize, + subsampling: JpegSubsampling, + generator: fn(usize) -> Vec, +} + +fn encoded_fixture(spec: WsiFixtureSpec) -> Vec { + let rgb = (spec.generator)(spec.dim); + encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: spec.dim as u32, + height: spec.dim as u32, + }, + JpegEncodeOptions { + quality: 90, + subsampling: spec.subsampling, + restart_interval: Some((spec.dim / 8) as u16), + backend: JpegBackend::Cpu, + }, + ) + .expect("encode benchmark JPEG fixture") + .data +} + +fn tile_batch_inputs(jpeg: &[u8], batch_size: usize) -> Vec> { + vec![JpegTileBatchInput { bytes: jpeg }; batch_size] +} + +fn expect_successful_batch( + batch: EncodedTranscodeBatch, + context: &'static str, +) -> EncodedTranscodeBatch { + assert_eq!( + batch.report.failed_tiles, 0, + "{context} produced {} failed tiles", + batch.report.failed_tiles + ); + assert_eq!( + batch.report.successful_tiles, batch.report.tile_count, + "{context} produced an incomplete successful tile count" + ); + emit_transcode_batch_profile(&batch, context); + batch +} + +fn emit_transcode_batch_profile(batch: &EncodedTranscodeBatch, context: &'static str) { + if std::env::var_os(TRANSCODE_PROFILE_STAGES_ENV).is_none() { + return; + } + + let report = &batch.report; + let timings = report.timings; + let context = context.replace(' ', "_"); + let coefficient_path = format!("{:?}", report.coefficient_path); + let total_us = report + .extract_us + .saturating_add(report.transform_us) + .saturating_add(report.encode_us); + eprintln!( + "j2k_profile codec=transcode op=transcode_batch path=metal_cpu_hybrid pipeline=jpeg_to_htj2k_hybrid context={context} coefficient_path={coefficient_path} extract_processor=cpu transform_processor=metal encode_processor=cpu tile_count={} successful_tiles={} failed_tiles={} transformed_components={} reversible_dwt53_batches={} reversible_dwt53_batch_jobs={} extract_us={} transform_us={} encode_us={} total_us={} source_raw_probe_us={} read_region_decode_us={} compose_pad_us={} generated_jpeg_encode_us={} jpeg_dct_extract_us={} jpeg_dct_repack_us={} dct_to_wavelet_total_us={} dct_to_wavelet_accelerator_us={} dct_to_wavelet_cpu_fallback_us={} dwt_decompose_us={} dwt97_batch_pack_upload_us={} dwt97_batch_idct_row_lift_us={} dwt97_batch_column_lift_us={} dwt97_batch_quantize_codeblock_us={} dwt97_batch_ht_encode_us={} dwt97_batch_ht_codeblock_dispatches={} dwt97_batch_readback_us={} htj2k_encode_us={} htj2k_encode_accelerator_dispatches={} htj2k_encode_ht_code_block_dispatches={} htj2k_encode_packetization_dispatches={} component_count={} batch_count={} batch_jobs={} accelerator_attempts={} accelerator_jobs={} accelerator_dispatches={} accelerator_dispatched_jobs={} cpu_fallback_jobs={}", + report.tile_count, + report.successful_tiles, + report.failed_tiles, + report.transformed_components, + report.reversible_dwt53_batches, + report.reversible_dwt53_batch_jobs, + report.extract_us, + report.transform_us, + report.encode_us, + total_us, + timings.source_raw_probe_us, + timings.read_region_decode_us, + timings.compose_pad_us, + timings.generated_jpeg_encode_us, + timings.jpeg_dct_extract_us, + timings.jpeg_dct_repack_us, + timings.dct_to_wavelet_total_us, + timings.dct_to_wavelet_accelerator_us, + timings.dct_to_wavelet_cpu_fallback_us, + timings.dwt_decompose_us, + timings.dwt97_batch_pack_upload_us, + timings.dwt97_batch_idct_row_lift_us, + timings.dwt97_batch_column_lift_us, + timings.dwt97_batch_quantize_codeblock_us, + timings.dwt97_batch_ht_encode_us, + timings.dwt97_batch_ht_codeblock_dispatches, + timings.dwt97_batch_readback_us, + timings.htj2k_encode_us, + timings.htj2k_encode_accelerator_dispatches, + timings.htj2k_encode_ht_code_block_dispatches, + timings.htj2k_encode_packetization_dispatches, + timings.component_count, + timings.batch_count, + timings.batch_jobs, + timings.accelerator_attempts, + timings.accelerator_jobs, + timings.accelerator_dispatches, + timings.accelerator_dispatched_jobs, + timings.cpu_fallback_jobs, + ); + eprint!("{}", report.pipeline_map().debug_report()); +} + +fn metal_available() -> bool { + #[cfg(target_os = "macos")] + { + metal::Device::system_default().is_some() + } + #[cfg(not(target_os = "macos"))] + { + false + } +} + +fn rgb_srgb_pattern(dim: usize) -> Vec { + let mut data = Vec::with_capacity(dim * dim * 3); + for y in 0..dim { + for x in 0..dim { + data.push(((x * 5 + y * 3 + 17) & 0xff) as u8); + data.push(((x * 2 + y * 7 + 41) & 0xff) as u8); + data.push(((x * 11 + y * 13 + 73) & 0xff) as u8); + } + } + data +} + +fn rgb_p3_like_pattern(dim: usize) -> Vec { + let mut data = Vec::with_capacity(dim * dim * 3); + for y in 0..dim { + for x in 0..dim { + let radial = ((x ^ y) & 0xff) as u8; + data.push(radial.saturating_add(32)); + data.push(((x * 9 + y * 5 + 19) & 0xff) as u8); + data.push(((x * 3 + y * 15 + 97) & 0xff) as u8); + } + } + data +} + +fn rgb_ycbcr_like_pattern(dim: usize) -> Vec { + let mut data = Vec::with_capacity(dim * dim * 3); + for y in 0..dim { + for x in 0..dim { + let y_sample = i32::from(((x * 3 + y * 2 + ((x / 31 + y / 17) * 23)) & 0xff) as u8); + let cb = i32::from((((x / 8) * 9 + y * 2 + 96) & 0xff) as u8) - 128; + let cr = i32::from(((x * 2 + (y / 8) * 11 + 160) & 0xff) as u8) - 128; + let r = y_sample + ((91_881 * cr) >> 16); + let g = y_sample - ((22_554 * cb + 46_802 * cr) >> 16); + let b = y_sample + ((116_130 * cb) >> 16); + data.push(clamp_u8(r)); + data.push(clamp_u8(g)); + data.push(clamp_u8(b)); + } + } + data +} + +fn clamp_u8(value: i32) -> u8 { + u8::try_from(value.clamp(0, 255)).expect("clamped value fits in u8") +} + +fn explicit_metal_accepts(job: DctGridToDwt97Job<'_>) -> bool { + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + matches!(accelerator.dct_grid_to_dwt97(job), Ok(Some(_))) +} + +fn explicit_metal_accepts_53(job: DctGridToDwt53Job<'_>) -> bool { + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + matches!(accelerator.dct_grid_to_dwt53(job), Ok(Some(_))) +} + +fn explicit_metal_accepts_reversible_53(job: DctGridToReversibleDwt53Job<'_>) -> bool { + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + matches!(accelerator.dct_grid_to_reversible_dwt53(job), Ok(Some(_))) +} + +fn explicit_metal_accepts_reversible_53_batch(jobs: &[DctGridToReversibleDwt53Job<'_>]) -> bool { + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + matches!( + accelerator.dct_grid_to_reversible_dwt53_batch(jobs), + Ok(Some(_)) + ) +} + +fn structured_blocks(block_cols: usize, block_rows: usize) -> Vec<[[f64; 8]; 8]> { + let mut blocks = Vec::with_capacity(block_cols * block_rows); + for block_y in 0..block_rows { + for block_x in 0..block_cols { + let mut block = [[0.0; 8]; 8]; + block[0][0] = 384.0 + (block_x * 19 + block_y * 23) as f64; + block[0][1] = -17.0 + block_x as f64; + block[1][0] = 11.0 - block_y as f64; + block[2][3] = 7.0; + block[4][4] = -3.0; + block[7][7] = 2.0; + blocks.push(block); + } + } + blocks +} + +fn structured_i16_blocks(block_cols: usize, block_rows: usize) -> Vec<[i16; 64]> { + structured_i16_blocks_with_offset(block_cols, block_rows, 0) +} + +fn structured_i16_blocks_with_offset( + block_cols: usize, + block_rows: usize, + base_offset: i16, +) -> Vec<[i16; 64]> { + let mut blocks = Vec::with_capacity(block_cols * block_rows); + for block_y in 0..block_rows { + for block_x in 0..block_cols { + let mut block = [0i16; 64]; + let block_offset = + i16::try_from(block_x * 19 + block_y * 23).expect("fixture offset fits i16"); + let x_offset = i16::try_from(block_x).expect("fixture x offset fits i16"); + let y_offset = i16::try_from(block_y).expect("fixture y offset fits i16"); + block[0] = 384 + base_offset + block_offset; + block[1] = -17 + x_offset; + block[8] = 11 - y_offset; + block[19] = 7; + block[36] = -3; + block[63] = 2; + blocks.push(block); + } + } + blocks +} + +criterion_group!(dct53_metal_projection, bench_dct53_projection); +criterion_group!(dct97_metal_idct_dwt, bench_dct97_idct_dwt); +criterion_group!(jpeg_to_htj2k_wsi_53, bench_jpeg_to_htj2k_wsi_53); +criterion_group!( + reversible_dct53_metal_projection, + bench_reversible_dct53_projection +); +criterion_group!( + reversible_dct53_batch_metal_projection, + bench_reversible_dct53_batch_projection +); +criterion_group!( + jpeg_to_htj2k_wsi_integer_53, + bench_jpeg_to_htj2k_wsi_integer_53 +); +criterion_group!( + jpeg_to_htj2k_wsi_integer_53_tile_batch, + bench_jpeg_to_htj2k_wsi_integer_53_tile_batch +); +criterion_group!(jpeg_to_htj2k_wsi_97, bench_jpeg_to_htj2k_wsi); +criterion_main!( + dct53_metal_projection, + dct97_metal_idct_dwt, + reversible_dct53_metal_projection, + reversible_dct53_batch_metal_projection, + jpeg_to_htj2k_wsi_53, + jpeg_to_htj2k_wsi_integer_53, + jpeg_to_htj2k_wsi_integer_53_tile_batch, + jpeg_to_htj2k_wsi_97 +); diff --git a/crates/j2k-transcode-metal/fixtures/conformance/baseline_420_16x16.jpg b/crates/j2k-transcode-metal/fixtures/conformance/baseline_420_16x16.jpg new file mode 100644 index 00000000..af176169 Binary files /dev/null and b/crates/j2k-transcode-metal/fixtures/conformance/baseline_420_16x16.jpg differ diff --git a/crates/j2k-transcode-metal/fixtures/conformance/baseline_422_16x8.jpg b/crates/j2k-transcode-metal/fixtures/conformance/baseline_422_16x8.jpg new file mode 100644 index 00000000..60b479c5 Binary files /dev/null and b/crates/j2k-transcode-metal/fixtures/conformance/baseline_422_16x8.jpg differ diff --git a/crates/j2k-transcode-metal/fixtures/conformance/baseline_444_8x8.jpg b/crates/j2k-transcode-metal/fixtures/conformance/baseline_444_8x8.jpg new file mode 100644 index 00000000..0f4b494a Binary files /dev/null and b/crates/j2k-transcode-metal/fixtures/conformance/baseline_444_8x8.jpg differ diff --git a/crates/j2k-transcode-metal/fixtures/conformance/grayscale_8x8.jpg b/crates/j2k-transcode-metal/fixtures/conformance/grayscale_8x8.jpg new file mode 100644 index 00000000..80cecf54 Binary files /dev/null and b/crates/j2k-transcode-metal/fixtures/conformance/grayscale_8x8.jpg differ diff --git a/crates/j2k-transcode-metal/src/dct97.metal b/crates/j2k-transcode-metal/src/dct97.metal new file mode 100644 index 00000000..737657d5 --- /dev/null +++ b/crates/j2k-transcode-metal/src/dct97.metal @@ -0,0 +1,572 @@ +// SPDX-License-Identifier: Apache-2.0 + +#include + +using namespace metal; + +struct Dct97ProjectionParams { + uint width; + uint height; + uint block_cols; + uint band_width; + uint band_height; +}; + +struct Dct97BatchProjectionParams { + uint width; + uint height; + uint block_cols; + uint blocks_per_item; + uint band_width; + uint band_height; + uint output_stride; +}; + +struct Dct97IdctRowLiftParams { + uint width; + uint height; + uint block_cols; + uint blocks_per_item; + uint low_width; + uint high_width; +}; + +struct Dct97ColumnLiftParams { + uint height; + uint low_width; + uint high_width; + uint low_height; + uint high_height; + uint row_low_stride; + uint row_high_stride; + uint ll_stride; + uint hl_stride; + uint lh_stride; + uint hh_stride; +}; + +struct Dct97QuantizeCodeblocksParams { + uint band_width; + uint band_height; + uint output_stride; + uint code_block_width; + uint code_block_height; + float inv_delta; +}; + +struct Reversible53ProjectionParams { + uint width; + uint height; + uint block_cols; + uint blocks_per_item; + uint band_width; + uint band_height; + uint output_stride; + uint vertical_low; + uint horizontal_low; +}; + +struct Dct97SparseRow { + uint offset; + uint count; +}; + +struct Dct97WeightTap { + uint sample_idx; + float weight; +}; + +#define DCT97_STAGED_MAX_AXIS 1024 +#define DCT97_ROWS_PER_GROUP 2 +#define DCT97_COLUMNS_PER_GROUP 4 +#define DCT97_THREADS_PER_GROUP 256 + +constant float DCT97_ALPHA = -1.586134342059924f; +constant float DCT97_BETA = -0.052980118572961f; +constant float DCT97_GAMMA = 0.882911075530934f; +constant float DCT97_DELTA = 0.443506852043971f; +constant float DCT97_KAPPA = 1.230174104914001f; +constant float DCT97_INV_KAPPA = 1.0f / DCT97_KAPPA; + +static inline float dct97_idct_sample( + device const float *blocks, + device const float *idct_basis, + constant Dct97IdctRowLiftParams ¶ms, + uint item_idx, + uint x, + uint y +) { + const uint block_x = x / 8u; + const uint block_y = y / 8u; + const uint local_x = x % 8u; + const uint local_y = y % 8u; + const uint block_base = + (item_idx * params.blocks_per_item + block_y * params.block_cols + block_x) * 64u; + + float sample = 0.0f; + for (uint freq_y = 0; freq_y < 8u; ++freq_y) { + const float y_basis = idct_basis[local_y * 8u + freq_y]; + for (uint freq_x = 0; freq_x < 8u; ++freq_x) { + const float coefficient = blocks[block_base + freq_y * 8u + freq_x]; + sample += coefficient * y_basis * idct_basis[local_x * 8u + freq_x]; + } + } + return sample; +} + +static inline void dct97_forward_lift_in_threadgroup( + threadgroup float *data, + uint n, + uint thread_idx, + uint threads_per_group +) { + if (n < 2u) { + return; + } + + const uint last_even = ((n % 2u) == 0u) ? n - 2u : n - 1u; + + for (uint i = 1u + thread_idx * 2u; i < n; i += threads_per_group * 2u) { + const float left = data[i - 1u]; + const float right = (i + 1u < n) ? data[i + 1u] : data[last_even]; + data[i] += DCT97_ALPHA * (left + right); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint i = thread_idx * 2u; i < n; i += threads_per_group * 2u) { + const float left = (i > 0u) ? data[i - 1u] : data[1u]; + const float right = (i + 1u < n) ? data[i + 1u] : left; + data[i] += DCT97_BETA * (left + right); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint i = 1u + thread_idx * 2u; i < n; i += threads_per_group * 2u) { + const float left = data[i - 1u]; + const float right = (i + 1u < n) ? data[i + 1u] : data[last_even]; + data[i] += DCT97_GAMMA * (left + right); + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint i = thread_idx * 2u; i < n; i += threads_per_group * 2u) { + const float left = (i > 0u) ? data[i - 1u] : data[1u]; + const float right = (i + 1u < n) ? data[i + 1u] : left; + data[i] += DCT97_DELTA * (left + right); + } + threadgroup_barrier(mem_flags::mem_threadgroup); +} + +kernel void dct97_idct_row_lift_batch( + device const float *blocks [[buffer(0)]], + device const float *idct_basis [[buffer(1)]], + device float *row_low [[buffer(2)]], + device float *row_high [[buffer(3)]], + constant Dct97IdctRowLiftParams ¶ms [[buffer(4)]], + uint3 group_id [[threadgroup_position_in_grid]], + uint thread_idx [[thread_index_in_threadgroup]] +) { + threadgroup float rows[DCT97_ROWS_PER_GROUP][DCT97_STAGED_MAX_AXIS]; + + const uint row_group = group_id.x; + const uint item_idx = group_id.y; + if (params.width > DCT97_STAGED_MAX_AXIS) { + return; + } + + for (uint row_offset = 0u; row_offset < DCT97_ROWS_PER_GROUP; ++row_offset) { + const uint y = row_group * DCT97_ROWS_PER_GROUP + row_offset; + if (y >= params.height) { + continue; + } + for (uint x = thread_idx; x < params.width; x += DCT97_THREADS_PER_GROUP) { + rows[row_offset][x] = dct97_idct_sample(blocks, idct_basis, params, item_idx, x, y); + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint row_offset = 0u; row_offset < DCT97_ROWS_PER_GROUP; ++row_offset) { + const uint y = row_group * DCT97_ROWS_PER_GROUP + row_offset; + if (y < params.height) { + dct97_forward_lift_in_threadgroup( + rows[row_offset], params.width, thread_idx, DCT97_THREADS_PER_GROUP); + } + } + + for (uint row_offset = 0u; row_offset < DCT97_ROWS_PER_GROUP; ++row_offset) { + const uint y = row_group * DCT97_ROWS_PER_GROUP + row_offset; + if (y >= params.height) { + continue; + } + const uint low_base = item_idx * params.height * params.low_width + y * params.low_width; + const uint high_base = item_idx * params.height * params.high_width + y * params.high_width; + for (uint low_x = thread_idx; low_x < params.low_width; low_x += DCT97_THREADS_PER_GROUP) { + row_low[low_base + low_x] = rows[row_offset][low_x * 2u] * DCT97_INV_KAPPA; + } + for (uint high_x = thread_idx; high_x < params.high_width; high_x += DCT97_THREADS_PER_GROUP) { + row_high[high_base + high_x] = rows[row_offset][high_x * 2u + 1u] * DCT97_KAPPA; + } + } +} + +kernel void dct97_column_lift_batch( + device const float *row_low [[buffer(0)]], + device const float *row_high [[buffer(1)]], + device float *ll [[buffer(2)]], + device float *hl [[buffer(3)]], + device float *lh [[buffer(4)]], + device float *hh [[buffer(5)]], + constant Dct97ColumnLiftParams ¶ms [[buffer(6)]], + uint3 group_id [[threadgroup_position_in_grid]], + uint thread_idx [[thread_index_in_threadgroup]] +) { + threadgroup float columns[DCT97_COLUMNS_PER_GROUP][DCT97_STAGED_MAX_AXIS]; + + const uint column_group = group_id.x; + const uint item_idx = group_id.y; + const bool horizontal_low = group_id.z == 0u; + const uint band_width = horizontal_low ? params.low_width : params.high_width; + if (params.height > DCT97_STAGED_MAX_AXIS) { + return; + } + + for (uint column_offset = 0u; column_offset < DCT97_COLUMNS_PER_GROUP; ++column_offset) { + const uint x = column_group * DCT97_COLUMNS_PER_GROUP + column_offset; + if (x >= band_width) { + continue; + } + const uint source_stride = horizontal_low ? params.row_low_stride : params.row_high_stride; + device const float *source = horizontal_low ? row_low : row_high; + const uint source_width = band_width; + const uint source_base = item_idx * source_stride + x; + for (uint y = thread_idx; y < params.height; y += DCT97_THREADS_PER_GROUP) { + columns[column_offset][y] = source[source_base + y * source_width]; + } + } + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint column_offset = 0u; column_offset < DCT97_COLUMNS_PER_GROUP; ++column_offset) { + const uint x = column_group * DCT97_COLUMNS_PER_GROUP + column_offset; + if (x < band_width) { + dct97_forward_lift_in_threadgroup( + columns[column_offset], params.height, thread_idx, DCT97_THREADS_PER_GROUP); + } + } + + for (uint column_offset = 0u; column_offset < DCT97_COLUMNS_PER_GROUP; ++column_offset) { + const uint x = column_group * DCT97_COLUMNS_PER_GROUP + column_offset; + if (x >= band_width) { + continue; + } + for (uint low_y = thread_idx; low_y < params.low_height; low_y += DCT97_THREADS_PER_GROUP) { + const float value = columns[column_offset][low_y * 2u] * DCT97_INV_KAPPA; + if (horizontal_low) { + ll[item_idx * params.ll_stride + low_y * params.low_width + x] = value; + } else { + hl[item_idx * params.hl_stride + low_y * params.high_width + x] = value; + } + } + for (uint high_y = thread_idx; high_y < params.high_height; high_y += DCT97_THREADS_PER_GROUP) { + const float value = columns[column_offset][high_y * 2u + 1u] * DCT97_KAPPA; + if (horizontal_low) { + lh[item_idx * params.lh_stride + high_y * params.low_width + x] = value; + } else { + hh[item_idx * params.hh_stride + high_y * params.high_width + x] = value; + } + } + } +} + +kernel void dct97_quantize_codeblocks_batch( + device const float *band [[buffer(0)]], + device int *output [[buffer(1)]], + constant Dct97QuantizeCodeblocksParams ¶ms [[buffer(2)]], + uint3 gid [[thread_position_in_grid]] +) { + const uint x = gid.x; + const uint y = gid.y; + const uint item_idx = gid.z; + if (x >= params.band_width || y >= params.band_height) { + return; + } + + const float value = + band[item_idx * params.band_width * params.band_height + y * params.band_width + x]; + const int sign = value < 0.0f ? -1 : 1; + const int magnitude = int(floor(fabs(value) * params.inv_delta)); + + const uint cbx = x / params.code_block_width; + const uint cby = y / params.code_block_height; + const uint local_x = x - cbx * params.code_block_width; + const uint local_y = y - cby * params.code_block_height; + const uint block_x0 = cbx * params.code_block_width; + const uint block_y0 = cby * params.code_block_height; + const uint block_width = min(params.code_block_width, params.band_width - block_x0); + const uint block_height = min(params.code_block_height, params.band_height - block_y0); + const uint item_base = item_idx * params.output_stride; + const uint codeblock_row_base = cby * params.code_block_height * params.band_width; + const uint codeblock_base = codeblock_row_base + cbx * params.code_block_width * block_height; + const uint block_offset = local_y * block_width + local_x; + + output[item_base + codeblock_base + block_offset] = sign * magnitude; +} + +kernel void dct97_project_band( + device const float *blocks [[buffer(0)]], + device const Dct97SparseRow *x_rows [[buffer(1)]], + device const Dct97WeightTap *x_taps [[buffer(2)]], + device const Dct97SparseRow *y_rows [[buffer(3)]], + device const Dct97WeightTap *y_taps [[buffer(4)]], + device const float *idct_basis [[buffer(5)]], + device float *output [[buffer(6)]], + constant Dct97ProjectionParams ¶ms [[buffer(7)]], + uint2 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.band_width || gid.y >= params.band_height) { + return; + } + + const Dct97SparseRow x_row = x_rows[gid.x]; + const Dct97SparseRow y_row = y_rows[gid.y]; + float value = 0.0f; + for (uint y_tap_idx = 0; y_tap_idx < y_row.count; ++y_tap_idx) { + const Dct97WeightTap y_tap = y_taps[y_row.offset + y_tap_idx]; + const uint sample_y = y_tap.sample_idx; + const float y_weight = y_tap.weight; + const uint block_y = sample_y / 8u; + const uint local_y = sample_y % 8u; + + for (uint x_tap_idx = 0; x_tap_idx < x_row.count; ++x_tap_idx) { + const Dct97WeightTap x_tap = x_taps[x_row.offset + x_tap_idx]; + const uint sample_x = x_tap.sample_idx; + const float x_weight = x_tap.weight; + const uint block_x = sample_x / 8u; + const uint local_x = sample_x % 8u; + const uint block_base = (block_y * params.block_cols + block_x) * 64u; + const float sample_weight = y_weight * x_weight; + + for (uint freq_y = 0; freq_y < 8u; ++freq_y) { + const float y_basis = idct_basis[local_y * 8u + freq_y]; + for (uint freq_x = 0; freq_x < 8u; ++freq_x) { + const float coefficient = blocks[block_base + freq_y * 8u + freq_x]; + const float x_basis = idct_basis[local_x * 8u + freq_x]; + value += sample_weight * y_basis * x_basis * coefficient; + } + } + } + } + + output[gid.y * params.band_width + gid.x] = value; +} + +kernel void dct97_project_band_batch( + device const float *blocks [[buffer(0)]], + device const Dct97SparseRow *x_rows [[buffer(1)]], + device const Dct97WeightTap *x_taps [[buffer(2)]], + device const Dct97SparseRow *y_rows [[buffer(3)]], + device const Dct97WeightTap *y_taps [[buffer(4)]], + device const float *idct_basis [[buffer(5)]], + device float *output [[buffer(6)]], + constant Dct97BatchProjectionParams ¶ms [[buffer(7)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.band_width || gid.y >= params.band_height) { + return; + } + + const Dct97SparseRow x_row = x_rows[gid.x]; + const Dct97SparseRow y_row = y_rows[gid.y]; + const uint item_base = gid.z * params.blocks_per_item; + float value = 0.0f; + for (uint y_tap_idx = 0; y_tap_idx < y_row.count; ++y_tap_idx) { + const Dct97WeightTap y_tap = y_taps[y_row.offset + y_tap_idx]; + const uint sample_y = y_tap.sample_idx; + const float y_weight = y_tap.weight; + const uint block_y = sample_y / 8u; + const uint local_y = sample_y % 8u; + + for (uint x_tap_idx = 0; x_tap_idx < x_row.count; ++x_tap_idx) { + const Dct97WeightTap x_tap = x_taps[x_row.offset + x_tap_idx]; + const uint sample_x = x_tap.sample_idx; + const float x_weight = x_tap.weight; + const uint block_x = sample_x / 8u; + const uint local_x = sample_x % 8u; + const uint block_base = (item_base + block_y * params.block_cols + block_x) * 64u; + const float sample_weight = y_weight * x_weight; + + for (uint freq_y = 0; freq_y < 8u; ++freq_y) { + const float y_basis = idct_basis[local_y * 8u + freq_y]; + for (uint freq_x = 0; freq_x < 8u; ++freq_x) { + const float coefficient = blocks[block_base + freq_y * 8u + freq_x]; + const float x_basis = idct_basis[local_x * 8u + freq_x]; + value += sample_weight * y_basis * x_basis * coefficient; + } + } + } + } + + output[gid.z * params.output_stride + gid.y * params.band_width + gid.x] = value; +} + +static inline int floor_div_i32(int numerator, int denominator) { + const int quotient = numerator / denominator; + const int remainder = numerator % denominator; + return (remainder < 0) ? quotient - 1 : quotient; +} + +static inline int reversible53_sample( + device const int *blocks, + uint block_cols, + uint blocks_per_item, + uint item_idx, + uint x, + uint y +) { + const uint block_x = x / 8u; + const uint block_y = y / 8u; + const uint local_x = x % 8u; + const uint local_y = y % 8u; + const uint item_block_base = item_idx * blocks_per_item; + const uint block_base = (item_block_base + block_y * block_cols + block_x) * 64u; + return blocks[block_base + local_y * 8u + local_x]; +} + +static inline int reversible53_vertical_high( + device const int *blocks, + constant Reversible53ProjectionParams ¶ms, + uint item_idx, + uint x, + uint high_idx +) { + const uint odd_idx = high_idx * 2u + 1u; + const int current = reversible53_sample( + blocks, params.block_cols, params.blocks_per_item, item_idx, x, odd_idx); + const int left = reversible53_sample( + blocks, params.block_cols, params.blocks_per_item, item_idx, x, odd_idx - 1u); + if ((params.height % 2u) == 0u && odd_idx + 1u == params.height) { + return current - left; + } + + const uint right_idx = (odd_idx + 1u < params.height) ? odd_idx + 1u : params.height - 1u; + const int right = reversible53_sample( + blocks, params.block_cols, params.blocks_per_item, item_idx, x, right_idx); + return current - floor_div_i32(left + right, 2); +} + +static inline int reversible53_vertical_low( + device const int *blocks, + constant Reversible53ProjectionParams ¶ms, + uint item_idx, + uint x, + uint low_idx +) { + const uint even_idx = low_idx * 2u; + const int current = reversible53_sample( + blocks, params.block_cols, params.blocks_per_item, item_idx, x, even_idx); + if (params.height < 2u) { + return current; + } + + if ((params.height % 2u) == 0u) { + const int right = reversible53_vertical_high(blocks, params, item_idx, x, low_idx); + if (low_idx == 0u) { + return current + floor_div_i32(right + 1, 2); + } + const int left = reversible53_vertical_high(blocks, params, item_idx, x, low_idx - 1u); + return current + floor_div_i32(left + right + 2, 4); + } + + const uint high_len = params.height / 2u; + if (high_len == 0u) { + return current; + } + const int left = low_idx > 0u + ? reversible53_vertical_high(blocks, params, item_idx, x, low_idx - 1u) + : reversible53_vertical_high(blocks, params, item_idx, x, 0u); + const int right = low_idx < high_len + ? reversible53_vertical_high(blocks, params, item_idx, x, low_idx) + : left; + return current + floor_div_i32(left + right + 2, 4); +} + +static inline int reversible53_vertical_value( + device const int *blocks, + constant Reversible53ProjectionParams ¶ms, + uint item_idx, + uint x, + uint output_y +) { + return params.vertical_low != 0u + ? reversible53_vertical_low(blocks, params, item_idx, x, output_y) + : reversible53_vertical_high(blocks, params, item_idx, x, output_y); +} + +static inline int reversible53_horizontal_high( + device const int *blocks, + constant Reversible53ProjectionParams ¶ms, + uint item_idx, + uint high_idx, + uint output_y +) { + const uint odd_idx = high_idx * 2u + 1u; + const int current = reversible53_vertical_value(blocks, params, item_idx, odd_idx, output_y); + const int left = reversible53_vertical_value(blocks, params, item_idx, odd_idx - 1u, output_y); + if ((params.width % 2u) == 0u && odd_idx + 1u == params.width) { + return current - left; + } + + const uint right_idx = (odd_idx + 1u < params.width) ? odd_idx + 1u : params.width - 1u; + const int right = reversible53_vertical_value(blocks, params, item_idx, right_idx, output_y); + return current - floor_div_i32(left + right, 2); +} + +static inline int reversible53_horizontal_low( + device const int *blocks, + constant Reversible53ProjectionParams ¶ms, + uint item_idx, + uint low_idx, + uint output_y +) { + const uint even_idx = low_idx * 2u; + const int current = reversible53_vertical_value(blocks, params, item_idx, even_idx, output_y); + if (params.width < 2u) { + return current; + } + + if ((params.width % 2u) == 0u) { + const int right = reversible53_horizontal_high( + blocks, params, item_idx, low_idx, output_y); + if (low_idx == 0u) { + return current + floor_div_i32(right + 1, 2); + } + const int left = reversible53_horizontal_high( + blocks, params, item_idx, low_idx - 1u, output_y); + return current + floor_div_i32(left + right + 2, 4); + } + + const uint high_len = params.width / 2u; + if (high_len == 0u) { + return current; + } + const int left = low_idx > 0u + ? reversible53_horizontal_high(blocks, params, item_idx, low_idx - 1u, output_y) + : reversible53_horizontal_high(blocks, params, item_idx, 0u, output_y); + const int right = low_idx < high_len + ? reversible53_horizontal_high(blocks, params, item_idx, low_idx, output_y) + : left; + return current + floor_div_i32(left + right + 2, 4); +} + +kernel void reversible53_project_band( + device const int *blocks [[buffer(0)]], + device int *output [[buffer(1)]], + constant Reversible53ProjectionParams ¶ms [[buffer(2)]], + uint3 gid [[thread_position_in_grid]] +) { + if (gid.x >= params.band_width || gid.y >= params.band_height) { + return; + } + + const int value = params.horizontal_low != 0u + ? reversible53_horizontal_low(blocks, params, gid.z, gid.x, gid.y) + : reversible53_horizontal_high(blocks, params, gid.z, gid.x, gid.y); + output[gid.z * params.output_stride + gid.y * params.band_width + gid.x] = value; +} diff --git a/crates/j2k-transcode-metal/src/lib.rs b/crates/j2k-transcode-metal/src/lib.rs new file mode 100644 index 00000000..e3c7779f --- /dev/null +++ b/crates/j2k-transcode-metal/src/lib.rs @@ -0,0 +1,680 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Metal acceleration for coefficient-domain JPEG to HTJ2K transcode stages. +//! +//! The supported targets are direct DCT-grid to one-level 5/3 and 9/7 wavelet +//! projections used by `j2k-transcode`'s HTJ2K paths. CPU scalar code +//! remains the oracle and fallback. + +#[cfg(target_os = "macos")] +mod metal; + +#[doc(hidden)] +pub mod weights; + +#[cfg(target_os = "macos")] +pub use metal::MetalTranscodeSession; + +use core::fmt; + +use j2k_transcode::accelerator::{ + DctGridToDwt53Job, DctGridToDwt97Job, DctGridToHtj2k97CodeBlockJob, + DctGridToReversibleDwt53Job, DctToWaveletStageAccelerator, Dwt97BatchStageTimings, + Htj2k97CodeBlockOptions, PrequantizedHtj2k97Component, ReversibleDwt53FirstLevel, + TranscodeStageError, +}; +use j2k_transcode::dct53_2d::Dwt53TwoDimensional; +use j2k_transcode::dct97_2d::Dwt97TwoDimensional; + +/// Stable message returned when Metal is unavailable. +pub const METAL_UNAVAILABLE: &str = "Metal is unavailable on this host"; + +const DEFAULT_AUTO_MIN_SAMPLES: usize = 224 * 224; +const DEFAULT_AUTO_DWT97_MIN_SAMPLES: usize = usize::MAX; +const DEFAULT_AUTO_REVERSIBLE_MIN_SAMPLES: usize = usize::MAX; +const DEFAULT_AUTO_REVERSIBLE_BATCH_MIN_JOBS: usize = 32; +const DEFAULT_AUTO_REVERSIBLE_BATCH_MIN_SAMPLES: usize = 224 * 224 * 32; +const DEFAULT_AUTO_DWT97_BATCH_MIN_JOBS: usize = 32; +const DEFAULT_AUTO_DWT97_BATCH_MIN_SAMPLES: usize = 224 * 224 * 32; +const MAX_AUTO_DWT97_STAGED_BATCH_AXIS: usize = 1024; + +/// Error returned by the Metal transcode accelerator. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MetalTranscodeError { + /// Metal is unavailable on this host or target. + MetalUnavailable, + /// The request is outside the current Metal implementation. + UnsupportedJob(&'static str), + /// Metal runtime creation or device setup failed. + Runtime(&'static str), + /// Metal runtime or kernel execution failed. + Kernel(&'static str), +} + +impl fmt::Display for MetalTranscodeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MetalUnavailable => f.write_str(METAL_UNAVAILABLE), + Self::UnsupportedJob(reason) | Self::Runtime(reason) | Self::Kernel(reason) => { + f.write_str(reason) + } + } + } +} + +impl From for TranscodeStageError { + fn from(error: MetalTranscodeError) -> Self { + match error { + MetalTranscodeError::MetalUnavailable => Self::DeviceUnavailable, + MetalTranscodeError::UnsupportedJob(reason) => Self::Unsupported(reason), + MetalTranscodeError::Runtime(reason) | MetalTranscodeError::Kernel(reason) => { + Self::Backend(reason.to_string()) + } + } + } +} + +impl std::error::Error for MetalTranscodeError {} + +#[cfg(not(target_os = "macos"))] +#[derive(Clone, Copy, Debug, Default)] +/// Placeholder Metal transcode session for hosts without Metal support. +pub struct MetalTranscodeSession { + _private: (), +} + +#[cfg(not(target_os = "macos"))] +impl MetalTranscodeSession { + /// Return `MetalUnavailable` on hosts without Metal support. + pub const fn system_default() -> Result { + Err(MetalTranscodeError::MetalUnavailable) + } +} + +/// Optional Metal accelerator for `j2k-transcode` transform stages. +#[derive(Debug, Clone)] +pub struct MetalDctToWaveletStageAccelerator { + mode: MetalDispatchMode, + min_auto_samples: usize, + min_auto_dwt97_samples: usize, + min_auto_reversible_samples: usize, + min_auto_reversible_batch_jobs: usize, + min_auto_reversible_batch_samples: usize, + reversible_dwt53_attempts: usize, + reversible_dwt53_dispatches: usize, + reversible_dwt53_batch_attempts: usize, + reversible_dwt53_batch_dispatches: usize, + dwt53_attempts: usize, + dwt53_dispatches: usize, + dwt97_attempts: usize, + dwt97_dispatches: usize, + dwt97_batch_attempts: usize, + dwt97_batch_dispatches: usize, + htj2k97_codeblock_batch_attempts: usize, + htj2k97_codeblock_batch_dispatches: usize, + last_dwt97_batch_stage_timings: Option, + min_auto_dwt97_batch_jobs: usize, + min_auto_dwt97_batch_samples: usize, + #[cfg(target_os = "macos")] + session: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MetalDispatchMode { + Explicit, + Auto, +} + +impl MetalDctToWaveletStageAccelerator { + /// Create an accelerator that treats unsupported Metal dispatch as an + /// error. + #[must_use] + pub const fn new_explicit() -> Self { + Self { + mode: MetalDispatchMode::Explicit, + min_auto_samples: 0, + min_auto_dwt97_samples: 0, + min_auto_reversible_samples: 0, + min_auto_reversible_batch_jobs: 0, + min_auto_reversible_batch_samples: 0, + reversible_dwt53_attempts: 0, + reversible_dwt53_dispatches: 0, + reversible_dwt53_batch_attempts: 0, + reversible_dwt53_batch_dispatches: 0, + dwt53_attempts: 0, + dwt53_dispatches: 0, + dwt97_attempts: 0, + dwt97_dispatches: 0, + dwt97_batch_attempts: 0, + dwt97_batch_dispatches: 0, + htj2k97_codeblock_batch_attempts: 0, + htj2k97_codeblock_batch_dispatches: 0, + last_dwt97_batch_stage_timings: None, + min_auto_dwt97_batch_jobs: 0, + min_auto_dwt97_batch_samples: 0, + #[cfg(target_os = "macos")] + session: None, + } + } + + /// Create an accelerator that falls back to scalar CPU for small or + /// unsupported jobs. + #[must_use] + pub const fn for_auto() -> Self { + Self { + mode: MetalDispatchMode::Auto, + min_auto_samples: DEFAULT_AUTO_MIN_SAMPLES, + min_auto_dwt97_samples: DEFAULT_AUTO_DWT97_MIN_SAMPLES, + min_auto_reversible_samples: DEFAULT_AUTO_REVERSIBLE_MIN_SAMPLES, + min_auto_reversible_batch_jobs: DEFAULT_AUTO_REVERSIBLE_BATCH_MIN_JOBS, + min_auto_reversible_batch_samples: DEFAULT_AUTO_REVERSIBLE_BATCH_MIN_SAMPLES, + reversible_dwt53_attempts: 0, + reversible_dwt53_dispatches: 0, + reversible_dwt53_batch_attempts: 0, + reversible_dwt53_batch_dispatches: 0, + dwt53_attempts: 0, + dwt53_dispatches: 0, + dwt97_attempts: 0, + dwt97_dispatches: 0, + dwt97_batch_attempts: 0, + dwt97_batch_dispatches: 0, + htj2k97_codeblock_batch_attempts: 0, + htj2k97_codeblock_batch_dispatches: 0, + last_dwt97_batch_stage_timings: None, + min_auto_dwt97_batch_jobs: DEFAULT_AUTO_DWT97_BATCH_MIN_JOBS, + min_auto_dwt97_batch_samples: DEFAULT_AUTO_DWT97_BATCH_MIN_SAMPLES, + #[cfg(target_os = "macos")] + session: None, + } + } + + /// Create an explicit-dispatch accelerator bound to a caller-owned Metal session. + #[cfg(target_os = "macos")] + #[must_use] + pub fn new_explicit_with_session(session: MetalTranscodeSession) -> Self { + Self::new_explicit().with_session(session) + } + + /// Create an Auto-mode accelerator bound to a caller-owned Metal session. + #[cfg(target_os = "macos")] + #[must_use] + pub fn for_auto_with_session(session: MetalTranscodeSession) -> Self { + Self::for_auto().with_session(session) + } + + /// Create an explicit-dispatch accelerator bound to an existing Metal device. + #[cfg(target_os = "macos")] + #[must_use] + pub fn new_explicit_with_device(device: ::metal::Device) -> Self { + Self::new_explicit_with_session(MetalTranscodeSession::new(device)) + } + + /// Create an Auto-mode accelerator bound to an existing Metal device. + #[cfg(target_os = "macos")] + #[must_use] + pub fn for_auto_with_device(device: ::metal::Device) -> Self { + Self::for_auto_with_session(MetalTranscodeSession::new(device)) + } + + /// Bind this accelerator to a caller-owned Metal session. + #[cfg(target_os = "macos")] + #[must_use] + pub fn with_session(mut self, session: MetalTranscodeSession) -> Self { + self.session = Some(session); + self + } + + /// Bind this accelerator to an existing Metal device. + #[cfg(target_os = "macos")] + #[must_use] + pub fn with_device(self, device: ::metal::Device) -> Self { + self.with_session(MetalTranscodeSession::new(device)) + } + + #[cfg(target_os = "macos")] + fn metal_session(&mut self) -> &mut MetalTranscodeSession { + self.session + .get_or_insert_with(MetalTranscodeSession::default) + } + + /// Override the minimum component sample count used before Auto mode + /// dispatches non-reversible projection jobs to Metal. + #[must_use] + pub const fn with_auto_min_samples(mut self, min_samples: usize) -> Self { + self.min_auto_samples = min_samples; + self.min_auto_dwt97_samples = min_samples; + self + } + + /// Override the minimum component sample count used before Auto mode + /// dispatches 9/7 transform jobs to Metal. + #[must_use] + pub const fn with_auto_dwt97_min_samples(mut self, min_samples: usize) -> Self { + self.min_auto_dwt97_samples = min_samples; + self + } + + /// Override the 9/7 batch thresholds used before Auto mode dispatches a + /// same-geometry batch to Metal. + #[must_use] + pub const fn with_auto_dwt97_batch_thresholds( + mut self, + min_jobs: usize, + min_samples: usize, + ) -> Self { + self.min_auto_dwt97_batch_jobs = min_jobs; + self.min_auto_dwt97_batch_samples = min_samples; + self + } + + /// Override the minimum component sample count used before Auto mode + /// dispatches single reversible 5/3 jobs to Metal. + #[must_use] + pub const fn with_auto_reversible_min_samples(mut self, min_samples: usize) -> Self { + self.min_auto_reversible_samples = min_samples; + self + } + + /// Override the reversible 5/3 batch thresholds used before Auto mode + /// dispatches a same-geometry batch to Metal. + #[must_use] + pub const fn with_auto_reversible_batch_thresholds( + mut self, + min_jobs: usize, + min_samples: usize, + ) -> Self { + self.min_auto_reversible_batch_jobs = min_jobs; + self.min_auto_reversible_batch_samples = min_samples; + self + } + + /// Number of reversible integer 5/3 jobs offered to this accelerator. + #[must_use] + pub const fn reversible_dwt53_attempts(&self) -> usize { + self.reversible_dwt53_attempts + } + + /// Number of reversible integer 5/3 jobs handled by Metal. + #[must_use] + pub const fn reversible_dwt53_dispatches(&self) -> usize { + self.reversible_dwt53_dispatches + } + + /// Number of reversible integer 5/3 batches offered to this accelerator. + #[must_use] + pub const fn reversible_dwt53_batch_attempts(&self) -> usize { + self.reversible_dwt53_batch_attempts + } + + /// Number of reversible integer 5/3 batches handled by Metal. + #[must_use] + pub const fn reversible_dwt53_batch_dispatches(&self) -> usize { + self.reversible_dwt53_batch_dispatches + } + + /// Number of 5/3 projection jobs offered to this accelerator. + #[must_use] + pub const fn dwt53_attempts(&self) -> usize { + self.dwt53_attempts + } + + /// Number of 5/3 projection jobs handled by Metal. + #[must_use] + pub const fn dwt53_dispatches(&self) -> usize { + self.dwt53_dispatches + } + + /// Number of 9/7 transform jobs offered to this accelerator. + #[must_use] + pub const fn dwt97_attempts(&self) -> usize { + self.dwt97_attempts + } + + /// Number of 9/7 transform jobs handled by Metal. + #[must_use] + pub const fn dwt97_dispatches(&self) -> usize { + self.dwt97_dispatches + } + + /// Number of 9/7 transform batches offered to this accelerator. + #[must_use] + pub const fn dwt97_batch_attempts(&self) -> usize { + self.dwt97_batch_attempts + } + + /// Number of 9/7 transform batches handled by Metal. + #[must_use] + pub const fn dwt97_batch_dispatches(&self) -> usize { + self.dwt97_batch_dispatches + } + + /// Number of 9/7 code-block-ready batches offered to this accelerator. + #[must_use] + pub const fn htj2k97_codeblock_batch_attempts(&self) -> usize { + self.htj2k97_codeblock_batch_attempts + } + + /// Number of 9/7 code-block-ready batches handled by Metal. + #[must_use] + pub const fn htj2k97_codeblock_batch_dispatches(&self) -> usize { + self.htj2k97_codeblock_batch_dispatches + } + + /// Backend stage timings for the most recent 9/7 batch dispatch. + #[must_use] + pub const fn last_dwt97_batch_stage_timings(&self) -> Option { + self.last_dwt97_batch_stage_timings + } + + /// Dispatch a same-geometry batch of reversible integer 5/3 DCT-grid + /// projection jobs. This is an experimental Metal-specific extension used + /// for WSI tile-component queues; scalar/Rayon remains the portable oracle. + pub fn dct_grid_to_reversible_dwt53_batch( + &mut self, + jobs: &[DctGridToReversibleDwt53Job<'_>], + ) -> Result>, TranscodeStageError> { + self.dispatch_reversible_dwt53_batch(jobs) + } + + fn dispatch_reversible_dwt53_batch( + &mut self, + jobs: &[DctGridToReversibleDwt53Job<'_>], + ) -> Result>, TranscodeStageError> { + self.reversible_dwt53_batch_attempts = + self.reversible_dwt53_batch_attempts.saturating_add(1); + + if jobs.is_empty() { + return Ok(Some(Vec::new())); + } + + let total_samples = jobs.iter().fold(0usize, |total, job| { + total.saturating_add(job.width.saturating_mul(job.height)) + }); + // Auto declines with `Ok(None)` (small batches, unavailable Metal, + // unsupported jobs) so the caller runs its scalar fallback — the same + // contract as the float 5/3 path and the CUDA accelerator, instead of + // silently computing a Rayon fallback inside the backend. + if self.mode == MetalDispatchMode::Auto + && (jobs.len() < self.min_auto_reversible_batch_jobs + || total_samples < self.min_auto_reversible_batch_samples) + { + return Ok(None); + } + + #[cfg(not(target_os = "macos"))] + { + match self.mode { + MetalDispatchMode::Explicit => Err(TranscodeStageError::DeviceUnavailable), + MetalDispatchMode::Auto => Ok(None), + } + } + + #[cfg(target_os = "macos")] + { + match metal::dispatch_dct_grid_to_reversible_dwt53_batch(self.metal_session(), jobs) { + Ok(output) => { + self.reversible_dwt53_batch_dispatches = + self.reversible_dwt53_batch_dispatches.saturating_add(1); + Ok(Some(output)) + } + Err( + MetalTranscodeError::MetalUnavailable | MetalTranscodeError::UnsupportedJob(_), + ) if self.mode == MetalDispatchMode::Auto => Ok(None), + Err(error) => Err(error.into()), + } + } + } +} + +impl Default for MetalDctToWaveletStageAccelerator { + fn default() -> Self { + Self::for_auto() + } +} + +impl DctToWaveletStageAccelerator for MetalDctToWaveletStageAccelerator { + fn supports_dwt97_batch(&self) -> bool { + true + } + + fn supports_htj2k97_codeblock_batch(&self) -> bool { + true + } + + fn dct_grid_to_reversible_dwt53( + &mut self, + job: DctGridToReversibleDwt53Job<'_>, + ) -> Result, TranscodeStageError> { + self.reversible_dwt53_attempts = self.reversible_dwt53_attempts.saturating_add(1); + + // Auto declines with `Ok(None)`; see dispatch_reversible_dwt53_batch. + if self.mode == MetalDispatchMode::Auto + && job.width.saturating_mul(job.height) < self.min_auto_reversible_samples + { + return Ok(None); + } + + #[cfg(not(target_os = "macos"))] + { + match self.mode { + MetalDispatchMode::Explicit => Err(TranscodeStageError::DeviceUnavailable), + MetalDispatchMode::Auto => Ok(None), + } + } + + #[cfg(target_os = "macos")] + { + match metal::dispatch_dct_grid_to_reversible_dwt53(self.metal_session(), job) { + Ok(output) => { + self.reversible_dwt53_dispatches = + self.reversible_dwt53_dispatches.saturating_add(1); + Ok(Some(output)) + } + Err( + MetalTranscodeError::MetalUnavailable | MetalTranscodeError::UnsupportedJob(_), + ) if self.mode == MetalDispatchMode::Auto => Ok(None), + Err(error) => Err(error.into()), + } + } + } + + fn dct_grid_to_reversible_dwt53_batch( + &mut self, + jobs: &[DctGridToReversibleDwt53Job<'_>], + ) -> Result>, TranscodeStageError> { + self.dispatch_reversible_dwt53_batch(jobs) + } + + fn dct_grid_to_dwt53( + &mut self, + job: DctGridToDwt53Job<'_>, + ) -> Result>, TranscodeStageError> { + self.dwt53_attempts = self.dwt53_attempts.saturating_add(1); + + if self.mode == MetalDispatchMode::Auto + && job.width.saturating_mul(job.height) < self.min_auto_samples + { + return Ok(None); + } + + #[cfg(not(target_os = "macos"))] + { + let _ = job; + match self.mode { + MetalDispatchMode::Explicit => Err(TranscodeStageError::DeviceUnavailable), + MetalDispatchMode::Auto => Ok(None), + } + } + + #[cfg(target_os = "macos")] + { + match metal::dispatch_dct_grid_to_dwt53(self.metal_session(), job) { + Ok(output) => { + self.dwt53_dispatches = self.dwt53_dispatches.saturating_add(1); + Ok(Some(output)) + } + Err( + MetalTranscodeError::MetalUnavailable | MetalTranscodeError::UnsupportedJob(_), + ) if self.mode == MetalDispatchMode::Auto => Ok(None), + Err(error) => Err(error.into()), + } + } + } + + fn dct_grid_to_dwt97( + &mut self, + job: DctGridToDwt97Job<'_>, + ) -> Result>, TranscodeStageError> { + self.dwt97_attempts = self.dwt97_attempts.saturating_add(1); + + if self.mode == MetalDispatchMode::Auto + && job.width.saturating_mul(job.height) < self.min_auto_dwt97_samples + { + return Ok(None); + } + + #[cfg(not(target_os = "macos"))] + { + let _ = job; + match self.mode { + MetalDispatchMode::Explicit => Err(TranscodeStageError::DeviceUnavailable), + MetalDispatchMode::Auto => Ok(None), + } + } + + #[cfg(target_os = "macos")] + { + match metal::dispatch_dct_grid_to_dwt97(self.metal_session(), job) { + Ok(output) => { + self.dwt97_dispatches = self.dwt97_dispatches.saturating_add(1); + Ok(Some(output)) + } + Err( + MetalTranscodeError::MetalUnavailable | MetalTranscodeError::UnsupportedJob(_), + ) if self.mode == MetalDispatchMode::Auto => Ok(None), + Err(error) => Err(error.into()), + } + } + } + + fn dct_grid_to_dwt97_batch( + &mut self, + jobs: &[DctGridToDwt97Job<'_>], + ) -> Result>>, TranscodeStageError> { + self.dwt97_batch_attempts = self.dwt97_batch_attempts.saturating_add(1); + self.last_dwt97_batch_stage_timings = None; + + if jobs.is_empty() { + return Ok(Some(Vec::new())); + } + + let total_samples = jobs.iter().fold(0usize, |total, job| { + total.saturating_add(job.width.saturating_mul(job.height)) + }); + if self.mode == MetalDispatchMode::Auto + && (jobs.len() < self.min_auto_dwt97_batch_jobs + || total_samples < self.min_auto_dwt97_batch_samples) + { + return Ok(None); + } + if self.mode == MetalDispatchMode::Auto + && jobs.iter().any(|job| { + job.width > MAX_AUTO_DWT97_STAGED_BATCH_AXIS + || job.height > MAX_AUTO_DWT97_STAGED_BATCH_AXIS + }) + { + return Ok(None); + } + + #[cfg(not(target_os = "macos"))] + { + let _ = jobs; + match self.mode { + MetalDispatchMode::Explicit => Err(TranscodeStageError::DeviceUnavailable), + MetalDispatchMode::Auto => Ok(None), + } + } + + #[cfg(target_os = "macos")] + { + match metal::dispatch_dct_grid_to_dwt97_batch(self.metal_session(), jobs) { + Ok((output, timings)) => { + self.dwt97_batch_dispatches = self.dwt97_batch_dispatches.saturating_add(1); + self.last_dwt97_batch_stage_timings = Some(timings); + Ok(Some(output)) + } + Err( + MetalTranscodeError::MetalUnavailable | MetalTranscodeError::UnsupportedJob(_), + ) if self.mode == MetalDispatchMode::Auto => Ok(None), + Err(error) => Err(error.into()), + } + } + } + + fn dct_grid_to_htj2k97_codeblock_batch( + &mut self, + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, + ) -> Result>, TranscodeStageError> { + self.dwt97_batch_attempts = self.dwt97_batch_attempts.saturating_add(1); + self.htj2k97_codeblock_batch_attempts = + self.htj2k97_codeblock_batch_attempts.saturating_add(1); + self.last_dwt97_batch_stage_timings = None; + + if jobs.is_empty() { + return Ok(Some(Vec::new())); + } + + let total_samples = jobs.iter().fold(0usize, |total, job| { + total.saturating_add(job.width.saturating_mul(job.height)) + }); + if self.mode == MetalDispatchMode::Auto + && (jobs.len() < self.min_auto_dwt97_batch_jobs + || total_samples < self.min_auto_dwt97_batch_samples) + { + return Ok(None); + } + if self.mode == MetalDispatchMode::Auto + && jobs.iter().any(|job| { + job.width > MAX_AUTO_DWT97_STAGED_BATCH_AXIS + || job.height > MAX_AUTO_DWT97_STAGED_BATCH_AXIS + }) + { + return Ok(None); + } + + #[cfg(not(target_os = "macos"))] + { + let _ = (jobs, options); + match self.mode { + MetalDispatchMode::Explicit => Err(TranscodeStageError::DeviceUnavailable), + MetalDispatchMode::Auto => Ok(None), + } + } + + #[cfg(target_os = "macos")] + { + match metal::dispatch_dct_grid_to_htj2k97_codeblock_batch( + self.metal_session(), + jobs, + options, + ) { + Ok((output, timings)) => { + self.dwt97_batch_dispatches = self.dwt97_batch_dispatches.saturating_add(1); + self.htj2k97_codeblock_batch_dispatches = + self.htj2k97_codeblock_batch_dispatches.saturating_add(1); + self.last_dwt97_batch_stage_timings = Some(timings); + Ok(Some(output)) + } + Err( + MetalTranscodeError::MetalUnavailable | MetalTranscodeError::UnsupportedJob(_), + ) if self.mode == MetalDispatchMode::Auto => Ok(None), + Err(error) => Err(error.into()), + } + } + } + + fn last_dwt97_batch_stage_timings(&self) -> Option { + self.last_dwt97_batch_stage_timings + } +} diff --git a/crates/j2k-transcode-metal/src/metal.rs b/crates/j2k-transcode-metal/src/metal.rs new file mode 100644 index 00000000..528cfcf9 --- /dev/null +++ b/crates/j2k-transcode-metal/src/metal.rs @@ -0,0 +1,2289 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Metal runtime for direct DCT-grid to one-level wavelet projection. + +use std::sync::Arc; +use std::time::Instant; + +use core::f32::consts::PI; +use core::mem::{size_of, size_of_val}; + +use j2k_metal_support::{ + checked_buffer_contents_slice, checked_command_queue, shared_buffer_for_len, + shared_buffer_with_slice, system_default_device, MetalPipelineLoader, +}; +use j2k_transcode::accelerator::{ + idct_blocks_to_signed_samples_rayon, DctGridToDwt53Job, DctGridToDwt97Job, + DctGridToHtj2k97CodeBlockJob, DctGridToReversibleDwt53Job, Dwt97BatchStageTimings, + Htj2k97CodeBlockOptions, J2kSubBandType, PrequantizedHtj2k97CodeBlock, + PrequantizedHtj2k97Component, PrequantizedHtj2k97Resolution, PrequantizedHtj2k97Subband, + ReversibleDwt53FirstLevel, +}; +use j2k_transcode::dct53_2d::Dwt53TwoDimensional; +use j2k_transcode::dct97_2d::Dwt97TwoDimensional; +use j2k_transcode::htj2k97_codeblock_oracle::{ + htj2k97_subband_delta, htj2k97_subband_total_bitplanes, +}; +use metal::{ + Buffer, CommandQueue, ComputeCommandEncoderRef, ComputePipelineState, Device, + MTLResourceOptions, MTLSize, +}; + +use crate::weights::{SparseDwt53WeightRows, SparseDwt97WeightRows, SparseWeightRow}; +use crate::MetalTranscodeError; + +const SHADER_SOURCE: &str = include_str!("dct97.metal"); +const METAL_DCT_KERNEL_FAILED: &str = "Metal DCT wavelet projection failed"; +const METAL_DCT_RUNTIME_FAILED: &str = "Metal DCT wavelet runtime setup failed"; +const METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID: &str = + "Metal reversible DCT 5/3 job has unsupported grid geometry"; +const METAL_DCT53_UNSUPPORTED_GRID: &str = "Metal DCT 5/3 job has unsupported grid geometry"; +const METAL_DCT97_UNSUPPORTED_GRID: &str = "Metal DCT 9/7 job has unsupported grid geometry"; +const DWT97_STAGED_MAX_AXIS: usize = 1024; +const DWT97_STAGED_ROWS_PER_GROUP: usize = 2; +const DWT97_STAGED_COLUMNS_PER_GROUP: usize = 4; +const DWT97_STAGED_THREADS_PER_GROUP: u64 = 256; +const DWT97_BLOCK_COEFFICIENTS: usize = 64; + +struct MetalRuntime { + device: Device, + queue: CommandQueue, + dct_project_band: ComputePipelineState, + dct_project_band_batch: ComputePipelineState, + dct97_idct_row_lift_batch: ComputePipelineState, + dct97_column_lift_batch: ComputePipelineState, + dct97_quantize_codeblocks_batch: ComputePipelineState, + reversible53_project_band: ComputePipelineState, + idct_basis: Buffer, +} + +#[derive(Clone, Default)] +/// Reusable Metal session for transcode-stage accelerator dispatch. +pub struct MetalTranscodeSession { + device: Option, + runtime: Option>, +} + +impl MetalTranscodeSession { + /// Create a transcode session bound to an existing Metal device. + pub fn new(device: Device) -> Self { + Self { + device: Some(device), + runtime: None, + } + } + + /// Create a transcode session bound to the system default Metal device. + pub fn system_default() -> Result { + system_default_device() + .map(Self::new) + .map_err(|_| MetalTranscodeError::MetalUnavailable) + } + + fn runtime(&mut self) -> Result, MetalTranscodeError> { + if let Some(runtime) = &self.runtime { + return Ok(Arc::clone(runtime)); + } + let runtime = Arc::new(match &self.device { + Some(device) => MetalRuntime::new_with_device(device.clone())?, + None => MetalRuntime::new()?, + }); + self.runtime = Some(Arc::clone(&runtime)); + Ok(runtime) + } + + fn with_runtime( + &mut self, + f: impl FnOnce(&MetalRuntime) -> Result, + ) -> Result { + let runtime = self.runtime()?; + f(&runtime) + } +} + +impl core::fmt::Debug for MetalTranscodeSession { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("MetalTranscodeSession") + .field("device", &self.device.as_ref().map(|device| device.name())) + .field("runtime_initialized", &self.runtime.is_some()) + .finish() + } +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct DctProjectionParams { + width: u32, + height: u32, + block_cols: u32, + band_width: u32, + band_height: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct DctBatchProjectionParams { + width: u32, + height: u32, + block_cols: u32, + blocks_per_item: u32, + band_width: u32, + band_height: u32, + output_stride: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct Dct97IdctRowLiftParams { + width: u32, + height: u32, + block_cols: u32, + blocks_per_item: u32, + low_width: u32, + high_width: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct Dct97ColumnLiftParams { + height: u32, + low_width: u32, + high_width: u32, + low_height: u32, + high_height: u32, + row_low_stride: u32, + row_high_stride: u32, + ll_stride: u32, + hl_stride: u32, + lh_stride: u32, + hh_stride: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct Dct97QuantizeCodeblocksParams { + band_width: u32, + band_height: u32, + output_stride: u32, + code_block_width: u32, + code_block_height: u32, + inv_delta: f32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct Reversible53ProjectionParams { + width: u32, + height: u32, + block_cols: u32, + blocks_per_item: u32, + band_width: u32, + band_height: u32, + output_stride: u32, + vertical_low: u32, + horizontal_low: u32, +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct MetalSparseRow { + offset: u32, + count: u32, +} + +// SAFETY: Metal ABI structs are repr(C) plain data matching shader layouts. +unsafe impl j2k_core::GpuAbi for MetalSparseRow { + const NAME: &'static str = "MetalSparseRow"; +} + +#[repr(C)] +#[derive(Clone, Copy)] +struct MetalWeightTap { + sample_idx: u32, + weight: f32, +} + +// SAFETY: Metal ABI structs are repr(C) plain data matching shader layouts. +unsafe impl j2k_core::GpuAbi for MetalWeightTap { + const NAME: &'static str = "MetalWeightTap"; +} + +struct MetalSparseRows { + rows: Vec, + taps: Vec, +} + +impl MetalRuntime { + fn new() -> Result { + let device = system_default_device().map_err(|_| MetalTranscodeError::MetalUnavailable)?; + Self::new_with_device(device) + } + + fn new_with_device(device: Device) -> Result { + let loader = MetalPipelineLoader::new(&device, SHADER_SOURCE) + .map_err(|_| MetalTranscodeError::Runtime(METAL_DCT_RUNTIME_FAILED))?; + let pipeline = |name| { + loader + .pipeline(name) + .map_err(|_| MetalTranscodeError::Runtime(METAL_DCT_RUNTIME_FAILED)) + }; + let dct_project_band = pipeline("dct97_project_band")?; + let dct_project_band_batch = pipeline("dct97_project_band_batch")?; + let dct97_idct_row_lift_batch = pipeline("dct97_idct_row_lift_batch")?; + let dct97_column_lift_batch = pipeline("dct97_column_lift_batch")?; + let dct97_quantize_codeblocks_batch = pipeline("dct97_quantize_codeblocks_batch")?; + let reversible53_project_band = pipeline("reversible53_project_band")?; + let queue = checked_command_queue(&device) + .map_err(|_| MetalTranscodeError::Runtime(METAL_DCT_RUNTIME_FAILED))?; + let idct_basis_data = idct8_basis_table(); + let idct_basis = device.new_buffer_with_data( + idct_basis_data.as_ptr().cast(), + size_of_val(&idct_basis_data) as u64, + MTLResourceOptions::StorageModeShared, + ); + + Ok(Self { + device, + queue, + dct_project_band, + dct_project_band_batch, + dct97_idct_row_lift_batch, + dct97_column_lift_batch, + dct97_quantize_codeblocks_batch, + reversible53_project_band, + idct_basis, + }) + } +} + +pub(crate) fn dispatch_dct_grid_to_reversible_dwt53( + session: &mut MetalTranscodeSession, + job: DctGridToReversibleDwt53Job<'_>, +) -> Result { + let mut outputs = + dispatch_dct_grid_to_reversible_dwt53_batch(session, core::slice::from_ref(&job))?; + outputs + .pop() + .ok_or(MetalTranscodeError::Kernel(METAL_DCT_KERNEL_FAILED)) +} + +pub(crate) fn dispatch_dct_grid_to_reversible_dwt53_batch( + session: &mut MetalTranscodeSession, + jobs: &[DctGridToReversibleDwt53Job<'_>], +) -> Result, MetalTranscodeError> { + let Some(first) = jobs.first() else { + return Ok(Vec::new()); + }; + validate_reversible_batch_geometry(jobs)?; + + let blocks_per_item = first.block_cols.checked_mul(first.block_rows).ok_or( + MetalTranscodeError::UnsupportedJob(METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID), + )?; + let mut block_samples = Vec::with_capacity(blocks_per_item.saturating_mul(jobs.len())); + for job in jobs { + block_samples.extend(idct_blocks_to_signed_samples_rayon(job.dequantized_blocks)); + } + + session.with_runtime(|runtime| { + dispatch_reversible_dwt53_batch_with_runtime( + runtime, + &block_samples, + jobs.len(), + first.block_cols, + first.width, + first.height, + ) + }) +} + +pub(crate) fn dispatch_dct_grid_to_dwt53( + session: &mut MetalTranscodeSession, + job: DctGridToDwt53Job<'_>, +) -> Result, MetalTranscodeError> { + validate_grid( + job.blocks.len(), + job.block_cols, + job.block_rows, + job.width, + job.height, + METAL_DCT53_UNSUPPORTED_GRID, + )?; + session.with_runtime(|runtime| dispatch_dct_grid_to_dwt53_with_runtime(runtime, job)) +} + +#[allow(clippy::similar_names)] +fn dispatch_reversible_dwt53_batch_with_runtime( + runtime: &MetalRuntime, + block_samples: &[[i32; 64]], + batch_count: usize, + block_cols: usize, + width: usize, + height: usize, +) -> Result, MetalTranscodeError> { + if batch_count == 0 { + return Ok(Vec::new()); + } + if !block_samples.len().is_multiple_of(batch_count) { + return Err(MetalTranscodeError::UnsupportedJob( + METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID, + )); + } + + let blocks_per_item = block_samples.len() / batch_count; + let blocks_per_item_u32 = u32_param(blocks_per_item, METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID)?; + let batch_count_u32 = u32_param(batch_count, METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID)?; + let width_u32 = u32_param(width, METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID)?; + let height_u32 = u32_param(height, METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID)?; + let block_cols_u32 = u32_param(block_cols, METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID)?; + let kernel_geometry = ReversibleBatchKernelGeometry { + width: width_u32, + height: height_u32, + block_cols: block_cols_u32, + blocks_per_item: blocks_per_item_u32, + batch_count: batch_count_u32, + }; + let low_width = width.div_ceil(2); + let high_width = width / 2; + let low_height = height.div_ceil(2); + let high_height = height / 2; + let ll_len = low_width * low_height; + let hl_len = high_width * low_height; + let lh_len = low_width * high_height; + let hh_len = high_width * high_height; + let output_shape = ReversibleBatchOutputShape { + low_width, + low_height, + high_width, + high_height, + ll_len, + hl_len, + lh_len, + hh_len, + batch_count, + }; + let blocks = buffer_with_slice(&runtime.device, block_samples); + + let ll_buffer = output_i32_buffer( + &runtime.device, + checked_batch_len(ll_len, batch_count, METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID)?, + ); + let hl_buffer = output_i32_buffer( + &runtime.device, + checked_batch_len(hl_len, batch_count, METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID)?, + ); + let lh_buffer = output_i32_buffer( + &runtime.device, + checked_batch_len(lh_len, batch_count, METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID)?, + ); + let hh_buffer = output_i32_buffer( + &runtime.device, + checked_batch_len(hh_len, batch_count, METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID)?, + ); + let output_buffers = ReversibleOutputBuffers { + ll: &ll_buffer, + hl: &hl_buffer, + lh: &lh_buffer, + hh: &hh_buffer, + }; + + let command_buffer = runtime.queue.new_command_buffer(); + command_buffer.set_label("j2k-transcode-metal reversible dct53 projection"); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.reversible53_project_band); + encoder.set_buffer(0, Some(&blocks), 0); + + dispatch_reversible_band( + encoder, + &ll_buffer, + reversible_band_geometry(kernel_geometry, low_width, low_height, ll_len, true, true)?, + ); + dispatch_reversible_band( + encoder, + &hl_buffer, + reversible_band_geometry(kernel_geometry, high_width, low_height, hl_len, true, false)?, + ); + dispatch_reversible_band( + encoder, + &lh_buffer, + reversible_band_geometry(kernel_geometry, low_width, high_height, lh_len, false, true)?, + ); + dispatch_reversible_band( + encoder, + &hh_buffer, + reversible_band_geometry( + kernel_geometry, + high_width, + high_height, + hh_len, + false, + false, + )?, + ); + + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + read_reversible_batch_outputs(output_buffers, output_shape) +} + +pub(crate) fn dispatch_dct_grid_to_dwt97( + session: &mut MetalTranscodeSession, + job: DctGridToDwt97Job<'_>, +) -> Result, MetalTranscodeError> { + validate_grid( + job.blocks.len(), + job.block_cols, + job.block_rows, + job.width, + job.height, + METAL_DCT97_UNSUPPORTED_GRID, + )?; + session.with_runtime(|runtime| dispatch_dct_grid_to_dwt97_with_runtime(runtime, job)) +} + +pub(crate) fn dispatch_dct_grid_to_dwt97_batch( + session: &mut MetalTranscodeSession, + jobs: &[DctGridToDwt97Job<'_>], +) -> Result<(Vec>, Dwt97BatchStageTimings), MetalTranscodeError> { + let Some(first) = jobs.first() else { + return Ok((Vec::new(), Dwt97BatchStageTimings::default())); + }; + validate_dwt97_batch_geometry(jobs)?; + session + .with_runtime(|runtime| dispatch_dct_grid_to_dwt97_batch_with_runtime(runtime, jobs, first)) +} + +pub(crate) fn dispatch_dct_grid_to_htj2k97_codeblock_batch( + session: &mut MetalTranscodeSession, + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, +) -> Result<(Vec, Dwt97BatchStageTimings), MetalTranscodeError> { + let Some(first) = jobs.first() else { + return Ok((Vec::new(), Dwt97BatchStageTimings::default())); + }; + validate_dwt97_codeblock_batch_geometry(jobs)?; + validate_htj2k97_codeblock_options(options)?; + session.with_runtime(|runtime| { + dispatch_dct_grid_to_htj2k97_codeblock_batch_with_runtime(runtime, jobs, first, options) + }) +} + +#[allow(clippy::similar_names)] +fn dispatch_dct_grid_to_dwt53_with_runtime( + runtime: &MetalRuntime, + job: DctGridToDwt53Job<'_>, +) -> Result, MetalTranscodeError> { + let x_weights = SparseDwt53WeightRows::for_len(job.width); + let y_weights = SparseDwt53WeightRows::for_len(job.height); + let bands = dispatch_projected_bands_with_runtime( + runtime, + ProjectionJob { + blocks: job.blocks, + block_cols: job.block_cols, + width: job.width, + height: job.height, + x_low: &x_weights.low, + x_high: &x_weights.high, + y_low: &y_weights.low, + y_high: &y_weights.high, + unsupported_grid: METAL_DCT53_UNSUPPORTED_GRID, + label: "j2k-transcode-metal dct53 projection", + }, + )?; + + Ok(Dwt53TwoDimensional { + ll: bands.ll, + hl: bands.hl, + lh: bands.lh, + hh: bands.hh, + low_width: bands.low_width, + low_height: bands.low_height, + high_width: bands.high_width, + high_height: bands.high_height, + }) +} + +#[allow(clippy::similar_names)] +fn dispatch_dct_grid_to_dwt97_with_runtime( + runtime: &MetalRuntime, + job: DctGridToDwt97Job<'_>, +) -> Result, MetalTranscodeError> { + let x_weights = SparseDwt97WeightRows::for_len(job.width); + let y_weights = SparseDwt97WeightRows::for_len(job.height); + let bands = dispatch_projected_bands_with_runtime( + runtime, + ProjectionJob { + blocks: job.blocks, + block_cols: job.block_cols, + width: job.width, + height: job.height, + x_low: &x_weights.low, + x_high: &x_weights.high, + y_low: &y_weights.low, + y_high: &y_weights.high, + unsupported_grid: METAL_DCT97_UNSUPPORTED_GRID, + label: "j2k-transcode-metal dct97 projection", + }, + )?; + + Ok(Dwt97TwoDimensional { + ll: bands.ll, + hl: bands.hl, + lh: bands.lh, + hh: bands.hh, + low_width: bands.low_width, + low_height: bands.low_height, + high_width: bands.high_width, + high_height: bands.high_height, + }) +} + +#[allow(clippy::similar_names)] +fn dispatch_dct_grid_to_dwt97_batch_with_runtime( + runtime: &MetalRuntime, + jobs: &[DctGridToDwt97Job<'_>], + first: &DctGridToDwt97Job<'_>, +) -> Result<(Vec>, Dwt97BatchStageTimings), MetalTranscodeError> { + if staged_dwt97_batch_supported(first) { + return dispatch_dct_grid_to_dwt97_batch_staged_with_runtime(runtime, jobs, first); + } + + let x_weights = SparseDwt97WeightRows::for_len(first.width); + let y_weights = SparseDwt97WeightRows::for_len(first.height); + let bands = dispatch_projected_bands_batch_with_runtime( + runtime, + ProjectionBatchJob { + jobs, + block_cols: first.block_cols, + block_rows: first.block_rows, + width: first.width, + height: first.height, + x_low: &x_weights.low, + x_high: &x_weights.high, + y_low: &y_weights.low, + y_high: &y_weights.high, + unsupported_grid: METAL_DCT97_UNSUPPORTED_GRID, + label: "j2k-transcode-metal batched dct97 projection", + }, + )?; + + Ok(( + bands + .into_iter() + .map(|bands| Dwt97TwoDimensional { + ll: bands.ll, + hl: bands.hl, + lh: bands.lh, + hh: bands.hh, + low_width: bands.low_width, + low_height: bands.low_height, + high_width: bands.high_width, + high_height: bands.high_height, + }) + .collect(), + Dwt97BatchStageTimings::default(), + )) +} + +fn staged_dwt97_batch_supported(first: &DctGridToDwt97Job<'_>) -> bool { + first.width <= DWT97_STAGED_MAX_AXIS && first.height <= DWT97_STAGED_MAX_AXIS +} + +fn staged_dwt97_codeblock_batch_supported(first: &DctGridToHtj2k97CodeBlockJob<'_>) -> bool { + first.width <= DWT97_STAGED_MAX_AXIS && first.height <= DWT97_STAGED_MAX_AXIS +} + +fn dispatch_dct_grid_to_dwt97_batch_staged_with_runtime( + runtime: &MetalRuntime, + jobs: &[DctGridToDwt97Job<'_>], + first: &DctGridToDwt97Job<'_>, +) -> Result<(Vec>, Dwt97BatchStageTimings), MetalTranscodeError> { + let shape = dwt97_staged_batch_shape(jobs, first)?; + let mut timings = Dwt97BatchStageTimings::default(); + + let pack_upload_start = Instant::now(); + let blocks = dwt97_batch_blocks_buffer(&runtime.device, jobs)?; + let row_buffers = dwt97_staged_row_buffers(runtime, shape)?; + let output_buffers = + projection_batch_output_buffers(runtime, shape, METAL_DCT97_UNSUPPORTED_GRID)?; + timings.pack_upload_us = pack_upload_start.elapsed().as_micros(); + + let row_start = Instant::now(); + dispatch_dwt97_staged_row_lift(runtime, first.height, shape, &blocks, &row_buffers)?; + timings.idct_row_lift_us = row_start.elapsed().as_micros(); + + let column_start = Instant::now(); + dispatch_dwt97_staged_column_lift(runtime, shape, &row_buffers, &output_buffers)?; + timings.column_lift_us = column_start.elapsed().as_micros(); + + let readback_start = Instant::now(); + let bands = read_projected_batch_outputs(&output_buffers, shape, METAL_DCT97_UNSUPPORTED_GRID)?; + timings.readback_us = readback_start.elapsed().as_micros(); + + Ok(( + bands + .into_iter() + .map(|bands| Dwt97TwoDimensional { + ll: bands.ll, + hl: bands.hl, + lh: bands.lh, + hh: bands.hh, + low_width: bands.low_width, + low_height: bands.low_height, + high_width: bands.high_width, + high_height: bands.high_height, + }) + .collect(), + timings, + )) +} + +fn dispatch_dct_grid_to_htj2k97_codeblock_batch_with_runtime( + runtime: &MetalRuntime, + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + first: &DctGridToHtj2k97CodeBlockJob<'_>, + options: Htj2k97CodeBlockOptions, +) -> Result<(Vec, Dwt97BatchStageTimings), MetalTranscodeError> { + if !staged_dwt97_codeblock_batch_supported(first) { + return Err(MetalTranscodeError::UnsupportedJob( + METAL_DCT97_UNSUPPORTED_GRID, + )); + } + + let shape = dwt97_codeblock_batch_shape(jobs, first)?; + let mut timings = Dwt97BatchStageTimings::default(); + + let pack_upload_start = Instant::now(); + let blocks = dwt97_codeblock_batch_blocks_buffer(&runtime.device, jobs)?; + let row_buffers = dwt97_staged_row_buffers(runtime, shape)?; + let band_buffers = + projection_batch_output_buffers(runtime, shape, METAL_DCT97_UNSUPPORTED_GRID)?; + let codeblock_buffers = + dwt97_codeblock_output_buffers(runtime, shape, METAL_DCT97_UNSUPPORTED_GRID)?; + timings.pack_upload_us = pack_upload_start.elapsed().as_micros(); + + let row_start = Instant::now(); + dispatch_dwt97_staged_row_lift(runtime, first.height, shape, &blocks, &row_buffers)?; + timings.idct_row_lift_us = row_start.elapsed().as_micros(); + + let column_start = Instant::now(); + dispatch_dwt97_staged_column_lift(runtime, shape, &row_buffers, &band_buffers)?; + timings.column_lift_us = column_start.elapsed().as_micros(); + + let quantize_start = Instant::now(); + dispatch_dwt97_quantize_codeblocks(runtime, shape, options, &band_buffers, &codeblock_buffers)?; + timings.quantize_codeblock_us = quantize_start.elapsed().as_micros(); + + let readback_start = Instant::now(); + let components = read_prequantized_97_codeblock_outputs( + &codeblock_buffers, + jobs, + shape, + options, + METAL_DCT97_UNSUPPORTED_GRID, + )?; + timings.readback_us = readback_start.elapsed().as_micros(); + + Ok((components, timings)) +} + +fn dwt97_staged_batch_shape( + jobs: &[DctGridToDwt97Job<'_>], + first: &DctGridToDwt97Job<'_>, +) -> Result { + let low_width = first.width.div_ceil(2); + let high_width = first.width / 2; + let low_height = first.height.div_ceil(2); + let high_height = first.height / 2; + let blocks_per_item = first.block_cols.checked_mul(first.block_rows).ok_or( + MetalTranscodeError::UnsupportedJob(METAL_DCT97_UNSUPPORTED_GRID), + )?; + + Ok(ProjectionBatchShape { + batch_count: jobs.len(), + batch_count_u32: u32_param(jobs.len(), METAL_DCT97_UNSUPPORTED_GRID)?, + width: u32_param(first.width, METAL_DCT97_UNSUPPORTED_GRID)?, + height: u32_param(first.height, METAL_DCT97_UNSUPPORTED_GRID)?, + block_cols: u32_param(first.block_cols, METAL_DCT97_UNSUPPORTED_GRID)?, + blocks_per_item: u32_param(blocks_per_item, METAL_DCT97_UNSUPPORTED_GRID)?, + low_width, + low_height, + high_width, + high_height, + ll_len: low_width * low_height, + hl_len: high_width * low_height, + lh_len: low_width * high_height, + hh_len: high_width * high_height, + }) +} + +fn dwt97_codeblock_batch_shape( + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + first: &DctGridToHtj2k97CodeBlockJob<'_>, +) -> Result { + let low_width = first.width.div_ceil(2); + let high_width = first.width / 2; + let low_height = first.height.div_ceil(2); + let high_height = first.height / 2; + let blocks_per_item = first.block_cols.checked_mul(first.block_rows).ok_or( + MetalTranscodeError::UnsupportedJob(METAL_DCT97_UNSUPPORTED_GRID), + )?; + + Ok(ProjectionBatchShape { + batch_count: jobs.len(), + batch_count_u32: u32_param(jobs.len(), METAL_DCT97_UNSUPPORTED_GRID)?, + width: u32_param(first.width, METAL_DCT97_UNSUPPORTED_GRID)?, + height: u32_param(first.height, METAL_DCT97_UNSUPPORTED_GRID)?, + block_cols: u32_param(first.block_cols, METAL_DCT97_UNSUPPORTED_GRID)?, + blocks_per_item: u32_param(blocks_per_item, METAL_DCT97_UNSUPPORTED_GRID)?, + low_width, + low_height, + high_width, + high_height, + ll_len: low_width * low_height, + hl_len: high_width * low_height, + lh_len: low_width * high_height, + hh_len: high_width * high_height, + }) +} + +struct Dwt97StagedRowBuffers { + low: Buffer, + high: Buffer, +} + +fn dwt97_staged_row_buffers( + runtime: &MetalRuntime, + shape: ProjectionBatchShape, +) -> Result { + let height = shape.height as usize; + Ok(Dwt97StagedRowBuffers { + low: output_buffer( + &runtime.device, + checked_batch_len( + height * shape.low_width, + shape.batch_count, + METAL_DCT97_UNSUPPORTED_GRID, + )?, + ), + high: output_buffer( + &runtime.device, + checked_batch_len( + height * shape.high_width, + shape.batch_count, + METAL_DCT97_UNSUPPORTED_GRID, + )?, + ), + }) +} + +fn dispatch_dwt97_staged_row_lift( + runtime: &MetalRuntime, + height: usize, + shape: ProjectionBatchShape, + blocks: &Buffer, + row_buffers: &Dwt97StagedRowBuffers, +) -> Result<(), MetalTranscodeError> { + let params = Dct97IdctRowLiftParams { + width: shape.width, + height: shape.height, + block_cols: shape.block_cols, + blocks_per_item: shape.blocks_per_item, + low_width: u32_param(shape.low_width, METAL_DCT97_UNSUPPORTED_GRID)?, + high_width: u32_param(shape.high_width, METAL_DCT97_UNSUPPORTED_GRID)?, + }; + let row_groups = height.div_ceil(DWT97_STAGED_ROWS_PER_GROUP); + + let command_buffer = runtime.queue.new_command_buffer(); + command_buffer.set_label("j2k-transcode-metal dct97 idct row lift batch"); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.dct97_idct_row_lift_batch); + encoder.set_buffer(0, Some(blocks), 0); + encoder.set_buffer(1, Some(&runtime.idct_basis), 0); + encoder.set_buffer(2, Some(&row_buffers.low), 0); + encoder.set_buffer(3, Some(&row_buffers.high), 0); + encoder.set_bytes( + 4, + size_of::() as u64, + (&raw const params).cast(), + ); + encoder.dispatch_thread_groups( + MTLSize { + width: row_groups as u64, + height: u64::from(shape.batch_count_u32), + depth: 1, + }, + staged_threads_per_group(), + ); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + Ok(()) +} + +fn dispatch_dwt97_staged_column_lift( + runtime: &MetalRuntime, + shape: ProjectionBatchShape, + row_buffers: &Dwt97StagedRowBuffers, + output_buffers: &ProjectionBatchOutputBuffers, +) -> Result<(), MetalTranscodeError> { + let row_low_stride = (shape.height as usize).checked_mul(shape.low_width).ok_or( + MetalTranscodeError::UnsupportedJob(METAL_DCT97_UNSUPPORTED_GRID), + )?; + let row_high_stride = (shape.height as usize) + .checked_mul(shape.high_width) + .ok_or(MetalTranscodeError::UnsupportedJob( + METAL_DCT97_UNSUPPORTED_GRID, + ))?; + let params = Dct97ColumnLiftParams { + height: shape.height, + low_width: u32_param(shape.low_width, METAL_DCT97_UNSUPPORTED_GRID)?, + high_width: u32_param(shape.high_width, METAL_DCT97_UNSUPPORTED_GRID)?, + low_height: u32_param(shape.low_height, METAL_DCT97_UNSUPPORTED_GRID)?, + high_height: u32_param(shape.high_height, METAL_DCT97_UNSUPPORTED_GRID)?, + row_low_stride: u32_param(row_low_stride, METAL_DCT97_UNSUPPORTED_GRID)?, + row_high_stride: u32_param(row_high_stride, METAL_DCT97_UNSUPPORTED_GRID)?, + ll_stride: u32_param(shape.ll_len, METAL_DCT97_UNSUPPORTED_GRID)?, + hl_stride: u32_param(shape.hl_len, METAL_DCT97_UNSUPPORTED_GRID)?, + lh_stride: u32_param(shape.lh_len, METAL_DCT97_UNSUPPORTED_GRID)?, + hh_stride: u32_param(shape.hh_len, METAL_DCT97_UNSUPPORTED_GRID)?, + }; + let column_groups = shape + .low_width + .max(shape.high_width) + .div_ceil(DWT97_STAGED_COLUMNS_PER_GROUP); + + let command_buffer = runtime.queue.new_command_buffer(); + command_buffer.set_label("j2k-transcode-metal dct97 column lift batch"); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.dct97_column_lift_batch); + encoder.set_buffer(0, Some(&row_buffers.low), 0); + encoder.set_buffer(1, Some(&row_buffers.high), 0); + encoder.set_buffer(2, Some(&output_buffers.ll), 0); + encoder.set_buffer(3, Some(&output_buffers.hl), 0); + encoder.set_buffer(4, Some(&output_buffers.lh), 0); + encoder.set_buffer(5, Some(&output_buffers.hh), 0); + encoder.set_bytes( + 6, + size_of::() as u64, + (&raw const params).cast(), + ); + encoder.dispatch_thread_groups( + MTLSize { + width: column_groups as u64, + height: u64::from(shape.batch_count_u32), + depth: 2, + }, + staged_threads_per_group(), + ); + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + Ok(()) +} + +fn dispatch_dwt97_quantize_codeblocks( + runtime: &MetalRuntime, + shape: ProjectionBatchShape, + options: Htj2k97CodeBlockOptions, + band_buffers: &ProjectionBatchOutputBuffers, + codeblock_buffers: &Dwt97CodeBlockOutputBuffers, +) -> Result<(), MetalTranscodeError> { + let cb_width = code_block_len_from_exp(options.code_block_width_exp)?; + let cb_height = code_block_len_from_exp(options.code_block_height_exp)?; + let command_buffer = runtime.queue.new_command_buffer(); + command_buffer.set_label("j2k-transcode-metal dct97 quantize codeblocks batch"); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.dct97_quantize_codeblocks_batch); + dispatch_dwt97_quantize_codeblock_band( + encoder, + &band_buffers.ll, + &codeblock_buffers.ll, + Dwt97QuantizeBand { + width: shape.low_width, + height: shape.low_height, + stride: shape.ll_len, + cb_width, + cb_height, + inv_delta: dwt97_quantize_inv_delta(options, J2kSubBandType::LowLow), + batch_count: shape.batch_count_u32, + }, + )?; + dispatch_dwt97_quantize_codeblock_band( + encoder, + &band_buffers.hl, + &codeblock_buffers.hl, + Dwt97QuantizeBand { + width: shape.high_width, + height: shape.low_height, + stride: shape.hl_len, + cb_width, + cb_height, + inv_delta: dwt97_quantize_inv_delta(options, J2kSubBandType::HighLow), + batch_count: shape.batch_count_u32, + }, + )?; + dispatch_dwt97_quantize_codeblock_band( + encoder, + &band_buffers.lh, + &codeblock_buffers.lh, + Dwt97QuantizeBand { + width: shape.low_width, + height: shape.high_height, + stride: shape.lh_len, + cb_width, + cb_height, + inv_delta: dwt97_quantize_inv_delta(options, J2kSubBandType::LowHigh), + batch_count: shape.batch_count_u32, + }, + )?; + dispatch_dwt97_quantize_codeblock_band( + encoder, + &band_buffers.hh, + &codeblock_buffers.hh, + Dwt97QuantizeBand { + width: shape.high_width, + height: shape.high_height, + stride: shape.hh_len, + cb_width, + cb_height, + inv_delta: dwt97_quantize_inv_delta(options, J2kSubBandType::HighHigh), + batch_count: shape.batch_count_u32, + }, + )?; + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + Ok(()) +} + +#[derive(Clone, Copy)] +struct Dwt97QuantizeBand { + width: usize, + height: usize, + stride: usize, + cb_width: usize, + cb_height: usize, + inv_delta: f32, + batch_count: u32, +} + +fn dispatch_dwt97_quantize_codeblock_band( + encoder: &ComputeCommandEncoderRef, + band_buffer: &Buffer, + codeblock_buffer: &Buffer, + band: Dwt97QuantizeBand, +) -> Result<(), MetalTranscodeError> { + if band.width == 0 || band.height == 0 { + return Ok(()); + } + let params = Dct97QuantizeCodeblocksParams { + band_width: u32_param(band.width, METAL_DCT97_UNSUPPORTED_GRID)?, + band_height: u32_param(band.height, METAL_DCT97_UNSUPPORTED_GRID)?, + output_stride: u32_param(band.stride, METAL_DCT97_UNSUPPORTED_GRID)?, + code_block_width: u32_param(band.cb_width, METAL_DCT97_UNSUPPORTED_GRID)?, + code_block_height: u32_param(band.cb_height, METAL_DCT97_UNSUPPORTED_GRID)?, + inv_delta: band.inv_delta, + }; + encoder.set_buffer(0, Some(band_buffer), 0); + encoder.set_buffer(1, Some(codeblock_buffer), 0); + encoder.set_bytes( + 2, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_projection_threads( + encoder, + band.width as u64, + band.height as u64, + u64::from(band.batch_count), + ); + Ok(()) +} + +fn staged_threads_per_group() -> MTLSize { + MTLSize { + width: DWT97_STAGED_THREADS_PER_GROUP, + height: 1, + depth: 1, + } +} + +#[inline] +fn projection_thread_grid(width: u64, height: u64, depth: u64) -> MTLSize { + MTLSize { + width, + height, + depth, + } +} + +#[inline] +fn projection_threads_per_group() -> MTLSize { + projection_thread_grid(16, 8, 1) +} + +#[inline] +fn projection_dispatch_sizes(width: u64, height: u64, depth: u64) -> (MTLSize, MTLSize) { + ( + projection_thread_grid(width, height, depth), + projection_threads_per_group(), + ) +} + +#[inline] +fn dispatch_projection_threads( + encoder: &ComputeCommandEncoderRef, + width: u64, + height: u64, + depth: u64, +) { + let (threads, threads_per_group) = projection_dispatch_sizes(width, height, depth); + encoder.dispatch_threads(threads, threads_per_group); +} + +#[inline] +fn bind_projection_input_buffers( + encoder: &ComputeCommandEncoderRef, + blocks: &Buffer, + idct_basis: &Buffer, +) { + encoder.set_buffer(0, Some(blocks), 0); + encoder.set_buffer(5, Some(idct_basis), 0); +} + +#[inline] +fn bind_projection_band_buffers( + encoder: &ComputeCommandEncoderRef, + x_weights: (&Buffer, &Buffer), + y_weights: (&Buffer, &Buffer), + output: &Buffer, +) { + encoder.set_buffer(1, Some(x_weights.0), 0); + encoder.set_buffer(2, Some(x_weights.1), 0); + encoder.set_buffer(3, Some(y_weights.0), 0); + encoder.set_buffer(4, Some(y_weights.1), 0); + encoder.set_buffer(6, Some(output), 0); +} + +#[derive(Clone, Copy)] +struct ProjectionJob<'a> { + blocks: &'a [[[f64; 8]; 8]], + block_cols: usize, + width: usize, + height: usize, + x_low: &'a [SparseWeightRow], + x_high: &'a [SparseWeightRow], + y_low: &'a [SparseWeightRow], + y_high: &'a [SparseWeightRow], + unsupported_grid: &'static str, + label: &'static str, +} + +#[derive(Clone, Copy)] +struct ProjectionBatchJob<'a, 'b> { + jobs: &'a [DctGridToDwt97Job<'b>], + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + x_low: &'a [SparseWeightRow], + x_high: &'a [SparseWeightRow], + y_low: &'a [SparseWeightRow], + y_high: &'a [SparseWeightRow], + unsupported_grid: &'static str, + label: &'static str, +} + +struct ProjectedBands { + ll: Vec, + hl: Vec, + lh: Vec, + hh: Vec, + low_width: usize, + low_height: usize, + high_width: usize, + high_height: usize, +} + +#[derive(Clone, Copy)] +struct ReversibleBatchOutputShape { + low_width: usize, + low_height: usize, + high_width: usize, + high_height: usize, + ll_len: usize, + hl_len: usize, + lh_len: usize, + hh_len: usize, + batch_count: usize, +} + +#[derive(Clone, Copy)] +struct ReversibleOutputBuffers<'a> { + ll: &'a Buffer, + hl: &'a Buffer, + lh: &'a Buffer, + hh: &'a Buffer, +} + +fn read_reversible_batch_outputs( + buffers: ReversibleOutputBuffers<'_>, + shape: ReversibleBatchOutputShape, +) -> Result, MetalTranscodeError> { + let ll = read_i32_buffer( + buffers.ll, + checked_batch_len( + shape.ll_len, + shape.batch_count, + METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID, + )?, + )?; + let hl = read_i32_buffer( + buffers.hl, + checked_batch_len( + shape.hl_len, + shape.batch_count, + METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID, + )?, + )?; + let lh = read_i32_buffer( + buffers.lh, + checked_batch_len( + shape.lh_len, + shape.batch_count, + METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID, + )?, + )?; + let hh = read_i32_buffer( + buffers.hh, + checked_batch_len( + shape.hh_len, + shape.batch_count, + METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID, + )?, + )?; + + let mut outputs = Vec::with_capacity(shape.batch_count); + for idx in 0..shape.batch_count { + outputs.push(ReversibleDwt53FirstLevel { + ll: ll[idx * shape.ll_len..idx * shape.ll_len + shape.ll_len].to_vec(), + hl: hl[idx * shape.hl_len..idx * shape.hl_len + shape.hl_len].to_vec(), + lh: lh[idx * shape.lh_len..idx * shape.lh_len + shape.lh_len].to_vec(), + hh: hh[idx * shape.hh_len..idx * shape.hh_len + shape.hh_len].to_vec(), + low_width: shape.low_width, + low_height: shape.low_height, + high_width: shape.high_width, + high_height: shape.high_height, + }); + } + + Ok(outputs) +} + +#[allow(clippy::similar_names)] +fn dispatch_projected_bands_with_runtime( + runtime: &MetalRuntime, + job: ProjectionJob<'_>, +) -> Result { + let width = u32_param(job.width, job.unsupported_grid)?; + let height = u32_param(job.height, job.unsupported_grid)?; + let block_cols = u32_param(job.block_cols, job.unsupported_grid)?; + let low_width = job.width.div_ceil(2); + let high_width = job.width / 2; + let low_height = job.height.div_ceil(2); + let high_height = job.height / 2; + + let x_low = metal_sparse_rows(job.x_low, job.unsupported_grid)?; + let x_high = metal_sparse_rows(job.x_high, job.unsupported_grid)?; + let y_low = metal_sparse_rows(job.y_low, job.unsupported_grid)?; + let y_high = metal_sparse_rows(job.y_high, job.unsupported_grid)?; + let x_low_rows = buffer_with_slice(&runtime.device, &x_low.rows); + let x_low_taps = buffer_with_slice(&runtime.device, &x_low.taps); + let x_high_rows = buffer_with_slice(&runtime.device, &x_high.rows); + let x_high_taps = buffer_with_slice(&runtime.device, &x_high.taps); + let y_low_rows = buffer_with_slice(&runtime.device, &y_low.rows); + let y_low_taps = buffer_with_slice(&runtime.device, &y_low.taps); + let y_high_rows = buffer_with_slice(&runtime.device, &y_high.rows); + let y_high_taps = buffer_with_slice(&runtime.device, &y_high.taps); + let blocks = dwt97_blocks_buffer(&runtime.device, job.blocks)?; + + let ll_buffer = output_buffer(&runtime.device, low_width * low_height); + let hl_buffer = output_buffer(&runtime.device, high_width * low_height); + let lh_buffer = output_buffer(&runtime.device, low_width * high_height); + let hh_buffer = output_buffer(&runtime.device, high_width * high_height); + + let command_buffer = runtime.queue.new_command_buffer(); + command_buffer.set_label(job.label); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.dct_project_band); + bind_projection_input_buffers(encoder, &blocks, &runtime.idct_basis); + + dispatch_band( + encoder, + (&x_low_rows, &x_low_taps), + (&y_low_rows, &y_low_taps), + &ll_buffer, + BandGeometry { + width, + height, + block_cols, + band_width: u32_param(low_width, job.unsupported_grid)?, + band_height: u32_param(low_height, job.unsupported_grid)?, + }, + ); + dispatch_band( + encoder, + (&x_high_rows, &x_high_taps), + (&y_low_rows, &y_low_taps), + &hl_buffer, + BandGeometry { + width, + height, + block_cols, + band_width: u32_param(high_width, job.unsupported_grid)?, + band_height: u32_param(low_height, job.unsupported_grid)?, + }, + ); + dispatch_band( + encoder, + (&x_low_rows, &x_low_taps), + (&y_high_rows, &y_high_taps), + &lh_buffer, + BandGeometry { + width, + height, + block_cols, + band_width: u32_param(low_width, job.unsupported_grid)?, + band_height: u32_param(high_height, job.unsupported_grid)?, + }, + ); + dispatch_band( + encoder, + (&x_high_rows, &x_high_taps), + (&y_high_rows, &y_high_taps), + &hh_buffer, + BandGeometry { + width, + height, + block_cols, + band_width: u32_param(high_width, job.unsupported_grid)?, + band_height: u32_param(high_height, job.unsupported_grid)?, + }, + ); + + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + + Ok(ProjectedBands { + ll: read_f32_buffer(&ll_buffer, low_width * low_height)?, + hl: read_f32_buffer(&hl_buffer, high_width * low_height)?, + lh: read_f32_buffer(&lh_buffer, low_width * high_height)?, + hh: read_f32_buffer(&hh_buffer, high_width * high_height)?, + low_width, + low_height, + high_width, + high_height, + }) +} + +#[allow(clippy::similar_names)] +fn dispatch_projected_bands_batch_with_runtime( + runtime: &MetalRuntime, + job: ProjectionBatchJob<'_, '_>, +) -> Result, MetalTranscodeError> { + let Some(shape) = projection_batch_shape(job)? else { + return Ok(Vec::new()); + }; + + let weights = projection_batch_weight_buffers(runtime, job)?; + let blocks = dwt97_batch_blocks_buffer(&runtime.device, job.jobs)?; + let outputs = projection_batch_output_buffers(runtime, shape, job.unsupported_grid)?; + + dispatch_projection_batch_bands(runtime, job, shape, &weights, &blocks, &outputs)?; + read_projected_batch_outputs(&outputs, shape, job.unsupported_grid) +} + +#[derive(Clone, Copy)] +struct ProjectionBatchShape { + batch_count: usize, + batch_count_u32: u32, + width: u32, + height: u32, + block_cols: u32, + blocks_per_item: u32, + low_width: usize, + low_height: usize, + high_width: usize, + high_height: usize, + ll_len: usize, + hl_len: usize, + lh_len: usize, + hh_len: usize, +} + +fn projection_batch_shape( + job: ProjectionBatchJob<'_, '_>, +) -> Result, MetalTranscodeError> { + let batch_count = job.jobs.len(); + if batch_count == 0 { + return Ok(None); + } + + let low_width = job.width.div_ceil(2); + let high_width = job.width / 2; + let low_height = job.height.div_ceil(2); + let high_height = job.height / 2; + let blocks_per_item = job + .block_cols + .checked_mul(job.block_rows) + .ok_or(MetalTranscodeError::UnsupportedJob(job.unsupported_grid))?; + + Ok(Some(ProjectionBatchShape { + batch_count, + batch_count_u32: u32_param(batch_count, job.unsupported_grid)?, + width: u32_param(job.width, job.unsupported_grid)?, + height: u32_param(job.height, job.unsupported_grid)?, + block_cols: u32_param(job.block_cols, job.unsupported_grid)?, + blocks_per_item: u32_param(blocks_per_item, job.unsupported_grid)?, + low_width, + low_height, + high_width, + high_height, + ll_len: low_width * low_height, + hl_len: high_width * low_height, + lh_len: low_width * high_height, + hh_len: high_width * high_height, + })) +} + +struct ProjectionBatchWeightBuffers { + x_low_rows: Buffer, + x_low_taps: Buffer, + x_high_rows: Buffer, + x_high_taps: Buffer, + y_low_rows: Buffer, + y_low_taps: Buffer, + y_high_rows: Buffer, + y_high_taps: Buffer, +} + +fn projection_batch_weight_buffers( + runtime: &MetalRuntime, + job: ProjectionBatchJob<'_, '_>, +) -> Result { + let x_low = metal_sparse_rows(job.x_low, job.unsupported_grid)?; + let x_high = metal_sparse_rows(job.x_high, job.unsupported_grid)?; + let y_low = metal_sparse_rows(job.y_low, job.unsupported_grid)?; + let y_high = metal_sparse_rows(job.y_high, job.unsupported_grid)?; + + Ok(ProjectionBatchWeightBuffers { + x_low_rows: buffer_with_slice(&runtime.device, &x_low.rows), + x_low_taps: buffer_with_slice(&runtime.device, &x_low.taps), + x_high_rows: buffer_with_slice(&runtime.device, &x_high.rows), + x_high_taps: buffer_with_slice(&runtime.device, &x_high.taps), + y_low_rows: buffer_with_slice(&runtime.device, &y_low.rows), + y_low_taps: buffer_with_slice(&runtime.device, &y_low.taps), + y_high_rows: buffer_with_slice(&runtime.device, &y_high.rows), + y_high_taps: buffer_with_slice(&runtime.device, &y_high.taps), + }) +} + +struct ProjectionBatchOutputBuffers { + ll: Buffer, + hl: Buffer, + lh: Buffer, + hh: Buffer, +} + +struct Dwt97CodeBlockOutputBuffers { + ll: Buffer, + hl: Buffer, + lh: Buffer, + hh: Buffer, +} + +fn projection_batch_output_buffers( + runtime: &MetalRuntime, + shape: ProjectionBatchShape, + unsupported_grid: &'static str, +) -> Result { + Ok(ProjectionBatchOutputBuffers { + ll: output_buffer( + &runtime.device, + checked_batch_len(shape.ll_len, shape.batch_count, unsupported_grid)?, + ), + hl: output_buffer( + &runtime.device, + checked_batch_len(shape.hl_len, shape.batch_count, unsupported_grid)?, + ), + lh: output_buffer( + &runtime.device, + checked_batch_len(shape.lh_len, shape.batch_count, unsupported_grid)?, + ), + hh: output_buffer( + &runtime.device, + checked_batch_len(shape.hh_len, shape.batch_count, unsupported_grid)?, + ), + }) +} + +fn dwt97_codeblock_output_buffers( + runtime: &MetalRuntime, + shape: ProjectionBatchShape, + unsupported_grid: &'static str, +) -> Result { + Ok(Dwt97CodeBlockOutputBuffers { + ll: output_i32_buffer( + &runtime.device, + checked_batch_len(shape.ll_len, shape.batch_count, unsupported_grid)?, + ), + hl: output_i32_buffer( + &runtime.device, + checked_batch_len(shape.hl_len, shape.batch_count, unsupported_grid)?, + ), + lh: output_i32_buffer( + &runtime.device, + checked_batch_len(shape.lh_len, shape.batch_count, unsupported_grid)?, + ), + hh: output_i32_buffer( + &runtime.device, + checked_batch_len(shape.hh_len, shape.batch_count, unsupported_grid)?, + ), + }) +} + +fn dispatch_projection_batch_bands( + runtime: &MetalRuntime, + job: ProjectionBatchJob<'_, '_>, + shape: ProjectionBatchShape, + weights: &ProjectionBatchWeightBuffers, + blocks: &Buffer, + outputs: &ProjectionBatchOutputBuffers, +) -> Result<(), MetalTranscodeError> { + let command_buffer = runtime.queue.new_command_buffer(); + command_buffer.set_label(job.label); + let encoder = command_buffer.new_compute_command_encoder(); + encoder.set_compute_pipeline_state(&runtime.dct_project_band_batch); + bind_projection_input_buffers(encoder, blocks, &runtime.idct_basis); + + dispatch_band_batch( + encoder, + (&weights.x_low_rows, &weights.x_low_taps), + (&weights.y_low_rows, &weights.y_low_taps), + &outputs.ll, + BatchBandGeometry { + width: shape.width, + height: shape.height, + block_cols: shape.block_cols, + blocks_per_item: shape.blocks_per_item, + band_width: u32_param(shape.low_width, job.unsupported_grid)?, + band_height: u32_param(shape.low_height, job.unsupported_grid)?, + output_stride: u32_param(shape.ll_len, job.unsupported_grid)?, + batch_count: shape.batch_count_u32, + }, + ); + dispatch_band_batch( + encoder, + (&weights.x_high_rows, &weights.x_high_taps), + (&weights.y_low_rows, &weights.y_low_taps), + &outputs.hl, + BatchBandGeometry { + width: shape.width, + height: shape.height, + block_cols: shape.block_cols, + blocks_per_item: shape.blocks_per_item, + band_width: u32_param(shape.high_width, job.unsupported_grid)?, + band_height: u32_param(shape.low_height, job.unsupported_grid)?, + output_stride: u32_param(shape.hl_len, job.unsupported_grid)?, + batch_count: shape.batch_count_u32, + }, + ); + dispatch_band_batch( + encoder, + (&weights.x_low_rows, &weights.x_low_taps), + (&weights.y_high_rows, &weights.y_high_taps), + &outputs.lh, + BatchBandGeometry { + width: shape.width, + height: shape.height, + block_cols: shape.block_cols, + blocks_per_item: shape.blocks_per_item, + band_width: u32_param(shape.low_width, job.unsupported_grid)?, + band_height: u32_param(shape.high_height, job.unsupported_grid)?, + output_stride: u32_param(shape.lh_len, job.unsupported_grid)?, + batch_count: shape.batch_count_u32, + }, + ); + dispatch_band_batch( + encoder, + (&weights.x_high_rows, &weights.x_high_taps), + (&weights.y_high_rows, &weights.y_high_taps), + &outputs.hh, + BatchBandGeometry { + width: shape.width, + height: shape.height, + block_cols: shape.block_cols, + blocks_per_item: shape.blocks_per_item, + band_width: u32_param(shape.high_width, job.unsupported_grid)?, + band_height: u32_param(shape.high_height, job.unsupported_grid)?, + output_stride: u32_param(shape.hh_len, job.unsupported_grid)?, + batch_count: shape.batch_count_u32, + }, + ); + + encoder.end_encoding(); + command_buffer.commit(); + command_buffer.wait_until_completed(); + Ok(()) +} + +fn read_projected_batch_outputs( + buffers: &ProjectionBatchOutputBuffers, + shape: ProjectionBatchShape, + unsupported_grid: &'static str, +) -> Result, MetalTranscodeError> { + let ll = shared_f32_slice( + &buffers.ll, + checked_batch_len(shape.ll_len, shape.batch_count, unsupported_grid)?, + )?; + let hl = shared_f32_slice( + &buffers.hl, + checked_batch_len(shape.hl_len, shape.batch_count, unsupported_grid)?, + )?; + let lh = shared_f32_slice( + &buffers.lh, + checked_batch_len(shape.lh_len, shape.batch_count, unsupported_grid)?, + )?; + let hh = shared_f32_slice( + &buffers.hh, + checked_batch_len(shape.hh_len, shape.batch_count, unsupported_grid)?, + )?; + + let mut outputs = Vec::with_capacity(shape.batch_count); + for idx in 0..shape.batch_count { + outputs.push(ProjectedBands { + ll: f32_slice_to_f64(&ll[idx * shape.ll_len..idx * shape.ll_len + shape.ll_len]), + hl: f32_slice_to_f64(&hl[idx * shape.hl_len..idx * shape.hl_len + shape.hl_len]), + lh: f32_slice_to_f64(&lh[idx * shape.lh_len..idx * shape.lh_len + shape.lh_len]), + hh: f32_slice_to_f64(&hh[idx * shape.hh_len..idx * shape.hh_len + shape.hh_len]), + low_width: shape.low_width, + low_height: shape.low_height, + high_width: shape.high_width, + high_height: shape.high_height, + }); + } + + Ok(outputs) +} + +fn read_prequantized_97_codeblock_outputs( + buffers: &Dwt97CodeBlockOutputBuffers, + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + shape: ProjectionBatchShape, + options: Htj2k97CodeBlockOptions, + unsupported_grid: &'static str, +) -> Result, MetalTranscodeError> { + let ll = shared_i32_slice( + &buffers.ll, + checked_batch_len(shape.ll_len, shape.batch_count, unsupported_grid)?, + )?; + let hl = shared_i32_slice( + &buffers.hl, + checked_batch_len(shape.hl_len, shape.batch_count, unsupported_grid)?, + )?; + let lh = shared_i32_slice( + &buffers.lh, + checked_batch_len(shape.lh_len, shape.batch_count, unsupported_grid)?, + )?; + let hh = shared_i32_slice( + &buffers.hh, + checked_batch_len(shape.hh_len, shape.batch_count, unsupported_grid)?, + )?; + + let mut components = Vec::with_capacity(shape.batch_count); + for (idx, job) in jobs.iter().enumerate() { + components.push(PrequantizedHtj2k97Component { + x_rsiz: job.x_rsiz, + y_rsiz: job.y_rsiz, + resolutions: vec![ + PrequantizedHtj2k97Resolution { + subbands: vec![prequantized_subband_from_codeblock_buffer( + codeblock_item_slice(ll, idx, shape.ll_len, unsupported_grid)?, + shape.low_width, + shape.low_height, + J2kSubBandType::LowLow, + dwt97_total_bitplanes(options, J2kSubBandType::LowLow), + options, + )?], + }, + PrequantizedHtj2k97Resolution { + subbands: vec![ + prequantized_subband_from_codeblock_buffer( + codeblock_item_slice(hl, idx, shape.hl_len, unsupported_grid)?, + shape.high_width, + shape.low_height, + J2kSubBandType::HighLow, + dwt97_total_bitplanes(options, J2kSubBandType::HighLow), + options, + )?, + prequantized_subband_from_codeblock_buffer( + codeblock_item_slice(lh, idx, shape.lh_len, unsupported_grid)?, + shape.low_width, + shape.high_height, + J2kSubBandType::LowHigh, + dwt97_total_bitplanes(options, J2kSubBandType::LowHigh), + options, + )?, + prequantized_subband_from_codeblock_buffer( + codeblock_item_slice(hh, idx, shape.hh_len, unsupported_grid)?, + shape.high_width, + shape.high_height, + J2kSubBandType::HighHigh, + dwt97_total_bitplanes(options, J2kSubBandType::HighHigh), + options, + )?, + ], + }, + ], + }); + } + + Ok(components) +} + +fn codeblock_item_slice<'a>( + values: &'a [i32], + item_idx: usize, + stride: usize, + unsupported_grid: &'static str, +) -> Result<&'a [i32], MetalTranscodeError> { + let start = item_idx + .checked_mul(stride) + .ok_or(MetalTranscodeError::UnsupportedJob(unsupported_grid))?; + let end = start + .checked_add(stride) + .ok_or(MetalTranscodeError::UnsupportedJob(unsupported_grid))?; + values + .get(start..end) + .ok_or(MetalTranscodeError::UnsupportedJob(unsupported_grid)) +} + +fn prequantized_subband_from_codeblock_buffer( + values: &[i32], + width: usize, + height: usize, + sub_band_type: J2kSubBandType, + total_bitplanes: u8, + options: Htj2k97CodeBlockOptions, +) -> Result { + if width == 0 || height == 0 { + return Ok(PrequantizedHtj2k97Subband { + sub_band_type, + num_cbs_x: 0, + num_cbs_y: 0, + total_bitplanes: 0, + code_blocks: Vec::new(), + }); + } + + let cb_width = code_block_len_from_exp(options.code_block_width_exp)?; + let cb_height = code_block_len_from_exp(options.code_block_height_exp)?; + let num_cbs_x = width.div_ceil(cb_width); + let num_cbs_y = height.div_ceil(cb_height); + let mut offset = 0usize; + let mut code_blocks = Vec::with_capacity(num_cbs_x.saturating_mul(num_cbs_y)); + for cby in 0..num_cbs_y { + for cbx in 0..num_cbs_x { + let x0 = cbx * cb_width; + let y0 = cby * cb_height; + let block_width = (width - x0).min(cb_width); + let block_height = (height - y0).min(cb_height); + let len = block_width.checked_mul(block_height).ok_or( + MetalTranscodeError::UnsupportedJob(METAL_DCT97_UNSUPPORTED_GRID), + )?; + let end = offset + .checked_add(len) + .ok_or(MetalTranscodeError::UnsupportedJob( + METAL_DCT97_UNSUPPORTED_GRID, + ))?; + let coefficients = values + .get(offset..end) + .ok_or(MetalTranscodeError::UnsupportedJob( + METAL_DCT97_UNSUPPORTED_GRID, + ))? + .to_vec(); + code_blocks.push(PrequantizedHtj2k97CodeBlock { + coefficients, + width: u32_param(block_width, METAL_DCT97_UNSUPPORTED_GRID)?, + height: u32_param(block_height, METAL_DCT97_UNSUPPORTED_GRID)?, + }); + offset = end; + } + } + + Ok(PrequantizedHtj2k97Subband { + sub_band_type, + num_cbs_x: u32_param(num_cbs_x, METAL_DCT97_UNSUPPORTED_GRID)?, + num_cbs_y: u32_param(num_cbs_y, METAL_DCT97_UNSUPPORTED_GRID)?, + total_bitplanes, + code_blocks, + }) +} + +#[derive(Clone, Copy)] +struct BandGeometry { + width: u32, + height: u32, + block_cols: u32, + band_width: u32, + band_height: u32, +} + +#[derive(Clone, Copy)] +struct BatchBandGeometry { + width: u32, + height: u32, + block_cols: u32, + blocks_per_item: u32, + band_width: u32, + band_height: u32, + output_stride: u32, + batch_count: u32, +} + +#[derive(Clone, Copy)] +struct ReversibleBandGeometry { + width: u32, + height: u32, + block_cols: u32, + blocks_per_item: u32, + band_width: u32, + band_height: u32, + output_stride: u32, + batch_count: u32, + vertical_low: bool, + horizontal_low: bool, +} + +#[derive(Clone, Copy)] +struct ReversibleBatchKernelGeometry { + width: u32, + height: u32, + block_cols: u32, + blocks_per_item: u32, + batch_count: u32, +} + +fn reversible_band_geometry( + base: ReversibleBatchKernelGeometry, + band_width: usize, + band_height: usize, + output_stride: usize, + vertical_low: bool, + horizontal_low: bool, +) -> Result { + Ok(ReversibleBandGeometry { + width: base.width, + height: base.height, + block_cols: base.block_cols, + blocks_per_item: base.blocks_per_item, + band_width: u32_param(band_width, METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID)?, + band_height: u32_param(band_height, METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID)?, + output_stride: u32_param(output_stride, METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID)?, + batch_count: base.batch_count, + vertical_low, + horizontal_low, + }) +} + +fn dispatch_reversible_band( + encoder: &ComputeCommandEncoderRef, + output: &Buffer, + geometry: ReversibleBandGeometry, +) { + if geometry.band_width == 0 || geometry.band_height == 0 { + return; + } + + let params = Reversible53ProjectionParams { + width: geometry.width, + height: geometry.height, + block_cols: geometry.block_cols, + blocks_per_item: geometry.blocks_per_item, + band_width: geometry.band_width, + band_height: geometry.band_height, + output_stride: geometry.output_stride, + vertical_low: u32::from(geometry.vertical_low), + horizontal_low: u32::from(geometry.horizontal_low), + }; + encoder.set_buffer(1, Some(output), 0); + encoder.set_bytes( + 2, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_projection_threads( + encoder, + u64::from(geometry.band_width), + u64::from(geometry.band_height), + u64::from(geometry.batch_count), + ); +} + +fn dispatch_band( + encoder: &ComputeCommandEncoderRef, + x_weights: (&Buffer, &Buffer), + y_weights: (&Buffer, &Buffer), + output: &Buffer, + geometry: BandGeometry, +) { + if geometry.band_width == 0 || geometry.band_height == 0 { + return; + } + + let params = DctProjectionParams { + width: geometry.width, + height: geometry.height, + block_cols: geometry.block_cols, + band_width: geometry.band_width, + band_height: geometry.band_height, + }; + bind_projection_band_buffers(encoder, x_weights, y_weights, output); + encoder.set_bytes( + 7, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_projection_threads( + encoder, + u64::from(geometry.band_width), + u64::from(geometry.band_height), + 1, + ); +} + +fn dispatch_band_batch( + encoder: &ComputeCommandEncoderRef, + x_weights: (&Buffer, &Buffer), + y_weights: (&Buffer, &Buffer), + output: &Buffer, + geometry: BatchBandGeometry, +) { + if geometry.band_width == 0 || geometry.band_height == 0 { + return; + } + + let params = DctBatchProjectionParams { + width: geometry.width, + height: geometry.height, + block_cols: geometry.block_cols, + blocks_per_item: geometry.blocks_per_item, + band_width: geometry.band_width, + band_height: geometry.band_height, + output_stride: geometry.output_stride, + }; + bind_projection_band_buffers(encoder, x_weights, y_weights, output); + encoder.set_bytes( + 7, + size_of::() as u64, + (&raw const params).cast(), + ); + dispatch_projection_threads( + encoder, + u64::from(geometry.band_width), + u64::from(geometry.band_height), + u64::from(geometry.batch_count), + ); +} + +fn validate_grid( + block_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + unsupported_grid: &'static str, +) -> Result<(), MetalTranscodeError> { + let expected_blocks = block_cols + .checked_mul(block_rows) + .ok_or(MetalTranscodeError::UnsupportedJob(unsupported_grid))?; + let covered_width = block_cols + .checked_mul(8) + .ok_or(MetalTranscodeError::UnsupportedJob(unsupported_grid))?; + let covered_height = block_rows + .checked_mul(8) + .ok_or(MetalTranscodeError::UnsupportedJob(unsupported_grid))?; + + if block_count != expected_blocks + || width == 0 + || height == 0 + || width > covered_width + || height > covered_height + { + return Err(MetalTranscodeError::UnsupportedJob(unsupported_grid)); + } + Ok(()) +} + +fn validate_reversible_batch_geometry( + jobs: &[DctGridToReversibleDwt53Job<'_>], +) -> Result<(), MetalTranscodeError> { + let Some(first) = jobs.first() else { + return Ok(()); + }; + + for job in jobs { + validate_grid( + job.dequantized_blocks.len(), + job.block_cols, + job.block_rows, + job.width, + job.height, + METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID, + )?; + + if job.block_cols != first.block_cols + || job.block_rows != first.block_rows + || job.width != first.width + || job.height != first.height + { + return Err(MetalTranscodeError::UnsupportedJob( + METAL_REVERSIBLE_DCT53_UNSUPPORTED_GRID, + )); + } + } + + Ok(()) +} + +fn validate_dwt97_batch_geometry( + jobs: &[DctGridToDwt97Job<'_>], +) -> Result<(), MetalTranscodeError> { + let Some(first) = jobs.first() else { + return Ok(()); + }; + + for job in jobs { + validate_grid( + job.blocks.len(), + job.block_cols, + job.block_rows, + job.width, + job.height, + METAL_DCT97_UNSUPPORTED_GRID, + )?; + + if job.block_cols != first.block_cols + || job.block_rows != first.block_rows + || job.width != first.width + || job.height != first.height + { + return Err(MetalTranscodeError::UnsupportedJob( + METAL_DCT97_UNSUPPORTED_GRID, + )); + } + } + + Ok(()) +} + +fn validate_dwt97_codeblock_batch_geometry( + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], +) -> Result<(), MetalTranscodeError> { + let Some(first) = jobs.first() else { + return Ok(()); + }; + + for job in jobs { + validate_grid( + job.blocks.len(), + job.block_cols, + job.block_rows, + job.width, + job.height, + METAL_DCT97_UNSUPPORTED_GRID, + )?; + + if job.block_cols != first.block_cols + || job.block_rows != first.block_rows + || job.width != first.width + || job.height != first.height + { + return Err(MetalTranscodeError::UnsupportedJob( + METAL_DCT97_UNSUPPORTED_GRID, + )); + } + } + + Ok(()) +} + +fn validate_htj2k97_codeblock_options( + options: Htj2k97CodeBlockOptions, +) -> Result<(), MetalTranscodeError> { + // Shared with CUDA so the two backends accept/reject identical options. + // Option failures keep their own message instead of the grid-geometry one. + j2k_transcode::htj2k97_codeblock_oracle::validate_htj2k97_codeblock_options(options) + .map(|_| ()) + .map_err(MetalTranscodeError::UnsupportedJob) +} + +fn code_block_len_from_exp(exp: u8) -> Result { + 1usize + .checked_shl(u32::from(exp) + 2) + .filter(|&value| value > 0) + .ok_or(MetalTranscodeError::UnsupportedJob( + METAL_DCT97_UNSUPPORTED_GRID, + )) +} + +fn dwt97_total_bitplanes(options: Htj2k97CodeBlockOptions, sub_band_type: J2kSubBandType) -> u8 { + htj2k97_subband_total_bitplanes(options, sub_band_type) +} + +fn dwt97_quantize_inv_delta( + options: Htj2k97CodeBlockOptions, + sub_band_type: J2kSubBandType, +) -> f32 { + (1.0 / htj2k97_subband_delta(options, sub_band_type)) as f32 +} + +fn checked_batch_len( + value_len: usize, + batch_count: usize, + unsupported_grid: &'static str, +) -> Result { + value_len + .checked_mul(batch_count) + .ok_or(MetalTranscodeError::UnsupportedJob(unsupported_grid)) +} + +fn u32_param(value: usize, unsupported_grid: &'static str) -> Result { + u32::try_from(value).map_err(|_| MetalTranscodeError::UnsupportedJob(unsupported_grid)) +} + +fn metal_sparse_rows( + rows: &[SparseWeightRow], + unsupported_grid: &'static str, +) -> Result { + let mut metal_rows = Vec::with_capacity(rows.len()); + let mut taps = Vec::new(); + for row in rows { + let offset = u32_param(taps.len(), unsupported_grid)?; + let count = u32_param(row.taps.len(), unsupported_grid)?; + metal_rows.push(MetalSparseRow { offset, count }); + for tap in &row.taps { + taps.push(MetalWeightTap { + sample_idx: u32_param(tap.sample_idx, unsupported_grid)?, + weight: tap.weight, + }); + } + } + Ok(MetalSparseRows { + rows: metal_rows, + taps, + }) +} + +fn buffer_with_slice(device: &Device, values: &[T]) -> Buffer { + shared_buffer_with_slice(device, values) +} + +fn dwt97_blocks_buffer( + device: &Device, + blocks: &[[[f64; 8]; 8]], +) -> Result { + let value_count = dwt97_block_value_count(blocks.len())?; + let buffer = output_buffer(device, value_count); + write_dwt97_blocks_to_buffer(&buffer, blocks)?; + Ok(buffer) +} + +fn dwt97_batch_blocks_buffer( + device: &Device, + jobs: &[DctGridToDwt97Job<'_>], +) -> Result { + let value_count = dwt97_jobs_value_count(jobs.iter().map(|job| job.blocks.len()))?; + let buffer = output_buffer(device, value_count); + let mut offset = 0; + for job in jobs { + offset += write_dwt97_blocks_to_buffer_at(&buffer, offset, job.blocks)?; + } + debug_assert_eq!(offset, value_count); + Ok(buffer) +} + +fn dwt97_codeblock_batch_blocks_buffer( + device: &Device, + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], +) -> Result { + let value_count = dwt97_jobs_value_count(jobs.iter().map(|job| job.blocks.len()))?; + let buffer = output_buffer(device, value_count); + let mut offset = 0; + for job in jobs { + offset += write_dwt97_blocks_to_buffer_at(&buffer, offset, job.blocks)?; + } + debug_assert_eq!(offset, value_count); + Ok(buffer) +} + +fn dwt97_jobs_value_count( + mut block_counts: impl Iterator, +) -> Result { + block_counts.try_fold(0_usize, |total, block_count| { + let block_values = dwt97_block_value_count(block_count)?; + total + .checked_add(block_values) + .ok_or(MetalTranscodeError::UnsupportedJob( + METAL_DCT97_UNSUPPORTED_GRID, + )) + }) +} + +fn dwt97_block_value_count(block_count: usize) -> Result { + let value_count = block_count.checked_mul(DWT97_BLOCK_COEFFICIENTS).ok_or( + MetalTranscodeError::UnsupportedJob(METAL_DCT97_UNSUPPORTED_GRID), + )?; + value_count + .checked_mul(size_of::()) + .ok_or(MetalTranscodeError::UnsupportedJob( + METAL_DCT97_UNSUPPORTED_GRID, + ))?; + Ok(value_count) +} + +fn write_dwt97_blocks_to_buffer( + buffer: &Buffer, + blocks: &[[[f64; 8]; 8]], +) -> Result<(), MetalTranscodeError> { + let written = write_dwt97_blocks_to_buffer_at(buffer, 0, blocks)?; + debug_assert_eq!(written, dwt97_block_value_count(blocks.len())?); + Ok(()) +} + +fn write_dwt97_blocks_to_buffer_at( + buffer: &Buffer, + start: usize, + blocks: &[[[f64; 8]; 8]], +) -> Result { + let value_count = dwt97_block_value_count(blocks.len())?; + let end = start + .checked_add(value_count) + .ok_or(MetalTranscodeError::UnsupportedJob( + METAL_DCT97_UNSUPPORTED_GRID, + ))?; + if end > buffer_f32_capacity(buffer) { + return Err(MetalTranscodeError::UnsupportedJob( + METAL_DCT97_UNSUPPORTED_GRID, + )); + } + + let mut offset = start; + // SAFETY: Metal ABI structs are repr(C) plain data matching shader layouts. + unsafe { + // SAFETY: `buffer_f32_capacity` above proves every write from + // `start..end` is within the shared f32 buffer. The buffer is newly + // allocated by these packers and is not aliased for CPU writes while + // this function fills it. + let values = buffer.contents().cast::(); + for block in blocks { + for row in block { + for &coefficient in row { + values.add(offset).write(coefficient as f32); + offset += 1; + } + } + } + } + Ok(offset - start) +} + +fn buffer_f32_capacity(buffer: &Buffer) -> usize { + let element_size = size_of::() as u64; + usize::try_from(buffer.length() / element_size).unwrap_or(usize::MAX) +} + +fn output_buffer(device: &Device, value_count: usize) -> Buffer { + shared_buffer_for_len::(device, value_count) +} + +fn output_i32_buffer(device: &Device, value_count: usize) -> Buffer { + shared_buffer_for_len::(device, value_count) +} + +fn read_f32_buffer(buffer: &Buffer, value_count: usize) -> Result, MetalTranscodeError> { + shared_f32_slice(buffer, value_count).map(f32_slice_to_f64) +} + +fn read_i32_buffer(buffer: &Buffer, value_count: usize) -> Result, MetalTranscodeError> { + shared_i32_slice(buffer, value_count).map(<[i32]>::to_vec) +} + +fn shared_f32_slice(buffer: &Buffer, value_count: usize) -> Result<&[f32], MetalTranscodeError> { + if value_count == 0 { + return Ok(&[]); + } + checked_buffer_contents_slice(buffer, 0, value_count) + .map_err(|_| MetalTranscodeError::Kernel(METAL_DCT_KERNEL_FAILED)) +} + +fn shared_i32_slice(buffer: &Buffer, value_count: usize) -> Result<&[i32], MetalTranscodeError> { + if value_count == 0 { + return Ok(&[]); + } + checked_buffer_contents_slice(buffer, 0, value_count) + .map_err(|_| MetalTranscodeError::Kernel(METAL_DCT_KERNEL_FAILED)) +} + +fn f32_slice_to_f64(values: &[f32]) -> Vec { + values.iter().map(|&value| f64::from(value)).collect() +} + +fn idct8_basis_table() -> [f32; 64] { + let mut table = [0.0; 64]; + for sample_idx in 0..8 { + for freq in 0..8 { + table[sample_idx * 8 + freq] = idct8_basis(sample_idx, freq); + } + } + table +} + +fn idct8_basis(sample_idx: usize, freq: usize) -> f32 { + let scale = if freq == 0 { + (1.0_f32 / 8.0).sqrt() + } else { + (2.0_f32 / 8.0).sqrt() + }; + scale * (((sample_idx as f32 + 0.5) * freq as f32 * PI) / 8.0).cos() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn projection_dispatch_sizes_use_16_by_8_threadgroups() { + let (threads, threadgroup) = projection_dispatch_sizes(5, 6, 7); + + assert_eq!((threads.width, threads.height, threads.depth), (5, 6, 7)); + assert_eq!( + (threadgroup.width, threadgroup.height, threadgroup.depth), + (16, 8, 1) + ); + } + + #[test] + fn dwt97_block_value_count_rejects_overflow() { + assert_eq!(dwt97_block_value_count(2), Ok(128)); + assert_eq!( + dwt97_block_value_count(usize::MAX), + Err(MetalTranscodeError::UnsupportedJob( + METAL_DCT97_UNSUPPORTED_GRID + )) + ); + } +} diff --git a/crates/j2k-transcode-metal/src/weights.rs b/crates/j2k-transcode-metal/src/weights.rs new file mode 100644 index 00000000..7bdbfb3b --- /dev/null +++ b/crates/j2k-transcode-metal/src/weights.rs @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Scalar-derived wavelet projection weight rows for Metal kernels. + +const ALPHA: f64 = -1.586_134_342_059_924; +const BETA: f64 = -0.052_980_118_572_961; +const GAMMA: f64 = 0.882_911_075_530_934; +const DELTA: f64 = 0.443_506_852_043_971; +const KAPPA: f64 = 1.230_174_104_914_001; +const INV_KAPPA: f64 = 1.0 / KAPPA; + +/// One-dimensional 9/7 projection weights for every output row. +#[derive(Debug, Clone, PartialEq)] +pub struct Dwt97WeightRows { + /// Low-pass output rows, each indexed by input sample position. + pub low: Vec>, + /// High-pass output rows, each indexed by input sample position. + pub high: Vec>, +} + +impl Dwt97WeightRows { + /// Build deterministic 9/7 projection rows for a one-dimensional sample + /// extent. + #[must_use] + pub fn for_len(sample_len: usize) -> Self { + let mut low = vec![vec![0.0; sample_len]; low_len(sample_len)]; + let mut high = vec![vec![0.0; sample_len]; high_len(sample_len)]; + + for sample_idx in 0..sample_len { + let mut basis = vec![0.0; sample_len]; + basis[sample_idx] = 1.0; + let transformed = linearized_97_from_sample_slice(&basis); + + for (row, &weight) in low.iter_mut().zip(transformed.low.iter()) { + row[sample_idx] = weight as f32; + } + for (row, &weight) in high.iter_mut().zip(transformed.high.iter()) { + row[sample_idx] = weight as f32; + } + } + + Self { low, high } + } +} + +/// One-dimensional 5/3 projection weights for every output row. +#[derive(Debug, Clone, PartialEq)] +pub struct Dwt53WeightRows { + /// Low-pass output rows, each indexed by input sample position. + pub low: Vec>, + /// High-pass output rows, each indexed by input sample position. + pub high: Vec>, +} + +impl Dwt53WeightRows { + /// Build deterministic 5/3 projection rows for a one-dimensional sample + /// extent. + #[must_use] + pub fn for_len(sample_len: usize) -> Self { + let mut low = vec![vec![0.0; sample_len]; low_len(sample_len)]; + let mut high = vec![vec![0.0; sample_len]; high_len(sample_len)]; + + for sample_idx in 0..sample_len { + let mut basis = vec![0.0; sample_len]; + basis[sample_idx] = 1.0; + let transformed = linearized_53_from_sample_slice(&basis); + + for (row, &weight) in low.iter_mut().zip(transformed.low.iter()) { + row[sample_idx] = weight as f32; + } + for (row, &weight) in high.iter_mut().zip(transformed.high.iter()) { + row[sample_idx] = weight as f32; + } + } + + Self { low, high } + } +} + +/// Sparse one-dimensional 9/7 projection rows. +#[derive(Debug, Clone, PartialEq)] +pub struct SparseDwt97WeightRows { + /// Low-pass sparse output rows. + pub low: Vec, + /// High-pass sparse output rows. + pub high: Vec, +} + +impl SparseDwt97WeightRows { + /// Build sparse 9/7 projection rows for a one-dimensional sample extent. + #[must_use] + pub fn for_len(sample_len: usize) -> Self { + let dense = Dwt97WeightRows::for_len(sample_len); + Self { + low: sparse_rows_from_dense(&dense.low), + high: sparse_rows_from_dense(&dense.high), + } + } + + /// Largest tap count across low-pass and high-pass rows. + #[must_use] + pub fn max_taps_per_row(&self) -> usize { + self.low + .iter() + .chain(self.high.iter()) + .map(|row| row.taps.len()) + .max() + .unwrap_or(0) + } +} + +/// Sparse one-dimensional 5/3 projection rows. +#[derive(Debug, Clone, PartialEq)] +pub struct SparseDwt53WeightRows { + /// Low-pass sparse output rows. + pub low: Vec, + /// High-pass sparse output rows. + pub high: Vec, +} + +impl SparseDwt53WeightRows { + /// Build sparse 5/3 projection rows for a one-dimensional sample extent. + #[must_use] + pub fn for_len(sample_len: usize) -> Self { + let dense = Dwt53WeightRows::for_len(sample_len); + Self { + low: sparse_rows_from_dense(&dense.low), + high: sparse_rows_from_dense(&dense.high), + } + } + + /// Largest tap count across low-pass and high-pass rows. + #[must_use] + pub fn max_taps_per_row(&self) -> usize { + self.low + .iter() + .chain(self.high.iter()) + .map(|row| row.taps.len()) + .max() + .unwrap_or(0) + } +} + +/// Sparse row of sample-position weights. +#[derive(Debug, Clone, PartialEq)] +pub struct SparseWeightRow { + /// Nonzero taps in sample-index order. + pub taps: Vec, +} + +/// One nonzero sample-position weight. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SparseWeightTap { + /// Input sample index. + pub sample_idx: usize, + /// Weight applied to that sample. + pub weight: f32, +} + +fn sparse_rows_from_dense(rows: &[Vec]) -> Vec { + rows.iter() + .map(|row| SparseWeightRow { + taps: row + .iter() + .copied() + .enumerate() + .filter(|&(_, weight)| weight.to_bits() != 0) + .map(|(sample_idx, weight)| SparseWeightTap { sample_idx, weight }) + .collect(), + }) + .collect() +} + +fn linearized_53_from_sample_slice(samples: &[f64]) -> Dwt53OneDimensional { + let mut high = Vec::with_capacity(high_len(samples.len())); + for odd_idx in (1..samples.len()).step_by(2) { + let left = samples[odd_idx - 1]; + let right = samples.get(odd_idx + 1).copied().unwrap_or(left); + high.push(samples[odd_idx] - ((left + right) * 0.5)); + } + + let mut low = Vec::with_capacity(low_len(samples.len())); + for even_idx in (0..samples.len()).step_by(2) { + let current = samples[even_idx]; + let even_output_idx = even_idx / 2; + let left_high = even_output_idx.checked_sub(1).and_then(|idx| high.get(idx)); + let right_high = high.get(even_output_idx); + let update = match (left_high, right_high) { + (Some(left), Some(right)) => (*left + *right) * 0.25, + (None, Some(right)) => *right * 0.5, + (Some(left), None) => *left * 0.5, + (None, None) => 0.0, + }; + low.push(current + update); + } + + Dwt53OneDimensional { low, high } +} + +fn linearized_97_from_sample_slice(samples: &[f64]) -> Dwt97OneDimensional { + let mut lifted = samples.to_vec(); + forward_lift_97(&mut lifted); + + Dwt97OneDimensional { + low: lifted.iter().step_by(2).copied().collect(), + high: lifted.iter().skip(1).step_by(2).copied().collect(), + } +} + +fn forward_lift_97(data: &mut [f64]) { + let sample_count = data.len(); + if sample_count < 2 { + return; + } + + let last_even = if sample_count.is_multiple_of(2) { + sample_count - 2 + } else { + sample_count - 1 + }; + + for sample_idx in (1..sample_count).step_by(2) { + let left = data[sample_idx - 1]; + let right = if sample_idx + 1 < sample_count { + data[sample_idx + 1] + } else { + data[last_even] + }; + data[sample_idx] += ALPHA * (left + right); + } + + for sample_idx in (0..sample_count).step_by(2) { + let left = if sample_idx > 0 { + data[sample_idx - 1] + } else { + data[1] + }; + let right = if sample_idx + 1 < sample_count { + data[sample_idx + 1] + } else { + left + }; + data[sample_idx] += BETA * (left + right); + } + + for sample_idx in (1..sample_count).step_by(2) { + let left = data[sample_idx - 1]; + let right = if sample_idx + 1 < sample_count { + data[sample_idx + 1] + } else { + data[last_even] + }; + data[sample_idx] += GAMMA * (left + right); + } + + for sample_idx in (0..sample_count).step_by(2) { + let left = if sample_idx > 0 { + data[sample_idx - 1] + } else { + data[1] + }; + let right = if sample_idx + 1 < sample_count { + data[sample_idx + 1] + } else { + left + }; + data[sample_idx] += DELTA * (left + right); + } + + for sample_idx in (0..sample_count).step_by(2) { + data[sample_idx] *= INV_KAPPA; + } + for sample_idx in (1..sample_count).step_by(2) { + data[sample_idx] *= KAPPA; + } +} + +const fn low_len(sample_len: usize) -> usize { + sample_len.div_ceil(2) +} + +const fn high_len(sample_len: usize) -> usize { + sample_len / 2 +} + +struct Dwt97OneDimensional { + low: Vec, + high: Vec, +} + +struct Dwt53OneDimensional { + low: Vec, + high: Vec, +} diff --git a/crates/j2k-transcode-metal/tests/bench_harness.rs b/crates/j2k-transcode-metal/tests/bench_harness.rs new file mode 100644 index 00000000..163a67f7 --- /dev/null +++ b/crates/j2k-transcode-metal/tests/bench_harness.rs @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: Apache-2.0 + +const DCT97_BENCH: &str = include_str!("../benches/dct97.rs"); + +#[test] +fn dct97_benchmark_groups_are_stable() { + for expected in [ + "dct53_metal_projection", + "jpeg_to_htj2k_wsi_53", + "reversible_dct53_metal_projection", + "reversible_dct53_batch_metal_projection", + "jpeg_to_htj2k_wsi_integer_53", + "jpeg_to_htj2k_wsi_integer_53_tile_batch", + "metal_auto", + "rayon_batch", + "metal_auto_batch", + "metal_explicit_batch", + "rayon_224x224", + "rayon_512x512", + "rayon_1024x1024", + "rayon_2048x2048", + "batch_1", + "batch_8", + "batch_32", + "batch_128", + "batch_256", + "batch_512", + "rayon_224x224_batch_1", + "metal_explicit_224x224_batch_1", + "rayon_512x512_batch_512", + "rayon_1024x1024_batch_128", + "rayon_2048x2048_batch_32", + "dct97_metal_idct_dwt", + "cpu_idct_dwt_224x224", + "metal_explicit_224x224", + "cpu_idct_dwt_512x512", + "metal_explicit_512x512", + "cpu_idct_dwt_1024x1024", + "metal_explicit_1024x1024", + "cpu_idct_dwt_2048x2048", + "metal_explicit_2048x2048", + "jpeg_to_htj2k_wsi_97", + "srgb_ybr420_224", + "srgb_ybr420_512", + "srgb_ybr420_1024", + "srgb_ybr420_2048", + "srgb_ybr420_224_batch_128", + "srgb_ybr420_512_batch_128", + "srgb_ybr420_1024_batch_128", + "srgb_ybr420_2048_batch_128", + "srgb_ybr420_2048_batch_256", + "srgb_ybr420_2048_batch_512", + "p3_like_ybr444_224", + "p3_like_ybr444_512", + "p3_like_ybr444_1024", + "p3_like_ybr444_2048", + "p3_like_ybr444_224_batch_128", + "p3_like_ybr444_512_batch_256", + "p3_like_ybr444_1024_batch_512", + "p3_like_ybr444_2048_batch_512", + "ycbcr_like_ybr420_224", + "ycbcr_like_ybr420_512", + "ycbcr_like_ybr420_1024", + "ycbcr_like_ybr420_2048", + "ycbcr_like_ybr420_224_batch_512", + "ycbcr_like_ybr420_512_batch_512", + "ycbcr_like_ybr420_1024_batch_512", + "ycbcr_like_ybr420_2048_batch_512", + ] { + assert!( + DCT97_BENCH.contains(expected), + "missing benchmark marker {expected}" + ); + } +} + +#[test] +fn dct97_benchmark_emits_transcode_batch_profile_rows() { + for expected in [ + "J2K_TRANSCODE_METAL_PROFILE_STAGES", + "emit_transcode_batch_profile", + "j2k_profile codec=transcode op=transcode_batch path=metal_cpu_hybrid", + "pipeline=jpeg_to_htj2k_hybrid", + "extract_processor=cpu", + "transform_processor=metal", + "encode_processor=cpu", + "pipeline_map", + "debug_report", + "jpeg_dct_extract_us", + "dct_to_wavelet_total_us", + "htj2k_encode_us", + "accelerator_dispatches", + "cpu_fallback_jobs", + ] { + assert!( + DCT97_BENCH.contains(expected), + "missing transcode batch profile marker {expected}" + ); + } +} diff --git a/crates/j2k-transcode-metal/tests/dct53.rs b/crates/j2k-transcode-metal/tests/dct53.rs new file mode 100644 index 00000000..d1c29dbe --- /dev/null +++ b/crates/j2k-transcode-metal/tests/dct53.rs @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_transcode::accelerator::TranscodeStageError; +use j2k_transcode::accelerator::{ + DctGridToDwt53Job, DctGridToReversibleDwt53Job, DctToWaveletStageAccelerator, +}; +#[cfg(target_os = "macos")] +use j2k_transcode::accelerator::{RayonReversibleDwt53Accelerator, ReversibleDwt53FirstLevel}; +#[cfg(target_os = "macos")] +use j2k_transcode::dct53_2d::{ + dct8x8_blocks_to_dwt53_float_linear_with_scratch, Dct53GridScratch, Dwt53TwoDimensional, +}; +use j2k_transcode_metal::weights::{Dwt53WeightRows, SparseDwt53WeightRows}; +use j2k_transcode_metal::MetalDctToWaveletStageAccelerator; + +#[test] +fn explicit_metal_53_reports_unavailable_on_non_macos() { + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + let blocks = vec![[[0.0; 8]; 8]]; + let result = accelerator.dct_grid_to_dwt53(DctGridToDwt53Job { + blocks: &blocks, + block_cols: 1, + block_rows: 1, + width: 8, + height: 8, + }); + + #[cfg(not(target_os = "macos"))] + assert_eq!( + result.expect_err("explicit Metal is unavailable off macOS"), + TranscodeStageError::DeviceUnavailable + ); + + #[cfg(target_os = "macos")] + let _ = result; +} + +#[test] +fn explicit_metal_reversible_53_reports_unavailable_on_non_macos() { + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + let blocks = vec![[0i16; 64]]; + let result = accelerator.dct_grid_to_reversible_dwt53(DctGridToReversibleDwt53Job { + dequantized_blocks: &blocks, + block_cols: 1, + block_rows: 1, + width: 8, + height: 8, + }); + + #[cfg(not(target_os = "macos"))] + assert_eq!( + result.expect_err("explicit Metal is unavailable off macOS"), + TranscodeStageError::DeviceUnavailable + ); + + #[cfg(target_os = "macos")] + let _ = result; +} + +#[test] +fn auto_metal_53_falls_back_for_tiny_jobs() { + let mut accelerator = MetalDctToWaveletStageAccelerator::for_auto(); + let blocks = vec![[[0.0; 8]; 8]]; + let output = accelerator + .dct_grid_to_dwt53(DctGridToDwt53Job { + blocks: &blocks, + block_cols: 1, + block_rows: 1, + width: 8, + height: 8, + }) + .expect("auto accelerator can decline tiny 5/3 job"); + + assert!(output.is_none()); + assert_eq!(accelerator.dwt53_attempts(), 1); + assert_eq!(accelerator.dwt53_dispatches(), 0); +} + +#[test] +fn auto_metal_reversible_53_declines_tiny_jobs() { + let mut accelerator = MetalDctToWaveletStageAccelerator::for_auto(); + let blocks = vec![[0i16; 64]]; + let output = accelerator + .dct_grid_to_reversible_dwt53(DctGridToReversibleDwt53Job { + dequantized_blocks: &blocks, + block_cols: 1, + block_rows: 1, + width: 8, + height: 8, + }) + .expect("auto accelerator can decline tiny reversible 5/3 job"); + + // `None` hands the job to the caller's scalar fallback — the same + // contract as the float 5/3 path and the CUDA accelerator. + assert!(output.is_none()); + assert_eq!(accelerator.reversible_dwt53_attempts(), 1); + assert_eq!(accelerator.reversible_dwt53_dispatches(), 0); +} + +#[test] +fn auto_metal_reversible_53_batch_declines_tiny_jobs() { + let mut accelerator = MetalDctToWaveletStageAccelerator::for_auto(); + let blocks = vec![[0i16; 64]]; + let jobs = [DctGridToReversibleDwt53Job { + dequantized_blocks: &blocks, + block_cols: 1, + block_rows: 1, + width: 8, + height: 8, + }]; + let output = accelerator + .dct_grid_to_reversible_dwt53_batch(&jobs) + .expect("auto accelerator can decline tiny reversible 5/3 batch"); + + assert!(output.is_none()); + assert_eq!(accelerator.reversible_dwt53_batch_attempts(), 1); + assert_eq!(accelerator.reversible_dwt53_batch_dispatches(), 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn explicit_metal_dct53_matches_scalar_for_structured_cases() { + let blocks = structured_blocks(2, 2); + let mut scalar_scratch = Dct53GridScratch::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + for (width, height) in [(8, 8), (13, 11), (16, 16)] { + let actual = match accelerator.dct_grid_to_dwt53(DctGridToDwt53Job { + blocks: &blocks, + block_cols: 2, + block_rows: 2, + width, + height, + }) { + Ok(Some(output)) => output, + Ok(None) => panic!("explicit Metal accelerator must not silently fall back"), + Err(TranscodeStageError::DeviceUnavailable) => { + eprintln!( + "skipping Metal 5/3 coefficient test because no Metal device is available" + ); + return; + } + Err(message) => panic!("explicit Metal 5/3 accelerator failed: {message}"), + }; + let expected = dct8x8_blocks_to_dwt53_float_linear_with_scratch( + &blocks, + 2, + 2, + width, + height, + &mut scalar_scratch, + ) + .expect("scalar 5/3 projection accepts covered grid"); + + let max_diff = max_abs_diff(&actual, &expected); + assert!( + max_diff <= 2.0e-2, + "Metal 5/3 DCT projection diverged for {width}x{height}: {max_diff}" + ); + } + + assert_eq!(accelerator.dwt53_dispatches(), 3); +} + +#[cfg(target_os = "macos")] +#[test] +fn explicit_metal_reversible_dct53_matches_rayon_for_structured_cases() { + let blocks = structured_i16_blocks(2, 2); + let mut expected_accelerator = RayonReversibleDwt53Accelerator::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + for (width, height) in [(8, 8), (13, 11), (16, 16)] { + let job = DctGridToReversibleDwt53Job { + dequantized_blocks: &blocks, + block_cols: 2, + block_rows: 2, + width, + height, + }; + let actual = match accelerator.dct_grid_to_reversible_dwt53(job) { + Ok(Some(output)) => output, + Ok(None) => panic!("explicit Metal accelerator must not silently fall back"), + Err(TranscodeStageError::DeviceUnavailable) => { + eprintln!( + "skipping Metal reversible 5/3 test because no Metal device is available" + ); + return; + } + Err(message) => panic!("explicit Metal reversible 5/3 accelerator failed: {message}"), + }; + let expected = expected_accelerator + .dct_grid_to_reversible_dwt53(job) + .expect("rayon reversible 5/3 accepts covered grid") + .expect("rayon handles reversible 5/3 job"); + + assert_reversible_eq(&actual, &expected, width, height); + } + + assert_eq!(accelerator.reversible_dwt53_dispatches(), 3); +} + +#[cfg(target_os = "macos")] +#[test] +fn explicit_metal_reversible_dct53_batch_matches_rayon_for_structured_cases() { + let batch_blocks = [ + structured_i16_blocks_with_offset(2, 2, 0), + structured_i16_blocks_with_offset(2, 2, 31), + structured_i16_blocks_with_offset(2, 2, -27), + structured_i16_blocks_with_offset(2, 2, 59), + ]; + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + for (width, height) in [(8, 8), (13, 11), (16, 16)] { + let jobs: Vec<_> = batch_blocks + .iter() + .map(|blocks| DctGridToReversibleDwt53Job { + dequantized_blocks: blocks, + block_cols: 2, + block_rows: 2, + width, + height, + }) + .collect(); + let actual = match accelerator.dct_grid_to_reversible_dwt53_batch(&jobs) { + Ok(Some(output)) => output, + Ok(None) => panic!("explicit Metal batch accelerator must not silently fall back"), + Err(TranscodeStageError::DeviceUnavailable) => { + eprintln!( + "skipping Metal reversible 5/3 batch test because no Metal device is available" + ); + return; + } + Err(message) => { + panic!("explicit Metal reversible 5/3 batch accelerator failed: {message}"); + } + }; + + assert_eq!(actual.len(), jobs.len()); + for (idx, (actual, job)) in actual.iter().zip(jobs.iter()).enumerate() { + let mut expected_accelerator = RayonReversibleDwt53Accelerator::default(); + let expected = expected_accelerator + .dct_grid_to_reversible_dwt53(*job) + .expect("rayon reversible 5/3 accepts covered grid") + .expect("rayon handles reversible 5/3 job"); + assert_eq!( + actual, &expected, + "reversible 5/3 batch mismatch for item {idx} at {width}x{height}" + ); + } + } + + assert_eq!(accelerator.reversible_dwt53_batch_dispatches(), 3); +} + +#[test] +fn dwt53_weight_rows_match_expected_geometry_for_supported_lengths() { + for sample_len in [8_usize, 13, 16] { + let rows = Dwt53WeightRows::for_len(sample_len); + + assert_eq!(rows.low.len(), sample_len.div_ceil(2)); + assert_eq!(rows.high.len(), sample_len / 2); + assert!(rows.low.iter().all(|row| row.len() == sample_len)); + assert!(rows.high.iter().all(|row| row.len() == sample_len)); + assert!(rows + .low + .iter() + .all(|row| row.iter().any(|&value| value.to_bits() != 0))); + assert!(rows + .high + .iter() + .all(|row| row.iter().any(|&value| value.to_bits() != 0))); + } +} + +#[test] +fn dwt53_weight_rows_are_deterministic() { + let first = Dwt53WeightRows::for_len(13); + let second = Dwt53WeightRows::for_len(13); + + assert_eq!(f32_rows_to_bits(&first.low), f32_rows_to_bits(&second.low)); + assert_eq!( + f32_rows_to_bits(&first.high), + f32_rows_to_bits(&second.high) + ); +} + +#[test] +fn sparse_dwt53_weight_rows_reconstruct_dense_rows_for_wsi_lengths() { + for sample_len in [8_usize, 13, 16, 224, 512, 1024, 2048] { + let dense = Dwt53WeightRows::for_len(sample_len); + let sparse = SparseDwt53WeightRows::for_len(sample_len); + + assert!(sparse.max_taps_per_row() <= 5); + assert_eq!(sparse.low.len(), dense.low.len()); + assert_eq!(sparse.high.len(), dense.high.len()); + assert_eq!(reconstruct_sparse_rows(&sparse.low, sample_len), dense.low); + assert_eq!( + reconstruct_sparse_rows(&sparse.high, sample_len), + dense.high + ); + } +} + +fn f32_rows_to_bits(rows: &[Vec]) -> Vec> { + rows.iter() + .map(|row| row.iter().map(|value| value.to_bits()).collect()) + .collect() +} + +fn reconstruct_sparse_rows( + rows: &[j2k_transcode_metal::weights::SparseWeightRow], + sample_len: usize, +) -> Vec> { + rows.iter() + .map(|row| { + let mut dense = vec![0.0; sample_len]; + for tap in &row.taps { + dense[tap.sample_idx] = tap.weight; + } + dense + }) + .collect() +} + +#[cfg(target_os = "macos")] +fn assert_reversible_eq( + actual: &ReversibleDwt53FirstLevel, + expected: &ReversibleDwt53FirstLevel, + width: usize, + height: usize, +) { + assert_eq!( + actual, expected, + "reversible 5/3 mismatch for {width}x{height}" + ); +} + +#[cfg(target_os = "macos")] +fn max_abs_diff(actual: &Dwt53TwoDimensional, expected: &Dwt53TwoDimensional) -> f64 { + assert_eq!(actual.low_width, expected.low_width); + assert_eq!(actual.low_height, expected.low_height); + assert_eq!(actual.high_width, expected.high_width); + assert_eq!(actual.high_height, expected.high_height); + + actual + .ll + .iter() + .zip(expected.ll.iter()) + .chain(actual.hl.iter().zip(expected.hl.iter())) + .chain(actual.lh.iter().zip(expected.lh.iter())) + .chain(actual.hh.iter().zip(expected.hh.iter())) + .map(|(actual, expected)| (actual - expected).abs()) + .fold(0.0, f64::max) +} + +#[cfg(target_os = "macos")] +fn structured_blocks(block_cols: usize, block_rows: usize) -> Vec<[[f64; 8]; 8]> { + let mut blocks = Vec::with_capacity(block_cols * block_rows); + for block_y in 0..block_rows { + for block_x in 0..block_cols { + let mut block = [[0.0; 8]; 8]; + block[0][0] = 384.0 + (block_x * 19 + block_y * 23) as f64; + block[0][1] = -17.0 + block_x as f64; + block[1][0] = 11.0 - block_y as f64; + block[2][3] = 7.0; + block[4][4] = -3.0; + block[7][7] = 2.0; + blocks.push(block); + } + } + blocks +} + +#[cfg(target_os = "macos")] +fn structured_i16_blocks(block_cols: usize, block_rows: usize) -> Vec<[i16; 64]> { + structured_i16_blocks_with_offset(block_cols, block_rows, 0) +} + +#[cfg(target_os = "macos")] +fn structured_i16_blocks_with_offset( + block_cols: usize, + block_rows: usize, + base_offset: i16, +) -> Vec<[i16; 64]> { + let mut blocks = Vec::with_capacity(block_cols * block_rows); + for block_y in 0..block_rows { + for block_x in 0..block_cols { + let mut block = [0i16; 64]; + let block_offset = + i16::try_from(block_x * 19 + block_y * 23).expect("fixture offset fits i16"); + let x_offset = i16::try_from(block_x).expect("fixture x offset fits i16"); + let y_offset = i16::try_from(block_y).expect("fixture y offset fits i16"); + block[0] = 384 + base_offset + block_offset; + block[1] = -17 + x_offset; + block[8] = 11 - y_offset; + block[19] = 7; + block[36] = -3; + block[63] = 2; + blocks.push(block); + } + } + blocks +} diff --git a/crates/j2k-transcode-metal/tests/dct97.rs b/crates/j2k-transcode-metal/tests/dct97.rs new file mode 100644 index 00000000..6b978c0b --- /dev/null +++ b/crates/j2k-transcode-metal/tests/dct97.rs @@ -0,0 +1,558 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_transcode::accelerator::TranscodeStageError; +use j2k_transcode::accelerator::{DctGridToDwt97Job, DctToWaveletStageAccelerator}; +#[cfg(target_os = "macos")] +use j2k_transcode::accelerator::{ + DctGridToHtj2k97CodeBlockJob, Htj2k97CodeBlockOptions, IrreversibleQuantizationSubbandScales, + PrequantizedHtj2k97Component, +}; +#[cfg(target_os = "macos")] +use j2k_transcode::dct97_2d::{ + dct8x8_blocks_then_dwt97_float_with_scratch, Dct97GridScratch, Dwt97TwoDimensional, +}; +#[cfg(target_os = "macos")] +use j2k_transcode::htj2k97_codeblock_oracle::prequantized_component_from_dwt97; +use j2k_transcode_metal::weights::{Dwt97WeightRows, SparseDwt97WeightRows}; +use j2k_transcode_metal::MetalDctToWaveletStageAccelerator; + +#[test] +fn explicit_metal_reports_unavailable_on_non_macos() { + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + let blocks = vec![[[0.0; 8]; 8]]; + let result = accelerator.dct_grid_to_dwt97(DctGridToDwt97Job { + blocks: &blocks, + block_cols: 1, + block_rows: 1, + width: 8, + height: 8, + }); + + #[cfg(not(target_os = "macos"))] + assert_eq!( + result.expect_err("explicit Metal is unavailable off macOS"), + TranscodeStageError::DeviceUnavailable + ); + + #[cfg(target_os = "macos")] + let _ = result; +} + +#[test] +fn auto_metal_falls_back_for_tiny_jobs() { + let mut accelerator = MetalDctToWaveletStageAccelerator::for_auto(); + let blocks = vec![[[0.0; 8]; 8]]; + let output = accelerator + .dct_grid_to_dwt97(DctGridToDwt97Job { + blocks: &blocks, + block_cols: 1, + block_rows: 1, + width: 8, + height: 8, + }) + .expect("auto accelerator can decline tiny job"); + + assert!(output.is_none()); + assert_eq!(accelerator.dwt97_attempts(), 1); + assert_eq!(accelerator.dwt97_dispatches(), 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn auto_metal_uses_cpu_for_97_jobs_by_default() { + let blocks = structured_blocks(64, 64); + let mut accelerator = MetalDctToWaveletStageAccelerator::for_auto(); + + match accelerator.dct_grid_to_dwt97(DctGridToDwt97Job { + blocks: &blocks, + block_cols: 64, + block_rows: 64, + width: 512, + height: 512, + }) { + Ok(None) | Err(TranscodeStageError::DeviceUnavailable) => {} + Ok(Some(_)) => panic!("auto Metal should leave 9/7 jobs on the optimized CPU path"), + Err(message) => panic!("auto Metal 9/7 accelerator failed: {message}"), + } + + assert_eq!(accelerator.dwt97_attempts(), 1); + assert_eq!(accelerator.dwt97_dispatches(), 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn explicit_metal_dct97_matches_scalar_for_structured_cases() { + let blocks = structured_blocks(2, 2); + let mut scalar_scratch = Dct97GridScratch::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + for (width, height) in [(8, 8), (13, 11), (16, 16)] { + let actual = match accelerator.dct_grid_to_dwt97(DctGridToDwt97Job { + blocks: &blocks, + block_cols: 2, + block_rows: 2, + width, + height, + }) { + Ok(Some(output)) => output, + Ok(None) => panic!("explicit Metal accelerator must not silently fall back"), + Err(TranscodeStageError::DeviceUnavailable) => { + eprintln!("skipping Metal coefficient test because no Metal device is available"); + return; + } + Err(message) => panic!("explicit Metal accelerator failed: {message}"), + }; + let expected = dct8x8_blocks_then_dwt97_float_with_scratch( + &blocks, + 2, + 2, + width, + height, + &mut scalar_scratch, + ) + .expect("scalar 9/7 IDCT path accepts covered grid"); + + let max_diff = max_abs_diff(&actual, &expected); + assert!( + max_diff <= 2.0e-2, + "Metal 9/7 DCT transform diverged for {width}x{height}: {max_diff}" + ); + } + + assert_eq!(accelerator.dwt97_dispatches(), 3); +} + +#[cfg(target_os = "macos")] +#[test] +fn explicit_metal_dct97_batch_matches_scalar_for_structured_cases() { + let first = structured_blocks(2, 2); + let second = structured_blocks_with_offset(2, 2, 97.0); + let jobs = [ + DctGridToDwt97Job { + blocks: &first, + block_cols: 2, + block_rows: 2, + width: 13, + height: 11, + }, + DctGridToDwt97Job { + blocks: &second, + block_cols: 2, + block_rows: 2, + width: 13, + height: 11, + }, + ]; + let mut scalar_scratch = Dct97GridScratch::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + let actual = match accelerator.dct_grid_to_dwt97_batch(&jobs) { + Ok(Some(output)) => output, + Ok(None) => panic!("explicit Metal batch accelerator must not silently fall back"), + Err(TranscodeStageError::DeviceUnavailable) => { + eprintln!("skipping Metal batch coefficient test because no Metal device is available"); + return; + } + Err(message) => panic!("explicit Metal batch accelerator failed: {message}"), + }; + + assert_eq!(actual.len(), jobs.len()); + for (actual, job) in actual.iter().zip(jobs.iter()) { + let expected = dct8x8_blocks_then_dwt97_float_with_scratch( + job.blocks, + job.block_cols, + job.block_rows, + job.width, + job.height, + &mut scalar_scratch, + ) + .expect("scalar 9/7 IDCT path accepts covered grid"); + + let max_diff = max_abs_diff(actual, &expected); + assert!( + max_diff <= 2.0e-2, + "Metal 9/7 batch transform diverged: {max_diff}" + ); + } + + assert_eq!(accelerator.dwt97_batch_dispatches(), 1); +} + +#[cfg(target_os = "macos")] +#[test] +fn explicit_metal_dct97_batch_reports_idct_row_and_column_stage_timings() { + let batch_blocks = [ + structured_blocks_with_offset(4, 4, 0.0), + structured_blocks_with_offset(4, 4, 3.0), + ]; + let jobs: Vec<_> = batch_blocks + .iter() + .map(|blocks| DctGridToDwt97Job { + blocks, + block_cols: 4, + block_rows: 4, + width: 29, + height: 31, + }) + .collect(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + match accelerator.dct_grid_to_dwt97_batch(&jobs) { + Ok(Some(_)) => {} + Ok(None) => panic!("explicit Metal batch accelerator must not silently fall back"), + Err(TranscodeStageError::DeviceUnavailable) => { + eprintln!("skipping Metal batch timing test because no Metal device is available"); + return; + } + Err(message) => panic!("explicit Metal batch accelerator failed: {message}"), + } + + let timings = accelerator + .last_dwt97_batch_stage_timings() + .expect("Metal 9/7 batch records backend stage timings"); + assert!(timings.pack_upload_us > 0); + assert!(timings.idct_row_lift_us > 0); + assert!(timings.column_lift_us > 0); + assert_eq!(timings.quantize_codeblock_us, 0); + assert!(timings.readback_us > 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn explicit_metal_dct97_codeblock_batch_matches_scalar_quantized_layout() { + let batch_blocks = [ + structured_blocks_with_offset(4, 4, 0.0), + structured_blocks_with_offset(4, 4, 37.0), + ]; + let jobs: Vec<_> = batch_blocks + .iter() + .map(|blocks| DctGridToHtj2k97CodeBlockJob { + blocks, + block_cols: 4, + block_rows: 4, + width: 29, + height: 31, + x_rsiz: 1, + y_rsiz: 1, + }) + .collect(); + let options = Htj2k97CodeBlockOptions { + bit_depth: 8, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + irreversible_quantization_scale: 2.5, + irreversible_quantization_subband_scales: IrreversibleQuantizationSubbandScales { + low_low: 0.9, + high_low: 1.1, + low_high: 1.2, + high_high: 1.5, + }, + }; + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + let actual = match accelerator.dct_grid_to_htj2k97_codeblock_batch(&jobs, options) { + Ok(Some(output)) => output, + Ok(None) => { + panic!("explicit Metal code-block batch accelerator must not silently fall back") + } + Err(TranscodeStageError::DeviceUnavailable) => { + eprintln!("skipping Metal code-block batch test because no Metal device is available"); + return; + } + Err(message) => panic!("explicit Metal code-block batch accelerator failed: {message}"), + }; + + assert_eq!(actual.len(), jobs.len()); + let mut scalar_scratch = Dct97GridScratch::default(); + for (actual, job) in actual.iter().zip(jobs.iter()) { + let dwt = dct8x8_blocks_then_dwt97_float_with_scratch( + job.blocks, + job.block_cols, + job.block_rows, + job.width, + job.height, + &mut scalar_scratch, + ) + .expect("scalar 9/7 IDCT path accepts covered grid"); + let expected = prequantized_component_from_dwt97(&dwt, options, job.x_rsiz, job.y_rsiz); + + assert_prequantized_component_layout_eq(actual, &expected); + assert_prequantized_component_coefficients_close(actual, &expected, 1); + } + + assert_eq!(accelerator.dwt97_batch_dispatches(), 1); + assert_eq!(accelerator.htj2k97_codeblock_batch_attempts(), 1); + assert_eq!(accelerator.htj2k97_codeblock_batch_dispatches(), 1); + let timings = accelerator + .last_dwt97_batch_stage_timings() + .expect("Metal code-block batch records backend stage timings"); + assert!(timings.pack_upload_us > 0); + assert!(timings.idct_row_lift_us > 0); + assert!(timings.column_lift_us > 0); + assert!(timings.quantize_codeblock_us > 0); + assert!(timings.readback_us > 0); +} + +#[cfg(target_os = "macos")] +#[test] +fn explicit_metal_dct97_codeblock_batch_accepts_zero_guard_bits_and_matches_scalar() { + // The old Metal-only validator rejected guard_bits == 0; the shared + // validator accepts it (CUDA and the native encoder always did). This + // pins the Metal kernel against the CPU oracle for the newly reachable + // option so the widening cannot silently produce divergent code-blocks. + let blocks = structured_blocks_with_offset(4, 4, 19.0); + let jobs = [DctGridToHtj2k97CodeBlockJob { + blocks: &blocks, + block_cols: 4, + block_rows: 4, + width: 29, + height: 31, + x_rsiz: 1, + y_rsiz: 1, + }]; + let options = Htj2k97CodeBlockOptions { + bit_depth: 8, + guard_bits: 0, + code_block_width_exp: 2, + code_block_height_exp: 2, + irreversible_quantization_scale: 2.5, + irreversible_quantization_subband_scales: IrreversibleQuantizationSubbandScales { + low_low: 0.9, + high_low: 1.1, + low_high: 1.2, + high_high: 1.5, + }, + }; + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + let actual = match accelerator.dct_grid_to_htj2k97_codeblock_batch(&jobs, options) { + Ok(Some(output)) => output, + Ok(None) => { + panic!("explicit Metal code-block batch accelerator must not silently fall back") + } + Err(TranscodeStageError::DeviceUnavailable) => { + eprintln!("skipping Metal zero-guard-bits test because no Metal device is available"); + return; + } + Err(message) => panic!("explicit Metal rejected guard_bits == 0: {message}"), + }; + + assert_eq!(actual.len(), jobs.len()); + let mut scalar_scratch = Dct97GridScratch::default(); + let job = &jobs[0]; + let dwt = dct8x8_blocks_then_dwt97_float_with_scratch( + job.blocks, + job.block_cols, + job.block_rows, + job.width, + job.height, + &mut scalar_scratch, + ) + .expect("scalar 9/7 IDCT path accepts covered grid"); + let expected = prequantized_component_from_dwt97(&dwt, options, job.x_rsiz, job.y_rsiz); + + assert_prequantized_component_layout_eq(&actual[0], &expected); + assert_prequantized_component_coefficients_close(&actual[0], &expected, 1); +} + +#[test] +fn weight_rows_match_expected_geometry_for_supported_lengths() { + for sample_len in [8_usize, 13, 16] { + let rows = Dwt97WeightRows::for_len(sample_len); + + assert_eq!(rows.low.len(), sample_len.div_ceil(2)); + assert_eq!(rows.high.len(), sample_len / 2); + assert!(rows.low.iter().all(|row| row.len() == sample_len)); + assert!(rows.high.iter().all(|row| row.len() == sample_len)); + assert!(rows + .low + .iter() + .all(|row| row.iter().any(|&value| value.to_bits() != 0))); + assert!(rows + .high + .iter() + .all(|row| row.iter().any(|&value| value.to_bits() != 0))); + } +} + +#[test] +fn weight_rows_are_deterministic() { + let first = Dwt97WeightRows::for_len(13); + let second = Dwt97WeightRows::for_len(13); + + assert_eq!(f32_rows_to_bits(&first.low), f32_rows_to_bits(&second.low)); + assert_eq!( + f32_rows_to_bits(&first.high), + f32_rows_to_bits(&second.high) + ); +} + +#[test] +fn sparse_weight_rows_reconstruct_dense_rows_for_wsi_lengths() { + for sample_len in [8_usize, 13, 16, 512, 1024, 2048] { + let dense = Dwt97WeightRows::for_len(sample_len); + let sparse = SparseDwt97WeightRows::for_len(sample_len); + + assert!(sparse.max_taps_per_row() <= 16); + assert_eq!(sparse.low.len(), dense.low.len()); + assert_eq!(sparse.high.len(), dense.high.len()); + assert_eq!(reconstruct_sparse_rows(&sparse.low, sample_len), dense.low); + assert_eq!( + reconstruct_sparse_rows(&sparse.high, sample_len), + dense.high + ); + } +} + +fn f32_rows_to_bits(rows: &[Vec]) -> Vec> { + rows.iter() + .map(|row| row.iter().map(|value| value.to_bits()).collect()) + .collect() +} + +fn reconstruct_sparse_rows( + rows: &[j2k_transcode_metal::weights::SparseWeightRow], + sample_len: usize, +) -> Vec> { + rows.iter() + .map(|row| { + let mut dense = vec![0.0; sample_len]; + for tap in &row.taps { + dense[tap.sample_idx] = tap.weight; + } + dense + }) + .collect() +} + +#[cfg(target_os = "macos")] +fn max_abs_diff(actual: &Dwt97TwoDimensional, expected: &Dwt97TwoDimensional) -> f64 { + assert_eq!(actual.low_width, expected.low_width); + assert_eq!(actual.low_height, expected.low_height); + assert_eq!(actual.high_width, expected.high_width); + assert_eq!(actual.high_height, expected.high_height); + + actual + .ll + .iter() + .zip(expected.ll.iter()) + .chain(actual.hl.iter().zip(expected.hl.iter())) + .chain(actual.lh.iter().zip(expected.lh.iter())) + .chain(actual.hh.iter().zip(expected.hh.iter())) + .map(|(actual, expected)| (actual - expected).abs()) + .fold(0.0, f64::max) +} + +#[cfg(target_os = "macos")] +fn assert_prequantized_component_layout_eq( + actual: &PrequantizedHtj2k97Component, + expected: &PrequantizedHtj2k97Component, +) { + assert_eq!(actual.x_rsiz, expected.x_rsiz); + assert_eq!(actual.y_rsiz, expected.y_rsiz); + assert_eq!(actual.resolutions.len(), expected.resolutions.len()); + for (actual_resolution, expected_resolution) in + actual.resolutions.iter().zip(expected.resolutions.iter()) + { + assert_eq!( + actual_resolution.subbands.len(), + expected_resolution.subbands.len() + ); + for (actual_subband, expected_subband) in actual_resolution + .subbands + .iter() + .zip(expected_resolution.subbands.iter()) + { + assert_eq!(actual_subband.sub_band_type, expected_subband.sub_band_type); + assert_eq!(actual_subband.num_cbs_x, expected_subband.num_cbs_x); + assert_eq!(actual_subband.num_cbs_y, expected_subband.num_cbs_y); + assert_eq!( + actual_subband.total_bitplanes, + expected_subband.total_bitplanes + ); + assert_eq!( + actual_subband.code_blocks.len(), + expected_subband.code_blocks.len() + ); + for (actual_block, expected_block) in actual_subband + .code_blocks + .iter() + .zip(expected_subband.code_blocks.iter()) + { + assert_eq!(actual_block.width, expected_block.width); + assert_eq!(actual_block.height, expected_block.height); + assert_eq!( + actual_block.coefficients.len(), + expected_block.coefficients.len() + ); + } + } + } +} + +#[cfg(target_os = "macos")] +fn assert_prequantized_component_coefficients_close( + actual: &PrequantizedHtj2k97Component, + expected: &PrequantizedHtj2k97Component, + max_abs_error: i32, +) { + for (actual_resolution, expected_resolution) in + actual.resolutions.iter().zip(expected.resolutions.iter()) + { + for (actual_subband, expected_subband) in actual_resolution + .subbands + .iter() + .zip(expected_resolution.subbands.iter()) + { + for (actual_block, expected_block) in actual_subband + .code_blocks + .iter() + .zip(expected_subband.code_blocks.iter()) + { + for (&actual_coefficient, &expected_coefficient) in actual_block + .coefficients + .iter() + .zip(expected_block.coefficients.iter()) + { + assert!( + (actual_coefficient - expected_coefficient).abs() <= max_abs_error, + "quantized coefficient diverged: actual {actual_coefficient}, expected {expected_coefficient}" + ); + } + } + } + } +} + +#[cfg(target_os = "macos")] +fn structured_blocks(block_cols: usize, block_rows: usize) -> Vec<[[f64; 8]; 8]> { + let mut blocks = Vec::with_capacity(block_cols * block_rows); + for block_y in 0..block_rows { + for block_x in 0..block_cols { + let mut block = [[0.0; 8]; 8]; + block[0][0] = 384.0 + (block_x * 19 + block_y * 23) as f64; + block[0][1] = -17.0 + block_x as f64; + block[1][0] = 11.0 - block_y as f64; + block[2][3] = 7.0; + block[4][4] = -3.0; + block[7][7] = 2.0; + blocks.push(block); + } + } + blocks +} + +#[cfg(target_os = "macos")] +fn structured_blocks_with_offset( + block_cols: usize, + block_rows: usize, + offset: f64, +) -> Vec<[[f64; 8]; 8]> { + let mut blocks = structured_blocks(block_cols, block_rows); + for block in &mut blocks { + block[0][0] += offset; + block[3][2] -= offset / 7.0; + } + blocks +} diff --git a/crates/j2k-transcode-metal/tests/jpeg_to_htj2k.rs b/crates/j2k-transcode-metal/tests/jpeg_to_htj2k.rs new file mode 100644 index 00000000..72f7c4ce --- /dev/null +++ b/crates/j2k-transcode-metal/tests/jpeg_to_htj2k.rs @@ -0,0 +1,450 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(target_os = "macos")] +use j2k_native::{DecodeSettings, Image}; +#[cfg(target_os = "macos")] +use j2k_test_support::{ + jpeg_baseline_420_16x16, jpeg_baseline_422_16x8, jpeg_baseline_444_8x8, jpeg_grayscale_8x8, +}; +#[cfg(target_os = "macos")] +use j2k_transcode::accelerator::TranscodeStageError; +#[cfg(target_os = "macos")] +use j2k_transcode::JpegToHtj2kError; +#[cfg(target_os = "macos")] +use j2k_transcode::{ + JpegTileBatchInput, JpegToHtj2kCoefficientPath, JpegToHtj2kOptions, JpegToHtj2kTranscoder, +}; +#[cfg(target_os = "macos")] +use j2k_transcode_metal::MetalDctToWaveletStageAccelerator; + +#[cfg(target_os = "macos")] +#[test] +fn ycbcr_420_jpeg_transcodes_to_htj2k_with_explicit_metal_97_and_native_sampling() { + let jpeg = jpeg_baseline_420_16x16(); + let options = JpegToHtj2kOptions { + validate_against_float_reference: true, + ..JpegToHtj2kOptions::lossy_97() + }; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + let encoded = match transcoder.transcode_with_accelerator(&jpeg, &options, &mut accelerator) { + Ok(encoded) => encoded, + Err(JpegToHtj2kError::Accelerator(TranscodeStageError::DeviceUnavailable)) => { + eprintln!( + "skipping Metal transcode integration test because no Metal device is available" + ); + return; + } + Err(error) => panic!("explicit Metal 9/7 transcode failed: {error}"), + }; + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated Metal 9/7 HTJ2K") + .decode_native() + .expect("native decoder accepts generated Metal 9/7 HTJ2K"); + let metrics = encoded + .report + .float_reference_metrics + .as_ref() + .expect("float reference metrics are reported"); + + assert_eq!( + encoded.report.coefficient_path, + JpegToHtj2kCoefficientPath::FloatDirectLinear97 + ); + assert_eq!( + encoded.report.path, + "native_component_sampling_float_direct_97" + ); + assert_eq!(metrics.total, 384); + assert_eq!(metrics.max_abs_error, 0); + assert_eq!(accelerator.dwt97_attempts(), 3); + assert_eq!(accelerator.dwt97_dispatches(), 3); + assert_eq!((decoded.width, decoded.height), (16, 16)); + assert_eq!(decoded.num_components, 3); + assert_report_sampling( + &encoded.report.components, + &[(16, 16, 1, 1), (8, 8, 2, 2), (8, 8, 2, 2)], + ); + assert_component_sampling(&encoded.codestream, &[(1, 1), (2, 2), (2, 2)]); +} + +#[cfg(target_os = "macos")] +#[test] +fn ycbcr_420_jpeg_transcodes_to_htj2k_with_explicit_metal_53_and_native_sampling() { + let jpeg = jpeg_baseline_420_16x16(); + let options = JpegToHtj2kOptions { + coefficient_path: JpegToHtj2kCoefficientPath::FloatDirectLinear53, + validate_against_float_reference: true, + ..JpegToHtj2kOptions::lossless_53() + }; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + let encoded = match transcoder.transcode_with_accelerator(&jpeg, &options, &mut accelerator) { + Ok(encoded) => encoded, + Err(JpegToHtj2kError::Accelerator(TranscodeStageError::DeviceUnavailable)) => { + eprintln!( + "skipping Metal transcode integration test because no Metal device is available" + ); + return; + } + Err(error) => panic!("explicit Metal 5/3 transcode failed: {error}"), + }; + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated Metal 5/3 HTJ2K") + .decode_native() + .expect("native decoder accepts generated Metal 5/3 HTJ2K"); + let metrics = encoded + .report + .float_reference_metrics + .as_ref() + .expect("float reference metrics are reported"); + + assert_eq!( + encoded.report.coefficient_path, + JpegToHtj2kCoefficientPath::FloatDirectLinear53 + ); + assert_eq!( + encoded.report.path, + "native_component_sampling_float_direct_53" + ); + assert_eq!(metrics.total, 384); + assert_eq!(metrics.max_abs_error, 0); + assert_eq!(accelerator.dwt53_attempts(), 3); + assert_eq!(accelerator.dwt53_dispatches(), 3); + assert_eq!((decoded.width, decoded.height), (16, 16)); + assert_eq!(decoded.num_components, 3); + assert_report_sampling( + &encoded.report.components, + &[(16, 16, 1, 1), (8, 8, 2, 2), (8, 8, 2, 2)], + ); + assert_component_sampling(&encoded.codestream, &[(1, 1), (2, 2), (2, 2)]); +} + +#[cfg(target_os = "macos")] +#[test] +fn grayscale_jpeg_transcodes_to_htj2k_with_explicit_metal_reversible_53_batch() { + assert_explicit_metal_integer53_matches_scalar( + &jpeg_grayscale_8x8(), + "full_resolution_components_integer_direct_53", + &[(8, 8, 1, 1)], + &[(1, 1)], + 1, + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn ycbcr_444_jpeg_transcodes_to_htj2k_with_explicit_metal_reversible_53_batch() { + assert_explicit_metal_integer53_matches_scalar( + &jpeg_baseline_444_8x8(), + "full_resolution_components_integer_direct_53", + &[(8, 8, 1, 1), (8, 8, 1, 1), (8, 8, 1, 1)], + &[(1, 1), (1, 1), (1, 1)], + 1, + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn ycbcr_422_jpeg_transcodes_to_htj2k_with_explicit_metal_reversible_53_batch() { + assert_explicit_metal_integer53_matches_scalar( + &jpeg_baseline_422_16x8(), + "native_component_sampling_integer_direct_53", + &[(16, 8, 1, 1), (8, 8, 2, 1), (8, 8, 2, 1)], + &[(1, 1), (2, 1), (2, 1)], + 2, + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn ycbcr_420_jpeg_transcodes_to_htj2k_with_explicit_metal_reversible_53_batch() { + assert_explicit_metal_integer53_matches_scalar( + &jpeg_baseline_420_16x16(), + "native_component_sampling_integer_direct_53", + &[(16, 16, 1, 1), (8, 8, 2, 2), (8, 8, 2, 2)], + &[(1, 1), (2, 2), (2, 2)], + 2, + ); +} + +#[cfg(target_os = "macos")] +#[test] +fn ycbcr_420_batch_transcodes_with_explicit_metal_reversible_53_across_tiles() { + let jpeg = jpeg_baseline_420_16x16(); + let inputs = vec![ + JpegTileBatchInput { + bytes: jpeg.as_slice(), + }; + 4 + ]; + let options = JpegToHtj2kOptions { + validate_against_integer_reference: true, + ..JpegToHtj2kOptions::lossless_53() + }; + let mut scalar_transcoder = JpegToHtj2kTranscoder::default(); + let scalar = scalar_transcoder + .transcode(&jpeg, &options) + .expect("scalar IntegerDirect53 transcode succeeds"); + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + let batch = match transcoder.transcode_batch_with_accelerator( + &inputs, + &options, + &mut accelerator, + ) { + Ok(batch) => batch, + Err(JpegToHtj2kError::Accelerator(TranscodeStageError::DeviceUnavailable)) => { + eprintln!( + "skipping Metal reversible batch transcode integration test because no Metal device is available" + ); + return; + } + Err(error) => panic!("explicit Metal reversible 5/3 batch transcode failed: {error}"), + }; + + assert_eq!(batch.report.tile_count, inputs.len()); + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!(batch.report.failed_tiles, 0); + assert_eq!(batch.report.reversible_dwt53_batches, 3); + assert_eq!(batch.report.reversible_dwt53_batch_jobs, 12); + assert_eq!(accelerator.reversible_dwt53_attempts(), 0); + assert_eq!(accelerator.reversible_dwt53_batch_attempts(), 3); + assert_eq!(accelerator.reversible_dwt53_batch_dispatches(), 3); + for tile in batch.tiles { + let tile = tile.expect("valid tile transcodes"); + assert_eq!(tile.codestream, scalar.codestream); + assert_component_sampling(&tile.codestream, &[(1, 1), (2, 2), (2, 2)]); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn ycbcr_420_batch_transcodes_with_explicit_metal_97_across_tiles() { + let jpeg = jpeg_baseline_420_16x16(); + let inputs = vec![ + JpegTileBatchInput { + bytes: jpeg.as_slice(), + }; + 4 + ]; + let options = JpegToHtj2kOptions { + validate_against_float_reference: true, + ..JpegToHtj2kOptions::lossy_97() + }; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + let batch = match transcoder.transcode_batch_with_accelerator( + &inputs, + &options, + &mut accelerator, + ) { + Ok(batch) => batch, + Err(JpegToHtj2kError::Accelerator(TranscodeStageError::DeviceUnavailable)) => { + eprintln!("skipping Metal 9/7 batch transcode integration test because no Metal device is available"); + return; + } + Err(error) => panic!("explicit Metal 9/7 batch transcode failed: {error}"), + }; + + assert_eq!(batch.report.tile_count, inputs.len()); + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!(batch.report.failed_tiles, 0); + assert_eq!(batch.report.timings.batch_jobs, 12); + assert_eq!(batch.report.timings.accelerator_dispatches, 2); + assert_eq!(batch.report.timings.accelerator_dispatched_jobs, 12); + assert_eq!(batch.report.timings.cpu_fallback_jobs, 0); + assert!(batch.report.timings.dwt97_batch_pack_upload_us > 0); + assert!(batch.report.timings.dwt97_batch_idct_row_lift_us > 0); + assert!(batch.report.timings.dwt97_batch_column_lift_us > 0); + assert_eq!(batch.report.timings.dwt97_batch_quantize_codeblock_us, 0); + assert!(batch.report.timings.dwt97_batch_readback_us > 0); + assert_eq!(accelerator.dwt97_attempts(), 0); + assert_eq!(accelerator.dwt97_batch_attempts(), 2); + assert_eq!(accelerator.dwt97_batch_dispatches(), 2); + for tile in batch.tiles { + let tile = tile.expect("valid 9/7 tile transcodes"); + assert_eq!( + tile.report.coefficient_path, + JpegToHtj2kCoefficientPath::FloatDirectLinear97 + ); + assert_eq!( + tile.report.path, + "native_component_sampling_float_direct_97" + ); + assert_eq!( + tile.report + .float_reference_metrics + .as_ref() + .expect("float reference metrics are reported") + .max_abs_error, + 0 + ); + assert_component_sampling(&tile.codestream, &[(1, 1), (2, 2), (2, 2)]); + } +} + +#[cfg(target_os = "macos")] +#[test] +fn ycbcr_420_batch_transcodes_with_explicit_metal_97_codeblock_path() { + let jpeg = jpeg_baseline_420_16x16(); + let inputs = vec![ + JpegTileBatchInput { + bytes: jpeg.as_slice(), + }; + 4 + ]; + let options = JpegToHtj2kOptions::lossy_97(); + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + let batch = match transcoder.transcode_batch_with_accelerator( + &inputs, + &options, + &mut accelerator, + ) { + Ok(batch) => batch, + Err(JpegToHtj2kError::Accelerator(TranscodeStageError::DeviceUnavailable)) => { + eprintln!("skipping Metal 9/7 code-block batch transcode integration test because no Metal device is available"); + return; + } + Err(error) => panic!("explicit Metal 9/7 code-block batch transcode failed: {error}"), + }; + + assert_eq!(batch.report.tile_count, inputs.len()); + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!(batch.report.failed_tiles, 0); + assert_eq!(batch.report.timings.batch_jobs, 12); + assert_eq!(batch.report.timings.accelerator_dispatches, 2); + assert_eq!(batch.report.timings.accelerator_dispatched_jobs, 12); + assert_eq!(batch.report.timings.cpu_fallback_jobs, 0); + assert!(batch.report.timings.dwt97_batch_pack_upload_us > 0); + assert!(batch.report.timings.dwt97_batch_idct_row_lift_us > 0); + assert!(batch.report.timings.dwt97_batch_column_lift_us > 0); + assert!(batch.report.timings.dwt97_batch_quantize_codeblock_us > 0); + assert!(batch.report.timings.dwt97_batch_readback_us > 0); + assert_eq!(accelerator.dwt97_attempts(), 0); + assert_eq!(accelerator.dwt97_batch_attempts(), 2); + assert_eq!(accelerator.dwt97_batch_dispatches(), 2); + assert_eq!(accelerator.htj2k97_codeblock_batch_attempts(), 2); + assert_eq!(accelerator.htj2k97_codeblock_batch_dispatches(), 2); + for tile in batch.tiles { + let tile = tile.expect("valid 9/7 code-block tile transcodes"); + let decoded = Image::new(&tile.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated Metal code-block 9/7 HTJ2K") + .decode_native() + .expect("native decoder accepts generated Metal code-block 9/7 HTJ2K"); + assert_eq!((decoded.width, decoded.height), (16, 16)); + assert_eq!(decoded.num_components, 3); + assert_eq!( + tile.report.coefficient_path, + JpegToHtj2kCoefficientPath::FloatDirectLinear97 + ); + assert_eq!( + tile.report.path, + "native_component_sampling_float_direct_97" + ); + assert!(tile.report.float_reference_metrics.is_none()); + assert_component_sampling(&tile.codestream, &[(1, 1), (2, 2), (2, 2)]); + } +} + +#[cfg(target_os = "macos")] +fn assert_explicit_metal_integer53_matches_scalar( + jpeg: &[u8], + expected_path: &str, + expected_report_sampling: &[(u32, u32, u8, u8)], + expected_codestream_sampling: &[(u8, u8)], + expected_batch_dispatches: usize, +) { + let options = JpegToHtj2kOptions { + validate_against_integer_reference: true, + ..JpegToHtj2kOptions::lossless_53() + }; + let mut scalar_transcoder = JpegToHtj2kTranscoder::default(); + let scalar = scalar_transcoder + .transcode(jpeg, &options) + .expect("scalar IntegerDirect53 transcode succeeds"); + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = MetalDctToWaveletStageAccelerator::new_explicit(); + + let encoded = match transcoder.transcode_with_accelerator(jpeg, &options, &mut accelerator) { + Ok(encoded) => encoded, + Err(JpegToHtj2kError::Accelerator(TranscodeStageError::DeviceUnavailable)) => { + eprintln!( + "skipping Metal reversible transcode integration test because no Metal device is available" + ); + return; + } + Err(error) => panic!("explicit Metal reversible 5/3 transcode failed: {error}"), + }; + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated Metal reversible 5/3 HTJ2K") + .decode_native() + .expect("native decoder accepts generated Metal reversible 5/3 HTJ2K"); + let metrics = encoded + .report + .integer_reference_metrics + .as_ref() + .expect("integer reference metrics are reported"); + + assert_eq!(encoded.codestream, scalar.codestream); + assert_eq!( + encoded.report.coefficient_path, + JpegToHtj2kCoefficientPath::IntegerDirect53 + ); + assert_eq!(encoded.report.path, expected_path); + assert_eq!(metrics.max_abs_error, 0); + assert_eq!(metrics.exact_matches, metrics.total); + assert_eq!(accelerator.reversible_dwt53_attempts(), 0); + assert_eq!(accelerator.reversible_dwt53_dispatches(), 0); + assert_eq!( + accelerator.reversible_dwt53_batch_attempts(), + expected_batch_dispatches + ); + assert_eq!( + accelerator.reversible_dwt53_batch_dispatches(), + expected_batch_dispatches + ); + assert_eq!( + (decoded.width, decoded.height), + (encoded.report.width, encoded.report.height) + ); + assert_eq!(decoded.num_components, expected_report_sampling.len() as u8); + assert_report_sampling(&encoded.report.components, expected_report_sampling); + assert_component_sampling(&encoded.codestream, expected_codestream_sampling); +} + +#[cfg(target_os = "macos")] +fn assert_report_sampling( + components: &[j2k_transcode::TranscodeComponentReport], + expected: &[(u32, u32, u8, u8)], +) { + assert_eq!(components.len(), expected.len()); + for (component, &(width, height, x_rsiz, y_rsiz)) in components.iter().zip(expected.iter()) { + assert_eq!((component.width, component.height), (width, height)); + assert_eq!((component.x_rsiz, component.y_rsiz), (x_rsiz, y_rsiz)); + } +} + +#[cfg(target_os = "macos")] +fn assert_component_sampling(codestream: &[u8], expected: &[(u8, u8)]) { + let siz = find_marker(codestream, 0x51).expect("SIZ marker"); + let component_info = siz + 40; + for (component_index, &(x_rsiz, y_rsiz)) in expected.iter().enumerate() { + let offset = component_info + component_index * 3; + assert_eq!(codestream[offset + 1], x_rsiz); + assert_eq!(codestream[offset + 2], y_rsiz); + } +} + +#[cfg(target_os = "macos")] +fn find_marker(codestream: &[u8], marker: u8) -> Option { + codestream + .windows(2) + .position(|window| window == [0xff, marker]) +} diff --git a/crates/j2k-transcode-metal/tests/shader_integrity.rs b/crates/j2k-transcode-metal/tests/shader_integrity.rs new file mode 100644 index 00000000..086b401b --- /dev/null +++ b/crates/j2k-transcode-metal/tests/shader_integrity.rs @@ -0,0 +1,14 @@ +use j2k_test_support::unwired_metal_kernels; + +const METAL_SOURCE: &str = include_str!("../src/metal.rs"); +const SHADER_SOURCE: &str = include_str!("../src/dct97.metal"); + +#[test] +fn metal_kernels_are_wired_to_host_pipelines() { + let unused = unwired_metal_kernels([SHADER_SOURCE], METAL_SOURCE); + + assert!( + unused.is_empty(), + "Metal kernels must be compiled by host pipeline setup or removed: {unused:?}" + ); +} diff --git a/crates/j2k-transcode/Cargo.toml b/crates/j2k-transcode/Cargo.toml new file mode 100644 index 00000000..5a591f18 --- /dev/null +++ b/crates/j2k-transcode/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "j2k-transcode" +description = "JPEG to J2K and HTJ2K transcode primitives for j2k" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +readme = "README.md" + +[package.metadata.docs.rs] +all-features = true +targets = [] + +[lib] +name = "j2k_transcode" +path = "src/lib.rs" + +[dependencies] +rayon = { workspace = true } +j2k-jpeg = { path = "../j2k-jpeg", version = "=0.6.0" } +j2k = { path = "../j2k", version = "=0.6.0" } +j2k-native = { path = "../j2k-native", version = "=0.6.0" } + +[dev-dependencies] +criterion = { workspace = true } +proptest = { workspace = true } +j2k-test-support = { path = "../j2k-test-support" } + +[[bench]] +name = "dct53" +harness = false + +[lints.rust] +unsafe_code = "forbid" +unreachable_pub = "warn" + +[lints.clippy] +pedantic = { level = "warn", priority = -1 } +module_name_repetitions = "allow" +must_use_candidate = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +cast_possible_truncation = "allow" +cast_precision_loss = "allow" diff --git a/crates/j2k-transcode/README.md b/crates/j2k-transcode/README.md new file mode 100644 index 00000000..6ea02256 --- /dev/null +++ b/crates/j2k-transcode/README.md @@ -0,0 +1,7 @@ +# j2k-transcode + +JPEG to J2K/HTJ2K transcode crate for J2K. + +The crate owns CPU transcode algorithms and shared accelerator hooks. CUDA and +Metal acceleration live in adapter crates. Unsupported source classes and +unsupported transcode modes return explicit errors. diff --git a/crates/j2k-transcode/benches/dct53.rs b/crates/j2k-transcode/benches/dct53.rs new file mode 100644 index 00000000..fa9cc08c --- /dev/null +++ b/crates/j2k-transcode/benches/dct53.rs @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: Apache-2.0 + +use criterion::{criterion_group, criterion_main, Criterion}; +use j2k_jpeg::transcode::{extract_dct_blocks, DctExtractOptions}; +use j2k_jpeg::{ + encode_jpeg_baseline, JpegBackend, JpegEncodeOptions, JpegSamples, JpegSubsampling, +}; +use j2k_test_support::{ + JPEG_BASELINE_420_16X16, JPEG_BASELINE_420_RESTART_32X16, JPEG_BASELINE_422_16X8, + JPEG_BASELINE_444_8X8, JPEG_GRAYSCALE_8X8, +}; +use j2k_transcode::dct53_1d::{ + dct8_blocks_to_dwt53_float_linear, dct8_to_dwt53_float_linear, idct8_blocks_then_dwt53_float, + idct8_then_dwt53_float, +}; +use j2k_transcode::dct53_2d::{ + dct8x8_blocks_then_dwt53_float, dct8x8_blocks_to_dwt53_float_linear, + dct8x8_blocks_to_dwt53_float_linear_with_scratch, dct8x8_to_dwt53_float_linear, + idct8x8_then_dwt53_float, Dct53GridScratch, +}; +use j2k_transcode::dct53_multilevel::{ + dct8x8_to_dwt53_multilevel_float_linear, idct8x8_then_dwt53_multilevel_float, +}; +use j2k_transcode::dct97_2d::{ + dct8x8_blocks_then_dwt97_float, dct8x8_blocks_then_dwt97_float_with_scratch, Dct97GridScratch, +}; +use j2k_transcode::{jpeg_to_htj2k, JpegToHtj2kOptions, JpegToHtj2kTranscoder}; +use std::hint::black_box; + +fn bench_dct53_math(c: &mut Criterion) { + let coeffs = [91.0, -36.0, 14.0, -9.0, 3.0, 22.0, -11.0, 4.0]; + + let mut single_block = c.benchmark_group("dct53_1d_single_block_scalar"); + single_block.bench_function("direct_linear", |b| { + b.iter(|| dct8_to_dwt53_float_linear(black_box(coeffs))); + }); + single_block.bench_function("idct_then_dwt_reference", |b| { + b.iter(|| idct8_then_dwt53_float(black_box(coeffs))); + }); + single_block.finish(); + + let blocks = pseudo_random_blocks(32); + + let mut multi_block = c.benchmark_group("dct53_1d_multi_block_scalar"); + multi_block.bench_function("direct_linear", |b| { + b.iter(|| dct8_blocks_to_dwt53_float_linear(black_box(&blocks))); + }); + multi_block.bench_function("idct_then_dwt_reference", |b| { + b.iter(|| idct8_blocks_then_dwt53_float(black_box(&blocks))); + }); + multi_block.finish(); + + let block_2d = synthetic_8x8_block(); + + let mut two_dimensional = c.benchmark_group("dct53_2d_single_level_scalar"); + two_dimensional.bench_function("direct_linear", |b| { + b.iter(|| dct8x8_to_dwt53_float_linear(black_box(block_2d))); + }); + two_dimensional.bench_function("idct_then_dwt_reference", |b| { + b.iter(|| idct8x8_then_dwt53_float(black_box(block_2d))); + }); + two_dimensional.finish(); + + let grid_blocks = synthetic_8x8_grid_blocks(2, 2); + let mut two_dimensional_grid = c.benchmark_group("dct53_2d_grid_scalar"); + two_dimensional_grid.bench_function("direct_linear_13x11", |b| { + b.iter(|| { + dct8x8_blocks_to_dwt53_float_linear( + black_box(&grid_blocks), + black_box(2), + black_box(2), + black_box(13), + black_box(11), + ) + .expect("valid DCT grid"); + }); + }); + let mut grid_scratch = Dct53GridScratch::default(); + two_dimensional_grid.bench_function("direct_linear_13x11_scratch_reuse", |b| { + b.iter(|| { + dct8x8_blocks_to_dwt53_float_linear_with_scratch( + black_box(&grid_blocks), + black_box(2), + black_box(2), + black_box(13), + black_box(11), + black_box(&mut grid_scratch), + ) + .expect("valid DCT grid"); + }); + }); + two_dimensional_grid.bench_function("idct_then_dwt_reference_13x11", |b| { + b.iter(|| { + dct8x8_blocks_then_dwt53_float( + black_box(&grid_blocks), + black_box(2), + black_box(2), + black_box(13), + black_box(11), + ) + .expect("valid DCT grid"); + }); + }); + two_dimensional_grid.finish(); + + bench_dct97_grid(c, &grid_blocks); + + let mut multilevel = c.benchmark_group("dct53_multilevel_scalar"); + multilevel.bench_function("direct_level1_then_ll_recursion", |b| { + b.iter(|| { + dct8x8_to_dwt53_multilevel_float_linear(black_box(block_2d), black_box(2)) + .expect("valid decomposition levels"); + }); + }); + multilevel.bench_function("idct_then_dwt_reference", |b| { + b.iter(|| { + idct8x8_then_dwt53_multilevel_float(black_box(block_2d), black_box(2)) + .expect("valid decomposition levels"); + }); + }); + multilevel.finish(); +} + +fn bench_dct97_grid(c: &mut Criterion, grid_blocks: &[[[f64; 8]; 8]]) { + let mut two_dimensional_grid_97 = c.benchmark_group("dct97_2d_grid_scalar"); + let mut grid_97_scratch = Dct97GridScratch::default(); + two_dimensional_grid_97.bench_function("idct_then_dwt_reference_13x11", |b| { + b.iter(|| { + dct8x8_blocks_then_dwt97_float( + black_box(grid_blocks), + black_box(2), + black_box(2), + black_box(13), + black_box(11), + ) + .expect("valid DCT grid"); + }); + }); + two_dimensional_grid_97.bench_function("idct_then_dwt_reference_13x11_scratch_reuse", |b| { + b.iter(|| { + dct8x8_blocks_then_dwt97_float_with_scratch( + black_box(grid_blocks), + black_box(2), + black_box(2), + black_box(13), + black_box(11), + black_box(&mut grid_97_scratch), + ) + .expect("valid DCT grid"); + }); + }); + two_dimensional_grid_97.finish(); +} + +fn bench_layout_candidates(c: &mut Criterion) { + let block_cols = 8; + let block_rows = 8; + let blocks = synthetic_natural_i16_blocks(block_cols * block_rows); + + let mut layout = c.benchmark_group("dct53_layout_candidates"); + layout.bench_function("aos_8x8_f64", |b| { + b.iter(|| pack_aos_8x8_f64(black_box(&blocks))); + }); + layout.bench_function("row_window_packed_f64", |b| { + b.iter(|| { + pack_row_window_packed_f64( + black_box(&blocks), + black_box(block_cols), + black_box(block_rows), + ); + }); + }); + layout.bench_function("soa_coefficient_major_f64", |b| { + b.iter(|| pack_soa_coefficient_major_f64(black_box(&blocks))); + }); + layout.finish(); +} + +fn bench_jpeg_paths(c: &mut Criterion) { + let jpeg_420 = JPEG_BASELINE_420_16X16; + let jpeg_restart = JPEG_BASELINE_420_RESTART_32X16; + + let mut jpeg_extract = c.benchmark_group("jpeg_dct_extract"); + jpeg_extract.bench_function("baseline_420_16x16", |b| { + b.iter(|| { + extract_dct_blocks(black_box(jpeg_420), DctExtractOptions::default()) + .expect("extract baseline 4:2:0 DCT blocks"); + }); + }); + jpeg_extract.bench_function("baseline_420_restart_32x16", |b| { + b.iter(|| { + extract_dct_blocks(black_box(jpeg_restart), DctExtractOptions::default()) + .expect("extract restart-coded 4:2:0 DCT blocks"); + }); + }); + jpeg_extract.finish(); + + let jpeg_gray = JPEG_GRAYSCALE_8X8; + let jpeg_444 = JPEG_BASELINE_444_8X8; + let jpeg_422 = JPEG_BASELINE_422_16X8; + let jpeg_420 = JPEG_BASELINE_420_16X16; + let transcode_options = JpegToHtj2kOptions::default(); + let transcode_97_options = JpegToHtj2kOptions::lossy_97(); + let mut jpeg_to_htj2k_group = c.benchmark_group("jpeg_to_htj2k"); + jpeg_to_htj2k_group.bench_function("grayscale_8x8", |b| { + b.iter(|| { + jpeg_to_htj2k(black_box(jpeg_gray), black_box(&transcode_options)) + .expect("transcode grayscale JPEG to HTJ2K"); + }); + }); + let mut stateful_transcoder = JpegToHtj2kTranscoder::default(); + jpeg_to_htj2k_group.bench_function("grayscale_8x8_stateful_reuse", |b| { + b.iter(|| { + stateful_transcoder + .transcode(black_box(jpeg_gray), black_box(&transcode_options)) + .expect("stateful transcode grayscale JPEG to HTJ2K"); + }); + }); + let jpeg_gray_multiblock = grayscale_jpeg(13, 11); + jpeg_to_htj2k_group.bench_function("grayscale_13x11", |b| { + b.iter(|| { + jpeg_to_htj2k( + black_box(&jpeg_gray_multiblock), + black_box(&transcode_options), + ) + .expect("transcode multi-block grayscale JPEG to HTJ2K"); + }); + }); + jpeg_to_htj2k_group.bench_function("ycbcr_444_8x8", |b| { + b.iter(|| { + jpeg_to_htj2k(black_box(jpeg_444), black_box(&transcode_options)) + .expect("transcode 4:4:4 YCbCr JPEG to HTJ2K"); + }); + }); + jpeg_to_htj2k_group.bench_function("ycbcr_422_16x8", |b| { + b.iter(|| { + jpeg_to_htj2k(black_box(jpeg_422), black_box(&transcode_options)) + .expect("transcode 4:2:2 YCbCr JPEG to HTJ2K"); + }); + }); + jpeg_to_htj2k_group.bench_function("ycbcr_420_16x16", |b| { + b.iter(|| { + jpeg_to_htj2k(black_box(jpeg_420), black_box(&transcode_options)) + .expect("transcode 4:2:0 YCbCr JPEG to HTJ2K"); + }); + }); + jpeg_to_htj2k_group.bench_function("grayscale_8x8_float_direct_97", |b| { + b.iter(|| { + jpeg_to_htj2k(black_box(jpeg_gray), black_box(&transcode_97_options)) + .expect("transcode grayscale JPEG to 9/7 HTJ2K"); + }); + }); + jpeg_to_htj2k_group.bench_function("ycbcr_420_16x16_float_direct_97", |b| { + b.iter(|| { + jpeg_to_htj2k(black_box(jpeg_420), black_box(&transcode_97_options)) + .expect("transcode 4:2:0 YCbCr JPEG to 9/7 HTJ2K"); + }); + }); + jpeg_to_htj2k_group.finish(); +} + +fn bench_dct53(c: &mut Criterion) { + bench_dct53_math(c); + bench_layout_candidates(c); + bench_jpeg_paths(c); +} + +fn pseudo_random_blocks(block_count: usize) -> Vec<[f64; 8]> { + let mut state = 0x384f_921d_u32; + (0..block_count) + .map(|_| { + let mut block = [0.0; 8]; + for coeff in &mut block { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + let bounded = u16::try_from(state % 257).expect("modulo result fits u16"); + *coeff = f64::from(i32::from(bounded) - 128); + } + block + }) + .collect() +} + +fn synthetic_8x8_block() -> [[f64; 8]; 8] { + let mut block = [[0.0; 8]; 8]; + block[0][0] = 512.0; + block[0][1] = -31.0; + block[1][0] = 27.0; + block[2][3] = 9.0; + block[7][7] = -6.0; + block +} + +fn synthetic_8x8_grid_blocks(block_cols: usize, block_rows: usize) -> Vec<[[f64; 8]; 8]> { + let mut blocks = Vec::with_capacity(block_cols * block_rows); + for block_y in 0..block_rows { + for block_x in 0..block_cols { + let mut block = synthetic_8x8_block(); + block[0][0] += (block_x * 17 + block_y * 23) as f64; + block[0][1] += block_x as f64; + block[1][0] -= block_y as f64; + blocks.push(block); + } + } + blocks +} + +fn synthetic_natural_i16_blocks(block_count: usize) -> Vec<[i16; 64]> { + let mut state = 0x91af_b33d_u32; + (0..block_count) + .map(|_| { + let mut block = [0; 64]; + for coefficient in &mut block { + state = state.wrapping_mul(22_695_477).wrapping_add(1); + let bounded = u16::try_from((state >> 8) % 2049).expect("modulo result fits u16"); + let signed = i32::from(bounded) - 1024; + *coefficient = i16::try_from(signed).expect("bounded coefficient fits i16"); + } + block + }) + .collect() +} + +fn pack_aos_8x8_f64(blocks: &[[i16; 64]]) -> Vec<[[f64; 8]; 8]> { + blocks + .iter() + .map(|block| { + let mut output = [[0.0; 8]; 8]; + for (idx, &coefficient) in block.iter().enumerate() { + output[idx / 8][idx % 8] = f64::from(coefficient); + } + output + }) + .collect() +} + +fn pack_row_window_packed_f64( + blocks: &[[i16; 64]], + block_cols: usize, + block_rows: usize, +) -> Vec { + assert_eq!(blocks.len(), block_cols * block_rows); + + let mut output = Vec::with_capacity(blocks.len() * 64); + for block_y in 0..block_rows { + let row_start = block_y * block_cols; + let row_blocks = &blocks[row_start..row_start + block_cols]; + for coefficient_y in 0..8 { + for block in row_blocks { + let coefficient_row = &block[coefficient_y * 8..coefficient_y * 8 + 8]; + output.extend( + coefficient_row + .iter() + .map(|&coefficient| f64::from(coefficient)), + ); + } + } + } + output +} + +fn pack_soa_coefficient_major_f64(blocks: &[[i16; 64]]) -> Vec { + let mut output = Vec::with_capacity(blocks.len() * 64); + for coefficient_idx in 0..64 { + output.extend(blocks.iter().map(|block| f64::from(block[coefficient_idx]))); + } + output +} + +fn grayscale_jpeg(width: u32, height: u32) -> Vec { + let samples = patterned_gray(width, height); + encode_jpeg_baseline( + JpegSamples::Gray8 { + data: &samples, + width, + height, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Gray, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode grayscale JPEG") + .data +} + +fn patterned_gray(width: u32, height: u32) -> Vec { + let mut out = Vec::with_capacity(width as usize * height as usize); + for y in 0..height { + for x in 0..width { + out.push(((x * 7 + y * 11 + 19) & 0xff) as u8); + } + } + out +} + +criterion_group!(dct53_benches, bench_dct53); +criterion_main!(dct53_benches); diff --git a/crates/j2k-transcode/examples/jpeg_to_htj2k.rs b/crates/j2k-transcode/examples/jpeg_to_htj2k.rs new file mode 100644 index 00000000..a425cb63 --- /dev/null +++ b/crates/j2k-transcode/examples/jpeg_to_htj2k.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Transcode a committed grayscale JPEG fixture into an HTJ2K codestream. +//! +//! Run with: +//! `cargo run -p j2k-transcode --example jpeg_to_htj2k` + +use j2k_test_support::JPEG_GRAYSCALE_8X8; +use j2k_transcode::{jpeg_to_htj2k, JpegToHtj2kOptions}; + +fn main() -> Result<(), Box> { + let encoded = jpeg_to_htj2k(JPEG_GRAYSCALE_8X8, &JpegToHtj2kOptions::lossless_53())?; + let report = &encoded.report; + + assert!(!encoded.codestream.is_empty()); + println!( + "transcoded {}x{} JPEG with {} component(s) into {} HTJ2K bytes", + report.width, + report.height, + report.component_count, + encoded.codestream.len() + ); + Ok(()) +} diff --git a/crates/j2k-transcode/fixtures/conformance/baseline_420_16x16.jpg b/crates/j2k-transcode/fixtures/conformance/baseline_420_16x16.jpg new file mode 100644 index 00000000..af176169 Binary files /dev/null and b/crates/j2k-transcode/fixtures/conformance/baseline_420_16x16.jpg differ diff --git a/crates/j2k-transcode/fixtures/conformance/baseline_420_16x16.rgb b/crates/j2k-transcode/fixtures/conformance/baseline_420_16x16.rgb new file mode 100644 index 00000000..16d0fa17 Binary files /dev/null and b/crates/j2k-transcode/fixtures/conformance/baseline_420_16x16.rgb differ diff --git a/crates/j2k-transcode/fixtures/conformance/baseline_420_restart_32x16.jpg b/crates/j2k-transcode/fixtures/conformance/baseline_420_restart_32x16.jpg new file mode 100644 index 00000000..e794eaaf Binary files /dev/null and b/crates/j2k-transcode/fixtures/conformance/baseline_420_restart_32x16.jpg differ diff --git a/crates/j2k-transcode/fixtures/conformance/baseline_420_restart_32x16.rgb b/crates/j2k-transcode/fixtures/conformance/baseline_420_restart_32x16.rgb new file mode 100644 index 00000000..6f46bf03 Binary files /dev/null and b/crates/j2k-transcode/fixtures/conformance/baseline_420_restart_32x16.rgb differ diff --git a/crates/j2k-transcode/fixtures/conformance/baseline_422_16x8.jpg b/crates/j2k-transcode/fixtures/conformance/baseline_422_16x8.jpg new file mode 100644 index 00000000..60b479c5 Binary files /dev/null and b/crates/j2k-transcode/fixtures/conformance/baseline_422_16x8.jpg differ diff --git a/crates/j2k-transcode/fixtures/conformance/baseline_422_16x8.rgb b/crates/j2k-transcode/fixtures/conformance/baseline_422_16x8.rgb new file mode 100644 index 00000000..bdffccfd Binary files /dev/null and b/crates/j2k-transcode/fixtures/conformance/baseline_422_16x8.rgb differ diff --git a/crates/j2k-transcode/fixtures/conformance/baseline_444_8x8.jpg b/crates/j2k-transcode/fixtures/conformance/baseline_444_8x8.jpg new file mode 100644 index 00000000..0f4b494a Binary files /dev/null and b/crates/j2k-transcode/fixtures/conformance/baseline_444_8x8.jpg differ diff --git a/crates/j2k-transcode/fixtures/conformance/baseline_444_8x8.rgb b/crates/j2k-transcode/fixtures/conformance/baseline_444_8x8.rgb new file mode 100644 index 00000000..52ee9b37 Binary files /dev/null and b/crates/j2k-transcode/fixtures/conformance/baseline_444_8x8.rgb differ diff --git a/crates/j2k-transcode/fixtures/conformance/grayscale_8x8.gray b/crates/j2k-transcode/fixtures/conformance/grayscale_8x8.gray new file mode 100644 index 00000000..2ceeb10f Binary files /dev/null and b/crates/j2k-transcode/fixtures/conformance/grayscale_8x8.gray differ diff --git a/crates/j2k-transcode/fixtures/conformance/grayscale_8x8.jpg b/crates/j2k-transcode/fixtures/conformance/grayscale_8x8.jpg new file mode 100644 index 00000000..80cecf54 Binary files /dev/null and b/crates/j2k-transcode/fixtures/conformance/grayscale_8x8.jpg differ diff --git a/crates/j2k-transcode/fuzz/.gitignore b/crates/j2k-transcode/fuzz/.gitignore new file mode 100644 index 00000000..1a45eee7 --- /dev/null +++ b/crates/j2k-transcode/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/crates/j2k-transcode/fuzz/Cargo.lock b/crates/j2k-transcode/fuzz/Cargo.lock new file mode 100644 index 00000000..31cb2887 --- /dev/null +++ b/crates/j2k-transcode/fuzz/Cargo.lock @@ -0,0 +1,287 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "cc" +version = "1.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "fearless_simd" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "j2k" +version = "0.6.0" +dependencies = [ + "j2k-core", + "j2k-native", + "j2k-types", + "thiserror", +] + +[[package]] +name = "j2k-core" +version = "0.6.0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "j2k-jpeg" +version = "0.6.0" +dependencies = [ + "j2k-core", + "j2k-profile", + "memchr", + "rayon", + "thiserror", +] + +[[package]] +name = "j2k-native" +version = "0.6.0" +dependencies = [ + "fearless_simd", + "j2k-profile", + "j2k-types", + "libm", + "rayon", +] + +[[package]] +name = "j2k-profile" +version = "0.6.0" + +[[package]] +name = "j2k-transcode" +version = "0.6.0" +dependencies = [ + "j2k", + "j2k-jpeg", + "j2k-native", + "rayon", +] + +[[package]] +name = "j2k-transcode-fuzz" +version = "0.1.0" +dependencies = [ + "j2k-transcode", + "libfuzzer-sys", +] + +[[package]] +name = "j2k-types" +version = "0.6.0" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" diff --git a/crates/j2k-transcode/fuzz/Cargo.toml b/crates/j2k-transcode/fuzz/Cargo.toml new file mode 100644 index 00000000..29396cda --- /dev/null +++ b/crates/j2k-transcode/fuzz/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "j2k-transcode-fuzz" +version = "0.1.0" +edition = "2021" +publish = false + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" +j2k-transcode = { path = ".." } + +[[bin]] +name = "jpeg_to_htj2k_fuzz" +path = "fuzz_targets/jpeg_to_htj2k_fuzz.rs" +test = false +doc = false +bench = false + +[profile.release] +debug = 1 + +[workspace] diff --git a/crates/j2k-transcode/fuzz/fuzz_targets/jpeg_to_htj2k_fuzz.rs b/crates/j2k-transcode/fuzz/fuzz_targets/jpeg_to_htj2k_fuzz.rs new file mode 100644 index 00000000..aa6f65ea --- /dev/null +++ b/crates/j2k-transcode/fuzz/fuzz_targets/jpeg_to_htj2k_fuzz.rs @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0 +#![no_main] + +use j2k_transcode::{jpeg_to_htj2k, JpegToHtj2kOptions}; +use libfuzzer_sys::fuzz_target; + +const MAX_INPUT_BYTES: usize = 256 * 1024; +const MAX_OUTPUT_BYTES: usize = 4 * 1024 * 1024; + +fuzz_target!(|data: &[u8]| { + if data.len() > MAX_INPUT_BYTES { + return; + } + + let Ok(encoded) = jpeg_to_htj2k(data, &JpegToHtj2kOptions::lossless_53()) else { + return; + }; + + if encoded.codestream.len() > MAX_OUTPUT_BYTES { + return; + } +}); diff --git a/crates/j2k-transcode/src/accelerator.rs b/crates/j2k-transcode/src/accelerator.rs new file mode 100644 index 00000000..be0e8fe4 --- /dev/null +++ b/crates/j2k-transcode/src/accelerator.rs @@ -0,0 +1,924 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Optional acceleration hooks for coefficient-domain transform stages. +//! +//! These hooks are intentionally narrow: accelerated backends may replace the +//! direct DCT-grid to one-level wavelet projection, while the scalar path +//! remains the default oracle and fallback. + +use core::fmt; + +use crate::dct53_2d::Dwt53TwoDimensional; +use crate::dct97_2d::Dwt97TwoDimensional; +use crate::dct_grid::validate_dct_block_grid; +use crate::reversible53::{ + reversible_lift_53_high_at, reversible_lift_53_i32, reversible_lift_53_low_at, +}; +pub use j2k::adapter::encode_stage::{ + EncodedHtJ2kCodeBlock, IrreversibleQuantizationSubbandScales, J2kSubBandType, + PreencodedHtj2k97CodeBlock, PreencodedHtj2k97CompactCodeBlock, + PreencodedHtj2k97CompactComponent, PreencodedHtj2k97CompactImage, + PreencodedHtj2k97CompactResolution, PreencodedHtj2k97CompactSubband, + PreencodedHtj2k97Component, PreencodedHtj2k97Resolution, PreencodedHtj2k97Subband, + PrequantizedHtj2k97CodeBlock, PrequantizedHtj2k97Component, PrequantizedHtj2k97Image, + PrequantizedHtj2k97Resolution, PrequantizedHtj2k97Subband, +}; +use j2k_jpeg::transcode::idct_islow_block; +use rayon::prelude::*; + +const REVERSIBLE_DWT53_UNSUPPORTED_GRID: &str = + "reversible DCT 5/3 job has unsupported grid geometry"; + +/// Direct DCT-grid to one-level reversible integer 5/3 projection job. +#[derive(Debug, Clone, Copy)] +pub struct DctGridToReversibleDwt53Job<'a> { + /// Natural-order, dequantized 8x8 DCT blocks. + pub dequantized_blocks: &'a [[i16; 64]], + /// Number of DCT block columns in `dequantized_blocks`. + pub block_cols: usize, + /// Number of DCT block rows in `dequantized_blocks`. + pub block_rows: usize, + /// Logical component width in samples. + pub width: usize, + /// Logical component height in samples. + pub height: usize, +} + +/// One separable single-level reversible integer 5/3 transform result. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReversibleDwt53FirstLevel { + /// Low-horizontal, low-vertical band. + pub ll: Vec, + /// High-horizontal, low-vertical band. + pub hl: Vec, + /// Low-horizontal, high-vertical band. + pub lh: Vec, + /// High-horizontal, high-vertical band. + pub hh: Vec, + /// Width of horizontally low-pass bands. + pub low_width: usize, + /// Height of vertically low-pass bands. + pub low_height: usize, + /// Width of horizontally high-pass bands. + pub high_width: usize, + /// Height of vertically high-pass bands. + pub high_height: usize, +} + +/// Direct DCT-grid to one-level 5/3 projection job. +#[derive(Debug, Clone, Copy)] +pub struct DctGridToDwt53Job<'a> { + /// Natural-order, dequantized 8x8 DCT blocks. + pub blocks: &'a [[[f64; 8]; 8]], + /// Number of DCT block columns in `blocks`. + pub block_cols: usize, + /// Number of DCT block rows in `blocks`. + pub block_rows: usize, + /// Logical component width in samples. + pub width: usize, + /// Logical component height in samples. + pub height: usize, +} + +/// Direct DCT-grid to one-level 9/7 transform job. +#[derive(Debug, Clone, Copy)] +pub struct DctGridToDwt97Job<'a> { + /// Natural-order, dequantized 8x8 DCT blocks. + pub blocks: &'a [[[f64; 8]; 8]], + /// Number of DCT block columns in `blocks`. + pub block_cols: usize, + /// Number of DCT block rows in `blocks`. + pub block_rows: usize, + /// Logical component width in samples. + pub width: usize, + /// Logical component height in samples. + pub height: usize, +} + +/// Direct DCT-grid to prequantized one-level 9/7 HTJ2K code-block job. +#[derive(Debug, Clone, Copy)] +pub struct DctGridToHtj2k97CodeBlockJob<'a> { + /// Natural-order, dequantized 8x8 DCT blocks. + pub blocks: &'a [[[f64; 8]; 8]], + /// Number of DCT block columns in `blocks`. + pub block_cols: usize, + /// Number of DCT block rows in `blocks`. + pub block_rows: usize, + /// Logical component width in samples. + pub width: usize, + /// Logical component height in samples. + pub height: usize, + /// Horizontal SIZ sampling factor (`XRsiz`). + pub x_rsiz: u8, + /// Vertical SIZ sampling factor (`YRsiz`). + pub y_rsiz: u8, +} + +/// Direct dequantized i16 DCT-grid to one-level 9/7 HTJ2K code-block job. +/// +/// This is for accelerators that consume the JPEG coefficient extraction +/// output directly and do not need the generic f64 block representation. +#[derive(Debug, Clone, Copy)] +pub struct DctGridI16ToHtj2k97CodeBlockJob<'a> { + /// Natural-order, dequantized 8x8 DCT blocks. + pub dequantized_blocks: &'a [[i16; 64]], + /// Number of DCT block columns in `dequantized_blocks`. + pub block_cols: usize, + /// Number of DCT block rows in `dequantized_blocks`. + pub block_rows: usize, + /// Logical component width in samples. + pub width: usize, + /// Logical component height in samples. + pub height: usize, + /// Horizontal SIZ sampling factor (`XRsiz`). + pub x_rsiz: u8, + /// Vertical SIZ sampling factor (`YRsiz`). + pub y_rsiz: u8, +} + +/// One same-geometry i16 DCT-grid HTJ2K preencode batch. +#[derive(Debug, Clone, Copy)] +pub struct DctGridI16ToHtj2k97CodeBlockBatch<'a, 'j> { + /// Jobs in this same-geometry batch. + pub jobs: &'j [DctGridI16ToHtj2k97CodeBlockJob<'a>], +} + +/// Compact preencoded HTJ2K components backed by one payload buffer. +#[derive(Debug, Clone)] +pub struct PreencodedHtj2k97CompactBatch { + /// Contiguous encoded code-block payload bytes for every component. + pub payload: Vec, + /// Compact components in the same order as the submitted jobs. + pub components: Vec, +} + +/// Compact preencoded HTJ2K grouped-batch output backed by one payload buffer. +#[derive(Debug, Clone)] +pub struct PreencodedHtj2k97CompactBatchGroups { + /// Contiguous encoded code-block payload bytes for every returned group. + pub payload: Vec, + /// Compact components grouped in the same order as submitted batches. + pub groups: Vec>, +} + +/// Encode parameters needed to quantize 9/7 output directly into HTJ2K +/// code-block coefficient layout. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Htj2k97CodeBlockOptions { + /// Component precision in bits. + pub bit_depth: u8, + /// JPEG 2000 guard bits used for QCD and code-block bitplane counts. + pub guard_bits: u8, + /// Code-block width exponent minus two. + pub code_block_width_exp: u8, + /// Code-block height exponent minus two. + pub code_block_height_exp: u8, + /// Multiplier applied to irreversible 9/7 scalar quantization step sizes. + pub irreversible_quantization_scale: f32, + /// Per-subband multipliers applied on top of + /// [`irreversible_quantization_scale`](Self::irreversible_quantization_scale). + pub irreversible_quantization_subband_scales: IrreversibleQuantizationSubbandScales, +} + +/// Backend-specific timing breakdown for a same-geometry 9/7 batch. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Dwt97BatchStageTimings { + /// Host packing, buffer allocation, and upload time in microseconds. + pub pack_upload_us: u128, + /// Time spent in the IDCT plus horizontal 9/7 row-lift stage. + pub idct_row_lift_us: u128, + /// Time spent in the vertical 9/7 column-lift stage. + pub column_lift_us: u128, + /// Time spent quantizing 9/7 bands into HTJ2K code-block layout. + pub quantize_codeblock_us: u128, + /// Time spent HT-encoding resident code-block coefficients. + pub ht_encode_us: u128, + /// Resident HT cleanup-pass encode kernel time in microseconds. + pub ht_kernel_us: u128, + /// Resident HT status-buffer device-to-host readback time in microseconds. + pub ht_status_readback_us: u128, + /// Resident HT encoded-byte compaction kernel time in microseconds. + pub ht_compact_us: u128, + /// Resident HT compacted encoded-byte device-to-host readback time in microseconds. + pub ht_output_readback_us: u128, + /// Number of HT code-block encode kernel dispatches in this batch. + pub ht_codeblock_dispatches: usize, + /// Time spent reading and unpacking Metal band buffers into host outputs. + pub readback_us: u128, +} + +/// Error returned by accelerated transcode stage backends. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TranscodeStageError { + /// The job shape, options, or environment are outside what this backend + /// supports. + Unsupported(&'static str), + /// The backend failed while executing the stage. + Backend(String), + /// The device or runtime backing this accelerator is unavailable. + DeviceUnavailable, +} + +impl fmt::Display for TranscodeStageError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Unsupported(reason) => f.write_str(reason), + Self::Backend(reason) => f.write_str(reason), + Self::DeviceUnavailable => f.write_str("accelerator device is unavailable"), + } + } +} + +impl std::error::Error for TranscodeStageError {} + +impl From<&'static str> for TranscodeStageError { + fn from(reason: &'static str) -> Self { + Self::Unsupported(reason) + } +} + +/// Optional backend for SIMD, GPU, or other accelerated transform stages. +pub trait DctToWaveletStageAccelerator { + /// Whether this accelerator wants same-geometry 9/7 batch jobs offered. + /// + /// The default is false so CPU-only fallback paths do not pay the memory + /// cost of materializing batch-owned float DCT blocks before immediately + /// falling back. + fn supports_dwt97_batch(&self) -> bool { + false + } + + /// Whether this accelerator wants same-geometry 9/7 batches offered as + /// prequantized HTJ2K code-block jobs before the float-band hook. + fn supports_htj2k97_codeblock_batch(&self) -> bool { + false + } + + /// Whether this accelerator wants same-geometry 9/7 preencoded HTJ2K + /// batches offered with dequantized i16 DCT blocks before materializing the + /// generic f64 block representation. + fn supports_htj2k97_i16_preencoded_batch(&self) -> bool { + false + } + + /// Whether this accelerator wants the compact i16 preencoded HTJ2K batch + /// hook offered before the owned preencoded hook. + fn supports_htj2k97_compact_preencoded_batch(&self) -> bool { + self.supports_htj2k97_i16_preencoded_batch() + } + + /// Optionally compute the direct DCT-grid to one-level reversible integer + /// 5/3 projection. + /// + /// Return `Ok(Some(output))` when the backend handled the job bit-exactly + /// relative to j2k's scalar integer oracle. Return `Ok(None)` to use + /// the scalar fallback. + fn dct_grid_to_reversible_dwt53( + &mut self, + _job: DctGridToReversibleDwt53Job<'_>, + ) -> Result, TranscodeStageError> { + Ok(None) + } + + /// Optionally compute a same-geometry batch of direct DCT-grid to + /// one-level reversible integer 5/3 projections. + /// + /// Backends should return outputs in the same order as `jobs`. Return + /// `Ok(None)` to use the scalar per-component fallback. + fn dct_grid_to_reversible_dwt53_batch( + &mut self, + _jobs: &[DctGridToReversibleDwt53Job<'_>], + ) -> Result>, TranscodeStageError> { + Ok(None) + } + + /// Optionally compute the direct DCT-grid to one-level 5/3 projection. + /// + /// Return `Ok(Some(output))` when the backend handled the job. Return + /// `Ok(None)` to use the scalar fallback. + fn dct_grid_to_dwt53( + &mut self, + _job: DctGridToDwt53Job<'_>, + ) -> Result>, TranscodeStageError> { + Ok(None) + } + + /// Optionally compute the direct DCT-grid to one-level 9/7 transform. + /// + /// Return `Ok(Some(output))` when the backend handled the job. Return + /// `Ok(None)` to use the scalar fallback. + fn dct_grid_to_dwt97( + &mut self, + _job: DctGridToDwt97Job<'_>, + ) -> Result>, TranscodeStageError> { + Ok(None) + } + + /// Optionally compute a same-geometry batch of direct DCT-grid to + /// one-level 9/7 transforms. + /// + /// Backends should return outputs in the same order as `jobs`. Return + /// `Ok(None)` to use the scalar per-component fallback. + fn dct_grid_to_dwt97_batch( + &mut self, + _jobs: &[DctGridToDwt97Job<'_>], + ) -> Result>>, TranscodeStageError> { + Ok(None) + } + + /// Optionally compute same-geometry DCT-grid 9/7 jobs directly into + /// prequantized HTJ2K code-block components. + /// + /// Backends should return one component per input job in the same order as + /// `jobs`. Return `Ok(None)` to use the float-band path. + fn dct_grid_to_htj2k97_codeblock_batch( + &mut self, + _jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + _options: Htj2k97CodeBlockOptions, + ) -> Result>, TranscodeStageError> { + Ok(None) + } + + /// Optionally compute same-geometry DCT-grid 9/7 jobs directly into + /// preencoded HTJ2K code-block payloads. + /// + /// Backends should return one component per input job in the same order as + /// `jobs`. Return `Ok(None)` to use the prequantized or float-band path. + fn dct_grid_to_htj2k97_preencoded_batch( + &mut self, + _jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + _options: Htj2k97CodeBlockOptions, + ) -> Result>, TranscodeStageError> { + Ok(None) + } + + /// Optionally compute same-geometry dequantized i16 DCT-grid 9/7 jobs + /// directly into preencoded HTJ2K code-block payloads. + /// + /// Backends should return one component per input job in the same order as + /// `jobs`. Return `Ok(None)` to use the generic f64 preencoded path. + fn dct_grid_i16_to_htj2k97_preencoded_batch( + &mut self, + _jobs: &[DctGridI16ToHtj2k97CodeBlockJob<'_>], + _options: Htj2k97CodeBlockOptions, + ) -> Result>, TranscodeStageError> { + Ok(None) + } + + /// Optionally compute same-geometry dequantized i16 DCT-grid 9/7 jobs into + /// compact preencoded HTJ2K code-block payloads. + /// + /// Backends should return one component per input job in the same order as + /// `jobs`, with all component ranges pointing into the returned payload. + /// Return `Ok(None)` to use the owned preencoded path. + fn dct_grid_i16_to_htj2k97_compact_preencoded_batch( + &mut self, + _jobs: &[DctGridI16ToHtj2k97CodeBlockJob<'_>], + _options: Htj2k97CodeBlockOptions, + ) -> Result, TranscodeStageError> { + Ok(None) + } + + /// Optionally compute multiple same-geometry dequantized i16 DCT-grid + /// batches directly into preencoded HTJ2K code-block payloads. + /// + /// Each input batch is internally same-geometry, but different batches may + /// have different component dimensions. Backends should return one output + /// vector per input batch, in order. Return `Ok(None)` to use the per-group + /// fallback hooks. + fn dct_grid_i16_to_htj2k97_preencoded_batch_groups( + &mut self, + _groups: &[DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], + _options: Htj2k97CodeBlockOptions, + ) -> Result>>, TranscodeStageError> { + Ok(None) + } + + /// Optionally compute multiple same-geometry dequantized i16 DCT-grid 9/7 + /// batches into compact preencoded HTJ2K code-block payloads. + /// + /// Each returned item corresponds to one input batch and contains one + /// component per job in that batch. Return `Ok(None)` to use the owned + /// preencoded grouped hook. + fn dct_grid_i16_to_htj2k97_compact_preencoded_batch_groups( + &mut self, + _groups: &[DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], + _options: Htj2k97CodeBlockOptions, + ) -> Result, TranscodeStageError> { + Ok(None) + } + + /// Return backend stage timings for the most recent 9/7 batch dispatch. + fn last_dwt97_batch_stage_timings(&self) -> Option { + None + } +} + +/// Accelerator that always uses the scalar CPU fallback. +#[derive(Debug, Default, Clone, Copy)] +pub struct CpuOnlyDctToWaveletStageAccelerator; + +impl DctToWaveletStageAccelerator for CpuOnlyDctToWaveletStageAccelerator {} + +/// CPU/Rayon accelerator for the exact reversible integer 5/3 first level. +/// +/// This backend keeps j2k's scalar ISLOW IDCT semantics as the oracle: +/// each 8x8 block is decoded with `j2k-jpeg`, level-shifted to signed +/// component samples, then transformed with reversible integer 5/3 lifting. +#[derive(Debug, Default, Clone)] +pub struct RayonReversibleDwt53Accelerator { + attempts: usize, + dispatches: usize, + batch_attempts: usize, + batch_dispatches: usize, +} + +impl RayonReversibleDwt53Accelerator { + /// Number of reversible 5/3 jobs offered to this accelerator. + #[must_use] + pub const fn reversible_dwt53_attempts(&self) -> usize { + self.attempts + } + + /// Number of reversible 5/3 jobs handled by this accelerator. + #[must_use] + pub const fn reversible_dwt53_dispatches(&self) -> usize { + self.dispatches + } + + /// Number of reversible 5/3 batches offered to this accelerator. + #[must_use] + pub const fn reversible_dwt53_batch_attempts(&self) -> usize { + self.batch_attempts + } + + /// Number of reversible 5/3 batches handled by this accelerator. + #[must_use] + pub const fn reversible_dwt53_batch_dispatches(&self) -> usize { + self.batch_dispatches + } +} + +impl DctToWaveletStageAccelerator for RayonReversibleDwt53Accelerator { + fn dct_grid_to_reversible_dwt53( + &mut self, + job: DctGridToReversibleDwt53Job<'_>, + ) -> Result, TranscodeStageError> { + self.attempts = self.attempts.saturating_add(1); + let output = reversible_dwt53_first_level_rayon(job)?; + self.dispatches = self.dispatches.saturating_add(1); + Ok(Some(output)) + } + + fn dct_grid_to_reversible_dwt53_batch( + &mut self, + jobs: &[DctGridToReversibleDwt53Job<'_>], + ) -> Result>, TranscodeStageError> { + self.batch_attempts = self.batch_attempts.saturating_add(1); + let mut output = Vec::with_capacity(jobs.len()); + for job in jobs { + output.push(reversible_dwt53_first_level_rayon(*job)?); + } + self.batch_dispatches = self.batch_dispatches.saturating_add(1); + Ok(Some(output)) + } +} + +/// Decode the job's dequantized DCT blocks into j2k's signed integer +/// component sample blocks. +/// +/// This is public so hybrid GPU backends can keep JPEG parsing and exact IDCT +/// on CPU while offloading the reversible 5/3 projection. +pub fn idct_blocks_to_signed_samples_rayon(blocks: &[[i16; 64]]) -> Vec<[i32; 64]> { + blocks + .par_iter() + .map(|block| { + let decoded = idct_islow_block(block); + decoded.map(|sample| i32::from(sample) - 128) + }) + .collect() +} + +/// Compute one exact reversible integer 5/3 level from already decoded +/// block-local signed samples. +pub fn reversible_dwt53_first_level_from_block_samples( + block_samples: &[[i32; 64]], + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, +) -> Result { + validate_reversible_grid(block_samples.len(), block_cols, block_rows, width, height)?; + + let low_width = width.div_ceil(2); + let low_height = height.div_ceil(2); + let high_width = width / 2; + let high_height = height / 2; + + let low_rows: Vec<(Vec, Vec)> = (0..low_height) + .into_par_iter() + .map(|output_y| { + let mut row = Vec::with_capacity(width); + for x in 0..width { + row.push(vertical_low_53_i32_at( + block_samples, + block_cols, + width, + height, + x, + output_y, + )); + } + reversible_lift_53_i32(&mut row); + ( + row.iter().step_by(2).copied().collect(), + row.iter().skip(1).step_by(2).copied().collect(), + ) + }) + .collect(); + let high_rows: Vec<(Vec, Vec)> = (0..high_height) + .into_par_iter() + .map(|output_y| { + let mut row = Vec::with_capacity(width); + for x in 0..width { + row.push(vertical_high_53_i32_at( + block_samples, + block_cols, + width, + height, + x, + output_y, + )); + } + reversible_lift_53_i32(&mut row); + ( + row.iter().step_by(2).copied().collect(), + row.iter().skip(1).step_by(2).copied().collect(), + ) + }) + .collect(); + + let mut ll = Vec::with_capacity(low_width * low_height); + let mut hl = Vec::with_capacity(high_width * low_height); + for (low, high) in low_rows { + ll.extend(low); + hl.extend(high); + } + + let mut lh = Vec::with_capacity(low_width * high_height); + let mut hh = Vec::with_capacity(high_width * high_height); + for (low, high) in high_rows { + lh.extend(low); + hh.extend(high); + } + + Ok(ReversibleDwt53FirstLevel { + ll, + hl, + lh, + hh, + low_width, + low_height, + high_width, + high_height, + }) +} + +fn reversible_dwt53_first_level_rayon( + job: DctGridToReversibleDwt53Job<'_>, +) -> Result { + validate_reversible_grid( + job.dequantized_blocks.len(), + job.block_cols, + job.block_rows, + job.width, + job.height, + )?; + let block_samples = idct_blocks_to_signed_samples_rayon(job.dequantized_blocks); + reversible_dwt53_first_level_from_block_samples( + &block_samples, + job.block_cols, + job.block_rows, + job.width, + job.height, + ) +} + +fn validate_reversible_grid( + block_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, +) -> Result<(), &'static str> { + validate_dct_block_grid(block_count, block_cols, block_rows, width, height) + .map_err(|_| REVERSIBLE_DWT53_UNSUPPORTED_GRID) +} + +fn vertical_low_53_i32_at( + block_samples: &[[i32; 64]], + block_cols: usize, + width: usize, + height: usize, + x: usize, + low_idx: usize, +) -> i32 { + reversible_lift_53_low_at(height, low_idx, |y| { + component_sample_i32(block_samples, block_cols, width, height, x, y) + }) +} + +fn vertical_high_53_i32_at( + block_samples: &[[i32; 64]], + block_cols: usize, + width: usize, + height: usize, + x: usize, + high_idx: usize, +) -> i32 { + reversible_lift_53_high_at(height, high_idx, |y| { + component_sample_i32(block_samples, block_cols, width, height, x, y) + }) +} + +fn component_sample_i32( + block_samples: &[[i32; 64]], + block_cols: usize, + width: usize, + height: usize, + x: usize, + y: usize, +) -> i32 { + debug_assert!(x < width); + debug_assert!(y < height); + let block_x = x / 8; + let block_y = y / 8; + let block_idx = block_y * block_cols + block_x; + let local_idx = (y % 8) * 8 + (x % 8); + block_samples[block_idx][local_idx] +} + +#[cfg(test)] +mod ground_truth_tests { + //! Independent ground truth for the reversible integer 5/3. + //! + //! The CUDA 5/3 kernel is parity-tested against the lifting in this module, + //! so a boundary/indexing/band-split bug here would be faithfully copied by + //! the kernel and pass parity. Validate the lifting against the canonical + //! JPEG2000 reversible 5/3 (ISO/IEC 15444-1 Annex F.3.8.1) evaluated per + //! output index from a whole-sample-symmetrically extended signal — a + //! structurally different implementation than the in-place two-pass loops. + + use super::{ + reversible_dwt53_first_level_from_block_samples, reversible_lift_53_i32, + ReversibleDwt53FirstLevel, + }; + + fn floor2(a: i32, b: i32) -> i32 { + a.div_euclid(b) + } + + /// Whole-sample symmetric reflection (mirror about 0 and `n - 1`, endpoints + /// not repeated) — the boundary extension the lifting realizes at the edges. + fn ws_reflect(i: isize, n: usize) -> usize { + if n == 1 { + return 0; + } + let n = isize::try_from(n).unwrap(); + let period = 2 * (n - 1); + let mut k = i.rem_euclid(period); + if k >= n { + k = period - k; + } + usize::try_from(k).unwrap() + } + + /// Canonical forward 5/3: `(low, high)` where `low[m]` is the even/approx + /// coefficient and `high[m]` the odd/detail coefficient. Every index is read + /// through whole-sample symmetric extension of the original signal, so the + /// detail-boundary behavior follows automatically (no special cases). + fn ref_53_forward(signal: &[i32]) -> (Vec, Vec) { + let n = signal.len(); + if n < 2 { + return (signal.to_vec(), Vec::new()); + } + let sig = |i: isize| signal[ws_reflect(i, n)]; + let detail = |m: isize| { + let c = 2 * m + 1; + sig(c) - floor2(sig(c - 1) + sig(c + 1), 2) + }; + let low: Vec = (0..n.div_ceil(2)) + .map(|m| { + let mi = isize::try_from(m).unwrap(); + sig(2 * mi) + floor2(detail(mi - 1) + detail(mi) + 2, 4) + }) + .collect(); + let high: Vec = (0..n / 2) + .map(|m| detail(isize::try_from(m).unwrap())) + .collect(); + (low, high) + } + + /// Separable 2D reference matching the oracle's vertical-then-horizontal + /// order (integer floor lifting is NOT order-independent, so order matters). + fn ref_53_2d(plane: &[i32], width: usize, height: usize) -> ReversibleDwt53FirstLevel { + let low_width = width.div_ceil(2); + let high_width = width / 2; + let low_height = height.div_ceil(2); + let high_height = height / 2; + + let mut v_low = vec![0i32; width * low_height]; + let mut v_high = vec![0i32; width * high_height]; + for x in 0..width { + let column: Vec = (0..height).map(|y| plane[y * width + x]).collect(); + let (lo, hi) = ref_53_forward(&column); + for (oy, &value) in lo.iter().enumerate() { + v_low[oy * width + x] = value; + } + for (oy, &value) in hi.iter().enumerate() { + v_high[oy * width + x] = value; + } + } + + let horizontal = |source: &[i32], rows: usize| -> (Vec, Vec) { + let mut low = vec![0i32; low_width * rows]; + let mut high = vec![0i32; high_width * rows]; + for oy in 0..rows { + let (lo, hi) = ref_53_forward(&source[oy * width..oy * width + width]); + low[oy * low_width..oy * low_width + low_width].copy_from_slice(&lo); + high[oy * high_width..oy * high_width + high_width].copy_from_slice(&hi); + } + (low, high) + }; + + let (ll, hl) = horizontal(&v_low, low_height); + let (lh, hh) = horizontal(&v_high, high_height); + + ReversibleDwt53FirstLevel { + ll, + hl, + lh, + hh, + low_width, + low_height, + high_width, + high_height, + } + } + + /// Pack a flat `width x height` sample plane into the block-major + /// `[[i32; 64]]` layout `reversible_dwt53_first_level_from_block_samples` + /// consumes (local index `(y % 8) * 8 + (x % 8)`). + fn pack_plane(plane: &[i32], width: usize, height: usize) -> (Vec<[i32; 64]>, usize, usize) { + let block_cols = width.div_ceil(8); + let block_rows = height.div_ceil(8); + let mut blocks = vec![[0i32; 64]; block_cols * block_rows]; + for y in 0..height { + for x in 0..width { + let block = (y / 8) * block_cols + (x / 8); + blocks[block][(y % 8) * 8 + (x % 8)] = plane[y * width + x]; + } + } + (blocks, block_cols, block_rows) + } + + fn next_sample(state: &mut u64) -> i32 { + *state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + ((*state >> 40) & 0x1ff) as i32 - 256 + } + + #[test] + fn reversible_lift_53_matches_canonical_formula_1d() { + let mut state = 0x0a11_ce5e_ed00_d001u64; + for n in [2usize, 3, 4, 5, 8, 9, 12, 15, 16, 23, 32, 33, 64, 65] { + let signal: Vec = (0..n).map(|_| next_sample(&mut state)).collect(); + let mut lifted = signal.clone(); + reversible_lift_53_i32(&mut lifted); + let lifted_low: Vec = lifted.iter().step_by(2).copied().collect(); + let lifted_high: Vec = lifted.iter().skip(1).step_by(2).copied().collect(); + let (low, high) = ref_53_forward(&signal); + assert_eq!(lifted_low, low, "low band mismatch for n={n}"); + assert_eq!(lifted_high, high, "high band mismatch for n={n}"); + } + } + + #[test] + fn reversible_lift_53_shared_helper_matches_canonical_formula_1d() { + let mut state = 0x5a53_5a53_5a53_5a53u64; + for n in [2usize, 3, 4, 5, 8, 9, 16, 17, 31, 32, 65] { + let signal: Vec = (0..n).map(|_| next_sample(&mut state)).collect(); + let mut lifted = signal.clone(); + crate::reversible53::reversible_lift_53_i32(&mut lifted); + let lifted_low: Vec = lifted.iter().step_by(2).copied().collect(); + let lifted_high: Vec = lifted.iter().skip(1).step_by(2).copied().collect(); + let (low, high) = ref_53_forward(&signal); + assert_eq!(lifted_low, low, "low band mismatch for n={n}"); + assert_eq!(lifted_high, high, "high band mismatch for n={n}"); + } + } + + #[test] + fn reversible_dwt53_2d_matches_canonical_separable() { + let mut state = 0xfeed_5eed_d00d_face_u64; + for (width, height) in [ + (8usize, 8usize), + (16, 16), + (24, 16), + (15, 13), + (16, 23), + (9, 7), + (32, 32), + ] { + let plane: Vec = (0..width * height) + .map(|_| next_sample(&mut state)) + .collect(); + let (blocks, block_cols, block_rows) = pack_plane(&plane, width, height); + let got = reversible_dwt53_first_level_from_block_samples( + &blocks, block_cols, block_rows, width, height, + ) + .expect("oracle accepts the packed grid"); + let want = ref_53_2d(&plane, width, height); + assert_eq!( + ( + got.low_width, + got.low_height, + got.high_width, + got.high_height + ), + ( + want.low_width, + want.low_height, + want.high_width, + want.high_height + ), + "band dimensions for {width}x{height}" + ); + assert_eq!(got.ll, want.ll, "LL mismatch for {width}x{height}"); + assert_eq!(got.hl, want.hl, "HL mismatch for {width}x{height}"); + assert_eq!(got.lh, want.lh, "LH mismatch for {width}x{height}"); + assert_eq!(got.hh, want.hh, "HH mismatch for {width}x{height}"); + } + } + + #[test] + fn reversible_lift_53_kills_dc_and_linear_detail() { + // Constant -> low = constant, detail exactly zero. + let mut constant = vec![7i32; 32]; + reversible_lift_53_i32(&mut constant); + assert!( + constant.iter().skip(1).step_by(2).all(|&v| v == 0), + "constant produced nonzero detail" + ); + assert!( + constant.iter().step_by(2).all(|&v| v == 7), + "constant low band drifted from 7" + ); + + // Linear ramp -> interior detail exactly zero (two vanishing moments). + let ramp: Vec = (0..40_i32).map(|k| 3 * k - 5).collect(); + let mut lifted = ramp; + reversible_lift_53_i32(&mut lifted); + let detail: Vec = lifted.iter().skip(1).step_by(2).copied().collect(); + for &value in &detail[1..detail.len() - 1] { + assert_eq!(value, 0, "linear ramp produced interior detail {value}"); + } + } + + #[test] + fn reversible_dwt53_2d_separates_horizontal_and_vertical_detail() { + // Varies only along x -> no vertical detail (LH and HH vanish). + let (width, height) = (16usize, 16usize); + let varies_in_x: Vec = (0..width * height) + .map(|i| 3 * i32::try_from(i % width).unwrap() - 7) + .collect(); + let (blocks, bc, br) = pack_plane(&varies_in_x, width, height); + let t = reversible_dwt53_first_level_from_block_samples(&blocks, bc, br, width, height) + .expect("oracle accepts grid"); + assert!( + t.lh.iter().all(|&v| v == 0), + "x-only plane produced LH detail" + ); + assert!( + t.hh.iter().all(|&v| v == 0), + "x-only plane produced HH detail" + ); + + // Varies only along y -> no horizontal detail (HL and HH vanish). + let varies_in_y: Vec = (0..width * height) + .map(|i| 3 * i32::try_from(i / width).unwrap() - 7) + .collect(); + let (blocks, bc, br) = pack_plane(&varies_in_y, width, height); + let t = reversible_dwt53_first_level_from_block_samples(&blocks, bc, br, width, height) + .expect("oracle accepts grid"); + assert!( + t.hl.iter().all(|&v| v == 0), + "y-only plane produced HL detail" + ); + assert!( + t.hh.iter().all(|&v| v == 0), + "y-only plane produced HH detail" + ); + } +} diff --git a/crates/j2k-transcode/src/corpus_validation.rs b/crates/j2k-transcode/src/corpus_validation.rs new file mode 100644 index 00000000..2b1f876f --- /dev/null +++ b/crates/j2k-transcode/src/corpus_validation.rs @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Validation harness helpers for the experimental JPEG-DCT to HTJ2K path. + +use core::fmt; +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::{ + jpeg_to_htj2k, JpegToHtj2kError, JpegToHtj2kOptions, TranscodeValidationClassification, +}; + +/// External WSI JPEG roots, separated using the platform path separator. +pub const TRANSCODE_WSI_ROOT_ENV: &str = "J2K_TRANSCODE_WSI_ROOT"; +/// Require `TRANSCODE_WSI_ROOT_ENV` to be configured and non-empty. +pub const REQUIRE_TRANSCODE_WSI_ROOT_ENV: &str = "J2K_REQUIRE_TRANSCODE_WSI_ROOT"; +/// Maximum number of external WSI tiles to include. `0` means no limit. +pub const TRANSCODE_WSI_TILE_LIMIT_ENV: &str = "J2K_TRANSCODE_WSI_TILE_LIMIT"; +/// Maximum accepted external JPEG payload size in bytes. +pub const TRANSCODE_WSI_MAX_PAYLOAD_BYTES_ENV: &str = "J2K_TRANSCODE_WSI_MAX_PAYLOAD_BYTES"; + +const DEFAULT_EXTERNAL_TILE_LIMIT: usize = 8; +const DEFAULT_MAX_EXTERNAL_PAYLOAD_BYTES: u64 = 67_108_864; + +/// Borrowed JPEG fixture for corpus validation. +#[derive(Debug, Clone, Copy)] +pub struct CorpusFixture<'a> { + /// Human-readable fixture name used in failure messages and reports. + pub name: &'a str, + /// JPEG codestream bytes. + pub bytes: &'a [u8], +} + +/// Owned JPEG fixture loaded from a local corpus path. +#[derive(Debug, Clone)] +pub struct OwnedCorpusFixture { + /// Human-readable fixture name, usually a path. + pub name: String, + /// JPEG codestream bytes. + pub bytes: Vec, +} + +impl OwnedCorpusFixture { + /// Borrow this owned fixture for validation. + #[must_use] + pub fn as_fixture(&self) -> CorpusFixture<'_> { + CorpusFixture { + name: &self.name, + bytes: &self.bytes, + } + } +} + +/// Options for deterministic and optional external corpus validation. +#[derive(Debug, Clone)] +pub struct CorpusValidationOptions { + /// Options passed to `jpeg_to_htj2k`. Validation metrics are enabled by the + /// corpus harness regardless of this value. + pub transcode_options: JpegToHtj2kOptions, + /// Optional local roots containing extracted WSI JPEG tiles. + pub external_wsi_roots: Vec, + /// Whether missing or empty external roots are hard failures. + pub require_external_wsi: bool, + /// Maximum number of external JPEG tiles to load. `0` means no limit. + pub external_tile_limit: usize, + /// Maximum accepted external JPEG payload size in bytes. + pub max_external_payload_bytes: u64, +} + +impl Default for CorpusValidationOptions { + fn default() -> Self { + Self { + transcode_options: JpegToHtj2kOptions::default(), + external_wsi_roots: Vec::new(), + require_external_wsi: false, + external_tile_limit: DEFAULT_EXTERNAL_TILE_LIMIT, + max_external_payload_bytes: DEFAULT_MAX_EXTERNAL_PAYLOAD_BYTES, + } + } +} + +impl CorpusValidationOptions { + /// Build options from the opt-in external corpus environment variables. + #[must_use] + pub fn from_env() -> Self { + let mut options = Self::default(); + if let Some(roots) = std::env::var_os(TRANSCODE_WSI_ROOT_ENV) { + options.external_wsi_roots = std::env::split_paths(&roots).collect(); + } + options.require_external_wsi = std::env::var_os(REQUIRE_TRANSCODE_WSI_ROOT_ENV).is_some(); + if let Ok(limit) = std::env::var(TRANSCODE_WSI_TILE_LIMIT_ENV) { + if let Ok(limit) = limit.parse() { + options.external_tile_limit = limit; + } + } + if let Ok(max_bytes) = std::env::var(TRANSCODE_WSI_MAX_PAYLOAD_BYTES_ENV) { + if let Ok(max_bytes) = max_bytes.parse() { + options.max_external_payload_bytes = max_bytes; + } + } + options + } +} + +/// Aggregate validation report across all tested fixtures. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CorpusValidationReport { + /// Number of JPEG fixtures validated. + pub fixture_count: usize, + /// Number of compared wavelet coefficients. + pub sample_count: usize, + /// Number of coefficients matching the float oracle exactly after + /// rounding. + pub exact_match_count: usize, + /// Maximum absolute rounded-coefficient error. + pub max_abs_error: i64, + /// Threshold classification for the aggregate corpus metrics. + pub classification: TranscodeValidationClassification, + /// Absolute-error histogram keyed by LSB distance. + pub histogram_buckets: BTreeMap, + /// Per-fixture summaries. + pub fixtures: Vec, +} + +/// Validation summary for one JPEG fixture. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CorpusFixtureReport { + /// Fixture name. + pub name: String, + /// Source reference-grid dimensions. + pub dimensions: (u32, u32), + /// Number of JPEG/HTJ2K components. + pub component_count: usize, + /// Compared coefficient count. + pub sample_count: usize, + /// Exact coefficient matches after rounding. + pub exact_match_count: usize, + /// Maximum absolute rounded-coefficient error. + pub max_abs_error: i64, + /// Threshold classification for this fixture's integer-reference metrics. + pub classification: TranscodeValidationClassification, +} + +/// Corpus validation failure. +#[derive(Debug)] +pub enum CorpusValidationError { + /// No fixtures were provided to the deterministic validation pass. + EmptyCorpus, + /// External WSI roots were required but not configured or empty. + MissingRequiredExternalCorpus(&'static str), + /// Reading an external fixture failed. + Io { + /// Path that failed. + path: PathBuf, + /// Source IO error. + source: std::io::Error, + }, + /// Transcoding one fixture failed. + Transcode { + /// Fixture name. + name: String, + /// Source transcode error. + source: JpegToHtj2kError, + }, + /// The transcode completed without the required validation metrics. + MissingMetrics { + /// Fixture name. + name: String, + }, +} + +impl fmt::Display for CorpusValidationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::EmptyCorpus => write!(f, "corpus validation requires at least one fixture"), + Self::MissingRequiredExternalCorpus(reason) => { + write!(f, "required external corpus is unavailable: {reason}") + } + Self::Io { path, source } => { + write!( + f, + "failed to read external fixture {}: {source}", + path.display() + ) + } + Self::Transcode { name, source } => { + write!(f, "failed to transcode fixture {name}: {source}") + } + Self::MissingMetrics { name } => { + write!(f, "fixture {name} did not report validation metrics") + } + } + } +} + +impl std::error::Error for CorpusValidationError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io { source, .. } => Some(source), + Self::Transcode { source, .. } => Some(source), + Self::EmptyCorpus + | Self::MissingRequiredExternalCorpus(_) + | Self::MissingMetrics { .. } => None, + } + } +} + +/// Validate a deterministic set of JPEG fixtures against the integer +/// ISLOW-IDCT-then-reversible-5/3 oracle and aggregate coefficient error +/// metrics. +pub fn validate_transcode_corpus( + fixtures: &[CorpusFixture<'_>], + options: &CorpusValidationOptions, +) -> Result { + if fixtures.is_empty() { + return Err(CorpusValidationError::EmptyCorpus); + } + + let mut report = CorpusValidationReport { + fixture_count: 0, + sample_count: 0, + exact_match_count: 0, + max_abs_error: 0, + classification: TranscodeValidationClassification::Exact, + histogram_buckets: BTreeMap::new(), + fixtures: Vec::with_capacity(fixtures.len()), + }; + + for fixture in fixtures.iter().copied() { + let validated = validate_fixture(fixture, options)?; + report.fixture_count += 1; + report.sample_count += validated.report.sample_count; + report.exact_match_count += validated.report.exact_match_count; + report.max_abs_error = report.max_abs_error.max(validated.report.max_abs_error); + for (error, count) in validated.histogram_buckets { + *report.histogram_buckets.entry(error).or_insert(0) += count; + } + report.fixtures.push(validated.report); + } + report.classification = classify_corpus_report(&report); + + Ok(report) +} + +struct ValidatedFixture { + report: CorpusFixtureReport, + histogram_buckets: BTreeMap, +} + +fn validate_fixture( + fixture: CorpusFixture<'_>, + options: &CorpusValidationOptions, +) -> Result { + let mut transcode_options = options.transcode_options.clone(); + transcode_options.validate_against_integer_reference = true; + let encoded = jpeg_to_htj2k(fixture.bytes, &transcode_options).map_err(|source| { + CorpusValidationError::Transcode { + name: fixture.name.to_string(), + source, + } + })?; + let metrics = encoded + .report + .integer_reference_metrics + .as_ref() + .ok_or_else(|| CorpusValidationError::MissingMetrics { + name: fixture.name.to_string(), + })?; + + Ok(ValidatedFixture { + report: CorpusFixtureReport { + name: fixture.name.to_string(), + dimensions: (encoded.report.width, encoded.report.height), + component_count: encoded.report.component_count, + sample_count: metrics.total, + exact_match_count: metrics.exact_matches, + max_abs_error: metrics.max_abs_error, + classification: TranscodeValidationClassification::classify_metrics(metrics), + }, + histogram_buckets: metrics.absolute_error_histogram.clone(), + }) +} + +fn classify_corpus_report(report: &CorpusValidationReport) -> TranscodeValidationClassification { + if report.sample_count == 0 { + return TranscodeValidationClassification::Exact; + } + if report.exact_match_count == report.sample_count && report.max_abs_error == 0 { + TranscodeValidationClassification::Exact + } else { + let exact_match_rate = report.exact_match_count as f64 / report.sample_count as f64; + if report.max_abs_error <= 1 && exact_match_rate >= 0.999 { + TranscodeValidationClassification::OneLsbBounded + } else { + TranscodeValidationClassification::OutsideThreshold + } + } +} + +/// Load optional external WSI JPEG fixtures from `options.external_wsi_roots`. +/// +/// Normal CI should leave the root list empty. Signoff hosts can set +/// `J2K_TRANSCODE_WSI_ROOT` and build options with +/// [`CorpusValidationOptions::from_env`]. +pub fn load_external_wsi_fixtures( + options: &CorpusValidationOptions, +) -> Result, CorpusValidationError> { + if options.external_wsi_roots.is_empty() { + if options.require_external_wsi { + return Err(CorpusValidationError::MissingRequiredExternalCorpus( + "no roots configured", + )); + } + return Ok(Vec::new()); + } + + let mut paths = Vec::new(); + for root in &options.external_wsi_roots { + collect_jpegs(root, &mut paths); + } + paths.sort(); + if paths.is_empty() && options.require_external_wsi { + return Err(CorpusValidationError::MissingRequiredExternalCorpus( + "configured roots contained no JPEG files", + )); + } + + let limit = if options.external_tile_limit == 0 { + usize::MAX + } else { + options.external_tile_limit + }; + let mut fixtures = Vec::new(); + for path in paths.into_iter().take(limit) { + let metadata = fs::metadata(&path).map_err(|source| CorpusValidationError::Io { + path: path.clone(), + source, + })?; + if metadata.len() > options.max_external_payload_bytes { + continue; + } + let bytes = fs::read(&path).map_err(|source| CorpusValidationError::Io { + path: path.clone(), + source, + })?; + fixtures.push(OwnedCorpusFixture { + name: path.display().to_string(), + bytes, + }); + } + + Ok(fixtures) +} + +fn collect_jpegs(root: &Path, out: &mut Vec) { + if root.is_file() { + if is_jpeg_path(root) { + out.push(root.to_path_buf()); + } + return; + } + let Ok(entries) = fs::read_dir(root) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + collect_jpegs(&path, out); + } else if is_jpeg_path(&path) { + out.push(path); + } + } +} + +fn is_jpeg_path(path: &Path) -> bool { + path.extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| matches!(ext.to_ascii_lowercase().as_str(), "jpg" | "jpeg")) +} diff --git a/crates/j2k-transcode/src/dct53_1d.rs b/crates/j2k-transcode/src/dct53_1d.rs new file mode 100644 index 00000000..5b5b6437 --- /dev/null +++ b/crates/j2k-transcode/src/dct53_1d.rs @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Constrained 1D DCT to 5/3 wavelet experiments. +//! +//! This module intentionally works on one synthetic 8-coefficient DCT block and +//! one single-level 5/3 transform. The float path is a linear composition of +//! the inverse DCT basis with a linearized 5/3 analysis step. The reversible +//! path is bit-exact against the rounded-IDCT reference, but it is piecewise +//! integer arithmetic rather than a single linear matrix. + +use core::fmt; + +use crate::dct_grid::{high_len, idct8_basis, low_len}; +use crate::reversible53::reversible_lift_53_i32; + +/// One single-level 5/3 transform result for an 8-sample 1D signal. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Dwt53OneLevel { + /// Low-pass samples, corresponding to even positions after lifting. + pub low: [T; 4], + /// High-pass samples, corresponding to odd positions after lifting. + pub high: [T; 4], +} + +/// One single-level 5/3 transform result for an arbitrary-length 1D row. +#[derive(Debug, Clone, PartialEq)] +pub struct Dwt53Row { + /// Low-pass samples, corresponding to even positions after lifting. + pub low: Vec, + /// High-pass samples, corresponding to odd positions after lifting. + pub high: Vec, +} + +/// Error returned when a logical row length cannot be covered by DCT blocks. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Dct53RowLengthError { + sample_len: usize, + capacity: usize, +} + +impl Dct53RowLengthError { + /// Requested logical sample length. + #[must_use] + pub const fn sample_len(self) -> usize { + self.sample_len + } + + /// Number of samples covered by the provided 8-point DCT blocks. + #[must_use] + pub const fn capacity(self) -> usize { + self.capacity + } +} + +impl fmt::Display for Dct53RowLengthError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "row length {} exceeds DCT block sample capacity {}", + self.sample_len, self.capacity + ) + } +} + +impl std::error::Error for Dct53RowLengthError {} + +/// Map one 8-point DCT coefficient vector directly into a linearized one-level +/// 5/3 wavelet result. +/// +/// This proves the linear composition: +/// +/// `DWT53_linear * IDCT8 * dct_coefficients` +#[must_use] +pub fn dct8_to_dwt53_float_linear(coefficients: [f64; 8]) -> Dwt53OneLevel { + let rows = linearized_53_rows(); + let mut low = [0.0; 4]; + let mut high = [0.0; 4]; + + for (dst, row) in low.iter_mut().zip(rows[..4].iter()) { + *dst = dct_row_projection(row, &coefficients); + } + for (dst, row) in high.iter_mut().zip(rows[4..].iter()) { + *dst = dct_row_projection(row, &coefficients); + } + + Dwt53OneLevel { low, high } +} + +/// Reference path for the linearized 1D experiment: +/// DCT coefficients -> float IDCT samples -> linearized 5/3. +#[must_use] +pub fn idct8_then_dwt53_float(coefficients: [f64; 8]) -> Dwt53OneLevel { + let mut samples = [0.0; 8]; + for (idx, sample) in samples.iter_mut().enumerate() { + *sample = idct8_sample(&coefficients, idx); + } + linearized_53_from_samples(samples) +} + +/// Map adjacent 8-point DCT blocks directly into a linearized one-level 5/3 +/// wavelet row. +/// +/// This keeps the production direction honest: output coefficients are +/// projected from the DCT basis without first creating a row of spatial samples. +#[must_use] +pub fn dct8_blocks_to_dwt53_float_linear(blocks: &[[f64; 8]]) -> Dwt53Row { + dct8_blocks_to_dwt53_float_linear_with_len(blocks, blocks.len() * 8) + .expect("full row length is always covered by the provided blocks") +} + +/// Map adjacent 8-point DCT blocks directly into a linearized one-level 5/3 +/// wavelet row using a logical sample length. +/// +/// Use this for image component rows whose JPEG block storage includes padded +/// samples beyond the real component width. +pub fn dct8_blocks_to_dwt53_float_linear_with_len( + blocks: &[[f64; 8]], + sample_len: usize, +) -> Result, Dct53RowLengthError> { + validate_sample_len(blocks, sample_len)?; + + let low_len = low_len(sample_len); + let high_len = high_len(sample_len); + let mut low = Vec::with_capacity(low_len); + let mut high = Vec::with_capacity(high_len); + + for output_idx in 0..low_len { + low.push(project_blocks_with_linearized_53_weights( + blocks, sample_len, true, output_idx, + )); + } + for output_idx in 0..high_len { + high.push(project_blocks_with_linearized_53_weights( + blocks, sample_len, false, output_idx, + )); + } + + Ok(Dwt53Row { low, high }) +} + +/// Reference path for an arbitrary-length row: +/// DCT coefficients -> float IDCT samples -> linearized 5/3. +#[must_use] +pub fn idct8_blocks_then_dwt53_float(blocks: &[[f64; 8]]) -> Dwt53Row { + idct8_blocks_then_dwt53_float_with_len(blocks, blocks.len() * 8) + .expect("full row length is always covered by the provided blocks") +} + +/// Reference path for a logical row length: +/// DCT coefficients -> float IDCT samples -> linearized 5/3. +pub fn idct8_blocks_then_dwt53_float_with_len( + blocks: &[[f64; 8]], + sample_len: usize, +) -> Result, Dct53RowLengthError> { + validate_sample_len(blocks, sample_len)?; + + let mut samples = Vec::with_capacity(sample_len); + for sample_idx in 0..sample_len { + let block = &blocks[sample_idx / 8]; + samples.push(idct8_sample(block, sample_idx % 8)); + } + + Ok(linearized_53_from_sample_slice(&samples)) +} + +/// Map one 8-point integer DCT coefficient vector into one reversible 5/3 +/// wavelet result after rounded IDCT sample evaluation. +/// +/// This path is not a linear matrix. It keeps the integer rounding points that +/// the reversible 5/3 path requires, while avoiding a reusable spatial-domain +/// image buffer. +#[must_use] +pub fn dct8_to_dwt53_reversible_i16(coefficients: [i16; 8]) -> Dwt53OneLevel { + let mut samples = [0; 8]; + for (idx, sample) in samples.iter_mut().enumerate() { + *sample = rounded_idct8_sample(&coefficients, idx); + } + reversible_53_from_samples(samples) +} + +/// Reference path for the reversible 1D experiment: +/// DCT coefficients -> rounded IDCT samples -> reversible 5/3. +#[must_use] +pub fn idct8_rounded_then_dwt53_reversible(coefficients: [i16; 8]) -> Dwt53OneLevel { + let mut samples = [0; 8]; + for (idx, sample) in samples.iter_mut().enumerate() { + *sample = rounded_idct8_sample(&coefficients, idx); + } + reversible_53_from_samples(samples) +} + +fn dct_row_projection(sample_weights: &[f64; 8], coefficients: &[f64; 8]) -> f64 { + let mut coefficient_weights = [0.0; 8]; + for (sample_idx, sample_weight) in sample_weights.iter().copied().enumerate() { + for (freq, coefficient_weight) in coefficient_weights.iter_mut().enumerate() { + *coefficient_weight += sample_weight * idct8_basis(sample_idx, freq); + } + } + + coefficient_weights + .iter() + .zip(coefficients.iter()) + .map(|(weight, coefficient)| weight * coefficient) + .sum() +} + +fn project_blocks_with_linearized_53_weights( + blocks: &[[f64; 8]], + sample_len: usize, + is_low: bool, + output_idx: usize, +) -> f64 { + let mut output = 0.0; + + for sample_idx in 0..sample_len { + let sample_weight = linearized_53_sample_weight(sample_len, is_low, output_idx, sample_idx); + if sample_weight == 0.0 { + continue; + } + + let block_idx = sample_idx / 8; + let local_sample_idx = sample_idx % 8; + let block = &blocks[block_idx]; + for (freq, coefficient) in block.iter().copied().enumerate() { + output += sample_weight * idct8_basis(local_sample_idx, freq) * coefficient; + } + } + + output +} + +fn idct8_sample(coefficients: &[f64; 8], sample_idx: usize) -> f64 { + coefficients + .iter() + .enumerate() + .map(|(freq, coefficient)| coefficient * idct8_basis(sample_idx, freq)) + .sum() +} + +fn rounded_idct8_sample(coefficients: &[i16; 8], sample_idx: usize) -> i32 { + let float_coefficients = coefficients.map(f64::from); + idct8_sample(&float_coefficients, sample_idx).round() as i32 +} + +fn linearized_53_sample_weight( + sample_len: usize, + is_low: bool, + output_idx: usize, + sample_idx: usize, +) -> f64 { + let mut basis = vec![0.0; sample_len]; + basis[sample_idx] = 1.0; + let row = linearized_53_from_sample_slice(&basis); + if is_low { + row.low[output_idx] + } else { + row.high[output_idx] + } +} + +fn linearized_53_from_samples(samples: [f64; 8]) -> Dwt53OneLevel { + let row = linearized_53_from_sample_slice(&samples); + Dwt53OneLevel { + low: row + .low + .try_into() + .expect("8 samples produce exactly 4 low-pass outputs"), + high: row + .high + .try_into() + .expect("8 samples produce exactly 4 high-pass outputs"), + } +} + +fn linearized_53_from_sample_slice(samples: &[f64]) -> Dwt53Row { + let mut high = Vec::with_capacity(high_len(samples.len())); + for odd_idx in (1..samples.len()).step_by(2) { + let left = samples[odd_idx - 1]; + let right = samples.get(odd_idx + 1).copied().unwrap_or(left); + high.push(samples[odd_idx] - ((left + right) * 0.5)); + } + + let mut low = Vec::with_capacity(low_len(samples.len())); + for even_idx in (0..samples.len()).step_by(2) { + let current = samples[even_idx]; + let even_output_idx = even_idx / 2; + let left_high = even_output_idx.checked_sub(1).and_then(|idx| high.get(idx)); + let right_high = high.get(even_output_idx); + let update = match (left_high, right_high) { + (Some(left), Some(right)) => (*left + *right) * 0.25, + (None, Some(right)) => *right * 0.5, + (Some(left), None) => *left * 0.5, + (None, None) => 0.0, + }; + low.push(current + update); + } + + Dwt53Row { low, high } +} + +fn reversible_53_from_samples(mut samples: [i32; 8]) -> Dwt53OneLevel { + reversible_lift_53_i32(&mut samples); + Dwt53OneLevel { + low: [samples[0], samples[2], samples[4], samples[6]], + high: [samples[1], samples[3], samples[5], samples[7]], + } +} + +fn validate_sample_len(blocks: &[[f64; 8]], sample_len: usize) -> Result<(), Dct53RowLengthError> { + let capacity = blocks.len() * 8; + if sample_len > capacity { + return Err(Dct53RowLengthError { + sample_len, + capacity, + }); + } + + Ok(()) +} + +fn linearized_53_rows() -> [[f64; 8]; 8] { + [ + [0.75, 0.5, -0.25, 0.0, 0.0, 0.0, 0.0, 0.0], + [-0.125, 0.25, 0.75, 0.25, -0.125, 0.0, 0.0, 0.0], + [0.0, 0.0, -0.125, 0.25, 0.75, 0.25, -0.125, 0.0], + [0.0, 0.0, 0.0, 0.0, -0.125, 0.25, 0.625, 0.25], + [-0.5, 1.0, -0.5, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, -0.5, 1.0, -0.5, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, -0.5, 1.0, -0.5, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0, 1.0], + ] +} diff --git a/crates/j2k-transcode/src/dct53_2d.rs b/crates/j2k-transcode/src/dct53_2d.rs new file mode 100644 index 00000000..4948e5b5 --- /dev/null +++ b/crates/j2k-transcode/src/dct53_2d.rs @@ -0,0 +1,533 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Constrained 2D DCT to 5/3 wavelet experiments. +//! +//! The direct float path projects an 8x8 DCT block into one separable +//! single-level 5/3 result without first storing the 8x8 spatial samples. The +//! reference path materializes samples to keep the oracle easy to audit. + +use crate::dct_grid::{high_len, idct8_basis, low_len, validate_dct_block_grid}; +pub use crate::DctGridError as Dct53GridError; + +/// One separable single-level 2D 5/3 transform result. +#[derive(Debug, Clone, PartialEq)] +pub struct Dwt53TwoDimensional { + /// Low-horizontal, low-vertical band. + pub ll: Vec, + /// High-horizontal, low-vertical band. + pub hl: Vec, + /// Low-horizontal, high-vertical band. + pub lh: Vec, + /// High-horizontal, high-vertical band. + pub hh: Vec, + /// Width of horizontally low-pass bands. + pub low_width: usize, + /// Height of vertically low-pass bands. + pub low_height: usize, + /// Width of horizontally high-pass bands. + pub high_width: usize, + /// Height of vertically high-pass bands. + pub high_height: usize, +} + +impl Dwt53TwoDimensional { + /// Maximum absolute coefficient difference across matching bands. + #[must_use] + pub fn max_abs_diff(&self, other: &Self) -> f64 { + assert_eq!(self.low_width, other.low_width); + assert_eq!(self.low_height, other.low_height); + assert_eq!(self.high_width, other.high_width); + assert_eq!(self.high_height, other.high_height); + + self.ll + .iter() + .zip(other.ll.iter()) + .chain(self.hl.iter().zip(other.hl.iter())) + .chain(self.lh.iter().zip(other.lh.iter())) + .chain(self.hh.iter().zip(other.hh.iter())) + .map(|(actual, expected)| (actual - expected).abs()) + .fold(0.0, f64::max) + } +} + +/// Scratch storage for repeated DCT-grid to 5/3 projection calls. +/// +/// Reuse one value per worker when transforming many components or tiles with +/// matching geometry. The scratch caches linearized 5/3 weight rows; it does +/// not store spatial samples. +#[derive(Debug, Default)] +pub struct Dct53GridScratch { + x_weights: Dwt53WeightRows, + y_weights: Dwt53WeightRows, +} + +impl Dct53GridScratch { + /// Aggregate capacity of cached weight rows. + /// + /// This is intended for experimental tests and benchmark instrumentation. + #[must_use] + pub fn weight_row_capacity(&self) -> usize { + self.x_weights.weight_capacity() + self.y_weights.weight_capacity() + } +} + +/// Map one 8x8 DCT block directly into a linearized one-level 2D 5/3 result. +#[must_use] +pub fn dct8x8_to_dwt53_float_linear(block: [[f64; 8]; 8]) -> Dwt53TwoDimensional { + let width = 8; + let height = 8; + let low_width = low_len(width); + let low_height = low_len(height); + let high_width = high_len(width); + let high_height = high_len(height); + + let mut ll = Vec::with_capacity(low_width * low_height); + let mut hl = Vec::with_capacity(high_width * low_height); + let mut lh = Vec::with_capacity(low_width * high_height); + let mut hh = Vec::with_capacity(high_width * high_height); + + for y in 0..low_height { + for x in 0..low_width { + ll.push(project_dct_block(&block, true, y, true, x)); + } + for x in 0..high_width { + hl.push(project_dct_block(&block, true, y, false, x)); + } + } + + for y in 0..high_height { + for x in 0..low_width { + lh.push(project_dct_block(&block, false, y, true, x)); + } + for x in 0..high_width { + hh.push(project_dct_block(&block, false, y, false, x)); + } + } + + Dwt53TwoDimensional { + ll, + hl, + lh, + hh, + low_width, + low_height, + high_width, + high_height, + } +} + +/// Map an adjacent 8x8 DCT block grid directly into a linearized one-level 2D +/// 5/3 result for the logical component dimensions. +/// +/// Padded JPEG edge samples outside `width x height` are ignored. +pub fn dct8x8_blocks_to_dwt53_float_linear( + blocks: &[[[f64; 8]; 8]], + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, +) -> Result, Dct53GridError> { + let mut scratch = Dct53GridScratch::default(); + dct8x8_blocks_to_dwt53_float_linear_with_scratch( + blocks, + block_cols, + block_rows, + width, + height, + &mut scratch, + ) +} + +/// Map an adjacent 8x8 DCT block grid directly into a linearized one-level 2D +/// 5/3 result using caller-owned scratch for reusable weight rows. +pub fn dct8x8_blocks_to_dwt53_float_linear_with_scratch( + blocks: &[[[f64; 8]; 8]], + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + scratch: &mut Dct53GridScratch, +) -> Result, Dct53GridError> { + validate_grid(blocks.len(), block_cols, block_rows, width, height)?; + + let low_width = low_len(width); + let low_height = low_len(height); + let high_width = high_len(width); + let high_height = high_len(height); + scratch.x_weights.ensure_sample_len(width); + scratch.y_weights.ensure_sample_len(height); + let x_weights = &scratch.x_weights; + let y_weights = &scratch.y_weights; + + let mut ll = Vec::with_capacity(low_width * low_height); + let mut hl = Vec::with_capacity(high_width * low_height); + let mut lh = Vec::with_capacity(low_width * high_height); + let mut hh = Vec::with_capacity(high_width * high_height); + + for y in 0..low_height { + for x in 0..low_width { + ll.push(project_dct_grid( + blocks, + block_cols, + &y_weights.low[y].taps, + &x_weights.low[x].taps, + )); + } + for x in 0..high_width { + hl.push(project_dct_grid( + blocks, + block_cols, + &y_weights.low[y].taps, + &x_weights.high[x].taps, + )); + } + } + + for y in 0..high_height { + for x in 0..low_width { + lh.push(project_dct_grid( + blocks, + block_cols, + &y_weights.high[y].taps, + &x_weights.low[x].taps, + )); + } + for x in 0..high_width { + hh.push(project_dct_grid( + blocks, + block_cols, + &y_weights.high[y].taps, + &x_weights.high[x].taps, + )); + } + } + + Ok(Dwt53TwoDimensional { + ll, + hl, + lh, + hh, + low_width, + low_height, + high_width, + high_height, + }) +} + +/// Reference path for the 2D experiment: +/// DCT coefficients -> float IDCT samples -> separable linearized 5/3. +#[must_use] +pub fn idct8x8_then_dwt53_float(block: [[f64; 8]; 8]) -> Dwt53TwoDimensional { + let mut samples = Vec::with_capacity(64); + for y in 0..8 { + for x in 0..8 { + samples.push(idct8x8_sample(&block, x, y)); + } + } + + linearized_53_2d_from_plane(&samples, 8, 8) +} + +/// Reference path for a DCT block grid: +/// DCT coefficients -> float IDCT samples -> separable linearized 5/3. +pub fn dct8x8_blocks_then_dwt53_float( + blocks: &[[[f64; 8]; 8]], + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, +) -> Result, Dct53GridError> { + validate_grid(blocks.len(), block_cols, block_rows, width, height)?; + + let mut samples = Vec::with_capacity(width * height); + for y in 0..height { + let block_y = y / 8; + let local_y = y % 8; + for x in 0..width { + let block_x = x / 8; + let local_x = x % 8; + let block = &blocks[block_y * block_cols + block_x]; + samples.push(idct8x8_sample(block, local_x, local_y)); + } + } + + Ok(linearized_53_2d_from_plane(&samples, width, height)) +} + +fn project_dct_block( + block: &[[f64; 8]; 8], + vertical_low: bool, + output_y: usize, + horizontal_low: bool, + output_x: usize, +) -> f64 { + let mut output = 0.0; + + for sample_y in 0..8 { + let y_weight = linearized_53_sample_weight(8, vertical_low, output_y, sample_y); + if y_weight == 0.0 { + continue; + } + + for sample_x in 0..8 { + let x_weight = linearized_53_sample_weight(8, horizontal_low, output_x, sample_x); + if x_weight == 0.0 { + continue; + } + + let sample_weight = y_weight * x_weight; + for (freq_y, coefficient_row) in block.iter().enumerate() { + let y_basis = idct8_basis(sample_y, freq_y); + for (freq_x, coefficient) in coefficient_row.iter().copied().enumerate() { + output += sample_weight * y_basis * idct8_basis(sample_x, freq_x) * coefficient; + } + } + } + } + + output +} + +fn project_dct_grid( + blocks: &[[[f64; 8]; 8]], + block_cols: usize, + y_weights: &[SparseWeightTap], + x_weights: &[SparseWeightTap], +) -> f64 { + let mut output = 0.0; + + for &SparseWeightTap { + sample_idx: sample_y, + weight: y_weight, + } in y_weights + { + let block_y = sample_y / 8; + let local_y = sample_y % 8; + + for &SparseWeightTap { + sample_idx: sample_x, + weight: x_weight, + } in x_weights + { + let block_x = sample_x / 8; + let local_x = sample_x % 8; + let block = &blocks[block_y * block_cols + block_x]; + let sample_weight = y_weight * x_weight; + + for (freq_y, coefficient_row) in block.iter().enumerate() { + let y_basis = idct8_basis(local_y, freq_y); + for (freq_x, coefficient) in coefficient_row.iter().copied().enumerate() { + output += sample_weight * y_basis * idct8_basis(local_x, freq_x) * coefficient; + } + } + } + } + + output +} + +fn idct8x8_sample(block: &[[f64; 8]; 8], x: usize, y: usize) -> f64 { + let mut sample = 0.0; + for (freq_y, row) in block.iter().enumerate() { + let y_basis = idct8_basis(y, freq_y); + for (freq_x, coefficient) in row.iter().copied().enumerate() { + sample += coefficient * y_basis * idct8_basis(x, freq_x); + } + } + sample +} + +pub(crate) fn linearized_53_2d_from_plane( + samples: &[f64], + width: usize, + height: usize, +) -> Dwt53TwoDimensional { + debug_assert_eq!(samples.len(), width * height); + + let low_width = low_len(width); + let low_height = low_len(height); + let high_width = high_len(width); + let high_height = high_len(height); + + let mut row_low = Vec::with_capacity(height * low_width); + let mut row_high = Vec::with_capacity(height * high_width); + for y in 0..height { + let start = y * width; + let row = &samples[start..start + width]; + let transformed = linearized_53_from_sample_slice(row); + row_low.extend(transformed.low); + row_high.extend(transformed.high); + } + + let mut ll = Vec::with_capacity(low_width * low_height); + let mut lh = Vec::with_capacity(low_width * high_height); + for x in 0..low_width { + let column = column_from_rows(&row_low, low_width, x, height); + let transformed = linearized_53_from_sample_slice(&column); + ll.extend(transformed.low); + lh.extend(transformed.high); + } + + let mut hl = Vec::with_capacity(high_width * low_height); + let mut hh = Vec::with_capacity(high_width * high_height); + for x in 0..high_width { + let column = column_from_rows(&row_high, high_width, x, height); + let transformed = linearized_53_from_sample_slice(&column); + hl.extend(transformed.low); + hh.extend(transformed.high); + } + + Dwt53TwoDimensional { + ll: transpose_band(&ll, low_height, low_width), + hl: transpose_band(&hl, low_height, high_width), + lh: transpose_band(&lh, high_height, low_width), + hh: transpose_band(&hh, high_height, high_width), + low_width, + low_height, + high_width, + high_height, + } +} + +fn column_from_rows(rows: &[f64], stride: usize, x: usize, height: usize) -> Vec { + (0..height).map(|y| rows[y * stride + x]).collect() +} + +fn transpose_band(column_major: &[f64], height: usize, width: usize) -> Vec { + let mut row_major = Vec::with_capacity(width * height); + for y in 0..height { + for x in 0..width { + row_major.push(column_major[x * height + y]); + } + } + row_major +} + +fn linearized_53_sample_weight( + sample_len: usize, + is_low: bool, + output_idx: usize, + sample_idx: usize, +) -> f64 { + let mut basis = vec![0.0; sample_len]; + basis[sample_idx] = 1.0; + let row = linearized_53_from_sample_slice(&basis); + if is_low { + row.low[output_idx] + } else { + row.high[output_idx] + } +} + +fn linearized_53_from_sample_slice(samples: &[f64]) -> Dwt53OneDimensional { + let mut high = Vec::with_capacity(high_len(samples.len())); + for odd_idx in (1..samples.len()).step_by(2) { + let left = samples[odd_idx - 1]; + let right = samples.get(odd_idx + 1).copied().unwrap_or(left); + high.push(samples[odd_idx] - ((left + right) * 0.5)); + } + + let mut low = Vec::with_capacity(low_len(samples.len())); + for even_idx in (0..samples.len()).step_by(2) { + let current = samples[even_idx]; + let even_output_idx = even_idx / 2; + let left_high = even_output_idx.checked_sub(1).and_then(|idx| high.get(idx)); + let right_high = high.get(even_output_idx); + let update = match (left_high, right_high) { + (Some(left), Some(right)) => (*left + *right) * 0.25, + (None, Some(right)) => *right * 0.5, + (Some(left), None) => *left * 0.5, + (None, None) => 0.0, + }; + low.push(current + update); + } + + Dwt53OneDimensional { low, high } +} + +fn validate_grid( + block_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, +) -> Result<(), Dct53GridError> { + validate_dct_block_grid(block_count, block_cols, block_rows, width, height) +} + +#[derive(Debug, Default)] +struct Dwt53WeightRows { + sample_len: Option, + low: Vec, + high: Vec, +} + +impl Dwt53WeightRows { + fn ensure_sample_len(&mut self, sample_len: usize) { + if self.sample_len == Some(sample_len) { + return; + } + + resize_weight_rows(&mut self.low, low_len(sample_len), 5); + resize_weight_rows(&mut self.high, high_len(sample_len), 3); + + for sample_idx in 0..sample_len { + let mut basis = vec![0.0; sample_len]; + basis[sample_idx] = 1.0; + let transformed = linearized_53_from_sample_slice(&basis); + for (row, &weight) in self.low.iter_mut().zip(transformed.low.iter()) { + if weight != 0.0 { + row.taps.push(SparseWeightTap { sample_idx, weight }); + } + } + for (row, &weight) in self.high.iter_mut().zip(transformed.high.iter()) { + if weight != 0.0 { + row.taps.push(SparseWeightTap { sample_idx, weight }); + } + } + } + + self.sample_len = Some(sample_len); + } + + fn weight_capacity(&self) -> usize { + self.low + .iter() + .map(|row| row.taps.capacity()) + .sum::() + + self + .high + .iter() + .map(|row| row.taps.capacity()) + .sum::() + } +} + +fn resize_weight_rows(rows: &mut Vec, row_count: usize, max_taps: usize) { + if rows.len() < row_count { + rows.resize_with(row_count, SparseWeightRow::default); + } + for row in rows.iter_mut().take(row_count) { + row.taps.clear(); + if row.taps.capacity() < max_taps { + row.taps.reserve_exact(max_taps - row.taps.capacity()); + } + } + rows.truncate(row_count); +} + +#[derive(Debug, Default)] +struct SparseWeightRow { + taps: Vec, +} + +#[derive(Debug, Clone, Copy)] +struct SparseWeightTap { + sample_idx: usize, + weight: f64, +} + +struct Dwt53OneDimensional { + low: Vec, + high: Vec, +} diff --git a/crates/j2k-transcode/src/dct53_multilevel.rs b/crates/j2k-transcode/src/dct53_multilevel.rs new file mode 100644 index 00000000..397ff73c --- /dev/null +++ b/crates/j2k-transcode/src/dct53_multilevel.rs @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Multilevel DCT-to-5/3 experiments. +//! +//! The first decomposition level is produced by the direct DCT-domain mapping. +//! Additional levels use a conventional 5/3 transform over the LL band, which +//! keeps the validation surface smaller until profiling justifies direct LL2+ +//! mappings. + +use core::fmt; + +use crate::dct53_2d::{ + dct8x8_to_dwt53_float_linear, idct8x8_then_dwt53_float, linearized_53_2d_from_plane, + Dwt53TwoDimensional, +}; + +/// Multilevel 5/3 decomposition result for one component plane. +#[derive(Debug, Clone, PartialEq)] +pub struct Dwt53MultiLevel { + /// Decomposition levels in order from full resolution to lowest LL. + pub levels: Vec>, + /// Lowest-resolution LL band after all requested levels. + pub final_ll: Vec, + /// Width of `final_ll`. + pub final_ll_width: usize, + /// Height of `final_ll`. + pub final_ll_height: usize, +} + +impl Dwt53MultiLevel { + /// Maximum absolute coefficient difference across matching levels. + #[must_use] + pub fn max_abs_diff(&self, other: &Self) -> f64 { + assert_eq!(self.levels.len(), other.levels.len()); + assert_eq!(self.final_ll_width, other.final_ll_width); + assert_eq!(self.final_ll_height, other.final_ll_height); + + let level_diff = self + .levels + .iter() + .zip(other.levels.iter()) + .map(|(actual, expected)| actual.max_abs_diff(expected)) + .fold(0.0, f64::max); + + self.final_ll + .iter() + .zip(other.final_ll.iter()) + .map(|(actual, expected)| (actual - expected).abs()) + .fold(level_diff, f64::max) + } +} + +/// Error returned when the requested decomposition level count is invalid. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Dwt53MultiLevelError { + requested_levels: usize, + available_levels: usize, +} + +impl Dwt53MultiLevelError { + /// Requested decomposition levels. + #[must_use] + pub const fn requested_levels(self) -> usize { + self.requested_levels + } + + /// Maximum levels supported by the supplied starting plane. + #[must_use] + pub const fn available_levels(self) -> usize { + self.available_levels + } +} + +impl fmt::Display for Dwt53MultiLevelError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "requested {} decomposition levels, but only {} are available", + self.requested_levels, self.available_levels + ) + } +} + +impl std::error::Error for Dwt53MultiLevelError {} + +/// Direct level-1 DCT-to-5/3 followed by conventional LL recursion. +pub fn dct8x8_to_dwt53_multilevel_float_linear( + block: [[f64; 8]; 8], + levels: usize, +) -> Result, Dwt53MultiLevelError> { + validate_levels(levels, 8, 8)?; + Ok(decompose_from_first_level( + dct8x8_to_dwt53_float_linear(block), + levels, + )) +} + +/// Reference multilevel path: +/// DCT coefficients -> float IDCT samples -> conventional 5/3 recursion. +pub fn idct8x8_then_dwt53_multilevel_float( + block: [[f64; 8]; 8], + levels: usize, +) -> Result, Dwt53MultiLevelError> { + validate_levels(levels, 8, 8)?; + Ok(decompose_from_first_level( + idct8x8_then_dwt53_float(block), + levels, + )) +} + +fn decompose_from_first_level( + first_level: Dwt53TwoDimensional, + levels: usize, +) -> Dwt53MultiLevel { + let mut decomposition = Dwt53MultiLevel { + final_ll: first_level.ll.clone(), + final_ll_width: first_level.low_width, + final_ll_height: first_level.low_height, + levels: vec![first_level], + }; + + while decomposition.levels.len() < levels { + let next = linearized_53_2d_from_plane( + &decomposition.final_ll, + decomposition.final_ll_width, + decomposition.final_ll_height, + ); + decomposition.final_ll.clone_from(&next.ll); + decomposition.final_ll_width = next.low_width; + decomposition.final_ll_height = next.low_height; + decomposition.levels.push(next); + } + + decomposition +} + +fn validate_levels( + requested_levels: usize, + width: usize, + height: usize, +) -> Result<(), Dwt53MultiLevelError> { + let available_levels = available_levels(width, height); + if requested_levels == 0 || requested_levels > available_levels { + return Err(Dwt53MultiLevelError { + requested_levels, + available_levels, + }); + } + + Ok(()) +} + +fn available_levels(mut width: usize, mut height: usize) -> usize { + let mut levels = 0; + while width >= 2 && height >= 2 { + levels += 1; + width = width.div_ceil(2); + height = height.div_ceil(2); + } + levels +} diff --git a/crates/j2k-transcode/src/dct97_2d.rs b/crates/j2k-transcode/src/dct97_2d.rs new file mode 100644 index 00000000..1f5ef7ca --- /dev/null +++ b/crates/j2k-transcode/src/dct97_2d.rs @@ -0,0 +1,836 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Constrained 2D DCT to irreversible 9/7 wavelet transforms. +//! +//! The production float path performs a separable 8x8 IDCT into a reusable +//! spatial plane, then applies the separable single-level 9/7 transform. + +use rayon::prelude::*; + +use crate::dct_grid::{high_len, idct8_basis, idct8_basis_table, low_len, validate_dct_block_grid}; +pub use crate::DctGridError as Dct97GridError; + +const ALPHA: f64 = -1.586_134_342_059_924; +const BETA: f64 = -0.052_980_118_572_961; +const GAMMA: f64 = 0.882_911_075_530_934; +const DELTA: f64 = 0.443_506_852_043_971; +const KAPPA: f64 = 1.230_174_104_914_001; +const INV_KAPPA: f64 = 1.0 / KAPPA; +const PARALLEL_IDCT_MIN_SAMPLES: usize = 64 * 64; + +/// One separable single-level 2D 9/7 transform result. +#[derive(Debug, Clone, PartialEq)] +pub struct Dwt97TwoDimensional { + /// Low-horizontal, low-vertical band. + pub ll: Vec, + /// High-horizontal, low-vertical band. + pub hl: Vec, + /// Low-horizontal, high-vertical band. + pub lh: Vec, + /// High-horizontal, high-vertical band. + pub hh: Vec, + /// Width of horizontally low-pass bands. + pub low_width: usize, + /// Height of vertically low-pass bands. + pub low_height: usize, + /// Width of horizontally high-pass bands. + pub high_width: usize, + /// Height of vertically high-pass bands. + pub high_height: usize, +} + +/// Scratch storage for repeated DCT-grid to 9/7 transform calls. +#[derive(Debug, Default)] +pub struct Dct97GridScratch { + spatial_samples: Vec, + plane: Dwt97PlaneScratch, +} + +#[derive(Debug, Default)] +struct Dwt97PlaneScratch { + row_low: Vec, + row_high: Vec, + lift_workspace: Vec, +} + +impl Dct97GridScratch { + /// Capacity of the reusable spatial-sample buffer used by the IDCT-then + /// 9/7 path. + #[must_use] + pub fn spatial_sample_capacity(&self) -> usize { + self.spatial_samples.capacity() + } +} + +/// Reference path for a DCT block grid: +/// DCT coefficients -> float IDCT samples -> separable linearized 9/7. +pub fn dct8x8_blocks_then_dwt97_float( + blocks: &[[[f64; 8]; 8]], + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, +) -> Result, Dct97GridError> { + validate_grid(blocks.len(), block_cols, block_rows, width, height)?; + + let mut samples = Vec::with_capacity(width * height); + for y in 0..height { + let block_y = y / 8; + let local_y = y % 8; + for x in 0..width { + let block_x = x / 8; + let local_x = x % 8; + let block = &blocks[block_y * block_cols + block_x]; + samples.push(idct8x8_sample(block, local_x, local_y)); + } + } + + Ok(linearized_97_2d_from_plane(&samples, width, height)) +} + +/// Reference 9/7 path with caller-owned spatial-sample scratch: +/// DCT coefficients -> float IDCT samples -> separable linearized 9/7. +pub fn dct8x8_blocks_then_dwt97_float_with_scratch( + blocks: &[[[f64; 8]; 8]], + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + scratch: &mut Dct97GridScratch, +) -> Result, Dct97GridError> { + validate_grid(blocks.len(), block_cols, block_rows, width, height)?; + + let sample_count = width.saturating_mul(height); + scratch.spatial_samples.clear(); + scratch.spatial_samples.resize(sample_count, 0.0); + idct8x8_blocks_to_samples( + blocks, + block_cols, + width, + height, + &mut scratch.spatial_samples, + ); + + Ok(linearized_97_2d_from_plane_with_plane_scratch( + &scratch.spatial_samples, + width, + height, + &mut scratch.plane, + )) +} + +pub(crate) fn linearized_97_2d_from_plane( + samples: &[f64], + width: usize, + height: usize, +) -> Dwt97TwoDimensional { + let mut scratch = Dct97GridScratch::default(); + linearized_97_2d_from_plane_with_scratch(samples, width, height, &mut scratch) +} + +pub(crate) fn linearized_97_2d_from_plane_with_scratch( + samples: &[f64], + width: usize, + height: usize, + scratch: &mut Dct97GridScratch, +) -> Dwt97TwoDimensional { + linearized_97_2d_from_plane_with_plane_scratch(samples, width, height, &mut scratch.plane) +} + +fn linearized_97_2d_from_plane_with_plane_scratch( + samples: &[f64], + width: usize, + height: usize, + scratch: &mut Dwt97PlaneScratch, +) -> Dwt97TwoDimensional { + debug_assert_eq!(samples.len(), width * height); + + let low_width = low_len(width); + let low_height = low_len(height); + let high_width = high_len(width); + let high_height = high_len(height); + + scratch.row_low.clear(); + scratch.row_low.resize(height * low_width, 0.0); + scratch.row_high.clear(); + scratch.row_high.resize(height * high_width, 0.0); + + for y in 0..height { + let start = y * width; + let row = &samples[start..start + width]; + let low_start = y * low_width; + let high_start = y * high_width; + linearized_97_split_contiguous_into( + row, + &mut scratch.row_low[low_start..low_start + low_width], + &mut scratch.row_high[high_start..high_start + high_width], + &mut scratch.lift_workspace, + ); + } + + let mut ll = vec![0.0; low_width * low_height]; + let mut lh = vec![0.0; low_width * high_height]; + for x in 0..low_width { + linearized_97_split_strided_into( + &scratch.row_low, + low_width, + x, + height, + &mut ll, + &mut lh, + low_width, + &mut scratch.lift_workspace, + ); + } + + let mut hl = vec![0.0; high_width * low_height]; + let mut hh = vec![0.0; high_width * high_height]; + for x in 0..high_width { + linearized_97_split_strided_into( + &scratch.row_high, + high_width, + x, + height, + &mut hl, + &mut hh, + high_width, + &mut scratch.lift_workspace, + ); + } + + Dwt97TwoDimensional { + ll, + hl, + lh, + hh, + low_width, + low_height, + high_width, + high_height, + } +} + +fn idct8x8_sample(block: &[[f64; 8]; 8], x: usize, y: usize) -> f64 { + let mut sample = 0.0; + for (freq_y, row) in block.iter().enumerate() { + let y_basis = idct8_basis(y, freq_y); + for (freq_x, coefficient) in row.iter().copied().enumerate() { + sample += coefficient * y_basis * idct8_basis(x, freq_x); + } + } + sample +} + +fn idct8x8_blocks_to_samples( + blocks: &[[[f64; 8]; 8]], + block_cols: usize, + width: usize, + height: usize, + samples: &mut [f64], +) { + debug_assert_eq!(samples.len(), width * height); + let basis = idct8_basis_table(); + let active_block_cols = width.div_ceil(8); + let active_block_rows = height.div_ceil(8); + + if width * height >= PARALLEL_IDCT_MIN_SAMPLES { + samples + .par_chunks_mut(width * 8) + .enumerate() + .take(active_block_rows) + .for_each(|(block_y, sample_rows)| { + idct8x8_block_row_to_samples( + blocks, + block_cols, + width, + height, + basis, + active_block_cols, + block_y, + sample_rows, + ); + }); + } else { + for block_y in 0..active_block_rows { + let block_sample_y = block_y * 8; + let output_rows = (height - block_sample_y).min(8); + let row_start = block_sample_y * width; + let row_end = row_start + output_rows * width; + idct8x8_block_row_to_samples( + blocks, + block_cols, + width, + height, + basis, + active_block_cols, + block_y, + &mut samples[row_start..row_end], + ); + } + } +} + +#[allow(clippy::too_many_arguments)] +fn idct8x8_block_row_to_samples( + blocks: &[[[f64; 8]; 8]], + block_cols: usize, + width: usize, + height: usize, + basis: &[[f64; 8]; 8], + active_block_cols: usize, + block_y: usize, + sample_rows: &mut [f64], +) { + let block_sample_y = block_y * 8; + let output_rows = (height - block_sample_y).min(8); + for block_x in 0..active_block_cols { + let block_sample_x = block_x * 8; + let output_cols = (width - block_sample_x).min(8); + let block = &blocks[block_y * block_cols + block_x]; + let mut vertical = [[0.0; 8]; 8]; + + for (local_y, basis_row) in basis.iter().enumerate() { + for freq_x in 0..8 { + let mut sum = 0.0; + for (freq_y, block_row) in block.iter().enumerate() { + sum += basis_row[freq_y] * block_row[freq_x]; + } + vertical[local_y][freq_x] = sum; + } + } + + for (local_y, vertical_row) in vertical.iter().enumerate().take(output_rows) { + let row_offset = local_y * width + block_sample_x; + for local_x in 0..output_cols { + let mut sample = 0.0; + for (freq_x, vertical_value) in vertical_row.iter().enumerate() { + sample += *vertical_value * basis[local_x][freq_x]; + } + sample_rows[row_offset + local_x] = sample; + } + } + } +} + +#[cfg(test)] +fn linearized_97_from_sample_slice(samples: &[f64]) -> Dwt97OneDimensional { + let mut lifted = samples.to_vec(); + forward_lift_97(&mut lifted); + + Dwt97OneDimensional { + low: lifted.iter().step_by(2).copied().collect(), + high: lifted.iter().skip(1).step_by(2).copied().collect(), + } +} + +fn forward_lift_97(data: &mut [f64]) { + let n = data.len(); + if n < 2 { + return; + } + + let last_even = if n.is_multiple_of(2) { n - 2 } else { n - 1 }; + + for i in (1..n).step_by(2) { + let left = data[i - 1]; + let right = if i + 1 < n { + data[i + 1] + } else { + data[last_even] + }; + data[i] += ALPHA * (left + right); + } + + for i in (0..n).step_by(2) { + let left = if i > 0 { data[i - 1] } else { data[1] }; + let right = if i + 1 < n { data[i + 1] } else { left }; + data[i] += BETA * (left + right); + } + + for i in (1..n).step_by(2) { + let left = data[i - 1]; + let right = if i + 1 < n { + data[i + 1] + } else { + data[last_even] + }; + data[i] += GAMMA * (left + right); + } + + for i in (0..n).step_by(2) { + let left = if i > 0 { data[i - 1] } else { data[1] }; + let right = if i + 1 < n { data[i + 1] } else { left }; + data[i] += DELTA * (left + right); + } + + for i in (0..n).step_by(2) { + data[i] *= INV_KAPPA; + } + for i in (1..n).step_by(2) { + data[i] *= KAPPA; + } +} + +fn linearized_97_split_contiguous_into( + samples: &[f64], + low: &mut [f64], + high: &mut [f64], + workspace: &mut Vec, +) { + debug_assert_eq!(low.len(), low_len(samples.len())); + debug_assert_eq!(high.len(), high_len(samples.len())); + + workspace.clear(); + workspace.extend_from_slice(samples); + forward_lift_97(workspace); + + for (target, value) in low.iter_mut().zip(workspace.iter().step_by(2)) { + *target = *value; + } + for (target, value) in high.iter_mut().zip(workspace.iter().skip(1).step_by(2)) { + *target = *value; + } +} + +#[allow(clippy::too_many_arguments)] +fn linearized_97_split_strided_into( + samples: &[f64], + stride: usize, + x: usize, + height: usize, + low: &mut [f64], + high: &mut [f64], + band_width: usize, + workspace: &mut Vec, +) { + debug_assert_eq!(low.len(), band_width * low_len(height)); + debug_assert_eq!(high.len(), band_width * high_len(height)); + + workspace.clear(); + workspace.extend((0..height).map(|y| samples[y * stride + x])); + forward_lift_97(workspace); + + for (low_y, value) in workspace.iter().step_by(2).enumerate() { + low[low_y * band_width + x] = *value; + } + for (high_y, value) in workspace.iter().skip(1).step_by(2).enumerate() { + high[high_y * band_width + x] = *value; + } +} + +fn validate_grid( + block_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, +) -> Result<(), Dct97GridError> { + validate_dct_block_grid(block_count, block_cols, block_rows, width, height) +} + +#[cfg(test)] +struct Dwt97OneDimensional { + low: Vec, + high: Vec, +} + +#[cfg(test)] +mod tests { + use core::f64::consts::PI; + + use super::*; + + fn assert_all_close(values: &[f64], expected: f64, epsilon: f64) { + for &value in values { + assert!( + (value - expected).abs() < epsilon, + "value={value} expected={expected} values={values:?}" + ); + } + } + + #[test] + fn linearized_97_from_constant_signal_places_dc_in_low_pass() { + for len in [2usize, 3, 8, 9, 64, 65] { + let samples = vec![50.0; len]; + + let transformed = linearized_97_from_sample_slice(&samples); + + assert_all_close(&transformed.low, 50.0, 0.001); + assert_all_close(&transformed.high, 0.0, 0.001); + } + } + + #[test] + fn linearized_97_2d_from_constant_plane_places_dc_in_ll() { + for (width, height) in [(8usize, 8usize), (9, 7)] { + let samples = vec![50.0; width * height]; + + let transformed = linearized_97_2d_from_plane(&samples, width, height); + + assert_all_close(&transformed.ll, 50.0, 0.001); + assert_all_close(&transformed.hl, 0.0, 0.001); + assert_all_close(&transformed.lh, 0.0, 0.001); + assert_all_close(&transformed.hh, 0.0, 0.001); + } + } + + // ------------------------------------------------------------------------- + // Independent CDF 9/7 ground truth. + // + // The CUDA 9/7 kernel is parity-tested against `forward_lift_97` / + // `linearized_97_2d_from_plane`, so a bug in the lifting would be faithfully + // reproduced by the kernel and pass that parity test unnoticed. These tests + // close that gap by validating the lifting against an *independent* + // implementation: a direct FIR filter bank using the canonical, fully + // normalized CDF 9/7 analysis taps and JPEG2000 whole-sample symmetric + // extension. Different arithmetic, same transform. + // + // The taps themselves are checked against their defining mathematical + // properties (DC gains and high-pass vanishing moments) so they cannot + // silently drift to "match" a buggy lifting. + + /// Canonical CDF 9/7 analysis low-pass filter (9 taps, even-symmetric). + /// Fully normalized so its DC gain is 1 (a constant maps unchanged into the + /// low band, matching the lifting's `INV_KAPPA` scaling). + const REF_LP: [f64; 9] = [ + 0.026_748_757_410_810, + -0.016_864_118_442_875, + -0.078_223_266_528_990, + 0.266_864_118_442_875, + 0.602_949_018_236_360, + 0.266_864_118_442_875, + -0.078_223_266_528_990, + -0.016_864_118_442_875, + 0.026_748_757_410_810, + ]; + + /// Canonical CDF 9/7 analysis high-pass filter (7 taps, even-symmetric). + /// Fully normalized so its DC gain is 0 (matching the lifting's `KAPPA` + /// scaling); it has four vanishing moments. + const REF_HP: [f64; 7] = [ + 0.091_271_763_114_250, + -0.057_543_526_228_500, + -0.591_271_763_114_247, + 1.115_087_052_456_994, + -0.591_271_763_114_247, + -0.057_543_526_228_500, + 0.091_271_763_114_250, + ]; + + /// Whole-sample symmetric reflection: mirror about index 0 and `n - 1` + /// without repeating the endpoints. This is the boundary extension + /// `forward_lift_97` implements at the array edges. + fn ws_reflect(i: isize, n: usize) -> usize { + debug_assert!(n >= 1); + if n == 1 { + return 0; + } + let n = isize::try_from(n).expect("signal length fits in isize"); + let period = 2 * (n - 1); + let mut k = i.rem_euclid(period); + if k >= n { + k = period - k; + } + usize::try_from(k).expect("reflected index is non-negative") + } + + /// Independent single-level forward 9/7 analysis via direct convolution. + /// Returns `(low, high)` interleaved-position bands matching `forward_lift_97` + /// (`low[m]` centered at sample `2m`, `high[m]` centered at sample `2m + 1`). + fn ref_analysis_1d(signal: &[f64]) -> (Vec, Vec) { + let n = signal.len(); + if n < 2 { + // The lifting leaves <2-length signals unchanged (low = the sample). + return (signal.to_vec(), Vec::new()); + } + let mut low = vec![0.0; low_len(n)]; + let mut high = vec![0.0; high_len(n)]; + for (m, out) in low.iter_mut().enumerate() { + let center = 2 * isize::try_from(m).unwrap(); + *out = REF_LP + .iter() + .enumerate() + .map(|(t, &tap)| { + tap * signal[ws_reflect(center + isize::try_from(t).unwrap() - 4, n)] + }) + .sum(); + } + for (m, out) in high.iter_mut().enumerate() { + let center = 2 * isize::try_from(m).unwrap() + 1; + *out = REF_HP + .iter() + .enumerate() + .map(|(t, &tap)| { + tap * signal[ws_reflect(center + isize::try_from(t).unwrap() - 3, n)] + }) + .sum(); + } + (low, high) + } + + /// Independent separable 2D forward 9/7 (rows then columns) producing the + /// same four-band layout as `linearized_97_2d_from_plane`. + fn ref_analysis_2d(samples: &[f64], width: usize, height: usize) -> Dwt97TwoDimensional { + let low_width = low_len(width); + let high_width = high_len(width); + let low_height = low_len(height); + let high_height = high_len(height); + + let mut row_low = vec![0.0; height * low_width]; + let mut row_high = vec![0.0; height * high_width]; + for y in 0..height { + let (lo, hi) = ref_analysis_1d(&samples[y * width..y * width + width]); + row_low[y * low_width..y * low_width + low_width].copy_from_slice(&lo); + row_high[y * high_width..y * high_width + high_width].copy_from_slice(&hi); + } + + let vertical_split = |source: &[f64], band_width: usize| -> (Vec, Vec) { + let mut low = vec![0.0; band_width * low_height]; + let mut high = vec![0.0; band_width * high_height]; + for x in 0..band_width { + let column: Vec = (0..height).map(|y| source[y * band_width + x]).collect(); + let (lo, hi) = ref_analysis_1d(&column); + for (vy, &value) in lo.iter().enumerate() { + low[vy * band_width + x] = value; + } + for (vy, &value) in hi.iter().enumerate() { + high[vy * band_width + x] = value; + } + } + (low, high) + }; + + let (ll, lh) = vertical_split(&row_low, low_width); + let (hl, hh) = vertical_split(&row_high, high_width); + + Dwt97TwoDimensional { + ll, + hl, + lh, + hh, + low_width, + low_height, + high_width, + high_height, + } + } + + /// Small deterministic PRNG (LCG) for reproducible test signals in [-1, 1). + fn next_unit(state: &mut u64) -> f64 { + *state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + ((*state >> 11) as f64 / (1u64 << 53) as f64).mul_add(2.0, -1.0) + } + + fn assert_bands_close(actual: &[f64], expected: &[f64], label: &str, epsilon: f64) { + assert_eq!(actual.len(), expected.len(), "{label} band length"); + for (i, (a, b)) in actual.iter().zip(expected.iter()).enumerate() { + assert!( + (a - b).abs() <= epsilon, + "{label}[{i}] diverged: lifting={a} reference={b} (diff {})", + (a - b).abs() + ); + } + } + + #[test] + fn reference_cdf97_taps_satisfy_their_defining_properties() { + // Low-pass DC gain 1, high-pass DC gain 0 — the normalization the + // lifting's KAPPA scaling targets. + let lp_dc: f64 = REF_LP.iter().sum(); + assert!((lp_dc - 1.0).abs() < 1e-9, "low-pass DC gain = {lp_dc}"); + let hp_dc: f64 = REF_HP.iter().sum(); + assert!(hp_dc.abs() < 1e-9, "high-pass DC gain = {hp_dc}"); + + // Even symmetry. + for k in 0..4 { + assert!( + (REF_LP[k] - REF_LP[8 - k]).abs() < 1e-15, + "low-pass asymmetric at {k}" + ); + } + for k in 0..3 { + assert!( + (REF_HP[k] - REF_HP[6 - k]).abs() < 1e-15, + "high-pass asymmetric at {k}" + ); + } + + // Four vanishing moments: the high-pass annihilates polynomials of + // degree <= 3 (so a wrong predict coefficient or sign cannot pass). + for m in 1..=3 { + let moment: f64 = REF_HP + .iter() + .enumerate() + .map(|(k, &tap)| (k as f64 - 3.0).powi(m) * tap) + .sum(); + assert!(moment.abs() < 1e-9, "high-pass moment {m} = {moment}"); + } + } + + #[test] + fn forward_lift_97_matches_independent_filter_bank_1d() { + let mut state = 0x1234_5678_9abc_def0u64; + for n in [2usize, 3, 4, 5, 8, 9, 12, 15, 16, 23, 32, 33, 64, 65] { + let signal: Vec = (0..n).map(|_| next_unit(&mut state) * 100.0).collect(); + let lifted = linearized_97_from_sample_slice(&signal); + let (low, high) = ref_analysis_1d(&signal); + assert_bands_close(&lifted.low, &low, &format!("n={n} low"), 1e-9); + assert_bands_close(&lifted.high, &high, &format!("n={n} high"), 1e-9); + } + } + + #[test] + fn forward_lift_97_annihilates_low_degree_polynomials() { + // Independent of the filter bank: a correct 9/7 high-pass kills cubics in + // the interior (boundary coefficients use symmetric extension). This pins + // the predict-step coefficients and signs directly from wavelet theory. + let n = 40usize; + let polynomials: [[f64; 4]; 4] = [ + [5.0, 0.0, 0.0, 0.0], + [0.0, 2.5, 0.0, 0.0], + [1.0, -0.7, 0.3, 0.0], + [0.0, 0.0, 0.0, 0.05], + ]; + for coeffs in polynomials { + let signal: Vec = (0..n) + .map(|i| { + let x = i as f64; + coeffs[3].mul_add( + x * x * x, + coeffs[2].mul_add(x * x, coeffs[1].mul_add(x, coeffs[0])), + ) + }) + .collect(); + let lifted = linearized_97_from_sample_slice(&signal); + // Skip the first/last high-pass coefficients (boundary support). + let interior = &lifted.high[3..lifted.high.len() - 3]; + assert_all_close(interior, 0.0, 1e-6); + } + } + + #[test] + fn linearized_97_2d_matches_independent_separable_filter_bank() { + let mut state = 0xfeed_face_dead_beefu64; + for (width, height) in [ + (8usize, 8usize), + (16, 16), + (24, 16), + (15, 13), + (16, 23), + (9, 7), + (32, 32), + ] { + let samples: Vec = (0..width * height) + .map(|_| next_unit(&mut state) * 100.0) + .collect(); + let got = linearized_97_2d_from_plane(&samples, width, height); + let want = ref_analysis_2d(&samples, width, height); + assert_eq!( + ( + got.low_width, + got.low_height, + got.high_width, + got.high_height + ), + ( + want.low_width, + want.low_height, + want.high_width, + want.high_height + ), + "band dimensions for {width}x{height}" + ); + assert_bands_close(&got.ll, &want.ll, &format!("{width}x{height} ll"), 1e-9); + assert_bands_close(&got.hl, &want.hl, &format!("{width}x{height} hl"), 1e-9); + assert_bands_close(&got.lh, &want.lh, &format!("{width}x{height} lh"), 1e-9); + assert_bands_close(&got.hh, &want.hh, &format!("{width}x{height} hh"), 1e-9); + } + } + + #[test] + fn linearized_97_2d_separates_horizontal_and_vertical_detail() { + // Catches an HL/LH swap or a row/column transpose independently of the + // filter bank: a plane that varies only along x has no vertical detail + // (LH and HH must vanish), and vice versa. + let (width, height) = (16usize, 16usize); + + let varies_in_x: Vec = (0..width * height) + .map(|i| ((i % width) as f64).sin().mul_add(30.0, 5.0)) + .collect(); + let t = linearized_97_2d_from_plane(&varies_in_x, width, height); + assert_all_close(&t.lh, 0.0, 1e-9); + assert_all_close(&t.hh, 0.0, 1e-9); + + let varies_in_y: Vec = (0..width * height) + .map(|i| ((i / width) as f64).cos().mul_add(30.0, 5.0)) + .collect(); + let t = linearized_97_2d_from_plane(&varies_in_y, width, height); + assert_all_close(&t.hl, 0.0, 1e-9); + assert_all_close(&t.hh, 0.0, 1e-9); + } + + // ------------------------------------------------------------------------- + // Ground truth: exact mathematical inverse DCT for the float 9/7 path. + // + // The 9/7 transcode oracle (`dct8x8_blocks_then_dwt97_float`) feeds + // `idct8x8_sample` into the wavelet. Validate that IDCT against the defining + // DCT-III cosine sum so a basis/normalization/transpose bug cannot hide + // inside both the oracle and its CUDA port. + fn exact_idct_sample(block: &[[f64; 8]; 8], x: usize, y: usize) -> f64 { + let alpha = |k: usize| { + if k == 0 { + (1.0_f64 / 8.0).sqrt() + } else { + (2.0_f64 / 8.0).sqrt() + } + }; + let cos_term = |sample: usize, freq: usize| { + (((2 * sample + 1) as f64) * freq as f64 * PI / 16.0).cos() + }; + let mut acc = 0.0; + for (v, row) in block.iter().enumerate() { + for (u, &coeff) in row.iter().enumerate() { + acc += alpha(u) * alpha(v) * coeff * cos_term(x, u) * cos_term(y, v); + } + } + acc + } + + #[test] + fn idct8x8_sample_matches_exact_cosine_sum() { + let mut state = 0x5151_aaaa_bbbb_ccccu64; + for _ in 0..64 { + let mut block = [[0.0f64; 8]; 8]; + for row in &mut block { + for coeff in row { + *coeff = next_unit(&mut state) * 64.0; + } + } + for y in 0..8 { + for x in 0..8 { + let got = idct8x8_sample(&block, x, y); + let want = exact_idct_sample(&block, x, y); + assert!( + (got - want).abs() < 1e-9, + "idct8x8_sample({x},{y})={got} exact={want}" + ); + } + } + } + } + + #[test] + fn idct8x8_sample_dc_only_is_uniform() { + // DC-only block -> uniform plane equal to F(0,0) / 8. + let mut block = [[0.0f64; 8]; 8]; + block[0][0] = 320.0; + for y in 0..8 { + for x in 0..8 { + assert!((idct8x8_sample(&block, x, y) - 40.0).abs() < 1e-9); + } + } + } +} diff --git a/crates/j2k-transcode/src/dct_grid.rs b/crates/j2k-transcode/src/dct_grid.rs new file mode 100644 index 00000000..ce25f8e1 --- /dev/null +++ b/crates/j2k-transcode/src/dct_grid.rs @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: Apache-2.0 + +use core::f64::consts::PI; +use core::fmt; +use std::sync::LazyLock; + +pub(crate) const fn low_len(sample_len: usize) -> usize { + sample_len.div_ceil(2) +} + +pub(crate) const fn high_len(sample_len: usize) -> usize { + sample_len / 2 +} + +pub(crate) fn idct8_basis(sample_idx: usize, freq: usize) -> f64 { + debug_assert!(sample_idx < 8); + debug_assert!(freq < 8); + + idct8_basis_table()[sample_idx][freq] +} + +pub(crate) fn idct8_basis_table() -> &'static [[f64; 8]; 8] { + static BASIS: LazyLock<[[f64; 8]; 8]> = LazyLock::new(|| { + let mut basis = [[0.0; 8]; 8]; + for (sample_idx, row) in basis.iter_mut().enumerate() { + for (freq, value) in row.iter_mut().enumerate() { + *value = idct8_basis_uncached(sample_idx, freq); + } + } + basis + }); + &BASIS +} + +fn idct8_basis_uncached(sample_idx: usize, freq: usize) -> f64 { + let scale = if freq == 0 { + (1.0_f64 / 8.0).sqrt() + } else { + (2.0_f64 / 8.0).sqrt() + }; + scale * (((sample_idx as f64 + 0.5) * freq as f64 * PI) / 8.0).cos() +} + +/// Error returned when a DCT block grid cannot cover the requested component. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DctGridError { + block_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, +} + +impl DctGridError { + const fn new( + block_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, + ) -> Self { + Self { + block_count, + block_cols, + block_rows, + width, + height, + } + } + + /// Number of supplied 8x8 DCT blocks. + #[must_use] + pub const fn block_count(self) -> usize { + self.block_count + } + + /// Declared block columns. + #[must_use] + pub const fn block_cols(self) -> usize { + self.block_cols + } + + /// Declared block rows. + #[must_use] + pub const fn block_rows(self) -> usize { + self.block_rows + } + + /// Requested component width. + #[must_use] + pub const fn width(self) -> usize { + self.width + } + + /// Requested component height. + #[must_use] + pub const fn height(self) -> usize { + self.height + } +} + +impl fmt::Display for DctGridError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "DCT grid has {} blocks for {}x{} grid covering requested {}x{} samples", + self.block_count, self.block_cols, self.block_rows, self.width, self.height + ) + } +} + +impl std::error::Error for DctGridError {} + +pub(crate) fn validate_dct_block_grid( + block_count: usize, + block_cols: usize, + block_rows: usize, + width: usize, + height: usize, +) -> Result<(), DctGridError> { + let expected_blocks = block_cols + .checked_mul(block_rows) + .ok_or_else(|| DctGridError::new(block_count, block_cols, block_rows, width, height))?; + let covered_width = block_cols + .checked_mul(8) + .ok_or_else(|| DctGridError::new(block_count, block_cols, block_rows, width, height))?; + let covered_height = block_rows + .checked_mul(8) + .ok_or_else(|| DctGridError::new(block_count, block_cols, block_rows, width, height))?; + + if block_count != expected_blocks + || width == 0 + || height == 0 + || width > covered_width + || height > covered_height + { + return Err(DctGridError::new( + block_count, + block_cols, + block_rows, + width, + height, + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{high_len, idct8_basis, low_len, validate_dct_block_grid}; + + #[test] + fn band_lengths_split_even_and_odd_samples() { + assert_eq!((low_len(0), high_len(0)), (0, 0)); + assert_eq!((low_len(1), high_len(1)), (1, 0)); + assert_eq!((low_len(8), high_len(8)), (4, 4)); + assert_eq!((low_len(9), high_len(9)), (5, 4)); + } + + #[test] + fn idct8_basis_is_orthonormal_for_dc_and_first_ac() { + assert!((idct8_basis(0, 0) - (1.0_f64 / 8.0).sqrt()).abs() < 1e-12); + assert!((idct8_basis(0, 1) - 0.490_392_640_201_615_2).abs() < 1e-12); + } + + #[test] + fn validates_non_empty_grid_covered_by_blocks() { + assert_eq!(validate_dct_block_grid(6, 3, 2, 24, 16), Ok(())); + } + + #[test] + fn rejects_overflowing_grid_dimensions() { + let err = validate_dct_block_grid(1, usize::MAX, usize::MAX, 8, 8) + .expect_err("overflowing dimensions must fail"); + assert_eq!(err.block_count(), 1); + assert_eq!(err.block_cols(), usize::MAX); + assert_eq!(err.block_rows(), usize::MAX); + } + + #[test] + fn rejects_zero_image_extent() { + assert!(validate_dct_block_grid(1, 1, 1, 0, 8).is_err()); + assert!(validate_dct_block_grid(1, 1, 1, 8, 0).is_err()); + } +} diff --git a/crates/j2k-transcode/src/htj2k97_codeblock_oracle.rs b/crates/j2k-transcode/src/htj2k97_codeblock_oracle.rs new file mode 100644 index 00000000..745e8941 --- /dev/null +++ b/crates/j2k-transcode/src/htj2k97_codeblock_oracle.rs @@ -0,0 +1,515 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Shared scalar oracle: float 9/7 bands into prequantized HTJ2K code-blocks. +//! +//! This module uses the native encoder's public irreversible 9/7 quantization +//! step helper plus the native code-block layout rules, so both GPU backends can +//! compare their fused code-block kernels against one authoritative CPU +//! reference instead of each re-deriving the math. +//! +//! The re-derivation is anchored to native truth by a codestream pin test (see +//! the module tests): encoding the oracle's prequantized output reproduces the +//! native precomputed-DWT codestream byte-for-byte. + +use crate::accelerator::Htj2k97CodeBlockOptions; +use crate::dct97_2d::Dwt97TwoDimensional; +use j2k::adapter::encode_stage::{ + J2kSubBandType, PrequantizedHtj2k97CodeBlock, PrequantizedHtj2k97Component, + PrequantizedHtj2k97Resolution, PrequantizedHtj2k97Subband, +}; +use j2k_native::irreversible_quantization_step_for_subband; + +/// Quantize one level of float 9/7 bands into a prequantized HTJ2K component. +/// +/// Resolution nesting matches the native encoder for a single decomposition +/// level: resolution 0 holds `[LL]`, resolution 1 holds `[HL, LH, HH]`. +#[must_use] +pub fn prequantized_component_from_dwt97( + dwt: &Dwt97TwoDimensional, + options: Htj2k97CodeBlockOptions, + x_rsiz: u8, + y_rsiz: u8, +) -> PrequantizedHtj2k97Component { + PrequantizedHtj2k97Component { + x_rsiz, + y_rsiz, + resolutions: vec![ + PrequantizedHtj2k97Resolution { + subbands: vec![quantize_codeblock_subband( + &dwt.ll, + dwt.low_width, + dwt.low_height, + J2kSubBandType::LowLow, + options, + )], + }, + PrequantizedHtj2k97Resolution { + subbands: vec![ + quantize_codeblock_subband( + &dwt.hl, + dwt.high_width, + dwt.low_height, + J2kSubBandType::HighLow, + options, + ), + quantize_codeblock_subband( + &dwt.lh, + dwt.low_width, + dwt.high_height, + J2kSubBandType::LowHigh, + options, + ), + quantize_codeblock_subband( + &dwt.hh, + dwt.high_width, + dwt.high_height, + J2kSubBandType::HighHigh, + options, + ), + ], + }, + ], + } +} + +/// Quantize a single float subband and slice it into code-block-major layout. +/// +/// Code-blocks are emitted outer `cby`, inner `cbx`; each block's coefficients +/// are row-major, matching the native encoder's `copy_code_block_coefficients`. +#[must_use] +pub fn quantize_codeblock_subband( + coefficients: &[f64], + width: usize, + height: usize, + sub_band_type: J2kSubBandType, + options: Htj2k97CodeBlockOptions, +) -> PrequantizedHtj2k97Subband { + let quantized = quantize_subband_coefficients(coefficients, sub_band_type, options); + let cb_width = htj2k97_code_block_dim(options.code_block_width_exp); + let cb_height = htj2k97_code_block_dim(options.code_block_height_exp); + let num_cbs_x = width.div_ceil(cb_width); + let num_cbs_y = height.div_ceil(cb_height); + let mut code_blocks = Vec::with_capacity(num_cbs_x * num_cbs_y); + + for cby in 0..num_cbs_y { + for cbx in 0..num_cbs_x { + let x0 = cbx * cb_width; + let y0 = cby * cb_height; + let block_width = (width - x0).min(cb_width); + let block_height = (height - y0).min(cb_height); + let mut block_coefficients = Vec::with_capacity(block_width * block_height); + for y in 0..block_height { + let row_start = (y0 + y) * width + x0; + block_coefficients + .extend_from_slice(&quantized[row_start..row_start + block_width]); + } + code_blocks.push(PrequantizedHtj2k97CodeBlock { + coefficients: block_coefficients, + width: block_width as u32, + height: block_height as u32, + }); + } + } + + PrequantizedHtj2k97Subband { + sub_band_type, + num_cbs_x: num_cbs_x as u32, + num_cbs_y: num_cbs_y as u32, + total_bitplanes: htj2k97_subband_total_bitplanes(options, sub_band_type), + code_blocks, + } +} + +/// Deadzone quantization step size `Δ` for a subband. +/// +/// `Δ = 2^(range_bits − exponent) · (1 + mantissa/2048)`, with +/// `range_bits = bit_depth + {LL:0, HL:1, LH:1, HH:2}` and the shared +/// `(exponent, mantissa)` derived by this module's quantizer. +#[must_use] +pub fn htj2k97_subband_delta( + options: Htj2k97CodeBlockOptions, + sub_band_type: J2kSubBandType, +) -> f64 { + let log_gain = match sub_band_type { + J2kSubBandType::LowLow => 0, + J2kSubBandType::HighLow | J2kSubBandType::LowHigh => 1, + J2kSubBandType::HighHigh => 2, + }; + let range_bits = i32::from(options.bit_depth) + log_gain; + let (exponent, mantissa) = htj2k97_step(options, sub_band_type); + pow2i_f64(range_bits - i32::from(exponent)) * (1.0 + f64::from(mantissa) / 2048.0) +} + +/// Total declared bitplanes for every code-block in a subband. +/// +/// `saturating(guard_bits + exponent - 1)`. The exponent is derived from the +/// effective global plus per-subband quantization profile, so callers must pass +/// the actual subband kind. +#[must_use] +pub fn htj2k97_subband_total_bitplanes( + options: Htj2k97CodeBlockOptions, + sub_band_type: J2kSubBandType, +) -> u8 { + let (exponent, _) = htj2k97_step(options, sub_band_type); + options + .guard_bits + .saturating_add(exponent) + .saturating_sub(1) +} + +/// Validate 9/7 code-block options against the numeric limits both GPU +/// backends must agree on, returning the decoded `(cb_width, cb_height)`. +/// +/// One shared implementation keeps Metal and CUDA from drifting: the same +/// options must be accepted or rejected identically by every backend. Errors +/// are backend-neutral static strings for the caller's unsupported-job error. +/// +/// # Errors +/// Rejects zero/oversized bit depths and guard bits, non-finite or +/// non-positive quantization scales, code-block dimensions beyond the HTJ2K +/// limits (sides ≤ 1024, area ≤ 4096), and subband deltas or total bitplane +/// counts outside the supported range. +pub fn validate_htj2k97_codeblock_options( + options: Htj2k97CodeBlockOptions, +) -> Result<(usize, usize), &'static str> { + if options.bit_depth == 0 + || options.bit_depth > 30 + || options.guard_bits > 30 + || !options.irreversible_quantization_scale.is_finite() + || options.irreversible_quantization_scale <= 0.0 + { + return Err("9/7 code-block options are outside supported numeric range"); + } + let subband_scales = options.irreversible_quantization_subband_scales; + if [ + subband_scales.low_low, + subband_scales.high_low, + subband_scales.low_high, + subband_scales.high_high, + ] + .iter() + .any(|scale| !scale.is_finite() || *scale <= 0.0) + { + return Err("9/7 code-block quantization options are outside supported range"); + } + + let cb_width = checked_code_block_dim(options.code_block_width_exp)?; + let cb_height = checked_code_block_dim(options.code_block_height_exp)?; + if cb_width > 1024 + || cb_height > 1024 + || cb_width + .checked_mul(cb_height) + .is_none_or(|area| area > 4096) + { + return Err("9/7 code-block dimensions exceed HTJ2K limits"); + } + + for subband in [ + J2kSubBandType::LowLow, + J2kSubBandType::HighLow, + J2kSubBandType::LowHigh, + J2kSubBandType::HighHigh, + ] { + let delta = htj2k97_subband_delta(options, subband); + if !delta.is_finite() + || delta <= 0.0 + || htj2k97_subband_total_bitplanes(options, subband) > 30 + { + return Err("9/7 code-block quantization options are outside supported range"); + } + } + + Ok((cb_width, cb_height)) +} + +fn checked_code_block_dim(exp_minus_two: u8) -> Result { + 1usize + .checked_shl(u32::from(exp_minus_two) + 2) + .ok_or("9/7 code-block dimension exponent is unsupported") +} + +fn quantize_subband_coefficients( + coefficients: &[f64], + sub_band_type: J2kSubBandType, + options: Htj2k97CodeBlockOptions, +) -> Vec { + let delta = htj2k97_subband_delta(options, sub_band_type); + let inv_delta = 1.0 / delta; + + coefficients + .iter() + .map(|&coefficient| { + // Deadzone quantization: q = sign(c) · floor(|c| · (1/Δ)), sign(0) = +1. + let sign = if coefficient < 0.0 { -1 } else { 1 }; + sign * (coefficient.abs() * inv_delta).floor() as i32 + }) + .collect() +} + +/// Shared `(exponent, mantissa)` for the irreversible 9/7 quantizer. +fn htj2k97_step(options: Htj2k97CodeBlockOptions, sub_band_type: J2kSubBandType) -> (u8, u16) { + let step = irreversible_quantization_step_for_subband( + options.bit_depth, + options.guard_bits, + options.irreversible_quantization_scale, + options.irreversible_quantization_subband_scales, + sub_band_type, + ); + (step.exponent, step.mantissa) +} + +fn pow2i_f64(exp: i32) -> f64 { + 2.0f64.powi(exp) +} + +fn htj2k97_code_block_dim(exp_minus_two: u8) -> usize { + 1usize + .checked_shl(u32::from(exp_minus_two) + 2) + .unwrap_or(usize::MAX) +} + +#[cfg(test)] +mod tests { + use super::*; + use j2k::adapter::encode_stage::{ + IrreversibleQuantizationSubbandScales, PrequantizedHtj2k97Image, + }; + use j2k_native::{ + encode_precomputed_htj2k_97, encode_prequantized_htj2k_97, EncodeOptions, + J2kForwardDwt97Level, J2kForwardDwt97Output, PrecomputedHtj2k97Component, + PrecomputedHtj2k97Image, + }; + + // Boundary-free coefficients on a 0.25 grid: exact in both f32 and f64, and + // every product with the scale-1.0 inverse deltas (4, 2, 1) lands on an exact + // integer/half-integer. So the f64 oracle and native's f32 quantizer agree + // bit-for-bit here and the codestream pin is exact, not merely close. + fn sample_band(len: usize, offset: f64) -> Vec { + (0..len) + .map(|idx| ((idx % 17) as f64 - 8.0) * 0.5 + offset) + .collect() + } + + #[test] + fn oracle_prequantized_component_matches_native_precomputed_codestream() { + let width = 17u32; + let height = 13u32; + let low_width = width.div_ceil(2) as usize; + let low_height = height.div_ceil(2) as usize; + let high_width = (width / 2) as usize; + let high_height = (height / 2) as usize; + + let ll = sample_band(low_width * low_height, 0.25); + let hl = sample_band(high_width * low_height, -0.75); + let lh = sample_band(low_width * high_height, 1.25); + let hh = sample_band(high_width * high_height, -1.5); + + let options = EncodeOptions { + num_decomposition_levels: 1, + reversible: false, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + ..EncodeOptions::default() + }; + + // Native precomputed-DWT path quantizes the f32 bands internally. + let precomputed_image = PrecomputedHtj2k97Image { + width, + height, + bit_depth: 8, + signed: false, + components: vec![PrecomputedHtj2k97Component { + x_rsiz: 1, + y_rsiz: 1, + dwt: J2kForwardDwt97Output { + ll: ll.iter().map(|&v| v as f32).collect(), + ll_width: low_width as u32, + ll_height: low_height as u32, + levels: vec![J2kForwardDwt97Level { + hl: hl.iter().map(|&v| v as f32).collect(), + lh: lh.iter().map(|&v| v as f32).collect(), + hh: hh.iter().map(|&v| v as f32).collect(), + width, + height, + low_width: low_width as u32, + low_height: low_height as u32, + high_width: high_width as u32, + high_height: high_height as u32, + }], + }, + }], + }; + + // Oracle prequantized path (f64) over the same bands. + let dwt = Dwt97TwoDimensional { + ll, + hl, + lh, + hh, + low_width, + low_height, + high_width, + high_height, + }; + let codeblock_options = Htj2k97CodeBlockOptions { + bit_depth: 8, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + irreversible_quantization_scale: 1.0, + irreversible_quantization_subband_scales: + IrreversibleQuantizationSubbandScales::default(), + }; + let component = prequantized_component_from_dwt97(&dwt, codeblock_options, 1, 1); + let prequantized_image = PrequantizedHtj2k97Image { + width, + height, + bit_depth: 8, + signed: false, + components: vec![component], + }; + + let expected = encode_precomputed_htj2k_97(&precomputed_image, &options) + .expect("native precomputed 9/7 encode"); + let native_prequantized_image = native_prequantized_image(prequantized_image); + let actual = encode_prequantized_htj2k_97(&native_prequantized_image, &options) + .expect("oracle prequantized 9/7 encode"); + + assert_eq!( + actual, expected, + "oracle prequantized component must reproduce the native precomputed-DWT codestream" + ); + } + + #[test] + fn shared_validator_accepts_standard_options_and_returns_dims() { + let options = Htj2k97CodeBlockOptions { + bit_depth: 8, + guard_bits: 2, + code_block_width_exp: 4, + code_block_height_exp: 4, + irreversible_quantization_scale: 1.0, + irreversible_quantization_subband_scales: + IrreversibleQuantizationSubbandScales::default(), + }; + assert_eq!(validate_htj2k97_codeblock_options(options), Ok((64, 64))); + } + + #[test] + fn shared_validator_rejects_out_of_spec_options_on_every_backend() { + let valid = Htj2k97CodeBlockOptions { + bit_depth: 8, + guard_bits: 2, + code_block_width_exp: 4, + code_block_height_exp: 4, + irreversible_quantization_scale: 1.0, + irreversible_quantization_subband_scales: + IrreversibleQuantizationSubbandScales::default(), + }; + + // Each case was accepted by the old Metal-only validator. + let oversized_bit_depth = Htj2k97CodeBlockOptions { + bit_depth: 31, + ..valid + }; + let oversized_guard_bits = Htj2k97CodeBlockOptions { + guard_bits: 31, + ..valid + }; + // 1024x1024: each side passes the per-side cap, area breaks the + // HTJ2K 4096 limit. + let oversized_area = Htj2k97CodeBlockOptions { + code_block_width_exp: 8, + code_block_height_exp: 8, + ..valid + }; + for options in [oversized_bit_depth, oversized_guard_bits, oversized_area] { + assert!( + validate_htj2k97_codeblock_options(options).is_err(), + "options must be rejected: {options:?}" + ); + } + + // guard_bits == 0 stays accepted (the old Metal validator rejected it, + // CUDA and the native encoder accept it). + let zero_guard_bits = Htj2k97CodeBlockOptions { + guard_bits: 0, + ..valid + }; + assert!(validate_htj2k97_codeblock_options(zero_guard_bits).is_ok()); + } + + #[test] + fn oracle_subband_profile_changes_only_selected_delta_and_bitplanes() { + let mut options = Htj2k97CodeBlockOptions { + bit_depth: 8, + guard_bits: 2, + code_block_width_exp: 2, + code_block_height_exp: 2, + irreversible_quantization_scale: 1.9, + irreversible_quantization_subband_scales: + IrreversibleQuantizationSubbandScales::default(), + }; + let high_low_delta = htj2k97_subband_delta(options, J2kSubBandType::HighLow); + let high_high_delta = htj2k97_subband_delta(options, J2kSubBandType::HighHigh); + let default_hh_bitplanes = + htj2k97_subband_total_bitplanes(options, J2kSubBandType::HighHigh); + + options.irreversible_quantization_subband_scales.high_high = 1.5; + + assert_eq!( + htj2k97_subband_delta(options, J2kSubBandType::HighLow).to_bits(), + high_low_delta.to_bits() + ); + assert!(htj2k97_subband_delta(options, J2kSubBandType::HighHigh) > high_high_delta); + assert_ne!( + htj2k97_subband_total_bitplanes(options, J2kSubBandType::HighHigh), + default_hh_bitplanes + ); + } + + fn native_prequantized_image( + image: PrequantizedHtj2k97Image, + ) -> j2k_native::PrequantizedHtj2k97Image { + j2k_native::PrequantizedHtj2k97Image { + width: image.width, + height: image.height, + bit_depth: image.bit_depth, + signed: image.signed, + components: image + .components + .into_iter() + .map(|component| j2k_native::PrequantizedHtj2k97Component { + x_rsiz: component.x_rsiz, + y_rsiz: component.y_rsiz, + resolutions: component + .resolutions + .into_iter() + .map(|resolution| j2k_native::PrequantizedHtj2k97Resolution { + subbands: resolution + .subbands + .into_iter() + .map(|subband| j2k_native::PrequantizedHtj2k97Subband { + sub_band_type: subband.sub_band_type, + num_cbs_x: subband.num_cbs_x, + num_cbs_y: subband.num_cbs_y, + total_bitplanes: subband.total_bitplanes, + code_blocks: subband + .code_blocks + .into_iter() + .map(|block| j2k_native::PrequantizedHtj2k97CodeBlock { + coefficients: block.coefficients, + width: block.width, + height: block.height, + }) + .collect(), + }) + .collect(), + }) + .collect(), + }) + .collect(), + } + } +} diff --git a/crates/j2k-transcode/src/jpeg_to_htj2k.rs b/crates/j2k-transcode/src/jpeg_to_htj2k.rs new file mode 100644 index 00000000..606144bf --- /dev/null +++ b/crates/j2k-transcode/src/jpeg_to_htj2k.rs @@ -0,0 +1,4454 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Experimental JPEG DCT to HTJ2K codestream transcode entry point. + +use core::fmt; +use std::time::Instant; + +use j2k::adapter::encode_stage::{ + CpuOnlyJ2kEncodeStageAccelerator, IrreversibleQuantizationSubbandScales, + J2kEncodeDispatchReport, J2kEncodeStageAccelerator, J2kForwardDwt53Level, + J2kForwardDwt53Output, J2kForwardDwt97Level, J2kForwardDwt97Output, NativeEncodeStageAdapter, + PrecomputedHtj2k53Component, PrecomputedHtj2k53Image, PrecomputedHtj2k97Component, + PrecomputedHtj2k97Image, PreencodedHtj2k97CompactComponent, PreencodedHtj2k97CompactImage, + PreencodedHtj2k97Component, PreencodedHtj2k97Image, PrequantizedHtj2k97Component, + PrequantizedHtj2k97Image, +}; +use j2k::J2kProgressionOrder; +use j2k_jpeg::transcode::{ + extract_dct_blocks, idct_islow_block, DctExtractOptions, JpegDctComponent, JpegDctImage, +}; +use j2k_native::{ + encode_precomputed_htj2k_53_with_accelerator, + encode_precomputed_htj2k_97_batch_with_accelerator, + encode_precomputed_htj2k_97_with_accelerator, + encode_preencoded_htj2k_97_compact_owned_with_accelerator, + encode_preencoded_htj2k_97_owned_with_accelerator, + encode_prequantized_htj2k_97_with_accelerator, +}; +use rayon::prelude::*; + +use crate::accelerator::{ + CpuOnlyDctToWaveletStageAccelerator, DctGridI16ToHtj2k97CodeBlockBatch, + DctGridI16ToHtj2k97CodeBlockJob, DctGridToDwt53Job, DctGridToDwt97Job, + DctGridToHtj2k97CodeBlockJob, DctGridToReversibleDwt53Job, DctToWaveletStageAccelerator, + Dwt97BatchStageTimings, Htj2k97CodeBlockOptions, ReversibleDwt53FirstLevel, + TranscodeStageError, +}; +use crate::dct53_2d::{ + dct8x8_blocks_then_dwt53_float, dct8x8_blocks_to_dwt53_float_linear_with_scratch, + linearized_53_2d_from_plane, Dct53GridScratch, Dwt53TwoDimensional, +}; +use crate::dct97_2d::{ + dct8x8_blocks_then_dwt97_float, dct8x8_blocks_then_dwt97_float_with_scratch, + linearized_97_2d_from_plane_with_scratch, Dct97GridScratch, Dwt97TwoDimensional, +}; +use crate::metrics::{error_metrics_i32, ErrorMetrics, MetricsLengthError}; +use crate::reversible53::{ + reversible_lift_53_high_at_fallible, reversible_lift_53_i32, reversible_lift_53_low_at_fallible, +}; +use crate::DctGridError; + +/// Default irreversible quantization multiplier for JPEG direct 9/7 HTJ2K. +/// +/// Empirically rate-match the explicit lossy comparison profile near the +/// external comparator output size on the bundled WSI tiles. Lower values +/// produce larger/higher-quality codestreams; `1.0` matches the native encoder +/// default but overshoots the external baseline size for this transcode path. +pub const JPEG_TO_HTJ2K_LOSSY_97_QUANTIZATION_SCALE: f32 = 1.9; + +/// HTJ2K encode options used after JPEG coefficient-domain wavelet bands are produced. +#[derive(Debug, Clone, PartialEq)] +#[allow(clippy::struct_excessive_bools)] +pub struct JpegToHtj2kEncodeOptions { + /// Number of wavelet decomposition levels. + pub num_decomposition_levels: u8, + /// Whether to emit reversible/lossless coding. + pub reversible: bool, + /// Code-block width exponent minus two. + pub code_block_width_exp: u8, + /// Code-block height exponent minus two. + pub code_block_height_exp: u8, + /// JPEG 2000 guard bits. + pub guard_bits: u8, + /// Whether to encode HTJ2K code blocks instead of classic EBCOT. + pub use_ht_block_coding: bool, + /// Packet progression order. + pub progression_order: J2kProgressionOrder, + /// Whether to write a TLM marker segment. + pub write_tlm: bool, + /// Whether to write PLT packet-length marker segments. + pub write_plt: bool, + /// Whether to write PLM packet-length marker segments. + pub write_plm: bool, + /// Whether to write SOP marker segments before packets. + pub write_sop: bool, + /// Whether to write EPH markers after packet headers. + pub write_eph: bool, + /// Whether to apply JPEG 2000 multi-component transform. + pub use_mct: bool, + /// Number of cumulative quality layers. + pub num_layers: u8, + /// Optional cumulative packet-body byte targets for each quality layer. + pub quality_layer_byte_targets: Vec, + /// Whether native HTJ2K validation is enabled after encode. + pub validate_high_throughput_codestream: bool, + /// Global irreversible 9/7 quantization scale. + pub irreversible_quantization_scale: f32, + /// Per-subband irreversible 9/7 quantization scales. + pub irreversible_quantization_subband_scales: IrreversibleQuantizationSubbandScales, + /// Optional per-component SIZ sampling factors (`XRsiz`, `YRsiz`). + pub component_sampling: Option>, + /// Optional tile size for multi-tile codestreams. + pub tile_size: Option<(u32, u32)>, + /// Optional precinct exponents in COD order. + pub precinct_exponents: Vec<(u8, u8)>, +} + +impl Default for JpegToHtj2kEncodeOptions { + fn default() -> Self { + Self { + num_decomposition_levels: 5, + reversible: true, + code_block_width_exp: 4, + code_block_height_exp: 4, + guard_bits: 1, + use_ht_block_coding: false, + progression_order: J2kProgressionOrder::Lrcp, + write_tlm: false, + write_plt: false, + write_plm: false, + write_sop: false, + write_eph: false, + use_mct: true, + num_layers: 1, + quality_layer_byte_targets: Vec::new(), + validate_high_throughput_codestream: true, + irreversible_quantization_scale: 1.0, + irreversible_quantization_subband_scales: + IrreversibleQuantizationSubbandScales::default(), + component_sampling: None, + tile_size: None, + precinct_exponents: Vec::new(), + } + } +} + +impl JpegToHtj2kEncodeOptions { + fn to_native(&self) -> j2k_native::EncodeOptions { + j2k_native::EncodeOptions { + num_decomposition_levels: self.num_decomposition_levels, + reversible: self.reversible, + code_block_width_exp: self.code_block_width_exp, + code_block_height_exp: self.code_block_height_exp, + guard_bits: self.guard_bits, + use_ht_block_coding: self.use_ht_block_coding, + progression_order: native_progression_order(self.progression_order), + write_tlm: self.write_tlm, + write_plt: self.write_plt, + write_plm: self.write_plm, + write_sop: self.write_sop, + write_eph: self.write_eph, + use_mct: self.use_mct, + num_layers: self.num_layers, + quality_layer_byte_targets: self.quality_layer_byte_targets.clone(), + validate_high_throughput_codestream: self.validate_high_throughput_codestream, + irreversible_quantization_scale: self.irreversible_quantization_scale, + irreversible_quantization_subband_scales: self.irreversible_quantization_subband_scales, + component_sampling: self.component_sampling.clone(), + tile_size: self.tile_size, + precinct_exponents: self.precinct_exponents.clone(), + } + } +} + +/// Options for the experimental JPEG-to-HTJ2K path. +#[derive(Debug, Clone)] +pub struct JpegToHtj2kOptions { + /// HTJ2K encode options used after wavelet bands are produced. + pub encode_options: JpegToHtj2kEncodeOptions, + /// Coefficient production path used for HTJ2K precomputed bands. + pub coefficient_path: JpegToHtj2kCoefficientPath, + /// Materialize the float IDCT-then-DWT oracle and report rounded + /// coefficient differences. This is intended for validation and tests, not + /// the production direct path. + pub validate_against_float_reference: bool, + /// Materialize j2k-jpeg scalar ISLOW samples and report reversible + /// integer 5/3 coefficient differences against the rounded direct path. + /// This is intended for validation and tests, not the production direct + /// path. + pub validate_against_integer_reference: bool, +} + +impl Default for JpegToHtj2kOptions { + fn default() -> Self { + Self::lossless_53() + } +} + +impl JpegToHtj2kOptions { + /// Options for the default reversible 5/3 HTJ2K coefficient path. + #[must_use] + pub fn lossless_53() -> Self { + Self { + encode_options: transcode_encode_options(true), + coefficient_path: JpegToHtj2kCoefficientPath::IntegerDirect53, + validate_against_float_reference: false, + validate_against_integer_reference: false, + } + } + + /// Options for the irreversible 9/7 HTJ2K float-linear coefficient path. + #[must_use] + pub fn lossy_97() -> Self { + let mut encode_options = transcode_encode_options(false); + encode_options.irreversible_quantization_scale = JPEG_TO_HTJ2K_LOSSY_97_QUANTIZATION_SCALE; + Self { + encode_options, + coefficient_path: JpegToHtj2kCoefficientPath::FloatDirectLinear97, + validate_against_float_reference: false, + validate_against_integer_reference: false, + } + } +} + +fn transcode_encode_options(reversible: bool) -> JpegToHtj2kEncodeOptions { + JpegToHtj2kEncodeOptions { + num_decomposition_levels: 1, + reversible, + use_ht_block_coding: true, + use_mct: false, + validate_high_throughput_codestream: false, + ..JpegToHtj2kEncodeOptions::default() + } +} + +/// Experimental production path used to generate HTJ2K wavelet coefficients. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JpegToHtj2kCoefficientPath { + /// Exact reversible 5/3 coefficients relative to `j2k-jpeg` scalar + /// ISLOW block decode semantics. The first 5/3 level is computed from DCT + /// blocks without materializing a full spatial image plane; later levels + /// recurse conventionally over the LL coefficient band. + IntegerDirect53, + /// Floating-point linear composition of IDCT and 5/3 analysis. This is the + /// linear math oracle path and remains useful for validating the direct + /// matrix composition, but it is not the integer reversible production + /// default. + FloatDirectLinear53, + /// Floating-point linear composition of IDCT and irreversible 9/7 + /// analysis. This is a lossy experimental path and must be paired with an + /// irreversible HTJ2K encode. + FloatDirectLinear97, +} + +/// Reusable experimental JPEG-to-HTJ2K transcoder state. +/// +/// Create one value per worker thread and reuse it across many tiles to keep +/// scratch buffers allocated between calls. The scalar math and output are the +/// same as [`jpeg_to_htj2k`]. +#[derive(Debug, Default)] +pub struct JpegToHtj2kTranscoder { + scratch: JpegToHtj2kScratch, +} + +impl JpegToHtj2kTranscoder { + /// Transcode a constrained baseline JPEG tile into HTJ2K using this + /// instance's reusable scratch buffers. + pub fn transcode( + &mut self, + bytes: &[u8], + options: &JpegToHtj2kOptions, + ) -> Result { + let mut accelerator = CpuOnlyDctToWaveletStageAccelerator; + self.transcode_with_accelerator(bytes, options, &mut accelerator) + } + + /// Transcode with an optional stage accelerator. + /// + /// Accelerators may handle direct DCT-grid projection stages and return + /// `None` for scalar fallback. Integer-direct 5/3 is offered in + /// same-geometry batches before falling back to per-component work. + pub fn transcode_with_accelerator( + &mut self, + bytes: &[u8], + options: &JpegToHtj2kOptions, + accelerator: &mut A, + ) -> Result { + let mut encode_accelerator = CpuOnlyJ2kEncodeStageAccelerator; + self.transcode_with_accelerators(bytes, options, accelerator, &mut encode_accelerator) + } + + /// Transcode with separate transform-stage and HTJ2K encode-stage + /// accelerators. + pub fn transcode_with_accelerators< + A: DctToWaveletStageAccelerator, + E: J2kEncodeStageAccelerator, + >( + &mut self, + bytes: &[u8], + options: &JpegToHtj2kOptions, + transform_accelerator: &mut A, + encode_accelerator: &mut E, + ) -> Result { + jpeg_to_htj2k_with_scratch( + bytes, + options, + &mut self.scratch, + transform_accelerator, + encode_accelerator, + ) + } + + /// Transcode many JPEG tiles, preserving per-tile failures in the returned + /// batch. Integer-direct 5/3 groups same-geometry components across tiles + /// before calling the accelerator. + pub fn transcode_batch( + &mut self, + tiles: &[JpegTileBatchInput<'_>], + options: &JpegToHtj2kOptions, + ) -> Result { + let mut accelerator = CpuOnlyDctToWaveletStageAccelerator; + self.transcode_batch_with_accelerator(tiles, options, &mut accelerator) + } + + /// Transcode many JPEG tiles with an optional stage accelerator. + pub fn transcode_batch_with_accelerator( + &mut self, + tiles: &[JpegTileBatchInput<'_>], + options: &JpegToHtj2kOptions, + accelerator: &mut A, + ) -> Result { + let mut encode_accelerator = CpuOnlyJ2kEncodeStageAccelerator; + self.transcode_batch_with_accelerators(tiles, options, accelerator, &mut encode_accelerator) + } + + /// Transcode many JPEG tiles with separate transform-stage and HTJ2K + /// encode-stage accelerators. + pub fn transcode_batch_with_accelerators< + A: DctToWaveletStageAccelerator, + E: J2kEncodeStageAccelerator, + >( + &mut self, + tiles: &[JpegTileBatchInput<'_>], + options: &JpegToHtj2kOptions, + transform_accelerator: &mut A, + encode_accelerator: &mut E, + ) -> Result { + jpeg_tile_batch_to_htj2k_with_scratch( + tiles, + options, + &mut self.scratch, + transform_accelerator, + encode_accelerator, + ) + } + + /// Current capacity of the reusable DCT block conversion scratch. + /// + /// This is exposed for benchmark and validation harnesses while the API is + /// experimental. + #[must_use] + pub fn dct_block_scratch_capacity(&self) -> usize { + self.scratch.dct_blocks_f64.capacity() + } + + /// Current capacity of the reusable integer block-local IDCT sample cache. + /// + /// This cache stores level-shifted 8x8 block samples for the integer-direct + /// path. It is block-local scratch, not a full spatial image plane. + #[must_use] + pub fn integer_idct_block_scratch_capacity(&self) -> usize { + self.scratch.integer_idct_blocks.capacity() + } +} + +#[derive(Debug, Default)] +struct JpegToHtj2kScratch { + dct_blocks_f64: Vec<[[f64; 8]; 8]>, + dct53_grid: Dct53GridScratch, + dct97_grid: Dct97GridScratch, + integer_idct_blocks: Vec>, + integer_row: Vec, +} + +/// Encoded transcode output and validation/report metadata. +#[derive(Debug, Clone)] +pub struct EncodedTranscode { + /// HTJ2K codestream bytes. + pub codestream: Vec, + /// Summary of the experimental path used. + pub report: TranscodeReport, +} + +/// One JPEG tile input for batch transcode. +#[derive(Debug, Clone, Copy)] +pub struct JpegTileBatchInput<'a> { + /// JPEG codestream bytes for one tile. + pub bytes: &'a [u8], +} + +/// Batch transcode output. Tile-level parse/encode failures are preserved so a +/// WSI ingest queue can continue past isolated bad tiles. +#[derive(Debug)] +pub struct EncodedTranscodeBatch { + /// Per-input tile result in input order. + pub tiles: Vec>, + /// Aggregate batch report. + pub report: BatchTranscodeReport, +} + +/// Aggregate report for multi-tile transcode. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BatchTranscodeReport { + /// Number of input tiles. + pub tile_count: usize, + /// Number of successfully encoded output tiles. + pub successful_tiles: usize, + /// Number of tile-local failures. + pub failed_tiles: usize, + /// Number of transformed components across successful extracted tiles. + pub transformed_components: usize, + /// Number of same-geometry reversible 5/3 batches submitted. + pub reversible_dwt53_batches: usize, + /// Number of reversible 5/3 component jobs in submitted batches. + pub reversible_dwt53_batch_jobs: usize, + /// Batch extraction time in microseconds. + pub extract_us: u128, + /// Batch DCT-to-wavelet time in microseconds. + pub transform_us: u128, + /// Batch HTJ2K encode time in microseconds. + pub encode_us: u128, + /// Detailed stage timings for the batch. Batch-accelerated 5/3 transform + /// timings stay here instead of being copied into every tile report. + pub timings: TranscodeTimingReport, + /// Coefficient path used by the batch. + pub coefficient_path: JpegToHtj2kCoefficientPath, +} + +/// Detailed timing and dispatch counters for JPEG-to-HTJ2K transcode. +/// +/// Durations are wall-clock microseconds measured around the current Rust API +/// boundaries. Accelerator time includes backend submission and wait overhead +/// visible to this crate; backend-specific hardware counters are not exposed +/// here. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TranscodeTimingReport { + /// Raw compressed-tile probe/read time before JPEG DCT extraction. + pub source_raw_probe_us: u128, + /// Source region decode time for strip/retile workflows. + pub read_region_decode_us: u128, + /// Region compose/pad time for generated regular tiles. + pub compose_pad_us: u128, + /// JPEG encode time when the workflow generates regular JPEG tiles. + pub generated_jpeg_encode_us: u128, + /// JPEG DCT extraction time in microseconds. + pub jpeg_dct_extract_us: u128, + /// Time spent repacking integer DCT coefficients into float block grids. + pub jpeg_dct_repack_us: u128, + /// Total wall time spent producing DWT bands from JPEG DCT coefficients. + pub dct_to_wavelet_total_us: u128, + /// Wall time spent inside accelerator hook calls. + pub dct_to_wavelet_accelerator_us: u128, + /// Wall time spent in scalar CPU fallback transforms. + pub dct_to_wavelet_cpu_fallback_us: u128, + /// Time spent decomposing first-level DWT output into requested levels. + pub dwt_decompose_us: u128, + /// Backend 9/7 batch host pack/upload time in microseconds. + pub dwt97_batch_pack_upload_us: u128, + /// Backend 9/7 batch IDCT plus horizontal row-lift time in microseconds. + pub dwt97_batch_idct_row_lift_us: u128, + /// Backend 9/7 batch vertical column-lift time in microseconds. + pub dwt97_batch_column_lift_us: u128, + /// Backend 9/7 batch quantize/code-block layout time in microseconds. + pub dwt97_batch_quantize_codeblock_us: u128, + /// Backend 9/7 resident HT code-block encode time in microseconds. + pub dwt97_batch_ht_encode_us: u128, + /// Backend 9/7 resident HT cleanup-pass encode kernel time in microseconds. + pub dwt97_batch_ht_kernel_us: u128, + /// Backend 9/7 resident HT status-buffer device-to-host readback time in microseconds. + pub dwt97_batch_ht_status_readback_us: u128, + /// Backend 9/7 resident HT encoded-byte compaction kernel time in microseconds. + pub dwt97_batch_ht_compact_us: u128, + /// Backend 9/7 resident HT compacted encoded-byte device-to-host readback time in microseconds. + pub dwt97_batch_ht_output_readback_us: u128, + /// Backend 9/7 resident HT code-block encode dispatches. + pub dwt97_batch_ht_codeblock_dispatches: usize, + /// Backend 9/7 batch output readback/unpack time in microseconds. + pub dwt97_batch_readback_us: u128, + /// HTJ2K encode time in microseconds. + pub htj2k_encode_us: u128, + /// Encode-stage accelerator dispatches during HTJ2K encode. + pub htj2k_encode_accelerator_dispatches: usize, + /// HT cleanup code-block accelerator dispatches during HTJ2K encode. + pub htj2k_encode_ht_code_block_dispatches: usize, + /// Packetization accelerator dispatches during HTJ2K encode. + pub htj2k_encode_packetization_dispatches: usize, + /// Time spent writing compressed frames to a DICOM `PixelData` spool. + pub dicom_spool_write_us: u128, + /// Time spent writing final DICOM instances. + pub dicom_final_write_us: u128, + /// Number of source tiles represented by this timing report. + pub tile_count: usize, + /// Number of components transformed into wavelet bands. + pub component_count: usize, + /// Number of same-geometry transform batches offered to the accelerator. + pub batch_count: usize, + /// Number of component jobs in same-geometry transform batches. + pub batch_jobs: usize, + /// Number of accelerator hook calls. + pub accelerator_attempts: usize, + /// Number of component jobs offered through accelerator hook calls. + pub accelerator_jobs: usize, + /// Number of accelerator hook calls that returned an accelerated result. + pub accelerator_dispatches: usize, + /// Number of component jobs completed by accelerated results. + pub accelerator_dispatched_jobs: usize, + /// Number of component jobs completed by scalar CPU fallback transforms. + pub cpu_fallback_jobs: usize, +} + +impl TranscodeTimingReport { + fn add_assign(&mut self, other: Self) { + macro_rules! saturating_add_fields { + ($($field:ident),+ $(,)?) => { + $( + self.$field = self.$field.saturating_add(other.$field); + )+ + }; + } + + saturating_add_fields!( + source_raw_probe_us, + read_region_decode_us, + compose_pad_us, + generated_jpeg_encode_us, + jpeg_dct_extract_us, + jpeg_dct_repack_us, + dct_to_wavelet_total_us, + dct_to_wavelet_accelerator_us, + dct_to_wavelet_cpu_fallback_us, + dwt_decompose_us, + dwt97_batch_pack_upload_us, + dwt97_batch_idct_row_lift_us, + dwt97_batch_column_lift_us, + dwt97_batch_quantize_codeblock_us, + dwt97_batch_ht_encode_us, + dwt97_batch_ht_kernel_us, + dwt97_batch_ht_status_readback_us, + dwt97_batch_ht_compact_us, + dwt97_batch_ht_output_readback_us, + dwt97_batch_ht_codeblock_dispatches, + dwt97_batch_readback_us, + htj2k_encode_us, + htj2k_encode_accelerator_dispatches, + htj2k_encode_ht_code_block_dispatches, + htj2k_encode_packetization_dispatches, + dicom_spool_write_us, + dicom_final_write_us, + tile_count, + component_count, + batch_count, + batch_jobs, + accelerator_attempts, + accelerator_jobs, + accelerator_dispatches, + accelerator_dispatched_jobs, + cpu_fallback_jobs, + ); + } +} + +/// Per-component transcode geometry preserved in the generated codestream. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TranscodeComponentReport { + /// Component index in JPEG SOF order. + pub component_index: usize, + /// Native component width in samples before HTJ2K SIZ expansion. + pub width: u32, + /// Native component height in samples before HTJ2K SIZ expansion. + pub height: u32, + /// Number of DCT blocks per component row, including padded edge blocks. + pub block_cols: u32, + /// Number of DCT block rows, including padded edge blocks. + pub block_rows: u32, + /// HTJ2K SIZ horizontal sampling factor. + pub x_rsiz: u8, + /// HTJ2K SIZ vertical sampling factor. + pub y_rsiz: u8, +} + +/// Error metrics from an optional validation oracle. +pub type TranscodeValidationMetrics = ErrorMetrics; + +/// Classification for optional coefficient-validation metrics. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TranscodeValidationClassification { + /// All compared coefficients match the selected oracle exactly. + Exact, + /// Coefficients satisfy the experimental one-LSB-bounded threshold: + /// maximum absolute error is at most one LSB and at least 99.9% of + /// coefficients match exactly. + OneLsbBounded, + /// Coefficients do not satisfy the exact or one-LSB-bounded thresholds. + OutsideThreshold, +} + +impl TranscodeValidationClassification { + /// Classify validation metrics using the experimental acceptance + /// thresholds documented for this coefficient-domain path. + #[must_use] + pub fn classify_metrics(metrics: &TranscodeValidationMetrics) -> Self { + if metrics.exact_matches == metrics.total && metrics.max_abs_error == 0 { + Self::Exact + } else if metrics.is_one_lsb_bounded(0.999) { + Self::OneLsbBounded + } else { + Self::OutsideThreshold + } + } +} + +/// Transcode summary for validation and benchmarking. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TranscodeReport { + /// Source reference-grid width. + pub width: u32, + /// Source reference-grid height. + pub height: u32, + /// Number of transformed components. + pub component_count: usize, + /// Native transformed component geometry and SIZ sampling. + pub components: Vec, + /// Rounded coefficient metrics against the optional float IDCT-then-DWT + /// oracle. + pub float_reference_metrics: Option, + /// Threshold classification for `float_reference_metrics`. + pub float_reference_classification: Option, + /// Rounded direct coefficients compared with j2k-jpeg scalar + /// ISLOW-IDCT-then-reversible-5/3 coefficients. + pub integer_reference_metrics: Option, + /// Threshold classification for `integer_reference_metrics`. + pub integer_reference_classification: Option, + /// Number of DWT decomposition levels encoded. + pub decomposition_levels: u8, + /// Coefficient path used to generate the HTJ2K bands. + pub coefficient_path: JpegToHtj2kCoefficientPath, + /// Name of the experimental path used. + pub path: &'static str, + /// Wall-clock extraction time in microseconds. + pub extract_us: u128, + /// Wall-clock DCT-to-wavelet time in microseconds. + pub transform_us: u128, + /// Wall-clock HTJ2K encode time in microseconds. + pub encode_us: u128, + /// Detailed stage timings and accelerator/fallback counters. + pub timings: TranscodeTimingReport, +} + +/// Error returned by the experimental transcode path. +#[derive(Debug)] +pub enum JpegToHtj2kError { + /// JPEG parse or entropy decode failed. + Jpeg(j2k_jpeg::JpegError), + /// Input is outside the currently implemented experimental slice. + Unsupported(&'static str), + /// DCT block grid metadata did not cover the component dimensions. + Grid(String), + /// DCT block grid metadata did not cover the component dimensions for the + /// 9/7 path. + Grid97(String), + /// Optional transform acceleration failed. + Accelerator(TranscodeStageError), + /// Validation metric inputs were inconsistent. + Metrics(String), + /// Validation encountered an out-of-range or non-finite coefficient. + Validation(&'static str), + /// Native HTJ2K encode failed. + Encode(&'static str), +} + +impl fmt::Display for JpegToHtj2kError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Jpeg(err) => write!(f, "JPEG extraction failed: {err}"), + Self::Unsupported(reason) => write!(f, "unsupported transcode input: {reason}"), + Self::Grid(reason) | Self::Grid97(reason) => { + write!(f, "DCT grid transform failed: {reason}") + } + Self::Accelerator(reason) => write!(f, "transform accelerator failed: {reason}"), + Self::Metrics(reason) => write!(f, "validation metrics failed: {reason}"), + Self::Validation(reason) => write!(f, "validation failed: {reason}"), + Self::Encode(reason) => write!(f, "HTJ2K encode failed: {reason}"), + } + } +} + +impl std::error::Error for JpegToHtj2kError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Jpeg(err) => Some(err), + Self::Unsupported(_) + | Self::Grid(_) + | Self::Grid97(_) + | Self::Accelerator(_) + | Self::Metrics(_) + | Self::Validation(_) + | Self::Encode(_) => None, + } + } +} + +impl From for JpegToHtj2kError { + fn from(value: j2k_jpeg::JpegError) -> Self { + Self::Jpeg(value) + } +} + +fn dct53_grid_error(value: DctGridError) -> JpegToHtj2kError { + JpegToHtj2kError::Grid(value.to_string()) +} + +fn dct97_grid_error(value: DctGridError) -> JpegToHtj2kError { + JpegToHtj2kError::Grid97(value.to_string()) +} + +impl From for JpegToHtj2kError { + fn from(value: MetricsLengthError) -> Self { + Self::Metrics(value.to_string()) + } +} + +/// Transcode a constrained baseline grayscale JPEG tile into an HTJ2K +/// codestream using direct DCT-domain wavelet coefficients. +/// +/// Current implementation scope is baseline JPEG with one or more components +/// at native JPEG component resolution. Component subsampling is preserved +/// through SIZ `XRsiz`/`YRsiz` instead of chroma upsampling. +pub fn jpeg_to_htj2k( + bytes: &[u8], + options: &JpegToHtj2kOptions, +) -> Result { + JpegToHtj2kTranscoder::default().transcode(bytes, options) +} + +/// Transcode many JPEG tiles into HTJ2K codestreams. +pub fn jpeg_to_htj2k_batch( + tiles: &[JpegTileBatchInput<'_>], + options: &JpegToHtj2kOptions, +) -> Result { + JpegToHtj2kTranscoder::default().transcode_batch(tiles, options) +} + +fn jpeg_tile_batch_to_htj2k_with_scratch< + A: DctToWaveletStageAccelerator, + E: J2kEncodeStageAccelerator, +>( + tiles: &[JpegTileBatchInput<'_>], + options: &JpegToHtj2kOptions, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut A, + encode_accelerator: &mut E, +) -> Result { + validate_transcode_options(options)?; + match options.coefficient_path { + JpegToHtj2kCoefficientPath::IntegerDirect53 => {} + JpegToHtj2kCoefficientPath::FloatDirectLinear97 + if accelerator.supports_dwt97_batch() + || accelerator.supports_htj2k97_codeblock_batch() => + { + return jpeg_float97_tile_batch_to_htj2k_with_scratch( + tiles, + options, + scratch, + accelerator, + encode_accelerator, + ); + } + JpegToHtj2kCoefficientPath::FloatDirectLinear53 + | JpegToHtj2kCoefficientPath::FloatDirectLinear97 => { + return Ok(transcode_tile_batch_individually( + tiles, + options, + scratch, + accelerator, + encode_accelerator, + )); + } + } + + let extract_start = Instant::now(); + let prepared_results = tiles + .par_iter() + .enumerate() + .map(|(tile_index, tile)| { + ( + tile_index, + prepare_integer_batch_tile(tile_index, tile.bytes, options), + ) + }) + .collect::>(); + let extract_us = extract_start.elapsed().as_micros(); + let mut tile_results: Vec>> = + (0..tiles.len()).map(|_| None).collect(); + let mut prepared_tiles = Vec::new(); + for (tile_index, result) in prepared_results { + match result { + Ok(prepared) => prepared_tiles.push(prepared), + Err(error) => tile_results[tile_index] = Some(Err(error)), + } + } + + let transform_start = Instant::now(); + let mut timings = TranscodeTimingReport::default(); + let (reversible_dwt53_batches, reversible_dwt53_batch_jobs) = transform_integer_batch_tiles( + &mut prepared_tiles, + options, + scratch, + accelerator, + &mut timings, + )?; + let transform_us = transform_start.elapsed().as_micros(); + timings.jpeg_dct_extract_us = extract_us; + timings.dct_to_wavelet_total_us = transform_us; + timings.tile_count = prepared_tiles.len(); + + let encode_start = Instant::now(); + let encoded_tiles = encode_integer_prepared_tiles(prepared_tiles, options, encode_accelerator); + for (tile_index, encoded) in encoded_tiles { + add_encode_timing_counters_from_result(&mut timings, &encoded); + tile_results[tile_index] = Some(encoded); + } + let encode_us = encode_start.elapsed().as_micros(); + timings.htj2k_encode_us = encode_us; + + let output_tiles = tile_results + .into_iter() + .map(|tile| { + tile.unwrap_or(Err(JpegToHtj2kError::Validation( + "batch transcode did not produce a tile result", + ))) + }) + .collect::>(); + Ok(batch_output( + output_tiles, + BatchTranscodeReport { + tile_count: tiles.len(), + successful_tiles: 0, + failed_tiles: 0, + transformed_components: reversible_dwt53_batch_jobs, + reversible_dwt53_batches, + reversible_dwt53_batch_jobs, + extract_us, + transform_us, + encode_us, + timings, + coefficient_path: options.coefficient_path, + }, + )) +} + +fn jpeg_float97_tile_batch_to_htj2k_with_scratch< + A: DctToWaveletStageAccelerator, + E: J2kEncodeStageAccelerator, +>( + tiles: &[JpegTileBatchInput<'_>], + options: &JpegToHtj2kOptions, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut A, + encode_accelerator: &mut E, +) -> Result { + let extract_start = Instant::now(); + let prepared_results = tiles + .par_iter() + .enumerate() + .map(|(tile_index, tile)| { + ( + tile_index, + prepare_float97_batch_tile(tile_index, tile.bytes, options), + ) + }) + .collect::>(); + let extract_us = extract_start.elapsed().as_micros(); + let mut tile_results: Vec>> = + (0..tiles.len()).map(|_| None).collect(); + let mut prepared_tiles = Vec::new(); + for (tile_index, result) in prepared_results { + match result { + Ok(prepared) => prepared_tiles.push(prepared), + Err(error) => tile_results[tile_index] = Some(Err(error)), + } + } + + let transform_start = Instant::now(); + let mut timings = TranscodeTimingReport::default(); + let (_dwt97_batches, dwt97_batch_jobs) = transform_float97_batch_tiles( + &mut prepared_tiles, + options, + scratch, + accelerator, + &mut timings, + )?; + let transform_us = transform_start.elapsed().as_micros(); + timings.jpeg_dct_extract_us = extract_us; + timings.dct_to_wavelet_total_us = transform_us; + timings.tile_count = prepared_tiles.len(); + + let encode_start = Instant::now(); + let encoded_tiles = encode_float97_prepared_tiles(prepared_tiles, options, encode_accelerator); + for (tile_index, encoded) in encoded_tiles { + add_encode_timing_counters_from_result(&mut timings, &encoded); + tile_results[tile_index] = Some(encoded); + } + let encode_us = encode_start.elapsed().as_micros(); + timings.htj2k_encode_us = encode_us; + + let output_tiles = tile_results + .into_iter() + .map(|tile| { + tile.unwrap_or(Err(JpegToHtj2kError::Validation( + "9/7 batch transcode did not produce a tile result", + ))) + }) + .collect::>(); + Ok(batch_output( + output_tiles, + BatchTranscodeReport { + tile_count: tiles.len(), + successful_tiles: 0, + failed_tiles: 0, + transformed_components: dwt97_batch_jobs, + reversible_dwt53_batches: 0, + reversible_dwt53_batch_jobs: 0, + extract_us, + transform_us, + encode_us, + timings, + coefficient_path: options.coefficient_path, + }, + )) +} + +fn transcode_tile_batch_individually< + A: DctToWaveletStageAccelerator, + E: J2kEncodeStageAccelerator, +>( + tiles: &[JpegTileBatchInput<'_>], + options: &JpegToHtj2kOptions, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut A, + encode_accelerator: &mut E, +) -> EncodedTranscodeBatch { + let start = Instant::now(); + let output_tiles = tiles + .iter() + .map(|tile| { + jpeg_to_htj2k_with_scratch( + tile.bytes, + options, + scratch, + accelerator, + encode_accelerator, + ) + }) + .collect::>(); + let mut timings = aggregate_tile_timings(&output_tiles); + timings.tile_count = output_tiles.iter().filter(|tile| tile.is_ok()).count(); + let elapsed_us = start.elapsed().as_micros(); + if timings.dct_to_wavelet_total_us == 0 { + timings.dct_to_wavelet_total_us = elapsed_us + .saturating_sub(timings.jpeg_dct_extract_us) + .saturating_sub(timings.htj2k_encode_us); + } + batch_output( + output_tiles, + BatchTranscodeReport { + tile_count: tiles.len(), + successful_tiles: 0, + failed_tiles: 0, + transformed_components: timings.component_count, + reversible_dwt53_batches: 0, + reversible_dwt53_batch_jobs: 0, + extract_us: timings.jpeg_dct_extract_us, + transform_us: timings.dct_to_wavelet_total_us, + encode_us: timings.htj2k_encode_us, + timings, + coefficient_path: options.coefficient_path, + }, + ) +} + +fn aggregate_tile_timings( + tiles: &[Result], +) -> TranscodeTimingReport { + let mut timings = TranscodeTimingReport::default(); + for tile in tiles.iter().filter_map(|tile| tile.as_ref().ok()) { + timings.add_assign(tile.report.timings); + } + timings +} + +fn batch_output( + tiles: Vec>, + mut report: BatchTranscodeReport, +) -> EncodedTranscodeBatch { + report.successful_tiles = tiles.iter().filter(|tile| tile.is_ok()).count(); + report.failed_tiles = tiles.len().saturating_sub(report.successful_tiles); + EncodedTranscodeBatch { tiles, report } +} + +struct IntegerBatchTile { + tile_index: usize, + jpeg: JpegDctImage, + component_sampling: Vec<(u8, u8)>, + decomposition_levels: u8, + all_unit_sampled: bool, + component_reports: Vec, + precomputed_components: Vec>, + float_validation_actual: Vec, + float_validation_expected: Vec, + integer_validation_actual: Vec, + integer_validation_expected: Vec, + timings: TranscodeTimingReport, +} + +struct Float97BatchTile { + tile_index: usize, + jpeg: JpegDctImage, + component_sampling: Vec<(u8, u8)>, + decomposition_levels: u8, + all_unit_sampled: bool, + component_reports: Vec, + precomputed_components: Vec>, + preencoded_compact_payload: Vec, + preencoded_compact_components: Vec>, + preencoded_components: Vec>, + prequantized_components: Vec>, + float_validation_actual: Vec, + float_validation_expected: Vec, + timings: TranscodeTimingReport, +} + +struct Float97PrecomputedBatchRecord { + tile_index: usize, + jpeg: JpegDctImage, + decomposition_levels: u8, + all_unit_sampled: bool, + component_reports: Vec, + float_validation_actual: Vec, + float_validation_expected: Vec, + timings: TranscodeTimingReport, +} + +#[derive(Clone, Copy)] +struct BatchComponentRef { + tile_index: usize, + component_index: usize, +} + +fn prepare_integer_batch_tile( + tile_index: usize, + bytes: &[u8], + options: &JpegToHtj2kOptions, +) -> Result { + let extract_start = Instant::now(); + let jpeg = extract_dct_blocks(bytes, DctExtractOptions::default())?; + let timings = TranscodeTimingReport { + jpeg_dct_extract_us: extract_start.elapsed().as_micros(), + tile_count: 1, + ..TranscodeTimingReport::default() + }; + if jpeg.components.is_empty() || jpeg.components.len() > 4 { + return Err(JpegToHtj2kError::Unsupported( + "unsupported JPEG component count for jpeg_to_htj2k", + )); + } + let component_sampling = + component_sampling_for_jpeg(&jpeg.components, jpeg.width, jpeg.height)?; + let decomposition_levels = decomposition_levels_for_components( + &jpeg.components, + options.encode_options.num_decomposition_levels, + )?; + let all_unit_sampled = component_sampling + .iter() + .all(|&(x_rsiz, y_rsiz)| x_rsiz == 1 && y_rsiz == 1); + let component_reports = jpeg + .components + .iter() + .zip(component_sampling.iter().copied()) + .map(|(component, (x_rsiz, y_rsiz))| TranscodeComponentReport { + component_index: component.component_index, + width: component.width, + height: component.height, + block_cols: component.block_cols, + block_rows: component.block_rows, + x_rsiz, + y_rsiz, + }) + .collect::>(); + let precomputed_components = (0..jpeg.components.len()).map(|_| None).collect(); + + Ok(IntegerBatchTile { + tile_index, + jpeg, + component_sampling, + decomposition_levels, + all_unit_sampled, + component_reports, + precomputed_components, + float_validation_actual: Vec::new(), + float_validation_expected: Vec::new(), + integer_validation_actual: Vec::new(), + integer_validation_expected: Vec::new(), + timings, + }) +} + +fn prepare_float97_batch_tile( + tile_index: usize, + bytes: &[u8], + options: &JpegToHtj2kOptions, +) -> Result { + let extract_start = Instant::now(); + let jpeg = extract_dct_blocks(bytes, DctExtractOptions::dequantized_only())?; + let timings = TranscodeTimingReport { + jpeg_dct_extract_us: extract_start.elapsed().as_micros(), + tile_count: 1, + ..TranscodeTimingReport::default() + }; + if jpeg.components.is_empty() || jpeg.components.len() > 4 { + return Err(JpegToHtj2kError::Unsupported( + "unsupported JPEG component count for jpeg_to_htj2k", + )); + } + let component_sampling = + component_sampling_for_jpeg(&jpeg.components, jpeg.width, jpeg.height)?; + let decomposition_levels = decomposition_levels_for_components( + &jpeg.components, + options.encode_options.num_decomposition_levels, + )?; + let all_unit_sampled = component_sampling + .iter() + .all(|&(x_rsiz, y_rsiz)| x_rsiz == 1 && y_rsiz == 1); + let component_reports = jpeg + .components + .iter() + .zip(component_sampling.iter().copied()) + .map(|(component, (x_rsiz, y_rsiz))| TranscodeComponentReport { + component_index: component.component_index, + width: component.width, + height: component.height, + block_cols: component.block_cols, + block_rows: component.block_rows, + x_rsiz, + y_rsiz, + }) + .collect::>(); + let precomputed_components = (0..jpeg.components.len()).map(|_| None).collect(); + let preencoded_compact_components = (0..jpeg.components.len()).map(|_| None).collect(); + let preencoded_components = (0..jpeg.components.len()).map(|_| None).collect(); + let prequantized_components = (0..jpeg.components.len()).map(|_| None).collect(); + + Ok(Float97BatchTile { + tile_index, + jpeg, + component_sampling, + decomposition_levels, + all_unit_sampled, + component_reports, + precomputed_components, + preencoded_compact_payload: Vec::new(), + preencoded_compact_components, + preencoded_components, + prequantized_components, + float_validation_actual: Vec::new(), + float_validation_expected: Vec::new(), + timings, + }) +} + +fn transform_integer_batch_tiles( + tiles: &mut [IntegerBatchTile], + options: &JpegToHtj2kOptions, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut A, + timings: &mut TranscodeTimingReport, +) -> Result<(usize, usize), JpegToHtj2kError> { + let groups = batch_component_groups(tiles); + let mut batch_count = 0usize; + let mut job_count = 0usize; + + for group in groups { + batch_count = batch_count.saturating_add(1); + job_count = job_count.saturating_add(group.len()); + let wavelets = + integer_wavelets_for_batch_group(&group, tiles, scratch, accelerator, timings)?; + for (component_ref, wavelet) in group.into_iter().zip(wavelets) { + store_integer_batch_wavelet(component_ref, &wavelet, tiles, options, scratch)?; + } + } + + Ok((batch_count, job_count)) +} + +fn transform_float97_batch_tiles( + tiles: &mut [Float97BatchTile], + options: &JpegToHtj2kOptions, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut A, + timings: &mut TranscodeTimingReport, +) -> Result<(usize, usize), JpegToHtj2kError> { + let groups = float97_batch_component_groups(tiles); + let grouped_i16_preencoded = try_store_grouped_i16_preencoded_float97_batches( + &groups, + tiles, + options, + accelerator, + timings, + )?; + let mut batch_count = 0usize; + let mut job_count = 0usize; + + for (group_index, group) in groups.into_iter().enumerate() { + batch_count = batch_count.saturating_add(1); + job_count = job_count.saturating_add(group.len()); + if grouped_i16_preencoded + .get(group_index) + .copied() + .unwrap_or(false) + { + continue; + } + if try_store_prequantized_float97_batch_group(&group, tiles, options, accelerator, timings)? + { + continue; + } + let wavelets = + float97_wavelets_for_batch_group(&group, tiles, scratch, accelerator, timings)?; + for (component_ref, wavelet) in group.into_iter().zip(wavelets) { + store_float97_batch_wavelet(component_ref, &wavelet, tiles, options, scratch)?; + } + } + + Ok((batch_count, job_count)) +} + +fn batch_component_groups(tiles: &[IntegerBatchTile]) -> Vec> { + let mut groups: Vec> = Vec::new(); + + for (tile_index, tile) in tiles.iter().enumerate() { + for (component_index, component) in tile.jpeg.components.iter().enumerate() { + let component_ref = BatchComponentRef { + tile_index, + component_index, + }; + if let Some(group) = groups.iter_mut().find(|group| { + let first = group[0]; + same_batch_component_key( + &tiles[first.tile_index], + first.component_index, + tile, + component_index, + ) + }) { + group.push(component_ref); + } else { + let _ = component; + groups.push(vec![component_ref]); + } + } + } + + groups +} + +fn float97_batch_component_groups(tiles: &[Float97BatchTile]) -> Vec> { + let mut groups: Vec> = Vec::new(); + + for (tile_index, tile) in tiles.iter().enumerate() { + for component_index in 0..tile.jpeg.components.len() { + let component_ref = BatchComponentRef { + tile_index, + component_index, + }; + if let Some(group) = groups.iter_mut().find(|group| { + let first = group[0]; + same_float97_batch_component_key( + &tiles[first.tile_index], + first.component_index, + tile, + component_index, + ) + }) { + group.push(component_ref); + } else { + groups.push(vec![component_ref]); + } + } + } + + groups +} + +fn same_batch_component_key( + left_tile: &IntegerBatchTile, + left_component_index: usize, + right_tile: &IntegerBatchTile, + right_component_index: usize, +) -> bool { + let left = &left_tile.jpeg.components[left_component_index]; + let right = &right_tile.jpeg.components[right_component_index]; + left.component_index == right.component_index + && left.width == right.width + && left.height == right.height + && left.block_cols == right.block_cols + && left.block_rows == right.block_rows + && left_tile.component_sampling[left_component_index] + == right_tile.component_sampling[right_component_index] +} + +fn same_float97_batch_component_key( + left_tile: &Float97BatchTile, + left_component_index: usize, + right_tile: &Float97BatchTile, + right_component_index: usize, +) -> bool { + let left = &left_tile.jpeg.components[left_component_index]; + let right = &right_tile.jpeg.components[right_component_index]; + left.width == right.width + && left.height == right.height + && left.block_cols == right.block_cols + && left.block_rows == right.block_rows + && left_tile.component_sampling[left_component_index] + == right_tile.component_sampling[right_component_index] +} + +fn integer_wavelets_for_batch_group( + group: &[BatchComponentRef], + tiles: &[IntegerBatchTile], + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut A, + timings: &mut TranscodeTimingReport, +) -> Result, JpegToHtj2kError> { + let jobs = group + .iter() + .map(|component_ref| { + integer_dct_job_for_component( + &tiles[component_ref.tile_index].jpeg.components[component_ref.component_index], + ) + }) + .collect::, _>>()?; + record_batch_attempt(timings, group.len()); + let accelerator_start = Instant::now(); + let accelerated = accelerator + .dct_grid_to_reversible_dwt53_batch(&jobs) + .map_err(JpegToHtj2kError::Accelerator)?; + timings.dct_to_wavelet_accelerator_us = timings + .dct_to_wavelet_accelerator_us + .saturating_add(accelerator_start.elapsed().as_micros()); + + if let Some(first_levels) = accelerated { + if first_levels.len() != group.len() { + return Err(JpegToHtj2kError::Validation( + "reversible 5/3 batch accelerator returned wrong component count", + )); + } + timings.component_count = timings.component_count.saturating_add(group.len()); + record_accelerator_dispatch(timings, group.len()); + let decompose_start = Instant::now(); + let wavelets = first_levels + .into_iter() + .zip(group.iter().copied()) + .map(|(first_level, component_ref)| { + integer_wavelet_from_first_level( + first_level, + tiles[component_ref.tile_index].decomposition_levels, + ) + }) + .collect(); + timings.dwt_decompose_us = timings + .dwt_decompose_us + .saturating_add(decompose_start.elapsed().as_micros()); + return Ok(wavelets); + } + + group + .iter() + .map(|component_ref| { + integer_direct_wavelet_from_component( + &tiles[component_ref.tile_index].jpeg.components[component_ref.component_index], + tiles[component_ref.tile_index].decomposition_levels, + scratch, + accelerator, + timings, + ) + }) + .collect() +} + +fn i16_htj2k97_jobs_for_batch_group<'a>( + group: &[BatchComponentRef], + tiles: &'a [Float97BatchTile], +) -> Result>, JpegToHtj2kError> { + group + .iter() + .map(|component_ref| { + let tile = &tiles[component_ref.tile_index]; + let component = &tile.jpeg.components[component_ref.component_index]; + let (x_rsiz, y_rsiz) = tile.component_sampling[component_ref.component_index]; + validate_component_block_grid(component)?; + Ok(DctGridI16ToHtj2k97CodeBlockJob { + dequantized_blocks: &component.dequantized_blocks, + block_cols: component.block_cols as usize, + block_rows: component.block_rows as usize, + width: component.width as usize, + height: component.height as usize, + x_rsiz, + y_rsiz, + }) + }) + .collect() +} + +fn store_compact_preencoded_component( + tile: &mut Float97BatchTile, + component_index: usize, + batch_payload: &[u8], + mut component: PreencodedHtj2k97CompactComponent, +) -> Result<(), JpegToHtj2kError> { + if component_index >= tile.preencoded_compact_components.len() { + return Err(JpegToHtj2kError::Validation( + "compact preencoded component index out of range", + )); + } + + for resolution in &mut component.resolutions { + for subband in &mut resolution.subbands { + for block in &mut subband.code_blocks { + if block.payload_range.start > block.payload_range.end + || block.payload_range.end > batch_payload.len() + { + return Err(JpegToHtj2kError::Validation( + "compact preencoded payload range out of bounds", + )); + } + let start = tile.preencoded_compact_payload.len(); + tile.preencoded_compact_payload + .extend_from_slice(&batch_payload[block.payload_range.clone()]); + let end = tile.preencoded_compact_payload.len(); + block.payload_range = start..end; + } + } + } + + tile.preencoded_compact_components[component_index] = Some(component); + Ok(()) +} + +#[allow(clippy::too_many_lines)] +fn try_store_grouped_i16_preencoded_float97_batches( + groups: &[Vec], + tiles: &mut [Float97BatchTile], + options: &JpegToHtj2kOptions, + accelerator: &mut A, + timings: &mut TranscodeTimingReport, +) -> Result, JpegToHtj2kError> { + let mut handled = vec![false; groups.len()]; + if !accelerator.supports_htj2k97_i16_preencoded_batch() + || options.validate_against_float_reference + || groups.len() <= 1 + { + return Ok(handled); + } + + let eligible_indices = groups + .iter() + .enumerate() + .filter_map(|(index, group)| { + let eligible = group + .iter() + .all(|component_ref| tiles[component_ref.tile_index].decomposition_levels == 1); + eligible.then_some(index) + }) + .collect::>(); + if eligible_indices.len() <= 1 { + return Ok(handled); + } + + let codeblock_options = htj2k97_codeblock_options(&options.encode_options); + let total_jobs = eligible_indices + .iter() + .map(|&index| groups[index].len()) + .sum::(); + record_accelerator_attempt(timings, total_jobs); + let accelerator_start = Instant::now(); + let jobs_by_group = eligible_indices + .iter() + .map(|&index| i16_htj2k97_jobs_for_batch_group(&groups[index], tiles)) + .collect::, JpegToHtj2kError>>()?; + let batches = jobs_by_group + .iter() + .map(|jobs| DctGridI16ToHtj2k97CodeBlockBatch { jobs }) + .collect::>(); + let compact_grouped_components = if accelerator.supports_htj2k97_compact_preencoded_batch() { + accelerator + .dct_grid_i16_to_htj2k97_compact_preencoded_batch_groups(&batches, codeblock_options) + .map_err(JpegToHtj2kError::Accelerator)? + } else { + None + }; + if let Some(stage_timings) = accelerator.last_dwt97_batch_stage_timings() { + add_dwt97_batch_stage_timings(timings, stage_timings); + } + if let Some(compact_grouped_components) = compact_grouped_components { + timings.dct_to_wavelet_accelerator_us = timings + .dct_to_wavelet_accelerator_us + .saturating_add(accelerator_start.elapsed().as_micros()); + let compact_payload = compact_grouped_components.payload; + let compact_groups = compact_grouped_components.groups; + if compact_groups.len() != eligible_indices.len() { + return Err(JpegToHtj2kError::Validation( + "9/7 grouped i16 compact preencoded accelerator returned wrong group count", + )); + } + for (&group_index, components) in eligible_indices.iter().zip(compact_groups) { + let group = &groups[group_index]; + if components.len() != group.len() { + return Err(JpegToHtj2kError::Validation( + "9/7 grouped i16 compact preencoded accelerator returned wrong component count", + )); + } + + timings.component_count = timings.component_count.saturating_add(group.len()); + record_batch_dispatch(timings, group.len()); + for (component_ref, component) in group.iter().copied().zip(components) { + store_compact_preencoded_component( + &mut tiles[component_ref.tile_index], + component_ref.component_index, + &compact_payload, + component, + )?; + } + handled[group_index] = true; + } + return Ok(handled); + } + + let grouped_components = accelerator + .dct_grid_i16_to_htj2k97_preencoded_batch_groups(&batches, codeblock_options) + .map_err(JpegToHtj2kError::Accelerator)?; + if let Some(stage_timings) = accelerator.last_dwt97_batch_stage_timings() { + add_dwt97_batch_stage_timings(timings, stage_timings); + } + timings.dct_to_wavelet_accelerator_us = timings + .dct_to_wavelet_accelerator_us + .saturating_add(accelerator_start.elapsed().as_micros()); + + let Some(grouped_components) = grouped_components else { + return Ok(handled); + }; + if grouped_components.len() != eligible_indices.len() { + return Err(JpegToHtj2kError::Validation( + "9/7 grouped i16 preencoded accelerator returned wrong group count", + )); + } + + for (&group_index, components) in eligible_indices.iter().zip(grouped_components) { + let group = &groups[group_index]; + if components.len() != group.len() { + return Err(JpegToHtj2kError::Validation( + "9/7 grouped i16 preencoded accelerator returned wrong component count", + )); + } + + timings.component_count = timings.component_count.saturating_add(group.len()); + record_batch_dispatch(timings, group.len()); + for (component_ref, component) in group.iter().copied().zip(components) { + tiles[component_ref.tile_index].preencoded_components[component_ref.component_index] = + Some(component); + } + handled[group_index] = true; + } + + Ok(handled) +} + +#[allow(clippy::too_many_lines)] +fn try_store_prequantized_float97_batch_group( + group: &[BatchComponentRef], + tiles: &mut [Float97BatchTile], + options: &JpegToHtj2kOptions, + accelerator: &mut A, + timings: &mut TranscodeTimingReport, +) -> Result { + if !(accelerator.supports_htj2k97_codeblock_batch() + || accelerator.supports_htj2k97_i16_preencoded_batch()) + || options.validate_against_float_reference + || group + .iter() + .any(|component_ref| tiles[component_ref.tile_index].decomposition_levels != 1) + { + return Ok(false); + } + + let codeblock_options = htj2k97_codeblock_options(&options.encode_options); + if accelerator.supports_htj2k97_i16_preencoded_batch() { + let jobs = i16_htj2k97_jobs_for_batch_group(group, tiles)?; + + record_accelerator_attempt(timings, group.len()); + let accelerator_start = Instant::now(); + let compact_preencoded_components = + if accelerator.supports_htj2k97_compact_preencoded_batch() { + accelerator + .dct_grid_i16_to_htj2k97_compact_preencoded_batch(&jobs, codeblock_options) + .map_err(JpegToHtj2kError::Accelerator)? + } else { + None + }; + if let Some(stage_timings) = accelerator.last_dwt97_batch_stage_timings() { + add_dwt97_batch_stage_timings(timings, stage_timings); + } + if let Some(compact_batch) = compact_preencoded_components { + timings.dct_to_wavelet_accelerator_us = timings + .dct_to_wavelet_accelerator_us + .saturating_add(accelerator_start.elapsed().as_micros()); + if compact_batch.components.len() != group.len() { + return Err(JpegToHtj2kError::Validation( + "9/7 i16 compact preencoded accelerator returned wrong component count", + )); + } + + timings.component_count = timings.component_count.saturating_add(group.len()); + record_batch_dispatch(timings, group.len()); + for (component_ref, component) in group.iter().copied().zip(compact_batch.components) { + store_compact_preencoded_component( + &mut tiles[component_ref.tile_index], + component_ref.component_index, + &compact_batch.payload, + component, + )?; + } + + return Ok(true); + } + + let preencoded_components = accelerator + .dct_grid_i16_to_htj2k97_preencoded_batch(&jobs, codeblock_options) + .map_err(JpegToHtj2kError::Accelerator)?; + if let Some(stage_timings) = accelerator.last_dwt97_batch_stage_timings() { + add_dwt97_batch_stage_timings(timings, stage_timings); + } + timings.dct_to_wavelet_accelerator_us = timings + .dct_to_wavelet_accelerator_us + .saturating_add(accelerator_start.elapsed().as_micros()); + if let Some(components) = preencoded_components { + if components.len() != group.len() { + return Err(JpegToHtj2kError::Validation( + "9/7 i16 preencoded accelerator returned wrong component count", + )); + } + + timings.component_count = timings.component_count.saturating_add(group.len()); + record_batch_dispatch(timings, group.len()); + for (component_ref, component) in group.iter().copied().zip(components) { + tiles[component_ref.tile_index].preencoded_components + [component_ref.component_index] = Some(component); + } + + return Ok(true); + } + } + + let repack_start = Instant::now(); + let block_storage = group + .par_iter() + .map(|component_ref| { + dct_blocks_to_8x8_f64( + &tiles[component_ref.tile_index].jpeg.components[component_ref.component_index] + .dequantized_blocks, + ) + }) + .collect::>(); + timings.jpeg_dct_repack_us = timings + .jpeg_dct_repack_us + .saturating_add(repack_start.elapsed().as_micros()); + + let jobs = group + .iter() + .zip(block_storage.iter()) + .map(|(component_ref, blocks)| { + let tile = &tiles[component_ref.tile_index]; + let component = &tile.jpeg.components[component_ref.component_index]; + let (x_rsiz, y_rsiz) = tile.component_sampling[component_ref.component_index]; + validate_component_block_grid(component)?; + Ok(DctGridToHtj2k97CodeBlockJob { + blocks, + block_cols: component.block_cols as usize, + block_rows: component.block_rows as usize, + width: component.width as usize, + height: component.height as usize, + x_rsiz, + y_rsiz, + }) + }) + .collect::, JpegToHtj2kError>>()?; + + record_accelerator_attempt(timings, group.len()); + let accelerator_start = Instant::now(); + let preencoded_components = accelerator + .dct_grid_to_htj2k97_preencoded_batch(&jobs, codeblock_options) + .map_err(JpegToHtj2kError::Accelerator)?; + if let Some(components) = preencoded_components { + if let Some(stage_timings) = accelerator.last_dwt97_batch_stage_timings() { + add_dwt97_batch_stage_timings(timings, stage_timings); + } + timings.dct_to_wavelet_accelerator_us = timings + .dct_to_wavelet_accelerator_us + .saturating_add(accelerator_start.elapsed().as_micros()); + if components.len() != group.len() { + return Err(JpegToHtj2kError::Validation( + "9/7 preencoded accelerator returned wrong component count", + )); + } + + timings.component_count = timings.component_count.saturating_add(group.len()); + record_batch_dispatch(timings, group.len()); + for (component_ref, component) in group.iter().copied().zip(components) { + tiles[component_ref.tile_index].preencoded_components[component_ref.component_index] = + Some(component); + } + + return Ok(true); + } + + let accelerated_components = accelerator + .dct_grid_to_htj2k97_codeblock_batch(&jobs, codeblock_options) + .map_err(JpegToHtj2kError::Accelerator)?; + if let Some(stage_timings) = accelerator.last_dwt97_batch_stage_timings() { + add_dwt97_batch_stage_timings(timings, stage_timings); + } + timings.dct_to_wavelet_accelerator_us = timings + .dct_to_wavelet_accelerator_us + .saturating_add(accelerator_start.elapsed().as_micros()); + + let Some(components) = accelerated_components else { + return Ok(false); + }; + if components.len() != group.len() { + return Err(JpegToHtj2kError::Validation( + "9/7 code-block accelerator returned wrong component count", + )); + } + + timings.component_count = timings.component_count.saturating_add(group.len()); + record_batch_dispatch(timings, group.len()); + for (component_ref, component) in group.iter().copied().zip(components) { + tiles[component_ref.tile_index].prequantized_components[component_ref.component_index] = + Some(component); + } + + Ok(true) +} + +fn htj2k97_codeblock_options(options: &JpegToHtj2kEncodeOptions) -> Htj2k97CodeBlockOptions { + Htj2k97CodeBlockOptions { + bit_depth: 8, + guard_bits: options.guard_bits.max(2), + code_block_width_exp: options.code_block_width_exp, + code_block_height_exp: options.code_block_height_exp, + irreversible_quantization_scale: options.irreversible_quantization_scale, + irreversible_quantization_subband_scales: options.irreversible_quantization_subband_scales, + } +} + +fn native_progression_order( + progression: J2kProgressionOrder, +) -> j2k_native::EncodeProgressionOrder { + match progression { + J2kProgressionOrder::Lrcp => j2k_native::EncodeProgressionOrder::Lrcp, + J2kProgressionOrder::Rlcp => j2k_native::EncodeProgressionOrder::Rlcp, + J2kProgressionOrder::Rpcl => j2k_native::EncodeProgressionOrder::Rpcl, + J2kProgressionOrder::Pcrl => j2k_native::EncodeProgressionOrder::Pcrl, + J2kProgressionOrder::Cprl => j2k_native::EncodeProgressionOrder::Cprl, + } +} + +fn float97_wavelets_for_batch_group( + group: &[BatchComponentRef], + tiles: &[Float97BatchTile], + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut A, + timings: &mut TranscodeTimingReport, +) -> Result, JpegToHtj2kError> { + let repack_start = Instant::now(); + let block_storage = group + .iter() + .map(|component_ref| { + dct_blocks_to_8x8_f64( + &tiles[component_ref.tile_index].jpeg.components[component_ref.component_index] + .dequantized_blocks, + ) + }) + .collect::>(); + timings.jpeg_dct_repack_us = timings + .jpeg_dct_repack_us + .saturating_add(repack_start.elapsed().as_micros()); + + let jobs = group + .iter() + .zip(block_storage.iter()) + .map(|(component_ref, blocks)| { + let component = + &tiles[component_ref.tile_index].jpeg.components[component_ref.component_index]; + validate_component_block_grid(component)?; + Ok(DctGridToDwt97Job { + blocks, + block_cols: component.block_cols as usize, + block_rows: component.block_rows as usize, + width: component.width as usize, + height: component.height as usize, + }) + }) + .collect::, JpegToHtj2kError>>()?; + + record_batch_attempt(timings, group.len()); + let accelerator_start = Instant::now(); + let accelerated_first_levels = accelerator + .dct_grid_to_dwt97_batch(&jobs) + .map_err(JpegToHtj2kError::Accelerator)?; + if let Some(stage_timings) = accelerator.last_dwt97_batch_stage_timings() { + add_dwt97_batch_stage_timings(timings, stage_timings); + } + timings.dct_to_wavelet_accelerator_us = timings + .dct_to_wavelet_accelerator_us + .saturating_add(accelerator_start.elapsed().as_micros()); + + if let Some(first_levels) = accelerated_first_levels { + if first_levels.len() != group.len() { + return Err(JpegToHtj2kError::Validation( + "9/7 batch accelerator returned wrong component count", + )); + } + timings.component_count = timings.component_count.saturating_add(group.len()); + record_accelerator_dispatch(timings, group.len()); + let decompose_start = Instant::now(); + let wavelets = first_levels + .into_par_iter() + .zip(group.par_iter().copied()) + .map(|(first_level, component_ref)| { + decompose_97_from_first_level( + first_level, + usize::from(tiles[component_ref.tile_index].decomposition_levels), + ) + }) + .collect::>(); + timings.dwt_decompose_us = timings + .dwt_decompose_us + .saturating_add(decompose_start.elapsed().as_micros()); + return Ok(wavelets); + } + + group + .iter() + .map(|component_ref| { + float_direct_97_wavelet_from_component( + &tiles[component_ref.tile_index].jpeg.components[component_ref.component_index], + tiles[component_ref.tile_index].decomposition_levels, + scratch, + accelerator, + timings, + ) + }) + .collect() +} + +fn add_dwt97_batch_stage_timings( + timings: &mut TranscodeTimingReport, + stage_timings: Dwt97BatchStageTimings, +) { + timings.dwt97_batch_pack_upload_us = timings + .dwt97_batch_pack_upload_us + .saturating_add(stage_timings.pack_upload_us); + timings.dwt97_batch_idct_row_lift_us = timings + .dwt97_batch_idct_row_lift_us + .saturating_add(stage_timings.idct_row_lift_us); + timings.dwt97_batch_column_lift_us = timings + .dwt97_batch_column_lift_us + .saturating_add(stage_timings.column_lift_us); + timings.dwt97_batch_quantize_codeblock_us = timings + .dwt97_batch_quantize_codeblock_us + .saturating_add(stage_timings.quantize_codeblock_us); + timings.dwt97_batch_ht_encode_us = timings + .dwt97_batch_ht_encode_us + .saturating_add(stage_timings.ht_encode_us); + timings.dwt97_batch_ht_kernel_us = timings + .dwt97_batch_ht_kernel_us + .saturating_add(stage_timings.ht_kernel_us); + timings.dwt97_batch_ht_status_readback_us = timings + .dwt97_batch_ht_status_readback_us + .saturating_add(stage_timings.ht_status_readback_us); + timings.dwt97_batch_ht_compact_us = timings + .dwt97_batch_ht_compact_us + .saturating_add(stage_timings.ht_compact_us); + timings.dwt97_batch_ht_output_readback_us = timings + .dwt97_batch_ht_output_readback_us + .saturating_add(stage_timings.ht_output_readback_us); + timings.dwt97_batch_ht_codeblock_dispatches = timings + .dwt97_batch_ht_codeblock_dispatches + .saturating_add(stage_timings.ht_codeblock_dispatches); + timings.dwt97_batch_readback_us = timings + .dwt97_batch_readback_us + .saturating_add(stage_timings.readback_us); +} + +fn record_accelerator_attempt(timings: &mut TranscodeTimingReport, job_count: usize) { + timings.accelerator_attempts = timings.accelerator_attempts.saturating_add(1); + timings.accelerator_jobs = timings.accelerator_jobs.saturating_add(job_count); +} + +fn record_accelerator_dispatch(timings: &mut TranscodeTimingReport, job_count: usize) { + timings.accelerator_dispatches = timings.accelerator_dispatches.saturating_add(1); + timings.accelerator_dispatched_jobs = timings + .accelerator_dispatched_jobs + .saturating_add(job_count); +} + +fn record_batch_attempt(timings: &mut TranscodeTimingReport, job_count: usize) { + timings.batch_count = timings.batch_count.saturating_add(1); + timings.batch_jobs = timings.batch_jobs.saturating_add(job_count); + record_accelerator_attempt(timings, job_count); +} + +fn record_batch_dispatch(timings: &mut TranscodeTimingReport, job_count: usize) { + timings.batch_count = timings.batch_count.saturating_add(1); + timings.batch_jobs = timings.batch_jobs.saturating_add(job_count); + record_accelerator_dispatch(timings, job_count); +} + +fn record_cpu_fallback(timings: &mut TranscodeTimingReport, job_count: usize) { + timings.cpu_fallback_jobs = timings.cpu_fallback_jobs.saturating_add(job_count); +} + +fn store_integer_batch_wavelet( + component_ref: BatchComponentRef, + wavelet: &IntegerWavelet, + tiles: &mut [IntegerBatchTile], + options: &JpegToHtj2kOptions, + scratch: &mut JpegToHtj2kScratch, +) -> Result<(), JpegToHtj2kError> { + let tile = &mut tiles[component_ref.tile_index]; + let component = &tile.jpeg.components[component_ref.component_index]; + let (x_rsiz, y_rsiz) = tile.component_sampling[component_ref.component_index]; + let actual_coefficients = flatten_integer_wavelet(wavelet); + tile.precomputed_components[component_ref.component_index] = + Some(PrecomputedHtj2k53Component { + x_rsiz, + y_rsiz, + dwt: j2k_dwt_from_integer_wavelet(wavelet), + }); + + if options.validate_against_float_reference { + tile.float_validation_actual + .extend(actual_coefficients.clone()); + tile.float_validation_expected + .extend(float_reference_coefficients( + component, + tile.decomposition_levels, + scratch, + )?); + } + if options.validate_against_integer_reference { + tile.integer_validation_actual.extend(actual_coefficients); + tile.integer_validation_expected + .extend(integer_reference_coefficients( + component, + tile.decomposition_levels, + )?); + } + + Ok(()) +} + +fn store_float97_batch_wavelet( + component_ref: BatchComponentRef, + wavelet: &ComponentWavelet97, + tiles: &mut [Float97BatchTile], + options: &JpegToHtj2kOptions, + scratch: &mut JpegToHtj2kScratch, +) -> Result<(), JpegToHtj2kError> { + let tile = &mut tiles[component_ref.tile_index]; + let component = &tile.jpeg.components[component_ref.component_index]; + let (x_rsiz, y_rsiz) = tile.component_sampling[component_ref.component_index]; + tile.precomputed_components[component_ref.component_index] = + Some(PrecomputedHtj2k97Component { + x_rsiz, + y_rsiz, + dwt: j2k_dwt97_from_wavelet( + wavelet, + component.width as usize, + component.height as usize, + ), + }); + + if options.validate_against_float_reference { + let actual_coefficients = rounded_wavelet97_i32(wavelet)?; + tile.float_validation_actual.extend(actual_coefficients); + tile.float_validation_expected + .extend(float97_reference_coefficients( + component, + tile.decomposition_levels, + scratch, + )?); + } + + Ok(()) +} + +fn record_encode_dispatch_delta( + timings: &mut TranscodeTimingReport, + before: J2kEncodeDispatchReport, + after: J2kEncodeDispatchReport, +) { + let delta = after.saturating_delta(before); + timings.htj2k_encode_accelerator_dispatches = timings + .htj2k_encode_accelerator_dispatches + .saturating_add(delta.total()); + timings.htj2k_encode_ht_code_block_dispatches = timings + .htj2k_encode_ht_code_block_dispatches + .saturating_add(delta.ht_code_block); + timings.htj2k_encode_packetization_dispatches = timings + .htj2k_encode_packetization_dispatches + .saturating_add(delta.packetization); +} + +fn add_encode_timing_counters_from_result( + timings: &mut TranscodeTimingReport, + tile: &Result, +) { + let Ok(tile) = tile else { + return; + }; + timings.htj2k_encode_accelerator_dispatches = timings + .htj2k_encode_accelerator_dispatches + .saturating_add(tile.report.timings.htj2k_encode_accelerator_dispatches); + timings.htj2k_encode_ht_code_block_dispatches = timings + .htj2k_encode_ht_code_block_dispatches + .saturating_add(tile.report.timings.htj2k_encode_ht_code_block_dispatches); + timings.htj2k_encode_packetization_dispatches = timings + .htj2k_encode_packetization_dispatches + .saturating_add(tile.report.timings.htj2k_encode_packetization_dispatches); +} + +fn encode_integer_prepared_tiles( + prepared_tiles: Vec, + options: &JpegToHtj2kOptions, + encode_accelerator: &mut E, +) -> Vec<(usize, Result)> { + if encode_accelerator.prefer_parallel_cpu_tile_encode() { + return prepared_tiles + .into_par_iter() + .map(|prepared| { + let tile_index = prepared.tile_index; + let mut cpu_accelerator = CpuOnlyJ2kEncodeStageAccelerator; + ( + tile_index, + encode_integer_batch_tile(prepared, options, &mut cpu_accelerator), + ) + }) + .collect(); + } + + prepared_tiles + .into_iter() + .map(|prepared| { + let tile_index = prepared.tile_index; + ( + tile_index, + encode_integer_batch_tile(prepared, options, encode_accelerator), + ) + }) + .collect() +} + +fn encode_float97_prepared_tiles( + prepared_tiles: Vec, + options: &JpegToHtj2kOptions, + encode_accelerator: &mut E, +) -> Vec<(usize, Result)> { + if !encode_accelerator.prefer_parallel_cpu_tile_encode() + && can_encode_float97_precomputed_tiles_batch(&prepared_tiles, options) + { + return encode_float97_precomputed_tiles_batch(prepared_tiles, options, encode_accelerator); + } + + if encode_accelerator.prefer_parallel_cpu_tile_encode() { + return prepared_tiles + .into_par_iter() + .map(|prepared| { + let tile_index = prepared.tile_index; + let mut cpu_accelerator = CpuOnlyJ2kEncodeStageAccelerator; + ( + tile_index, + encode_float97_batch_tile(prepared, options, &mut cpu_accelerator), + ) + }) + .collect(); + } + + prepared_tiles + .into_iter() + .map(|prepared| { + let tile_index = prepared.tile_index; + ( + tile_index, + encode_float97_batch_tile(prepared, options, encode_accelerator), + ) + }) + .collect() +} + +fn can_encode_float97_precomputed_tiles_batch( + prepared_tiles: &[Float97BatchTile], + options: &JpegToHtj2kOptions, +) -> bool { + options.encode_options.num_layers == 1 + && prepared_tiles.iter().all(|tile| { + tile.precomputed_components.iter().all(Option::is_some) + && tile.preencoded_compact_payload.is_empty() + && tile + .preencoded_compact_components + .iter() + .all(Option::is_none) + && tile.preencoded_components.iter().all(Option::is_none) + && tile.prequantized_components.iter().all(Option::is_none) + }) +} + +#[allow(clippy::too_many_lines)] +fn encode_float97_precomputed_tiles_batch( + prepared_tiles: Vec, + options: &JpegToHtj2kOptions, + encode_accelerator: &mut E, +) -> Vec<(usize, Result)> { + let mut records = Vec::with_capacity(prepared_tiles.len()); + let mut images = Vec::with_capacity(prepared_tiles.len()); + + for tile in prepared_tiles { + let Float97BatchTile { + tile_index, + jpeg, + decomposition_levels, + all_unit_sampled, + component_reports, + precomputed_components, + preencoded_compact_payload: _, + preencoded_compact_components: _, + preencoded_components: _, + prequantized_components: _, + float_validation_actual, + float_validation_expected, + timings, + .. + } = tile; + let components = match precomputed_components + .into_iter() + .map(|component| { + component.ok_or(JpegToHtj2kError::Validation( + "9/7 precomputed batch transcode did not produce all components", + )) + }) + .collect::, _>>() + { + Ok(components) => components, + Err(error) => return vec![(tile_index, Err(error))], + }; + images.push(PrecomputedHtj2k97Image { + width: jpeg.width, + height: jpeg.height, + bit_depth: 8, + signed: false, + components, + }); + records.push(Float97PrecomputedBatchRecord { + tile_index, + jpeg, + decomposition_levels, + all_unit_sampled, + component_reports, + float_validation_actual, + float_validation_expected, + timings, + }); + } + + let encode_start = Instant::now(); + let encode_dispatch_before = encode_accelerator.dispatch_report(); + let native_images = images; + let codestreams = { + let mut native_encode_accelerator = NativeEncodeStageAdapter::new(encode_accelerator); + let native_encode_options = options.encode_options.to_native(); + match encode_precomputed_htj2k_97_batch_with_accelerator( + &native_images, + &native_encode_options, + &mut native_encode_accelerator, + ) { + Ok(codestreams) => codestreams, + Err(error) => { + return records + .into_iter() + .map(|record| (record.tile_index, Err(JpegToHtj2kError::Encode(error)))) + .collect(); + } + } + }; + let encode_dispatch_after = encode_accelerator.dispatch_report(); + let encode_us = encode_start.elapsed().as_micros(); + + if codestreams.len() != records.len() { + return records + .into_iter() + .map(|record| { + ( + record.tile_index, + Err(JpegToHtj2kError::Validation( + "9/7 precomputed batch encode returned the wrong tile count", + )), + ) + }) + .collect(); + } + + records + .into_iter() + .zip(codestreams) + .enumerate() + .map(|(batch_index, (record, codestream))| { + let encode_measurement = (batch_index == 0).then_some(( + encode_dispatch_before, + encode_dispatch_after, + encode_us, + )); + ( + record.tile_index, + encoded_float97_precomputed_batch_record( + record, + codestream, + options, + encode_measurement, + ), + ) + }) + .collect() +} + +fn encoded_float97_precomputed_batch_record( + record: Float97PrecomputedBatchRecord, + codestream: Vec, + options: &JpegToHtj2kOptions, + encode_measurement: Option<(J2kEncodeDispatchReport, J2kEncodeDispatchReport, u128)>, +) -> Result { + let Float97PrecomputedBatchRecord { + jpeg, + decomposition_levels, + all_unit_sampled, + component_reports, + float_validation_actual, + float_validation_expected, + mut timings, + .. + } = record; + + if let Some((encode_dispatch_before, encode_dispatch_after, encode_us)) = encode_measurement { + record_encode_dispatch_delta(&mut timings, encode_dispatch_before, encode_dispatch_after); + timings.htj2k_encode_us = encode_us; + } + let encode_us = timings.htj2k_encode_us; + let float_reference_metrics = if options.validate_against_float_reference { + Some(error_metrics_i32( + &float_validation_actual, + &float_validation_expected, + )?) + } else { + None + }; + + Ok(EncodedTranscode { + codestream, + report: TranscodeReport { + width: jpeg.width, + height: jpeg.height, + component_count: jpeg.components.len(), + components: component_reports, + float_reference_classification: float_reference_metrics + .as_ref() + .map(TranscodeValidationClassification::classify_metrics), + float_reference_metrics, + integer_reference_classification: None, + integer_reference_metrics: None, + decomposition_levels, + coefficient_path: options.coefficient_path, + path: transcode_path_name(all_unit_sampled, options.coefficient_path), + extract_us: timings.jpeg_dct_extract_us, + transform_us: 0, + encode_us, + timings, + }, + }) +} + +fn encode_integer_batch_tile( + tile: IntegerBatchTile, + options: &JpegToHtj2kOptions, + encode_accelerator: &mut E, +) -> Result { + let mut timings = tile.timings; + let components = tile + .precomputed_components + .into_iter() + .map(|component| { + component.ok_or(JpegToHtj2kError::Validation( + "integer batch transcode did not produce all components", + )) + }) + .collect::, _>>()?; + let encode_start = Instant::now(); + let precomputed = PrecomputedHtj2k53Image { + width: tile.jpeg.width, + height: tile.jpeg.height, + bit_depth: 8, + signed: false, + components, + }; + let encode_dispatch_before = encode_accelerator.dispatch_report(); + let native_precomputed = precomputed; + let codestream = { + let mut native_encode_accelerator = NativeEncodeStageAdapter::new(encode_accelerator); + let native_encode_options = options.encode_options.to_native(); + encode_precomputed_htj2k_53_with_accelerator( + &native_precomputed, + &native_encode_options, + &mut native_encode_accelerator, + ) + .map_err(JpegToHtj2kError::Encode)? + }; + record_encode_dispatch_delta( + &mut timings, + encode_dispatch_before, + encode_accelerator.dispatch_report(), + ); + let encode_us = encode_start.elapsed().as_micros(); + timings.htj2k_encode_us = encode_us; + let integer_reference_metrics = if options.validate_against_integer_reference { + Some(error_metrics_i32( + &tile.integer_validation_actual, + &tile.integer_validation_expected, + )?) + } else { + None + }; + let float_reference_metrics = if options.validate_against_float_reference { + Some(error_metrics_i32( + &tile.float_validation_actual, + &tile.float_validation_expected, + )?) + } else { + None + }; + + Ok(EncodedTranscode { + codestream, + report: TranscodeReport { + width: tile.jpeg.width, + height: tile.jpeg.height, + component_count: tile.jpeg.components.len(), + components: tile.component_reports, + float_reference_classification: float_reference_metrics + .as_ref() + .map(TranscodeValidationClassification::classify_metrics), + float_reference_metrics, + integer_reference_classification: integer_reference_metrics + .as_ref() + .map(TranscodeValidationClassification::classify_metrics), + integer_reference_metrics, + decomposition_levels: tile.decomposition_levels, + coefficient_path: options.coefficient_path, + path: transcode_path_name(tile.all_unit_sampled, options.coefficient_path), + extract_us: timings.jpeg_dct_extract_us, + transform_us: 0, + encode_us, + timings, + }, + }) +} + +#[allow(clippy::too_many_lines)] +fn encode_float97_batch_tile( + tile: Float97BatchTile, + options: &JpegToHtj2kOptions, + encode_accelerator: &mut E, +) -> Result { + let Float97BatchTile { + jpeg, + decomposition_levels, + all_unit_sampled, + component_reports, + precomputed_components, + preencoded_compact_payload, + preencoded_compact_components, + preencoded_components, + prequantized_components, + float_validation_actual, + float_validation_expected, + mut timings, + .. + } = tile; + + let encode_start = Instant::now(); + let encode_dispatch_before = encode_accelerator.dispatch_report(); + let codestream = { + let mut native_encode_accelerator = NativeEncodeStageAdapter::new(encode_accelerator); + let native_encode_options = options.encode_options.to_native(); + if preencoded_compact_components.iter().any(Option::is_some) { + let components = preencoded_compact_components + .into_iter() + .map(|component| { + component.ok_or(JpegToHtj2kError::Validation( + "9/7 compact preencoded batch transcode did not produce all components", + )) + }) + .collect::, _>>()?; + let preencoded = PreencodedHtj2k97CompactImage { + width: jpeg.width, + height: jpeg.height, + bit_depth: 8, + signed: false, + payload: preencoded_compact_payload, + components, + }; + encode_preencoded_htj2k_97_compact_owned_with_accelerator( + preencoded, + &native_encode_options, + &mut native_encode_accelerator, + ) + .map_err(JpegToHtj2kError::Encode)? + } else if preencoded_components.iter().any(Option::is_some) { + let components = preencoded_components + .into_iter() + .map(|component| { + component.ok_or(JpegToHtj2kError::Validation( + "9/7 preencoded batch transcode did not produce all components", + )) + }) + .collect::, _>>()?; + let preencoded = PreencodedHtj2k97Image { + width: jpeg.width, + height: jpeg.height, + bit_depth: 8, + signed: false, + components, + }; + encode_preencoded_htj2k_97_owned_with_accelerator( + preencoded, + &native_encode_options, + &mut native_encode_accelerator, + ) + .map_err(JpegToHtj2kError::Encode)? + } else if prequantized_components.iter().any(Option::is_some) { + let components = prequantized_components + .into_iter() + .map(|component| { + component.ok_or(JpegToHtj2kError::Validation( + "9/7 code-block batch transcode did not produce all components", + )) + }) + .collect::, _>>()?; + let prequantized = PrequantizedHtj2k97Image { + width: jpeg.width, + height: jpeg.height, + bit_depth: 8, + signed: false, + components, + }; + let native_prequantized = prequantized; + encode_prequantized_htj2k_97_with_accelerator( + &native_prequantized, + &native_encode_options, + &mut native_encode_accelerator, + ) + .map_err(JpegToHtj2kError::Encode)? + } else { + let components = precomputed_components + .into_iter() + .map(|component| { + component.ok_or(JpegToHtj2kError::Validation( + "9/7 batch transcode did not produce all components", + )) + }) + .collect::, _>>()?; + let precomputed = PrecomputedHtj2k97Image { + width: jpeg.width, + height: jpeg.height, + bit_depth: 8, + signed: false, + components, + }; + let native_precomputed = precomputed; + encode_precomputed_htj2k_97_with_accelerator( + &native_precomputed, + &native_encode_options, + &mut native_encode_accelerator, + ) + .map_err(JpegToHtj2kError::Encode)? + } + }; + record_encode_dispatch_delta( + &mut timings, + encode_dispatch_before, + encode_accelerator.dispatch_report(), + ); + let encode_us = encode_start.elapsed().as_micros(); + timings.htj2k_encode_us = encode_us; + let float_reference_metrics = if options.validate_against_float_reference { + Some(error_metrics_i32( + &float_validation_actual, + &float_validation_expected, + )?) + } else { + None + }; + + Ok(EncodedTranscode { + codestream, + report: TranscodeReport { + width: jpeg.width, + height: jpeg.height, + component_count: jpeg.components.len(), + components: component_reports, + float_reference_classification: float_reference_metrics + .as_ref() + .map(TranscodeValidationClassification::classify_metrics), + float_reference_metrics, + integer_reference_classification: None, + integer_reference_metrics: None, + decomposition_levels, + coefficient_path: options.coefficient_path, + path: transcode_path_name(all_unit_sampled, options.coefficient_path), + extract_us: timings.jpeg_dct_extract_us, + transform_us: 0, + encode_us, + timings, + }, + }) +} + +#[allow(clippy::too_many_lines)] +fn jpeg_to_htj2k_with_scratch( + bytes: &[u8], + options: &JpegToHtj2kOptions, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut A, + encode_accelerator: &mut E, +) -> Result { + validate_transcode_options(options)?; + let mut timings = TranscodeTimingReport { + tile_count: 1, + ..TranscodeTimingReport::default() + }; + + let extract_start = Instant::now(); + let jpeg = extract_dct_blocks(bytes, DctExtractOptions::default())?; + let extract_us = extract_start.elapsed().as_micros(); + timings.jpeg_dct_extract_us = extract_us; + + if jpeg.components.is_empty() || jpeg.components.len() > 4 { + return Err(JpegToHtj2kError::Unsupported( + "unsupported JPEG component count for jpeg_to_htj2k", + )); + } + let component_sampling = + component_sampling_for_jpeg(&jpeg.components, jpeg.width, jpeg.height)?; + let decomposition_levels = decomposition_levels_for_components( + &jpeg.components, + options.encode_options.num_decomposition_levels, + )?; + let all_unit_sampled = component_sampling + .iter() + .all(|&(x_rsiz, y_rsiz)| x_rsiz == 1 && y_rsiz == 1); + let component_reports = jpeg + .components + .iter() + .zip(component_sampling.iter().copied()) + .map(|(component, (x_rsiz, y_rsiz))| TranscodeComponentReport { + component_index: component.component_index, + width: component.width, + height: component.height, + block_cols: component.block_cols, + block_rows: component.block_rows, + x_rsiz, + y_rsiz, + }) + .collect(); + + let transform_start = Instant::now(); + let component_batch = transcode_component_batch( + &jpeg.components, + &component_sampling, + decomposition_levels, + options, + scratch, + accelerator, + &mut timings, + )?; + let transform_us = transform_start.elapsed().as_micros(); + timings.dct_to_wavelet_total_us = transform_us; + + let encode_start = Instant::now(); + let encode_dispatch_before = encode_accelerator.dispatch_report(); + let native_encode_options = options.encode_options.to_native(); + let codestream = match component_batch.precomputed_components { + PrecomputedComponentBatch::Dwt53(components) => { + let precomputed = PrecomputedHtj2k53Image { + width: jpeg.width, + height: jpeg.height, + bit_depth: 8, + signed: false, + components, + }; + let native_precomputed = precomputed; + let mut native_encode_accelerator = NativeEncodeStageAdapter::new(encode_accelerator); + encode_precomputed_htj2k_53_with_accelerator( + &native_precomputed, + &native_encode_options, + &mut native_encode_accelerator, + ) + .map_err(JpegToHtj2kError::Encode)? + } + PrecomputedComponentBatch::Dwt97(components) => { + let precomputed = PrecomputedHtj2k97Image { + width: jpeg.width, + height: jpeg.height, + bit_depth: 8, + signed: false, + components, + }; + let native_precomputed = precomputed; + let mut native_encode_accelerator = NativeEncodeStageAdapter::new(encode_accelerator); + encode_precomputed_htj2k_97_with_accelerator( + &native_precomputed, + &native_encode_options, + &mut native_encode_accelerator, + ) + .map_err(JpegToHtj2kError::Encode)? + } + }; + record_encode_dispatch_delta( + &mut timings, + encode_dispatch_before, + encode_accelerator.dispatch_report(), + ); + let encode_us = encode_start.elapsed().as_micros(); + timings.htj2k_encode_us = encode_us; + + Ok(EncodedTranscode { + codestream, + report: TranscodeReport { + width: jpeg.width, + height: jpeg.height, + component_count: jpeg.components.len(), + components: component_reports, + float_reference_classification: component_batch + .float_reference_metrics + .as_ref() + .map(TranscodeValidationClassification::classify_metrics), + float_reference_metrics: component_batch.float_reference_metrics, + integer_reference_classification: component_batch + .integer_reference_metrics + .as_ref() + .map(TranscodeValidationClassification::classify_metrics), + integer_reference_metrics: component_batch.integer_reference_metrics, + decomposition_levels, + coefficient_path: options.coefficient_path, + path: transcode_path_name(all_unit_sampled, options.coefficient_path), + extract_us, + transform_us, + encode_us, + timings, + }, + }) +} + +fn validate_transcode_options(options: &JpegToHtj2kOptions) -> Result<(), JpegToHtj2kError> { + if !options.encode_options.use_ht_block_coding { + return Err(JpegToHtj2kError::Unsupported( + "jpeg_to_htj2k requires HT block coding", + )); + } + if options.encode_options.use_mct { + return Err(JpegToHtj2kError::Unsupported( + "jpeg_to_htj2k requires use_mct=false because JPEG components stay in native color space", + )); + } + + match (options.coefficient_path, options.encode_options.reversible) { + ( + JpegToHtj2kCoefficientPath::IntegerDirect53 + | JpegToHtj2kCoefficientPath::FloatDirectLinear53, + true, + ) + | (JpegToHtj2kCoefficientPath::FloatDirectLinear97, false) => Ok(()), + ( + JpegToHtj2kCoefficientPath::IntegerDirect53 + | JpegToHtj2kCoefficientPath::FloatDirectLinear53, + false, + ) => Err(JpegToHtj2kError::Unsupported( + "5/3 coefficient path requires reversible HTJ2K encode", + )), + (JpegToHtj2kCoefficientPath::FloatDirectLinear97, true) => { + Err(JpegToHtj2kError::Unsupported( + "9/7 coefficient path requires irreversible HTJ2K encode", + )) + } + } +} + +struct ComponentTranscodeBatch { + precomputed_components: PrecomputedComponentBatch, + float_reference_metrics: Option, + integer_reference_metrics: Option, +} + +enum PrecomputedComponentBatch { + Dwt53(Vec), + Dwt97(Vec), +} + +struct ComponentTranscodeResult { + precomputed: PrecomputedComponent, + float_validation_coefficients: Option<(Vec, Vec)>, + integer_validation_coefficients: Option<(Vec, Vec)>, +} + +enum PrecomputedComponent { + Dwt53(PrecomputedHtj2k53Component), + Dwt97(PrecomputedHtj2k97Component), +} + +struct ComponentWavelet { + final_ll: Vec, + final_ll_width: usize, + final_ll_height: usize, + levels: Vec>, +} + +struct ComponentWavelet97 { + final_ll: Vec, + final_ll_width: usize, + final_ll_height: usize, + levels: Vec>, +} + +struct IntegerWaveletLevel { + width: usize, + height: usize, + low_width: usize, + low_height: usize, + high_width: usize, + high_height: usize, + hl: Vec, + lh: Vec, + hh: Vec, +} + +struct IntegerWavelet { + final_ll: Vec, + final_ll_width: usize, + final_ll_height: usize, + levels: Vec, +} + +fn transcode_component_batch( + components: &[JpegDctComponent], + component_sampling: &[(u8, u8)], + decomposition_levels: u8, + options: &JpegToHtj2kOptions, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut impl DctToWaveletStageAccelerator, + timings: &mut TranscodeTimingReport, +) -> Result { + if matches!( + options.coefficient_path, + JpegToHtj2kCoefficientPath::FloatDirectLinear97 + ) && options.validate_against_integer_reference + { + return Err(JpegToHtj2kError::Unsupported( + "integer reversible validation is only defined for 5/3 coefficient paths", + )); + } + + if matches!( + options.coefficient_path, + JpegToHtj2kCoefficientPath::IntegerDirect53 + ) { + return transcode_integer_component_batch( + components, + component_sampling, + decomposition_levels, + options, + scratch, + accelerator, + timings, + ); + } + + let mut precomputed_53 = Vec::with_capacity(components.len()); + let mut precomputed_97 = Vec::with_capacity(components.len()); + let mut float_validation_actual = Vec::new(); + let mut float_validation_expected = Vec::new(); + let mut integer_validation_actual = Vec::new(); + let mut integer_validation_expected = Vec::new(); + + for (component, (x_rsiz, y_rsiz)) in components.iter().zip(component_sampling.iter().copied()) { + let component_result = component_to_precomputed_htj2k( + component, + x_rsiz, + y_rsiz, + decomposition_levels, + options, + scratch, + accelerator, + timings, + )?; + match component_result.precomputed { + PrecomputedComponent::Dwt53(precomputed) => precomputed_53.push(precomputed), + PrecomputedComponent::Dwt97(precomputed) => precomputed_97.push(precomputed), + } + if let Some((actual, expected)) = component_result.float_validation_coefficients { + float_validation_actual.extend(actual); + float_validation_expected.extend(expected); + } + if let Some((actual, expected)) = component_result.integer_validation_coefficients { + integer_validation_actual.extend(actual); + integer_validation_expected.extend(expected); + } + } + + let float_reference_metrics = if options.validate_against_float_reference { + Some(error_metrics_i32( + &float_validation_actual, + &float_validation_expected, + )?) + } else { + None + }; + let integer_reference_metrics = if options.validate_against_integer_reference { + Some(error_metrics_i32( + &integer_validation_actual, + &integer_validation_expected, + )?) + } else { + None + }; + + let precomputed_components = if matches!( + options.coefficient_path, + JpegToHtj2kCoefficientPath::FloatDirectLinear97 + ) { + PrecomputedComponentBatch::Dwt97(precomputed_97) + } else { + PrecomputedComponentBatch::Dwt53(precomputed_53) + }; + + Ok(ComponentTranscodeBatch { + precomputed_components, + float_reference_metrics, + integer_reference_metrics, + }) +} + +fn transcode_integer_component_batch( + components: &[JpegDctComponent], + component_sampling: &[(u8, u8)], + decomposition_levels: u8, + options: &JpegToHtj2kOptions, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut impl DctToWaveletStageAccelerator, + timings: &mut TranscodeTimingReport, +) -> Result { + let mut precomputed_53: Vec> = + (0..components.len()).map(|_| None).collect(); + let mut float_validation_actual = Vec::new(); + let mut float_validation_expected = Vec::new(); + let mut integer_validation_actual = Vec::new(); + let mut integer_validation_expected = Vec::new(); + + for group in same_geometry_component_groups(components) { + let group_wavelets = integer_wavelets_for_component_group( + &group, + components, + decomposition_levels, + scratch, + accelerator, + timings, + )?; + for (component_index, wavelet) in group.into_iter().zip(group_wavelets) { + let component = &components[component_index]; + let (x_rsiz, y_rsiz) = component_sampling[component_index]; + let actual_coefficients = flatten_integer_wavelet(&wavelet); + precomputed_53[component_index] = Some(PrecomputedHtj2k53Component { + x_rsiz, + y_rsiz, + dwt: j2k_dwt_from_integer_wavelet(&wavelet), + }); + + if options.validate_against_float_reference { + float_validation_actual.extend(actual_coefficients.clone()); + float_validation_expected.extend(float_reference_coefficients( + component, + decomposition_levels, + scratch, + )?); + } + if options.validate_against_integer_reference { + integer_validation_actual.extend(actual_coefficients); + integer_validation_expected.extend(integer_reference_coefficients( + component, + decomposition_levels, + )?); + } + } + } + + let float_reference_metrics = if options.validate_against_float_reference { + Some(error_metrics_i32( + &float_validation_actual, + &float_validation_expected, + )?) + } else { + None + }; + let integer_reference_metrics = if options.validate_against_integer_reference { + Some(error_metrics_i32( + &integer_validation_actual, + &integer_validation_expected, + )?) + } else { + None + }; + let precomputed_components = precomputed_53 + .into_iter() + .map(|component| { + component.ok_or(JpegToHtj2kError::Validation( + "integer transcode did not produce all components", + )) + }) + .collect::, _>>()?; + + Ok(ComponentTranscodeBatch { + precomputed_components: PrecomputedComponentBatch::Dwt53(precomputed_components), + float_reference_metrics, + integer_reference_metrics, + }) +} + +fn integer_wavelets_for_component_group( + group: &[usize], + components: &[JpegDctComponent], + decomposition_levels: u8, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut impl DctToWaveletStageAccelerator, + timings: &mut TranscodeTimingReport, +) -> Result, JpegToHtj2kError> { + let jobs = group + .iter() + .map(|&component_index| integer_dct_job_for_component(&components[component_index])) + .collect::, _>>()?; + record_batch_attempt(timings, group.len()); + let accelerator_start = Instant::now(); + let accelerated_first_levels = accelerator + .dct_grid_to_reversible_dwt53_batch(&jobs) + .map_err(JpegToHtj2kError::Accelerator)?; + timings.dct_to_wavelet_accelerator_us = timings + .dct_to_wavelet_accelerator_us + .saturating_add(accelerator_start.elapsed().as_micros()); + + if let Some(first_levels) = accelerated_first_levels { + if first_levels.len() != group.len() { + return Err(JpegToHtj2kError::Validation( + "reversible 5/3 batch accelerator returned wrong component count", + )); + } + timings.component_count = timings.component_count.saturating_add(group.len()); + record_accelerator_dispatch(timings, group.len()); + let decompose_start = Instant::now(); + let wavelets = first_levels + .into_iter() + .map(|first_level| integer_wavelet_from_first_level(first_level, decomposition_levels)) + .collect(); + timings.dwt_decompose_us = timings + .dwt_decompose_us + .saturating_add(decompose_start.elapsed().as_micros()); + return Ok(wavelets); + } + + group + .iter() + .map(|&component_index| { + integer_direct_wavelet_from_component( + &components[component_index], + decomposition_levels, + scratch, + accelerator, + timings, + ) + }) + .collect() +} + +fn same_geometry_component_groups(components: &[JpegDctComponent]) -> Vec> { + let mut assigned = vec![false; components.len()]; + let mut groups = Vec::new(); + + for component_index in 0..components.len() { + if assigned[component_index] { + continue; + } + assigned[component_index] = true; + let mut group = vec![component_index]; + for candidate_index in component_index + 1..components.len() { + if !assigned[candidate_index] + && same_component_geometry( + &components[component_index], + &components[candidate_index], + ) + { + assigned[candidate_index] = true; + group.push(candidate_index); + } + } + groups.push(group); + } + + groups +} + +fn same_component_geometry(left: &JpegDctComponent, right: &JpegDctComponent) -> bool { + left.width == right.width + && left.height == right.height + && left.block_cols == right.block_cols + && left.block_rows == right.block_rows +} + +fn integer_dct_job_for_component( + component: &JpegDctComponent, +) -> Result, JpegToHtj2kError> { + validate_component_block_grid(component)?; + Ok(DctGridToReversibleDwt53Job { + dequantized_blocks: &component.dequantized_blocks, + block_cols: component.block_cols as usize, + block_rows: component.block_rows as usize, + width: component.width as usize, + height: component.height as usize, + }) +} + +#[allow(clippy::too_many_arguments)] +fn component_to_precomputed_htj2k( + component: &JpegDctComponent, + x_rsiz: u8, + y_rsiz: u8, + decomposition_levels: u8, + options: &JpegToHtj2kOptions, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut impl DctToWaveletStageAccelerator, + timings: &mut TranscodeTimingReport, +) -> Result { + let (dwt, actual_coefficients) = match options.coefficient_path { + JpegToHtj2kCoefficientPath::IntegerDirect53 => { + let wavelet = integer_direct_wavelet_from_component( + component, + decomposition_levels, + scratch, + accelerator, + timings, + )?; + ( + PrecomputedComponent::Dwt53(PrecomputedHtj2k53Component { + x_rsiz, + y_rsiz, + dwt: j2k_dwt_from_integer_wavelet(&wavelet), + }), + flatten_integer_wavelet(&wavelet), + ) + } + JpegToHtj2kCoefficientPath::FloatDirectLinear53 => { + let wavelet = float_direct_wavelet_from_component( + component, + decomposition_levels, + scratch, + accelerator, + timings, + )?; + ( + PrecomputedComponent::Dwt53(PrecomputedHtj2k53Component { + x_rsiz, + y_rsiz, + dwt: j2k_dwt_from_wavelet( + &wavelet, + component.width as usize, + component.height as usize, + ), + }), + rounded_wavelet_i32(&wavelet)?, + ) + } + JpegToHtj2kCoefficientPath::FloatDirectLinear97 => { + let wavelet = float_direct_97_wavelet_from_component( + component, + decomposition_levels, + scratch, + accelerator, + timings, + )?; + ( + PrecomputedComponent::Dwt97(PrecomputedHtj2k97Component { + x_rsiz, + y_rsiz, + dwt: j2k_dwt97_from_wavelet( + &wavelet, + component.width as usize, + component.height as usize, + ), + }), + rounded_wavelet97_i32(&wavelet)?, + ) + } + }; + let float_validation_coefficients = if options.validate_against_float_reference { + let expected = match options.coefficient_path { + JpegToHtj2kCoefficientPath::FloatDirectLinear97 => { + float97_reference_coefficients(component, decomposition_levels, scratch)? + } + JpegToHtj2kCoefficientPath::IntegerDirect53 + | JpegToHtj2kCoefficientPath::FloatDirectLinear53 => { + float_reference_coefficients(component, decomposition_levels, scratch)? + } + }; + Some((actual_coefficients.clone(), expected)) + } else { + None + }; + let integer_validation_coefficients = if options.validate_against_integer_reference { + let expected = integer_reference_coefficients(component, decomposition_levels)?; + Some((actual_coefficients, expected)) + } else { + None + }; + + Ok(ComponentTranscodeResult { + precomputed: dwt, + float_validation_coefficients, + integer_validation_coefficients, + }) +} + +fn transcode_path_name( + all_unit_sampled: bool, + coefficient_path: JpegToHtj2kCoefficientPath, +) -> &'static str { + match (all_unit_sampled, coefficient_path) { + (true, JpegToHtj2kCoefficientPath::IntegerDirect53) => { + "full_resolution_components_integer_direct_53" + } + (false, JpegToHtj2kCoefficientPath::IntegerDirect53) => { + "native_component_sampling_integer_direct_53" + } + (true, JpegToHtj2kCoefficientPath::FloatDirectLinear53) => { + "full_resolution_components_float_direct_53" + } + (false, JpegToHtj2kCoefficientPath::FloatDirectLinear53) => { + "native_component_sampling_float_direct_53" + } + (true, JpegToHtj2kCoefficientPath::FloatDirectLinear97) => { + "full_resolution_components_float_direct_97" + } + (false, JpegToHtj2kCoefficientPath::FloatDirectLinear97) => { + "native_component_sampling_float_direct_97" + } + } +} + +fn float_direct_wavelet_from_component( + component: &JpegDctComponent, + decomposition_levels: u8, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut impl DctToWaveletStageAccelerator, + timings: &mut TranscodeTimingReport, +) -> Result { + timings.component_count = timings.component_count.saturating_add(1); + let repack_start = Instant::now(); + dct_blocks_to_8x8_f64_into(&component.dequantized_blocks, &mut scratch.dct_blocks_f64); + timings.jpeg_dct_repack_us = timings + .jpeg_dct_repack_us + .saturating_add(repack_start.elapsed().as_micros()); + let blocks = &scratch.dct_blocks_f64; + let job = DctGridToDwt53Job { + blocks, + block_cols: component.block_cols as usize, + block_rows: component.block_rows as usize, + width: component.width as usize, + height: component.height as usize, + }; + record_accelerator_attempt(timings, 1); + let accelerator_start = Instant::now(); + let accelerated = accelerator + .dct_grid_to_dwt53(job) + .map_err(JpegToHtj2kError::Accelerator)?; + timings.dct_to_wavelet_accelerator_us = timings + .dct_to_wavelet_accelerator_us + .saturating_add(accelerator_start.elapsed().as_micros()); + let bands = if let Some(bands) = accelerated { + record_accelerator_dispatch(timings, 1); + bands + } else { + record_cpu_fallback(timings, 1); + let fallback_start = Instant::now(); + let bands = dct8x8_blocks_to_dwt53_float_linear_with_scratch( + blocks, + component.block_cols as usize, + component.block_rows as usize, + component.width as usize, + component.height as usize, + &mut scratch.dct53_grid, + ) + .map_err(dct53_grid_error)?; + timings.dct_to_wavelet_cpu_fallback_us = timings + .dct_to_wavelet_cpu_fallback_us + .saturating_add(fallback_start.elapsed().as_micros()); + bands + }; + let decompose_start = Instant::now(); + let wavelet = decompose_from_first_level(bands, usize::from(decomposition_levels)); + timings.dwt_decompose_us = timings + .dwt_decompose_us + .saturating_add(decompose_start.elapsed().as_micros()); + Ok(wavelet) +} + +fn float_direct_97_wavelet_from_component( + component: &JpegDctComponent, + decomposition_levels: u8, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut impl DctToWaveletStageAccelerator, + timings: &mut TranscodeTimingReport, +) -> Result { + timings.component_count = timings.component_count.saturating_add(1); + let repack_start = Instant::now(); + dct_blocks_to_8x8_f64_into(&component.dequantized_blocks, &mut scratch.dct_blocks_f64); + timings.jpeg_dct_repack_us = timings + .jpeg_dct_repack_us + .saturating_add(repack_start.elapsed().as_micros()); + let blocks = &scratch.dct_blocks_f64; + let job = DctGridToDwt97Job { + blocks, + block_cols: component.block_cols as usize, + block_rows: component.block_rows as usize, + width: component.width as usize, + height: component.height as usize, + }; + record_accelerator_attempt(timings, 1); + let accelerator_start = Instant::now(); + let accelerated = accelerator + .dct_grid_to_dwt97(job) + .map_err(JpegToHtj2kError::Accelerator)?; + timings.dct_to_wavelet_accelerator_us = timings + .dct_to_wavelet_accelerator_us + .saturating_add(accelerator_start.elapsed().as_micros()); + let bands = if let Some(bands) = accelerated { + record_accelerator_dispatch(timings, 1); + bands + } else { + record_cpu_fallback(timings, 1); + let fallback_start = Instant::now(); + let bands = dct8x8_blocks_then_dwt97_float_with_scratch( + blocks, + component.block_cols as usize, + component.block_rows as usize, + component.width as usize, + component.height as usize, + &mut scratch.dct97_grid, + ) + .map_err(dct97_grid_error)?; + timings.dct_to_wavelet_cpu_fallback_us = timings + .dct_to_wavelet_cpu_fallback_us + .saturating_add(fallback_start.elapsed().as_micros()); + bands + }; + let decompose_start = Instant::now(); + let wavelet = decompose_97_from_first_level_with_scratch( + bands, + usize::from(decomposition_levels), + &mut scratch.dct97_grid, + ); + timings.dwt_decompose_us = timings + .dwt_decompose_us + .saturating_add(decompose_start.elapsed().as_micros()); + Ok(wavelet) +} + +fn float_reference_coefficients( + component: &JpegDctComponent, + decomposition_levels: u8, + scratch: &mut JpegToHtj2kScratch, +) -> Result, JpegToHtj2kError> { + dct_blocks_to_8x8_f64_into(&component.dequantized_blocks, &mut scratch.dct_blocks_f64); + let blocks = &scratch.dct_blocks_f64; + let first_reference_level = dct8x8_blocks_then_dwt53_float( + blocks, + component.block_cols as usize, + component.block_rows as usize, + component.width as usize, + component.height as usize, + ) + .map_err(dct53_grid_error)?; + let reference = + decompose_from_first_level(first_reference_level, usize::from(decomposition_levels)); + rounded_wavelet_i32(&reference) +} + +fn float97_reference_coefficients( + component: &JpegDctComponent, + decomposition_levels: u8, + scratch: &mut JpegToHtj2kScratch, +) -> Result, JpegToHtj2kError> { + dct_blocks_to_8x8_f64_into(&component.dequantized_blocks, &mut scratch.dct_blocks_f64); + let blocks = &scratch.dct_blocks_f64; + let first_reference_level = dct8x8_blocks_then_dwt97_float( + blocks, + component.block_cols as usize, + component.block_rows as usize, + component.width as usize, + component.height as usize, + ) + .map_err(dct97_grid_error)?; + let reference = + decompose_97_from_first_level(first_reference_level, usize::from(decomposition_levels)); + rounded_wavelet97_i32(&reference) +} + +fn decompose_from_first_level( + first_level: Dwt53TwoDimensional, + decomposition_levels: usize, +) -> ComponentWavelet { + let mut wavelet = ComponentWavelet { + final_ll: first_level.ll.clone(), + final_ll_width: first_level.low_width, + final_ll_height: first_level.low_height, + levels: vec![first_level], + }; + + while wavelet.levels.len() < decomposition_levels { + let next = linearized_53_2d_from_plane( + &wavelet.final_ll, + wavelet.final_ll_width, + wavelet.final_ll_height, + ); + wavelet.final_ll.clone_from(&next.ll); + wavelet.final_ll_width = next.low_width; + wavelet.final_ll_height = next.low_height; + wavelet.levels.push(next); + } + + wavelet +} + +fn decompose_97_from_first_level( + first_level: Dwt97TwoDimensional, + decomposition_levels: usize, +) -> ComponentWavelet97 { + let mut scratch = Dct97GridScratch::default(); + decompose_97_from_first_level_with_scratch(first_level, decomposition_levels, &mut scratch) +} + +fn decompose_97_from_first_level_with_scratch( + first_level: Dwt97TwoDimensional, + decomposition_levels: usize, + scratch: &mut Dct97GridScratch, +) -> ComponentWavelet97 { + let mut wavelet = ComponentWavelet97 { + final_ll: first_level.ll.clone(), + final_ll_width: first_level.low_width, + final_ll_height: first_level.low_height, + levels: vec![first_level], + }; + + while wavelet.levels.len() < decomposition_levels { + let next = linearized_97_2d_from_plane_with_scratch( + &wavelet.final_ll, + wavelet.final_ll_width, + wavelet.final_ll_height, + scratch, + ); + wavelet.final_ll.clone_from(&next.ll); + wavelet.final_ll_width = next.low_width; + wavelet.final_ll_height = next.low_height; + wavelet.levels.push(next); + } + + wavelet +} + +fn j2k_dwt_from_wavelet( + wavelet: &ComponentWavelet, + width: usize, + height: usize, +) -> J2kForwardDwt53Output { + let mut current_width = width; + let mut current_height = height; + let mut levels = Vec::with_capacity(wavelet.levels.len()); + + for level in &wavelet.levels { + levels.push(J2kForwardDwt53Level { + hl: level.hl.iter().map(|&value| value as f32).collect(), + lh: level.lh.iter().map(|&value| value as f32).collect(), + hh: level.hh.iter().map(|&value| value as f32).collect(), + width: current_width as u32, + height: current_height as u32, + low_width: level.low_width as u32, + low_height: level.low_height as u32, + high_width: level.high_width as u32, + high_height: level.high_height as u32, + }); + current_width = level.low_width; + current_height = level.low_height; + } + levels.reverse(); + + J2kForwardDwt53Output { + ll: wavelet.final_ll.iter().map(|&value| value as f32).collect(), + ll_width: wavelet.final_ll_width as u32, + ll_height: wavelet.final_ll_height as u32, + levels, + } +} + +fn j2k_dwt97_from_wavelet( + wavelet: &ComponentWavelet97, + width: usize, + height: usize, +) -> J2kForwardDwt97Output { + let mut current_width = width; + let mut current_height = height; + let mut levels = Vec::with_capacity(wavelet.levels.len()); + + for level in &wavelet.levels { + levels.push(J2kForwardDwt97Level { + hl: level.hl.iter().map(|&value| value as f32).collect(), + lh: level.lh.iter().map(|&value| value as f32).collect(), + hh: level.hh.iter().map(|&value| value as f32).collect(), + width: current_width as u32, + height: current_height as u32, + low_width: level.low_width as u32, + low_height: level.low_height as u32, + high_width: level.high_width as u32, + high_height: level.high_height as u32, + }); + current_width = level.low_width; + current_height = level.low_height; + } + levels.reverse(); + + J2kForwardDwt97Output { + ll: wavelet.final_ll.iter().map(|&value| value as f32).collect(), + ll_width: wavelet.final_ll_width as u32, + ll_height: wavelet.final_ll_height as u32, + levels, + } +} + +fn j2k_dwt_from_integer_wavelet(wavelet: &IntegerWavelet) -> J2kForwardDwt53Output { + let mut levels = Vec::with_capacity(wavelet.levels.len()); + for level in &wavelet.levels { + levels.push(J2kForwardDwt53Level { + hl: level.hl.iter().map(|&value| value as f32).collect(), + lh: level.lh.iter().map(|&value| value as f32).collect(), + hh: level.hh.iter().map(|&value| value as f32).collect(), + width: level.width as u32, + height: level.height as u32, + low_width: level.low_width as u32, + low_height: level.low_height as u32, + high_width: level.high_width as u32, + high_height: level.high_height as u32, + }); + } + levels.reverse(); + + J2kForwardDwt53Output { + ll: wavelet.final_ll.iter().map(|&value| value as f32).collect(), + ll_width: wavelet.final_ll_width as u32, + ll_height: wavelet.final_ll_height as u32, + levels, + } +} + +fn rounded_wavelet_i32(wavelet: &ComponentWavelet) -> Result, JpegToHtj2kError> { + let coefficient_count = wavelet.final_ll.len() + + wavelet + .levels + .iter() + .map(|level| level.hl.len() + level.lh.len() + level.hh.len()) + .sum::(); + let mut output = Vec::with_capacity(coefficient_count); + append_rounded_i32(&wavelet.final_ll, &mut output)?; + for level in wavelet.levels.iter().rev() { + append_rounded_i32(&level.hl, &mut output)?; + append_rounded_i32(&level.lh, &mut output)?; + append_rounded_i32(&level.hh, &mut output)?; + } + Ok(output) +} + +fn rounded_wavelet97_i32(wavelet: &ComponentWavelet97) -> Result, JpegToHtj2kError> { + let coefficient_count = wavelet.final_ll.len() + + wavelet + .levels + .iter() + .map(|level| level.hl.len() + level.lh.len() + level.hh.len()) + .sum::(); + let mut output = Vec::with_capacity(coefficient_count); + append_rounded_i32(&wavelet.final_ll, &mut output)?; + for level in wavelet.levels.iter().rev() { + append_rounded_i32(&level.hl, &mut output)?; + append_rounded_i32(&level.lh, &mut output)?; + append_rounded_i32(&level.hh, &mut output)?; + } + Ok(output) +} + +fn integer_direct_wavelet_from_component( + component: &JpegDctComponent, + decomposition_levels: u8, + scratch: &mut JpegToHtj2kScratch, + accelerator: &mut impl DctToWaveletStageAccelerator, + timings: &mut TranscodeTimingReport, +) -> Result { + let job = integer_dct_job_for_component(component)?; + timings.component_count = timings.component_count.saturating_add(1); + record_accelerator_attempt(timings, 1); + let accelerator_start = Instant::now(); + let accelerated_first_level = accelerator + .dct_grid_to_reversible_dwt53(job) + .map_err(JpegToHtj2kError::Accelerator)?; + timings.dct_to_wavelet_accelerator_us = timings + .dct_to_wavelet_accelerator_us + .saturating_add(accelerator_start.elapsed().as_micros()); + if let Some(first_level) = accelerated_first_level { + record_accelerator_dispatch(timings, 1); + let decompose_start = Instant::now(); + let wavelet = integer_wavelet_from_first_level(first_level, decomposition_levels); + timings.dwt_decompose_us = timings + .dwt_decompose_us + .saturating_add(decompose_start.elapsed().as_micros()); + return Ok(wavelet); + } + + scratch.integer_idct_blocks.clear(); + scratch + .integer_idct_blocks + .resize_with(component.dequantized_blocks.len(), || None); + record_cpu_fallback(timings, 1); + let fallback_start = Instant::now(); + let (final_ll, final_ll_width, final_ll_height, first_level) = + integer_direct_first_level_from_component( + component, + &mut scratch.integer_idct_blocks, + &mut scratch.integer_row, + )?; + timings.dct_to_wavelet_cpu_fallback_us = timings + .dct_to_wavelet_cpu_fallback_us + .saturating_add(fallback_start.elapsed().as_micros()); + let decompose_start = Instant::now(); + let wavelet = integer_wavelet_from_first_parts( + final_ll, + final_ll_width, + final_ll_height, + first_level, + decomposition_levels, + ); + timings.dwt_decompose_us = timings + .dwt_decompose_us + .saturating_add(decompose_start.elapsed().as_micros()); + Ok(wavelet) +} + +fn integer_wavelet_from_first_level( + first_level: ReversibleDwt53FirstLevel, + decomposition_levels: u8, +) -> IntegerWavelet { + let (final_ll, final_ll_width, final_ll_height, first_level) = + integer_wavelet_first_level_from_accelerated(first_level); + integer_wavelet_from_first_parts( + final_ll, + final_ll_width, + final_ll_height, + first_level, + decomposition_levels, + ) +} + +fn integer_wavelet_from_first_parts( + mut final_ll: Vec, + mut final_ll_width: usize, + mut final_ll_height: usize, + first_level: IntegerWaveletLevel, + decomposition_levels: u8, +) -> IntegerWavelet { + let mut levels = vec![first_level]; + + let remaining_levels = usize::from(decomposition_levels.saturating_sub(1)); + if remaining_levels > 0 { + let tail = + reversible_dwt53_i32(final_ll, final_ll_width, final_ll_height, remaining_levels); + final_ll = tail.final_ll; + final_ll_width = tail.final_ll_width; + final_ll_height = tail.final_ll_height; + levels.extend(tail.levels); + } + + IntegerWavelet { + final_ll, + final_ll_width, + final_ll_height, + levels, + } +} + +fn integer_wavelet_first_level_from_accelerated( + first_level: ReversibleDwt53FirstLevel, +) -> (Vec, usize, usize, IntegerWaveletLevel) { + let level = IntegerWaveletLevel { + width: first_level.low_width + first_level.high_width, + height: first_level.low_height + first_level.high_height, + low_width: first_level.low_width, + low_height: first_level.low_height, + high_width: first_level.high_width, + high_height: first_level.high_height, + hl: first_level.hl, + lh: first_level.lh, + hh: first_level.hh, + }; + ( + first_level.ll, + first_level.low_width, + first_level.low_height, + level, + ) +} + +fn integer_direct_first_level_from_component( + component: &JpegDctComponent, + idct_blocks: &mut [Option<[i32; 64]>], + row: &mut Vec, +) -> Result<(Vec, usize, usize, IntegerWaveletLevel), JpegToHtj2kError> { + let width = component.width as usize; + let height = component.height as usize; + let low_width = width.div_ceil(2); + let low_height = height.div_ceil(2); + let high_width = width / 2; + let high_height = height / 2; + + let mut ll = Vec::with_capacity(low_width * low_height); + let mut hl = Vec::with_capacity(high_width * low_height); + let mut lh = Vec::with_capacity(low_width * high_height); + let mut hh = Vec::with_capacity(high_width * high_height); + row.clear(); + if row.capacity() < width { + row.reserve(width - row.capacity()); + } + + for output_y in 0..low_height { + row.clear(); + for x in 0..width { + row.push(vertical_53_i32_at( + component, + idct_blocks, + x, + output_y, + true, + )?); + } + reversible_lift_53_i32(row); + ll.extend(row.iter().step_by(2).copied()); + hl.extend(row.iter().skip(1).step_by(2).copied()); + } + + for output_y in 0..high_height { + row.clear(); + for x in 0..width { + row.push(vertical_53_i32_at( + component, + idct_blocks, + x, + output_y, + false, + )?); + } + reversible_lift_53_i32(row); + lh.extend(row.iter().step_by(2).copied()); + hh.extend(row.iter().skip(1).step_by(2).copied()); + } + + let level = IntegerWaveletLevel { + width, + height, + low_width, + low_height, + high_width, + high_height, + hl, + lh, + hh, + }; + + Ok((ll, low_width, low_height, level)) +} + +fn vertical_53_i32_at( + component: &JpegDctComponent, + idct_blocks: &mut [Option<[i32; 64]>], + x: usize, + output_y: usize, + low_pass: bool, +) -> Result { + if low_pass { + vertical_low_53_i32_at(component, idct_blocks, x, output_y) + } else { + vertical_high_53_i32_at(component, idct_blocks, x, output_y) + } +} + +fn vertical_low_53_i32_at( + component: &JpegDctComponent, + idct_blocks: &mut [Option<[i32; 64]>], + x: usize, + low_idx: usize, +) -> Result { + let height = component.height as usize; + reversible_lift_53_low_at_fallible(height, low_idx, |y| { + component_sample_i32(component, idct_blocks, x, y) + }) +} + +fn vertical_high_53_i32_at( + component: &JpegDctComponent, + idct_blocks: &mut [Option<[i32; 64]>], + x: usize, + high_idx: usize, +) -> Result { + let height = component.height as usize; + reversible_lift_53_high_at_fallible(height, high_idx, |y| { + component_sample_i32(component, idct_blocks, x, y) + }) +} + +fn component_sample_i32( + component: &JpegDctComponent, + idct_blocks: &mut [Option<[i32; 64]>], + x: usize, + y: usize, +) -> Result { + if x >= component.width as usize || y >= component.height as usize { + return Err(JpegToHtj2kError::Validation( + "component sample coordinate exceeds dimensions", + )); + } + let block_cols = component.block_cols as usize; + let block_x = x / 8; + let block_y = y / 8; + let block_idx = block_y * block_cols + block_x; + let block = component + .dequantized_blocks + .get(block_idx) + .ok_or(JpegToHtj2kError::Validation( + "component block grid does not cover requested sample", + ))?; + let cached = idct_blocks + .get_mut(block_idx) + .ok_or(JpegToHtj2kError::Validation( + "integer IDCT cache does not cover requested block", + ))?; + let block_samples = cached.get_or_insert_with(|| { + let decoded = idct_islow_block(block); + decoded.map(|sample| i32::from(sample) - 128) + }); + let local_idx = (y % 8) * 8 + (x % 8); + Ok(block_samples[local_idx]) +} + +fn integer_reference_coefficients( + component: &JpegDctComponent, + decomposition_levels: u8, +) -> Result, JpegToHtj2kError> { + let samples = idct_component_samples_i32(component)?; + let wavelet = reversible_dwt53_i32( + samples, + component.width as usize, + component.height as usize, + usize::from(decomposition_levels), + ); + Ok(flatten_integer_wavelet(&wavelet)) +} + +fn idct_component_samples_i32(component: &JpegDctComponent) -> Result, JpegToHtj2kError> { + validate_component_block_grid(component)?; + + let width = component.width as usize; + let height = component.height as usize; + let block_cols = component.block_cols as usize; + let block_rows = component.block_rows as usize; + let mut samples = vec![0; width * height]; + for block_y in 0..block_rows { + for block_x in 0..block_cols { + let block = &component.dequantized_blocks[block_y * block_cols + block_x]; + let block_samples = idct_islow_block(block); + for local_y in 0..8 { + let y = block_y * 8 + local_y; + if y >= height { + continue; + } + for local_x in 0..8 { + let x = block_x * 8 + local_x; + if x >= width { + continue; + } + samples[y * width + x] = i32::from(block_samples[local_y * 8 + local_x]) - 128; + } + } + } + } + + Ok(samples) +} + +fn validate_component_block_grid(component: &JpegDctComponent) -> Result<(), JpegToHtj2kError> { + let block_cols = component.block_cols as usize; + let block_rows = component.block_rows as usize; + let expected_blocks = + block_cols + .checked_mul(block_rows) + .ok_or(JpegToHtj2kError::Validation( + "component block grid overflow", + ))?; + if component.dequantized_blocks.len() != expected_blocks { + return Err(JpegToHtj2kError::Validation( + "component block count does not match block grid", + )); + } + + Ok(()) +} + +fn reversible_dwt53_i32( + mut buffer: Vec, + width: usize, + height: usize, + decomposition_levels: usize, +) -> IntegerWavelet { + let mut current_width = width; + let mut current_height = height; + let mut levels = Vec::with_capacity(decomposition_levels); + + for _ in 0..decomposition_levels { + for x in 0..current_width { + let mut column = Vec::with_capacity(current_height); + for y in 0..current_height { + column.push(buffer[y * width + x]); + } + reversible_lift_53_i32(&mut column); + let low_len = current_height.div_ceil(2); + for (idx, value) in column.iter().step_by(2).copied().enumerate() { + buffer[idx * width + x] = value; + } + for (idx, value) in column.iter().skip(1).step_by(2).copied().enumerate() { + buffer[(low_len + idx) * width + x] = value; + } + } + + for y in 0..current_height { + let row_start = y * width; + let mut row = buffer[row_start..row_start + current_width].to_vec(); + reversible_lift_53_i32(&mut row); + let low_len = current_width.div_ceil(2); + for (idx, value) in row.iter().step_by(2).copied().enumerate() { + buffer[row_start + idx] = value; + } + for (idx, value) in row.iter().skip(1).step_by(2).copied().enumerate() { + buffer[row_start + low_len + idx] = value; + } + } + + let low_width = current_width.div_ceil(2); + let low_height = current_height.div_ceil(2); + let high_width = current_width / 2; + let high_height = current_height / 2; + let mut hl = Vec::with_capacity(high_width * low_height); + let mut lh = Vec::with_capacity(low_width * high_height); + let mut hh = Vec::with_capacity(high_width * high_height); + + for y in 0..low_height { + for x in 0..high_width { + hl.push(buffer[y * width + low_width + x]); + } + } + for y in 0..high_height { + for x in 0..low_width { + lh.push(buffer[(low_height + y) * width + x]); + } + } + for y in 0..high_height { + for x in 0..high_width { + hh.push(buffer[(low_height + y) * width + low_width + x]); + } + } + + levels.push(IntegerWaveletLevel { + width: current_width, + height: current_height, + low_width, + low_height, + high_width, + high_height, + hl, + lh, + hh, + }); + current_width = low_width; + current_height = low_height; + } + + let mut final_ll = Vec::with_capacity(current_width * current_height); + for y in 0..current_height { + for x in 0..current_width { + final_ll.push(buffer[y * width + x]); + } + } + + IntegerWavelet { + final_ll, + final_ll_width: current_width, + final_ll_height: current_height, + levels, + } +} + +fn flatten_integer_wavelet(wavelet: &IntegerWavelet) -> Vec { + let coefficient_count = wavelet.final_ll.len() + + wavelet + .levels + .iter() + .map(|level| level.hl.len() + level.lh.len() + level.hh.len()) + .sum::(); + let mut output = Vec::with_capacity(coefficient_count); + output.extend_from_slice(&wavelet.final_ll); + for level in wavelet.levels.iter().rev() { + output.extend_from_slice(&level.hl); + output.extend_from_slice(&level.lh); + output.extend_from_slice(&level.hh); + } + output +} + +fn append_rounded_i32(values: &[f64], output: &mut Vec) -> Result<(), JpegToHtj2kError> { + for &value in values { + output.push(round_f64_to_i32(value)?); + } + Ok(()) +} + +fn round_f64_to_i32(value: f64) -> Result { + let rounded = value.round(); + if !rounded.is_finite() { + return Err(JpegToHtj2kError::Validation( + "float reference coefficient is not finite", + )); + } + if rounded < f64::from(i32::MIN) || rounded > f64::from(i32::MAX) { + return Err(JpegToHtj2kError::Validation( + "float reference coefficient exceeds i32 range", + )); + } + Ok(rounded as i32) +} + +fn decomposition_levels_for_components( + components: &[JpegDctComponent], + requested_levels: u8, +) -> Result { + if requested_levels == 0 { + return Err(JpegToHtj2kError::Unsupported( + "jpeg_to_htj2k requires at least one decomposition level", + )); + } + + let available_levels = components + .iter() + .map(|component| available_decomposition_levels(component.width, component.height)) + .min() + .ok_or(JpegToHtj2kError::Unsupported("missing JPEG components"))?; + let decomposition_levels = requested_levels.min(available_levels); + if decomposition_levels == 0 { + return Err(JpegToHtj2kError::Unsupported( + "component dimensions are too small for a DWT decomposition", + )); + } + + Ok(decomposition_levels) +} + +fn available_decomposition_levels(width: u32, height: u32) -> u8 { + let min_dim = width.min(height); + if min_dim <= 1 { + 0 + } else { + min_dim.ilog2() as u8 + } +} + +fn component_sampling_for_jpeg( + components: &[JpegDctComponent], + reference_width: u32, + reference_height: u32, +) -> Result, JpegToHtj2kError> { + let max_h = components + .iter() + .map(|component| component.h_samp) + .max() + .ok_or(JpegToHtj2kError::Unsupported("missing JPEG components"))?; + let max_v = components + .iter() + .map(|component| component.v_samp) + .max() + .ok_or(JpegToHtj2kError::Unsupported("missing JPEG components"))?; + + components + .iter() + .map(|component| { + if component.h_samp == 0 || component.v_samp == 0 { + return Err(JpegToHtj2kError::Unsupported( + "JPEG component sampling factors must be non-zero", + )); + } + if max_h % component.h_samp != 0 || max_v % component.v_samp != 0 { + return Err(JpegToHtj2kError::Unsupported( + "fractional JPEG component sampling is not supported", + )); + } + + let x_rsiz = max_h / component.h_samp; + let y_rsiz = max_v / component.v_samp; + let expected_width = reference_width.div_ceil(u32::from(x_rsiz)); + let expected_height = reference_height.div_ceil(u32::from(y_rsiz)); + if component.width != expected_width || component.height != expected_height { + return Err(JpegToHtj2kError::Unsupported( + "JPEG component dimensions do not match derived SIZ sampling", + )); + } + + Ok((x_rsiz, y_rsiz)) + }) + .collect() +} + +fn dct_blocks_to_8x8_f64_into(blocks: &[[i16; 64]], output: &mut Vec<[[f64; 8]; 8]>) { + output.clear(); + output.reserve(blocks.len()); + for block in blocks { + let mut converted = [[0.0; 8]; 8]; + for (idx, &coefficient) in block.iter().enumerate() { + converted[idx / 8][idx % 8] = f64::from(coefficient); + } + output.push(converted); + } +} + +fn dct_blocks_to_8x8_f64(blocks: &[[i16; 64]]) -> Vec<[[f64; 8]; 8]> { + let mut output = Vec::with_capacity(blocks.len()); + dct_blocks_to_8x8_f64_into(blocks, &mut output); + output +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::accelerator::{ + DctGridI16ToHtj2k97CodeBlockBatch, PreencodedHtj2k97CodeBlock, + PreencodedHtj2k97CompactCodeBlock, PreencodedHtj2k97CompactComponent, + PreencodedHtj2k97CompactResolution, PreencodedHtj2k97CompactSubband, + PreencodedHtj2k97Resolution, PreencodedHtj2k97Subband, + }; + use j2k::adapter::encode_stage::{EncodedHtJ2kCodeBlock, J2kHtCodeBlockEncodeJob}; + use j2k_jpeg::transcode::JpegDctCodingMode; + use j2k_jpeg::ColorSpace; + + #[test] + fn timing_report_add_assign_saturates_and_adds_all_counter_kinds() { + let mut report = TranscodeTimingReport { + source_raw_probe_us: u128::MAX - 1, + dwt97_batch_ht_codeblock_dispatches: usize::MAX - 1, + tile_count: 2, + accelerator_jobs: 3, + cpu_fallback_jobs: 4, + ..TranscodeTimingReport::default() + }; + report.add_assign(TranscodeTimingReport { + source_raw_probe_us: 10, + dwt97_batch_ht_codeblock_dispatches: 10, + tile_count: 5, + accelerator_jobs: 7, + cpu_fallback_jobs: 11, + ..TranscodeTimingReport::default() + }); + + assert_eq!(report.source_raw_probe_us, u128::MAX); + assert_eq!(report.dwt97_batch_ht_codeblock_dispatches, usize::MAX); + assert_eq!(report.tile_count, 7); + assert_eq!(report.accelerator_jobs, 10); + assert_eq!(report.cpu_fallback_jobs, 15); + } + + #[derive(Default)] + struct GroupedI16Accelerator { + grouped_calls: usize, + single_calls: usize, + grouped_lengths: Vec>, + } + + impl DctToWaveletStageAccelerator for GroupedI16Accelerator { + fn supports_htj2k97_i16_preencoded_batch(&self) -> bool { + true + } + + fn dct_grid_i16_to_htj2k97_preencoded_batch( + &mut self, + jobs: &[DctGridI16ToHtj2k97CodeBlockJob<'_>], + _options: Htj2k97CodeBlockOptions, + ) -> Result>, TranscodeStageError> { + self.single_calls = self.single_calls.saturating_add(1); + Ok(Some( + jobs.iter() + .map(|job| dummy_preencoded_component(job.x_rsiz, job.y_rsiz)) + .collect(), + )) + } + + fn dct_grid_i16_to_htj2k97_preencoded_batch_groups( + &mut self, + groups: &[DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], + _options: Htj2k97CodeBlockOptions, + ) -> Result>>, TranscodeStageError> { + self.grouped_calls = self.grouped_calls.saturating_add(1); + self.grouped_lengths + .push(groups.iter().map(|group| group.jobs.len()).collect()); + Ok(Some( + groups + .iter() + .map(|group| { + group + .jobs + .iter() + .map(|job| dummy_preencoded_component(job.x_rsiz, job.y_rsiz)) + .collect() + }) + .collect(), + )) + } + } + + #[test] + fn float97_batch_offers_i16_preencoded_geometry_groups_together() { + let mut tiles = vec![test_float97_tile()]; + let options = JpegToHtj2kOptions::lossy_97(); + let mut scratch = JpegToHtj2kScratch::default(); + let mut accelerator = GroupedI16Accelerator::default(); + let mut timings = TranscodeTimingReport::default(); + + let (batch_count, job_count) = transform_float97_batch_tiles( + &mut tiles, + &options, + &mut scratch, + &mut accelerator, + &mut timings, + ) + .expect("grouped i16 preencoded transform"); + + assert_eq!(batch_count, 2); + assert_eq!(job_count, 3); + assert_eq!(accelerator.grouped_calls, 1); + assert_eq!(accelerator.single_calls, 0); + assert_eq!(accelerator.grouped_lengths, vec![vec![1, 2]]); + assert!(tiles[0].preencoded_components.iter().all(Option::is_some)); + } + + #[derive(Default)] + struct CountingHtBatchEncodeAccelerator { + batches: usize, + jobs: usize, + single_blocks: usize, + } + + impl J2kEncodeStageAccelerator for CountingHtBatchEncodeAccelerator { + fn encode_ht_code_blocks( + &mut self, + jobs: &[J2kHtCodeBlockEncodeJob<'_>], + ) -> Result>, &'static str> { + self.batches = self.batches.saturating_add(1); + self.jobs = self.jobs.saturating_add(jobs.len()); + Ok(None) + } + + fn encode_ht_code_block( + &mut self, + _job: J2kHtCodeBlockEncodeJob<'_>, + ) -> Result, &'static str> { + self.single_blocks = self.single_blocks.saturating_add(1); + Ok(None) + } + } + + #[test] + fn float97_precomputed_prepared_tiles_offer_all_tiles_to_one_ht_batch() { + let tiles = vec![ + test_float97_precomputed_tile(0), + test_float97_precomputed_tile(1), + ]; + let mut options = JpegToHtj2kOptions::lossy_97(); + options.encode_options.code_block_width_exp = 2; + options.encode_options.code_block_height_exp = 2; + let mut accelerator = CountingHtBatchEncodeAccelerator::default(); + + let encoded_tiles = encode_float97_prepared_tiles(tiles, &options, &mut accelerator); + + assert_eq!(encoded_tiles.len(), 2); + for (expected_tile_index, (actual_tile_index, encoded)) in + encoded_tiles.into_iter().enumerate() + { + assert_eq!(actual_tile_index, expected_tile_index); + let encoded = encoded.expect("precomputed batch tile encodes"); + assert!(encoded.codestream.starts_with(&[0xff, 0x4f])); + } + assert_eq!(accelerator.batches, 1); + assert!(accelerator.jobs > 0); + assert_eq!(accelerator.single_blocks, accelerator.jobs); + } + + #[test] + fn compact_preencoded_component_storage_rebases_ranges_into_tile_payload() { + let mut tile = test_float97_tile(); + let batch_payload = vec![1, 2, 3, 4, 5, 6]; + let component = PreencodedHtj2k97CompactComponent { + x_rsiz: 1, + y_rsiz: 1, + resolutions: vec![PreencodedHtj2k97CompactResolution { + subbands: vec![PreencodedHtj2k97CompactSubband { + sub_band_type: crate::accelerator::J2kSubBandType::LowLow, + num_cbs_x: 2, + num_cbs_y: 1, + total_bitplanes: 1, + code_blocks: vec![ + PreencodedHtj2k97CompactCodeBlock { + width: 1, + height: 1, + payload_range: 1..3, + cleanup_length: 2, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 0, + }, + PreencodedHtj2k97CompactCodeBlock { + width: 1, + height: 1, + payload_range: 3..6, + cleanup_length: 3, + refinement_length: 0, + num_coding_passes: 1, + num_zero_bitplanes: 0, + }, + ], + }], + }], + }; + + store_compact_preencoded_component(&mut tile, 1, &batch_payload, component) + .expect("compact component storage"); + + let stored = tile.preencoded_compact_components[1] + .as_ref() + .expect("stored compact component"); + assert_eq!(tile.preencoded_compact_payload, vec![2, 3, 4, 5, 6]); + assert_eq!( + stored.resolutions[0].subbands[0].code_blocks[0].payload_range, + 0..2 + ); + assert_eq!( + stored.resolutions[0].subbands[0].code_blocks[1].payload_range, + 2..5 + ); + } + + fn test_float97_tile() -> Float97BatchTile { + let components = vec![ + test_component(0, 16, 16, 2, 2), + test_component(1, 8, 8, 1, 1), + test_component(2, 8, 8, 1, 1), + ]; + Float97BatchTile { + tile_index: 0, + jpeg: JpegDctImage { + width: 16, + height: 16, + color_space: ColorSpace::YCbCr, + coding_mode: JpegDctCodingMode::BaselineSequential, + scan_count: 1, + components, + restart_index: None, + }, + component_sampling: vec![(1, 1), (2, 2), (2, 2)], + decomposition_levels: 1, + all_unit_sampled: false, + component_reports: Vec::new(), + precomputed_components: vec![None, None, None], + preencoded_compact_payload: Vec::new(), + preencoded_compact_components: vec![None, None, None], + preencoded_components: vec![None, None, None], + prequantized_components: vec![None, None, None], + float_validation_actual: Vec::new(), + float_validation_expected: Vec::new(), + timings: TranscodeTimingReport::default(), + } + } + + fn test_float97_precomputed_tile(tile_index: usize) -> Float97BatchTile { + let width = 17; + let height = 13; + let component = test_component(0, width, height, 1, 1); + Float97BatchTile { + tile_index, + jpeg: JpegDctImage { + width, + height, + color_space: ColorSpace::Grayscale, + coding_mode: JpegDctCodingMode::BaselineSequential, + scan_count: 1, + components: vec![component], + restart_index: None, + }, + component_sampling: vec![(1, 1)], + decomposition_levels: 1, + all_unit_sampled: true, + component_reports: vec![TranscodeComponentReport { + component_index: 0, + width, + height, + block_cols: width.div_ceil(8), + block_rows: height.div_ceil(8), + x_rsiz: 1, + y_rsiz: 1, + }], + precomputed_components: vec![Some(dummy_precomputed_component(1, 1, width, height))], + preencoded_compact_payload: Vec::new(), + preencoded_compact_components: vec![None], + preencoded_components: vec![None], + prequantized_components: vec![None], + float_validation_actual: Vec::new(), + float_validation_expected: Vec::new(), + timings: TranscodeTimingReport::default(), + } + } + + fn test_component( + component_index: usize, + width: u32, + height: u32, + h_samp: u8, + v_samp: u8, + ) -> JpegDctComponent { + let block_cols = width.div_ceil(8); + let block_rows = height.div_ceil(8); + let block_count = (block_cols * block_rows) as usize; + JpegDctComponent { + component_index, + width, + height, + h_samp, + v_samp, + block_cols, + block_rows, + quant_table: [1u16; 64], + quantized_blocks: vec![[0i16; 64]; block_count], + dequantized_blocks: vec![[0i16; 64]; block_count], + } + } + + fn dummy_precomputed_component( + x_rsiz: u8, + y_rsiz: u8, + width: u32, + height: u32, + ) -> PrecomputedHtj2k97Component { + let low_width = width.div_ceil(2); + let low_height = height.div_ceil(2); + let high_width = width / 2; + let high_height = height / 2; + PrecomputedHtj2k97Component { + x_rsiz, + y_rsiz, + dwt: J2kForwardDwt97Output { + ll: sample_f32_coefficients(low_width * low_height, 0.25), + ll_width: low_width, + ll_height: low_height, + levels: vec![J2kForwardDwt97Level { + hl: sample_f32_coefficients(high_width * low_height, -0.75), + lh: sample_f32_coefficients(low_width * high_height, 1.25), + hh: sample_f32_coefficients(high_width * high_height, -1.5), + width, + height, + low_width, + low_height, + high_width, + high_height, + }], + }, + } + } + + fn sample_f32_coefficients(count: u32, seed: f32) -> Vec { + (0..count) + .map(|idx| seed + (idx as f32).sin() * 0.125) + .collect() + } + + fn dummy_preencoded_component(x_rsiz: u8, y_rsiz: u8) -> PreencodedHtj2k97Component { + PreencodedHtj2k97Component { + x_rsiz, + y_rsiz, + resolutions: vec![PreencodedHtj2k97Resolution { + subbands: vec![PreencodedHtj2k97Subband { + sub_band_type: crate::accelerator::J2kSubBandType::LowLow, + num_cbs_x: 1, + num_cbs_y: 1, + total_bitplanes: 1, + code_blocks: vec![PreencodedHtj2k97CodeBlock { + width: 1, + height: 1, + encoded: EncodedHtJ2kCodeBlock { + data: Vec::new(), + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 0, + num_zero_bitplanes: 1, + }, + }], + }], + }], + } + } +} diff --git a/crates/j2k-transcode/src/lib.rs b/crates/j2k-transcode/src/lib.rs new file mode 100644 index 00000000..de9cece5 --- /dev/null +++ b/crates/j2k-transcode/src/lib.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! JPEG-to-HTJ2K coefficient-domain transcode workflow. + +pub mod accelerator; + +#[doc(hidden)] +pub mod corpus_validation; +#[doc(hidden)] +pub mod dct53_1d; +pub mod dct53_2d; +#[doc(hidden)] +pub mod dct53_multilevel; +pub mod dct97_2d; +mod dct_grid; +pub mod htj2k97_codeblock_oracle; +#[doc(hidden)] +mod jpeg_to_htj2k; +#[doc(hidden)] +pub mod metrics; +mod pipeline_map; +mod reversible53; + +pub use j2k::J2kProgressionOrder as EncodeProgressionOrder; + +pub use dct_grid::DctGridError; +pub use jpeg_to_htj2k::{ + jpeg_to_htj2k, jpeg_to_htj2k_batch, BatchTranscodeReport, EncodedTranscode, + EncodedTranscodeBatch, JpegTileBatchInput, JpegToHtj2kCoefficientPath, + JpegToHtj2kEncodeOptions, JpegToHtj2kError, JpegToHtj2kOptions, JpegToHtj2kTranscoder, + TranscodeComponentReport, TranscodeReport, TranscodeTimingReport, + TranscodeValidationClassification, TranscodeValidationMetrics, + JPEG_TO_HTJ2K_LOSSY_97_QUANTIZATION_SCALE, +}; +pub use pipeline_map::{ + TranscodePipelineMap, TranscodePipelineStageKind, TranscodePipelineStageReport, + TranscodeResidentStageRecommendation, TranscodeStageProcessor, +}; diff --git a/crates/j2k-transcode/src/metrics.rs b/crates/j2k-transcode/src/metrics.rs new file mode 100644 index 00000000..a47c46cd --- /dev/null +++ b/crates/j2k-transcode/src/metrics.rs @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Error metrics for coefficient-domain validation. + +use core::fmt; +use std::collections::BTreeMap; + +/// Difference summary between two integer coefficient vectors. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ErrorMetrics { + /// Number of compared coefficients. + pub total: usize, + /// Number of coefficients with exact equality. + pub exact_matches: usize, + /// Maximum absolute coefficient error. + pub max_abs_error: i64, + /// Absolute-error histogram keyed by LSB distance. + pub absolute_error_histogram: BTreeMap, +} + +impl ErrorMetrics { + /// Fraction of coefficients that match exactly. + #[must_use] + pub fn exact_match_rate(&self) -> f64 { + if self.total == 0 { + return 1.0; + } + + self.exact_matches as f64 / self.total as f64 + } + + /// Number of coefficients at the given absolute error. + #[must_use] + pub fn absolute_error_count(&self, absolute_error: i64) -> usize { + self.absolute_error_histogram + .get(&absolute_error) + .copied() + .unwrap_or(0) + } + + /// Whether the metrics satisfy a one-LSB-bounded claim at the requested + /// exact-match threshold. + #[must_use] + pub fn is_one_lsb_bounded(&self, exact_match_threshold: f64) -> bool { + self.max_abs_error <= 1 && self.exact_match_rate() >= exact_match_threshold + } +} + +/// Error returned when metric inputs do not describe the same coefficient set. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MetricsLengthError { + actual_len: usize, + expected_len: usize, +} + +impl MetricsLengthError { + /// Length of the actual coefficient slice. + #[must_use] + pub const fn actual_len(self) -> usize { + self.actual_len + } + + /// Length of the expected coefficient slice. + #[must_use] + pub const fn expected_len(self) -> usize { + self.expected_len + } +} + +impl fmt::Display for MetricsLengthError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "metric input lengths differ: actual {}, expected {}", + self.actual_len, self.expected_len + ) + } +} + +impl std::error::Error for MetricsLengthError {} + +/// Compute exact-match rate, max absolute error, and absolute-LSB histogram for +/// two integer coefficient vectors. +pub fn error_metrics_i32( + actual: &[i32], + expected: &[i32], +) -> Result { + if actual.len() != expected.len() { + return Err(MetricsLengthError { + actual_len: actual.len(), + expected_len: expected.len(), + }); + } + + let mut exact_matches = 0; + let mut max_abs_error = 0; + let mut absolute_error_histogram = BTreeMap::new(); + + for (&actual, &expected) in actual.iter().zip(expected.iter()) { + let abs_error = (i64::from(actual) - i64::from(expected)).abs(); + if abs_error == 0 { + exact_matches += 1; + } + max_abs_error = max_abs_error.max(abs_error); + *absolute_error_histogram.entry(abs_error).or_insert(0) += 1; + } + + Ok(ErrorMetrics { + total: actual.len(), + exact_matches, + max_abs_error, + absolute_error_histogram, + }) +} diff --git a/crates/j2k-transcode/src/pipeline_map.rs b/crates/j2k-transcode/src/pipeline_map.rs new file mode 100644 index 00000000..fb2e9c7b --- /dev/null +++ b/crates/j2k-transcode/src/pipeline_map.rs @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Stage-level residency report for JPEG-to-HTJ2K transcode timings. + +use core::fmt::{self, Write as _}; + +use crate::{BatchTranscodeReport, TranscodeReport, TranscodeTimingReport}; + +/// Logical stages in the JPEG-to-J2K/HTJ2K transcode pipeline. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TranscodePipelineStageKind { + /// JPEG marker parsing, entropy decode, and DCT coefficient extraction. + EntropyDecode, + /// Coefficient repacking and host/device input preparation. + CoefficientPrep, + /// DCT-grid to wavelet-domain transform. + Transform, + /// Quantization, code-block layout, and pre-packet code-block work. + QuantizationCodeBlockPrep, + /// Packet header and packet body formation. + Packetization, + /// Final marker, tile-part, and codestream byte assembly. + CodestreamAssembly, +} + +impl TranscodePipelineStageKind { + /// Stable snake-case label used by debug reports and logs. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::EntropyDecode => "entropy_decode", + Self::CoefficientPrep => "coefficient_prep", + Self::Transform => "transform", + Self::QuantizationCodeBlockPrep => "quantization_code_block_prep", + Self::Packetization => "packetization", + Self::CodestreamAssembly => "codestream_assembly", + } + } +} + +impl fmt::Display for TranscodePipelineStageKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Observed residency for a transcode stage. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TranscodeStageProcessor { + /// Work is currently observed at CPU/native Rust or native encoder boundaries. + Cpu, + /// Work is observed through the Metal/accelerator stage counters. + Metal, + /// Existing counters show both CPU and Metal/accelerator work for this stage. + Hybrid, +} + +impl TranscodeStageProcessor { + /// Stable label used by debug reports and logs. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Cpu => "Cpu", + Self::Metal => "Metal", + Self::Hybrid => "Hybrid", + } + } +} + +impl fmt::Display for TranscodeStageProcessor { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One stage in a transcode pipeline residency map. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TranscodePipelineStageReport { + /// Logical transcode stage. + pub stage: TranscodePipelineStageKind, + /// Observed CPU/Metal residency for this stage. + pub processor: TranscodeStageProcessor, + /// CPU/native time currently visible for this stage, in microseconds. + pub cpu_us: u128, + /// Metal/accelerator time currently visible for this stage, in microseconds. + pub metal_us: u128, + /// Host/device transfer time visible for this stage, in microseconds. + pub transfer_us: u128, + /// Dispatches observed for this stage. + pub dispatches: usize, + /// Component jobs that used CPU fallback at this stage. + pub fallback_jobs: usize, + /// Short interpretation of the counters behind this stage. + pub note: &'static str, +} + +/// Recommended next stage to evaluate for Metal residency. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TranscodeResidentStageRecommendation { + /// Stage that should be evaluated next. + pub stage: TranscodePipelineStageKind, + /// Existing measured time supporting the recommendation, in microseconds. + pub evidence_us: u128, + /// Existing dispatch count supporting the recommendation. + pub evidence_dispatches: usize, + /// Why this stage is the next candidate. + pub reason: &'static str, +} + +/// Stage-by-stage transcode residency map derived from existing timings. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TranscodePipelineMap { + /// Ordered stage reports from JPEG input through codestream output. + pub stages: Vec, + /// Next resident-stage candidate derived from the observed counters. + pub recommendation: TranscodeResidentStageRecommendation, +} + +impl TranscodePipelineMap { + /// Build a pipeline map from an existing timing report. + #[must_use] + pub fn from_timings(timings: &TranscodeTimingReport) -> Self { + Self { + stages: vec![ + entropy_decode_stage(timings), + coefficient_prep_stage(timings), + transform_stage(timings), + quantization_code_block_stage(timings), + packetization_stage(timings), + codestream_assembly_stage(timings), + ], + recommendation: recommend_next_resident_stage(timings), + } + } + + /// Render a compact, line-oriented report for benchmark/debug output. + #[must_use] + pub fn debug_report(&self) -> String { + let mut output = String::from("jpeg_to_htj2k_pipeline_map\n"); + for stage in &self.stages { + writeln!( + output, + "stage={} processor={} cpu_us={} metal_us={} transfer_us={} dispatches={} fallback_jobs={} note={}", + stage.stage, + stage.processor, + stage.cpu_us, + stage.metal_us, + stage.transfer_us, + stage.dispatches, + stage.fallback_jobs, + stage.note, + ) + .expect("writing a transcode pipeline debug report to a String cannot fail"); + } + writeln!( + output, + "recommend_next_stage={} evidence_us={} evidence_dispatches={} reason={}", + self.recommendation.stage, + self.recommendation.evidence_us, + self.recommendation.evidence_dispatches, + self.recommendation.reason, + ) + .expect("writing a transcode pipeline recommendation to a String cannot fail"); + output + } +} + +impl TranscodeTimingReport { + /// Convert this timing report into a CPU/Metal transcode pipeline map. + #[must_use] + pub fn pipeline_map(&self) -> TranscodePipelineMap { + TranscodePipelineMap::from_timings(self) + } +} + +impl TranscodeReport { + /// Convert this transcode report into a CPU/Metal pipeline map. + #[must_use] + pub fn pipeline_map(&self) -> TranscodePipelineMap { + self.timings.pipeline_map() + } +} + +impl BatchTranscodeReport { + /// Convert this batch transcode report into a CPU/Metal pipeline map. + #[must_use] + pub fn pipeline_map(&self) -> TranscodePipelineMap { + self.timings.pipeline_map() + } +} + +fn entropy_decode_stage(timings: &TranscodeTimingReport) -> TranscodePipelineStageReport { + TranscodePipelineStageReport { + stage: TranscodePipelineStageKind::EntropyDecode, + processor: TranscodeStageProcessor::Cpu, + cpu_us: timings.jpeg_dct_extract_us, + metal_us: 0, + transfer_us: 0, + dispatches: 0, + fallback_jobs: 0, + note: "JPEG marker parsing, entropy decode, dequantization, and DCT coefficient extraction stay on CPU", + } +} + +fn coefficient_prep_stage(timings: &TranscodeTimingReport) -> TranscodePipelineStageReport { + let transfer_us = timings.dwt97_batch_pack_upload_us; + TranscodePipelineStageReport { + stage: TranscodePipelineStageKind::CoefficientPrep, + processor: processor_for( + timings.jpeg_dct_repack_us, + 0, + transfer_us, + usize::from(transfer_us > 0), + 0, + ), + cpu_us: timings.jpeg_dct_repack_us, + metal_us: 0, + transfer_us, + dispatches: usize::from(transfer_us > 0), + fallback_jobs: 0, + note: "DCT coefficient repack and Metal buffer pack/upload are visible before transform dispatch", + } +} + +fn transform_stage(timings: &TranscodeTimingReport) -> TranscodePipelineStageReport { + let dwt97_kernel_us = timings + .dwt97_batch_idct_row_lift_us + .saturating_add(timings.dwt97_batch_column_lift_us); + let metal_us = if dwt97_kernel_us > 0 { + dwt97_kernel_us + } else if timings.accelerator_dispatches > 0 { + timings.dct_to_wavelet_accelerator_us + } else { + 0 + }; + let cpu_us = timings + .dct_to_wavelet_cpu_fallback_us + .saturating_add(timings.dwt_decompose_us); + TranscodePipelineStageReport { + stage: TranscodePipelineStageKind::Transform, + processor: processor_for( + cpu_us, + metal_us, + timings.dwt97_batch_readback_us, + timings.accelerator_dispatches, + timings.cpu_fallback_jobs, + ), + cpu_us, + metal_us, + transfer_us: timings.dwt97_batch_readback_us, + dispatches: timings.accelerator_dispatches, + fallback_jobs: timings.cpu_fallback_jobs, + note: "DCT-grid to DWT projection uses accelerator dispatches when available; Ok(None) jobs remain caller CPU fallback", + } +} + +fn quantization_code_block_stage(timings: &TranscodeTimingReport) -> TranscodePipelineStageReport { + let metal_us = timings + .dwt97_batch_quantize_codeblock_us + .saturating_add(timings.dwt97_batch_ht_encode_us) + .saturating_add(timings.dwt97_batch_ht_kernel_us) + .saturating_add(timings.dwt97_batch_ht_compact_us); + let transfer_us = timings + .dwt97_batch_ht_status_readback_us + .saturating_add(timings.dwt97_batch_ht_output_readback_us); + let dispatches = timings + .dwt97_batch_ht_codeblock_dispatches + .saturating_add(timings.htj2k_encode_ht_code_block_dispatches); + TranscodePipelineStageReport { + stage: TranscodePipelineStageKind::QuantizationCodeBlockPrep, + processor: processor_for(0, metal_us, transfer_us, dispatches, 0), + cpu_us: 0, + metal_us, + transfer_us, + dispatches, + fallback_jobs: 0, + note: "9/7 quantization/code-block layout is only isolated when a backend reports resident stage timings; otherwise it is inside native encode time", + } +} + +fn packetization_stage(timings: &TranscodeTimingReport) -> TranscodePipelineStageReport { + let dispatches = timings.htj2k_encode_packetization_dispatches; + TranscodePipelineStageReport { + stage: TranscodePipelineStageKind::Packetization, + processor: processor_for(0, 0, 0, dispatches, 0), + cpu_us: 0, + metal_us: 0, + transfer_us: 0, + dispatches, + fallback_jobs: 0, + note: "Packetization dispatches are counted separately when an encode-stage accelerator handles them; CPU time is otherwise inside native encode time", + } +} + +fn codestream_assembly_stage(timings: &TranscodeTimingReport) -> TranscodePipelineStageReport { + TranscodePipelineStageReport { + stage: TranscodePipelineStageKind::CodestreamAssembly, + processor: TranscodeStageProcessor::Cpu, + cpu_us: timings.htj2k_encode_us, + metal_us: 0, + transfer_us: 0, + dispatches: 0, + fallback_jobs: 0, + note: "Final marker, tile-part, packet byte ordering, and codestream assembly remain at the CPU/native encoder boundary", + } +} + +fn recommend_next_resident_stage( + timings: &TranscodeTimingReport, +) -> TranscodeResidentStageRecommendation { + let transform_readback_us = timings.dwt97_batch_readback_us; + let code_block_readback_us = timings + .dwt97_batch_ht_status_readback_us + .saturating_add(timings.dwt97_batch_ht_output_readback_us); + let has_resident_transform = timings.accelerator_dispatches > 0 + && (timings.dwt97_batch_idct_row_lift_us > 0 + || timings.dwt97_batch_column_lift_us > 0 + || timings.dct_to_wavelet_accelerator_us > 0); + + if timings.cpu_fallback_jobs > 0 && timings.accelerator_dispatches == 0 { + return TranscodeResidentStageRecommendation { + stage: TranscodePipelineStageKind::Transform, + evidence_us: timings.dct_to_wavelet_cpu_fallback_us, + evidence_dispatches: timings.accelerator_attempts, + reason: "transform jobs are still completing through caller CPU fallback", + }; + } + + if has_resident_transform && timings.dwt97_batch_quantize_codeblock_us == 0 { + return TranscodeResidentStageRecommendation { + stage: TranscodePipelineStageKind::QuantizationCodeBlockPrep, + evidence_us: transform_readback_us.saturating_add(timings.htj2k_encode_us), + evidence_dispatches: timings.accelerator_dispatches, + reason: "resident transform output is read back before quantization/code-block prep and native encode", + }; + } + + if timings.dwt97_batch_quantize_codeblock_us > 0 + && timings.dwt97_batch_ht_codeblock_dispatches == 0 + { + return TranscodeResidentStageRecommendation { + stage: TranscodePipelineStageKind::QuantizationCodeBlockPrep, + evidence_us: transform_readback_us + .saturating_add(code_block_readback_us) + .saturating_add(timings.htj2k_encode_us), + evidence_dispatches: timings.accelerator_dispatches, + reason: "Metal already reaches 9/7 code-block prep; extend residency through HT code-block encode before packetization", + }; + } + + if timings.cpu_fallback_jobs > 0 { + return TranscodeResidentStageRecommendation { + stage: TranscodePipelineStageKind::Transform, + evidence_us: timings.dct_to_wavelet_cpu_fallback_us, + evidence_dispatches: timings.accelerator_attempts, + reason: "some transform jobs still use CPU fallback after accelerator attempts", + }; + } + + let coefficient_prep_us = timings + .jpeg_dct_repack_us + .saturating_add(timings.dwt97_batch_pack_upload_us); + if coefficient_prep_us > 0 { + return TranscodeResidentStageRecommendation { + stage: TranscodePipelineStageKind::CoefficientPrep, + evidence_us: coefficient_prep_us, + evidence_dispatches: timings.accelerator_dispatches, + reason: + "coefficient repack and host-to-device upload are visible before Metal work starts", + }; + } + + if timings.htj2k_encode_packetization_dispatches == 0 && timings.htj2k_encode_us > 0 { + return TranscodeResidentStageRecommendation { + stage: TranscodePipelineStageKind::Packetization, + evidence_us: timings.htj2k_encode_us, + evidence_dispatches: timings.htj2k_encode_accelerator_dispatches, + reason: + "packetization and codestream assembly remain inside the CPU/native encode boundary", + }; + } + + TranscodeResidentStageRecommendation { + stage: TranscodePipelineStageKind::CodestreamAssembly, + evidence_us: timings.htj2k_encode_us, + evidence_dispatches: timings.htj2k_encode_accelerator_dispatches, + reason: "no stronger resident-stage candidate is visible from the current counters", + } +} + +fn processor_for( + cpu_us: u128, + metal_us: u128, + transfer_us: u128, + dispatches: usize, + fallback_jobs: usize, +) -> TranscodeStageProcessor { + let has_cpu = cpu_us > 0 || fallback_jobs > 0; + let has_metal = metal_us > 0 || transfer_us > 0 || dispatches > 0; + match (has_cpu, has_metal) { + (true, true) => TranscodeStageProcessor::Hybrid, + (false, true) => TranscodeStageProcessor::Metal, + (_, false) => TranscodeStageProcessor::Cpu, + } +} diff --git a/crates/j2k-transcode/src/reversible53.rs b/crates/j2k-transcode/src/reversible53.rs new file mode 100644 index 00000000..0a09e6fd --- /dev/null +++ b/crates/j2k-transcode/src/reversible53.rs @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Shared reversible integer 5/3 lifting helpers for transcode paths. + +use core::convert::Infallible; + +pub(crate) fn reversible_lift_53_i32(values: &mut [i32]) { + let n = values.len(); + if n < 2 { + return; + } + + if n.is_multiple_of(2) { + for i in (1..n - 1).step_by(2) { + values[i] -= floor_div_i32(values[i - 1] + values[i + 1], 2); + } + values[n - 1] -= values[n - 2]; + + values[0] += floor_div_i32(values[1] + 1, 2); + for i in (2..n).step_by(2) { + values[i] += floor_div_i32(values[i - 1] + values[i + 1] + 2, 4); + } + return; + } + + let last_even = n - 1; + for i in (1..n).step_by(2) { + let right = values.get(i + 1).copied().unwrap_or(values[last_even]); + values[i] -= floor_div_i32(values[i - 1] + right, 2); + } + for i in (0..n).step_by(2) { + let left = if i > 0 { values[i - 1] } else { values[1] }; + let right = values.get(i + 1).copied().unwrap_or(left); + values[i] += floor_div_i32(left + right + 2, 4); + } +} + +pub(crate) fn reversible_lift_53_low_at( + sample_len: usize, + low_idx: usize, + mut sample: impl FnMut(usize) -> i32, +) -> i32 { + let mut fallible_sample = |idx| Ok::(sample(idx)); + match reversible_lift_53_low_at_fallible(sample_len, low_idx, &mut fallible_sample) { + Ok(value) => value, + Err(err) => match err {}, + } +} + +pub(crate) fn reversible_lift_53_high_at( + sample_len: usize, + high_idx: usize, + mut sample: impl FnMut(usize) -> i32, +) -> i32 { + let mut fallible_sample = |idx| Ok::(sample(idx)); + match reversible_lift_53_high_at_fallible(sample_len, high_idx, &mut fallible_sample) { + Ok(value) => value, + Err(err) => match err {}, + } +} + +pub(crate) fn reversible_lift_53_low_at_fallible( + sample_len: usize, + low_idx: usize, + mut sample: impl FnMut(usize) -> Result, +) -> Result { + reversible_lift_53_low_at_with(sample_len, low_idx, &mut sample) +} + +pub(crate) fn reversible_lift_53_high_at_fallible( + sample_len: usize, + high_idx: usize, + mut sample: impl FnMut(usize) -> Result, +) -> Result { + reversible_lift_53_high_at_with(sample_len, high_idx, &mut sample) +} + +fn reversible_lift_53_low_at_with( + sample_len: usize, + low_idx: usize, + sample: &mut impl FnMut(usize) -> Result, +) -> Result { + let even_idx = low_idx * 2; + let current = sample(even_idx)?; + if sample_len < 2 { + return Ok(current); + } + + if sample_len.is_multiple_of(2) { + let right = reversible_lift_53_high_at_with(sample_len, low_idx, sample)?; + if low_idx == 0 { + return Ok(current + floor_div_i32(right + 1, 2)); + } + let left = reversible_lift_53_high_at_with(sample_len, low_idx - 1, sample)?; + return Ok(current + floor_div_i32(left + right + 2, 4)); + } + + let high_len = sample_len / 2; + if high_len == 0 { + return Ok(current); + } + let left = if low_idx > 0 { + reversible_lift_53_high_at_with(sample_len, low_idx - 1, sample)? + } else { + reversible_lift_53_high_at_with(sample_len, 0, sample)? + }; + let right = if low_idx < high_len { + reversible_lift_53_high_at_with(sample_len, low_idx, sample)? + } else { + left + }; + Ok(current + floor_div_i32(left + right + 2, 4)) +} + +fn reversible_lift_53_high_at_with( + sample_len: usize, + high_idx: usize, + sample: &mut impl FnMut(usize) -> Result, +) -> Result { + let odd_idx = high_idx * 2 + 1; + let current = sample(odd_idx)?; + let left = sample(odd_idx - 1)?; + if sample_len.is_multiple_of(2) && odd_idx + 1 == sample_len { + return Ok(current - left); + } + + let right_idx = if odd_idx + 1 < sample_len { + odd_idx + 1 + } else { + sample_len - 1 + }; + let right = sample(right_idx)?; + Ok(current - floor_div_i32(left + right, 2)) +} + +pub(crate) fn floor_div_i32(numerator: i32, denominator: i32) -> i32 { + numerator.div_euclid(denominator) +} + +#[cfg(test)] +mod tests { + use super::{reversible_lift_53_high_at, reversible_lift_53_i32, reversible_lift_53_low_at}; + + #[test] + fn indexed_lift_matches_in_place_lift_for_varied_lengths() { + for sample_len in 1_usize..=17 { + let samples: Vec = (0..sample_len) + .map(|idx| { + let idx = i32::try_from(idx).expect("test index fits in i32"); + let value = (idx * 37) - (idx % 5) * 19; + if idx % 2 == 0 { + value + } else { + -value + } + }) + .collect(); + let mut lifted = samples.clone(); + reversible_lift_53_i32(&mut lifted); + + let low_len = sample_len.div_ceil(2); + for low_idx in 0..low_len { + assert_eq!( + reversible_lift_53_low_at(sample_len, low_idx, |idx| samples[idx]), + lifted[low_idx * 2], + "low {low_idx} for len {sample_len}" + ); + } + let high_len = sample_len / 2; + for high_idx in 0..high_len { + assert_eq!( + reversible_lift_53_high_at(sample_len, high_idx, |idx| samples[idx]), + lifted[high_idx * 2 + 1], + "high {high_idx} for len {sample_len}" + ); + } + } + } +} diff --git a/crates/j2k-transcode/tests/bench_harness.rs b/crates/j2k-transcode/tests/bench_harness.rs new file mode 100644 index 00000000..fe43d1c7 --- /dev/null +++ b/crates/j2k-transcode/tests/bench_harness.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 + +const DCT53_BENCH: &str = include_str!("../benches/dct53.rs"); + +#[test] +fn dct53_benchmark_group_names_are_stable() { + for group_name in [ + "dct53_1d_single_block_scalar", + "dct53_1d_multi_block_scalar", + "dct53_2d_single_level_scalar", + "dct53_2d_grid_scalar", + "dct97_2d_grid_scalar", + "direct_linear_13x11_scratch_reuse", + "idct_then_dwt_reference_13x11_scratch_reuse", + "dct53_multilevel_scalar", + "dct53_layout_candidates", + "jpeg_dct_extract", + "jpeg_to_htj2k", + "grayscale_8x8_stateful_reuse", + "grayscale_8x8_float_direct_97", + "ycbcr_420_16x16_float_direct_97", + ] { + assert!( + DCT53_BENCH.contains(group_name), + "missing Criterion group {group_name}" + ); + } +} diff --git a/crates/j2k-transcode/tests/corpus_validation.rs b/crates/j2k-transcode/tests/corpus_validation.rs new file mode 100644 index 00000000..f040976c --- /dev/null +++ b/crates/j2k-transcode/tests/corpus_validation.rs @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_test_support::{ + JPEG_BASELINE_420_16X16, JPEG_BASELINE_422_16X8, JPEG_BASELINE_444_8X8, JPEG_GRAYSCALE_8X8, +}; +use j2k_transcode::corpus_validation::{ + load_external_wsi_fixtures, validate_transcode_corpus, CorpusFixture, CorpusValidationError, + CorpusValidationOptions, +}; +use j2k_transcode::TranscodeValidationClassification; +use std::{fs, path::PathBuf}; + +#[test] +fn committed_conformance_fixtures_produce_error_report() { + let fixtures = conformance_fixtures(); + + let report = validate_transcode_corpus(&fixtures, &CorpusValidationOptions::default()) + .expect("validate committed transcode corpus"); + + assert_eq!(report.fixture_count, fixtures.len()); + assert!(report.sample_count >= 64); + assert_eq!(report.exact_match_count, report.sample_count); + assert_eq!(report.max_abs_error, 0); + assert_eq!( + report.classification, + TranscodeValidationClassification::Exact + ); + assert_eq!( + report.histogram_buckets.get(&0).copied(), + Some(report.sample_count) + ); + assert!(report + .fixtures + .iter() + .all(|fixture| fixture.classification == TranscodeValidationClassification::Exact)); +} + +#[test] +fn external_wsi_loader_discovers_limited_jpeg_inputs() { + let root = unique_temp_dir("j2k-transcode-corpus"); + let nested = root.join("nested"); + fs::create_dir_all(&nested).expect("create temp corpus"); + fs::write(root.join("ignore.txt"), b"not a jpeg").expect("write ignored file"); + fs::write(root.join("a.jpg"), conformance_fixtures()[0].bytes).expect("write first jpeg"); + fs::write(nested.join("b.jpeg"), conformance_fixtures()[1].bytes).expect("write nested jpeg"); + + let options = CorpusValidationOptions { + external_wsi_roots: vec![root.clone()], + external_tile_limit: 1, + ..CorpusValidationOptions::default() + }; + + let fixtures = load_external_wsi_fixtures(&options).expect("load external WSI fixtures"); + + assert_eq!(fixtures.len(), 1); + assert!(fixtures[0].name.ends_with("a.jpg")); + let report = validate_transcode_corpus(&[fixtures[0].as_fixture()], &options) + .expect("validate loaded external fixture"); + assert_eq!(report.fixture_count, 1); + + let _ = fs::remove_dir_all(root); +} + +#[test] +fn external_wsi_loader_can_require_configured_roots() { + let options = CorpusValidationOptions { + require_external_wsi: true, + ..CorpusValidationOptions::default() + }; + + let err = load_external_wsi_fixtures(&options).expect_err("missing roots should fail"); + + assert!(matches!( + err, + CorpusValidationError::MissingRequiredExternalCorpus(_) + )); +} + +fn conformance_fixtures() -> Vec> { + vec![ + CorpusFixture { + name: "grayscale_8x8", + bytes: JPEG_GRAYSCALE_8X8, + }, + CorpusFixture { + name: "baseline_444_8x8", + bytes: JPEG_BASELINE_444_8X8, + }, + CorpusFixture { + name: "baseline_422_16x8", + bytes: JPEG_BASELINE_422_16X8, + }, + CorpusFixture { + name: "baseline_420_16x16", + bytes: JPEG_BASELINE_420_16X16, + }, + ] +} + +fn unique_temp_dir(prefix: &str) -> PathBuf { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time after epoch") + .as_nanos(); + std::env::temp_dir().join(format!("{prefix}-{}-{unique}", std::process::id())) +} diff --git a/crates/j2k-transcode/tests/dct53_1d.rs b/crates/j2k-transcode/tests/dct53_1d.rs new file mode 100644 index 00000000..83aa828a --- /dev/null +++ b/crates/j2k-transcode/tests/dct53_1d.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_transcode::dct53_1d::{ + dct8_blocks_to_dwt53_float_linear, dct8_blocks_to_dwt53_float_linear_with_len, + dct8_to_dwt53_float_linear, dct8_to_dwt53_reversible_i16, idct8_blocks_then_dwt53_float, + idct8_blocks_then_dwt53_float_with_len, idct8_rounded_then_dwt53_reversible, + idct8_then_dwt53_float, Dwt53OneLevel, Dwt53Row, +}; +use proptest::prelude::*; + +proptest! { + #[test] + fn linear_multi_block_mapping_matches_reference_for_generated_coefficients( + blocks in proptest::collection::vec(proptest::array::uniform8(-256_i16..=256), 1..5) + ) { + let blocks: Vec<[f64; 8]> = blocks + .into_iter() + .map(|block| block.map(f64::from)) + .collect(); + + let direct = dct8_blocks_to_dwt53_float_linear(&blocks); + let reference = idct8_blocks_then_dwt53_float(&blocks); + + prop_assert_eq!(direct.low.len(), reference.low.len()); + prop_assert_eq!(direct.high.len(), reference.high.len()); + for (actual, expected) in direct.low.iter().zip(reference.low.iter()) { + prop_assert!((actual - expected).abs() <= 1.0e-9); + } + for (actual, expected) in direct.high.iter().zip(reference.high.iter()) { + prop_assert!((actual - expected).abs() <= 1.0e-9); + } + } +} + +#[test] +fn linear_single_level_mapping_matches_float_reference_for_synthetic_blocks() { + for coeffs in synthetic_float_coefficients() { + let direct = dct8_to_dwt53_float_linear(coeffs); + let reference = idct8_then_dwt53_float(coeffs); + + assert_dwt53_close(direct, reference, 1.0e-10); + } +} + +#[test] +fn linear_mapping_crosses_dct_block_boundary_for_two_blocks() { + let blocks = [ + [52.0, 11.0, -4.0, 7.0, 0.0, -3.0, 2.0, 1.0], + [47.0, -9.0, 5.0, -2.0, 8.0, 0.0, -1.0, 3.0], + ]; + + let direct = dct8_blocks_to_dwt53_float_linear(&blocks); + let reference = idct8_blocks_then_dwt53_float(&blocks); + + assert_eq!(direct.low.len(), 8); + assert_eq!(direct.high.len(), 8); + assert_dwt53_row_close(&direct, &reference, 1.0e-10); +} + +#[test] +fn linear_mapping_handles_cropped_even_and_odd_row_lengths() { + let blocks = [ + [52.0, 11.0, -4.0, 7.0, 0.0, -3.0, 2.0, 1.0], + [47.0, -9.0, 5.0, -2.0, 8.0, 0.0, -1.0, 3.0], + [21.0, 3.0, -8.0, 1.0, 2.0, -4.0, 6.0, -5.0], + ]; + + for sample_len in [15_usize, 16, 17] { + let direct = + dct8_blocks_to_dwt53_float_linear_with_len(&blocks, sample_len).expect("valid row"); + let reference = + idct8_blocks_then_dwt53_float_with_len(&blocks, sample_len).expect("valid row"); + + assert_eq!(direct.low.len(), sample_len.div_ceil(2)); + assert_eq!(direct.high.len(), sample_len / 2); + assert_dwt53_row_close(&direct, &reference, 1.0e-10); + } +} + +#[test] +fn linear_mapping_matches_reference_for_dc_high_frequency_and_random_rows() { + for blocks in multi_block_float_coefficients() { + let direct = dct8_blocks_to_dwt53_float_linear(&blocks); + let reference = idct8_blocks_then_dwt53_float(&blocks); + + assert_dwt53_row_close(&direct, &reference, 1.0e-10); + } +} + +#[test] +fn reversible_single_level_mapping_matches_rounded_reference_for_synthetic_blocks() { + for coeffs in synthetic_i16_coefficients() { + let direct = dct8_to_dwt53_reversible_i16(coeffs); + let reference = idct8_rounded_then_dwt53_reversible(coeffs); + + assert_eq!(direct, reference); + } +} + +#[test] +fn cropped_row_length_rejects_missing_dct_coverage() { + let blocks = [[1.0; 8]]; + let err = dct8_blocks_to_dwt53_float_linear_with_len(&blocks, 9).unwrap_err(); + + assert_eq!(err.sample_len(), 9); + assert_eq!(err.capacity(), 8); +} + +fn synthetic_float_coefficients() -> Vec<[f64; 8]> { + vec![ + [0.0; 8], + [32.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 18.0, -7.0, 0.0, 5.0, 0.0, 0.0, 0.0], + [91.0, -36.0, 14.0, -9.0, 3.0, 22.0, -11.0, 4.0], + [-40.0, 12.0, 28.0, -17.0, 6.0, -3.0, 2.0, -1.0], + ] +} + +fn synthetic_i16_coefficients() -> Vec<[i16; 8]> { + vec![ + [0; 8], + [32, 0, 0, 0, 0, 0, 0, 0], + [0, 18, -7, 0, 5, 0, 0, 0], + [91, -36, 14, -9, 3, 22, -11, 4], + [-40, 12, 28, -17, 6, -3, 2, -1], + ] +} + +fn multi_block_float_coefficients() -> Vec> { + vec![ + vec![ + [64.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [32.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + [-16.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + ], + vec![ + [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 24.0], + [0.0, 0.0, 0.0, 0.0, -17.0, 0.0, 13.0, 0.0], + [0.0, -19.0, 0.0, 11.0, 0.0, -7.0, 0.0, 5.0], + ], + pseudo_random_blocks(5), + ] +} + +fn pseudo_random_blocks(block_count: usize) -> Vec<[f64; 8]> { + let mut state = 0x7a37_4c21_u32; + (0..block_count) + .map(|_| { + let mut block = [0.0; 8]; + for coeff in &mut block { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + let bounded = u16::try_from(state % 97).expect("modulo result fits u16"); + *coeff = f64::from(i32::from(bounded) - 48); + } + block + }) + .collect() +} + +fn assert_dwt53_close(actual: Dwt53OneLevel, expected: Dwt53OneLevel, tolerance: f64) { + for (idx, (actual, expected)) in actual.low.iter().zip(expected.low.iter()).enumerate() { + assert!( + (actual - expected).abs() <= tolerance, + "low[{idx}] differs: actual={actual}, expected={expected}" + ); + } + for (idx, (actual, expected)) in actual.high.iter().zip(expected.high.iter()).enumerate() { + assert!( + (actual - expected).abs() <= tolerance, + "high[{idx}] differs: actual={actual}, expected={expected}" + ); + } +} + +fn assert_dwt53_row_close(actual: &Dwt53Row, expected: &Dwt53Row, tolerance: f64) { + assert_eq!(actual.low.len(), expected.low.len()); + assert_eq!(actual.high.len(), expected.high.len()); + + for (idx, (actual, expected)) in actual.low.iter().zip(expected.low.iter()).enumerate() { + assert!( + (actual - expected).abs() <= tolerance, + "low[{idx}] differs: actual={actual}, expected={expected}" + ); + } + for (idx, (actual, expected)) in actual.high.iter().zip(expected.high.iter()).enumerate() { + assert!( + (actual - expected).abs() <= tolerance, + "high[{idx}] differs: actual={actual}, expected={expected}" + ); + } +} diff --git a/crates/j2k-transcode/tests/dct53_2d.rs b/crates/j2k-transcode/tests/dct53_2d.rs new file mode 100644 index 00000000..306f60ad --- /dev/null +++ b/crates/j2k-transcode/tests/dct53_2d.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_transcode::dct53_2d::{ + dct8x8_blocks_then_dwt53_float, dct8x8_blocks_to_dwt53_float_linear, + dct8x8_blocks_to_dwt53_float_linear_with_scratch, dct8x8_to_dwt53_float_linear, + idct8x8_then_dwt53_float, Dct53GridScratch, +}; + +#[test] +fn dct8x8_to_single_level_2d_53_matches_reference() { + let mut block = [[0.0; 8]; 8]; + block[0][0] = 512.0; + block[0][1] = -31.0; + block[1][0] = 27.0; + block[2][3] = 9.0; + + let direct = dct8x8_to_dwt53_float_linear(block); + let reference = idct8x8_then_dwt53_float(block); + + assert_eq!(direct.low_width, 4); + assert_eq!(direct.low_height, 4); + assert_eq!(direct.high_width, 4); + assert_eq!(direct.high_height, 4); + assert!(direct.max_abs_diff(&reference) <= 1.0e-9); +} + +#[test] +fn dct8x8_to_2d_53_matches_reference_for_structured_cases() { + for block in [ + dc_only_block(), + high_frequency_block(), + checkerboard_like_block(), + gradient_like_block(), + ] { + let direct = dct8x8_to_dwt53_float_linear(block); + let reference = idct8x8_then_dwt53_float(block); + + assert!( + direct.max_abs_diff(&reference) <= 1.0e-9, + "max diff {}", + direct.max_abs_diff(&reference) + ); + } +} + +#[test] +fn dct8x8_grid_to_2d_53_crosses_block_boundaries() { + let blocks = synthetic_grid_blocks(2, 2); + + let direct = + dct8x8_blocks_to_dwt53_float_linear(&blocks, 2, 2, 13, 11).expect("valid DCT grid"); + let reference = dct8x8_blocks_then_dwt53_float(&blocks, 2, 2, 13, 11).expect("valid DCT grid"); + + assert_eq!(direct.low_width, 7); + assert_eq!(direct.low_height, 6); + assert_eq!(direct.high_width, 6); + assert_eq!(direct.high_height, 5); + assert!( + direct.max_abs_diff(&reference) <= 1.0e-9, + "max diff {}", + direct.max_abs_diff(&reference) + ); +} + +#[test] +fn dct8x8_grid_scratch_reuses_weight_rows_for_same_geometry() { + let blocks = synthetic_grid_blocks(2, 2); + let mut scratch = Dct53GridScratch::default(); + + let direct = + dct8x8_blocks_to_dwt53_float_linear_with_scratch(&blocks, 2, 2, 13, 11, &mut scratch) + .expect("valid DCT grid"); + let stateless = + dct8x8_blocks_to_dwt53_float_linear(&blocks, 2, 2, 13, 11).expect("valid DCT grid"); + let capacity_after_first = scratch.weight_row_capacity(); + + let repeated = + dct8x8_blocks_to_dwt53_float_linear_with_scratch(&blocks, 2, 2, 13, 11, &mut scratch) + .expect("valid DCT grid"); + + assert!(capacity_after_first > 0); + assert_eq!(scratch.weight_row_capacity(), capacity_after_first); + assert!(direct.max_abs_diff(&stateless) <= 1.0e-9); + assert!(repeated.max_abs_diff(&stateless) <= 1.0e-9); +} + +#[test] +fn dct8x8_grid_scratch_uses_sparse_weight_rows_for_wsi_tile() { + let dim = 224_usize; + let block_cols = dim / 8; + let block_rows = dim / 8; + let blocks = vec![[[0.0; 8]; 8]; block_cols * block_rows]; + let mut scratch = Dct53GridScratch::default(); + + dct8x8_blocks_to_dwt53_float_linear_with_scratch( + &blocks, + block_cols, + block_rows, + dim, + dim, + &mut scratch, + ) + .expect("valid DCT grid"); + + assert!( + scratch.weight_row_capacity() <= dim * 10, + "5/3 grid weights should stay sparse at WSI tile sizes, got capacity {}", + scratch.weight_row_capacity() + ); +} + +fn dc_only_block() -> [[f64; 8]; 8] { + let mut block = [[0.0; 8]; 8]; + block[0][0] = 384.0; + block +} + +fn high_frequency_block() -> [[f64; 8]; 8] { + let mut block = [[0.0; 8]; 8]; + block[7][7] = 64.0; + block[6][7] = -31.0; + block[7][6] = 29.0; + block +} + +fn checkerboard_like_block() -> [[f64; 8]; 8] { + let mut block = [[0.0; 8]; 8]; + for (y, row) in block.iter_mut().enumerate() { + for (x, coeff) in row.iter_mut().enumerate() { + if (x + y) % 2 == 0 { + *coeff = 8.0; + } else { + *coeff = -8.0; + } + } + } + block +} + +fn gradient_like_block() -> [[f64; 8]; 8] { + let mut block = [[0.0; 8]; 8]; + block[0][0] = 256.0; + block[0][1] = -48.0; + block[1][0] = 36.0; + block[0][2] = 12.0; + block[2][0] = -9.0; + block +} + +fn synthetic_grid_blocks(block_cols: usize, block_rows: usize) -> Vec<[[f64; 8]; 8]> { + let mut blocks = Vec::with_capacity(block_cols * block_rows); + for block_y in 0..block_rows { + for block_x in 0..block_cols { + let mut block = [[0.0; 8]; 8]; + block[0][0] = 192.0 + (block_x * 17 + block_y * 23) as f64; + block[0][1] = -31.0 + block_x as f64; + block[1][0] = 27.0 - block_y as f64; + block[2][3] = 9.0; + block[7][7] = -6.0; + blocks.push(block); + } + } + blocks +} diff --git a/crates/j2k-transcode/tests/dct53_multilevel.rs b/crates/j2k-transcode/tests/dct53_multilevel.rs new file mode 100644 index 00000000..67c8b3bc --- /dev/null +++ b/crates/j2k-transcode/tests/dct53_multilevel.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_transcode::dct53_multilevel::{ + dct8x8_to_dwt53_multilevel_float_linear, idct8x8_then_dwt53_multilevel_float, +}; + +#[test] +fn dct8x8_multilevel_uses_direct_level_one_and_conventional_ll_recursion() { + let mut block = [[0.0; 8]; 8]; + block[0][0] = 512.0; + block[0][1] = -31.0; + block[1][0] = 27.0; + block[2][3] = 9.0; + block[7][7] = -6.0; + + let direct = dct8x8_to_dwt53_multilevel_float_linear(block, 2).expect("valid levels"); + let reference = idct8x8_then_dwt53_multilevel_float(block, 2).expect("valid levels"); + + assert_eq!(direct.levels.len(), 2); + assert_eq!(direct.final_ll_width, 2); + assert_eq!(direct.final_ll_height, 2); + assert!(direct.max_abs_diff(&reference) <= 1.0e-9); +} + +#[test] +fn dct8x8_multilevel_rejects_zero_levels() { + let err = dct8x8_to_dwt53_multilevel_float_linear([[0.0; 8]; 8], 0).unwrap_err(); + + assert_eq!(err.requested_levels(), 0); +} diff --git a/crates/j2k-transcode/tests/dct97_2d.rs b/crates/j2k-transcode/tests/dct97_2d.rs new file mode 100644 index 00000000..f214fdce --- /dev/null +++ b/crates/j2k-transcode/tests/dct97_2d.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_transcode::dct97_2d::{ + dct8x8_blocks_then_dwt97_float, dct8x8_blocks_then_dwt97_float_with_scratch, Dct97GridScratch, + Dwt97TwoDimensional, +}; + +#[test] +fn dct8x8_grid_to_2d_97_idct_scratch_path_matches_reference_for_structured_cases() { + let blocks = structured_blocks(2, 2); + let mut scratch = Dct97GridScratch::default(); + + for (width, height) in [(8, 8), (13, 11), (16, 16)] { + let scratch_path = + dct8x8_blocks_then_dwt97_float_with_scratch(&blocks, 2, 2, width, height, &mut scratch) + .expect("scratch 9/7 IDCT path accepts covered grid"); + let reference = dct8x8_blocks_then_dwt97_float(&blocks, 2, 2, width, height) + .expect("reference 9/7 IDCT path accepts covered grid"); + + assert!( + max_abs_diff(&scratch_path, &reference) < 1.0e-9, + "scratch 9/7 IDCT path diverged for {width}x{height}" + ); + } +} + +#[test] +fn dct8x8_grid_to_2d_97_idct_scratch_path_matches_reference_and_reuses_storage() { + let large_blocks = structured_blocks(32, 32); + let small_blocks = structured_blocks(2, 2); + let mut scratch = Dct97GridScratch::default(); + + let large = + dct8x8_blocks_then_dwt97_float_with_scratch(&large_blocks, 32, 32, 255, 241, &mut scratch) + .expect("scratch 9/7 IDCT path accepts covered large grid"); + let expected_large = dct8x8_blocks_then_dwt97_float(&large_blocks, 32, 32, 255, 241) + .expect("reference 9/7 IDCT path accepts covered large grid"); + let capacity_after_large = scratch.spatial_sample_capacity(); + + let small = + dct8x8_blocks_then_dwt97_float_with_scratch(&small_blocks, 2, 2, 13, 11, &mut scratch) + .expect("scratch 9/7 IDCT path accepts covered small grid"); + let expected_small = dct8x8_blocks_then_dwt97_float(&small_blocks, 2, 2, 13, 11) + .expect("reference 9/7 IDCT path accepts covered small grid"); + + assert!( + max_abs_diff(&large, &expected_large) < 1.0e-9, + "scratch 9/7 IDCT path diverged for large grid" + ); + assert!( + max_abs_diff(&small, &expected_small) < 1.0e-9, + "scratch 9/7 IDCT path diverged for small grid" + ); + assert_eq!(scratch.spatial_sample_capacity(), capacity_after_large); +} + +fn max_abs_diff(actual: &Dwt97TwoDimensional, expected: &Dwt97TwoDimensional) -> f64 { + assert_eq!(actual.low_width, expected.low_width); + assert_eq!(actual.low_height, expected.low_height); + assert_eq!(actual.high_width, expected.high_width); + assert_eq!(actual.high_height, expected.high_height); + + actual + .ll + .iter() + .zip(expected.ll.iter()) + .chain(actual.hl.iter().zip(expected.hl.iter())) + .chain(actual.lh.iter().zip(expected.lh.iter())) + .chain(actual.hh.iter().zip(expected.hh.iter())) + .map(|(actual, expected)| (actual - expected).abs()) + .fold(0.0, f64::max) +} + +fn structured_blocks(block_cols: usize, block_rows: usize) -> Vec<[[f64; 8]; 8]> { + let mut blocks = Vec::with_capacity(block_cols * block_rows); + for block_y in 0..block_rows { + for block_x in 0..block_cols { + let mut block = [[0.0; 8]; 8]; + block[0][0] = 384.0 + (block_x * 19 + block_y * 23) as f64; + block[0][1] = -17.0 + block_x as f64; + block[1][0] = 11.0 - block_y as f64; + block[2][3] = 7.0; + block[4][4] = -3.0; + block[7][7] = 2.0; + blocks.push(block); + } + } + blocks +} diff --git a/crates/j2k-transcode/tests/error_metrics.rs b/crates/j2k-transcode/tests/error_metrics.rs new file mode 100644 index 00000000..b4d9b3c7 --- /dev/null +++ b/crates/j2k-transcode/tests/error_metrics.rs @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_transcode::metrics::error_metrics_i32; +use j2k_transcode::TranscodeValidationClassification; + +#[test] +fn error_metrics_report_exact_rate_max_abs_error_and_histogram() { + let actual = [10, 11, 12, 13, 14]; + let expected = [10, 12, 12, 12, 16]; + + let metrics = error_metrics_i32(&actual, &expected).expect("matching lengths"); + + assert_eq!(metrics.total, 5); + assert_eq!(metrics.exact_matches, 2); + assert!((metrics.exact_match_rate() - 0.4).abs() <= f64::EPSILON); + assert_eq!(metrics.max_abs_error, 2); + assert_eq!(metrics.absolute_error_count(0), 2); + assert_eq!(metrics.absolute_error_count(1), 2); + assert_eq!(metrics.absolute_error_count(2), 1); + assert!(!metrics.is_one_lsb_bounded(0.999)); +} + +#[test] +fn error_metrics_accept_one_lsb_bounded_claim_when_thresholds_pass() { + let actual = [10, 11, 12, 13]; + let expected = [10, 11, 12, 14]; + + let metrics = error_metrics_i32(&actual, &expected).expect("matching lengths"); + + assert!(metrics.is_one_lsb_bounded(0.75)); + assert!(!metrics.is_one_lsb_bounded(0.999)); +} + +#[test] +fn transcode_validation_classification_applies_thresholds() { + let exact = error_metrics_i32(&[4, 5, 6], &[4, 5, 6]).expect("matching lengths"); + assert_eq!( + TranscodeValidationClassification::classify_metrics(&exact), + TranscodeValidationClassification::Exact + ); + + let mut actual = vec![7; 1000]; + let expected = vec![7; 1000]; + actual[999] = 8; + let one_lsb = error_metrics_i32(&actual, &expected).expect("matching lengths"); + assert_eq!( + TranscodeValidationClassification::classify_metrics(&one_lsb), + TranscodeValidationClassification::OneLsbBounded + ); + + actual[998] = 8; + let outside = error_metrics_i32(&actual, &expected).expect("matching lengths"); + assert_eq!( + TranscodeValidationClassification::classify_metrics(&outside), + TranscodeValidationClassification::OutsideThreshold + ); +} + +#[test] +fn error_metrics_reject_mismatched_lengths() { + let err = error_metrics_i32(&[1, 2, 3], &[1, 2]).unwrap_err(); + + assert_eq!(err.actual_len(), 3); + assert_eq!(err.expected_len(), 2); +} diff --git a/crates/signinum-jpeg/tests/fixtures/mod.rs b/crates/j2k-transcode/tests/fixtures/mod.rs similarity index 91% rename from crates/signinum-jpeg/tests/fixtures/mod.rs rename to crates/j2k-transcode/tests/fixtures/mod.rs index b10a2241..5afcea79 100644 --- a/crates/signinum-jpeg/tests/fixtures/mod.rs +++ b/crates/j2k-transcode/tests/fixtures/mod.rs @@ -8,42 +8,42 @@ /// A 16×16 baseline JPEG with 4:2:0 sampling. pub(crate) fn minimal_baseline_420_jpeg() -> Vec { - include_bytes!("../../fixtures/conformance/baseline_420_16x16.jpg").to_vec() + j2k_test_support::jpeg_baseline_420_16x16() } /// An 8×8 grayscale (single-component) baseline JPEG. pub(crate) fn grayscale_8x8_jpeg() -> Vec { - include_bytes!("../../fixtures/conformance/grayscale_8x8.jpg").to_vec() + j2k_test_support::jpeg_grayscale_8x8() } /// An 8×8 baseline JPEG with 4:4:4 sampling. pub(crate) fn baseline_444_8x8_jpeg() -> Vec { - include_bytes!("../../fixtures/conformance/baseline_444_8x8.jpg").to_vec() + j2k_test_support::jpeg_baseline_444_8x8() } /// Reference pixels for [`baseline_444_8x8_jpeg`]. pub(crate) fn baseline_444_8x8_rgb() -> Vec { - include_bytes!("../../fixtures/conformance/baseline_444_8x8.rgb").to_vec() + j2k_test_support::jpeg_baseline_444_8x8_rgb() } /// A 16×8 baseline JPEG with 4:2:2 sampling. pub(crate) fn baseline_422_16x8_jpeg() -> Vec { - include_bytes!("../../fixtures/conformance/baseline_422_16x8.jpg").to_vec() + j2k_test_support::jpeg_baseline_422_16x8() } /// Reference pixels for [`baseline_422_16x8_jpeg`]. pub(crate) fn baseline_422_16x8_rgb() -> Vec { - include_bytes!("../../fixtures/conformance/baseline_422_16x8.rgb").to_vec() + j2k_test_support::jpeg_baseline_422_16x8_rgb() } /// A 32×16 baseline JPEG with 4:2:0 sampling and restart coding. pub(crate) fn baseline_420_restart_32x16_jpeg() -> Vec { - include_bytes!("../../fixtures/conformance/baseline_420_restart_32x16.jpg").to_vec() + j2k_test_support::jpeg_baseline_420_restart_32x16() } /// Reference pixels for [`baseline_420_restart_32x16_jpeg`]. pub(crate) fn baseline_420_restart_32x16_rgb() -> Vec { - include_bytes!("../../fixtures/conformance/baseline_420_restart_32x16.rgb").to_vec() + j2k_test_support::jpeg_baseline_420_restart_32x16_rgb() } /// An 8×8 APP14 RGB JPEG with constant pixel value `(200, 20, 10)`. diff --git a/crates/j2k-transcode/tests/jpeg_to_htj2k.rs b/crates/j2k-transcode/tests/jpeg_to_htj2k.rs new file mode 100644 index 00000000..bdf0f3ca --- /dev/null +++ b/crates/j2k-transcode/tests/jpeg_to_htj2k.rs @@ -0,0 +1,1758 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::adapter::encode_stage::{ + EncodedHtJ2kCodeBlock, IrreversibleQuantizationSubbandScales, J2kEncodeStageAccelerator, + J2kHtCodeBlockEncodeJob, +}; +use j2k_jpeg::{ + encode_jpeg_baseline, JpegBackend, JpegEncodeOptions, JpegSamples, JpegSubsampling, +}; +use j2k_native::{DecodeSettings, Image}; +use j2k_transcode::accelerator::{ + DctGridToDwt53Job, DctGridToDwt97Job, DctGridToHtj2k97CodeBlockJob, + DctGridToReversibleDwt53Job, DctToWaveletStageAccelerator, Dwt97BatchStageTimings, + Htj2k97CodeBlockOptions, J2kSubBandType, PreencodedHtj2k97CodeBlock, + PreencodedHtj2k97Component, PreencodedHtj2k97Resolution, PreencodedHtj2k97Subband, + PrequantizedHtj2k97CodeBlock, PrequantizedHtj2k97Component, PrequantizedHtj2k97Resolution, + PrequantizedHtj2k97Subband, RayonReversibleDwt53Accelerator, ReversibleDwt53FirstLevel, + TranscodeStageError, +}; +use j2k_transcode::dct53_2d::{ + dct8x8_blocks_to_dwt53_float_linear_with_scratch, Dct53GridScratch, Dwt53TwoDimensional, +}; +use j2k_transcode::dct97_2d::{ + dct8x8_blocks_then_dwt97_float_with_scratch, Dct97GridScratch, Dwt97TwoDimensional, +}; +use j2k_transcode::{ + jpeg_to_htj2k, EncodedTranscode, JpegTileBatchInput, JpegToHtj2kCoefficientPath, + JpegToHtj2kOptions, JpegToHtj2kTranscoder, TranscodePipelineStageKind, TranscodeStageProcessor, + TranscodeTimingReport, TranscodeValidationClassification, + JPEG_TO_HTJ2K_LOSSY_97_QUANTIZATION_SCALE, +}; +use std::{ + env, fs, + path::PathBuf, + process::{Command, Output}, + time::{SystemTime, UNIX_EPOCH}, +}; + +#[path = "fixtures/mod.rs"] +mod jpeg_fixtures; +use j2k_test_support::{ + JPEG_BASELINE_420_16X16, JPEG_BASELINE_422_16X8, JPEG_BASELINE_444_8X8, JPEG_GRAYSCALE_8X8, +}; + +#[test] +fn grayscale_8x8_jpeg_transcodes_to_decodable_htj2k() { + let jpeg = JPEG_GRAYSCALE_8X8; + + let encoded = jpeg_to_htj2k(jpeg, &JpegToHtj2kOptions::default()) + .expect("transcode grayscale JPEG to HTJ2K"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated HTJ2K") + .decode_native() + .expect("native decoder accepts generated HTJ2K"); + + assert_eq!((encoded.report.width, encoded.report.height), (8, 8)); + assert_eq!(encoded.report.component_count, 1); + assert_eq!((decoded.width, decoded.height), (8, 8)); + assert_eq!(decoded.num_components, 1); +} + +#[test] +fn grayscale_8x8_transcode_reports_opt_in_float_reference_metrics() { + let jpeg = JPEG_GRAYSCALE_8X8; + let options = JpegToHtj2kOptions { + coefficient_path: JpegToHtj2kCoefficientPath::FloatDirectLinear53, + validate_against_float_reference: true, + ..JpegToHtj2kOptions::default() + }; + + let encoded = + jpeg_to_htj2k(jpeg, &options).expect("transcode grayscale JPEG with validation enabled"); + let metrics = encoded + .report + .float_reference_metrics + .as_ref() + .expect("float reference metrics are reported"); + + assert_eq!(metrics.total, 64); + assert_eq!(metrics.exact_matches, 64); + assert_eq!(metrics.max_abs_error, 0); +} + +#[test] +fn grayscale_8x8_jpeg_transcodes_to_decodable_lossy_97_htj2k() { + let jpeg = JPEG_GRAYSCALE_8X8; + let options = JpegToHtj2kOptions { + validate_against_float_reference: true, + ..JpegToHtj2kOptions::lossy_97() + }; + + let encoded = + jpeg_to_htj2k(jpeg, &options).expect("transcode grayscale JPEG to lossy 9/7 HTJ2K"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated 9/7 HTJ2K") + .decode_native() + .expect("native decoder accepts generated 9/7 HTJ2K"); + let metrics = encoded + .report + .float_reference_metrics + .as_ref() + .expect("float reference metrics are reported"); + + assert_eq!( + encoded.report.path, + "full_resolution_components_float_direct_97" + ); + assert_eq!( + encoded.report.coefficient_path, + JpegToHtj2kCoefficientPath::FloatDirectLinear97 + ); + assert_eq!(encoded.report.decomposition_levels, 1); + assert_eq!(metrics.total, 64); + assert_eq!(metrics.exact_matches, 64); + assert_eq!(metrics.max_abs_error, 0); + assert_eq!((decoded.width, decoded.height), (8, 8)); + assert_eq!(decoded.num_components, 1); +} + +#[test] +fn option_constructors_select_consistent_default_codec_modes() { + let lossless = JpegToHtj2kOptions::lossless_53(); + assert_eq!( + lossless.coefficient_path, + JpegToHtj2kCoefficientPath::IntegerDirect53 + ); + assert!(lossless.encode_options.reversible); + assert!(!lossless.encode_options.use_mct); + + let lossy = JpegToHtj2kOptions::lossy_97(); + assert_eq!( + lossy.coefficient_path, + JpegToHtj2kCoefficientPath::FloatDirectLinear97 + ); + assert!(!lossy.encode_options.reversible); + assert!(!lossy.encode_options.use_mct); + assert_eq!( + lossy + .encode_options + .irreversible_quantization_scale + .to_bits(), + JPEG_TO_HTJ2K_LOSSY_97_QUANTIZATION_SCALE.to_bits() + ); + assert_eq!( + lossy + .encode_options + .irreversible_quantization_scale + .to_bits(), + 1.9f32.to_bits() + ); + assert_eq!( + lossy + .encode_options + .irreversible_quantization_subband_scales, + IrreversibleQuantizationSubbandScales::default() + ); +} + +#[test] +fn transcode_rejects_inconsistent_codec_mode_options() { + let jpeg = JPEG_GRAYSCALE_8X8; + let options = JpegToHtj2kOptions { + coefficient_path: JpegToHtj2kCoefficientPath::FloatDirectLinear97, + ..JpegToHtj2kOptions::default() + }; + + let err = jpeg_to_htj2k(jpeg, &options).expect_err("9/7 path requires irreversible encode"); + + assert!( + err.to_string() + .contains("9/7 coefficient path requires irreversible HTJ2K encode"), + "{err}" + ); +} + +#[test] +fn ycbcr_420_jpeg_transcodes_to_decodable_lossy_97_htj2k_with_native_sampling() { + let jpeg = JPEG_BASELINE_420_16X16; + let options = JpegToHtj2kOptions { + validate_against_float_reference: true, + ..JpegToHtj2kOptions::lossy_97() + }; + + let encoded = jpeg_to_htj2k(jpeg, &options).expect("transcode 4:2:0 JPEG to lossy 9/7 HTJ2K"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated 9/7 HTJ2K") + .decode_native() + .expect("native decoder accepts generated 9/7 HTJ2K"); + let metrics = encoded + .report + .float_reference_metrics + .as_ref() + .expect("float reference metrics are reported"); + + assert_eq!( + encoded.report.path, + "native_component_sampling_float_direct_97" + ); + assert_eq!(metrics.total, 384); + assert_eq!(metrics.exact_matches, 384); + assert_eq!(metrics.max_abs_error, 0); + assert_report_sampling(&encoded, &[(16, 16, 1, 1), (8, 8, 2, 2), (8, 8, 2, 2)]); + assert_eq!((decoded.width, decoded.height), (16, 16)); + assert_eq!(decoded.num_components, 3); + assert_component_sampling(&encoded.codestream, &[(1, 1), (2, 2), (2, 2)]); +} + +#[test] +fn grayscale_8x8_transcode_reports_opt_in_integer_reference_metrics() { + let jpeg = JPEG_GRAYSCALE_8X8; + let options = JpegToHtj2kOptions { + validate_against_integer_reference: true, + ..JpegToHtj2kOptions::default() + }; + + let encoded = + jpeg_to_htj2k(jpeg, &options).expect("transcode grayscale JPEG with integer validation"); + let metrics = encoded + .report + .integer_reference_metrics + .as_ref() + .expect("integer reference metrics are reported"); + + assert_eq!(metrics.total, 64); + assert_eq!(metrics.exact_matches, metrics.total); + assert_eq!(metrics.max_abs_error, 0); + assert_eq!( + encoded.report.coefficient_path, + JpegToHtj2kCoefficientPath::IntegerDirect53 + ); + assert_eq!( + encoded.report.integer_reference_classification, + Some(TranscodeValidationClassification::Exact) + ); + assert_eq!(encoded.report.float_reference_classification, None); +} + +#[test] +fn default_transcode_uses_integer_direct_coefficients() { + let jpeg = JPEG_BASELINE_420_16X16; + let options = JpegToHtj2kOptions { + validate_against_integer_reference: true, + ..JpegToHtj2kOptions::default() + }; + + let encoded = jpeg_to_htj2k(jpeg, &options) + .expect("transcode 4:2:0 JPEG with default integer direct path"); + let metrics = encoded + .report + .integer_reference_metrics + .as_ref() + .expect("integer reference metrics are reported"); + + assert_eq!( + encoded.report.path, + "native_component_sampling_integer_direct_53" + ); + assert_eq!(metrics.total, 384); + assert_eq!(metrics.exact_matches, metrics.total); + assert_eq!(metrics.max_abs_error, 0); +} + +#[test] +fn grayscale_8x8_jpeg_transcodes_with_two_decomposition_levels() { + let jpeg = JPEG_GRAYSCALE_8X8; + let mut encode_options = JpegToHtj2kOptions::default().encode_options; + encode_options.num_decomposition_levels = 2; + let options = JpegToHtj2kOptions { + encode_options, + coefficient_path: JpegToHtj2kCoefficientPath::FloatDirectLinear53, + validate_against_float_reference: true, + ..JpegToHtj2kOptions::default() + }; + + let encoded = + jpeg_to_htj2k(jpeg, &options).expect("transcode grayscale JPEG with two DWT levels"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated HTJ2K") + .decode_native() + .expect("native decoder accepts generated HTJ2K"); + let metrics = encoded + .report + .float_reference_metrics + .as_ref() + .expect("float reference metrics are reported"); + + assert_eq!(encoded.report.decomposition_levels, 2); + assert_eq!((decoded.width, decoded.height), (8, 8)); + assert_eq!(decoded.num_components, 1); + assert_eq!(metrics.total, 64); + assert_eq!(metrics.exact_matches, 64); +} + +#[test] +fn integer_direct_transcode_matches_integer_oracle_with_two_decomposition_levels() { + let jpeg = JPEG_GRAYSCALE_8X8; + let mut encode_options = JpegToHtj2kOptions::default().encode_options; + encode_options.num_decomposition_levels = 2; + let options = JpegToHtj2kOptions { + encode_options, + validate_against_integer_reference: true, + ..JpegToHtj2kOptions::default() + }; + + let encoded = + jpeg_to_htj2k(jpeg, &options).expect("integer-direct transcode supports two DWT levels"); + let metrics = encoded + .report + .integer_reference_metrics + .as_ref() + .expect("integer reference metrics are reported"); + + assert_eq!( + encoded.report.path, + "full_resolution_components_integer_direct_53" + ); + assert_eq!(encoded.report.decomposition_levels, 2); + assert_eq!(metrics.total, 64); + assert_eq!(metrics.exact_matches, metrics.total); + assert_eq!(metrics.max_abs_error, 0); +} + +#[test] +fn generated_htj2k_is_accepted_by_available_external_decoder() { + let jpeg = JPEG_GRAYSCALE_8X8; + let encoded = jpeg_to_htj2k(jpeg, &JpegToHtj2kOptions::default()) + .expect("transcode grayscale JPEG to HTJ2K"); + let decoders = available_external_decoders(); + if decoders.is_empty() { + eprintln!("skipping external HTJ2K decoder check: no supported decoder executable found"); + return; + } + + let mut failures = Vec::new(); + for decoder in decoders { + match run_external_decoder(decoder, &encoded.codestream) { + Ok(()) => return, + Err(err) => failures.push(format!("{decoder:?}: {err}")), + } + } + + panic!( + "generated HTJ2K codestream was rejected by all available external decoders:\n{}", + failures.join("\n") + ); +} + +#[test] +fn progressive_ycbcr_420_jpeg_transcodes_with_native_component_sampling() { + let jpeg = jpeg_fixtures::progressive_8x8_jpeg(); + + let encoded = jpeg_to_htj2k(&jpeg, &JpegToHtj2kOptions::default()) + .expect("transcode progressive 4:2:0 JPEG to HTJ2K"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated HTJ2K") + .decode_native() + .expect("native decoder accepts generated HTJ2K"); + + assert_eq!((encoded.report.width, encoded.report.height), (8, 8)); + assert_eq!(encoded.report.component_count, 3); + assert_report_sampling(&encoded, &[(8, 8, 1, 1), (4, 4, 2, 2), (4, 4, 2, 2)]); + assert_eq!((decoded.width, decoded.height), (8, 8)); + assert_eq!(decoded.num_components, 3); + assert_component_sampling(&encoded.codestream, &[(1, 1), (2, 2), (2, 2)]); +} + +#[test] +fn grayscale_multiblock_jpeg_transcodes_to_decodable_htj2k() { + let width = 13; + let height = 11; + let gray = patterned_gray(width, height); + let jpeg = encode_jpeg_baseline( + JpegSamples::Gray8 { + data: &gray, + width, + height, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Gray, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode grayscale JPEG fixture"); + + let encoded = jpeg_to_htj2k(&jpeg.data, &JpegToHtj2kOptions::default()) + .expect("transcode multi-block grayscale JPEG to HTJ2K"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated HTJ2K") + .decode_native() + .expect("native decoder accepts generated HTJ2K"); + + assert_eq!( + (encoded.report.width, encoded.report.height), + (width, height) + ); + assert_eq!(encoded.report.component_count, 1); + assert_eq!((decoded.width, decoded.height), (width, height)); + assert_eq!(decoded.num_components, 1); +} + +#[test] +fn integer_direct_transcode_with_rayon_accelerator_matches_scalar_for_grayscale_dimensions() { + for (width, height) in [(8, 8), (13, 11), (16, 16)] { + let jpeg = encoded_gray_jpeg(width, height); + let options = JpegToHtj2kOptions { + validate_against_integer_reference: true, + ..JpegToHtj2kOptions::default() + }; + let scalar = jpeg_to_htj2k(&jpeg, &options).expect("scalar integer-direct transcode"); + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = RayonReversibleDwt53Accelerator::default(); + + let accelerated = transcoder + .transcode_with_accelerator(&jpeg, &options, &mut accelerator) + .expect("rayon integer-direct transcode"); + + assert_eq!( + accelerated.codestream, scalar.codestream, + "accelerated IntegerDirect53 must match scalar oracle for {width}x{height}" + ); + assert_eq!( + accelerated.report.integer_reference_classification, + Some(TranscodeValidationClassification::Exact) + ); + assert_eq!(accelerator.reversible_dwt53_attempts(), 0); + assert_eq!(accelerator.reversible_dwt53_dispatches(), 0); + assert_eq!(accelerator.reversible_dwt53_batch_attempts(), 1); + assert_eq!(accelerator.reversible_dwt53_batch_dispatches(), 1); + } +} + +#[test] +fn integer_direct_transcode_with_rayon_accelerator_matches_scalar_for_ycbcr_420() { + let jpeg = JPEG_BASELINE_420_16X16; + let options = JpegToHtj2kOptions { + validate_against_integer_reference: true, + ..JpegToHtj2kOptions::default() + }; + let scalar = jpeg_to_htj2k(jpeg, &options).expect("scalar 4:2:0 integer-direct transcode"); + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = RayonReversibleDwt53Accelerator::default(); + + let accelerated = transcoder + .transcode_with_accelerator(jpeg, &options, &mut accelerator) + .expect("rayon 4:2:0 integer-direct transcode"); + + assert_eq!(accelerated.codestream, scalar.codestream); + assert_eq!( + accelerated.report.integer_reference_classification, + Some(TranscodeValidationClassification::Exact) + ); + assert_eq!(accelerator.reversible_dwt53_attempts(), 0); + assert_eq!(accelerator.reversible_dwt53_dispatches(), 0); + assert_eq!(accelerator.reversible_dwt53_batch_attempts(), 2); + assert_eq!(accelerator.reversible_dwt53_batch_dispatches(), 2); + assert_report_sampling(&accelerated, &[(16, 16, 1, 1), (8, 8, 2, 2), (8, 8, 2, 2)]); + assert_component_sampling(&accelerated.codestream, &[(1, 1), (2, 2), (2, 2)]); +} + +#[test] +fn ycbcr_444_jpeg_transcodes_to_decodable_htj2k_without_mct() { + let jpeg = JPEG_BASELINE_444_8X8; + + let encoded = jpeg_to_htj2k(jpeg, &JpegToHtj2kOptions::default()) + .expect("transcode 4:4:4 YCbCr JPEG to HTJ2K"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated HTJ2K") + .decode_native() + .expect("native decoder accepts generated HTJ2K"); + + assert_eq!((encoded.report.width, encoded.report.height), (8, 8)); + assert_eq!(encoded.report.component_count, 3); + assert_report_sampling(&encoded, &[(8, 8, 1, 1), (8, 8, 1, 1), (8, 8, 1, 1)]); + assert_eq!((decoded.width, decoded.height), (8, 8)); + assert_eq!(decoded.num_components, 3); +} + +#[test] +fn ycbcr_422_jpeg_transcodes_with_native_component_sampling() { + let jpeg = JPEG_BASELINE_422_16X8; + + let encoded = jpeg_to_htj2k(jpeg, &JpegToHtj2kOptions::default()) + .expect("transcode 4:2:2 YCbCr JPEG to HTJ2K"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated HTJ2K") + .decode_native() + .expect("native decoder accepts generated HTJ2K"); + + assert_eq!((encoded.report.width, encoded.report.height), (16, 8)); + assert_eq!(encoded.report.component_count, 3); + assert_report_sampling(&encoded, &[(16, 8, 1, 1), (8, 8, 2, 1), (8, 8, 2, 1)]); + assert_eq!((decoded.width, decoded.height), (16, 8)); + assert_eq!(decoded.num_components, 3); + assert_component_sampling(&encoded.codestream, &[(1, 1), (2, 1), (2, 1)]); +} + +#[test] +fn ycbcr_420_jpeg_transcodes_with_native_component_sampling() { + let jpeg = JPEG_BASELINE_420_16X16; + + let encoded = jpeg_to_htj2k(jpeg, &JpegToHtj2kOptions::default()) + .expect("transcode 4:2:0 YCbCr JPEG to HTJ2K"); + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native parser accepts generated HTJ2K") + .decode_native() + .expect("native decoder accepts generated HTJ2K"); + + assert_eq!((encoded.report.width, encoded.report.height), (16, 16)); + assert_eq!(encoded.report.component_count, 3); + assert_report_sampling(&encoded, &[(16, 16, 1, 1), (8, 8, 2, 2), (8, 8, 2, 2)]); + assert!(encoded.report.float_reference_metrics.is_none()); + assert_eq!((decoded.width, decoded.height), (16, 16)); + assert_eq!(decoded.num_components, 3); + assert_component_sampling(&encoded.codestream, &[(1, 1), (2, 2), (2, 2)]); +} + +#[test] +fn ycbcr_420_validation_metrics_cover_native_component_coefficients() { + let jpeg = JPEG_BASELINE_420_16X16; + let options = JpegToHtj2kOptions { + coefficient_path: JpegToHtj2kCoefficientPath::FloatDirectLinear53, + validate_against_float_reference: true, + ..JpegToHtj2kOptions::default() + }; + + let encoded = + jpeg_to_htj2k(jpeg, &options).expect("transcode 4:2:0 JPEG with validation enabled"); + let metrics = encoded + .report + .float_reference_metrics + .as_ref() + .expect("float reference metrics are reported"); + + assert_eq!(metrics.total, 384); + assert_eq!(metrics.exact_matches, 384); + assert_eq!(metrics.max_abs_error, 0); +} + +#[test] +fn stateful_transcoder_reuses_dct_block_scratch_across_tiles() { + let larger_jpeg = JPEG_BASELINE_420_16X16; + let smaller_jpeg = JPEG_GRAYSCALE_8X8; + let options = JpegToHtj2kOptions { + coefficient_path: JpegToHtj2kCoefficientPath::FloatDirectLinear53, + ..JpegToHtj2kOptions::default() + }; + let mut transcoder = JpegToHtj2kTranscoder::default(); + + let larger = transcoder + .transcode(larger_jpeg, &options) + .expect("stateful transcode accepts 4:2:0 JPEG"); + let capacity_after_larger = transcoder.dct_block_scratch_capacity(); + assert!(capacity_after_larger >= 4); + + let smaller = transcoder + .transcode(smaller_jpeg, &options) + .expect("stateful transcode accepts grayscale JPEG"); + let stateless = + jpeg_to_htj2k(smaller_jpeg, &options).expect("stateless transcode accepts grayscale JPEG"); + + assert_eq!(larger.report.component_count, 3); + assert_eq!(smaller.report.component_count, 1); + assert_eq!( + transcoder.dct_block_scratch_capacity(), + capacity_after_larger + ); + assert_eq!(smaller.codestream, stateless.codestream); +} + +#[test] +fn float_direct_transcode_paths_use_acceleration_hooks_when_available() { + let jpeg = JPEG_GRAYSCALE_8X8; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = CountingAccelerator::default(); + + let options_53 = JpegToHtj2kOptions { + coefficient_path: JpegToHtj2kCoefficientPath::FloatDirectLinear53, + ..JpegToHtj2kOptions::default() + }; + let encoded_53 = transcoder + .transcode_with_accelerator(jpeg, &options_53, &mut accelerator) + .expect("accelerated 5/3 float transcode succeeds"); + assert_eq!(encoded_53.report.timings.component_count, 1); + assert_eq!(encoded_53.report.timings.accelerator_attempts, 1); + assert_eq!(encoded_53.report.timings.accelerator_dispatches, 1); + assert_eq!(encoded_53.report.timings.cpu_fallback_jobs, 0); + + let options_97 = JpegToHtj2kOptions { + ..JpegToHtj2kOptions::lossy_97() + }; + let encoded_97 = transcoder + .transcode_with_accelerator(jpeg, &options_97, &mut accelerator) + .expect("accelerated 9/7 float transcode succeeds"); + assert_eq!(encoded_97.report.timings.component_count, 1); + assert_eq!(encoded_97.report.timings.accelerator_attempts, 1); + assert_eq!(encoded_97.report.timings.accelerator_dispatches, 1); + assert_eq!(encoded_97.report.timings.cpu_fallback_jobs, 0); + + assert_eq!(accelerator.dwt53_calls, 1); + assert_eq!(accelerator.dwt97_calls, 1); +} + +#[test] +fn lossy_97_cpu_report_includes_transform_fallback_timing_breakdown() { + let jpeg = JPEG_GRAYSCALE_8X8; + let mut transcoder = JpegToHtj2kTranscoder::default(); + + let encoded = transcoder + .transcode(jpeg, &JpegToHtj2kOptions::lossy_97()) + .expect("CPU-only 9/7 transcode succeeds"); + let timings = encoded.report.timings; + + assert_eq!(timings.jpeg_dct_extract_us, encoded.report.extract_us); + assert_eq!(timings.dct_to_wavelet_total_us, encoded.report.transform_us); + assert_eq!(timings.htj2k_encode_us, encoded.report.encode_us); + assert_eq!(timings.component_count, 1); + assert_eq!(timings.accelerator_attempts, 1); + assert_eq!(timings.accelerator_jobs, 1); + assert_eq!(timings.accelerator_dispatches, 0); + assert_eq!(timings.accelerator_dispatched_jobs, 0); + assert_eq!(timings.cpu_fallback_jobs, 1); +} + +#[test] +fn transcode_pipeline_map_covers_cpu_fallback_stages() { + let jpeg = JPEG_GRAYSCALE_8X8; + let mut transcoder = JpegToHtj2kTranscoder::default(); + + let encoded = transcoder + .transcode(jpeg, &JpegToHtj2kOptions::lossy_97()) + .expect("CPU-only 9/7 transcode succeeds"); + let map = encoded.report.pipeline_map(); + let stages = map + .stages + .iter() + .map(|stage| stage.stage) + .collect::>(); + + assert_eq!( + stages, + vec![ + TranscodePipelineStageKind::EntropyDecode, + TranscodePipelineStageKind::CoefficientPrep, + TranscodePipelineStageKind::Transform, + TranscodePipelineStageKind::QuantizationCodeBlockPrep, + TranscodePipelineStageKind::Packetization, + TranscodePipelineStageKind::CodestreamAssembly, + ] + ); + let transform = map + .stages + .iter() + .find(|stage| stage.stage == TranscodePipelineStageKind::Transform) + .expect("pipeline map includes transform stage"); + assert_eq!(transform.processor, TranscodeStageProcessor::Cpu); + assert_eq!( + map.recommendation.stage, + TranscodePipelineStageKind::Transform + ); + assert!(map.debug_report().contains("stage=transform processor=Cpu")); +} + +#[test] +fn transcode_pipeline_map_reports_metal_residency_and_next_stage() { + let timings = TranscodeTimingReport { + jpeg_dct_extract_us: 11, + jpeg_dct_repack_us: 7, + dct_to_wavelet_accelerator_us: 100, + dwt97_batch_pack_upload_us: 9, + dwt97_batch_idct_row_lift_us: 31, + dwt97_batch_column_lift_us: 29, + dwt97_batch_quantize_codeblock_us: 13, + dwt97_batch_readback_us: 17, + htj2k_encode_us: 101, + accelerator_attempts: 1, + accelerator_jobs: 4, + accelerator_dispatches: 1, + accelerator_dispatched_jobs: 4, + ..TranscodeTimingReport::default() + }; + + let map = timings.pipeline_map(); + let coefficient_prep = map + .stages + .iter() + .find(|stage| stage.stage == TranscodePipelineStageKind::CoefficientPrep) + .expect("pipeline map includes coefficient prep stage"); + let transform = map + .stages + .iter() + .find(|stage| stage.stage == TranscodePipelineStageKind::Transform) + .expect("pipeline map includes transform stage"); + let code_block_prep = map + .stages + .iter() + .find(|stage| stage.stage == TranscodePipelineStageKind::QuantizationCodeBlockPrep) + .expect("pipeline map includes code-block prep stage"); + let debug = map.debug_report(); + + assert_eq!(coefficient_prep.processor, TranscodeStageProcessor::Hybrid); + assert_eq!(transform.processor, TranscodeStageProcessor::Metal); + assert_eq!(code_block_prep.processor, TranscodeStageProcessor::Metal); + assert_eq!( + map.recommendation.stage, + TranscodePipelineStageKind::QuantizationCodeBlockPrep + ); + assert!( + map.recommendation.evidence_us >= timings.dwt97_batch_readback_us, + "recommendation should cite readback or encode timing evidence" + ); + assert!(debug.contains("stage=entropy_decode processor=Cpu")); + assert!(debug.contains("stage=transform processor=Metal")); + assert!(debug.contains("recommend_next_stage=quantization_code_block_prep")); +} + +#[test] +fn integer_direct_transcode_path_uses_reversible_acceleration_hook_when_available() { + let jpeg = JPEG_GRAYSCALE_8X8; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = CountingAccelerator::default(); + + let encoded = transcoder + .transcode_with_accelerator(jpeg, &JpegToHtj2kOptions::default(), &mut accelerator) + .expect("accelerated integer-direct transcode succeeds"); + + assert_eq!(accelerator.reversible_dwt53_calls, 0); + assert_eq!(accelerator.reversible_dwt53_batch_calls, 1); + assert_eq!(accelerator.reversible_dwt53_batch_sizes, vec![1]); + assert_eq!(encoded.report.timings.batch_count, 1); + assert_eq!(encoded.report.timings.batch_jobs, 1); + assert_eq!(encoded.report.timings.accelerator_attempts, 1); + assert_eq!(encoded.report.timings.accelerator_dispatches, 1); + assert_eq!(encoded.report.timings.cpu_fallback_jobs, 0); +} + +#[test] +fn integer_direct_transcode_batches_same_geometry_components_when_available() { + for fixture in [ + BatchFixture { + name: "grayscale", + jpeg: JPEG_GRAYSCALE_8X8, + expected_batch_sizes: &[1], + }, + BatchFixture { + name: "ycbcr_444", + jpeg: JPEG_BASELINE_444_8X8, + expected_batch_sizes: &[3], + }, + BatchFixture { + name: "ycbcr_422", + jpeg: JPEG_BASELINE_422_16X8, + expected_batch_sizes: &[1, 2], + }, + BatchFixture { + name: "ycbcr_420", + jpeg: JPEG_BASELINE_420_16X16, + expected_batch_sizes: &[1, 2], + }, + ] { + let options = JpegToHtj2kOptions { + validate_against_integer_reference: true, + ..JpegToHtj2kOptions::default() + }; + let scalar = + jpeg_to_htj2k(fixture.jpeg, &options).expect("scalar integer-direct transcode"); + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = CountingAccelerator::default(); + + let accelerated = transcoder + .transcode_with_accelerator(fixture.jpeg, &options, &mut accelerator) + .expect("batched integer-direct transcode succeeds"); + + assert_eq!( + accelerated.codestream, scalar.codestream, + "batched IntegerDirect53 must match scalar oracle for {}", + fixture.name + ); + assert_eq!( + accelerated.report.integer_reference_classification, + Some(TranscodeValidationClassification::Exact) + ); + assert_eq!( + accelerator.reversible_dwt53_calls, 0, + "batch-capable accelerator should not need single-job fallback for {}", + fixture.name + ); + assert_eq!( + accelerator.reversible_dwt53_batch_sizes, fixture.expected_batch_sizes, + "unexpected same-geometry batch grouping for {}", + fixture.name + ); + } +} + +#[test] +fn integer_direct_batch_transcode_groups_components_across_tiles() { + for fixture in [ + BatchFixture { + name: "grayscale", + jpeg: JPEG_GRAYSCALE_8X8, + expected_batch_sizes: &[4], + }, + BatchFixture { + name: "ycbcr_444", + jpeg: JPEG_BASELINE_444_8X8, + expected_batch_sizes: &[4, 4, 4], + }, + BatchFixture { + name: "ycbcr_422", + jpeg: JPEG_BASELINE_422_16X8, + expected_batch_sizes: &[4, 4, 4], + }, + BatchFixture { + name: "ycbcr_420", + jpeg: JPEG_BASELINE_420_16X16, + expected_batch_sizes: &[4, 4, 4], + }, + ] { + let options = JpegToHtj2kOptions { + validate_against_integer_reference: true, + ..JpegToHtj2kOptions::default() + }; + let inputs = vec![ + JpegTileBatchInput { + bytes: fixture.jpeg + }; + 4 + ]; + let expected = jpeg_to_htj2k(fixture.jpeg, &options) + .expect("scalar integer-direct transcode succeeds"); + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = CountingAccelerator::default(); + + let batch = transcoder + .transcode_batch_with_accelerator(&inputs, &options, &mut accelerator) + .expect("batched transcode accepts valid options"); + + assert_eq!(batch.tiles.len(), inputs.len()); + assert_eq!(batch.report.tile_count, inputs.len()); + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!(batch.report.failed_tiles, 0); + assert_eq!( + accelerator.reversible_dwt53_batch_sizes, fixture.expected_batch_sizes, + "unexpected cross-tile component grouping for {}", + fixture.name + ); + assert_eq!(accelerator.reversible_dwt53_calls, 0); + assert_eq!( + batch.report.timings.batch_jobs, + fixture.expected_batch_sizes.iter().sum::(), + "batch report should count accelerated component jobs for {}", + fixture.name + ); + assert_eq!( + batch.report.timings.accelerator_dispatches, + fixture.expected_batch_sizes.len(), + "batch report should count accelerator batch dispatches for {}", + fixture.name + ); + for tile in batch.tiles { + let tile = tile.expect("valid tile transcodes"); + assert_eq!( + tile.codestream, expected.codestream, + "batch tile must match scalar output for {}", + fixture.name + ); + assert_eq!( + tile.report.integer_reference_classification, + Some(TranscodeValidationClassification::Exact) + ); + assert_eq!( + tile.report.timings.batch_jobs, 0, + "tile report must not duplicate shared batch timing context for {}", + fixture.name + ); + assert_eq!(tile.report.transform_us, 0); + } + } +} + +#[test] +fn float97_batch_transcode_groups_components_across_tiles() { + for fixture in [ + BatchFixture { + name: "grayscale", + jpeg: JPEG_GRAYSCALE_8X8, + expected_batch_sizes: &[4], + }, + BatchFixture { + name: "ycbcr_444", + jpeg: JPEG_BASELINE_444_8X8, + expected_batch_sizes: &[12], + }, + BatchFixture { + name: "ycbcr_422", + jpeg: JPEG_BASELINE_422_16X8, + expected_batch_sizes: &[4, 8], + }, + BatchFixture { + name: "ycbcr_420", + jpeg: JPEG_BASELINE_420_16X16, + expected_batch_sizes: &[4, 8], + }, + ] { + let options = JpegToHtj2kOptions::lossy_97(); + let inputs = vec![ + JpegTileBatchInput { + bytes: fixture.jpeg + }; + 4 + ]; + let expected = + jpeg_to_htj2k(fixture.jpeg, &options).expect("scalar 9/7 transcode succeeds"); + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = CountingAccelerator::default(); + + let batch = transcoder + .transcode_batch_with_accelerator(&inputs, &options, &mut accelerator) + .expect("batched 9/7 transcode accepts valid options"); + + assert_eq!(batch.tiles.len(), inputs.len()); + assert_eq!(batch.report.tile_count, inputs.len()); + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!(batch.report.failed_tiles, 0); + assert_eq!( + accelerator.dwt97_batch_sizes, fixture.expected_batch_sizes, + "unexpected cross-tile 9/7 component grouping for {}", + fixture.name + ); + assert_eq!(accelerator.dwt97_calls, 0); + assert_eq!( + batch.report.timings.batch_jobs, + fixture.expected_batch_sizes.iter().sum::(), + "batch report should count 9/7 component jobs for {}", + fixture.name + ); + assert_eq!( + batch.report.timings.accelerator_dispatches, + fixture.expected_batch_sizes.len(), + "batch report should count 9/7 accelerator batch dispatches for {}", + fixture.name + ); + for tile in batch.tiles { + let tile = tile.expect("valid 9/7 tile transcodes"); + assert_eq!( + tile.codestream, expected.codestream, + "batch 9/7 tile must match scalar output for {}", + fixture.name + ); + assert_eq!( + tile.report.timings.batch_jobs, 0, + "tile report must not duplicate shared 9/7 batch timing context for {}", + fixture.name + ); + assert_eq!(tile.report.transform_us, 0); + } + } +} + +#[test] +fn float97_batch_transcode_prefers_prequantized_codeblock_batches() { + let jpeg = JPEG_GRAYSCALE_8X8; + let options = JpegToHtj2kOptions::lossy_97(); + let inputs = vec![JpegTileBatchInput { bytes: jpeg }; 4]; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = Prequantized97Accelerator::default(); + + let batch = transcoder + .transcode_batch_with_accelerator(&inputs, &options, &mut accelerator) + .expect("prequantized 9/7 batch transcode accepts valid options"); + + assert_eq!(batch.tiles.len(), inputs.len()); + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!(accelerator.prequantized_batch_sizes, vec![4]); + assert_eq!(accelerator.dwt97_batch_calls, 0); + assert_eq!(batch.report.timings.accelerator_dispatches, 1); + assert_eq!(batch.report.timings.accelerator_dispatched_jobs, 4); + for tile in batch.tiles { + let tile = tile.expect("valid prequantized tile transcodes"); + assert!(tile.codestream.starts_with(&[0xFF, 0x4F])); + } +} + +#[test] +fn float97_prequantized_batch_receives_lossy_subband_profile() { + let jpeg = JPEG_GRAYSCALE_8X8; + let mut options = JpegToHtj2kOptions::lossy_97(); + options + .encode_options + .irreversible_quantization_subband_scales + .high_high = 1.5; + let inputs = vec![JpegTileBatchInput { bytes: jpeg }; 2]; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = Prequantized97Accelerator::default(); + + let batch = transcoder + .transcode_batch_with_accelerator(&inputs, &options, &mut accelerator) + .expect("prequantized 9/7 batch transcode accepts subband profile"); + + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!( + accelerator + .last_options + .expect("accelerator received code-block options") + .irreversible_quantization_subband_scales + .high_high + .to_bits(), + 1.5f32.to_bits() + ); +} + +#[test] +fn integer_direct_batch_transcode_offers_ht_blocks_to_encode_accelerator() { + let jpeg = JPEG_GRAYSCALE_8X8; + let options = JpegToHtj2kOptions::lossless_53(); + let inputs = vec![JpegTileBatchInput { bytes: jpeg }; 4]; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut transform_accelerator = RayonReversibleDwt53Accelerator::default(); + let mut encode_accelerator = CountingHtEncodeAccelerator::default(); + + let batch = transcoder + .transcode_batch_with_accelerators( + &inputs, + &options, + &mut transform_accelerator, + &mut encode_accelerator, + ) + .expect("5/3 batch transcode accepts separate encode accelerator"); + + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!(encode_accelerator.batches, inputs.len()); + assert!(encode_accelerator.jobs > 0); + assert_eq!(encode_accelerator.single_blocks, encode_accelerator.jobs); +} + +#[test] +fn batch_transcode_preserves_encode_hooks_when_parallel_cpu_fallback_requested() { + let jpeg = JPEG_GRAYSCALE_8X8; + let options = JpegToHtj2kOptions::lossless_53(); + let inputs = vec![JpegTileBatchInput { bytes: jpeg }; 4]; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut transform_accelerator = RayonReversibleDwt53Accelerator::default(); + let mut encode_accelerator = CountingHtEncodeAccelerator { + parallel_cpu_code_block_fallback: true, + ..CountingHtEncodeAccelerator::default() + }; + + let batch = transcoder + .transcode_batch_with_accelerators( + &inputs, + &options, + &mut transform_accelerator, + &mut encode_accelerator, + ) + .expect("batch transcode preserves encode accelerator hooks"); + + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!(encode_accelerator.batches, inputs.len()); + assert!(encode_accelerator.jobs > 0); + assert_eq!(encode_accelerator.single_blocks, 0); +} + +#[test] +fn float97_batch_transcode_offers_prequantized_ht_blocks_to_encode_accelerator() { + let jpeg = JPEG_GRAYSCALE_8X8; + let options = JpegToHtj2kOptions::lossy_97(); + let inputs = vec![JpegTileBatchInput { bytes: jpeg }; 4]; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut transform_accelerator = Prequantized97Accelerator::default(); + let mut encode_accelerator = CountingHtEncodeAccelerator::default(); + + let batch = transcoder + .transcode_batch_with_accelerators( + &inputs, + &options, + &mut transform_accelerator, + &mut encode_accelerator, + ) + .expect("9/7 batch transcode accepts separate encode accelerator"); + + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!(transform_accelerator.prequantized_batch_sizes, vec![4]); + assert_eq!(encode_accelerator.batches, inputs.len()); + assert!(encode_accelerator.jobs > 0); + assert_eq!(encode_accelerator.single_blocks, encode_accelerator.jobs); +} + +#[test] +fn float97_preencoded_batch_skips_encode_codeblock_hooks() { + let jpeg = JPEG_GRAYSCALE_8X8; + let options = JpegToHtj2kOptions::lossy_97(); + let inputs = vec![JpegTileBatchInput { bytes: jpeg }; 4]; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut transform_accelerator = Preencoded97Accelerator::default(); + let mut encode_accelerator = CountingHtEncodeAccelerator::default(); + + let batch = transcoder + .transcode_batch_with_accelerators( + &inputs, + &options, + &mut transform_accelerator, + &mut encode_accelerator, + ) + .expect("9/7 preencoded batch transcode accepts separate encode accelerator"); + + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!(transform_accelerator.preencoded_batch_sizes, vec![4]); + assert_eq!(encode_accelerator.batches, 0); + assert_eq!(encode_accelerator.jobs, 0); + assert_eq!(batch.report.timings.dwt97_batch_ht_codeblock_dispatches, 1); +} + +#[test] +fn float97_preencoded_batch_groups_compatible_color_components() { + let jpeg = JPEG_BASELINE_444_8X8; + let options = JpegToHtj2kOptions::lossy_97(); + let inputs = vec![JpegTileBatchInput { bytes: jpeg }; 4]; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut transform_accelerator = Preencoded97Accelerator::default(); + let mut encode_accelerator = CountingHtEncodeAccelerator::default(); + + let batch = transcoder + .transcode_batch_with_accelerators( + &inputs, + &options, + &mut transform_accelerator, + &mut encode_accelerator, + ) + .expect("9/7 preencoded batch groups compatible color components"); + + assert_eq!(batch.report.successful_tiles, inputs.len()); + assert_eq!(transform_accelerator.preencoded_batch_sizes, vec![12]); + assert_eq!(encode_accelerator.batches, 0); + assert_eq!(encode_accelerator.jobs, 0); +} + +#[test] +fn batch_transcode_reports_bad_tiles_without_aborting_valid_tiles() { + let good = JPEG_GRAYSCALE_8X8; + let inputs = [ + JpegTileBatchInput { bytes: good }, + JpegTileBatchInput { + bytes: b"not a jpeg", + }, + JpegTileBatchInput { bytes: good }, + ]; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = CountingAccelerator::default(); + + let batch = transcoder + .transcode_batch_with_accelerator(&inputs, &JpegToHtj2kOptions::default(), &mut accelerator) + .expect("valid batch options do not fail globally"); + + assert_eq!(batch.report.tile_count, 3); + assert_eq!(batch.report.successful_tiles, 2); + assert_eq!(batch.report.failed_tiles, 1); + assert!(batch.tiles[0].is_ok()); + assert!(batch.tiles[1].is_err()); + assert!(batch.tiles[2].is_ok()); + assert_eq!(accelerator.reversible_dwt53_batch_sizes, vec![2]); +} + +#[test] +fn stateful_transcoder_reuses_integer_idct_block_scratch_across_tiles() { + let larger_jpeg = JPEG_BASELINE_420_16X16; + let smaller_jpeg = JPEG_GRAYSCALE_8X8; + let options = JpegToHtj2kOptions::default(); + let mut transcoder = JpegToHtj2kTranscoder::default(); + + let larger = transcoder + .transcode(larger_jpeg, &options) + .expect("stateful integer-direct transcode accepts 4:2:0 JPEG"); + let capacity_after_larger = transcoder.integer_idct_block_scratch_capacity(); + assert!(capacity_after_larger >= 4); + + let smaller = transcoder + .transcode(smaller_jpeg, &options) + .expect("stateful integer-direct transcode accepts grayscale JPEG"); + let stateless = jpeg_to_htj2k(smaller_jpeg, &options) + .expect("stateless integer-direct transcode accepts grayscale JPEG"); + + assert_eq!(larger.report.component_count, 3); + assert_eq!(smaller.report.component_count, 1); + assert_eq!( + transcoder.integer_idct_block_scratch_capacity(), + capacity_after_larger + ); + assert_eq!(smaller.codestream, stateless.codestream); +} + +#[derive(Default)] +struct CountingAccelerator { + reversible_dwt53_calls: usize, + reversible_dwt53_batch_calls: usize, + reversible_dwt53_batch_sizes: Vec, + dwt53_calls: usize, + dwt97_calls: usize, + dwt97_batch_calls: usize, + dwt97_batch_sizes: Vec, + dwt53_scratch: Dct53GridScratch, + dwt97_scratch: Dct97GridScratch, +} + +impl DctToWaveletStageAccelerator for CountingAccelerator { + fn supports_dwt97_batch(&self) -> bool { + true + } + + fn dct_grid_to_reversible_dwt53( + &mut self, + job: DctGridToReversibleDwt53Job<'_>, + ) -> Result, TranscodeStageError> { + self.reversible_dwt53_calls += 1; + Ok(Some( + RayonReversibleDwt53Accelerator::default() + .dct_grid_to_reversible_dwt53(job)? + .expect("rayon accelerator handles test job"), + )) + } + + fn dct_grid_to_reversible_dwt53_batch( + &mut self, + jobs: &[DctGridToReversibleDwt53Job<'_>], + ) -> Result>, TranscodeStageError> { + self.reversible_dwt53_batch_calls += 1; + self.reversible_dwt53_batch_sizes.push(jobs.len()); + let mut output = Vec::with_capacity(jobs.len()); + let mut rayon = RayonReversibleDwt53Accelerator::default(); + for job in jobs { + output.push( + rayon + .dct_grid_to_reversible_dwt53(*job)? + .expect("rayon accelerator handles batched test job"), + ); + } + Ok(Some(output)) + } + + fn dct_grid_to_dwt53( + &mut self, + job: DctGridToDwt53Job<'_>, + ) -> Result>, TranscodeStageError> { + self.dwt53_calls += 1; + let dwt = dct8x8_blocks_to_dwt53_float_linear_with_scratch( + job.blocks, + job.block_cols, + job.block_rows, + job.width, + job.height, + &mut self.dwt53_scratch, + ) + .map_err(|_| "test DCT 5/3 grid failed")?; + Ok(Some(dwt)) + } + + fn dct_grid_to_dwt97( + &mut self, + job: DctGridToDwt97Job<'_>, + ) -> Result>, TranscodeStageError> { + self.dwt97_calls += 1; + let dwt = dct8x8_blocks_then_dwt97_float_with_scratch( + job.blocks, + job.block_cols, + job.block_rows, + job.width, + job.height, + &mut self.dwt97_scratch, + ) + .map_err(|_| "test DCT 9/7 grid failed")?; + Ok(Some(dwt)) + } + + fn dct_grid_to_dwt97_batch( + &mut self, + jobs: &[DctGridToDwt97Job<'_>], + ) -> Result>>, TranscodeStageError> { + self.dwt97_batch_calls += 1; + self.dwt97_batch_sizes.push(jobs.len()); + let mut output = Vec::with_capacity(jobs.len()); + for job in jobs { + output.push( + dct8x8_blocks_then_dwt97_float_with_scratch( + job.blocks, + job.block_cols, + job.block_rows, + job.width, + job.height, + &mut self.dwt97_scratch, + ) + .map_err(|_| "test batched DCT 9/7 grid failed")?, + ); + } + Ok(Some(output)) + } +} + +struct BatchFixture { + name: &'static str, + jpeg: &'static [u8], + expected_batch_sizes: &'static [usize], +} + +#[derive(Default)] +struct Prequantized97Accelerator { + prequantized_batch_sizes: Vec, + dwt97_batch_calls: usize, + last_options: Option, +} + +#[derive(Default)] +struct Preencoded97Accelerator { + preencoded_batch_sizes: Vec, + last_timings: Option, +} + +#[derive(Default)] +struct CountingHtEncodeAccelerator { + batches: usize, + jobs: usize, + single_blocks: usize, + parallel_cpu_code_block_fallback: bool, +} + +impl J2kEncodeStageAccelerator for CountingHtEncodeAccelerator { + fn encode_ht_code_blocks( + &mut self, + jobs: &[J2kHtCodeBlockEncodeJob<'_>], + ) -> Result>, &'static str> { + self.batches += 1; + self.jobs += jobs.len(); + Ok(None) + } + + fn encode_ht_code_block( + &mut self, + _job: J2kHtCodeBlockEncodeJob<'_>, + ) -> Result, &'static str> { + self.single_blocks += 1; + Ok(None) + } + + fn prefer_parallel_cpu_code_block_fallback(&self) -> bool { + self.parallel_cpu_code_block_fallback + } +} + +impl DctToWaveletStageAccelerator for Prequantized97Accelerator { + fn supports_htj2k97_codeblock_batch(&self) -> bool { + true + } + + fn dct_grid_to_htj2k97_codeblock_batch( + &mut self, + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, + ) -> Result>, TranscodeStageError> { + self.prequantized_batch_sizes.push(jobs.len()); + self.last_options = Some(options); + Ok(Some( + jobs.iter() + .map(|job| zero_prequantized_component(job, options)) + .collect(), + )) + } + + fn dct_grid_to_dwt97_batch( + &mut self, + _jobs: &[DctGridToDwt97Job<'_>], + ) -> Result>>, TranscodeStageError> { + self.dwt97_batch_calls += 1; + Ok(None) + } +} + +impl DctToWaveletStageAccelerator for Preencoded97Accelerator { + fn supports_htj2k97_codeblock_batch(&self) -> bool { + true + } + + fn dct_grid_to_htj2k97_preencoded_batch( + &mut self, + jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + options: Htj2k97CodeBlockOptions, + ) -> Result>, TranscodeStageError> { + self.preencoded_batch_sizes.push(jobs.len()); + self.last_timings = Some(Dwt97BatchStageTimings { + ht_encode_us: 7, + ht_codeblock_dispatches: 1, + ..Dwt97BatchStageTimings::default() + }); + Ok(Some( + jobs.iter() + .map(|job| zero_preencoded_component(job, options)) + .collect(), + )) + } + + fn dct_grid_to_htj2k97_codeblock_batch( + &mut self, + _jobs: &[DctGridToHtj2k97CodeBlockJob<'_>], + _options: Htj2k97CodeBlockOptions, + ) -> Result>, TranscodeStageError> { + panic!("preencoded accelerator should be offered before prequantized fallback") + } + + fn last_dwt97_batch_stage_timings(&self) -> Option { + self.last_timings + } +} + +fn zero_prequantized_component( + job: &DctGridToHtj2k97CodeBlockJob<'_>, + options: Htj2k97CodeBlockOptions, +) -> PrequantizedHtj2k97Component { + let low_width = job.width.div_ceil(2); + let low_height = job.height.div_ceil(2); + let high_width = job.width / 2; + let high_height = job.height / 2; + PrequantizedHtj2k97Component { + x_rsiz: job.x_rsiz, + y_rsiz: job.y_rsiz, + resolutions: vec![ + PrequantizedHtj2k97Resolution { + subbands: vec![zero_prequantized_subband( + low_width, + low_height, + J2kSubBandType::LowLow, + zero_prequantized_total_bitplanes(options, J2kSubBandType::LowLow), + options, + )], + }, + PrequantizedHtj2k97Resolution { + subbands: vec![ + zero_prequantized_subband( + high_width, + low_height, + J2kSubBandType::HighLow, + zero_prequantized_total_bitplanes(options, J2kSubBandType::HighLow), + options, + ), + zero_prequantized_subband( + low_width, + high_height, + J2kSubBandType::LowHigh, + zero_prequantized_total_bitplanes(options, J2kSubBandType::LowHigh), + options, + ), + zero_prequantized_subband( + high_width, + high_height, + J2kSubBandType::HighHigh, + zero_prequantized_total_bitplanes(options, J2kSubBandType::HighHigh), + options, + ), + ], + }, + ], + } +} + +fn zero_preencoded_component( + job: &DctGridToHtj2k97CodeBlockJob<'_>, + options: Htj2k97CodeBlockOptions, +) -> PreencodedHtj2k97Component { + let low_width = job.width.div_ceil(2); + let low_height = job.height.div_ceil(2); + let high_width = job.width / 2; + let high_height = job.height / 2; + PreencodedHtj2k97Component { + x_rsiz: job.x_rsiz, + y_rsiz: job.y_rsiz, + resolutions: vec![ + PreencodedHtj2k97Resolution { + subbands: vec![zero_preencoded_subband( + low_width, + low_height, + J2kSubBandType::LowLow, + zero_prequantized_total_bitplanes(options, J2kSubBandType::LowLow), + options, + )], + }, + PreencodedHtj2k97Resolution { + subbands: vec![ + zero_preencoded_subband( + high_width, + low_height, + J2kSubBandType::HighLow, + zero_prequantized_total_bitplanes(options, J2kSubBandType::HighLow), + options, + ), + zero_preencoded_subband( + low_width, + high_height, + J2kSubBandType::LowHigh, + zero_prequantized_total_bitplanes(options, J2kSubBandType::LowHigh), + options, + ), + zero_preencoded_subband( + high_width, + high_height, + J2kSubBandType::HighHigh, + zero_prequantized_total_bitplanes(options, J2kSubBandType::HighHigh), + options, + ), + ], + }, + ], + } +} + +fn zero_prequantized_total_bitplanes( + options: Htj2k97CodeBlockOptions, + sub_band_type: J2kSubBandType, +) -> u8 { + let base_delta = pow2i_f32_for_test(-i32::from(options.guard_bits)) + * options.irreversible_quantization_scale + * match sub_band_type { + J2kSubBandType::LowLow => options.irreversible_quantization_subband_scales.low_low, + J2kSubBandType::HighLow => options.irreversible_quantization_subband_scales.high_low, + J2kSubBandType::LowHigh => options.irreversible_quantization_subband_scales.low_high, + J2kSubBandType::HighHigh => options.irreversible_quantization_subband_scales.high_high, + }; + let floor_log2 = base_delta.log2().floor() as i32; + let mut exponent = i32::from(options.bit_depth) - floor_log2; + let normalized = base_delta / pow2i_f32_for_test(floor_log2); + let mantissa = ((normalized - 1.0) * 2048.0).round() as i32; + + if mantissa >= 2048 { + exponent -= 1; + } + + options + .guard_bits + .saturating_add(u8::try_from(exponent.clamp(0, 31)).expect("clamped exponent fits u8")) + .saturating_sub(1) +} + +fn pow2i_f32_for_test(exp: i32) -> f32 { + if exp >= 0 { + (1u32 << exp.cast_unsigned()) as f32 + } else { + 1.0 / (1u32 << (-exp).cast_unsigned()) as f32 + } +} + +fn zero_preencoded_subband( + width: usize, + height: usize, + sub_band_type: J2kSubBandType, + total_bitplanes: u8, + options: Htj2k97CodeBlockOptions, +) -> PreencodedHtj2k97Subband { + let cb_width = 1usize << (options.code_block_width_exp + 2); + let cb_height = 1usize << (options.code_block_height_exp + 2); + let num_cbs_x = width.div_ceil(cb_width); + let num_cbs_y = height.div_ceil(cb_height); + let mut code_blocks = Vec::with_capacity(num_cbs_x * num_cbs_y); + for cby in 0..num_cbs_y { + for cbx in 0..num_cbs_x { + let x0 = cbx * cb_width; + let y0 = cby * cb_height; + let block_width = (width - x0).min(cb_width); + let block_height = (height - y0).min(cb_height); + code_blocks.push(PreencodedHtj2k97CodeBlock { + width: block_width as u32, + height: block_height as u32, + encoded: EncodedHtJ2kCodeBlock { + data: Vec::new(), + cleanup_length: 0, + refinement_length: 0, + num_coding_passes: 0, + num_zero_bitplanes: total_bitplanes, + }, + }); + } + } + + PreencodedHtj2k97Subband { + sub_band_type, + num_cbs_x: num_cbs_x as u32, + num_cbs_y: num_cbs_y as u32, + total_bitplanes, + code_blocks, + } +} + +fn zero_prequantized_subband( + width: usize, + height: usize, + sub_band_type: J2kSubBandType, + total_bitplanes: u8, + options: Htj2k97CodeBlockOptions, +) -> PrequantizedHtj2k97Subband { + let cb_width = 1usize << (options.code_block_width_exp + 2); + let cb_height = 1usize << (options.code_block_height_exp + 2); + let num_cbs_x = width.div_ceil(cb_width); + let num_cbs_y = height.div_ceil(cb_height); + let mut code_blocks = Vec::with_capacity(num_cbs_x * num_cbs_y); + for cby in 0..num_cbs_y { + for cbx in 0..num_cbs_x { + let x0 = cbx * cb_width; + let y0 = cby * cb_height; + let block_width = (width - x0).min(cb_width); + let block_height = (height - y0).min(cb_height); + code_blocks.push(PrequantizedHtj2k97CodeBlock { + coefficients: vec![0; block_width * block_height], + width: block_width as u32, + height: block_height as u32, + }); + } + } + + PrequantizedHtj2k97Subband { + sub_band_type, + num_cbs_x: num_cbs_x as u32, + num_cbs_y: num_cbs_y as u32, + total_bitplanes, + code_blocks, + } +} + +fn encoded_gray_jpeg(width: u32, height: u32) -> Vec { + let gray = patterned_gray(width, height); + encode_jpeg_baseline( + JpegSamples::Gray8 { + data: &gray, + width, + height, + }, + JpegEncodeOptions { + quality: 90, + subsampling: JpegSubsampling::Gray, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + ) + .expect("encode grayscale JPEG fixture") + .data +} + +fn patterned_gray(width: u32, height: u32) -> Vec { + let mut out = Vec::with_capacity(width as usize * height as usize); + for y in 0..height { + for x in 0..width { + out.push(((x * 7 + y * 11 + 19) & 0xff) as u8); + } + } + out +} + +fn assert_component_sampling(codestream: &[u8], expected: &[(u8, u8)]) { + let siz = find_marker(codestream, 0x51).expect("SIZ marker"); + let component_info = siz + 40; + for (component_index, &(x_rsiz, y_rsiz)) in expected.iter().enumerate() { + let offset = component_info + component_index * 3; + assert_eq!(codestream[offset + 1], x_rsiz); + assert_eq!(codestream[offset + 2], y_rsiz); + } +} + +fn assert_report_sampling(encoded: &EncodedTranscode, expected: &[(u32, u32, u8, u8)]) { + assert_eq!(encoded.report.components.len(), expected.len()); + for (component, &(width, height, x_rsiz, y_rsiz)) in + encoded.report.components.iter().zip(expected) + { + assert_eq!((component.width, component.height), (width, height)); + assert_eq!((component.x_rsiz, component.y_rsiz), (x_rsiz, y_rsiz)); + } +} + +#[derive(Debug, Clone, Copy)] +enum ExternalDecoder { + Grok, + OpenJpeg, +} + +fn available_external_decoders() -> Vec { + let mut decoders = Vec::new(); + if Command::new("grk_decompress").arg("-h").output().is_ok() { + decoders.push(ExternalDecoder::Grok); + } + if Command::new("opj_decompress").arg("-h").output().is_ok() { + decoders.push(ExternalDecoder::OpenJpeg); + } + decoders +} + +fn run_external_decoder(decoder: ExternalDecoder, codestream: &[u8]) -> Result<(), String> { + let ExternalDecodeFiles { + input_path, + output_path, + } = write_external_decode_input(codestream)?; + let output = match decoder { + ExternalDecoder::Grok => Command::new("grk_decompress") + .arg("-i") + .arg(&input_path) + .arg("-o") + .arg(&output_path) + .arg("-O") + .arg("PNM") + .output(), + ExternalDecoder::OpenJpeg => Command::new("opj_decompress") + .arg("-quiet") + .arg("-i") + .arg(&input_path) + .arg("-o") + .arg(&output_path) + .output(), + }; + let output = output.map_err(|err| err.to_string()); + let _ = fs::remove_file(&input_path); + let _ = fs::remove_file(&output_path); + + let output = output?; + if output.status.success() { + Ok(()) + } else { + Err(format_command_output(&output)) + } +} + +struct ExternalDecodeFiles { + input_path: PathBuf, + output_path: PathBuf, +} + +fn write_external_decode_input(codestream: &[u8]) -> Result { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|err| err.to_string())? + .as_nanos(); + let stem = format!("j2k-transcode-{}-{unique}", std::process::id()); + let input_path = env::temp_dir().join(format!("{stem}.j2k")); + let output_path = env::temp_dir().join(format!("{stem}.pgm")); + fs::write(&input_path, codestream).map_err(|err| err.to_string())?; + + Ok(ExternalDecodeFiles { + input_path, + output_path, + }) +} + +fn format_command_output(output: &Output) -> String { + format!( + "status: {}; stdout: {}; stderr: {}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +fn find_marker(codestream: &[u8], marker: u8) -> Option { + codestream + .windows(2) + .position(|window| window == [0xff, marker]) +} diff --git a/crates/j2k-types/Cargo.toml b/crates/j2k-types/Cargo.toml new file mode 100644 index 00000000..39ceb615 --- /dev/null +++ b/crates/j2k-types/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "j2k-types" +description = "Shared JPEG 2000 and HTJ2K encode-stage contract types for j2k" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +readme = "README.md" + +[package.metadata.docs.rs] +all-features = true +targets = [] + +[lib] +name = "j2k_types" +path = "src/lib.rs" + +[dependencies] + +[lints] +workspace = true diff --git a/crates/j2k-types/README.md b/crates/j2k-types/README.md new file mode 100644 index 00000000..22d99d23 --- /dev/null +++ b/crates/j2k-types/README.md @@ -0,0 +1,10 @@ +# j2k-types + +Shared JPEG 2000 encode-stage contract types for the j2k workspace. + +This crate is the neutral public contract between the `j2k` adapter +surface and the `j2k-native` codec engine: encode-stage job, output, +and report types are defined once here so neither crate mirrors the other's +definitions. It contains plain data types only — codec behavior lives in +`j2k-native`, and the encode-stage accelerator traits stay in their +owning crates. diff --git a/crates/j2k-types/src/lib.rs b/crates/j2k-types/src/lib.rs new file mode 100644 index 00000000..8d511193 --- /dev/null +++ b/crates/j2k-types/src/lib.rs @@ -0,0 +1,775 @@ +//! Shared JPEG 2000 encode-stage contract types for j2k. +//! +//! This crate is the neutral public contract between the `j2k` +//! adapter surface and the `j2k-native` codec engine: job, output, +//! and report types cross the boundary here so neither crate mirrors the +//! other's definitions. It intentionally contains plain data types only - +//! codec behavior stays in `j2k-native` and adapter traits stay in +//! their owning crates. + +#![no_std] +#![forbid(unsafe_code)] +#![forbid(missing_docs)] + +extern crate alloc; + +use alloc::vec::Vec; +use core::ops::Range; + +/// Adapter classic J2K sub-band kind for backend experimentation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum J2kSubBandType { + /// Low-low sub-band. + LowLow, + /// High-low sub-band. + HighLow, + /// Low-high sub-band. + LowHigh, + /// High-high sub-band. + HighHigh, +} + +/// Adapter classic J2K code-block style for backend experimentation. +#[derive(Debug, Clone, Copy)] +#[allow(clippy::struct_excessive_bools)] // models the five independent COD code-block style flags +pub struct J2kCodeBlockStyle { + /// Selective arithmetic coding bypass was enabled. + pub selective_arithmetic_coding_bypass: bool, + /// Context probabilities reset after each pass. + pub reset_context_probabilities: bool, + /// Coding terminated after each pass. + pub termination_on_each_pass: bool, + /// Vertically causal context was enabled. + pub vertically_causal_context: bool, + /// Segmentation symbols were enabled. + pub segmentation_symbols: bool, +} + +/// Adapter classic J2K coded segment for backend experimentation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct J2kCodeBlockSegment { + /// Byte offset of this segment within the combined payload. + pub data_offset: u32, + /// Segment payload length in bytes. + pub data_length: u32, + /// First coding pass covered by this segment. + pub start_coding_pass: u8, + /// One-past-last coding pass covered by this segment. + pub end_coding_pass: u8, + /// Whether this segment is decoded through the arithmetic path. + pub use_arithmetic: bool, +} + +/// Adapter encoded classic J2K code-block payload for backend experimentation. +#[derive(Debug, Clone)] +pub struct EncodedJ2kCodeBlock { + /// Combined payload bytes for all coded segments in this code block. + pub data: Vec, + /// Coded segments for the code block. + pub segments: Vec, + /// Number of coding passes present for this code block. + pub number_of_coding_passes: u8, + /// Missing most-significant bit planes for this code block. + pub missing_bit_planes: u8, +} + +/// Adapter encoded HTJ2K cleanup code-block payload for backend experimentation. +#[derive(Debug, Clone)] +pub struct EncodedHtJ2kCodeBlock { + /// Combined cleanup/refinement bytes for this code block. + pub data: Vec, + /// Cleanup segment length in bytes. + pub cleanup_length: u32, + /// Refinement segment length in bytes. + pub refinement_length: u32, + /// Number of coding passes present for this code block. + pub num_coding_passes: u8, + /// Number of zero most-significant bitplanes before first inclusion. + pub num_zero_bitplanes: u8, +} + +/// Adapter pixel deinterleave/level-shift job for backend experimentation. +#[derive(Debug, Clone, Copy)] +pub struct J2kDeinterleaveToF32Job<'a> { + /// Interleaved source pixel bytes. + pub pixels: &'a [u8], + /// Number of pixels to convert. + pub num_pixels: usize, + /// Number of interleaved components per pixel. + pub num_components: u8, + /// Source sample bit depth. + pub bit_depth: u8, + /// Whether source samples are signed. + pub signed: bool, +} + +/// Adapter forward RCT job for backend experimentation. +#[derive(Debug)] +pub struct J2kForwardRctJob<'a> { + /// First component plane, updated in place. + pub plane0: &'a mut [f32], + /// Second component plane, updated in place. + pub plane1: &'a mut [f32], + /// Third component plane, updated in place. + pub plane2: &'a mut [f32], +} + +/// Adapter forward ICT job for backend experimentation. +#[derive(Debug)] +pub struct J2kForwardIctJob<'a> { + /// First component plane, updated in place. + pub plane0: &'a mut [f32], + /// Second component plane, updated in place. + pub plane1: &'a mut [f32], + /// Third component plane, updated in place. + pub plane2: &'a mut [f32], +} + +/// Adapter forward 5/3 DWT job for backend experimentation. +#[derive(Debug, Clone, Copy)] +pub struct J2kForwardDwt53Job<'a> { + /// Source samples in row-major order. + pub samples: &'a [f32], + /// Source width in samples. + pub width: u32, + /// Source height in samples. + pub height: u32, + /// Number of decomposition levels requested. + pub num_levels: u8, +} + +/// Adapter forward 5/3 DWT output for backend experimentation. +#[derive(Debug, Clone)] +pub struct J2kForwardDwt53Output { + /// LL subband coefficients from the lowest decomposition level. + pub ll: Vec, + /// LL subband width. + pub ll_width: u32, + /// LL subband height. + pub ll_height: u32, + /// Higher resolution detail levels, ordered from lowest to highest. + pub levels: Vec, +} + +/// Adapter forward 5/3 DWT detail level for backend experimentation. +#[derive(Debug, Clone)] +pub struct J2kForwardDwt53Level { + /// HL subband coefficients. + pub hl: Vec, + /// LH subband coefficients. + pub lh: Vec, + /// HH subband coefficients. + pub hh: Vec, + /// Full-resolution width represented by this level. + pub width: u32, + /// Full-resolution height represented by this level. + pub height: u32, + /// Low-pass width at this level. + pub low_width: u32, + /// Low-pass height at this level. + pub low_height: u32, + /// High-pass width at this level. + pub high_width: u32, + /// High-pass height at this level. + pub high_height: u32, +} + +/// Adapter forward irreversible 9/7 DWT job for backend experimentation. +#[derive(Debug, Clone, Copy)] +pub struct J2kForwardDwt97Job<'a> { + /// Source samples in row-major order. + pub samples: &'a [f32], + /// Source width in samples. + pub width: u32, + /// Source height in samples. + pub height: u32, + /// Number of decomposition levels requested. + pub num_levels: u8, +} + +/// Adapter forward 9/7 DWT output for backend experimentation. +#[derive(Debug, Clone)] +pub struct J2kForwardDwt97Output { + /// LL subband coefficients from the lowest decomposition level. + pub ll: Vec, + /// LL subband width. + pub ll_width: u32, + /// LL subband height. + pub ll_height: u32, + /// Higher resolution detail levels, ordered from lowest to highest. + pub levels: Vec, +} + +/// Adapter forward 9/7 DWT detail level for backend experimentation. +#[derive(Debug, Clone)] +pub struct J2kForwardDwt97Level { + /// HL subband coefficients. + pub hl: Vec, + /// LH subband coefficients. + pub lh: Vec, + /// HH subband coefficients. + pub hh: Vec, + /// Full-resolution width represented by this level. + pub width: u32, + /// Full-resolution height represented by this level. + pub height: u32, + /// Low-pass width at this level. + pub low_width: u32, + /// Low-pass height at this level. + pub low_height: u32, + /// High-pass width at this level. + pub high_width: u32, + /// High-pass height at this level. + pub high_height: u32, +} + +/// Adapter sub-band quantization job for backend experimentation. +#[derive(Debug, Clone, Copy)] +pub struct J2kQuantizeSubbandJob<'a> { + /// Source sub-band coefficients in row-major order. + pub coefficients: &'a [f32], + /// Quantization step-size exponent. + pub step_exponent: u16, + /// Quantization step-size mantissa. + pub step_mantissa: u16, + /// Nominal range bits for this sub-band. + pub range_bits: u8, + /// Whether to use reversible integer quantization. + pub reversible: bool, +} + +/// Adapter Tier-1 classic J2K code-block encode job for backend experimentation. +#[derive(Debug, Clone, Copy)] +pub struct J2kTier1CodeBlockEncodeJob<'a> { + /// Quantized coefficients in row-major order. + pub coefficients: &'a [i32], + /// Code-block width in samples. + pub width: u32, + /// Code-block height in samples. + pub height: u32, + /// Subband kind containing this code-block. + pub sub_band_type: J2kSubBandType, + /// Total bitplanes for this subband/code-block. + pub total_bitplanes: u8, + /// Classic J2K code-block style flags. + pub style: J2kCodeBlockStyle, +} + +/// Adapter HTJ2K code-block encode job for backend experimentation. +#[derive(Debug, Clone, Copy)] +pub struct J2kHtCodeBlockEncodeJob<'a> { + /// Quantized coefficients in row-major order. + pub coefficients: &'a [i32], + /// Code-block width in samples. + pub width: u32, + /// Code-block height in samples. + pub height: u32, + /// Total bitplanes for this subband/code-block. + pub total_bitplanes: u8, + /// Requested HT coding passes for this contribution. + /// + /// `1` is cleanup-only. Higher values require an accelerator that can + /// encode those passes and must not be silently reduced by CPU fallback. + pub target_coding_passes: u8, +} + +/// Adapter HTJ2K cleanup encode job for one unquantized sub-band. +#[derive(Debug, Clone, Copy)] +pub struct J2kHtSubbandEncodeJob<'a> { + /// Source sub-band coefficients in row-major order. + pub coefficients: &'a [f32], + /// Sub-band width in samples. + pub width: u32, + /// Sub-band height in samples. + pub height: u32, + /// Quantization step-size exponent. + pub step_exponent: u16, + /// Quantization step-size mantissa. + pub step_mantissa: u16, + /// Nominal range bits for this sub-band. + pub range_bits: u8, + /// Whether to use reversible integer quantization. + pub reversible: bool, + /// Code-block width in samples. + pub code_block_width: u32, + /// Code-block height in samples. + pub code_block_height: u32, + /// Total coded bitplanes for this sub-band. + pub total_bitplanes: u8, +} + +/// Adapter HTJ2K tile-body encode job for backend-resident full-tile paths. +#[derive(Debug, Clone, Copy)] +pub struct J2kHtj2kTileEncodeJob<'a> { + /// Interleaved source pixel bytes. + pub pixels: &'a [u8], + /// Tile/image width in samples. + pub width: u32, + /// Tile/image height in samples. + pub height: u32, + /// Number of interleaved image components. + pub num_components: u8, + /// Source component bit depth. + pub bit_depth: u8, + /// Whether source samples are signed. + pub signed: bool, + /// Number of DWT decomposition levels. + pub num_decomposition_levels: u8, + /// Whether the codestream uses reversible coding. + pub reversible: bool, + /// Whether a multi-component transform should be applied. + pub use_mct: bool, + /// JPEG 2000 guard bits used to derive total coded bitplanes. + pub guard_bits: u8, + /// Code-block width in samples. + pub code_block_width: u32, + /// Code-block height in samples. + pub code_block_height: u32, + /// Packet progression order to emit. + pub progression_order: J2kPacketizationProgressionOrder, + /// Per-component sampling factors, as `(x_rsiz, y_rsiz)`. + pub component_sampling: &'a [(u8, u8)], + /// Quantization step sizes, as `(exponent, mantissa)`, in codestream order. + pub quantization_steps: &'a [(u16, u16)], +} + +/// Adapter LRCP packetization code-block contribution for backend experimentation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct J2kPacketizationCodeBlock<'a> { + /// Encoded Tier-1 bitstream bytes for this packet contribution. + pub data: &'a [u8], + /// HTJ2K cleanup segment length in bytes when using high-throughput coding. + pub ht_cleanup_length: u32, + /// HTJ2K refinement segment length in bytes when using high-throughput coding. + pub ht_refinement_length: u32, + /// Number of coding passes in this contribution. + pub num_coding_passes: u8, + /// Number of zero most-significant bitplanes before first inclusion. + pub num_zero_bitplanes: u8, + /// Whether this code-block was included in a previous packet. + pub previously_included: bool, + /// L-block value used for segment length coding. + pub l_block: u32, + /// Block coder used for this contribution. + pub block_coding_mode: J2kPacketizationBlockCodingMode, +} + +/// Adapter packetization block coding mode for backend experimentation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum J2kPacketizationBlockCodingMode { + /// Classic JPEG 2000 Part 1 EBCOT block coding. + Classic, + /// High-throughput JPEG 2000 Part 15 block coding. + HighThroughput, +} + +/// Adapter packet progression order for backend packetization experimentation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum J2kPacketizationProgressionOrder { + /// Layer-resolution-component-position progression. + Lrcp, + /// Resolution-layer-component-position progression. + Rlcp, + /// Resolution-position-component-layer progression. + Rpcl, + /// Position-component-resolution-layer progression. + Pcrl, + /// Component-position-resolution-layer progression. + Cprl, +} + +/// Adapter LRCP packetization subband precinct for backend experimentation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct J2kPacketizationSubband<'a> { + /// Code-block contributions in row-major order. + pub code_blocks: Vec>, + /// Number of code-blocks in the x direction. + pub num_cbs_x: u32, + /// Number of code-blocks in the y direction. + pub num_cbs_y: u32, +} + +/// Adapter LRCP packetization resolution packet for backend experimentation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct J2kPacketizationResolution<'a> { + /// Subbands in packet order: LL for resolution 0, then HL/LH/HH. + pub subbands: Vec>, +} + +/// Adapter explicit packet descriptor for backend packetization experimentation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct J2kPacketizationPacketDescriptor { + /// Index into the packet contribution array. + pub packet_index: u32, + /// Persistent packet-state index for repeated layer/precinct packets. + pub state_index: u32, + /// Quality layer for inclusion tag-tree thresholds. + pub layer: u8, + /// Resolution index in the output progression. + pub resolution: u32, + /// Component index in the output progression. + pub component: u8, + /// Precinct index in the output progression. + pub precinct: u64, +} + +/// Adapter LRCP packetization job for backend experimentation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct J2kPacketizationEncodeJob<'a> { + /// Number of resolution packets prepared for packetization. + pub resolution_count: u32, + /// Number of layers to write. + pub num_layers: u8, + /// Number of image components. + pub num_components: u8, + /// Total number of code-block contributions. + pub code_block_count: u32, + /// Packet progression order to emit. + pub progression_order: J2kPacketizationProgressionOrder, + /// Explicit packet descriptors in output progression order. + pub packet_descriptors: &'a [J2kPacketizationPacketDescriptor], + /// Packet payload prepared by Tier-1, in LRCP packet order. + pub resolutions: &'a [J2kPacketizationResolution<'a>], +} + +/// Adapter encode-stage dispatch counters for backend experimentation. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct J2kEncodeDispatchReport { + /// Pixel deinterleave/level-shift dispatch count. + pub deinterleave: usize, + /// Forward RCT kernel dispatch count. + pub forward_rct: usize, + /// Forward ICT kernel dispatch count. + pub forward_ict: usize, + /// Forward reversible 5/3 DWT kernel dispatch count. + pub forward_dwt53: usize, + /// Forward irreversible 9/7 DWT kernel dispatch count. + pub forward_dwt97: usize, + /// Sub-band quantization dispatch count. + pub quantize_subband: usize, + /// Tier-1 code-block encode dispatch count. + pub tier1_code_block: usize, + /// HTJ2K code-block encode dispatch count. + pub ht_code_block: usize, + /// Packetization dispatch count. + pub packetization: usize, +} + +impl J2kEncodeDispatchReport { + /// Return the saturating per-stage delta from `before` to `self`. + #[must_use] + pub fn saturating_delta(self, before: Self) -> Self { + Self { + deinterleave: self.deinterleave.saturating_sub(before.deinterleave), + forward_rct: self.forward_rct.saturating_sub(before.forward_rct), + forward_ict: self.forward_ict.saturating_sub(before.forward_ict), + forward_dwt53: self.forward_dwt53.saturating_sub(before.forward_dwt53), + forward_dwt97: self.forward_dwt97.saturating_sub(before.forward_dwt97), + quantize_subband: self + .quantize_subband + .saturating_sub(before.quantize_subband), + tier1_code_block: self + .tier1_code_block + .saturating_sub(before.tier1_code_block), + ht_code_block: self.ht_code_block.saturating_sub(before.ht_code_block), + packetization: self.packetization.saturating_sub(before.packetization), + } + } + + /// Return total dispatches across all encode stages. + #[must_use] + pub fn total(self) -> usize { + self.forward_rct + .saturating_add(self.deinterleave) + .saturating_add(self.forward_ict) + .saturating_add(self.forward_dwt53) + .saturating_add(self.forward_dwt97) + .saturating_add(self.quantize_subband) + .saturating_add(self.tier1_code_block) + .saturating_add(self.ht_code_block) + .saturating_add(self.packetization) + } + + /// Return whether at least one encode stage dispatched. + #[must_use] + pub fn any(self) -> bool { + self.total() > 0 + } +} + +/// Adapter CPU-only encode accelerator that always falls back to native stages. +#[derive(Debug, Default, Clone, Copy)] +pub struct CpuOnlyJ2kEncodeStageAccelerator; + +/// Multipliers applied to irreversible 9/7 quantization step sizes by subband. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct IrreversibleQuantizationSubbandScales { + /// Multiplier for the LL subband. + pub low_low: f32, + /// Multiplier for HL subbands. + pub high_low: f32, + /// Multiplier for LH subbands. + pub low_high: f32, + /// Multiplier for HH subbands. + pub high_high: f32, +} + +/// Public JPEG 2000 irreversible quantization step-size tuple. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IrreversibleQuantizationStep { + /// Quantization step-size exponent. + pub exponent: u8, + /// Quantization step-size mantissa. + pub mantissa: u16, +} + +impl Default for IrreversibleQuantizationSubbandScales { + fn default() -> Self { + Self { + low_low: 1.0, + high_low: 1.0, + low_high: 1.0, + high_high: 1.0, + } + } +} + +/// Precomputed reversible 5/3 wavelet coefficients for one component. +#[derive(Debug, Clone)] +pub struct PrecomputedHtj2k53Component { + /// Horizontal SIZ sampling factor (`XRsiz`). + pub x_rsiz: u8, + /// Vertical SIZ sampling factor (`YRsiz`). + pub y_rsiz: u8, + /// Forward 5/3 DWT output, ordered as the encoder expects. + pub dwt: J2kForwardDwt53Output, +} + +/// Precomputed reversible 5/3 wavelet image. +#[derive(Debug, Clone)] +pub struct PrecomputedHtj2k53Image { + /// Reference-grid image width. + pub width: u32, + /// Reference-grid image height. + pub height: u32, + /// Component precision in bits. + pub bit_depth: u8, + /// Whether component samples are signed. + pub signed: bool, + /// Components at their native resolution. + pub components: Vec, +} + +/// Precomputed irreversible 9/7 wavelet coefficients for one component. +#[derive(Debug, Clone)] +pub struct PrecomputedHtj2k97Component { + /// Horizontal SIZ sampling factor (`XRsiz`). + pub x_rsiz: u8, + /// Vertical SIZ sampling factor (`YRsiz`). + pub y_rsiz: u8, + /// Forward 9/7 DWT output, ordered as the encoder expects. + pub dwt: J2kForwardDwt97Output, +} + +/// Precomputed irreversible 9/7 wavelet image. +#[derive(Debug, Clone)] +pub struct PrecomputedHtj2k97Image { + /// Reference-grid image width. + pub width: u32, + /// Reference-grid image height. + pub height: u32, + /// Component precision in bits. + pub bit_depth: u8, + /// Whether component samples are signed. + pub signed: bool, + /// Components at their native resolution. + pub components: Vec, +} + +/// Prequantized irreversible 9/7 HTJ2K code-block image. +#[derive(Debug, Clone)] +pub struct PrequantizedHtj2k97Image { + /// Reference-grid image width. + pub width: u32, + /// Reference-grid image height. + pub height: u32, + /// Component precision in bits. + pub bit_depth: u8, + /// Whether component samples are signed. + pub signed: bool, + /// Components at their native resolution. + pub components: Vec, +} + +/// Prequantized irreversible 9/7 HTJ2K component. +#[derive(Debug, Clone)] +pub struct PrequantizedHtj2k97Component { + /// Horizontal SIZ sampling factor (`XRsiz`). + pub x_rsiz: u8, + /// Vertical SIZ sampling factor (`YRsiz`). + pub y_rsiz: u8, + /// Resolution packets for this component, ordered from lowest to highest. + pub resolutions: Vec, +} + +/// One component resolution's prequantized HTJ2K subbands. +#[derive(Debug, Clone)] +pub struct PrequantizedHtj2k97Resolution { + /// Subbands in packet order: LL for resolution 0, then HL/LH/HH. + pub subbands: Vec, +} + +/// One prequantized HTJ2K subband split into code-blocks. +#[derive(Debug, Clone)] +pub struct PrequantizedHtj2k97Subband { + /// Subband kind. + pub sub_band_type: J2kSubBandType, + /// Number of code-blocks in the x direction. + pub num_cbs_x: u32, + /// Number of code-blocks in the y direction. + pub num_cbs_y: u32, + /// Total bitplanes declared for every code-block in this subband. + pub total_bitplanes: u8, + /// Code-block coefficients in row-major code-block order. + pub code_blocks: Vec, +} + +/// One prequantized HTJ2K code-block. +#[derive(Debug, Clone)] +pub struct PrequantizedHtj2k97CodeBlock { + /// Quantized coefficients in row-major order. + pub coefficients: Vec, + /// Code-block width in coefficients. + pub width: u32, + /// Code-block height in coefficients. + pub height: u32, +} + +/// Preencoded irreversible 9/7 HTJ2K code-block image. +#[derive(Debug, Clone)] +pub struct PreencodedHtj2k97Image { + /// Reference-grid image width. + pub width: u32, + /// Reference-grid image height. + pub height: u32, + /// Component precision in bits. + pub bit_depth: u8, + /// Whether component samples are signed. + pub signed: bool, + /// Components at their native resolution. + pub components: Vec, +} + +/// Preencoded irreversible 9/7 HTJ2K component. +#[derive(Debug, Clone)] +pub struct PreencodedHtj2k97Component { + /// Horizontal SIZ sampling factor (`XRsiz`). + pub x_rsiz: u8, + /// Vertical SIZ sampling factor (`YRsiz`). + pub y_rsiz: u8, + /// Resolution packets for this component, ordered from lowest to highest. + pub resolutions: Vec, +} + +/// One component resolution's preencoded HTJ2K subbands. +#[derive(Debug, Clone)] +pub struct PreencodedHtj2k97Resolution { + /// Subbands in packet order: LL for resolution 0, then HL/LH/HH. + pub subbands: Vec, +} + +/// One preencoded HTJ2K subband split into code-blocks. +#[derive(Debug, Clone)] +pub struct PreencodedHtj2k97Subband { + /// Subband kind. + pub sub_band_type: J2kSubBandType, + /// Number of code-blocks in the x direction. + pub num_cbs_x: u32, + /// Number of code-blocks in the y direction. + pub num_cbs_y: u32, + /// Total bitplanes declared for every code-block in this subband. + pub total_bitplanes: u8, + /// Encoded code-block payloads in row-major code-block order. + pub code_blocks: Vec, +} + +/// One preencoded HTJ2K code-block. +#[derive(Debug, Clone)] +pub struct PreencodedHtj2k97CodeBlock { + /// Code-block width in coefficients. + pub width: u32, + /// Code-block height in coefficients. + pub height: u32, + /// Encoded cleanup/refinement payload and packet metadata. + pub encoded: EncodedHtJ2kCodeBlock, +} + +/// Preencoded irreversible 9/7 HTJ2K code-block image backed by one compact +/// payload buffer. +#[derive(Debug, Clone)] +pub struct PreencodedHtj2k97CompactImage { + /// Reference-grid image width. + pub width: u32, + /// Reference-grid image height. + pub height: u32, + /// Component precision in bits. + pub bit_depth: u8, + /// Whether component samples are signed. + pub signed: bool, + /// Contiguous encoded code-block payload bytes. + pub payload: Vec, + /// Components at their native resolution. + pub components: Vec, +} + +/// Preencoded compact irreversible 9/7 HTJ2K component. +#[derive(Debug, Clone)] +pub struct PreencodedHtj2k97CompactComponent { + /// Horizontal SIZ sampling factor (`XRsiz`). + pub x_rsiz: u8, + /// Vertical SIZ sampling factor (`YRsiz`). + pub y_rsiz: u8, + /// Resolution packets for this component, ordered from lowest to highest. + pub resolutions: Vec, +} + +/// One component resolution's compact preencoded HTJ2K subbands. +#[derive(Debug, Clone)] +pub struct PreencodedHtj2k97CompactResolution { + /// Subbands in packet order: LL for resolution 0, then HL/LH/HH. + pub subbands: Vec, +} + +/// One compact preencoded HTJ2K subband split into code-blocks. +#[derive(Debug, Clone)] +pub struct PreencodedHtj2k97CompactSubband { + /// Subband kind. + pub sub_band_type: J2kSubBandType, + /// Number of code-blocks in the x direction. + pub num_cbs_x: u32, + /// Number of code-blocks in the y direction. + pub num_cbs_y: u32, + /// Total bitplanes declared for every code-block in this subband. + pub total_bitplanes: u8, + /// Code-block metadata in row-major code-block order. + pub code_blocks: Vec, +} + +/// One compact preencoded HTJ2K code-block. +#[derive(Debug, Clone)] +pub struct PreencodedHtj2k97CompactCodeBlock { + /// Code-block width in coefficients. + pub width: u32, + /// Code-block height in coefficients. + pub height: u32, + /// Byte range into the image-level compact payload. + pub payload_range: Range, + /// HTJ2K cleanup segment length in bytes. + pub cleanup_length: u32, + /// HTJ2K refinement segment length in bytes. + pub refinement_length: u32, + /// Number of coding passes in the encoded payload. + pub num_coding_passes: u8, + /// Number of missing most-significant bitplanes. + pub num_zero_bitplanes: u8, +} diff --git a/crates/signinum-j2k/Cargo.toml b/crates/j2k/Cargo.toml similarity index 63% rename from crates/signinum-j2k/Cargo.toml rename to crates/j2k/Cargo.toml index e8be72cc..4880f262 100644 --- a/crates/signinum-j2k/Cargo.toml +++ b/crates/j2k/Cargo.toml @@ -1,7 +1,7 @@ [package] -name = "signinum-j2k" -description = "JPEG 2000 inspector and decoder for whole-slide images" -version = "0.4.2" +name = "j2k" +description = "GPU-aware JPEG 2000 and HTJ2K decoder/encoder APIs in Rust" +version.workspace = true edition.workspace = true rust-version.workspace = true license.workspace = true @@ -10,20 +10,25 @@ keywords.workspace = true categories.workspace = true readme = "README.md" +[package.metadata.docs.rs] +all-features = true +targets = [] + [lib] -name = "signinum_j2k" +name = "j2k" path = "src/lib.rs" [dependencies] -signinum-core = { path = "../signinum-core", version = "=0.4.2" } -signinum-j2k-native = { path = "../signinum-j2k-native", version = "=0.4.2" } +j2k-core = { path = "../j2k-core", version = "=0.6.0" } +j2k-native = { path = "../j2k-native", version = "=0.6.0" } +j2k-types = { path = "../j2k-types", version = "=0.6.0" } thiserror = { workspace = true } [dev-dependencies] proptest = { workspace = true } criterion = { workspace = true } -signinum-j2k-native = { path = "../signinum-j2k-native", version = "=0.4.2" } -signinum-test-support = { path = "../signinum-test-support" } +j2k-native = { path = "../j2k-native", version = "=0.6.0" } +j2k-test-support = { path = "../j2k-test-support" } [[bench]] name = "public_api" diff --git a/crates/j2k/README.md b/crates/j2k/README.md new file mode 100644 index 00000000..4b022f42 --- /dev/null +++ b/crates/j2k/README.md @@ -0,0 +1,13 @@ +# j2k + +JPEG 2000 and HTJ2K public decoder/encoder crate for J2K. + +This crate exposes inspect, decode, encode, recode, device-surface, and +encode-stage adapter contracts backed by the native J2K engine and optional +device adapters. + +The encode-stage adapter module is a backend SPI for CUDA, Metal, and transcode +integration. It is not the primary end-user encode API. + +For JPEG 2000 / HTJ2K application code, including CPU and supported GPU-backed +paths, use this crate directly. diff --git a/crates/j2k/benches/public_api.rs b/crates/j2k/benches/public_api.rs new file mode 100644 index 00000000..99eb80ea --- /dev/null +++ b/crates/j2k/benches/public_api.rs @@ -0,0 +1,1019 @@ +// SPDX-License-Identifier: Apache-2.0 + +use criterion::{criterion_group, criterion_main, Criterion}; +use j2k::{ + decode_tiles_region_scaled_into, encode_j2k_lossless, recode_j2k_to_htj2k_lossless, + CpuDecodeParallelism, DecoderContext, Downscale, EncodeBackendPreference, ImageDecodeRows, + J2kBlockCodingMode, J2kCodec, J2kContext, J2kDecoder, J2kEncodeValidation, + J2kLosslessEncodeOptions, J2kLosslessSamples, J2kScratchPool, J2kToHtj2kOptions, PixelFormat, + Rect, RowSink, TileBatchDecode, TileBatchOptions, TileRegionScaledDecodeJob, +}; +use j2k_test_support::{patterned_gray8, patterned_rgb8, wrap_codestream_jp2}; + +const TILE_SIDE: u32 = 128; +const ROI_SIDE: u32 = 64; +const HT_TILE_SIDE: u32 = 512; +const CPU_MATRIX_SIDE: u32 = 512; +const BATCH_SIZE: usize = 16; + +fn bench_encode_options() -> J2kLosslessEncodeOptions { + let mut options = J2kLosslessEncodeOptions::default(); + options.backend = EncodeBackendPreference::CpuOnly; + options.validation = J2kEncodeValidation::External; + options +} + +fn ht_encode_options() -> J2kLosslessEncodeOptions { + let mut options = bench_encode_options(); + options.block_coding_mode = J2kBlockCodingMode::HighThroughput; + options +} + +fn recode_options() -> J2kToHtj2kOptions { + let mut options = J2kToHtj2kOptions::default(); + options.validation = J2kEncodeValidation::External; + options +} + +fn cpu_matrix_encode_options( + block_coding_mode: J2kBlockCodingMode, + validation: J2kEncodeValidation, +) -> J2kLosslessEncodeOptions { + let mut options = J2kLosslessEncodeOptions::default(); + options.backend = EncodeBackendPreference::CpuOnly; + options.validation = validation; + options.block_coding_mode = block_coding_mode; + options +} + +fn encode_gray8_codestream(width: u32, height: u32) -> Vec { + let pixels = patterned_gray8(width, height); + encode_gray8_codestream_from_pixels(width, height, &pixels, bench_encode_options()) +} + +fn encode_gray16_codestream(width: u32, height: u32) -> Vec { + let mut pixels = Vec::with_capacity(width as usize * height as usize * 2); + for y in 0..height { + for x in 0..width { + let sample = ((x * 257 + y * 911) & 0xffff) as u16; + pixels.extend_from_slice(&sample.to_le_bytes()); + } + } + let samples = J2kLosslessSamples::new(&pixels, width, height, 1, 16, false) + .expect("valid gray16 samples"); + encode_j2k_lossless(samples, &bench_encode_options()) + .expect("encode gray16 codestream") + .codestream +} + +fn encode_ht_gray8_codestream(width: u32, height: u32) -> Vec { + let pixels = patterned_gray8(width, height); + encode_gray8_codestream_from_pixels(width, height, &pixels, ht_encode_options()) +} + +fn encode_ht_rgb8_codestream(width: u32, height: u32) -> Vec { + let pixels = patterned_rgb8(width, height); + encode_rgb8_codestream_from_pixels(width, height, &pixels, ht_encode_options()) +} + +fn encode_gray8_codestream_from_pixels( + width: u32, + height: u32, + pixels: &[u8], + options: J2kLosslessEncodeOptions, +) -> Vec { + let samples = + J2kLosslessSamples::new(pixels, width, height, 1, 8, false).expect("valid gray8 samples"); + encode_j2k_lossless(samples, &options) + .expect("encode gray8 codestream") + .codestream +} + +fn encode_rgb8_codestream(width: u32, height: u32) -> Vec { + let pixels = patterned_rgb8(width, height); + encode_rgb8_codestream_from_pixels(width, height, &pixels, bench_encode_options()) +} + +fn encode_rgb8_codestream_with_levels(width: u32, height: u32, levels: u8) -> Vec { + let pixels = patterned_rgb8(width, height); + encode_rgb8_codestream_from_pixels( + width, + height, + &pixels, + bench_encode_options().with_max_decomposition_levels(Some(levels)), + ) +} + +fn encode_rgb8_codestream_from_pixels( + width: u32, + height: u32, + pixels: &[u8], + options: J2kLosslessEncodeOptions, +) -> Vec { + let samples = + J2kLosslessSamples::new(pixels, width, height, 3, 8, false).expect("valid rgb8 samples"); + encode_j2k_lossless(samples, &options) + .expect("encode rgb8 codestream") + .codestream +} + +fn bench_lossless_encode(c: &mut Criterion) { + let mut group = c.benchmark_group("j2k_public_lossless_encode"); + + let gray = patterned_gray8(TILE_SIDE, TILE_SIDE); + let options = bench_encode_options(); + group.bench_function("gray8_128x128", |b| { + b.iter(|| { + let samples = J2kLosslessSamples::new( + std::hint::black_box(gray.as_slice()), + TILE_SIDE, + TILE_SIDE, + 1, + 8, + false, + ) + .expect("valid gray8 samples"); + let encoded = encode_j2k_lossless(samples, &options).expect("encode gray8"); + std::hint::black_box(encoded.codestream.len()); + }); + }); + + let rgb = patterned_rgb8(TILE_SIDE, TILE_SIDE); + group.bench_function("rgb8_128x128", |b| { + b.iter(|| { + let samples = J2kLosslessSamples::new( + std::hint::black_box(rgb.as_slice()), + TILE_SIDE, + TILE_SIDE, + 3, + 8, + false, + ) + .expect("valid rgb8 samples"); + let encoded = encode_j2k_lossless(samples, &options).expect("encode rgb8"); + std::hint::black_box(encoded.codestream.len()); + }); + }); + + group.finish(); +} + +fn bench_inspect(c: &mut Criterion) { + let codestream = encode_rgb8_codestream(TILE_SIDE, TILE_SIDE); + + let mut group = c.benchmark_group("j2k_public_inspect"); + group.bench_function("rgb8_128x128", |b| { + b.iter(|| { + let info = + J2kDecoder::inspect(std::hint::black_box(codestream.as_slice())).expect("inspect"); + std::hint::black_box(info); + }); + }); + group.finish(); +} + +fn bench_decode(c: &mut Criterion) { + let codestream = encode_rgb8_codestream(TILE_SIDE, TILE_SIDE); + let ht_codestream = encode_ht_gray8_codestream(HT_TILE_SIDE, HT_TILE_SIDE); + let mut group = c.benchmark_group("j2k_public_decode"); + + let full_stride = TILE_SIDE as usize * 3; + let mut full = vec![0u8; full_stride * TILE_SIDE as usize]; + group.bench_function("rgb8_full_128x128", |b| { + b.iter(|| { + let mut decoder = + J2kDecoder::new(std::hint::black_box(codestream.as_slice())).expect("rgb8 decoder"); + decoder + .decode_into(&mut full, full_stride, PixelFormat::Rgb8) + .expect("decode full rgb8"); + std::hint::black_box(&full); + }); + }); + + let roi = Rect { + x: 32, + y: 32, + w: ROI_SIDE, + h: ROI_SIDE, + }; + let roi_stride = ROI_SIDE as usize * 3; + let mut roi_out = vec![0u8; roi_stride * ROI_SIDE as usize]; + let mut pool = J2kScratchPool::new(); + group.bench_function("rgb8_roi_64x64", |b| { + b.iter(|| { + let mut decoder = + J2kDecoder::new(std::hint::black_box(codestream.as_slice())).expect("rgb8 decoder"); + decoder + .decode_region_into(&mut pool, &mut roi_out, roi_stride, PixelFormat::Rgb8, roi) + .expect("decode rgb8 roi"); + std::hint::black_box(&roi_out); + }); + }); + + let ht_stride = HT_TILE_SIDE as usize; + let mut ht_out = vec![0u8; ht_stride * HT_TILE_SIDE as usize]; + group.bench_function("htj2k_gray8_full_512x512", |b| { + b.iter(|| { + let mut decoder = J2kDecoder::new(std::hint::black_box(ht_codestream.as_slice())) + .expect("htj2k decoder"); + decoder + .decode_into(&mut ht_out, ht_stride, PixelFormat::Gray8) + .expect("decode full htj2k gray8"); + std::hint::black_box(&ht_out); + }); + }); + + group.finish(); +} + +fn bench_recode(c: &mut Criterion) { + let classic = encode_rgb8_codestream(CPU_MATRIX_SIDE, CPU_MATRIX_SIDE); + let htj2k = encode_ht_rgb8_codestream(CPU_MATRIX_SIDE, CPU_MATRIX_SIDE); + let options = recode_options(); + let mut group = c.benchmark_group("j2k_public_recode"); + + group.bench_function("classic_rgb8_512_to_htj2k_53_coefficients", |b| { + b.iter(|| { + let recoded = + recode_j2k_to_htj2k_lossless(std::hint::black_box(classic.as_slice()), options) + .expect("coefficient-domain recode"); + std::hint::black_box(recoded.bytes.len()); + }); + }); + + group.bench_function("raw_htj2k_rgb8_512_passthrough", |b| { + b.iter(|| { + let recoded = + recode_j2k_to_htj2k_lossless(std::hint::black_box(htj2k.as_slice()), options) + .expect("HTJ2K passthrough"); + std::hint::black_box(recoded.bytes.len()); + }); + }); + + group.finish(); +} + +fn bench_region_scaled(c: &mut Criterion) { + let codestream = encode_rgb8_codestream_with_levels(TILE_SIDE, TILE_SIDE, 2); + let roi = Rect { + x: 32, + y: 32, + w: ROI_SIDE, + h: ROI_SIDE, + }; + let out_side = ROI_SIDE.div_ceil(Downscale::Quarter.denominator()); + let stride = out_side as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut out = vec![0u8; stride * out_side as usize]; + let mut pool = J2kScratchPool::new(); + + let mut group = c.benchmark_group("j2k_public_decode_region_scaled"); + group.bench_function("rgb8_region_scaled_64x64_q4", |b| { + b.iter(|| { + let mut decoder = + J2kDecoder::new(std::hint::black_box(codestream.as_slice())).expect("rgb8 decoder"); + decoder + .decode_region_scaled_into( + &mut pool, + &mut out, + stride, + PixelFormat::Rgb8, + roi, + Downscale::Quarter, + ) + .expect("decode rgb8 region scaled"); + std::hint::black_box(&out); + }); + }); + group.finish(); +} + +fn bench_scaled_reuse(c: &mut Criterion) { + let codestream = encode_rgb8_codestream_with_levels(CPU_MATRIX_SIDE, CPU_MATRIX_SIDE, 2); + let scale = Downscale::Quarter; + let out_side = CPU_MATRIX_SIDE.div_ceil(scale.denominator()); + let stride = out_side as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut setup_out = vec![0u8; stride * out_side as usize]; + let mut reused_out = vec![0u8; stride * out_side as usize]; + let mut reused_decoder = J2kDecoder::new(codestream.as_slice()).expect("reused scaled decoder"); + let mut reused_pool = J2kScratchPool::new(); + + let mut group = c.benchmark_group("j2k_public_decode_scaled_reuse"); + group.bench_function("rgb8_512_q4_setup_inclusive", |b| { + b.iter(|| { + let mut decoder = + J2kDecoder::new(std::hint::black_box(codestream.as_slice())).expect("rgb8 decoder"); + let mut pool = J2kScratchPool::new(); + decoder + .decode_scaled_into(&mut pool, &mut setup_out, stride, PixelFormat::Rgb8, scale) + .expect("setup-inclusive scaled decode"); + std::hint::black_box(&setup_out); + }); + }); + group.bench_function("rgb8_512_q4_decoder_setup_excluded", |b| { + b.iter(|| { + reused_decoder + .decode_scaled_into( + &mut reused_pool, + &mut reused_out, + stride, + PixelFormat::Rgb8, + scale, + ) + .expect("reused scaled decode"); + std::hint::black_box(&reused_out); + }); + }); + group.finish(); +} + +fn bench_region_scaled_reuse(c: &mut Criterion) { + let codestream = encode_rgb8_codestream_with_levels(CPU_MATRIX_SIDE, CPU_MATRIX_SIDE, 2); + let roi = Rect { + x: 128, + y: 128, + w: 256, + h: 256, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut setup_out = vec![0u8; stride * scaled.h as usize]; + let mut reused_out = vec![0u8; stride * scaled.h as usize]; + let mut reused_decoder = + J2kDecoder::new(codestream.as_slice()).expect("reused region-scaled decoder"); + let mut reused_pool = J2kScratchPool::new(); + + let mut group = c.benchmark_group("j2k_public_decode_region_scaled_reuse"); + group.bench_function("rgb8_512_roi256_q4_setup_inclusive", |b| { + b.iter(|| { + let mut decoder = + J2kDecoder::new(std::hint::black_box(codestream.as_slice())).expect("rgb8 decoder"); + let mut pool = J2kScratchPool::new(); + decoder + .decode_region_scaled_into( + &mut pool, + &mut setup_out, + stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("setup-inclusive region scaled decode"); + std::hint::black_box(&setup_out); + }); + }); + group.bench_function("rgb8_512_roi256_q4_decoder_setup_excluded", |b| { + b.iter(|| { + reused_decoder + .decode_region_scaled_into( + &mut reused_pool, + &mut reused_out, + stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("reused region scaled decode"); + std::hint::black_box(&reused_out); + }); + }); + group.finish(); +} + +fn bench_mixed_scale_reuse(c: &mut Criterion) { + let codestream = encode_rgb8_codestream_with_levels(CPU_MATRIX_SIDE, CPU_MATRIX_SIDE, 2); + let mut decoder = J2kDecoder::new(codestream.as_slice()).expect("mixed-scale decoder"); + let mut pool = J2kScratchPool::new(); + let half_side = CPU_MATRIX_SIDE.div_ceil(Downscale::Half.denominator()); + let half_stride = half_side as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut half_out = vec![0u8; half_stride * half_side as usize]; + let quarter_side = CPU_MATRIX_SIDE.div_ceil(Downscale::Quarter.denominator()); + let quarter_stride = quarter_side as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut quarter_out = vec![0u8; quarter_stride * quarter_side as usize]; + + let mut group = c.benchmark_group("j2k_public_decode_mixed_scale_reuse"); + group.bench_function("rgb8_512_q4_q2_q4_single_decoder", |b| { + b.iter(|| { + decoder + .decode_scaled_into( + &mut pool, + &mut quarter_out, + quarter_stride, + PixelFormat::Rgb8, + Downscale::Quarter, + ) + .expect("quarter decode"); + decoder + .decode_scaled_into( + &mut pool, + &mut half_out, + half_stride, + PixelFormat::Rgb8, + Downscale::Half, + ) + .expect("half decode"); + decoder + .decode_scaled_into( + &mut pool, + &mut quarter_out, + quarter_stride, + PixelFormat::Rgb8, + Downscale::Quarter, + ) + .expect("quarter decode after half"); + std::hint::black_box((&quarter_out, &half_out)); + }); + }); + group.finish(); +} + +fn bench_rows(c: &mut Criterion) { + let codestream = encode_gray8_codestream(TILE_SIDE, TILE_SIDE); + let gray16_codestream = encode_gray16_codestream(TILE_SIDE, TILE_SIDE); + let mut group = c.benchmark_group("j2k_public_decode_rows"); + group.bench_function("gray8_rows_128x128", |b| { + b.iter(|| { + let mut decoder = J2kDecoder::new(std::hint::black_box(codestream.as_slice())) + .expect("gray8 decoder"); + let mut sink = VecRowSink::new(TILE_SIDE, TILE_SIDE); + decoder.decode_rows(&mut sink).expect("decode gray8 rows"); + std::hint::black_box(sink.rows); + }); + }); + group.bench_function("gray8_rows_128x128_reused_decoder", |b| { + let mut decoder = J2kDecoder::new(codestream.as_slice()).expect("gray8 reused decoder"); + b.iter(|| { + let mut sink = VecRowSink::new(TILE_SIDE, TILE_SIDE); + decoder.decode_rows(&mut sink).expect("decode gray8 rows"); + std::hint::black_box(sink.rows); + }); + }); + group.bench_function("gray16_rows_128x128_reused_decoder", |b| { + let mut decoder = + J2kDecoder::new(gray16_codestream.as_slice()).expect("gray16 reused decoder"); + b.iter(|| { + let mut sink = VecRowSinkU16::new(TILE_SIDE, TILE_SIDE); + as ImageDecodeRows<'_, u16>>::decode_rows(&mut decoder, &mut sink) + .expect("decode gray16 rows"); + std::hint::black_box(sink.rows); + }); + }); + group.finish(); +} + +fn bench_tile_batch(c: &mut Criterion) { + let repeated = encode_gray8_codestream(TILE_SIDE, TILE_SIDE); + let ht_repeated = encode_ht_gray8_codestream(TILE_SIDE, TILE_SIDE); + let mut distinct = Vec::with_capacity(BATCH_SIZE); + let mut ht_distinct = Vec::with_capacity(BATCH_SIZE); + for idx in 0..BATCH_SIZE { + let mut pixels = patterned_gray8(TILE_SIDE, TILE_SIDE); + pixels[0] = pixels[0].wrapping_add(idx as u8); + distinct.push(encode_gray8_codestream_from_pixels( + TILE_SIDE, + TILE_SIDE, + &pixels, + bench_encode_options(), + )); + ht_distinct.push(encode_gray8_codestream_from_pixels( + TILE_SIDE, + TILE_SIDE, + &pixels, + ht_encode_options(), + )); + } + + let stride = TILE_SIDE as usize; + let mut out = vec![0u8; stride * TILE_SIDE as usize]; + let mut group = c.benchmark_group("j2k_public_tile_batch"); + + group.bench_function("gray8_repeated_batch_16", |b| { + b.iter(|| { + let mut ctx = DecoderContext::::default(); + let mut pool = J2kScratchPool::new(); + for _ in 0..BATCH_SIZE { + ::decode_tile( + &mut ctx, + &mut pool, + std::hint::black_box(repeated.as_slice()), + &mut out, + stride, + PixelFormat::Gray8, + ) + .expect("decode repeated gray8 tile"); + } + std::hint::black_box(&out); + }); + }); + + group.bench_function("htj2k_gray8_repeated_batch_16", |b| { + b.iter(|| { + let mut ctx = DecoderContext::::default(); + let mut pool = J2kScratchPool::new(); + for _ in 0..BATCH_SIZE { + ::decode_tile( + &mut ctx, + &mut pool, + std::hint::black_box(ht_repeated.as_slice()), + &mut out, + stride, + PixelFormat::Gray8, + ) + .expect("decode repeated htj2k gray8 tile"); + } + std::hint::black_box(&out); + }); + }); + + group.bench_function("gray8_distinct_batch_16", |b| { + b.iter(|| { + let mut ctx = DecoderContext::::default(); + let mut pool = J2kScratchPool::new(); + for codestream in &distinct { + ::decode_tile( + &mut ctx, + &mut pool, + std::hint::black_box(codestream.as_slice()), + &mut out, + stride, + PixelFormat::Gray8, + ) + .expect("decode distinct gray8 tile"); + } + std::hint::black_box(&out); + }); + }); + + group.bench_function("htj2k_gray8_distinct_batch_16", |b| { + b.iter(|| { + let mut ctx = DecoderContext::::default(); + let mut pool = J2kScratchPool::new(); + for codestream in &ht_distinct { + ::decode_tile( + &mut ctx, + &mut pool, + std::hint::black_box(codestream.as_slice()), + &mut out, + stride, + PixelFormat::Gray8, + ) + .expect("decode distinct htj2k gray8 tile"); + } + std::hint::black_box(&out); + }); + }); + + group.finish(); +} + +fn bench_tile_batch_region_scaled_rgb(c: &mut Criterion) { + let repeated_classic = encode_rgb8_codestream(CPU_MATRIX_SIDE, CPU_MATRIX_SIDE); + let repeated_htj2k = encode_ht_rgb8_codestream(CPU_MATRIX_SIDE, CPU_MATRIX_SIDE); + let repeated_htj2k_jp2 = + wrap_codestream_jp2(&repeated_htj2k, CPU_MATRIX_SIDE, CPU_MATRIX_SIDE, 3, 8, 16); + let repeated_htj2k_256 = encode_ht_rgb8_codestream(256, 256); + let repeated_htj2k_256_jp2 = wrap_codestream_jp2(&repeated_htj2k_256, 256, 256, 3, 8, 16); + let mut distinct_classic = Vec::with_capacity(BATCH_SIZE); + let mut distinct_htj2k = Vec::with_capacity(BATCH_SIZE); + for idx in 0..BATCH_SIZE { + let mut pixels = patterned_rgb8(CPU_MATRIX_SIDE, CPU_MATRIX_SIDE); + pixels[0] = pixels[0].wrapping_add(idx as u8); + distinct_classic.push(encode_rgb8_codestream_from_pixels( + CPU_MATRIX_SIDE, + CPU_MATRIX_SIDE, + &pixels, + bench_encode_options(), + )); + distinct_htj2k.push(encode_rgb8_codestream_from_pixels( + CPU_MATRIX_SIDE, + CPU_MATRIX_SIDE, + &pixels, + ht_encode_options(), + )); + } + + let roi = Rect { + x: 128, + y: 128, + w: 256, + h: 256, + }; + let scale = Downscale::Quarter; + let scaled = roi.scaled_covering(scale); + let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let output_len = stride * scaled.h as usize; + let rgba_stride = scaled.w as usize * PixelFormat::Rgba8.bytes_per_pixel(); + let rgba_output_len = rgba_stride * scaled.h as usize; + + let roi_256 = Rect { + x: 64, + y: 64, + w: 128, + h: 128, + }; + let scaled_256 = roi_256.scaled_covering(scale); + let stride_256 = scaled_256.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let output_len_256 = stride_256 * scaled_256.h as usize; + + let mut group = c.benchmark_group("j2k_public_tile_batch_region_scaled_rgb_q4"); + group.bench_function("classic_repeated_512_roi256_batch16", |b| { + b.iter(|| { + let mut outputs = vec![vec![0_u8; output_len]; BATCH_SIZE]; + let mut jobs = outputs + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: std::hint::black_box(repeated_classic.as_slice()), + out, + stride, + roi, + scale, + }) + .collect::>(); + let outcomes = decode_tiles_region_scaled_into( + &mut jobs, + PixelFormat::Rgb8, + TileBatchOptions::default(), + ) + .expect("decode repeated classic RGB ROI+scale batch"); + std::hint::black_box((outputs, outcomes)); + }); + }); + group.bench_function("classic_distinct_512_roi256_batch16", |b| { + b.iter(|| { + let mut outputs = vec![vec![0_u8; output_len]; BATCH_SIZE]; + let mut jobs = outputs + .iter_mut() + .zip(distinct_classic.iter()) + .map(|(out, input)| TileRegionScaledDecodeJob { + input: std::hint::black_box(input.as_slice()), + out, + stride, + roi, + scale, + }) + .collect::>(); + let outcomes = decode_tiles_region_scaled_into( + &mut jobs, + PixelFormat::Rgb8, + TileBatchOptions::default(), + ) + .expect("decode distinct classic RGB ROI+scale batch"); + std::hint::black_box((outputs, outcomes)); + }); + }); + group.bench_function("htj2k_repeated_512_roi256_batch16", |b| { + b.iter(|| { + let mut outputs = vec![vec![0_u8; output_len]; BATCH_SIZE]; + let mut jobs = outputs + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: std::hint::black_box(repeated_htj2k.as_slice()), + out, + stride, + roi, + scale, + }) + .collect::>(); + let outcomes = decode_tiles_region_scaled_into( + &mut jobs, + PixelFormat::Rgb8, + TileBatchOptions::default(), + ) + .expect("decode repeated HTJ2K RGB ROI+scale batch"); + std::hint::black_box((outputs, outcomes)); + }); + }); + group.bench_function("htj2k_jp2_rgb8_repeated_512_roi256_batch16", |b| { + b.iter(|| { + let mut outputs = vec![vec![0_u8; output_len]; BATCH_SIZE]; + let mut jobs = outputs + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: std::hint::black_box(repeated_htj2k_jp2.as_slice()), + out, + stride, + roi, + scale, + }) + .collect::>(); + let outcomes = decode_tiles_region_scaled_into( + &mut jobs, + PixelFormat::Rgb8, + TileBatchOptions::default(), + ) + .expect("decode repeated HTJ2K JP2 RGB ROI+scale batch"); + std::hint::black_box((outputs, outcomes)); + }); + }); + group.bench_function("htj2k_jp2_rgba8_repeated_512_roi256_batch16", |b| { + b.iter(|| { + let mut outputs = vec![vec![0_u8; rgba_output_len]; BATCH_SIZE]; + let mut jobs = outputs + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: std::hint::black_box(repeated_htj2k_jp2.as_slice()), + out, + stride: rgba_stride, + roi, + scale, + }) + .collect::>(); + let outcomes = decode_tiles_region_scaled_into( + &mut jobs, + PixelFormat::Rgba8, + TileBatchOptions::default(), + ) + .expect("decode repeated HTJ2K JP2 RGBA ROI+scale batch"); + std::hint::black_box((outputs, outcomes)); + }); + }); + group.bench_function("htj2k_jp2_rgb8_repeated_256_roi128_batch16", |b| { + b.iter(|| { + let mut outputs = vec![vec![0_u8; output_len_256]; BATCH_SIZE]; + let mut jobs = outputs + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: std::hint::black_box(repeated_htj2k_256_jp2.as_slice()), + out, + stride: stride_256, + roi: roi_256, + scale, + }) + .collect::>(); + let outcomes = decode_tiles_region_scaled_into( + &mut jobs, + PixelFormat::Rgb8, + TileBatchOptions::default(), + ) + .expect("decode repeated 256 HTJ2K JP2 RGB ROI+scale batch"); + std::hint::black_box((outputs, outcomes)); + }); + }); + group.bench_function("htj2k_distinct_512_roi256_batch16", |b| { + b.iter(|| { + let mut outputs = vec![vec![0_u8; output_len]; BATCH_SIZE]; + let mut jobs = outputs + .iter_mut() + .zip(distinct_htj2k.iter()) + .map(|(out, input)| TileRegionScaledDecodeJob { + input: std::hint::black_box(input.as_slice()), + out, + stride, + roi, + scale, + }) + .collect::>(); + let outcomes = decode_tiles_region_scaled_into( + &mut jobs, + PixelFormat::Rgb8, + TileBatchOptions::default(), + ) + .expect("decode distinct HTJ2K RGB ROI+scale batch"); + std::hint::black_box((outputs, outcomes)); + }); + }); + group.finish(); +} + +fn bench_decode_gray_setup(c: &mut Criterion) { + let codestream = encode_gray8_codestream(TILE_SIDE, TILE_SIDE); + let stride = TILE_SIDE as usize; + let mut out = vec![0u8; stride * TILE_SIDE as usize]; + + let mut group = c.benchmark_group("j2k_public_decode_gray"); + group.bench_function("gray8_full_128x128", |b| { + b.iter(|| { + let mut decoder = J2kDecoder::new(std::hint::black_box(codestream.as_slice())) + .expect("gray8 decoder"); + decoder + .decode_into(&mut out, stride, PixelFormat::Gray8) + .expect("decode full gray8"); + std::hint::black_box(&out); + }); + }); + group.finish(); +} + +fn bench_cpu_encode_matrix(c: &mut Criterion) { + let pixels = patterned_rgb8(CPU_MATRIX_SIDE, CPU_MATRIX_SIDE); + let classic_external = + cpu_matrix_encode_options(J2kBlockCodingMode::Classic, J2kEncodeValidation::External); + let htj2k_external = cpu_matrix_encode_options( + J2kBlockCodingMode::HighThroughput, + J2kEncodeValidation::External, + ); + let classic_roundtrip = cpu_matrix_encode_options( + J2kBlockCodingMode::Classic, + J2kEncodeValidation::CpuRoundTrip, + ); + + let mut group = c.benchmark_group("j2k_public_cpu_encode_matrix"); + group.bench_function("rgb8_512_classic_external", |b| { + b.iter(|| { + let samples = J2kLosslessSamples::new( + std::hint::black_box(pixels.as_slice()), + CPU_MATRIX_SIDE, + CPU_MATRIX_SIDE, + 3, + 8, + false, + ) + .expect("valid rgb8 samples"); + let encoded = + encode_j2k_lossless(samples, &classic_external).expect("classic CPU encode"); + std::hint::black_box(encoded.codestream.len()); + }); + }); + + group.bench_function("rgb8_512_htj2k_external", |b| { + b.iter(|| { + let samples = J2kLosslessSamples::new( + std::hint::black_box(pixels.as_slice()), + CPU_MATRIX_SIDE, + CPU_MATRIX_SIDE, + 3, + 8, + false, + ) + .expect("valid rgb8 samples"); + let encoded = encode_j2k_lossless(samples, &htj2k_external).expect("HTJ2K CPU encode"); + std::hint::black_box(encoded.codestream.len()); + }); + }); + + group.bench_function("rgb8_512_classic_roundtrip", |b| { + b.iter(|| { + let samples = J2kLosslessSamples::new( + std::hint::black_box(pixels.as_slice()), + CPU_MATRIX_SIDE, + CPU_MATRIX_SIDE, + 3, + 8, + false, + ) + .expect("valid rgb8 samples"); + let encoded = + encode_j2k_lossless(samples, &classic_roundtrip).expect("classic CPU encode"); + std::hint::black_box(encoded.codestream.len()); + }); + }); + group.finish(); +} + +fn bench_cpu_decode_matrix(c: &mut Criterion) { + let pixels = patterned_gray8(CPU_MATRIX_SIDE, CPU_MATRIX_SIDE); + let classic_codestream = encode_gray8_codestream_from_pixels( + CPU_MATRIX_SIDE, + CPU_MATRIX_SIDE, + &pixels, + cpu_matrix_encode_options(J2kBlockCodingMode::Classic, J2kEncodeValidation::External), + ); + let htj2k_codestream = encode_gray8_codestream_from_pixels( + CPU_MATRIX_SIDE, + CPU_MATRIX_SIDE, + &pixels, + cpu_matrix_encode_options( + J2kBlockCodingMode::HighThroughput, + J2kEncodeValidation::External, + ), + ); + let rgb_classic_codestream = encode_rgb8_codestream(CPU_MATRIX_SIDE, CPU_MATRIX_SIDE); + let rgb_htj2k_codestream = encode_ht_rgb8_codestream(CPU_MATRIX_SIDE, CPU_MATRIX_SIDE); + + let stride = CPU_MATRIX_SIDE as usize; + let mut classic_out = vec![0u8; stride * CPU_MATRIX_SIDE as usize]; + let mut htj2k_out = vec![0u8; stride * CPU_MATRIX_SIDE as usize]; + let rgb_stride = CPU_MATRIX_SIDE as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut rgb_classic_out = vec![0u8; rgb_stride * CPU_MATRIX_SIDE as usize]; + let mut rgb_htj2k_out = vec![0u8; rgb_stride * CPU_MATRIX_SIDE as usize]; + + let mut group = c.benchmark_group("j2k_public_cpu_decode_matrix"); + group.bench_function("gray8_512_classic_decode", |b| { + b.iter(|| { + let mut decoder = J2kDecoder::new(std::hint::black_box(classic_codestream.as_slice())) + .expect("J2K decoder"); + decoder + .decode_into(&mut classic_out, stride, PixelFormat::Gray8) + .expect("decode classic gray8"); + std::hint::black_box(&classic_out); + }); + }); + + group.bench_function("gray8_512_htj2k_decode", |b| { + b.iter(|| { + let mut decoder = J2kDecoder::new(std::hint::black_box(htj2k_codestream.as_slice())) + .expect("HTJ2K decoder"); + decoder + .decode_into(&mut htj2k_out, stride, PixelFormat::Gray8) + .expect("decode htj2k gray8"); + std::hint::black_box(&htj2k_out); + }); + }); + + group.bench_function("rgb8_512_classic_decode", |b| { + b.iter(|| { + let mut decoder = + J2kDecoder::new(std::hint::black_box(rgb_classic_codestream.as_slice())) + .expect("J2K decoder"); + decoder + .decode_into(&mut rgb_classic_out, rgb_stride, PixelFormat::Rgb8) + .expect("decode classic rgb8"); + std::hint::black_box(&rgb_classic_out); + }); + }); + + group.bench_function("rgb8_512_classic_decode_serial", |b| { + b.iter(|| { + let mut decoder = + J2kDecoder::new(std::hint::black_box(rgb_classic_codestream.as_slice())) + .expect("J2K decoder"); + decoder.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + decoder + .decode_into(&mut rgb_classic_out, rgb_stride, PixelFormat::Rgb8) + .expect("decode serial classic rgb8"); + std::hint::black_box(&rgb_classic_out); + }); + }); + + group.bench_function("rgb8_512_htj2k_decode", |b| { + b.iter(|| { + let mut decoder = + J2kDecoder::new(std::hint::black_box(rgb_htj2k_codestream.as_slice())) + .expect("HTJ2K decoder"); + decoder + .decode_into(&mut rgb_htj2k_out, rgb_stride, PixelFormat::Rgb8) + .expect("decode htj2k rgb8"); + std::hint::black_box(&rgb_htj2k_out); + }); + }); + group.finish(); +} + +criterion_group!( + benches, + bench_lossless_encode, + bench_inspect, + bench_decode, + bench_recode, + bench_region_scaled, + bench_scaled_reuse, + bench_region_scaled_reuse, + bench_mixed_scale_reuse, + bench_rows, + bench_tile_batch, + bench_tile_batch_region_scaled_rgb, + bench_decode_gray_setup, + bench_cpu_encode_matrix, + bench_cpu_decode_matrix +); +criterion_main!(benches); + +struct VecRowSink { + rows: Vec, + width: usize, +} + +impl VecRowSink { + fn new(width: u32, height: u32) -> Self { + Self { + rows: vec![0; width as usize * height as usize], + width: width as usize, + } + } +} + +impl RowSink for VecRowSink { + type Error = std::convert::Infallible; + + fn write_row(&mut self, y: u32, row: &[u8]) -> Result<(), Self::Error> { + let start = y as usize * self.width; + let end = start + row.len(); + self.rows[start..end].copy_from_slice(row); + Ok(()) + } +} + +struct VecRowSinkU16 { + rows: Vec, + width: usize, +} + +impl VecRowSinkU16 { + fn new(width: u32, height: u32) -> Self { + Self { + rows: vec![0; width as usize * height as usize], + width: width as usize, + } + } +} + +impl RowSink for VecRowSinkU16 { + type Error = std::convert::Infallible; + + fn write_row(&mut self, y: u32, row: &[u16]) -> Result<(), Self::Error> { + let start = y as usize * self.width; + let end = start + row.len(); + self.rows[start..end].copy_from_slice(row); + Ok(()) + } +} diff --git a/crates/signinum-j2k/examples/decode_generated.rs b/crates/j2k/examples/decode_generated.rs similarity index 87% rename from crates/signinum-j2k/examples/decode_generated.rs rename to crates/j2k/examples/decode_generated.rs index b7398258..327ff011 100644 --- a/crates/signinum-j2k/examples/decode_generated.rs +++ b/crates/j2k/examples/decode_generated.rs @@ -4,10 +4,10 @@ //! WSI-shaped API. //! //! Run with: -//! `cargo run -p signinum-j2k --example decode_generated` +//! `cargo run -p j2k --example decode_generated` -use signinum_j2k::{Downscale, J2kDecoder, J2kScratchPool, PixelFormat, Rect}; -use signinum_j2k_native::{encode_htj2k, EncodeOptions}; +use j2k::{Downscale, J2kDecoder, J2kScratchPool, PixelFormat, Rect}; +use j2k_native::{encode_htj2k, EncodeOptions}; fn main() -> Result<(), Box> { let width = 16_u32; diff --git a/crates/j2k/fuzz/.gitignore b/crates/j2k/fuzz/.gitignore new file mode 100644 index 00000000..d476c62b --- /dev/null +++ b/crates/j2k/fuzz/.gitignore @@ -0,0 +1,11 @@ +target +artifacts +corpus/* +!corpus/parse_fuzz/ +!corpus/decode_fuzz/ +corpus/parse_fuzz/* +corpus/decode_fuzz/* +!corpus/parse_fuzz/high_csiz_16385.j2c +!corpus/parse_fuzz/spec_max_csiz_16384.j2c +!corpus/decode_fuzz/max_tile_grid_small_payload.j2c +!corpus/decode_fuzz/tile_component_structural_bomb.j2c diff --git a/crates/signinum-jpeg/fuzz/Cargo.lock b/crates/j2k/fuzz/Cargo.lock similarity index 56% rename from crates/signinum-jpeg/fuzz/Cargo.lock rename to crates/j2k/fuzz/Cargo.lock index bb951b7c..1b49f7a4 100644 --- a/crates/signinum-jpeg/fuzz/Cargo.lock +++ b/crates/j2k/fuzz/Cargo.lock @@ -10,9 +10,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "cc" -version = "1.2.60" +version = "1.2.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", "jobserver", @@ -26,6 +26,43 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "fearless_simd" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -44,6 +81,50 @@ dependencies = [ "wasip2", ] +[[package]] +name = "j2k" +version = "0.6.0" +dependencies = [ + "j2k-core", + "j2k-native", + "j2k-types", + "thiserror", +] + +[[package]] +name = "j2k-core" +version = "0.6.0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "j2k-fuzz" +version = "0.1.0" +dependencies = [ + "j2k", + "libfuzzer-sys", +] + +[[package]] +name = "j2k-native" +version = "0.6.0" +dependencies = [ + "fearless_simd", + "j2k-profile", + "j2k-types", + "libm", + "rayon", +] + +[[package]] +name = "j2k-profile" +version = "0.6.0" + +[[package]] +name = "j2k-types" +version = "0.6.0" + [[package]] name = "jobserver" version = "0.1.34" @@ -56,25 +137,25 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libfuzzer-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f12a681b7dd8ce12bff52488013ba614b869148d54dd79836ab85aafdd53f08d" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" dependencies = [ "arbitrary", "cc", ] [[package]] -name = "memchr" -version = "2.8.0" +name = "libm" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "proc-macro2" @@ -101,40 +182,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] -name = "shlex" -version = "1.3.0" +name = "rayon" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signinum-core" -version = "1.0.0" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ - "thiserror", + "either", + "rayon-core", ] [[package]] -name = "signinum-jpeg" -version = "1.0.0" +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" dependencies = [ - "memchr", - "signinum-core", - "thiserror", + "crossbeam-deque", + "crossbeam-utils", ] [[package]] -name = "signinum-jpeg-fuzz" -version = "0.1.0" -dependencies = [ - "libfuzzer-sys", - "signinum-jpeg", -] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -169,15 +246,15 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen", ] [[package]] name = "wit-bindgen" -version = "0.51.0" +version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" diff --git a/crates/signinum-j2k/fuzz/Cargo.toml b/crates/j2k/fuzz/Cargo.toml similarity index 87% rename from crates/signinum-j2k/fuzz/Cargo.toml rename to crates/j2k/fuzz/Cargo.toml index 6ce77087..7fb83938 100644 --- a/crates/signinum-j2k/fuzz/Cargo.toml +++ b/crates/j2k/fuzz/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "signinum-j2k-fuzz" +name = "j2k-fuzz" version = "0.1.0" edition = "2021" publish = false @@ -9,7 +9,7 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" -signinum-j2k = { path = ".." } +j2k = { path = ".." } [[bin]] name = "parse_fuzz" diff --git a/crates/j2k/fuzz/corpus/decode_fuzz/max_tile_grid_small_payload.j2c b/crates/j2k/fuzz/corpus/decode_fuzz/max_tile_grid_small_payload.j2c new file mode 100644 index 00000000..9881ad4f Binary files /dev/null and b/crates/j2k/fuzz/corpus/decode_fuzz/max_tile_grid_small_payload.j2c differ diff --git a/crates/j2k/fuzz/corpus/decode_fuzz/tile_component_structural_bomb.j2c b/crates/j2k/fuzz/corpus/decode_fuzz/tile_component_structural_bomb.j2c new file mode 100644 index 00000000..f4f624e9 Binary files /dev/null and b/crates/j2k/fuzz/corpus/decode_fuzz/tile_component_structural_bomb.j2c differ diff --git a/crates/j2k/fuzz/corpus/parse_fuzz/high_csiz_16385.j2c b/crates/j2k/fuzz/corpus/parse_fuzz/high_csiz_16385.j2c new file mode 100644 index 00000000..09beaf4d Binary files /dev/null and b/crates/j2k/fuzz/corpus/parse_fuzz/high_csiz_16385.j2c differ diff --git a/crates/j2k/fuzz/corpus/parse_fuzz/spec_max_csiz_16384.j2c b/crates/j2k/fuzz/corpus/parse_fuzz/spec_max_csiz_16384.j2c new file mode 100644 index 00000000..c5b02189 Binary files /dev/null and b/crates/j2k/fuzz/corpus/parse_fuzz/spec_max_csiz_16384.j2c differ diff --git a/crates/signinum-j2k/fuzz/fuzz_targets/decode_fuzz.rs b/crates/j2k/fuzz/fuzz_targets/decode_fuzz.rs similarity index 93% rename from crates/signinum-j2k/fuzz/fuzz_targets/decode_fuzz.rs rename to crates/j2k/fuzz/fuzz_targets/decode_fuzz.rs index 770df021..b45d1fec 100644 --- a/crates/signinum-j2k/fuzz/fuzz_targets/decode_fuzz.rs +++ b/crates/j2k/fuzz/fuzz_targets/decode_fuzz.rs @@ -1,7 +1,7 @@ #![no_main] +use j2k::{J2kDecoder, PixelFormat}; use libfuzzer_sys::fuzz_target; -use signinum_j2k::{J2kDecoder, PixelFormat}; const MAX_PIXELS: u32 = 1 << 20; diff --git a/crates/signinum-j2k/fuzz/fuzz_targets/parse_fuzz.rs b/crates/j2k/fuzz/fuzz_targets/parse_fuzz.rs similarity index 79% rename from crates/signinum-j2k/fuzz/fuzz_targets/parse_fuzz.rs rename to crates/j2k/fuzz/fuzz_targets/parse_fuzz.rs index ce0e7ccb..e61cb7b3 100644 --- a/crates/signinum-j2k/fuzz/fuzz_targets/parse_fuzz.rs +++ b/crates/j2k/fuzz/fuzz_targets/parse_fuzz.rs @@ -1,7 +1,7 @@ #![no_main] +use j2k::J2kDecoder; use libfuzzer_sys::fuzz_target; -use signinum_j2k::J2kDecoder; fuzz_target!(|data: &[u8]| { let _ = J2kDecoder::inspect(data); diff --git a/crates/j2k/src/adapter/adaptive_route.rs b/crates/j2k/src/adapter/adaptive_route.rs new file mode 100644 index 00000000..e716204d --- /dev/null +++ b/crates/j2k/src/adapter/adaptive_route.rs @@ -0,0 +1,1096 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Adaptive JPEG 2000 / HTJ2K CPU-device route planning. + +use alloc::vec::Vec; + +use crate::{J2kBlockCodingMode, J2kError, J2kLosslessEncodeOptions, J2kLosslessSamples}; +use j2k_core::{BackendCapabilities, BackendKind, BackendRequest, Unsupported}; + +/// Caller intent for adaptive JPEG 2000-family routing. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kAdaptiveBackendRequest { + /// Use the best benchmark-approved CPU/device split available on the host. + Accelerated, + /// Force all stages onto the portable CPU route. + CpuOnly, + /// Require proof of the requested device path and fail if unavailable. + StrictDevice(BackendKind), +} + +impl J2kAdaptiveBackendRequest { + /// Convert a shared backend request into adaptive JPEG 2000 route intent. + #[must_use] + pub const fn from_backend_request(request: BackendRequest) -> Self { + match request { + BackendRequest::Auto => Self::Accelerated, + BackendRequest::Cpu => Self::CpuOnly, + BackendRequest::Metal => Self::StrictDevice(BackendKind::Metal), + BackendRequest::Cuda => Self::StrictDevice(BackendKind::Cuda), + } + } +} + +#[derive(Debug, Clone, Copy)] +struct LosslessEncodeReferencePolicy { + components: u8, + min_pixels: u64, + dwt_cpu_ns: u64, + dwt_accelerated_ns: u64, + ht_cpu_ns: u64, + ht_accelerated_ns: u64, + end_to_end_cpu_ns: u64, + end_to_end_accelerated_ns: u64, + criterion_noise_percent: f64, +} + +/// Reference policies captured during the v0.5 release-maturity audit on the +/// CUDA host-output HTJ2K encode path. They are route-gating policy evidence, +/// not live measurements from the current machine. +const CUDA_HTJ2K_HOST_ENCODE_REFERENCE_POLICIES: [LosslessEncodeReferencePolicy; 2] = [ + LosslessEncodeReferencePolicy { + components: 3, + min_pixels: 1024 * 1024, + dwt_cpu_ns: 19_506_000, + dwt_accelerated_ns: 2_616_000, + ht_cpu_ns: 4_566_000, + ht_accelerated_ns: 2_002_000, + end_to_end_cpu_ns: 81_419_000, + end_to_end_accelerated_ns: 41_307_000, + criterion_noise_percent: 2.0, + }, + LosslessEncodeReferencePolicy { + components: 4, + min_pixels: 1024 * 1024, + dwt_cpu_ns: 19_506_000, + dwt_accelerated_ns: 2_616_000, + ht_cpu_ns: 4_566_000, + ht_accelerated_ns: 2_002_000, + end_to_end_cpu_ns: 108_350_000, + end_to_end_accelerated_ns: 53_360_000, + criterion_noise_percent: 2.0, + }, +]; + +fn cuda_htj2k_host_encode_reference_policy( + workload: J2kAdaptiveWorkload, +) -> Option<&'static LosslessEncodeReferencePolicy> { + let pixels = u64::from(workload.tile_size.0).saturating_mul(u64::from(workload.tile_size.1)); + CUDA_HTJ2K_HOST_ENCODE_REFERENCE_POLICIES + .iter() + .find(|policy| { + workload.operation == J2kAdaptiveOperation::Encode + && workload.codec_mode == J2kAdaptiveCodecMode::Htj2k + && workload.quality_mode == J2kAdaptiveQualityMode::Lossless + && workload.components == policy.components + && workload.bit_depth == 8 + && workload.batch_size == 1 + && !workload.roi + && !workload.scaled + && workload.quality_layers == 1 + && workload.output_residency == J2kAdaptiveOutputResidency::Host + && pixels >= policy.min_pixels + }) +} + +/// High-level operation being routed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kAdaptiveOperation { + /// JPEG 2000-family encode. + Encode, + /// JPEG 2000-family decode. + Decode, + /// JPEG 2000-family transcode or recode. + Transcode, +} + +/// Codestream family being routed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kAdaptiveCodecMode { + /// Classic JPEG 2000 Part 1 block coding. + ClassicJ2k, + /// High-throughput JPEG 2000 Part 15 block coding. + Htj2k, +} + +/// Quality mode being routed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kAdaptiveQualityMode { + /// Reversible/lossless path. + Lossless, + /// Irreversible/lossy path. + Lossy, +} + +/// Desired ownership of produced buffers. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kAdaptiveOutputResidency { + /// Return host-visible output. + Host, + /// Keep output resident on the selected device when the adapter supports it. + Device, +} + +/// One JPEG 2000-family workload shape used by the adaptive planner. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct J2kAdaptiveWorkload { + /// Operation being routed. + pub operation: J2kAdaptiveOperation, + /// Classic J2K versus HTJ2K mode. + pub codec_mode: J2kAdaptiveCodecMode, + /// Lossless versus lossy mode. + pub quality_mode: J2kAdaptiveQualityMode, + /// Number of image components. + pub components: u8, + /// Significant bits per component sample. + pub bit_depth: u8, + /// Tile dimensions in pixels. + pub tile_size: (u32, u32), + /// Number of same-shaped tiles or frames in the route. + pub batch_size: u16, + /// Whether this route decodes or transcodes a source ROI. + pub roi: bool, + /// Whether this route decodes or transcodes at reduced resolution. + pub scaled: bool, + /// Number of cumulative quality layers requested or present. + pub quality_layers: u16, + /// Desired output residency. + pub output_residency: J2kAdaptiveOutputResidency, +} + +impl J2kAdaptiveWorkload { + /// Build a workload with full-frame host output and one quality layer. + #[must_use] + pub const fn new( + operation: J2kAdaptiveOperation, + codec_mode: J2kAdaptiveCodecMode, + quality_mode: J2kAdaptiveQualityMode, + components: u8, + bit_depth: u8, + tile_size: (u32, u32), + batch_size: u16, + ) -> Self { + Self { + operation, + codec_mode, + quality_mode, + components, + bit_depth, + tile_size, + batch_size, + roi: false, + scaled: false, + quality_layers: 1, + output_residency: J2kAdaptiveOutputResidency::Host, + } + } + + /// Return the workload with ROI routing enabled or disabled. + #[must_use] + pub const fn with_roi(mut self, roi: bool) -> Self { + self.roi = roi; + self + } + + /// Return the workload with scaled routing enabled or disabled. + #[must_use] + pub const fn with_scaled(mut self, scaled: bool) -> Self { + self.scaled = scaled; + self + } + + /// Return the workload with a quality-layer count. + #[must_use] + pub const fn with_quality_layers(mut self, quality_layers: u16) -> Self { + self.quality_layers = quality_layers; + self + } + + /// Return the workload with the requested output residency. + #[must_use] + pub const fn with_output_residency( + mut self, + output_residency: J2kAdaptiveOutputResidency, + ) -> Self { + self.output_residency = output_residency; + self + } + + /// Classify a stage for this workload before benchmark gating. + #[must_use] + pub fn logical_owner_for(self, stage: J2kAdaptiveStage) -> J2kAdaptiveStageOwner { + if self.is_small_cpu_workload() { + return J2kAdaptiveStageOwner::Cpu; + } + + match stage { + J2kAdaptiveStage::MarkerParsing + | J2kAdaptiveStage::CopySync + | J2kAdaptiveStage::Validation => J2kAdaptiveStageOwner::Cpu, + J2kAdaptiveStage::Mct => { + if self.components >= 3 && self.is_wsi_shaped() { + J2kAdaptiveStageOwner::Gpu + } else { + J2kAdaptiveStageOwner::Variable + } + } + J2kAdaptiveStage::Dwt => match self.operation { + J2kAdaptiveOperation::Encode | J2kAdaptiveOperation::Transcode + if self.is_wsi_shaped() => + { + J2kAdaptiveStageOwner::Gpu + } + _ => J2kAdaptiveStageOwner::Cpu, + }, + J2kAdaptiveStage::Idwt => match self.operation { + J2kAdaptiveOperation::Decode | J2kAdaptiveOperation::Transcode + if self.is_wsi_shaped() => + { + J2kAdaptiveStageOwner::Gpu + } + _ => J2kAdaptiveStageOwner::Cpu, + }, + J2kAdaptiveStage::Quantization => { + if self.quality_mode == J2kAdaptiveQualityMode::Lossy && self.is_wsi_shaped() { + J2kAdaptiveStageOwner::Gpu + } else { + J2kAdaptiveStageOwner::Cpu + } + } + J2kAdaptiveStage::HtBlockCoding => { + if self.codec_mode == J2kAdaptiveCodecMode::Htj2k && self.is_wsi_shaped() { + J2kAdaptiveStageOwner::Gpu + } else { + J2kAdaptiveStageOwner::Cpu + } + } + J2kAdaptiveStage::Tier1 + | J2kAdaptiveStage::PcrdRateControl + | J2kAdaptiveStage::Packetization + | J2kAdaptiveStage::CodestreamAssembly => J2kAdaptiveStageOwner::Variable, + } + } + + fn is_wsi_shaped(self) -> bool { + let pixels = u64::from(self.tile_size.0).saturating_mul(u64::from(self.tile_size.1)); + pixels >= 512 * 512 || self.batch_size >= 16 + } + + fn is_small_cpu_workload(self) -> bool { + let pixels = u64::from(self.tile_size.0).saturating_mul(u64::from(self.tile_size.1)); + pixels < 512 * 512 && self.batch_size <= 1 + } +} + +/// Pipeline stage represented in adaptive route reports. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kAdaptiveStage { + /// Marker parsing and main/tile header processing. + MarkerParsing, + /// Multi-component transform, including fused deinterleave plus RCT/ICT. + Mct, + /// Forward wavelet transform. + Dwt, + /// Inverse wavelet transform. + Idwt, + /// Irreversible quantization or dequantization. + Quantization, + /// Classic EBCOT Tier-1 block coding. + Tier1, + /// HTJ2K cleanup/refinement block coding. + HtBlockCoding, + /// PCRD and rate-control decisions. + PcrdRateControl, + /// Packet ordering and packet body assembly. + Packetization, + /// Codestream marker and tile-part assembly. + CodestreamAssembly, + /// Host-device copies, synchronization, and residency transitions. + CopySync, + /// Decode/round-trip/output validation. + Validation, +} + +impl J2kAdaptiveStage { + /// Every stage emitted by adaptive route reports. + pub const ALL: [Self; 12] = [ + Self::MarkerParsing, + Self::Mct, + Self::Dwt, + Self::Idwt, + Self::Quantization, + Self::Tier1, + Self::HtBlockCoding, + Self::PcrdRateControl, + Self::Packetization, + Self::CodestreamAssembly, + Self::CopySync, + Self::Validation, + ]; + + /// Stable profiling label for this stage. + #[must_use] + pub const fn profile_label(self) -> &'static str { + match self { + Self::MarkerParsing => "marker_parsing", + Self::Mct => "mct_rct_ict", + Self::Dwt => "dwt", + Self::Idwt => "idwt", + Self::Quantization => "quantization", + Self::Tier1 => "tier1", + Self::HtBlockCoding => "ht_block_coding", + Self::PcrdRateControl => "pcrd_rate_control", + Self::Packetization => "packetization", + Self::CodestreamAssembly => "codestream_assembly", + Self::CopySync => "copy_sync", + Self::Validation => "validation", + } + } +} + +/// Logical owner class before benchmark gates are applied. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kAdaptiveStageOwner { + /// CPU-shaped work. + Cpu, + /// GPU-shaped work that must still pass stage and end-to-end gates. + Gpu, + /// Workload-dependent stage requiring benchmark evidence before default device routing. + Variable, +} + +/// RCA reason for a logical GPU stage that did not pass its gate. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kAdaptiveRcaReason { + /// The device implementation uses the wrong algorithmic structure. + AlgorithmicMismatch, + /// Transfer or synchronization overhead dominates useful device work. + TransferSyncOverhead, + /// The implementation does not batch enough independent work. + MissingBatching, + /// The route loses residency and pays unnecessary host-device movement. + MissingResidency, + /// This exact workload is too small for device routing. + TooSmallWorkload, + /// The benchmark does not measure the intended workload shape. + BenchmarkMismatch, + /// Evidence shows the optimized CPU is genuinely better for this exact shape. + CpuGenuinelyBetter, +} + +/// RCA classification for a blocked stage/backend pair. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct J2kAdaptiveRcaFinding { + /// Stage covered by the finding. + pub stage: J2kAdaptiveStage, + /// Device backend covered by the finding. + pub backend: BackendKind, + /// RCA reason. + pub reason: J2kAdaptiveRcaReason, + /// Whether the finding permits CPU routing for this exact stage/backend. + pub reclassify_cpu: bool, +} + +impl J2kAdaptiveRcaFinding { + /// Record that RCA permits CPU routing for this exact stage/backend. + #[must_use] + pub const fn reclassify_cpu( + stage: J2kAdaptiveStage, + backend: BackendKind, + reason: J2kAdaptiveRcaReason, + ) -> Self { + Self { + stage, + backend, + reason, + reclassify_cpu: true, + } + } +} + +/// Benchmark evidence for one stage or one end-to-end adaptive route. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct J2kAdaptiveBenchmarkEvidence { + /// Scope covered by the evidence. + pub scope: J2kAdaptiveBenchmarkScope, + /// Device backend measured against CPU. + pub backend: BackendKind, + /// Optimized CPU time in nanoseconds. + pub cpu_ns: u64, + /// Optimized accelerated or adaptive time in nanoseconds. + pub accelerated_ns: u64, + /// Criterion noise bound in percentage points. + pub criterion_noise_percent: f64, +} + +impl J2kAdaptiveBenchmarkEvidence { + /// Build stage benchmark evidence. + #[must_use] + pub const fn stage( + stage: J2kAdaptiveStage, + backend: BackendKind, + cpu_ns: u64, + accelerated_ns: u64, + criterion_noise_percent: f64, + ) -> Self { + Self { + scope: J2kAdaptiveBenchmarkScope::Stage(stage), + backend, + cpu_ns, + accelerated_ns, + criterion_noise_percent, + } + } + + /// Build end-to-end route benchmark evidence. + #[must_use] + pub const fn end_to_end( + backend: BackendKind, + cpu_ns: u64, + accelerated_ns: u64, + criterion_noise_percent: f64, + ) -> Self { + Self { + scope: J2kAdaptiveBenchmarkScope::EndToEnd, + backend, + cpu_ns, + accelerated_ns, + criterion_noise_percent, + } + } + + /// Percent speedup of accelerated over CPU. + #[must_use] + #[allow(clippy::cast_precision_loss)] + pub fn improvement_percent(self) -> f64 { + if self.accelerated_ns == 0 { + return f64::INFINITY; + } + ((self.cpu_ns as f64 / self.accelerated_ns as f64) - 1.0) * 100.0 + } + + fn passes(self, policy: J2kAdaptiveGatePolicy) -> bool { + self.cpu_ns > 0 + && self.accelerated_ns > 0 + && self.improvement_percent() + >= policy.min_speedup_percent + self.criterion_noise_percent + } +} + +/// Scope covered by benchmark evidence. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kAdaptiveBenchmarkScope { + /// Evidence for one stage. + Stage(J2kAdaptiveStage), + /// Evidence for the full adaptive route. + EndToEnd, +} + +/// Benchmark evidence set used by the planner. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct J2kAdaptiveBenchmarks { + stage: Vec, + end_to_end: Vec, +} + +impl J2kAdaptiveBenchmarks { + /// Add stage evidence. Later evidence for the same stage/backend takes precedence. + pub fn push_stage(&mut self, evidence: J2kAdaptiveBenchmarkEvidence) { + debug_assert!(matches!( + evidence.scope, + J2kAdaptiveBenchmarkScope::Stage(_) + )); + self.stage.push(evidence); + } + + /// Add end-to-end evidence. Later evidence for the same backend takes precedence. + pub fn push_end_to_end(&mut self, evidence: J2kAdaptiveBenchmarkEvidence) { + debug_assert!(matches!( + evidence.scope, + J2kAdaptiveBenchmarkScope::EndToEnd + )); + self.end_to_end.push(evidence); + } + + fn stage_for( + &self, + stage: J2kAdaptiveStage, + backend: BackendKind, + ) -> Option { + self.stage.iter().rev().copied().find(|evidence| { + evidence.backend == backend && evidence.scope == J2kAdaptiveBenchmarkScope::Stage(stage) + }) + } + + fn end_to_end_for(&self, backend: BackendKind) -> Option { + self.end_to_end + .iter() + .rev() + .copied() + .find(|evidence| evidence.backend == backend) + } + + fn has_evidence_for(&self, backend: BackendKind) -> bool { + self.end_to_end_for(backend).is_some() + || self + .stage + .iter() + .any(|evidence| evidence.backend == backend) + } + + fn best_observed_ns_for(&self, backend: BackendKind) -> Option { + let end_to_end = self + .end_to_end_for(backend) + .map(|evidence| evidence.accelerated_ns); + let stage = self + .stage + .iter() + .rev() + .find(|evidence| evidence.backend == backend) + .map(|evidence| evidence.accelerated_ns); + end_to_end.or(stage) + } +} + +/// Adaptive route gate policy. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct J2kAdaptiveGatePolicy { + /// Minimum speedup required before Criterion noise is added. + pub min_speedup_percent: f64, +} + +impl Default for J2kAdaptiveGatePolicy { + fn default() -> Self { + Self { + min_speedup_percent: 10.0, + } + } +} + +/// Route selected by the planner. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kAdaptiveRouteKind { + /// Portable CPU route. + CpuOnly, + /// CPU plus benchmark-approved device stages. + Hybrid, + /// Strict device proof route. + StrictDevice, +} + +/// Gate status for one stage decision. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kAdaptiveStageGateStatus { + /// CPU-shaped stage, or CPU requested explicitly. + CpuShaped, + /// Variable stage kept on CPU because no passing stage benchmark exists. + VariableCpuDefault, + /// Stage passed its benchmark gate and may route to device. + Approved, + /// Required benchmark evidence is missing. + BenchmarkGateMissing, + /// A logical GPU stage failed its gate and needs RCA before default routing. + BlockedNeedsRca, + /// RCA permits CPU routing for this exact stage/backend/workload. + ReclassifiedCpu, + /// Strict device proof route bypasses adaptive performance gates. + StrictDeviceProof, + /// End-to-end route evidence is missing or below threshold. + EndToEndGateBlocked, +} + +/// One stage placement and gate result. +#[derive(Debug, Clone, PartialEq)] +pub struct J2kAdaptiveStageDecision { + /// Pipeline stage. + pub stage: J2kAdaptiveStage, + /// Logical owner before gates. + pub logical_owner: J2kAdaptiveStageOwner, + /// Backend selected for this stage in the returned route. + pub selected_backend: BackendKind, + /// Gate status for this stage. + pub gate_status: J2kAdaptiveStageGateStatus, + /// Measured stage speedup when stage evidence was available. + pub improvement_percent: Option, + /// RCA finding applied to this stage, if any. + pub rca_reason: Option, +} + +impl J2kAdaptiveStageDecision { + /// Return true when this stage blocks default GPU routing pending evidence or RCA. + #[must_use] + pub fn requires_rca(&self) -> bool { + matches!( + self.gate_status, + J2kAdaptiveStageGateStatus::BenchmarkGateMissing + | J2kAdaptiveStageGateStatus::BlockedNeedsRca + ) + } +} + +/// Adaptive planner output. +#[derive(Debug, Clone, PartialEq)] +pub struct J2kAdaptiveRouteReport { + /// Requested route intent. + pub request: J2kAdaptiveBackendRequest, + /// Selected route kind. + pub route_kind: J2kAdaptiveRouteKind, + /// Device used by the route, when any stage is device-backed. + pub selected_device: Option, + /// Stage decisions, always covering every [`J2kAdaptiveStage::ALL`] entry. + pub stages: Vec, +} + +impl J2kAdaptiveRouteReport { + /// Return the decision for a stage. + #[must_use] + pub fn stage(&self, stage: J2kAdaptiveStage) -> Option<&J2kAdaptiveStageDecision> { + self.stages.iter().find(|decision| decision.stage == stage) + } + + /// Return true when any logical GPU stage remains unresolved. + #[must_use] + pub fn has_unresolved_rca(&self) -> bool { + self.stages + .iter() + .any(J2kAdaptiveStageDecision::requires_rca) + } +} + +/// Adaptive JPEG 2000 route planner. +#[derive(Debug, Clone, PartialEq)] +pub struct J2kAdaptiveRoutePlanner { + capabilities: BackendCapabilities, + policy: J2kAdaptiveGatePolicy, + rca_findings: Vec, +} + +impl J2kAdaptiveRoutePlanner { + /// Build a planner from caller-provided capabilities. + #[must_use] + pub fn new(capabilities: BackendCapabilities) -> Self { + Self { + capabilities, + policy: J2kAdaptiveGatePolicy::default(), + rca_findings: Vec::new(), + } + } + + /// Build a planner with the default lossless encode policy evidence. + #[must_use] + pub fn lossless_encode(capabilities: BackendCapabilities) -> Self { + Self::new(capabilities).with_lossless_encode_policy() + } + + /// Build a planner with compile-time default capabilities. + #[must_use] + pub fn compile_time_defaults() -> Self { + Self::new(BackendCapabilities::compile_time_defaults()) + } + + /// Return a planner with a different gate policy. + #[must_use] + pub const fn with_policy(mut self, policy: J2kAdaptiveGatePolicy) -> Self { + self.policy = policy; + self + } + + /// Return a planner with an RCA finding. + #[must_use] + pub fn with_rca_finding(mut self, finding: J2kAdaptiveRcaFinding) -> Self { + self.rca_findings.push(finding); + self + } + + /// Return a planner with lossless host-pixel encode RCA defaults. + #[must_use] + pub fn with_lossless_encode_policy(self) -> Self { + self.with_rca_finding(J2kAdaptiveRcaFinding::reclassify_cpu( + J2kAdaptiveStage::Mct, + BackendKind::Cuda, + J2kAdaptiveRcaReason::CpuGenuinelyBetter, + )) + } + + /// Plan the default lossless JPEG 2000/HTJ2K encode route. + /// + /// The default route is conservative: CPU-shaped work stays on CPU, and + /// CUDA HTJ2K host-output encode is approved only for shapes covered by the + /// documented reference-policy table in this module. + pub fn plan_lossless_encode( + &self, + samples: J2kLosslessSamples<'_>, + options: J2kLosslessEncodeOptions, + ) -> Result { + let workload = Self::lossless_encode_workload(samples, options); + let benchmarks = Self::lossless_encode_benchmarks(workload); + self.plan( + workload, + J2kAdaptiveBackendRequest::Accelerated, + &benchmarks, + ) + } + + /// Build the adaptive workload for lossless encode options. + #[must_use] + pub fn lossless_encode_workload( + samples: J2kLosslessSamples<'_>, + options: J2kLosslessEncodeOptions, + ) -> J2kAdaptiveWorkload { + let codec_mode = match options.block_coding_mode { + J2kBlockCodingMode::Classic => J2kAdaptiveCodecMode::ClassicJ2k, + J2kBlockCodingMode::HighThroughput => J2kAdaptiveCodecMode::Htj2k, + }; + J2kAdaptiveWorkload::new( + J2kAdaptiveOperation::Encode, + codec_mode, + J2kAdaptiveQualityMode::Lossless, + samples.components, + samples.bit_depth, + (samples.width, samples.height), + 1, + ) + } + + /// Benchmark evidence for default lossless encode auto-routing. + #[must_use] + pub fn lossless_encode_benchmarks(workload: J2kAdaptiveWorkload) -> J2kAdaptiveBenchmarks { + let mut benchmarks = J2kAdaptiveBenchmarks::default(); + if let Some(policy) = cuda_htj2k_host_encode_reference_policy(workload) { + benchmarks.push_stage(J2kAdaptiveBenchmarkEvidence::stage( + J2kAdaptiveStage::Dwt, + BackendKind::Cuda, + policy.dwt_cpu_ns, + policy.dwt_accelerated_ns, + policy.criterion_noise_percent, + )); + benchmarks.push_stage(J2kAdaptiveBenchmarkEvidence::stage( + J2kAdaptiveStage::HtBlockCoding, + BackendKind::Cuda, + policy.ht_cpu_ns, + policy.ht_accelerated_ns, + policy.criterion_noise_percent, + )); + benchmarks.push_end_to_end(J2kAdaptiveBenchmarkEvidence::end_to_end( + BackendKind::Cuda, + policy.end_to_end_cpu_ns, + policy.end_to_end_accelerated_ns, + policy.criterion_noise_percent, + )); + } + benchmarks + } + + /// Plan a route for the workload and benchmark evidence. + pub fn plan( + &self, + workload: J2kAdaptiveWorkload, + request: J2kAdaptiveBackendRequest, + benchmarks: &J2kAdaptiveBenchmarks, + ) -> Result { + match request { + J2kAdaptiveBackendRequest::CpuOnly => Ok(Self::cpu_only_report(workload, request)), + J2kAdaptiveBackendRequest::StrictDevice(backend) => { + self.strict_device_report(workload, request, backend) + } + J2kAdaptiveBackendRequest::Accelerated => { + Ok(self.accelerated_report(workload, request, benchmarks)) + } + } + } + + fn cpu_only_report( + workload: J2kAdaptiveWorkload, + request: J2kAdaptiveBackendRequest, + ) -> J2kAdaptiveRouteReport { + let stages = J2kAdaptiveStage::ALL + .into_iter() + .map(|stage| J2kAdaptiveStageDecision { + stage, + logical_owner: workload.logical_owner_for(stage), + selected_backend: BackendKind::Cpu, + gate_status: J2kAdaptiveStageGateStatus::CpuShaped, + improvement_percent: None, + rca_reason: None, + }) + .collect(); + J2kAdaptiveRouteReport { + request, + route_kind: J2kAdaptiveRouteKind::CpuOnly, + selected_device: None, + stages, + } + } + + fn strict_device_report( + &self, + workload: J2kAdaptiveWorkload, + request: J2kAdaptiveBackendRequest, + backend: BackendKind, + ) -> Result { + if !self.supports_backend(backend) { + return Err(Unsupported { + what: "strict JPEG 2000 device route is unavailable", + } + .into()); + } + + let stages = J2kAdaptiveStage::ALL + .into_iter() + .map(|stage| { + let logical_owner = workload.logical_owner_for(stage); + let selected_backend = if logical_owner == J2kAdaptiveStageOwner::Cpu { + BackendKind::Cpu + } else { + backend + }; + J2kAdaptiveStageDecision { + stage, + logical_owner, + selected_backend, + gate_status: if selected_backend == BackendKind::Cpu { + J2kAdaptiveStageGateStatus::CpuShaped + } else { + J2kAdaptiveStageGateStatus::StrictDeviceProof + }, + improvement_percent: None, + rca_reason: None, + } + }) + .collect(); + + Ok(J2kAdaptiveRouteReport { + request, + route_kind: J2kAdaptiveRouteKind::StrictDevice, + selected_device: Some(backend), + stages, + }) + } + + fn accelerated_report( + &self, + workload: J2kAdaptiveWorkload, + request: J2kAdaptiveBackendRequest, + benchmarks: &J2kAdaptiveBenchmarks, + ) -> J2kAdaptiveRouteReport { + let Some(backend) = self.best_approved_device(workload, benchmarks) else { + return self.gated_cpu_report( + workload, + request, + self.best_candidate_device(benchmarks), + benchmarks, + ); + }; + + let mut stages = Vec::with_capacity(J2kAdaptiveStage::ALL.len()); + let mut unresolved = false; + for stage in J2kAdaptiveStage::ALL { + let decision = self.stage_decision(workload, stage, backend, benchmarks, true); + unresolved |= decision.requires_rca(); + stages.push(decision); + } + + if unresolved { + for decision in &mut stages { + decision.selected_backend = BackendKind::Cpu; + } + return J2kAdaptiveRouteReport { + request, + route_kind: J2kAdaptiveRouteKind::CpuOnly, + selected_device: None, + stages, + }; + } + + let has_device_stage = stages + .iter() + .any(|decision| decision.selected_backend == backend); + J2kAdaptiveRouteReport { + request, + route_kind: if has_device_stage { + J2kAdaptiveRouteKind::Hybrid + } else { + J2kAdaptiveRouteKind::CpuOnly + }, + selected_device: has_device_stage.then_some(backend), + stages, + } + } + + fn gated_cpu_report( + &self, + workload: J2kAdaptiveWorkload, + request: J2kAdaptiveBackendRequest, + backend: Option, + benchmarks: &J2kAdaptiveBenchmarks, + ) -> J2kAdaptiveRouteReport { + let stages = J2kAdaptiveStage::ALL + .into_iter() + .map(|stage| { + let mut decision = backend.map_or_else( + || { + let logical_owner = workload.logical_owner_for(stage); + J2kAdaptiveStageDecision { + stage, + logical_owner, + selected_backend: BackendKind::Cpu, + gate_status: if logical_owner == J2kAdaptiveStageOwner::Gpu { + J2kAdaptiveStageGateStatus::BenchmarkGateMissing + } else { + J2kAdaptiveStageGateStatus::CpuShaped + }, + improvement_percent: None, + rca_reason: None, + } + }, + |backend| { + let end_to_end_passed = benchmarks + .end_to_end_for(backend) + .is_some_and(|evidence| evidence.passes(self.policy)); + self.stage_decision(workload, stage, backend, benchmarks, end_to_end_passed) + }, + ); + decision.selected_backend = BackendKind::Cpu; + decision + }) + .collect(); + + J2kAdaptiveRouteReport { + request, + route_kind: J2kAdaptiveRouteKind::CpuOnly, + selected_device: None, + stages, + } + } + + fn stage_decision( + &self, + workload: J2kAdaptiveWorkload, + stage: J2kAdaptiveStage, + backend: BackendKind, + benchmarks: &J2kAdaptiveBenchmarks, + end_to_end_passed: bool, + ) -> J2kAdaptiveStageDecision { + let logical_owner = workload.logical_owner_for(stage); + match logical_owner { + J2kAdaptiveStageOwner::Cpu => J2kAdaptiveStageDecision { + stage, + logical_owner, + selected_backend: BackendKind::Cpu, + gate_status: J2kAdaptiveStageGateStatus::CpuShaped, + improvement_percent: None, + rca_reason: None, + }, + J2kAdaptiveStageOwner::Variable => { + let evidence = benchmarks.stage_for(stage, backend); + let approved = end_to_end_passed + && evidence.is_some_and(|evidence| evidence.passes(self.policy)); + J2kAdaptiveStageDecision { + stage, + logical_owner, + selected_backend: if approved { backend } else { BackendKind::Cpu }, + gate_status: if approved { + J2kAdaptiveStageGateStatus::Approved + } else { + J2kAdaptiveStageGateStatus::VariableCpuDefault + }, + improvement_percent: evidence + .map(J2kAdaptiveBenchmarkEvidence::improvement_percent), + rca_reason: None, + } + } + J2kAdaptiveStageOwner::Gpu => { + if let Some(finding) = self.rca_for(stage, backend) { + return J2kAdaptiveStageDecision { + stage, + logical_owner, + selected_backend: BackendKind::Cpu, + gate_status: J2kAdaptiveStageGateStatus::ReclassifiedCpu, + improvement_percent: benchmarks + .stage_for(stage, backend) + .map(J2kAdaptiveBenchmarkEvidence::improvement_percent), + rca_reason: Some(finding.reason), + }; + } + + let evidence = benchmarks.stage_for(stage, backend); + let gate_status = match (end_to_end_passed, evidence) { + (false, _) => J2kAdaptiveStageGateStatus::EndToEndGateBlocked, + (true, None) => J2kAdaptiveStageGateStatus::BenchmarkGateMissing, + (true, Some(evidence)) if evidence.passes(self.policy) => { + J2kAdaptiveStageGateStatus::Approved + } + (true, Some(_)) => J2kAdaptiveStageGateStatus::BlockedNeedsRca, + }; + J2kAdaptiveStageDecision { + stage, + logical_owner, + selected_backend: if gate_status == J2kAdaptiveStageGateStatus::Approved { + backend + } else { + BackendKind::Cpu + }, + gate_status, + improvement_percent: evidence + .map(J2kAdaptiveBenchmarkEvidence::improvement_percent), + rca_reason: None, + } + } + } + } + + fn best_approved_device( + &self, + workload: J2kAdaptiveWorkload, + benchmarks: &J2kAdaptiveBenchmarks, + ) -> Option { + [BackendKind::Metal, BackendKind::Cuda] + .into_iter() + .filter(|backend| self.supports_backend(*backend)) + .filter_map(|backend| { + benchmarks + .end_to_end_for(backend) + .filter(|evidence| evidence.passes(self.policy)) + .map(|evidence| (backend, evidence.accelerated_ns)) + }) + .filter(|(backend, _)| { + J2kAdaptiveStage::ALL.into_iter().all(|stage| { + !self + .stage_decision(workload, stage, *backend, benchmarks, true) + .requires_rca() + }) + }) + .min_by_key(|(_, accelerated_ns)| *accelerated_ns) + .map(|(backend, _)| backend) + } + + fn best_candidate_device(&self, benchmarks: &J2kAdaptiveBenchmarks) -> Option { + [BackendKind::Metal, BackendKind::Cuda] + .into_iter() + .filter(|backend| self.supports_backend(*backend)) + .filter(|backend| benchmarks.has_evidence_for(*backend)) + .min_by_key(|backend| { + benchmarks + .best_observed_ns_for(*backend) + .unwrap_or(u64::MAX) + }) + } + + fn supports_backend(&self, backend: BackendKind) -> bool { + match backend { + BackendKind::Cpu => true, + BackendKind::Metal => self.capabilities.metal, + BackendKind::Cuda => self.capabilities.cuda, + } + } + + fn rca_for( + &self, + stage: J2kAdaptiveStage, + backend: BackendKind, + ) -> Option { + self.rca_findings.iter().copied().find(|finding| { + finding.stage == stage && finding.backend == backend && finding.reclassify_cpu + }) + } +} diff --git a/crates/signinum-j2k/src/adapter/device_plan.rs b/crates/j2k/src/adapter/device_plan.rs similarity index 65% rename from crates/signinum-j2k/src/adapter/device_plan.rs rename to crates/j2k/src/adapter/device_plan.rs index 8431edf2..cc2bae47 100644 --- a/crates/signinum-j2k/src/adapter/device_plan.rs +++ b/crates/j2k/src/adapter/device_plan.rs @@ -1,16 +1,33 @@ // SPDX-License-Identifier: Apache-2.0 use crate::error::J2kError; -use signinum_core::{Downscale, Rect}; +use j2k_core::{Downscale, Rect}; +/// Device decode shape requested by a GPU adapter. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DeviceDecodeRequest { + /// Decode the full image at full resolution. Full, - Region { roi: Rect }, - Scaled { scale: Downscale }, - RegionScaled { roi: Rect, scale: Downscale }, + /// Decode a full-resolution source region. + Region { + /// Source region of interest. + roi: Rect, + }, + /// Decode the full image at reduced resolution. + Scaled { + /// Requested downscale factor. + scale: Downscale, + }, + /// Decode a source region at reduced resolution. + RegionScaled { + /// Source region of interest. + roi: Rect, + /// Requested downscale factor. + scale: Downscale, + }, } +/// Normalized device decode plan derived from source dimensions and request. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DeviceDecodePlan { source_dims: (u32, u32), @@ -20,6 +37,7 @@ pub struct DeviceDecodePlan { } impl DeviceDecodePlan { + /// Build a normalized plan for an image. pub fn for_image( source_dims: (u32, u32), request: DeviceDecodeRequest, @@ -50,30 +68,37 @@ impl DeviceDecodePlan { }) } + /// Original image dimensions. pub fn source_dims(self) -> (u32, u32) { self.source_dims } + /// Full-resolution source rectangle to read. pub fn source_rect(self) -> Rect { self.source_rect } + /// Requested downscale factor. pub fn scale(self) -> Downscale { self.scale } + /// Output rectangle in reduced-resolution coordinates. pub fn output_rect(self) -> Rect { self.output_rect } + /// Output dimensions in pixels. pub fn output_dims(self) -> (u32, u32) { (self.output_rect.w, self.output_rect.h) } + /// Target resolution hint for native decoders that accept one. pub fn target_resolution(self) -> Option<(u32, u32)> { (self.scale != Downscale::None).then_some(self.output_dims()) } + /// Return true when the request is an unscaled full-frame decode. pub fn is_full_frame(self) -> bool { self.source_rect == Rect::full(self.source_dims) && self.scale == Downscale::None } diff --git a/crates/j2k/src/adapter/encode_stage.rs b/crates/j2k/src/adapter/encode_stage.rs new file mode 100644 index 00000000..b2d4b48b --- /dev/null +++ b/crates/j2k/src/adapter/encode_stage.rs @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Public JPEG 2000 encode-stage adapter contracts. +//! +//! The encode-stage job, output, and report types are shared with +//! `j2k-native` through the neutral `j2k-types` contract +//! crate; this module re-exports them and keeps the adapter-side accelerator +//! trait plus the hidden native bridge. + +use alloc::vec::Vec; + +pub use j2k_types::{ + CpuOnlyJ2kEncodeStageAccelerator, EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, + IrreversibleQuantizationStep, IrreversibleQuantizationSubbandScales, J2kCodeBlockSegment, + J2kCodeBlockStyle, J2kDeinterleaveToF32Job, J2kEncodeDispatchReport, J2kForwardDwt53Job, + J2kForwardDwt53Level, J2kForwardDwt53Output, J2kForwardDwt97Job, J2kForwardDwt97Level, + J2kForwardDwt97Output, J2kForwardIctJob, J2kForwardRctJob, J2kHtCodeBlockEncodeJob, + J2kHtSubbandEncodeJob, J2kHtj2kTileEncodeJob, J2kPacketizationBlockCodingMode, + J2kPacketizationCodeBlock, J2kPacketizationEncodeJob, J2kPacketizationPacketDescriptor, + J2kPacketizationProgressionOrder, J2kPacketizationResolution, J2kPacketizationSubband, + J2kQuantizeSubbandJob, J2kSubBandType, J2kTier1CodeBlockEncodeJob, PrecomputedHtj2k53Component, + PrecomputedHtj2k53Image, PrecomputedHtj2k97Component, PrecomputedHtj2k97Image, + PreencodedHtj2k97CodeBlock, PreencodedHtj2k97CompactCodeBlock, + PreencodedHtj2k97CompactComponent, PreencodedHtj2k97CompactImage, + PreencodedHtj2k97CompactResolution, PreencodedHtj2k97CompactSubband, + PreencodedHtj2k97Component, PreencodedHtj2k97Image, PreencodedHtj2k97Resolution, + PreencodedHtj2k97Subband, PrequantizedHtj2k97CodeBlock, PrequantizedHtj2k97Component, + PrequantizedHtj2k97Image, PrequantizedHtj2k97Resolution, PrequantizedHtj2k97Subband, +}; + +/// Adapter JPEG 2000 encode-stage accelerator for backend experimentation. +pub trait J2kEncodeStageAccelerator { + /// Report cumulative backend dispatches completed by this accelerator. + fn dispatch_report(&self) -> J2kEncodeDispatchReport { + J2kEncodeDispatchReport::default() + } + + /// Optionally deinterleave interleaved pixel bytes into f32 component planes. + fn encode_deinterleave( + &mut self, + _job: J2kDeinterleaveToF32Job<'_>, + ) -> core::result::Result>>, &'static str> { + Ok(None) + } + + /// Optionally apply forward RCT in place. + fn encode_forward_rct( + &mut self, + _job: J2kForwardRctJob<'_>, + ) -> core::result::Result { + Ok(false) + } + + /// Optionally apply forward ICT in place. + fn encode_forward_ict( + &mut self, + _job: J2kForwardIctJob<'_>, + ) -> core::result::Result { + Ok(false) + } + + /// Optionally run a forward reversible 5/3 DWT. + fn encode_forward_dwt53( + &mut self, + _job: J2kForwardDwt53Job<'_>, + ) -> core::result::Result, &'static str> { + Ok(None) + } + + /// Optionally run a forward irreversible 9/7 DWT. + fn encode_forward_dwt97( + &mut self, + _job: J2kForwardDwt97Job<'_>, + ) -> core::result::Result, &'static str> { + Ok(None) + } + + /// Optionally quantize one sub-band. + fn encode_quantize_subband( + &mut self, + _job: J2kQuantizeSubbandJob<'_>, + ) -> core::result::Result>, &'static str> { + Ok(None) + } + + /// Optionally encode one classic Tier-1 code-block. + fn encode_tier1_code_block( + &mut self, + _job: J2kTier1CodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + Ok(None) + } + + /// Optionally encode multiple classic Tier-1 code-blocks in one backend dispatch. + fn encode_tier1_code_blocks( + &mut self, + _jobs: &[J2kTier1CodeBlockEncodeJob<'_>], + ) -> core::result::Result>, &'static str> { + Ok(None) + } + + /// Optionally encode one HTJ2K code-block. + fn encode_ht_code_block( + &mut self, + _job: J2kHtCodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + Ok(None) + } + + /// Optionally encode multiple HTJ2K code-blocks in one backend dispatch. + fn encode_ht_code_blocks( + &mut self, + _jobs: &[J2kHtCodeBlockEncodeJob<'_>], + ) -> core::result::Result>, &'static str> { + Ok(None) + } + + /// Optionally quantize and encode one HTJ2K cleanup-only sub-band. + fn encode_ht_subband( + &mut self, + _job: J2kHtSubbandEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + Ok(None) + } + + /// Optionally encode the complete HTJ2K tile packet body. + fn encode_htj2k_tile( + &mut self, + _job: J2kHtj2kTileEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + Ok(None) + } + + /// Return whether native CPU code-block fallback should use internal rayon parallelism. + fn prefer_parallel_cpu_code_block_fallback(&self) -> bool { + false + } + + /// Return whether whole-tile CPU-only batch encode may be parallelized by callers. + fn prefer_parallel_cpu_tile_encode(&self) -> bool { + false + } + + /// Optionally packetize prepared packet contributions. + fn encode_packetization( + &mut self, + _job: J2kPacketizationEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + Ok(None) + } +} + +impl J2kEncodeStageAccelerator for CpuOnlyJ2kEncodeStageAccelerator { + fn prefer_parallel_cpu_code_block_fallback(&self) -> bool { + true + } + + fn prefer_parallel_cpu_tile_encode(&self) -> bool { + true + } +} + +/// Adapter that lets native encoder internals call a public encode-stage accelerator. +/// +/// With the shared `j2k-types` contract this is a direct +/// passthrough: the job, output, and report types on both sides are the same +/// types. +#[doc(hidden)] +pub struct NativeEncodeStageAdapter<'a, A: J2kEncodeStageAccelerator + ?Sized> { + inner: &'a mut A, +} + +impl<'a, A: J2kEncodeStageAccelerator + ?Sized> NativeEncodeStageAdapter<'a, A> { + /// Create an adapter around a public encode-stage accelerator. + pub fn new(inner: &'a mut A) -> Self { + Self { inner } + } +} + +#[doc(hidden)] +impl j2k_native::J2kEncodeStageAccelerator + for NativeEncodeStageAdapter<'_, A> +{ + fn dispatch_report(&self) -> J2kEncodeDispatchReport { + self.inner.dispatch_report() + } + + fn encode_deinterleave( + &mut self, + job: J2kDeinterleaveToF32Job<'_>, + ) -> core::result::Result>>, &'static str> { + self.inner.encode_deinterleave(job) + } + + fn encode_forward_rct( + &mut self, + job: J2kForwardRctJob<'_>, + ) -> core::result::Result { + self.inner.encode_forward_rct(job) + } + + fn encode_forward_ict( + &mut self, + job: J2kForwardIctJob<'_>, + ) -> core::result::Result { + self.inner.encode_forward_ict(job) + } + + fn encode_forward_dwt53( + &mut self, + job: J2kForwardDwt53Job<'_>, + ) -> core::result::Result, &'static str> { + self.inner.encode_forward_dwt53(job) + } + + fn encode_forward_dwt97( + &mut self, + job: J2kForwardDwt97Job<'_>, + ) -> core::result::Result, &'static str> { + self.inner.encode_forward_dwt97(job) + } + + fn encode_quantize_subband( + &mut self, + job: J2kQuantizeSubbandJob<'_>, + ) -> core::result::Result>, &'static str> { + self.inner.encode_quantize_subband(job) + } + + fn encode_tier1_code_block( + &mut self, + job: J2kTier1CodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + self.inner.encode_tier1_code_block(job) + } + + fn encode_tier1_code_blocks( + &mut self, + jobs: &[J2kTier1CodeBlockEncodeJob<'_>], + ) -> core::result::Result>, &'static str> { + self.inner.encode_tier1_code_blocks(jobs) + } + + fn encode_ht_code_block( + &mut self, + job: J2kHtCodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + self.inner.encode_ht_code_block(job) + } + + fn encode_ht_code_blocks( + &mut self, + jobs: &[J2kHtCodeBlockEncodeJob<'_>], + ) -> core::result::Result>, &'static str> { + self.inner.encode_ht_code_blocks(jobs) + } + + fn encode_ht_subband( + &mut self, + job: J2kHtSubbandEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + self.inner.encode_ht_subband(job) + } + + fn encode_htj2k_tile( + &mut self, + job: J2kHtj2kTileEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + self.inner.encode_htj2k_tile(job) + } + + fn prefer_parallel_cpu_code_block_fallback(&self) -> bool { + self.inner.prefer_parallel_cpu_code_block_fallback() + } + + fn prefer_parallel_cpu_tile_encode(&self) -> bool { + self.inner.prefer_parallel_cpu_tile_encode() + } + + fn encode_packetization( + &mut self, + job: J2kPacketizationEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + self.inner.encode_packetization(job) + } +} + +#[cfg(test)] +mod tests { + use super::J2kEncodeStageAccelerator; + use super::{J2kEncodeDispatchReport, NativeEncodeStageAdapter}; + + #[derive(Default)] + struct ReportingAccelerator { + report: J2kEncodeDispatchReport, + } + + impl J2kEncodeStageAccelerator for ReportingAccelerator { + fn dispatch_report(&self) -> J2kEncodeDispatchReport { + self.report + } + } + + #[test] + fn native_adapter_forwards_dispatch_report() { + let mut accelerator = ReportingAccelerator { + report: J2kEncodeDispatchReport { + deinterleave: 1, + forward_rct: 2, + forward_ict: 3, + forward_dwt53: 4, + forward_dwt97: 5, + quantize_subband: 6, + tier1_code_block: 7, + ht_code_block: 8, + packetization: 9, + }, + }; + let adapter = NativeEncodeStageAdapter::new(&mut accelerator); + + let report = j2k_native::J2kEncodeStageAccelerator::dispatch_report(&adapter); + + assert_eq!(report.deinterleave, 1); + assert_eq!(report.forward_rct, 2); + assert_eq!(report.forward_ict, 3); + assert_eq!(report.forward_dwt53, 4); + assert_eq!(report.forward_dwt97, 5); + assert_eq!(report.quantize_subband, 6); + assert_eq!(report.tier1_code_block, 7); + assert_eq!(report.ht_code_block, 8); + assert_eq!(report.packetization, 9); + } +} diff --git a/crates/j2k/src/adapter/mod.rs b/crates/j2k/src/adapter/mod.rs new file mode 100644 index 00000000..432ba015 --- /dev/null +++ b/crates/j2k/src/adapter/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Public adapter-facing JPEG 2000 planning APIs. + +/// Adaptive CPU/device route planning. +pub mod adaptive_route; + +/// Device decode request normalization. +pub mod device_plan; + +/// Encode-stage adapter contracts. +pub mod encode_stage; diff --git a/crates/signinum-j2k/src/backend.rs b/crates/j2k/src/backend.rs similarity index 90% rename from crates/signinum-j2k/src/backend.rs rename to crates/j2k/src/backend.rs index a1cbedae..02b1a445 100644 --- a/crates/signinum-j2k/src/backend.rs +++ b/crates/j2k/src/backend.rs @@ -2,9 +2,9 @@ use crate::J2kError; use alloc::string::ToString; -use signinum_core::{Colorspace, Info}; +use j2k_core::{Colorspace, Info}; -pub(crate) use signinum_j2k_native::{ColorSpace, DecodeSettings, Image, RawBitmap}; +pub(crate) use j2k_native::{ColorSpace, DecodeSettings, DecodedComponents, Image, RawBitmap}; pub(crate) fn image(bytes: &[u8], settings: DecodeSettings) -> Result, J2kError> { Image::new(bytes, &settings).map_err(|err| J2kError::Backend(err.to_string())) diff --git a/crates/j2k/src/batch.rs b/crates/j2k/src/batch.rs new file mode 100644 index 00000000..1f0338b0 --- /dev/null +++ b/crates/j2k/src/batch.rs @@ -0,0 +1,799 @@ +// SPDX-License-Identifier: Apache-2.0 + +use core::convert::Infallible; +use std::num::NonZeroUsize; + +pub use j2k_core::TileBatchOptions; +use j2k_core::{ + collect_indexed_batch_results, tile_batch_worker_count, CompressedTransferSyntax, + DecodeOutcome, DecoderContext, Downscale, IndexedBatchResult, PixelFormat, Rect, + TileBatchDecode, +}; +use j2k_native::{ + execute_direct_color_plan_rgb8_into, execute_direct_color_plan_rgba8_into, + DecodeError as NativeDecodeError, DecodingError as NativeDecodingError, J2kDirectColorPlan, + J2kDirectCpuScratch, J2kRect, +}; + +use crate::backend::{self, DecodeSettings}; +use crate::decode::{validate_buffer, validate_region}; +use crate::parse::parse_image_info; +use crate::{CpuDecodeParallelism, J2kCodec, J2kContext, J2kError, J2kScratchPool}; + +/// One full-tile decode request for [`decode_tiles_into`]. +pub type TileDecodeJob<'i, 'o> = j2k_core::TileDecodeJob<'i, 'o>; + +/// One ROI tile decode request for [`decode_tiles_region_into`]. +pub type TileRegionDecodeJob<'i, 'o> = j2k_core::TileRegionDecodeJob<'i, 'o>; + +/// One scaled tile decode request for [`decode_tiles_scaled_into`]. +pub type TileScaledDecodeJob<'i, 'o> = j2k_core::TileScaledDecodeJob<'i, 'o>; + +/// One ROI+scaled tile decode request for [`decode_tiles_region_scaled_into`]. +pub type TileRegionScaledDecodeJob<'i, 'o> = j2k_core::TileRegionScaledDecodeJob<'i, 'o>; + +/// Error returned by J2K CPU tile batches, annotated with the first failing +/// tile index from the caller's input order. +pub type TileBatchError = j2k_core::TileBatchError; + +type BatchOutcome = DecodeOutcome; +type J2kIndexedBatchResult = IndexedBatchResult; + +/// One-shot parse-plus-decode of an independent J2K/HTJ2K tile into the +/// caller's buffer, reusing both caller-owned [`DecoderContext`] and +/// caller-owned [`J2kScratchPool`]. +pub fn decode_tile_into_in_context( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut J2kScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, +) -> Result { + ::decode_tile(ctx, pool, bytes, out, stride, fmt) +} + +/// One-shot parse-plus-ROI-decode of an independent J2K/HTJ2K tile into the +/// caller's buffer, reusing both caller-owned [`DecoderContext`] and +/// caller-owned [`J2kScratchPool`]. +pub fn decode_tile_region_into_in_context( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut J2kScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, +) -> Result { + ::decode_tile_region(ctx, pool, bytes, out, stride, fmt, roi) +} + +/// One-shot parse-plus-scaled-decode of an independent J2K/HTJ2K tile into the +/// caller's buffer, reusing both caller-owned [`DecoderContext`] and +/// caller-owned [`J2kScratchPool`]. +pub fn decode_tile_scaled_into_in_context( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut J2kScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + scale: Downscale, +) -> Result { + ::decode_tile_scaled(ctx, pool, bytes, out, stride, fmt, scale) +} + +/// One-shot parse-plus-ROI-scaled-decode of an independent J2K/HTJ2K tile +/// into the caller's buffer, reusing both caller-owned [`DecoderContext`] and +/// caller-owned [`J2kScratchPool`]. +#[allow(clippy::too_many_arguments)] +pub fn decode_tile_region_scaled_into_in_context( + bytes: &[u8], + ctx: &mut DecoderContext, + pool: &mut J2kScratchPool, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, + scale: Downscale, +) -> Result { + ::decode_tile_region_scaled( + ctx, pool, bytes, out, stride, fmt, roi, scale, + ) +} + +/// Decode independent J2K/HTJ2K tiles into caller-owned output buffers using +/// a scoped CPU worker pool. +/// +/// Each worker owns one [`DecoderContext`] and one [`J2kScratchPool`]. Returned +/// outcomes preserve caller input order. +pub fn decode_tiles_into( + jobs: &mut [TileDecodeJob<'_, '_>], + fmt: PixelFormat, + options: TileBatchOptions, +) -> Result, TileBatchError> { + if jobs.is_empty() { + return Ok(Vec::new()); + } + + let job_count = jobs.len(); + let worker_count = tile_batch_worker_count(job_count, options, available_tile_batch_workers()); + let chunk_size = job_count.div_ceil(worker_count); + let results = + std::thread::scope(|scope| { + let mut handles = Vec::with_capacity(worker_count); + for (chunk_index, chunk) in jobs.chunks_mut(chunk_size).enumerate() { + let start_index = chunk_index * chunk_size; + let inner_parallelism = inner_parallelism_for_batch(job_count); + handles.push(scope.spawn(move || { + decode_tile_job_chunk(start_index, chunk, fmt, inner_parallelism) + })); + } + + let mut results = Vec::with_capacity(job_count); + for handle in handles { + match handle.join() { + Ok(chunk_results) => results.extend(chunk_results), + Err(payload) => std::panic::resume_unwind(payload), + } + } + results + }); + + collect_indexed_batch_results(job_count, results, |index, source| TileBatchError { + index, + source, + }) +} + +/// Decode independent J2K/HTJ2K tile regions into caller-owned output buffers +/// using a scoped CPU worker pool. +pub fn decode_tiles_region_into( + jobs: &mut [TileRegionDecodeJob<'_, '_>], + fmt: PixelFormat, + options: TileBatchOptions, +) -> Result, TileBatchError> { + if jobs.is_empty() { + return Ok(Vec::new()); + } + + let job_count = jobs.len(); + let worker_count = tile_batch_worker_count(job_count, options, available_tile_batch_workers()); + let chunk_size = job_count.div_ceil(worker_count); + let results = std::thread::scope(|scope| { + let mut handles = Vec::with_capacity(worker_count); + for (chunk_index, chunk) in jobs.chunks_mut(chunk_size).enumerate() { + let start_index = chunk_index * chunk_size; + let inner_parallelism = inner_parallelism_for_batch(job_count); + handles.push(scope.spawn(move || { + decode_tile_region_job_chunk(start_index, chunk, fmt, inner_parallelism) + })); + } + + let mut results = Vec::with_capacity(job_count); + for handle in handles { + match handle.join() { + Ok(chunk_results) => results.extend(chunk_results), + Err(payload) => std::panic::resume_unwind(payload), + } + } + results + }); + + collect_indexed_batch_results(job_count, results, |index, source| TileBatchError { + index, + source, + }) +} + +/// Decode independent J2K/HTJ2K tiles at reduced resolution into caller-owned +/// output buffers using a scoped CPU worker pool. +pub fn decode_tiles_scaled_into( + jobs: &mut [TileScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + options: TileBatchOptions, +) -> Result, TileBatchError> { + if jobs.is_empty() { + return Ok(Vec::new()); + } + + let job_count = jobs.len(); + let worker_count = tile_batch_worker_count(job_count, options, available_tile_batch_workers()); + let chunk_size = job_count.div_ceil(worker_count); + let results = std::thread::scope(|scope| { + let mut handles = Vec::with_capacity(worker_count); + for (chunk_index, chunk) in jobs.chunks_mut(chunk_size).enumerate() { + let start_index = chunk_index * chunk_size; + let inner_parallelism = inner_parallelism_for_batch(job_count); + handles.push(scope.spawn(move || { + decode_tile_scaled_job_chunk(start_index, chunk, fmt, inner_parallelism) + })); + } + + let mut results = Vec::with_capacity(job_count); + for handle in handles { + match handle.join() { + Ok(chunk_results) => results.extend(chunk_results), + Err(payload) => std::panic::resume_unwind(payload), + } + } + results + }); + + collect_indexed_batch_results(job_count, results, |index, source| TileBatchError { + index, + source, + }) +} + +/// Decode independent J2K/HTJ2K tile regions at reduced resolution into +/// caller-owned output buffers using a scoped CPU worker pool. +pub fn decode_tiles_region_scaled_into( + jobs: &mut [TileRegionScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + options: TileBatchOptions, +) -> Result, TileBatchError> { + if jobs.is_empty() { + return Ok(Vec::new()); + } + + let job_count = jobs.len(); + let worker_count = tile_batch_worker_count(job_count, options, available_tile_batch_workers()); + let chunk_size = job_count.div_ceil(worker_count); + let shared_direct_plan = build_repeated_direct_color_region_plan(jobs, fmt) + .map_err(|source| TileBatchError { index: 0, source })?; + let results = std::thread::scope(|scope| { + let mut handles = Vec::with_capacity(worker_count); + for (chunk_index, chunk) in jobs.chunks_mut(chunk_size).enumerate() { + let start_index = chunk_index * chunk_size; + let shared_direct_plan = shared_direct_plan.as_ref(); + handles.push(scope.spawn(move || { + decode_tile_region_scaled_job_chunk( + start_index, + chunk, + fmt, + inner_parallelism_for_batch(job_count), + shared_direct_plan, + ) + })); + } + + let mut results = Vec::with_capacity(job_count); + for handle in handles { + match handle.join() { + Ok(chunk_results) => results.extend(chunk_results), + Err(payload) => std::panic::resume_unwind(payload), + } + } + results + }); + + collect_indexed_batch_results(job_count, results, |index, source| TileBatchError { + index, + source, + }) +} + +fn available_tile_batch_workers() -> usize { + std::thread::available_parallelism().map_or(1, NonZeroUsize::get) +} + +fn inner_parallelism_for_batch(batch_size: usize) -> CpuDecodeParallelism { + if batch_size > 1 { + CpuDecodeParallelism::Serial + } else { + CpuDecodeParallelism::Auto + } +} + +fn decode_tile_job_chunk( + start_index: usize, + jobs: &mut [TileDecodeJob<'_, '_>], + fmt: PixelFormat, + inner_parallelism: CpuDecodeParallelism, +) -> Vec { + let mut ctx = DecoderContext::::new(); + ctx.codec_mut() + .set_cpu_decode_parallelism(inner_parallelism); + let mut pool = J2kScratchPool::new(); + let mut results = Vec::with_capacity(jobs.len()); + for (local_index, job) in jobs.iter_mut().enumerate() { + let outcome = + decode_tile_into_in_context(job.input, &mut ctx, &mut pool, job.out, job.stride, fmt); + results.push((start_index + local_index, outcome)); + } + results +} + +fn decode_tile_region_job_chunk( + start_index: usize, + jobs: &mut [TileRegionDecodeJob<'_, '_>], + fmt: PixelFormat, + inner_parallelism: CpuDecodeParallelism, +) -> Vec { + let mut ctx = DecoderContext::::new(); + ctx.codec_mut() + .set_cpu_decode_parallelism(inner_parallelism); + let mut pool = J2kScratchPool::new(); + let mut results = Vec::with_capacity(jobs.len()); + for (local_index, job) in jobs.iter_mut().enumerate() { + let outcome = decode_tile_region_into_in_context( + job.input, &mut ctx, &mut pool, job.out, job.stride, fmt, job.roi, + ); + results.push((start_index + local_index, outcome)); + } + results +} + +fn decode_tile_scaled_job_chunk( + start_index: usize, + jobs: &mut [TileScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + inner_parallelism: CpuDecodeParallelism, +) -> Vec { + let mut ctx = DecoderContext::::new(); + ctx.codec_mut() + .set_cpu_decode_parallelism(inner_parallelism); + let mut pool = J2kScratchPool::new(); + let mut results = Vec::with_capacity(jobs.len()); + for (local_index, job) in jobs.iter_mut().enumerate() { + let outcome = decode_tile_scaled_into_in_context( + job.input, &mut ctx, &mut pool, job.out, job.stride, fmt, job.scale, + ); + results.push((start_index + local_index, outcome)); + } + results +} + +fn decode_tile_region_scaled_job_chunk( + start_index: usize, + jobs: &mut [TileRegionScaledDecodeJob<'_, '_>], + fmt: PixelFormat, + inner_parallelism: CpuDecodeParallelism, + shared_direct_plan: Option<&DirectColorRegionCache>, +) -> Vec { + let mut ctx = DecoderContext::::new(); + ctx.codec_mut() + .set_cpu_decode_parallelism(inner_parallelism); + let mut pool = J2kScratchPool::new(); + let mut direct_scratch = J2kDirectCpuScratch::new(); + let mut direct_cache = None; + let mut results = Vec::with_capacity(jobs.len()); + for (local_index, job) in jobs.iter_mut().enumerate() { + let outcome = match decode_tile_region_scaled_shared_direct_color_u8_in_context( + job, + &mut ctx, + fmt, + &mut direct_scratch, + shared_direct_plan, + ) + .and_then(|outcome| { + if outcome.is_some() { + Ok(outcome) + } else { + decode_tile_region_scaled_direct_color_u8_in_context( + job, + &mut ctx, + fmt, + &mut direct_scratch, + &mut direct_cache, + ) + } + }) { + Ok(Some(outcome)) => Ok(outcome), + Ok(None) => decode_tile_region_scaled_into_in_context( + job.input, &mut ctx, &mut pool, job.out, job.stride, fmt, job.roi, job.scale, + ), + Err(error) => Err(error), + }; + results.push((start_index + local_index, outcome)); + } + results +} + +struct DirectColorRegionCache { + input_ptr: usize, + input_len: usize, + roi: Rect, + scale: Downscale, + output_region: J2kRect, + plan: J2kDirectColorPlan, +} + +fn build_repeated_direct_color_region_plan( + jobs: &[TileRegionScaledDecodeJob<'_, '_>], + fmt: PixelFormat, +) -> Result, J2kError> { + if !is_direct_color_u8_format(fmt) { + return Ok(None); + } + let Some(first) = jobs.first() else { + return Ok(None); + }; + if first.scale == Downscale::None { + return Ok(None); + } + let key = DirectColorRegionKey { + input_ptr: first.input.as_ptr() as usize, + input_len: first.input.len(), + roi: first.roi, + scale: first.scale, + }; + if !jobs.iter().all(|job| { + job.input.as_ptr() as usize == key.input_ptr + && job.input.len() == key.input_len + && job.roi == key.roi + && job.scale == key.scale + }) { + return Ok(None); + } + + let Some((plan, output_region)) = + build_direct_color_region_plan(first.input, first.roi, first.scale)? + else { + return Ok(None); + }; + Ok(Some(DirectColorRegionCache { + input_ptr: key.input_ptr, + input_len: key.input_len, + roi: key.roi, + scale: key.scale, + output_region, + plan, + })) +} + +fn decode_tile_region_scaled_shared_direct_color_u8_in_context( + job: &mut TileRegionScaledDecodeJob<'_, '_>, + _ctx: &mut DecoderContext, + fmt: PixelFormat, + scratch: &mut J2kDirectCpuScratch, + shared_direct_plan: Option<&DirectColorRegionCache>, +) -> Result, J2kError> { + let Some(shared_direct_plan) = shared_direct_plan else { + return Ok(None); + }; + if !is_direct_color_u8_format(fmt) + || !shared_direct_plan.matches(DirectColorRegionKey { + input_ptr: job.input.as_ptr() as usize, + input_len: job.input.len(), + roi: job.roi, + scale: job.scale, + }) + { + return Ok(None); + } + + let decoded = job.roi.scaled_covering(job.scale); + validate_buffer((decoded.w, decoded.h), job.out.len(), job.stride, fmt)?; + execute_direct_color_plan_u8_into( + &shared_direct_plan.plan, + shared_direct_plan.output_region, + scratch, + job.out, + job.stride, + fmt, + )?; + Ok(Some(DecodeOutcome::new(decoded, Vec::new()))) +} + +fn decode_tile_region_scaled_direct_color_u8_in_context( + job: &mut TileRegionScaledDecodeJob<'_, '_>, + _ctx: &mut DecoderContext, + fmt: PixelFormat, + scratch: &mut J2kDirectCpuScratch, + cache: &mut Option, +) -> Result, J2kError> { + if !is_direct_color_u8_format(fmt) || job.scale == Downscale::None { + return Ok(None); + } + + let decoded = job.roi.scaled_covering(job.scale); + validate_buffer((decoded.w, decoded.h), job.out.len(), job.stride, fmt)?; + let key = DirectColorRegionKey { + input_ptr: job.input.as_ptr() as usize, + input_len: job.input.len(), + roi: job.roi, + scale: job.scale, + }; + if !cache.as_ref().is_some_and(|cache| cache.matches(key)) { + let Some((plan, output_region)) = + build_direct_color_region_plan(job.input, job.roi, job.scale)? + else { + return Ok(None); + }; + *cache = Some(DirectColorRegionCache { + input_ptr: key.input_ptr, + input_len: key.input_len, + roi: key.roi, + scale: key.scale, + output_region, + plan, + }); + } + + let cache = cache + .as_ref() + .ok_or_else(|| J2kError::Backend("internal direct color plan cache missing".to_string()))?; + execute_direct_color_plan_u8_into( + &cache.plan, + cache.output_region, + scratch, + job.out, + job.stride, + fmt, + )?; + Ok(Some(DecodeOutcome::new(decoded, Vec::new()))) +} + +#[derive(Clone, Copy)] +struct DirectColorRegionKey { + input_ptr: usize, + input_len: usize, + roi: Rect, + scale: Downscale, +} + +impl DirectColorRegionCache { + fn matches(&self, key: DirectColorRegionKey) -> bool { + self.input_ptr == key.input_ptr + && self.input_len == key.input_len + && self.roi == key.roi + && self.scale == key.scale + } +} + +fn is_direct_color_u8_format(fmt: PixelFormat) -> bool { + matches!(fmt, PixelFormat::Rgb8 | PixelFormat::Rgba8) +} + +fn execute_direct_color_plan_u8_into( + plan: &J2kDirectColorPlan, + output_region: J2kRect, + scratch: &mut J2kDirectCpuScratch, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, +) -> Result<(), J2kError> { + match fmt { + PixelFormat::Rgb8 => { + execute_direct_color_plan_rgb8_into(plan, output_region, scratch, out, stride) + } + PixelFormat::Rgba8 => { + execute_direct_color_plan_rgba8_into(plan, output_region, scratch, out, stride) + } + _ => unreachable!("validated direct color output format"), + } + .map_err(|error| J2kError::Backend(error.to_string())) +} + +fn build_direct_color_region_plan( + input: &[u8], + roi: Rect, + scale: Downscale, +) -> Result, J2kError> { + if !input_declares_htj2k(input) { + return Ok(None); + } + + let Ok(parsed) = parse_image_info(input) else { + return Ok(None); + }; + if !matches!( + parsed.transfer_syntax, + CompressedTransferSyntax::HtJpeg2000Lossless | CompressedTransferSyntax::HtJpeg2000Lossy + ) { + return Ok(None); + } + + validate_region(roi, parsed.info.dimensions)?; + let target_dims = ( + parsed.info.dimensions.0.div_ceil(scale.denominator()), + parsed.info.dimensions.1.div_ceil(scale.denominator()), + ); + let output_region = roi.scaled_covering(scale); + let image = backend::image( + input, + DecodeSettings { + target_resolution: Some(target_dims), + ..DecodeSettings::default() + }, + )?; + validate_region(output_region, (image.width(), image.height()))?; + + let mut native_context = j2k_native::DecoderContext::default(); + match image.build_direct_color_plan_region_with_context( + &mut native_context, + ( + output_region.x, + output_region.y, + output_region.w, + output_region.h, + ), + ) { + Ok(plan) if direct_color_plan_uses_only_htj2k(&plan) => Ok(Some(( + plan, + J2kRect { + x0: output_region.x, + y0: output_region.y, + x1: output_region.x + output_region.w, + y1: output_region.y + output_region.h, + }, + ))), + Ok(_) => Ok(None), + Err(error) if is_unsupported_direct_color_plan_error(error) => Ok(None), + Err(error) => Err(J2kError::Backend(error.to_string())), + } +} + +fn input_declares_htj2k(input: &[u8]) -> bool { + const JP2_SIGNATURE_PREFIX: [u8; 8] = [0, 0, 0, 12, b'j', b'P', b' ', b' ']; + + if raw_codestream_declares_htj2k(input) { + return true; + } + if !input.starts_with(&JP2_SIGNATURE_PREFIX) { + return false; + } + jp2_codestream_declares_htj2k(input) +} + +fn jp2_codestream_declares_htj2k(input: &[u8]) -> bool { + let mut offset = 0usize; + while offset < input.len() { + let Some((box_type, payload_start, end)) = read_jp2_box_header(input, offset) else { + return false; + }; + if end > input.len() { + return false; + } + if &box_type == b"jp2c" { + return raw_codestream_declares_htj2k(&input[payload_start..end]); + } + offset = end; + } + false +} + +fn read_jp2_box_header(input: &[u8], offset: usize) -> Option<([u8; 4], usize, usize)> { + let header = input.get(offset..offset.checked_add(8)?)?; + let lbox = u32::from_be_bytes([header[0], header[1], header[2], header[3]]); + let box_type = [header[4], header[5], header[6], header[7]]; + match lbox { + 0 => Some((box_type, offset.checked_add(8)?, input.len())), + 1 => { + let extended = input.get(offset.checked_add(8)?..offset.checked_add(16)?)?; + let xlbox = u64::from_be_bytes([ + extended[0], + extended[1], + extended[2], + extended[3], + extended[4], + extended[5], + extended[6], + extended[7], + ]); + if xlbox < 16 || xlbox > usize::MAX as u64 { + return None; + } + let end = offset.checked_add(xlbox as usize)?; + Some((box_type, offset.checked_add(16)?, end)) + } + length if length < 8 => None, + length => { + let end = offset.checked_add(length as usize)?; + Some((box_type, offset.checked_add(8)?, end)) + } + } +} + +fn raw_codestream_declares_htj2k(input: &[u8]) -> bool { + const MARKER_SOC: u8 = 0x4f; + const MARKER_CAP: u8 = 0x50; + const MARKER_COD: u8 = 0x52; + const MARKER_SOT: u8 = 0x90; + const MARKER_SOD: u8 = 0x93; + const MARKER_EOC: u8 = 0xd9; + + if input.len() < 2 || input[0] != 0xff || input[1] != MARKER_SOC { + return false; + } + + let mut offset = 2usize; + while offset + 2 <= input.len() { + if input[offset] != 0xff { + return false; + } + let marker = input[offset + 1]; + offset += 2; + match marker { + MARKER_SOT | MARKER_SOD | MARKER_EOC => return false, + MARKER_CAP => return true, + MARKER_COD => { + let Some(payload) = read_codestream_segment_payload(input, &mut offset) else { + return false; + }; + if payload.get(8).is_some_and(|style| style & 0x40 != 0) { + return true; + } + } + _ => { + if read_codestream_segment_payload(input, &mut offset).is_none() { + return false; + } + } + } + } + false +} + +fn read_codestream_segment_payload<'a>(input: &'a [u8], offset: &mut usize) -> Option<&'a [u8]> { + let len_bytes = input.get(*offset..offset.checked_add(2)?)?; + let len = u16::from_be_bytes([len_bytes[0], len_bytes[1]]) as usize; + if len < 2 { + return None; + } + let payload_start = offset.checked_add(2)?; + let payload_end = offset.checked_add(len)?; + let payload = input.get(payload_start..payload_end)?; + *offset = payload_end; + Some(payload) +} + +fn direct_color_plan_uses_only_htj2k(plan: &J2kDirectColorPlan) -> bool { + plan.component_plans.iter().all(|component| { + component.steps.iter().any(|step| { + matches!( + step, + j2k_native::J2kDirectGrayscaleStep::HtSubBand(sub_band) + if !sub_band.jobs.is_empty() + ) + }) && component + .steps + .iter() + .all(|step| !matches!(step, j2k_native::J2kDirectGrayscaleStep::ClassicSubBand(_))) + }) +} + +fn is_unsupported_direct_color_plan_error(error: NativeDecodeError) -> bool { + matches!( + error, + NativeDecodeError::Decoding(NativeDecodingError::UnsupportedFeature(_)) + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use j2k_native::{encode, encode_htj2k, EncodeOptions}; + use j2k_test_support::wrap_codestream_jp2; + + fn encode_rgb_codestream(htj2k: bool) -> Vec { + let pixels = (0..16 * 16 * 3) + .map(|idx| ((idx * 11 + idx / 3) & 0xff) as u8) + .collect::>(); + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + if htj2k { + encode_htj2k(&pixels, 16, 16, 3, 8, false, &options).expect("encode HTJ2K") + } else { + encode(&pixels, 16, 16, 3, 8, false, &options).expect("encode J2K") + } + } + + #[test] + fn htj2k_eligibility_accepts_raw_and_jp2_wrapped_inputs() { + let raw_htj2k = encode_rgb_codestream(true); + let jp2_htj2k = wrap_codestream_jp2(&raw_htj2k, 16, 16, 3, 8, 16); + let raw_classic = encode_rgb_codestream(false); + let jp2_classic = wrap_codestream_jp2(&raw_classic, 16, 16, 3, 8, 16); + + assert!(input_declares_htj2k(&raw_htj2k)); + assert!(input_declares_htj2k(&jp2_htj2k)); + assert!(!input_declares_htj2k(&raw_classic)); + assert!(!input_declares_htj2k(&jp2_classic)); + } +} diff --git a/crates/j2k/src/context.rs b/crates/j2k/src/context.rs new file mode 100644 index 00000000..45ff03f7 --- /dev/null +++ b/crates/j2k/src/context.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_core::{CacheStats, CodecContext}; + +use crate::CpuDecodeParallelism; + +/// Reusable JPEG 2000 decode context and cache state. +#[derive(Debug, Default, Clone)] +pub struct J2kContext { + cpu_decode_parallelism: CpuDecodeParallelism, +} + +impl J2kContext { + /// Create an empty JPEG 2000 context. + pub const fn new() -> Self { + Self { + cpu_decode_parallelism: CpuDecodeParallelism::Auto, + } + } + + /// Return the CPU decode parallelism policy. + pub fn cpu_decode_parallelism(&self) -> CpuDecodeParallelism { + self.cpu_decode_parallelism + } + + /// Set the CPU decode parallelism policy. + pub fn set_cpu_decode_parallelism(&mut self, parallelism: CpuDecodeParallelism) { + self.cpu_decode_parallelism = parallelism; + } +} + +impl CodecContext for J2kContext { + fn clear(&mut self) { + self.cpu_decode_parallelism = CpuDecodeParallelism::Auto; + } + + fn cache_stats(&self) -> CacheStats { + CacheStats::default() + } +} diff --git a/crates/j2k/src/decode.rs b/crates/j2k/src/decode.rs new file mode 100644 index 00000000..8972afe8 --- /dev/null +++ b/crates/j2k/src/decode.rs @@ -0,0 +1,636 @@ +// SPDX-License-Identifier: Apache-2.0 + +use crate::backend::{ColorSpace, DecodedComponents, Image, RawBitmap}; +use crate::J2kError; +use alloc::string::ToString; +use core::convert::Infallible; +use j2k_core::{validate_strided_output_buffer, DecodeOutcome, PixelFormat, Rect, Unsupported}; +pub(crate) type J2kDecodeOutcome = DecodeOutcome; + +pub(crate) fn decode_image_into_with_native_context<'a>( + image: &Image<'a>, + native_context: &mut j2k_native::DecoderContext<'a>, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, +) -> Result<(), J2kError> { + let dims = (image.width(), image.height()); + match fmt { + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => { + if can_decode_u8_directly(image.color_space(), image.has_alpha(), dims, stride, fmt) { + image + .decode_into(out, native_context) + .map_err(|err| J2kError::Backend(err.to_string()))?; + return Ok(()); + } + let decoded = image + .decode_with_context(native_context) + .map_err(|err| J2kError::Backend(err.to_string()))?; + write_u8_output( + image.color_space(), + image.has_alpha(), + dims, + &decoded.data, + out, + stride, + fmt, + ) + } + PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16 => { + let raw = image + .decode_native_with_context(native_context) + .map_err(|err| J2kError::Backend(err.to_string()))?; + write_u16_output( + image.color_space(), + image.has_alpha(), + &raw, + out, + stride, + fmt, + ) + } + _ => Err(Unsupported { + what: "pixel format is not yet supported by j2k", + } + .into()), + } +} + +fn can_decode_u8_directly( + color_space: &ColorSpace, + has_alpha: bool, + dims: (u32, u32), + stride: usize, + fmt: PixelFormat, +) -> bool { + let width = dims.0 as usize; + match (color_space, has_alpha, fmt) { + (ColorSpace::RGB, false, PixelFormat::Rgb8) => stride == width * 3, + (ColorSpace::RGB, true, PixelFormat::Rgba8) => stride == width * 4, + (ColorSpace::Gray, false, PixelFormat::Gray8) => stride == width, + _ => false, + } +} + +pub(crate) fn decode_image_region_into_with_native_context<'a>( + image: &Image<'a>, + native_context: &mut j2k_native::DecoderContext<'a>, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, + roi: Rect, +) -> Result<(), J2kError> { + match fmt { + PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => { + let components = image + .decode_region_components_with_context((roi.x, roi.y, roi.w, roi.h), native_context) + .map_err(|err| J2kError::Backend(err.to_string()))?; + write_components_u8_output(&components, out, stride, fmt) + } + PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16 => { + let raw = image + .decode_native_region_with_context((roi.x, roi.y, roi.w, roi.h), native_context) + .map_err(|err| J2kError::Backend(err.to_string()))?; + write_u16_output( + image.color_space(), + image.has_alpha(), + &raw, + out, + stride, + fmt, + ) + } + _ => Err(Unsupported { + what: "pixel format is not yet supported by j2k", + } + .into()), + } +} + +fn write_components_u8_output( + components: &DecodedComponents<'_>, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, +) -> Result<(), J2kError> { + let dims = components.dimensions(); + let expected_samples = component_sample_count(dims)?; + let width = dims.0 as usize; + let height = dims.1 as usize; + let planes = components.planes(); + match ( + components.color_space(), + components.has_alpha(), + planes.len(), + fmt, + ) { + (ColorSpace::Gray, false, 1, PixelFormat::Gray8) => { + validate_component_planes(&planes[..1], expected_samples)?; + write_component_rows_u8(&planes[0], out, stride, width, height); + Ok(()) + } + (ColorSpace::RGB, false, 3, PixelFormat::Rgb8) + | (ColorSpace::RGB, true, 4, PixelFormat::Rgb8) => { + validate_component_planes(&planes[..3], expected_samples)?; + write_rgb_component_rows_u8(planes, out, stride, width, height); + Ok(()) + } + (ColorSpace::RGB, false, 3, PixelFormat::Rgba8) => { + validate_component_planes(&planes[..3], expected_samples)?; + write_rgba_component_rows_u8(planes, out, stride, width, height, true); + Ok(()) + } + (ColorSpace::RGB, true, 4, PixelFormat::Rgba8) => { + validate_component_planes(&planes[..4], expected_samples)?; + write_rgba_component_rows_u8(planes, out, stride, width, height, false); + Ok(()) + } + _ => Err(Unsupported { + what: "backend color space cannot be mapped to requested 8-bit pixel format", + } + .into()), + } +} + +fn component_sample_count(dims: (u32, u32)) -> Result { + (dims.0 as usize) + .checked_mul(dims.1 as usize) + .ok_or(J2kError::DimensionOverflow { + width: dims.0, + height: dims.1, + }) +} + +fn validate_component_planes( + planes: &[j2k_native::ComponentPlane<'_>], + expected_samples: usize, +) -> Result<(), J2kError> { + for (index, plane) in planes.iter().enumerate() { + let samples = plane.samples().len(); + if samples < expected_samples { + return Err(J2kError::Backend(format!( + "backend component plane {index} has {samples} samples, expected at least {expected_samples}" + ))); + } + } + Ok(()) +} + +fn write_component_rows_u8( + plane: &j2k_native::ComponentPlane<'_>, + out: &mut [u8], + stride: usize, + width: usize, + height: usize, +) { + for y in 0..height { + let src = &plane.samples()[y * width..(y + 1) * width]; + let dst = &mut out[y * stride..y * stride + width]; + write_samples_as_u8(src, plane.bit_depth(), dst); + } +} + +fn write_rgb_component_rows_u8( + planes: &[j2k_native::ComponentPlane<'_>], + out: &mut [u8], + stride: usize, + width: usize, + height: usize, +) { + for y in 0..height { + let row = y * width; + let dst = &mut out[y * stride..y * stride + width * 3]; + for x in 0..width { + let dst = &mut dst[x * 3..x * 3 + 3]; + for channel in 0..3 { + dst[channel] = sample_as_u8( + planes[channel].samples()[row + x], + planes[channel].bit_depth(), + ); + } + } + } +} + +fn write_rgba_component_rows_u8( + planes: &[j2k_native::ComponentPlane<'_>], + out: &mut [u8], + stride: usize, + width: usize, + height: usize, + synthesize_alpha: bool, +) { + for y in 0..height { + let row = y * width; + let dst = &mut out[y * stride..y * stride + width * 4]; + for x in 0..width { + let dst = &mut dst[x * 4..x * 4 + 4]; + for channel in 0..3 { + dst[channel] = sample_as_u8( + planes[channel].samples()[row + x], + planes[channel].bit_depth(), + ); + } + dst[3] = if synthesize_alpha { + u8::MAX + } else { + sample_as_u8(planes[3].samples()[row + x], planes[3].bit_depth()) + }; + } + } +} + +fn write_samples_as_u8(src: &[f32], bit_depth: u8, dst: &mut [u8]) { + for (sample, dst) in src.iter().zip(dst.iter_mut()) { + *dst = sample_as_u8(*sample, bit_depth); + } +} + +fn sample_as_u8(sample: f32, bit_depth: u8) -> u8 { + let rounded = sample.round(); + if bit_depth == 8 { + return rounded.clamp(0.0, f32::from(u8::MAX)) as u8; + } + let max_value = if bit_depth >= 16 { + f32::from(u16::MAX) + } else { + f32::from(((1_u16 << bit_depth) - 1).max(1)) + }; + ((rounded.clamp(0.0, max_value) / max_value) * f32::from(u8::MAX)).round() as u8 +} + +pub(crate) fn validate_buffer( + dims: (u32, u32), + out_len: usize, + stride: usize, + fmt: PixelFormat, +) -> Result<(), J2kError> { + validate_strided_output_buffer(dims, out_len, stride, fmt).map_err(Into::into) +} + +pub(crate) fn validate_region(roi: Rect, dims: (u32, u32)) -> Result<(), J2kError> { + if roi.is_within(dims) { + return Ok(()); + } + Err(J2kError::InvalidRegion { + x: roi.x, + y: roi.y, + w: roi.w, + h: roi.h, + image_w: dims.0, + image_h: dims.1, + }) +} + +fn write_u8_output( + color_space: &ColorSpace, + has_alpha: bool, + dims: (u32, u32), + decoded: &[u8], + out: &mut [u8], + stride: usize, + fmt: PixelFormat, +) -> Result<(), J2kError> { + let width = dims.0 as usize; + let height = dims.1 as usize; + match (color_space, has_alpha, fmt) { + (ColorSpace::RGB, false, PixelFormat::Rgb8) => { + copy_rows_exact(decoded, out, stride, width * 3, height); + Ok(()) + } + (ColorSpace::RGB, true, PixelFormat::Rgb8) => { + drop_alpha_u8(decoded, out, stride, width, height); + Ok(()) + } + (ColorSpace::RGB, false, PixelFormat::Rgba8) => { + add_opaque_alpha_u8(decoded, out, stride, width, height); + Ok(()) + } + (ColorSpace::RGB, true, PixelFormat::Rgba8) => { + copy_rows_exact(decoded, out, stride, width * 4, height); + Ok(()) + } + (ColorSpace::Gray, false, PixelFormat::Gray8) => { + copy_rows_exact(decoded, out, stride, width, height); + Ok(()) + } + _ => Err(Unsupported { + what: "backend color space cannot be mapped to requested 8-bit pixel format", + } + .into()), + } +} + +fn write_u16_output( + color_space: &ColorSpace, + has_alpha: bool, + raw: &RawBitmap, + out: &mut [u8], + stride: usize, + fmt: PixelFormat, +) -> Result<(), J2kError> { + let width = raw.width as usize; + let height = raw.height as usize; + match (color_space, has_alpha, raw.num_components, fmt) { + (ColorSpace::RGB, false, 3, PixelFormat::Rgb16) => { + convert_or_copy_u16( + &raw.data, + raw.bytes_per_sample, + raw.bit_depth, + 3, + out, + stride, + (width, height), + ); + Ok(()) + } + (ColorSpace::RGB, true, 4, PixelFormat::Rgb16) => { + write_u16_channel_rows(U16ChannelRows { + src: &raw.data, + bytes_per_sample: raw.bytes_per_sample, + bit_depth: raw.bit_depth, + source_channels: 4, + layout: U16ChannelLayout::Drop, + out, + stride, + dims: (width, height), + }); + Ok(()) + } + (ColorSpace::RGB, false, 3, PixelFormat::Rgba16) => { + write_u16_channel_rows(U16ChannelRows { + src: &raw.data, + bytes_per_sample: raw.bytes_per_sample, + bit_depth: raw.bit_depth, + source_channels: 3, + layout: U16ChannelLayout::Synthesize, + out, + stride, + dims: (width, height), + }); + Ok(()) + } + (ColorSpace::RGB, true, 4, PixelFormat::Rgba16) => { + write_u16_channel_rows(U16ChannelRows { + src: &raw.data, + bytes_per_sample: raw.bytes_per_sample, + bit_depth: raw.bit_depth, + source_channels: 4, + layout: U16ChannelLayout::Preserve, + out, + stride, + dims: (width, height), + }); + Ok(()) + } + (ColorSpace::Gray, false, 1, PixelFormat::Gray16) => { + convert_or_copy_u16( + &raw.data, + raw.bytes_per_sample, + raw.bit_depth, + 1, + out, + stride, + (width, height), + ); + Ok(()) + } + _ => Err(Unsupported { + what: "backend color space cannot be mapped to requested 16-bit pixel format", + } + .into()), + } +} + +#[derive(Debug, Clone, Copy)] +enum U16ChannelLayout { + Drop, + Synthesize, + Preserve, +} + +struct U16ChannelRows<'src, 'out> { + src: &'src [u8], + bytes_per_sample: u8, + bit_depth: u8, + source_channels: usize, + layout: U16ChannelLayout, + out: &'out mut [u8], + stride: usize, + dims: (usize, usize), +} + +fn copy_rows_exact(src: &[u8], out: &mut [u8], stride: usize, row_bytes: usize, height: usize) { + for (src_row, dst_row) in src + .chunks_exact(row_bytes) + .zip(out.chunks_exact_mut(stride)) + .take(height) + { + dst_row[..row_bytes].copy_from_slice(src_row); + } +} + +fn add_opaque_alpha_u8(src: &[u8], out: &mut [u8], stride: usize, width: usize, height: usize) { + let src_row_bytes = width * 3; + let dst_row_bytes = width * 4; + for (src_row, dst_row) in src + .chunks_exact(src_row_bytes) + .zip(out.chunks_exact_mut(stride)) + .take(height) + { + for (rgb, rgba) in src_row + .chunks_exact(3) + .zip(dst_row[..dst_row_bytes].chunks_exact_mut(4)) + { + rgba[..3].copy_from_slice(rgb); + rgba[3] = u8::MAX; + } + } +} + +fn drop_alpha_u8(src: &[u8], out: &mut [u8], stride: usize, width: usize, height: usize) { + let src_row_bytes = width * 4; + let dst_row_bytes = width * 3; + for (src_row, dst_row) in src + .chunks_exact(src_row_bytes) + .zip(out.chunks_exact_mut(stride)) + .take(height) + { + for (rgba, rgb) in src_row + .chunks_exact(4) + .zip(dst_row[..dst_row_bytes].chunks_exact_mut(3)) + { + rgb.copy_from_slice(&rgba[..3]); + } + } +} + +fn write_u16_channel_rows(job: U16ChannelRows<'_, '_>) { + let U16ChannelRows { + src, + bytes_per_sample, + bit_depth, + source_channels, + layout, + out, + stride, + dims, + } = job; + let (width, height) = dims; + let dst_channels = match layout { + U16ChannelLayout::Drop => 3, + U16ChannelLayout::Synthesize | U16ChannelLayout::Preserve => 4, + }; + let bytes_per_sample = usize::from(bytes_per_sample); + let src_row_bytes = width * source_channels * bytes_per_sample; + let dst_row_bytes = width * dst_channels * 2; + let alpha = opaque_alpha_u16(bytes_per_sample, bit_depth); + + for (src_row, dst_row) in src + .chunks_exact(src_row_bytes) + .zip(out.chunks_exact_mut(stride)) + .take(height) + { + let dst_row = &mut dst_row[..dst_row_bytes]; + for x in 0..width { + let src_pixel = &src_row[x * source_channels * bytes_per_sample..]; + let dst_pixel = &mut dst_row[x * dst_channels * 2..(x + 1) * dst_channels * 2]; + for channel in 0..3 { + let sample = output_u16_sample(src_pixel, channel, bytes_per_sample, bit_depth); + dst_pixel[channel * 2..channel * 2 + 2].copy_from_slice(&sample.to_le_bytes()); + } + match layout { + U16ChannelLayout::Drop => {} + U16ChannelLayout::Synthesize => { + dst_pixel[6..8].copy_from_slice(&alpha.to_le_bytes()); + } + U16ChannelLayout::Preserve => { + let sample = output_u16_sample(src_pixel, 3, bytes_per_sample, bit_depth); + dst_pixel[6..8].copy_from_slice(&sample.to_le_bytes()); + } + } + } + } +} + +fn opaque_alpha_u16(bytes_per_sample: usize, bit_depth: u8) -> u16 { + if bytes_per_sample == 1 { + u16::MAX + } else { + ((1_u32 << bit_depth.min(16)) - 1).max(1) as u16 + } +} + +fn output_u16_sample( + src_pixel: &[u8], + channel: usize, + bytes_per_sample: usize, + bit_depth: u8, +) -> u16 { + let offset = channel * bytes_per_sample; + if bytes_per_sample == 2 { + return u16::from_le_bytes([src_pixel[offset], src_pixel[offset + 1]]); + } + widen_u8_sample_to_u16(src_pixel[offset], bit_depth) +} + +fn widen_u8_sample_to_u16(sample: u8, bit_depth: u8) -> u16 { + let max_value = ((1_u32 << bit_depth.min(16)) - 1).max(1); + ((u32::from(sample) * u32::from(u16::MAX) + (max_value / 2)) / max_value) as u16 +} + +fn convert_or_copy_u16( + src: &[u8], + bytes_per_sample: u8, + bit_depth: u8, + channels: usize, + out: &mut [u8], + stride: usize, + dims: (usize, usize), +) { + let (width, height) = dims; + let dst_row_bytes = width * channels * 2; + let src_row_bytes = width * channels * usize::from(bytes_per_sample); + for (src_row, dst_row) in src + .chunks_exact(src_row_bytes) + .zip(out.chunks_exact_mut(stride)) + .take(height) + { + let dst_row = &mut dst_row[..dst_row_bytes]; + if bytes_per_sample == 2 { + dst_row.copy_from_slice(src_row); + continue; + } + for (sample, dst_sample) in src_row.iter().zip(dst_row.chunks_exact_mut(2)) { + let widened = widen_u8_sample_to_u16(*sample, bit_depth); + dst_sample.copy_from_slice(&widened.to_le_bytes()); + } + } +} + +#[cfg(test)] +mod tests { + #[cfg(target_pointer_width = "32")] + use super::J2kError; + use super::{can_decode_u8_directly, component_sample_count, ColorSpace, PixelFormat}; + + #[test] + fn direct_u8_decode_accepts_exact_rgb_and_gray_layouts() { + assert!(can_decode_u8_directly( + &ColorSpace::RGB, + false, + (128, 64), + 128 * 3, + PixelFormat::Rgb8 + )); + assert!(can_decode_u8_directly( + &ColorSpace::Gray, + false, + (128, 64), + 128, + PixelFormat::Gray8 + )); + } + + #[test] + fn direct_u8_decode_rejects_format_conversion_and_padded_stride() { + assert!(!can_decode_u8_directly( + &ColorSpace::RGB, + false, + (128, 64), + 128 * 4, + PixelFormat::Rgba8 + )); + assert!(!can_decode_u8_directly( + &ColorSpace::RGB, + true, + (128, 64), + 128 * 3, + PixelFormat::Rgb8 + )); + assert!(!can_decode_u8_directly( + &ColorSpace::Gray, + false, + (128, 64), + 160, + PixelFormat::Gray8 + )); + } + + #[test] + fn component_sample_count_matches_image_dimensions() { + assert_eq!(component_sample_count((16, 8)).expect("sample count"), 128); + } + + #[cfg(target_pointer_width = "32")] + #[test] + fn component_sample_count_reports_dimension_overflow() { + assert!(matches!( + component_sample_count((u32::MAX, u32::MAX)), + Err(J2kError::DimensionOverflow { + width: u32::MAX, + height: u32::MAX + }) + )); + } +} diff --git a/crates/j2k/src/encode.rs b/crates/j2k/src/encode.rs new file mode 100644 index 00000000..1624bc71 --- /dev/null +++ b/crates/j2k/src/encode.rs @@ -0,0 +1,1690 @@ +// SPDX-License-Identifier: Apache-2.0 + +use alloc::vec::Vec; + +use j2k_core::{BackendKind, Unsupported}; +use j2k_native::{DecodeSettings, EncodeOptions, EncodeProgressionOrder, Image}; + +use crate::{ + adapter::encode_stage::{ + J2kEncodeDispatchReport, J2kEncodeStageAccelerator, NativeEncodeStageAdapter, + }, + J2kError, +}; + +/// Backend preference for JPEG 2000 lossless encoding. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum EncodeBackendPreference { + /// Pick the fastest safe backend exposed by the caller, falling back to CPU. + #[default] + Auto, + /// Require the pure Rust CPU encoder. + CpuOnly, + /// Require a device encoder and fail if unavailable or unsupported. + RequireDevice, +} + +/// Supported JPEG 2000 progression orders for the lossless encode facade. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum J2kProgressionOrder { + /// Layer-resolution-component-position progression. + #[default] + Lrcp, + /// Resolution-layer-component-position progression. + Rlcp, + /// Resolution-position-component-layer progression. + Rpcl, + /// Position-component-resolution-layer progression. + Pcrl, + /// Component-position-resolution-layer progression. + Cprl, +} + +/// Supported code-block coding modes for the lossless encode facade. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum J2kBlockCodingMode { + /// Classic JPEG 2000 Part 1 EBCOT block coding. + #[default] + Classic, + /// High-throughput JPEG 2000 Part 15 block coding. + HighThroughput, +} + +/// Reversible transform profile for lossless JPEG 2000 output. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum ReversibleTransform { + /// Reversible color transform with 5/3 wavelet transform. + #[default] + Rct53, + /// No color transform with 5/3 wavelet transform. + None53, +} + +/// Validation policy for the lossless encode facade. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum J2kEncodeValidation { + /// Decode the produced codestream with the native CPU decoder and compare + /// decoded samples before returning. + #[default] + CpuRoundTrip, + /// Skip facade validation because the caller performs equivalent external + /// validation, for example by decoding on a device backend. + External, +} + +/// Options controlling JPEG 2000 lossless encoding. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub struct J2kLosslessEncodeOptions { + /// Backend preference for encode stages. + pub backend: EncodeBackendPreference, + /// Code-block coding mode for the codestream. + pub block_coding_mode: J2kBlockCodingMode, + /// Packet progression order. + pub progression: J2kProgressionOrder, + /// Optional explicit lossless decomposition level request. + /// + /// Requests are clamped to the geometry-safe maximum for the tile. + pub max_decomposition_levels: Option, + /// Reversible transform profile. + pub reversible_transform: ReversibleTransform, + /// Validation policy applied before returning encoded bytes. + pub validation: J2kEncodeValidation, +} + +impl Default for J2kLosslessEncodeOptions { + fn default() -> Self { + Self { + backend: EncodeBackendPreference::Auto, + block_coding_mode: J2kBlockCodingMode::Classic, + progression: J2kProgressionOrder::Lrcp, + max_decomposition_levels: None, + reversible_transform: ReversibleTransform::Rct53, + validation: J2kEncodeValidation::CpuRoundTrip, + } + } +} + +impl J2kLosslessEncodeOptions { + /// Create JPEG 2000 lossless encode options. + pub const fn new( + backend: EncodeBackendPreference, + block_coding_mode: J2kBlockCodingMode, + progression: J2kProgressionOrder, + max_decomposition_levels: Option, + reversible_transform: ReversibleTransform, + validation: J2kEncodeValidation, + ) -> Self { + Self { + backend, + block_coding_mode, + progression, + max_decomposition_levels, + reversible_transform, + validation, + } + } + + /// Return options with a different backend preference. + #[must_use] + pub const fn with_backend(mut self, backend: EncodeBackendPreference) -> Self { + self.backend = backend; + self + } + + /// Return options using adaptive accelerated routing. + #[must_use] + pub const fn with_accelerated_backend(self) -> Self { + self.with_backend(EncodeBackendPreference::Auto) + } + + /// Return options using the portable CPU route. + #[must_use] + pub const fn with_cpu_only_backend(self) -> Self { + self.with_backend(EncodeBackendPreference::CpuOnly) + } + + /// Return options requiring a strict device route. + #[must_use] + pub const fn with_strict_device_backend(self) -> Self { + self.with_backend(EncodeBackendPreference::RequireDevice) + } + + /// Return options with a different code-block coding mode. + #[must_use] + pub const fn with_block_coding_mode(mut self, block_coding_mode: J2kBlockCodingMode) -> Self { + self.block_coding_mode = block_coding_mode; + self + } + + /// Return options with a different packet progression order. + #[must_use] + pub const fn with_progression(mut self, progression: J2kProgressionOrder) -> Self { + self.progression = progression; + self + } + + /// Return options with a different maximum decomposition-level request. + #[must_use] + pub const fn with_max_decomposition_levels( + mut self, + max_decomposition_levels: Option, + ) -> Self { + self.max_decomposition_levels = max_decomposition_levels; + self + } + + /// Return options with a different reversible transform. + #[must_use] + pub const fn with_reversible_transform( + mut self, + reversible_transform: ReversibleTransform, + ) -> Self { + self.reversible_transform = reversible_transform; + self + } + + /// Return options with a different validation policy. + #[must_use] + pub const fn with_validation(mut self, validation: J2kEncodeValidation) -> Self { + self.validation = validation; + self + } +} + +/// Rate target for stable lossy JPEG 2000 encoding. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum J2kRateTarget { + /// Target total codestream bits per image pixel. + BitsPerPixel(f64), + /// Target total codestream byte size. + Bytes(u64), + /// Target decoded peak signal-to-noise ratio in dB. + PsnrDb(f64), +} + +/// One cumulative lossy quality layer request. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct J2kQualityLayer { + /// Cumulative target for this quality layer. + pub target: J2kRateTarget, +} + +impl J2kQualityLayer { + /// Create a cumulative lossy quality layer target. + #[must_use] + pub const fn new(target: J2kRateTarget) -> Self { + Self { target } + } +} + +/// Optional JPEG 2000 marker segment requested for lossy encode output. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kMarkerSegment { + /// SOP packet marker segments. + Sop, + /// EPH packet header termination markers. + Eph, + /// TLM tile-part length marker segment. + Tlm, + /// PLT packet length marker segments. + Plt, + /// PLM packet length marker segments. + Plm, +} + +/// Options controlling stable lossy JPEG 2000 encoding. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub struct J2kLossyEncodeOptions { + /// Backend preference for encode stages. + pub backend: EncodeBackendPreference, + /// Code-block coding mode for the codestream. + pub block_coding_mode: J2kBlockCodingMode, + /// Packet progression order. + pub progression: J2kProgressionOrder, + /// Optional explicit lossy decomposition level request. + pub max_decomposition_levels: Option, + /// Single codestream rate target. + pub rate_target: Option, + /// Cumulative quality layer targets. + pub quality_layers: Vec, + /// Optional tile width and height. + pub tile_size: Option<(u32, u32)>, + /// Optional precinct exponents in COD/COC order. + pub precinct_exponents: Vec<(u8, u8)>, + /// Optional marker segments requested for the codestream. + pub marker_segments: Vec, + /// Allowed PSNR target tolerance in dB. + pub psnr_tolerance_db: f64, + /// Iteration budget for lossy target searches. + pub psnr_iteration_budget: u8, + /// Validation policy applied before returning encoded bytes. + pub validation: J2kEncodeValidation, +} + +impl Default for J2kLossyEncodeOptions { + fn default() -> Self { + Self { + backend: EncodeBackendPreference::Auto, + block_coding_mode: J2kBlockCodingMode::Classic, + progression: J2kProgressionOrder::Lrcp, + max_decomposition_levels: None, + rate_target: None, + quality_layers: Vec::new(), + tile_size: None, + precinct_exponents: Vec::new(), + marker_segments: Vec::new(), + psnr_tolerance_db: 0.25, + psnr_iteration_budget: 8, + validation: J2kEncodeValidation::CpuRoundTrip, + } + } +} + +impl J2kLossyEncodeOptions { + /// Return options with a different backend preference. + #[must_use] + pub fn with_backend(mut self, backend: EncodeBackendPreference) -> Self { + self.backend = backend; + self + } + + /// Return options using adaptive accelerated routing. + #[must_use] + pub fn with_accelerated_backend(self) -> Self { + self.with_backend(EncodeBackendPreference::Auto) + } + + /// Return options using the portable CPU route. + #[must_use] + pub fn with_cpu_only_backend(self) -> Self { + self.with_backend(EncodeBackendPreference::CpuOnly) + } + + /// Return options requiring a strict device route. + #[must_use] + pub fn with_strict_device_backend(self) -> Self { + self.with_backend(EncodeBackendPreference::RequireDevice) + } + + /// Return options with a different code-block coding mode. + #[must_use] + pub fn with_block_coding_mode(mut self, block_coding_mode: J2kBlockCodingMode) -> Self { + self.block_coding_mode = block_coding_mode; + self + } + + /// Return options with a different packet progression order. + #[must_use] + pub fn with_progression(mut self, progression: J2kProgressionOrder) -> Self { + self.progression = progression; + self + } + + /// Return options with a different maximum decomposition-level request. + #[must_use] + pub fn with_max_decomposition_levels(mut self, max_decomposition_levels: Option) -> Self { + self.max_decomposition_levels = max_decomposition_levels; + self + } + + /// Return options with a different single codestream rate target. + #[must_use] + pub fn with_rate_target(mut self, rate_target: Option) -> Self { + self.rate_target = rate_target; + self + } + + /// Return options with different cumulative quality layer targets. + #[must_use] + pub fn with_quality_layers(mut self, quality_layers: Vec) -> Self { + self.quality_layers = quality_layers; + self + } + + /// Return options with different optional marker segment requests. + #[must_use] + pub fn with_marker_segments(mut self, marker_segments: Vec) -> Self { + self.marker_segments = marker_segments; + self + } + + /// Return options with a different validation policy. + #[must_use] + pub fn with_validation(mut self, validation: J2kEncodeValidation) -> Self { + self.validation = validation; + self + } +} + +/// Borrowed interleaved samples and image geometry for lossless encoding. +#[derive(Debug, Clone, Copy)] +pub struct J2kLosslessSamples<'a> { + /// Interleaved sample bytes. + pub data: &'a [u8], + /// Image width in pixels. + pub width: u32, + /// Image height in pixels. + pub height: u32, + /// Component count. The stable facade accepts 1-4 independent component + /// samples. Two-component output is written without MCT. + pub components: u8, + /// Significant bits per component sample. + pub bit_depth: u8, + /// Whether component samples are signed. + pub signed: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct SampleGeometry { + expected_bytes: usize, +} + +fn validate_sample_geometry( + data: &[u8], + width: u32, + height: u32, + components: u8, + bit_depth: u8, + component_what: &'static str, + bit_depth_what: &'static str, +) -> Result { + if width == 0 || height == 0 { + return Err(J2kError::InvalidSamples { + what: "dimensions must be non-zero".to_string(), + }); + } + if !(1..=4).contains(&components) { + return Err(J2kError::Unsupported(Unsupported { + what: component_what, + })); + } + if bit_depth == 0 || bit_depth > 16 { + return Err(J2kError::Unsupported(Unsupported { + what: bit_depth_what, + })); + } + let bytes_per_sample = if bit_depth <= 8 { 1usize } else { 2usize }; + let expected_bytes = (width as usize) + .checked_mul(height as usize) + .and_then(|px| px.checked_mul(components as usize)) + .and_then(|samples| samples.checked_mul(bytes_per_sample)) + .ok_or(J2kError::DimensionOverflow { width, height })?; + if data.len() != expected_bytes { + let what = if data.len() < expected_bytes { + format!( + "pixel data too short: expected {expected_bytes} bytes, got {}", + data.len() + ) + } else { + format!( + "pixel data has trailing bytes: expected {expected_bytes} bytes, got {}", + data.len() + ) + }; + return Err(J2kError::InvalidSamples { what }); + } + Ok(SampleGeometry { expected_bytes }) +} + +impl<'a> J2kLosslessSamples<'a> { + /// Validate and construct a sample descriptor. + pub fn new( + data: &'a [u8], + width: u32, + height: u32, + components: u8, + bit_depth: u8, + signed: bool, + ) -> Result { + let geometry = validate_sample_geometry( + data, + width, + height, + components, + bit_depth, + "JPEG 2000 lossless encode supports 1-4 component samples", + "JPEG 2000 lossless encode supports 1-16 bits per sample", + )?; + debug_assert_eq!(geometry.expected_bytes, data.len()); + Ok(Self { + data, + width, + height, + components, + bit_depth, + signed, + }) + } +} + +/// Borrowed interleaved samples and image geometry for lossy encoding. +#[derive(Debug, Clone, Copy)] +pub struct J2kLossySamples<'a> { + /// Interleaved sample bytes. + pub data: &'a [u8], + /// Image width in pixels. + pub width: u32, + /// Image height in pixels. + pub height: u32, + /// Component count. The stable facade accepts 1-4 component samples. + pub components: u8, + /// Significant bits per component sample. + pub bit_depth: u8, + /// Whether component samples are signed. + pub signed: bool, +} + +impl<'a> J2kLossySamples<'a> { + /// Validate and construct a lossy sample descriptor. + pub fn new( + data: &'a [u8], + width: u32, + height: u32, + components: u8, + bit_depth: u8, + signed: bool, + ) -> Result { + let geometry = validate_sample_geometry( + data, + width, + height, + components, + bit_depth, + "JPEG 2000 lossy encode supports 1-4 component samples", + "JPEG 2000 lossy encode supports 1-16 bits per sample; 17-38 bit encode is not supported", + )?; + debug_assert_eq!(geometry.expected_bytes, data.len()); + Ok(Self { + data, + width, + height, + components, + bit_depth, + signed, + }) + } +} + +/// Encoded JPEG 2000 lossless codestream and encode metadata. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncodedJ2k { + /// Raw JPEG 2000 codestream bytes. + pub codestream: Vec, + /// Backend that satisfied the encode contract. + pub backend: BackendKind, + /// Encode-stage dispatches observed while producing this codestream. + /// + /// This can be nonzero even when [`Self::backend`] is [`BackendKind::Cpu`] + /// for Auto routes that used one or more device stages but did not satisfy + /// every stage required for a fully device-backed encode contract. + pub dispatch_report: J2kEncodeDispatchReport, + /// Encoded image width in pixels. + pub width: u32, + /// Encoded image height in pixels. + pub height: u32, + /// Encoded component count. + pub components: u8, + /// Encoded significant bits per sample. + pub bit_depth: u8, + /// Whether encoded samples are signed. + pub signed: bool, +} + +/// Metrics reported by stable lossy JPEG 2000 encoding. +#[derive(Debug, Clone, PartialEq)] +pub struct J2kLossyEncodeReport { + /// Requested effective rate target. + pub target: Option, + /// Number of cumulative quality layers emitted. + pub quality_layers: u16, + /// Final native irreversible quantization scale. + pub quantization_scale: f32, + /// Total encoded codestream bytes. + pub actual_bytes: u64, + /// Total codestream bits per image pixel. + pub actual_bits_per_pixel: f64, + /// Decoded PSNR in dB when CPU validation was requested. + pub psnr_db: Option, + /// HTJ2K rate granularity in bytes when HT block coding is used. + pub ht_rate_granularity_bytes: Option, +} + +/// Encoded JPEG 2000 lossy codestream and encode metadata. +#[derive(Debug, Clone, PartialEq)] +pub struct EncodedLossyJ2k { + /// Raw JPEG 2000 codestream bytes. + pub codestream: Vec, + /// Backend that satisfied the encode contract. + pub backend: BackendKind, + /// Encode-stage dispatches observed while producing this codestream. + /// + /// This can be nonzero even when [`Self::backend`] is [`BackendKind::Cpu`] + /// for Auto routes that used one or more device stages but did not satisfy + /// every stage required for a fully device-backed encode contract. + pub dispatch_report: J2kEncodeDispatchReport, + /// Encoded image width in pixels. + pub width: u32, + /// Encoded image height in pixels. + pub height: u32, + /// Encoded component count. + pub components: u8, + /// Encoded significant bits per sample. + pub bit_depth: u8, + /// Whether encoded samples are signed. + pub signed: bool, + /// Lossy encode metrics. + pub report: J2kLossyEncodeReport, +} + +/// Encode interleaved samples into a raw JPEG 2000 lossless codestream. +pub fn encode_j2k_lossless( + samples: J2kLosslessSamples<'_>, + options: &J2kLosslessEncodeOptions, +) -> Result { + let backend = resolve_encode_backend(options.backend)?; + let codestream = encode_cpu(samples, *options)?; + validate_lossless_roundtrip(samples, &codestream, options.validation)?; + Ok(EncodedJ2k { + codestream, + backend, + dispatch_report: J2kEncodeDispatchReport::default(), + width: samples.width, + height: samples.height, + components: samples.components, + bit_depth: samples.bit_depth, + signed: samples.signed, + }) +} + +/// Encode interleaved samples with an optional device encode-stage accelerator. +/// +/// Accelerators return CPU fallback by reporting no dispatch. `Auto` accepts +/// that fallback; `RequireDevice` requires at least one dispatch. Any +/// accelerator error or codestream validation error is returned to the caller. +pub fn encode_j2k_lossless_with_accelerator( + samples: J2kLosslessSamples<'_>, + options: &J2kLosslessEncodeOptions, + accelerated_backend: BackendKind, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result { + if options.backend == EncodeBackendPreference::CpuOnly { + return encode_j2k_lossless(samples, options); + } + + let before = accelerator.dispatch_report(); + let required_stages = required_encode_stages(samples, *options, accelerated_backend); + let codestream = encode_with_native_accelerator(samples, *options, accelerator)?; + let dispatch = accelerator.dispatch_report().saturating_delta(before); + validate_lossless_roundtrip(samples, &codestream, options.validation)?; + + let backend = resolve_accelerated_encode_backend( + options.backend, + accelerated_backend, + dispatch, + required_stages, + )?; + Ok(EncodedJ2k { + codestream, + backend, + dispatch_report: dispatch, + width: samples.width, + height: samples.height, + components: samples.components, + bit_depth: samples.bit_depth, + signed: samples.signed, + }) +} + +/// Encode interleaved samples into a raw JPEG 2000 lossy codestream. +pub fn encode_j2k_lossy( + samples: J2kLossySamples<'_>, + options: &J2kLossyEncodeOptions, +) -> Result { + validate_lossy_options(options)?; + let target = effective_lossy_target(options)?; + let attempt = encode_lossy_targeted(samples, options, target, |scale| { + encode_cpu_lossy(samples, options, scale) + })?; + let report = lossy_report(samples, options, target, &attempt)?; + Ok(EncodedLossyJ2k { + codestream: attempt.codestream, + backend: resolve_encode_backend(options.backend)?, + dispatch_report: J2kEncodeDispatchReport::default(), + width: samples.width, + height: samples.height, + components: samples.components, + bit_depth: samples.bit_depth, + signed: samples.signed, + report, + }) +} + +/// Encode interleaved lossy samples with an optional device encode-stage accelerator. +pub fn encode_j2k_lossy_with_accelerator( + samples: J2kLossySamples<'_>, + options: &J2kLossyEncodeOptions, + accelerated_backend: BackendKind, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result { + if options.backend == EncodeBackendPreference::CpuOnly { + return encode_j2k_lossy(samples, options); + } + + validate_lossy_options(options)?; + let target = effective_lossy_target(options)?; + let before = accelerator.dispatch_report(); + let required_stages = required_lossy_encode_stages(samples, options, accelerated_backend); + let attempt = encode_lossy_targeted(samples, options, target, |scale| { + encode_lossy_with_native_accelerator(samples, options, scale, accelerator) + })?; + let dispatch = accelerator.dispatch_report().saturating_delta(before); + let backend = resolve_accelerated_encode_backend( + options.backend, + accelerated_backend, + dispatch, + required_stages, + )?; + let report = lossy_report(samples, options, target, &attempt)?; + Ok(EncodedLossyJ2k { + codestream: attempt.codestream, + backend, + dispatch_report: dispatch, + width: samples.width, + height: samples.height, + components: samples.components, + bit_depth: samples.bit_depth, + signed: samples.signed, + report, + }) +} + +fn resolve_encode_backend(preference: EncodeBackendPreference) -> Result { + match preference { + EncodeBackendPreference::Auto | EncodeBackendPreference::CpuOnly => Ok(BackendKind::Cpu), + EncodeBackendPreference::RequireDevice => Err(J2kError::Unsupported(Unsupported { + what: "device JPEG 2000 lossless encode backend is unavailable", + })), + } +} + +fn resolve_accelerated_encode_backend( + preference: EncodeBackendPreference, + accelerated_backend: BackendKind, + dispatch: J2kEncodeDispatchReport, + required_stages: RequiredEncodeStages, +) -> Result { + if required_stages.satisfied_by(dispatch) { + return Ok(accelerated_backend); + } + match preference { + EncodeBackendPreference::RequireDevice => Err(J2kError::Unsupported(Unsupported { + what: required_stages.missing_message(dispatch), + })), + EncodeBackendPreference::Auto | EncodeBackendPreference::CpuOnly => Ok(BackendKind::Cpu), + } +} + +fn encode_cpu( + samples: J2kLosslessSamples<'_>, + options: J2kLosslessEncodeOptions, +) -> Result, J2kError> { + let options = native_lossless_options(samples, options); + j2k_native::encode( + samples.data, + samples.width, + samples.height, + samples.components, + samples.bit_depth, + samples.signed, + &options, + ) + .map_err(|err| J2kError::Backend(format!("JPEG 2000 lossless encode failed: {err}"))) +} + +fn encode_with_native_accelerator( + samples: J2kLosslessSamples<'_>, + options: J2kLosslessEncodeOptions, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, J2kError> { + let options = native_lossless_options(samples, options); + let mut native_accelerator = NativeEncodeStageAdapter::new(accelerator); + j2k_native::encode_with_accelerator( + samples.data, + samples.width, + samples.height, + samples.components, + samples.bit_depth, + samples.signed, + &options, + &mut native_accelerator, + ) + .map_err(|err| J2kError::Backend(format!("JPEG 2000 lossless encode failed: {err}"))) +} + +struct LossyAttempt { + codestream: Vec, + quantization_scale: f32, +} + +fn encode_cpu_lossy( + samples: J2kLossySamples<'_>, + options: &J2kLossyEncodeOptions, + quantization_scale: f32, +) -> Result, J2kError> { + let options = native_lossy_options(samples, options, quantization_scale)?; + j2k_native::encode( + samples.data, + samples.width, + samples.height, + samples.components, + samples.bit_depth, + samples.signed, + &options, + ) + .map_err(|err| J2kError::Backend(format!("JPEG 2000 lossy encode failed: {err}"))) +} + +fn encode_lossy_with_native_accelerator( + samples: J2kLossySamples<'_>, + options: &J2kLossyEncodeOptions, + quantization_scale: f32, + accelerator: &mut impl J2kEncodeStageAccelerator, +) -> Result, J2kError> { + let options = native_lossy_options(samples, options, quantization_scale)?; + let mut native_accelerator = NativeEncodeStageAdapter::new(accelerator); + j2k_native::encode_with_accelerator( + samples.data, + samples.width, + samples.height, + samples.components, + samples.bit_depth, + samples.signed, + &options, + &mut native_accelerator, + ) + .map_err(|err| J2kError::Backend(format!("JPEG 2000 lossy encode failed: {err}"))) +} + +fn encode_lossy_targeted( + samples: J2kLossySamples<'_>, + options: &J2kLossyEncodeOptions, + target: Option, + mut encode_at_scale: impl FnMut(f32) -> Result, J2kError>, +) -> Result { + match target { + None => { + let codestream = encode_at_scale(1.0)?; + Ok(LossyAttempt { + codestream, + quantization_scale: 1.0, + }) + } + Some(J2kRateTarget::Bytes(bytes)) => { + encode_lossy_to_byte_target(samples, options, bytes, encode_at_scale) + } + Some(J2kRateTarget::BitsPerPixel(bits_per_pixel)) => { + let target_bytes = target_bytes_for_bpp(samples, bits_per_pixel)?; + encode_lossy_to_byte_target(samples, options, target_bytes, encode_at_scale) + } + Some(J2kRateTarget::PsnrDb(psnr_db)) => { + encode_lossy_to_psnr_target(samples, options, psnr_db, encode_at_scale) + } + } +} + +fn encode_lossy_to_byte_target( + _samples: J2kLossySamples<'_>, + options: &J2kLossyEncodeOptions, + target_bytes: u64, + mut encode_at_scale: impl FnMut(f32) -> Result, J2kError>, +) -> Result { + let tolerance = byte_target_tolerance(target_bytes); + let mut low = 1.0f32; + let mut high = 1.0f32; + let mut best = LossyAttempt { + codestream: encode_at_scale(high)?, + quantization_scale: high, + }; + let mut best_diff = byte_target_diff(best.codestream.len() as u64, target_bytes); + + while best.codestream.len() as u64 > target_bytes.saturating_add(tolerance) + && high < 1_048_576.0 + { + low = high; + high *= 2.0; + let codestream = encode_at_scale(high)?; + let diff = byte_target_diff(codestream.len() as u64, target_bytes); + if diff < best_diff { + best = LossyAttempt { + codestream, + quantization_scale: high, + }; + best_diff = diff; + } + } + + if best.codestream.len() as u64 > target_bytes.saturating_add(tolerance) { + return Err(J2kError::RateTargetUnreachable { + target: format!("{target_bytes} bytes"), + best: format!("{} bytes", best.codestream.len()), + }); + } + + for _ in 0..options.psnr_iteration_budget.max(1) { + let mid = (low + high) * 0.5; + let codestream = encode_at_scale(mid)?; + let len = codestream.len() as u64; + let diff = byte_target_diff(len, target_bytes); + if diff < best_diff { + best = LossyAttempt { + codestream, + quantization_scale: mid, + }; + best_diff = diff; + } + if len > target_bytes { + low = mid; + } else { + high = mid; + } + } + + Ok(best) +} + +fn encode_lossy_to_psnr_target( + samples: J2kLossySamples<'_>, + options: &J2kLossyEncodeOptions, + target_psnr_db: f64, + mut encode_at_scale: impl FnMut(f32) -> Result, J2kError>, +) -> Result { + let tolerance = options.psnr_tolerance_db; + let mut low = 1.0f32; + let mut high = 1.0f32; + let mut best = LossyAttempt { + codestream: encode_at_scale(high)?, + quantization_scale: high, + }; + let mut best_psnr = decoded_psnr(samples, &best.codestream)?; + if best_psnr + tolerance < target_psnr_db { + return Err(J2kError::RateTargetUnreachable { + target: format!("{target_psnr_db:.3} dB"), + best: format!("{best_psnr:.3} dB"), + }); + } + + for _ in 0..options.psnr_iteration_budget.max(1) { + high *= 2.0; + let codestream = encode_at_scale(high)?; + let psnr = decoded_psnr(samples, &codestream)?; + if psnr + tolerance >= target_psnr_db { + best = LossyAttempt { + codestream, + quantization_scale: high, + }; + best_psnr = psnr; + low = high; + } else { + break; + } + } + + for _ in 0..options.psnr_iteration_budget.max(1) { + let mid = (low + high) * 0.5; + let codestream = encode_at_scale(mid)?; + let psnr = decoded_psnr(samples, &codestream)?; + if psnr + tolerance >= target_psnr_db { + best = LossyAttempt { + codestream, + quantization_scale: mid, + }; + best_psnr = psnr; + low = mid; + } else { + high = mid; + } + } + + let _ = best_psnr; + Ok(best) +} + +fn native_lossless_options( + samples: J2kLosslessSamples<'_>, + options: J2kLosslessEncodeOptions, +) -> EncodeOptions { + let progression_order = native_progression_order(options.progression); + EncodeOptions { + reversible: true, + num_decomposition_levels: j2k_lossless_decomposition_levels_for_options(samples, options), + use_ht_block_coding: options.block_coding_mode == J2kBlockCodingMode::HighThroughput, + progression_order, + write_tlm: options.progression == J2kProgressionOrder::Rpcl, + use_mct: options.reversible_transform == ReversibleTransform::Rct53, + validate_high_throughput_codestream: false, + ..EncodeOptions::default() + } +} + +fn native_lossy_options( + samples: J2kLossySamples<'_>, + options: &J2kLossyEncodeOptions, + quantization_scale: f32, +) -> Result { + let num_layers = lossy_quality_layer_count(options); + Ok(EncodeOptions { + reversible: false, + num_decomposition_levels: j2k_lossy_decomposition_levels_for_options(samples, options), + use_ht_block_coding: options.block_coding_mode == J2kBlockCodingMode::HighThroughput, + progression_order: native_progression_order(options.progression), + write_tlm: options.marker_segments.contains(&J2kMarkerSegment::Tlm), + write_plt: options.marker_segments.contains(&J2kMarkerSegment::Plt), + write_plm: options.marker_segments.contains(&J2kMarkerSegment::Plm), + write_sop: options.marker_segments.contains(&J2kMarkerSegment::Sop), + write_eph: options.marker_segments.contains(&J2kMarkerSegment::Eph), + use_mct: samples.components >= 3, + num_layers, + quality_layer_byte_targets: lossy_quality_layer_byte_targets(samples, options)?, + tile_size: options.tile_size, + precinct_exponents: options.precinct_exponents.clone(), + validate_high_throughput_codestream: false, + irreversible_quantization_scale: quantization_scale, + ..EncodeOptions::default() + }) +} + +fn lossy_quality_layer_byte_targets( + samples: J2kLossySamples<'_>, + options: &J2kLossyEncodeOptions, +) -> Result, J2kError> { + if options.quality_layers.len() <= 1 { + return Ok(Vec::new()); + } + + let mut targets = Vec::with_capacity(options.quality_layers.len()); + for layer in &options.quality_layers { + match layer.target { + J2kRateTarget::Bytes(bytes) => targets.push(bytes), + J2kRateTarget::BitsPerPixel(bits_per_pixel) => { + targets.push(target_bytes_for_bpp(samples, bits_per_pixel)?); + } + J2kRateTarget::PsnrDb(_) => return Ok(Vec::new()), + } + } + if targets.windows(2).any(|pair| pair[0] > pair[1]) { + return Err(J2kError::Unsupported(Unsupported { + what: "JPEG 2000 lossy quality layer targets must be cumulative and monotonic", + })); + } + Ok(targets) +} + +pub(crate) fn native_progression_order(progression: J2kProgressionOrder) -> EncodeProgressionOrder { + match progression { + J2kProgressionOrder::Lrcp => EncodeProgressionOrder::Lrcp, + J2kProgressionOrder::Rlcp => EncodeProgressionOrder::Rlcp, + J2kProgressionOrder::Rpcl => EncodeProgressionOrder::Rpcl, + J2kProgressionOrder::Pcrl => EncodeProgressionOrder::Pcrl, + J2kProgressionOrder::Cprl => EncodeProgressionOrder::Cprl, + } +} + +const MIN_LOSSLESS_DWT_DIMENSION: u32 = 64; + +/// Return the default lossless decomposition level policy used by the facade. +pub fn j2k_lossless_decomposition_levels(samples: J2kLosslessSamples<'_>) -> u8 { + j2k_lossless_decomposition_levels_for_progression(samples, J2kProgressionOrder::Lrcp) +} + +/// Return the default lossless decomposition level policy for a progression. +pub fn j2k_lossless_decomposition_levels_for_progression( + samples: J2kLosslessSamples<'_>, + progression: J2kProgressionOrder, +) -> u8 { + if matches!( + progression, + J2kProgressionOrder::Rpcl | J2kProgressionOrder::Pcrl | J2kProgressionOrder::Cprl + ) { + return j2k_rpcl_lossless_decomposition_levels(samples); + } + + if samples.width.min(samples.height) < MIN_LOSSLESS_DWT_DIMENSION { + return 0; + } + + 1 +} + +fn j2k_lossy_decomposition_levels_for_options( + samples: J2kLossySamples<'_>, + options: &J2kLossyEncodeOptions, +) -> u8 { + let levels = if matches!( + options.progression, + J2kProgressionOrder::Rpcl | J2kProgressionOrder::Pcrl | J2kProgressionOrder::Cprl + ) { + j2k_lossy_position_progression_decomposition_levels(samples) + } else { + u8::from(samples.width.min(samples.height) >= MIN_LOSSLESS_DWT_DIMENSION) + }; + options.max_decomposition_levels.map_or(levels, |max| { + levels + .min(max) + .min(max_decomposition_levels(samples.width, samples.height)) + }) +} + +fn j2k_lossy_position_progression_decomposition_levels(samples: J2kLossySamples<'_>) -> u8 { + j2k_rpcl_lossless_decomposition_levels(J2kLosslessSamples { + data: samples.data, + width: samples.width, + height: samples.height, + components: samples.components, + bit_depth: samples.bit_depth, + signed: samples.signed, + }) +} + +/// Return the effective lossless decomposition level policy for encode options. +pub fn j2k_lossless_decomposition_levels_for_options( + samples: J2kLosslessSamples<'_>, + options: J2kLosslessEncodeOptions, +) -> u8 { + let levels = j2k_lossless_decomposition_levels_for_progression(samples, options.progression); + options + .max_decomposition_levels + .map_or(levels, |requested| { + if samples.width.min(samples.height) < MIN_LOSSLESS_DWT_DIMENSION { + return 0; + } + requested.min(max_decomposition_levels(samples.width, samples.height)) + }) +} + +fn j2k_rpcl_lossless_decomposition_levels(samples: J2kLosslessSamples<'_>) -> u8 { + let mut levels = 0u8; + let mut width = samples.width; + let mut height = samples.height; + let max_levels = max_decomposition_levels(samples.width, samples.height); + + while width.min(height) > MIN_LOSSLESS_DWT_DIMENSION && levels < max_levels { + width = width.div_ceil(2); + height = height.div_ceil(2); + levels += 1; + } + + levels +} + +fn max_decomposition_levels(width: u32, height: u32) -> u8 { + let min_dim = width.min(height); + if min_dim <= 1 { + return 0; + } + min_dim.ilog2() as u8 +} + +#[derive(Debug, Clone, Copy)] +struct RequiredEncodeStages { + bits: u16, +} + +impl RequiredEncodeStages { + const DEINTERLEAVE: u16 = 1 << 0; + const FORWARD_RCT: u16 = 1 << 1; + const FORWARD_DWT53: u16 = 1 << 2; + const TIER1_CODE_BLOCK: u16 = 1 << 3; + const HT_CODE_BLOCK: u16 = 1 << 4; + const PACKETIZATION: u16 = 1 << 5; + const QUANTIZE_SUBBAND: u16 = 1 << 6; + const FORWARD_ICT: u16 = 1 << 7; + const FORWARD_DWT97: u16 = 1 << 8; + + fn satisfied_by(self, dispatch: J2kEncodeDispatchReport) -> bool { + self.missing_stage(dispatch).is_none() + } + + fn missing_message(self, dispatch: J2kEncodeDispatchReport) -> &'static str { + match self.missing_stage(dispatch) { + Some("deinterleave") => { + "requested JPEG 2000 device encode backend did not dispatch deinterleave" + } + Some("forward_rct") => { + "requested JPEG 2000 device encode backend did not dispatch forward_rct" + } + Some("forward_ict") => { + "requested JPEG 2000 device encode backend did not dispatch forward_ict" + } + Some("forward_dwt53") => { + "requested JPEG 2000 device encode backend did not dispatch forward_dwt53" + } + Some("forward_dwt97") => { + "requested JPEG 2000 device encode backend did not dispatch forward_dwt97" + } + Some("tier1_code_block") => { + "requested JPEG 2000 device encode backend did not dispatch tier1_code_block" + } + Some("ht_code_block") => { + "requested JPEG 2000 device encode backend did not dispatch ht_code_block" + } + Some("quantize_subband") => { + "requested JPEG 2000 device encode backend did not dispatch quantize_subband" + } + Some("packetization") => { + "requested JPEG 2000 device encode backend did not dispatch packetization" + } + _ => "requested JPEG 2000 device encode backend did not dispatch", + } + } + + fn missing_stage(self, dispatch: J2kEncodeDispatchReport) -> Option<&'static str> { + if self.contains(Self::DEINTERLEAVE) && dispatch.deinterleave == 0 { + return Some("deinterleave"); + } + if self.contains(Self::FORWARD_RCT) && dispatch.forward_rct == 0 { + return Some("forward_rct"); + } + if self.contains(Self::FORWARD_ICT) && dispatch.forward_ict == 0 { + return Some("forward_ict"); + } + if self.contains(Self::FORWARD_DWT53) && dispatch.forward_dwt53 == 0 { + return Some("forward_dwt53"); + } + if self.contains(Self::FORWARD_DWT97) && dispatch.forward_dwt97 == 0 { + return Some("forward_dwt97"); + } + if self.contains(Self::TIER1_CODE_BLOCK) && dispatch.tier1_code_block == 0 { + return Some("tier1_code_block"); + } + if self.contains(Self::HT_CODE_BLOCK) && dispatch.ht_code_block == 0 { + return Some("ht_code_block"); + } + if self.contains(Self::QUANTIZE_SUBBAND) && dispatch.quantize_subband == 0 { + return Some("quantize_subband"); + } + if self.contains(Self::PACKETIZATION) && dispatch.packetization == 0 { + return Some("packetization"); + } + None + } + + fn contains(self, stage: u16) -> bool { + self.bits & stage != 0 + } +} + +fn required_encode_stages( + samples: J2kLosslessSamples<'_>, + options: J2kLosslessEncodeOptions, + accelerated_backend: BackendKind, +) -> RequiredEncodeStages { + let decomposition_levels = j2k_lossless_decomposition_levels_for_options(samples, options); + let high_throughput = options.block_coding_mode == J2kBlockCodingMode::HighThroughput; + + let mut bits = RequiredEncodeStages::PACKETIZATION; + if matches!(accelerated_backend, BackendKind::Cuda | BackendKind::Metal) { + bits |= RequiredEncodeStages::DEINTERLEAVE | RequiredEncodeStages::QUANTIZE_SUBBAND; + } + if samples.components >= 3 && options.reversible_transform == ReversibleTransform::Rct53 { + bits |= RequiredEncodeStages::FORWARD_RCT; + } + if decomposition_levels > 0 { + bits |= RequiredEncodeStages::FORWARD_DWT53; + } + if high_throughput { + bits |= RequiredEncodeStages::HT_CODE_BLOCK; + } else { + bits |= RequiredEncodeStages::TIER1_CODE_BLOCK; + } + + RequiredEncodeStages { bits } +} + +fn required_lossy_encode_stages( + samples: J2kLossySamples<'_>, + options: &J2kLossyEncodeOptions, + accelerated_backend: BackendKind, +) -> RequiredEncodeStages { + let decomposition_levels = j2k_lossy_decomposition_levels_for_options(samples, options); + let high_throughput = options.block_coding_mode == J2kBlockCodingMode::HighThroughput; + + let scalar_packetization_required = lossy_quality_layer_count(options) > 1 + || options.marker_segments.contains(&J2kMarkerSegment::Plt) + || options.marker_segments.contains(&J2kMarkerSegment::Plm) + || options.marker_segments.contains(&J2kMarkerSegment::Sop) + || options.marker_segments.contains(&J2kMarkerSegment::Eph); + let mut bits = 0; + if !scalar_packetization_required || accelerated_backend == BackendKind::Metal { + bits |= RequiredEncodeStages::PACKETIZATION; + } + if matches!(accelerated_backend, BackendKind::Cuda | BackendKind::Metal) { + bits |= RequiredEncodeStages::DEINTERLEAVE | RequiredEncodeStages::QUANTIZE_SUBBAND; + if samples.components >= 3 { + bits |= RequiredEncodeStages::FORWARD_ICT; + } + if decomposition_levels > 0 { + bits |= RequiredEncodeStages::FORWARD_DWT97; + } + } + if high_throughput { + bits |= RequiredEncodeStages::HT_CODE_BLOCK; + } else { + bits |= RequiredEncodeStages::TIER1_CODE_BLOCK; + } + + RequiredEncodeStages { bits } +} + +fn validate_lossy_options(options: &J2kLossyEncodeOptions) -> Result<(), J2kError> { + if options.quality_layers.len() > 32 { + return Err(J2kError::Unsupported(Unsupported { + what: "JPEG 2000 lossy encode supports 1-32 quality layers", + })); + } + if let Some((tile_width, tile_height)) = options.tile_size { + if tile_width == 0 || tile_height == 0 { + return Err(J2kError::Unsupported(Unsupported { + what: "JPEG 2000 lossy tile dimensions must be non-zero", + })); + } + } + if options + .precinct_exponents + .iter() + .any(|&(ppx, ppy)| ppx > 15 || ppy > 15) + { + return Err(J2kError::Unsupported(Unsupported { + what: "JPEG 2000 lossy precinct exponents must be 0-15", + })); + } + if !(options.psnr_tolerance_db.is_finite() && options.psnr_tolerance_db >= 0.0) { + return Err(J2kError::Unsupported(Unsupported { + what: "JPEG 2000 lossy PSNR tolerance must be finite and non-negative", + })); + } + if options.psnr_iteration_budget == 0 { + return Err(J2kError::Unsupported(Unsupported { + what: "JPEG 2000 lossy PSNR iteration budget must be greater than zero", + })); + } + validate_rate_target(options.rate_target)?; + for layer in &options.quality_layers { + validate_rate_target(Some(layer.target))?; + } + Ok(()) +} + +fn effective_lossy_target( + options: &J2kLossyEncodeOptions, +) -> Result, J2kError> { + match (options.rate_target, options.quality_layers.as_slice()) { + (target, []) => Ok(target), + (None, [layer]) => Ok(Some(layer.target)), + (Some(target), [layer]) if target == layer.target => Ok(Some(target)), + (Some(_), [_]) => Err(J2kError::Unsupported(Unsupported { + what: + "specify either a JPEG 2000 lossy rate target or one quality layer target, not both", + })), + (None, layers) => Ok(layers.last().map(|layer| layer.target)), + (Some(target), layers) if layers.last().is_some_and(|layer| layer.target == target) => { + Ok(Some(target)) + } + (Some(_), _) => Err(J2kError::Unsupported(Unsupported { + what: "when multiple JPEG 2000 quality layers are specified, the single rate target must match the final cumulative layer target", + })), + } +} + +fn validate_rate_target(target: Option) -> Result<(), J2kError> { + match target { + None => Ok(()), + Some(J2kRateTarget::BitsPerPixel(bits_per_pixel)) + if bits_per_pixel.is_finite() && bits_per_pixel > 0.0 => + { + Ok(()) + } + Some(J2kRateTarget::Bytes(bytes)) if bytes > 0 => Ok(()), + Some(J2kRateTarget::PsnrDb(psnr_db)) if psnr_db.is_finite() && psnr_db > 0.0 => Ok(()), + Some(J2kRateTarget::BitsPerPixel(_)) => Err(J2kError::Unsupported(Unsupported { + what: "JPEG 2000 lossy bits-per-pixel target must be finite and greater than zero", + })), + Some(J2kRateTarget::Bytes(_)) => Err(J2kError::Unsupported(Unsupported { + what: "JPEG 2000 lossy byte target must be greater than zero", + })), + Some(J2kRateTarget::PsnrDb(_)) => Err(J2kError::Unsupported(Unsupported { + what: "JPEG 2000 lossy PSNR target must be finite and greater than zero", + })), + } +} + +fn lossy_report( + samples: J2kLossySamples<'_>, + options: &J2kLossyEncodeOptions, + target: Option, + attempt: &LossyAttempt, +) -> Result { + let actual_bytes = attempt.codestream.len() as u64; + Ok(J2kLossyEncodeReport { + target, + quality_layers: u16::from(lossy_quality_layer_count(options)), + quantization_scale: attempt.quantization_scale, + actual_bytes, + actual_bits_per_pixel: bits_per_pixel(samples, actual_bytes), + psnr_db: validate_lossy_roundtrip(samples, &attempt.codestream, options.validation)?, + ht_rate_granularity_bytes: (options.block_coding_mode + == J2kBlockCodingMode::HighThroughput) + .then_some(actual_bytes), + }) +} + +fn lossy_quality_layer_count(options: &J2kLossyEncodeOptions) -> u8 { + u8::try_from(options.quality_layers.len().max(1)).unwrap_or(32) +} + +fn validate_lossy_roundtrip( + samples: J2kLossySamples<'_>, + codestream: &[u8], + validation: J2kEncodeValidation, +) -> Result, J2kError> { + if validation == J2kEncodeValidation::External { + return Ok(None); + } + + let decoded = Image::new(codestream, &DecodeSettings::default()) + .map_err(|err| J2kError::Backend(format!("encoded codestream validation failed: {err}")))? + .decode_native() + .map_err(|err| J2kError::Backend(format!("encoded codestream validation failed: {err}")))?; + + if decoded.width != samples.width + || decoded.height != samples.height + || decoded.num_components != samples.components + || decoded.bit_depth != samples.bit_depth + { + return Err(J2kError::InvalidSamples { + what: "JPEG 2000 lossy encode failed round-trip geometry validation".to_string(), + }); + } + + Ok(Some(psnr_from_decoded(samples, &decoded.data)?)) +} + +fn decoded_psnr(samples: J2kLossySamples<'_>, codestream: &[u8]) -> Result { + let decoded = Image::new(codestream, &DecodeSettings::default()) + .map_err(|err| J2kError::Backend(format!("encoded codestream validation failed: {err}")))? + .decode_native() + .map_err(|err| J2kError::Backend(format!("encoded codestream validation failed: {err}")))?; + psnr_from_decoded(samples, &decoded.data) +} + +fn psnr_from_decoded(samples: J2kLossySamples<'_>, decoded: &[u8]) -> Result { + if decoded.len() != samples.data.len() { + return Err(J2kError::InvalidSamples { + what: format!( + "JPEG 2000 lossy encode validation length mismatch: expected {} bytes, got {} bytes", + samples.data.len(), + decoded.len() + ), + }); + } + let bytes_per_sample = if samples.bit_depth <= 8 { + 1usize + } else { + 2usize + }; + let sample_count = samples.data.len() / bytes_per_sample; + let mut squared_error = 0.0f64; + for sample_idx in 0..sample_count { + let original = sample_value(samples.data, sample_idx, samples.bit_depth, samples.signed); + let decoded = sample_value(decoded, sample_idx, samples.bit_depth, samples.signed); + let error = original - decoded; + squared_error += error * error; + } + if squared_error == 0.0 { + return Ok(f64::INFINITY); + } + let mse = squared_error / usize_to_f64(sample_count); + let peak = f64::from((1u32 << u32::from(samples.bit_depth)) - 1); + Ok(10.0 * ((peak * peak) / mse).log10()) +} + +fn sample_value(data: &[u8], sample_idx: usize, bit_depth: u8, signed: bool) -> f64 { + if bit_depth <= 8 { + if signed { + f64::from(data[sample_idx] as i8) + } else { + f64::from(data[sample_idx]) + } + } else { + let byte_idx = sample_idx * 2; + let bytes = [data[byte_idx], data[byte_idx + 1]]; + if signed { + f64::from(i16::from_le_bytes(bytes)) + } else { + f64::from(u16::from_le_bytes(bytes)) + } + } +} + +fn target_bytes_for_bpp( + samples: J2kLossySamples<'_>, + bits_per_pixel: f64, +) -> Result { + let pixels = f64::from(samples.width) * f64::from(samples.height); + let bytes = (pixels * bits_per_pixel / 8.0).ceil(); + if bytes.is_finite() && bytes > 0.0 && bytes <= 18_446_744_073_709_551_615.0 { + Ok(bytes as u64) + } else { + Err(J2kError::Unsupported(Unsupported { + what: "JPEG 2000 lossy bits-per-pixel target overflows byte target", + })) + } +} + +fn byte_target_tolerance(target_bytes: u64) -> u64 { + target_bytes.div_ceil(100).max(512) +} + +fn byte_target_diff(actual: u64, target: u64) -> u64 { + actual.abs_diff(target) +} + +fn bits_per_pixel(samples: J2kLossySamples<'_>, bytes: u64) -> f64 { + (u64_to_f64(bytes) * 8.0) / (f64::from(samples.width) * f64::from(samples.height)) +} + +#[allow(clippy::cast_precision_loss)] +fn usize_to_f64(value: usize) -> f64 { + value as f64 +} + +#[allow(clippy::cast_precision_loss)] +fn u64_to_f64(value: u64) -> f64 { + value as f64 +} + +fn validate_lossless_roundtrip( + samples: J2kLosslessSamples<'_>, + codestream: &[u8], + validation: J2kEncodeValidation, +) -> Result<(), J2kError> { + if validation == J2kEncodeValidation::External { + return Ok(()); + } + + let decoded = Image::new(codestream, &DecodeSettings::default()) + .map_err(|err| J2kError::Backend(format!("encoded codestream validation failed: {err}")))? + .decode_native() + .map_err(|err| J2kError::Backend(format!("encoded codestream validation failed: {err}")))?; + + if decoded.width != samples.width + || decoded.height != samples.height + || decoded.num_components != samples.components + || decoded.bit_depth != samples.bit_depth + { + return Err(J2kError::InvalidSamples { + what: "JPEG 2000 lossless encode failed round-trip geometry validation".to_string(), + }); + } + if decoded.data != samples.data { + let mismatch = decoded + .data + .iter() + .zip(samples.data.iter()) + .position(|(actual, expected)| actual != expected); + return Err(J2kError::InvalidSamples { + what: match mismatch { + Some(index) => format!( + "JPEG 2000 lossless encode failed round-trip validation at byte {index}: expected {}, got {}", + samples.data[index], decoded.data[index] + ), + None => format!( + "JPEG 2000 lossless encode failed round-trip validation: expected {} bytes, got {} bytes", + samples.data.len(), + decoded.data.len() + ), + }}); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ + encode_j2k_lossless, j2k_lossless_decomposition_levels_for_options, + native_lossless_options, DecodeSettings, EncodeBackendPreference, Image, + J2kBlockCodingMode, J2kEncodeValidation, J2kLosslessEncodeOptions, J2kLosslessSamples, + J2kProgressionOrder, ReversibleTransform, + }; + + fn cod_mct(codestream: &[u8]) -> u8 { + let cod_offset = codestream + .windows(2) + .position(|window| window == [0xFF, 0x52]) + .expect("COD marker"); + codestream[cod_offset + 8] + } + + #[test] + fn lossless_encode_can_disable_component_transform() { + let pixels: Vec = (0..4 * 4 * 3) + .map(|value| ((value * 17) & 0xFF) as u8) + .collect(); + let samples = J2kLosslessSamples::new(&pixels, 4, 4, 3, 8, false).unwrap(); + let encoded = encode_j2k_lossless( + samples, + &J2kLosslessEncodeOptions { + block_coding_mode: J2kBlockCodingMode::Classic, + progression: J2kProgressionOrder::Lrcp, + max_decomposition_levels: Some(0), + reversible_transform: ReversibleTransform::None53, + validation: J2kEncodeValidation::CpuRoundTrip, + ..J2kLosslessEncodeOptions::default() + }, + ) + .unwrap(); + + assert_eq!(cod_mct(&encoded.codestream), 0); + } + + #[test] + fn explicit_decomposition_levels_override_default_lrcp_policy() { + let pixels = vec![0; 128 * 128]; + let samples = J2kLosslessSamples::new(&pixels, 128, 128, 1, 8, false).unwrap(); + + let levels = j2k_lossless_decomposition_levels_for_options( + samples, + J2kLosslessEncodeOptions { + block_coding_mode: J2kBlockCodingMode::Classic, + progression: J2kProgressionOrder::Lrcp, + max_decomposition_levels: Some(5), + ..J2kLosslessEncodeOptions::default() + }, + ); + + assert_eq!(levels, 5); + } + + #[test] + fn facade_native_options_skip_internal_ht_validation_for_external_validation() { + let pixels = vec![0; 64 * 64]; + let samples = J2kLosslessSamples::new(&pixels, 64, 64, 1, 8, false).unwrap(); + + let external = native_lossless_options( + samples, + J2kLosslessEncodeOptions { + block_coding_mode: J2kBlockCodingMode::HighThroughput, + validation: J2kEncodeValidation::External, + ..J2kLosslessEncodeOptions::default() + }, + ); + let roundtrip = native_lossless_options( + samples, + J2kLosslessEncodeOptions { + block_coding_mode: J2kBlockCodingMode::HighThroughput, + validation: J2kEncodeValidation::CpuRoundTrip, + ..J2kLosslessEncodeOptions::default() + }, + ); + + assert!(!external.validate_high_throughput_codestream); + assert!(!roundtrip.validate_high_throughput_codestream); + } + + #[test] + fn lossless_facade_roundtrips_four_component_via_public_api() { + let width: u32 = 32; + let height: u32 = 24; + let components: u8 = 4; + + // Deterministic 4-component (RGBA/CMYK) 8-bit input, distinct per plane. + let mut pixels = Vec::with_capacity((width * height * u32::from(components)) as usize); + for y in 0..height { + for x in 0..width { + for c in 0..u32::from(components) { + let value = (x.wrapping_mul(7) ^ y.wrapping_mul(13)).wrapping_add(c * 41); + pixels.push((value & 0xFF) as u8); + } + } + } + + // MUST go through the real public constructor. + let samples = J2kLosslessSamples::new(&pixels, width, height, components, 8, false) + .expect("4-component samples must be accepted by the public constructor"); + + // Encode via the public CPU lossless entry. + let encoded = encode_j2k_lossless( + samples, + &J2kLosslessEncodeOptions { + backend: EncodeBackendPreference::CpuOnly, + validation: J2kEncodeValidation::CpuRoundTrip, + ..J2kLosslessEncodeOptions::default() + }, + ) + .expect("4-component CPU lossless encode must succeed"); + + assert_eq!(encoded.components, components); + + // Decode the bytes with the native decoder and assert an exact round-trip. + let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) + .expect("native decode of 4-component codestream must construct") + .decode_native() + .expect("native decode of 4-component codestream must succeed"); + + assert_eq!(decoded.width, width); + assert_eq!(decoded.height, height); + assert_eq!(decoded.num_components, components); + assert_eq!(decoded.bit_depth, 8); + assert_eq!( + decoded.data, pixels, + "4-component pixels must round-trip exactly" + ); + + // 2-component is accepted and handled as independent channels without MCT. + let two_component = vec![0u8; (width * height * 2) as usize]; + let two_component = J2kLosslessSamples::new(&two_component, width, height, 2, 8, false) + .expect("2-component samples must be accepted by the public constructor"); + assert_eq!(two_component.components, 2); + } +} diff --git a/crates/j2k/src/error.rs b/crates/j2k/src/error.rs new file mode 100644 index 00000000..7b41f350 --- /dev/null +++ b/crates/j2k/src/error.rs @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k_core::{BufferError, CodecError, InputError, NotImplemented, Unsupported}; + +/// Error returned by JPEG 2000 inspect, decode, encode, and recode APIs. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum J2kError { + /// Caller-owned buffers were too small or malformed. + #[error(transparent)] + Buffer(#[from] BufferError), + + /// Input was too short or truncated while parsing. + #[error(transparent)] + Input(#[from] InputError), + + /// Requested feature is planned but not implemented. + #[error(transparent)] + NotImplemented(#[from] NotImplemented), + + /// Requested input feature or option is unsupported. + #[error(transparent)] + Unsupported(#[from] Unsupported), + + /// Native backend or encode/decode stage failed. + #[error("backend failed: {0}")] + Backend(String), + + /// Caller-provided encode samples were malformed. + #[error("invalid JPEG 2000 samples: {what}")] + InvalidSamples { + /// Description of the invalid sample condition. + what: String, + }, + + /// Lossy rate-control search could not satisfy the requested target. + #[error("JPEG 2000 lossy rate target unreachable: {target}, best {best}")] + RateTargetUnreachable { + /// Requested rate target. + target: String, + /// Best achievable result observed by the search. + best: String, + }, + + /// Requested region lies outside image bounds. + #[error("region ({x},{y} {w}x{h}) is outside image bounds {image_w}x{image_h}")] + InvalidRegion { + /// Region left coordinate. + x: u32, + /// Region top coordinate. + y: u32, + /// Region width. + w: u32, + /// Region height. + h: u32, + /// Image width. + image_w: u32, + /// Image height. + image_h: u32, + }, + + /// JP2 box structure was invalid. + #[error("invalid JP2 box at offset {offset}: {what}")] + InvalidBox { + /// Byte offset of the invalid box. + offset: usize, + /// Description of the invalid box condition. + what: &'static str, + }, + + /// Required JP2 box was absent. + #[error("missing required JP2 box {box_type}")] + MissingRequiredBox { + /// Missing box type. + box_type: &'static str, + }, + + /// Codestream marker was invalid. + #[error("invalid codestream marker FF{marker:02X} at offset {offset}")] + InvalidMarker { + /// Byte offset of the invalid marker. + offset: usize, + /// Marker byte following the `0xFF` prefix. + marker: u8, + }, + + /// Required codestream marker was absent. + #[error("missing required codestream marker {marker}")] + MissingRequiredMarker { + /// Missing marker name. + marker: &'static str, + }, + + /// SIZ segment was invalid. + #[error("invalid SIZ segment: {what}")] + InvalidSiz { + /// Description of the invalid SIZ condition. + what: &'static str, + }, + + /// COD segment was invalid. + #[error("invalid COD segment: {what}")] + InvalidCod { + /// Description of the invalid COD condition. + what: &'static str, + }, + + /// Image dimensions overflowed a size computation. + #[error("dimension overflow: {width}x{height}")] + DimensionOverflow { + /// Image width. + width: u32, + /// Image height. + height: u32, + }, +} + +impl CodecError for J2kError { + fn is_truncated(&self) -> bool { + matches!( + self, + Self::Input(InputError::TooShort { .. } | InputError::TruncatedAt { .. }) + ) + } + + fn is_not_implemented(&self) -> bool { + matches!(self, Self::NotImplemented(_)) + } + + fn is_unsupported(&self) -> bool { + matches!(self, Self::Unsupported(_)) + } + + fn is_buffer_error(&self) -> bool { + matches!(self, Self::Buffer(_)) + } +} diff --git a/crates/j2k/src/lib.rs b/crates/j2k/src/lib.rs new file mode 100644 index 00000000..5c62e051 --- /dev/null +++ b/crates/j2k/src/lib.rs @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! JPEG 2000 inspect support for j2k. + +#![deny(missing_docs)] + +extern crate alloc; + +mod backend; +mod batch; +mod decode; +mod encode; +mod parallelism; +mod recode; + +/// Reusable JPEG 2000 decode context. +pub mod context; +pub use context::J2kContext; + +/// JPEG 2000 error type. +pub mod error; +pub use error::J2kError; + +/// Caller-owned JPEG 2000 scratch pool. +pub mod scratch; +pub use scratch::J2kScratchPool; + +/// Adapter-facing planning APIs shared with GPU crates. +/// +/// This module is public so device adapters can use the same route planning and +/// encode-stage contracts as the facade without depending on root re-exports. +pub mod adapter; + +/// Borrowed view and decoder entry points. +pub mod view; +pub use view::{J2kCodec, J2kDecoder, J2kRowDecodeOptions, J2kView}; + +pub use batch::{ + decode_tile_into_in_context, decode_tile_region_into_in_context, + decode_tile_region_scaled_into_in_context, decode_tile_scaled_into_in_context, + decode_tiles_into, decode_tiles_region_into, decode_tiles_region_scaled_into, + decode_tiles_scaled_into, TileBatchError, TileBatchOptions, TileDecodeJob, TileRegionDecodeJob, + TileRegionScaledDecodeJob, TileScaledDecodeJob, +}; + +pub use parallelism::CpuDecodeParallelism; + +pub use encode::{ + encode_j2k_lossless, encode_j2k_lossless_with_accelerator, encode_j2k_lossy, + encode_j2k_lossy_with_accelerator, j2k_lossless_decomposition_levels, + j2k_lossless_decomposition_levels_for_options, + j2k_lossless_decomposition_levels_for_progression, EncodeBackendPreference, EncodedJ2k, + EncodedLossyJ2k, J2kBlockCodingMode, J2kEncodeValidation, J2kLosslessEncodeOptions, + J2kLosslessSamples, J2kLossyEncodeOptions, J2kLossyEncodeReport, J2kLossySamples, + J2kMarkerSegment, J2kProgressionOrder, J2kQualityLayer, J2kRateTarget, ReversibleTransform, +}; + +pub use recode::{ + recode_j2k_to_htj2k_lossless, J2kToHtj2kMode, J2kToHtj2kOptions, J2kToHtj2kReport, + ReencodedHtj2k, +}; + +pub use j2k_core::{ + BackendKind, BackendRequest, BufferError, CodecError, CompressedPayloadKind, + CompressedTransferSyntax, DecodeOutcome, DecodeRowsError, DecoderContext, Downscale, + ImageCodec, ImageDecode, ImageDecodeRows, PassthroughCandidate, PassthroughDecision, + PassthroughRejectReason, PassthroughRequirements, PixelFormat, Rect, RowSink, TileBatchDecode, +}; + +pub(crate) mod parse; diff --git a/crates/j2k/src/parallelism.rs b/crates/j2k/src/parallelism.rs new file mode 100644 index 00000000..df50948e --- /dev/null +++ b/crates/j2k/src/parallelism.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 + +/// CPU parallelism policy for JPEG 2000 decode. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum CpuDecodeParallelism { + /// Allow a single tile decode to use internal code-block parallelism. + #[default] + Auto, + /// Keep code-block decode serial for callers that already parallelize tiles. + Serial, +} + +impl CpuDecodeParallelism { + pub(crate) const fn to_native(self) -> j2k_native::CpuDecodeParallelism { + match self { + Self::Auto => j2k_native::CpuDecodeParallelism::Auto, + Self::Serial => j2k_native::CpuDecodeParallelism::Serial, + } + } + + pub(crate) const fn from_native(parallelism: j2k_native::CpuDecodeParallelism) -> Self { + match parallelism { + j2k_native::CpuDecodeParallelism::Auto => Self::Auto, + j2k_native::CpuDecodeParallelism::Serial => Self::Serial, + } + } +} diff --git a/crates/signinum-j2k/src/parse/boxes.rs b/crates/j2k/src/parse/boxes.rs similarity index 94% rename from crates/signinum-j2k/src/parse/boxes.rs rename to crates/j2k/src/parse/boxes.rs index ed1162bd..176b14bb 100644 --- a/crates/signinum-j2k/src/parse/boxes.rs +++ b/crates/j2k/src/parse/boxes.rs @@ -2,7 +2,7 @@ use super::{codestream::parse_codestream, ParsedImageInfo}; use crate::J2kError; -use signinum_core::{Colorspace, CompressedPayloadKind, InputError}; +use j2k_core::{Colorspace, CompressedPayloadKind, InputError}; const JP2_SIGNATURE: [u8; 12] = [0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0D, 0x0A, 0x87, 0x0A]; const JP2_SIGNATURE_PREFIX: [u8; 8] = [0, 0, 0, 12, b'j', b'P', b' ', b' ']; @@ -109,10 +109,13 @@ pub(crate) fn parse_jp2(input: &[u8]) -> Result { } let codestream = codestream.ok_or(J2kError::MissingRequiredBox { box_type: "jp2c" })?; let parsed = parse_codestream(codestream)?; + let info = parsed.clone().into_info(colorspace); + let components = parsed.siz.component_info.clone(); Ok(ParsedImageInfo { - info: parsed.into_info(colorspace), + info, transfer_syntax: parsed.transfer_syntax(), payload_kind: CompressedPayloadKind::Jp2File, + components, }) } @@ -123,6 +126,13 @@ fn parse_jp2h(payload: &[u8], base_offset: usize) -> Result<(bool, Option payload.len() { + return Err(InputError::TruncatedAt { + offset: base_offset + offset, + segment: "box payload", + } + .into()); + } let inner = &payload[header.payload_start..header.end]; match &header.box_type { b"ihdr" => { diff --git a/crates/j2k/src/parse/codestream.rs b/crates/j2k/src/parse/codestream.rs new file mode 100644 index 00000000..7ecd0b7f --- /dev/null +++ b/crates/j2k/src/parse/codestream.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 + +use super::{ParsedCod, ParsedComponentInfo, ParsedSiz}; +use crate::J2kError; +use j2k_core::{InputError, TileLayout, Unsupported}; +use j2k_native::{ + inspect_j2k_codestream_header, looks_like_j2k_codestream, J2kCodestreamHeaderError, +}; + +#[derive(Debug, Clone)] +pub(crate) struct CodestreamInfo { + pub(crate) siz: ParsedSiz, + pub(crate) cod: ParsedCod, +} + +pub(crate) fn looks_like_codestream(input: &[u8]) -> bool { + looks_like_j2k_codestream(input) +} + +pub(crate) fn parse_codestream(input: &[u8]) -> Result { + let header = inspect_j2k_codestream_header(input).map_err(map_header_error)?; + let components = u8::try_from(header.components).map_err(|_| { + J2kError::Unsupported(Unsupported { + what: "component count > 255", + }) + })?; + let component_info = header + .component_info + .into_iter() + .map(|component| ParsedComponentInfo { + bit_depth: component.bit_depth, + signed: component.signed, + x_rsiz: component.x_rsiz, + y_rsiz: component.y_rsiz, + }) + .collect(); + + Ok(CodestreamInfo { + siz: ParsedSiz { + dimensions: header.dimensions, + components, + bit_depth: header.bit_depth, + tile_layout: TileLayout { + tile_width: header.tile_size.0, + tile_height: header.tile_size.1, + tiles_x: header.tile_count.0, + tiles_y: header.tile_count.1, + }, + component_info, + }, + cod: ParsedCod { + resolution_levels: header.resolution_levels, + has_mct: header.has_mct, + reversible: header.reversible, + high_throughput: header.high_throughput, + }, + }) +} + +fn map_header_error(error: J2kCodestreamHeaderError) -> J2kError { + match error { + J2kCodestreamHeaderError::TooShort { need, have } => { + InputError::TooShort { need, have }.into() + } + J2kCodestreamHeaderError::TruncatedAt { offset, segment } => { + InputError::TruncatedAt { offset, segment }.into() + } + J2kCodestreamHeaderError::InvalidMarker { offset, marker } => { + J2kError::InvalidMarker { offset, marker } + } + J2kCodestreamHeaderError::MissingRequiredMarker { marker } => { + J2kError::MissingRequiredMarker { marker } + } + J2kCodestreamHeaderError::InvalidSegment { offset, what } => { + J2kError::InvalidBox { offset, what } + } + J2kCodestreamHeaderError::InvalidSiz { what } => J2kError::InvalidSiz { what }, + J2kCodestreamHeaderError::InvalidCod { what } => J2kError::InvalidCod { what }, + J2kCodestreamHeaderError::Unsupported { what } => { + J2kError::Unsupported(Unsupported { what }) + } + error => J2kError::Backend(error.to_string()), + } +} diff --git a/crates/signinum-j2k/src/parse/mod.rs b/crates/j2k/src/parse/mod.rs similarity index 85% rename from crates/signinum-j2k/src/parse/mod.rs rename to crates/j2k/src/parse/mod.rs index c07dce5b..90092519 100644 --- a/crates/signinum-j2k/src/parse/mod.rs +++ b/crates/j2k/src/parse/mod.rs @@ -6,7 +6,7 @@ mod codestream; use self::boxes::parse_jp2; use self::codestream::{parse_codestream, CodestreamInfo}; use crate::J2kError; -use signinum_core::{ +use j2k_core::{ Colorspace, CompressedPayloadKind, CompressedTransferSyntax, Info, TileLayout, Unsupported, }; @@ -20,10 +20,13 @@ pub(crate) fn parse_image_info(input: &[u8]) -> Result, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ParsedComponentInfo { + pub(crate) bit_depth: u8, + pub(crate) signed: bool, + pub(crate) x_rsiz: u8, + pub(crate) y_rsiz: u8, } fn infer_colorspace(components: u8, has_mct: bool, reversible: bool) -> Colorspace { @@ -48,12 +60,13 @@ fn infer_colorspace(components: u8, has_mct: bool, reversible: bool) -> Colorspa } } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] struct ParsedSiz { dimensions: (u32, u32), components: u8, bit_depth: u8, tile_layout: TileLayout, + component_info: Vec, } #[derive(Debug, Clone, Copy)] diff --git a/crates/j2k/src/recode.rs b/crates/j2k/src/recode.rs new file mode 100644 index 00000000..a8de9e9f --- /dev/null +++ b/crates/j2k/src/recode.rs @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! JPEG 2000-family coefficient-domain recoding APIs. +//! +//! The direct 5/3 path decodes classic JPEG 2000 Tier-1 code-blocks into +//! reversible wavelet coefficients and re-encodes those coefficients with +//! HTJ2K block coding. It does not run inverse DWT, forward DWT, or a +//! pixel-domain lossless encode. The output is coefficient-preserving for the +//! supported reversible 5/3 profile, not byte-preserving unless passthrough is +//! reported. + +use alloc::vec::Vec; + +use j2k_core::{ + Colorspace, CompressedPayloadKind, CompressedTransferSyntax, PassthroughRequirements, + Unsupported, +}; +use j2k_native::{DecodeSettings, EncodeOptions, Image}; + +use crate::{ + encode::{native_progression_order, J2kEncodeValidation, J2kProgressionOrder}, + parse::{parse_image_info, ParsedImageInfo}, + J2kError, J2kView, +}; + +/// Options for classic JPEG 2000 reversible 5/3 to HTJ2K lossless recoding. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub struct J2kToHtj2kOptions { + /// Requested output payload shape. + /// + /// DICOM encapsulated WSI frames use raw JPEG 2000-family codestreams, so + /// the default is [`CompressedPayloadKind::Jpeg2000Codestream`]. JP2 output + /// is not produced by the coefficient-domain recoder yet. + pub output_payload_kind: CompressedPayloadKind, + /// Output packet progression order. + pub progression: J2kProgressionOrder, + /// Optional decoded-pixel validation of the produced codestream. + pub validation: J2kEncodeValidation, +} + +impl Default for J2kToHtj2kOptions { + fn default() -> Self { + Self { + output_payload_kind: CompressedPayloadKind::Jpeg2000Codestream, + progression: J2kProgressionOrder::Lrcp, + validation: J2kEncodeValidation::CpuRoundTrip, + } + } +} + +impl J2kToHtj2kOptions { + /// Create J2K/JP2 to HTJ2K recode options. + pub const fn new( + output_payload_kind: CompressedPayloadKind, + progression: J2kProgressionOrder, + validation: J2kEncodeValidation, + ) -> Self { + Self { + output_payload_kind, + progression, + validation, + } + } +} + +/// Recode path used for a J2K/JP2 to HTJ2K request. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum J2kToHtj2kMode { + /// Input bytes already matched the requested HTJ2K transfer syntax and + /// payload kind, so bytes were copied unchanged. + Passthrough, + /// Classic reversible 5/3 code-blocks were entropy-decoded to quantized + /// wavelet coefficients and re-encoded with HT block coding. + CoefficientPreserving, + /// Reserved for an explicit decode-pixels/re-encode fallback. + PixelPreserving, +} + +/// Metadata describing a J2K/JP2 to HTJ2K recode. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct J2kToHtj2kReport { + /// Recode path used for this output. + pub mode: J2kToHtj2kMode, + /// Classified input transfer syntax. + pub input_transfer_syntax: CompressedTransferSyntax, + /// Output transfer syntax. + pub output_transfer_syntax: CompressedTransferSyntax, + /// Classified input payload/container kind. + pub input_payload_kind: CompressedPayloadKind, + /// Output payload/container kind. + pub output_payload_kind: CompressedPayloadKind, + /// Image width in pixels. + pub width: u32, + /// Image height in pixels. + pub height: u32, + /// Component count. + pub components: u8, + /// Significant bits per component. + pub bit_depth: u8, +} + +/// HTJ2K codestream bytes and recode metadata. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReencodedHtj2k { + /// Encoded HTJ2K bytes. + pub bytes: Vec, + /// Recode metadata and selected path. + pub report: J2kToHtj2kReport, +} + +/// Recode a classic JPEG 2000 reversible 5/3 J2K/JP2 input to lossless HTJ2K. +/// +/// This is a JPEG 2000-family coefficient-domain recode. For supported classic +/// lossless 5/3 sources it preserves decoded quantized wavelet coefficients and +/// changes only the block coding and packetized codestream representation. It +/// is not a DCT JPEG transcode and does not claim byte preservation except when +/// [`J2kToHtj2kMode::Passthrough`] is reported. +pub fn recode_j2k_to_htj2k_lossless( + bytes: &[u8], + options: J2kToHtj2kOptions, +) -> Result { + let view = J2kView::parse(bytes)?; + let parsed = parse_image_info(bytes)?; + let info = view.info().clone(); + let output_transfer_syntax = CompressedTransferSyntax::HtJpeg2000Lossless; + + if let Some(candidate) = view.passthrough_candidate() { + let requirements = + PassthroughRequirements::new(output_transfer_syntax, options.output_payload_kind); + if let Ok(copy) = candidate.copy_bytes_if_eligible(&requirements) { + return Ok(ReencodedHtj2k { + bytes: copy.to_vec(), + report: J2kToHtj2kReport { + mode: J2kToHtj2kMode::Passthrough, + input_transfer_syntax: candidate.transfer_syntax(), + output_transfer_syntax, + input_payload_kind: candidate.payload_kind(), + output_payload_kind: options.output_payload_kind, + width: info.dimensions.0, + height: info.dimensions.1, + components: info.components, + bit_depth: info.bit_depth, + }, + }); + } + } + + validate_recode_request(&parsed, options)?; + + let source = Image::new(bytes, &DecodeSettings::default()) + .map_err(|err| map_native_decode_error(err, "source JPEG 2000 parse failed"))?; + let coefficients = source + .decode_reversible_53_coefficients() + .map_err(|err| map_native_decode_error(err, "source coefficient extraction failed"))?; + + let encode_options = native_encode_options(options, &coefficients); + let codestream = j2k_native::encode_precomputed_htj2k_53_with_mct( + &coefficients.image, + &encode_options, + coefficients.use_mct, + ) + .map_err(|err| J2kError::Backend(format!("HTJ2K coefficient recode failed: {err}")))?; + + if options.validation == J2kEncodeValidation::CpuRoundTrip { + validate_recode_roundtrip(bytes, &codestream)?; + } + + Ok(ReencodedHtj2k { + bytes: codestream, + report: J2kToHtj2kReport { + mode: J2kToHtj2kMode::CoefficientPreserving, + input_transfer_syntax: parsed.transfer_syntax, + output_transfer_syntax, + input_payload_kind: parsed.payload_kind, + output_payload_kind: options.output_payload_kind, + width: parsed.info.dimensions.0, + height: parsed.info.dimensions.1, + components: parsed.info.components, + bit_depth: parsed.info.bit_depth, + }, + }) +} + +fn validate_recode_request( + parsed: &ParsedImageInfo, + options: J2kToHtj2kOptions, +) -> Result<(), J2kError> { + if options.output_payload_kind != CompressedPayloadKind::Jpeg2000Codestream { + return Err(Unsupported { + what: "coefficient-domain J2K to HTJ2K recode currently emits only raw codestreams", + } + .into()); + } + if parsed.transfer_syntax != CompressedTransferSyntax::Jpeg2000Lossless { + return Err(Unsupported { + what: "coefficient-domain lossless recode currently supports only classic lossless J2K", + } + .into()); + } + if !matches!(parsed.info.components, 1 | 3) { + return Err(Unsupported { + what: "coefficient-domain lossless recode supports only grayscale or RGB component counts", + } + .into()); + } + if !matches!(parsed.info.bit_depth, 8 | 16) { + return Err(Unsupported { + what: "coefficient-domain lossless recode supports only 8-bit or 16-bit sources", + } + .into()); + } + if !matches!( + parsed.info.colorspace, + Colorspace::Grayscale + | Colorspace::SGray + | Colorspace::Rgb + | Colorspace::SRgb + | Colorspace::Rct + ) { + return Err(Unsupported { + what: "coefficient-domain lossless recode supports only Gray/RGB/RCT colorspaces", + } + .into()); + } + if parsed.components.iter().any(|component| component.signed) { + return Err(Unsupported { + what: "signed JPEG 2000 sources are not supported for coefficient-domain recode yet", + } + .into()); + } + if parsed + .components + .iter() + .any(|component| component.bit_depth != parsed.info.bit_depth) + { + return Err(Unsupported { + what: "mixed component bit depths are not supported for coefficient-domain recode", + } + .into()); + } + if parsed + .components + .iter() + .any(|component| component.x_rsiz != 1 || component.y_rsiz != 1) + { + return Err(Unsupported { + what: "component subsampling is not supported for coefficient-domain recode yet", + } + .into()); + } + Ok(()) +} + +fn native_encode_options( + options: J2kToHtj2kOptions, + coefficients: &j2k_native::Reversible53CoefficientImage, +) -> EncodeOptions { + EncodeOptions { + reversible: true, + use_ht_block_coding: true, + use_mct: coefficients.use_mct, + code_block_width_exp: coefficients.code_block_width_exp, + code_block_height_exp: coefficients.code_block_height_exp, + guard_bits: coefficients.guard_bits, + progression_order: native_progression_order(options.progression), + write_tlm: options.progression == J2kProgressionOrder::Rpcl, + validate_high_throughput_codestream: false, + ..EncodeOptions::default() + } +} + +fn validate_recode_roundtrip(source: &[u8], encoded: &[u8]) -> Result<(), J2kError> { + let source = Image::new(source, &DecodeSettings::default()) + .map_err(|err| map_native_decode_error(err, "source JPEG 2000 validation parse failed"))? + .decode_native() + .map_err(|err| map_native_decode_error(err, "source JPEG 2000 validation decode failed"))?; + let encoded = Image::new(encoded, &DecodeSettings::default()) + .map_err(|err| map_native_decode_error(err, "HTJ2K validation parse failed"))? + .decode_native() + .map_err(|err| map_native_decode_error(err, "HTJ2K validation decode failed"))?; + + if source.width != encoded.width + || source.height != encoded.height + || source.bit_depth != encoded.bit_depth + || source.num_components != encoded.num_components + || source.data != encoded.data + { + return Err(J2kError::Backend( + "HTJ2K coefficient recode failed pixel validation".to_string(), + )); + } + Ok(()) +} + +fn map_native_decode_error(err: j2k_native::DecodeError, context: &'static str) -> J2kError { + match err { + j2k_native::DecodeError::Decoding(j2k_native::DecodingError::UnsupportedFeature(what)) => { + J2kError::Unsupported(Unsupported { what }) + } + _ => J2kError::Backend(format!("{context}: {err}")), + } +} diff --git a/crates/signinum-j2k/src/scratch.rs b/crates/j2k/src/scratch.rs similarity index 91% rename from crates/signinum-j2k/src/scratch.rs rename to crates/j2k/src/scratch.rs index 1a824e37..3777713b 100644 --- a/crates/signinum-j2k/src/scratch.rs +++ b/crates/j2k/src/scratch.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 use alloc::vec::Vec; -use signinum_core::ScratchPool; +use j2k_core::ScratchPool; -/// Caller-owned reusable scratch for `signinum-j2k`. +/// Caller-owned reusable scratch for `j2k`. #[derive(Debug, Default, PartialEq, Eq)] pub struct J2kScratchPool { packed_bytes: Vec, @@ -11,6 +11,7 @@ pub struct J2kScratchPool { } impl J2kScratchPool { + /// Create an empty JPEG 2000 scratch pool. pub const fn new() -> Self { Self { packed_bytes: Vec::new(), diff --git a/crates/signinum-j2k/src/view.rs b/crates/j2k/src/view.rs similarity index 56% rename from crates/signinum-j2k/src/view.rs rename to crates/j2k/src/view.rs index 78e68365..04556811 100644 --- a/crates/signinum-j2k/src/view.rs +++ b/crates/j2k/src/view.rs @@ -7,19 +7,18 @@ use crate::{ context::J2kContext, decode::{ decode_image_into_with_native_context, decode_image_region_into_with_native_context, - decode_region_scaled_from_info, decode_scaled_from_info, validate_buffer, validate_region, - validate_supported_format, J2kDecodeOutcome, + validate_buffer, validate_region, J2kDecodeOutcome, }, parse::{parse_image_info, parse_info}, scratch::J2kScratchPool, - J2kError, + CpuDecodeParallelism, J2kError, }; use alloc::vec::Vec; use core::convert::Infallible; -use signinum_core::{ - CompressedPayloadKind, CompressedTransferSyntax, DecodeRowsError, DecoderContext, Downscale, - ImageCodec, ImageDecode, ImageDecodeRows, Info, PassthroughCandidate, PixelFormat, Rect, - RowSink, TileBatchDecode, +use j2k_core::{ + BufferError, CompressedPayloadKind, CompressedTransferSyntax, DecodeRowsError, DecoderContext, + Downscale, ImageCodec, ImageDecode, ImageDecodeRows, Info, PassthroughCandidate, PixelFormat, + Rect, RowSink, TileBatchDecode, DEFAULT_MAX_HOST_ALLOCATION_BYTES, }; /// Borrowed parse result for a JP2 or raw JPEG 2000 / HTJ2K codestream. @@ -85,10 +84,67 @@ pub struct J2kDecoder<'a> { bytes: &'a [u8], info: Info, image: Option>, - native_context: signinum_j2k_native::DecoderContext<'a>, + native_context: j2k_native::DecoderContext<'a>, passthrough: Option<(CompressedTransferSyntax, CompressedPayloadKind)>, } +/// Options for bounded J2K row decoding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct J2kRowDecodeOptions { + max_rows_per_stripe: u32, + max_stripe_bytes: usize, +} + +impl J2kRowDecodeOptions { + /// Create row decode options with the requested maximum stripe height. + /// + /// A zero value is normalized to one row. + pub const fn new(max_rows_per_stripe: u32) -> Self { + Self { + max_rows_per_stripe, + max_stripe_bytes: DEFAULT_MAX_HOST_ALLOCATION_BYTES, + } + } + + /// Create row decode options with explicit stripe height and byte caps. + pub const fn new_with_max_stripe_bytes( + max_rows_per_stripe: u32, + max_stripe_bytes: usize, + ) -> Self { + Self { + max_rows_per_stripe, + max_stripe_bytes, + } + } + + /// Return a copy of these options with an explicit stripe byte cap. + #[must_use] + pub const fn with_max_stripe_bytes(mut self, max_stripe_bytes: usize) -> Self { + self.max_stripe_bytes = max_stripe_bytes; + self + } + + /// Maximum number of decoded rows held per bounded row-decode stripe. + pub const fn max_rows_per_stripe(self) -> u32 { + if self.max_rows_per_stripe == 0 { + 1 + } else { + self.max_rows_per_stripe + } + } + + /// Maximum number of packed bytes held per bounded row-decode stripe. + pub const fn max_stripe_bytes(self) -> usize { + self.max_stripe_bytes + } +} + +impl Default for J2kRowDecodeOptions { + fn default() -> Self { + Self::new(64) + } +} + #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] /// Marker type used by generic tile-batch decode traits. pub struct J2kCodec; @@ -124,7 +180,7 @@ impl<'a> J2kDecoder<'a> { bytes: view.bytes, info: view.info, image: view.image, - native_context: signinum_j2k_native::DecoderContext::default(), + native_context: j2k_native::DecoderContext::default(), passthrough: view.passthrough, }) } @@ -134,17 +190,15 @@ impl<'a> J2kDecoder<'a> { &self.info } - /// Return the native CPU decode parallelism policy for this decoder. - pub fn cpu_decode_parallelism(&self) -> signinum_j2k_native::CpuDecodeParallelism { - self.native_context.cpu_decode_parallelism() + /// Return the CPU decode parallelism policy for this decoder. + pub fn cpu_decode_parallelism(&self) -> CpuDecodeParallelism { + CpuDecodeParallelism::from_native(self.native_context.cpu_decode_parallelism()) } - /// Set the native CPU decode parallelism policy for this decoder. - pub fn set_cpu_decode_parallelism( - &mut self, - parallelism: signinum_j2k_native::CpuDecodeParallelism, - ) { - self.native_context.set_cpu_decode_parallelism(parallelism); + /// Set the CPU decode parallelism policy for this decoder. + pub fn set_cpu_decode_parallelism(&mut self, parallelism: CpuDecodeParallelism) { + self.native_context + .set_cpu_decode_parallelism(parallelism.to_native()); } /// Original compressed bytes backing this decoder. @@ -198,7 +252,6 @@ impl<'a> J2kDecoder<'a> { stride: usize, fmt: PixelFormat, ) -> Result { - validate_supported_format(fmt)?; validate_buffer(self.info.dimensions, out.len(), stride, fmt)?; self.ensure_image()?; let (Some(image), native_context) = (self.image.as_ref(), &mut self.native_context) else { @@ -207,10 +260,10 @@ impl<'a> J2kDecoder<'a> { )); }; decode_image_into_with_native_context(image, native_context, out, stride, fmt)?; - Ok(signinum_core::DecodeOutcome { - decoded: Rect::full(self.info.dimensions), - warnings: Vec::new(), - }) + Ok(j2k_core::DecodeOutcome::new( + Rect::full(self.info.dimensions), + Vec::new(), + )) } /// Decode a source-coordinate region into `out`. @@ -240,7 +293,6 @@ impl<'a> J2kDecoder<'a> { fmt: PixelFormat, roi: Rect, ) -> Result { - validate_supported_format(fmt)?; validate_region(roi, self.info.dimensions)?; validate_buffer((roi.w, roi.h), out.len(), stride, fmt)?; self.ensure_image()?; @@ -250,10 +302,7 @@ impl<'a> J2kDecoder<'a> { )); }; decode_image_region_into_with_native_context(image, native_context, out, stride, fmt, roi)?; - Ok(signinum_core::DecodeOutcome { - decoded: roi, - warnings: Vec::new(), - }) + Ok(j2k_core::DecodeOutcome::new(roi, Vec::new())) } /// Decode the full image at a reduced resolution. @@ -275,17 +324,30 @@ impl<'a> J2kDecoder<'a> { if scale == Downscale::None { return self.decode_into_with_scratch(pool, out, stride, fmt); } - decode_scaled_from_info( - self.bytes, - self.info.dimensions, - pool, - out, - stride, - fmt, - scale, - ) - } - + let settings = DecodeSettings { + target_resolution: Some(self.scaled_target_dims(scale)), + ..DecodeSettings::default() + }; + let image = backend_image(self.bytes, settings)?; + let image_dims = (image.width(), image.height()); + validate_buffer(image_dims, out.len(), stride, fmt)?; + let mut native_context = self.scaled_decode_native_context(); + decode_image_into_with_native_context(&image, &mut native_context, out, stride, fmt)?; + Ok(j2k_core::DecodeOutcome::new( + Rect::full(image_dims), + Vec::new(), + )) + } + + /// Decode a source-coordinate region at a reduced resolution. + /// + /// `roi` is expressed in full-resolution source pixels. The decoded output + /// covers `roi.scaled_covering(scale)` in reduced-resolution coordinates. + /// + /// # Errors + /// Returns [`J2kError`] when the region is out of bounds, the scale or + /// pixel format is unsupported, the output buffer is too small, or decode + /// fails. pub fn decode_region_scaled_into( &mut self, pool: &mut J2kScratchPool, @@ -298,16 +360,26 @@ impl<'a> J2kDecoder<'a> { if scale == Downscale::None { return self.decode_region_into(pool, out, stride, fmt, roi); } - decode_region_scaled_from_info( - self.bytes, - self.info.dimensions, - pool, + validate_region(roi, self.info.dimensions)?; + let scaled_roi = roi.scaled_covering(scale); + validate_buffer((scaled_roi.w, scaled_roi.h), out.len(), stride, fmt)?; + let settings = DecodeSettings { + target_resolution: Some(self.scaled_target_dims(scale)), + ..DecodeSettings::default() + }; + let image = backend_image(self.bytes, settings)?; + let image_dims = (image.width(), image.height()); + validate_region(scaled_roi, image_dims)?; + let mut native_context = self.scaled_decode_native_context(); + decode_image_region_into_with_native_context( + &image, + &mut native_context, out, stride, fmt, - roi, - scale, - ) + scaled_roi, + )?; + Ok(j2k_core::DecodeOutcome::new(scaled_roi, Vec::new())) } fn ensure_image(&mut self) -> Result<(), J2kError> { @@ -325,6 +397,128 @@ impl<'a> J2kDecoder<'a> { .as_ref() .ok_or_else(|| J2kError::Backend("internal image cache missing".to_string())) } + + fn scaled_target_dims(&self, scale: Downscale) -> (u32, u32) { + ( + self.info.dimensions.0.div_ceil(scale.denominator()), + self.info.dimensions.1.div_ceil(scale.denominator()), + ) + } + + fn scaled_decode_native_context(&self) -> j2k_native::DecoderContext<'a> { + let mut native_context = j2k_native::DecoderContext::default(); + native_context.set_cpu_decode_parallelism(self.native_context.cpu_decode_parallelism()); + native_context + } + + /// Decode rows into a `u8` row sink while bounding host output scratch to + /// at most `options.max_rows_per_stripe()` rows. + /// + /// # Errors + /// Returns a decode error for unsupported formats or malformed input, and + /// forwards sink errors without converting them to successful decodes. + pub fn decode_rows_u8_bounded>( + &mut self, + sink: &mut R, + options: J2kRowDecodeOptions, + ) -> Result> { + let fmt = row_format_u8(self.info()).map_err(DecodeRowsError::Decode)?; + let row_bytes = row_bytes_for(self.info(), fmt).map_err(DecodeRowsError::Decode)?; + let width = self.info.dimensions.0; + let height = self.info.dimensions.1; + let (stripe_rows, max_stripe_len) = bounded_row_stripe_layout(row_bytes, height, options) + .map_err(DecodeRowsError::Decode)?; + let mut pool = J2kScratchPool::new(); + let mut y = 0_u32; + while y < height { + let rows = stripe_rows.min(height - y); + let stripe_len = row_bytes.checked_mul(rows as usize).ok_or_else(|| { + DecodeRowsError::Decode(J2kError::Buffer(BufferError::SizeOverflow { + what: "J2K bounded row decode stripe buffer", + })) + })?; + let stripe = pool.packed_bytes(max_stripe_len); + self.decode_region_into_cached( + &mut stripe[..stripe_len], + row_bytes, + fmt, + Rect { + x: 0, + y, + w: width, + h: rows, + }, + ) + .map_err(DecodeRowsError::Decode)?; + for row_index in 0..rows { + let start = row_index as usize * row_bytes; + sink.write_row(y + row_index, &stripe[start..start + row_bytes]) + .map_err(DecodeRowsError::Sink)?; + } + y += rows; + } + Ok(j2k_core::DecodeOutcome::new( + Rect::full(self.info.dimensions), + Vec::new(), + )) + } + + /// Decode rows into a `u16` row sink while bounding host output scratch to + /// at most `options.max_rows_per_stripe()` rows. + /// + /// # Errors + /// Returns a decode error for unsupported formats or malformed input, and + /// forwards sink errors without converting them to successful decodes. + pub fn decode_rows_u16_bounded>( + &mut self, + sink: &mut R, + options: J2kRowDecodeOptions, + ) -> Result> { + let fmt = row_format_u16(self.info()).map_err(DecodeRowsError::Decode)?; + let row_bytes = row_bytes_for(self.info(), fmt).map_err(DecodeRowsError::Decode)?; + let samples_per_row = row_samples_for(self.info(), fmt).map_err(DecodeRowsError::Decode)?; + let width = self.info.dimensions.0; + let height = self.info.dimensions.1; + let (stripe_rows, max_stripe_len) = bounded_row_stripe_layout(row_bytes, height, options) + .map_err(DecodeRowsError::Decode)?; + let mut pool = J2kScratchPool::new(); + let mut y = 0_u32; + while y < height { + let rows = stripe_rows.min(height - y); + let stripe_len = row_bytes.checked_mul(rows as usize).ok_or_else(|| { + DecodeRowsError::Decode(J2kError::Buffer(BufferError::SizeOverflow { + what: "J2K bounded row decode stripe buffer", + })) + })?; + let (packed, row) = pool.packed_bytes_and_row_u16(max_stripe_len, samples_per_row); + self.decode_region_into_cached( + &mut packed[..stripe_len], + row_bytes, + fmt, + Rect { + x: 0, + y, + w: width, + h: rows, + }, + ) + .map_err(DecodeRowsError::Decode)?; + for row_index in 0..rows { + let start = row_index as usize * row_bytes; + let packed_row = &packed[start..start + row_bytes]; + for (dst, src) in row.iter_mut().zip(packed_row.chunks_exact(2)) { + *dst = u16::from_le_bytes([src[0], src[1]]); + } + sink.write_row(y + row_index, row) + .map_err(DecodeRowsError::Sink)?; + } + y += rows; + } + Ok(j2k_core::DecodeOutcome::new( + Rect::full(self.info.dimensions), + Vec::new(), + )) + } } impl ImageCodec for J2kDecoder<'_> { @@ -353,7 +547,7 @@ impl<'a> ImageDecode<'a> for J2kDecoder<'a> { out: &mut [u8], stride: usize, fmt: PixelFormat, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { J2kDecoder::decode_into(self, out, stride, fmt) } @@ -363,7 +557,7 @@ impl<'a> ImageDecode<'a> for J2kDecoder<'a> { out: &mut [u8], stride: usize, fmt: PixelFormat, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { J2kDecoder::decode_into_with_scratch(self, pool, out, stride, fmt) } @@ -374,7 +568,7 @@ impl<'a> ImageDecode<'a> for J2kDecoder<'a> { stride: usize, fmt: PixelFormat, roi: Rect, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { J2kDecoder::decode_region_into(self, pool, out, stride, fmt, roi) } @@ -385,7 +579,7 @@ impl<'a> ImageDecode<'a> for J2kDecoder<'a> { stride: usize, fmt: PixelFormat, scale: Downscale, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { J2kDecoder::decode_scaled_into(self, pool, out, stride, fmt, scale) } @@ -397,7 +591,7 @@ impl<'a> ImageDecode<'a> for J2kDecoder<'a> { fmt: PixelFormat, roi: Rect, scale: Downscale, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { J2kDecoder::decode_region_scaled_into(self, pool, out, stride, fmt, roi, scale) } } @@ -406,31 +600,9 @@ impl<'a> ImageDecodeRows<'a, u8> for J2kDecoder<'a> { fn decode_rows>( &mut self, sink: &mut R, - ) -> Result, DecodeRowsError> + ) -> Result, DecodeRowsError> { - let fmt = row_format_u8(self.info()).map_err(DecodeRowsError::Decode)?; - let row_bytes = row_bytes_for(self.info(), fmt).map_err(DecodeRowsError::Decode)?; - let mut pool = J2kScratchPool::new(); - let row = pool.packed_bytes(row_bytes); - for y in 0..self.info.dimensions.1 { - self.decode_region_into_cached( - row, - row_bytes, - fmt, - Rect { - x: 0, - y, - w: self.info.dimensions.0, - h: 1, - }, - ) - .map_err(DecodeRowsError::Decode)?; - sink.write_row(y, row).map_err(DecodeRowsError::Sink)?; - } - Ok(signinum_core::DecodeOutcome { - decoded: Rect::full(self.info.dimensions), - warnings: Vec::new(), - }) + self.decode_rows_u8_bounded(sink, J2kRowDecodeOptions::default()) } } @@ -438,35 +610,9 @@ impl<'a> ImageDecodeRows<'a, u16> for J2kDecoder<'a> { fn decode_rows>( &mut self, sink: &mut R, - ) -> Result, DecodeRowsError> + ) -> Result, DecodeRowsError> { - let fmt = row_format_u16(self.info()).map_err(DecodeRowsError::Decode)?; - let row_bytes = row_bytes_for(self.info(), fmt).map_err(DecodeRowsError::Decode)?; - let samples_per_row = row_samples_for(self.info(), fmt).map_err(DecodeRowsError::Decode)?; - let mut pool = J2kScratchPool::new(); - let (packed, row) = pool.packed_bytes_and_row_u16(row_bytes, samples_per_row); - for y in 0..self.info.dimensions.1 { - self.decode_region_into_cached( - packed, - row_bytes, - fmt, - Rect { - x: 0, - y, - w: self.info.dimensions.0, - h: 1, - }, - ) - .map_err(DecodeRowsError::Decode)?; - for (dst, src) in row.iter_mut().zip(packed.chunks_exact(2)) { - *dst = u16::from_le_bytes([src[0], src[1]]); - } - sink.write_row(y, row).map_err(DecodeRowsError::Sink)?; - } - Ok(signinum_core::DecodeOutcome { - decoded: Rect::full(self.info.dimensions), - warnings: Vec::new(), - }) + self.decode_rows_u16_bounded(sink, J2kRowDecodeOptions::default()) } } @@ -486,8 +632,7 @@ impl TileBatchDecode for J2kCodec { out: &mut [u8], stride: usize, fmt: PixelFormat, - ) -> Result, Self::Error> { - ctx.codec_mut().record_tile_decode(); + ) -> Result, Self::Error> { let mut decoder = J2kDecoder::new(input)?; decoder.set_cpu_decode_parallelism(ctx.codec().cpu_decode_parallelism()); decoder.decode_into_with_scratch(pool, out, stride, fmt) @@ -501,8 +646,7 @@ impl TileBatchDecode for J2kCodec { stride: usize, fmt: PixelFormat, roi: Rect, - ) -> Result, Self::Error> { - ctx.codec_mut().record_tile_decode(); + ) -> Result, Self::Error> { let mut decoder = J2kDecoder::new(input)?; decoder.set_cpu_decode_parallelism(ctx.codec().cpu_decode_parallelism()); decoder.decode_region_into(pool, out, stride, fmt, roi) @@ -516,8 +660,7 @@ impl TileBatchDecode for J2kCodec { stride: usize, fmt: PixelFormat, scale: Downscale, - ) -> Result, Self::Error> { - ctx.codec_mut().record_tile_decode(); + ) -> Result, Self::Error> { let mut decoder = J2kDecoder::new(input)?; decoder.set_cpu_decode_parallelism(ctx.codec().cpu_decode_parallelism()); decoder.decode_scaled_into(pool, out, stride, fmt, scale) @@ -532,8 +675,7 @@ impl TileBatchDecode for J2kCodec { fmt: PixelFormat, roi: Rect, scale: Downscale, - ) -> Result, Self::Error> { - ctx.codec_mut().record_tile_decode(); + ) -> Result, Self::Error> { let mut decoder = J2kDecoder::new(input)?; decoder.set_cpu_decode_parallelism(ctx.codec().cpu_decode_parallelism()); decoder.decode_region_scaled_into(pool, out, stride, fmt, roi, scale) @@ -545,7 +687,7 @@ fn row_format_u8(info: &Info) -> Result { 1 => Ok(PixelFormat::Gray8), 3 => Ok(PixelFormat::Rgb8), 4 => Ok(PixelFormat::Rgba8), - _ => Err(signinum_core::Unsupported { + _ => Err(j2k_core::Unsupported { what: "row decode only supports Gray/RGB/RGBA images in J2K-M2", } .into()), @@ -556,17 +698,43 @@ fn row_format_u16(info: &Info) -> Result { match info.components { 1 => Ok(PixelFormat::Gray16), 3 => Ok(PixelFormat::Rgb16), - 4 => Err(signinum_core::Unsupported { - what: "Rgba16 row decode is not supported by signinum-j2k", - } - .into()), - _ => Err(signinum_core::Unsupported { - what: "row decode only supports Gray/RGB images in J2K-M2", + 4 => Ok(PixelFormat::Rgba16), + _ => Err(j2k_core::Unsupported { + what: "row decode only supports Gray/RGB/RGBA images in J2K-M2", } .into()), } } +fn bounded_row_stripe_layout( + row_bytes: usize, + height: u32, + options: J2kRowDecodeOptions, +) -> Result<(u32, usize), J2kError> { + let cap = options.max_stripe_bytes(); + if row_bytes > cap { + return Err(J2kError::Buffer(BufferError::AllocationTooLarge { + requested: row_bytes, + cap, + what: "J2K bounded row decode stripe buffer", + })); + } + + let max_rows = options.max_rows_per_stripe(); + let image_rows = height.max(1); + let rows_by_cap = cap.checked_div(row_bytes).map_or(max_rows, |capped_rows| { + u32::try_from(capped_rows).unwrap_or(u32::MAX).max(1) + }); + let stripe_rows = max_rows.min(image_rows).min(rows_by_cap); + let max_stripe_len = row_bytes + .checked_mul(stripe_rows as usize) + .ok_or(J2kError::Buffer(BufferError::SizeOverflow { + what: "J2K bounded row decode stripe buffer", + }))?; + + Ok((stripe_rows, max_stripe_len)) +} + fn row_bytes_for(info: &Info, fmt: PixelFormat) -> Result { (info.dimensions.0 as usize) .checked_mul(fmt.bytes_per_pixel()) @@ -609,3 +777,63 @@ fn should_retry_with_backend(error: &J2kError) -> bool { } ) } + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn scaled_decode_native_context_preserves_configured_parallelism() { + let mut decoder = J2kDecoder { + bytes: &[], + info: Info { + dimensions: (1, 1), + components: 1, + colorspace: j2k_core::Colorspace::SGray, + bit_depth: 8, + tile_layout: None, + coded_unit_layout: None, + restart_interval: None, + resolution_levels: 1, + }, + image: None, + native_context: j2k_native::DecoderContext::default(), + passthrough: None, + }; + decoder.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); + + let native_context = decoder.scaled_decode_native_context(); + + assert_eq!( + native_context.cpu_decode_parallelism(), + CpuDecodeParallelism::Serial.to_native() + ); + } + + #[test] + fn bounded_row_stripe_layout_clamps_rows_to_byte_cap() { + let options = J2kRowDecodeOptions::new_with_max_stripe_bytes(100, 1_024); + + let (rows, bytes) = + bounded_row_stripe_layout(100, 50, options).expect("stripe layout should fit"); + + assert_eq!(rows, 10); + assert_eq!(bytes, 1_000); + } + + #[test] + fn bounded_row_stripe_layout_rejects_single_row_over_cap() { + let options = J2kRowDecodeOptions::new_with_max_stripe_bytes(100, 99); + + let err = + bounded_row_stripe_layout(100, 50, options).expect_err("single row should exceed cap"); + + assert!(matches!( + err, + J2kError::Buffer(BufferError::AllocationTooLarge { + requested: 100, + cap: 99, + what: "J2K bounded row decode stripe buffer", + }) + )); + } +} diff --git a/crates/j2k/tests/adaptive_route.rs b/crates/j2k/tests/adaptive_route.rs new file mode 100644 index 00000000..24cfc236 --- /dev/null +++ b/crates/j2k/tests/adaptive_route.rs @@ -0,0 +1,423 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::adapter::adaptive_route::{ + J2kAdaptiveBackendRequest, J2kAdaptiveBenchmarkEvidence, J2kAdaptiveBenchmarks, + J2kAdaptiveCodecMode, J2kAdaptiveOperation, J2kAdaptiveOutputResidency, J2kAdaptiveQualityMode, + J2kAdaptiveRcaFinding, J2kAdaptiveRcaReason, J2kAdaptiveRouteKind, J2kAdaptiveRoutePlanner, + J2kAdaptiveStage, J2kAdaptiveStageGateStatus, J2kAdaptiveStageOwner, J2kAdaptiveWorkload, +}; +use j2k::{EncodeBackendPreference, J2kLosslessEncodeOptions, J2kLossyEncodeOptions}; +use j2k_core::{BackendCapabilities, BackendKind, CpuFeatures}; + +fn metal_caps() -> BackendCapabilities { + BackendCapabilities { + cpu: CpuFeatures { + avx2: false, + sse41: false, + neon: true, + }, + metal: true, + cuda: false, + } +} + +fn metal_cuda_caps() -> BackendCapabilities { + BackendCapabilities { + cpu: CpuFeatures { + avx2: false, + sse41: false, + neon: true, + }, + metal: true, + cuda: true, + } +} + +fn cpu_caps() -> BackendCapabilities { + BackendCapabilities { + cpu: CpuFeatures::default(), + metal: false, + cuda: false, + } +} + +fn rgb_wsi_htj2k_encode() -> J2kAdaptiveWorkload { + J2kAdaptiveWorkload::new( + J2kAdaptiveOperation::Encode, + J2kAdaptiveCodecMode::Htj2k, + J2kAdaptiveQualityMode::Lossless, + 3, + 8, + (512, 512), + 16, + ) + .with_output_residency(J2kAdaptiveOutputResidency::Host) +} + +fn approved_metal_benchmarks_for(workload: J2kAdaptiveWorkload) -> J2kAdaptiveBenchmarks { + let mut benchmarks = J2kAdaptiveBenchmarks::default(); + for stage in J2kAdaptiveStage::ALL { + if workload.logical_owner_for(stage) == J2kAdaptiveStageOwner::Gpu { + benchmarks.push_stage(J2kAdaptiveBenchmarkEvidence::stage( + stage, + BackendKind::Metal, + 100_000, + 80_000, + 1.0, + )); + } + } + benchmarks.push_end_to_end(J2kAdaptiveBenchmarkEvidence::end_to_end( + BackendKind::Metal, + 2_000_000, + 1_600_000, + 1.0, + )); + benchmarks +} + +fn approved_cuda_benchmarks_for(workload: J2kAdaptiveWorkload) -> J2kAdaptiveBenchmarks { + let mut benchmarks = J2kAdaptiveBenchmarks::default(); + for stage in J2kAdaptiveStage::ALL { + if workload.logical_owner_for(stage) == J2kAdaptiveStageOwner::Gpu { + benchmarks.push_stage(J2kAdaptiveBenchmarkEvidence::stage( + stage, + BackendKind::Cuda, + 100_000, + 80_000, + 1.0, + )); + } + } + benchmarks.push_end_to_end(J2kAdaptiveBenchmarkEvidence::end_to_end( + BackendKind::Cuda, + 2_000_000, + 1_500_000, + 1.0, + )); + benchmarks +} + +fn metal_stage_candidate_benchmarks_for(stage: J2kAdaptiveStage) -> J2kAdaptiveBenchmarks { + let mut benchmarks = J2kAdaptiveBenchmarks::default(); + benchmarks.push_stage(J2kAdaptiveBenchmarkEvidence::stage( + stage, + BackendKind::Metal, + 100_000, + 70_000, + 1.0, + )); + benchmarks +} + +#[test] +fn encode_backend_preference_helpers_select_clear_routes() { + assert_eq!( + J2kLosslessEncodeOptions::default() + .with_accelerated_backend() + .backend, + EncodeBackendPreference::Auto + ); + assert_eq!( + J2kLosslessEncodeOptions::default() + .with_cpu_only_backend() + .backend, + EncodeBackendPreference::CpuOnly + ); + assert_eq!( + J2kLosslessEncodeOptions::default() + .with_strict_device_backend() + .backend, + EncodeBackendPreference::RequireDevice + ); + assert_eq!( + J2kLossyEncodeOptions::default() + .with_accelerated_backend() + .backend, + EncodeBackendPreference::Auto + ); +} + +#[test] +fn adaptive_planner_keeps_small_workloads_on_cpu_without_benchmark_gate() { + let workload = J2kAdaptiveWorkload::new( + J2kAdaptiveOperation::Encode, + J2kAdaptiveCodecMode::Htj2k, + J2kAdaptiveQualityMode::Lossless, + 3, + 8, + (128, 128), + 1, + ); + let report = J2kAdaptiveRoutePlanner::new(metal_caps()) + .plan( + workload, + J2kAdaptiveBackendRequest::Accelerated, + &J2kAdaptiveBenchmarks::default(), + ) + .expect("accelerated CPU route should plan"); + + assert_eq!(report.route_kind, J2kAdaptiveRouteKind::CpuOnly); + assert_eq!(report.selected_device, None); + assert!(report.stage(J2kAdaptiveStage::MarkerParsing).is_some()); + assert!(report.stages.len() >= J2kAdaptiveStage::ALL.len()); + assert!( + report + .stages + .iter() + .all(|stage| stage.selected_backend == BackendKind::Cpu), + "small ungated workload must stay CPU-only" + ); +} + +#[test] +fn stage_candidate_remains_cpu_when_end_to_end_gate_is_missing() { + let workload = rgb_wsi_htj2k_encode(); + let benchmarks = metal_stage_candidate_benchmarks_for(J2kAdaptiveStage::Dwt); + + let report = J2kAdaptiveRoutePlanner::new(metal_caps()) + .plan( + workload, + J2kAdaptiveBackendRequest::Accelerated, + &benchmarks, + ) + .expect("route should plan with stage evidence only"); + + let dwt = report.stage(J2kAdaptiveStage::Dwt).expect("DWT decision"); + assert_eq!(report.route_kind, J2kAdaptiveRouteKind::CpuOnly); + assert_eq!(report.selected_device, None); + assert_eq!(dwt.logical_owner, J2kAdaptiveStageOwner::Gpu); + assert_eq!(dwt.selected_backend, BackendKind::Cpu); + assert_eq!( + dwt.gate_status, + J2kAdaptiveStageGateStatus::EndToEndGateBlocked + ); + let improvement = dwt + .improvement_percent + .expect("stage candidate evidence should remain visible for RCA"); + assert!((improvement - 42.857_142_857_142_854).abs() < 1e-12); +} + +#[test] +fn stage_candidate_remains_cpu_when_end_to_end_gate_fails() { + let workload = rgb_wsi_htj2k_encode(); + let mut benchmarks = metal_stage_candidate_benchmarks_for(J2kAdaptiveStage::HtBlockCoding); + benchmarks.push_end_to_end(J2kAdaptiveBenchmarkEvidence::end_to_end( + BackendKind::Metal, + 2_000_000, + 1_950_000, + 1.0, + )); + + let report = J2kAdaptiveRoutePlanner::new(metal_caps()) + .plan( + workload, + J2kAdaptiveBackendRequest::Accelerated, + &benchmarks, + ) + .expect("route should plan with a failing end-to-end gate"); + + let ht = report + .stage(J2kAdaptiveStage::HtBlockCoding) + .expect("HT block decision"); + assert_eq!(report.route_kind, J2kAdaptiveRouteKind::CpuOnly); + assert_eq!(report.selected_device, None); + assert_eq!(ht.selected_backend, BackendKind::Cpu); + assert_eq!( + ht.gate_status, + J2kAdaptiveStageGateStatus::EndToEndGateBlocked + ); + assert!(ht.improvement_percent.is_some()); +} + +#[test] +fn approved_backend_is_not_masked_by_faster_stage_only_candidate() { + let workload = rgb_wsi_htj2k_encode(); + let mut benchmarks = approved_cuda_benchmarks_for(workload); + benchmarks.push_stage(J2kAdaptiveBenchmarkEvidence::stage( + J2kAdaptiveStage::Dwt, + BackendKind::Metal, + 100_000, + 70_000, + 1.0, + )); + + let report = J2kAdaptiveRoutePlanner::new(metal_cuda_caps()) + .plan( + workload, + J2kAdaptiveBackendRequest::Accelerated, + &benchmarks, + ) + .expect("route should plan with mixed backend evidence"); + + assert_eq!(report.route_kind, J2kAdaptiveRouteKind::Hybrid); + assert_eq!(report.selected_device, Some(BackendKind::Cuda)); + assert_eq!( + report + .stage(J2kAdaptiveStage::Dwt) + .expect("DWT decision") + .selected_backend, + BackendKind::Cuda + ); + assert_eq!( + report + .stage(J2kAdaptiveStage::HtBlockCoding) + .expect("HT block decision") + .selected_backend, + BackendKind::Cuda + ); +} + +#[test] +fn rca_reclassification_is_exact_to_stage_and_backend() { + let workload = rgb_wsi_htj2k_encode(); + let mut benchmarks = approved_metal_benchmarks_for(workload); + benchmarks.push_stage(J2kAdaptiveBenchmarkEvidence::stage( + J2kAdaptiveStage::Dwt, + BackendKind::Metal, + 100_000, + 96_000, + 1.0, + )); + + let report = J2kAdaptiveRoutePlanner::new(metal_caps()) + .with_rca_finding(J2kAdaptiveRcaFinding::reclassify_cpu( + J2kAdaptiveStage::HtBlockCoding, + BackendKind::Metal, + J2kAdaptiveRcaReason::TransferSyncOverhead, + )) + .plan( + workload, + J2kAdaptiveBackendRequest::Accelerated, + &benchmarks, + ) + .expect("route should plan with non-matching RCA"); + + let dwt = report.stage(J2kAdaptiveStage::Dwt).expect("DWT decision"); + assert_eq!(dwt.gate_status, J2kAdaptiveStageGateStatus::BlockedNeedsRca); + assert_eq!(dwt.selected_backend, BackendKind::Cpu); + assert!(report.has_unresolved_rca()); + + let report = J2kAdaptiveRoutePlanner::new(metal_caps()) + .with_rca_finding(J2kAdaptiveRcaFinding::reclassify_cpu( + J2kAdaptiveStage::Dwt, + BackendKind::Cuda, + J2kAdaptiveRcaReason::TransferSyncOverhead, + )) + .plan( + workload, + J2kAdaptiveBackendRequest::Accelerated, + &benchmarks, + ) + .expect("route should plan with backend-mismatched RCA"); + + let dwt = report.stage(J2kAdaptiveStage::Dwt).expect("DWT decision"); + assert_eq!(dwt.gate_status, J2kAdaptiveStageGateStatus::BlockedNeedsRca); + assert_eq!(dwt.selected_backend, BackendKind::Cpu); + assert!(report.has_unresolved_rca()); +} + +#[test] +fn adaptive_planner_requires_stage_and_end_to_end_gates_before_default_gpu() { + let workload = rgb_wsi_htj2k_encode(); + let planner = J2kAdaptiveRoutePlanner::new(metal_caps()); + + let ungated = planner + .plan( + workload, + J2kAdaptiveBackendRequest::Accelerated, + &J2kAdaptiveBenchmarks::default(), + ) + .expect("ungated route should still plan"); + assert_eq!(ungated.route_kind, J2kAdaptiveRouteKind::CpuOnly); + assert!(ungated + .stage(J2kAdaptiveStage::Dwt) + .expect("DWT decision") + .requires_rca()); + + let gated = planner + .plan( + workload, + J2kAdaptiveBackendRequest::Accelerated, + &approved_metal_benchmarks_for(workload), + ) + .expect("gated route should plan"); + + assert_eq!(gated.route_kind, J2kAdaptiveRouteKind::Hybrid); + assert_eq!(gated.selected_device, Some(BackendKind::Metal)); + assert_eq!( + gated + .stage(J2kAdaptiveStage::MarkerParsing) + .expect("marker parsing decision") + .selected_backend, + BackendKind::Cpu + ); + assert_eq!( + gated + .stage(J2kAdaptiveStage::Dwt) + .expect("DWT decision") + .selected_backend, + BackendKind::Metal + ); + assert_eq!( + gated + .stage(J2kAdaptiveStage::HtBlockCoding) + .expect("HT block decision") + .selected_backend, + BackendKind::Metal + ); +} + +#[test] +fn logical_gpu_loss_requires_rca_before_reclassification() { + let workload = rgb_wsi_htj2k_encode(); + let mut benchmarks = approved_metal_benchmarks_for(workload); + benchmarks.push_stage(J2kAdaptiveBenchmarkEvidence::stage( + J2kAdaptiveStage::Dwt, + BackendKind::Metal, + 100_000, + 96_000, + 1.0, + )); + + let unresolved = J2kAdaptiveRoutePlanner::new(metal_caps()) + .plan( + workload, + J2kAdaptiveBackendRequest::Accelerated, + &benchmarks, + ) + .expect("route should plan with a blocked GPU stage"); + let dwt = unresolved + .stage(J2kAdaptiveStage::Dwt) + .expect("DWT decision"); + assert_eq!(dwt.gate_status, J2kAdaptiveStageGateStatus::BlockedNeedsRca); + assert!(unresolved.has_unresolved_rca()); + + let resolved = J2kAdaptiveRoutePlanner::new(metal_caps()) + .with_rca_finding(J2kAdaptiveRcaFinding::reclassify_cpu( + J2kAdaptiveStage::Dwt, + BackendKind::Metal, + J2kAdaptiveRcaReason::TooSmallWorkload, + )) + .plan( + workload, + J2kAdaptiveBackendRequest::Accelerated, + &benchmarks, + ) + .expect("route should plan after RCA"); + let dwt = resolved.stage(J2kAdaptiveStage::Dwt).expect("DWT decision"); + assert_eq!(dwt.gate_status, J2kAdaptiveStageGateStatus::ReclassifiedCpu); + assert_eq!(dwt.selected_backend, BackendKind::Cpu); + assert!(!resolved.has_unresolved_rca()); +} + +#[test] +fn strict_device_request_fails_when_backend_is_unavailable() { + let result = J2kAdaptiveRoutePlanner::new(cpu_caps()).plan( + rgb_wsi_htj2k_encode(), + J2kAdaptiveBackendRequest::StrictDevice(BackendKind::Metal), + &J2kAdaptiveBenchmarks::default(), + ); + + assert!(result.is_err(), "strict Metal must not silently fall back"); +} diff --git a/crates/j2k/tests/batch.rs b/crates/j2k/tests/batch.rs new file mode 100644 index 00000000..9bdb3ce4 --- /dev/null +++ b/crates/j2k/tests/batch.rs @@ -0,0 +1,455 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::{ + decode_tiles_into, decode_tiles_region_into, decode_tiles_region_scaled_into, + decode_tiles_scaled_into, Downscale, J2kDecoder, PixelFormat, Rect, TileBatchOptions, + TileDecodeJob, TileRegionDecodeJob, TileRegionScaledDecodeJob, TileScaledDecodeJob, +}; +use j2k_native::{encode, encode_htj2k, EncodeOptions}; +use j2k_test_support::wrap_codestream_jp2; +use std::num::NonZeroUsize; + +fn encode_codestream( + pixels: &[u8], + width: u32, + height: u32, + components: u8, + bit_depth: u8, +) -> Vec { + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + encode( + pixels, width, height, components, bit_depth, false, &options, + ) + .expect("encode") +} + +fn encode_ht_codestream( + pixels: &[u8], + width: u32, + height: u32, + components: u8, + bit_depth: u8, +) -> Vec { + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 2, + ..EncodeOptions::default() + }; + encode_htj2k( + pixels, width, height, components, bit_depth, false, &options, + ) + .expect("encode HTJ2K") +} + +fn rgb_fixture() -> Vec { + let pixels = (0_u8..48).collect::>(); + encode_codestream(&pixels, 4, 4, 3, 8) +} + +fn ht_rgb_fixture() -> Vec { + let pixels = (0..16 * 16 * 3) + .map(|idx| ((idx * 13 + idx / 3) & 0xff) as u8) + .collect::>(); + encode_ht_codestream(&pixels, 16, 16, 3, 8) +} + +fn ht_rgb_jp2_fixture() -> Vec { + wrap_codestream_jp2(&ht_rgb_fixture(), 16, 16, 3, 8, 16) +} + +fn decode_rgb8_reference(bytes: &[u8]) -> (Vec, usize) { + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + let (width, height) = decoder.info().dimensions; + let stride = width as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut out = vec![0_u8; stride * height as usize]; + decoder + .decode_into(&mut out, stride, PixelFormat::Rgb8) + .expect("decode reference"); + (out, stride) +} + +fn assert_region_scaled_batch_matches_single_decode(bytes: &[u8], fmt: PixelFormat) { + const JOBS: usize = 8; + let roi = Rect { + x: 4, + y: 4, + w: 8, + h: 8, + }; + let scale = Downscale::Half; + let scaled_roi = roi.scaled_covering(scale); + let stride = scaled_roi.w as usize * fmt.bytes_per_pixel(); + + let mut decoder = J2kDecoder::new(bytes).expect("decoder"); + let mut pool = j2k::J2kScratchPool::new(); + let mut expected = vec![0_u8; stride * scaled_roi.h as usize]; + decoder + .decode_region_scaled_into(&mut pool, &mut expected, stride, fmt, roi, scale) + .expect("decode reference"); + + let mut outputs = (0..JOBS) + .map(|_| vec![0_u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions::new(NonZeroUsize::new(4)); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: bytes, + out: out.as_mut_slice(), + stride, + roi, + scale, + }) + .collect::>(); + decode_tiles_region_scaled_into(&mut jobs, fmt, options).expect("batch decode") + }; + + assert_eq!(outcomes.len(), JOBS); + for outcome in &outcomes { + assert_eq!(outcome.decoded, scaled_roi); + } + for (index, out) in outputs.iter().enumerate() { + assert_eq!(out, &expected, "tile {index} output diverged"); + } +} + +#[test] +fn production_batch_decode_empty_input_succeeds() { + let mut jobs: Vec> = Vec::new(); + + let outcomes = decode_tiles_into(&mut jobs, PixelFormat::Rgb8, TileBatchOptions::default()) + .expect("empty batch succeeds"); + + assert!(outcomes.is_empty()); +} + +#[test] +fn production_batch_decode_worker_one_matches_single_tile_decode() { + let codestream = rgb_fixture(); + let (expected, stride) = decode_rgb8_reference(&codestream); + let mut actual = vec![0_u8; expected.len()]; + let options = TileBatchOptions::new(NonZeroUsize::new(1)); + + let outcomes = { + let mut jobs = vec![TileDecodeJob { + input: &codestream, + out: actual.as_mut_slice(), + stride, + }]; + decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect("batch decode") + }; + + assert_eq!(outcomes.len(), 1); + assert_eq!(actual, expected); +} + +#[test] +fn production_batch_decode_parallel_preserves_order_and_output() { + const JOBS: usize = 16; + let codestream = rgb_fixture(); + let (expected, stride) = decode_rgb8_reference(&codestream); + let mut outputs = (0..JOBS) + .map(|_| vec![0_u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions::new(NonZeroUsize::new(4)); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileDecodeJob { + input: codestream.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect("batch decode") + }; + + assert_eq!(outcomes.len(), JOBS); + for (index, out) in outputs.iter().enumerate() { + assert_eq!(out, &expected, "tile {index} output diverged"); + } +} + +#[test] +fn production_batch_decode_matches_repeated_single_tile_decodes() { + let inputs = [ + rgb_fixture(), + encode_codestream(&(48_u8..96).collect::>(), 4, 4, 3, 8), + encode_codestream(&(96_u8..144).collect::>(), 4, 4, 3, 8), + ]; + let expected = inputs + .iter() + .map(|input| decode_rgb8_reference(input).0) + .collect::>(); + let stride = 4 * PixelFormat::Rgb8.bytes_per_pixel(); + let mut outputs = expected + .iter() + .map(|tile| vec![0_u8; tile.len()]) + .collect::>(); + let options = TileBatchOptions::new(NonZeroUsize::new(2)); + + let outcomes = { + let mut jobs = inputs + .iter() + .zip(outputs.iter_mut()) + .map(|(input, out)| TileDecodeJob { + input: input.as_slice(), + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect("batch decode") + }; + + assert_eq!(outcomes.len(), inputs.len()); + assert_eq!(outputs, expected); +} + +#[test] +fn production_batch_region_scaled_decode_parallel_preserves_order_and_output() { + const JOBS: usize = 12; + let codestream = rgb_fixture(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let scale = Downscale::Half; + let scaled_roi = roi.scaled_covering(scale); + let stride = scaled_roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + + let mut decoder = J2kDecoder::new(&codestream).expect("decoder"); + let mut pool = j2k::J2kScratchPool::new(); + let mut expected = vec![0_u8; stride * scaled_roi.h as usize]; + decoder + .decode_region_scaled_into( + &mut pool, + &mut expected, + stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("decode reference"); + + let mut outputs = (0..JOBS) + .map(|_| vec![0_u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions::new(NonZeroUsize::new(3)); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: codestream.as_slice(), + out: out.as_mut_slice(), + stride, + roi, + scale, + }) + .collect::>(); + decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8, options) + .expect("batch decode") + }; + + assert_eq!(outcomes.len(), JOBS); + for outcome in &outcomes { + assert_eq!(outcome.decoded, scaled_roi); + } + for (index, out) in outputs.iter().enumerate() { + assert_eq!(out, &expected, "tile {index} output diverged"); + } +} + +#[test] +fn production_batch_region_decode_parallel_preserves_order_and_output() { + const JOBS: usize = 12; + let codestream = rgb_fixture(); + let roi = Rect { + x: 1, + y: 0, + w: 2, + h: 3, + }; + let stride = roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + + let mut decoder = J2kDecoder::new(&codestream).expect("decoder"); + let mut pool = j2k::J2kScratchPool::new(); + let mut expected = vec![0_u8; stride * roi.h as usize]; + decoder + .decode_region_into(&mut pool, &mut expected, stride, PixelFormat::Rgb8, roi) + .expect("decode reference"); + + let mut outputs = (0..JOBS) + .map(|_| vec![0_u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions::new(NonZeroUsize::new(3)); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileRegionDecodeJob { + input: codestream.as_slice(), + out: out.as_mut_slice(), + stride, + roi, + }) + .collect::>(); + decode_tiles_region_into(&mut jobs, PixelFormat::Rgb8, options).expect("batch decode") + }; + + assert_eq!(outcomes.len(), JOBS); + for outcome in &outcomes { + assert_eq!(outcome.decoded, roi); + } + for (index, out) in outputs.iter().enumerate() { + assert_eq!(out, &expected, "tile {index} output diverged"); + } +} + +#[test] +fn production_batch_scaled_decode_parallel_preserves_order_and_output() { + const JOBS: usize = 12; + let codestream = ht_rgb_fixture(); + let scale = Downscale::Half; + let scaled = Rect::full((16, 16)).scaled_covering(scale); + let stride = scaled.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + + let mut decoder = J2kDecoder::new(&codestream).expect("decoder"); + let mut pool = j2k::J2kScratchPool::new(); + let mut expected = vec![0_u8; stride * scaled.h as usize]; + decoder + .decode_scaled_into(&mut pool, &mut expected, stride, PixelFormat::Rgb8, scale) + .expect("decode reference"); + + let mut outputs = (0..JOBS) + .map(|_| vec![0_u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions::new(NonZeroUsize::new(3)); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileScaledDecodeJob { + input: codestream.as_slice(), + out: out.as_mut_slice(), + stride, + scale, + }) + .collect::>(); + decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgb8, options).expect("batch decode") + }; + + assert_eq!(outcomes.len(), JOBS); + for outcome in &outcomes { + assert_eq!(outcome.decoded, scaled); + } + for (index, out) in outputs.iter().enumerate() { + assert_eq!(out, &expected, "tile {index} output diverged"); + } +} + +#[test] +fn production_batch_region_scaled_htj2k_rgb_matches_single_decode() { + const JOBS: usize = 8; + let codestream = ht_rgb_fixture(); + let roi = Rect { + x: 4, + y: 4, + w: 8, + h: 8, + }; + let scale = Downscale::Half; + let scaled_roi = roi.scaled_covering(scale); + let stride = scaled_roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + + let mut decoder = J2kDecoder::new(&codestream).expect("decoder"); + let mut pool = j2k::J2kScratchPool::new(); + let mut expected = vec![0_u8; stride * scaled_roi.h as usize]; + decoder + .decode_region_scaled_into( + &mut pool, + &mut expected, + stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("decode reference"); + + let mut outputs = (0..JOBS) + .map(|_| vec![0_u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions::new(NonZeroUsize::new(4)); + + let outcomes = { + let mut jobs = outputs + .iter_mut() + .map(|out| TileRegionScaledDecodeJob { + input: codestream.as_slice(), + out: out.as_mut_slice(), + stride, + roi, + scale, + }) + .collect::>(); + decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8, options) + .expect("batch decode") + }; + + assert_eq!(outcomes.len(), JOBS); + for outcome in &outcomes { + assert_eq!(outcome.decoded, scaled_roi); + } + for (index, out) in outputs.iter().enumerate() { + assert_eq!(out, &expected, "tile {index} output diverged"); + } +} + +#[test] +fn production_batch_region_scaled_htj2k_jp2_rgb_matches_single_decode() { + let jp2 = ht_rgb_jp2_fixture(); + + assert_region_scaled_batch_matches_single_decode(&jp2, PixelFormat::Rgb8); +} + +#[test] +fn production_batch_region_scaled_htj2k_jp2_rgba_matches_single_decode() { + let jp2 = ht_rgb_jp2_fixture(); + + assert_region_scaled_batch_matches_single_decode(&jp2, PixelFormat::Rgba8); +} + +#[test] +fn production_batch_decode_reports_first_failing_tile_index() { + let codestream = rgb_fixture(); + let (expected, stride) = decode_rgb8_reference(&codestream); + let mut outputs = (0..3) + .map(|_| vec![0_u8; expected.len()]) + .collect::>(); + let options = TileBatchOptions::new(NonZeroUsize::new(2)); + + let err = { + let inputs: [&[u8]; 3] = [codestream.as_slice(), b"not j2k", codestream.as_slice()]; + let mut jobs = inputs + .into_iter() + .zip(outputs.iter_mut()) + .map(|(input, out)| TileDecodeJob { + input, + out: out.as_mut_slice(), + stride, + }) + .collect::>(); + decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect_err("bad tile fails") + }; + + assert_eq!(err.index, 1); +} diff --git a/crates/j2k/tests/bench_harness.rs b/crates/j2k/tests/bench_harness.rs new file mode 100644 index 00000000..e9890c97 --- /dev/null +++ b/crates/j2k/tests/bench_harness.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[test] +fn public_api_bench_exposes_cpu_decode_regression_surface() { + let bench = include_str!("../benches/public_api.rs"); + + for expected in [ + "j2k_public_cpu_encode_matrix", + "j2k_public_cpu_decode_matrix", + "rgb8_512_classic_external", + "rgb8_512_htj2k_external", + "rgb8_512_classic_roundtrip", + "gray8_512_classic_decode", + "gray8_512_htj2k_decode", + "rgb8_512_classic_decode", + "rgb8_512_classic_decode_serial", + "rgb8_512_htj2k_decode", + "j2k_public_recode", + "classic_rgb8_512_to_htj2k_53_coefficients", + "raw_htj2k_rgb8_512_passthrough", + "j2k_public_decode_region_scaled", + "j2k_public_decode_rows", + "j2k_public_tile_batch", + "j2k_public_tile_batch_region_scaled_rgb_q4", + "rgb8_region_scaled_64x64_q4", + "gray8_rows_128x128", + "gray8_repeated_batch_16", + "gray8_distinct_batch_16", + "htj2k_gray8_repeated_batch_16", + "htj2k_gray8_distinct_batch_16", + "classic_repeated_512_roi256_batch16", + "classic_distinct_512_roi256_batch16", + "htj2k_repeated_512_roi256_batch16", + "htj2k_jp2_rgb8_repeated_512_roi256_batch16", + "htj2k_jp2_rgba8_repeated_512_roi256_batch16", + "htj2k_jp2_rgb8_repeated_256_roi128_batch16", + "htj2k_distinct_512_roi256_batch16", + "htj2k_gray8_full_512x512", + ] { + assert!( + bench.contains(expected), + "public API benchmark is missing `{expected}`" + ); + } +} diff --git a/crates/signinum-j2k/tests/decode.rs b/crates/j2k/tests/decode.rs similarity index 71% rename from crates/signinum-j2k/tests/decode.rs rename to crates/j2k/tests/decode.rs index 653075ab..09026ed4 100644 --- a/crates/signinum-j2k/tests/decode.rs +++ b/crates/j2k/tests/decode.rs @@ -1,11 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_core::{ +use j2k::{J2kCodec, J2kContext, J2kDecoder, J2kError, J2kRowDecodeOptions}; +use j2k_core::{ BufferError, DecoderContext, Downscale, ImageDecodeRows, PixelFormat, Rect, RowSink, TileBatchDecode, }; -use signinum_j2k::{J2kCodec, J2kContext, J2kDecoder, J2kError}; -use signinum_j2k_native::{encode, encode_htj2k, DecodeSettings, EncodeOptions, Image}; +use j2k_native::{encode, encode_htj2k, DecodeSettings, EncodeOptions, Image}; +use j2k_test_support::{ + crop_interleaved_bytes, crop_interleaved_u8, wrap_jp2_codestream, wrap_jp2_rgba_codestream, + PixelRect, +}; fn encode_codestream( pixels: &[u8], @@ -26,55 +30,42 @@ fn encode_codestream( .expect("encode") } -fn encode_ht_codestream( +fn encode_codestream_with_levels( pixels: &[u8], width: u32, height: u32, components: u8, bit_depth: u8, + reversible: bool, + levels: u8, ) -> Vec { let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, + reversible, + num_decomposition_levels: levels, ..EncodeOptions::default() }; - encode_htj2k( + encode( pixels, width, height, components, bit_depth, false, &options, ) - .expect("encode ht") + .expect("encode") } -fn wrap_codestream_jp2( - codestream: &[u8], +fn encode_ht_codestream( + pixels: &[u8], width: u32, height: u32, - components: u16, + components: u8, bit_depth: u8, - colorspace_enum: u32, ) -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0D, 0x0A, 0x87, 0x0A]); - bytes.extend_from_slice(&[ - 0, 0, 0, 20, b'f', b't', b'y', b'p', b'j', b'p', b'2', b' ', 0, 0, 0, 0, b'j', b'p', b'2', - b' ', - ]); - - let bpc = bit_depth.saturating_sub(1); - bytes.extend_from_slice(&[ - 0, 0, 0, 45, b'j', b'p', b'2', b'h', 0, 0, 0, 22, b'i', b'h', b'd', b'r', - ]); - bytes.extend_from_slice(&height.to_be_bytes()); - bytes.extend_from_slice(&width.to_be_bytes()); - bytes.extend_from_slice(&components.to_be_bytes()); - bytes.extend_from_slice(&[bpc, 7, 0, 0]); - bytes.extend_from_slice(&[0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0]); - bytes.extend_from_slice(&colorspace_enum.to_be_bytes()); - - let len = (8 + codestream.len()) as u32; - bytes.extend_from_slice(&len.to_be_bytes()); - bytes.extend_from_slice(b"jp2c"); - bytes.extend_from_slice(codestream); - bytes + let options = EncodeOptions { + reversible: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + encode_htj2k( + pixels, width, height, components, bit_depth, false, &options, + ) + .expect("encode ht") } fn backend_decode_u8(bytes: &[u8]) -> Vec { @@ -96,7 +87,7 @@ fn backend_decode_u8_scaled(bytes: &[u8], target_resolution: (u32, u32)) -> Vec< } fn backend_decode_u8_region(bytes: &[u8], roi: Rect) -> Vec { - let mut context = signinum_j2k_native::DecoderContext::default(); + let mut context = j2k_native::DecoderContext::default(); Image::new(bytes, &DecodeSettings::default()) .expect("backend image") .decode_region_with_context((roi.x, roi.y, roi.w, roi.h), &mut context) @@ -131,25 +122,15 @@ fn locally_inspectable_codestream_without_decode_headers() -> Vec { } fn crop_u8(full: &[u8], full_width: usize, channels: usize, roi: Rect) -> Vec { - let mut out = Vec::with_capacity(roi.w as usize * roi.h as usize * channels); - let row_bytes = full_width * channels; - let roi_row_bytes = roi.w as usize * channels; - for y in roi.y as usize..(roi.y + roi.h) as usize { - let start = y * row_bytes + roi.x as usize * channels; - out.extend_from_slice(&full[start..start + roi_row_bytes]); - } - out + crop_interleaved_u8(full, full_width, channels, pixel_rect(roi)) } fn crop_bytes(full: &[u8], full_width: usize, bytes_per_pixel: usize, roi: Rect) -> Vec { - let mut out = Vec::with_capacity(roi.w as usize * roi.h as usize * bytes_per_pixel); - let row_bytes = full_width * bytes_per_pixel; - let roi_row_bytes = roi.w as usize * bytes_per_pixel; - for y in roi.y as usize..(roi.y + roi.h) as usize { - let start = y * row_bytes + roi.x as usize * bytes_per_pixel; - out.extend_from_slice(&full[start..start + roi_row_bytes]); - } - out + crop_interleaved_bytes(full, full_width, bytes_per_pixel, pixel_rect(roi)) +} + +fn pixel_rect(roi: Rect) -> PixelRect { + PixelRect::new(roi.x, roi.y, roi.w, roi.h) } #[derive(Default)] @@ -190,7 +171,7 @@ fn decode_rgb8_codestream_roundtrips_reversible_pixels() { let outcome = decoder .decode_into(&mut out, 2 * 3, PixelFormat::Rgb8) .expect("decode"); - assert_eq!(outcome.decoded, signinum_core::Rect::full((2, 2))); + assert_eq!(outcome.decoded, j2k_core::Rect::full((2, 2))); assert_eq!(out, expected.as_slice()); } @@ -225,7 +206,7 @@ fn decoder_reuses_native_context_across_multiple_decode_calls() { let mut scaled = [0_u8; 3]; decoder .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut scaled, 3, PixelFormat::Rgb8, @@ -256,7 +237,7 @@ fn decode_rgba8_fills_opaque_alpha_for_rgb_source() { fn decode_gray8_jp2_roundtrips_reversible_pixels() { let pixels = [3, 9, 27, 81]; let codestream = encode_codestream(&pixels, 2, 2, 1, 8, true); - let jp2 = wrap_codestream_jp2(&codestream, 2, 2, 1, 8, 17); + let jp2 = wrap_jp2_codestream(&codestream, 2, 2, 1, 8, 17); let mut decoder = J2kDecoder::new(&jp2).expect("decoder"); let mut out = [0_u8; 4]; decoder @@ -308,15 +289,108 @@ fn decode_rgb16_roundtrips_native_samples() { } #[test] -fn decode_rejects_unsupported_rgba16_output() { - let pixels = [1, 2, 3, 4, 5, 6]; - let codestream = encode_codestream(&pixels, 2, 1, 3, 8, true); +fn decode_rgba16_fills_opaque_alpha_for_rgb_source() { + let samples = [0_u16, 1, 2, 1024, 2048, 3072]; + let pixels: Vec = samples.into_iter().flat_map(u16::to_le_bytes).collect(); + let codestream = encode_codestream(&pixels, 2, 1, 3, 12, true); let mut decoder = J2kDecoder::new(&codestream).expect("decoder"); let mut out = [0_u8; 16]; - let err = decoder + decoder .decode_into(&mut out, 2 * 4 * 2, PixelFormat::Rgba16) - .unwrap_err(); - assert!(matches!(err, J2kError::Unsupported(_))); + .expect("decode"); + let expected: Vec = [0_u16, 1, 2, 4095, 1024, 2048, 3072, 4095] + .into_iter() + .flat_map(u16::to_le_bytes) + .collect(); + assert_eq!(out, expected.as_slice()); +} + +#[test] +fn decode_rgba16_preserves_jp2_alpha_channel() { + let samples = [0_u16, 1, 2, 3, 1024, 2048, 3072, 4095]; + let pixels: Vec = samples.into_iter().flat_map(u16::to_le_bytes).collect(); + let codestream = encode_codestream(&pixels, 2, 1, 4, 12, true); + let jp2 = wrap_jp2_rgba_codestream(&codestream, 2, 1, 12); + let mut decoder = J2kDecoder::new(&jp2).expect("decoder"); + let mut out = [0_u8; 16]; + decoder + .decode_into(&mut out, 2 * 4 * 2, PixelFormat::Rgba16) + .expect("decode"); + assert_eq!(out, pixels.as_slice()); +} + +#[test] +fn decode_rgba16_roi_scaled_and_region_scaled_preserve_alpha() { + let samples: Vec = (0..4 * 4 * 4).map(|sample| sample * 3).collect(); + let pixels: Vec = samples.into_iter().flat_map(u16::to_le_bytes).collect(); + let codestream = encode_codestream(&pixels, 4, 4, 4, 12, true); + let jp2 = wrap_jp2_rgba_codestream(&codestream, 4, 4, 12); + let fmt = PixelFormat::Rgba16; + let bytes_per_pixel = fmt.bytes_per_pixel(); + + let mut full_decoder = J2kDecoder::new(&jp2).expect("full decoder"); + let full_stride = 4 * bytes_per_pixel; + let mut full = vec![0_u8; full_stride * 4]; + full_decoder + .decode_into(&mut full, full_stride, fmt) + .expect("full decode"); + + let roi = Rect { + x: 1, + y: 1, + w: 2, + h: 2, + }; + let mut region_decoder = J2kDecoder::new(&jp2).expect("region decoder"); + let region_stride = roi.w as usize * bytes_per_pixel; + let mut region = vec![0_u8; region_stride * roi.h as usize]; + let region_outcome = region_decoder + .decode_region_into( + &mut j2k::J2kScratchPool::new(), + &mut region, + region_stride, + fmt, + roi, + ) + .expect("region decode"); + assert_eq!(region_outcome.decoded, roi); + assert_eq!(region, crop_bytes(&full, 4, bytes_per_pixel, roi)); + + let scale = Downscale::Half; + let scaled_dims = (2, 2); + let scaled_stride = scaled_dims.0 as usize * bytes_per_pixel; + let mut scaled_decoder = J2kDecoder::new(&jp2).expect("scaled decoder"); + let mut scaled = vec![0_u8; scaled_stride * scaled_dims.1 as usize]; + let scaled_outcome = scaled_decoder + .decode_scaled_into( + &mut j2k::J2kScratchPool::new(), + &mut scaled, + scaled_stride, + fmt, + scale, + ) + .expect("scaled decode"); + assert_eq!(scaled_outcome.decoded, Rect::full(scaled_dims)); + + let scaled_roi = roi.scaled_covering(scale); + let mut region_scaled_decoder = J2kDecoder::new(&jp2).expect("region scaled decoder"); + let region_scaled_stride = scaled_roi.w as usize * bytes_per_pixel; + let mut region_scaled = vec![0_u8; region_scaled_stride * scaled_roi.h as usize]; + let region_scaled_outcome = region_scaled_decoder + .decode_region_scaled_into( + &mut j2k::J2kScratchPool::new(), + &mut region_scaled, + region_scaled_stride, + fmt, + roi, + scale, + ) + .expect("region scaled decode"); + assert_eq!(region_scaled_outcome.decoded, scaled_roi); + assert_eq!( + region_scaled, + crop_bytes(&scaled, scaled_dims.0 as usize, bytes_per_pixel, scaled_roi) + ); } #[test] @@ -355,7 +429,7 @@ fn decode_scaled_into_matches_backend_target_resolution_decode() { let codestream = encode_codestream(&pixels, 4, 4, 3, 8, true); let expected = backend_decode_u8_scaled(&codestream, (2, 2)); let mut decoder = J2kDecoder::new(&codestream).expect("decoder"); - let mut pool = signinum_j2k::J2kScratchPool::new(); + let mut pool = j2k::J2kScratchPool::new(); let mut out = [0_u8; 12]; let outcome = decoder .decode_scaled_into( @@ -370,6 +444,97 @@ fn decode_scaled_into_matches_backend_target_resolution_decode() { assert_eq!(out, expected.as_slice()); } +#[test] +fn reused_decoder_scaled_decodes_match_fresh_decodes_across_scales() { + let pixels: Vec = (0..16 * 16 * 3) + .map(|index| ((index * 13 + 17) & 0xFF) as u8) + .collect(); + let codestream = encode_codestream_with_levels(&pixels, 16, 16, 3, 8, true, 2); + let mut reused = J2kDecoder::new(&codestream).expect("reused decoder"); + let mut reused_pool = j2k::J2kScratchPool::new(); + + for scale in [Downscale::Half, Downscale::Quarter, Downscale::Half] { + let dims = ( + 16_u32.div_ceil(scale.denominator()), + 16_u32.div_ceil(scale.denominator()), + ); + let stride = dims.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut expected = vec![0_u8; stride * dims.1 as usize]; + let mut fresh = J2kDecoder::new(&codestream).expect("fresh decoder"); + fresh + .decode_scaled_into( + &mut j2k::J2kScratchPool::new(), + &mut expected, + stride, + PixelFormat::Rgb8, + scale, + ) + .expect("fresh scaled decode"); + + let mut actual = vec![0_u8; stride * dims.1 as usize]; + let outcome = reused + .decode_scaled_into( + &mut reused_pool, + &mut actual, + stride, + PixelFormat::Rgb8, + scale, + ) + .expect("reused scaled decode"); + + assert_eq!(outcome.decoded, Rect::full(dims)); + assert_eq!(actual, expected, "scale {scale:?}"); + } +} + +#[test] +fn reused_decoder_region_scaled_decodes_match_fresh_decodes_across_scales() { + let pixels: Vec = (0..16 * 16 * 3) + .map(|index| ((index * 11 + 29) & 0xFF) as u8) + .collect(); + let codestream = encode_codestream_with_levels(&pixels, 16, 16, 3, 8, true, 2); + let roi = Rect { + x: 3, + y: 2, + w: 9, + h: 10, + }; + let mut reused = J2kDecoder::new(&codestream).expect("reused decoder"); + let mut reused_pool = j2k::J2kScratchPool::new(); + + for scale in [Downscale::Quarter, Downscale::Half, Downscale::Quarter] { + let scaled_roi = roi.scaled_covering(scale); + let stride = scaled_roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let mut expected = vec![0_u8; stride * scaled_roi.h as usize]; + let mut fresh = J2kDecoder::new(&codestream).expect("fresh decoder"); + fresh + .decode_region_scaled_into( + &mut j2k::J2kScratchPool::new(), + &mut expected, + stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("fresh region scaled decode"); + + let mut actual = vec![0_u8; stride * scaled_roi.h as usize]; + let outcome = reused + .decode_region_scaled_into( + &mut reused_pool, + &mut actual, + stride, + PixelFormat::Rgb8, + roi, + scale, + ) + .expect("reused region scaled decode"); + + assert_eq!(outcome.decoded, scaled_roi); + assert_eq!(actual, expected, "scale {scale:?}"); + } +} + #[test] fn decode_region_into_matches_cropping_full_decode() { let pixels = [0_u8, 1, 2, 3, 4, 5, 6, 7, 8]; @@ -387,7 +552,7 @@ fn decode_region_into_matches_cropping_full_decode() { h: 2, }; let expected = crop_u8(&full, 3, 1, roi); - let mut pool = signinum_j2k::J2kScratchPool::new(); + let mut pool = j2k::J2kScratchPool::new(); let mut out = [0_u8; 4]; let outcome = decoder .decode_region_into(&mut pool, &mut out, 2, PixelFormat::Gray8, roi) @@ -415,7 +580,7 @@ fn decode_region_scaled_into_matches_cropping_scaled_decode_for_supported_format let mut scaled = vec![0_u8; scaled_stride * 2]; scaled_decoder .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut scaled, scaled_stride, fmt, @@ -429,7 +594,7 @@ fn decode_region_scaled_into_matches_cropping_scaled_decode_for_supported_format let mut out = vec![0_u8; stride * scaled_roi.h as usize]; let outcome = decoder .decode_region_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, stride, fmt, @@ -448,7 +613,7 @@ fn decode_region_scaled_into_matches_cropping_scaled_decode_for_supported_format let mut gray8_scaled = vec![0_u8; 2 * 2]; gray8_scaled_decoder .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut gray8_scaled, 2, PixelFormat::Gray8, @@ -466,7 +631,7 @@ fn decode_region_scaled_into_matches_cropping_scaled_decode_for_supported_format let gray8_stride = scaled_roi.w as usize * PixelFormat::Gray8.bytes_per_pixel(); let outcome = gray8_decoder .decode_region_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut gray8_out, gray8_stride, PixelFormat::Gray8, @@ -490,7 +655,7 @@ fn decode_region_scaled_into_matches_cropping_scaled_decode_for_supported_format let mut gray16_scaled = vec![0_u8; 2 * 2 * 2]; gray16_scaled_decoder .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut gray16_scaled, 2 * 2, PixelFormat::Gray16, @@ -508,7 +673,7 @@ fn decode_region_scaled_into_matches_cropping_scaled_decode_for_supported_format let gray16_stride = scaled_roi.w as usize * PixelFormat::Gray16.bytes_per_pixel(); let outcome = gray16_decoder .decode_region_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut gray16_out, gray16_stride, PixelFormat::Gray16, @@ -534,7 +699,7 @@ fn decode_region_scaled_into_matches_cropping_scaled_decode_for_supported_format let mut rgb16_scaled = vec![0_u8; 2 * 2 * 3 * 2]; rgb16_scaled_decoder .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut rgb16_scaled, 2 * 3 * 2, PixelFormat::Rgb16, @@ -552,7 +717,7 @@ fn decode_region_scaled_into_matches_cropping_scaled_decode_for_supported_format let rgb16_stride = scaled_roi.w as usize * PixelFormat::Rgb16.bytes_per_pixel(); let outcome = rgb16_decoder .decode_region_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut rgb16_out, rgb16_stride, PixelFormat::Rgb16, @@ -581,7 +746,7 @@ fn decode_region_scaled_htj2k_gray8_matches_cropping_scaled_decode() { let mut scaled = vec![0_u8; 2 * 2]; scaled_decoder .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut scaled, 2, PixelFormat::Gray8, @@ -595,7 +760,7 @@ fn decode_region_scaled_htj2k_gray8_matches_cropping_scaled_decode() { let mut out = vec![0_u8; stride * scaled_roi.h as usize]; let outcome = decoder .decode_region_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, stride, PixelFormat::Gray8, @@ -623,7 +788,7 @@ fn decode_region_scaled_none_matches_region_decode() { let mut expected = [0_u8; 4]; expected_decoder .decode_region_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut expected, 2, PixelFormat::Gray8, @@ -635,7 +800,7 @@ fn decode_region_scaled_none_matches_region_decode() { let mut out = [0_u8; 4]; let outcome = decoder .decode_region_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, 2, PixelFormat::Gray8, @@ -697,6 +862,44 @@ fn decode_rows_u16_matches_full_gray16_decode() { assert_eq!(collected, full); } +#[test] +fn decode_rows_u8_bounded_matches_full_rgba8_decode_one_row_at_a_time() { + let pixels: Vec = (0_u8..32).collect(); + let codestream = encode_codestream(&pixels, 4, 2, 4, 8, true); + let jp2 = wrap_jp2_rgba_codestream(&codestream, 4, 2, 8); + let mut decoder = J2kDecoder::new(&jp2).expect("decoder"); + let mut full = [0_u8; 32]; + decoder + .decode_into(&mut full, 4 * 4, PixelFormat::Rgba8) + .expect("full decode"); + + let mut sink = CollectRowsU8::default(); + decoder + .decode_rows_u8_bounded(&mut sink, J2kRowDecodeOptions::new(1)) + .expect("bounded row decode"); + assert_eq!(sink.rows, full); +} + +#[test] +fn decode_rows_u16_bounded_matches_full_rgba16_decode_one_row_at_a_time() { + let samples: Vec = (0..4 * 2 * 4).map(|sample| sample * 11).collect(); + let pixels: Vec = samples.into_iter().flat_map(u16::to_le_bytes).collect(); + let codestream = encode_codestream(&pixels, 4, 2, 4, 12, true); + let jp2 = wrap_jp2_rgba_codestream(&codestream, 4, 2, 12); + let mut decoder = J2kDecoder::new(&jp2).expect("decoder"); + let mut full = vec![0_u8; 4 * 2 * 4 * 2]; + decoder + .decode_into(&mut full, 4 * 4 * 2, PixelFormat::Rgba16) + .expect("full decode"); + + let mut sink = CollectRowsU16::default(); + decoder + .decode_rows_u16_bounded(&mut sink, J2kRowDecodeOptions::new(1)) + .expect("bounded row decode"); + let collected: Vec = sink.rows.into_iter().flat_map(u16::to_le_bytes).collect(); + assert_eq!(collected, full); +} + #[test] fn tile_batch_decode_matches_borrowed_decoder_decode() { let pixels = [10_u8, 20, 30, 40, 50, 60]; @@ -708,7 +911,7 @@ fn tile_batch_decode_matches_borrowed_decoder_decode() { .expect("decoder decode"); let mut ctx = DecoderContext::::new(); - let mut pool = signinum_j2k::J2kScratchPool::new(); + let mut pool = j2k::J2kScratchPool::new(); let mut out = [0_u8; 6]; let outcome = ::decode_tile( &mut ctx, @@ -721,7 +924,7 @@ fn tile_batch_decode_matches_borrowed_decoder_decode() { .expect("tile decode"); assert_eq!(outcome.decoded, Rect::full((2, 1))); assert_eq!(out, expected); - assert_eq!(ctx.cache_stats().misses, 1); + assert_eq!(ctx.cache_stats(), j2k_core::CacheStats::default()); } #[test] @@ -735,7 +938,7 @@ fn tile_batch_region_decode_matches_decoder_region_decode() { h: 2, }; let mut decoder = J2kDecoder::new(&codestream).expect("decoder"); - let mut pool = signinum_j2k::J2kScratchPool::new(); + let mut pool = j2k::J2kScratchPool::new(); let mut expected = [0_u8; 4]; decoder .decode_region_into(&mut pool, &mut expected, 2, PixelFormat::Gray8, roi) @@ -761,7 +964,7 @@ fn tile_batch_scaled_decode_matches_decoder_scaled_decode() { let pixels: Vec = (0_u8..48).collect(); let codestream = encode_codestream(&pixels, 4, 4, 3, 8, true); let mut decoder = J2kDecoder::new(&codestream).expect("decoder"); - let mut pool = signinum_j2k::J2kScratchPool::new(); + let mut pool = j2k::J2kScratchPool::new(); let mut expected = [0_u8; 12]; decoder .decode_scaled_into( @@ -803,7 +1006,7 @@ fn tile_batch_region_scaled_decode_matches_decoder_region_scaled_decode() { let stride = scaled_roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); let mut decoder = J2kDecoder::new(&codestream).expect("decoder"); - let mut pool = signinum_j2k::J2kScratchPool::new(); + let mut pool = j2k::J2kScratchPool::new(); let mut expected = vec![0_u8; stride * scaled_roi.h as usize]; decoder .decode_region_scaled_into( @@ -838,7 +1041,7 @@ fn decode_region_into_rejects_out_of_bounds_roi() { let pixels = [0_u8, 1, 2, 3]; let codestream = encode_codestream(&pixels, 2, 2, 1, 8, true); let mut decoder = J2kDecoder::new(&codestream).expect("decoder"); - let mut pool = signinum_j2k::J2kScratchPool::new(); + let mut pool = j2k::J2kScratchPool::new(); let mut out = [0_u8; 4]; let err = decoder .decode_region_into( @@ -875,7 +1078,7 @@ fn decode_htj2k_scaled_into_matches_native_target_resolution_decode() { let codestream = encode_ht_codestream(&pixels, 4, 4, 1, 8); let expected = backend_decode_u8_scaled(&codestream, (2, 2)); let mut decoder = J2kDecoder::new(&codestream).expect("decoder"); - let mut pool = signinum_j2k::J2kScratchPool::new(); + let mut pool = j2k::J2kScratchPool::new(); let mut out = [0_u8; 4]; decoder .decode_scaled_into(&mut pool, &mut out, 2, PixelFormat::Gray8, Downscale::Half) @@ -910,7 +1113,7 @@ fn tile_batch_decode_matches_borrowed_decoder_for_htj2k() { .expect("decoder decode"); let mut ctx = DecoderContext::::new(); - let mut pool = signinum_j2k::J2kScratchPool::new(); + let mut pool = j2k::J2kScratchPool::new(); let mut out = [0_u8; 4]; ::decode_tile( &mut ctx, diff --git a/crates/signinum-j2k/tests/device_plan.rs b/crates/j2k/tests/device_plan.rs similarity index 93% rename from crates/signinum-j2k/tests/device_plan.rs rename to crates/j2k/tests/device_plan.rs index bcf64174..c01abb86 100644 --- a/crates/signinum-j2k/tests/device_plan.rs +++ b/crates/j2k/tests/device_plan.rs @@ -1,5 +1,5 @@ -use signinum_core::{Downscale, Rect}; -use signinum_j2k::adapter::device_plan::{DeviceDecodePlan, DeviceDecodeRequest}; +use j2k::adapter::device_plan::{DeviceDecodePlan, DeviceDecodeRequest}; +use j2k_core::{Downscale, Rect}; #[test] fn full_request_plan_uses_full_source_rect() { diff --git a/crates/signinum-j2k/tests/encode_lossless.rs b/crates/j2k/tests/encode_lossless.rs similarity index 55% rename from crates/signinum-j2k/tests/encode_lossless.rs rename to crates/j2k/tests/encode_lossless.rs index e5281849..a0e366e0 100644 --- a/crates/signinum-j2k/tests/encode_lossless.rs +++ b/crates/j2k/tests/encode_lossless.rs @@ -2,24 +2,130 @@ use std::path::PathBuf; -use signinum_core::{BackendKind, CodecError}; -use signinum_j2k::{ +use j2k::adapter::encode_stage::{ + EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, J2kCodeBlockSegment, J2kCodeBlockStyle, + J2kDeinterleaveToF32Job, J2kEncodeDispatchReport, J2kEncodeStageAccelerator, + J2kHtCodeBlockEncodeJob, J2kPacketizationEncodeJob, J2kQuantizeSubbandJob, J2kSubBandType, + J2kTier1CodeBlockEncodeJob, +}; +use j2k::{ encode_j2k_lossless, encode_j2k_lossless_with_accelerator, j2k_lossless_decomposition_levels, j2k_lossless_decomposition_levels_for_options, - j2k_lossless_decomposition_levels_for_progression, EncodeBackendPreference, - EncodedHtJ2kCodeBlock, J2kBlockCodingMode, J2kEncodeDispatchReport, J2kEncodeStageAccelerator, - J2kEncodeValidation, J2kHtCodeBlockEncodeJob, J2kLosslessEncodeOptions, J2kLosslessSamples, - J2kPacketizationEncodeJob, J2kProgressionOrder, ReversibleTransform, + j2k_lossless_decomposition_levels_for_progression, EncodeBackendPreference, J2kBlockCodingMode, + J2kEncodeValidation, J2kLosslessEncodeOptions, J2kLosslessSamples, J2kProgressionOrder, + ReversibleTransform, }; -use signinum_j2k_native::{DecodeSettings, Image}; +use j2k_core::{BackendKind, CodecError}; +use j2k_native::{DecodeSettings, Image}; -fn decode_native(codestream: &[u8]) -> signinum_j2k_native::RawBitmap { +fn decode_native(codestream: &[u8]) -> j2k_native::RawBitmap { Image::new(codestream, &DecodeSettings::default()) .expect("encoded codestream should parse") .decode_native() .expect("encoded codestream should decode") } +fn cpu_options() -> J2kLosslessEncodeOptions { + J2kLosslessEncodeOptions::default().with_backend(EncodeBackendPreference::CpuOnly) +} + +fn auto_options() -> J2kLosslessEncodeOptions { + J2kLosslessEncodeOptions::default().with_backend(EncodeBackendPreference::Auto) +} + +fn require_device_options() -> J2kLosslessEncodeOptions { + J2kLosslessEncodeOptions::default().with_backend(EncodeBackendPreference::RequireDevice) +} + +fn deinterleave_to_f32_for_test(job: J2kDeinterleaveToF32Job<'_>) -> Vec> { + let num_components = usize::from(job.num_components); + let bytes_per_sample = if job.bit_depth <= 8 { 1 } else { 2 }; + assert_eq!( + job.pixels.len(), + job.num_pixels * num_components * bytes_per_sample + ); + + let unsigned_offset = if job.signed { + 0 + } else { + 1_i32 << u32::from(job.bit_depth.saturating_sub(1)) + }; + let mut components = vec![vec![0.0; job.num_pixels]; num_components]; + for pixel_idx in 0..job.num_pixels { + for (component_idx, component) in components.iter_mut().enumerate() { + let sample_idx = pixel_idx * num_components + component_idx; + let sample = if job.bit_depth <= 8 { + let byte = job.pixels[sample_idx]; + if job.signed { + i16::from(byte as i8) + } else { + i16::try_from(i32::from(byte) - unsigned_offset) + .expect("level-shifted 8-bit sample fits in i16") + } + } else { + let byte_idx = sample_idx * 2; + let bytes = [job.pixels[byte_idx], job.pixels[byte_idx + 1]]; + if job.signed { + i16::from_le_bytes(bytes) + } else { + i16::try_from(i32::from(u16::from_le_bytes(bytes)) - unsigned_offset) + .expect("level-shifted 16-bit sample fits in i16") + } + }; + component[pixel_idx] = f32::from(sample); + } + } + components +} + +fn native_subband(subband: J2kSubBandType) -> j2k_native::J2kSubBandType { + match subband { + J2kSubBandType::LowLow => j2k_native::J2kSubBandType::LowLow, + J2kSubBandType::HighLow => j2k_native::J2kSubBandType::HighLow, + J2kSubBandType::LowHigh => j2k_native::J2kSubBandType::LowHigh, + J2kSubBandType::HighHigh => j2k_native::J2kSubBandType::HighHigh, + } +} + +fn native_code_block_style(style: J2kCodeBlockStyle) -> j2k_native::J2kCodeBlockStyle { + j2k_native::J2kCodeBlockStyle { + selective_arithmetic_coding_bypass: style.selective_arithmetic_coding_bypass, + reset_context_probabilities: style.reset_context_probabilities, + termination_on_each_pass: style.termination_on_each_pass, + vertically_causal_context: style.vertically_causal_context, + segmentation_symbols: style.segmentation_symbols, + } +} + +fn public_encoded_j2k(block: j2k_native::EncodedJ2kCodeBlock) -> EncodedJ2kCodeBlock { + EncodedJ2kCodeBlock { + data: block.data, + segments: block + .segments + .into_iter() + .map(|segment| J2kCodeBlockSegment { + data_offset: segment.data_offset, + data_length: segment.data_length, + start_coding_pass: segment.start_coding_pass, + end_coding_pass: segment.end_coding_pass, + use_arithmetic: segment.use_arithmetic, + }) + .collect(), + number_of_coding_passes: block.number_of_coding_passes, + missing_bit_planes: block.missing_bit_planes, + } +} + +fn public_encoded_ht(block: j2k_native::EncodedHtJ2kCodeBlock) -> EncodedHtJ2kCodeBlock { + EncodedHtJ2kCodeBlock { + data: block.data, + cleanup_length: block.cleanup_length, + refinement_length: block.refinement_length, + num_coding_passes: block.num_coding_passes, + num_zero_bitplanes: block.num_zero_bitplanes, + } +} + #[test] fn default_lossless_options_use_auto_cpu_safe_profile() { let options = J2kLosslessEncodeOptions::default(); @@ -39,11 +145,7 @@ fn lossless_encode_can_skip_facade_cpu_validation_for_external_validation() { let encoded = encode_j2k_lossless( samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - }, + &cpu_options().with_validation(J2kEncodeValidation::External), ) .expect("lossless encode without facade CPU validation"); @@ -58,11 +160,7 @@ fn cpu_htj2k_lossless_round_trips_gray8() { let encoded = encode_j2k_lossless( samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - block_coding_mode: J2kBlockCodingMode::HighThroughput, - ..J2kLosslessEncodeOptions::default() - }, + &cpu_options().with_block_coding_mode(J2kBlockCodingMode::HighThroughput), ) .expect("HTJ2K lossless encode"); @@ -83,12 +181,9 @@ fn cpu_htj2k_rpcl_writes_cod_rpcl_and_tlm() { let encoded = encode_j2k_lossless( samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - block_coding_mode: J2kBlockCodingMode::HighThroughput, - progression: J2kProgressionOrder::Rpcl, - ..J2kLosslessEncodeOptions::default() - }, + &cpu_options() + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_progression(J2kProgressionOrder::Rpcl), ) .expect("HTJ2K RPCL lossless encode"); @@ -98,6 +193,43 @@ fn cpu_htj2k_rpcl_writes_cod_rpcl_and_tlm() { assert_eq!(decode_native(&encoded.codestream).data, pixels); } +#[test] +fn cpu_lossless_all_progression_orders_write_cod_marker_and_round_trip() { + let mut pixels = Vec::with_capacity(64 * 64 * 3); + for y in 0..64u8 { + for x in 0..64u8 { + pixels.push(x.wrapping_mul(3).wrapping_add(y)); + pixels.push(y.wrapping_mul(5).wrapping_add(x / 2)); + pixels.push(x.wrapping_mul(7).wrapping_sub(y.wrapping_mul(2))); + } + } + + for (progression, marker_byte) in [ + (J2kProgressionOrder::Lrcp, 0x00), + (J2kProgressionOrder::Rlcp, 0x01), + (J2kProgressionOrder::Rpcl, 0x02), + (J2kProgressionOrder::Pcrl, 0x03), + (J2kProgressionOrder::Cprl, 0x04), + ] { + let samples = J2kLosslessSamples::new(&pixels, 64, 64, 3, 8, false).unwrap(); + let encoded = encode_j2k_lossless( + samples, + &cpu_options() + .with_progression(progression) + .with_reversible_transform(ReversibleTransform::None53), + ) + .unwrap_or_else(|err| panic!("{progression:?} encode failed: {err}")); + + let cod_offset = marker_offset(&encoded.codestream, 0x52).expect("COD marker"); + assert_eq!(encoded.codestream[cod_offset + 5], marker_byte); + assert_eq!( + decode_native(&encoded.codestream).data, + pixels, + "{progression:?} round trip" + ); + } +} + #[test] fn default_lossless_policy_enables_one_reversible_dwt_level_for_wsi_tiles() { let gray = vec![0; 64 * 64]; @@ -140,11 +272,9 @@ fn max_decomposition_level_option_caps_rpcl_without_forcing_small_tiles() { assert_eq!( j2k_lossless_decomposition_levels_for_options( large, - J2kLosslessEncodeOptions { - progression: J2kProgressionOrder::Rpcl, - max_decomposition_levels: Some(1), - ..J2kLosslessEncodeOptions::default() - } + J2kLosslessEncodeOptions::default() + .with_progression(J2kProgressionOrder::Rpcl) + .with_max_decomposition_levels(Some(1)) ), 1 ); @@ -155,11 +285,9 @@ fn max_decomposition_level_option_caps_rpcl_without_forcing_small_tiles() { assert_eq!( j2k_lossless_decomposition_levels_for_options( small, - J2kLosslessEncodeOptions { - progression: J2kProgressionOrder::Rpcl, - max_decomposition_levels: Some(1), - ..J2kLosslessEncodeOptions::default() - } + J2kLosslessEncodeOptions::default() + .with_progression(J2kProgressionOrder::Rpcl) + .with_max_decomposition_levels(Some(1)) ), 0 ); @@ -170,14 +298,7 @@ fn cpu_lossless_round_trips_gray8() { let pixels: Vec = (0..35).map(|v| (v * 7) as u8).collect(); let samples = J2kLosslessSamples::new(&pixels, 7, 5, 1, 8, false).unwrap(); - let encoded = encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - ..J2kLosslessEncodeOptions::default() - }, - ) - .expect("lossless encode"); + let encoded = encode_j2k_lossless(samples, &cpu_options()).expect("lossless encode"); assert_eq!(encoded.backend, BackendKind::Cpu); assert_eq!(encoded.width, 7); @@ -195,18 +316,56 @@ fn cpu_lossless_round_trips_gray8() { } #[test] -fn cpu_classic_lossless_cod_marker_length_reaches_next_marker() { - let pixels = vec![127u8; 64 * 64 * 3]; - let samples = J2kLosslessSamples::new(&pixels, 64, 64, 3, 8, false).unwrap(); +fn cpu_lossless_round_trips_two_component_no_mct_with_strict_decode() { + let mut pixels = Vec::with_capacity(11 * 7 * 2); + for y in 0..7u8 { + for x in 0..11u8 { + pixels.push(x.wrapping_mul(17).wrapping_add(y.wrapping_mul(3))); + pixels.push(255u8.wrapping_sub(x.wrapping_mul(5).wrapping_add(y.wrapping_mul(11)))); + } + } + let samples = + J2kLosslessSamples::new(&pixels, 11, 7, 2, 8, false).expect("2-component samples"); let encoded = encode_j2k_lossless( samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - ..J2kLosslessEncodeOptions::default() + &cpu_options().with_reversible_transform(ReversibleTransform::None53), + ) + .expect("2-component lossless encode"); + + let cod_offset = marker_offset(&encoded.codestream, 0x52).expect("COD marker"); + assert_eq!( + encoded.codestream[cod_offset + 8], + 0, + "2-component output must not use MCT" + ); + + let image = Image::new( + &encoded.codestream, + &DecodeSettings { + resolve_palette_indices: true, + strict: true, + target_resolution: None, }, ) - .expect("cpu lossless encode"); + .expect("strict parse of 2-component codestream"); + let decoded = image + .decode_native() + .expect("strict decode of 2-component codestream"); + + assert_eq!(decoded.width, 11); + assert_eq!(decoded.height, 7); + assert_eq!(decoded.num_components, 2); + assert_eq!(decoded.bit_depth, 8); + assert_eq!(decoded.data, pixels); +} + +#[test] +fn cpu_classic_lossless_cod_marker_length_reaches_next_marker() { + let pixels = vec![127u8; 64 * 64 * 3]; + let samples = J2kLosslessSamples::new(&pixels, 64, 64, 3, 8, false).unwrap(); + + let encoded = encode_j2k_lossless(samples, &cpu_options()).expect("cpu lossless encode"); let cod_offset = marker_offset(&encoded.codestream, 0x52).expect("COD marker"); let qcd_offset = marker_offset(&encoded.codestream, 0x5C).expect("QCD marker"); @@ -255,14 +414,7 @@ fn cpu_lossless_round_trips_rgb8_high_variance_512() { } let samples = J2kLosslessSamples::new(&pixels, 512, 512, 3, 8, false).unwrap(); - let encoded = encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - ..J2kLosslessEncodeOptions::default() - }, - ) - .expect("cpu lossless encode"); + let encoded = encode_j2k_lossless(samples, &cpu_options()).expect("cpu lossless encode"); let decoded = decode_native(&encoded.codestream); assert_eq!(decoded.data, pixels); @@ -273,14 +425,7 @@ fn cpu_lossless_round_trips_rgb8_constant_gray_512() { let pixels = vec![243u8; 512 * 512 * 3]; let samples = J2kLosslessSamples::new(&pixels, 512, 512, 3, 8, false).unwrap(); - let encoded = encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - ..J2kLosslessEncodeOptions::default() - }, - ) - .expect("cpu lossless encode"); + let encoded = encode_j2k_lossless(samples, &cpu_options()).expect("cpu lossless encode"); let decoded = decode_native(&encoded.codestream); assert_eq!(decoded.data, pixels); @@ -299,14 +444,7 @@ fn cpu_lossless_round_trips_rgb8_low_variance_slide_like_512() { } let samples = J2kLosslessSamples::new(&pixels, 512, 512, 3, 8, false).unwrap(); - let encoded = encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - ..J2kLosslessEncodeOptions::default() - }, - ) - .expect("cpu lossless encode"); + let encoded = encode_j2k_lossless(samples, &cpu_options()).expect("cpu lossless encode"); let decoded = decode_native(&encoded.codestream); assert_eq!(decoded.data, pixels); @@ -327,40 +465,33 @@ fn cpu_lossless_round_trips_rgb8_variable_chroma_512() { } let samples = J2kLosslessSamples::new(&pixels, 512, 512, 3, 8, false).unwrap(); - let encoded = encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - ..J2kLosslessEncodeOptions::default() - }, - ) - .expect("cpu lossless encode"); + let encoded = encode_j2k_lossless(samples, &cpu_options()).expect("cpu lossless encode"); let decoded = decode_native(&encoded.codestream); assert_eq!(decoded.data, pixels); } #[test] -#[ignore = "requires SIGNINUM_J2K_APERIO_TILE_FIXTURE"] +#[ignore = "requires J2K_APERIO_TILE_FIXTURE"] fn cpu_lossless_round_trips_aperio_jp2k_problem_tile_512() { - let Some(path) = std::env::var_os("SIGNINUM_J2K_APERIO_TILE_FIXTURE").map(PathBuf::from) else { + let Some(path) = std::env::var_os("J2K_APERIO_TILE_FIXTURE").map(PathBuf::from) else { return; }; let pixels = std::fs::read(&path).expect("problem tile fixture"); assert_eq!(pixels.len(), 512 * 512 * 3); let samples = J2kLosslessSamples::new(&pixels, 512, 512, 3, 8, false).unwrap(); - let codestream = signinum_j2k_native::encode( + let codestream = j2k_native::encode( samples.data, samples.width, samples.height, samples.components, samples.bit_depth, samples.signed, - &signinum_j2k_native::EncodeOptions { + &j2k_native::EncodeOptions { reversible: true, num_decomposition_levels: 0, - ..signinum_j2k_native::EncodeOptions::default() + ..j2k_native::EncodeOptions::default() }, ) .expect("cpu lossless encode"); @@ -389,14 +520,7 @@ fn cpu_lossless_round_trips_rgb8_seed_130_64() { } let samples = J2kLosslessSamples::new(&pixels, 64, 64, 3, 8, false).unwrap(); - let encoded = encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - ..J2kLosslessEncodeOptions::default() - }, - ) - .expect("cpu lossless encode"); + let encoded = encode_j2k_lossless(samples, &cpu_options()).expect("cpu lossless encode"); let decoded = decode_native(&encoded.codestream); assert_eq!(decoded.data, pixels); @@ -412,32 +536,19 @@ fn cpu_lossless_round_trips_gray8_seed_104_64() { } let samples = J2kLosslessSamples::new(&pixels, 64, 64, 1, 8, false).unwrap(); - let encoded = encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - ..J2kLosslessEncodeOptions::default() - }, - ) - .expect("cpu lossless encode"); + let encoded = encode_j2k_lossless(samples, &cpu_options()).expect("cpu lossless encode"); let decoded = decode_native(&encoded.codestream); assert_eq!(decoded.data, pixels); } #[test] -fn prefer_device_falls_back_to_validated_cpu_until_device_encode_is_complete() { +fn auto_falls_back_to_validated_cpu_until_device_encode_is_complete() { let pixels: Vec = (0..27).map(|v| (v * 3) as u8).collect(); let samples = J2kLosslessSamples::new(&pixels, 3, 3, 3, 8, false).unwrap(); - let encoded = encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::PreferDevice, - ..J2kLosslessEncodeOptions::default() - }, - ) - .expect("prefer-device lossless encode"); + let encoded = + encode_j2k_lossless(samples, &auto_options()).expect("prefer-device lossless encode"); assert_eq!(encoded.backend, BackendKind::Cpu); let decoded = decode_native(&encoded.codestream); @@ -449,14 +560,7 @@ fn require_device_errors_clearly_when_encode_backend_is_unavailable() { let pixels = vec![0u8; 4 * 4]; let samples = J2kLosslessSamples::new(&pixels, 4, 4, 1, 8, false).unwrap(); - let err = encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - ) - .unwrap_err(); + let err = encode_j2k_lossless(samples, &require_device_options()).unwrap_err(); assert!(err.is_unsupported()); assert!(err.to_string().contains("device")); @@ -464,7 +568,7 @@ fn require_device_errors_clearly_when_encode_backend_is_unavailable() { } #[test] -fn accelerator_facade_prefer_device_falls_back_when_no_stage_dispatches() { +fn accelerator_facade_auto_falls_back_when_no_stage_dispatches() { #[derive(Default)] struct NoDispatchAccelerator; @@ -476,10 +580,7 @@ fn accelerator_facade_prefer_device_falls_back_when_no_stage_dispatches() { let encoded = encode_j2k_lossless_with_accelerator( samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::PreferDevice, - ..J2kLosslessEncodeOptions::default() - }, + &auto_options(), BackendKind::Metal, &mut accelerator, ) @@ -502,10 +603,7 @@ fn accelerator_facade_require_device_errors_when_no_stage_dispatches() { let err = encode_j2k_lossless_with_accelerator( samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, + &require_device_options(), BackendKind::Metal, &mut accelerator, ) @@ -519,22 +617,47 @@ fn accelerator_facade_require_device_errors_when_no_stage_dispatches() { fn accelerator_facade_require_device_errors_when_any_required_stage_is_missing() { #[derive(Default)] struct PacketizationDispatchAccelerator { - packetization_dispatches: usize, + deinterleave: usize, + quantize_subband: usize, + packetization: usize, } impl J2kEncodeStageAccelerator for PacketizationDispatchAccelerator { fn dispatch_report(&self) -> J2kEncodeDispatchReport { J2kEncodeDispatchReport { - packetization: self.packetization_dispatches, + deinterleave: self.deinterleave, + quantize_subband: self.quantize_subband, + packetization: self.packetization, ..J2kEncodeDispatchReport::default() } } + fn encode_deinterleave( + &mut self, + job: J2kDeinterleaveToF32Job<'_>, + ) -> core::result::Result>>, &'static str> { + self.deinterleave = self.deinterleave.saturating_add(1); + Ok(Some(deinterleave_to_f32_for_test(job))) + } + + fn encode_quantize_subband( + &mut self, + job: J2kQuantizeSubbandJob<'_>, + ) -> core::result::Result>, &'static str> { + self.quantize_subband = self.quantize_subband.saturating_add(1); + Ok(Some( + job.coefficients + .iter() + .map(|sample| sample.round() as i32) + .collect(), + )) + } + fn encode_packetization( &mut self, _job: J2kPacketizationEncodeJob<'_>, ) -> core::result::Result>, &'static str> { - self.packetization_dispatches = self.packetization_dispatches.saturating_add(1); + self.packetization = self.packetization.saturating_add(1); Ok(None) } } @@ -545,10 +668,7 @@ fn accelerator_facade_require_device_errors_when_any_required_stage_is_missing() let err = encode_j2k_lossless_with_accelerator( samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, + &require_device_options(), BackendKind::Metal, &mut accelerator, ) @@ -562,32 +682,58 @@ fn accelerator_facade_require_device_errors_when_any_required_stage_is_missing() fn accelerator_facade_reports_requested_backend_after_all_required_stages_dispatch() { #[derive(Default)] struct FullClassicAccelerator { - tier1_code_block_dispatches: usize, - packetization_dispatches: usize, + deinterleave: usize, + quantize_subband: usize, + tier1_code_block: usize, + packetization: usize, } impl J2kEncodeStageAccelerator for FullClassicAccelerator { fn dispatch_report(&self) -> J2kEncodeDispatchReport { J2kEncodeDispatchReport { - tier1_code_block: self.tier1_code_block_dispatches, - packetization: self.packetization_dispatches, + deinterleave: self.deinterleave, + quantize_subband: self.quantize_subband, + tier1_code_block: self.tier1_code_block, + packetization: self.packetization, ..J2kEncodeDispatchReport::default() } } + fn encode_deinterleave( + &mut self, + job: J2kDeinterleaveToF32Job<'_>, + ) -> core::result::Result>>, &'static str> { + self.deinterleave = self.deinterleave.saturating_add(1); + Ok(Some(deinterleave_to_f32_for_test(job))) + } + + fn encode_quantize_subband( + &mut self, + job: J2kQuantizeSubbandJob<'_>, + ) -> core::result::Result>, &'static str> { + self.quantize_subband = self.quantize_subband.saturating_add(1); + Ok(Some( + job.coefficients + .iter() + .map(|sample| sample.round() as i32) + .collect(), + )) + } + fn encode_tier1_code_block( &mut self, - job: signinum_j2k::J2kTier1CodeBlockEncodeJob<'_>, - ) -> core::result::Result, &'static str> { - self.tier1_code_block_dispatches = self.tier1_code_block_dispatches.saturating_add(1); - signinum_j2k_native::encode_j2k_code_block_scalar_with_style( + job: J2kTier1CodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + self.tier1_code_block = self.tier1_code_block.saturating_add(1); + j2k_native::encode_j2k_code_block_scalar_with_style( job.coefficients, job.width, job.height, - job.sub_band_type, + native_subband(job.sub_band_type), job.total_bitplanes, - job.style, + native_code_block_style(job.style), ) + .map(public_encoded_j2k) .map(Some) } @@ -595,7 +741,7 @@ fn accelerator_facade_reports_requested_backend_after_all_required_stages_dispat &mut self, _job: J2kPacketizationEncodeJob<'_>, ) -> core::result::Result>, &'static str> { - self.packetization_dispatches = self.packetization_dispatches.saturating_add(1); + self.packetization = self.packetization.saturating_add(1); Ok(None) } } @@ -606,10 +752,7 @@ fn accelerator_facade_reports_requested_backend_after_all_required_stages_dispat let encoded = encode_j2k_lossless_with_accelerator( samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::PreferDevice, - ..J2kLosslessEncodeOptions::default() - }, + &auto_options(), BackendKind::Metal, &mut accelerator, ) @@ -623,44 +766,64 @@ fn accelerator_facade_reports_requested_backend_after_all_required_stages_dispat fn accelerator_facade_ht_require_device_checks_ht_code_block_stage() { #[derive(Default)] struct FullHtAccelerator { - ht_code_block_dispatches: usize, - packetization_dispatches: usize, + deinterleave: usize, + quantize_subband: usize, + ht_code_block: usize, + packetization: usize, } impl J2kEncodeStageAccelerator for FullHtAccelerator { fn dispatch_report(&self) -> J2kEncodeDispatchReport { J2kEncodeDispatchReport { - ht_code_block: self.ht_code_block_dispatches, - packetization: self.packetization_dispatches, + deinterleave: self.deinterleave, + quantize_subband: self.quantize_subband, + ht_code_block: self.ht_code_block, + packetization: self.packetization, ..J2kEncodeDispatchReport::default() } } + fn encode_deinterleave( + &mut self, + job: J2kDeinterleaveToF32Job<'_>, + ) -> core::result::Result>>, &'static str> { + self.deinterleave = self.deinterleave.saturating_add(1); + Ok(Some(deinterleave_to_f32_for_test(job))) + } + + fn encode_quantize_subband( + &mut self, + job: J2kQuantizeSubbandJob<'_>, + ) -> core::result::Result>, &'static str> { + self.quantize_subband = self.quantize_subband.saturating_add(1); + Ok(Some( + job.coefficients + .iter() + .map(|sample| sample.round() as i32) + .collect(), + )) + } + fn encode_ht_code_block( &mut self, job: J2kHtCodeBlockEncodeJob<'_>, ) -> core::result::Result, &'static str> { - self.ht_code_block_dispatches = self.ht_code_block_dispatches.saturating_add(1); - signinum_j2k_native::encode_ht_code_block_scalar( + self.ht_code_block = self.ht_code_block.saturating_add(1); + j2k_native::encode_ht_code_block_scalar( job.coefficients, job.width, job.height, job.total_bitplanes, ) - .map(|encoded| { - Some(EncodedHtJ2kCodeBlock { - data: encoded.data, - num_coding_passes: encoded.num_coding_passes, - num_zero_bitplanes: encoded.num_zero_bitplanes, - }) - }) + .map(public_encoded_ht) + .map(Some) } fn encode_packetization( &mut self, _job: J2kPacketizationEncodeJob<'_>, ) -> core::result::Result>, &'static str> { - self.packetization_dispatches = self.packetization_dispatches.saturating_add(1); + self.packetization = self.packetization.saturating_add(1); Ok(None) } } @@ -671,11 +834,7 @@ fn accelerator_facade_ht_require_device_checks_ht_code_block_stage() { let encoded = encode_j2k_lossless_with_accelerator( samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - block_coding_mode: J2kBlockCodingMode::HighThroughput, - ..J2kLosslessEncodeOptions::default() - }, + &require_device_options().with_block_coding_mode(J2kBlockCodingMode::HighThroughput), BackendKind::Metal, &mut accelerator, ) diff --git a/crates/j2k/tests/encode_lossy.rs b/crates/j2k/tests/encode_lossy.rs new file mode 100644 index 00000000..efe7e0dd --- /dev/null +++ b/crates/j2k/tests/encode_lossy.rs @@ -0,0 +1,641 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::adapter::encode_stage::{ + EncodedHtJ2kCodeBlock, J2kDeinterleaveToF32Job, J2kEncodeDispatchReport, + J2kEncodeStageAccelerator, J2kHtCodeBlockEncodeJob, J2kPacketizationEncodeJob, + J2kQuantizeSubbandJob, +}; +use j2k::{ + encode_j2k_lossy, encode_j2k_lossy_with_accelerator, EncodeBackendPreference, + J2kBlockCodingMode, J2kEncodeValidation, J2kLossyEncodeOptions, J2kLossySamples, + J2kMarkerSegment, J2kProgressionOrder, J2kQualityLayer, J2kRateTarget, +}; +use j2k_core::{BackendKind, CodecError}; +use j2k_native::{DecodeSettings, Image}; + +fn decode_native(codestream: &[u8]) -> j2k_native::RawBitmap { + Image::new(codestream, &DecodeSettings::default()) + .expect("encoded codestream should parse") + .decode_native() + .expect("encoded codestream should decode") +} + +fn strict_decode_native(codestream: &[u8]) -> j2k_native::RawBitmap { + Image::new( + codestream, + &DecodeSettings { + strict: true, + ..DecodeSettings::default() + }, + ) + .expect("encoded codestream should parse strictly") + .decode_native() + .expect("encoded codestream should decode strictly") +} + +fn public_encoded_ht(block: j2k_native::EncodedHtJ2kCodeBlock) -> EncodedHtJ2kCodeBlock { + EncodedHtJ2kCodeBlock { + data: block.data, + cleanup_length: block.cleanup_length, + refinement_length: block.refinement_length, + num_coding_passes: block.num_coding_passes, + num_zero_bitplanes: block.num_zero_bitplanes, + } +} + +fn plt_packet_length_count(codestream: &[u8]) -> usize { + plt_packet_lengths(codestream).len() +} + +fn plt_packet_lengths(codestream: &[u8]) -> Vec { + let plt = codestream + .windows(2) + .position(|window| window == [0xFF, 0x58]) + .expect("PLT marker"); + let marker_len = u16::from_be_bytes([codestream[plt + 2], codestream[plt + 3]]) as usize; + let payload = &codestream[plt + 5..plt + 2 + marker_len]; + let mut lengths = Vec::new(); + let mut value = 0_u32; + let mut in_progress = false; + for &byte in payload { + value = (value << 7) + u32::from(byte & 0x7F); + in_progress = true; + if byte & 0x80 == 0 { + lengths.push(value); + value = 0; + in_progress = false; + } + } + assert!(!in_progress, "PLT packet length is incomplete"); + lengths +} + +#[test] +fn lossy_sample_descriptor_rejects_more_than_sixteen_bits_explicitly() { + let pixels = vec![0u8; 2 * 2 * 2]; + + let err = J2kLossySamples::new(&pixels, 2, 2, 1, 17, false).unwrap_err(); + + assert!(err.is_unsupported()); + assert!(err.to_string().contains("1-16 bits per sample")); +} + +#[test] +fn lossy_quality_layer_targets_must_be_cumulative() { + let pixels = vec![0u8; 16 * 16]; + let samples = J2kLossySamples::new(&pixels, 16, 16, 1, 8, false).unwrap(); + + let err = encode_j2k_lossy( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_quality_layers(vec![ + J2kQualityLayer::new(J2kRateTarget::Bytes(2048)), + J2kQualityLayer::new(J2kRateTarget::Bytes(1024)), + ]), + ) + .unwrap_err(); + + assert!(err.is_unsupported()); + assert!(err.to_string().contains("cumulative")); +} + +#[test] +fn cpu_classic_lossy_bits_per_pixel_target_encodes_parseable_codestream() { + let pixels: Vec = (0..64 * 64) + .map(|index| (((index * 13) ^ (index / 5)) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 64, 64, 1, 8, false).unwrap(); + + let encoded = encode_j2k_lossy( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_rate_target(Some(J2kRateTarget::BitsPerPixel(2.0))) + .with_progression(J2kProgressionOrder::Rlcp), + ) + .expect("classic lossy encode"); + + assert_eq!(encoded.backend, BackendKind::Cpu); + assert_eq!(encoded.width, 64); + assert_eq!(encoded.height, 64); + assert_eq!(encoded.components, 1); + assert_eq!(encoded.bit_depth, 8); + assert!(encoded.codestream.starts_with(&[0xFF, 0x4F])); + assert!(encoded.report.actual_bits_per_pixel.is_finite()); + assert!(encoded.report.psnr_db.expect("PSNR report").is_finite()); + + let decoded = strict_decode_native(&encoded.codestream); + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.num_components, 1); +} + +#[test] +fn cpu_classic_lossy_multiple_quality_layers_encode_scalable_codestream() { + let pixels: Vec = (0..64 * 64) + .map(|index| (((index * 17) + (index / 3)) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 64, 64, 1, 8, false).unwrap(); + + let encoded = encode_j2k_lossy( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_quality_layers(vec![ + J2kQualityLayer::new(J2kRateTarget::BitsPerPixel(0.75)), + J2kQualityLayer::new(J2kRateTarget::BitsPerPixel(1.5)), + ]), + ) + .expect("classic lossy multilayer encode"); + + assert_eq!(encoded.report.quality_layers, 2); + let decoded = decode_native(&encoded.codestream); + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.num_components, 1); + + let cod = encoded + .codestream + .windows(2) + .position(|window| window == [0xFF, 0x52]) + .expect("COD marker"); + assert_eq!( + u16::from_be_bytes([encoded.codestream[cod + 6], encoded.codestream[cod + 7]]), + 2 + ); + assert_eq!(encoded.codestream[cod + 12] & 0x04, 0x04); +} + +#[test] +fn cpu_classic_lossy_round_trips_two_component_no_mct_with_strict_decode() { + let mut pixels = Vec::with_capacity(17 * 13 * 2); + for y in 0..13u8 { + for x in 0..17u8 { + pixels.push(x.wrapping_mul(11).wrapping_add(y.wrapping_mul(7))); + pixels.push(255u8.wrapping_sub(x.wrapping_mul(3).wrapping_add(y.wrapping_mul(19)))); + } + } + let samples = J2kLossySamples::new(&pixels, 17, 13, 2, 8, false).unwrap(); + + let encoded = encode_j2k_lossy( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_max_decomposition_levels(Some(0)), + ) + .expect("2-component lossy encode"); + + let cod = encoded + .codestream + .windows(2) + .position(|window| window == [0xFF, 0x52]) + .expect("COD marker"); + assert_eq!( + encoded.codestream[cod + 8], + 0, + "2-component lossy output must not use MCT" + ); + + let decoded = strict_decode_native(&encoded.codestream); + assert_eq!(decoded.width, 17); + assert_eq!(decoded.height, 13); + assert_eq!(decoded.num_components, 2); + assert_eq!(decoded.bit_depth, 8); +} + +#[test] +fn cpu_classic_lossy_quality_layer_byte_targets_bound_early_layer_packets() { + let pixels: Vec = (0..128 * 128) + .map(|index| (((index * 53) ^ (index / 11)) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 128, 128, 1, 8, false).unwrap(); + + let encoded = encode_j2k_lossy( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_max_decomposition_levels(Some(0)) + .with_marker_segments(vec![J2kMarkerSegment::Plt]) + .with_quality_layers(vec![ + J2kQualityLayer::new(J2kRateTarget::Bytes(256)), + J2kQualityLayer::new(J2kRateTarget::Bytes(20_000)), + ]), + ) + .expect("classic lossy PCRD encode"); + + let packet_lengths = plt_packet_lengths(&encoded.codestream); + assert_eq!(packet_lengths.len(), 2); + assert!( + packet_lengths[0] > 1, + "first layer packet should carry legal pass-truncated data" + ); + assert!( + packet_lengths[0] <= 768, + "first layer packet length {} exceeded target+tolerance", + packet_lengths[0] + ); + assert!(packet_lengths[1] > 0); + + let decoded = strict_decode_native(&encoded.codestream); + assert_eq!(decoded.width, 128); + assert_eq!(decoded.height, 128); + assert_eq!(decoded.num_components, 1); +} + +#[test] +fn cpu_classic_lossy_multiple_quality_layers_decode_all_progressions() { + let pixels: Vec = (0..64 * 64) + .map(|index| (((index * 29) ^ (index / 7)) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 64, 64, 1, 8, false).unwrap(); + + for progression in [ + J2kProgressionOrder::Lrcp, + J2kProgressionOrder::Rlcp, + J2kProgressionOrder::Rpcl, + J2kProgressionOrder::Pcrl, + J2kProgressionOrder::Cprl, + ] { + let encoded = encode_j2k_lossy( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_progression(progression) + .with_quality_layers(vec![ + J2kQualityLayer::new(J2kRateTarget::BitsPerPixel(0.75)), + J2kQualityLayer::new(J2kRateTarget::BitsPerPixel(1.5)), + ]), + ) + .unwrap_or_else(|err| panic!("{progression:?} multilayer encode failed: {err}")); + + let decoded = decode_native(&encoded.codestream); + assert_eq!(decoded.width, 64, "{progression:?}"); + assert_eq!(decoded.height, 64, "{progression:?}"); + assert_eq!(decoded.num_components, 1, "{progression:?}"); + } +} + +#[test] +fn cpu_classic_lossy_emits_plt_and_plm_that_strict_decode_uses() { + let pixels: Vec = (0..64 * 64) + .map(|index| (((index * 11) + (index / 9)) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 64, 64, 1, 8, false).unwrap(); + + let encoded = encode_j2k_lossy( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_marker_segments(vec![J2kMarkerSegment::Plt, J2kMarkerSegment::Plm]), + ) + .expect("classic lossy PLT/PLM encode"); + + assert!( + encoded + .codestream + .windows(2) + .any(|marker| marker == [0xFF, 0x58]), + "PLT marker must be emitted" + ); + assert!( + encoded + .codestream + .windows(2) + .any(|marker| marker == [0xFF, 0x57]), + "PLM marker must be emitted" + ); + + let decoded = strict_decode_native(&encoded.codestream); + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.num_components, 1); +} + +#[test] +fn cpu_classic_lossy_emits_sop_and_eph_that_strict_decode_uses() { + let pixels: Vec = (0..64 * 64) + .map(|index| (((index * 19) ^ (index / 11)) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 64, 64, 1, 8, false).unwrap(); + + let encoded = encode_j2k_lossy( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_marker_segments(vec![J2kMarkerSegment::Sop, J2kMarkerSegment::Eph]), + ) + .expect("classic lossy SOP/EPH encode"); + + assert!( + encoded + .codestream + .windows(2) + .any(|marker| marker == [0xFF, 0x91]), + "SOP marker must be emitted" + ); + assert!( + encoded + .codestream + .windows(2) + .any(|marker| marker == [0xFF, 0x92]), + "EPH marker must be emitted" + ); + let cod = encoded + .codestream + .windows(2) + .position(|window| window == [0xFF, 0x52]) + .expect("COD marker"); + assert_eq!(encoded.codestream[cod + 4] & 0x06, 0x06); + + let decoded = strict_decode_native(&encoded.codestream); + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.num_components, 1); +} + +#[test] +fn cpu_classic_lossy_multi_tile_codestream_decodes() { + let pixels: Vec = (0..96 * 80) + .map(|index| (((index * 23) + (index / 13)) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 96, 80, 1, 8, false).unwrap(); + let mut options = J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_rate_target(Some(J2kRateTarget::BitsPerPixel(1.5))) + .with_marker_segments(vec![J2kMarkerSegment::Tlm]); + options.tile_size = Some((48, 40)); + + let encoded = encode_j2k_lossy(samples, &options).expect("classic lossy multi-tile encode"); + + let sot_count = encoded + .codestream + .windows(2) + .filter(|marker| *marker == [0xFF, 0x90]) + .count(); + assert_eq!(sot_count, 4); + let tlm_count = encoded + .codestream + .windows(2) + .filter(|marker| *marker == [0xFF, 0x55]) + .count(); + assert_eq!(tlm_count, 4); + + let decoded = decode_native(&encoded.codestream); + assert_eq!(decoded.width, 96); + assert_eq!(decoded.height, 80); + assert_eq!(decoded.num_components, 1); +} + +#[test] +fn cpu_classic_lossy_multi_tile_emits_plt_and_plm() { + let pixels: Vec = (0..96 * 80) + .map(|index| (((index * 41) + (index / 23)) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 96, 80, 1, 8, false).unwrap(); + let mut options = J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_rate_target(Some(J2kRateTarget::BitsPerPixel(1.5))) + .with_marker_segments(vec![J2kMarkerSegment::Plt, J2kMarkerSegment::Plm]); + options.tile_size = Some((48, 40)); + + let encoded = + encode_j2k_lossy(samples, &options).expect("classic lossy multi-tile PLT/PLM encode"); + + let plt_count = encoded + .codestream + .windows(2) + .filter(|marker| *marker == [0xFF, 0x58]) + .count(); + assert_eq!(plt_count, 4); + assert!( + encoded + .codestream + .windows(2) + .any(|marker| marker == [0xFF, 0x57]), + "PLM marker must be emitted" + ); + + let decoded = strict_decode_native(&encoded.codestream); + assert_eq!(decoded.width, 96); + assert_eq!(decoded.height, 80); + assert_eq!(decoded.num_components, 1); +} + +#[test] +fn cpu_classic_lossy_writes_explicit_single_precinct_exponents() { + let pixels: Vec = (0..64 * 64) + .map(|index| (((index * 31) + (index / 17)) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 64, 64, 1, 8, false).unwrap(); + let mut options = J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_max_decomposition_levels(Some(1)); + options.precinct_exponents = vec![(15, 15), (15, 15)]; + + let encoded = encode_j2k_lossy(samples, &options).expect("classic lossy precinct encode"); + + let cod = encoded + .codestream + .windows(2) + .position(|window| window == [0xFF, 0x52]) + .expect("COD marker"); + assert_eq!(encoded.codestream[cod + 4] & 0x01, 0x01); + assert_eq!( + u16::from_be_bytes([encoded.codestream[cod + 2], encoded.codestream[cod + 3]]), + 14 + ); + assert_eq!(&encoded.codestream[cod + 14..cod + 16], &[0xFF, 0xFF]); + + let decoded = strict_decode_native(&encoded.codestream); + assert_eq!(decoded.width, 64); + assert_eq!(decoded.height, 64); + assert_eq!(decoded.num_components, 1); +} + +#[test] +fn cpu_classic_lossy_splits_packets_by_precinct() { + let pixels: Vec = (0..128 * 128) + .map(|index| (((index * 37) + (index / 19)) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 128, 128, 1, 8, false).unwrap(); + let mut options = J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_max_decomposition_levels(Some(0)) + .with_marker_segments(vec![J2kMarkerSegment::Plt]); + options.precinct_exponents = vec![(6, 6)]; + + let encoded = encode_j2k_lossy(samples, &options).expect("classic lossy precinct encode"); + + let cod = encoded + .codestream + .windows(2) + .position(|window| window == [0xFF, 0x52]) + .expect("COD marker"); + assert_eq!(encoded.codestream[cod + 4] & 0x01, 0x01); + assert_eq!( + u16::from_be_bytes([encoded.codestream[cod + 2], encoded.codestream[cod + 3]]), + 13 + ); + assert_eq!(encoded.codestream[cod + 14], 0x66); + assert_eq!(plt_packet_length_count(&encoded.codestream), 4); + + let decoded = strict_decode_native(&encoded.codestream); + assert_eq!(decoded.width, 128); + assert_eq!(decoded.height, 128); + assert_eq!(decoded.num_components, 1); +} + +#[test] +fn cpu_htj2k_lossy_reports_rate_granularity() { + let pixels: Vec = (0..32 * 32).map(|index| (index & 0xFF) as u8).collect(); + let samples = J2kLossySamples::new(&pixels, 32, 32, 1, 8, false).unwrap(); + + let encoded = encode_j2k_lossy( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_validation(J2kEncodeValidation::CpuRoundTrip), + ) + .expect("HTJ2K lossy encode"); + + assert_eq!( + encoded.report.ht_rate_granularity_bytes, + Some(encoded.codestream.len() as u64) + ); + assert_eq!(decode_native(&encoded.codestream).num_components, 1); +} + +#[test] +fn cpu_htj2k_lossy_multiple_quality_layers_use_segment_granularity() { + let pixels: Vec = (0..64 * 64) + .map(|index| (((index * 43) ^ (index / 29)) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 64, 64, 1, 8, false).unwrap(); + + let encoded = encode_j2k_lossy( + samples, + &J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::CpuOnly) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_quality_layers(vec![ + J2kQualityLayer::new(J2kRateTarget::BitsPerPixel(0.75)), + J2kQualityLayer::new(J2kRateTarget::BitsPerPixel(1.5)), + ]) + .with_validation(J2kEncodeValidation::CpuRoundTrip), + ) + .expect("HTJ2K multilayer lossy encode"); + + assert_eq!(encoded.report.quality_layers, 2); + assert!(encoded.report.ht_rate_granularity_bytes.is_some()); + let cod = encoded + .codestream + .windows(2) + .position(|window| window == [0xFF, 0x52]) + .expect("COD marker"); + assert_eq!( + u16::from_be_bytes([encoded.codestream[cod + 6], encoded.codestream[cod + 7]]), + 2 + ); + assert_eq!(decode_native(&encoded.codestream).num_components, 1); +} + +#[test] +fn accelerator_facade_htj2k_lossy_multilayer_require_device_checks_supported_stages() { + #[derive(Default)] + struct FullHtj2kLossyAccelerator { + deinterleave: usize, + quantize_subband: usize, + ht_code_block: usize, + packetization: usize, + } + + impl J2kEncodeStageAccelerator for FullHtj2kLossyAccelerator { + fn dispatch_report(&self) -> J2kEncodeDispatchReport { + J2kEncodeDispatchReport { + deinterleave: self.deinterleave, + quantize_subband: self.quantize_subband, + ht_code_block: self.ht_code_block, + packetization: self.packetization, + ..J2kEncodeDispatchReport::default() + } + } + + fn encode_deinterleave( + &mut self, + job: J2kDeinterleaveToF32Job<'_>, + ) -> core::result::Result>>, &'static str> { + self.deinterleave = self.deinterleave.saturating_add(1); + assert_eq!(job.bit_depth, 8); + assert!(!job.signed); + let mut component = Vec::with_capacity(job.num_pixels); + for &sample in job.pixels { + component.push(f32::from(sample) - 128.0); + } + Ok(Some(vec![component])) + } + + fn encode_quantize_subband( + &mut self, + job: J2kQuantizeSubbandJob<'_>, + ) -> core::result::Result>, &'static str> { + self.quantize_subband = self.quantize_subband.saturating_add(1); + Ok(Some( + job.coefficients + .iter() + .map(|sample| sample.round() as i32) + .collect(), + )) + } + + fn encode_ht_code_block( + &mut self, + job: J2kHtCodeBlockEncodeJob<'_>, + ) -> core::result::Result, &'static str> { + self.ht_code_block = self.ht_code_block.saturating_add(1); + j2k_native::encode_ht_code_block_scalar( + job.coefficients, + job.width, + job.height, + job.total_bitplanes, + ) + .map(public_encoded_ht) + .map(Some) + } + + fn encode_packetization( + &mut self, + _job: J2kPacketizationEncodeJob<'_>, + ) -> core::result::Result>, &'static str> { + self.packetization = self.packetization.saturating_add(1); + Ok(None) + } + } + + let pixels: Vec = (0..32 * 32) + .map(|index| (((index * 47) + (index / 31)) & 0xFF) as u8) + .collect(); + let samples = J2kLossySamples::new(&pixels, 32, 32, 1, 8, false).unwrap(); + let mut options = J2kLossyEncodeOptions::default() + .with_backend(EncodeBackendPreference::RequireDevice) + .with_block_coding_mode(J2kBlockCodingMode::HighThroughput) + .with_max_decomposition_levels(Some(0)) + .with_quality_layers(vec![ + J2kQualityLayer::new(J2kRateTarget::Bytes(100_000)), + J2kQualityLayer::new(J2kRateTarget::Bytes(100_000)), + ]) + .with_validation(J2kEncodeValidation::External); + options.psnr_iteration_budget = 1; + let mut accelerator = FullHtj2kLossyAccelerator::default(); + + let encoded = + encode_j2k_lossy_with_accelerator(samples, &options, BackendKind::Cuda, &mut accelerator) + .expect("HTJ2K lossy multilayer required stages should dispatch"); + + assert_eq!(encoded.backend, BackendKind::Cuda); + assert!(accelerator.deinterleave > 0); + assert!(accelerator.quantize_subband > 0); + assert!(accelerator.ht_code_block > 0); + assert_eq!(decode_native(&encoded.codestream).num_components, 1); +} diff --git a/crates/signinum-j2k/tests/grok_parity.rs b/crates/j2k/tests/grok_parity.rs similarity index 70% rename from crates/signinum-j2k/tests/grok_parity.rs rename to crates/j2k/tests/grok_parity.rs index d3b3fc07..aeff38bc 100644 --- a/crates/signinum-j2k/tests/grok_parity.rs +++ b/crates/j2k/tests/grok_parity.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_core::{Downscale, PixelFormat, Rect}; -use signinum_j2k::J2kDecoder; -use signinum_j2k_native::{encode_htj2k, EncodeOptions}; -use signinum_test_support::gradient_u8; +use j2k::J2kDecoder; +use j2k_core::{Downscale, PixelFormat, Rect}; +use j2k_native::{encode_htj2k, EncodeOptions}; +use j2k_test_support::{gradient_u8, read_pnm_pixels, wrap_codestream_jp2, write_pnm}; use std::{ fs, path::{Path, PathBuf}, @@ -26,7 +26,7 @@ fn classic_gray_full_decode_matches_grok() { let mut out = vec![0_u8; 128 * 128]; decoder .decode_into(&mut out, 128, PixelFormat::Gray8) - .expect("signinum decode"); + .expect("j2k decode"); let expected = decode_with_grok(&path, "grok_full_gray", &jp2, ".pgm", &[]); assert_eq!(out, expected); @@ -44,7 +44,7 @@ fn classic_rgb_full_decode_matches_grok() { let mut out = vec![0_u8; 128 * 128 * 3]; decoder .decode_into(&mut out, 128 * 3, PixelFormat::Rgb8) - .expect("signinum decode"); + .expect("j2k decode"); let expected = decode_with_grok(&path, "grok_full_rgb", &jp2, ".ppm", &[]); assert_eq!(out, expected); @@ -68,13 +68,13 @@ fn classic_gray_region_decode_matches_grok_area_decode() { let mut out = vec![0_u8; roi.w as usize * roi.h as usize]; decoder .decode_region_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, roi.w as usize, PixelFormat::Gray8, roi, ) - .expect("signinum region decode"); + .expect("j2k region decode"); let expected = decode_with_grok( &path, @@ -107,13 +107,13 @@ fn classic_rgb_region_decode_matches_grok_area_decode() { let mut out = vec![0_u8; roi.w as usize * roi.h as usize * 3]; decoder .decode_region_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, roi.w as usize * 3, PixelFormat::Rgb8, roi, ) - .expect("signinum region decode"); + .expect("j2k region decode"); let expected = decode_with_grok( &path, @@ -140,13 +140,13 @@ fn classic_gray_scaled_decode_matches_grok_reduce() { let mut out = vec![0_u8; 32 * 32]; decoder .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, 32, PixelFormat::Gray8, Downscale::Quarter, ) - .expect("signinum scaled decode"); + .expect("j2k scaled decode"); let expected = decode_with_grok(&path, "grok_scaled_gray", &jp2, ".pgm", &["-r", "2"]); assert_eq!(out, expected); @@ -164,13 +164,13 @@ fn classic_rgb_scaled_decode_matches_grok_reduce() { let mut out = vec![0_u8; 32 * 32 * 3]; decoder .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, 32 * 3, PixelFormat::Rgb8, Downscale::Quarter, ) - .expect("signinum scaled decode"); + .expect("j2k scaled decode"); let expected = decode_with_grok(&path, "grok_scaled_rgb", &jp2, ".ppm", &["-r", "2"]); assert_eq!(out, expected); @@ -188,7 +188,7 @@ fn ht_gray_full_decode_matches_grok() { let mut out = vec![0_u8; 128 * 128]; decoder .decode_into(&mut out, 128, PixelFormat::Gray8) - .expect("signinum decode"); + .expect("j2k decode"); let expected = decode_with_grok(&path, "grok_full_ht_gray", &jp2, ".pgm", &[]); assert_eq!(out, expected); @@ -212,13 +212,13 @@ fn ht_gray_region_decode_matches_grok_area_decode() { let mut out = vec![0_u8; roi.w as usize * roi.h as usize]; decoder .decode_region_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, roi.w as usize, PixelFormat::Gray8, roi, ) - .expect("signinum region decode"); + .expect("j2k region decode"); let expected = decode_with_grok( &path, @@ -245,13 +245,13 @@ fn ht_gray_scaled_decode_matches_grok_reduce() { let mut out = vec![0_u8; 32 * 32]; decoder .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, 32, PixelFormat::Gray8, Downscale::Quarter, ) - .expect("signinum scaled decode"); + .expect("j2k scaled decode"); let expected = decode_with_grok(&path, "grok_scaled_ht_gray", &jp2, ".pgm", &["-r", "2"]); assert_eq!(out, expected); @@ -269,7 +269,7 @@ fn ht_rgb_full_decode_matches_grok() { let mut out = vec![0_u8; 128 * 128 * 3]; decoder .decode_into(&mut out, 128 * 3, PixelFormat::Rgb8) - .expect("signinum decode"); + .expect("j2k decode"); let expected = decode_with_grok(&path, "grok_full_ht_rgb", &jp2, ".ppm", &[]); assert_eq!(out, expected); @@ -293,13 +293,13 @@ fn ht_rgb_region_decode_matches_grok_area_decode() { let mut out = vec![0_u8; roi.w as usize * roi.h as usize * 3]; decoder .decode_region_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, roi.w as usize * 3, PixelFormat::Rgb8, roi, ) - .expect("signinum region decode"); + .expect("j2k region decode"); let expected = decode_with_grok( &path, @@ -326,13 +326,13 @@ fn ht_rgb_scaled_decode_matches_grok_reduce() { let mut out = vec![0_u8; 32 * 32 * 3]; decoder .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, 32 * 3, PixelFormat::Rgb8, Downscale::Quarter, ) - .expect("signinum scaled decode"); + .expect("j2k scaled decode"); let expected = decode_with_grok(&path, "grok_scaled_ht_rgb", &jp2, ".ppm", &["-r", "2"]); assert_eq!(out, expected); @@ -348,7 +348,7 @@ fn classic_jp2(pixels: &[u8], width: u32, height: u32, components: u8) -> Option format!("grok_classic_input_{id}.ppm") }); let out_path = dir.join(format!("grok_classic_output_{id}.jp2")); - write_pnm(&src_path, pixels, width, height, components).ok()?; + write_pnm(&src_path, pixels, width, height, usize::from(components)).ok()?; let status = Command::new(bin) .arg("-i") .arg(&src_path) @@ -361,7 +361,7 @@ fn classic_jp2(pixels: &[u8], width: u32, height: u32, components: u8) -> Option if !status.success() { assert!( !require_grok(), - "SIGNINUM_REQUIRE_GROK is set but grk_compress failed" + "J2K_REQUIRE_GROK is set but grk_compress failed" ); return None; } @@ -384,45 +384,12 @@ fn ht_jp2(pixels: &[u8], width: u32, height: u32, components: u8) -> Vec { wrap_codestream_jp2(&codestream, width, height, u16::from(components), 8, 17) } -fn wrap_codestream_jp2( - codestream: &[u8], - width: u32, - height: u32, - components: u16, - bit_depth: u8, - colorspace_enum: u32, -) -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0D, 0x0A, 0x87, 0x0A]); - bytes.extend_from_slice(&[ - 0, 0, 0, 20, b'f', b't', b'y', b'p', b'j', b'p', b'2', b' ', 0, 0, 0, 0, b'j', b'p', b'2', - b' ', - ]); - - let bpc = bit_depth.saturating_sub(1); - bytes.extend_from_slice(&[ - 0, 0, 0, 45, b'j', b'p', b'2', b'h', 0, 0, 0, 22, b'i', b'h', b'd', b'r', - ]); - bytes.extend_from_slice(&height.to_be_bytes()); - bytes.extend_from_slice(&width.to_be_bytes()); - bytes.extend_from_slice(&components.to_be_bytes()); - bytes.extend_from_slice(&[bpc, 7, 0, 0]); - bytes.extend_from_slice(&[0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0]); - bytes.extend_from_slice(&colorspace_enum.to_be_bytes()); - - let len = (8 + codestream.len()) as u32; - bytes.extend_from_slice(&len.to_be_bytes()); - bytes.extend_from_slice(b"jp2c"); - bytes.extend_from_slice(codestream); - bytes -} - fn grok_decompress_bin() -> Option { static GROK: OnceLock> = OnceLock::new(); let path = GROK.get_or_init(discover_grok_decompress_bin).clone(); assert!( path.is_some() || !require_grok(), - "SIGNINUM_REQUIRE_GROK is set but grk_decompress was not found" + "J2K_REQUIRE_GROK is set but grk_decompress was not found" ); path } @@ -432,17 +399,17 @@ fn grok_compress_bin() -> Option { let path = GROK.get_or_init(discover_grok_compress_bin).clone(); assert!( path.is_some() || !require_grok(), - "SIGNINUM_REQUIRE_GROK is set but grk_compress was not found" + "J2K_REQUIRE_GROK is set but grk_compress was not found" ); path } fn require_grok() -> bool { - std::env::var_os("SIGNINUM_REQUIRE_GROK").is_some() + std::env::var_os("J2K_REQUIRE_GROK").is_some() } fn discover_grok_decompress_bin() -> Option { - std::env::var_os("SIGNINUM_GROK_BIN") + std::env::var_os("J2K_GROK_BIN") .map(PathBuf::from) .or_else(|| Some(PathBuf::from("/opt/homebrew/bin/grk_decompress"))) .or_else(|| Some(PathBuf::from("/usr/local/bin/grk_decompress"))) @@ -450,13 +417,13 @@ fn discover_grok_decompress_bin() -> Option { } fn discover_grok_compress_bin() -> Option { - if let Some(path) = std::env::var_os("SIGNINUM_GROK_COMPRESS_BIN") + if let Some(path) = std::env::var_os("J2K_GROK_COMPRESS_BIN") .map(PathBuf::from) .filter(|path| path.exists()) { return Some(path); } - if let Some(path) = std::env::var_os("SIGNINUM_GROK_BIN") + if let Some(path) = std::env::var_os("J2K_GROK_BIN") .map(PathBuf::from) .filter(|path| path.exists()) { @@ -491,89 +458,13 @@ fn decode_with_grok( command.args(extra_args); let status = command.status().expect("run grk_decompress"); assert!(status.success(), "grk_decompress failed"); - read_pnm_pixels(&output_path) -} - -fn write_pnm( - path: &Path, - pixels: &[u8], - width: u32, - height: u32, - channels: u8, -) -> std::io::Result<()> { - let mut bytes = Vec::new(); - if channels == 1 { - bytes.extend_from_slice(format!("P5\n{width} {height}\n255\n").as_bytes()); - } else { - bytes.extend_from_slice(format!("P6\n{width} {height}\n255\n").as_bytes()); - } - bytes.extend_from_slice(pixels); - fs::write(path, bytes) -} - -fn read_pnm_pixels(path: &Path) -> Vec { - let bytes = fs::read(path).expect("read pnm"); - let mut cursor = 0; - - let magic = read_pnm_token(&bytes, &mut cursor).expect("pnm magic"); - assert!(magic == b"P5" || magic == b"P6", "unexpected pnm magic"); - - let _width = read_pnm_token(&bytes, &mut cursor).expect("pnm width"); - let _height = read_pnm_token(&bytes, &mut cursor).expect("pnm height"); - let _maxval = read_pnm_token(&bytes, &mut cursor).expect("pnm maxval"); - - while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() { - cursor += 1; - } - - bytes[cursor..].to_vec() -} - -fn read_pnm_token<'a>(bytes: &'a [u8], cursor: &mut usize) -> Option<&'a [u8]> { - skip_pnm_separators(bytes, cursor); - if *cursor >= bytes.len() { - return None; - } - - let start = *cursor; - while *cursor < bytes.len() { - let byte = bytes[*cursor]; - if byte.is_ascii_whitespace() || byte == b'#' { - break; - } - *cursor += 1; - } - - if start == *cursor { - None - } else { - Some(&bytes[start..*cursor]) - } -} - -fn skip_pnm_separators(bytes: &[u8], cursor: &mut usize) { - while *cursor < bytes.len() { - let byte = bytes[*cursor]; - if byte.is_ascii_whitespace() { - *cursor += 1; - continue; - } - if byte == b'#' { - *cursor += 1; - while *cursor < bytes.len() && bytes[*cursor] != b'\n' { - *cursor += 1; - } - continue; - } - break; - } + read_pnm_pixels(&output_path).expect("read pnm") } fn temp_dir() -> &'static Path { static DIR: OnceLock = OnceLock::new(); DIR.get_or_init(|| { - let dir = - std::env::temp_dir().join(format!("signinum-j2k-grok-parity-{}", std::process::id())); + let dir = std::env::temp_dir().join(format!("j2k-grok-parity-{}", std::process::id())); fs::create_dir_all(&dir).expect("create temp dir"); dir }) diff --git a/crates/signinum-j2k/tests/inspect.rs b/crates/j2k/tests/inspect.rs similarity index 60% rename from crates/signinum-j2k/tests/inspect.rs rename to crates/j2k/tests/inspect.rs index 30e1c90b..02ed0161 100644 --- a/crates/signinum-j2k/tests/inspect.rs +++ b/crates/j2k/tests/inspect.rs @@ -1,39 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_core::{ - Colorspace, CompressedPayloadKind, CompressedTransferSyntax, PassthroughDecision, +use j2k::{J2kDecoder, J2kError, J2kView}; +use j2k_core::{ + Colorspace, CompressedPayloadKind, CompressedTransferSyntax, InputError, PassthroughDecision, PassthroughRequirements, }; -use signinum_j2k::{J2kDecoder, J2kError, J2kView}; -use signinum_j2k_native::{encode, encode_htj2k, EncodeOptions}; - -fn minimal_codestream() -> Vec { - let mut bytes = vec![0xFF, 0x4F]; - let mut siz = Vec::new(); - push_u16(&mut siz, 0); - push_u32(&mut siz, 128); - push_u32(&mut siz, 64); - push_u32(&mut siz, 0); - push_u32(&mut siz, 0); - push_u32(&mut siz, 64); - push_u32(&mut siz, 64); - push_u32(&mut siz, 0); - push_u32(&mut siz, 0); - push_u16(&mut siz, 3); - for _ in 0..3 { - siz.extend_from_slice(&[0x07, 0x01, 0x01]); - } - bytes.extend_from_slice(&[0xFF, 0x51]); - push_u16(&mut bytes, (siz.len() + 2) as u16); - bytes.extend_from_slice(&siz); - - let cod = [0x00, 0x00, 0x00, 0x01, 0x01, 0x05, 0x04, 0x04, 0x00, 0x01]; - bytes.extend_from_slice(&[0xFF, 0x52]); - push_u16(&mut bytes, (cod.len() + 2) as u16); - bytes.extend_from_slice(&cod); - bytes.extend_from_slice(&[0xFF, 0x90, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); - bytes -} +use j2k_native::{encode, encode_htj2k, EncodeOptions}; +use j2k_test_support::{minimal_j2k_codestream, minimal_jp2, wrap_jp2_codestream}; fn codestream_without_siz() -> Vec { let mut bytes = vec![0xFF, 0x4F]; @@ -68,13 +41,34 @@ fn codestream_without_cod() -> Vec { } fn codestream_truncated_after_main_header() -> Vec { - let mut bytes = minimal_codestream(); + let mut bytes = minimal_j2k_codestream(); bytes.truncate(bytes.len() - 10); bytes } -fn minimal_jp2() -> Vec { - let codestream = minimal_codestream(); +fn codestream_with_component_count(component_count: u16) -> Vec { + let mut bytes = minimal_j2k_codestream(); + let siz = bytes + .windows(2) + .position(|marker| marker == [0xFF, 0x51]) + .expect("SIZ marker"); + let lsiz = u16::from_be_bytes([bytes[siz + 2], bytes[siz + 3]]) as usize; + let component_start = siz + 40; + let component_end = siz + 2 + lsiz; + let first_component = bytes[component_start..component_start + 3].to_vec(); + let new_lsiz = 38 + usize::from(component_count) * 3; + + bytes[siz + 2..siz + 4].copy_from_slice(&(new_lsiz as u16).to_be_bytes()); + bytes[siz + 38..siz + 40].copy_from_slice(&component_count.to_be_bytes()); + bytes.splice( + component_start..component_end, + first_component.repeat(usize::from(component_count)), + ); + bytes +} + +fn jp2_with_truncated_jp2h_child_box() -> Vec { + let codestream = minimal_j2k_codestream(); let mut bytes = Vec::new(); bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0D, 0x0A, 0x87, 0x0A]); bytes.extend_from_slice(&[ @@ -82,8 +76,7 @@ fn minimal_jp2() -> Vec { b' ', ]); bytes.extend_from_slice(&[ - 0, 0, 0, 45, b'j', b'p', b'2', b'h', 0, 0, 0, 22, b'i', b'h', b'd', b'r', 0, 0, 0, 64, 0, - 0, 0, 128, 0, 3, 7, 7, 0, 0, 0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0, 0, 0, 0, 16, + 0, 0, 0, 16, b'j', b'p', b'2', b'h', 0, 0, 0, 32, b'i', b'h', b'd', b'r', ]); let len = (8 + codestream.len()) as u32; bytes.extend_from_slice(&len.to_be_bytes()); @@ -108,45 +101,15 @@ fn classic_lossless_codestream() -> Vec { } fn ht_jp2() -> Vec { - let codestream = ht_codestream(); - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0D, 0x0A, 0x87, 0x0A]); - bytes.extend_from_slice(&[ - 0, 0, 0, 20, b'f', b't', b'y', b'p', b'j', b'p', b'2', b' ', 0, 0, 0, 0, b'j', b'p', b'2', - b' ', - ]); - bytes.extend_from_slice(&[ - 0, 0, 0, 45, b'j', b'p', b'2', b'h', 0, 0, 0, 22, b'i', b'h', b'd', b'r', 0, 0, 0, 2, 0, 0, - 0, 2, 0, 1, 7, 7, 0, 0, 0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0, 0, 0, 0, 17, - ]); - let len = (8 + codestream.len()) as u32; - bytes.extend_from_slice(&len.to_be_bytes()); - bytes.extend_from_slice(b"jp2c"); - bytes.extend_from_slice(&codestream); - bytes + wrap_jp2_codestream(&ht_codestream(), 2, 2, 1, 8, 17) } fn classic_lossless_jp2() -> Vec { - let codestream = classic_lossless_codestream(); - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0D, 0x0A, 0x87, 0x0A]); - bytes.extend_from_slice(&[ - 0, 0, 0, 20, b'f', b't', b'y', b'p', b'j', b'p', b'2', b' ', 0, 0, 0, 0, b'j', b'p', b'2', - b' ', - ]); - bytes.extend_from_slice(&[ - 0, 0, 0, 45, b'j', b'p', b'2', b'h', 0, 0, 0, 22, b'i', b'h', b'd', b'r', 0, 0, 0, 2, 0, 0, - 0, 2, 0, 1, 7, 7, 0, 0, 0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0, 0, 0, 0, 17, - ]); - let len = (8 + codestream.len()) as u32; - bytes.extend_from_slice(&len.to_be_bytes()); - bytes.extend_from_slice(b"jp2c"); - bytes.extend_from_slice(&codestream); - bytes + wrap_jp2_codestream(&classic_lossless_codestream(), 2, 2, 1, 8, 17) } fn jp2_with_jp2c_before_jp2h() -> Vec { - let codestream = minimal_codestream(); + let codestream = minimal_j2k_codestream(); let mut bytes = Vec::new(); bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0D, 0x0A, 0x87, 0x0A]); bytes.extend_from_slice(&[ @@ -172,9 +135,28 @@ fn push_u32(out: &mut Vec, value: u32) { out.extend_from_slice(&value.to_be_bytes()); } +fn rewrite_siz_u32(bytes: &mut [u8], payload_offset: usize, value: u32) { + let siz = bytes + .windows(2) + .position(|marker| marker == [0xFF, 0x51]) + .expect("SIZ marker"); + let offset = siz + 4 + payload_offset; + bytes[offset..offset + 4].copy_from_slice(&value.to_be_bytes()); +} + +fn rewrite_component_sampling(bytes: &mut [u8], component: usize, x_rsiz: u8, y_rsiz: u8) { + let siz = bytes + .windows(2) + .position(|marker| marker == [0xFF, 0x51]) + .expect("SIZ marker"); + let component_offset = siz + 40 + component * 3; + bytes[component_offset + 1] = x_rsiz; + bytes[component_offset + 2] = y_rsiz; +} + #[test] fn inspect_raw_codestream_reports_core_info() { - let info = J2kDecoder::inspect(&minimal_codestream()).expect("codestream inspect"); + let info = J2kDecoder::inspect(&minimal_j2k_codestream()).expect("codestream inspect"); assert_eq!(info.dimensions, (128, 64)); assert_eq!(info.components, 3); assert_eq!(info.bit_depth, 8); @@ -187,6 +169,73 @@ fn inspect_raw_codestream_reports_core_info() { assert_eq!(tiles.tiles_y, 1); } +#[test] +fn inspect_raw_codestream_rejects_zero_component_sampling() { + let mut bytes = minimal_j2k_codestream(); + rewrite_component_sampling(&mut bytes, 0, 0, 1); + + let err = J2kDecoder::inspect(&bytes).expect_err("zero sampling must reject"); + + assert!(matches!(err, J2kError::InvalidSiz { .. })); +} + +#[test] +fn inspect_raw_codestream_rejects_oversized_dimensions() { + let mut bytes = minimal_j2k_codestream(); + rewrite_siz_u32(&mut bytes, 2, 60_001); + + let err = J2kDecoder::inspect(&bytes).expect_err("oversized width must reject"); + + assert!(matches!(err, J2kError::InvalidSiz { .. })); +} + +#[test] +fn inspect_raw_codestream_rejects_bad_tile_origin() { + let mut bytes = minimal_j2k_codestream(); + rewrite_siz_u32(&mut bytes, 26, 1); + + let err = J2kDecoder::inspect(&bytes).expect_err("bad tile origin must reject"); + + assert!(matches!(err, J2kError::InvalidSiz { .. })); +} + +#[test] +fn inspect_raw_codestream_rejects_tile_extent_overflow() { + let mut bytes = minimal_j2k_codestream(); + rewrite_siz_u32(&mut bytes, 2, u32::MAX); + rewrite_siz_u32(&mut bytes, 10, u32::MAX - 1); + rewrite_siz_u32(&mut bytes, 18, 10); + rewrite_siz_u32(&mut bytes, 26, u32::MAX - 2); + + let err = J2kDecoder::inspect(&bytes).expect_err("overflow must reject"); + + assert!(matches!(err, J2kError::InvalidSiz { .. })); +} + +#[test] +fn inspect_raw_codestream_rejects_excessive_tile_count() { + let mut bytes = minimal_j2k_codestream(); + rewrite_siz_u32(&mut bytes, 2, 257); + rewrite_siz_u32(&mut bytes, 6, 257); + rewrite_siz_u32(&mut bytes, 18, 1); + rewrite_siz_u32(&mut bytes, 22, 1); + + let err = J2kDecoder::inspect(&bytes).expect_err("tile count must reject"); + + assert!(matches!(err, J2kError::InvalidSiz { .. })); +} + +#[test] +fn inspect_spec_valid_component_count_above_u8_is_unsupported() { + let bytes = codestream_with_component_count(256); + let err = J2kDecoder::inspect(&bytes).expect_err("component count unsupported"); + + let J2kError::Unsupported(unsupported) = err else { + panic!("expected unsupported component count, got {err:?}"); + }; + assert_eq!(unsupported.what, "component count > 255"); +} + #[test] fn inspect_jp2_uses_container_colorspace() { let info = J2kDecoder::inspect(&minimal_jp2()).expect("jp2 inspect"); @@ -234,7 +283,21 @@ fn codestream_truncated_after_main_header_is_rejected() { let err = J2kDecoder::inspect(&codestream_truncated_after_main_header()).unwrap_err(); assert!(matches!( err, - J2kError::Input(signinum_core::InputError::TruncatedAt { .. }) + J2kError::Input(j2k_core::InputError::TruncatedAt { .. }) + )); +} + +#[test] +fn jp2_with_truncated_nested_header_box_returns_error() { + let err = J2kDecoder::inspect(&jp2_with_truncated_jp2h_child_box()) + .expect_err("truncated nested jp2h child box"); + + assert!(matches!( + err, + J2kError::Input(InputError::TruncatedAt { + segment: "box payload", + .. + }) )); } @@ -318,11 +381,9 @@ fn jp2_file_is_not_eligible_for_raw_dicom_codestream_copy() { view.passthrough_candidate() .expect("jp2 passthrough candidate") .copy_bytes_if_eligible(&requirements), - Err( - signinum_core::PassthroughRejectReason::PayloadKindMismatch { - source: CompressedPayloadKind::Jp2File, - destination: CompressedPayloadKind::Jpeg2000Codestream, - } - ) + Err(j2k_core::PassthroughRejectReason::PayloadKindMismatch { + source: CompressedPayloadKind::Jp2File, + destination: CompressedPayloadKind::Jpeg2000Codestream, + }) ); } diff --git a/crates/j2k/tests/iso_conformance.rs b/crates/j2k/tests/iso_conformance.rs new file mode 100644 index 00000000..5a223acb --- /dev/null +++ b/crates/j2k/tests/iso_conformance.rs @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + env, fs, + path::{Component, Path, PathBuf}, +}; + +use j2k_native::{DecodeSettings, Image}; + +const CONFORMANCE_ENV: &str = "J2K_ISO_CONFORMANCE_DIR"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Classification { + Blocking, + KnownLimitation, + Investigate, +} + +#[derive(Debug)] +struct Vector { + id: String, + path: PathBuf, + classification: Classification, + features: String, + reason: String, +} + +fn repo_root() -> &'static Path { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("workspace root") +} + +fn manifest_path() -> PathBuf { + repo_root().join("corpus/j2k-conformance/manifest.tsv") +} + +fn load_manifest() -> Vec { + let text = fs::read_to_string(manifest_path()).expect("read J2K conformance manifest"); + text.lines() + .enumerate() + .filter_map(|(line_idx, line)| { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + Some(parse_manifest_line(line_idx + 1, line)) + }) + .collect() +} + +fn parse_manifest_line(line_number: usize, line: &str) -> Vector { + let fields: Vec<_> = line.split('\t').collect(); + assert_eq!( + fields.len(), + 5, + "manifest line {line_number} must contain id, path, classification, features, reason" + ); + let path = PathBuf::from(fields[1]); + assert!( + !path.is_absolute() + && !path + .components() + .any(|component| matches!(component, Component::ParentDir)), + "manifest line {line_number} path must stay relative to the ISO vector root" + ); + Vector { + id: fields[0].to_string(), + path, + classification: parse_classification(line_number, fields[2]), + features: fields[3].to_string(), + reason: fields[4].to_string(), + } +} + +fn parse_classification(line_number: usize, value: &str) -> Classification { + match value { + "blocking" => Classification::Blocking, + "known-limitation" => Classification::KnownLimitation, + "investigate" => Classification::Investigate, + _ => panic!("manifest line {line_number} has invalid classification {value:?}"), + } +} + +#[test] +fn iso_conformance_manifest_is_release_classified() { + let vectors = load_manifest(); + assert!( + vectors + .iter() + .any(|vector| vector.classification == Classification::Blocking), + "manifest must contain at least one blocking vector" + ); + for vector in vectors { + assert!(!vector.id.is_empty(), "vector id must not be empty"); + assert!( + !vector.features.is_empty(), + "{} must list exercised features", + vector.id + ); + if vector.classification == Classification::KnownLimitation { + assert!( + !vector.reason.is_empty(), + "{} known limitation must document the deferred feature", + vector.id + ); + } + assert_ne!( + vector.classification, + Classification::Investigate, + "{} must be classified before release signoff", + vector.id + ); + } +} + +#[test] +fn iso_conformance_manifest_blocks_release_shipped_features() { + let vectors = load_manifest(); + for required_feature in [ + "part1-core", + "part15-core", + "poc", + "precincts", + "progression-orders", + "tlm", + "plt", + "sop", + "eph", + ] { + assert!( + vectors.iter().any(|vector| { + vector.classification == Classification::Blocking + && vector + .features + .split(';') + .any(|feature| feature == required_feature) + }), + "manifest must include a blocking vector for shipped feature {required_feature}" + ); + } + assert!( + vectors.iter().any(|vector| { + vector.features.split(';').any(|feature| feature == "plm") + && (vector.classification == Classification::Blocking + || (vector.classification == Classification::KnownLimitation + && vector + .features + .split(';') + .any(|feature| feature == "conformance-coverage-gap"))) + }), + "manifest must include a blocking PLM vector or document the ISO coverage gap" + ); +} + +#[test] +fn iso_conformance_flags_missing_blocking_vectors_as_signoff_failures() { + let vector_root = env::temp_dir().join(format!("j2k-missing-blocking-{}", std::process::id())); + fs::create_dir_all(&vector_root).expect("create temporary vector root"); + let vectors = vec![Vector { + id: "missing_blocking".to_string(), + path: PathBuf::from("part1/missing.j2k"), + classification: Classification::Blocking, + features: "part1-core".to_string(), + reason: "blocking vector required".to_string(), + }]; + + let missing = missing_blocking_vectors(&vectors, &vector_root); + + fs::remove_dir_all(&vector_root).expect("remove temporary vector root"); + assert_eq!(missing, vec!["missing_blocking".to_string()]); +} + +fn missing_blocking_vectors(vectors: &[Vector], vector_root: &Path) -> Vec { + vectors + .iter() + .filter(|vector| vector.classification == Classification::Blocking) + .filter(|vector| !vector_root.join(&vector.path).exists()) + .map(|vector| vector.id.clone()) + .collect() +} + +#[test] +fn env_gated_iso_conformance_blocks_only_shipped_features() { + let Some(vector_root) = env::var_os(CONFORMANCE_ENV).map(PathBuf::from) else { + return; + }; + let vectors = load_manifest(); + let missing_blocking = missing_blocking_vectors(&vectors, &vector_root); + assert!( + missing_blocking.is_empty(), + "blocking ISO conformance vectors are missing from {}: {}", + vector_root.display(), + missing_blocking.join(", ") + ); + + for vector in vectors { + match vector.classification { + Classification::Investigate => { + panic!("{} is still investigate at release signoff", vector.id); + } + Classification::KnownLimitation => { + eprintln!( + "known limitation {}: {} ({})", + vector.id, vector.reason, vector.features + ); + } + Classification::Blocking => { + let path = vector_root.join(&vector.path); + let bytes = fs::read(&path) + .unwrap_or_else(|err| panic!("read blocking vector {}: {err}", vector.id)); + let image = Image::new(&bytes, &DecodeSettings::default()) + .unwrap_or_else(|err| panic!("parse blocking vector {}: {err}", vector.id)); + image + .decode_native() + .unwrap_or_else(|err| panic!("decode blocking vector {}: {err}", vector.id)); + } + } + } +} diff --git a/crates/signinum-j2k/tests/openjpeg_parity.rs b/crates/j2k/tests/openjpeg_parity.rs similarity index 70% rename from crates/signinum-j2k/tests/openjpeg_parity.rs rename to crates/j2k/tests/openjpeg_parity.rs index fe93c0ca..abce11a0 100644 --- a/crates/signinum-j2k/tests/openjpeg_parity.rs +++ b/crates/j2k/tests/openjpeg_parity.rs @@ -1,11 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 -use signinum_core::{Downscale, PixelFormat, Rect}; -use signinum_j2k::{ +use j2k::{ encode_j2k_lossless, EncodeBackendPreference, J2kDecoder, J2kLosslessEncodeOptions, J2kLosslessSamples, }; -use signinum_test_support::gradient_u8; +use j2k_core::{Downscale, PixelFormat, Rect}; +use j2k_test_support::{gradient_u8, read_pnm_pixels, write_pnm}; use std::{ fs, path::{Path, PathBuf}, @@ -25,7 +25,7 @@ fn classic_gray_full_decode_matches_openjpeg() { let mut out = vec![0_u8; 128 * 128]; decoder .decode_into(&mut out, 128, PixelFormat::Gray8) - .expect("signinum decode"); + .expect("j2k decode"); let expected = decode_with_openjpeg(&paths, "parity_full_gray", &jp2, ".pgm", &[]); assert_eq!(out, expected); @@ -43,7 +43,7 @@ fn classic_rgb_full_decode_matches_openjpeg() { let mut out = vec![0_u8; 128 * 128 * 3]; decoder .decode_into(&mut out, 128 * 3, PixelFormat::Rgb8) - .expect("signinum decode"); + .expect("j2k decode"); let expected = decode_with_openjpeg(&paths, "parity_full_rgb", &jp2, ".ppm", &[]); assert_eq!(out, expected); @@ -67,13 +67,13 @@ fn classic_gray_region_decode_matches_openjpeg_area_decode() { let mut out = vec![0_u8; roi.w as usize * roi.h as usize]; decoder .decode_region_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, roi.w as usize, PixelFormat::Gray8, roi, ) - .expect("signinum region decode"); + .expect("j2k region decode"); let expected = decode_with_openjpeg( &paths, @@ -106,13 +106,13 @@ fn classic_rgb_region_decode_matches_openjpeg_area_decode() { let mut out = vec![0_u8; roi.w as usize * roi.h as usize * 3]; decoder .decode_region_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, roi.w as usize * 3, PixelFormat::Rgb8, roi, ) - .expect("signinum region decode"); + .expect("j2k region decode"); let expected = decode_with_openjpeg( &paths, @@ -139,13 +139,13 @@ fn classic_gray_scaled_decode_matches_openjpeg_reduce() { let mut out = vec![0_u8; 32 * 32]; decoder .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, 32, PixelFormat::Gray8, Downscale::Quarter, ) - .expect("signinum scaled decode"); + .expect("j2k scaled decode"); let expected = decode_with_openjpeg(&paths, "parity_scaled_gray", &jp2, ".pgm", &["-r", "2"]); assert_eq!(out, expected); @@ -163,13 +163,13 @@ fn classic_rgb_scaled_decode_matches_openjpeg_reduce() { let mut out = vec![0_u8; 32 * 32 * 3]; decoder .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), + &mut j2k::J2kScratchPool::new(), &mut out, 32 * 3, PixelFormat::Rgb8, Downscale::Quarter, ) - .expect("signinum scaled decode"); + .expect("j2k scaled decode"); let expected = decode_with_openjpeg(&paths, "parity_scaled_rgb", &jp2, ".ppm", &["-r", "2"]); assert_eq!(out, expected); @@ -185,14 +185,11 @@ fn classic_lossless_encode_decodes_with_openjpeg() { let encoded = encode_j2k_lossless( samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - ..J2kLosslessEncodeOptions::default() - }, + &J2kLosslessEncodeOptions::default().with_backend(EncodeBackendPreference::CpuOnly), ) - .expect("signinum encode"); + .expect("j2k encode"); - let decoded = decode_j2k_with_openjpeg(&paths, "signinum_encode_rgb", &encoded.codestream); + let decoded = decode_j2k_with_openjpeg(&paths, "j2k_encode_rgb", &encoded.codestream); assert_eq!(decoded, pixels); } @@ -207,18 +204,18 @@ impl OpenJpegPaths { let paths = discover_openjpeg_paths(); assert!( paths.is_some() || !require_openjpeg(), - "SIGNINUM_REQUIRE_OPENJPEG is set but opj_compress/opj_decompress were not found" + "J2K_REQUIRE_OPENJPEG is set but opj_compress/opj_decompress were not found" ); paths } } fn discover_openjpeg_paths() -> Option { - let compress = std::env::var_os("SIGNINUM_OPENJPEG_COMPRESS_BIN") + let compress = std::env::var_os("J2K_OPENJPEG_COMPRESS_BIN") .map(PathBuf::from) .or_else(|| Some(PathBuf::from("/opt/homebrew/bin/opj_compress"))) .filter(|path| path.exists())?; - let decompress = std::env::var_os("SIGNINUM_OPENJPEG_BIN") + let decompress = std::env::var_os("J2K_OPENJPEG_BIN") .map(PathBuf::from) .or_else(|| Some(PathBuf::from("/opt/homebrew/bin/opj_decompress"))) .filter(|path| path.exists())?; @@ -229,7 +226,7 @@ fn discover_openjpeg_paths() -> Option { } fn require_openjpeg() -> bool { - std::env::var_os("SIGNINUM_REQUIRE_OPENJPEG").is_some() + std::env::var_os("J2K_REQUIRE_OPENJPEG").is_some() } fn encode_with_openjpeg( @@ -277,7 +274,7 @@ fn decode_with_openjpeg( command.args(extra_args); let status = command.status().expect("run opj_decompress"); assert!(status.success(), "opj_decompress failed"); - read_pnm_pixels(&output_path) + read_pnm_pixels(&output_path).expect("read pnm") } fn decode_j2k_with_openjpeg(paths: &OpenJpegPaths, stem: &str, codestream: &[u8]) -> Vec { @@ -294,88 +291,13 @@ fn decode_j2k_with_openjpeg(paths: &OpenJpegPaths, stem: &str, codestream: &[u8] .status() .expect("run opj_decompress"); assert!(status.success(), "opj_decompress failed"); - read_pnm_pixels(&output_path) -} - -fn write_pnm( - path: &Path, - pixels: &[u8], - width: u32, - height: u32, - channels: usize, -) -> std::io::Result<()> { - let mut bytes = Vec::new(); - if channels == 1 { - bytes.extend_from_slice(format!("P5\n{width} {height}\n255\n").as_bytes()); - } else { - bytes.extend_from_slice(format!("P6\n{width} {height}\n255\n").as_bytes()); - } - bytes.extend_from_slice(pixels); - fs::write(path, bytes) -} - -fn read_pnm_pixels(path: &Path) -> Vec { - let bytes = fs::read(path).expect("read pnm"); - let mut cursor = 0; - - let magic = read_pnm_token(&bytes, &mut cursor).expect("pnm magic"); - assert!(magic == b"P5" || magic == b"P6", "unexpected pnm magic"); - - let _width = read_pnm_token(&bytes, &mut cursor).expect("pnm width"); - let _height = read_pnm_token(&bytes, &mut cursor).expect("pnm height"); - let _maxval = read_pnm_token(&bytes, &mut cursor).expect("pnm maxval"); - - while cursor < bytes.len() && bytes[cursor].is_ascii_whitespace() { - cursor += 1; - } - - bytes[cursor..].to_vec() -} - -fn read_pnm_token<'a>(bytes: &'a [u8], cursor: &mut usize) -> Option<&'a [u8]> { - skip_pnm_separators(bytes, cursor); - if *cursor >= bytes.len() { - return None; - } - - let start = *cursor; - while *cursor < bytes.len() { - let byte = bytes[*cursor]; - if byte.is_ascii_whitespace() || byte == b'#' { - break; - } - *cursor += 1; - } - - if start == *cursor { - None - } else { - Some(&bytes[start..*cursor]) - } -} - -fn skip_pnm_separators(bytes: &[u8], cursor: &mut usize) { - while *cursor < bytes.len() { - let byte = bytes[*cursor]; - if byte.is_ascii_whitespace() { - *cursor += 1; - continue; - } - if byte == b'#' { - *cursor += 1; - while *cursor < bytes.len() && bytes[*cursor] != b'\n' { - *cursor += 1; - } - continue; - } - break; - } + read_pnm_pixels(&output_path).expect("read pnm") } fn temp_dir() -> &'static Path { static DIR: OnceLock = OnceLock::new(); DIR.get_or_init(|| { - let dir = std::env::temp_dir().join(format!("signinum-j2k-parity-{}", std::process::id())); + let dir = std::env::temp_dir().join(format!("j2k-parity-{}", std::process::id())); fs::create_dir_all(&dir).expect("create temp dir"); dir }) diff --git a/crates/signinum-j2k/tests/proptest_inspect.rs b/crates/j2k/tests/proptest_inspect.rs similarity index 94% rename from crates/signinum-j2k/tests/proptest_inspect.rs rename to crates/j2k/tests/proptest_inspect.rs index b198f32c..17ee06f9 100644 --- a/crates/signinum-j2k/tests/proptest_inspect.rs +++ b/crates/j2k/tests/proptest_inspect.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 +use j2k::J2kDecoder; use proptest::prelude::*; -use signinum_j2k::J2kDecoder; proptest! { #[test] diff --git a/crates/j2k/tests/recode.rs b/crates/j2k/tests/recode.rs new file mode 100644 index 00000000..0dd00cad --- /dev/null +++ b/crates/j2k/tests/recode.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: Apache-2.0 + +use j2k::{ + encode_j2k_lossless, J2kBlockCodingMode, J2kEncodeValidation, J2kError, + J2kLosslessEncodeOptions, J2kLosslessSamples, J2kToHtj2kMode, J2kToHtj2kOptions, + ReversibleTransform, +}; +use j2k_core::{CodecError, CompressedPayloadKind, CompressedTransferSyntax}; +use j2k_native::{DecodeSettings, EncodeOptions, Image}; +use j2k_test_support::{patterned_gray8, patterned_rgb8, wrap_jp2_codestream}; + +fn decode_native(codestream: &[u8]) -> j2k_native::RawBitmap { + Image::new(codestream, &DecodeSettings::default()) + .expect("codestream should parse") + .decode_native() + .expect("codestream should decode") +} + +fn lossless_options(block_coding_mode: J2kBlockCodingMode) -> J2kLosslessEncodeOptions { + J2kLosslessEncodeOptions::default() + .with_block_coding_mode(block_coding_mode) + .with_validation(J2kEncodeValidation::External) +} + +fn native_encode_options(reversible: bool, use_mct: bool) -> EncodeOptions { + EncodeOptions { + reversible, + use_mct, + use_ht_block_coding: false, + num_decomposition_levels: 1, + validate_high_throughput_codestream: false, + ..EncodeOptions::default() + } +} + +#[test] +fn classic_lossless_53_rgb_recode_to_htj2k_decodes_pixel_exact() { + let width = 64; + let height = 64; + let pixels = patterned_rgb8(width, height); + let samples = + J2kLosslessSamples::new(&pixels, width, height, 3, 8, false).expect("valid RGB samples"); + let classic = encode_j2k_lossless( + samples, + &lossless_options(J2kBlockCodingMode::Classic) + .with_reversible_transform(ReversibleTransform::Rct53), + ) + .expect("classic lossless encode") + .codestream; + + let recoded = j2k::recode_j2k_to_htj2k_lossless(&classic, J2kToHtj2kOptions::default()) + .expect("coefficient-domain recode"); + + assert_eq!(recoded.report.mode, J2kToHtj2kMode::CoefficientPreserving); + assert_eq!( + recoded.report.output_transfer_syntax, + CompressedTransferSyntax::HtJpeg2000Lossless + ); + assert!(recoded.bytes.starts_with(&[0xff, 0x4f])); + + let decoded = decode_native(&recoded.bytes); + assert_eq!((decoded.width, decoded.height), (width, height)); + assert_eq!(decoded.num_components, 3); + assert_eq!(decoded.bit_depth, 8); + assert_eq!(decoded.data, pixels); +} + +#[test] +fn classic_lossless_53_gray16_recode_to_htj2k_decodes_pixel_exact() { + let width = 64; + let height = 64; + let mut pixels = Vec::new(); + for sample in patterned_gray8(width, height) { + let value = u16::from(sample) * 257; + pixels.extend_from_slice(&value.to_le_bytes()); + } + let samples = J2kLosslessSamples::new(&pixels, width, height, 1, 16, false) + .expect("valid gray16 samples"); + let classic = encode_j2k_lossless(samples, &lossless_options(J2kBlockCodingMode::Classic)) + .expect("classic lossless encode") + .codestream; + + let recoded = j2k::recode_j2k_to_htj2k_lossless(&classic, J2kToHtj2kOptions::default()) + .expect("coefficient-domain recode"); + + assert_eq!(recoded.report.mode, J2kToHtj2kMode::CoefficientPreserving); + let decoded = decode_native(&recoded.bytes); + assert_eq!(decoded.data, pixels); +} + +#[test] +fn jp2_wrapped_classic_lossless_53_recode_emits_raw_htj2k_codestream() { + let width = 64; + let height = 64; + let pixels = patterned_rgb8(width, height); + let samples = + J2kLosslessSamples::new(&pixels, width, height, 3, 8, false).expect("valid RGB samples"); + let classic = encode_j2k_lossless( + samples, + &lossless_options(J2kBlockCodingMode::Classic) + .with_reversible_transform(ReversibleTransform::Rct53), + ) + .expect("classic lossless encode") + .codestream; + let jp2 = wrap_jp2_codestream(&classic, width, height, 3, 8, 16); + + let recoded = j2k::recode_j2k_to_htj2k_lossless(&jp2, J2kToHtj2kOptions::default()) + .expect("JP2 coefficient-domain recode"); + + assert_eq!( + recoded.report.input_payload_kind, + CompressedPayloadKind::Jp2File + ); + assert_eq!( + recoded.report.output_payload_kind, + CompressedPayloadKind::Jpeg2000Codestream + ); + assert!(recoded.bytes.starts_with(&[0xff, 0x4f])); + assert_eq!(decode_native(&recoded.bytes).data, pixels); +} + +#[test] +fn already_raw_htj2k_lossless_returns_passthrough() { + let width = 32; + let height = 32; + let pixels = patterned_gray8(width, height); + let samples = + J2kLosslessSamples::new(&pixels, width, height, 1, 8, false).expect("valid gray samples"); + let htj2k = encode_j2k_lossless( + samples, + &lossless_options(J2kBlockCodingMode::HighThroughput), + ) + .expect("HTJ2K encode") + .codestream; + + let recoded = j2k::recode_j2k_to_htj2k_lossless(&htj2k, J2kToHtj2kOptions::default()) + .expect("passthrough recode"); + + assert_eq!(recoded.report.mode, J2kToHtj2kMode::Passthrough); + assert_eq!(recoded.bytes, htj2k); +} + +#[test] +fn malformed_input_returns_explicit_error() { + let err = j2k::recode_j2k_to_htj2k_lossless(b"not jpeg 2000", J2kToHtj2kOptions::default()) + .expect_err("malformed input should fail"); + + assert!(matches!(err, J2kError::Unsupported(_)) || err.is_truncated()); +} + +#[test] +fn lossy_97_source_is_rejected_for_lossless_53_coefficient_recode() { + let width = 32; + let height = 32; + let pixels = patterned_gray8(width, height); + let lossy = j2k_native::encode( + &pixels, + width, + height, + 1, + 8, + false, + &native_encode_options(false, false), + ) + .expect("lossy 9/7 encode"); + + let err = j2k::recode_j2k_to_htj2k_lossless(&lossy, J2kToHtj2kOptions::default()) + .expect_err("lossy source should fail"); + + assert!(matches!(err, J2kError::Unsupported(_))); +} + +#[test] +fn signed_source_is_rejected_before_recode() { + let pixels = [0_u8, 1, 255, 127]; + let signed = j2k_native::encode( + &pixels, + 2, + 2, + 1, + 8, + true, + &native_encode_options(true, false), + ) + .expect("signed classic encode"); + + let err = j2k::recode_j2k_to_htj2k_lossless(&signed, J2kToHtj2kOptions::default()) + .expect_err("signed source should fail"); + + assert!(matches!(err, J2kError::Unsupported(_))); +} + +#[test] +fn unsupported_component_count_is_rejected() { + let pixels = vec![127_u8; 16 * 16 * 4]; + let four_component = j2k_native::encode( + &pixels, + 16, + 16, + 4, + 8, + false, + &native_encode_options(true, false), + ) + .expect("four-component classic encode"); + + let err = j2k::recode_j2k_to_htj2k_lossless(&four_component, J2kToHtj2kOptions::default()) + .expect_err("four-component source should fail"); + + assert!(matches!(err, J2kError::Unsupported(_))); +} diff --git a/crates/signinum-cli/README.md b/crates/signinum-cli/README.md deleted file mode 100644 index 21b15b20..00000000 --- a/crates/signinum-cli/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# signinum-cli - -Command-line inspection utility for `signinum`. - -Install: - -```sh -cargo install signinum-cli -``` - -The CPU-first 1.0 CLI provides: - -```sh -signinum inspect -``` - -It parses JPEG and JPEG 2000 headers and prints decoded metadata. It does not -own WSI container parsing, caching, prefetch, or image decode workflows. diff --git a/crates/signinum-cli/tests/inspect_cli.rs b/crates/signinum-cli/tests/inspect_cli.rs deleted file mode 100644 index 10ec4c6e..00000000 --- a/crates/signinum-cli/tests/inspect_cli.rs +++ /dev/null @@ -1,169 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - fs, - path::{Path, PathBuf}, - process::{Command, Output}, -}; - -fn signinum_bin() -> &'static str { - env!("CARGO_BIN_EXE_signinum") -} - -fn run_signinum(args: &[&str]) -> Output { - Command::new(signinum_bin()) - .args(args) - .output() - .expect("run signinum CLI") -} - -fn write_temp_file(name: &str, bytes: &[u8]) -> PathBuf { - let dir = std::env::temp_dir().join(format!("signinum-cli-tests-{}", std::process::id())); - fs::create_dir_all(&dir).expect("create CLI test temp dir"); - let path = dir.join(name); - fs::write(&path, bytes).expect("write CLI test input"); - path -} - -fn stdout(output: &Output) -> String { - String::from_utf8_lossy(&output.stdout).into_owned() -} - -fn stderr(output: &Output) -> String { - String::from_utf8_lossy(&output.stderr).into_owned() -} - -#[test] -fn inspect_cli_reports_jpeg_info() { - let jpeg = minimal_jpeg(); - let input = write_temp_file("grayscale_8x8.jpg", &jpeg); - - let output = run_signinum(&["inspect", path_str(&input)]); - - assert!(output.status.success(), "stderr: {}", stderr(&output)); - let stdout = stdout(&output); - assert!(stdout.contains("8")); - assert!(stdout.contains("Grayscale")); - assert!(stdout.contains("bit=8")); -} - -#[test] -fn inspect_cli_reports_jp2_info() { - let input = write_temp_file("minimal.jp2", &minimal_jp2()); - - let output = run_signinum(&["inspect", path_str(&input)]); - - assert!(output.status.success(), "stderr: {}", stderr(&output)); - let stdout = stdout(&output); - assert!(stdout.contains("128")); - assert!(stdout.contains("64")); - assert!(stdout.contains("levels=6")); -} - -#[test] -fn inspect_cli_rejects_unknown_subcommand() { - let output = run_signinum(&["unknown"]); - - assert_eq!(output.status.code(), Some(2)); - assert!(stderr(&output).contains("unknown subcommand: unknown")); -} - -#[test] -fn inspect_cli_reports_missing_file() { - let missing = std::env::temp_dir() - .join(format!("signinum-cli-tests-{}", std::process::id())) - .join("missing.jpg"); - - let output = run_signinum(&["inspect", path_str(&missing)]); - - assert_eq!(output.status.code(), Some(1)); - assert!(stderr(&output).contains("error reading")); -} - -#[test] -fn inspect_cli_reports_invalid_jpeg() { - let input = write_temp_file("invalid.jpg", b"not a jpeg"); - - let output = run_signinum(&["inspect", path_str(&input)]); - - assert_eq!(output.status.code(), Some(1)); - assert!(stderr(&output).contains("error:")); -} - -fn path_str(path: &Path) -> &str { - path.to_str().expect("test path is UTF-8") -} - -fn minimal_jpeg() -> Vec { - let gray = (0..64).map(|value| (value * 3) as u8).collect::>(); - signinum_jpeg::encode_jpeg_baseline( - signinum_jpeg::JpegSamples::Gray8 { - data: &gray, - width: 8, - height: 8, - }, - signinum_jpeg::JpegEncodeOptions { - quality: 90, - subsampling: signinum_jpeg::JpegSubsampling::Gray, - restart_interval: None, - backend: signinum_jpeg::JpegBackend::Cpu, - }, - ) - .expect("encode CLI test JPEG") - .data -} - -fn minimal_j2k_codestream() -> Vec { - let mut bytes = vec![0xFF, 0x4F]; - let mut siz = Vec::new(); - push_u16(&mut siz, 0); - push_u32(&mut siz, 128); - push_u32(&mut siz, 64); - push_u32(&mut siz, 0); - push_u32(&mut siz, 0); - push_u32(&mut siz, 64); - push_u32(&mut siz, 64); - push_u32(&mut siz, 0); - push_u32(&mut siz, 0); - push_u16(&mut siz, 3); - for _ in 0..3 { - siz.extend_from_slice(&[0x07, 0x01, 0x01]); - } - bytes.extend_from_slice(&[0xFF, 0x51]); - push_u16(&mut bytes, (siz.len() + 2) as u16); - bytes.extend_from_slice(&siz); - - let cod = [0x00, 0x00, 0x00, 0x01, 0x01, 0x05, 0x04, 0x04, 0x00, 0x01]; - bytes.extend_from_slice(&[0xFF, 0x52]); - push_u16(&mut bytes, (cod.len() + 2) as u16); - bytes.extend_from_slice(&cod); - bytes.extend_from_slice(&[0xFF, 0x90, 0x00, 0x0A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); - bytes -} - -fn minimal_jp2() -> Vec { - let codestream = minimal_j2k_codestream(); - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0D, 0x0A, 0x87, 0x0A]); - bytes.extend_from_slice(&[ - 0, 0, 0, 20, b'f', b't', b'y', b'p', b'j', b'p', b'2', b' ', 0, 0, 0, 0, b'j', b'p', b'2', - b' ', - ]); - bytes.extend_from_slice(&[ - 0, 0, 0, 45, b'j', b'p', b'2', b'h', 0, 0, 0, 22, b'i', b'h', b'd', b'r', 0, 0, 0, 64, 0, - 0, 0, 128, 0, 3, 7, 7, 0, 0, 0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0, 0, 0, 0, 16, - ]); - let len = (8 + codestream.len()) as u32; - bytes.extend_from_slice(&len.to_be_bytes()); - bytes.extend_from_slice(b"jp2c"); - bytes.extend_from_slice(&codestream); - bytes -} - -fn push_u16(out: &mut Vec, value: u16) { - out.extend_from_slice(&value.to_be_bytes()); -} - -fn push_u32(out: &mut Vec, value: u32) { - out.extend_from_slice(&value.to_be_bytes()); -} diff --git a/crates/signinum-core/README.md b/crates/signinum-core/README.md deleted file mode 100644 index e1869954..00000000 --- a/crates/signinum-core/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# signinum-core - -Shared CPU-first 1.0 decode contracts for the `signinum` workspace. - -Install: - -```sh -cargo add signinum-core -``` - -This crate contains the stable value types and traits used by the CPU codec -crates: - -- pixel/sample formats -- ROI and downscale geometry -- caller-owned scratch and decoder context traits -- row streaming and tile-batch decode traits -- backend request and device-surface contracts - -It contains no image-format parser or decoder. diff --git a/crates/signinum-core/src/context.rs b/crates/signinum-core/src/context.rs deleted file mode 100644 index 57ee464f..00000000 --- a/crates/signinum-core/src/context.rs +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub struct CacheStats { - pub hits: u64, - pub misses: u64, -} - -pub trait CodecContext: Default + Send { - fn clear(&mut self); - - fn cache_stats(&self) -> CacheStats { - CacheStats::default() - } -} - -#[derive(Debug, Default)] -pub struct DecoderContext { - codec: C, -} - -impl DecoderContext { - pub fn new() -> Self { - Self { - codec: C::default(), - } - } - - pub fn codec(&self) -> &C { - &self.codec - } - - pub fn codec_mut(&mut self) -> &mut C { - &mut self.codec - } - - pub fn clear(&mut self) { - self.codec.clear(); - } - - pub fn cache_stats(&self) -> CacheStats { - self.codec.cache_stats() - } - - pub fn into_inner(self) -> C { - self.codec - } -} diff --git a/crates/signinum-core/src/error.rs b/crates/signinum-core/src/error.rs deleted file mode 100644 index dca24847..00000000 --- a/crates/signinum-core/src/error.rs +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use crate::{pixel::PixelFormat, sample::SampleType}; - -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum BufferError { - #[error("output buffer too small: required {required} bytes, have {have}")] - OutputTooSmall { required: usize, have: usize }, - #[error("input buffer too small: required {required} bytes, have {have}")] - InputTooSmall { required: usize, have: usize }, - #[error("buffer size overflow while computing {what}")] - SizeOverflow { what: &'static str }, - #[error("stride {stride} is smaller than row width {row_bytes}")] - StrideTooSmall { row_bytes: usize, stride: usize }, - #[error("stride {stride} is not aligned to {align}")] - StrideNotAligned { stride: usize, align: usize }, - #[error("pixel format {fmt:?} does not match sample type {sample_type:?}")] - SampleTypeMismatch { - fmt: PixelFormat, - sample_type: SampleType, - }, -} - -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum InputError { - #[error("input too short: need {need} bytes, have {have}")] - TooShort { need: usize, have: usize }, - #[error("input truncated at offset {offset} while reading {segment}")] - TruncatedAt { - offset: usize, - segment: &'static str, - }, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -#[error("not yet implemented: {what}")] -pub struct NotImplemented { - pub what: &'static str, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] -#[error("unsupported: {what}")] -pub struct Unsupported { - pub what: &'static str, -} - -pub trait CodecError: core::error::Error + Send + Sync + 'static { - fn is_truncated(&self) -> bool; - fn is_not_implemented(&self) -> bool; - fn is_unsupported(&self) -> bool; - fn is_buffer_error(&self) -> bool; -} diff --git a/crates/signinum-core/src/lib.rs b/crates/signinum-core/src/lib.rs deleted file mode 100644 index 76f42ffb..00000000 --- a/crates/signinum-core/src/lib.rs +++ /dev/null @@ -1,49 +0,0 @@ -//! Shared traits and value types for the `signinum` workspace. -//! -//! Codec crates use this crate to expose common pixel formats, decode -//! outcomes, row sinks, caller-owned scratch pools, and CPU/GPU backend -//! selection contracts without depending on each other. - -#![no_std] -#![warn(unreachable_pub)] - -extern crate alloc; - -pub mod backend; -pub mod batch; -mod buffer; -pub mod context; -pub mod error; -pub mod passthrough; -pub mod pixel; -pub mod row_sink; -pub mod sample; -pub mod scale; -pub mod scratch; -pub mod traits; -pub mod types; - -pub use backend::{BackendCapabilities, BackendKind, BackendRequest, CpuFeatures}; -pub use batch::{ - collect_indexed_batch_results, tile_batch_worker_count, IndexedBatchResult, TileBatchOptions, -}; -pub use buffer::{ - copy_tight_pixels_to_strided_output, strided_output_len, validate_strided_output_buffer, -}; -pub use context::{CacheStats, CodecContext, DecoderContext}; -pub use error::{BufferError, CodecError, InputError, NotImplemented, Unsupported}; -pub use passthrough::{ - CompressedPayloadKind, CompressedTransferSyntax, PassthroughCandidate, PassthroughDecision, - PassthroughRejectReason, PassthroughRequirements, -}; -pub use pixel::{PixelFormat, PixelLayout}; -pub use row_sink::RowSink; -pub use sample::{Sample, SampleType}; -pub use scale::Downscale; -pub use scratch::ScratchPool; -pub use traits::{ - DecodeRowsError, DeviceSubmission, DeviceSurface, ImageCodec, ImageDecode, ImageDecodeDevice, - ImageDecodeRows, ImageDecodeSubmit, ReadySubmission, TileBatchDecode, TileBatchDecodeDevice, - TileBatchDecodeManyDevice, TileBatchDecodeSubmit, TileDecompress, -}; -pub use types::{CodedUnitLayout, Colorspace, DecodeOutcome, Info, Rect, TileLayout, WarningKind}; diff --git a/crates/signinum-core/src/scratch.rs b/crates/signinum-core/src/scratch.rs deleted file mode 100644 index f9cf2813..00000000 --- a/crates/signinum-core/src/scratch.rs +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -pub trait ScratchPool: Send { - fn bytes_allocated(&self) -> usize; - fn reset(&mut self); -} diff --git a/crates/signinum-core/src/types.rs b/crates/signinum-core/src/types.rs deleted file mode 100644 index 0415e592..00000000 --- a/crates/signinum-core/src/types.rs +++ /dev/null @@ -1,110 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use alloc::vec::Vec; - -use crate::scale::Downscale; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[non_exhaustive] -pub enum Colorspace { - Grayscale, - YCbCr, - Rgb, - Cmyk, - Ycck, - SRgb, - SGray, - IccTagged, - Rct, - Ict, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct TileLayout { - pub tile_width: u32, - pub tile_height: u32, - pub tiles_x: u32, - pub tiles_y: u32, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct CodedUnitLayout { - pub unit_width: u32, - pub unit_height: u32, - pub units_x: u32, - pub units_y: u32, -} - -impl CodedUnitLayout { - pub const fn unit_count(&self) -> u32 { - self.units_x.saturating_mul(self.units_y) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Rect { - pub x: u32, - pub y: u32, - pub w: u32, - pub h: u32, -} - -impl Rect { - pub const fn full(dims: (u32, u32)) -> Self { - Self { - x: 0, - y: 0, - w: dims.0, - h: dims.1, - } - } - - pub fn is_within(&self, dims: (u32, u32)) -> bool { - let (w, h) = dims; - self.x.checked_add(self.w).is_some_and(|r| r <= w) - && self.y.checked_add(self.h).is_some_and(|b| b <= h) - } - - #[must_use] - pub fn scaled_covering(&self, scale: Downscale) -> Self { - let denom = scale.denominator(); - let x_end = self.x.saturating_add(self.w); - let y_end = self.y.saturating_add(self.h); - let x0 = self.x / denom; - let y0 = self.y / denom; - let x1 = x_end.div_ceil(denom); - let y1 = y_end.div_ceil(denom); - Self { - x: x0, - y: y0, - w: x1.saturating_sub(x0), - h: y1.saturating_sub(y0), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Info { - pub dimensions: (u32, u32), - pub components: u8, - pub colorspace: Colorspace, - pub bit_depth: u8, - pub tile_layout: Option, - pub coded_unit_layout: Option, - pub restart_interval: Option, - pub resolution_levels: u8, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[non_exhaustive] -pub enum WarningKind { - MinorCompliance, - NonFatalTruncation, - UnusualFeature, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DecodeOutcome { - pub decoded: Rect, - pub warnings: Vec, -} diff --git a/crates/signinum-core/tests/repo_integrity.rs b/crates/signinum-core/tests/repo_integrity.rs deleted file mode 100644 index a9c1ebbc..00000000 --- a/crates/signinum-core/tests/repo_integrity.rs +++ /dev/null @@ -1,1055 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - collections::BTreeSet, - ffi::OsStr, - fs, - path::{Component, Path, PathBuf}, - process::Command, -}; - -fn repo_root() -> &'static Path { - Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .and_then(Path::parent) - .expect("workspace root") -} - -#[test] -fn conformance_manifest_lists_all_committed_jpeg_inputs() { - let root = repo_root(); - let conformance = root.join("corpus/conformance"); - let manifest = - fs::read_to_string(conformance.join("manifest.json")).expect("read conformance manifest"); - - for entry in fs::read_dir(&conformance).expect("read conformance dir") { - let entry = entry.expect("read conformance entry"); - let path = entry.path(); - if path.extension().and_then(|ext| ext.to_str()) != Some("jpg") { - continue; - } - let filename = path - .file_name() - .and_then(|name| name.to_str()) - .expect("utf-8 fixture filename"); - assert!( - manifest.contains(&format!("\"{filename}\"")), - "conformance fixture {filename} is missing from manifest.json" - ); - } -} - -#[test] -fn corpus_readme_does_not_claim_committed_fixtures_are_absent() { - let readme = - fs::read_to_string(repo_root().join("corpus/README.md")).expect("read corpus README"); - - assert!( - !readme.contains("intentionally empty"), - "corpus README still claims the committed fixture corpus is empty" - ); -} - -#[test] -fn adapter_crates_do_not_import_codec_private_modules() { - let root = repo_root(); - let adapter_crates = [ - "crates/signinum-jpeg-metal", - "crates/signinum-jpeg-cuda", - "crates/signinum-j2k-metal", - "crates/signinum-j2k-cuda", - ]; - - for crate_dir in adapter_crates { - for path in rust_sources(&root.join(crate_dir)) { - let source = fs::read_to_string(&path) - .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); - assert!( - !source.contains("::__private") && !source.contains(" __private::"), - "adapter source {} imports a codec __private module; use the public adapter API", - path.strip_prefix(root).unwrap_or(&path).display() - ); - } - } -} - -#[test] -fn cuda_adapter_crates_keep_public_libs_as_module_shells() { - let root = repo_root(); - let expected_modules = [ - ( - "crates/signinum-jpeg-cuda", - [ - "codec.rs", - "decoder.rs", - "error.rs", - "runtime.rs", - "session.rs", - "surface.rs", - ] - .as_slice(), - ), - ( - "crates/signinum-j2k-cuda", - [ - "codec.rs", - "decoder.rs", - "encode.rs", - "error.rs", - "runtime.rs", - "session.rs", - "surface.rs", - ] - .as_slice(), - ), - ]; - - for (crate_dir, modules) in expected_modules { - let src_dir = root.join(crate_dir).join("src"); - let lib_path = src_dir.join("lib.rs"); - let lib = fs::read_to_string(&lib_path) - .unwrap_or_else(|err| panic!("read {}: {err}", lib_path.display())); - let line_count = lib.lines().count(); - assert!( - line_count <= 220, - "{} should stay a thin public module shell; found {line_count} lines", - lib_path.strip_prefix(root).unwrap_or(&lib_path).display() - ); - - for module in modules { - let module_path = src_dir.join(module); - assert!( - module_path.exists(), - "{} must exist to keep CUDA adapter responsibilities focused", - module_path - .strip_prefix(root) - .unwrap_or(&module_path) - .display() - ); - } - } -} - -#[test] -fn reusable_benchmark_generators_live_in_test_support() { - let root = repo_root(); - let support = fs::read_to_string(root.join("crates/signinum-test-support/src/lib.rs")) - .expect("read signinum-test-support"); - - for required in [ - "pub fn gradient_u8", - "pub fn patterned_rgb8_tiles", - "pub fn gpu_bench_rgb8", - ] { - assert!( - support.contains(required), - "signinum-test-support must expose reusable generator `{required}`" - ); - } -} - -#[test] -fn workspace_contains_public_signinum_facade_crate() { - let root = repo_root(); - let manifest_path = root.join("crates/signinum/Cargo.toml"); - let manifest = fs::read_to_string(&manifest_path).unwrap_or_else(|err| { - panic!("read {}: {err}", manifest_path.display()); - }); - - for required in [ - "name = \"signinum\"", - "signinum-core", - "signinum-jpeg", - "signinum-j2k", - "signinum-tilecodec", - ] { - assert!( - manifest.contains(required), - "{} must contain `{required}`", - manifest_path - .strip_prefix(root) - .unwrap_or(&manifest_path) - .display() - ); - } - - let root_manifest = - fs::read_to_string(root.join("Cargo.toml")).expect("read workspace manifest"); - assert!( - root_manifest.contains("\"crates/signinum\""), - "workspace members must include the public signinum facade crate" - ); -} - -#[test] -fn architecture_dependency_graph_matches_cargo_metadata() { - let root = repo_root(); - let metadata_edges = cargo_metadata_workspace_edges(root); - let docs = - fs::read_to_string(root.join("docs/architecture.md")).expect("read architecture docs"); - let docs_edges = architecture_doc_dependency_edges(&docs); - - let missing = metadata_edges - .difference(&docs_edges) - .map(format_edge) - .collect::>(); - let extra = docs_edges - .difference(&metadata_edges) - .map(format_edge) - .collect::>(); - - assert!( - missing.is_empty() && extra.is_empty(), - "docs/architecture.md crate dependency graph drifted from cargo metadata\n\ - missing from docs: {missing:#?}\n\ - not in cargo metadata: {extra:#?}" - ); -} - -#[test] -fn wsi_decode_api_guide_covers_public_surfaces() { - let root = repo_root(); - let readme = fs::read_to_string(root.join("README.md")).expect("read README"); - let architecture = - fs::read_to_string(root.join("docs/architecture.md")).expect("read architecture docs"); - let guide_path = root.join("docs/wsi-decode-api.md"); - let guide = fs::read_to_string(&guide_path).expect("read WSI decode API guide"); - - assert!( - readme.contains("docs/wsi-decode-api.md"), - "README must link the WSI decode API guide" - ); - assert!( - architecture.contains("wsi-decode-api.md"), - "architecture docs must link the WSI decode API guide" - ); - - for required in [ - "decode_region_scaled_into", - "decode_rows", - "TileBatchDecode", - "BackendRequest::Auto", - "BackendRequest::Metal", - "BackendRequest::Cuda", - "DeviceSurface", - "ScratchPool", - "DecoderContext", - ] { - assert!( - guide.contains(required), - "{} must document {required}", - guide_path - .strip_prefix(root) - .unwrap_or(&guide_path) - .display() - ); - } -} - -#[test] -fn ci_workflow_keeps_docs_and_benchmark_compile_gates() { - let workflow = - fs::read_to_string(repo_root().join(".github/workflows/ci.yml")).expect("read CI workflow"); - let xtask = fs::read_to_string(repo_root().join("xtask/src/main.rs")).expect("read xtask"); - - for required in ["cargo xtask doc", "cargo xtask bench-build", "macos-latest"] { - assert!( - workflow.contains(required), - "CI workflow must contain `{required}`" - ); - } - assert!( - !workflow.contains("macos-13"), - "hosted CI must not gate releases on unsupported Intel macOS runners" - ); - - for required in [ - "\"doc\"", - "\"--workspace\"", - "\"--all-features\"", - "\"--no-deps\"", - "\"signinum-jpeg-metal\"", - "\"signinum-j2k-metal\"", - "\"--no-run\"", - ] { - assert!(xtask.contains(required), "xtask must contain `{required}`"); - } -} - -#[test] -fn xtask_test_does_not_run_benchmarks_as_tests() { - let xtask = fs::read_to_string(repo_root().join("xtask/src/main.rs")).expect("read xtask"); - let test_section = xtask - .split("fn test()") - .nth(1) - .and_then(|rest| rest.split("fn doc()").next()) - .expect("xtask test section"); - - for required in ["\"--lib\"", "\"--bins\"", "\"--tests\"", "\"--doc\""] { - assert!( - test_section.contains(required), - "xtask test must include cargo test selector `{required}`" - ); - } - assert!( - !test_section.contains("\"--all-targets\""), - "xtask test must not pass --all-targets because harness=false benchmark binaries would run as tests" - ); -} - -#[test] -fn xtask_fuzz_build_checks_every_fuzz_manifest() { - let root = repo_root(); - let xtask = fs::read_to_string(root.join("xtask/src/main.rs")).expect("read xtask"); - let mut manifests = Vec::new(); - - for entry in fs::read_dir(root.join("crates")).expect("read crates dir") { - let entry = entry.expect("read crate entry"); - let manifest = entry.path().join("fuzz/Cargo.toml"); - if manifest.exists() { - manifests.push(manifest); - } - } - manifests.sort(); - assert!( - !manifests.is_empty(), - "repository must keep fuzz targets under crates/*/fuzz" - ); - - for manifest in manifests { - let relative_path = manifest - .strip_prefix(root) - .expect("fuzz manifest under repo root"); - let relative = relative_path - .iter() - .map(|part| part.to_string_lossy()) - .collect::>() - .join("/"); - assert!( - xtask.contains(&relative), - "xtask fuzz-build must check {relative}" - ); - } -} - -#[test] -fn ci_coverage_job_is_a_required_gate() { - let workflow = - fs::read_to_string(repo_root().join(".github/workflows/ci.yml")).expect("read CI workflow"); - let coverage_job = workflow_job(&workflow, "coverage"); - - assert!( - coverage_job.contains("taiki-e/install-action@cargo-llvm-cov") - && coverage_job.contains("cargo xtask coverage"), - "coverage job must install cargo-llvm-cov and run xtask coverage" - ); - assert!( - !coverage_job.contains("continue-on-error"), - "coverage job must not be allowed to fail silently" - ); -} - -#[test] -fn gpu_validation_workflow_is_self_hosted_and_explicit() { - let root = repo_root(); - let workflow_path = root.join(".github/workflows/gpu-validation.yml"); - let workflow = fs::read_to_string(&workflow_path).expect("read GPU validation workflow"); - - for required in [ - "workflow_dispatch", - "run-timed-benchmarks", - "run-metal-validation", - "self-hosted", - "metal", - "cuda", - "cargo test -p signinum-jpeg-metal", - "cargo test -p signinum-j2k-metal", - "cargo test -p signinum-jpeg-cuda", - "cargo test -p signinum-j2k-cuda", - ] { - assert!( - workflow.contains(required), - "{} must contain `{required}`", - workflow_path - .strip_prefix(root) - .unwrap_or(&workflow_path) - .display() - ); - } -} - -#[test] -fn cuda_gpu_validation_job_stays_cuda_focused() { - let root = repo_root(); - let workflow_path = root.join(".github/workflows/gpu-validation.yml"); - let workflow = fs::read_to_string(&workflow_path).expect("read GPU validation workflow"); - let cuda_job = workflow_job(&workflow, "cuda-x86_64-compatibility"); - - for required in [ - "runs-on: [self-hosted, Linux, X64, cuda]", - "SIGNINUM_REQUIRE_CUDA_RUNTIME", - "SIGNINUM_REQUIRE_CUDA_JPEG_HARDWARE_DECODE", - "SIGNINUM_GPU_BENCH_DIM", - "SIGNINUM_GPU_BENCH_BATCH", - "SIGNINUM_GPU_BENCH_BATCH_DIM", - "uname -a", - "rustc -Vv", - "cargo -V", - "nvidia-smi", - "ldconfig -p | grep -i nvjpeg", - "CUDA runtime validation requires a working CUDA driver", - "cargo test -p signinum-jpeg-cuda --all-targets --features cuda-runtime", - "cargo test -p signinum-j2k-cuda --all-targets --features cuda-runtime", - "cargo bench -p signinum-jpeg-cuda --bench device_decode --features cuda-runtime --no-run", - "cargo bench -p signinum-jpeg-cuda --bench device_decode --features cuda-runtime -- --noplot", - ] { - assert!( - cuda_job.contains(required), - "{} CUDA job must contain `{required}`", - workflow_path - .strip_prefix(root) - .unwrap_or(&workflow_path) - .display() - ); - } - - for forbidden in [ - "cargo bench -p signinum-j2k-metal --bench compare --no-run", - "cargo bench -p signinum-jpeg --no-run", - "cargo test -p signinum-jpeg-metal", - "cargo test -p signinum-j2k-metal", - ] { - assert!( - !cuda_job.contains(forbidden), - "{} CUDA job must not contain Metal validation command `{forbidden}`", - workflow_path - .strip_prefix(root) - .unwrap_or(&workflow_path) - .display() - ); - } -} - -#[test] -fn crates_io_publish_policy_is_explicit() { - let root = repo_root(); - let workspace = fs::read_to_string(root.join("Cargo.toml")).expect("read workspace manifest"); - let xtask = fs::read_to_string(root.join("xtask/src/main.rs")).expect("read xtask"); - let publishable = const_array_block(&xtask, "PUBLISHABLE_PACKAGES"); - let publish_workflow = fs::read_to_string(root.join(".github/workflows/publish.yml")) - .expect("read publish workflow"); - - assert!( - workspace.contains("version = \"0.4.2\""), - "workspace package version must match the current staged release version" - ); - - for package in [ - "signinum-core", - "signinum-cuda-runtime", - "signinum-profile", - "signinum-j2k-native", - "signinum-tilecodec", - "signinum-jpeg", - "signinum-j2k", - "signinum-jpeg-metal", - "signinum-jpeg-cuda", - "signinum-j2k-metal", - "signinum-j2k-cuda", - "signinum-cli", - "signinum", - ] { - assert!( - publishable.contains(&format!("\"{package}\"")), - "xtask package gate must include publishable package {package}" - ); - assert!( - publish_workflow.contains(&format!("publish-{package}:")), - "publish workflow must include package {package}" - ); - } - - let package = "signinum-j2k-compare"; - assert!( - !publishable.contains(&format!("\"{package}\"")), - "xtask package gate must not package local comparator package {package}" - ); - assert!( - !publish_workflow.contains(&format!("publish-{package}:")), - "publish workflow must not publish local comparator package {package}" - ); -} - -#[test] -fn release_docs_use_manifest_versions_for_publish_order() { - let release = - fs::read_to_string(repo_root().join("docs/release.md")).expect("read release docs"); - - assert!( - release.contains("manifest versions"), - "release docs must describe publishing the current manifest versions instead of stale hard-coded versions" - ); - assert!( - !release.contains("`signinum-j2k` `1.1.0`") - && !release.contains("`signinum-j2k-native` `0.3.0`") - && !release.contains("`signinum` `1.0.0`"), - "release docs must not carry stale pre-facade publish versions" - ); -} - -fn const_array_block<'a>(source: &'a str, name: &str) -> &'a str { - let start = source - .find(&format!("const {name}:")) - .unwrap_or_else(|| panic!("missing const {name}")); - let rest = &source[start..]; - let end = rest - .find("];") - .unwrap_or_else(|| panic!("unterminated const {name}")); - &rest[..end] -} - -#[test] -fn j2k_compare_stays_unpublished_and_out_of_j2k_package_deps() { - let root = repo_root(); - let compare_manifest = fs::read_to_string(root.join("crates/signinum-j2k-compare/Cargo.toml")) - .expect("read signinum-j2k-compare manifest"); - let j2k_manifest = fs::read_to_string(root.join("crates/signinum-j2k/Cargo.toml")) - .expect("read signinum-j2k manifest"); - - assert!( - compare_manifest.contains("publish = false"), - "signinum-j2k-compare must remain an unpublished local oracle helper" - ); - assert!( - !j2k_manifest.contains("signinum-j2k-compare"), - "signinum-j2k must not package a dev-dependency on signinum-j2k-compare" - ); -} - -#[test] -fn package_preflight_is_staged_dependency_aware() { - let root = repo_root(); - let xtask = fs::read_to_string(root.join("xtask/src/main.rs")).expect("read xtask"); - let publish_script = - fs::read_to_string(root.join("scripts/publish-crate.sh")).expect("read publish script"); - let release = fs::read_to_string(root.join("docs/release.md")).expect("read release docs"); - - assert!( - xtask.contains("STAGED_DEPENDENCY_PACKAGES"), - "xtask package preflight must explicitly model crates blocked by unpublished staged dependencies" - ); - assert!( - xtask.contains("\"--list\"") && xtask.contains("unpublished workspace dependencies"), - "xtask package preflight must validate package contents for staged downstream crates without hiding why strict packaging is skipped" - ); - assert!( - publish_script.contains("dry-run package list only") - && publish_script.contains("signinum-cli") - && publish_script.contains("cargo package -p \"$crate\" --list"), - "publish workflow dry-run must not fail downstream crates only because staged dependency versions are not published yet" - ); - assert!( - release.contains("cargo package --list") - && release.contains("cargo publish --dry-run") - && release.contains("unpublished workspace dependencies"), - "release docs must explain the pre-publish package validation limits" - ); -} - -#[test] -fn public_docs_describe_facade_auto_and_cuda_runtime_surface_scope() { - let root = repo_root(); - let readme = fs::read_to_string(root.join("README.md")).expect("read README"); - let changelog = fs::read_to_string(root.join("CHANGELOG.md")).expect("read changelog"); - let architecture = - fs::read_to_string(root.join("docs/architecture.md")).expect("read architecture docs"); - let release = fs::read_to_string(root.join("docs/release.md")).expect("read release docs"); - let wsi_api = - fs::read_to_string(root.join("docs/wsi-decode-api.md")).expect("read WSI API docs"); - - for (name, docs) in [ - ("README.md", readme.as_str()), - ("docs/architecture.md", architecture.as_str()), - ("docs/release.md", release.as_str()), - ] { - assert!( - docs.contains("facade release") - && docs.contains("Runtime backend selection defaults to `Auto`"), - "{name} must name the facade release posture and Auto backend policy" - ); - } - - for (name, docs) in [ - ("README.md", readme.as_str()), - ("CHANGELOG.md", changelog.as_str()), - ("docs/wsi-decode-api.md", wsi_api.as_str()), - ("docs/release.md", release.as_str()), - ] { - assert!( - docs.contains("cuda-runtime") - && docs.contains("CUDA device memory") - && docs.contains("nvJPEG") - && docs.contains("NVIDIA performance"), - "{name} must describe CUDA device-memory output and nvJPEG scope without overclaiming NVIDIA performance" - ); - assert!( - !docs.contains("compatibility-only with no runtime CUDA decode"), - "{name} must not describe CUDA as compatibility-only after runtime surface support exists" - ); - } -} - -#[test] -fn public_docs_route_users_to_current_crates() { - let root = repo_root(); - let readme = fs::read_to_string(root.join("README.md")).expect("read README"); - - for required in [ - "Which crate should I use?", - "Fast Path For LLM-Assisted Use", - "cargo add signinum", - "statumen", - "wsi-dicom", - "signinum-jpeg", - "signinum-j2k", - "signinum-cli", - ] { - assert!( - readme.contains(required), - "README.md must route users to `{required}` after the rename" - ); - } - - let legacy_terms = [ - format!("{}{}", "ash", "lar"), - format!("{}{}", "zig", "gurat"), - ]; - for legacy in &legacy_terms { - assert!( - !readme.to_ascii_lowercase().contains(legacy), - "README.md must use current package names only" - ); - } -} - -#[test] -fn public_repo_excludes_agent_private_artifacts() { - let root = repo_root(); - let private_docs_name = ["super", "powers"].concat(); - let private_dir = ["docs", private_docs_name.as_str()].join("/"); - let migration_doc = ["MIGRATION", ".md"].concat(); - let migration_doc_lower = migration_doc.to_ascii_lowercase(); - let mut offenders = Vec::new(); - - for path in repo_text_files(root) { - let relative = path.strip_prefix(root).unwrap_or(&path); - let relative_text = relative.to_string_lossy(); - let file_name = path - .file_name() - .and_then(OsStr::to_str) - .unwrap_or_default() - .to_ascii_lowercase(); - if relative_text.starts_with(&private_dir) || file_name == migration_doc_lower { - offenders.push(relative_text.to_string()); - } - } - - assert!( - offenders.is_empty(), - "public repo must not track agent-private planning docs or migration scratch files: {offenders:?}" - ); -} - -#[test] -fn published_crates_have_crates_io_landing_readmes() { - let root = repo_root(); - - for crate_dir in [ - "crates/signinum-core", - "crates/signinum-cuda-runtime", - "crates/signinum-profile", - "crates/signinum-j2k-native", - "crates/signinum-tilecodec", - "crates/signinum-jpeg", - "crates/signinum-j2k", - "crates/signinum-jpeg-metal", - "crates/signinum-jpeg-cuda", - "crates/signinum-j2k-metal", - "crates/signinum-j2k-cuda", - "crates/signinum-cli", - ] { - let manifest_path = root.join(crate_dir).join("Cargo.toml"); - let manifest = fs::read_to_string(&manifest_path) - .unwrap_or_else(|err| panic!("read {}: {err}", manifest_path.display())); - let readme_path = root.join(crate_dir).join("README.md"); - - assert!( - manifest.contains("readme"), - "{} must declare a readme for crates.io landing pages", - manifest_path - .strip_prefix(root) - .unwrap_or(&manifest_path) - .display() - ); - assert!( - readme_path.exists(), - "{} must exist for crates.io landing pages", - readme_path - .strip_prefix(root) - .unwrap_or(&readme_path) - .display() - ); - } -} - -#[test] -fn public_text_does_not_embed_local_user_home_paths() { - let root = repo_root(); - let mut offenders = Vec::new(); - - for path in repo_text_files(root) { - if is_archived_handoff(&path) { - continue; - } - if path.ends_with("crates/signinum-core/tests/repo_integrity.rs") { - continue; - } - let source = fs::read_to_string(&path) - .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); - if source.contains("/Users/") || source.contains("C:\\Users\\") { - offenders.push( - path.strip_prefix(root) - .unwrap_or(&path) - .display() - .to_string(), - ); - } - } - - assert!( - offenders.is_empty(), - "public text must not embed local user-home paths; use env vars or repo-relative defaults: {offenders:?}" - ); -} - -#[test] -fn referenced_shell_scripts_exist() { - let root = repo_root(); - let mut missing = Vec::new(); - - for path in repo_text_files(root) { - if is_archived_handoff(&path) { - continue; - } - let source = fs::read_to_string(&path) - .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); - for script in referenced_shell_scripts(&source) { - let root_relative = root.join(&script); - let file_relative = path.parent().expect("text file has parent").join(&script); - if !root_relative.exists() && !file_relative.exists() { - missing.push(format!( - "{} references missing script {script}", - path.strip_prefix(root).unwrap_or(&path).display() - )); - } - } - } - - assert!( - missing.is_empty(), - "all referenced shell scripts must exist: {missing:?}" - ); -} - -#[test] -fn public_narrative_docs_do_not_carry_stale_zeiss_claims() { - let root = repo_root(); - let mut offenders = Vec::new(); - - for relative in [ - "README.md", - "docs/architecture.md", - "docs/bench.md", - "docs/parity.md", - "docs/release.md", - "docs/wsi-decode-api.md", - "paper/paper.md", - "paper/arxiv/main.tex", - ] { - let path = root.join(relative); - let Ok(source) = fs::read_to_string(&path) else { - if relative.starts_with("paper/") { - continue; - } - panic!("read {}: missing required narrative doc", path.display()); - }; - if source.contains("Zeiss") { - offenders.push(relative); - } - } - - assert!( - offenders.is_empty(), - "public narrative docs must not carry stale Zeiss integration claims: {offenders:?}" - ); -} - -#[test] -fn packaged_rust_sources_do_not_include_files_outside_their_crate() { - let root = repo_root(); - let workspace_crates = root.join("crates"); - let mut escaping = Vec::new(); - - for source_path in rust_sources(&workspace_crates) { - let Ok(relative_to_crates) = source_path.strip_prefix(&workspace_crates) else { - continue; - }; - let Some(crate_name) = relative_to_crates.components().next() else { - continue; - }; - let member_root = workspace_crates.join(crate_name.as_os_str()); - let source = fs::read_to_string(&source_path) - .unwrap_or_else(|err| panic!("read {}: {err}", source_path.display())); - - for include_path in rust_include_paths(&source) { - let resolved = normalize_path( - &source_path - .parent() - .expect("source file has parent") - .join(&include_path), - ); - if !resolved.starts_with(&member_root) { - escaping.push(format!( - "{} includes {} outside package root", - source_path - .strip_prefix(root) - .unwrap_or(&source_path) - .display(), - include_path - )); - } - } - } - - assert!( - escaping.is_empty(), - "package source include paths must stay inside their crate so packaged tests/benches/examples are not dead: {escaping:?}" - ); -} - -fn workflow_job<'a>(workflow: &'a str, job_name: &str) -> &'a str { - let marker = format!(" {job_name}:"); - let start = workflow - .find(&marker) - .unwrap_or_else(|| panic!("missing workflow job {job_name}")); - let rest = &workflow[start..]; - let mut search_start = marker.len(); - let mut end = rest.len(); - while let Some(relative) = rest[search_start..].find("\n ") { - let candidate = search_start + relative + 1; - if !rest[candidate..].starts_with(" ") { - end = candidate; - break; - } - search_start = candidate + 1; - } - &rest[..end] -} - -fn cargo_metadata_workspace_edges(root: &Path) -> BTreeSet<(String, String)> { - let output = Command::new("cargo") - .args(["metadata", "--no-deps", "--format-version=1"]) - .current_dir(root) - .output() - .expect("run cargo metadata"); - assert!( - output.status.success(), - "cargo metadata failed\nstdout:\n{}\nstderr:\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - - let metadata = - serde_json::from_slice::(&output.stdout).expect("parse cargo metadata"); - let workspace_members = metadata["workspace_members"] - .as_array() - .expect("metadata workspace_members array") - .iter() - .map(|id| { - id.as_str() - .expect("workspace member id is string") - .to_owned() - }) - .collect::>(); - let workspace_packages = metadata["packages"] - .as_array() - .expect("metadata packages array") - .iter() - .filter(|package| { - package["id"] - .as_str() - .is_some_and(|id| workspace_members.contains(id)) - }) - .filter_map(|package| package["name"].as_str()) - .collect::>(); - - let mut edges = BTreeSet::new(); - for package in metadata["packages"] - .as_array() - .expect("metadata packages array") - .iter() - .filter(|package| { - package["id"] - .as_str() - .is_some_and(|id| workspace_members.contains(id)) - }) - { - let source = package["name"].as_str().expect("package name"); - for dependency in package["dependencies"] - .as_array() - .expect("package dependencies array") - .iter() - .filter(|dependency| dependency["kind"].is_null()) - .filter(|dependency| dependency["source"].is_null()) - .filter_map(|dependency| dependency["name"].as_str()) - .filter(|dependency| workspace_packages.contains(dependency)) - { - edges.insert((source.to_owned(), dependency.to_owned())); - } - } - edges -} - -fn architecture_doc_dependency_edges(docs: &str) -> BTreeSet<(String, String)> { - let graph_section = docs - .split("## Crate dependency graph") - .nth(1) - .expect("architecture dependency graph section"); - let graph_block = graph_section - .split("```") - .nth(1) - .expect("architecture dependency graph code block"); - let mut edges = BTreeSet::new(); - - for line in graph_block.lines().filter(|line| line.contains("->")) { - let (source, dependencies) = line.split_once("->").expect("graph edge line"); - let source = source.trim(); - for dependency in dependencies.split(',') { - let dependency = dependency - .split_whitespace() - .next() - .expect("graph dependency token"); - edges.insert((source.to_owned(), dependency.to_owned())); - } - } - - edges -} - -fn format_edge(edge: &(String, String)) -> String { - format!("{} -> {}", edge.0, edge.1) -} - -fn rust_sources(dir: &Path) -> Vec { - let mut out = Vec::new(); - collect_rust_sources(dir, &mut out); - out -} - -fn collect_rust_sources(dir: &Path, out: &mut Vec) { - for entry in fs::read_dir(dir).unwrap_or_else(|err| panic!("read {}: {err}", dir.display())) { - let entry = entry.expect("read directory entry"); - let path = entry.path(); - if path.is_dir() { - collect_rust_sources(&path, out); - } else if path.extension().and_then(|ext| ext.to_str()) == Some("rs") { - out.push(path); - } - } -} - -fn repo_text_files(root: &Path) -> Vec { - let mut out = Vec::new(); - collect_repo_text_files(root, &mut out); - out -} - -fn collect_repo_text_files(dir: &Path, out: &mut Vec) { - for entry in fs::read_dir(dir).unwrap_or_else(|err| panic!("read {}: {err}", dir.display())) { - let entry = entry.expect("read directory entry"); - let path = entry.path(); - if path.is_dir() { - if should_skip_repo_dir(&path) { - continue; - } - collect_repo_text_files(&path, out); - continue; - } - if is_repo_text_file(&path) { - out.push(path); - } - } -} - -fn should_skip_repo_dir(path: &Path) -> bool { - path.file_name() - .and_then(OsStr::to_str) - .is_some_and(|name| matches!(name, ".git" | ".venv" | "target")) -} - -fn is_repo_text_file(path: &Path) -> bool { - matches!( - path.extension().and_then(OsStr::to_str), - Some("bib" | "json" | "md" | "rs" | "sh" | "tex" | "toml" | "txt" | "yaml" | "yml") - ) -} - -fn is_archived_handoff(path: &Path) -> bool { - path.file_name() - .and_then(OsStr::to_str) - .is_some_and(|name| name.starts_with("HANDOFF-")) -} - -fn referenced_shell_scripts(source: &str) -> Vec { - source - .split(|ch: char| !(ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_' | '/'))) - .filter(|token| { - Path::new(token) - .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("sh")) - && token.contains('/') - }) - .filter(|token| !token.starts_with("http://") && !token.starts_with("https://")) - .map(str::to_string) - .collect() -} - -fn rust_include_paths(source: &str) -> Vec { - let mut out = Vec::new(); - for marker in ["include_bytes!(\"", "include_str!(\""] { - let mut rest = source; - while let Some(start) = rest.find(marker) { - let after_marker = &rest[start + marker.len()..]; - let Some(end) = after_marker.find('"') else { - break; - }; - out.push(after_marker[..end].to_string()); - rest = &after_marker[end + 1..]; - } - } - out -} - -fn normalize_path(path: &Path) -> PathBuf { - let mut normalized = PathBuf::new(); - for component in path.components() { - match component { - Component::ParentDir => { - normalized.pop(); - } - Component::CurDir => {} - other => normalized.push(other.as_os_str()), - } - } - normalized -} diff --git a/crates/signinum-cuda-runtime/README.md b/crates/signinum-cuda-runtime/README.md deleted file mode 100644 index d79d4818..00000000 --- a/crates/signinum-cuda-runtime/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# signinum-cuda-runtime - -CUDA Driver API runtime helpers for the `signinum` CUDA adapter crates. - -Most downstream users should depend on `signinum-jpeg-cuda` or -`signinum-j2k-cuda` instead of using this crate directly. This crate owns the -small runtime layer used by those adapters to allocate CUDA device memory, -copy bytes between host and device, launch bundled CUDA kernels, call nvJPEG -when it is available, and report CUDA driver errors clearly. - -The runtime currently exposes full-frame RGB8 JPEG decode through NVIDIA -nvJPEG, including the legacy batched API used by `signinum-jpeg-cuda` for -full-tile RGB8 batches. JPEG 2000 / HTJ2K CUDA adapters still upload -CPU-decoded bytes into CUDA device memory and do not provide CUDA codestream -decode kernels. diff --git a/crates/signinum-cuda-runtime/build.rs b/crates/signinum-cuda-runtime/build.rs deleted file mode 100644 index 0761dfb0..00000000 --- a/crates/signinum-cuda-runtime/build.rs +++ /dev/null @@ -1,37 +0,0 @@ -use std::env; -use std::fs; -use std::path::PathBuf; -use std::process::Command; - -fn main() { - println!("cargo:rerun-if-changed=src/j2k_encode_kernels.cu"); - println!("cargo:rerun-if-changed=src/j2k_encode_kernels.ptx"); - println!("cargo:rerun-if-env-changed=NVCC"); - - let out_dir = PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR is set by cargo")); - let ptx = out_dir.join("j2k_encode_kernels.ptx"); - let source = PathBuf::from("src/j2k_encode_kernels.cu"); - let nvcc = env::var_os("NVCC").unwrap_or_else(|| "nvcc".into()); - - let compiled = Command::new(&nvcc) - .args(["--ptx", "-O3", "--std=c++14"]) - .arg(&source) - .arg("-o") - .arg(&ptx) - .status() - .is_ok_and(|status| status.success()); - - if compiled { - let mut bytes = fs::read(&ptx).expect("read generated CUDA PTX"); - if bytes.last().copied() != Some(0) { - bytes.push(0); - fs::write(&ptx, bytes).expect("NUL-terminate generated CUDA PTX"); - } - } else { - let mut bytes = fs::read("src/j2k_encode_kernels.ptx").expect("read fallback CUDA PTX"); - if bytes.last().copied() != Some(0) { - bytes.push(0); - } - fs::write(&ptx, bytes).expect("write fallback CUDA PTX"); - } -} diff --git a/crates/signinum-cuda-runtime/src/j2k_encode_kernels.cu b/crates/signinum-cuda-runtime/src/j2k_encode_kernels.cu deleted file mode 100644 index 783ec119..00000000 --- a/crates/signinum-cuda-runtime/src/j2k_encode_kernels.cu +++ /dev/null @@ -1,121 +0,0 @@ -#include - -extern "C" __global__ void signinum_j2k_forward_rct( - float *plane0, - float *plane1, - float *plane2, - unsigned long long len -) { - const unsigned long long idx = - static_cast(blockIdx.x) * blockDim.x + threadIdx.x; - if (idx >= len) { - return; - } - - const float r = plane0[idx]; - const float g = plane1[idx]; - const float b = plane2[idx]; - plane0[idx] = floorf((r + 2.0f * g + b) * 0.25f); - plane1[idx] = b - g; - plane2[idx] = r - g; -} - -__device__ float signinum_j2k_fdwt53_predict_row( - const float *src, - unsigned int row_base, - unsigned int width, - unsigned int high_index -) { - const unsigned int odd = high_index * 2u + 1u; - const unsigned int last_even = (width % 2u == 0u) ? width - 2u : width - 1u; - const float left = src[row_base + odd - 1u]; - const float right = (odd + 1u < width) ? src[row_base + odd + 1u] : src[row_base + last_even]; - return src[row_base + odd] - floorf((left + right) * 0.5f); -} - -__device__ float signinum_j2k_fdwt53_predict_col( - const float *src, - unsigned int x, - unsigned int full_width, - unsigned int height, - unsigned int high_index -) { - const unsigned int odd = high_index * 2u + 1u; - const unsigned int last_even = (height % 2u == 0u) ? height - 2u : height - 1u; - const float top = src[(odd - 1u) * full_width + x]; - const float bottom = (odd + 1u < height) - ? src[(odd + 1u) * full_width + x] - : src[last_even * full_width + x]; - return src[odd * full_width + x] - floorf((top + bottom) * 0.5f); -} - -extern "C" __global__ void signinum_j2k_forward_dwt53_horizontal( - const float *src, - float *dst, - unsigned int full_width, - unsigned int current_width, - unsigned int current_height, - unsigned int low_width -) { - const unsigned int x = blockIdx.x * blockDim.x + threadIdx.x; - const unsigned int y = blockIdx.y * blockDim.y + threadIdx.y; - if (x >= current_width || y >= current_height) { - return; - } - - const unsigned int row_base = y * full_width; - if (x < low_width) { - const unsigned int even = x * 2u; - const float left = x > 0u - ? signinum_j2k_fdwt53_predict_row(src, row_base, current_width, x - 1u) - : signinum_j2k_fdwt53_predict_row(src, row_base, current_width, 0u); - const float right = even + 1u < current_width - ? signinum_j2k_fdwt53_predict_row(src, row_base, current_width, x) - : left; - dst[row_base + x] = src[row_base + even] + floorf((left + right) * 0.25f + 0.5f); - return; - } - - dst[row_base + x] = signinum_j2k_fdwt53_predict_row( - src, - row_base, - current_width, - x - low_width - ); -} - -extern "C" __global__ void signinum_j2k_forward_dwt53_vertical( - const float *src, - float *dst, - unsigned int full_width, - unsigned int current_width, - unsigned int current_height, - unsigned int low_height -) { - const unsigned int x = blockIdx.x * blockDim.x + threadIdx.x; - const unsigned int y = blockIdx.y * blockDim.y + threadIdx.y; - if (x >= current_width || y >= current_height) { - return; - } - - if (y < low_height) { - const unsigned int even = y * 2u; - const float top = y > 0u - ? signinum_j2k_fdwt53_predict_col(src, x, full_width, current_height, y - 1u) - : signinum_j2k_fdwt53_predict_col(src, x, full_width, current_height, 0u); - const float bottom = even + 1u < current_height - ? signinum_j2k_fdwt53_predict_col(src, x, full_width, current_height, y) - : top; - dst[y * full_width + x] = - src[even * full_width + x] + floorf((top + bottom) * 0.25f + 0.5f); - return; - } - - dst[y * full_width + x] = signinum_j2k_fdwt53_predict_col( - src, - x, - full_width, - current_height, - y - low_height - ); -} diff --git a/crates/signinum-cuda-runtime/src/kernels.rs b/crates/signinum-cuda-runtime/src/kernels.rs deleted file mode 100644 index 0d9ef800..00000000 --- a/crates/signinum-cuda-runtime/src/kernels.rs +++ /dev/null @@ -1,150 +0,0 @@ -use std::os::raw::c_uint; - -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub(crate) enum CudaKernel { - CopyU8, - J2kForwardRct, - J2kForwardDwt53Horizontal, - J2kForwardDwt53Vertical, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(crate) struct CudaLaunchGeometry { - pub grid: (c_uint, c_uint, c_uint), - pub block: (c_uint, c_uint, c_uint), -} - -impl CudaKernel { - pub(crate) fn ptx(self) -> &'static [u8] { - match self { - Self::CopyU8 => COPY_U8_PTX, - Self::J2kForwardRct - | Self::J2kForwardDwt53Horizontal - | Self::J2kForwardDwt53Vertical => J2K_ENCODE_PTX, - } - } - - pub(crate) fn entrypoint(self) -> &'static [u8] { - match self { - Self::CopyU8 => b"signinum_copy_u8\0", - Self::J2kForwardRct => b"signinum_j2k_forward_rct\0", - Self::J2kForwardDwt53Horizontal => b"signinum_j2k_forward_dwt53_horizontal\0", - Self::J2kForwardDwt53Vertical => b"signinum_j2k_forward_dwt53_vertical\0", - } - } -} - -pub(crate) fn copy_u8_launch_geometry(len: usize) -> Option { - let blocks = c_uint::try_from(len.div_ceil(COPY_U8_THREADS)).ok()?; - Some(CudaLaunchGeometry { - grid: (blocks, 1, 1), - block: (COPY_U8_THREADS_CUDA, 1, 1), - }) -} - -const COPY_U8_THREADS: usize = 256; -const COPY_U8_THREADS_CUDA: c_uint = 256; -const J2K_ENCODE_THREADS_X: c_uint = 16; -const J2K_ENCODE_THREADS_Y: c_uint = 16; -const J2K_ENCODE_PTX: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/j2k_encode_kernels.ptx")); - -pub(crate) fn j2k_forward_rct_launch_geometry(len: usize) -> Option { - let blocks = c_uint::try_from(len.div_ceil(COPY_U8_THREADS)).ok()?; - Some(CudaLaunchGeometry { - grid: (blocks, 1, 1), - block: (COPY_U8_THREADS_CUDA, 1, 1), - }) -} - -pub(crate) fn j2k_dwt53_launch_geometry(width: u32, height: u32) -> Option { - let grid_x = c_uint::try_from(width.div_ceil(J2K_ENCODE_THREADS_X)).ok()?; - let grid_y = c_uint::try_from(height.div_ceil(J2K_ENCODE_THREADS_Y)).ok()?; - Some(CudaLaunchGeometry { - grid: (grid_x, grid_y, 1), - block: (J2K_ENCODE_THREADS_X, J2K_ENCODE_THREADS_Y, 1), - }) -} - -const COPY_U8_PTX: &[u8] = concat!( - r" -.version 7.0 -.target sm_52 -.address_size 64 - -.visible .entry signinum_copy_u8( - .param .u64 dst, - .param .u64 src, - .param .u64 len -) -{ - .reg .pred %p; - .reg .b32 %r<5>; - .reg .b64 %rd<7>; - .reg .b16 %u; - - ld.param.u64 %rd1, [dst]; - ld.param.u64 %rd2, [src]; - ld.param.u64 %rd3, [len]; - mov.u32 %r1, %tid.x; - mov.u32 %r2, %ctaid.x; - mov.u32 %r3, %ntid.x; - mad.lo.s32 %r4, %r2, %r3, %r1; - cvt.u64.u32 %rd4, %r4; - setp.ge.u64 %p, %rd4, %rd3; - @%p bra DONE; - add.u64 %rd5, %rd2, %rd4; - ld.global.u8 %u, [%rd5]; - add.u64 %rd6, %rd1, %rd4; - st.global.u8 [%rd6], %u; -DONE: - ret; -} -", - "\0" -) -.as_bytes(); - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn copy_u8_kernel_metadata_matches_embedded_ptx() { - let ptx = CudaKernel::CopyU8.ptx(); - assert_eq!(ptx.last(), Some(&0)); - let source = std::str::from_utf8(&ptx[..ptx.len() - 1]).expect("ptx utf8"); - assert!(source.contains(".visible .entry signinum_copy_u8(")); - assert_eq!(CudaKernel::CopyU8.entrypoint(), b"signinum_copy_u8\0"); - } - - #[test] - fn j2k_encode_kernel_metadata_matches_generated_ptx() { - assert_eq!(J2K_ENCODE_PTX.last(), Some(&0)); - assert_eq!( - CudaKernel::J2kForwardRct.entrypoint(), - b"signinum_j2k_forward_rct\0" - ); - assert_eq!( - CudaKernel::J2kForwardDwt53Horizontal.entrypoint(), - b"signinum_j2k_forward_dwt53_horizontal\0" - ); - assert_eq!( - CudaKernel::J2kForwardDwt53Vertical.entrypoint(), - b"signinum_j2k_forward_dwt53_vertical\0" - ); - } - - #[test] - fn copy_u8_launch_geometry_rounds_up_to_256_thread_blocks() { - assert_eq!(copy_u8_launch_geometry(1).unwrap().grid, (1, 1, 1)); - assert_eq!(copy_u8_launch_geometry(256).unwrap().grid, (1, 1, 1)); - assert_eq!(copy_u8_launch_geometry(257).unwrap().grid, (2, 1, 1)); - } - - #[test] - fn j2k_dwt53_launch_geometry_uses_16_by_16_thread_blocks() { - let geometry = j2k_dwt53_launch_geometry(17, 33).unwrap(); - assert_eq!(geometry.grid, (2, 3, 1)); - assert_eq!(geometry.block, (16, 16, 1)); - } -} diff --git a/crates/signinum-cuda-runtime/src/lib.rs b/crates/signinum-cuda-runtime/src/lib.rs deleted file mode 100644 index f0b7e5cc..00000000 --- a/crates/signinum-cuda-runtime/src/lib.rs +++ /dev/null @@ -1,1160 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Thin CUDA Driver API runtime used by signinum CUDA adapter crates. - -#![deny(unsafe_op_in_unsafe_fn)] -#![warn(unreachable_pub)] - -mod kernels; -mod nvjpeg; - -use std::{ - collections::HashMap, - ffi::c_void, - os::raw::{c_char, c_int, c_uint}, - sync::{Arc, Mutex}, -}; - -use kernels::{ - copy_u8_launch_geometry, j2k_dwt53_launch_geometry, j2k_forward_rct_launch_geometry, CudaKernel, -}; -use libloading::Library; - -type CuResult = c_int; -type CuDevice = c_int; -type CuContext = *mut c_void; -type CuDevicePtr = u64; -type CuModule = *mut c_void; -type CuFunction = *mut c_void; - -const CUDA_SUCCESS: CuResult = 0; - -type CuInit = unsafe extern "C" fn(c_uint) -> CuResult; -type CuDeviceGetCount = unsafe extern "C" fn(*mut c_int) -> CuResult; -type CuDeviceGet = unsafe extern "C" fn(*mut CuDevice, c_int) -> CuResult; -type CuCtxCreate = unsafe extern "C" fn(*mut CuContext, c_uint, CuDevice) -> CuResult; -type CuCtxDestroy = unsafe extern "C" fn(CuContext) -> CuResult; -type CuCtxSetCurrent = unsafe extern "C" fn(CuContext) -> CuResult; -type CuMemAlloc = unsafe extern "C" fn(*mut CuDevicePtr, usize) -> CuResult; -type CuMemFree = unsafe extern "C" fn(CuDevicePtr) -> CuResult; -type CuMemcpyHtoD = unsafe extern "C" fn(CuDevicePtr, *const c_void, usize) -> CuResult; -type CuMemcpyDtoH = unsafe extern "C" fn(*mut c_void, CuDevicePtr, usize) -> CuResult; -type CuGetErrorName = unsafe extern "C" fn(CuResult, *mut *const c_char) -> CuResult; -type CuModuleLoadData = unsafe extern "C" fn(*mut CuModule, *const c_void) -> CuResult; -type CuModuleUnload = unsafe extern "C" fn(CuModule) -> CuResult; -type CuModuleGetFunction = - unsafe extern "C" fn(*mut CuFunction, CuModule, *const c_char) -> CuResult; -type CuLaunchKernel = unsafe extern "C" fn( - CuFunction, - c_uint, - c_uint, - c_uint, - c_uint, - c_uint, - c_uint, - c_uint, - *mut c_void, - *mut *mut c_void, - *mut *mut c_void, -) -> CuResult; -type CuCtxSynchronize = unsafe extern "C" fn() -> CuResult; - -#[derive(Debug, thiserror::Error)] -pub enum CudaError { - #[error("CUDA driver is unavailable: {message}")] - Unavailable { message: String }, - #[error("CUDA driver call {operation} failed with CUresult {code}{name}")] - Driver { - operation: &'static str, - code: CuResult, - name: String, - }, - #[error("CUDA copy output buffer too small: required {required}, have {have}")] - OutputTooSmall { required: usize, have: usize }, - #[error("CUDA byte length is too large for kernel launch: {len}")] - LengthTooLarge { len: usize }, - #[error("CUDA image allocation size overflow for {width}x{height}x{channels}")] - ImageTooLarge { - width: u32, - height: u32, - channels: usize, - }, - #[error("nvJPEG is unavailable: {message}")] - NvjpegUnavailable { message: String }, - #[error("nvJPEG call {operation} failed with nvjpegStatus_t {code}{name}")] - Nvjpeg { - operation: &'static str, - code: i32, - name: String, - }, - #[error("nvJPEG decoded dimensions mismatch: expected {expected:?}, got {actual:?}")] - NvjpegDimensions { - expected: (u32, u32), - actual: (u32, u32), - }, - #[error("CUDA runtime state lock is poisoned: {message}")] - StatePoisoned { message: String }, -} - -struct Driver { - _library: Library, - cu_init: CuInit, - cu_device_get_count: CuDeviceGetCount, - cu_device_get: CuDeviceGet, - cu_ctx_create: CuCtxCreate, - cu_ctx_destroy: CuCtxDestroy, - cu_ctx_set_current: CuCtxSetCurrent, - cu_mem_alloc: CuMemAlloc, - cu_mem_free: CuMemFree, - cu_memcpy_htod: CuMemcpyHtoD, - cu_memcpy_dtoh: CuMemcpyDtoH, - cu_get_error_name: CuGetErrorName, - cu_module_load_data: CuModuleLoadData, - cu_module_unload: CuModuleUnload, - cu_module_get_function: CuModuleGetFunction, - cu_launch_kernel: CuLaunchKernel, - cu_ctx_synchronize: CuCtxSynchronize, -} - -impl Driver { - fn load() -> Result { - #[cfg(target_os = "linux")] - const LIBRARY_CANDIDATES: &[&str] = &["libcuda.so.1", "libcuda.so"]; - #[cfg(target_os = "windows")] - const LIBRARY_CANDIDATES: &[&str] = &["nvcuda.dll"]; - #[cfg(not(any(target_os = "linux", target_os = "windows")))] - const LIBRARY_CANDIDATES: &[&str] = &[]; - - let mut last_error = None; - for candidate in LIBRARY_CANDIDATES { - // SAFETY: Loading the CUDA driver library is required before symbol - // lookup. The resulting Library is owned by Driver and outlives all - // copied function pointers. - match unsafe { Library::new(candidate) } { - Ok(library) => return Self::from_library(library), - Err(error) => last_error = Some(error.to_string()), - } - } - - Err(CudaError::Unavailable { - message: last_error.unwrap_or_else(|| "unsupported CUDA host platform".to_string()), - }) - } - - fn from_library(library: Library) -> Result { - Ok(Self { - cu_init: load_symbol(&library, b"cuInit\0")?, - cu_device_get_count: load_symbol(&library, b"cuDeviceGetCount\0")?, - cu_device_get: load_symbol(&library, b"cuDeviceGet\0")?, - cu_ctx_create: load_symbol(&library, b"cuCtxCreate_v2\0")?, - cu_ctx_destroy: load_symbol(&library, b"cuCtxDestroy_v2\0")?, - cu_ctx_set_current: load_symbol(&library, b"cuCtxSetCurrent\0")?, - cu_mem_alloc: load_symbol(&library, b"cuMemAlloc_v2\0")?, - cu_mem_free: load_symbol(&library, b"cuMemFree_v2\0")?, - cu_memcpy_htod: load_symbol(&library, b"cuMemcpyHtoD_v2\0")?, - cu_memcpy_dtoh: load_symbol(&library, b"cuMemcpyDtoH_v2\0")?, - cu_get_error_name: load_symbol(&library, b"cuGetErrorName\0")?, - cu_module_load_data: load_symbol(&library, b"cuModuleLoadData\0")?, - cu_module_unload: load_symbol(&library, b"cuModuleUnload\0")?, - cu_module_get_function: load_symbol(&library, b"cuModuleGetFunction\0")?, - cu_launch_kernel: load_symbol(&library, b"cuLaunchKernel\0")?, - cu_ctx_synchronize: load_symbol(&library, b"cuCtxSynchronize\0")?, - _library: library, - }) - } - - fn check(&self, operation: &'static str, result: CuResult) -> Result<(), CudaError> { - if result == CUDA_SUCCESS { - Ok(()) - } else { - Err(CudaError::Driver { - operation, - code: result, - name: self.error_name(result), - }) - } - } - - fn error_name(&self, result: CuResult) -> String { - let mut name = std::ptr::null(); - // SAFETY: cuGetErrorName writes a borrowed static C string pointer for - // a CUDA result code. A failure here is non-critical for diagnostics. - let status = unsafe { (self.cu_get_error_name)(result, &raw mut name) }; - if status == CUDA_SUCCESS && !name.is_null() { - // SAFETY: CUDA returns a NUL-terminated static string on success. - let cstr = unsafe { std::ffi::CStr::from_ptr(name) }; - format!(" ({})", cstr.to_string_lossy()) - } else { - String::new() - } - } -} - -fn load_symbol(library: &Library, name: &'static [u8]) -> Result { - // SAFETY: Symbol names are NUL-terminated CUDA Driver API entry points. The - // symbol value is copied, and Driver keeps the Library alive. - unsafe { library.get::(name) } - .map(|symbol| *symbol) - .map_err(|error| CudaError::Unavailable { - message: format!( - "missing CUDA driver symbol {}: {error}", - String::from_utf8_lossy(name) - ), - }) -} - -// SAFETY: CUDA Driver API handles are process resources guarded by the driver. -// The struct stores copied function pointers and owns the loaded library. -unsafe impl Send for Driver {} -// SAFETY: Driver entry points are immutable function pointers, and mutable CUDA -// state is always addressed through explicit CUDA context calls. -unsafe impl Sync for Driver {} - -struct ContextInner { - driver: Driver, - context: CuContext, - modules: Mutex>, - nvjpeg: Mutex>, -} - -impl ContextInner { - fn set_current(&self) -> Result<(), CudaError> { - // SAFETY: context is created by cuCtxCreate_v2 and remains valid while - // ContextInner is alive. - self.driver.check("cuCtxSetCurrent", unsafe { - (self.driver.cu_ctx_set_current)(self.context) - }) - } - - fn kernel_function(&self, kernel: CudaKernel) -> Result { - self.set_current()?; - let mut modules = self - .modules - .lock() - .map_err(|error| CudaError::StatePoisoned { - message: error.to_string(), - })?; - if let Some(compiled) = modules.get(&kernel) { - return Ok(compiled.function); - } - - let compiled = CompiledKernel::load(self, kernel)?; - let function = compiled.function; - modules.insert(kernel, compiled); - Ok(function) - } -} - -impl Drop for ContextInner { - fn drop(&mut self) { - if !self.context.is_null() { - let _ = self.set_current(); - let nvjpeg = match self.nvjpeg.get_mut() { - Ok(nvjpeg) => nvjpeg, - Err(poisoned) => poisoned.into_inner(), - }; - drop(nvjpeg.take()); - let modules = match self.modules.get_mut() { - Ok(modules) => modules, - Err(poisoned) => poisoned.into_inner(), - }; - for compiled in modules.drain().map(|(_, compiled)| compiled) { - // SAFETY: modules were loaded into this CUDA context. Drop - // cannot surface errors, so cleanup failures are ignored. - let _ = unsafe { (self.driver.cu_module_unload)(compiled.module) }; - } - // SAFETY: context was created by this ContextInner and cached - // modules have already been unloaded. - let _ = unsafe { (self.driver.cu_ctx_destroy)(self.context) }; - } - } -} - -// SAFETY: ContextInner owns an opaque CUDA context handle and synchronizes its -// Rust-side mutable caches with mutexes. -unsafe impl Send for ContextInner {} -// SAFETY: All shared Rust state is mutex-protected, and CUDA operations set the -// current context before touching context-owned resources. -unsafe impl Sync for ContextInner {} - -#[derive(Clone)] -pub struct CudaContext { - inner: Arc, -} - -impl CudaContext { - pub fn system_default() -> Result { - let driver = Driver::load()?; - - // SAFETY: cuInit is the CUDA Driver API process initializer. - driver.check("cuInit", unsafe { (driver.cu_init)(0) })?; - - let mut count = 0; - // SAFETY: CUDA writes one integer device count to the provided pointer. - driver.check("cuDeviceGetCount", unsafe { - (driver.cu_device_get_count)(&raw mut count) - })?; - if count <= 0 { - return Err(CudaError::Unavailable { - message: "no CUDA devices reported by driver".to_string(), - }); - } - - let mut device = 0; - // SAFETY: device 0 is valid when count is greater than zero. - driver.check("cuDeviceGet", unsafe { - (driver.cu_device_get)(&raw mut device, 0) - })?; - - let mut context = std::ptr::null_mut(); - // SAFETY: CUDA writes a newly-created context handle for a valid device. - driver.check("cuCtxCreate_v2", unsafe { - (driver.cu_ctx_create)(&raw mut context, 0, device) - })?; - - Ok(Self { - inner: Arc::new(ContextInner { - driver, - context, - modules: Mutex::new(HashMap::new()), - nvjpeg: Mutex::new(None), - }), - }) - } - - pub fn upload(&self, bytes: &[u8]) -> Result { - self.inner.set_current()?; - - let mut ptr = 0; - let buffer = if bytes.is_empty() { - CudaDeviceBuffer { - context: self.clone(), - ptr, - len: bytes.len(), - } - } else { - // SAFETY: CUDA writes a device pointer for the requested byte size. - self.inner.driver.check("cuMemAlloc_v2", unsafe { - (self.inner.driver.cu_mem_alloc)(&raw mut ptr, bytes.len()) - })?; - - CudaDeviceBuffer { - context: self.clone(), - ptr, - len: bytes.len(), - } - }; - - if !bytes.is_empty() { - // SAFETY: ptr is a valid device allocation of bytes.len(), and the - // host pointer is valid for bytes.len(). - self.inner.driver.check("cuMemcpyHtoD_v2", unsafe { - (self.inner.driver.cu_memcpy_htod)( - ptr, - bytes.as_ptr().cast::(), - bytes.len(), - ) - })?; - } - - Ok(buffer) - } - - pub fn copy_with_kernel(&self, bytes: &[u8]) -> Result { - let staging = self.upload(bytes)?; - let output = self.copy_device_to_device_with_kernel(&staging)?; - let copy_dispatches = usize::from(!bytes.is_empty()); - Ok(CudaKernelOutput { - buffer: output, - execution: CudaExecutionStats { - kernel_dispatches: copy_dispatches, - copy_kernel_dispatches: copy_dispatches, - decode_kernel_dispatches: 0, - hardware_decode: false, - }, - }) - } - - pub fn decode_jpeg_rgb8_with_nvjpeg( - &self, - bytes: &[u8], - dimensions: (u32, u32), - ) -> Result { - self.inner.set_current()?; - let (pitch_bytes, byte_len) = rgb8_layout(dimensions)?; - let output = self.allocate(byte_len)?; - if byte_len == 0 { - return Ok(CudaKernelOutput { - buffer: output, - execution: CudaExecutionStats::default(), - }); - } - - let mut state = self - .inner - .nvjpeg - .lock() - .map_err(|error| CudaError::StatePoisoned { - message: error.to_string(), - })?; - if state.is_none() { - *state = Some(nvjpeg::NvjpegState::new()?); - } - let state = state.as_mut().ok_or_else(|| CudaError::NvjpegUnavailable { - message: "nvJPEG state did not initialize".to_string(), - })?; - state.decode_rgb8(bytes, dimensions, output.device_ptr(), pitch_bytes)?; - - // SAFETY: A CUDA context is current for this ContextInner before nvJPEG - // decode work is submitted; synchronize waits for that context's work. - let status = unsafe { (self.inner.driver.cu_ctx_synchronize)() }; - self.inner.driver.check("cuCtxSynchronize", status)?; - - Ok(CudaKernelOutput { - buffer: output, - execution: CudaExecutionStats { - kernel_dispatches: 1, - copy_kernel_dispatches: 0, - decode_kernel_dispatches: 1, - hardware_decode: true, - }, - }) - } - - pub fn decode_jpeg_rgb8_batch_with_nvjpeg( - &self, - inputs: &[(&[u8], (u32, u32))], - ) -> Result, CudaError> { - self.inner.set_current()?; - if inputs.is_empty() { - return Ok(Vec::new()); - } - - let mut buffers = Vec::with_capacity(inputs.len()); - let mut pointers = Vec::with_capacity(inputs.len()); - let mut pitches = Vec::with_capacity(inputs.len()); - for (_, dimensions) in inputs { - let (pitch_bytes, byte_len) = rgb8_layout(*dimensions)?; - let buffer = self.allocate(byte_len)?; - pointers.push(buffer.device_ptr()); - pitches.push(pitch_bytes); - buffers.push(buffer); - } - - let mut state = nvjpeg::NvjpegState::new_batched()?; - state.decode_rgb8_batch(inputs, &pointers, &pitches)?; - - // SAFETY: nvJPEG batched decode submits work to the current CUDA - // context; synchronize waits for completion before buffers are exposed. - let status = unsafe { (self.inner.driver.cu_ctx_synchronize)() }; - self.inner.driver.check("cuCtxSynchronize", status)?; - - let execution = CudaExecutionStats { - kernel_dispatches: 1, - copy_kernel_dispatches: 0, - decode_kernel_dispatches: 1, - hardware_decode: true, - }; - Ok(buffers - .into_iter() - .map(|buffer| CudaKernelOutput { buffer, execution }) - .collect()) - } - - pub fn j2k_forward_rct( - &self, - plane0: &mut [f32], - plane1: &mut [f32], - plane2: &mut [f32], - ) -> Result { - if plane0.len() != plane1.len() || plane0.len() != plane2.len() { - return Err(CudaError::ImageTooLarge { - width: u32::try_from(plane0.len()).unwrap_or(u32::MAX), - height: 1, - channels: 3, - }); - } - if plane0.is_empty() { - return Ok(CudaExecutionStats::default()); - } - - self.inner.set_current()?; - let buffer0 = self.upload(f32_slice_as_bytes(plane0))?; - let buffer1 = self.upload(f32_slice_as_bytes(plane1))?; - let buffer2 = self.upload(f32_slice_as_bytes(plane2))?; - self.launch_j2k_forward_rct_buffers(&buffer0, &buffer1, &buffer2, plane0.len())?; - buffer0.copy_to_host(f32_slice_as_bytes_mut(plane0))?; - buffer1.copy_to_host(f32_slice_as_bytes_mut(plane1))?; - buffer2.copy_to_host(f32_slice_as_bytes_mut(plane2))?; - - Ok(CudaExecutionStats { - kernel_dispatches: 1, - copy_kernel_dispatches: 0, - decode_kernel_dispatches: 0, - hardware_decode: false, - }) - } - - pub fn j2k_forward_dwt53( - &self, - samples: &[f32], - width: u32, - height: u32, - num_levels: u8, - ) -> Result { - let expected_len = - (width as usize) - .checked_mul(height as usize) - .ok_or(CudaError::ImageTooLarge { - width, - height, - channels: 1, - })?; - if expected_len != samples.len() { - return Err(CudaError::ImageTooLarge { - width, - height, - channels: 1, - }); - } - if samples.is_empty() || num_levels == 0 { - return Ok(CudaDwt53Output { - transformed: samples.to_vec(), - levels: Vec::new(), - ll_width: width, - ll_height: height, - execution: CudaExecutionStats::default(), - }); - } - - self.inner.set_current()?; - let buffer_a = self.upload(f32_slice_as_bytes(samples))?; - let buffer_b = self.allocate(std::mem::size_of_val(samples))?; - let mut current_width = width; - let mut current_height = height; - let mut levels = Vec::new(); - let mut dispatches = 0usize; - let mut active_is_a = true; - - for _ in 0..num_levels { - if current_width < 2 && current_height < 2 { - break; - } - let (level_dispatches, level_shape) = self.launch_j2k_forward_dwt53_level( - &buffer_a, - &buffer_b, - &mut active_is_a, - CudaDwt53LevelPass { - full_width: width, - current_width, - current_height, - }, - )?; - dispatches = dispatches.saturating_add(level_dispatches); - levels.push(level_shape); - current_width = level_shape.low_width; - current_height = level_shape.low_height; - } - - let mut transformed = vec![0f32; samples.len()]; - if active_is_a { - buffer_a.copy_to_host(f32_slice_as_bytes_mut(&mut transformed))?; - } else { - buffer_b.copy_to_host(f32_slice_as_bytes_mut(&mut transformed))?; - } - Ok(CudaDwt53Output { - transformed, - levels, - ll_width: current_width, - ll_height: current_height, - execution: CudaExecutionStats { - kernel_dispatches: dispatches, - copy_kernel_dispatches: 0, - decode_kernel_dispatches: 0, - hardware_decode: false, - }, - }) - } - - fn launch_j2k_forward_dwt53_level( - &self, - buffer_a: &CudaDeviceBuffer, - buffer_b: &CudaDeviceBuffer, - active_is_a: &mut bool, - pass: CudaDwt53LevelPass, - ) -> Result<(usize, CudaDwt53LevelShape), CudaError> { - let low_width = pass.current_width.div_ceil(2); - let low_height = pass.current_height.div_ceil(2); - let mut dispatches = 0usize; - - if pass.current_height >= 2 { - let (input, output) = active_dwt53_buffers(buffer_a, buffer_b, *active_is_a); - self.launch_j2k_forward_dwt53_pass( - CudaKernel::J2kForwardDwt53Vertical, - input, - output, - CudaDwt53Pass { - full_width: pass.full_width, - current_width: pass.current_width, - current_height: pass.current_height, - low_extent: low_height, - }, - )?; - *active_is_a = !*active_is_a; - dispatches = dispatches.saturating_add(1); - } - - if pass.current_width >= 2 { - let (input, output) = active_dwt53_buffers(buffer_a, buffer_b, *active_is_a); - self.launch_j2k_forward_dwt53_pass( - CudaKernel::J2kForwardDwt53Horizontal, - input, - output, - CudaDwt53Pass { - full_width: pass.full_width, - current_width: pass.current_width, - current_height: pass.current_height, - low_extent: low_width, - }, - )?; - *active_is_a = !*active_is_a; - dispatches = dispatches.saturating_add(1); - } - - Ok(( - dispatches, - CudaDwt53LevelShape { - width: pass.current_width, - height: pass.current_height, - low_width, - low_height, - high_width: pass.current_width / 2, - high_height: pass.current_height / 2, - }, - )) - } - - fn launch_j2k_forward_rct_buffers( - &self, - plane0: &CudaDeviceBuffer, - plane1: &CudaDeviceBuffer, - plane2: &CudaDeviceBuffer, - len: usize, - ) -> Result<(), CudaError> { - let function = self.inner.kernel_function(CudaKernel::J2kForwardRct)?; - let mut plane0_ptr = plane0.device_ptr(); - let mut plane1_ptr = plane1.device_ptr(); - let mut plane2_ptr = plane2.device_ptr(); - let mut len_u64 = u64::try_from(len).map_err(|_| CudaError::LengthTooLarge { len })?; - let mut params = [ - (&raw mut plane0_ptr).cast::(), - (&raw mut plane1_ptr).cast::(), - (&raw mut plane2_ptr).cast::(), - (&raw mut len_u64).cast::(), - ]; - let geometry = - j2k_forward_rct_launch_geometry(len).ok_or(CudaError::LengthTooLarge { len })?; - - self.launch_kernel(function, geometry, &mut params) - } - - fn launch_j2k_forward_dwt53_pass( - &self, - kernel: CudaKernel, - input: &CudaDeviceBuffer, - output: &CudaDeviceBuffer, - pass: CudaDwt53Pass, - ) -> Result<(), CudaError> { - let function = self.inner.kernel_function(kernel)?; - let mut input_ptr = input.device_ptr(); - let mut output_ptr = output.device_ptr(); - let mut full_width = pass.full_width; - let mut current_width = pass.current_width; - let mut current_height = pass.current_height; - let mut low_extent = pass.low_extent; - let mut params = [ - (&raw mut input_ptr).cast::(), - (&raw mut output_ptr).cast::(), - (&raw mut full_width).cast::(), - (&raw mut current_width).cast::(), - (&raw mut current_height).cast::(), - (&raw mut low_extent).cast::(), - ]; - let geometry = j2k_dwt53_launch_geometry(current_width, current_height).ok_or( - CudaError::ImageTooLarge { - width: pass.current_width, - height: pass.current_height, - channels: 1, - }, - )?; - self.launch_kernel(function, geometry, &mut params) - } - - fn launch_kernel( - &self, - function: CuFunction, - geometry: kernels::CudaLaunchGeometry, - params: &mut [*mut c_void], - ) -> Result<(), CudaError> { - // SAFETY: `function` was loaded from a live module in this context, and - // `params` contains kernel argument pointers valid for the launch call. - let launch_status = unsafe { - (self.inner.driver.cu_launch_kernel)( - function, - geometry.grid.0, - geometry.grid.1, - geometry.grid.2, - geometry.block.0, - geometry.block.1, - geometry.block.2, - 0, - std::ptr::null_mut(), - params.as_mut_ptr(), - std::ptr::null_mut(), - ) - }; - self.inner.driver.check("cuLaunchKernel", launch_status)?; - // SAFETY: The kernel was launched on the current context; synchronize - // waits for completion before callers inspect outputs. - let sync_status = unsafe { (self.inner.driver.cu_ctx_synchronize)() }; - self.inner.driver.check("cuCtxSynchronize", sync_status) - } - - pub fn copy_device_to_device_with_kernel( - &self, - src: &CudaDeviceBuffer, - ) -> Result { - self.inner.set_current()?; - let dst = self.allocate(src.byte_len())?; - if src.byte_len() == 0 { - return Ok(dst); - } - - let function = self.inner.kernel_function(CudaKernel::CopyU8)?; - let mut dst_ptr = dst.device_ptr(); - let mut src_ptr = src.device_ptr(); - let mut len = u64::try_from(src.byte_len()).map_err(|_| CudaError::LengthTooLarge { - len: src.byte_len(), - })?; - let mut params = [ - (&raw mut dst_ptr).cast::(), - (&raw mut src_ptr).cast::(), - (&raw mut len).cast::(), - ]; - let geometry = - copy_u8_launch_geometry(src.byte_len()).ok_or(CudaError::LengthTooLarge { - len: src.byte_len(), - })?; - - self.launch_kernel(function, geometry, &mut params)?; - - Ok(dst) - } - - pub fn allocate(&self, len: usize) -> Result { - self.inner.set_current()?; - let mut ptr = 0; - if len != 0 { - // SAFETY: CUDA writes a device pointer for the requested byte size. - self.inner.driver.check("cuMemAlloc_v2", unsafe { - (self.inner.driver.cu_mem_alloc)(&raw mut ptr, len) - })?; - } - Ok(CudaDeviceBuffer { - context: self.clone(), - ptr, - len, - }) - } -} - -impl std::fmt::Debug for CudaContext { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("CudaContext").finish_non_exhaustive() - } -} - -#[derive(Debug)] -pub struct CudaDeviceBuffer { - context: CudaContext, - ptr: CuDevicePtr, - len: usize, -} - -#[derive(Debug)] -pub struct CudaKernelOutput { - buffer: CudaDeviceBuffer, - execution: CudaExecutionStats, -} - -#[derive(Debug)] -pub struct CudaDwt53Output { - transformed: Vec, - levels: Vec, - ll_width: u32, - ll_height: u32, - execution: CudaExecutionStats, -} - -impl CudaDwt53Output { - pub fn transformed(&self) -> &[f32] { - &self.transformed - } - - pub fn levels(&self) -> &[CudaDwt53LevelShape] { - &self.levels - } - - pub fn ll_dimensions(&self) -> (u32, u32) { - (self.ll_width, self.ll_height) - } - - pub fn execution(&self) -> CudaExecutionStats { - self.execution - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct CudaDwt53LevelShape { - pub width: u32, - pub height: u32, - pub low_width: u32, - pub low_height: u32, - pub high_width: u32, - pub high_height: u32, -} - -#[derive(Clone, Copy, Debug)] -struct CudaDwt53Pass { - full_width: u32, - current_width: u32, - current_height: u32, - low_extent: u32, -} - -#[derive(Clone, Copy, Debug)] -struct CudaDwt53LevelPass { - full_width: u32, - current_width: u32, - current_height: u32, -} - -fn active_dwt53_buffers<'a>( - buffer_a: &'a CudaDeviceBuffer, - buffer_b: &'a CudaDeviceBuffer, - active_is_a: bool, -) -> (&'a CudaDeviceBuffer, &'a CudaDeviceBuffer) { - if active_is_a { - (buffer_a, buffer_b) - } else { - (buffer_b, buffer_a) - } -} - -impl CudaKernelOutput { - pub fn into_parts(self) -> (CudaDeviceBuffer, CudaExecutionStats) { - (self.buffer, self.execution) - } -} - -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub struct CudaExecutionStats { - kernel_dispatches: usize, - copy_kernel_dispatches: usize, - decode_kernel_dispatches: usize, - hardware_decode: bool, -} - -impl CudaExecutionStats { - pub fn kernel_dispatches(self) -> usize { - self.kernel_dispatches - } - - pub fn copy_kernel_dispatches(self) -> usize { - self.copy_kernel_dispatches - } - - pub fn decode_kernel_dispatches(self) -> usize { - self.decode_kernel_dispatches - } - - pub fn used_hardware_decode(self) -> bool { - self.hardware_decode - } -} - -#[derive(Debug)] -struct CompiledKernel { - module: CuModule, - function: CuFunction, -} - -impl CompiledKernel { - fn load(context: &ContextInner, kernel: CudaKernel) -> Result { - context.set_current()?; - let mut module = std::ptr::null_mut(); - // SAFETY: image is a NUL-terminated PTX string. CUDA copies or parses - // it during module load, and the context cache unloads the module on - // context drop. - context.driver.check("cuModuleLoadData", unsafe { - (context.driver.cu_module_load_data)( - &raw mut module, - kernel.ptx().as_ptr().cast::(), - ) - })?; - let mut function = std::ptr::null_mut(); - // SAFETY: name is a NUL-terminated kernel symbol in this module. - context.driver.check("cuModuleGetFunction", unsafe { - (context.driver.cu_module_get_function)( - &raw mut function, - module, - kernel.entrypoint().as_ptr().cast::(), - ) - })?; - Ok(Self { module, function }) - } -} - -// SAFETY: CompiledKernel stores opaque CUDA module/function handles. Lifetime -// and unloading are coordinated by ContextInner's module cache mutex. -unsafe impl Send for CompiledKernel {} - -impl CudaDeviceBuffer { - pub fn device_ptr(&self) -> u64 { - self.ptr - } - - pub fn byte_len(&self) -> usize { - self.len - } - - pub fn copy_to_host(&self, out: &mut [u8]) -> Result<(), CudaError> { - if out.len() < self.len { - return Err(CudaError::OutputTooSmall { - required: self.len, - have: out.len(), - }); - } - if self.len == 0 { - return Ok(()); - } - - self.context.inner.set_current()?; - // SAFETY: ptr is a live device allocation of self.len bytes, and out is - // valid for at least self.len bytes. - self.context.inner.driver.check("cuMemcpyDtoH_v2", unsafe { - (self.context.inner.driver.cu_memcpy_dtoh)( - out.as_mut_ptr().cast::(), - self.ptr, - self.len, - ) - }) - } -} - -impl Drop for CudaDeviceBuffer { - fn drop(&mut self) { - if self.ptr != 0 { - let _ = self.context.inner.set_current(); - // SAFETY: ptr was allocated by this CUDA context. Drop cannot - // surface errors, so failures are ignored during cleanup. - let _ = unsafe { (self.context.inner.driver.cu_mem_free)(self.ptr) }; - } - } -} - -fn f32_slice_as_bytes(samples: &[f32]) -> &[u8] { - // SAFETY: f32 has no invalid bit patterns, and the output byte slice is - // read-only with the same lifetime as the input samples. - unsafe { - std::slice::from_raw_parts( - samples.as_ptr().cast::(), - std::mem::size_of_val(samples), - ) - } -} - -fn f32_slice_as_bytes_mut(samples: &mut [f32]) -> &mut [u8] { - // SAFETY: the returned byte slice covers exactly the same initialized f32 - // storage and is used only for CUDA copies into the existing allocation. - unsafe { - std::slice::from_raw_parts_mut( - samples.as_mut_ptr().cast::(), - std::mem::size_of_val(samples), - ) - } -} - -fn rgb8_layout(dimensions: (u32, u32)) -> Result<(usize, usize), CudaError> { - let row_bytes = dimensions - .0 - .try_into() - .ok() - .and_then(|width: usize| width.checked_mul(3)) - .ok_or(CudaError::ImageTooLarge { - width: dimensions.0, - height: dimensions.1, - channels: 3, - })?; - let byte_len = - row_bytes - .checked_mul(dimensions.1 as usize) - .ok_or(CudaError::ImageTooLarge { - width: dimensions.0, - height: dimensions.1, - channels: 3, - })?; - Ok((row_bytes, byte_len)) -} - -#[cfg(test)] -mod tests { - use super::CudaContext; - - fn cuda_runtime_required() -> bool { - std::env::var_os("SIGNINUM_REQUIRE_CUDA_RUNTIME").is_some() - } - - #[test] - fn j2k_forward_rct_matches_cpu_when_runtime_required() { - if !cuda_runtime_required() { - return; - } - - let mut plane0 = vec![10.0, 1.0, 0.0, 255.0, 128.0]; - let mut plane1 = vec![20.0, 2.0, 255.0, 0.0, 64.0]; - let mut plane2 = vec![30.0, 3.0, 128.0, 127.0, 32.0]; - let mut expected0 = plane0.clone(); - let mut expected1 = plane1.clone(); - let mut expected2 = plane2.clone(); - for ((r, g), b) in expected0 - .iter_mut() - .zip(expected1.iter_mut()) - .zip(expected2.iter_mut()) - { - let r0 = *r; - let g0 = *g; - let b0 = *b; - *r = ((r0 + 2.0_f32 * g0 + b0) * 0.25_f32).floor(); - *g = b0 - g0; - *b = r0 - g0; - } - - let context = CudaContext::system_default().expect("CUDA context"); - let execution = context - .j2k_forward_rct(&mut plane0, &mut plane1, &mut plane2) - .expect("CUDA forward RCT"); - - assert_eq!(execution.kernel_dispatches(), 1); - assert_eq!(plane0, expected0); - assert_eq!(plane1, expected1); - assert_eq!(plane2, expected2); - } - - #[test] - fn j2k_forward_dwt53_matches_cpu_when_runtime_required() { - if !cuda_runtime_required() { - return; - } - - let width = 5usize; - let height = 3usize; - let samples: Vec = (0..width * height) - .map(|value| { - let sample = u16::try_from((value * 7 + 3) % 19).expect("sample fits in u16"); - f32::from(sample) - }) - .collect(); - let expected = cpu_forward_dwt53_buffer(&samples, width, height, 1); - - let context = CudaContext::system_default().expect("CUDA context"); - let output = context - .j2k_forward_dwt53( - &samples, - u32::try_from(width).expect("width fits in u32"), - u32::try_from(height).expect("height fits in u32"), - 1, - ) - .expect("CUDA forward 5/3 DWT"); - - assert_eq!(output.execution().kernel_dispatches(), 2); - assert_eq!(output.transformed(), expected.as_slice()); - assert_eq!(output.ll_dimensions(), (3, 2)); - } - - fn cpu_forward_dwt53_buffer( - samples: &[f32], - width: usize, - height: usize, - levels: u8, - ) -> Vec { - let mut buffer = samples.to_vec(); - let mut current_width = width; - let mut current_height = height; - - for _ in 0..levels { - if current_width < 2 && current_height < 2 { - break; - } - if current_height >= 2 { - let low_height = current_height.div_ceil(2); - let mut col = vec![0.0; current_height]; - for x in 0..current_width { - for y in 0..current_height { - col[y] = buffer[y * width + x]; - } - forward_lift_53(&mut col); - for y in 0..low_height { - buffer[y * width + x] = col[y * 2]; - } - for y in 0..current_height / 2 { - buffer[(low_height + y) * width + x] = col[y * 2 + 1]; - } - } - } - if current_width >= 2 { - let mut row = vec![0.0; current_width]; - for y in 0..current_height { - let row_start = y * width; - row.copy_from_slice(&buffer[row_start..row_start + current_width]); - forward_lift_53(&mut row); - let low_width = current_width.div_ceil(2); - for x in 0..low_width { - buffer[row_start + x] = row[x * 2]; - } - for x in 0..current_width / 2 { - buffer[row_start + low_width + x] = row[x * 2 + 1]; - } - } - } - current_width = current_width.div_ceil(2); - current_height = current_height.div_ceil(2); - } - - buffer - } - - fn forward_lift_53(data: &mut [f32]) { - let n = data.len(); - if n < 2 { - return; - } - - let last_even = if n.is_multiple_of(2) { n - 2 } else { n - 1 }; - for i in (1..n).step_by(2) { - let left = data[i - 1]; - let right = if i + 1 < n { - data[i + 1] - } else { - data[last_even] - }; - data[i] -= ((left + right) * 0.5).floor(); - } - - for i in (0..n).step_by(2) { - let left = if i > 0 { data[i - 1] } else { data[1] }; - let right = if i + 1 < n { data[i + 1] } else { left }; - data[i] += ((left + right) * 0.25 + 0.5).floor(); - } - } -} diff --git a/crates/signinum-cuda-runtime/src/nvjpeg.rs b/crates/signinum-cuda-runtime/src/nvjpeg.rs deleted file mode 100644 index 603b69ed..00000000 --- a/crates/signinum-cuda-runtime/src/nvjpeg.rs +++ /dev/null @@ -1,419 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - ffi::c_void, - os::raw::c_int, - sync::{Arc, OnceLock}, -}; - -use libloading::Library; - -use crate::CudaError; - -type NvjpegStatus = c_int; -type NvjpegHandle = *mut c_void; -type NvjpegJpegState = *mut c_void; -type CudaStream = *mut c_void; - -const NVJPEG_STATUS_SUCCESS: NvjpegStatus = 0; -const NVJPEG_OUTPUT_RGBI: c_int = 5; -const NVJPEG_MAX_COMPONENT: usize = 4; -const NVJPEG_BACKEND_GPU_HYBRID: c_int = 2; - -type NvjpegCreateEx = - unsafe extern "C" fn(c_int, *mut c_void, *mut c_void, u32, *mut NvjpegHandle) -> NvjpegStatus; -type NvjpegCreateSimple = unsafe extern "C" fn(*mut NvjpegHandle) -> NvjpegStatus; -type NvjpegDestroy = unsafe extern "C" fn(NvjpegHandle) -> NvjpegStatus; -type NvjpegJpegStateCreate = - unsafe extern "C" fn(NvjpegHandle, *mut NvjpegJpegState) -> NvjpegStatus; -type NvjpegJpegStateDestroy = unsafe extern "C" fn(NvjpegJpegState) -> NvjpegStatus; -type NvjpegGetImageInfo = unsafe extern "C" fn( - NvjpegHandle, - *const u8, - usize, - *mut c_int, - *mut c_int, - *mut c_int, - *mut c_int, -) -> NvjpegStatus; -type NvjpegDecode = unsafe extern "C" fn( - NvjpegHandle, - NvjpegJpegState, - *const u8, - usize, - c_int, - *mut NvjpegImage, - CudaStream, -) -> NvjpegStatus; -type NvjpegDecodeBatchedInitialize = - unsafe extern "C" fn(NvjpegHandle, NvjpegJpegState, c_int, c_int, c_int) -> NvjpegStatus; -type NvjpegDecodeBatched = unsafe extern "C" fn( - NvjpegHandle, - NvjpegJpegState, - *const *const u8, - *const usize, - *mut NvjpegImage, - CudaStream, -) -> NvjpegStatus; - -#[repr(C)] -struct NvjpegImage { - channel: [*mut u8; NVJPEG_MAX_COMPONENT], - pitch: [usize; NVJPEG_MAX_COMPONENT], -} - -pub(crate) struct NvjpegLibrary { - _library: Library, - create_ex: Option, - create_simple: NvjpegCreateSimple, - destroy: NvjpegDestroy, - jpeg_state_create: NvjpegJpegStateCreate, - jpeg_state_destroy: NvjpegJpegStateDestroy, - get_image_info: NvjpegGetImageInfo, - decode: NvjpegDecode, - decode_batched_initialize: NvjpegDecodeBatchedInitialize, - decode_batched: NvjpegDecodeBatched, -} - -impl NvjpegLibrary { - fn load() -> Result { - #[cfg(target_os = "linux")] - const LIBRARY_CANDIDATES: &[&str] = &[ - "libnvjpeg.so.13", - "libnvjpeg.so.12", - "libnvjpeg.so.11", - "libnvjpeg.so", - ]; - #[cfg(target_os = "windows")] - const LIBRARY_CANDIDATES: &[&str] = &[ - "nvjpeg64_130.dll", - "nvjpeg64_120.dll", - "nvjpeg64_110.dll", - "nvjpeg64.dll", - ]; - #[cfg(not(any(target_os = "linux", target_os = "windows")))] - const LIBRARY_CANDIDATES: &[&str] = &[]; - - let mut last_error = None; - for candidate in LIBRARY_CANDIDATES { - // SAFETY: Loading nvJPEG is required for runtime-only CUDA JPEG - // decode support. NvjpegLibrary owns the handle for all copied - // function pointers. - match unsafe { Library::new(candidate) } { - Ok(library) => return Self::from_library(library), - Err(error) => last_error = Some(error.to_string()), - } - } - - Err(CudaError::NvjpegUnavailable { - message: last_error.unwrap_or_else(|| "unsupported nvJPEG host platform".to_string()), - }) - } - - fn from_library(library: Library) -> Result { - Ok(Self { - create_ex: load_optional_nvjpeg_symbol(&library, b"nvjpegCreateEx\0"), - create_simple: load_nvjpeg_symbol(&library, b"nvjpegCreateSimple\0")?, - destroy: load_nvjpeg_symbol(&library, b"nvjpegDestroy\0")?, - jpeg_state_create: load_nvjpeg_symbol(&library, b"nvjpegJpegStateCreate\0")?, - jpeg_state_destroy: load_nvjpeg_symbol(&library, b"nvjpegJpegStateDestroy\0")?, - get_image_info: load_nvjpeg_symbol(&library, b"nvjpegGetImageInfo\0")?, - decode: load_nvjpeg_symbol(&library, b"nvjpegDecode\0")?, - decode_batched_initialize: load_nvjpeg_symbol( - &library, - b"nvjpegDecodeBatchedInitialize\0", - )?, - decode_batched: load_nvjpeg_symbol(&library, b"nvjpegDecodeBatched\0")?, - _library: library, - }) - } - - fn check(operation: &'static str, status: NvjpegStatus) -> Result<(), CudaError> { - if status == NVJPEG_STATUS_SUCCESS { - Ok(()) - } else { - Err(CudaError::Nvjpeg { - operation, - code: status, - name: nvjpeg_status_name(status), - }) - } - } -} - -// SAFETY: NvjpegLibrary stores immutable copied function pointers and owns the -// loaded shared library for at least as long as those pointers are used. -unsafe impl Send for NvjpegLibrary {} -// SAFETY: The library wrapper has no interior mutable Rust state; nvJPEG state -// is kept in NvjpegState instances. -unsafe impl Sync for NvjpegLibrary {} - -pub(crate) struct NvjpegState { - library: Arc, - handle: NvjpegHandle, - state: NvjpegJpegState, -} - -impl NvjpegState { - pub(crate) fn new() -> Result { - Self::new_with_backend(None) - } - - pub(crate) fn new_batched() -> Result { - Self::new_with_backend(Some(NVJPEG_BACKEND_GPU_HYBRID)) - } - - fn new_with_backend(backend: Option) -> Result { - let library = shared_library()?; - let mut handle = std::ptr::null_mut(); - if let Some(backend) = backend { - let Some(create_ex) = library.create_ex else { - return Err(CudaError::NvjpegUnavailable { - message: "nvJPEG library does not export nvjpegCreateEx".to_string(), - }); - }; - // SAFETY: `handle` is a valid out-pointer, and null allocator - // arguments request nvJPEG defaults. - let status = unsafe { - (create_ex)( - backend, - std::ptr::null_mut(), - std::ptr::null_mut(), - 0, - &raw mut handle, - ) - }; - NvjpegLibrary::check("nvjpegCreateEx", status)?; - } else { - // SAFETY: `handle` is a valid out-pointer for nvJPEG to fill. - let status = unsafe { (library.create_simple)(&raw mut handle) }; - NvjpegLibrary::check("nvjpegCreateSimple", status)?; - } - if handle.is_null() { - return Err(CudaError::NvjpegUnavailable { - message: "nvJPEG handle creation returned a null handle".to_string(), - }); - } - let mut state = std::ptr::null_mut(); - // SAFETY: `handle` is a live nvJPEG handle and `state` is a valid - // out-pointer for the associated JPEG state. - let state_status = unsafe { (library.jpeg_state_create)(handle, &raw mut state) }; - if let Err(error) = NvjpegLibrary::check("nvjpegJpegStateCreate", state_status) { - // SAFETY: handle was created above and state creation failed before - // ownership could be moved into NvjpegState. - let _ = unsafe { (library.destroy)(handle) }; - return Err(error); - } - if state.is_null() { - // SAFETY: handle was created above and no state was returned. - let _ = unsafe { (library.destroy)(handle) }; - return Err(CudaError::NvjpegUnavailable { - message: "nvjpegJpegStateCreate returned a null state".to_string(), - }); - } - Ok(Self { - library, - handle, - state, - }) - } - - pub(crate) fn decode_rgb8( - &mut self, - bytes: &[u8], - dimensions: (u32, u32), - device_ptr: u64, - pitch_bytes: usize, - ) -> Result<(), CudaError> { - self.validate_dimensions(bytes, dimensions)?; - let mut image = rgb8_destination(device_ptr, pitch_bytes)?; - - // SAFETY: `bytes` is live for the duration of the call, `image` points - // at caller-owned device memory, and stream null selects the default. - let status = unsafe { - (self.library.decode)( - self.handle, - self.state, - bytes.as_ptr(), - bytes.len(), - NVJPEG_OUTPUT_RGBI, - &raw mut image, - std::ptr::null_mut(), - ) - }; - NvjpegLibrary::check("nvjpegDecode", status) - } - - pub(crate) fn decode_rgb8_batch( - &mut self, - inputs: &[(&[u8], (u32, u32))], - outputs: &[u64], - pitches: &[usize], - ) -> Result<(), CudaError> { - if inputs.len() != outputs.len() || inputs.len() != pitches.len() { - return Err(CudaError::NvjpegUnavailable { - message: "nvJPEG batch inputs and outputs have different lengths".to_string(), - }); - } - let batch_size = - c_int::try_from(inputs.len()).map_err(|_| CudaError::NvjpegUnavailable { - message: format!("nvJPEG batch size {} does not fit c_int", inputs.len()), - })?; - if batch_size == 0 { - return Ok(()); - } - - for (bytes, dimensions) in inputs { - self.validate_dimensions(bytes, *dimensions)?; - } - - let mut data = Vec::with_capacity(inputs.len()); - let mut lengths = Vec::with_capacity(inputs.len()); - let mut destinations = Vec::with_capacity(inputs.len()); - for ((bytes, _dimensions), (device_ptr, pitch_bytes)) in - inputs.iter().zip(outputs.iter().zip(pitches.iter())) - { - data.push(bytes.as_ptr()); - lengths.push(bytes.len()); - destinations.push(rgb8_destination(*device_ptr, *pitch_bytes)?); - } - - // SAFETY: `self.handle`/`self.state` are live nvJPEG objects; the batch - // size was checked to fit c_int and match the prepared arrays. - let init_status = unsafe { - (self.library.decode_batched_initialize)( - self.handle, - self.state, - batch_size, - 1, - NVJPEG_OUTPUT_RGBI, - ) - }; - NvjpegLibrary::check("nvjpegDecodeBatchedInitialize", init_status)?; - // SAFETY: data/length/destination arrays have `batch_size` entries and - // remain live for the synchronous nvJPEG API call. - let decode_status = unsafe { - (self.library.decode_batched)( - self.handle, - self.state, - data.as_ptr(), - lengths.as_ptr(), - destinations.as_mut_ptr(), - std::ptr::null_mut(), - ) - }; - NvjpegLibrary::check("nvjpegDecodeBatched", decode_status) - } - - fn validate_dimensions(&self, bytes: &[u8], dimensions: (u32, u32)) -> Result<(), CudaError> { - let mut components = 0; - let mut subsampling = 0; - let mut widths = [0; NVJPEG_MAX_COMPONENT]; - let mut heights = [0; NVJPEG_MAX_COMPONENT]; - // SAFETY: `bytes` is live for the duration of the call and all output - // pointers refer to local storage sized for nvJPEG component metadata. - let status = unsafe { - (self.library.get_image_info)( - self.handle, - bytes.as_ptr(), - bytes.len(), - &raw mut components, - &raw mut subsampling, - widths.as_mut_ptr(), - heights.as_mut_ptr(), - ) - }; - NvjpegLibrary::check("nvjpegGetImageInfo", status)?; - let actual = ( - u32::try_from(widths[0]).unwrap_or(0), - u32::try_from(heights[0]).unwrap_or(0), - ); - if actual != dimensions { - return Err(CudaError::NvjpegDimensions { - expected: dimensions, - actual, - }); - } - Ok(()) - } -} - -impl Drop for NvjpegState { - fn drop(&mut self) { - if !self.state.is_null() { - // SAFETY: state was created by this nvJPEG handle. Drop cannot - // report failures, so cleanup errors are ignored. - let _ = unsafe { (self.library.jpeg_state_destroy)(self.state) }; - } - if !self.handle.is_null() { - // SAFETY: handle was created by nvJPEG and outlives the JPEG state - // destroyed above. - let _ = unsafe { (self.library.destroy)(self.handle) }; - } - } -} - -// SAFETY: NvjpegState owns opaque nvJPEG handles and is only used through -// mutable methods or external synchronization by CudaContext. -unsafe impl Send for NvjpegState {} - -fn load_nvjpeg_symbol(library: &Library, name: &'static [u8]) -> Result { - // SAFETY: Symbol names are NUL-terminated nvJPEG entry points. The symbol - // value is copied, and NvjpegLibrary keeps the Library alive. - unsafe { library.get::(name) } - .map(|symbol| *symbol) - .map_err(|error| CudaError::NvjpegUnavailable { - message: format!( - "missing nvJPEG symbol {}: {error}", - String::from_utf8_lossy(name) - ), - }) -} - -fn load_optional_nvjpeg_symbol(library: &Library, name: &'static [u8]) -> Option { - // SAFETY: Symbol names are NUL-terminated nvJPEG entry points. The symbol - // value is copied, and NvjpegLibrary keeps the Library alive. - unsafe { library.get::(name) }.map(|symbol| *symbol).ok() -} - -fn rgb8_destination(device_ptr: u64, pitch_bytes: usize) -> Result { - let mut image = NvjpegImage { - channel: [std::ptr::null_mut(); NVJPEG_MAX_COMPONENT], - pitch: [0; NVJPEG_MAX_COMPONENT], - }; - image.channel[0] = usize::try_from(device_ptr).map_err(|_| CudaError::NvjpegUnavailable { - message: "CUDA device pointer does not fit host pointer width".to_string(), - })? as *mut u8; - image.pitch[0] = pitch_bytes; - Ok(image) -} - -fn shared_library() -> Result, CudaError> { - static LIBRARY: OnceLock, String>> = OnceLock::new(); - match LIBRARY.get_or_init(|| { - NvjpegLibrary::load() - .map(Arc::new) - .map_err(|error| error.to_string()) - }) { - Ok(library) => Ok(library.clone()), - Err(message) => Err(CudaError::NvjpegUnavailable { - message: message.clone(), - }), - } -} - -fn nvjpeg_status_name(status: NvjpegStatus) -> String { - let name = match status { - 1 => "NVJPEG_STATUS_NOT_INITIALIZED", - 2 => "NVJPEG_STATUS_INVALID_PARAMETER", - 3 => "NVJPEG_STATUS_BAD_JPEG", - 4 => "NVJPEG_STATUS_JPEG_NOT_SUPPORTED", - 5 => "NVJPEG_STATUS_ALLOCATOR_FAILURE", - 6 => "NVJPEG_STATUS_EXECUTION_FAILED", - 7 => "NVJPEG_STATUS_ARCH_MISMATCH", - 8 => "NVJPEG_STATUS_INTERNAL_ERROR", - 9 => "NVJPEG_STATUS_IMPLEMENTATION_NOT_SUPPORTED", - _ => return String::new(), - }; - format!(" ({name})") -} diff --git a/crates/signinum-j2k-compare/build.rs b/crates/signinum-j2k-compare/build.rs deleted file mode 100644 index 30fe5cd2..00000000 --- a/crates/signinum-j2k-compare/build.rs +++ /dev/null @@ -1,126 +0,0 @@ -use std::path::{Path, PathBuf}; - -fn main() { - println!("cargo:rustc-check-cfg=cfg(have_grok)"); - println!("cargo:rerun-if-env-changed=SIGNINUM_GROK_ROOT"); - println!("cargo:rerun-if-env-changed=SIGNINUM_GROK_SOURCE"); - println!("cargo:rerun-if-changed=src/grok_shim.c"); - - if let Some(config) = grok_config() { - let staged_lib_dir = stage_grok_runtime(&config.lib_dir) - .unwrap_or_else(|err| panic!("failed to stage Grok runtime libraries: {err}")); - println!("cargo:rustc-cfg=have_grok"); - println!( - "cargo:rustc-link-search=native={}", - staged_lib_dir.display() - ); - println!( - "cargo:rustc-link-search=native={}", - config.lib_dir.display() - ); - println!("cargo:rustc-link-lib=dylib=grokj2k"); - #[cfg(target_os = "macos")] - println!( - "cargo:rustc-link-arg=-Wl,-rpath,{}", - staged_lib_dir.display() - ); - - cc::Build::new() - .file("src/grok_shim.c") - .include(config.source_include) - .include(config.build_include) - .warnings(false) - .compile("signinum_j2k_grok_shim"); - } -} - -fn stage_grok_runtime(lib_dir: &Path) -> Result { - let out_dir = - PathBuf::from(std::env::var_os("OUT_DIR").ok_or_else(|| "OUT_DIR missing".to_string())?); - #[cfg(target_os = "macos")] - { - stage_family( - lib_dir, - &out_dir, - "libgrokj2k.20.3.0.dylib", - "libgrokj2k.1.dylib", - "libgrokj2k.dylib", - )?; - } - #[cfg(not(target_os = "macos"))] - { - let _ = lib_dir; - } - Ok(out_dir) -} - -#[cfg(target_os = "macos")] -fn stage_family( - lib_dir: &Path, - out_dir: &Path, - real_name: &str, - compat_name: &str, - link_name: &str, -) -> Result<(), String> { - let real_src = lib_dir.join(real_name); - if !real_src.exists() { - return Err(format!("missing {real_name} in {}", lib_dir.display())); - } - let real_dst = out_dir.join(real_name); - if real_dst.exists() { - std::fs::remove_file(&real_dst) - .map_err(|err| format!("remove {}: {err}", real_dst.display()))?; - } - std::fs::copy(&real_src, &real_dst).map_err(|err| format!("copy {real_name}: {err}"))?; - symlink_in_dir(real_name, &out_dir.join(compat_name))?; - symlink_in_dir(compat_name, &out_dir.join(link_name))?; - Ok(()) -} - -#[cfg(target_os = "macos")] -fn symlink_in_dir(target_name: &str, dst: &PathBuf) -> Result<(), String> { - if dst.exists() { - std::fs::remove_file(dst).map_err(|err| format!("remove {}: {err}", dst.display()))?; - } - std::os::unix::fs::symlink(target_name, dst) - .map_err(|err| format!("symlink {} -> {target_name}: {err}", dst.display())) -} - -struct GrokConfig { - lib_dir: PathBuf, - source_include: PathBuf, - build_include: PathBuf, -} - -fn grok_config() -> Option { - let source_root = std::env::var_os("SIGNINUM_GROK_SOURCE") - .map_or_else(|| PathBuf::from("/tmp/grok-signinum"), PathBuf::from); - let lib_dir = std::env::var_os("SIGNINUM_GROK_ROOT") - .map_or_else(|| source_root.join("build/bin"), PathBuf::from); - let source_include = source_root.join("src/lib/core"); - let build_include = source_root.join("build/src/lib/core"); - if has_grok_artifacts(&lib_dir, &source_include, &build_include) { - Some(GrokConfig { - lib_dir, - source_include, - build_include, - }) - } else { - None - } -} - -fn has_grok_artifacts(lib_dir: &Path, source_include: &Path, build_include: &Path) -> bool { - let header = source_include.join("grok.h"); - let config_header = build_include.join("grk_config.h"); - if !(header.exists() && config_header.exists()) { - return false; - } - [ - lib_dir.join("libgrokj2k.dylib"), - lib_dir.join("libgrokj2k.so"), - lib_dir.join("grokj2k.lib"), - ] - .into_iter() - .any(|path| path.exists()) -} diff --git a/crates/signinum-j2k-compare/src/lib.rs b/crates/signinum-j2k-compare/src/lib.rs deleted file mode 100644 index b969c615..00000000 --- a/crates/signinum-j2k-compare/src/lib.rs +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! In-process external JPEG 2000 comparators for benches and parity tests. - -pub mod grok; -pub mod openjpeg; diff --git a/crates/signinum-j2k-compare/tests/in_process_parity.rs b/crates/signinum-j2k-compare/tests/in_process_parity.rs deleted file mode 100644 index e1bbbc2b..00000000 --- a/crates/signinum-j2k-compare/tests/in_process_parity.rs +++ /dev/null @@ -1,196 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use signinum_core::{Downscale, PixelFormat, Rect}; -use signinum_j2k::J2kDecoder; -use signinum_test_support::gradient_u8; -use std::{ - fs, - path::{Path, PathBuf}, - process::Command, - sync::{ - atomic::{AtomicUsize, Ordering}, - OnceLock, - }, -}; - -#[test] -fn openjpeg_in_process_matches_signinum_rgb_fixture() { - let Some(input) = bench_fixture_rgb() else { - return; - }; - let ours = signinum_rgb(&input); - let theirs = signinum_j2k_compare::openjpeg::decode_rgb(&input).expect("openjpeg"); - assert_eq!(ours, theirs); -} - -#[test] -fn openjpeg_in_process_region_matches_signinum_rgb_fixture() { - let Some(input) = bench_fixture_rgb() else { - return; - }; - let roi = Rect { - x: 16, - y: 24, - w: 64, - h: 64, - }; - let ours = signinum_rgb_region(&input, roi); - let theirs = signinum_j2k_compare::openjpeg::decode_rgb_region(&input, roi).expect("openjpeg"); - assert_eq!(ours, theirs); -} - -#[test] -fn grok_in_process_matches_signinum_rgb_fixture() { - if !signinum_j2k_compare::grok::is_available() { - assert!( - !require_grok(), - "SIGNINUM_REQUIRE_GROK is set but in-process Grok is unavailable" - ); - return; - } - let Some(input) = bench_fixture_rgb() else { - return; - }; - let ours = signinum_rgb(&input); - let theirs = signinum_j2k_compare::grok::decode_rgb(&input).expect("grok"); - assert_eq!(ours, theirs); -} - -#[test] -fn grok_in_process_scaled_matches_signinum_rgb_fixture() { - if !signinum_j2k_compare::grok::is_available() { - assert!( - !require_grok(), - "SIGNINUM_REQUIRE_GROK is set but in-process Grok is unavailable" - ); - return; - } - let Some(input) = bench_fixture_rgb() else { - return; - }; - let ours = signinum_rgb_scaled_q4(&input); - let theirs = signinum_j2k_compare::grok::decode_rgb_scaled(&input, 2).expect("grok"); - assert_eq!(ours, theirs); -} - -fn bench_fixture_rgb() -> Option> { - let pixels = gradient_u8(128, 128, 3); - openjpeg_encode_jp2("in_process_parity_rgb", &pixels, 128, 128) -} - -fn signinum_rgb(bytes: &[u8]) -> Vec { - let mut decoder = J2kDecoder::new(bytes).expect("decoder"); - let dims = decoder.info().dimensions; - let mut out = vec![0_u8; dims.0 as usize * dims.1 as usize * 3]; - decoder - .decode_into(&mut out, dims.0 as usize * 3, PixelFormat::Rgb8) - .expect("decode"); - out -} - -fn signinum_rgb_region(bytes: &[u8], roi: Rect) -> Vec { - let mut decoder = J2kDecoder::new(bytes).expect("decoder"); - let mut out = vec![0_u8; roi.w as usize * roi.h as usize * 3]; - decoder - .decode_region_into( - &mut signinum_j2k::J2kScratchPool::new(), - &mut out, - roi.w as usize * 3, - PixelFormat::Rgb8, - roi, - ) - .expect("region decode"); - out -} - -fn signinum_rgb_scaled_q4(bytes: &[u8]) -> Vec { - let mut decoder = J2kDecoder::new(bytes).expect("decoder"); - let dims = decoder.info().dimensions; - let scaled = (dims.0.div_ceil(4), dims.1.div_ceil(4)); - let mut out = vec![0_u8; scaled.0 as usize * scaled.1 as usize * 3]; - decoder - .decode_scaled_into( - &mut signinum_j2k::J2kScratchPool::new(), - &mut out, - scaled.0 as usize * 3, - PixelFormat::Rgb8, - Downscale::Quarter, - ) - .expect("scaled decode"); - out -} - -fn openjpeg_encode_jp2(name: &str, pixels: &[u8], width: u32, height: u32) -> Option> { - let Some(bin) = openjpeg_compress_bin() else { - assert!( - !require_openjpeg(), - "SIGNINUM_REQUIRE_OPENJPEG is set but opj_compress was not found" - ); - return None; - }; - let dir = openjpeg_temp_dir(); - let unique = next_temp_suffix(); - let src_path = dir.join(format!("{name}-{unique}.ppm")); - let out_path = dir.join(format!("{name}-{unique}.jp2")); - write_ppm(&src_path, pixels, width, height).ok()?; - let status = Command::new(bin) - .arg("-i") - .arg(&src_path) - .arg("-o") - .arg(&out_path) - .status() - .ok()?; - if !status.success() { - assert!( - !require_openjpeg(), - "SIGNINUM_REQUIRE_OPENJPEG is set but opj_compress failed" - ); - return None; - } - fs::read(out_path).ok() -} - -fn next_temp_suffix() -> usize { - static COUNTER: AtomicUsize = AtomicUsize::new(0); - COUNTER.fetch_add(1, Ordering::Relaxed) -} - -fn openjpeg_compress_bin() -> Option { - static OPENJPEG_COMPRESS: OnceLock> = OnceLock::new(); - OPENJPEG_COMPRESS - .get_or_init(|| { - if let Some(path) = std::env::var_os("SIGNINUM_OPENJPEG_COMPRESS_BIN") { - let path = PathBuf::from(path); - if path.exists() { - return Some(path); - } - } - let default = PathBuf::from("/opt/homebrew/bin/opj_compress"); - default.exists().then_some(default) - }) - .clone() -} - -fn require_openjpeg() -> bool { - std::env::var_os("SIGNINUM_REQUIRE_OPENJPEG").is_some() -} - -fn require_grok() -> bool { - std::env::var_os("SIGNINUM_REQUIRE_GROK").is_some() -} - -fn openjpeg_temp_dir() -> PathBuf { - static DIR: OnceLock = OnceLock::new(); - DIR.get_or_init(|| { - let dir = std::env::temp_dir().join("signinum-j2k-openjpeg-tests"); - fs::create_dir_all(&dir).expect("create openjpeg temp dir"); - dir - }) - .clone() -} - -fn write_ppm(path: &Path, pixels: &[u8], width: u32, height: u32) -> std::io::Result<()> { - let mut bytes = format!("P6\n{width} {height}\n255\n").into_bytes(); - bytes.extend_from_slice(pixels); - fs::write(path, bytes) -} diff --git a/crates/signinum-j2k-cuda/Cargo.toml b/crates/signinum-j2k-cuda/Cargo.toml deleted file mode 100644 index a4bf8f2a..00000000 --- a/crates/signinum-j2k-cuda/Cargo.toml +++ /dev/null @@ -1,48 +0,0 @@ -[package] -name = "signinum-j2k-cuda" -description = "CUDA device-output adapter for signinum-j2k" -version = "0.4.2" -edition.workspace = true -rust-version.workspace = true -license.workspace = true -repository.workspace = true -keywords.workspace = true -categories.workspace = true -readme = "README.md" - -[lib] -name = "signinum_j2k_cuda" -path = "src/lib.rs" - -[features] -default = [] -cuda-runtime = ["dep:signinum-cuda-runtime"] - -[dependencies] -signinum-core = { path = "../signinum-core", version = "=0.4.2" } -signinum-cuda-runtime = { path = "../signinum-cuda-runtime", version = "=0.4.2", optional = true } -signinum-j2k = { path = "../signinum-j2k", version = "=0.4.2" } -signinum-j2k-native = { path = "../signinum-j2k-native", version = "=0.4.2" } -signinum-profile = { path = "../signinum-profile", version = "=0.4.2" } -thiserror = { workspace = true } - -[dev-dependencies] -criterion = { workspace = true } -signinum-j2k-native = { path = "../signinum-j2k-native", version = "=0.4.2" } - -[[bench]] -name = "encode_stages" -harness = false - -[lints.rust] -unsafe_code = "allow" -unsafe_op_in_unsafe_fn = "deny" -unreachable_pub = "warn" - -[lints.clippy] -pedantic = { level = "warn", priority = -1 } -module_name_repetitions = "allow" -must_use_candidate = "allow" -missing_errors_doc = "allow" -missing_panics_doc = "allow" -too_many_lines = "allow" diff --git a/crates/signinum-j2k-cuda/README.md b/crates/signinum-j2k-cuda/README.md deleted file mode 100644 index 6dd87133..00000000 --- a/crates/signinum-j2k-cuda/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# signinum-j2k-cuda - -CUDA-facing device-output adapter for `signinum-j2k`. - -Install this crate when a pipeline needs JPEG 2000 / HTJ2K output copied into -CUDA device memory: - -```sh -cargo add signinum-j2k-cuda --features cuda-runtime -``` - -`BackendRequest::Cpu` and `BackendRequest::Auto` return host-backed CPU -surfaces. `BackendRequest::Cuda` requires the `cuda-runtime` feature and an -available CUDA driver; when both are present, scalar and full-tile batch calls -upload CPU-decoded JPEG 2000 / HTJ2K bytes into CUDA device memory and return -CUDA-backed `DeviceSurface` values. - -This crate does not provide CUDA kernel JPEG 2000 / HTJ2K decode and makes no -NVIDIA performance claim. The stable CPU decode API lives in `signinum-j2k`. diff --git a/crates/signinum-j2k-cuda/src/decoder.rs b/crates/signinum-j2k-cuda/src/decoder.rs deleted file mode 100644 index bc57ecbb..00000000 --- a/crates/signinum-j2k-cuda/src/decoder.rs +++ /dev/null @@ -1,280 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use core::convert::Infallible; - -use signinum_core::{ - BackendRequest, DecodeOutcome, Downscale, ImageCodec, ImageDecode, ImageDecodeDevice, - ImageDecodeSubmit, PixelFormat, ReadySubmission, Rect, -}; -use signinum_j2k::{ - adapter::device_plan::{DeviceDecodePlan, DeviceDecodeRequest}, - J2kDecoder as CpuDecoder, J2kScratchPool as CpuJ2kScratchPool, J2kView, -}; - -use crate::runtime::{validate_surface_request, wrap_surface}; -use crate::{profile, CudaSession, Error, Surface}; - -pub struct J2kDecoder<'a> { - inner: CpuDecoder<'a>, - pool: CpuJ2kScratchPool, -} - -impl<'a> J2kDecoder<'a> { - pub fn new(input: &'a [u8]) -> Result { - Ok(Self { - inner: CpuDecoder::new(input)?, - pool: CpuJ2kScratchPool::new(), - }) - } - - fn decode_to_surface_impl( - &mut self, - session: &mut CudaSession, - fmt: PixelFormat, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - let dims = self.inner.info().dimensions; - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut out = vec![0u8; stride * dims.1 as usize]; - if profile::gpu_route_profile_enabled() { - let request_s = format!("{backend:?}"); - let fmt_s = format!("{fmt:?}"); - let width_s = dims.0.to_string(); - let height_s = dims.1.to_string(); - profile::emit_gpu_route_profile( - "j2k", - "gpu_route", - "cuda", - &[ - ("op", "full"), - ("request", request_s.as_str()), - ("fmt", fmt_s.as_str()), - ("width", width_s.as_str()), - ("height", height_s.as_str()), - ("decision", "cpu_decode_then_wrap"), - ], - ); - } - self.inner - .decode_into_with_scratch(&mut self.pool, &mut out, stride, fmt)?; - wrap_surface(out, dims, fmt, backend, session) - } - - fn decode_region_to_surface_impl( - &mut self, - session: &mut CudaSession, - fmt: PixelFormat, - roi: Rect, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - let plan = DeviceDecodePlan::for_image( - self.inner.info().dimensions, - DeviceDecodeRequest::Region { roi }, - )?; - let dims = plan.output_dims(); - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut out = vec![0u8; stride * dims.1 as usize]; - self.inner - .decode_region_into(&mut self.pool, &mut out, stride, fmt, plan.source_rect())?; - wrap_surface(out, dims, fmt, backend, session) - } - - fn decode_scaled_to_surface_impl( - &mut self, - session: &mut CudaSession, - fmt: PixelFormat, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - let dims = DeviceDecodePlan::for_image( - self.inner.info().dimensions, - DeviceDecodeRequest::Scaled { scale }, - )? - .output_dims(); - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut out = vec![0u8; stride * dims.1 as usize]; - self.inner - .decode_scaled_into(&mut self.pool, &mut out, stride, fmt, scale)?; - wrap_surface(out, dims, fmt, backend, session) - } - - fn decode_region_scaled_to_surface_impl( - &mut self, - session: &mut CudaSession, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - let plan = DeviceDecodePlan::for_image( - self.inner.info().dimensions, - DeviceDecodeRequest::RegionScaled { roi, scale }, - )?; - let dims = plan.output_dims(); - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut out = vec![0u8; stride * dims.1 as usize]; - self.inner.decode_region_scaled_into( - &mut self.pool, - &mut out, - stride, - fmt, - plan.source_rect(), - scale, - )?; - wrap_surface(out, dims, fmt, backend, session) - } -} - -impl ImageCodec for J2kDecoder<'_> { - type Error = Error; - type Warning = Infallible; - type Pool = CpuJ2kScratchPool; -} - -impl<'a> ImageDecode<'a> for J2kDecoder<'a> { - type View = J2kView<'a>; - - fn inspect(input: &'a [u8]) -> Result { - Ok(CpuDecoder::inspect(input)?) - } - - fn parse(input: &'a [u8]) -> Result { - Ok(J2kView::parse(input)?) - } - - fn from_view(view: Self::View) -> Result { - Ok(Self { - inner: CpuDecoder::from_view(view)?, - pool: CpuJ2kScratchPool::new(), - }) - } - - fn decode_into( - &mut self, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - ) -> Result, Self::Error> { - Ok(self.inner.decode_into(out, stride, fmt)?) - } - - fn decode_into_with_scratch( - &mut self, - pool: &mut Self::Pool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - ) -> Result, Self::Error> { - Ok(self - .inner - .decode_into_with_scratch(pool, out, stride, fmt)?) - } - - fn decode_region_into( - &mut self, - pool: &mut Self::Pool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - ) -> Result, Self::Error> { - Ok(self.inner.decode_region_into(pool, out, stride, fmt, roi)?) - } - - fn decode_scaled_into( - &mut self, - pool: &mut Self::Pool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - scale: Downscale, - ) -> Result, Self::Error> { - Ok(self - .inner - .decode_scaled_into(pool, out, stride, fmt, scale)?) - } - - fn decode_region_scaled_into( - &mut self, - pool: &mut Self::Pool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - ) -> Result, Self::Error> { - Ok(self - .inner - .decode_region_scaled_into(pool, out, stride, fmt, roi, scale)?) - } -} - -impl<'a> ImageDecodeDevice<'a> for J2kDecoder<'a> { - type DeviceSurface = Surface; -} - -impl<'a> ImageDecodeSubmit<'a> for J2kDecoder<'a> { - type Session = CudaSession; - type DeviceSurface = Surface; - type SubmittedSurface = ReadySubmission; - - fn submit_to_device( - &mut self, - session: &mut Self::Session, - fmt: PixelFormat, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - self.decode_to_surface_impl(session, fmt, backend), - )) - } - - fn submit_region_to_device( - &mut self, - session: &mut Self::Session, - fmt: PixelFormat, - roi: Rect, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - self.decode_region_to_surface_impl(session, fmt, roi, backend), - )) - } - - fn submit_scaled_to_device( - &mut self, - session: &mut Self::Session, - fmt: PixelFormat, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - self.decode_scaled_to_surface_impl(session, fmt, scale, backend), - )) - } - - fn submit_region_scaled_to_device( - &mut self, - session: &mut Self::Session, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - self.decode_region_scaled_to_surface_impl(session, fmt, roi, scale, backend), - )) - } -} diff --git a/crates/signinum-j2k-cuda/src/encode.rs b/crates/signinum-j2k-cuda/src/encode.rs deleted file mode 100644 index 70b52aab..00000000 --- a/crates/signinum-j2k-cuda/src/encode.rs +++ /dev/null @@ -1,444 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#[cfg(feature = "cuda-runtime")] -use signinum_cuda_runtime::{CudaContext, CudaDwt53Output}; -use signinum_j2k_native::{ - EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, J2kEncodeDispatchReport, J2kEncodeStageAccelerator, - J2kForwardDwt53Job, J2kForwardDwt53Output, J2kForwardRctJob, J2kHtCodeBlockEncodeJob, - J2kPacketizationEncodeJob, J2kTier1CodeBlockEncodeJob, -}; - -use crate::profile; - -#[derive(Debug, Default, Clone)] -pub struct CudaEncodeStageAccelerator { - #[cfg(feature = "cuda-runtime")] - context: Option, - forward_rct_attempts: usize, - forward_dwt53_attempts: usize, - tier1_code_block_attempts: usize, - ht_code_block_attempts: usize, - packetization_attempts: usize, - forward_rct_dispatches: usize, - forward_dwt53_dispatches: usize, - tier1_code_block_dispatches: usize, - ht_code_block_dispatches: usize, - packetization_dispatches: usize, -} - -impl CudaEncodeStageAccelerator { - #[cfg(feature = "cuda-runtime")] - fn cuda_context(&mut self) -> core::result::Result, &'static str> { - if self.context.is_none() { - match CudaContext::system_default() { - Ok(context) => self.context = Some(context), - Err(_) if cuda_runtime_required() => return Err("CUDA encode stage unavailable"), - Err(_) => return Ok(None), - } - } - Ok(self.context.clone()) - } - - pub fn forward_rct_attempts(&self) -> usize { - self.forward_rct_attempts - } - - pub fn forward_dwt53_attempts(&self) -> usize { - self.forward_dwt53_attempts - } - - pub fn tier1_code_block_attempts(&self) -> usize { - self.tier1_code_block_attempts - } - - pub fn ht_code_block_attempts(&self) -> usize { - self.ht_code_block_attempts - } - - pub fn packetization_attempts(&self) -> usize { - self.packetization_attempts - } - - pub fn forward_rct_dispatches(&self) -> usize { - self.forward_rct_dispatches - } - - pub fn forward_dwt53_dispatches(&self) -> usize { - self.forward_dwt53_dispatches - } - - pub fn tier1_code_block_dispatches(&self) -> usize { - self.tier1_code_block_dispatches - } - - pub fn ht_code_block_dispatches(&self) -> usize { - self.ht_code_block_dispatches - } - - pub fn packetization_dispatches(&self) -> usize { - self.packetization_dispatches - } -} - -#[cfg(feature = "cuda-runtime")] -fn cuda_runtime_required() -> bool { - std::env::var_os("SIGNINUM_REQUIRE_CUDA_RUNTIME").is_some() -} - -impl J2kEncodeStageAccelerator for CudaEncodeStageAccelerator { - fn dispatch_report(&self) -> J2kEncodeDispatchReport { - J2kEncodeDispatchReport { - forward_rct: self.forward_rct_dispatches, - forward_dwt53: self.forward_dwt53_dispatches, - tier1_code_block: self.tier1_code_block_dispatches, - ht_code_block: self.ht_code_block_dispatches, - packetization: self.packetization_dispatches, - } - } - - fn encode_forward_rct( - &mut self, - job: J2kForwardRctJob<'_>, - ) -> core::result::Result { - self.forward_rct_attempts = self.forward_rct_attempts.saturating_add(1); - #[cfg(feature = "cuda-runtime")] - if let Some(context) = self.cuda_context()? { - context - .j2k_forward_rct(job.plane0, job.plane1, job.plane2) - .map_err(|_| "CUDA forward RCT encode kernel failed")?; - self.forward_rct_dispatches = self.forward_rct_dispatches.saturating_add(1); - if profile::gpu_route_profile_enabled() { - profile::emit_gpu_route_profile( - "j2k", - "gpu_route", - "cuda", - &[ - ("op", "encode_forward_rct"), - ("decision", "cuda_dispatch"), - ("dispatches", "1"), - ], - ); - } - return Ok(true); - } - #[cfg(not(feature = "cuda-runtime"))] - let _ = job; - if profile::gpu_route_profile_enabled() { - profile::emit_gpu_route_profile( - "j2k", - "gpu_route", - "cuda", - &[ - ("op", "encode_forward_rct"), - ("decision", "cpu_fallback"), - ("reason", "cuda_unavailable"), - ], - ); - } - Ok(false) - } - - fn encode_forward_dwt53( - &mut self, - job: J2kForwardDwt53Job<'_>, - ) -> core::result::Result, &'static str> { - self.forward_dwt53_attempts = self.forward_dwt53_attempts.saturating_add(1); - if job.num_levels == 0 { - if profile::gpu_route_profile_enabled() { - profile::emit_gpu_route_profile( - "j2k", - "gpu_route", - "cuda", - &[ - ("op", "encode_forward_dwt53"), - ("decision", "cpu_fallback"), - ("reason", "zero_levels"), - ], - ); - } - return Ok(None); - } - #[cfg(feature = "cuda-runtime")] - if let Some(context) = self.cuda_context()? { - let output = context - .j2k_forward_dwt53(job.samples, job.width, job.height, job.num_levels) - .map_err(|_| "CUDA forward 5/3 DWT encode kernel failed")?; - let dispatches = output.execution().kernel_dispatches(); - self.forward_dwt53_dispatches = - self.forward_dwt53_dispatches.saturating_add(dispatches); - if profile::gpu_route_profile_enabled() { - let width_s = job.width.to_string(); - let height_s = job.height.to_string(); - let levels_s = job.num_levels.to_string(); - let dispatches_s = dispatches.to_string(); - profile::emit_gpu_route_profile( - "j2k", - "gpu_route", - "cuda", - &[ - ("op", "encode_forward_dwt53"), - ("decision", "cuda_dispatch"), - ("width", width_s.as_str()), - ("height", height_s.as_str()), - ("levels", levels_s.as_str()), - ("dispatches", dispatches_s.as_str()), - ], - ); - } - return Ok(Some(cuda_dwt53_output_to_j2k(&output)?)); - } - #[cfg(not(feature = "cuda-runtime"))] - let _ = job; - if profile::gpu_route_profile_enabled() { - profile::emit_gpu_route_profile( - "j2k", - "gpu_route", - "cuda", - &[ - ("op", "encode_forward_dwt53"), - ("decision", "cpu_fallback"), - ("reason", "cuda_unavailable"), - ], - ); - } - Ok(None) - } - - fn encode_tier1_code_block( - &mut self, - _job: J2kTier1CodeBlockEncodeJob<'_>, - ) -> core::result::Result, &'static str> { - self.tier1_code_block_attempts = self.tier1_code_block_attempts.saturating_add(1); - if profile::gpu_route_profile_enabled() { - profile::emit_gpu_route_profile( - "j2k", - "gpu_route", - "cuda", - &[ - ("op", "encode_tier1_code_block"), - ("decision", "cpu_fallback"), - ("reason", "unsupported_stage"), - ], - ); - } - Ok(None) - } - - fn encode_ht_code_block( - &mut self, - _job: J2kHtCodeBlockEncodeJob<'_>, - ) -> core::result::Result, &'static str> { - self.ht_code_block_attempts = self.ht_code_block_attempts.saturating_add(1); - if profile::gpu_route_profile_enabled() { - profile::emit_gpu_route_profile( - "j2k", - "gpu_route", - "cuda", - &[ - ("op", "encode_ht_code_block"), - ("decision", "cpu_fallback"), - ("reason", "unsupported_stage"), - ], - ); - } - Ok(None) - } - - fn encode_packetization( - &mut self, - _job: J2kPacketizationEncodeJob<'_>, - ) -> core::result::Result>, &'static str> { - self.packetization_attempts = self.packetization_attempts.saturating_add(1); - if profile::gpu_route_profile_enabled() { - profile::emit_gpu_route_profile( - "j2k", - "gpu_route", - "cuda", - &[ - ("op", "encode_packetization"), - ("decision", "cpu_fallback"), - ("reason", "unsupported_stage"), - ], - ); - } - Ok(None) - } -} - -#[cfg(feature = "cuda-runtime")] -fn cuda_dwt53_output_to_j2k( - output: &CudaDwt53Output, -) -> core::result::Result { - let (ll_width, ll_height) = output.ll_dimensions(); - let transformed = output.transformed(); - let full_width = output - .levels() - .first() - .map_or(ll_width, |level| level.width) as usize; - let mut ll = Vec::with_capacity((ll_width as usize) * (ll_height as usize)); - for y in 0..ll_height as usize { - let row_start = y - .checked_mul(full_width) - .ok_or("CUDA DWT LL row offset overflow")?; - ll.extend_from_slice(&transformed[row_start..row_start + ll_width as usize]); - } - - let mut levels = Vec::with_capacity(output.levels().len()); - for shape in output.levels() { - levels.push(signinum_j2k_native::J2kForwardDwt53Level { - hl: extract_cuda_subband( - transformed, - full_width, - shape.low_width, - 0, - shape.high_width, - shape.low_height, - )?, - lh: extract_cuda_subband( - transformed, - full_width, - 0, - shape.low_height, - shape.low_width, - shape.high_height, - )?, - hh: extract_cuda_subband( - transformed, - full_width, - shape.low_width, - shape.low_height, - shape.high_width, - shape.high_height, - )?, - width: shape.width, - height: shape.height, - low_width: shape.low_width, - low_height: shape.low_height, - high_width: shape.high_width, - high_height: shape.high_height, - }); - } - levels.reverse(); - - Ok(J2kForwardDwt53Output { - ll, - ll_width, - ll_height, - levels, - }) -} - -#[cfg(feature = "cuda-runtime")] -fn extract_cuda_subband( - transformed: &[f32], - full_width: usize, - x0: u32, - y0: u32, - width: u32, - height: u32, -) -> core::result::Result, &'static str> { - let mut out = Vec::with_capacity((width as usize) * (height as usize)); - for y in 0..height as usize { - let row_start = (y0 as usize) - .checked_add(y) - .and_then(|row| row.checked_mul(full_width)) - .and_then(|row| row.checked_add(x0 as usize)) - .ok_or("CUDA DWT subband offset overflow")?; - out.extend_from_slice(&transformed[row_start..row_start + width as usize]); - } - Ok(out) -} - -#[cfg(test)] -mod tests { - use super::CudaEncodeStageAccelerator; - use signinum_j2k_native::{encode_with_accelerator, DecodeSettings, EncodeOptions, Image}; - - #[test] - fn cuda_encode_stage_accelerator_preserves_cpu_codestream_validity() { - let pixels: Vec = (0u8..192).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let mut accelerator = CudaEncodeStageAccelerator::default(); - - let codestream = - encode_with_accelerator(&pixels, 8, 8, 3, 8, false, &options, &mut accelerator) - .expect("encode with CUDA stage accelerator"); - let decoded = Image::new(&codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - - assert_eq!(decoded.width, 8); - assert_eq!(decoded.height, 8); - assert_eq!(decoded.num_components, 3); - assert_eq!(decoded.bit_depth, 8); - assert_eq!(accelerator.forward_rct_attempts(), 1); - assert_eq!(accelerator.forward_dwt53_attempts(), 3); - assert!(accelerator.tier1_code_block_attempts() > 0); - assert_eq!(accelerator.packetization_attempts(), 1); - } - - #[cfg(feature = "cuda-runtime")] - #[test] - fn cuda_forward_rct_dispatches_when_runtime_required() { - if std::env::var_os("SIGNINUM_REQUIRE_CUDA_RUNTIME").is_none() { - return; - } - - let pixels: Vec = (0u16..7 * 5 * 3) - .map(|i| u8::try_from((i * 17) & 0xFF).expect("masked value fits in u8")) - .collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 0, - ..EncodeOptions::default() - }; - let mut accelerator = CudaEncodeStageAccelerator::default(); - - let codestream = - encode_with_accelerator(&pixels, 7, 5, 3, 8, false, &options, &mut accelerator) - .expect("encode with CUDA forward RCT"); - let decoded = Image::new(&codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - - assert_eq!(decoded.data, pixels); - assert_eq!(accelerator.forward_rct_attempts(), 1); - assert_eq!(accelerator.forward_rct_dispatches(), 1); - } - - #[cfg(feature = "cuda-runtime")] - #[test] - fn cuda_forward_dwt53_dispatches_when_runtime_required() { - if std::env::var_os("SIGNINUM_REQUIRE_CUDA_RUNTIME").is_none() { - return; - } - - let pixels: Vec = (0u16..8 * 8) - .map(|i| u8::try_from((i * 5) & 0xFF).expect("masked value fits in u8")) - .collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let mut accelerator = CudaEncodeStageAccelerator::default(); - - let codestream = - encode_with_accelerator(&pixels, 8, 8, 1, 8, false, &options, &mut accelerator) - .expect("encode with CUDA forward DWT 5/3"); - let decoded = Image::new(&codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - - assert_eq!(decoded.data, pixels); - assert_eq!(accelerator.forward_dwt53_attempts(), 1); - assert_eq!(accelerator.forward_dwt53_dispatches(), 2); - } -} diff --git a/crates/signinum-j2k-cuda/src/error.rs b/crates/signinum-j2k-cuda/src/error.rs deleted file mode 100644 index 45586078..00000000 --- a/crates/signinum-j2k-cuda/src/error.rs +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use signinum_core::{BackendRequest, BufferError, CodecError}; -use signinum_j2k::J2kError; - -#[derive(Debug, thiserror::Error)] -pub enum Error { - #[error(transparent)] - Decode(#[from] J2kError), - #[error(transparent)] - Buffer(#[from] BufferError), - #[error("backend request {request:?} is not supported by signinum-j2k-cuda")] - UnsupportedBackend { request: BackendRequest }, - #[error("CUDA is unavailable on this host")] - CudaUnavailable, - #[cfg(feature = "cuda-runtime")] - #[error("CUDA runtime error: {message}")] - CudaRuntime { message: String }, -} - -impl CodecError for Error { - fn is_truncated(&self) -> bool { - matches!(self, Self::Decode(inner) if inner.is_truncated()) - } - - fn is_not_implemented(&self) -> bool { - matches!(self, Self::Decode(inner) if inner.is_not_implemented()) - } - - fn is_unsupported(&self) -> bool { - matches!( - self, - Self::UnsupportedBackend { .. } | Self::CudaUnavailable - ) || matches!(self, Self::Decode(inner) if inner.is_unsupported()) - } - - fn is_buffer_error(&self) -> bool { - matches!(self, Self::Buffer(_)) - || matches!(self, Self::Decode(inner) if inner.is_buffer_error()) - } -} diff --git a/crates/signinum-j2k-cuda/src/lib.rs b/crates/signinum-j2k-cuda/src/lib.rs deleted file mode 100644 index 5d574929..00000000 --- a/crates/signinum-j2k-cuda/src/lib.rs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! CUDA-facing device-output adapter for `signinum-j2k`. -//! -//! This crate intentionally exposes the same backend-selection surface as the -//! Metal adapter. CPU and auto requests return host-backed surfaces, while -//! explicit CUDA requests upload decoded output into CUDA device memory when -//! the `cuda-runtime` feature and a CUDA driver are available. - -#![warn(unreachable_pub)] - -mod codec; -mod decoder; -mod encode; -mod error; -mod profile; -mod runtime; -mod session; -mod surface; - -pub use codec::Codec; -pub use decoder::J2kDecoder; -pub use encode::CudaEncodeStageAccelerator; -pub use error::Error; -pub use session::CudaSession; -pub use signinum_j2k::{J2kContext, J2kScratchPool}; -pub use surface::{CudaSurface, CudaSurfaceStats, Surface}; diff --git a/crates/signinum-j2k-cuda/src/profile.rs b/crates/signinum-j2k-cuda/src/profile.rs deleted file mode 100644 index f4000840..00000000 --- a/crates/signinum-j2k-cuda/src/profile.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -pub(crate) fn gpu_route_profile_enabled() -> bool { - signinum_profile::gpu_route_profile_enabled() -} - -pub(crate) fn emit_gpu_route_profile(codec: &str, op: &str, path: &str, fields: &[(K, V)]) -where - K: AsRef, - V: AsRef, -{ - debug_assert_eq!(op, "gpu_route"); - signinum_profile::emit_gpu_route_profile(codec, path, fields); -} diff --git a/crates/signinum-j2k-cuda/src/session.rs b/crates/signinum-j2k-cuda/src/session.rs deleted file mode 100644 index c36baefb..00000000 --- a/crates/signinum-j2k-cuda/src/session.rs +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#[cfg(feature = "cuda-runtime")] -use signinum_cuda_runtime::CudaContext; - -#[cfg(feature = "cuda-runtime")] -use crate::runtime::cuda_error; -#[cfg(feature = "cuda-runtime")] -use crate::Error; - -#[derive(Clone, Default)] -pub struct CudaSession { - submissions: u64, - #[cfg(feature = "cuda-runtime")] - context: Option, -} - -impl CudaSession { - pub fn submissions(&self) -> u64 { - self.submissions - } - - #[cfg(feature = "cuda-runtime")] - pub fn is_runtime_initialized(&self) -> bool { - self.context.is_some() - } - - pub(crate) fn record_submit(&mut self) { - self.submissions = self.submissions.saturating_add(1); - } - - #[cfg(feature = "cuda-runtime")] - pub(crate) fn cuda_context(&mut self) -> Result { - if self.context.is_none() { - self.context = Some(CudaContext::system_default().map_err(cuda_error)?); - } - self.context.clone().ok_or(Error::CudaUnavailable) - } -} - -impl std::fmt::Debug for CudaSession { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut debug = f.debug_struct("CudaSession"); - debug.field("submissions", &self.submissions); - #[cfg(feature = "cuda-runtime")] - debug.field("runtime_initialized", &self.is_runtime_initialized()); - debug.finish_non_exhaustive() - } -} diff --git a/crates/signinum-j2k-cuda/src/surface.rs b/crates/signinum-j2k-cuda/src/surface.rs deleted file mode 100644 index 3c3ecf75..00000000 --- a/crates/signinum-j2k-cuda/src/surface.rs +++ /dev/null @@ -1,127 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use signinum_core::{copy_tight_pixels_to_strided_output, BackendKind, DeviceSurface, PixelFormat}; -#[cfg(feature = "cuda-runtime")] -use signinum_cuda_runtime::CudaDeviceBuffer; - -#[cfg(feature = "cuda-runtime")] -use crate::runtime::cuda_error; -use crate::Error; - -#[derive(Debug)] -pub(crate) enum Storage { - Host(Vec), - #[cfg(feature = "cuda-runtime")] - Cuda(CudaDeviceBuffer), -} - -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] -pub struct CudaSurfaceStats { - pub(crate) kernel_dispatches: usize, -} - -impl CudaSurfaceStats { - pub fn kernel_dispatches(self) -> usize { - self.kernel_dispatches - } -} - -#[derive(Clone, Copy, Debug)] -pub struct CudaSurface<'a> { - #[cfg(feature = "cuda-runtime")] - buffer: &'a CudaDeviceBuffer, - #[cfg(not(feature = "cuda-runtime"))] - _marker: core::marker::PhantomData<&'a ()>, - pub(crate) stats: CudaSurfaceStats, -} - -impl CudaSurface<'_> { - pub fn device_ptr(&self) -> u64 { - #[cfg(feature = "cuda-runtime")] - { - self.buffer.device_ptr() - } - #[cfg(not(feature = "cuda-runtime"))] - { - unreachable!("CudaSurface cannot be constructed without cuda-runtime support") - } - } - - pub fn stats(&self) -> CudaSurfaceStats { - self.stats - } -} - -#[derive(Debug)] -pub struct Surface { - pub(crate) backend: BackendKind, - pub(crate) dimensions: (u32, u32), - pub(crate) fmt: PixelFormat, - pub(crate) pitch_bytes: usize, - pub(crate) stats: CudaSurfaceStats, - pub(crate) storage: Storage, -} - -impl Surface { - pub fn pitch_bytes(&self) -> usize { - self.pitch_bytes - } - - pub fn as_host_bytes(&self) -> Option<&[u8]> { - match &self.storage { - Storage::Host(bytes) => Some(bytes), - #[cfg(feature = "cuda-runtime")] - Storage::Cuda(_) => None, - } - } - - pub fn download_into(&self, out: &mut [u8], stride: usize) -> Result<(), Error> { - match &self.storage { - Storage::Host(bytes) => { - copy_tight_pixels_to_strided_output(bytes, self.dimensions, self.fmt, out, stride) - .map_err(Error::from) - } - #[cfg(feature = "cuda-runtime")] - Storage::Cuda(buffer) => { - let mut tight = vec![0u8; self.byte_len()]; - buffer.copy_to_host(&mut tight).map_err(cuda_error)?; - copy_tight_pixels_to_strided_output(&tight, self.dimensions, self.fmt, out, stride) - .map_err(Error::from) - } - } - } - - pub fn cuda_surface(&self) -> Option> { - #[cfg(feature = "cuda-runtime")] - match &self.storage { - Storage::Cuda(buffer) => Some(CudaSurface { - buffer, - stats: self.stats, - }), - Storage::Host(_) => None, - } - #[cfg(not(feature = "cuda-runtime"))] - { - let _ = self.stats; - None - } - } -} - -impl DeviceSurface for Surface { - fn backend_kind(&self) -> BackendKind { - self.backend - } - - fn dimensions(&self) -> (u32, u32) { - self.dimensions - } - - fn pixel_format(&self) -> PixelFormat { - self.fmt - } - - fn byte_len(&self) -> usize { - self.pitch_bytes * self.dimensions.1 as usize - } -} diff --git a/crates/signinum-j2k-cuda/tests/host_surface.rs b/crates/signinum-j2k-cuda/tests/host_surface.rs deleted file mode 100644 index 6c6950f8..00000000 --- a/crates/signinum-j2k-cuda/tests/host_surface.rs +++ /dev/null @@ -1,453 +0,0 @@ -use signinum_core::{ - BackendRequest, CodecError, DecoderContext, DeviceSubmission, DeviceSurface, Downscale, - ImageDecode, ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, Rect, TileBatchDecodeDevice, - TileBatchDecodeManyDevice, -}; -use signinum_j2k_cuda::{Codec, CudaSession, Error, J2kDecoder}; -use signinum_j2k_native::{encode, encode_htj2k, EncodeOptions}; - -fn fixture() -> Vec { - let pixels = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]; - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - encode(&pixels, 2, 2, 3, 8, false, &options).expect("encode") -} - -fn fixture_ht_gray8() -> Vec { - let pixels: Vec = (0..16).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - encode_htj2k(&pixels, 4, 4, 1, 8, false, &options).expect("encode ht gray8") -} - -#[test] -fn auto_falls_back_to_cpu_surface() { - let bytes = fixture(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_to_device(PixelFormat::Rgb8, BackendRequest::Auto) - .expect("surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cpu); - assert!(surface.as_host_bytes().is_some()); -} - -#[test] -fn explicit_cuda_request_returns_cuda_surface_or_clear_unavailable_error() { - let bytes = fixture(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - match decoder.decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) { - Ok(surface) => { - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cuda); - assert_eq!(surface.as_host_bytes(), None); - #[cfg(feature = "cuda-runtime")] - assert_ne!( - surface.cuda_surface().expect("cuda surface").device_ptr(), - 0 - ); - } - Err(error) => assert!(error.is_unsupported()), - } -} - -#[test] -fn explicit_cuda_request_validates_decode_before_upload() { - let bytes = fixture(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - - let error = decoder - .decode_to_device(PixelFormat::Rgba16, BackendRequest::Cuda) - .expect_err("unsupported decode"); - assert!(error.is_unsupported()); - assert!(!matches!(error, Error::CudaUnavailable)); -} - -#[test] -fn explicit_cuda_request_returns_cuda_surface_when_runtime_required() { - if !runtime_required() { - return; - } - - let bytes = fixture(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) - .expect("cuda surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cuda); - assert_eq!(surface.as_host_bytes(), None); - assert_cuda_surface(&surface); - assert_eq!(surface.dimensions(), (2, 2)); - - let mut downloaded = vec![0u8; surface.byte_len()]; - surface - .download_into(&mut downloaded, surface.pitch_bytes()) - .expect("download cuda surface"); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut expected = [0u8; 12]; - host_decoder - .decode_into(&mut expected, 6, PixelFormat::Rgb8) - .expect("host decode"); - assert_eq!(downloaded, expected); -} - -#[test] -fn explicit_cuda_region_scaled_surface_matches_host_when_runtime_required() { - if !runtime_required() { - return; - } - - let bytes = fixture_ht_gray8(); - let roi = Rect { - x: 1, - y: 0, - w: 2, - h: 3, - }; - let scale = Downscale::Half; - let scaled = roi.scaled_covering(scale); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Cuda) - .expect("cuda region+scaled surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cuda); - assert_eq!(surface.as_host_bytes(), None); - assert_cuda_surface(&surface); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - - let mut downloaded = vec![0u8; surface.byte_len()]; - surface - .download_into(&mut downloaded, surface.pitch_bytes()) - .expect("download cuda surface"); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut expected = vec![0u8; scaled.w as usize * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut signinum_j2k_cuda::J2kScratchPool::new(), - &mut expected, - scaled.w as usize, - PixelFormat::Gray8, - roi, - scale, - ) - .expect("host decode"); - assert_eq!(downloaded, expected); -} - -#[test] -fn explicit_cuda_download_respects_padded_stride_when_runtime_required() { - if !runtime_required() { - return; - } - - let bytes = fixture(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) - .expect("cuda surface"); - assert_cuda_surface(&surface); - let row_bytes = surface.pitch_bytes(); - let stride = row_bytes + 5; - let mut downloaded = vec![0xCD; stride * surface.dimensions().1 as usize]; - surface - .download_into(&mut downloaded, stride) - .expect("download cuda surface"); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut expected = [0u8; 12]; - host_decoder - .decode_into(&mut expected, row_bytes, PixelFormat::Rgb8) - .expect("host decode"); - for (row, expected_row) in expected.chunks(row_bytes).enumerate() { - let start = row * stride; - assert_eq!(&downloaded[start..start + row_bytes], expected_row); - assert_eq!(&downloaded[start + row_bytes..start + stride], &[0xCD; 5]); - } -} - -fn runtime_required() -> bool { - std::env::var_os("SIGNINUM_REQUIRE_CUDA_RUNTIME").is_some() -} - -fn assert_cuda_surface(surface: &signinum_j2k_cuda::Surface) { - let cuda = surface.cuda_surface().expect("cuda surface"); - assert_ne!(cuda.device_ptr(), 0); - assert!(cuda.stats().kernel_dispatches() > 0); -} - -#[test] -fn submit_to_device_auto_falls_back_to_cpu_surface() { - let bytes = fixture(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let mut session = CudaSession::default(); - let surface = as ImageDecodeSubmit<'_>>::submit_to_device( - &mut decoder, - &mut session, - PixelFormat::Rgb8, - BackendRequest::Auto, - ) - .expect("submission") - .wait() - .expect("surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cpu); - assert!(surface.as_host_bytes().is_some()); - assert!(session.submissions() >= 1); -} - -#[cfg(feature = "cuda-runtime")] -#[test] -fn submit_to_device_auto_does_not_initialize_cuda_runtime() { - let bytes = fixture(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let mut session = CudaSession::default(); - let surface = as ImageDecodeSubmit<'_>>::submit_to_device( - &mut decoder, - &mut session, - PixelFormat::Rgb8, - BackendRequest::Auto, - ) - .expect("submission") - .wait() - .expect("surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cpu); - assert_eq!(session.submissions(), 1); - assert!(!session.is_runtime_initialized()); -} - -#[cfg(feature = "cuda-runtime")] -#[test] -fn explicit_cuda_submissions_reuse_session_runtime_when_required() { - if !runtime_required() { - return; - } - - let bytes = fixture(); - let mut session = CudaSession::default(); - assert!(!session.is_runtime_initialized()); - - let mut first = J2kDecoder::new(&bytes).expect("decoder"); - let first_surface = as ImageDecodeSubmit<'_>>::submit_to_device( - &mut first, - &mut session, - PixelFormat::Rgb8, - BackendRequest::Cuda, - ) - .expect("first submission") - .wait() - .expect("first surface"); - assert_eq!( - first_surface.backend_kind(), - signinum_core::BackendKind::Cuda - ); - assert_cuda_surface(&first_surface); - assert!(session.is_runtime_initialized()); - - let mut second = J2kDecoder::new(&bytes).expect("decoder"); - let second_surface = as ImageDecodeSubmit<'_>>::submit_to_device( - &mut second, - &mut session, - PixelFormat::Rgb8, - BackendRequest::Cuda, - ) - .expect("second submission") - .wait() - .expect("second surface"); - assert_eq!( - second_surface.backend_kind(), - signinum_core::BackendKind::Cuda - ); - assert_cuda_surface(&second_surface); - assert_eq!(session.submissions(), 2); - assert!(session.is_runtime_initialized()); -} - -#[test] -fn auto_classic_full_frame_surface_matches_host_decode() { - let bytes = fixture(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_to_device(PixelFormat::Rgb8, BackendRequest::Auto) - .expect("surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cpu); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 12]; - host_decoder - .decode_into(&mut host, 6, PixelFormat::Rgb8) - .expect("host decode"); - assert_eq!(surface.as_host_bytes(), Some(host.as_slice())); -} - -#[test] -fn auto_htj2k_full_frame_surface_matches_host_decode() { - let bytes = fixture_ht_gray8(); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_to_device(PixelFormat::Gray8, BackendRequest::Auto) - .expect("surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cpu); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = [0u8; 16]; - host_decoder - .decode_into(&mut host, 4, PixelFormat::Gray8) - .expect("host decode"); - assert_eq!(surface.as_host_bytes(), Some(host.as_slice())); -} - -#[test] -fn auto_region_scaled_surface_matches_host_decode() { - let bytes = fixture_ht_gray8(); - let roi = Rect { - x: 1, - y: 0, - w: 2, - h: 3, - }; - let scale = Downscale::Half; - let scaled = roi.scaled_covering(scale); - let mut decoder = J2kDecoder::new(&bytes).expect("decoder"); - let surface = decoder - .decode_region_scaled_to_device(PixelFormat::Gray8, roi, scale, BackendRequest::Auto) - .expect("surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cpu); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut host = vec![0u8; scaled.w as usize * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut signinum_j2k_cuda::J2kScratchPool::new(), - &mut host, - scaled.w as usize, - PixelFormat::Gray8, - roi, - scale, - ) - .expect("host decode"); - assert_eq!(surface.as_host_bytes(), Some(host.as_slice())); -} - -#[test] -fn tile_batch_region_scaled_cuda_surface_matches_host_when_runtime_required() { - if !runtime_required() { - return; - } - - let bytes = fixture_ht_gray8(); - let roi = Rect { - x: 1, - y: 0, - w: 2, - h: 3, - }; - let scale = Downscale::Half; - let scaled = roi.scaled_covering(scale); - let mut ctx = DecoderContext::::new(); - let mut pool = signinum_j2k_cuda::J2kScratchPool::new(); - let surface = Codec::decode_tile_region_scaled_to_device( - &mut ctx, - &mut pool, - &bytes, - PixelFormat::Gray8, - roi, - scale, - BackendRequest::Cuda, - ) - .expect("cuda tile batch surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cuda); - assert_eq!(surface.as_host_bytes(), None); - assert_cuda_surface(&surface); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - - let mut downloaded = vec![0u8; surface.byte_len()]; - surface - .download_into(&mut downloaded, surface.pitch_bytes()) - .expect("download cuda surface"); - - let mut host_decoder = J2kDecoder::new(&bytes).expect("host decoder"); - let mut expected = vec![0u8; scaled.w as usize * scaled.h as usize]; - host_decoder - .decode_region_scaled_into( - &mut signinum_j2k_cuda::J2kScratchPool::new(), - &mut expected, - scaled.w as usize, - PixelFormat::Gray8, - roi, - scale, - ) - .expect("host decode"); - assert_eq!(downloaded, expected); -} - -#[test] -fn decode_tiles_to_device_auto_preserves_order_and_matches_host_bytes() { - let bytes = fixture_ht_gray8(); - let mut ctx = DecoderContext::::new(); - let mut pool = signinum_j2k_cuda::J2kScratchPool::new(); - let inputs = [bytes.as_slice(), bytes.as_slice()]; - - let surfaces = Codec::decode_tiles_to_device( - &mut ctx, - &mut pool, - &inputs, - PixelFormat::Gray8, - BackendRequest::Auto, - ) - .expect("batch surfaces"); - - assert_eq!(surfaces.len(), inputs.len()); - let mut expected = [0u8; 16]; - J2kDecoder::new(&bytes) - .expect("host decoder") - .decode_into(&mut expected, 4, PixelFormat::Gray8) - .expect("host decode"); - for surface in surfaces { - assert_eq!(surface.dimensions(), (4, 4)); - match surface.backend_kind() { - signinum_core::BackendKind::Cpu => { - assert_eq!(surface.as_host_bytes(), Some(expected.as_slice())); - } - signinum_core::BackendKind::Cuda => { - let mut downloaded = vec![0u8; surface.byte_len()]; - surface - .download_into(&mut downloaded, surface.pitch_bytes()) - .expect("download cuda surface"); - assert_eq!(downloaded, expected); - } - signinum_core::BackendKind::Metal => panic!("J2K CUDA batch returned Metal surface"), - } - } -} - -#[test] -fn decode_tiles_to_device_explicit_cuda_returns_cuda_surfaces_or_clear_unavailable_error() { - let bytes = fixture_ht_gray8(); - let mut ctx = DecoderContext::::new(); - let mut pool = signinum_j2k_cuda::J2kScratchPool::new(); - let inputs = [bytes.as_slice(), bytes.as_slice()]; - - match Codec::decode_tiles_to_device( - &mut ctx, - &mut pool, - &inputs, - PixelFormat::Gray8, - BackendRequest::Cuda, - ) { - Ok(surfaces) => { - assert_eq!(surfaces.len(), inputs.len()); - for surface in surfaces { - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cuda); - assert_eq!(surface.as_host_bytes(), None); - assert_cuda_surface(&surface); - } - } - Err(error) => assert!(error.is_unsupported()), - } -} diff --git a/crates/signinum-j2k-metal/Cargo.toml b/crates/signinum-j2k-metal/Cargo.toml deleted file mode 100644 index fabc3404..00000000 --- a/crates/signinum-j2k-metal/Cargo.toml +++ /dev/null @@ -1,62 +0,0 @@ -[package] -name = "signinum-j2k-metal" -description = "Metal device-output adapter for signinum-j2k" -version = "0.4.2" -edition.workspace = true -rust-version.workspace = true -license.workspace = true -repository.workspace = true -keywords.workspace = true -categories.workspace = true -readme = "README.md" - -[lib] -name = "signinum_j2k_metal" -path = "src/lib.rs" - -[dependencies] -signinum-core = { path = "../signinum-core", version = "=0.4.2" } -signinum-j2k = { path = "../signinum-j2k", version = "=0.4.2" } -signinum-j2k-native = { path = "../signinum-j2k-native", version = "=0.4.2" } -signinum-profile = { path = "../signinum-profile", version = "=0.4.2" } -thiserror = { workspace = true } - -[target.'cfg(target_os = "macos")'.dependencies] -libc = "0.2" -metal = "0.31" -rayon = "1" - -[dev-dependencies] -criterion = { workspace = true } -rayon = "1" -signinum-j2k-compare = { path = "../signinum-j2k-compare" } -signinum-test-support = { path = "../signinum-test-support" } - -[[bench]] -name = "device_upload" -harness = false - -[[bench]] -name = "compare" -harness = false - -[[bench]] -name = "encode_stages" -harness = false - -[lints.rust] -unsafe_code = "allow" -unsafe_op_in_unsafe_fn = "deny" -unreachable_pub = "warn" - -[lints.clippy] -pedantic = { level = "warn", priority = -1 } -module_name_repetitions = "allow" -must_use_candidate = "allow" -missing_errors_doc = "allow" -missing_panics_doc = "allow" -too_many_lines = "allow" -cast_possible_truncation = "allow" -cast_sign_loss = "allow" -cast_possible_wrap = "allow" -doc_markdown = "allow" diff --git a/crates/signinum-j2k-metal/README.md b/crates/signinum-j2k-metal/README.md deleted file mode 100644 index c201ffd4..00000000 --- a/crates/signinum-j2k-metal/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# signinum-j2k-metal - -Apple Metal device-output adapter for `signinum-j2k`. - -Install this crate when a macOS pipeline needs JPEG 2000 / HTJ2K output as a -Metal-backed `DeviceSurface`: - -```sh -cargo add signinum-j2k-metal -``` - -The adapter exposes full, ROI, reduced-resolution, and combined -ROI+reduced-resolution device surfaces. `BackendRequest::Auto` may choose a -validated Metal path for supported shapes and otherwise returns host-backed CPU -output. `BackendRequest::Metal` is strict: it returns resident Metal decode -surfaces only, and reports unsupported or unavailable Metal requests as errors. -Use the explicit `decode_*_cpu_staged_metal_surface_with_session` APIs when -CPU-decoded bytes need to be uploaded into a Metal buffer. - -The stable CPU decode API lives in `signinum-j2k`. This adapter remains -pre-1.0 while runtime validation and routing policies continue to harden. diff --git a/crates/signinum-j2k-metal/benches/common/mod.rs b/crates/signinum-j2k-metal/benches/common/mod.rs deleted file mode 100644 index e680d3f3..00000000 --- a/crates/signinum-j2k-metal/benches/common/mod.rs +++ /dev/null @@ -1,1567 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#![allow(dead_code)] - -use criterion::black_box; -use rayon::prelude::*; -use signinum_core::{ - BackendRequest, DeviceSubmission, ImageDecodeDevice, TileBatchDecodeDevice, - TileBatchDecodeSubmit, -}; -use signinum_j2k::{ - decode_tiles_into, decode_tiles_region_scaled_into, CompressedTransferSyntax, DecoderContext, - Downscale, J2kContext, J2kDecoder, J2kScratchPool, PixelFormat, Rect, TileBatchOptions, - TileDecodeJob, TileRegionScaledDecodeJob, -}; -use signinum_j2k_compare::{grok, openjpeg}; -use signinum_j2k_metal::{ - extract_dicom_encapsulated_frames_with_limit, Codec as MetalJ2kCodec, - J2kDecoder as MetalJ2kDecoder, J2kScratchPool as MetalJ2kScratchPool, MetalSession, -}; -use signinum_j2k_native::{encode, encode_htj2k, EncodeOptions}; -use std::{ - collections::BTreeSet, - env, fs, - path::{Path, PathBuf}, -}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum DecodeMode { - Gray8, - Gray16, - Rgb8, -} - -#[derive(Clone, Debug)] -pub(crate) struct BenchInput { - pub name: &'static str, - pub bytes: Vec, - pub dimensions: (u32, u32), - pub mode: DecodeMode, - pub is_ht: bool, -} - -#[derive(Clone, Debug)] -pub(crate) struct ExternalTileBatch { - pub name: String, - pub inputs: Vec>, - pub dimensions: Vec<(u32, u32)>, - pub mode: DecodeMode, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum ExternalCodecFamily { - J2k, - Htj2k, - Unknown, -} - -const AUTO_REPEATED_GRAYSCALE_MIN_DIM: u32 = 512; -const AUTO_REPEATED_GRAYSCALE_MIN_COUNT: usize = 16; -const EXTERNAL_WSI_TILE_DIR_ENV: &str = "SIGNINUM_J2K_METAL_WSI_TILE_DIR"; -const J2K_TILE_BATCH_SIZES_ENV: &str = "SIGNINUM_J2K_TILE_BATCH_SIZES"; -const J2K_REGION_EDGES_ENV: &str = "SIGNINUM_J2K_REGION_EDGES"; -const DEFAULT_J2K_TILE_BATCH_SIZES: &[usize] = &[16, 32, 64, 128]; -const DEFAULT_J2K_REGION_EDGES: &[u32] = &[256]; - -pub(crate) fn j2k_tile_batch_sizes() -> Vec { - env::var(J2K_TILE_BATCH_SIZES_ENV) - .ok() - .map_or_else(default_j2k_tile_batch_sizes, |raw| { - let parsed = raw - .split(',') - .filter_map(|value| value.trim().parse::().ok()) - .filter(|&value| value > 0) - .collect::>(); - if parsed.is_empty() { - default_j2k_tile_batch_sizes() - } else { - parsed - } - }) -} - -fn default_j2k_tile_batch_sizes() -> Vec { - DEFAULT_J2K_TILE_BATCH_SIZES.to_vec() -} - -pub(crate) fn j2k_region_edges() -> Vec { - env::var(J2K_REGION_EDGES_ENV) - .ok() - .map_or_else(default_j2k_region_edges, |raw| { - let parsed = raw - .split(',') - .filter_map(|value| value.trim().parse::().ok()) - .filter(|&value| value > 0) - .collect::>(); - if parsed.is_empty() { - default_j2k_region_edges() - } else { - parsed - } - }) -} - -fn default_j2k_region_edges() -> Vec { - DEFAULT_J2K_REGION_EDGES.to_vec() -} - -pub(crate) fn bench_inputs() -> Vec { - let mut inputs = vec![ - BenchInput { - name: "j2k_gray_1024", - bytes: classic_bench_bytes( - "j2k_gray_1024", - &signinum_test_support::gradient_u8(1024, 1024, 1), - 1024, - 1024, - DecodeMode::Gray8, - ), - dimensions: (1024, 1024), - mode: DecodeMode::Gray8, - is_ht: false, - }, - BenchInput { - name: "j2k_gray_512", - bytes: classic_bench_bytes( - "j2k_gray_512", - &signinum_test_support::gradient_u8(512, 512, 1), - 512, - 512, - DecodeMode::Gray8, - ), - dimensions: (512, 512), - mode: DecodeMode::Gray8, - is_ht: false, - }, - BenchInput { - name: "j2k_rgb_1024", - bytes: classic_bench_bytes( - "j2k_rgb_1024", - &signinum_test_support::gradient_u8(1024, 1024, 3), - 1024, - 1024, - DecodeMode::Rgb8, - ), - dimensions: (1024, 1024), - mode: DecodeMode::Rgb8, - is_ht: false, - }, - BenchInput { - name: "j2k_rgb_256", - bytes: classic_bench_bytes( - "j2k_rgb_256", - &signinum_test_support::gradient_u8(256, 256, 3), - 256, - 256, - DecodeMode::Rgb8, - ), - dimensions: (256, 256), - mode: DecodeMode::Rgb8, - is_ht: false, - }, - ]; - - inputs.extend(ht_bench_inputs()); - - inputs -} - -pub(crate) fn external_wsi_tile_batches(max_count: usize) -> Vec { - let Some(root) = external_tile_root() else { - eprintln!( - "skipping external WSI J2K tile benchmarks: set {EXTERNAL_WSI_TILE_DIR_ENV} to a directory of JP2/J2K/JPH/JHC tiles or DICOM WSI files" - ); - return Vec::new(); - }; - - let mut paths = collect_external_tile_paths(&root); - if paths.is_empty() { - eprintln!( - "skipping external WSI J2K tile benchmarks: no JP2/J2K/JPH/JHC tiles or DICOM WSI files found under {}", - root.display() - ); - return Vec::new(); - } - paths.sort(); - - let mut j2k_gray8 = Vec::new(); - let mut j2k_gray16 = Vec::new(); - let mut j2k_rgb8 = Vec::new(); - let mut htj2k_gray8 = Vec::new(); - let mut htj2k_gray16 = Vec::new(); - let mut htj2k_rgb8 = Vec::new(); - let mut unknown_gray8 = Vec::new(); - let mut unknown_gray16 = Vec::new(); - let mut unknown_rgb8 = Vec::new(); - for path in paths { - let Ok(source_bytes) = fs::read(&path) else { - eprintln!( - "skipping unreadable external WSI source: {}", - path.display() - ); - continue; - }; - let frames = external_source_frames(&path, source_bytes, max_count); - for bytes in frames { - let Ok(info) = J2kDecoder::inspect(&bytes) else { - eprintln!( - "skipping unsupported external J2K tile/frame: {}", - path.display() - ); - continue; - }; - let Some(mode) = external_decode_mode(info.components, info.bit_depth) else { - continue; - }; - let family = external_codec_family(&bytes); - let entry = (bytes, info.dimensions); - match (family, mode) { - (ExternalCodecFamily::J2k, DecodeMode::Gray8) => j2k_gray8.push(entry), - (ExternalCodecFamily::J2k, DecodeMode::Gray16) => j2k_gray16.push(entry), - (ExternalCodecFamily::J2k, DecodeMode::Rgb8) => j2k_rgb8.push(entry), - (ExternalCodecFamily::Htj2k, DecodeMode::Gray8) => htj2k_gray8.push(entry), - (ExternalCodecFamily::Htj2k, DecodeMode::Gray16) => htj2k_gray16.push(entry), - (ExternalCodecFamily::Htj2k, DecodeMode::Rgb8) => htj2k_rgb8.push(entry), - (ExternalCodecFamily::Unknown, DecodeMode::Gray8) => unknown_gray8.push(entry), - (ExternalCodecFamily::Unknown, DecodeMode::Gray16) => unknown_gray16.push(entry), - (ExternalCodecFamily::Unknown, DecodeMode::Rgb8) => unknown_rgb8.push(entry), - } - } - } - - [ - external_tile_batch(&root, "j2k_gray8", DecodeMode::Gray8, j2k_gray8, max_count), - external_tile_batch( - &root, - "j2k_gray16", - DecodeMode::Gray16, - j2k_gray16, - max_count, - ), - external_tile_batch(&root, "j2k_rgb8", DecodeMode::Rgb8, j2k_rgb8, max_count), - external_tile_batch( - &root, - "htj2k_gray8", - DecodeMode::Gray8, - htj2k_gray8, - max_count, - ), - external_tile_batch( - &root, - "htj2k_gray16", - DecodeMode::Gray16, - htj2k_gray16, - max_count, - ), - external_tile_batch(&root, "htj2k_rgb8", DecodeMode::Rgb8, htj2k_rgb8, max_count), - external_tile_batch( - &root, - "j2k_unknown_gray8", - DecodeMode::Gray8, - unknown_gray8, - max_count, - ), - external_tile_batch( - &root, - "j2k_unknown_gray16", - DecodeMode::Gray16, - unknown_gray16, - max_count, - ), - external_tile_batch( - &root, - "j2k_unknown_rgb8", - DecodeMode::Rgb8, - unknown_rgb8, - max_count, - ), - ] - .into_iter() - .flatten() - .collect() -} - -fn external_tile_root() -> Option { - let raw = env::var_os(EXTERNAL_WSI_TILE_DIR_ENV)?; - if raw.is_empty() { - return None; - } - let root = PathBuf::from(raw); - if root.is_dir() { - Some(root) - } else { - eprintln!( - "skipping external WSI J2K tile benchmarks: {} is not a directory", - root.display() - ); - None - } -} - -fn collect_external_tile_paths(root: &Path) -> Vec { - let mut out = Vec::new(); - let mut pending = vec![root.to_path_buf()]; - while let Some(dir) = pending.pop() { - let Ok(entries) = fs::read_dir(&dir) else { - eprintln!( - "skipping unreadable external tile directory: {}", - dir.display() - ); - continue; - }; - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - pending.push(path); - } else if is_external_wsi_source_path(&path) { - out.push(path); - } - } - } - out -} - -fn is_external_wsi_source_path(path: &Path) -> bool { - path.extension() - .and_then(|ext| ext.to_str()) - .is_some_and(|ext| { - matches!( - ext.to_ascii_lowercase().as_str(), - "jp2" | "j2k" | "j2c" | "jpc" | "jph" | "jhc" | "dcm" | "dicom" - ) - }) -} - -fn external_source_frames(path: &Path, bytes: Vec, max_count: usize) -> Vec> { - if is_dicom_path(path) { - match extract_dicom_encapsulated_frames_with_limit(&bytes, max_count) { - Ok(frames) => frames, - Err(error) => { - eprintln!( - "skipping external DICOM WSI source {}: {error}", - path.display() - ); - Vec::new() - } - } - } else { - vec![bytes] - } -} - -fn is_dicom_path(path: &Path) -> bool { - path.extension() - .and_then(|ext| ext.to_str()) - .is_some_and(|ext| matches!(ext.to_ascii_lowercase().as_str(), "dcm" | "dicom")) -} - -fn external_decode_mode(components: u8, bit_depth: u8) -> Option { - match (components, bit_depth) { - (1, 1..=8) => Some(DecodeMode::Gray8), - (1, 9..=16) => Some(DecodeMode::Gray16), - (3, 1..=8) => Some(DecodeMode::Rgb8), - _ => None, - } -} - -fn external_codec_family(bytes: &[u8]) -> ExternalCodecFamily { - let Ok(decoder) = J2kDecoder::new(bytes) else { - return ExternalCodecFamily::Unknown; - }; - let Some(candidate) = decoder.passthrough_candidate() else { - return ExternalCodecFamily::Unknown; - }; - match candidate.transfer_syntax() { - CompressedTransferSyntax::Jpeg2000Lossless | CompressedTransferSyntax::Jpeg2000Lossy => { - ExternalCodecFamily::J2k - } - CompressedTransferSyntax::HtJpeg2000Lossless - | CompressedTransferSyntax::HtJpeg2000Lossy => ExternalCodecFamily::Htj2k, - _ => ExternalCodecFamily::Unknown, - } -} - -fn external_tile_batch( - root: &Path, - label: &str, - mode: DecodeMode, - entries: Vec<(Vec, (u32, u32))>, - max_count: usize, -) -> Option { - if entries.is_empty() || max_count == 0 { - return None; - } - - let count = entries.len().min(max_count); - let (inputs, dimensions): (Vec<_>, Vec<_>) = entries.into_iter().take(count).unzip(); - let distinct_dims = dimensions.iter().copied().collect::>().len(); - let root_name = root - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("external_wsi"); - Some(ExternalTileBatch { - name: format!("{root_name}_{label}_{count}tiles_{distinct_dims}dims"), - inputs, - dimensions, - mode, - }) -} - -fn ht_bench_inputs() -> Vec { - let candidates = [ - ("htj2k_gray_1024", 1024_u32, 1024_u32), - ("htj2k_gray_512", 512_u32, 512_u32), - ]; - - let mut inputs = Vec::with_capacity(candidates.len()); - let mut errors = Vec::new(); - for (name, width, height) in candidates { - let pixels = ht_bench_pixels(width, height, 1); - match try_encode_ht(&pixels, width, height, 1, 8) { - Ok(codestream) => inputs.push(BenchInput { - name, - bytes: wrap_codestream_jp2(&codestream, width, height, 1, 8, 17), - dimensions: (width, height), - mode: DecodeMode::Gray8, - is_ht: true, - }), - Err(error) => errors.push(format!("{name}: {error}")), - } - } - - if inputs.is_empty() { - eprintln!( - "skipping HTJ2K bench inputs: {}", - errors - .last() - .map_or("no HTJ2K benchmark candidate succeeded", String::as_str) - ); - } - inputs -} - -fn ht_bench_pixels(width: u32, height: u32, channels: usize) -> Vec { - let mut out = Vec::with_capacity(width as usize * height as usize * channels); - let width_denom = width.saturating_sub(1).max(1); - let height_denom = height.saturating_sub(1).max(1); - for y in 0..height { - let y_base = (y * 29) / height_denom; - for x in 0..width { - let x_base = (x * 31) / width_denom; - for c in 0..channels { - out.push((x_base + y_base + c as u32 * 17) as u8); - } - } - } - out -} - -pub(crate) fn signinum_inspect(bytes: &[u8]) { - black_box(J2kDecoder::inspect(bytes).expect("signinum inspect")); -} - -pub(crate) fn signinum_decode(bytes: &[u8], mode: DecodeMode) { - let mut decoder = J2kDecoder::new(bytes).expect("signinum decoder"); - let info = decoder.info().dimensions; - let (fmt, stride) = mode_geometry(mode, info); - let mut out = vec![0_u8; stride * info.1 as usize]; - decoder - .decode_into(&mut out, stride, fmt) - .expect("signinum decode"); - black_box(out); -} - -pub(crate) fn signinum_decode_region(bytes: &[u8], mode: DecodeMode, edge: u32) { - let mut decoder = J2kDecoder::new(bytes).expect("signinum decoder"); - let roi = centered_roi(decoder.info().dimensions, edge); - let fmt = mode_format(mode); - let stride = roi.w as usize * fmt.bytes_per_pixel(); - let mut pool = J2kScratchPool::new(); - let mut out = vec![0_u8; stride * roi.h as usize]; - decoder - .decode_region_into(&mut pool, &mut out, stride, fmt, roi) - .expect("signinum region decode"); - black_box(out); -} - -pub(crate) fn signinum_decode_scaled(bytes: &[u8], mode: DecodeMode, scale: Downscale) { - let mut decoder = J2kDecoder::new(bytes).expect("signinum decoder"); - let dims = scaled_dims(decoder.info().dimensions, scale); - let fmt = mode_format(mode); - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut pool = J2kScratchPool::new(); - let mut out = vec![0_u8; stride * dims.1 as usize]; - decoder - .decode_scaled_into(&mut pool, &mut out, stride, fmt, scale) - .expect("signinum scaled decode"); - black_box(out); -} - -pub(crate) fn signinum_decode_region_scaled( - bytes: &[u8], - mode: DecodeMode, - edge: u32, - scale: Downscale, -) { - let mut decoder = J2kDecoder::new(bytes).expect("signinum decoder"); - let roi = centered_roi(decoder.info().dimensions, edge); - let scaled = roi.scaled_covering(scale); - let fmt = mode_format(mode); - let stride = scaled.w as usize * fmt.bytes_per_pixel(); - let mut pool = J2kScratchPool::new(); - let mut out = vec![0_u8; stride * scaled.h as usize]; - decoder - .decode_region_scaled_into(&mut pool, &mut out, stride, fmt, roi, scale) - .expect("signinum region scaled decode"); - black_box(out); -} - -pub(crate) fn signinum_decode_tile_batch(bytes: &[u8], mode: DecodeMode, count: usize) { - let decoder = J2kDecoder::new(bytes).expect("signinum decoder"); - let dims = decoder.info().dimensions; - let (fmt, stride) = mode_geometry(mode, dims); - let mut outputs = (0..count) - .map(|_| vec![0_u8; stride * dims.1 as usize]) - .collect::>(); - let outcomes = { - let mut jobs = outputs - .iter_mut() - .map(|out| TileDecodeJob { - input: bytes, - out: out.as_mut_slice(), - stride, - }) - .collect::>(); - decode_tiles_into(&mut jobs, fmt, TileBatchOptions::default()).expect("tile decode") - }; - black_box((outputs, outcomes)); -} - -pub(crate) fn openjpeg_decode_tile_batch(bytes: &[u8], mode: DecodeMode, count: usize) { - let outputs = (0..count) - .into_par_iter() - .map(|_| match mode { - DecodeMode::Gray8 => openjpeg::decode_gray(bytes), - DecodeMode::Rgb8 => openjpeg::decode_rgb(bytes), - DecodeMode::Gray16 => { - Err("openjpeg: Gray16 benchmark output is not implemented".to_string()) - } - }) - .collect::, _>>() - .expect("OpenJPEG tile batch decode"); - black_box(outputs); -} - -pub(crate) fn signinum_decode_tile_batch_region_scaled( - bytes: &[u8], - mode: DecodeMode, - edge: u32, - scale: Downscale, - count: usize, -) { - let decoder = J2kDecoder::new(bytes).expect("signinum decoder"); - let roi = centered_roi(decoder.info().dimensions, edge); - let scaled = roi.scaled_covering(scale); - let fmt = mode_format(mode); - let stride = scaled.w as usize * fmt.bytes_per_pixel(); - let mut outputs = (0..count) - .map(|_| vec![0_u8; stride * scaled.h as usize]) - .collect::>(); - let outcomes = { - let mut jobs = outputs - .iter_mut() - .map(|out| TileRegionScaledDecodeJob { - input: bytes, - out: out.as_mut_slice(), - stride, - roi, - scale, - }) - .collect::>(); - decode_tiles_region_scaled_into(&mut jobs, fmt, TileBatchOptions::default()) - .expect("tile region scaled decode") - }; - black_box((outputs, outcomes)); -} - -pub(crate) fn distinct_rgb_tile_batch_inputs(input: &BenchInput, count: usize) -> Vec> { - assert_eq!(input.mode, DecodeMode::Rgb8); - (0..count) - .map(|index| { - let name = format!("{}_distinct_{index}", input.name); - classic_bench_bytes( - &name, - &signinum_test_support::gradient_variant_u8( - input.dimensions.0, - input.dimensions.1, - 3, - index as u32, - ), - input.dimensions.0, - input.dimensions.1, - input.mode, - ) - }) - .collect() -} - -pub(crate) fn distinct_gray_tile_batch_inputs(input: &BenchInput, count: usize) -> Vec> { - assert_eq!(input.mode, DecodeMode::Gray8); - (0..count) - .map(|index| { - let pixels = signinum_test_support::gradient_variant_u8( - input.dimensions.0, - input.dimensions.1, - 1, - index as u32, - ); - if input.is_ht { - wrap_codestream_jp2( - &try_encode_ht(&pixels, input.dimensions.0, input.dimensions.1, 1, 8) - .expect("encode distinct HTJ2K grayscale benchmark tile"), - input.dimensions.0, - input.dimensions.1, - 1, - 8, - 17, - ) - } else { - let name = format!("{}_distinct_{index}", input.name); - classic_bench_bytes( - &name, - &pixels, - input.dimensions.0, - input.dimensions.1, - input.mode, - ) - } - }) - .collect() -} - -pub(crate) fn signinum_decode_tile_batch_distinct(inputs: &[Vec], mode: DecodeMode) { - let Some(first) = inputs.first() else { - return; - }; - let decoder = J2kDecoder::new(first).expect("signinum decoder"); - let dims = decoder.info().dimensions; - let (fmt, stride) = mode_geometry(mode, dims); - let mut outputs = inputs - .iter() - .map(|_| vec![0_u8; stride * dims.1 as usize]) - .collect::>(); - let outcomes = { - let mut jobs = inputs - .iter() - .zip(outputs.iter_mut()) - .map(|(bytes, out)| TileDecodeJob { - input: bytes, - out: out.as_mut_slice(), - stride, - }) - .collect::>(); - decode_tiles_into(&mut jobs, fmt, TileBatchOptions::default()).expect("tile decode") - }; - black_box((outputs, outcomes)); -} - -pub(crate) fn openjpeg_decode_tile_batch_distinct(inputs: &[Vec], mode: DecodeMode) { - let outputs = inputs - .par_iter() - .map(|bytes| match mode { - DecodeMode::Gray8 => openjpeg::decode_gray(bytes), - DecodeMode::Rgb8 => openjpeg::decode_rgb(bytes), - DecodeMode::Gray16 => { - Err("openjpeg: Gray16 benchmark output is not implemented".to_string()) - } - }) - .collect::, _>>() - .expect("OpenJPEG distinct tile batch decode"); - black_box(outputs); -} - -pub(crate) fn signinum_decode_tile_batch_region_scaled_distinct( - inputs: &[Vec], - mode: DecodeMode, - edge: u32, - scale: Downscale, -) { - let Some(first) = inputs.first() else { - return; - }; - let decoder = J2kDecoder::new(first).expect("signinum decoder"); - let roi = centered_roi(decoder.info().dimensions, edge); - let scaled = roi.scaled_covering(scale); - let fmt = mode_format(mode); - let stride = scaled.w as usize * fmt.bytes_per_pixel(); - let mut outputs = inputs - .iter() - .map(|_| vec![0_u8; stride * scaled.h as usize]) - .collect::>(); - let outcomes = { - let mut jobs = inputs - .iter() - .zip(outputs.iter_mut()) - .map(|(bytes, out)| TileRegionScaledDecodeJob { - input: bytes, - out: out.as_mut_slice(), - stride, - roi, - scale, - }) - .collect::>(); - decode_tiles_region_scaled_into(&mut jobs, fmt, TileBatchOptions::default()) - .expect("tile region scaled decode") - }; - black_box((outputs, outcomes)); -} - -pub(crate) fn signinum_decode_external_tile_batch_region_scaled( - batch: &ExternalTileBatch, - count: usize, - edge: u32, - scale: Downscale, -) { - let count = count.min(batch.inputs.len()); - if count == 0 { - return; - } - - let fmt = mode_format(batch.mode); - let (rois, stride, height) = external_batch_output_geometry(batch, count, edge, scale, fmt); - let mut outputs = (0..count) - .map(|_| vec![0_u8; stride * height as usize]) - .collect::>(); - let outcomes = { - let mut jobs = batch - .inputs - .iter() - .zip(rois.iter()) - .take(count) - .zip(outputs.iter_mut()) - .map(|((bytes, roi), out)| TileRegionScaledDecodeJob { - input: bytes, - out: out.as_mut_slice(), - stride, - roi: *roi, - scale, - }) - .collect::>(); - decode_tiles_region_scaled_into(&mut jobs, fmt, TileBatchOptions::default()) - .expect("external tile region scaled decode") - }; - black_box((outputs, outcomes)); -} - -pub(crate) fn openjpeg_decode_external_tile_batch_region_scaled( - batch: &ExternalTileBatch, - count: usize, - edge: u32, - scale: Downscale, -) { - let count = count.min(batch.inputs.len()); - if count == 0 { - return; - } - - let reduce = downscale_reduction_factor(scale); - let (rois, _, _) = - external_batch_output_geometry(batch, count, edge, scale, mode_format(batch.mode)); - let outputs = batch - .inputs - .par_iter() - .zip(rois.par_iter()) - .take(count) - .map(|(bytes, roi)| match batch.mode { - DecodeMode::Gray8 => openjpeg::decode_gray_region_scaled(bytes, *roi, reduce), - DecodeMode::Rgb8 => openjpeg::decode_rgb_region_scaled(bytes, *roi, reduce), - DecodeMode::Gray16 => { - Err("openjpeg: Gray16 benchmark output is not implemented".to_string()) - } - }) - .collect::, _>>() - .expect("OpenJPEG external tile region scaled decode"); - black_box(outputs); -} - -pub(crate) fn grok_decode_external_tile_batch_region_scaled( - batch: &ExternalTileBatch, - count: usize, - edge: u32, - scale: Downscale, -) { - let count = count.min(batch.inputs.len()); - if count == 0 { - return; - } - - let reduce = downscale_reduction_factor(scale); - let (rois, _, _) = - external_batch_output_geometry(batch, count, edge, scale, mode_format(batch.mode)); - let outputs = batch - .inputs - .par_iter() - .zip(rois.par_iter()) - .take(count) - .map(|(bytes, roi)| match batch.mode { - DecodeMode::Gray8 => grok::decode_gray_region_scaled(bytes, *roi, reduce), - DecodeMode::Rgb8 => grok::decode_rgb_region_scaled(bytes, *roi, reduce), - DecodeMode::Gray16 => { - Err("grok: Gray16 benchmark output is not implemented".to_string()) - } - }) - .collect::, _>>() - .expect("Grok external tile region scaled decode"); - black_box(outputs); -} - -pub(crate) fn metal_available() -> bool { - cfg!(target_os = "macos") -} - -pub(crate) fn signinum_metal_decode(bytes: &[u8], mode: DecodeMode) { - let mut decoder = MetalJ2kDecoder::new(bytes).expect("signinum metal decoder"); - let surface = decoder - .decode_to_device(mode_format(mode), BackendRequest::Metal) - .expect("signinum metal decode"); - black_box(surface); -} - -pub(crate) fn signinum_adaptive_decode(bytes: &[u8], mode: DecodeMode) { - signinum_decode(bytes, mode); -} - -pub(crate) fn signinum_metal_supports_decode(bytes: &[u8], mode: DecodeMode) -> bool { - let mut decoder = MetalJ2kDecoder::new(bytes).expect("signinum metal decoder"); - decoder - .decode_to_device(mode_format(mode), BackendRequest::Metal) - .is_ok() -} - -pub(crate) fn signinum_metal_decode_region(bytes: &[u8], mode: DecodeMode, edge: u32) { - let cpu_decoder = J2kDecoder::new(bytes).expect("signinum decoder"); - let roi = centered_roi(cpu_decoder.info().dimensions, edge); - let mut decoder = MetalJ2kDecoder::new(bytes).expect("signinum metal decoder"); - let surface = decoder - .decode_region_to_device(mode_format(mode), roi, BackendRequest::Metal) - .expect("signinum metal region decode"); - black_box(surface); -} - -pub(crate) fn signinum_adaptive_decode_region(bytes: &[u8], mode: DecodeMode, edge: u32) { - signinum_decode_region(bytes, mode, edge); -} - -pub(crate) fn signinum_metal_supports_region(bytes: &[u8], mode: DecodeMode, edge: u32) -> bool { - let cpu_decoder = J2kDecoder::new(bytes).expect("signinum decoder"); - let roi = centered_roi(cpu_decoder.info().dimensions, edge); - let mut decoder = MetalJ2kDecoder::new(bytes).expect("signinum metal decoder"); - decoder - .decode_region_to_device(mode_format(mode), roi, BackendRequest::Metal) - .is_ok() -} - -pub(crate) fn signinum_metal_decode_scaled(bytes: &[u8], mode: DecodeMode, scale: Downscale) { - let mut decoder = MetalJ2kDecoder::new(bytes).expect("signinum metal decoder"); - let surface = decoder - .decode_scaled_to_device(mode_format(mode), scale, BackendRequest::Metal) - .expect("signinum metal scaled decode"); - black_box(surface); -} - -pub(crate) fn signinum_adaptive_decode_scaled(bytes: &[u8], mode: DecodeMode, scale: Downscale) { - signinum_decode_scaled(bytes, mode, scale); -} - -pub(crate) fn signinum_metal_decode_region_scaled( - bytes: &[u8], - mode: DecodeMode, - edge: u32, - scale: Downscale, -) { - let cpu_decoder = J2kDecoder::new(bytes).expect("signinum decoder"); - let roi = centered_roi(cpu_decoder.info().dimensions, edge); - let mut decoder = MetalJ2kDecoder::new(bytes).expect("signinum metal decoder"); - let surface = decoder - .decode_region_scaled_to_device(mode_format(mode), roi, scale, BackendRequest::Metal) - .expect("signinum metal region scaled decode"); - black_box(surface); -} - -pub(crate) fn signinum_adaptive_decode_region_scaled( - bytes: &[u8], - mode: DecodeMode, - edge: u32, - scale: Downscale, -) { - signinum_decode_region_scaled(bytes, mode, edge, scale); -} - -pub(crate) fn signinum_metal_supports_scaled( - bytes: &[u8], - mode: DecodeMode, - scale: Downscale, -) -> bool { - let mut decoder = MetalJ2kDecoder::new(bytes).expect("signinum metal decoder"); - decoder - .decode_scaled_to_device(mode_format(mode), scale, BackendRequest::Metal) - .is_ok() -} - -pub(crate) fn signinum_metal_supports_region_scaled( - bytes: &[u8], - mode: DecodeMode, - edge: u32, - scale: Downscale, -) -> bool { - if !supports_metal_region_scaled_mode(mode) { - return false; - } - let cpu_decoder = J2kDecoder::new(bytes).expect("signinum decoder"); - let roi = centered_roi(cpu_decoder.info().dimensions, edge); - let mut decoder = MetalJ2kDecoder::new(bytes).expect("signinum metal decoder"); - decoder - .decode_region_scaled_to_device(mode_format(mode), roi, scale, BackendRequest::Metal) - .is_ok() -} - -pub(crate) fn signinum_metal_decode_tile_batch(bytes: &[u8], mode: DecodeMode, count: usize) { - let mut ctx = DecoderContext::::new(); - let mut session = MetalSession::default(); - let mut pool = MetalJ2kScratchPool::new(); - let submissions = (0..count) - .map(|_| { - MetalJ2kCodec::submit_tile_to_device( - &mut ctx, - &mut session, - &mut pool, - bytes, - mode_format(mode), - BackendRequest::Metal, - ) - .expect("signinum metal tile submit") - }) - .collect::>(); - let surfaces = submissions - .into_iter() - .map(|submission| submission.wait().expect("signinum metal tile decode")) - .collect::>(); - black_box(surfaces); -} - -pub(crate) fn signinum_metal_decode_tile_batch_region_scaled( - bytes: &[u8], - mode: DecodeMode, - edge: u32, - scale: Downscale, - count: usize, -) { - let cpu_decoder = J2kDecoder::new(bytes).expect("signinum decoder"); - let roi = centered_roi(cpu_decoder.info().dimensions, edge); - let mut ctx = DecoderContext::::new(); - let mut session = MetalSession::default(); - let mut pool = MetalJ2kScratchPool::new(); - let submissions = (0..count) - .map(|_| { - MetalJ2kCodec::submit_tile_region_scaled_to_device( - &mut ctx, - &mut session, - &mut pool, - bytes, - mode_format(mode), - roi, - scale, - BackendRequest::Metal, - ) - .expect("signinum metal tile region scaled submit") - }) - .collect::>(); - let surfaces = submissions - .into_iter() - .map(|submission| { - submission - .wait() - .expect("signinum metal tile region scaled decode") - }) - .collect::>(); - black_box(surfaces); -} - -pub(crate) fn signinum_metal_decode_tile_batch_distinct(inputs: &[Vec], mode: DecodeMode) { - let mut ctx = DecoderContext::::new(); - let mut session = MetalSession::default(); - let mut pool = MetalJ2kScratchPool::new(); - let submissions = inputs - .iter() - .map(|bytes| { - MetalJ2kCodec::submit_tile_to_device( - &mut ctx, - &mut session, - &mut pool, - bytes, - mode_format(mode), - BackendRequest::Metal, - ) - .expect("signinum metal tile submit") - }) - .collect::>(); - let surfaces = submissions - .into_iter() - .map(|submission| submission.wait().expect("signinum metal tile decode")) - .collect::>(); - black_box(surfaces); -} - -pub(crate) fn signinum_metal_decode_tile_batch_region_scaled_distinct( - inputs: &[Vec], - mode: DecodeMode, - edge: u32, - scale: Downscale, -) { - let Some(first) = inputs.first() else { - return; - }; - let cpu_decoder = J2kDecoder::new(first).expect("signinum decoder"); - let roi = centered_roi(cpu_decoder.info().dimensions, edge); - let mut ctx = DecoderContext::::new(); - let mut session = MetalSession::default(); - let mut pool = MetalJ2kScratchPool::new(); - let submissions = inputs - .iter() - .map(|bytes| { - MetalJ2kCodec::submit_tile_region_scaled_to_device( - &mut ctx, - &mut session, - &mut pool, - bytes, - mode_format(mode), - roi, - scale, - BackendRequest::Metal, - ) - .expect("signinum metal tile region scaled submit") - }) - .collect::>(); - let surfaces = submissions - .into_iter() - .map(|submission| { - submission - .wait() - .expect("signinum metal tile region scaled decode") - }) - .collect::>(); - black_box(surfaces); -} - -pub(crate) fn signinum_metal_decode_external_tile_batch_region_scaled( - batch: &ExternalTileBatch, - count: usize, - edge: u32, - scale: Downscale, -) { - let count = count.min(batch.inputs.len()); - if count == 0 { - return; - } - - let fmt = mode_format(batch.mode); - let (rois, _, _) = external_batch_output_geometry(batch, count, edge, scale, fmt); - let mut ctx = DecoderContext::::new(); - let mut session = MetalSession::default(); - let mut pool = MetalJ2kScratchPool::new(); - let submissions = batch - .inputs - .iter() - .zip(rois.iter()) - .take(count) - .map(|(bytes, roi)| { - MetalJ2kCodec::submit_tile_region_scaled_to_device( - &mut ctx, - &mut session, - &mut pool, - bytes, - fmt, - *roi, - scale, - BackendRequest::Metal, - ) - .expect("signinum metal external tile region scaled submit") - }) - .collect::>(); - let surfaces = submissions - .into_iter() - .map(|submission| { - submission - .wait() - .expect("signinum metal external tile region scaled decode") - }) - .collect::>(); - black_box(surfaces); -} - -fn signinum_adaptive_decode_tile_batch_to_device(input: &BenchInput, count: usize) { - let mut ctx = DecoderContext::::new(); - let mut session = MetalSession::default(); - let mut pool = MetalJ2kScratchPool::new(); - let submissions = (0..count) - .map(|_| { - MetalJ2kCodec::submit_tile_to_device( - &mut ctx, - &mut session, - &mut pool, - &input.bytes, - mode_format(input.mode), - BackendRequest::Auto, - ) - .expect("signinum auto tile submit") - }) - .collect::>(); - let surfaces = submissions - .into_iter() - .map(|submission| submission.wait().expect("signinum auto tile decode")) - .collect::>(); - black_box(surfaces); -} - -fn signinum_adaptive_decode_tile_batch_region_scaled_to_device( - input: &BenchInput, - edge: u32, - scale: Downscale, - count: usize, -) { - let decoder = J2kDecoder::new(&input.bytes).expect("signinum decoder"); - let roi = centered_roi(decoder.info().dimensions, edge); - let mut ctx = DecoderContext::::new(); - let mut session = MetalSession::default(); - let mut pool = MetalJ2kScratchPool::new(); - let submissions = (0..count) - .map(|_| { - MetalJ2kCodec::submit_tile_region_scaled_to_device( - &mut ctx, - &mut session, - &mut pool, - &input.bytes, - mode_format(input.mode), - roi, - scale, - BackendRequest::Auto, - ) - .expect("signinum auto tile region scaled submit") - }) - .collect::>(); - let surfaces = submissions - .into_iter() - .map(|submission| { - submission - .wait() - .expect("signinum auto tile region scaled decode") - }) - .collect::>(); - black_box(surfaces); -} - -pub(crate) fn signinum_adaptive_decode_tile_batch(input: &BenchInput, count: usize) { - #[cfg(target_os = "macos")] - if should_auto_use_direct_grayscale_input(input, count) { - signinum_adaptive_decode_tile_batch_to_device(input, count); - return; - } - - signinum_decode_tile_batch(&input.bytes, input.mode, count); -} - -pub(crate) fn signinum_adaptive_decode_tile_batch_region_scaled( - input: &BenchInput, - edge: u32, - scale: Downscale, - count: usize, -) { - signinum_adaptive_decode_tile_batch_region_scaled_to_device(input, edge, scale, count); -} - -pub(crate) fn signinum_adaptive_decode_tile_batch_region_scaled_distinct( - inputs: &[Vec], - mode: DecodeMode, - edge: u32, - scale: Downscale, -) { - let Some(first) = inputs.first() else { - return; - }; - let decoder = J2kDecoder::new(first).expect("signinum decoder"); - let roi = centered_roi(decoder.info().dimensions, edge); - let mut ctx = DecoderContext::::new(); - let mut session = MetalSession::default(); - let mut pool = MetalJ2kScratchPool::new(); - let submissions = inputs - .iter() - .map(|bytes| { - MetalJ2kCodec::submit_tile_region_scaled_to_device( - &mut ctx, - &mut session, - &mut pool, - bytes, - mode_format(mode), - roi, - scale, - BackendRequest::Auto, - ) - .expect("signinum auto distinct tile region scaled submit") - }) - .collect::>(); - let surfaces = submissions - .into_iter() - .map(|submission| { - submission - .wait() - .expect("signinum auto distinct tile region scaled decode") - }) - .collect::>(); - black_box(surfaces); -} - -pub(crate) fn signinum_adaptive_decode_external_tile_batch_region_scaled( - batch: &ExternalTileBatch, - count: usize, - edge: u32, - scale: Downscale, -) { - let count = count.min(batch.inputs.len()); - if count == 0 { - return; - } - - let fmt = mode_format(batch.mode); - let (rois, _, _) = external_batch_output_geometry(batch, count, edge, scale, fmt); - let mut ctx = DecoderContext::::new(); - let mut session = MetalSession::default(); - let mut pool = MetalJ2kScratchPool::new(); - let submissions = batch - .inputs - .iter() - .zip(rois.iter()) - .take(count) - .map(|(bytes, roi)| { - MetalJ2kCodec::submit_tile_region_scaled_to_device( - &mut ctx, - &mut session, - &mut pool, - bytes, - fmt, - *roi, - scale, - BackendRequest::Auto, - ) - .expect("signinum auto external tile region scaled submit") - }) - .collect::>(); - let surfaces = submissions - .into_iter() - .map(|submission| { - submission - .wait() - .expect("signinum auto external tile region scaled decode") - }) - .collect::>(); - black_box(surfaces); -} - -fn should_auto_use_direct_grayscale_input(input: &BenchInput, count: usize) -> bool { - if !matches!(input.mode, DecodeMode::Gray8 | DecodeMode::Gray16) || count == 0 { - return false; - } - if input.dimensions.0.max(input.dimensions.1) < AUTO_REPEATED_GRAYSCALE_MIN_DIM { - return false; - } - count >= AUTO_REPEATED_GRAYSCALE_MIN_COUNT -} - -pub(crate) fn signinum_metal_supports_tile_batch(bytes: &[u8], mode: DecodeMode) -> bool { - let mut ctx = DecoderContext::::new(); - let mut pool = MetalJ2kScratchPool::new(); - MetalJ2kCodec::decode_tile_to_device( - &mut ctx, - &mut pool, - bytes, - mode_format(mode), - BackendRequest::Metal, - ) - .is_ok() -} - -pub(crate) fn signinum_metal_supports_tile_batch_region_scaled( - bytes: &[u8], - mode: DecodeMode, - edge: u32, - scale: Downscale, -) -> bool { - let cpu_decoder = J2kDecoder::new(bytes).expect("signinum decoder"); - let roi = centered_roi(cpu_decoder.info().dimensions, edge); - let mut ctx = DecoderContext::::new(); - let mut pool = MetalJ2kScratchPool::new(); - MetalJ2kCodec::decode_tile_region_scaled_to_device( - &mut ctx, - &mut pool, - bytes, - mode_format(mode), - roi, - scale, - BackendRequest::Metal, - ) - .is_ok() -} - -pub(crate) fn signinum_metal_supports_tile_batch_distinct( - inputs: &[Vec], - mode: DecodeMode, -) -> bool { - inputs - .iter() - .all(|bytes| signinum_metal_supports_tile_batch(bytes, mode)) -} - -pub(crate) fn signinum_metal_supports_tile_batch_region_scaled_distinct( - inputs: &[Vec], - mode: DecodeMode, - edge: u32, - scale: Downscale, -) -> bool { - if !supports_metal_region_scaled_mode(mode) { - return false; - } - let Some(first) = inputs.first() else { - return true; - }; - let cpu_decoder = J2kDecoder::new(first).expect("signinum decoder"); - let roi = centered_roi(cpu_decoder.info().dimensions, edge); - let mut ctx = DecoderContext::::new(); - let mut pool = MetalJ2kScratchPool::new(); - inputs.iter().all(|bytes| { - MetalJ2kCodec::decode_tile_region_scaled_to_device( - &mut ctx, - &mut pool, - bytes, - mode_format(mode), - roi, - scale, - BackendRequest::Metal, - ) - .is_ok() - }) -} - -pub(crate) fn signinum_metal_supports_external_tile_batch_region_scaled( - batch: &ExternalTileBatch, - count: usize, - edge: u32, - scale: Downscale, -) -> bool { - if !supports_metal_region_scaled_mode(batch.mode) { - return false; - } - let count = count.min(batch.inputs.len()); - if count == 0 { - return true; - } - - let fmt = mode_format(batch.mode); - let (rois, _, _) = external_batch_output_geometry(batch, count, edge, scale, fmt); - let mut ctx = DecoderContext::::new(); - let mut pool = MetalJ2kScratchPool::new(); - batch - .inputs - .iter() - .zip(rois.iter()) - .take(count) - .all(|(bytes, roi)| { - MetalJ2kCodec::decode_tile_region_scaled_to_device( - &mut ctx, - &mut pool, - bytes, - fmt, - *roi, - scale, - BackendRequest::Metal, - ) - .is_ok() - }) -} - -pub(crate) fn openjpeg_supports_external_tile_batch_region_scaled( - batch: &ExternalTileBatch, - count: usize, -) -> bool { - matches!(batch.mode, DecodeMode::Gray8 | DecodeMode::Rgb8) - && batch - .inputs - .iter() - .take(count.min(batch.inputs.len())) - .all(|bytes| matches!(external_codec_family(bytes), ExternalCodecFamily::J2k)) -} - -pub(crate) fn grok_supports_external_tile_batch_region_scaled( - batch: &ExternalTileBatch, - count: usize, -) -> bool { - grok::is_available() - && matches!(batch.mode, DecodeMode::Gray8 | DecodeMode::Rgb8) - && batch - .inputs - .iter() - .take(count.min(batch.inputs.len())) - .all(|bytes| { - matches!( - external_codec_family(bytes), - ExternalCodecFamily::J2k | ExternalCodecFamily::Htj2k - ) - }) -} - -fn encode_j2k(pixels: &[u8], width: u32, height: u32, components: u8, bit_depth: u8) -> Vec { - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 3, - guard_bits: 2, - ..EncodeOptions::default() - }; - encode( - pixels, width, height, components, bit_depth, false, &options, - ) - .expect("encode") -} - -fn try_encode_ht( - pixels: &[u8], - width: u32, - height: u32, - components: u8, - bit_depth: u8, -) -> Result, String> { - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 3, - guard_bits: 2, - ..EncodeOptions::default() - }; - encode_htj2k( - pixels, width, height, components, bit_depth, false, &options, - ) - .map_err(std::string::ToString::to_string) -} - -fn classic_bench_bytes( - _name: &str, - pixels: &[u8], - width: u32, - height: u32, - mode: DecodeMode, -) -> Vec { - let (components, colorspace) = match mode { - DecodeMode::Gray8 | DecodeMode::Gray16 => (1_u16, 17_u32), - DecodeMode::Rgb8 => (3_u16, 16_u32), - }; - wrap_codestream_jp2( - &encode_j2k(pixels, width, height, components as u8, 8), - width, - height, - components, - 8, - colorspace, - ) -} - -fn wrap_codestream_jp2( - codestream: &[u8], - width: u32, - height: u32, - components: u16, - bit_depth: u8, - colorspace_enum: u32, -) -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0, 0, 0, 12, b'j', b'P', b' ', b' ', 0x0D, 0x0A, 0x87, 0x0A]); - bytes.extend_from_slice(&[ - 0, 0, 0, 20, b'f', b't', b'y', b'p', b'j', b'p', b'2', b' ', 0, 0, 0, 0, b'j', b'p', b'2', - b' ', - ]); - - let bpc = bit_depth.saturating_sub(1); - bytes.extend_from_slice(&[ - 0, 0, 0, 45, b'j', b'p', b'2', b'h', 0, 0, 0, 22, b'i', b'h', b'd', b'r', - ]); - bytes.extend_from_slice(&height.to_be_bytes()); - bytes.extend_from_slice(&width.to_be_bytes()); - bytes.extend_from_slice(&components.to_be_bytes()); - bytes.extend_from_slice(&[bpc, 7, 0, 0]); - bytes.extend_from_slice(&[0, 0, 0, 15, b'c', b'o', b'l', b'r', 1, 0, 0]); - bytes.extend_from_slice(&colorspace_enum.to_be_bytes()); - - let len = (8 + codestream.len()) as u32; - bytes.extend_from_slice(&len.to_be_bytes()); - bytes.extend_from_slice(b"jp2c"); - bytes.extend_from_slice(codestream); - bytes -} - -pub(crate) fn centered_roi(dims: (u32, u32), edge: u32) -> Rect { - let w = edge.min(dims.0); - let h = edge.min(dims.1); - Rect { - x: (dims.0 - w) / 2, - y: (dims.1 - h) / 2, - w, - h, - } -} - -fn external_batch_output_geometry( - batch: &ExternalTileBatch, - count: usize, - edge: u32, - scale: Downscale, - fmt: PixelFormat, -) -> (Vec, usize, u32) { - let rois = batch - .dimensions - .iter() - .take(count) - .map(|&dims| centered_roi(dims, edge)) - .collect::>(); - let (max_width, max_height) = rois - .iter() - .map(|roi| { - let scaled = roi.scaled_covering(scale); - (scaled.w, scaled.h) - }) - .fold((0_u32, 0_u32), |(max_w, max_h), (w, h)| { - (max_w.max(w), max_h.max(h)) - }); - (rois, max_width as usize * fmt.bytes_per_pixel(), max_height) -} - -fn downscale_reduction_factor(scale: Downscale) -> u32 { - scale.denominator().trailing_zeros() -} - -fn mode_format(mode: DecodeMode) -> PixelFormat { - match mode { - DecodeMode::Gray8 => PixelFormat::Gray8, - DecodeMode::Gray16 => PixelFormat::Gray16, - DecodeMode::Rgb8 => PixelFormat::Rgb8, - } -} - -fn supports_metal_region_scaled_mode(mode: DecodeMode) -> bool { - matches!(mode, DecodeMode::Gray8 | DecodeMode::Gray16) -} - -fn mode_geometry(mode: DecodeMode, dims: (u32, u32)) -> (PixelFormat, usize) { - let fmt = mode_format(mode); - (fmt, dims.0 as usize * fmt.bytes_per_pixel()) -} - -fn scaled_dims(dims: (u32, u32), scale: Downscale) -> (u32, u32) { - let denom = scale.denominator(); - (dims.0.div_ceil(denom), dims.1.div_ceil(denom)) -} diff --git a/crates/signinum-j2k-metal/benches/compare.rs b/crates/signinum-j2k-metal/benches/compare.rs deleted file mode 100644 index 45a270e1..00000000 --- a/crates/signinum-j2k-metal/benches/compare.rs +++ /dev/null @@ -1,538 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -mod common; - -use common::{ - bench_inputs, distinct_gray_tile_batch_inputs, distinct_rgb_tile_batch_inputs, - external_wsi_tile_batches, grok_decode_external_tile_batch_region_scaled, - grok_supports_external_tile_batch_region_scaled, j2k_region_edges, j2k_tile_batch_sizes, - metal_available, openjpeg_decode_external_tile_batch_region_scaled, openjpeg_decode_tile_batch, - openjpeg_decode_tile_batch_distinct, openjpeg_supports_external_tile_batch_region_scaled, - signinum_adaptive_decode, signinum_adaptive_decode_external_tile_batch_region_scaled, - signinum_adaptive_decode_region, signinum_adaptive_decode_region_scaled, - signinum_adaptive_decode_scaled, signinum_adaptive_decode_tile_batch, - signinum_adaptive_decode_tile_batch_region_scaled, - signinum_adaptive_decode_tile_batch_region_scaled_distinct, signinum_decode, - signinum_decode_external_tile_batch_region_scaled, signinum_decode_region, - signinum_decode_region_scaled, signinum_decode_scaled, signinum_decode_tile_batch, - signinum_decode_tile_batch_distinct, signinum_decode_tile_batch_region_scaled, - signinum_decode_tile_batch_region_scaled_distinct, signinum_inspect, signinum_metal_decode, - signinum_metal_decode_external_tile_batch_region_scaled, signinum_metal_decode_region, - signinum_metal_decode_region_scaled, signinum_metal_decode_scaled, - signinum_metal_decode_tile_batch, signinum_metal_decode_tile_batch_distinct, - signinum_metal_decode_tile_batch_region_scaled, - signinum_metal_decode_tile_batch_region_scaled_distinct, signinum_metal_supports_decode, - signinum_metal_supports_external_tile_batch_region_scaled, signinum_metal_supports_region, - signinum_metal_supports_region_scaled, signinum_metal_supports_scaled, - signinum_metal_supports_tile_batch, signinum_metal_supports_tile_batch_distinct, - signinum_metal_supports_tile_batch_region_scaled, - signinum_metal_supports_tile_batch_region_scaled_distinct, DecodeMode, -}; -use criterion::{criterion_group, criterion_main, Criterion}; -use signinum_core::Rect; -use signinum_j2k::Downscale; -use signinum_j2k_compare::{grok, openjpeg}; - -fn bench_compare(c: &mut Criterion) { - let inputs = bench_inputs(); - let batch_sizes = j2k_tile_batch_sizes(); - let max_batch_size = batch_sizes.iter().copied().max().unwrap_or(16); - let region_edges = j2k_region_edges(); - - let mut inspect = c.benchmark_group("inspect"); - for input in &inputs { - inspect.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_inspect(&input.bytes)); - }); - } - inspect.finish(); - - let mut decode_gray = c.benchmark_group("decode_gray"); - for input in inputs - .iter() - .filter(|input| input.mode == DecodeMode::Gray8) - { - decode_gray.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_decode(&input.bytes, input.mode)); - }); - decode_gray.bench_function(format!("signinum-adaptive/{}", input.name), |b| { - b.iter(|| signinum_adaptive_decode(&input.bytes, input.mode)); - }); - if !input.is_ht && openjpeg::is_available() { - decode_gray.bench_function(format!("openjpeg/{}", input.name), |b| { - b.iter(|| openjpeg::decode_gray(&input.bytes).expect("OpenJPEG decode")); - }); - } - if grok::is_available() { - decode_gray.bench_function(format!("grok/{}", input.name), |b| { - b.iter(|| grok::decode_gray(&input.bytes).expect("Grok decode")); - }); - } - if metal_available() && signinum_metal_supports_decode(&input.bytes, input.mode) { - decode_gray.bench_function(format!("signinum-metal/{}", input.name), |b| { - b.iter(|| signinum_metal_decode(&input.bytes, input.mode)); - }); - } - } - decode_gray.finish(); - - let mut decode_rgb = c.benchmark_group("decode_rgb"); - for input in inputs.iter().filter(|input| input.mode == DecodeMode::Rgb8) { - decode_rgb.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_decode(&input.bytes, input.mode)); - }); - if !input.is_ht && openjpeg::is_available() { - decode_rgb.bench_function(format!("openjpeg/{}", input.name), |b| { - b.iter(|| openjpeg::decode_rgb(&input.bytes).expect("OpenJPEG decode")); - }); - } - if grok::is_available() { - decode_rgb.bench_function(format!("grok/{}", input.name), |b| { - b.iter(|| grok::decode_rgb(&input.bytes).expect("Grok decode")); - }); - } - if metal_available() && signinum_metal_supports_decode(&input.bytes, input.mode) { - decode_rgb.bench_function(format!("signinum-metal/{}", input.name), |b| { - b.iter(|| signinum_metal_decode(&input.bytes, input.mode)); - }); - } - } - decode_rgb.finish(); - - let mut wsi_region = c.benchmark_group("wsi_region_gray"); - for input in inputs - .iter() - .filter(|input| input.mode == DecodeMode::Gray8) - { - wsi_region.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_decode_region(&input.bytes, input.mode, 256)); - }); - wsi_region.bench_function(format!("signinum-adaptive/{}", input.name), |b| { - b.iter(|| signinum_adaptive_decode_region(&input.bytes, input.mode, 256)); - }); - if !input.is_ht && openjpeg::is_available() { - wsi_region.bench_function(format!("openjpeg/{}", input.name), |b| { - let roi = compare_roi(input.dimensions, 256); - b.iter(|| { - openjpeg::decode_gray_region(&input.bytes, roi).expect("OpenJPEG region decode") - }); - }); - } - if grok::is_available() { - wsi_region.bench_function(format!("grok/{}", input.name), |b| { - let roi = compare_roi(input.dimensions, 256); - b.iter(|| grok::decode_gray_region(&input.bytes, roi).expect("Grok region decode")); - }); - } - if metal_available() && signinum_metal_supports_region(&input.bytes, input.mode, 256) { - wsi_region.bench_function(format!("signinum-metal/{}", input.name), |b| { - b.iter(|| signinum_metal_decode_region(&input.bytes, input.mode, 256)); - }); - } - } - wsi_region.finish(); - - let mut wsi_scaled = c.benchmark_group("wsi_scaled_gray_q4"); - for input in inputs - .iter() - .filter(|input| input.mode == DecodeMode::Gray8) - { - wsi_scaled.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| signinum_decode_scaled(&input.bytes, input.mode, Downscale::Quarter)); - }); - wsi_scaled.bench_function(format!("signinum-adaptive/{}", input.name), |b| { - b.iter(|| { - signinum_adaptive_decode_scaled(&input.bytes, input.mode, Downscale::Quarter); - }); - }); - if !input.is_ht && openjpeg::is_available() { - wsi_scaled.bench_function(format!("openjpeg/{}", input.name), |b| { - b.iter(|| { - openjpeg::decode_gray_scaled(&input.bytes, 2).expect("OpenJPEG scaled decode") - }); - }); - } - if grok::is_available() { - wsi_scaled.bench_function(format!("grok/{}", input.name), |b| { - b.iter(|| grok::decode_gray_scaled(&input.bytes, 2).expect("Grok scaled decode")); - }); - } - if metal_available() - && signinum_metal_supports_scaled(&input.bytes, input.mode, Downscale::Quarter) - { - wsi_scaled.bench_function(format!("signinum-metal/{}", input.name), |b| { - b.iter(|| { - signinum_metal_decode_scaled(&input.bytes, input.mode, Downscale::Quarter); - }); - }); - } - } - wsi_scaled.finish(); - - let mut wsi_region_scaled = c.benchmark_group("wsi_region_scaled_gray_q4"); - for input in inputs - .iter() - .filter(|input| input.mode == DecodeMode::Gray8) - { - wsi_region_scaled.bench_function(format!("signinum/{}", input.name), |b| { - b.iter(|| { - signinum_decode_region_scaled(&input.bytes, input.mode, 256, Downscale::Quarter); - }); - }); - wsi_region_scaled.bench_function(format!("signinum-adaptive/{}", input.name), |b| { - b.iter(|| { - signinum_adaptive_decode_region_scaled( - &input.bytes, - input.mode, - 256, - Downscale::Quarter, - ); - }); - }); - if metal_available() - && signinum_metal_supports_region_scaled( - &input.bytes, - input.mode, - 256, - Downscale::Quarter, - ) - { - wsi_region_scaled.bench_function(format!("signinum-metal/{}", input.name), |b| { - b.iter(|| { - signinum_metal_decode_region_scaled( - &input.bytes, - input.mode, - 256, - Downscale::Quarter, - ); - }); - }); - } - } - wsi_region_scaled.finish(); - - let mut wsi_tile_batch = c.benchmark_group("wsi_tile_batch_gray"); - for input in inputs - .iter() - .filter(|input| input.mode == DecodeMode::Gray8) - { - for &count in &batch_sizes { - wsi_tile_batch.bench_function(format!("signinum/{}/batch_{count}", input.name), |b| { - b.iter(|| signinum_decode_tile_batch(&input.bytes, input.mode, count)); - }); - wsi_tile_batch.bench_function( - format!("signinum-adaptive/{}/batch_{count}", input.name), - |b| { - b.iter(|| signinum_adaptive_decode_tile_batch(input, count)); - }, - ); - if metal_available() && signinum_metal_supports_tile_batch(&input.bytes, input.mode) { - wsi_tile_batch.bench_function( - format!("signinum-metal/{}/batch_{count}", input.name), - |b| { - b.iter(|| { - signinum_metal_decode_tile_batch(&input.bytes, input.mode, count); - }); - }, - ); - } - } - } - wsi_tile_batch.finish(); - - let mut wsi_tile_batch_region_scaled = - c.benchmark_group("wsi_tile_batch_region_scaled_gray_q4"); - for input in inputs - .iter() - .filter(|input| input.mode == DecodeMode::Gray8) - { - for &count in &batch_sizes { - wsi_tile_batch_region_scaled.bench_function( - format!("signinum/{}/batch_{count}", input.name), - |b| { - b.iter(|| { - signinum_decode_tile_batch_region_scaled( - &input.bytes, - input.mode, - 256, - Downscale::Quarter, - count, - ); - }); - }, - ); - wsi_tile_batch_region_scaled.bench_function( - format!("signinum-adaptive/{}/batch_{count}", input.name), - |b| { - b.iter(|| { - signinum_adaptive_decode_tile_batch_region_scaled( - input, - 256, - Downscale::Quarter, - count, - ); - }); - }, - ); - if metal_available() - && signinum_metal_supports_tile_batch_region_scaled( - &input.bytes, - input.mode, - 256, - Downscale::Quarter, - ) - { - wsi_tile_batch_region_scaled.bench_function( - format!("signinum-metal/{}/batch_{count}", input.name), - |b| { - b.iter(|| { - signinum_metal_decode_tile_batch_region_scaled( - &input.bytes, - input.mode, - 256, - Downscale::Quarter, - count, - ); - }); - }, - ); - } - } - } - wsi_tile_batch_region_scaled.finish(); - - let mut wsi_tile_batch_region_scaled_distinct = - c.benchmark_group("wsi_tile_batch_region_scaled_gray_distinct_q4"); - for input in inputs - .iter() - .filter(|input| input.mode == DecodeMode::Gray8) - { - for &count in &batch_sizes { - let distinct_inputs = distinct_gray_tile_batch_inputs(input, count); - wsi_tile_batch_region_scaled_distinct.bench_function( - format!("signinum/{}/batch_{count}", input.name), - |b| { - b.iter(|| { - signinum_decode_tile_batch_region_scaled_distinct( - &distinct_inputs, - input.mode, - 256, - Downscale::Quarter, - ); - }); - }, - ); - wsi_tile_batch_region_scaled_distinct.bench_function( - format!("signinum-adaptive/{}/batch_{count}", input.name), - |b| { - b.iter(|| { - signinum_adaptive_decode_tile_batch_region_scaled_distinct( - &distinct_inputs, - input.mode, - 256, - Downscale::Quarter, - ); - }); - }, - ); - if metal_available() - && signinum_metal_supports_tile_batch_region_scaled_distinct( - &distinct_inputs, - input.mode, - 256, - Downscale::Quarter, - ) - { - wsi_tile_batch_region_scaled_distinct.bench_function( - format!("signinum-metal/{}/batch_{count}", input.name), - |b| { - b.iter(|| { - signinum_metal_decode_tile_batch_region_scaled_distinct( - &distinct_inputs, - input.mode, - 256, - Downscale::Quarter, - ); - }); - }, - ); - } - } - } - wsi_tile_batch_region_scaled_distinct.finish(); - - let external_batches = external_wsi_tile_batches(max_batch_size); - if !external_batches.is_empty() { - let mut external_wsi_region_scaled = - c.benchmark_group("external_wsi_tile_batch_region_scaled_q4"); - for batch in &external_batches { - for &count in &batch_sizes { - if batch.inputs.len() < count { - continue; - } - for &edge in ®ion_edges { - external_wsi_region_scaled.bench_function( - format!("signinum/{}/edge_{edge}/batch_{count}", batch.name), - |b| { - b.iter(|| { - signinum_decode_external_tile_batch_region_scaled( - batch, - count, - edge, - Downscale::Quarter, - ); - }); - }, - ); - external_wsi_region_scaled.bench_function( - format!("signinum-adaptive/{}/edge_{edge}/batch_{count}", batch.name), - |b| { - b.iter(|| { - signinum_adaptive_decode_external_tile_batch_region_scaled( - batch, - count, - edge, - Downscale::Quarter, - ); - }); - }, - ); - if openjpeg::is_available() - && openjpeg_supports_external_tile_batch_region_scaled(batch, count) - { - external_wsi_region_scaled.bench_function( - format!("openjpeg/{}/edge_{edge}/batch_{count}", batch.name), - |b| { - b.iter(|| { - openjpeg_decode_external_tile_batch_region_scaled( - batch, - count, - edge, - Downscale::Quarter, - ); - }); - }, - ); - } - if grok_supports_external_tile_batch_region_scaled(batch, count) { - external_wsi_region_scaled.bench_function( - format!("grok/{}/edge_{edge}/batch_{count}", batch.name), - |b| { - b.iter(|| { - grok_decode_external_tile_batch_region_scaled( - batch, - count, - edge, - Downscale::Quarter, - ); - }); - }, - ); - } - if metal_available() - && signinum_metal_supports_external_tile_batch_region_scaled( - batch, - count, - edge, - Downscale::Quarter, - ) - { - external_wsi_region_scaled.bench_function( - format!("signinum-metal/{}/edge_{edge}/batch_{count}", batch.name), - |b| { - b.iter(|| { - signinum_metal_decode_external_tile_batch_region_scaled( - batch, - count, - edge, - Downscale::Quarter, - ); - }); - }, - ); - } - } - } - } - external_wsi_region_scaled.finish(); - } - - let mut wsi_tile_batch_rgb = c.benchmark_group("wsi_tile_batch_rgb"); - for input in inputs.iter().filter(|input| input.mode == DecodeMode::Rgb8) { - for &count in &batch_sizes { - wsi_tile_batch_rgb.bench_function( - format!("signinum/{}/batch_{count}", input.name), - |b| { - b.iter(|| signinum_decode_tile_batch(&input.bytes, input.mode, count)); - }, - ); - if !input.is_ht && openjpeg::is_available() { - wsi_tile_batch_rgb.bench_function( - format!("openjpeg/{}/batch_{count}", input.name), - |b| { - b.iter(|| openjpeg_decode_tile_batch(&input.bytes, input.mode, count)); - }, - ); - } - if metal_available() && signinum_metal_supports_tile_batch(&input.bytes, input.mode) { - wsi_tile_batch_rgb.bench_function( - format!("signinum-metal/{}/batch_{count}", input.name), - |b| { - b.iter(|| { - signinum_metal_decode_tile_batch(&input.bytes, input.mode, count); - }); - }, - ); - } - } - } - wsi_tile_batch_rgb.finish(); - - let mut wsi_tile_batch_rgb_distinct = c.benchmark_group("wsi_tile_batch_rgb_distinct"); - for input in inputs.iter().filter(|input| input.mode == DecodeMode::Rgb8) { - for &count in &batch_sizes { - let distinct_inputs = distinct_rgb_tile_batch_inputs(input, count); - wsi_tile_batch_rgb_distinct.bench_function( - format!("signinum/{}/batch_{count}", input.name), - |b| { - b.iter(|| signinum_decode_tile_batch_distinct(&distinct_inputs, input.mode)); - }, - ); - if !input.is_ht && openjpeg::is_available() { - wsi_tile_batch_rgb_distinct.bench_function( - format!("openjpeg/{}/batch_{count}", input.name), - |b| { - b.iter(|| { - openjpeg_decode_tile_batch_distinct(&distinct_inputs, input.mode); - }); - }, - ); - } - if metal_available() - && signinum_metal_supports_tile_batch_distinct(&distinct_inputs, input.mode) - { - wsi_tile_batch_rgb_distinct.bench_function( - format!("signinum-metal/{}/batch_{count}", input.name), - |b| { - b.iter(|| { - signinum_metal_decode_tile_batch_distinct(&distinct_inputs, input.mode); - }); - }, - ); - } - } - } - wsi_tile_batch_rgb_distinct.finish(); -} - -fn compare_roi(dimensions: (u32, u32), extent: u32) -> Rect { - Rect { - x: dimensions.0.saturating_sub(extent) / 2, - y: dimensions.1.saturating_sub(extent) / 2, - w: dimensions.0.min(extent), - h: dimensions.1.min(extent), - } -} - -criterion_group!(benches, bench_compare); -criterion_main!(benches); diff --git a/crates/signinum-j2k-metal/benches/device_upload.rs b/crates/signinum-j2k-metal/benches/device_upload.rs deleted file mode 100644 index 923b2314..00000000 --- a/crates/signinum-j2k-metal/benches/device_upload.rs +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use criterion::{criterion_group, criterion_main, Criterion}; -use signinum_core::{BackendRequest, ImageDecodeDevice, PixelFormat}; -use signinum_j2k::J2kDecoder as CpuDecoder; -use signinum_j2k_metal::J2kDecoder as MetalDecoder; -use signinum_j2k_native::{encode, EncodeOptions}; - -fn fixture() -> Vec { - let pixels = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]; - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - encode(&pixels, 2, 2, 3, 8, false, &options).expect("encode") -} - -fn bench_device_upload(c: &mut Criterion) { - let bytes = fixture(); - let mut group = c.benchmark_group("j2k_metal_device"); - - group.bench_function("cpu_decode_rgb8", |b| { - let mut decoder = CpuDecoder::new(&bytes).expect("cpu decoder"); - b.iter(|| { - let mut out = [0u8; 12]; - decoder - .decode_into(&mut out, 6, PixelFormat::Rgb8) - .expect("cpu decode") - }); - }); - - if metal_decode_available() { - group.bench_function("metal_surface_rgb8", |b| { - let mut decoder = MetalDecoder::new(&bytes).expect("metal decoder"); - b.iter(|| { - decoder - .decode_to_device(PixelFormat::Rgb8, BackendRequest::Metal) - .expect("device decode") - }); - }); - } - - group.finish(); -} - -fn metal_decode_available() -> bool { - #[cfg(target_os = "macos")] - { - metal::Device::system_default().is_some() - } - #[cfg(not(target_os = "macos"))] - { - assert!( - std::env::var_os("SIGNINUM_REQUIRE_METAL_BENCH").is_none(), - "SIGNINUM_REQUIRE_METAL_BENCH is set but this is not a Metal host" - ); - false - } -} - -criterion_group!(benches, bench_device_upload); -criterion_main!(benches); diff --git a/crates/signinum-j2k-metal/benches/encode_stages.rs b/crates/signinum-j2k-metal/benches/encode_stages.rs deleted file mode 100644 index 3e664944..00000000 --- a/crates/signinum-j2k-metal/benches/encode_stages.rs +++ /dev/null @@ -1,379 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; -#[cfg(target_os = "macos")] -use signinum_core::PixelFormat; -use signinum_j2k::{ - encode_j2k_lossless, EncodeBackendPreference, J2kBlockCodingMode, J2kEncodeValidation, - J2kLosslessEncodeOptions, J2kLosslessSamples, -}; -use signinum_j2k_metal::MetalEncodeStageAccelerator; -#[cfg(target_os = "macos")] -use signinum_j2k_metal::{ - encode_lossless_from_padded_metal_buffer_with_report, MetalBackendSession, - MetalLosslessEncodeTile, -}; -use signinum_j2k_native::{J2kEncodeStageAccelerator, J2kForwardDwt53Job, J2kForwardRctJob}; - -const BENCH_DIMS: &[u32] = &[512, 1024, 2048]; -const ENCODE_BENCH_DIMS: &[u32] = &[512, 1024]; - -fn bench_encode_stages(c: &mut Criterion) { - let mut rct = c.benchmark_group("j2k_metal_forward_rct"); - for &dim in BENCH_DIMS { - let pixels = generate_rgb_planes(dim, dim); - rct.bench_with_input(BenchmarkId::new("cpu", dim), &pixels, |b, planes| { - b.iter(|| { - let (mut plane0, mut plane1, mut plane2) = clone_planes(planes); - cpu_forward_rct(&mut plane0, &mut plane1, &mut plane2); - (plane0, plane1, plane2) - }); - }); - - if metal_encode_available() { - rct.bench_with_input(BenchmarkId::new("metal", dim), &pixels, |b, planes| { - let mut accelerator = MetalEncodeStageAccelerator::default(); - b.iter(|| { - let (mut plane0, mut plane1, mut plane2) = clone_planes(planes); - let dispatched = accelerator - .encode_forward_rct(J2kForwardRctJob { - plane0: &mut plane0, - plane1: &mut plane1, - plane2: &mut plane2, - }) - .expect("Metal forward RCT"); - assert!(dispatched, "Metal forward RCT did not dispatch"); - (plane0, plane1, plane2) - }); - }); - } - } - rct.finish(); - - let mut dwt = c.benchmark_group("j2k_metal_forward_dwt53"); - for &dim in BENCH_DIMS { - let samples = generate_gray_plane(dim, dim); - dwt.bench_with_input(BenchmarkId::new("cpu", dim), &samples, |b, samples| { - b.iter(|| cpu_forward_dwt53(samples, dim, dim, 1)); - }); - - if metal_encode_available() { - dwt.bench_with_input(BenchmarkId::new("metal", dim), &samples, |b, samples| { - let mut accelerator = MetalEncodeStageAccelerator::default(); - b.iter(|| { - let output = accelerator - .encode_forward_dwt53(J2kForwardDwt53Job { - samples, - width: dim, - height: dim, - num_levels: 1, - }) - .expect("Metal forward DWT 5/3") - .expect("Metal forward DWT 5/3 dispatch"); - assert_eq!(output.ll_width, dim / 2); - output - }); - }); - } - } - dwt.finish(); - - let mut encode = c.benchmark_group("j2k_metal_lossless_rgb8_encode"); - for &dim in ENCODE_BENCH_DIMS { - let pixels = generate_rgb8_pixels(dim, dim); - let cpu_options = J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - }; - encode.bench_with_input(BenchmarkId::new("cpu", dim), &pixels, |b, pixels| { - b.iter(|| { - let samples = J2kLosslessSamples::new(pixels, dim, dim, 3, 8, false) - .expect("valid RGB8 samples"); - encode_j2k_lossless(samples, &cpu_options).expect("CPU J2K lossless encode") - }); - }); - let cpu_ht_options = J2kLosslessEncodeOptions { - block_coding_mode: J2kBlockCodingMode::HighThroughput, - ..cpu_options - }; - encode.bench_with_input(BenchmarkId::new("cpu_htj2k", dim), &pixels, |b, pixels| { - b.iter(|| { - let samples = J2kLosslessSamples::new(pixels, dim, dim, 3, 8, false) - .expect("valid RGB8 samples"); - encode_j2k_lossless(samples, &cpu_ht_options).expect("CPU HTJ2K lossless encode") - }); - }); - - #[cfg(target_os = "macos")] - if metal_encode_available() { - let session = MetalBackendSession::system_default().expect("Metal session"); - let buffer = private_buffer_with_bytes(&session, &pixels); - let metal_options = J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - }; - let auto_options = J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::Auto, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - }; - let auto_ht_options = J2kLosslessEncodeOptions { - block_coding_mode: J2kBlockCodingMode::HighThroughput, - ..auto_options - }; - encode.bench_with_input(BenchmarkId::new("resident_metal", dim), &pixels, |b, _| { - b.iter(|| { - let encoded = encode_lossless_from_padded_metal_buffer_with_report( - MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: dim, - height: dim, - pitch_bytes: dim as usize * 3, - output_width: dim, - output_height: dim, - format: PixelFormat::Rgb8, - }, - &metal_options, - &session, - ) - .expect("resident Metal J2K lossless encode"); - assert!(encoded.resident.coefficient_prep_used); - assert!(encoded.resident.packetization_used); - assert!(encoded.resident.codestream_assembly_used); - encoded.encoded - }); - }); - encode.bench_with_input( - BenchmarkId::new("auto_host_metal_buffer", dim), - &pixels, - |b, _| { - b.iter(|| { - let encoded = encode_lossless_from_padded_metal_buffer_with_report( - MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: dim, - height: dim, - pitch_bytes: dim as usize * 3, - output_width: dim, - output_height: dim, - format: PixelFormat::Rgb8, - }, - &auto_options, - &session, - ) - .expect("Auto J2K lossless encode from Metal buffer"); - assert!(!encoded.resident.coefficient_prep_used); - assert!(!encoded.resident.packetization_used); - assert!(!encoded.resident.codestream_assembly_used); - encoded.encoded - }); - }, - ); - encode.bench_with_input( - BenchmarkId::new("auto_host_metal_buffer_htj2k", dim), - &pixels, - |b, _| { - b.iter(|| { - let encoded = encode_lossless_from_padded_metal_buffer_with_report( - MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: dim, - height: dim, - pitch_bytes: dim as usize * 3, - output_width: dim, - output_height: dim, - format: PixelFormat::Rgb8, - }, - &auto_ht_options, - &session, - ) - .expect("Auto HTJ2K lossless encode from Metal buffer"); - assert!(!encoded.resident.coefficient_prep_used); - assert!(!encoded.resident.packetization_used); - assert!(!encoded.resident.codestream_assembly_used); - encoded.encoded - }); - }, - ); - } - } - encode.finish(); -} - -fn metal_encode_available() -> bool { - #[cfg(target_os = "macos")] - { - metal::Device::system_default().is_some() - } - #[cfg(not(target_os = "macos"))] - { - assert!( - std::env::var_os("SIGNINUM_REQUIRE_METAL_BENCH").is_none(), - "SIGNINUM_REQUIRE_METAL_BENCH is set but this is not a Metal host" - ); - false - } -} - -fn generate_rgb_planes(width: u32, height: u32) -> (Vec, Vec, Vec) { - let len = width as usize * height as usize; - let mut plane0 = Vec::with_capacity(len); - let mut plane1 = Vec::with_capacity(len); - let mut plane2 = Vec::with_capacity(len); - for y in 0..height { - for x in 0..width { - plane0.push(centered_sample(x * 13 + y * 3)); - plane1.push(centered_sample(x * 5 + y * 11 + (x ^ y))); - plane2.push(centered_sample(x * 7 + y * 17 + x.wrapping_mul(y) / 31)); - } - } - (plane0, plane1, plane2) -} - -fn generate_gray_plane(width: u32, height: u32) -> Vec { - let len = width as usize * height as usize; - let mut samples = Vec::with_capacity(len); - for y in 0..height { - for x in 0..width { - samples.push(centered_sample(x * 9 + y * 15 + x.wrapping_mul(y) / 17)); - } - } - samples -} - -fn generate_rgb8_pixels(width: u32, height: u32) -> Vec { - let len = width as usize * height as usize * 3; - let mut pixels = Vec::with_capacity(len); - for y in 0..height { - for x in 0..width { - pixels.push(((x * 3 + y * 5) & 0xff) as u8); - pixels.push(((x * 7 + y * 11 + (x ^ y)) & 0xff) as u8); - pixels.push(((x * 13 + y * 17 + x.wrapping_mul(y) / 31) & 0xff) as u8); - } - } - pixels -} - -#[cfg(target_os = "macos")] -fn private_buffer_with_bytes(session: &MetalBackendSession, bytes: &[u8]) -> metal::Buffer { - let upload = session.device().new_buffer_with_data( - bytes.as_ptr().cast(), - bytes.len() as u64, - metal::MTLResourceOptions::StorageModeShared, - ); - let private = session.device().new_buffer( - bytes.len() as u64, - metal::MTLResourceOptions::StorageModePrivate, - ); - let queue = session.device().new_command_queue(); - let command_buffer = queue.new_command_buffer(); - let blit = command_buffer.new_blit_command_encoder(); - blit.copy_from_buffer(&upload, 0, &private, 0, bytes.len() as u64); - blit.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - private -} - -fn centered_sample(value: u32) -> f32 { - f32::from(u8::try_from(value & 0xff).expect("masked sample fits in u8")) - 128.0 -} - -fn clone_planes(planes: &(Vec, Vec, Vec)) -> (Vec, Vec, Vec) { - (planes.0.clone(), planes.1.clone(), planes.2.clone()) -} - -fn cpu_forward_rct(plane0: &mut [f32], plane1: &mut [f32], plane2: &mut [f32]) { - for ((r, g), b) in plane0 - .iter_mut() - .zip(plane1.iter_mut()) - .zip(plane2.iter_mut()) - { - let original_r = *r; - let original_g = *g; - let original_b = *b; - *r = ((original_r + 2.0 * original_g + original_b) * 0.25).floor(); - *g = original_b - original_g; - *b = original_r - original_g; - } -} - -fn cpu_forward_dwt53(samples: &[f32], width: u32, height: u32, num_levels: u8) -> Vec { - let full_width = width as usize; - let mut buffer = samples.to_vec(); - let mut current_width = width as usize; - let mut current_height = height as usize; - - for _ in 0..num_levels { - if current_width < 2 && current_height < 2 { - break; - } - if current_width >= 2 { - let mut row = vec![0.0; current_width]; - for y in 0..current_height { - let start = y * full_width; - row.copy_from_slice(&buffer[start..start + current_width]); - forward_lift_53(&mut row); - let low_width = current_width.div_ceil(2); - for i in 0..low_width { - buffer[start + i] = row[i * 2]; - } - for i in 0..(current_width / 2) { - buffer[start + low_width + i] = row[i * 2 + 1]; - } - } - } - if current_height >= 2 { - let mut col = vec![0.0; current_height]; - for x in 0..current_width { - for y in 0..current_height { - col[y] = buffer[y * full_width + x]; - } - forward_lift_53(&mut col); - let low_height = current_height.div_ceil(2); - for i in 0..low_height { - buffer[i * full_width + x] = col[i * 2]; - } - for i in 0..(current_height / 2) { - buffer[(low_height + i) * full_width + x] = col[i * 2 + 1]; - } - } - } - current_width = current_width.div_ceil(2); - current_height = current_height.div_ceil(2); - } - - buffer -} - -fn forward_lift_53(data: &mut [f32]) { - let n = data.len(); - if n < 2 { - return; - } - - let last_even = if n.is_multiple_of(2) { n - 2 } else { n - 1 }; - for i in (1..n).step_by(2) { - let left = data[i - 1]; - let right = if i + 1 < n { - data[i + 1] - } else { - data[last_even] - }; - data[i] -= ((left + right) * 0.5).floor(); - } - - for i in (0..n).step_by(2) { - let left = if i > 0 { data[i - 1] } else { data[1] }; - let right = if i + 1 < n { data[i + 1] } else { left }; - data[i] += ((left + right) * 0.25 + 0.5).floor(); - } -} - -criterion_group!(benches, bench_encode_stages); -criterion_main!(benches); diff --git a/crates/signinum-j2k-metal/src/batch.rs b/crates/signinum-j2k-metal/src/batch.rs deleted file mode 100644 index 12fc36c2..00000000 --- a/crates/signinum-j2k-metal/src/batch.rs +++ /dev/null @@ -1,811 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::{Arc, Mutex}; - -use signinum_core::{BackendKind, BackendRequest, DeviceSubmission, Downscale, PixelFormat, Rect}; -use signinum_j2k::{ - decode_tiles_into, decode_tiles_region_scaled_into, TileBatchOptions, TileDecodeJob, - TileRegionScaledDecodeJob, -}; - -use crate::{Error, J2kDecoder, MetalSession, Storage, Surface, SurfaceResidency}; - -const AUTO_REGION_SCALED_GRAYSCALE_BATCH64_MIN_DIM: u32 = 512; -const AUTO_REGION_SCALED_GRAYSCALE_BATCH64_MIN_COUNT: usize = 64; -const AUTO_REGION_SCALED_GRAYSCALE_BATCH16_MIN_DIM: u32 = 1024; -const AUTO_REGION_SCALED_GRAYSCALE_BATCH16_MIN_COUNT: usize = 16; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum BatchOp { - Full, - Region(Rect), - Scaled(Downscale), - RegionScaled { roi: Rect, scale: Downscale }, -} - -#[derive(Clone)] -struct QueuedRequest { - input: Arc<[u8]>, - fmt: PixelFormat, - backend: BackendRequest, - op: BatchOp, - output_slot: usize, -} - -impl QueuedRequest { - fn max_image_dim(&self) -> Option { - let decoder = J2kDecoder::new(self.input.as_ref()).ok()?; - let dims = decoder.inner.info().dimensions; - Some(dims.0.max(dims.1)) - } -} - -#[derive(Default)] -pub(crate) struct SessionState { - pub(crate) submissions: u64, - queued: Vec, - completed: Vec>>, -} - -#[derive(Clone, Default)] -pub(crate) struct SharedSession(pub(crate) Arc>); - -pub struct MetalSubmission { - session: SharedSession, - slot: usize, -} - -impl DeviceSubmission for MetalSubmission { - type Output = Surface; - type Error = Error; - - fn wait(self) -> Result { - let mut session = self.session.0.lock().expect("J2K Metal session"); - flush_if_needed(&mut session); - take_surface(&mut session, self.slot) - } -} - -pub(crate) fn queue_tile_request( - session: &mut MetalSession, - input: &[u8], - fmt: PixelFormat, - backend: BackendRequest, - op: BatchOp, -) -> MetalSubmission { - queue_tile_request_shared(session, Arc::<[u8]>::from(input), fmt, backend, op) -} - -pub(crate) fn queue_tile_request_shared( - session: &mut MetalSession, - input: Arc<[u8]>, - fmt: PixelFormat, - backend: BackendRequest, - op: BatchOp, -) -> MetalSubmission { - let mut state = session.shared.0.lock().expect("J2K Metal session"); - let slot = state.completed.len(); - state.completed.push(None); - state.queued.push(QueuedRequest { - input, - fmt, - backend, - op, - output_slot: slot, - }); - MetalSubmission { - session: session.shared.clone(), - slot, - } -} - -fn flush_if_needed(session: &mut SessionState) { - if session.queued.is_empty() { - return; - } - - for batch in group_metal_requests(std::mem::take(&mut session.queued)) { - process_batch(session, batch); - } -} - -fn group_metal_requests(queued: Vec) -> Vec> { - coalesce_cpu_host_batches(coalesce_distinct_region_scaled_grayscale_metal_requests( - coalesce_distinct_full_color_metal_requests( - coalesce_distinct_full_grayscale_metal_requests(group_repeated_full_metal_requests( - queued, - )), - ), - )) -} - -fn group_repeated_full_metal_requests(queued: Vec) -> Vec> { - let mut batches: Vec> = Vec::new(); - for request in queued { - if let Some(batch) = batches - .iter_mut() - .find(|batch| can_decode_as_repeated_full_metal_batch(&batch[0], &request)) - { - batch.push(request); - } else { - batches.push(vec![request]); - } - } - batches -} - -fn coalesce_distinct_full_grayscale_metal_requests( - repeated_batches: Vec>, -) -> Vec> { - let mut batches = Vec::new(); - let mut gray8 = Vec::new(); - let mut gray16 = Vec::new(); - - for batch in repeated_batches { - if batch.len() == 1 && is_distinct_full_grayscale_metal_candidate(&batch[0]) { - let request = batch - .into_iter() - .next() - .expect("single-entry batch has request"); - match request.fmt { - PixelFormat::Gray8 => gray8.push(request), - PixelFormat::Gray16 => gray16.push(request), - _ => unreachable!("candidate pixel format is restricted above"), - } - } else { - batches.push(batch); - } - } - - push_coalesced_or_single(&mut batches, gray8); - push_coalesced_or_single(&mut batches, gray16); - batches -} - -fn coalesce_distinct_region_scaled_grayscale_metal_requests( - repeated_batches: Vec>, -) -> Vec> { - let mut batches = Vec::new(); - let mut metal_gray8 = Vec::new(); - let mut metal_gray16 = Vec::new(); - let mut auto_gray8 = Vec::new(); - let mut auto_gray16 = Vec::new(); - - for batch in repeated_batches { - if batch.len() == 1 && is_region_scaled_grayscale_batch_candidate(&batch[0]) { - let request = batch - .into_iter() - .next() - .expect("single-entry batch has request"); - match (request.backend, request.fmt) { - (BackendRequest::Metal, PixelFormat::Gray8) => metal_gray8.push(request), - (BackendRequest::Metal, PixelFormat::Gray16) => metal_gray16.push(request), - (BackendRequest::Auto, PixelFormat::Gray8) => auto_gray8.push(request), - (BackendRequest::Auto, PixelFormat::Gray16) => auto_gray16.push(request), - _ => unreachable!("candidate backend and pixel format are restricted above"), - } - } else { - batches.push(batch); - } - } - - push_coalesced_or_single(&mut batches, metal_gray8); - push_coalesced_or_single(&mut batches, metal_gray16); - push_auto_region_scaled_grayscale_batches(&mut batches, auto_gray8); - push_auto_region_scaled_grayscale_batches(&mut batches, auto_gray16); - batches -} - -fn push_coalesced_or_single(batches: &mut Vec>, requests: Vec) { - if requests.is_empty() { - return; - } - if requests.len() == 1 { - batches.extend(requests.into_iter().map(|request| vec![request])); - } else { - batches.push(requests); - } -} - -fn push_auto_region_scaled_grayscale_batches( - batches: &mut Vec>, - requests: Vec, -) { - let Some(min_dim) = auto_region_scaled_grayscale_metal_min_dim(&requests) else { - push_coalesced_or_single(batches, requests); - return; - }; - - let mut metal_requests = Vec::new(); - let mut cpu_requests = Vec::new(); - for request in requests { - if request - .max_image_dim() - .is_some_and(|max_dim| max_dim >= min_dim) - { - metal_requests.push(request); - } else { - cpu_requests.push(request); - } - } - push_coalesced_or_single(batches, metal_requests); - push_coalesced_or_single(batches, cpu_requests); -} - -#[allow(clippy::similar_names)] -fn coalesce_distinct_full_color_metal_requests( - repeated_batches: Vec>, -) -> Vec> { - let mut batches = Vec::new(); - let mut rgb8 = Vec::new(); - let mut rgba8 = Vec::new(); - let mut rgb16 = Vec::new(); - - for batch in repeated_batches { - if batch.len() == 1 && is_distinct_full_color_metal_candidate(&batch[0]) { - let request = batch - .into_iter() - .next() - .expect("single-entry batch has request"); - match request.fmt { - PixelFormat::Rgb8 => rgb8.push(request), - PixelFormat::Rgba8 => rgba8.push(request), - PixelFormat::Rgb16 => rgb16.push(request), - _ => unreachable!("candidate pixel format is restricted above"), - } - } else { - batches.push(batch); - } - } - - push_coalesced_or_single(&mut batches, rgb8); - push_coalesced_or_single(&mut batches, rgba8); - push_coalesced_or_single(&mut batches, rgb16); - batches -} - -fn coalesce_cpu_host_batches(batches: Vec>) -> Vec> { - let mut coalesced: Vec> = Vec::new(); - let mut cpu_groups: Vec> = Vec::new(); - for batch in batches { - if batch.len() == 1 && is_cpu_host_batch_candidate(&batch[0]) { - let request = batch - .into_iter() - .next() - .expect("single-entry batch has request"); - if let Some(existing) = cpu_groups - .iter_mut() - .find(|existing| can_coalesce_cpu_host_batch(&existing[0], &request)) - { - existing.push(request); - } else { - cpu_groups.push(vec![request]); - } - } else { - coalesced.push(batch); - } - } - coalesced.extend(cpu_groups); - coalesced -} - -fn is_cpu_host_batch_candidate(request: &QueuedRequest) -> bool { - matches!(request.op, BatchOp::Full | BatchOp::RegionScaled { .. }) - && matches!(request.backend, BackendRequest::Cpu | BackendRequest::Auto) -} - -fn can_coalesce_cpu_host_batch(first: &QueuedRequest, next: &QueuedRequest) -> bool { - is_cpu_host_batch_candidate(first) - && is_cpu_host_batch_candidate(next) - && first.fmt == next.fmt - && matches!( - (&first.op, &next.op), - (BatchOp::Full, BatchOp::Full) - | (BatchOp::RegionScaled { .. }, BatchOp::RegionScaled { .. }) - ) -} - -fn can_decode_as_repeated_full_grayscale_batch( - first: &QueuedRequest, - next: &QueuedRequest, -) -> bool { - is_repeated_full_grayscale_candidate(first) - && is_repeated_full_grayscale_candidate(next) - && first.fmt == next.fmt - && first.backend == next.backend - && first.input.as_ref() == next.input.as_ref() -} - -fn can_decode_as_repeated_full_color_batch(first: &QueuedRequest, next: &QueuedRequest) -> bool { - is_repeated_full_color_candidate(first) - && is_repeated_full_color_candidate(next) - && first.fmt == next.fmt - && first.backend == next.backend - && first.input.as_ref() == next.input.as_ref() -} - -fn can_decode_as_repeated_full_metal_batch(first: &QueuedRequest, next: &QueuedRequest) -> bool { - can_decode_as_repeated_full_grayscale_batch(first, next) - || can_decode_as_repeated_full_color_batch(first, next) -} - -fn is_repeated_full_grayscale_candidate(request: &QueuedRequest) -> bool { - matches!(request.op, BatchOp::Full) - && matches!(request.fmt, PixelFormat::Gray8 | PixelFormat::Gray16) - && matches!( - request.backend, - BackendRequest::Auto | BackendRequest::Metal - ) -} - -fn is_repeated_full_color_candidate(request: &QueuedRequest) -> bool { - matches!(request.op, BatchOp::Full) - && matches!( - request.fmt, - PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 - ) - && request.backend == BackendRequest::Metal -} - -fn is_distinct_full_grayscale_metal_candidate(request: &QueuedRequest) -> bool { - matches!(request.op, BatchOp::Full) - && matches!(request.fmt, PixelFormat::Gray8 | PixelFormat::Gray16) - && request.backend == BackendRequest::Metal -} - -fn is_distinct_full_color_metal_candidate(request: &QueuedRequest) -> bool { - matches!(request.op, BatchOp::Full) - && matches!( - request.fmt, - PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Rgb16 - ) - && request.backend == BackendRequest::Metal -} - -fn is_region_scaled_grayscale_batch_candidate(request: &QueuedRequest) -> bool { - matches!(request.op, BatchOp::RegionScaled { .. }) - && matches!(request.fmt, PixelFormat::Gray8 | PixelFormat::Gray16) - && matches!( - request.backend, - BackendRequest::Auto | BackendRequest::Metal - ) -} - -fn should_auto_use_metal_for_region_scaled_grayscale_batch(requests: &[QueuedRequest]) -> bool { - auto_region_scaled_grayscale_metal_min_dim(requests).is_some() -} - -fn auto_region_scaled_grayscale_metal_min_dim(requests: &[QueuedRequest]) -> Option { - let mut count_512_class = 0usize; - let mut count_1024_class = 0usize; - for request in requests { - let Some(max_dim) = request.max_image_dim() else { - continue; - }; - if max_dim >= AUTO_REGION_SCALED_GRAYSCALE_BATCH64_MIN_DIM { - count_512_class += 1; - } - if max_dim >= AUTO_REGION_SCALED_GRAYSCALE_BATCH16_MIN_DIM { - count_1024_class += 1; - } - } - - if count_512_class >= AUTO_REGION_SCALED_GRAYSCALE_BATCH64_MIN_COUNT { - Some(AUTO_REGION_SCALED_GRAYSCALE_BATCH64_MIN_DIM) - } else if count_1024_class >= AUTO_REGION_SCALED_GRAYSCALE_BATCH16_MIN_COUNT { - Some(AUTO_REGION_SCALED_GRAYSCALE_BATCH16_MIN_DIM) - } else { - None - } -} - -fn can_decode_requests_as_repeated_full_grayscale_batch(requests: &[QueuedRequest]) -> bool { - let Some((first, rest)) = requests.split_first() else { - return false; - }; - !rest.is_empty() - && rest - .iter() - .all(|request| can_decode_as_repeated_full_grayscale_batch(first, request)) -} - -fn can_decode_requests_as_repeated_full_color_batch(requests: &[QueuedRequest]) -> bool { - let Some((first, rest)) = requests.split_first() else { - return false; - }; - !rest.is_empty() - && rest - .iter() - .all(|request| can_decode_as_repeated_full_color_batch(first, request)) -} - -fn process_batch(session: &mut SessionState, requests: Vec) { - if can_decode_requests_as_repeated_full_grayscale_batch(&requests) { - if let Some(Ok(surfaces)) = decode_repeated_full_grayscale(&requests[0], requests.len()) { - if surfaces.len() == requests.len() { - session.submissions = session.submissions.saturating_add(1); - for (request, surface) in requests.into_iter().zip(surfaces) { - session.completed[request.output_slot] = Some(Ok(surface)); - } - return; - } - } - } - - if can_decode_requests_as_repeated_full_color_batch(&requests) { - if let Some(Ok(surfaces)) = decode_repeated_full_color(&requests[0], requests.len()) { - if surfaces.len() == requests.len() { - session.submissions = session.submissions.saturating_add(1); - for (request, surface) in requests.into_iter().zip(surfaces) { - session.completed[request.output_slot] = Some(Ok(surface)); - } - return; - } - } - } - - if requests.len() > 1 { - if let Some(Ok(surfaces)) = decode_distinct_full_grayscale_batch(&requests) { - if surfaces.len() == requests.len() { - session.submissions = session.submissions.saturating_add(1); - for (request, surface) in requests.into_iter().zip(surfaces) { - session.completed[request.output_slot] = Some(Ok(surface)); - } - return; - } - } - } - - if requests.len() > 1 { - if let Some(result) = decode_distinct_full_color_batch(&requests) { - match result { - Ok(surfaces) if surfaces.len() == requests.len() => { - session.submissions = session.submissions.saturating_add(1); - for (request, surface) in requests.into_iter().zip(surfaces) { - session.completed[request.output_slot] = Some(Ok(surface)); - } - return; - } - Ok(_) | Err(_) => {} - } - } - } - - if requests.len() > 1 { - if let Some(Ok(surfaces)) = decode_distinct_region_scaled_grayscale_batch(&requests) { - if surfaces.len() == requests.len() { - session.submissions = session.submissions.saturating_add(1); - for (request, surface) in requests.into_iter().zip(surfaces) { - session.completed[request.output_slot] = Some(Ok(surface)); - } - return; - } - } - } - - if requests.len() > 1 { - if let Some(Ok(surfaces)) = decode_cpu_host_batch(&requests) { - if surfaces.len() == requests.len() { - session.submissions = session.submissions.saturating_add(1); - for (request, surface) in requests.into_iter().zip(surfaces) { - session.completed[request.output_slot] = Some(Ok(surface)); - } - return; - } - } - } - - for request in requests { - session.submissions = session.submissions.saturating_add(1); - session.completed[request.output_slot] = Some(decode_individual(&request)); - } -} - -fn decode_cpu_host_batch(requests: &[QueuedRequest]) -> Option, Error>> { - decode_cpu_full_batch(requests).or_else(|| decode_cpu_region_scaled_batch(requests)) -} - -fn decode_cpu_full_batch(requests: &[QueuedRequest]) -> Option, Error>> { - let first = requests.first()?; - if requests.len() <= 1 - || !requests - .iter() - .all(|request| is_cpu_host_full_batch_candidate(request) && request.fmt == first.fmt) - { - return None; - } - - Some(decode_cpu_full_batch_inner(requests, first.fmt)) -} - -fn is_cpu_host_full_batch_candidate(request: &QueuedRequest) -> bool { - matches!(request.op, BatchOp::Full) - && matches!(request.backend, BackendRequest::Cpu | BackendRequest::Auto) -} - -fn decode_cpu_full_batch_inner( - requests: &[QueuedRequest], - fmt: PixelFormat, -) -> Result, Error> { - let mut dims = Vec::with_capacity(requests.len()); - let mut outputs = Vec::with_capacity(requests.len()); - for request in requests { - let decoder = J2kDecoder::new(request.input.as_ref())?; - let tile_dims = decoder.inner.info().dimensions; - let stride = tile_dims.0 as usize * fmt.bytes_per_pixel(); - dims.push(tile_dims); - outputs.push(vec![0_u8; stride * tile_dims.1 as usize]); - } - - { - let mut jobs = requests - .iter() - .zip(dims.iter()) - .zip(outputs.iter_mut()) - .map(|((request, dims), out)| TileDecodeJob { - input: request.input.as_ref(), - out: out.as_mut_slice(), - stride: dims.0 as usize * fmt.bytes_per_pixel(), - }) - .collect::>(); - decode_tiles_into(&mut jobs, fmt, TileBatchOptions::default()) - .map_err(|err| Error::Decode(err.source))?; - } - - Ok(outputs - .into_iter() - .zip(dims) - .map(|(bytes, dimensions)| host_surface(bytes, dimensions, fmt)) - .collect()) -} - -fn decode_cpu_region_scaled_batch( - requests: &[QueuedRequest], -) -> Option, Error>> { - let first = requests.first()?; - if requests.len() <= 1 - || !requests.iter().all(|request| { - is_cpu_host_region_scaled_batch_candidate(request) && request.fmt == first.fmt - }) - { - return None; - } - - Some(decode_cpu_region_scaled_batch_inner(requests, first.fmt)) -} - -fn is_cpu_host_region_scaled_batch_candidate(request: &QueuedRequest) -> bool { - matches!(request.op, BatchOp::RegionScaled { .. }) - && matches!(request.backend, BackendRequest::Cpu | BackendRequest::Auto) -} - -fn decode_cpu_region_scaled_batch_inner( - requests: &[QueuedRequest], - fmt: PixelFormat, -) -> Result, Error> { - let mut dims = Vec::with_capacity(requests.len()); - let mut outputs = Vec::with_capacity(requests.len()); - for request in requests { - let BatchOp::RegionScaled { roi, scale } = request.op else { - unreachable!("candidate op is restricted above"); - }; - let dimensions = roi.scaled_covering(scale); - let stride = dimensions.w as usize * fmt.bytes_per_pixel(); - dims.push((dimensions.w, dimensions.h)); - outputs.push(vec![0_u8; stride * dimensions.h as usize]); - } - - { - let mut jobs = requests - .iter() - .zip(outputs.iter_mut()) - .map(|(request, out)| { - let BatchOp::RegionScaled { roi, scale } = request.op else { - unreachable!("candidate op is restricted above"); - }; - let dimensions = roi.scaled_covering(scale); - TileRegionScaledDecodeJob { - input: request.input.as_ref(), - out: out.as_mut_slice(), - stride: dimensions.w as usize * fmt.bytes_per_pixel(), - roi, - scale, - } - }) - .collect::>(); - decode_tiles_region_scaled_into(&mut jobs, fmt, TileBatchOptions::default()) - .map_err(|err| Error::Decode(err.source))?; - } - - Ok(outputs - .into_iter() - .zip(dims) - .map(|(bytes, dimensions)| host_surface(bytes, dimensions, fmt)) - .collect()) -} - -fn host_surface(bytes: Vec, dimensions: (u32, u32), fmt: PixelFormat) -> Surface { - Surface { - backend: BackendKind::Cpu, - residency: SurfaceResidency::Host, - dimensions, - fmt, - pitch_bytes: dimensions.0 as usize * fmt.bytes_per_pixel(), - byte_offset: 0, - storage: Storage::Host(bytes), - } -} - -fn decode_repeated_full_grayscale( - request: &QueuedRequest, - count: usize, -) -> Option, Error>> { - if !is_repeated_full_grayscale_candidate(request) || count <= 1 { - return None; - } - - #[cfg(target_os = "macos")] - { - let result = - J2kDecoder::new(request.input.as_ref()).and_then(|mut decoder| match request.backend { - BackendRequest::Auto => { - decoder.decode_repeated_grayscale_auto_to_device(request.fmt, count) - } - BackendRequest::Metal => { - decoder.decode_repeated_grayscale_direct_to_device(request.fmt, count) - } - _ => unreachable!("candidate backend is restricted above"), - }); - Some(result) - } - - #[cfg(not(target_os = "macos"))] - { - None - } -} - -fn decode_repeated_full_color( - request: &QueuedRequest, - count: usize, -) -> Option, Error>> { - if !is_repeated_full_color_candidate(request) || count <= 1 { - return None; - } - - #[cfg(target_os = "macos")] - { - let result = J2kDecoder::new(request.input.as_ref()).and_then(|mut decoder| { - decoder.decode_repeated_color_direct_to_device(request.fmt, count) - }); - Some(result) - } - - #[cfg(not(target_os = "macos"))] - { - None - } -} - -fn decode_distinct_full_grayscale_batch( - requests: &[QueuedRequest], -) -> Option, Error>> { - let first = requests.first()?; - if requests.len() <= 1 - || !requests.iter().all(|request| { - is_distinct_full_grayscale_metal_candidate(request) && request.fmt == first.fmt - }) - { - return None; - } - - #[cfg(target_os = "macos")] - { - let inputs = requests - .iter() - .map(|request| request.input.clone()) - .collect::>(); - Some(crate::decode_full_grayscale_batch_direct_to_device( - &inputs, first.fmt, - )) - } - - #[cfg(not(target_os = "macos"))] - { - None - } -} - -fn decode_distinct_full_color_batch( - requests: &[QueuedRequest], -) -> Option, Error>> { - let first = requests.first()?; - if requests.len() <= 1 - || !requests.iter().all(|request| { - is_distinct_full_color_metal_candidate(request) && request.fmt == first.fmt - }) - { - return None; - } - - #[cfg(target_os = "macos")] - { - let inputs = requests - .iter() - .map(|request| request.input.clone()) - .collect::>(); - Some(crate::decode_full_color_batch_direct_to_device( - &inputs, first.fmt, - )) - } - - #[cfg(not(target_os = "macos"))] - { - None - } -} - -fn decode_distinct_region_scaled_grayscale_batch( - requests: &[QueuedRequest], -) -> Option, Error>> { - let first = requests.first()?; - if requests.len() <= 1 - || !requests.iter().all(|request| { - is_region_scaled_grayscale_batch_candidate(request) - && request.fmt == first.fmt - && request.backend == first.backend - }) - { - return None; - } - if first.backend == BackendRequest::Auto - && !should_auto_use_metal_for_region_scaled_grayscale_batch(requests) - { - return None; - } - - #[cfg(target_os = "macos")] - { - let request_specs = requests - .iter() - .map(|request| match request.op { - BatchOp::RegionScaled { roi, scale } => (request.input.clone(), roi, scale), - _ => unreachable!("candidate op is restricted above"), - }) - .collect::>(); - Some( - crate::decode_region_scaled_grayscale_batch_direct_to_device(&request_specs, first.fmt), - ) - } - - #[cfg(not(target_os = "macos"))] - { - None - } -} - -fn decode_individual(request: &QueuedRequest) -> Result { - let mut decoder = J2kDecoder::new(request.input.as_ref())?; - match request.op { - BatchOp::Full => decoder.decode_to_surface_impl(request.fmt, request.backend), - BatchOp::Region(roi) => { - decoder.decode_region_to_surface_impl(request.fmt, roi, request.backend) - } - BatchOp::Scaled(scale) => { - decoder.decode_scaled_to_surface_impl(request.fmt, scale, request.backend) - } - BatchOp::RegionScaled { roi, scale } => { - decoder.decode_region_scaled_to_surface_impl(request.fmt, roi, scale, request.backend) - } - } -} - -fn take_surface(session: &mut SessionState, slot: usize) -> Result { - session - .completed - .get_mut(slot) - .and_then(Option::take) - .ok_or_else(|| Error::MetalKernel { - message: format!("missing queued J2K Metal surface for slot {slot}"), - })? -} diff --git a/crates/signinum-j2k-metal/src/dicom.rs b/crates/signinum-j2k-metal/src/dicom.rs deleted file mode 100644 index 7398584c..00000000 --- a/crates/signinum-j2k-metal/src/dicom.rs +++ /dev/null @@ -1,599 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#[derive(Debug, thiserror::Error, PartialEq, Eq)] -pub enum DicomFrameExtractError { - #[error("DICOM Pixel Data element was not found")] - PixelDataNotFound, - #[error("DICOM Pixel Data is not encapsulated")] - PixelDataNotEncapsulated, - #[error("DICOM Pixel Data is truncated while reading {what}")] - Truncated { what: &'static str }, - #[error("DICOM Pixel Data is malformed: {what}")] - Malformed { what: &'static str }, -} - -const PIXEL_DATA_TAG: [u8; 4] = [0xE0, 0x7F, 0x10, 0x00]; -const EXTENDED_OFFSET_TABLE_TAG: [u8; 4] = [0xE0, 0x7F, 0x01, 0x00]; -const EXTENDED_OFFSET_TABLE_LENGTHS_TAG: [u8; 4] = [0xE0, 0x7F, 0x02, 0x00]; -const ITEM_TAG: [u8; 4] = [0xFE, 0xFF, 0x00, 0xE0]; -const SEQUENCE_DELIMITATION_TAG: [u8; 4] = [0xFE, 0xFF, 0xDD, 0xE0]; -const UNDEFINED_LENGTH: u32 = u32::MAX; - -pub fn extract_dicom_encapsulated_frames( - input: &[u8], -) -> Result>, DicomFrameExtractError> { - extract_dicom_encapsulated_frames_with_limit(input, usize::MAX) -} - -pub fn extract_dicom_encapsulated_frames_with_limit( - input: &[u8], - limit: usize, -) -> Result>, DicomFrameExtractError> { - if limit == 0 { - return Ok(Vec::new()); - } - let pixel_data = find_encapsulated_pixel_data_value(input)?; - let extended_offsets = extended_offset_table_entries(input, pixel_data.tag_offset)?; - let extended_lengths = extended_offset_table_length_entries(input, pixel_data.tag_offset)?; - if let (Some(offsets), Some(lengths)) = (&extended_offsets, &extended_lengths) { - if offsets.len() != lengths.len() { - return Err(DicomFrameExtractError::Malformed { - what: "Extended Offset Table and Lengths have different entry counts", - }); - } - } - - let (bot, fragment_cursor) = read_basic_offset_table(input, pixel_data.value_offset)?; - let offsets = basic_offset_table_entries(bot)?; - let extended_offsets = offsets - .is_empty() - .then_some(extended_offsets) - .flatten() - .unwrap_or_default(); - let stop_offset = offsets - .get(limit) - .map(|offset| u64::from(*offset)) - .or_else(|| extended_offsets.get(limit).copied()); - let fragments = read_pixel_data_fragments( - input, - fragment_cursor, - stop_offset, - (offsets.is_empty() && extended_offsets.is_empty()).then_some(limit), - )?; - if fragments.is_empty() { - return Ok(Vec::new()); - } - - if offsets.is_empty() && extended_offsets.is_empty() { - return Ok(fragments - .into_iter() - .map(|fragment| fragment.payload.to_vec()) - .collect()); - } - - if !offsets.is_empty() { - let frame_count = offsets.len().min(limit); - let offsets = offsets[..frame_count] - .iter() - .copied() - .map(u64::from) - .collect::>(); - return frames_from_offset_table(&fragments, &offsets, None); - } - - let frame_count = extended_offsets.len().min(limit); - let lengths = extended_lengths - .as_ref() - .map(|lengths| &lengths[..frame_count]); - frames_from_offset_table(&fragments, &extended_offsets[..frame_count], lengths) -} - -#[derive(Clone, Copy)] -struct PixelDataLocation { - tag_offset: usize, - value_offset: usize, -} - -fn find_encapsulated_pixel_data_value( - input: &[u8], -) -> Result { - let Some(tag_offset) = input - .windows(PIXEL_DATA_TAG.len()) - .position(|window| window == PIXEL_DATA_TAG) - else { - return Err(DicomFrameExtractError::PixelDataNotFound); - }; - let header = input - .get(tag_offset..) - .ok_or(DicomFrameExtractError::Truncated { - what: "Pixel Data header", - })?; - if header.len() < 8 { - return Err(DicomFrameExtractError::Truncated { - what: "Pixel Data header", - }); - } - - let after_tag = tag_offset + 4; - let vr = input - .get(after_tag..after_tag + 2) - .ok_or(DicomFrameExtractError::Truncated { - what: "Pixel Data VR", - })?; - let (value_offset, length) = if is_explicit_vr(vr) { - if is_long_explicit_vr(vr) { - let length_offset = after_tag + 4; - ( - after_tag + 8, - read_u32(input, length_offset, "Pixel Data length")?, - ) - } else { - let length = u32::from(read_u16(input, after_tag + 2, "Pixel Data length")?); - (after_tag + 4, length) - } - } else { - ( - after_tag + 4, - read_u32(input, after_tag, "Pixel Data length")?, - ) - }; - - if length != UNDEFINED_LENGTH { - return Err(DicomFrameExtractError::PixelDataNotEncapsulated); - } - if value_offset > input.len() { - return Err(DicomFrameExtractError::Truncated { - what: "Pixel Data value", - }); - } - Ok(PixelDataLocation { - tag_offset, - value_offset, - }) -} - -#[derive(Clone, Copy)] -struct DicomFragment<'a> { - item_offset: u64, - payload: &'a [u8], -} - -fn read_basic_offset_table( - input: &[u8], - cursor: usize, -) -> Result<(&[u8], usize), DicomFrameExtractError> { - let tag = input - .get(cursor..cursor + 4) - .ok_or(DicomFrameExtractError::Truncated { - what: "Basic Offset Table item tag", - })?; - if tag != ITEM_TAG { - return Err(DicomFrameExtractError::Malformed { - what: "encapsulated Pixel Data does not start with a Basic Offset Table item", - }); - } - let length = usize::try_from(read_u32(input, cursor + 4, "Basic Offset Table length")?) - .map_err(|_| DicomFrameExtractError::Malformed { - what: "Basic Offset Table length exceeds usize", - })?; - let payload_start = cursor + 8; - let payload_end = - payload_start - .checked_add(length) - .ok_or(DicomFrameExtractError::Malformed { - what: "Basic Offset Table length overflow", - })?; - let payload = - input - .get(payload_start..payload_end) - .ok_or(DicomFrameExtractError::Truncated { - what: "Basic Offset Table payload", - })?; - Ok((payload, payload_end)) -} - -fn read_pixel_data_fragments( - input: &[u8], - mut cursor: usize, - stop_item_offset: Option, - max_fragments: Option, -) -> Result>, DicomFrameExtractError> { - let mut fragments = Vec::new(); - let mut first_fragment_item_offset = None; - loop { - let tag = input - .get(cursor..cursor + 4) - .ok_or(DicomFrameExtractError::Truncated { - what: "Pixel Data item tag", - })?; - let length = read_u32(input, cursor + 4, "Pixel Data item length")?; - cursor += 8; - if tag == SEQUENCE_DELIMITATION_TAG { - if length != 0 { - return Err(DicomFrameExtractError::Malformed { - what: "non-zero sequence delimitation length", - }); - } - break; - } - if tag != ITEM_TAG { - return Err(DicomFrameExtractError::Malformed { - what: "expected item tag in encapsulated Pixel Data", - }); - } - if length == UNDEFINED_LENGTH { - return Err(DicomFrameExtractError::Malformed { - what: "undefined-length Pixel Data item", - }); - } - let length = usize::try_from(length).map_err(|_| DicomFrameExtractError::Malformed { - what: "Pixel Data item length exceeds usize", - })?; - let payload_end = cursor - .checked_add(length) - .ok_or(DicomFrameExtractError::Malformed { - what: "Pixel Data item length overflow", - })?; - let payload = input - .get(cursor..payload_end) - .ok_or(DicomFrameExtractError::Truncated { - what: "Pixel Data item payload", - })?; - - let item_tag_offset = cursor - 8; - let first = *first_fragment_item_offset.get_or_insert(item_tag_offset); - let item_offset = u64::try_from(item_tag_offset - first).map_err(|_| { - DicomFrameExtractError::Malformed { - what: "fragment item offset exceeds u64", - } - })?; - if stop_item_offset.is_some_and(|stop| item_offset >= stop) { - break; - } - fragments.push(DicomFragment { - item_offset, - payload, - }); - if max_fragments.is_some_and(|max| fragments.len() >= max) { - break; - } - cursor = payload_end; - } - - Ok(fragments) -} - -fn extended_offset_table_entries( - input: &[u8], - before: usize, -) -> Result>, DicomFrameExtractError> { - let Some(payload) = read_element_value_before( - input, - EXTENDED_OFFSET_TABLE_TAG, - before, - "Extended Offset Table", - )? - else { - return Ok(None); - }; - Ok(Some(u64_entries(payload, "Extended Offset Table")?)) -} - -fn extended_offset_table_length_entries( - input: &[u8], - before: usize, -) -> Result>, DicomFrameExtractError> { - let Some(payload) = read_element_value_before( - input, - EXTENDED_OFFSET_TABLE_LENGTHS_TAG, - before, - "Extended Offset Table Lengths", - )? - else { - return Ok(None); - }; - Ok(Some(u64_entries(payload, "Extended Offset Table Lengths")?)) -} - -fn read_element_value_before<'a>( - input: &'a [u8], - tag: [u8; 4], - before: usize, - name: &'static str, -) -> Result, DicomFrameExtractError> { - let search = input - .get(..before) - .ok_or(DicomFrameExtractError::Truncated { - what: "DICOM element search range", - })?; - let Some(tag_offset) = search.windows(tag.len()).position(|window| window == tag) else { - return Ok(None); - }; - read_element_value_at(input, tag_offset, name).map(Some) -} - -fn read_element_value_at<'a>( - input: &'a [u8], - tag_offset: usize, - name: &'static str, -) -> Result<&'a [u8], DicomFrameExtractError> { - let after_tag = tag_offset + 4; - let vr = input - .get(after_tag..after_tag + 2) - .ok_or(DicomFrameExtractError::Truncated { what: name })?; - let (value_offset, length) = if is_explicit_vr(vr) { - if is_long_explicit_vr(vr) { - ( - after_tag + 8, - read_u32(input, after_tag + 4, "DICOM element length")?, - ) - } else { - ( - after_tag + 4, - u32::from(read_u16(input, after_tag + 2, "DICOM element length")?), - ) - } - } else { - ( - after_tag + 4, - read_u32(input, after_tag, "DICOM element length")?, - ) - }; - if length == UNDEFINED_LENGTH { - return Err(DicomFrameExtractError::Malformed { - what: "Extended Offset Table element has undefined length", - }); - } - let length = usize::try_from(length).map_err(|_| DicomFrameExtractError::Malformed { - what: "DICOM element length exceeds usize", - })?; - let value_end = value_offset - .checked_add(length) - .ok_or(DicomFrameExtractError::Malformed { - what: "DICOM element length overflow", - })?; - input - .get(value_offset..value_end) - .ok_or(DicomFrameExtractError::Truncated { what: name }) -} - -fn u64_entries(payload: &[u8], name: &'static str) -> Result, DicomFrameExtractError> { - if !payload.len().is_multiple_of(8) { - return Err(DicomFrameExtractError::Malformed { what: name }); - } - Ok(payload - .chunks_exact(8) - .map(|chunk| { - u64::from_le_bytes([ - chunk[0], chunk[1], chunk[2], chunk[3], chunk[4], chunk[5], chunk[6], chunk[7], - ]) - }) - .collect()) -} - -fn basic_offset_table_entries(bot: &[u8]) -> Result, DicomFrameExtractError> { - if bot.is_empty() { - return Ok(Vec::new()); - } - if !bot.len().is_multiple_of(4) { - return Err(DicomFrameExtractError::Malformed { - what: "Basic Offset Table length is not a multiple of four", - }); - } - Ok(bot - .chunks_exact(4) - .map(|chunk| u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) - .collect()) -} - -fn frames_from_offset_table( - fragments: &[DicomFragment<'_>], - offsets: &[u64], - lengths: Option<&[u64]>, -) -> Result>, DicomFrameExtractError> { - let mut frames = Vec::with_capacity(offsets.len()); - for (index, start) in offsets.iter().copied().enumerate() { - let end = offsets.get(index + 1).copied().unwrap_or(u64::MAX); - if end <= start { - return Err(DicomFrameExtractError::Malformed { - what: "offset table offsets are not strictly increasing", - }); - } - let mut frame = Vec::new(); - let max_payload_len = lengths - .and_then(|entries| entries.get(index)) - .and_then(|length| usize::try_from(*length).ok()); - for fragment in fragments - .iter() - .filter(|fragment| fragment.item_offset >= start && fragment.item_offset < end) - { - if let Some(max_payload_len) = max_payload_len { - let remaining = max_payload_len.saturating_sub(frame.len()); - frame.extend_from_slice(&fragment.payload[..fragment.payload.len().min(remaining)]); - if frame.len() >= max_payload_len { - break; - } - } else { - frame.extend_from_slice(fragment.payload); - } - } - if frame.is_empty() { - return Err(DicomFrameExtractError::Malformed { - what: "offset table frame has no fragments", - }); - } - frames.push(frame); - } - Ok(frames) -} - -fn is_explicit_vr(vr: &[u8]) -> bool { - vr.len() == 2 && vr.iter().all(u8::is_ascii_uppercase) -} - -fn is_long_explicit_vr(vr: &[u8]) -> bool { - matches!( - vr, - b"OB" | b"OD" | b"OF" | b"OL" | b"OV" | b"OW" | b"SQ" | b"UC" | b"UR" | b"UT" | b"UN" - ) -} - -fn read_u16( - input: &[u8], - offset: usize, - what: &'static str, -) -> Result { - let bytes = input - .get(offset..offset + 2) - .ok_or(DicomFrameExtractError::Truncated { what })?; - Ok(u16::from_le_bytes([bytes[0], bytes[1]])) -} - -fn read_u32( - input: &[u8], - offset: usize, - what: &'static str, -) -> Result { - let bytes = input - .get(offset..offset + 4) - .ok_or(DicomFrameExtractError::Truncated { what })?; - Ok(u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn push_item(bytes: &mut Vec, payload: &[u8]) { - bytes.extend_from_slice(&[0xFE, 0xFF, 0x00, 0xE0]); - bytes.extend_from_slice(&(payload.len() as u32).to_le_bytes()); - bytes.extend_from_slice(payload); - } - - fn explicit_ob_pixel_data(bot: &[u8], fragments: &[&[u8]]) -> Vec { - let mut bytes = vec![0; 128]; - bytes.extend_from_slice(b"DICM"); - bytes.extend_from_slice(&[0xE0, 0x7F, 0x10, 0x00]); - bytes.extend_from_slice(b"OB"); - bytes.extend_from_slice(&[0, 0]); - bytes.extend_from_slice(&u32::MAX.to_le_bytes()); - push_item(&mut bytes, bot); - for fragment in fragments { - push_item(&mut bytes, fragment); - } - bytes.extend_from_slice(&[0xFE, 0xFF, 0xDD, 0xE0]); - bytes.extend_from_slice(&0_u32.to_le_bytes()); - bytes - } - - fn push_explicit_ov_element(bytes: &mut Vec, tag: [u8; 4], payload: &[u8]) { - bytes.extend_from_slice(&tag); - bytes.extend_from_slice(b"OV"); - bytes.extend_from_slice(&[0, 0]); - bytes.extend_from_slice(&(payload.len() as u32).to_le_bytes()); - bytes.extend_from_slice(payload); - } - - fn explicit_ob_pixel_data_with_extended_offsets( - offsets: &[u64], - lengths: &[u64], - bot: &[u8], - fragments: &[&[u8]], - ) -> Vec { - let mut bytes = vec![0; 128]; - bytes.extend_from_slice(b"DICM"); - let mut offset_payload = Vec::new(); - for offset in offsets { - offset_payload.extend_from_slice(&offset.to_le_bytes()); - } - push_explicit_ov_element(&mut bytes, [0xE0, 0x7F, 0x01, 0x00], &offset_payload); - let mut length_payload = Vec::new(); - for length in lengths { - length_payload.extend_from_slice(&length.to_le_bytes()); - } - push_explicit_ov_element(&mut bytes, [0xE0, 0x7F, 0x02, 0x00], &length_payload); - bytes.extend_from_slice(&[0xE0, 0x7F, 0x10, 0x00]); - bytes.extend_from_slice(b"OB"); - bytes.extend_from_slice(&[0, 0]); - bytes.extend_from_slice(&u32::MAX.to_le_bytes()); - push_item(&mut bytes, bot); - for fragment in fragments { - push_item(&mut bytes, fragment); - } - bytes.extend_from_slice(&[0xFE, 0xFF, 0xDD, 0xE0]); - bytes.extend_from_slice(&0_u32.to_le_bytes()); - bytes - } - - #[test] - fn dicom_empty_basic_offset_table_extracts_each_fragment_as_frame() { - let bytes = explicit_ob_pixel_data(&[], &[b"first", b"second"]); - - let frames = extract_dicom_encapsulated_frames(&bytes).expect("extract frames"); - - assert_eq!(frames, vec![b"first".to_vec(), b"second".to_vec()]); - } - - #[test] - fn dicom_basic_offset_table_groups_fragments_into_frames() { - let first = b"aa"; - let second = b"bb"; - let third = b"cc"; - let second_frame_offset = 8 + first.len() as u32 + 8 + second.len() as u32; - let mut bot = Vec::new(); - bot.extend_from_slice(&0_u32.to_le_bytes()); - bot.extend_from_slice(&second_frame_offset.to_le_bytes()); - let bytes = explicit_ob_pixel_data(&bot, &[first, second, third]); - - let frames = extract_dicom_encapsulated_frames(&bytes).expect("extract frames"); - - assert_eq!(frames, vec![b"aabb".to_vec(), b"cc".to_vec()]); - } - - #[test] - fn dicom_extended_offset_table_lengths_trim_padded_frame_payloads() { - let first_with_padding = b"abc\0"; - let second = b"de"; - let second_frame_offset = 8 + first_with_padding.len() as u64; - let bytes = explicit_ob_pixel_data_with_extended_offsets( - &[0, second_frame_offset], - &[3, second.len() as u64], - &[], - &[first_with_padding, second], - ); - - let frames = extract_dicom_encapsulated_frames(&bytes).expect("extract frames"); - - assert_eq!(frames, vec![b"abc".to_vec(), b"de".to_vec()]); - } - - #[test] - fn dicom_extract_limit_stops_after_requested_empty_bot_frames() { - let bytes = explicit_ob_pixel_data(&[], &[b"first", b"second", b"third"]); - - let frames = - extract_dicom_encapsulated_frames_with_limit(&bytes, 2).expect("extract frames"); - - assert_eq!(frames, vec![b"first".to_vec(), b"second".to_vec()]); - } - - #[test] - fn dicom_extract_limit_stops_after_requested_basic_offset_frames() { - let first = b"aa"; - let second = b"bb"; - let third = b"cc"; - let fourth = b"dd"; - let second_frame_offset = 8 + first.len() as u32 + 8 + second.len() as u32; - let third_frame_offset = second_frame_offset + 8 + third.len() as u32; - let mut bot = Vec::new(); - bot.extend_from_slice(&0_u32.to_le_bytes()); - bot.extend_from_slice(&second_frame_offset.to_le_bytes()); - bot.extend_from_slice(&third_frame_offset.to_le_bytes()); - let bytes = explicit_ob_pixel_data(&bot, &[first, second, third, fourth]); - - let frames = - extract_dicom_encapsulated_frames_with_limit(&bytes, 2).expect("extract frames"); - - assert_eq!(frames, vec![b"aabb".to_vec(), b"cc".to_vec()]); - } -} diff --git a/crates/signinum-j2k-metal/src/encode.rs b/crates/signinum-j2k-metal/src/encode.rs deleted file mode 100644 index 7c3f1ffb..00000000 --- a/crates/signinum-j2k-metal/src/encode.rs +++ /dev/null @@ -1,5801 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#[cfg(target_os = "macos")] -use crate::compute; -#[cfg(target_os = "macos")] -use metal::Buffer; -#[cfg(target_os = "macos")] -use rayon::prelude::*; -use signinum_core::DeviceSubmission; -#[cfg(target_os = "macos")] -use signinum_core::{BackendKind, DeviceSurface, PixelFormat}; -#[cfg(target_os = "macos")] -use signinum_j2k::{ - EncodeBackendPreference, J2kBlockCodingMode, J2kEncodeValidation, J2kProgressionOrder, -}; -use signinum_j2k::{EncodedJ2k, J2kLosslessEncodeOptions, J2kLosslessSamples}; -#[cfg(target_os = "macos")] -use signinum_j2k_native::{ - EncodeProgressionOrder, J2kPacketizationPacketDescriptor, J2kSubBandType, -}; -use signinum_j2k_native::{ - EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, J2kEncodeDispatchReport, J2kEncodeStageAccelerator, - J2kForwardDwt53Job, J2kForwardDwt53Output, J2kForwardRctJob, J2kHtCodeBlockEncodeJob, - J2kPacketizationEncodeJob, J2kTier1CodeBlockEncodeJob, -}; -#[cfg(all(test, target_os = "macos"))] -use std::cell::Cell; -#[cfg(target_os = "macos")] -use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, -}; -use std::time::Duration; -#[cfg(target_os = "macos")] -use std::time::Instant; - -/// Encode-stage accelerator for JPEG 2000 Metal work. -/// -/// The type is wired into the native encoder hook interface and reports -/// dispatches for each required encode stage. -#[derive(Debug, Clone)] -pub struct MetalEncodeStageAccelerator { - dispatch_stages: MetalEncodeDispatchStages, - parallel_cpu_code_block_fallback: bool, - forward_rct_attempts: usize, - forward_dwt53_attempts: usize, - tier1_code_block_attempts: usize, - ht_code_block_attempts: usize, - packetization_attempts: usize, - forward_rct_dispatches: usize, - forward_dwt53_dispatches: usize, - tier1_code_block_dispatches: usize, - ht_code_block_dispatches: usize, - packetization_dispatches: usize, -} - -impl Default for MetalEncodeStageAccelerator { - fn default() -> Self { - Self { - dispatch_stages: MetalEncodeDispatchStages::ALL, - parallel_cpu_code_block_fallback: false, - forward_rct_attempts: 0, - forward_dwt53_attempts: 0, - tier1_code_block_attempts: 0, - ht_code_block_attempts: 0, - packetization_attempts: 0, - forward_rct_dispatches: 0, - forward_dwt53_dispatches: 0, - tier1_code_block_dispatches: 0, - ht_code_block_dispatches: 0, - packetization_dispatches: 0, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct MetalEncodeDispatchStages(u8); - -impl MetalEncodeDispatchStages { - const FORWARD_RCT: Self = Self(1 << 0); - const FORWARD_DWT53: Self = Self(1 << 1); - const TIER1_CODE_BLOCK: Self = Self(1 << 2); - const HT_CODE_BLOCK: Self = Self(1 << 3); - const PACKETIZATION: Self = Self(1 << 4); - const ALL: Self = Self( - Self::FORWARD_RCT.0 - | Self::FORWARD_DWT53.0 - | Self::TIER1_CODE_BLOCK.0 - | Self::HT_CODE_BLOCK.0 - | Self::PACKETIZATION.0, - ); - - fn contains(self, stage: Self) -> bool { - self.0 & stage.0 != 0 - } - - fn without(self, stage: Self) -> Self { - Self(self.0 & !stage.0) - } -} - -impl MetalEncodeStageAccelerator { - pub fn with_cpu_forward_rct() -> Self { - Self { - dispatch_stages: MetalEncodeDispatchStages::ALL - .without(MetalEncodeDispatchStages::FORWARD_RCT), - ..Self::default() - } - } - - pub fn for_auto_host_output() -> Self { - Self { - dispatch_stages: MetalEncodeDispatchStages::FORWARD_DWT53, - parallel_cpu_code_block_fallback: true, - ..Self::default() - } - } - - #[cfg(target_os = "macos")] - fn for_host_output(options: J2kLosslessEncodeOptions) -> Self { - if options.backend == EncodeBackendPreference::Auto { - Self::for_auto_host_output() - } else { - Self::with_cpu_forward_rct() - } - } - - pub fn forward_rct_attempts(&self) -> usize { - self.forward_rct_attempts - } - - pub fn forward_dwt53_attempts(&self) -> usize { - self.forward_dwt53_attempts - } - - pub fn tier1_code_block_attempts(&self) -> usize { - self.tier1_code_block_attempts - } - - pub fn ht_code_block_attempts(&self) -> usize { - self.ht_code_block_attempts - } - - pub fn packetization_attempts(&self) -> usize { - self.packetization_attempts - } - - pub fn forward_rct_dispatches(&self) -> usize { - self.forward_rct_dispatches - } - - pub fn forward_dwt53_dispatches(&self) -> usize { - self.forward_dwt53_dispatches - } - - pub fn tier1_code_block_dispatches(&self) -> usize { - self.tier1_code_block_dispatches - } - - pub fn ht_code_block_dispatches(&self) -> usize { - self.ht_code_block_dispatches - } - - pub fn packetization_dispatches(&self) -> usize { - self.packetization_dispatches - } -} - -#[cfg(target_os = "macos")] -fn metal_dispatch_result( - result: &Result<(), crate::Error>, - message: &'static str, -) -> Result { - match result { - Ok(()) => Ok(true), - Err(crate::Error::MetalUnavailable) => Ok(false), - Err(_) => Err(message), - } -} - -#[cfg(target_os = "macos")] -fn metal_dispatch_option( - result: Result, - message: &'static str, -) -> Result, &'static str> { - match result { - Ok(value) => Ok(Some(value)), - Err(crate::Error::MetalUnavailable) => Ok(None), - Err(_) => Err(message), - } -} - -impl J2kEncodeStageAccelerator for MetalEncodeStageAccelerator { - fn dispatch_report(&self) -> J2kEncodeDispatchReport { - J2kEncodeDispatchReport { - forward_rct: self.forward_rct_dispatches, - forward_dwt53: self.forward_dwt53_dispatches, - tier1_code_block: self.tier1_code_block_dispatches, - ht_code_block: self.ht_code_block_dispatches, - packetization: self.packetization_dispatches, - } - } - - fn prefer_parallel_cpu_code_block_fallback(&self) -> bool { - self.parallel_cpu_code_block_fallback - } - - fn encode_forward_rct( - &mut self, - job: J2kForwardRctJob<'_>, - ) -> core::result::Result { - self.forward_rct_attempts = self.forward_rct_attempts.saturating_add(1); - if !self - .dispatch_stages - .contains(MetalEncodeDispatchStages::FORWARD_RCT) - { - let _ = job; - return Ok(false); - } - #[cfg(target_os = "macos")] - { - let result = compute::encode_forward_rct(job.plane0, job.plane1, job.plane2); - let dispatched = - metal_dispatch_result(&result, "Metal forward RCT encode kernel failed")?; - if dispatched { - self.forward_rct_dispatches = self.forward_rct_dispatches.saturating_add(1); - } - Ok(dispatched) - } - #[cfg(not(target_os = "macos"))] - { - let _ = job; - Ok(false) - } - } - - fn encode_forward_dwt53( - &mut self, - job: J2kForwardDwt53Job<'_>, - ) -> core::result::Result, &'static str> { - self.forward_dwt53_attempts = self.forward_dwt53_attempts.saturating_add(1); - if job.num_levels == 0 { - return Ok(None); - } - if !self - .dispatch_stages - .contains(MetalEncodeDispatchStages::FORWARD_DWT53) - { - let _ = job; - return Ok(None); - } - #[cfg(target_os = "macos")] - { - let output = metal_dispatch_option( - compute::encode_forward_dwt53(job.samples, job.width, job.height, job.num_levels), - "Metal forward 5/3 DWT encode kernel failed", - )?; - if output.is_some() { - self.forward_dwt53_dispatches = self.forward_dwt53_dispatches.saturating_add(1); - } - Ok(output) - } - #[cfg(not(target_os = "macos"))] - { - let _ = job; - Ok(None) - } - } - - fn encode_tier1_code_block( - &mut self, - job: J2kTier1CodeBlockEncodeJob<'_>, - ) -> core::result::Result, &'static str> { - self.tier1_code_block_attempts = self.tier1_code_block_attempts.saturating_add(1); - if !self - .dispatch_stages - .contains(MetalEncodeDispatchStages::TIER1_CODE_BLOCK) - { - let _ = job; - return Ok(None); - } - #[cfg(target_os = "macos")] - { - let encoded = metal_dispatch_option( - compute::encode_classic_tier1_code_block(job), - "Metal classic Tier-1 encode kernel failed", - )?; - if encoded.is_some() { - self.tier1_code_block_dispatches = - self.tier1_code_block_dispatches.saturating_add(1); - } - Ok(encoded) - } - #[cfg(not(target_os = "macos"))] - { - let _ = job; - Ok(None) - } - } - - fn encode_tier1_code_blocks( - &mut self, - jobs: &[J2kTier1CodeBlockEncodeJob<'_>], - ) -> core::result::Result>, &'static str> { - self.tier1_code_block_attempts = self.tier1_code_block_attempts.saturating_add(jobs.len()); - if !self - .dispatch_stages - .contains(MetalEncodeDispatchStages::TIER1_CODE_BLOCK) - { - let _ = jobs; - return Ok(None); - } - #[cfg(target_os = "macos")] - { - let encoded = metal_dispatch_option( - compute::encode_classic_tier1_code_blocks(jobs), - "Metal classic Tier-1 encode batch kernel failed", - )?; - if encoded.is_some() && !jobs.is_empty() { - self.tier1_code_block_dispatches = - self.tier1_code_block_dispatches.saturating_add(1); - } - Ok(encoded) - } - #[cfg(not(target_os = "macos"))] - { - let _ = jobs; - Ok(None) - } - } - - fn encode_ht_code_block( - &mut self, - job: J2kHtCodeBlockEncodeJob<'_>, - ) -> core::result::Result, &'static str> { - self.ht_code_block_attempts = self.ht_code_block_attempts.saturating_add(1); - if !self - .dispatch_stages - .contains(MetalEncodeDispatchStages::HT_CODE_BLOCK) - { - let _ = job; - return Ok(None); - } - #[cfg(target_os = "macos")] - { - let encoded = metal_dispatch_option( - compute::encode_ht_cleanup_code_block(job), - "Metal HTJ2K code-block encode kernel failed", - )?; - if encoded.is_some() { - self.ht_code_block_dispatches = self.ht_code_block_dispatches.saturating_add(1); - } - Ok(encoded) - } - #[cfg(not(target_os = "macos"))] - { - let _ = job; - Ok(None) - } - } - - fn encode_ht_code_blocks( - &mut self, - jobs: &[J2kHtCodeBlockEncodeJob<'_>], - ) -> core::result::Result>, &'static str> { - self.ht_code_block_attempts = self.ht_code_block_attempts.saturating_add(jobs.len()); - if !self - .dispatch_stages - .contains(MetalEncodeDispatchStages::HT_CODE_BLOCK) - { - let _ = jobs; - return Ok(None); - } - #[cfg(target_os = "macos")] - { - let encoded = metal_dispatch_option( - compute::encode_ht_cleanup_code_blocks(jobs), - "Metal HTJ2K code-block encode batch kernel failed", - )?; - if encoded.is_some() && !jobs.is_empty() { - self.ht_code_block_dispatches = self.ht_code_block_dispatches.saturating_add(1); - } - Ok(encoded) - } - #[cfg(not(target_os = "macos"))] - { - let _ = jobs; - Ok(None) - } - } - - fn encode_packetization( - &mut self, - job: J2kPacketizationEncodeJob<'_>, - ) -> core::result::Result>, &'static str> { - self.packetization_attempts = self.packetization_attempts.saturating_add(1); - if !self - .dispatch_stages - .contains(MetalEncodeDispatchStages::PACKETIZATION) - { - let _ = job; - return Ok(None); - } - #[cfg(target_os = "macos")] - { - let encoded = metal_dispatch_option( - compute::encode_tier2_packetization(job), - "Metal Tier-2 packetization encode kernel failed", - )?; - if encoded.is_some() { - self.packetization_dispatches = self.packetization_dispatches.saturating_add(1); - } - Ok(encoded) - } - #[cfg(not(target_os = "macos"))] - { - let _ = job; - Ok(None) - } - } -} - -#[cfg(target_os = "macos")] -#[derive(Debug, Clone, Copy)] -pub struct MetalLosslessEncodeTile<'a> { - pub buffer: &'a Buffer, - pub byte_offset: usize, - pub width: u32, - pub height: u32, - pub pitch_bytes: usize, - pub output_width: u32, - pub output_height: u32, - pub format: PixelFormat, -} - -#[cfg(not(target_os = "macos"))] -#[derive(Debug, Clone, Copy)] -pub struct MetalLosslessEncodeTile<'a> { - _private: core::marker::PhantomData<&'a ()>, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MetalLosslessEncodeResidency { - pub coefficient_prep_used: bool, - pub packetization_used: bool, - pub codestream_assembly_used: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MetalLosslessEncodeOutcome { - pub encoded: EncodedJ2k, - pub input_copy_used: bool, - pub resident: MetalLosslessEncodeResidency, - pub input_copy_duration: Duration, - pub encode_duration: Duration, - pub gpu_duration: Option, - pub validation_duration: Duration, -} - -#[cfg(target_os = "macos")] -/// JPEG 2000 codestream bytes owned by a Metal buffer. -/// -/// The buffer is CPU-readable for the current padded resident encode API, so -/// callers can stream `codestream_bytes()` into file or network writers without -/// first materializing an owned `Vec`. -pub struct MetalEncodedJ2k { - pub codestream_buffer: Buffer, - pub byte_offset: usize, - pub byte_len: usize, - pub capacity: usize, - pub width: u32, - pub height: u32, - pub components: u8, - pub bit_depth: u8, - pub signed: bool, -} - -#[cfg(target_os = "macos")] -impl MetalEncodedJ2k { - /// Borrow the finished codestream bytes from the backing Metal buffer. - pub fn codestream_bytes(&self) -> Result<&[u8], crate::Error> { - let end = self.byte_offset.checked_add(self.byte_len).ok_or_else(|| { - crate::Error::MetalKernel { - message: "J2K Metal codestream byte range overflow".to_string(), - } - })?; - let buffer_len = usize::try_from(self.codestream_buffer.length()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal codestream buffer length exceeds usize".to_string(), - } - })?; - if end > buffer_len { - return Err(crate::Error::MetalKernel { - message: "J2K Metal codestream byte range exceeds buffer length".to_string(), - }); - } - let ptr = self.codestream_buffer.contents().cast::(); - if ptr.is_null() { - return Err(crate::Error::MetalKernel { - message: "J2K Metal codestream buffer is not CPU-readable".to_string(), - }); - } - Ok(unsafe { core::slice::from_raw_parts(ptr.add(self.byte_offset), self.byte_len) }) - } - - /// Materialize the buffer-backed codestream into the compatibility `Vec` API shape. - pub fn to_encoded_j2k(&self) -> Result { - Ok(EncodedJ2k { - codestream: self.codestream_bytes()?.to_vec(), - backend: BackendKind::Metal, - width: self.width, - height: self.height, - components: self.components, - bit_depth: self.bit_depth, - signed: self.signed, - }) - } -} - -#[cfg(not(target_os = "macos"))] -pub struct MetalEncodedJ2k { - _private: (), -} - -/// Metal lossless encode report for buffer-backed codestream output. -pub struct MetalLosslessBufferEncodeOutcome { - pub encoded: MetalEncodedJ2k, - pub input_copy_used: bool, - pub resident: MetalLosslessEncodeResidency, - pub input_copy_duration: Duration, - pub encode_duration: Duration, - pub gpu_duration: Option, - pub validation_duration: Duration, -} - -/// Tuning knobs for resident Metal lossless J2K/HTJ2K tile batch encode. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct MetalLosslessEncodeConfig { - /// Requested maximum number of tiles submitted concurrently. - /// - /// `None` uses the crate default and still clamps by the memory budget. - pub gpu_encode_inflight_tiles: Option, - /// Resident encode memory budget in bytes. - /// - /// `None` uses `min(10 GiB, hw_memsize * 0.40)` when host memory can be - /// discovered. - pub gpu_encode_memory_budget_bytes: Option, -} - -/// Resolved resident Metal lossless J2K/HTJ2K tile batch encode metrics. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct MetalLosslessEncodeBatchStats { - pub configured_inflight_tiles: Option, - pub effective_inflight_tiles: usize, - pub configured_memory_budget_bytes: Option, - pub effective_memory_budget_bytes: usize, - pub estimated_peak_bytes_per_tile: usize, - pub max_observed_inflight_tiles: usize, - pub encode_wall_duration: Duration, -} - -/// Resident Metal lossless J2K/HTJ2K tile batch output and batch-level metrics. -pub struct MetalLosslessBufferEncodeBatchOutcome { - pub outcomes: Vec, - pub stats: MetalLosslessEncodeBatchStats, -} - -#[cfg(target_os = "macos")] -pub struct SubmittedJ2kLosslessMetalEncode { - inner: SubmittedJ2kLosslessMetalEncodeBatch, -} - -#[cfg(target_os = "macos")] -pub struct SubmittedJ2kLosslessMetalEncodeBatch { - state: SubmittedJ2kLosslessMetalEncodeBatchState, -} - -#[cfg(target_os = "macos")] -enum SubmittedJ2kLosslessMetalEncodeBatchState { - Ready(Vec), - Deferred { - tiles: Vec, - options: J2kLosslessEncodeOptions, - session: crate::MetalBackendSession, - staging: MetalEncodeInputStaging, - config: MetalLosslessEncodeConfig, - }, -} - -#[cfg(target_os = "macos")] -struct OwnedMetalLosslessEncodeTile { - buffer: Buffer, - byte_offset: usize, - width: u32, - height: u32, - pitch_bytes: usize, - output_width: u32, - output_height: u32, - format: PixelFormat, -} - -#[cfg(target_os = "macos")] -impl OwnedMetalLosslessEncodeTile { - fn from_tile(tile: MetalLosslessEncodeTile<'_>) -> Self { - Self { - buffer: tile.buffer.to_owned(), - byte_offset: tile.byte_offset, - width: tile.width, - height: tile.height, - pitch_bytes: tile.pitch_bytes, - output_width: tile.output_width, - output_height: tile.output_height, - format: tile.format, - } - } - - fn as_tile(&self) -> MetalLosslessEncodeTile<'_> { - MetalLosslessEncodeTile { - buffer: &self.buffer, - byte_offset: self.byte_offset, - width: self.width, - height: self.height, - pitch_bytes: self.pitch_bytes, - output_width: self.output_width, - output_height: self.output_height, - format: self.format, - } - } -} - -#[cfg(not(target_os = "macos"))] -pub struct SubmittedJ2kLosslessMetalEncode { - _private: (), -} - -#[cfg(not(target_os = "macos"))] -pub struct SubmittedJ2kLosslessMetalEncodeBatch { - _private: (), -} - -#[cfg(target_os = "macos")] -impl DeviceSubmission for SubmittedJ2kLosslessMetalEncode { - type Output = EncodedJ2k; - type Error = crate::Error; - - fn wait(self) -> Result { - let mut encoded = self.inner.wait()?; - if encoded.len() != 1 { - return Err(crate::Error::MetalKernel { - message: "submitted J2K Metal single encode produced an unexpected batch length" - .to_string(), - }); - } - Ok(encoded.remove(0)) - } -} - -#[cfg(target_os = "macos")] -impl DeviceSubmission for SubmittedJ2kLosslessMetalEncodeBatch { - type Output = Vec; - type Error = crate::Error; - - fn wait(self) -> Result { - match self.state { - SubmittedJ2kLosslessMetalEncodeBatchState::Ready(encoded) => Ok(encoded), - SubmittedJ2kLosslessMetalEncodeBatchState::Deferred { - tiles, - options, - session, - staging, - config, - } => { - encode_lossless_owned_tiles_with_report(&tiles, options, &session, staging, config) - .map(|outcomes| { - outcomes - .into_iter() - .map(|outcome| outcome.encoded) - .collect() - }) - } - } - } -} - -#[cfg(not(target_os = "macos"))] -impl DeviceSubmission for SubmittedJ2kLosslessMetalEncode { - type Output = EncodedJ2k; - type Error = crate::Error; - - fn wait(self) -> Result { - let _ = self; - Err(crate::Error::MetalUnavailable) - } -} - -#[cfg(not(target_os = "macos"))] -impl DeviceSubmission for SubmittedJ2kLosslessMetalEncodeBatch { - type Output = Vec; - type Error = crate::Error; - - fn wait(self) -> Result { - let _ = self; - Err(crate::Error::MetalUnavailable) - } -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_metal_buffer( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - submit_lossless_from_metal_buffer(tile, options, session)?.wait() -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_metal_buffer_to_metal( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - Ok(encode_lossless_from_metal_buffer_to_metal_with_report(tile, options, session)?.encoded) -} - -#[cfg(target_os = "macos")] -pub fn submit_lossless_from_metal_buffer( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let inner = submit_lossless_from_metal_buffers(&[tile], options, session)?; - Ok(SubmittedJ2kLosslessMetalEncode { inner }) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_metal_buffer_with_report( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let mut accelerator = MetalEncodeStageAccelerator::for_host_output(*options); - encode_lossless_tile_with_report( - tile, - *options, - session, - MetalEncodeInputStaging::CopyAndPad, - &mut accelerator, - ) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_metal_buffer_to_metal_with_report( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let mut outcomes = - encode_lossless_from_metal_buffers_to_metal_with_report(&[tile], options, session)?; - if outcomes.len() != 1 { - return Err(crate::Error::MetalKernel { - message: "J2K Metal single buffer encode produced an unexpected batch length" - .to_string(), - }); - } - Ok(outcomes.remove(0)) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_padded_metal_buffer( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - submit_lossless_from_padded_metal_buffer(tile, options, session)?.wait() -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_padded_metal_buffer_to_metal( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - Ok( - encode_lossless_from_padded_metal_buffer_to_metal_with_report(tile, options, session)? - .encoded, - ) -} - -#[cfg(target_os = "macos")] -pub fn submit_lossless_from_padded_metal_buffer( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let inner = submit_lossless_from_padded_metal_buffers(&[tile], options, session)?; - Ok(SubmittedJ2kLosslessMetalEncode { inner }) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_padded_metal_buffer_with_report( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let mut accelerator = MetalEncodeStageAccelerator::for_host_output(*options); - encode_lossless_tile_with_report( - tile, - *options, - session, - MetalEncodeInputStaging::AlreadyPaddedContiguous, - &mut accelerator, - ) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_padded_metal_buffer_to_metal_with_report( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let mut outcomes = - encode_lossless_from_padded_metal_buffers_to_metal_with_report(&[tile], options, session)?; - if outcomes.len() != 1 { - return Err(crate::Error::MetalKernel { - message: "J2K Metal single buffer encode produced an unexpected batch length" - .to_string(), - }); - } - Ok(outcomes.remove(0)) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_metal_buffers( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - submit_lossless_from_metal_buffers(tiles, options, session)?.wait() -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_metal_buffers_to_metal( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - Ok( - encode_lossless_from_metal_buffers_to_metal_with_report(tiles, options, session)? - .into_iter() - .map(|outcome| outcome.encoded) - .collect(), - ) -} - -#[cfg(target_os = "macos")] -pub fn submit_lossless_from_metal_buffers( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - submit_lossless_from_metal_buffers_with_config( - tiles, - options, - session, - MetalLosslessEncodeConfig::default(), - ) -} - -#[cfg(target_os = "macos")] -pub fn submit_lossless_from_metal_buffers_with_config( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - config: MetalLosslessEncodeConfig, -) -> Result { - submit_lossless_tiles( - tiles, - *options, - session, - MetalEncodeInputStaging::CopyAndPad, - config, - ) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_metal_buffers_with_report( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - encode_lossless_tiles_with_report( - tiles, - *options, - session, - MetalEncodeInputStaging::CopyAndPad, - ) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_metal_buffers_to_metal_with_report( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - Ok(encode_lossless_from_metal_buffers_to_metal_batch( - tiles, - options, - session, - MetalLosslessEncodeConfig::default(), - )? - .outcomes) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_metal_buffers_to_metal_batch( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - config: MetalLosslessEncodeConfig, -) -> Result { - encode_lossless_tiles_to_metal_buffer_batch( - tiles, - *options, - session, - MetalEncodeInputStaging::CopyAndPad, - config, - ) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_padded_metal_buffers( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - submit_lossless_from_padded_metal_buffers(tiles, options, session)?.wait() -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_padded_metal_buffers_to_metal( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - Ok( - encode_lossless_from_padded_metal_buffers_to_metal_with_report(tiles, options, session)? - .into_iter() - .map(|outcome| outcome.encoded) - .collect(), - ) -} - -#[cfg(target_os = "macos")] -pub fn submit_lossless_from_padded_metal_buffers( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - submit_lossless_from_padded_metal_buffers_with_config( - tiles, - options, - session, - MetalLosslessEncodeConfig::default(), - ) -} - -#[cfg(target_os = "macos")] -pub fn submit_lossless_from_padded_metal_buffers_with_config( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - config: MetalLosslessEncodeConfig, -) -> Result { - submit_lossless_tiles( - tiles, - *options, - session, - MetalEncodeInputStaging::AlreadyPaddedContiguous, - config, - ) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_padded_metal_buffers_with_report( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - encode_lossless_tiles_with_report( - tiles, - *options, - session, - MetalEncodeInputStaging::AlreadyPaddedContiguous, - ) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_padded_metal_buffers_to_metal_with_report( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - Ok(encode_lossless_from_padded_metal_buffers_to_metal_batch( - tiles, - options, - session, - MetalLosslessEncodeConfig::default(), - )? - .outcomes) -} - -#[cfg(target_os = "macos")] -pub fn encode_lossless_from_padded_metal_buffers_to_metal_batch( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - config: MetalLosslessEncodeConfig, -) -> Result { - encode_lossless_tiles_to_metal_buffer_batch( - tiles, - *options, - session, - MetalEncodeInputStaging::AlreadyPaddedContiguous, - config, - ) -} - -#[cfg(target_os = "macos")] -fn encode_lossless_tiles_with_report( - tiles: &[MetalLosslessEncodeTile<'_>], - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - staging: MetalEncodeInputStaging, -) -> Result, crate::Error> { - if should_try_resident_lossless_host_encode(options) { - let batch = try_encode_resident_lossless_tiles_to_metal_buffer_batch( - tiles, - options, - session, - staging, - MetalLosslessEncodeConfig::default(), - )?; - if let Some(outcomes) = batch { - return outcomes - .outcomes - .into_iter() - .map(|outcome| { - Ok(MetalLosslessEncodeOutcome { - encoded: outcome.encoded.to_encoded_j2k()?, - input_copy_used: outcome.input_copy_used, - resident: outcome.resident, - input_copy_duration: outcome.input_copy_duration, - encode_duration: outcome.encode_duration, - gpu_duration: outcome.gpu_duration, - validation_duration: outcome.validation_duration, - }) - }) - .collect(); - } - } - - let mut accelerator = MetalEncodeStageAccelerator::for_host_output(options); - tiles - .iter() - .map(|&tile| { - encode_lossless_tile_with_report(tile, options, session, staging, &mut accelerator) - }) - .collect() -} - -#[cfg(target_os = "macos")] -fn encode_lossless_owned_tiles_with_report( - tiles: &[OwnedMetalLosslessEncodeTile], - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - staging: MetalEncodeInputStaging, - config: MetalLosslessEncodeConfig, -) -> Result, crate::Error> { - let borrowed = tiles - .iter() - .map(OwnedMetalLosslessEncodeTile::as_tile) - .collect::>(); - if should_try_resident_lossless_host_encode(options) { - let batch = try_encode_resident_lossless_tiles_to_metal_buffer_batch( - &borrowed, options, session, staging, config, - )?; - if let Some(outcomes) = batch { - return outcomes - .outcomes - .into_iter() - .map(|outcome| { - Ok(MetalLosslessEncodeOutcome { - encoded: outcome.encoded.to_encoded_j2k()?, - input_copy_used: outcome.input_copy_used, - resident: outcome.resident, - input_copy_duration: outcome.input_copy_duration, - encode_duration: outcome.encode_duration, - gpu_duration: outcome.gpu_duration, - validation_duration: outcome.validation_duration, - }) - }) - .collect(); - } - } - - let mut accelerator = MetalEncodeStageAccelerator::for_host_output(options); - borrowed - .iter() - .map(|&tile| { - encode_lossless_tile_with_report(tile, options, session, staging, &mut accelerator) - }) - .collect() -} - -#[cfg(target_os = "macos")] -fn encode_lossless_tiles_to_metal_buffer_batch( - tiles: &[MetalLosslessEncodeTile<'_>], - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - staging: MetalEncodeInputStaging, - config: MetalLosslessEncodeConfig, -) -> Result { - if options.backend != EncodeBackendPreference::CpuOnly { - if let Some(batch) = try_encode_resident_lossless_tiles_to_metal_buffer_batch( - tiles, options, session, staging, config, - )? { - return Ok(batch); - } - } - - let mut outcomes = Vec::with_capacity(tiles.len()); - for &tile in tiles { - outcomes.push(encode_lossless_tile_to_metal_buffer_with_report( - tile, options, session, staging, - )?); - } - Ok(MetalLosslessBufferEncodeBatchOutcome { - outcomes, - stats: MetalLosslessEncodeBatchStats::default(), - }) -} - -#[cfg(target_os = "macos")] -fn try_encode_resident_lossless_tiles_to_metal_buffer_batch( - tiles: &[MetalLosslessEncodeTile<'_>], - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - staging: MetalEncodeInputStaging, - config: MetalLosslessEncodeConfig, -) -> Result, crate::Error> { - if tiles.is_empty() { - return Ok(Some(MetalLosslessBufferEncodeBatchOutcome { - outcomes: Vec::new(), - stats: resolve_lossless_encode_config(0, 1, config)?, - })); - } - - let mut planned = Vec::with_capacity(tiles.len()); - for (index, &tile) in tiles.iter().enumerate() { - let Some(item) = plan_resident_lossless_buffer_encode(index, tile, options, staging)? - else { - return Ok(None); - }; - planned.push(item); - } - let estimated_peak_bytes_per_tile = planned - .iter() - .map(PlannedResidentLosslessBufferEncode::estimated_peak_bytes) - .max() - .unwrap_or(1); - let mut stats = - resolve_lossless_encode_config(tiles.len(), estimated_peak_bytes_per_tile, config)?; - let encode_started = Instant::now(); - let outcomes = encode_planned_resident_lossless_tiles( - planned, - options, - session, - stats.effective_inflight_tiles, - &mut stats, - )?; - stats.encode_wall_duration = encode_started.elapsed(); - Ok(Some(MetalLosslessBufferEncodeBatchOutcome { - outcomes, - stats, - })) -} - -#[cfg(any(test, target_os = "macos"))] -const GPU_ENCODE_DEFAULT_INFLIGHT_TILES: usize = 64; -#[cfg(any(test, target_os = "macos"))] -const GPU_ENCODE_FALLBACK_HW_MEM_BYTES: usize = 8 * 1024 * 1024 * 1024; -#[cfg(any(test, target_os = "macos"))] -const GPU_ENCODE_MAX_DEFAULT_MEMORY_BUDGET_BYTES: usize = 10 * 1024 * 1024 * 1024; -#[cfg(any(test, target_os = "macos"))] -const GPU_ENCODE_MEMORY_BUDGET_PERCENT: usize = 40; - -#[cfg(any(test, target_os = "macos"))] -fn default_gpu_encode_memory_budget_bytes_for_hw_mem(hw_memsize: usize) -> usize { - hw_memsize - .saturating_mul(GPU_ENCODE_MEMORY_BUDGET_PERCENT) - .checked_div(100) - .unwrap_or(0) - .clamp(1, GPU_ENCODE_MAX_DEFAULT_MEMORY_BUDGET_BYTES) -} - -#[cfg(any(test, target_os = "macos"))] -fn default_gpu_encode_memory_budget_bytes() -> usize { - let hw_memsize = host_memory_bytes().unwrap_or(GPU_ENCODE_FALLBACK_HW_MEM_BYTES); - default_gpu_encode_memory_budget_bytes_for_hw_mem(hw_memsize) -} - -#[cfg(target_os = "macos")] -fn host_memory_bytes() -> Option { - let mut value = 0u64; - let mut len = core::mem::size_of::(); - let name = b"hw.memsize\0"; - let rc = unsafe { - libc::sysctlbyname( - name.as_ptr().cast(), - (&raw mut value).cast(), - &raw mut len, - core::ptr::null_mut(), - 0, - ) - }; - (rc == 0 && len == core::mem::size_of::()) - .then(|| usize::try_from(value).ok()) - .flatten() -} - -#[cfg(all(test, not(target_os = "macos")))] -fn host_memory_bytes() -> Option { - None -} - -#[cfg(any(test, target_os = "macos"))] -fn resolve_lossless_encode_config( - tile_count: usize, - estimated_peak_bytes_per_tile: usize, - config: MetalLosslessEncodeConfig, -) -> Result { - if config.gpu_encode_inflight_tiles == Some(0) { - return Err(crate::Error::UnsupportedMetalRequest { - reason: "J2K Metal encode in-flight tile cap must be greater than zero", - }); - } - if config.gpu_encode_memory_budget_bytes == Some(0) { - return Err(crate::Error::UnsupportedMetalRequest { - reason: "J2K Metal encode memory budget must be greater than zero", - }); - } - - let effective_memory_budget_bytes = config - .gpu_encode_memory_budget_bytes - .unwrap_or_else(default_gpu_encode_memory_budget_bytes) - .max(1); - let estimated_peak_bytes_per_tile = estimated_peak_bytes_per_tile.max(1); - let memory_limited_tiles = - (effective_memory_budget_bytes / estimated_peak_bytes_per_tile).max(1); - let configured_or_default = config - .gpu_encode_inflight_tiles - .unwrap_or(GPU_ENCODE_DEFAULT_INFLIGHT_TILES); - let effective_inflight_tiles = configured_or_default - .min(memory_limited_tiles) - .min(tile_count.max(1)) - .max(1); - - Ok(MetalLosslessEncodeBatchStats { - configured_inflight_tiles: config.gpu_encode_inflight_tiles, - effective_inflight_tiles, - configured_memory_budget_bytes: config.gpu_encode_memory_budget_bytes, - effective_memory_budget_bytes, - estimated_peak_bytes_per_tile, - max_observed_inflight_tiles: 0, - encode_wall_duration: Duration::ZERO, - }) -} - -#[cfg(test)] -fn resolve_lossless_encode_config_for_test( - tile_count: usize, - estimated_peak_bytes_per_tile: usize, - config: MetalLosslessEncodeConfig, -) -> Result { - resolve_lossless_encode_config(tile_count, estimated_peak_bytes_per_tile, config) -} - -#[cfg(target_os = "macos")] -fn checked_add_bytes(lhs: usize, rhs: usize) -> usize { - lhs.saturating_add(rhs) -} - -#[cfg(target_os = "macos")] -fn checked_mul_bytes(lhs: usize, rhs: usize) -> usize { - lhs.saturating_mul(rhs) -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Copy)] -struct LosslessSubbandPlan { - num_cbs_x: u32, - num_cbs_y: u32, - code_block_start: usize, - code_block_count: usize, -} - -#[cfg(target_os = "macos")] -#[derive(Clone)] -struct LosslessResolutionPlan { - subbands: Vec, -} - -#[cfg(target_os = "macos")] -struct LosslessDeviceEncodePlan { - components: u8, - bit_depth: u8, - block_coding_mode: J2kBlockCodingMode, - num_decomposition_levels: u8, - use_mct: bool, - guard_bits: u8, - code_blocks: Vec, - resolutions: Vec, - progression_order: EncodeProgressionOrder, - write_tlm: bool, -} - -#[cfg(target_os = "macos")] -struct ResidentLosslessBufferEncodeMetadata { - tile: OwnedMetalLosslessEncodeTile, - components: u8, - bit_depth: u8, - bytes_per_pixel: usize, - plan: LosslessDeviceEncodePlan, - packet_descriptors: Vec, - packetization_resolutions: Vec, -} - -#[cfg(target_os = "macos")] -struct PreparedResidentLosslessBufferEncode { - metadata: ResidentLosslessBufferEncodeMetadata, - prepared: compute::J2kPreparedLosslessDeviceCodeBlocks, -} - -#[cfg(target_os = "macos")] -struct PlannedResidentLosslessBufferEncode { - index: usize, - metadata: ResidentLosslessBufferEncodeMetadata, - coefficient_count: usize, - bytes_per_sample: u8, - estimated_peak_bytes: usize, - #[cfg(test)] - failure_injection_index: Option, -} - -#[cfg(target_os = "macos")] -impl PlannedResidentLosslessBufferEncode { - fn estimated_peak_bytes(&self) -> usize { - self.estimated_peak_bytes - } -} - -#[cfg(target_os = "macos")] -enum ResidentLosslessTier1 { - Classic(compute::J2kResidentLosslessTier1CodeBlocks), - HighThroughput(compute::J2kResidentLosslessHtCodeBlocks), -} - -#[cfg(target_os = "macos")] -struct SubmittedResidentLosslessBufferEncode { - metadata: ResidentLosslessBufferEncodeMetadata, - pending_codestream: compute::J2kPendingResidentLosslessCodestream, -} - -#[cfg(target_os = "macos")] -struct FinishedResidentLosslessBufferEncode { - metadata: ResidentLosslessBufferEncodeMetadata, - encoded: MetalEncodedJ2k, - encode_duration: Duration, - gpu_duration: Option, -} - -#[cfg(target_os = "macos")] -fn lossless_device_encode_levels(width: u32, height: u32, options: J2kLosslessEncodeOptions) -> u8 { - const MIN_LOSSLESS_DWT_DIMENSION: u32 = 64; - let levels = if options.progression == J2kProgressionOrder::Rpcl { - let mut levels = 0u8; - let mut w = width; - let mut h = height; - let max_levels = if width.min(height) <= 1 { - 0 - } else { - width.min(height).ilog2() as u8 - }; - while w.min(h) > MIN_LOSSLESS_DWT_DIMENSION && levels < max_levels { - w = w.div_ceil(2); - h = h.div_ceil(2); - levels = levels.saturating_add(1); - } - levels - } else { - u8::from(width.min(height) >= MIN_LOSSLESS_DWT_DIMENSION) - }; - - options - .max_decomposition_levels - .map_or(levels, |requested| { - let max_levels = if width.min(height) <= 1 { - 0 - } else { - width.min(height).ilog2() as u8 - }; - requested.min(max_levels) - }) -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Copy)] -struct LosslessSubbandInput { - component: u32, - subband_x: u32, - subband_y: u32, - width: u32, - height: u32, - sub_band_type: J2kSubBandType, - total_bitplanes: u8, -} - -#[cfg(target_os = "macos")] -fn push_lossless_subband_plan( - resolution: &mut LosslessResolutionPlan, - code_blocks: &mut Vec, - coefficient_offset: &mut u32, - subband: LosslessSubbandInput, -) -> Result<(), crate::Error> { - if subband.width == 0 || subband.height == 0 { - resolution.subbands.push(LosslessSubbandPlan { - num_cbs_x: 0, - num_cbs_y: 0, - code_block_start: code_blocks.len(), - code_block_count: 0, - }); - return Ok(()); - } - let cb_width = 64u32; - let cb_height = 64u32; - let num_cbs_x = subband.width.div_ceil(cb_width); - let num_cbs_y = subband.height.div_ceil(cb_height); - let code_block_start = code_blocks.len(); - for cby in 0..num_cbs_y { - for cbx in 0..num_cbs_x { - let block_x = cbx * cb_width; - let block_y = cby * cb_height; - let block_width = (block_x + cb_width).min(subband.width) - block_x; - let block_height = (block_y + cb_height).min(subband.height) - block_y; - let coeff_count = - block_width - .checked_mul(block_height) - .ok_or_else(|| crate::Error::MetalKernel { - message: "J2K Metal resident encode code-block size overflow".to_string(), - })?; - code_blocks.push(compute::J2kLosslessDeviceCodeBlock { - coefficient_offset: *coefficient_offset, - component: subband.component, - subband_x: subband.subband_x, - subband_y: subband.subband_y, - block_x, - block_y, - width: block_width, - height: block_height, - sub_band_type: subband.sub_band_type, - total_bitplanes: subband.total_bitplanes, - }); - *coefficient_offset = coefficient_offset.checked_add(coeff_count).ok_or_else(|| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode coefficient offset overflow".to_string(), - } - })?; - } - } - resolution.subbands.push(LosslessSubbandPlan { - num_cbs_x, - num_cbs_y, - code_block_start, - code_block_count: code_blocks.len() - code_block_start, - }); - Ok(()) -} - -#[cfg(target_os = "macos")] -fn lossless_device_encode_plan( - width: u32, - height: u32, - components: u8, - bit_depth: u8, - options: J2kLosslessEncodeOptions, -) -> Result, crate::Error> { - if !matches!( - options.block_coding_mode, - J2kBlockCodingMode::Classic | J2kBlockCodingMode::HighThroughput - ) { - return Ok(None); - } - let num_decomposition_levels = lossless_device_encode_levels(width, height, options); - if num_decomposition_levels > 1 { - return Ok(None); - } - let progression_order = match options.progression { - J2kProgressionOrder::Lrcp => EncodeProgressionOrder::Lrcp, - J2kProgressionOrder::Rpcl => EncodeProgressionOrder::Rpcl, - }; - let use_mct = components >= 3; - let guard_bits: u8 = if use_mct { 2 } else { 1 }; - let mut code_blocks = Vec::new(); - let mut coefficient_offset = 0u32; - let mut component_resolutions = Vec::>::new(); - for component in 0..components { - let mut component_packets = Vec::new(); - let mut base_packet = LosslessResolutionPlan { - subbands: Vec::new(), - }; - if num_decomposition_levels == 0 { - push_lossless_subband_plan( - &mut base_packet, - &mut code_blocks, - &mut coefficient_offset, - LosslessSubbandInput { - component: u32::from(component), - subband_x: 0, - subband_y: 0, - width, - height, - sub_band_type: J2kSubBandType::LowLow, - total_bitplanes: guard_bits.saturating_add(bit_depth).saturating_sub(1), - }, - )?; - component_packets.push(base_packet); - } else { - let low_width = width.div_ceil(2); - let low_height = height.div_ceil(2); - let high_width = width / 2; - let high_height = height / 2; - push_lossless_subband_plan( - &mut base_packet, - &mut code_blocks, - &mut coefficient_offset, - LosslessSubbandInput { - component: u32::from(component), - subband_x: 0, - subband_y: 0, - width: low_width, - height: low_height, - sub_band_type: J2kSubBandType::LowLow, - total_bitplanes: guard_bits.saturating_add(bit_depth).saturating_sub(1), - }, - )?; - component_packets.push(base_packet); - - let mut detail_packet = LosslessResolutionPlan { - subbands: Vec::new(), - }; - push_lossless_subband_plan( - &mut detail_packet, - &mut code_blocks, - &mut coefficient_offset, - LosslessSubbandInput { - component: u32::from(component), - subband_x: low_width, - subband_y: 0, - width: high_width, - height: low_height, - sub_band_type: J2kSubBandType::HighLow, - total_bitplanes: guard_bits.saturating_add(bit_depth), - }, - )?; - push_lossless_subband_plan( - &mut detail_packet, - &mut code_blocks, - &mut coefficient_offset, - LosslessSubbandInput { - component: u32::from(component), - subband_x: 0, - subband_y: low_height, - width: low_width, - height: high_height, - sub_band_type: J2kSubBandType::LowHigh, - total_bitplanes: guard_bits.saturating_add(bit_depth), - }, - )?; - push_lossless_subband_plan( - &mut detail_packet, - &mut code_blocks, - &mut coefficient_offset, - LosslessSubbandInput { - component: u32::from(component), - subband_x: low_width, - subband_y: low_height, - width: high_width, - height: high_height, - sub_band_type: J2kSubBandType::HighHigh, - total_bitplanes: guard_bits.saturating_add(bit_depth).saturating_add(1), - }, - )?; - component_packets.push(detail_packet); - } - component_resolutions.push(component_packets); - } - - let resolution_count = component_resolutions.first().map_or(0usize, Vec::len); - let mut resolutions = - Vec::with_capacity(resolution_count.saturating_mul(usize::from(components))); - for resolution in 0..resolution_count { - for component in &component_resolutions { - resolutions.push(component[resolution].clone()); - } - } - - Ok(Some(LosslessDeviceEncodePlan { - components, - bit_depth, - block_coding_mode: options.block_coding_mode, - num_decomposition_levels, - use_mct, - guard_bits, - code_blocks, - resolutions, - progression_order, - write_tlm: options.progression == J2kProgressionOrder::Rpcl, - })) -} - -#[cfg(target_os = "macos")] -#[derive(Debug, Clone, Copy)] -enum MetalEncodeInputStaging { - CopyAndPad, - AlreadyPaddedContiguous, -} - -#[cfg(target_os = "macos")] -fn submit_lossless_tiles( - tiles: &[MetalLosslessEncodeTile<'_>], - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - staging: MetalEncodeInputStaging, - config: MetalLosslessEncodeConfig, -) -> Result { - if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) - && should_try_resident_lossless_host_encode(options) - { - let mut ready = Vec::with_capacity(tiles.len()); - let mut all_ready = true; - for &tile in tiles { - validate_metal_encode_tile(tile)?; - lossless_sample_shape(tile.format)?; - validate_padded_contiguous_metal_encode_tile(tile, tile.format.bytes_per_pixel())?; - if let Some(outcome) = try_encode_lossless_tile_device_resident_with_report( - tile, options, session, staging, - )? { - ready.push(outcome.encoded); - } else { - all_ready = false; - break; - } - } - if all_ready { - return Ok(SubmittedJ2kLosslessMetalEncodeBatch { - state: SubmittedJ2kLosslessMetalEncodeBatchState::Ready(ready), - }); - } - if options.backend == EncodeBackendPreference::RequireDevice { - return Err(crate::Error::UnsupportedMetalRequest { - reason: "J2K Metal resident encode requires classic padded contiguous Gray/RGB lossless input with at most one DWT level", - }); - } - } - - let mut owned = Vec::with_capacity(tiles.len()); - for &tile in tiles { - validate_metal_encode_tile(tile)?; - if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { - lossless_sample_shape(tile.format)?; - validate_padded_contiguous_metal_encode_tile(tile, tile.format.bytes_per_pixel())?; - } - owned.push(OwnedMetalLosslessEncodeTile::from_tile(tile)); - } - Ok(SubmittedJ2kLosslessMetalEncodeBatch { - state: SubmittedJ2kLosslessMetalEncodeBatchState::Deferred { - tiles: owned, - options, - session: session.clone(), - staging, - config, - }, - }) -} - -#[cfg(target_os = "macos")] -fn should_try_resident_lossless_host_encode(options: J2kLosslessEncodeOptions) -> bool { - options.backend == EncodeBackendPreference::RequireDevice -} - -#[cfg(target_os = "macos")] -fn host_output_encode_options(mut options: J2kLosslessEncodeOptions) -> J2kLosslessEncodeOptions { - options.validation = J2kEncodeValidation::External; - options -} - -#[cfg(target_os = "macos")] -fn packet_descriptors_for_lossless_device_order( - packet_count: usize, - num_components: u8, -) -> Result, crate::Error> { - let component_count = usize::from(num_components).max(1); - (0..packet_count) - .map(|packet_index| { - Ok(J2kPacketizationPacketDescriptor { - packet_index: u32::try_from(packet_index).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode packet index exceeds u32".to_string(), - } - })?, - state_index: u32::try_from(packet_index).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode packet state index exceeds u32" - .to_string(), - } - })?, - layer: 0, - resolution: u32::try_from(packet_index / component_count).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode packet resolution exceeds u32" - .to_string(), - } - })?, - component: u8::try_from(packet_index % component_count).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode packet component exceeds u8" - .to_string(), - } - })?, - precinct: 0, - }) - }) - .collect() -} - -#[cfg(target_os = "macos")] -fn resident_packetization_resolutions_from_lossless_device_plan( - plan: &LosslessDeviceEncodePlan, -) -> Result, crate::Error> { - plan.resolutions - .iter() - .map(|resolution| { - let subbands = resolution - .subbands - .iter() - .map(|subband| { - let code_block_end = subband - .code_block_start - .checked_add(subband.code_block_count) - .ok_or_else(|| crate::Error::MetalKernel { - message: "J2K Metal resident encode code-block range overflow" - .to_string(), - })?; - if code_block_end > plan.code_blocks.len() { - return Err(crate::Error::MetalKernel { - message: "J2K Metal resident encode code-block range out of bounds" - .to_string(), - }); - } - Ok(compute::J2kResidentPacketizationSubband { - code_block_start: u32::try_from(subband.code_block_start).map_err( - |_| crate::Error::MetalKernel { - message: "J2K Metal resident encode code-block offset exceeds u32" - .to_string(), - }, - )?, - code_block_count: u32::try_from(subband.code_block_count).map_err( - |_| crate::Error::MetalKernel { - message: "J2K Metal resident encode code-block count exceeds u32" - .to_string(), - }, - )?, - num_cbs_x: subband.num_cbs_x, - num_cbs_y: subband.num_cbs_y, - }) - }) - .collect::, crate::Error>>()?; - Ok(compute::J2kResidentPacketizationResolution { subbands }) - }) - .collect() -} - -#[cfg(target_os = "macos")] -fn lossless_device_coefficient_count( - code_blocks: &[compute::J2kLosslessDeviceCodeBlock], -) -> Result { - let mut count = 0usize; - for block in code_blocks { - let offset = - usize::try_from(block.coefficient_offset).map_err(|_| crate::Error::MetalKernel { - message: "J2K Metal resident encode coefficient offset exceeds usize".to_string(), - })?; - let block_count = (block.width as usize) - .checked_mul(block.height as usize) - .ok_or_else(|| crate::Error::MetalKernel { - message: "J2K Metal resident encode coefficient count overflow".to_string(), - })?; - count = count.max(offset.checked_add(block_count).ok_or_else(|| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode coefficient count overflow".to_string(), - } - })?); - } - Ok(count) -} - -#[cfg(target_os = "macos")] -fn plan_resident_lossless_buffer_encode( - index: usize, - tile: MetalLosslessEncodeTile<'_>, - options: J2kLosslessEncodeOptions, - staging: MetalEncodeInputStaging, -) -> Result, crate::Error> { - validate_metal_encode_tile(tile)?; - if options.backend == EncodeBackendPreference::CpuOnly { - return Ok(None); - } - let (components, bit_depth) = lossless_sample_shape(tile.format)?; - let bytes_per_pixel = tile.format.bytes_per_pixel(); - let bytes_per_sample = - u8::try_from(tile.format.bytes_per_sample()).map_err(|_| crate::Error::MetalKernel { - message: "J2K Metal resident encode bytes per sample exceeds u8".to_string(), - })?; - if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { - validate_padded_contiguous_metal_encode_tile(tile, bytes_per_pixel)?; - } - let Some(plan) = lossless_device_encode_plan( - tile.output_width, - tile.output_height, - components, - bit_depth, - options, - )? - else { - return Ok(None); - }; - let coefficient_count = lossless_device_coefficient_count(&plan.code_blocks)?; - let packetization_resolutions = - resident_packetization_resolutions_from_lossless_device_plan(&plan)?; - let packet_descriptors = - packet_descriptors_for_lossless_device_order(plan.resolutions.len(), plan.components)?; - let metadata = ResidentLosslessBufferEncodeMetadata { - tile: OwnedMetalLosslessEncodeTile::from_tile(tile), - components, - bit_depth, - bytes_per_pixel, - plan, - packet_descriptors, - packetization_resolutions, - }; - let estimated_peak_bytes = - estimate_resident_lossless_encode_peak_bytes(&metadata, coefficient_count, staging); - Ok(Some(PlannedResidentLosslessBufferEncode { - index, - metadata, - coefficient_count, - bytes_per_sample, - estimated_peak_bytes, - #[cfg(test)] - failure_injection_index: test_resident_encode_failure_index(), - })) -} - -#[cfg(target_os = "macos")] -fn estimate_resident_lossless_encode_peak_bytes( - metadata: &ResidentLosslessBufferEncodeMetadata, - coefficient_count: usize, - staging: MetalEncodeInputStaging, -) -> usize { - let pixels = checked_mul_bytes( - metadata.tile.output_width as usize, - metadata.tile.output_height as usize, - ) - .max(1); - let plane_bytes = checked_mul_bytes(pixels, core::mem::size_of::()); - let code_block_count = metadata.plan.code_blocks.len().max(1); - let packet_count = metadata - .packet_descriptors - .len() - .max(metadata.plan.resolutions.len()) - .max(1); - let input_bytes = checked_mul_bytes( - checked_mul_bytes(metadata.tile.width as usize, metadata.tile.height as usize), - metadata.bytes_per_pixel, - ); - let staged_input_bytes = if matches!(staging, MetalEncodeInputStaging::CopyAndPad) { - checked_mul_bytes(pixels, metadata.bytes_per_pixel) - } else { - 0 - }; - let coefficient_bytes = - checked_mul_bytes(coefficient_count.max(1), core::mem::size_of::()); - let plane_buffers = checked_mul_bytes(3, plane_bytes); - let scratch_buffers = checked_mul_bytes(usize::from(metadata.components), plane_bytes); - let code_block_tables = checked_mul_bytes(code_block_count, 256); - let tier1_output = estimated_tier1_output_bytes(&metadata.plan); - let packet_header = checked_add_bytes(checked_mul_bytes(code_block_count, 256), 4096); - let packet_output = checked_add_bytes( - checked_add_bytes(tier1_output, checked_mul_bytes(packet_header, packet_count)), - 1024, - ); - let codestream_capacity = checked_add_bytes( - packet_output, - checked_add_bytes(4096, checked_mul_bytes(pixels, metadata.bytes_per_pixel)), - ); - let validation_bytes = checked_mul_bytes(pixels, metadata.bytes_per_pixel).saturating_mul( - usize::from(metadata.plan.write_tlm || metadata.plan.use_mct || metadata.components > 0), - ); - - [ - input_bytes / 4, - staged_input_bytes, - plane_buffers, - scratch_buffers, - coefficient_bytes, - code_block_tables, - tier1_output, - packet_output, - codestream_capacity, - validation_bytes, - 4 * 1024 * 1024, - ] - .into_iter() - .fold(0usize, checked_add_bytes) -} - -#[cfg(target_os = "macos")] -fn estimated_tier1_output_bytes(plan: &LosslessDeviceEncodePlan) -> usize { - const HT_ENCODE_OUTPUT_CAPACITY_PER_BLOCK: usize = - (16_384usize * 16).div_ceil(15) + 192 + (3072 - 192); - plan.code_blocks - .iter() - .map(|block| match plan.block_coding_mode { - J2kBlockCodingMode::HighThroughput => HT_ENCODE_OUTPUT_CAPACITY_PER_BLOCK, - J2kBlockCodingMode::Classic => { - let samples = checked_mul_bytes(block.width as usize, block.height as usize); - checked_add_bytes( - checked_mul_bytes( - checked_mul_bytes(samples, usize::from(block.total_bitplanes).max(1)), - 8, - ), - 4097, - ) - .max(4097) - } - }) - .fold(0usize, checked_add_bytes) - .max(1) -} - -#[cfg(target_os = "macos")] -fn encode_prepared_resident_lossless_tier1( - prepared: PreparedResidentLosslessBufferEncode, - session: &crate::MetalBackendSession, -) -> Result<(ResidentLosslessBufferEncodeMetadata, ResidentLosslessTier1), crate::Error> { - let tier1 = match prepared.metadata.plan.block_coding_mode { - J2kBlockCodingMode::Classic => ResidentLosslessTier1::Classic( - compute::encode_classic_tier1_prepared_device_code_blocks_resident( - session, - prepared.prepared, - )?, - ), - J2kBlockCodingMode::HighThroughput => ResidentLosslessTier1::HighThroughput( - compute::encode_ht_prepared_device_code_blocks_resident(session, prepared.prepared)?, - ), - }; - - Ok((prepared.metadata, tier1)) -} - -#[cfg(target_os = "macos")] -fn resident_packetization_job_for_metadata( - metadata: &ResidentLosslessBufferEncodeMetadata, -) -> Result, crate::Error> { - Ok(compute::J2kResidentPacketizationEncodeJob { - resolution_count: u32::try_from(metadata.plan.resolutions.len()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode resolution count exceeds u32".to_string(), - } - })?, - num_layers: 1, - num_components: metadata.plan.components, - code_block_count: u32::try_from(metadata.plan.code_blocks.len()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode code-block count exceeds u32".to_string(), - } - })?, - packet_descriptors: &metadata.packet_descriptors, - resolutions: &metadata.packetization_resolutions, - }) -} - -#[cfg(target_os = "macos")] -fn resident_codestream_assembly_job_for_metadata( - metadata: &ResidentLosslessBufferEncodeMetadata, -) -> compute::J2kLosslessCodestreamAssemblyJob { - compute::J2kLosslessCodestreamAssemblyJob { - width: metadata.tile.output_width, - height: metadata.tile.output_height, - num_components: metadata.plan.components, - bit_depth: metadata.plan.bit_depth, - signed: false, - num_decomposition_levels: metadata.plan.num_decomposition_levels, - use_mct: metadata.plan.use_mct, - guard_bits: metadata.plan.guard_bits, - progression_order: metadata.plan.progression_order, - write_tlm: metadata.plan.write_tlm, - block_coding_mode: match metadata.plan.block_coding_mode { - J2kBlockCodingMode::Classic => compute::J2kLosslessCodestreamBlockCodingMode::Classic, - J2kBlockCodingMode::HighThroughput => { - compute::J2kLosslessCodestreamBlockCodingMode::HighThroughput - } - }, - } -} - -#[cfg(target_os = "macos")] -fn submit_resident_lossless_buffer_encode( - metadata: ResidentLosslessBufferEncodeMetadata, - tier1: &ResidentLosslessTier1, - session: &crate::MetalBackendSession, -) -> Result { - let packetization_job = resident_packetization_job_for_metadata(&metadata)?; - let assembly_job = resident_codestream_assembly_job_for_metadata(&metadata); - let pending_codestream = match tier1 { - ResidentLosslessTier1::Classic(tier1) => { - compute::submit_lossless_codestream_buffer_from_resident_classic_tier1( - session, - tier1, - packetization_job, - assembly_job, - )? - } - ResidentLosslessTier1::HighThroughput(tier1) => { - compute::submit_lossless_codestream_buffer_from_resident_ht_tier1( - session, - tier1, - packetization_job, - assembly_job, - )? - } - }; - Ok(SubmittedResidentLosslessBufferEncode { - metadata, - pending_codestream, - }) -} - -#[cfg(target_os = "macos")] -fn wait_submitted_resident_lossless_buffer_encode( - submitted: SubmittedResidentLosslessBufferEncode, -) -> Result { - let encode_started = Instant::now(); - let codestream = compute::wait_resident_lossless_codestream(submitted.pending_codestream)?; - let encode_duration = encode_started.elapsed(); - Ok(finished_resident_lossless_buffer_encode( - submitted.metadata, - codestream, - encode_duration, - )) -} - -#[cfg(target_os = "macos")] -fn finished_resident_lossless_buffer_encode( - metadata: ResidentLosslessBufferEncodeMetadata, - codestream: compute::J2kResidentLosslessCodestream, - encode_duration: Duration, -) -> FinishedResidentLosslessBufferEncode { - let encoded = MetalEncodedJ2k { - codestream_buffer: codestream.buffer, - byte_offset: codestream.byte_offset, - byte_len: codestream.byte_len, - capacity: codestream.capacity, - width: metadata.tile.output_width, - height: metadata.tile.output_height, - components: metadata.components, - bit_depth: metadata.bit_depth, - signed: false, - }; - - FinishedResidentLosslessBufferEncode { - metadata, - encoded, - encode_duration, - gpu_duration: codestream.gpu_duration, - } -} - -#[cfg(target_os = "macos")] -fn validate_finished_resident_lossless_buffer_encode( - finished: FinishedResidentLosslessBufferEncode, - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let FinishedResidentLosslessBufferEncode { - metadata, - encoded, - encode_duration, - gpu_duration, - } = finished; - - let validation_duration = if options.validation == J2kEncodeValidation::CpuRoundTrip { - let validation_started = Instant::now(); - let tile = metadata.tile.as_tile(); - if tile.width == tile.output_width - && tile.height == tile.output_height - && tile.pitch_bytes == tile.output_width as usize * metadata.bytes_per_pixel - { - validate_lossless_roundtrip_on_metal_tile_with_session( - tile, - encoded.codestream_bytes()?, - session, - )?; - } else { - validate_lossless_roundtrip_on_metal_region_with_session( - tile, - tile.output_width, - tile.output_height, - metadata.bytes_per_pixel, - encoded.codestream_bytes()?, - session, - )?; - } - validation_started.elapsed() - } else { - Duration::ZERO - }; - - Ok(MetalLosslessBufferEncodeOutcome { - encoded, - input_copy_used: false, - resident: MetalLosslessEncodeResidency { - coefficient_prep_used: true, - packetization_used: true, - codestream_assembly_used: true, - }, - input_copy_duration: Duration::ZERO, - encode_duration, - gpu_duration, - validation_duration, - }) -} - -#[cfg(target_os = "macos")] -struct InflightLimitedOrderedItems { - items: Vec, - max_observed_inflight_items: usize, -} - -#[cfg(target_os = "macos")] -fn collect_inflight_limited_ordered( - items: Vec, - inflight_items: usize, - f: F, -) -> Result, crate::Error> -where - T: Send, - O: Send, - F: Fn(usize, T) -> Result + Sync, -{ - if items.is_empty() { - return Ok(InflightLimitedOrderedItems { - items: Vec::new(), - max_observed_inflight_items: 0, - }); - } - - let active = Arc::new(AtomicUsize::new(0)); - let observed = Arc::new(AtomicUsize::new(0)); - let pool = rayon::ThreadPoolBuilder::new() - .num_threads(inflight_items.max(1)) - .build() - .map_err(|err| crate::Error::MetalKernel { - message: format!("J2K Metal encode worker pool initialization failed: {err}"), - })?; - - let active_for_tasks = Arc::clone(&active); - let observed_for_tasks = Arc::clone(&observed); - let results = pool.install(|| { - items - .into_par_iter() - .enumerate() - .map(|(index, item)| { - let _guard = ActiveTileGuard::new(&active_for_tasks, &observed_for_tasks); - f(index, item) - }) - .collect::>() - }); - - let max_observed_inflight_items = observed.load(Ordering::Relaxed); - let mut ordered = Vec::with_capacity(results.len()); - let mut first_error = None; - for result in results { - match result { - Ok(item) if first_error.is_none() => ordered.push(item), - Ok(_) => {} - Err(err) => { - if first_error.is_none() { - first_error = Some(err); - } - } - } - } - - if let Some(err) = first_error { - return Err(err); - } - - Ok(InflightLimitedOrderedItems { - items: ordered, - max_observed_inflight_items, - }) -} - -#[cfg(target_os = "macos")] -fn encode_planned_resident_lossless_tiles( - planned: Vec, - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - inflight_tiles: usize, - stats: &mut MetalLosslessEncodeBatchStats, -) -> Result, crate::Error> { - if planned.is_empty() { - return Ok(Vec::new()); - } - if planned.iter().all(|planned| { - planned.metadata.plan.block_coding_mode == J2kBlockCodingMode::HighThroughput - }) { - return encode_planned_resident_ht_lossless_tiles_batch( - planned, - options, - session, - inflight_tiles, - stats, - ); - } - if planned - .iter() - .all(|planned| planned.metadata.plan.block_coding_mode == J2kBlockCodingMode::Classic) - { - return encode_planned_resident_classic_lossless_tiles_batch( - planned, - options, - session, - inflight_tiles, - stats, - ); - } - - let encoded = collect_inflight_limited_ordered(planned, inflight_tiles, |_, planned| { - let index = planned.index; - encode_planned_resident_lossless_tile(planned, options, session).map_err(|err| { - crate::Error::MetalKernel { - message: format!("J2K Metal resident encode failed at tile {index}: {err}"), - } - }) - })?; - stats.max_observed_inflight_tiles = stats - .max_observed_inflight_tiles - .max(encoded.max_observed_inflight_items); - Ok(encoded.items) -} - -#[cfg(target_os = "macos")] -struct PreparedResidentLosslessBatchItem { - prepared: PreparedResidentLosslessBufferEncode, - prepare_duration: Duration, -} - -#[cfg(target_os = "macos")] -fn prepare_planned_resident_ht_lossless_tiles_batch( - planned: Vec, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - struct HtBatchPlanInfo { - index: usize, - coefficient_count: usize, - bytes_per_sample: u8, - code_blocks: Vec, - } - - let started = Instant::now(); - let mut metadatas = Vec::with_capacity(planned.len()); - let mut plan_infos = Vec::with_capacity(planned.len()); - for planned in planned { - #[cfg(test)] - if planned.failure_injection_index == Some(planned.index) { - return Err(crate::Error::MetalKernel { - message: format!( - "injected J2K Metal resident encode failure at tile {}", - planned.index - ), - }); - } - - plan_infos.push(HtBatchPlanInfo { - index: planned.index, - coefficient_count: planned.coefficient_count, - bytes_per_sample: planned.bytes_per_sample, - code_blocks: planned.metadata.plan.code_blocks.clone(), - }); - metadatas.push(planned.metadata); - } - - let mut batch_items = Vec::with_capacity(metadatas.len()); - for (metadata, plan_info) in metadatas.iter().zip(plan_infos) { - let tile = metadata.tile.as_tile(); - batch_items.push(compute::J2kLosslessDeviceBatchPrepareItem { - tile_index: plan_info.index, - job: compute::J2kLosslessDevicePrepareJob { - input: tile.buffer, - input_byte_offset: tile.byte_offset, - input_width: tile.width, - input_height: tile.height, - input_pitch_bytes: tile.pitch_bytes, - output_width: tile.output_width, - output_height: tile.output_height, - components: metadata.components, - bytes_per_sample: plan_info.bytes_per_sample, - bit_depth: metadata.bit_depth, - num_decomposition_levels: metadata.plan.num_decomposition_levels, - coefficient_count: plan_info.coefficient_count, - }, - code_blocks: plan_info.code_blocks, - }); - } - - let prepared = compute::prepare_lossless_device_code_blocks_batch(session, batch_items)?; - let prepare_duration = duration_share(started.elapsed(), prepared.len()); - Ok(metadatas - .into_iter() - .zip(prepared) - .map(|(metadata, prepared)| PreparedResidentLosslessBatchItem { - prepared: PreparedResidentLosslessBufferEncode { metadata, prepared }, - prepare_duration, - }) - .collect()) -} - -#[cfg(target_os = "macos")] -fn encode_planned_resident_ht_lossless_tiles_batch( - planned: Vec, - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - inflight_tiles: usize, - stats: &mut MetalLosslessEncodeBatchStats, -) -> Result, crate::Error> { - let planned_len = planned.len(); - let prepared = - prepare_planned_resident_ht_lossless_tiles_batch(planned, session).map_err(|err| { - crate::Error::MetalKernel { - message: format!("J2K Metal resident HT batch encode failed: {err}"), - } - })?; - stats.max_observed_inflight_tiles = stats.max_observed_inflight_tiles.max( - planned_len - .min(inflight_tiles) - .max(usize::from(planned_len > 0)), - ); - - let mut metadatas = Vec::with_capacity(prepared.len()); - let mut prepare_durations = Vec::with_capacity(prepared.len()); - let mut batch_items = Vec::with_capacity(prepared.len()); - for item in prepared { - let PreparedResidentLosslessBatchItem { - prepared, - prepare_duration, - } = item; - let metadata = prepared.metadata; - let codestream = resident_codestream_assembly_job_for_metadata(&metadata); - batch_items.push(compute::J2kResidentHtBatchEncodeItem { - prepared: prepared.prepared, - resolution_count: u32::try_from(metadata.plan.resolutions.len()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode resolution count exceeds u32".to_string(), - } - })?, - num_layers: 1, - num_components: metadata.plan.components, - code_block_count: u32::try_from(metadata.plan.code_blocks.len()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode code-block count exceeds u32".to_string(), - } - })?, - packet_descriptors: metadata.packet_descriptors.clone(), - resolutions: metadata.packetization_resolutions.clone(), - codestream, - }); - prepare_durations.push(prepare_duration); - metadatas.push(metadata); - } - - let batch_started = Instant::now(); - let pending = - compute::submit_lossless_codestream_buffers_from_prepared_ht_batch(session, batch_items)?; - let codestreams = compute::wait_resident_lossless_codestream_batch(pending)?; - let batch_duration = duration_share(batch_started.elapsed(), codestreams.len()); - metadatas - .into_iter() - .zip(prepare_durations) - .zip(codestreams) - .map(|((metadata, prepare_duration), codestream)| { - let finished = finished_resident_lossless_buffer_encode( - metadata, - codestream, - prepare_duration.saturating_add(batch_duration), - ); - validate_finished_resident_lossless_buffer_encode(finished, options, session) - }) - .collect() -} - -#[cfg(target_os = "macos")] -fn encode_planned_resident_classic_lossless_tiles_batch( - planned: Vec, - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - inflight_tiles: usize, - stats: &mut MetalLosslessEncodeBatchStats, -) -> Result, crate::Error> { - let prepared = collect_inflight_limited_ordered(planned, inflight_tiles, |_, planned| { - let index = planned.index; - let started = Instant::now(); - prepare_planned_resident_lossless_tile(planned, session) - .map(|prepared| PreparedResidentLosslessBatchItem { - prepared, - prepare_duration: started.elapsed(), - }) - .map_err(|err| crate::Error::MetalKernel { - message: format!("J2K Metal resident encode failed at tile {index}: {err}"), - }) - })?; - stats.max_observed_inflight_tiles = stats - .max_observed_inflight_tiles - .max(prepared.max_observed_inflight_items); - - let mut metadatas = Vec::with_capacity(prepared.items.len()); - let mut prepare_durations = Vec::with_capacity(prepared.items.len()); - let mut batch_items = Vec::with_capacity(prepared.items.len()); - for item in prepared.items { - let PreparedResidentLosslessBatchItem { - prepared, - prepare_duration, - } = item; - let metadata = prepared.metadata; - let codestream = resident_codestream_assembly_job_for_metadata(&metadata); - batch_items.push(compute::J2kResidentClassicBatchEncodeItem { - prepared: prepared.prepared, - resolution_count: u32::try_from(metadata.plan.resolutions.len()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode resolution count exceeds u32".to_string(), - } - })?, - num_layers: 1, - num_components: metadata.plan.components, - code_block_count: u32::try_from(metadata.plan.code_blocks.len()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode code-block count exceeds u32".to_string(), - } - })?, - packet_descriptors: metadata.packet_descriptors.clone(), - resolutions: metadata.packetization_resolutions.clone(), - codestream, - }); - prepare_durations.push(prepare_duration); - metadatas.push(metadata); - } - - let batch_limit = inflight_tiles.max(1); - let mut outcomes = Vec::with_capacity(metadatas.len()); - while !batch_items.is_empty() { - let take = batch_items.len().min(batch_limit); - let chunk_items = batch_items.drain(..take).collect(); - let chunk_metadatas = metadatas.drain(..take).collect::>(); - let chunk_prepare_durations = prepare_durations.drain(..take).collect::>(); - let batch_started = Instant::now(); - let pending = compute::submit_lossless_codestream_buffers_from_prepared_classic_batch( - session, - chunk_items, - )?; - let codestreams = compute::wait_resident_lossless_codestream_batch(pending)?; - let batch_duration = duration_share(batch_started.elapsed(), codestreams.len()); - for ((metadata, prepare_duration), codestream) in chunk_metadatas - .into_iter() - .zip(chunk_prepare_durations) - .zip(codestreams) - { - let finished = finished_resident_lossless_buffer_encode( - metadata, - codestream, - prepare_duration.saturating_add(batch_duration), - ); - outcomes.push(validate_finished_resident_lossless_buffer_encode( - finished, options, session, - )?); - } - } - Ok(outcomes) -} - -#[cfg(target_os = "macos")] -fn prepare_planned_resident_lossless_tile( - planned: PlannedResidentLosslessBufferEncode, - session: &crate::MetalBackendSession, -) -> Result { - #[cfg(test)] - if planned.failure_injection_index == Some(planned.index) { - return Err(crate::Error::MetalKernel { - message: format!( - "injected J2K Metal resident encode failure at tile {}", - planned.index - ), - }); - } - - let tile = planned.metadata.tile.as_tile(); - let prepared = compute::prepare_lossless_device_code_blocks( - session, - compute::J2kLosslessDevicePrepareJob { - input: tile.buffer, - input_byte_offset: tile.byte_offset, - input_width: tile.width, - input_height: tile.height, - input_pitch_bytes: tile.pitch_bytes, - output_width: tile.output_width, - output_height: tile.output_height, - components: planned.metadata.components, - bytes_per_sample: planned.bytes_per_sample, - bit_depth: planned.metadata.bit_depth, - num_decomposition_levels: planned.metadata.plan.num_decomposition_levels, - coefficient_count: planned.coefficient_count, - }, - planned.metadata.plan.code_blocks.clone(), - )?; - Ok(PreparedResidentLosslessBufferEncode { - metadata: planned.metadata, - prepared, - }) -} - -#[cfg(target_os = "macos")] -fn duration_share(duration: Duration, count: usize) -> Duration { - if count == 0 { - return Duration::ZERO; - } - let nanos = duration.as_nanos() / count as u128; - Duration::from_nanos(nanos.min(u128::from(u64::MAX)) as u64) -} - -#[cfg(target_os = "macos")] -struct ActiveTileGuard<'a> { - active: &'a AtomicUsize, -} - -#[cfg(target_os = "macos")] -impl<'a> ActiveTileGuard<'a> { - fn new(active: &'a AtomicUsize, observed: &AtomicUsize) -> Self { - let now = active.fetch_add(1, Ordering::AcqRel).saturating_add(1); - let mut current = observed.load(Ordering::Relaxed); - while now > current { - match observed.compare_exchange(current, now, Ordering::AcqRel, Ordering::Relaxed) { - Ok(_) => break, - Err(next) => current = next, - } - } - Self { active } - } -} - -#[cfg(target_os = "macos")] -impl Drop for ActiveTileGuard<'_> { - fn drop(&mut self) { - self.active.fetch_sub(1, Ordering::AcqRel); - } -} - -#[cfg(target_os = "macos")] -fn encode_planned_resident_lossless_tile( - planned: PlannedResidentLosslessBufferEncode, - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let encode_started = Instant::now(); - let prepared = prepare_planned_resident_lossless_tile(planned, session)?; - let (metadata, tier1) = encode_prepared_resident_lossless_tier1(prepared, session)?; - let submitted = submit_resident_lossless_buffer_encode(metadata, &tier1, session)?; - let mut finished = wait_submitted_resident_lossless_buffer_encode(submitted)?; - finished.encode_duration = encode_started.elapsed(); - validate_finished_resident_lossless_buffer_encode(finished, options, session) -} - -#[cfg(all(test, target_os = "macos"))] -thread_local! { - static TEST_RESIDENT_ENCODE_FAILURE_INDEX: Cell> = const { Cell::new(None) }; -} - -#[cfg(all(test, target_os = "macos"))] -fn set_test_resident_encode_failure_index(index: Option) { - TEST_RESIDENT_ENCODE_FAILURE_INDEX.set(index); -} - -#[cfg(all(test, target_os = "macos"))] -fn test_resident_encode_failure_index() -> Option { - TEST_RESIDENT_ENCODE_FAILURE_INDEX.get() -} - -#[cfg(target_os = "macos")] -fn validate_lossless_roundtrip_on_metal_tile_with_session( - tile: MetalLosslessEncodeTile<'_>, - codestream: &[u8], - session: &crate::MetalBackendSession, -) -> Result<(), crate::Error> { - let mut decoder = crate::J2kDecoder::new(codestream)?; - let surface = decoder.decode_to_device_with_session(tile.format, session)?; - if surface.dimensions() != (tile.output_width, tile.output_height) { - return Err(crate::Error::MetalKernel { - message: format!( - "J2K Metal resident validation geometry mismatch: expected {}x{}, got {}x{}", - tile.output_width, - tile.output_height, - surface.dimensions().0, - surface.dimensions().1 - ), - }); - } - if surface.pixel_format() != tile.format { - return Err(crate::Error::MetalKernel { - message: format!( - "J2K Metal resident validation format mismatch: expected {:?}, got {:?}", - tile.format, - surface.pixel_format() - ), - }); - } - let expected_pitch = tile.output_width as usize * tile.format.bytes_per_pixel(); - if surface.pitch_bytes() != expected_pitch || tile.pitch_bytes != expected_pitch { - return Err(crate::Error::MetalKernel { - message: "J2K Metal resident validation requires contiguous source and decoded rows" - .to_string(), - }); - } - let byte_len = expected_pitch - .checked_mul(tile.output_height as usize) - .ok_or_else(|| crate::Error::MetalKernel { - message: "J2K Metal resident validation byte length overflow".to_string(), - })?; - let (decoded_buffer, decoded_offset) = - surface - .metal_buffer() - .ok_or(crate::Error::UnsupportedMetalRequest { - reason: "J2K Metal resident validation decode did not return a Metal buffer", - })?; - compute::validate_metal_buffers_match( - tile.buffer, - tile.byte_offset, - decoded_buffer, - decoded_offset, - byte_len, - session, - ) -} - -#[cfg(target_os = "macos")] -#[allow(clippy::too_many_arguments)] -fn validate_lossless_roundtrip_on_metal_region_with_session( - source: MetalLosslessEncodeTile<'_>, - output_width: u32, - output_height: u32, - bytes_per_pixel: usize, - codestream: &[u8], - session: &crate::MetalBackendSession, -) -> Result<(), crate::Error> { - let staged_buffer = compute::copy_interleaved_padded_to_shared_buffer( - source.buffer, - source.byte_offset, - source.width, - source.height, - source.pitch_bytes, - output_width, - output_height, - bytes_per_pixel, - session, - )?; - let staged_tile = MetalLosslessEncodeTile { - buffer: &staged_buffer, - byte_offset: 0, - width: output_width, - height: output_height, - pitch_bytes: output_width as usize * bytes_per_pixel, - output_width, - output_height, - format: source.format, - }; - validate_lossless_roundtrip_on_metal_tile_with_session(staged_tile, codestream, session) -} - -#[cfg(target_os = "macos")] -fn try_encode_lossless_tile_device_resident_with_report( - tile: MetalLosslessEncodeTile<'_>, - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - staging: MetalEncodeInputStaging, -) -> Result, crate::Error> { - let Some(outcome) = try_encode_lossless_tile_device_resident_to_metal_buffer_with_report( - tile, options, session, staging, - )? - else { - return Ok(None); - }; - Ok(Some(MetalLosslessEncodeOutcome { - encoded: outcome.encoded.to_encoded_j2k()?, - input_copy_used: outcome.input_copy_used, - resident: outcome.resident, - input_copy_duration: outcome.input_copy_duration, - encode_duration: outcome.encode_duration, - gpu_duration: outcome.gpu_duration, - validation_duration: outcome.validation_duration, - })) -} - -#[cfg(target_os = "macos")] -fn try_encode_lossless_tile_device_resident_to_metal_buffer_with_report( - tile: MetalLosslessEncodeTile<'_>, - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - staging: MetalEncodeInputStaging, -) -> Result, crate::Error> { - if options.backend == EncodeBackendPreference::CpuOnly { - return Ok(None); - } - let (components, bit_depth) = lossless_sample_shape(tile.format)?; - let bytes_per_pixel = tile.format.bytes_per_pixel(); - let bytes_per_sample = - u8::try_from(tile.format.bytes_per_sample()).map_err(|_| crate::Error::MetalKernel { - message: "J2K Metal resident encode bytes per sample exceeds u8".to_string(), - })?; - if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { - validate_padded_contiguous_metal_encode_tile(tile, bytes_per_pixel)?; - } - let Some(plan) = lossless_device_encode_plan( - tile.output_width, - tile.output_height, - components, - bit_depth, - options, - )? - else { - return Ok(None); - }; - - let encode_started = Instant::now(); - let coefficient_count = lossless_device_coefficient_count(&plan.code_blocks)?; - let prepared = compute::prepare_lossless_device_code_blocks( - session, - compute::J2kLosslessDevicePrepareJob { - input: tile.buffer, - input_byte_offset: tile.byte_offset, - input_width: tile.width, - input_height: tile.height, - input_pitch_bytes: tile.pitch_bytes, - output_width: tile.output_width, - output_height: tile.output_height, - components, - bytes_per_sample, - bit_depth, - num_decomposition_levels: plan.num_decomposition_levels, - coefficient_count, - }, - plan.code_blocks.clone(), - )?; - let packetization_resolutions = - resident_packetization_resolutions_from_lossless_device_plan(&plan)?; - let packet_descriptors = - packet_descriptors_for_lossless_device_order(plan.resolutions.len(), plan.components)?; - let packetization_job = compute::J2kResidentPacketizationEncodeJob { - resolution_count: u32::try_from(plan.resolutions.len()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode resolution count exceeds u32".to_string(), - } - })?, - num_layers: 1, - num_components: plan.components, - code_block_count: u32::try_from(plan.code_blocks.len()).map_err(|_| { - crate::Error::MetalKernel { - message: "J2K Metal resident encode code-block count exceeds u32".to_string(), - } - })?, - packet_descriptors: &packet_descriptors, - resolutions: &packetization_resolutions, - }; - let assembly_job = compute::J2kLosslessCodestreamAssemblyJob { - width: tile.output_width, - height: tile.output_height, - num_components: plan.components, - bit_depth: plan.bit_depth, - signed: false, - num_decomposition_levels: plan.num_decomposition_levels, - use_mct: plan.use_mct, - guard_bits: plan.guard_bits, - progression_order: plan.progression_order, - write_tlm: plan.write_tlm, - block_coding_mode: match plan.block_coding_mode { - J2kBlockCodingMode::Classic => compute::J2kLosslessCodestreamBlockCodingMode::Classic, - J2kBlockCodingMode::HighThroughput => { - compute::J2kLosslessCodestreamBlockCodingMode::HighThroughput - } - }, - }; - let codestream = match plan.block_coding_mode { - J2kBlockCodingMode::Classic => { - let resident_tier1 = - compute::encode_classic_tier1_prepared_device_code_blocks_resident( - session, prepared, - )?; - compute::encode_lossless_codestream_buffer_from_resident_classic_tier1( - session, - &resident_tier1, - packetization_job, - assembly_job, - )? - } - J2kBlockCodingMode::HighThroughput => { - let resident_tier1 = - compute::encode_ht_prepared_device_code_blocks_resident(session, prepared)?; - compute::encode_lossless_codestream_buffer_from_resident_ht_tier1( - session, - &resident_tier1, - packetization_job, - assembly_job, - )? - } - }; - let encode_duration = encode_started.elapsed(); - - let encoded = MetalEncodedJ2k { - codestream_buffer: codestream.buffer, - byte_offset: codestream.byte_offset, - byte_len: codestream.byte_len, - capacity: codestream.capacity, - width: tile.output_width, - height: tile.output_height, - components, - bit_depth, - signed: false, - }; - - let validation_duration = if options.validation == J2kEncodeValidation::CpuRoundTrip { - let validation_started = Instant::now(); - if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { - validate_lossless_roundtrip_on_metal_tile_with_session( - tile, - encoded.codestream_bytes()?, - session, - )?; - } else { - validate_lossless_roundtrip_on_metal_region_with_session( - tile, - tile.output_width, - tile.output_height, - bytes_per_pixel, - encoded.codestream_bytes()?, - session, - )?; - } - validation_started.elapsed() - } else { - Duration::ZERO - }; - - Ok(Some(MetalLosslessBufferEncodeOutcome { - encoded, - input_copy_used: false, - resident: MetalLosslessEncodeResidency { - coefficient_prep_used: true, - packetization_used: true, - codestream_assembly_used: true, - }, - input_copy_duration: Duration::ZERO, - encode_duration, - gpu_duration: codestream.gpu_duration, - validation_duration, - })) -} - -#[cfg(target_os = "macos")] -fn encode_lossless_tile_to_metal_buffer_with_report( - tile: MetalLosslessEncodeTile<'_>, - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - staging: MetalEncodeInputStaging, -) -> Result { - validate_metal_encode_tile(tile)?; - lossless_sample_shape(tile.format)?; - if options.backend == EncodeBackendPreference::CpuOnly { - return Err(crate::Error::UnsupportedMetalRequest { - reason: "J2K Metal buffer output encode requires a device backend", - }); - } - let bytes_per_pixel = tile.format.bytes_per_pixel(); - if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { - validate_padded_contiguous_metal_encode_tile(tile, bytes_per_pixel)?; - } - if let Some(outcome) = try_encode_lossless_tile_device_resident_to_metal_buffer_with_report( - tile, options, session, staging, - )? { - return Ok(outcome); - } - Err(crate::Error::UnsupportedMetalRequest { - reason: "J2K Metal buffer output encode requires classic padded contiguous Gray/RGB lossless input with at most one DWT level", - }) -} - -#[cfg(target_os = "macos")] -fn encode_lossless_tile_with_report( - tile: MetalLosslessEncodeTile<'_>, - options: J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - staging: MetalEncodeInputStaging, - accelerator: &mut MetalEncodeStageAccelerator, -) -> Result { - validate_metal_encode_tile(tile)?; - let (components, bit_depth) = lossless_sample_shape(tile.format)?; - let bytes_per_pixel = tile.format.bytes_per_pixel(); - if should_try_resident_lossless_host_encode(options) { - if let Some(outcome) = - try_encode_lossless_tile_device_resident_with_report(tile, options, session, staging)? - { - return Ok(outcome); - } - } - if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) - && options.backend == EncodeBackendPreference::RequireDevice - { - return Err(crate::Error::UnsupportedMetalRequest { - reason: "J2K Metal resident encode requires classic padded contiguous Gray/RGB lossless input with at most one DWT level", - }); - } - let mut input_copy_used = false; - let mut input_copy_duration = Duration::ZERO; - let mut staged_buffer = None; - let mut source_byte_offset = tile.byte_offset; - if matches!(staging, MetalEncodeInputStaging::AlreadyPaddedContiguous) { - validate_padded_contiguous_metal_encode_tile(tile, bytes_per_pixel)?; - if tile.buffer.contents().is_null() { - let copy_started = Instant::now(); - staged_buffer = Some(compute::copy_interleaved_padded_to_shared_buffer( - tile.buffer, - tile.byte_offset, - tile.width, - tile.height, - tile.pitch_bytes, - tile.output_width, - tile.output_height, - bytes_per_pixel, - session, - )?); - input_copy_duration = copy_started.elapsed(); - input_copy_used = true; - source_byte_offset = 0; - } - } else { - let copy_started = Instant::now(); - staged_buffer = Some(compute::copy_interleaved_padded_to_shared_buffer( - tile.buffer, - tile.byte_offset, - tile.width, - tile.height, - tile.pitch_bytes, - tile.output_width, - tile.output_height, - bytes_per_pixel, - session, - )?); - input_copy_duration = copy_started.elapsed(); - input_copy_used = true; - source_byte_offset = 0; - } - let buffer = staged_buffer.as_ref().unwrap_or(tile.buffer); - let len = tile.output_width as usize * tile.output_height as usize * bytes_per_pixel; - let ptr = buffer.contents().cast::(); - if ptr.is_null() { - return Err(crate::Error::UnsupportedMetalRequest { - reason: "J2K Metal encode input buffer is not host-visible", - }); - } - let data = unsafe { core::slice::from_raw_parts(ptr.add(source_byte_offset), len) }; - let samples = J2kLosslessSamples::new( - data, - tile.output_width, - tile.output_height, - components, - bit_depth, - false, - ) - .map_err(crate::Error::Decode)?; - - let encode_options = host_output_encode_options(options); - let encode_started = Instant::now(); - let encoded = signinum_j2k::encode_j2k_lossless_with_accelerator( - samples, - &encode_options, - BackendKind::Metal, - accelerator, - ) - .map_err(crate::Error::Decode)?; - let encode_duration = encode_started.elapsed(); - let validation_duration = if options.validation == J2kEncodeValidation::CpuRoundTrip { - let validation_started = Instant::now(); - validate_lossless_roundtrip_on_metal_with_session(samples, &encoded.codestream, session)?; - validation_started.elapsed() - } else { - Duration::ZERO - }; - Ok(MetalLosslessEncodeOutcome { - encoded, - input_copy_used, - resident: MetalLosslessEncodeResidency { - coefficient_prep_used: false, - packetization_used: false, - codestream_assembly_used: false, - }, - input_copy_duration, - encode_duration, - gpu_duration: None, - validation_duration, - }) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_metal_buffer( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - submit_lossless_from_metal_buffer(tile, options, session)?.wait() -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_metal_buffer_to_metal( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let _ = (tile, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn submit_lossless_from_metal_buffer( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let _ = (tile, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_metal_buffer_with_report( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let _ = (tile, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_metal_buffer_to_metal_with_report( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let _ = (tile, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_padded_metal_buffer( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - submit_lossless_from_padded_metal_buffer(tile, options, session)?.wait() -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_padded_metal_buffer_to_metal( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let _ = (tile, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn submit_lossless_from_padded_metal_buffer( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let _ = (tile, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_padded_metal_buffer_with_report( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let _ = (tile, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_padded_metal_buffer_to_metal_with_report( - tile: MetalLosslessEncodeTile<'_>, - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let _ = (tile, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_metal_buffers( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - submit_lossless_from_metal_buffers(tiles, options, session)?.wait() -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_metal_buffers_to_metal( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - let _ = (tiles, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn submit_lossless_from_metal_buffers( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let _ = (tiles, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn submit_lossless_from_metal_buffers_with_config( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - config: MetalLosslessEncodeConfig, -) -> Result { - let _ = (tiles, options, session, config); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_metal_buffers_with_report( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - let _ = (tiles, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_metal_buffers_to_metal_with_report( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - let _ = (tiles, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_metal_buffers_to_metal_batch( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - config: MetalLosslessEncodeConfig, -) -> Result { - let _ = (tiles, options, session, config); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_padded_metal_buffers( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - submit_lossless_from_padded_metal_buffers(tiles, options, session)?.wait() -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_padded_metal_buffers_to_metal( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - let _ = (tiles, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn submit_lossless_from_padded_metal_buffers( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result { - let _ = (tiles, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn submit_lossless_from_padded_metal_buffers_with_config( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - config: MetalLosslessEncodeConfig, -) -> Result { - let _ = (tiles, options, session, config); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_padded_metal_buffers_with_report( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - let _ = (tiles, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_padded_metal_buffers_to_metal_batch( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, - config: MetalLosslessEncodeConfig, -) -> Result { - let _ = (tiles, options, session, config); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(not(target_os = "macos"))] -pub fn encode_lossless_from_padded_metal_buffers_to_metal_with_report( - tiles: &[MetalLosslessEncodeTile<'_>], - options: &J2kLosslessEncodeOptions, - session: &crate::MetalBackendSession, -) -> Result, crate::Error> { - let _ = (tiles, options, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(target_os = "macos")] -pub fn validate_lossless_roundtrip_on_metal( - samples: J2kLosslessSamples<'_>, - codestream: &[u8], -) -> Result<(), crate::Error> { - let session = crate::MetalBackendSession::system_default()?; - validate_lossless_roundtrip_on_metal_with_session(samples, codestream, &session) -} - -#[cfg(not(target_os = "macos"))] -pub fn validate_lossless_roundtrip_on_metal( - samples: J2kLosslessSamples<'_>, - codestream: &[u8], -) -> Result<(), crate::Error> { - let _ = (samples, codestream); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(target_os = "macos")] -pub fn validate_lossless_roundtrip_on_metal_with_session( - samples: J2kLosslessSamples<'_>, - codestream: &[u8], - session: &crate::MetalBackendSession, -) -> Result<(), crate::Error> { - let fmt = validation_pixel_format(samples)?; - let mut decoder = crate::J2kDecoder::new(codestream)?; - let surface = decoder.decode_to_device_with_session(fmt, session)?; - - if surface.dimensions() != (samples.width, samples.height) { - return Err(crate::Error::MetalKernel { - message: format!( - "J2K Metal validation geometry mismatch: expected {}x{}, got {}x{}", - samples.width, - samples.height, - surface.dimensions().0, - surface.dimensions().1 - ), - }); - } - if surface.pixel_format() != fmt { - return Err(crate::Error::MetalKernel { - message: format!( - "J2K Metal validation format mismatch: expected {:?}, got {:?}", - fmt, - surface.pixel_format() - ), - }); - } - let expected_pitch = samples.width as usize * fmt.bytes_per_pixel(); - if surface.pitch_bytes() != expected_pitch { - return Err(crate::Error::MetalKernel { - message: format!( - "J2K Metal validation pitch mismatch: expected {expected_pitch}, got {}", - surface.pitch_bytes() - ), - }); - } - if surface.byte_len() != samples.data.len() { - return Err(crate::Error::MetalKernel { - message: format!( - "J2K Metal validation length mismatch: expected {} bytes, got {} bytes", - samples.data.len(), - surface.byte_len() - ), - }); - } - - let (buffer, byte_offset) = - surface - .metal_buffer() - .ok_or(crate::Error::UnsupportedMetalRequest { - reason: "J2K Metal validation decode did not return a Metal buffer", - })?; - compute::validate_metal_buffer_matches_bytes(samples.data, buffer, byte_offset, session) -} - -#[cfg(not(target_os = "macos"))] -pub fn validate_lossless_roundtrip_on_metal_with_session( - samples: J2kLosslessSamples<'_>, - codestream: &[u8], - session: &crate::MetalBackendSession, -) -> Result<(), crate::Error> { - let _ = (samples, codestream, session); - Err(crate::Error::MetalUnavailable) -} - -#[cfg(target_os = "macos")] -fn validation_pixel_format(samples: J2kLosslessSamples<'_>) -> Result { - match (samples.components, samples.bit_depth) { - (1, 1..=8) => Ok(PixelFormat::Gray8), - (3, 1..=8) => Ok(PixelFormat::Rgb8), - (1, 9..=16) => Ok(PixelFormat::Gray16), - (3, 9..=16) => Ok(PixelFormat::Rgb16), - _ => Err(crate::Error::UnsupportedMetalRequest { - reason: "J2K Metal validation supports only grayscale or RGB samples up to 16 bits", - }), - } -} - -#[cfg(target_os = "macos")] -fn lossless_sample_shape(format: PixelFormat) -> Result<(u8, u8), crate::Error> { - match format { - PixelFormat::Gray8 => Ok((1, 8)), - PixelFormat::Rgb8 => Ok((3, 8)), - PixelFormat::Gray16 => Ok((1, 16)), - PixelFormat::Rgb16 => Ok((3, 16)), - PixelFormat::Rgba8 | PixelFormat::Rgba16 => Err(crate::Error::UnsupportedMetalRequest { - reason: "J2K Metal encode from RGBA tiles requires explicit alpha handling", - }), - _ => Err(crate::Error::UnsupportedMetalRequest { - reason: "J2K Metal encode received an unknown pixel format", - }), - } -} - -#[cfg(target_os = "macos")] -fn validate_metal_encode_tile(tile: MetalLosslessEncodeTile<'_>) -> Result<(), crate::Error> { - if tile.width == 0 || tile.height == 0 || tile.output_width == 0 || tile.output_height == 0 { - return Err(crate::Error::MetalKernel { - message: "J2K Metal encode tile dimensions must be nonzero".to_string(), - }); - } - if tile.width > tile.output_width || tile.height > tile.output_height { - return Err(crate::Error::MetalKernel { - message: "J2K Metal encode input tile exceeds output tile dimensions".to_string(), - }); - } - let row_bytes = tile - .width - .checked_mul(tile.format.bytes_per_pixel() as u32) - .ok_or_else(|| crate::Error::MetalKernel { - message: "J2K Metal encode row byte count overflow".to_string(), - })? as usize; - if tile.pitch_bytes < row_bytes { - return Err(crate::Error::MetalKernel { - message: "J2K Metal encode tile pitch is shorter than one row".to_string(), - }); - } - let required_end = tile - .byte_offset - .checked_add( - tile.pitch_bytes - .checked_mul(tile.height.saturating_sub(1) as usize) - .and_then(|prefix| prefix.checked_add(row_bytes)) - .ok_or_else(|| crate::Error::MetalKernel { - message: "J2K Metal encode input byte range overflow".to_string(), - })?, - ) - .ok_or_else(|| crate::Error::MetalKernel { - message: "J2K Metal encode input byte range overflow".to_string(), - })?; - let buffer_len = - usize::try_from(tile.buffer.length()).map_err(|_| crate::Error::MetalKernel { - message: "J2K Metal encode buffer length exceeds usize".to_string(), - })?; - if required_end > buffer_len { - return Err(crate::Error::MetalKernel { - message: format!( - "J2K Metal encode input byte range exceeds buffer length: need {required_end}, buffer has {buffer_len}" - ), - }); - } - Ok(()) -} - -#[cfg(target_os = "macos")] -fn validate_padded_contiguous_metal_encode_tile( - tile: MetalLosslessEncodeTile<'_>, - bytes_per_pixel: usize, -) -> Result<(), crate::Error> { - if tile.width != tile.output_width || tile.height != tile.output_height { - return Err(crate::Error::MetalKernel { - message: - "J2K Metal no-copy encode requires input dimensions to match output dimensions" - .to_string(), - }); - } - let expected_pitch = (tile.output_width as usize) - .checked_mul(bytes_per_pixel) - .ok_or_else(|| crate::Error::MetalKernel { - message: "J2K Metal no-copy encode pitch overflow".to_string(), - })?; - if tile.pitch_bytes != expected_pitch { - return Err(crate::Error::MetalKernel { - message: format!( - "J2K Metal no-copy encode requires contiguous rows: expected pitch {expected_pitch}, got {}", - tile.pitch_bytes - ), - }); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::MetalEncodeStageAccelerator; - #[cfg(target_os = "macos")] - use crate::compute; - #[cfg(target_os = "macos")] - use metal::foreign_types::ForeignType; - #[cfg(target_os = "macos")] - use metal::Buffer; - use signinum_core::DeviceSubmission; - #[cfg(target_os = "macos")] - use signinum_core::{BackendKind, PixelFormat}; - #[cfg(target_os = "macos")] - use signinum_j2k::{ - encode_j2k_lossless_with_accelerator, EncodeBackendPreference, J2kBlockCodingMode, - J2kEncodeValidation, J2kLosslessSamples, J2kProgressionOrder, - }; - use signinum_j2k::{EncodedJ2k, J2kLosslessEncodeOptions}; - use signinum_j2k_native::{ - encode_with_accelerator, DecodeSettings, EncodeOptions, Image, J2kEncodeStageAccelerator, - J2kForwardRctJob, - }; - #[cfg(target_os = "macos")] - use signinum_j2k_native::{J2kCodeBlockStyle, J2kForwardDwt53Job}; - - #[cfg(target_os = "macos")] - fn private_buffer_with_bytes(session: &crate::MetalBackendSession, bytes: &[u8]) -> Buffer { - let upload = session.device().new_buffer_with_data( - bytes.as_ptr().cast(), - bytes.len() as u64, - metal::MTLResourceOptions::StorageModeShared, - ); - let private = session.device().new_buffer( - bytes.len() as u64, - metal::MTLResourceOptions::StorageModePrivate, - ); - let queue = session.device().new_command_queue(); - let command_buffer = queue.new_command_buffer(); - let blit = command_buffer.new_blit_command_encoder(); - blit.copy_from_buffer(&upload, 0, &private, 0, bytes.len() as u64); - blit.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - private - } - - #[cfg(target_os = "macos")] - fn overwrite_private_buffer_with_bytes( - session: &crate::MetalBackendSession, - dst: &Buffer, - bytes: &[u8], - ) { - let upload = session.device().new_buffer_with_data( - bytes.as_ptr().cast(), - bytes.len() as u64, - metal::MTLResourceOptions::StorageModeShared, - ); - let queue = session.device().new_command_queue(); - let command_buffer = queue.new_command_buffer(); - let blit = command_buffer.new_blit_command_encoder(); - blit.copy_from_buffer(&upload, 0, dst, 0, bytes.len() as u64); - blit.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - } - - #[cfg(target_os = "macos")] - #[test] - fn inflight_limited_runner_starts_next_item_before_slow_peer_finishes() { - use std::sync::{Arc, Condvar, Mutex}; - use std::time::Duration; - - #[derive(Default)] - struct Probe { - third_item_started: bool, - } - - let probe = Arc::new((Mutex::new(Probe::default()), Condvar::new())); - let task_probe = Arc::clone(&probe); - - let outcomes = super::collect_inflight_limited_ordered(vec![0usize, 1, 2], 2, move |_, item| { - match item { - 0 => Ok(item), - 1 => { - let (lock, cvar) = &*task_probe; - let state = lock.lock().expect("probe mutex"); - let (state, _timeout) = cvar - .wait_timeout_while(state, Duration::from_millis(250), |state| { - !state.third_item_started - }) - .expect("probe wait"); - if !state.third_item_started { - return Err(crate::Error::MetalKernel { - message: - "runner waited for the whole in-flight chunk before scheduling more work" - .to_string(), - }); - } - Ok(item) - } - 2 => { - let (lock, cvar) = &*task_probe; - let mut state = lock.lock().expect("probe mutex"); - state.third_item_started = true; - cvar.notify_all(); - Ok(item) - } - _ => unreachable!("unexpected test item"), - } - }) - .expect("in-flight runner should slide past a slow peer"); - - assert_eq!(outcomes.items, vec![0, 1, 2]); - assert!(outcomes.max_observed_inflight_items <= 2); - assert!(outcomes.max_observed_inflight_items > 0); - } - - #[test] - fn submitted_lossless_metal_encode_public_api_is_available() { - fn assert_single_submission< - S: DeviceSubmission, - >() { - } - fn assert_batch_submission< - S: DeviceSubmission, Error = crate::Error>, - >() { - } - fn assert_submit_single_fn( - _submit: for<'tile, 'options, 'session> fn( - super::MetalLosslessEncodeTile<'tile>, - &'options J2kLosslessEncodeOptions, - &'session crate::MetalBackendSession, - ) -> Result< - crate::SubmittedJ2kLosslessMetalEncode, - crate::Error, - >, - ) { - } - fn assert_submit_batch_fn( - _submit: for<'slice, 'tile, 'options, 'session> fn( - &'slice [super::MetalLosslessEncodeTile<'tile>], - &'options J2kLosslessEncodeOptions, - &'session crate::MetalBackendSession, - ) -> Result< - crate::SubmittedJ2kLosslessMetalEncodeBatch, - crate::Error, - >, - ) { - } - - assert_single_submission::(); - assert_batch_submission::(); - assert_submit_single_fn(crate::submit_lossless_from_metal_buffer); - assert_submit_single_fn(crate::submit_lossless_from_padded_metal_buffer); - assert_submit_batch_fn(crate::submit_lossless_from_metal_buffers); - assert_submit_batch_fn(crate::submit_lossless_from_padded_metal_buffers); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_dispatch_option_treats_unavailable_as_no_dispatch() { - let result: Result, &'static str> = - super::metal_dispatch_option(Err(crate::Error::MetalUnavailable), "kernel failed"); - - assert_eq!(result, Ok(None)); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_dispatch_option_preserves_kernel_errors() { - let result: Result, &'static str> = super::metal_dispatch_option( - Err(crate::Error::MetalKernel { - message: "bad status".to_string(), - }), - "kernel failed", - ); - - assert_eq!(result, Err("kernel failed")); - } - - #[test] - fn metal_encode_stage_accelerator_preserves_cpu_codestream_validity() { - let pixels: Vec = (0..8 * 8 * 3).map(|i| (i & 0xFF) as u8).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let mut accelerator = MetalEncodeStageAccelerator::default(); - - let codestream = - encode_with_accelerator(&pixels, 8, 8, 3, 8, false, &options, &mut accelerator) - .expect("encode with metal stage accelerator"); - let decoded = Image::new(&codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - - assert_eq!(decoded.width, 8); - assert_eq!(decoded.height, 8); - assert_eq!(decoded.num_components, 3); - assert_eq!(decoded.bit_depth, 8); - assert_eq!(accelerator.forward_rct_attempts(), 1); - assert_eq!(accelerator.forward_dwt53_attempts(), 3); - assert!(accelerator.tier1_code_block_attempts() > 0); - assert_eq!(accelerator.packetization_attempts(), 1); - } - - #[test] - fn metal_encode_stage_accelerator_can_leave_forward_rct_on_cpu() { - let mut plane0 = vec![0.0, 64.0, 128.0, 255.0]; - let mut plane1 = vec![3.0, 67.0, 131.0, 252.0]; - let mut plane2 = vec![7.0, 71.0, 135.0, 248.0]; - let original = (plane0.clone(), plane1.clone(), plane2.clone()); - let mut accelerator = MetalEncodeStageAccelerator::with_cpu_forward_rct(); - - let dispatched = accelerator - .encode_forward_rct(J2kForwardRctJob { - plane0: &mut plane0, - plane1: &mut plane1, - plane2: &mut plane2, - }) - .expect("CPU RCT fallback should be selectable"); - - assert!(!dispatched); - assert_eq!(accelerator.forward_rct_attempts(), 1); - assert_eq!(accelerator.forward_rct_dispatches(), 0); - assert_eq!((plane0, plane1, plane2), original); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_forward_rct_dispatch_round_trips_rgb8_lossless_tile() { - let pixels: Vec = (0..7 * 5 * 3).map(|i| ((i * 17) & 0xFF) as u8).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 0, - ..EncodeOptions::default() - }; - let mut accelerator = MetalEncodeStageAccelerator::default(); - - let codestream = - encode_with_accelerator(&pixels, 7, 5, 3, 8, false, &options, &mut accelerator) - .expect("encode with metal forward RCT"); - let decoded = Image::new(&codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - - assert_eq!(decoded.data, pixels); - assert_eq!(accelerator.forward_rct_attempts(), 1); - assert_eq!(accelerator.forward_rct_dispatches(), 1); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_validation_decodes_and_compares_lossless_codestream_on_device() { - let pixels: Vec = (0..16 * 16 * 3).map(|i| ((i * 29) & 0xFF) as u8).collect(); - let samples = J2kLosslessSamples::new(&pixels, 16, 16, 3, 8, false).unwrap(); - let encoded = signinum_j2k::encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - ..J2kLosslessEncodeOptions::default() - }, - ) - .expect("lossless encode"); - - super::validate_lossless_roundtrip_on_metal(samples, &encoded.codestream) - .expect("Metal lossless validation"); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_buffer_lossless_encode_pads_edge_tile_on_device() { - let pixels: Vec = (0..7 * 5 * 3).map(|i| ((i * 19) & 0xFF) as u8).collect(); - let device = metal::Device::system_default().expect("Metal device"); - if !compute::ht_simd_prototype_available_for_device_for_test(&device) - .expect("HTJ2K SIMD prototype availability query") - { - return; - } - let session = crate::MetalBackendSession::new(device); - let buffer = session.device().new_buffer_with_data( - pixels.as_ptr().cast(), - pixels.len() as u64, - metal::MTLResourceOptions::StorageModeShared, - ); - - let encoded = super::encode_lossless_from_metal_buffer( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 7, - height: 5, - pitch_bytes: 7 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal buffer lossless encode"); - - assert_eq!(encoded.backend, BackendKind::Metal); - let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.width, 8); - assert_eq!(decoded.height, 8); - for y in 0..8usize { - for x in 0..8usize { - let dst = (y * 8 + x) * 3; - if x < 7 && y < 5 { - let src = (y * 7 + x) * 3; - assert_eq!(&decoded.data[dst..dst + 3], &pixels[src..src + 3]); - } else { - assert_eq!(&decoded.data[dst..dst + 3], &[0, 0, 0]); - } - } - } - } - - #[cfg(target_os = "macos")] - #[test] - fn submitted_metal_buffer_lossless_encode_wait_round_trips() { - let pixels: Vec = (0..7 * 5 * 3).map(|i| ((i * 19) & 0xFF) as u8).collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffer = session.device().new_buffer_with_data( - pixels.as_ptr().cast(), - pixels.len() as u64, - metal::MTLResourceOptions::StorageModeShared, - ); - - let submitted: crate::SubmittedJ2kLosslessMetalEncode = - crate::submit_lossless_from_metal_buffer( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 7, - height: 5, - pitch_bytes: 7 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("submit Metal buffer lossless encode"); - let encoded = submitted.wait().expect("wait Metal buffer lossless encode"); - - assert_eq!(encoded.backend, BackendKind::Metal); - let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.width, 8); - assert_eq!(decoded.height, 8); - for y in 0..8usize { - for x in 0..8usize { - let dst = (y * 8 + x) * 3; - if x < 7 && y < 5 { - let src = (y * 7 + x) * 3; - assert_eq!(&decoded.data[dst..dst + 3], &pixels[src..src + 3]); - } else { - assert_eq!(&decoded.data[dst..dst + 3], &[0, 0, 0]); - } - } - } - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_buffer_lossless_encode_accepts_padded_contiguous_input_without_copy() { - let pixels: Vec = (0..8 * 8 * 3).map(|i| ((i * 31) & 0xFF) as u8).collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffer = session.device().new_buffer_with_data( - pixels.as_ptr().cast(), - pixels.len() as u64, - metal::MTLResourceOptions::StorageModeShared, - ); - - let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal padded buffer lossless encode"); - - assert_eq!(encoded.encoded.backend, BackendKind::Metal); - assert!(!encoded.input_copy_used); - assert_eq!(encoded.input_copy_duration, std::time::Duration::ZERO); - let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.width, 8); - assert_eq!(decoded.height, 8); - assert_eq!(decoded.data, pixels); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_padded_private_rgb8_encode_uses_resident_coefficient_prep() { - let pixels: Vec = (0..8 * 8 * 3).map(|i| ((i * 31) & 0xFF) as u8).collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffer = private_buffer_with_bytes(&session, &pixels); - - let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal private padded buffer lossless encode"); - - assert_eq!(encoded.encoded.backend, BackendKind::Metal); - assert!(!encoded.input_copy_used); - assert!(encoded.resident.coefficient_prep_used); - assert!(encoded.resident.packetization_used); - assert!(encoded.resident.codestream_assembly_used); - let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, pixels); - } - - #[cfg(target_os = "macos")] - #[test] - fn auto_host_output_encode_options_preserve_auto_for_hybrid_path() { - let routed = super::host_output_encode_options(J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::Auto, - validation: J2kEncodeValidation::CpuRoundTrip, - ..J2kLosslessEncodeOptions::default() - }); - - assert_eq!(routed.backend, EncodeBackendPreference::Auto); - assert_eq!(routed.validation, J2kEncodeValidation::External); - - let prefer_device = super::host_output_encode_options(J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::PreferDevice, - validation: J2kEncodeValidation::CpuRoundTrip, - ..J2kLosslessEncodeOptions::default() - }); - assert_eq!(prefer_device.backend, EncodeBackendPreference::PreferDevice); - assert_eq!(prefer_device.validation, J2kEncodeValidation::External); - } - - #[cfg(target_os = "macos")] - #[test] - fn auto_host_output_accelerator_uses_metal_dwt_with_cpu_block_fallback() { - let pixels: Vec = (0..64 * 64).map(|i| ((i * 17) & 0xff) as u8).collect(); - let samples = - J2kLosslessSamples::new(&pixels, 64, 64, 1, 8, false).expect("valid gray samples"); - let options = J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::Auto, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - }; - let mut accelerator = MetalEncodeStageAccelerator::for_auto_host_output(); - - let encoded = encode_j2k_lossless_with_accelerator( - samples, - &options, - BackendKind::Metal, - &mut accelerator, - ) - .expect("hybrid host-output encode"); - - assert_eq!(encoded.backend, BackendKind::Cpu); - assert_eq!(accelerator.forward_dwt53_dispatches(), 1); - assert_eq!(accelerator.tier1_code_block_dispatches(), 0); - assert_eq!(accelerator.packetization_dispatches(), 0); - assert!(accelerator.prefer_parallel_cpu_code_block_fallback()); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_padded_private_rgb8_auto_host_encode_routes_away_from_resident_prep() { - let pixels: Vec = (0..8 * 8 * 3).map(|i| ((i * 43) & 0xFF) as u8).collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffer = private_buffer_with_bytes(&session, &pixels); - - let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::Auto, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Auto host-output encode should avoid resident prep and still succeed"); - - assert_eq!(encoded.encoded.backend, BackendKind::Cpu); - assert!(!encoded.resident.coefficient_prep_used); - assert!(!encoded.resident.packetization_used); - assert!(!encoded.resident.codestream_assembly_used); - let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, pixels); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_padded_private_rgb8_encode_to_metal_buffer_exposes_finished_bytes() { - let pixels: Vec = (0..8 * 8 * 3).map(|i| ((i * 37) & 0xFF) as u8).collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffer = private_buffer_with_bytes(&session, &pixels); - - let encoded = super::encode_lossless_from_padded_metal_buffer_to_metal_with_report( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal private padded buffer lossless encode to Metal buffer"); - - assert!(!encoded.input_copy_used); - assert!(encoded.resident.coefficient_prep_used); - assert!(encoded.resident.packetization_used); - assert!(encoded.resident.codestream_assembly_used); - assert!( - encoded.gpu_duration.is_some(), - "resident Metal encode should report command-buffer GPU duration" - ); - assert_eq!(encoded.encoded.byte_offset, 0); - assert!(encoded.encoded.byte_len > 0); - assert!(encoded.encoded.capacity >= encoded.encoded.byte_len); - let codestream = encoded - .encoded - .codestream_bytes() - .expect("Metal codestream bytes are CPU-readable"); - assert!(codestream.starts_with(&[0xFF, 0x4F])); - let decoded = Image::new(codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, pixels); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_edge_private_rgb8_encode_to_metal_buffer_pads_and_stays_resident() { - let pixels: Vec = (0..7 * 5 * 3).map(|i| ((i * 41) & 0xFF) as u8).collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffer = private_buffer_with_bytes(&session, &pixels); - - let encoded = super::encode_lossless_from_metal_buffer_to_metal_with_report( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 7, - height: 5, - pitch_bytes: 7 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal private edge buffer lossless encode to Metal buffer"); - - assert!(!encoded.input_copy_used); - assert!(encoded.resident.coefficient_prep_used); - assert!(encoded.resident.packetization_used); - assert!(encoded.resident.codestream_assembly_used); - let codestream = encoded - .encoded - .codestream_bytes() - .expect("Metal codestream bytes are CPU-readable"); - assert!(codestream.starts_with(&[0xFF, 0x4F])); - let decoded = Image::new(codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.width, 8); - assert_eq!(decoded.height, 8); - for y in 0..8usize { - for x in 0..8usize { - let dst = (y * 8 + x) * 3; - if x < 7 && y < 5 { - let src = (y * 7 + x) * 3; - assert_eq!(&decoded.data[dst..dst + 3], &pixels[src..src + 3]); - } else { - assert_eq!(&decoded.data[dst..dst + 3], &[0, 0, 0]); - } - } - } - } - - #[cfg(target_os = "macos")] - #[test] - fn submitted_private_padded_rgb8_encode_snapshots_before_wait() { - let pixels: Vec = (0..8 * 8 * 3).map(|i| ((i * 31) & 0xFF) as u8).collect(); - let replacement = vec![0u8; pixels.len()]; - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffer = private_buffer_with_bytes(&session, &pixels); - - let submitted = super::submit_lossless_from_padded_metal_buffer( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("submit Metal private padded RGB8 encode"); - overwrite_private_buffer_with_bytes(&session, &buffer, &replacement); - - let encoded = submitted.wait().expect("wait submitted encode"); - let decoded = Image::new(&encoded.codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, pixels); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_padded_private_gray8_dwt_encode_uses_resident_coefficient_prep() { - let mut pixels = Vec::with_capacity(128 * 128); - for y in 0..128u32 { - for x in 0..128u32 { - pixels.push(((x * 7 + y * 11 + (x ^ y)) & 0xFF) as u8); - } - } - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffer = private_buffer_with_bytes(&session, &pixels); - - let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 128, - height: 128, - pitch_bytes: 128, - output_width: 128, - output_height: 128, - format: PixelFormat::Gray8, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal private padded DWT buffer lossless encode"); - - assert_eq!(encoded.encoded.backend, BackendKind::Metal); - assert!(!encoded.input_copy_used); - assert!(encoded.resident.coefficient_prep_used); - assert!(encoded.resident.packetization_used); - assert!(encoded.resident.codestream_assembly_used); - let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, pixels); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_padded_private_rgb8_dwt_encode_uses_resident_coefficient_prep() { - let mut pixels = Vec::with_capacity(128 * 128 * 3); - for y in 0..128u32 { - for x in 0..128u32 { - pixels.push(((x * 3 + y * 5) & 0xFF) as u8); - pixels.push(((x * 7 + y * 11) & 0xFF) as u8); - pixels.push(((x * 13 + y * 17) & 0xFF) as u8); - } - } - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffer = private_buffer_with_bytes(&session, &pixels); - - let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 128, - height: 128, - pitch_bytes: 128 * 3, - output_width: 128, - output_height: 128, - format: PixelFormat::Rgb8, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal private padded RGB8 DWT buffer lossless encode"); - - assert_eq!(encoded.encoded.backend, BackendKind::Metal); - assert!(encoded.resident.coefficient_prep_used); - assert!(encoded.resident.packetization_used); - assert!(encoded.resident.codestream_assembly_used); - let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, pixels); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_padded_private_gray8_rpcl_encode_uses_resident_coefficient_prep() { - let mut pixels = Vec::with_capacity(128 * 128); - for y in 0..128u32 { - for x in 0..128u32 { - pixels.push(((x * 5 + y * 9 + (x ^ y)) & 0xFF) as u8); - } - } - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffer = private_buffer_with_bytes(&session, &pixels); - - let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 128, - height: 128, - pitch_bytes: 128, - output_width: 128, - output_height: 128, - format: PixelFormat::Gray8, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - progression: J2kProgressionOrder::Rpcl, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal private padded RPCL buffer lossless encode"); - - assert_eq!(encoded.encoded.backend, BackendKind::Metal); - assert!(encoded.resident.coefficient_prep_used); - assert!(encoded.resident.packetization_used); - assert!(encoded.resident.codestream_assembly_used); - let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, pixels); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_padded_private_gray16_encode_uses_resident_coefficient_prep() { - let mut pixels = Vec::with_capacity(8 * 8 * 2); - for idx in 0..64u16 { - let value = idx.wrapping_mul(997).wrapping_add(123); - pixels.extend_from_slice(&value.to_le_bytes()); - } - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffer = private_buffer_with_bytes(&session, &pixels); - - let encoded = super::encode_lossless_from_padded_metal_buffer_with_report( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8 * 2, - output_width: 8, - output_height: 8, - format: PixelFormat::Gray16, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal private padded Gray16 buffer lossless encode"); - - assert_eq!(encoded.encoded.backend, BackendKind::Metal); - assert!(!encoded.input_copy_used); - assert!(encoded.resident.coefficient_prep_used); - assert!(encoded.resident.packetization_used); - assert!(encoded.resident.codestream_assembly_used); - let decoded = Image::new(&encoded.encoded.codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, pixels); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_padded_private_ht_encode_to_metal_buffer_stays_resident() { - let pixels: Vec = (0..8 * 8).map(|i| ((i * 31) & 0xFF) as u8).collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffer = private_buffer_with_bytes(&session, &pixels); - - let encoded = super::encode_lossless_from_padded_metal_buffer_to_metal_with_report( - super::MetalLosslessEncodeTile { - buffer: &buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8, - output_width: 8, - output_height: 8, - format: PixelFormat::Gray8, - }, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - block_coding_mode: J2kBlockCodingMode::HighThroughput, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal private padded HTJ2K buffer lossless encode"); - - assert!(!encoded.input_copy_used); - assert!(encoded.resident.coefficient_prep_used); - assert!(encoded.resident.packetization_used); - assert!(encoded.resident.codestream_assembly_used); - let codestream = encoded - .encoded - .codestream_bytes() - .expect("Metal codestream bytes are CPU-readable"); - assert!(codestream.windows(2).any(|window| window == [0xFF, 0x50])); - let cod_marker = codestream - .windows(2) - .position(|window| window == [0xFF, 0x52]) - .expect("COD marker"); - assert_eq!(codestream[cod_marker + 12], 0x40); - let decoded = Image::new(codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, pixels); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_buffer_lossless_batch_encodes_padded_contiguous_inputs() { - let first: Vec = (0..8 * 8 * 3).map(|i| ((i * 7) & 0xFF) as u8).collect(); - let second: Vec = (0..8 * 8 * 3) - .map(|i| ((i * 13 + 5) & 0xFF) as u8) - .collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let first_buffer = session.device().new_buffer_with_data( - first.as_ptr().cast(), - first.len() as u64, - metal::MTLResourceOptions::StorageModeShared, - ); - let second_buffer = session.device().new_buffer_with_data( - second.as_ptr().cast(), - second.len() as u64, - metal::MTLResourceOptions::StorageModeShared, - ); - let tiles = [ - super::MetalLosslessEncodeTile { - buffer: &first_buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - super::MetalLosslessEncodeTile { - buffer: &second_buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - ]; - - let encoded = super::encode_lossless_from_padded_metal_buffers_with_report( - &tiles, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal padded buffer batch lossless encode"); - - assert_eq!(encoded.len(), 2); - for (frame, expected) in encoded.iter().zip([first, second]) { - assert_eq!(frame.encoded.backend, BackendKind::Metal); - assert!(!frame.input_copy_used); - let decoded = Image::new(&frame.encoded.codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, expected); - } - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_padded_private_batch_encode_to_metal_buffers_exposes_per_frame_bytes() { - let first: Vec = (0..8 * 8 * 3).map(|i| ((i * 17) & 0xFF) as u8).collect(); - let second: Vec = (0..8 * 8 * 3) - .map(|i| 255u8.wrapping_sub(((i * 23) & 0xFF) as u8)) - .collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let first_buffer = private_buffer_with_bytes(&session, &first); - let second_buffer = private_buffer_with_bytes(&session, &second); - let tiles = [ - super::MetalLosslessEncodeTile { - buffer: &first_buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - super::MetalLosslessEncodeTile { - buffer: &second_buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - ]; - - let encoded = super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( - &tiles, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal padded buffer batch lossless encode to Metal buffers"); - - assert_eq!(encoded.len(), 2); - assert_eq!( - encoded[0].encoded.codestream_buffer.as_ptr(), - encoded[1].encoded.codestream_buffer.as_ptr(), - "classic J2K resident batch encode should assemble codestreams into one shared batch buffer" - ); - assert_eq!(encoded[0].encoded.byte_offset, 0); - assert!( - encoded[1].encoded.byte_offset > 0, - "second classic J2K batch codestream should be a nonzero slice into the shared batch buffer" - ); - for (frame, expected) in encoded.iter().zip([first, second]) { - assert!(!frame.input_copy_used); - assert!(frame.resident.coefficient_prep_used); - assert!(frame.resident.packetization_used); - assert!(frame.resident.codestream_assembly_used); - let codestream = frame - .encoded - .codestream_bytes() - .expect("Metal codestream bytes are CPU-readable"); - let decoded = Image::new(codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, expected); - } - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_edge_private_batch_encode_to_metal_buffers_stays_resident() { - let first: Vec = (0..7 * 5 * 3).map(|i| ((i * 17) & 0xFF) as u8).collect(); - let second: Vec = (0..6 * 8 * 3) - .map(|i| 255u8.wrapping_sub(((i * 19) & 0xFF) as u8)) - .collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let first_buffer = private_buffer_with_bytes(&session, &first); - let second_buffer = private_buffer_with_bytes(&session, &second); - compute::reset_ht_batch_coefficient_copy_blits_for_test(); - let tiles = [ - super::MetalLosslessEncodeTile { - buffer: &first_buffer, - byte_offset: 0, - width: 7, - height: 5, - pitch_bytes: 7 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - super::MetalLosslessEncodeTile { - buffer: &second_buffer, - byte_offset: 0, - width: 6, - height: 8, - pitch_bytes: 6 * 3, - output_width: 8, - output_height: 8, - format: PixelFormat::Rgb8, - }, - ]; - - let encoded = super::encode_lossless_from_metal_buffers_to_metal_with_report( - &tiles, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal edge buffer batch lossless encode to Metal buffers"); - - assert_eq!(encoded.len(), 2); - for frame in &encoded { - assert!(!frame.input_copy_used); - assert!(frame.resident.coefficient_prep_used); - assert!(frame.resident.packetization_used); - assert!(frame.resident.codestream_assembly_used); - } - - for (frame, (expected, width, height)) in encoded - .iter() - .zip([(first, 7usize, 5usize), (second, 6usize, 8usize)]) - { - let codestream = frame - .encoded - .codestream_bytes() - .expect("Metal codestream bytes are CPU-readable"); - let decoded = Image::new(codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - for y in 0..8usize { - for x in 0..8usize { - let dst = (y * 8 + x) * 3; - if x < width && y < height { - let src = (y * width + x) * 3; - assert_eq!(&decoded.data[dst..dst + 3], &expected[src..src + 3]); - } else { - assert_eq!(&decoded.data[dst..dst + 3], &[0, 0, 0]); - } - } - } - } - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_ht_private_batch_encode_to_metal_buffers_stays_resident() { - let first: Vec = (0..8 * 8).map(|i| ((i * 11) & 0xFF) as u8).collect(); - let second: Vec = (0..8 * 8) - .map(|i| 255u8.wrapping_sub(((i * 13) & 0xFF) as u8)) - .collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let first_buffer = private_buffer_with_bytes(&session, &first); - let second_buffer = private_buffer_with_bytes(&session, &second); - let tiles = [ - super::MetalLosslessEncodeTile { - buffer: &first_buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8, - output_width: 8, - output_height: 8, - format: PixelFormat::Gray8, - }, - super::MetalLosslessEncodeTile { - buffer: &second_buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8, - output_width: 8, - output_height: 8, - format: PixelFormat::Gray8, - }, - ]; - - let encoded = super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( - &tiles, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - block_coding_mode: J2kBlockCodingMode::HighThroughput, - ..J2kLosslessEncodeOptions::default() - }, - &session, - ) - .expect("Metal HTJ2K batch lossless encode to Metal buffers"); - - assert_eq!(encoded.len(), 2); - assert_eq!( - compute::ht_batch_coefficient_copy_blits_for_test(), - 0, - "HTJ2K resident batch prep should write directly into the batch coefficient buffer" - ); - assert_eq!( - encoded[0].encoded.codestream_buffer.as_ptr(), - encoded[1].encoded.codestream_buffer.as_ptr(), - "HTJ2K resident batch encode should assemble codestreams into one shared batch buffer" - ); - assert_eq!(encoded[0].encoded.byte_offset, 0); - assert!( - encoded[1].encoded.byte_offset > 0, - "second HTJ2K batch codestream should be a nonzero slice into the shared batch buffer" - ); - for (frame, expected) in encoded.iter().zip([first, second]) { - assert!(!frame.input_copy_used); - assert!(frame.resident.coefficient_prep_used); - assert!(frame.resident.packetization_used); - assert!(frame.resident.codestream_assembly_used); - let codestream = frame - .encoded - .codestream_bytes() - .expect("Metal codestream bytes are CPU-readable"); - assert!(codestream.windows(2).any(|window| window == [0xFF, 0x50])); - let decoded = Image::new(codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, expected); - } - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_ht_private_batch_encode_reuses_private_arenas_between_batches() { - const WIDTH: usize = 37; - const HEIGHT: usize = 41; - let first: Vec = (0..WIDTH * HEIGHT) - .map(|i| ((i * 7 + 3) & 0xFF) as u8) - .collect(); - let second: Vec = (0..WIDTH * HEIGHT) - .map(|i| 255u8.wrapping_sub(((i * 5 + 11) & 0xFF) as u8)) - .collect(); - let device = metal::Device::system_default().expect("Metal device"); - let session = crate::MetalBackendSession::new(device.clone()); - let first_buffer = private_buffer_with_bytes(&session, &first); - let second_buffer = private_buffer_with_bytes(&session, &second); - let tiles = [ - super::MetalLosslessEncodeTile { - buffer: &first_buffer, - byte_offset: 0, - width: WIDTH as u32, - height: HEIGHT as u32, - pitch_bytes: WIDTH, - output_width: WIDTH as u32, - output_height: HEIGHT as u32, - format: PixelFormat::Gray8, - }, - super::MetalLosslessEncodeTile { - buffer: &second_buffer, - byte_offset: 0, - width: WIDTH as u32, - height: HEIGHT as u32, - pitch_bytes: WIDTH, - output_width: WIDTH as u32, - output_height: HEIGHT as u32, - format: PixelFormat::Gray8, - }, - ]; - let options = J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - block_coding_mode: J2kBlockCodingMode::HighThroughput, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - }; - - compute::with_isolated_runtime_for_device_for_test(&device, || { - compute::reset_private_buffer_pool_misses_for_test(); - super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( - &tiles, &options, &session, - )?; - let first_misses = compute::private_buffer_pool_misses_for_test(); - assert!( - first_misses > 0, - "first unique HTJ2K batch should populate reusable private arenas" - ); - - compute::reset_private_buffer_pool_misses_for_test(); - let encoded = super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( - &tiles, &options, &session, - )?; - - assert_eq!( - compute::private_buffer_pool_misses_for_test(), - 0, - "second same-shape HTJ2K batch should reuse private arenas" - ); - assert_eq!(encoded.len(), 2); - Ok(()) - }) - .expect("isolated HTJ2K Metal runtime"); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_ht_private_batch_encode_uses_simd_prototype_when_enabled() { - const WIDTH: usize = 32; - const HEIGHT: usize = 32; - let first: Vec = (0..WIDTH * HEIGHT) - .map(|i| ((i * 3 + 17) & 0xFF) as u8) - .collect(); - let second: Vec = (0..WIDTH * HEIGHT) - .map(|i| ((i * 11 + 5) & 0xFF) as u8) - .collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let first_buffer = private_buffer_with_bytes(&session, &first); - let second_buffer = private_buffer_with_bytes(&session, &second); - let tiles = [ - super::MetalLosslessEncodeTile { - buffer: &first_buffer, - byte_offset: 0, - width: WIDTH as u32, - height: HEIGHT as u32, - pitch_bytes: WIDTH, - output_width: WIDTH as u32, - output_height: HEIGHT as u32, - format: PixelFormat::Gray8, - }, - super::MetalLosslessEncodeTile { - buffer: &second_buffer, - byte_offset: 0, - width: WIDTH as u32, - height: HEIGHT as u32, - pitch_bytes: WIDTH, - output_width: WIDTH as u32, - output_height: HEIGHT as u32, - format: PixelFormat::Gray8, - }, - ]; - let options = J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - block_coding_mode: J2kBlockCodingMode::HighThroughput, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - }; - - let _route = compute::force_ht_simd_prototype_route_for_test(true); - compute::reset_ht_simd_prototype_dispatches_for_test(); - let encoded = super::encode_lossless_from_padded_metal_buffers_to_metal_with_report( - &tiles, &options, &session, - ) - .expect("HTJ2K SIMD prototype batch encode"); - - assert_eq!(encoded.len(), 2); - assert!( - compute::ht_simd_prototype_dispatches_for_test() > 0, - "enabled HTJ2K batch encode should route through the SIMD prototype" - ); - for (frame, expected) in encoded.iter().zip([first, second]) { - let codestream = frame - .encoded - .codestream_bytes() - .expect("Metal codestream bytes are CPU-readable"); - let decoded = Image::new(codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(decoded.data, expected); - } - } - - #[test] - fn default_gpu_encode_memory_budget_uses_forty_percent_capped_at_ten_gib() { - const GIB: usize = 1024 * 1024 * 1024; - - assert_eq!( - super::default_gpu_encode_memory_budget_bytes_for_hw_mem(8 * GIB), - 8 * GIB * 40 / 100 - ); - assert_eq!( - super::default_gpu_encode_memory_budget_bytes_for_hw_mem(16 * GIB), - 16 * GIB * 40 / 100 - ); - assert_eq!( - super::default_gpu_encode_memory_budget_bytes_for_hw_mem(24 * GIB), - 24 * GIB * 40 / 100 - ); - assert_eq!( - super::default_gpu_encode_memory_budget_bytes_for_hw_mem(64 * GIB), - 10 * GIB - ); - } - - #[test] - fn gpu_encode_inflight_resolution_clamps_requested_tiles_by_memory_budget() { - let stats = super::resolve_lossless_encode_config_for_test( - 100, - 1_000, - super::MetalLosslessEncodeConfig { - gpu_encode_inflight_tiles: Some(32), - gpu_encode_memory_budget_bytes: Some(4_500), - }, - ) - .expect("resolved config"); - - assert_eq!(stats.configured_inflight_tiles, Some(32)); - assert_eq!(stats.effective_inflight_tiles, 4); - assert_eq!(stats.configured_memory_budget_bytes, Some(4_500)); - assert_eq!(stats.effective_memory_budget_bytes, 4_500); - assert_eq!(stats.estimated_peak_bytes_per_tile, 1_000); - } - - #[test] - fn gpu_encode_default_inflight_uses_sixty_four_when_memory_allows() { - let stats = super::resolve_lossless_encode_config_for_test( - 100, - 1_000, - super::MetalLosslessEncodeConfig { - gpu_encode_inflight_tiles: None, - gpu_encode_memory_budget_bytes: Some(1_000_000), - }, - ) - .expect("resolved config"); - - assert_eq!(stats.configured_inflight_tiles, None); - assert_eq!(stats.effective_inflight_tiles, 64); - } - - #[test] - fn gpu_encode_inflight_resolution_rejects_zero_overrides() { - let err = super::resolve_lossless_encode_config_for_test( - 4, - 1_000, - super::MetalLosslessEncodeConfig { - gpu_encode_inflight_tiles: Some(0), - gpu_encode_memory_budget_bytes: Some(4_000), - }, - ) - .unwrap_err(); - assert!( - err.to_string().contains("in-flight"), - "unexpected error: {err}" - ); - - let err = super::resolve_lossless_encode_config_for_test( - 4, - 1_000, - super::MetalLosslessEncodeConfig { - gpu_encode_inflight_tiles: Some(2), - gpu_encode_memory_budget_bytes: Some(0), - }, - ) - .unwrap_err(); - assert!( - err.to_string().contains("memory budget"), - "unexpected error: {err}" - ); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_ht_batch_encode_preserves_order_and_matches_inflight_one() { - let inputs = [ - (0..8 * 8) - .map(|i| ((i * 11 + 3) & 0xFF) as u8) - .collect::>(), - (0..8 * 8) - .map(|i| ((i * 13 + 5) & 0xFF) as u8) - .collect::>(), - (0..8 * 8) - .map(|i| ((i * 17 + 7) & 0xFF) as u8) - .collect::>(), - (0..8 * 8) - .map(|i| ((i * 19 + 9) & 0xFF) as u8) - .collect::>(), - ]; - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let buffers = inputs - .iter() - .map(|bytes| private_buffer_with_bytes(&session, bytes)) - .collect::>(); - let tiles = buffers - .iter() - .map(|buffer| super::MetalLosslessEncodeTile { - buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8, - output_width: 8, - output_height: 8, - format: PixelFormat::Gray8, - }) - .collect::>(); - let options = J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - block_coding_mode: J2kBlockCodingMode::HighThroughput, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - }; - - let serial = super::encode_lossless_from_padded_metal_buffers_to_metal_batch( - &tiles, - &options, - &session, - super::MetalLosslessEncodeConfig { - gpu_encode_inflight_tiles: Some(1), - gpu_encode_memory_budget_bytes: Some(1024 * 1024 * 1024), - }, - ) - .expect("serial Metal HTJ2K batch"); - let parallel = super::encode_lossless_from_padded_metal_buffers_to_metal_batch( - &tiles, - &options, - &session, - super::MetalLosslessEncodeConfig { - gpu_encode_inflight_tiles: Some(2), - gpu_encode_memory_budget_bytes: Some(1024 * 1024 * 1024), - }, - ) - .expect("parallel Metal HTJ2K batch"); - let repeated_parallel = super::encode_lossless_from_padded_metal_buffers_to_metal_batch( - &tiles, - &options, - &session, - super::MetalLosslessEncodeConfig { - gpu_encode_inflight_tiles: Some(2), - gpu_encode_memory_budget_bytes: Some(1024 * 1024 * 1024), - }, - ) - .expect("repeated parallel Metal HTJ2K batch"); - - assert_eq!(serial.outcomes.len(), inputs.len()); - assert_eq!(parallel.outcomes.len(), inputs.len()); - assert_eq!(parallel.stats.effective_inflight_tiles, 2); - assert!(parallel.stats.max_observed_inflight_tiles <= 2); - assert!(parallel.stats.max_observed_inflight_tiles > 0); - for (((serial_outcome, parallel_outcome), repeated_outcome), expected) in serial - .outcomes - .iter() - .zip(parallel.outcomes.iter()) - .zip(repeated_parallel.outcomes.iter()) - .zip(inputs.iter()) - { - let serial_bytes = serial_outcome - .encoded - .codestream_bytes() - .expect("serial codestream"); - let parallel_bytes = parallel_outcome - .encoded - .codestream_bytes() - .expect("parallel codestream"); - let repeated_bytes = repeated_outcome - .encoded - .codestream_bytes() - .expect("repeated parallel codestream"); - assert_eq!(parallel_bytes, serial_bytes); - assert_eq!(repeated_bytes, serial_bytes); - - let decoded = Image::new(parallel_bytes, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - assert_eq!(&decoded.data, expected); - } - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_parallel_batch_returns_indexed_injected_failure() { - let first: Vec = (0..8 * 8).map(|i| ((i * 3) & 0xFF) as u8).collect(); - let second: Vec = (0..8 * 8).map(|i| ((i * 5) & 0xFF) as u8).collect(); - let session = crate::MetalBackendSession::system_default().expect("Metal session"); - let first_buffer = private_buffer_with_bytes(&session, &first); - let second_buffer = private_buffer_with_bytes(&session, &second); - let tiles = [ - super::MetalLosslessEncodeTile { - buffer: &first_buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8, - output_width: 8, - output_height: 8, - format: PixelFormat::Gray8, - }, - super::MetalLosslessEncodeTile { - buffer: &second_buffer, - byte_offset: 0, - width: 8, - height: 8, - pitch_bytes: 8, - output_width: 8, - output_height: 8, - format: PixelFormat::Gray8, - }, - ]; - let options = J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - block_coding_mode: J2kBlockCodingMode::HighThroughput, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - }; - - super::set_test_resident_encode_failure_index(Some(1)); - let Err(err) = super::encode_lossless_from_padded_metal_buffers_to_metal_batch( - &tiles, - &options, - &session, - super::MetalLosslessEncodeConfig { - gpu_encode_inflight_tiles: Some(2), - gpu_encode_memory_budget_bytes: Some(1024 * 1024 * 1024), - }, - ) else { - panic!("injected failure should fail the batch"); - }; - super::set_test_resident_encode_failure_index(None); - - assert!( - err.to_string().contains("tile 1"), - "unexpected error: {err}" - ); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_forward_dwt53_dispatch_round_trips_gray8_lossless_tile() { - let pixels: Vec = (0..8 * 8).map(|i| ((i * 5) & 0xFF) as u8).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let mut accelerator = MetalEncodeStageAccelerator::default(); - - let codestream = - encode_with_accelerator(&pixels, 8, 8, 1, 8, false, &options, &mut accelerator) - .expect("encode with metal forward DWT 5/3"); - let decoded = Image::new(&codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - - assert_eq!(decoded.data, pixels); - assert_eq!(accelerator.forward_dwt53_attempts(), 1); - assert_eq!(accelerator.forward_dwt53_dispatches(), 1); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_lossless_facade_dispatches_rct_and_dwt_for_wsi_sized_rgb_tile() { - let mut pixels = Vec::with_capacity(128 * 128 * 3); - for y in 0..128u32 { - for x in 0..128u32 { - pixels.push(((x * 3 + y * 5) & 0xFF) as u8); - pixels.push(((x * 7 + y * 11) & 0xFF) as u8); - pixels.push(((x * 13 + y * 17) & 0xFF) as u8); - } - } - let samples = - J2kLosslessSamples::new(&pixels, 128, 128, 3, 8, false).expect("valid RGB samples"); - let mut accelerator = MetalEncodeStageAccelerator::default(); - - let encoded = encode_j2k_lossless_with_accelerator( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::PreferDevice, - ..J2kLosslessEncodeOptions::default() - }, - BackendKind::Metal, - &mut accelerator, - ) - .expect("Metal-accelerated lossless encode"); - - assert_eq!(encoded.backend, BackendKind::Metal); - assert_eq!(accelerator.forward_rct_dispatches(), 1); - assert_eq!(accelerator.forward_dwt53_dispatches(), 3); - assert!(accelerator.tier1_code_block_attempts() > 0); - assert_eq!(accelerator.packetization_attempts(), 1); - assert!(accelerator.tier1_code_block_dispatches() > 0); - assert_eq!(accelerator.packetization_dispatches(), 1); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_classic_tier1_uses_one_batched_dispatch_for_multiple_code_blocks() { - let pixels: Vec = (0..256 * 256) - .map(|idx| ((idx * 17 + 3) & 0xFF) as u8) - .collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 0, - ..EncodeOptions::default() - }; - let mut accelerator = MetalEncodeStageAccelerator::default(); - - let codestream = - encode_with_accelerator(&pixels, 256, 256, 1, 8, false, &options, &mut accelerator) - .expect("encode with batched Metal classic Tier-1"); - let decoded = Image::new(&codestream, &DecodeSettings::default()) - .expect("codestream parses") - .decode_native() - .expect("codestream decodes"); - - assert_eq!(decoded.data, pixels); - assert!(accelerator.tier1_code_block_attempts() > 1); - assert_eq!(accelerator.tier1_code_block_dispatches(), 1); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_htj2k_uses_one_batched_dispatch_for_multiple_code_blocks() { - let pixels: Vec = (0..256 * 256) - .map(|idx| ((idx * 23 + 9) & 0xFF) as u8) - .collect(); - let samples = - J2kLosslessSamples::new(&pixels, 256, 256, 1, 8, false).expect("valid gray samples"); - let mut accelerator = MetalEncodeStageAccelerator::default(); - - let encoded = encode_j2k_lossless_with_accelerator( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - block_coding_mode: J2kBlockCodingMode::HighThroughput, - ..J2kLosslessEncodeOptions::default() - }, - BackendKind::Metal, - &mut accelerator, - ) - .expect("Metal-accelerated HTJ2K lossless encode"); - - assert_eq!(encoded.backend, BackendKind::Metal); - assert!(accelerator.ht_code_block_attempts() > 1); - assert_eq!(accelerator.ht_code_block_dispatches(), 1); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_htj2k_lossless_facade_dispatches_ht_code_blocks_and_packetization() { - let pixels: Vec = (0..64).map(|value| ((value * 13) & 0xFF) as u8).collect(); - let samples = - J2kLosslessSamples::new(&pixels, 8, 8, 1, 8, false).expect("valid gray samples"); - let mut accelerator = MetalEncodeStageAccelerator::default(); - - let encoded = encode_j2k_lossless_with_accelerator( - samples, - &J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::RequireDevice, - block_coding_mode: J2kBlockCodingMode::HighThroughput, - ..J2kLosslessEncodeOptions::default() - }, - BackendKind::Metal, - &mut accelerator, - ) - .expect("Metal-accelerated HTJ2K lossless encode"); - - assert_eq!(encoded.backend, BackendKind::Metal); - assert!(accelerator.ht_code_block_attempts() > 0); - assert!(accelerator.ht_code_block_dispatches() > 0); - assert_eq!(accelerator.packetization_attempts(), 1); - assert_eq!(accelerator.packetization_dispatches(), 1); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_classic_tier1_kernel_matches_scalar_oracle() { - let coeffs: Vec = (0..64) - .map(|idx| { - let value = ((idx * 37 + 11) & 0x1ff) - 255; - if idx % 5 == 0 { - 0 - } else { - value - } - }) - .collect(); - let style = J2kCodeBlockStyle { - selective_arithmetic_coding_bypass: false, - reset_context_probabilities: false, - termination_on_each_pass: false, - vertically_causal_context: false, - segmentation_symbols: false, - }; - let job = signinum_j2k_native::J2kTier1CodeBlockEncodeJob { - coefficients: &coeffs, - width: 8, - height: 8, - sub_band_type: signinum_j2k_native::J2kSubBandType::HighHigh, - total_bitplanes: 9, - style, - }; - - let gpu = compute::encode_classic_tier1_code_block(job).expect("Metal classic encode"); - let cpu = signinum_j2k_native::encode_j2k_code_block_scalar_with_style( - &coeffs, - 8, - 8, - signinum_j2k_native::J2kSubBandType::HighHigh, - 9, - style, - ) - .expect("scalar classic encode"); - - assert_eq!(gpu.data, cpu.data); - assert_eq!(gpu.segments.len(), cpu.segments.len()); - for (gpu_segment, cpu_segment) in gpu.segments.iter().zip(cpu.segments.iter()) { - assert_eq!(gpu_segment.data_offset, cpu_segment.data_offset); - assert_eq!(gpu_segment.data_length, cpu_segment.data_length); - assert_eq!(gpu_segment.start_coding_pass, cpu_segment.start_coding_pass); - assert_eq!(gpu_segment.end_coding_pass, cpu_segment.end_coding_pass); - assert_eq!(gpu_segment.use_arithmetic, cpu_segment.use_arithmetic); - } - assert_eq!(gpu.number_of_coding_passes, cpu.number_of_coding_passes); - assert_eq!(gpu.missing_bit_planes, cpu.missing_bit_planes); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_classic_tier1_kernel_matches_scalar_for_terminated_passes() { - let coeffs: Vec = (0..64) - .map(|idx| { - let value = ((idx * 43 + 5) & 0x3ff) - 511; - if idx % 6 == 0 { - 0 - } else { - value - } - }) - .collect(); - let style = J2kCodeBlockStyle { - selective_arithmetic_coding_bypass: false, - reset_context_probabilities: true, - termination_on_each_pass: true, - vertically_causal_context: false, - segmentation_symbols: true, - }; - let job = signinum_j2k_native::J2kTier1CodeBlockEncodeJob { - coefficients: &coeffs, - width: 8, - height: 8, - sub_band_type: signinum_j2k_native::J2kSubBandType::LowHigh, - total_bitplanes: 10, - style, - }; - - let gpu = - compute::encode_classic_tier1_code_block(job).expect("Metal classic terminated encode"); - let cpu = signinum_j2k_native::encode_j2k_code_block_scalar_with_style( - &coeffs, - 8, - 8, - signinum_j2k_native::J2kSubBandType::LowHigh, - 10, - style, - ) - .expect("scalar classic terminated encode"); - - assert_eq!(gpu.data, cpu.data); - assert_eq!(gpu.segments.len(), cpu.segments.len()); - for (gpu_segment, cpu_segment) in gpu.segments.iter().zip(cpu.segments.iter()) { - assert_eq!(gpu_segment.data_offset, cpu_segment.data_offset); - assert_eq!(gpu_segment.data_length, cpu_segment.data_length); - assert_eq!(gpu_segment.start_coding_pass, cpu_segment.start_coding_pass); - assert_eq!(gpu_segment.end_coding_pass, cpu_segment.end_coding_pass); - assert_eq!(gpu_segment.use_arithmetic, cpu_segment.use_arithmetic); - } - assert_eq!(gpu.number_of_coding_passes, cpu.number_of_coding_passes); - assert_eq!(gpu.missing_bit_planes, cpu.missing_bit_planes); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_classic_tier1_kernel_matches_scalar_for_selective_bypass() { - let coeffs: Vec = (0..64) - .map(|idx| { - let value = ((idx * 61 + 29) & 0x7ff) - 1023; - if idx % 4 == 0 { - 0 - } else { - value - } - }) - .collect(); - let style = J2kCodeBlockStyle { - selective_arithmetic_coding_bypass: true, - reset_context_probabilities: false, - termination_on_each_pass: false, - vertically_causal_context: false, - segmentation_symbols: false, - }; - let job = signinum_j2k_native::J2kTier1CodeBlockEncodeJob { - coefficients: &coeffs, - width: 8, - height: 8, - sub_band_type: signinum_j2k_native::J2kSubBandType::HighLow, - total_bitplanes: 11, - style, - }; - - let gpu = - compute::encode_classic_tier1_code_block(job).expect("Metal classic bypass encode"); - let cpu = signinum_j2k_native::encode_j2k_code_block_scalar_with_style( - &coeffs, - 8, - 8, - signinum_j2k_native::J2kSubBandType::HighLow, - 11, - style, - ) - .expect("scalar classic bypass encode"); - - assert_eq!(gpu.data, cpu.data); - assert_eq!(gpu.segments.len(), cpu.segments.len()); - for (gpu_segment, cpu_segment) in gpu.segments.iter().zip(cpu.segments.iter()) { - assert_eq!(gpu_segment.data_offset, cpu_segment.data_offset); - assert_eq!(gpu_segment.data_length, cpu_segment.data_length); - assert_eq!(gpu_segment.start_coding_pass, cpu_segment.start_coding_pass); - assert_eq!(gpu_segment.end_coding_pass, cpu_segment.end_coding_pass); - assert_eq!(gpu_segment.use_arithmetic, cpu_segment.use_arithmetic); - } - assert_eq!(gpu.number_of_coding_passes, cpu.number_of_coding_passes); - assert_eq!(gpu.missing_bit_planes, cpu.missing_bit_planes); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_htj2k_cleanup_kernel_matches_scalar_oracle() { - let coeffs: Vec = (0..64) - .map(|idx| { - let value = ((idx * 19 + 7) & 0xff) - 127; - if idx % 7 == 0 { - 0 - } else { - value - } - }) - .collect(); - let job = signinum_j2k_native::J2kHtCodeBlockEncodeJob { - coefficients: &coeffs, - width: 8, - height: 8, - total_bitplanes: 8, - }; - - let gpu = compute::encode_ht_cleanup_code_block(job).expect("Metal HT encode"); - let cpu = signinum_j2k_native::encode_ht_code_block_scalar(&coeffs, 8, 8, 8) - .expect("scalar HT encode"); - - assert_eq!(gpu.data, cpu.data); - assert_eq!(gpu.num_coding_passes, cpu.num_coding_passes); - assert_eq!(gpu.num_zero_bitplanes, cpu.num_zero_bitplanes); - } - - #[cfg(target_os = "macos")] - #[test] - fn ht_simd_prototype_matches_scalar_for_64x64_block() { - if !compute::ht_simd_prototype_available_for_test() - .expect("HTJ2K SIMD prototype availability query") - { - return; - } - let coeffs: Vec = (0..4096) - .map(|idx| { - let value = ((idx * 37 + idx / 11 + 13) & 0xff) - 127; - if idx % 17 == 0 || idx % 29 == 0 { - 0 - } else { - value - } - }) - .collect(); - let job = signinum_j2k_native::J2kHtCodeBlockEncodeJob { - coefficients: &coeffs, - width: 64, - height: 64, - total_bitplanes: 8, - }; - - let scalar = { - let _route = compute::force_ht_simd_prototype_route_for_test(false); - compute::encode_ht_cleanup_code_blocks(&[job]) - .expect("scalar Metal HT encode") - .remove(0) - }; - let simd = compute::encode_ht_cleanup_code_blocks_simd_prototype_for_test(&[job]) - .expect("SIMD prototype Metal HT encode") - .remove(0); - - assert_eq!(simd.data, scalar.data); - assert_eq!(simd.num_coding_passes, scalar.num_coding_passes); - assert_eq!(simd.num_zero_bitplanes, scalar.num_zero_bitplanes); - } - - #[cfg(target_os = "macos")] - #[test] - fn ht_simd_prototype_matches_scalar_for_mixed_block_batch() { - if !compute::ht_simd_prototype_available_for_test() - .expect("HTJ2K SIMD prototype availability query") - { - return; - } - let all_zero = vec![0; 64]; - let non_square: Vec = (0..512) - .map(|idx| { - let value = (idx * 23 + 9) & 0x7f; - if idx % 5 == 0 { - -value - } else { - value - } - }) - .collect(); - let bitplane_edge: Vec = (0..256) - .map(|idx| match idx % 4 { - 0 => 255, - 1 => -255, - 2 => 1, - _ => 0, - }) - .collect(); - let wide: Vec = (0..4096) - .map(|idx| { - let value = ((idx * 41 + idx / 7 + 3) & 0xff) - 128; - if idx % 31 == 0 { - 0 - } else { - value - } - }) - .collect(); - let jobs = [ - signinum_j2k_native::J2kHtCodeBlockEncodeJob { - coefficients: &all_zero, - width: 8, - height: 8, - total_bitplanes: 8, - }, - signinum_j2k_native::J2kHtCodeBlockEncodeJob { - coefficients: &non_square, - width: 16, - height: 32, - total_bitplanes: 8, - }, - signinum_j2k_native::J2kHtCodeBlockEncodeJob { - coefficients: &bitplane_edge, - width: 16, - height: 16, - total_bitplanes: 8, - }, - signinum_j2k_native::J2kHtCodeBlockEncodeJob { - coefficients: &wide, - width: 64, - height: 64, - total_bitplanes: 8, - }, - ]; - - let scalar = { - let _route = compute::force_ht_simd_prototype_route_for_test(false); - compute::encode_ht_cleanup_code_blocks(&jobs).expect("scalar Metal HT batch") - }; - let simd = compute::encode_ht_cleanup_code_blocks_simd_prototype_for_test(&jobs) - .expect("SIMD prototype Metal HT batch"); - - assert_eq!(simd.len(), scalar.len()); - for (simd_block, scalar_block) in simd.iter().zip(scalar.iter()) { - assert_eq!(simd_block.data, scalar_block.data); - assert_eq!(simd_block.num_coding_passes, scalar_block.num_coding_passes); - assert_eq!( - simd_block.num_zero_bitplanes, - scalar_block.num_zero_bitplanes - ); - } - } - - #[cfg(target_os = "macos")] - #[test] - fn ht_simd_prototype_length_estimate_matches_scalar() { - if !compute::ht_simd_prototype_available_for_test() - .expect("HTJ2K SIMD prototype availability query") - { - return; - } - let all_zero = vec![0; 64]; - let patterned: Vec = (0..4096) - .map(|idx| { - let value = ((idx * 53 + idx / 13 + 21) & 0xff) - 127; - if idx % 19 == 0 || idx % 37 == 0 { - 0 - } else { - value - } - }) - .collect(); - let jobs = [ - signinum_j2k_native::J2kHtCodeBlockEncodeJob { - coefficients: &all_zero, - width: 8, - height: 8, - total_bitplanes: 8, - }, - signinum_j2k_native::J2kHtCodeBlockEncodeJob { - coefficients: &patterned, - width: 64, - height: 64, - total_bitplanes: 8, - }, - ]; - - let scalar = - compute::encode_ht_cleanup_code_blocks_with_segment_lengths_for_test(&jobs, false) - .expect("scalar Metal HT segment lengths"); - let simd = - compute::encode_ht_cleanup_code_blocks_with_segment_lengths_for_test(&jobs, true) - .expect("SIMD prototype Metal HT segment lengths"); - - assert_eq!(simd.len(), scalar.len()); - for ((simd_block, simd_lengths), (scalar_block, scalar_lengths)) in - simd.iter().zip(scalar.iter()) - { - assert_eq!(simd_block.data, scalar_block.data); - assert_eq!(simd_block.num_coding_passes, scalar_block.num_coding_passes); - assert_eq!( - simd_block.num_zero_bitplanes, - scalar_block.num_zero_bitplanes - ); - assert_eq!(simd_lengths, scalar_lengths); - assert_eq!( - simd_lengths.magnitude_sign + simd_lengths.mel + simd_lengths.vlc, - u32::try_from(simd_block.data.len()).expect("HT data length fits u32") - ); - } - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_tier2_packetization_kernel_matches_scalar_oracle() { - let block0 = [0x12, 0x34, 0x56, 0x78]; - let block1 = [0x9a, 0xbc]; - let code_blocks = vec![ - signinum_j2k_native::J2kPacketizationCodeBlock { - data: &block0, - num_coding_passes: 1, - num_zero_bitplanes: 2, - previously_included: false, - l_block: 3, - block_coding_mode: signinum_j2k_native::J2kPacketizationBlockCodingMode::Classic, - }, - signinum_j2k_native::J2kPacketizationCodeBlock { - data: &block1, - num_coding_passes: 1, - num_zero_bitplanes: 1, - previously_included: false, - l_block: 3, - block_coding_mode: - signinum_j2k_native::J2kPacketizationBlockCodingMode::HighThroughput, - }, - ]; - let subband = signinum_j2k_native::J2kPacketizationSubband { - code_blocks, - num_cbs_x: 2, - num_cbs_y: 1, - }; - let resolution = signinum_j2k_native::J2kPacketizationResolution { - subbands: vec![subband], - }; - let resolutions = [resolution]; - let packet_descriptors = [signinum_j2k_native::J2kPacketizationPacketDescriptor { - packet_index: 0, - state_index: 0, - layer: 0, - resolution: 0, - component: 0, - precinct: 0, - }]; - let job = signinum_j2k_native::J2kPacketizationEncodeJob { - resolution_count: 1, - num_layers: 1, - num_components: 1, - code_block_count: 2, - progression_order: signinum_j2k_native::J2kPacketizationProgressionOrder::Lrcp, - packet_descriptors: &packet_descriptors, - resolutions: &resolutions, - }; - - let gpu = compute::encode_tier2_packetization(job).expect("Metal packet encode"); - let cpu = signinum_j2k_native::encode_j2k_packetization_scalar(job) - .expect("scalar packet encode"); - - assert_eq!(gpu, cpu); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_tier2_packetization_reuses_descriptor_state_across_layers() { - let block0 = vec![0x11]; - let block1 = vec![0x22]; - let first = signinum_j2k_native::J2kPacketizationResolution { - subbands: vec![signinum_j2k_native::J2kPacketizationSubband { - code_blocks: vec![signinum_j2k_native::J2kPacketizationCodeBlock { - data: &block0, - num_coding_passes: 1, - num_zero_bitplanes: 0, - previously_included: false, - l_block: 3, - block_coding_mode: - signinum_j2k_native::J2kPacketizationBlockCodingMode::Classic, - }], - num_cbs_x: 1, - num_cbs_y: 1, - }], - }; - let second = signinum_j2k_native::J2kPacketizationResolution { - subbands: vec![signinum_j2k_native::J2kPacketizationSubband { - code_blocks: vec![signinum_j2k_native::J2kPacketizationCodeBlock { - data: &block1, - num_coding_passes: 1, - num_zero_bitplanes: 0, - previously_included: false, - l_block: 3, - block_coding_mode: - signinum_j2k_native::J2kPacketizationBlockCodingMode::Classic, - }], - num_cbs_x: 1, - num_cbs_y: 1, - }], - }; - let resolutions = [first, second]; - let packet_descriptors = [ - signinum_j2k_native::J2kPacketizationPacketDescriptor { - packet_index: 0, - state_index: 0, - layer: 0, - resolution: 0, - component: 0, - precinct: 0, - }, - signinum_j2k_native::J2kPacketizationPacketDescriptor { - packet_index: 1, - state_index: 0, - layer: 1, - resolution: 0, - component: 0, - precinct: 0, - }, - ]; - let job = signinum_j2k_native::J2kPacketizationEncodeJob { - resolution_count: 2, - num_layers: 2, - num_components: 1, - code_block_count: 2, - progression_order: signinum_j2k_native::J2kPacketizationProgressionOrder::Rpcl, - packet_descriptors: &packet_descriptors, - resolutions: &resolutions, - }; - - let gpu = compute::encode_tier2_packetization(job).expect("Metal packet encode"); - let cpu = signinum_j2k_native::encode_j2k_packetization_scalar(job) - .expect("scalar packet encode"); - - assert_eq!(gpu, cpu); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_tier2_packetization_honors_explicit_descriptor_order() { - let block0 = vec![0xA0]; - let block1 = vec![0xB0]; - let first = signinum_j2k_native::J2kPacketizationResolution { - subbands: vec![signinum_j2k_native::J2kPacketizationSubband { - code_blocks: vec![signinum_j2k_native::J2kPacketizationCodeBlock { - data: &block0, - num_coding_passes: 1, - num_zero_bitplanes: 0, - previously_included: false, - l_block: 3, - block_coding_mode: - signinum_j2k_native::J2kPacketizationBlockCodingMode::Classic, - }], - num_cbs_x: 1, - num_cbs_y: 1, - }], - }; - let second = signinum_j2k_native::J2kPacketizationResolution { - subbands: vec![signinum_j2k_native::J2kPacketizationSubband { - code_blocks: vec![signinum_j2k_native::J2kPacketizationCodeBlock { - data: &block1, - num_coding_passes: 1, - num_zero_bitplanes: 0, - previously_included: false, - l_block: 3, - block_coding_mode: - signinum_j2k_native::J2kPacketizationBlockCodingMode::Classic, - }], - num_cbs_x: 1, - num_cbs_y: 1, - }], - }; - let resolutions = [first, second]; - let packet_descriptors = [ - signinum_j2k_native::J2kPacketizationPacketDescriptor { - packet_index: 1, - state_index: 1, - layer: 0, - resolution: 1, - component: 0, - precinct: 0, - }, - signinum_j2k_native::J2kPacketizationPacketDescriptor { - packet_index: 0, - state_index: 0, - layer: 0, - resolution: 0, - component: 0, - precinct: 0, - }, - ]; - let job = signinum_j2k_native::J2kPacketizationEncodeJob { - resolution_count: 2, - num_layers: 1, - num_components: 1, - code_block_count: 2, - progression_order: signinum_j2k_native::J2kPacketizationProgressionOrder::Rpcl, - packet_descriptors: &packet_descriptors, - resolutions: &resolutions, - }; - - let gpu = compute::encode_tier2_packetization(job).expect("Metal packet encode"); - let cpu = signinum_j2k_native::encode_j2k_packetization_scalar(job) - .expect("scalar packet encode"); - - assert_eq!(gpu, cpu); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_forward_dwt53_handles_single_sample_edge_dimensions() { - for (width, height) in [(1, 8), (8, 1)] { - let samples: Vec = (0..width * height) - .map(|i| { - f32::from( - u8::try_from((i * 11 + width * 3 + height * 5) & 0xFF) - .expect("masked sample fits in u8"), - ) - 128.0 - }) - .collect(); - let mut accelerator = MetalEncodeStageAccelerator::default(); - - let output = accelerator - .encode_forward_dwt53(J2kForwardDwt53Job { - samples: &samples, - width, - height, - num_levels: 1, - }) - .expect("metal DWT 5/3 stage") - .expect("metal DWT 5/3 dispatch"); - - assert_eq!(output.ll_width, width.div_ceil(2)); - assert_eq!(output.ll_height, height.div_ceil(2)); - assert_eq!(output.levels.len(), 1); - assert_eq!(accelerator.forward_dwt53_attempts(), 1); - assert_eq!(accelerator.forward_dwt53_dispatches(), 1); - } - } -} diff --git a/crates/signinum-j2k-metal/src/encode_bitstream.metal b/crates/signinum-j2k-metal/src/encode_bitstream.metal deleted file mode 100644 index 07c6a4d0..00000000 --- a/crates/signinum-j2k-metal/src/encode_bitstream.metal +++ /dev/null @@ -1,3426 +0,0 @@ -#include -using namespace metal; - -constant uint J2K_ENCODE_STATUS_OK = 0u; -constant uint J2K_ENCODE_STATUS_FAIL = 1u; -constant uint J2K_ENCODE_STATUS_UNSUPPORTED = 2u; - -constant uchar J2K_ENCODE_SIGNIFICANT = uchar(1u << 7u); -constant uchar J2K_ENCODE_MAGNITUDE_REFINED = uchar(1u << 6u); -constant uchar J2K_ENCODE_SIGN = uchar(1u << 5u); - -struct J2kClassicEncodeParams { - uint width; - uint height; - uint sub_band_type; - uint total_bitplanes; - uint style_flags; - uint output_capacity; - uint segment_capacity; -}; - -struct J2kClassicEncodeStatus { - uint code; - uint detail; - uint data_len; - uint number_of_coding_passes; - uint missing_bit_planes; - uint segment_count; - uint reserved0; - uint reserved1; -}; - -struct J2kMqEncoder { - device uchar *data; - uint max_len; - uint len; - uint a; - uint c; - uint ct; - uint failed; -}; - -struct J2kRawBitWriter { - device uchar *data; - uint max_len; - uint len; - uint buffer; - uint bits_in_buffer; - uint last_byte_was_ff; - uint failed; -}; - -inline void j2k_set_encode_status( - device J2kClassicEncodeStatus *status, - uint code, - uint detail, - uint data_len, - uint passes, - uint missing, - uint segments -) { - status->code = code; - status->detail = detail; - status->data_len = data_len; - status->number_of_coding_passes = passes; - status->missing_bit_planes = missing; - status->segment_count = segments; - status->reserved0 = 0u; - status->reserved1 = 0u; -} - -inline void j2k_mq_init(thread J2kMqEncoder &encoder, device uchar *out, uint capacity) { - encoder.data = out; - encoder.max_len = capacity; - encoder.len = 0u; - encoder.a = 0x8000u; - encoder.c = 0u; - encoder.ct = 12u; - encoder.failed = 0u; - if (capacity == 0u) { - encoder.failed = 1u; - return; - } - encoder.data[0] = uchar(0); - encoder.len = 1u; -} - -inline void j2k_mq_push(thread J2kMqEncoder &encoder, uchar value) { - if (encoder.len >= encoder.max_len) { - encoder.failed = 1u; - return; - } - encoder.data[encoder.len] = value; - encoder.len += 1u; -} - -inline void j2k_mq_byte_out(thread J2kMqEncoder &encoder) { - if (encoder.failed != 0u || encoder.len == 0u) { - encoder.failed = 1u; - return; - } - - uchar last_byte = encoder.data[encoder.len - 1u]; - if (last_byte == uchar(0xFFu)) { - const uchar b = uchar(encoder.c >> 20u); - j2k_mq_push(encoder, b); - encoder.c &= 0xFFFFFu; - encoder.ct = 7u; - } else if ((encoder.c & 0x8000000u) == 0u) { - const uchar b = uchar(encoder.c >> 19u); - j2k_mq_push(encoder, b); - encoder.c &= 0x7FFFFu; - encoder.ct = 8u; - } else { - encoder.data[encoder.len - 1u] = uchar(encoder.data[encoder.len - 1u] + uchar(1u)); - encoder.c &= 0x7FFFFFFu; - if (encoder.data[encoder.len - 1u] == uchar(0xFFu)) { - const uchar b = uchar(encoder.c >> 20u); - j2k_mq_push(encoder, b); - encoder.c &= 0xFFFFFu; - encoder.ct = 7u; - } else { - const uchar b = uchar(encoder.c >> 19u); - j2k_mq_push(encoder, b); - encoder.c &= 0x7FFFFu; - encoder.ct = 8u; - } - } -} - -inline void j2k_mq_renormalize(thread J2kMqEncoder &encoder) { - do { - encoder.a <<= 1u; - encoder.c <<= 1u; - encoder.ct -= 1u; - if (encoder.ct == 0u) { - j2k_mq_byte_out(encoder); - } - } while ((encoder.a & 0x8000u) == 0u && encoder.failed == 0u); -} - -inline void j2k_mq_encode( - thread J2kMqEncoder &encoder, - thread uchar *contexts, - uint ctx_label, - uint bit -) { - uchar ctx = contexts[ctx_label]; - const J2kQeData qe = J2K_QE_TABLE[ctx & uchar(0x7Fu)]; - const uint mps = uint(ctx >> 7u); - encoder.a -= qe.qe; - - if (bit == mps) { - if ((encoder.a & 0x8000u) != 0u) { - encoder.c += qe.qe; - return; - } - if (encoder.a < qe.qe) { - encoder.a = qe.qe; - } else { - encoder.c += qe.qe; - } - ctx = uchar((ctx & uchar(0x80u)) | qe.nmps); - } else { - if (encoder.a < qe.qe) { - encoder.c += qe.qe; - } else { - encoder.a = qe.qe; - } - if (qe.switch_mps != 0u) { - ctx ^= uchar(0x80u); - } - ctx = uchar((ctx & uchar(0x80u)) | qe.nlps); - } - - contexts[ctx_label] = ctx; - j2k_mq_renormalize(encoder); -} - -inline void j2k_mq_set_bits(thread J2kMqEncoder &encoder) { - const uint temp = encoder.c + encoder.a; - encoder.c |= 0xFFFFu; - if (encoder.c >= temp) { - encoder.c -= 0x8000u; - } -} - -inline void j2k_mq_finish(thread J2kMqEncoder &encoder) { - j2k_mq_set_bits(encoder); - encoder.c <<= encoder.ct; - j2k_mq_byte_out(encoder); - encoder.c <<= encoder.ct; - j2k_mq_byte_out(encoder); -} - -inline void j2k_raw_writer_init(thread J2kRawBitWriter &writer, device uchar *out, uint capacity) { - writer.data = out; - writer.max_len = capacity; - writer.len = 0u; - writer.buffer = 0u; - writer.bits_in_buffer = 0u; - writer.last_byte_was_ff = 0u; - writer.failed = 0u; -} - -inline void j2k_raw_writer_push(thread J2kRawBitWriter &writer, uchar value) { - if (writer.len >= writer.max_len) { - writer.failed = 1u; - return; - } - writer.data[writer.len] = value; - writer.len += 1u; - writer.last_byte_was_ff = value == uchar(0xFFu) ? 1u : 0u; -} - -inline void j2k_raw_writer_flush_byte(thread J2kRawBitWriter &writer) { - const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; - const uchar byte = uchar(writer.buffer >> (writer.bits_in_buffer - limit)); - j2k_raw_writer_push(writer, byte); - writer.bits_in_buffer -= limit; - writer.buffer &= writer.bits_in_buffer == 0u ? 0u : ((1u << writer.bits_in_buffer) - 1u); -} - -inline void j2k_raw_writer_write_bit(thread J2kRawBitWriter &writer, uint bit) { - writer.buffer = (writer.buffer << 1u) | (bit & 1u); - writer.bits_in_buffer += 1u; - const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; - if (writer.bits_in_buffer >= limit) { - j2k_raw_writer_flush_byte(writer); - } -} - -inline void j2k_raw_writer_finish(thread J2kRawBitWriter &writer) { - if (writer.bits_in_buffer == 0u) { - return; - } - const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; - const uint shift = limit - writer.bits_in_buffer; - j2k_raw_writer_push(writer, uchar(writer.buffer << shift)); - writer.buffer = 0u; - writer.bits_in_buffer = 0u; -} - -inline uint j2k_classic_magnitude(int value) { - return value < 0 ? uint(-value) : uint(value); -} - -inline uchar j2k_classic_effective_neighbors( - thread const uchar *states, - uint padded_width, - uint index_x, - uint index_y, - uint height, - uint style_flags -) { - return effective_neighborhood_states(states, padded_width, index_x, index_y, height, style_flags); -} - -inline void j2k_classic_encode_sign( - uint idx, - thread const uchar *states, - thread J2kMqEncoder &encoder, - thread uchar *contexts, - uint padded_width, - uint index_x, - uint index_y, - uint height, - uint style_flags -) { - const uchar2 sign_ctx = sign_context( - states, - padded_width, - index_x, - index_y, - height, - style_flags - ); - const uint sign_bit = (uint(states[idx]) >> 5u) & 1u; - j2k_mq_encode(encoder, contexts, uint(sign_ctx.x), sign_bit ^ uint(sign_ctx.y)); -} - -inline void j2k_classic_significance_pass( - thread const uint *magnitudes, - thread uchar *states, - thread uchar *coded, - thread J2kMqEncoder &encoder, - thread uchar *contexts, - uint width, - uint height, - uint padded_width, - uint bit_mask, - uint sub_band_type, - uint style_flags -) { - for (uint y_base = 0u; y_base < height; y_base += 4u) { - for (uint x = 0u; x < width; ++x) { - const uint y_end = min(y_base + 4u, height); - for (uint y = y_base; y < y_end; ++y) { - const uint ix = x + 1u; - const uint iy = y + 1u; - const uint idx = coeff_index(padded_width, ix, iy); - const uchar neighbor_sig = - j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); - if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && neighbor_sig != 0u) { - const uint ctx_label = uint(zero_context_label(neighbor_sig, sub_band_type)); - const uint bit = (magnitudes[idx] & bit_mask) != 0u ? 1u : 0u; - j2k_mq_encode(encoder, contexts, ctx_label, bit); - coded[idx] = uchar(1u); - if (bit != 0u) { - j2k_classic_encode_sign( - idx, - states, - encoder, - contexts, - padded_width, - ix, - iy, - height, - style_flags - ); - set_significant(states, padded_width, ix, iy); - } - } - } - } - } -} - -inline void j2k_classic_magnitude_refinement_pass( - thread const uint *magnitudes, - thread uchar *states, - thread uchar *coded, - thread J2kMqEncoder &encoder, - thread uchar *contexts, - uint width, - uint height, - uint padded_width, - uint bit_mask, - uint style_flags -) { - for (uint y_base = 0u; y_base < height; y_base += 4u) { - for (uint x = 0u; x < width; ++x) { - const uint y_end = min(y_base + 4u, height); - for (uint y = y_base; y < y_end; ++y) { - const uint ix = x + 1u; - const uint iy = y + 1u; - const uint idx = coeff_index(padded_width, ix, iy); - if ((states[idx] & J2K_ENCODE_SIGNIFICANT) != 0u && coded[idx] == 0u) { - const uint ctx_label = - uint(magnitude_refinement_context(states, padded_width, ix, iy, height, style_flags)); - const uint bit = (magnitudes[idx] & bit_mask) != 0u ? 1u : 0u; - j2k_mq_encode(encoder, contexts, ctx_label, bit); - states[idx] |= J2K_ENCODE_MAGNITUDE_REFINED; - } - } - } - } -} - -inline void j2k_classic_significance_pass_raw( - thread const uint *magnitudes, - thread uchar *states, - thread uchar *coded, - thread J2kRawBitWriter &writer, - uint width, - uint height, - uint padded_width, - uint bit_mask, - uint style_flags -) { - for (uint y_base = 0u; y_base < height; y_base += 4u) { - for (uint x = 0u; x < width; ++x) { - const uint y_end = min(y_base + 4u, height); - for (uint y = y_base; y < y_end; ++y) { - const uint ix = x + 1u; - const uint iy = y + 1u; - const uint idx = coeff_index(padded_width, ix, iy); - const uchar neighbor_sig = - j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); - if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && neighbor_sig != 0u) { - const uint bit = (magnitudes[idx] & bit_mask) != 0u ? 1u : 0u; - j2k_raw_writer_write_bit(writer, bit); - coded[idx] = uchar(1u); - if (bit != 0u) { - j2k_raw_writer_write_bit(writer, (uint(states[idx]) >> 5u) & 1u); - set_significant(states, padded_width, ix, iy); - } - } - } - } - } -} - -inline void j2k_classic_magnitude_refinement_pass_raw( - thread const uint *magnitudes, - thread uchar *states, - thread uchar *coded, - thread J2kRawBitWriter &writer, - uint width, - uint height, - uint padded_width, - uint bit_mask -) { - for (uint y_base = 0u; y_base < height; y_base += 4u) { - for (uint x = 0u; x < width; ++x) { - const uint y_end = min(y_base + 4u, height); - for (uint y = y_base; y < y_end; ++y) { - const uint ix = x + 1u; - const uint iy = y + 1u; - const uint idx = coeff_index(padded_width, ix, iy); - if ((states[idx] & J2K_ENCODE_SIGNIFICANT) != 0u && coded[idx] == 0u) { - const uint bit = (magnitudes[idx] & bit_mask) != 0u ? 1u : 0u; - j2k_raw_writer_write_bit(writer, bit); - states[idx] |= J2K_ENCODE_MAGNITUDE_REFINED; - } - } - } - } -} - -inline void j2k_classic_cleanup_pass( - thread const uint *magnitudes, - thread uchar *states, - thread uchar *coded, - thread J2kMqEncoder &encoder, - thread uchar *contexts, - uint width, - uint height, - uint padded_width, - uint bit_mask, - uint sub_band_type, - uint style_flags -) { - for (uint y_base = 0u; y_base < height; y_base += 4u) { - for (uint x = 0u; x < width; ++x) { - const uint y_end = min(y_base + 4u, height); - const uint stripe_height = y_end - y_base; - - if (stripe_height == 4u) { - bool all_zero_uncoded = true; - for (uint y = y_base; y < y_end; ++y) { - const uint ix = x + 1u; - const uint iy = y + 1u; - const uint idx = coeff_index(padded_width, ix, iy); - const uchar neighbor_sig = - j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); - if ((states[idx] & J2K_ENCODE_SIGNIFICANT) != 0u || coded[idx] != 0u || neighbor_sig != 0u) { - all_zero_uncoded = false; - break; - } - } - - if (all_zero_uncoded) { - uint first_sig = 4u; - for (uint pos = 0u; pos < 4u; ++pos) { - const uint idx = coeff_index(padded_width, x + 1u, y_base + pos + 1u); - if ((magnitudes[idx] & bit_mask) != 0u) { - first_sig = pos; - break; - } - } - - if (first_sig < 4u) { - j2k_mq_encode(encoder, contexts, 17u, 1u); - j2k_mq_encode(encoder, contexts, 18u, (first_sig >> 1u) & 1u); - j2k_mq_encode(encoder, contexts, 18u, first_sig & 1u); - - const uint sig_y = y_base + first_sig; - const uint sig_idx = coeff_index(padded_width, x + 1u, sig_y + 1u); - j2k_classic_encode_sign( - sig_idx, - states, - encoder, - contexts, - padded_width, - x + 1u, - sig_y + 1u, - height, - style_flags - ); - set_significant(states, padded_width, x + 1u, sig_y + 1u); - - for (uint y = sig_y + 1u; y < y_end; ++y) { - const uint ix = x + 1u; - const uint iy = y + 1u; - const uint idx = coeff_index(padded_width, ix, iy); - if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && coded[idx] == 0u) { - const uchar neighbor_sig = - j2k_classic_effective_neighbors( - states, - padded_width, - ix, - iy, - height, - style_flags - ); - const uint ctx_label = uint(zero_context_label(neighbor_sig, sub_band_type)); - const uint bit = (magnitudes[idx] & bit_mask) != 0u ? 1u : 0u; - j2k_mq_encode(encoder, contexts, ctx_label, bit); - if (bit != 0u) { - j2k_classic_encode_sign( - idx, - states, - encoder, - contexts, - padded_width, - ix, - iy, - height, - style_flags - ); - set_significant(states, padded_width, ix, iy); - } - } - } - continue; - } - - j2k_mq_encode(encoder, contexts, 17u, 0u); - continue; - } - } - - for (uint y = y_base; y < y_end; ++y) { - const uint ix = x + 1u; - const uint iy = y + 1u; - const uint idx = coeff_index(padded_width, ix, iy); - if ((states[idx] & J2K_ENCODE_SIGNIFICANT) == 0u && coded[idx] == 0u) { - const uchar neighbor_sig = - j2k_classic_effective_neighbors(states, padded_width, ix, iy, height, style_flags); - const uint ctx_label = uint(zero_context_label(neighbor_sig, sub_band_type)); - const uint bit = (magnitudes[idx] & bit_mask) != 0u ? 1u : 0u; - j2k_mq_encode(encoder, contexts, ctx_label, bit); - if (bit != 0u) { - j2k_classic_encode_sign( - idx, - states, - encoder, - contexts, - padded_width, - ix, - iy, - height, - style_flags - ); - set_significant(states, padded_width, ix, iy); - } - } - } - } - } -} - -inline void j2k_classic_encode_segmentation_symbols( - thread J2kMqEncoder &encoder, - thread uchar *contexts -) { - j2k_mq_encode(encoder, contexts, 18u, 1u); - j2k_mq_encode(encoder, contexts, 18u, 0u); - j2k_mq_encode(encoder, contexts, 18u, 1u); - j2k_mq_encode(encoder, contexts, 18u, 0u); -} - -inline uint j2k_classic_bypass_segment_idx(uint pass_idx) { - if (pass_idx < 10u) { - return 0u; - } - return 1u + (2u * ((pass_idx - 10u) / 3u)) + (((pass_idx - 10u) % 3u) == 2u ? 1u : 0u); -} - -inline bool j2k_classic_push_segment( - device J2kClassicSegment *segments, - uint segment_capacity, - thread uint &segment_count, - uint data_offset, - uint data_length, - uint start_pass, - uint end_pass, - bool use_arithmetic -) { - if (segment_count >= segment_capacity) { - return false; - } - segments[segment_count].data_offset = data_offset; - segments[segment_count].data_length = data_length; - segments[segment_count].start_coding_pass = start_pass; - segments[segment_count].end_coding_pass = end_pass; - segments[segment_count].use_arithmetic = use_arithmetic ? 1u : 0u; - segment_count += 1u; - return true; -} - -inline uint j2k_classic_finish_arithmetic_segment(thread J2kMqEncoder &encoder) { - j2k_mq_finish(encoder); - if (encoder.failed != 0u || encoder.len == 0u) { - encoder.failed = 1u; - return 0u; - } - const uint data_len = encoder.len - 1u; - for (uint idx = 0u; idx < data_len; ++idx) { - encoder.data[idx] = encoder.data[idx + 1u]; - } - return data_len; -} - -inline void j2k_encode_classic_code_block_impl( - device const int *coefficients, - device uchar *out, - J2kClassicEncodeParams params, - device J2kClassicEncodeStatus *status, - device J2kClassicSegment *segments -) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u, 0u, 0u, 0u); - - if (params.width == 0u || params.height == 0u || - params.width > J2K_CLASSIC_MAX_WIDTH || - params.height > J2K_CLASSIC_MAX_HEIGHT || - params.total_bitplanes > 31u) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 1u, 0u, 0u, 0u, 0u); - return; - } - - const uint padded_width = params.width + 2u; - const uint padded_height = params.height + 2u; - const uint padded_count = padded_width * padded_height; - - thread uint magnitudes[J2K_CLASSIC_MAX_COEFF_COUNT]; - thread uchar states[J2K_CLASSIC_MAX_COEFF_COUNT]; - thread uchar coded[J2K_CLASSIC_MAX_COEFF_COUNT]; - - for (uint idx = 0u; idx < padded_count; ++idx) { - magnitudes[idx] = 0u; - states[idx] = uchar(0u); - coded[idx] = uchar(0u); - } - - uint max_magnitude = 0u; - for (uint y = 0u; y < params.height; ++y) { - for (uint x = 0u; x < params.width; ++x) { - const uint src_idx = y * params.width + x; - const int value = coefficients[src_idx]; - const uint dst_idx = coeff_index(padded_width, x + 1u, y + 1u); - const uint magnitude = j2k_classic_magnitude(value); - magnitudes[dst_idx] = magnitude; - if (value < 0) { - states[dst_idx] |= J2K_ENCODE_SIGN; - } - max_magnitude = max(max_magnitude, magnitude); - } - } - - if (max_magnitude == 0u) { - j2k_set_encode_status( - status, - J2K_ENCODE_STATUS_OK, - 0u, - 0u, - 0u, - params.total_bitplanes, - 0u - ); - return; - } - - const uint num_bitplanes = 32u - clz(max_magnitude); - if (num_bitplanes > params.total_bitplanes) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 3u, 0u, 0u, 0u, 0u); - return; - } - const uint missing_bit_planes = params.total_bitplanes - num_bitplanes; - - thread uchar contexts[19]; - reset_contexts(contexts); - - if ((params.style_flags & (J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS | - J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS)) != 0u) { - const uint total_passes = 1u + 3u * (num_bitplanes - 1u); - uint data_cursor = 0u; - uint segment_count = 0u; - uint current_segment_idx = 0xFFFFFFFFu; - uint current_segment_start_pass = 0u; - bool current_use_arithmetic = true; - bool have_segment = false; - thread J2kMqEncoder arithmetic_encoder; - thread J2kRawBitWriter raw_writer; - - for (uint coding_pass = 0u; coding_pass < total_passes; ++coding_pass) { - const uint segment_idx = - (params.style_flags & J2K_CLASSIC_STYLE_TERMINATION_ON_EACH_PASS) != 0u - ? coding_pass - : j2k_classic_bypass_segment_idx(coding_pass); - const bool use_arithmetic = - (params.style_flags & J2K_CLASSIC_STYLE_SELECTIVE_ARITHMETIC_CODING_BYPASS) == 0u || - coding_pass <= 9u || - (coding_pass % 3u) == 0u; - - if (!have_segment || current_segment_idx != segment_idx) { - if (have_segment) { - uint segment_len = 0u; - if (current_use_arithmetic) { - segment_len = j2k_classic_finish_arithmetic_segment(arithmetic_encoder); - if (arithmetic_encoder.failed != 0u) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 6u, 0u, 0u, 0u, 0u); - return; - } - } else { - j2k_raw_writer_finish(raw_writer); - if (raw_writer.failed != 0u) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 7u, 0u, 0u, 0u, 0u); - return; - } - segment_len = raw_writer.len; - } - if (!j2k_classic_push_segment( - segments, - params.segment_capacity, - segment_count, - data_cursor, - segment_len, - current_segment_start_pass, - coding_pass, - current_use_arithmetic)) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 8u, 0u, 0u, 0u, 0u); - return; - } - data_cursor += segment_len; - } - - current_segment_idx = segment_idx; - current_segment_start_pass = coding_pass; - current_use_arithmetic = use_arithmetic; - have_segment = true; - if (data_cursor > params.output_capacity) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 9u, 0u, 0u, 0u, 0u); - return; - } - const uint remaining_capacity = params.output_capacity - data_cursor; - if (use_arithmetic) { - j2k_mq_init(arithmetic_encoder, out + data_cursor, remaining_capacity); - } else { - j2k_raw_writer_init(raw_writer, out + data_cursor, remaining_capacity); - } - } - - const uint current_bitplane = (coding_pass + 2u) / 3u; - const uint bit_mask = 1u << (num_bitplanes - 1u - current_bitplane); - switch (coding_pass % 3u) { - case 0u: - j2k_classic_cleanup_pass( - magnitudes, - states, - coded, - arithmetic_encoder, - contexts, - params.width, - params.height, - padded_width, - bit_mask, - params.sub_band_type, - params.style_flags - ); - if ((params.style_flags & J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS) != 0u) { - j2k_classic_encode_segmentation_symbols(arithmetic_encoder, contexts); - } - for (uint idx = 0u; idx < padded_count; ++idx) { - coded[idx] = uchar(0u); - } - break; - case 1u: - if (use_arithmetic) { - j2k_classic_significance_pass( - magnitudes, - states, - coded, - arithmetic_encoder, - contexts, - params.width, - params.height, - padded_width, - bit_mask, - params.sub_band_type, - params.style_flags - ); - } else { - j2k_classic_significance_pass_raw( - magnitudes, - states, - coded, - raw_writer, - params.width, - params.height, - padded_width, - bit_mask, - params.style_flags - ); - } - break; - default: - if (use_arithmetic) { - j2k_classic_magnitude_refinement_pass( - magnitudes, - states, - coded, - arithmetic_encoder, - contexts, - params.width, - params.height, - padded_width, - bit_mask, - params.style_flags - ); - } else { - j2k_classic_magnitude_refinement_pass_raw( - magnitudes, - states, - coded, - raw_writer, - params.width, - params.height, - padded_width, - bit_mask - ); - } - break; - } - - if ((params.style_flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0u) { - reset_contexts(contexts); - } - const bool current_failed = use_arithmetic - ? arithmetic_encoder.failed != 0u - : raw_writer.failed != 0u; - if (current_failed) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 10u, 0u, 0u, 0u, 0u); - return; - } - } - - if (have_segment) { - uint segment_len = 0u; - if (current_use_arithmetic) { - segment_len = j2k_classic_finish_arithmetic_segment(arithmetic_encoder); - if (arithmetic_encoder.failed != 0u) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 11u, 0u, 0u, 0u, 0u); - return; - } - } else { - j2k_raw_writer_finish(raw_writer); - if (raw_writer.failed != 0u) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 12u, 0u, 0u, 0u, 0u); - return; - } - segment_len = raw_writer.len; - } - if (!j2k_classic_push_segment( - segments, - params.segment_capacity, - segment_count, - data_cursor, - segment_len, - current_segment_start_pass, - total_passes, - current_use_arithmetic)) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 13u, 0u, 0u, 0u, 0u); - return; - } - data_cursor += segment_len; - } - - j2k_set_encode_status( - status, - J2K_ENCODE_STATUS_OK, - 0u, - data_cursor, - total_passes, - missing_bit_planes, - segment_count - ); - return; - } - - thread J2kMqEncoder encoder; - j2k_mq_init(encoder, out, params.output_capacity); - - uint pass_count = 0u; - for (int bp = int(num_bitplanes) - 1; bp >= 0; --bp) { - const uint bit_mask = 1u << uint(bp); - const bool first_bitplane = uint(bp) == num_bitplanes - 1u; - - if (first_bitplane) { - j2k_classic_cleanup_pass( - magnitudes, - states, - coded, - encoder, - contexts, - params.width, - params.height, - padded_width, - bit_mask, - params.sub_band_type, - params.style_flags - ); - if ((params.style_flags & J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS) != 0u) { - j2k_classic_encode_segmentation_symbols(encoder, contexts); - } - pass_count += 1u; - if ((params.style_flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0u) { - reset_contexts(contexts); - } - } else { - j2k_classic_significance_pass( - magnitudes, - states, - coded, - encoder, - contexts, - params.width, - params.height, - padded_width, - bit_mask, - params.sub_band_type, - params.style_flags - ); - pass_count += 1u; - if ((params.style_flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0u) { - reset_contexts(contexts); - } - - j2k_classic_magnitude_refinement_pass( - magnitudes, - states, - coded, - encoder, - contexts, - params.width, - params.height, - padded_width, - bit_mask, - params.style_flags - ); - pass_count += 1u; - if ((params.style_flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0u) { - reset_contexts(contexts); - } - - j2k_classic_cleanup_pass( - magnitudes, - states, - coded, - encoder, - contexts, - params.width, - params.height, - padded_width, - bit_mask, - params.sub_band_type, - params.style_flags - ); - if ((params.style_flags & J2K_CLASSIC_STYLE_SEGMENTATION_SYMBOLS) != 0u) { - j2k_classic_encode_segmentation_symbols(encoder, contexts); - } - pass_count += 1u; - if ((params.style_flags & J2K_CLASSIC_STYLE_RESET_CONTEXT_PROBABILITIES) != 0u) { - reset_contexts(contexts); - } - } - - for (uint idx = 0u; idx < padded_count; ++idx) { - coded[idx] = uchar(0u); - } - - if (encoder.failed != 0u) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 4u, 0u, 0u, 0u, 0u); - return; - } - } - - const uint data_len = j2k_classic_finish_arithmetic_segment(encoder); - if (encoder.failed != 0u || encoder.len == 0u) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 5u, 0u, 0u, 0u, 0u); - return; - } - uint segment_count = 0u; - if (!j2k_classic_push_segment( - segments, - params.segment_capacity, - segment_count, - 0u, - data_len, - 0u, - pass_count, - true)) { - j2k_set_encode_status(status, J2K_ENCODE_STATUS_FAIL, 14u, 0u, 0u, 0u, 0u); - return; - } - - j2k_set_encode_status( - status, - J2K_ENCODE_STATUS_OK, - 0u, - data_len, - pass_count, - missing_bit_planes, - segment_count - ); -} - -struct J2kClassicEncodeBatchJob { - uint coefficient_offset; - uint output_offset; - uint segment_offset; - uint width; - uint height; - uint sub_band_type; - uint total_bitplanes; - uint style_flags; - uint output_capacity; - uint segment_capacity; -}; - -kernel void j2k_encode_classic_code_block( - device const int *coefficients [[buffer(0)]], - device uchar *out [[buffer(1)]], - constant J2kClassicEncodeParams ¶ms [[buffer(2)]], - device J2kClassicEncodeStatus *status [[buffer(3)]], - device J2kClassicSegment *segments [[buffer(4)]], - uint gid [[thread_position_in_grid]] -) { - if (gid != 0u) { - return; - } - j2k_encode_classic_code_block_impl(coefficients, out, params, status, segments); -} - -kernel void j2k_encode_classic_code_blocks( - device const int *coefficients [[buffer(0)]], - device uchar *out [[buffer(1)]], - device const J2kClassicEncodeBatchJob *jobs [[buffer(2)]], - device J2kClassicEncodeStatus *statuses [[buffer(3)]], - device J2kClassicSegment *segments [[buffer(4)]], - constant uint &job_count [[buffer(5)]], - uint gid [[thread_position_in_grid]] -) { - if (gid >= job_count) { - return; - } - const J2kClassicEncodeBatchJob job = jobs[gid]; - J2kClassicEncodeParams params; - params.width = job.width; - params.height = job.height; - params.sub_band_type = job.sub_band_type; - params.total_bitplanes = job.total_bitplanes; - params.style_flags = job.style_flags; - params.output_capacity = job.output_capacity; - params.segment_capacity = job.segment_capacity; - j2k_encode_classic_code_block_impl( - coefficients + job.coefficient_offset, - out + job.output_offset, - params, - statuses + gid, - segments + job.segment_offset - ); -} - -constant uint J2K_HT_MAX_BITPLANES = 30u; -constant uint J2K_HT_MEL_SIZE = 192u; -constant uint J2K_HT_VLC_SIZE = 3072u - J2K_HT_MEL_SIZE; -constant uint J2K_HT_MS_SIZE = ((16384u * 16u) + 14u) / 15u; -constant uint J2K_HT_MEL_OFFSET = J2K_HT_MS_SIZE; -constant uint J2K_HT_VLC_OFFSET = J2K_HT_MS_SIZE + J2K_HT_MEL_SIZE; - -struct J2kHtEncodeParams { - uint width; - uint height; - uint total_bitplanes; - uint output_capacity; -}; - -struct J2kHtEncodeStatus { - uint code; - uint detail; - uint data_len; - uint num_coding_passes; - uint num_zero_bitplanes; - uint reserved0; - uint reserved1; - uint reserved2; -}; - -struct J2kHtMelEncoder { - uint pos; - uint remaining_bits; - uchar tmp; - uint run; - uint k; - uint threshold; - uint failed; -}; - -struct J2kHtVlcEncoder { - uint pos; - uint used_bits; - uchar tmp; - uint last_greater_than_8f; - uint failed; -}; - -struct J2kHtMagSgnEncoder { - uint pos; - uint max_bits; - uint used_bits; - uint tmp; - uint failed; -}; - -constant uint J2K_HT_MEL_EXP[13] = { - 0u, 0u, 0u, 1u, 1u, 1u, 2u, 2u, 2u, 3u, 3u, 4u, 5u -}; - -inline void j2k_set_ht_encode_status( - device J2kHtEncodeStatus *status, - uint code, - uint detail, - uint data_len, - uint passes, - uint zbp -) { - status->code = code; - status->detail = detail; - status->data_len = data_len; - status->num_coding_passes = passes; - status->num_zero_bitplanes = zbp; - status->reserved0 = 0u; - status->reserved1 = 0u; - status->reserved2 = 0u; -} - -inline void j2k_set_ht_encode_status_with_segments( - device J2kHtEncodeStatus *status, - uint code, - uint detail, - uint data_len, - uint passes, - uint zbp, - uint ms_len, - uint mel_len, - uint vlc_len -) { - status->code = code; - status->detail = detail; - status->data_len = data_len; - status->num_coding_passes = passes; - status->num_zero_bitplanes = zbp; - status->reserved0 = ms_len; - status->reserved1 = mel_len; - status->reserved2 = vlc_len; -} - -inline uint j2k_ht_aligned_sign_magnitude(int coefficient, uint total_bitplanes) { - if (coefficient == 0) { - return 0u; - } - const uint sign = coefficient < 0 ? 0x80000000u : 0u; - const uint magnitude = (coefficient < 0 ? uint(-coefficient) : uint(coefficient)) - << (31u - total_bitplanes); - return sign | magnitude; -} - -inline void j2k_ht_mel_init(thread J2kHtMelEncoder &mel) { - mel.pos = 0u; - mel.remaining_bits = 8u; - mel.tmp = uchar(0u); - mel.run = 0u; - mel.k = 0u; - mel.threshold = 1u; - mel.failed = 0u; -} - -inline void j2k_ht_vlc_init(thread J2kHtVlcEncoder &vlc, device uchar *out) { - vlc.pos = 1u; - vlc.used_bits = 4u; - vlc.tmp = uchar(0x0Fu); - vlc.last_greater_than_8f = 1u; - vlc.failed = 0u; - out[J2K_HT_VLC_OFFSET + J2K_HT_VLC_SIZE - 1u] = uchar(0xFFu); -} - -inline void j2k_ht_ms_init(thread J2kHtMagSgnEncoder &ms) { - ms.pos = 0u; - ms.max_bits = 8u; - ms.used_bits = 0u; - ms.tmp = 0u; - ms.failed = 0u; -} - -inline void j2k_ht_mel_emit_bit(thread J2kHtMelEncoder &mel, device uchar *out, bool bit) { - mel.tmp = uchar((uint(mel.tmp) << 1u) | (bit ? 1u : 0u)); - mel.remaining_bits -= 1u; - if (mel.remaining_bits == 0u) { - if (mel.pos >= J2K_HT_MEL_SIZE) { - mel.failed = 1u; - return; - } - out[J2K_HT_MEL_OFFSET + mel.pos] = mel.tmp; - mel.pos += 1u; - mel.remaining_bits = mel.tmp == uchar(0xFFu) ? 7u : 8u; - mel.tmp = uchar(0u); - } -} - -inline void j2k_ht_mel_encode(thread J2kHtMelEncoder &mel, device uchar *out, bool bit) { - if (!bit) { - mel.run += 1u; - if (mel.run >= mel.threshold) { - j2k_ht_mel_emit_bit(mel, out, true); - mel.run = 0u; - mel.k = min(mel.k + 1u, 12u); - mel.threshold = 1u << J2K_HT_MEL_EXP[mel.k]; - } - } else { - j2k_ht_mel_emit_bit(mel, out, false); - uint t = J2K_HT_MEL_EXP[mel.k]; - while (t > 0u) { - t -= 1u; - j2k_ht_mel_emit_bit(mel, out, ((mel.run >> t) & 1u) != 0u); - } - mel.run = 0u; - mel.k = mel.k == 0u ? 0u : mel.k - 1u; - mel.threshold = 1u << J2K_HT_MEL_EXP[mel.k]; - } -} - -inline void j2k_ht_vlc_encode( - thread J2kHtVlcEncoder &vlc, - device uchar *out, - uint codeword, - uint codeword_len -) { - while (codeword_len > 0u) { - if (vlc.pos >= J2K_HT_VLC_SIZE) { - vlc.failed = 1u; - return; - } - - uint available_bits = 8u - vlc.last_greater_than_8f - vlc.used_bits; - const uint take = min(available_bits, codeword_len); - const uint mask = take == 32u ? 0xFFFFFFFFu : ((1u << take) - 1u); - vlc.tmp = uchar(uint(vlc.tmp) | ((codeword & mask) << vlc.used_bits)); - vlc.used_bits += take; - available_bits -= take; - codeword_len -= take; - codeword >>= take; - - if (available_bits == 0u) { - if (vlc.last_greater_than_8f != 0u && vlc.tmp != uchar(0x7Fu)) { - vlc.last_greater_than_8f = 0u; - continue; - } - - const uint write_index = J2K_HT_VLC_SIZE - 1u - vlc.pos; - out[J2K_HT_VLC_OFFSET + write_index] = vlc.tmp; - vlc.pos += 1u; - vlc.last_greater_than_8f = vlc.tmp > uchar(0x8Fu) ? 1u : 0u; - vlc.tmp = uchar(0u); - vlc.used_bits = 0u; - } - } -} - -inline void j2k_ht_ms_encode( - thread J2kHtMagSgnEncoder &ms, - device uchar *out, - uint codeword, - uint codeword_len -) { - while (codeword_len > 0u) { - if (ms.pos >= J2K_HT_MS_SIZE) { - ms.failed = 1u; - return; - } - - const uint take = min(ms.max_bits - ms.used_bits, codeword_len); - const uint mask = take == 32u ? 0xFFFFFFFFu : ((1u << take) - 1u); - ms.tmp |= (codeword & mask) << ms.used_bits; - ms.used_bits += take; - codeword >>= take; - codeword_len -= take; - - if (ms.used_bits >= ms.max_bits) { - out[ms.pos] = uchar(ms.tmp); - ms.pos += 1u; - ms.max_bits = ms.tmp == 0xFFu ? 7u : 8u; - ms.tmp = 0u; - ms.used_bits = 0u; - } - } -} - -inline void j2k_ht_ms_terminate(thread J2kHtMagSgnEncoder &ms, device uchar *out) { - if (ms.used_bits > 0u) { - const uint unused = ms.max_bits - ms.used_bits; - ms.tmp |= (0xFFu & ((1u << unused) - 1u)) << ms.used_bits; - ms.used_bits += unused; - if (ms.tmp != 0xFFu) { - if (ms.pos >= J2K_HT_MS_SIZE) { - ms.failed = 1u; - return; - } - out[ms.pos] = uchar(ms.tmp); - ms.pos += 1u; - } - } else if (ms.max_bits == 7u) { - ms.pos = ms.pos == 0u ? 0u : ms.pos - 1u; - } -} - -inline void j2k_ht_process_sample( - uint slot, - uint value, - uint p, - thread int *rho_acc, - thread int *e_q, - thread int &e_qmax, - thread uint *s -) { - uint val = value + value; - val >>= p; - val &= ~1u; - if (val != 0u) { - rho_acc[0] |= int(1u << (slot & 0x3u)); - val -= 1u; - e_q[slot] = int(32u - clz(val)); - e_qmax = max(e_qmax, e_q[slot]); - val -= 1u; - s[slot] = val + (value >> 31u); - } -} - -inline uchar j2k_ht_uvlc_byte(device const uchar *table, uint index, uint field) { - return table[index * 6u + field]; -} - -inline void j2k_ht_encode_uvlc_pair( - thread J2kHtVlcEncoder &vlc, - device uchar *out, - device const uchar *uvlc_table, - uint first_index, - uint second_index -) { - const uchar first_pre = j2k_ht_uvlc_byte(uvlc_table, first_index, 0u); - const uchar first_pre_len = j2k_ht_uvlc_byte(uvlc_table, first_index, 1u); - const uchar first_suf = j2k_ht_uvlc_byte(uvlc_table, first_index, 2u); - const uchar first_suf_len = j2k_ht_uvlc_byte(uvlc_table, first_index, 3u); - const uchar second_pre = j2k_ht_uvlc_byte(uvlc_table, second_index, 0u); - const uchar second_pre_len = j2k_ht_uvlc_byte(uvlc_table, second_index, 1u); - const uchar second_suf = j2k_ht_uvlc_byte(uvlc_table, second_index, 2u); - const uchar second_suf_len = j2k_ht_uvlc_byte(uvlc_table, second_index, 3u); - j2k_ht_vlc_encode(vlc, out, uint(first_pre), uint(first_pre_len)); - j2k_ht_vlc_encode(vlc, out, uint(second_pre), uint(second_pre_len)); - j2k_ht_vlc_encode(vlc, out, uint(first_suf), uint(first_suf_len)); - j2k_ht_vlc_encode(vlc, out, uint(second_suf), uint(second_suf_len)); -} - -inline void j2k_ht_encode_uvlc( - int u_q0, - int u_q1, - thread J2kHtVlcEncoder &vlc, - device uchar *out, - device const uchar *uvlc_table -) { - if (u_q0 > 2 && u_q1 > 2) { - j2k_ht_encode_uvlc_pair(vlc, out, uvlc_table, uint(u_q0 - 2), uint(u_q1 - 2)); - } else if (u_q0 > 2 && u_q1 > 0) { - const uint first_index = uint(u_q0); - const uchar first_pre = j2k_ht_uvlc_byte(uvlc_table, first_index, 0u); - const uchar first_pre_len = j2k_ht_uvlc_byte(uvlc_table, first_index, 1u); - const uchar first_suf = j2k_ht_uvlc_byte(uvlc_table, first_index, 2u); - const uchar first_suf_len = j2k_ht_uvlc_byte(uvlc_table, first_index, 3u); - j2k_ht_vlc_encode(vlc, out, uint(first_pre), uint(first_pre_len)); - j2k_ht_vlc_encode(vlc, out, uint(u_q1 - 1), 1u); - j2k_ht_vlc_encode(vlc, out, uint(first_suf), uint(first_suf_len)); - } else { - j2k_ht_encode_uvlc_pair( - vlc, - out, - uvlc_table, - uint(max(u_q0, 0)), - uint(max(u_q1, 0)) - ); - } -} - -inline void j2k_ht_encode_uvlc_non_initial( - int u_q0, - int u_q1, - thread J2kHtVlcEncoder &vlc, - device uchar *out, - device const uchar *uvlc_table -) { - j2k_ht_encode_uvlc_pair( - vlc, - out, - uvlc_table, - uint(max(u_q0, 0)), - uint(max(u_q1, 0)) - ); -} - -inline void j2k_ht_encode_mag_signs( - int rho, - int u_q, - ushort tuple, - thread const uint *s, - uint offset, - thread J2kHtMagSgnEncoder &ms, - device uchar *out -) { - const uint e_k = uint(tuple & ushort(0xFu)); - for (uint bit = 0u; bit < 4u; ++bit) { - const int sample_mask = int(1u << bit); - if ((rho & sample_mask) == 0) { - continue; - } - const int reduction = int((e_k >> bit) & 1u); - const uint magnitude_bits = uint(u_q - reduction); - const uint payload = magnitude_bits == 0u - ? 0u - : (s[offset + bit] & ((1u << magnitude_bits) - 1u)); - j2k_ht_ms_encode(ms, out, payload, magnitude_bits); - } -} - -inline int j2k_ht_encode_quad_initial_row( - uint offset, - uint c_q, - int rho, - int e_qmax, - thread const int *e_q, - thread const uint *s, - uint lep, - uint lcxp, - thread uchar *e_val, - thread uchar *cx_val, - thread J2kHtMelEncoder &mel, - thread J2kHtVlcEncoder &vlc, - thread J2kHtMagSgnEncoder &ms, - device uchar *out, - device const ushort *vlc_table0 -) { - const int u_q = max(e_qmax, 1) - 1; - uint eps = 0u; - if (u_q > 0) { - eps |= uint(e_q[offset] == e_qmax); - eps |= uint(e_q[offset + 1u] == e_qmax) << 1u; - eps |= uint(e_q[offset + 2u] == e_qmax) << 2u; - eps |= uint(e_q[offset + 3u] == e_qmax) << 3u; - } - - e_val[lep] = max(e_val[lep], uchar(e_q[offset + 1u])); - e_val[lep + 1u] = uchar(e_q[offset + 3u]); - cx_val[lcxp] = uchar(uint(cx_val[lcxp]) | uint((rho & 2) >> 1)); - cx_val[lcxp + 1u] = uchar((rho & 8) >> 3); - - const ushort tuple = vlc_table0[(c_q << 8u) | (uint(rho) << 4u) | eps]; - j2k_ht_vlc_encode(vlc, out, uint(tuple >> 8u), uint((tuple >> 4u) & ushort(0x7u))); - if (c_q == 0u) { - j2k_ht_mel_encode(mel, out, rho != 0); - } - j2k_ht_encode_mag_signs(rho, max(e_qmax, 1), tuple, s, offset, ms, out); - return u_q; -} - -inline int j2k_ht_encode_quad_non_initial_row( - uint offset, - uint c_q, - int rho, - int e_qmax, - int max_e, - thread const int *e_q, - thread const uint *s, - thread J2kHtMelEncoder &mel, - thread J2kHtVlcEncoder &vlc, - thread J2kHtMagSgnEncoder &ms, - device uchar *out, - device const ushort *vlc_table1 -) { - const int kappa = (rho & (rho - 1)) != 0 ? max(max_e, 1) : 1; - const int u_q = max(e_qmax, kappa) - kappa; - uint eps = 0u; - if (u_q > 0) { - eps |= uint(e_q[offset] == e_qmax); - eps |= uint(e_q[offset + 1u] == e_qmax) << 1u; - eps |= uint(e_q[offset + 2u] == e_qmax) << 2u; - eps |= uint(e_q[offset + 3u] == e_qmax) << 3u; - } - - const ushort tuple = vlc_table1[(c_q << 8u) | (uint(rho) << 4u) | eps]; - j2k_ht_vlc_encode(vlc, out, uint(tuple >> 8u), uint((tuple >> 4u) & ushort(0x7u))); - if (c_q == 0u) { - j2k_ht_mel_encode(mel, out, rho != 0); - } - j2k_ht_encode_mag_signs(rho, max(e_qmax, kappa), tuple, s, offset, ms, out); - return u_q; -} - -inline void j2k_ht_clear_quad_state(thread int *rho, thread int *e_q, thread int *e_qmax, thread uint *s) { - rho[0] = 0; - rho[1] = 0; - for (uint idx = 0u; idx < 8u; ++idx) { - e_q[idx] = 0; - s[idx] = 0u; - } - e_qmax[0] = 0; - e_qmax[1] = 0; -} - -inline int j2k_ht_encode_first_quad_pair( - device const int *coefficients, - uint stride, - uint height, - uint total_bitplanes, - uint p, - thread uint &sp, - uint x, - thread uchar *e_val, - thread uchar *cx_val, - thread uint &c_q0, - thread int *rho, - thread int *e_q, - thread int *e_qmax, - thread uint *s, - thread J2kHtMelEncoder &mel, - thread J2kHtVlcEncoder &vlc, - thread J2kHtMagSgnEncoder &ms, - device uchar *out, - device const ushort *vlc_table0, - device const uchar *uvlc_table -) { - const uint lep = x / 2u; - const uint lcxp = x / 2u; - - j2k_ht_process_sample(0u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[0], e_q, e_qmax[0], s); - j2k_ht_process_sample( - 1u, - height > 1u ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, - p, - &rho[0], - e_q, - e_qmax[0], - s - ); - sp += 1u; - - if (x + 1u < stride) { - j2k_ht_process_sample(2u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[0], e_q, e_qmax[0], s); - j2k_ht_process_sample( - 3u, - height > 1u ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, - p, - &rho[0], - e_q, - e_qmax[0], - s - ); - sp += 1u; - } - - const int u_q0 = j2k_ht_encode_quad_initial_row( - 0u, c_q0, rho[0], e_qmax[0], e_q, s, lep, lcxp, e_val, cx_val, mel, vlc, ms, out, vlc_table0 - ); - - if (x + 2u < stride) { - j2k_ht_process_sample(4u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[1], e_q, e_qmax[1], s); - j2k_ht_process_sample( - 5u, - height > 1u ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, - p, - &rho[1], - e_q, - e_qmax[1], - s - ); - sp += 1u; - - if (x + 3u < stride) { - j2k_ht_process_sample(6u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[1], e_q, e_qmax[1], s); - j2k_ht_process_sample( - 7u, - height > 1u ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, - p, - &rho[1], - e_q, - e_qmax[1], - s - ); - sp += 1u; - } - - const uint c_q1 = uint((rho[0] >> 1) | (rho[0] & 1)); - const int u_q1 = j2k_ht_encode_quad_initial_row( - 4u, c_q1, rho[1], e_qmax[1], e_q, s, lep + 1u, lcxp + 1u, e_val, cx_val, mel, vlc, ms, out, vlc_table0 - ); - - if (u_q0 > 0 && u_q1 > 0) { - j2k_ht_mel_encode(mel, out, min(u_q0, u_q1) > 2); - } - j2k_ht_encode_uvlc(u_q0, u_q1, vlc, out, uvlc_table); - c_q0 = uint((rho[1] >> 1) | (rho[1] & 1)); - } else { - j2k_ht_encode_uvlc(u_q0, 0, vlc, out, uvlc_table); - c_q0 = 0u; - } - - j2k_ht_clear_quad_state(rho, e_q, e_qmax, s); - return 0; -} - -inline int j2k_ht_encode_non_initial_quad_pair( - device const int *coefficients, - uint stride, - uint width, - uint height, - uint y, - uint total_bitplanes, - uint p, - thread uint &sp, - uint x, - thread uchar *e_val, - thread uchar *cx_val, - thread uint &lep, - thread uint &lcxp, - thread int &max_e, - thread uint &c_q0, - thread int *rho, - thread int *e_q, - thread int *e_qmax, - thread uint *s, - thread J2kHtMelEncoder &mel, - thread J2kHtVlcEncoder &vlc, - thread J2kHtMagSgnEncoder &ms, - device uchar *out, - device const ushort *vlc_table1, - device const uchar *uvlc_table -) { - j2k_ht_process_sample(0u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[0], e_q, e_qmax[0], s); - j2k_ht_process_sample( - 1u, - y + 1u < height ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, - p, - &rho[0], - e_q, - e_qmax[0], - s - ); - sp += 1u; - - if (x + 1u < width) { - j2k_ht_process_sample(2u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[0], e_q, e_qmax[0], s); - j2k_ht_process_sample( - 3u, - y + 1u < height ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, - p, - &rho[0], - e_q, - e_qmax[0], - s - ); - sp += 1u; - } - - const int prev_max = max_e; - const int u_q0 = j2k_ht_encode_quad_non_initial_row( - 0u, c_q0, rho[0], e_qmax[0], prev_max, e_q, s, mel, vlc, ms, out, vlc_table1 - ); - - e_val[lep] = max(e_val[lep], uchar(e_q[1])); - lep += 1u; - max_e = int(max(e_val[lep], e_val[lep + 1u])) - 1; - e_val[lep] = uchar(e_q[3]); - cx_val[lcxp] = uchar(uint(cx_val[lcxp]) | uint((rho[0] & 2) >> 1)); - lcxp += 1u; - uint c_q1 = uint(cx_val[lcxp]) + (uint(cx_val[lcxp + 1u]) << 2u); - cx_val[lcxp] = uchar((rho[0] & 8) >> 3); - - int u_q1 = 0; - if (x + 2u < width) { - j2k_ht_process_sample(4u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[1], e_q, e_qmax[1], s); - j2k_ht_process_sample( - 5u, - y + 1u < height ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, - p, - &rho[1], - e_q, - e_qmax[1], - s - ); - sp += 1u; - - if (x + 3u < width) { - j2k_ht_process_sample(6u, j2k_ht_aligned_sign_magnitude(coefficients[sp], total_bitplanes), p, &rho[1], e_q, e_qmax[1], s); - j2k_ht_process_sample( - 7u, - y + 1u < height ? j2k_ht_aligned_sign_magnitude(coefficients[sp + stride], total_bitplanes) : 0u, - p, - &rho[1], - e_q, - e_qmax[1], - s - ); - sp += 1u; - } - - c_q1 |= uint((rho[0] & 4) >> 1); - c_q1 |= uint((rho[0] & 8) >> 2); - u_q1 = j2k_ht_encode_quad_non_initial_row( - 4u, c_q1, rho[1], e_qmax[1], max_e, e_q, s, mel, vlc, ms, out, vlc_table1 - ); - - e_val[lep] = max(e_val[lep], uchar(e_q[5])); - lep += 1u; - max_e = int(max(e_val[lep], e_val[lep + 1u])) - 1; - e_val[lep] = uchar(e_q[7]); - cx_val[lcxp] = uchar(uint(cx_val[lcxp]) | uint((rho[1] & 2) >> 1)); - lcxp += 1u; - c_q0 = uint(cx_val[lcxp]) + (uint(cx_val[lcxp + 1u]) << 2u); - cx_val[lcxp] = uchar((rho[1] & 8) >> 3); - c_q0 |= uint((rho[1] & 4) >> 1); - c_q0 |= uint((rho[1] & 8) >> 2); - } else { - c_q0 = 0u; - } - - j2k_ht_encode_uvlc_non_initial(u_q0, u_q1, vlc, out, uvlc_table); - j2k_ht_clear_quad_state(rho, e_q, e_qmax, s); - return 0; -} - -inline void j2k_ht_terminate_mel_vlc( - thread J2kHtMelEncoder &mel, - thread J2kHtVlcEncoder &vlc, - device uchar *out -) { - if (mel.run > 0u) { - j2k_ht_mel_emit_bit(mel, out, true); - } - - mel.tmp = uchar(uint(mel.tmp) << mel.remaining_bits); - const uchar mel_mask = uchar((0xFFu << mel.remaining_bits) & 0xFFu); - const uchar vlc_mask = vlc.used_bits == 0u - ? uchar(0u) - : uchar((1u << vlc.used_bits) - 1u); - - if ((mel_mask | vlc_mask) == uchar(0u)) { - return; - } - - const uchar fused = mel.tmp | vlc.tmp; - const bool fused_ok = - ((((fused ^ mel.tmp) & mel_mask) | ((fused ^ vlc.tmp) & vlc_mask)) == uchar(0u)) && - fused != uchar(0xFFu); - - if (fused_ok && vlc.pos > 1u) { - if (mel.pos >= J2K_HT_MEL_SIZE) { - mel.failed = 1u; - return; - } - out[J2K_HT_MEL_OFFSET + mel.pos] = fused; - mel.pos += 1u; - } else { - if (mel.pos >= J2K_HT_MEL_SIZE || vlc.pos >= J2K_HT_VLC_SIZE) { - mel.failed = 1u; - vlc.failed = 1u; - return; - } - out[J2K_HT_MEL_OFFSET + mel.pos] = mel.tmp; - mel.pos += 1u; - const uint write_index = J2K_HT_VLC_SIZE - 1u - vlc.pos; - out[J2K_HT_VLC_OFFSET + write_index] = vlc.tmp; - vlc.pos += 1u; - } -} - -inline void j2k_encode_ht_code_block_impl_with_max_and_assembly( - device const int *coefficients, - device uchar *out, - J2kHtEncodeParams params, - device const ushort *vlc_table0, - device const ushort *vlc_table1, - device const uchar *uvlc_table, - device J2kHtEncodeStatus *status, - uint max_magnitude, - bool assemble_final -) { - j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u, 0u, 0u); - - if (params.width == 0u || params.height == 0u || - params.total_bitplanes == 0u || params.total_bitplanes > J2K_HT_MAX_BITPLANES || - params.output_capacity < J2K_HT_MS_SIZE + J2K_HT_MEL_SIZE + J2K_HT_VLC_SIZE) { - j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 1u, 0u, 0u, 0u); - return; - } - - if (max_magnitude == 0u) { - j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_OK, 0u, 0u, 0u, params.total_bitplanes); - return; - } - - const uint block_bitplanes = 32u - clz(max_magnitude); - if (block_bitplanes > params.total_bitplanes) { - j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_FAIL, 2u, 0u, 0u, 0u); - return; - } - - const uint missing_msbs = params.total_bitplanes - 1u; - const uint p = 30u - missing_msbs; - - thread J2kHtMelEncoder mel; - thread J2kHtVlcEncoder vlc; - thread J2kHtMagSgnEncoder ms; - j2k_ht_mel_init(mel); - j2k_ht_vlc_init(vlc, out); - j2k_ht_ms_init(ms); - - thread uchar e_val[513]; - thread uchar cx_val[513]; - for (uint idx = 0u; idx < 513u; ++idx) { - e_val[idx] = uchar(0u); - cx_val[idx] = uchar(0u); - } - - thread int e_qmax[2]; - thread int e_q[8]; - thread int rho[2]; - thread uint s[8]; - j2k_ht_clear_quad_state(rho, e_q, e_qmax, s); - - uint c_q0 = 0u; - uint sp = 0u; - uint x = 0u; - while (x < params.width) { - j2k_ht_encode_first_quad_pair( - coefficients, - params.width, - params.height, - params.total_bitplanes, - p, - sp, - x, - e_val, - cx_val, - c_q0, - rho, - e_q, - e_qmax, - s, - mel, - vlc, - ms, - out, - vlc_table0, - uvlc_table - ); - x += 4u; - } - - const uint e_val_sentinel = (params.width + 1u) / 2u + 1u; - if (e_val_sentinel < 513u) { - e_val[e_val_sentinel] = uchar(0u); - } - - uint y = 2u; - while (y < params.height) { - uint lep = 0u; - int max_e = int(max(e_val[lep], e_val[lep + 1u])) - 1; - e_val[lep] = uchar(0u); - - uint lcxp = 0u; - c_q0 = uint(cx_val[lcxp]) + (uint(cx_val[lcxp + 1u]) << 2u); - cx_val[lcxp] = uchar(0u); - - sp = y * params.width; - x = 0u; - while (x < params.width) { - j2k_ht_encode_non_initial_quad_pair( - coefficients, - params.width, - params.width, - params.height, - y, - params.total_bitplanes, - p, - sp, - x, - e_val, - cx_val, - lep, - lcxp, - max_e, - c_q0, - rho, - e_q, - e_qmax, - s, - mel, - vlc, - ms, - out, - vlc_table1, - uvlc_table - ); - x += 4u; - } - - y += 2u; - } - - j2k_ht_terminate_mel_vlc(mel, vlc, out); - j2k_ht_ms_terminate(ms, out); - - if (mel.failed != 0u || vlc.failed != 0u || ms.failed != 0u) { - j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_FAIL, 3u, 0u, 0u, 0u); - return; - } - - const uint ms_len = ms.pos; - const uint mel_len = mel.pos; - const uint vlc_len = vlc.pos; - const uint total_len = ms_len + mel_len + vlc_len; - if (total_len < 2u || total_len > params.output_capacity) { - j2k_set_ht_encode_status(status, J2K_ENCODE_STATUS_FAIL, 4u, 0u, 0u, 0u); - return; - } - - if (assemble_final) { - for (uint idx = 0u; idx < mel_len; ++idx) { - out[ms_len + idx] = out[J2K_HT_MEL_OFFSET + idx]; - } - const uint vlc_start = J2K_HT_VLC_SIZE - vlc_len; - for (uint idx = 0u; idx < vlc_len; ++idx) { - out[ms_len + mel_len + idx] = out[J2K_HT_VLC_OFFSET + vlc_start + idx]; - } - - const uint last = total_len - 1u; - const uint prev = total_len - 2u; - const uint locator_bytes = mel_len + vlc_len; - out[last] = uchar(locator_bytes >> 4u); - out[prev] = uchar((out[prev] & uchar(0xF0u)) | uchar(locator_bytes & 0x0Fu)); - } - - j2k_set_ht_encode_status_with_segments( - status, - J2K_ENCODE_STATUS_OK, - 0u, - total_len, - 1u, - missing_msbs, - ms_len, - mel_len, - vlc_len - ); -} - -inline void j2k_encode_ht_code_block_impl_with_max( - device const int *coefficients, - device uchar *out, - J2kHtEncodeParams params, - device const ushort *vlc_table0, - device const ushort *vlc_table1, - device const uchar *uvlc_table, - device J2kHtEncodeStatus *status, - uint max_magnitude -) { - j2k_encode_ht_code_block_impl_with_max_and_assembly( - coefficients, - out, - params, - vlc_table0, - vlc_table1, - uvlc_table, - status, - max_magnitude, - true - ); -} - -inline void j2k_encode_ht_code_block_impl( - device const int *coefficients, - device uchar *out, - J2kHtEncodeParams params, - device const ushort *vlc_table0, - device const ushort *vlc_table1, - device const uchar *uvlc_table, - device J2kHtEncodeStatus *status -) { - uint max_magnitude = 0u; - for (uint y = 0u; y < params.height; ++y) { - for (uint x = 0u; x < params.width; ++x) { - max_magnitude = max(max_magnitude, j2k_classic_magnitude(coefficients[y * params.width + x])); - } - } - j2k_encode_ht_code_block_impl_with_max( - coefficients, - out, - params, - vlc_table0, - vlc_table1, - uvlc_table, - status, - max_magnitude - ); -} - -struct J2kHtEncodeBatchJob { - uint coefficient_offset; - uint output_offset; - uint width; - uint height; - uint total_bitplanes; - uint output_capacity; -}; - -kernel void j2k_encode_ht_code_block( - device const int *coefficients [[buffer(0)]], - device uchar *out [[buffer(1)]], - constant J2kHtEncodeParams ¶ms [[buffer(2)]], - device const ushort *vlc_table0 [[buffer(3)]], - device const ushort *vlc_table1 [[buffer(4)]], - device const uchar *uvlc_table [[buffer(5)]], - device J2kHtEncodeStatus *status [[buffer(6)]], - uint gid [[thread_position_in_grid]] -) { - if (gid != 0u) { - return; - } - j2k_encode_ht_code_block_impl( - coefficients, - out, - params, - vlc_table0, - vlc_table1, - uvlc_table, - status - ); -} - -kernel void j2k_encode_ht_code_blocks( - device const int *coefficients [[buffer(0)]], - device uchar *out [[buffer(1)]], - device const J2kHtEncodeBatchJob *jobs [[buffer(2)]], - device const ushort *vlc_table0 [[buffer(3)]], - device const ushort *vlc_table1 [[buffer(4)]], - device const uchar *uvlc_table [[buffer(5)]], - device J2kHtEncodeStatus *statuses [[buffer(6)]], - constant uint &job_count [[buffer(7)]], - uint gid [[thread_position_in_grid]] -) { - if (gid >= job_count) { - return; - } - const J2kHtEncodeBatchJob job = jobs[gid]; - J2kHtEncodeParams params; - params.width = job.width; - params.height = job.height; - params.total_bitplanes = job.total_bitplanes; - params.output_capacity = job.output_capacity; - j2k_encode_ht_code_block_impl( - coefficients + job.coefficient_offset, - out + job.output_offset, - params, - vlc_table0, - vlc_table1, - uvlc_table, - statuses + gid - ); -} - -#if defined(SIGNINUM_J2K_METAL_HT_SIMD_PROTOTYPE) -kernel void j2k_encode_ht_code_blocks_simd_prototype( - device const int *coefficients [[buffer(0)]], - device uchar *out [[buffer(1)]], - device const J2kHtEncodeBatchJob *jobs [[buffer(2)]], - device const ushort *vlc_table0 [[buffer(3)]], - device const ushort *vlc_table1 [[buffer(4)]], - device const uchar *uvlc_table [[buffer(5)]], - device J2kHtEncodeStatus *statuses [[buffer(6)]], - constant uint &job_count [[buffer(7)]], - uint tg [[threadgroup_position_in_grid]], - uint tid [[thread_index_in_threadgroup]] -) { - if (tg >= job_count) { - return; - } - - const J2kHtEncodeBatchJob job = jobs[tg]; - device const int *block = coefficients + job.coefficient_offset; - const uint sample_count = job.width * job.height; - uint local_max = 0u; - for (uint idx = tid; idx < sample_count; idx += 32u) { - local_max = max(local_max, j2k_classic_magnitude(block[idx])); - } - const uint block_max = simd_max(local_max); - - J2kHtEncodeParams params; - params.width = job.width; - params.height = job.height; - params.total_bitplanes = job.total_bitplanes; - params.output_capacity = job.output_capacity; - device uchar *block_out = out + job.output_offset; - device J2kHtEncodeStatus *status = statuses + tg; - if (tid == 0u) { - j2k_encode_ht_code_block_impl_with_max_and_assembly( - block, - block_out, - params, - vlc_table0, - vlc_table1, - uvlc_table, - status, - block_max, - false - ); - } - - threadgroup_barrier(mem_flags::mem_device); - - if (status->code != J2K_ENCODE_STATUS_OK || status->data_len == 0u) { - return; - } - - const uint ms_len = status->reserved0; - const uint mel_len = status->reserved1; - const uint vlc_len = status->reserved2; - const uint mel_src = J2K_HT_MEL_OFFSET; - const uint mel_dst = ms_len; - const uint vlc_src = J2K_HT_VLC_OFFSET + (J2K_HT_VLC_SIZE - vlc_len); - const uint vlc_dst = ms_len + mel_len; - const bool mel_nonoverlap = - (mel_dst + mel_len <= mel_src) || (mel_src + mel_len <= mel_dst); - const bool vlc_nonoverlap = - (vlc_dst + vlc_len <= vlc_src) || (vlc_src + vlc_len <= vlc_dst); - - if (mel_nonoverlap) { - for (uint idx = tid; idx < mel_len; idx += 32u) { - block_out[mel_dst + idx] = block_out[mel_src + idx]; - } - } else if (tid == 0u) { - for (uint idx = 0u; idx < mel_len; ++idx) { - block_out[mel_dst + idx] = block_out[mel_src + idx]; - } - } - - threadgroup_barrier(mem_flags::mem_device); - - if (vlc_nonoverlap) { - for (uint idx = tid; idx < vlc_len; idx += 32u) { - block_out[vlc_dst + idx] = block_out[vlc_src + idx]; - } - } else if (tid == 0u) { - for (uint idx = 0u; idx < vlc_len; ++idx) { - block_out[vlc_dst + idx] = block_out[vlc_src + idx]; - } - } - - threadgroup_barrier(mem_flags::mem_device); - - if (tid == 0u) { - const uint total_len = status->data_len; - const uint last = total_len - 1u; - const uint prev = total_len - 2u; - const uint locator_bytes = mel_len + vlc_len; - block_out[last] = uchar(locator_bytes >> 4u); - block_out[prev] = uchar((block_out[prev] & uchar(0xF0u)) | uchar(locator_bytes & 0x0Fu)); - } -} -#endif - -struct J2kPacketEncodeParams { - uint resolution_count; - uint num_layers; - uint num_components; - uint code_block_count; - uint subband_count; - uint descriptor_count; - uint output_capacity; - uint header_capacity; - uint scratch_node_capacity; -}; - -struct J2kPacketDescriptor { - uint packet_index; - uint state_index; - uint layer; - uint resolution; - uint component; - uint precinct_lo; - uint precinct_hi; - uint state_block_offset; -}; - -struct J2kPacketResolution { - uint subband_offset; - uint subband_count; -}; - -struct J2kPacketSubband { - uint block_offset; - uint block_count; - uint num_cbs_x; - uint num_cbs_y; -}; - -struct J2kPacketBlock { - uint data_offset; - uint data_len; - uint num_coding_passes; - uint num_zero_bitplanes; - uint previously_included; - uint l_block; - uint block_coding_mode; - uint reserved0; -}; - -struct J2kResidentPacketBlock { - uint tier1_job_index; - uint previously_included; - uint l_block; - uint block_coding_mode; -}; - -struct J2kResidentPacketBlockParams { - uint block_count; - uint tier1_job_count; -}; - -struct J2kPacketStateBlock { - uint previously_included; - uint l_block; -}; - -struct J2kPacketEncodeStatus { - uint code; - uint detail; - uint data_len; - uint reserved0; -}; - -struct J2kLosslessCodestreamAssemblyParams { - uint width; - uint height; - uint num_components; - uint bit_depth; - uint signed_samples; - uint num_decomposition_levels; - uint use_mct; - uint guard_bits; - uint progression_order; - uint write_tlm; - uint high_throughput; - uint output_capacity; -}; - -struct J2kCodestreamAssemblyStatus { - uint code; - uint detail; - uint data_len; - uint reserved0; -}; - -struct J2kPacketBitWriter { - device uchar *data; - uint capacity; - uint len; - uint buffer; - uint bits_in_buffer; - uint last_byte_was_ff; - uint failed; -}; - -inline void j2k_set_packet_status(device J2kPacketEncodeStatus *status, uint code, uint detail, uint len) { - status->code = code; - status->detail = detail; - status->data_len = len; - status->reserved0 = 0u; -} - -inline void j2k_set_codestream_status( - device J2kCodestreamAssemblyStatus *status, - uint code, - uint detail, - uint len -) { - status->code = code; - status->detail = detail; - status->data_len = len; - status->reserved0 = 0u; -} - -inline bool j2k_codestream_write_u8( - device uchar *out, - uint capacity, - thread uint &cursor, - uint value -) { - if (cursor >= capacity) { - return false; - } - out[cursor] = uchar(value & 0xFFu); - cursor += 1u; - return true; -} - -inline bool j2k_codestream_write_u16( - device uchar *out, - uint capacity, - thread uint &cursor, - uint value -) { - return j2k_codestream_write_u8(out, capacity, cursor, value >> 8u) && - j2k_codestream_write_u8(out, capacity, cursor, value); -} - -inline bool j2k_codestream_write_u32( - device uchar *out, - uint capacity, - thread uint &cursor, - uint value -) { - return j2k_codestream_write_u8(out, capacity, cursor, value >> 24u) && - j2k_codestream_write_u8(out, capacity, cursor, value >> 16u) && - j2k_codestream_write_u8(out, capacity, cursor, value >> 8u) && - j2k_codestream_write_u8(out, capacity, cursor, value); -} - -inline bool j2k_codestream_write_marker( - device uchar *out, - uint capacity, - thread uint &cursor, - uint marker -) { - return j2k_codestream_write_u8(out, capacity, cursor, 0xFFu) && - j2k_codestream_write_u8(out, capacity, cursor, marker); -} - -kernel void j2k_assemble_lossless_classic_codestream( - device const uchar *tile_data [[buffer(0)]], - device const J2kPacketEncodeStatus *tile_status [[buffer(1)]], - device uchar *out [[buffer(2)]], - constant J2kLosslessCodestreamAssemblyParams ¶ms [[buffer(3)]], - device J2kCodestreamAssemblyStatus *status [[buffer(4)]], - uint gid [[thread_position_in_grid]] -) { - if (gid != 0u) { - return; - } - - j2k_set_codestream_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u); - const J2kPacketEncodeStatus packet_status = tile_status[0]; - if (packet_status.code != J2K_ENCODE_STATUS_OK) { - j2k_set_codestream_status(status, J2K_ENCODE_STATUS_FAIL, packet_status.detail, 0u); - return; - } - if (params.num_components == 0u || params.num_components > 255u || - params.bit_depth == 0u || params.bit_depth > 16u || - params.num_decomposition_levels > 31u) { - j2k_set_codestream_status(status, J2K_ENCODE_STATUS_UNSUPPORTED, 1u, 0u); - return; - } - - const uint tile_len = packet_status.data_len; - const uint tile_part_len = 14u + tile_len; - uint cursor = 0u; - bool ok = true; - - ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x4Fu); - - ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x51u); - const uint siz_len = 38u + 3u * params.num_components; - ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, siz_len); - ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, params.width); - ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, params.height); - ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, params.width); - ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, params.height); - ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, params.num_components); - const uint ssiz = (params.bit_depth - 1u) | (params.signed_samples != 0u ? 0x80u : 0u); - for (uint comp = 0u; comp < params.num_components && ok; ++comp) { - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, ssiz); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 1u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 1u); - } - - if (params.high_throughput != 0u) { - const uint magnitude_bits = params.bit_depth - 1u; - const uint bp = magnitude_bits <= 8u ? 0u : - (magnitude_bits < 28u ? magnitude_bits - 8u : 13u + (magnitude_bits >> 2u)); - ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x50u); - ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 8u); - ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, 0x00020000u); - ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, bp); - } - - ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x52u); - ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 12u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.progression_order); - ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 1u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.use_mct != 0u ? 1u : 0u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.num_decomposition_levels); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 4u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 4u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.high_throughput != 0u ? 0x40u : 0u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 1u); - - ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x5Cu); - const uint qcd_steps = 1u + 3u * params.num_decomposition_levels; - ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 3u + qcd_steps); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.guard_bits << 5u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, params.bit_depth << 3u); - for (uint level = 0u; level < params.num_decomposition_levels && ok; ++level) { - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, (params.bit_depth + 1u) << 3u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, (params.bit_depth + 1u) << 3u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, (params.bit_depth + 2u) << 3u); - } - - if (params.write_tlm != 0u) { - ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x55u); - ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 10u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 0x22u); - ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, tile_part_len); - } - - ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x90u); - ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 10u); - ok = ok && j2k_codestream_write_u16(out, params.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u32(out, params.output_capacity, cursor, tile_part_len); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u8(out, params.output_capacity, cursor, 1u); - ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0x93u); - - if (!ok || cursor + tile_len + 2u > params.output_capacity) { - j2k_set_codestream_status(status, J2K_ENCODE_STATUS_FAIL, 2u, cursor); - return; - } - for (uint idx = 0u; idx < tile_len; ++idx) { - out[cursor + idx] = tile_data[idx]; - } - cursor += tile_len; - ok = ok && j2k_codestream_write_marker(out, params.output_capacity, cursor, 0xD9u); - if (!ok) { - j2k_set_codestream_status(status, J2K_ENCODE_STATUS_FAIL, 3u, cursor); - return; - } - - j2k_set_codestream_status(status, J2K_ENCODE_STATUS_OK, 0u, cursor); -} - -struct J2kBatchedCodestreamAssemblyJob { - uint tile_data_offset; - uint codestream_offset; - uint width; - uint height; - uint num_components; - uint bit_depth; - uint signed_samples; - uint num_decomposition_levels; - uint use_mct; - uint guard_bits; - uint progression_order; - uint write_tlm; - uint high_throughput; - uint output_capacity; -}; - -kernel void j2k_assemble_lossless_codestream_batched( - device const uchar *tile_data [[buffer(0)]], - device const J2kPacketEncodeStatus *tile_status [[buffer(1)]], - device uchar *out [[buffer(2)]], - device const J2kBatchedCodestreamAssemblyJob *jobs [[buffer(3)]], - device J2kCodestreamAssemblyStatus *status [[buffer(4)]], - uint gid [[thread_position_in_grid]] -) { - const J2kBatchedCodestreamAssemblyJob job = jobs[gid]; - device J2kCodestreamAssemblyStatus *tile_status_out = status + gid; - device uchar *tile_out = out + job.codestream_offset; - device const uchar *packet_data = tile_data + job.tile_data_offset; - const J2kPacketEncodeStatus packet_status = tile_status[gid]; - - j2k_set_codestream_status(tile_status_out, J2K_ENCODE_STATUS_FAIL, 0u, 0u); - if (packet_status.code != J2K_ENCODE_STATUS_OK) { - j2k_set_codestream_status(tile_status_out, J2K_ENCODE_STATUS_FAIL, packet_status.detail, 0u); - return; - } - if (job.num_components == 0u || job.num_components > 255u || - job.bit_depth == 0u || job.bit_depth > 16u || - job.num_decomposition_levels > 31u) { - j2k_set_codestream_status(tile_status_out, J2K_ENCODE_STATUS_UNSUPPORTED, 1u, 0u); - return; - } - - const uint tile_len = packet_status.data_len; - const uint tile_part_len = 14u + tile_len; - uint cursor = 0u; - bool ok = true; - - ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x4Fu); - - ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x51u); - const uint siz_len = 38u + 3u * job.num_components; - ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, siz_len); - ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, job.width); - ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, job.height); - ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, job.width); - ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, job.height); - ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, job.num_components); - const uint ssiz = (job.bit_depth - 1u) | (job.signed_samples != 0u ? 0x80u : 0u); - for (uint comp = 0u; comp < job.num_components && ok; ++comp) { - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, ssiz); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 1u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 1u); - } - - if (job.high_throughput != 0u) { - const uint magnitude_bits = job.bit_depth - 1u; - const uint bp = magnitude_bits <= 8u ? 0u : - (magnitude_bits < 28u ? magnitude_bits - 8u : 13u + (magnitude_bits >> 2u)); - ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x50u); - ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 8u); - ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, 0x00020000u); - ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, bp); - } - - ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x52u); - ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 12u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.progression_order); - ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 1u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.use_mct != 0u ? 1u : 0u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.num_decomposition_levels); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 4u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 4u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.high_throughput != 0u ? 0x40u : 0u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 1u); - - ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x5Cu); - const uint qcd_steps = 1u + 3u * job.num_decomposition_levels; - ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 3u + qcd_steps); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.guard_bits << 5u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, job.bit_depth << 3u); - for (uint level = 0u; level < job.num_decomposition_levels && ok; ++level) { - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, (job.bit_depth + 1u) << 3u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, (job.bit_depth + 1u) << 3u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, (job.bit_depth + 2u) << 3u); - } - - if (job.write_tlm != 0u) { - ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x55u); - ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 10u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 0x22u); - ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, tile_part_len); - } - - ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x90u); - ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 10u); - ok = ok && j2k_codestream_write_u16(tile_out, job.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u32(tile_out, job.output_capacity, cursor, tile_part_len); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 0u); - ok = ok && j2k_codestream_write_u8(tile_out, job.output_capacity, cursor, 1u); - ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0x93u); - - if (!ok || cursor + tile_len + 2u > job.output_capacity) { - j2k_set_codestream_status(tile_status_out, J2K_ENCODE_STATUS_FAIL, 2u, cursor); - return; - } - for (uint idx = 0u; idx < tile_len; ++idx) { - tile_out[cursor + idx] = packet_data[idx]; - } - cursor += tile_len; - ok = ok && j2k_codestream_write_marker(tile_out, job.output_capacity, cursor, 0xD9u); - if (!ok) { - j2k_set_codestream_status(tile_status_out, J2K_ENCODE_STATUS_FAIL, 3u, cursor); - return; - } - - j2k_set_codestream_status(tile_status_out, J2K_ENCODE_STATUS_OK, 0u, cursor); -} - -kernel void j2k_prepare_packet_blocks_from_classic_status( - device const J2kResidentPacketBlock *resident_blocks [[buffer(0)]], - device const J2kClassicEncodeBatchJob *tier1_jobs [[buffer(1)]], - device const J2kClassicEncodeStatus *tier1_statuses [[buffer(2)]], - device J2kPacketBlock *packet_blocks [[buffer(3)]], - constant J2kResidentPacketBlockParams ¶ms [[buffer(4)]], - uint gid [[thread_position_in_grid]] -) { - if (gid >= params.block_count) { - return; - } - - const J2kResidentPacketBlock resident = resident_blocks[gid]; - J2kPacketBlock packet; - packet.data_offset = 0u; - packet.data_len = 0u; - packet.num_coding_passes = 1u; - packet.num_zero_bitplanes = 0u; - packet.previously_included = resident.previously_included; - packet.l_block = resident.l_block; - packet.block_coding_mode = 0xFFFFFFFFu; - packet.reserved0 = 0u; - - if (resident.tier1_job_index < params.tier1_job_count) { - const J2kClassicEncodeBatchJob job = tier1_jobs[resident.tier1_job_index]; - const J2kClassicEncodeStatus tier1_status = tier1_statuses[resident.tier1_job_index]; - if (tier1_status.code == J2K_ENCODE_STATUS_OK && - tier1_status.data_len <= job.output_capacity && - tier1_status.segment_count <= job.segment_capacity) { - packet.data_offset = job.output_offset; - packet.data_len = tier1_status.data_len; - packet.num_coding_passes = tier1_status.number_of_coding_passes; - packet.num_zero_bitplanes = tier1_status.missing_bit_planes; - packet.block_coding_mode = resident.block_coding_mode; - } else { - packet.reserved0 = tier1_status.detail; - } - } - - packet_blocks[gid] = packet; -} - -kernel void j2k_prepare_packet_blocks_from_ht_status( - device const J2kResidentPacketBlock *resident_blocks [[buffer(0)]], - device const J2kHtEncodeBatchJob *tier1_jobs [[buffer(1)]], - device const J2kHtEncodeStatus *tier1_statuses [[buffer(2)]], - device J2kPacketBlock *packet_blocks [[buffer(3)]], - constant J2kResidentPacketBlockParams ¶ms [[buffer(4)]], - uint gid [[thread_position_in_grid]] -) { - if (gid >= params.block_count) { - return; - } - - const J2kResidentPacketBlock resident = resident_blocks[gid]; - J2kPacketBlock packet; - packet.data_offset = 0u; - packet.data_len = 0u; - packet.num_coding_passes = 1u; - packet.num_zero_bitplanes = 0u; - packet.previously_included = resident.previously_included; - packet.l_block = resident.l_block; - packet.block_coding_mode = 0xFFFFFFFFu; - packet.reserved0 = 0u; - - if (resident.tier1_job_index < params.tier1_job_count) { - const J2kHtEncodeBatchJob job = tier1_jobs[resident.tier1_job_index]; - const J2kHtEncodeStatus tier1_status = tier1_statuses[resident.tier1_job_index]; - if (tier1_status.code == J2K_ENCODE_STATUS_OK && - tier1_status.data_len <= job.output_capacity) { - packet.data_offset = job.output_offset; - packet.data_len = tier1_status.data_len; - packet.num_coding_passes = tier1_status.num_coding_passes; - packet.num_zero_bitplanes = tier1_status.num_zero_bitplanes; - packet.block_coding_mode = resident.block_coding_mode; - } else { - packet.reserved0 = tier1_status.detail; - } - } - - packet_blocks[gid] = packet; -} - -inline void j2k_packet_writer_init(thread J2kPacketBitWriter &writer, device uchar *data, uint capacity) { - writer.data = data; - writer.capacity = capacity; - writer.len = 0u; - writer.buffer = 0u; - writer.bits_in_buffer = 0u; - writer.last_byte_was_ff = 0u; - writer.failed = 0u; -} - -inline void j2k_packet_flush_byte(thread J2kPacketBitWriter &writer) { - const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; - const uchar byte = uchar(writer.buffer >> (writer.bits_in_buffer - limit)); - if (writer.len >= writer.capacity) { - writer.failed = 1u; - return; - } - writer.data[writer.len] = byte; - writer.len += 1u; - writer.last_byte_was_ff = byte == uchar(0xFFu) ? 1u : 0u; - writer.bits_in_buffer -= limit; - writer.buffer &= writer.bits_in_buffer == 0u ? 0u : ((1u << writer.bits_in_buffer) - 1u); -} - -inline void j2k_packet_write_bit(thread J2kPacketBitWriter &writer, uint bit) { - writer.buffer = (writer.buffer << 1u) | (bit & 1u); - writer.bits_in_buffer += 1u; - const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; - if (writer.bits_in_buffer >= limit) { - j2k_packet_flush_byte(writer); - } -} - -inline void j2k_packet_write_bits(thread J2kPacketBitWriter &writer, uint value, uint count) { - for (int bit = int(count) - 1; bit >= 0; --bit) { - j2k_packet_write_bit(writer, (value >> uint(bit)) & 1u); - } -} - -inline void j2k_packet_writer_finish(thread J2kPacketBitWriter &writer) { - if (writer.bits_in_buffer > 0u) { - const uint limit = writer.last_byte_was_ff != 0u ? 7u : 8u; - const uint shift = limit - writer.bits_in_buffer; - const uchar byte = uchar(writer.buffer << shift); - if (writer.len >= writer.capacity) { - writer.failed = 1u; - return; - } - writer.data[writer.len] = byte; - writer.len += 1u; - writer.last_byte_was_ff = byte == uchar(0xFFu) ? 1u : 0u; - writer.buffer = 0u; - writer.bits_in_buffer = 0u; - } -} - -inline uint j2k_packet_ilog2(uint value) { - return value == 0u ? 0u : 31u - clz(value); -} - -inline bool j2k_packet_value_fits(uint value, uint bits) { - return bits >= 32u || value < (1u << bits); -} - -inline uint j2k_packet_bits_for_length(uint l_block, uint passes) { - const uint log2_passes = passes <= 1u ? 0u : j2k_packet_ilog2(passes); - return l_block + log2_passes; -} - -inline uint j2k_packet_bits_for_ht_length(uint l_block, uint passes) { - const uint placeholder_groups = (passes > 0u ? passes - 1u : 0u) / 3u; - const uint placeholder_passes = placeholder_groups * 3u; - return l_block + j2k_packet_ilog2(placeholder_passes + 1u); -} - -inline void j2k_packet_encode_num_passes(uint passes, thread J2kPacketBitWriter &writer) { - if (passes == 1u) { - j2k_packet_write_bit(writer, 0u); - } else if (passes == 2u) { - j2k_packet_write_bits(writer, 0b10u, 2u); - } else if (passes == 3u) { - j2k_packet_write_bits(writer, 0b1100u, 4u); - } else if (passes == 4u) { - j2k_packet_write_bits(writer, 0b1101u, 4u); - } else if (passes == 5u) { - j2k_packet_write_bits(writer, 0b1110u, 4u); - } else if (passes <= 36u) { - j2k_packet_write_bits(writer, 0b1111u, 4u); - j2k_packet_write_bits(writer, passes - 6u, 5u); - } else { - j2k_packet_write_bits(writer, 0x1FFu, 9u); - j2k_packet_write_bits(writer, passes - 37u, 7u); - } -} - -inline void j2k_packet_encode_num_ht_passes(uint passes, thread J2kPacketBitWriter &writer) { - if (passes == 1u) { - j2k_packet_write_bit(writer, 0u); - } else if (passes == 2u) { - j2k_packet_write_bits(writer, 0b10u, 2u); - } else if (passes <= 5u) { - j2k_packet_write_bits(writer, 0b11u, 2u); - j2k_packet_write_bits(writer, passes - 3u, 2u); - } else if (passes <= 36u) { - j2k_packet_write_bits(writer, 0b11u, 2u); - j2k_packet_write_bits(writer, 0b11u, 2u); - j2k_packet_write_bits(writer, passes - 6u, 5u); - } else { - j2k_packet_write_bits(writer, 0b11u, 2u); - j2k_packet_write_bits(writer, 0b11u, 2u); - j2k_packet_write_bits(writer, 31u, 5u); - j2k_packet_write_bits(writer, passes - 37u, 7u); - } -} - -inline void j2k_packet_encode_length( - uint length, - thread uint &l_block, - uint num_bits, - thread J2kPacketBitWriter &writer -) { - while (!j2k_packet_value_fits(length, num_bits)) { - j2k_packet_write_bit(writer, 1u); - l_block += 1u; - num_bits += 1u; - } - j2k_packet_write_bit(writer, 0u); - j2k_packet_write_bits(writer, length, num_bits); -} - -inline uint j2k_packet_tree_offsets( - uint width, - uint height, - thread uint *level_offsets, - thread uint *level_widths, - thread uint *level_heights, - thread uint &levels -) { - uint total = 0u; - uint w = width; - uint h = height; - levels = 0u; - while (true) { - level_offsets[levels] = total; - level_widths[levels] = w; - level_heights[levels] = h; - total += w * h; - levels += 1u; - if (w <= 1u && h <= 1u) { - break; - } - w = (w + 1u) / 2u; - h = (h + 1u) / 2u; - } - return total; -} - -inline bool j2k_packet_prepare_tree( - device const J2kPacketBlock *blocks, - uint block_offset, - uint block_count, - uint num_cbs_x, - uint num_cbs_y, - bool zero_bitplanes, - uint inclusion_layer, - device uint *value, - device uint *current, - device uint *known, - uint node_capacity, - thread uint *level_offsets, - thread uint *level_widths, - thread uint *level_heights, - thread uint &levels -) { - if (num_cbs_x == 0u || num_cbs_y == 0u || num_cbs_x * num_cbs_y != block_count) { - return false; - } - const uint node_count = - j2k_packet_tree_offsets(num_cbs_x, num_cbs_y, level_offsets, level_widths, level_heights, levels); - if (node_count > node_capacity || levels > 16u) { - return false; - } - for (uint idx = 0u; idx < node_count; ++idx) { - value[idx] = 0u; - current[idx] = 0u; - known[idx] = 0u; - } - for (uint idx = 0u; idx < block_count; ++idx) { - const J2kPacketBlock block = blocks[block_offset + idx]; - value[idx] = zero_bitplanes - ? block.num_zero_bitplanes - : (block.num_coding_passes > 0u ? inclusion_layer : 0x7FFFFFFFu); - } - for (uint level = 1u; level < levels; ++level) { - const uint prev_w = level_widths[level - 1u]; - const uint prev_h = level_heights[level - 1u]; - const uint cur_w = level_widths[level]; - const uint cur_h = level_heights[level]; - for (uint py = 0u; py < cur_h; ++py) { - for (uint px = 0u; px < cur_w; ++px) { - uint min_value = 0xFFFFFFFFu; - for (uint dy = 0u; dy < 2u; ++dy) { - const uint cy = py * 2u + dy; - if (cy >= prev_h) { - continue; - } - for (uint dx = 0u; dx < 2u; ++dx) { - const uint cx = px * 2u + dx; - if (cx >= prev_w) { - continue; - } - const uint child = level_offsets[level - 1u] + cy * prev_w + cx; - min_value = min(min_value, value[child]); - } - } - value[level_offsets[level] + py * cur_w + px] = min_value; - } - } - } - return true; -} - -inline void j2k_packet_tree_encode( - uint x, - uint y, - uint threshold, - device uint *value, - device uint *current, - device uint *known, - thread uint *level_offsets, - thread uint *level_widths, - uint levels, - thread J2kPacketBitWriter &writer -) { - thread uint path[16]; - uint cx = x; - uint cy = y; - for (uint level = 0u; level < levels; ++level) { - path[level] = level_offsets[level] + cy * level_widths[level] + cx; - cx /= 2u; - cy /= 2u; - } - - uint parent_val = 0u; - for (int level = int(levels) - 1; level >= 0; --level) { - const uint node = path[uint(level)]; - const uint start = max(current[node], parent_val); - if (known[node] == 0u) { - const uint target = min(value[node], threshold); - for (uint v = start; v < target; ++v) { - j2k_packet_write_bit(writer, 0u); - } - if (value[node] < threshold) { - j2k_packet_write_bit(writer, 1u); - known[node] = 1u; - } - current[node] = target; - } - parent_val = current[node]; - } -} - -kernel void j2k_encode_packetization( - device const J2kPacketResolution *resolutions [[buffer(0)]], - device const J2kPacketSubband *subbands [[buffer(1)]], - device const J2kPacketBlock *blocks [[buffer(2)]], - device const uchar *payload [[buffer(3)]], - device uchar *out [[buffer(4)]], - device uchar *header [[buffer(5)]], - device uint *tree_scratch [[buffer(6)]], - constant J2kPacketEncodeParams ¶ms [[buffer(7)]], - device J2kPacketEncodeStatus *status [[buffer(8)]], - device const J2kPacketDescriptor *descriptors [[buffer(9)]], - device J2kPacketStateBlock *state_blocks [[buffer(10)]], - uint gid [[thread_position_in_grid]] -) { - if (gid != 0u) { - return; - } - - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u); - - const uint node_capacity = params.scratch_node_capacity; - device uint *inc_value = tree_scratch; - device uint *inc_current = tree_scratch + node_capacity; - device uint *inc_known = tree_scratch + node_capacity * 2u; - device uint *zbp_value = tree_scratch + node_capacity * 3u; - device uint *zbp_current = tree_scratch + node_capacity * 4u; - device uint *zbp_known = tree_scratch + node_capacity * 5u; - - uint out_len = 0u; - const uint packet_count = - params.descriptor_count > 0u ? params.descriptor_count : params.resolution_count; - for (uint packet_order_idx = 0u; packet_order_idx < packet_count; ++packet_order_idx) { - const bool has_descriptor = params.descriptor_count > 0u; - const J2kPacketDescriptor descriptor = has_descriptor - ? descriptors[packet_order_idx] - : J2kPacketDescriptor{packet_order_idx, packet_order_idx, 0u, packet_order_idx, 0u, 0u, 0u, 0u}; - const uint packet_index = descriptor.packet_index; - if (packet_index >= params.resolution_count) { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 6u, 0u); - return; - } - const J2kPacketResolution resolution = resolutions[packet_index]; - uint state_block_cursor = descriptor.state_block_offset; - bool any_data = false; - for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { - const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; - for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { - if (blocks[subband.block_offset + block_idx].num_coding_passes > 0u) { - any_data = true; - break; - } - } - } - - thread J2kPacketBitWriter writer; - j2k_packet_writer_init(writer, header, params.header_capacity); - if (!any_data) { - j2k_packet_write_bit(writer, 0u); - j2k_packet_writer_finish(writer); - } else { - j2k_packet_write_bit(writer, 1u); - for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { - const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; - const uint subband_state_block_offset = state_block_cursor; - state_block_cursor += subband.block_count; - thread uint level_offsets[16]; - thread uint level_widths[16]; - thread uint level_heights[16]; - uint levels = 0u; - if (!j2k_packet_prepare_tree( - blocks, - subband.block_offset, - subband.block_count, - subband.num_cbs_x, - subband.num_cbs_y, - false, - descriptor.layer, - inc_value, - inc_current, - inc_known, - node_capacity, - level_offsets, - level_widths, - level_heights, - levels)) { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 1u, 0u); - return; - } - thread uint z_level_offsets[16]; - thread uint z_level_widths[16]; - thread uint z_level_heights[16]; - uint z_levels = 0u; - if (!j2k_packet_prepare_tree( - blocks, - subband.block_offset, - subband.block_count, - subband.num_cbs_x, - subband.num_cbs_y, - true, - descriptor.layer, - zbp_value, - zbp_current, - zbp_known, - node_capacity, - z_level_offsets, - z_level_widths, - z_level_heights, - z_levels)) { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 2u, 0u); - return; - } - - for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { - const uint x = block_idx % subband.num_cbs_x; - const uint y = block_idx / subband.num_cbs_x; - const J2kPacketBlock block = blocks[subband.block_offset + block_idx]; - const uint state_block_index = subband_state_block_offset + block_idx; - uint previously_included = block.previously_included; - uint local_l_block = block.l_block; - if (has_descriptor) { - previously_included = state_blocks[state_block_index].previously_included; - local_l_block = state_blocks[state_block_index].l_block; - } - if (previously_included == 0u) { - j2k_packet_tree_encode( - x, - y, - descriptor.layer + 1u, - inc_value, - inc_current, - inc_known, - level_offsets, - level_widths, - levels, - writer - ); - if (block.num_coding_passes == 0u) { - continue; - } - j2k_packet_tree_encode( - x, - y, - block.num_zero_bitplanes + 1u, - zbp_value, - zbp_current, - zbp_known, - z_level_offsets, - z_level_widths, - z_levels, - writer - ); - } else if (block.num_coding_passes > 0u) { - j2k_packet_write_bit(writer, 1u); - } else { - j2k_packet_write_bit(writer, 0u); - continue; - } - - if (block.block_coding_mode == 0u) { - const uint num_bits = - j2k_packet_bits_for_length(local_l_block, block.num_coding_passes); - j2k_packet_encode_num_passes(block.num_coding_passes, writer); - j2k_packet_encode_length(block.data_len, local_l_block, num_bits, writer); - } else if (block.block_coding_mode == 1u) { - const uint num_bits = - j2k_packet_bits_for_ht_length(local_l_block, block.num_coding_passes); - j2k_packet_encode_num_ht_passes(block.num_coding_passes, writer); - j2k_packet_encode_length(block.data_len, local_l_block, num_bits, writer); - } else { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 7u, block.reserved0); - return; - } - if (has_descriptor) { - state_blocks[state_block_index].previously_included = 1u; - state_blocks[state_block_index].l_block = local_l_block; - } - } - } - j2k_packet_writer_finish(writer); - } - - if (writer.failed != 0u || out_len + writer.len > params.output_capacity) { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 3u, 0u); - return; - } - for (uint idx = 0u; idx < writer.len; ++idx) { - out[out_len + idx] = header[idx]; - } - out_len += writer.len; - if (writer.len > 0u && header[writer.len - 1u] == uchar(0xFFu)) { - if (out_len >= params.output_capacity) { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 4u, 0u); - return; - } - out[out_len] = uchar(0u); - out_len += 1u; - } - - if (any_data) { - for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { - const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; - for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { - const J2kPacketBlock block = blocks[subband.block_offset + block_idx]; - if (block.num_coding_passes == 0u) { - continue; - } - if (out_len + block.data_len > params.output_capacity) { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 5u, 0u); - return; - } - for (uint byte_idx = 0u; byte_idx < block.data_len; ++byte_idx) { - out[out_len + byte_idx] = payload[block.data_offset + byte_idx]; - } - out_len += block.data_len; - } - } - } - } - - j2k_set_packet_status(status, J2K_ENCODE_STATUS_OK, 0u, out_len); -} - -struct J2kBatchedPacketEncodeJob { - uint resolution_offset; - uint subband_offset; - uint block_offset; - uint descriptor_offset; - uint state_block_offset; - uint output_offset; - uint header_offset; - uint scratch_offset; - uint resolution_count; - uint num_layers; - uint num_components; - uint code_block_count; - uint subband_count; - uint descriptor_count; - uint output_capacity; - uint header_capacity; - uint scratch_node_capacity; -}; - -kernel void j2k_encode_packetization_batched( - device const J2kPacketResolution *all_resolutions [[buffer(0)]], - device const J2kPacketSubband *all_subbands [[buffer(1)]], - device const J2kPacketBlock *all_blocks [[buffer(2)]], - device const uchar *payload [[buffer(3)]], - device uchar *all_out [[buffer(4)]], - device uchar *all_header [[buffer(5)]], - device uint *all_tree_scratch [[buffer(6)]], - device const J2kBatchedPacketEncodeJob *jobs [[buffer(7)]], - device J2kPacketEncodeStatus *all_status [[buffer(8)]], - device const J2kPacketDescriptor *all_descriptors [[buffer(9)]], - device J2kPacketStateBlock *all_state_blocks [[buffer(10)]], - uint gid [[thread_position_in_grid]] -) { - const J2kBatchedPacketEncodeJob job = jobs[gid]; - device const J2kPacketResolution *resolutions = all_resolutions + job.resolution_offset; - device const J2kPacketSubband *subbands = all_subbands + job.subband_offset; - device const J2kPacketBlock *blocks = all_blocks + job.block_offset; - device uchar *out = all_out + job.output_offset; - device uchar *header = all_header + job.header_offset; - device uint *tree_scratch = all_tree_scratch + job.scratch_offset; - device J2kPacketEncodeStatus *status = all_status + gid; - device const J2kPacketDescriptor *descriptors = all_descriptors + job.descriptor_offset; - device J2kPacketStateBlock *state_blocks = all_state_blocks + job.state_block_offset; - - J2kPacketEncodeParams params; - params.resolution_count = job.resolution_count; - params.num_layers = job.num_layers; - params.num_components = job.num_components; - params.code_block_count = job.code_block_count; - params.subband_count = job.subband_count; - params.descriptor_count = job.descriptor_count; - params.output_capacity = job.output_capacity; - params.header_capacity = job.header_capacity; - params.scratch_node_capacity = job.scratch_node_capacity; - - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 0u, 0u); - - const uint node_capacity = params.scratch_node_capacity; - device uint *inc_value = tree_scratch; - device uint *inc_current = tree_scratch + node_capacity; - device uint *inc_known = tree_scratch + node_capacity * 2u; - device uint *zbp_value = tree_scratch + node_capacity * 3u; - device uint *zbp_current = tree_scratch + node_capacity * 4u; - device uint *zbp_known = tree_scratch + node_capacity * 5u; - - uint out_len = 0u; - const uint packet_count = - params.descriptor_count > 0u ? params.descriptor_count : params.resolution_count; - for (uint packet_order_idx = 0u; packet_order_idx < packet_count; ++packet_order_idx) { - const bool has_descriptor = params.descriptor_count > 0u; - const J2kPacketDescriptor descriptor = has_descriptor - ? descriptors[packet_order_idx] - : J2kPacketDescriptor{packet_order_idx, packet_order_idx, 0u, packet_order_idx, 0u, 0u, 0u, 0u}; - const uint packet_index = descriptor.packet_index; - if (packet_index >= params.resolution_count) { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 6u, 0u); - return; - } - const J2kPacketResolution resolution = resolutions[packet_index]; - uint state_block_cursor = descriptor.state_block_offset; - bool any_data = false; - for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { - const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; - for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { - if (blocks[subband.block_offset + block_idx].num_coding_passes > 0u) { - any_data = true; - break; - } - } - } - - thread J2kPacketBitWriter writer; - j2k_packet_writer_init(writer, header, params.header_capacity); - if (!any_data) { - j2k_packet_write_bit(writer, 0u); - j2k_packet_writer_finish(writer); - } else { - j2k_packet_write_bit(writer, 1u); - for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { - const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; - const uint subband_state_block_offset = state_block_cursor; - state_block_cursor += subband.block_count; - thread uint level_offsets[16]; - thread uint level_widths[16]; - thread uint level_heights[16]; - uint levels = 0u; - if (!j2k_packet_prepare_tree( - blocks, - subband.block_offset, - subband.block_count, - subband.num_cbs_x, - subband.num_cbs_y, - false, - descriptor.layer, - inc_value, - inc_current, - inc_known, - node_capacity, - level_offsets, - level_widths, - level_heights, - levels)) { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 1u, 0u); - return; - } - thread uint z_level_offsets[16]; - thread uint z_level_widths[16]; - thread uint z_level_heights[16]; - uint z_levels = 0u; - if (!j2k_packet_prepare_tree( - blocks, - subband.block_offset, - subband.block_count, - subband.num_cbs_x, - subband.num_cbs_y, - true, - descriptor.layer, - zbp_value, - zbp_current, - zbp_known, - node_capacity, - z_level_offsets, - z_level_widths, - z_level_heights, - z_levels)) { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 2u, 0u); - return; - } - - for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { - const uint x = block_idx % subband.num_cbs_x; - const uint y = block_idx / subband.num_cbs_x; - const J2kPacketBlock block = blocks[subband.block_offset + block_idx]; - const uint state_block_index = subband_state_block_offset + block_idx; - uint previously_included = block.previously_included; - uint local_l_block = block.l_block; - if (has_descriptor) { - previously_included = state_blocks[state_block_index].previously_included; - local_l_block = state_blocks[state_block_index].l_block; - } - if (previously_included == 0u) { - j2k_packet_tree_encode( - x, - y, - descriptor.layer + 1u, - inc_value, - inc_current, - inc_known, - level_offsets, - level_widths, - levels, - writer - ); - if (block.num_coding_passes == 0u) { - continue; - } - j2k_packet_tree_encode( - x, - y, - block.num_zero_bitplanes + 1u, - zbp_value, - zbp_current, - zbp_known, - z_level_offsets, - z_level_widths, - z_levels, - writer - ); - } else if (block.num_coding_passes > 0u) { - j2k_packet_write_bit(writer, 1u); - } else { - j2k_packet_write_bit(writer, 0u); - continue; - } - - if (block.block_coding_mode == 0u) { - const uint num_bits = - j2k_packet_bits_for_length(local_l_block, block.num_coding_passes); - j2k_packet_encode_num_passes(block.num_coding_passes, writer); - j2k_packet_encode_length(block.data_len, local_l_block, num_bits, writer); - } else if (block.block_coding_mode == 1u) { - const uint num_bits = - j2k_packet_bits_for_ht_length(local_l_block, block.num_coding_passes); - j2k_packet_encode_num_ht_passes(block.num_coding_passes, writer); - j2k_packet_encode_length(block.data_len, local_l_block, num_bits, writer); - } else { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 7u, block.reserved0); - return; - } - if (has_descriptor) { - state_blocks[state_block_index].previously_included = 1u; - state_blocks[state_block_index].l_block = local_l_block; - } - } - } - j2k_packet_writer_finish(writer); - } - - if (writer.failed != 0u || out_len + writer.len > params.output_capacity) { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 3u, 0u); - return; - } - for (uint idx = 0u; idx < writer.len; ++idx) { - out[out_len + idx] = header[idx]; - } - out_len += writer.len; - if (writer.len > 0u && header[writer.len - 1u] == uchar(0xFFu)) { - if (out_len >= params.output_capacity) { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 4u, 0u); - return; - } - out[out_len] = uchar(0u); - out_len += 1u; - } - - if (any_data) { - for (uint sb_idx = 0u; sb_idx < resolution.subband_count; ++sb_idx) { - const J2kPacketSubband subband = subbands[resolution.subband_offset + sb_idx]; - for (uint block_idx = 0u; block_idx < subband.block_count; ++block_idx) { - const J2kPacketBlock block = blocks[subband.block_offset + block_idx]; - if (block.num_coding_passes == 0u) { - continue; - } - if (out_len + block.data_len > params.output_capacity) { - j2k_set_packet_status(status, J2K_ENCODE_STATUS_FAIL, 5u, 0u); - return; - } - for (uint byte_idx = 0u; byte_idx < block.data_len; ++byte_idx) { - out[out_len + byte_idx] = payload[block.data_offset + byte_idx]; - } - out_len += block.data_len; - } - } - } - } - - j2k_set_packet_status(status, J2K_ENCODE_STATUS_OK, 0u, out_len); -} diff --git a/crates/signinum-j2k-metal/src/fdwt.metal b/crates/signinum-j2k-metal/src/fdwt.metal deleted file mode 100644 index 3d42960e..00000000 --- a/crates/signinum-j2k-metal/src/fdwt.metal +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#include -using namespace metal; - -struct J2kForwardDwt53Params { - uint full_width; - uint current_width; - uint current_height; - uint low_width; - uint low_height; -}; - -inline float j2k_fdwt53_predict_row( - device const float *src, - uint row_base, - uint width, - uint high_index -) { - const uint odd = high_index * 2u + 1u; - const uint last_even = (width % 2u == 0u) ? width - 2u : width - 1u; - const float left = src[row_base + odd - 1u]; - const float right = (odd + 1u < width) ? src[row_base + odd + 1u] : src[row_base + last_even]; - return src[row_base + odd] - floor((left + right) * 0.5f); -} - -inline float j2k_fdwt53_predict_col( - device const float *src, - uint x, - uint full_width, - uint height, - uint high_index -) { - const uint odd = high_index * 2u + 1u; - const uint last_even = (height % 2u == 0u) ? height - 2u : height - 1u; - const float top = src[(odd - 1u) * full_width + x]; - const float bottom = (odd + 1u < height) - ? src[(odd + 1u) * full_width + x] - : src[last_even * full_width + x]; - return src[odd * full_width + x] - floor((top + bottom) * 0.5f); -} - -kernel void j2k_forward_dwt53_horizontal( - device const float *src [[buffer(0)]], - device float *dst [[buffer(1)]], - constant J2kForwardDwt53Params ¶ms [[buffer(2)]], - uint2 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.current_width || gid.y >= params.current_height) { - return; - } - - const uint row_base = gid.y * params.full_width; - if (gid.x < params.low_width) { - const uint even = gid.x * 2u; - const float left = gid.x > 0u - ? j2k_fdwt53_predict_row(src, row_base, params.current_width, gid.x - 1u) - : j2k_fdwt53_predict_row(src, row_base, params.current_width, 0u); - const float right = even + 1u < params.current_width - ? j2k_fdwt53_predict_row(src, row_base, params.current_width, gid.x) - : left; - dst[row_base + gid.x] = src[row_base + even] + floor((left + right) * 0.25f + 0.5f); - return; - } - - const uint high_index = gid.x - params.low_width; - dst[row_base + gid.x] = j2k_fdwt53_predict_row( - src, - row_base, - params.current_width, - high_index - ); -} - -kernel void j2k_forward_dwt53_vertical( - device const float *src [[buffer(0)]], - device float *dst [[buffer(1)]], - constant J2kForwardDwt53Params ¶ms [[buffer(2)]], - uint2 gid [[thread_position_in_grid]] -) { - if (gid.x >= params.current_width || gid.y >= params.current_height) { - return; - } - - if (gid.y < params.low_height) { - const uint even = gid.y * 2u; - const float top = gid.y > 0u - ? j2k_fdwt53_predict_col(src, gid.x, params.full_width, params.current_height, gid.y - 1u) - : j2k_fdwt53_predict_col(src, gid.x, params.full_width, params.current_height, 0u); - const float bottom = even + 1u < params.current_height - ? j2k_fdwt53_predict_col(src, gid.x, params.full_width, params.current_height, gid.y) - : top; - dst[gid.y * params.full_width + gid.x] = - src[even * params.full_width + gid.x] + floor((top + bottom) * 0.25f + 0.5f); - return; - } - - const uint high_index = gid.y - params.low_height; - dst[gid.y * params.full_width + gid.x] = j2k_fdwt53_predict_col( - src, - gid.x, - params.full_width, - params.current_height, - high_index - ); -} diff --git a/crates/signinum-j2k-metal/src/profile.rs b/crates/signinum-j2k-metal/src/profile.rs deleted file mode 100644 index f4000840..00000000 --- a/crates/signinum-j2k-metal/src/profile.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -pub(crate) fn gpu_route_profile_enabled() -> bool { - signinum_profile::gpu_route_profile_enabled() -} - -pub(crate) fn emit_gpu_route_profile(codec: &str, op: &str, path: &str, fields: &[(K, V)]) -where - K: AsRef, - V: AsRef, -{ - debug_assert_eq!(op, "gpu_route"); - signinum_profile::emit_gpu_route_profile(codec, path, fields); -} diff --git a/crates/signinum-j2k-metal/tests/bench_harness.rs b/crates/signinum-j2k-metal/tests/bench_harness.rs deleted file mode 100644 index 213386e1..00000000 --- a/crates/signinum-j2k-metal/tests/bench_harness.rs +++ /dev/null @@ -1,156 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#[test] -fn compare_bench_adaptive_region_scaled_uses_auto_device_submit() { - let source = include_str!("../benches/common/mod.rs"); - - assert!( - source.contains("fn signinum_adaptive_decode_tile_batch_region_scaled_to_device("), - "compare bench must expose a device Auto helper for adaptive ROI+scaled tile batches" - ); - - let device_helper = source - .split("fn signinum_adaptive_decode_tile_batch_region_scaled_to_device(") - .nth(1) - .expect("adaptive ROI+scaled device helper must exist"); - let device_helper_body = device_helper - .split("pub(crate) fn signinum_adaptive_decode_tile_batch(") - .next() - .expect("adaptive ROI+scaled device helper body must be delimited by the next helper"); - assert!( - device_helper_body.contains("BackendRequest::Auto"), - "adaptive ROI+scaled device helper must submit through BackendRequest::Auto" - ); - - let adaptive_fn = source - .split("pub(crate) fn signinum_adaptive_decode_tile_batch_region_scaled(") - .nth(1) - .expect("adaptive ROI+scaled benchmark helper must exist"); - let adaptive_body = adaptive_fn - .split("fn should_auto_use_direct_grayscale_input(") - .next() - .expect("adaptive ROI+scaled helper body must be delimited by the next helper"); - assert!( - adaptive_body.contains("signinum_adaptive_decode_tile_batch_region_scaled_to_device("), - "adaptive ROI+scaled benchmark helper must call the Auto device-submit helper" - ); -} - -#[test] -fn compare_bench_distinct_and_external_region_scaled_expose_auto_variants() { - let common = include_str!("../benches/common/mod.rs"); - let compare = include_str!("../benches/compare.rs"); - - for helper in [ - "signinum_adaptive_decode_tile_batch_region_scaled_distinct", - "signinum_adaptive_decode_external_tile_batch_region_scaled", - ] { - assert!( - common.contains(&format!("pub(crate) fn {helper}(")), - "common benchmark helpers must expose {helper}" - ); - - let helper_body = common - .split(&format!("pub(crate) fn {helper}(")) - .nth(1) - .expect("helper must exist") - .split("pub(crate) fn signinum_metal_supports") - .next() - .expect("helper body must be delimited before support helpers"); - assert!( - helper_body.contains("BackendRequest::Auto"), - "{helper} must submit through BackendRequest::Auto" - ); - assert!( - compare.matches(helper).count() >= 2, - "compare bench must import and register {helper}" - ); - } -} - -#[test] -fn compare_bench_batch_sizes_are_env_configurable() { - let common = include_str!("../benches/common/mod.rs"); - let compare = include_str!("../benches/compare.rs"); - - assert!( - common.contains("SIGNINUM_J2K_TILE_BATCH_SIZES"), - "J2K compare benches must expose env-controlled batch-size sweeps" - ); - assert!( - common.contains("DEFAULT_J2K_TILE_BATCH_SIZES") && common.contains("&[16, 32, 64, 128]"), - "default J2K batch-size sweep must cover 16/32/64/128" - ); - assert!( - compare.contains("let batch_sizes = j2k_tile_batch_sizes();"), - "compare bench must parse batch sizes once and reuse them across batch groups" - ); - assert!( - compare.contains("external_wsi_tile_batches(max_batch_size)"), - "external WSI loader must load enough tiles for the largest requested batch" - ); -} - -#[test] -fn compare_bench_keeps_multiple_htj2k_gray_sizes() { - let common = include_str!("../benches/common/mod.rs"); - - assert!( - common.contains("inputs.extend(ht_bench_inputs()"), - "bench_inputs must include all generated HTJ2K size candidates, not only the first success" - ); - assert!( - common.contains("(\"htj2k_gray_1024\", 1024_u32, 1024_u32)") - && common.contains("(\"htj2k_gray_512\", 512_u32, 512_u32)"), - "HTJ2K bench coverage must include 1024 and 512 grayscale tiles" - ); -} - -#[test] -fn external_wsi_bench_groups_by_codec_family() { - let common = include_str!("../benches/common/mod.rs"); - - assert!( - common.contains("enum ExternalCodecFamily"), - "external WSI batches must track classic J2K versus HTJ2K separately" - ); - assert!( - common.contains("external_codec_family("), - "external WSI loader must classify each tile/frame by compressed transfer syntax" - ); - assert!( - common.contains("\"htj2k_gray8\"") && common.contains("\"j2k_gray8\""), - "external WSI batch labels must keep HTJ2K and classic J2K timing separate" - ); -} - -#[test] -fn metal_region_scaled_benches_gate_to_grayscale_direct_modes() { - let common = include_str!("../benches/common/mod.rs"); - - assert!( - common.contains("fn supports_metal_region_scaled_mode("), - "ROI+scaled Metal benchmark support must have a cheap format gate" - ); - assert!( - common.contains("matches!(mode, DecodeMode::Gray8 | DecodeMode::Gray16)"), - "ROI+scaled Metal benchmarks must only advertise direct grayscale modes until RGB is GPU-native" - ); - for helper in [ - "signinum_metal_supports_region_scaled", - "signinum_metal_supports_tile_batch_region_scaled_distinct", - "signinum_metal_supports_external_tile_batch_region_scaled", - ] { - let helper_body = common - .split(&format!("pub(crate) fn {helper}(")) - .nth(1) - .expect("support helper must exist") - .split("pub(crate) fn") - .next() - .expect("support helper body must be delimited"); - assert!( - helper_body.contains("supports_metal_region_scaled_mode("), - "{helper} must skip unsupported ROI+scaled Metal formats before probing decode" - ); - } -} diff --git a/crates/signinum-j2k-metal/tests/shader_integrity.rs b/crates/signinum-j2k-metal/tests/shader_integrity.rs deleted file mode 100644 index a23d275c..00000000 --- a/crates/signinum-j2k-metal/tests/shader_integrity.rs +++ /dev/null @@ -1,49 +0,0 @@ -const COMPUTE_SOURCE: &str = include_str!("../src/compute.rs"); -const SHADER_SOURCES: &[&str] = &[ - COMPUTE_SOURCE, - include_str!("../src/classic.metal"), - include_str!("../src/encode_bitstream.metal"), - include_str!("../src/fdwt.metal"), - include_str!("../src/ht_cleanup.metal"), - include_str!("../src/idwt.metal"), - include_str!("../src/mct.metal"), - include_str!("../src/store.metal"), -]; - -#[test] -fn metal_kernels_are_wired_to_host_pipelines() { - let unused = SHADER_SOURCES - .iter() - .flat_map(|source| kernel_names(source)) - .filter(|name| !host_compiles_pipeline(name)) - .collect::>(); - - assert!( - unused.is_empty(), - "Metal kernels must be compiled by host pipeline setup or removed: {unused:?}" - ); -} - -fn kernel_names(source: &str) -> Vec { - source - .lines() - .filter_map(|line| { - let line = line.trim_start(); - let rest = line.strip_prefix("kernel void ")?; - let name = rest - .split(|ch: char| !(ch.is_ascii_alphanumeric() || ch == '_')) - .next() - .expect("kernel name follows kernel declaration"); - Some(name.to_owned()) - }) - .collect() -} - -fn host_compiles_pipeline(name: &str) -> bool { - let quoted = format!("\"{name}\""); - COMPUTE_SOURCE.match_indices("ed).any(|(index, _)| { - let context_start = index.saturating_sub(96); - let context = &COMPUTE_SOURCE[context_start..index]; - context.contains("get_function(") || context.contains("pipeline(") - }) -} diff --git a/crates/signinum-j2k-native/README.md b/crates/signinum-j2k-native/README.md deleted file mode 100644 index c7999e30..00000000 --- a/crates/signinum-j2k-native/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# signinum-j2k-native - -Implementation crate for `signinum-j2k`. - -Most users should install `signinum-j2k` instead: - -```sh -cargo add signinum-j2k -``` - -This crate contains the pure-Rust JPEG 2000 / HTJ2K engine used by the public -`signinum-j2k` CPU-first 1.0 API. It is published so `signinum-j2k` can be consumed -from crates.io, but downstream users should prefer the stable `signinum-j2k` -wrapper unless they intentionally need engine-level APIs. diff --git a/crates/signinum-j2k-native/benches/tier1_bitplane.rs b/crates/signinum-j2k-native/benches/tier1_bitplane.rs deleted file mode 100644 index 0cfcbda6..00000000 --- a/crates/signinum-j2k-native/benches/tier1_bitplane.rs +++ /dev/null @@ -1,167 +0,0 @@ -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; -use signinum_j2k_native::{ - decode_j2k_code_block_scalar, encode_j2k_code_block_scalar_with_style, J2kCodeBlockDecodeJob, - J2kCodeBlockStyle, J2kSubBandType, -}; - -fn generated_coefficients(width: u32, height: u32, seed: u32) -> Vec { - let mut state = seed ^ 0xa24b_aed4; - let mut coefficients = Vec::with_capacity(width as usize * height as usize); - for idx in 0..width * height { - state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); - let value = ((state >> 16) & 0x01ff) as i32 - 255; - coefficients.push(if (idx + seed).is_multiple_of(13) { - 0 - } else { - value - }); - } - coefficients -} - -fn default_style() -> J2kCodeBlockStyle { - J2kCodeBlockStyle { - selective_arithmetic_coding_bypass: false, - reset_context_probabilities: false, - termination_on_each_pass: false, - vertically_causal_context: false, - segmentation_symbols: false, - } -} - -fn bench_tier1_decode(c: &mut Criterion) { - let mut group = c.benchmark_group("tier1_bitplane_decode"); - let cases = [ - ("default", default_style(), J2kSubBandType::LowLow), - ( - "bypass", - J2kCodeBlockStyle { - selective_arithmetic_coding_bypass: true, - ..default_style() - }, - J2kSubBandType::HighLow, - ), - ( - "segmented", - J2kCodeBlockStyle { - termination_on_each_pass: true, - reset_context_probabilities: true, - segmentation_symbols: true, - ..default_style() - }, - J2kSubBandType::HighHigh, - ), - ( - "vertically_causal", - J2kCodeBlockStyle { - vertically_causal_context: true, - ..default_style() - }, - J2kSubBandType::LowHigh, - ), - ]; - - for (name, style, sub_band_type) in cases { - let width = 64; - let height = 64; - let total_bitplanes = 10; - let coefficients = generated_coefficients(width, height, 0x5151_0000); - let encoded = encode_j2k_code_block_scalar_with_style( - &coefficients, - width, - height, - sub_band_type, - total_bitplanes, - style, - ) - .expect("encode code block"); - let job = J2kCodeBlockDecodeJob { - data: &encoded.data, - segments: &encoded.segments, - width, - height, - output_stride: width as usize, - missing_bit_planes: encoded.missing_bit_planes, - number_of_coding_passes: encoded.number_of_coding_passes, - total_bitplanes, - sub_band_type, - style, - strict: true, - dequantization_step: 1.0, - }; - let mut output = vec![0.0; width as usize * height as usize]; - - group.bench_with_input(BenchmarkId::new("decode_64x64", name), &job, |b, job| { - b.iter(|| { - decode_j2k_code_block_scalar(*job, &mut output).expect("decode code block"); - black_box(&output); - }); - }); - } - - group.finish(); -} - -fn bench_tier1_encode(c: &mut Criterion) { - let mut group = c.benchmark_group("tier1_bitplane_encode"); - let cases = [ - ("default", default_style(), J2kSubBandType::LowLow), - ( - "bypass", - J2kCodeBlockStyle { - selective_arithmetic_coding_bypass: true, - ..default_style() - }, - J2kSubBandType::HighLow, - ), - ( - "segmented", - J2kCodeBlockStyle { - termination_on_each_pass: true, - reset_context_probabilities: true, - segmentation_symbols: true, - ..default_style() - }, - J2kSubBandType::HighHigh, - ), - ( - "vertically_causal", - J2kCodeBlockStyle { - vertically_causal_context: true, - ..default_style() - }, - J2kSubBandType::LowHigh, - ), - ]; - - for (name, style, sub_band_type) in cases { - let width = 64; - let height = 64; - let total_bitplanes = 10; - let coefficients = generated_coefficients(width, height, 0x7171_0000); - - group.bench_with_input( - BenchmarkId::new("encode_64x64", name), - &coefficients, - |b, coefficients| { - b.iter(|| { - let encoded = encode_j2k_code_block_scalar_with_style( - black_box(coefficients), - width, - height, - sub_band_type, - total_bitplanes, - style, - ) - .expect("encode code block"); - black_box(encoded); - }); - }, - ); - } - - group.finish(); -} - -criterion_group!(benches, bench_tier1_decode, bench_tier1_encode); -criterion_main!(benches); diff --git a/crates/signinum-j2k-native/src/j2c/encode.rs b/crates/signinum-j2k-native/src/j2c/encode.rs deleted file mode 100644 index c64dd5d7..00000000 --- a/crates/signinum-j2k-native/src/j2c/encode.rs +++ /dev/null @@ -1,1799 +0,0 @@ -//! Top-level JPEG 2000 encode orchestration. -//! -//! Coordinates the full encoding pipeline: -//! pixels → MCT → DWT → quantize → EBCOT T1 → T2 → codestream -//! -//! Supports both lossless (5-3 reversible) and lossy (9-7 irreversible) encoding. - -use alloc::vec; -use alloc::vec::Vec; - -#[cfg(feature = "parallel")] -use rayon::prelude::*; - -use super::bitplane_encode; -use super::build::SubBandType; -use super::codestream_write::{self, BlockCodingMode, EncodeParams}; -use super::fdwt::{self, DwtDecomposition}; -use super::forward_mct; -use super::ht_block_encode; -use super::packet_encode::{self, CodeBlockPacketData, ResolutionPacket, SubbandPrecinct}; -use super::quantize::{self, QuantStepSize}; -use crate::math::{floor_f32, log2_f32}; -use crate::profile; -use crate::{ - CpuOnlyJ2kEncodeStageAccelerator, EncodedJ2kCodeBlock, J2kEncodeStageAccelerator, - J2kForwardDwt53Job, J2kForwardDwt53Level, J2kForwardDwt53Output, J2kForwardRctJob, - J2kPacketizationBlockCodingMode, J2kPacketizationCodeBlock, J2kPacketizationEncodeJob, - J2kPacketizationPacketDescriptor, J2kPacketizationResolution, J2kPacketizationSubband, - J2kSubBandType, J2kTier1CodeBlockEncodeJob, -}; -use crate::{DecodeSettings, Image}; - -/// Encoding options for JPEG 2000. -#[derive(Debug, Clone)] -pub struct EncodeOptions { - /// Number of decomposition levels (default: 5). - pub num_decomposition_levels: u8, - /// Use reversible (lossless) transform (default: true). - pub reversible: bool, - /// Code-block width exponent minus 2 (default: 4, meaning 2^6=64). - pub code_block_width_exp: u8, - /// Code-block height exponent minus 2 (default: 4, meaning 2^6=64). - pub code_block_height_exp: u8, - /// Number of guard bits (default: 1 for reversible, 2 for irreversible). - pub guard_bits: u8, - /// Encode using HT block coding (HTJ2K / Part 15) instead of classic EBCOT. - pub use_ht_block_coding: bool, - /// Packet progression order to write in COD and use for packetization. - pub progression_order: EncodeProgressionOrder, - /// Write a TLM marker segment for the single tile-part. - pub write_tlm: bool, - /// Apply the JPEG 2000 multi-component color transform for 3+ component inputs. - pub use_mct: bool, - /// Decode and verify HTJ2K codestreams inside the native encoder. - pub validate_high_throughput_codestream: bool, -} - -impl Default for EncodeOptions { - fn default() -> Self { - Self { - num_decomposition_levels: 5, - reversible: true, - code_block_width_exp: 4, - code_block_height_exp: 4, - guard_bits: 1, - use_ht_block_coding: false, - progression_order: EncodeProgressionOrder::Lrcp, - write_tlm: false, - use_mct: true, - validate_high_throughput_codestream: true, - } - } -} - -/// JPEG 2000 packet progression orders supported by the encoder. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum EncodeProgressionOrder { - /// Layer-resolution-component-position progression. - #[default] - Lrcp, - /// Resolution-position-component-layer progression. - Rpcl, -} - -/// Encode pixel data into a JPEG 2000 codestream. -/// -/// # Arguments -/// * `pixels` — Raw pixel data. For 8-bit: one byte per sample. For >8-bit: two bytes per sample (little-endian u16). -/// * `width` — Image width in pixels. -/// * `height` — Image height in pixels. -/// * `num_components` — Number of components (1 for grayscale, 3 for RGB). -/// * `bit_depth` — Bits per sample (e.g., 8, 12, 16). -/// * `signed` — Whether samples are signed. -/// * `options` — Encoding parameters. -/// -/// # Returns -/// The encoded JPEG 2000 codestream bytes (`.j2c` format). -pub fn encode( - pixels: &[u8], - width: u32, - height: u32, - num_components: u8, - bit_depth: u8, - signed: bool, - options: &EncodeOptions, -) -> Result, &'static str> { - let mut accelerator = CpuOnlyJ2kEncodeStageAccelerator; - encode_with_accelerator( - pixels, - width, - height, - num_components, - bit_depth, - signed, - options, - &mut accelerator, - ) -} - -/// Encode pixel data into a JPEG 2000 codestream using optional encode-stage hooks. -/// -/// Stage hooks may accelerate forward RCT, forward 5/3 DWT, Tier-1 code-block -/// encode, and packetization. Returning fallback from a hook preserves the CPU -/// baseline for that stage. -pub fn encode_with_accelerator( - pixels: &[u8], - width: u32, - height: u32, - num_components: u8, - bit_depth: u8, - signed: bool, - options: &EncodeOptions, - accelerator: &mut impl J2kEncodeStageAccelerator, -) -> Result, &'static str> { - let block_coding_mode = block_coding_mode(options); - let codestream = encode_impl( - pixels, - width, - height, - num_components, - bit_depth, - signed, - options, - block_coding_mode, - accelerator, - )?; - - if block_coding_mode == BlockCodingMode::HighThroughput - && options.validate_high_throughput_codestream - { - validate_htj2k_codestream( - &codestream, - pixels, - width, - height, - num_components, - bit_depth, - signed, - options.reversible, - )?; - } - - Ok(codestream) -} - -/// Encode pixel data into an HTJ2K codestream. -/// -/// Lossless HTJ2K output is self-validated before it is returned. -pub fn encode_htj2k( - pixels: &[u8], - width: u32, - height: u32, - num_components: u8, - bit_depth: u8, - signed: bool, - options: &EncodeOptions, -) -> Result, &'static str> { - let mut options = options.clone(); - options.use_ht_block_coding = true; - encode( - pixels, - width, - height, - num_components, - bit_depth, - signed, - &options, - ) -} - -fn block_coding_mode(options: &EncodeOptions) -> BlockCodingMode { - if options.use_ht_block_coding { - BlockCodingMode::HighThroughput - } else { - BlockCodingMode::Classic - } -} - -fn validate_htj2k_codestream( - codestream: &[u8], - pixels: &[u8], - width: u32, - height: u32, - num_components: u8, - bit_depth: u8, - signed: bool, - reversible: bool, -) -> Result<(), &'static str> { - let image = Image::new(codestream, &DecodeSettings::default()) - .map_err(|_| "generated HTJ2K codestream failed self-validation")?; - let decoded = image - .decode_native() - .map_err(|_| "generated HTJ2K codestream failed self-validation")?; - - if decoded.width != width - || decoded.height != height - || decoded.bit_depth != bit_depth - || decoded.num_components != num_components - { - return Err("generated HTJ2K codestream failed self-validation"); - } - - if reversible && !native_samples_equal(pixels, &decoded.data, bit_depth, signed) { - return Err("generated HTJ2K codestream did not roundtrip"); - } - - Ok(()) -} - -fn native_samples_equal(expected: &[u8], actual: &[u8], bit_depth: u8, signed: bool) -> bool { - if expected.len() != actual.len() { - return false; - } - - let bytes_per_sample = if bit_depth <= 8 { 1 } else { 2 }; - let sample_count = expected.len() / bytes_per_sample; - (0..sample_count).all(|sample_index| { - decode_native_sample(expected, sample_index, bit_depth, signed) - == decode_native_sample(actual, sample_index, bit_depth, signed) - }) -} - -fn decode_native_sample(bytes: &[u8], sample_index: usize, bit_depth: u8, signed: bool) -> i32 { - let byte_offset = sample_index * if bit_depth <= 8 { 1 } else { 2 }; - let mask = (1u32 << u32::from(bit_depth)) - 1; - let raw = if bit_depth <= 8 { - u32::from(bytes[byte_offset]) - } else { - u32::from(u16::from_le_bytes([ - bytes[byte_offset], - bytes[byte_offset + 1], - ])) - } & mask; - - if signed { - let shift = 32 - u32::from(bit_depth); - ((raw << shift) as i32) >> shift - } else { - raw as i32 - } -} - -fn encode_impl( - pixels: &[u8], - width: u32, - height: u32, - num_components: u8, - bit_depth: u8, - signed: bool, - options: &EncodeOptions, - block_coding_mode: BlockCodingMode, - accelerator: &mut impl J2kEncodeStageAccelerator, -) -> Result, &'static str> { - if width == 0 || height == 0 { - return Err("invalid dimensions"); - } - if num_components == 0 || num_components > 4 { - return Err("unsupported component count"); - } - if bit_depth == 0 || bit_depth > 16 { - return Err("unsupported bit depth"); - } - - let num_pixels = (width * height) as usize; - let bytes_per_sample = if bit_depth <= 8 { 1 } else { 2 }; - let expected_len = num_pixels * num_components as usize * bytes_per_sample; - if pixels.len() < expected_len { - return Err("pixel data too short"); - } - - let profile_enabled = profile::profile_stages_enabled(); - let total_start = profile::profile_now(profile_enabled); - - // Step 1: Convert pixel bytes to f32 component arrays - let stage_start = profile::profile_now(profile_enabled); - let mut components = deinterleave_to_f32(pixels, num_pixels, num_components, bit_depth, signed); - let deinterleave_us = profile::elapsed_us(stage_start); - - // Step 2: Apply forward MCT if RGB with 3+ components - let stage_start = profile::profile_now(profile_enabled); - let use_mct = options.use_mct && num_components >= 3; - if use_mct { - if options.reversible { - if !try_encode_forward_rct(&mut components, accelerator)? { - forward_mct::forward_rct(&mut components); - } - } else { - forward_mct::forward_ict(&mut components); - } - } - let mct_us = profile::elapsed_us(stage_start); - - // Step 3: Apply forward DWT to each component - let num_levels = options.num_decomposition_levels.min( - // Don't decompose more than the image supports - max_decomposition_levels(width, height), - ); - - let stage_start = profile::profile_now(profile_enabled); - let decompositions: Vec = components - .iter() - .map(|comp| { - encode_forward_dwt( - comp, - width, - height, - num_levels, - options.reversible, - accelerator, - ) - }) - .collect::, _>>()?; - let dwt_us = profile::elapsed_us(stage_start); - - // Step 4: Compute quantization step sizes - let guard_bits = if options.reversible { - if use_mct { - options.guard_bits.max(2) - } else { - options.guard_bits - } - } else { - options.guard_bits.max(2) - }; - - let step_sizes = - quantize::compute_step_sizes(bit_depth, num_levels, options.reversible, guard_bits); - - // Step 5: Quantize and encode code-blocks for each component - let cb_width = 1u32 << (options.code_block_width_exp + 2); - let cb_height = 1u32 << (options.code_block_height_exp + 2); - - let mut component_resolution_packets: Vec> = - Vec::with_capacity(num_components as usize); - - let stage_start = profile::profile_now(profile_enabled); - for decomp in decompositions.iter().take(num_components as usize) { - let mut packets = Vec::with_capacity(num_levels as usize + 1); - - // LL subband (resolution 0) - let ll_subband = prepare_subband( - &decomp.ll, - decomp.ll_width, - decomp.ll_height, - &step_sizes[0], - guard_bits, - options.reversible, - block_coding_mode, - cb_width, - cb_height, - SubBandType::LowLow, - )?; - packets.push(PreparedResolutionPacket { - subbands: vec![ll_subband], - }); - - // Higher resolution levels - for (level_idx, level) in decomp.levels.iter().enumerate() { - let step_base = 1 + level_idx * 3; - - // HL subband - let hl_subband = prepare_subband( - &level.hl, - level.high_width, - level.low_height, - &step_sizes[step_base], - guard_bits, - options.reversible, - block_coding_mode, - cb_width, - cb_height, - SubBandType::HighLow, - )?; - - // LH subband - let lh_subband = prepare_subband( - &level.lh, - level.low_width, - level.high_height, - &step_sizes[step_base + 1], - guard_bits, - options.reversible, - block_coding_mode, - cb_width, - cb_height, - SubBandType::LowHigh, - )?; - - // HH subband - let hh_subband = prepare_subband( - &level.hh, - level.high_width, - level.high_height, - &step_sizes[step_base + 2], - guard_bits, - options.reversible, - block_coding_mode, - cb_width, - cb_height, - SubBandType::HighHigh, - )?; - - packets.push(PreparedResolutionPacket { - subbands: vec![hl_subband, lh_subband, hh_subband], - }); - } - - component_resolution_packets.push(packets); - } - let subband_prepare_us = profile::elapsed_us(stage_start); - - let prepared_resolution_packets = - ordered_prepared_resolution_packets(component_resolution_packets, options)?; - let stage_start = profile::profile_now(profile_enabled); - let resolution_packets = - encode_prepared_resolution_packets(prepared_resolution_packets, accelerator)?; - let block_encode_us = profile::elapsed_us(stage_start); - - // Step 6: Form tile bitstream (T2) - let stage_start = profile::profile_now(profile_enabled); - let mut resolution_packets = resolution_packets; - let packetization_resolutions = public_packetization_resolutions(&resolution_packets); - let packet_descriptors = - packet_descriptors_for_order(resolution_packets.len(), 1, num_components)?; - let packetization_job = J2kPacketizationEncodeJob { - resolution_count: resolution_packets.len() as u32, - num_layers: 1, - num_components, - code_block_count: count_code_blocks(&resolution_packets)?, - progression_order: public_packetization_progression_order(options.progression_order), - packet_descriptors: &packet_descriptors, - resolutions: &packetization_resolutions, - }; - let tile_data = accelerator - .encode_packetization(packetization_job)? - .unwrap_or_else(|| { - packet_encode::form_tile_bitstream(&mut resolution_packets, 1, num_components) - }); - let packetize_us = profile::elapsed_us(stage_start); - - // Step 7: Write codestream - let stage_start = profile::profile_now(profile_enabled); - let quant_params: Vec<(u16, u16)> = step_sizes - .iter() - .map(|s| (s.exponent, s.mantissa)) - .collect(); - - let params = EncodeParams { - width, - height, - num_components, - bit_depth, - signed, - num_decomposition_levels: num_levels, - reversible: options.reversible, - code_block_width_exp: options.code_block_width_exp, - code_block_height_exp: options.code_block_height_exp, - num_layers: 1, - use_mct, - guard_bits, - block_coding_mode, - progression_order: options.progression_order, - write_tlm: options.write_tlm, - }; - - let codestream = codestream_write::write_codestream(¶ms, &tile_data, &quant_params); - let codestream_us = profile::elapsed_us(stage_start); - - if profile_enabled { - profile::emit_profile_row( - "encode", - "cpu", - &[ - ("deinterleave_us", deinterleave_us), - ("mct_us", mct_us), - ("dwt_us", dwt_us), - ("subband_prepare_us", subband_prepare_us), - ("block_encode_us", block_encode_us), - ("packetize_us", packetize_us), - ("codestream_us", codestream_us), - ("total_us", profile::elapsed_us(total_start)), - ], - ); - } - - Ok(codestream) -} - -fn try_encode_forward_rct( - components: &mut [Vec], - accelerator: &mut impl J2kEncodeStageAccelerator, -) -> Result { - debug_assert!(components.len() >= 3); - let (plane0, rest) = components.split_at_mut(1); - let (plane1, plane2) = rest.split_at_mut(1); - accelerator.encode_forward_rct(J2kForwardRctJob { - plane0: &mut plane0[0], - plane1: &mut plane1[0], - plane2: &mut plane2[0], - }) -} - -fn encode_forward_dwt( - component: &[f32], - width: u32, - height: u32, - num_levels: u8, - reversible: bool, - accelerator: &mut impl J2kEncodeStageAccelerator, -) -> Result { - if reversible { - if let Some(output) = accelerator.encode_forward_dwt53(J2kForwardDwt53Job { - samples: component, - width, - height, - num_levels, - })? { - return convert_forward_dwt53_output(output); - } - } - - Ok(fdwt::forward_dwt( - component, width, height, num_levels, reversible, - )) -} - -fn convert_forward_dwt53_output( - output: J2kForwardDwt53Output, -) -> Result { - validate_band_len(output.ll.len(), output.ll_width, output.ll_height)?; - let mut levels = Vec::with_capacity(output.levels.len()); - for level in output.levels { - validate_dwt53_level(&level)?; - levels.push(fdwt::DwtLevel { - hl: level.hl, - lh: level.lh, - hh: level.hh, - low_width: level.low_width, - low_height: level.low_height, - high_width: level.high_width, - high_height: level.high_height, - }); - } - Ok(DwtDecomposition { - ll: output.ll, - ll_width: output.ll_width, - ll_height: output.ll_height, - levels, - }) -} - -fn validate_dwt53_level(level: &J2kForwardDwt53Level) -> Result<(), &'static str> { - validate_band_len(level.hl.len(), level.high_width, level.low_height)?; - validate_band_len(level.lh.len(), level.low_width, level.high_height)?; - validate_band_len(level.hh.len(), level.high_width, level.high_height)?; - Ok(()) -} - -fn validate_band_len(actual: usize, width: u32, height: u32) -> Result<(), &'static str> { - let expected = (width as usize) - .checked_mul(height as usize) - .ok_or("accelerated DWT output dimensions overflow")?; - if actual != expected { - return Err("accelerated DWT output length mismatch"); - } - Ok(()) -} - -fn count_code_blocks(resolution_packets: &[ResolutionPacket]) -> Result { - let count = resolution_packets - .iter() - .flat_map(|resolution| resolution.subbands.iter()) - .try_fold(0usize, |acc, subband| { - acc.checked_add(subband.code_blocks.len()) - .ok_or("packetization code-block count overflow") - })?; - u32::try_from(count).map_err(|_| "packetization code-block count exceeds u32") -} - -fn packet_descriptors_for_order( - packet_count: usize, - num_layers: u8, - num_components: u8, -) -> Result, &'static str> { - if num_layers != 1 { - return Err("encode currently prepares one packet contribution layer"); - } - let component_count = usize::from(num_components).max(1); - (0..packet_count) - .map(|packet_index| { - Ok(J2kPacketizationPacketDescriptor { - packet_index: u32::try_from(packet_index) - .map_err(|_| "packet descriptor index exceeds u32")?, - state_index: u32::try_from(packet_index) - .map_err(|_| "packet descriptor state index exceeds u32")?, - layer: 0, - resolution: u32::try_from(packet_index / component_count) - .map_err(|_| "packet descriptor resolution exceeds u32")?, - component: u8::try_from(packet_index % component_count) - .map_err(|_| "packet descriptor component exceeds u8")?, - precinct: 0, - }) - }) - .collect() -} - -fn ordered_prepared_resolution_packets( - component_resolution_packets: Vec>, - options: &EncodeOptions, -) -> Result, &'static str> { - match options.progression_order { - EncodeProgressionOrder::Lrcp | EncodeProgressionOrder::Rpcl => { - lrcp_ordered_prepared_resolution_packets(component_resolution_packets) - } - } -} - -fn lrcp_ordered_prepared_resolution_packets( - component_resolution_packets: Vec>, -) -> Result, &'static str> { - let resolution_count = component_resolution_packets - .first() - .map_or(0usize, alloc::vec::Vec::len); - let mut component_iters: Vec<_> = component_resolution_packets - .into_iter() - .map(alloc::vec::Vec::into_iter) - .collect(); - let mut resolution_packets = - Vec::with_capacity(resolution_count.saturating_mul(component_iters.len())); - - for _resolution in 0..resolution_count { - for component in &mut component_iters { - resolution_packets.push( - component - .next() - .ok_or("component packet resolution count mismatch")?, - ); - } - } - - if component_iters - .iter_mut() - .any(|component| component.next().is_some()) - { - return Err("component packet resolution count mismatch"); - } - - Ok(resolution_packets) -} - -fn public_packetization_progression_order( - progression_order: EncodeProgressionOrder, -) -> crate::J2kPacketizationProgressionOrder { - match progression_order { - EncodeProgressionOrder::Lrcp => crate::J2kPacketizationProgressionOrder::Lrcp, - EncodeProgressionOrder::Rpcl => crate::J2kPacketizationProgressionOrder::Rpcl, - } -} - -fn public_packetization_resolutions( - resolution_packets: &[ResolutionPacket], -) -> Vec> { - resolution_packets - .iter() - .map(|resolution| J2kPacketizationResolution { - subbands: resolution - .subbands - .iter() - .map(|subband| J2kPacketizationSubband { - code_blocks: subband - .code_blocks - .iter() - .map(|code_block| J2kPacketizationCodeBlock { - data: &code_block.data, - num_coding_passes: code_block.num_coding_passes, - num_zero_bitplanes: code_block.num_zero_bitplanes, - previously_included: code_block.previously_included, - l_block: code_block.l_block, - block_coding_mode: public_packetization_block_coding_mode( - code_block.block_coding_mode, - ), - }) - .collect(), - num_cbs_x: subband.num_cbs_x, - num_cbs_y: subband.num_cbs_y, - }) - .collect(), - }) - .collect() -} - -fn public_packetization_block_coding_mode( - block_coding_mode: BlockCodingMode, -) -> J2kPacketizationBlockCodingMode { - match block_coding_mode { - BlockCodingMode::Classic => J2kPacketizationBlockCodingMode::Classic, - BlockCodingMode::HighThroughput => J2kPacketizationBlockCodingMode::HighThroughput, - } -} - -struct PreparedEncodeCodeBlock { - coefficients: Vec, - width: u32, - height: u32, -} - -struct PreparedEncodeSubband { - code_blocks: Vec, - num_cbs_x: u32, - num_cbs_y: u32, - sub_band_type: SubBandType, - total_bitplanes: u8, - block_coding_mode: BlockCodingMode, -} - -struct PreparedResolutionPacket { - subbands: Vec, -} - -fn prepare_subband( - coefficients: &[f32], - width: u32, - height: u32, - step_size: &QuantStepSize, - guard_bits: u8, - reversible: bool, - block_coding_mode: BlockCodingMode, - cb_width: u32, - cb_height: u32, - sub_band_type: SubBandType, -) -> Result { - if width == 0 || height == 0 { - return Ok(PreparedEncodeSubband { - code_blocks: Vec::new(), - num_cbs_x: 0, - num_cbs_y: 0, - sub_band_type, - total_bitplanes: 0, - block_coding_mode, - }); - } - - // Quantize - let quantized = quantize::quantize_subband(coefficients, step_size, guard_bits, reversible); - debug_assert!(step_size.exponent <= u16::from(u8::MAX)); - let total_bitplanes = guard_bits - .saturating_add(step_size.exponent as u8) - .saturating_sub(1); - - // Split into code-blocks - let num_cbs_x = width.div_ceil(cb_width); - let num_cbs_y = height.div_ceil(cb_height); - let mut code_blocks = Vec::with_capacity((num_cbs_x * num_cbs_y) as usize); - - for cby in 0..num_cbs_y { - for cbx in 0..num_cbs_x { - let x0 = cbx * cb_width; - let y0 = cby * cb_height; - let x1 = (x0 + cb_width).min(width); - let y1 = (y0 + cb_height).min(height); - let cbw = x1 - x0; - let cbh = y1 - y0; - - // Extract code-block coefficients - let mut cb_coeffs = vec![0i32; (cbw * cbh) as usize]; - for y in 0..cbh { - for x in 0..cbw { - cb_coeffs[(y * cbw + x) as usize] = - quantized[((y0 + y) * width + (x0 + x)) as usize]; - } - } - - code_blocks.push(PreparedEncodeCodeBlock { - coefficients: cb_coeffs, - width: cbw, - height: cbh, - }); - } - } - - Ok(PreparedEncodeSubband { - code_blocks, - num_cbs_x, - num_cbs_y, - sub_band_type, - total_bitplanes, - block_coding_mode, - }) -} - -fn encode_prepared_resolution_packets( - prepared_packets: Vec, - accelerator: &mut impl J2kEncodeStageAccelerator, -) -> Result, &'static str> { - let subband_counts: Vec<_> = prepared_packets - .iter() - .map(|packet| packet.subbands.len()) - .collect(); - let prepared_subbands: Vec<_> = prepared_packets - .into_iter() - .flat_map(|packet| packet.subbands) - .collect(); - let mut encoded_subbands = - encode_prepared_subbands(prepared_subbands, accelerator)?.into_iter(); - - subband_counts - .into_iter() - .map(|subband_count| { - let mut subbands = Vec::with_capacity(subband_count); - for _ in 0..subband_count { - subbands.push( - encoded_subbands - .next() - .ok_or("encoded subband count mismatch")?, - ); - } - Ok(ResolutionPacket { subbands }) - }) - .collect() -} - -fn encode_prepared_subbands( - prepared_subbands: Vec, - accelerator: &mut impl J2kEncodeStageAccelerator, -) -> Result, &'static str> { - let block_coding_mode = prepared_subbands - .iter() - .find(|subband| !subband.code_blocks.is_empty()) - .map(|subband| subband.block_coding_mode); - let encoded_blocks = match block_coding_mode { - Some(BlockCodingMode::HighThroughput) => { - encode_all_ht_code_blocks(&prepared_subbands, accelerator)? - } - Some(BlockCodingMode::Classic) => { - encode_all_tier1_code_blocks(&prepared_subbands, accelerator)? - } - None => Vec::new(), - }; - - let mut encoded_iter = encoded_blocks.into_iter(); - let mut precincts = Vec::with_capacity(prepared_subbands.len()); - for subband in prepared_subbands { - let mut code_blocks = Vec::with_capacity(subband.code_blocks.len()); - for _ in 0..subband.code_blocks.len() { - let encoded = encoded_iter - .next() - .ok_or("encoded code-block count mismatch")?; - code_blocks.push(CodeBlockPacketData { - data: encoded.data, - num_coding_passes: encoded.num_coding_passes, - num_zero_bitplanes: encoded.num_zero_bitplanes, - previously_included: false, - l_block: 3, - block_coding_mode: subband.block_coding_mode, - }); - } - precincts.push(SubbandPrecinct { - code_blocks, - num_cbs_x: subband.num_cbs_x, - num_cbs_y: subband.num_cbs_y, - }); - } - if encoded_iter.next().is_some() { - return Err("encoded code-block count mismatch"); - } - - Ok(precincts) -} - -fn encode_all_ht_code_blocks( - prepared_subbands: &[PreparedEncodeSubband], - accelerator: &mut impl J2kEncodeStageAccelerator, -) -> Result, &'static str> { - let jobs: Vec<_> = prepared_subbands - .iter() - .flat_map(|subband| { - subband - .code_blocks - .iter() - .map(move |block| crate::J2kHtCodeBlockEncodeJob { - coefficients: &block.coefficients, - width: block.width, - height: block.height, - total_bitplanes: subband.total_bitplanes, - }) - }) - .collect(); - - if let Some(encoded) = accelerator.encode_ht_code_blocks(&jobs)? { - if encoded.len() != jobs.len() { - return Err("accelerated HT code-block batch length mismatch"); - } - return Ok(encoded - .into_iter() - .map(ht_encoded_code_block_from_accelerator) - .collect()); - } - - if accelerator.prefer_parallel_cpu_code_block_fallback() { - return encode_all_ht_code_blocks_parallel(&jobs); - } - - jobs.iter() - .map(|job| { - encode_ht_code_block( - job.coefficients, - job.width, - job.height, - job.total_bitplanes, - accelerator, - ) - }) - .collect() -} - -fn encode_all_tier1_code_blocks( - prepared_subbands: &[PreparedEncodeSubband], - accelerator: &mut impl J2kEncodeStageAccelerator, -) -> Result, &'static str> { - let style = default_public_code_block_style(); - let jobs: Vec<_> = prepared_subbands - .iter() - .flat_map(|subband| { - let public_sub_band_type = public_sub_band_type(subband.sub_band_type); - subband - .code_blocks - .iter() - .map(move |block| J2kTier1CodeBlockEncodeJob { - coefficients: &block.coefficients, - width: block.width, - height: block.height, - sub_band_type: public_sub_band_type, - total_bitplanes: subband.total_bitplanes, - style, - }) - }) - .collect(); - - if let Some(encoded) = accelerator.encode_tier1_code_blocks(&jobs)? { - if encoded.len() != jobs.len() { - return Err("accelerated classic code-block batch length mismatch"); - } - return Ok(encoded - .into_iter() - .map(encoded_code_block_from_accelerator) - .collect()); - } - - if accelerator.prefer_parallel_cpu_code_block_fallback() { - return encode_all_tier1_code_blocks_parallel(&jobs); - } - - let mut encoded = Vec::with_capacity(jobs.len()); - for subband in prepared_subbands { - for block in &subband.code_blocks { - encoded.push(encode_tier1_code_block( - &block.coefficients, - block.width, - block.height, - subband.sub_band_type, - subband.total_bitplanes, - accelerator, - )?); - } - } - Ok(encoded) -} - -#[cfg(feature = "parallel")] -fn encode_all_ht_code_blocks_parallel( - jobs: &[crate::J2kHtCodeBlockEncodeJob<'_>], -) -> Result, &'static str> { - jobs.par_iter() - .map(|job| { - ht_block_encode::encode_code_block( - job.coefficients, - job.width, - job.height, - job.total_bitplanes, - ) - }) - .collect() -} - -#[cfg(not(feature = "parallel"))] -fn encode_all_ht_code_blocks_parallel( - jobs: &[crate::J2kHtCodeBlockEncodeJob<'_>], -) -> Result, &'static str> { - jobs.iter() - .map(|job| { - ht_block_encode::encode_code_block( - job.coefficients, - job.width, - job.height, - job.total_bitplanes, - ) - }) - .collect() -} - -#[cfg(feature = "parallel")] -fn encode_all_tier1_code_blocks_parallel( - jobs: &[J2kTier1CodeBlockEncodeJob<'_>], -) -> Result, &'static str> { - jobs.par_iter() - .map(|job| { - Ok(bitplane_encode::encode_code_block( - job.coefficients, - job.width, - job.height, - internal_sub_band_type(job.sub_band_type), - job.total_bitplanes, - )) - }) - .collect() -} - -#[cfg(not(feature = "parallel"))] -fn encode_all_tier1_code_blocks_parallel( - jobs: &[J2kTier1CodeBlockEncodeJob<'_>], -) -> Result, &'static str> { - jobs.iter() - .map(|job| { - Ok(bitplane_encode::encode_code_block( - job.coefficients, - job.width, - job.height, - internal_sub_band_type(job.sub_band_type), - job.total_bitplanes, - )) - }) - .collect() -} - -fn encode_ht_code_block( - coefficients: &[i32], - width: u32, - height: u32, - total_bitplanes: u8, - accelerator: &mut impl J2kEncodeStageAccelerator, -) -> Result { - if let Some(encoded) = accelerator.encode_ht_code_block(crate::J2kHtCodeBlockEncodeJob { - coefficients, - width, - height, - total_bitplanes, - })? { - return Ok(ht_encoded_code_block_from_accelerator(encoded)); - } - - ht_block_encode::encode_code_block(coefficients, width, height, total_bitplanes) -} - -fn ht_encoded_code_block_from_accelerator( - encoded: crate::EncodedHtJ2kCodeBlock, -) -> bitplane_encode::EncodedCodeBlock { - bitplane_encode::EncodedCodeBlock { - data: encoded.data, - num_coding_passes: encoded.num_coding_passes, - num_zero_bitplanes: encoded.num_zero_bitplanes, - } -} - -fn encode_tier1_code_block( - coefficients: &[i32], - width: u32, - height: u32, - sub_band_type: SubBandType, - total_bitplanes: u8, - accelerator: &mut impl J2kEncodeStageAccelerator, -) -> Result { - if let Some(encoded) = accelerator.encode_tier1_code_block(J2kTier1CodeBlockEncodeJob { - coefficients, - width, - height, - sub_band_type: public_sub_band_type(sub_band_type), - total_bitplanes, - style: default_public_code_block_style(), - })? { - return Ok(encoded_code_block_from_accelerator(encoded)); - } - - Ok(bitplane_encode::encode_code_block( - coefficients, - width, - height, - sub_band_type, - total_bitplanes, - )) -} - -fn encoded_code_block_from_accelerator( - encoded: EncodedJ2kCodeBlock, -) -> bitplane_encode::EncodedCodeBlock { - bitplane_encode::EncodedCodeBlock { - data: encoded.data, - num_coding_passes: encoded.number_of_coding_passes, - num_zero_bitplanes: encoded.missing_bit_planes, - } -} - -fn public_sub_band_type(sub_band_type: SubBandType) -> J2kSubBandType { - match sub_band_type { - SubBandType::LowLow => J2kSubBandType::LowLow, - SubBandType::HighLow => J2kSubBandType::HighLow, - SubBandType::LowHigh => J2kSubBandType::LowHigh, - SubBandType::HighHigh => J2kSubBandType::HighHigh, - } -} - -fn internal_sub_band_type(sub_band_type: J2kSubBandType) -> SubBandType { - match sub_band_type { - J2kSubBandType::LowLow => SubBandType::LowLow, - J2kSubBandType::HighLow => SubBandType::HighLow, - J2kSubBandType::LowHigh => SubBandType::LowHigh, - J2kSubBandType::HighHigh => SubBandType::HighHigh, - } -} - -fn default_public_code_block_style() -> crate::J2kCodeBlockStyle { - crate::J2kCodeBlockStyle { - selective_arithmetic_coding_bypass: false, - reset_context_probabilities: false, - termination_on_each_pass: false, - vertically_causal_context: false, - segmentation_symbols: false, - } -} - -/// Convert interleaved pixel bytes to per-component f32 arrays. -fn deinterleave_to_f32( - pixels: &[u8], - num_pixels: usize, - num_components: u8, - bit_depth: u8, - signed: bool, -) -> Vec> { - let nc = num_components as usize; - let mut components = vec![vec![0.0f32; num_pixels]; nc]; - let unsigned_offset = if signed { - 0.0 - } else { - (1u32 << (bit_depth as u32 - 1)) as f32 - }; - - if bit_depth <= 8 { - for (i, pixel) in pixels.chunks_exact(nc).take(num_pixels).enumerate() { - for (c, component) in components.iter_mut().enumerate().take(nc) { - let val = pixel[c]; - component[i] = if signed { - (val as i8) as f32 - } else { - val as f32 - unsigned_offset - }; - } - } - } else { - // 16-bit samples (little-endian) - for (i, pixel) in pixels.chunks_exact(nc * 2).take(num_pixels).enumerate() { - for (c, component) in components.iter_mut().enumerate().take(nc) { - let offset = c * 2; - let val = u16::from_le_bytes([pixel[offset], pixel[offset + 1]]); - component[i] = if signed { - (val as i16) as f32 - } else { - val as f32 - unsigned_offset - }; - } - } - } - - components -} - -/// Calculate the maximum number of decomposition levels for given dimensions. -fn max_decomposition_levels(width: u32, height: u32) -> u8 { - let min_dim = width.min(height); - if min_dim <= 1 { - return 0; - } - floor_f32(log2_f32(min_dim as f32)) as u8 -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_encode_8bit_gray() { - let width = 8u32; - let height = 8u32; - let pixels: Vec = (0..64).collect(); - - let result = encode( - &pixels, - width, - height, - 1, - 8, - false, - &EncodeOptions { - num_decomposition_levels: 2, - ..Default::default() - }, - ); - - assert!(result.is_ok()); - let codestream = result.unwrap(); - // Verify SOC marker - assert_eq!(codestream[0], 0xFF); - assert_eq!(codestream[1], 0x4F); - // Verify EOC marker - let len = codestream.len(); - assert_eq!(codestream[len - 2], 0xFF); - assert_eq!(codestream[len - 1], 0xD9); - } - - #[test] - fn test_encode_16bit_gray() { - let width = 8u32; - let height = 8u32; - let mut pixels = Vec::with_capacity(128); - for i in 0..64u16 { - let val = i * 100; - pixels.extend_from_slice(&val.to_le_bytes()); - } - - let result = encode( - &pixels, - width, - height, - 1, - 16, - false, - &EncodeOptions { - num_decomposition_levels: 2, - ..Default::default() - }, - ); - - assert!(result.is_ok()); - } - - #[test] - fn test_encode_rgb() { - let width = 16u32; - let height = 16u32; - let pixels: Vec = (0..width * height * 3).map(|i| (i & 0xFF) as u8).collect(); - - let result = encode( - &pixels, - width, - height, - 3, - 8, - false, - &EncodeOptions { - num_decomposition_levels: 3, - ..Default::default() - }, - ); - - assert!(result.is_ok(), "RGB encode failed: {:?}", result.err()); - } - - #[test] - fn encode_with_accelerator_calls_lossless_stage_hooks() { - #[derive(Default)] - struct CountingAccelerator { - forward_rct: usize, - forward_dwt53: usize, - tier1_code_blocks: usize, - tier1_code_block_batches: usize, - tier1_batched_jobs: usize, - packetization: usize, - packetization_resolution_count: u32, - packetization_code_block_count: u32, - packetization_saw_payload: bool, - } - - impl crate::J2kEncodeStageAccelerator for CountingAccelerator { - fn encode_forward_rct( - &mut self, - _job: crate::J2kForwardRctJob<'_>, - ) -> core::result::Result { - self.forward_rct += 1; - Ok(false) - } - - fn encode_forward_dwt53( - &mut self, - _job: crate::J2kForwardDwt53Job<'_>, - ) -> core::result::Result, &'static str> - { - self.forward_dwt53 += 1; - Ok(None) - } - - fn encode_tier1_code_block( - &mut self, - _job: crate::J2kTier1CodeBlockEncodeJob<'_>, - ) -> core::result::Result, &'static str> - { - self.tier1_code_blocks += 1; - Ok(None) - } - - fn encode_tier1_code_blocks( - &mut self, - jobs: &[crate::J2kTier1CodeBlockEncodeJob<'_>], - ) -> core::result::Result>, &'static str> - { - self.tier1_code_block_batches += 1; - self.tier1_batched_jobs += jobs.len(); - Ok(None) - } - - fn encode_packetization( - &mut self, - job: crate::J2kPacketizationEncodeJob<'_>, - ) -> core::result::Result>, &'static str> { - self.packetization += 1; - self.packetization_resolution_count = job.resolution_count; - self.packetization_code_block_count = job.code_block_count; - self.packetization_saw_payload = job - .resolutions - .iter() - .flat_map(|resolution| resolution.subbands.iter()) - .flat_map(|subband| subband.code_blocks.iter()) - .any(|code_block| !code_block.data.is_empty()); - Ok(None) - } - } - - let pixels: Vec = (0..8 * 8 * 3).map(|i| (i & 0xFF) as u8).collect(); - let options = EncodeOptions { - num_decomposition_levels: 1, - reversible: true, - ..EncodeOptions::default() - }; - let mut accelerator = CountingAccelerator::default(); - - let codestream = - encode_with_accelerator(&pixels, 8, 8, 3, 8, false, &options, &mut accelerator) - .expect("encode with accelerator hooks"); - - assert!(codestream.starts_with(&[0xFF, 0x4F])); - assert_eq!(accelerator.forward_rct, 1); - assert_eq!(accelerator.forward_dwt53, 3); - assert!(accelerator.tier1_code_block_batches > 0); - assert_eq!( - accelerator.tier1_code_blocks, - accelerator.tier1_batched_jobs - ); - assert_eq!(accelerator.packetization, 1); - assert_eq!(accelerator.packetization_resolution_count, 6); - assert_eq!( - accelerator.packetization_code_block_count, - u32::try_from(accelerator.tier1_code_blocks).expect("test code-block count fits u32") - ); - assert!(accelerator.packetization_saw_payload); - } - - #[test] - fn cpu_only_accelerator_opts_into_parallel_block_fallback_only_for_native_cpu() { - #[derive(Default)] - struct ExternalAccelerator; - - impl crate::J2kEncodeStageAccelerator for ExternalAccelerator {} - - let cpu = crate::CpuOnlyJ2kEncodeStageAccelerator; - let external = ExternalAccelerator; - - assert!(cpu.prefer_parallel_cpu_code_block_fallback()); - assert!(!external.prefer_parallel_cpu_code_block_fallback()); - } - - #[test] - fn cpu_parallel_block_fallback_matches_serial_classic_and_htj2k_output() { - #[derive(Default)] - struct SerialCpuFallbackAccelerator; - - impl crate::J2kEncodeStageAccelerator for SerialCpuFallbackAccelerator {} - - let pixels = gradient_u8(96, 80); - for use_ht_block_coding in [false, true] { - let options = EncodeOptions { - num_decomposition_levels: 1, - code_block_width_exp: 2, - code_block_height_exp: 2, - use_ht_block_coding, - ..EncodeOptions::default() - }; - let parallel = encode(&pixels, 96, 80, 1, 8, false, &options) - .expect("parallel CPU fallback encode"); - let mut serial_accelerator = SerialCpuFallbackAccelerator; - let serial = encode_with_accelerator( - &pixels, - 96, - 80, - 1, - 8, - false, - &options, - &mut serial_accelerator, - ) - .expect("serial CPU fallback encode"); - - assert_eq!(parallel, serial); - } - } - - #[test] - fn test_encode_lossy() { - let pixels: Vec = (0..64).collect(); - - let result = encode( - &pixels, - 8, - 8, - 1, - 8, - false, - &EncodeOptions { - num_decomposition_levels: 2, - reversible: false, - guard_bits: 2, - ..Default::default() - }, - ); - - assert!(result.is_ok()); - } - - fn assert_htj2k_lossless_roundtrip( - pixels: &[u8], - width: u32, - height: u32, - bit_depth: u8, - num_decomposition_levels: u8, - ) { - let codestream = encode_htj2k( - pixels, - width, - height, - 1, - bit_depth, - false, - &EncodeOptions { - num_decomposition_levels, - ..Default::default() - }, - ) - .expect("HTJ2K encode"); - - assert!(codestream.windows(2).any(|window| window == [0xFF, 0x50])); - let cod_offset = codestream - .windows(2) - .position(|window| window == [0xFF, 0x52]) - .expect("COD marker"); - assert_eq!(codestream[cod_offset + 12], 0x40); - - let image = Image::new( - &codestream, - &DecodeSettings { - resolve_palette_indices: true, - strict: true, - target_resolution: None, - }, - ) - .expect("parse HT codestream"); - let decoded = image.decode_native().expect("decode HT codestream"); - - assert_eq!(decoded.width, width); - assert_eq!(decoded.height, height); - assert_eq!(decoded.bit_depth, bit_depth); - assert_eq!(decoded.data, pixels); - } - - fn gradient_u8(width: u32, height: u32) -> Vec { - let mut pixels = Vec::with_capacity((width * height) as usize); - for y in 0..height { - for x in 0..width { - pixels.push(((x * 17 + y * 31) % 256) as u8); - } - } - pixels - } - - #[test] - fn test_encode_high_throughput_zero_image_roundtrip() { - let width = 4u32; - let height = 4u32; - let sample = 2048u16.to_le_bytes(); - let mut pixels = Vec::with_capacity((width * height * 2) as usize); - for _ in 0..(width * height) { - pixels.extend_from_slice(&sample); - } - - let codestream = encode( - &pixels, - width, - height, - 1, - 12, - false, - &EncodeOptions { - num_decomposition_levels: 2, - use_ht_block_coding: true, - ..Default::default() - }, - ) - .expect("HT all-zero encode"); - - assert!(codestream.windows(2).any(|window| window == [0xFF, 0x50])); - let cod_offset = codestream - .windows(2) - .position(|window| window == [0xFF, 0x52]) - .expect("COD marker"); - assert_eq!(codestream[cod_offset + 12], 0x40); - - let image = - Image::new(&codestream, &DecodeSettings::default()).expect("parse HT codestream"); - let decoded = image.decode_native().expect("decode HT codestream"); - - assert_eq!(decoded.width, width); - assert_eq!(decoded.height, height); - assert_eq!(decoded.bit_depth, 12); - assert_eq!(decoded.data, pixels); - } - - #[test] - fn test_encode_high_throughput_nonzero_roundtrip() { - let width = 1u32; - let height = 1u32; - let pixels = 2049u16.to_le_bytes().to_vec(); - - let codestream = encode_htj2k( - &pixels, - width, - height, - 1, - 12, - false, - &EncodeOptions { - num_decomposition_levels: 0, - ..Default::default() - }, - ) - .expect("HT non-zero encode"); - - assert!(codestream.windows(2).any(|window| window == [0xFF, 0x50])); - let image = - Image::new(&codestream, &DecodeSettings::default()).expect("parse HT codestream"); - let decoded = image.decode_native().expect("decode HT codestream"); - - assert_eq!(decoded.width, width); - assert_eq!(decoded.height, height); - assert_eq!(decoded.bit_depth, 12); - assert_eq!(decoded.data, pixels); - } - - #[test] - fn test_encode_high_throughput_varied_12bit_roundtrip() { - let mut pixels = Vec::with_capacity(32); - for i in 0u16..16 { - pixels.extend_from_slice(&((i * 257) & 0x0FFF).to_le_bytes()); - } - - let codestream = encode_htj2k( - &pixels, - 4, - 4, - 1, - 12, - false, - &EncodeOptions { - num_decomposition_levels: 1, - ..Default::default() - }, - ) - .expect("HT varied encode"); - - let image = - Image::new(&codestream, &DecodeSettings::default()).expect("parse HT codestream"); - let decoded = image.decode_native().expect("decode HT codestream"); - - assert_eq!(decoded.width, 4); - assert_eq!(decoded.height, 4); - assert_eq!(decoded.bit_depth, 12); - assert_eq!(decoded.data, pixels); - } - - #[test] - fn test_encode_high_throughput_gradient_8bit_roundtrip() { - let pixels: Vec = (0..64).collect(); - - let codestream = encode_htj2k( - &pixels, - 8, - 8, - 1, - 8, - false, - &EncodeOptions { - num_decomposition_levels: 3, - ..Default::default() - }, - ) - .expect("HT gradient encode"); - - let image = - Image::new(&codestream, &DecodeSettings::default()).expect("parse HT codestream"); - let decoded = image.decode_native().expect("decode HT codestream"); - - assert_eq!(decoded.width, 8); - assert_eq!(decoded.height, 8); - assert_eq!(decoded.bit_depth, 8); - assert_eq!(decoded.data, pixels); - } - - #[test] - fn test_encode_high_throughput_varied_12bit_large_roundtrip() { - let width = 16u32; - let height = 8u32; - let mut pixels = Vec::with_capacity((width * height * 2) as usize); - for y in 0u16..height as u16 { - for x in 0u16..width as u16 { - let value = (x * 257 + y * 17) & 0x0FFF; - pixels.extend_from_slice(&value.to_le_bytes()); - } - } - - assert_htj2k_lossless_roundtrip(&pixels, width, height, 12, 4); - } - - #[test] - fn test_encode_high_throughput_ramp_16bit_roundtrip() { - let width = 48u32; - let height = 24u32; - let mut pixels = Vec::with_capacity((width * height * 2) as usize); - for y in 0u16..height as u16 { - for x in 0u16..width as u16 { - let value = x * 521 + y * 997; - pixels.extend_from_slice(&value.to_le_bytes()); - } - } - - assert_htj2k_lossless_roundtrip(&pixels, width, height, 16, 4); - } - - #[test] - fn test_encode_high_throughput_lossy_large_gradient_is_parseable() { - let pixels = gradient_u8(128, 128); - - let codestream = encode_htj2k( - &pixels, - 128, - 128, - 1, - 8, - false, - &EncodeOptions { - num_decomposition_levels: 5, - reversible: false, - guard_bits: 2, - ..Default::default() - }, - ) - .expect("lossy HT encode"); - - assert!(codestream.windows(2).any(|window| window == [0xFF, 0x50])); - assert!(!codestream.is_empty()); - - let image = Image::new( - &codestream, - &DecodeSettings { - resolve_palette_indices: true, - strict: true, - target_resolution: None, - }, - ) - .expect("parse lossy HT codestream"); - let decoded = image.decode_native().expect("decode lossy HT codestream"); - - assert_eq!(decoded.width, 128); - assert_eq!(decoded.height, 128); - assert_eq!(decoded.bit_depth, 8); - } - - #[test] - fn test_encode_invalid_dimensions() { - let result = encode(&[], 0, 0, 1, 8, false, &EncodeOptions::default()); - assert!(result.is_err()); - } - - #[test] - fn test_encode_too_short() { - let pixels = vec![0u8; 10]; // Way too short for 8x8 - let result = encode(&pixels, 8, 8, 1, 8, false, &EncodeOptions::default()); - assert!(result.is_err()); - } - - #[test] - fn test_deinterleave_rgb() { - let pixels = vec![ - 10u8, 20, 30, // pixel 0: R=10, G=20, B=30 - 40, 50, 60, // pixel 1: R=40, G=50, B=60 - ]; - let comps = deinterleave_to_f32(&pixels, 2, 3, 8, false); - assert_eq!(comps[0], vec![-118.0, -88.0]); // R - assert_eq!(comps[1], vec![-108.0, -78.0]); // G - assert_eq!(comps[2], vec![-98.0, -68.0]); // B - } - - #[test] - fn test_encode_decode_roundtrip_gray_8bit() { - use crate::{DecodeSettings, Image}; - - // Constant image: all pixels = 42 — simplest possible test - let original: Vec = vec![42u8; 64]; // 8x8, all same value - let encoded = encode( - &original, - 8, - 8, - 1, - 8, - false, - &EncodeOptions { - num_decomposition_levels: 0, - reversible: true, - ..Default::default() - }, - ) - .expect("encode failed"); - - let settings = DecodeSettings { - resolve_palette_indices: false, - strict: false, - target_resolution: None, - }; - let image = Image::new(&encoded, &settings).expect("parse failed"); - let decoded = image.decode_native().expect("decode failed"); - - assert_eq!(decoded.width, 8); - assert_eq!(decoded.height, 8); - assert_eq!(decoded.data, original, "round-trip mismatch"); - } - - #[test] - fn test_encode_decode_roundtrip_gray_8bit_single_dwt_level() { - use crate::{DecodeSettings, Image}; - - let original: Vec = (0..64 * 64) - .map(|value| ((value * 37 + value / 7) & 0xFF) as u8) - .collect(); - let encoded = encode( - &original, - 64, - 64, - 1, - 8, - false, - &EncodeOptions { - num_decomposition_levels: 1, - reversible: true, - ..Default::default() - }, - ) - .expect("encode failed"); - - let image = Image::new(&encoded, &DecodeSettings::default()).expect("parse failed"); - let decoded = image.decode_native().expect("decode failed"); - - assert_eq!(decoded.width, 64); - assert_eq!(decoded.height, 64); - assert_eq!(decoded.data, original, "round-trip mismatch"); - } -} diff --git a/crates/signinum-j2k-native/src/j2c/ht_block_decode.rs b/crates/signinum-j2k-native/src/j2c/ht_block_decode.rs deleted file mode 100644 index 97949f40..00000000 --- a/crates/signinum-j2k-native/src/j2c/ht_block_decode.rs +++ /dev/null @@ -1,1126 +0,0 @@ -//! Scalar HTJ2K block decoding. - -use alloc::vec; -use alloc::vec::Vec; - -use super::build::CodeBlock; -use super::decode::DecompositionStorage; -use super::ht_tables::{UVLC_TABLE0, UVLC_TABLE1, VLC_TABLE0, VLC_TABLE1}; -use crate::error::{bail, DecodingError, Result}; - -#[derive(Default)] -pub(crate) struct HtBlockDecodeContext { - coefficients: Vec, - width: u32, - height: u32, -} - -impl HtBlockDecodeContext { - fn reset(&mut self, code_block: &CodeBlock) { - self.width = code_block.rect.width(); - self.height = code_block.rect.height(); - self.coefficients.clear(); - self.coefficients - .resize((self.width * self.height) as usize, 0); - } - - pub(crate) fn coefficient_rows(&self) -> impl Iterator { - self.coefficients.chunks_exact(self.width as usize) - } -} - -pub(crate) fn coefficient_to_i32(value: u32, k_max: u8) -> i32 { - let shift = 31_u32.saturating_sub(k_max as u32); - let magnitude = ((value & 0x7FFF_FFFF) >> shift) as i32; - - if (value & 0x8000_0000) != 0 { - -magnitude - } else { - magnitude - } -} - -pub(crate) fn decode( - code_block: &CodeBlock, - total_bitplanes: u8, - stripe_causal: bool, - ctx: &mut HtBlockDecodeContext, - storage: &DecompositionStorage<'_>, - strict: bool, -) -> Result<()> { - ctx.reset(code_block); - - if total_bitplanes == 0 { - return Ok(()); - } - - if total_bitplanes > 31 { - bail!(DecodingError::TooManyBitplanes); - } - - let actual_bitplanes = if strict { - total_bitplanes - .checked_sub(code_block.missing_bit_planes) - .ok_or(DecodingError::InvalidBitplaneCount)? - } else { - total_bitplanes.saturating_sub(code_block.missing_bit_planes) - }; - - let max_coding_passes = if actual_bitplanes == 0 { - 0 - } else { - 1 + 3 * (actual_bitplanes - 1) - }; - - if code_block.number_of_coding_passes > max_coding_passes && strict { - bail!(DecodingError::TooManyCodingPasses); - } - - if code_block.number_of_coding_passes == 0 || actual_bitplanes == 0 { - return Ok(()); - } - - let combined = collect_code_block_data(code_block, storage)?; - decode_combined_validated( - &combined, - code_block.missing_bit_planes, - total_bitplanes, - code_block.number_of_coding_passes, - stripe_causal, - strict, - &mut ctx.coefficients, - code_block.rect.width(), - code_block.rect.height(), - code_block.rect.width(), - ) -} - -pub(crate) struct CombinedCodeBlockData { - pub(crate) data: Vec, - pub(crate) cleanup_length: u32, - pub(crate) refinement_length: u32, -} - -pub(crate) fn collect_code_block_data( - code_block: &CodeBlock, - storage: &DecompositionStorage<'_>, -) -> Result { - let mut data = Vec::new(); - let mut cleanup_length = 0; - let mut refinement_length = 0; - let mut saw_cleanup = false; - let mut saw_refinement = false; - - for layer in &storage.layers[code_block.layers.start..code_block.layers.end] { - let Some(range) = layer.segments.clone() else { - continue; - }; - - for segment in &storage.segments[range] { - match segment.idx { - 0 if !saw_cleanup => { - cleanup_length = segment.data_length; - data.extend_from_slice(segment.data); - saw_cleanup = true; - } - 1 if !saw_refinement => { - refinement_length = segment.data_length; - data.extend_from_slice(segment.data); - saw_refinement = true; - } - _ => bail!(DecodingError::UnsupportedFeature( - "unexpected HTJ2K segment layout" - )), - } - } - } - - if !saw_cleanup { - bail!(DecodingError::CodeBlockDecodeFailure); - } - - Ok(CombinedCodeBlockData { - data, - cleanup_length, - refinement_length, - }) -} - -pub(crate) fn decode_combined( - combined: &CombinedCodeBlockData, - missing_bit_planes: u8, - number_of_coding_passes: u8, - width: u32, - height: u32, - stride: u32, - stripe_causal: bool, - decoded_data: &mut [u32], -) -> Result<()> { - decode_impl( - &combined.data, - decoded_data, - missing_bit_planes as u32, - number_of_coding_passes as u32, - combined.cleanup_length, - combined.refinement_length, - width, - height, - stride, - stripe_causal, - ) - .ok_or(DecodingError::CodeBlockDecodeFailure.into()) -} - -pub(crate) fn decode_combined_validated( - combined: &CombinedCodeBlockData, - missing_bit_planes: u8, - total_bitplanes: u8, - number_of_coding_passes: u8, - stripe_causal: bool, - strict: bool, - decoded_data: &mut [u32], - width: u32, - height: u32, - stride: u32, -) -> Result<()> { - if total_bitplanes == 0 { - return Ok(()); - } - - if total_bitplanes > 31 { - bail!(DecodingError::TooManyBitplanes); - } - - let actual_bitplanes = if strict { - total_bitplanes - .checked_sub(missing_bit_planes) - .ok_or(DecodingError::InvalidBitplaneCount)? - } else { - total_bitplanes.saturating_sub(missing_bit_planes) - }; - - let max_coding_passes = if actual_bitplanes == 0 { - 0 - } else { - 1 + 3 * (actual_bitplanes - 1) - }; - - if number_of_coding_passes > max_coding_passes && strict { - bail!(DecodingError::TooManyCodingPasses); - } - - if number_of_coding_passes == 0 || actual_bitplanes == 0 { - return Ok(()); - } - - decode_combined( - combined, - missing_bit_planes, - number_of_coding_passes, - width, - height, - stride, - stripe_causal, - decoded_data, - ) -} - -struct MelDecoder<'a> { - data: &'a [u8], - pos: usize, - remaining: usize, - unstuff: bool, - current_byte: u8, - bits_left: u8, - k: usize, - num_runs: usize, - runs: u64, -} - -impl<'a> MelDecoder<'a> { - fn new(data: &'a [u8], lcup: usize, scup: usize) -> Self { - Self { - data, - pos: lcup - scup, - remaining: scup - 1, - unstuff: false, - current_byte: 0, - bits_left: 0, - k: 0, - num_runs: 0, - runs: 0, - } - } - - fn read_bit(&mut self) -> Option { - if self.bits_left == 0 { - let mut byte = if self.remaining > 0 { - let byte = self.data.get(self.pos).copied()?; - self.pos += 1; - self.remaining -= 1; - byte - } else { - 0xFF - }; - - if self.remaining == 0 { - byte |= 0x0F; - } - - self.current_byte = byte; - self.bits_left = 8 - u8::from(self.unstuff); - self.unstuff = byte == 0xFF; - } - - self.bits_left -= 1; - Some(((self.current_byte >> self.bits_left) & 1) as u32) - } - - fn read_bits(&mut self, count: usize) -> Option { - let mut value = 0; - - for _ in 0..count { - value = (value << 1) | self.read_bit()?; - } - - Some(value) - } - - fn decode_more_runs(&mut self) -> Option<()> { - const MEL_EXP: [usize; 13] = [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5]; - - while self.num_runs < 8 { - let eval = MEL_EXP[self.k]; - let first = self.read_bit()?; - let run = if first == 1 { - self.k = (self.k + 1).min(12); - ((1usize << eval) - 1) << 1 - } else { - self.k = self.k.saturating_sub(1); - (self.read_bits(eval)? as usize) << 1 | 1 - }; - - self.runs |= (run as u64) << (self.num_runs * 7); - self.num_runs += 1; - - if eval == 5 && first == 0 && self.num_runs >= 8 { - break; - } - } - - Some(()) - } - - fn get_run(&mut self) -> Option { - if self.num_runs == 0 { - self.decode_more_runs()?; - } - - let run = (self.runs & 0x7F) as i32; - self.runs >>= 7; - self.num_runs -= 1; - Some(run) - } -} - -struct ForwardBitReader<'a, const PAD: u8> { - data: &'a [u8], - pos: usize, - tmp: u64, - bits: u32, - unstuff: bool, -} - -impl<'a, const PAD: u8> ForwardBitReader<'a, PAD> { - fn new(data: &'a [u8]) -> Self { - Self { - data, - pos: 0, - tmp: 0, - bits: 0, - unstuff: false, - } - } - - fn fill(&mut self) { - while self.bits <= 32 { - let byte = if self.pos < self.data.len() { - let byte = self.data[self.pos]; - self.pos += 1; - byte - } else { - PAD - }; - - self.tmp |= (byte as u64) << self.bits; - self.bits += 8 - u32::from(self.unstuff); - self.unstuff = byte == 0xFF; - } - } - - fn fetch(&mut self) -> u32 { - if self.bits < 32 { - self.fill(); - } - - self.tmp as u32 - } - - fn advance(&mut self, count: u32) { - debug_assert!(count <= self.bits); - self.tmp >>= count; - self.bits -= count; - } -} - -struct ReverseBitReader<'a> { - data: &'a [u8], - pos: isize, - remaining: usize, - tmp: u64, - bits: u32, - unstuff: bool, -} - -impl<'a> ReverseBitReader<'a> { - fn new_vlc(data: &'a [u8], lcup: usize, scup: usize) -> Self { - let d = data[lcup - 2]; - let tmp = u64::from(d >> 4); - let bits = 4 - u32::from((tmp & 0x7) == 0x7); - - Self { - data, - pos: lcup as isize - 3, - remaining: scup - 2, - tmp, - bits, - unstuff: (d | 0x0F) > 0x8F, - } - } - - fn new_mrp(data: &'a [u8], lcup: usize, len2: usize) -> Self { - Self { - data, - pos: (lcup + len2) as isize - 1, - remaining: len2, - tmp: 0, - bits: 0, - unstuff: true, - } - } - - fn fill(&mut self) { - while self.bits <= 32 { - let byte = if self.remaining > 0 { - let byte = self.data[self.pos as usize]; - self.pos -= 1; - self.remaining -= 1; - byte - } else { - 0 - }; - - let d_bits = 8 - u32::from(self.unstuff && (byte & 0x7F) == 0x7F); - self.tmp |= (byte as u64) << self.bits; - self.bits += d_bits; - self.unstuff = byte > 0x8F; - } - } - - fn fetch(&mut self) -> u32 { - if self.bits < 32 { - self.fill(); - } - - self.tmp as u32 - } - - fn advance(&mut self, count: u32) -> u32 { - debug_assert!(count <= self.bits); - self.tmp >>= count; - self.bits -= count; - self.tmp as u32 - } -} - -fn read_u32_pair(values: &[u16], index: usize) -> u32 { - u32::from(values[index]) | (u32::from(values[index + 1]) << 16) -} - -fn sample_mask(bit: u32) -> u32 { - 1 << (4 + bit) -} - -fn decode_mag_sgn_sample_with_vn( - magsgn: &mut ForwardBitReader<0xFF>, - inf: u32, - bit: u32, - uq: u32, - p: u32, -) -> (u32, u32) { - if (inf & sample_mask(bit)) == 0 { - return (0, 0); - } - - let ms_val = magsgn.fetch(); - let m_n = uq - ((inf >> (12 + bit)) & 1); - magsgn.advance(m_n); - - let mut value = ms_val << 31; - let mask = if m_n == 0 { 0 } else { (1_u32 << m_n) - 1 }; - let mut v_n = ms_val & mask; - v_n |= ((inf >> (8 + bit)) & 1) << m_n; - v_n |= 1; - value |= (v_n + 2) << (p - 1); - (value, v_n) -} - -fn decode_impl( - coded_data: &[u8], - decoded_data: &mut [u32], - missing_msbs: u32, - mut num_passes: u32, - lengths1: u32, - lengths2: u32, - width: u32, - height: u32, - stride: u32, - stripe_causal: bool, -) -> Option<()> { - if num_passes > 1 && lengths2 == 0 { - num_passes = 1; - } - - if num_passes > 3 || missing_msbs > 30 { - return None; - } - - if missing_msbs == 30 { - return None; - } - - if missing_msbs == 29 && num_passes > 1 { - num_passes = 1; - } - - let p = 30 - missing_msbs; - let lcup = lengths1 as usize; - let len2 = lengths2 as usize; - - if lcup < 2 || coded_data.len() < lcup.saturating_add(len2) { - return None; - } - - let scup = ((coded_data[lcup - 1] as usize) << 4) + usize::from(coded_data[lcup - 2] & 0x0F); - if !(2..=lcup).contains(&scup) || scup > 4079 { - return None; - } - - let quad_rows = height.div_ceil(2) as usize; - let sstr = ((width + 2 + 7) & !7) as usize; - let mut scratch = vec![0u16; sstr * (quad_rows + 1)]; - let mmsbp2 = missing_msbs + 2; - - { - let mut mel = MelDecoder::new(coded_data, lcup, scup); - let mut vlc = ReverseBitReader::new_vlc(coded_data, lcup, scup); - let mut run = mel.get_run()?; - let mut c_q = 0u32; - let mut row_offset = 0usize; - let mut x = 0u32; - - while x < width { - let mut vlc_val = vlc.fetch(); - let mut t0 = u32::from(VLC_TABLE0[(c_q + (vlc_val & 0x7F)) as usize]); - if c_q == 0 { - run -= 2; - t0 = if run == -1 { t0 } else { 0 }; - if run < 0 { - run = mel.get_run()?; - } - } - scratch[row_offset] = t0 as u16; - x += 2; - c_q = ((t0 & 0x10) << 3) | ((t0 & 0xE0) << 2); - vlc_val = vlc.advance(t0 & 0x7); - - let mut t1 = u32::from(VLC_TABLE0[(c_q + (vlc_val & 0x7F)) as usize]); - if c_q == 0 && x < width { - run -= 2; - t1 = if run == -1 { t1 } else { 0 }; - if run < 0 { - run = mel.get_run()?; - } - } - if x >= width { - t1 = 0; - } - scratch[row_offset + 2] = t1 as u16; - x += 2; - c_q = ((t1 & 0x10) << 3) | ((t1 & 0xE0) << 2); - vlc_val = vlc.advance(t1 & 0x7); - - let mut uvlc_mode = ((t0 & 0x8) << 3) | ((t1 & 0x8) << 4); - if uvlc_mode == 0xC0 { - run -= 2; - if run == -1 { - uvlc_mode += 0x40; - } - if run < 0 { - run = mel.get_run()?; - } - } - - let mut uvlc_entry = u32::from(UVLC_TABLE0[(uvlc_mode + (vlc_val & 0x3F)) as usize]); - vlc_val = vlc.advance(uvlc_entry & 0x7); - uvlc_entry >>= 3; - let mut len = uvlc_entry & 0xF; - let tmp = vlc_val & ((1_u32 << len) - 1); - vlc_val = vlc.advance(len); - uvlc_entry >>= 4; - len = uvlc_entry & 0x7; - uvlc_entry >>= 3; - scratch[row_offset + 1] = (1 + (uvlc_entry & 0x7) + (tmp & !(0xFF_u32 << len))) as u16; - scratch[row_offset + 3] = (1 + (uvlc_entry >> 3) + (tmp >> len)) as u16; - - row_offset += 4; - } - scratch[row_offset] = 0; - scratch[row_offset + 1] = 0; - - for y in (2..height).step_by(2) { - let row_base = (y >> 1) as usize * sstr; - let prev_base = row_base - sstr; - let mut x = 0u32; - let mut c_q = 0u32; - let mut row_offset = row_base; - - while x < width { - c_q |= (u32::from(scratch[prev_base + (row_offset - row_base)]) & 0xA0) << 2; - c_q |= (u32::from(scratch[prev_base + (row_offset - row_base) + 2]) & 0x20) << 4; - - let mut vlc_val = vlc.fetch(); - let mut t0 = u32::from(VLC_TABLE1[(c_q + (vlc_val & 0x7F)) as usize]); - if c_q == 0 { - run -= 2; - t0 = if run == -1 { t0 } else { 0 }; - if run < 0 { - run = mel.get_run()?; - } - } - scratch[row_offset] = t0 as u16; - x += 2; - - c_q = ((t0 & 0x40) << 2) | ((t0 & 0x80) << 1); - c_q |= u32::from(scratch[prev_base + (row_offset - row_base)]) & 0x80; - c_q |= (u32::from(scratch[prev_base + (row_offset - row_base) + 2]) & 0xA0) << 2; - c_q |= (u32::from(scratch[prev_base + (row_offset - row_base) + 4]) & 0x20) << 4; - vlc_val = vlc.advance(t0 & 0x7); - - let mut t1 = u32::from(VLC_TABLE1[(c_q + (vlc_val & 0x7F)) as usize]); - if c_q == 0 && x < width { - run -= 2; - t1 = if run == -1 { t1 } else { 0 }; - if run < 0 { - run = mel.get_run()?; - } - } - if x >= width { - t1 = 0; - } - scratch[row_offset + 2] = t1 as u16; - x += 2; - - c_q = ((t1 & 0x40) << 2) | ((t1 & 0x80) << 1); - c_q |= u32::from(scratch[prev_base + (row_offset - row_base) + 2]) & 0x80; - vlc_val = vlc.advance(t1 & 0x7); - - let uvlc_mode = ((t0 & 0x8) << 3) | ((t1 & 0x8) << 4); - let mut uvlc_entry = - u32::from(UVLC_TABLE1[(uvlc_mode + (vlc_val & 0x3F)) as usize]); - vlc_val = vlc.advance(uvlc_entry & 0x7); - uvlc_entry >>= 3; - let mut len = uvlc_entry & 0xF; - let tmp = vlc_val & ((1_u32 << len) - 1); - vlc_val = vlc.advance(len); - uvlc_entry >>= 4; - len = uvlc_entry & 0x7; - uvlc_entry >>= 3; - scratch[row_offset + 1] = ((uvlc_entry & 0x7) + (tmp & !(0xFF_u32 << len))) as u16; - scratch[row_offset + 3] = ((uvlc_entry >> 3) + (tmp >> len)) as u16; - - row_offset += 4; - } - - scratch[row_offset] = 0; - scratch[row_offset + 1] = 0; - } - } - - { - let mut magsgn = ForwardBitReader::<0xFF>::new(&coded_data[..lcup - scup]); - let v_n_width = width.div_ceil(2) as usize + 2; - let mut v_n_scratch = vec![0u32; v_n_width]; - let mut prev_v_n = 0u32; - let mut x = 0u32; - let mut sp = 0usize; - let mut vp = 0usize; - let mut dp = 0usize; - let second_row_present = height > 1; - - while x < width { - let inf = u32::from(scratch[sp]); - let uq = u32::from(scratch[sp + 1]); - if uq > mmsbp2 { - return None; - } - - let (val0, _) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 0, uq, p); - decoded_data[dp] = val0; - - let (val1, v_n1) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 1, uq, p); - if second_row_present { - decoded_data[dp + stride as usize] = val1; - } - v_n_scratch[vp] = prev_v_n | v_n1; - prev_v_n = 0; - dp += 1; - x += 1; - - if x >= width { - vp += 1; - break; - } - - let (val2, _) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 2, uq, p); - decoded_data[dp] = val2; - - let (val3, v_n3) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 3, uq, p); - if second_row_present { - decoded_data[dp + stride as usize] = val3; - } - prev_v_n = v_n3; - dp += 1; - x += 1; - sp += 2; - vp += 1; - } - v_n_scratch[vp] = prev_v_n; - - for y in (2..height).step_by(2) { - let row_base = (y >> 1) as usize * sstr; - let mut sp = row_base; - let mut vp = 0usize; - let mut dp = (y * stride) as usize; - let mut prev_v_n = 0u32; - let mut x = 0u32; - let second_row_present = y + 1 < height; - - while x < width { - let inf = u32::from(scratch[sp]); - let u_q = u32::from(scratch[sp + 1]); - let mut gamma = inf & 0xF0; - gamma &= gamma.wrapping_sub(0x10); - let mut emax = v_n_scratch[vp] | v_n_scratch[vp + 1]; - emax = 31 - (emax | 2).leading_zeros(); - let kappa = if gamma != 0 { emax } else { 1 }; - let uq = u_q + kappa; - if uq > mmsbp2 { - return None; - } - - let (val0, _) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 0, uq, p); - decoded_data[dp] = val0; - - let (val1, v_n1) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 1, uq, p); - if second_row_present { - decoded_data[dp + stride as usize] = val1; - } - v_n_scratch[vp] = prev_v_n | v_n1; - prev_v_n = 0; - dp += 1; - x += 1; - - if x >= width { - vp += 1; - break; - } - - let (val2, _) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 2, uq, p); - decoded_data[dp] = val2; - - let (val3, v_n3) = decode_mag_sgn_sample_with_vn(&mut magsgn, inf, 3, uq, p); - if second_row_present { - decoded_data[dp + stride as usize] = val3; - } - prev_v_n = v_n3; - dp += 1; - x += 1; - sp += 2; - vp += 1; - } - - v_n_scratch[vp] = prev_v_n; - } - } - - if num_passes > 1 { - let sigma_rows = height.div_ceil(4) as usize + 1; - let mstr = ((width.div_ceil(4) + 2 + 7) & !7) as usize; - let mut sigma = vec![0u16; sigma_rows * mstr]; - - { - let mut y = 0u32; - while y < height { - let sp_base = (y >> 1) as usize * sstr; - let dp_base = (y >> 2) as usize * mstr; - let mut x = 0u32; - let mut sp = sp_base; - let mut dp = dp_base; - - while x < width { - let mut t0 = ((u32::from(scratch[sp]) & 0x30) >> 4) - | ((u32::from(scratch[sp]) & 0xC0) >> 2); - t0 |= ((u32::from(scratch[sp + 2]) & 0x30) << 4) - | ((u32::from(scratch[sp + 2]) & 0xC0) << 6); - let mut t1 = ((u32::from(scratch[sp + sstr]) & 0x30) >> 2) - | (u32::from(scratch[sp + sstr]) & 0xC0); - t1 |= ((u32::from(scratch[sp + sstr + 2]) & 0x30) << 6) - | ((u32::from(scratch[sp + sstr + 2]) & 0xC0) << 8); - sigma[dp] = (t0 | t1) as u16; - - x += 4; - sp += 4; - dp += 1; - } - - sigma[dp] = 0; - y += 4; - } - - let dp_base = (height.div_ceil(4) as usize) * mstr; - for x in 0..=width.div_ceil(4) as usize { - sigma[dp_base + x] = 0; - } - } - - { - let mut prev_row_sig = vec![0u16; width.div_ceil(4) as usize + 8]; - let mut sigprop = ForwardBitReader::<0>::new(&coded_data[lcup..lcup + len2]); - - for y in (0..height).step_by(4) { - let mut pattern = 0xFFFFu32; - if height - y < 4 { - pattern = 0x7777; - if height - y < 3 { - pattern = 0x3333; - if height - y < 2 { - pattern = 0x1111; - } - } - } - - let mut prev = 0u32; - let cur_row = (y >> 2) as usize * mstr; - let next_row = cur_row + mstr; - let dpp = (y * stride) as usize; - - for x in (0..width).step_by(4) { - let mut col_pattern = pattern; - let mut s = x as i32 + 4 - width as i32; - s = s.max(0); - col_pattern >>= (s * 4) as u32; - - let idx = (x >> 2) as usize; - let ps = - u32::from(prev_row_sig[idx]) | (u32::from(prev_row_sig[idx + 1]) << 16); - let ns = read_u32_pair(&sigma, next_row + idx); - let mut u = (ps & 0x8888_8888) >> 3; - if !stripe_causal { - u |= (ns & 0x1111_1111) << 3; - } - - let cs = read_u32_pair(&sigma, cur_row + idx); - let mut mbr = cs; - mbr |= (cs & 0x7777_7777) << 1; - mbr |= (cs & 0xEEEE_EEEE) >> 1; - mbr |= u; - let t = mbr; - mbr |= t << 4; - mbr |= t >> 4; - mbr |= prev >> 12; - mbr &= col_pattern; - mbr &= !cs; - - let mut new_sig = mbr; - if new_sig != 0 { - let mut cwd = sigprop.fetch(); - let mut cnt = 0u32; - let mut col_mask = 0xFu32; - let inv_sig = !cs & col_pattern; - - for i in (0..16).step_by(4) { - if (col_mask & new_sig) == 0 { - col_mask <<= 4; - continue; - } - - let mut sample_mask = 0x1111u32 & col_mask; - if (new_sig & sample_mask) != 0 { - new_sig &= !sample_mask; - if (cwd & 1) != 0 { - let t = 0x33u32 << i; - new_sig |= t & inv_sig; - } - cwd >>= 1; - cnt += 1; - } - - sample_mask <<= 1; - if (new_sig & sample_mask) != 0 { - new_sig &= !sample_mask; - if (cwd & 1) != 0 { - let t = 0x76u32 << i; - new_sig |= t & inv_sig; - } - cwd >>= 1; - cnt += 1; - } - - sample_mask <<= 1; - if (new_sig & sample_mask) != 0 { - new_sig &= !sample_mask; - if (cwd & 1) != 0 { - let t = 0xECu32 << i; - new_sig |= t & inv_sig; - } - cwd >>= 1; - cnt += 1; - } - - sample_mask <<= 1; - if (new_sig & sample_mask) != 0 { - new_sig &= !sample_mask; - if (cwd & 1) != 0 { - let t = 0xC8u32 << i; - new_sig |= t & inv_sig; - } - cwd >>= 1; - cnt += 1; - } - - col_mask <<= 4; - } - - if new_sig != 0 { - let mut dp = dpp + x as usize; - let value = 3u32 << (p - 2); - let mut col_mask = 0xFu32; - - for _ in 0..4 { - if (col_mask & new_sig) == 0 { - col_mask <<= 4; - dp += 1; - continue; - } - - let mut sample_mask = 0x1111u32 & col_mask; - if (new_sig & sample_mask) != 0 { - decoded_data[dp] = (cwd << 31) | value; - cwd >>= 1; - cnt += 1; - } - - sample_mask <<= 1; - if (new_sig & sample_mask) != 0 { - decoded_data[dp + stride as usize] = (cwd << 31) | value; - cwd >>= 1; - cnt += 1; - } - - sample_mask <<= 1; - if (new_sig & sample_mask) != 0 { - decoded_data[dp + 2 * stride as usize] = (cwd << 31) | value; - cwd >>= 1; - cnt += 1; - } - - sample_mask <<= 1; - if (new_sig & sample_mask) != 0 { - decoded_data[dp + 3 * stride as usize] = (cwd << 31) | value; - cwd >>= 1; - cnt += 1; - } - - col_mask <<= 4; - dp += 1; - } - } - - sigprop.advance(cnt); - } - - let combined_sig = new_sig | cs; - prev_row_sig[idx] = combined_sig as u16; - if idx + 1 < prev_row_sig.len() { - prev_row_sig[idx + 1] = (combined_sig >> 16) as u16; - } - - let t = combined_sig; - let mut next_prev = combined_sig; - next_prev |= (t & 0x7777) << 1; - next_prev |= (t & 0xEEEE) >> 1; - prev = (next_prev | u) & 0xF000; - } - } - } - - if num_passes > 2 { - let mut magref = ReverseBitReader::new_mrp(coded_data, lcup, len2); - let half = 1u32 << (p - 2); - let mstr = ((width.div_ceil(4) + 2 + 7) & !7) as usize; - let sigma_rows = height.div_ceil(4) as usize + 1; - let mut sigma = vec![0u16; sigma_rows * mstr]; - - let mut y = 0u32; - while y < height { - let sp_base = (y >> 1) as usize * sstr; - let dp_base = (y >> 2) as usize * mstr; - let mut x = 0u32; - let mut sp = sp_base; - let mut dp = dp_base; - while x < width { - let mut t0 = ((u32::from(scratch[sp]) & 0x30) >> 4) - | ((u32::from(scratch[sp]) & 0xC0) >> 2); - t0 |= ((u32::from(scratch[sp + 2]) & 0x30) << 4) - | ((u32::from(scratch[sp + 2]) & 0xC0) << 6); - let mut t1 = ((u32::from(scratch[sp + sstr]) & 0x30) >> 2) - | (u32::from(scratch[sp + sstr]) & 0xC0); - t1 |= ((u32::from(scratch[sp + sstr + 2]) & 0x30) << 6) - | ((u32::from(scratch[sp + sstr + 2]) & 0xC0) << 8); - sigma[dp] = (t0 | t1) as u16; - x += 4; - sp += 4; - dp += 1; - } - sigma[dp] = 0; - y += 4; - } - - let dp_base = (height.div_ceil(4) as usize) * mstr; - for x in 0..=width.div_ceil(4) as usize { - sigma[dp_base + x] = 0; - } - - for y in (0..height).step_by(4) { - let mut cur_sig_idx = (y >> 2) as usize * mstr; - let dpp = (y * stride) as usize; - - for i in (0..width).step_by(8) { - let cwd = magref.fetch(); - let sig = read_u32_pair(&sigma, cur_sig_idx); - cur_sig_idx += 2; - let mut col_mask = 0xFu32; - let mut cwd_mut = cwd; - - if sig != 0 { - for j in 0..8 { - if (sig & col_mask) != 0 { - let mut dp = dpp + i as usize + j; - let mut sample_mask = 0x1111_1111u32 & col_mask; - - for _ in 0..4 { - if (sig & sample_mask) != 0 { - let mut sym = cwd_mut & 1; - sym = (1 - sym) << (p - 1); - sym |= half; - decoded_data[dp] ^= sym; - cwd_mut >>= 1; - } - sample_mask <<= 1; - dp += stride as usize; - } - } - col_mask <<= 4; - } - } - - magref.advance(sig.count_ones()); - } - } - } - } - - Some(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::j2c::ht_block_encode::encode_code_block; - - #[test] - fn test_coefficient_to_i32_shifted_alignment() { - let aligned = 3u32 << (31 - 5); - assert_eq!(coefficient_to_i32(aligned, 5), 3); - assert_eq!(coefficient_to_i32(0x8000_0000 | aligned, 5), -3); - } - - #[test] - fn test_direct_ht_block_roundtrip_varied_4x4() { - let original: Vec = (0..16).map(|i| (i * 3) - 20).collect(); - let total_bitplanes = 6u8; - let encoded = encode_code_block(&original, 4, 4, total_bitplanes).expect("encode HT block"); - assert_eq!(encoded.num_coding_passes, 1); - - let mut decoded = vec![0u32; original.len()]; - let decoded_ok = decode_impl( - &encoded.data, - &mut decoded, - u32::from(encoded.num_zero_bitplanes), - u32::from(encoded.num_coding_passes), - encoded.data.len() as u32, - 0, - 4, - 4, - 4, - false, - ); - assert!(decoded_ok.is_some(), "encoded={:02x?}", encoded.data); - - let decoded_i32: Vec = decoded - .into_iter() - .map(|value| coefficient_to_i32(value, total_bitplanes)) - .collect(); - assert_eq!(decoded_i32, original, "encoded={:02x?}", encoded.data); - } - - #[test] - fn test_direct_ht_block_roundtrip_positive_varied_4x4() { - let original: Vec = (0..16).map(|i| i * 3).collect(); - let total_bitplanes = 6u8; - let encoded = encode_code_block(&original, 4, 4, total_bitplanes).expect("encode HT block"); - assert_eq!(encoded.num_coding_passes, 1); - - let mut decoded = vec![0u32; original.len()]; - let decoded_ok = decode_impl( - &encoded.data, - &mut decoded, - u32::from(encoded.num_zero_bitplanes), - u32::from(encoded.num_coding_passes), - encoded.data.len() as u32, - 0, - 4, - 4, - 4, - false, - ); - assert!(decoded_ok.is_some(), "encoded={:02x?}", encoded.data); - - let decoded_i32: Vec = decoded - .into_iter() - .map(|value| coefficient_to_i32(value, total_bitplanes)) - .collect(); - assert_eq!(decoded_i32, original, "encoded={:02x?}", encoded.data); - } -} diff --git a/crates/signinum-j2k-native/src/j2c/ht_block_encode.rs b/crates/signinum-j2k-native/src/j2c/ht_block_encode.rs deleted file mode 100644 index a06a7bab..00000000 --- a/crates/signinum-j2k-native/src/j2c/ht_block_encode.rs +++ /dev/null @@ -1,894 +0,0 @@ -//! Scalar HTJ2K cleanup-only block encoding. - -use alloc::vec; -use alloc::vec::Vec; - -use super::bitplane_encode::EncodedCodeBlock; -use super::ht_encode_tables::{ - HtUvlcTableEntry, HT_UVLC_ENCODE_TABLE, HT_VLC_ENCODE_TABLE0, HT_VLC_ENCODE_TABLE1, -}; - -const MEL_EXP: [usize; 13] = [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5]; -const MAX_HT_BITPLANES: u8 = 30; -const MEL_SIZE: usize = 192; -const VLC_SIZE: usize = 3072 - MEL_SIZE; -const MS_SIZE: usize = (16384usize * 16).div_ceil(15); - -struct MelEncoder { - buffer: Vec, - pos: usize, - remaining_bits: u8, - tmp: u8, - run: usize, - k: usize, - threshold: usize, -} - -impl MelEncoder { - fn new() -> Self { - Self { - buffer: vec![0; MEL_SIZE], - pos: 0, - remaining_bits: 8, - tmp: 0, - run: 0, - k: 0, - threshold: 1, - } - } - - fn emit_bit(&mut self, bit: bool) -> Result<(), &'static str> { - self.tmp = (self.tmp << 1) | u8::from(bit); - self.remaining_bits -= 1; - - if self.remaining_bits == 0 { - if self.pos >= self.buffer.len() { - return Err("HTJ2K MEL encoder buffer is full"); - } - - self.buffer[self.pos] = self.tmp; - self.pos += 1; - self.remaining_bits = if self.tmp == 0xFF { 7 } else { 8 }; - self.tmp = 0; - } - - Ok(()) - } - - fn encode(&mut self, bit: bool) -> Result<(), &'static str> { - if !bit { - self.run += 1; - if self.run >= self.threshold { - self.emit_bit(true)?; - self.run = 0; - self.k = (self.k + 1).min(MEL_EXP.len() - 1); - self.threshold = 1 << MEL_EXP[self.k]; - } - } else { - self.emit_bit(false)?; - let mut t = MEL_EXP[self.k]; - while t > 0 { - t -= 1; - self.emit_bit(((self.run >> t) & 1) != 0)?; - } - self.run = 0; - self.k = self.k.saturating_sub(1); - self.threshold = 1 << MEL_EXP[self.k]; - } - - Ok(()) - } -} - -struct VlcEncoder { - buffer: Vec, - pos: usize, - used_bits: u8, - tmp: u8, - last_greater_than_8f: bool, -} - -impl VlcEncoder { - fn new() -> Self { - let mut buffer = vec![0; VLC_SIZE]; - let last = buffer.len() - 1; - buffer[last] = 0xFF; - - Self { - buffer, - pos: 1, - used_bits: 4, - tmp: 0x0F, - last_greater_than_8f: true, - } - } - - fn encode(&mut self, mut codeword: u32, mut codeword_len: u8) -> Result<(), &'static str> { - while codeword_len > 0 { - if self.pos >= self.buffer.len() { - return Err("HTJ2K VLC encoder buffer is full"); - } - - let mut available_bits = 8 - u8::from(self.last_greater_than_8f) - self.used_bits; - let take = available_bits.min(codeword_len); - let mask = if take == 32 { - u32::MAX - } else { - (1u32 << take) - 1 - }; - self.tmp |= ((codeword & mask) as u8) << self.used_bits; - self.used_bits += take; - available_bits -= take; - codeword_len -= take; - codeword >>= take; - - if available_bits == 0 { - if self.last_greater_than_8f && self.tmp != 0x7F { - self.last_greater_than_8f = false; - continue; - } - - let write_index = self.buffer.len() - 1 - self.pos; - self.buffer[write_index] = self.tmp; - self.pos += 1; - self.last_greater_than_8f = self.tmp > 0x8F; - self.tmp = 0; - self.used_bits = 0; - } - } - - Ok(()) - } -} - -struct MagSgnEncoder { - buffer: Vec, - pos: usize, - max_bits: u8, - used_bits: u8, - tmp: u32, -} - -impl MagSgnEncoder { - fn new() -> Self { - Self { - buffer: vec![0; MS_SIZE], - pos: 0, - max_bits: 8, - used_bits: 0, - tmp: 0, - } - } - - fn encode(&mut self, mut codeword: u32, mut codeword_len: u32) -> Result<(), &'static str> { - while codeword_len > 0 { - if self.pos >= self.buffer.len() { - return Err("HTJ2K magnitude/sign encoder buffer is full"); - } - - let take = u32::from(self.max_bits - self.used_bits).min(codeword_len); - let mask = if take == 32 { - u32::MAX - } else { - (1u32 << take) - 1 - }; - self.tmp |= (codeword & mask) << self.used_bits; - self.used_bits += take as u8; - codeword >>= take; - codeword_len -= take; - - if self.used_bits >= self.max_bits { - self.buffer[self.pos] = self.tmp as u8; - self.pos += 1; - self.max_bits = if self.tmp == 0xFF { 7 } else { 8 }; - self.tmp = 0; - self.used_bits = 0; - } - } - - Ok(()) - } - - fn terminate(&mut self) -> Result<(), &'static str> { - if self.used_bits > 0 { - let unused = self.max_bits - self.used_bits; - self.tmp |= (0xFF & ((1u32 << unused) - 1)) << self.used_bits; - self.used_bits += unused; - - if self.tmp != 0xFF { - if self.pos >= self.buffer.len() { - return Err("HTJ2K magnitude/sign encoder buffer is full"); - } - - self.buffer[self.pos] = self.tmp as u8; - self.pos += 1; - } - } else if self.max_bits == 7 { - self.pos = self.pos.saturating_sub(1); - } - - Ok(()) - } -} - -pub(crate) fn encode_code_block( - coefficients: &[i32], - width: u32, - height: u32, - total_bitplanes: u8, -) -> Result { - if total_bitplanes == 0 || total_bitplanes > MAX_HT_BITPLANES { - return Err("HTJ2K scalar encoder currently supports 1..=30 bitplanes"); - } - - let max_magnitude = coefficients - .iter() - .map(|coefficient| coefficient.unsigned_abs()) - .max() - .unwrap_or(0); - - if max_magnitude == 0 { - return Ok(EncodedCodeBlock { - data: Vec::new(), - num_coding_passes: 0, - num_zero_bitplanes: total_bitplanes, - }); - } - - let block_bitplanes = (u32::BITS - max_magnitude.leading_zeros()) as u8; - if block_bitplanes > total_bitplanes { - return Err("HTJ2K block magnitude exceeds configured bitplane count"); - } - - let missing_msbs = total_bitplanes.saturating_sub(1); - let aligned = convert_to_aligned_sign_magnitude(coefficients, total_bitplanes); - let data = encode_cleanup_segment(&aligned, missing_msbs, width as usize, height as usize)?; - - Ok(EncodedCodeBlock { - data, - num_coding_passes: 1, - num_zero_bitplanes: missing_msbs, - }) -} - -fn convert_to_aligned_sign_magnitude(coefficients: &[i32], k_max: u8) -> Vec { - let shift = u32::from(31_u8.saturating_sub(k_max)); - - coefficients - .iter() - .map(|&coefficient| { - if coefficient == 0 { - 0 - } else { - let sign = if coefficient < 0 { 0x8000_0000 } else { 0 }; - let magnitude = coefficient.unsigned_abs() << shift; - sign | magnitude - } - }) - .collect() -} - -fn encode_cleanup_segment( - coefficients: &[u32], - missing_msbs: u8, - width: usize, - height: usize, -) -> Result, &'static str> { - let mut mel = MelEncoder::new(); - let mut vlc = VlcEncoder::new(); - let mut ms = MagSgnEncoder::new(); - - let p = 30_u32.saturating_sub(u32::from(missing_msbs)); - let stride = width; - - let mut e_val = [0u8; 513]; - let mut cx_val = [0u8; 513]; - - let mut e_qmax = [0i32; 2]; - let mut e_q = [0i32; 8]; - let mut rho = [0i32; 2]; - let mut c_q0 = 0usize; - let mut s = [0u32; 8]; - let mut sp = 0usize; - let mut x = 0usize; - - while x < width { - encode_first_quad_pair( - coefficients, - stride, - height, - p, - &mut sp, - x, - &mut e_val, - &mut cx_val, - &mut c_q0, - &mut rho, - &mut e_q, - &mut e_qmax, - &mut s, - &mut mel, - &mut vlc, - &mut ms, - )?; - x += 4; - } - - let e_val_sentinel = width.div_ceil(2) + 1; - e_val[e_val_sentinel] = 0; - - let mut y = 2usize; - while y < height { - let mut lep = 0usize; - let mut max_e = i32::from(e_val[lep].max(e_val[lep + 1])) - 1; - e_val[lep] = 0; - - let mut lcxp = 0usize; - c_q0 = usize::from(cx_val[lcxp]) + (usize::from(cx_val[lcxp + 1]) << 2); - cx_val[lcxp] = 0; - - sp = y * stride; - x = 0; - while x < width { - encode_non_initial_quad_pair( - coefficients, - stride, - width, - height, - y, - p, - &mut sp, - x, - &mut e_val, - &mut cx_val, - &mut lep, - &mut lcxp, - &mut max_e, - &mut c_q0, - &mut rho, - &mut e_q, - &mut e_qmax, - &mut s, - &mut mel, - &mut vlc, - &mut ms, - )?; - x += 4; - } - - y += 2; - } - - terminate_mel_vlc(&mut mel, &mut vlc)?; - ms.terminate()?; - - let total_len = ms.pos + mel.pos + vlc.pos; - if total_len < 2 { - return Err("HTJ2K cleanup segment is too short"); - } - - let mut data = Vec::with_capacity(total_len); - data.extend_from_slice(&ms.buffer[..ms.pos]); - data.extend_from_slice(&mel.buffer[..mel.pos]); - let vlc_start = vlc.buffer.len() - vlc.pos; - data.extend_from_slice(&vlc.buffer[vlc_start..]); - - let locator_bytes = mel.pos + vlc.pos; - let last = data.len() - 1; - let prev = data.len() - 2; - data[last] = (locator_bytes >> 4) as u8; - data[prev] = (data[prev] & 0xF0) | ((locator_bytes as u8) & 0x0F); - - Ok(data) -} - -fn process_sample( - slot: usize, - value: u32, - p: u32, - rho_acc: &mut i32, - e_q: &mut [i32; 8], - e_qmax: &mut i32, - s: &mut [u32; 8], -) { - let mut val = value.wrapping_add(value); - val >>= p; - val &= !1u32; - if val != 0 { - *rho_acc |= 1 << (slot & 0x3); - val -= 1; - e_q[slot] = (u32::BITS - val.leading_zeros()) as i32; - *e_qmax = (*e_qmax).max(e_q[slot]); - val -= 1; - s[slot] = val + (value >> 31); - } -} - -#[allow(clippy::too_many_arguments)] -fn encode_first_quad_pair( - coefficients: &[u32], - stride: usize, - height: usize, - p: u32, - sp: &mut usize, - x: usize, - e_val: &mut [u8; 513], - cx_val: &mut [u8; 513], - c_q0: &mut usize, - rho: &mut [i32; 2], - e_q: &mut [i32; 8], - e_qmax: &mut [i32; 2], - s: &mut [u32; 8], - mel: &mut MelEncoder, - vlc: &mut VlcEncoder, - ms: &mut MagSgnEncoder, -) -> Result<(), &'static str> { - let lep = x / 2; - let lcxp = x / 2; - - process_sample(0, coefficients[*sp], p, &mut rho[0], e_q, &mut e_qmax[0], s); - process_sample( - 1, - if height > 1 { - coefficients[*sp + stride] - } else { - 0 - }, - p, - &mut rho[0], - e_q, - &mut e_qmax[0], - s, - ); - *sp += 1; - - if x + 1 < stride { - process_sample(2, coefficients[*sp], p, &mut rho[0], e_q, &mut e_qmax[0], s); - process_sample( - 3, - if height > 1 { - coefficients[*sp + stride] - } else { - 0 - }, - p, - &mut rho[0], - e_q, - &mut e_qmax[0], - s, - ); - *sp += 1; - } - - let u_q0 = encode_quad_initial_row( - 0, *c_q0, rho[0], e_qmax[0], e_q, s, lep, lcxp, e_val, cx_val, mel, vlc, ms, - )?; - - if x + 2 < stride { - process_sample(4, coefficients[*sp], p, &mut rho[1], e_q, &mut e_qmax[1], s); - process_sample( - 5, - if height > 1 { - coefficients[*sp + stride] - } else { - 0 - }, - p, - &mut rho[1], - e_q, - &mut e_qmax[1], - s, - ); - *sp += 1; - - if x + 3 < stride { - process_sample(6, coefficients[*sp], p, &mut rho[1], e_q, &mut e_qmax[1], s); - process_sample( - 7, - if height > 1 { - coefficients[*sp + stride] - } else { - 0 - }, - p, - &mut rho[1], - e_q, - &mut e_qmax[1], - s, - ); - *sp += 1; - } - - let c_q1 = ((rho[0] >> 1) | (rho[0] & 1)) as usize; - let u_q1 = encode_quad_initial_row( - 4, - c_q1, - rho[1], - e_qmax[1], - e_q, - s, - lep + 1, - lcxp + 1, - e_val, - cx_val, - mel, - vlc, - ms, - )?; - - if u_q0 > 0 && u_q1 > 0 { - mel.encode(u_q0.min(u_q1) > 2)?; - } - encode_uvlc(u_q0, u_q1, &mut *vlc)?; - *c_q0 = ((rho[1] >> 1) | (rho[1] & 1)) as usize; - } else { - encode_uvlc(u_q0, 0, &mut *vlc)?; - *c_q0 = 0; - } - - *rho = [0; 2]; - *e_q = [0; 8]; - *e_qmax = [0; 2]; - *s = [0; 8]; - - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -fn encode_non_initial_quad_pair( - coefficients: &[u32], - stride: usize, - width: usize, - height: usize, - y: usize, - p: u32, - sp: &mut usize, - x: usize, - e_val: &mut [u8; 513], - cx_val: &mut [u8; 513], - lep: &mut usize, - lcxp: &mut usize, - max_e: &mut i32, - c_q0: &mut usize, - rho: &mut [i32; 2], - e_q: &mut [i32; 8], - e_qmax: &mut [i32; 2], - s: &mut [u32; 8], - mel: &mut MelEncoder, - vlc: &mut VlcEncoder, - ms: &mut MagSgnEncoder, -) -> Result<(), &'static str> { - process_sample(0, coefficients[*sp], p, &mut rho[0], e_q, &mut e_qmax[0], s); - process_sample( - 1, - if y + 1 < height { - coefficients[*sp + stride] - } else { - 0 - }, - p, - &mut rho[0], - e_q, - &mut e_qmax[0], - s, - ); - *sp += 1; - - if x + 1 < width { - process_sample(2, coefficients[*sp], p, &mut rho[0], e_q, &mut e_qmax[0], s); - process_sample( - 3, - if y + 1 < height { - coefficients[*sp + stride] - } else { - 0 - }, - p, - &mut rho[0], - e_q, - &mut e_qmax[0], - s, - ); - *sp += 1; - } - - let prev_max = *max_e; - let u_q0 = encode_quad_non_initial_row( - 0, *c_q0, rho[0], e_qmax[0], prev_max, e_q, s, *lep, *lcxp, e_val, cx_val, mel, vlc, ms, - )?; - - e_val[*lep] = e_val[*lep].max(e_q[1] as u8); - *lep += 1; - *max_e = i32::from(e_val[*lep].max(e_val[*lep + 1])) - 1; - e_val[*lep] = e_q[3] as u8; - cx_val[*lcxp] |= ((rho[0] & 2) >> 1) as u8; - *lcxp += 1; - let c_q1 = usize::from(cx_val[*lcxp]) + (usize::from(cx_val[*lcxp + 1]) << 2); - cx_val[*lcxp] = ((rho[0] & 8) >> 3) as u8; - - let mut u_q1 = 0; - if x + 2 < width { - process_sample(4, coefficients[*sp], p, &mut rho[1], e_q, &mut e_qmax[1], s); - process_sample( - 5, - if y + 1 < height { - coefficients[*sp + stride] - } else { - 0 - }, - p, - &mut rho[1], - e_q, - &mut e_qmax[1], - s, - ); - *sp += 1; - - if x + 3 < width { - process_sample(6, coefficients[*sp], p, &mut rho[1], e_q, &mut e_qmax[1], s); - process_sample( - 7, - if y + 1 < height { - coefficients[*sp + stride] - } else { - 0 - }, - p, - &mut rho[1], - e_q, - &mut e_qmax[1], - s, - ); - *sp += 1; - } - - let mut c_q1_local = c_q1; - c_q1_local |= ((rho[0] & 4) >> 1) as usize; - c_q1_local |= ((rho[0] & 8) >> 2) as usize; - - u_q1 = encode_quad_non_initial_row( - 4, c_q1_local, rho[1], e_qmax[1], *max_e, e_q, s, *lep, *lcxp, e_val, cx_val, mel, vlc, - ms, - )?; - - e_val[*lep] = e_val[*lep].max(e_q[5] as u8); - *lep += 1; - *max_e = i32::from(e_val[*lep].max(e_val[*lep + 1])) - 1; - e_val[*lep] = e_q[7] as u8; - cx_val[*lcxp] |= ((rho[1] & 2) >> 1) as u8; - *lcxp += 1; - *c_q0 = usize::from(cx_val[*lcxp]) + (usize::from(cx_val[*lcxp + 1]) << 2); - cx_val[*lcxp] = ((rho[1] & 8) >> 3) as u8; - - *c_q0 |= ((rho[1] & 4) >> 1) as usize; - *c_q0 |= ((rho[1] & 8) >> 2) as usize; - } else { - *c_q0 = 0; - } - - encode_uvlc_non_initial(u_q0, u_q1, &mut *vlc)?; - - *rho = [0; 2]; - *e_q = [0; 8]; - *e_qmax = [0; 2]; - *s = [0; 8]; - - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -fn encode_quad_initial_row( - offset: usize, - c_q: usize, - rho: i32, - e_qmax: i32, - e_q: &[i32; 8], - s: &[u32; 8], - lep: usize, - lcxp: usize, - e_val: &mut [u8; 513], - cx_val: &mut [u8; 513], - mel: &mut MelEncoder, - vlc: &mut VlcEncoder, - ms: &mut MagSgnEncoder, -) -> Result { - let u_q = e_qmax.max(1) - 1; - let mut eps = 0u16; - - if u_q > 0 { - eps |= u16::from((e_q[offset] == e_qmax) as u8); - eps |= u16::from((e_q[offset + 1] == e_qmax) as u8) << 1; - eps |= u16::from((e_q[offset + 2] == e_qmax) as u8) << 2; - eps |= u16::from((e_q[offset + 3] == e_qmax) as u8) << 3; - } - - e_val[lep] = e_val[lep].max(e_q[offset + 1] as u8); - e_val[lep + 1] = e_q[offset + 3] as u8; - cx_val[lcxp] |= ((rho & 2) >> 1) as u8; - cx_val[lcxp + 1] = ((rho & 8) >> 3) as u8; - - let tuple = HT_VLC_ENCODE_TABLE0[(c_q << 8) | ((rho as usize) << 4) | eps as usize]; - vlc.encode(u32::from(tuple >> 8), ((tuple >> 4) & 0x7) as u8)?; - - if c_q == 0 { - mel.encode(rho != 0)?; - } - - encode_mag_signs(rho, e_qmax.max(1), tuple, s, offset, ms)?; - Ok(u_q) -} - -#[allow(clippy::too_many_arguments)] -fn encode_quad_non_initial_row( - offset: usize, - c_q: usize, - rho: i32, - e_qmax: i32, - max_e: i32, - e_q: &[i32; 8], - s: &[u32; 8], - _lep: usize, - _lcxp: usize, - _e_val: &mut [u8; 513], - _cx_val: &mut [u8; 513], - mel: &mut MelEncoder, - vlc: &mut VlcEncoder, - ms: &mut MagSgnEncoder, -) -> Result { - let kappa = if (rho & (rho - 1)) != 0 { - max_e.max(1) - } else { - 1 - }; - let u_q = e_qmax.max(kappa) - kappa; - let mut eps = 0u16; - - if u_q > 0 { - eps |= u16::from((e_q[offset] == e_qmax) as u8); - eps |= u16::from((e_q[offset + 1] == e_qmax) as u8) << 1; - eps |= u16::from((e_q[offset + 2] == e_qmax) as u8) << 2; - eps |= u16::from((e_q[offset + 3] == e_qmax) as u8) << 3; - } - - let tuple = HT_VLC_ENCODE_TABLE1[(c_q << 8) | ((rho as usize) << 4) | eps as usize]; - vlc.encode(u32::from(tuple >> 8), ((tuple >> 4) & 0x7) as u8)?; - - if c_q == 0 { - mel.encode(rho != 0)?; - } - - encode_mag_signs(rho, e_qmax.max(kappa), tuple, s, offset, ms)?; - Ok(u_q) -} - -fn encode_mag_signs( - rho: i32, - u_q: i32, - tuple: u16, - s: &[u32; 8], - offset: usize, - ms: &mut MagSgnEncoder, -) -> Result<(), &'static str> { - let e_k = tuple & 0xF; - - let mut encode = |bit: i32, shift: u32, sample_offset: usize| -> Result<(), &'static str> { - let sample_mask = 1 << bit; - if (rho & sample_mask) == 0 { - return Ok(()); - } - - let reduction = ((u32::from(e_k) >> shift) & 1) as i32; - let magnitude_bits = (u_q - reduction) as u32; - let payload = if magnitude_bits == 0 { - 0 - } else { - s[offset + sample_offset] & ((1u32 << magnitude_bits) - 1) - }; - ms.encode(payload, magnitude_bits) - }; - - encode(0, 0, 0)?; - encode(1, 1, 1)?; - encode(2, 2, 2)?; - encode(3, 3, 3)?; - Ok(()) -} - -fn encode_uvlc(u_q0: i32, u_q1: i32, vlc: &mut VlcEncoder) -> Result<(), &'static str> { - if u_q0 > 2 && u_q1 > 2 { - let first = HT_UVLC_ENCODE_TABLE[(u_q0 - 2) as usize]; - let second = HT_UVLC_ENCODE_TABLE[(u_q1 - 2) as usize]; - encode_uvlc_pair(vlc, first, second) - } else if u_q0 > 2 && u_q1 > 0 { - let first = HT_UVLC_ENCODE_TABLE[u_q0 as usize]; - vlc.encode(u32::from(first.pre), first.pre_len)?; - vlc.encode((u_q1 - 1) as u32, 1)?; - vlc.encode(u32::from(first.suf), first.suf_len) - } else { - let first = HT_UVLC_ENCODE_TABLE[u_q0.max(0) as usize]; - let second = HT_UVLC_ENCODE_TABLE[u_q1.max(0) as usize]; - encode_uvlc_pair(vlc, first, second) - } -} - -fn encode_uvlc_non_initial(u_q0: i32, u_q1: i32, vlc: &mut VlcEncoder) -> Result<(), &'static str> { - let first = HT_UVLC_ENCODE_TABLE[u_q0.max(0) as usize]; - let second = HT_UVLC_ENCODE_TABLE[u_q1.max(0) as usize]; - encode_uvlc_pair(vlc, first, second) -} - -fn encode_uvlc_pair( - vlc: &mut VlcEncoder, - first: HtUvlcTableEntry, - second: HtUvlcTableEntry, -) -> Result<(), &'static str> { - vlc.encode(u32::from(first.pre), first.pre_len)?; - vlc.encode(u32::from(second.pre), second.pre_len)?; - vlc.encode(u32::from(first.suf), first.suf_len)?; - vlc.encode(u32::from(second.suf), second.suf_len) -} - -fn terminate_mel_vlc(mel: &mut MelEncoder, vlc: &mut VlcEncoder) -> Result<(), &'static str> { - if mel.run > 0 { - mel.emit_bit(true)?; - } - - mel.tmp = (u16::from(mel.tmp) << mel.remaining_bits) as u8; - let mel_mask = ((0xFFu16 << mel.remaining_bits) & 0xFF) as u8; - let vlc_mask = if vlc.used_bits == 0 { - 0 - } else { - ((1u16 << vlc.used_bits) - 1) as u8 - }; - - if (mel_mask | vlc_mask) == 0 { - return Ok(()); - } - - let fused = mel.tmp | vlc.tmp; - let fused_ok = - (((fused ^ mel.tmp) & mel_mask) | ((fused ^ vlc.tmp) & vlc_mask)) == 0 && fused != 0xFF; - - if fused_ok && vlc.pos > 1 { - if mel.pos >= mel.buffer.len() { - return Err("HTJ2K MEL encoder buffer is full"); - } - - mel.buffer[mel.pos] = fused; - mel.pos += 1; - } else { - if mel.pos >= mel.buffer.len() { - return Err("HTJ2K MEL encoder buffer is full"); - } - if vlc.pos >= vlc.buffer.len() { - return Err("HTJ2K VLC encoder buffer is full"); - } - - mel.buffer[mel.pos] = mel.tmp; - mel.pos += 1; - let write_index = vlc.buffer.len() - 1 - vlc.pos; - vlc.buffer[write_index] = vlc.tmp; - vlc.pos += 1; - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_convert_to_aligned_sign_magnitude() { - let aligned = convert_to_aligned_sign_magnitude(&[0, 1, -2, 3], 2); - assert_eq!(aligned, vec![0, 0x2000_0000, 0xC000_0000, 0x6000_0000]); - } - - #[test] - fn test_encode_cleanup_only_nonzero_block() { - let encoded = encode_code_block(&[1], 1, 1, 5).expect("encode HT block"); - assert_eq!(encoded.num_coding_passes, 1); - assert_eq!(encoded.num_zero_bitplanes, 4); - assert!(encoded.data.len() >= 2); - } -} diff --git a/crates/signinum-j2k-native/src/j2c/quantize.rs b/crates/signinum-j2k-native/src/j2c/quantize.rs deleted file mode 100644 index a1817125..00000000 --- a/crates/signinum-j2k-native/src/j2c/quantize.rs +++ /dev/null @@ -1,250 +0,0 @@ -//! Forward quantization for JPEG 2000 encoding. -//! -//! - Lossless (reversible 5-3): No quantization, just sign/magnitude conversion -//! - Lossy (irreversible 9-7): Scalar deadzone quantization with step sizes -//! derived from the DWT subband gain norms. - -use alloc::vec; -use alloc::vec::Vec; - -use crate::math::{floor_f32, floor_f64, log2_f64, powi_f64, round_f32, round_f64}; - -/// Quantization parameters for a single subband. -#[derive(Debug, Clone, Copy)] -pub(crate) struct QuantStepSize { - pub(crate) exponent: u16, - pub(crate) mantissa: u16, -} - -impl QuantStepSize { - /// Compute the actual step size Δ = 2^(exp - guard_bits) × (1 + mantissa/2048). - fn delta(&self, guard_bits: u8) -> f32 { - let rb = self.exponent as i32 - guard_bits as i32; - let base = if rb >= 0 { - (1u32 << rb) as f32 - } else { - 1.0 / (1u32 << (-rb)) as f32 - }; - base * (1.0 + self.mantissa as f32 / 2048.0) - } -} - -/// Compute default quantization step sizes for the irreversible 9-7 transform. -/// -/// The step sizes are derived from the DWT 9-7 subband gain norms (Table E.1 in T.800). -/// For lossless mode, step sizes are not used (exponents store bit depth info only). -pub(crate) fn compute_step_sizes( - bit_depth: u8, - num_decompositions: u8, - reversible: bool, - guard_bits: u8, -) -> Vec { - let mut step_sizes = Vec::new(); - - if reversible { - // For reversible 5-3, QCD stores the subband exponent only. - // The decoder reconstructs the number of bitplanes as: - // Mb = guard_bits + exponent - 1 - // For lossless coding we therefore need exponents that reproduce the - // reversible subband dynamic range: - // LL => bit_depth + 0 - // HL/LH => bit_depth + 1 - // HH => bit_depth + 2 - // This gain depends on subband orientation, not decomposition level. - step_sizes.push(QuantStepSize { - exponent: bit_depth as u16, - mantissa: 0, - }); - - for _ in 0..num_decompositions { - step_sizes.push(QuantStepSize { - exponent: bit_depth as u16 + 1, - mantissa: 0, - }); - step_sizes.push(QuantStepSize { - exponent: bit_depth as u16 + 1, - mantissa: 0, - }); - step_sizes.push(QuantStepSize { - exponent: bit_depth as u16 + 2, - mantissa: 0, - }); - } - } else { - // For irreversible 9-7: compute step sizes from norm gains - // Base step size for the image - let base_step = 1.0 / (1u32 << bit_depth) as f32; - - // DWT 9-7 analysis gain norms (squared), per Table E.1 - // These are the L2 norms of the analysis basis functions. - let ll_gain = 1.0f64; - let hl_gains: Vec = (0..num_decompositions) - .map(|l| dwt97_subband_gain(l, false, true)) - .collect(); - let lh_gains: Vec = (0..num_decompositions) - .map(|l| dwt97_subband_gain(l, true, false)) - .collect(); - let hh_gains: Vec = (0..num_decompositions) - .map(|l| dwt97_subband_gain(l, true, true)) - .collect(); - - // LL subband - step_sizes.push(step_from_gain( - base_step as f64 / ll_gain, - guard_bits, - bit_depth, - )); - - for level in 0..num_decompositions as usize { - step_sizes.push(step_from_gain( - base_step as f64 / hl_gains[level], - guard_bits, - bit_depth, - )); - step_sizes.push(step_from_gain( - base_step as f64 / lh_gains[level], - guard_bits, - bit_depth, - )); - step_sizes.push(step_from_gain( - base_step as f64 / hh_gains[level], - guard_bits, - bit_depth, - )); - } - } - - step_sizes -} - -/// Approximate DWT 9-7 subband gain for a given decomposition level. -fn dwt97_subband_gain(level: u8, high_row: bool, high_col: bool) -> f64 { - // Approximate gains based on analysis filter norms - // Low-pass analysis gain ≈ 1.0 per level (normalized) - // High-pass analysis gain increases with level - let low = 1.0f64; - let high = 2.0f64; - - let row_gain = if high_row { - high * (1u64 << level) as f64 - } else { - low - }; - let col_gain = if high_col { - high * (1u64 << level) as f64 - } else { - low - }; - - row_gain * col_gain -} - -fn step_from_gain(step: f64, guard_bits: u8, bit_depth: u8) -> QuantStepSize { - if step <= 0.0 { - return QuantStepSize { - exponent: guard_bits as u16 + bit_depth as u16, - mantissa: 0, - }; - } - - let log2_step = log2_f64(step); - let exponent = -(floor_f64(log2_step) as i32) + guard_bits as i32; - let exponent = exponent.clamp(0, 31) as u16; - - // mantissa = (step / 2^(-exponent+guard_bits) - 1) * 2048 - let reconstructed_base = powi_f64(2.0, -(exponent as i32) + guard_bits as i32); - let mantissa = if reconstructed_base > 0.0 { - round_f64((step / reconstructed_base - 1.0) * 2048.0) as u16 - } else { - 0 - }; - - QuantStepSize { - exponent, - mantissa: mantissa.min(2047), - } -} - -/// Quantize wavelet coefficients for a single subband. -/// -/// For lossless: converts f32 to i32 (round to nearest integer). -/// For lossy: applies scalar deadzone quantization. -/// -/// Returns (magnitude, sign) pairs packed as i32 values. -pub(crate) fn quantize_subband( - coefficients: &[f32], - step_size: &QuantStepSize, - guard_bits: u8, - reversible: bool, -) -> Vec { - if reversible { - // No quantization: round to nearest integer - coefficients.iter().map(|&c| round_f32(c) as i32).collect() - } else { - let delta = step_size.delta(guard_bits); - if delta <= 0.0 { - return vec![0i32; coefficients.len()]; - } - let inv_delta = 1.0 / delta; - - coefficients - .iter() - .map(|&c| { - // Deadzone quantization: q = sign(c) * floor(|c| / Δ) - let sign = if c < 0.0 { -1 } else { 1 }; - let magnitude = floor_f32(c.abs() * inv_delta) as i32; - sign * magnitude - }) - .collect() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_lossless_quantize() { - let coeffs = vec![10.0, -5.0, 3.7, -8.2, 0.0]; - let step = QuantStepSize { - exponent: 12, - mantissa: 0, - }; - let result = quantize_subband(&coeffs, &step, 1, true); - assert_eq!(result, vec![10, -5, 4, -8, 0]); - } - - #[test] - fn test_lossy_quantize() { - let coeffs = vec![10.0, -5.0, 0.3, -0.1]; - let step = QuantStepSize { - exponent: 1, - mantissa: 0, - }; - let delta = step.delta(1); - assert!((delta - 1.0).abs() < 0.01); - - let result = quantize_subband(&coeffs, &step, 1, false); - assert_eq!(result[0], 10); - assert_eq!(result[1], -5); - assert_eq!(result[2], 0); // Below deadzone - assert_eq!(result[3], 0); // Below deadzone - } - - #[test] - fn test_compute_step_sizes_reversible() { - let steps = compute_step_sizes(8, 3, true, 1); - // 1 LL + 3 levels × 3 subbands = 10 - assert_eq!(steps.len(), 10); - // All mantissas should be 0 for reversible - assert!(steps.iter().all(|s| s.mantissa == 0)); - let exponents: Vec = steps.iter().map(|s| s.exponent).collect(); - assert_eq!(exponents, vec![8, 9, 9, 10, 9, 9, 10, 9, 9, 10]); - } - - #[test] - fn test_compute_step_sizes_irreversible() { - let steps = compute_step_sizes(8, 3, false, 1); - assert_eq!(steps.len(), 10); - } -} diff --git a/crates/signinum-j2k-native/src/lib.rs b/crates/signinum-j2k-native/src/lib.rs deleted file mode 100644 index 269b83a5..00000000 --- a/crates/signinum-j2k-native/src/lib.rs +++ /dev/null @@ -1,2856 +0,0 @@ -/*! -Internal pure-Rust JPEG 2000 codec engine for `signinum`. - -This module tree was imported from the `dicom-toolkit-jpeg2000` 0.5.0 crate -and adapted in-repo so `signinum-j2k` no longer depends on an external -production decoder crate. - -`dicom-toolkit-jpeg2000` is the JPEG 2000 engine used by `dicom-toolkit-rs`. -It is a maintained fork of the original `hayro-jpeg2000` project with -DICOM-focused extensions, including native-bit-depth decode for 8/12/16-bit -images and pure-Rust JPEG 2000 encoding. - -The crate can decode both raw JPEG 2000 codestreams (`.j2c`) and images wrapped -inside the JP2 container format. The decoder supports the vast majority of features -defined in the JPEG 2000 core coding system (ISO/IEC 15444-1) as well as some color -spaces from the extensions (ISO/IEC 15444-2). There are still some missing pieces -for some "obscure" features (for example support for progression order -changes in tile-parts), but the features that commonly appear in real-world -images are supported. - -The crate offers both a high-level 8-bit decode path for general image use and -a native-bit-depth decode path for integrations such as DICOM, plus encoder APIs -for emitting raw JPEG 2000 and HTJ2K codestreams. - -# Example -```rust,no_run -use signinum_j2k_native::{DecodeSettings, Image}; - -let data = std::fs::read("image.jp2").unwrap(); -let image = Image::new(&data, &DecodeSettings::default()).unwrap(); - -println!( - "{}x{} image in {:?} with alpha={}", - image.width(), - image.height(), - image.color_space(), - image.has_alpha(), -); - -let bitmap = image.decode().unwrap(); -``` - -If you want to see a more comprehensive example, please take a look -at the example in [GitHub](https://github.com/knopkem/dicom-toolkit-rs/blob/main/crates/dicom-toolkit-jpeg2000/examples/png.rs), -which shows the main steps needed to convert a JPEG 2000 image into PNG. - -# Testing -The decoder has been tested against 20.000+ images scraped from random PDFs -on the internet and also passes a large part of the `OpenJPEG` test suite. So you -can expect the crate to perform decently in terms of decoding correctness. - -# Performance -A decent amount of effort has already been put into optimizing this crate -(both in terms of raw performance but also memory allocations). However, there -are some more important optimizations that have not been implemented yet, so -there is definitely still room for improvement (and I am planning on implementing -them eventually). - -Overall, you should expect this crate to have worse performance than `OpenJPEG`, -but the difference gap should not be too large. - -# Safety -By default, the crate has the `simd` feature enabled, which uses the -[`fearless_simd`](https://github.com/linebender/fearless_simd) crate to accelerate -important parts of the pipeline. If you want to eliminate any usage of unsafe -in this crate as well as its dependencies, you can simply disable this -feature, at the cost of worse decoding performance. Unsafe code is forbidden -via a crate-level attribute. - -The crate is `no_std` compatible but requires an allocator to be available. -*/ - -#![cfg_attr(not(feature = "std"), no_std)] -#![forbid(unsafe_code)] -#![forbid(missing_docs)] -#![allow(clippy::too_many_arguments)] - -extern crate alloc; - -use alloc::vec; -use alloc::vec::Vec; - -use crate::error::{bail, err}; -use crate::j2c::{ComponentData, Header}; -use crate::jp2::cdef::{ChannelAssociation, ChannelType}; -use crate::jp2::cmap::ComponentMappingType; -use crate::jp2::colr::{CieLab, EnumeratedColorspace}; -use crate::jp2::icc::ICCMetadata; -use crate::jp2::{DecodedImage, ImageBoxes}; - -pub mod error; -#[macro_use] -pub(crate) mod log; -mod direct_plan; -pub(crate) mod math; -pub(crate) mod profile; -pub(crate) mod writer; - -use crate::math::{dispatch, f32x8, Level, Simd, SIMD_WIDTH}; -#[doc(hidden)] -pub use direct_plan::{ - HtOwnedCodeBlockBatchJob, HtOwnedSubBandPlan, J2kDirectBandId, J2kDirectColorPlan, - J2kDirectGrayscalePlan, J2kDirectGrayscaleStep, J2kDirectIdwtStep, J2kDirectStoreStep, - J2kOwnedCodeBlockBatchJob, J2kOwnedSubBandPlan, -}; - -/// Maps an output coordinate within an IDWT step to the source sub-band index. -/// -/// `origin` is the global coordinate of the IDWT output rectangle, -/// `local_coord` is the coordinate within that output rectangle, and -/// `low_pass` selects the low-pass (`LL`/`LH`) or high-pass (`HL`/`HH`) band -/// along one axis. This helper is exposed so backend adapters can compute -/// required input windows with the same odd-origin rounding as the native IDWT. -#[must_use] -pub fn idwt_band_index(origin: u32, local_coord: u32, low_pass: bool) -> u32 { - let global = u64::from(origin) + u64::from(local_coord); - let origin = u64::from(origin); - let index = if low_pass { - global.div_ceil(2).saturating_sub(origin.div_ceil(2)) - } else { - (global / 2).saturating_sub(origin / 2) - }; - u32::try_from(index).unwrap_or(u32::MAX) -} - -pub use error::{ - ColorError, DecodeError, DecodingError, FormatError, MarkerError, Result, TileError, - ValidationError, -}; -pub use j2c::encode::{ - encode, encode_htj2k, encode_with_accelerator, EncodeOptions, EncodeProgressionOrder, -}; -pub use j2c::{CpuDecodeParallelism, DecoderContext}; - -mod j2c; -mod jp2; -pub(crate) mod reader; -#[doc(hidden)] -pub use j2c::ht_encode_tables::HtUvlcTableEntry; - -/// Hidden HTJ2K code-block job description for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct HtCodeBlockDecodeJob<'a> { - /// Combined cleanup/refinement bytes for the code block. - pub data: &'a [u8], - /// Cleanup segment length in bytes. - pub cleanup_length: u32, - /// Refinement segment length in bytes. - pub refinement_length: u32, - /// Code-block width in samples. - pub width: u32, - /// Code-block height in samples. - pub height: u32, - /// Output row stride, in samples, for the target sub-band storage. - pub output_stride: usize, - /// Missing most-significant bit planes for this code block. - pub missing_bit_planes: u8, - /// Number of coding passes present for this code block. - pub number_of_coding_passes: u8, - /// Total coded bitplanes for the parent sub-band. - pub num_bitplanes: u8, - /// Whether vertically causal context was enabled. - pub stripe_causal: bool, - /// Whether strict decode validation is enabled for the parent image. - pub strict: bool, - /// Dequantization step to apply to decoded coefficients. - pub dequantization_step: f32, -} - -/// Hidden HTJ2K batched code-block decode job for one sub-band. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct HtCodeBlockBatchJob<'a> { - /// X offset within the target sub-band coefficient buffer. - pub output_x: u32, - /// Y offset within the target sub-band coefficient buffer. - pub output_y: u32, - /// The actual code-block decode parameters. - pub code_block: HtCodeBlockDecodeJob<'a>, -} - -/// Hidden HTJ2K batched sub-band decode request for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct HtSubBandDecodeJob<'a> { - /// Sub-band width in samples. - pub width: u32, - /// Sub-band height in samples. - pub height: u32, - /// Code blocks to decode into this sub-band. - pub jobs: &'a [HtCodeBlockBatchJob<'a>], -} - -/// Hidden classic J2K sub-band kind for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum J2kSubBandType { - /// Low-low sub-band. - LowLow, - /// High-low sub-band. - HighLow, - /// Low-high sub-band. - LowHigh, - /// High-high sub-band. - HighHigh, -} - -/// Hidden classic J2K code-block style for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct J2kCodeBlockStyle { - /// Selective arithmetic coding bypass was enabled. - pub selective_arithmetic_coding_bypass: bool, - /// Context probabilities reset after each pass. - pub reset_context_probabilities: bool, - /// Coding terminated after each pass. - pub termination_on_each_pass: bool, - /// Vertically causal context was enabled. - pub vertically_causal_context: bool, - /// Segmentation symbols were enabled. - pub segmentation_symbols: bool, -} - -/// Hidden classic J2K coded segment for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct J2kCodeBlockSegment { - /// Byte offset of this segment within the combined payload. - pub data_offset: u32, - /// Segment payload length in bytes. - pub data_length: u32, - /// First coding pass covered by this segment. - pub start_coding_pass: u8, - /// One-past-last coding pass covered by this segment. - pub end_coding_pass: u8, - /// Whether this segment is decoded through the arithmetic path. - pub use_arithmetic: bool, -} - -/// Hidden classic J2K code-block job description for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct J2kCodeBlockDecodeJob<'a> { - /// Combined payload bytes for all coded segments in this code block. - pub data: &'a [u8], - /// Coded segments for the code block. - pub segments: &'a [J2kCodeBlockSegment], - /// Code-block width in samples. - pub width: u32, - /// Code-block height in samples. - pub height: u32, - /// Output row stride, in samples, for the target sub-band storage. - pub output_stride: usize, - /// Missing most-significant bit planes for this code block. - pub missing_bit_planes: u8, - /// Number of coding passes present for this code block. - pub number_of_coding_passes: u8, - /// Total coded bitplanes for the parent sub-band. - pub total_bitplanes: u8, - /// The sub-band type containing this code block. - pub sub_band_type: J2kSubBandType, - /// The code-block style flags. - pub style: J2kCodeBlockStyle, - /// Whether strict decode validation is enabled for the parent image. - pub strict: bool, - /// Dequantization step to apply to decoded coefficients. - pub dequantization_step: f32, -} - -/// Hidden encoded classic J2K code-block payload for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone)] -pub struct EncodedJ2kCodeBlock { - /// Combined payload bytes for all coded segments in this code block. - pub data: Vec, - /// Coded segments for the code block. - pub segments: Vec, - /// Number of coding passes present for this code block. - pub number_of_coding_passes: u8, - /// Missing most-significant bit planes for this code block. - pub missing_bit_planes: u8, -} - -/// Hidden encoded HTJ2K cleanup code-block payload for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone)] -pub struct EncodedHtJ2kCodeBlock { - /// Combined cleanup/refinement bytes for this code block. - pub data: Vec, - /// Number of coding passes present for this code block. - pub num_coding_passes: u8, - /// Number of zero most-significant bitplanes before first inclusion. - pub num_zero_bitplanes: u8, -} - -/// Hidden forward RCT job for backend experimentation. -#[doc(hidden)] -#[derive(Debug)] -pub struct J2kForwardRctJob<'a> { - /// First component plane, updated in place. - pub plane0: &'a mut [f32], - /// Second component plane, updated in place. - pub plane1: &'a mut [f32], - /// Third component plane, updated in place. - pub plane2: &'a mut [f32], -} - -/// Hidden forward 5/3 DWT job for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct J2kForwardDwt53Job<'a> { - /// Source samples in row-major order. - pub samples: &'a [f32], - /// Source width in samples. - pub width: u32, - /// Source height in samples. - pub height: u32, - /// Number of decomposition levels requested. - pub num_levels: u8, -} - -/// Hidden forward 5/3 DWT output for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone)] -pub struct J2kForwardDwt53Output { - /// LL subband coefficients from the lowest decomposition level. - pub ll: Vec, - /// LL subband width. - pub ll_width: u32, - /// LL subband height. - pub ll_height: u32, - /// Higher resolution detail levels, ordered from lowest to highest. - pub levels: Vec, -} - -/// Hidden forward 5/3 DWT detail level for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone)] -pub struct J2kForwardDwt53Level { - /// HL subband coefficients. - pub hl: Vec, - /// LH subband coefficients. - pub lh: Vec, - /// HH subband coefficients. - pub hh: Vec, - /// Full-resolution width represented by this level. - pub width: u32, - /// Full-resolution height represented by this level. - pub height: u32, - /// Low-pass width at this level. - pub low_width: u32, - /// Low-pass height at this level. - pub low_height: u32, - /// High-pass width at this level. - pub high_width: u32, - /// High-pass height at this level. - pub high_height: u32, -} - -/// Hidden Tier-1 classic J2K code-block encode job for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct J2kTier1CodeBlockEncodeJob<'a> { - /// Quantized coefficients in row-major order. - pub coefficients: &'a [i32], - /// Code-block width in samples. - pub width: u32, - /// Code-block height in samples. - pub height: u32, - /// Subband kind containing this code-block. - pub sub_band_type: J2kSubBandType, - /// Total bitplanes for this subband/code-block. - pub total_bitplanes: u8, - /// Classic J2K code-block style flags. - pub style: J2kCodeBlockStyle, -} - -/// Hidden HTJ2K cleanup-only code-block encode job for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct J2kHtCodeBlockEncodeJob<'a> { - /// Quantized coefficients in row-major order. - pub coefficients: &'a [i32], - /// Code-block width in samples. - pub width: u32, - /// Code-block height in samples. - pub height: u32, - /// Total bitplanes for this subband/code-block. - pub total_bitplanes: u8, -} - -/// Hidden LRCP packetization code-block contribution for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct J2kPacketizationCodeBlock<'a> { - /// Encoded Tier-1 bitstream bytes for this packet contribution. - pub data: &'a [u8], - /// Number of coding passes in this contribution. - pub num_coding_passes: u8, - /// Number of zero most-significant bitplanes before first inclusion. - pub num_zero_bitplanes: u8, - /// Whether this code-block was included in a previous packet. - pub previously_included: bool, - /// L-block value used for segment length coding. - pub l_block: u32, - /// Block coder used for this contribution. - pub block_coding_mode: J2kPacketizationBlockCodingMode, -} - -/// Hidden packetization block coding mode for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum J2kPacketizationBlockCodingMode { - /// Classic JPEG 2000 Part 1 EBCOT block coding. - Classic, - /// High-throughput JPEG 2000 Part 15 block coding. - HighThroughput, -} - -/// Hidden packet progression order for backend packetization experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum J2kPacketizationProgressionOrder { - /// Layer-resolution-component-position progression. - Lrcp, - /// Resolution-position-component-layer progression. - Rpcl, -} - -/// Hidden LRCP packetization subband precinct for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct J2kPacketizationSubband<'a> { - /// Code-block contributions in row-major order. - pub code_blocks: Vec>, - /// Number of code-blocks in the x direction. - pub num_cbs_x: u32, - /// Number of code-blocks in the y direction. - pub num_cbs_y: u32, -} - -/// Hidden LRCP packetization resolution packet for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct J2kPacketizationResolution<'a> { - /// Subbands in packet order: LL for resolution 0, then HL/LH/HH. - pub subbands: Vec>, -} - -/// Hidden explicit packet descriptor for backend packetization experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct J2kPacketizationPacketDescriptor { - /// Index into the packet contribution array. - pub packet_index: u32, - /// Persistent packet-state index for repeated layer/precinct packets. - pub state_index: u32, - /// Quality layer for inclusion tag-tree thresholds. - pub layer: u8, - /// Resolution index in the output progression. - pub resolution: u32, - /// Component index in the output progression. - pub component: u8, - /// Precinct index in the output progression. - pub precinct: u64, -} - -/// Hidden LRCP packetization job for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct J2kPacketizationEncodeJob<'a> { - /// Number of resolution packets prepared for packetization. - pub resolution_count: u32, - /// Number of layers to write. - pub num_layers: u8, - /// Number of image components. - pub num_components: u8, - /// Total number of code-block contributions. - pub code_block_count: u32, - /// Packet progression order to emit. - pub progression_order: J2kPacketizationProgressionOrder, - /// Explicit packet descriptors in output progression order. - pub packet_descriptors: &'a [J2kPacketizationPacketDescriptor], - /// Packet payload prepared by Tier-1, in LRCP packet order. - pub resolutions: &'a [J2kPacketizationResolution<'a>], -} - -/// Hidden encode-stage dispatch counters for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub struct J2kEncodeDispatchReport { - /// Forward RCT kernel dispatch count. - pub forward_rct: usize, - /// Forward reversible 5/3 DWT kernel dispatch count. - pub forward_dwt53: usize, - /// Tier-1 code-block encode dispatch count. - pub tier1_code_block: usize, - /// HTJ2K cleanup-only code-block encode dispatch count. - pub ht_code_block: usize, - /// Packetization dispatch count. - pub packetization: usize, -} - -impl J2kEncodeDispatchReport { - /// Return the saturating per-stage delta from `before` to `self`. - #[must_use] - pub fn saturating_delta(self, before: Self) -> Self { - Self { - forward_rct: self.forward_rct.saturating_sub(before.forward_rct), - forward_dwt53: self.forward_dwt53.saturating_sub(before.forward_dwt53), - tier1_code_block: self - .tier1_code_block - .saturating_sub(before.tier1_code_block), - ht_code_block: self.ht_code_block.saturating_sub(before.ht_code_block), - packetization: self.packetization.saturating_sub(before.packetization), - } - } - - /// Return total dispatches across all encode stages. - #[must_use] - pub fn total(self) -> usize { - self.forward_rct - .saturating_add(self.forward_dwt53) - .saturating_add(self.tier1_code_block) - .saturating_add(self.ht_code_block) - .saturating_add(self.packetization) - } - - /// Return whether at least one encode stage dispatched. - #[must_use] - pub fn any(self) -> bool { - self.total() > 0 - } -} - -/// Hidden JPEG 2000 encode-stage accelerator for backend experimentation. -#[doc(hidden)] -pub trait J2kEncodeStageAccelerator { - /// Report cumulative backend dispatches completed by this accelerator. - fn dispatch_report(&self) -> J2kEncodeDispatchReport { - J2kEncodeDispatchReport::default() - } - - /// Optionally apply forward RCT in place. - /// - /// Return `Ok(true)` after writing transformed planes. Return `Ok(false)` - /// to use the CPU fallback. - fn encode_forward_rct( - &mut self, - _job: J2kForwardRctJob<'_>, - ) -> core::result::Result { - Ok(false) - } - - /// Optionally run a forward reversible 5/3 DWT. - /// - /// Return `Ok(Some(output))` with all subbands populated. Return - /// `Ok(None)` to use the CPU fallback. - fn encode_forward_dwt53( - &mut self, - _job: J2kForwardDwt53Job<'_>, - ) -> core::result::Result, &'static str> { - Ok(None) - } - - /// Optionally encode one classic Tier-1 code-block. - /// - /// Return `Ok(Some(output))` with encoded bytes and pass metadata. Return - /// `Ok(None)` to use the CPU fallback. - fn encode_tier1_code_block( - &mut self, - _job: J2kTier1CodeBlockEncodeJob<'_>, - ) -> core::result::Result, &'static str> { - Ok(None) - } - - /// Optionally encode multiple classic Tier-1 code-blocks in one backend dispatch. - /// - /// Return `Ok(Some(outputs))` with one encoded output per input job. Return - /// `Ok(None)` to use the per-block hook or CPU fallback. - fn encode_tier1_code_blocks( - &mut self, - _jobs: &[J2kTier1CodeBlockEncodeJob<'_>], - ) -> core::result::Result>, &'static str> { - Ok(None) - } - - /// Optionally encode one HTJ2K cleanup-only code-block. - /// - /// Return `Ok(Some(output))` with encoded bytes and pass metadata. Return - /// `Ok(None)` to use the CPU fallback. - fn encode_ht_code_block( - &mut self, - _job: J2kHtCodeBlockEncodeJob<'_>, - ) -> core::result::Result, &'static str> { - Ok(None) - } - - /// Optionally encode multiple HTJ2K cleanup-only code-blocks in one backend dispatch. - /// - /// Return `Ok(Some(outputs))` with one encoded output per input job. Return - /// `Ok(None)` to use the per-block hook or CPU fallback. - fn encode_ht_code_blocks( - &mut self, - _jobs: &[J2kHtCodeBlockEncodeJob<'_>], - ) -> core::result::Result>, &'static str> { - Ok(None) - } - - /// Return whether native CPU code-block fallback should use internal rayon parallelism. - /// - /// External accelerators keep serial per-block fallback so their hooks still - /// observe every fallback block after a declined batch hook. - #[doc(hidden)] - fn prefer_parallel_cpu_code_block_fallback(&self) -> bool { - false - } - - /// Optionally packetize prepared packet contributions. - /// - /// Return `Ok(Some(bytes))` with the complete tile bitstream. Return - /// `Ok(None)` to use the CPU fallback. - fn encode_packetization( - &mut self, - _job: J2kPacketizationEncodeJob<'_>, - ) -> core::result::Result>, &'static str> { - Ok(None) - } -} - -/// Hidden CPU-only encode accelerator that always falls back to native stages. -#[doc(hidden)] -#[derive(Debug, Default, Clone, Copy)] -pub struct CpuOnlyJ2kEncodeStageAccelerator; - -impl J2kEncodeStageAccelerator for CpuOnlyJ2kEncodeStageAccelerator { - fn prefer_parallel_cpu_code_block_fallback(&self) -> bool { - true - } -} - -/// Hidden classic J2K batched code-block decode job for one sub-band. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct J2kCodeBlockBatchJob<'a> { - /// X offset within the target sub-band coefficient buffer. - pub output_x: u32, - /// Y offset within the target sub-band coefficient buffer. - pub output_y: u32, - /// The actual code-block decode parameters. - pub code_block: J2kCodeBlockDecodeJob<'a>, -} - -/// Hidden classic J2K batched sub-band decode request for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct J2kSubBandDecodeJob<'a> { - /// Sub-band width in samples. - pub width: u32, - /// Sub-band height in samples. - pub height: u32, - /// Code blocks to decode into this sub-band. - pub jobs: &'a [J2kCodeBlockBatchJob<'a>], -} - -/// Hidden integer rectangle for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct J2kRect { - /// Inclusive minimum x coordinate. - pub x0: u32, - /// Inclusive minimum y coordinate. - pub y0: u32, - /// Exclusive maximum x coordinate. - pub x1: u32, - /// Exclusive maximum y coordinate. - pub y1: u32, -} - -impl J2kRect { - /// Rectangle width in samples. - pub fn width(self) -> u32 { - self.x1.saturating_sub(self.x0) - } - - /// Rectangle height in samples. - pub fn height(self) -> u32 { - self.y1.saturating_sub(self.y0) - } -} - -/// Hidden wavelet transform selector for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum J2kWaveletTransform { - /// Reversible 5/3 transform. - Reversible53, - /// Irreversible 9/7 transform. - Irreversible97, -} - -/// Hidden single sub-band payload for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct J2kIdwtBand<'a> { - /// Rect covered by this band. - pub rect: J2kRect, - /// Band coefficients in row-major order. - pub coefficients: &'a [f32], -} - -/// Hidden single-decomposition IDWT job for backend experimentation. -#[doc(hidden)] -#[derive(Debug, Clone, Copy)] -pub struct J2kSingleDecompositionIdwtJob<'a> { - /// Output rect of the decomposition level. - pub rect: J2kRect, - /// Transform to apply. - pub transform: J2kWaveletTransform, - /// LL band input. - pub ll: J2kIdwtBand<'a>, - /// HL band input. - pub hl: J2kIdwtBand<'a>, - /// LH band input. - pub lh: J2kIdwtBand<'a>, - /// HH band input. - pub hh: J2kIdwtBand<'a>, -} - -/// Hidden inverse MCT job for backend experimentation. -#[doc(hidden)] -#[derive(Debug)] -pub struct J2kInverseMctJob<'a> { - /// Transform to apply. - pub transform: J2kWaveletTransform, - /// First component plane, updated in place. - pub plane0: &'a mut [f32], - /// Second component plane, updated in place. - pub plane1: &'a mut [f32], - /// Third component plane, updated in place. - pub plane2: &'a mut [f32], - /// Constant sign-shift addend applied to the first plane after inverse MCT. - pub addend0: f32, - /// Constant sign-shift addend applied to the second plane after inverse MCT. - pub addend1: f32, - /// Constant sign-shift addend applied to the third plane after inverse MCT. - pub addend2: f32, -} - -/// Hidden component-store job for backend experimentation. -#[doc(hidden)] -#[derive(Debug)] -pub struct J2kStoreComponentJob<'a> { - /// Source IDWT coefficients in row-major order. - pub input: &'a [f32], - /// Source row width. - pub input_width: u32, - /// Source x offset to begin copying from. - pub source_x: u32, - /// Source y offset to begin copying from. - pub source_y: u32, - /// Number of samples to copy per row. - pub copy_width: u32, - /// Number of rows to copy. - pub copy_height: u32, - /// Destination component plane in row-major order. - pub output: &'a mut [f32], - /// Destination row width. - pub output_width: u32, - /// Destination x offset to begin writing at. - pub output_x: u32, - /// Destination y offset to begin writing at. - pub output_y: u32, - /// Constant value added to every copied sample. - pub addend: f32, -} - -/// Hidden HTJ2K code-block decode hook for backend experimentation. -#[doc(hidden)] -pub trait HtCodeBlockDecoder { - /// Optionally decode a full classic J2K sub-band in one batch. - /// - /// Implementations should return `Ok(true)` if they handled the request and - /// wrote the decoded coefficients into `output`. Returning `Ok(false)` - /// falls back to per-code-block decode via `decode_j2k_code_block`. - fn decode_j2k_sub_band( - &mut self, - _job: J2kSubBandDecodeJob<'_>, - _output: &mut [f32], - ) -> Result { - Ok(false) - } - - /// Optionally decode one classic J2K code block. - /// - /// Implementations should return `Ok(true)` if they handled the request - /// and wrote the decoded coefficients into `output`. Returning `Ok(false)` - /// falls back to the scalar bitplane decoder. - fn decode_j2k_code_block( - &mut self, - _job: J2kCodeBlockDecodeJob<'_>, - _output: &mut [f32], - ) -> Result { - Ok(false) - } - - /// Optionally decode a full HTJ2K sub-band in one batch. - /// - /// Implementations should return `Ok(true)` if they handled the request and - /// wrote the decoded coefficients into `output`. Returning `Ok(false)` - /// falls back to per-code-block decode via `decode_code_block`. - fn decode_sub_band( - &mut self, - _job: HtSubBandDecodeJob<'_>, - _output: &mut [f32], - ) -> Result { - Ok(false) - } - - /// Optionally decode one single-decomposition IDWT level on a backend. - /// - /// Implementations should return `Ok(true)` if they handled the request - /// and wrote the transformed coefficients into `output`. Returning - /// `Ok(false)` falls back to the scalar/SIMD CPU IDWT path. - fn decode_single_decomposition_idwt( - &mut self, - _job: J2kSingleDecompositionIdwtJob<'_>, - _output: &mut [f32], - ) -> Result { - Ok(false) - } - - /// Optionally apply inverse MCT on a backend. - /// - /// Implementations should return `Ok(true)` if they handled the request - /// and updated the component planes in place. Returning `Ok(false)` falls - /// back to the scalar/SIMD CPU MCT path. - fn decode_inverse_mct(&mut self, _job: J2kInverseMctJob<'_>) -> Result { - Ok(false) - } - - /// Optionally store one component plane on a backend. - /// - /// Implementations should return `Ok(true)` if they handled the request - /// and updated the destination plane in place. Returning `Ok(false)` falls - /// back to the CPU store path. - fn decode_store_component(&mut self, _job: J2kStoreComponentJob<'_>) -> Result { - Ok(false) - } - - /// Decode one HTJ2K code block into `output`, writing `job.width` samples per row. - fn decode_code_block( - &mut self, - job: HtCodeBlockDecodeJob<'_>, - output: &mut [f32], - ) -> Result<()>; -} - -fn internal_j2k_sub_band_type(sub_band_type: J2kSubBandType) -> j2c::build::SubBandType { - match sub_band_type { - J2kSubBandType::LowLow => j2c::build::SubBandType::LowLow, - J2kSubBandType::HighLow => j2c::build::SubBandType::HighLow, - J2kSubBandType::LowHigh => j2c::build::SubBandType::LowHigh, - J2kSubBandType::HighHigh => j2c::build::SubBandType::HighHigh, - } -} - -fn internal_j2k_code_block_style(style: J2kCodeBlockStyle) -> j2c::codestream::CodeBlockStyle { - j2c::codestream::CodeBlockStyle { - selective_arithmetic_coding_bypass: style.selective_arithmetic_coding_bypass, - reset_context_probabilities: style.reset_context_probabilities, - termination_on_each_pass: style.termination_on_each_pass, - vertically_causal_context: style.vertically_causal_context, - segmentation_symbols: style.segmentation_symbols, - high_throughput_block_coding: false, - } -} - -/// Hidden scalar classic J2K encoder helper for backend experimentation. -#[doc(hidden)] -pub fn encode_j2k_code_block_scalar_with_style( - coefficients: &[i32], - width: u32, - height: u32, - sub_band_type: J2kSubBandType, - total_bitplanes: u8, - style: J2kCodeBlockStyle, -) -> core::result::Result { - let encoded = j2c::bitplane_encode::encode_code_block_segments_with_style( - coefficients, - width, - height, - internal_j2k_sub_band_type(sub_band_type), - total_bitplanes, - &internal_j2k_code_block_style(style), - ); - let segments = encoded - .segments - .into_iter() - .map(|segment| J2kCodeBlockSegment { - data_offset: segment.data_offset, - data_length: segment.data_length, - start_coding_pass: segment.start_coding_pass, - end_coding_pass: segment.end_coding_pass, - use_arithmetic: segment.use_arithmetic, - }) - .collect(); - - Ok(EncodedJ2kCodeBlock { - data: encoded.data, - segments, - number_of_coding_passes: encoded.num_coding_passes, - missing_bit_planes: encoded.num_zero_bitplanes, - }) -} - -/// Hidden scalar HTJ2K cleanup-only encoder helper for backend experimentation. -#[doc(hidden)] -pub fn encode_ht_code_block_scalar( - coefficients: &[i32], - width: u32, - height: u32, - total_bitplanes: u8, -) -> core::result::Result { - let encoded = - j2c::ht_block_encode::encode_code_block(coefficients, width, height, total_bitplanes)?; - Ok(EncodedHtJ2kCodeBlock { - data: encoded.data, - num_coding_passes: encoded.num_coding_passes, - num_zero_bitplanes: encoded.num_zero_bitplanes, - }) -} - -/// Hidden scalar Tier-2 packetization helper for backend experimentation. -#[doc(hidden)] -pub fn encode_j2k_packetization_scalar( - job: J2kPacketizationEncodeJob<'_>, -) -> core::result::Result, &'static str> { - let mut resolutions = job - .resolutions - .iter() - .map(|resolution| j2c::packet_encode::ResolutionPacket { - subbands: resolution - .subbands - .iter() - .map(|subband| j2c::packet_encode::SubbandPrecinct { - code_blocks: subband - .code_blocks - .iter() - .map(|code_block| j2c::packet_encode::CodeBlockPacketData { - data: code_block.data.to_vec(), - num_coding_passes: code_block.num_coding_passes, - num_zero_bitplanes: code_block.num_zero_bitplanes, - previously_included: code_block.previously_included, - l_block: code_block.l_block, - block_coding_mode: match code_block.block_coding_mode { - J2kPacketizationBlockCodingMode::Classic => { - j2c::codestream_write::BlockCodingMode::Classic - } - J2kPacketizationBlockCodingMode::HighThroughput => { - j2c::codestream_write::BlockCodingMode::HighThroughput - } - }, - }) - .collect(), - num_cbs_x: subband.num_cbs_x, - num_cbs_y: subband.num_cbs_y, - }) - .collect(), - }) - .collect::>(); - - let descriptors = job - .packet_descriptors - .iter() - .map(|descriptor| j2c::packet_encode::PacketDescriptor { - packet_index: descriptor.packet_index, - state_index: descriptor.state_index, - layer: descriptor.layer, - resolution: descriptor.resolution, - component: descriptor.component, - precinct: descriptor.precinct, - }) - .collect::>(); - - if descriptors.is_empty() { - Ok(j2c::packet_encode::form_tile_bitstream_for_progression( - &mut resolutions, - job.num_layers, - job.num_components, - job.progression_order, - )) - } else { - j2c::packet_encode::form_tile_bitstream_with_descriptors(&mut resolutions, &descriptors) - } -} - -/// Hidden scalar classic J2K decoder helper for backend experimentation. -#[doc(hidden)] -pub fn decode_j2k_code_block_scalar( - job: J2kCodeBlockDecodeJob<'_>, - output: &mut [f32], -) -> Result<()> { - let required_len = if job.height == 0 { - 0 - } else { - job.output_stride - .checked_mul(job.height as usize - 1) - .and_then(|prefix| prefix.checked_add(job.width as usize)) - .ok_or(DecodingError::CodeBlockDecodeFailure)? - }; - if output.len() < required_len { - bail!(DecodingError::CodeBlockDecodeFailure); - } - - let style = internal_j2k_code_block_style(job.style); - let sub_band_type = internal_j2k_sub_band_type(job.sub_band_type); - let code_block_stride = - usize::try_from(job.width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; - let mut bit_plane_decode_context = j2c::bitplane::BitPlaneDecodeContext::default(); - - j2c::bitplane::decode_code_block_segments_validated( - job.data, - job.segments, - job.width, - job.height, - job.missing_bit_planes, - job.number_of_coding_passes, - job.total_bitplanes, - sub_band_type, - &style, - job.strict, - &mut bit_plane_decode_context, - )?; - - for (row_idx, coeff_row) in bit_plane_decode_context - .coefficient_rows() - .enumerate() - .take(job.height as usize) - { - let row_start = row_idx * job.output_stride; - let output_row = &mut output[row_start..row_start + code_block_stride]; - for (coefficient, sample) in coeff_row.iter().zip(output_row.iter_mut()) { - *sample = coefficient.get() as f32 * job.dequantization_step; - } - } - - Ok(()) -} - -/// Hidden scalar classic J2K batched decoder helper for backend experimentation. -#[doc(hidden)] -pub fn decode_j2k_sub_band_scalar(job: J2kSubBandDecodeJob<'_>, output: &mut [f32]) -> Result<()> { - let required_len = if job.height == 0 { - 0 - } else { - usize::try_from(job.width) - .ok() - .and_then(|width| width.checked_mul(job.height as usize)) - .ok_or(DecodingError::CodeBlockDecodeFailure)? - }; - if output.len() < required_len { - bail!(DecodingError::CodeBlockDecodeFailure); - } - - let sub_band_width = - usize::try_from(job.width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; - - for batch_job in job.jobs { - let code_block = batch_job.code_block; - if code_block.output_stride != sub_band_width { - bail!(DecodingError::CodeBlockDecodeFailure); - } - if batch_job - .output_x - .checked_add(code_block.width) - .is_none_or(|x1| x1 > job.width) - || batch_job - .output_y - .checked_add(code_block.height) - .is_none_or(|y1| y1 > job.height) - { - bail!(DecodingError::CodeBlockDecodeFailure); - } - - let base_idx = usize::try_from(batch_job.output_y) - .ok() - .and_then(|y| y.checked_mul(sub_band_width)) - .and_then(|row| row.checked_add(batch_job.output_x as usize)) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - let block_output_len = if code_block.height == 0 { - 0 - } else { - code_block - .output_stride - .checked_mul(code_block.height as usize - 1) - .and_then(|prefix| prefix.checked_add(code_block.width as usize)) - .ok_or(DecodingError::CodeBlockDecodeFailure)? - }; - let end_idx = base_idx - .checked_add(block_output_len) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - if end_idx > output.len() { - bail!(DecodingError::CodeBlockDecodeFailure); - } - - decode_j2k_code_block_scalar(code_block, &mut output[base_idx..end_idx])?; - } - - Ok(()) -} - -/// Hidden scalar HTJ2K decoder helper for backend experimentation. -#[doc(hidden)] -pub fn decode_ht_code_block_scalar( - job: HtCodeBlockDecodeJob<'_>, - output: &mut [f32], -) -> Result<()> { - let required_len = if job.height == 0 { - 0 - } else { - job.output_stride - .checked_mul(job.height as usize - 1) - .and_then(|prefix| prefix.checked_add(job.width as usize)) - .ok_or(DecodingError::CodeBlockDecodeFailure)? - }; - if output.len() < required_len { - bail!(DecodingError::CodeBlockDecodeFailure); - } - let code_block_stride = - usize::try_from(job.width).map_err(|_| DecodingError::CodeBlockDecodeFailure)?; - let code_block_len = code_block_stride - .checked_mul(job.height as usize) - .ok_or(DecodingError::CodeBlockDecodeFailure)?; - - let combined = j2c::ht_block_decode::CombinedCodeBlockData { - data: job.data.to_vec(), - cleanup_length: job.cleanup_length, - refinement_length: job.refinement_length, - }; - let mut coefficients = vec![0u32; code_block_len]; - j2c::ht_block_decode::decode_combined_validated( - &combined, - job.missing_bit_planes, - job.num_bitplanes, - job.number_of_coding_passes, - job.stripe_causal, - job.strict, - &mut coefficients, - job.width, - job.height, - job.width, - )?; - - for (row_idx, coeff_row) in coefficients - .chunks_exact(code_block_stride) - .enumerate() - .take(job.height as usize) - { - let row_start = row_idx * job.output_stride; - let output_row = &mut output[row_start..row_start + code_block_stride]; - for (coefficient, sample) in coeff_row.iter().copied().zip(output_row.iter_mut()) { - *sample = j2c::ht_block_decode::coefficient_to_i32(coefficient, job.num_bitplanes) - as f32 - * job.dequantization_step; - } - } - - Ok(()) -} - -/// Hidden HTJ2K VLC table 0 for backend experimentation. -#[doc(hidden)] -pub fn ht_vlc_table0() -> &'static [u16; 1024] { - &j2c::ht_tables::VLC_TABLE0 -} - -/// Hidden HTJ2K VLC table 1 for backend experimentation. -#[doc(hidden)] -pub fn ht_vlc_table1() -> &'static [u16; 1024] { - &j2c::ht_tables::VLC_TABLE1 -} - -/// Hidden HTJ2K UVLC table 0 for backend experimentation. -#[doc(hidden)] -pub fn ht_uvlc_table0() -> &'static [u16; 320] { - &j2c::ht_tables::UVLC_TABLE0 -} - -/// Hidden HTJ2K UVLC table 1 for backend experimentation. -#[doc(hidden)] -pub fn ht_uvlc_table1() -> &'static [u16; 256] { - &j2c::ht_tables::UVLC_TABLE1 -} - -/// Hidden HTJ2K cleanup encoder VLC table 0 for backend experimentation. -#[doc(hidden)] -pub fn ht_vlc_encode_table0() -> &'static [u16; 2048] { - &j2c::ht_encode_tables::HT_VLC_ENCODE_TABLE0 -} - -/// Hidden HTJ2K cleanup encoder VLC table 1 for backend experimentation. -#[doc(hidden)] -pub fn ht_vlc_encode_table1() -> &'static [u16; 2048] { - &j2c::ht_encode_tables::HT_VLC_ENCODE_TABLE1 -} - -/// Hidden HTJ2K cleanup encoder UVLC table for backend experimentation. -#[doc(hidden)] -pub fn ht_uvlc_encode_table() -> &'static [HtUvlcTableEntry; 75] { - &j2c::ht_encode_tables::HT_UVLC_ENCODE_TABLE -} - -/// JP2 signature box: 00 00 00 0C 6A 50 20 20 -pub(crate) const JP2_MAGIC: &[u8] = b"\x00\x00\x00\x0C\x6A\x50\x20\x20"; -/// Codestream signature: FF 4F FF 51 (SOC + SIZ markers) -pub(crate) const CODESTREAM_MAGIC: &[u8] = b"\xFF\x4F\xFF\x51"; - -/// Settings to apply during decoding. -#[derive(Debug, Copy, Clone)] -pub struct DecodeSettings { - /// Whether palette indices should be resolved. - /// - /// JPEG2000 images can be stored in two different ways. First, by storing - /// RGB values (depending on the color space) for each pixel. Secondly, by - /// only storing a single index for each channel, and then resolving the - /// actual color using the index. - /// - /// If you disable this option, in case you have an image with palette - /// indices, they will not be resolved, but instead a grayscale image - /// will be returned, with each pixel value corresponding to the palette - /// index of the location. - pub resolve_palette_indices: bool, - /// Whether strict mode should be enabled when decoding. - /// - /// It is recommended to leave this flag disabled, unless you have a - /// specific reason not to. - pub strict: bool, - /// A hint for the target resolution that the image should be decoded at. - pub target_resolution: Option<(u32, u32)>, -} - -impl Default for DecodeSettings { - fn default() -> Self { - Self { - resolve_palette_indices: true, - strict: false, - target_resolution: None, - } - } -} - -/// A JPEG2000 image or codestream. -pub struct Image<'a> { - /// The tile-part payload used by the legacy JPEG 2000 decoder. - pub(crate) codestream: &'a [u8], - /// The header of the J2C codestream. - pub(crate) header: Header<'a>, - /// The JP2 boxes of the image. In the case of a raw codestream, we - /// will synthesize the necessary boxes. - pub(crate) boxes: ImageBoxes, - /// Settings that should be applied during decoding. - pub(crate) settings: DecodeSettings, - /// Whether the image has an alpha channel. - pub(crate) has_alpha: bool, - /// The color space of the image. - pub(crate) color_space: ColorSpace, -} - -impl<'a> Image<'a> { - /// Try to create a new JPEG2000 image from the given data. - pub fn new(data: &'a [u8], settings: &DecodeSettings) -> Result { - if data.starts_with(JP2_MAGIC) { - jp2::parse(data, *settings) - } else if data.starts_with(CODESTREAM_MAGIC) { - j2c::parse(data, settings) - } else { - err!(FormatError::InvalidSignature) - } - } - - /// Whether the image has an alpha channel. - pub fn has_alpha(&self) -> bool { - self.has_alpha - } - - /// The color space of the image. - pub fn color_space(&self) -> &ColorSpace { - &self.color_space - } - - /// The width of the image. - pub fn width(&self) -> u32 { - self.header.size_data.image_width() - } - - /// The height of the image. - pub fn height(&self) -> u32 { - self.header.size_data.image_height() - } - - /// The original bit depth of the image. You usually don't need to do anything - /// with this parameter, it just exists for informational purposes. - pub fn original_bit_depth(&self) -> u8 { - // Note that this only works if all components have the same precision. - self.header.component_infos[0].size_info.precision - } - - /// Whether decode finishes with additional host-side component mutation or reordering. - #[doc(hidden)] - pub fn supports_direct_device_plane_reuse(&self) -> bool { - if self.settings.resolve_palette_indices && self.boxes.palette.is_some() { - return false; - } - if self.boxes.channel_definition.is_some() { - return false; - } - !matches!( - self.boxes - .color_specification - .as_ref() - .map(|spec| &spec.color_space), - Some(jp2::colr::ColorSpace::Enumerated( - EnumeratedColorspace::Sycc | EnumeratedColorspace::CieLab(_) - )) - ) - } - - /// Decode the image and return its decoded result as a `Vec`, with each - /// channel interleaved. - pub fn decode(&self) -> Result> { - let bitmap = self.decode_with_context(&mut DecoderContext::default())?; - Ok(bitmap.data) - } - - /// Decode the image and return its decoded result using a caller-provided - /// decoder context so allocations can be reused across repeated decodes. - pub fn decode_with_context(&self, decoder_context: &mut DecoderContext<'a>) -> Result { - let buffer_size = self.width() as usize - * self.height() as usize - * (self.color_space.num_channels() as usize + if self.has_alpha { 1 } else { 0 }); - let mut buf = vec![0; buffer_size]; - self.decode_into(&mut buf, decoder_context)?; - - Ok(Bitmap { - color_space: self.color_space.clone(), - data: buf, - has_alpha: self.has_alpha, - width: self.width(), - height: self.height(), - original_bit_depth: self.original_bit_depth(), - }) - } - - /// Decode the image into borrowed component planes using a caller-provided - /// decoder context so allocations can be reused across repeated decodes. - pub fn decode_components_with_context<'ctx>( - &self, - decoder_context: &'ctx mut DecoderContext<'a>, - ) -> Result> { - let decoded_image = self.prepare_decoded_image(decoder_context)?; - let planes = decoded_image - .decoded_components - .iter() - .map(|component| ComponentPlane { - samples: component.container.truncated(), - bit_depth: component.bit_depth, - }) - .collect(); - - Ok(DecodedComponents { - dimensions: (self.width(), self.height()), - color_space: self.color_space.clone(), - has_alpha: self.has_alpha, - planes, - }) - } - - /// Build a hidden grayscale direct device plan without materializing host component planes. - #[doc(hidden)] - pub fn build_direct_grayscale_plan_with_context( - &self, - decoder_context: &mut DecoderContext<'a>, - ) -> Result { - if !matches!(self.color_space, ColorSpace::Gray) || self.has_alpha { - bail!(DecodingError::UnsupportedFeature( - "direct grayscale plan only supports grayscale images without alpha" - )); - } - - j2c::build_direct_grayscale_plan(self.codestream, &self.header, decoder_context) - } - - /// Build a hidden RGB direct device plan without materializing host component planes. - #[doc(hidden)] - pub fn build_direct_color_plan_with_context( - &self, - decoder_context: &mut DecoderContext<'a>, - ) -> Result { - if !matches!(self.color_space, ColorSpace::RGB) || self.has_alpha { - bail!(DecodingError::UnsupportedFeature( - "direct color plan only supports RGB images without alpha" - )); - } - - j2c::build_direct_color_plan(self.codestream, &self.header, decoder_context) - } - - /// Decode borrowed component planes while delegating HTJ2K code-block decode. - #[doc(hidden)] - pub fn decode_components_with_ht_decoder<'ctx>( - &self, - decoder_context: &'ctx mut DecoderContext<'a>, - ht_decoder: &mut dyn HtCodeBlockDecoder, - ) -> Result> { - let decoded_image = - self.prepare_decoded_image_with_ht_decoder(decoder_context, ht_decoder)?; - let planes = decoded_image - .decoded_components - .iter() - .map(|component| ComponentPlane { - samples: component.container.truncated(), - bit_depth: component.bit_depth, - }) - .collect(); - - Ok(DecodedComponents { - dimensions: (self.width(), self.height()), - color_space: self.color_space.clone(), - has_alpha: self.has_alpha, - planes, - }) - } - - /// Decode borrowed component planes for a requested region while - /// delegating code-block/transform stages through the hidden backend hook. - #[doc(hidden)] - pub fn decode_region_components_with_ht_decoder<'ctx>( - &self, - decoder_context: &'ctx mut DecoderContext<'a>, - roi: (u32, u32, u32, u32), - ht_decoder: &mut dyn HtCodeBlockDecoder, - ) -> Result> { - validate_roi((self.width(), self.height()), roi)?; - let (_x, _y, width, height) = roi; - let decoded_image = self.prepare_decoded_image_with_region_and_ht_decoder( - decoder_context, - Some(roi), - Some(ht_decoder), - )?; - let planes = decoded_image - .decoded_components - .iter() - .map(|component| ComponentPlane { - samples: component.container.truncated(), - bit_depth: component.bit_depth, - }) - .collect(); - - Ok(DecodedComponents { - dimensions: (width, height), - color_space: self.color_space.clone(), - has_alpha: self.has_alpha, - planes, - }) - } - - /// Decode a region of the image and return it as an 8-bit interleaved bitmap. - pub fn decode_region(&self, roi: (u32, u32, u32, u32)) -> Result { - self.decode_region_with_context(roi, &mut DecoderContext::default()) - } - - /// Decode a region of the image and return it as an 8-bit interleaved bitmap - /// using a caller-provided decoder context. - pub fn decode_region_with_context( - &self, - roi: (u32, u32, u32, u32), - decoder_context: &mut DecoderContext<'a>, - ) -> Result { - validate_roi((self.width(), self.height()), roi)?; - let mut decoded_image = - self.prepare_decoded_image_with_region(decoder_context, Some(roi))?; - let (_x, _y, width, height) = roi; - let channels = - self.color_space.num_channels() as usize + if self.has_alpha { 1 } else { 0 }; - let mut data = vec![0; width as usize * height as usize * channels]; - interleave_and_convert_region( - &mut decoded_image, - width as usize, - (0, 0, width, height), - &mut data, - ); - Ok(Bitmap { - color_space: self.color_space.clone(), - data, - has_alpha: self.has_alpha, - width, - height, - original_bit_depth: self.original_bit_depth(), - }) - } - - /// Decode the image at native bit depth without scaling to 8-bit. - /// - /// For images with bit depth ≤ 8, returns pixel data as `Vec`. - /// For images with bit depth > 8 (e.g., 12-bit or 16-bit), returns - /// pixel data as little-endian `u16` values packed into `Vec`. - /// - /// This is essential for medical imaging (DICOM) where 12-bit and 16-bit - /// images must preserve their full dynamic range. - pub fn decode_native(&self) -> Result { - let mut decoder_context = DecoderContext::default(); - self.decode_native_with_context(&mut decoder_context) - } - - /// Decode a region of the image at native bit depth. - pub fn decode_native_region(&self, roi: (u32, u32, u32, u32)) -> Result { - self.decode_native_region_with_context(roi, &mut DecoderContext::default()) - } - - /// Decode the image at native bit depth using a caller-provided decoder - /// context so allocations can be reused across repeated decodes. - pub fn decode_native_with_context( - &self, - decoder_context: &mut DecoderContext<'a>, - ) -> Result { - self.decode_with_output_region(decoder_context, None)?; - - let components = &decoder_context.tile_decode_context.channel_data; - let bit_depth = self.original_bit_depth(); - let num_components = components.len() as u8; - let width = self.width(); - let height = self.height(); - let pixel_count = width as usize * height as usize; - - if bit_depth <= 8 { - let max_val = ((1u32 << bit_depth) - 1) as f32; - let mut data = Vec::with_capacity(pixel_count * num_components as usize); - for i in 0..pixel_count { - for component in components.iter() { - let v = math::round_f32(component.container.truncated()[i]); - let clamped = if v < 0.0 { - 0.0 - } else if v > max_val { - max_val - } else { - v - }; - data.push(clamped as u8); - } - } - Ok(RawBitmap { - data, - width, - height, - bit_depth, - num_components, - bytes_per_sample: 1, - }) - } else { - let max_val = ((1u32 << bit_depth) - 1) as f32; - let mut data = Vec::with_capacity(pixel_count * num_components as usize * 2); - for i in 0..pixel_count { - for component in components.iter() { - let v = math::round_f32(component.container.truncated()[i]); - let clamped = if v < 0.0 { - 0.0 - } else if v > max_val { - max_val - } else { - v - }; - let val = clamped as u16; - data.extend_from_slice(&val.to_le_bytes()); - } - } - Ok(RawBitmap { - data, - width, - height, - bit_depth, - num_components, - bytes_per_sample: 2, - }) - } - } - - /// Decode a region of the image at native bit depth using a caller-provided - /// decoder context. - pub fn decode_native_region_with_context( - &self, - roi: (u32, u32, u32, u32), - decoder_context: &mut DecoderContext<'a>, - ) -> Result { - validate_roi((self.width(), self.height()), roi)?; - self.decode_with_output_region(decoder_context, Some(roi))?; - - let components = &decoder_context.tile_decode_context.channel_data; - let bit_depth = self.original_bit_depth(); - let num_components = components.len() as u8; - let bytes_per_sample = if bit_depth <= 8 { 1 } else { 2 }; - let (_x, _y, width, height) = roi; - let mut data = Vec::with_capacity( - width as usize * height as usize * num_components as usize * bytes_per_sample, - ); - let max_val = ((1u32 << bit_depth) - 1) as f32; - - for row in 0..height as usize { - for col in 0..width as usize { - let idx = row * width as usize + col; - for component in components { - let v = math::round_f32(component.container.truncated()[idx]); - let clamped = if v < 0.0 { - 0.0 - } else if v > max_val { - max_val - } else { - v - }; - if bit_depth <= 8 { - data.push(clamped as u8); - } else { - data.extend_from_slice(&(clamped as u16).to_le_bytes()); - } - } - } - } - - Ok(RawBitmap { - data, - width, - height, - bit_depth, - num_components, - bytes_per_sample: bytes_per_sample as u8, - }) - } - - /// Decode the image into the given buffer. - /// - /// This method does the same as [`Image::decode`], but you can provide - /// a custom buffer for the output, as well as a decoder context. Doing - /// so allows the internal decode engine to reuse memory allocations, so - /// this is especially recommended if you plan on converting multiple - /// images in the same session. - /// - /// The buffer must have the correct size. - pub fn decode_into( - &self, - buf: &mut [u8], - decoder_context: &mut DecoderContext<'a>, - ) -> Result<()> { - let mut decoded_image = self.prepare_decoded_image(decoder_context)?; - interleave_and_convert(&mut decoded_image, buf); - - Ok(()) - } - - fn prepare_decoded_image<'ctx>( - &self, - decoder_context: &'ctx mut DecoderContext<'a>, - ) -> Result> { - self.prepare_decoded_image_with_region(decoder_context, None) - } - - fn prepare_decoded_image_with_ht_decoder<'ctx>( - &self, - decoder_context: &'ctx mut DecoderContext<'a>, - ht_decoder: &mut dyn HtCodeBlockDecoder, - ) -> Result> { - self.prepare_decoded_image_with_region_and_ht_decoder( - decoder_context, - None, - Some(ht_decoder), - ) - } - - fn prepare_decoded_image_with_region<'ctx>( - &self, - decoder_context: &'ctx mut DecoderContext<'a>, - output_region: Option<(u32, u32, u32, u32)>, - ) -> Result> { - self.prepare_decoded_image_with_region_and_ht_decoder(decoder_context, output_region, None) - } - - fn prepare_decoded_image_with_region_and_ht_decoder<'ctx>( - &self, - decoder_context: &'ctx mut DecoderContext<'a>, - output_region: Option<(u32, u32, u32, u32)>, - ht_decoder: Option<&mut dyn HtCodeBlockDecoder>, - ) -> Result> { - let settings = &self.settings; - self.decode_with_output_region_and_ht_decoder(decoder_context, output_region, ht_decoder)?; - let mut decoded_image = DecodedImage { - decoded_components: &mut decoder_context.tile_decode_context.channel_data, - boxes: self.boxes.clone(), - }; - - if settings.resolve_palette_indices { - let components = core::mem::take(decoded_image.decoded_components); - *decoded_image.decoded_components = - resolve_palette_indices(components, &decoded_image.boxes)?; - } - - if let Some(cdef) = &decoded_image.boxes.channel_definition { - let mut components = decoded_image - .decoded_components - .iter() - .cloned() - .zip( - cdef.channel_definitions - .iter() - .map(|c| match c._association { - ChannelAssociation::WholeImage => u16::MAX, - ChannelAssociation::Colour(c) => c, - }), - ) - .collect::>(); - components.sort_by_key(|component| component.1); - *decoded_image.decoded_components = components.into_iter().map(|c| c.0).collect(); - } - - let bit_depth = decoded_image.decoded_components[0].bit_depth; - convert_color_space(&mut decoded_image, bit_depth)?; - Ok(decoded_image) - } - - fn decode_with_output_region( - &self, - decoder_context: &mut DecoderContext<'a>, - output_region: Option<(u32, u32, u32, u32)>, - ) -> Result<()> { - self.decode_with_output_region_and_ht_decoder(decoder_context, output_region, None) - } - - fn decode_with_output_region_and_ht_decoder( - &self, - decoder_context: &mut DecoderContext<'a>, - output_region: Option<(u32, u32, u32, u32)>, - mut ht_decoder: Option<&mut dyn HtCodeBlockDecoder>, - ) -> Result<()> { - decoder_context.set_output_region(output_region); - let decode_result = j2c::decode( - self.codestream, - &self.header, - decoder_context, - &mut ht_decoder, - ); - decoder_context.set_output_region(None); - decode_result - } -} - -pub(crate) fn resolve_alpha_and_color_space( - boxes: &ImageBoxes, - header: &Header<'_>, - settings: &DecodeSettings, -) -> Result<(ColorSpace, bool)> { - let mut num_components = header.component_infos.len(); - - // Override number of components with what is actually in the palette box - // in case we resolve them. - if settings.resolve_palette_indices { - if let Some(palette_box) = &boxes.palette { - num_components = palette_box.columns.len(); - } - } - - let mut has_alpha = false; - - if let Some(cdef) = &boxes.channel_definition { - let last = cdef.channel_definitions.last().unwrap(); - has_alpha = last.channel_type == ChannelType::Opacity; - } - - let mut color_space = get_color_space(boxes, num_components)?; - - // If we didn't resolve palette indices, we need to assume grayscale image. - if !settings.resolve_palette_indices && boxes.palette.is_some() { - has_alpha = false; - color_space = ColorSpace::Gray; - } - - let actual_num_components = header.component_infos.len(); - - // Validate the number of channels. - if boxes.palette.is_none() - && actual_num_components - != (color_space.num_channels() + if has_alpha { 1 } else { 0 }) as usize - { - if !settings.strict - && actual_num_components == color_space.num_channels() as usize + 1 - && !has_alpha - { - // See OPENJPEG test case orb-blue10-lin-j2k. Assume that we have an - // alpha channel in this case. - has_alpha = true; - } else { - // Color space is invalid, attempt to repair. - if actual_num_components == 1 || (actual_num_components == 2 && has_alpha) { - color_space = ColorSpace::Gray; - } else if actual_num_components == 3 { - color_space = ColorSpace::RGB; - } else if actual_num_components == 4 { - if has_alpha { - color_space = ColorSpace::RGB; - } else { - color_space = ColorSpace::CMYK; - } - } else { - bail!(ValidationError::TooManyChannels); - } - } - } - - Ok((color_space, has_alpha)) -} - -/// The color space of the image. -#[derive(Debug, Clone)] -pub enum ColorSpace { - /// A grayscale image. - Gray, - /// An RGB image. - RGB, - /// A CMYK image. - CMYK, - /// An unknown color space. - Unknown { - /// The number of channels of the color space. - num_channels: u8, - }, - /// An image based on an ICC profile. - Icc { - /// The raw data of the ICC profile. - profile: Vec, - /// The number of channels used by the ICC profile. - num_channels: u8, - }, -} - -impl ColorSpace { - /// Return the number of expected channels for the color space. - pub fn num_channels(&self) -> u8 { - match self { - Self::Gray => 1, - Self::RGB => 3, - Self::CMYK => 4, - Self::Unknown { num_channels } => *num_channels, - Self::Icc { - num_channels: num_components, - .. - } => *num_components, - } - } -} - -/// A bitmap storing the decoded result of the image. -pub struct Bitmap { - /// The color space of the image. - pub color_space: ColorSpace, - /// The raw pixel data of the image. The result will always be in - /// 8-bit (in case the original image had a different bit-depth, this - /// decode path scales it to 8-bit). - /// - /// The size is guaranteed to equal - /// `width * height * (num_channels + (if has_alpha { 1 } else { 0 })`. - /// Pixels are interleaved on a per-channel basis, the alpha channel always - /// appearing as the last channel, if available. - pub data: Vec, - /// Whether the image has an alpha channel. - pub has_alpha: bool, - /// The width of the image. - pub width: u32, - /// The height of the image. - pub height: u32, - /// The original bit depth of the image. You usually don't need to do anything - /// with this parameter, it just exists for informational purposes. - pub original_bit_depth: u8, -} - -/// Raw decoded pixel data at native bit depth (no 8-bit scaling). -/// -/// For bit depths ≤ 8, `data` contains one byte per sample. -/// For bit depths > 8 (e.g., 12 or 16), `data` contains two bytes per sample -/// in little-endian byte order (`u16` LE). -/// -/// Samples are interleaved: for a 3-component image, the layout is -/// `[R0, G0, B0, R1, G1, B1, ...]`. -pub struct RawBitmap { - /// The raw pixel data at native bit depth. - pub data: Vec, - /// The width of the image in pixels. - pub width: u32, - /// The height of the image in pixels. - pub height: u32, - /// The original bit depth per sample (e.g., 8, 12, 16). - pub bit_depth: u8, - /// The number of components (e.g., 1 for grayscale, 3 for RGB). - pub num_components: u8, - /// Bytes per sample: 1 for bit_depth ≤ 8, 2 for bit_depth > 8. - pub bytes_per_sample: u8, -} - -/// A borrowed decoded component plane. -pub struct ComponentPlane<'a> { - samples: &'a [f32], - bit_depth: u8, -} - -impl<'a> ComponentPlane<'a> { - /// Component samples in row-major order. - pub fn samples(&self) -> &'a [f32] { - self.samples - } - - /// Bit depth of this component plane. - pub fn bit_depth(&self) -> u8 { - self.bit_depth - } -} - -/// Borrowed decoded component planes for an image. -pub struct DecodedComponents<'a> { - dimensions: (u32, u32), - color_space: ColorSpace, - has_alpha: bool, - planes: Vec>, -} - -impl<'a> DecodedComponents<'a> { - /// Dimensions of the decoded image represented by these planes. - pub fn dimensions(&self) -> (u32, u32) { - self.dimensions - } - - /// Color space after JPEG 2000 color conversion has been applied. - pub fn color_space(&self) -> &ColorSpace { - &self.color_space - } - - /// Whether the decoded image has an alpha channel. - pub fn has_alpha(&self) -> bool { - self.has_alpha - } - - /// Borrowed decoded component planes in display order. - pub fn planes(&self) -> &[ComponentPlane<'a>] { - &self.planes - } -} - -fn interleave_and_convert(image: &mut DecodedImage<'_>, buf: &mut [u8]) { - let components = &mut *image.decoded_components; - let num_components = components.len(); - - let mut all_same_bit_depth = Some(components[0].bit_depth); - - for component in components.iter().skip(1) { - if Some(component.bit_depth) != all_same_bit_depth { - all_same_bit_depth = None; - } - } - - let max_len = components[0].container.truncated().len(); - - let mut output_iter = buf.iter_mut(); - - if all_same_bit_depth == Some(8) && num_components <= 4 { - // Fast path for the common case. - match num_components { - // Gray-scale. - 1 => { - for (output, input) in output_iter.zip( - components[0] - .container - .iter() - .map(|v| math::round_f32(*v) as u8), - ) { - *output = input; - } - } - // Gray-scale with alpha. - 2 => { - let c0 = &components[0]; - let c1 = &components[1]; - - let c0 = &c0.container[..max_len]; - let c1 = &c1.container[..max_len]; - - for i in 0..max_len { - *output_iter.next().unwrap() = math::round_f32(c0[i]) as u8; - *output_iter.next().unwrap() = math::round_f32(c1[i]) as u8; - } - } - // RGB - 3 => { - let c0 = &components[0]; - let c1 = &components[1]; - let c2 = &components[2]; - - let c0 = &c0.container[..max_len]; - let c1 = &c1.container[..max_len]; - let c2 = &c2.container[..max_len]; - - for i in 0..max_len { - *output_iter.next().unwrap() = math::round_f32(c0[i]) as u8; - *output_iter.next().unwrap() = math::round_f32(c1[i]) as u8; - *output_iter.next().unwrap() = math::round_f32(c2[i]) as u8; - } - } - // RGBA or CMYK. - 4 => { - let c0 = &components[0]; - let c1 = &components[1]; - let c2 = &components[2]; - let c3 = &components[3]; - - let c0 = &c0.container[..max_len]; - let c1 = &c1.container[..max_len]; - let c2 = &c2.container[..max_len]; - let c3 = &c3.container[..max_len]; - - for i in 0..max_len { - *output_iter.next().unwrap() = math::round_f32(c0[i]) as u8; - *output_iter.next().unwrap() = math::round_f32(c1[i]) as u8; - *output_iter.next().unwrap() = math::round_f32(c2[i]) as u8; - *output_iter.next().unwrap() = math::round_f32(c3[i]) as u8; - } - } - _ => unreachable!(), - } - } else { - // Slow path that also requires us to scale to 8 bit. - let mul_factor = ((1 << 8) - 1) as f32; - - for sample in 0..max_len { - for channel in components.iter() { - *output_iter.next().unwrap() = math::round_f32( - (channel.container[sample] / ((1_u32 << channel.bit_depth) - 1) as f32) - * mul_factor, - ) as u8; - } - } - } -} - -fn interleave_and_convert_region( - image: &mut DecodedImage<'_>, - image_width: usize, - roi: (u32, u32, u32, u32), - buf: &mut [u8], -) { - let components = &mut *image.decoded_components; - let num_components = components.len(); - let (x, y, width, height) = roi; - let mut output_iter = buf.iter_mut(); - - let mut all_same_bit_depth = Some(components[0].bit_depth); - for component in components.iter().skip(1) { - if Some(component.bit_depth) != all_same_bit_depth { - all_same_bit_depth = None; - } - } - - if all_same_bit_depth == Some(8) && num_components <= 4 { - for row in y as usize..(y + height) as usize { - let row_base = row * image_width; - for col in x as usize..(x + width) as usize { - let idx = row_base + col; - for component in components.iter() { - *output_iter.next().unwrap() = math::round_f32(component.container[idx]) as u8; - } - } - } - } else { - let mul_factor = ((1 << 8) - 1) as f32; - for row in y as usize..(y + height) as usize { - let row_base = row * image_width; - for col in x as usize..(x + width) as usize { - let idx = row_base + col; - for component in components.iter() { - *output_iter.next().unwrap() = math::round_f32( - (component.container[idx] / ((1_u32 << component.bit_depth) - 1) as f32) - * mul_factor, - ) as u8; - } - } - } - } -} - -fn validate_roi(dims: (u32, u32), roi: (u32, u32, u32, u32)) -> Result<()> { - let (image_width, image_height) = dims; - let (x, y, width, height) = roi; - let x_end = x - .checked_add(width) - .ok_or(ValidationError::InvalidDimensions)?; - let y_end = y - .checked_add(height) - .ok_or(ValidationError::InvalidDimensions)?; - if x_end > image_width || y_end > image_height { - return Err(ValidationError::InvalidDimensions.into()); - } - Ok(()) -} - -fn convert_color_space(image: &mut DecodedImage<'_>, bit_depth: u8) -> Result<()> { - if let Some(jp2::colr::ColorSpace::Enumerated(e)) = &image - .boxes - .color_specification - .as_ref() - .map(|i| &i.color_space) - { - match e { - EnumeratedColorspace::Sycc => { - dispatch!(Level::new(), simd => { - sycc_to_rgb(simd, image.decoded_components, bit_depth) - })?; - } - EnumeratedColorspace::CieLab(cielab) => { - dispatch!(Level::new(), simd => { - cielab_to_rgb(simd, image.decoded_components, bit_depth, cielab) - })?; - } - _ => {} - } - } - - Ok(()) -} - -fn get_color_space(boxes: &ImageBoxes, num_components: usize) -> Result { - let cs = match boxes - .color_specification - .as_ref() - .map(|c| &c.color_space) - .unwrap_or(&jp2::colr::ColorSpace::Unknown) - { - jp2::colr::ColorSpace::Enumerated(e) => { - match e { - EnumeratedColorspace::Cmyk => ColorSpace::CMYK, - EnumeratedColorspace::Srgb => ColorSpace::RGB, - EnumeratedColorspace::RommRgb => { - // Use an ICC profile to process the RommRGB color space. - ColorSpace::Icc { - profile: include_bytes!("../assets/ProPhoto-v2-micro.icc").to_vec(), - num_channels: 3, - } - } - EnumeratedColorspace::EsRgb => ColorSpace::RGB, - EnumeratedColorspace::Greyscale => ColorSpace::Gray, - EnumeratedColorspace::Sycc => ColorSpace::RGB, - EnumeratedColorspace::CieLab(_) => ColorSpace::Icc { - profile: include_bytes!("../assets/LAB.icc").to_vec(), - num_channels: 3, - }, - _ => bail!(FormatError::Unsupported), - } - } - jp2::colr::ColorSpace::Icc(icc) => { - if let Some(metadata) = ICCMetadata::from_data(icc) { - ColorSpace::Icc { - profile: icc.clone(), - num_channels: metadata.color_space.num_components(), - } - } else { - // See OPENJPEG test orb-blue10-lin-jp2.jp2. They seem to - // assume RGB in this case (even though the image has 4 - // components with no opacity channel, they assume RGBA instead - // of CMYK). - ColorSpace::RGB - } - } - jp2::colr::ColorSpace::Unknown => match num_components { - 1 => ColorSpace::Gray, - 3 => ColorSpace::RGB, - 4 => ColorSpace::CMYK, - _ => ColorSpace::Unknown { - num_channels: num_components as u8, - }, - }, - }; - - Ok(cs) -} - -fn resolve_palette_indices( - components: Vec, - boxes: &ImageBoxes, -) -> Result> { - let Some(palette) = boxes.palette.as_ref() else { - // Nothing to resolve. - return Ok(components); - }; - - let mapping = boxes.component_mapping.as_ref().unwrap(); - let mut resolved = Vec::with_capacity(mapping.entries.len()); - - for entry in &mapping.entries { - let component_idx = entry.component_index as usize; - let component = components - .get(component_idx) - .ok_or(ColorError::PaletteResolutionFailed)?; - - match entry.mapping_type { - ComponentMappingType::Direct => resolved.push(component.clone()), - ComponentMappingType::Palette { column } => { - let column_idx = column as usize; - let column_info = palette - .columns - .get(column_idx) - .ok_or(ColorError::PaletteResolutionFailed)?; - - let mut mapped = - Vec::with_capacity(component.container.truncated().len() + SIMD_WIDTH); - - for &sample in component.container.truncated() { - let index = math::round_f32(sample) as i64; - let value = palette - .map(index as usize, column_idx) - .ok_or(ColorError::PaletteResolutionFailed)?; - mapped.push(value as f32); - } - - resolved.push(ComponentData { - container: math::SimdBuffer::new(mapped), - bit_depth: column_info.bit_depth, - }); - } - } - } - - Ok(resolved) -} - -#[inline(always)] -fn cielab_to_rgb( - simd: S, - components: &mut [ComponentData], - bit_depth: u8, - lab: &CieLab, -) -> Result<()> { - let (head, _) = components - .split_at_mut_checked(3) - .ok_or(ColorError::LabConversionFailed)?; - - let [l, a, b] = head else { - unreachable!(); - }; - - let prec0 = l.bit_depth; - let prec1 = a.bit_depth; - let prec2 = b.bit_depth; - - // Prevent underflows/divisions by zero further below. - if prec0 < 4 || prec1 < 4 || prec2 < 4 { - bail!(ColorError::LabConversionFailed); - } - - let rl = lab.rl.unwrap_or(100); - let ra = lab.ra.unwrap_or(170); - let rb = lab.ra.unwrap_or(200); - let ol = lab.ol.unwrap_or(0); - let oa = lab.oa.unwrap_or(1 << (bit_depth - 1)); - let ob = lab - .ob - .unwrap_or((1 << (bit_depth - 2)) + (1 << (bit_depth - 3))); - - // Copied from OpenJPEG. - let min_l = -(rl as f32 * ol as f32) / ((1 << prec0) - 1) as f32; - let max_l = min_l + rl as f32; - let min_a = -(ra as f32 * oa as f32) / ((1 << prec1) - 1) as f32; - let max_a = min_a + ra as f32; - let min_b = -(rb as f32 * ob as f32) / ((1 << prec2) - 1) as f32; - let max_b = min_b + rb as f32; - - let bit_max = (1_u32 << bit_depth) - 1; - - // Note that we are not doing the actual conversion with the ICC profile yet, - // just decoding the raw LAB values. - // We leave applying the ICC profile to the user. - let divisor_l = ((1 << prec0) - 1) as f32; - let divisor_a = ((1 << prec1) - 1) as f32; - let divisor_b = ((1 << prec2) - 1) as f32; - - let scale_l_final = bit_max as f32 / 100.0; - let scale_ab_final = bit_max as f32 / 255.0; - - let l_offset = min_l * scale_l_final; - let l_scale = (max_l - min_l) / divisor_l * scale_l_final; - let a_offset = (min_a + 128.0) * scale_ab_final; - let a_scale = (max_a - min_a) / divisor_a * scale_ab_final; - let b_offset = (min_b + 128.0) * scale_ab_final; - let b_scale = (max_b - min_b) / divisor_b * scale_ab_final; - - let l_offset_v = f32x8::splat(simd, l_offset); - let l_scale_v = f32x8::splat(simd, l_scale); - let a_offset_v = f32x8::splat(simd, a_offset); - let a_scale_v = f32x8::splat(simd, a_scale); - let b_offset_v = f32x8::splat(simd, b_offset); - let b_scale_v = f32x8::splat(simd, b_scale); - - // Note that we are not doing the actual conversion with the ICC profile yet, - // just decoding the raw LAB values. - // We leave applying the ICC profile to the user. - for ((l_chunk, a_chunk), b_chunk) in l - .container - .chunks_exact_mut(SIMD_WIDTH) - .zip(a.container.chunks_exact_mut(SIMD_WIDTH)) - .zip(b.container.chunks_exact_mut(SIMD_WIDTH)) - { - let l_v = f32x8::from_slice(simd, l_chunk); - let a_v = f32x8::from_slice(simd, a_chunk); - let b_v = f32x8::from_slice(simd, b_chunk); - - l_v.mul_add(l_scale_v, l_offset_v).store(l_chunk); - a_v.mul_add(a_scale_v, a_offset_v).store(a_chunk); - b_v.mul_add(b_scale_v, b_offset_v).store(b_chunk); - } - - Ok(()) -} - -#[inline(always)] -fn sycc_to_rgb(simd: S, components: &mut [ComponentData], bit_depth: u8) -> Result<()> { - let offset = (1_u32 << (bit_depth as u32 - 1)) as f32; - let max_value = ((1_u32 << bit_depth as u32) - 1) as f32; - - let (head, _) = components - .split_at_mut_checked(3) - .ok_or(ColorError::SyccConversionFailed)?; - - let [y, cb, cr] = head else { - unreachable!(); - }; - - let offset_v = f32x8::splat(simd, offset); - let max_v = f32x8::splat(simd, max_value); - let zero_v = f32x8::splat(simd, 0.0); - let cr_to_r = f32x8::splat(simd, 1.402); - let cb_to_g = f32x8::splat(simd, -0.344136); - let cr_to_g = f32x8::splat(simd, -0.714136); - let cb_to_b = f32x8::splat(simd, 1.772); - - for ((y_chunk, cb_chunk), cr_chunk) in y - .container - .chunks_exact_mut(SIMD_WIDTH) - .zip(cb.container.chunks_exact_mut(SIMD_WIDTH)) - .zip(cr.container.chunks_exact_mut(SIMD_WIDTH)) - { - let y_v = f32x8::from_slice(simd, y_chunk); - let cb_v = f32x8::from_slice(simd, cb_chunk) - offset_v; - let cr_v = f32x8::from_slice(simd, cr_chunk) - offset_v; - - // r = y + 1.402 * cr - let r = cr_v.mul_add(cr_to_r, y_v); - // g = y - 0.344136 * cb - 0.714136 * cr - let g = cr_v.mul_add(cr_to_g, cb_v.mul_add(cb_to_g, y_v)); - // b = y + 1.772 * cb - let b = cb_v.mul_add(cb_to_b, y_v); - - r.min(max_v).max(zero_v).store(y_chunk); - g.min(max_v).max(zero_v).store(cb_chunk); - b.min(max_v).max(zero_v).store(cr_chunk); - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - #[derive(Default)] - struct DecodeWorkCounter { - classic_code_blocks: usize, - ht_code_blocks: usize, - idwt_output_samples: usize, - } - - impl DecodeWorkCounter { - fn code_blocks(&self) -> usize { - self.classic_code_blocks + self.ht_code_blocks - } - } - - struct FailingHtDecoder { - called: bool, - } - - impl HtCodeBlockDecoder for FailingHtDecoder { - fn decode_code_block( - &mut self, - _job: HtCodeBlockDecodeJob<'_>, - _output: &mut [f32], - ) -> Result<()> { - self.called = true; - Err(DecodingError::CodeBlockDecodeFailure.into()) - } - } - - struct FailingClassicDecoder { - called: bool, - } - - impl HtCodeBlockDecoder for FailingClassicDecoder { - fn decode_code_block( - &mut self, - _job: HtCodeBlockDecodeJob<'_>, - _output: &mut [f32], - ) -> Result<()> { - panic!("HT hook must not be used for classic J2K test") - } - - fn decode_j2k_code_block( - &mut self, - _job: J2kCodeBlockDecodeJob<'_>, - _output: &mut [f32], - ) -> Result { - self.called = true; - Err(DecodingError::CodeBlockDecodeFailure.into()) - } - } - - struct FailingClassicBatchDecoder { - called: bool, - } - - impl HtCodeBlockDecoder for FailingClassicBatchDecoder { - fn decode_code_block( - &mut self, - _job: HtCodeBlockDecodeJob<'_>, - _output: &mut [f32], - ) -> Result<()> { - panic!("HT hook must not be used for classic J2K batch test") - } - - fn decode_j2k_code_block( - &mut self, - _job: J2kCodeBlockDecodeJob<'_>, - _output: &mut [f32], - ) -> Result { - panic!( - "per-block classic hook must not be used when the batch hook handles the sub-band" - ) - } - - fn decode_j2k_sub_band( - &mut self, - _job: J2kSubBandDecodeJob<'_>, - _output: &mut [f32], - ) -> Result { - self.called = true; - Err(DecodingError::CodeBlockDecodeFailure.into()) - } - } - - fn fixture() -> Vec { - let pixels = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120]; - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - encode(&pixels, 2, 2, 3, 8, false, &options).expect("encode") - } - - fn fixture_multi_block() -> Vec { - let pixels: Vec = (0..64).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 0, - code_block_width_exp: 0, - code_block_height_exp: 0, - ..EncodeOptions::default() - }; - encode(&pixels, 8, 8, 1, 8, false, &options).expect("encode multi-block classic") - } - - fn fixture_gray() -> Vec { - let pixels: Vec = (0..16).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - encode(&pixels, 4, 4, 1, 8, false, &options).expect("encode classic gray8") - } - - fn fixture_ht_gray() -> Vec { - let pixels: Vec = (0..16).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - encode_htj2k(&pixels, 4, 4, 1, 8, false, &options).expect("encode ht gray8") - } - - fn gradient_pixels(width: u32, height: u32, components: u8) -> Vec { - let mut pixels = Vec::with_capacity(width as usize * height as usize * components as usize); - for y in 0..height { - for x in 0..width { - for component in 0..components { - pixels.push(((x * 3 + y * 5 + u32::from(component) * 41) & 0xff) as u8); - } - } - } - pixels - } - - fn roi_fixture(classic: bool, components: u8) -> Vec { - let width = 64; - let height = 64; - let pixels = gradient_pixels(width, height, components); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 2, - code_block_width_exp: 0, - code_block_height_exp: 0, - ..EncodeOptions::default() - }; - if classic { - encode(&pixels, width, height, components, 8, false, &options) - .expect("encode ROI classic fixture") - } else { - encode_htj2k(&pixels, width, height, components, 8, false, &options) - .expect("encode ROI HT fixture") - } - } - - fn crop_interleaved( - full: &[u8], - full_width: u32, - channels: usize, - roi: (u32, u32, u32, u32), - ) -> Vec { - let (x, y, width, height) = roi; - let mut out = Vec::with_capacity(width as usize * height as usize * channels); - let row_bytes = full_width as usize * channels; - let roi_row_bytes = width as usize * channels; - for row in y as usize..(y + height) as usize { - let start = row * row_bytes + x as usize * channels; - out.extend_from_slice(&full[start..start + roi_row_bytes]); - } - out - } - - fn count_decode_work(bytes: &[u8], roi: Option<(u32, u32, u32, u32)>) -> DecodeWorkCounter { - let image = Image::new(bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - match roi { - Some(roi) => { - image - .decode_region_with_context(roi, &mut context) - .expect("region decode with counter"); - } - None => { - image - .decode_with_context(&mut context) - .expect("full decode with counter"); - } - } - let counters = context.tile_decode_context.debug_counters; - DecodeWorkCounter { - classic_code_blocks: counters.decoded_code_blocks, - ht_code_blocks: 0, - idwt_output_samples: counters.idwt_output_samples, - } - } - - #[test] - fn roi_decode_matches_full_crop_for_classic_and_htj2k_gray_and_rgb() { - let cases = [ - (true, 1_u8, true, false), - (true, 3_u8, false, false), - (false, 1_u8, true, false), - (false, 3_u8, false, false), - ]; - let rois = [ - (20, 18, 17, 19), - (0, 0, 9, 11), - (63, 63, 1, 1), - (7, 5, 13, 9), - (0, 0, 64, 64), - ]; - - for (classic, components, expect_gray, has_alpha) in cases { - let bytes = roi_fixture(classic, components); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let full = image.decode().expect("full decode"); - let channels = components as usize; - for roi in rois { - let region = image.decode_region(roi).expect("region decode"); - assert_eq!(matches!(region.color_space, ColorSpace::Gray), expect_gray); - assert_eq!(region.has_alpha, has_alpha); - assert_eq!( - region.data, - crop_interleaved(&full, 64, channels, roi), - "classic={classic} components={components} roi={roi:?}" - ); - } - } - } - - #[test] - fn roi_decode_prunes_code_blocks_and_idwt_work_for_classic_and_htj2k() { - let roi = (48, 48, 16, 16); - for classic in [true, false] { - let bytes = { - let pixels = gradient_pixels(128, 128, 1); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 3, - code_block_width_exp: 0, - code_block_height_exp: 0, - ..EncodeOptions::default() - }; - if classic { - encode(&pixels, 128, 128, 1, 8, false, &options) - .expect("encode classic work fixture") - } else { - encode_htj2k(&pixels, 128, 128, 1, 8, false, &options) - .expect("encode ht work fixture") - } - }; - let full = count_decode_work(&bytes, None); - let region = count_decode_work(&bytes, Some(roi)); - - assert!( - region.code_blocks() > 0 && region.code_blocks() < full.code_blocks(), - "ROI should decode fewer code-blocks for classic={classic}; full={}, region={}", - full.code_blocks(), - region.code_blocks() - ); - assert!( - region.idwt_output_samples > 0 - && region.idwt_output_samples < full.idwt_output_samples, - "ROI should produce fewer IDWT output samples for classic={classic}; full={}, region={}", - full.idwt_output_samples, - region.idwt_output_samples - ); - } - } - - #[test] - fn region_decode_reuses_region_sized_component_storage() { - let bytes = fixture(); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - - let bitmap = image - .decode_region_with_context((1, 0, 1, 2), &mut context) - .expect("region decode"); - - assert_eq!((bitmap.width, bitmap.height), (1, 2)); - assert!(context - .tile_decode_context - .channel_data - .iter() - .all(|component| component.container.truncated().len() == 2)); - } - - #[test] - fn native_region_decode_reuses_region_sized_component_storage() { - let bytes = fixture(); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - - let bitmap = image - .decode_native_region_with_context((1, 0, 1, 2), &mut context) - .expect("native region decode"); - - assert_eq!((bitmap.width, bitmap.height), (1, 2)); - assert!(context - .tile_decode_context - .channel_data - .iter() - .all(|component| component.container.truncated().len() == 2)); - } - - #[test] - fn decoder_context_defaults_to_auto_cpu_parallelism() { - let context = DecoderContext::default(); - - assert_eq!(context.cpu_decode_parallelism(), CpuDecodeParallelism::Auto); - } - - #[test] - fn classic_j2k_auto_and_serial_cpu_parallelism_match_pixels() { - let bytes = fixture_multi_block(); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut auto_context = DecoderContext::default(); - let mut serial_context = DecoderContext::default(); - serial_context.set_cpu_decode_parallelism(CpuDecodeParallelism::Serial); - - let auto = image - .decode_with_context(&mut auto_context) - .expect("auto decode"); - let serial = image - .decode_with_context(&mut serial_context) - .expect("serial decode"); - - assert_eq!(auto.data, serial.data); - } - - #[test] - fn serial_cpu_parallelism_disables_classic_sub_band_parallel_branch() { - assert!(!j2c::should_decode_classic_sub_band_in_parallel( - CpuDecodeParallelism::Serial, - 16 - )); - } - - #[test] - fn grayscale_direct_plan_is_built_without_materializing_channel_data() { - let bytes = fixture_gray(); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("build direct plan"); - - assert_eq!(plan.dimensions, (4, 4)); - assert_eq!(plan.bit_depth, 8); - assert!( - !plan.steps.is_empty(), - "direct plan must contain executable steps" - ); - assert!( - plan.steps.iter().any(|step| matches!( - step, - J2kDirectGrayscaleStep::ClassicSubBand(plan) if !plan.jobs.is_empty() - )), - "classic J2K direct plan must contain at least one non-empty classic sub-band job" - ); - assert!( - context.tile_decode_context.channel_data.is_empty(), - "building a direct plan must not materialize host component planes" - ); - } - - #[test] - fn grayscale_direct_plan_honors_target_resolution() { - let bytes = fixture_ht_gray(); - let image = Image::new( - &bytes, - &DecodeSettings { - target_resolution: Some((2, 2)), - ..DecodeSettings::default() - }, - ) - .expect("scaled image"); - let mut context = DecoderContext::default(); - - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("build scaled direct plan"); - - assert_eq!(plan.dimensions, (2, 2)); - assert!(plan.steps.iter().any(|step| matches!( - step, - J2kDirectGrayscaleStep::HtSubBand(plan) if !plan.jobs.is_empty() - ))); - assert!(plan.steps.iter().any(|step| matches!( - step, - J2kDirectGrayscaleStep::Store(store) - if store.output_width == 2 && store.output_height == 2 - ))); - assert!( - context.tile_decode_context.channel_data.is_empty(), - "building a scaled direct plan must not materialize host component planes" - ); - } - - #[test] - fn htj2k_grayscale_direct_plan_contains_ht_sub_band_steps() { - let bytes = fixture_ht_gray(); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut context = DecoderContext::default(); - - let plan = image - .build_direct_grayscale_plan_with_context(&mut context) - .expect("build direct plan"); - - assert!( - plan.steps.iter().any(|step| matches!( - step, - J2kDirectGrayscaleStep::HtSubBand(plan) if !plan.jobs.is_empty() - )), - "HTJ2K direct plan must contain at least one non-empty HT sub-band decode step" - ); - } - - #[test] - fn ht_decoder_hook_is_used_for_htj2k_codeblocks() { - let pixels: Vec = (0..16).collect(); - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - let bytes = encode_htj2k(&pixels, 4, 4, 1, 8, false, &options).expect("encode ht"); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut hooked_context = DecoderContext::default(); - let mut hook = FailingHtDecoder { called: false }; - let error = match image.decode_components_with_ht_decoder(&mut hooked_context, &mut hook) { - Ok(_) => panic!("hooked decode must use external HT decoder"), - Err(error) => error, - }; - - assert!(hook.called, "HT decoder hook must be invoked"); - assert_eq!( - error, - DecodeError::Decoding(DecodingError::CodeBlockDecodeFailure) - ); - } - - #[test] - fn classic_decoder_hook_is_used_for_j2k_codeblocks() { - let bytes = fixture(); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut hooked_context = DecoderContext::default(); - let mut hook = FailingClassicDecoder { called: false }; - let error = match image.decode_components_with_ht_decoder(&mut hooked_context, &mut hook) { - Ok(_) => panic!("hooked decode must use external classic decoder"), - Err(error) => error, - }; - - assert!(hook.called, "classic decoder hook must be invoked"); - assert_eq!( - error, - DecodeError::Decoding(DecodingError::CodeBlockDecodeFailure) - ); - } - - #[test] - fn classic_sub_band_decoder_hook_is_used_for_j2k_codeblocks() { - let bytes = fixture_multi_block(); - let image = Image::new(&bytes, &DecodeSettings::default()).expect("image"); - let mut hooked_context = DecoderContext::default(); - let mut hook = FailingClassicBatchDecoder { called: false }; - let error = match image.decode_components_with_ht_decoder(&mut hooked_context, &mut hook) { - Ok(_) => panic!("hooked decode must use external classic batch decoder"), - Err(error) => error, - }; - - assert!(hook.called, "classic sub-band decoder hook must be invoked"); - assert_eq!( - error, - DecodeError::Decoding(DecodingError::CodeBlockDecodeFailure) - ); - } -} diff --git a/crates/signinum-j2k/README.md b/crates/signinum-j2k/README.md deleted file mode 100644 index 393acfc1..00000000 --- a/crates/signinum-j2k/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# signinum-j2k - -JPEG 2000 / HTJ2K inspect and CPU decode for whole-slide imaging workloads. - -Install: - -```sh -cargo add signinum-j2k -``` - -The CPU-first 1.0 surface covers borrowed inspect/parse, compressed -passthrough candidates, full-frame decode, ROI decode, reduced-resolution -decode, combined ROI+reduced-resolution decode, row-bounded decode, and -tile-batch decode through the shared `signinum-core` traits. - -GPU adapter crates are versioned separately and are not part of the CPU-first -1.0 stability promise. diff --git a/crates/signinum-j2k/benches/public_api.rs b/crates/signinum-j2k/benches/public_api.rs deleted file mode 100644 index d926c7c9..00000000 --- a/crates/signinum-j2k/benches/public_api.rs +++ /dev/null @@ -1,159 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use signinum_j2k::{ - encode_j2k_lossless, EncodeBackendPreference, J2kDecoder, J2kEncodeValidation, - J2kLosslessEncodeOptions, J2kLosslessSamples, J2kScratchPool, PixelFormat, Rect, -}; -use signinum_test_support::{patterned_gray8, patterned_rgb8}; - -const TILE_SIDE: u32 = 128; -const ROI_SIDE: u32 = 64; - -fn bench_encode_options() -> J2kLosslessEncodeOptions { - J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - } -} - -fn encode_gray8_codestream(width: u32, height: u32) -> Vec { - let pixels = patterned_gray8(width, height); - let samples = - J2kLosslessSamples::new(&pixels, width, height, 1, 8, false).expect("valid gray8 samples"); - encode_j2k_lossless(samples, &bench_encode_options()) - .expect("encode gray8 codestream") - .codestream -} - -fn encode_rgb8_codestream(width: u32, height: u32) -> Vec { - let pixels = patterned_rgb8(width, height); - let samples = - J2kLosslessSamples::new(&pixels, width, height, 3, 8, false).expect("valid rgb8 samples"); - encode_j2k_lossless(samples, &bench_encode_options()) - .expect("encode rgb8 codestream") - .codestream -} - -fn bench_lossless_encode(c: &mut Criterion) { - let mut group = c.benchmark_group("j2k_public_lossless_encode"); - - let gray = patterned_gray8(TILE_SIDE, TILE_SIDE); - let options = bench_encode_options(); - group.bench_function("gray8_128x128", |b| { - b.iter(|| { - let samples = J2kLosslessSamples::new( - black_box(gray.as_slice()), - TILE_SIDE, - TILE_SIDE, - 1, - 8, - false, - ) - .expect("valid gray8 samples"); - let encoded = encode_j2k_lossless(samples, &options).expect("encode gray8"); - black_box(encoded.codestream.len()); - }); - }); - - let rgb = patterned_rgb8(TILE_SIDE, TILE_SIDE); - group.bench_function("rgb8_128x128", |b| { - b.iter(|| { - let samples = J2kLosslessSamples::new( - black_box(rgb.as_slice()), - TILE_SIDE, - TILE_SIDE, - 3, - 8, - false, - ) - .expect("valid rgb8 samples"); - let encoded = encode_j2k_lossless(samples, &options).expect("encode rgb8"); - black_box(encoded.codestream.len()); - }); - }); - - group.finish(); -} - -fn bench_inspect(c: &mut Criterion) { - let codestream = encode_rgb8_codestream(TILE_SIDE, TILE_SIDE); - - let mut group = c.benchmark_group("j2k_public_inspect"); - group.bench_function("rgb8_128x128", |b| { - b.iter(|| { - let info = J2kDecoder::inspect(black_box(codestream.as_slice())).expect("inspect"); - black_box(info); - }); - }); - group.finish(); -} - -fn bench_decode(c: &mut Criterion) { - let codestream = encode_rgb8_codestream(TILE_SIDE, TILE_SIDE); - let mut group = c.benchmark_group("j2k_public_decode"); - - let full_stride = TILE_SIDE as usize * 3; - let mut full = vec![0u8; full_stride * TILE_SIDE as usize]; - group.bench_function("rgb8_full_128x128", |b| { - b.iter(|| { - let mut decoder = - J2kDecoder::new(black_box(codestream.as_slice())).expect("rgb8 decoder"); - decoder - .decode_into(&mut full, full_stride, PixelFormat::Rgb8) - .expect("decode full rgb8"); - black_box(&full); - }); - }); - - let roi = Rect { - x: 32, - y: 32, - w: ROI_SIDE, - h: ROI_SIDE, - }; - let roi_stride = ROI_SIDE as usize * 3; - let mut roi_out = vec![0u8; roi_stride * ROI_SIDE as usize]; - let mut pool = J2kScratchPool::new(); - group.bench_function("rgb8_roi_64x64", |b| { - b.iter(|| { - let mut decoder = - J2kDecoder::new(black_box(codestream.as_slice())).expect("rgb8 decoder"); - decoder - .decode_region_into(&mut pool, &mut roi_out, roi_stride, PixelFormat::Rgb8, roi) - .expect("decode rgb8 roi"); - black_box(&roi_out); - }); - }); - - group.finish(); -} - -fn bench_decode_gray_setup(c: &mut Criterion) { - let codestream = encode_gray8_codestream(TILE_SIDE, TILE_SIDE); - let stride = TILE_SIDE as usize; - let mut out = vec![0u8; stride * TILE_SIDE as usize]; - - let mut group = c.benchmark_group("j2k_public_decode_gray"); - group.bench_function("gray8_full_128x128", |b| { - b.iter(|| { - let mut decoder = - J2kDecoder::new(black_box(codestream.as_slice())).expect("gray8 decoder"); - decoder - .decode_into(&mut out, stride, PixelFormat::Gray8) - .expect("decode full gray8"); - black_box(&out); - }); - }); - group.finish(); -} - -criterion_group!( - benches, - bench_lossless_encode, - bench_inspect, - bench_decode, - bench_decode_gray_setup -); -criterion_main!(benches); diff --git a/crates/signinum-j2k/src/adapter/mod.rs b/crates/signinum-j2k/src/adapter/mod.rs deleted file mode 100644 index 6f481e43..00000000 --- a/crates/signinum-j2k/src/adapter/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Public adapter-facing JPEG 2000 planning APIs. - -pub mod device_plan; diff --git a/crates/signinum-j2k/src/batch.rs b/crates/signinum-j2k/src/batch.rs deleted file mode 100644 index 5595290d..00000000 --- a/crates/signinum-j2k/src/batch.rs +++ /dev/null @@ -1,233 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use core::convert::Infallible; -use std::num::NonZeroUsize; - -pub use signinum_core::TileBatchOptions; -use signinum_core::{ - collect_indexed_batch_results, tile_batch_worker_count, DecodeOutcome, DecoderContext, - Downscale, IndexedBatchResult, PixelFormat, Rect, TileBatchDecode, -}; - -use crate::{CpuDecodeParallelism, J2kCodec, J2kContext, J2kError, J2kScratchPool}; - -/// One full-tile decode request for [`decode_tiles_into`]. -pub struct TileDecodeJob<'i, 'o> { - /// Compressed J2K/HTJ2K tile bytes. - pub input: &'i [u8], - /// Caller-owned output buffer for this tile. - pub out: &'o mut [u8], - /// Distance in bytes between output rows. - pub stride: usize, -} - -/// One ROI+scaled tile decode request for [`decode_tiles_region_scaled_into`]. -pub struct TileRegionScaledDecodeJob<'i, 'o> { - /// Compressed J2K/HTJ2K tile bytes. - pub input: &'i [u8], - /// Caller-owned output buffer for this tile. - pub out: &'o mut [u8], - /// Distance in bytes between output rows. - pub stride: usize, - /// Region of interest in source-image coordinates. - pub roi: Rect, - /// Downscale factor applied to the region decode. - pub scale: Downscale, -} - -/// Error returned by J2K CPU tile batches, annotated with the first failing -/// tile index from the caller's input order. -#[derive(Debug)] -pub struct TileBatchError { - /// Index of the first failing tile in input order. - pub index: usize, - /// Decode error reported for that tile. - pub source: J2kError, -} - -impl core::fmt::Display for TileBatchError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "tile {} decode failed: {}", self.index, self.source) - } -} - -impl std::error::Error for TileBatchError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - Some(&self.source) - } -} - -type BatchOutcome = DecodeOutcome; -type J2kIndexedBatchResult = IndexedBatchResult; - -/// One-shot parse-plus-decode of an independent J2K/HTJ2K tile into the -/// caller's buffer, reusing both caller-owned [`DecoderContext`] and -/// caller-owned [`J2kScratchPool`]. -pub fn decode_tile_into_in_context( - bytes: &[u8], - ctx: &mut DecoderContext, - pool: &mut J2kScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, -) -> Result { - ::decode_tile(ctx, pool, bytes, out, stride, fmt) -} - -/// One-shot parse-plus-ROI-scaled-decode of an independent J2K/HTJ2K tile -/// into the caller's buffer, reusing both caller-owned [`DecoderContext`] and -/// caller-owned [`J2kScratchPool`]. -#[allow(clippy::too_many_arguments)] -pub fn decode_tile_region_scaled_into_in_context( - bytes: &[u8], - ctx: &mut DecoderContext, - pool: &mut J2kScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, -) -> Result { - ::decode_tile_region_scaled( - ctx, pool, bytes, out, stride, fmt, roi, scale, - ) -} - -/// Decode independent J2K/HTJ2K tiles into caller-owned output buffers using -/// a scoped CPU worker pool. -/// -/// Each worker owns one [`DecoderContext`] and one [`J2kScratchPool`]. Returned -/// outcomes preserve caller input order. -pub fn decode_tiles_into( - jobs: &mut [TileDecodeJob<'_, '_>], - fmt: PixelFormat, - options: TileBatchOptions, -) -> Result, TileBatchError> { - if jobs.is_empty() { - return Ok(Vec::new()); - } - - let job_count = jobs.len(); - let worker_count = tile_batch_worker_count(job_count, options, available_tile_batch_workers()); - let chunk_size = job_count.div_ceil(worker_count); - let results = - std::thread::scope(|scope| { - let mut handles = Vec::with_capacity(worker_count); - for (chunk_index, chunk) in jobs.chunks_mut(chunk_size).enumerate() { - let start_index = chunk_index * chunk_size; - let inner_parallelism = inner_parallelism_for_batch(job_count); - handles.push(scope.spawn(move || { - decode_tile_job_chunk(start_index, chunk, fmt, inner_parallelism) - })); - } - - let mut results = Vec::with_capacity(job_count); - for handle in handles { - match handle.join() { - Ok(chunk_results) => results.extend(chunk_results), - Err(payload) => std::panic::resume_unwind(payload), - } - } - results - }); - - collect_indexed_batch_results(job_count, results, |index, source| TileBatchError { - index, - source, - }) -} - -/// Decode independent J2K/HTJ2K tile regions at reduced resolution into -/// caller-owned output buffers using a scoped CPU worker pool. -pub fn decode_tiles_region_scaled_into( - jobs: &mut [TileRegionScaledDecodeJob<'_, '_>], - fmt: PixelFormat, - options: TileBatchOptions, -) -> Result, TileBatchError> { - if jobs.is_empty() { - return Ok(Vec::new()); - } - - let job_count = jobs.len(); - let worker_count = tile_batch_worker_count(job_count, options, available_tile_batch_workers()); - let chunk_size = job_count.div_ceil(worker_count); - let results = std::thread::scope(|scope| { - let mut handles = Vec::with_capacity(worker_count); - for (chunk_index, chunk) in jobs.chunks_mut(chunk_size).enumerate() { - let start_index = chunk_index * chunk_size; - handles.push(scope.spawn(move || { - decode_tile_region_scaled_job_chunk( - start_index, - chunk, - fmt, - inner_parallelism_for_batch(job_count), - ) - })); - } - - let mut results = Vec::with_capacity(job_count); - for handle in handles { - match handle.join() { - Ok(chunk_results) => results.extend(chunk_results), - Err(payload) => std::panic::resume_unwind(payload), - } - } - results - }); - - collect_indexed_batch_results(job_count, results, |index, source| TileBatchError { - index, - source, - }) -} - -fn available_tile_batch_workers() -> usize { - std::thread::available_parallelism().map_or(1, NonZeroUsize::get) -} - -fn inner_parallelism_for_batch(batch_size: usize) -> CpuDecodeParallelism { - if batch_size > 1 { - CpuDecodeParallelism::Serial - } else { - CpuDecodeParallelism::Auto - } -} - -fn decode_tile_job_chunk( - start_index: usize, - jobs: &mut [TileDecodeJob<'_, '_>], - fmt: PixelFormat, - inner_parallelism: CpuDecodeParallelism, -) -> Vec { - let mut ctx = DecoderContext::::new(); - ctx.codec_mut() - .set_cpu_decode_parallelism(inner_parallelism); - let mut pool = J2kScratchPool::new(); - let mut results = Vec::with_capacity(jobs.len()); - for (local_index, job) in jobs.iter_mut().enumerate() { - let outcome = - decode_tile_into_in_context(job.input, &mut ctx, &mut pool, job.out, job.stride, fmt); - results.push((start_index + local_index, outcome)); - } - results -} - -fn decode_tile_region_scaled_job_chunk( - start_index: usize, - jobs: &mut [TileRegionScaledDecodeJob<'_, '_>], - fmt: PixelFormat, - inner_parallelism: CpuDecodeParallelism, -) -> Vec { - let mut ctx = DecoderContext::::new(); - ctx.codec_mut() - .set_cpu_decode_parallelism(inner_parallelism); - let mut pool = J2kScratchPool::new(); - let mut results = Vec::with_capacity(jobs.len()); - for (local_index, job) in jobs.iter_mut().enumerate() { - let outcome = decode_tile_region_scaled_into_in_context( - job.input, &mut ctx, &mut pool, job.out, job.stride, fmt, job.roi, job.scale, - ); - results.push((start_index + local_index, outcome)); - } - results -} diff --git a/crates/signinum-j2k/src/context.rs b/crates/signinum-j2k/src/context.rs deleted file mode 100644 index a7b39acf..00000000 --- a/crates/signinum-j2k/src/context.rs +++ /dev/null @@ -1,84 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use signinum_core::{CacheStats, CodecContext}; -use signinum_j2k_native::CpuDecodeParallelism; -#[cfg(target_os = "macos")] -use signinum_j2k_native::J2kDirectGrayscalePlan; - -#[cfg(target_os = "macos")] -#[derive(Debug, Clone)] -struct DirectGrayPlanCache { - key: u64, - plan: J2kDirectGrayscalePlan, -} - -#[derive(Debug, Default, Clone)] -pub struct J2kContext { - hits: u64, - misses: u64, - cpu_decode_parallelism: CpuDecodeParallelism, - #[cfg(target_os = "macos")] - direct_gray_plan: Option, -} - -impl J2kContext { - pub const fn new() -> Self { - Self { - hits: 0, - misses: 0, - cpu_decode_parallelism: CpuDecodeParallelism::Auto, - #[cfg(target_os = "macos")] - direct_gray_plan: None, - } - } - - pub(crate) fn record_tile_decode(&mut self) { - self.misses = self.misses.saturating_add(1); - } - - pub fn cpu_decode_parallelism(&self) -> CpuDecodeParallelism { - self.cpu_decode_parallelism - } - - pub fn set_cpu_decode_parallelism(&mut self, parallelism: CpuDecodeParallelism) { - self.cpu_decode_parallelism = parallelism; - } - - #[cfg(target_os = "macos")] - #[doc(hidden)] - pub fn cached_direct_gray_plan(&mut self, key: u64) -> Option { - if let Some(cache) = &self.direct_gray_plan { - if cache.key == key { - self.hits = self.hits.saturating_add(1); - return Some(cache.plan.clone()); - } - } - self.misses = self.misses.saturating_add(1); - None - } - - #[cfg(target_os = "macos")] - #[doc(hidden)] - pub fn store_direct_gray_plan(&mut self, key: u64, plan: J2kDirectGrayscalePlan) { - self.direct_gray_plan = Some(DirectGrayPlanCache { key, plan }); - } -} - -impl CodecContext for J2kContext { - fn clear(&mut self) { - self.hits = 0; - self.misses = 0; - self.cpu_decode_parallelism = CpuDecodeParallelism::Auto; - #[cfg(target_os = "macos")] - { - self.direct_gray_plan = None; - } - } - - fn cache_stats(&self) -> CacheStats { - CacheStats { - hits: self.hits, - misses: self.misses, - } - } -} diff --git a/crates/signinum-j2k/src/decode.rs b/crates/signinum-j2k/src/decode.rs deleted file mode 100644 index 14cec9f4..00000000 --- a/crates/signinum-j2k/src/decode.rs +++ /dev/null @@ -1,409 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use crate::backend::{ColorSpace, DecodeSettings, Image, RawBitmap}; -use crate::{backend, J2kError, J2kScratchPool}; -use alloc::{string::ToString, vec::Vec}; -use core::convert::Infallible; -use signinum_core::{ - validate_strided_output_buffer, DecodeOutcome, Downscale, PixelFormat, Rect, Unsupported, -}; -pub(crate) type J2kDecodeOutcome = DecodeOutcome; - -pub(crate) fn decode_scaled_from_info( - bytes: &[u8], - full_dims: (u32, u32), - pool: &mut J2kScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - scale: Downscale, -) -> Result { - validate_supported_format(fmt)?; - let target_dims = ( - full_dims.0.div_ceil(scale.denominator()), - full_dims.1.div_ceil(scale.denominator()), - ); - let settings = DecodeSettings { - target_resolution: Some(target_dims), - ..DecodeSettings::default() - }; - let _ = pool; - let _ = full_dims; - decode_with_settings(bytes, settings, out, stride, fmt) -} - -#[allow(clippy::too_many_arguments)] -pub(crate) fn decode_region_scaled_from_info( - bytes: &[u8], - full_dims: (u32, u32), - pool: &mut J2kScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, -) -> Result { - validate_supported_format(fmt)?; - validate_region(roi, full_dims)?; - if scale == Downscale::None { - return decode_region(bytes, pool, out, stride, fmt, roi); - } - - let target_dims = ( - full_dims.0.div_ceil(scale.denominator()), - full_dims.1.div_ceil(scale.denominator()), - ); - let scaled_roi = roi.scaled_covering(scale); - validate_buffer((scaled_roi.w, scaled_roi.h), out.len(), stride, fmt)?; - let settings = DecodeSettings { - target_resolution: Some(target_dims), - ..DecodeSettings::default() - }; - let image = backend::image(bytes, settings)?; - let image_dims = (image.width(), image.height()); - validate_region(scaled_roi, image_dims)?; - decode_image_region_into(&image, out, stride, fmt, scaled_roi)?; - - Ok(DecodeOutcome { - decoded: scaled_roi, - warnings: Vec::new(), - }) -} - -pub(crate) fn decode_region( - bytes: &[u8], - _pool: &mut J2kScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, -) -> Result { - validate_supported_format(fmt)?; - let image = backend::image(bytes, DecodeSettings::default())?; - let dims = (image.width(), image.height()); - validate_region(roi, dims)?; - validate_buffer((roi.w, roi.h), out.len(), stride, fmt)?; - decode_image_region_into(&image, out, stride, fmt, roi)?; - - Ok(DecodeOutcome { - decoded: roi, - warnings: Vec::new(), - }) -} - -pub(crate) fn decode_with_settings( - bytes: &[u8], - settings: DecodeSettings, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, -) -> Result { - validate_supported_format(fmt)?; - let image = backend::image(bytes, settings)?; - let dims = (image.width(), image.height()); - validate_buffer(dims, out.len(), stride, fmt)?; - decode_image_into(&image, out, stride, fmt)?; - Ok(DecodeOutcome { - decoded: Rect::full(dims), - warnings: Vec::new(), - }) -} - -fn decode_image_into( - image: &Image<'_>, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, -) -> Result<(), J2kError> { - let mut native_context = signinum_j2k_native::DecoderContext::default(); - decode_image_into_with_native_context(image, &mut native_context, out, stride, fmt) -} - -pub(crate) fn decode_image_into_with_native_context<'a>( - image: &Image<'a>, - native_context: &mut signinum_j2k_native::DecoderContext<'a>, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, -) -> Result<(), J2kError> { - let dims = (image.width(), image.height()); - match fmt { - PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => { - let decoded = image - .decode_with_context(native_context) - .map_err(|err| J2kError::Backend(err.to_string()))?; - write_u8_output( - image.color_space(), - image.has_alpha(), - dims, - &decoded.data, - out, - stride, - fmt, - ) - } - PixelFormat::Rgb16 | PixelFormat::Gray16 => { - let raw = image - .decode_native_with_context(native_context) - .map_err(|err| J2kError::Backend(err.to_string()))?; - write_u16_output( - image.color_space(), - image.has_alpha(), - &raw, - out, - stride, - fmt, - ) - } - PixelFormat::Rgba16 => unreachable!("validated above"), - _ => Err(Unsupported { - what: "pixel format is not yet supported by signinum-j2k", - } - .into()), - } -} - -fn decode_image_region_into( - image: &Image<'_>, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, -) -> Result<(), J2kError> { - let mut native_context = signinum_j2k_native::DecoderContext::default(); - decode_image_region_into_with_native_context(image, &mut native_context, out, stride, fmt, roi) -} - -pub(crate) fn decode_image_region_into_with_native_context<'a>( - image: &Image<'a>, - native_context: &mut signinum_j2k_native::DecoderContext<'a>, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, -) -> Result<(), J2kError> { - let dims = (roi.w, roi.h); - match fmt { - PixelFormat::Rgb8 | PixelFormat::Rgba8 | PixelFormat::Gray8 => { - let decoded = image - .decode_region_with_context((roi.x, roi.y, roi.w, roi.h), native_context) - .map_err(|err| J2kError::Backend(err.to_string()))?; - write_u8_output( - image.color_space(), - image.has_alpha(), - dims, - &decoded.data, - out, - stride, - fmt, - ) - } - PixelFormat::Rgb16 | PixelFormat::Gray16 => { - let raw = image - .decode_native_region_with_context((roi.x, roi.y, roi.w, roi.h), native_context) - .map_err(|err| J2kError::Backend(err.to_string()))?; - write_u16_output( - image.color_space(), - image.has_alpha(), - &raw, - out, - stride, - fmt, - ) - } - PixelFormat::Rgba16 => unreachable!("validated above"), - _ => Err(Unsupported { - what: "pixel format is not yet supported by signinum-j2k", - } - .into()), - } -} - -pub(crate) fn validate_supported_format(fmt: PixelFormat) -> Result<(), J2kError> { - if matches!(fmt, PixelFormat::Rgba16) { - return Err(Unsupported { - what: "Rgba16 output is not supported by signinum-j2k M1", - } - .into()); - } - Ok(()) -} - -pub(crate) fn validate_buffer( - dims: (u32, u32), - out_len: usize, - stride: usize, - fmt: PixelFormat, -) -> Result<(), J2kError> { - validate_strided_output_buffer(dims, out_len, stride, fmt).map_err(Into::into) -} - -pub(crate) fn validate_region(roi: Rect, dims: (u32, u32)) -> Result<(), J2kError> { - if roi.is_within(dims) { - return Ok(()); - } - Err(J2kError::InvalidRegion { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - image_w: dims.0, - image_h: dims.1, - }) -} - -fn write_u8_output( - color_space: &ColorSpace, - has_alpha: bool, - dims: (u32, u32), - decoded: &[u8], - out: &mut [u8], - stride: usize, - fmt: PixelFormat, -) -> Result<(), J2kError> { - let width = dims.0 as usize; - let height = dims.1 as usize; - match (color_space, has_alpha, fmt) { - (ColorSpace::RGB, false, PixelFormat::Rgb8) => { - copy_rows_exact(decoded, out, stride, width * 3, height); - Ok(()) - } - (ColorSpace::RGB, true, PixelFormat::Rgb8) => { - drop_alpha_u8(decoded, out, stride, width, height); - Ok(()) - } - (ColorSpace::RGB, false, PixelFormat::Rgba8) => { - add_opaque_alpha_u8(decoded, out, stride, width, height); - Ok(()) - } - (ColorSpace::RGB, true, PixelFormat::Rgba8) => { - copy_rows_exact(decoded, out, stride, width * 4, height); - Ok(()) - } - (ColorSpace::Gray, false, PixelFormat::Gray8) => { - copy_rows_exact(decoded, out, stride, width, height); - Ok(()) - } - _ => Err(Unsupported { - what: "backend color space cannot be mapped to requested 8-bit pixel format", - } - .into()), - } -} - -fn write_u16_output( - color_space: &ColorSpace, - has_alpha: bool, - raw: &RawBitmap, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, -) -> Result<(), J2kError> { - let width = raw.width as usize; - let height = raw.height as usize; - match (color_space, has_alpha, raw.num_components, fmt) { - (ColorSpace::RGB, false, 3, PixelFormat::Rgb16) => { - convert_or_copy_u16( - &raw.data, - raw.bytes_per_sample, - raw.bit_depth, - 3, - out, - stride, - (width, height), - ); - Ok(()) - } - (ColorSpace::Gray, false, 1, PixelFormat::Gray16) => { - convert_or_copy_u16( - &raw.data, - raw.bytes_per_sample, - raw.bit_depth, - 1, - out, - stride, - (width, height), - ); - Ok(()) - } - _ => Err(Unsupported { - what: "backend color space cannot be mapped to requested 16-bit pixel format", - } - .into()), - } -} - -fn copy_rows_exact(src: &[u8], out: &mut [u8], stride: usize, row_bytes: usize, height: usize) { - for (src_row, dst_row) in src - .chunks_exact(row_bytes) - .zip(out.chunks_exact_mut(stride)) - .take(height) - { - dst_row[..row_bytes].copy_from_slice(src_row); - } -} - -fn add_opaque_alpha_u8(src: &[u8], out: &mut [u8], stride: usize, width: usize, height: usize) { - let src_row_bytes = width * 3; - let dst_row_bytes = width * 4; - for (src_row, dst_row) in src - .chunks_exact(src_row_bytes) - .zip(out.chunks_exact_mut(stride)) - .take(height) - { - for (rgb, rgba) in src_row - .chunks_exact(3) - .zip(dst_row[..dst_row_bytes].chunks_exact_mut(4)) - { - rgba[..3].copy_from_slice(rgb); - rgba[3] = u8::MAX; - } - } -} - -fn drop_alpha_u8(src: &[u8], out: &mut [u8], stride: usize, width: usize, height: usize) { - let src_row_bytes = width * 4; - let dst_row_bytes = width * 3; - for (src_row, dst_row) in src - .chunks_exact(src_row_bytes) - .zip(out.chunks_exact_mut(stride)) - .take(height) - { - for (rgba, rgb) in src_row - .chunks_exact(4) - .zip(dst_row[..dst_row_bytes].chunks_exact_mut(3)) - { - rgb.copy_from_slice(&rgba[..3]); - } - } -} - -fn convert_or_copy_u16( - src: &[u8], - bytes_per_sample: u8, - bit_depth: u8, - channels: usize, - out: &mut [u8], - stride: usize, - dims: (usize, usize), -) { - let (width, height) = dims; - let dst_row_bytes = width * channels * 2; - let src_row_bytes = width * channels * usize::from(bytes_per_sample); - let max_value = ((1_u32 << bit_depth.min(16)) - 1).max(1); - for (src_row, dst_row) in src - .chunks_exact(src_row_bytes) - .zip(out.chunks_exact_mut(stride)) - .take(height) - { - let dst_row = &mut dst_row[..dst_row_bytes]; - if bytes_per_sample == 2 { - dst_row.copy_from_slice(src_row); - continue; - } - for (sample, dst_sample) in src_row.iter().zip(dst_row.chunks_exact_mut(2)) { - let widened = (u32::from(*sample) * u32::from(u16::MAX) + (max_value / 2)) / max_value; - dst_sample.copy_from_slice(&(widened as u16).to_le_bytes()); - } - } -} diff --git a/crates/signinum-j2k/src/encode.rs b/crates/signinum-j2k/src/encode.rs deleted file mode 100644 index 3e1189aa..00000000 --- a/crates/signinum-j2k/src/encode.rs +++ /dev/null @@ -1,584 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use alloc::vec::Vec; - -use signinum_core::{BackendKind, Unsupported}; -use signinum_j2k_native::{ - DecodeSettings, EncodeOptions, EncodeProgressionOrder, Image, J2kEncodeDispatchReport, - J2kEncodeStageAccelerator, -}; - -use crate::J2kError; - -/// Backend preference for JPEG 2000 lossless encoding. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum EncodeBackendPreference { - /// Pick the fastest safe backend exposed by the caller, falling back to CPU. - #[default] - Auto, - /// Require the pure Rust CPU encoder. - CpuOnly, - /// Prefer a device encoder, but fall back to CPU when unavailable. - PreferDevice, - /// Require a device encoder and fail if unavailable or unsupported. - RequireDevice, -} - -/// Supported JPEG 2000 progression orders for the lossless encode facade. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum J2kProgressionOrder { - /// Layer-resolution-component-position progression. - #[default] - Lrcp, - /// Resolution-position-component-layer progression. - Rpcl, -} - -/// Supported code-block coding modes for the lossless encode facade. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum J2kBlockCodingMode { - /// Classic JPEG 2000 Part 1 EBCOT block coding. - #[default] - Classic, - /// High-throughput JPEG 2000 Part 15 block coding. - HighThroughput, -} - -/// Reversible transform profile for lossless JPEG 2000 output. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum ReversibleTransform { - /// Reversible color transform with 5/3 wavelet transform. - #[default] - Rct53, - /// No color transform with 5/3 wavelet transform. - None53, -} - -/// Validation policy for the lossless encode facade. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum J2kEncodeValidation { - /// Decode the produced codestream with the native CPU decoder and compare - /// decoded samples before returning. - #[default] - CpuRoundTrip, - /// Skip facade validation because the caller performs equivalent external - /// validation, for example by decoding on a device backend. - External, -} - -/// Options controlling JPEG 2000 lossless encoding. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct J2kLosslessEncodeOptions { - pub backend: EncodeBackendPreference, - pub block_coding_mode: J2kBlockCodingMode, - pub progression: J2kProgressionOrder, - /// Optional explicit lossless decomposition level request. - /// - /// Requests are clamped to the geometry-safe maximum for the tile. - pub max_decomposition_levels: Option, - pub reversible_transform: ReversibleTransform, - pub validation: J2kEncodeValidation, -} - -impl Default for J2kLosslessEncodeOptions { - fn default() -> Self { - Self { - backend: EncodeBackendPreference::Auto, - block_coding_mode: J2kBlockCodingMode::Classic, - progression: J2kProgressionOrder::Lrcp, - max_decomposition_levels: None, - reversible_transform: ReversibleTransform::Rct53, - validation: J2kEncodeValidation::CpuRoundTrip, - } - } -} - -/// Borrowed interleaved samples and image geometry for lossless encoding. -#[derive(Debug, Clone, Copy)] -pub struct J2kLosslessSamples<'a> { - pub data: &'a [u8], - pub width: u32, - pub height: u32, - pub components: u8, - pub bit_depth: u8, - pub signed: bool, -} - -impl<'a> J2kLosslessSamples<'a> { - /// Validate and construct a sample descriptor. - pub fn new( - data: &'a [u8], - width: u32, - height: u32, - components: u8, - bit_depth: u8, - signed: bool, - ) -> Result { - if width == 0 || height == 0 { - return Err(J2kError::Backend("invalid dimensions".to_string())); - } - if !matches!(components, 1 | 3) { - return Err(J2kError::Unsupported(Unsupported { - what: "JPEG 2000 lossless encode supports only grayscale or RGB samples", - })); - } - if bit_depth == 0 || bit_depth > 16 { - return Err(J2kError::Unsupported(Unsupported { - what: "JPEG 2000 lossless encode supports 1-16 bits per sample", - })); - } - let bytes_per_sample = if bit_depth <= 8 { 1usize } else { 2usize }; - let expected = (width as usize) - .checked_mul(height as usize) - .and_then(|px| px.checked_mul(components as usize)) - .and_then(|samples| samples.checked_mul(bytes_per_sample)) - .ok_or(J2kError::DimensionOverflow { width, height })?; - if data.len() != expected { - return Err(J2kError::Backend(format!( - "pixel data too short: expected {expected} bytes, got {}", - data.len() - ))); - } - Ok(Self { - data, - width, - height, - components, - bit_depth, - signed, - }) - } -} - -/// Encoded JPEG 2000 lossless codestream and encode metadata. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct EncodedJ2k { - pub codestream: Vec, - pub backend: BackendKind, - pub width: u32, - pub height: u32, - pub components: u8, - pub bit_depth: u8, - pub signed: bool, -} - -/// Encode interleaved samples into a raw JPEG 2000 lossless codestream. -pub fn encode_j2k_lossless( - samples: J2kLosslessSamples<'_>, - options: &J2kLosslessEncodeOptions, -) -> Result { - let backend = resolve_encode_backend(options.backend)?; - let codestream = encode_cpu(samples, *options)?; - validate_lossless_roundtrip(samples, &codestream, options.validation)?; - Ok(EncodedJ2k { - codestream, - backend, - width: samples.width, - height: samples.height, - components: samples.components, - bit_depth: samples.bit_depth, - signed: samples.signed, - }) -} - -/// Encode interleaved samples with an optional device encode-stage accelerator. -/// -/// Accelerators return CPU fallback by reporting no dispatch. `Auto` and -/// `PreferDevice` accept that fallback; `RequireDevice` requires at least one -/// dispatch. Any accelerator error or codestream validation error is returned to -/// the caller. -pub fn encode_j2k_lossless_with_accelerator( - samples: J2kLosslessSamples<'_>, - options: &J2kLosslessEncodeOptions, - accelerated_backend: BackendKind, - accelerator: &mut impl J2kEncodeStageAccelerator, -) -> Result { - if options.backend == EncodeBackendPreference::CpuOnly { - return encode_j2k_lossless(samples, options); - } - - let before = accelerator.dispatch_report(); - let required_stages = required_encode_stages(samples, *options); - let codestream = encode_with_native_accelerator(samples, *options, accelerator)?; - let dispatch = accelerator.dispatch_report().saturating_delta(before); - validate_lossless_roundtrip(samples, &codestream, options.validation)?; - - let backend = resolve_accelerated_encode_backend( - options.backend, - accelerated_backend, - dispatch, - required_stages, - )?; - Ok(EncodedJ2k { - codestream, - backend, - width: samples.width, - height: samples.height, - components: samples.components, - bit_depth: samples.bit_depth, - signed: samples.signed, - }) -} - -fn resolve_encode_backend(preference: EncodeBackendPreference) -> Result { - match preference { - EncodeBackendPreference::Auto - | EncodeBackendPreference::CpuOnly - | EncodeBackendPreference::PreferDevice => Ok(BackendKind::Cpu), - EncodeBackendPreference::RequireDevice => Err(J2kError::Unsupported(Unsupported { - what: "device JPEG 2000 lossless encode backend is unavailable", - })), - } -} - -fn resolve_accelerated_encode_backend( - preference: EncodeBackendPreference, - accelerated_backend: BackendKind, - dispatch: J2kEncodeDispatchReport, - required_stages: RequiredEncodeStages, -) -> Result { - if required_stages.satisfied_by(dispatch) { - return Ok(accelerated_backend); - } - match preference { - EncodeBackendPreference::RequireDevice => Err(J2kError::Unsupported(Unsupported { - what: required_stages.missing_message(dispatch), - })), - EncodeBackendPreference::Auto - | EncodeBackendPreference::CpuOnly - | EncodeBackendPreference::PreferDevice => Ok(BackendKind::Cpu), - } -} - -fn encode_cpu( - samples: J2kLosslessSamples<'_>, - options: J2kLosslessEncodeOptions, -) -> Result, J2kError> { - let options = native_lossless_options(samples, options); - signinum_j2k_native::encode( - samples.data, - samples.width, - samples.height, - samples.components, - samples.bit_depth, - samples.signed, - &options, - ) - .map_err(|err| J2kError::Backend(format!("JPEG 2000 lossless encode failed: {err}"))) -} - -fn encode_with_native_accelerator( - samples: J2kLosslessSamples<'_>, - options: J2kLosslessEncodeOptions, - accelerator: &mut impl J2kEncodeStageAccelerator, -) -> Result, J2kError> { - let options = native_lossless_options(samples, options); - signinum_j2k_native::encode_with_accelerator( - samples.data, - samples.width, - samples.height, - samples.components, - samples.bit_depth, - samples.signed, - &options, - accelerator, - ) - .map_err(|err| J2kError::Backend(format!("JPEG 2000 lossless encode failed: {err}"))) -} - -fn native_lossless_options( - samples: J2kLosslessSamples<'_>, - options: J2kLosslessEncodeOptions, -) -> EncodeOptions { - let progression_order = native_progression_order(options.progression); - EncodeOptions { - reversible: true, - num_decomposition_levels: j2k_lossless_decomposition_levels_for_options(samples, options), - use_ht_block_coding: options.block_coding_mode == J2kBlockCodingMode::HighThroughput, - progression_order, - write_tlm: options.progression == J2kProgressionOrder::Rpcl, - use_mct: options.reversible_transform == ReversibleTransform::Rct53, - validate_high_throughput_codestream: false, - ..EncodeOptions::default() - } -} - -fn native_progression_order(progression: J2kProgressionOrder) -> EncodeProgressionOrder { - match progression { - J2kProgressionOrder::Lrcp => EncodeProgressionOrder::Lrcp, - J2kProgressionOrder::Rpcl => EncodeProgressionOrder::Rpcl, - } -} - -const MIN_LOSSLESS_DWT_DIMENSION: u32 = 64; - -/// Return the default lossless decomposition level policy used by the facade. -pub fn j2k_lossless_decomposition_levels(samples: J2kLosslessSamples<'_>) -> u8 { - j2k_lossless_decomposition_levels_for_progression(samples, J2kProgressionOrder::Lrcp) -} - -/// Return the default lossless decomposition level policy for a progression. -pub fn j2k_lossless_decomposition_levels_for_progression( - samples: J2kLosslessSamples<'_>, - progression: J2kProgressionOrder, -) -> u8 { - if progression == J2kProgressionOrder::Rpcl { - return j2k_rpcl_lossless_decomposition_levels(samples); - } - - if samples.width.min(samples.height) < MIN_LOSSLESS_DWT_DIMENSION { - return 0; - } - - 1 -} - -/// Return the effective lossless decomposition level policy for encode options. -pub fn j2k_lossless_decomposition_levels_for_options( - samples: J2kLosslessSamples<'_>, - options: J2kLosslessEncodeOptions, -) -> u8 { - let levels = j2k_lossless_decomposition_levels_for_progression(samples, options.progression); - options - .max_decomposition_levels - .map_or(levels, |requested| { - if samples.width.min(samples.height) < MIN_LOSSLESS_DWT_DIMENSION { - return 0; - } - requested.min(max_decomposition_levels(samples.width, samples.height)) - }) -} - -fn j2k_rpcl_lossless_decomposition_levels(samples: J2kLosslessSamples<'_>) -> u8 { - let mut levels = 0u8; - let mut width = samples.width; - let mut height = samples.height; - let max_levels = max_decomposition_levels(samples.width, samples.height); - - while width.min(height) > MIN_LOSSLESS_DWT_DIMENSION && levels < max_levels { - width = width.div_ceil(2); - height = height.div_ceil(2); - levels += 1; - } - - levels -} - -fn max_decomposition_levels(width: u32, height: u32) -> u8 { - let min_dim = width.min(height); - if min_dim <= 1 { - return 0; - } - min_dim.ilog2() as u8 -} - -#[derive(Debug, Clone, Copy)] -struct RequiredEncodeStages { - bits: u8, -} - -impl RequiredEncodeStages { - const FORWARD_RCT: u8 = 1 << 0; - const FORWARD_DWT53: u8 = 1 << 1; - const TIER1_CODE_BLOCK: u8 = 1 << 2; - const HT_CODE_BLOCK: u8 = 1 << 3; - const PACKETIZATION: u8 = 1 << 4; - - fn satisfied_by(self, dispatch: J2kEncodeDispatchReport) -> bool { - self.missing_stage(dispatch).is_none() - } - - fn missing_message(self, dispatch: J2kEncodeDispatchReport) -> &'static str { - match self.missing_stage(dispatch) { - Some("forward_rct") => { - "requested JPEG 2000 lossless device encode backend did not dispatch forward_rct" - } - Some("forward_dwt53") => { - "requested JPEG 2000 lossless device encode backend did not dispatch forward_dwt53" - } - Some("tier1_code_block") => { - "requested JPEG 2000 lossless device encode backend did not dispatch tier1_code_block" - } - Some("ht_code_block") => { - "requested JPEG 2000 lossless device encode backend did not dispatch ht_code_block" - } - Some("packetization") => { - "requested JPEG 2000 lossless device encode backend did not dispatch packetization" - } - _ => "requested JPEG 2000 lossless device encode backend did not dispatch", - } - } - - fn missing_stage(self, dispatch: J2kEncodeDispatchReport) -> Option<&'static str> { - if self.contains(Self::FORWARD_RCT) && dispatch.forward_rct == 0 { - return Some("forward_rct"); - } - if self.contains(Self::FORWARD_DWT53) && dispatch.forward_dwt53 == 0 { - return Some("forward_dwt53"); - } - if self.contains(Self::TIER1_CODE_BLOCK) && dispatch.tier1_code_block == 0 { - return Some("tier1_code_block"); - } - if self.contains(Self::HT_CODE_BLOCK) && dispatch.ht_code_block == 0 { - return Some("ht_code_block"); - } - if self.contains(Self::PACKETIZATION) && dispatch.packetization == 0 { - return Some("packetization"); - } - None - } - - fn contains(self, stage: u8) -> bool { - self.bits & stage != 0 - } -} - -fn required_encode_stages( - samples: J2kLosslessSamples<'_>, - options: J2kLosslessEncodeOptions, -) -> RequiredEncodeStages { - let decomposition_levels = j2k_lossless_decomposition_levels_for_options(samples, options); - let high_throughput = options.block_coding_mode == J2kBlockCodingMode::HighThroughput; - - let mut bits = RequiredEncodeStages::PACKETIZATION; - if samples.components >= 3 && options.reversible_transform == ReversibleTransform::Rct53 { - bits |= RequiredEncodeStages::FORWARD_RCT; - } - if decomposition_levels > 0 { - bits |= RequiredEncodeStages::FORWARD_DWT53; - } - if high_throughput { - bits |= RequiredEncodeStages::HT_CODE_BLOCK; - } else { - bits |= RequiredEncodeStages::TIER1_CODE_BLOCK; - } - - RequiredEncodeStages { bits } -} - -fn validate_lossless_roundtrip( - samples: J2kLosslessSamples<'_>, - codestream: &[u8], - validation: J2kEncodeValidation, -) -> Result<(), J2kError> { - if validation == J2kEncodeValidation::External { - return Ok(()); - } - - let decoded = Image::new(codestream, &DecodeSettings::default()) - .map_err(|err| J2kError::Backend(format!("encoded codestream validation failed: {err}")))? - .decode_native() - .map_err(|err| J2kError::Backend(format!("encoded codestream validation failed: {err}")))?; - - if decoded.width != samples.width - || decoded.height != samples.height - || decoded.num_components != samples.components - || decoded.bit_depth != samples.bit_depth - { - return Err(J2kError::Backend( - "JPEG 2000 lossless encode failed round-trip geometry validation".to_string(), - )); - } - if decoded.data != samples.data { - let mismatch = decoded - .data - .iter() - .zip(samples.data.iter()) - .position(|(actual, expected)| actual != expected); - return Err(J2kError::Backend(match mismatch { - Some(index) => format!( - "JPEG 2000 lossless encode failed round-trip validation at byte {index}: expected {}, got {}", - samples.data[index], decoded.data[index] - ), - None => format!( - "JPEG 2000 lossless encode failed round-trip validation: expected {} bytes, got {} bytes", - samples.data.len(), - decoded.data.len() - ), - })); - } - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::{ - encode_j2k_lossless, j2k_lossless_decomposition_levels_for_options, - native_lossless_options, J2kBlockCodingMode, J2kEncodeValidation, J2kLosslessEncodeOptions, - J2kLosslessSamples, J2kProgressionOrder, ReversibleTransform, - }; - - fn cod_mct(codestream: &[u8]) -> u8 { - let cod_offset = codestream - .windows(2) - .position(|window| window == [0xFF, 0x52]) - .expect("COD marker"); - codestream[cod_offset + 8] - } - - #[test] - fn lossless_encode_can_disable_component_transform() { - let pixels: Vec = (0..4 * 4 * 3) - .map(|value| ((value * 17) & 0xFF) as u8) - .collect(); - let samples = J2kLosslessSamples::new(&pixels, 4, 4, 3, 8, false).unwrap(); - let encoded = encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions { - block_coding_mode: J2kBlockCodingMode::Classic, - progression: J2kProgressionOrder::Lrcp, - max_decomposition_levels: Some(0), - reversible_transform: ReversibleTransform::None53, - validation: J2kEncodeValidation::CpuRoundTrip, - ..J2kLosslessEncodeOptions::default() - }, - ) - .unwrap(); - - assert_eq!(cod_mct(&encoded.codestream), 0); - } - - #[test] - fn explicit_decomposition_levels_override_default_lrcp_policy() { - let pixels = vec![0; 128 * 128]; - let samples = J2kLosslessSamples::new(&pixels, 128, 128, 1, 8, false).unwrap(); - - let levels = j2k_lossless_decomposition_levels_for_options( - samples, - J2kLosslessEncodeOptions { - block_coding_mode: J2kBlockCodingMode::Classic, - progression: J2kProgressionOrder::Lrcp, - max_decomposition_levels: Some(5), - ..J2kLosslessEncodeOptions::default() - }, - ); - - assert_eq!(levels, 5); - } - - #[test] - fn facade_native_options_skip_internal_ht_validation_for_external_validation() { - let pixels = vec![0; 64 * 64]; - let samples = J2kLosslessSamples::new(&pixels, 64, 64, 1, 8, false).unwrap(); - - let external = native_lossless_options( - samples, - J2kLosslessEncodeOptions { - block_coding_mode: J2kBlockCodingMode::HighThroughput, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - }, - ); - let roundtrip = native_lossless_options( - samples, - J2kLosslessEncodeOptions { - block_coding_mode: J2kBlockCodingMode::HighThroughput, - validation: J2kEncodeValidation::CpuRoundTrip, - ..J2kLosslessEncodeOptions::default() - }, - ); - - assert!(!external.validate_high_throughput_codestream); - assert!(!roundtrip.validate_high_throughput_codestream); - } -} diff --git a/crates/signinum-j2k/src/error.rs b/crates/signinum-j2k/src/error.rs deleted file mode 100644 index 8180294f..00000000 --- a/crates/signinum-j2k/src/error.rs +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use signinum_core::{BufferError, CodecError, InputError, NotImplemented, Unsupported}; - -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -#[non_exhaustive] -pub enum J2kError { - #[error(transparent)] - Buffer(#[from] BufferError), - - #[error(transparent)] - Input(#[from] InputError), - - #[error(transparent)] - NotImplemented(#[from] NotImplemented), - - #[error(transparent)] - Unsupported(#[from] Unsupported), - - #[error("backend decode failed: {0}")] - Backend(String), - - #[error("region ({x},{y} {w}x{h}) is outside image bounds {image_w}x{image_h}")] - InvalidRegion { - x: u32, - y: u32, - w: u32, - h: u32, - image_w: u32, - image_h: u32, - }, - - #[error("invalid JP2 box at offset {offset}: {what}")] - InvalidBox { offset: usize, what: &'static str }, - - #[error("missing required JP2 box {box_type}")] - MissingRequiredBox { box_type: &'static str }, - - #[error("invalid codestream marker FF{marker:02X} at offset {offset}")] - InvalidMarker { offset: usize, marker: u8 }, - - #[error("missing required codestream marker {marker}")] - MissingRequiredMarker { marker: &'static str }, - - #[error("invalid SIZ segment: {what}")] - InvalidSiz { what: &'static str }, - - #[error("invalid COD segment: {what}")] - InvalidCod { what: &'static str }, - - #[error("dimension overflow: {width}x{height}")] - DimensionOverflow { width: u32, height: u32 }, -} - -impl CodecError for J2kError { - fn is_truncated(&self) -> bool { - matches!( - self, - Self::Input(InputError::TooShort { .. } | InputError::TruncatedAt { .. }) - ) - } - - fn is_not_implemented(&self) -> bool { - matches!(self, Self::NotImplemented(_)) - } - - fn is_unsupported(&self) -> bool { - matches!(self, Self::Unsupported(_)) - } - - fn is_buffer_error(&self) -> bool { - matches!(self, Self::Buffer(_)) - } -} diff --git a/crates/signinum-j2k/src/lib.rs b/crates/signinum-j2k/src/lib.rs deleted file mode 100644 index 986c91c9..00000000 --- a/crates/signinum-j2k/src/lib.rs +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! JPEG 2000 inspect support for signinum. - -extern crate alloc; - -mod backend; -mod batch; -mod decode; -mod encode; - -pub mod context; -pub use context::J2kContext; - -pub mod error; -pub use error::J2kError; - -pub mod scratch; -pub use scratch::J2kScratchPool; - -pub mod adapter; - -pub mod view; -pub use view::{J2kCodec, J2kDecoder, J2kView}; - -pub use batch::{ - decode_tile_into_in_context, decode_tile_region_scaled_into_in_context, decode_tiles_into, - decode_tiles_region_scaled_into, TileBatchError, TileBatchOptions, TileDecodeJob, - TileRegionScaledDecodeJob, -}; - -pub use signinum_j2k_native::CpuDecodeParallelism; - -pub use encode::{ - encode_j2k_lossless, encode_j2k_lossless_with_accelerator, j2k_lossless_decomposition_levels, - j2k_lossless_decomposition_levels_for_options, - j2k_lossless_decomposition_levels_for_progression, EncodeBackendPreference, EncodedJ2k, - J2kBlockCodingMode, J2kEncodeValidation, J2kLosslessEncodeOptions, J2kLosslessSamples, - J2kProgressionOrder, ReversibleTransform, -}; - -#[doc(hidden)] -pub use signinum_j2k_native::{ - EncodedHtJ2kCodeBlock, EncodedJ2kCodeBlock, J2kEncodeDispatchReport, J2kEncodeStageAccelerator, - J2kForwardDwt53Job, J2kForwardDwt53Level, J2kForwardDwt53Output, J2kForwardRctJob, - J2kHtCodeBlockEncodeJob, J2kPacketizationBlockCodingMode, J2kPacketizationCodeBlock, - J2kPacketizationEncodeJob, J2kPacketizationProgressionOrder, J2kPacketizationResolution, - J2kPacketizationSubband, J2kTier1CodeBlockEncodeJob, -}; - -pub use signinum_core::{ - BackendKind, BackendRequest, BufferError, CodecError, CompressedPayloadKind, - CompressedTransferSyntax, DecodeOutcome, DecodeRowsError, DecoderContext, Downscale, - ImageCodec, ImageDecode, ImageDecodeRows, PassthroughCandidate, PassthroughDecision, - PassthroughRejectReason, PassthroughRequirements, PixelFormat, Rect, RowSink, TileBatchDecode, -}; - -pub(crate) mod parse; diff --git a/crates/signinum-j2k/src/parse/codestream.rs b/crates/signinum-j2k/src/parse/codestream.rs deleted file mode 100644 index 6d03bddc..00000000 --- a/crates/signinum-j2k/src/parse/codestream.rs +++ /dev/null @@ -1,242 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use super::{ParsedCod, ParsedSiz}; -use crate::J2kError; -use signinum_core::{InputError, TileLayout}; - -const MARKER_SOC: u8 = 0x4F; -const MARKER_CAP: u8 = 0x50; -const MARKER_SIZ: u8 = 0x51; -const MARKER_COD: u8 = 0x52; -const MARKER_SOT: u8 = 0x90; -const MARKER_SOD: u8 = 0x93; -const MARKER_EOC: u8 = 0xD9; - -#[derive(Debug, Clone, Copy)] -pub(crate) struct CodestreamInfo { - pub(crate) siz: ParsedSiz, - pub(crate) cod: ParsedCod, -} - -pub(crate) fn looks_like_codestream(input: &[u8]) -> bool { - input.len() >= 2 && input[0] == 0xFF && input[1] == MARKER_SOC -} - -pub(crate) fn parse_codestream(input: &[u8]) -> Result { - if input.len() < 2 { - return Err(InputError::TooShort { - need: 2, - have: input.len(), - } - .into()); - } - if !looks_like_codestream(input) { - return Err(J2kError::InvalidMarker { - offset: 0, - marker: input[1], - }); - } - - let mut offset = 2usize; - let mut siz = None; - let mut cod = None; - let mut high_throughput_cap = false; - let mut terminated = false; - - while offset < input.len() { - let marker = read_marker(input, &mut offset)?; - match marker { - MARKER_SOT | MARKER_SOD | MARKER_EOC => { - terminated = true; - break; - } - MARKER_SIZ => { - let payload = read_segment_payload(input, &mut offset, "SIZ")?; - siz = Some(parse_siz(payload)?); - } - MARKER_COD => { - let payload = read_segment_payload(input, &mut offset, "COD")?; - cod = Some(parse_cod(payload)?); - } - MARKER_CAP => { - let _ = read_segment_payload(input, &mut offset, "CAP")?; - high_throughput_cap = true; - } - _ => { - let _ = read_segment_payload(input, &mut offset, "segment")?; - } - } - } - - if !terminated { - return Err(InputError::TruncatedAt { - offset, - segment: "main header terminator", - } - .into()); - } - - Ok(CodestreamInfo { - siz: siz.ok_or(J2kError::MissingRequiredMarker { marker: "SIZ" })?, - cod: cod - .ok_or(J2kError::MissingRequiredMarker { marker: "COD" })? - .with_high_throughput_cap(high_throughput_cap), - }) -} - -fn read_marker(input: &[u8], offset: &mut usize) -> Result { - if *offset + 2 > input.len() { - return Err(InputError::TruncatedAt { - offset: *offset, - segment: "marker", - } - .into()); - } - if input[*offset] != 0xFF { - return Err(J2kError::InvalidMarker { - offset: *offset, - marker: input[*offset], - }); - } - let marker = input[*offset + 1]; - *offset += 2; - Ok(marker) -} - -fn read_segment_payload<'a>( - input: &'a [u8], - offset: &mut usize, - segment: &'static str, -) -> Result<&'a [u8], J2kError> { - if *offset + 2 > input.len() { - return Err(InputError::TruncatedAt { - offset: *offset, - segment, - } - .into()); - } - let length = u16::from_be_bytes([input[*offset], input[*offset + 1]]) as usize; - if length < 2 { - return Err(J2kError::InvalidBox { - offset: *offset, - what: "segment length smaller than header", - }); - } - let start = *offset + 2; - let end = *offset + length; - if end > input.len() { - return Err(InputError::TruncatedAt { - offset: *offset, - segment, - } - .into()); - } - *offset = end; - Ok(&input[start..end]) -} - -#[allow(clippy::similar_names)] -fn parse_siz(payload: &[u8]) -> Result { - if payload.len() < 36 { - return Err(J2kError::InvalidSiz { - what: "payload shorter than fixed SIZ header", - }); - } - let x_size = read_u32(payload, 2); - let y_size = read_u32(payload, 6); - let x_origin = read_u32(payload, 10); - let y_origin = read_u32(payload, 14); - let tile_width = read_u32(payload, 18); - let tile_height = read_u32(payload, 22); - let tile_x_origin = read_u32(payload, 26); - let tile_y_origin = read_u32(payload, 30); - let component_count = read_u16(payload, 34); - - let component_bytes = usize::from(component_count) * 3; - if payload.len() < 36 + component_bytes { - return Err(J2kError::InvalidSiz { - what: "component descriptors truncated", - }); - } - if component_count == 0 { - return Err(J2kError::InvalidSiz { - what: "component count must be non-zero", - }); - } - if component_count > u16::from(u8::MAX) { - return Err(J2kError::Unsupported(signinum_core::Unsupported { - what: "component count > 255", - })); - } - if x_size <= x_origin || y_size <= y_origin { - return Err(J2kError::InvalidSiz { - what: "image origin must be smaller than image size", - }); - } - if tile_width == 0 || tile_height == 0 { - return Err(J2kError::InvalidSiz { - what: "tile size must be non-zero", - }); - } - if tile_x_origin > x_size || tile_y_origin > y_size { - return Err(J2kError::InvalidSiz { - what: "tile origin must be within image bounds", - }); - } - - let width = x_size - x_origin; - let height = y_size - y_origin; - let tiles_x = (x_size - tile_x_origin).div_ceil(tile_width); - let tiles_y = (y_size - tile_y_origin).div_ceil(tile_height); - let mut bit_depth = 0u8; - for idx in 0..usize::from(component_count) { - let ssiz = payload[36 + idx * 3]; - bit_depth = bit_depth.max((ssiz & 0x7F) + 1); - } - - Ok(ParsedSiz { - dimensions: (width, height), - components: component_count as u8, - bit_depth, - tile_layout: TileLayout { - tile_width, - tile_height, - tiles_x, - tiles_y, - }, - }) -} - -fn parse_cod(payload: &[u8]) -> Result { - if payload.len() < 10 { - return Err(J2kError::InvalidCod { - what: "payload shorter than fixed COD header", - }); - } - Ok(ParsedCod { - resolution_levels: payload[5].saturating_add(1), - has_mct: payload[4] != 0, - reversible: payload[9] == 1, - high_throughput: payload[8] & 0x40 != 0, - }) -} - -impl ParsedCod { - const fn with_high_throughput_cap(mut self, high_throughput_cap: bool) -> Self { - self.high_throughput |= high_throughput_cap; - self - } -} - -fn read_u16(bytes: &[u8], offset: usize) -> u16 { - u16::from_be_bytes([bytes[offset], bytes[offset + 1]]) -} - -fn read_u32(bytes: &[u8], offset: usize) -> u32 { - u32::from_be_bytes([ - bytes[offset], - bytes[offset + 1], - bytes[offset + 2], - bytes[offset + 3], - ]) -} diff --git a/crates/signinum-j2k/tests/batch.rs b/crates/signinum-j2k/tests/batch.rs deleted file mode 100644 index 525c781b..00000000 --- a/crates/signinum-j2k/tests/batch.rs +++ /dev/null @@ -1,228 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use signinum_j2k::{ - decode_tiles_into, decode_tiles_region_scaled_into, Downscale, J2kDecoder, PixelFormat, Rect, - TileBatchOptions, TileDecodeJob, TileRegionScaledDecodeJob, -}; -use signinum_j2k_native::{encode, EncodeOptions}; -use std::num::NonZeroUsize; - -fn encode_codestream( - pixels: &[u8], - width: u32, - height: u32, - components: u8, - bit_depth: u8, -) -> Vec { - let options = EncodeOptions { - reversible: true, - num_decomposition_levels: 1, - ..EncodeOptions::default() - }; - encode( - pixels, width, height, components, bit_depth, false, &options, - ) - .expect("encode") -} - -fn rgb_fixture() -> Vec { - let pixels = (0_u8..48).collect::>(); - encode_codestream(&pixels, 4, 4, 3, 8) -} - -fn decode_rgb8_reference(bytes: &[u8]) -> (Vec, usize) { - let mut decoder = J2kDecoder::new(bytes).expect("decoder"); - let (width, height) = decoder.info().dimensions; - let stride = width as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let mut out = vec![0_u8; stride * height as usize]; - decoder - .decode_into(&mut out, stride, PixelFormat::Rgb8) - .expect("decode reference"); - (out, stride) -} - -#[test] -fn production_batch_decode_empty_input_succeeds() { - let mut jobs: Vec> = Vec::new(); - - let outcomes = decode_tiles_into(&mut jobs, PixelFormat::Rgb8, TileBatchOptions::default()) - .expect("empty batch succeeds"); - - assert!(outcomes.is_empty()); -} - -#[test] -fn production_batch_decode_worker_one_matches_single_tile_decode() { - let codestream = rgb_fixture(); - let (expected, stride) = decode_rgb8_reference(&codestream); - let mut actual = vec![0_u8; expected.len()]; - let options = TileBatchOptions { - workers: NonZeroUsize::new(1), - }; - - let outcomes = { - let mut jobs = vec![TileDecodeJob { - input: &codestream, - out: actual.as_mut_slice(), - stride, - }]; - decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect("batch decode") - }; - - assert_eq!(outcomes.len(), 1); - assert_eq!(actual, expected); -} - -#[test] -fn production_batch_decode_parallel_preserves_order_and_output() { - const JOBS: usize = 16; - let codestream = rgb_fixture(); - let (expected, stride) = decode_rgb8_reference(&codestream); - let mut outputs = (0..JOBS) - .map(|_| vec![0_u8; expected.len()]) - .collect::>(); - let options = TileBatchOptions { - workers: NonZeroUsize::new(4), - }; - - let outcomes = { - let mut jobs = outputs - .iter_mut() - .map(|out| TileDecodeJob { - input: codestream.as_slice(), - out: out.as_mut_slice(), - stride, - }) - .collect::>(); - decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect("batch decode") - }; - - assert_eq!(outcomes.len(), JOBS); - for (index, out) in outputs.iter().enumerate() { - assert_eq!(out, &expected, "tile {index} output diverged"); - } -} - -#[test] -fn production_batch_decode_matches_repeated_single_tile_decodes() { - let inputs = [ - rgb_fixture(), - encode_codestream(&(48_u8..96).collect::>(), 4, 4, 3, 8), - encode_codestream(&(96_u8..144).collect::>(), 4, 4, 3, 8), - ]; - let expected = inputs - .iter() - .map(|input| decode_rgb8_reference(input).0) - .collect::>(); - let stride = 4 * PixelFormat::Rgb8.bytes_per_pixel(); - let mut outputs = expected - .iter() - .map(|tile| vec![0_u8; tile.len()]) - .collect::>(); - let options = TileBatchOptions { - workers: NonZeroUsize::new(2), - }; - - let outcomes = { - let mut jobs = inputs - .iter() - .zip(outputs.iter_mut()) - .map(|(input, out)| TileDecodeJob { - input: input.as_slice(), - out: out.as_mut_slice(), - stride, - }) - .collect::>(); - decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect("batch decode") - }; - - assert_eq!(outcomes.len(), inputs.len()); - assert_eq!(outputs, expected); -} - -#[test] -fn production_batch_region_scaled_decode_parallel_preserves_order_and_output() { - const JOBS: usize = 12; - let codestream = rgb_fixture(); - let roi = Rect { - x: 1, - y: 0, - w: 2, - h: 3, - }; - let scale = Downscale::Half; - let scaled_roi = roi.scaled_covering(scale); - let stride = scaled_roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); - - let mut decoder = J2kDecoder::new(&codestream).expect("decoder"); - let mut pool = signinum_j2k::J2kScratchPool::new(); - let mut expected = vec![0_u8; stride * scaled_roi.h as usize]; - decoder - .decode_region_scaled_into( - &mut pool, - &mut expected, - stride, - PixelFormat::Rgb8, - roi, - scale, - ) - .expect("decode reference"); - - let mut outputs = (0..JOBS) - .map(|_| vec![0_u8; expected.len()]) - .collect::>(); - let options = TileBatchOptions { - workers: NonZeroUsize::new(3), - }; - - let outcomes = { - let mut jobs = outputs - .iter_mut() - .map(|out| TileRegionScaledDecodeJob { - input: codestream.as_slice(), - out: out.as_mut_slice(), - stride, - roi, - scale, - }) - .collect::>(); - decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8, options) - .expect("batch decode") - }; - - assert_eq!(outcomes.len(), JOBS); - for outcome in &outcomes { - assert_eq!(outcome.decoded, scaled_roi); - } - for (index, out) in outputs.iter().enumerate() { - assert_eq!(out, &expected, "tile {index} output diverged"); - } -} - -#[test] -fn production_batch_decode_reports_first_failing_tile_index() { - let codestream = rgb_fixture(); - let (expected, stride) = decode_rgb8_reference(&codestream); - let mut outputs = (0..3) - .map(|_| vec![0_u8; expected.len()]) - .collect::>(); - let options = TileBatchOptions { - workers: NonZeroUsize::new(2), - }; - - let err = { - let inputs: [&[u8]; 3] = [codestream.as_slice(), b"not j2k", codestream.as_slice()]; - let mut jobs = inputs - .into_iter() - .zip(outputs.iter_mut()) - .map(|(input, out)| TileDecodeJob { - input, - out: out.as_mut_slice(), - stride, - }) - .collect::>(); - decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect_err("bad tile fails") - }; - - assert_eq!(err.index, 1); -} diff --git a/crates/signinum-jpeg-cuda/README.md b/crates/signinum-jpeg-cuda/README.md deleted file mode 100644 index a3621c44..00000000 --- a/crates/signinum-jpeg-cuda/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# signinum-jpeg-cuda - -CUDA-facing device-output adapter for `signinum-jpeg`. - -Install this crate when a pipeline needs JPEG output in CUDA device memory: - -```sh -cargo add signinum-jpeg-cuda --features cuda-runtime -``` - -`BackendRequest::Cpu` returns host-backed CPU surfaces. `BackendRequest::Auto` -uses CPU for scalar calls, but the full-tile batch helper may choose CUDA when -the runtime and nvJPEG are available. `BackendRequest::Cuda` requires the -`cuda-runtime` feature and an available CUDA driver. For full-frame RGB8 JPEG -decode, the adapter uses NVIDIA nvJPEG when `libnvjpeg` is available and -returns a CUDA-backed `DeviceSurface` without first decoding to a host RGB -buffer. Region, scaled, non-RGB8, and nvJPEG-unsupported requests fall back to -CPU decode plus CUDA device-memory upload. - -Use `cargo bench -p signinum-jpeg-cuda --bench device_decode --features -cuda-runtime` on an NVIDIA host to compare CPU decode, nvJPEG surface -production through a reused `CudaSession`, and decode-plus-download timing. -Set `SIGNINUM_GPU_BENCH_DIM=4096` for the generated large-tile benchmark, or -set `SIGNINUM_CUDA_BENCH_JPEG` to a large WSI-shaped JPEG tile. The same bench -also compares a CPU batch loop, the low-level nvJPEG batch call, and the -public `TileBatchDecodeManyDevice` adapter path; tune it with -`SIGNINUM_GPU_BENCH_BATCH` and `SIGNINUM_GPU_BENCH_BATCH_DIM`. - -The stable CPU decode API lives in `signinum-jpeg`. diff --git a/crates/signinum-jpeg-cuda/benches/device_decode.rs b/crates/signinum-jpeg-cuda/benches/device_decode.rs deleted file mode 100644 index 57e5e5d8..00000000 --- a/crates/signinum-jpeg-cuda/benches/device_decode.rs +++ /dev/null @@ -1,282 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use criterion::{criterion_group, criterion_main, Criterion}; -use jpeg_encoder::{ColorType, Encoder}; -use signinum_core::{ - BackendRequest, DeviceSubmission, DeviceSurface, ImageDecodeDevice, ImageDecodeSubmit, - PixelFormat, -}; -#[cfg(feature = "cuda-runtime")] -use signinum_core::{DecoderContext, TileBatchDecodeManyDevice}; -#[cfg(feature = "cuda-runtime")] -use signinum_cuda_runtime::CudaContext; -use signinum_jpeg::Decoder as CpuDecoder; -#[cfg(feature = "cuda-runtime")] -use signinum_jpeg_cuda::Codec as CudaCodec; -use signinum_jpeg_cuda::{CudaSession, Decoder as CudaDecoder}; - -const DEFAULT_JPEG: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); -const DEFAULT_GENERATED_DIM: u16 = 2048; -#[cfg(feature = "cuda-runtime")] -const DEFAULT_BATCH_DIM: u16 = 1024; -#[cfg(feature = "cuda-runtime")] -const DEFAULT_BATCH_SIZE: usize = 64; - -fn bench_device_decode(c: &mut Criterion) { - let input = bench_input(); - let mut group = c.benchmark_group("jpeg_cuda_device_decode"); - - group.bench_function("cpu_decode_rgb8", |b| { - b.iter(|| { - let decoder = CpuDecoder::new(&input).expect("cpu decoder"); - decoder.decode(PixelFormat::Rgb8).expect("cpu decode") - }); - }); - - match cuda_probe(&input) { - Some(probe) => { - let label = if probe.used_hardware_decode { - "cuda_nvjpeg_rgb8_surface" - } else { - "cuda_upload_fallback_rgb8_surface" - }; - group.bench_function(label, |b| { - let mut session = CudaSession::default(); - b.iter(|| { - let mut decoder = CudaDecoder::new(&input).expect("cuda decoder"); - as ImageDecodeSubmit<'_>>::submit_to_device( - &mut decoder, - &mut session, - PixelFormat::Rgb8, - BackendRequest::Cuda, - ) - .expect("cuda submit") - .wait() - .expect("cuda decode") - }); - }); - - let label = if probe.used_hardware_decode { - "cuda_nvjpeg_rgb8_download" - } else { - "cuda_upload_fallback_rgb8_download" - }; - group.bench_function(label, |b| { - let mut session = CudaSession::default(); - b.iter(|| { - let mut decoder = CudaDecoder::new(&input).expect("cuda decoder"); - let surface = as ImageDecodeSubmit<'_>>::submit_to_device( - &mut decoder, - &mut session, - PixelFormat::Rgb8, - BackendRequest::Cuda, - ) - .expect("cuda submit") - .wait() - .expect("cuda decode"); - let mut out = vec![0u8; surface.byte_len()]; - surface - .download_into(&mut out, surface.pitch_bytes()) - .expect("cuda download"); - out - }); - }); - } - None if std::env::var_os("SIGNINUM_REQUIRE_CUDA_BENCH").is_some() => { - panic!("SIGNINUM_REQUIRE_CUDA_BENCH is set but CUDA decode is unavailable") - } - None => { - eprintln!("skipping CUDA decode benches: CUDA runtime is unavailable"); - } - } - - group.finish(); - - bench_batch_decode(c); -} - -fn bench_input() -> Vec { - let path = std::env::var_os("SIGNINUM_CUDA_BENCH_JPEG") - .or_else(|| std::env::var_os("SIGNINUM_GPU_BENCH_JPEG")); - match path { - Some(path) => std::fs::read(&path).unwrap_or_else(|error| { - panic!( - "failed to read SIGNINUM_CUDA_BENCH_JPEG={}: {error}", - path.to_string_lossy() - ) - }), - None if std::env::var_os("SIGNINUM_GPU_BENCH_SMALL_FIXTURE").is_some() => { - DEFAULT_JPEG.to_vec() - } - None => generated_jpeg(generated_dim()), - } -} - -fn generated_jpeg(dim: u16) -> Vec { - let rgb = signinum_test_support::gpu_bench_rgb8(u32::from(dim), u32::from(dim)); - - let mut jpeg = Vec::new(); - Encoder::new(&mut jpeg, 90) - .encode(&rgb, dim, dim, ColorType::Rgb) - .expect("encode generated benchmark JPEG"); - jpeg -} - -fn generated_dim() -> u16 { - let Some(value) = std::env::var_os("SIGNINUM_GPU_BENCH_DIM") else { - return DEFAULT_GENERATED_DIM; - }; - let value = value - .to_string_lossy() - .parse::() - .expect("SIGNINUM_GPU_BENCH_DIM must be a u16"); - assert!( - (256..=8192).contains(&value), - "SIGNINUM_GPU_BENCH_DIM must be between 256 and 8192" - ); - value -} - -#[cfg(feature = "cuda-runtime")] -fn bench_batch_decode(c: &mut Criterion) { - let dim = batch_dim(); - let input = generated_jpeg(dim); - let dimensions = (u32::from(dim), u32::from(dim)); - let batch_size = batch_size(); - let batch_inputs = vec![(input.as_slice(), dimensions); batch_size]; - let batch_refs = vec![input.as_slice(); batch_size]; - - let mut group = c.benchmark_group("jpeg_cuda_batch_decode"); - group.sample_size(10); - - group.bench_function(format!("cpu_decode_rgb8_batch{batch_size}"), |b| { - b.iter(|| { - let mut total = 0usize; - for _ in 0..batch_size { - let decoder = CpuDecoder::new(&input).expect("cpu decoder"); - let decoded_rgb = decoder.decode(PixelFormat::Rgb8).expect("cpu decode"); - total = total.saturating_add(decoded_rgb.0.len()); - std::hint::black_box(decoded_rgb); - } - total - }); - }); - - let context = match CudaContext::system_default() { - Ok(context) => context, - Err(error) if std::env::var_os("SIGNINUM_REQUIRE_CUDA_BENCH").is_some() => { - panic!( - "SIGNINUM_REQUIRE_CUDA_BENCH is set but CUDA batch decode is unavailable: {error}" - ) - } - Err(error) => { - eprintln!("skipping CUDA batch decode bench: {error}"); - group.finish(); - return; - } - }; - if let Err(error) = context.decode_jpeg_rgb8_batch_with_nvjpeg(&batch_inputs) { - assert!( - std::env::var_os("SIGNINUM_REQUIRE_CUDA_BENCH").is_none(), - "SIGNINUM_REQUIRE_CUDA_BENCH is set but nvJPEG batch decode is unavailable: {error}" - ); - eprintln!("skipping CUDA batch decode bench: {error}"); - group.finish(); - return; - } - - group.bench_function( - format!("cuda_nvjpeg_runtime_rgb8_batch{batch_size}_surfaces"), - |b| { - b.iter(|| { - let outputs = context - .decode_jpeg_rgb8_batch_with_nvjpeg(&batch_inputs) - .expect("cuda batch decode"); - std::hint::black_box(outputs) - }); - }, - ); - - group.bench_function( - format!("cuda_adapter_rgb8_batch{batch_size}_surfaces"), - |b| { - let mut ctx = DecoderContext::::new(); - let mut pool = signinum_jpeg::ScratchPool::new(); - b.iter(|| { - let outputs = CudaCodec::decode_tiles_to_device( - &mut ctx, - &mut pool, - &batch_refs, - PixelFormat::Rgb8, - BackendRequest::Cuda, - ) - .expect("cuda adapter batch decode"); - std::hint::black_box(outputs) - }); - }, - ); - - group.finish(); -} - -#[cfg(not(feature = "cuda-runtime"))] -fn bench_batch_decode(_c: &mut Criterion) {} - -#[cfg(feature = "cuda-runtime")] -fn batch_size() -> usize { - let Some(value) = std::env::var_os("SIGNINUM_GPU_BENCH_BATCH") else { - return DEFAULT_BATCH_SIZE; - }; - let value = value - .to_string_lossy() - .parse::() - .expect("SIGNINUM_GPU_BENCH_BATCH must be a usize"); - assert!( - (1..=256).contains(&value), - "SIGNINUM_GPU_BENCH_BATCH must be between 1 and 256" - ); - value -} - -#[cfg(feature = "cuda-runtime")] -fn batch_dim() -> u16 { - let Some(value) = std::env::var_os("SIGNINUM_GPU_BENCH_BATCH_DIM") else { - return DEFAULT_BATCH_DIM; - }; - let value = value - .to_string_lossy() - .parse::() - .expect("SIGNINUM_GPU_BENCH_BATCH_DIM must be a u16"); - assert!( - (128..=4096).contains(&value), - "SIGNINUM_GPU_BENCH_BATCH_DIM must be between 128 and 4096" - ); - value -} - -struct CudaProbe { - used_hardware_decode: bool, -} - -fn cuda_probe(input: &[u8]) -> Option { - let mut decoder = CudaDecoder::new(input).expect("cuda decoder"); - let surface = match decoder.decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) { - Ok(surface) => surface, - Err(error) => { - eprintln!("skipping CUDA decode benches: {error}"); - return None; - } - }; - let stats = surface.cuda_surface().expect("cuda surface").stats(); - if std::env::var_os("SIGNINUM_REQUIRE_CUDA_JPEG_HARDWARE_DECODE").is_some() - && !stats.used_hardware_decode() - { - panic!("SIGNINUM_REQUIRE_CUDA_JPEG_HARDWARE_DECODE is set but nvJPEG was not used"); - } - Some(CudaProbe { - used_hardware_decode: stats.used_hardware_decode(), - }) -} - -criterion_group!(benches, bench_device_decode); -criterion_main!(benches); diff --git a/crates/signinum-jpeg-cuda/src/codec.rs b/crates/signinum-jpeg-cuda/src/codec.rs deleted file mode 100644 index 77b0631f..00000000 --- a/crates/signinum-jpeg-cuda/src/codec.rs +++ /dev/null @@ -1,401 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#[cfg(feature = "cuda-runtime")] -use signinum_core::BackendKind; -use signinum_core::{ - BackendRequest, Downscale, ImageCodec, PixelFormat, ReadySubmission, Rect, - TileBatchDecodeDevice, TileBatchDecodeManyDevice, TileBatchDecodeSubmit, -}; -#[cfg(feature = "cuda-runtime")] -use signinum_cuda_runtime::CudaError; -use signinum_jpeg::{ - decode_tile_into_in_context, decode_tile_region_into_in_context, - decode_tile_region_scaled_into_in_context, decode_tile_scaled_into_in_context, - Decoder as CpuDecoder, DecoderContext as CpuDecoderContext, ScratchPool as CpuScratchPool, - Warning as CpuWarning, -}; - -#[cfg(feature = "cuda-runtime")] -use crate::runtime::cuda_error; -use crate::runtime::{validate_surface_request, wrap_surface}; -#[cfg(feature = "cuda-runtime")] -use crate::surface::{CudaSurfaceStats, Storage}; -use crate::{profile, CudaSession, Error, Surface}; - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct Codec; - -impl ImageCodec for Codec { - type Error = Error; - type Warning = CpuWarning; - type Pool = CpuScratchPool; -} - -impl Codec { - fn decode_tile_to_surface_impl( - ctx: &mut signinum_core::DecoderContext, - session: &mut CudaSession, - pool: &mut CpuScratchPool, - input: &[u8], - fmt: PixelFormat, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - let dims = CpuDecoder::inspect(input)?.dimensions; - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut out = vec![0u8; stride * dims.1 as usize]; - decode_tile_into_in_context(input, ctx.codec_mut(), pool, &mut out, stride, fmt)?; - wrap_surface(out, dims, fmt, backend, session) - } - - fn decode_tile_region_to_surface_impl( - ctx: &mut signinum_core::DecoderContext, - session: &mut CudaSession, - pool: &mut CpuScratchPool, - input: &[u8], - fmt: PixelFormat, - roi: Rect, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - let dims = (roi.w, roi.h); - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut out = vec![0u8; stride * dims.1 as usize]; - decode_tile_region_into_in_context( - input, - ctx.codec_mut(), - pool, - &mut out, - stride, - fmt, - roi.into(), - )?; - wrap_surface(out, dims, fmt, backend, session) - } - - fn decode_tile_scaled_to_surface_impl( - ctx: &mut signinum_core::DecoderContext, - session: &mut CudaSession, - pool: &mut CpuScratchPool, - input: &[u8], - fmt: PixelFormat, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - let dims = ( - CpuDecoder::inspect(input)? - .dimensions - .0 - .div_ceil(scale.denominator()), - CpuDecoder::inspect(input)? - .dimensions - .1 - .div_ceil(scale.denominator()), - ); - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut out = vec![0u8; stride * dims.1 as usize]; - decode_tile_scaled_into_in_context( - input, - ctx.codec_mut(), - pool, - &mut out, - stride, - fmt, - scale, - )?; - wrap_surface(out, dims, fmt, backend, session) - } - - #[allow(clippy::too_many_arguments)] - fn decode_tile_region_scaled_to_surface_impl( - ctx: &mut signinum_core::DecoderContext, - session: &mut CudaSession, - pool: &mut CpuScratchPool, - input: &[u8], - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - let dims = { - let scaled = roi.scaled_covering(scale); - (scaled.w, scaled.h) - }; - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut out = vec![0u8; stride * dims.1 as usize]; - decode_tile_region_scaled_into_in_context( - input, - ctx.codec_mut(), - pool, - &mut out, - stride, - fmt, - roi.into(), - scale, - )?; - wrap_surface(out, dims, fmt, backend, session) - } -} - -impl TileBatchDecodeSubmit for Codec { - type Context = CpuDecoderContext; - type Session = CudaSession; - type DeviceSurface = Surface; - type SubmittedSurface = ReadySubmission; - - fn submit_tile_to_device( - ctx: &mut signinum_core::DecoderContext, - session: &mut Self::Session, - pool: &mut Self::Pool, - input: &[u8], - fmt: PixelFormat, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - Self::decode_tile_to_surface_impl(ctx, session, pool, input, fmt, backend), - )) - } - - fn submit_tile_region_to_device( - ctx: &mut signinum_core::DecoderContext, - session: &mut Self::Session, - pool: &mut Self::Pool, - input: &[u8], - fmt: PixelFormat, - roi: Rect, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - Self::decode_tile_region_to_surface_impl(ctx, session, pool, input, fmt, roi, backend), - )) - } - - fn submit_tile_scaled_to_device( - ctx: &mut signinum_core::DecoderContext, - session: &mut Self::Session, - pool: &mut Self::Pool, - input: &[u8], - fmt: PixelFormat, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - Self::decode_tile_scaled_to_surface_impl( - ctx, session, pool, input, fmt, scale, backend, - ), - )) - } - - fn submit_tile_region_scaled_to_device( - ctx: &mut signinum_core::DecoderContext, - session: &mut Self::Session, - pool: &mut Self::Pool, - input: &[u8], - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - validate_surface_request(backend)?; - session.record_submit(); - Ok(ReadySubmission::from_result( - Self::decode_tile_region_scaled_to_surface_impl( - ctx, session, pool, input, fmt, roi, scale, backend, - ), - )) - } -} - -impl TileBatchDecodeDevice for Codec { - type Context = CpuDecoderContext; - type DeviceSurface = Surface; -} - -impl TileBatchDecodeManyDevice for Codec { - type Context = CpuDecoderContext; - type DeviceSurface = Surface; - - fn decode_tiles_to_device( - ctx: &mut signinum_core::DecoderContext, - pool: &mut Self::Pool, - inputs: &[&[u8]], - fmt: PixelFormat, - backend: BackendRequest, - ) -> Result, Self::Error> { - validate_surface_request(backend)?; - if inputs.is_empty() { - return Ok(Vec::new()); - } - - let mut session = CudaSession::default(); - if let Some(surfaces) = try_decode_tiles_nvjpeg_batch(inputs, fmt, backend, &mut session)? { - return Ok(surfaces); - } - - inputs - .iter() - .map(|input| { - Self::decode_tile_to_surface_impl(ctx, &mut session, pool, input, fmt, backend) - }) - .collect() - } -} - -#[cfg(feature = "cuda-runtime")] -fn try_decode_tiles_nvjpeg_batch( - inputs: &[&[u8]], - fmt: PixelFormat, - backend: BackendRequest, - session: &mut CudaSession, -) -> Result>, Error> { - if fmt != PixelFormat::Rgb8 || !matches!(backend, BackendRequest::Auto | BackendRequest::Cuda) { - if profile::gpu_route_profile_enabled() { - let request_s = format!("{backend:?}"); - let fmt_s = format!("{fmt:?}"); - let tiles_s = inputs.len().to_string(); - profile::emit_gpu_route_profile( - "jpeg", - "gpu_route", - "cuda", - &[ - ("op", "batch_full"), - ("request", request_s.as_str()), - ("fmt", fmt_s.as_str()), - ("tiles", tiles_s.as_str()), - ("decision", "nvjpeg_batch_ineligible"), - ], - ); - } - return Ok(None); - } - - let mut batch_inputs = Vec::with_capacity(inputs.len()); - for input in inputs { - let dimensions = CpuDecoder::inspect(input)?.dimensions; - batch_inputs.push((*input, dimensions)); - } - - let context = match session.cuda_context() { - Ok(context) => context, - Err(_) if backend == BackendRequest::Auto => { - if profile::gpu_route_profile_enabled() { - let tiles_s = inputs.len().to_string(); - profile::emit_gpu_route_profile( - "jpeg", - "gpu_route", - "cuda", - &[ - ("op", "batch_full"), - ("request", "Auto"), - ("fmt", "Rgb8"), - ("tiles", tiles_s.as_str()), - ("decision", "nvjpeg_batch_fallback"), - ("reason", "cuda_unavailable"), - ], - ); - } - return Ok(None); - } - Err(error) => return Err(error), - }; - - match context.decode_jpeg_rgb8_batch_with_nvjpeg(&batch_inputs) { - Ok(outputs) => { - if profile::gpu_route_profile_enabled() { - let tiles_s = outputs.len().to_string(); - profile::emit_gpu_route_profile( - "jpeg", - "gpu_route", - "cuda", - &[ - ("op", "batch_full"), - ("request", "AutoOrCuda"), - ("fmt", "Rgb8"), - ("tiles", tiles_s.as_str()), - ("decision", "nvjpeg_batch"), - ], - ); - } - let mut surfaces = Vec::with_capacity(outputs.len()); - for (output, (_, dimensions)) in outputs.into_iter().zip(batch_inputs) { - let pitch_bytes = dimensions.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let (buffer, stats) = output.into_parts(); - surfaces.push(Surface { - backend: BackendKind::Cuda, - dimensions, - fmt: PixelFormat::Rgb8, - pitch_bytes, - stats: CudaSurfaceStats { - kernel_dispatches: stats.kernel_dispatches(), - copy_kernel_dispatches: stats.copy_kernel_dispatches(), - decode_kernel_dispatches: stats.decode_kernel_dispatches(), - hardware_decode: stats.used_hardware_decode(), - }, - storage: Storage::Cuda(buffer), - }); - } - Ok(Some(surfaces)) - } - Err( - CudaError::NvjpegUnavailable { .. } - | CudaError::Nvjpeg { .. } - | CudaError::NvjpegDimensions { .. }, - ) => { - if profile::gpu_route_profile_enabled() { - let tiles_s = inputs.len().to_string(); - profile::emit_gpu_route_profile( - "jpeg", - "gpu_route", - "cuda", - &[ - ("op", "batch_full"), - ("request", "AutoOrCuda"), - ("fmt", "Rgb8"), - ("tiles", tiles_s.as_str()), - ("decision", "nvjpeg_batch_fallback"), - ("reason", "nvjpeg_unavailable_or_rejected"), - ], - ); - } - Ok(None) - } - Err(error) => Err(cuda_error(error)), - } -} - -#[cfg(not(feature = "cuda-runtime"))] -#[allow(clippy::unnecessary_wraps)] -fn try_decode_tiles_nvjpeg_batch( - inputs: &[&[u8]], - fmt: PixelFormat, - backend: BackendRequest, - _session: &mut CudaSession, -) -> Result>, Error> { - if profile::gpu_route_profile_enabled() { - let request_s = format!("{backend:?}"); - let fmt_s = format!("{fmt:?}"); - let tiles_s = inputs.len().to_string(); - profile::emit_gpu_route_profile( - "jpeg", - "gpu_route", - "cuda", - &[ - ("op", "batch_full"), - ("request", request_s.as_str()), - ("fmt", fmt_s.as_str()), - ("tiles", tiles_s.as_str()), - ("decision", "nvjpeg_batch_fallback"), - ("reason", "cuda_runtime_feature_disabled"), - ], - ); - } - Ok(None) -} diff --git a/crates/signinum-jpeg-cuda/src/error.rs b/crates/signinum-jpeg-cuda/src/error.rs deleted file mode 100644 index 8a6b2762..00000000 --- a/crates/signinum-jpeg-cuda/src/error.rs +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use signinum_core::{BackendRequest, BufferError, CodecError}; -use signinum_jpeg::JpegError; - -#[derive(Debug, thiserror::Error)] -pub enum Error { - #[error(transparent)] - Decode(#[from] JpegError), - #[error(transparent)] - Buffer(#[from] BufferError), - #[error("backend request {request:?} is not supported by signinum-jpeg-cuda")] - UnsupportedBackend { request: BackendRequest }, - #[error("CUDA is unavailable on this host")] - CudaUnavailable, - #[cfg(feature = "cuda-runtime")] - #[error("CUDA runtime error: {message}")] - CudaRuntime { message: String }, -} - -impl CodecError for Error { - fn is_truncated(&self) -> bool { - matches!(self, Self::Decode(inner) if inner.is_truncated()) - } - - fn is_not_implemented(&self) -> bool { - matches!(self, Self::Decode(inner) if inner.is_not_implemented()) - } - - fn is_unsupported(&self) -> bool { - matches!( - self, - Self::UnsupportedBackend { .. } | Self::CudaUnavailable - ) || matches!(self, Self::Decode(inner) if inner.is_unsupported()) - } - - fn is_buffer_error(&self) -> bool { - matches!(self, Self::Buffer(_)) - || matches!(self, Self::Decode(inner) if inner.is_buffer_error()) - } -} diff --git a/crates/signinum-jpeg-cuda/src/lib.rs b/crates/signinum-jpeg-cuda/src/lib.rs deleted file mode 100644 index 9d11d25f..00000000 --- a/crates/signinum-jpeg-cuda/src/lib.rs +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! CUDA-facing device-output adapter for `signinum-jpeg`. -//! -//! This crate intentionally exposes the same backend-selection surface as the -//! Metal adapter. CPU requests return host-backed surfaces. Scalar auto -//! requests stay on CPU, while full-tile batch auto requests may use nvJPEG -//! when the CUDA runtime and library are available. Explicit CUDA requests -//! return CUDA-backed surfaces or a clear unavailable error. - -#![warn(unreachable_pub)] - -mod codec; -mod decoder; -mod error; -mod profile; -mod runtime; -mod session; -mod surface; - -pub use codec::Codec; -pub use decoder::Decoder; -pub use error::Error; -pub use session::CudaSession; -pub use signinum_jpeg::{DecoderContext, ScratchPool}; -pub use surface::{CudaSurface, CudaSurfaceStats, Surface}; diff --git a/crates/signinum-jpeg-cuda/src/profile.rs b/crates/signinum-jpeg-cuda/src/profile.rs deleted file mode 100644 index f4000840..00000000 --- a/crates/signinum-jpeg-cuda/src/profile.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -pub(crate) fn gpu_route_profile_enabled() -> bool { - signinum_profile::gpu_route_profile_enabled() -} - -pub(crate) fn emit_gpu_route_profile(codec: &str, op: &str, path: &str, fields: &[(K, V)]) -where - K: AsRef, - V: AsRef, -{ - debug_assert_eq!(op, "gpu_route"); - signinum_profile::emit_gpu_route_profile(codec, path, fields); -} diff --git a/crates/signinum-jpeg-cuda/src/session.rs b/crates/signinum-jpeg-cuda/src/session.rs deleted file mode 100644 index c36baefb..00000000 --- a/crates/signinum-jpeg-cuda/src/session.rs +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#[cfg(feature = "cuda-runtime")] -use signinum_cuda_runtime::CudaContext; - -#[cfg(feature = "cuda-runtime")] -use crate::runtime::cuda_error; -#[cfg(feature = "cuda-runtime")] -use crate::Error; - -#[derive(Clone, Default)] -pub struct CudaSession { - submissions: u64, - #[cfg(feature = "cuda-runtime")] - context: Option, -} - -impl CudaSession { - pub fn submissions(&self) -> u64 { - self.submissions - } - - #[cfg(feature = "cuda-runtime")] - pub fn is_runtime_initialized(&self) -> bool { - self.context.is_some() - } - - pub(crate) fn record_submit(&mut self) { - self.submissions = self.submissions.saturating_add(1); - } - - #[cfg(feature = "cuda-runtime")] - pub(crate) fn cuda_context(&mut self) -> Result { - if self.context.is_none() { - self.context = Some(CudaContext::system_default().map_err(cuda_error)?); - } - self.context.clone().ok_or(Error::CudaUnavailable) - } -} - -impl std::fmt::Debug for CudaSession { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut debug = f.debug_struct("CudaSession"); - debug.field("submissions", &self.submissions); - #[cfg(feature = "cuda-runtime")] - debug.field("runtime_initialized", &self.is_runtime_initialized()); - debug.finish_non_exhaustive() - } -} diff --git a/crates/signinum-jpeg-cuda/tests/host_surface.rs b/crates/signinum-jpeg-cuda/tests/host_surface.rs deleted file mode 100644 index 9d63db2b..00000000 --- a/crates/signinum-jpeg-cuda/tests/host_surface.rs +++ /dev/null @@ -1,527 +0,0 @@ -use signinum_core::{ - BackendRequest, CodecError, DecoderContext, DeviceSubmission, DeviceSurface, Downscale, - ImageDecode, ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, Rect, TileBatchDecodeDevice, - TileBatchDecodeManyDevice, -}; -use signinum_jpeg_cuda::{Codec, CudaSession, Decoder, Error}; - -const BASELINE_420: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); -const NVJPEG_RGB8_MAX_CHANNEL_DELTA: u8 = 16; - -#[test] -fn auto_falls_back_to_cpu_surface() { - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - let surface = decoder - .decode_to_device(PixelFormat::Rgb8, BackendRequest::Auto) - .expect("surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cpu); - assert!(surface.as_host_bytes().is_some()); -} - -#[test] -fn explicit_cuda_request_returns_cuda_surface_or_clear_unavailable_error() { - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - match decoder.decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) { - Ok(surface) => { - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cuda); - assert_eq!(surface.as_host_bytes(), None); - #[cfg(feature = "cuda-runtime")] - assert_ne!( - surface.cuda_surface().expect("cuda surface").device_ptr(), - 0 - ); - } - Err(error) => assert!(error.is_unsupported()), - } -} - -#[test] -fn explicit_cuda_request_validates_decode_before_upload() { - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - - let error = decoder - .decode_to_device(PixelFormat::Rgba16, BackendRequest::Cuda) - .expect_err("unsupported decode"); - assert!(error.is_unsupported()); - assert!(!matches!(error, Error::CudaUnavailable)); -} - -#[test] -fn explicit_cuda_request_returns_cuda_surface_when_runtime_required() { - if !runtime_required() { - return; - } - - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - let surface = decoder - .decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) - .expect("cuda surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cuda); - assert_eq!(surface.as_host_bytes(), None); - assert_cuda_surface(&surface); - assert_eq!(surface.dimensions(), (16, 16)); - - let mut downloaded = vec![0u8; surface.byte_len()]; - surface - .download_into(&mut downloaded, surface.pitch_bytes()) - .expect("download cuda surface"); - - let (expected, _) = signinum_jpeg::Decoder::new(BASELINE_420) - .expect("host decoder") - .decode(PixelFormat::Rgb8) - .expect("host decode"); - assert_surface_bytes_match_or_are_close(&surface, &downloaded, &expected); -} - -#[test] -fn explicit_cuda_region_scaled_surface_matches_host_when_runtime_required() { - if !runtime_required() { - return; - } - - let roi = Rect { - x: 4, - y: 4, - w: 10, - h: 10, - }; - let scale = Downscale::Quarter; - let scaled = roi.scaled_covering(scale); - - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - let surface = decoder - .decode_region_scaled_to_device(PixelFormat::Rgb8, roi, scale, BackendRequest::Cuda) - .expect("cuda region+scaled surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cuda); - assert_eq!(surface.as_host_bytes(), None); - assert_cuda_surface(&surface); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - - let mut downloaded = vec![0u8; surface.byte_len()]; - surface - .download_into(&mut downloaded, surface.pitch_bytes()) - .expect("download cuda surface"); - - let (expected, _) = signinum_jpeg::Decoder::new(BASELINE_420) - .expect("host decoder") - .decode_region_scaled( - PixelFormat::Rgb8, - signinum_jpeg::Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }, - scale, - ) - .expect("host decode"); - assert_eq!(downloaded, expected); -} - -#[test] -fn explicit_cuda_download_respects_padded_stride_when_runtime_required() { - if !runtime_required() { - return; - } - - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - let surface = decoder - .decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) - .expect("cuda surface"); - assert_cuda_surface(&surface); - let row_bytes = surface.pitch_bytes(); - let stride = row_bytes + 5; - let mut downloaded = vec![0xCD; stride * surface.dimensions().1 as usize]; - surface - .download_into(&mut downloaded, stride) - .expect("download cuda surface"); - - let (expected, _) = signinum_jpeg::Decoder::new(BASELINE_420) - .expect("host decoder") - .decode(PixelFormat::Rgb8) - .expect("host decode"); - for (row, expected_row) in expected.chunks(row_bytes).enumerate() { - let start = row * stride; - assert_surface_bytes_match_or_are_close( - &surface, - &downloaded[start..start + row_bytes], - expected_row, - ); - assert_eq!(&downloaded[start + row_bytes..start + stride], &[0xCD; 5]); - } -} - -#[test] -fn explicit_cuda_full_frame_uses_hardware_decode_when_required() { - if !hardware_decode_required() { - return; - } - - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - let surface = decoder - .decode_to_device(PixelFormat::Rgb8, BackendRequest::Cuda) - .expect("cuda surface"); - let cuda = surface.cuda_surface().expect("cuda surface"); - let stats = cuda.stats(); - assert!( - stats.used_hardware_decode(), - "explicit full-frame RGB8 CUDA decode must use a CUDA JPEG decode path when required" - ); - assert!( - stats.decode_kernel_dispatches() > 0, - "hardware decode path must report decode kernel dispatches" - ); - assert_eq!( - stats.copy_kernel_dispatches(), - 0, - "hardware decode path should not be reported as the CPU decode plus copy fallback" - ); -} - -fn runtime_required() -> bool { - std::env::var_os("SIGNINUM_REQUIRE_CUDA_RUNTIME").is_some() -} - -fn hardware_decode_required() -> bool { - std::env::var_os("SIGNINUM_REQUIRE_CUDA_JPEG_HARDWARE_DECODE").is_some() -} - -fn assert_cuda_surface(surface: &signinum_jpeg_cuda::Surface) { - let cuda = surface.cuda_surface().expect("cuda surface"); - assert_ne!(cuda.device_ptr(), 0); - assert!(cuda.stats().kernel_dispatches() > 0); -} - -fn assert_surface_bytes_match_or_are_close( - surface: &signinum_jpeg_cuda::Surface, - actual: &[u8], - expected: &[u8], -) { - assert_eq!(actual.len(), expected.len()); - let stats = surface.cuda_surface().expect("cuda surface").stats(); - if !stats.used_hardware_decode() { - assert_eq!(actual, expected); - return; - } - - let max_delta = actual - .iter() - .zip(expected) - .map(|(actual, expected)| actual.abs_diff(*expected)) - .max() - .unwrap_or(0); - assert!( - max_delta <= NVJPEG_RGB8_MAX_CHANNEL_DELTA, - "nvJPEG decode differed from the CPU reference by max channel delta {max_delta}" - ); -} - -#[test] -fn submit_to_device_auto_falls_back_to_cpu_surface() { - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - let mut session = CudaSession::default(); - let surface = as ImageDecodeSubmit<'_>>::submit_to_device( - &mut decoder, - &mut session, - PixelFormat::Rgb8, - BackendRequest::Auto, - ) - .expect("submission") - .wait() - .expect("surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cpu); - assert!(surface.as_host_bytes().is_some()); - assert!(session.submissions() >= 1); -} - -#[cfg(feature = "cuda-runtime")] -#[test] -fn submit_to_device_auto_does_not_initialize_cuda_runtime() { - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - let mut session = CudaSession::default(); - let surface = as ImageDecodeSubmit<'_>>::submit_to_device( - &mut decoder, - &mut session, - PixelFormat::Rgb8, - BackendRequest::Auto, - ) - .expect("submission") - .wait() - .expect("surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cpu); - assert_eq!(session.submissions(), 1); - assert!(!session.is_runtime_initialized()); -} - -#[cfg(feature = "cuda-runtime")] -#[test] -fn explicit_cuda_submissions_reuse_session_runtime_when_required() { - if !runtime_required() { - return; - } - - let mut session = CudaSession::default(); - assert!(!session.is_runtime_initialized()); - - let mut first = Decoder::new(BASELINE_420).expect("decoder"); - let first_surface = as ImageDecodeSubmit<'_>>::submit_to_device( - &mut first, - &mut session, - PixelFormat::Rgb8, - BackendRequest::Cuda, - ) - .expect("first submission") - .wait() - .expect("first surface"); - assert_eq!( - first_surface.backend_kind(), - signinum_core::BackendKind::Cuda - ); - assert_cuda_surface(&first_surface); - assert!(session.is_runtime_initialized()); - - let mut second = Decoder::new(BASELINE_420).expect("decoder"); - let second_surface = as ImageDecodeSubmit<'_>>::submit_to_device( - &mut second, - &mut session, - PixelFormat::Rgb8, - BackendRequest::Cuda, - ) - .expect("second submission") - .wait() - .expect("second surface"); - assert_eq!( - second_surface.backend_kind(), - signinum_core::BackendKind::Cuda - ); - assert_cuda_surface(&second_surface); - assert_eq!(session.submissions(), 2); - assert!(session.is_runtime_initialized()); -} - -#[test] -fn auto_region_scaled_surface_matches_host_decode() { - let roi = Rect { - x: 4, - y: 4, - w: 10, - h: 10, - }; - let scale = Downscale::Quarter; - let scaled = roi.scaled_covering(scale); - - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - let surface = decoder - .decode_region_scaled_to_device(PixelFormat::Rgb8, roi, scale, BackendRequest::Auto) - .expect("surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cpu); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - - let mut host_decoder = Decoder::new(BASELINE_420).expect("host decoder"); - let mut host = vec![0u8; scaled.w as usize * scaled.h as usize * 3]; - host_decoder - .decode_region_scaled_into( - &mut signinum_jpeg::ScratchPool::new(), - &mut host, - scaled.w as usize * 3, - PixelFormat::Rgb8, - roi, - scale, - ) - .expect("host decode"); - assert_eq!(surface.as_host_bytes(), Some(host.as_slice())); -} - -#[test] -fn tile_batch_region_scaled_auto_surface_matches_host_decode() { - let roi = Rect { - x: 4, - y: 4, - w: 10, - h: 10, - }; - let scale = Downscale::Quarter; - let scaled = roi.scaled_covering(scale); - let mut ctx = DecoderContext::::new(); - let mut pool = signinum_jpeg::ScratchPool::new(); - let surface = Codec::decode_tile_region_scaled_to_device( - &mut ctx, - &mut pool, - BASELINE_420, - PixelFormat::Rgb8, - roi, - scale, - BackendRequest::Auto, - ) - .expect("surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cpu); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - - let (expected, _) = signinum_jpeg::Decoder::new(BASELINE_420) - .expect("host decoder") - .decode_region_scaled( - PixelFormat::Rgb8, - signinum_jpeg::Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }, - scale, - ) - .expect("host decode"); - assert_eq!(surface.as_host_bytes(), Some(expected.as_slice())); -} - -#[test] -fn tile_batch_region_scaled_cuda_surface_matches_host_when_runtime_required() { - if !runtime_required() { - return; - } - - let roi = Rect { - x: 4, - y: 4, - w: 10, - h: 10, - }; - let scale = Downscale::Quarter; - let scaled = roi.scaled_covering(scale); - let mut ctx = DecoderContext::::new(); - let mut pool = signinum_jpeg::ScratchPool::new(); - let surface = Codec::decode_tile_region_scaled_to_device( - &mut ctx, - &mut pool, - BASELINE_420, - PixelFormat::Rgb8, - roi, - scale, - BackendRequest::Cuda, - ) - .expect("cuda tile batch surface"); - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cuda); - assert_eq!(surface.as_host_bytes(), None); - assert_cuda_surface(&surface); - assert_eq!(surface.dimensions(), (scaled.w, scaled.h)); - - let mut downloaded = vec![0u8; surface.byte_len()]; - surface - .download_into(&mut downloaded, surface.pitch_bytes()) - .expect("download cuda surface"); - - let (expected, _) = signinum_jpeg::Decoder::new(BASELINE_420) - .expect("host decoder") - .decode_region_scaled( - PixelFormat::Rgb8, - signinum_jpeg::Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }, - scale, - ) - .expect("host decode"); - assert_eq!(downloaded, expected); -} - -#[test] -fn decode_tiles_to_device_auto_preserves_order_and_matches_host_bytes() { - let mut ctx = DecoderContext::::new(); - let mut pool = signinum_jpeg::ScratchPool::new(); - let inputs = [BASELINE_420, BASELINE_420]; - - let surfaces = Codec::decode_tiles_to_device( - &mut ctx, - &mut pool, - &inputs, - PixelFormat::Rgb8, - BackendRequest::Auto, - ) - .expect("batch surfaces"); - - assert_eq!(surfaces.len(), inputs.len()); - let (expected, _) = signinum_jpeg::Decoder::new(BASELINE_420) - .expect("host decoder") - .decode(PixelFormat::Rgb8) - .expect("host decode"); - for surface in surfaces { - assert_eq!(surface.dimensions(), (16, 16)); - match surface.backend_kind() { - signinum_core::BackendKind::Cpu => { - assert_eq!(surface.as_host_bytes(), Some(expected.as_slice())); - } - signinum_core::BackendKind::Cuda => { - let mut downloaded = vec![0u8; surface.byte_len()]; - surface - .download_into(&mut downloaded, surface.pitch_bytes()) - .expect("download cuda surface"); - assert_surface_bytes_match_or_are_close(&surface, &downloaded, &expected); - } - signinum_core::BackendKind::Metal => panic!("JPEG CUDA batch returned Metal surface"), - } - } -} - -#[test] -fn decode_tiles_to_device_explicit_cuda_returns_cuda_surfaces_or_clear_unavailable_error() { - let mut ctx = DecoderContext::::new(); - let mut pool = signinum_jpeg::ScratchPool::new(); - let inputs = [BASELINE_420, BASELINE_420]; - - match Codec::decode_tiles_to_device( - &mut ctx, - &mut pool, - &inputs, - PixelFormat::Rgb8, - BackendRequest::Cuda, - ) { - Ok(surfaces) => { - assert_eq!(surfaces.len(), inputs.len()); - for surface in surfaces { - assert_eq!(surface.backend_kind(), signinum_core::BackendKind::Cuda); - assert_eq!(surface.as_host_bytes(), None); - assert_cuda_surface(&surface); - } - } - Err(error) => assert!(error.is_unsupported()), - } -} - -#[test] -fn decode_tiles_to_device_explicit_cuda_uses_hardware_decode_when_required() { - if !hardware_decode_required() { - return; - } - - let mut ctx = DecoderContext::::new(); - let mut pool = signinum_jpeg::ScratchPool::new(); - let inputs = [BASELINE_420, BASELINE_420]; - - let surfaces = Codec::decode_tiles_to_device( - &mut ctx, - &mut pool, - &inputs, - PixelFormat::Rgb8, - BackendRequest::Cuda, - ) - .expect("cuda batch surfaces"); - - assert_eq!(surfaces.len(), inputs.len()); - for surface in surfaces { - let stats = surface.cuda_surface().expect("cuda surface").stats(); - assert!( - stats.used_hardware_decode(), - "explicit full-tile RGB8 CUDA batch decode must use nvJPEG when required" - ); - assert!( - stats.decode_kernel_dispatches() > 0, - "hardware batch decode path must report decode dispatches" - ); - assert_eq!( - stats.copy_kernel_dispatches(), - 0, - "hardware batch decode path should not be reported as CPU decode plus copy" - ); - } -} diff --git a/crates/signinum-jpeg-metal/README.md b/crates/signinum-jpeg-metal/README.md deleted file mode 100644 index b83dc4f1..00000000 --- a/crates/signinum-jpeg-metal/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# signinum-jpeg-metal - -Apple Metal device-output adapter for `signinum-jpeg`. - -Install this crate when a macOS pipeline needs JPEG tile output as a -Metal-backed `DeviceSurface`: - -```sh -cargo add signinum-jpeg-metal -``` - -`BackendRequest::Auto` may choose a validated Metal path for supported JPEG -tile shapes and otherwise returns host-backed CPU output. `BackendRequest::Metal` -is strict: supported requests return resident Metal decode surfaces on macOS, -while unsupported shapes, CPU-staged upload fallbacks, or hosts without Metal -return an error. Check `Surface::residency()` when a caller needs to distinguish -`MetalResidentDecode` from explicit CPU-staged upload buffers. - -The stable CPU decode API lives in `signinum-jpeg`. This adapter remains -pre-1.0 while runtime validation and routing policies continue to harden. diff --git a/crates/signinum-jpeg-metal/src/compute.rs b/crates/signinum-jpeg-metal/src/compute.rs deleted file mode 100644 index be62c37a..00000000 --- a/crates/signinum-jpeg-metal/src/compute.rs +++ /dev/null @@ -1,10342 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#![allow(clippy::similar_names)] - -#[cfg(all(target_os = "macos", test))] -use std::cell::Cell; -#[cfg(target_os = "macos")] -use std::{ - cell::RefCell, - ffi::OsStr, - mem::{size_of, size_of_val}, - time::{Duration, Instant}, -}; - -#[cfg(target_os = "macos")] -use metal::{ - Buffer, CommandBuffer, CommandBufferRef, CommandQueue, CompileOptions, ComputePipelineState, - Device, MTLResourceOptions, MTLSize, -}; -use signinum_core::{BackendRequest, PixelFormat, Rect}; -use signinum_jpeg::{ - adapter::{ - JpegMetalEntropyCheckpointV1, JpegMetalFast420PacketV1, JpegMetalFast422PacketV1, - JpegMetalFast444PacketV1, MetalHuffmanTable as PacketHuffmanTable, - }, - ColorSpace as JpegColorSpace, ComponentRowWriter, Decoder as CpuDecoder, -}; - -use crate::viewport::ViewportTile; -use crate::{batch, Error, Surface}; - -#[cfg(target_os = "macos")] -const SHADER_SOURCE: &str = include_str!("shaders.metal"); - -#[cfg(target_os = "macos")] -const MODE_GRAY: u32 = 0; -#[cfg(target_os = "macos")] -const MODE_YCBCR: u32 = 1; -#[cfg(target_os = "macos")] -const MODE_RGB: u32 = 2; - -#[cfg(target_os = "macos")] -const OUT_GRAY: u32 = 0; -#[cfg(target_os = "macos")] -const OUT_RGB: u32 = 1; -#[cfg(target_os = "macos")] -const OUT_RGBA: u32 = 2; - -#[cfg(target_os = "macos")] -pub(crate) const JPEG_BASELINE_ENCODE_FORMAT_GRAY8: u32 = 0; -#[cfg(target_os = "macos")] -pub(crate) const JPEG_BASELINE_ENCODE_FORMAT_RGB8: u32 = 1; -#[cfg(target_os = "macos")] -const JPEG_BASELINE_ENCODE_STATUS_OK: u32 = 0; -#[cfg(target_os = "macos")] -const JPEG_BASELINE_ENCODE_STATUS_OVERFLOW: u32 = 1; -#[cfg(target_os = "macos")] -const JPEG_BASELINE_ENCODE_STATUS_MISSING_HUFFMAN: u32 = 2; -#[cfg(target_os = "macos")] -const JPEG_BASELINE_ENCODE_STATUS_INVALID_PARAMS: u32 = 3; - -#[cfg(target_os = "macos")] -const FAST420_STATUS_OK: u32 = 0; -#[cfg(target_os = "macos")] -const FAST420_STATUS_TRUNCATED: u32 = 1; -#[cfg(target_os = "macos")] -const FAST420_STATUS_HUFFMAN: u32 = 2; -#[cfg(target_os = "macos")] -const AUTO_METAL_MIN_BATCH_REQUESTS: usize = 8; -#[cfg(target_os = "macos")] -const AUTO_METAL_MIN_BATCH_EDGE: u32 = 512; -#[cfg(target_os = "macos")] -const REGION_SCALED_BATCH_CHUNK: usize = 8; -#[cfg(target_os = "macos")] -const SPLIT_FAST420_BATCH_ENV: &str = "SIGNINUM_JPEG_METAL_SPLIT_FAST420_BATCH"; -#[cfg(target_os = "macos")] -const FAST420_BATCH_TIMING_ENV: &str = "SIGNINUM_JPEG_METAL_FAST420_BATCH_TIMING"; - -#[cfg(all(target_os = "macos", test))] -std::thread_local! { - static JPEG_PRIVATE_BUFFER_ALLOCATIONS: Cell = const { Cell::new(0) }; -} - -#[cfg(all(target_os = "macos", test))] -pub(crate) fn reset_jpeg_private_buffer_allocations_for_test() { - JPEG_PRIVATE_BUFFER_ALLOCATIONS.with(|allocations| allocations.set(0)); -} - -#[cfg(all(target_os = "macos", test))] -pub(crate) fn jpeg_private_buffer_allocations_for_test() -> usize { - JPEG_PRIVATE_BUFFER_ALLOCATIONS.with(Cell::get) -} - -#[cfg(target_os = "macos")] -fn new_shared_buffer(device: &Device, bytes: usize) -> Buffer { - device.new_buffer(bytes.max(1) as u64, MTLResourceOptions::StorageModeShared) -} - -#[cfg(target_os = "macos")] -fn new_private_buffer(device: &Device, bytes: usize) -> Buffer { - #[cfg(test)] - JPEG_PRIVATE_BUFFER_ALLOCATIONS.with(|allocations| allocations.set(allocations.get() + 1)); - device.new_buffer(bytes.max(1) as u64, MTLResourceOptions::StorageModePrivate) -} - -#[cfg(target_os = "macos")] -fn new_decode_plane_buffer(device: &Device, bytes: usize, returned_publicly: bool) -> Buffer { - if returned_publicly { - new_shared_buffer(device, bytes) - } else { - new_private_buffer(device, bytes) - } -} - -#[cfg(target_os = "macos")] -struct FastRgbDecodeBuffer { - buffer: Buffer, - dimensions: (u32, u32), - status_buffer: Buffer, - command_buffer: CommandBuffer, -} - -#[cfg(target_os = "macos")] -fn private_jpeg_tile_from_fast_rgb_buffer( - decoded: FastRgbDecodeBuffer, -) -> crate::ResidentPrivateJpegTile { - crate::ResidentPrivateJpegTile { - buffer: decoded.buffer, - byte_offset: 0, - dimensions: decoded.dimensions, - pixel_format: PixelFormat::Rgb8, - pitch_bytes: decoded.dimensions.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(), - status_buffer: decoded.status_buffer, - command_buffer: decoded.command_buffer, - } -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct JpegPackParams { - width: u32, - height: u32, - out_stride: u32, - alpha: u32, - mode: u32, - out_format: u32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -pub(crate) struct JpegBaselineEncodeParams { - pub(crate) input_offset_bytes: u32, - pub(crate) input_width: u32, - pub(crate) input_height: u32, - pub(crate) output_width: u32, - pub(crate) output_height: u32, - pub(crate) pitch_bytes: u32, - pub(crate) mcus_per_row: u32, - pub(crate) mcu_rows: u32, - pub(crate) restart_interval_mcus: u32, - pub(crate) format: u32, - pub(crate) components: u32, - pub(crate) max_h: u32, - pub(crate) max_v: u32, - pub(crate) h0: u32, - pub(crate) v0: u32, - pub(crate) h1: u32, - pub(crate) v1: u32, - pub(crate) h2: u32, - pub(crate) v2: u32, - pub(crate) entropy_offset_bytes: u32, - pub(crate) entropy_capacity: u32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -pub(crate) struct JpegBaselineEncodeHuffmanTable { - pub(crate) codes: [u16; 256], - pub(crate) lens: [u8; 256], -} - -#[cfg(target_os = "macos")] -impl Default for JpegBaselineEncodeHuffmanTable { - fn default() -> Self { - Self { - codes: [0; 256], - lens: [0; 256], - } - } -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct JpegBaselineEncodeStatus { - code: u32, - entropy_len: u32, - detail: u32, - reserved: u32, -} - -#[cfg(target_os = "macos")] -pub(crate) struct JpegBaselineEntropyEncodeJob<'a> { - pub(crate) input: &'a Buffer, - pub(crate) input_offset: usize, - pub(crate) params: JpegBaselineEncodeParams, - pub(crate) q_luma: [u8; 64], - pub(crate) q_chroma: [u8; 64], - pub(crate) huff_dc_luma: JpegBaselineEncodeHuffmanTable, - pub(crate) huff_ac_luma: JpegBaselineEncodeHuffmanTable, - pub(crate) huff_dc_chroma: JpegBaselineEncodeHuffmanTable, - pub(crate) huff_ac_chroma: JpegBaselineEncodeHuffmanTable, - pub(crate) entropy_capacity: usize, -} - -#[cfg(target_os = "macos")] -pub(crate) struct JpegBaselineEntropyEncodeBatchJob<'a> { - pub(crate) input: &'a Buffer, - pub(crate) params: Vec, - pub(crate) q_luma: [u8; 64], - pub(crate) q_chroma: [u8; 64], - pub(crate) huff_dc_luma: JpegBaselineEncodeHuffmanTable, - pub(crate) huff_ac_luma: JpegBaselineEncodeHuffmanTable, - pub(crate) huff_dc_chroma: JpegBaselineEncodeHuffmanTable, - pub(crate) huff_ac_chroma: JpegBaselineEncodeHuffmanTable, - pub(crate) entropy_capacity: usize, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct JpegFast420Params { - width: u32, - height: u32, - chroma_width: u32, - chroma_height: u32, - mcus_per_row: u32, - mcu_rows: u32, - restart_interval_mcus: u32, - restart_offset_count: u32, - restart_start_mcu: u32, - entropy_len: u32, - out_stride: u32, - alpha: u32, - out_format: u32, - origin_x: u32, - origin_y: u32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, PartialEq, Eq)] -struct JpegFast420ScaledParams { - scaled_width: u32, - scaled_height: u32, - chroma_width: u32, - chroma_height: u32, - mcus_per_row: u32, - mcu_rows: u32, - restart_interval_mcus: u32, - restart_offset_count: u32, - restart_start_mcu: u32, - entropy_len: u32, - scale_shift: u32, - origin_x: u32, - origin_y: u32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct JpegFast444Params { - width: u32, - height: u32, - mcus_per_row: u32, - mcu_rows: u32, - restart_interval_mcus: u32, - restart_offset_count: u32, - restart_start_mcu: u32, - entropy_len: u32, - origin_x: u32, - origin_y: u32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, PartialEq, Eq)] -struct JpegFast444ScaledParams { - scaled_width: u32, - scaled_height: u32, - mcus_per_row: u32, - mcu_rows: u32, - restart_interval_mcus: u32, - restart_offset_count: u32, - restart_start_mcu: u32, - entropy_len: u32, - scale_shift: u32, - origin_x: u32, - origin_y: u32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, PartialEq, Eq)] -struct JpegFast420WindowedPackParams { - src_width: u32, - src_height: u32, - chroma_width: u32, - chroma_height: u32, - src_x: u32, - src_y: u32, - width: u32, - height: u32, - out_stride: u32, - alpha: u32, - out_format: u32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct JpegFast420BatchParams { - width: u32, - height: u32, - chroma_width: u32, - chroma_height: u32, - mcus_per_row: u32, - mcu_rows: u32, - segment_count: u32, - tile_count: u32, - out_stride: u32, - alpha: u32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, PartialEq, Eq)] -struct JpegFastRegionScaledBatchParams { - scaled_width: u32, - scaled_height: u32, - chroma_width: u32, - chroma_height: u32, - mcus_per_row: u32, - mcu_rows: u32, - segment_count: u32, - tile_count: u32, - scale_shift: u32, - origin_x: u32, - origin_y: u32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, PartialEq, Eq)] -struct JpegWindowedPackBatchParams { - src_width: u32, - src_height: u32, - chroma_width: u32, - chroma_height: u32, - src_x: u32, - src_y: u32, - width: u32, - height: u32, - tile_count: u32, - out_stride: u32, - alpha: u32, - mode: u32, - out_format: u32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct PreparedHuffmanHost { - min_code: [i32; 17], - max_code: [i32; 17], - val_offset: [i32; 17], - values: [u8; 256], - fast_symbol: [u8; 512], - fast_len: [u8; 512], - values_len: u16, - reserved: u16, -} - -#[cfg(target_os = "macos")] -impl From<&PacketHuffmanTable> for PreparedHuffmanHost { - fn from(value: &PacketHuffmanTable) -> Self { - let mut min_code = [i32::MAX; 17]; - let mut max_code = [-1i32; 17]; - let mut val_offset = [0i32; 17]; - let mut values = [0u8; 256]; - let mut fast_symbol = [0u8; 512]; - let mut fast_len = [0u8; 512]; - let values_len = usize::from(value.values_len); - values[..values_len].copy_from_slice(&value.values[..values_len]); - - let mut huffsize = [0u8; 256]; - let mut huffsize_len = 0usize; - for (len_minus_1, &count) in value.bits.iter().enumerate() { - let len = u8::try_from(len_minus_1 + 1).expect("JPEG Huffman code length fits in u8"); - for _ in 0..count { - huffsize[huffsize_len] = len; - huffsize_len += 1; - } - } - - let mut huffcode = [0u16; 256]; - let mut code = 0u32; - let mut si = huffsize.first().copied().unwrap_or(0); - for (idx, &size) in huffsize[..huffsize_len].iter().enumerate() { - while size != si { - code <<= 1; - si += 1; - } - huffcode[idx] = u16::try_from(code).expect("JPEG Huffman code fits in u16"); - code += 1; - } - - let mut idx = 0usize; - for (len_minus_1, &count) in value.bits.iter().enumerate() { - let len = len_minus_1 + 1; - let count = usize::from(count); - if count == 0 { - continue; - } - min_code[len] = i32::from(huffcode[idx]); - max_code[len] = i32::from(huffcode[idx + count - 1]); - val_offset[len] = - i32::try_from(idx).expect("JPEG Huffman value index fits in i32") - min_code[len]; - idx += count; - } - - for idx in 0..huffsize_len { - let len = usize::from(huffsize[idx]); - if len == 0 || len > 9 { - continue; - } - let code = usize::from(huffcode[idx]); - let prefix = code << (9 - len); - let fill = 1usize << (9 - len); - for suffix in 0..fill { - fast_symbol[prefix | suffix] = values[idx]; - fast_len[prefix | suffix] = huffsize[idx]; - } - } - - Self { - min_code, - max_code, - val_offset, - values, - fast_symbol, - fast_len, - values_len: value.values_len, - reserved: 0, - } - } -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy, Default)] -struct JpegDecodeStatus { - code: u32, - detail: u32, - position: u32, - reserved: u32, -} - -#[cfg(target_os = "macos")] -#[repr(C)] -#[derive(Clone, Copy)] -struct JpegEntropyCheckpointHost { - mcu_index: u32, - entropy_pos: u32, - bit_acc: u64, - bit_count: u32, - y_prev_dc: i32, - cb_prev_dc: i32, - cr_prev_dc: i32, - reserved: u32, -} - -#[cfg(target_os = "macos")] -impl From for JpegEntropyCheckpointHost { - fn from(value: JpegMetalEntropyCheckpointV1) -> Self { - Self { - mcu_index: value.mcu_index, - entropy_pos: value.entropy_pos, - bit_acc: value.bit_acc, - bit_count: value.bit_count, - y_prev_dc: value.y_prev_dc, - cb_prev_dc: value.cb_prev_dc, - cr_prev_dc: value.cr_prev_dc, - reserved: value.reserved, - } - } -} - -#[cfg(target_os = "macos")] -thread_local! { - static METAL_RUNTIME: RefCell>> = const { RefCell::new(None) }; - static VIEWPORT_PLANE_CACHE: RefCell> = const { RefCell::new(None) }; -} - -#[cfg(target_os = "macos")] -pub(crate) struct MetalRuntime { - device: Device, - queue: CommandQueue, - pack_pipeline: ComputePipelineState, - jpeg_baseline_encode_pipeline: ComputePipelineState, - jpeg_baseline_encode_batch_pipeline: ComputePipelineState, - pack_420_pipeline: ComputePipelineState, - pack_420_rgb_pipeline: ComputePipelineState, - pack_420_rgba_pipeline: ComputePipelineState, - pack_420_rgb_batch_pipeline: ComputePipelineState, - pack_420_windowed_rgb_batch_pipeline: ComputePipelineState, - pack_422_rgb_pipeline: ComputePipelineState, - pack_422_rgba_pipeline: ComputePipelineState, - pack_422_rgb_batch_pipeline: ComputePipelineState, - pack_422_windowed_rgb_batch_pipeline: ComputePipelineState, - pack_444_rgb_batch_pipeline: ComputePipelineState, - pack_422_windowed_pipeline: ComputePipelineState, - pack_422_windowed_rgb_pipeline: ComputePipelineState, - pack_422_windowed_rgba_pipeline: ComputePipelineState, - pack_420_windowed_pipeline: ComputePipelineState, - pack_420_windowed_rgb_pipeline: ComputePipelineState, - pack_420_windowed_rgba_pipeline: ComputePipelineState, - fast420_decode_pipeline: ComputePipelineState, - fast420_batch_decode_pipeline: ComputePipelineState, - fast420_batch_coeffs_decode_pipeline: ComputePipelineState, - fast420_batch_idct_deposit_pipeline: ComputePipelineState, - fast420_scaled_region_batch_decode_pipeline: ComputePipelineState, - fast422_decode_pipeline: ComputePipelineState, - fast422_batch_decode_pipeline: ComputePipelineState, - fast422_scaled_region_batch_decode_pipeline: ComputePipelineState, - fast422_region_decode_pipeline: ComputePipelineState, - fast422_scaled_decode_pipeline: ComputePipelineState, - fast422_scaled_region_decode_pipeline: ComputePipelineState, - fast420_region_decode_pipeline: ComputePipelineState, - fast420_scaled_decode_pipeline: ComputePipelineState, - fast420_scaled_region_decode_pipeline: ComputePipelineState, - fast444_decode_pipeline: ComputePipelineState, - fast444_region_decode_pipeline: ComputePipelineState, - fast444_scaled_decode_pipeline: ComputePipelineState, - fast444_scaled_region_decode_pipeline: ComputePipelineState, - fast444_scaled_region_batch_decode_pipeline: ComputePipelineState, -} - -#[cfg(target_os = "macos")] -impl MetalRuntime { - fn new() -> Result { - let device = Device::system_default() - .ok_or_else(|| "Metal is unavailable on this host".to_string())?; - Self::new_with_device(device) - } - - pub(crate) fn new_with_device(device: Device) -> Result { - let options = CompileOptions::new(); - let library = device.new_library_with_source(SHADER_SOURCE, &options)?; - let pack_function = library.get_function("jpeg_pack", None)?; - let pack_pipeline = device.new_compute_pipeline_state_with_function(&pack_function)?; - let jpeg_baseline_encode_function = - library.get_function("jpeg_encode_baseline_entropy", None)?; - let jpeg_baseline_encode_pipeline = - device.new_compute_pipeline_state_with_function(&jpeg_baseline_encode_function)?; - let jpeg_baseline_encode_batch_function = - library.get_function("jpeg_encode_baseline_entropy_batch", None)?; - let jpeg_baseline_encode_batch_pipeline = device - .new_compute_pipeline_state_with_function(&jpeg_baseline_encode_batch_function)?; - let pack_420_function = library.get_function("jpeg_pack_420", None)?; - let pack_420_pipeline = - device.new_compute_pipeline_state_with_function(&pack_420_function)?; - let pack_420_rgb_function = library.get_function("jpeg_pack_420_rgb", None)?; - let pack_420_rgb_pipeline = - device.new_compute_pipeline_state_with_function(&pack_420_rgb_function)?; - let pack_420_rgba_function = library.get_function("jpeg_pack_420_rgba", None)?; - let pack_420_rgba_pipeline = - device.new_compute_pipeline_state_with_function(&pack_420_rgba_function)?; - let pack_420_rgb_batch_function = library.get_function("jpeg_pack_420_rgb_batch", None)?; - let pack_420_rgb_batch_pipeline = - device.new_compute_pipeline_state_with_function(&pack_420_rgb_batch_function)?; - let pack_420_windowed_rgb_batch_function = - library.get_function("jpeg_pack_420_windowed_rgb_batch", None)?; - let pack_420_windowed_rgb_batch_pipeline = device - .new_compute_pipeline_state_with_function(&pack_420_windowed_rgb_batch_function)?; - let pack_422_rgb_function = library.get_function("jpeg_pack_422_rgb", None)?; - let pack_422_rgb_pipeline = - device.new_compute_pipeline_state_with_function(&pack_422_rgb_function)?; - let pack_422_rgba_function = library.get_function("jpeg_pack_422_rgba", None)?; - let pack_422_rgba_pipeline = - device.new_compute_pipeline_state_with_function(&pack_422_rgba_function)?; - let pack_422_rgb_batch_function = library.get_function("jpeg_pack_422_rgb_batch", None)?; - let pack_422_rgb_batch_pipeline = - device.new_compute_pipeline_state_with_function(&pack_422_rgb_batch_function)?; - let pack_422_windowed_rgb_batch_function = - library.get_function("jpeg_pack_422_windowed_rgb_batch", None)?; - let pack_422_windowed_rgb_batch_pipeline = device - .new_compute_pipeline_state_with_function(&pack_422_windowed_rgb_batch_function)?; - let pack_444_rgb_batch_function = library.get_function("jpeg_pack_444_rgb_batch", None)?; - let pack_444_rgb_batch_pipeline = - device.new_compute_pipeline_state_with_function(&pack_444_rgb_batch_function)?; - let pack_422_windowed_function = library.get_function("jpeg_pack_422_windowed", None)?; - let pack_422_windowed_pipeline = - device.new_compute_pipeline_state_with_function(&pack_422_windowed_function)?; - let pack_422_windowed_rgb_function = - library.get_function("jpeg_pack_422_windowed_rgb", None)?; - let pack_422_windowed_rgb_pipeline = - device.new_compute_pipeline_state_with_function(&pack_422_windowed_rgb_function)?; - let pack_422_windowed_rgba_function = - library.get_function("jpeg_pack_422_windowed_rgba", None)?; - let pack_422_windowed_rgba_pipeline = - device.new_compute_pipeline_state_with_function(&pack_422_windowed_rgba_function)?; - let pack_420_windowed_function = library.get_function("jpeg_pack_420_windowed", None)?; - let pack_420_windowed_pipeline = - device.new_compute_pipeline_state_with_function(&pack_420_windowed_function)?; - let pack_420_windowed_rgb_function = - library.get_function("jpeg_pack_420_windowed_rgb", None)?; - let pack_420_windowed_rgb_pipeline = - device.new_compute_pipeline_state_with_function(&pack_420_windowed_rgb_function)?; - let pack_420_windowed_rgba_function = - library.get_function("jpeg_pack_420_windowed_rgba", None)?; - let pack_420_windowed_rgba_pipeline = - device.new_compute_pipeline_state_with_function(&pack_420_windowed_rgba_function)?; - let fast420_decode_function = library.get_function("jpeg_decode_fast420", None)?; - let fast420_decode_pipeline = - device.new_compute_pipeline_state_with_function(&fast420_decode_function)?; - let fast420_batch_decode_function = - library.get_function("jpeg_decode_fast420_batch", None)?; - let fast420_batch_decode_pipeline = - device.new_compute_pipeline_state_with_function(&fast420_batch_decode_function)?; - let fast420_batch_coeffs_decode_function = - library.get_function("jpeg_decode_fast420_batch_coeffs", None)?; - let fast420_batch_coeffs_decode_pipeline = device - .new_compute_pipeline_state_with_function(&fast420_batch_coeffs_decode_function)?; - let fast420_batch_idct_deposit_function = - library.get_function("jpeg_idct_deposit_fast420_batch", None)?; - let fast420_batch_idct_deposit_pipeline = device - .new_compute_pipeline_state_with_function(&fast420_batch_idct_deposit_function)?; - let fast420_scaled_region_batch_decode_function = - library.get_function("jpeg_decode_fast420_scaled_region_batch", None)?; - let fast420_scaled_region_batch_decode_pipeline = device - .new_compute_pipeline_state_with_function( - &fast420_scaled_region_batch_decode_function, - )?; - let fast422_decode_function = library.get_function("jpeg_decode_fast422", None)?; - let fast422_decode_pipeline = - device.new_compute_pipeline_state_with_function(&fast422_decode_function)?; - let fast422_batch_decode_function = - library.get_function("jpeg_decode_fast422_batch", None)?; - let fast422_batch_decode_pipeline = - device.new_compute_pipeline_state_with_function(&fast422_batch_decode_function)?; - let fast422_scaled_region_batch_decode_function = - library.get_function("jpeg_decode_fast422_scaled_region_batch", None)?; - let fast422_scaled_region_batch_decode_pipeline = device - .new_compute_pipeline_state_with_function( - &fast422_scaled_region_batch_decode_function, - )?; - let fast422_region_decode_function = - library.get_function("jpeg_decode_fast422_region", None)?; - let fast422_region_decode_pipeline = - device.new_compute_pipeline_state_with_function(&fast422_region_decode_function)?; - let fast422_scaled_decode_function = - library.get_function("jpeg_decode_fast422_scaled", None)?; - let fast422_scaled_decode_pipeline = - device.new_compute_pipeline_state_with_function(&fast422_scaled_decode_function)?; - let fast422_scaled_region_decode_function = - library.get_function("jpeg_decode_fast422_scaled_region", None)?; - let fast422_scaled_region_decode_pipeline = device - .new_compute_pipeline_state_with_function(&fast422_scaled_region_decode_function)?; - let fast420_region_decode_function = - library.get_function("jpeg_decode_fast420_region", None)?; - let fast420_region_decode_pipeline = - device.new_compute_pipeline_state_with_function(&fast420_region_decode_function)?; - let fast420_scaled_decode_function = - library.get_function("jpeg_decode_fast420_scaled", None)?; - let fast420_scaled_decode_pipeline = - device.new_compute_pipeline_state_with_function(&fast420_scaled_decode_function)?; - let fast420_scaled_region_decode_function = - library.get_function("jpeg_decode_fast420_scaled_region", None)?; - let fast420_scaled_region_decode_pipeline = device - .new_compute_pipeline_state_with_function(&fast420_scaled_region_decode_function)?; - let fast444_decode_function = library.get_function("jpeg_decode_fast444", None)?; - let fast444_decode_pipeline = - device.new_compute_pipeline_state_with_function(&fast444_decode_function)?; - let fast444_region_decode_function = - library.get_function("jpeg_decode_fast444_region", None)?; - let fast444_region_decode_pipeline = - device.new_compute_pipeline_state_with_function(&fast444_region_decode_function)?; - let fast444_scaled_decode_function = - library.get_function("jpeg_decode_fast444_scaled", None)?; - let fast444_scaled_decode_pipeline = - device.new_compute_pipeline_state_with_function(&fast444_scaled_decode_function)?; - let fast444_scaled_region_decode_function = - library.get_function("jpeg_decode_fast444_scaled_region", None)?; - let fast444_scaled_region_decode_pipeline = device - .new_compute_pipeline_state_with_function(&fast444_scaled_region_decode_function)?; - let fast444_scaled_region_batch_decode_function = - library.get_function("jpeg_decode_fast444_scaled_region_batch", None)?; - let fast444_scaled_region_batch_decode_pipeline = device - .new_compute_pipeline_state_with_function( - &fast444_scaled_region_batch_decode_function, - )?; - let queue = device.new_command_queue(); - Ok(Self { - device, - queue, - pack_pipeline, - jpeg_baseline_encode_pipeline, - jpeg_baseline_encode_batch_pipeline, - pack_420_pipeline, - pack_420_rgb_pipeline, - pack_420_rgba_pipeline, - pack_420_rgb_batch_pipeline, - pack_420_windowed_rgb_batch_pipeline, - pack_422_rgb_pipeline, - pack_422_rgba_pipeline, - pack_422_rgb_batch_pipeline, - pack_422_windowed_rgb_batch_pipeline, - pack_444_rgb_batch_pipeline, - pack_422_windowed_pipeline, - pack_422_windowed_rgb_pipeline, - pack_422_windowed_rgba_pipeline, - pack_420_windowed_pipeline, - pack_420_windowed_rgb_pipeline, - pack_420_windowed_rgba_pipeline, - fast420_decode_pipeline, - fast420_batch_decode_pipeline, - fast420_batch_coeffs_decode_pipeline, - fast420_batch_idct_deposit_pipeline, - fast420_scaled_region_batch_decode_pipeline, - fast422_decode_pipeline, - fast422_batch_decode_pipeline, - fast422_scaled_region_batch_decode_pipeline, - fast422_region_decode_pipeline, - fast422_scaled_decode_pipeline, - fast422_scaled_region_decode_pipeline, - fast420_region_decode_pipeline, - fast420_scaled_decode_pipeline, - fast420_scaled_region_decode_pipeline, - fast444_decode_pipeline, - fast444_region_decode_pipeline, - fast444_scaled_decode_pipeline, - fast444_scaled_region_decode_pipeline, - fast444_scaled_region_batch_decode_pipeline, - }) - } -} - -#[cfg(target_os = "macos")] -fn pack_420_pipeline_for_format(runtime: &MetalRuntime, fmt: PixelFormat) -> &ComputePipelineState { - match fmt { - PixelFormat::Rgb8 => &runtime.pack_420_rgb_pipeline, - PixelFormat::Rgba8 => &runtime.pack_420_rgba_pipeline, - _ => &runtime.pack_420_pipeline, - } -} - -#[cfg(target_os = "macos")] -fn pack_420_windowed_pipeline_for_format( - runtime: &MetalRuntime, - fmt: PixelFormat, -) -> &ComputePipelineState { - match fmt { - PixelFormat::Rgb8 => &runtime.pack_420_windowed_rgb_pipeline, - PixelFormat::Rgba8 => &runtime.pack_420_windowed_rgba_pipeline, - _ => &runtime.pack_420_windowed_pipeline, - } -} - -#[cfg(target_os = "macos")] -fn pack_422_pipeline_for_format( - runtime: &MetalRuntime, - fmt: PixelFormat, -) -> Option<&ComputePipelineState> { - match fmt { - PixelFormat::Rgb8 => Some(&runtime.pack_422_rgb_pipeline), - PixelFormat::Rgba8 => Some(&runtime.pack_422_rgba_pipeline), - _ => None, - } -} - -#[cfg(target_os = "macos")] -fn pack_422_windowed_pipeline_for_format( - runtime: &MetalRuntime, - fmt: PixelFormat, -) -> &ComputePipelineState { - match fmt { - PixelFormat::Rgb8 => &runtime.pack_422_windowed_rgb_pipeline, - PixelFormat::Rgba8 => &runtime.pack_422_windowed_rgba_pipeline, - _ => &runtime.pack_422_windowed_pipeline, - } -} - -#[cfg(target_os = "macos")] -fn with_runtime(f: impl FnOnce(&MetalRuntime) -> Result) -> Result { - METAL_RUNTIME.with(|runtime| { - let mut runtime = runtime.borrow_mut(); - if runtime.is_none() { - *runtime = Some(MetalRuntime::new()); - } - match runtime.as_ref().expect("runtime initialized") { - Ok(runtime) => f(runtime), - Err(message) => Err(runtime_initialization_error(message)), - } - }) -} - -#[cfg(target_os = "macos")] -fn with_runtime_for_session( - session: &crate::MetalBackendSession, - f: impl FnOnce(&MetalRuntime) -> Result, -) -> Result { - let runtime = session - .runtime - .get_or_init(|| MetalRuntime::new_with_device(session.device.clone())); - match runtime { - Ok(runtime) => f(runtime), - Err(message) => Err(runtime_initialization_error(message)), - } -} - -#[cfg(target_os = "macos")] -pub(crate) fn runtime_initialization_error(message: &str) -> Error { - if message == "Metal is unavailable on this host" { - Error::MetalUnavailable - } else { - Error::MetalKernel { - message: message.to_string(), - } - } -} - -#[cfg(target_os = "macos")] -pub(crate) fn encode_jpeg_baseline_entropy_with_session( - session: &crate::MetalBackendSession, - job: &JpegBaselineEntropyEncodeJob<'_>, -) -> Result, Error> { - with_runtime_for_session(session, |runtime| { - let entropy_buffer = runtime.device.new_buffer( - job.entropy_capacity as u64, - MTLResourceOptions::StorageModeShared, - ); - let status = JpegBaselineEncodeStatus::default(); - let status_buffer = runtime.device.new_buffer_with_data( - (&raw const status).cast(), - size_of::() as u64, - MTLResourceOptions::StorageModeShared, - ); - - let command_buffer = runtime.queue.new_command_buffer(); - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.jpeg_baseline_encode_pipeline); - encoder.set_buffer(0, Some(job.input), job.input_offset as u64); - encoder.set_buffer(1, Some(&entropy_buffer), 0); - encoder.set_buffer(2, Some(&status_buffer), 0); - encoder.set_bytes( - 3, - size_of::() as u64, - (&raw const job.params).cast(), - ); - encoder.set_bytes( - 4, - size_of_val(&job.q_luma) as u64, - job.q_luma.as_ptr().cast(), - ); - encoder.set_bytes( - 5, - size_of_val(&job.q_chroma) as u64, - job.q_chroma.as_ptr().cast(), - ); - encoder.set_bytes( - 6, - size_of::() as u64, - (&raw const job.huff_dc_luma).cast(), - ); - encoder.set_bytes( - 7, - size_of::() as u64, - (&raw const job.huff_ac_luma).cast(), - ); - encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const job.huff_dc_chroma).cast(), - ); - encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const job.huff_ac_chroma).cast(), - ); - encoder.dispatch_threads( - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - ); - encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - - let status = unsafe { *(status_buffer.contents().cast::()) }; - if status.code != JPEG_BASELINE_ENCODE_STATUS_OK { - return Err(jpeg_baseline_encode_status_error(status)); - } - let entropy_len = usize::try_from(status.entropy_len).map_err(|_| Error::MetalKernel { - message: "JPEG Baseline Metal encode entropy length exceeds usize".to_string(), - })?; - if entropy_len > job.entropy_capacity { - return Err(Error::MetalKernel { - message: "JPEG Baseline Metal encode reported length exceeds output capacity" - .to_string(), - }); - } - let entropy = unsafe { - core::slice::from_raw_parts(entropy_buffer.contents().cast::(), entropy_len) - }; - Ok(entropy.to_vec()) - }) -} - -#[cfg(target_os = "macos")] -pub(crate) fn encode_jpeg_baseline_entropy_batch_with_session( - session: &crate::MetalBackendSession, - job: &JpegBaselineEntropyEncodeBatchJob<'_>, -) -> Result>, Error> { - if job.params.is_empty() { - return Ok(Vec::new()); - } - with_runtime_for_session(session, |runtime| { - let entropy_buffer = runtime.device.new_buffer( - job.entropy_capacity as u64, - MTLResourceOptions::StorageModeShared, - ); - let statuses = vec![JpegBaselineEncodeStatus::default(); job.params.len()]; - let status_buffer = runtime.device.new_buffer_with_data( - statuses.as_ptr().cast(), - size_of::() as u64 * statuses.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let params_buffer = runtime.device.new_buffer_with_data( - job.params.as_ptr().cast(), - size_of::() as u64 * job.params.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let tile_count = u32::try_from(job.params.len()).map_err(|_| Error::MetalKernel { - message: "JPEG Baseline Metal batch tile count exceeds u32".to_string(), - })?; - - let command_buffer = runtime.queue.new_command_buffer(); - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.jpeg_baseline_encode_batch_pipeline); - encoder.set_buffer(0, Some(job.input), 0); - encoder.set_buffer(1, Some(&entropy_buffer), 0); - encoder.set_buffer(2, Some(&status_buffer), 0); - encoder.set_buffer(3, Some(¶ms_buffer), 0); - encoder.set_bytes( - 4, - size_of_val(&job.q_luma) as u64, - job.q_luma.as_ptr().cast(), - ); - encoder.set_bytes( - 5, - size_of_val(&job.q_chroma) as u64, - job.q_chroma.as_ptr().cast(), - ); - encoder.set_bytes( - 6, - size_of::() as u64, - (&raw const job.huff_dc_luma).cast(), - ); - encoder.set_bytes( - 7, - size_of::() as u64, - (&raw const job.huff_ac_luma).cast(), - ); - encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const job.huff_dc_chroma).cast(), - ); - encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const job.huff_ac_chroma).cast(), - ); - encoder.set_bytes(10, size_of::() as u64, (&raw const tile_count).cast()); - encoder.dispatch_threads( - MTLSize { - width: u64::from(tile_count), - height: 1, - depth: 1, - }, - MTLSize { - width: 1, - height: 1, - depth: 1, - }, - ); - encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - - let status_slice = unsafe { - core::slice::from_raw_parts( - status_buffer.contents().cast::(), - job.params.len(), - ) - }; - let entropy_bytes = unsafe { - core::slice::from_raw_parts( - entropy_buffer.contents().cast::(), - job.entropy_capacity, - ) - }; - let mut out = Vec::with_capacity(job.params.len()); - for (status, params) in status_slice.iter().copied().zip(job.params.iter()) { - if status.code != JPEG_BASELINE_ENCODE_STATUS_OK { - return Err(jpeg_baseline_encode_status_error(status)); - } - let entropy_len = - usize::try_from(status.entropy_len).map_err(|_| Error::MetalKernel { - message: "JPEG Baseline Metal encode entropy length exceeds usize".to_string(), - })?; - let offset = - usize::try_from(params.entropy_offset_bytes).map_err(|_| Error::MetalKernel { - message: "JPEG Baseline Metal batch entropy offset exceeds usize".to_string(), - })?; - let capacity = - usize::try_from(params.entropy_capacity).map_err(|_| Error::MetalKernel { - message: "JPEG Baseline Metal batch entropy capacity exceeds usize".to_string(), - })?; - if entropy_len > capacity { - return Err(Error::MetalKernel { - message: - "JPEG Baseline Metal encode reported length exceeds tile output capacity" - .to_string(), - }); - } - let end = offset - .checked_add(entropy_len) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Baseline Metal batch entropy range overflow".to_string(), - })?; - if end > entropy_bytes.len() { - return Err(Error::MetalKernel { - message: "JPEG Baseline Metal batch entropy range exceeds buffer".to_string(), - }); - } - out.push(entropy_bytes[offset..end].to_vec()); - } - Ok(out) - }) -} - -#[cfg(target_os = "macos")] -fn jpeg_baseline_encode_status_error(status: JpegBaselineEncodeStatus) -> Error { - let message = match status.code { - JPEG_BASELINE_ENCODE_STATUS_OVERFLOW => { - "JPEG Baseline Metal encode entropy output exceeded capacity".to_string() - } - JPEG_BASELINE_ENCODE_STATUS_MISSING_HUFFMAN => format!( - "JPEG Baseline Metal encode missing Huffman code for symbol {}", - status.detail - ), - JPEG_BASELINE_ENCODE_STATUS_INVALID_PARAMS => { - "JPEG Baseline Metal encode received invalid kernel parameters".to_string() - } - other => format!("JPEG Baseline Metal encode failed with status {other}"), - }; - Error::MetalKernel { message } -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Copy, PartialEq, Eq)] -enum PlaneMode { - Gray, - YCbCr, - Rgb, -} - -#[cfg(target_os = "macos")] -struct PlaneStage { - dims: (u32, u32), - mode: PlaneMode, - plane0: Buffer, - plane1: Option, - plane2: Option, -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Copy)] -enum PlaneStageResidency { - CpuStagedMetalUpload, - MetalResidentDecode, -} - -#[cfg(target_os = "macos")] -struct ViewportPlaneWriter<'a> { - stage: &'a mut PlaneStage, - dest: Rect, -} - -#[cfg(target_os = "macos")] -struct CachedViewportPlanes { - dims: (u32, u32), - mode: PlaneMode, - plane0: Buffer, - plane1: Option, - plane2: Option, -} - -#[cfg(target_os = "macos")] -impl PlaneStage { - fn new(device: &Device, color_space: JpegColorSpace, dims: (u32, u32)) -> Result { - let len = dims.0 as usize * dims.1 as usize; - let plane0 = device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared); - let (mode, plane1, plane2) = match color_space { - JpegColorSpace::Grayscale => (PlaneMode::Gray, None, None), - JpegColorSpace::YCbCr => ( - PlaneMode::YCbCr, - Some(device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared)), - Some(device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared)), - ), - JpegColorSpace::Rgb => ( - PlaneMode::Rgb, - Some(device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared)), - Some(device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared)), - ), - JpegColorSpace::Cmyk | JpegColorSpace::Ycck => { - return Err(Error::MetalKernel { - message: "Metal compute path does not support CMYK/YCCK JPEG output" - .to_string(), - }) - } - }; - - Ok(Self { - dims, - mode, - plane0, - plane1, - plane2, - }) - } - - fn finish_with_runtime( - self, - runtime: &MetalRuntime, - fmt: PixelFormat, - ) -> Result { - self.finish_with_runtime_and_residency( - runtime, - fmt, - PlaneStageResidency::CpuStagedMetalUpload, - ) - } - - fn finish_resident_with_runtime( - self, - runtime: &MetalRuntime, - fmt: PixelFormat, - ) -> Result { - self.finish_with_runtime_and_residency( - runtime, - fmt, - PlaneStageResidency::MetalResidentDecode, - ) - } - - fn finish_with_runtime_and_residency( - self, - runtime: &MetalRuntime, - fmt: PixelFormat, - residency: PlaneStageResidency, - ) -> Result { - match (self.mode, fmt) { - (PlaneMode::Gray | PlaneMode::YCbCr, PixelFormat::Gray8) => Ok( - surface_from_plane_buffer(self.plane0, self.dims, fmt, residency), - ), - ( - PlaneMode::Gray | PlaneMode::YCbCr | PlaneMode::Rgb, - PixelFormat::Rgb8 | PixelFormat::Rgba8, - ) - | (PlaneMode::Rgb, PixelFormat::Gray8) => { - Ok(self.dispatch_with_runtime(runtime, fmt, residency)) - } - _ => Err(Error::MetalKernel { - message: format!("unsupported JPEG Metal pixel format {fmt:?}"), - }), - } - } - - fn dispatch_with_runtime( - self, - runtime: &MetalRuntime, - fmt: PixelFormat, - residency: PlaneStageResidency, - ) -> Surface { - let pitch_bytes = self.dims.0 as usize * fmt.bytes_per_pixel(); - let out_buffer = runtime.device.new_buffer( - (pitch_bytes * self.dims.1 as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - let params = JpegPackParams { - width: self.dims.0, - height: self.dims.1, - out_stride: u32::try_from(pitch_bytes).expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - mode: match self.mode { - PlaneMode::Gray => MODE_GRAY, - PlaneMode::YCbCr => MODE_YCBCR, - PlaneMode::Rgb => MODE_RGB, - }, - out_format: match fmt { - PixelFormat::Gray8 => OUT_GRAY, - PixelFormat::Rgb8 => OUT_RGB, - PixelFormat::Rgba8 => OUT_RGBA, - _ => unreachable!("validated by finish"), - }, - }; - - let command_buffer = runtime.queue.new_command_buffer(); - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.pack_pipeline); - encoder.set_buffer(0, Some(&self.plane0), 0); - encoder.set_buffer(1, self.plane1.as_ref().map(std::convert::AsRef::as_ref), 0); - encoder.set_buffer(2, self.plane2.as_ref().map(std::convert::AsRef::as_ref), 0); - encoder.set_buffer(3, Some(&out_buffer), 0); - encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - dispatch_2d_pipeline(encoder, &runtime.pack_pipeline, self.dims); - encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - - surface_from_plane_buffer(out_buffer, self.dims, fmt, residency) - } - - fn dispatch_private_rgb8_with_runtime( - self, - runtime: &MetalRuntime, - status_buffer: Buffer, - ) -> crate::ResidentPrivateJpegTile { - let fmt = PixelFormat::Rgb8; - let pitch_bytes = self.dims.0 as usize * fmt.bytes_per_pixel(); - let out_buffer = new_private_buffer(&runtime.device, pitch_bytes * self.dims.1 as usize); - let params = JpegPackParams { - width: self.dims.0, - height: self.dims.1, - out_stride: u32::try_from(pitch_bytes).expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - mode: match self.mode { - PlaneMode::Gray => MODE_GRAY, - PlaneMode::YCbCr => MODE_YCBCR, - PlaneMode::Rgb => MODE_RGB, - }, - out_format: OUT_RGB, - }; - - let command_buffer = runtime.queue.new_command_buffer(); - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.pack_pipeline); - encoder.set_buffer(0, Some(&self.plane0), 0); - encoder.set_buffer(1, self.plane1.as_ref().map(std::convert::AsRef::as_ref), 0); - encoder.set_buffer(2, self.plane2.as_ref().map(std::convert::AsRef::as_ref), 0); - encoder.set_buffer(3, Some(&out_buffer), 0); - encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - dispatch_2d_pipeline(encoder, &runtime.pack_pipeline, self.dims); - encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - let command_buffer = command_buffer.to_owned(); - - crate::ResidentPrivateJpegTile { - buffer: out_buffer, - byte_offset: 0, - dimensions: self.dims, - pixel_format: fmt, - pitch_bytes, - status_buffer, - command_buffer, - } - } -} - -#[cfg(target_os = "macos")] -fn surface_from_plane_buffer( - buffer: Buffer, - dims: (u32, u32), - fmt: PixelFormat, - residency: PlaneStageResidency, -) -> Surface { - match residency { - PlaneStageResidency::CpuStagedMetalUpload => { - Surface::from_cpu_staged_metal_buffer(buffer, dims, fmt) - } - PlaneStageResidency::MetalResidentDecode => Surface::from_metal_buffer(buffer, dims, fmt), - } -} - -#[cfg(target_os = "macos")] -impl ComponentRowWriter for PlaneStage { - fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), signinum_jpeg::JpegError> { - let width = self.dims.0 as usize; - write_row_u8(&self.plane0, y, width, gray_row); - Ok(()) - } - - fn write_ycbcr_row( - &mut self, - y: u32, - y_row: &[u8], - chroma_blue_row: &[u8], - chroma_red_row: &[u8], - ) -> Result<(), signinum_jpeg::JpegError> { - let width = self.dims.0 as usize; - write_row_u8(&self.plane0, y, width, y_row); - write_row_u8( - self.plane1.as_ref().expect("Cb plane"), - y, - width, - chroma_blue_row, - ); - write_row_u8( - self.plane2.as_ref().expect("Cr plane"), - y, - width, - chroma_red_row, - ); - Ok(()) - } - - fn write_rgb_row( - &mut self, - y: u32, - r_row: &[u8], - g_row: &[u8], - b_row: &[u8], - ) -> Result<(), signinum_jpeg::JpegError> { - let width = self.dims.0 as usize; - write_row_u8(&self.plane0, y, width, r_row); - write_row_u8(self.plane1.as_ref().expect("G plane"), y, width, g_row); - write_row_u8(self.plane2.as_ref().expect("B plane"), y, width, b_row); - Ok(()) - } -} - -#[cfg(target_os = "macos")] -fn write_row_u8(buffer: &Buffer, y: u32, width: usize, src: &[u8]) { - let row_start = y as usize * width; - let row_end = row_start + width; - let len = width * (y as usize + 1); - let dst = unsafe { - core::slice::from_raw_parts_mut(buffer.contents().cast::(), len.max(row_end)) - }; - dst[row_start..row_end].copy_from_slice(&src[..width]); -} - -#[cfg(target_os = "macos")] -fn write_row_u8_at(buffer: &Buffer, y: u32, x: u32, full_width: usize, src: &[u8]) { - let row_start = y as usize * full_width + x as usize; - let row_end = row_start + src.len(); - let len = full_width * (y as usize + 1); - let dst = unsafe { - core::slice::from_raw_parts_mut(buffer.contents().cast::(), len.max(row_end)) - }; - dst[row_start..row_end].copy_from_slice(src); -} - -#[cfg(target_os = "macos")] -fn plane_mode_for_color_space(color_space: JpegColorSpace) -> Result { - match color_space { - JpegColorSpace::Grayscale => Ok(PlaneMode::Gray), - JpegColorSpace::YCbCr => Ok(PlaneMode::YCbCr), - JpegColorSpace::Rgb => Ok(PlaneMode::Rgb), - JpegColorSpace::Cmyk | JpegColorSpace::Ycck => Err(Error::MetalKernel { - message: "Metal compute path does not support CMYK/YCCK JPEG output".to_string(), - }), - } -} - -#[cfg(target_os = "macos")] -fn clear_buffer(buffer: &Buffer, len: usize) { - unsafe { - core::ptr::write_bytes(buffer.contents().cast::(), 0, len); - } -} - -#[cfg(target_os = "macos")] -fn cached_plane_stage( - device: &Device, - color_space: JpegColorSpace, - dims: (u32, u32), -) -> Result { - let mode = plane_mode_for_color_space(color_space)?; - VIEWPORT_PLANE_CACHE.with(|slot| { - let mut slot = slot.borrow_mut(); - let len = dims.0 as usize * dims.1 as usize; - let refresh = slot - .as_ref() - .is_none_or(|cached| cached.dims != dims || cached.mode != mode); - if refresh { - let plane0 = device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared); - let (plane1, plane2) = match mode { - PlaneMode::Gray => (None, None), - PlaneMode::YCbCr | PlaneMode::Rgb => ( - Some(device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared)), - Some(device.new_buffer(len as u64, MTLResourceOptions::StorageModeShared)), - ), - }; - *slot = Some(CachedViewportPlanes { - dims, - mode, - plane0, - plane1, - plane2, - }); - } - - let cached = slot.as_ref().expect("viewport plane cache"); - let stage = PlaneStage { - dims, - mode, - plane0: cached.plane0.clone(), - plane1: cached.plane1.clone(), - plane2: cached.plane2.clone(), - }; - clear_buffer(&stage.plane0, len); - if let Some(plane1) = &stage.plane1 { - clear_buffer(plane1, len); - } - if let Some(plane2) = &stage.plane2 { - clear_buffer(plane2, len); - } - Ok(stage) - }) -} - -#[cfg(target_os = "macos")] -impl ComponentRowWriter for ViewportPlaneWriter<'_> { - fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), signinum_jpeg::JpegError> { - write_row_u8_at( - &self.stage.plane0, - self.dest.y + y, - self.dest.x, - self.stage.dims.0 as usize, - gray_row, - ); - Ok(()) - } - - fn write_ycbcr_row( - &mut self, - y: u32, - y_row: &[u8], - chroma_blue_row: &[u8], - chroma_red_row: &[u8], - ) -> Result<(), signinum_jpeg::JpegError> { - let width = self.stage.dims.0 as usize; - write_row_u8_at( - &self.stage.plane0, - self.dest.y + y, - self.dest.x, - width, - y_row, - ); - write_row_u8_at( - self.stage.plane1.as_ref().expect("Cb plane"), - self.dest.y + y, - self.dest.x, - width, - chroma_blue_row, - ); - write_row_u8_at( - self.stage.plane2.as_ref().expect("Cr plane"), - self.dest.y + y, - self.dest.x, - width, - chroma_red_row, - ); - Ok(()) - } - - fn write_rgb_row( - &mut self, - y: u32, - r_row: &[u8], - g_row: &[u8], - b_row: &[u8], - ) -> Result<(), signinum_jpeg::JpegError> { - let width = self.stage.dims.0 as usize; - write_row_u8_at( - &self.stage.plane0, - self.dest.y + y, - self.dest.x, - width, - r_row, - ); - write_row_u8_at( - self.stage.plane1.as_ref().expect("G plane"), - self.dest.y + y, - self.dest.x, - width, - g_row, - ); - write_row_u8_at( - self.stage.plane2.as_ref().expect("B plane"), - self.dest.y + y, - self.dest.x, - width, - b_row, - ); - Ok(()) - } -} - -#[cfg(target_os = "macos")] -fn dispatch_2d_pipeline( - encoder: &metal::ComputeCommandEncoderRef, - pipeline: &ComputePipelineState, - dims: (u32, u32), -) { - let width = pipeline.thread_execution_width().max(1); - let max_threads = pipeline.max_total_threads_per_threadgroup().max(width); - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(dims.0), - height: u64::from(dims.1), - depth: 1, - }, - MTLSize { - width, - height, - depth: 1, - }, - ); -} - -#[cfg(target_os = "macos")] -fn dispatch_3d_pipeline( - encoder: &metal::ComputeCommandEncoderRef, - pipeline: &ComputePipelineState, - dims: (u32, u32, u32), -) { - let width = pipeline.thread_execution_width().max(1); - let max_threads = pipeline.max_total_threads_per_threadgroup().max(width); - let height = (max_threads / width).max(1); - encoder.dispatch_threads( - MTLSize { - width: u64::from(dims.0), - height: u64::from(dims.1), - depth: u64::from(dims.2), - }, - MTLSize { - width, - height, - depth: 1, - }, - ); -} - -#[cfg(target_os = "macos")] -fn packed_pair_extent(value: u32) -> u32 { - value.div_ceil(2).max(1) -} - -#[cfg(target_os = "macos")] -fn dispatch_1d_pipeline( - encoder: &metal::ComputeCommandEncoderRef, - pipeline: &ComputePipelineState, - threads: u32, -) { - let threadgroup_width = choose_1d_threadgroup_width( - pipeline.thread_execution_width(), - pipeline.max_total_threads_per_threadgroup(), - threads, - ); - encoder.dispatch_threads( - MTLSize { - width: u64::from(threads.max(1)), - height: 1, - depth: 1, - }, - MTLSize { - width: threadgroup_width, - height: 1, - depth: 1, - }, - ); -} - -#[cfg(target_os = "macos")] -fn choose_1d_threadgroup_width(simd_width: u64, max_threads: u64, threads: u32) -> u64 { - let simd_width = simd_width.max(1); - let max_threads = max_threads.max(simd_width); - let requested = u64::from(threads.max(1)); - let rounded = requested.div_ceil(simd_width) * simd_width; - rounded.clamp(simd_width, max_threads.min(256).max(simd_width)) -} - -#[cfg(target_os = "macos")] -fn pixel_format_to_out_format(fmt: PixelFormat) -> Option { - match fmt { - PixelFormat::Gray8 => Some(OUT_GRAY), - PixelFormat::Rgb8 => Some(OUT_RGB), - PixelFormat::Rgba8 => Some(OUT_RGBA), - _ => None, - } -} - -#[cfg(target_os = "macos")] -fn plane_mode_to_u32(mode: PlaneMode) -> u32 { - match mode { - PlaneMode::Gray => MODE_GRAY, - PlaneMode::YCbCr => MODE_YCBCR, - PlaneMode::Rgb => MODE_RGB, - } -} - -#[cfg(target_os = "macos")] -fn fast420_params( - packet: &JpegMetalFast420PacketV1, - fmt: PixelFormat, -) -> Result { - let out_format = pixel_format_to_out_format(fmt).ok_or_else(|| Error::MetalKernel { - message: format!("unsupported JPEG Metal fast420 pixel format {fmt:?}"), - })?; - let out_stride = packet.dimensions.0 as usize * fmt.bytes_per_pixel(); - Ok(JpegFast420Params { - width: packet.dimensions.0, - height: packet.dimensions.1, - chroma_width: packet.dimensions.0.div_ceil(2), - chroma_height: packet.dimensions.1.div_ceil(2), - mcus_per_row: packet.mcus_per_row, - mcu_rows: packet.mcu_rows, - restart_interval_mcus: packet.restart_interval_mcus, - restart_offset_count: entropy_segment_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ), - restart_start_mcu: 0, - entropy_len: u32::try_from(packet.entropy_bytes.len()) - .expect("JPEG Metal entropy payload fits in u32"), - out_stride: u32::try_from(out_stride).expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - out_format, - origin_x: 0, - origin_y: 0, - }) -} - -#[cfg(target_os = "macos")] -fn fast422_params( - packet: &JpegMetalFast422PacketV1, - fmt: PixelFormat, -) -> Result { - let out_format = pixel_format_to_out_format(fmt).ok_or_else(|| Error::MetalKernel { - message: format!("unsupported JPEG Metal fast422 pixel format {fmt:?}"), - })?; - let out_stride = packet.dimensions.0 as usize * fmt.bytes_per_pixel(); - Ok(JpegFast420Params { - width: packet.dimensions.0, - height: packet.dimensions.1, - chroma_width: packet.dimensions.0.div_ceil(2), - chroma_height: packet.dimensions.1, - mcus_per_row: packet.mcus_per_row, - mcu_rows: packet.mcu_rows, - restart_interval_mcus: packet.restart_interval_mcus, - restart_offset_count: entropy_segment_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ), - restart_start_mcu: 0, - entropy_len: u32::try_from(packet.entropy_bytes.len()) - .expect("JPEG Metal entropy payload fits in u32"), - out_stride: u32::try_from(out_stride).expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - out_format, - origin_x: 0, - origin_y: 0, - }) -} - -#[cfg(target_os = "macos")] -fn full_mcu_window_422(dims: (u32, u32), roi: signinum_jpeg::Rect) -> signinum_jpeg::Rect { - let x0 = (roi.x / 16) * 16; - let y0 = (roi.y / 8) * 8; - let x1 = (roi.x + roi.w).div_ceil(16) * 16; - let y1 = (roi.y + roi.h).div_ceil(8) * 8; - signinum_jpeg::Rect { - x: x0, - y: y0, - w: x1.min(dims.0).saturating_sub(x0), - h: y1.min(dims.1).saturating_sub(y0), - } -} - -#[cfg(target_os = "macos")] -fn fast422_region_params( - packet: &JpegMetalFast422PacketV1, - fmt: PixelFormat, - source_window: signinum_jpeg::Rect, -) -> Result { - let out_format = pixel_format_to_out_format(fmt).ok_or_else(|| Error::MetalKernel { - message: format!("unsupported JPEG Metal fast422 pixel format {fmt:?}"), - })?; - let out_stride = source_window.w as usize * fmt.bytes_per_pixel(); - Ok(JpegFast420Params { - width: source_window.w, - height: source_window.h, - chroma_width: source_window.w.div_ceil(2), - chroma_height: source_window.h, - mcus_per_row: packet.mcus_per_row, - mcu_rows: packet.mcu_rows, - restart_interval_mcus: packet.restart_interval_mcus, - restart_offset_count: entropy_segment_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ), - restart_start_mcu: 0, - entropy_len: u32::try_from(packet.entropy_bytes.len()) - .expect("JPEG Metal entropy payload fits in u32"), - out_stride: u32::try_from(out_stride).expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - out_format, - origin_x: source_window.x, - origin_y: source_window.y, - }) -} - -#[cfg(target_os = "macos")] -fn fast422_scaled_params( - packet: &JpegMetalFast422PacketV1, - scale: signinum_core::Downscale, -) -> Option { - let scale_shift = match scale { - signinum_core::Downscale::Half => 1, - signinum_core::Downscale::Quarter => 2, - signinum_core::Downscale::Eighth => 3, - _ => return None, - }; - let denom = 1u32 << scale_shift; - let scaled_width = packet.dimensions.0.div_ceil(denom); - let scaled_height = packet.dimensions.1.div_ceil(denom); - Some(JpegFast420ScaledParams { - scaled_width, - scaled_height, - chroma_width: scaled_width.div_ceil(2), - chroma_height: scaled_height, - mcus_per_row: packet.mcus_per_row, - mcu_rows: packet.mcu_rows, - restart_interval_mcus: packet.restart_interval_mcus, - restart_offset_count: entropy_segment_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ), - restart_start_mcu: 0, - entropy_len: u32::try_from(packet.entropy_bytes.len()) - .expect("JPEG Metal entropy payload fits in u32"), - scale_shift, - origin_x: 0, - origin_y: 0, - }) -} - -#[cfg(target_os = "macos")] -fn full_mcu_scaled_window_422( - scaled_dims: (u32, u32), - roi: signinum_jpeg::Rect, - scale_shift: u32, -) -> signinum_jpeg::Rect { - let mcu_width = 16u32 >> scale_shift; - let mcu_height = 8u32 >> scale_shift; - let x0 = (roi.x / mcu_width) * mcu_width; - let y0 = (roi.y / mcu_height) * mcu_height; - let x1 = (roi.x + roi.w).div_ceil(mcu_width) * mcu_width; - let y1 = (roi.y + roi.h).div_ceil(mcu_height) * mcu_height; - signinum_jpeg::Rect { - x: x0, - y: y0, - w: x1.min(scaled_dims.0).saturating_sub(x0), - h: y1.min(scaled_dims.1).saturating_sub(y0), - } -} - -#[cfg(target_os = "macos")] -fn fast422_scaled_region_params( - packet: &JpegMetalFast422PacketV1, - scale: signinum_core::Downscale, - source_window: signinum_jpeg::Rect, -) -> Option { - let full = fast422_scaled_params(packet, scale)?; - Some(JpegFast420ScaledParams { - scaled_width: source_window.w, - scaled_height: source_window.h, - chroma_width: source_window.w.div_ceil(2), - chroma_height: source_window.h, - origin_x: source_window.x, - origin_y: source_window.y, - ..full - }) -} - -#[cfg(target_os = "macos")] -fn full_mcu_window_420(dims: (u32, u32), roi: signinum_jpeg::Rect) -> signinum_jpeg::Rect { - let x0 = (roi.x / 16) * 16; - let y0 = (roi.y / 16) * 16; - let x1 = (roi.x + roi.w).div_ceil(16) * 16; - let y1 = (roi.y + roi.h).div_ceil(16) * 16; - signinum_jpeg::Rect { - x: x0, - y: y0, - w: x1.min(dims.0).saturating_sub(x0), - h: y1.min(dims.1).saturating_sub(y0), - } -} - -#[cfg(target_os = "macos")] -fn fast420_region_params( - packet: &JpegMetalFast420PacketV1, - fmt: PixelFormat, - source_window: signinum_jpeg::Rect, -) -> Result { - let out_format = pixel_format_to_out_format(fmt).ok_or_else(|| Error::MetalKernel { - message: format!("unsupported JPEG Metal fast420 pixel format {fmt:?}"), - })?; - let out_stride = source_window.w as usize * fmt.bytes_per_pixel(); - Ok(JpegFast420Params { - width: source_window.w, - height: source_window.h, - chroma_width: source_window.w.div_ceil(2), - chroma_height: source_window.h.div_ceil(2), - mcus_per_row: packet.mcus_per_row, - mcu_rows: packet.mcu_rows, - restart_interval_mcus: packet.restart_interval_mcus, - restart_offset_count: entropy_segment_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ), - restart_start_mcu: 0, - entropy_len: u32::try_from(packet.entropy_bytes.len()) - .expect("JPEG Metal entropy payload fits in u32"), - out_stride: u32::try_from(out_stride).expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - out_format, - origin_x: source_window.x, - origin_y: source_window.y, - }) -} - -#[cfg(target_os = "macos")] -fn fast420_scaled_params( - packet: &JpegMetalFast420PacketV1, - scale: signinum_core::Downscale, -) -> Option { - let scale_shift = match scale { - signinum_core::Downscale::Half => 1, - signinum_core::Downscale::Quarter => 2, - signinum_core::Downscale::Eighth => 3, - _ => return None, - }; - let denom = 1u32 << scale_shift; - let scaled_width = packet.dimensions.0.div_ceil(denom); - let scaled_height = packet.dimensions.1.div_ceil(denom); - Some(JpegFast420ScaledParams { - scaled_width, - scaled_height, - chroma_width: scaled_width.div_ceil(2), - chroma_height: scaled_height.div_ceil(2), - mcus_per_row: packet.mcus_per_row, - mcu_rows: packet.mcu_rows, - restart_interval_mcus: packet.restart_interval_mcus, - restart_offset_count: entropy_segment_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ), - restart_start_mcu: 0, - entropy_len: u32::try_from(packet.entropy_bytes.len()) - .expect("JPEG Metal entropy payload fits in u32"), - scale_shift, - origin_x: 0, - origin_y: 0, - }) -} - -#[cfg(target_os = "macos")] -fn full_mcu_scaled_window_420( - scaled_dims: (u32, u32), - roi: signinum_jpeg::Rect, - scale_shift: u32, -) -> signinum_jpeg::Rect { - let mcu_size = 16u32 >> scale_shift; - let x0 = (roi.x / mcu_size) * mcu_size; - let y0 = (roi.y / mcu_size) * mcu_size; - let x1 = (roi.x + roi.w).div_ceil(mcu_size) * mcu_size; - let y1 = (roi.y + roi.h).div_ceil(mcu_size) * mcu_size; - signinum_jpeg::Rect { - x: x0, - y: y0, - w: x1.min(scaled_dims.0).saturating_sub(x0), - h: y1.min(scaled_dims.1).saturating_sub(y0), - } -} - -#[cfg(target_os = "macos")] -fn fast420_scaled_region_params( - packet: &JpegMetalFast420PacketV1, - scale: signinum_core::Downscale, - source_window: signinum_jpeg::Rect, -) -> Option { - let full = fast420_scaled_params(packet, scale)?; - Some(JpegFast420ScaledParams { - scaled_width: source_window.w, - scaled_height: source_window.h, - chroma_width: source_window.w.div_ceil(2), - chroma_height: source_window.h.div_ceil(2), - origin_x: source_window.x, - origin_y: source_window.y, - ..full - }) -} - -#[cfg(target_os = "macos")] -fn fast444_params(packet: &JpegMetalFast444PacketV1) -> JpegFast444Params { - JpegFast444Params { - width: packet.dimensions.0, - height: packet.dimensions.1, - mcus_per_row: packet.mcus_per_row, - mcu_rows: packet.mcu_rows, - restart_interval_mcus: packet.restart_interval_mcus, - restart_offset_count: entropy_segment_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ), - restart_start_mcu: 0, - entropy_len: u32::try_from(packet.entropy_bytes.len()) - .expect("JPEG Metal fast444 entropy payload fits in u32"), - origin_x: 0, - origin_y: 0, - } -} - -#[cfg(target_os = "macos")] -fn fast444_region_params( - packet: &JpegMetalFast444PacketV1, - roi: signinum_jpeg::Rect, -) -> JpegFast444Params { - JpegFast444Params { - width: roi.w, - height: roi.h, - origin_x: roi.x, - origin_y: roi.y, - ..fast444_params(packet) - } -} - -#[cfg(target_os = "macos")] -fn fast444_scaled_params( - packet: &JpegMetalFast444PacketV1, - scale: signinum_core::Downscale, -) -> Option { - let scale_shift = match scale { - signinum_core::Downscale::Half => 1, - signinum_core::Downscale::Quarter => 2, - signinum_core::Downscale::Eighth => 3, - _ => return None, - }; - let denom = 1u32 << scale_shift; - Some(JpegFast444ScaledParams { - scaled_width: packet.dimensions.0.div_ceil(denom), - scaled_height: packet.dimensions.1.div_ceil(denom), - mcus_per_row: packet.mcus_per_row, - mcu_rows: packet.mcu_rows, - restart_interval_mcus: packet.restart_interval_mcus, - restart_offset_count: entropy_segment_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ), - restart_start_mcu: 0, - entropy_len: u32::try_from(packet.entropy_bytes.len()) - .expect("JPEG Metal fast444 entropy payload fits in u32"), - scale_shift, - origin_x: 0, - origin_y: 0, - }) -} - -#[cfg(target_os = "macos")] -fn fast444_scaled_region_params( - packet: &JpegMetalFast444PacketV1, - scale: signinum_core::Downscale, - roi: signinum_jpeg::Rect, -) -> Option { - Some(JpegFast444ScaledParams { - scaled_width: roi.w, - scaled_height: roi.h, - origin_x: roi.x, - origin_y: roi.y, - ..fast444_scaled_params(packet, scale)? - }) -} - -#[cfg(target_os = "macos")] -fn fast420_windowed_pack_params_for_dims( - dims: (u32, u32), - fmt: PixelFormat, - roi: signinum_jpeg::Rect, -) -> JpegFast420WindowedPackParams { - let out_format = pixel_format_to_out_format(fmt) - .ok_or_else(|| Error::MetalKernel { - message: format!("unsupported JPEG Metal fast420 pixel format {fmt:?}"), - }) - .expect("validated JPEG Metal fast420 pixel format"); - let out_stride = roi.w as usize * fmt.bytes_per_pixel(); - JpegFast420WindowedPackParams { - src_width: dims.0, - src_height: dims.1, - chroma_width: dims.0.div_ceil(2), - chroma_height: dims.1.div_ceil(2), - src_x: roi.x, - src_y: roi.y, - width: roi.w, - height: roi.h, - out_stride: u32::try_from(out_stride).expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - out_format, - } -} - -#[cfg(target_os = "macos")] -fn fast422_windowed_pack_params_for_dims( - dims: (u32, u32), - fmt: PixelFormat, - roi: signinum_jpeg::Rect, -) -> JpegFast420WindowedPackParams { - let out_format = pixel_format_to_out_format(fmt) - .ok_or_else(|| Error::MetalKernel { - message: format!("unsupported JPEG Metal fast422 pixel format {fmt:?}"), - }) - .expect("validated JPEG Metal fast422 pixel format"); - let out_stride = roi.w as usize * fmt.bytes_per_pixel(); - JpegFast420WindowedPackParams { - src_width: dims.0, - src_height: dims.1, - chroma_width: dims.0.div_ceil(2), - chroma_height: dims.1, - src_x: roi.x, - src_y: roi.y, - width: roi.w, - height: roi.h, - out_stride: u32::try_from(out_stride).expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - out_format, - } -} - -#[cfg(target_os = "macos")] -fn decode_error_from_cpu( - decoder: &CpuDecoder<'_>, - fmt: PixelFormat, - status: JpegDecodeStatus, -) -> Error { - if let Err(err) = decoder.decode(fmt) { - Error::Decode(err) - } else { - let reason = match status.code { - FAST420_STATUS_TRUNCATED => "truncated entropy stream", - FAST420_STATUS_HUFFMAN => "invalid Huffman stream", - _ => "unexpected Metal fast420 failure", - }; - Error::MetalKernel { - message: format!("{reason} at entropy byte {}", status.position), - } - } -} - -#[cfg(target_os = "macos")] -fn restart_offsets_buffer(device: &Device, restart_offsets: &[u32]) -> Result { - if restart_offsets.is_empty() { - return Err(Error::MetalKernel { - message: "JPEG Metal restart offsets must contain at least one entry".to_string(), - }); - } - Ok(device.new_buffer_with_data( - restart_offsets.as_ptr().cast(), - size_of_val(restart_offsets) as u64, - MTLResourceOptions::StorageModeShared, - )) -} - -#[cfg(target_os = "macos")] -fn entropy_checkpoints_buffer( - device: &Device, - entropy_checkpoints: &[JpegMetalEntropyCheckpointV1], -) -> Result { - if entropy_checkpoints.is_empty() { - return Err(Error::MetalKernel { - message: "JPEG Metal entropy checkpoints must contain at least one entry".to_string(), - }); - } - let checkpoints = entropy_checkpoints - .iter() - .copied() - .map(JpegEntropyCheckpointHost::from) - .collect::>(); - Ok(device.new_buffer_with_data( - checkpoints.as_ptr().cast(), - size_of_val(checkpoints.as_slice()) as u64, - MTLResourceOptions::StorageModeShared, - )) -} - -#[cfg(target_os = "macos")] -fn u32_buffer(device: &Device, values: &[u32], label: &str) -> Result { - if values.is_empty() { - return Err(Error::MetalKernel { - message: format!("JPEG Metal {label} buffer must contain at least one entry"), - }); - } - Ok(device.new_buffer_with_data( - values.as_ptr().cast(), - size_of_val(values) as u64, - MTLResourceOptions::StorageModeShared, - )) -} - -#[cfg(target_os = "macos")] -fn entropy_segment_count( - restart_interval_mcus: u32, - restart_offsets_len: usize, - entropy_checkpoints_len: usize, -) -> u32 { - let len = if restart_interval_mcus == 0 { - entropy_checkpoints_len - } else { - restart_offsets_len - }; - u32::try_from(len) - .expect("JPEG Metal entropy segment count fits in u32") - .max(1) -} - -#[cfg(target_os = "macos")] -fn restart_work_for_mcu_range( - restart_offsets: &[u32], - restart_interval_mcus: u32, - total_mcus: u32, - first_mcu: u32, - end_mcu: u32, -) -> (u32, &[u32]) { - if restart_interval_mcus == 0 || restart_offsets.len() <= 1 { - return (0, restart_offsets); - } - - let first_mcu = first_mcu.min(total_mcus); - let end_mcu = end_mcu.min(total_mcus).max(first_mcu + 1); - let restart_offset_count = - u32::try_from(restart_offsets.len()).expect("JPEG Metal restart offsets fit in u32"); - let first_segment = (first_mcu / restart_interval_mcus).min(restart_offset_count - 1); - let end_segment = end_mcu - .div_ceil(restart_interval_mcus) - .min(restart_offset_count) - .max(first_segment + 1); - ( - first_segment * restart_interval_mcus, - &restart_offsets[first_segment as usize..end_segment as usize], - ) -} - -#[cfg(target_os = "macos")] -fn mcu_range_for_rect( - rect: signinum_jpeg::Rect, - mcus_per_row: u32, - mcu_rows: u32, - mcu_width: u32, - mcu_height: u32, -) -> (u32, u32) { - if rect.w == 0 || rect.h == 0 || mcus_per_row == 0 || mcu_rows == 0 { - return (0, 0); - } - - let max_col = mcus_per_row - 1; - let max_row = mcu_rows - 1; - let last_x = rect.x.saturating_add(rect.w).saturating_sub(1); - let last_y = rect.y.saturating_add(rect.h).saturating_sub(1); - let first_col = (rect.x / mcu_width).min(max_col); - let last_col = (last_x / mcu_width).min(max_col); - let first_row = (rect.y / mcu_height).min(max_row); - let last_row = (last_y / mcu_height).min(max_row); - let first_mcu = first_row * mcus_per_row + first_col; - let end_mcu = last_row * mcus_per_row + last_col + 1; - (first_mcu, end_mcu) -} - -#[cfg(target_os = "macos")] -fn entropy_decode_thread_count( - restart_interval_mcus: u32, - restart_offsets_len: usize, - entropy_checkpoints_len: usize, -) -> u32 { - entropy_segment_count( - restart_interval_mcus, - restart_offsets_len, - entropy_checkpoints_len, - ) -} - -#[cfg(target_os = "macos")] -fn decode_status_buffer(device: &Device, count: u32) -> Buffer { - let statuses = vec![JpegDecodeStatus::default(); count as usize]; - device.new_buffer_with_data( - statuses.as_ptr().cast(), - size_of_val(statuses.as_slice()) as u64, - MTLResourceOptions::StorageModeShared, - ) -} - -#[cfg(target_os = "macos")] -fn first_decode_error_status(buffer: &Buffer, count: u32) -> Option { - let statuses = unsafe { - core::slice::from_raw_parts(buffer.contents().cast::(), count as usize) - }; - statuses - .iter() - .copied() - .find(|status| status.code != FAST420_STATUS_OK) -} - -#[cfg(target_os = "macos")] -enum BatchedFastPacket<'a> { - Fast420(&'a JpegMetalFast420PacketV1), - Fast422(&'a JpegMetalFast422PacketV1), - Fast444(&'a JpegMetalFast444PacketV1, PlaneMode), -} - -#[cfg(target_os = "macos")] -struct BatchedDecodeItem { - request_index: usize, - surface: Surface, - status_buffer: Buffer, - decode_threads: u32, - _decode_resources: Vec, -} - -#[cfg(target_os = "macos")] -#[derive(Default)] -struct BatchDeviceBufferCache { - packet_buffers: Vec, -} - -#[cfg(target_os = "macos")] -struct SharedPacketDeviceBuffers { - entropy_ptr: usize, - entropy_len: usize, - checkpoints_ptr: usize, - checkpoints_len: usize, - entropy_buffer: Buffer, - entropy_checkpoints_buffer: Buffer, -} - -#[cfg(target_os = "macos")] -impl BatchDeviceBufferCache { - fn packet_buffers( - &mut self, - runtime: &MetalRuntime, - entropy_bytes: &[u8], - entropy_checkpoints: &[JpegMetalEntropyCheckpointV1], - ) -> Result<(Buffer, Buffer), Error> { - let entropy_ptr = entropy_bytes.as_ptr() as usize; - let entropy_len = entropy_bytes.len(); - let checkpoints_ptr = entropy_checkpoints.as_ptr() as usize; - let checkpoints_len = entropy_checkpoints.len(); - if let Some(entry) = self.packet_buffers.iter().find(|entry| { - entry.entropy_ptr == entropy_ptr - && entry.entropy_len == entropy_len - && entry.checkpoints_ptr == checkpoints_ptr - && entry.checkpoints_len == checkpoints_len - }) { - return Ok(( - entry.entropy_buffer.clone(), - entry.entropy_checkpoints_buffer.clone(), - )); - } - - let entropy_buffer = runtime.device.new_buffer_with_data( - entropy_bytes.as_ptr().cast(), - entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, entropy_checkpoints)?; - self.packet_buffers.push(SharedPacketDeviceBuffers { - entropy_ptr, - entropy_len, - checkpoints_ptr, - checkpoints_len, - entropy_buffer: entropy_buffer.clone(), - entropy_checkpoints_buffer: entropy_checkpoints_buffer.clone(), - }); - Ok((entropy_buffer, entropy_checkpoints_buffer)) - } -} - -#[cfg(target_os = "macos")] -fn request_allows_batched_packet( - requests: &[batch::QueuedRequest], - request: &batch::QueuedRequest, - restart_interval_mcus: u32, - dimensions: (u32, u32), -) -> bool { - match request.backend { - BackendRequest::Metal => true, - BackendRequest::Auto => match request.op { - batch::BatchOp::RegionScaled { .. } => false, - _ => { - requests.len() >= AUTO_METAL_MIN_BATCH_REQUESTS - && (restart_interval_mcus != 0 - || auto_batch_work_is_large_enough(request, dimensions)) - } - }, - BackendRequest::Cpu | BackendRequest::Cuda => false, - } -} - -#[cfg(target_os = "macos")] -fn auto_batch_work_is_large_enough(request: &batch::QueuedRequest, dimensions: (u32, u32)) -> bool { - let dims = match request.op { - batch::BatchOp::Full | batch::BatchOp::Scaled(_) => dimensions, - batch::BatchOp::Region(roi) => (roi.w, roi.h), - batch::BatchOp::RegionScaled { .. } => return false, - }; - dims.0 >= AUTO_METAL_MIN_BATCH_EDGE && dims.1 >= AUTO_METAL_MIN_BATCH_EDGE -} - -#[cfg(target_os = "macos")] -fn batched_fast_packets( - requests: &[batch::QueuedRequest], -) -> Result>>, Error> { - if requests.len() < 2 { - return Ok(None); - } - - let mut packets = Vec::with_capacity(requests.len()); - for request in requests { - let batchable_op = match request.op { - batch::BatchOp::Full - | batch::BatchOp::Region(_) - | batch::BatchOp::Scaled( - signinum_core::Downscale::Half - | signinum_core::Downscale::Quarter - | signinum_core::Downscale::Eighth, - ) - | batch::BatchOp::RegionScaled { - scale: - signinum_core::Downscale::Half - | signinum_core::Downscale::Quarter - | signinum_core::Downscale::Eighth, - .. - } => true, - batch::BatchOp::Scaled(_) | batch::BatchOp::RegionScaled { .. } => false, - }; - if !batchable_op - || !matches!( - request.backend, - BackendRequest::Auto | BackendRequest::Metal - ) - || pixel_format_to_out_format(request.fmt).is_none() - { - return Ok(None); - } - - if let Some(packet) = request.fast420_packet.as_deref() { - if !request_allows_batched_packet( - requests, - request, - packet.restart_interval_mcus, - packet.dimensions, - ) { - return Ok(None); - } - packets.push(BatchedFastPacket::Fast420(packet)); - continue; - } - - if let Some(packet) = request.fast422_packet.as_deref() { - if !request_allows_batched_packet( - requests, - request, - packet.restart_interval_mcus, - packet.dimensions, - ) { - return Ok(None); - } - packets.push(BatchedFastPacket::Fast422(packet)); - continue; - } - - if let Some(packet) = request.fast444_packet.as_deref() { - if !request_allows_batched_packet( - requests, - request, - packet.restart_interval_mcus, - packet.dimensions, - ) { - return Ok(None); - } - let decoder = CpuDecoder::new(request.input.as_ref())?; - packets.push(BatchedFastPacket::Fast444( - packet, - fast444_plane_mode(&decoder), - )); - continue; - } - - return Ok(None); - } - - Ok(Some(packets)) -} - -#[cfg(target_os = "macos")] -fn core_rect_to_jpeg(rect: Rect) -> signinum_jpeg::Rect { - signinum_jpeg::Rect { - x: rect.x, - y: rect.y, - w: rect.w, - h: rect.h, - } -} - -#[cfg(target_os = "macos")] -#[allow(clippy::too_many_arguments)] -fn encode_jpeg_pack_to_surface_in_command_buffer( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - plane0: &Buffer, - plane1: Option<&Buffer>, - plane2: Option<&Buffer>, - dims: (u32, u32), - mode: PlaneMode, - fmt: PixelFormat, -) -> Result { - match (mode, fmt) { - (PlaneMode::Gray | PlaneMode::YCbCr, PixelFormat::Gray8) => { - return Ok(Surface::from_metal_buffer(plane0.clone(), dims, fmt)); - } - ( - PlaneMode::Gray | PlaneMode::YCbCr | PlaneMode::Rgb, - PixelFormat::Rgb8 | PixelFormat::Rgba8, - ) - | (PlaneMode::Rgb, PixelFormat::Gray8) => {} - _ => { - return Err(Error::MetalKernel { - message: format!("unsupported JPEG Metal pixel format {fmt:?}"), - }); - } - } - - let pitch_bytes = dims.0 as usize * fmt.bytes_per_pixel(); - let out_buffer = runtime.device.new_buffer( - (pitch_bytes * dims.1 as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - let params = JpegPackParams { - width: dims.0, - height: dims.1, - out_stride: u32::try_from(pitch_bytes).expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - mode: match mode { - PlaneMode::Gray => MODE_GRAY, - PlaneMode::YCbCr => MODE_YCBCR, - PlaneMode::Rgb => MODE_RGB, - }, - out_format: match fmt { - PixelFormat::Gray8 => OUT_GRAY, - PixelFormat::Rgb8 => OUT_RGB, - PixelFormat::Rgba8 => OUT_RGBA, - _ => unreachable!("validated by caller"), - }, - }; - - let encoder = command_buffer.new_compute_command_encoder(); - encoder.set_compute_pipeline_state(&runtime.pack_pipeline); - encoder.set_buffer(0, Some(plane0), 0); - encoder.set_buffer(1, plane1.map(std::convert::AsRef::as_ref), 0); - encoder.set_buffer(2, plane2.map(std::convert::AsRef::as_ref), 0); - encoder.set_buffer(3, Some(&out_buffer), 0); - encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - dispatch_2d_pipeline(encoder, &runtime.pack_pipeline, dims); - encoder.end_encoding(); - - Ok(Surface::from_metal_buffer(out_buffer, dims, fmt)) -} - -#[cfg(target_os = "macos")] -fn encode_fast420_region_batch_item( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - request_index: usize, - packet: &JpegMetalFast420PacketV1, - fmt: PixelFormat, - roi: Rect, -) -> Result { - let roi = core_rect_to_jpeg(roi); - let source_window = full_mcu_window_420(packet.dimensions, roi); - let mut params = fast420_region_params(packet, fmt, source_window)?; - let (first_mcu, end_mcu) = - mcu_range_for_rect(source_window, packet.mcus_per_row, packet.mcu_rows, 16, 16); - let total_mcus = packet.mcus_per_row * packet.mcu_rows; - let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( - &packet.restart_offsets, - packet.restart_interval_mcus, - total_mcus, - first_mcu, - end_mcu, - ); - params.restart_start_mcu = restart_start_mcu; - params.restart_offset_count = entropy_segment_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - - let local_roi = signinum_jpeg::Rect { - x: roi.x - source_window.x, - y: roi.y - source_window.y, - w: roi.w, - h: roi.h, - }; - let pack_params = - fast420_windowed_pack_params_for_dims((source_window.w, source_window.h), fmt, local_roi); - let y_len = source_window.w as usize * source_window.h as usize; - let chroma_len = source_window.w.div_ceil(2) as usize * source_window.h.div_ceil(2) as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, false); - let cb_plane = new_private_buffer(&runtime.device, chroma_len); - let cr_plane = new_private_buffer(&runtime.device, chroma_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast420_region_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast420_region_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let out_buffer = runtime.device.new_buffer( - (pack_params.out_stride as usize * roi.h as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - let pack_encoder = command_buffer.new_compute_command_encoder(); - let pack_pipeline = pack_420_windowed_pipeline_for_format(runtime, fmt); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const pack_params).cast(), - ); - dispatch_2d_pipeline(pack_encoder, pack_pipeline, (roi.w, roi.h)); - pack_encoder.end_encoding(); - - Ok(BatchedDecodeItem { - request_index, - surface: Surface::from_metal_buffer(out_buffer, (roi.w, roi.h), fmt), - status_buffer: status_buffer.clone(), - decode_threads, - _decode_resources: vec![ - y_plane, - cb_plane, - cr_plane, - entropy_buffer, - restart_offsets_buffer, - entropy_checkpoints_buffer, - status_buffer, - ], - }) -} - -#[cfg(target_os = "macos")] -fn encode_fast420_scaled_batch_item( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - request_index: usize, - packet: &JpegMetalFast420PacketV1, - fmt: PixelFormat, - scale: signinum_core::Downscale, -) -> Result { - let Some(params) = fast420_scaled_params(packet, scale) else { - return Err(Error::MetalKernel { - message: format!("unsupported JPEG Metal fast420 scale {scale:?}"), - }); - }; - - let y_len = params.scaled_width as usize * params.scaled_height as usize; - let chroma_len = params.chroma_width as usize * params.chroma_height as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, fmt == PixelFormat::Gray8); - let cb_plane = new_private_buffer(&runtime.device, chroma_len); - let cr_plane = new_private_buffer(&runtime.device, chroma_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast420_scaled_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast420_scaled_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let out_buffer = (fmt != PixelFormat::Gray8).then(|| { - runtime.device.new_buffer( - (params.scaled_width as usize * fmt.bytes_per_pixel() * params.scaled_height as usize) - as u64, - MTLResourceOptions::StorageModeShared, - ) - }); - - if let Some(out_buffer) = out_buffer.as_ref() { - let pack_params = JpegFast420Params { - width: params.scaled_width, - height: params.scaled_height, - chroma_width: params.chroma_width, - chroma_height: params.chroma_height, - mcus_per_row: params.mcus_per_row, - mcu_rows: params.mcu_rows, - restart_interval_mcus: params.restart_interval_mcus, - restart_offset_count: params.restart_offset_count, - restart_start_mcu: params.restart_start_mcu, - entropy_len: params.entropy_len, - out_stride: u32::try_from(params.scaled_width as usize * fmt.bytes_per_pixel()) - .expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - out_format: pixel_format_to_out_format(fmt).expect("validated output format"), - origin_x: 0, - origin_y: 0, - }; - let pack_encoder = command_buffer.new_compute_command_encoder(); - let pack_pipeline = pack_420_pipeline_for_format(runtime, fmt); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const pack_params).cast(), - ); - dispatch_2d_pipeline( - pack_encoder, - pack_pipeline, - (params.scaled_width, params.scaled_height), - ); - pack_encoder.end_encoding(); - } - - let surface = match out_buffer { - Some(out_buffer) => { - Surface::from_metal_buffer(out_buffer, (params.scaled_width, params.scaled_height), fmt) - } - None => Surface::from_metal_buffer( - y_plane.clone(), - (params.scaled_width, params.scaled_height), - fmt, - ), - }; - - Ok(BatchedDecodeItem { - request_index, - surface, - status_buffer: status_buffer.clone(), - decode_threads, - _decode_resources: vec![ - y_plane, - cb_plane, - cr_plane, - entropy_buffer, - restart_offsets_buffer, - entropy_checkpoints_buffer, - status_buffer, - ], - }) -} - -#[cfg(target_os = "macos")] -#[allow(clippy::too_many_arguments)] -fn encode_fast420_scaled_region_batch_item( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - device_buffer_cache: &mut BatchDeviceBufferCache, - request_index: usize, - packet: &JpegMetalFast420PacketV1, - fmt: PixelFormat, - roi: Rect, - scale: signinum_core::Downscale, -) -> Result { - let Some(full_params) = fast420_scaled_params(packet, scale) else { - return Err(Error::MetalKernel { - message: format!("unsupported JPEG Metal fast420 scale {scale:?}"), - }); - }; - let scaled_roi = roi.scaled_covering(scale); - let scaled_roi = signinum_jpeg::Rect { - x: scaled_roi.x, - y: scaled_roi.y, - w: scaled_roi.w, - h: scaled_roi.h, - }; - let source_window = full_mcu_scaled_window_420( - (full_params.scaled_width, full_params.scaled_height), - scaled_roi, - full_params.scale_shift, - ); - let Some(mut decode_params) = fast420_scaled_region_params(packet, scale, source_window) else { - return Err(Error::MetalKernel { - message: format!("unsupported JPEG Metal fast420 scaled region {scale:?}"), - }); - }; - let mcu_size = 16u32 >> decode_params.scale_shift; - let (first_mcu, end_mcu) = mcu_range_for_rect( - source_window, - packet.mcus_per_row, - packet.mcu_rows, - mcu_size, - mcu_size, - ); - let total_mcus = packet.mcus_per_row * packet.mcu_rows; - let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( - &packet.restart_offsets, - packet.restart_interval_mcus, - total_mcus, - first_mcu, - end_mcu, - ); - decode_params.restart_start_mcu = restart_start_mcu; - decode_params.restart_offset_count = entropy_segment_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let local_roi = signinum_jpeg::Rect { - x: scaled_roi.x - source_window.x, - y: scaled_roi.y - source_window.y, - w: scaled_roi.w, - h: scaled_roi.h, - }; - let pack_params = - fast420_windowed_pack_params_for_dims((source_window.w, source_window.h), fmt, local_roi); - let y_len = source_window.w as usize * source_window.h as usize; - let chroma_len = source_window.w.div_ceil(2) as usize * source_window.h.div_ceil(2) as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, false); - let cb_plane = new_private_buffer(&runtime.device, chroma_len); - let cr_plane = new_private_buffer(&runtime.device, chroma_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; - let (entropy_buffer, entropy_checkpoints_buffer) = device_buffer_cache.packet_buffers( - runtime, - &packet.entropy_bytes, - &packet.entropy_checkpoints, - )?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast420_scaled_region_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const decode_params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast420_scaled_region_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let out_buffer = runtime.device.new_buffer( - (pack_params.out_stride as usize * scaled_roi.h as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - let pack_encoder = command_buffer.new_compute_command_encoder(); - let pack_pipeline = pack_420_windowed_pipeline_for_format(runtime, fmt); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const pack_params).cast(), - ); - dispatch_2d_pipeline(pack_encoder, pack_pipeline, (scaled_roi.w, scaled_roi.h)); - pack_encoder.end_encoding(); - - Ok(BatchedDecodeItem { - request_index, - surface: Surface::from_metal_buffer(out_buffer, (scaled_roi.w, scaled_roi.h), fmt), - status_buffer: status_buffer.clone(), - decode_threads, - _decode_resources: vec![ - y_plane, - cb_plane, - cr_plane, - entropy_buffer, - restart_offsets_buffer, - entropy_checkpoints_buffer, - status_buffer, - ], - }) -} - -#[cfg(target_os = "macos")] -fn encode_fast420_batch_item( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - request_index: usize, - packet: &JpegMetalFast420PacketV1, - fmt: PixelFormat, -) -> Result { - let params = fast420_params(packet, fmt)?; - let y_len = params.width as usize * params.height as usize; - let chroma_len = params.chroma_width as usize * params.chroma_height as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, fmt == PixelFormat::Gray8); - let cb_plane = new_private_buffer(&runtime.device, chroma_len); - let cr_plane = new_private_buffer(&runtime.device, chroma_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast420_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast420_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let surface = if fmt == PixelFormat::Gray8 { - Surface::from_metal_buffer(y_plane.clone(), packet.dimensions, fmt) - } else { - let out_buffer = runtime.device.new_buffer( - (params.out_stride as usize * params.height as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - let pack_encoder = command_buffer.new_compute_command_encoder(); - let pack_pipeline = pack_420_pipeline_for_format(runtime, fmt); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - dispatch_2d_pipeline(pack_encoder, pack_pipeline, packet.dimensions); - pack_encoder.end_encoding(); - Surface::from_metal_buffer(out_buffer, packet.dimensions, fmt) - }; - - Ok(BatchedDecodeItem { - request_index, - surface, - status_buffer: status_buffer.clone(), - decode_threads, - _decode_resources: vec![ - y_plane, - cb_plane, - cr_plane, - entropy_buffer, - restart_offsets_buffer, - entropy_checkpoints_buffer, - status_buffer, - ], - }) -} - -#[cfg(target_os = "macos")] -fn encode_fast422_batch_item( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - request_index: usize, - packet: &JpegMetalFast422PacketV1, - fmt: PixelFormat, -) -> Result { - let params = fast422_params(packet, fmt)?; - let y_len = params.width as usize * params.height as usize; - let chroma_len = params.chroma_width as usize * params.chroma_height as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, fmt == PixelFormat::Gray8); - let cb_plane = new_private_buffer(&runtime.device, chroma_len); - let cr_plane = new_private_buffer(&runtime.device, chroma_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast422_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast422_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let surface = if fmt == PixelFormat::Gray8 { - Surface::from_metal_buffer(y_plane.clone(), packet.dimensions, fmt) - } else { - let Some(pack_pipeline) = pack_422_pipeline_for_format(runtime, fmt) else { - return Err(Error::MetalKernel { - message: format!("unsupported JPEG Metal fast422 pixel format {fmt:?}"), - }); - }; - let out_buffer = runtime.device.new_buffer( - (params.out_stride as usize * params.height as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - let pack_encoder = command_buffer.new_compute_command_encoder(); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - dispatch_2d_pipeline(pack_encoder, pack_pipeline, packet.dimensions); - pack_encoder.end_encoding(); - Surface::from_metal_buffer(out_buffer, packet.dimensions, fmt) - }; - - Ok(BatchedDecodeItem { - request_index, - surface, - status_buffer: status_buffer.clone(), - decode_threads, - _decode_resources: vec![ - y_plane, - cb_plane, - cr_plane, - entropy_buffer, - restart_offsets_buffer, - entropy_checkpoints_buffer, - status_buffer, - ], - }) -} - -#[cfg(target_os = "macos")] -fn encode_fast422_region_batch_item( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - request_index: usize, - packet: &JpegMetalFast422PacketV1, - fmt: PixelFormat, - roi: Rect, -) -> Result { - let roi = core_rect_to_jpeg(roi); - let source_window = full_mcu_window_422(packet.dimensions, roi); - let mut params = fast422_region_params(packet, fmt, source_window)?; - let (first_mcu, end_mcu) = - mcu_range_for_rect(source_window, packet.mcus_per_row, packet.mcu_rows, 16, 8); - let total_mcus = packet.mcus_per_row * packet.mcu_rows; - let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( - &packet.restart_offsets, - packet.restart_interval_mcus, - total_mcus, - first_mcu, - end_mcu, - ); - params.restart_start_mcu = restart_start_mcu; - params.restart_offset_count = entropy_segment_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - - let local_roi = signinum_jpeg::Rect { - x: roi.x - source_window.x, - y: roi.y - source_window.y, - w: roi.w, - h: roi.h, - }; - let pack_params = - fast422_windowed_pack_params_for_dims((source_window.w, source_window.h), fmt, local_roi); - let y_len = source_window.w as usize * source_window.h as usize; - let chroma_len = source_window.w.div_ceil(2) as usize * source_window.h as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, false); - let cb_plane = new_private_buffer(&runtime.device, chroma_len); - let cr_plane = new_private_buffer(&runtime.device, chroma_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast422_region_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast422_region_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let out_buffer = runtime.device.new_buffer( - (pack_params.out_stride as usize * roi.h as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - let pack_encoder = command_buffer.new_compute_command_encoder(); - let pack_pipeline = pack_422_windowed_pipeline_for_format(runtime, fmt); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const pack_params).cast(), - ); - dispatch_2d_pipeline(pack_encoder, pack_pipeline, (roi.w, roi.h)); - pack_encoder.end_encoding(); - - Ok(BatchedDecodeItem { - request_index, - surface: Surface::from_metal_buffer(out_buffer, (roi.w, roi.h), fmt), - status_buffer: status_buffer.clone(), - decode_threads, - _decode_resources: vec![ - y_plane, - cb_plane, - cr_plane, - entropy_buffer, - restart_offsets_buffer, - entropy_checkpoints_buffer, - status_buffer, - ], - }) -} - -#[cfg(target_os = "macos")] -fn encode_fast422_scaled_batch_item( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - request_index: usize, - packet: &JpegMetalFast422PacketV1, - fmt: PixelFormat, - scale: signinum_core::Downscale, -) -> Result { - let Some(params) = fast422_scaled_params(packet, scale) else { - return Err(Error::MetalKernel { - message: format!("unsupported JPEG Metal fast422 scale {scale:?}"), - }); - }; - - let y_len = params.scaled_width as usize * params.scaled_height as usize; - let chroma_len = params.chroma_width as usize * params.chroma_height as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, fmt == PixelFormat::Gray8); - let cb_plane = new_private_buffer(&runtime.device, chroma_len); - let cr_plane = new_private_buffer(&runtime.device, chroma_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast422_scaled_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast422_scaled_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let out_buffer = (fmt != PixelFormat::Gray8).then(|| { - runtime.device.new_buffer( - (params.scaled_width as usize * fmt.bytes_per_pixel() * params.scaled_height as usize) - as u64, - MTLResourceOptions::StorageModeShared, - ) - }); - - if let Some(out_buffer) = out_buffer.as_ref() { - let pack_params = JpegFast420Params { - width: params.scaled_width, - height: params.scaled_height, - chroma_width: params.chroma_width, - chroma_height: params.chroma_height, - mcus_per_row: params.mcus_per_row, - mcu_rows: params.mcu_rows, - restart_interval_mcus: params.restart_interval_mcus, - restart_offset_count: params.restart_offset_count, - restart_start_mcu: params.restart_start_mcu, - entropy_len: params.entropy_len, - out_stride: u32::try_from(params.scaled_width as usize * fmt.bytes_per_pixel()) - .expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - out_format: pixel_format_to_out_format(fmt).expect("validated output format"), - origin_x: 0, - origin_y: 0, - }; - let pack_encoder = command_buffer.new_compute_command_encoder(); - let Some(pack_pipeline) = pack_422_pipeline_for_format(runtime, fmt) else { - return Err(Error::MetalKernel { - message: format!("unsupported JPEG Metal fast422 pixel format {fmt:?}"), - }); - }; - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const pack_params).cast(), - ); - dispatch_2d_pipeline( - pack_encoder, - pack_pipeline, - (params.scaled_width, params.scaled_height), - ); - pack_encoder.end_encoding(); - } - - let surface = match out_buffer { - Some(out_buffer) => { - Surface::from_metal_buffer(out_buffer, (params.scaled_width, params.scaled_height), fmt) - } - None => Surface::from_metal_buffer( - y_plane.clone(), - (params.scaled_width, params.scaled_height), - fmt, - ), - }; - - Ok(BatchedDecodeItem { - request_index, - surface, - status_buffer: status_buffer.clone(), - decode_threads, - _decode_resources: vec![ - y_plane, - cb_plane, - cr_plane, - entropy_buffer, - restart_offsets_buffer, - entropy_checkpoints_buffer, - status_buffer, - ], - }) -} - -#[cfg(target_os = "macos")] -#[allow(clippy::too_many_arguments)] -fn encode_fast422_scaled_region_batch_item( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - device_buffer_cache: &mut BatchDeviceBufferCache, - request_index: usize, - packet: &JpegMetalFast422PacketV1, - fmt: PixelFormat, - roi: Rect, - scale: signinum_core::Downscale, -) -> Result { - let Some(full_params) = fast422_scaled_params(packet, scale) else { - return Err(Error::MetalKernel { - message: format!("unsupported JPEG Metal fast422 scale {scale:?}"), - }); - }; - let scaled_roi = roi.scaled_covering(scale); - let scaled_roi = signinum_jpeg::Rect { - x: scaled_roi.x, - y: scaled_roi.y, - w: scaled_roi.w, - h: scaled_roi.h, - }; - let source_window = full_mcu_scaled_window_422( - (full_params.scaled_width, full_params.scaled_height), - scaled_roi, - full_params.scale_shift, - ); - let Some(mut decode_params) = fast422_scaled_region_params(packet, scale, source_window) else { - return Err(Error::MetalKernel { - message: format!("unsupported JPEG Metal fast422 scaled region {scale:?}"), - }); - }; - let mcu_width = 16u32 >> decode_params.scale_shift; - let mcu_height = 8u32 >> decode_params.scale_shift; - let (first_mcu, end_mcu) = mcu_range_for_rect( - source_window, - packet.mcus_per_row, - packet.mcu_rows, - mcu_width, - mcu_height, - ); - let total_mcus = packet.mcus_per_row * packet.mcu_rows; - let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( - &packet.restart_offsets, - packet.restart_interval_mcus, - total_mcus, - first_mcu, - end_mcu, - ); - decode_params.restart_start_mcu = restart_start_mcu; - decode_params.restart_offset_count = entropy_segment_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let local_roi = signinum_jpeg::Rect { - x: scaled_roi.x - source_window.x, - y: scaled_roi.y - source_window.y, - w: scaled_roi.w, - h: scaled_roi.h, - }; - let pack_params = - fast422_windowed_pack_params_for_dims((source_window.w, source_window.h), fmt, local_roi); - let y_len = source_window.w as usize * source_window.h as usize; - let chroma_len = source_window.w.div_ceil(2) as usize * source_window.h as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, false); - let cb_plane = new_private_buffer(&runtime.device, chroma_len); - let cr_plane = new_private_buffer(&runtime.device, chroma_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; - let (entropy_buffer, entropy_checkpoints_buffer) = device_buffer_cache.packet_buffers( - runtime, - &packet.entropy_bytes, - &packet.entropy_checkpoints, - )?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast422_scaled_region_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const decode_params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast422_scaled_region_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let out_buffer = runtime.device.new_buffer( - (pack_params.out_stride as usize * scaled_roi.h as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - let pack_encoder = command_buffer.new_compute_command_encoder(); - let pack_pipeline = pack_422_windowed_pipeline_for_format(runtime, fmt); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const pack_params).cast(), - ); - dispatch_2d_pipeline(pack_encoder, pack_pipeline, (scaled_roi.w, scaled_roi.h)); - pack_encoder.end_encoding(); - - Ok(BatchedDecodeItem { - request_index, - surface: Surface::from_metal_buffer(out_buffer, (scaled_roi.w, scaled_roi.h), fmt), - status_buffer: status_buffer.clone(), - decode_threads, - _decode_resources: vec![ - y_plane, - cb_plane, - cr_plane, - entropy_buffer, - restart_offsets_buffer, - entropy_checkpoints_buffer, - status_buffer, - ], - }) -} - -#[cfg(target_os = "macos")] -fn encode_fast444_region_batch_item( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - request_index: usize, - packet: &JpegMetalFast444PacketV1, - mode: PlaneMode, - fmt: PixelFormat, - roi: Rect, -) -> Result { - let roi = core_rect_to_jpeg(roi); - let mut params = fast444_region_params(packet, roi); - let (first_mcu, end_mcu) = mcu_range_for_rect(roi, packet.mcus_per_row, packet.mcu_rows, 8, 8); - let total_mcus = packet.mcus_per_row * packet.mcu_rows; - let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( - &packet.restart_offsets, - packet.restart_interval_mcus, - total_mcus, - first_mcu, - end_mcu, - ); - params.restart_start_mcu = restart_start_mcu; - params.restart_offset_count = entropy_segment_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - - let plane_len = params.width as usize * params.height as usize; - let y_plane = new_decode_plane_buffer( - &runtime.device, - plane_len, - fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, - ); - let cb_plane = new_private_buffer(&runtime.device, plane_len); - let cr_plane = new_private_buffer(&runtime.device, plane_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast444_region_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast444_region_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let surface = encode_jpeg_pack_to_surface_in_command_buffer( - runtime, - command_buffer, - &y_plane, - Some(&cb_plane), - Some(&cr_plane), - (roi.w, roi.h), - mode, - fmt, - )?; - - Ok(BatchedDecodeItem { - request_index, - surface, - status_buffer: status_buffer.clone(), - decode_threads, - _decode_resources: vec![ - y_plane, - cb_plane, - cr_plane, - entropy_buffer, - restart_offsets_buffer, - entropy_checkpoints_buffer, - status_buffer, - ], - }) -} - -#[cfg(target_os = "macos")] -fn encode_fast444_scaled_batch_item( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - request_index: usize, - packet: &JpegMetalFast444PacketV1, - mode: PlaneMode, - fmt: PixelFormat, - scale: signinum_core::Downscale, -) -> Result { - let Some(params) = fast444_scaled_params(packet, scale) else { - return Err(Error::MetalKernel { - message: format!("unsupported JPEG Metal fast444 scale {scale:?}"), - }); - }; - - let plane_len = params.scaled_width as usize * params.scaled_height as usize; - let y_plane = new_decode_plane_buffer( - &runtime.device, - plane_len, - fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, - ); - let cb_plane = new_private_buffer(&runtime.device, plane_len); - let cr_plane = new_private_buffer(&runtime.device, plane_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast444_scaled_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast444_scaled_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let surface = encode_jpeg_pack_to_surface_in_command_buffer( - runtime, - command_buffer, - &y_plane, - Some(&cb_plane), - Some(&cr_plane), - (params.scaled_width, params.scaled_height), - mode, - fmt, - )?; - - Ok(BatchedDecodeItem { - request_index, - surface, - status_buffer: status_buffer.clone(), - decode_threads, - _decode_resources: vec![ - y_plane, - cb_plane, - cr_plane, - entropy_buffer, - restart_offsets_buffer, - entropy_checkpoints_buffer, - status_buffer, - ], - }) -} - -#[cfg(target_os = "macos")] -#[allow(clippy::too_many_arguments)] -fn encode_fast444_scaled_region_batch_item( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - device_buffer_cache: &mut BatchDeviceBufferCache, - request_index: usize, - packet: &JpegMetalFast444PacketV1, - mode: PlaneMode, - fmt: PixelFormat, - roi: Rect, - scale: signinum_core::Downscale, -) -> Result { - let scaled_roi = roi.scaled_covering(scale); - let scaled_roi = signinum_jpeg::Rect { - x: scaled_roi.x, - y: scaled_roi.y, - w: scaled_roi.w, - h: scaled_roi.h, - }; - let Some(mut params) = fast444_scaled_region_params(packet, scale, scaled_roi) else { - return Err(Error::MetalKernel { - message: format!("unsupported JPEG Metal fast444 scaled region {scale:?}"), - }); - }; - let mcu_size = 8u32 >> params.scale_shift; - let (first_mcu, end_mcu) = mcu_range_for_rect( - scaled_roi, - packet.mcus_per_row, - packet.mcu_rows, - mcu_size, - mcu_size, - ); - let total_mcus = packet.mcus_per_row * packet.mcu_rows; - let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( - &packet.restart_offsets, - packet.restart_interval_mcus, - total_mcus, - first_mcu, - end_mcu, - ); - params.restart_start_mcu = restart_start_mcu; - params.restart_offset_count = entropy_segment_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - - let plane_len = params.scaled_width as usize * params.scaled_height as usize; - let y_plane = new_decode_plane_buffer( - &runtime.device, - plane_len, - fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, - ); - let cb_plane = new_private_buffer(&runtime.device, plane_len); - let cr_plane = new_private_buffer(&runtime.device, plane_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; - let (entropy_buffer, entropy_checkpoints_buffer) = device_buffer_cache.packet_buffers( - runtime, - &packet.entropy_bytes, - &packet.entropy_checkpoints, - )?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast444_scaled_region_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast444_scaled_region_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let surface = encode_jpeg_pack_to_surface_in_command_buffer( - runtime, - command_buffer, - &y_plane, - Some(&cb_plane), - Some(&cr_plane), - (scaled_roi.w, scaled_roi.h), - mode, - fmt, - )?; - - Ok(BatchedDecodeItem { - request_index, - surface, - status_buffer: status_buffer.clone(), - decode_threads, - _decode_resources: vec![ - y_plane, - cb_plane, - cr_plane, - entropy_buffer, - restart_offsets_buffer, - entropy_checkpoints_buffer, - status_buffer, - ], - }) -} - -#[cfg(target_os = "macos")] -fn encode_fast444_batch_item( - runtime: &MetalRuntime, - command_buffer: &CommandBufferRef, - request_index: usize, - packet: &JpegMetalFast444PacketV1, - mode: PlaneMode, - fmt: PixelFormat, -) -> Result { - let params = fast444_params(packet); - let plane_len = params.width as usize * params.height as usize; - let y_plane = new_decode_plane_buffer( - &runtime.device, - plane_len, - fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, - ); - let cb_plane = new_private_buffer(&runtime.device, plane_len); - let cr_plane = new_private_buffer(&runtime.device, plane_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast444_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast444_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let surface = encode_jpeg_pack_to_surface_in_command_buffer( - runtime, - command_buffer, - &y_plane, - Some(&cb_plane), - Some(&cr_plane), - packet.dimensions, - mode, - fmt, - )?; - - Ok(BatchedDecodeItem { - request_index, - surface, - status_buffer: status_buffer.clone(), - decode_threads, - _decode_resources: vec![ - y_plane, - cb_plane, - cr_plane, - entropy_buffer, - restart_offsets_buffer, - entropy_checkpoints_buffer, - status_buffer, - ], - }) -} - -#[cfg(target_os = "macos")] -fn fast420_packets_share_full_rgb_batch_shape( - first: &JpegMetalFast420PacketV1, - packet: &JpegMetalFast420PacketV1, - segment_count: usize, -) -> bool { - packet.restart_interval_mcus == 0 - && packet.dimensions == first.dimensions - && packet.mcus_per_row == first.mcus_per_row - && packet.mcu_rows == first.mcu_rows - && packet.entropy_checkpoints.len() == segment_count - && packet.y_quant == first.y_quant - && packet.cb_quant == first.cb_quant - && packet.cr_quant == first.cr_quant - && packet.y_dc_table == first.y_dc_table - && packet.y_ac_table == first.y_ac_table - && packet.cb_dc_table == first.cb_dc_table - && packet.cb_ac_table == first.cb_ac_table - && packet.cr_dc_table == first.cr_dc_table - && packet.cr_ac_table == first.cr_ac_table -} - -#[cfg(target_os = "macos")] -fn checked_u32(value: usize, label: &str) -> Result { - u32::try_from(value).map_err(|_| Error::MetalKernel { - message: format!("JPEG Metal {label} does not fit in u32"), - }) -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Fast420BatchDecodeMode { - Fused, - SplitCoeffIdct, -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Copy, Debug, Default)] -struct Fast420BatchTiming { - accepted: Duration, - entropy_concat: Duration, - buffer_alloc: Duration, - encode_decode: Duration, - wait_decode: Duration, - encode_pack: Duration, - wait_pack: Duration, - total: Duration, -} - -#[cfg(target_os = "macos")] -impl Fast420BatchTiming { - fn micros(duration: Duration) -> u128 { - duration.as_micros() - } - - fn log(self, label: &str, tile_count: usize, dimensions: (u32, u32), segment_count: usize) { - eprintln!( - concat!( - "JPEG Metal fast420 batch timing ", - "mode={} tiles={} dims={}x{} segments={} ", - "accepted_us={} entropy_concat_us={} buffer_alloc_us={} ", - "encode_decode_us={} wait_decode_us={} ", - "encode_pack_us={} wait_pack_us={} total_us={}" - ), - label, - tile_count, - dimensions.0, - dimensions.1, - segment_count, - Self::micros(self.accepted), - Self::micros(self.entropy_concat), - Self::micros(self.buffer_alloc), - Self::micros(self.encode_decode), - Self::micros(self.wait_decode), - Self::micros(self.encode_pack), - Self::micros(self.wait_pack), - Self::micros(self.total) - ); - } -} - -#[cfg(target_os = "macos")] -fn fast420_batch_decode_mode() -> Fast420BatchDecodeMode { - if split_fast420_batch_enabled() { - Fast420BatchDecodeMode::SplitCoeffIdct - } else { - Fast420BatchDecodeMode::Fused - } -} - -#[cfg(target_os = "macos")] -struct BatchEntropyBuffers { - payload: Buffer, - offsets: Buffer, - lens: Buffer, - checkpoints: Buffer, -} - -#[cfg(target_os = "macos")] -fn batch_entropy_buffers<'a>( - runtime: &MetalRuntime, - entropy_bytes_iter: impl Iterator + Clone, - entropy_checkpoints_iter: impl Iterator + Clone, - tile_count: usize, - segment_count: usize, -) -> Result, Error> { - let total_entropy_len = entropy_bytes_iter - .clone() - .map(<[u8]>::len) - .try_fold(0usize, usize::checked_add) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal region scaled batch entropy length overflowed".to_string(), - })?; - if total_entropy_len == 0 { - return Ok(None); - } - - let mut entropy_bytes = Vec::with_capacity(total_entropy_len); - let mut entropy_offsets = Vec::with_capacity(tile_count); - let mut entropy_lens = Vec::with_capacity(tile_count); - let mut entropy_checkpoints = Vec::with_capacity(tile_count * segment_count); - for (bytes, checkpoints) in entropy_bytes_iter.zip(entropy_checkpoints_iter) { - entropy_offsets.push(checked_u32( - entropy_bytes.len(), - "region scaled batch entropy offset", - )?); - entropy_lens.push(checked_u32( - bytes.len(), - "region scaled batch entropy length", - )?); - entropy_bytes.extend_from_slice(bytes); - entropy_checkpoints.extend(checkpoints.iter().copied()); - } - - let entropy_buffer = runtime.device.new_buffer_with_data( - entropy_bytes.as_ptr().cast(), - entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - Ok(Some(BatchEntropyBuffers { - payload: entropy_buffer, - offsets: u32_buffer( - &runtime.device, - &entropy_offsets, - "region scaled batch entropy offsets", - )?, - lens: u32_buffer( - &runtime.device, - &entropy_lens, - "region scaled batch entropy lengths", - )?, - checkpoints: entropy_checkpoints_buffer(&runtime.device, &entropy_checkpoints)?, - })) -} - -#[cfg(target_os = "macos")] -fn region_scaled_batch_error_results( - requests: &[batch::QueuedRequest], - status_buffer: &Buffer, - total_decode_threads: u32, -) -> Result>>, Error> { - let Some(status) = first_decode_error_status(status_buffer, total_decode_threads) else { - return Ok(None); - }; - let mut results = Vec::with_capacity(requests.len()); - for request in requests { - let decoder = CpuDecoder::new(request.input.as_ref())?; - results.push(Err(decode_error_from_cpu(&decoder, request.fmt, status))); - } - Ok(Some(results)) -} - -#[cfg(target_os = "macos")] -#[derive(Clone, Copy, PartialEq, Eq)] -struct RegionScaledBatchPlan { - decode_params: JpegFastRegionScaledBatchParams, - pack_params: JpegWindowedPackBatchParams, - y_len: usize, - chroma_len: usize, - out_tile_len: usize, - out_dims: (u32, u32), -} - -#[cfg(target_os = "macos")] -fn fast420_region_scaled_batch_plan( - packet: &JpegMetalFast420PacketV1, - roi: Rect, - scale: signinum_core::Downscale, - tile_count: u32, - segment_count: u32, -) -> Option { - let full_params = fast420_scaled_params(packet, scale)?; - let scaled_roi = roi.scaled_covering(scale); - let scaled_roi = signinum_jpeg::Rect { - x: scaled_roi.x, - y: scaled_roi.y, - w: scaled_roi.w, - h: scaled_roi.h, - }; - let source_window = full_mcu_scaled_window_420( - (full_params.scaled_width, full_params.scaled_height), - scaled_roi, - full_params.scale_shift, - ); - let decode_params = fast420_scaled_region_params(packet, scale, source_window)?; - let local_roi = signinum_jpeg::Rect { - x: scaled_roi.x - source_window.x, - y: scaled_roi.y - source_window.y, - w: scaled_roi.w, - h: scaled_roi.h, - }; - let pack_params = fast420_windowed_pack_params_for_dims( - (source_window.w, source_window.h), - PixelFormat::Rgb8, - local_roi, - ); - let out_stride = scaled_roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); - Some(RegionScaledBatchPlan { - decode_params: JpegFastRegionScaledBatchParams { - scaled_width: decode_params.scaled_width, - scaled_height: decode_params.scaled_height, - chroma_width: decode_params.chroma_width, - chroma_height: decode_params.chroma_height, - mcus_per_row: decode_params.mcus_per_row, - mcu_rows: decode_params.mcu_rows, - segment_count, - tile_count, - scale_shift: decode_params.scale_shift, - origin_x: decode_params.origin_x, - origin_y: decode_params.origin_y, - }, - pack_params: JpegWindowedPackBatchParams { - src_width: pack_params.src_width, - src_height: pack_params.src_height, - chroma_width: pack_params.chroma_width, - chroma_height: pack_params.chroma_height, - src_x: pack_params.src_x, - src_y: pack_params.src_y, - width: pack_params.width, - height: pack_params.height, - tile_count, - out_stride: u32::try_from(out_stride).expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - mode: MODE_YCBCR, - out_format: OUT_RGB, - }, - y_len: source_window.w as usize * source_window.h as usize, - chroma_len: source_window.w.div_ceil(2) as usize * source_window.h.div_ceil(2) as usize, - out_tile_len: out_stride * scaled_roi.h as usize, - out_dims: (scaled_roi.w, scaled_roi.h), - }) -} - -#[cfg(target_os = "macos")] -fn fast422_region_scaled_batch_plan( - packet: &JpegMetalFast422PacketV1, - roi: Rect, - scale: signinum_core::Downscale, - tile_count: u32, - segment_count: u32, -) -> Option { - let full_params = fast422_scaled_params(packet, scale)?; - let scaled_roi = roi.scaled_covering(scale); - let scaled_roi = signinum_jpeg::Rect { - x: scaled_roi.x, - y: scaled_roi.y, - w: scaled_roi.w, - h: scaled_roi.h, - }; - let source_window = full_mcu_scaled_window_422( - (full_params.scaled_width, full_params.scaled_height), - scaled_roi, - full_params.scale_shift, - ); - let decode_params = fast422_scaled_region_params(packet, scale, source_window)?; - let local_roi = signinum_jpeg::Rect { - x: scaled_roi.x - source_window.x, - y: scaled_roi.y - source_window.y, - w: scaled_roi.w, - h: scaled_roi.h, - }; - let pack_params = fast422_windowed_pack_params_for_dims( - (source_window.w, source_window.h), - PixelFormat::Rgb8, - local_roi, - ); - let out_stride = scaled_roi.w as usize * PixelFormat::Rgb8.bytes_per_pixel(); - Some(RegionScaledBatchPlan { - decode_params: JpegFastRegionScaledBatchParams { - scaled_width: decode_params.scaled_width, - scaled_height: decode_params.scaled_height, - chroma_width: decode_params.chroma_width, - chroma_height: decode_params.chroma_height, - mcus_per_row: decode_params.mcus_per_row, - mcu_rows: decode_params.mcu_rows, - segment_count, - tile_count, - scale_shift: decode_params.scale_shift, - origin_x: decode_params.origin_x, - origin_y: decode_params.origin_y, - }, - pack_params: JpegWindowedPackBatchParams { - src_width: pack_params.src_width, - src_height: pack_params.src_height, - chroma_width: pack_params.chroma_width, - chroma_height: pack_params.chroma_height, - src_x: pack_params.src_x, - src_y: pack_params.src_y, - width: pack_params.width, - height: pack_params.height, - tile_count, - out_stride: u32::try_from(out_stride).expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - mode: MODE_YCBCR, - out_format: OUT_RGB, - }, - y_len: source_window.w as usize * source_window.h as usize, - chroma_len: source_window.w.div_ceil(2) as usize * source_window.h as usize, - out_tile_len: out_stride * scaled_roi.h as usize, - out_dims: (scaled_roi.w, scaled_roi.h), - }) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast420_full_rgb_batch_to_surfaces( - runtime: &MetalRuntime, - requests: &[batch::QueuedRequest], - packets: &[BatchedFastPacket<'_>], -) -> Result>>, Error> { - try_decode_fast420_full_rgb_batch_to_surfaces_with_mode( - runtime, - requests, - packets, - fast420_batch_decode_mode(), - ) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast420_full_rgb_batch_to_surfaces_with_mode( - runtime: &MetalRuntime, - requests: &[batch::QueuedRequest], - packets: &[BatchedFastPacket<'_>], - decode_mode: Fast420BatchDecodeMode, -) -> Result>>, Error> { - let timing_enabled = - decode_mode == Fast420BatchDecodeMode::Fused && fast420_batch_timing_enabled(); - let timing_total_start = timing_enabled.then(Instant::now); - let mut timing = Fast420BatchTiming::default(); - - if requests.len() < 2 - || requests - .iter() - .any(|request| request.op != batch::BatchOp::Full || request.fmt != PixelFormat::Rgb8) - { - return Ok(None); - } - - let mut fast420_packets = Vec::with_capacity(packets.len()); - for packet in packets { - let BatchedFastPacket::Fast420(packet) = packet else { - return Ok(None); - }; - fast420_packets.push(*packet); - } - - let Some(first) = fast420_packets.first().copied() else { - return Ok(None); - }; - if first.restart_interval_mcus != 0 || first.entropy_checkpoints.is_empty() { - return Ok(None); - } - - let segment_count = first.entropy_checkpoints.len(); - if !fast420_packets - .iter() - .all(|packet| fast420_packets_share_full_rgb_batch_shape(first, packet, segment_count)) - { - return Ok(None); - } - - let tile_count = fast420_packets.len(); - let tile_count_u32 = checked_u32(tile_count, "batch tile count")?; - let segment_count_u32 = checked_u32(segment_count, "batch segment count")?; - let total_decode_threads = checked_u32( - tile_count - .checked_mul(segment_count) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal batch decode thread count overflowed".to_string(), - })?, - "batch decode thread count", - )?; - - let width = first.dimensions.0; - let height = first.dimensions.1; - let chroma_width = width.div_ceil(2); - let chroma_height = height.div_ceil(2); - let y_len = width as usize * height as usize; - let chroma_len = chroma_width as usize * chroma_height as usize; - let out_stride = width as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let out_tile_len = out_stride * height as usize; - let total_mcus = first.mcus_per_row as usize * first.mcu_rows as usize; - let blocks_per_tile = total_mcus - .checked_mul(6) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal fast420 batch block count overflowed".to_string(), - })?; - let total_blocks = - blocks_per_tile - .checked_mul(tile_count) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal fast420 batch total block count overflowed".to_string(), - })?; - let _total_blocks_u32 = checked_u32(total_blocks, "fast420 batch block count")?; - - let params = JpegFast420BatchParams { - width, - height, - chroma_width, - chroma_height, - mcus_per_row: first.mcus_per_row, - mcu_rows: first.mcu_rows, - segment_count: segment_count_u32, - tile_count: tile_count_u32, - out_stride: checked_u32(out_stride, "batch output stride")?, - alpha: u32::from(u8::MAX), - }; - if timing_enabled { - timing.accepted = timing_total_start - .expect("timing start is set when timing is enabled") - .elapsed(); - } - - let timing_entropy_start = timing_enabled.then(Instant::now); - let total_entropy_len = fast420_packets - .iter() - .map(|packet| packet.entropy_bytes.len()) - .try_fold(0usize, usize::checked_add) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal batch entropy length overflowed".to_string(), - })?; - if total_entropy_len == 0 { - return Ok(None); - } - - let mut entropy_bytes = Vec::with_capacity(total_entropy_len); - let mut entropy_offsets = Vec::with_capacity(tile_count); - let mut entropy_lens = Vec::with_capacity(tile_count); - let mut entropy_checkpoints = Vec::with_capacity(tile_count * segment_count); - for packet in &fast420_packets { - entropy_offsets.push(checked_u32(entropy_bytes.len(), "batch entropy offset")?); - entropy_lens.push(checked_u32( - packet.entropy_bytes.len(), - "batch entropy length", - )?); - entropy_bytes.extend_from_slice(&packet.entropy_bytes); - entropy_checkpoints.extend(packet.entropy_checkpoints.iter().copied()); - } - if timing_enabled { - timing.entropy_concat = timing_entropy_start - .expect("timing start is set when timing is enabled") - .elapsed(); - } - - let timing_buffer_start = timing_enabled.then(Instant::now); - let y_plane = new_private_buffer(&runtime.device, y_len * tile_count); - let cb_plane = new_private_buffer(&runtime.device, chroma_len * tile_count); - let cr_plane = new_private_buffer(&runtime.device, chroma_len * tile_count); - let out_buffer = runtime.device.new_buffer( - (out_tile_len * tile_count) as u64, - MTLResourceOptions::StorageModeShared, - ); - let status_buffer = decode_status_buffer(&runtime.device, total_decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - entropy_bytes.as_ptr().cast(), - entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let entropy_offsets_buffer = - u32_buffer(&runtime.device, &entropy_offsets, "batch entropy offsets")?; - let entropy_lens_buffer = u32_buffer(&runtime.device, &entropy_lens, "batch entropy lengths")?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &entropy_checkpoints)?; - if timing_enabled { - timing.buffer_alloc = timing_buffer_start - .expect("timing start is set when timing is enabled") - .elapsed(); - } - - let dc_tables = [ - PreparedHuffmanHost::from(&first.y_dc_table), - PreparedHuffmanHost::from(&first.cb_dc_table), - PreparedHuffmanHost::from(&first.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&first.y_ac_table), - PreparedHuffmanHost::from(&first.cb_ac_table), - PreparedHuffmanHost::from(&first.cr_ac_table), - ]; - - let mut command_buffer = runtime.queue.new_command_buffer(); - let mut _split_scratch: Option<(Buffer, Buffer)> = None; - match decode_mode { - Fast420BatchDecodeMode::Fused => { - let timing_encode_start = timing_enabled.then(Instant::now); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast420_batch_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - first.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - first.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - first.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&entropy_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&entropy_lens_buffer), 0); - decoder_encoder.set_buffer(16, Some(&status_buffer), 0); - decoder_encoder.set_buffer(17, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast420_batch_decode_pipeline, - total_decode_threads, - ); - decoder_encoder.end_encoding(); - if timing_enabled { - timing.encode_decode = timing_encode_start - .expect("timing start is set when timing is enabled") - .elapsed(); - command_buffer.commit(); - let timing_wait_start = Instant::now(); - command_buffer.wait_until_completed(); - timing.wait_decode = timing_wait_start.elapsed(); - command_buffer = runtime.queue.new_command_buffer(); - } - } - Fast420BatchDecodeMode::SplitCoeffIdct => { - let coeff_bytes = total_blocks - .checked_mul(64) - .and_then(|bytes| bytes.checked_mul(size_of::())) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal fast420 batch coefficient scratch overflowed".to_string(), - })?; - let idct_component_depth = - tile_count_u32 - .checked_mul(6) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal fast420 batch IDCT dispatch overflowed".to_string(), - })?; - let coeff_blocks = runtime - .device - .new_buffer(coeff_bytes as u64, MTLResourceOptions::StorageModePrivate); - let dc_only_flags = runtime - .device - .new_buffer(total_blocks as u64, MTLResourceOptions::StorageModePrivate); - - let coeff_encoder = command_buffer.new_compute_command_encoder(); - coeff_encoder.set_compute_pipeline_state(&runtime.fast420_batch_coeffs_decode_pipeline); - coeff_encoder.set_buffer(0, Some(&entropy_buffer), 0); - coeff_encoder.set_buffer(1, Some(&coeff_blocks), 0); - coeff_encoder.set_buffer(2, Some(&dc_only_flags), 0); - coeff_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - coeff_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - first.y_quant.as_ptr().cast(), - ); - coeff_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - first.cb_quant.as_ptr().cast(), - ); - coeff_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - first.cr_quant.as_ptr().cast(), - ); - coeff_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - coeff_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - coeff_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - coeff_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - coeff_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - coeff_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - coeff_encoder.set_buffer(14, Some(&entropy_offsets_buffer), 0); - coeff_encoder.set_buffer(15, Some(&entropy_lens_buffer), 0); - coeff_encoder.set_buffer(16, Some(&status_buffer), 0); - coeff_encoder.set_buffer(17, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - coeff_encoder, - &runtime.fast420_batch_coeffs_decode_pipeline, - total_decode_threads, - ); - coeff_encoder.end_encoding(); - - let idct_encoder = command_buffer.new_compute_command_encoder(); - idct_encoder.set_compute_pipeline_state(&runtime.fast420_batch_idct_deposit_pipeline); - idct_encoder.set_buffer(0, Some(&coeff_blocks), 0); - idct_encoder.set_buffer(1, Some(&dc_only_flags), 0); - idct_encoder.set_buffer(2, Some(&y_plane), 0); - idct_encoder.set_buffer(3, Some(&cb_plane), 0); - idct_encoder.set_buffer(4, Some(&cr_plane), 0); - idct_encoder.set_bytes( - 5, - size_of::() as u64, - (&raw const params).cast(), - ); - dispatch_3d_pipeline( - idct_encoder, - &runtime.fast420_batch_idct_deposit_pipeline, - (first.mcus_per_row, first.mcu_rows, idct_component_depth), - ); - idct_encoder.end_encoding(); - - _split_scratch = Some((coeff_blocks, dc_only_flags)); - } - } - - let timing_pack_encode_start = timing_enabled.then(Instant::now); - let pack_encoder = command_buffer.new_compute_command_encoder(); - pack_encoder.set_compute_pipeline_state(&runtime.pack_420_rgb_batch_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - dispatch_3d_pipeline( - pack_encoder, - &runtime.pack_420_rgb_batch_pipeline, - ( - packed_pair_extent(width), - packed_pair_extent(height), - tile_count_u32, - ), - ); - pack_encoder.end_encoding(); - if timing_enabled { - timing.encode_pack = timing_pack_encode_start - .expect("timing start is set when timing is enabled") - .elapsed(); - } - - command_buffer.commit(); - if timing_enabled { - let timing_wait_start = Instant::now(); - command_buffer.wait_until_completed(); - timing.wait_pack = timing_wait_start.elapsed(); - timing.total = timing_total_start - .expect("timing start is set when timing is enabled") - .elapsed(); - timing.log("fused-stages", tile_count, first.dimensions, segment_count); - } else { - command_buffer.wait_until_completed(); - } - - if let Some(status) = first_decode_error_status(&status_buffer, total_decode_threads) { - let mut results = Vec::with_capacity(requests.len()); - for request in requests { - let decoder = CpuDecoder::new(request.input.as_ref())?; - results.push(Err(decode_error_from_cpu(&decoder, request.fmt, status))); - } - return Ok(Some(results)); - } - - let mut results = Vec::with_capacity(requests.len()); - for index in 0..requests.len() { - results.push(Ok(Surface::from_metal_buffer_offset( - out_buffer.clone(), - first.dimensions, - PixelFormat::Rgb8, - index * out_tile_len, - ))); - } - Ok(Some(results)) -} - -#[cfg(target_os = "macos")] -fn split_fast420_batch_enabled() -> bool { - split_fast420_batch_value_enabled(std::env::var_os(SPLIT_FAST420_BATCH_ENV).as_deref()) -} - -#[cfg(target_os = "macos")] -fn split_fast420_batch_value_enabled(value: Option<&OsStr>) -> bool { - value.is_some_and(|value| value == OsStr::new("1")) -} - -#[cfg(target_os = "macos")] -fn fast420_batch_timing_enabled() -> bool { - fast420_batch_timing_value_enabled(std::env::var_os(FAST420_BATCH_TIMING_ENV).as_deref()) -} - -#[cfg(target_os = "macos")] -fn fast420_batch_timing_value_enabled(value: Option<&OsStr>) -> bool { - value.is_some_and(|value| value == OsStr::new("1")) -} - -#[cfg(target_os = "macos")] -fn fast422_packets_share_full_rgb_batch_shape( - first: &JpegMetalFast422PacketV1, - packet: &JpegMetalFast422PacketV1, - segment_count: usize, -) -> bool { - packet.restart_interval_mcus == 0 - && packet.dimensions == first.dimensions - && packet.mcus_per_row == first.mcus_per_row - && packet.mcu_rows == first.mcu_rows - && packet.entropy_checkpoints.len() == segment_count - && packet.y_quant == first.y_quant - && packet.cb_quant == first.cb_quant - && packet.cr_quant == first.cr_quant - && packet.y_dc_table == first.y_dc_table - && packet.y_ac_table == first.y_ac_table - && packet.cb_dc_table == first.cb_dc_table - && packet.cb_ac_table == first.cb_ac_table - && packet.cr_dc_table == first.cr_dc_table - && packet.cr_ac_table == first.cr_ac_table -} - -#[cfg(target_os = "macos")] -fn try_decode_fast422_full_rgb_batch_to_surfaces( - runtime: &MetalRuntime, - requests: &[batch::QueuedRequest], - packets: &[BatchedFastPacket<'_>], -) -> Result>>, Error> { - if requests.len() < 2 - || requests - .iter() - .any(|request| request.op != batch::BatchOp::Full || request.fmt != PixelFormat::Rgb8) - { - return Ok(None); - } - - let mut fast422_packets = Vec::with_capacity(packets.len()); - for packet in packets { - let BatchedFastPacket::Fast422(packet) = packet else { - return Ok(None); - }; - fast422_packets.push(*packet); - } - - let Some(first) = fast422_packets.first().copied() else { - return Ok(None); - }; - if first.restart_interval_mcus != 0 || first.entropy_checkpoints.is_empty() { - return Ok(None); - } - - let segment_count = first.entropy_checkpoints.len(); - if !fast422_packets - .iter() - .all(|packet| fast422_packets_share_full_rgb_batch_shape(first, packet, segment_count)) - { - return Ok(None); - } - - let tile_count = fast422_packets.len(); - let tile_count_u32 = checked_u32(tile_count, "batch tile count")?; - let segment_count_u32 = checked_u32(segment_count, "batch segment count")?; - let total_decode_threads = checked_u32( - tile_count - .checked_mul(segment_count) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal batch decode thread count overflowed".to_string(), - })?, - "batch decode thread count", - )?; - - let width = first.dimensions.0; - let height = first.dimensions.1; - let chroma_width = width.div_ceil(2); - let chroma_height = height; - let y_len = width as usize * height as usize; - let chroma_len = chroma_width as usize * chroma_height as usize; - let out_stride = width as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let out_tile_len = out_stride * height as usize; - - let params = JpegFast420BatchParams { - width, - height, - chroma_width, - chroma_height, - mcus_per_row: first.mcus_per_row, - mcu_rows: first.mcu_rows, - segment_count: segment_count_u32, - tile_count: tile_count_u32, - out_stride: checked_u32(out_stride, "batch output stride")?, - alpha: u32::from(u8::MAX), - }; - - let total_entropy_len = fast422_packets - .iter() - .map(|packet| packet.entropy_bytes.len()) - .try_fold(0usize, usize::checked_add) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal batch entropy length overflowed".to_string(), - })?; - if total_entropy_len == 0 { - return Ok(None); - } - - let mut entropy_bytes = Vec::with_capacity(total_entropy_len); - let mut entropy_offsets = Vec::with_capacity(tile_count); - let mut entropy_lens = Vec::with_capacity(tile_count); - let mut entropy_checkpoints = Vec::with_capacity(tile_count * segment_count); - for packet in &fast422_packets { - entropy_offsets.push(checked_u32(entropy_bytes.len(), "batch entropy offset")?); - entropy_lens.push(checked_u32( - packet.entropy_bytes.len(), - "batch entropy length", - )?); - entropy_bytes.extend_from_slice(&packet.entropy_bytes); - entropy_checkpoints.extend(packet.entropy_checkpoints.iter().copied()); - } - - let y_plane = new_private_buffer(&runtime.device, y_len * tile_count); - let cb_plane = new_private_buffer(&runtime.device, chroma_len * tile_count); - let cr_plane = new_private_buffer(&runtime.device, chroma_len * tile_count); - let out_buffer = runtime.device.new_buffer( - (out_tile_len * tile_count) as u64, - MTLResourceOptions::StorageModeShared, - ); - let status_buffer = decode_status_buffer(&runtime.device, total_decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - entropy_bytes.as_ptr().cast(), - entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let entropy_offsets_buffer = - u32_buffer(&runtime.device, &entropy_offsets, "batch entropy offsets")?; - let entropy_lens_buffer = u32_buffer(&runtime.device, &entropy_lens, "batch entropy lengths")?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&first.y_dc_table), - PreparedHuffmanHost::from(&first.cb_dc_table), - PreparedHuffmanHost::from(&first.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&first.y_ac_table), - PreparedHuffmanHost::from(&first.cb_ac_table), - PreparedHuffmanHost::from(&first.cr_ac_table), - ]; - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast422_batch_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - first.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - first.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - first.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&entropy_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&entropy_lens_buffer), 0); - decoder_encoder.set_buffer(16, Some(&status_buffer), 0); - decoder_encoder.set_buffer(17, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast422_batch_decode_pipeline, - total_decode_threads, - ); - decoder_encoder.end_encoding(); - - let pack_encoder = command_buffer.new_compute_command_encoder(); - pack_encoder.set_compute_pipeline_state(&runtime.pack_422_rgb_batch_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - dispatch_3d_pipeline( - pack_encoder, - &runtime.pack_422_rgb_batch_pipeline, - (packed_pair_extent(width), height, tile_count_u32), - ); - pack_encoder.end_encoding(); - - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(status) = first_decode_error_status(&status_buffer, total_decode_threads) { - let mut results = Vec::with_capacity(requests.len()); - for request in requests { - let decoder = CpuDecoder::new(request.input.as_ref())?; - results.push(Err(decode_error_from_cpu(&decoder, request.fmt, status))); - } - return Ok(Some(results)); - } - - let mut results = Vec::with_capacity(requests.len()); - for index in 0..requests.len() { - results.push(Ok(Surface::from_metal_buffer_offset( - out_buffer.clone(), - first.dimensions, - PixelFormat::Rgb8, - index * out_tile_len, - ))); - } - Ok(Some(results)) -} - -#[cfg(target_os = "macos")] -fn fast444_packets_share_region_scaled_batch_shape( - first: &JpegMetalFast444PacketV1, - packet: &JpegMetalFast444PacketV1, - segment_count: usize, -) -> bool { - packet.restart_interval_mcus == 0 - && packet.dimensions == first.dimensions - && packet.mcus_per_row == first.mcus_per_row - && packet.mcu_rows == first.mcu_rows - && packet.entropy_checkpoints.len() == segment_count - && packet.y_quant == first.y_quant - && packet.cb_quant == first.cb_quant - && packet.cr_quant == first.cr_quant - && packet.y_dc_table == first.y_dc_table - && packet.y_ac_table == first.y_ac_table - && packet.cb_dc_table == first.cb_dc_table - && packet.cb_ac_table == first.cb_ac_table - && packet.cr_dc_table == first.cr_dc_table - && packet.cr_ac_table == first.cr_ac_table -} - -#[cfg(target_os = "macos")] -fn try_decode_fast444_region_scaled_rgb_batch_to_surfaces( - runtime: &MetalRuntime, - requests: &[batch::QueuedRequest], - packets: &[BatchedFastPacket<'_>], -) -> Result>>, Error> { - if requests.len() < 2 - || requests - .iter() - .any(|request| request.fmt != PixelFormat::Rgb8) - { - return Ok(None); - } - - let mut fast444_packets = Vec::with_capacity(packets.len()); - for packet in packets { - let BatchedFastPacket::Fast444(packet, mode) = packet else { - return Ok(None); - }; - fast444_packets.push((*packet, *mode)); - } - - let Some((first, first_mode)) = fast444_packets.first().copied() else { - return Ok(None); - }; - let batch::BatchOp::RegionScaled { - roi: first_roi, - scale: first_scale, - } = requests[0].op - else { - return Ok(None); - }; - if first.restart_interval_mcus != 0 || first.entropy_checkpoints.is_empty() { - return Ok(None); - } - - let first_scaled = first_roi.scaled_covering(first_scale); - let first_scaled_roi = signinum_jpeg::Rect { - x: first_scaled.x, - y: first_scaled.y, - w: first_scaled.w, - h: first_scaled.h, - }; - let Some(first_decode_params) = - fast444_scaled_region_params(first, first_scale, first_scaled_roi) - else { - return Ok(None); - }; - - let segment_count = first.entropy_checkpoints.len(); - let tile_count = fast444_packets.len(); - let tile_count_u32 = checked_u32(tile_count, "region scaled batch tile count")?; - let segment_count_u32 = checked_u32(segment_count, "region scaled batch segment count")?; - let total_decode_threads = checked_u32( - tile_count - .checked_mul(segment_count) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal region scaled batch decode thread count overflowed" - .to_string(), - })?, - "region scaled batch decode thread count", - )?; - - for (request, (packet, mode)) in requests.iter().zip(fast444_packets.iter().copied()) { - let batch::BatchOp::RegionScaled { roi, scale } = request.op else { - return Ok(None); - }; - if scale != first_scale - || mode != first_mode - || !fast444_packets_share_region_scaled_batch_shape(first, packet, segment_count) - { - return Ok(None); - } - let scaled = roi.scaled_covering(scale); - let scaled_roi = signinum_jpeg::Rect { - x: scaled.x, - y: scaled.y, - w: scaled.w, - h: scaled.h, - }; - if fast444_scaled_region_params(packet, scale, scaled_roi) != Some(first_decode_params) { - return Ok(None); - } - } - - let out_stride = - first_decode_params.scaled_width as usize * PixelFormat::Rgb8.bytes_per_pixel(); - let out_tile_len = out_stride * first_decode_params.scaled_height as usize; - - let plane_len = - first_decode_params.scaled_width as usize * first_decode_params.scaled_height as usize; - let decode_params = JpegFastRegionScaledBatchParams { - scaled_width: first_decode_params.scaled_width, - scaled_height: first_decode_params.scaled_height, - chroma_width: first_decode_params.scaled_width, - chroma_height: first_decode_params.scaled_height, - mcus_per_row: first_decode_params.mcus_per_row, - mcu_rows: first_decode_params.mcu_rows, - segment_count: segment_count_u32, - tile_count: tile_count_u32, - scale_shift: first_decode_params.scale_shift, - origin_x: first_decode_params.origin_x, - origin_y: first_decode_params.origin_y, - }; - let pack_params = JpegWindowedPackBatchParams { - src_width: first_decode_params.scaled_width, - src_height: first_decode_params.scaled_height, - chroma_width: first_decode_params.scaled_width, - chroma_height: first_decode_params.scaled_height, - src_x: 0, - src_y: 0, - width: first_decode_params.scaled_width, - height: first_decode_params.scaled_height, - tile_count: tile_count_u32, - out_stride: checked_u32(out_stride, "region scaled batch output stride")?, - alpha: u32::from(u8::MAX), - mode: plane_mode_to_u32(first_mode), - out_format: OUT_RGB, - }; - - let Some(entropy_buffers) = batch_entropy_buffers( - runtime, - fast444_packets - .iter() - .map(|(packet, _)| packet.entropy_bytes.as_slice()), - fast444_packets - .iter() - .map(|(packet, _)| packet.entropy_checkpoints.as_slice()), - tile_count, - segment_count, - )? - else { - return Ok(None); - }; - - let y_plane = new_private_buffer(&runtime.device, plane_len * tile_count); - let cb_plane = new_private_buffer(&runtime.device, plane_len * tile_count); - let cr_plane = new_private_buffer(&runtime.device, plane_len * tile_count); - let out_buffer = runtime.device.new_buffer( - (out_tile_len * tile_count) as u64, - MTLResourceOptions::StorageModeShared, - ); - let status_buffer = decode_status_buffer(&runtime.device, total_decode_threads); - let dc_tables = [ - PreparedHuffmanHost::from(&first.y_dc_table), - PreparedHuffmanHost::from(&first.cb_dc_table), - PreparedHuffmanHost::from(&first.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&first.y_ac_table), - PreparedHuffmanHost::from(&first.cb_ac_table), - PreparedHuffmanHost::from(&first.cr_ac_table), - ]; - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder - .set_compute_pipeline_state(&runtime.fast444_scaled_region_batch_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffers.payload), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const decode_params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - first.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - first.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - first.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&entropy_buffers.offsets), 0); - decoder_encoder.set_buffer(15, Some(&entropy_buffers.lens), 0); - decoder_encoder.set_buffer(16, Some(&status_buffer), 0); - decoder_encoder.set_buffer(17, Some(&entropy_buffers.checkpoints), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast444_scaled_region_batch_decode_pipeline, - total_decode_threads, - ); - decoder_encoder.end_encoding(); - - let pack_encoder = command_buffer.new_compute_command_encoder(); - pack_encoder.set_compute_pipeline_state(&runtime.pack_444_rgb_batch_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const pack_params).cast(), - ); - dispatch_3d_pipeline( - pack_encoder, - &runtime.pack_444_rgb_batch_pipeline, - ( - first_decode_params.scaled_width, - first_decode_params.scaled_height, - tile_count_u32, - ), - ); - pack_encoder.end_encoding(); - - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(results) = - region_scaled_batch_error_results(requests, &status_buffer, total_decode_threads)? - { - return Ok(Some(results)); - } - - let mut results = Vec::with_capacity(requests.len()); - for index in 0..requests.len() { - results.push(Ok(Surface::from_metal_buffer_offset( - out_buffer.clone(), - ( - first_decode_params.scaled_width, - first_decode_params.scaled_height, - ), - PixelFormat::Rgb8, - index * out_tile_len, - ))); - } - Ok(Some(results)) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast420_region_scaled_rgb_batch_to_surfaces( - runtime: &MetalRuntime, - requests: &[batch::QueuedRequest], - packets: &[BatchedFastPacket<'_>], -) -> Result>>, Error> { - if requests.len() < 2 - || requests - .iter() - .any(|request| request.fmt != PixelFormat::Rgb8) - { - return Ok(None); - } - - let mut fast420_packets = Vec::with_capacity(packets.len()); - for packet in packets { - let BatchedFastPacket::Fast420(packet) = packet else { - return Ok(None); - }; - fast420_packets.push(*packet); - } - - let Some(first) = fast420_packets.first().copied() else { - return Ok(None); - }; - let batch::BatchOp::RegionScaled { - roi: first_roi, - scale: first_scale, - } = requests[0].op - else { - return Ok(None); - }; - if first.restart_interval_mcus != 0 || first.entropy_checkpoints.is_empty() { - return Ok(None); - } - - let segment_count = first.entropy_checkpoints.len(); - let tile_count = fast420_packets.len(); - let tile_count_u32 = checked_u32(tile_count, "region scaled batch tile count")?; - let segment_count_u32 = checked_u32(segment_count, "region scaled batch segment count")?; - let Some(first_plan) = fast420_region_scaled_batch_plan( - first, - first_roi, - first_scale, - tile_count_u32, - segment_count_u32, - ) else { - return Ok(None); - }; - - let total_decode_threads = checked_u32( - tile_count - .checked_mul(segment_count) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal fast420 region scaled batch decode thread count overflowed" - .to_string(), - })?, - "fast420 region scaled batch decode thread count", - )?; - - for (request, packet) in requests.iter().zip(fast420_packets.iter().copied()) { - let batch::BatchOp::RegionScaled { roi, scale } = request.op else { - return Ok(None); - }; - if scale != first_scale - || !fast420_packets_share_full_rgb_batch_shape(first, packet, segment_count) - || fast420_region_scaled_batch_plan( - packet, - roi, - scale, - tile_count_u32, - segment_count_u32, - ) != Some(first_plan) - { - return Ok(None); - } - } - - let Some(entropy_buffers) = batch_entropy_buffers( - runtime, - fast420_packets - .iter() - .map(|packet| packet.entropy_bytes.as_slice()), - fast420_packets - .iter() - .map(|packet| packet.entropy_checkpoints.as_slice()), - tile_count, - segment_count, - )? - else { - return Ok(None); - }; - - let y_plane = new_private_buffer(&runtime.device, first_plan.y_len * tile_count); - let cb_plane = new_private_buffer(&runtime.device, first_plan.chroma_len * tile_count); - let cr_plane = new_private_buffer(&runtime.device, first_plan.chroma_len * tile_count); - let out_buffer = runtime.device.new_buffer( - (first_plan.out_tile_len * tile_count) as u64, - MTLResourceOptions::StorageModeShared, - ); - let status_buffer = decode_status_buffer(&runtime.device, total_decode_threads); - let dc_tables = [ - PreparedHuffmanHost::from(&first.y_dc_table), - PreparedHuffmanHost::from(&first.cb_dc_table), - PreparedHuffmanHost::from(&first.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&first.y_ac_table), - PreparedHuffmanHost::from(&first.cb_ac_table), - PreparedHuffmanHost::from(&first.cr_ac_table), - ]; - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder - .set_compute_pipeline_state(&runtime.fast420_scaled_region_batch_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffers.payload), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const first_plan.decode_params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - first.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - first.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - first.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&entropy_buffers.offsets), 0); - decoder_encoder.set_buffer(15, Some(&entropy_buffers.lens), 0); - decoder_encoder.set_buffer(16, Some(&status_buffer), 0); - decoder_encoder.set_buffer(17, Some(&entropy_buffers.checkpoints), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast420_scaled_region_batch_decode_pipeline, - total_decode_threads, - ); - decoder_encoder.end_encoding(); - - let pack_encoder = command_buffer.new_compute_command_encoder(); - pack_encoder.set_compute_pipeline_state(&runtime.pack_420_windowed_rgb_batch_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const first_plan.pack_params).cast(), - ); - dispatch_3d_pipeline( - pack_encoder, - &runtime.pack_420_windowed_rgb_batch_pipeline, - (first_plan.out_dims.0, first_plan.out_dims.1, tile_count_u32), - ); - pack_encoder.end_encoding(); - - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(results) = - region_scaled_batch_error_results(requests, &status_buffer, total_decode_threads)? - { - return Ok(Some(results)); - } - - let mut results = Vec::with_capacity(requests.len()); - for index in 0..requests.len() { - results.push(Ok(Surface::from_metal_buffer_offset( - out_buffer.clone(), - first_plan.out_dims, - PixelFormat::Rgb8, - index * first_plan.out_tile_len, - ))); - } - Ok(Some(results)) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast422_region_scaled_rgb_batch_to_surfaces( - runtime: &MetalRuntime, - requests: &[batch::QueuedRequest], - packets: &[BatchedFastPacket<'_>], -) -> Result>>, Error> { - if requests.len() < 2 - || requests - .iter() - .any(|request| request.fmt != PixelFormat::Rgb8) - { - return Ok(None); - } - - let mut fast422_packets = Vec::with_capacity(packets.len()); - for packet in packets { - let BatchedFastPacket::Fast422(packet) = packet else { - return Ok(None); - }; - fast422_packets.push(*packet); - } - - let Some(first) = fast422_packets.first().copied() else { - return Ok(None); - }; - let batch::BatchOp::RegionScaled { - roi: first_roi, - scale: first_scale, - } = requests[0].op - else { - return Ok(None); - }; - if first.restart_interval_mcus != 0 || first.entropy_checkpoints.is_empty() { - return Ok(None); - } - - let segment_count = first.entropy_checkpoints.len(); - let tile_count = fast422_packets.len(); - let tile_count_u32 = checked_u32(tile_count, "region scaled batch tile count")?; - let segment_count_u32 = checked_u32(segment_count, "region scaled batch segment count")?; - let Some(first_plan) = fast422_region_scaled_batch_plan( - first, - first_roi, - first_scale, - tile_count_u32, - segment_count_u32, - ) else { - return Ok(None); - }; - - let total_decode_threads = checked_u32( - tile_count - .checked_mul(segment_count) - .ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal fast422 region scaled batch decode thread count overflowed" - .to_string(), - })?, - "fast422 region scaled batch decode thread count", - )?; - - for (request, packet) in requests.iter().zip(fast422_packets.iter().copied()) { - let batch::BatchOp::RegionScaled { roi, scale } = request.op else { - return Ok(None); - }; - if scale != first_scale - || !fast422_packets_share_full_rgb_batch_shape(first, packet, segment_count) - || fast422_region_scaled_batch_plan( - packet, - roi, - scale, - tile_count_u32, - segment_count_u32, - ) != Some(first_plan) - { - return Ok(None); - } - } - - let Some(entropy_buffers) = batch_entropy_buffers( - runtime, - fast422_packets - .iter() - .map(|packet| packet.entropy_bytes.as_slice()), - fast422_packets - .iter() - .map(|packet| packet.entropy_checkpoints.as_slice()), - tile_count, - segment_count, - )? - else { - return Ok(None); - }; - - let y_plane = new_private_buffer(&runtime.device, first_plan.y_len * tile_count); - let cb_plane = new_private_buffer(&runtime.device, first_plan.chroma_len * tile_count); - let cr_plane = new_private_buffer(&runtime.device, first_plan.chroma_len * tile_count); - let out_buffer = runtime.device.new_buffer( - (first_plan.out_tile_len * tile_count) as u64, - MTLResourceOptions::StorageModeShared, - ); - let status_buffer = decode_status_buffer(&runtime.device, total_decode_threads); - let dc_tables = [ - PreparedHuffmanHost::from(&first.y_dc_table), - PreparedHuffmanHost::from(&first.cb_dc_table), - PreparedHuffmanHost::from(&first.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&first.y_ac_table), - PreparedHuffmanHost::from(&first.cb_ac_table), - PreparedHuffmanHost::from(&first.cr_ac_table), - ]; - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder - .set_compute_pipeline_state(&runtime.fast422_scaled_region_batch_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffers.payload), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const first_plan.decode_params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - first.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - first.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - first.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&entropy_buffers.offsets), 0); - decoder_encoder.set_buffer(15, Some(&entropy_buffers.lens), 0); - decoder_encoder.set_buffer(16, Some(&status_buffer), 0); - decoder_encoder.set_buffer(17, Some(&entropy_buffers.checkpoints), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast422_scaled_region_batch_decode_pipeline, - total_decode_threads, - ); - decoder_encoder.end_encoding(); - - let pack_encoder = command_buffer.new_compute_command_encoder(); - pack_encoder.set_compute_pipeline_state(&runtime.pack_422_windowed_rgb_batch_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const first_plan.pack_params).cast(), - ); - dispatch_3d_pipeline( - pack_encoder, - &runtime.pack_422_windowed_rgb_batch_pipeline, - (first_plan.out_dims.0, first_plan.out_dims.1, tile_count_u32), - ); - pack_encoder.end_encoding(); - - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(results) = - region_scaled_batch_error_results(requests, &status_buffer, total_decode_threads)? - { - return Ok(Some(results)); - } - - let mut results = Vec::with_capacity(requests.len()); - for index in 0..requests.len() { - results.push(Ok(Surface::from_metal_buffer_offset( - out_buffer.clone(), - first_plan.out_dims, - PixelFormat::Rgb8, - index * first_plan.out_tile_len, - ))); - } - Ok(Some(results)) -} - -#[cfg(target_os = "macos")] -fn requests_share_one_input(requests: &[batch::QueuedRequest]) -> bool { - let Some(first) = requests.first() else { - return false; - }; - requests.iter().all(|request| { - request.input.as_ptr() == first.input.as_ptr() && request.input.len() == first.input.len() - }) -} - -#[cfg(target_os = "macos")] -fn requests_share_one_region_scaled_work(requests: &[batch::QueuedRequest]) -> bool { - let Some(first) = requests.first() else { - return false; - }; - requests_share_one_input(requests) - && requests.iter().all(|request| { - request.fmt == first.fmt && request.backend == first.backend && request.op == first.op - }) -} - -#[cfg(target_os = "macos")] -fn decode_region_scaled_packet_surface( - runtime: &MetalRuntime, - decoder: &CpuDecoder<'_>, - request: &batch::QueuedRequest, - packet: &BatchedFastPacket<'_>, -) -> Result { - let batch::BatchOp::RegionScaled { roi, scale } = request.op else { - return Err(Error::MetalKernel { - message: "JPEG Metal expected a region scaled batch request".to_string(), - }); - }; - let scaled = roi.scaled_covering(scale); - let scaled_roi = signinum_jpeg::Rect { - x: scaled.x, - y: scaled.y, - w: scaled.w, - h: scaled.h, - }; - match packet { - BatchedFastPacket::Fast420(packet) => try_decode_fast420_scaled_region_to_surface( - runtime, - decoder, - Some(packet), - request.fmt, - scaled_roi, - scale, - ), - BatchedFastPacket::Fast422(packet) => try_decode_fast422_scaled_region_to_surface( - runtime, - Some(packet), - request.fmt, - scaled_roi, - scale, - ), - BatchedFastPacket::Fast444(packet, _) => try_decode_fast444_scaled_region_to_surface( - runtime, - decoder, - Some(packet), - request.fmt, - scaled_roi, - scale, - ), - } - .and_then(|surface| { - surface.ok_or_else(|| Error::MetalKernel { - message: "JPEG Metal repeated region scaled batch was not packet-decodable".to_string(), - }) - }) -} - -#[cfg(target_os = "macos")] -fn try_decode_repeated_region_scaled_batch_to_surfaces( - runtime: &MetalRuntime, - requests: &[batch::QueuedRequest], - packets: &[BatchedFastPacket<'_>], -) -> Result>>, Error> { - if requests.len() <= REGION_SCALED_BATCH_CHUNK - || !requests_share_one_input(requests) - || !requests - .iter() - .all(|request| matches!(request.op, batch::BatchOp::RegionScaled { .. })) - { - return Ok(None); - } - - let decoder = CpuDecoder::new(requests[0].input.as_ref())?; - if requests_share_one_region_scaled_work(requests) { - let surface = - decode_region_scaled_packet_surface(runtime, &decoder, &requests[0], &packets[0])?; - return Ok(Some( - (0..requests.len()) - .map(|_| Ok(surface.clone())) - .collect::>(), - )); - } - - let mut results = Vec::with_capacity(requests.len()); - for (request, packet) in requests.iter().zip(packets.iter()) { - results.push(decode_region_scaled_packet_surface( - runtime, &decoder, request, packet, - )); - } - - Ok(Some(results)) -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_full_batch_to_surfaces( - requests: &[batch::QueuedRequest], -) -> Result>>, Error> { - let Some(packets) = batched_fast_packets(requests)? else { - return Ok(None); - }; - - with_runtime(|runtime| decode_full_batch_to_surfaces_with_runtime(runtime, requests, &packets)) -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_full_batch_to_surfaces_with_session( - requests: &[batch::QueuedRequest], - session: &crate::MetalBackendSession, -) -> Result>>, Error> { - let Some(packets) = batched_fast_packets(requests)? else { - return Ok(None); - }; - - with_runtime_for_session(session, |runtime| { - decode_full_batch_to_surfaces_with_runtime(runtime, requests, &packets) - }) -} - -#[cfg(target_os = "macos")] -fn decode_full_batch_to_surfaces_with_runtime( - runtime: &MetalRuntime, - requests: &[batch::QueuedRequest], - packets: &[BatchedFastPacket<'_>], -) -> Result>>, Error> { - if let Some(results) = - try_decode_fast420_full_rgb_batch_to_surfaces(runtime, requests, packets)? - { - return Ok(Some(results)); - } - if let Some(results) = - try_decode_fast422_full_rgb_batch_to_surfaces(runtime, requests, packets)? - { - return Ok(Some(results)); - } - if let Some(results) = - try_decode_repeated_region_scaled_batch_to_surfaces(runtime, requests, packets)? - { - return Ok(Some(results)); - } - if let Some(results) = - try_decode_fast444_region_scaled_rgb_batch_to_surfaces(runtime, requests, packets)? - { - return Ok(Some(results)); - } - if let Some(results) = - try_decode_fast420_region_scaled_rgb_batch_to_surfaces(runtime, requests, packets)? - { - return Ok(Some(results)); - } - if let Some(results) = - try_decode_fast422_region_scaled_rgb_batch_to_surfaces(runtime, requests, packets)? - { - return Ok(Some(results)); - } - - let mut results = Vec::with_capacity(requests.len()); - let has_region_scaled = requests - .iter() - .any(|request| matches!(request.op, batch::BatchOp::RegionScaled { .. })); - let chunk_size = if has_region_scaled { - REGION_SCALED_BATCH_CHUNK - } else { - requests.len().max(1) - }; - for chunk_start in (0..requests.len()).step_by(chunk_size) { - let chunk_end = (chunk_start + chunk_size).min(requests.len()); - let command_buffer = runtime.queue.new_command_buffer(); - let mut encoded = Vec::with_capacity(chunk_end - chunk_start); - let mut device_buffer_cache = BatchDeviceBufferCache::default(); - for index in chunk_start..chunk_end { - let request = &requests[index]; - let packet = &packets[index]; - let item = match packet { - BatchedFastPacket::Fast420(packet) => match request.op { - batch::BatchOp::Full => encode_fast420_batch_item( - runtime, - command_buffer, - index, - packet, - request.fmt, - )?, - batch::BatchOp::Region(roi) => encode_fast420_region_batch_item( - runtime, - command_buffer, - index, - packet, - request.fmt, - roi, - )?, - batch::BatchOp::Scaled(scale) => encode_fast420_scaled_batch_item( - runtime, - command_buffer, - index, - packet, - request.fmt, - scale, - )?, - batch::BatchOp::RegionScaled { roi, scale } => { - encode_fast420_scaled_region_batch_item( - runtime, - command_buffer, - &mut device_buffer_cache, - index, - packet, - request.fmt, - roi, - scale, - )? - } - }, - BatchedFastPacket::Fast422(packet) => match request.op { - batch::BatchOp::Full => encode_fast422_batch_item( - runtime, - command_buffer, - index, - packet, - request.fmt, - )?, - batch::BatchOp::Region(roi) => encode_fast422_region_batch_item( - runtime, - command_buffer, - index, - packet, - request.fmt, - roi, - )?, - batch::BatchOp::Scaled(scale) => encode_fast422_scaled_batch_item( - runtime, - command_buffer, - index, - packet, - request.fmt, - scale, - )?, - batch::BatchOp::RegionScaled { roi, scale } => { - encode_fast422_scaled_region_batch_item( - runtime, - command_buffer, - &mut device_buffer_cache, - index, - packet, - request.fmt, - roi, - scale, - )? - } - }, - BatchedFastPacket::Fast444(packet, mode) => match request.op { - batch::BatchOp::Full => encode_fast444_batch_item( - runtime, - command_buffer, - index, - packet, - *mode, - request.fmt, - )?, - batch::BatchOp::Region(roi) => encode_fast444_region_batch_item( - runtime, - command_buffer, - index, - packet, - *mode, - request.fmt, - roi, - )?, - batch::BatchOp::Scaled(scale) => encode_fast444_scaled_batch_item( - runtime, - command_buffer, - index, - packet, - *mode, - request.fmt, - scale, - )?, - batch::BatchOp::RegionScaled { roi, scale } => { - encode_fast444_scaled_region_batch_item( - runtime, - command_buffer, - &mut device_buffer_cache, - index, - packet, - *mode, - request.fmt, - roi, - scale, - )? - } - }, - }; - encoded.push(item); - } - - command_buffer.commit(); - command_buffer.wait_until_completed(); - - for item in encoded { - if let Some(status) = - first_decode_error_status(&item.status_buffer, item.decode_threads) - { - let request = &requests[item.request_index]; - let decoder = CpuDecoder::new(request.input.as_ref())?; - results.push(Err(decode_error_from_cpu(&decoder, request.fmt, status))); - } else { - results.push(Ok(item.surface)); - } - } - } - Ok(Some(results)) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast422_to_surface( - runtime: &MetalRuntime, - packet: Option<&JpegMetalFast422PacketV1>, - fmt: PixelFormat, -) -> Result, Error> { - let Some(decoded) = - decode_fast422_to_rgb_buffer(runtime, packet, fmt, MTLResourceOptions::StorageModeShared)? - else { - return Ok(None); - }; - Ok(Some(Surface::from_metal_buffer( - decoded.buffer, - decoded.dimensions, - fmt, - ))) -} - -#[cfg(target_os = "macos")] -fn decode_fast422_to_rgb_buffer( - runtime: &MetalRuntime, - packet: Option<&JpegMetalFast422PacketV1>, - fmt: PixelFormat, - output_storage: MTLResourceOptions, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - let Some(_out_format) = pixel_format_to_out_format(fmt) else { - return Ok(None); - }; - - let params = fast422_params(packet, fmt)?; - let y_len = params.width as usize * params.height as usize; - let chroma_len = params.chroma_width as usize * params.chroma_height as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, fmt == PixelFormat::Gray8); - let cb_plane = new_private_buffer(&runtime.device, chroma_len); - let cr_plane = new_private_buffer(&runtime.device, chroma_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let out_buffer = (fmt != PixelFormat::Gray8).then(|| { - runtime.device.new_buffer( - (params.out_stride as usize * params.height as usize) as u64, - output_storage, - ) - }); - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast422_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast422_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - if let Some(out_buffer) = out_buffer.as_ref() { - let Some(pack_pipeline) = pack_422_pipeline_for_format(runtime, fmt) else { - return Ok(None); - }; - let pack_encoder = command_buffer.new_compute_command_encoder(); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - dispatch_2d_pipeline(pack_encoder, pack_pipeline, packet.dimensions); - pack_encoder.end_encoding(); - } - - command_buffer.commit(); - command_buffer.wait_until_completed(); - let command_buffer = command_buffer.to_owned(); - - if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { - return Err(Error::MetalKernel { - message: format!( - "unexpected Metal fast422 failure at entropy byte {}", - status.position - ), - }); - } - - Ok(Some(FastRgbDecodeBuffer { - buffer: out_buffer.unwrap_or(y_plane), - dimensions: packet.dimensions, - status_buffer, - command_buffer, - })) -} - -#[cfg(target_os = "macos")] -fn fast422_status_error(status: JpegDecodeStatus) -> Error { - Error::MetalKernel { - message: format!( - "unexpected Metal fast422 failure at entropy byte {}", - status.position - ), - } -} - -#[cfg(target_os = "macos")] -fn try_decode_fast422_region_to_surface( - runtime: &MetalRuntime, - packet: Option<&JpegMetalFast422PacketV1>, - fmt: PixelFormat, - roi: signinum_jpeg::Rect, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - let Some(_) = pixel_format_to_out_format(fmt) else { - return Ok(None); - }; - - let command_buffer = runtime.queue.new_command_buffer(); - let item = encode_fast422_region_batch_item( - runtime, - command_buffer, - 0, - packet, - fmt, - Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }, - )?; - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(status) = first_decode_error_status(&item.status_buffer, item.decode_threads) { - return Err(fast422_status_error(status)); - } - - Ok(Some(item.surface)) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast422_scaled_to_surface( - runtime: &MetalRuntime, - packet: Option<&JpegMetalFast422PacketV1>, - fmt: PixelFormat, - scale: signinum_core::Downscale, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - let Some(_) = pixel_format_to_out_format(fmt) else { - return Ok(None); - }; - if fast422_scaled_params(packet, scale).is_none() { - return Ok(None); - } - - let command_buffer = runtime.queue.new_command_buffer(); - let item = encode_fast422_scaled_batch_item(runtime, command_buffer, 0, packet, fmt, scale)?; - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(status) = first_decode_error_status(&item.status_buffer, item.decode_threads) { - return Err(fast422_status_error(status)); - } - - Ok(Some(item.surface)) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast422_scaled_region_to_surface( - runtime: &MetalRuntime, - packet: Option<&JpegMetalFast422PacketV1>, - fmt: PixelFormat, - scaled_roi: signinum_jpeg::Rect, - scale: signinum_core::Downscale, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - let Some(_) = pixel_format_to_out_format(fmt) else { - return Ok(None); - }; - let Some(full_params) = fast422_scaled_params(packet, scale) else { - return Ok(None); - }; - let source_window = full_mcu_scaled_window_422( - (full_params.scaled_width, full_params.scaled_height), - scaled_roi, - full_params.scale_shift, - ); - let Some(mut decode_params) = fast422_scaled_region_params(packet, scale, source_window) else { - return Ok(None); - }; - let mcu_width = 16u32 >> decode_params.scale_shift; - let mcu_height = 8u32 >> decode_params.scale_shift; - let (first_mcu, end_mcu) = mcu_range_for_rect( - source_window, - packet.mcus_per_row, - packet.mcu_rows, - mcu_width, - mcu_height, - ); - let total_mcus = packet.mcus_per_row * packet.mcu_rows; - let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( - &packet.restart_offsets, - packet.restart_interval_mcus, - total_mcus, - first_mcu, - end_mcu, - ); - decode_params.restart_start_mcu = restart_start_mcu; - decode_params.restart_offset_count = entropy_segment_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let local_roi = signinum_jpeg::Rect { - x: scaled_roi.x - source_window.x, - y: scaled_roi.y - source_window.y, - w: scaled_roi.w, - h: scaled_roi.h, - }; - let pack_params = - fast422_windowed_pack_params_for_dims((source_window.w, source_window.h), fmt, local_roi); - let y_len = source_window.w as usize * source_window.h as usize; - let chroma_len = source_window.w.div_ceil(2) as usize * source_window.h as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, false); - let cb_plane = new_private_buffer(&runtime.device, chroma_len); - let cr_plane = new_private_buffer(&runtime.device, chroma_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let out_buffer = runtime.device.new_buffer( - (pack_params.out_stride as usize * scaled_roi.h as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast422_scaled_region_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&cb_plane), 0); - decoder_encoder.set_buffer(3, Some(&cr_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const decode_params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast422_scaled_region_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let pack_encoder = command_buffer.new_compute_command_encoder(); - let pack_pipeline = pack_422_windowed_pipeline_for_format(runtime, fmt); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&cb_plane), 0); - pack_encoder.set_buffer(2, Some(&cr_plane), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const pack_params).cast(), - ); - dispatch_2d_pipeline(pack_encoder, pack_pipeline, (scaled_roi.w, scaled_roi.h)); - pack_encoder.end_encoding(); - - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { - return Err(fast422_status_error(status)); - } - - Ok(Some(Surface::from_metal_buffer( - out_buffer, - (scaled_roi.w, scaled_roi.h), - fmt, - ))) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast420_to_surface( - runtime: &MetalRuntime, - decoder: &CpuDecoder<'_>, - packet: Option<&JpegMetalFast420PacketV1>, - fmt: PixelFormat, -) -> Result, Error> { - let Some(decoded) = decode_fast420_to_rgb_buffer( - runtime, - decoder, - packet, - fmt, - MTLResourceOptions::StorageModeShared, - )? - else { - return Ok(None); - }; - Ok(Some(Surface::from_metal_buffer( - decoded.buffer, - decoded.dimensions, - fmt, - ))) -} - -#[cfg(target_os = "macos")] -fn decode_fast420_to_rgb_buffer( - runtime: &MetalRuntime, - decoder: &CpuDecoder<'_>, - packet: Option<&JpegMetalFast420PacketV1>, - fmt: PixelFormat, - output_storage: MTLResourceOptions, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - let Some(_out_format) = pixel_format_to_out_format(fmt) else { - return Ok(None); - }; - - let params = fast420_params(packet, fmt)?; - let y_len = params.width as usize * params.height as usize; - let chroma_len = params.chroma_width as usize * params.chroma_height as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, fmt == PixelFormat::Gray8); - let chroma_buffers = [ - new_private_buffer(&runtime.device, chroma_len), - new_private_buffer(&runtime.device, chroma_len), - ]; - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let out_buffer = (fmt != PixelFormat::Gray8).then(|| { - runtime.device.new_buffer( - (params.out_stride as usize * params.height as usize) as u64, - output_storage, - ) - }); - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast420_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&chroma_buffers[0]), 0); - decoder_encoder.set_buffer(3, Some(&chroma_buffers[1]), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast420_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - if let Some(out_buffer) = out_buffer.as_ref() { - let pack_encoder = command_buffer.new_compute_command_encoder(); - let pack_pipeline = pack_420_pipeline_for_format(runtime, fmt); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&chroma_buffers[0]), 0); - pack_encoder.set_buffer(2, Some(&chroma_buffers[1]), 0); - pack_encoder.set_buffer(3, Some(out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - dispatch_2d_pipeline(pack_encoder, pack_pipeline, packet.dimensions); - pack_encoder.end_encoding(); - } - - command_buffer.commit(); - command_buffer.wait_until_completed(); - let command_buffer = command_buffer.to_owned(); - - if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { - return Err(decode_error_from_cpu(decoder, fmt, status)); - } - - Ok(Some(FastRgbDecodeBuffer { - buffer: out_buffer.unwrap_or(y_plane), - dimensions: packet.dimensions, - status_buffer, - command_buffer, - })) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast420_region_to_surface( - runtime: &MetalRuntime, - decoder: &CpuDecoder<'_>, - packet: Option<&JpegMetalFast420PacketV1>, - fmt: PixelFormat, - roi: signinum_jpeg::Rect, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - let Some(_) = pixel_format_to_out_format(fmt) else { - return Ok(None); - }; - - let source_window = full_mcu_window_420(packet.dimensions, roi); - let mut decode_params = fast420_region_params(packet, fmt, source_window)?; - let (first_mcu, end_mcu) = - mcu_range_for_rect(source_window, packet.mcus_per_row, packet.mcu_rows, 16, 16); - let total_mcus = packet.mcus_per_row * packet.mcu_rows; - let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( - &packet.restart_offsets, - packet.restart_interval_mcus, - total_mcus, - first_mcu, - end_mcu, - ); - decode_params.restart_start_mcu = restart_start_mcu; - decode_params.restart_offset_count = entropy_segment_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let local_roi = signinum_jpeg::Rect { - x: roi.x - source_window.x, - y: roi.y - source_window.y, - w: roi.w, - h: roi.h, - }; - let pack_params = - fast420_windowed_pack_params_for_dims((source_window.w, source_window.h), fmt, local_roi); - let y_len = source_window.w as usize * source_window.h as usize; - let chroma_len = source_window.w.div_ceil(2) as usize * source_window.h.div_ceil(2) as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, false); - let chroma_buffers = [ - new_private_buffer(&runtime.device, chroma_len), - new_private_buffer(&runtime.device, chroma_len), - ]; - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let out_buffer = runtime.device.new_buffer( - (pack_params.out_stride as usize * roi.h as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast420_region_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&chroma_buffers[0]), 0); - decoder_encoder.set_buffer(3, Some(&chroma_buffers[1]), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const decode_params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast420_region_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let pack_encoder = command_buffer.new_compute_command_encoder(); - let pack_pipeline = pack_420_windowed_pipeline_for_format(runtime, fmt); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&chroma_buffers[0]), 0); - pack_encoder.set_buffer(2, Some(&chroma_buffers[1]), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const pack_params).cast(), - ); - dispatch_2d_pipeline(pack_encoder, pack_pipeline, (roi.w, roi.h)); - pack_encoder.end_encoding(); - - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { - return Err(decode_error_from_cpu(decoder, fmt, status)); - } - - Ok(Some(Surface::from_metal_buffer( - out_buffer, - (roi.w, roi.h), - fmt, - ))) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast420_scaled_to_surface( - runtime: &MetalRuntime, - decoder: &CpuDecoder<'_>, - packet: Option<&JpegMetalFast420PacketV1>, - fmt: PixelFormat, - scale: signinum_core::Downscale, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - let Some(_out_format) = pixel_format_to_out_format(fmt) else { - return Ok(None); - }; - let Some(params) = fast420_scaled_params(packet, scale) else { - return Ok(None); - }; - - let y_len = params.scaled_width as usize * params.scaled_height as usize; - let chroma_len = params.chroma_width as usize * params.chroma_height as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, fmt == PixelFormat::Gray8); - let chroma_buffers = [ - new_private_buffer(&runtime.device, chroma_len), - new_private_buffer(&runtime.device, chroma_len), - ]; - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let out_buffer = (fmt != PixelFormat::Gray8).then(|| { - runtime.device.new_buffer( - (params.scaled_width as usize * fmt.bytes_per_pixel() * params.scaled_height as usize) - as u64, - MTLResourceOptions::StorageModeShared, - ) - }); - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast420_scaled_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&chroma_buffers[0]), 0); - decoder_encoder.set_buffer(3, Some(&chroma_buffers[1]), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast420_scaled_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - if let Some(out_buffer) = out_buffer.as_ref() { - let pack_params = JpegFast420Params { - width: params.scaled_width, - height: params.scaled_height, - chroma_width: params.chroma_width, - chroma_height: params.chroma_height, - mcus_per_row: params.mcus_per_row, - mcu_rows: params.mcu_rows, - restart_interval_mcus: params.restart_interval_mcus, - restart_offset_count: params.restart_offset_count, - restart_start_mcu: params.restart_start_mcu, - entropy_len: params.entropy_len, - out_stride: u32::try_from(params.scaled_width as usize * fmt.bytes_per_pixel()) - .expect("JPEG Metal output stride fits in u32"), - alpha: u32::from(u8::MAX), - out_format: pixel_format_to_out_format(fmt).expect("validated output format"), - origin_x: 0, - origin_y: 0, - }; - let pack_encoder = command_buffer.new_compute_command_encoder(); - let pack_pipeline = pack_420_pipeline_for_format(runtime, fmt); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&chroma_buffers[0]), 0); - pack_encoder.set_buffer(2, Some(&chroma_buffers[1]), 0); - pack_encoder.set_buffer(3, Some(out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const pack_params).cast(), - ); - dispatch_2d_pipeline( - pack_encoder, - pack_pipeline, - (params.scaled_width, params.scaled_height), - ); - pack_encoder.end_encoding(); - } - - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { - return Err(decode_error_from_cpu(decoder, fmt, status)); - } - - Ok(Some(match out_buffer { - Some(out_buffer) => { - Surface::from_metal_buffer(out_buffer, (params.scaled_width, params.scaled_height), fmt) - } - None => { - Surface::from_metal_buffer(y_plane, (params.scaled_width, params.scaled_height), fmt) - } - })) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast420_scaled_region_to_surface( - runtime: &MetalRuntime, - decoder: &CpuDecoder<'_>, - packet: Option<&JpegMetalFast420PacketV1>, - fmt: PixelFormat, - scaled_roi: signinum_jpeg::Rect, - scale: signinum_core::Downscale, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - let Some(_) = pixel_format_to_out_format(fmt) else { - return Ok(None); - }; - let Some(full_params) = fast420_scaled_params(packet, scale) else { - return Ok(None); - }; - let source_window = full_mcu_scaled_window_420( - (full_params.scaled_width, full_params.scaled_height), - scaled_roi, - full_params.scale_shift, - ); - let Some(mut decode_params) = fast420_scaled_region_params(packet, scale, source_window) else { - return Ok(None); - }; - let mcu_size = 16u32 >> decode_params.scale_shift; - let (first_mcu, end_mcu) = mcu_range_for_rect( - source_window, - packet.mcus_per_row, - packet.mcu_rows, - mcu_size, - mcu_size, - ); - let total_mcus = packet.mcus_per_row * packet.mcu_rows; - let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( - &packet.restart_offsets, - packet.restart_interval_mcus, - total_mcus, - first_mcu, - end_mcu, - ); - decode_params.restart_start_mcu = restart_start_mcu; - decode_params.restart_offset_count = entropy_segment_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let local_roi = signinum_jpeg::Rect { - x: scaled_roi.x - source_window.x, - y: scaled_roi.y - source_window.y, - w: scaled_roi.w, - h: scaled_roi.h, - }; - let pack_params = - fast420_windowed_pack_params_for_dims((source_window.w, source_window.h), fmt, local_roi); - let y_len = source_window.w as usize * source_window.h as usize; - let chroma_len = source_window.w.div_ceil(2) as usize * source_window.h.div_ceil(2) as usize; - let y_plane = new_decode_plane_buffer(&runtime.device, y_len, false); - let chroma_buffers = [ - new_private_buffer(&runtime.device, chroma_len), - new_private_buffer(&runtime.device, chroma_len), - ]; - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let out_buffer = runtime.device.new_buffer( - (pack_params.out_stride as usize * scaled_roi.h as usize) as u64, - MTLResourceOptions::StorageModeShared, - ); - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast420_scaled_region_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&chroma_buffers[0]), 0); - decoder_encoder.set_buffer(3, Some(&chroma_buffers[1]), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const decode_params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast420_scaled_region_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - - let pack_encoder = command_buffer.new_compute_command_encoder(); - let pack_pipeline = pack_420_windowed_pipeline_for_format(runtime, fmt); - pack_encoder.set_compute_pipeline_state(pack_pipeline); - pack_encoder.set_buffer(0, Some(&y_plane), 0); - pack_encoder.set_buffer(1, Some(&chroma_buffers[0]), 0); - pack_encoder.set_buffer(2, Some(&chroma_buffers[1]), 0); - pack_encoder.set_buffer(3, Some(&out_buffer), 0); - pack_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const pack_params).cast(), - ); - dispatch_2d_pipeline(pack_encoder, pack_pipeline, (scaled_roi.w, scaled_roi.h)); - pack_encoder.end_encoding(); - - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { - return Err(decode_error_from_cpu(decoder, fmt, status)); - } - - Ok(Some(Surface::from_metal_buffer( - out_buffer, - (scaled_roi.w, scaled_roi.h), - fmt, - ))) -} - -#[cfg(target_os = "macos")] -fn fast444_plane_mode(decoder: &CpuDecoder<'_>) -> PlaneMode { - match decoder.info().color_space { - JpegColorSpace::Rgb => PlaneMode::Rgb, - _ => PlaneMode::YCbCr, - } -} - -#[cfg(target_os = "macos")] -fn try_decode_fast444_to_surface( - runtime: &MetalRuntime, - decoder: &CpuDecoder<'_>, - packet: Option<&JpegMetalFast444PacketV1>, - fmt: PixelFormat, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - let Some(_) = pixel_format_to_out_format(fmt) else { - return Ok(None); - }; - - let params = fast444_params(packet); - let mode = fast444_plane_mode(decoder); - let plane_len = params.width as usize * params.height as usize; - let y_plane = new_decode_plane_buffer( - &runtime.device, - plane_len, - fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, - ); - let chroma_blue_plane = new_private_buffer(&runtime.device, plane_len); - let chroma_red_plane = new_private_buffer(&runtime.device, plane_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast444_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&chroma_blue_plane), 0); - decoder_encoder.set_buffer(3, Some(&chroma_red_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast444_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { - return Err(decode_error_from_cpu(decoder, fmt, status)); - } - - PlaneStage { - dims: packet.dimensions, - mode, - plane0: y_plane, - plane1: Some(chroma_blue_plane), - plane2: Some(chroma_red_plane), - } - .finish_resident_with_runtime(runtime, fmt) - .map(Some) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast444_to_private_rgb8_tile( - runtime: &MetalRuntime, - decoder: &CpuDecoder<'_>, - packet: Option<&JpegMetalFast444PacketV1>, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - - let params = fast444_params(packet); - let mode = fast444_plane_mode(decoder); - let plane_len = params.width as usize * params.height as usize; - let y_plane = new_private_buffer(&runtime.device, plane_len); - let chroma_blue_plane = new_private_buffer(&runtime.device, plane_len); - let chroma_red_plane = new_private_buffer(&runtime.device, plane_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast444_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&chroma_blue_plane), 0); - decoder_encoder.set_buffer(3, Some(&chroma_red_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast444_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { - return Err(decode_error_from_cpu(decoder, PixelFormat::Rgb8, status)); - } - - Ok(Some( - PlaneStage { - dims: packet.dimensions, - mode, - plane0: y_plane, - plane1: Some(chroma_blue_plane), - plane2: Some(chroma_red_plane), - } - .dispatch_private_rgb8_with_runtime(runtime, status_buffer), - )) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast444_region_to_surface( - runtime: &MetalRuntime, - decoder: &CpuDecoder<'_>, - packet: Option<&JpegMetalFast444PacketV1>, - fmt: PixelFormat, - roi: signinum_jpeg::Rect, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - let Some(_) = pixel_format_to_out_format(fmt) else { - return Ok(None); - }; - - let mut params = fast444_region_params(packet, roi); - let (first_mcu, end_mcu) = mcu_range_for_rect(roi, packet.mcus_per_row, packet.mcu_rows, 8, 8); - let total_mcus = packet.mcus_per_row * packet.mcu_rows; - let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( - &packet.restart_offsets, - packet.restart_interval_mcus, - total_mcus, - first_mcu, - end_mcu, - ); - params.restart_start_mcu = restart_start_mcu; - params.restart_offset_count = entropy_segment_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let mode = fast444_plane_mode(decoder); - let plane_len = params.width as usize * params.height as usize; - let y_plane = new_decode_plane_buffer( - &runtime.device, - plane_len, - fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, - ); - let chroma_blue_plane = new_private_buffer(&runtime.device, plane_len); - let chroma_red_plane = new_private_buffer(&runtime.device, plane_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast444_region_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&chroma_blue_plane), 0); - decoder_encoder.set_buffer(3, Some(&chroma_red_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast444_region_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { - return Err(decode_error_from_cpu(decoder, fmt, status)); - } - - PlaneStage { - dims: (roi.w, roi.h), - mode, - plane0: y_plane, - plane1: Some(chroma_blue_plane), - plane2: Some(chroma_red_plane), - } - .finish_resident_with_runtime(runtime, fmt) - .map(Some) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast444_scaled_to_surface( - runtime: &MetalRuntime, - decoder: &CpuDecoder<'_>, - packet: Option<&JpegMetalFast444PacketV1>, - fmt: PixelFormat, - scale: signinum_core::Downscale, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - let Some(_) = pixel_format_to_out_format(fmt) else { - return Ok(None); - }; - let Some(params) = fast444_scaled_params(packet, scale) else { - return Ok(None); - }; - - let mode = fast444_plane_mode(decoder); - let plane_len = params.scaled_width as usize * params.scaled_height as usize; - let y_plane = new_decode_plane_buffer( - &runtime.device, - plane_len, - fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, - ); - let chroma_blue_plane = new_private_buffer(&runtime.device, plane_len); - let chroma_red_plane = new_private_buffer(&runtime.device, plane_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - packet.restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, &packet.restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast444_scaled_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&chroma_blue_plane), 0); - decoder_encoder.set_buffer(3, Some(&chroma_red_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast444_scaled_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { - return Err(decode_error_from_cpu(decoder, fmt, status)); - } - - PlaneStage { - dims: (params.scaled_width, params.scaled_height), - mode, - plane0: y_plane, - plane1: Some(chroma_blue_plane), - plane2: Some(chroma_red_plane), - } - .finish_resident_with_runtime(runtime, fmt) - .map(Some) -} - -#[cfg(target_os = "macos")] -fn try_decode_fast444_scaled_region_to_surface( - runtime: &MetalRuntime, - decoder: &CpuDecoder<'_>, - packet: Option<&JpegMetalFast444PacketV1>, - fmt: PixelFormat, - scaled_roi: signinum_jpeg::Rect, - scale: signinum_core::Downscale, -) -> Result, Error> { - let Some(packet) = packet else { - return Ok(None); - }; - let Some(_) = pixel_format_to_out_format(fmt) else { - return Ok(None); - }; - let Some(mut params) = fast444_scaled_region_params(packet, scale, scaled_roi) else { - return Ok(None); - }; - let mcu_size = 8u32 >> params.scale_shift; - let (first_mcu, end_mcu) = mcu_range_for_rect( - scaled_roi, - packet.mcus_per_row, - packet.mcu_rows, - mcu_size, - mcu_size, - ); - let total_mcus = packet.mcus_per_row * packet.mcu_rows; - let (restart_start_mcu, restart_offsets) = restart_work_for_mcu_range( - &packet.restart_offsets, - packet.restart_interval_mcus, - total_mcus, - first_mcu, - end_mcu, - ); - params.restart_start_mcu = restart_start_mcu; - params.restart_offset_count = entropy_segment_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - - let mode = fast444_plane_mode(decoder); - let plane_len = params.scaled_width as usize * params.scaled_height as usize; - let y_plane = new_decode_plane_buffer( - &runtime.device, - plane_len, - fmt == PixelFormat::Gray8 && mode != PlaneMode::Rgb, - ); - let chroma_blue_plane = new_private_buffer(&runtime.device, plane_len); - let chroma_red_plane = new_private_buffer(&runtime.device, plane_len); - let decode_threads = entropy_decode_thread_count( - packet.restart_interval_mcus, - restart_offsets.len(), - packet.entropy_checkpoints.len(), - ); - let status_buffer = decode_status_buffer(&runtime.device, decode_threads); - let entropy_buffer = runtime.device.new_buffer_with_data( - packet.entropy_bytes.as_ptr().cast(), - packet.entropy_bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - let restart_offsets_buffer = restart_offsets_buffer(&runtime.device, restart_offsets)?; - let entropy_checkpoints_buffer = - entropy_checkpoints_buffer(&runtime.device, &packet.entropy_checkpoints)?; - - let dc_tables = [ - PreparedHuffmanHost::from(&packet.y_dc_table), - PreparedHuffmanHost::from(&packet.cb_dc_table), - PreparedHuffmanHost::from(&packet.cr_dc_table), - ]; - let ac_tables = [ - PreparedHuffmanHost::from(&packet.y_ac_table), - PreparedHuffmanHost::from(&packet.cb_ac_table), - PreparedHuffmanHost::from(&packet.cr_ac_table), - ]; - - let command_buffer = runtime.queue.new_command_buffer(); - let decoder_encoder = command_buffer.new_compute_command_encoder(); - decoder_encoder.set_compute_pipeline_state(&runtime.fast444_scaled_region_decode_pipeline); - decoder_encoder.set_buffer(0, Some(&entropy_buffer), 0); - decoder_encoder.set_buffer(1, Some(&y_plane), 0); - decoder_encoder.set_buffer(2, Some(&chroma_blue_plane), 0); - decoder_encoder.set_buffer(3, Some(&chroma_red_plane), 0); - decoder_encoder.set_bytes( - 4, - size_of::() as u64, - (&raw const params).cast(), - ); - decoder_encoder.set_bytes( - 5, - size_of::<[u16; 64]>() as u64, - packet.y_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 6, - size_of::<[u16; 64]>() as u64, - packet.cb_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 7, - size_of::<[u16; 64]>() as u64, - packet.cr_quant.as_ptr().cast(), - ); - decoder_encoder.set_bytes( - 8, - size_of::() as u64, - (&raw const dc_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 9, - size_of::() as u64, - (&raw const ac_tables[0]).cast(), - ); - decoder_encoder.set_bytes( - 10, - size_of::() as u64, - (&raw const dc_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 11, - size_of::() as u64, - (&raw const ac_tables[1]).cast(), - ); - decoder_encoder.set_bytes( - 12, - size_of::() as u64, - (&raw const dc_tables[2]).cast(), - ); - decoder_encoder.set_bytes( - 13, - size_of::() as u64, - (&raw const ac_tables[2]).cast(), - ); - decoder_encoder.set_buffer(14, Some(&restart_offsets_buffer), 0); - decoder_encoder.set_buffer(15, Some(&status_buffer), 0); - decoder_encoder.set_buffer(16, Some(&entropy_checkpoints_buffer), 0); - dispatch_1d_pipeline( - decoder_encoder, - &runtime.fast444_scaled_region_decode_pipeline, - decode_threads, - ); - decoder_encoder.end_encoding(); - command_buffer.commit(); - command_buffer.wait_until_completed(); - - if let Some(status) = first_decode_error_status(&status_buffer, decode_threads) { - return Err(decode_error_from_cpu(decoder, fmt, status)); - } - - PlaneStage { - dims: (scaled_roi.w, scaled_roi.h), - mode, - plane0: y_plane, - plane1: Some(chroma_blue_plane), - plane2: Some(chroma_red_plane), - } - .finish_resident_with_runtime(runtime, fmt) - .map(Some) -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_to_surface( - decoder: &CpuDecoder<'_>, - pool: &mut signinum_jpeg::ScratchPool, - fmt: PixelFormat, - fast444_packet: Option<&JpegMetalFast444PacketV1>, - fast422_packet: Option<&JpegMetalFast422PacketV1>, - fast420_packet: Option<&JpegMetalFast420PacketV1>, -) -> Result { - with_runtime(|runtime| { - decode_to_surface_with_runtime( - runtime, - decoder, - pool, - fmt, - fast444_packet, - fast422_packet, - fast420_packet, - ) - }) -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_to_surface_with_session( - decoder: &CpuDecoder<'_>, - pool: &mut signinum_jpeg::ScratchPool, - fmt: PixelFormat, - fast444_packet: Option<&JpegMetalFast444PacketV1>, - fast422_packet: Option<&JpegMetalFast422PacketV1>, - fast420_packet: Option<&JpegMetalFast420PacketV1>, - session: &crate::MetalBackendSession, -) -> Result { - with_runtime_for_session(session, |runtime| { - decode_to_surface_with_runtime( - runtime, - decoder, - pool, - fmt, - fast444_packet, - fast422_packet, - fast420_packet, - ) - }) -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_private_rgb8_tile_with_session( - decoder: &CpuDecoder<'_>, - fast444_packet: Option<&JpegMetalFast444PacketV1>, - fast422_packet: Option<&JpegMetalFast422PacketV1>, - fast420_packet: Option<&JpegMetalFast420PacketV1>, - session: &crate::MetalBackendSession, -) -> Result { - with_runtime_for_session(session, |runtime| { - if let Some(tile) = - try_decode_fast444_to_private_rgb8_tile(runtime, decoder, fast444_packet)? - { - return Ok(tile); - } - if let Some(decoded) = decode_fast422_to_rgb_buffer( - runtime, - fast422_packet, - PixelFormat::Rgb8, - MTLResourceOptions::StorageModePrivate, - )? { - return Ok(private_jpeg_tile_from_fast_rgb_buffer(decoded)); - } - if let Some(decoded) = decode_fast420_to_rgb_buffer( - runtime, - decoder, - fast420_packet, - PixelFormat::Rgb8, - MTLResourceOptions::StorageModePrivate, - )? { - return Ok(private_jpeg_tile_from_fast_rgb_buffer(decoded)); - } - Err(Error::UnsupportedMetalRequest { - reason: - "private JPEG Metal output supports only fast baseline 4:4:4, 4:2:2, or 4:2:0 RGB8 full-tile decode", - }) - }) -} - -#[cfg(target_os = "macos")] -fn decode_to_surface_with_runtime( - runtime: &MetalRuntime, - decoder: &CpuDecoder<'_>, - pool: &mut signinum_jpeg::ScratchPool, - fmt: PixelFormat, - fast444_packet: Option<&JpegMetalFast444PacketV1>, - fast422_packet: Option<&JpegMetalFast422PacketV1>, - fast420_packet: Option<&JpegMetalFast420PacketV1>, -) -> Result { - if let Some(surface) = try_decode_fast444_to_surface(runtime, decoder, fast444_packet, fmt)? { - return Ok(surface); - } - if let Some(surface) = try_decode_fast422_to_surface(runtime, fast422_packet, fmt)? { - return Ok(surface); - } - if let Some(surface) = try_decode_fast420_to_surface(runtime, decoder, fast420_packet, fmt)? { - return Ok(surface); - } - let mut stage = PlaneStage::new( - &runtime.device, - decoder.info().color_space, - decoder.info().dimensions, - )?; - decoder.decode_component_rows_with_scratch(pool, &mut stage)?; - stage.finish_with_runtime(runtime, fmt) -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_region_to_surface( - decoder: &CpuDecoder<'_>, - pool: &mut signinum_jpeg::ScratchPool, - fmt: PixelFormat, - roi: signinum_jpeg::Rect, - fast444_packet: Option<&JpegMetalFast444PacketV1>, - fast422_packet: Option<&JpegMetalFast422PacketV1>, - fast420_packet: Option<&JpegMetalFast420PacketV1>, -) -> Result { - with_runtime(|runtime| { - if let Some(surface) = - try_decode_fast444_region_to_surface(runtime, decoder, fast444_packet, fmt, roi)? - { - return Ok(surface); - } - if let Some(surface) = - try_decode_fast422_region_to_surface(runtime, fast422_packet, fmt, roi)? - { - return Ok(surface); - } - if let Some(surface) = - try_decode_fast420_region_to_surface(runtime, decoder, fast420_packet, fmt, roi)? - { - return Ok(surface); - } - let dims = (roi.w, roi.h); - let mut stage = cached_plane_stage(&runtime.device, decoder.info().color_space, dims)?; - decoder.decode_region_component_rows_with_scratch( - pool, - &mut stage, - roi, - signinum_core::Downscale::None, - )?; - stage.finish_with_runtime(runtime, fmt) - }) -} - -#[cfg(target_os = "macos")] -pub(crate) fn decode_scaled_to_surface( - decoder: &CpuDecoder<'_>, - pool: &mut signinum_jpeg::ScratchPool, - fmt: PixelFormat, - scale: signinum_core::Downscale, - fast444_packet: Option<&JpegMetalFast444PacketV1>, - fast422_packet: Option<&JpegMetalFast422PacketV1>, - fast420_packet: Option<&JpegMetalFast420PacketV1>, -) -> Result { - with_runtime(|runtime| { - if let Some(surface) = - try_decode_fast444_scaled_to_surface(runtime, decoder, fast444_packet, fmt, scale)? - { - return Ok(surface); - } - if let Some(surface) = - try_decode_fast422_scaled_to_surface(runtime, fast422_packet, fmt, scale)? - { - return Ok(surface); - } - if let Some(surface) = - try_decode_fast420_scaled_to_surface(runtime, decoder, fast420_packet, fmt, scale)? - { - return Ok(surface); - } - let full = decoder.info().dimensions; - let roi = signinum_jpeg::Rect { - x: 0, - y: 0, - w: full.0, - h: full.1, - }; - let scaled = (Rect { - x: 0, - y: 0, - w: full.0, - h: full.1, - }) - .scaled_covering(scale); - let mut stage = cached_plane_stage( - &runtime.device, - decoder.info().color_space, - (scaled.w, scaled.h), - )?; - decoder.decode_region_component_rows_with_scratch(pool, &mut stage, roi, scale)?; - stage.finish_with_runtime(runtime, fmt) - }) -} - -#[cfg(target_os = "macos")] -#[allow(clippy::too_many_arguments)] -pub(crate) fn decode_region_scaled_to_surface( - decoder: &CpuDecoder<'_>, - pool: &mut signinum_jpeg::ScratchPool, - fmt: PixelFormat, - roi: signinum_jpeg::Rect, - scale: signinum_core::Downscale, - fast444_packet: Option<&JpegMetalFast444PacketV1>, - fast422_packet: Option<&JpegMetalFast422PacketV1>, - fast420_packet: Option<&JpegMetalFast420PacketV1>, -) -> Result { - with_runtime(|runtime| { - let scaled_roi = (Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }) - .scaled_covering(scale); - if let Some(surface) = try_decode_fast444_scaled_region_to_surface( - runtime, - decoder, - fast444_packet, - fmt, - signinum_jpeg::Rect { - x: scaled_roi.x, - y: scaled_roi.y, - w: scaled_roi.w, - h: scaled_roi.h, - }, - scale, - )? { - return Ok(surface); - } - if let Some(surface) = try_decode_fast422_scaled_region_to_surface( - runtime, - fast422_packet, - fmt, - signinum_jpeg::Rect { - x: scaled_roi.x, - y: scaled_roi.y, - w: scaled_roi.w, - h: scaled_roi.h, - }, - scale, - )? { - return Ok(surface); - } - if let Some(surface) = try_decode_fast420_scaled_region_to_surface( - runtime, - decoder, - fast420_packet, - fmt, - signinum_jpeg::Rect { - x: scaled_roi.x, - y: scaled_roi.y, - w: scaled_roi.w, - h: scaled_roi.h, - }, - scale, - )? { - return Ok(surface); - } - let scaled = (Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }) - .scaled_covering(scale); - let mut stage = cached_plane_stage( - &runtime.device, - decoder.info().color_space, - (scaled.w, scaled.h), - )?; - decoder.decode_region_component_rows_with_scratch(pool, &mut stage, roi, scale)?; - stage.finish_with_runtime(runtime, fmt) - }) -} - -#[cfg(target_os = "macos")] -pub(crate) fn compose_rgb_viewport_from_regions( - decoder: &CpuDecoder<'_>, - pool: &mut signinum_jpeg::ScratchPool, - scale: signinum_core::Downscale, - viewport_dims: (u32, u32), - tiles: &[ViewportTile], -) -> Result { - with_runtime(|runtime| { - let mut stage = - cached_plane_stage(&runtime.device, decoder.info().color_space, viewport_dims)?; - for tile in tiles { - let dims = tile.source_roi.scaled_covering(scale); - if (dims.w, dims.h) != (tile.dest.w, tile.dest.h) { - return Err(Error::MetalKernel { - message: format!( - "viewport tile dims {:?} do not match destination rect {:?}", - (dims.w, dims.h), - tile.dest - ), - }); - } - let mut writer = ViewportPlaneWriter { - stage: &mut stage, - dest: tile.dest, - }; - decoder.decode_region_component_rows_with_scratch( - pool, - &mut writer, - signinum_jpeg::Rect { - x: tile.source_roi.x, - y: tile.source_roi.y, - w: tile.source_roi.w, - h: tile.source_roi.h, - }, - scale, - )?; - } - stage.finish_with_runtime(runtime, PixelFormat::Rgb8) - }) -} - -#[cfg(all(test, target_os = "macos"))] -mod tests { - use super::*; - use crate::Storage; - use std::sync::Arc; - - const BASELINE_420: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); - const BASELINE_420_RESTART: &[u8] = - include_bytes!("../fixtures/jpeg/baseline_420_restart_32x16.jpg"); - const BASELINE_422: &[u8] = include_bytes!("../fixtures/jpeg/baseline_422_16x8.jpg"); - const BASELINE_444: &[u8] = include_bytes!("../fixtures/jpeg/baseline_444_8x8.jpg"); - - #[test] - fn mcu_range_for_rect_covers_only_touched_rows_and_columns() { - let roi = signinum_jpeg::Rect { - x: 19, - y: 35, - w: 11, - h: 34, - }; - - assert_eq!(mcu_range_for_rect(roi, 8, 6, 16, 16), (17, 34)); - } - - #[test] - fn restart_work_for_mcu_range_slices_to_overlapping_restart_segments() { - let restart_offsets = [0, 10, 20, 30, 40, 50]; - - let (restart_start_mcu, offsets) = - restart_work_for_mcu_range(&restart_offsets, 4, 24, 9, 15); - - assert_eq!(restart_start_mcu, 8); - assert_eq!(offsets, &[20, 30]); - } - - #[test] - fn runtime_initialization_error_classifies_device_unavailable() { - assert!(matches!( - runtime_initialization_error("Metal is unavailable on this host"), - Error::MetalUnavailable - )); - assert!(matches!( - runtime_initialization_error("failed to compile Metal library"), - Error::MetalKernel { .. } - )); - } - - #[test] - fn shader_decode_block_clears_coefficients_with_vector_stores() { - assert!( - SHADER_SOURCE.contains("thread short4 *coeff_chunks"), - "decode_block should clear coeffs with packed short4 stores" - ); - assert!( - SHADER_SOURCE.contains("coeff_chunks[i] = short4(0);"), - "decode_block should zero each packed coefficient chunk" - ); - } - - #[test] - fn shader_source_keeps_entropy_fast_paths() { - assert!(SHADER_SOURCE.contains("inline bool refill_four_bytes(")); - assert!( - SHADER_SOURCE.contains("return refill_four_bytes(br, bytes, len) || refill_one_byte") - ); - assert!(SHADER_SOURCE.contains("ensure_bits_padded(br, bytes, len, 9)")); - assert!(SHADER_SOURCE.contains("table.fast_len[fast_index]")); - assert!(SHADER_SOURCE.contains("inline bool decode_block_skip(")); - assert!(SHADER_SOURCE.contains("skip_receive_extend(br, bytes, len, ssss, status)")); - assert!(SHADER_SOURCE.contains("inline bool configure_batch_entropy_thread(")); - } - - #[test] - fn shader_kernels_use_incremental_mx_my() { - assert!( - SHADER_SOURCE.contains("inline void init_mcu_cursor("), - "fast decode kernels should seed mx/my via init_mcu_cursor instead of dividing per MCU" - ); - assert!( - SHADER_SOURCE.contains("inline void advance_mcu_cursor("), - "fast decode kernels should carry mx/my via advance_mcu_cursor instead of dividing per MCU" - ); - assert!( - !SHADER_SOURCE.contains("mcu_index / params.mcus_per_row"), - "no fast kernel should still divide mcu_index by mcus_per_row inside the MCU loop" - ); - assert!( - !SHADER_SOURCE.contains("mcu_index % params.mcus_per_row"), - "no fast kernel should still modulo mcu_index by mcus_per_row inside the MCU loop" - ); - } - - #[test] - fn split_fast420_batch_env_requires_explicit_one() { - assert!(split_fast420_batch_value_enabled(Some( - std::ffi::OsStr::new("1") - ))); - assert!(!split_fast420_batch_value_enabled(Some( - std::ffi::OsStr::new("true") - ))); - assert!(!split_fast420_batch_value_enabled(None)); - } - - #[test] - fn fast420_batch_timing_env_requires_explicit_one() { - assert!(fast420_batch_timing_value_enabled(Some( - std::ffi::OsStr::new("1") - ))); - assert!(!fast420_batch_timing_value_enabled(Some( - std::ffi::OsStr::new("true") - ))); - assert!(!fast420_batch_timing_value_enabled(None)); - } - - #[test] - fn one_dimensional_dispatch_width_tracks_work_without_full_threadgroup_waste() { - assert_eq!(choose_1d_threadgroup_width(32, 1024, 1), 32); - assert_eq!(choose_1d_threadgroup_width(32, 1024, 33), 64); - assert_eq!(choose_1d_threadgroup_width(32, 1024, 256), 256); - assert_eq!(choose_1d_threadgroup_width(32, 1024, 257), 256); - } - - #[test] - fn auto_batched_packets_skip_distinct_region_scaled_requests() { - let packet = signinum_jpeg::adapter::build_metal_fast420_packet(BASELINE_420_RESTART) - .expect("packet"); - let roi = Rect { - x: 0, - y: 0, - w: 16, - h: 16, - }; - let requests = vec![ - batch::QueuedRequest::new( - Arc::<[u8]>::from(BASELINE_420_RESTART), - PixelFormat::Rgb8, - BackendRequest::Auto, - batch::BatchOp::RegionScaled { - roi, - scale: signinum_core::Downscale::Quarter, - }, - None, - None, - Some(packet.clone()), - ), - batch::QueuedRequest::new( - Arc::<[u8]>::from(BASELINE_420_RESTART), - PixelFormat::Rgb8, - BackendRequest::Auto, - batch::BatchOp::RegionScaled { - roi, - scale: signinum_core::Downscale::Quarter, - }, - None, - None, - Some(packet), - ), - ]; - - assert!(batched_fast_packets(&requests) - .expect("packet lookup") - .is_none()); - } - - #[test] - fn auto_batched_packets_keep_large_repeated_region_scaled_requests_off_metal() { - let input = Arc::<[u8]>::from(BASELINE_420); - let packet = - signinum_jpeg::adapter::build_metal_fast420_packet(BASELINE_420).expect("packet"); - let roi = Rect { - x: 0, - y: 0, - w: 16, - h: 16, - }; - let requests = (0..=REGION_SCALED_BATCH_CHUNK) - .map(|_| { - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Auto, - batch::BatchOp::RegionScaled { - roi, - scale: signinum_core::Downscale::Quarter, - }, - None, - None, - Some(packet.clone()), - ) - }) - .collect::>(); - - assert!(batched_fast_packets(&requests) - .expect("packet lookup") - .is_none()); - } - - #[test] - fn auto_batched_packets_require_wsi_batch_threshold() { - let input = Arc::<[u8]>::from(BASELINE_420_RESTART); - let packet = signinum_jpeg::adapter::build_metal_fast420_packet(BASELINE_420_RESTART) - .expect("packet"); - let requests = (0..7) - .map(|_| { - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Auto, - batch::BatchOp::Full, - None, - None, - Some(packet.clone()), - ) - }) - .collect::>(); - - assert!(batched_fast_packets(&requests) - .expect("packet lookup") - .is_none()); - } - - #[test] - fn auto_batched_packets_accept_restart_wsi_batch_at_threshold() { - let input = Arc::<[u8]>::from(BASELINE_420_RESTART); - let packet = signinum_jpeg::adapter::build_metal_fast420_packet(BASELINE_420_RESTART) - .expect("packet"); - let requests = (0..8) - .map(|_| { - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Auto, - batch::BatchOp::Full, - None, - None, - Some(packet.clone()), - ) - }) - .collect::>(); - - assert!(batched_fast_packets(&requests) - .expect("packet lookup") - .is_some()); - } - - #[test] - fn auto_batched_packets_accept_large_nonrestart_wsi_batch_at_threshold() { - let input = Arc::<[u8]>::from(generated_rgb_jpeg(512)); - let fast444_packet = - signinum_jpeg::adapter::build_metal_fast444_packet(input.as_ref()).ok(); - let fast422_packet = - signinum_jpeg::adapter::build_metal_fast422_packet(input.as_ref()).ok(); - let fast420_packet = - signinum_jpeg::adapter::build_metal_fast420_packet(input.as_ref()).ok(); - assert!( - fast444_packet.is_some() || fast422_packet.is_some() || fast420_packet.is_some(), - "generated JPEG must be packet-decodable" - ); - let requests = (0..8) - .map(|_| { - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Auto, - batch::BatchOp::Full, - fast444_packet.clone(), - fast422_packet.clone(), - fast420_packet.clone(), - ) - }) - .collect::>(); - - assert!(batched_fast_packets(&requests) - .expect("packet lookup") - .is_some()); - } - - fn generated_rgb_jpeg(dim: u16) -> Vec { - let mut rgb = Vec::with_capacity(dim as usize * dim as usize * 3); - for y in 0..dim { - for x in 0..dim { - let xf = u32::from(x); - let yf = u32::from(y); - rgb.push(((xf * 13 + yf * 3) & 0xff) as u8); - rgb.push(((xf * 5 + yf * 11 + (xf ^ yf)) & 0xff) as u8); - rgb.push(((xf * 7 + yf * 17 + (xf.wrapping_mul(yf) >> 5)) & 0xff) as u8); - } - } - - let mut jpeg = Vec::new(); - let mut encoder = jpeg_encoder::Encoder::new(&mut jpeg, 90); - encoder.set_sampling_factor(jpeg_encoder::SamplingFactor::F_2_2); - encoder - .encode(&rgb, dim, dim, jpeg_encoder::ColorType::Rgb) - .expect("encode generated JPEG"); - jpeg - } - - #[test] - fn fast420_packet_scaled_decode_matches_cpu_scaled_bytes() { - let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); - let packet = - signinum_jpeg::adapter::build_metal_fast420_packet(BASELINE_420).expect("packet"); - for scale in [ - signinum_core::Downscale::Half, - signinum_core::Downscale::Quarter, - ] { - let (expected, _) = decoder - .decode_scaled(PixelFormat::Rgb8, scale) - .expect("cpu scaled"); - - let surface = with_runtime(|runtime| { - let surface = try_decode_fast420_scaled_to_surface( - runtime, - &decoder, - Some(&packet), - PixelFormat::Rgb8, - scale, - )? - .expect("fast420 scaled surface"); - Ok::<_, Error>(surface) - }) - .expect("metal scaled"); - - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast420_packet_region_decode_matches_cpu_region_bytes() { - let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); - let packet = - signinum_jpeg::adapter::build_metal_fast420_packet(BASELINE_420).expect("packet"); - let roi = signinum_jpeg::Rect { - x: 3, - y: 2, - w: 9, - h: 10, - }; - let (expected, _) = decoder - .decode_region(PixelFormat::Rgb8, roi) - .expect("cpu region"); - - let surface = with_runtime(|runtime| { - let surface = try_decode_fast420_region_to_surface( - runtime, - &decoder, - Some(&packet), - PixelFormat::Rgb8, - roi, - )? - .expect("fast420 region surface"); - Ok::<_, Error>(surface) - }) - .expect("metal region"); - - assert_eq!(surface.dimensions, (roi.w, roi.h)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - - #[test] - fn fast420_region_batch_decode_matches_cpu_region_bytes() { - let input = Arc::<[u8]>::from(BASELINE_420); - let packet = - signinum_jpeg::adapter::build_metal_fast420_packet(BASELINE_420).expect("packet"); - let roi = Rect { - x: 4, - y: 4, - w: 8, - h: 8, - }; - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Region(roi), - None, - None, - Some(packet.clone()), - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Region(roi), - None, - None, - Some(packet), - ), - ]; - let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); - let (expected, _) = decoder - .decode_region( - PixelFormat::Rgb8, - signinum_jpeg::Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }, - ) - .expect("cpu region"); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("region batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.dimensions, (roi.w, roi.h)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast420_full_batch_decode_uses_shared_surface_offsets() { - let input = Arc::<[u8]>::from(BASELINE_420); - let packet = - signinum_jpeg::adapter::build_metal_fast420_packet(BASELINE_420).expect("packet"); - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Full, - None, - None, - Some(packet.clone()), - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Full, - None, - None, - Some(packet), - ), - ]; - let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); - let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("fast420 full batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for (index, result) in results.iter().enumerate() { - let surface = result.as_ref().expect("surface"); - assert_eq!(surface.dimensions, (16, 16)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - let Storage::Metal { offset, .. } = &surface.storage else { - panic!("expected Metal storage"); - }; - assert_eq!(*offset, index * expected.len()); - } - } - - #[test] - fn fast420_split_full_batch_decode_matches_cpu_bytes() { - let jpeg = generated_rgb_jpeg(32); - let input = Arc::<[u8]>::from(jpeg.into_boxed_slice()); - let packet = - signinum_jpeg::adapter::build_metal_fast420_packet(input.as_ref()).expect("packet"); - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Full, - None, - None, - Some(packet.clone()), - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Full, - None, - None, - Some(packet), - ), - ]; - let packets = batched_fast_packets(&requests) - .expect("packet lookup") - .expect("packets"); - let decoder = CpuDecoder::new(input.as_ref()).expect("decoder"); - let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); - - let results = with_runtime(|runtime| { - try_decode_fast420_full_rgb_batch_to_surfaces_with_mode( - runtime, - &requests, - &packets, - Fast420BatchDecodeMode::SplitCoeffIdct, - ) - }) - .expect("batch result") - .expect("split fast420 full batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.dimensions, (32, 32)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast420_batch_clears_high_ac_before_dc_only_blocks() { - let input = Arc::<[u8]>::from(fast420_high_ac_then_dc_only_jpeg(1)); - let packet = - signinum_jpeg::adapter::build_metal_fast420_packet(input.as_ref()).expect("packet"); - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Full, - None, - None, - Some(packet.clone()), - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Full, - None, - None, - Some(packet), - ), - ]; - let decoder = CpuDecoder::new(input.as_ref()).expect("decoder"); - let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("fast420 full batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.dimensions, (16, 16)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast420_batch_matches_cpu_for_high_ac_overflow_coefficients() { - let input = Arc::<[u8]>::from(fast420_high_ac_then_dc_only_jpeg(u8::MAX)); - let packet = - signinum_jpeg::adapter::build_metal_fast420_packet(input.as_ref()).expect("packet"); - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Full, - None, - None, - Some(packet.clone()), - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Full, - None, - None, - Some(packet), - ), - ]; - let decoder = CpuDecoder::new(input.as_ref()).expect("decoder"); - let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("fast420 full batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.dimensions, (16, 16)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast420_packet_region_scaled_decode_matches_cpu_region_scaled_bytes() { - let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); - let packet = - signinum_jpeg::adapter::build_metal_fast420_packet(BASELINE_420).expect("packet"); - let roi = signinum_jpeg::Rect { - x: 3, - y: 2, - w: 9, - h: 10, - }; - let scale = signinum_core::Downscale::Quarter; - let (expected, _) = decoder - .decode_region_scaled(PixelFormat::Rgb8, roi, scale) - .expect("cpu region scaled"); - let scaled_roi = signinum_jpeg::Rect { - x: roi.x / 4, - y: roi.y / 4, - w: (roi.x + roi.w).div_ceil(4) - (roi.x / 4), - h: (roi.y + roi.h).div_ceil(4) - (roi.y / 4), - }; - - let surface = with_runtime(|runtime| { - let surface = try_decode_fast420_scaled_region_to_surface( - runtime, - &decoder, - Some(&packet), - PixelFormat::Rgb8, - scaled_roi, - scale, - )? - .expect("fast420 scaled region surface"); - Ok::<_, Error>(surface) - }) - .expect("metal region scaled"); - - assert_eq!(surface.dimensions, (scaled_roi.w, scaled_roi.h)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - - #[test] - fn fast420_region_scaled_batch_decode_matches_cpu_region_scaled_bytes() { - let input = Arc::<[u8]>::from(BASELINE_420); - let packet = - signinum_jpeg::adapter::build_metal_fast420_packet(BASELINE_420).expect("packet"); - let roi = Rect { - x: 3, - y: 2, - w: 9, - h: 10, - }; - let scale = signinum_core::Downscale::Quarter; - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::RegionScaled { roi, scale }, - None, - None, - Some(packet.clone()), - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::RegionScaled { roi, scale }, - None, - None, - Some(packet), - ), - ]; - let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); - let (expected, _) = decoder - .decode_region_scaled( - PixelFormat::Rgb8, - signinum_jpeg::Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }, - scale, - ) - .expect("cpu region scaled"); - let scaled = roi.scaled_covering(scale); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("region scaled batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.dimensions, (scaled.w, scaled.h)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast420_scaled_batch_decode_matches_cpu_scaled_bytes() { - let input = Arc::<[u8]>::from(BASELINE_420); - let packet = - signinum_jpeg::adapter::build_metal_fast420_packet(BASELINE_420).expect("packet"); - let scale = signinum_core::Downscale::Quarter; - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Scaled(scale), - None, - None, - Some(packet.clone()), - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Scaled(scale), - None, - None, - Some(packet), - ), - ]; - let decoder = CpuDecoder::new(BASELINE_420).expect("decoder"); - let (expected, _) = decoder - .decode_scaled(PixelFormat::Rgb8, scale) - .expect("cpu scaled"); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("scaled batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.dimensions, (4, 4)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast422_packet_full_decode_matches_cpu_bytes() { - let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); - let packet = - signinum_jpeg::adapter::build_metal_fast422_packet(BASELINE_422).expect("packet"); - let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); - - let surface = with_runtime(|runtime| { - let surface = try_decode_fast422_to_surface(runtime, Some(&packet), PixelFormat::Rgb8)? - .expect("fast422 surface"); - Ok::<_, Error>(surface) - }) - .expect("metal full decode"); - - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - - #[test] - fn fast422_full_batch_decode_matches_cpu_bytes() { - let input = Arc::<[u8]>::from(BASELINE_422); - let packet = - signinum_jpeg::adapter::build_metal_fast422_packet(BASELINE_422).expect("packet"); - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Full, - None, - Some(packet.clone()), - None, - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Full, - None, - Some(packet), - None, - ), - ]; - let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); - let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("fast422 batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for (index, result) in results.iter().enumerate() { - let surface = result.as_ref().expect("surface"); - assert_eq!(surface.dimensions, (16, 8)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - let Storage::Metal { offset, .. } = &surface.storage else { - panic!("expected Metal storage"); - }; - assert_eq!(*offset, index * expected.len()); - } - } - - #[test] - fn fast422_packet_region_decode_matches_cpu_region_bytes() { - let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); - let packet = - signinum_jpeg::adapter::build_metal_fast422_packet(BASELINE_422).expect("packet"); - let roi = signinum_jpeg::Rect { - x: 3, - y: 1, - w: 9, - h: 5, - }; - let (expected, _) = decoder - .decode_region(PixelFormat::Rgb8, roi) - .expect("cpu region"); - - let surface = with_runtime(|runtime| { - let surface = try_decode_fast422_region_to_surface( - runtime, - Some(&packet), - PixelFormat::Rgb8, - roi, - )? - .expect("fast422 region surface"); - Ok::<_, Error>(surface) - }) - .expect("metal region"); - - assert_eq!(surface.dimensions, (roi.w, roi.h)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - - #[test] - fn fast422_region_batch_decode_matches_cpu_region_bytes() { - let input = Arc::<[u8]>::from(BASELINE_422); - let packet = - signinum_jpeg::adapter::build_metal_fast422_packet(BASELINE_422).expect("packet"); - let roi = Rect { - x: 3, - y: 1, - w: 9, - h: 5, - }; - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Region(roi), - None, - Some(packet.clone()), - None, - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Region(roi), - None, - Some(packet), - None, - ), - ]; - let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); - let (expected, _) = decoder - .decode_region( - PixelFormat::Rgb8, - signinum_jpeg::Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }, - ) - .expect("cpu region"); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("fast422 region batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.dimensions, (roi.w, roi.h)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast422_packet_scaled_decode_matches_cpu_scaled_bytes() { - let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); - let packet = - signinum_jpeg::adapter::build_metal_fast422_packet(BASELINE_422).expect("packet"); - for (scale, dims) in [ - (signinum_core::Downscale::Half, (8, 4)), - (signinum_core::Downscale::Quarter, (4, 2)), - ] { - let (expected, _) = decoder - .decode_scaled(PixelFormat::Rgb8, scale) - .expect("cpu scaled"); - - let surface = with_runtime(|runtime| { - let surface = try_decode_fast422_scaled_to_surface( - runtime, - Some(&packet), - PixelFormat::Rgb8, - scale, - )? - .expect("fast422 scaled surface"); - Ok::<_, Error>(surface) - }) - .expect("metal scaled"); - - assert_eq!(surface.dimensions, dims); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast422_scaled_batch_decode_matches_cpu_scaled_bytes() { - let input = Arc::<[u8]>::from(BASELINE_422); - let packet = - signinum_jpeg::adapter::build_metal_fast422_packet(BASELINE_422).expect("packet"); - let scale = signinum_core::Downscale::Quarter; - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Scaled(scale), - None, - Some(packet.clone()), - None, - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Scaled(scale), - None, - Some(packet), - None, - ), - ]; - let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); - let (expected, _) = decoder - .decode_scaled(PixelFormat::Rgb8, scale) - .expect("cpu scaled"); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("fast422 scaled batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.dimensions, (4, 2)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast422_packet_region_scaled_decode_matches_cpu_region_scaled_bytes() { - let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); - let packet = - signinum_jpeg::adapter::build_metal_fast422_packet(BASELINE_422).expect("packet"); - let roi = signinum_jpeg::Rect { - x: 3, - y: 1, - w: 9, - h: 5, - }; - let scale = signinum_core::Downscale::Half; - let (expected, _) = decoder - .decode_region_scaled(PixelFormat::Rgb8, roi, scale) - .expect("cpu region scaled"); - let scaled_roi = signinum_jpeg::Rect { - x: roi.x / 2, - y: roi.y / 2, - w: (roi.x + roi.w).div_ceil(2) - (roi.x / 2), - h: (roi.y + roi.h).div_ceil(2) - (roi.y / 2), - }; - - let surface = with_runtime(|runtime| { - let surface = try_decode_fast422_scaled_region_to_surface( - runtime, - Some(&packet), - PixelFormat::Rgb8, - scaled_roi, - scale, - )? - .expect("fast422 scaled region surface"); - Ok::<_, Error>(surface) - }) - .expect("metal region scaled"); - - assert_eq!(surface.dimensions, (scaled_roi.w, scaled_roi.h)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - - #[test] - fn fast422_region_scaled_batch_decode_matches_cpu_region_scaled_bytes() { - let input = Arc::<[u8]>::from(BASELINE_422); - let packet = - signinum_jpeg::adapter::build_metal_fast422_packet(BASELINE_422).expect("packet"); - let roi = Rect { - x: 3, - y: 1, - w: 9, - h: 5, - }; - let scale = signinum_core::Downscale::Half; - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::RegionScaled { roi, scale }, - None, - Some(packet.clone()), - None, - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::RegionScaled { roi, scale }, - None, - Some(packet), - None, - ), - ]; - let decoder = CpuDecoder::new(BASELINE_422).expect("decoder"); - let (expected, _) = decoder - .decode_region_scaled( - PixelFormat::Rgb8, - signinum_jpeg::Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }, - scale, - ) - .expect("cpu region scaled"); - let scaled = roi.scaled_covering(scale); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("fast422 region scaled batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.dimensions, (scaled.w, scaled.h)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast444_packet_full_decode_matches_cpu_bytes() { - let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); - let packet = - signinum_jpeg::adapter::build_metal_fast444_packet(BASELINE_444).expect("packet"); - let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("cpu full decode"); - - let surface = with_runtime(|runtime| { - let surface = - try_decode_fast444_to_surface(runtime, &decoder, Some(&packet), PixelFormat::Rgb8)? - .expect("fast444 surface"); - Ok::<_, Error>(surface) - }) - .expect("metal full decode"); - - assert_eq!( - surface.residency(), - crate::SurfaceResidency::MetalResidentDecode - ); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - - #[test] - fn fast444_packet_scaled_decode_matches_cpu_scaled_bytes() { - let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); - let packet = - signinum_jpeg::adapter::build_metal_fast444_packet(BASELINE_444).expect("packet"); - for scale in [ - signinum_core::Downscale::Half, - signinum_core::Downscale::Quarter, - ] { - let (expected, _) = decoder - .decode_scaled(PixelFormat::Rgb8, scale) - .expect("cpu scaled"); - - let surface = with_runtime(|runtime| { - let surface = try_decode_fast444_scaled_to_surface( - runtime, - &decoder, - Some(&packet), - PixelFormat::Rgb8, - scale, - )? - .expect("fast444 scaled surface"); - Ok::<_, Error>(surface) - }) - .expect("metal scaled"); - - assert_eq!( - surface.residency(), - crate::SurfaceResidency::MetalResidentDecode - ); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast444_packet_region_decode_matches_cpu_region_bytes() { - let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); - let packet = - signinum_jpeg::adapter::build_metal_fast444_packet(BASELINE_444).expect("packet"); - let roi = signinum_jpeg::Rect { - x: 1, - y: 2, - w: 5, - h: 4, - }; - let (expected, _) = decoder - .decode_region(PixelFormat::Rgb8, roi) - .expect("cpu region"); - - let surface = with_runtime(|runtime| { - let surface = try_decode_fast444_region_to_surface( - runtime, - &decoder, - Some(&packet), - PixelFormat::Rgb8, - roi, - )? - .expect("fast444 region surface"); - Ok::<_, Error>(surface) - }) - .expect("metal region"); - - assert_eq!(surface.dimensions, (roi.w, roi.h)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!( - surface.residency(), - crate::SurfaceResidency::MetalResidentDecode - ); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - - #[test] - fn fast444_region_batch_decode_matches_cpu_region_bytes() { - let input = Arc::<[u8]>::from(BASELINE_444); - let packet = - signinum_jpeg::adapter::build_metal_fast444_packet(BASELINE_444).expect("packet"); - let roi = Rect { - x: 1, - y: 2, - w: 5, - h: 4, - }; - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Region(roi), - Some(packet.clone()), - None, - None, - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Region(roi), - Some(packet), - None, - None, - ), - ]; - let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); - let (expected, _) = decoder - .decode_region( - PixelFormat::Rgb8, - signinum_jpeg::Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }, - ) - .expect("cpu region"); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("region batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.dimensions, (roi.w, roi.h)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!( - surface.residency(), - crate::SurfaceResidency::MetalResidentDecode - ); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast444_packet_region_scaled_decode_matches_cpu_region_scaled_bytes() { - let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); - let packet = - signinum_jpeg::adapter::build_metal_fast444_packet(BASELINE_444).expect("packet"); - let roi = signinum_jpeg::Rect { - x: 1, - y: 2, - w: 5, - h: 4, - }; - let scale = signinum_core::Downscale::Quarter; - let (expected, _) = decoder - .decode_region_scaled(PixelFormat::Rgb8, roi, scale) - .expect("cpu region scaled"); - let scaled_roi = signinum_jpeg::Rect { - x: roi.x / 4, - y: roi.y / 4, - w: (roi.x + roi.w).div_ceil(4) - (roi.x / 4), - h: (roi.y + roi.h).div_ceil(4) - (roi.y / 4), - }; - - let surface = with_runtime(|runtime| { - let surface = try_decode_fast444_scaled_region_to_surface( - runtime, - &decoder, - Some(&packet), - PixelFormat::Rgb8, - scaled_roi, - scale, - )? - .expect("fast444 scaled region surface"); - Ok::<_, Error>(surface) - }) - .expect("metal region scaled"); - - assert_eq!(surface.dimensions, (scaled_roi.w, scaled_roi.h)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!( - surface.residency(), - crate::SurfaceResidency::MetalResidentDecode - ); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - - #[test] - fn fast444_region_scaled_batch_decode_matches_cpu_region_scaled_bytes() { - let input = Arc::<[u8]>::from(BASELINE_444); - let packet = - signinum_jpeg::adapter::build_metal_fast444_packet(BASELINE_444).expect("packet"); - let roi = Rect { - x: 1, - y: 2, - w: 5, - h: 4, - }; - let scale = signinum_core::Downscale::Quarter; - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::RegionScaled { roi, scale }, - Some(packet.clone()), - None, - None, - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::RegionScaled { roi, scale }, - Some(packet), - None, - None, - ), - ]; - let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); - let (expected, _) = decoder - .decode_region_scaled( - PixelFormat::Rgb8, - signinum_jpeg::Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }, - scale, - ) - .expect("cpu region scaled"); - let scaled = roi.scaled_covering(scale); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("region scaled batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.dimensions, (scaled.w, scaled.h)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!( - surface.residency(), - crate::SurfaceResidency::MetalResidentDecode - ); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - #[test] - fn fast444_scaled_batch_decode_matches_cpu_scaled_bytes() { - let input = Arc::<[u8]>::from(BASELINE_444); - let packet = - signinum_jpeg::adapter::build_metal_fast444_packet(BASELINE_444).expect("packet"); - let scale = signinum_core::Downscale::Quarter; - let requests = vec![ - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Scaled(scale), - Some(packet.clone()), - None, - None, - ), - batch::QueuedRequest::new( - Arc::clone(&input), - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Scaled(scale), - Some(packet), - None, - None, - ), - ]; - let decoder = CpuDecoder::new(BASELINE_444).expect("decoder"); - let (expected, _) = decoder - .decode_scaled(PixelFormat::Rgb8, scale) - .expect("cpu scaled"); - - let results = decode_full_batch_to_surfaces(&requests) - .expect("batch result") - .expect("scaled batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.dimensions, (2, 2)); - assert_eq!(surface.fmt, PixelFormat::Rgb8); - assert_eq!( - surface.residency(), - crate::SurfaceResidency::MetalResidentDecode - ); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - } - - fn fast420_high_ac_then_dc_only_jpeg(ac_quant: u8) -> Vec { - assert!(ac_quant > 0, "JPEG quant entries must be nonzero"); - - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0xff, 0xd8]); - - let mut quant = [1u8; 64]; - quant[63] = ac_quant; - let mut dqt = Vec::with_capacity(65); - dqt.push(0x00); - dqt.extend_from_slice(&quant); - append_jpeg_segment(&mut bytes, 0xdb, &dqt); - - append_jpeg_segment( - &mut bytes, - 0xc0, - &[ - 8, - 0, - 16, - 0, - 16, - 3, - 1, - (2 << 4) | 2, - 0, - 2, - (1 << 4) | 1, - 0, - 3, - (1 << 4) | 1, - 0, - ], - ); - - let mut dc_bits = [0u8; 16]; - dc_bits[0] = 1; - let mut dht_dc = Vec::with_capacity(18); - dht_dc.push(0x00); - dht_dc.extend_from_slice(&dc_bits); - dht_dc.push(0x00); - append_jpeg_segment(&mut bytes, 0xc4, &dht_dc); - - let mut ac_bits = [0u8; 16]; - ac_bits[1] = 3; - let mut dht_ac = Vec::with_capacity(20); - dht_ac.push(0x10); - dht_ac.extend_from_slice(&ac_bits); - dht_ac.extend_from_slice(&[0x00, 0xf0, 0xea]); - append_jpeg_segment(&mut bytes, 0xc4, &dht_ac); - - append_jpeg_segment(&mut bytes, 0xda, &[3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0]); - - bytes.extend_from_slice(&fast420_high_ac_entropy()); - bytes.extend_from_slice(&[0xff, 0xd9]); - bytes - } - - fn append_jpeg_segment(bytes: &mut Vec, marker: u8, payload: &[u8]) { - bytes.extend_from_slice(&[0xff, marker]); - let len = u16::try_from(payload.len() + 2).expect("JPEG segment length fits in u16"); - bytes.extend_from_slice(&len.to_be_bytes()); - bytes.extend_from_slice(payload); - } - - fn fast420_high_ac_entropy() -> Vec { - let mut writer = EntropyBitWriter::default(); - emit_high_ac_block(&mut writer); - for _ in 0..5 { - emit_dc_only_block(&mut writer); - } - writer.finish() - } - - fn emit_high_ac_block(writer: &mut EntropyBitWriter) { - writer.push_bits(0, 1); - for _ in 0..3 { - writer.push_bits(0b01, 2); - } - writer.push_bits(0b10, 2); - writer.push_bits(0b11_1111_1111, 10); - } - - fn emit_dc_only_block(writer: &mut EntropyBitWriter) { - writer.push_bits(0, 1); - writer.push_bits(0b00, 2); - } - - #[derive(Default)] - struct EntropyBitWriter { - bytes: Vec, - current: u8, - bit_count: u8, - } - - impl EntropyBitWriter { - fn push_bits(&mut self, bits: u16, len: u8) { - for shift in (0..len).rev() { - let bit = u8::from(((bits >> shift) & 1) != 0); - self.current = (self.current << 1) | bit; - self.bit_count += 1; - if self.bit_count == 8 { - self.push_current_byte(); - } - } - } - - fn finish(mut self) -> Vec { - if self.bit_count != 0 { - let pad_bits = 8 - self.bit_count; - self.current = (self.current << pad_bits) | ((1u8 << pad_bits) - 1); - self.push_current_byte(); - } - self.bytes - } - - fn push_current_byte(&mut self) { - self.bytes.push(self.current); - if self.current == 0xff { - self.bytes.push(0x00); - } - self.current = 0; - self.bit_count = 0; - } - } -} diff --git a/crates/signinum-jpeg-metal/src/lib.rs b/crates/signinum-jpeg-metal/src/lib.rs deleted file mode 100644 index 20115c1a..00000000 --- a/crates/signinum-jpeg-metal/src/lib.rs +++ /dev/null @@ -1,1896 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#![deny(unsafe_op_in_unsafe_fn)] -#![warn(unreachable_pub)] - -mod batch; -#[cfg(target_os = "macos")] -mod compute; -mod encode; -mod profile; -mod routing; -mod session; -pub mod viewport; - -use std::sync::Arc; -#[cfg(target_os = "macos")] -use std::sync::OnceLock; - -use signinum_core::{ - copy_tight_pixels_to_strided_output, BackendKind, BackendRequest, BufferError, CodecError, - DecodeOutcome, DeviceSubmission, DeviceSurface, Downscale, ImageCodec, ImageDecode, - ImageDecodeDevice, ImageDecodeSubmit, PixelFormat, Rect, TileBatchDecodeDevice, - TileBatchDecodeSubmit, -}; -use signinum_jpeg::{ - adapter::{ - build_metal_fast420_packet, build_metal_fast420_packet_for_decoder, - build_metal_fast422_packet, build_metal_fast422_packet_for_decoder, - build_metal_fast444_packet, build_metal_fast444_packet_for_decoder, decoder_bytes, - JpegMetalFast420PacketV1, JpegMetalFast422PacketV1, JpegMetalFast444PacketV1, - }, - Decoder as CpuDecoder, DecoderContext as CpuDecoderContext, JpegError, JpegView, - ScratchPool as CpuScratchPool, Warning as CpuWarning, -}; - -pub use encode::{ - encode_jpeg_baseline_batch_from_metal_buffers, encode_jpeg_baseline_from_metal_buffer, - JpegBaselineMetalEncodeTile, -}; - -#[cfg(target_os = "macos")] -use metal::{Buffer, CommandBuffer, Device, MTLResourceOptions}; - -#[derive(Debug, thiserror::Error)] -pub enum Error { - #[error(transparent)] - Decode(#[from] JpegError), - #[error(transparent)] - Encode(#[from] signinum_jpeg::JpegEncodeError), - #[error(transparent)] - Buffer(#[from] BufferError), - #[error("backend request {request:?} is not supported by signinum-jpeg-metal")] - UnsupportedBackend { request: BackendRequest }, - #[error("unsupported JPEG Metal request: {reason}")] - UnsupportedMetalRequest { reason: &'static str }, - #[error("Metal is unavailable on this host")] - MetalUnavailable, - #[error("Metal kernel error: {message}")] - MetalKernel { message: String }, -} - -impl CodecError for Error { - fn is_truncated(&self) -> bool { - matches!(self, Self::Decode(inner) if inner.is_truncated()) - } - - fn is_not_implemented(&self) -> bool { - matches!(self, Self::Decode(inner) if inner.is_not_implemented()) - } - - fn is_unsupported(&self) -> bool { - matches!( - self, - Self::UnsupportedBackend { .. } - | Self::MetalUnavailable - | Self::MetalKernel { .. } - | Self::UnsupportedMetalRequest { .. } - ) || matches!(self, Self::Decode(inner) if inner.is_unsupported()) - } - - fn is_buffer_error(&self) -> bool { - matches!(self, Self::Buffer(_)) - || matches!(self, Self::Decode(inner) if inner.is_buffer_error()) - } -} - -#[derive(Clone)] -pub(crate) enum Storage { - Host(Vec), - #[cfg(target_os = "macos")] - Metal { - buffer: Buffer, - offset: usize, - }, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SurfaceResidency { - Host, - MetalResidentDecode, - CpuStagedMetalUpload, -} - -#[derive(Clone)] -pub struct Surface { - backend: BackendKind, - residency: SurfaceResidency, - dimensions: (u32, u32), - fmt: PixelFormat, - pitch_bytes: usize, - storage: Storage, -} - -impl Surface { - pub fn pitch_bytes(&self) -> usize { - self.pitch_bytes - } - - pub fn residency(&self) -> SurfaceResidency { - self.residency - } - - pub fn as_bytes(&self) -> &[u8] { - match &self.storage { - Storage::Host(bytes) => bytes, - #[cfg(target_os = "macos")] - Storage::Metal { buffer, offset } => { - let len = self.byte_len(); - unsafe { - core::slice::from_raw_parts(buffer.contents().cast::().add(*offset), len) - } - } - } - } - - pub fn download_into(&self, out: &mut [u8], stride: usize) -> Result<(), Error> { - copy_tight_pixels_to_strided_output(self.as_bytes(), self.dimensions, self.fmt, out, stride) - .map_err(Error::from) - } - - #[cfg(target_os = "macos")] - pub fn metal_buffer(&self) -> Option<(&Buffer, usize)> { - match &self.storage { - Storage::Metal { buffer, offset } => Some((buffer, *offset)), - Storage::Host(_) => None, - } - } - - #[cfg(target_os = "macos")] - pub(crate) fn from_metal_buffer( - buffer: Buffer, - dimensions: (u32, u32), - fmt: PixelFormat, - ) -> Self { - Self::from_metal_buffer_offset(buffer, dimensions, fmt, 0) - } - - #[cfg(target_os = "macos")] - pub(crate) fn from_metal_buffer_offset( - buffer: Buffer, - dimensions: (u32, u32), - fmt: PixelFormat, - offset: usize, - ) -> Self { - Self { - backend: BackendKind::Metal, - residency: SurfaceResidency::MetalResidentDecode, - dimensions, - fmt, - pitch_bytes: dimensions.0 as usize * fmt.bytes_per_pixel(), - storage: Storage::Metal { buffer, offset }, - } - } - - #[cfg(target_os = "macos")] - pub(crate) fn from_cpu_staged_metal_buffer( - buffer: Buffer, - dimensions: (u32, u32), - fmt: PixelFormat, - ) -> Self { - Self::from_cpu_staged_metal_buffer_offset(buffer, dimensions, fmt, 0) - } - - #[cfg(target_os = "macos")] - pub(crate) fn from_cpu_staged_metal_buffer_offset( - buffer: Buffer, - dimensions: (u32, u32), - fmt: PixelFormat, - offset: usize, - ) -> Self { - Self { - backend: BackendKind::Metal, - residency: SurfaceResidency::CpuStagedMetalUpload, - dimensions, - fmt, - pitch_bytes: dimensions.0 as usize * fmt.bytes_per_pixel(), - storage: Storage::Metal { buffer, offset }, - } - } -} - -impl DeviceSurface for Surface { - fn backend_kind(&self) -> BackendKind { - self.backend - } - - fn dimensions(&self) -> (u32, u32) { - self.dimensions - } - - fn pixel_format(&self) -> PixelFormat { - self.fmt - } - - fn byte_len(&self) -> usize { - self.pitch_bytes * self.dimensions.1 as usize - } -} - -#[cfg(target_os = "macos")] -#[doc(hidden)] -#[derive(Clone)] -pub struct ResidentPrivateJpegTile { - pub buffer: Buffer, - pub byte_offset: usize, - pub dimensions: (u32, u32), - pub pixel_format: PixelFormat, - pub pitch_bytes: usize, - pub status_buffer: Buffer, - pub command_buffer: CommandBuffer, -} - -#[cfg(target_os = "macos")] -#[derive(Clone)] -pub struct MetalBackendSession { - device: Device, - runtime: Arc>>, -} - -#[cfg(target_os = "macos")] -impl MetalBackendSession { - pub fn new(device: Device) -> Self { - Self { - device, - runtime: Arc::new(OnceLock::new()), - } - } - - pub fn system_default() -> Result { - Device::system_default() - .map(Self::new) - .ok_or(Error::MetalUnavailable) - } - - pub fn device(&self) -> &metal::DeviceRef { - self.device.as_ref() - } -} - -#[cfg(target_os = "macos")] -impl core::fmt::Debug for MetalBackendSession { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("MetalBackendSession") - .field("device", &self.device.name()) - .field("runtime_initialized", &self.runtime.get().is_some()) - .finish() - } -} - -#[cfg(not(target_os = "macos"))] -#[derive(Clone, Copy, Debug, Default)] -pub struct MetalBackendSession { - _private: (), -} - -#[cfg(not(target_os = "macos"))] -impl MetalBackendSession { - pub fn system_default() -> Result { - Err(Error::MetalUnavailable) - } -} - -#[derive(Default)] -pub struct MetalSession { - shared: session::SharedSession, -} - -impl MetalSession { - pub fn submissions(&self) -> u64 { - self.shared.0.lock().expect("metal session").submissions - } -} - -impl core::fmt::Debug for MetalSession { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("MetalSession") - .field("submissions", &self.submissions()) - .finish() - } -} - -/// Convenience wrapper for submitting a group of JPEG tiles to one decoder -/// session. -/// -/// The batch preserves submission order and lets compatible requests share a -/// Metal submission. Callers still own slide metadata, level selection, cache -/// policy, and viewport planning. -#[derive(Default)] -pub struct JpegTileBatch { - session: MetalSession, - submissions: Vec, -} - -impl JpegTileBatch { - /// Create an empty tile batch. - pub fn new() -> Self { - Self::default() - } - - /// Create an empty tile batch with capacity for `capacity` submissions. - pub fn with_capacity(capacity: usize) -> Self { - Self { - submissions: Vec::with_capacity(capacity), - ..Self::default() - } - } - - /// Number of queued tile requests. - pub fn len(&self) -> usize { - self.submissions.len() - } - - /// Whether the batch has no queued tile requests. - pub fn is_empty(&self) -> bool { - self.submissions.is_empty() - } - - /// Number of Metal session submissions already flushed. - /// - /// Queued requests normally do not increment this until `decode_all` waits - /// on the first result. - pub fn submissions(&self) -> u64 { - self.session.submissions() - } - - /// Queue a full-tile decode request, copying the compressed tile bytes into - /// the batch. - pub fn push_tile( - &mut self, - input: &[u8], - fmt: PixelFormat, - backend: BackendRequest, - ) -> Result { - self.push_shared_tile(Arc::<[u8]>::from(input), fmt, backend) - } - - /// Queue a full-tile decode request backed by shared compressed tile bytes. - pub fn push_shared_tile( - &mut self, - input: Arc<[u8]>, - fmt: PixelFormat, - backend: BackendRequest, - ) -> Result { - Ok(self.push_shared_request(input, fmt, backend, batch::BatchOp::Full)) - } - - /// Queue a region decode request, copying the compressed tile bytes into - /// the batch. - pub fn push_tile_region( - &mut self, - input: &[u8], - fmt: PixelFormat, - roi: Rect, - backend: BackendRequest, - ) -> Result { - self.push_shared_tile_region(Arc::<[u8]>::from(input), fmt, roi, backend) - } - - /// Queue a region decode request backed by shared compressed tile bytes. - pub fn push_shared_tile_region( - &mut self, - input: Arc<[u8]>, - fmt: PixelFormat, - roi: Rect, - backend: BackendRequest, - ) -> Result { - Ok(self.push_shared_request(input, fmt, backend, batch::BatchOp::Region(roi))) - } - - /// Queue a scaled decode request, copying the compressed tile bytes into - /// the batch. - pub fn push_tile_scaled( - &mut self, - input: &[u8], - fmt: PixelFormat, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - self.push_shared_tile_scaled(Arc::<[u8]>::from(input), fmt, scale, backend) - } - - /// Queue a scaled decode request backed by shared compressed tile bytes. - pub fn push_shared_tile_scaled( - &mut self, - input: Arc<[u8]>, - fmt: PixelFormat, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - Ok(self.push_shared_request(input, fmt, backend, batch::BatchOp::Scaled(scale))) - } - - /// Queue a region decode at reduced resolution, copying the compressed tile - /// bytes into the batch. - pub fn push_tile_region_scaled( - &mut self, - input: &[u8], - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - self.push_shared_tile_region_scaled(Arc::<[u8]>::from(input), fmt, roi, scale, backend) - } - - /// Queue a region decode at reduced resolution backed by shared compressed - /// tile bytes. - pub fn push_shared_tile_region_scaled( - &mut self, - input: Arc<[u8]>, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - Ok(self.push_shared_request( - input, - fmt, - backend, - batch::BatchOp::RegionScaled { roi, scale }, - )) - } - - /// Decode all queued tile requests and return surfaces in submission order. - pub fn decode_all(self) -> Result, Error> { - let mut surfaces = Vec::with_capacity(self.submissions.len()); - for submission in self.submissions { - surfaces.push(submission.wait()?); - } - Ok(surfaces) - } - - fn push_shared_request( - &mut self, - input: Arc<[u8]>, - fmt: PixelFormat, - backend: BackendRequest, - op: batch::BatchOp, - ) -> usize { - let slot = self.submissions.len(); - let submission = { - let mut state = self.session.shared.0.lock().expect("metal session"); - let (fast444_packet, fast422_packet, fast420_packet) = - state.resolve_fast_packets(&input, backend); - let slot = state.queue_request(batch::QueuedRequest::new_shared( - input, - fmt, - backend, - op, - fast444_packet, - fast422_packet, - fast420_packet, - )); - batch::MetalSubmission { - session: self.session.shared.clone(), - slot, - } - }; - self.submissions.push(submission); - slot - } -} - -pub struct Decoder<'a> { - inner: CpuDecoder<'a>, - source: Arc<[u8]>, - fast444_packet: Option>, - fast422_packet: Option>, - fast420_packet: Option>, -} - -impl<'a> Decoder<'a> { - pub fn new(input: &'a [u8]) -> Result { - let inner = CpuDecoder::new(input)?; - Ok(Self { - fast444_packet: build_metal_fast444_packet(input).ok().map(Arc::new), - fast422_packet: build_metal_fast422_packet(input).ok().map(Arc::new), - fast420_packet: build_metal_fast420_packet(input).ok().map(Arc::new), - inner, - source: Arc::<[u8]>::from(input), - }) - } - - pub fn from_view(view: JpegView<'a>) -> Result { - let inner = CpuDecoder::from_view(view)?; - let source = Arc::<[u8]>::from(decoder_bytes(&inner)); - let fast444_packet = build_metal_fast444_packet_for_decoder(&inner) - .ok() - .map(Arc::new); - let fast422_packet = build_metal_fast422_packet_for_decoder(&inner) - .ok() - .map(Arc::new); - let fast420_packet = build_metal_fast420_packet_for_decoder(&inner) - .ok() - .map(Arc::new); - Ok(Self { - inner, - source, - fast444_packet, - fast422_packet, - fast420_packet, - }) - } - - pub fn inner(&self) -> &CpuDecoder<'a> { - &self.inner - } - - pub fn into_inner(self) -> CpuDecoder<'a> { - self.inner - } - - pub fn decode_region_scaled_to_device( - &mut self, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - let mut pool = CpuScratchPool::new(); - decode_region_scaled_surface_from_decoder( - &self.inner, - &mut pool, - fmt, - roi, - scale, - backend, - self.fast444_packet.as_deref(), - self.fast422_packet.as_deref(), - self.fast420_packet.as_deref(), - ) - } - - pub fn decode_to_device_with_session( - &mut self, - fmt: PixelFormat, - session: &MetalBackendSession, - ) -> Result { - #[cfg(target_os = "macos")] - { - let mut pool = CpuScratchPool::new(); - let decision = choose_route( - &self.inner, - BackendRequest::Metal, - fmt, - batch::BatchOp::Full, - self.fast444_packet.as_deref(), - self.fast422_packet.as_deref(), - self.fast420_packet.as_deref(), - ); - if let Some(err) = routing::decision_error(decision) { - return Err(err); - } - match decision { - routing::RouteDecision::MetalKernel => { - reject_cpu_staged_metal_upload(compute::decode_to_surface_with_session( - &self.inner, - &mut pool, - fmt, - self.fast444_packet.as_deref(), - self.fast422_packet.as_deref(), - self.fast420_packet.as_deref(), - session, - )?) - } - routing::RouteDecision::CpuHost - | routing::RouteDecision::RejectExplicitMetal { .. } - | routing::RouteDecision::RejectUnsupportedBackend { .. } - | routing::RouteDecision::MetalUnavailable => unreachable!("handled above"), - } - } - #[cfg(not(target_os = "macos"))] - { - let _ = session; - let decision = choose_route( - &self.inner, - BackendRequest::Metal, - fmt, - batch::BatchOp::Full, - self.fast444_packet.as_deref(), - self.fast422_packet.as_deref(), - self.fast420_packet.as_deref(), - ); - if let Some(err) = routing::decision_error(decision) { - return Err(err); - } - Err(Error::MetalUnavailable) - } - } - - #[cfg(target_os = "macos")] - #[doc(hidden)] - pub fn decode_private_rgb8_tile_with_session( - &mut self, - session: &MetalBackendSession, - ) -> Result { - let decision = choose_route( - &self.inner, - BackendRequest::Metal, - PixelFormat::Rgb8, - batch::BatchOp::Full, - self.fast444_packet.as_deref(), - self.fast422_packet.as_deref(), - self.fast420_packet.as_deref(), - ); - if let Some(err) = routing::decision_error(decision) { - return Err(err); - } - match decision { - routing::RouteDecision::MetalKernel => compute::decode_private_rgb8_tile_with_session( - &self.inner, - self.fast444_packet.as_deref(), - self.fast422_packet.as_deref(), - self.fast420_packet.as_deref(), - session, - ), - routing::RouteDecision::CpuHost - | routing::RouteDecision::RejectExplicitMetal { .. } - | routing::RouteDecision::RejectUnsupportedBackend { .. } - | routing::RouteDecision::MetalUnavailable => unreachable!("handled above"), - } - } -} - -impl ImageCodec for Decoder<'_> { - type Error = Error; - type Warning = CpuWarning; - type Pool = CpuScratchPool; -} - -impl<'a> ImageDecode<'a> for Decoder<'a> { - type View = JpegView<'a>; - - fn inspect(input: &'a [u8]) -> Result { - Ok(CpuDecoder::inspect(input)?.to_core_info()) - } - - fn parse(input: &'a [u8]) -> Result { - Ok(JpegView::parse(input)?) - } - - fn from_view(view: Self::View) -> Result { - Self::from_view(view) - } - - fn decode_into( - &mut self, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - ) -> Result, Self::Error> { - Ok(self.inner.decode_into(out, stride, fmt)?.into()) - } - - fn decode_into_with_scratch( - &mut self, - pool: &mut Self::Pool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - ) -> Result, Self::Error> { - Ok(self - .inner - .decode_into_with_scratch(pool, out, stride, fmt)? - .into()) - } - - fn decode_region_into( - &mut self, - pool: &mut Self::Pool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - ) -> Result, Self::Error> { - Ok(self - .inner - .decode_region_into_with_scratch(pool, out, stride, fmt, roi.into())? - .into()) - } - - fn decode_scaled_into( - &mut self, - pool: &mut Self::Pool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - scale: Downscale, - ) -> Result, Self::Error> { - Ok(self - .inner - .decode_scaled_into_with_scratch(pool, out, stride, fmt, scale)? - .into()) - } - - fn decode_region_scaled_into( - &mut self, - pool: &mut Self::Pool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - ) -> Result, Self::Error> { - Ok(self - .inner - .decode_region_scaled_into_with_scratch(pool, out, stride, fmt, roi.into(), scale)? - .into()) - } -} - -impl<'a> ImageDecodeDevice<'a> for Decoder<'a> { - type DeviceSurface = Surface; -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub struct Codec; - -impl ImageCodec for Codec { - type Error = Error; - type Warning = CpuWarning; - type Pool = CpuScratchPool; -} - -impl Codec { - #[allow(clippy::too_many_arguments)] - pub fn submit_tile_region_scaled_to_device( - ctx: &mut signinum_core::DecoderContext, - session: &mut MetalSession, - pool: &mut CpuScratchPool, - input: &[u8], - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - backend: BackendRequest, - ) -> Result<::SubmittedSurface, Error> { - let _ = (ctx, pool); - let slot = { - let mut state = session.shared.0.lock().expect("metal session"); - let input = state.intern_input_slice(input); - let (fast444_packet, fast422_packet, fast420_packet) = - state.resolve_fast_packets(&input, backend); - state.queue_request(batch::QueuedRequest::new_shared( - input, - fmt, - backend, - batch::BatchOp::RegionScaled { roi, scale }, - fast444_packet, - fast422_packet, - fast420_packet, - )) - }; - Ok(batch::MetalSubmission { - session: session.shared.clone(), - slot, - }) - } -} - -impl<'a> ImageDecodeSubmit<'a> for Decoder<'a> { - type Session = MetalSession; - type DeviceSurface = Surface; - type SubmittedSurface = batch::MetalSubmission; - - fn submit_to_device( - &mut self, - session: &mut Self::Session, - fmt: PixelFormat, - backend: BackendRequest, - ) -> Result { - let fast444_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { - self.fast444_packet.clone() - } else { - None - }; - let fast422_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { - self.fast422_packet.clone() - } else { - None - }; - let fast420_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { - self.fast420_packet.clone() - } else { - None - }; - let slot = session - .shared - .0 - .lock() - .expect("metal session") - .queue_request(batch::QueuedRequest::new_shared( - Arc::clone(&self.source), - fmt, - backend, - batch::BatchOp::Full, - fast444_packet, - fast422_packet, - fast420_packet, - )); - Ok(batch::MetalSubmission { - session: session.shared.clone(), - slot, - }) - } - - fn submit_region_to_device( - &mut self, - session: &mut Self::Session, - fmt: PixelFormat, - roi: Rect, - backend: BackendRequest, - ) -> Result { - let fast444_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { - self.fast444_packet.clone() - } else { - None - }; - let fast422_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { - self.fast422_packet.clone() - } else { - None - }; - let fast420_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { - self.fast420_packet.clone() - } else { - None - }; - let slot = session - .shared - .0 - .lock() - .expect("metal session") - .queue_request(batch::QueuedRequest::new_shared( - Arc::clone(&self.source), - fmt, - backend, - batch::BatchOp::Region(roi), - fast444_packet, - fast422_packet, - fast420_packet, - )); - Ok(batch::MetalSubmission { - session: session.shared.clone(), - slot, - }) - } - - fn submit_scaled_to_device( - &mut self, - session: &mut Self::Session, - fmt: PixelFormat, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - let fast444_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { - self.fast444_packet.clone() - } else { - None - }; - let fast422_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { - self.fast422_packet.clone() - } else { - None - }; - let fast420_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { - self.fast420_packet.clone() - } else { - None - }; - let slot = session - .shared - .0 - .lock() - .expect("metal session") - .queue_request(batch::QueuedRequest::new_shared( - Arc::clone(&self.source), - fmt, - backend, - batch::BatchOp::Scaled(scale), - fast444_packet, - fast422_packet, - fast420_packet, - )); - Ok(batch::MetalSubmission { - session: session.shared.clone(), - slot, - }) - } - - fn submit_region_scaled_to_device( - &mut self, - session: &mut Self::Session, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - let fast444_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { - self.fast444_packet.clone() - } else { - None - }; - let fast422_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { - self.fast422_packet.clone() - } else { - None - }; - let fast420_packet = if matches!(backend, BackendRequest::Auto | BackendRequest::Metal) { - self.fast420_packet.clone() - } else { - None - }; - let slot = session - .shared - .0 - .lock() - .expect("metal session") - .queue_request(batch::QueuedRequest::new_shared( - Arc::clone(&self.source), - fmt, - backend, - batch::BatchOp::RegionScaled { roi, scale }, - fast444_packet, - fast422_packet, - fast420_packet, - )); - Ok(batch::MetalSubmission { - session: session.shared.clone(), - slot, - }) - } -} - -impl TileBatchDecodeSubmit for Codec { - type Context = CpuDecoderContext; - type Session = MetalSession; - type DeviceSurface = Surface; - type SubmittedSurface = batch::MetalSubmission; - - fn submit_tile_to_device( - ctx: &mut signinum_core::DecoderContext, - session: &mut Self::Session, - pool: &mut Self::Pool, - input: &[u8], - fmt: PixelFormat, - backend: BackendRequest, - ) -> Result { - let _ = (ctx, pool); - let slot = { - let mut state = session.shared.0.lock().expect("metal session"); - let input = state.intern_input_slice(input); - let (fast444_packet, fast422_packet, fast420_packet) = - state.resolve_fast_packets(&input, backend); - state.queue_request(batch::QueuedRequest::new_shared( - input, - fmt, - backend, - batch::BatchOp::Full, - fast444_packet, - fast422_packet, - fast420_packet, - )) - }; - Ok(batch::MetalSubmission { - session: session.shared.clone(), - slot, - }) - } - - fn submit_tile_region_to_device( - ctx: &mut signinum_core::DecoderContext, - session: &mut Self::Session, - pool: &mut Self::Pool, - input: &[u8], - fmt: PixelFormat, - roi: Rect, - backend: BackendRequest, - ) -> Result { - let _ = (ctx, pool); - let slot = { - let mut state = session.shared.0.lock().expect("metal session"); - let input = state.intern_input_slice(input); - let (fast444_packet, fast422_packet, fast420_packet) = - state.resolve_fast_packets(&input, backend); - state.queue_request(batch::QueuedRequest::new_shared( - input, - fmt, - backend, - batch::BatchOp::Region(roi), - fast444_packet, - fast422_packet, - fast420_packet, - )) - }; - Ok(batch::MetalSubmission { - session: session.shared.clone(), - slot, - }) - } - - fn submit_tile_scaled_to_device( - ctx: &mut signinum_core::DecoderContext, - session: &mut Self::Session, - pool: &mut Self::Pool, - input: &[u8], - fmt: PixelFormat, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - let _ = (ctx, pool); - let slot = { - let mut state = session.shared.0.lock().expect("metal session"); - let input = state.intern_input_slice(input); - let (fast444_packet, fast422_packet, fast420_packet) = - state.resolve_fast_packets(&input, backend); - state.queue_request(batch::QueuedRequest::new_shared( - input, - fmt, - backend, - batch::BatchOp::Scaled(scale), - fast444_packet, - fast422_packet, - fast420_packet, - )) - }; - Ok(batch::MetalSubmission { - session: session.shared.clone(), - slot, - }) - } - - fn submit_tile_region_scaled_to_device( - ctx: &mut signinum_core::DecoderContext, - session: &mut Self::Session, - pool: &mut Self::Pool, - input: &[u8], - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - backend: BackendRequest, - ) -> Result { - Codec::submit_tile_region_scaled_to_device( - ctx, session, pool, input, fmt, roi, scale, backend, - ) - } -} - -impl TileBatchDecodeDevice for Codec { - type Context = CpuDecoderContext; - type DeviceSurface = Surface; -} - -pub(crate) fn decode_surface_from_bytes( - input: &[u8], - fmt: PixelFormat, - backend: BackendRequest, - op: batch::BatchOp, - fast444_packet: Option>, - fast422_packet: Option>, - fast420_packet: Option>, -) -> Result { - let decoder = CpuDecoder::new(input)?; - let mut pool = CpuScratchPool::new(); - let build_auto_packets = - matches!(backend, BackendRequest::Auto) && decoder.info().restart_interval.is_some(); - let build_metal_packets = matches!(backend, BackendRequest::Metal); - let fast444_packet = if build_auto_packets || build_metal_packets { - fast444_packet.or_else(|| { - build_metal_fast444_packet_for_decoder(&decoder) - .ok() - .map(Arc::new) - }) - } else { - None - }; - let fast422_packet = if build_auto_packets || build_metal_packets { - fast422_packet.or_else(|| { - build_metal_fast422_packet_for_decoder(&decoder) - .ok() - .map(Arc::new) - }) - } else { - None - }; - let fast420_packet = if build_auto_packets || build_metal_packets { - fast420_packet.or_else(|| { - build_metal_fast420_packet_for_decoder(&decoder) - .ok() - .map(Arc::new) - }) - } else { - None - }; - decode_surface_from_decoder( - &decoder, - &mut pool, - fmt, - backend, - op, - fast444_packet.as_deref(), - fast422_packet.as_deref(), - fast420_packet.as_deref(), - ) -} - -#[allow(clippy::unnecessary_wraps)] -pub(crate) fn decode_compatible_batch( - requests: &[batch::QueuedRequest], -) -> Result>>, Error> { - #[cfg(target_os = "macos")] - { - compute::decode_full_batch_to_surfaces(requests) - } - #[cfg(not(target_os = "macos"))] - { - let _ = requests; - Ok(None) - } -} - -#[cfg(target_os = "macos")] -#[doc(hidden)] -pub fn decode_rgb8_batch_to_device_with_session( - inputs: &[&[u8]], - session: &MetalBackendSession, -) -> Result>>, Error> { - if inputs.len() < 2 { - return Ok(None); - } - - let mut state = session::SessionState::default(); - let mut requests = Vec::with_capacity(inputs.len()); - for input in inputs { - let input = state.intern_input_slice(input); - let (fast444_packet, fast422_packet, fast420_packet) = - state.resolve_fast_packets(&input, BackendRequest::Metal); - requests.push(batch::QueuedRequest::new_shared( - input, - PixelFormat::Rgb8, - BackendRequest::Metal, - batch::BatchOp::Full, - fast444_packet, - fast422_packet, - fast420_packet, - )); - } - - compute::decode_full_batch_to_surfaces_with_session(&requests, session) -} - -#[allow(clippy::too_many_arguments)] -fn decode_surface_from_decoder( - decoder: &CpuDecoder<'_>, - pool: &mut CpuScratchPool, - fmt: PixelFormat, - backend: BackendRequest, - op: batch::BatchOp, - fast444_packet: Option<&JpegMetalFast444PacketV1>, - fast422_packet: Option<&JpegMetalFast422PacketV1>, - fast420_packet: Option<&JpegMetalFast420PacketV1>, -) -> Result { - match op { - batch::BatchOp::Full => match backend { - BackendRequest::Cpu => decode_full_cpu_upload(decoder, pool, fmt), - BackendRequest::Auto | BackendRequest::Metal => { - let decision = choose_route( - decoder, - backend, - fmt, - op, - fast444_packet, - fast422_packet, - fast420_packet, - ); - if let Some(err) = routing::decision_error(decision) { - return Err(err); - } - match decision { - routing::RouteDecision::CpuHost => decode_full_cpu_upload(decoder, pool, fmt), - routing::RouteDecision::MetalKernel => { - #[cfg(target_os = "macos")] - { - reject_cpu_staged_metal_upload(compute::decode_to_surface( - decoder, - pool, - fmt, - fast444_packet, - fast422_packet, - fast420_packet, - )?) - } - #[cfg(not(target_os = "macos"))] - { - let _ = ( - decoder, - pool, - fmt, - fast444_packet, - fast422_packet, - fast420_packet, - ); - Err(Error::MetalUnavailable) - } - } - routing::RouteDecision::RejectExplicitMetal { .. } - | routing::RouteDecision::RejectUnsupportedBackend { .. } - | routing::RouteDecision::MetalUnavailable => unreachable!("handled above"), - } - } - BackendRequest::Cuda => Err(Error::UnsupportedBackend { request: backend }), - }, - batch::BatchOp::Region(roi) => match backend { - BackendRequest::Cpu => decode_region_cpu_upload(decoder, pool, fmt, roi), - BackendRequest::Auto | BackendRequest::Metal => { - let decision = choose_route( - decoder, - backend, - fmt, - op, - fast444_packet, - fast422_packet, - fast420_packet, - ); - if let Some(err) = routing::decision_error(decision) { - return Err(err); - } - match decision { - routing::RouteDecision::CpuHost => { - decode_region_cpu_upload(decoder, pool, fmt, roi) - } - routing::RouteDecision::MetalKernel => { - #[cfg(target_os = "macos")] - { - reject_cpu_staged_metal_upload(compute::decode_region_to_surface( - decoder, - pool, - fmt, - roi.into(), - fast444_packet, - fast422_packet, - fast420_packet, - )?) - } - #[cfg(not(target_os = "macos"))] - { - let _ = ( - decoder, - pool, - fmt, - roi, - fast444_packet, - fast422_packet, - fast420_packet, - ); - Err(Error::MetalUnavailable) - } - } - routing::RouteDecision::RejectExplicitMetal { .. } - | routing::RouteDecision::RejectUnsupportedBackend { .. } - | routing::RouteDecision::MetalUnavailable => unreachable!("handled above"), - } - } - BackendRequest::Cuda => Err(Error::UnsupportedBackend { request: backend }), - }, - batch::BatchOp::Scaled(scale) => match backend { - BackendRequest::Cpu => decode_scaled_cpu_upload(decoder, pool, fmt, scale), - BackendRequest::Auto | BackendRequest::Metal => { - let decision = choose_route( - decoder, - backend, - fmt, - op, - fast444_packet, - fast422_packet, - fast420_packet, - ); - if let Some(err) = routing::decision_error(decision) { - return Err(err); - } - match decision { - routing::RouteDecision::CpuHost => { - decode_scaled_cpu_upload(decoder, pool, fmt, scale) - } - routing::RouteDecision::MetalKernel => { - #[cfg(target_os = "macos")] - { - reject_cpu_staged_metal_upload(compute::decode_scaled_to_surface( - decoder, - pool, - fmt, - scale, - fast444_packet, - fast422_packet, - fast420_packet, - )?) - } - #[cfg(not(target_os = "macos"))] - { - let _ = ( - decoder, - pool, - fmt, - scale, - fast444_packet, - fast422_packet, - fast420_packet, - ); - Err(Error::MetalUnavailable) - } - } - routing::RouteDecision::RejectExplicitMetal { .. } - | routing::RouteDecision::RejectUnsupportedBackend { .. } - | routing::RouteDecision::MetalUnavailable => unreachable!("handled above"), - } - } - BackendRequest::Cuda => Err(Error::UnsupportedBackend { request: backend }), - }, - batch::BatchOp::RegionScaled { roi, scale } => decode_region_scaled_surface_from_decoder( - decoder, - pool, - fmt, - roi, - scale, - backend, - fast444_packet, - fast422_packet, - fast420_packet, - ), - } -} - -fn decode_full_cpu_upload( - decoder: &CpuDecoder<'_>, - pool: &mut CpuScratchPool, - fmt: PixelFormat, -) -> Result { - let dims = decoder.info().dimensions; - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut out = vec![0u8; stride * dims.1 as usize]; - decoder.decode_into_with_scratch(pool, &mut out, stride, fmt)?; - upload_surface(out, dims, fmt, BackendRequest::Cpu) -} - -fn decode_region_cpu_upload( - decoder: &CpuDecoder<'_>, - pool: &mut CpuScratchPool, - fmt: PixelFormat, - roi: Rect, -) -> Result { - let dims = (roi.w, roi.h); - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut out = vec![0u8; stride * dims.1 as usize]; - decoder.decode_region_into_with_scratch(pool, &mut out, stride, fmt, roi.into())?; - upload_surface(out, dims, fmt, BackendRequest::Cpu) -} - -fn decode_scaled_cpu_upload( - decoder: &CpuDecoder<'_>, - pool: &mut CpuScratchPool, - fmt: PixelFormat, - scale: Downscale, -) -> Result { - let dims = scaled_dims(decoder.info().dimensions, scale); - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut out = vec![0u8; stride * dims.1 as usize]; - decoder.decode_scaled_into_with_scratch(pool, &mut out, stride, fmt, scale)?; - upload_surface(out, dims, fmt, BackendRequest::Cpu) -} - -#[allow(clippy::too_many_arguments)] -fn decode_region_scaled_surface_from_decoder( - decoder: &CpuDecoder<'_>, - pool: &mut CpuScratchPool, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - backend: BackendRequest, - fast444_packet: Option<&JpegMetalFast444PacketV1>, - fast422_packet: Option<&JpegMetalFast422PacketV1>, - fast420_packet: Option<&JpegMetalFast420PacketV1>, -) -> Result { - match backend { - BackendRequest::Cpu => { - decode_region_scaled_cpu_upload(decoder, pool, fmt, roi, scale, BackendRequest::Cpu) - } - BackendRequest::Auto | BackendRequest::Metal => { - let decision = choose_route( - decoder, - backend, - fmt, - batch::BatchOp::RegionScaled { roi, scale }, - fast444_packet, - fast422_packet, - fast420_packet, - ); - if let Some(err) = routing::decision_error(decision) { - return Err(err); - } - match decision { - routing::RouteDecision::CpuHost => decode_region_scaled_cpu_upload( - decoder, - pool, - fmt, - roi, - scale, - BackendRequest::Cpu, - ), - routing::RouteDecision::MetalKernel => { - #[cfg(target_os = "macos")] - { - reject_cpu_staged_metal_upload(compute::decode_region_scaled_to_surface( - decoder, - pool, - fmt, - roi.into(), - scale, - fast444_packet, - fast422_packet, - fast420_packet, - )?) - } - #[cfg(not(target_os = "macos"))] - { - let _ = ( - decoder, - pool, - fmt, - roi, - scale, - fast444_packet, - fast422_packet, - fast420_packet, - ); - Err(Error::MetalUnavailable) - } - } - routing::RouteDecision::RejectExplicitMetal { .. } - | routing::RouteDecision::RejectUnsupportedBackend { .. } - | routing::RouteDecision::MetalUnavailable => unreachable!("handled above"), - } - } - BackendRequest::Cuda => Err(Error::UnsupportedBackend { request: backend }), - } -} - -#[cfg(target_os = "macos")] -fn reject_cpu_staged_metal_upload(surface: Surface) -> Result { - if surface.residency() == SurfaceResidency::CpuStagedMetalUpload { - return Err(Error::UnsupportedMetalRequest { - reason: "JPEG Metal explicit device decode requires a direct resident Metal decode; use the CPU path for CPU-staged output", - }); - } - Ok(surface) -} - -#[allow(clippy::too_many_arguments)] -fn choose_route( - decoder: &CpuDecoder<'_>, - backend: BackendRequest, - fmt: PixelFormat, - op: batch::BatchOp, - fast444_packet: Option<&JpegMetalFast444PacketV1>, - fast422_packet: Option<&JpegMetalFast422PacketV1>, - fast420_packet: Option<&JpegMetalFast420PacketV1>, -) -> routing::RouteDecision { - let capabilities = routing::JpegMetalCapabilities::for_request( - decoder, - fmt, - op, - fast444_packet, - fast422_packet, - fast420_packet, - ); - let decision = routing::decide_route(backend, capabilities); - if profile::gpu_route_profile_enabled() { - let request_s = format!("{backend:?}"); - let fmt_s = format!("{fmt:?}"); - let has_fast_packet_s = capabilities.has_fast_packet().to_string(); - let supports_format_s = capabilities.supports_output_format().to_string(); - let (decision_s, reason_s) = jpeg_route_decision_profile(decision); - profile::emit_gpu_route_profile( - "jpeg", - "gpu_route", - "metal", - &[ - ("request", request_s.as_str()), - ("fmt", fmt_s.as_str()), - ("op", jpeg_batch_op_profile(op)), - ("has_fast_packet", has_fast_packet_s.as_str()), - ("supports_output_format", supports_format_s.as_str()), - ("decision", decision_s), - ("reason", reason_s), - ], - ); - } - decision -} - -fn jpeg_batch_op_profile(op: batch::BatchOp) -> &'static str { - match op { - batch::BatchOp::Full => "full", - batch::BatchOp::Region(_) => "region", - batch::BatchOp::Scaled(_) => "scaled", - batch::BatchOp::RegionScaled { .. } => "region_scaled", - } -} - -fn jpeg_route_decision_profile(decision: routing::RouteDecision) -> (&'static str, &'static str) { - match decision { - routing::RouteDecision::CpuHost => ("cpu_host", "none"), - routing::RouteDecision::MetalKernel => ("metal_kernel", "none"), - routing::RouteDecision::RejectExplicitMetal { reason } => { - let reason_code = if reason.contains("fast") { - "no_fast_packet" - } else { - "unsupported_format" - }; - ("reject_explicit_metal", reason_code) - } - routing::RouteDecision::RejectUnsupportedBackend { .. } => { - ("reject_unsupported_backend", "unsupported_backend") - } - routing::RouteDecision::MetalUnavailable => ("metal_unavailable", "metal_unavailable"), - } -} - -fn decode_region_scaled_cpu_upload( - decoder: &CpuDecoder<'_>, - pool: &mut CpuScratchPool, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - backend: BackendRequest, -) -> Result { - let scaled = roi.scaled_covering(scale); - let dims = (scaled.w, scaled.h); - let stride = dims.0 as usize * fmt.bytes_per_pixel(); - let mut out = vec![0u8; stride * dims.1 as usize]; - decoder.decode_region_scaled_into_with_scratch( - pool, - &mut out, - stride, - fmt, - roi.into(), - scale, - )?; - upload_surface(out, dims, fmt, backend) -} - -fn scaled_dims(full: (u32, u32), scale: Downscale) -> (u32, u32) { - ( - full.0.div_ceil(scale.denominator()), - full.1.div_ceil(scale.denominator()), - ) -} - -pub(crate) fn upload_surface( - bytes: Vec, - dimensions: (u32, u32), - fmt: PixelFormat, - backend: BackendRequest, -) -> Result { - let pitch_bytes = dimensions.0 as usize * fmt.bytes_per_pixel(); - match backend { - BackendRequest::Cpu => Ok(Surface { - backend: BackendKind::Cpu, - residency: SurfaceResidency::Host, - dimensions, - fmt, - pitch_bytes, - storage: Storage::Host(bytes), - }), - BackendRequest::Auto | BackendRequest::Metal => { - #[cfg(target_os = "macos")] - { - let device = Device::system_default().ok_or(Error::MetalUnavailable)?; - let buffer = device.new_buffer_with_data( - bytes.as_ptr().cast(), - bytes.len() as u64, - MTLResourceOptions::StorageModeShared, - ); - Ok(Surface { - backend: BackendKind::Metal, - residency: SurfaceResidency::CpuStagedMetalUpload, - dimensions, - fmt, - pitch_bytes, - storage: Storage::Metal { buffer, offset: 0 }, - }) - } - #[cfg(not(target_os = "macos"))] - { - if matches!(backend, BackendRequest::Auto) { - Ok(Surface { - backend: BackendKind::Cpu, - residency: SurfaceResidency::Host, - dimensions, - fmt, - pitch_bytes, - storage: Storage::Host(bytes), - }) - } else { - Err(Error::MetalUnavailable) - } - } - } - BackendRequest::Cuda => Err(Error::UnsupportedBackend { request: backend }), - } -} - -pub use signinum_jpeg::{ - DecoderContext, Downscale as JpegDownscale, PixelFormat as JpegPixelFormat, ScratchPool, -}; -pub use signinum_jpeg::{Info, Rect as JpegRectPublic}; - -#[cfg(test)] -mod tests { - use super::*; - use signinum_jpeg::adapter::{build_metal_fast420_packet, build_metal_fast444_packet}; - - const BASELINE_420: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); - const BASELINE_420_RESTART: &[u8] = - include_bytes!("../fixtures/jpeg/baseline_420_restart_32x16.jpg"); - const BASELINE_444: &[u8] = include_bytes!("../fixtures/jpeg/baseline_444_8x8.jpg"); - #[cfg(not(target_os = "macos"))] - const GRAYSCALE: &[u8] = include_bytes!("../fixtures/jpeg/grayscale_8x8.jpg"); - - #[test] - fn auto_route_prefers_cpu_host_for_nonrestart_packets() { - let decoder_420 = CpuDecoder::new(BASELINE_420).expect("420 decoder"); - let packet_420 = build_metal_fast420_packet(BASELINE_420).expect("420 packet"); - assert_eq!( - choose_route( - &decoder_420, - BackendRequest::Auto, - PixelFormat::Rgb8, - batch::BatchOp::Full, - None, - None, - Some(&packet_420), - ), - routing::RouteDecision::CpuHost - ); - - let decoder_444 = CpuDecoder::new(BASELINE_444).expect("444 decoder"); - let packet_444 = build_metal_fast444_packet(BASELINE_444).expect("444 packet"); - assert_eq!( - choose_route( - &decoder_444, - BackendRequest::Auto, - PixelFormat::Rgb8, - batch::BatchOp::Scaled(Downscale::Quarter), - Some(&packet_444), - None, - None, - ), - routing::RouteDecision::CpuHost - ); - } - - #[test] - fn auto_route_keeps_small_single_restart_packets_on_cpu_host() { - let decoder = CpuDecoder::new(BASELINE_420_RESTART).expect("restart decoder"); - let packet = build_metal_fast420_packet(BASELINE_420_RESTART).expect("restart packet"); - - assert_eq!( - choose_route( - &decoder, - BackendRequest::Auto, - PixelFormat::Rgb8, - batch::BatchOp::Full, - None, - None, - Some(&packet) - ), - routing::RouteDecision::CpuHost - ); - assert_eq!( - choose_route( - &decoder, - BackendRequest::Auto, - PixelFormat::Rgb8, - batch::BatchOp::Region(Rect { - x: 0, - y: 0, - w: 16, - h: 16, - }), - None, - None, - Some(&packet), - ), - routing::RouteDecision::CpuHost - ); - } - - #[cfg(target_os = "macos")] - #[test] - fn metal_backend_session_reuses_compiled_runtime() { - let session = MetalBackendSession::system_default().expect("Metal backend session"); - assert!(session.runtime.get().is_none()); - - let mut first = Decoder::new(BASELINE_420).expect("first decoder"); - let first_surface = first - .decode_to_device_with_session(PixelFormat::Rgb8, &session) - .expect("first session decode"); - assert_eq!( - first_surface.residency(), - SurfaceResidency::MetalResidentDecode - ); - let first_runtime = session - .runtime - .get() - .and_then(|runtime| runtime.as_ref().ok()) - .map(std::ptr::from_ref::) - .expect("session runtime after first decode"); - - let mut second = Decoder::new(BASELINE_420).expect("second decoder"); - second - .decode_to_device_with_session(PixelFormat::Rgb8, &session) - .expect("second session decode"); - let second_runtime = session - .runtime - .get() - .and_then(|runtime| runtime.as_ref().ok()) - .map(std::ptr::from_ref::) - .expect("session runtime after second decode"); - - assert_eq!(first_runtime, second_runtime); - } - - #[cfg(target_os = "macos")] - #[test] - fn jpeg_rgb8_batch_decode_uses_backend_session_runtime() { - let session = MetalBackendSession::system_default().expect("Metal backend session"); - assert!(session.runtime.get().is_none()); - - let inputs = [BASELINE_420, BASELINE_420]; - let results = decode_rgb8_batch_to_device_with_session(&inputs, &session) - .expect("session batch decode") - .expect("baseline JPEG batch should use Metal batch path"); - - assert_eq!(results.len(), 2); - assert!(session.runtime.get().is_some()); - for result in results { - let surface = result.expect("surface"); - assert_eq!(surface.backend_kind(), BackendKind::Metal); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - assert_eq!(surface.dimensions(), (16, 16)); - assert_eq!(surface.pixel_format(), PixelFormat::Rgb8); - } - } - - #[cfg(target_os = "macos")] - #[test] - fn jpeg_device_decode_uses_private_internal_planes() { - let session = MetalBackendSession::system_default().expect("Metal backend session"); - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - - compute::reset_jpeg_private_buffer_allocations_for_test(); - let surface = decoder - .decode_to_device_with_session(PixelFormat::Rgb8, &session) - .expect("resident JPEG Metal decode"); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - assert!( - compute::jpeg_private_buffer_allocations_for_test() > 0, - "resident JPEG Metal decode should use Private internal planes" - ); - let _ = surface.as_bytes(); - } - - #[cfg(target_os = "macos")] - #[test] - fn jpeg_private_rgb8_tile_uses_private_output_buffer() { - let session = MetalBackendSession::system_default().expect("Metal backend session"); - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - - let tile = decoder - .decode_private_rgb8_tile_with_session(&session) - .expect("resident private JPEG Metal decode"); - - assert_eq!(tile.dimensions, (16, 16)); - assert_eq!(tile.pixel_format, PixelFormat::Rgb8); - assert_eq!(tile.pitch_bytes, 16 * PixelFormat::Rgb8.bytes_per_pixel()); - assert_eq!(tile.byte_offset, 0); - assert_eq!(tile.buffer.storage_mode(), metal::MTLStorageMode::Private); - assert!(tile.status_buffer.length() > 0); - } - - #[cfg(target_os = "macos")] - #[test] - fn jpeg_gray_region_decode_uses_private_internal_planes() { - let roi = Rect { - x: 4, - y: 4, - w: 8, - h: 8, - }; - let mut expected_decoder = Decoder::new(BASELINE_420).expect("expected decoder"); - let mut expected = vec![0; roi.w as usize * roi.h as usize]; - expected_decoder - .decode_region_into( - &mut CpuScratchPool::new(), - &mut expected, - roi.w as usize, - PixelFormat::Gray8, - roi, - ) - .expect("expected CPU region decode"); - - let mut decoder = Decoder::new(BASELINE_420).expect("decoder"); - compute::reset_jpeg_private_buffer_allocations_for_test(); - let surface = decoder - .decode_region_to_device(PixelFormat::Gray8, roi, BackendRequest::Metal) - .expect("resident JPEG Metal region decode"); - assert_eq!(surface.residency(), SurfaceResidency::MetalResidentDecode); - assert!( - compute::jpeg_private_buffer_allocations_for_test() >= 3, - "resident Gray8 region decode should keep decoded Y/Cb/Cr planes Private" - ); - assert_eq!(surface.as_bytes(), expected.as_slice()); - } - - #[cfg(target_os = "macos")] - #[test] - fn uploaded_metal_surface_is_marked_cpu_staged() { - let surface = upload_surface( - vec![1, 2, 3], - (1, 1), - PixelFormat::Rgb8, - BackendRequest::Metal, - ) - .expect("CPU staged Metal upload"); - - assert_eq!(surface.residency(), SurfaceResidency::CpuStagedMetalUpload); - } - - #[test] - fn auto_route_prefers_cpu_host_for_region_scaled_even_with_restart_packets() { - let decoder = CpuDecoder::new(BASELINE_420_RESTART).expect("restart decoder"); - let packet = build_metal_fast420_packet(BASELINE_420_RESTART).expect("restart packet"); - - assert_eq!( - choose_route( - &decoder, - BackendRequest::Auto, - PixelFormat::Rgb8, - batch::BatchOp::RegionScaled { - roi: Rect { - x: 0, - y: 0, - w: 16, - h: 16, - }, - scale: Downscale::Quarter, - }, - None, - None, - Some(&packet), - ), - routing::RouteDecision::CpuHost - ); - } - - #[cfg(not(target_os = "macos"))] - #[test] - fn session_decode_rejects_unsupported_shape_before_host_unavailability() { - let mut decoder = Decoder::new(GRAYSCALE).expect("decoder"); - let session = MetalBackendSession::default(); - - assert!(matches!( - decoder.decode_to_device_with_session(PixelFormat::Gray8, &session), - Err(Error::UnsupportedMetalRequest { .. }) - )); - } -} diff --git a/crates/signinum-jpeg-metal/src/profile.rs b/crates/signinum-jpeg-metal/src/profile.rs deleted file mode 100644 index f4000840..00000000 --- a/crates/signinum-jpeg-metal/src/profile.rs +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -pub(crate) fn gpu_route_profile_enabled() -> bool { - signinum_profile::gpu_route_profile_enabled() -} - -pub(crate) fn emit_gpu_route_profile(codec: &str, op: &str, path: &str, fields: &[(K, V)]) -where - K: AsRef, - V: AsRef, -{ - debug_assert_eq!(op, "gpu_route"); - signinum_profile::emit_gpu_route_profile(codec, path, fields); -} diff --git a/crates/signinum-jpeg-metal/tests/shader_integrity.rs b/crates/signinum-jpeg-metal/tests/shader_integrity.rs deleted file mode 100644 index 70fb1077..00000000 --- a/crates/signinum-jpeg-metal/tests/shader_integrity.rs +++ /dev/null @@ -1,113 +0,0 @@ -const SHADER_SOURCE: &str = include_str!("../src/shaders.metal"); -const COMPUTE_SOURCE: &str = include_str!("../src/compute.rs"); - -#[test] -fn decode_loops_advance_mcu_coordinates_incrementally() { - let marker = "inline void init_mcu_cursor("; - let fast_decode_source = &SHADER_SOURCE[SHADER_SOURCE - .find(marker) - .expect("shader source should define init_mcu_cursor")..]; - assert!( - !fast_decode_source.contains("mcu_index / params.mcus_per_row"), - "decode hot loops must not divide every MCU to recover my" - ); - assert!( - !fast_decode_source.contains("mcu_index % params.mcus_per_row"), - "decode hot loops must not modulo every MCU to recover mx" - ); -} - -#[test] -fn batch_rgb_pack_kernels_process_pixel_groups() { - let compact_compute = COMPUTE_SOURCE - .chars() - .filter(|ch| !ch.is_whitespace()) - .collect::(); - - assert!( - SHADER_SOURCE.contains("const uint x0 = gid.x * 2u;"), - "batch RGB pack kernels should group horizontal pixels per thread" - ); - assert!( - SHADER_SOURCE.contains("const uint y0 = gid.y * 2u;"), - "420 batch RGB pack should group 2x2 output pixels per thread" - ); - assert!( - compact_compute - .contains("(packed_pair_extent(width),packed_pair_extent(height),tile_count_u32,)"), - "420 batch RGB pack dispatch should use grouped 2x2 grid dimensions" - ); - assert!( - compact_compute.contains("(packed_pair_extent(width),height,tile_count_u32)"), - "422 batch RGB pack dispatch should use grouped horizontal grid dimensions" - ); -} - -#[test] -fn fast420_batch_split_path_stays_wired() { - assert!( - SHADER_SOURCE.contains("kernel void jpeg_decode_fast420_batch_coeffs"), - "split fast420 batch must keep the entropy-to-coefficients kernel" - ); - assert!( - SHADER_SOURCE.contains("kernel void jpeg_idct_deposit_fast420_batch"), - "split fast420 batch must keep the IDCT/deposit kernel" - ); - assert!( - COMPUTE_SOURCE.contains("SIGNINUM_JPEG_METAL_SPLIT_FAST420_BATCH"), - "split fast420 batch must stay opt-in until benchmarks promote it" - ); -} - -#[test] -fn entropy_fast_paths_stay_wired() { - assert!( - SHADER_SOURCE.contains( - "return refill_four_bytes(br, bytes, len) || refill_one_byte(br, bytes, len);" - ), - "bit refill must try a 4-byte load before falling back to byte refill" - ); - assert!( - SHADER_SOURCE.contains("const uchar len9 = table.fast_len[fast_index];") - && SHADER_SOURCE.contains("symbol = table.fast_symbol[fast_index];"), - "Huffman decode must keep the 9-bit fast table path" - ); - assert!( - COMPUTE_SOURCE.contains("fast_symbol: [u8; 512]") - && COMPUTE_SOURCE.contains("fast_len: [u8; 512]"), - "host PreparedHuffman layout must include the 9-bit fast table" - ); -} - -#[test] -fn jpeg_encode_batch_entropy_path_stays_wired() { - assert!( - SHADER_SOURCE.contains("kernel void jpeg_encode_baseline_entropy_batch"), - "JPEG encode should keep the batched entropy kernel" - ); - assert!( - COMPUTE_SOURCE.contains("jpeg_baseline_encode_batch_pipeline"), - "host runtime should keep a compiled batch entropy pipeline" - ); - assert!( - COMPUTE_SOURCE.contains("encode_jpeg_baseline_entropy_batch_with_session"), - "public batch encode path should dispatch through the Metal batch helper" - ); -} - -#[test] -fn jpeg_encode_parallel_path_accepts_restart_segments() { - assert!( - SHADER_SOURCE.contains("jpeg_encode_push_restart_marker"), - "JPEG encode shader should emit restart markers on GPU" - ); - assert!( - SHADER_SOURCE.contains("if (params.restart_interval_mcus != 0u && mcus_since_restart == params.restart_interval_mcus)"), - "JPEG encode shader should handle restart intervals inside the entropy kernel" - ); - assert!( - !COMPUTE_SOURCE - .contains("if first.restart_interval_mcus != 0 {\n return Ok(None);\n }"), - "restart intervals must not force the JPEG batch encoder off the optimized Metal path" - ); -} diff --git a/crates/signinum-jpeg-metal/tests/viewport.rs b/crates/signinum-jpeg-metal/tests/viewport.rs deleted file mode 100644 index 4b6668ee..00000000 --- a/crates/signinum-jpeg-metal/tests/viewport.rs +++ /dev/null @@ -1,461 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use signinum_core::{BackendRequest, Downscale, PixelFormat, Rect}; -use signinum_jpeg::{Decoder, ScratchPool}; -use signinum_jpeg_metal::viewport::{ - choose_viewport_surface_strategy, compose_viewport_cpu, decode_viewport_region_cpu, - decode_viewport_to_surface, is_contiguous_viewport_workload, suggest_viewport_workload, - viewport_source_bounds, ViewportSurfaceStrategy, ViewportTile, -}; -#[cfg(target_os = "macos")] -use signinum_jpeg_metal::viewport::{compose_viewport_hybrid, decode_viewport_region_hybrid}; - -const BASELINE_420: &[u8] = include_bytes!("../fixtures/jpeg/baseline_420_16x16.jpg"); -const GRAYSCALE: &[u8] = include_bytes!("../fixtures/jpeg/grayscale_8x8.jpg"); - -fn quadrant_tiles() -> [ViewportTile; 4] { - [ - ViewportTile { - source_roi: Rect { - x: 0, - y: 0, - w: 8, - h: 8, - }, - dest: Rect { - x: 0, - y: 0, - w: 8, - h: 8, - }, - }, - ViewportTile { - source_roi: Rect { - x: 8, - y: 0, - w: 8, - h: 8, - }, - dest: Rect { - x: 8, - y: 0, - w: 8, - h: 8, - }, - }, - ViewportTile { - source_roi: Rect { - x: 0, - y: 8, - w: 8, - h: 8, - }, - dest: Rect { - x: 0, - y: 8, - w: 8, - h: 8, - }, - }, - ViewportTile { - source_roi: Rect { - x: 8, - y: 8, - w: 8, - h: 8, - }, - dest: Rect { - x: 8, - y: 8, - w: 8, - h: 8, - }, - }, - ] -} - -#[test] -fn cpu_viewport_quadrants_match_full_decode() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let mut pool = ScratchPool::new(); - - let actual = compose_viewport_cpu( - &decoder, - &mut pool, - PixelFormat::Rgb8, - Downscale::None, - (16, 16), - &quadrant_tiles(), - ) - .expect("viewport"); - let (expected, _) = decoder.decode(PixelFormat::Rgb8).expect("full decode"); - - assert_eq!(actual, expected); -} - -#[test] -fn suggested_viewport_workload_is_fixed_for_macro_like_input() { - let workload = suggest_viewport_workload((1_191, 408)).expect("workload"); - - assert_eq!(workload.scale, Downscale::Half); - assert_eq!(workload.viewport_dims, (576, 192)); - assert_eq!(workload.tiles.len(), 12); - assert_eq!( - workload.tiles.first(), - Some(&ViewportTile { - source_roi: Rect { - x: 18, - y: 12, - w: 192, - h: 192, - }, - dest: Rect { - x: 0, - y: 0, - w: 96, - h: 96, - }, - }) - ); - assert_eq!( - workload.tiles.last(), - Some(&ViewportTile { - source_roi: Rect { - x: 978, - y: 204, - w: 192, - h: 192, - }, - dest: Rect { - x: 480, - y: 96, - w: 96, - h: 96, - }, - }) - ); - assert!(is_contiguous_viewport_workload(&workload)); -} - -#[test] -fn cpu_viewport_misaligned_scaled_tile_matches_direct_decode() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let mut cpu_pool = ScratchPool::new(); - let roi = Rect { - x: 1, - y: 1, - w: 10, - h: 10, - }; - let tiles = [ViewportTile { - source_roi: roi, - dest: Rect { - x: 0, - y: 0, - w: 6, - h: 6, - }, - }]; - - let viewport = compose_viewport_cpu( - &decoder, - &mut cpu_pool, - PixelFormat::Rgb8, - Downscale::Half, - (6, 6), - &tiles, - ) - .expect("cpu viewport"); - let (expected, _outcome) = decoder - .decode_region_scaled( - PixelFormat::Rgb8, - signinum_jpeg::Rect { - x: roi.x, - y: roi.y, - w: roi.w, - h: roi.h, - }, - Downscale::Half, - ) - .expect("direct decode"); - - assert_eq!(expected.len(), 6 * 6 * 3); - assert_eq!(viewport, expected); -} - -#[test] -fn cpu_contiguous_viewport_region_matches_direct_decode() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let mut pool = ScratchPool::new(); - let workload = signinum_jpeg_metal::viewport::ViewportWorkload { - scale: Downscale::None, - viewport_dims: (16, 16), - tiles: quadrant_tiles().to_vec(), - }; - - let actual = decode_viewport_region_cpu(&decoder, &mut pool, PixelFormat::Rgb8, &workload) - .expect("cpu viewport region"); - let (expected, _) = decoder - .decode_region_scaled( - PixelFormat::Rgb8, - signinum_jpeg::Rect { - x: viewport_source_bounds(&workload).x, - y: viewport_source_bounds(&workload).y, - w: viewport_source_bounds(&workload).w, - h: viewport_source_bounds(&workload).h, - }, - workload.scale, - ) - .expect("direct decode"); - - assert_eq!(actual, expected); -} - -#[test] -fn gapped_tiles_are_not_contiguous() { - let workload = signinum_jpeg_metal::viewport::ViewportWorkload { - scale: Downscale::None, - viewport_dims: (16, 16), - tiles: vec![ - ViewportTile { - source_roi: Rect { - x: 0, - y: 0, - w: 8, - h: 8, - }, - dest: Rect { - x: 0, - y: 0, - w: 8, - h: 8, - }, - }, - ViewportTile { - source_roi: Rect { - x: 8, - y: 8, - w: 8, - h: 8, - }, - dest: Rect { - x: 8, - y: 8, - w: 8, - h: 8, - }, - }, - ], - }; - - assert!(!is_contiguous_viewport_workload(&workload)); - assert_eq!( - choose_viewport_surface_strategy(&workload, BackendRequest::Cpu).expect("cpu strategy"), - ViewportSurfaceStrategy::CpuComposite - ); -} - -#[test] -fn cpu_auto_strategy_prefers_contiguous_when_available() { - let workload = signinum_jpeg_metal::viewport::ViewportWorkload { - scale: Downscale::None, - viewport_dims: (16, 16), - tiles: quadrant_tiles().to_vec(), - }; - - assert!(is_contiguous_viewport_workload(&workload)); - assert_eq!( - choose_viewport_surface_strategy(&workload, BackendRequest::Cpu).expect("cpu strategy"), - ViewportSurfaceStrategy::CpuContiguous - ); -} - -#[cfg(target_os = "macos")] -#[test] -fn hybrid_viewport_quadrants_match_cpu_viewport() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let mut cpu_pool = ScratchPool::new(); - let mut hybrid_pool = ScratchPool::new(); - - let expected = compose_viewport_cpu( - &decoder, - &mut cpu_pool, - PixelFormat::Rgb8, - Downscale::None, - (16, 16), - &quadrant_tiles(), - ) - .expect("cpu viewport"); - let actual = compose_viewport_hybrid( - &decoder, - &mut hybrid_pool, - Downscale::None, - (16, 16), - &quadrant_tiles(), - ) - .expect("hybrid viewport"); - - assert_eq!(actual.as_bytes(), expected.as_slice()); -} - -#[cfg(target_os = "macos")] -#[test] -fn hybrid_viewport_misaligned_scaled_tile_matches_cpu_viewport() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let mut cpu_pool = ScratchPool::new(); - let mut hybrid_pool = ScratchPool::new(); - let tiles = [ViewportTile { - source_roi: Rect { - x: 1, - y: 1, - w: 10, - h: 10, - }, - dest: Rect { - x: 0, - y: 0, - w: 6, - h: 6, - }, - }]; - - let expected = compose_viewport_cpu( - &decoder, - &mut cpu_pool, - PixelFormat::Rgb8, - Downscale::Half, - (6, 6), - &tiles, - ) - .expect("cpu viewport"); - let actual = - compose_viewport_hybrid(&decoder, &mut hybrid_pool, Downscale::Half, (6, 6), &tiles) - .expect("hybrid viewport"); - - assert_eq!(actual.as_bytes(), expected.as_slice()); -} - -#[cfg(target_os = "macos")] -#[test] -fn hybrid_contiguous_viewport_region_matches_cpu_region() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let mut cpu_pool = ScratchPool::new(); - let mut hybrid_pool = ScratchPool::new(); - let workload = signinum_jpeg_metal::viewport::ViewportWorkload { - scale: Downscale::None, - viewport_dims: (16, 16), - tiles: quadrant_tiles().to_vec(), - }; - - let expected = - decode_viewport_region_cpu(&decoder, &mut cpu_pool, PixelFormat::Rgb8, &workload) - .expect("cpu viewport region"); - let actual = decode_viewport_region_hybrid(&decoder, &mut hybrid_pool, &workload) - .expect("hybrid viewport region"); - - assert_eq!(actual.as_bytes(), expected.as_slice()); -} - -#[cfg(target_os = "macos")] -#[test] -fn auto_viewport_surface_path_prefers_cpu_for_small_contiguous_workloads() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let mut direct_pool = ScratchPool::new(); - let mut auto_pool = ScratchPool::new(); - let workload = signinum_jpeg_metal::viewport::ViewportWorkload { - scale: Downscale::None, - viewport_dims: (16, 16), - tiles: quadrant_tiles().to_vec(), - }; - - let expected = signinum_jpeg_metal::viewport::decode_viewport_region_cpu_to_surface( - &decoder, - &mut direct_pool, - &workload, - ) - .expect("cpu viewport surface"); - let actual = - decode_viewport_to_surface(&decoder, &mut auto_pool, &workload, BackendRequest::Auto) - .expect("auto viewport surface"); - - assert_eq!(actual.as_bytes(), expected.as_bytes()); -} - -#[cfg(not(target_os = "macos"))] -#[test] -fn non_macos_auto_viewport_surface_returns_cpu_surface() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let mut pool = ScratchPool::new(); - let workload = signinum_jpeg_metal::viewport::ViewportWorkload { - scale: Downscale::None, - viewport_dims: (16, 16), - tiles: quadrant_tiles().to_vec(), - }; - - let surface = decode_viewport_to_surface(&decoder, &mut pool, &workload, BackendRequest::Auto) - .expect("auto viewport surface"); - - assert_eq!( - signinum_core::DeviceSurface::backend_kind(&surface), - signinum_core::BackendKind::Cpu - ); -} - -#[cfg(not(target_os = "macos"))] -#[test] -fn non_macos_explicit_metal_viewport_surface_is_unavailable() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let mut pool = ScratchPool::new(); - let workload = signinum_jpeg_metal::viewport::ViewportWorkload { - scale: Downscale::None, - viewport_dims: (16, 16), - tiles: quadrant_tiles().to_vec(), - }; - - let result = decode_viewport_to_surface(&decoder, &mut pool, &workload, BackendRequest::Metal); - assert!(matches!( - result, - Err(signinum_jpeg_metal::Error::MetalUnavailable) - )); -} - -#[test] -fn explicit_metal_viewport_unsupported_shape_is_rejected() { - let decoder = Decoder::new(GRAYSCALE).expect("decoder"); - let mut pool = ScratchPool::new(); - let workload = signinum_jpeg_metal::viewport::ViewportWorkload { - scale: Downscale::None, - viewport_dims: (8, 8), - tiles: vec![ViewportTile { - source_roi: Rect { - x: 0, - y: 0, - w: 8, - h: 8, - }, - dest: Rect { - x: 0, - y: 0, - w: 8, - h: 8, - }, - }], - }; - - let result = decode_viewport_to_surface(&decoder, &mut pool, &workload, BackendRequest::Metal); - - match result { - Err(signinum_jpeg_metal::Error::UnsupportedMetalRequest { reason }) => { - assert!(reason.contains("JPEG Metal")); - } - #[cfg(not(target_os = "macos"))] - Err(signinum_jpeg_metal::Error::MetalUnavailable) => { - panic!("unsupported shape should be rejected before host availability") - } - Err(other) => panic!("unexpected explicit Metal viewport error: {other:?}"), - Ok(surface) => panic!( - "explicit Metal viewport must not fall back; got {:?}", - signinum_core::DeviceSurface::backend_kind(&surface) - ), - } -} diff --git a/crates/signinum-jpeg/README.md b/crates/signinum-jpeg/README.md deleted file mode 100644 index 4558b257..00000000 --- a/crates/signinum-jpeg/README.md +++ /dev/null @@ -1,94 +0,0 @@ -# signinum-jpeg - -JPEG tile inspect and CPU decode for whole-slide imaging workloads. -The baseline JPEG encoder is kept for generated fixtures and explicit fallback -or derived-output use. WSI/DICOM storage conversion should prefer compressed -tile passthrough, or lossless JPEG 2000 / HTJ2K encode when a new diagnostic -codestream is required. - -Install: - -```sh -cargo add signinum-jpeg -``` - -Use this crate when you need codec primitives directly. Use -[`statumen`](https://github.com/frames-sg/statumen) when you need a whole-slide -reader/container layer. - -```rust -use signinum_jpeg::{Decoder, JpegError, JpegView, RowSink}; - -let info = Decoder::inspect(bytes)?; -println!( - "{}×{} {:?} mcu={:?} restart={:?}", - info.dimensions.0, - info.dimensions.1, - info.sof_kind, - info.mcu_geometry, - info.restart_interval -); - -let view = JpegView::parse(bytes)?; -if let Some(candidate) = view.passthrough_candidate() { - println!( - "passthrough syntax={:?} payload={:?}", - candidate.transfer_syntax(), - candidate.payload_kind() - ); -} -if let Some(index) = view.restart_index()? { - println!("restart segments={}", index.segments.len()); -} -let decoder = Decoder::from_view(view)?; - -struct Sink; - -impl RowSink for Sink { - type Error = JpegError; - - fn write_row(&mut self, _y: u32, _row: &[u8]) -> Result<(), JpegError> { - Ok(()) - } -} - -decoder.decode_rows(&mut Sink)?; -``` - -For WSI viewers decoding many independent JPEG tiles per frame, use the -production batch APIs. `TileBatchOptions::default()` uses host parallelism and -one reusable decoder context plus scratch pool per worker. The `_with_options` -variants are the intended path when TIFF or DICOM metadata has already resolved -ambiguous three-component JPEG data to RGB or YCbCr. - -```rust -use signinum_jpeg::{ - decode_tiles_into_with_options, ColorTransform, DecodeOptions, PixelFormat, - TileBatchOptions, TileDecodeJob, -}; - -let decode_options = DecodeOptions::default().with_color_transform(ColorTransform::ForceYCbCr); -let stride = tile_width as usize * 3; -let mut outputs = compressed_tiles - .iter() - .map(|_| vec![0_u8; stride * tile_height as usize]) - .collect::>(); -let mut jobs = compressed_tiles - .iter() - .zip(outputs.iter_mut()) - .map(|(input, out)| TileDecodeJob { - input: input.as_ref(), - out: out.as_mut_slice(), - stride, - }) - .collect::>(); - -decode_tiles_into_with_options( - &mut jobs, - PixelFormat::Rgb8, - decode_options, - TileBatchOptions::default(), -)?; -``` - -Current decode targets are native `x86_64` and `aarch64` hosts. diff --git a/crates/signinum-jpeg/src/adapter/metal_fast420.rs b/crates/signinum-jpeg/src/adapter/metal_fast420.rs deleted file mode 100644 index 2bb568df..00000000 --- a/crates/signinum-jpeg/src/adapter/metal_fast420.rs +++ /dev/null @@ -1,993 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use crate::error::{HuffmanFailure, JpegError, MarkerKind}; -use crate::info::{ColorSpace, SamplingFactors, SofKind}; -use crate::parse::header::parse_header; -use crate::parse::scan::ScanComponent; -use crate::parse::tables::RawHuffmanTable; -use alloc::vec::Vec; - -const PLANNER_FAST_BITS: u8 = 12; -const PLANNER_FAST_ENTRIES: usize = 1 << PLANNER_FAST_BITS; -const MAX_NONRESTART_ENTROPY_CHECKPOINTS: u32 = 2048; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum MetalFast420PacketError { - Decode(JpegError), - UnsupportedSof(SofKind), - UnsupportedColorSpace(ColorSpace), - UnsupportedSampling, - UnsupportedComponentOrder, - MissingScan, - MissingQuantTable { slot: u8 }, - MissingHuffmanTable { kind: TableKind, slot: u8 }, - EntropyMarkerUnsupported { marker: u8 }, - TruncatedEntropy, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TableKind { - Dc, - Ac, -} - -impl From for MetalFast420PacketError { - fn from(value: JpegError) -> Self { - Self::Decode(value) - } -} - -#[repr(C)] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MetalHuffmanTable { - pub bits: [u8; 16], - pub values_len: u16, - pub values: [u8; 256], -} - -#[repr(C)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct JpegMetalEntropyCheckpointV1 { - pub mcu_index: u32, - pub entropy_pos: u32, - pub bit_acc: u64, - pub bit_count: u32, - pub y_prev_dc: i32, - pub cb_prev_dc: i32, - pub cr_prev_dc: i32, - pub reserved: u32, -} - -impl JpegMetalEntropyCheckpointV1 { - fn restart(mcu_index: u32, entropy_pos: u32) -> Self { - Self { - mcu_index, - entropy_pos, - bit_acc: 0, - bit_count: 0, - y_prev_dc: 0, - cb_prev_dc: 0, - cr_prev_dc: 0, - reserved: 0, - } - } -} - -impl MetalHuffmanTable { - fn from_raw(raw: &RawHuffmanTable) -> Self { - let mut values = [0u8; 256]; - let slice = raw.values.as_slice(); - values[..slice.len()].copy_from_slice(slice); - Self { - bits: raw.bits, - values_len: slice.len() as u16, - values, - } - } -} - -#[repr(C)] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JpegMetalFast420PacketV1 { - pub dimensions: (u32, u32), - pub mcus_per_row: u32, - pub mcu_rows: u32, - pub restart_interval_mcus: u32, - pub restart_offsets: Vec, - pub entropy_checkpoints: Vec, - pub y_quant: [u16; 64], - pub cb_quant: [u16; 64], - pub cr_quant: [u16; 64], - pub y_dc_table: MetalHuffmanTable, - pub y_ac_table: MetalHuffmanTable, - pub cb_dc_table: MetalHuffmanTable, - pub cb_ac_table: MetalHuffmanTable, - pub cr_dc_table: MetalHuffmanTable, - pub cr_ac_table: MetalHuffmanTable, - pub entropy_bytes: Vec, -} - -#[repr(C)] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JpegMetalFast422PacketV1 { - pub dimensions: (u32, u32), - pub mcus_per_row: u32, - pub mcu_rows: u32, - pub restart_interval_mcus: u32, - pub restart_offsets: Vec, - pub entropy_checkpoints: Vec, - pub y_quant: [u16; 64], - pub cb_quant: [u16; 64], - pub cr_quant: [u16; 64], - pub y_dc_table: MetalHuffmanTable, - pub y_ac_table: MetalHuffmanTable, - pub cb_dc_table: MetalHuffmanTable, - pub cb_ac_table: MetalHuffmanTable, - pub cr_dc_table: MetalHuffmanTable, - pub cr_ac_table: MetalHuffmanTable, - pub entropy_bytes: Vec, -} - -#[repr(C)] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JpegMetalFast444PacketV1 { - pub dimensions: (u32, u32), - pub mcus_per_row: u32, - pub mcu_rows: u32, - pub restart_interval_mcus: u32, - pub restart_offsets: Vec, - pub entropy_checkpoints: Vec, - pub y_quant: [u16; 64], - pub cb_quant: [u16; 64], - pub cr_quant: [u16; 64], - pub y_dc_table: MetalHuffmanTable, - pub y_ac_table: MetalHuffmanTable, - pub cb_dc_table: MetalHuffmanTable, - pub cb_ac_table: MetalHuffmanTable, - pub cr_dc_table: MetalHuffmanTable, - pub cr_ac_table: MetalHuffmanTable, - pub entropy_bytes: Vec, -} - -#[repr(C)] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct JpegMetalGrayPacketV1 { - pub dimensions: (u32, u32), - pub mcus_per_row: u32, - pub mcu_rows: u32, - pub restart_interval_mcus: u32, - pub restart_offsets: Vec, - pub y_quant: [u16; 64], - pub y_dc_table: MetalHuffmanTable, - pub y_ac_table: MetalHuffmanTable, - pub entropy_bytes: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct EntropySegments { - entropy_bytes: Vec, - restart_offsets: Vec, -} - -#[derive(Debug, Clone, Copy)] -enum PlannerLayout { - Fast420, - Fast422, - Fast444, -} - -#[derive(Debug, Clone)] -struct PlannerHuffman { - fast: [(u8, u8); PLANNER_FAST_ENTRIES], - max_code: [i32; 17], - val_offset: [i32; 17], - values: [u8; 256], - values_len: usize, -} - -impl PlannerHuffman { - fn from_metal(raw: &MetalHuffmanTable) -> Result { - let mut fast = [(0u8, 0u8); PLANNER_FAST_ENTRIES]; - let mut max_code = [-1i32; 17]; - let mut val_offset = [0i32; 17]; - let mut values = [0u8; 256]; - let values_len = usize::from(raw.values_len); - values[..values_len].copy_from_slice(&raw.values[..values_len]); - - let mut huffsize = [0u8; 256]; - let mut huffsize_len = 0usize; - for (len_minus_1, &count) in raw.bits.iter().enumerate() { - let len = (len_minus_1 + 1) as u8; - for _ in 0..count { - huffsize[huffsize_len] = len; - huffsize_len += 1; - } - } - - let mut huffcode = [0u16; 256]; - let mut code: u32 = 0; - let mut si = huffsize.first().copied().unwrap_or(0); - for (k, &s) in huffsize[..huffsize_len].iter().enumerate() { - while s != si { - code <<= 1; - si += 1; - } - huffcode[k] = code as u16; - code = code.checked_add(1).ok_or_else(planner_code_overflow)?; - } - if si > 0 && (code - 1) >= (1u32 << si) { - return Err(planner_code_overflow()); - } - - let mut k = 0usize; - for len_minus_1 in 0..16 { - let len = len_minus_1 + 1; - let count = raw.bits[len_minus_1] as usize; - if count == 0 { - continue; - } - let min_code = i32::from(huffcode[k]); - max_code[len] = i32::from(huffcode[k + count - 1]); - val_offset[len] = k as i32 - min_code; - k += count; - } - - k = 0; - for len_minus_1 in 0..PLANNER_FAST_BITS as usize { - let len = (len_minus_1 + 1) as u8; - let count = raw.bits[len_minus_1] as usize; - for _ in 0..count { - let c = huffcode[k]; - let fast_index_base = (c as usize) << (PLANNER_FAST_BITS - len); - let fast_count = 1 << (PLANNER_FAST_BITS - len); - for j in 0..fast_count { - fast[fast_index_base + j] = (raw.values[k], len); - } - k += 1; - } - } - - Ok(Self { - fast, - max_code, - val_offset, - values, - values_len, - }) - } - - fn decode(&self, reader: &mut PlannerBitReader<'_>) -> Result { - reader.ensure_bits_padded(PLANNER_FAST_BITS); - let peek = reader.peek_bits(PLANNER_FAST_BITS) as usize; - let (sym, len) = self.fast[peek]; - if len != 0 { - reader.consume_bits(len); - return Ok(sym); - } - - reader.ensure_bits_padded(16); - let code16 = reader.peek_bits(16) as i32; - for len in (PLANNER_FAST_BITS as usize + 1)..=16 { - let l = len as u8; - let c = code16 >> (16 - l); - if c <= self.max_code[len] { - reader.consume_bits(l); - let idx = (c + self.val_offset[len]) as usize; - if idx >= self.values_len { - return Err(planner_invalid_symbol()); - } - return Ok(self.values[idx]); - } - } - Err(planner_code_overflow()) - } -} - -struct PlannerBitReader<'a> { - bytes: &'a [u8], - pos: usize, - acc: u64, - bits: u8, -} - -impl<'a> PlannerBitReader<'a> { - fn new(bytes: &'a [u8]) -> Self { - Self { - bytes, - pos: 0, - acc: 0, - bits: 0, - } - } - - fn checkpoint( - &self, - mcu_index: u32, - y_prev_dc: i32, - cb_prev_dc: i32, - cr_prev_dc: i32, - ) -> Result { - Ok(JpegMetalEntropyCheckpointV1 { - mcu_index, - entropy_pos: u32::try_from(self.pos).map_err(|_| planner_truncated_entropy())?, - bit_acc: self.acc, - bit_count: u32::from(self.bits), - y_prev_dc, - cb_prev_dc, - cr_prev_dc, - reserved: 0, - }) - } - - fn ensure_bits(&mut self, n: u8) -> Result<(), MetalFast420PacketError> { - while self.bits < n { - if !self.refill_one_byte() { - return Err(planner_table_exhausted()); - } - } - Ok(()) - } - - fn ensure_bits_padded(&mut self, n: u8) { - while self.bits < n { - if !self.refill_one_byte() { - self.acc |= 1u64 << (63 - self.bits); - self.bits += 1; - } - } - } - - fn refill_one_byte(&mut self) -> bool { - let Some(&byte) = self.bytes.get(self.pos) else { - return false; - }; - let shift = 64 - 8 - self.bits; - self.acc |= u64::from(byte) << shift; - self.pos += 1; - self.bits += 8; - true - } - - fn peek_bits(&self, n: u8) -> u32 { - if n == 0 { - 0 - } else { - (self.acc >> (64 - n)) as u32 - } - } - - fn consume_bits(&mut self, n: u8) { - self.acc <<= n; - self.bits -= n; - } - - fn receive_extend(&mut self, ssss: u8) -> Result { - if ssss == 0 { - return Ok(0); - } - self.ensure_bits(ssss)?; - let value = self.peek_bits(ssss) as i32; - self.consume_bits(ssss); - let threshold = 1i32 << (ssss - 1); - Ok(if value < threshold { - value + ((-1i32) << ssss) + 1 - } else { - value - }) - } -} - -pub fn build_metal_fast420_packet( - bytes: &[u8], -) -> Result { - let header = parse_header(bytes)?; - if !matches!(header.sof_kind, SofKind::Baseline8 | SofKind::Extended8) { - return Err(MetalFast420PacketError::UnsupportedSof(header.sof_kind)); - } - if header.bit_depth != 8 { - return Err(MetalFast420PacketError::Decode( - JpegError::UnsupportedBitDepth { - depth: header.bit_depth, - }, - )); - } - if header.color_space() != ColorSpace::YCbCr { - return Err(MetalFast420PacketError::UnsupportedColorSpace( - header.color_space(), - )); - } - if header.sampling != SamplingFactors::from_components(&[(2, 2), (1, 1), (1, 1)]) { - return Err(MetalFast420PacketError::UnsupportedSampling); - } - let scan = header - .scan - .as_ref() - .ok_or(MetalFast420PacketError::MissingScan)?; - let [y_scan, cb_scan, cr_scan] = ordered_scan_triplet(&header.component_ids, &scan.components)?; - - let y_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 0)?; - let cb_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 1)?; - let cr_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 2)?; - let y_dc_table = huffman_table(&header.huffman_tables.dc, TableKind::Dc, y_scan.dc_table)?; - let y_ac_table = huffman_table(&header.huffman_tables.ac, TableKind::Ac, y_scan.ac_table)?; - let cb_dc_table = huffman_table(&header.huffman_tables.dc, TableKind::Dc, cb_scan.dc_table)?; - let cb_ac_table = huffman_table(&header.huffman_tables.ac, TableKind::Ac, cb_scan.ac_table)?; - let cr_dc_table = huffman_table(&header.huffman_tables.dc, TableKind::Dc, cr_scan.dc_table)?; - let cr_ac_table = huffman_table(&header.huffman_tables.ac, TableKind::Ac, cr_scan.ac_table)?; - - let entropy_offset = header - .sos_offset - .ok_or(MetalFast420PacketError::MissingScan)?; - let restart_interval_mcus = u32::from(header.restart_interval.unwrap_or(0)); - let EntropySegments { - entropy_bytes, - restart_offsets, - } = extract_entropy_segments(&bytes[entropy_offset..], header.restart_interval)?; - let (width, height) = header.dimensions; - let mcus_per_row = width.div_ceil(16); - let mcu_rows = height.div_ceil(16); - let entropy_checkpoints = build_triplet_entropy_checkpoints( - PlannerLayout::Fast420, - &entropy_bytes, - mcus_per_row - .checked_mul(mcu_rows) - .expect("JPEG Metal fast420 MCU count fits in u32"), - restart_interval_mcus, - &restart_offsets, - [&y_dc_table, &cb_dc_table, &cr_dc_table], - [&y_ac_table, &cb_ac_table, &cr_ac_table], - )?; - - Ok(JpegMetalFast420PacketV1 { - dimensions: header.dimensions, - mcus_per_row, - mcu_rows, - restart_interval_mcus, - restart_offsets, - entropy_checkpoints, - y_quant, - cb_quant, - cr_quant, - y_dc_table, - y_ac_table, - cb_dc_table, - cb_ac_table, - cr_dc_table, - cr_ac_table, - entropy_bytes, - }) -} - -pub fn build_metal_fast444_packet( - bytes: &[u8], -) -> Result { - let header = parse_header(bytes)?; - if !matches!(header.sof_kind, SofKind::Baseline8 | SofKind::Extended8) { - return Err(MetalFast420PacketError::UnsupportedSof(header.sof_kind)); - } - if header.bit_depth != 8 { - return Err(MetalFast420PacketError::Decode( - JpegError::UnsupportedBitDepth { - depth: header.bit_depth, - }, - )); - } - if !matches!(header.color_space(), ColorSpace::YCbCr | ColorSpace::Rgb) { - return Err(MetalFast420PacketError::UnsupportedColorSpace( - header.color_space(), - )); - } - if header.sampling != SamplingFactors::from_components(&[(1, 1), (1, 1), (1, 1)]) { - return Err(MetalFast420PacketError::UnsupportedSampling); - } - let scan = header - .scan - .as_ref() - .ok_or(MetalFast420PacketError::MissingScan)?; - let [y_scan, cb_scan, cr_scan] = ordered_scan_triplet(&header.component_ids, &scan.components)?; - - let y_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 0)?; - let cb_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 1)?; - let cr_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 2)?; - let y_dc_table = huffman_table(&header.huffman_tables.dc, TableKind::Dc, y_scan.dc_table)?; - let y_ac_table = huffman_table(&header.huffman_tables.ac, TableKind::Ac, y_scan.ac_table)?; - let cb_dc_table = huffman_table(&header.huffman_tables.dc, TableKind::Dc, cb_scan.dc_table)?; - let cb_ac_table = huffman_table(&header.huffman_tables.ac, TableKind::Ac, cb_scan.ac_table)?; - let cr_dc_table = huffman_table(&header.huffman_tables.dc, TableKind::Dc, cr_scan.dc_table)?; - let cr_ac_table = huffman_table(&header.huffman_tables.ac, TableKind::Ac, cr_scan.ac_table)?; - - let entropy_offset = header - .sos_offset - .ok_or(MetalFast420PacketError::MissingScan)?; - let restart_interval_mcus = u32::from(header.restart_interval.unwrap_or(0)); - let EntropySegments { - entropy_bytes, - restart_offsets, - } = extract_entropy_segments(&bytes[entropy_offset..], header.restart_interval)?; - let (width, height) = header.dimensions; - let mcus_per_row = width.div_ceil(8); - let mcu_rows = height.div_ceil(8); - let entropy_checkpoints = build_triplet_entropy_checkpoints( - PlannerLayout::Fast444, - &entropy_bytes, - mcus_per_row - .checked_mul(mcu_rows) - .expect("JPEG Metal fast444 MCU count fits in u32"), - restart_interval_mcus, - &restart_offsets, - [&y_dc_table, &cb_dc_table, &cr_dc_table], - [&y_ac_table, &cb_ac_table, &cr_ac_table], - )?; - - Ok(JpegMetalFast444PacketV1 { - dimensions: header.dimensions, - mcus_per_row, - mcu_rows, - restart_interval_mcus, - restart_offsets, - entropy_checkpoints, - y_quant, - cb_quant, - cr_quant, - y_dc_table, - y_ac_table, - cb_dc_table, - cb_ac_table, - cr_dc_table, - cr_ac_table, - entropy_bytes, - }) -} - -pub fn build_metal_fast422_packet( - bytes: &[u8], -) -> Result { - let header = parse_header(bytes)?; - if !matches!(header.sof_kind, SofKind::Baseline8 | SofKind::Extended8) { - return Err(MetalFast420PacketError::UnsupportedSof(header.sof_kind)); - } - if header.bit_depth != 8 { - return Err(MetalFast420PacketError::Decode( - JpegError::UnsupportedBitDepth { - depth: header.bit_depth, - }, - )); - } - if header.color_space() != ColorSpace::YCbCr { - return Err(MetalFast420PacketError::UnsupportedColorSpace( - header.color_space(), - )); - } - if header.sampling != SamplingFactors::from_components(&[(2, 1), (1, 1), (1, 1)]) { - return Err(MetalFast420PacketError::UnsupportedSampling); - } - let scan = header - .scan - .as_ref() - .ok_or(MetalFast420PacketError::MissingScan)?; - let [y_scan, cb_scan, cr_scan] = ordered_scan_triplet(&header.component_ids, &scan.components)?; - - let y_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 0)?; - let cb_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 1)?; - let cr_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 2)?; - let y_dc_table = huffman_table(&header.huffman_tables.dc, TableKind::Dc, y_scan.dc_table)?; - let y_ac_table = huffman_table(&header.huffman_tables.ac, TableKind::Ac, y_scan.ac_table)?; - let cb_dc_table = huffman_table(&header.huffman_tables.dc, TableKind::Dc, cb_scan.dc_table)?; - let cb_ac_table = huffman_table(&header.huffman_tables.ac, TableKind::Ac, cb_scan.ac_table)?; - let cr_dc_table = huffman_table(&header.huffman_tables.dc, TableKind::Dc, cr_scan.dc_table)?; - let cr_ac_table = huffman_table(&header.huffman_tables.ac, TableKind::Ac, cr_scan.ac_table)?; - - let entropy_offset = header - .sos_offset - .ok_or(MetalFast420PacketError::MissingScan)?; - let restart_interval_mcus = u32::from(header.restart_interval.unwrap_or(0)); - let EntropySegments { - entropy_bytes, - restart_offsets, - } = extract_entropy_segments(&bytes[entropy_offset..], header.restart_interval)?; - let (width, height) = header.dimensions; - let mcus_per_row = width.div_ceil(16); - let mcu_rows = height.div_ceil(8); - let entropy_checkpoints = build_triplet_entropy_checkpoints( - PlannerLayout::Fast422, - &entropy_bytes, - mcus_per_row - .checked_mul(mcu_rows) - .expect("JPEG Metal fast422 MCU count fits in u32"), - restart_interval_mcus, - &restart_offsets, - [&y_dc_table, &cb_dc_table, &cr_dc_table], - [&y_ac_table, &cb_ac_table, &cr_ac_table], - )?; - - Ok(JpegMetalFast422PacketV1 { - dimensions: header.dimensions, - mcus_per_row, - mcu_rows, - restart_interval_mcus, - restart_offsets, - entropy_checkpoints, - y_quant, - cb_quant, - cr_quant, - y_dc_table, - y_ac_table, - cb_dc_table, - cb_ac_table, - cr_dc_table, - cr_ac_table, - entropy_bytes, - }) -} - -pub fn build_metal_gray_packet( - bytes: &[u8], -) -> Result { - let header = parse_header(bytes)?; - if !matches!(header.sof_kind, SofKind::Baseline8 | SofKind::Extended8) { - return Err(MetalFast420PacketError::UnsupportedSof(header.sof_kind)); - } - if header.bit_depth != 8 { - return Err(MetalFast420PacketError::Decode( - JpegError::UnsupportedBitDepth { - depth: header.bit_depth, - }, - )); - } - if header.color_space() != ColorSpace::Grayscale { - return Err(MetalFast420PacketError::UnsupportedColorSpace( - header.color_space(), - )); - } - if header.sampling != SamplingFactors::from_components(&[(1, 1)]) { - return Err(MetalFast420PacketError::UnsupportedSampling); - } - - let scan = header - .scan - .as_ref() - .ok_or(MetalFast420PacketError::MissingScan)?; - if header.component_ids.len() != 1 || scan.components.len() != 1 { - return Err(MetalFast420PacketError::UnsupportedComponentOrder); - } - if scan.components[0].id != header.component_ids[0] { - return Err(MetalFast420PacketError::UnsupportedComponentOrder); - } - - let y_quant = quant_for_component(&header.quant_table_ids, &header.quant_tables.entries, 0)?; - let y_dc_table = huffman_table( - &header.huffman_tables.dc, - TableKind::Dc, - scan.components[0].dc_table, - )?; - let y_ac_table = huffman_table( - &header.huffman_tables.ac, - TableKind::Ac, - scan.components[0].ac_table, - )?; - - let entropy_offset = header - .sos_offset - .ok_or(MetalFast420PacketError::MissingScan)?; - let restart_interval_mcus = u32::from(header.restart_interval.unwrap_or(0)); - let EntropySegments { - entropy_bytes, - restart_offsets, - } = extract_entropy_segments(&bytes[entropy_offset..], header.restart_interval)?; - let (width, height) = header.dimensions; - - Ok(JpegMetalGrayPacketV1 { - dimensions: header.dimensions, - mcus_per_row: width.div_ceil(8), - mcu_rows: height.div_ceil(8), - restart_interval_mcus, - restart_offsets, - y_quant, - y_dc_table, - y_ac_table, - entropy_bytes, - }) -} - -pub fn build_metal_fast420_packet_for_decoder( - decoder: &crate::decoder::Decoder<'_>, -) -> Result { - build_metal_fast420_packet(decoder.bytes) -} - -pub fn build_metal_fast444_packet_for_decoder( - decoder: &crate::decoder::Decoder<'_>, -) -> Result { - build_metal_fast444_packet(decoder.bytes) -} - -pub fn build_metal_fast422_packet_for_decoder( - decoder: &crate::decoder::Decoder<'_>, -) -> Result { - build_metal_fast422_packet(decoder.bytes) -} - -pub fn build_metal_gray_packet_for_decoder( - decoder: &crate::decoder::Decoder<'_>, -) -> Result { - build_metal_gray_packet(decoder.bytes) -} - -fn quant_for_component( - quant_table_ids: &[u8], - tables: &[Option<[u16; 64]>; 4], - component_idx: usize, -) -> Result<[u16; 64], MetalFast420PacketError> { - let slot = *quant_table_ids - .get(component_idx) - .ok_or(MetalFast420PacketError::UnsupportedComponentOrder)?; - tables[slot as usize].ok_or(MetalFast420PacketError::MissingQuantTable { slot }) -} - -fn ordered_scan_triplet( - component_ids: &[u8], - scan_components: &[ScanComponent], -) -> Result<[ScanComponent; 3], MetalFast420PacketError> { - if component_ids.len() != 3 || scan_components.len() != 3 { - return Err(MetalFast420PacketError::UnsupportedComponentOrder); - } - - let mut ordered = [None; 3]; - for (index, &component_id) in component_ids.iter().enumerate() { - let Some(component) = scan_components - .iter() - .copied() - .find(|component| component.id == component_id) - else { - return Err(MetalFast420PacketError::UnsupportedComponentOrder); - }; - ordered[index] = Some(component); - } - - match ordered { - [Some(first), Some(second), Some(third)] => Ok([first, second, third]), - _ => Err(MetalFast420PacketError::UnsupportedComponentOrder), - } -} - -fn huffman_table( - tables: &[Option; 4], - kind: TableKind, - slot: u8, -) -> Result { - let raw = tables[slot as usize] - .as_ref() - .ok_or(MetalFast420PacketError::MissingHuffmanTable { kind, slot })?; - Ok(MetalHuffmanTable::from_raw(raw)) -} - -fn planner_error(reason: HuffmanFailure) -> MetalFast420PacketError { - MetalFast420PacketError::Decode(JpegError::HuffmanDecode { mcu: 0, reason }) -} - -fn planner_code_overflow() -> MetalFast420PacketError { - planner_error(HuffmanFailure::CodeOverflow) -} - -fn planner_invalid_symbol() -> MetalFast420PacketError { - planner_error(HuffmanFailure::InvalidSymbol) -} - -fn planner_table_exhausted() -> MetalFast420PacketError { - planner_error(HuffmanFailure::TableExhausted) -} - -fn planner_truncated_entropy() -> MetalFast420PacketError { - MetalFast420PacketError::TruncatedEntropy -} - -fn nonrestart_entropy_chunk_mcus(total_mcus: u32) -> u32 { - total_mcus - .div_ceil(MAX_NONRESTART_ENTROPY_CHECKPOINTS) - .max(1) -} - -fn restart_entropy_checkpoints( - total_mcus: u32, - restart_interval_mcus: u32, - restart_offsets: &[u32], -) -> Vec { - restart_offsets - .iter() - .enumerate() - .filter_map(|(index, &offset)| { - let Ok(index) = u32::try_from(index) else { - return None; - }; - let mcu_index = index.saturating_mul(restart_interval_mcus); - (mcu_index < total_mcus) - .then_some(JpegMetalEntropyCheckpointV1::restart(mcu_index, offset)) - }) - .collect() -} - -fn build_triplet_entropy_checkpoints( - layout: PlannerLayout, - entropy_bytes: &[u8], - total_mcus: u32, - restart_interval_mcus: u32, - restart_offsets: &[u32], - dc_tables: [&MetalHuffmanTable; 3], - ac_tables: [&MetalHuffmanTable; 3], -) -> Result, MetalFast420PacketError> { - if total_mcus == 0 { - return Ok(vec![JpegMetalEntropyCheckpointV1::restart(0, 0)]); - } - if restart_interval_mcus != 0 { - let checkpoints = - restart_entropy_checkpoints(total_mcus, restart_interval_mcus, restart_offsets); - if !checkpoints.is_empty() { - return Ok(checkpoints); - } - return Ok(vec![JpegMetalEntropyCheckpointV1::restart(0, 0)]); - } - - let dc_tables = alloc::vec![ - PlannerHuffman::from_metal(dc_tables[0])?, - PlannerHuffman::from_metal(dc_tables[1])?, - PlannerHuffman::from_metal(dc_tables[2])?, - ] - .into_boxed_slice(); - let ac_tables = alloc::vec![ - PlannerHuffman::from_metal(ac_tables[0])?, - PlannerHuffman::from_metal(ac_tables[1])?, - PlannerHuffman::from_metal(ac_tables[2])?, - ] - .into_boxed_slice(); - let mut reader = PlannerBitReader::new(entropy_bytes); - let mut checkpoints = Vec::new(); - let chunk_mcus = nonrestart_entropy_chunk_mcus(total_mcus); - let mut next_checkpoint_mcu = 0u32; - let mut y_prev_dc = 0i32; - let mut cb_prev_dc = 0i32; - let mut cr_prev_dc = 0i32; - - for mcu_index in 0..total_mcus { - if mcu_index == next_checkpoint_mcu { - checkpoints.push(reader.checkpoint(mcu_index, y_prev_dc, cb_prev_dc, cr_prev_dc)?); - next_checkpoint_mcu = next_checkpoint_mcu.saturating_add(chunk_mcus); - } - skip_triplet_mcu( - layout, - &mut reader, - [&dc_tables[0], &dc_tables[1], &dc_tables[2]], - [&ac_tables[0], &ac_tables[1], &ac_tables[2]], - &mut y_prev_dc, - &mut cb_prev_dc, - &mut cr_prev_dc, - )?; - } - - if checkpoints.is_empty() { - checkpoints.push(JpegMetalEntropyCheckpointV1::restart(0, 0)); - } - Ok(checkpoints) -} - -fn skip_triplet_mcu( - layout: PlannerLayout, - reader: &mut PlannerBitReader<'_>, - dc_tables: [&PlannerHuffman; 3], - ac_tables: [&PlannerHuffman; 3], - y_prev_dc: &mut i32, - cb_prev_dc: &mut i32, - cr_prev_dc: &mut i32, -) -> Result<(), MetalFast420PacketError> { - match layout { - PlannerLayout::Fast420 => { - for _ in 0..4 { - skip_block(reader, dc_tables[0], ac_tables[0], y_prev_dc)?; - } - skip_block(reader, dc_tables[1], ac_tables[1], cb_prev_dc)?; - skip_block(reader, dc_tables[2], ac_tables[2], cr_prev_dc)?; - } - PlannerLayout::Fast422 => { - for _ in 0..2 { - skip_block(reader, dc_tables[0], ac_tables[0], y_prev_dc)?; - } - skip_block(reader, dc_tables[1], ac_tables[1], cb_prev_dc)?; - skip_block(reader, dc_tables[2], ac_tables[2], cr_prev_dc)?; - } - PlannerLayout::Fast444 => { - skip_block(reader, dc_tables[0], ac_tables[0], y_prev_dc)?; - skip_block(reader, dc_tables[1], ac_tables[1], cb_prev_dc)?; - skip_block(reader, dc_tables[2], ac_tables[2], cr_prev_dc)?; - } - } - Ok(()) -} - -fn skip_block( - reader: &mut PlannerBitReader<'_>, - dc_table: &PlannerHuffman, - ac_table: &PlannerHuffman, - prev_dc: &mut i32, -) -> Result<(), MetalFast420PacketError> { - let ssss = dc_table.decode(reader)?; - if ssss > 15 { - return Err(planner_invalid_symbol()); - } - let diff = reader.receive_extend(ssss)?; - *prev_dc = prev_dc.wrapping_add(diff); - - let mut k = 1usize; - while k < 64 { - let sym = ac_table.decode(reader)?; - let run = usize::from(sym >> 4); - let ssss = sym & 0x0F; - if ssss == 0 { - if run == 15 { - k += 16; - continue; - } - break; - } - - k += run; - if k >= 64 { - return Err(planner_invalid_symbol()); - } - let _ = reader.receive_extend(ssss)?; - k += 1; - } - Ok(()) -} - -fn extract_entropy_segments( - bytes: &[u8], - restart_interval: Option, -) -> Result { - let mut out = Vec::with_capacity(bytes.len()); - let mut restart_offsets = vec![0u32]; - let mut pos = 0usize; - let mut expected_rst = 0xD0u8; - while pos < bytes.len() { - let byte = bytes[pos]; - if byte != 0xFF { - out.push(byte); - pos += 1; - continue; - } - let next = *bytes - .get(pos + 1) - .ok_or(MetalFast420PacketError::TruncatedEntropy)?; - match next { - 0x00 => { - out.push(0xFF); - pos += 2; - } - 0xD9 => { - return Ok(EntropySegments { - entropy_bytes: out, - restart_offsets, - }); - } - 0xD0..=0xD7 if restart_interval.unwrap_or(0) != 0 => { - if next != expected_rst { - return Err(MetalFast420PacketError::EntropyMarkerUnsupported { marker: next }); - } - restart_offsets.push( - u32::try_from(out.len()) - .map_err(|_| MetalFast420PacketError::TruncatedEntropy)?, - ); - expected_rst = if expected_rst == 0xD7 { - 0xD0 - } else { - expected_rst + 1 - }; - pos += 2; - } - marker => { - return Err(MetalFast420PacketError::EntropyMarkerUnsupported { marker }); - } - } - } - Err(MetalFast420PacketError::Decode(JpegError::MissingMarker { - marker: MarkerKind::Eoi, - })) -} diff --git a/crates/signinum-jpeg/src/color/ycbcr.rs b/crates/signinum-jpeg/src/color/ycbcr.rs deleted file mode 100644 index 99af67ca..00000000 --- a/crates/signinum-jpeg/src/color/ycbcr.rs +++ /dev/null @@ -1,82 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! YCbCr → RGB conversion. Scalar implementation uses libjpeg-turbo's 16-bit -//! fixed-point coefficients (`jdcolor.c`), so outputs match their ISLOW path -//! byte-for-byte. -//! -//! Coefficients (all × 2^16, rounded): -//! R = Y + 1.40200 * (Cr - 128) -//! G = Y - 0.34414 * (Cb - 128) - 0.71414 * (Cr - 128) -//! B = Y + 1.77200 * (Cb - 128) - -const FIX_1_40200: i32 = 91_881; // (int)(1.40200 * 65536 + 0.5) -const FIX_0_34414: i32 = 22_554; // (int)(0.34414 * 65536 + 0.5) -const FIX_0_71414: i32 = 46_802; // (int)(0.71414 * 65536 + 0.5) -const FIX_1_77200: i32 = 116_130; // (int)(1.77200 * 65536 + 0.5) -const ROUND: i32 = 1 << 15; // 0.5 in 16-bit fixed point - -const fn clamp_to_u8(v: i32) -> u8 { - if v < 0 { - 0 - } else if v > 255 { - 255 - } else { - v as u8 - } -} - -/// Convert one YCbCr pixel to RGB. `y`, `cb`, `cr` are the 8-bit component -/// values as read from the decoded block after IDCT and upsample. -/// -/// Returns `(R, G, B)` clamped to `[0, 255]`. -pub(crate) fn ycbcr_to_rgb(y: u8, cb: u8, cr: u8) -> (u8, u8, u8) { - let y = y as i32; - let cb_centered = cb as i32 - 128; - let cr_centered = cr as i32 - 128; - let r = y + ((FIX_1_40200 * cr_centered + ROUND) >> 16); - let g = y - ((FIX_0_34414 * cb_centered + FIX_0_71414 * cr_centered + ROUND) >> 16); - let b = y + ((FIX_1_77200 * cb_centered + ROUND) >> 16); - - (clamp_to_u8(r), clamp_to_u8(g), clamp_to_u8(b)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn neutral_gray_roundtrips_to_equal_rgb_channels() { - let (r, g, b) = ycbcr_to_rgb(128, 128, 128); - assert_eq!((r, g, b), (128, 128, 128)); - } - - #[test] - fn bright_red_maps_to_high_r_low_gb() { - // libjpeg-turbo: Y=76 Cb=85 Cr=255 ≈ pure red (255, 0, 0). - let (r, g, b) = ycbcr_to_rgb(76, 85, 255); - assert!(r > 240 && g < 15 && b < 15, "got ({r}, {g}, {b})"); - } - - #[test] - fn clamps_out_of_range_arithmetic_to_0_255() { - // Y=255, large Cr pushes R arithmetic above 255 → saturate high. - let (r, _, _) = ycbcr_to_rgb(255, 128, 255); - assert_eq!(r, 255, "R must saturate at 255"); - // Y=255, large Cb pushes B arithmetic above 255 → saturate high. - let (_, _, b) = ycbcr_to_rgb(255, 255, 128); - assert_eq!(b, 255, "B must saturate at 255"); - // Y=0, small Cr pushes R arithmetic below 0 → saturate low. - let (r, _, _) = ycbcr_to_rgb(0, 128, 0); - assert_eq!(r, 0, "R must saturate at 0"); - } - - #[test] - fn matches_libjpeg_turbo_fixed_point_expectations() { - // Sampled checks against libjpeg-turbo jdcolor.c computed values. - // Y=100 Cb=150 Cr=200 → R=201, G=41, B=139 (16-bit fixed point). - let (r, g, b) = ycbcr_to_rgb(100, 150, 200); - assert!((r as i32 - 201).abs() <= 1, "R={r}, expected ≈201"); - assert!((g as i32 - 41).abs() <= 1, "G={g}, expected ≈41"); - assert!((b as i32 - 139).abs() <= 1, "B={b}, expected ≈139"); - } -} diff --git a/crates/signinum-jpeg/src/decoder.rs b/crates/signinum-jpeg/src/decoder.rs deleted file mode 100644 index 0b67b3af..00000000 --- a/crates/signinum-jpeg/src/decoder.rs +++ /dev/null @@ -1,3145 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Public [`Decoder`] entry points. - -use crate::backend::Backend; -use crate::context::DecoderContext; -use crate::entropy::huffman::HuffmanTable; -use crate::entropy::progressive::{ - decode_progressive, PreparedProgressiveComponentPlan, PreparedProgressivePlan, - PreparedProgressiveScan, PreparedProgressiveScanComponent, -}; -use crate::entropy::sequential::{ - decode_scan_baseline, decode_scan_baseline_rgb, decode_scan_fast_rgb_444, - decode_scan_fast_tile_rgb, decode_scan_fast_tile_rgb_region, - decode_scan_fast_tile_rgb_region_scaled, fast_tile_region_first_decode_mcu, - stripe_region_layout, PreparedComponentPlan, PreparedDecodePlan, -}; -use crate::error::{JpegError, MarkerKind, Warning}; -use crate::info::{ - ColorSpace, DecodeOptions, DownscaleFactor, Info, OutputFormat, Rect, RestartIndex, - RestartSegment, SofKind, -}; -use crate::internal::checkpoint::{checkpoint_before_mcu, CpuCheckpointCache, DeviceCheckpoint}; -use crate::internal::scratch::{ScratchPool, SinkRows}; -use crate::output::{ - validate_buffer, Gray8Writer, InterleavedRgbWriter, OutputWriter, Rgb8Writer, Rgba8Writer, -}; -use crate::parse::header::{parse_header, parse_info, ParsedHeader}; -use crate::parse::tables::{HuffmanValues, RawHuffmanTable}; -use crate::profile::{duration_us_string, emit_jpeg_profile_row, jpeg_profile_stages_enabled}; -use crate::JpegCodec; -use alloc::sync::Arc; -use alloc::vec::Vec; -use core::cell::RefCell; -use core::num::NonZeroUsize; -pub use signinum_core::TileBatchOptions; -use signinum_core::{ - collect_indexed_batch_results, tile_batch_worker_count, CompressedPayloadKind, - CompressedTransferSyntax, DecodeOutcome as CoreDecodeOutcome, DecodeRowsError, - DecoderContext as CoreDecoderContext, Downscale, ImageCodec, ImageDecode, ImageDecodeRows, - IndexedBatchResult, PassthroughCandidate, PixelFormat, RowSink, TileBatchDecode, -}; -use std::sync::Mutex; -use std::time::{Duration, Instant}; - -const DEFAULT_MAX_DECODE_BYTES: usize = 512 * 1024 * 1024; -const CPU_ROI_CHECKPOINT_CADENCE_MCUS: u32 = 1024; -const CPU_ROI_CHECKPOINT_MIN_TARGET_MCUS: u32 = 4096; - -std::thread_local! { - static DEFAULT_SCRATCH: RefCell = RefCell::new(ScratchPool::new()); - static DEFAULT_CONTEXT: RefCell = RefCell::new(DecoderContext::new()); -} - -/// Non-fatal outcome of a successful decode. See spec Section 2. -/// -/// `DecodeOutcome` lives on `decoder.rs` rather than `info.rs` because it -/// carries `Warning` values from `error.rs`, and moving it into `info` would -/// create a `info → error` cycle (see `info.rs` header note). -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DecodeOutcome { - /// The rectangle actually written to the output buffer. For `decode_into` - /// this is always `Rect::full(info.dimensions)`; later milestones add - /// `decode_region_into` which can return a narrower rect. - pub decoded: Rect, - /// Warnings emitted during parse or decode. Empty when the stream is - /// syntactically clean and every capability was exercised without fallback. - pub warnings: Vec, -} - -impl From for CoreDecodeOutcome { - fn from(outcome: DecodeOutcome) -> Self { - Self { - decoded: outcome.decoded.into(), - warnings: outcome.warnings, - } - } -} - -/// One tile decode request for [`decode_tiles_into`]. -pub struct TileDecodeJob<'i, 'o> { - /// Compressed JPEG tile bytes. - pub input: &'i [u8], - /// Caller-owned output buffer for this tile. - pub out: &'o mut [u8], - /// Distance in bytes between output rows. - pub stride: usize, -} - -/// One scaled tile decode request for [`decode_tiles_scaled_into`]. -pub struct TileScaledDecodeJob<'i, 'o> { - /// Compressed JPEG tile bytes. - pub input: &'i [u8], - /// Caller-owned output buffer for this tile. - pub out: &'o mut [u8], - /// Distance in bytes between output rows. - pub stride: usize, - /// Downscale factor applied to the full-tile decode. - pub scale: Downscale, -} - -/// One ROI+scaled tile decode request for -/// [`decode_tiles_region_scaled_into`]. -pub struct TileRegionScaledDecodeJob<'i, 'o> { - /// Compressed JPEG tile bytes. - pub input: &'i [u8], - /// Caller-owned output buffer for this tile. - pub out: &'o mut [u8], - /// Distance in bytes between output rows. - pub stride: usize, - /// Region of interest in source-image coordinates. - pub roi: Rect, - /// Downscale factor applied to the region decode. - pub scale: Downscale, -} - -/// Error returned by [`decode_tiles_into`], annotated with the failing tile -/// index from the caller's input order. -#[derive(Debug)] -pub struct TileBatchError { - /// Index of the first failing tile in input order. - pub index: usize, - /// Decode error reported for that tile. - pub source: JpegError, -} - -impl core::fmt::Display for TileBatchError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "tile {} decode failed: {}", self.index, self.source) - } -} - -impl std::error::Error for TileBatchError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - Some(&self.source) - } -} - -/// Receives decoded component rows before they are packed into the final -/// interleaved pixel format. -pub trait ComponentRowWriter { - /// Receive one grayscale row. - fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError>; - - /// Receive one full-width Y/Cb/Cr row. - fn write_ycbcr_row( - &mut self, - y: u32, - y_row: &[u8], - cb_row: &[u8], - cr_row: &[u8], - ) -> Result<(), JpegError>; - - /// Receive one full-width planar RGB row. - fn write_rgb_row( - &mut self, - y: u32, - r_row: &[u8], - g_row: &[u8], - b_row: &[u8], - ) -> Result<(), JpegError>; -} - -/// A parsed borrowed view of a JPEG stream. -#[derive(Debug)] -pub struct JpegView<'a> { - bytes: &'a [u8], - header: ParsedHeader, - info: Info, - options: DecodeOptions, -} - -impl<'a> JpegView<'a> { - /// Parse the stream into a borrowed view that can later build a decoder. - pub fn parse(input: &'a [u8]) -> Result { - Self::parse_with_options(input, DecodeOptions::default()) - } - - /// Parse the stream with explicit decode options. - pub fn parse_with_options(input: &'a [u8], options: DecodeOptions) -> Result { - let header = parse_header(input)?; - let mut info = header.info(); - options.apply_to_info(&mut info); - Ok(Self { - bytes: input, - header, - info, - options, - }) - } - - /// Header-derived metadata for the parsed stream. - pub fn info(&self) -> &Info { - &self.info - } - - /// Original compressed bytes backing this view. - pub fn bytes(&self) -> &'a [u8] { - self.bytes - } - - /// Return a byte-preserving passthrough candidate for active DICOM/WSI - /// transfer syntaxes. - /// - /// Progressive JPEG is intentionally not exposed here because the active - /// conversion path should transcode it rather than introduce a retired or - /// unsupported destination syntax. - pub fn passthrough_candidate(&self) -> Option> { - jpeg_passthrough_syntax(&self.info).map(|transfer_syntax| { - PassthroughCandidate::new( - self.bytes, - transfer_syntax, - CompressedPayloadKind::JpegInterchange, - self.info.to_core_info(), - ) - }) - } - - /// Build a restart-marker byte-offset index for the first scan. - /// - /// Offsets are absolute byte positions in the original JPEG byte slice. - /// Returns `Ok(None)` when the stream has no non-zero DRI marker. - pub fn restart_index(&self) -> Result, JpegError> { - restart_index_for_stream( - self.bytes, - self.header.sos_offset, - &self.info, - self.info.restart_interval, - ) - } -} - -/// A borrowed view of a JPEG stream ready to decode. Constructed via -/// [`Decoder::new`] or [`Decoder::from_view`]. `Decoder<'a>: Send + Sync`. -#[derive(Debug)] -pub struct Decoder<'a> { - pub(crate) bytes: &'a [u8], - pub(crate) info: Info, - pub(crate) warnings: Arc<[Warning]>, - pub(crate) backend: Backend, - pub(crate) plan: PreparedDecodePlan, - pub(crate) progressive_plan: Option, - pub(crate) cpu_entropy_checkpoints: Mutex, -} - -impl<'a> Decoder<'a> { - /// Parse the headers without decoding pixels. The parser walks headers up - /// to the first SOS and then performs a lightweight marker scan so - /// `Info::scan_count` reflects all scans in the file. - /// - /// # Errors - /// Returns any structural, unsupported-SOF, or sanity-check error - /// encountered before the Start-of-Scan marker. See [`JpegError`]. - pub fn inspect(input: &'a [u8]) -> Result { - Self::inspect_with_options(input, DecodeOptions::default()) - } - - /// Parse headers with explicit decode options, without decoding pixels. - /// - /// The options are applied to the returned [`Info`] exactly as they would - /// be for [`Self::new_with_options`]. - pub fn inspect_with_options( - input: &'a [u8], - options: DecodeOptions, - ) -> Result { - let mut info = parse_info(input)?; - options.apply_to_info(&mut info); - Ok(info) - } - - /// Build a decoder with explicit decode options. - pub fn new_with_options(input: &'a [u8], options: DecodeOptions) -> Result { - let view = JpegView::parse_with_options(input, options)?; - DEFAULT_CONTEXT.with(|ctx| Self::from_view_in_context(view, &mut ctx.borrow_mut())) - } - - /// Build a decoder ready for `decode_into`. Parses the full header, pre- - /// builds every referenced Huffman table, and validates that the stream is - /// one of the SOFs this release implements. - /// - /// # Errors - /// - Any parse error encountered before SOS (see [`Self::inspect`]). - /// - [`JpegError::NotImplemented`] for SOFs that parse but are not yet - /// decodable (Extended12, Progressive12, Lossless — all land in M3). - /// - [`JpegError::MissingHuffmanTable`] if the scan references a DC/AC - /// table slot that was never defined by a DHT segment. - pub fn new(input: &'a [u8]) -> Result { - Self::new_with_options(input, DecodeOptions::default()) - } - - /// Build a decoder from a previously parsed [`JpegView`]. - pub fn from_view(view: JpegView<'a>) -> Result { - DEFAULT_CONTEXT.with(|ctx| Self::from_view_in_context(view, &mut ctx.borrow_mut())) - } - - /// Build a decoder from a previously parsed [`JpegView`], reusing shared - /// compiled DHT/DQT state from `ctx` when table contents repeat. - pub fn from_view_in_context( - view: JpegView<'a>, - ctx: &mut DecoderContext, - ) -> Result { - let JpegView { - bytes, - header, - info, - options, - } = view; - let backend = Backend::detect(); - let (info, warnings, plan, progressive_plan) = if info.sof_kind == SofKind::Progressive8 { - let progressive_plan = Self::build_progressive_plan(&header, &info, ctx)?; - let plan = Self::build_progressive_placeholder_plan(&header, &info, ctx)?; - ( - info, - Arc::<[Warning]>::from(header.warnings.as_slice()), - plan, - Some(progressive_plan), - ) - } else if options == DecodeOptions::default() { - if let Some(scan_offset) = header.sos_offset { - let header_prefix = &bytes[..scan_offset]; - let (info, warnings, plan) = ctx.resolve_decode_plan(header_prefix, |ctx| { - let plan = Self::build_prepared_plan(&header, &info, ctx)?; - Ok(( - info.clone(), - Arc::<[Warning]>::from(header.warnings.as_slice()), - plan, - )) - })?; - (info, warnings, plan, None) - } else { - let plan = Self::build_prepared_plan(&header, &info, ctx)?; - ( - info, - Arc::<[Warning]>::from(header.warnings.as_slice()), - plan, - None, - ) - } - } else { - let plan = Self::build_prepared_plan(&header, &info, ctx)?; - ( - info, - Arc::<[Warning]>::from(header.warnings.as_slice()), - plan, - None, - ) - }; - Ok(Self { - bytes, - info, - warnings, - backend, - plan, - progressive_plan, - cpu_entropy_checkpoints: Mutex::new(CpuCheckpointCache::default()), - }) - } - - fn build_prepared_plan( - header: &ParsedHeader, - info: &Info, - ctx: &mut DecoderContext, - ) -> Result { - match info.sof_kind { - SofKind::Baseline8 | SofKind::Extended8 => {} - other => return Err(JpegError::NotImplemented { sof: other }), - } - match info.color_space { - ColorSpace::Grayscale | ColorSpace::YCbCr | ColorSpace::Rgb => {} - color_space => return Err(JpegError::UnsupportedColorSpace { color_space }), - } - - let mut dc_tables: [Option>; 4] = Default::default(); - let mut ac_tables: [Option>; 4] = Default::default(); - let scan = header.scan.as_ref().ok_or(JpegError::MissingMarker { - marker: MarkerKind::Sos, - })?; - if header.scan_count != 1 { - return Err(JpegError::InvalidSequentialScanCount { - sof: info.sof_kind, - count: header.scan_count, - }); - } - // M1b requires the first component (Y for 3-component, single for - // grayscale) to be the maximally-sampled component. Non-luma-leading - // layouts are pathological; real baselines always satisfy this. - if let Some((h, v)) = header.sampling.component(0) { - if h != header.sampling.max_h || v != header.sampling.max_v { - return Err(JpegError::NotImplemented { sof: info.sof_kind }); - } - } - // Every component must declare H,V in 1..=4 per T.81 §B.2.2, and max_h - // must actually divide every component's H (same for V). Malformed - // streams can set H=0 (div-by-zero in upsample ratio), non-divisors - // (arbitrary ratios M2 handles), or ratios that don't produce planes - // that cover the image width. - for (h, v) in header.sampling.iter() { - if h == 0 || v == 0 || h > 4 || v > 4 { - return Err(JpegError::NotImplemented { sof: info.sof_kind }); - } - if !header.sampling.max_h.is_multiple_of(h) || !header.sampling.max_v.is_multiple_of(v) - { - return Err(JpegError::NotImplemented { sof: info.sof_kind }); - } - } - for comp in &scan.components { - let di = comp.dc_table as usize; - let ai = comp.ac_table as usize; - if dc_tables[di].is_none() { - let raw = header.huffman_tables.dc[di].as_ref().ok_or( - JpegError::MissingHuffmanTable { - component: comp.id, - class: 0, - id: comp.dc_table, - }, - )?; - dc_tables[di] = Some(ctx.resolve_huffman_table(raw)?); - } - if ac_tables[ai].is_none() { - let raw = header.huffman_tables.ac[ai].as_ref().ok_or( - JpegError::MissingHuffmanTable { - component: comp.id, - class: 1, - id: comp.ac_table, - }, - )?; - ac_tables[ai] = Some(ctx.resolve_huffman_table(raw)?); - } - } - - build_decode_plan(header, info, &dc_tables, &ac_tables, ctx) - } - - fn build_progressive_plan( - header: &ParsedHeader, - info: &Info, - ctx: &mut DecoderContext, - ) -> Result { - if info.sof_kind != SofKind::Progressive8 { - return Err(JpegError::NotImplemented { sof: info.sof_kind }); - } - match info.color_space { - ColorSpace::Grayscale | ColorSpace::YCbCr | ColorSpace::Rgb => {} - color_space => return Err(JpegError::UnsupportedColorSpace { color_space }), - } - validate_sampling_factors(header, info)?; - if header.progressive_scans.is_empty() { - return Err(JpegError::MissingMarker { - marker: MarkerKind::Sos, - }); - } - - let max_h = u32::from(header.sampling.max_h); - let max_v = u32::from(header.sampling.max_v); - let mcu_cols = info.dimensions.0.div_ceil(8 * max_h); - let mcu_rows = info.dimensions.1.div_ceil(8 * max_v); - let mut components = Vec::with_capacity(header.component_ids.len()); - for (output_index, &id) in header.component_ids.iter().enumerate() { - let (h, v) = - header - .sampling - .component(output_index) - .ok_or(JpegError::MissingMarker { - marker: MarkerKind::Sof, - })?; - let quant_id = - *header - .quant_table_ids - .get(output_index) - .ok_or(JpegError::MissingMarker { - marker: MarkerKind::Sof, - })? as usize; - let quant = *header - .quant_tables - .entries - .get(quant_id) - .and_then(|q| q.as_ref()) - .ok_or(JpegError::MissingQuantTable { - component: id, - table_id: quant_id as u8, - })?; - components.push(PreparedProgressiveComponentPlan { - h, - v, - output_index, - quant: ctx.resolve_quant_table(quant), - block_cols: mcu_cols * u32::from(h), - block_rows: mcu_rows * u32::from(v), - sample_width: info - .dimensions - .0 - .saturating_mul(u32::from(h)) - .div_ceil(max_h), - sample_height: info - .dimensions - .1 - .saturating_mul(u32::from(v)) - .div_ceil(max_v), - }); - } - - let mut scans = Vec::with_capacity(header.progressive_scans.len()); - for parsed in &header.progressive_scans { - let mut scan_components = Vec::with_capacity(parsed.scan.components.len()); - for component in &parsed.scan.components { - let component_index = find_component_index(&header.component_ids, component.id) - .ok_or(JpegError::UnknownScanComponent { - offset: parsed.entropy_offset, - component: component.id, - })?; - let quant_id = *header.quant_table_ids.get(component_index).ok_or( - JpegError::MissingMarker { - marker: MarkerKind::Sof, - }, - )?; - let _ = parsed - .quant_tables - .entries - .get(quant_id as usize) - .and_then(|q| q.as_ref()) - .ok_or(JpegError::MissingQuantTable { - component: component.id, - table_id: quant_id, - })?; - let dc_table = if parsed.scan.ss == 0 { - Some(resolve_progressive_huffman( - ctx, - &parsed.huffman_tables.dc, - component.id, - 0, - component.dc_table, - )?) - } else { - None - }; - let ac_table = if parsed.scan.ss > 0 { - Some(resolve_progressive_huffman( - ctx, - &parsed.huffman_tables.ac, - component.id, - 1, - component.ac_table, - )?) - } else { - None - }; - scan_components.push(PreparedProgressiveScanComponent { - component_index, - dc_table, - ac_table, - }); - } - scans.push(PreparedProgressiveScan { - components: scan_components, - ss: parsed.scan.ss, - se: parsed.scan.se, - ah: parsed.scan.ah, - al: parsed.scan.al, - entropy_offset: parsed.entropy_offset, - restart_interval: parsed.restart_interval, - }); - } - - let scratch_bytes = - compute_progressive_scratch_bytes(&components, info.dimensions.0 as usize)?; - Ok(PreparedProgressivePlan { - components, - scans, - sampling: info.sampling, - color_space: info.color_space, - dimensions: info.dimensions, - mcu_cols, - mcu_rows, - scratch_bytes, - }) - } - - fn build_progressive_placeholder_plan( - header: &ParsedHeader, - info: &Info, - ctx: &mut DecoderContext, - ) -> Result { - let empty_raw = RawHuffmanTable { - bits: [0; 16], - values: HuffmanValues::default(), - }; - let empty_huffman = ctx.resolve_huffman_table(&empty_raw)?; - let mut components = Vec::with_capacity(header.component_ids.len()); - for (output_index, &id) in header.component_ids.iter().enumerate() { - let (h, v) = - header - .sampling - .component(output_index) - .ok_or(JpegError::MissingMarker { - marker: MarkerKind::Sof, - })?; - let quant_id = - *header - .quant_table_ids - .get(output_index) - .ok_or(JpegError::MissingMarker { - marker: MarkerKind::Sof, - })? as usize; - let quant = *header - .quant_tables - .entries - .get(quant_id) - .and_then(|q| q.as_ref()) - .ok_or(JpegError::MissingQuantTable { - component: id, - table_id: quant_id as u8, - })?; - components.push(PreparedComponentPlan { - h, - v, - output_index, - quant: ctx.resolve_quant_table(quant), - dc_table: Arc::clone(&empty_huffman), - ac_table: Arc::clone(&empty_huffman), - }); - } - Ok(PreparedDecodePlan { - components, - sampling: info.sampling, - color_space: info.color_space, - restart_interval: header.restart_interval, - dimensions: info.dimensions, - scan_offset: header.sos_offset.ok_or(JpegError::MissingMarker { - marker: MarkerKind::Sos, - })?, - scratch_bytes: compute_decode_scratch_bytes( - info.dimensions, - info.sampling, - DEFAULT_MAX_DECODE_BYTES, - )?, - }) - } - - /// The parsed header as a public [`Info`]. - pub fn info(&self) -> &Info { - &self.info - } - - /// Return a byte-preserving passthrough candidate for this decoded stream. - pub fn passthrough_candidate(&self) -> Option> { - jpeg_passthrough_syntax(&self.info).map(|transfer_syntax| { - PassthroughCandidate::new( - self.bytes, - transfer_syntax, - CompressedPayloadKind::JpegInterchange, - self.info.to_core_info(), - ) - }) - } - - /// Build a restart-marker byte-offset index for the first scan. - /// - /// Offsets are absolute byte positions in the original JPEG byte slice. - /// Returns `Ok(None)` when the stream has no non-zero DRI marker. - pub fn restart_index(&self) -> Result, JpegError> { - restart_index_for_stream( - self.bytes, - Some(self.plan.scan_offset), - &self.info, - self.plan.restart_interval, - ) - } - - /// Decode the full image into the caller's buffer. - /// - /// # Errors - /// - [`JpegError::OutputBufferTooSmall`] or [`JpegError::InvalidStride`] - /// if the provided buffer/stride cannot hold the image at `fmt`. - /// - [`JpegError::NotImplemented`] if `fmt` requests a raw output the - /// current release does not emit (e.g. `RawYCbCr8`). - /// - Any entropy- or structural-decode error from the scan walker. - pub fn decode_into( - &self, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - ) -> Result { - DEFAULT_SCRATCH - .with(|pool| self.decode_into_with_scratch(&mut pool.borrow_mut(), out, stride, fmt)) - } - - /// Decode the full image into a freshly allocated tightly-packed buffer. - /// - /// This is the owned-output counterpart to [`Self::decode_into`]. It - /// avoids pre-zeroing the destination buffer, which matters on WSI-sized - /// RGB outputs where the allocation itself can otherwise dominate the - /// benchmark. - pub fn decode(&self, fmt: PixelFormat) -> Result<(Vec, DecodeOutcome), JpegError> { - DEFAULT_SCRATCH.with(|pool| self.decode_with_scratch(&mut pool.borrow_mut(), fmt)) - } - - /// Decode the full image into the caller's buffer using the core - /// `PixelFormat` + `Downscale` contract. - pub fn decode_scaled_into( - &self, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - scale: Downscale, - ) -> Result { - DEFAULT_SCRATCH.with(|pool| { - self.decode_scaled_into_with_scratch(&mut pool.borrow_mut(), out, stride, fmt, scale) - }) - } - - /// Decode the full image into a freshly allocated tightly-packed buffer - /// using the core `PixelFormat` + `Downscale` contract. - pub fn decode_scaled( - &self, - fmt: PixelFormat, - scale: Downscale, - ) -> Result<(Vec, DecodeOutcome), JpegError> { - DEFAULT_SCRATCH - .with(|pool| self.decode_scaled_with_scratch(&mut pool.borrow_mut(), fmt, scale)) - } - - /// [`Self::decode`] with caller-owned scratch. - pub fn decode_with_scratch( - &self, - pool: &mut ScratchPool, - fmt: PixelFormat, - ) -> Result<(Vec, DecodeOutcome), JpegError> { - self.decode_scaled_with_scratch(pool, fmt, Downscale::None) - } - - /// [`Self::decode_scaled`] with caller-owned scratch. - pub fn decode_scaled_with_scratch( - &self, - pool: &mut ScratchPool, - fmt: PixelFormat, - scale: Downscale, - ) -> Result<(Vec, DecodeOutcome), JpegError> { - let legacy = output_format_from_parts(self.info.sof_kind, fmt, scale)?; - let downscale = legacy.downscale(); - let (width, height) = scaled_dimensions(self.info.dimensions, downscale); - let stride = width as usize * legacy.bytes_per_pixel(); - let len = stride * height as usize; - let mut out = allocate_output_buffer(len); - let outcome = self.decode_scaled_into_with_scratch(pool, &mut out, stride, fmt, scale)?; - Ok((out, outcome)) - } - - /// Decode the full image into the caller's buffer, reusing the supplied - /// [`ScratchPool`]. On a long-running tile batch this eliminates the - /// per-tile allocation of stripe buffers, the DC predictor, and the - /// chroma upsample rows — the realistic WSI reader shape. The first - /// call against a fresh pool does the allocation; subsequent calls at - /// the same-or-smaller shape reuse the underlying `Vec`s. - /// - /// # Errors - /// Identical to [`Self::decode_into`]. - pub fn decode_into_with_scratch( - &self, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - ) -> Result { - self.decode_scaled_into_with_scratch(pool, out, stride, fmt, Downscale::None) - } - - fn decode_into_output_format_with_scratch( - &self, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: OutputFormat, - ) -> Result { - let profile_enabled = jpeg_profile_stages_enabled(); - let total_start = profile_enabled.then(Instant::now); - let downscale = fmt.downscale(); - let (w, h) = scaled_dimensions(self.info.dimensions, downscale); - let scratch_bytes = self.decode_scratch_bytes(DEFAULT_MAX_DECODE_BYTES)?; - let bpp = fmt.bytes_per_pixel(); - validate_buffer(out, stride, w, h, bpp)?; - let decode_start = profile_enabled.then(Instant::now); - let result = match fmt { - OutputFormat::Rgb8 | OutputFormat::Rgb8Scaled { .. } => { - let mut writer = Rgb8Writer::new(out, stride, w); - self.decode_rgb_with_writer( - pool, - &mut writer, - downscale, - Rect::full(self.info.dimensions), - ) - } - OutputFormat::Rgba8 { alpha } => { - let mut writer = Rgba8Writer::new(out, stride, w, alpha); - self.decode_with_writer( - pool, - &mut writer, - downscale, - Rect::full(self.info.dimensions), - ) - } - OutputFormat::Gray8 | OutputFormat::Gray8Scaled { .. } => { - let mut writer = Gray8Writer::new(out, stride, w); - self.decode_with_writer( - pool, - &mut writer, - downscale, - Rect::full(self.info.dimensions), - ) - } - }; - if let (Some(total_start), Some(decode_start), Ok(outcome)) = - (total_start, decode_start, &result) - { - let source_width_s = self.info.dimensions.0.to_string(); - let source_height_s = self.info.dimensions.1.to_string(); - let output_width_s = w.to_string(); - let output_height_s = h.to_string(); - let stride_s = stride.to_string(); - let bpp_s = bpp.to_string(); - let output_bytes_s = stride.saturating_mul(h as usize).to_string(); - let scratch_bytes_s = scratch_bytes.to_string(); - let warning_count_s = outcome.warnings.len().to_string(); - let decode_us = duration_us_string(decode_start.elapsed()); - let total_us = duration_us_string(total_start.elapsed()); - emit_jpeg_profile_row( - "decode", - "cpu", - &[ - ("mode", "full"), - ("fmt", output_format_profile_name(fmt)), - ("downscale", downscale_profile_name(downscale)), - ("source_width", source_width_s.as_str()), - ("source_height", source_height_s.as_str()), - ("output_width", output_width_s.as_str()), - ("output_height", output_height_s.as_str()), - ("stride", stride_s.as_str()), - ("bpp", bpp_s.as_str()), - ("scratch_bytes", scratch_bytes_s.as_str()), - ("output_bytes", output_bytes_s.as_str()), - ("decode_us", decode_us.as_str()), - ("total_us", total_us.as_str()), - ("warnings", warning_count_s.as_str()), - ], - ); - } - result - } - - /// [`Self::decode_scaled_into`] with caller-owned scratch. - pub fn decode_scaled_into_with_scratch( - &self, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - scale: Downscale, - ) -> Result { - self.decode_into_output_format_with_scratch( - pool, - out, - stride, - output_format_from_parts(self.info.sof_kind, fmt, scale)?, - ) - } - - /// Decode the full image into interleaved RGB rows delivered to `sink`. - pub fn decode_rows(&self, sink: &mut S) -> Result - where - S: RowSink, - { - DEFAULT_SCRATCH.with(|pool| self.decode_rows_with_scratch(&mut pool.borrow_mut(), sink)) - } - - /// [`Self::decode_rows`] with caller-owned scratch. See - /// [`Self::decode_into_with_scratch`] for the reuse contract. - pub fn decode_rows_with_scratch( - &self, - pool: &mut ScratchPool, - sink: &mut S, - ) -> Result - where - S: RowSink, - { - let width = self.info.dimensions.0 as usize; - let rows = pool.take_sink_rows(width); - let mut writer = SinkWriter::new(sink, rows, self.backend); - let result = self.decode_rgb_with_writer( - pool, - &mut writer, - DownscaleFactor::Full, - Rect::full(self.info.dimensions), - ); - pool.restore_sink_rows(writer.into_rows()); - result - } - - /// Decode the full image into component rows. - pub fn decode_component_rows_with_scratch( - &self, - pool: &mut ScratchPool, - writer: &mut W, - ) -> Result - where - W: ComponentRowWriter, - { - self.decode_region_component_rows_with_scratch( - pool, - writer, - Rect::full(self.info.dimensions), - Downscale::None, - ) - } - - /// Decode `roi` into component rows, optionally at a reduced scale. - pub fn decode_region_component_rows_with_scratch( - &self, - pool: &mut ScratchPool, - writer: &mut W, - roi: Rect, - scale: Downscale, - ) -> Result - where - W: ComponentRowWriter, - { - if !roi.is_within(self.info.dimensions) { - return Err(JpegError::RectOutOfBounds { - rect: roi, - width: self.info.dimensions.0, - height: self.info.dimensions.1, - }); - } - - let downscale = jpeg_downscale(scale); - let scaled_roi = scaled_rect_covering(roi, downscale)?; - let mut adapter = ComponentWriterAdapter { inner: writer }; - - if roi == Rect::full(self.info.dimensions) { - self.decode_with_writer(pool, &mut adapter, downscale, roi) - } else { - let (source_x0, source_width) = - self.source_window_for_output_rect(downscale, scaled_roi); - let mut cropped = CroppedWriter::new(adapter, scaled_roi, source_x0, source_width); - self.decode_with_writer(pool, &mut cropped, downscale, roi) - } - } - - /// Decode a rectangular region of the image into the caller's buffer. - /// - /// `roi` is expressed in source-image coordinates. If `fmt` requests a - /// downscaled output, the written pixels cover the corresponding bounding - /// box in the scaled image grid. - pub fn decode_region_into( - &self, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - ) -> Result { - DEFAULT_SCRATCH.with(|pool| { - self.decode_region_into_with_scratch(&mut pool.borrow_mut(), out, stride, fmt, roi) - }) - } - - /// Decode `roi` into a freshly allocated tightly-packed buffer. - pub fn decode_region( - &self, - fmt: PixelFormat, - roi: Rect, - ) -> Result<(Vec, DecodeOutcome), JpegError> { - DEFAULT_SCRATCH - .with(|pool| self.decode_region_with_scratch(&mut pool.borrow_mut(), fmt, roi)) - } - - /// Decode `roi` into a freshly allocated tightly-packed buffer using the - /// core `PixelFormat` + `Downscale` contract. - pub fn decode_region_scaled( - &self, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - ) -> Result<(Vec, DecodeOutcome), JpegError> { - DEFAULT_SCRATCH.with(|pool| { - self.decode_region_scaled_with_scratch(&mut pool.borrow_mut(), fmt, roi, scale) - }) - } - - /// [`Self::decode_region`] with caller-owned scratch. - pub fn decode_region_with_scratch( - &self, - pool: &mut ScratchPool, - fmt: PixelFormat, - roi: Rect, - ) -> Result<(Vec, DecodeOutcome), JpegError> { - self.decode_region_scaled_with_scratch(pool, fmt, roi, Downscale::None) - } - - /// [`Self::decode_region_scaled`] with caller-owned scratch. - pub fn decode_region_scaled_with_scratch( - &self, - pool: &mut ScratchPool, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - ) -> Result<(Vec, DecodeOutcome), JpegError> { - let legacy = output_format_from_parts(self.info.sof_kind, fmt, scale)?; - let scaled_roi = scaled_rect_covering(roi, legacy.downscale())?; - let stride = scaled_roi.w as usize * legacy.bytes_per_pixel(); - let len = stride * scaled_roi.h as usize; - let mut out = allocate_output_buffer(len); - let outcome = - self.decode_region_scaled_into_with_scratch(pool, &mut out, stride, fmt, roi, scale)?; - Ok((out, outcome)) - } - - /// [`Self::decode_region_into`] with caller-owned scratch. - pub fn decode_region_into_with_scratch( - &self, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - ) -> Result { - self.decode_region_scaled_into_with_scratch(pool, out, stride, fmt, roi, Downscale::None) - } - - fn decode_region_into_output_format_with_scratch( - &self, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: OutputFormat, - roi: Rect, - ) -> Result { - let profile_enabled = jpeg_profile_stages_enabled(); - let total_start = profile_enabled.then(Instant::now); - if !roi.is_within(self.info.dimensions) { - return Err(JpegError::RectOutOfBounds { - rect: roi, - width: self.info.dimensions.0, - height: self.info.dimensions.1, - }); - } - - if roi == Rect::full(self.info.dimensions) { - return self.decode_into_output_format_with_scratch(pool, out, stride, fmt); - } - - let downscale = fmt.downscale(); - let scaled_roi = scaled_rect_covering(roi, downscale)?; - let scratch_bytes = self.decode_scratch_bytes(DEFAULT_MAX_DECODE_BYTES)?; - validate_buffer( - out, - stride, - scaled_roi.w, - scaled_roi.h, - fmt.bytes_per_pixel(), - )?; - - let decode_start = profile_enabled.then(Instant::now); - let result = match fmt { - OutputFormat::Rgb8 | OutputFormat::Rgb8Scaled { .. } => { - if fmt == OutputFormat::Rgb8 - && downscale == DownscaleFactor::Full - && self.plan.matches_fast_tile_shape() - { - let mut writer = Rgb8Writer::new(out, stride, scaled_roi.w); - let scan_bytes = &self.bytes[self.plan.scan_offset..]; - let checkpoint = self.checkpoint_for_mcu( - scan_bytes, - fast_tile_region_first_decode_mcu(&self.plan, roi, DownscaleFactor::Full), - )?; - let scan_warnings = decode_scan_fast_tile_rgb_region( - &self.plan, - self.backend, - scan_bytes, - pool, - &mut writer, - roi, - checkpoint.as_ref(), - )?; - Ok(DecodeOutcome { - decoded: roi, - warnings: merged_warnings(&self.warnings, scan_warnings), - }) - } else if matches!(fmt, OutputFormat::Rgb8Scaled { .. }) - && self.plan.matches_fast_tile_shape() - { - let mut writer = Rgb8Writer::new(out, stride, scaled_roi.w); - let scan_bytes = &self.bytes[self.plan.scan_offset..]; - let checkpoint = self.checkpoint_for_mcu( - scan_bytes, - fast_tile_region_first_decode_mcu(&self.plan, scaled_roi, downscale), - )?; - let scan_warnings = decode_scan_fast_tile_rgb_region_scaled( - &self.plan, - self.backend, - scan_bytes, - pool, - &mut writer, - scaled_roi, - downscale, - checkpoint.as_ref(), - )?; - Ok(DecodeOutcome { - decoded: scaled_roi, - warnings: merged_warnings(&self.warnings, scan_warnings), - }) - } else { - let base = Rgb8Writer::new(out, stride, scaled_roi.w); - let (source_x0, source_width) = - self.source_window_for_output_rect(downscale, scaled_roi); - let mut writer = CroppedWriter::new(base, scaled_roi, source_x0, source_width); - self.decode_rgb_with_writer(pool, &mut writer, downscale, roi) - } - } - OutputFormat::Rgba8 { alpha } => { - let base = Rgba8Writer::new(out, stride, scaled_roi.w, alpha); - let (source_x0, source_width) = - self.source_window_for_output_rect(downscale, scaled_roi); - let mut writer = CroppedWriter::new(base, scaled_roi, source_x0, source_width); - self.decode_with_writer(pool, &mut writer, downscale, roi) - } - OutputFormat::Gray8 | OutputFormat::Gray8Scaled { .. } => { - let base = Gray8Writer::new(out, stride, scaled_roi.w); - let (source_x0, source_width) = - self.source_window_for_output_rect(downscale, scaled_roi); - let mut writer = CroppedWriter::new(base, scaled_roi, source_x0, source_width); - self.decode_with_writer(pool, &mut writer, downscale, roi) - } - }; - if let (Some(total_start), Some(decode_start), Ok(outcome)) = - (total_start, decode_start, &result) - { - let source_width_s = self.info.dimensions.0.to_string(); - let source_height_s = self.info.dimensions.1.to_string(); - let roi_x_s = roi.x.to_string(); - let roi_y_s = roi.y.to_string(); - let roi_w_s = roi.w.to_string(); - let roi_h_s = roi.h.to_string(); - let output_width_s = scaled_roi.w.to_string(); - let output_height_s = scaled_roi.h.to_string(); - let stride_s = stride.to_string(); - let bpp_s = fmt.bytes_per_pixel().to_string(); - let output_bytes_s = stride.saturating_mul(scaled_roi.h as usize).to_string(); - let scratch_bytes_s = scratch_bytes.to_string(); - let warning_count_s = outcome.warnings.len().to_string(); - let decode_us = duration_us_string(decode_start.elapsed()); - let total_us = duration_us_string(total_start.elapsed()); - let mode = if downscale == DownscaleFactor::Full { - "region" - } else { - "region_scaled" - }; - emit_jpeg_profile_row( - "decode", - "cpu", - &[ - ("mode", mode), - ("fmt", output_format_profile_name(fmt)), - ("downscale", downscale_profile_name(downscale)), - ("source_width", source_width_s.as_str()), - ("source_height", source_height_s.as_str()), - ("roi_x", roi_x_s.as_str()), - ("roi_y", roi_y_s.as_str()), - ("roi_w", roi_w_s.as_str()), - ("roi_h", roi_h_s.as_str()), - ("output_width", output_width_s.as_str()), - ("output_height", output_height_s.as_str()), - ("stride", stride_s.as_str()), - ("bpp", bpp_s.as_str()), - ("scratch_bytes", scratch_bytes_s.as_str()), - ("output_bytes", output_bytes_s.as_str()), - ("decode_us", decode_us.as_str()), - ("total_us", total_us.as_str()), - ("warnings", warning_count_s.as_str()), - ], - ); - } - result - } - - /// Decode `roi` into the caller's buffer using the core `PixelFormat` + - /// `Downscale` contract. - pub fn decode_region_scaled_into( - &self, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - ) -> Result { - DEFAULT_SCRATCH.with(|pool| { - self.decode_region_scaled_into_with_scratch( - &mut pool.borrow_mut(), - out, - stride, - fmt, - roi, - scale, - ) - }) - } - - /// [`Self::decode_region_scaled_into`] with caller-owned scratch. - pub fn decode_region_scaled_into_with_scratch( - &self, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - ) -> Result { - self.decode_region_into_output_format_with_scratch( - pool, - out, - stride, - output_format_from_parts(self.info.sof_kind, fmt, scale)?, - roi, - ) - } - - /// Decode the full image into RGBA with a caller-chosen alpha byte. - pub fn decode_rgba8_into_with_alpha( - &self, - out: &mut [u8], - stride: usize, - alpha: u8, - ) -> Result { - DEFAULT_SCRATCH.with(|pool| { - self.decode_rgba8_into_with_alpha_with_scratch( - &mut pool.borrow_mut(), - out, - stride, - alpha, - ) - }) - } - - /// [`Self::decode_rgba8_into_with_alpha`] with caller-owned scratch. - pub fn decode_rgba8_into_with_alpha_with_scratch( - &self, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - alpha: u8, - ) -> Result { - self.decode_into_output_format_with_scratch( - pool, - out, - stride, - OutputFormat::Rgba8 { alpha }, - ) - } - - /// Decode a region into RGBA with a caller-chosen alpha byte. - pub fn decode_region_rgba8_into_with_alpha( - &self, - out: &mut [u8], - stride: usize, - roi: Rect, - alpha: u8, - ) -> Result { - DEFAULT_SCRATCH.with(|pool| { - self.decode_region_rgba8_into_with_alpha_with_scratch( - &mut pool.borrow_mut(), - out, - stride, - roi, - alpha, - ) - }) - } - - /// [`Self::decode_region_rgba8_into_with_alpha`] with caller-owned scratch. - pub fn decode_region_rgba8_into_with_alpha_with_scratch( - &self, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - roi: Rect, - alpha: u8, - ) -> Result { - self.decode_region_into_output_format_with_scratch( - pool, - out, - stride, - OutputFormat::Rgba8 { alpha }, - roi, - ) - } -} - -/// One-shot parse-plus-decode of an independent JPEG tile into the caller's -/// buffer, reusing a pre-allocated [`ScratchPool`]. This is the primitive -/// WSI tile-batch readers want: one function call per tile, with all -/// heap state external. -/// -/// Parallelism is the caller's responsibility for this primitive. For -/// production batch decode, use [`decode_tiles_into`]. -/// -/// # Example -/// -/// ```no_run -/// use signinum_jpeg::{decode_tile_into, PixelFormat, ScratchPool}; -/// -/// let bytes: &[u8] = todo!("read tile bytes"); -/// let mut out = vec![0u8; 256 * 256 * 3]; -/// let mut pool = ScratchPool::new(); -/// decode_tile_into(bytes, &mut pool, &mut out, 256 * 3, PixelFormat::Rgb8)?; -/// # Ok::<(), signinum_jpeg::JpegError>(()) -/// ``` -/// -/// # Errors -/// Forwarded from [`Decoder::new`] (parse) and -/// [`Decoder::decode_into_with_scratch`] (decode). -pub fn decode_tile_into( - bytes: &[u8], - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, -) -> Result { - DEFAULT_CONTEXT.with(|ctx| { - decode_tile_into_in_context(bytes, &mut ctx.borrow_mut(), pool, out, stride, fmt) - }) -} - -/// One-shot parse-plus-decode of an independent JPEG tile into the caller's -/// buffer, reusing both caller-owned [`DecoderContext`] and caller-owned -/// [`ScratchPool`]. -pub fn decode_tile_into_in_context( - bytes: &[u8], - ctx: &mut DecoderContext, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, -) -> Result { - decode_tile_into_in_context_with_options( - bytes, - ctx, - pool, - out, - stride, - fmt, - DecodeOptions::default(), - ) -} - -/// One-shot parse-plus-decode of an independent JPEG tile into the caller's -/// buffer, reusing both caller-owned [`DecoderContext`] and caller-owned -/// [`ScratchPool`], with explicit JPEG decode options. -pub fn decode_tile_into_in_context_with_options( - bytes: &[u8], - ctx: &mut DecoderContext, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - options: DecodeOptions, -) -> Result { - let dec = Decoder::from_view_in_context(JpegView::parse_with_options(bytes, options)?, ctx)?; - dec.decode_into_with_scratch(pool, out, stride, fmt) -} - -/// Decode independent JPEG tiles into caller-owned output buffers using a -/// scoped CPU worker pool. -/// -/// Each worker owns one [`DecoderContext`] and one [`ScratchPool`], so repeated -/// tiles reuse parsed table state and heap scratch within that worker without -/// sharing mutable decoder state across threads. Returned outcomes preserve -/// the caller's input order. -/// -/// # Errors -/// Returns [`TileBatchError`] with the first failing tile index in input order. -pub fn decode_tiles_into( - jobs: &mut [TileDecodeJob<'_, '_>], - fmt: PixelFormat, - options: TileBatchOptions, -) -> Result, TileBatchError> { - decode_tiles_into_with_options(jobs, fmt, DecodeOptions::default(), options) -} - -/// Decode independent JPEG tiles into caller-owned output buffers using a -/// scoped CPU worker pool and explicit JPEG decode options. -/// -/// Use this variant when container metadata has already resolved ambiguous -/// three-component JPEG data to RGB or YCbCr via [`DecodeOptions`]. -/// -/// # Errors -/// Returns [`TileBatchError`] with the first failing tile index in input order. -pub fn decode_tiles_into_with_options( - jobs: &mut [TileDecodeJob<'_, '_>], - fmt: PixelFormat, - decode_options: DecodeOptions, - options: TileBatchOptions, -) -> Result, TileBatchError> { - if jobs.is_empty() { - return Ok(Vec::new()); - } - - let job_count = jobs.len(); - let worker_count = tile_batch_worker_count(job_count, options, available_tile_batch_workers()); - let chunk_size = job_count.div_ceil(worker_count); - let results = std::thread::scope(|scope| { - let mut handles = Vec::with_capacity(worker_count); - for (chunk_index, chunk) in jobs.chunks_mut(chunk_size).enumerate() { - let start_index = chunk_index * chunk_size; - handles.push( - scope.spawn(move || decode_tile_job_chunk(start_index, chunk, fmt, decode_options)), - ); - } - - let mut results = Vec::with_capacity(job_count); - for handle in handles { - match handle.join() { - Ok(chunk_results) => results.extend(chunk_results), - Err(payload) => std::panic::resume_unwind(payload), - } - } - results - }); - - collect_indexed_batch_results(job_count, results, |index, source| TileBatchError { - index, - source, - }) -} - -/// Decode independent JPEG tiles at reduced resolution into caller-owned -/// output buffers using a scoped CPU worker pool. -/// -/// Each worker owns one [`DecoderContext`] and one [`ScratchPool`], so repeated -/// tiles reuse parsed table state and heap scratch within that worker without -/// sharing mutable decoder state across threads. Returned outcomes preserve -/// the caller's input order. -/// -/// # Errors -/// Returns [`TileBatchError`] with the first failing tile index in input order. -pub fn decode_tiles_scaled_into( - jobs: &mut [TileScaledDecodeJob<'_, '_>], - fmt: PixelFormat, - options: TileBatchOptions, -) -> Result, TileBatchError> { - decode_tiles_scaled_into_with_options(jobs, fmt, DecodeOptions::default(), options) -} - -/// Decode independent JPEG tiles at reduced resolution into caller-owned -/// output buffers using a scoped CPU worker pool and explicit JPEG decode -/// options. -/// -/// # Errors -/// Returns [`TileBatchError`] with the first failing tile index in input order. -pub fn decode_tiles_scaled_into_with_options( - jobs: &mut [TileScaledDecodeJob<'_, '_>], - fmt: PixelFormat, - decode_options: DecodeOptions, - options: TileBatchOptions, -) -> Result, TileBatchError> { - if jobs.is_empty() { - return Ok(Vec::new()); - } - - let job_count = jobs.len(); - let worker_count = tile_batch_worker_count(job_count, options, available_tile_batch_workers()); - let chunk_size = job_count.div_ceil(worker_count); - let results = std::thread::scope(|scope| { - let mut handles = Vec::with_capacity(worker_count); - for (chunk_index, chunk) in jobs.chunks_mut(chunk_size).enumerate() { - let start_index = chunk_index * chunk_size; - handles.push(scope.spawn(move || { - decode_tile_scaled_job_chunk(start_index, chunk, fmt, decode_options) - })); - } - - let mut results = Vec::with_capacity(job_count); - for handle in handles { - match handle.join() { - Ok(chunk_results) => results.extend(chunk_results), - Err(payload) => std::panic::resume_unwind(payload), - } - } - results - }); - - collect_indexed_batch_results(job_count, results, |index, source| TileBatchError { - index, - source, - }) -} - -/// Decode independent JPEG tile regions at reduced resolution into -/// caller-owned output buffers using a scoped CPU worker pool. -/// -/// Each worker owns one [`DecoderContext`] and one [`ScratchPool`], so repeated -/// tiles reuse parsed table state and heap scratch within that worker without -/// sharing mutable decoder state across threads. Returned outcomes preserve -/// the caller's input order. -/// -/// # Errors -/// Returns [`TileBatchError`] with the first failing tile index in input order. -pub fn decode_tiles_region_scaled_into( - jobs: &mut [TileRegionScaledDecodeJob<'_, '_>], - fmt: PixelFormat, - options: TileBatchOptions, -) -> Result, TileBatchError> { - decode_tiles_region_scaled_into_with_options(jobs, fmt, DecodeOptions::default(), options) -} - -/// Decode independent JPEG tile regions at reduced resolution into -/// caller-owned output buffers using a scoped CPU worker pool and explicit JPEG -/// decode options. -/// -/// # Errors -/// Returns [`TileBatchError`] with the first failing tile index in input order. -pub fn decode_tiles_region_scaled_into_with_options( - jobs: &mut [TileRegionScaledDecodeJob<'_, '_>], - fmt: PixelFormat, - decode_options: DecodeOptions, - options: TileBatchOptions, -) -> Result, TileBatchError> { - if jobs.is_empty() { - return Ok(Vec::new()); - } - - let job_count = jobs.len(); - let worker_count = tile_batch_worker_count(job_count, options, available_tile_batch_workers()); - let chunk_size = job_count.div_ceil(worker_count); - let results = std::thread::scope(|scope| { - let mut handles = Vec::with_capacity(worker_count); - for (chunk_index, chunk) in jobs.chunks_mut(chunk_size).enumerate() { - let start_index = chunk_index * chunk_size; - handles.push(scope.spawn(move || { - decode_tile_region_scaled_job_chunk(start_index, chunk, fmt, decode_options) - })); - } - - let mut results = Vec::with_capacity(job_count); - for handle in handles { - match handle.join() { - Ok(chunk_results) => results.extend(chunk_results), - Err(payload) => std::panic::resume_unwind(payload), - } - } - results - }); - - collect_indexed_batch_results(job_count, results, |index, source| TileBatchError { - index, - source, - }) -} - -fn available_tile_batch_workers() -> usize { - std::thread::available_parallelism().map_or(1, NonZeroUsize::get) -} - -fn decode_tile_job_chunk( - start_index: usize, - jobs: &mut [TileDecodeJob<'_, '_>], - fmt: PixelFormat, - options: DecodeOptions, -) -> Vec> { - let mut ctx = DecoderContext::new(); - let mut pool = ScratchPool::new(); - let mut results = Vec::with_capacity(jobs.len()); - for (local_index, job) in jobs.iter_mut().enumerate() { - let outcome = decode_tile_into_in_context_with_options( - job.input, &mut ctx, &mut pool, job.out, job.stride, fmt, options, - ); - results.push((start_index + local_index, outcome)); - } - results -} - -fn decode_tile_scaled_job_chunk( - start_index: usize, - jobs: &mut [TileScaledDecodeJob<'_, '_>], - fmt: PixelFormat, - options: DecodeOptions, -) -> Vec> { - let mut ctx = DecoderContext::new(); - let mut pool = ScratchPool::new(); - let mut results = Vec::with_capacity(jobs.len()); - for (local_index, job) in jobs.iter_mut().enumerate() { - let outcome = decode_tile_scaled_into_in_context_with_options( - job.input, &mut ctx, &mut pool, job.out, job.stride, fmt, job.scale, options, - ); - results.push((start_index + local_index, outcome)); - } - results -} - -fn decode_tile_region_scaled_job_chunk( - start_index: usize, - jobs: &mut [TileRegionScaledDecodeJob<'_, '_>], - fmt: PixelFormat, - options: DecodeOptions, -) -> Vec> { - let mut ctx = DecoderContext::new(); - let mut pool = ScratchPool::new(); - let mut results = Vec::with_capacity(jobs.len()); - for (local_index, job) in jobs.iter_mut().enumerate() { - let outcome = decode_tile_region_scaled_into_in_context_with_options( - job.input, &mut ctx, &mut pool, job.out, job.stride, fmt, job.roi, job.scale, options, - ); - results.push((start_index + local_index, outcome)); - } - results -} - -/// One-shot parse-plus-region-decode of an independent JPEG tile into the -/// caller's buffer, reusing both caller-owned [`DecoderContext`] and -/// caller-owned [`ScratchPool`]. -pub fn decode_tile_region_into_in_context( - bytes: &[u8], - ctx: &mut DecoderContext, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, -) -> Result { - decode_tile_region_into_in_context_with_options( - bytes, - ctx, - pool, - out, - stride, - fmt, - roi, - DecodeOptions::default(), - ) -} - -/// One-shot parse-plus-region-decode of an independent JPEG tile into the -/// caller's buffer, reusing caller-owned state and explicit JPEG decode -/// options. -#[allow(clippy::too_many_arguments)] -pub fn decode_tile_region_into_in_context_with_options( - bytes: &[u8], - ctx: &mut DecoderContext, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - options: DecodeOptions, -) -> Result { - let dec = Decoder::from_view_in_context(JpegView::parse_with_options(bytes, options)?, ctx)?; - dec.decode_region_into_with_scratch(pool, out, stride, fmt, roi) -} - -/// One-shot parse-plus-scaled-decode of an independent JPEG tile into the -/// caller's buffer, reusing both caller-owned [`DecoderContext`] and -/// caller-owned [`ScratchPool`]. -pub fn decode_tile_scaled_into_in_context( - bytes: &[u8], - ctx: &mut DecoderContext, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - scale: Downscale, -) -> Result { - decode_tile_scaled_into_in_context_with_options( - bytes, - ctx, - pool, - out, - stride, - fmt, - scale, - DecodeOptions::default(), - ) -} - -/// One-shot parse-plus-scaled-decode of an independent JPEG tile into the -/// caller's buffer, reusing caller-owned state and explicit JPEG decode -/// options. -#[allow(clippy::too_many_arguments)] -pub fn decode_tile_scaled_into_in_context_with_options( - bytes: &[u8], - ctx: &mut DecoderContext, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - scale: Downscale, - options: DecodeOptions, -) -> Result { - let dec = Decoder::from_view_in_context(JpegView::parse_with_options(bytes, options)?, ctx)?; - dec.decode_scaled_into_with_scratch(pool, out, stride, fmt, scale) -} - -/// One-shot parse-plus-region-scaled-decode of an independent JPEG tile into -/// the caller's buffer, reusing both caller-owned [`DecoderContext`] and -/// caller-owned [`ScratchPool`]. -#[allow(clippy::too_many_arguments)] -pub fn decode_tile_region_scaled_into_in_context( - bytes: &[u8], - ctx: &mut DecoderContext, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, -) -> Result { - decode_tile_region_scaled_into_in_context_with_options( - bytes, - ctx, - pool, - out, - stride, - fmt, - roi, - scale, - DecodeOptions::default(), - ) -} - -/// One-shot parse-plus-region-scaled-decode of an independent JPEG tile into -/// the caller's buffer, reusing caller-owned state and explicit JPEG decode -/// options. -#[allow(clippy::too_many_arguments)] -pub fn decode_tile_region_scaled_into_in_context_with_options( - bytes: &[u8], - ctx: &mut DecoderContext, - pool: &mut ScratchPool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: Rect, - scale: Downscale, - options: DecodeOptions, -) -> Result { - let dec = Decoder::from_view_in_context(JpegView::parse_with_options(bytes, options)?, ctx)?; - dec.decode_region_scaled_into_with_scratch(pool, out, stride, fmt, roi, scale) -} - -impl Decoder<'_> { - /// One-shot parse-plus-row-decode of a JPEG tile using caller-owned shared - /// table context and caller-owned scratch. - pub fn decode_tile( - bytes: &[u8], - ctx: &mut DecoderContext, - pool: &mut ScratchPool, - sink: &mut S, - ) -> Result - where - S: RowSink, - { - let dec = Decoder::from_view_in_context(JpegView::parse(bytes)?, ctx)?; - dec.decode_rows_with_scratch(pool, sink) - } -} - -impl Decoder<'_> { - fn decode_scratch_bytes(&self, cap: usize) -> Result { - let scratch_bytes = self - .progressive_plan - .as_ref() - .map_or(self.plan.scratch_bytes, |plan| plan.scratch_bytes); - if scratch_bytes > cap { - return Err(JpegError::MemoryCapExceeded { - requested: scratch_bytes, - cap, - }); - } - Ok(scratch_bytes) - } - - fn checkpoint_for_mcu( - &self, - scan_bytes: &[u8], - target_mcu: u32, - ) -> Result, JpegError> { - if self.plan.restart_interval.is_some() || target_mcu < CPU_ROI_CHECKPOINT_MIN_TARGET_MCUS { - return Ok(None); - } - - let mut cache = self - .cpu_entropy_checkpoints - .lock() - .expect("CPU entropy checkpoint cache mutex poisoned"); - checkpoint_before_mcu( - &self.plan, - scan_bytes, - CPU_ROI_CHECKPOINT_CADENCE_MCUS, - target_mcu, - &mut cache, - ) - } - - fn source_window_for_output_rect( - &self, - downscale: DownscaleFactor, - output_rect: Rect, - ) -> (u32, u32) { - let layout = stripe_region_layout(&self.plan, downscale, output_rect); - (layout.source_x0, layout.source_width) - } - - fn decode_with_writer( - &self, - pool: &mut ScratchPool, - writer: &mut W, - downscale: DownscaleFactor, - decoded: Rect, - ) -> Result { - let _ = self.decode_scratch_bytes(DEFAULT_MAX_DECODE_BYTES)?; - let profile_enabled = jpeg_profile_stages_enabled(); - if let Some(plan) = &self.progressive_plan { - if downscale != DownscaleFactor::Full || decoded != Rect::full(self.info.dimensions) { - return Err(JpegError::NotImplemented { - sof: self.info.sof_kind, - }); - } - let scan_start = profile_enabled.then(Instant::now); - let scan_warnings = decode_progressive(plan, self.backend, self.bytes, writer)?; - if let Some(start) = scan_start { - emit_decode_scan_profile( - "progressive", - self.info.dimensions, - decoded, - downscale, - start.elapsed(), - ); - } - return Ok(DecodeOutcome { - decoded, - warnings: merged_warnings(&self.warnings, scan_warnings), - }); - } - let output_rect = scaled_rect_covering(decoded, downscale)?; - let scan_bytes = &self.bytes[self.plan.scan_offset..]; - let scan_start = profile_enabled.then(Instant::now); - let scan_warnings = decode_scan_baseline( - &self.plan, - self.backend, - scan_bytes, - pool, - writer, - downscale, - output_rect, - )?; - if let Some(start) = scan_start { - emit_decode_scan_profile( - "baseline", - self.info.dimensions, - decoded, - downscale, - start.elapsed(), - ); - } - Ok(DecodeOutcome { - decoded, - warnings: merged_warnings(&self.warnings, scan_warnings), - }) - } - - fn decode_rgb_with_writer( - &self, - pool: &mut ScratchPool, - writer: &mut W, - downscale: DownscaleFactor, - decoded: Rect, - ) -> Result { - let _ = self.decode_scratch_bytes(DEFAULT_MAX_DECODE_BYTES)?; - let profile_enabled = jpeg_profile_stages_enabled(); - if let Some(plan) = &self.progressive_plan { - if downscale != DownscaleFactor::Full || decoded != Rect::full(self.info.dimensions) { - return Err(JpegError::NotImplemented { - sof: self.info.sof_kind, - }); - } - let scan_start = profile_enabled.then(Instant::now); - let scan_warnings = decode_progressive(plan, self.backend, self.bytes, writer)?; - if let Some(start) = scan_start { - emit_decode_scan_profile( - "progressive_rgb", - self.info.dimensions, - decoded, - downscale, - start.elapsed(), - ); - } - return Ok(DecodeOutcome { - decoded, - warnings: merged_warnings(&self.warnings, scan_warnings), - }); - } - let output_rect = scaled_rect_covering(decoded, downscale)?; - let scan_bytes = &self.bytes[self.plan.scan_offset..]; - let scan_start = profile_enabled.then(Instant::now); - let (scan_path, scan_warnings) = - if downscale == DownscaleFactor::Full && self.plan.matches_fast_tile_shape() { - ( - "fast420_rgb", - decode_scan_fast_tile_rgb(&self.plan, self.backend, scan_bytes, pool, writer)?, - ) - } else if downscale == DownscaleFactor::Full - && decoded == Rect::full(self.info.dimensions) - && self.plan.matches_fast_rgb444_shape() - { - ( - "fast444_rgb", - decode_scan_fast_rgb_444(&self.plan, self.backend, scan_bytes, pool, writer)?, - ) - } else { - ( - "baseline_rgb", - decode_scan_baseline_rgb( - &self.plan, - self.backend, - scan_bytes, - pool, - writer, - downscale, - output_rect, - )?, - ) - }; - if let Some(start) = scan_start { - emit_decode_scan_profile( - scan_path, - self.info.dimensions, - decoded, - downscale, - start.elapsed(), - ); - } - Ok(DecodeOutcome { - decoded, - warnings: merged_warnings(&self.warnings, scan_warnings), - }) - } -} - -fn restart_index_for_stream( - bytes: &[u8], - scan_data_offset: Option, - info: &Info, - restart_interval: Option, -) -> Result, JpegError> { - let Some(interval_mcus) = restart_interval - .filter(|&interval| interval > 0) - .map(u32::from) - else { - return Ok(None); - }; - let scan_data_offset = scan_data_offset.ok_or(JpegError::MissingMarker { - marker: MarkerKind::Sos, - })?; - if !matches!(info.sof_kind, SofKind::Baseline8 | SofKind::Extended8) || info.scan_count != 1 { - return Err(JpegError::NotImplemented { sof: info.sof_kind }); - } - let total_mcus = info.mcu_geometry.count; - let expected_restarts = total_mcus.saturating_sub(1) / interval_mcus; - let mut segments = Vec::new(); - segments.push(RestartSegment { - start_mcu: 0, - entropy_offset: scan_data_offset, - marker_offset: None, - marker: None, - }); - - let mut found_restarts = 0u32; - let mut expected_rst = 0xd0u8; - let mut pos = scan_data_offset; - while pos < bytes.len() { - if bytes[pos] != 0xff { - pos += 1; - continue; - } - - let mut marker_code_pos = pos + 1; - while marker_code_pos < bytes.len() && bytes[marker_code_pos] == 0xff { - marker_code_pos += 1; - } - if marker_code_pos >= bytes.len() { - return Err(JpegError::Truncated { - offset: pos, - expected: 1, - }); - } - - let marker = bytes[marker_code_pos]; - let marker_offset = marker_code_pos - 1; - match marker { - 0x00 => pos = marker_code_pos + 1, - 0xd0..=0xd7 => { - if found_restarts >= expected_restarts { - return Err(JpegError::UnexpectedMarker { - offset: marker_offset, - expected: MarkerKind::Eoi, - found: marker, - }); - } - if marker != expected_rst { - return Err(JpegError::RestartMismatch { - offset: marker_offset, - expected: expected_rst & 0x07, - found: marker, - }); - } - found_restarts += 1; - segments.push(RestartSegment { - start_mcu: found_restarts.saturating_mul(interval_mcus), - entropy_offset: marker_code_pos + 1, - marker_offset: Some(marker_offset), - marker: Some(marker), - }); - expected_rst = if expected_rst == 0xd7 { - 0xd0 - } else { - expected_rst + 1 - }; - pos = marker_code_pos + 1; - } - 0xd9 => { - if found_restarts != expected_restarts { - return Err(JpegError::UnexpectedEoi { - mcu_at: found_restarts - .saturating_add(1) - .saturating_mul(interval_mcus), - mcu_total: total_mcus, - }); - } - return Ok(Some(RestartIndex { - scan_data_offset, - interval_mcus, - segments, - })); - } - found => { - return Err(JpegError::UnexpectedMarker { - offset: marker_offset, - expected: MarkerKind::Eoi, - found, - }); - } - } - } - - Err(JpegError::MissingMarker { - marker: MarkerKind::Eoi, - }) -} - -fn output_format_profile_name(fmt: OutputFormat) -> &'static str { - match fmt { - OutputFormat::Rgb8 | OutputFormat::Rgb8Scaled { .. } => "Rgb8", - OutputFormat::Rgba8 { .. } => "Rgba8", - OutputFormat::Gray8 | OutputFormat::Gray8Scaled { .. } => "Gray8", - } -} - -fn downscale_profile_name(downscale: DownscaleFactor) -> &'static str { - match downscale { - DownscaleFactor::Full => "full", - DownscaleFactor::Half => "half", - DownscaleFactor::Quarter => "quarter", - DownscaleFactor::Eighth => "eighth", - } -} - -fn emit_decode_scan_profile( - scan_path: &str, - dimensions: (u32, u32), - decoded: Rect, - downscale: DownscaleFactor, - elapsed: Duration, -) { - let source_width_s = dimensions.0.to_string(); - let source_height_s = dimensions.1.to_string(); - let decoded_x_s = decoded.x.to_string(); - let decoded_y_s = decoded.y.to_string(); - let decoded_w_s = decoded.w.to_string(); - let decoded_h_s = decoded.h.to_string(); - let scan_us = duration_us_string(elapsed); - emit_jpeg_profile_row( - "decode_scan", - "cpu", - &[ - ("scan_path", scan_path), - ("downscale", downscale_profile_name(downscale)), - ("source_width", source_width_s.as_str()), - ("source_height", source_height_s.as_str()), - ("decoded_x", decoded_x_s.as_str()), - ("decoded_y", decoded_y_s.as_str()), - ("decoded_w", decoded_w_s.as_str()), - ("decoded_h", decoded_h_s.as_str()), - ("scan_us", scan_us.as_str()), - ], - ); -} - -fn merged_warnings(header_warnings: &[Warning], scan_warnings: Vec) -> Vec { - if header_warnings.is_empty() { - return scan_warnings; - } - if scan_warnings.is_empty() { - return header_warnings.to_vec(); - } - let mut warnings = Vec::with_capacity(header_warnings.len() + scan_warnings.len()); - warnings.extend_from_slice(header_warnings); - warnings.extend(scan_warnings); - warnings -} - -fn jpeg_passthrough_syntax(info: &Info) -> Option { - match info.sof_kind { - SofKind::Baseline8 if info.bit_depth == 8 => Some(CompressedTransferSyntax::JpegBaseline8), - SofKind::Extended8 | SofKind::Extended12 => { - Some(CompressedTransferSyntax::JpegExtendedSequential) - } - SofKind::Baseline8 | SofKind::Progressive8 | SofKind::Progressive12 | SofKind::Lossless => { - None - } - } -} - -fn core_outcome(outcome: DecodeOutcome) -> CoreDecodeOutcome { - outcome.into() -} - -fn jpeg_downscale(scale: Downscale) -> DownscaleFactor { - match scale { - Downscale::None => DownscaleFactor::Full, - Downscale::Half => DownscaleFactor::Half, - Downscale::Quarter => DownscaleFactor::Quarter, - Downscale::Eighth => DownscaleFactor::Eighth, - _ => unreachable!("unsupported Downscale variant"), - } -} - -fn output_format_from_parts( - sof_kind: SofKind, - fmt: PixelFormat, - scale: Downscale, -) -> Result { - match (fmt, scale) { - (PixelFormat::Rgb8, Downscale::None) => Ok(OutputFormat::Rgb8), - (PixelFormat::Rgb8, scale) => Ok(OutputFormat::Rgb8Scaled { - factor: jpeg_downscale(scale), - }), - (PixelFormat::Gray8, Downscale::None) => Ok(OutputFormat::Gray8), - (PixelFormat::Gray8, scale) => Ok(OutputFormat::Gray8Scaled { - factor: jpeg_downscale(scale), - }), - (PixelFormat::Rgba8, Downscale::None) => Ok(OutputFormat::Rgba8 { alpha: 255 }), - (PixelFormat::Rgba8, _) => Err(JpegError::DownscaleUnsupported { sof: sof_kind }), - (PixelFormat::Rgb16 | PixelFormat::Rgba16 | PixelFormat::Gray16, _) => { - Err(JpegError::UnsupportedBitDepth { depth: 16 }) - } - _ => Err(JpegError::DownscaleUnsupported { sof: sof_kind }), - } -} - -impl ImageCodec for JpegCodec { - type Error = JpegError; - type Warning = Warning; - type Pool = ScratchPool; -} - -impl ImageCodec for Decoder<'_> { - type Error = JpegError; - type Warning = Warning; - type Pool = ScratchPool; -} - -impl<'a> ImageDecode<'a> for Decoder<'a> { - type View = JpegView<'a>; - - fn inspect(input: &'a [u8]) -> Result { - Ok(Decoder::inspect(input)?.to_core_info()) - } - - fn parse(input: &'a [u8]) -> Result { - JpegView::parse(input) - } - - fn from_view(view: Self::View) -> Result { - Decoder::from_view(view) - } - - fn decode_into( - &mut self, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - ) -> Result, Self::Error> { - Decoder::decode_into(self, out, stride, fmt).map(core_outcome) - } - - fn decode_into_with_scratch( - &mut self, - pool: &mut Self::Pool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - ) -> Result, Self::Error> { - Decoder::decode_into_with_scratch(self, pool, out, stride, fmt).map(core_outcome) - } - - fn decode_region_into( - &mut self, - pool: &mut Self::Pool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: signinum_core::Rect, - ) -> Result, Self::Error> { - Decoder::decode_region_into_with_scratch(self, pool, out, stride, fmt, roi.into()) - .map(core_outcome) - } - - fn decode_scaled_into( - &mut self, - pool: &mut Self::Pool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - scale: Downscale, - ) -> Result, Self::Error> { - Decoder::decode_scaled_into_with_scratch(self, pool, out, stride, fmt, scale) - .map(core_outcome) - } - - fn decode_region_scaled_into( - &mut self, - pool: &mut Self::Pool, - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: signinum_core::Rect, - scale: Downscale, - ) -> Result, Self::Error> { - Decoder::decode_region_scaled_into_with_scratch( - self, - pool, - out, - stride, - fmt, - roi.into(), - scale, - ) - .map(core_outcome) - } -} - -struct CoreRowSinkAdapter<'a, R: RowSink> { - sink: &'a mut R, - sink_error: Option, -} - -impl> RowSink for CoreRowSinkAdapter<'_, R> { - type Error = JpegError; - - fn write_row(&mut self, y: u32, row: &[u8]) -> Result<(), JpegError> { - match self.sink.write_row(y, row) { - Ok(()) => Ok(()), - Err(err) => { - self.sink_error = Some(err); - Err(JpegError::RowSinkAborted) - } - } - } -} - -impl<'a> ImageDecodeRows<'a, u8> for Decoder<'a> { - fn decode_rows>( - &mut self, - sink: &mut R, - ) -> Result, DecodeRowsError> { - let mut adapter = CoreRowSinkAdapter { - sink, - sink_error: None, - }; - match Decoder::decode_rows(self, &mut adapter) { - Ok(outcome) => Ok(core_outcome(outcome)), - Err(JpegError::RowSinkAborted) => Err(DecodeRowsError::Sink( - adapter - .sink_error - .expect("row sink abort stores the original sink error"), - )), - Err(err) => Err(DecodeRowsError::Decode(err)), - } - } -} - -impl TileBatchDecode for JpegCodec { - type Context = DecoderContext; - - fn decode_tile( - ctx: &mut CoreDecoderContext, - pool: &mut Self::Pool, - input: &[u8], - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - ) -> Result, Self::Error> { - let dec = Decoder::from_view_in_context(JpegView::parse(input)?, ctx.codec_mut())?; - dec.decode_into_with_scratch(pool, out, stride, fmt) - .map(core_outcome) - } - - fn decode_tile_region( - ctx: &mut CoreDecoderContext, - pool: &mut Self::Pool, - input: &[u8], - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: signinum_core::Rect, - ) -> Result, Self::Error> { - let dec = Decoder::from_view_in_context(JpegView::parse(input)?, ctx.codec_mut())?; - dec.decode_region_into_with_scratch(pool, out, stride, fmt, roi.into()) - .map(core_outcome) - } - - fn decode_tile_scaled( - ctx: &mut CoreDecoderContext, - pool: &mut Self::Pool, - input: &[u8], - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - scale: Downscale, - ) -> Result, Self::Error> { - let dec = Decoder::from_view_in_context(JpegView::parse(input)?, ctx.codec_mut())?; - dec.decode_scaled_into_with_scratch(pool, out, stride, fmt, scale) - .map(core_outcome) - } - - fn decode_tile_region_scaled( - ctx: &mut CoreDecoderContext, - pool: &mut Self::Pool, - input: &[u8], - out: &mut [u8], - stride: usize, - fmt: PixelFormat, - roi: signinum_core::Rect, - scale: Downscale, - ) -> Result, Self::Error> { - let dec = Decoder::from_view_in_context(JpegView::parse(input)?, ctx.codec_mut())?; - dec.decode_region_scaled_into_with_scratch(pool, out, stride, fmt, roi.into(), scale) - .map(core_outcome) - } -} - -#[allow(clippy::uninit_vec)] -fn allocate_output_buffer(len: usize) -> Vec { - let mut out = Vec::with_capacity(len); - // Safety: all owned-output entrypoints use tight row strides, and the - // decode writers fully initialize every byte in the destination on success. - // If decode returns an error, dropping a Vec with uninitialized bytes is - // still sound because `u8` has no drop glue. - unsafe { - out.set_len(len); - } - out -} - -fn scaled_dimensions(dims: (u32, u32), factor: DownscaleFactor) -> (u32, u32) { - let denom = factor.denominator(); - (dims.0.div_ceil(denom), dims.1.div_ceil(denom)) -} - -fn scaled_rect_covering(rect: Rect, factor: DownscaleFactor) -> Result { - let denom = factor.denominator(); - let x_end = rect - .x - .checked_add(rect.w) - .ok_or(JpegError::RectOutOfBounds { - rect, - width: u32::MAX, - height: u32::MAX, - })?; - let y_end = rect - .y - .checked_add(rect.h) - .ok_or(JpegError::RectOutOfBounds { - rect, - width: u32::MAX, - height: u32::MAX, - })?; - let x0 = rect.x / denom; - let y0 = rect.y / denom; - let x1 = x_end.div_ceil(denom); - let y1 = y_end.div_ceil(denom); - Ok(Rect { - x: x0, - y: y0, - w: x1.saturating_sub(x0), - h: y1.saturating_sub(y0), - }) -} - -struct CroppedWriter { - inner: W, - rect: Rect, - source_x0: u32, - source_width: u32, - top_row: Vec, - bottom_row: Vec, -} - -struct ComponentWriterAdapter<'a, W> { - inner: &'a mut W, -} - -impl OutputWriter for ComponentWriterAdapter<'_, W> { - fn write_rgb_row( - &mut self, - y: u32, - r_row: &[u8], - g_row: &[u8], - b_row: &[u8], - ) -> Result<(), JpegError> { - self.inner.write_rgb_row(y, r_row, g_row, b_row) - } - - fn write_ycbcr_row( - &mut self, - y: u32, - y_row: &[u8], - cb_row: &[u8], - cr_row: &[u8], - ) -> Result<(), JpegError> { - self.inner.write_ycbcr_row(y, y_row, cb_row, cr_row) - } - - fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError> { - self.inner.write_gray_row(y, gray_row) - } -} - -impl CroppedWriter { - fn new(inner: W, rect: Rect, source_x0: u32, source_width: u32) -> Self { - let row_len = source_width as usize * 3; - Self { - inner, - rect, - source_x0, - source_width, - top_row: vec![0; row_len], - bottom_row: vec![0; row_len], - } - } -} - -impl OutputWriter for CroppedWriter { - fn write_rgb_row( - &mut self, - y: u32, - r_row: &[u8], - g_row: &[u8], - b_row: &[u8], - ) -> Result<(), JpegError> { - if y < self.rect.y || y >= self.rect.y + self.rect.h { - return Ok(()); - } - let x0 = self - .rect - .x - .checked_sub(self.source_x0) - .expect("crop window must cover requested rect") as usize; - let x1 = x0 + self.rect.w as usize; - self.inner.write_rgb_row( - y - self.rect.y, - &r_row[x0..x1], - &g_row[x0..x1], - &b_row[x0..x1], - ) - } - - fn write_ycbcr_row( - &mut self, - y: u32, - y_row: &[u8], - cb_row: &[u8], - cr_row: &[u8], - ) -> Result<(), JpegError> { - if y < self.rect.y || y >= self.rect.y + self.rect.h { - return Ok(()); - } - let x0 = self - .rect - .x - .checked_sub(self.source_x0) - .expect("crop window must cover requested rect") as usize; - let x1 = x0 + self.rect.w as usize; - self.inner.write_ycbcr_row( - y - self.rect.y, - &y_row[x0..x1], - &cb_row[x0..x1], - &cr_row[x0..x1], - ) - } - - fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError> { - if y < self.rect.y || y >= self.rect.y + self.rect.h { - return Ok(()); - } - let x0 = self - .rect - .x - .checked_sub(self.source_x0) - .expect("crop window must cover requested rect") as usize; - let x1 = x0 + self.rect.w as usize; - self.inner - .write_gray_row(y - self.rect.y, &gray_row[x0..x1]) - } -} - -impl InterleavedRgbWriter for CroppedWriter { - fn with_rgb_rows(&mut self, y: u32, row_count: usize, fill: F) -> Result - where - F: FnOnce(&mut [u8], Option<&mut [u8]>) -> Result, - { - let row_len = self.source_width as usize * 3; - if self.top_row.len() != row_len { - self.top_row.resize(row_len, 0); - self.bottom_row.resize(row_len, 0); - } - - let result = match row_count { - 1 => fill(&mut self.top_row, None)?, - 2 => fill(&mut self.top_row, Some(&mut self.bottom_row))?, - _ => unreachable!("CroppedWriter only supports one or two rows"), - }; - - let top_in = y >= self.rect.y && y < self.rect.y + self.rect.h; - let bottom_y = y + 1; - let bottom_in = - row_count == 2 && bottom_y >= self.rect.y && bottom_y < self.rect.y + self.rect.h; - let x0 = self - .rect - .x - .checked_sub(self.source_x0) - .expect("crop window must cover requested rect") as usize - * 3; - let x1 = x0 + self.rect.w as usize * 3; - - match (top_in, bottom_in) { - (false, false) => {} - (true, false) => { - self.inner.with_rgb_rows(y - self.rect.y, 1, |dst, _| { - dst.copy_from_slice(&self.top_row[x0..x1]); - Ok(()) - })?; - } - (false, true) => { - self.inner - .with_rgb_rows(bottom_y - self.rect.y, 1, |dst, _| { - dst.copy_from_slice(&self.bottom_row[x0..x1]); - Ok(()) - })?; - } - (true, true) => { - self.inner - .with_rgb_rows(y - self.rect.y, 2, |dst_top, dst_bottom| { - dst_top.copy_from_slice(&self.top_row[x0..x1]); - dst_bottom - .expect("row_count=2 supplies bottom row") - .copy_from_slice(&self.bottom_row[x0..x1]); - Ok(()) - })?; - } - } - - Ok(result) - } -} - -fn build_decode_plan( - header: &ParsedHeader, - info: &Info, - dc_tables: &[Option>; 4], - ac_tables: &[Option>; 4], - ctx: &mut DecoderContext, -) -> Result { - let scan = header.scan.as_ref().ok_or(JpegError::MissingMarker { - marker: MarkerKind::Sos, - })?; - let scan_offset = header.sos_offset.ok_or(JpegError::MissingMarker { - marker: MarkerKind::Sos, - })?; - - let mut components = Vec::with_capacity(scan.components.len()); - for scan_comp in scan.components.iter().copied() { - let output_index = find_component_index(&header.component_ids, scan_comp.id).ok_or( - JpegError::UnknownScanComponent { - offset: scan_offset, - component: scan_comp.id, - }, - )?; - let (h, v) = header - .sampling - .component(output_index) - .ok_or(JpegError::MissingMarker { - marker: MarkerKind::Sof, - })?; - let quant_id = - *header - .quant_table_ids - .get(output_index) - .ok_or(JpegError::MissingMarker { - marker: MarkerKind::Sof, - })? as usize; - let quant = *header - .quant_tables - .entries - .get(quant_id) - .and_then(|q| q.as_ref()) - .ok_or(JpegError::MissingQuantTable { - component: scan_comp.id, - table_id: quant_id as u8, - })?; - let dc_table = dc_tables[scan_comp.dc_table as usize].as_ref().ok_or( - JpegError::MissingHuffmanTable { - component: scan_comp.id, - class: 0, - id: scan_comp.dc_table, - }, - )?; - let ac_table = ac_tables[scan_comp.ac_table as usize].as_ref().ok_or( - JpegError::MissingHuffmanTable { - component: scan_comp.id, - class: 1, - id: scan_comp.ac_table, - }, - )?; - components.push(PreparedComponentPlan { - h, - v, - output_index, - quant: ctx.resolve_quant_table(quant), - dc_table: Arc::clone(dc_table), - ac_table: Arc::clone(ac_table), - }); - } - - let scratch_bytes = - compute_decode_scratch_bytes(info.dimensions, info.sampling, DEFAULT_MAX_DECODE_BYTES)?; - - Ok(PreparedDecodePlan { - components, - sampling: info.sampling, - color_space: info.color_space, - restart_interval: header.restart_interval, - dimensions: info.dimensions, - scan_offset, - scratch_bytes, - }) -} - -fn validate_sampling_factors(header: &ParsedHeader, info: &Info) -> Result<(), JpegError> { - if let Some((h, v)) = header.sampling.component(0) { - if h != header.sampling.max_h || v != header.sampling.max_v { - return Err(JpegError::NotImplemented { sof: info.sof_kind }); - } - } - for (h, v) in header.sampling.iter() { - if h == 0 || v == 0 || h > 4 || v > 4 { - return Err(JpegError::NotImplemented { sof: info.sof_kind }); - } - if !header.sampling.max_h.is_multiple_of(h) || !header.sampling.max_v.is_multiple_of(v) { - return Err(JpegError::NotImplemented { sof: info.sof_kind }); - } - } - Ok(()) -} - -fn resolve_progressive_huffman( - ctx: &mut DecoderContext, - tables: &[Option; 4], - component: u8, - class: u8, - id: u8, -) -> Result, JpegError> { - let raw = tables - .get(id as usize) - .and_then(|table| table.as_ref()) - .ok_or(JpegError::MissingHuffmanTable { - component, - class, - id, - })?; - ctx.resolve_huffman_table(raw) -} - -fn compute_progressive_scratch_bytes( - components: &[PreparedProgressiveComponentPlan], - output_width: usize, -) -> Result { - let cap = DEFAULT_MAX_DECODE_BYTES; - let mut total = 0usize; - for component in components { - let blocks = checked_usize_product( - &[component.block_cols as usize, component.block_rows as usize], - cap, - )?; - let coeffs = checked_usize_product(&[blocks, 64, core::mem::size_of::()], cap)?; - total = total - .checked_add(coeffs) - .ok_or(JpegError::MemoryCapExceeded { - requested: usize::MAX, - cap, - })?; - - let plane = checked_usize_product( - &[ - component.block_cols as usize, - component.block_rows as usize, - 64, - ], - cap, - )?; - total = total - .checked_add(plane) - .ok_or(JpegError::MemoryCapExceeded { - requested: usize::MAX, - cap, - })?; - if total > cap { - return Err(JpegError::MemoryCapExceeded { - requested: total, - cap, - }); - } - } - total = - total - .checked_add(output_width.saturating_mul(3)) - .ok_or(JpegError::MemoryCapExceeded { - requested: usize::MAX, - cap, - })?; - if total > cap { - return Err(JpegError::MemoryCapExceeded { - requested: total, - cap, - }); - } - Ok(total) -} - -struct SinkWriter<'a, S> { - sink: &'a mut S, - rows: SinkRows, - backend: Backend, -} - -impl<'a, S> SinkWriter<'a, S> { - fn new(sink: &'a mut S, rows: SinkRows, backend: Backend) -> Self { - debug_assert_eq!(rows.top_row.len(), rows.bottom_row.len()); - Self { - sink, - rows, - backend, - } - } - - fn into_rows(self) -> SinkRows { - self.rows - } -} - -impl InterleavedRgbWriter for SinkWriter<'_, S> -where - S: RowSink, -{ - fn with_rgb_rows(&mut self, y: u32, row_count: usize, fill: F) -> Result - where - F: FnOnce(&mut [u8], Option<&mut [u8]>) -> Result, - { - let result = match row_count { - 1 => fill(&mut self.rows.top_row, None), - 2 => fill(&mut self.rows.top_row, Some(&mut self.rows.bottom_row)), - _ => unreachable!("SinkWriter only supports one or two rows"), - }?; - self.sink.write_row(y, &self.rows.top_row)?; - if row_count == 2 { - self.sink.write_row(y + 1, &self.rows.bottom_row)?; - } - Ok(result) - } -} - -impl OutputWriter for SinkWriter<'_, S> -where - S: RowSink, -{ - fn write_rgb_row( - &mut self, - y: u32, - r_row: &[u8], - g_row: &[u8], - b_row: &[u8], - ) -> Result<(), JpegError> { - self.backend - .fill_rgb_row_from_rgb(r_row, g_row, b_row, &mut self.rows.top_row); - self.sink.write_row(y, &self.rows.top_row) - } - - fn write_ycbcr_row( - &mut self, - y: u32, - y_row: &[u8], - cb_row: &[u8], - cr_row: &[u8], - ) -> Result<(), JpegError> { - self.backend - .fill_rgb_row_from_ycbcr(y_row, cb_row, cr_row, &mut self.rows.top_row); - self.sink.write_row(y, &self.rows.top_row) - } - - fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError> { - self.backend - .fill_rgb_row_from_gray(gray_row, &mut self.rows.top_row); - self.sink.write_row(y, &self.rows.top_row) - } -} - -fn find_component_index(component_ids: &[u8], id: u8) -> Option { - component_ids - .iter() - .position(|&component_id| component_id == id) -} - -fn compute_decode_scratch_bytes( - (width, height): (u32, u32), - sampling: crate::info::SamplingFactors, - cap: usize, -) -> Result { - let max_h = u32::from(sampling.max_h); - let max_v = u32::from(sampling.max_v); - let mcu_width = 8u32 - .checked_mul(max_h) - .ok_or(JpegError::MemoryCapExceeded { - requested: usize::MAX, - cap, - })?; - let mcu_height = 8u32 - .checked_mul(max_v) - .ok_or(JpegError::MemoryCapExceeded { - requested: usize::MAX, - cap, - })?; - let mcus_per_row = width.div_ceil(mcu_width); - let _mcu_rows = height.div_ceil(mcu_height); - - let mut stripe_total = 0usize; - for (h, v) in sampling.iter() { - let cols = checked_usize_product(&[mcus_per_row as usize, usize::from(h), 8usize], cap)?; - let rows = checked_usize_product(&[usize::from(v), 8usize], cap)?; - let plane = cols.checked_mul(rows).ok_or(JpegError::MemoryCapExceeded { - requested: usize::MAX, - cap, - })?; - stripe_total = stripe_total - .checked_add(plane) - .ok_or(JpegError::MemoryCapExceeded { - requested: usize::MAX, - cap, - })?; - if stripe_total > cap { - return Err(JpegError::MemoryCapExceeded { - requested: stripe_total, - cap, - }); - } - } - - let stripe_buffers = checked_usize_product(&[stripe_total, 3], cap)?; - let row_scratch = checked_usize_product(&[width as usize, 7], cap)?; - let total = stripe_buffers - .checked_add(row_scratch) - .ok_or(JpegError::MemoryCapExceeded { - requested: usize::MAX, - cap, - })?; - if total > cap { - return Err(JpegError::MemoryCapExceeded { - requested: total, - cap, - }); - } - - Ok(total) -} - -fn checked_usize_product(factors: &[usize], cap: usize) -> Result { - let mut value = 1usize; - for factor in factors { - value = value - .checked_mul(*factor) - .ok_or(JpegError::MemoryCapExceeded { - requested: usize::MAX, - cap, - })?; - } - Ok(value) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::error::Warning; - use crate::output::OutputWriter; - use alloc::vec; - use alloc::vec::Vec; - - fn minimal_baseline_jpeg() -> Vec { - let mut v = Vec::new(); - v.extend_from_slice(&[0xFF, 0xD8]); - v.extend_from_slice(&[0xFF, 0xDB, 0x00, 67, 0x00]); - v.extend(core::iter::repeat_n(1u8, 64)); - v.extend_from_slice(&[ - 0xFF, - 0xC0, - 0x00, - 17, - 8, - 0, - 16, - 0, - 16, - 3, - 1, - (2 << 4) | 2, - 0, - 2, - (1 << 4) | 1, - 0, - 3, - (1 << 4) | 1, - 0, - ]); - v.extend_from_slice(&[ - 0xFF, 0xC4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xAA, - ]); - v.extend_from_slice(&[ - 0xFF, 0xC4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xBB, - ]); - v.extend_from_slice(&[0xFF, 0xDA, 0x00, 12, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0]); - v.extend_from_slice(&[0x00, 0xFF, 0xD9]); - v - } - - fn dc_only_420_jpeg(width: u16, height: u16) -> Vec { - let mut v = Vec::new(); - v.extend_from_slice(&[0xFF, 0xD8]); - v.extend_from_slice(&[0xFF, 0xDB, 0x00, 67, 0x00]); - v.extend(core::iter::repeat_n(1u8, 64)); - v.extend_from_slice(&[ - 0xFF, - 0xC0, - 0x00, - 17, - 8, - (height >> 8) as u8, - height as u8, - (width >> 8) as u8, - width as u8, - 3, - 1, - (2 << 4) | 2, - 0, - 2, - (1 << 4) | 1, - 0, - 3, - (1 << 4) | 1, - 0, - ]); - v.extend_from_slice(&[ - 0xFF, 0xC4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - v.extend_from_slice(&[ - 0xFF, 0xC4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - v.extend_from_slice(&[0xFF, 0xDA, 0x00, 12, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0]); - - let mcus_per_row = u32::from(width).div_ceil(16); - let mcu_rows = u32::from(height).div_ceil(16); - let entropy_bits = mcus_per_row * mcu_rows * 12; - let entropy_bytes = (entropy_bits as usize).div_ceil(8) + 8; - v.extend(core::iter::repeat_n(0u8, entropy_bytes)); - v.extend_from_slice(&[0xFF, 0xD9]); - v - } - - #[test] - fn decoder_new_succeeds_on_baseline_stream() { - let bytes = minimal_baseline_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - assert_eq!(dec.info().dimensions, (16, 16)); - } - - #[test] - fn decoder_new_rejects_extended12_with_not_implemented() { - let mut bytes = minimal_baseline_jpeg(); - let p = bytes.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap(); - bytes[p + 1] = 0xC1; - bytes[p + 4] = 12; - let err = Decoder::new(&bytes).unwrap_err(); - assert!(err.is_not_implemented()); - } - - #[test] - fn decoder_new_rejects_arithmetic_as_unsupported() { - let mut bytes = minimal_baseline_jpeg(); - let p = bytes.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap(); - bytes[p + 1] = 0xC9; - let err = Decoder::new(&bytes).unwrap_err(); - assert!(err.is_unsupported()); - } - - #[test] - fn decode_outcome_carries_rect_and_warnings() { - let outcome = DecodeOutcome { - decoded: Rect { - x: 0, - y: 0, - w: 16, - h: 16, - }, - warnings: vec![Warning::MissingEoi], - }; - assert_eq!(outcome.decoded.w, 16); - assert_eq!(outcome.warnings.len(), 1); - } - - #[test] - fn decode_into_rejects_undersized_buffer() { - let bytes = minimal_baseline_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let mut buf = vec![0u8; 4]; - let err = dec - .decode_into(&mut buf, 48, PixelFormat::Rgb8) - .unwrap_err(); - assert!(matches!(err, JpegError::OutputBufferTooSmall { .. })); - } - - #[test] - fn decode_into_rejects_invalid_stride() { - let bytes = minimal_baseline_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let mut buf = vec![0u8; 16 * 16 * 3]; - let err = dec - .decode_into(&mut buf, 10, PixelFormat::Rgb8) - .unwrap_err(); - assert!(matches!(err, JpegError::InvalidStride { .. })); - } - - #[test] - fn large_fast_420_region_decode_populates_cpu_entropy_checkpoints() { - let bytes = dc_only_420_jpeg(1024, 2048); - let dec = Decoder::new(&bytes).expect("decoder"); - assert!(dec.plan.matches_fast_tile_shape()); - - let roi = Rect { - x: 64, - y: 1536, - w: 64, - h: 64, - }; - let mut out = vec![0u8; roi.w as usize * roi.h as usize * 3]; - let mut pool = ScratchPool::new(); - dec.decode_region_into_with_scratch( - &mut pool, - &mut out, - roi.w as usize * 3, - PixelFormat::Rgb8, - roi, - ) - .expect("deep ROI decode"); - - let cache = dec - .cpu_entropy_checkpoints - .lock() - .expect("checkpoint cache mutex"); - assert!(cache - .checkpoints - .iter() - .any(|checkpoint| checkpoint.mcu_index >= CPU_ROI_CHECKPOINT_MIN_TARGET_MCUS)); - } - - #[derive(Default)] - struct GrayRows { - rows: Vec<(u32, Vec)>, - } - - impl OutputWriter for GrayRows { - fn write_rgb_row( - &mut self, - _y: u32, - _r_row: &[u8], - _g_row: &[u8], - _b_row: &[u8], - ) -> Result<(), JpegError> { - unreachable!("gray test writer should not receive rgb rows"); - } - - fn write_ycbcr_row( - &mut self, - _y: u32, - _y_row: &[u8], - _cb_row: &[u8], - _cr_row: &[u8], - ) -> Result<(), JpegError> { - unreachable!("gray test writer should not receive ycbcr rows"); - } - - fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError> { - self.rows.push((y, gray_row.to_vec())); - Ok(()) - } - } - - #[test] - fn cropped_writer_honors_source_window_origin() { - let inner = GrayRows::default(); - let rect = Rect { - x: 6, - y: 1, - w: 2, - h: 1, - }; - let mut writer = CroppedWriter::new(inner, rect, 4, 4); - - writer - .write_gray_row(1, &[10, 20, 30, 40]) - .expect("crop write must succeed"); - - assert_eq!(writer.inner.rows, vec![(0, vec![30, 40])]); - } -} diff --git a/crates/signinum-jpeg/src/lossless/mod.rs b/crates/signinum-jpeg/src/lossless/mod.rs deleted file mode 100644 index ae41aa6f..00000000 --- a/crates/signinum-jpeg/src/lossless/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! lossless module — see lib.rs. diff --git a/crates/signinum-jpeg/src/segment.rs b/crates/signinum-jpeg/src/segment.rs deleted file mode 100644 index 70476143..00000000 --- a/crates/signinum-jpeg/src/segment.rs +++ /dev/null @@ -1,3 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! segment module — see lib.rs. diff --git a/crates/signinum-jpeg/tests/batch.rs b/crates/signinum-jpeg/tests/batch.rs deleted file mode 100644 index b77b2e6c..00000000 --- a/crates/signinum-jpeg/tests/batch.rs +++ /dev/null @@ -1,463 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Batch decode via [`Decoder::decode_tile`]: sequential output must match -//! parallel output byte-for-byte across a worker pool. Validates the -//! Phase 5 tile primitive under `std::thread::scope`. - -use signinum_jpeg::{ - decode_tile_into_in_context, decode_tile_region_scaled_into_in_context, decode_tiles_into, - decode_tiles_into_with_options, decode_tiles_region_scaled_into, decode_tiles_scaled_into, - decode_tiles_scaled_into_with_options, ColorTransform, DecodeOptions, Decoder, DecoderContext, - Downscale, PixelFormat, Rect, RowSink, ScratchPool, TileBatchOptions, TileDecodeJob, - TileRegionScaledDecodeJob, TileScaledDecodeJob, -}; -mod fixtures; -use fixtures::progressive_8x8_jpeg; -use std::num::NonZeroUsize; -use std::thread; - -const BASELINE_420: &[u8] = include_bytes!("../fixtures/conformance/baseline_420_16x16.jpg"); - -const BATCH_SIZE: usize = 100; - -#[derive(Default)] -struct CollectRows { - rows: Vec, -} - -impl RowSink for CollectRows { - type Error = signinum_jpeg::JpegError; - - fn write_row(&mut self, _y: u32, row: &[u8]) -> Result<(), signinum_jpeg::JpegError> { - self.rows.extend_from_slice(row); - Ok(()) - } -} - -fn decode_tile_bytes(bytes: &[u8], ctx: &mut DecoderContext, pool: &mut ScratchPool) -> Vec { - let mut sink = CollectRows::default(); - Decoder::decode_tile(bytes, ctx, pool, &mut sink).expect("Decoder::decode_tile"); - sink.rows -} - -fn decode_tile_rgb8_reference(bytes: &[u8]) -> (Vec, usize) { - let dec = Decoder::new(bytes).expect("fixture decoder"); - let (width, height) = dec.info().dimensions; - let stride = width as usize * 3; - let mut out = vec![0u8; stride * height as usize]; - dec.decode_into(&mut out, stride, PixelFormat::Rgb8) - .expect("fixture decode_into"); - (out, stride) -} - -#[test] -fn production_batch_decode_empty_input_succeeds() { - let mut jobs: Vec> = Vec::new(); - - let outcomes = decode_tiles_into(&mut jobs, PixelFormat::Rgb8, TileBatchOptions::default()) - .expect("empty batch succeeds"); - - assert!(outcomes.is_empty()); -} - -#[test] -fn production_batch_decode_worker_one_matches_single_tile_decode() { - let (expected, stride) = decode_tile_rgb8_reference(BASELINE_420); - let mut actual = vec![0u8; expected.len()]; - let options = TileBatchOptions { - workers: NonZeroUsize::new(1), - }; - - let outcomes = { - let mut jobs = vec![TileDecodeJob { - input: BASELINE_420, - out: actual.as_mut_slice(), - stride, - }]; - decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect("batch decode") - }; - - assert_eq!(outcomes.len(), 1); - assert_eq!(actual, expected); -} - -#[test] -fn production_batch_decode_progressive8_matches_single_tile_decode() { - let bytes = progressive_8x8_jpeg(); - let (expected, stride) = decode_tile_rgb8_reference(&bytes); - let mut actual = vec![0u8; expected.len()]; - - let outcomes = { - let mut jobs = vec![TileDecodeJob { - input: &bytes, - out: actual.as_mut_slice(), - stride, - }]; - decode_tiles_into( - &mut jobs, - PixelFormat::Rgb8, - TileBatchOptions { - workers: NonZeroUsize::new(1), - }, - ) - .expect("progressive batch decode") - }; - - assert_eq!(outcomes.len(), 1); - assert_eq!(actual, expected); -} - -#[test] -fn production_batch_decode_parallel_preserves_order_and_output() { - const JOBS: usize = 32; - let (expected, stride) = decode_tile_rgb8_reference(BASELINE_420); - let mut outputs = (0..JOBS) - .map(|_| vec![0u8; expected.len()]) - .collect::>(); - let options = TileBatchOptions { - workers: NonZeroUsize::new(4), - }; - - let outcomes = { - let mut jobs = outputs - .iter_mut() - .map(|out| TileDecodeJob { - input: BASELINE_420, - out: out.as_mut_slice(), - stride, - }) - .collect::>(); - decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect("batch decode") - }; - - assert_eq!(outcomes.len(), JOBS); - for (index, out) in outputs.iter().enumerate() { - assert_eq!(out, &expected, "tile {index} output diverged"); - } -} - -#[test] -fn production_batch_decode_with_options_preserves_forced_color_transform() { - const JOBS: usize = 8; - let decode_options = DecodeOptions::default().with_color_transform(ColorTransform::ForceRgb); - let dec = Decoder::new_with_options(BASELINE_420, decode_options).expect("fixture decoder"); - let (width, height) = dec.info().dimensions; - let stride = width as usize * 3; - let mut expected = vec![0u8; stride * height as usize]; - dec.decode_into(&mut expected, stride, PixelFormat::Rgb8) - .expect("reference forced-RGB decode"); - let mut outputs = (0..JOBS) - .map(|_| vec![0u8; expected.len()]) - .collect::>(); - let options = TileBatchOptions { - workers: NonZeroUsize::new(2), - }; - - let outcomes = { - let mut jobs = outputs - .iter_mut() - .map(|out| TileDecodeJob { - input: BASELINE_420, - out: out.as_mut_slice(), - stride, - }) - .collect::>(); - decode_tiles_into_with_options(&mut jobs, PixelFormat::Rgb8, decode_options, options) - .expect("batch decode with options") - }; - - assert_eq!(outcomes.len(), JOBS); - for (index, out) in outputs.iter().enumerate() { - assert_eq!(out, &expected, "tile {index} output diverged"); - } -} - -#[test] -fn production_batch_decode_reports_first_failing_tile_index() { - let (expected, stride) = decode_tile_rgb8_reference(BASELINE_420); - let mut outputs = (0..3) - .map(|_| vec![0u8; expected.len()]) - .collect::>(); - let options = TileBatchOptions { - workers: NonZeroUsize::new(2), - }; - - let err = { - let inputs: [&[u8]; 3] = [BASELINE_420, b"not a jpeg", BASELINE_420]; - let mut jobs = inputs - .into_iter() - .zip(outputs.iter_mut()) - .map(|(input, out)| TileDecodeJob { - input, - out: out.as_mut_slice(), - stride, - }) - .collect::>(); - decode_tiles_into(&mut jobs, PixelFormat::Rgb8, options).expect_err("bad tile fails") - }; - - assert_eq!(err.index, 1); -} - -#[test] -fn sequential_and_parallel_batch_produce_identical_output() { - let tiles: Vec<&[u8]> = (0..BATCH_SIZE).map(|_| BASELINE_420).collect(); - - let sequential: Vec> = { - let mut pool = ScratchPool::new(); - let mut ctx = DecoderContext::new(); - tiles - .iter() - .map(|bytes| decode_tile_bytes(bytes, &mut ctx, &mut pool)) - .collect() - }; - - let parallel: Vec> = thread::scope(|scope| { - const WORKERS: usize = 4; - let chunk_size = tiles.len().div_ceil(WORKERS); - let handles: Vec<_> = tiles - .chunks(chunk_size) - .map(|chunk| { - scope.spawn(|| { - let mut pool = ScratchPool::new(); - let mut ctx = DecoderContext::new(); - chunk - .iter() - .map(|bytes| decode_tile_bytes(bytes, &mut ctx, &mut pool)) - .collect::>() - }) - }) - .collect(); - handles - .into_iter() - .flat_map(|h| h.join().expect("worker panicked")) - .collect() - }); - - assert_eq!(sequential.len(), parallel.len()); - for (i, (seq, par)) in sequential.iter().zip(parallel.iter()).enumerate() { - assert_eq!( - seq, par, - "tile {i} diverged between sequential and parallel" - ); - } -} - -#[test] -fn pool_reuse_across_batch_matches_fresh_pool() { - let mut reused_pool = ScratchPool::new(); - let mut reused_ctx = DecoderContext::new(); - let reused_outputs: Vec> = (0..BATCH_SIZE) - .map(|_| decode_tile_bytes(BASELINE_420, &mut reused_ctx, &mut reused_pool)) - .collect(); - - let fresh_outputs: Vec> = (0..BATCH_SIZE) - .map(|_| { - let mut pool = ScratchPool::new(); - let mut ctx = DecoderContext::new(); - decode_tile_bytes(BASELINE_420, &mut ctx, &mut pool) - }) - .collect(); - - for (i, (reused, fresh)) in reused_outputs.iter().zip(fresh_outputs.iter()).enumerate() { - assert_eq!(reused, fresh, "iter {i} reused-pool output diverged"); - } -} - -#[test] -fn tile_buffer_decode_matches_decoder_decode_into() { - let dec = Decoder::new(BASELINE_420).expect("fixture decoder"); - let (width, height) = dec.info().dimensions; - let stride = width as usize * 3; - let mut expected = vec![0u8; stride * height as usize]; - let mut actual = vec![0u8; expected.len()]; - dec.decode_into(&mut expected, stride, PixelFormat::Rgb8) - .expect("baseline decode_into"); - - let mut ctx = DecoderContext::new(); - let mut pool = ScratchPool::new(); - decode_tile_into_in_context( - BASELINE_420, - &mut ctx, - &mut pool, - &mut actual, - stride, - PixelFormat::Rgb8, - ) - .expect("tile decode_into_in_context"); - - assert_eq!(actual, expected); -} - -#[test] -fn tile_region_scaled_decode_matches_decoder_region_decode() { - let dec = Decoder::new(BASELINE_420).expect("fixture decoder"); - let roi = Rect { - x: 4, - y: 4, - w: 8, - h: 8, - }; - let denom = 4; - let scaled_w = (roi.x + roi.w).div_ceil(denom) - roi.x / denom; - let scaled_h = (roi.y + roi.h).div_ceil(denom) - roi.y / denom; - let stride = scaled_w as usize * 3; - let mut expected = vec![0u8; stride * scaled_h as usize]; - let mut actual = vec![0u8; expected.len()]; - dec.decode_region_scaled_into( - &mut expected, - stride, - PixelFormat::Rgb8, - roi, - Downscale::Quarter, - ) - .expect("core region-scaled decode"); - - let mut ctx = DecoderContext::new(); - let mut pool = ScratchPool::new(); - decode_tile_region_scaled_into_in_context( - BASELINE_420, - &mut ctx, - &mut pool, - &mut actual, - stride, - PixelFormat::Rgb8, - roi, - Downscale::Quarter, - ) - .expect("tile region decode_into_in_context"); - - assert_eq!(actual, expected); -} - -#[test] -fn production_batch_region_scaled_decode_parallel_preserves_order_and_output() { - const JOBS: usize = 32; - let dec = Decoder::new(BASELINE_420).expect("fixture decoder"); - let roi = Rect { - x: 4, - y: 4, - w: 8, - h: 8, - }; - let denom = 4; - let scaled_w = (roi.x + roi.w).div_ceil(denom) - roi.x / denom; - let scaled_h = (roi.y + roi.h).div_ceil(denom) - roi.y / denom; - let stride = scaled_w as usize * 3; - let mut expected = vec![0u8; stride * scaled_h as usize]; - dec.decode_region_scaled_into( - &mut expected, - stride, - PixelFormat::Rgb8, - roi, - Downscale::Quarter, - ) - .expect("reference region-scaled decode"); - let mut outputs = (0..JOBS) - .map(|_| vec![0u8; expected.len()]) - .collect::>(); - let options = TileBatchOptions { - workers: NonZeroUsize::new(4), - }; - - let outcomes = { - let mut jobs = outputs - .iter_mut() - .map(|out| TileRegionScaledDecodeJob { - input: BASELINE_420, - out: out.as_mut_slice(), - stride, - roi, - scale: Downscale::Quarter, - }) - .collect::>(); - decode_tiles_region_scaled_into(&mut jobs, PixelFormat::Rgb8, options) - .expect("batch region-scaled decode") - }; - - assert_eq!(outcomes.len(), JOBS); - for (index, out) in outputs.iter().enumerate() { - assert_eq!(out, &expected, "tile {index} output diverged"); - } -} - -#[test] -fn production_batch_scaled_decode_parallel_preserves_order_and_output() { - const JOBS: usize = 32; - let dec = Decoder::new(BASELINE_420).expect("fixture decoder"); - let scale = Downscale::Quarter; - let denom = 4; - let (width, height) = dec.info().dimensions; - let scaled_w = width.div_ceil(denom); - let scaled_h = height.div_ceil(denom); - let stride = scaled_w as usize * 3; - let mut expected = vec![0u8; stride * scaled_h as usize]; - dec.decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, scale) - .expect("reference scaled decode"); - let mut outputs = (0..JOBS) - .map(|_| vec![0u8; expected.len()]) - .collect::>(); - let options = TileBatchOptions { - workers: NonZeroUsize::new(4), - }; - - let outcomes = { - let mut jobs = outputs - .iter_mut() - .map(|out| TileScaledDecodeJob { - input: BASELINE_420, - out: out.as_mut_slice(), - stride, - scale, - }) - .collect::>(); - decode_tiles_scaled_into(&mut jobs, PixelFormat::Rgb8, options) - .expect("batch scaled decode") - }; - - assert_eq!(outcomes.len(), JOBS); - for (index, out) in outputs.iter().enumerate() { - assert_eq!(out, &expected, "tile {index} output diverged"); - } -} - -#[test] -fn production_batch_scaled_decode_with_options_preserves_forced_color_transform() { - const JOBS: usize = 8; - let decode_options = DecodeOptions::default().with_color_transform(ColorTransform::ForceRgb); - let dec = Decoder::new_with_options(BASELINE_420, decode_options).expect("fixture decoder"); - let scale = Downscale::Quarter; - let denom = 4; - let (width, height) = dec.info().dimensions; - let scaled_w = width.div_ceil(denom); - let scaled_h = height.div_ceil(denom); - let stride = scaled_w as usize * 3; - let mut expected = vec![0u8; stride * scaled_h as usize]; - dec.decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, scale) - .expect("reference scaled forced-RGB decode"); - let mut outputs = (0..JOBS) - .map(|_| vec![0u8; expected.len()]) - .collect::>(); - let options = TileBatchOptions { - workers: NonZeroUsize::new(2), - }; - - let outcomes = { - let mut jobs = outputs - .iter_mut() - .map(|out| TileScaledDecodeJob { - input: BASELINE_420, - out: out.as_mut_slice(), - stride, - scale, - }) - .collect::>(); - decode_tiles_scaled_into_with_options(&mut jobs, PixelFormat::Rgb8, decode_options, options) - .expect("batch scaled decode with options") - }; - - assert_eq!(outcomes.len(), JOBS); - for (index, out) in outputs.iter().enumerate() { - assert_eq!(out, &expected, "tile {index} output diverged"); - } -} diff --git a/crates/signinum-jpeg/tests/decode_into.rs b/crates/signinum-jpeg/tests/decode_into.rs deleted file mode 100644 index f6dbdecd..00000000 --- a/crates/signinum-jpeg/tests/decode_into.rs +++ /dev/null @@ -1,381 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Integration tests for `Decoder::decode_into`. - -use signinum_jpeg::{Decoder, Downscale, JpegError, PixelFormat, Rect}; - -mod fixtures; -use fixtures::{ - grayscale_8x8_jpeg, minimal_baseline_420_jpeg, progressive_8x8_jpeg, rgb_app14_8x8_jpeg, - rgb_app14_8x8_rgb, -}; - -fn minimal_cmyk_baseline_jpeg() -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0xff, 0xd8]); - bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); - bytes.extend(std::iter::repeat_n(1u8, 64)); - bytes.extend_from_slice(&[ - 0xff, 0xc0, 0x00, 20, 8, 0, 8, 0, 8, 4, 1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0, 4, 0x11, 0, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xaa, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xbb, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xda, 0x00, 0x0e, 4, 1, 0x00, 2, 0x00, 3, 0x00, 4, 0x00, 0, 63, 0, 0x00, 0xff, 0xd9, - ]); - bytes -} - -#[test] -fn decode_into_rgb8_returns_decoded_rect_full_image() { - let bytes = minimal_baseline_420_jpeg(); - let dec = Decoder::new(&bytes).expect("baseline 4:2:0 must construct"); - let (w, h) = dec.info().dimensions; - let mut buf = vec![0u8; (w * h * 3) as usize]; - let outcome = dec - .decode_into(&mut buf, (w * 3) as usize, PixelFormat::Rgb8) - .expect("baseline 4:2:0 decode must succeed"); - assert_eq!(outcome.decoded.w, w); - assert_eq!(outcome.decoded.h, h); -} - -#[test] -fn decode_owned_rgb8_matches_decode_into() { - let bytes = minimal_baseline_420_jpeg(); - let dec = Decoder::new(&bytes).expect("baseline 4:2:0 must construct"); - let (w, h) = dec.info().dimensions; - let mut expected = vec![0u8; (w * h * 3) as usize]; - let expected_outcome = dec - .decode_into(&mut expected, (w * 3) as usize, PixelFormat::Rgb8) - .expect("baseline 4:2:0 decode must succeed"); - - let (owned, outcome) = dec.decode(PixelFormat::Rgb8).unwrap(); - assert_eq!(owned, expected); - assert_eq!(outcome, expected_outcome); -} - -#[test] -fn decode_into_rgba8_writes_alpha_byte() { - let bytes = minimal_baseline_420_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let (w, h) = dec.info().dimensions; - let mut buf = vec![0u8; (w * h * 4) as usize]; - dec.decode_rgba8_into_with_alpha(&mut buf, (w * 4) as usize, 200) - .unwrap(); - for y in 0..h as usize { - for x in 0..w as usize { - let idx = (y * w as usize + x) * 4; - assert_eq!(buf[idx + 3], 200, "pixel ({x},{y}) alpha"); - } - } -} - -#[test] -fn decode_into_rgba8_defaults_alpha_to_255() { - let bytes = minimal_baseline_420_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let (w, h) = dec.info().dimensions; - let mut buf = vec![0u8; (w * h * 4) as usize]; - dec.decode_into(&mut buf, (w * 4) as usize, PixelFormat::Rgba8) - .unwrap(); - for y in 0..h as usize { - for x in 0..w as usize { - let idx = (y * w as usize + x) * 4; - assert_eq!(buf[idx + 3], 255, "pixel ({x},{y}) alpha"); - } - } -} - -#[test] -fn decode_owned_region_scaled_matches_decode_region_into() { - let bytes = rgb_app14_8x8_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let roi = Rect { - x: 2, - y: 2, - w: 4, - h: 4, - }; - let mut expected = vec![0u8; 2 * 2 * 3]; - let expected_outcome = dec - .decode_region_scaled_into( - &mut expected, - 2 * 3, - PixelFormat::Rgb8, - roi, - Downscale::Half, - ) - .unwrap(); - - let (owned, outcome) = dec - .decode_region_scaled(PixelFormat::Rgb8, roi, Downscale::Half) - .unwrap(); - assert_eq!(owned, expected); - assert_eq!(outcome, expected_outcome); -} - -#[test] -fn decode_owned_scaled_matches_decode_scaled_into() { - let bytes = rgb_app14_8x8_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let mut expected = vec![0u8; 4 * 4 * 3]; - let expected_outcome = dec - .decode_scaled_into(&mut expected, 4 * 3, PixelFormat::Rgb8, Downscale::Half) - .unwrap(); - - let (owned, outcome) = dec - .decode_scaled(PixelFormat::Rgb8, Downscale::Half) - .unwrap(); - assert_eq!(owned, expected); - assert_eq!(outcome, expected_outcome); -} - -#[test] -fn full_tile_region_scaled_matches_full_scaled_decode() { - let bytes = minimal_baseline_420_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let (w, h) = dec.info().dimensions; - let roi = Rect { x: 0, y: 0, w, h }; - let stride = w.div_ceil(4) as usize * 3; - let mut expected = vec![0u8; stride * h.div_ceil(4) as usize]; - let expected_outcome = dec - .decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, Downscale::Quarter) - .unwrap(); - let mut actual = vec![0u8; expected.len()]; - - let actual_outcome = dec - .decode_region_scaled_into( - &mut actual, - stride, - PixelFormat::Rgb8, - roi, - Downscale::Quarter, - ) - .unwrap(); - - assert_eq!(actual, expected); - assert_eq!(actual_outcome, expected_outcome); - assert_eq!(actual_outcome.decoded, roi); -} - -#[test] -fn decode_into_gray8_produces_single_byte_per_pixel() { - let bytes = grayscale_8x8_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let (w, h) = dec.info().dimensions; - assert_eq!((w, h), (8, 8)); - let mut buf = vec![0u8; (w * h) as usize]; - let outcome = dec - .decode_into(&mut buf, w as usize, PixelFormat::Gray8) - .unwrap(); - assert_eq!(outcome.decoded.w, 8); - assert!(buf.iter().any(|&b| b != 0), "expected non-zero pixels"); -} - -#[test] -fn decode_into_rejects_undersized_buffer_with_api_misuse_error() { - let bytes = minimal_baseline_420_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let mut buf = vec![0u8; 4]; - let err = dec - .decode_into(&mut buf, 48, PixelFormat::Rgb8) - .unwrap_err(); - assert!(err.is_api_misuse()); - assert!(matches!(err, JpegError::OutputBufferTooSmall { .. })); -} - -#[test] -fn decode_into_rejects_stride_narrower_than_row_width() { - let bytes = minimal_baseline_420_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let mut buf = vec![0u8; 16 * 16 * 3]; - let err = dec - .decode_into(&mut buf, 10, PixelFormat::Rgb8) - .unwrap_err(); - assert!(err.is_api_misuse()); - assert!(matches!(err, JpegError::InvalidStride { .. })); -} - -#[test] -fn decode_into_tolerates_padded_stride() { - let bytes = minimal_baseline_420_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let (w, h) = dec.info().dimensions; - let padded_stride = (w as usize * 3) + 32; - let mut buf = vec![0xAAu8; padded_stride * h as usize]; - dec.decode_into(&mut buf, padded_stride, PixelFormat::Rgb8) - .unwrap(); - let last_row_start = (h as usize - 1) * padded_stride; - let last_row_end = last_row_start + w as usize * 3; - assert_eq!( - &buf[last_row_end..last_row_end + 16], - &[0xAA; 16], - "stride padding must not be overwritten" - ); -} - -#[test] -fn decode_into_rgb8_preserves_app14_rgb_pixels() { - let bytes = rgb_app14_8x8_jpeg(); - let dec = Decoder::new(&bytes).expect("APP14 RGB fixture must construct"); - let (w, h) = dec.info().dimensions; - assert_eq!((w, h), (8, 8)); - let mut buf = vec![0u8; (w * h * 3) as usize]; - dec.decode_into(&mut buf, (w * 3) as usize, PixelFormat::Rgb8) - .expect("APP14 RGB decode must succeed"); - assert_eq!(buf, rgb_app14_8x8_rgb()); -} - -#[test] -fn decode_into_rgb8_scaled_preserves_constant_app14_rgb_pixels() { - let bytes = rgb_app14_8x8_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - - for (factor, dims) in [ - (Downscale::Half, (4u32, 4u32)), - (Downscale::Quarter, (2u32, 2u32)), - (Downscale::Eighth, (1u32, 1u32)), - ] { - let mut buf = vec![0u8; dims.0 as usize * dims.1 as usize * 3]; - dec.decode_scaled_into(&mut buf, dims.0 as usize * 3, PixelFormat::Rgb8, factor) - .unwrap(); - let mut expected = Vec::with_capacity(buf.len()); - for _ in 0..(dims.0 * dims.1) { - expected.extend_from_slice(&[200, 20, 10]); - } - assert_eq!(buf, expected, "factor={factor:?}"); - } -} - -#[test] -fn decode_into_gray8_scaled_projects_constant_app14_rgb_pixels() { - let bytes = rgb_app14_8x8_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let expected = ((77 * 200 + 150 * 20 + 29 * 10 + 128) >> 8) as u8; - - for (factor, dims) in [ - (Downscale::Half, (4u32, 4u32)), - (Downscale::Quarter, (2u32, 2u32)), - (Downscale::Eighth, (1u32, 1u32)), - ] { - let mut buf = vec![0u8; dims.0 as usize * dims.1 as usize]; - dec.decode_scaled_into(&mut buf, dims.0 as usize, PixelFormat::Gray8, factor) - .unwrap(); - assert!(buf.iter().all(|&px| px == expected), "factor={factor:?}"); - } -} - -#[test] -fn decoder_new_accepts_progressive8() { - let bytes = progressive_8x8_jpeg(); - let dec = Decoder::new(&bytes).expect("progressive 8-bit JPEG must construct"); - - assert_eq!(dec.info().dimensions, (8, 8)); -} - -#[test] -fn decode_progressive8_rgb8_matches_jpeg_decoder_reference() { - let bytes = progressive_8x8_jpeg(); - let mut reference_decoder = jpeg_decoder::Decoder::new(std::io::Cursor::new(&bytes)); - let reference = reference_decoder - .decode() - .expect("jpeg-decoder reference decode"); - let info = reference_decoder.info().expect("jpeg-decoder info"); - assert_eq!((info.width, info.height), (8, 8)); - - let dec = Decoder::new(&bytes).expect("progressive 8-bit JPEG must construct"); - let (w, h) = dec.info().dimensions; - let mut actual = vec![0u8; (w * h * 3) as usize]; - dec.decode_into(&mut actual, (w * 3) as usize, PixelFormat::Rgb8) - .expect("progressive decode must succeed"); - - assert_eq!(actual.len(), reference.len()); - let max_delta = actual - .iter() - .zip(reference.iter()) - .map(|(&a, &b)| a.abs_diff(b)) - .max() - .unwrap_or(0); - assert!( - max_delta <= 3, - "progressive RGB max channel delta {max_delta} exceeds tolerance" - ); -} - -#[test] -fn decode_region_into_rgb8_crops_constant_app14_rgb_pixels() { - let bytes = rgb_app14_8x8_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let roi = Rect { - x: 2, - y: 1, - w: 3, - h: 4, - }; - let mut buf = vec![0u8; roi.w as usize * roi.h as usize * 3]; - let outcome = dec - .decode_region_into(&mut buf, roi.w as usize * 3, PixelFormat::Rgb8, roi) - .unwrap(); - assert_eq!(outcome.decoded, roi); - let mut expected = Vec::with_capacity(buf.len()); - for _ in 0..(roi.w * roi.h) { - expected.extend_from_slice(&[200, 20, 10]); - } - assert_eq!(buf, expected); -} - -#[test] -fn decode_region_into_rgb8_scaled_crops_constant_app14_rgb_pixels() { - let bytes = rgb_app14_8x8_jpeg(); - let dec = Decoder::new(&bytes).unwrap(); - let roi = Rect { - x: 2, - y: 2, - w: 4, - h: 4, - }; - let mut buf = vec![0u8; 2 * 2 * 3]; - let outcome = dec - .decode_region_scaled_into(&mut buf, 2 * 3, PixelFormat::Rgb8, roi, Downscale::Half) - .unwrap(); - assert_eq!(outcome.decoded, roi); - let mut expected = Vec::with_capacity(buf.len()); - for _ in 0..4 { - expected.extend_from_slice(&[200, 20, 10]); - } - assert_eq!(buf, expected); -} - -#[test] -fn decoder_new_rejects_cmyk_baseline_as_unsupported() { - let bytes = minimal_cmyk_baseline_jpeg(); - let err = Decoder::new(&bytes).expect_err("CMYK should not reach scalar decoder"); - assert!(matches!(err, JpegError::UnsupportedColorSpace { .. })); - assert!(err.is_unsupported()); -} - -#[test] -fn decoder_new_rejects_invalid_sequential_scan_parameters() { - let mut bytes = minimal_baseline_420_jpeg(); - let sos = bytes - .windows(2) - .position(|w| w == [0xff, 0xda]) - .expect("fixture SOS"); - bytes[sos + 2 + 2 + 1 + 3 * 2] = 1; - - let err = Decoder::new(&bytes).expect_err("baseline Ss=1 must be rejected"); - assert!(matches!( - err, - JpegError::InvalidScanParameters { - ss: 1, - se: 63, - ah: 0, - al: 0, - .. - } - )); -} diff --git a/crates/signinum-jpeg/tests/device_plan.rs b/crates/signinum-jpeg/tests/device_plan.rs deleted file mode 100644 index 6213b711..00000000 --- a/crates/signinum-jpeg/tests/device_plan.rs +++ /dev/null @@ -1,370 +0,0 @@ -use std::borrow::Cow; - -use signinum_jpeg::{ColorSpace, Decoder, Warning}; - -const BASELINE_420: &[u8] = include_bytes!("../fixtures/conformance/baseline_420_16x16.jpg"); -const BASELINE_422: &[u8] = include_bytes!("../fixtures/conformance/baseline_422_16x8.jpg"); - -#[test] -fn adapter_device_plan_exposes_scan_metadata() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let plan = signinum_jpeg::adapter::build_device_plan(&decoder, 4).expect("device plan"); - - assert_eq!(plan.dimensions, (16, 16)); - assert_eq!(plan.color_space, ColorSpace::YCbCr); - assert_eq!(plan.components.len(), 3); - assert_eq!(plan.checkpoints[0].mcu_index, 0); - assert!(!plan.scan_bytes.is_empty()); -} - -#[test] -fn adapter_device_plan_borrows_scan_bytes_for_well_formed_streams() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let plan = signinum_jpeg::adapter::build_device_plan(&decoder, 4).expect("device plan"); - - assert!(matches!(plan.scan_bytes, Cow::Borrowed(_))); -} - -#[test] -fn adapter_device_plan_keeps_fast_420_shape_information() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let plan = signinum_jpeg::adapter::build_device_plan(&decoder, 4).expect("device plan"); - - assert!(plan.matches_fast_420); - assert!(!plan.matches_fast_422); - assert!(!plan.matches_fast_444); -} - -#[test] -fn adapter_device_plan_keeps_fast_422_shape_information() { - let decoder = Decoder::new(BASELINE_422).expect("decoder"); - let plan = signinum_jpeg::adapter::build_device_plan(&decoder, 4).expect("device plan"); - - assert!(!plan.matches_fast_420); - assert!(plan.matches_fast_422); - assert!(!plan.matches_fast_444); -} - -#[test] -fn adapter_device_plan_scan_bytes_keep_terminal_eoi() { - let decoder = Decoder::new(BASELINE_420).expect("decoder"); - let plan = signinum_jpeg::adapter::build_device_plan(&decoder, 4).expect("device plan"); - - assert!(plan.scan_bytes.ends_with(&[0xff, 0xd9])); -} - -#[test] -fn adapter_device_plan_checkpoint_cadence_handles_multi_mcu_inputs() { - let bytes = grayscale_jpeg(24, 24); - let decoder = Decoder::new(&bytes).expect("grayscale decoder"); - - let cadence_zero = - signinum_jpeg::adapter::build_device_plan(&decoder, 0).expect("zero-cadence plan"); - let cadence_two = - signinum_jpeg::adapter::build_device_plan(&decoder, 2).expect("cadence-two plan"); - - assert_eq!( - cadence_zero - .checkpoints - .iter() - .map(|checkpoint| checkpoint.mcu_index) - .collect::>(), - vec![0, 1, 2, 3, 4, 5, 6, 7, 8] - ); - let zero_offsets = cadence_zero - .checkpoints - .iter() - .map(|checkpoint| checkpoint.scan_offset) - .collect::>(); - assert_eq!(zero_offsets.first(), Some(&0)); - assert!(zero_offsets.windows(2).all(|pair| pair[0] <= pair[1])); - assert_eq!( - cadence_two - .checkpoints - .iter() - .map(|checkpoint| checkpoint.mcu_index) - .collect::>(), - vec![0, 2, 4, 6, 8] - ); - let cadence_two_offsets = cadence_two - .checkpoints - .iter() - .map(|checkpoint| checkpoint.scan_offset) - .collect::>(); - assert_eq!(cadence_two_offsets.first(), Some(&0)); - assert!(cadence_two_offsets - .windows(2) - .all(|pair| pair[0] <= pair[1])); - assert!(cadence_two - .checkpoints - .iter() - .all(|checkpoint| checkpoint.bits_buffered <= 64 && checkpoint.expected_rst == 0)); -} - -#[test] -fn adapter_device_plan_restart_checkpoints_capture_resume_state() { - let bytes = restart_coded_grayscale_jpeg(24, 24); - let decoder = Decoder::new(&bytes).expect("restart-coded decoder"); - let plan = signinum_jpeg::adapter::build_device_plan(&decoder, 2).expect("device plan"); - - assert_eq!( - plan.checkpoints - .iter() - .map(|checkpoint| checkpoint.mcu_index) - .collect::>(), - vec![0, 1, 2, 3, 4, 5, 6, 7, 8] - ); - assert_eq!( - plan.checkpoints - .iter() - .map(|checkpoint| checkpoint.scan_offset) - .collect::>(), - vec![0, 3, 6, 9, 12, 15, 18, 21, 24] - ); - assert_eq!( - plan.checkpoints - .iter() - .map(|checkpoint| checkpoint.expected_rst) - .collect::>(), - vec![0, 1, 2, 3, 4, 5, 6, 7, 0] - ); - assert!(plan - .checkpoints - .iter() - .all(|checkpoint| checkpoint.bits_buffered == 0 && checkpoint.prev_dc == [0; 4])); -} - -#[test] -fn adapter_device_plan_treats_dri_zero_as_non_restart_fast_path() { - let bytes = insert_restart_interval(BASELINE_420.to_vec(), 0); - let decoder = Decoder::new(&bytes).expect("decoder"); - let plan = signinum_jpeg::adapter::build_device_plan(&decoder, 2).expect("device plan"); - - assert_eq!(plan.restart_interval, None); - assert!(plan.matches_fast_420); - assert_eq!( - plan.checkpoints - .iter() - .map(|checkpoint| checkpoint.expected_rst) - .collect::>(), - vec![0; plan.checkpoints.len()] - ); -} - -#[test] -fn adapter_device_plan_handles_restart_after_partial_entropy_byte() { - let bytes = grayscale_restart_jpeg(); - let decoder = Decoder::new(&bytes).expect("restart-coded decoder"); - let plan = signinum_jpeg::adapter::build_device_plan(&decoder, 2).expect("device plan"); - - assert_eq!(plan.checkpoints.len(), 2); - assert_eq!(plan.checkpoints[1].mcu_index, 1); - assert_eq!(plan.checkpoints[1].scan_offset, 3); - assert_eq!(plan.checkpoints[1].expected_rst, 1); -} - -#[test] -fn adapter_device_plan_surfaces_missing_eoi_warning() { - let mut bytes = grayscale_jpeg(24, 24); - bytes.truncate(bytes.len() - 2); - - let decoder = Decoder::new(&bytes).expect("decoder"); - let plan = signinum_jpeg::adapter::build_device_plan(&decoder, 2) - .expect("missing EOI should remain decodable"); - - assert!(plan.warnings.contains(&Warning::MissingEoi)); -} - -#[test] -fn adapter_device_plan_treats_trailing_ff_as_missing_eoi() { - let mut bytes = grayscale_jpeg(24, 24); - bytes.truncate(bytes.len() - 1); - - let decoder = Decoder::new(&bytes).expect("decoder"); - let plan = signinum_jpeg::adapter::build_device_plan(&decoder, 2) - .expect("trailing FF should remain decodable"); - - assert!(plan.warnings.contains(&Warning::MissingEoi)); - assert_eq!(plan.scan_bytes.last(), Some(&0xff)); -} - -#[test] -fn adapter_device_plan_rejects_non_eoi_marker_after_entropy() { - let mut bytes = restart_coded_grayscale_jpeg(24, 24); - let marker = bytes - .windows(2) - .position(|window| matches!(window, [0xff, 0xd0..=0xd7])) - .expect("restart marker"); - bytes[marker + 1] = 0xe0; - - let decoder = Decoder::new(&bytes).expect("restart-coded decoder"); - let err = signinum_jpeg::adapter::build_device_plan(&decoder, 2) - .expect_err("unexpected marker should fail"); - - assert!(matches!( - err, - signinum_jpeg::JpegError::UnexpectedMarker { - expected: signinum_jpeg::MarkerKind::Eoi, - found: 0xe0, - .. - } - )); -} - -#[test] -fn adapter_device_plan_rejects_restart_marker_without_dri() { - let bytes = insert_entropy_marker(BASELINE_420.to_vec(), 0xd0); - let decoder = Decoder::new(&bytes).expect("decoder"); - let err = signinum_jpeg::adapter::build_device_plan(&decoder, 2) - .expect_err("restart marker without DRI must fail"); - - assert!(matches!( - err, - signinum_jpeg::JpegError::UnexpectedMarker { - expected: signinum_jpeg::MarkerKind::Eoi, - found: 0xd0, - .. - } - )); -} - -#[test] -fn adapter_device_plan_rejects_doubled_ff_before_terminal_eoi() { - let mut bytes = grayscale_jpeg(24, 24); - bytes.insert(bytes.len() - 1, 0xff); - - let decoder = Decoder::new(&bytes).expect("decoder"); - let err = signinum_jpeg::adapter::build_device_plan(&decoder, 2) - .expect_err("double-FF terminal marker should fail"); - - assert!(matches!( - err, - signinum_jpeg::JpegError::UnexpectedMarker { - expected: signinum_jpeg::MarkerKind::Eoi, - found: 0xff, - .. - } - )); -} - -fn restart_coded_grayscale_jpeg(width: u16, height: u16) -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0xff, 0xd8]); - bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); - bytes.extend(std::iter::repeat_n(16u8, 64)); - bytes.extend_from_slice(&[ - 0xff, - 0xc0, - 0x00, - 11, - 8, - (height >> 8) as u8, - height as u8, - (width >> 8) as u8, - width as u8, - 1, - 1, - 0x11, - 0, - ]); - bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); - - let mcu_cols = u32::from(width).div_ceil(8); - let mcu_rows = u32::from(height).div_ceil(8); - let mcu_count = (mcu_cols * mcu_rows) as usize; - for mcu in 0..mcu_count { - bytes.push(0x00); - if mcu + 1 != mcu_count { - bytes.extend_from_slice(&[0xff, 0xd0 | ((mcu as u8) & 0x07)]); - } - } - - bytes.extend_from_slice(&[0xff, 0xd9]); - bytes -} - -fn grayscale_jpeg(width: u16, height: u16) -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0xff, 0xd8]); - bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); - bytes.extend(std::iter::repeat_n(16u8, 64)); - bytes.extend_from_slice(&[ - 0xff, - 0xc0, - 0x00, - 11, - 8, - (height >> 8) as u8, - height as u8, - (width >> 8) as u8, - width as u8, - 1, - 1, - 0x11, - 0, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); - - let mcu_cols = u32::from(width).div_ceil(8); - let mcu_rows = u32::from(height).div_ceil(8); - let mcu_count = (mcu_cols * mcu_rows) as usize; - bytes.extend(std::iter::repeat_n(0x00, mcu_count)); - - bytes.extend_from_slice(&[0xff, 0xd9]); - bytes -} - -fn grayscale_restart_jpeg() -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0xff, 0xd8]); - bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); - bytes.extend(std::iter::repeat_n(16u8, 64)); - bytes.extend_from_slice(&[0xff, 0xc0, 0x00, 11, 8, 0, 8, 0, 16, 1, 1, 0x11, 0]); - bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); - bytes.extend_from_slice(&[0x00, 0xff, 0xd0, 0x00, 0xff, 0xd9]); - bytes -} - -fn insert_restart_interval(mut bytes: Vec, interval: u16) -> Vec { - let sos = bytes - .windows(2) - .position(|window| window == [0xff, 0xda]) - .expect("SOS marker"); - bytes.splice( - sos..sos, - [ - 0xff, - 0xdd, - 0x00, - 0x04, - (interval >> 8) as u8, - interval as u8, - ], - ); - bytes -} - -fn insert_entropy_marker(mut bytes: Vec, marker: u8) -> Vec { - bytes.splice(bytes.len() - 2..bytes.len() - 2, [0xff, marker]); - bytes -} diff --git a/crates/signinum-jpeg/tests/inspect.rs b/crates/signinum-jpeg/tests/inspect.rs deleted file mode 100644 index fc438fc9..00000000 --- a/crates/signinum-jpeg/tests/inspect.rs +++ /dev/null @@ -1,341 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Integration tests for `Decoder::inspect`. - -use signinum_jpeg::{ - ColorSpace, ColorTransform, DecodeOptions, Decoder, JpegError, JpegView, McuGeometry, - RestartSegment, SofKind, -}; -use signinum_jpeg::{ - CompressedPayloadKind, CompressedTransferSyntax, PassthroughDecision, PassthroughRequirements, -}; - -mod fixtures; -use fixtures::progressive_8x8_jpeg; - -fn minimal_baseline_jpeg() -> Vec { - // Same construction as parse::header::tests — duplicated here because - // integration tests cannot access pub(crate) helpers. - let mut v = Vec::new(); - v.extend_from_slice(&[0xFF, 0xD8]); - v.extend_from_slice(&[0xFF, 0xDB, 0x00, 67, 0x00]); - v.extend(core::iter::repeat_n(1u8, 64)); - v.extend_from_slice(&[ - 0xFF, - 0xC0, - 0x00, - 17, - 8, - 0, - 16, - 0, - 16, - 3, - 1, - (2 << 4) | 2, - 0, - 2, - (1 << 4) | 1, - 0, - 3, - (1 << 4) | 1, - 0, - ]); - // DHT length = 2 (length field) + 1 (Tc/Th) + 16 (bits[]) + 1 (value) = 20 - v.extend_from_slice(&[ - 0xFF, 0xC4, 0x00, 20, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xAA, - ]); - v.extend_from_slice(&[ - 0xFF, 0xC4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xBB, - ]); - v.extend_from_slice(&[0xFF, 0xDA, 0x00, 12, 3, 1, 0x00, 2, 0x00, 3, 0x00, 0, 63, 0]); - v.extend_from_slice(&[0x00, 0xFF, 0xD9]); - v -} - -fn minimal_baseline_jpeg_with_restart_interval(interval: u16) -> Vec { - let mut bytes = minimal_baseline_jpeg(); - let sos_pos = bytes - .windows(2) - .position(|window| window == [0xff, 0xda]) - .expect("SOS marker"); - bytes.splice( - sos_pos..sos_pos, - [ - 0xff, - 0xdd, - 0x00, - 0x04, - (interval >> 8) as u8, - interval as u8, - ], - ); - bytes -} - -fn restart_coded_grayscale_jpeg(width: u16, height: u16) -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0xff, 0xd8]); - bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); - bytes.extend(std::iter::repeat_n(16u8, 64)); - bytes.extend_from_slice(&[ - 0xff, - 0xc0, - 0x00, - 11, - 8, - (height >> 8) as u8, - height as u8, - (width >> 8) as u8, - width as u8, - 1, - 1, - 0x11, - 0, - ]); - bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); - - let mcu_cols = u32::from(width).div_ceil(8); - let mcu_rows = u32::from(height).div_ceil(8); - let mcu_count = (mcu_cols * mcu_rows) as usize; - for mcu in 0..mcu_count { - bytes.push(0x00); - if mcu + 1 != mcu_count { - bytes.extend_from_slice(&[0xff, 0xd0 | ((mcu as u8) & 0x07)]); - } - } - - bytes.extend_from_slice(&[0xff, 0xd9]); - bytes -} - -fn scan_data_offset(bytes: &[u8]) -> usize { - let sos_pos = bytes - .windows(2) - .position(|window| window == [0xff, 0xda]) - .expect("SOS marker"); - let len = u16::from_be_bytes([bytes[sos_pos + 2], bytes[sos_pos + 3]]) as usize; - sos_pos + 2 + len -} - -fn restart_marker_offsets(bytes: &[u8]) -> Vec { - bytes - .windows(2) - .enumerate() - .filter_map(|(offset, window)| { - (window[0] == 0xff && (0xd0..=0xd7).contains(&window[1])).then_some(offset) - }) - .collect() -} - -#[test] -fn inspect_returns_info_for_valid_baseline_jpeg() { - let info = Decoder::inspect(&minimal_baseline_jpeg()).unwrap(); - assert_eq!(info.dimensions, (16, 16)); - assert_eq!(info.sof_kind, SofKind::Baseline8); - assert_eq!(info.color_space, ColorSpace::YCbCr); - assert_eq!(info.bit_depth, 8); - assert!(info.restart_interval.is_none()); - assert_eq!( - info.mcu_geometry, - McuGeometry { - width: 16, - height: 16, - columns: 1, - rows: 1, - count: 1, - } - ); - assert_eq!(info.scan_count, 1, "single SOS → scan_count must be 1"); -} - -#[test] -fn decode_options_color_transform_setter_round_trips() { - let mut options = DecodeOptions::default(); - options.set_color_transform(ColorTransform::ForceRgb); - assert!(matches!( - options.color_transform(), - ColorTransform::ForceRgb - )); -} - -#[test] -fn inspect_with_options_forces_three_component_color_space() { - let bytes = minimal_baseline_jpeg(); - let auto = Decoder::inspect(&bytes).unwrap(); - assert_eq!(auto.color_space, ColorSpace::YCbCr); - - let force_rgb = Decoder::inspect_with_options( - &bytes, - DecodeOptions::default().with_color_transform(ColorTransform::ForceRgb), - ) - .unwrap(); - assert_eq!(force_rgb.color_space, ColorSpace::Rgb); - - let force_ycbcr = Decoder::inspect_with_options( - &bytes, - DecodeOptions::default().with_color_transform(ColorTransform::ForceYCbCr), - ) - .unwrap(); - assert_eq!(force_ycbcr.color_space, ColorSpace::YCbCr); -} - -#[test] -fn inspect_returns_typed_error_for_empty_input() { - let err = Decoder::inspect(&[]).unwrap_err(); - assert!(matches!(err, JpegError::Truncated { .. })); -} - -#[test] -fn inspect_returns_typed_error_for_missing_sof() { - // SOI + EOI, nothing between - let bytes = &[0xFF, 0xD8, 0xFF, 0xD9]; - let err = Decoder::inspect(bytes).unwrap_err(); - assert!(matches!(err, JpegError::MissingMarker { .. })); -} - -#[test] -fn inspect_returns_typed_error_for_arithmetic_coding() { - // Swap SOF0 → SOF9 in the minimal JPEG - let mut bytes = minimal_baseline_jpeg(); - let pos = bytes.windows(2).position(|w| w == [0xFF, 0xC0]).unwrap(); - bytes[pos + 1] = 0xC9; - let err = Decoder::inspect(&bytes).unwrap_err(); - assert!(err.is_unsupported()); -} - -#[test] -fn inspect_is_api_misuse_predicate_negative_for_all_parse_errors() { - // Parse errors are never API misuse. - let err = Decoder::inspect(&[]).unwrap_err(); - assert!(!err.is_api_misuse()); -} - -#[test] -fn inspect_reports_all_progressive_scans() { - let info = Decoder::inspect(&progressive_8x8_jpeg()).unwrap(); - assert_eq!(info.sof_kind, SofKind::Progressive8); - assert_eq!(info.scan_count, 10); -} - -#[test] -fn inspect_treats_dri_zero_as_no_restart_interval() { - let info = Decoder::inspect(&minimal_baseline_jpeg_with_restart_interval(0)).unwrap(); - assert!(info.restart_interval.is_none()); -} - -#[test] -fn inspect_reports_restart_interval_and_mcu_geometry_for_wsi_planning() { - let info = Decoder::inspect(&fixtures::baseline_420_restart_32x16_jpeg()).unwrap(); - - assert_eq!(info.dimensions, (32, 16)); - assert_eq!(info.restart_interval, Some(2)); - assert_eq!( - info.mcu_geometry, - McuGeometry { - width: 16, - height: 16, - columns: 2, - rows: 1, - count: 2, - } - ); -} - -#[test] -fn jpeg_view_restart_index_reports_original_byte_offsets() { - let bytes = restart_coded_grayscale_jpeg(24, 8); - let view = JpegView::parse(&bytes).expect("view"); - let index = view - .restart_index() - .expect("restart index") - .expect("DRI should produce an index"); - let scan_data_offset = scan_data_offset(&bytes); - let rst_offsets = restart_marker_offsets(&bytes); - - assert_eq!(index.scan_data_offset, scan_data_offset); - assert_eq!(index.interval_mcus, 1); - assert_eq!( - index.segments, - vec![ - RestartSegment { - start_mcu: 0, - entropy_offset: scan_data_offset, - marker_offset: None, - marker: None, - }, - RestartSegment { - start_mcu: 1, - entropy_offset: rst_offsets[0] + 2, - marker_offset: Some(rst_offsets[0]), - marker: Some(0xd0), - }, - RestartSegment { - start_mcu: 2, - entropy_offset: rst_offsets[1] + 2, - marker_offset: Some(rst_offsets[1]), - marker: Some(0xd1), - }, - ] - ); - - let decoder_index = Decoder::new(&bytes) - .expect("decoder") - .restart_index() - .expect("decoder restart index"); - assert_eq!(decoder_index, Some(index)); -} - -#[test] -fn restart_index_is_none_without_dri() { - let bytes = minimal_baseline_jpeg(); - let view = JpegView::parse(&bytes).expect("view"); - assert_eq!(view.restart_index().expect("restart index"), None); -} - -#[test] -fn jpeg_view_exposes_baseline_passthrough_candidate_with_original_bytes() { - let bytes = minimal_baseline_jpeg(); - let view = JpegView::parse(&bytes).expect("view"); - let candidate = view - .passthrough_candidate() - .expect("baseline JPEG passthrough candidate"); - let requirements = PassthroughRequirements::new( - CompressedTransferSyntax::JpegBaseline8, - CompressedPayloadKind::JpegInterchange, - ) - .with_dimensions((16, 16)) - .with_components(3) - .with_bit_depth(8); - - assert_eq!(view.bytes(), bytes.as_slice()); - assert_eq!( - candidate.transfer_syntax(), - CompressedTransferSyntax::JpegBaseline8 - ); - assert_eq!( - candidate.payload_kind(), - CompressedPayloadKind::JpegInterchange - ); - assert_eq!( - candidate.evaluate(&requirements), - PassthroughDecision::Copy { - bytes: bytes.as_slice() - } - ); -} - -#[test] -fn jpeg_progressive_is_not_offered_as_active_passthrough_candidate() { - let bytes = progressive_8x8_jpeg(); - let view = JpegView::parse(&bytes).expect("progressive view"); - - assert!(view.passthrough_candidate().is_none()); -} diff --git a/crates/signinum-jpeg/tests/view_and_rows.rs b/crates/signinum-jpeg/tests/view_and_rows.rs deleted file mode 100644 index da549ec5..00000000 --- a/crates/signinum-jpeg/tests/view_and_rows.rs +++ /dev/null @@ -1,237 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Integration tests for the parsed-view API and row-streaming decode surface. - -use signinum_jpeg::{ - ComponentRowWriter, Decoder, Downscale, JpegError, JpegView, PixelFormat, Rect, RowSink, - ScratchPool, -}; - -mod fixtures; -use fixtures::{grayscale_8x8_jpeg, minimal_baseline_420_jpeg, rgb_app14_8x8_jpeg}; - -#[derive(Default)] -struct CollectRows { - rows: Vec<(u32, Vec)>, -} - -impl RowSink for CollectRows { - type Error = JpegError; - - fn write_row(&mut self, y: u32, row: &[u8]) -> Result<(), JpegError> { - self.rows.push((y, row.to_vec())); - Ok(()) - } -} - -#[derive(Default)] -struct CollectGrayComponentRows { - rows: Vec<(u32, Vec)>, -} - -impl ComponentRowWriter for CollectGrayComponentRows { - fn write_gray_row(&mut self, y: u32, gray_row: &[u8]) -> Result<(), JpegError> { - self.rows.push((y, gray_row.to_vec())); - Ok(()) - } - - fn write_ycbcr_row( - &mut self, - _y: u32, - _y_row: &[u8], - _cb_row: &[u8], - _cr_row: &[u8], - ) -> Result<(), JpegError> { - unreachable!("grayscale test writer should not receive ycbcr rows"); - } - - fn write_rgb_row( - &mut self, - _y: u32, - _r_row: &[u8], - _g_row: &[u8], - _b_row: &[u8], - ) -> Result<(), JpegError> { - unreachable!("grayscale test writer should not receive rgb rows"); - } -} - -#[test] -fn jpeg_view_parse_matches_decoder_inspect() { - let bytes = minimal_baseline_420_jpeg(); - let view = JpegView::parse(&bytes).expect("parsed view must construct"); - let info = Decoder::inspect(&bytes).expect("inspect must succeed"); - assert_eq!(view.info(), &info); -} - -#[test] -fn decoder_from_view_matches_decoder_new_rgb_output() { - let bytes = rgb_app14_8x8_jpeg(); - let dec_from_new = Decoder::new(&bytes).expect("decoder::new must succeed"); - let dec_from_view = Decoder::from_view(JpegView::parse(&bytes).unwrap()) - .expect("decoder::from_view must succeed"); - - let (w, h) = dec_from_new.info().dimensions; - let stride = (w * 3) as usize; - let mut new_out = vec![0u8; stride * h as usize]; - let mut view_out = vec![0u8; stride * h as usize]; - - dec_from_new - .decode_scaled_into(&mut new_out, stride, PixelFormat::Rgb8, Downscale::None) - .unwrap(); - dec_from_view - .decode_scaled_into(&mut view_out, stride, PixelFormat::Rgb8, Downscale::None) - .unwrap(); - - assert_eq!(view_out, new_out); -} - -#[test] -fn decode_rows_matches_decode_into_rgb8() { - let bytes = minimal_baseline_420_jpeg(); - let dec = Decoder::new(&bytes).expect("decoder::new must succeed"); - let (w, h) = dec.info().dimensions; - let stride = (w * 3) as usize; - - let mut expected = vec![0u8; stride * h as usize]; - dec.decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, Downscale::None) - .unwrap(); - - let mut sink = CollectRows::default(); - dec.decode_rows(&mut sink) - .expect("decode_rows must succeed"); - - assert_eq!(sink.rows.len(), h as usize); - for (row_idx, (y, row)) in sink.rows.iter().enumerate() { - assert_eq!(*y as usize, row_idx); - assert_eq!(row.len(), stride); - assert_eq!( - row.as_slice(), - &expected[row_idx * stride..(row_idx + 1) * stride] - ); - } -} - -#[test] -fn decode_rows_matches_decode_into_rgb8_for_grayscale_input() { - let bytes = grayscale_8x8_jpeg(); - let dec = Decoder::new(&bytes).expect("decoder::new must succeed"); - let (w, h) = dec.info().dimensions; - let stride = (w * 3) as usize; - - let mut expected = vec![0u8; stride * h as usize]; - dec.decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, Downscale::None) - .unwrap(); - - let mut sink = CollectRows::default(); - dec.decode_rows(&mut sink) - .expect("decode_rows must succeed"); - - assert_eq!(sink.rows.len(), h as usize); - for (row_idx, (y, row)) in sink.rows.iter().enumerate() { - assert_eq!(*y as usize, row_idx); - assert_eq!( - row.as_slice(), - &expected[row_idx * stride..(row_idx + 1) * stride] - ); - } -} - -#[test] -fn decode_rows_matches_decode_into_rgb8_for_restart_coded_grayscale_wsi_shape() { - let bytes = restart_coded_grayscale_jpeg(24, 24); - let dec = Decoder::new(&bytes).expect("restart-coded grayscale fixture must parse"); - let (w, h) = dec.info().dimensions; - let stride = (w * 3) as usize; - - let mut expected = vec![0u8; stride * h as usize]; - dec.decode_scaled_into(&mut expected, stride, PixelFormat::Rgb8, Downscale::None) - .expect("full decode must succeed"); - - let mut sink = CollectRows::default(); - dec.decode_rows(&mut sink) - .expect("decode_rows must succeed on restart-coded input"); - - assert_eq!(sink.rows.len(), h as usize); - for (row_idx, (y, row)) in sink.rows.iter().enumerate() { - assert_eq!(*y as usize, row_idx); - assert_eq!(row.len(), stride); - assert_eq!( - row.as_slice(), - &expected[row_idx * stride..(row_idx + 1) * stride] - ); - } -} - -#[test] -fn region_component_rows_scaled_matches_gray_region_decode_for_restart_fixture() { - let bytes = restart_coded_grayscale_jpeg(24, 24); - let dec = Decoder::new(&bytes).expect("restart-coded grayscale fixture must parse"); - let roi = Rect { - x: 5, - y: 6, - w: 11, - h: 10, - }; - - let mut pool = ScratchPool::new(); - let mut sink = CollectGrayComponentRows::default(); - dec.decode_region_component_rows_with_scratch(&mut pool, &mut sink, roi, Downscale::Half) - .expect("scaled region component rows must decode"); - - let expected = dec - .decode_region_scaled(PixelFormat::Gray8, roi, Downscale::Half) - .expect("scaled region decode must succeed") - .0; - - let mut collected = Vec::new(); - for (row_idx, (y, row)) in sink.rows.iter().enumerate() { - assert_eq!(*y as usize, row_idx); - collected.extend_from_slice(row); - } - - assert_eq!(collected, expected); -} - -fn restart_coded_grayscale_jpeg(width: u16, height: u16) -> Vec { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&[0xff, 0xd8]); - bytes.extend_from_slice(&[0xff, 0xdb, 0x00, 67, 0x00]); - bytes.extend(std::iter::repeat_n(16u8, 64)); - bytes.extend_from_slice(&[ - 0xff, - 0xc0, - 0x00, - 11, - 8, - (height >> 8) as u8, - height as u8, - (width >> 8) as u8, - width as u8, - 1, - 1, - 0x11, - 0, - ]); - bytes.extend_from_slice(&[0xff, 0xdd, 0x00, 0x04, 0x00, 0x01]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x00, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[ - 0xff, 0xc4, 0x00, 20, 0x10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]); - bytes.extend_from_slice(&[0xff, 0xda, 0x00, 0x08, 1, 1, 0x00, 0, 63, 0]); - - let mcu_cols = u32::from(width).div_ceil(8); - let mcu_rows = u32::from(height).div_ceil(8); - let mcu_count = (mcu_cols * mcu_rows) as usize; - for mcu in 0..mcu_count { - bytes.push(0x00); - if mcu + 1 != mcu_count { - bytes.extend_from_slice(&[0xff, 0xd0 | ((mcu as u8) & 0x07)]); - } - } - - bytes.extend_from_slice(&[0xff, 0xd9]); - bytes -} diff --git a/crates/signinum-profile/Cargo.toml b/crates/signinum-profile/Cargo.toml deleted file mode 100644 index 1e7d9c31..00000000 --- a/crates/signinum-profile/Cargo.toml +++ /dev/null @@ -1,27 +0,0 @@ -[package] -name = "signinum-profile" -description = "Internal profiling helpers for signinum crates" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -repository.workspace = true -readme = "README.md" - -[lib] -name = "signinum_profile" -path = "src/lib.rs" - -[features] -default = ["std"] -std = [] - -[lints.rust] -unsafe_code = "forbid" -unreachable_pub = "warn" - -[lints.clippy] -pedantic = { level = "warn", priority = -1 } -module_name_repetitions = "allow" -must_use_candidate = "allow" -missing_errors_doc = "allow" diff --git a/crates/signinum-profile/README.md b/crates/signinum-profile/README.md deleted file mode 100644 index 0529a03b..00000000 --- a/crates/signinum-profile/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# signinum-profile - -Internal profiling helpers shared by the `signinum` workspace crates. - -This crate is published only because public `signinum` crates depend on it at -runtime. It is not a user-facing API and should not be used directly by -applications. diff --git a/crates/signinum-test-support/src/lib.rs b/crates/signinum-test-support/src/lib.rs deleted file mode 100644 index d86fc99f..00000000 --- a/crates/signinum-test-support/src/lib.rs +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -#![forbid(unsafe_code)] - -/// Generates deterministic RGB8 pixels for tests and benches. -pub fn patterned_rgb8(width: u32, height: u32) -> Vec { - let mut pixels = Vec::with_capacity(width as usize * height as usize * 3); - for y in 0..height { - for x in 0..width { - pixels.push(((x * 17 + y * 3) & 0xFF) as u8); - pixels.push(((x * 5 + y * 11 + 40) & 0xFF) as u8); - pixels.push(((x * 13 + y * 7 + 90) & 0xFF) as u8); - } - } - pixels -} - -/// Generates deterministic grayscale pixels for tests and benches. -pub fn patterned_gray8(width: u32, height: u32) -> Vec { - let mut pixels = Vec::with_capacity(width as usize * height as usize); - for y in 0..height { - for x in 0..width { - pixels.push(((x * 19 + y * 23) & 0xFF) as u8); - } - } - pixels -} - -/// Generates a simple deterministic gradient for reference codec parity cases. -pub fn gradient_u8(width: u32, height: u32, channels: usize) -> Vec { - gradient_variant_u8(width, height, channels, 0) -} - -/// Generates a deterministic gradient variant keyed by `seed`. -pub fn gradient_variant_u8(width: u32, height: u32, channels: usize, seed: u32) -> Vec { - let mut out = Vec::with_capacity(width as usize * height as usize * channels); - for y in 0..height { - for x in 0..width { - for c in 0..channels { - out.push(((x + y + seed * 13 + (c as u32 * 17)) & 0xFF) as u8); - } - } - } - out -} - -/// Generates deterministic RGB8 pixels used by GPU upload/decode benches. -pub fn gpu_bench_rgb8(width: u32, height: u32) -> Vec { - let mut rgb = Vec::with_capacity(width as usize * height as usize * 3); - for y in 0..height { - for x in 0..width { - rgb.push(((x * 13 + y * 3) & 0xff) as u8); - rgb.push(((x * 5 + y * 11 + (x ^ y)) & 0xff) as u8); - rgb.push(((x * 7 + y * 17 + (x.wrapping_mul(y) >> 5)) & 0xff) as u8); - } - } - rgb -} - -/// Generates contiguous RGB8 tiles for JPEG baseline encode benches. -pub fn patterned_rgb8_tiles(width: u32, height: u32, tile_count: usize) -> Vec { - let mut rgb = Vec::with_capacity(width as usize * height as usize * 3 * tile_count); - for tile in 0..tile_count as u32 { - for y in 0..height { - for x in 0..width { - rgb.push(((x * 13 + y * 3 + tile * 29) & 0xff) as u8); - rgb.push(((x * 5 + y * 11 + (x ^ y) + tile * 17) & 0xff) as u8); - rgb.push(((x * 7 + y * 17 + (x.wrapping_mul(y) >> 5) + tile * 23) & 0xff) as u8); - } - } - } - rgb -} diff --git a/crates/signinum-tilecodec/Cargo.toml b/crates/signinum-tilecodec/Cargo.toml deleted file mode 100644 index 8ad4b381..00000000 --- a/crates/signinum-tilecodec/Cargo.toml +++ /dev/null @@ -1,44 +0,0 @@ -[package] -name = "signinum-tilecodec" -description = "Tile decompression codecs for pathology image containers" -version.workspace = true -edition.workspace = true -rust-version.workspace = true -license.workspace = true -repository.workspace = true -keywords.workspace = true -categories.workspace = true -readme = "README.md" - -[lib] -name = "signinum_tilecodec" -path = "src/lib.rs" - -[dependencies] -signinum-core = { path = "../signinum-core", version = "=0.4.2" } -thiserror = { workspace = true } -flate2 = { workspace = true } -zstd = { workspace = true } -weezl = { workspace = true } - -[dev-dependencies] -criterion = { workspace = true } - -[[bench]] -name = "compare" -harness = false - -[lints.rust] -unsafe_code = "forbid" -unreachable_pub = "warn" - -[lints.clippy] -pedantic = { level = "warn", priority = -1 } -module_name_repetitions = "allow" -must_use_candidate = "allow" -missing_errors_doc = "allow" -too_many_lines = "allow" -cast_possible_truncation = "allow" -cast_sign_loss = "allow" -cast_possible_wrap = "allow" -doc_markdown = "allow" diff --git a/crates/signinum-tilecodec/README.md b/crates/signinum-tilecodec/README.md deleted file mode 100644 index ea77ef94..00000000 --- a/crates/signinum-tilecodec/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# signinum-tilecodec - -Tile decompression primitives for pathology image containers. - -Install: - -```sh -cargo add signinum-tilecodec -``` - -The CPU-first 1.0 API provides `TileDecompress` implementations for Deflate, -Zstd, LZW, and Uncompressed payloads, with caller-owned scratch pools where a -codec benefits from reuse. diff --git a/crates/signinum-tilecodec/src/deflate.rs b/crates/signinum-tilecodec/src/deflate.rs deleted file mode 100644 index 27d72232..00000000 --- a/crates/signinum-tilecodec/src/deflate.rs +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use crate::{ - bounded::{copy_scratch_to_output, read_to_scratch_bounded, BoundedReadError}, - pool::DeflatePool, - TileCodecError, -}; -use flate2::read::{DeflateDecoder, ZlibDecoder}; -use signinum_core::TileDecompress; - -pub struct DeflateCodec; - -impl TileDecompress for DeflateCodec { - type Error = TileCodecError; - type Pool = DeflatePool; - - fn expected_size(_input: &[u8]) -> Result, Self::Error> { - Ok(None) - } - - fn decompress_into( - pool: &mut Self::Pool, - input: &[u8], - out: &mut [u8], - ) -> Result { - match read_to_scratch_bounded(ZlibDecoder::new(input), &mut pool.scratch, out.len()) { - Ok(written) => { - copy_scratch_to_output(&pool.scratch, out); - Ok(written) - } - Err(BoundedReadError::OutputTooSmall(error)) => Err(error.into()), - Err(BoundedReadError::Io(zlib_error)) => { - pool.scratch.clear(); - match read_to_scratch_bounded( - DeflateDecoder::new(input), - &mut pool.scratch, - out.len(), - ) { - Ok(written) => { - copy_scratch_to_output(&pool.scratch, out); - Ok(written) - } - Err(BoundedReadError::OutputTooSmall(error)) => Err(error.into()), - Err(BoundedReadError::Io(raw_error)) => Err(TileCodecError::Backend(format!( - "deflate decode failed (zlib: {zlib_error}; raw: {raw_error})" - ))), - } - } - } - } -} diff --git a/crates/signinum-tilecodec/src/error.rs b/crates/signinum-tilecodec/src/error.rs deleted file mode 100644 index cd7db4a9..00000000 --- a/crates/signinum-tilecodec/src/error.rs +++ /dev/null @@ -1,33 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use signinum_core::{BufferError, CodecError, InputError, Unsupported}; - -#[derive(Debug, thiserror::Error)] -pub enum TileCodecError { - #[error(transparent)] - Buffer(#[from] BufferError), - #[error(transparent)] - Input(#[from] InputError), - #[error(transparent)] - Unsupported(#[from] Unsupported), - #[error("{0}")] - Backend(String), -} - -impl CodecError for TileCodecError { - fn is_truncated(&self) -> bool { - matches!(self, Self::Input(_)) - } - - fn is_not_implemented(&self) -> bool { - false - } - - fn is_unsupported(&self) -> bool { - matches!(self, Self::Unsupported(_)) - } - - fn is_buffer_error(&self) -> bool { - matches!(self, Self::Buffer(_)) - } -} diff --git a/crates/signinum-tilecodec/src/pool.rs b/crates/signinum-tilecodec/src/pool.rs deleted file mode 100644 index 0e18d5dd..00000000 --- a/crates/signinum-tilecodec/src/pool.rs +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use signinum_core::ScratchPool; - -#[derive(Debug, Default)] -pub struct DeflatePool { - pub(crate) scratch: Vec, -} - -#[derive(Debug, Default)] -pub struct ZstdPool { - pub(crate) scratch: Vec, -} - -#[derive(Debug, Default)] -pub struct LzwPool { - pub(crate) scratch: Vec, -} - -#[derive(Debug, Default, Clone, Copy)] -pub struct NoPool; - -impl DeflatePool { - #[must_use] - pub fn new() -> Self { - Self::default() - } -} - -impl ZstdPool { - #[must_use] - pub fn new() -> Self { - Self::default() - } -} - -impl LzwPool { - #[must_use] - pub fn new() -> Self { - Self::default() - } -} - -impl ScratchPool for DeflatePool { - fn bytes_allocated(&self) -> usize { - self.scratch.capacity() - } - - fn reset(&mut self) { - self.scratch.clear(); - } -} - -impl ScratchPool for ZstdPool { - fn bytes_allocated(&self) -> usize { - self.scratch.capacity() - } - - fn reset(&mut self) { - self.scratch.clear(); - } -} - -impl ScratchPool for LzwPool { - fn bytes_allocated(&self) -> usize { - self.scratch.capacity() - } - - fn reset(&mut self) { - self.scratch.clear(); - } -} - -impl ScratchPool for NoPool { - fn bytes_allocated(&self) -> usize { - 0 - } - - fn reset(&mut self) {} -} diff --git a/crates/signinum-tilecodec/src/zstd_codec.rs b/crates/signinum-tilecodec/src/zstd_codec.rs deleted file mode 100644 index 39636862..00000000 --- a/crates/signinum-tilecodec/src/zstd_codec.rs +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use crate::{ - bounded::{copy_scratch_to_output, read_to_scratch_bounded, BoundedReadError}, - pool::ZstdPool, - TileCodecError, -}; -use signinum_core::TileDecompress; - -pub struct ZstdCodec; - -impl TileDecompress for ZstdCodec { - type Error = TileCodecError; - type Pool = ZstdPool; - - fn expected_size(_input: &[u8]) -> Result, Self::Error> { - Ok(None) - } - - fn decompress_into( - pool: &mut Self::Pool, - input: &[u8], - out: &mut [u8], - ) -> Result { - pool.scratch.clear(); - let mut decoder = zstd::stream::read::Decoder::new(input).map_err(|error| { - TileCodecError::Backend(format!("zstd decoder init failed: {error}")) - })?; - let written = match read_to_scratch_bounded(&mut decoder, &mut pool.scratch, out.len()) { - Ok(written) => written, - Err(BoundedReadError::OutputTooSmall(error)) => return Err(error.into()), - Err(BoundedReadError::Io(error)) => { - return Err(TileCodecError::Backend(format!( - "zstd decode failed: {error}" - ))); - } - }; - - copy_scratch_to_output(&pool.scratch, out); - Ok(written) - } -} diff --git a/crates/signinum/Cargo.toml b/crates/signinum/Cargo.toml deleted file mode 100644 index 50e61092..00000000 --- a/crates/signinum/Cargo.toml +++ /dev/null @@ -1,53 +0,0 @@ -[package] -name = "signinum" -description = "Facade crate for signinum pathology image codecs" -version = "0.4.2" -edition.workspace = true -rust-version.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true -keywords.workspace = true -categories.workspace = true -readme = "README.md" - -[lib] -name = "signinum" -path = "src/lib.rs" - -[features] -default = ["metal"] -gpu = ["metal", "cuda"] -metal = ["dep:signinum-jpeg-metal", "dep:signinum-j2k-metal"] -cuda = ["dep:signinum-jpeg-cuda", "dep:signinum-j2k-cuda"] - -[dependencies] -signinum-core = { path = "../signinum-core", version = "=0.4.2" } -signinum-jpeg = { path = "../signinum-jpeg", version = "=0.4.2" } -signinum-j2k = { path = "../signinum-j2k", version = "=0.4.2" } -signinum-tilecodec = { path = "../signinum-tilecodec", version = "=0.4.2" } -signinum-jpeg-metal = { path = "../signinum-jpeg-metal", version = "=0.4.2", optional = true } -signinum-j2k-metal = { path = "../signinum-j2k-metal", version = "=0.4.2", optional = true } -signinum-jpeg-cuda = { path = "../signinum-jpeg-cuda", version = "=0.4.2", optional = true } -signinum-j2k-cuda = { path = "../signinum-j2k-cuda", version = "=0.4.2", optional = true } - -[dev-dependencies] -criterion = { workspace = true } -signinum-j2k-native = { path = "../signinum-j2k-native", version = "=0.4.2" } -signinum-test-support = { path = "../signinum-test-support" } - -[[bench]] -name = "facade" -harness = false - -[lints.rust] -unsafe_code = "forbid" -unreachable_pub = "warn" - -[lints.clippy] -pedantic = { level = "warn", priority = -1 } -module_name_repetitions = "allow" -must_use_candidate = "allow" -missing_errors_doc = "allow" -missing_panics_doc = "allow" -doc_markdown = "allow" diff --git a/crates/signinum/README.md b/crates/signinum/README.md deleted file mode 100644 index 8d2d7857..00000000 --- a/crates/signinum/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# signinum - -Facade crate for the `signinum` pathology image codec workspace. - -The default build exposes CPU-portable JPEG, JPEG 2000, shared core contracts, -tile decompression APIs, and the portable Metal adapter. Runtime backend -selection defaults to `Auto`: device paths are used for supported workloads when -compiled and available, with CPU as the fallback. CUDA remains available through -the explicit `cuda` or `gpu` features. - -Install: - -```sh -cargo add signinum -``` - -Use this crate when an application wants one import surface for Signinum codec -primitives. Use `statumen` for whole-slide container parsing and `wsi-dicom` -for DICOM VL Whole Slide Microscopy export. diff --git a/crates/signinum/benches/facade.rs b/crates/signinum/benches/facade.rs deleted file mode 100644 index ce5e6000..00000000 --- a/crates/signinum/benches/facade.rs +++ /dev/null @@ -1,62 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use signinum::j2k::{ - encode_j2k_lossless as facade_encode_j2k_lossless, EncodeBackendPreference, - J2kEncodeValidation, J2kLosslessEncodeOptions, J2kLosslessSamples, -}; -use signinum_test_support::patterned_rgb8; - -const TILE_SIDE: u32 = 128; - -fn bench_encode_options() -> J2kLosslessEncodeOptions { - J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::CpuOnly, - validation: J2kEncodeValidation::External, - ..J2kLosslessEncodeOptions::default() - } -} - -fn bench_facade_j2k_encode(c: &mut Criterion) { - let pixels = patterned_rgb8(TILE_SIDE, TILE_SIDE); - let options = bench_encode_options(); - - let mut group = c.benchmark_group("facade_j2k_lossless_encode"); - group.bench_function("facade_cpu_only_rgb8_128x128", |b| { - b.iter(|| { - let samples = J2kLosslessSamples::new( - black_box(pixels.as_slice()), - TILE_SIDE, - TILE_SIDE, - 3, - 8, - false, - ) - .expect("valid rgb8 samples"); - let encoded = - facade_encode_j2k_lossless(samples, &options).expect("facade cpu-only encode"); - black_box(encoded.codestream.len()); - }); - }); - - group.bench_function("direct_cpu_only_rgb8_128x128", |b| { - b.iter(|| { - let samples = J2kLosslessSamples::new( - black_box(pixels.as_slice()), - TILE_SIDE, - TILE_SIDE, - 3, - 8, - false, - ) - .expect("valid rgb8 samples"); - let encoded = - signinum_j2k::encode_j2k_lossless(samples, &options).expect("direct cpu encode"); - black_box(encoded.codestream.len()); - }); - }); - group.finish(); -} - -criterion_group!(benches, bench_facade_j2k_encode); -criterion_main!(benches); diff --git a/crates/signinum/src/lib.rs b/crates/signinum/src/lib.rs deleted file mode 100644 index 90f825b5..00000000 --- a/crates/signinum/src/lib.rs +++ /dev/null @@ -1,221 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -//! Facade crate for the `signinum` pathology image codecs. -//! -//! Runtime backend requests default to [`BackendRequest::Auto`]. The facade -//! compiles portable CPU codecs plus the Metal adapter by default, then uses -//! device backends for supported workloads when they are compiled and available. -//! CPU is the fallback for `Auto`, not the policy default. -//! -//! # Examples -//! -//! JPEG decode imports: -//! -//! ```no_run -//! use signinum::jpeg::{Decoder, PixelFormat}; -//! -//! let bytes = std::fs::read("tile.jpg").unwrap(); -//! let mut decoder = Decoder::new(&bytes).unwrap(); -//! let info = decoder.info(); -//! let stride = info.dimensions.0 as usize * PixelFormat::Rgb8.bytes_per_pixel(); -//! let mut out = vec![0; stride * info.dimensions.1 as usize]; -//! decoder.decode_into(&mut out, stride, PixelFormat::Rgb8).unwrap(); -//! ``` -//! -//! JPEG 2000 lossless encode with the runtime default: -//! -//! ``` -//! use signinum::j2k::{encode_j2k_lossless, J2kLosslessEncodeOptions, J2kLosslessSamples}; -//! use signinum::BackendRequest; -//! -//! assert_eq!(BackendRequest::default(), BackendRequest::Auto); -//! let pixels = [0u8; 4 * 4]; -//! let samples = J2kLosslessSamples::new(&pixels, 4, 4, 1, 8, false).unwrap(); -//! let encoded = encode_j2k_lossless(samples, &J2kLosslessEncodeOptions::default()).unwrap(); -//! assert!(encoded.codestream.starts_with(&[0xFF, 0x4F])); -//! ``` -//! -//! Tile decompression imports: -//! -//! ``` -//! use signinum::tilecodec::UncompressedCodec; -//! use signinum::TileDecompress; -//! -//! let mut pool = ::Pool::default(); -//! let mut out = [0u8; 3]; -//! let written = UncompressedCodec::decompress_into(&mut pool, &[1, 2, 3], &mut out).unwrap(); -//! assert_eq!(written, 3); -//! ``` - -#![warn(unreachable_pub)] - -pub mod core { - //! Shared codec contracts and backend selection types. - - pub use signinum_core::*; -} - -pub mod jpeg { - //! Baseline JPEG decode APIs. - - pub use signinum_jpeg::*; - - #[cfg(feature = "cuda")] - pub mod cuda { - //! CUDA JPEG adapter APIs. - - pub use signinum_jpeg_cuda::*; - } - - #[cfg(feature = "metal")] - pub mod metal { - //! Metal JPEG adapter APIs. - - pub use signinum_jpeg_metal::*; - } -} - -pub mod j2k { - //! JPEG 2000 inspect, decode, and encode APIs. - - pub use signinum_j2k::{ - adapter, context, encode_j2k_lossless as encode_j2k_lossless_cpu, - encode_j2k_lossless_with_accelerator, error, j2k_lossless_decomposition_levels, scratch, - view, BackendKind, BackendRequest, BufferError, CodecError, CompressedPayloadKind, - CompressedTransferSyntax, DecodeOutcome, DecodeRowsError, DecoderContext, Downscale, - EncodeBackendPreference, EncodedJ2k, ImageCodec, ImageDecode, ImageDecodeRows, - J2kBlockCodingMode, J2kCodec, J2kContext, J2kDecoder, J2kEncodeDispatchReport, - J2kEncodeStageAccelerator, J2kEncodeValidation, J2kError, J2kLosslessEncodeOptions, - J2kLosslessSamples, J2kProgressionOrder, J2kScratchPool, J2kView, PassthroughCandidate, - PassthroughDecision, PassthroughRejectReason, PassthroughRequirements, PixelFormat, Rect, - ReversibleTransform, RowSink, TileBatchDecode, - }; - - #[cfg(feature = "cuda")] - pub mod cuda { - //! CUDA JPEG 2000 adapter APIs. - - pub use signinum_j2k_cuda::*; - } - - #[cfg(feature = "metal")] - pub mod metal { - //! Metal JPEG 2000 adapter APIs. - - pub use signinum_j2k_metal::*; - } - - /// Encode interleaved samples into a raw JPEG 2000 lossless codestream. - /// - /// With [`EncodeBackendPreference::Auto`] or - /// [`EncodeBackendPreference::PreferDevice`], the facade tries compiled - /// device encode-stage accelerators first and falls back to CPU only when - /// no device stage dispatches. Device kernel and validation failures are - /// returned to the caller. - pub fn encode_j2k_lossless( - samples: J2kLosslessSamples<'_>, - options: &J2kLosslessEncodeOptions, - ) -> Result { - if options.backend == EncodeBackendPreference::CpuOnly { - return signinum_j2k::encode_j2k_lossless(samples, options); - } - - if let Some(encoded) = try_metal_encode(samples, *options)? { - return Ok(encoded); - } - if let Some(encoded) = try_cuda_encode(samples, *options)? { - return Ok(encoded); - } - - signinum_j2k::encode_j2k_lossless(samples, options) - } - - #[cfg(feature = "metal")] - fn try_metal_encode( - samples: J2kLosslessSamples<'_>, - options: J2kLosslessEncodeOptions, - ) -> Result, J2kError> { - let mut accelerator = if options.backend == EncodeBackendPreference::Auto { - signinum_j2k_metal::MetalEncodeStageAccelerator::for_auto_host_output() - } else { - signinum_j2k_metal::MetalEncodeStageAccelerator::with_cpu_forward_rct() - }; - encode_with_device_accelerator(samples, options, BackendKind::Metal, &mut accelerator) - } - - #[cfg(not(feature = "metal"))] - #[allow(clippy::unnecessary_wraps)] - fn try_metal_encode( - _samples: J2kLosslessSamples<'_>, - _options: J2kLosslessEncodeOptions, - ) -> Result, J2kError> { - Ok(None) - } - - #[cfg(feature = "cuda")] - fn try_cuda_encode( - samples: J2kLosslessSamples<'_>, - options: J2kLosslessEncodeOptions, - ) -> Result, J2kError> { - let mut accelerator = signinum_j2k_cuda::CudaEncodeStageAccelerator::default(); - encode_with_device_accelerator(samples, options, BackendKind::Cuda, &mut accelerator) - } - - #[cfg(not(feature = "cuda"))] - #[allow(clippy::unnecessary_wraps)] - fn try_cuda_encode( - _samples: J2kLosslessSamples<'_>, - _options: J2kLosslessEncodeOptions, - ) -> Result, J2kError> { - Ok(None) - } - - #[cfg_attr(not(any(feature = "metal", feature = "cuda")), allow(dead_code))] - fn encode_with_device_accelerator( - samples: J2kLosslessSamples<'_>, - options: J2kLosslessEncodeOptions, - backend: BackendKind, - accelerator: &mut impl J2kEncodeStageAccelerator, - ) -> Result, J2kError> { - let device_options = J2kLosslessEncodeOptions { - backend: EncodeBackendPreference::PreferDevice, - ..options - }; - let encoded = signinum_j2k::encode_j2k_lossless_with_accelerator( - samples, - &device_options, - backend, - accelerator, - )?; - - Ok((encoded.backend == backend).then_some(encoded)) - } -} - -pub mod tilecodec { - //! Tile decompression codecs for container integrations. - - pub use signinum_tilecodec::*; -} - -pub use core::{ - BackendCapabilities, BackendKind, BackendRequest, BufferError, CodecError, DecodeOutcome, - DecodeRowsError, DecoderContext, DeviceSurface, Downscale, ImageCodec, ImageDecode, - ImageDecodeDevice, ImageDecodeRows, PixelFormat, Rect, RowSink, TileBatchDecode, - TileBatchDecodeManyDevice, TileDecompress, -}; -pub use core::{ - CompressedPayloadKind, CompressedTransferSyntax, PassthroughCandidate, PassthroughDecision, - PassthroughRejectReason, PassthroughRequirements, -}; -pub use j2k::{ - encode_j2k_lossless, encode_j2k_lossless_with_accelerator, j2k_lossless_decomposition_levels, - EncodeBackendPreference, EncodedJ2k, J2kBlockCodingMode, J2kCodec, J2kContext, J2kDecoder, - J2kEncodeDispatchReport, J2kEncodeStageAccelerator, J2kEncodeValidation, J2kError, - J2kLosslessEncodeOptions, J2kLosslessSamples, J2kProgressionOrder, ReversibleTransform, -}; -pub use jpeg::{ - ColorSpace, ColorTransform, DecodeOptions, Decoder as JpegDecoder, JpegCodec, JpegError, - JpegView, -}; -pub use tilecodec::{DeflateCodec, LzwCodec, TileCodecError, UncompressedCodec, ZstdCodec}; diff --git a/crates/signinum/tests/facade.rs b/crates/signinum/tests/facade.rs deleted file mode 100644 index 89528ef7..00000000 --- a/crates/signinum/tests/facade.rs +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -use signinum::{ - j2k::{encode_j2k_lossless, J2kLosslessEncodeOptions, J2kLosslessSamples}, - tilecodec::UncompressedCodec, - BackendKind, BackendRequest, CompressedPayloadKind, CompressedTransferSyntax, - PassthroughCandidate, PassthroughRequirements, TileDecompress, -}; - -#[test] -fn facade_default_features_enable_auto_device_adapters() { - let manifest = std::fs::read_to_string(env!("CARGO_MANIFEST_DIR").to_owned() + "/Cargo.toml") - .expect("read facade manifest"); - - assert!( - manifest.contains("default = [\"metal\"]"), - "signinum should compile the portable Metal adapter by default so Auto is not documented or packaged as CPU-only" - ); -} - -#[test] -fn facade_runtime_backend_default_is_auto() { - assert_eq!(BackendRequest::default(), BackendRequest::Auto); - assert_eq!( - J2kLosslessEncodeOptions::default().backend, - signinum::EncodeBackendPreference::Auto - ); -} - -#[test] -fn facade_auto_j2k_lossless_encode_uses_device_when_available() { - let pixels: Vec = (0..4 * 4 * 3) - .map(|value| u8::try_from((value * 11) & 0xFF).expect("masked sample fits")) - .collect(); - let samples = J2kLosslessSamples::new(&pixels, 4, 4, 3, 8, false).expect("valid samples"); - - let encoded = - encode_j2k_lossless(samples, &J2kLosslessEncodeOptions::default()).expect("encode"); - - #[cfg(all(feature = "metal", target_os = "macos"))] - match encoded.backend { - BackendKind::Metal => {} - BackendKind::Cpu => { - let samples = - J2kLosslessSamples::new(&pixels, 4, 4, 3, 8, false).expect("valid samples"); - let required = encode_j2k_lossless( - samples, - &J2kLosslessEncodeOptions { - backend: signinum::EncodeBackendPreference::RequireDevice, - ..J2kLosslessEncodeOptions::default() - }, - ); - assert!( - required.is_err(), - "Auto fell back to CPU even though RequireDevice succeeded" - ); - } - BackendKind::Cuda => panic!("unexpected facade backend: Cuda"), - } - #[cfg(not(all(feature = "metal", target_os = "macos")))] - assert_eq!(encoded.backend, BackendKind::Cpu); - assert!(encoded.codestream.starts_with(&[0xFF, 0x4F])); -} - -#[test] -fn facade_exports_tilecodec_contracts() { - let input = [1, 2, 3, 4]; - let mut output = [0; 4]; - let mut pool = ::Pool::default(); - - let written = UncompressedCodec::decompress_into(&mut pool, &input, &mut output) - .expect("uncompressed tile copy"); - - assert_eq!(written, input.len()); - assert_eq!(output, input); -} - -#[test] -fn facade_exports_passthrough_contracts() { - let info = signinum::core::Info { - dimensions: (1, 1), - components: 1, - colorspace: signinum::core::Colorspace::SGray, - bit_depth: 8, - tile_layout: None, - coded_unit_layout: None, - restart_interval: None, - resolution_levels: 1, - }; - let bytes = [0xff, 0x4f, 0xff, 0xd9]; - let candidate = PassthroughCandidate::new( - &bytes, - CompressedTransferSyntax::Jpeg2000Lossless, - CompressedPayloadKind::Jpeg2000Codestream, - info, - ); - let requirements = PassthroughRequirements::new( - CompressedTransferSyntax::Jpeg2000Lossless, - CompressedPayloadKind::Jpeg2000Codestream, - ); - - assert_eq!( - candidate - .copy_bytes_if_eligible(&requirements) - .expect("facade passthrough bytes"), - bytes - ); -} diff --git a/deny.toml b/deny.toml index 81c7f033..8c6878aa 100644 --- a/deny.toml +++ b/deny.toml @@ -1,10 +1,17 @@ [advisories] yanked = "deny" ignore = [ - # Transitive via metal 0.33.0, the current Metal binding release. + # Transitive via metal 0.33, the Metal binding release pinned in + # [workspace.dependencies]. # RUSTSEC-2024-0436 is an unmaintained advisory for paste, not a known - # memory-safety or code-execution vulnerability. Revisit when metal-rs - # removes paste or offers a maintained replacement path. + # memory-safety or code-execution vulnerability. Current metal 0.33 still + # depends on paste, and upstream closed paste removal as not planned. + # Review-by: 2026-12-31. + # Sources: + # - https://crates.io/crates/metal + # - https://github.com/gfx-rs/metal-rs/blob/master/Cargo.toml + # - https://rustsec.org/advisories/RUSTSEC-2024-0436.html + # - https://github.com/gfx-rs/metal-rs/issues/349 "RUSTSEC-2024-0436", ] diff --git a/docs/architecture.md b/docs/architecture.md index 6dfd8492..39142e77 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,380 +1,70 @@ # Architecture -This document is the system map for `signinum`. It is the first thing a new -contributor or coding agent should read before changing code, and it is the -single source of truth for how the workspace is shaped, where each -responsibility lives, and which dependency directions are legal. Anything that -is not described here, in a referenced design note, or in a crate-level README -should be treated as undocumented and verified before it is relied on. - -The structure of this file follows the harness-engineering convention of -keeping a repository-local map, an explicit layering, and cross-links to -design notes that an agent can reach without leaving the repo. - -## Companion documents - -- [`README.md`](../README.md) — public surface, supported APIs, MSRV, examples. -- [`CHANGELOG.md`](../CHANGELOG.md) — release history. -- [`docs/bench.md`](bench.md) — benchmark methodology and comparator policy. -- [`docs/parity.md`](parity.md) — parity expectations against reference decoders. -- [`docs/release.md`](release.md) — release staging notes. -- [`docs/wsi-decode-api.md`](wsi-decode-api.md) — public WSI decode API guide. -- [`docs/wsi-dicom-passthrough.md`](wsi-dicom-passthrough.md) — passthrough-first - policy for WSI/DICOM conversion layers built on these codec primitives. -- Crate-level `README.md` files where present — crate-scoped contracts and - feature notes. - -## System map - -The workspace is a single Cargo workspace defined in [`Cargo.toml`](../Cargo.toml). -All crates live under `crates/` and share `edition = 2021` and -`rust-version = 1.94`. Stable facade and codec crates use 1.x versions; -implementation and adapter crates stay on explicit pre-1.0 versions where their -backend APIs are still hardening. - -| Crate | Layer | Role | -|-------|-------|------| -| `signinum-core` | foundation | Shared traits, pixel/sample types, backend capability metadata, device-surface contracts, scratch/context contracts. No image-format logic. | -| `signinum-profile` | instrumentation helper | Shared profiling helpers used by implementation crates at runtime. Published only because public crates depend on it; not a user-facing API. | -| `signinum-cuda-runtime` | runtime helper | CUDA Driver API, CUDA memory, kernel launch, and nvJPEG runtime helpers used by CUDA adapters. Published support crate, not the primary user-facing API. | -| `signinum-tilecodec` | codec | Tile decompression primitives: Deflate, Zstd, LZW, Uncompressed. Implements `TileDecompress` from `core`. | -| `signinum-jpeg` | codec | Native pure-Rust JPEG inspect/decode for WSI tiles. CPU-first. Owns SIMD backends and fused entropy/IDCT/upsample paths. Its baseline JPEG encoder is a compatibility/fallback utility, not the diagnostic WSI/DICOM encode path. | -| `signinum-j2k-native` | codec engine | Published implementation dependency for `signinum-j2k`; not the stable user-facing API. Lives under `#![forbid(unsafe_code)]` and uses `fearless_simd`. | -| `signinum-j2k` | codec | Public JPEG 2000 / HTJ2K crate. Wraps `j2k-native` with the signinum-core trait surface (inspect, decode, ROI, scaled, row-bounded, tile-batch, and lossless encode). | -| `signinum-j2k-compare` | dev-only | OpenJPEG FFI bindings used as a reference decoder for conformance and parity testing. Unpublished. | -| `signinum-jpeg-metal` | adapter | Apple Metal device-output adapter for `signinum-jpeg`. Hosts compute kernels for color conversion, interleave/pack, and `MTLBuffer` production. | -| `signinum-j2k-metal` | adapter | Apple Metal device-output adapter for `signinum-j2k`. Same shape as the JPEG adapter. | -| `signinum-jpeg-cuda` | adapter | CUDA-facing API adapter for JPEG. `Auto`/`Cpu` stay host-backed; explicit full-frame RGB8 CUDA requests use nvJPEG when `cuda-runtime`, a CUDA driver, and `libnvjpeg` are available, with CPU decode plus CUDA upload fallback for unsupported shapes. | -| `signinum-j2k-cuda` | adapter | CUDA-facing API adapter for J2K. Explicit CUDA requests upload CPU-decoded output into CUDA device memory when `cuda-runtime` and a CUDA driver are available. | -| `signinum` | facade | Stable public import surface over `core`, the CPU codecs, tile decompression, and optional Metal/CUDA adapters behind facade features. | -| `signinum-cli` | binary | `signinum inspect ` entry point. Header parsing only, no decode. | - -Out-of-tree but in-repo: - -- `corpus/` — test data: `wsi-samples/`, `conformance/`, `regressions/`, - `fuzz-seeds/`, each with a manifest describing source, license, and tolerance. -- `paper/` — research paper materials. -- `target/` — build output (gitignored). - -## Layered architecture and dependency rules - -signinum is organized as foundation/helper crates, codec engines, codecs, -adapters, and facade/binary surfaces. Dependencies must flow inward only. An -agent adding a dependency edge that -points outward is changing the architecture and should stop and update this -document first. - -``` -foundation / helper crates → codec engines → codecs → adapters → facade / binary -``` - -| Layer | Members | May depend on | -|-------|---------|---------------| -| foundation | `signinum-core` | `thiserror` only. No other workspace crate. `no_std + alloc` posture. Contains only the x86 CPUID/XGETBV unsafe required for CPU feature detection. | -| helper crates | `signinum-profile`, `signinum-cuda-runtime` | No workspace crate. `signinum-profile` may be used by codec engines, codecs, and adapters for runtime instrumentation support. `signinum-cuda-runtime` may be used by CUDA adapters for driver/runtime integration. These crates must not become primary user-facing APIs and must not depend on codecs or adapters. | -| codec engines | `signinum-j2k-native` | helper crates. Internal only. Not re-exported. | -| codecs | `signinum-jpeg`, `signinum-j2k`, `signinum-tilecodec` | foundation, codec engines, helper crates. Must not depend on each other. Must not depend on adapters or `cli`. | -| adapters | `signinum-jpeg-metal`, `signinum-j2k-metal`, `signinum-jpeg-cuda`, `signinum-j2k-cuda` | foundation, helper crates, exactly one matching codec, optional engine for the matching codec. Adapters in different format families must not depend on each other. | -| facade | `signinum` | foundation, codecs, tilecodec, optional adapters behind feature gates. | -| binary | `signinum-cli` | codecs. Must not depend on adapters (kept host-neutral). | -| dev-only | `signinum-j2k-compare` | foundation and the codec under test. Used as a reference comparator in tests/benches; never a runtime dependency. | - -Hard rules enforced today (the goal is to mechanize these as the workspace -matures, mirroring harness-engineering structural tests): - -1. `signinum-core` is a leaf in the import graph. It does not import any - other workspace crate. -2. Codec crates do not import each other. Cross-format work goes through - `core` types or through caller code. -3. Adapter crates are additive. Removing all adapter crates must leave the - codec stack fully functional on CPU. -4. Metal sources are gated by `cfg(target_os = "macos")`. Non-macOS hosts - compile the adapter crate to a thin fallback that exercises the same - public API but reports unavailability. -5. CUDA sources expose the same device-output surface. Explicit CUDA requests - produce CUDA device memory when `cuda-runtime` and a CUDA driver are - available. JPEG full-frame RGB8 requests may use nvJPEG; unsupported JPEG - shapes and J2K CUDA requests use CPU decode plus CUDA upload. -6. `signinum-jpeg` keeps its NEON and x86 intrinsics scoped per-backend - in `crates/signinum-jpeg/src/backend/`. `signinum-j2k-native` keeps - its SIMD behind `fearless_simd` so the engine can stay - `#![forbid(unsafe_code)]`. -7. Adapter crates consume codec planning hooks through public `adapter` - modules. Imports from codec `__private` modules are not allowed. +This document records current workspace boundaries. It is not a roadmap. + +The public crate release centers on `j2k`. Runtime backend selection defaults to `Auto`: CPU remains the portable baseline, and explicit CUDA or Metal requests are strict. The workspace README records the public support and codec API policy. + +## Crate classes + +| Crate | Class | Role | +| --- | --- | --- | +| `j2k` | public codec | Primary user-facing JPEG 2000 / HTJ2K API. | +| `j2k-core` | core | Shared traits, errors, geometry, pixel formats, backend requests, and device-surface contracts. | +| `j2k-jpeg`, `j2k-tilecodec` | codec | CPU/native codec implementations and stable codec APIs. | +| `j2k-native` | engine | Native JPEG 2000 / HTJ2K engine used by J2K APIs and adapter validation. | +| `j2k-profile`, `j2k-metal-support` | support | Runtime/profile helpers used by adapters and codec crates. | +| `j2k-cuda-runtime` | CUDA engine | CUDA Driver API integration, J2K-owned kernel modules, launch orchestration, and CUDA memory helpers shared by CUDA adapters. | +| `j2k-jpeg-cuda`, `j2k-cuda`, `j2k-transcode-cuda` | CUDA adapter | Codec-facing CUDA APIs, route policy, and CUDA device memory integration for supported paths. | +| `j2k-jpeg-metal`, `j2k-metal`, `j2k-transcode-metal` | Metal adapter | macOS Metal runtime integration for supported paths. | +| `j2k-transcode` | transcode | JPEG to J2K/HTJ2K transcode algorithms. | +| `j2k-cli` | CLI | Command-line inspection entry point. | +| `j2k-test-support` | dev helper | Shared fixture and benchmark input helpers for tests, benches, and examples. | +| `j2k-compare` | tooling | Comparator tooling. | +| `xtask` | workspace tool | Repository automation under `xtask/`. | + +## Dependency rules + +- The public `j2k` crate owns the JPEG 2000 / HTJ2K API surface. +- Codec crates may depend on `j2k-core` and support crates. +- Adapter crates may depend inward on codec/core/support crates. +- Support crates must not depend on adapters. +- Test support and comparator crates must not become runtime dependencies of + stable public crates. +- CUDA paths must use J2K-owned CUDA kernels for codec stages they claim to + support. ## Crate dependency graph -Workspace edges (excluding external crates and `dev-dependencies`): - -``` -signinum-core (leaf) -signinum-profile (instrumentation helper) -signinum-cuda-runtime (CUDA runtime helper) - -signinum-tilecodec -> signinum-core - -signinum-jpeg -> signinum-profile, signinum-core -signinum-jpeg-metal -> signinum-jpeg, signinum-profile, signinum-core -signinum-jpeg-cuda -> signinum-jpeg, signinum-cuda-runtime, signinum-profile, signinum-core - -signinum-j2k-native -> signinum-profile -signinum-j2k -> signinum-j2k-native, signinum-core -signinum-j2k-metal -> signinum-j2k, signinum-j2k-native, signinum-profile, signinum-core -signinum-j2k-cuda -> signinum-j2k, signinum-j2k-native, signinum-cuda-runtime, signinum-profile, signinum-core - -signinum -> signinum-core, signinum-jpeg, signinum-j2k, signinum-tilecodec, signinum-jpeg-metal, signinum-j2k-metal, signinum-jpeg-cuda, signinum-j2k-cuda -signinum-cli -> signinum-jpeg, signinum-j2k - -signinum-j2k-compare -> signinum-core, signinum-j2k (test/bench reference only) +```text +j2k -> j2k-core, j2k-native, j2k-types +j2k-native -> j2k-types, j2k-profile +j2k-test-support -> j2k-native +j2k-cuda -> j2k-core, j2k-cuda-runtime, j2k, j2k-native, j2k-profile +j2k-metal -> j2k-core, j2k, j2k-native, j2k-metal-support, j2k-profile +j2k-jpeg -> j2k-core, j2k-profile +j2k-jpeg-cuda -> j2k-core, j2k-cuda-runtime, j2k-jpeg, j2k-profile +j2k-jpeg-metal -> j2k-core, j2k-jpeg, j2k-metal-support, j2k-profile +j2k-tilecodec -> j2k-core +j2k-compare -> j2k-core, j2k, j2k-test-support +j2k-transcode -> j2k, j2k-native, j2k-jpeg +j2k-metal-support -> j2k-core +j2k-cuda-runtime -> j2k-core +j2k-transcode-metal -> j2k-core, j2k-metal-support, j2k-transcode +j2k-transcode-cuda -> j2k-cuda-runtime, j2k-native, j2k-transcode +j2k-cli -> j2k, j2k-jpeg ``` -## Core abstractions - -These live in `signinum-core` and are the contract every codec and adapter -implements. New extension points belong here. - -### Codec entry traits - -- `ImageCodec` — base trait. Associated types: `Error`, `Warning`, `Pool`. -- `ImageDecode<'a>` — CPU decode surface. Methods include `inspect`, `parse`, - `decode_into`, `decode_into_with_scratch`, `decode_region_into`, - `decode_scaled_into`, and `decode_region_scaled_into`. -- `ImageDecodeRows<'a, S: RowSink<_>>` — row-bounded decode for large tiles. -- `ImageDecodeDevice<'a>` — synchronous device decode; returns a - `DeviceSurface`. -- `ImageDecodeSubmit<'a>` — asynchronous device decode; returns a - `DeviceSubmission` whose `wait()` produces a `DeviceSurface`. -- `TileBatchDecode` — stateless tile decode with a caller-owned - `DecoderContext` reused across tiles. -- `TileDecompress` — generic tile decompression (used by `tilecodec`). - -### Backend and surface model - -- `BackendKind` — `Cpu | Metal | Cuda`. -- `BackendRequest` — `Auto | Cpu | Metal | Cuda`. Callers state intent. - `Auto` may resolve to a CPU-backed surface; explicit unsupported device - requests return an error before decode work. -- `DeviceSurface` — trait describing decoded data sitting on a backend - (CPU buffer, `MTLBuffer`, etc.). Queryable for backend kind, dimensions, - pixel format, byte length. -- `DeviceSubmission` — async handle with `wait() -> DeviceSurface`. -- `ReadySubmission` — synchronous submission used by CPU fallbacks. -- `CpuFeatures` — runtime detection of AVX2 / SSE4.1 / NEON, cached. - -### Pixel and layout types - -- `PixelFormat` — `Rgb8 | Rgba8 | Gray8 | Rgb16 | Rgba16 | Gray16`. -- `PixelLayout` — `Rgb | Rgba | Gray`. -- `Sample`, `SampleType`. -- `Downscale` — reduced-resolution decode specifier. -- `RowSink` — caller-implemented sink for row streaming. -- `Rect` — ROI. -- `Info` — image metadata: dimensions, colorspace, components, bit depth, - tile layout, MCU/coded-unit layout, restart interval, resolution levels. -- `Colorspace` — `Grayscale | YCbCr | Rgb | Cmyk | Ycck | SRgb | SGray | - IccTagged | Rct | Ict`. -- `DecodeOutcome` — decoded `Rect` plus `Vec` warnings. - -### Caller-owned state - -- `ScratchPool` — caller-owned reusable allocations, reset per operation. -- `CodecContext` — codec-specific state (e.g., JPEG quantization tables). -- `DecoderContext` — wrapper holding a `CodecContext`, persistent across - tiles in a batch. - -These three types encode a deliberate harness contract: the codec never -hides allocation, threading, or runtime state. WSI readers compose those -themselves. - -## Decode pipeline - -The shape is the same for both image codecs, modulo the engine inside. - -``` -compressed bytes - │ - ▼ -inspect(bytes) → Info (no decode) - │ - ▼ -view = View::parse(bytes) → borrowed parse (no copy) - │ - ▼ -decoder = Decoder::from_view(view) - │ - ├─ CPU path ───────────────────────────────────────────────┐ - │ decode_into / decode_into_with_scratch │ - │ decode_region_into / decode_scaled_into / │ - │ decode_region_scaled_into / decode_rows │ - │ entropy → IDCT or DWT → color conv → output buffer │ - │ SIMD: AVX2 / SSE4.1 / NEON (jpeg) │ - │ fearless_simd (j2k-native) │ - │ returns DecodeOutcome │ - │ │ - └─ Device path (Metal today, CUDA API-only) ───────────────┘ - submit_to_device(session, fmt, BackendRequest::Metal) - │ - ▼ - capability check - │ - ├─ supported shape: prepare packet, upload to MTLBuffer, - │ dispatch compute kernel (color conv, interleave/pack) - │ → DeviceSubmission → wait() → Surface (with MTLBuffer) - │ - └─ unsupported explicit backend: fail before decode; - Auto/Cpu may wrap CPU output in a host-backed DeviceSurface -``` - -JPEG is fully fused on CPU: entropy decode, IDCT scheduling, upsampling, ROI, -and the fast 4:2:0 path live together in -`crates/signinum-jpeg/src/entropy/sequential.rs` because splitting them -regresses WSI tile-batch performance. Splitting that module is planned but -gated on stable benchmark and parity coverage. - -J2K parses boxes (COD, QCD, QCC, etc.) and codestream structure on CPU, then -drives either the native CPU reconstruction path or a MetalDirect plan. ROI and -reduced-resolution requests share the same core contract: the ROI is expressed -in source coordinates, and the returned decoded rectangle covers the -floor-start/ceil-end projection onto the requested reduced-resolution grid. - -The current MetalDirect path is first-class for grayscale J2K/HTJ2K full-tile -decode and grayscale ROI+scaled tile batches. Marker parsing and plan building -stay on CPU; supported classic Tier-1 or HT cleanup block jobs, grouped -sub-band decode, IDWT, and final store/pack run as one Metal command sequence -and return resident Metal surfaces. Distinct grayscale WSI-style ROI+scaled -batches are coalesced across separate codestreams. Cropped ROI+scaled plans -prune code-block jobs outside the requested store windows, compact retained -HTJ2K coded payloads, and crop every required IDWT output level. Cropped IDWT -outputs carry input-window origins and strides through the resident band graph, -so intermediate levels can feed later cropped levels without returning to broad -intermediate buffers. - -Unsupported formats, unsupported codestream features, RGB ROI+scaled paths, and -non-macOS hosts fall back through the CPU reconstruction and device-surface -upload path according to the requested backend. Explicit Metal requests fail -for unsupported Metal shapes; `Auto` is intentionally limited to measured -grayscale batch cases. - -## WSI/DICOM conversion policy - -WSI/DICOM container readers and writers live outside this codec workspace. They -should inspect compressed tile payloads and pass them through unchanged whenever -the destination transfer syntax and frame metadata make that legal. The shared -contract is `signinum-core::PassthroughCandidate` plus -`PassthroughRequirements`; codec views build candidates from borrowed source -bytes, and the container layer remains responsible for DICOM-specific frame -ordering and fragment writing. If a new diagnostic codestream is required, use -the lossless J2K/HTJ2K encode surfaces. Baseline JPEG encode is reserved for -explicit fallback, generated fixtures, or non-diagnostic derived output. - -Metal adapter routing is explicit after the facade release. -`BackendRequest::Cpu` returns host-backed CPU surfaces. `BackendRequest::Auto` -may select Metal only for validated adapter-supported shapes; otherwise it -falls back to a host-backed CPU surface. `BackendRequest::Metal` is strict: it -returns a resident Metal decode surface for supported shapes or a clear -unsupported/unavailable error. It does not silently return CPU output or -CPU-staged Metal upload. Adapters that expose staged upload do so as a separate -API, and resident-capable surfaces report residency so downstream WSI code can -reject CPU-staged buffers when end-to-end GPU residency is required. - -## Backend story - -There are three target backends. Selection is explicit in the public API. - -- **CPU** — always available. Pure Rust, with SIMD where it is profitable. - This is the baseline; every device path must have a working CPU fallback - with equivalent results to within documented tolerance. -- **Metal** — Apple Silicon macOS. Compute kernels live next to the adapter - crate. The adapter owns its `MTLDevice`/`MTLCommandQueue` session and - produces `MTLBuffer`-backed `DeviceSurface`s so downstream GPU pipelines - can consume the result without an extra download. -- **CUDA** — explicit device-memory output. `Auto` and `Cpu` return CPU-backed - host surfaces. `BackendRequest::Cuda` returns CUDA device memory when the - `cuda-runtime` feature and a CUDA driver are available, and otherwise reports - CUDA as unavailable. JPEG full-frame RGB8 can decode through nvJPEG when - `libnvjpeg` is available; other CUDA shapes use CPU decode plus CUDA upload. - -`BackendRequest::Auto` stays conservative: small or low-yield decodes are -served from CPU; larger batches with supported shapes can be routed to a -device backend. Public routing behavior is documented in the crate-level -READMEs, benchmark notes, and release notes. - -## Lifecycle and ownership - -signinum deliberately externalizes anything that resembles a runtime. - -- **Buffers** are caller-owned. Every `decode_*_into` writes into a slice the - caller provides. -- **Scratch** is caller-owned via `ScratchPool` and reset per operation. WSI - readers reuse pools across tiles to keep allocation off the hot path. -- **Decoder context** is caller-owned via `DecoderContext` and reused - across tiles in a batch. -- **Sessions** for device backends are owned by the adapter and held by the - caller for the lifetime of a batch. They wrap `MTLDevice`/`MTLCommandQueue` - state on Metal. -- **Threading and pyramid policy** are entirely the caller's responsibility. - No crate spawns threads or owns I/O. - -## Where to add things - -| Change | Crate | -|--------|-------| -| New shared trait, pixel format, passthrough contract, or backend kind | `signinum-core` | -| New JPEG decode shape, marker, or CPU optimization | `signinum-jpeg` | -| New JPEG GPU shape | `signinum-jpeg-metal` (or `-cuda`) | -| New diagnostic encode/transcode path | Prefer passthrough in the caller/container layer; otherwise `signinum-j2k-native`, surfaced through `signinum-j2k` | -| New J2K codestream feature, ROI/scaled support | `signinum-j2k-native`, surfaced through `signinum-j2k` | -| New tile decompression codec (e.g. LZ4) | `signinum-tilecodec` | -| New CLI subcommand | `signinum-cli` | -| New conformance fixture | `corpus/conformance/` plus its manifest | -| New regression repro | `corpus/regressions/issue-NNN.` | - -When in doubt, prefer adding to the lowest layer that the new behavior -genuinely requires, and never bypass `signinum-core` to share types -between codec crates. - -## Build and platform - -- Rust edition `2021`, MSRV `1.94`, pinned by [`rust-toolchain.toml`](../rust-toolchain.toml). -- Supported decode hosts: `x86_64` and `aarch64` only. Other targets fail - to build by design. -- Metal adapters compile and run on Apple Silicon macOS. On other hosts the - adapter crate compiles to a fallback surface. -- CUDA adapter crates expose runtime device-memory output for explicit CUDA - requests when built with `cuda-runtime` on hosts with a CUDA driver. Hosts - without CUDA return the documented unavailable error. JPEG full-frame RGB8 - CUDA decode additionally uses `libnvjpeg` when available. -- Release profile: `lto = "fat"`, `codegen-units = 1`, `strip = "symbols"`, - `opt-level = 3`. `release-bench` inherits `release` but keeps debug info. -- Notable feature flags: - - `signinum-j2k-native`: `std` (default), `simd` (default, requires - `std`), `logging`. - - `signinum-jpeg`: `scalar-only` retained for fuzzing and reference. - - `signinum-jpeg-cuda`, `signinum-j2k-cuda`: `cuda-runtime` enables CUDA Driver - API device allocation and explicit CUDA requests. `signinum-jpeg-cuda` - also loads nvJPEG at runtime for full-frame RGB8 JPEG decode when present. - -## Active areas - -These are the surfaces under active change. Treat anything here as -provisional and check the most recent commits before relying on it. +## Backend policy -- Metal adapter hardening: aligning the adapter session model with the - wgpu-hal style, exposing the underlying `MTLBuffer` from `DeviceSurface`, - and pushing more of the J2K full-tile path onto the GPU. -- Adaptive backend routing: deciding when `BackendRequest::Auto` should - upgrade a batch to Metal vs. stay on CPU. -- Keeping the public WSI decode API guide aligned with the core trait surface. -- Broadening release CI and adding self-hosted x86_64 GPU benchmark coverage. +CPU is the correctness baseline. Device adapters can add resident outputs and +stage acceleration, but they must preserve explicit unsupported errors for +unsupported requests. -## Stability posture +CUDA adapters use `j2k-cuda-runtime`, which owns the shared CUDA Driver API +runtime, kernel modules, and host launch orchestration for supported CUDA codec +stages. `cuda-runtime` support is an implementation dependency, not proof of +NVIDIA performance. -The facade release covers `signinum`, `signinum-core`, `signinum-jpeg`, -`signinum-j2k`, `signinum-tilecodec`, and `signinum-cli`. -`signinum-j2k-native`, `signinum-profile`, and `signinum-cuda-runtime` are -published support crates, not primary stable APIs. -Runtime backend selection defaults to `Auto`; supported compiled device paths -may run before falling back to CPU. The Metal and CUDA adapter APIs remain on -the hardening track while broader device decode, encode, and performance work -matures. Breaking changes to any of these surfaces should be reflected here and -in [`CHANGELOG.md`](../CHANGELOG.md). +Metal adapters use `j2k-metal-support` for device, queue, shader-library, +pipeline loading, and route-label helpers. Codec-specific kernels stay in codec +adapter crates. diff --git a/docs/bench.md b/docs/bench.md deleted file mode 100644 index 98e109b1..00000000 --- a/docs/bench.md +++ /dev/null @@ -1,711 +0,0 @@ -# Benchmarking methodology - -`signinum-jpeg` carries two Criterion benchmark targets: - -- `compare` compares `signinum-jpeg` against `jpeg-decoder`, `zune-jpeg`, - and direct `libjpeg-turbo` decode paths on the same JPEG byte streams, and - also carries a signinum-only `decode_rows_rgb` group for large - WSI-oriented inputs. -- `micro` measures signinum-only hot paths that are useful when tuning - regressions inside the crate: header inspect, Huffman symbol decode, and - scalar IDCT. - -The current benchmark contract is native-only: run and compare on `x86_64` -and `aarch64` hosts. wasm and `no_std` builds are no longer part of the -performance signoff path. - -Baseline JPEG encode benchmarks are fallback diagnostics only. WSI/DICOM -storage signoff should first prove compressed-tile passthrough eligibility and -then measure lossless J2K/HTJ2K encode paths for cases that must transcode. - -## Host setup - -The in-tree correctness tests do not require system codec libraries. Comparator -benchmark rows are optional and are enabled only when their local dependency can -be discovered. - -On macOS with Homebrew: - -```sh -brew install pkg-config jpeg-turbo openjpeg -``` - -On Ubuntu/Debian: - -```sh -sudo apt-get update -sudo apt-get install -y pkg-config libturbojpeg0-dev libjpeg-dev openjpeg-tools -``` - -JPEG comparator behavior: - -- `libjpeg-turbo` is discovered with `pkg-config --libs libturbojpeg libjpeg`. -- If it is not available, the `libjpeg-turbo` benchmark rows are skipped. -- Set `SIGNINUM_REQUIRE_LIBJPEG_TURBO=1` on signoff hosts to fail when the - direct comparator is missing. - -JPEG 2000 comparator behavior: - -- OpenJPEG in-process comparator code is provided by the Rust `openjpeg-sys` - dependency. -- The optional `opj_compress` binary is used only for generating one - OpenJPEG-shaped fixture path when it is present; otherwise the bench falls - back to the in-tree encoder for deterministic inputs. -- Grok rows are skipped unless `SIGNINUM_GROK_SOURCE` and - `SIGNINUM_GROK_ROOT` point to a local Grok build with headers and shared - libraries. - -## Compared operations - -- `inspect` - - `signinum-jpeg`: `Decoder::inspect` - - `jpeg-decoder`: `Decoder::read_info` - - `zune-jpeg`: `JpegDecoder::decode_headers` - - `libjpeg-turbo`: reused TurboJPEG handle + `tj3DecompressHeader` -- `decode_rgb` - - `signinum-jpeg`: `Decoder::new` + `decode_into(PixelFormat::Rgb8)` - - `jpeg-decoder`: `Decoder::decode` - - `zune-jpeg`: `JpegDecoder::decode` with RGB output - - `libjpeg-turbo`: reused TurboJPEG handle + `tj3Decompress8(..., TJPF_RGB)` -- `decode_gray` - - `signinum-jpeg`: `Decoder::new` + `decode_into(PixelFormat::Gray8)` - - `jpeg-decoder`: `Decoder::decode` - - `zune-jpeg`: `JpegDecoder::decode` with Luma output - - `libjpeg-turbo`: reused TurboJPEG handle + `tj3Decompress8(..., TJPF_GRAY)` -- `decode_rows_rgb` - - `signinum-jpeg`: `Decoder::new` + `decode_rows` into a `RowSink` - - no cross-crate comparator; TurboJPEG is a packed-output API, so this - remains a signinum-only streaming benchmark for very large WSI JPEGs -- `wsi_tile_batch_rgb` - - `signinum-jpeg`: repeated `Decoder::decode_tile` with a shared - `DecoderContext` and `ScratchPool` - - `libjpeg-turbo`: repeated `tj3Decompress8(..., TJPF_RGB)` with one reused - TurboJPEG handle across the batch -- `wsi_region_rgb` - - `signinum-jpeg`: `Decoder::decode_region_into(..., PixelFormat::Rgb8, roi)` - - `jpeg-decoder` / `zune-jpeg`: full RGB decode, then crop the centered - `256×256` region in memory - - `libjpeg-turbo`: TurboJPEG cropped decode, aligning the left crop boundary - to the scaled iMCU width and trimming the over-read columns in Rust -- `wsi_scaled_rgb_q4` - - `signinum-jpeg`: `decode_scaled(PixelFormat::Rgb8, Downscale::Quarter)` - - `jpeg-decoder` / `zune-jpeg`: full RGB decode, then spatially decimate by - `4×` in memory - - `libjpeg-turbo`: reused TurboJPEG handle + `tj3SetScalingFactor(1/4)` -- `wsi_scaled_rgb_q8` - - `signinum-jpeg`: `decode_scaled(PixelFormat::Rgb8, Downscale::Eighth)` - - `jpeg-decoder` / `zune-jpeg`: full RGB decode, then spatially decimate by - `8×` in memory - - `libjpeg-turbo`: reused TurboJPEG handle + `tj3SetScalingFactor(1/8)` -- `wsi_region_scaled_rgb_q4` - - `signinum-jpeg`: `decode_region_scaled(PixelFormat::Rgb8, roi, Downscale::Quarter)` - - `jpeg-decoder` / `zune-jpeg`: full RGB decode, crop the centered - `256×256` region, then spatially decimate by `4×` - - `libjpeg-turbo`: `tj3SetScalingFactor(1/4)` + cropped decode + left-edge - trim when the ROI is not scaled-iMCU aligned -- `wsi_region_scaled_rgb_q8` - - `signinum-jpeg`: `decode_region_scaled(PixelFormat::Rgb8, roi, Downscale::Eighth)` - - `jpeg-decoder` / `zune-jpeg`: full RGB decode, crop the centered - `256×256` region, then spatially decimate by `8×` - - `libjpeg-turbo`: `tj3SetScalingFactor(1/8)` + cropped decode + left-edge - trim when the ROI is not scaled-iMCU aligned -- `wsi_tile_batch_scaled_rgb_q4` - - `signinum-jpeg`: repeated `decode_tile_scaled_into_in_context(..., PixelFormat::Rgb8, Downscale::Quarter)` - with shared `DecoderContext`, shared `ScratchPool`, and one reused output - buffer - - `jpeg-decoder` / `zune-jpeg`: repeated fresh decode followed by in-memory - `4×` decimation per tile - - `libjpeg-turbo`: repeated scaled TurboJPEG decode with one reused handle -- `wsi_tile_batch_region_scaled_coalesced_rgb_q4` - - `signinum-jpeg`: repeated - `decode_tile_region_scaled_into_in_context(..., PixelFormat::Rgb8, roi, Downscale::Quarter)` - with shared `DecoderContext`, shared `ScratchPool`, and one reused output - buffer; the Metal adapter queues 64 identical region+scaled requests, so - the device path reports a `coalesce_hits_98p4pct` request hit rate by - construction -- `wsi_tile_batch_region_scaled_distinct_rgb_q4` - - `signinum-jpeg`: the same region+scaled tile-batch CPU path over 64 - different RGB JPEG byte streams from one compatible input directory - - `signinum-jpeg-metal`: queued `BatchOp::RegionScaled` over those 64 - distinct byte streams; this is the cold-pan WSI viewport case and should - report `coalesce_hits_0p0pct` - - `jpeg-decoder` / `zune-jpeg`: repeated fresh decode, centered - `256×256` crop, then `4×` in-memory decimation per tile - - `libjpeg-turbo`: repeated scaled cropped decode with one reused handle -- `decode_reused_rgb` / `decode_reused_gray` - - `signinum-jpeg`: `Decoder::new` once per input, then `decode_into` into a - pre-allocated buffer for every iteration — isolates pure decode cost from - `Decoder::new` and output allocation. - - No cross-crate comparator: neither `zune-jpeg` nor `jpeg-decoder` exposes a - reusable decoder, so the fair comparison against them stays in - `decode_rgb` / `decode_gray`. This group is the primary signal for the - WSI tile-batch workload (Phase 3+ scratch-pool path). - -`jpeg-decoder` is benchmarked with `default-features = false` so the comparison -stays single-threaded and does not fold Rayon scheduling into the baseline. -`libjpeg-turbo` is discovered through `pkg-config` at build time; if the local -machine does not expose both `libturbojpeg` and `libjpeg`, the `libjpeg-turbo` -rows are omitted from the compare bench. Set -`SIGNINUM_REQUIRE_LIBJPEG_TURBO=1` when running the direct comparator test on -a signoff host to fail loudly instead of silently skipping the native path. - -For WSI signoff, the primary performance surface is the reduced-output and -tile-batch groups: - -- `wsi_scaled_*` -- `wsi_region_scaled_*` -- `wsi_tile_batch_*` - -Those are the workloads that exploit signinum's structural advantages: -DCT-domain downscale, decode-time crop, and shared decode-state reuse across a -tile stream. Fresh full-frame `decode_rgb` remains a useful generic JPEG -comparison, but it is not the decisive WSI-viewer workload. - -## CPU-first JPEG proving policy - -- Apple Silicon CPU (`aarch64/NEON`) is the first proving host for JPEG - optimization work. -- The acceptance groups for CPU-first JPEG work are: - - `decode_rows_rgb` - - `wsi_region_rgb` - - `wsi_scaled_rgb_q4` - - `wsi_scaled_rgb_q8` - - `wsi_region_scaled_rgb_q4` - - `wsi_region_scaled_rgb_q8` - - `wsi_tile_batch_rgb` - - `wsi_tile_batch_scaled_rgb_q4` - - `wsi_tile_batch_region_scaled_coalesced_rgb_q4` - - `wsi_tile_batch_region_scaled_distinct_rgb_q4` -- Tiny committed fixtures remain useful for `micro_*` and correctness - regression only; they are not valid evidence for WSI performance claims. - -## Inputs - -The benchmark harness always includes the committed conformance fixtures: - -- `corpus/conformance/baseline_420_16x16.jpg` -- `corpus/conformance/grayscale_8x8.jpg` - -Optional local inputs are discovered from `SIGNINUM_BENCH_INPUTS`. The value -is parsed with the platform path separator, so it may contain one or more files -or directories: - -```sh -SIGNINUM_BENCH_INPUTS=/path/to/jpeg_dir:/path/to/extracted_wsi_tiles cargo bench -p signinum-jpeg --bench compare -``` - -Discovery rules: - -- recurse through directories -- accept only `.jpg` / `.jpeg` -- keep only files that `signinum_jpeg::Decoder::new` can decode today -- classify grayscale vs RGB using signinum header info so each file lands in - the matching benchmark group -- classify each file by estimated full-frame output bytes: - `width * height * bytes_per_pixel` -- `BoundedFullFrame` means the estimated full-frame output is `<= 512 MiB` -- `VeryLarge` means the estimate exceeds `512 MiB` or overflows the size math -- comparator `decode_rgb` / `decode_gray` benches include only - `BoundedFullFrame` files -- `decode_rows_rgb` includes only `VeryLarge` RGB files; it does not duplicate - the bounded full-frame cases - -Whole-slide containers such as `.svs` or `.ndpi` are intentionally not decoded -directly by this harness. Extract JPEG tiles first, then point -`SIGNINUM_BENCH_INPUTS` at the extracted tile directories. - -The optional external corpus regression test uses the same `BoundedFullFrame` / -`VeryLarge` classification. It still routes every `VeryLarge` JPEG through -`Decoder::decode_rows`, including grayscale inputs, because that test is -checking practical local-corpus coverage rather than mirroring the benchmark -group names exactly. - -The WSI-native groups (`wsi_region_*`, `wsi_scaled_*`, `wsi_tile_batch_*`) -intentionally compare complete viewer tasks rather than identical library APIs. -`signinum-jpeg` performs crop/downscale during decode and can reuse shared -decode state across a tile batch; the comparator crates do the equivalent work -after a full decode because they do not expose ROI, DCT-domain reduced output, -or shared table/scratch reuse surfaces. - -## Commands - -Compile-only checks: - -```sh -cargo bench -p signinum-j2k --bench public_api --no-run -cargo bench -p signinum --bench facade --no-run -cargo bench -p signinum-jpeg --no-run -``` - -Run the comparator benches: - -```sh -cargo bench -p signinum-jpeg --bench compare -``` - -Run the signinum-only microbenches: - -```sh -cargo bench -p signinum-jpeg --bench micro -``` - -Run the J2K public API benches: - -```sh -cargo bench -p signinum-j2k --bench public_api -``` - -Run the facade dispatch benches: - -```sh -cargo bench -p signinum --bench facade -``` - -Run against local extracted WSI JPEG tiles: - -```sh -SIGNINUM_BENCH_INPUTS="${SIGNINUM_WSI_ROOT:?set SIGNINUM_WSI_ROOT to extracted JPEG tiles}" \ - cargo bench -p signinum-jpeg --bench compare -``` - -Measure Metal fast 4:2:0 full-batch stages: - -```sh -SIGNINUM_JPEG_METAL_FAST420_BATCH_TIMING=1 \ -SIGNINUM_GPU_BENCH_BATCH_DIM=1024 \ -SIGNINUM_GPU_BENCH_BATCH=64 \ - cargo bench -p signinum-jpeg-metal --bench device_upload -- \ - jpeg_metal_batch_decode/metal_rgb8_batch64_surfaces -``` - -The timing mode prints `JPEG Metal fast420 batch timing` lines to stderr with -host setup and GPU wait timings for fused decode and RGB pack. It is diagnostic -only: enabling it splits fused decode and pack into separate command buffers, so -use normal benchmark runs without the env var for acceptance wall-clock numbers. - -## Stage profiling diagnostics - -Stage profiling is for local investigation only. Capture a release-bench -baseline first, enable the narrow diagnostic env var, then compare the stage -rows against Criterion and an external sampler before changing kernels. Use -`=summary` or `=aggregate` for Criterion runs so the profiler emits one -aggregate line per route/stage group instead of one stderr line per iteration. - -Baseline commands: - -```sh -cargo bench --profile release-bench -p signinum-j2k --bench public_api -cargo bench --profile release-bench -p signinum --bench facade -cargo bench --profile release-bench -p signinum-jpeg -cargo bench --profile release-bench -p signinum-j2k-native -cargo bench --profile release-bench -p signinum-jpeg-metal --bench compare --no-run -cargo bench --profile release-bench -p signinum-j2k-metal --bench compare --no-run -``` - -CPU codec stage rows: - -```sh -# One row per operation; useful for focused tests. -SIGNINUM_JPEG_PROFILE_STAGES=1 \ - cargo test -p signinum-jpeg cpu_encoder_round_trips_gray_and_writes_required_markers \ - --test encode_baseline -- --nocapture - -SIGNINUM_J2K_PROFILE_STAGES=1 \ - cargo test -p signinum-j2k-native j2c::encode::tests::test_encode_decode_roundtrip_gray_8bit \ - --features std -- --nocapture - -# Aggregated rows; suitable for release-bench investigation. -SIGNINUM_JPEG_PROFILE_STAGES=summary \ - cargo bench --profile release-bench -p signinum-jpeg --bench compare - -SIGNINUM_J2K_PROFILE_STAGES=summary \ - cargo bench --profile release-bench -p signinum-j2k-metal --bench encode_stages -``` - -Rows and summaries are compact key-value stderr lines: - -```text -signinum_profile codec=jpeg op=encode path=cpu width=... height=... entropy_us=... total_us=... -signinum_profile codec=jpeg op=decode path=cpu mode=region_scaled decode_us=... total_us=... -signinum_profile codec=j2k op=encode path=cpu dwt_us=... block_encode_us=... total_us=... -signinum_profile codec=j2k op=decode path=cpu codeblock_us=... idwt_us=... total_us=... -signinum_profile_summary codec=j2k op=encode path=cpu count=... dwt_us_sum=... dwt_us_avg=... -``` - -GPU route diagnostics: - -```sh -# Helper smoke tests for the env parser and row formatter. -SIGNINUM_GPU_ROUTE_PROFILE=1 cargo test -p signinum-jpeg-metal profile::tests --lib -- --nocapture -SIGNINUM_GPU_ROUTE_PROFILE=1 cargo test -p signinum-j2k-metal profile::tests --lib -- --nocapture -SIGNINUM_GPU_ROUTE_PROFILE=1 cargo test -p signinum-jpeg-cuda profile::tests --lib -- --nocapture -SIGNINUM_GPU_ROUTE_PROFILE=1 cargo test -p signinum-j2k-cuda profile::tests --lib -- --nocapture - -# Route-producing adapter tests on local hardware or fallback paths. -SIGNINUM_GPU_ROUTE_PROFILE=1 \ - cargo test -p signinum-jpeg-metal explicit_metal_unsupported_grayscale_shape_is_rejected \ - --test core_traits -- --nocapture -SIGNINUM_GPU_ROUTE_PROFILE=1 \ - cargo test -p signinum-j2k-metal explicit_metal_unsupported_rgba16_full_decode_is_rejected \ - --test device -- --nocapture -SIGNINUM_GPU_ROUTE_PROFILE=1 \ - cargo test -p signinum-jpeg-cuda explicit_cuda_request_returns_cuda_surface_or_clear_unavailable_error \ - --test host_surface -- --nocapture -SIGNINUM_GPU_ROUTE_PROFILE=1 \ - cargo test -p signinum-j2k-cuda explicit_cuda_request_returns_cuda_surface_or_clear_unavailable_error \ - --test host_surface -- --nocapture - -# Aggregated route decisions for benches. -SIGNINUM_GPU_ROUTE_PROFILE=summary \ - cargo bench --profile release-bench -p signinum-jpeg-metal --bench device_upload -``` - -The helper tests only prove formatting. The adapter tests and benches report -selected backend, fallback reason, batch eligibility, and CUDA/Metal upload or -nvJPEG decisions. They do not replace wall-clock bench numbers because route -logging itself is diagnostic output. - -Existing GPU timing env vars remain more specific: - -- `SIGNINUM_JPEG_METAL_FAST420_BATCH_TIMING=1` splits JPEG Metal fast 4:2:0 - batch work into host setup, GPU wait, and pack timings. -- `SIGNINUM_J2K_METAL_PROFILE_STAGES=1` labels J2K Metal command buffers and - reports Metal-stage GPU duration where available. - -Current J2K Metal lossless encode keeps the direct forward RCT, Tier-1, and -packetization kernels available for explicit device experiments. For host-output -`Auto` encode, route selection is deliberately hybrid: CPU forward RCT, Metal -forward 5/3 DWT, then parallel CPU code-block encode and packetization. The -forced resident Metal path remains available through explicit device requests, -but current single-tile host-output benchmarks route away from it. - -Classic J2K Tier-1 arithmetic coding remains the dominant host-output encode -cost after the hybrid route. When HTJ2K output is acceptable, set -`J2kBlockCodingMode::HighThroughput`; the facade disables the native encoder's -extra HTJ2K self-validation when the caller requested external validation, so -the benchmark measures encode work rather than a second decode of the generated -codestream. - -When stage rows point at a candidate bottleneck, confirm with platform tools -before optimizing: - -- macOS: Instruments Time Profiler and Metal System Trace. -- Linux: `perf record` / `perf report` for CPU paths. -- Cross-platform Firefox Profiler workflow: `samply record -- cargo bench ...`. - -## `signinum-j2k` - -`signinum-j2k` and `signinum-j2k-metal` carry a dedicated Criterion comparator -bench at `crates/signinum-j2k-metal/benches/compare.rs`. - -It uses deterministic runtime-generated codestreams so the bench is always -available without a checked-in J2K corpus: - -- classic grayscale J2K -- classic RGB J2K -- HTJ2K grayscale at 1024 and 512 tile sizes - -Bench groups: - -- `inspect` -- `decode_gray` -- `decode_rgb` -- `wsi_region_gray` -- `wsi_scaled_gray_q4` -- `wsi_region_scaled_gray_q4` -- `wsi_tile_batch_gray` -- `wsi_tile_batch_region_scaled_gray_q4` -- `wsi_tile_batch_region_scaled_gray_distinct_q4` -- `external_wsi_tile_batch_region_scaled_q4` when - `SIGNINUM_J2K_METAL_WSI_TILE_DIR` points at JP2/J2K/JPH/JHC tiles or DICOM - WSI files -- `wsi_tile_batch_gray_32` -- `wsi_tile_batch_gray_64` -- `wsi_tile_batch_rgb` -- `wsi_tile_batch_rgb_distinct` - -Comparator policy: - -- `signinum-j2k` is benchmarked through its public API -- OpenJPEG is benchmarked in-process through `openjpeg-sys` -- Grok is benchmarked in-process through the local `libgrokj2k` shared library - plus a thin C shim compiled into `signinum-j2k-compare` -- all three decoders produce packed `Gray8` or interleaved `Rgb8` output so - output-layout work is included equally in the timing -- the OpenJPEG and Grok comparator paths are forced single-threaded -- classic J2K bench inputs are generated through the local `opj_compress` - binary when available, so the RGB JP2 fixture path matches the OpenJPEG tool - chain; otherwise the bench falls back to the in-tree encoder path -- `opj_compress` is discovered from `SIGNINUM_OPENJPEG_COMPRESS_BIN`, - otherwise `/opt/homebrew/bin/opj_compress` is used when present -- Grok library discovery is controlled by `SIGNINUM_GROK_SOURCE` and - `SIGNINUM_GROK_ROOT`; by default the bench looks for a local Grok build at - `/tmp/grok-signinum` with shared libraries under `/tmp/grok-signinum/build/bin` - -Region and scale mapping: - -- region decode uses the native OpenJPEG decode-area API and Grok region fields -- scaled decode uses native OpenJPEG reduction-factor decode and Grok reduction - decode -- region+scaled decode projects the source-coordinate ROI onto the - reduced-resolution grid with floor-start/ceil-end coverage -- tile-batch decode includes repeated-tile groups and generated distinct-tile - groups; the external WSI group loads distinct JP2/J2K/JPH/JHC files or - encapsulated J2K frames from DICOM files when a local corpus is configured - -Compile the J2K compare bench: - -```sh -cargo bench -p signinum-j2k-metal --bench compare --no-run -``` - -Run it locally against in-process OpenJPEG: - -```sh -SIGNINUM_OPENJPEG_COMPRESS_BIN=/opt/homebrew/bin/opj_compress \ - cargo bench -p signinum-j2k-metal --bench compare -``` - -Run it locally against OpenJPEG and Grok: - -```sh -SIGNINUM_OPENJPEG_COMPRESS_BIN=/opt/homebrew/bin/opj_compress \ -SIGNINUM_GROK_SOURCE=/tmp/grok-signinum \ -SIGNINUM_GROK_ROOT=/tmp/grok-signinum/build/bin \ - cargo bench -p signinum-j2k-metal --bench compare -``` - -## GPU Benchmark Signoff - -Hosted CI compiles benchmark targets but does not provide GPU hardware for -runtime performance claims. Use `.github/workflows/gpu-validation.yml` on -self-hosted runners for GPU signoff: - -- Apple Silicon Metal runners validate Metal adapter tests and can run timed - `signinum-jpeg-metal` and `signinum-j2k-metal` Criterion benches. -- x86_64 CUDA runners validate CUDA device-memory output with `cuda-runtime` - and can run the `signinum-jpeg-cuda` nvJPEG Criterion bench. NVIDIA - performance claims require recorded timed-benchmark output from those hosts. - -Set the manual workflow input `run-timed-benchmarks=true` when collecting -release benchmark evidence. Leave it false for faster device/API validation. - -## Device-output adapters - -`signinum-jpeg-metal` and `signinum-j2k-metal` carry Apple-host device -benches that compare the CPU decode path against the corresponding -Metal-surface path. - -Current v1 scope is explicit: - -- JPEG: supported baseline WSI tile shapes can run Metal kernel paths for full, - region, scaled, region+scaled, and batched RGB device-output decode; - compatible queued region+scaled requests use a real `BatchOp::RegionScaled` - path. The coalesced benchmark intentionally queues 64 identical requests and - can collapse them to one immutable Metal surface; the distinct benchmark - queues 64 different JPEG byte streams so it measures cold-pan batch - throughput instead of duplicate-input reuse. Unsupported shapes fall back - through CPU decode plus device-surface upload according to the requested - backend -- J2K: grayscale full-tile and ROI+scaled MetalDirect paths keep marker parsing - and plan construction on CPU, then dispatch supported classic Tier-1 or - HTJ2K cleanup jobs, grouped sub-band work, IDWT, and store/pack in a resident - Metal command sequence. Distinct grayscale ROI+scaled tile batches are - coalesced across separate codestreams. Cropped ROI+scaled plans prune - irrelevant code-block jobs, compact retained HTJ2K coded payloads, and crop - every required IDWT output level, carrying input-window origins through the - resident band graph so intermediate IDWT levels can feed later cropped levels - safely. RGB ROI+scaled and unsupported codestream features still fall back - through CPU reconstruction plus device-surface upload -- J2K `signinum-adaptive` ROI+scaled batch benches submit through - `BackendRequest::Auto`; the batching layer chooses CPU for short/small - batches and Metal for measured grayscale batch thresholds -- these benches measure complete codec-device tasks, including surface - production; they do not include WSI container parsing, tile lookup, caching, - or prefetch policy -- JPEG baseline encode benches, where present, are compatibility/fallback - measurements and must not be used as evidence for the diagnostic storage - conversion path - -`signinum-jpeg-metal` compare bench names: - -- `decode_rgb` -- `wsi_tile_batch_rgb` -- `wsi_region_rgb` -- `wsi_scaled_rgb_q4` -- `wsi_scaled_rgb_q8` -- `wsi_region_scaled_rgb_q4` -- `wsi_region_scaled_rgb_q8` -- `wsi_tile_batch_scaled_rgb_q4` -- `wsi_tile_batch_region_scaled_coalesced_rgb_q4` -- `wsi_tile_batch_region_scaled_distinct_rgb_q4` -- viewer/composite groups for contiguous and sparse viewport-shaped device - output - -Compile the Metal benches: - -```sh -cargo bench -p signinum-jpeg-metal --bench compare --no-run -cargo bench -p signinum-jpeg-metal --bench device_upload --no-run -cargo bench -p signinum-j2k-metal --bench device_upload --no-run -``` - -Run them on Apple Silicon macOS: - -```sh -cargo bench -p signinum-jpeg-metal --bench compare -- --noplot -cargo bench -p signinum-jpeg-metal --bench device_upload -- --noplot -cargo bench -p signinum-j2k-metal --bench device_upload -- --noplot -``` - -`signinum-jpeg-cuda` and `signinum-j2k-cuda` expose the same device-output API -surface. On hosts with a CUDA driver and the `cuda-runtime` feature, explicit -CUDA requests return CUDA-backed surfaces: - -- `BackendRequest::Cpu` returns a host-backed surface -- `BackendRequest::Auto` falls back to the CPU surface -- `BackendRequest::Cuda` returns a CUDA-backed surface or fails explicitly as - unavailable on non-CUDA hosts - -`signinum-jpeg-cuda` has a full-frame RGB8 nvJPEG path when `libnvjpeg` is -available. Region, scaled, non-RGB8, nvJPEG-unsupported JPEG, and J2K CUDA -requests use CPU decode plus CUDA device-memory upload. - -Compile the CUDA JPEG bench: - -```sh -cargo bench -p signinum-jpeg-cuda --bench device_decode --features cuda-runtime --no-run -``` - -Run it on an NVIDIA host: - -```sh -SIGNINUM_GPU_BENCH_DIM=4096 \ -SIGNINUM_GPU_BENCH_BATCH=64 \ -SIGNINUM_GPU_BENCH_BATCH_DIM=1024 \ - cargo bench -p signinum-jpeg-cuda --bench device_decode --features cuda-runtime -- --noplot -``` - -The CUDA surface and download benchmark cases reuse one `CudaSession`, so they -measure steady-state nvJPEG decode after CUDA context and nvJPEG state -initialization. The CUDA batch case uses nvJPEG batched RGB8 decode and is the -throughput-oriented comparison for many same-sized WSI-style JPEG tiles. - -Set `SIGNINUM_GPU_BENCH_JPEG=/path/to/wsi_tile.jpg` or -`SIGNINUM_CUDA_BENCH_JPEG=/path/to/wsi_tile.jpg` to use a real tile instead of -the generated RGB benchmark JPEG. -Set `SIGNINUM_REQUIRE_CUDA_JPEG_HARDWARE_DECODE=1` when the run must fail -instead of benchmarking the CPU-upload fallback. Small committed fixtures are -useful for compile smoke tests, but realistic GPU comparisons need larger -WSI-shaped JPEG tiles. - -## `signinum-tilecodec` - -`signinum-tilecodec` carries a Criterion comparator bench at -`crates/signinum-tilecodec/benches/compare.rs`. - -It benchmarks four decompression paths: - -- `DeflateCodec` -- `ZstdCodec` -- `LzwCodec` -- `UncompressedCodec` - -Bench group: - -- `decompress_into` - -Comparator policy: - -- `signinum-tilecodec` is benchmarked through the public `TileDecompress` - implementations with reusable typed pools -- Deflate is compared against direct `flate2` decode using the same zlib-backed - implementation family -- Zstd is compared against direct `zstd` stream decode -- LZW is compared against direct `weezl` decode -- Uncompressed is compared against a plain `memcpy` - -Compile the tilecodec compare bench: - -```sh -cargo bench -p signinum-tilecodec --bench compare --no-run -``` - -Run it locally: - -```sh -cargo bench -p signinum-tilecodec --bench compare -``` - -## Recorded baselines - -All numbers on `aarch64-apple-darwin`, Criterion `--quick`, committed -fixtures only. Bigger inputs (local WSI tiles via `SIGNINUM_BENCH_INPUTS`) -are not stored here — rerun locally to capture them per commit. - -Pre-Phase-1 baseline (commit `9678d7d`, scalar-only decoder, audit snapshot): - -| group | input | signinum | jpeg-decoder | zune-jpeg | -|---|---|---|---|---| -| `decode_rgb` | `baseline_420_16x16` | 7.88 µs | 5.52 µs | 3.56 µs | -| `decode_gray` | `grayscale_8x8` | 2.89 µs | 1.68 µs | 1.26 µs | -| `decode_reused_rgb` | `baseline_420_16x16` | 1.19 µs | — | — | -| `decode_reused_gray` | `grayscale_8x8` | 0.22 µs | — | — | -| `micro/idct_reference_block` | — | 42 ns | — | — | -| `micro/upsample_h2v2_fancy_rows_128` | — | 473 ns | — | — | -| `micro/ycbcr_to_rgb_row_scalar_256` | — | 94 ns | — | — | -| `micro/huffman_luma_dc_zero_stream` (2048 syms) | — | 1.19 µs | — | — | - -Signinum is ~2.2× slower than zune on the fresh-mode groups at this -commit. The reused groups show the ceiling: per-tile `Decoder::new` and -output allocation eat 6–13× of the work on small fixtures, which Phase 3 -eliminates entirely. - -Post-Phase-1 snapshot (NEON SIMD ISLOW IDCT, same aarch64-apple-darwin): - -| group | input | signinum | Δ vs pre-Phase-1 | -|---|---|---|---| -| `decode_rgb` | `baseline_420_16x16` | 7.77 µs | -1.4% | -| `decode_gray` | `grayscale_8x8` | 2.92 µs | ~0 | -| `decode_reused_rgb` | `baseline_420_16x16` | 1.10 µs | -7.6% | -| `decode_reused_gray` | `grayscale_8x8` | 0.20 µs | -10.4% | -| `micro/idct_islow_scalar_block` | — | 38 ns | — | -| `micro/idct_islow_neon_block` | — | 18 ns | **-53%** (2.1× faster) | - -Fresh-mode barely moves because `Decoder::new` dominates the 16×16 fixture -(IDCT is 8–10% of the work). On real WSI tiles (256×256+) where hundreds -of IDCTs run per decode, the 2.1× kernel win will compound proportionally. - -Post-Phase-3 snapshot (ScratchPool + Phase 1 NEON IDCT): - -| group | input | signinum | zune-jpeg | signinum advantage | -|---|---|---|---|---| -| `decode_scratch_rgb` | `baseline_420_16x16` | **920 ns** | 3614 ns (fresh-mode) | **3.9× faster** | -| `decode_scratch_gray` | `grayscale_8x8` | **75 ns** | 1253 ns (fresh-mode) | **16.7× faster** | -| `decode_reused_rgb` | `baseline_420_16x16` | 1235 ns | — | — | -| `decode_reused_gray` | `grayscale_8x8` | 260 ns | — | — | - -The `decode_scratch_*` groups reuse both the `Decoder` and a -pre-allocated `ScratchPool` — the realistic WSI tile-batch shape. The -comparison against zune's fresh-mode decode isn't apples-to-apples on -allocation strategy (zune has no reusable decoder) but it does reflect -the real integration cost a WSI reader would pay. For a tile-batch -reader of 1000 tiles, we issue 1000 `decode_into_with_scratch` calls, -each at the scratch-group speed; zune's reader pays the fresh-mode -`JpegDecoder::new` + internal allocation on every tile. - -Phase 4 partial (AVX2 ISLOW IDCT on x86_64): - -AVX2 IDCT is wired in `src/idct/avx2.rs` using 128-bit SSE4.1 intrinsics -in the same 4-lane i32 structure as NEON. Coverage is validated by the -same proptest harness under `#[cfg(target_arch = "x86_64")]` plus -hand-picked edges. x86_64 runtime numbers land when CI (or a local Linux -host) runs `cargo bench -p signinum-jpeg --bench micro`; expected ratio -is ≥2× over scalar, matching NEON. - -## Policy - -- Benchmark results are report-only for now; CI compiles the benches but does - not fail on runtime performance deltas. -- libjpeg-turbo remains the primary JPEG parity oracle, and it is now also a - direct speed comparator when available locally through `pkg-config`. diff --git a/docs/env-vars.md b/docs/env-vars.md new file mode 100644 index 00000000..584c773b --- /dev/null +++ b/docs/env-vars.md @@ -0,0 +1,169 @@ +# Environment Variables + +This is the supported `J2K_*` environment-variable surface for the +workspace. Variables not listed here are internal symbols, generated metadata, +or test-only implementation details and must not be treated as user controls. + +Stability values: + +- Stable: supported for the v0.5.x release line. +- Experimental: accepted for diagnostics or adapter tuning, but may change + before 1.0. +- Test/CI: supported only for repository tests, CI, and release validation. +- Benchmark: supported only for benchmark harnesses and benchmark signoff. +- Generated: emitted by a build script for version reporting; do not set by + hand unless reproducing the build-script contract. + +## Library Runtime And Profiling + +| Variable | Effect | Default | Stability | +| --- | --- | --- | --- | +| `J2K_GPU_ROUTE_PROFILE` | Emits facade/adapter GPU route decisions. Use `1` for rows or `summary` for aggregate rows. | Disabled | Experimental | +| `J2K_JPEG_PROFILE_STAGES` | Emits JPEG CPU profiling rows. Use `1` for rows or `summary` where supported. | Disabled | Experimental | +| `J2K_PROFILE_STAGES` | Emits native/CUDA J2K profiling rows. Use `1` for rows or `summary` where supported. | Disabled | Experimental | +| `J2K_CUDA_TRACE` | Writes CUDA HTJ2K profile trace JSON to the configured path. | No trace file | Experimental | +| `J2K_CUDA_IDWT_TRACE` | Enables CUDA IDWT trace/profile output. | Disabled | Experimental | +| `J2K_CUDA_DISABLE_STAGE_TIMINGS` | Disables CUDA stage timing collection for benchmark runs. | Timings enabled | Experimental | +| `J2K_CUDA_DISABLE_DWT97_FUSED_COLUMN_QUANTIZE` | Disables the fused CUDA DWT 9/7 column quantize path. | Fused path enabled when supported | Experimental | +| `J2K_CUDA_DISABLE_COMPACT_PREENCODED` | Forces the CUDA transcode adapter to decline compact preencoded resident HT encode support. | Compact resident support enabled when supported | Experimental | +| `J2K_JPEG_METAL_FAST420_BATCH_TIMING` | Emits JPEG Metal fast 4:2:0 batch timing profile rows when set to `1`. | Disabled | Experimental | +| `J2K_METAL_PROFILE_STAGES` | Enables J2K Metal stage profile rows when set to `1`. | Disabled | Experimental | +| `J2K_METAL_PROFILE_SIGNPOSTS` | Enables J2K Metal OS signposts when set to `1` and stage profiling is enabled. | Disabled | Experimental | +| `J2K_METAL_PROFILE_DECODE_LABEL` | Labels J2K Metal decode profile rows. Non-alphanumeric characters are sanitized. | `unlabeled` | Experimental | +| `J2K_METAL_PROFILE_DECODE_SPLIT_COMMANDS` | Adds split-command decode timing rows when J2K Metal stage profiling is enabled. | Disabled | Experimental | +| `J2K_METAL_PROFILE_COEFFICIENT_PREP_SPLIT_COMMANDS` | Adds split-command coefficient-prep timing rows when J2K Metal stage profiling is enabled. | Disabled | Experimental | +| `J2K_METAL_PROFILE_CLASSIC_TIER1_DENSITY` | Emits classic Tier-1 density profiling when J2K Metal stage profiling is enabled. | Disabled | Experimental | +| `J2K_METAL_PROFILE_CLASSIC_TIER1_RAW_PACK` | Emits classic Tier-1 raw-pack profiling when J2K Metal stage profiling is enabled. | Disabled | Experimental | +| `J2K_METAL_PROFILE_CLASSIC_TIER1_ARITHMETIC_PACK` | Emits classic Tier-1 arithmetic-pack profiling when J2K Metal stage profiling is enabled. | Disabled | Experimental | +| `J2K_METAL_PROFILE_CLASSIC_TIER1_SYMBOL_PLAN` | Emits classic Tier-1 symbol-plan profiling when J2K Metal stage profiling is enabled. | Disabled | Experimental | +| `J2K_METAL_PROFILE_CLASSIC_TIER1_PASS_PLAN` | Emits classic Tier-1 pass-plan profiling and also enables symbol-plan profiling. | Disabled | Experimental | +| `J2K_METAL_PROFILE_CLASSIC_TIER1_TOKEN_EMIT` | Emits classic Tier-1 token-emission profiling when J2K Metal stage profiling is enabled. | Disabled | Experimental | +| `J2K_METAL_PROFILE_CLASSIC_TIER1_SPLIT_TOKEN_EMIT` | Emits classic Tier-1 split-token-emission profiling when J2K Metal stage profiling is enabled. | Disabled | Experimental | +| `J2K_METAL_PROFILE_CLASSIC_TIER1_TOKEN_PACK` | Emits classic Tier-1 token-pack profiling and also enables token-emission profiling. | Disabled | Experimental | +| `J2K_TRANSCODE_METAL_PROFILE_STAGES` | Enables transcode Metal profiling in the DCT 9/7 benchmark harness. | Disabled | Benchmark | + +## Experimental Backend Routing + +| Variable | Effect | Default | Stability | +| --- | --- | --- | --- | +| `J2K_METAL_HT_PACKET_CAPACITY` | Overrides J2K Metal HT packet buffer capacity. Invalid values are ignored by the fallback parser. | Built-in capacity | Experimental | +| `J2K_METAL_CLASSIC_SELECTIVE_BYPASS` | Set to `0` to disable classic selective arithmetic coding bypass in the Metal resident style flags. | Selective bypass enabled | Experimental | +| `J2K_METAL_CLASSIC_TIER1_GPU_TOKEN_PACK` | Requests the classic Tier-1 GPU token-pack route when supported. | Disabled | Experimental | +| `J2K_METAL_CLASSIC_TIER1_SPLIT_GPU_TOKEN_PACK` | Requests the split classic Tier-1 GPU token-pack route. | Disabled | Experimental | +| `J2K_METAL_CLASSIC_TIER1_SPLIT_MQ_BYTE_GPU_TOKEN_PACK` | Set to `1` to request, or `0` to disable, the split MQ-byte GPU token-pack route. | Auto | Experimental | +| `J2K_HYBRID_FLAT_CPU_TIER1` | Forces flat CPU Tier-1 batching for J2K Metal hybrid decode when set to a truthy value accepted by the adapter. | Adapter default | Experimental | +| `J2K_CUDA_OXIDE_ARCH` | Overrides the cuda-oxide build target when a `j2k-cuda-runtime/cuda-oxide-*` feature is enabled. | `sm_80` | Experimental | +| `J2K_CUDA_USE_OXIDE_J2K_DECODE_STORE` | Routes supported J2K decode store and inverse-MCT CUDA kernels through the cuda-oxide PTX when `j2k-cuda-runtime/cuda-oxide-j2k-decode-store` is enabled and PTX was generated. | Built-in CUDA C PTX | Experimental | +| `J2K_CUDA_USE_OXIDE_J2K_DEQUANTIZE` | Routes supported J2K HTJ2K dequantize CUDA kernels through the cuda-oxide PTX when `j2k-cuda-runtime/cuda-oxide-j2k-dequantize` is enabled and PTX was generated. | Built-in CUDA C PTX | Experimental | +| `J2K_CUDA_USE_OXIDE_J2K_ENCODE` | Routes supported J2K encode-stage CUDA kernels, HTJ2K encoded-byte compaction, and HTJ2K packetization through the cuda-oxide PTX when `j2k-cuda-runtime/cuda-oxide-j2k-encode` is enabled and PTX was generated. | Built-in CUDA C PTX | Experimental | +| `J2K_CUDA_USE_OXIDE_J2K_IDWT` | Routes supported generic J2K inverse-DWT CUDA kernels through the cuda-oxide PTX when `j2k-cuda-runtime/cuda-oxide-j2k-idwt` is enabled and PTX was generated. | Built-in CUDA C PTX | Experimental | +| `J2K_CUDA_USE_OXIDE_TRANSCODE` | Routes supported reversible 5/3 and irreversible 9/7 transcode CUDA kernels through the cuda-oxide PTX when `j2k-cuda-runtime/cuda-oxide-transcode` is enabled and PTX was generated. This includes staged, batched, code-block quantize, and fused i16 9/7 paths. | Built-in CUDA C PTX | Experimental | +| `J2K_REQUIRE_CUDA_OXIDE_COPY_U8` | Requires cuda-oxide CopyU8 PTX generation when `j2k-cuda-runtime/cuda-oxide-copy-u8` is enabled. | Disabled | Experimental | +| `J2K_REQUIRE_CUDA_OXIDE_J2K_DECODE_STORE` | Requires cuda-oxide J2K decode store/inverse-MCT PTX generation when `j2k-cuda-runtime/cuda-oxide-j2k-decode-store` is enabled. | Disabled | Experimental | +| `J2K_REQUIRE_CUDA_OXIDE_J2K_DEQUANTIZE` | Requires cuda-oxide J2K HTJ2K dequantize PTX generation when `j2k-cuda-runtime/cuda-oxide-j2k-dequantize` is enabled. | Disabled | Experimental | +| `J2K_REQUIRE_CUDA_OXIDE_J2K_ENCODE` | Requires cuda-oxide J2K encode-stage, HTJ2K compaction, and HTJ2K packetization PTX generation when `j2k-cuda-runtime/cuda-oxide-j2k-encode` is enabled. | Disabled | Experimental | +| `J2K_REQUIRE_CUDA_OXIDE_J2K_IDWT` | Requires cuda-oxide J2K generic inverse-DWT PTX generation when `j2k-cuda-runtime/cuda-oxide-j2k-idwt` is enabled. | Disabled | Experimental | +| `J2K_REQUIRE_CUDA_OXIDE_TRANSCODE` | Requires cuda-oxide reversible 5/3 transcode PTX generation when `j2k-cuda-runtime/cuda-oxide-transcode` is enabled. | Disabled | Experimental | + +## Test And CI Gates + +| Variable | Effect | Default | Stability | +| --- | --- | --- | --- | +| `J2K_REQUIRE_OPENJPEG` | Makes OpenJPEG parity tests fail instead of skip when OpenJPEG tools are unavailable. | Skip unavailable comparator paths | Test/CI | +| `J2K_REQUIRE_GROK` | Makes Grok parity tests fail instead of skip when Grok tools or libraries are unavailable. | Skip unavailable comparator paths | Test/CI | +| `J2K_REQUIRE_LIBJPEG_TURBO` | Makes libjpeg-turbo comparison tests fail instead of skip when the bench feature/tooling is unavailable. | Skip unavailable comparator path | Test/CI | +| `J2K_REQUIRE_CUDA_RUNTIME` | Makes CUDA tests and NVIDIA comparison harnesses require a usable CUDA runtime instead of skipping. | Skip runtime-only CUDA paths | Test/CI | +| `J2K_REQUIRE_CUDA_HTJ2K_STRICT` | Requires CUDA HTJ2K strict validation and makes CUDA kernel build failures fatal in the runtime build script. | Non-strict when runtime unavailable | Test/CI | +| `J2K_REQUIRE_CUDA_KERNEL_BUILD` | Makes CUDA kernel compilation failures fatal in the runtime build script. | Kernel build may be skipped when unsupported | Test/CI | +| `J2K_REQUIRE_CUDA_JPEG_HARDWARE_DECODE` | Requires CUDA JPEG hardware decode coverage in relevant CUDA tests/benches. | Hardware decode may skip | Test/CI | +| `J2K_REQUIRE_CUDA_BENCH` | Makes CUDA benchmark probes fail instead of skip when CUDA is unavailable or does not dispatch. | Skip unavailable CUDA benchmark paths | Benchmark | +| `J2K_REQUIRE_METAL_BENCH` | Makes Metal benchmark probes fail instead of skip when Metal is unavailable or does not dispatch. | Skip unavailable Metal benchmark paths | Benchmark | +| `J2K_REQUIRE_NV_BASELINE_BUILD` | Makes the standalone GPU baseline harness require its CUDA build dependencies. | Baseline build may skip unavailable pieces | Test/CI | +| `J2K_RUN_HOSTED_J2K_METAL_RUNTIME_TESTS` | Allows hosted macOS CI to run J2K Metal runtime tests that are otherwise skipped there. | Hosted runtime tests skipped | Test/CI | +| `J2K_REQUIRE_WSI_ROOT` | Makes external JPEG WSI tests fail if `J2K_WSI_ROOT` is missing or empty. | Skip external WSI tests | Test/CI | +| `J2K_REQUIRE_TRANSCODE_WSI_ROOT` | Makes transcode corpus validation require `J2K_TRANSCODE_WSI_ROOT`. | Skip external transcode WSI corpus | Test/CI | +| `J2K_REQUIRE_NDPI` | Makes NDPI passthrough tests fail if `J2K_NDPI_PATH` is missing. | Skip NDPI passthrough test | Test/CI | + +## External Data And Comparator Paths + +| Variable | Effect | Default | Stability | +| --- | --- | --- | --- | +| `J2K_WSI_ROOT` | Path list root for external JPEG WSI integration tests. | Not set | Test/CI | +| `J2K_TRANSCODE_WSI_ROOT` | Path list root for external JPEG-to-HTJ2K transcode corpus validation. | Not set | Test/CI | +| `J2K_TRANSCODE_WSI_TILE_LIMIT` | Maximum number of external WSI tiles used by transcode corpus validation. | Built-in validation default | Test/CI | +| `J2K_TRANSCODE_WSI_MAX_PAYLOAD_BYTES` | Maximum external WSI tile payload accepted by transcode corpus validation. | Built-in validation default | Test/CI | +| `J2K_NDPI_PATH` | Path to an NDPI slide for passthrough tests. | Not set | Test/CI | +| `J2K_NDPI_TILE_LIMIT` | Maximum NDPI tiles inspected by the passthrough test; `0` means no explicit tile-count cap. | Test default | Test/CI | +| `J2K_NDPI_MAX_PAYLOAD_BYTES` | Maximum NDPI tile payload accepted by the passthrough test. | Test default | Test/CI | +| `J2K_ISO_CONFORMANCE_DIR` | Path to ISO J2K conformance vectors for ignored/external tests. | Not set | Test/CI | +| `J2K_APERIO_TILE_FIXTURE` | Path to an Aperio J2K tile fixture for the ignored lossless encode test. | Not set | Test/CI | +| `J2K_PARITY_CORPUS_MANIFEST` | Manifest path used by `scripts/parity-corpus-fetch.sh`. | `corpus/wsi-samples/manifest.json` or first script argument | Test/CI | +| `J2K_PARITY_CORPUS_DIR` | Output directory used by `scripts/parity-corpus-fetch.sh`. | `corpus/wsi-samples` or second script argument | Test/CI | +| `J2K_PARITY_CORPUS_MAX_BYTES` | Maximum accepted byte size for each downloaded parity-corpus fixture. | `536870912` | Test/CI | +| `J2K_OPENJPEG_BIN` | Override for OpenJPEG `opj_decompress` in J2K parity tests. | `opj_decompress` on `PATH` | Test/CI | +| `J2K_OPENJPEG_DECOMPRESS_BIN` | Override for OpenJPEG `opj_decompress` in benchmark reports. | `opj_decompress` on `PATH` | Benchmark | +| `J2K_OPENJPEG_COMPRESS_BIN` | Override for OpenJPEG `opj_compress` in J2K parity tests. | `opj_compress` on `PATH` | Test/CI | +| `J2K_GROK_BIN` | Override for Grok `grk_decompress` in J2K parity tests. | `grk_decompress` on `PATH` | Test/CI | +| `J2K_GROK_COMPRESS_BIN` | Override for Grok `grk_compress` in J2K parity tests. | `grk_compress` on `PATH` | Test/CI | +| `J2K_GROK_ROOT` | Path to Grok installation/library root for in-process comparator builds and benchmark reports. | Not set | Test/CI | +| `J2K_GROK_SOURCE` | Path to Grok source used by the in-process comparator build script. | Not set | Test/CI | +| `J2K_GROK_VERSION` | Build-script emitted Grok version metadata consumed by the comparator crate. | `unavailable` if not emitted | Generated | +| `J2K_GROK_LIB_DIR` | Build-script emitted Grok library path metadata consumed by the comparator crate. | `unavailable` if not emitted | Generated | + +## Benchmark Harnesses + +| Variable | Effect | Default | Stability | +| --- | --- | --- | --- | +| `J2K_BENCH_INPUTS` | Path-list of JPEG inputs for JPEG and Metal benchmark/report harnesses. | Harness-specific generated fixtures | Benchmark | +| `J2K_BENCH_INPUT_SOURCE` | Input-source label recorded by `cargo xtask bench-report`. | Falls back to `J2K_BENCH_INPUTS`, otherwise `not recorded` | Benchmark | +| `J2K_BENCH_COMMAND` | Command label recorded by `cargo xtask bench-report`. | `not recorded` | Benchmark | +| `J2K_BENCH_SKIPPED_ROWS` | Semicolon-separated skipped-row reasons recorded by `cargo xtask bench-report`. | Empty | Benchmark | +| `J2K_BENCH_JPEG_DIR` | Directory of JPEG tiles for NVIDIA baseline and transcode comparison benches. | Harness-specific fixture fallback or required by workflow | Benchmark | +| `J2K_BENCH_SVS` | GPU workflow input SVS path used to extract benchmark JPEG tiles when `J2K_BENCH_JPEG_DIR` is unset. | Not set | Benchmark | +| `J2K_REPORT_ITERS` | Iteration count for custom report-style JPEG benchmarks. | Harness default | Benchmark | +| `J2K_ALLOC_REPORT` | Enables allocation report output in the CPU JPEG encode benchmark. | Disabled | Benchmark | +| `J2K_FORCE_FULL_FRAME` | Forces benchmark classification to full-frame mode. | Auto classification | Benchmark | +| `J2K_JPEG_BATCH_THREADS` | Overrides JPEG benchmark batch thread count. | Rayon/runtime default | Benchmark | +| `J2K_JPEG_TILE_BATCH_SIZE` | Overrides JPEG tile batch size in benchmark comparison runs. | Harness default | Benchmark | +| `J2K_JPEG_ENCODE_BENCH_DIM` | Image dimensions for the JPEG Metal baseline encode benchmark. | Harness default | Benchmark | +| `J2K_JPEG_ENCODE_BENCH_BATCH` | Batch size for the JPEG Metal baseline encode benchmark. | Harness default | Benchmark | +| `J2K_JPEG_ENCODE_BENCH_QUALITY` | Quality for the JPEG Metal baseline encode benchmark. | Harness default | Benchmark | +| `J2K_GPU_BENCH_JPEG` | JPEG fixture path for GPU JPEG upload/decode benchmarks. | Generated fixture unless strict bench requires a file | Benchmark | +| `J2K_CUDA_BENCH_JPEG` | CUDA-specific JPEG fixture path; falls back to `J2K_GPU_BENCH_JPEG` in CUDA JPEG benches. | Not set | Benchmark | +| `J2K_GPU_BENCH_SMALL_FIXTURE` | Allows GPU JPEG benches to use small generated fixtures. | Disabled | Benchmark | +| `J2K_GPU_BENCH_DIM` | Generated GPU benchmark dimensions, usually `N` or `WIDTHxHEIGHT`. | Harness default | Benchmark | +| `J2K_GPU_BENCH_BATCH` | Generated GPU benchmark batch count. | Harness default | Benchmark | +| `J2K_GPU_BENCH_BATCH_DIM` | Generated GPU benchmark batch tile dimensions. | Harness default | Benchmark | +| `J2K_GPU_BENCH_RESTART_INTERVAL` | Restart interval for generated GPU JPEG upload benchmark fixtures. | Harness default | Benchmark | +| `J2K_GPU_BENCH_SUBSAMPLING` | Generated GPU JPEG benchmark subsampling, accepted values depend on the harness. | Harness default | Benchmark | +| `J2K_CUDA_BENCH_SUBSAMPLING` | CUDA-specific generated JPEG benchmark subsampling; falls back with `J2K_GPU_BENCH_SUBSAMPLING`. | Harness default | Benchmark | +| `J2K_JPEG_COMPARE_QUALITY` | Quality for the standalone NVIDIA JPEG comparison fixture. | Harness default | Benchmark | +| `J2K_JPEG_COMPARE_ITERS` | Iteration count for the standalone NVIDIA JPEG comparison fixture. | Harness default | Benchmark | +| `J2K_JPEG_COMPARE_WARMUP` | Warmup iteration count for the standalone NVIDIA JPEG comparison fixture. | Harness default | Benchmark | +| `J2K_JPEG_COMPARE_SUBSAMPLING` | Subsampling for the standalone NVIDIA JPEG comparison fixture. | Falls back to `J2K_CUDA_BENCH_SUBSAMPLING` | Benchmark | +| `J2K_JPEG_COMPARE_PATTERN` | Generated image pattern for the standalone NVIDIA JPEG comparison fixture. | Harness default | Benchmark | +| `J2K_COMPARE_THREADS` | Thread count for J2K comparator signoff and benchmark reports. | Comparator default or `not set` in reports | Benchmark | +| `J2K_BATCH_COMPARE_REPEATS` | Repeat count for `jp2k_batch_compare`. | Tool default | Benchmark | +| `J2K_BATCH_COMPARE_THREADS` | Worker count for `jp2k_batch_compare`. | Tool default | Benchmark | +| `J2K_ROI_COMPARE_REPEATS` | Repeat count for `jp2k_roi_batch_compare`. | Tool default | Benchmark | +| `J2K_ROI_COMPARE_THREADS` | Worker count for `jp2k_roi_batch_compare`. | Tool default | Benchmark | +| `J2K_CUDA_DECODE_FORMATS` | Comma-separated CUDA J2K decode benchmark output formats such as `gray8,rgb8,rgba8`. | Harness default | Benchmark | +| `J2K_CUDA_DECODE_BATCH_SIZES` | Comma-separated CUDA J2K decode benchmark batch sizes. | Harness default | Benchmark | +| `J2K_CUDA_PROFILE_BATCH_SIZE` | Batch size for the CUDA HTJ2K decode profile example. | Example default | Benchmark | +| `J2K_CUDA_PROFILE_ITERATIONS` | Iteration count for the CUDA HTJ2K decode profile example. | Example default | Benchmark | +| `J2K_LEVEL1_CUDA_HT_MIN_MPS` | Level-1 CUDA HT throughput floor used by GPU validation workflow. | Workflow default `350` | Benchmark | +| `J2K_LEVEL1_CUDA_HT_MIN_SPEEDUP_VS_NVIDIA` | Level-1 CUDA HT speedup floor versus NVIDIA baseline in GPU validation workflow. | Workflow default `4.0` | Benchmark | +| `J2K_LEVEL2_CUDA_HT_MIN_MPS` | Level-2 CUDA HT throughput floor used by GPU validation workflow. | Workflow default `60` | Benchmark | +| `J2K_LEVEL2_CUDA_HT_MIN_SPEEDUP_VS_NVIDIA` | Level-2 CUDA HT speedup floor versus NVIDIA baseline in GPU validation workflow. | Workflow default `1.10` | Benchmark | + +## Xtask And Release Tooling + +| Variable | Effect | Default | Stability | +| --- | --- | --- | --- | +| `J2K_FUZZ_RUNS` | Number of runs passed to each `cargo xtask fuzz-run` target. | `1000` | Test/CI | +| `J2K_FUZZ_MAX_TOTAL_TIME_SECONDS` | Optional libFuzzer max total time for `cargo xtask fuzz-run`. | Not passed | Test/CI | +| `J2K_FUZZ_TARGET` | Target triple passed to `cargo fuzz run --target` by `cargo xtask fuzz-run`. | Nightly host target from `rustc -vV` | Test/CI | +| `J2K_SEMVER_TOOLCHAIN` | Rust toolchain used by `cargo xtask semver`. | `stable` | Test/CI | + +CI overrides the fuzz defaults to 512 runs / 60 seconds for pull requests and +20,000 runs / 900 seconds for the scheduled long fuzz job. diff --git a/docs/parity.md b/docs/parity.md deleted file mode 100644 index 48662fab..00000000 --- a/docs/parity.md +++ /dev/null @@ -1,35 +0,0 @@ -# Parity Strategy - -`signinum` keeps parity checks close to the codec surface instead of relying -on a single visual smoke test. - -## JPEG - -- Primary conformance fixtures live in `corpus/conformance/manifest.json` and - compare decoded bytes against libjpeg-turbo-generated raw outputs. -- WSI-shaped fixtures and policy checks live in the `signinum-jpeg` test and - bench suites. -- Tolerance is bit-exact for the committed baseline fixtures. Any future lossy - tolerance must be recorded per fixture in the manifest. - -## JPEG 2000 / HTJ2K - -- CPU parity tests compare generated codestreams against the in-repo native - engine and, where available, OpenJPEG/Grok comparator paths. -- ROI, scaled, combined ROI+scaled, row, and tile-batch surfaces are tested as - API behavior, not only as full-frame decode. -- Metal and CUDA-named adapter crates must preserve CPU parity for fallback - host surfaces. Metal crates must preserve decoded bytes for explicit - Metal-backed ROI+scaled surfaces. CUDA upload-fallback surfaces must preserve - decoded bytes after download. The nvJPEG full-frame RGB8 path in - `signinum-jpeg-cuda` is checked against the CPU reference with a small - vendor-IDCT tolerance and reports hardware decode separately from copy - fallback stats. - -## Maintenance Rules - -- Every committed conformance input must be listed in the matching manifest. -- Fixture generation scripts are maintainer tools; CI reads committed fixtures - and does not regenerate them. -- New codec behavior needs at least one focused parity or regression test - before benchmark numbers are updated. diff --git a/docs/release.md b/docs/release.md index c766ae05..7beb401c 100644 --- a/docs/release.md +++ b/docs/release.md @@ -1,88 +1,80 @@ -# Release Notes +# Release Policy -## Current State +The repository is staged for the `j2k` public crate release. Runtime backend selection defaults to `Auto`; CPU remains the portable baseline while supported device paths are selected only with validation and benchmark evidence. -The repository is staged for the `signinum` facade release. The stable release -artifacts are `signinum`, `signinum-core`, `signinum-jpeg`, `signinum-j2k`, -`signinum-tilecodec`, and `signinum-cli`. `signinum-cuda-runtime`, -`signinum-profile`, and `signinum-j2k-native` are published support crates so -the public runtime crates that depend on them can be installed from crates.io. +## Versions and publish order -Metal and CUDA adapter crates are published as pre-1.0 artifacts where their -APIs changed for the facade boundary. -Runtime backend selection defaults to `Auto`; supported compiled device paths -may run before CPU fallback. -CUDA explicit requests can produce CUDA device memory surfaces when built with -`cuda-runtime` on a host with a CUDA driver. `signinum-jpeg-cuda` can use -NVIDIA nvJPEG for full-frame RGB8 JPEG decode when `libnvjpeg` is installed; -unsupported JPEG shapes and the J2K CUDA adapter still use CPU decode plus -CUDA device memory upload. NVIDIA performance claims require self-hosted GPU -benchmark evidence. +Release scripts must use manifest versions. Do not publish from stale hard-coded crate/version pairs. -## Verification Gates +Real publishes must run from tag `v`. All publishable crates must share that workspace version. If a crate version is already on crates.io, the publish script fails by default; set `CRATES_IO_ALLOW_PUBLISHED_RERUN=true` only for an intentional idempotent rerun. + +Publish in this order: + +1. `j2k-core` +2. `j2k-profile` +3. `j2k-types` +4. `j2k-cuda-runtime` +5. `j2k-metal-support` +6. `j2k-native` +7. `j2k-jpeg` +8. `j2k-tilecodec` +9. `j2k` +10. `j2k-transcode` +11. `j2k-transcode-cuda` +12. `j2k-jpeg-metal` +13. `j2k-metal` +14. `j2k-transcode-metal` +15. `j2k-jpeg-cuda` +16. `j2k-cuda` +17. `j2k-cli` + +Publish preflight must account for staged unpublished workspace dependencies. +Use package listing and dry-run checks according to dependency availability: + +```bash +cargo package --list +cargo publish --dry-run +``` + +Some downstream packages may be validated with `cargo package --list` while +strict dry-run publishing is blocked by unpublished workspace dependencies. + +Run this before publishing: + +```bash +cargo xtask release-integrity +``` + +The integrity gate parses cargo metadata, manifests, `.github/workflows/publish.yml`, and this release document. It fails if a publishable workspace crate is missing from publish order, docs.rs metadata, semver/doc gates, or release docs, or if a workspace crate is neither publishable nor explicitly `publish = false`. + +## Required gates Hosted CI must pass before release staging: -1. `cargo fmt --all -- --check` -2. `cargo clippy --workspace --all-targets --all-features -- -D warnings` -3. `cargo xtask test` on Linux x86_64, Linux aarch64, and Apple Silicon - macOS runners -4. `cargo doc --workspace --all-features --no-deps` with rustdoc warnings - denied -5. Benchmark compile checks for JPEG, JPEG Metal, J2K Metal, and tilecodec - -Runtime GPU validation is intentionally separate because hosted GitHub runners -do not provide the required devices. Run `.github/workflows/gpu-validation.yml` -on self-hosted runners before claiming Metal runtime validation: - -1. Apple Silicon Metal runner labels: `self-hosted`, `macOS`, `ARM64`, - `metal` -2. x86_64 CUDA runner labels: `self-hosted`, `Linux`, `X64`, `cuda` -3. Use the `run-timed-benchmarks` workflow input when a release needs measured - GPU benchmark timing rather than compile-only coverage - -Passing the CUDA self-hosted job validates `cuda-runtime` device-memory output -and the opt-in nvJPEG JPEG decode path on a CUDA runner. Timed NVIDIA -performance claims require the `run-timed-benchmarks` workflow input and -recorded benchmark output. - -## Crates.io - -Crates.io publication is staged because workspace crates depend on each other. -Before publishing, run `cargo xtask package` from a clean worktree. The package -preflight runs `cargo package --list` for every publishable crate, -then runs strict `cargo package --no-verify` only for crates that do not depend -on unpublished workspace versions. Downstream crates such as -`signinum-j2k-native`, `signinum-jpeg`, `signinum-tilecodec`, `signinum-j2k`, -adapter crates, `signinum-cli`, and `signinum` cannot pass strict pre-publish -packaging until the prior staged crates exist on crates.io, because Cargo -resolves their versioned path dependencies against the registry during -packaging. - -This is an unpublished workspace dependencies limit, not a package content -failure. The publish workflow's dry-run mode mirrors that limit: it uses -`cargo publish --dry-run` for registry-independent crates and -`cargo package --list` for crates blocked only by unpublished workspace -dependencies. Real publishes still run `cargo publish` in dependency order. - -The crates.io publish order uses the current manifest versions and is enforced -by `.github/workflows/publish.yml`: - -1. `signinum-core` -2. `signinum-cuda-runtime` -3. `signinum-profile` -4. `signinum-j2k-native` -5. `signinum-jpeg` -6. `signinum-tilecodec` -7. `signinum-j2k` -8. `signinum-jpeg-metal` -9. `signinum-j2k-metal` -10. `signinum-jpeg-cuda` -11. `signinum-j2k-cuda` -12. `signinum-cli` -13. `signinum` - -Every package in this list must have a fresh manifest version before a -metadata-refresh release, because crates.io package metadata is immutable after -publication. `signinum-j2k-compare` remains `publish = false`; it is a local -parity oracle helper, not a released runtime dependency. +- formatting +- tests +- clippy +- release integrity +- package validation +- semver checks for stable packages +- docs and stable API inventory +- benchmark target compilation +- unsafe audit +- bounded fuzz run +- coverage via `cargo llvm-cov --fail-under-lines 80` + +Metal runtime validation runs on macOS where available. J2K Metal Criterion +bench signoff is reset until new narrow profiling benches are added. + +CUDA validation requires a self-hosted CUDA environment for runtime and NVIDIA performance evidence. CUDA paths use J2K-owned CUDA kernels, cuda-runtime integration, and CUDA device memory surfaces for supported shapes. NVIDIA performance claims require recorded self-hosted benchmark output. + +Coverage exclusions are limited to hardware-only GPU paths that cannot execute on hosted CI: `j2k-cuda-runtime`, CUDA adapter crates, Metal adapter crates, and `j2k-metal-support`. Those paths still require the Metal/CUDA validation gates before release. + +## Published and unpublished crates + +Published crates must declare package README files and docs.rs metadata. +Unpublished tooling and oracle helpers remain local even when versioned with the +workspace. + +`j2k-test-support` is an unpublished dev helper. Comparator crates and +automation-only tooling are not runtime API. diff --git a/docs/roadmap/cuda-oxide-kernel-path.md b/docs/roadmap/cuda-oxide-kernel-path.md new file mode 100644 index 00000000..c2a3aec3 --- /dev/null +++ b/docs/roadmap/cuda-oxide-kernel-path.md @@ -0,0 +1,69 @@ +# CUDA-Oxide Kernel Path + +## Goal + +Continue migrating practical J2K CUDA runtime kernels to Rust-authored +cuda-oxide equivalents. The existing CUDA C and checked-in PTX path remains the +default compatibility baseline; cuda-oxide routes are opt-in until build +stability, parity, and benchmark coverage are broad enough to promote them. + +## Current State + +- `crates/j2k-cuda-runtime/build.rs` invokes `nvcc` for `.cu` sources and + falls back to checked-in PTX for selected kernels. +- The `cuda-oxide-copy-u8` feature adds an opt-in Rust-authored `CopyU8` PTX + path without changing the default CUDA C/PTX runtime. +- The `cuda-oxide-j2k-encode`, `cuda-oxide-j2k-decode-store`, + `cuda-oxide-j2k-dequantize`, `cuda-oxide-j2k-idwt`, and + `cuda-oxide-transcode` features cover selected J2K CUDA paths where the GPU + work maps cleanly to cuda-oxide kernels. +- Unsupported host builds emit placeholder PTX by default so all-features + checks keep working. The documented require-build environment flag makes the + build fail loudly when cuda-oxide generation is required but unavailable. +- Runtime guards prevent placeholder PTX from being loaded accidentally. +- `crates/j2k-cuda-runtime/src/kernels.rs` exposes the shared kernel registry + used by the J2K, HTJ2K, JPEG, and transcode CUDA adapters. +- `crates/j2k-cuda`, `crates/j2k-jpeg-cuda`, and + `crates/j2k-transcode-cuda` already route through `j2k-cuda-runtime`. + +## Landed + +- Added feature-gated cuda-oxide build paths in `j2k-cuda-runtime`. +- Ported `CopyU8` as the initial low-risk Rust-authored kernel. +- Ported supported J2K encode-stage kernels: deinterleave, RCT/ICT, forward + DWT 5/3 and 9/7, quantization, HTJ2K encoded-byte compaction, and HTJ2K + packetization. +- Ported supported J2K decode-store, HTJ2K dequantize, generic IDWT, and + JPEG-to-J2K transcode kernels where the current CUDA path is data-parallel. +- Loaded generated PTX through the existing runtime module boundary. +- Added parity and metadata tests that stay host-safe when cuda-oxide is not + available. +- Documented the unsafe kernel source in `docs/unsafe-audit.md`. + +## Remaining Work + +1. Keep the remaining HTJ2K entropy encode/decode kernels on CUDA C unless a + measured cuda-oxide port shows a practical win. +2. Add benchmarks for the landed cuda-oxide routes on self-hosted CUDA + validation hardware before changing default routing. +3. Keep expanding parity tests against the existing CUDA C/PTX path and CPU + oracle before considering broader migration. +4. Treat JPEG decode kernels as a separate follow-up because the current goal + is J2K/HTJ2K coverage first. + +## Acceptance Criteria For The Next PR + +- The default build remains unchanged for users without cuda-oxide. +- The cuda-oxide path is opt-in and fails loudly when explicitly requested but + unavailable. +- The next kernel is generated from Rust, loaded by the existing runtime, and + covered by CPU/CUDA parity tests. +- CI or GPU validation documents build requirements, runtime behavior, and + performance evidence. +- The PR records whether cuda-oxide is ready for broader migration or should + remain limited to selected kernels. + +## Notes + +cuda-oxide is currently an experimental Rust-to-CUDA compiler. Treat this as a +controlled migration path, not an immediate removal of CUDA C. diff --git a/docs/roadmap/metal-encode-stage-coverage.md b/docs/roadmap/metal-encode-stage-coverage.md new file mode 100644 index 00000000..0986b084 --- /dev/null +++ b/docs/roadmap/metal-encode-stage-coverage.md @@ -0,0 +1,94 @@ +# Metal Encode-Stage Coverage + +## Goal + +Fill the missing Metal encode-stage acceleration gaps for JPEG 2000 and HTJ2K +encoding on macOS while keeping CPU fallback behavior explicit and testable. + +## Current State + +`crates/j2k-metal/src/encode/stage_accelerator.rs` currently reports Metal +dispatches for: + +- deinterleave for public 1-4 component, 1-16 bit host encode sample layouts +- forward RCT +- forward ICT +- forward 5/3 DWT +- forward 9/7 DWT +- subband quantization +- classic Tier-1 code-block encode +- HT code-block encode +- packetization + +The automatic host-output path is conservative and does not try every available +Metal stage. `MetalEncodeStageAccelerator::for_auto_host_output()` currently +allows benchmark-gated coefficient-prep stages only: + +- deinterleave, forward RCT, forward ICT, forward 5/3 DWT, forward 9/7 DWT, + and subband quantization may dispatch when the individual stage input has at + least `512 * 512` samples or coefficients. +- Classic Tier-1, HT code-block, and packetization stay disabled for + stage-by-stage Auto host-output routing. +- The resident HTJ2K RGB8 lossless shortcut remains separately gated to + 1,024 x 1,024 and larger tiles, with CPU packetization. +- Shapes below these gates fall back to CPU explicitly; no silent strict-device + fallback is used. + +## Benchmark Evidence + +Command: + +```bash +cargo test -p j2k-metal --test encode_auto_routing_benchmark -- --ignored --nocapture +``` + +Host: macOS 26.5 (25F71), Apple M4 Pro CPU, Apple M4 Pro 16-core GPU, +Metal 4. Timings are median wall-clock milliseconds from the ignored benchmark +harness and are intended as routing evidence, not a cross-machine performance +claim. + +Stage microbenchmarks: + +| Stage | 128 CPU / Metal | 512 CPU / Metal | 1024 CPU / Metal | Auto gate | +| --- | ---: | ---: | ---: | --- | +| deinterleave | 1.160 / 0.232 | 8.170 / 0.512 | 32.398 / 1.776 | >= 512 x 512 | +| forward RCT | 0.192 / 0.217 | 2.973 / 0.313 | 11.986 / 0.508 | >= 512 x 512 | +| forward ICT | 0.145 / 0.184 | 2.407 / 0.244 | 9.806 / 0.607 | >= 512 x 512 | +| forward 5/3 DWT | 0.946 / 0.199 | 15.392 / 0.434 | 62.272 / 0.873 | >= 512 x 512 | +| forward 9/7 DWT | 1.353 / 0.263 | 22.041 / 0.629 | 87.317 / 1.403 | >= 512 x 512 | +| subband quantization | 0.124 / 0.181 | 2.082 / 0.546 | 8.217 / 1.092 | >= 512 x 512 coefficients | + +Selected Auto-route results: + +| Route | 128 dispatch | 512 CPU / Auto | 512 dispatch | 1024 CPU / Auto | 1024 dispatch | +| --- | --- | ---: | --- | ---: | --- | +| lossless classic Gray8 | none | 42.776 / 21.572 | deinterleave=1, dwt53=1 | 169.933 / 61.478 | deinterleave=1, dwt53=1, quantize=4 | +| lossless classic RGB8 | none | 115.098 / 65.294 | deinterleave=1, rct=1, dwt53=3 | 429.063 / 187.509 | deinterleave=1, rct=1, dwt53=3, quantize=12 | +| lossless HTJ2K RGB8 | none below resident gate | 73.426 / 71.201 | none | 284.845 / 10.012 | rct=1, dwt53=3, ht=1 | +| lossy HTJ2K RGB8 | none | 186.633 / 39.586 | deinterleave=2, ict=2, dwt97=6 | 752.962 / 99.089 | deinterleave=2, ict=2, dwt97=6, quantize=24 | + +## Remaining Gap + +Subband quantization has landed for the existing contiguous encode-stage job +shape, with dispatch accounting, CPU parity tests, and explicit Metal errors for +unsupported quantization parameters. Auto routing now uses benchmark-gated +coefficient-prep stages and the existing resident HTJ2K RGB8 shortcut, but this +is not full end-to-end Metal encode coverage for every public encode route. +Tier-1, HT code-block, packetization, and codestream assembly remain CPU for +stage-by-stage Auto host-output routes unless a resident encode path explicitly +supports the shape. + +That boundary is intentional. Do not widen Auto routing simply to increase the +number of Metal stages. New automatic dispatch should land only when the stage +is parity-covered, benchmark-backed, and does not lose the benefit through +extra host/device transfer or dispatch overhead. + +## Acceptance Criteria + +- Explicit Metal encode requests either dispatch supported stages or return a + structured unsupported request error. +- Auto mode remains conservative unless benchmarks justify widening dispatch. +- CPU parity tests cover each new Metal stage. +- GPU validation records stage-level dispatch counts and performance artifacts. +- Public docs describe the supported Metal encode-stage surface without + implying full end-to-end Metal coverage for unsupported shapes. diff --git a/docs/roadmap/metal-jpeg-path-coverage.md b/docs/roadmap/metal-jpeg-path-coverage.md new file mode 100644 index 00000000..70455aa6 --- /dev/null +++ b/docs/roadmap/metal-jpeg-path-coverage.md @@ -0,0 +1,54 @@ +# Metal JPEG Path Coverage + +## Goal + +Expand Metal acceleration coverage in `j2k-jpeg-metal` for supported JPEG +decode and baseline encode workloads while keeping strict errors for unsupported +formats and packet shapes. + +## Current State + +- Explicit Metal JPEG decode requires fast baseline 4:2:0, 4:2:2, or 4:4:4 + packets. +- Explicit Metal JPEG decode currently supports `Gray8`, `Rgb8`, and `Rgba8` + output formats. +- Routing and benchmark evidence is documented in + `crates/j2k-jpeg-metal/docs/routing-benchmarks.md`. +- Single-request `BackendRequest::Auto` remains conservative unless benchmark + evidence shows Metal should be selected for a supported workload. +- Metal routing is not expanded for coverage alone; CPU stays preferred for + small, irregular, or entropy-bound work unless benchmarks show a device path + wins after transfer overhead. +- Viewport paths use hybrid strategies for selected contiguous or resident + workloads, but unsupported shapes fall back or return structured errors. +- Metal JPEG encode is baseline-only and accepts `Gray8` or `Rgb8` input + buffers. + +## Landed + +- Added benchmark harness coverage for JPEG Metal routing evidence. +- Documented where Metal should and should not be selected automatically. +- Preserved strict Metal errors for unsupported packet shapes. + +## Remaining Work + +1. Add a support matrix test suite for baseline sampling modes, restart + intervals, output formats, and viewport shapes. +2. Widen explicit Metal decode only where fast packet extraction and parity are + proven. +3. Add benchmark-backed Auto routing for full-frame or viewport workloads where + Metal consistently wins. +4. Extend baseline Metal encode coverage only after parity and output-size + bounds are stable. +5. Document unsupported JPEG features explicitly rather than silently falling + back for strict Metal requests. + +## Acceptance Criteria For The Next PR + +- Strict Metal requests reject unsupported JPEG shapes before launching kernels. +- Any Auto routing changes are backed by benchmark evidence and do not + initialize Metal for workloads that remain CPU-preferred. +- Decode and encode tests compare Metal output against the CPU JPEG oracle. +- Viewport tests cover contiguous, sparse, scaled, and resident-output paths. +- Public docs distinguish JPEG Metal acceleration from full JPEG feature + coverage. diff --git a/docs/roadmap/metal-transcode-resident-pipeline.md b/docs/roadmap/metal-transcode-resident-pipeline.md new file mode 100644 index 00000000..cc49da34 --- /dev/null +++ b/docs/roadmap/metal-transcode-resident-pipeline.md @@ -0,0 +1,52 @@ +# Metal Transcode Resident Pipeline + +## Goal + +Move JPEG-to-J2K/HTJ2K transcode toward a more resident Metal pipeline on +macOS, starting from the existing coefficient-domain transform accelerator and +adding adjacent stages only where correctness and benchmarks justify it. +JPEG-to-J2K/HTJ2K transcoding itself is implemented; this document tracks +optional Metal residency and performance work, not feature completion. + +## Current State + +- `j2k-transcode-metal` accelerates coefficient-domain DCT-grid to one-level + 5/3 and 9/7 wavelet projections used by HTJ2K paths. +- The Metal accelerator includes explicit and Auto modes, with Auto declining + small or unsupported jobs so the caller can use the scalar fallback. +- `j2k-transcode` exposes a pipeline residency map that reports CPU fallback + points and host/device transfer boundaries from timing reports. +- CPU scalar code remains the oracle and fallback. +- This is not yet a complete Metal JPEG entropy decode to final J2K/HTJ2K + codestream pipeline. + +## Landed + +- Added a transcode pipeline map that identifies stage residency and fallback + points from current timing data. +- Added tests and bench harness coverage for the pipeline map. +- Kept public wording at Metal-accelerated transcode stages rather than + claiming a complete resident pipeline. + +## Optional Metal Residency Work + +1. Add resident Metal handoff types for transform outputs that can flow into + downstream HTJ2K encode or packetization stages. +2. Extend Metal coverage for batches that currently decline in Auto mode only + because thresholds or shape gates are conservative. +3. Evaluate whether JPEG entropy decode, coefficient preparation, packetization, + or codestream assembly should be moved next, based on measured CPU time and + transfer overhead. Leave those stages on CPU when the workload is irregular, + small, or transfer-bound. +4. Add benchmark artifacts for JPEG-to-J2K and JPEG-to-HTJ2K workloads that are + not limited to WSI examples. + +## Acceptance Criteria For The Next PR + +- The pipeline map identifies every CPU fallback and host/device transfer. +- New resident stages preserve CPU oracle parity for 5/3 and 9/7 paths. +- Explicit Metal requests fail clearly when a pipeline segment is unsupported. +- Auto mode dispatch changes are benchmark-backed and keep scalar fallback + behavior explicit. +- Public docs describe this as Metal-accelerated transcode stages unless a + complete resident pipeline is actually implemented. diff --git a/docs/stable-api-1.0.md b/docs/stable-api-1.0.md new file mode 100644 index 00000000..8d53a691 --- /dev/null +++ b/docs/stable-api-1.0.md @@ -0,0 +1,30 @@ +# Stable API Policy + +The stable API inventory is generated. The human-maintained policy is small: +stable crates must preserve the public codec contracts, while experimental +adapters may evolve until promoted. + +## Generated snapshot + +The generated item-level companion is: + +- `docs/stable-api-1.0.public-api.txt` + +Regenerate or check it with: + +```bash +cargo xtask stable-api +cargo xtask stable-api --write +``` + +The snapshot records public items and exit-code contract expectations for the +stable public line. Manual prose in this file must not duplicate that inventory. + +## Stability tiers + +- Stable: `j2k`, `j2k-core`, `j2k-jpeg`, + `j2k-native`, `j2k-profile`, and `j2k-tilecodec`. +- Experimental: CUDA adapters, Metal adapters, and transcode crates. +- Unpublished tooling: test support, comparators, and xtask automation helpers. + +Breaking changes to stable crates require explicit semver review. diff --git a/docs/stable-api-1.0.public-api.txt b/docs/stable-api-1.0.public-api.txt new file mode 100644 index 00000000..2648250d --- /dev/null +++ b/docs/stable-api-1.0.public-api.txt @@ -0,0 +1,6688 @@ +# J2K 1.0 Public API Snapshot + +This file is generated by `cargo xtask stable-api --write` from `cargo public-api -p --all-features -sss --color never`. +It is generated on macOS so target-gated Metal APIs are included; non-macOS builds expose a smaller cfg-gated subset. + +Generator: `cargo-public-api 0.52.0`. + +It is the item-level companion to `docs/stable-api-1.0.md`: every public module, type, trait, function, method, constant, variant, and field reported here is semver-visible unless moved private before 1.0. + +## `j2k` + +```text +pub mod j2k +pub use j2k::BackendKind +pub use j2k::BackendRequest +pub use j2k::BufferError +pub use j2k::CodecError +pub use j2k::CompressedPayloadKind +pub use j2k::CompressedTransferSyntax +pub use j2k::DecodeOutcome +pub use j2k::DecodeRowsError +pub use j2k::DecoderContext +pub use j2k::Downscale +pub use j2k::ImageCodec +pub use j2k::ImageDecode +pub use j2k::ImageDecodeRows +pub use j2k::PassthroughCandidate +pub use j2k::PassthroughDecision +pub use j2k::PassthroughRejectReason +pub use j2k::PassthroughRequirements +pub use j2k::PixelFormat +pub use j2k::Rect +pub use j2k::RowSink +pub use j2k::TileBatchDecode +pub use j2k::TileBatchOptions +pub mod j2k::adapter +pub mod j2k::adapter::adaptive_route +pub enum j2k::adapter::adaptive_route::J2kAdaptiveBackendRequest +pub j2k::adapter::adaptive_route::J2kAdaptiveBackendRequest::Accelerated +pub j2k::adapter::adaptive_route::J2kAdaptiveBackendRequest::CpuOnly +pub j2k::adapter::adaptive_route::J2kAdaptiveBackendRequest::StrictDevice(j2k_core::backend::BackendKind) +impl j2k::adapter::adaptive_route::J2kAdaptiveBackendRequest +pub const fn j2k::adapter::adaptive_route::J2kAdaptiveBackendRequest::from_backend_request(j2k_core::backend::BackendRequest) -> Self +pub enum j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkScope +pub j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkScope::EndToEnd +pub j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkScope::Stage(j2k::adapter::adaptive_route::J2kAdaptiveStage) +pub enum j2k::adapter::adaptive_route::J2kAdaptiveCodecMode +pub j2k::adapter::adaptive_route::J2kAdaptiveCodecMode::ClassicJ2k +pub j2k::adapter::adaptive_route::J2kAdaptiveCodecMode::Htj2k +pub enum j2k::adapter::adaptive_route::J2kAdaptiveOperation +pub j2k::adapter::adaptive_route::J2kAdaptiveOperation::Decode +pub j2k::adapter::adaptive_route::J2kAdaptiveOperation::Encode +pub j2k::adapter::adaptive_route::J2kAdaptiveOperation::Transcode +pub enum j2k::adapter::adaptive_route::J2kAdaptiveOutputResidency +pub j2k::adapter::adaptive_route::J2kAdaptiveOutputResidency::Device +pub j2k::adapter::adaptive_route::J2kAdaptiveOutputResidency::Host +pub enum j2k::adapter::adaptive_route::J2kAdaptiveQualityMode +pub j2k::adapter::adaptive_route::J2kAdaptiveQualityMode::Lossless +pub j2k::adapter::adaptive_route::J2kAdaptiveQualityMode::Lossy +pub enum j2k::adapter::adaptive_route::J2kAdaptiveRcaReason +pub j2k::adapter::adaptive_route::J2kAdaptiveRcaReason::AlgorithmicMismatch +pub j2k::adapter::adaptive_route::J2kAdaptiveRcaReason::BenchmarkMismatch +pub j2k::adapter::adaptive_route::J2kAdaptiveRcaReason::CpuGenuinelyBetter +pub j2k::adapter::adaptive_route::J2kAdaptiveRcaReason::MissingBatching +pub j2k::adapter::adaptive_route::J2kAdaptiveRcaReason::MissingResidency +pub j2k::adapter::adaptive_route::J2kAdaptiveRcaReason::TooSmallWorkload +pub j2k::adapter::adaptive_route::J2kAdaptiveRcaReason::TransferSyncOverhead +pub enum j2k::adapter::adaptive_route::J2kAdaptiveRouteKind +pub j2k::adapter::adaptive_route::J2kAdaptiveRouteKind::CpuOnly +pub j2k::adapter::adaptive_route::J2kAdaptiveRouteKind::Hybrid +pub j2k::adapter::adaptive_route::J2kAdaptiveRouteKind::StrictDevice +pub enum j2k::adapter::adaptive_route::J2kAdaptiveStage +pub j2k::adapter::adaptive_route::J2kAdaptiveStage::CodestreamAssembly +pub j2k::adapter::adaptive_route::J2kAdaptiveStage::CopySync +pub j2k::adapter::adaptive_route::J2kAdaptiveStage::Dwt +pub j2k::adapter::adaptive_route::J2kAdaptiveStage::HtBlockCoding +pub j2k::adapter::adaptive_route::J2kAdaptiveStage::Idwt +pub j2k::adapter::adaptive_route::J2kAdaptiveStage::MarkerParsing +pub j2k::adapter::adaptive_route::J2kAdaptiveStage::Mct +pub j2k::adapter::adaptive_route::J2kAdaptiveStage::Packetization +pub j2k::adapter::adaptive_route::J2kAdaptiveStage::PcrdRateControl +pub j2k::adapter::adaptive_route::J2kAdaptiveStage::Quantization +pub j2k::adapter::adaptive_route::J2kAdaptiveStage::Tier1 +pub j2k::adapter::adaptive_route::J2kAdaptiveStage::Validation +impl j2k::adapter::adaptive_route::J2kAdaptiveStage +pub const j2k::adapter::adaptive_route::J2kAdaptiveStage::ALL: [Self; 12] +pub const fn j2k::adapter::adaptive_route::J2kAdaptiveStage::profile_label(self) -> &'static str +pub enum j2k::adapter::adaptive_route::J2kAdaptiveStageGateStatus +pub j2k::adapter::adaptive_route::J2kAdaptiveStageGateStatus::Approved +pub j2k::adapter::adaptive_route::J2kAdaptiveStageGateStatus::BenchmarkGateMissing +pub j2k::adapter::adaptive_route::J2kAdaptiveStageGateStatus::BlockedNeedsRca +pub j2k::adapter::adaptive_route::J2kAdaptiveStageGateStatus::CpuShaped +pub j2k::adapter::adaptive_route::J2kAdaptiveStageGateStatus::EndToEndGateBlocked +pub j2k::adapter::adaptive_route::J2kAdaptiveStageGateStatus::ReclassifiedCpu +pub j2k::adapter::adaptive_route::J2kAdaptiveStageGateStatus::StrictDeviceProof +pub j2k::adapter::adaptive_route::J2kAdaptiveStageGateStatus::VariableCpuDefault +pub enum j2k::adapter::adaptive_route::J2kAdaptiveStageOwner +pub j2k::adapter::adaptive_route::J2kAdaptiveStageOwner::Cpu +pub j2k::adapter::adaptive_route::J2kAdaptiveStageOwner::Gpu +pub j2k::adapter::adaptive_route::J2kAdaptiveStageOwner::Variable +pub struct j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkEvidence +pub j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkEvidence::accelerated_ns: u64 +pub j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkEvidence::backend: j2k_core::backend::BackendKind +pub j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkEvidence::cpu_ns: u64 +pub j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkEvidence::criterion_noise_percent: f64 +pub j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkEvidence::scope: j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkScope +impl j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkEvidence +pub const fn j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkEvidence::end_to_end(j2k_core::backend::BackendKind, u64, u64, f64) -> Self +pub fn j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkEvidence::improvement_percent(self) -> f64 +pub const fn j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkEvidence::stage(j2k::adapter::adaptive_route::J2kAdaptiveStage, j2k_core::backend::BackendKind, u64, u64, f64) -> Self +pub struct j2k::adapter::adaptive_route::J2kAdaptiveBenchmarks +impl j2k::adapter::adaptive_route::J2kAdaptiveBenchmarks +pub fn j2k::adapter::adaptive_route::J2kAdaptiveBenchmarks::push_end_to_end(&mut self, j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkEvidence) +pub fn j2k::adapter::adaptive_route::J2kAdaptiveBenchmarks::push_stage(&mut self, j2k::adapter::adaptive_route::J2kAdaptiveBenchmarkEvidence) +pub struct j2k::adapter::adaptive_route::J2kAdaptiveGatePolicy +pub j2k::adapter::adaptive_route::J2kAdaptiveGatePolicy::min_speedup_percent: f64 +impl core::default::Default for j2k::adapter::adaptive_route::J2kAdaptiveGatePolicy +pub fn j2k::adapter::adaptive_route::J2kAdaptiveGatePolicy::default() -> Self +pub struct j2k::adapter::adaptive_route::J2kAdaptiveRcaFinding +pub j2k::adapter::adaptive_route::J2kAdaptiveRcaFinding::backend: j2k_core::backend::BackendKind +pub j2k::adapter::adaptive_route::J2kAdaptiveRcaFinding::reason: j2k::adapter::adaptive_route::J2kAdaptiveRcaReason +pub j2k::adapter::adaptive_route::J2kAdaptiveRcaFinding::reclassify_cpu: bool +pub j2k::adapter::adaptive_route::J2kAdaptiveRcaFinding::stage: j2k::adapter::adaptive_route::J2kAdaptiveStage +impl j2k::adapter::adaptive_route::J2kAdaptiveRcaFinding +pub const fn j2k::adapter::adaptive_route::J2kAdaptiveRcaFinding::reclassify_cpu(j2k::adapter::adaptive_route::J2kAdaptiveStage, j2k_core::backend::BackendKind, j2k::adapter::adaptive_route::J2kAdaptiveRcaReason) -> Self +pub struct j2k::adapter::adaptive_route::J2kAdaptiveRoutePlanner +impl j2k::adapter::adaptive_route::J2kAdaptiveRoutePlanner +pub fn j2k::adapter::adaptive_route::J2kAdaptiveRoutePlanner::compile_time_defaults() -> Self +pub fn j2k::adapter::adaptive_route::J2kAdaptiveRoutePlanner::lossless_encode(j2k_core::backend::BackendCapabilities) -> Self +pub fn j2k::adapter::adaptive_route::J2kAdaptiveRoutePlanner::lossless_encode_benchmarks(j2k::adapter::adaptive_route::J2kAdaptiveWorkload) -> j2k::adapter::adaptive_route::J2kAdaptiveBenchmarks +pub fn j2k::adapter::adaptive_route::J2kAdaptiveRoutePlanner::lossless_encode_workload(j2k::J2kLosslessSamples<'_>, j2k::J2kLosslessEncodeOptions) -> j2k::adapter::adaptive_route::J2kAdaptiveWorkload +pub fn j2k::adapter::adaptive_route::J2kAdaptiveRoutePlanner::new(j2k_core::backend::BackendCapabilities) -> Self +pub fn j2k::adapter::adaptive_route::J2kAdaptiveRoutePlanner::plan(&self, j2k::adapter::adaptive_route::J2kAdaptiveWorkload, j2k::adapter::adaptive_route::J2kAdaptiveBackendRequest, &j2k::adapter::adaptive_route::J2kAdaptiveBenchmarks) -> core::result::Result +pub fn j2k::adapter::adaptive_route::J2kAdaptiveRoutePlanner::plan_lossless_encode(&self, j2k::J2kLosslessSamples<'_>, j2k::J2kLosslessEncodeOptions) -> core::result::Result +pub fn j2k::adapter::adaptive_route::J2kAdaptiveRoutePlanner::with_lossless_encode_policy(self) -> Self +pub const fn j2k::adapter::adaptive_route::J2kAdaptiveRoutePlanner::with_policy(self, j2k::adapter::adaptive_route::J2kAdaptiveGatePolicy) -> Self +pub fn j2k::adapter::adaptive_route::J2kAdaptiveRoutePlanner::with_rca_finding(self, j2k::adapter::adaptive_route::J2kAdaptiveRcaFinding) -> Self +pub struct j2k::adapter::adaptive_route::J2kAdaptiveRouteReport +pub j2k::adapter::adaptive_route::J2kAdaptiveRouteReport::request: j2k::adapter::adaptive_route::J2kAdaptiveBackendRequest +pub j2k::adapter::adaptive_route::J2kAdaptiveRouteReport::route_kind: j2k::adapter::adaptive_route::J2kAdaptiveRouteKind +pub j2k::adapter::adaptive_route::J2kAdaptiveRouteReport::selected_device: core::option::Option +pub j2k::adapter::adaptive_route::J2kAdaptiveRouteReport::stages: alloc::vec::Vec +impl j2k::adapter::adaptive_route::J2kAdaptiveRouteReport +pub fn j2k::adapter::adaptive_route::J2kAdaptiveRouteReport::has_unresolved_rca(&self) -> bool +pub fn j2k::adapter::adaptive_route::J2kAdaptiveRouteReport::stage(&self, j2k::adapter::adaptive_route::J2kAdaptiveStage) -> core::option::Option<&j2k::adapter::adaptive_route::J2kAdaptiveStageDecision> +pub struct j2k::adapter::adaptive_route::J2kAdaptiveStageDecision +pub j2k::adapter::adaptive_route::J2kAdaptiveStageDecision::gate_status: j2k::adapter::adaptive_route::J2kAdaptiveStageGateStatus +pub j2k::adapter::adaptive_route::J2kAdaptiveStageDecision::improvement_percent: core::option::Option +pub j2k::adapter::adaptive_route::J2kAdaptiveStageDecision::logical_owner: j2k::adapter::adaptive_route::J2kAdaptiveStageOwner +pub j2k::adapter::adaptive_route::J2kAdaptiveStageDecision::rca_reason: core::option::Option +pub j2k::adapter::adaptive_route::J2kAdaptiveStageDecision::selected_backend: j2k_core::backend::BackendKind +pub j2k::adapter::adaptive_route::J2kAdaptiveStageDecision::stage: j2k::adapter::adaptive_route::J2kAdaptiveStage +impl j2k::adapter::adaptive_route::J2kAdaptiveStageDecision +pub fn j2k::adapter::adaptive_route::J2kAdaptiveStageDecision::requires_rca(&self) -> bool +pub struct j2k::adapter::adaptive_route::J2kAdaptiveWorkload +pub j2k::adapter::adaptive_route::J2kAdaptiveWorkload::batch_size: u16 +pub j2k::adapter::adaptive_route::J2kAdaptiveWorkload::bit_depth: u8 +pub j2k::adapter::adaptive_route::J2kAdaptiveWorkload::codec_mode: j2k::adapter::adaptive_route::J2kAdaptiveCodecMode +pub j2k::adapter::adaptive_route::J2kAdaptiveWorkload::components: u8 +pub j2k::adapter::adaptive_route::J2kAdaptiveWorkload::operation: j2k::adapter::adaptive_route::J2kAdaptiveOperation +pub j2k::adapter::adaptive_route::J2kAdaptiveWorkload::output_residency: j2k::adapter::adaptive_route::J2kAdaptiveOutputResidency +pub j2k::adapter::adaptive_route::J2kAdaptiveWorkload::quality_layers: u16 +pub j2k::adapter::adaptive_route::J2kAdaptiveWorkload::quality_mode: j2k::adapter::adaptive_route::J2kAdaptiveQualityMode +pub j2k::adapter::adaptive_route::J2kAdaptiveWorkload::roi: bool +pub j2k::adapter::adaptive_route::J2kAdaptiveWorkload::scaled: bool +pub j2k::adapter::adaptive_route::J2kAdaptiveWorkload::tile_size: (u32, u32) +impl j2k::adapter::adaptive_route::J2kAdaptiveWorkload +pub fn j2k::adapter::adaptive_route::J2kAdaptiveWorkload::logical_owner_for(self, j2k::adapter::adaptive_route::J2kAdaptiveStage) -> j2k::adapter::adaptive_route::J2kAdaptiveStageOwner +pub const fn j2k::adapter::adaptive_route::J2kAdaptiveWorkload::new(j2k::adapter::adaptive_route::J2kAdaptiveOperation, j2k::adapter::adaptive_route::J2kAdaptiveCodecMode, j2k::adapter::adaptive_route::J2kAdaptiveQualityMode, u8, u8, (u32, u32), u16) -> Self +pub const fn j2k::adapter::adaptive_route::J2kAdaptiveWorkload::with_output_residency(self, j2k::adapter::adaptive_route::J2kAdaptiveOutputResidency) -> Self +pub const fn j2k::adapter::adaptive_route::J2kAdaptiveWorkload::with_quality_layers(self, u16) -> Self +pub const fn j2k::adapter::adaptive_route::J2kAdaptiveWorkload::with_roi(self, bool) -> Self +pub const fn j2k::adapter::adaptive_route::J2kAdaptiveWorkload::with_scaled(self, bool) -> Self +pub mod j2k::adapter::device_plan +pub enum j2k::adapter::device_plan::DeviceDecodeRequest +pub j2k::adapter::device_plan::DeviceDecodeRequest::Full +pub j2k::adapter::device_plan::DeviceDecodeRequest::Region +pub j2k::adapter::device_plan::DeviceDecodeRequest::Region::roi: j2k_core::types::Rect +pub j2k::adapter::device_plan::DeviceDecodeRequest::RegionScaled +pub j2k::adapter::device_plan::DeviceDecodeRequest::RegionScaled::roi: j2k_core::types::Rect +pub j2k::adapter::device_plan::DeviceDecodeRequest::RegionScaled::scale: j2k_core::scale::Downscale +pub j2k::adapter::device_plan::DeviceDecodeRequest::Scaled +pub j2k::adapter::device_plan::DeviceDecodeRequest::Scaled::scale: j2k_core::scale::Downscale +pub struct j2k::adapter::device_plan::DeviceDecodePlan +impl j2k::adapter::device_plan::DeviceDecodePlan +pub fn j2k::adapter::device_plan::DeviceDecodePlan::for_image((u32, u32), j2k::adapter::device_plan::DeviceDecodeRequest) -> core::result::Result +pub fn j2k::adapter::device_plan::DeviceDecodePlan::is_full_frame(self) -> bool +pub fn j2k::adapter::device_plan::DeviceDecodePlan::output_dims(self) -> (u32, u32) +pub fn j2k::adapter::device_plan::DeviceDecodePlan::output_rect(self) -> j2k_core::types::Rect +pub fn j2k::adapter::device_plan::DeviceDecodePlan::scale(self) -> j2k_core::scale::Downscale +pub fn j2k::adapter::device_plan::DeviceDecodePlan::source_dims(self) -> (u32, u32) +pub fn j2k::adapter::device_plan::DeviceDecodePlan::source_rect(self) -> j2k_core::types::Rect +pub fn j2k::adapter::device_plan::DeviceDecodePlan::target_resolution(self) -> core::option::Option<(u32, u32)> +pub mod j2k::adapter::encode_stage +pub use j2k::adapter::encode_stage::CpuOnlyJ2kEncodeStageAccelerator +pub use j2k::adapter::encode_stage::EncodedHtJ2kCodeBlock +pub use j2k::adapter::encode_stage::EncodedJ2kCodeBlock +pub use j2k::adapter::encode_stage::IrreversibleQuantizationStep +pub use j2k::adapter::encode_stage::IrreversibleQuantizationSubbandScales +pub use j2k::adapter::encode_stage::J2kCodeBlockSegment +pub use j2k::adapter::encode_stage::J2kCodeBlockStyle +pub use j2k::adapter::encode_stage::J2kDeinterleaveToF32Job +pub use j2k::adapter::encode_stage::J2kEncodeDispatchReport +pub use j2k::adapter::encode_stage::J2kForwardDwt53Job +pub use j2k::adapter::encode_stage::J2kForwardDwt53Level +pub use j2k::adapter::encode_stage::J2kForwardDwt53Output +pub use j2k::adapter::encode_stage::J2kForwardDwt97Job +pub use j2k::adapter::encode_stage::J2kForwardDwt97Level +pub use j2k::adapter::encode_stage::J2kForwardDwt97Output +pub use j2k::adapter::encode_stage::J2kForwardIctJob +pub use j2k::adapter::encode_stage::J2kForwardRctJob +pub use j2k::adapter::encode_stage::J2kHtCodeBlockEncodeJob +pub use j2k::adapter::encode_stage::J2kHtSubbandEncodeJob +pub use j2k::adapter::encode_stage::J2kHtj2kTileEncodeJob +pub use j2k::adapter::encode_stage::J2kPacketizationBlockCodingMode +pub use j2k::adapter::encode_stage::J2kPacketizationCodeBlock +pub use j2k::adapter::encode_stage::J2kPacketizationEncodeJob +pub use j2k::adapter::encode_stage::J2kPacketizationPacketDescriptor +pub use j2k::adapter::encode_stage::J2kPacketizationProgressionOrder +pub use j2k::adapter::encode_stage::J2kPacketizationResolution +pub use j2k::adapter::encode_stage::J2kPacketizationSubband +pub use j2k::adapter::encode_stage::J2kQuantizeSubbandJob +pub use j2k::adapter::encode_stage::J2kSubBandType +pub use j2k::adapter::encode_stage::J2kTier1CodeBlockEncodeJob +pub use j2k::adapter::encode_stage::PrecomputedHtj2k53Component +pub use j2k::adapter::encode_stage::PrecomputedHtj2k53Image +pub use j2k::adapter::encode_stage::PrecomputedHtj2k97Component +pub use j2k::adapter::encode_stage::PrecomputedHtj2k97Image +pub use j2k::adapter::encode_stage::PreencodedHtj2k97CodeBlock +pub use j2k::adapter::encode_stage::PreencodedHtj2k97CompactCodeBlock +pub use j2k::adapter::encode_stage::PreencodedHtj2k97CompactComponent +pub use j2k::adapter::encode_stage::PreencodedHtj2k97CompactImage +pub use j2k::adapter::encode_stage::PreencodedHtj2k97CompactResolution +pub use j2k::adapter::encode_stage::PreencodedHtj2k97CompactSubband +pub use j2k::adapter::encode_stage::PreencodedHtj2k97Component +pub use j2k::adapter::encode_stage::PreencodedHtj2k97Image +pub use j2k::adapter::encode_stage::PreencodedHtj2k97Resolution +pub use j2k::adapter::encode_stage::PreencodedHtj2k97Subband +pub use j2k::adapter::encode_stage::PrequantizedHtj2k97CodeBlock +pub use j2k::adapter::encode_stage::PrequantizedHtj2k97Component +pub use j2k::adapter::encode_stage::PrequantizedHtj2k97Image +pub use j2k::adapter::encode_stage::PrequantizedHtj2k97Resolution +pub use j2k::adapter::encode_stage::PrequantizedHtj2k97Subband +pub trait j2k::adapter::encode_stage::J2kEncodeStageAccelerator +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_deinterleave(&mut self, j2k_types::J2kDeinterleaveToF32Job<'_>) -> core::result::Result>>, &'static str> +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_forward_dwt53(&mut self, j2k_types::J2kForwardDwt53Job<'_>) -> core::result::Result, &'static str> +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_forward_dwt97(&mut self, j2k_types::J2kForwardDwt97Job<'_>) -> core::result::Result, &'static str> +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_forward_ict(&mut self, j2k_types::J2kForwardIctJob<'_>) -> core::result::Result +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_forward_rct(&mut self, j2k_types::J2kForwardRctJob<'_>) -> core::result::Result +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_ht_code_block(&mut self, j2k_types::J2kHtCodeBlockEncodeJob<'_>) -> core::result::Result, &'static str> +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_ht_code_blocks(&mut self, &[j2k_types::J2kHtCodeBlockEncodeJob<'_>]) -> core::result::Result>, &'static str> +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_ht_subband(&mut self, j2k_types::J2kHtSubbandEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_htj2k_tile(&mut self, j2k_types::J2kHtj2kTileEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_packetization(&mut self, j2k_types::J2kPacketizationEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_quantize_subband(&mut self, j2k_types::J2kQuantizeSubbandJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_tier1_code_block(&mut self, j2k_types::J2kTier1CodeBlockEncodeJob<'_>) -> core::result::Result, &'static str> +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::encode_tier1_code_blocks(&mut self, &[j2k_types::J2kTier1CodeBlockEncodeJob<'_>]) -> core::result::Result>, &'static str> +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::prefer_parallel_cpu_code_block_fallback(&self) -> bool +pub fn j2k::adapter::encode_stage::J2kEncodeStageAccelerator::prefer_parallel_cpu_tile_encode(&self) -> bool +impl j2k::adapter::encode_stage::J2kEncodeStageAccelerator for j2k_types::CpuOnlyJ2kEncodeStageAccelerator +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_deinterleave(&mut self, j2k_types::J2kDeinterleaveToF32Job<'_>) -> core::result::Result>>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_forward_dwt53(&mut self, j2k_types::J2kForwardDwt53Job<'_>) -> core::result::Result, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_forward_dwt97(&mut self, j2k_types::J2kForwardDwt97Job<'_>) -> core::result::Result, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_forward_ict(&mut self, j2k_types::J2kForwardIctJob<'_>) -> core::result::Result +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_forward_rct(&mut self, j2k_types::J2kForwardRctJob<'_>) -> core::result::Result +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_ht_code_block(&mut self, j2k_types::J2kHtCodeBlockEncodeJob<'_>) -> core::result::Result, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_ht_code_blocks(&mut self, &[j2k_types::J2kHtCodeBlockEncodeJob<'_>]) -> core::result::Result>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_ht_subband(&mut self, j2k_types::J2kHtSubbandEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_htj2k_tile(&mut self, j2k_types::J2kHtj2kTileEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_packetization(&mut self, j2k_types::J2kPacketizationEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_quantize_subband(&mut self, j2k_types::J2kQuantizeSubbandJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_tier1_code_block(&mut self, j2k_types::J2kTier1CodeBlockEncodeJob<'_>) -> core::result::Result, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_tier1_code_blocks(&mut self, &[j2k_types::J2kTier1CodeBlockEncodeJob<'_>]) -> core::result::Result>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::prefer_parallel_cpu_code_block_fallback(&self) -> bool +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::prefer_parallel_cpu_tile_encode(&self) -> bool +pub mod j2k::context +pub struct j2k::context::J2kContext +impl j2k::context::J2kContext +pub fn j2k::context::J2kContext::cpu_decode_parallelism(&self) -> j2k::CpuDecodeParallelism +pub const fn j2k::context::J2kContext::new() -> Self +pub fn j2k::context::J2kContext::set_cpu_decode_parallelism(&mut self, j2k::CpuDecodeParallelism) +impl j2k_core::context::CodecContext for j2k::context::J2kContext +pub fn j2k::context::J2kContext::cache_stats(&self) -> j2k_core::context::CacheStats +pub fn j2k::context::J2kContext::clear(&mut self) +pub mod j2k::error +#[non_exhaustive] pub enum j2k::error::J2kError +pub j2k::error::J2kError::Backend(alloc::string::String) +pub j2k::error::J2kError::Buffer(j2k_core::error::BufferError) +pub j2k::error::J2kError::DimensionOverflow +pub j2k::error::J2kError::DimensionOverflow::height: u32 +pub j2k::error::J2kError::DimensionOverflow::width: u32 +pub j2k::error::J2kError::Input(j2k_core::error::InputError) +pub j2k::error::J2kError::InvalidBox +pub j2k::error::J2kError::InvalidBox::offset: usize +pub j2k::error::J2kError::InvalidBox::what: &'static str +pub j2k::error::J2kError::InvalidCod +pub j2k::error::J2kError::InvalidCod::what: &'static str +pub j2k::error::J2kError::InvalidMarker +pub j2k::error::J2kError::InvalidMarker::marker: u8 +pub j2k::error::J2kError::InvalidMarker::offset: usize +pub j2k::error::J2kError::InvalidRegion +pub j2k::error::J2kError::InvalidRegion::h: u32 +pub j2k::error::J2kError::InvalidRegion::image_h: u32 +pub j2k::error::J2kError::InvalidRegion::image_w: u32 +pub j2k::error::J2kError::InvalidRegion::w: u32 +pub j2k::error::J2kError::InvalidRegion::x: u32 +pub j2k::error::J2kError::InvalidRegion::y: u32 +pub j2k::error::J2kError::InvalidSamples +pub j2k::error::J2kError::InvalidSamples::what: alloc::string::String +pub j2k::error::J2kError::InvalidSiz +pub j2k::error::J2kError::InvalidSiz::what: &'static str +pub j2k::error::J2kError::MissingRequiredBox +pub j2k::error::J2kError::MissingRequiredBox::box_type: &'static str +pub j2k::error::J2kError::MissingRequiredMarker +pub j2k::error::J2kError::MissingRequiredMarker::marker: &'static str +pub j2k::error::J2kError::NotImplemented(j2k_core::error::NotImplemented) +pub j2k::error::J2kError::RateTargetUnreachable +pub j2k::error::J2kError::RateTargetUnreachable::best: alloc::string::String +pub j2k::error::J2kError::RateTargetUnreachable::target: alloc::string::String +pub j2k::error::J2kError::Unsupported(j2k_core::error::Unsupported) +impl j2k_core::error::CodecError for j2k::error::J2kError +pub fn j2k::error::J2kError::is_buffer_error(&self) -> bool +pub fn j2k::error::J2kError::is_not_implemented(&self) -> bool +pub fn j2k::error::J2kError::is_truncated(&self) -> bool +pub fn j2k::error::J2kError::is_unsupported(&self) -> bool +pub mod j2k::scratch +pub struct j2k::scratch::J2kScratchPool +impl j2k::scratch::J2kScratchPool +pub const fn j2k::scratch::J2kScratchPool::new() -> Self +impl j2k_core::scratch::ScratchPool for j2k::scratch::J2kScratchPool +pub fn j2k::scratch::J2kScratchPool::bytes_allocated(&self) -> usize +pub fn j2k::scratch::J2kScratchPool::reset(&mut self) +pub mod j2k::view +pub struct j2k::view::J2kCodec +impl j2k_core::traits::ImageCodec for j2k::view::J2kCodec +pub type j2k::view::J2kCodec::Error = j2k::error::J2kError +pub type j2k::view::J2kCodec::Pool = j2k::scratch::J2kScratchPool +pub type j2k::view::J2kCodec::Warning = core::convert::Infallible +impl j2k_core::traits::TileBatchDecode for j2k::view::J2kCodec +pub type j2k::view::J2kCodec::Context = j2k::context::J2kContext +pub fn j2k::view::J2kCodec::decode_tile(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kCodec::decode_tile_region(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kCodec::decode_tile_region_scaled(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kCodec::decode_tile_scaled(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub struct j2k::view::J2kDecoder<'a> +impl<'a> j2k::view::J2kDecoder<'a> +pub fn j2k::view::J2kDecoder<'a>::bytes(&self) -> &'a [u8] +pub fn j2k::view::J2kDecoder<'a>::cpu_decode_parallelism(&self) -> j2k::CpuDecodeParallelism +pub fn j2k::view::J2kDecoder<'a>::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::view::J2kDecoder<'a>::decode_into_with_scratch(&mut self, &mut j2k::scratch::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::view::J2kDecoder<'a>::decode_region_into(&mut self, &mut j2k::scratch::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::view::J2kDecoder<'a>::decode_region_scaled_into(&mut self, &mut j2k::scratch::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::view::J2kDecoder<'a>::decode_rows_u16_bounded>(&mut self, &mut R, j2k::view::J2kRowDecodeOptions) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +pub fn j2k::view::J2kDecoder<'a>::decode_rows_u8_bounded>(&mut self, &mut R, j2k::view::J2kRowDecodeOptions) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +pub fn j2k::view::J2kDecoder<'a>::decode_scaled_into(&mut self, &mut j2k::scratch::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::view::J2kDecoder<'a>::from_view(j2k::view::J2kView<'a>) -> core::result::Result +pub fn j2k::view::J2kDecoder<'a>::info(&self) -> &j2k_core::types::Info +pub fn j2k::view::J2kDecoder<'a>::inspect(&'a [u8]) -> core::result::Result +pub fn j2k::view::J2kDecoder<'a>::new(&'a [u8]) -> core::result::Result +pub fn j2k::view::J2kDecoder<'a>::passthrough_candidate(&self) -> core::option::Option> +pub fn j2k::view::J2kDecoder<'a>::set_cpu_decode_parallelism(&mut self, j2k::CpuDecodeParallelism) +impl j2k_core::traits::ImageCodec for j2k::view::J2kDecoder<'_> +pub type j2k::view::J2kDecoder<'_>::Error = j2k::error::J2kError +pub type j2k::view::J2kDecoder<'_>::Pool = j2k::scratch::J2kScratchPool +pub type j2k::view::J2kDecoder<'_>::Warning = core::convert::Infallible +impl<'a> j2k_core::traits::ImageDecode<'a> for j2k::view::J2kDecoder<'a> +pub type j2k::view::J2kDecoder<'a>::View = j2k::view::J2kView<'a> +pub fn j2k::view::J2kDecoder<'a>::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kDecoder<'a>::decode_into_with_scratch(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kDecoder<'a>::decode_region_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kDecoder<'a>::decode_region_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kDecoder<'a>::decode_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kDecoder<'a>::from_view(Self::View) -> core::result::Result +pub fn j2k::view::J2kDecoder<'a>::inspect(&'a [u8]) -> core::result::Result +pub fn j2k::view::J2kDecoder<'a>::parse(&'a [u8]) -> core::result::Result +impl<'a> j2k_core::traits::ImageDecodeRows<'a, u16> for j2k::view::J2kDecoder<'a> +pub fn j2k::view::J2kDecoder<'a>::decode_rows>(&mut self, &mut R) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +impl<'a> j2k_core::traits::ImageDecodeRows<'a, u8> for j2k::view::J2kDecoder<'a> +pub fn j2k::view::J2kDecoder<'a>::decode_rows>(&mut self, &mut R) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +pub struct j2k::view::J2kRowDecodeOptions +impl j2k::view::J2kRowDecodeOptions +pub const fn j2k::view::J2kRowDecodeOptions::max_rows_per_stripe(self) -> u32 +pub const fn j2k::view::J2kRowDecodeOptions::max_stripe_bytes(self) -> usize +pub const fn j2k::view::J2kRowDecodeOptions::new(u32) -> Self +pub const fn j2k::view::J2kRowDecodeOptions::new_with_max_stripe_bytes(u32, usize) -> Self +pub const fn j2k::view::J2kRowDecodeOptions::with_max_stripe_bytes(self, usize) -> Self +impl core::default::Default for j2k::view::J2kRowDecodeOptions +pub fn j2k::view::J2kRowDecodeOptions::default() -> Self +pub struct j2k::view::J2kView<'a> +impl<'a> j2k::view::J2kView<'a> +pub fn j2k::view::J2kView<'a>::bytes(&self) -> &'a [u8] +pub fn j2k::view::J2kView<'a>::info(&self) -> &j2k_core::types::Info +pub fn j2k::view::J2kView<'a>::parse(&'a [u8]) -> core::result::Result +pub fn j2k::view::J2kView<'a>::passthrough_candidate(&self) -> core::option::Option> +pub enum j2k::CpuDecodeParallelism +pub j2k::CpuDecodeParallelism::Auto +pub j2k::CpuDecodeParallelism::Serial +pub enum j2k::EncodeBackendPreference +pub j2k::EncodeBackendPreference::Auto +pub j2k::EncodeBackendPreference::CpuOnly +pub j2k::EncodeBackendPreference::RequireDevice +pub enum j2k::J2kBlockCodingMode +pub j2k::J2kBlockCodingMode::Classic +pub j2k::J2kBlockCodingMode::HighThroughput +pub enum j2k::J2kEncodeValidation +pub j2k::J2kEncodeValidation::CpuRoundTrip +pub j2k::J2kEncodeValidation::External +#[non_exhaustive] pub enum j2k::J2kError +pub j2k::J2kError::Backend(alloc::string::String) +pub j2k::J2kError::Buffer(j2k_core::error::BufferError) +pub j2k::J2kError::DimensionOverflow +pub j2k::J2kError::DimensionOverflow::height: u32 +pub j2k::J2kError::DimensionOverflow::width: u32 +pub j2k::J2kError::Input(j2k_core::error::InputError) +pub j2k::J2kError::InvalidBox +pub j2k::J2kError::InvalidBox::offset: usize +pub j2k::J2kError::InvalidBox::what: &'static str +pub j2k::J2kError::InvalidCod +pub j2k::J2kError::InvalidCod::what: &'static str +pub j2k::J2kError::InvalidMarker +pub j2k::J2kError::InvalidMarker::marker: u8 +pub j2k::J2kError::InvalidMarker::offset: usize +pub j2k::J2kError::InvalidRegion +pub j2k::J2kError::InvalidRegion::h: u32 +pub j2k::J2kError::InvalidRegion::image_h: u32 +pub j2k::J2kError::InvalidRegion::image_w: u32 +pub j2k::J2kError::InvalidRegion::w: u32 +pub j2k::J2kError::InvalidRegion::x: u32 +pub j2k::J2kError::InvalidRegion::y: u32 +pub j2k::J2kError::InvalidSamples +pub j2k::J2kError::InvalidSamples::what: alloc::string::String +pub j2k::J2kError::InvalidSiz +pub j2k::J2kError::InvalidSiz::what: &'static str +pub j2k::J2kError::MissingRequiredBox +pub j2k::J2kError::MissingRequiredBox::box_type: &'static str +pub j2k::J2kError::MissingRequiredMarker +pub j2k::J2kError::MissingRequiredMarker::marker: &'static str +pub j2k::J2kError::NotImplemented(j2k_core::error::NotImplemented) +pub j2k::J2kError::RateTargetUnreachable +pub j2k::J2kError::RateTargetUnreachable::best: alloc::string::String +pub j2k::J2kError::RateTargetUnreachable::target: alloc::string::String +pub j2k::J2kError::Unsupported(j2k_core::error::Unsupported) +impl j2k_core::error::CodecError for j2k::error::J2kError +pub fn j2k::error::J2kError::is_buffer_error(&self) -> bool +pub fn j2k::error::J2kError::is_not_implemented(&self) -> bool +pub fn j2k::error::J2kError::is_truncated(&self) -> bool +pub fn j2k::error::J2kError::is_unsupported(&self) -> bool +pub enum j2k::J2kMarkerSegment +pub j2k::J2kMarkerSegment::Eph +pub j2k::J2kMarkerSegment::Plm +pub j2k::J2kMarkerSegment::Plt +pub j2k::J2kMarkerSegment::Sop +pub j2k::J2kMarkerSegment::Tlm +pub enum j2k::J2kProgressionOrder +pub j2k::J2kProgressionOrder::Cprl +pub j2k::J2kProgressionOrder::Lrcp +pub j2k::J2kProgressionOrder::Pcrl +pub j2k::J2kProgressionOrder::Rlcp +pub j2k::J2kProgressionOrder::Rpcl +pub enum j2k::J2kRateTarget +pub j2k::J2kRateTarget::BitsPerPixel(f64) +pub j2k::J2kRateTarget::Bytes(u64) +pub j2k::J2kRateTarget::PsnrDb(f64) +pub enum j2k::J2kToHtj2kMode +pub j2k::J2kToHtj2kMode::CoefficientPreserving +pub j2k::J2kToHtj2kMode::Passthrough +pub j2k::J2kToHtj2kMode::PixelPreserving +pub enum j2k::ReversibleTransform +pub j2k::ReversibleTransform::None53 +pub j2k::ReversibleTransform::Rct53 +pub struct j2k::EncodedJ2k +pub j2k::EncodedJ2k::backend: j2k_core::backend::BackendKind +pub j2k::EncodedJ2k::bit_depth: u8 +pub j2k::EncodedJ2k::codestream: alloc::vec::Vec +pub j2k::EncodedJ2k::components: u8 +pub j2k::EncodedJ2k::dispatch_report: j2k_types::J2kEncodeDispatchReport +pub j2k::EncodedJ2k::height: u32 +pub j2k::EncodedJ2k::signed: bool +pub j2k::EncodedJ2k::width: u32 +pub struct j2k::EncodedLossyJ2k +pub j2k::EncodedLossyJ2k::backend: j2k_core::backend::BackendKind +pub j2k::EncodedLossyJ2k::bit_depth: u8 +pub j2k::EncodedLossyJ2k::codestream: alloc::vec::Vec +pub j2k::EncodedLossyJ2k::components: u8 +pub j2k::EncodedLossyJ2k::dispatch_report: j2k_types::J2kEncodeDispatchReport +pub j2k::EncodedLossyJ2k::height: u32 +pub j2k::EncodedLossyJ2k::report: j2k::J2kLossyEncodeReport +pub j2k::EncodedLossyJ2k::signed: bool +pub j2k::EncodedLossyJ2k::width: u32 +pub struct j2k::J2kCodec +impl j2k_core::traits::ImageCodec for j2k::view::J2kCodec +pub type j2k::view::J2kCodec::Error = j2k::error::J2kError +pub type j2k::view::J2kCodec::Pool = j2k::scratch::J2kScratchPool +pub type j2k::view::J2kCodec::Warning = core::convert::Infallible +impl j2k_core::traits::TileBatchDecode for j2k::view::J2kCodec +pub type j2k::view::J2kCodec::Context = j2k::context::J2kContext +pub fn j2k::view::J2kCodec::decode_tile(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kCodec::decode_tile_region(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kCodec::decode_tile_region_scaled(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kCodec::decode_tile_scaled(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub struct j2k::J2kContext +impl j2k::context::J2kContext +pub fn j2k::context::J2kContext::cpu_decode_parallelism(&self) -> j2k::CpuDecodeParallelism +pub const fn j2k::context::J2kContext::new() -> Self +pub fn j2k::context::J2kContext::set_cpu_decode_parallelism(&mut self, j2k::CpuDecodeParallelism) +impl j2k_core::context::CodecContext for j2k::context::J2kContext +pub fn j2k::context::J2kContext::cache_stats(&self) -> j2k_core::context::CacheStats +pub fn j2k::context::J2kContext::clear(&mut self) +pub struct j2k::J2kDecoder<'a> +impl<'a> j2k::view::J2kDecoder<'a> +pub fn j2k::view::J2kDecoder<'a>::bytes(&self) -> &'a [u8] +pub fn j2k::view::J2kDecoder<'a>::cpu_decode_parallelism(&self) -> j2k::CpuDecodeParallelism +pub fn j2k::view::J2kDecoder<'a>::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::view::J2kDecoder<'a>::decode_into_with_scratch(&mut self, &mut j2k::scratch::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::view::J2kDecoder<'a>::decode_region_into(&mut self, &mut j2k::scratch::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::view::J2kDecoder<'a>::decode_region_scaled_into(&mut self, &mut j2k::scratch::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::view::J2kDecoder<'a>::decode_rows_u16_bounded>(&mut self, &mut R, j2k::view::J2kRowDecodeOptions) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +pub fn j2k::view::J2kDecoder<'a>::decode_rows_u8_bounded>(&mut self, &mut R, j2k::view::J2kRowDecodeOptions) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +pub fn j2k::view::J2kDecoder<'a>::decode_scaled_into(&mut self, &mut j2k::scratch::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::view::J2kDecoder<'a>::from_view(j2k::view::J2kView<'a>) -> core::result::Result +pub fn j2k::view::J2kDecoder<'a>::info(&self) -> &j2k_core::types::Info +pub fn j2k::view::J2kDecoder<'a>::inspect(&'a [u8]) -> core::result::Result +pub fn j2k::view::J2kDecoder<'a>::new(&'a [u8]) -> core::result::Result +pub fn j2k::view::J2kDecoder<'a>::passthrough_candidate(&self) -> core::option::Option> +pub fn j2k::view::J2kDecoder<'a>::set_cpu_decode_parallelism(&mut self, j2k::CpuDecodeParallelism) +impl j2k_core::traits::ImageCodec for j2k::view::J2kDecoder<'_> +pub type j2k::view::J2kDecoder<'_>::Error = j2k::error::J2kError +pub type j2k::view::J2kDecoder<'_>::Pool = j2k::scratch::J2kScratchPool +pub type j2k::view::J2kDecoder<'_>::Warning = core::convert::Infallible +impl<'a> j2k_core::traits::ImageDecode<'a> for j2k::view::J2kDecoder<'a> +pub type j2k::view::J2kDecoder<'a>::View = j2k::view::J2kView<'a> +pub fn j2k::view::J2kDecoder<'a>::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kDecoder<'a>::decode_into_with_scratch(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kDecoder<'a>::decode_region_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kDecoder<'a>::decode_region_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kDecoder<'a>::decode_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k::view::J2kDecoder<'a>::from_view(Self::View) -> core::result::Result +pub fn j2k::view::J2kDecoder<'a>::inspect(&'a [u8]) -> core::result::Result +pub fn j2k::view::J2kDecoder<'a>::parse(&'a [u8]) -> core::result::Result +impl<'a> j2k_core::traits::ImageDecodeRows<'a, u16> for j2k::view::J2kDecoder<'a> +pub fn j2k::view::J2kDecoder<'a>::decode_rows>(&mut self, &mut R) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +impl<'a> j2k_core::traits::ImageDecodeRows<'a, u8> for j2k::view::J2kDecoder<'a> +pub fn j2k::view::J2kDecoder<'a>::decode_rows>(&mut self, &mut R) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +#[non_exhaustive] pub struct j2k::J2kLosslessEncodeOptions +pub j2k::J2kLosslessEncodeOptions::backend: j2k::EncodeBackendPreference +pub j2k::J2kLosslessEncodeOptions::block_coding_mode: j2k::J2kBlockCodingMode +pub j2k::J2kLosslessEncodeOptions::max_decomposition_levels: core::option::Option +pub j2k::J2kLosslessEncodeOptions::progression: j2k::J2kProgressionOrder +pub j2k::J2kLosslessEncodeOptions::reversible_transform: j2k::ReversibleTransform +pub j2k::J2kLosslessEncodeOptions::validation: j2k::J2kEncodeValidation +impl j2k::J2kLosslessEncodeOptions +pub const fn j2k::J2kLosslessEncodeOptions::new(j2k::EncodeBackendPreference, j2k::J2kBlockCodingMode, j2k::J2kProgressionOrder, core::option::Option, j2k::ReversibleTransform, j2k::J2kEncodeValidation) -> Self +pub const fn j2k::J2kLosslessEncodeOptions::with_accelerated_backend(self) -> Self +pub const fn j2k::J2kLosslessEncodeOptions::with_backend(self, j2k::EncodeBackendPreference) -> Self +pub const fn j2k::J2kLosslessEncodeOptions::with_block_coding_mode(self, j2k::J2kBlockCodingMode) -> Self +pub const fn j2k::J2kLosslessEncodeOptions::with_cpu_only_backend(self) -> Self +pub const fn j2k::J2kLosslessEncodeOptions::with_max_decomposition_levels(self, core::option::Option) -> Self +pub const fn j2k::J2kLosslessEncodeOptions::with_progression(self, j2k::J2kProgressionOrder) -> Self +pub const fn j2k::J2kLosslessEncodeOptions::with_reversible_transform(self, j2k::ReversibleTransform) -> Self +pub const fn j2k::J2kLosslessEncodeOptions::with_strict_device_backend(self) -> Self +pub const fn j2k::J2kLosslessEncodeOptions::with_validation(self, j2k::J2kEncodeValidation) -> Self +impl core::default::Default for j2k::J2kLosslessEncodeOptions +pub fn j2k::J2kLosslessEncodeOptions::default() -> Self +pub struct j2k::J2kLosslessSamples<'a> +pub j2k::J2kLosslessSamples::bit_depth: u8 +pub j2k::J2kLosslessSamples::components: u8 +pub j2k::J2kLosslessSamples::data: &'a [u8] +pub j2k::J2kLosslessSamples::height: u32 +pub j2k::J2kLosslessSamples::signed: bool +pub j2k::J2kLosslessSamples::width: u32 +impl<'a> j2k::J2kLosslessSamples<'a> +pub fn j2k::J2kLosslessSamples<'a>::new(&'a [u8], u32, u32, u8, u8, bool) -> core::result::Result +#[non_exhaustive] pub struct j2k::J2kLossyEncodeOptions +pub j2k::J2kLossyEncodeOptions::backend: j2k::EncodeBackendPreference +pub j2k::J2kLossyEncodeOptions::block_coding_mode: j2k::J2kBlockCodingMode +pub j2k::J2kLossyEncodeOptions::marker_segments: alloc::vec::Vec +pub j2k::J2kLossyEncodeOptions::max_decomposition_levels: core::option::Option +pub j2k::J2kLossyEncodeOptions::precinct_exponents: alloc::vec::Vec<(u8, u8)> +pub j2k::J2kLossyEncodeOptions::progression: j2k::J2kProgressionOrder +pub j2k::J2kLossyEncodeOptions::psnr_iteration_budget: u8 +pub j2k::J2kLossyEncodeOptions::psnr_tolerance_db: f64 +pub j2k::J2kLossyEncodeOptions::quality_layers: alloc::vec::Vec +pub j2k::J2kLossyEncodeOptions::rate_target: core::option::Option +pub j2k::J2kLossyEncodeOptions::tile_size: core::option::Option<(u32, u32)> +pub j2k::J2kLossyEncodeOptions::validation: j2k::J2kEncodeValidation +impl j2k::J2kLossyEncodeOptions +pub fn j2k::J2kLossyEncodeOptions::with_accelerated_backend(self) -> Self +pub fn j2k::J2kLossyEncodeOptions::with_backend(self, j2k::EncodeBackendPreference) -> Self +pub fn j2k::J2kLossyEncodeOptions::with_block_coding_mode(self, j2k::J2kBlockCodingMode) -> Self +pub fn j2k::J2kLossyEncodeOptions::with_cpu_only_backend(self) -> Self +pub fn j2k::J2kLossyEncodeOptions::with_marker_segments(self, alloc::vec::Vec) -> Self +pub fn j2k::J2kLossyEncodeOptions::with_max_decomposition_levels(self, core::option::Option) -> Self +pub fn j2k::J2kLossyEncodeOptions::with_progression(self, j2k::J2kProgressionOrder) -> Self +pub fn j2k::J2kLossyEncodeOptions::with_quality_layers(self, alloc::vec::Vec) -> Self +pub fn j2k::J2kLossyEncodeOptions::with_rate_target(self, core::option::Option) -> Self +pub fn j2k::J2kLossyEncodeOptions::with_strict_device_backend(self) -> Self +pub fn j2k::J2kLossyEncodeOptions::with_validation(self, j2k::J2kEncodeValidation) -> Self +impl core::default::Default for j2k::J2kLossyEncodeOptions +pub fn j2k::J2kLossyEncodeOptions::default() -> Self +pub struct j2k::J2kLossyEncodeReport +pub j2k::J2kLossyEncodeReport::actual_bits_per_pixel: f64 +pub j2k::J2kLossyEncodeReport::actual_bytes: u64 +pub j2k::J2kLossyEncodeReport::ht_rate_granularity_bytes: core::option::Option +pub j2k::J2kLossyEncodeReport::psnr_db: core::option::Option +pub j2k::J2kLossyEncodeReport::quality_layers: u16 +pub j2k::J2kLossyEncodeReport::quantization_scale: f32 +pub j2k::J2kLossyEncodeReport::target: core::option::Option +pub struct j2k::J2kLossySamples<'a> +pub j2k::J2kLossySamples::bit_depth: u8 +pub j2k::J2kLossySamples::components: u8 +pub j2k::J2kLossySamples::data: &'a [u8] +pub j2k::J2kLossySamples::height: u32 +pub j2k::J2kLossySamples::signed: bool +pub j2k::J2kLossySamples::width: u32 +impl<'a> j2k::J2kLossySamples<'a> +pub fn j2k::J2kLossySamples<'a>::new(&'a [u8], u32, u32, u8, u8, bool) -> core::result::Result +pub struct j2k::J2kQualityLayer +pub j2k::J2kQualityLayer::target: j2k::J2kRateTarget +impl j2k::J2kQualityLayer +pub const fn j2k::J2kQualityLayer::new(j2k::J2kRateTarget) -> Self +pub struct j2k::J2kRowDecodeOptions +impl j2k::view::J2kRowDecodeOptions +pub const fn j2k::view::J2kRowDecodeOptions::max_rows_per_stripe(self) -> u32 +pub const fn j2k::view::J2kRowDecodeOptions::max_stripe_bytes(self) -> usize +pub const fn j2k::view::J2kRowDecodeOptions::new(u32) -> Self +pub const fn j2k::view::J2kRowDecodeOptions::new_with_max_stripe_bytes(u32, usize) -> Self +pub const fn j2k::view::J2kRowDecodeOptions::with_max_stripe_bytes(self, usize) -> Self +impl core::default::Default for j2k::view::J2kRowDecodeOptions +pub fn j2k::view::J2kRowDecodeOptions::default() -> Self +pub struct j2k::J2kScratchPool +impl j2k::scratch::J2kScratchPool +pub const fn j2k::scratch::J2kScratchPool::new() -> Self +impl j2k_core::scratch::ScratchPool for j2k::scratch::J2kScratchPool +pub fn j2k::scratch::J2kScratchPool::bytes_allocated(&self) -> usize +pub fn j2k::scratch::J2kScratchPool::reset(&mut self) +#[non_exhaustive] pub struct j2k::J2kToHtj2kOptions +pub j2k::J2kToHtj2kOptions::output_payload_kind: j2k_core::passthrough::CompressedPayloadKind +pub j2k::J2kToHtj2kOptions::progression: j2k::J2kProgressionOrder +pub j2k::J2kToHtj2kOptions::validation: j2k::J2kEncodeValidation +impl j2k::J2kToHtj2kOptions +pub const fn j2k::J2kToHtj2kOptions::new(j2k_core::passthrough::CompressedPayloadKind, j2k::J2kProgressionOrder, j2k::J2kEncodeValidation) -> Self +impl core::default::Default for j2k::J2kToHtj2kOptions +pub fn j2k::J2kToHtj2kOptions::default() -> Self +pub struct j2k::J2kToHtj2kReport +pub j2k::J2kToHtj2kReport::bit_depth: u8 +pub j2k::J2kToHtj2kReport::components: u8 +pub j2k::J2kToHtj2kReport::height: u32 +pub j2k::J2kToHtj2kReport::input_payload_kind: j2k_core::passthrough::CompressedPayloadKind +pub j2k::J2kToHtj2kReport::input_transfer_syntax: j2k_core::passthrough::CompressedTransferSyntax +pub j2k::J2kToHtj2kReport::mode: j2k::J2kToHtj2kMode +pub j2k::J2kToHtj2kReport::output_payload_kind: j2k_core::passthrough::CompressedPayloadKind +pub j2k::J2kToHtj2kReport::output_transfer_syntax: j2k_core::passthrough::CompressedTransferSyntax +pub j2k::J2kToHtj2kReport::width: u32 +pub struct j2k::J2kView<'a> +impl<'a> j2k::view::J2kView<'a> +pub fn j2k::view::J2kView<'a>::bytes(&self) -> &'a [u8] +pub fn j2k::view::J2kView<'a>::info(&self) -> &j2k_core::types::Info +pub fn j2k::view::J2kView<'a>::parse(&'a [u8]) -> core::result::Result +pub fn j2k::view::J2kView<'a>::passthrough_candidate(&self) -> core::option::Option> +pub struct j2k::ReencodedHtj2k +pub j2k::ReencodedHtj2k::bytes: alloc::vec::Vec +pub j2k::ReencodedHtj2k::report: j2k::J2kToHtj2kReport +pub fn j2k::decode_tile_into_in_context(&[u8], &mut j2k_core::context::DecoderContext, &mut j2k::scratch::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::decode_tile_region_into_in_context(&[u8], &mut j2k_core::context::DecoderContext, &mut j2k::scratch::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::decode_tile_region_scaled_into_in_context(&[u8], &mut j2k_core::context::DecoderContext, &mut j2k::scratch::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::decode_tile_scaled_into_in_context(&[u8], &mut j2k_core::context::DecoderContext, &mut j2k::scratch::J2kScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, j2k::error::J2kError> +pub fn j2k::decode_tiles_into(&mut [j2k::TileDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result>, j2k::TileBatchError> +pub fn j2k::decode_tiles_region_into(&mut [j2k::TileRegionDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result>, j2k::TileBatchError> +pub fn j2k::decode_tiles_region_scaled_into(&mut [j2k::TileRegionScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result>, j2k::TileBatchError> +pub fn j2k::decode_tiles_scaled_into(&mut [j2k::TileScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result>, j2k::TileBatchError> +pub fn j2k::encode_j2k_lossless(j2k::J2kLosslessSamples<'_>, &j2k::J2kLosslessEncodeOptions) -> core::result::Result +pub fn j2k::encode_j2k_lossless_with_accelerator(j2k::J2kLosslessSamples<'_>, &j2k::J2kLosslessEncodeOptions, j2k_core::backend::BackendKind, &mut impl j2k::adapter::encode_stage::J2kEncodeStageAccelerator) -> core::result::Result +pub fn j2k::encode_j2k_lossy(j2k::J2kLossySamples<'_>, &j2k::J2kLossyEncodeOptions) -> core::result::Result +pub fn j2k::encode_j2k_lossy_with_accelerator(j2k::J2kLossySamples<'_>, &j2k::J2kLossyEncodeOptions, j2k_core::backend::BackendKind, &mut impl j2k::adapter::encode_stage::J2kEncodeStageAccelerator) -> core::result::Result +pub fn j2k::j2k_lossless_decomposition_levels(j2k::J2kLosslessSamples<'_>) -> u8 +pub fn j2k::j2k_lossless_decomposition_levels_for_options(j2k::J2kLosslessSamples<'_>, j2k::J2kLosslessEncodeOptions) -> u8 +pub fn j2k::j2k_lossless_decomposition_levels_for_progression(j2k::J2kLosslessSamples<'_>, j2k::J2kProgressionOrder) -> u8 +pub fn j2k::recode_j2k_to_htj2k_lossless(&[u8], j2k::J2kToHtj2kOptions) -> core::result::Result +pub type j2k::TileBatchError = j2k_core::batch::TileBatchError +pub type j2k::TileDecodeJob<'i, 'o> = j2k_core::batch::TileDecodeJob<'i, 'o> +pub type j2k::TileRegionDecodeJob<'i, 'o> = j2k_core::batch::TileRegionDecodeJob<'i, 'o> +pub type j2k::TileRegionScaledDecodeJob<'i, 'o> = j2k_core::batch::TileRegionScaledDecodeJob<'i, 'o> +pub type j2k::TileScaledDecodeJob<'i, 'o> = j2k_core::batch::TileScaledDecodeJob<'i, 'o> +``` + +## `j2k-core` + +```text +pub mod j2k_core +pub mod j2k_core::accelerator +pub enum j2k_core::accelerator::BackendFailureKind +pub j2k_core::accelerator::BackendFailureKind::InvalidInput +pub j2k_core::accelerator::BackendFailureKind::Kernel +pub j2k_core::accelerator::BackendFailureKind::OutOfMemory +pub j2k_core::accelerator::BackendFailureKind::Runtime +pub j2k_core::accelerator::BackendFailureKind::StatePoisoned +pub j2k_core::accelerator::BackendFailureKind::Unavailable +pub j2k_core::accelerator::BackendFailureKind::Unsupported +#[non_exhaustive] pub enum j2k_core::accelerator::SurfaceResidency +pub j2k_core::accelerator::SurfaceResidency::CpuStagedCudaUpload +pub j2k_core::accelerator::SurfaceResidency::CpuStagedMetalUpload +pub j2k_core::accelerator::SurfaceResidency::CudaResidentDecode +pub j2k_core::accelerator::SurfaceResidency::Device(j2k_core::backend::BackendKind) +pub j2k_core::accelerator::SurfaceResidency::Host +pub j2k_core::accelerator::SurfaceResidency::MetalResidentDecode +pub j2k_core::accelerator::SurfaceResidency::Shared(j2k_core::backend::BackendKind) +impl j2k_core::accelerator::SurfaceResidency +pub const fn j2k_core::accelerator::SurfaceResidency::for_backend(j2k_core::backend::BackendKind) -> Self +pub struct j2k_core::accelerator::DeviceMemoryRange +pub j2k_core::accelerator::DeviceMemoryRange::allocation: u64 +pub j2k_core::accelerator::DeviceMemoryRange::backend: j2k_core::backend::BackendKind +pub j2k_core::accelerator::DeviceMemoryRange::len: usize +pub j2k_core::accelerator::DeviceMemoryRange::offset: usize +impl j2k_core::accelerator::DeviceMemoryRange +pub const fn j2k_core::accelerator::DeviceMemoryRange::new(j2k_core::backend::BackendKind, u64, usize, usize) -> Self +pub struct j2k_core::accelerator::ExecutionStats +pub j2k_core::accelerator::ExecutionStats::device_us: u128 +pub j2k_core::accelerator::ExecutionStats::kernel_dispatches: u64 +pub j2k_core::accelerator::ExecutionStats::readback_bytes: u64 +pub j2k_core::accelerator::ExecutionStats::submissions: u64 +pub j2k_core::accelerator::ExecutionStats::upload_bytes: u64 +impl j2k_core::accelerator::ExecutionStats +pub const fn j2k_core::accelerator::ExecutionStats::new() -> Self +pub const fn j2k_core::accelerator::ExecutionStats::saturating_add(self, Self) -> Self +pub trait j2k_core::accelerator::AcceleratorSession +pub fn j2k_core::accelerator::AcceleratorSession::backend_kind(&self) -> j2k_core::backend::BackendKind +pub fn j2k_core::accelerator::AcceleratorSession::execution_stats(&self) -> j2k_core::accelerator::ExecutionStats +pub unsafe trait j2k_core::accelerator::GpuAbi: core::marker::Copy + 'static +pub const j2k_core::accelerator::GpuAbi::NAME: &'static str +pub fn j2k_core::accelerator::GpuAbi::as_bytes(&Self) -> &[u8] +pub fn j2k_core::accelerator::GpuAbi::slice_as_bytes(&[Self]) -> &[u8] +pub fn j2k_core::accelerator::GpuAbi::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for f32 +pub const f32::NAME: &'static str +pub fn f32::as_bytes(&Self) -> &[u8] +pub fn f32::slice_as_bytes(&[Self]) -> &[u8] +pub fn f32::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for f64 +pub const f64::NAME: &'static str +pub fn f64::as_bytes(&Self) -> &[u8] +pub fn f64::slice_as_bytes(&[Self]) -> &[u8] +pub fn f64::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for i16 +pub const i16::NAME: &'static str +pub fn i16::as_bytes(&Self) -> &[u8] +pub fn i16::slice_as_bytes(&[Self]) -> &[u8] +pub fn i16::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for i32 +pub const i32::NAME: &'static str +pub fn i32::as_bytes(&Self) -> &[u8] +pub fn i32::slice_as_bytes(&[Self]) -> &[u8] +pub fn i32::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for i64 +pub const i64::NAME: &'static str +pub fn i64::as_bytes(&Self) -> &[u8] +pub fn i64::slice_as_bytes(&[Self]) -> &[u8] +pub fn i64::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for i8 +pub const i8::NAME: &'static str +pub fn i8::as_bytes(&Self) -> &[u8] +pub fn i8::slice_as_bytes(&[Self]) -> &[u8] +pub fn i8::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for u16 +pub const u16::NAME: &'static str +pub fn u16::as_bytes(&Self) -> &[u8] +pub fn u16::slice_as_bytes(&[Self]) -> &[u8] +pub fn u16::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for u32 +pub const u32::NAME: &'static str +pub fn u32::as_bytes(&Self) -> &[u8] +pub fn u32::slice_as_bytes(&[Self]) -> &[u8] +pub fn u32::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for u64 +pub const u64::NAME: &'static str +pub fn u64::as_bytes(&Self) -> &[u8] +pub fn u64::slice_as_bytes(&[Self]) -> &[u8] +pub fn u64::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for u8 +pub const u8::NAME: &'static str +pub fn u8::as_bytes(&Self) -> &[u8] +pub fn u8::slice_as_bytes(&[Self]) -> &[u8] +pub fn u8::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for [T; N] where T: j2k_core::accelerator::GpuAbi +pub const [T; N]::NAME: &'static str +pub fn [T; N]::as_bytes(&Self) -> &[u8] +pub fn [T; N]::slice_as_bytes(&[Self]) -> &[u8] +pub fn [T; N]::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +pub mod j2k_core::backend +pub enum j2k_core::backend::BackendKind +pub j2k_core::backend::BackendKind::Cpu +pub j2k_core::backend::BackendKind::Cuda +pub j2k_core::backend::BackendKind::Metal +pub enum j2k_core::backend::BackendRequest +pub j2k_core::backend::BackendRequest::Auto +pub j2k_core::backend::BackendRequest::Cpu +pub j2k_core::backend::BackendRequest::Cuda +pub j2k_core::backend::BackendRequest::Metal +impl j2k_core::backend::BackendRequest +pub const j2k_core::backend::BackendRequest::ACCELERATED: Self +pub const j2k_core::backend::BackendRequest::CPU_ONLY: Self +pub const j2k_core::backend::BackendRequest::STRICT_CUDA: Self +pub const j2k_core::backend::BackendRequest::STRICT_METAL: Self +pub struct j2k_core::backend::BackendCapabilities +pub j2k_core::backend::BackendCapabilities::cpu: j2k_core::backend::CpuFeatures +pub j2k_core::backend::BackendCapabilities::cuda: bool +pub j2k_core::backend::BackendCapabilities::metal: bool +impl j2k_core::backend::BackendCapabilities +pub fn j2k_core::backend::BackendCapabilities::compile_time_defaults() -> Self +pub const fn j2k_core::backend::BackendCapabilities::first_available_accelerator(self) -> core::option::Option +pub fn j2k_core::backend::BackendCapabilities::resolve(self, j2k_core::backend::BackendRequest) -> core::option::Option +pub const fn j2k_core::backend::BackendCapabilities::supports(self, j2k_core::backend::BackendRequest) -> bool +pub struct j2k_core::backend::CpuFeatures +pub j2k_core::backend::CpuFeatures::avx2: bool +pub j2k_core::backend::CpuFeatures::neon: bool +pub j2k_core::backend::CpuFeatures::sse41: bool +impl j2k_core::backend::CpuFeatures +pub fn j2k_core::backend::CpuFeatures::detect() -> Self +pub mod j2k_core::batch +pub struct j2k_core::batch::TileBatchError +pub j2k_core::batch::TileBatchError::index: usize +pub j2k_core::batch::TileBatchError::source: E +impl core::error::Error for j2k_core::batch::TileBatchError +pub fn j2k_core::batch::TileBatchError::source(&self) -> core::option::Option<&(dyn core::error::Error + 'static)> +impl core::fmt::Display for j2k_core::batch::TileBatchError +pub fn j2k_core::batch::TileBatchError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_core::batch::TileBatchOptions +pub j2k_core::batch::TileBatchOptions::workers: core::option::Option +impl j2k_core::batch::TileBatchOptions +pub const fn j2k_core::batch::TileBatchOptions::new(core::option::Option) -> Self +pub struct j2k_core::batch::TileDecodeJob<'i, 'o> +pub j2k_core::batch::TileDecodeJob::input: &'i [u8] +pub j2k_core::batch::TileDecodeJob::out: &'o mut [u8] +pub j2k_core::batch::TileDecodeJob::stride: usize +pub struct j2k_core::batch::TileRegionDecodeJob<'i, 'o> +pub j2k_core::batch::TileRegionDecodeJob::input: &'i [u8] +pub j2k_core::batch::TileRegionDecodeJob::out: &'o mut [u8] +pub j2k_core::batch::TileRegionDecodeJob::roi: j2k_core::types::Rect +pub j2k_core::batch::TileRegionDecodeJob::stride: usize +pub struct j2k_core::batch::TileRegionScaledDecodeJob<'i, 'o> +pub j2k_core::batch::TileRegionScaledDecodeJob::input: &'i [u8] +pub j2k_core::batch::TileRegionScaledDecodeJob::out: &'o mut [u8] +pub j2k_core::batch::TileRegionScaledDecodeJob::roi: j2k_core::types::Rect +pub j2k_core::batch::TileRegionScaledDecodeJob::scale: j2k_core::scale::Downscale +pub j2k_core::batch::TileRegionScaledDecodeJob::stride: usize +pub struct j2k_core::batch::TileScaledDecodeJob<'i, 'o> +pub j2k_core::batch::TileScaledDecodeJob::input: &'i [u8] +pub j2k_core::batch::TileScaledDecodeJob::out: &'o mut [u8] +pub j2k_core::batch::TileScaledDecodeJob::scale: j2k_core::scale::Downscale +pub j2k_core::batch::TileScaledDecodeJob::stride: usize +pub fn j2k_core::batch::collect_indexed_batch_results(usize, alloc::vec::Vec>, F) -> core::result::Result, B> where F: core::ops::function::FnOnce(usize, E) -> B +pub fn j2k_core::batch::tile_batch_worker_count(usize, j2k_core::batch::TileBatchOptions, usize) -> usize +pub type j2k_core::batch::IndexedBatchResult = (usize, core::result::Result) +pub mod j2k_core::context +pub struct j2k_core::context::CacheStats +pub j2k_core::context::CacheStats::evictions: u64 +pub j2k_core::context::CacheStats::hits: u64 +pub j2k_core::context::CacheStats::misses: u64 +pub j2k_core::context::CacheStats::occupied_slots: u64 +impl j2k_core::context::CacheStats +pub const fn j2k_core::context::CacheStats::new(u64, u64) -> Self +pub const fn j2k_core::context::CacheStats::with_slots(u64, u64, u64, u64) -> Self +pub struct j2k_core::context::DecoderContext +impl j2k_core::context::DecoderContext +pub fn j2k_core::context::DecoderContext::cache_stats(&self) -> j2k_core::context::CacheStats +pub fn j2k_core::context::DecoderContext::clear(&mut self) +pub fn j2k_core::context::DecoderContext::codec(&self) -> &C +pub fn j2k_core::context::DecoderContext::codec_mut(&mut self) -> &mut C +pub fn j2k_core::context::DecoderContext::into_inner(self) -> C +pub fn j2k_core::context::DecoderContext::new() -> Self +pub trait j2k_core::context::CodecContext: core::default::Default + core::marker::Send +pub fn j2k_core::context::CodecContext::cache_stats(&self) -> j2k_core::context::CacheStats +pub fn j2k_core::context::CodecContext::clear(&mut self) +pub mod j2k_core::device +pub const fn j2k_core::device::validate_cuda_surface_backend_request(j2k_core::backend::BackendRequest) -> core::result::Result<(), j2k_core::backend::BackendRequest> +pub mod j2k_core::error +pub enum j2k_core::error::BufferError +pub j2k_core::error::BufferError::AllocationTooLarge +pub j2k_core::error::BufferError::AllocationTooLarge::cap: usize +pub j2k_core::error::BufferError::AllocationTooLarge::requested: usize +pub j2k_core::error::BufferError::AllocationTooLarge::what: &'static str +pub j2k_core::error::BufferError::InputTooSmall +pub j2k_core::error::BufferError::InputTooSmall::have: usize +pub j2k_core::error::BufferError::InputTooSmall::required: usize +pub j2k_core::error::BufferError::OutputTooSmall +pub j2k_core::error::BufferError::OutputTooSmall::have: usize +pub j2k_core::error::BufferError::OutputTooSmall::required: usize +pub j2k_core::error::BufferError::SampleTypeMismatch +pub j2k_core::error::BufferError::SampleTypeMismatch::fmt: j2k_core::pixel::PixelFormat +pub j2k_core::error::BufferError::SampleTypeMismatch::sample_type: j2k_core::sample::SampleType +pub j2k_core::error::BufferError::SizeOverflow +pub j2k_core::error::BufferError::SizeOverflow::what: &'static str +pub j2k_core::error::BufferError::StrideNotAligned +pub j2k_core::error::BufferError::StrideNotAligned::align: usize +pub j2k_core::error::BufferError::StrideNotAligned::stride: usize +pub j2k_core::error::BufferError::StrideTooSmall +pub j2k_core::error::BufferError::StrideTooSmall::row_bytes: usize +pub j2k_core::error::BufferError::StrideTooSmall::stride: usize +pub enum j2k_core::error::InputError +pub j2k_core::error::InputError::TooShort +pub j2k_core::error::InputError::TooShort::have: usize +pub j2k_core::error::InputError::TooShort::need: usize +pub j2k_core::error::InputError::TruncatedAt +pub j2k_core::error::InputError::TruncatedAt::offset: usize +pub j2k_core::error::InputError::TruncatedAt::segment: &'static str +pub struct j2k_core::error::NotImplemented +pub j2k_core::error::NotImplemented::what: &'static str +pub struct j2k_core::error::Unsupported +pub j2k_core::error::Unsupported::what: &'static str +pub trait j2k_core::error::CodecError: core::error::Error + core::marker::Send + core::marker::Sync + 'static +pub fn j2k_core::error::CodecError::is_buffer_error(&self) -> bool +pub fn j2k_core::error::CodecError::is_not_implemented(&self) -> bool +pub fn j2k_core::error::CodecError::is_truncated(&self) -> bool +pub fn j2k_core::error::CodecError::is_unsupported(&self) -> bool +pub mod j2k_core::passthrough +#[non_exhaustive] pub enum j2k_core::passthrough::CompressedPayloadKind +pub j2k_core::passthrough::CompressedPayloadKind::Jp2File +pub j2k_core::passthrough::CompressedPayloadKind::Jpeg2000Codestream +pub j2k_core::passthrough::CompressedPayloadKind::JpegInterchange +#[non_exhaustive] pub enum j2k_core::passthrough::CompressedTransferSyntax +pub j2k_core::passthrough::CompressedTransferSyntax::HtJpeg2000Lossless +pub j2k_core::passthrough::CompressedTransferSyntax::HtJpeg2000Lossy +pub j2k_core::passthrough::CompressedTransferSyntax::Jpeg2000Lossless +pub j2k_core::passthrough::CompressedTransferSyntax::Jpeg2000Lossy +pub j2k_core::passthrough::CompressedTransferSyntax::JpegBaseline8 +pub j2k_core::passthrough::CompressedTransferSyntax::JpegExtendedSequential +impl j2k_core::passthrough::CompressedTransferSyntax +pub const fn j2k_core::passthrough::CompressedTransferSyntax::is_jpeg2000_family(self) -> bool +pub const fn j2k_core::passthrough::CompressedTransferSyntax::is_lossless(self) -> bool +pub enum j2k_core::passthrough::PassthroughDecision<'a> +pub j2k_core::passthrough::PassthroughDecision::Copy +pub j2k_core::passthrough::PassthroughDecision::Copy::bytes: &'a [u8] +pub j2k_core::passthrough::PassthroughDecision::Transcode +pub j2k_core::passthrough::PassthroughDecision::Transcode::reason: j2k_core::passthrough::PassthroughRejectReason +#[non_exhaustive] pub enum j2k_core::passthrough::PassthroughRejectReason +pub j2k_core::passthrough::PassthroughRejectReason::BitDepthMismatch +pub j2k_core::passthrough::PassthroughRejectReason::BitDepthMismatch::destination: u8 +pub j2k_core::passthrough::PassthroughRejectReason::BitDepthMismatch::source: u8 +pub j2k_core::passthrough::PassthroughRejectReason::ColorspaceMismatch +pub j2k_core::passthrough::PassthroughRejectReason::ColorspaceMismatch::destination: j2k_core::types::Colorspace +pub j2k_core::passthrough::PassthroughRejectReason::ColorspaceMismatch::source: j2k_core::types::Colorspace +pub j2k_core::passthrough::PassthroughRejectReason::ComponentsMismatch +pub j2k_core::passthrough::PassthroughRejectReason::ComponentsMismatch::destination: u8 +pub j2k_core::passthrough::PassthroughRejectReason::ComponentsMismatch::source: u8 +pub j2k_core::passthrough::PassthroughRejectReason::DimensionsMismatch +pub j2k_core::passthrough::PassthroughRejectReason::DimensionsMismatch::destination: (u32, u32) +pub j2k_core::passthrough::PassthroughRejectReason::DimensionsMismatch::source: (u32, u32) +pub j2k_core::passthrough::PassthroughRejectReason::EmptyPayload +pub j2k_core::passthrough::PassthroughRejectReason::PayloadKindMismatch +pub j2k_core::passthrough::PassthroughRejectReason::PayloadKindMismatch::destination: j2k_core::passthrough::CompressedPayloadKind +pub j2k_core::passthrough::PassthroughRejectReason::PayloadKindMismatch::source: j2k_core::passthrough::CompressedPayloadKind +pub j2k_core::passthrough::PassthroughRejectReason::TileLayoutMismatch +pub j2k_core::passthrough::PassthroughRejectReason::TileLayoutMismatch::destination: j2k_core::types::TileLayout +pub j2k_core::passthrough::PassthroughRejectReason::TileLayoutMismatch::source: core::option::Option +pub j2k_core::passthrough::PassthroughRejectReason::TransferSyntaxMismatch +pub j2k_core::passthrough::PassthroughRejectReason::TransferSyntaxMismatch::destination: j2k_core::passthrough::CompressedTransferSyntax +pub j2k_core::passthrough::PassthroughRejectReason::TransferSyntaxMismatch::source: j2k_core::passthrough::CompressedTransferSyntax +pub struct j2k_core::passthrough::PassthroughCandidate<'a> +impl<'a> j2k_core::passthrough::PassthroughCandidate<'a> +pub const fn j2k_core::passthrough::PassthroughCandidate<'a>::bytes(&self) -> &'a [u8] +pub fn j2k_core::passthrough::PassthroughCandidate<'a>::copy_bytes_if_eligible(&self, &j2k_core::passthrough::PassthroughRequirements) -> core::result::Result<&'a [u8], j2k_core::passthrough::PassthroughRejectReason> +pub fn j2k_core::passthrough::PassthroughCandidate<'a>::evaluate(&self, &j2k_core::passthrough::PassthroughRequirements) -> j2k_core::passthrough::PassthroughDecision<'a> +pub const fn j2k_core::passthrough::PassthroughCandidate<'a>::info(&self) -> &j2k_core::types::Info +pub const fn j2k_core::passthrough::PassthroughCandidate<'a>::new(&'a [u8], j2k_core::passthrough::CompressedTransferSyntax, j2k_core::passthrough::CompressedPayloadKind, j2k_core::types::Info) -> Self +pub const fn j2k_core::passthrough::PassthroughCandidate<'a>::payload_kind(&self) -> j2k_core::passthrough::CompressedPayloadKind +pub const fn j2k_core::passthrough::PassthroughCandidate<'a>::transfer_syntax(&self) -> j2k_core::passthrough::CompressedTransferSyntax +pub struct j2k_core::passthrough::PassthroughRequirements +pub j2k_core::passthrough::PassthroughRequirements::bit_depth: core::option::Option +pub j2k_core::passthrough::PassthroughRequirements::colorspace: core::option::Option +pub j2k_core::passthrough::PassthroughRequirements::components: core::option::Option +pub j2k_core::passthrough::PassthroughRequirements::dimensions: core::option::Option<(u32, u32)> +pub j2k_core::passthrough::PassthroughRequirements::payload_kind: j2k_core::passthrough::CompressedPayloadKind +pub j2k_core::passthrough::PassthroughRequirements::tile_layout: core::option::Option +pub j2k_core::passthrough::PassthroughRequirements::transfer_syntax: j2k_core::passthrough::CompressedTransferSyntax +impl j2k_core::passthrough::PassthroughRequirements +pub const fn j2k_core::passthrough::PassthroughRequirements::new(j2k_core::passthrough::CompressedTransferSyntax, j2k_core::passthrough::CompressedPayloadKind) -> Self +pub const fn j2k_core::passthrough::PassthroughRequirements::with_bit_depth(self, u8) -> Self +pub const fn j2k_core::passthrough::PassthroughRequirements::with_colorspace(self, j2k_core::types::Colorspace) -> Self +pub const fn j2k_core::passthrough::PassthroughRequirements::with_components(self, u8) -> Self +pub const fn j2k_core::passthrough::PassthroughRequirements::with_dimensions(self, (u32, u32)) -> Self +pub const fn j2k_core::passthrough::PassthroughRequirements::with_tile_layout(self, j2k_core::types::TileLayout) -> Self +pub mod j2k_core::pixel +#[non_exhaustive] pub enum j2k_core::pixel::PixelFormat +pub j2k_core::pixel::PixelFormat::Gray16 +pub j2k_core::pixel::PixelFormat::Gray8 +pub j2k_core::pixel::PixelFormat::Rgb16 +pub j2k_core::pixel::PixelFormat::Rgb8 +pub j2k_core::pixel::PixelFormat::Rgba16 +pub j2k_core::pixel::PixelFormat::Rgba8 +impl j2k_core::pixel::PixelFormat +pub const fn j2k_core::pixel::PixelFormat::bytes_per_pixel(self) -> usize +pub const fn j2k_core::pixel::PixelFormat::bytes_per_sample(self) -> usize +pub const fn j2k_core::pixel::PixelFormat::channels(self) -> usize +pub const fn j2k_core::pixel::PixelFormat::layout(self) -> j2k_core::pixel::PixelLayout +pub const fn j2k_core::pixel::PixelFormat::sample(self) -> j2k_core::sample::SampleType +#[non_exhaustive] pub enum j2k_core::pixel::PixelLayout +pub j2k_core::pixel::PixelLayout::Gray +pub j2k_core::pixel::PixelLayout::Rgb +pub j2k_core::pixel::PixelLayout::Rgba +pub mod j2k_core::row_sink +pub trait j2k_core::row_sink::RowSink +pub type j2k_core::row_sink::RowSink::Error: core::error::Error + core::marker::Send + core::marker::Sync + 'static +pub fn j2k_core::row_sink::RowSink::write_row(&mut self, u32, &[S]) -> core::result::Result<(), Self::Error> +pub mod j2k_core::sample +#[non_exhaustive] pub enum j2k_core::sample::SampleType +pub j2k_core::sample::SampleType::U16 +pub j2k_core::sample::SampleType::U8 +pub trait j2k_core::sample::Sample: core::marker::Copy + core::default::Default + core::marker::Send + core::marker::Sync + 'static +pub const j2k_core::sample::Sample::BITS: u8 +pub const j2k_core::sample::Sample::TYPE: j2k_core::sample::SampleType +impl j2k_core::sample::Sample for u16 +pub const u16::BITS: u8 +pub const u16::TYPE: j2k_core::sample::SampleType +impl j2k_core::sample::Sample for u8 +pub const u8::BITS: u8 +pub const u8::TYPE: j2k_core::sample::SampleType +pub mod j2k_core::scale +#[non_exhaustive] pub enum j2k_core::scale::Downscale +pub j2k_core::scale::Downscale::Eighth +pub j2k_core::scale::Downscale::Half +pub j2k_core::scale::Downscale::None +pub j2k_core::scale::Downscale::Quarter +impl j2k_core::scale::Downscale +pub const fn j2k_core::scale::Downscale::denominator(self) -> u32 +pub const fn j2k_core::scale::Downscale::output_block_size(self) -> u32 +pub mod j2k_core::scratch +pub trait j2k_core::scratch::ScratchPool: core::marker::Send +pub fn j2k_core::scratch::ScratchPool::bytes_allocated(&self) -> usize +pub fn j2k_core::scratch::ScratchPool::reset(&mut self) +pub mod j2k_core::traits +pub enum j2k_core::traits::DecodeRowsError where D: core::error::Error + 'static, E: core::error::Error + 'static +pub j2k_core::traits::DecodeRowsError::Decode(D) +pub j2k_core::traits::DecodeRowsError::Sink(E) +pub struct j2k_core::traits::ReadySubmission(_) +impl j2k_core::traits::ReadySubmission +pub fn j2k_core::traits::ReadySubmission::from_result(core::result::Result) -> Self +impl j2k_core::traits::DeviceSubmission for j2k_core::traits::ReadySubmission +pub type j2k_core::traits::ReadySubmission::Error = E +pub type j2k_core::traits::ReadySubmission::Output = T +pub fn j2k_core::traits::ReadySubmission::wait(self) -> core::result::Result +pub trait j2k_core::traits::DeviceSubmission +pub type j2k_core::traits::DeviceSubmission::Error +pub type j2k_core::traits::DeviceSubmission::Output +pub fn j2k_core::traits::DeviceSubmission::wait(self) -> core::result::Result +impl j2k_core::traits::DeviceSubmission for j2k_core::traits::ReadySubmission +pub type j2k_core::traits::ReadySubmission::Error = E +pub type j2k_core::traits::ReadySubmission::Output = T +pub fn j2k_core::traits::ReadySubmission::wait(self) -> core::result::Result +pub trait j2k_core::traits::DeviceSubmitSession +pub fn j2k_core::traits::DeviceSubmitSession::record_submit(&mut self) +pub trait j2k_core::traits::DeviceSurface +pub fn j2k_core::traits::DeviceSurface::backend_kind(&self) -> j2k_core::backend::BackendKind +pub fn j2k_core::traits::DeviceSurface::byte_len(&self) -> usize +pub fn j2k_core::traits::DeviceSurface::dimensions(&self) -> (u32, u32) +pub fn j2k_core::traits::DeviceSurface::execution_stats(&self) -> j2k_core::accelerator::ExecutionStats +pub fn j2k_core::traits::DeviceSurface::memory_range(&self) -> core::option::Option +pub fn j2k_core::traits::DeviceSurface::pixel_format(&self) -> j2k_core::pixel::PixelFormat +pub fn j2k_core::traits::DeviceSurface::residency(&self) -> j2k_core::accelerator::SurfaceResidency +pub trait j2k_core::traits::ImageCodec +pub type j2k_core::traits::ImageCodec::Error: j2k_core::error::CodecError +pub type j2k_core::traits::ImageCodec::Pool: j2k_core::scratch::ScratchPool +pub type j2k_core::traits::ImageCodec::Warning: core::fmt::Debug + core::fmt::Display + core::marker::Send + core::marker::Sync + 'static +pub trait j2k_core::traits::ImageDecode<'a>: j2k_core::traits::ImageCodec + core::marker::Sized + 'a +pub type j2k_core::traits::ImageDecode::View: 'a +pub fn j2k_core::traits::ImageDecode::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_core::traits::ImageDecode::decode_into_with_scratch(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_core::traits::ImageDecode::decode_region_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k_core::traits::ImageDecode::decode_region_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_core::traits::ImageDecode::decode_request_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::DecodeRequest) -> core::result::Result, Self::Error> +pub fn j2k_core::traits::ImageDecode::decode_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_core::traits::ImageDecode::from_view(Self::View) -> core::result::Result +pub fn j2k_core::traits::ImageDecode::inspect(&'a [u8]) -> core::result::Result +pub fn j2k_core::traits::ImageDecode::parse(&'a [u8]) -> core::result::Result +pub trait j2k_core::traits::ImageDecodeDevice<'a>: j2k_core::traits::ImageDecode<'a> +pub type j2k_core::traits::ImageDecodeDevice::DeviceSurface: j2k_core::traits::DeviceSurface +pub fn j2k_core::traits::ImageDecodeDevice::decode_region_scaled_to_device(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result<>::DeviceSurface, Self::Error> where Self: j2k_core::traits::ImageDecodeSubmit<'a, DeviceSurface = >::DeviceSurface> +pub fn j2k_core::traits::ImageDecodeDevice::decode_region_to_device(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result<>::DeviceSurface, Self::Error> where Self: j2k_core::traits::ImageDecodeSubmit<'a, DeviceSurface = >::DeviceSurface> +pub fn j2k_core::traits::ImageDecodeDevice::decode_scaled_to_device(&mut self, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result<>::DeviceSurface, Self::Error> where Self: j2k_core::traits::ImageDecodeSubmit<'a, DeviceSurface = >::DeviceSurface> +pub fn j2k_core::traits::ImageDecodeDevice::decode_to_device(&mut self, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result<>::DeviceSurface, Self::Error> where Self: j2k_core::traits::ImageDecodeSubmit<'a, DeviceSurface = >::DeviceSurface> +pub trait j2k_core::traits::ImageDecodeRows<'a, S: j2k_core::sample::Sample>: j2k_core::traits::ImageDecode<'a> +pub fn j2k_core::traits::ImageDecodeRows::decode_rows>(&mut self, &mut R) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +pub trait j2k_core::traits::ImageDecodeSubmit<'a>: j2k_core::traits::ImageDecode<'a> +pub type j2k_core::traits::ImageDecodeSubmit::DeviceSurface: j2k_core::traits::DeviceSurface +pub type j2k_core::traits::ImageDecodeSubmit::Session: core::default::Default + core::marker::Send +pub type j2k_core::traits::ImageDecodeSubmit::SubmittedSurface: j2k_core::traits::DeviceSubmission +pub fn j2k_core::traits::ImageDecodeSubmit::submit_region_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_core::traits::ImageDecodeSubmit::submit_region_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_core::traits::ImageDecodeSubmit::submit_request_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest, j2k_core::types::DecodeRequest) -> core::result::Result +pub fn j2k_core::traits::ImageDecodeSubmit::submit_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_core::traits::ImageDecodeSubmit::submit_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub trait j2k_core::traits::TileBatchDecode: j2k_core::traits::ImageCodec +pub type j2k_core::traits::TileBatchDecode::Context: j2k_core::context::CodecContext +pub fn j2k_core::traits::TileBatchDecode::decode_tile<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &'a [u8], &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_core::traits::TileBatchDecode::decode_tile_region<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &'a [u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k_core::traits::TileBatchDecode::decode_tile_region_scaled<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &'a [u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_core::traits::TileBatchDecode::decode_tile_request<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &'a [u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::DecodeRequest) -> core::result::Result, Self::Error> +pub fn j2k_core::traits::TileBatchDecode::decode_tile_scaled<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &'a [u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub trait j2k_core::traits::TileBatchDecodeDevice: j2k_core::traits::ImageCodec +pub type j2k_core::traits::TileBatchDecodeDevice::Context: j2k_core::context::CodecContext +pub type j2k_core::traits::TileBatchDecodeDevice::DeviceSurface: j2k_core::traits::DeviceSurface +pub fn j2k_core::traits::TileBatchDecodeDevice::decode_tile_region_scaled_to_device<'a>(&mut j2k_core::context::DecoderContext<::Context>, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result<::DeviceSurface, Self::Error> where Self: j2k_core::traits::TileBatchDecodeSubmit::Context, DeviceSurface = ::DeviceSurface> +pub fn j2k_core::traits::TileBatchDecodeDevice::decode_tile_region_to_device<'a>(&mut j2k_core::context::DecoderContext<::Context>, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result<::DeviceSurface, Self::Error> where Self: j2k_core::traits::TileBatchDecodeSubmit::Context, DeviceSurface = ::DeviceSurface> +pub fn j2k_core::traits::TileBatchDecodeDevice::decode_tile_scaled_to_device<'a>(&mut j2k_core::context::DecoderContext<::Context>, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result<::DeviceSurface, Self::Error> where Self: j2k_core::traits::TileBatchDecodeSubmit::Context, DeviceSurface = ::DeviceSurface> +pub fn j2k_core::traits::TileBatchDecodeDevice::decode_tile_to_device<'a>(&mut j2k_core::context::DecoderContext<::Context>, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result<::DeviceSurface, Self::Error> where Self: j2k_core::traits::TileBatchDecodeSubmit::Context, DeviceSurface = ::DeviceSurface> +pub trait j2k_core::traits::TileBatchDecodeManyDevice: j2k_core::traits::ImageCodec +pub type j2k_core::traits::TileBatchDecodeManyDevice::Context: j2k_core::context::CodecContext +pub type j2k_core::traits::TileBatchDecodeManyDevice::DeviceSurface: j2k_core::traits::DeviceSurface +pub fn j2k_core::traits::TileBatchDecodeManyDevice::decode_tiles_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[&[u8]], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result, Self::Error> +pub trait j2k_core::traits::TileBatchDecodeSubmit: j2k_core::traits::ImageCodec +pub type j2k_core::traits::TileBatchDecodeSubmit::Context: j2k_core::context::CodecContext +pub type j2k_core::traits::TileBatchDecodeSubmit::DeviceSurface: j2k_core::traits::DeviceSurface +pub type j2k_core::traits::TileBatchDecodeSubmit::Session: core::default::Default + core::marker::Send +pub type j2k_core::traits::TileBatchDecodeSubmit::SubmittedSurface: j2k_core::traits::DeviceSubmission +pub fn j2k_core::traits::TileBatchDecodeSubmit::submit_tile_region_scaled_to_device<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_core::traits::TileBatchDecodeSubmit::submit_tile_region_to_device<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_core::traits::TileBatchDecodeSubmit::submit_tile_request_to_device<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest, j2k_core::types::DecodeRequest) -> core::result::Result +pub fn j2k_core::traits::TileBatchDecodeSubmit::submit_tile_scaled_to_device<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_core::traits::TileBatchDecodeSubmit::submit_tile_to_device<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub trait j2k_core::traits::TileDecompress +pub type j2k_core::traits::TileDecompress::Error: j2k_core::error::CodecError +pub type j2k_core::traits::TileDecompress::Pool: j2k_core::scratch::ScratchPool +pub fn j2k_core::traits::TileDecompress::decompress_into(&mut Self::Pool, &[u8], &mut [u8]) -> core::result::Result +pub fn j2k_core::traits::TileDecompress::expected_size(&[u8]) -> core::result::Result, Self::Error> +pub fn j2k_core::traits::submit_ready_device(&mut S, impl core::ops::function::FnOnce(&mut S) -> core::result::Result) -> j2k_core::traits::ReadySubmission where S: j2k_core::traits::DeviceSubmitSession + ?core::marker::Sized +pub mod j2k_core::types +#[non_exhaustive] pub enum j2k_core::types::Colorspace +pub j2k_core::types::Colorspace::Cmyk +pub j2k_core::types::Colorspace::Grayscale +pub j2k_core::types::Colorspace::IccTagged +pub j2k_core::types::Colorspace::Ict +pub j2k_core::types::Colorspace::Rct +pub j2k_core::types::Colorspace::Rgb +pub j2k_core::types::Colorspace::SGray +pub j2k_core::types::Colorspace::SRgb +pub j2k_core::types::Colorspace::YCbCr +pub j2k_core::types::Colorspace::Ycck +#[non_exhaustive] pub enum j2k_core::types::WarningKind +pub j2k_core::types::WarningKind::MinorCompliance +pub j2k_core::types::WarningKind::NonFatalTruncation +pub j2k_core::types::WarningKind::UnusualFeature +pub struct j2k_core::types::CodedUnitLayout +pub j2k_core::types::CodedUnitLayout::unit_height: u32 +pub j2k_core::types::CodedUnitLayout::unit_width: u32 +pub j2k_core::types::CodedUnitLayout::units_x: u32 +pub j2k_core::types::CodedUnitLayout::units_y: u32 +impl j2k_core::types::CodedUnitLayout +pub const fn j2k_core::types::CodedUnitLayout::unit_count(&self) -> u32 +pub struct j2k_core::types::DecodeOutcome +pub j2k_core::types::DecodeOutcome::decoded: j2k_core::types::Rect +pub j2k_core::types::DecodeOutcome::warnings: alloc::vec::Vec +impl j2k_core::types::DecodeOutcome +pub fn j2k_core::types::DecodeOutcome::new(j2k_core::types::Rect, alloc::vec::Vec) -> Self +pub struct j2k_core::types::DecodeRequest +pub j2k_core::types::DecodeRequest::roi: core::option::Option +pub j2k_core::types::DecodeRequest::scale: j2k_core::scale::Downscale +impl j2k_core::types::DecodeRequest +pub const j2k_core::types::DecodeRequest::FULL: Self +pub fn j2k_core::types::DecodeRequest::decoded_rect(self, (u32, u32)) -> j2k_core::types::Rect +pub const fn j2k_core::types::DecodeRequest::full() -> Self +pub const fn j2k_core::types::DecodeRequest::is_full_resolution_full_image(self) -> bool +pub const fn j2k_core::types::DecodeRequest::region(j2k_core::types::Rect) -> Self +pub const fn j2k_core::types::DecodeRequest::region_scaled(j2k_core::types::Rect, j2k_core::scale::Downscale) -> Self +pub const fn j2k_core::types::DecodeRequest::scaled(j2k_core::scale::Downscale) -> Self +pub struct j2k_core::types::Info +pub j2k_core::types::Info::bit_depth: u8 +pub j2k_core::types::Info::coded_unit_layout: core::option::Option +pub j2k_core::types::Info::colorspace: j2k_core::types::Colorspace +pub j2k_core::types::Info::components: u8 +pub j2k_core::types::Info::dimensions: (u32, u32) +pub j2k_core::types::Info::resolution_levels: u8 +pub j2k_core::types::Info::restart_interval: core::option::Option +pub j2k_core::types::Info::tile_layout: core::option::Option +pub struct j2k_core::types::Rect +pub j2k_core::types::Rect::h: u32 +pub j2k_core::types::Rect::w: u32 +pub j2k_core::types::Rect::x: u32 +pub j2k_core::types::Rect::y: u32 +impl j2k_core::types::Rect +pub const fn j2k_core::types::Rect::full((u32, u32)) -> Self +pub fn j2k_core::types::Rect::is_within(&self, (u32, u32)) -> bool +pub fn j2k_core::types::Rect::scaled_covering(&self, j2k_core::scale::Downscale) -> Self +pub struct j2k_core::types::TileLayout +pub j2k_core::types::TileLayout::tile_height: u32 +pub j2k_core::types::TileLayout::tile_width: u32 +pub j2k_core::types::TileLayout::tiles_x: u32 +pub j2k_core::types::TileLayout::tiles_y: u32 +pub enum j2k_core::BackendFailureKind +pub j2k_core::BackendFailureKind::InvalidInput +pub j2k_core::BackendFailureKind::Kernel +pub j2k_core::BackendFailureKind::OutOfMemory +pub j2k_core::BackendFailureKind::Runtime +pub j2k_core::BackendFailureKind::StatePoisoned +pub j2k_core::BackendFailureKind::Unavailable +pub j2k_core::BackendFailureKind::Unsupported +pub enum j2k_core::BackendKind +pub j2k_core::BackendKind::Cpu +pub j2k_core::BackendKind::Cuda +pub j2k_core::BackendKind::Metal +pub enum j2k_core::BackendRequest +pub j2k_core::BackendRequest::Auto +pub j2k_core::BackendRequest::Cpu +pub j2k_core::BackendRequest::Cuda +pub j2k_core::BackendRequest::Metal +impl j2k_core::backend::BackendRequest +pub const j2k_core::backend::BackendRequest::ACCELERATED: Self +pub const j2k_core::backend::BackendRequest::CPU_ONLY: Self +pub const j2k_core::backend::BackendRequest::STRICT_CUDA: Self +pub const j2k_core::backend::BackendRequest::STRICT_METAL: Self +pub enum j2k_core::BufferError +pub j2k_core::BufferError::AllocationTooLarge +pub j2k_core::BufferError::AllocationTooLarge::cap: usize +pub j2k_core::BufferError::AllocationTooLarge::requested: usize +pub j2k_core::BufferError::AllocationTooLarge::what: &'static str +pub j2k_core::BufferError::InputTooSmall +pub j2k_core::BufferError::InputTooSmall::have: usize +pub j2k_core::BufferError::InputTooSmall::required: usize +pub j2k_core::BufferError::OutputTooSmall +pub j2k_core::BufferError::OutputTooSmall::have: usize +pub j2k_core::BufferError::OutputTooSmall::required: usize +pub j2k_core::BufferError::SampleTypeMismatch +pub j2k_core::BufferError::SampleTypeMismatch::fmt: j2k_core::pixel::PixelFormat +pub j2k_core::BufferError::SampleTypeMismatch::sample_type: j2k_core::sample::SampleType +pub j2k_core::BufferError::SizeOverflow +pub j2k_core::BufferError::SizeOverflow::what: &'static str +pub j2k_core::BufferError::StrideNotAligned +pub j2k_core::BufferError::StrideNotAligned::align: usize +pub j2k_core::BufferError::StrideNotAligned::stride: usize +pub j2k_core::BufferError::StrideTooSmall +pub j2k_core::BufferError::StrideTooSmall::row_bytes: usize +pub j2k_core::BufferError::StrideTooSmall::stride: usize +#[non_exhaustive] pub enum j2k_core::Colorspace +pub j2k_core::Colorspace::Cmyk +pub j2k_core::Colorspace::Grayscale +pub j2k_core::Colorspace::IccTagged +pub j2k_core::Colorspace::Ict +pub j2k_core::Colorspace::Rct +pub j2k_core::Colorspace::Rgb +pub j2k_core::Colorspace::SGray +pub j2k_core::Colorspace::SRgb +pub j2k_core::Colorspace::YCbCr +pub j2k_core::Colorspace::Ycck +#[non_exhaustive] pub enum j2k_core::CompressedPayloadKind +pub j2k_core::CompressedPayloadKind::Jp2File +pub j2k_core::CompressedPayloadKind::Jpeg2000Codestream +pub j2k_core::CompressedPayloadKind::JpegInterchange +#[non_exhaustive] pub enum j2k_core::CompressedTransferSyntax +pub j2k_core::CompressedTransferSyntax::HtJpeg2000Lossless +pub j2k_core::CompressedTransferSyntax::HtJpeg2000Lossy +pub j2k_core::CompressedTransferSyntax::Jpeg2000Lossless +pub j2k_core::CompressedTransferSyntax::Jpeg2000Lossy +pub j2k_core::CompressedTransferSyntax::JpegBaseline8 +pub j2k_core::CompressedTransferSyntax::JpegExtendedSequential +impl j2k_core::passthrough::CompressedTransferSyntax +pub const fn j2k_core::passthrough::CompressedTransferSyntax::is_jpeg2000_family(self) -> bool +pub const fn j2k_core::passthrough::CompressedTransferSyntax::is_lossless(self) -> bool +pub enum j2k_core::DecodeRowsError where D: core::error::Error + 'static, E: core::error::Error + 'static +pub j2k_core::DecodeRowsError::Decode(D) +pub j2k_core::DecodeRowsError::Sink(E) +#[non_exhaustive] pub enum j2k_core::Downscale +pub j2k_core::Downscale::Eighth +pub j2k_core::Downscale::Half +pub j2k_core::Downscale::None +pub j2k_core::Downscale::Quarter +impl j2k_core::scale::Downscale +pub const fn j2k_core::scale::Downscale::denominator(self) -> u32 +pub const fn j2k_core::scale::Downscale::output_block_size(self) -> u32 +pub enum j2k_core::InputError +pub j2k_core::InputError::TooShort +pub j2k_core::InputError::TooShort::have: usize +pub j2k_core::InputError::TooShort::need: usize +pub j2k_core::InputError::TruncatedAt +pub j2k_core::InputError::TruncatedAt::offset: usize +pub j2k_core::InputError::TruncatedAt::segment: &'static str +pub enum j2k_core::PassthroughDecision<'a> +pub j2k_core::PassthroughDecision::Copy +pub j2k_core::PassthroughDecision::Copy::bytes: &'a [u8] +pub j2k_core::PassthroughDecision::Transcode +pub j2k_core::PassthroughDecision::Transcode::reason: j2k_core::passthrough::PassthroughRejectReason +#[non_exhaustive] pub enum j2k_core::PassthroughRejectReason +pub j2k_core::PassthroughRejectReason::BitDepthMismatch +pub j2k_core::PassthroughRejectReason::BitDepthMismatch::destination: u8 +pub j2k_core::PassthroughRejectReason::BitDepthMismatch::source: u8 +pub j2k_core::PassthroughRejectReason::ColorspaceMismatch +pub j2k_core::PassthroughRejectReason::ColorspaceMismatch::destination: j2k_core::types::Colorspace +pub j2k_core::PassthroughRejectReason::ColorspaceMismatch::source: j2k_core::types::Colorspace +pub j2k_core::PassthroughRejectReason::ComponentsMismatch +pub j2k_core::PassthroughRejectReason::ComponentsMismatch::destination: u8 +pub j2k_core::PassthroughRejectReason::ComponentsMismatch::source: u8 +pub j2k_core::PassthroughRejectReason::DimensionsMismatch +pub j2k_core::PassthroughRejectReason::DimensionsMismatch::destination: (u32, u32) +pub j2k_core::PassthroughRejectReason::DimensionsMismatch::source: (u32, u32) +pub j2k_core::PassthroughRejectReason::EmptyPayload +pub j2k_core::PassthroughRejectReason::PayloadKindMismatch +pub j2k_core::PassthroughRejectReason::PayloadKindMismatch::destination: j2k_core::passthrough::CompressedPayloadKind +pub j2k_core::PassthroughRejectReason::PayloadKindMismatch::source: j2k_core::passthrough::CompressedPayloadKind +pub j2k_core::PassthroughRejectReason::TileLayoutMismatch +pub j2k_core::PassthroughRejectReason::TileLayoutMismatch::destination: j2k_core::types::TileLayout +pub j2k_core::PassthroughRejectReason::TileLayoutMismatch::source: core::option::Option +pub j2k_core::PassthroughRejectReason::TransferSyntaxMismatch +pub j2k_core::PassthroughRejectReason::TransferSyntaxMismatch::destination: j2k_core::passthrough::CompressedTransferSyntax +pub j2k_core::PassthroughRejectReason::TransferSyntaxMismatch::source: j2k_core::passthrough::CompressedTransferSyntax +#[non_exhaustive] pub enum j2k_core::PixelFormat +pub j2k_core::PixelFormat::Gray16 +pub j2k_core::PixelFormat::Gray8 +pub j2k_core::PixelFormat::Rgb16 +pub j2k_core::PixelFormat::Rgb8 +pub j2k_core::PixelFormat::Rgba16 +pub j2k_core::PixelFormat::Rgba8 +impl j2k_core::pixel::PixelFormat +pub const fn j2k_core::pixel::PixelFormat::bytes_per_pixel(self) -> usize +pub const fn j2k_core::pixel::PixelFormat::bytes_per_sample(self) -> usize +pub const fn j2k_core::pixel::PixelFormat::channels(self) -> usize +pub const fn j2k_core::pixel::PixelFormat::layout(self) -> j2k_core::pixel::PixelLayout +pub const fn j2k_core::pixel::PixelFormat::sample(self) -> j2k_core::sample::SampleType +#[non_exhaustive] pub enum j2k_core::PixelLayout +pub j2k_core::PixelLayout::Gray +pub j2k_core::PixelLayout::Rgb +pub j2k_core::PixelLayout::Rgba +#[non_exhaustive] pub enum j2k_core::SampleType +pub j2k_core::SampleType::U16 +pub j2k_core::SampleType::U8 +#[non_exhaustive] pub enum j2k_core::SurfaceResidency +pub j2k_core::SurfaceResidency::CpuStagedCudaUpload +pub j2k_core::SurfaceResidency::CpuStagedMetalUpload +pub j2k_core::SurfaceResidency::CudaResidentDecode +pub j2k_core::SurfaceResidency::Device(j2k_core::backend::BackendKind) +pub j2k_core::SurfaceResidency::Host +pub j2k_core::SurfaceResidency::MetalResidentDecode +pub j2k_core::SurfaceResidency::Shared(j2k_core::backend::BackendKind) +impl j2k_core::accelerator::SurfaceResidency +pub const fn j2k_core::accelerator::SurfaceResidency::for_backend(j2k_core::backend::BackendKind) -> Self +#[non_exhaustive] pub enum j2k_core::WarningKind +pub j2k_core::WarningKind::MinorCompliance +pub j2k_core::WarningKind::NonFatalTruncation +pub j2k_core::WarningKind::UnusualFeature +pub struct j2k_core::BackendCapabilities +pub j2k_core::BackendCapabilities::cpu: j2k_core::backend::CpuFeatures +pub j2k_core::BackendCapabilities::cuda: bool +pub j2k_core::BackendCapabilities::metal: bool +impl j2k_core::backend::BackendCapabilities +pub fn j2k_core::backend::BackendCapabilities::compile_time_defaults() -> Self +pub const fn j2k_core::backend::BackendCapabilities::first_available_accelerator(self) -> core::option::Option +pub fn j2k_core::backend::BackendCapabilities::resolve(self, j2k_core::backend::BackendRequest) -> core::option::Option +pub const fn j2k_core::backend::BackendCapabilities::supports(self, j2k_core::backend::BackendRequest) -> bool +pub struct j2k_core::CacheStats +pub j2k_core::CacheStats::evictions: u64 +pub j2k_core::CacheStats::hits: u64 +pub j2k_core::CacheStats::misses: u64 +pub j2k_core::CacheStats::occupied_slots: u64 +impl j2k_core::context::CacheStats +pub const fn j2k_core::context::CacheStats::new(u64, u64) -> Self +pub const fn j2k_core::context::CacheStats::with_slots(u64, u64, u64, u64) -> Self +pub struct j2k_core::CodedUnitLayout +pub j2k_core::CodedUnitLayout::unit_height: u32 +pub j2k_core::CodedUnitLayout::unit_width: u32 +pub j2k_core::CodedUnitLayout::units_x: u32 +pub j2k_core::CodedUnitLayout::units_y: u32 +impl j2k_core::types::CodedUnitLayout +pub const fn j2k_core::types::CodedUnitLayout::unit_count(&self) -> u32 +pub struct j2k_core::CpuFeatures +pub j2k_core::CpuFeatures::avx2: bool +pub j2k_core::CpuFeatures::neon: bool +pub j2k_core::CpuFeatures::sse41: bool +impl j2k_core::backend::CpuFeatures +pub fn j2k_core::backend::CpuFeatures::detect() -> Self +pub struct j2k_core::DecodeOutcome +pub j2k_core::DecodeOutcome::decoded: j2k_core::types::Rect +pub j2k_core::DecodeOutcome::warnings: alloc::vec::Vec +impl j2k_core::types::DecodeOutcome +pub fn j2k_core::types::DecodeOutcome::new(j2k_core::types::Rect, alloc::vec::Vec) -> Self +pub struct j2k_core::DecodeRequest +pub j2k_core::DecodeRequest::roi: core::option::Option +pub j2k_core::DecodeRequest::scale: j2k_core::scale::Downscale +impl j2k_core::types::DecodeRequest +pub const j2k_core::types::DecodeRequest::FULL: Self +pub fn j2k_core::types::DecodeRequest::decoded_rect(self, (u32, u32)) -> j2k_core::types::Rect +pub const fn j2k_core::types::DecodeRequest::full() -> Self +pub const fn j2k_core::types::DecodeRequest::is_full_resolution_full_image(self) -> bool +pub const fn j2k_core::types::DecodeRequest::region(j2k_core::types::Rect) -> Self +pub const fn j2k_core::types::DecodeRequest::region_scaled(j2k_core::types::Rect, j2k_core::scale::Downscale) -> Self +pub const fn j2k_core::types::DecodeRequest::scaled(j2k_core::scale::Downscale) -> Self +pub struct j2k_core::DecoderContext +impl j2k_core::context::DecoderContext +pub fn j2k_core::context::DecoderContext::cache_stats(&self) -> j2k_core::context::CacheStats +pub fn j2k_core::context::DecoderContext::clear(&mut self) +pub fn j2k_core::context::DecoderContext::codec(&self) -> &C +pub fn j2k_core::context::DecoderContext::codec_mut(&mut self) -> &mut C +pub fn j2k_core::context::DecoderContext::into_inner(self) -> C +pub fn j2k_core::context::DecoderContext::new() -> Self +pub struct j2k_core::DeviceMemoryRange +pub j2k_core::DeviceMemoryRange::allocation: u64 +pub j2k_core::DeviceMemoryRange::backend: j2k_core::backend::BackendKind +pub j2k_core::DeviceMemoryRange::len: usize +pub j2k_core::DeviceMemoryRange::offset: usize +impl j2k_core::accelerator::DeviceMemoryRange +pub const fn j2k_core::accelerator::DeviceMemoryRange::new(j2k_core::backend::BackendKind, u64, usize, usize) -> Self +pub struct j2k_core::ExecutionStats +pub j2k_core::ExecutionStats::device_us: u128 +pub j2k_core::ExecutionStats::kernel_dispatches: u64 +pub j2k_core::ExecutionStats::readback_bytes: u64 +pub j2k_core::ExecutionStats::submissions: u64 +pub j2k_core::ExecutionStats::upload_bytes: u64 +impl j2k_core::accelerator::ExecutionStats +pub const fn j2k_core::accelerator::ExecutionStats::new() -> Self +pub const fn j2k_core::accelerator::ExecutionStats::saturating_add(self, Self) -> Self +pub struct j2k_core::Info +pub j2k_core::Info::bit_depth: u8 +pub j2k_core::Info::coded_unit_layout: core::option::Option +pub j2k_core::Info::colorspace: j2k_core::types::Colorspace +pub j2k_core::Info::components: u8 +pub j2k_core::Info::dimensions: (u32, u32) +pub j2k_core::Info::resolution_levels: u8 +pub j2k_core::Info::restart_interval: core::option::Option +pub j2k_core::Info::tile_layout: core::option::Option +pub struct j2k_core::NotImplemented +pub j2k_core::NotImplemented::what: &'static str +pub struct j2k_core::PassthroughCandidate<'a> +impl<'a> j2k_core::passthrough::PassthroughCandidate<'a> +pub const fn j2k_core::passthrough::PassthroughCandidate<'a>::bytes(&self) -> &'a [u8] +pub fn j2k_core::passthrough::PassthroughCandidate<'a>::copy_bytes_if_eligible(&self, &j2k_core::passthrough::PassthroughRequirements) -> core::result::Result<&'a [u8], j2k_core::passthrough::PassthroughRejectReason> +pub fn j2k_core::passthrough::PassthroughCandidate<'a>::evaluate(&self, &j2k_core::passthrough::PassthroughRequirements) -> j2k_core::passthrough::PassthroughDecision<'a> +pub const fn j2k_core::passthrough::PassthroughCandidate<'a>::info(&self) -> &j2k_core::types::Info +pub const fn j2k_core::passthrough::PassthroughCandidate<'a>::new(&'a [u8], j2k_core::passthrough::CompressedTransferSyntax, j2k_core::passthrough::CompressedPayloadKind, j2k_core::types::Info) -> Self +pub const fn j2k_core::passthrough::PassthroughCandidate<'a>::payload_kind(&self) -> j2k_core::passthrough::CompressedPayloadKind +pub const fn j2k_core::passthrough::PassthroughCandidate<'a>::transfer_syntax(&self) -> j2k_core::passthrough::CompressedTransferSyntax +pub struct j2k_core::PassthroughRequirements +pub j2k_core::PassthroughRequirements::bit_depth: core::option::Option +pub j2k_core::PassthroughRequirements::colorspace: core::option::Option +pub j2k_core::PassthroughRequirements::components: core::option::Option +pub j2k_core::PassthroughRequirements::dimensions: core::option::Option<(u32, u32)> +pub j2k_core::PassthroughRequirements::payload_kind: j2k_core::passthrough::CompressedPayloadKind +pub j2k_core::PassthroughRequirements::tile_layout: core::option::Option +pub j2k_core::PassthroughRequirements::transfer_syntax: j2k_core::passthrough::CompressedTransferSyntax +impl j2k_core::passthrough::PassthroughRequirements +pub const fn j2k_core::passthrough::PassthroughRequirements::new(j2k_core::passthrough::CompressedTransferSyntax, j2k_core::passthrough::CompressedPayloadKind) -> Self +pub const fn j2k_core::passthrough::PassthroughRequirements::with_bit_depth(self, u8) -> Self +pub const fn j2k_core::passthrough::PassthroughRequirements::with_colorspace(self, j2k_core::types::Colorspace) -> Self +pub const fn j2k_core::passthrough::PassthroughRequirements::with_components(self, u8) -> Self +pub const fn j2k_core::passthrough::PassthroughRequirements::with_dimensions(self, (u32, u32)) -> Self +pub const fn j2k_core::passthrough::PassthroughRequirements::with_tile_layout(self, j2k_core::types::TileLayout) -> Self +pub struct j2k_core::ReadySubmission(_) +impl j2k_core::traits::ReadySubmission +pub fn j2k_core::traits::ReadySubmission::from_result(core::result::Result) -> Self +impl j2k_core::traits::DeviceSubmission for j2k_core::traits::ReadySubmission +pub type j2k_core::traits::ReadySubmission::Error = E +pub type j2k_core::traits::ReadySubmission::Output = T +pub fn j2k_core::traits::ReadySubmission::wait(self) -> core::result::Result +pub struct j2k_core::Rect +pub j2k_core::Rect::h: u32 +pub j2k_core::Rect::w: u32 +pub j2k_core::Rect::x: u32 +pub j2k_core::Rect::y: u32 +impl j2k_core::types::Rect +pub const fn j2k_core::types::Rect::full((u32, u32)) -> Self +pub fn j2k_core::types::Rect::is_within(&self, (u32, u32)) -> bool +pub fn j2k_core::types::Rect::scaled_covering(&self, j2k_core::scale::Downscale) -> Self +pub struct j2k_core::TileBatchError +pub j2k_core::TileBatchError::index: usize +pub j2k_core::TileBatchError::source: E +impl core::error::Error for j2k_core::batch::TileBatchError +pub fn j2k_core::batch::TileBatchError::source(&self) -> core::option::Option<&(dyn core::error::Error + 'static)> +impl core::fmt::Display for j2k_core::batch::TileBatchError +pub fn j2k_core::batch::TileBatchError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_core::TileBatchOptions +pub j2k_core::TileBatchOptions::workers: core::option::Option +impl j2k_core::batch::TileBatchOptions +pub const fn j2k_core::batch::TileBatchOptions::new(core::option::Option) -> Self +pub struct j2k_core::TileDecodeJob<'i, 'o> +pub j2k_core::TileDecodeJob::input: &'i [u8] +pub j2k_core::TileDecodeJob::out: &'o mut [u8] +pub j2k_core::TileDecodeJob::stride: usize +pub struct j2k_core::TileLayout +pub j2k_core::TileLayout::tile_height: u32 +pub j2k_core::TileLayout::tile_width: u32 +pub j2k_core::TileLayout::tiles_x: u32 +pub j2k_core::TileLayout::tiles_y: u32 +pub struct j2k_core::TileRegionDecodeJob<'i, 'o> +pub j2k_core::TileRegionDecodeJob::input: &'i [u8] +pub j2k_core::TileRegionDecodeJob::out: &'o mut [u8] +pub j2k_core::TileRegionDecodeJob::roi: j2k_core::types::Rect +pub j2k_core::TileRegionDecodeJob::stride: usize +pub struct j2k_core::TileRegionScaledDecodeJob<'i, 'o> +pub j2k_core::TileRegionScaledDecodeJob::input: &'i [u8] +pub j2k_core::TileRegionScaledDecodeJob::out: &'o mut [u8] +pub j2k_core::TileRegionScaledDecodeJob::roi: j2k_core::types::Rect +pub j2k_core::TileRegionScaledDecodeJob::scale: j2k_core::scale::Downscale +pub j2k_core::TileRegionScaledDecodeJob::stride: usize +pub struct j2k_core::TileScaledDecodeJob<'i, 'o> +pub j2k_core::TileScaledDecodeJob::input: &'i [u8] +pub j2k_core::TileScaledDecodeJob::out: &'o mut [u8] +pub j2k_core::TileScaledDecodeJob::scale: j2k_core::scale::Downscale +pub j2k_core::TileScaledDecodeJob::stride: usize +pub struct j2k_core::Unsupported +pub j2k_core::Unsupported::what: &'static str +pub const j2k_core::DEFAULT_MAX_HOST_ALLOCATION_BYTES: usize +pub trait j2k_core::AcceleratorSession +pub fn j2k_core::AcceleratorSession::backend_kind(&self) -> j2k_core::backend::BackendKind +pub fn j2k_core::AcceleratorSession::execution_stats(&self) -> j2k_core::accelerator::ExecutionStats +pub trait j2k_core::CodecContext: core::default::Default + core::marker::Send +pub fn j2k_core::CodecContext::cache_stats(&self) -> j2k_core::context::CacheStats +pub fn j2k_core::CodecContext::clear(&mut self) +pub trait j2k_core::CodecError: core::error::Error + core::marker::Send + core::marker::Sync + 'static +pub fn j2k_core::CodecError::is_buffer_error(&self) -> bool +pub fn j2k_core::CodecError::is_not_implemented(&self) -> bool +pub fn j2k_core::CodecError::is_truncated(&self) -> bool +pub fn j2k_core::CodecError::is_unsupported(&self) -> bool +pub trait j2k_core::DeviceSubmission +pub type j2k_core::DeviceSubmission::Error +pub type j2k_core::DeviceSubmission::Output +pub fn j2k_core::DeviceSubmission::wait(self) -> core::result::Result +impl j2k_core::traits::DeviceSubmission for j2k_core::traits::ReadySubmission +pub type j2k_core::traits::ReadySubmission::Error = E +pub type j2k_core::traits::ReadySubmission::Output = T +pub fn j2k_core::traits::ReadySubmission::wait(self) -> core::result::Result +pub trait j2k_core::DeviceSubmitSession +pub fn j2k_core::DeviceSubmitSession::record_submit(&mut self) +pub trait j2k_core::DeviceSurface +pub fn j2k_core::DeviceSurface::backend_kind(&self) -> j2k_core::backend::BackendKind +pub fn j2k_core::DeviceSurface::byte_len(&self) -> usize +pub fn j2k_core::DeviceSurface::dimensions(&self) -> (u32, u32) +pub fn j2k_core::DeviceSurface::execution_stats(&self) -> j2k_core::accelerator::ExecutionStats +pub fn j2k_core::DeviceSurface::memory_range(&self) -> core::option::Option +pub fn j2k_core::DeviceSurface::pixel_format(&self) -> j2k_core::pixel::PixelFormat +pub fn j2k_core::DeviceSurface::residency(&self) -> j2k_core::accelerator::SurfaceResidency +pub unsafe trait j2k_core::GpuAbi: core::marker::Copy + 'static +pub const j2k_core::GpuAbi::NAME: &'static str +pub fn j2k_core::GpuAbi::as_bytes(&Self) -> &[u8] +pub fn j2k_core::GpuAbi::slice_as_bytes(&[Self]) -> &[u8] +pub fn j2k_core::GpuAbi::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for f32 +pub const f32::NAME: &'static str +pub fn f32::as_bytes(&Self) -> &[u8] +pub fn f32::slice_as_bytes(&[Self]) -> &[u8] +pub fn f32::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for f64 +pub const f64::NAME: &'static str +pub fn f64::as_bytes(&Self) -> &[u8] +pub fn f64::slice_as_bytes(&[Self]) -> &[u8] +pub fn f64::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for i16 +pub const i16::NAME: &'static str +pub fn i16::as_bytes(&Self) -> &[u8] +pub fn i16::slice_as_bytes(&[Self]) -> &[u8] +pub fn i16::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for i32 +pub const i32::NAME: &'static str +pub fn i32::as_bytes(&Self) -> &[u8] +pub fn i32::slice_as_bytes(&[Self]) -> &[u8] +pub fn i32::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for i64 +pub const i64::NAME: &'static str +pub fn i64::as_bytes(&Self) -> &[u8] +pub fn i64::slice_as_bytes(&[Self]) -> &[u8] +pub fn i64::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for i8 +pub const i8::NAME: &'static str +pub fn i8::as_bytes(&Self) -> &[u8] +pub fn i8::slice_as_bytes(&[Self]) -> &[u8] +pub fn i8::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for u16 +pub const u16::NAME: &'static str +pub fn u16::as_bytes(&Self) -> &[u8] +pub fn u16::slice_as_bytes(&[Self]) -> &[u8] +pub fn u16::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for u32 +pub const u32::NAME: &'static str +pub fn u32::as_bytes(&Self) -> &[u8] +pub fn u32::slice_as_bytes(&[Self]) -> &[u8] +pub fn u32::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for u64 +pub const u64::NAME: &'static str +pub fn u64::as_bytes(&Self) -> &[u8] +pub fn u64::slice_as_bytes(&[Self]) -> &[u8] +pub fn u64::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for u8 +pub const u8::NAME: &'static str +pub fn u8::as_bytes(&Self) -> &[u8] +pub fn u8::slice_as_bytes(&[Self]) -> &[u8] +pub fn u8::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +impl j2k_core::accelerator::GpuAbi for [T; N] where T: j2k_core::accelerator::GpuAbi +pub const [T; N]::NAME: &'static str +pub fn [T; N]::as_bytes(&Self) -> &[u8] +pub fn [T; N]::slice_as_bytes(&[Self]) -> &[u8] +pub fn [T; N]::slice_as_bytes_mut(&mut [Self]) -> &mut [u8] +pub trait j2k_core::ImageCodec +pub type j2k_core::ImageCodec::Error: j2k_core::error::CodecError +pub type j2k_core::ImageCodec::Pool: j2k_core::scratch::ScratchPool +pub type j2k_core::ImageCodec::Warning: core::fmt::Debug + core::fmt::Display + core::marker::Send + core::marker::Sync + 'static +pub trait j2k_core::ImageDecode<'a>: j2k_core::traits::ImageCodec + core::marker::Sized + 'a +pub type j2k_core::ImageDecode::View: 'a +pub fn j2k_core::ImageDecode::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_core::ImageDecode::decode_into_with_scratch(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_core::ImageDecode::decode_region_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k_core::ImageDecode::decode_region_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_core::ImageDecode::decode_request_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::DecodeRequest) -> core::result::Result, Self::Error> +pub fn j2k_core::ImageDecode::decode_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_core::ImageDecode::from_view(Self::View) -> core::result::Result +pub fn j2k_core::ImageDecode::inspect(&'a [u8]) -> core::result::Result +pub fn j2k_core::ImageDecode::parse(&'a [u8]) -> core::result::Result +pub trait j2k_core::ImageDecodeDevice<'a>: j2k_core::traits::ImageDecode<'a> +pub type j2k_core::ImageDecodeDevice::DeviceSurface: j2k_core::traits::DeviceSurface +pub fn j2k_core::ImageDecodeDevice::decode_region_scaled_to_device(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result<>::DeviceSurface, Self::Error> where Self: j2k_core::traits::ImageDecodeSubmit<'a, DeviceSurface = >::DeviceSurface> +pub fn j2k_core::ImageDecodeDevice::decode_region_to_device(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result<>::DeviceSurface, Self::Error> where Self: j2k_core::traits::ImageDecodeSubmit<'a, DeviceSurface = >::DeviceSurface> +pub fn j2k_core::ImageDecodeDevice::decode_scaled_to_device(&mut self, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result<>::DeviceSurface, Self::Error> where Self: j2k_core::traits::ImageDecodeSubmit<'a, DeviceSurface = >::DeviceSurface> +pub fn j2k_core::ImageDecodeDevice::decode_to_device(&mut self, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result<>::DeviceSurface, Self::Error> where Self: j2k_core::traits::ImageDecodeSubmit<'a, DeviceSurface = >::DeviceSurface> +pub trait j2k_core::ImageDecodeRows<'a, S: j2k_core::sample::Sample>: j2k_core::traits::ImageDecode<'a> +pub fn j2k_core::ImageDecodeRows::decode_rows>(&mut self, &mut R) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +pub trait j2k_core::ImageDecodeSubmit<'a>: j2k_core::traits::ImageDecode<'a> +pub type j2k_core::ImageDecodeSubmit::DeviceSurface: j2k_core::traits::DeviceSurface +pub type j2k_core::ImageDecodeSubmit::Session: core::default::Default + core::marker::Send +pub type j2k_core::ImageDecodeSubmit::SubmittedSurface: j2k_core::traits::DeviceSubmission +pub fn j2k_core::ImageDecodeSubmit::submit_region_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_core::ImageDecodeSubmit::submit_region_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_core::ImageDecodeSubmit::submit_request_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest, j2k_core::types::DecodeRequest) -> core::result::Result +pub fn j2k_core::ImageDecodeSubmit::submit_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_core::ImageDecodeSubmit::submit_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub trait j2k_core::RowSink +pub type j2k_core::RowSink::Error: core::error::Error + core::marker::Send + core::marker::Sync + 'static +pub fn j2k_core::RowSink::write_row(&mut self, u32, &[S]) -> core::result::Result<(), Self::Error> +pub trait j2k_core::Sample: core::marker::Copy + core::default::Default + core::marker::Send + core::marker::Sync + 'static +pub const j2k_core::Sample::BITS: u8 +pub const j2k_core::Sample::TYPE: j2k_core::sample::SampleType +impl j2k_core::sample::Sample for u16 +pub const u16::BITS: u8 +pub const u16::TYPE: j2k_core::sample::SampleType +impl j2k_core::sample::Sample for u8 +pub const u8::BITS: u8 +pub const u8::TYPE: j2k_core::sample::SampleType +pub trait j2k_core::ScratchPool: core::marker::Send +pub fn j2k_core::ScratchPool::bytes_allocated(&self) -> usize +pub fn j2k_core::ScratchPool::reset(&mut self) +pub trait j2k_core::TileBatchDecode: j2k_core::traits::ImageCodec +pub type j2k_core::TileBatchDecode::Context: j2k_core::context::CodecContext +pub fn j2k_core::TileBatchDecode::decode_tile<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &'a [u8], &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_core::TileBatchDecode::decode_tile_region<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &'a [u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k_core::TileBatchDecode::decode_tile_region_scaled<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &'a [u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_core::TileBatchDecode::decode_tile_request<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &'a [u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::DecodeRequest) -> core::result::Result, Self::Error> +pub fn j2k_core::TileBatchDecode::decode_tile_scaled<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &'a [u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub trait j2k_core::TileBatchDecodeDevice: j2k_core::traits::ImageCodec +pub type j2k_core::TileBatchDecodeDevice::Context: j2k_core::context::CodecContext +pub type j2k_core::TileBatchDecodeDevice::DeviceSurface: j2k_core::traits::DeviceSurface +pub fn j2k_core::TileBatchDecodeDevice::decode_tile_region_scaled_to_device<'a>(&mut j2k_core::context::DecoderContext<::Context>, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result<::DeviceSurface, Self::Error> where Self: j2k_core::traits::TileBatchDecodeSubmit::Context, DeviceSurface = ::DeviceSurface> +pub fn j2k_core::TileBatchDecodeDevice::decode_tile_region_to_device<'a>(&mut j2k_core::context::DecoderContext<::Context>, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result<::DeviceSurface, Self::Error> where Self: j2k_core::traits::TileBatchDecodeSubmit::Context, DeviceSurface = ::DeviceSurface> +pub fn j2k_core::TileBatchDecodeDevice::decode_tile_scaled_to_device<'a>(&mut j2k_core::context::DecoderContext<::Context>, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result<::DeviceSurface, Self::Error> where Self: j2k_core::traits::TileBatchDecodeSubmit::Context, DeviceSurface = ::DeviceSurface> +pub fn j2k_core::TileBatchDecodeDevice::decode_tile_to_device<'a>(&mut j2k_core::context::DecoderContext<::Context>, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result<::DeviceSurface, Self::Error> where Self: j2k_core::traits::TileBatchDecodeSubmit::Context, DeviceSurface = ::DeviceSurface> +pub trait j2k_core::TileBatchDecodeManyDevice: j2k_core::traits::ImageCodec +pub type j2k_core::TileBatchDecodeManyDevice::Context: j2k_core::context::CodecContext +pub type j2k_core::TileBatchDecodeManyDevice::DeviceSurface: j2k_core::traits::DeviceSurface +pub fn j2k_core::TileBatchDecodeManyDevice::decode_tiles_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[&[u8]], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result, Self::Error> +pub trait j2k_core::TileBatchDecodeSubmit: j2k_core::traits::ImageCodec +pub type j2k_core::TileBatchDecodeSubmit::Context: j2k_core::context::CodecContext +pub type j2k_core::TileBatchDecodeSubmit::DeviceSurface: j2k_core::traits::DeviceSurface +pub type j2k_core::TileBatchDecodeSubmit::Session: core::default::Default + core::marker::Send +pub type j2k_core::TileBatchDecodeSubmit::SubmittedSurface: j2k_core::traits::DeviceSubmission +pub fn j2k_core::TileBatchDecodeSubmit::submit_tile_region_scaled_to_device<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_core::TileBatchDecodeSubmit::submit_tile_region_to_device<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_core::TileBatchDecodeSubmit::submit_tile_request_to_device<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest, j2k_core::types::DecodeRequest) -> core::result::Result +pub fn j2k_core::TileBatchDecodeSubmit::submit_tile_scaled_to_device<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_core::TileBatchDecodeSubmit::submit_tile_to_device<'a>(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &'a [u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub trait j2k_core::TileDecompress +pub type j2k_core::TileDecompress::Error: j2k_core::error::CodecError +pub type j2k_core::TileDecompress::Pool: j2k_core::scratch::ScratchPool +pub fn j2k_core::TileDecompress::decompress_into(&mut Self::Pool, &[u8], &mut [u8]) -> core::result::Result +pub fn j2k_core::TileDecompress::expected_size(&[u8]) -> core::result::Result, Self::Error> +pub fn j2k_core::collect_indexed_batch_results(usize, alloc::vec::Vec>, F) -> core::result::Result, B> where F: core::ops::function::FnOnce(usize, E) -> B +pub fn j2k_core::copy_tight_pixels_to_strided_output(&[u8], (u32, u32), j2k_core::pixel::PixelFormat, &mut [u8], usize) -> core::result::Result<(), j2k_core::error::BufferError> +pub fn j2k_core::ensure_allocation_within_cap(usize, usize, &'static str) -> core::result::Result +pub fn j2k_core::strided_output_len((u32, u32), usize, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_core::strided_output_len_capped((u32, u32), usize, j2k_core::pixel::PixelFormat, usize, &'static str) -> core::result::Result +pub fn j2k_core::submit_ready_device(&mut S, impl core::ops::function::FnOnce(&mut S) -> core::result::Result) -> j2k_core::traits::ReadySubmission where S: j2k_core::traits::DeviceSubmitSession + ?core::marker::Sized +pub fn j2k_core::tile_batch_worker_count(usize, j2k_core::batch::TileBatchOptions, usize) -> usize +pub const fn j2k_core::validate_cuda_surface_backend_request(j2k_core::backend::BackendRequest) -> core::result::Result<(), j2k_core::backend::BackendRequest> +pub fn j2k_core::validate_strided_output_buffer((u32, u32), usize, usize, j2k_core::pixel::PixelFormat) -> core::result::Result<(), j2k_core::error::BufferError> +pub type j2k_core::IndexedBatchResult = (usize, core::result::Result) +``` + +## `j2k-jpeg` + +```text +pub mod j2k_jpeg +pub use j2k_jpeg::CacheStats +pub use j2k_jpeg::CodecContext +pub use j2k_jpeg::CompressedPayloadKind +pub use j2k_jpeg::CompressedTransferSyntax +pub use j2k_jpeg::DecodeRowsError +pub use j2k_jpeg::Downscale +pub use j2k_jpeg::ImageCodec +pub use j2k_jpeg::ImageDecode +pub use j2k_jpeg::ImageDecodeRows +pub use j2k_jpeg::PassthroughCandidate +pub use j2k_jpeg::PassthroughDecision +pub use j2k_jpeg::PassthroughRejectReason +pub use j2k_jpeg::PassthroughRequirements +pub use j2k_jpeg::PixelFormat +pub use j2k_jpeg::PixelLayout +pub use j2k_jpeg::RowSink +pub use j2k_jpeg::Sample +pub use j2k_jpeg::SampleType +pub use j2k_jpeg::TileBatchDecode +pub use j2k_jpeg::TileBatchOptions +pub use j2k_jpeg::TileDecompress +pub mod j2k_jpeg::adapter +pub mod j2k_jpeg::adapter::fast_packet +pub enum j2k_jpeg::adapter::fast_packet::FastPacketError +pub j2k_jpeg::adapter::fast_packet::FastPacketError::Decode(j2k_jpeg::error::JpegError) +pub j2k_jpeg::adapter::fast_packet::FastPacketError::EntropyMarkerUnsupported +pub j2k_jpeg::adapter::fast_packet::FastPacketError::EntropyMarkerUnsupported::marker: u8 +pub j2k_jpeg::adapter::fast_packet::FastPacketError::MissingHuffmanTable +pub j2k_jpeg::adapter::fast_packet::FastPacketError::MissingHuffmanTable::kind: j2k_jpeg::adapter::fast_packet::TableKind +pub j2k_jpeg::adapter::fast_packet::FastPacketError::MissingHuffmanTable::slot: u8 +pub j2k_jpeg::adapter::fast_packet::FastPacketError::MissingQuantTable +pub j2k_jpeg::adapter::fast_packet::FastPacketError::MissingQuantTable::slot: u8 +pub j2k_jpeg::adapter::fast_packet::FastPacketError::MissingScan +pub j2k_jpeg::adapter::fast_packet::FastPacketError::TruncatedEntropy +pub j2k_jpeg::adapter::fast_packet::FastPacketError::UnsupportedColorSpace(j2k_jpeg::info::ColorSpace) +pub j2k_jpeg::adapter::fast_packet::FastPacketError::UnsupportedComponentOrder +pub j2k_jpeg::adapter::fast_packet::FastPacketError::UnsupportedSampling +pub j2k_jpeg::adapter::fast_packet::FastPacketError::UnsupportedSof(j2k_jpeg::info::SofKind) +impl core::convert::From for j2k_jpeg::adapter::fast_packet::FastPacketError +pub fn j2k_jpeg::adapter::fast_packet::FastPacketError::from(j2k_jpeg::error::JpegError) -> Self +pub enum j2k_jpeg::adapter::fast_packet::TableKind +pub j2k_jpeg::adapter::fast_packet::TableKind::Ac +pub j2k_jpeg::adapter::fast_packet::TableKind::Dc +#[repr(C)] pub struct j2k_jpeg::adapter::fast_packet::JpegEntropyCheckpointV1 +pub j2k_jpeg::adapter::fast_packet::JpegEntropyCheckpointV1::bit_acc: u64 +pub j2k_jpeg::adapter::fast_packet::JpegEntropyCheckpointV1::bit_count: u32 +pub j2k_jpeg::adapter::fast_packet::JpegEntropyCheckpointV1::cb_prev_dc: i32 +pub j2k_jpeg::adapter::fast_packet::JpegEntropyCheckpointV1::cr_prev_dc: i32 +pub j2k_jpeg::adapter::fast_packet::JpegEntropyCheckpointV1::entropy_pos: u32 +pub j2k_jpeg::adapter::fast_packet::JpegEntropyCheckpointV1::mcu_index: u32 +pub j2k_jpeg::adapter::fast_packet::JpegEntropyCheckpointV1::reserved: u32 +pub j2k_jpeg::adapter::fast_packet::JpegEntropyCheckpointV1::y_prev_dc: i32 +#[repr(C)] pub struct j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1 +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::cb_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::cb_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::cb_quant: [u16; 64] +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::cr_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::cr_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::cr_quant: [u16; 64] +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::dimensions: (u32, u32) +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::entropy_bytes: alloc::vec::Vec +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::entropy_checkpoints: alloc::vec::Vec +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::mcu_rows: u32 +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::mcus_per_row: u32 +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::restart_interval_mcus: u32 +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::restart_offsets: alloc::vec::Vec +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::y_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::y_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast420PacketV1::y_quant: [u16; 64] +#[repr(C)] pub struct j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1 +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::cb_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::cb_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::cb_quant: [u16; 64] +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::cr_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::cr_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::cr_quant: [u16; 64] +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::dimensions: (u32, u32) +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::entropy_bytes: alloc::vec::Vec +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::entropy_checkpoints: alloc::vec::Vec +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::mcu_rows: u32 +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::mcus_per_row: u32 +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::restart_interval_mcus: u32 +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::restart_offsets: alloc::vec::Vec +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::y_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::y_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast422PacketV1::y_quant: [u16; 64] +#[repr(C)] pub struct j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1 +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::cb_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::cb_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::cb_quant: [u16; 64] +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::cr_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::cr_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::cr_quant: [u16; 64] +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::dimensions: (u32, u32) +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::entropy_bytes: alloc::vec::Vec +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::entropy_checkpoints: alloc::vec::Vec +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::mcu_rows: u32 +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::mcus_per_row: u32 +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::restart_interval_mcus: u32 +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::restart_offsets: alloc::vec::Vec +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::y_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::y_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegFast444PacketV1::y_quant: [u16; 64] +#[repr(C)] pub struct j2k_jpeg::adapter::fast_packet::JpegGrayPacketV1 +pub j2k_jpeg::adapter::fast_packet::JpegGrayPacketV1::dimensions: (u32, u32) +pub j2k_jpeg::adapter::fast_packet::JpegGrayPacketV1::entropy_bytes: alloc::vec::Vec +pub j2k_jpeg::adapter::fast_packet::JpegGrayPacketV1::mcu_rows: u32 +pub j2k_jpeg::adapter::fast_packet::JpegGrayPacketV1::mcus_per_row: u32 +pub j2k_jpeg::adapter::fast_packet::JpegGrayPacketV1::restart_interval_mcus: u32 +pub j2k_jpeg::adapter::fast_packet::JpegGrayPacketV1::restart_offsets: alloc::vec::Vec +pub j2k_jpeg::adapter::fast_packet::JpegGrayPacketV1::y_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegGrayPacketV1::y_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegGrayPacketV1::y_quant: [u16; 64] +#[repr(C)] pub struct j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::fast_packet::JpegHuffmanTable::bits: [u8; 16] +pub j2k_jpeg::adapter::fast_packet::JpegHuffmanTable::values: [u8; 256] +pub j2k_jpeg::adapter::fast_packet::JpegHuffmanTable::values_len: u16 +pub fn j2k_jpeg::adapter::fast_packet::build_fast420_packet(&[u8]) -> core::result::Result +pub fn j2k_jpeg::adapter::fast_packet::build_fast420_packet_for_decoder(&j2k_jpeg::decoder::Decoder<'_>) -> core::result::Result +pub fn j2k_jpeg::adapter::fast_packet::build_fast422_packet(&[u8]) -> core::result::Result +pub fn j2k_jpeg::adapter::fast_packet::build_fast422_packet_for_decoder(&j2k_jpeg::decoder::Decoder<'_>) -> core::result::Result +pub fn j2k_jpeg::adapter::fast_packet::build_fast444_packet(&[u8]) -> core::result::Result +pub fn j2k_jpeg::adapter::fast_packet::build_fast444_packet_for_decoder(&j2k_jpeg::decoder::Decoder<'_>) -> core::result::Result +pub fn j2k_jpeg::adapter::fast_packet::build_gray_packet(&[u8]) -> core::result::Result +pub fn j2k_jpeg::adapter::fast_packet::build_gray_packet_for_decoder(&j2k_jpeg::decoder::Decoder<'_>) -> core::result::Result +pub enum j2k_jpeg::adapter::FastPacketError +pub j2k_jpeg::adapter::FastPacketError::Decode(j2k_jpeg::error::JpegError) +pub j2k_jpeg::adapter::FastPacketError::EntropyMarkerUnsupported +pub j2k_jpeg::adapter::FastPacketError::EntropyMarkerUnsupported::marker: u8 +pub j2k_jpeg::adapter::FastPacketError::MissingHuffmanTable +pub j2k_jpeg::adapter::FastPacketError::MissingHuffmanTable::kind: j2k_jpeg::adapter::fast_packet::TableKind +pub j2k_jpeg::adapter::FastPacketError::MissingHuffmanTable::slot: u8 +pub j2k_jpeg::adapter::FastPacketError::MissingQuantTable +pub j2k_jpeg::adapter::FastPacketError::MissingQuantTable::slot: u8 +pub j2k_jpeg::adapter::FastPacketError::MissingScan +pub j2k_jpeg::adapter::FastPacketError::TruncatedEntropy +pub j2k_jpeg::adapter::FastPacketError::UnsupportedColorSpace(j2k_jpeg::info::ColorSpace) +pub j2k_jpeg::adapter::FastPacketError::UnsupportedComponentOrder +pub j2k_jpeg::adapter::FastPacketError::UnsupportedSampling +pub j2k_jpeg::adapter::FastPacketError::UnsupportedSof(j2k_jpeg::info::SofKind) +impl core::convert::From for j2k_jpeg::adapter::fast_packet::FastPacketError +pub fn j2k_jpeg::adapter::fast_packet::FastPacketError::from(j2k_jpeg::error::JpegError) -> Self +pub struct j2k_jpeg::adapter::DeviceBatchSummary +pub j2k_jpeg::adapter::DeviceBatchSummary::checkpoint_count: usize +pub j2k_jpeg::adapter::DeviceBatchSummary::matches_fast_420: bool +pub j2k_jpeg::adapter::DeviceBatchSummary::matches_fast_422: bool +pub j2k_jpeg::adapter::DeviceBatchSummary::matches_fast_444: bool +pub j2k_jpeg::adapter::DeviceBatchSummary::restart_interval: core::option::Option +pub struct j2k_jpeg::adapter::DeviceCheckpoint +pub j2k_jpeg::adapter::DeviceCheckpoint::bit_accumulator: u64 +pub j2k_jpeg::adapter::DeviceCheckpoint::bits_buffered: u8 +pub j2k_jpeg::adapter::DeviceCheckpoint::expected_rst: u8 +pub j2k_jpeg::adapter::DeviceCheckpoint::mcu_index: u32 +pub j2k_jpeg::adapter::DeviceCheckpoint::prev_dc: [i32; 4] +pub j2k_jpeg::adapter::DeviceCheckpoint::scan_offset: usize +pub struct j2k_jpeg::adapter::DeviceComponentPlan +pub j2k_jpeg::adapter::DeviceComponentPlan::h: u8 +pub j2k_jpeg::adapter::DeviceComponentPlan::output_index: usize +pub j2k_jpeg::adapter::DeviceComponentPlan::v: u8 +pub struct j2k_jpeg::adapter::DeviceDecodePlan<'a> +pub j2k_jpeg::adapter::DeviceDecodePlan::checkpoints: alloc::vec::Vec +pub j2k_jpeg::adapter::DeviceDecodePlan::color_space: j2k_jpeg::info::ColorSpace +pub j2k_jpeg::adapter::DeviceDecodePlan::components: alloc::vec::Vec +pub j2k_jpeg::adapter::DeviceDecodePlan::dimensions: (u32, u32) +pub j2k_jpeg::adapter::DeviceDecodePlan::matches_fast_420: bool +pub j2k_jpeg::adapter::DeviceDecodePlan::matches_fast_422: bool +pub j2k_jpeg::adapter::DeviceDecodePlan::matches_fast_444: bool +pub j2k_jpeg::adapter::DeviceDecodePlan::restart_interval: core::option::Option +pub j2k_jpeg::adapter::DeviceDecodePlan::scan_bytes: alloc::borrow::Cow<'a, [u8]> +pub j2k_jpeg::adapter::DeviceDecodePlan::warnings: alloc::vec::Vec +pub struct j2k_jpeg::adapter::JpegBaselineEncodeTables +pub j2k_jpeg::adapter::JpegBaselineEncodeTables::huff_ac_chroma: j2k_jpeg::adapter::JpegBaselineHuffmanTable +pub j2k_jpeg::adapter::JpegBaselineEncodeTables::huff_ac_luma: j2k_jpeg::adapter::JpegBaselineHuffmanTable +pub j2k_jpeg::adapter::JpegBaselineEncodeTables::huff_dc_chroma: j2k_jpeg::adapter::JpegBaselineHuffmanTable +pub j2k_jpeg::adapter::JpegBaselineEncodeTables::huff_dc_luma: j2k_jpeg::adapter::JpegBaselineHuffmanTable +pub j2k_jpeg::adapter::JpegBaselineEncodeTables::q_chroma: [u8; 64] +pub j2k_jpeg::adapter::JpegBaselineEncodeTables::q_luma: [u8; 64] +pub j2k_jpeg::adapter::JpegBaselineEncodeTables::sampling: j2k_jpeg::adapter::JpegBaselineSampling +pub struct j2k_jpeg::adapter::JpegBaselineHuffmanTable +pub j2k_jpeg::adapter::JpegBaselineHuffmanTable::codes: [u16; 256] +pub j2k_jpeg::adapter::JpegBaselineHuffmanTable::lens: [u8; 256] +pub struct j2k_jpeg::adapter::JpegBaselineSampling +pub j2k_jpeg::adapter::JpegBaselineSampling::components: u8 +pub j2k_jpeg::adapter::JpegBaselineSampling::h: [u8; 3] +pub j2k_jpeg::adapter::JpegBaselineSampling::max_h: u8 +pub j2k_jpeg::adapter::JpegBaselineSampling::max_v: u8 +pub j2k_jpeg::adapter::JpegBaselineSampling::v: [u8; 3] +#[repr(C)] pub struct j2k_jpeg::adapter::JpegEntropyCheckpointV1 +pub j2k_jpeg::adapter::JpegEntropyCheckpointV1::bit_acc: u64 +pub j2k_jpeg::adapter::JpegEntropyCheckpointV1::bit_count: u32 +pub j2k_jpeg::adapter::JpegEntropyCheckpointV1::cb_prev_dc: i32 +pub j2k_jpeg::adapter::JpegEntropyCheckpointV1::cr_prev_dc: i32 +pub j2k_jpeg::adapter::JpegEntropyCheckpointV1::entropy_pos: u32 +pub j2k_jpeg::adapter::JpegEntropyCheckpointV1::mcu_index: u32 +pub j2k_jpeg::adapter::JpegEntropyCheckpointV1::reserved: u32 +pub j2k_jpeg::adapter::JpegEntropyCheckpointV1::y_prev_dc: i32 +#[repr(C)] pub struct j2k_jpeg::adapter::JpegFast420PacketV1 +pub j2k_jpeg::adapter::JpegFast420PacketV1::cb_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast420PacketV1::cb_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast420PacketV1::cb_quant: [u16; 64] +pub j2k_jpeg::adapter::JpegFast420PacketV1::cr_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast420PacketV1::cr_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast420PacketV1::cr_quant: [u16; 64] +pub j2k_jpeg::adapter::JpegFast420PacketV1::dimensions: (u32, u32) +pub j2k_jpeg::adapter::JpegFast420PacketV1::entropy_bytes: alloc::vec::Vec +pub j2k_jpeg::adapter::JpegFast420PacketV1::entropy_checkpoints: alloc::vec::Vec +pub j2k_jpeg::adapter::JpegFast420PacketV1::mcu_rows: u32 +pub j2k_jpeg::adapter::JpegFast420PacketV1::mcus_per_row: u32 +pub j2k_jpeg::adapter::JpegFast420PacketV1::restart_interval_mcus: u32 +pub j2k_jpeg::adapter::JpegFast420PacketV1::restart_offsets: alloc::vec::Vec +pub j2k_jpeg::adapter::JpegFast420PacketV1::y_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast420PacketV1::y_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast420PacketV1::y_quant: [u16; 64] +#[repr(C)] pub struct j2k_jpeg::adapter::JpegFast422PacketV1 +pub j2k_jpeg::adapter::JpegFast422PacketV1::cb_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast422PacketV1::cb_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast422PacketV1::cb_quant: [u16; 64] +pub j2k_jpeg::adapter::JpegFast422PacketV1::cr_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast422PacketV1::cr_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast422PacketV1::cr_quant: [u16; 64] +pub j2k_jpeg::adapter::JpegFast422PacketV1::dimensions: (u32, u32) +pub j2k_jpeg::adapter::JpegFast422PacketV1::entropy_bytes: alloc::vec::Vec +pub j2k_jpeg::adapter::JpegFast422PacketV1::entropy_checkpoints: alloc::vec::Vec +pub j2k_jpeg::adapter::JpegFast422PacketV1::mcu_rows: u32 +pub j2k_jpeg::adapter::JpegFast422PacketV1::mcus_per_row: u32 +pub j2k_jpeg::adapter::JpegFast422PacketV1::restart_interval_mcus: u32 +pub j2k_jpeg::adapter::JpegFast422PacketV1::restart_offsets: alloc::vec::Vec +pub j2k_jpeg::adapter::JpegFast422PacketV1::y_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast422PacketV1::y_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast422PacketV1::y_quant: [u16; 64] +#[repr(C)] pub struct j2k_jpeg::adapter::JpegFast444PacketV1 +pub j2k_jpeg::adapter::JpegFast444PacketV1::cb_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast444PacketV1::cb_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast444PacketV1::cb_quant: [u16; 64] +pub j2k_jpeg::adapter::JpegFast444PacketV1::cr_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast444PacketV1::cr_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast444PacketV1::cr_quant: [u16; 64] +pub j2k_jpeg::adapter::JpegFast444PacketV1::dimensions: (u32, u32) +pub j2k_jpeg::adapter::JpegFast444PacketV1::entropy_bytes: alloc::vec::Vec +pub j2k_jpeg::adapter::JpegFast444PacketV1::entropy_checkpoints: alloc::vec::Vec +pub j2k_jpeg::adapter::JpegFast444PacketV1::mcu_rows: u32 +pub j2k_jpeg::adapter::JpegFast444PacketV1::mcus_per_row: u32 +pub j2k_jpeg::adapter::JpegFast444PacketV1::restart_interval_mcus: u32 +pub j2k_jpeg::adapter::JpegFast444PacketV1::restart_offsets: alloc::vec::Vec +pub j2k_jpeg::adapter::JpegFast444PacketV1::y_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast444PacketV1::y_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegFast444PacketV1::y_quant: [u16; 64] +#[repr(C)] pub struct j2k_jpeg::adapter::JpegGrayPacketV1 +pub j2k_jpeg::adapter::JpegGrayPacketV1::dimensions: (u32, u32) +pub j2k_jpeg::adapter::JpegGrayPacketV1::entropy_bytes: alloc::vec::Vec +pub j2k_jpeg::adapter::JpegGrayPacketV1::mcu_rows: u32 +pub j2k_jpeg::adapter::JpegGrayPacketV1::mcus_per_row: u32 +pub j2k_jpeg::adapter::JpegGrayPacketV1::restart_interval_mcus: u32 +pub j2k_jpeg::adapter::JpegGrayPacketV1::restart_offsets: alloc::vec::Vec +pub j2k_jpeg::adapter::JpegGrayPacketV1::y_ac_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegGrayPacketV1::y_dc_table: j2k_jpeg::adapter::fast_packet::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegGrayPacketV1::y_quant: [u16; 64] +#[repr(C)] pub struct j2k_jpeg::adapter::JpegHuffmanTable +pub j2k_jpeg::adapter::JpegHuffmanTable::bits: [u8; 16] +pub j2k_jpeg::adapter::JpegHuffmanTable::values: [u8; 256] +pub j2k_jpeg::adapter::JpegHuffmanTable::values_len: u16 +pub const j2k_jpeg::adapter::JPEG_BASELINE_ZIGZAG: [u8; 64] +pub fn j2k_jpeg::adapter::assemble_jpeg_baseline_frame(&[u8], u32, u32, &j2k_jpeg::adapter::JpegBaselineEncodeTables, j2k_jpeg::encoder::JpegEncodeOptions, j2k_jpeg::encoder::JpegBackend) -> core::result::Result +pub fn j2k_jpeg::adapter::baseline_encode_tables(j2k_jpeg::encoder::JpegEncodeOptions) -> core::result::Result +pub fn j2k_jpeg::adapter::build_device_plan<'a>(&'a j2k_jpeg::decoder::Decoder<'a>, u32) -> core::result::Result, j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::adapter::build_fast420_packet(&[u8]) -> core::result::Result +pub fn j2k_jpeg::adapter::build_fast420_packet_for_decoder(&j2k_jpeg::decoder::Decoder<'_>) -> core::result::Result +pub fn j2k_jpeg::adapter::build_fast422_packet(&[u8]) -> core::result::Result +pub fn j2k_jpeg::adapter::build_fast422_packet_for_decoder(&j2k_jpeg::decoder::Decoder<'_>) -> core::result::Result +pub fn j2k_jpeg::adapter::build_fast444_packet(&[u8]) -> core::result::Result +pub fn j2k_jpeg::adapter::build_fast444_packet_for_decoder(&j2k_jpeg::decoder::Decoder<'_>) -> core::result::Result +pub fn j2k_jpeg::adapter::build_gray_packet(&[u8]) -> core::result::Result +pub fn j2k_jpeg::adapter::build_gray_packet_for_decoder(&j2k_jpeg::decoder::Decoder<'_>) -> core::result::Result +pub fn j2k_jpeg::adapter::decoder_bytes<'a>(&'a j2k_jpeg::decoder::Decoder<'a>) -> &'a [u8] +pub fn j2k_jpeg::adapter::jpeg_baseline_entropy_capacity_bytes(u32, u32, j2k_jpeg::adapter::JpegBaselineSampling, core::option::Option) -> core::result::Result +pub fn j2k_jpeg::adapter::jpeg_baseline_sampling_for(j2k_jpeg::encoder::JpegSubsampling) -> j2k_jpeg::adapter::JpegBaselineSampling +pub fn j2k_jpeg::adapter::summarize_device_batch(&j2k_jpeg::decoder::Decoder<'_>, u32) -> j2k_jpeg::adapter::DeviceBatchSummary +pub fn j2k_jpeg::adapter::validate_jpeg_baseline_dimensions(u32, u32) -> core::result::Result<(), j2k_jpeg::encoder::JpegEncodeError> +pub fn j2k_jpeg::adapter::validate_jpeg_baseline_restart_interval(core::option::Option) -> core::result::Result<(), j2k_jpeg::encoder::JpegEncodeError> +pub mod j2k_jpeg::batch_session +pub struct j2k_jpeg::batch_session::JpegBatchSession +impl j2k_jpeg::batch_session::JpegBatchSession +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_prepared_jpeg_tiles_rgb8(&mut self, &mut [j2k_jpeg::decoder::PreparedJpegTileJob<'_, '_>]) -> alloc::vec::Vec> +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_tiles_into(&mut self, &mut [j2k_jpeg::decoder::TileDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_tiles_into_with_options(&mut self, &mut [j2k_jpeg::decoder::TileDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_tiles_region_scaled_into(&mut self, &mut [j2k_jpeg::decoder::TileRegionScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_tiles_region_scaled_into_with_options(&mut self, &mut [j2k_jpeg::decoder::TileRegionScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_tiles_scaled_into(&mut self, &mut [j2k_jpeg::decoder::TileScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_tiles_scaled_into_with_options(&mut self, &mut [j2k_jpeg::decoder::TileScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::batch_session::JpegBatchSession::new(j2k_core::batch::TileBatchOptions) -> Self +pub fn j2k_jpeg::batch_session::JpegBatchSession::options(&self) -> j2k_core::batch::TileBatchOptions +pub fn j2k_jpeg::batch_session::JpegBatchSession::reset(&mut self) +pub fn j2k_jpeg::batch_session::JpegBatchSession::retained_worker_slots(&self) -> usize +pub fn j2k_jpeg::batch_session::JpegBatchSession::set_options(&mut self, j2k_core::batch::TileBatchOptions) +pub fn j2k_jpeg::batch_session::JpegBatchSession::worker_count(&self) -> usize +impl core::default::Default for j2k_jpeg::batch_session::JpegBatchSession +pub fn j2k_jpeg::batch_session::JpegBatchSession::default() -> Self +pub mod j2k_jpeg::capabilities +pub enum j2k_jpeg::capabilities::JpegDecodeOp +pub j2k_jpeg::capabilities::JpegDecodeOp::Full +pub j2k_jpeg::capabilities::JpegDecodeOp::Region(j2k_jpeg::info::Rect) +pub j2k_jpeg::capabilities::JpegDecodeOp::RegionScaled +pub j2k_jpeg::capabilities::JpegDecodeOp::RegionScaled::roi: j2k_jpeg::info::Rect +pub j2k_jpeg::capabilities::JpegDecodeOp::RegionScaled::scale: j2k_core::scale::Downscale +pub j2k_jpeg::capabilities::JpegDecodeOp::Scaled(j2k_core::scale::Downscale) +pub enum j2k_jpeg::capabilities::JpegResolvedDecodePath +pub j2k_jpeg::capabilities::JpegResolvedDecodePath::CpuHost +pub j2k_jpeg::capabilities::JpegResolvedDecodePath::MetalFast +pub j2k_jpeg::capabilities::JpegResolvedDecodePath::OwnedCudaRgb8 +pub j2k_jpeg::capabilities::JpegResolvedDecodePath::Rejected +pub j2k_jpeg::capabilities::JpegResolvedDecodePath::Rejected::backend: j2k_core::backend::BackendRequest +pub j2k_jpeg::capabilities::JpegResolvedDecodePath::Rejected::reason: &'static str +pub struct j2k_jpeg::capabilities::JpegBackendEligibility +pub j2k_jpeg::capabilities::JpegBackendEligibility::eligible: bool +pub j2k_jpeg::capabilities::JpegBackendEligibility::reason: core::option::Option<&'static str> +pub struct j2k_jpeg::capabilities::JpegCapabilityReport +pub j2k_jpeg::capabilities::JpegCapabilityReport::cpu: j2k_jpeg::capabilities::JpegBackendEligibility +pub j2k_jpeg::capabilities::JpegCapabilityReport::device: j2k_jpeg::adapter::DeviceBatchSummary +pub j2k_jpeg::capabilities::JpegCapabilityReport::info: j2k_jpeg::info::Info +pub j2k_jpeg::capabilities::JpegCapabilityReport::metal_fast: j2k_jpeg::capabilities::JpegBackendEligibility +pub j2k_jpeg::capabilities::JpegCapabilityReport::owned_cuda: j2k_jpeg::capabilities::JpegBackendEligibility +pub j2k_jpeg::capabilities::JpegCapabilityReport::request: j2k_jpeg::capabilities::JpegCapabilityRequest +impl j2k_jpeg::capabilities::JpegCapabilityReport +pub fn j2k_jpeg::capabilities::JpegCapabilityReport::for_decoder(&j2k_jpeg::decoder::Decoder<'_>, j2k_jpeg::capabilities::JpegCapabilityRequest) -> Self +pub fn j2k_jpeg::capabilities::JpegCapabilityReport::inspect(&[u8], j2k_jpeg::capabilities::JpegCapabilityRequest) -> core::result::Result +pub fn j2k_jpeg::capabilities::JpegCapabilityReport::metal_resident_rgb8_batch_output(&self) -> j2k_jpeg::capabilities::JpegBackendEligibility +pub fn j2k_jpeg::capabilities::JpegCapabilityReport::resolve_path(&self, j2k_core::backend::BackendRequest) -> j2k_jpeg::capabilities::JpegResolvedDecodePath +pub struct j2k_jpeg::capabilities::JpegCapabilityRequest +pub j2k_jpeg::capabilities::JpegCapabilityRequest::fmt: j2k_core::pixel::PixelFormat +pub j2k_jpeg::capabilities::JpegCapabilityRequest::op: j2k_jpeg::capabilities::JpegDecodeOp +pub struct j2k_jpeg::capabilities::JpegDecodeRequest +pub j2k_jpeg::capabilities::JpegDecodeRequest::backend: j2k_core::backend::BackendRequest +pub j2k_jpeg::capabilities::JpegDecodeRequest::fmt: j2k_core::pixel::PixelFormat +pub j2k_jpeg::capabilities::JpegDecodeRequest::op: j2k_jpeg::capabilities::JpegDecodeOp +impl j2k_jpeg::capabilities::JpegDecodeRequest +pub const fn j2k_jpeg::capabilities::JpegDecodeRequest::capability(self) -> j2k_jpeg::capabilities::JpegCapabilityRequest +pub struct j2k_jpeg::capabilities::JpegResolvedDecode +pub j2k_jpeg::capabilities::JpegResolvedDecode::capabilities: j2k_jpeg::capabilities::JpegCapabilityReport +pub j2k_jpeg::capabilities::JpegResolvedDecode::output_rect: j2k_jpeg::info::Rect +pub j2k_jpeg::capabilities::JpegResolvedDecode::path: j2k_jpeg::capabilities::JpegResolvedDecodePath +pub j2k_jpeg::capabilities::JpegResolvedDecode::request: j2k_jpeg::capabilities::JpegDecodeRequest +impl j2k_jpeg::capabilities::JpegResolvedDecode +pub fn j2k_jpeg::capabilities::JpegResolvedDecode::from_capabilities(j2k_jpeg::capabilities::JpegCapabilityReport, j2k_jpeg::capabilities::JpegDecodeRequest) -> Self +pub fn j2k_jpeg::capabilities::JpegResolvedDecode::inspect(&[u8], j2k_jpeg::capabilities::JpegDecodeRequest) -> core::result::Result +pub mod j2k_jpeg::context +pub struct j2k_jpeg::context::DecoderContext +impl j2k_jpeg::context::DecoderContext +pub fn j2k_jpeg::context::DecoderContext::new() -> Self +impl j2k_core::context::CodecContext for j2k_jpeg::context::DecoderContext +pub fn j2k_jpeg::context::DecoderContext::cache_stats(&self) -> j2k_core::context::CacheStats +pub fn j2k_jpeg::context::DecoderContext::clear(&mut self) +pub mod j2k_jpeg::decoder +pub use j2k_jpeg::decoder::TileBatchOptions +pub struct j2k_jpeg::decoder::DecodeOutcome +pub j2k_jpeg::decoder::DecodeOutcome::decoded: j2k_jpeg::info::Rect +pub j2k_jpeg::decoder::DecodeOutcome::warnings: alloc::vec::Vec +impl core::convert::From for j2k_core::types::DecodeOutcome +pub fn j2k_core::types::DecodeOutcome::from(j2k_jpeg::decoder::DecodeOutcome) -> Self +pub struct j2k_jpeg::decoder::DecodedTile +pub j2k_jpeg::decoder::DecodedTile::decoded: j2k_jpeg::info::Rect +pub j2k_jpeg::decoder::DecodedTile::dimensions: (u32, u32) +pub j2k_jpeg::decoder::DecodedTile::warnings: alloc::vec::Vec +pub struct j2k_jpeg::decoder::Decoder<'a> +impl j2k_jpeg::decoder::Decoder<'_> +pub fn j2k_jpeg::decoder::Decoder<'_>::decode_tile(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut S) -> core::result::Result where S: j2k_core::row_sink::RowSink +impl<'a> j2k_jpeg::decoder::Decoder<'a> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode(&self, j2k_core::pixel::PixelFormat) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_component_rows_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut W) -> core::result::Result where W: j2k_jpeg::decoder::ComponentRowWriter +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_into(&self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_into_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region(&self, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_component_rows_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut W, j2k_jpeg::info::Rect, j2k_core::scale::Downscale) -> core::result::Result where W: j2k_jpeg::decoder::ComponentRowWriter +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_into(&self, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_into_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_rgba8_into_with_alpha(&self, &mut [u8], usize, j2k_jpeg::info::Rect, u8) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_rgba8_into_with_alpha_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_jpeg::info::Rect, u8) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_scaled(&self, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_core::scale::Downscale) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_scaled_into(&self, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_scaled_into_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_scaled_with_scratch(&self, &mut j2k_jpeg::ScratchPool, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_core::scale::Downscale) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_with_scratch(&self, &mut j2k_jpeg::ScratchPool, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_rgba8_into_with_alpha(&self, &mut [u8], usize, u8) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_rgba8_into_with_alpha_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, u8) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_rows(&self, &mut S) -> core::result::Result where S: j2k_core::row_sink::RowSink +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_rows_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut S) -> core::result::Result where S: j2k_core::row_sink::RowSink +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_scaled(&self, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_scaled_into(&self, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_scaled_into_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_scaled_with_scratch(&self, &mut j2k_jpeg::ScratchPool, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_with_scratch(&self, &mut j2k_jpeg::ScratchPool, j2k_core::pixel::PixelFormat) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::from_view(j2k_jpeg::decoder::JpegView<'a>) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::from_view_in_context(j2k_jpeg::decoder::JpegView<'a>, &mut j2k_jpeg::context::DecoderContext) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::info(&self) -> &j2k_jpeg::info::Info +pub fn j2k_jpeg::decoder::Decoder<'a>::inspect(&'a [u8]) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::inspect_with_options(&'a [u8], j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::new(&'a [u8]) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::new_with_options(&'a [u8], j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::passthrough_candidate(&self) -> core::option::Option> +pub fn j2k_jpeg::decoder::Decoder<'a>::restart_index(&self) -> core::result::Result, j2k_jpeg::error::JpegError> +impl j2k_core::traits::ImageCodec for j2k_jpeg::decoder::Decoder<'_> +pub type j2k_jpeg::decoder::Decoder<'_>::Error = j2k_jpeg::error::JpegError +pub type j2k_jpeg::decoder::Decoder<'_>::Pool = j2k_jpeg::ScratchPool +pub type j2k_jpeg::decoder::Decoder<'_>::Warning = j2k_jpeg::error::Warning +impl<'a> j2k_core::traits::ImageDecode<'a> for j2k_jpeg::decoder::Decoder<'a> +pub type j2k_jpeg::decoder::Decoder<'a>::View = j2k_jpeg::decoder::JpegView<'a> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_into_with_scratch(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::decoder::Decoder<'a>::from_view(Self::View) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::inspect(&'a [u8]) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::parse(&'a [u8]) -> core::result::Result +impl<'a> j2k_core::traits::ImageDecodeRows<'a, u8> for j2k_jpeg::decoder::Decoder<'a> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_rows>(&mut self, &mut R) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +pub struct j2k_jpeg::decoder::JpegView<'a> +impl<'a> j2k_jpeg::decoder::JpegView<'a> +pub fn j2k_jpeg::decoder::JpegView<'a>::bytes(&self) -> &'a [u8] +pub fn j2k_jpeg::decoder::JpegView<'a>::info(&self) -> &j2k_jpeg::info::Info +pub fn j2k_jpeg::decoder::JpegView<'a>::parse(&'a [u8]) -> core::result::Result +pub fn j2k_jpeg::decoder::JpegView<'a>::parse_with_options(&'a [u8], j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decoder::JpegView<'a>::passthrough_candidate(&self) -> core::option::Option> +pub fn j2k_jpeg::decoder::JpegView<'a>::restart_index(&self) -> core::result::Result, j2k_jpeg::error::JpegError> +pub struct j2k_jpeg::decoder::PreparedJpegTileJob<'i, 'o> +pub j2k_jpeg::decoder::PreparedJpegTileJob::input: j2k_jpeg::segment::PreparedJpeg<'i> +pub j2k_jpeg::decoder::PreparedJpegTileJob::options: j2k_jpeg::info::DecodeOptions +pub j2k_jpeg::decoder::PreparedJpegTileJob::out: &'o mut [u8] +pub j2k_jpeg::decoder::PreparedJpegTileJob::stride: usize +pub trait j2k_jpeg::decoder::ComponentRowWriter +pub fn j2k_jpeg::decoder::ComponentRowWriter::write_gray_row(&mut self, u32, &[u8]) -> core::result::Result<(), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::ComponentRowWriter::write_rgb_row(&mut self, u32, &[u8], &[u8], &[u8]) -> core::result::Result<(), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::ComponentRowWriter::write_ycbcr_row(&mut self, u32, &[u8], &[u8], &[u8]) -> core::result::Result<(), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::decode_prepared_jpeg_tiles_rgb8(&mut [j2k_jpeg::decoder::PreparedJpegTileJob<'_, '_>]) -> alloc::vec::Vec> +pub fn j2k_jpeg::decoder::decode_tile_into(&[u8], &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_jpeg::decoder::decode_tile_into_in_context(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_jpeg::decoder::decode_tile_into_in_context_with_options(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decoder::decode_tile_region_into_in_context(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect) -> core::result::Result +pub fn j2k_jpeg::decoder::decode_tile_region_into_in_context_with_options(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decoder::decode_tile_region_scaled_into_in_context(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_jpeg::decoder::decode_tile_region_scaled_into_in_context_with_options(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_core::scale::Downscale, j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decoder::decode_tile_scaled_into_in_context(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_jpeg::decoder::decode_tile_scaled_into_in_context_with_options(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decoder::decode_tiles_into(&mut [j2k_jpeg::decoder::TileDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::decoder::decode_tiles_into_with_options(&mut [j2k_jpeg::decoder::TileDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions, j2k_core::batch::TileBatchOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::decoder::decode_tiles_region_scaled_into(&mut [j2k_jpeg::decoder::TileRegionScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::decoder::decode_tiles_region_scaled_into_with_options(&mut [j2k_jpeg::decoder::TileRegionScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions, j2k_core::batch::TileBatchOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::decoder::decode_tiles_scaled_into(&mut [j2k_jpeg::decoder::TileScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::decoder::decode_tiles_scaled_into_with_options(&mut [j2k_jpeg::decoder::TileScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions, j2k_core::batch::TileBatchOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub type j2k_jpeg::decoder::TileBatchError = j2k_core::batch::TileBatchError +pub type j2k_jpeg::decoder::TileDecodeJob<'i, 'o> = j2k_core::batch::TileDecodeJob<'i, 'o> +pub type j2k_jpeg::decoder::TileRegionScaledDecodeJob<'i, 'o> = j2k_core::batch::TileRegionScaledDecodeJob<'i, 'o> +pub type j2k_jpeg::decoder::TileScaledDecodeJob<'i, 'o> = j2k_core::batch::TileScaledDecodeJob<'i, 'o> +pub mod j2k_jpeg::encoder +pub enum j2k_jpeg::encoder::JpegBackend +pub j2k_jpeg::encoder::JpegBackend::Auto +pub j2k_jpeg::encoder::JpegBackend::Cpu +pub j2k_jpeg::encoder::JpegBackend::Metal +pub enum j2k_jpeg::encoder::JpegEncodeError +pub j2k_jpeg::encoder::JpegEncodeError::DimensionsTooLarge +pub j2k_jpeg::encoder::JpegEncodeError::DimensionsTooLarge::height: u32 +pub j2k_jpeg::encoder::JpegEncodeError::DimensionsTooLarge::width: u32 +pub j2k_jpeg::encoder::JpegEncodeError::EmptyDimensions +pub j2k_jpeg::encoder::JpegEncodeError::IncompatibleSubsampling +pub j2k_jpeg::encoder::JpegEncodeError::IncompatibleSubsampling::samples: &'static str +pub j2k_jpeg::encoder::JpegEncodeError::IncompatibleSubsampling::subsampling: j2k_jpeg::encoder::JpegSubsampling +pub j2k_jpeg::encoder::JpegEncodeError::Internal(alloc::string::String) +pub j2k_jpeg::encoder::JpegEncodeError::InvalidRestartInterval +pub j2k_jpeg::encoder::JpegEncodeError::MissingHuffmanCode +pub j2k_jpeg::encoder::JpegEncodeError::MissingHuffmanCode::symbol: u8 +pub j2k_jpeg::encoder::JpegEncodeError::SampleLength +pub j2k_jpeg::encoder::JpegEncodeError::SampleLength::actual: usize +pub j2k_jpeg::encoder::JpegEncodeError::SampleLength::expected: usize +pub j2k_jpeg::encoder::JpegEncodeError::SegmentTooLarge +pub j2k_jpeg::encoder::JpegEncodeError::SegmentTooLarge::name: &'static str +pub j2k_jpeg::encoder::JpegEncodeError::UnsupportedBackend +pub j2k_jpeg::encoder::JpegEncodeError::UnsupportedBackend::backend: j2k_jpeg::encoder::JpegBackend +pub enum j2k_jpeg::encoder::JpegSamples<'a> +pub j2k_jpeg::encoder::JpegSamples::Gray8 +pub j2k_jpeg::encoder::JpegSamples::Gray8::data: &'a [u8] +pub j2k_jpeg::encoder::JpegSamples::Gray8::height: u32 +pub j2k_jpeg::encoder::JpegSamples::Gray8::width: u32 +pub j2k_jpeg::encoder::JpegSamples::Rgb8 +pub j2k_jpeg::encoder::JpegSamples::Rgb8::data: &'a [u8] +pub j2k_jpeg::encoder::JpegSamples::Rgb8::height: u32 +pub j2k_jpeg::encoder::JpegSamples::Rgb8::width: u32 +pub enum j2k_jpeg::encoder::JpegSubsampling +pub j2k_jpeg::encoder::JpegSubsampling::Gray +pub j2k_jpeg::encoder::JpegSubsampling::Ybr420 +pub j2k_jpeg::encoder::JpegSubsampling::Ybr422 +pub j2k_jpeg::encoder::JpegSubsampling::Ybr444 +pub struct j2k_jpeg::encoder::EncodedJpeg +pub j2k_jpeg::encoder::EncodedJpeg::backend: j2k_jpeg::encoder::JpegBackend +pub j2k_jpeg::encoder::EncodedJpeg::data: alloc::vec::Vec +pub struct j2k_jpeg::encoder::JpegEncodeOptions +pub j2k_jpeg::encoder::JpegEncodeOptions::backend: j2k_jpeg::encoder::JpegBackend +pub j2k_jpeg::encoder::JpegEncodeOptions::quality: u8 +pub j2k_jpeg::encoder::JpegEncodeOptions::restart_interval: core::option::Option +pub j2k_jpeg::encoder::JpegEncodeOptions::subsampling: j2k_jpeg::encoder::JpegSubsampling +impl core::default::Default for j2k_jpeg::encoder::JpegEncodeOptions +pub fn j2k_jpeg::encoder::JpegEncodeOptions::default() -> Self +pub fn j2k_jpeg::encoder::encode_jpeg_baseline(j2k_jpeg::encoder::JpegSamples<'_>, j2k_jpeg::encoder::JpegEncodeOptions) -> core::result::Result +pub mod j2k_jpeg::error +pub enum j2k_jpeg::error::BuilderConflictReason +pub j2k_jpeg::error::BuilderConflictReason::InputAndScanFragments +pub j2k_jpeg::error::BuilderConflictReason::NoInput +pub j2k_jpeg::error::BuilderConflictReason::ScanFragmentsEmpty +pub enum j2k_jpeg::error::HuffmanFailure +pub j2k_jpeg::error::HuffmanFailure::CodeOverflow +pub j2k_jpeg::error::HuffmanFailure::InvalidSymbol +pub j2k_jpeg::error::HuffmanFailure::TableExhausted +#[non_exhaustive] pub enum j2k_jpeg::error::JpegError +pub j2k_jpeg::error::JpegError::BuilderConflict +pub j2k_jpeg::error::JpegError::BuilderConflict::reason: j2k_jpeg::error::BuilderConflictReason +pub j2k_jpeg::error::JpegError::CoefficientOverflow +pub j2k_jpeg::error::JpegError::CoefficientOverflow::component: u8 +pub j2k_jpeg::error::JpegError::CoefficientOverflow::mcu: u32 +pub j2k_jpeg::error::JpegError::ConflictingDri +pub j2k_jpeg::error::JpegError::ConflictingDri::existing: u16 +pub j2k_jpeg::error::JpegError::ConflictingDri::new: u16 +pub j2k_jpeg::error::JpegError::ConflictingDri::offset: usize +pub j2k_jpeg::error::JpegError::ConflictingDuplicateTable +pub j2k_jpeg::error::JpegError::ConflictingDuplicateTable::id: u8 +pub j2k_jpeg::error::JpegError::ConflictingDuplicateTable::offset: usize +pub j2k_jpeg::error::JpegError::ConflictingDuplicateTable::table: j2k_jpeg::error::TableKind +pub j2k_jpeg::error::JpegError::ConflictingExpectedDimensions +pub j2k_jpeg::error::JpegError::ConflictingExpectedDimensions::actual: (u16, u16) +pub j2k_jpeg::error::JpegError::ConflictingExpectedDimensions::expected: (u16, u16) +pub j2k_jpeg::error::JpegError::ConflictingExpectedDimensions::offset: usize +pub j2k_jpeg::error::JpegError::DimensionOverflow +pub j2k_jpeg::error::JpegError::DimensionOverflow::height: u32 +pub j2k_jpeg::error::JpegError::DimensionOverflow::width: u32 +pub j2k_jpeg::error::JpegError::DownscaleUnsupported +pub j2k_jpeg::error::JpegError::DownscaleUnsupported::sof: j2k_jpeg::info::SofKind +pub j2k_jpeg::error::JpegError::DuplicateMarker +pub j2k_jpeg::error::JpegError::DuplicateMarker::marker: j2k_jpeg::error::MarkerKind +pub j2k_jpeg::error::JpegError::DuplicateMarker::offset: usize +pub j2k_jpeg::error::JpegError::DuplicateScanComponent +pub j2k_jpeg::error::JpegError::DuplicateScanComponent::component: u8 +pub j2k_jpeg::error::JpegError::DuplicateScanComponent::offset: usize +pub j2k_jpeg::error::JpegError::ExpectedDimensionsRequired +pub j2k_jpeg::error::JpegError::ExpectedDimensionsRequired::offset: usize +pub j2k_jpeg::error::JpegError::HuffmanDecode +pub j2k_jpeg::error::JpegError::HuffmanDecode::mcu: u32 +pub j2k_jpeg::error::JpegError::HuffmanDecode::reason: j2k_jpeg::error::HuffmanFailure +pub j2k_jpeg::error::JpegError::InvalidJpegAssembly +pub j2k_jpeg::error::JpegError::InvalidJpegAssembly::offset: usize +pub j2k_jpeg::error::JpegError::InvalidJpegAssembly::reason: &'static str +pub j2k_jpeg::error::JpegError::InvalidMarker +pub j2k_jpeg::error::JpegError::InvalidMarker::marker: u8 +pub j2k_jpeg::error::JpegError::InvalidMarker::offset: usize +pub j2k_jpeg::error::JpegError::InvalidSampling +pub j2k_jpeg::error::JpegError::InvalidSampling::component: u8 +pub j2k_jpeg::error::JpegError::InvalidSampling::h: u8 +pub j2k_jpeg::error::JpegError::InvalidSampling::v: u8 +pub j2k_jpeg::error::JpegError::InvalidScanParameters +pub j2k_jpeg::error::JpegError::InvalidScanParameters::ah: u8 +pub j2k_jpeg::error::JpegError::InvalidScanParameters::al: u8 +pub j2k_jpeg::error::JpegError::InvalidScanParameters::offset: usize +pub j2k_jpeg::error::JpegError::InvalidScanParameters::se: u8 +pub j2k_jpeg::error::JpegError::InvalidScanParameters::ss: u8 +pub j2k_jpeg::error::JpegError::InvalidSegmentLength +pub j2k_jpeg::error::JpegError::InvalidSegmentLength::length: u16 +pub j2k_jpeg::error::JpegError::InvalidSegmentLength::marker: u8 +pub j2k_jpeg::error::JpegError::InvalidSegmentLength::offset: usize +pub j2k_jpeg::error::JpegError::InvalidSequentialComponentSet +pub j2k_jpeg::error::JpegError::InvalidSequentialComponentSet::expected: u8 +pub j2k_jpeg::error::JpegError::InvalidSequentialComponentSet::found: u8 +pub j2k_jpeg::error::JpegError::InvalidSequentialComponentSet::offset: usize +pub j2k_jpeg::error::JpegError::InvalidSequentialScanCount +pub j2k_jpeg::error::JpegError::InvalidSequentialScanCount::count: u16 +pub j2k_jpeg::error::JpegError::InvalidSequentialScanCount::sof: j2k_jpeg::info::SofKind +pub j2k_jpeg::error::JpegError::InvalidStride +pub j2k_jpeg::error::JpegError::InvalidStride::row: usize +pub j2k_jpeg::error::JpegError::InvalidStride::stride: usize +pub j2k_jpeg::error::JpegError::MemoryCapExceeded +pub j2k_jpeg::error::JpegError::MemoryCapExceeded::cap: usize +pub j2k_jpeg::error::JpegError::MemoryCapExceeded::requested: usize +pub j2k_jpeg::error::JpegError::MissingHuffmanTable +pub j2k_jpeg::error::JpegError::MissingHuffmanTable::class: u8 +pub j2k_jpeg::error::JpegError::MissingHuffmanTable::component: u8 +pub j2k_jpeg::error::JpegError::MissingHuffmanTable::id: u8 +pub j2k_jpeg::error::JpegError::MissingMarker +pub j2k_jpeg::error::JpegError::MissingMarker::marker: j2k_jpeg::error::MarkerKind +pub j2k_jpeg::error::JpegError::MissingQuantTable +pub j2k_jpeg::error::JpegError::MissingQuantTable::component: u8 +pub j2k_jpeg::error::JpegError::MissingQuantTable::table_id: u8 +pub j2k_jpeg::error::JpegError::NotImplemented +pub j2k_jpeg::error::JpegError::NotImplemented::sof: j2k_jpeg::info::SofKind +pub j2k_jpeg::error::JpegError::OutputBufferTooSmall +pub j2k_jpeg::error::JpegError::OutputBufferTooSmall::provided: usize +pub j2k_jpeg::error::JpegError::OutputBufferTooSmall::required: usize +pub j2k_jpeg::error::JpegError::RectOutOfBounds +pub j2k_jpeg::error::JpegError::RectOutOfBounds::height: u32 +pub j2k_jpeg::error::JpegError::RectOutOfBounds::rect: j2k_jpeg::info::Rect +pub j2k_jpeg::error::JpegError::RectOutOfBounds::width: u32 +pub j2k_jpeg::error::JpegError::RestartMismatch +pub j2k_jpeg::error::JpegError::RestartMismatch::expected: u8 +pub j2k_jpeg::error::JpegError::RestartMismatch::found: u8 +pub j2k_jpeg::error::JpegError::RestartMismatch::offset: usize +pub j2k_jpeg::error::JpegError::RowSinkAborted +pub j2k_jpeg::error::JpegError::ScanFragmentsOverlap +pub j2k_jpeg::error::JpegError::ScanFragmentsOverlap::mcu: u32 +pub j2k_jpeg::error::JpegError::Truncated +pub j2k_jpeg::error::JpegError::Truncated::expected: usize +pub j2k_jpeg::error::JpegError::Truncated::offset: usize +pub j2k_jpeg::error::JpegError::UnexpectedEoi +pub j2k_jpeg::error::JpegError::UnexpectedEoi::mcu_at: u32 +pub j2k_jpeg::error::JpegError::UnexpectedEoi::mcu_total: u32 +pub j2k_jpeg::error::JpegError::UnexpectedMarker +pub j2k_jpeg::error::JpegError::UnexpectedMarker::expected: j2k_jpeg::error::MarkerKind +pub j2k_jpeg::error::JpegError::UnexpectedMarker::found: u8 +pub j2k_jpeg::error::JpegError::UnexpectedMarker::offset: usize +pub j2k_jpeg::error::JpegError::UnknownScanComponent +pub j2k_jpeg::error::JpegError::UnknownScanComponent::component: u8 +pub j2k_jpeg::error::JpegError::UnknownScanComponent::offset: usize +pub j2k_jpeg::error::JpegError::UnsupportedBitDepth +pub j2k_jpeg::error::JpegError::UnsupportedBitDepth::depth: u8 +pub j2k_jpeg::error::JpegError::UnsupportedColorSpace +pub j2k_jpeg::error::JpegError::UnsupportedColorSpace::color_space: j2k_jpeg::info::ColorSpace +pub j2k_jpeg::error::JpegError::UnsupportedComponentCount +pub j2k_jpeg::error::JpegError::UnsupportedComponentCount::count: u8 +pub j2k_jpeg::error::JpegError::UnsupportedPredictor +pub j2k_jpeg::error::JpegError::UnsupportedPredictor::predictor: u8 +pub j2k_jpeg::error::JpegError::UnsupportedSof +pub j2k_jpeg::error::JpegError::UnsupportedSof::marker: u8 +pub j2k_jpeg::error::JpegError::UnsupportedSof::reason: j2k_jpeg::error::UnsupportedReason +pub j2k_jpeg::error::JpegError::ZeroDimension +pub j2k_jpeg::error::JpegError::ZeroDimension::height: u16 +pub j2k_jpeg::error::JpegError::ZeroDimension::width: u16 +impl j2k_jpeg::error::JpegError +pub fn j2k_jpeg::error::JpegError::is_api_misuse(&self) -> bool +pub fn j2k_jpeg::error::JpegError::is_not_implemented(&self) -> bool +pub fn j2k_jpeg::error::JpegError::is_truncated(&self) -> bool +pub fn j2k_jpeg::error::JpegError::is_unsupported(&self) -> bool +pub fn j2k_jpeg::error::JpegError::offset(&self) -> core::option::Option +impl core::convert::From for j2k_jpeg::adapter::fast_packet::FastPacketError +pub fn j2k_jpeg::adapter::fast_packet::FastPacketError::from(j2k_jpeg::error::JpegError) -> Self +impl j2k_core::error::CodecError for j2k_jpeg::error::JpegError +pub fn j2k_jpeg::error::JpegError::is_buffer_error(&self) -> bool +pub fn j2k_jpeg::error::JpegError::is_not_implemented(&self) -> bool +pub fn j2k_jpeg::error::JpegError::is_truncated(&self) -> bool +pub fn j2k_jpeg::error::JpegError::is_unsupported(&self) -> bool +pub enum j2k_jpeg::error::MarkerKind +pub j2k_jpeg::error::MarkerKind::App14 +pub j2k_jpeg::error::MarkerKind::Dht +pub j2k_jpeg::error::MarkerKind::Dqt +pub j2k_jpeg::error::MarkerKind::Dri +pub j2k_jpeg::error::MarkerKind::Eoi +pub j2k_jpeg::error::MarkerKind::Other(u8) +pub j2k_jpeg::error::MarkerKind::Sof +pub j2k_jpeg::error::MarkerKind::Soi +pub j2k_jpeg::error::MarkerKind::Sos +pub enum j2k_jpeg::error::TableKind +pub j2k_jpeg::error::TableKind::HuffmanAc +pub j2k_jpeg::error::TableKind::HuffmanDc +pub j2k_jpeg::error::TableKind::Quant +pub enum j2k_jpeg::error::UnsupportedReason +pub j2k_jpeg::error::UnsupportedReason::ArithmeticAndHierarchical +pub j2k_jpeg::error::UnsupportedReason::ArithmeticCoding +pub j2k_jpeg::error::UnsupportedReason::DifferentialBaseline +pub j2k_jpeg::error::UnsupportedReason::Hierarchical +#[non_exhaustive] pub enum j2k_jpeg::error::Warning +pub j2k_jpeg::error::Warning::AdobeApp14Ambiguous +pub j2k_jpeg::error::Warning::AdobeApp14Ambiguous::raw_transform: u8 +pub j2k_jpeg::error::Warning::IccProfileIgnored +pub j2k_jpeg::error::Warning::IccProfileIgnored::size: usize +pub j2k_jpeg::error::Warning::MissingEoi +pub j2k_jpeg::error::Warning::NonstandardTables +pub j2k_jpeg::error::Warning::PrecisionClamped +pub j2k_jpeg::error::Warning::PrecisionClamped::from_bits: u8 +pub j2k_jpeg::error::Warning::PrecisionClamped::to_bits: u8 +pub j2k_jpeg::error::Warning::RestartRecovered +pub j2k_jpeg::error::Warning::RestartRecovered::offset: usize +pub j2k_jpeg::error::Warning::SofDimensionsPatched +pub j2k_jpeg::error::Warning::SofDimensionsPatched::from: (u16, u16) +pub j2k_jpeg::error::Warning::SofDimensionsPatched::to: (u16, u16) +pub j2k_jpeg::error::Warning::TableCacheMismatch +pub j2k_jpeg::error::Warning::TableCacheMismatch::id: u8 +pub j2k_jpeg::error::Warning::TableCacheMismatch::which: j2k_jpeg::error::TableKind +pub j2k_jpeg::error::Warning::UnknownAppMarker +pub j2k_jpeg::error::Warning::UnknownAppMarker::marker: u8 +pub j2k_jpeg::error::Warning::UnknownAppMarker::size: usize +pub j2k_jpeg::error::Warning::UnknownColorProfile +impl core::fmt::Display for j2k_jpeg::error::Warning +pub fn j2k_jpeg::error::Warning::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub mod j2k_jpeg::info +pub enum j2k_jpeg::info::ColorSpace +pub j2k_jpeg::info::ColorSpace::Cmyk +pub j2k_jpeg::info::ColorSpace::Grayscale +pub j2k_jpeg::info::ColorSpace::Rgb +pub j2k_jpeg::info::ColorSpace::YCbCr +pub j2k_jpeg::info::ColorSpace::Ycck +pub enum j2k_jpeg::info::ColorTransform +pub j2k_jpeg::info::ColorTransform::Auto +pub j2k_jpeg::info::ColorTransform::ForceRgb +pub j2k_jpeg::info::ColorTransform::ForceYCbCr +pub enum j2k_jpeg::info::SamplingFactorsError +pub j2k_jpeg::info::SamplingFactorsError::Empty +pub j2k_jpeg::info::SamplingFactorsError::InvalidSampling +pub j2k_jpeg::info::SamplingFactorsError::InvalidSampling::component: usize +pub j2k_jpeg::info::SamplingFactorsError::InvalidSampling::h: u8 +pub j2k_jpeg::info::SamplingFactorsError::InvalidSampling::v: u8 +pub j2k_jpeg::info::SamplingFactorsError::TooManyComponents +pub j2k_jpeg::info::SamplingFactorsError::TooManyComponents::count: usize +pub enum j2k_jpeg::info::SofKind +pub j2k_jpeg::info::SofKind::Baseline8 +pub j2k_jpeg::info::SofKind::Extended12 +pub j2k_jpeg::info::SofKind::Extended8 +pub j2k_jpeg::info::SofKind::Lossless +pub j2k_jpeg::info::SofKind::Progressive12 +pub j2k_jpeg::info::SofKind::Progressive8 +pub struct j2k_jpeg::info::DecodeOptions +impl j2k_jpeg::info::DecodeOptions +pub fn j2k_jpeg::info::DecodeOptions::color_transform(&self) -> j2k_jpeg::info::ColorTransform +pub fn j2k_jpeg::info::DecodeOptions::set_color_transform(&mut self, j2k_jpeg::info::ColorTransform) +pub fn j2k_jpeg::info::DecodeOptions::with_color_transform(self, j2k_jpeg::info::ColorTransform) -> Self +impl core::default::Default for j2k_jpeg::info::DecodeOptions +pub fn j2k_jpeg::info::DecodeOptions::default() -> Self +pub struct j2k_jpeg::info::Info +pub j2k_jpeg::info::Info::bit_depth: u8 +pub j2k_jpeg::info::Info::color_space: j2k_jpeg::info::ColorSpace +pub j2k_jpeg::info::Info::dimensions: (u32, u32) +pub j2k_jpeg::info::Info::mcu_geometry: j2k_jpeg::info::McuGeometry +pub j2k_jpeg::info::Info::restart_interval: core::option::Option +pub j2k_jpeg::info::Info::sampling: j2k_jpeg::info::SamplingFactors +pub j2k_jpeg::info::Info::scan_count: u16 +pub j2k_jpeg::info::Info::sof_kind: j2k_jpeg::info::SofKind +impl j2k_jpeg::info::Info +pub fn j2k_jpeg::info::Info::to_core_info(&self) -> j2k_core::types::Info +pub struct j2k_jpeg::info::McuGeometry +pub j2k_jpeg::info::McuGeometry::columns: u32 +pub j2k_jpeg::info::McuGeometry::count: u32 +pub j2k_jpeg::info::McuGeometry::height: u32 +pub j2k_jpeg::info::McuGeometry::rows: u32 +pub j2k_jpeg::info::McuGeometry::width: u32 +pub struct j2k_jpeg::info::Rect +pub j2k_jpeg::info::Rect::h: u32 +pub j2k_jpeg::info::Rect::w: u32 +pub j2k_jpeg::info::Rect::x: u32 +pub j2k_jpeg::info::Rect::y: u32 +impl j2k_jpeg::info::Rect +pub fn j2k_jpeg::info::Rect::full((u32, u32)) -> Self +pub fn j2k_jpeg::info::Rect::is_within(&self, (u32, u32)) -> bool +impl core::convert::From for j2k_jpeg::info::Rect +pub fn j2k_jpeg::info::Rect::from(j2k_core::types::Rect) -> Self +impl core::convert::From for j2k_core::types::Rect +pub fn j2k_core::types::Rect::from(j2k_jpeg::info::Rect) -> Self +pub struct j2k_jpeg::info::RestartIndex +pub j2k_jpeg::info::RestartIndex::interval_mcus: u32 +pub j2k_jpeg::info::RestartIndex::scan_data_offset: usize +pub j2k_jpeg::info::RestartIndex::segments: alloc::vec::Vec +pub struct j2k_jpeg::info::RestartSegment +pub j2k_jpeg::info::RestartSegment::entropy_offset: usize +pub j2k_jpeg::info::RestartSegment::marker: core::option::Option +pub j2k_jpeg::info::RestartSegment::marker_offset: core::option::Option +pub j2k_jpeg::info::RestartSegment::start_mcu: u32 +pub struct j2k_jpeg::info::SamplingFactors +pub j2k_jpeg::info::SamplingFactors::max_h: u8 +pub j2k_jpeg::info::SamplingFactors::max_v: u8 +impl j2k_jpeg::info::SamplingFactors +pub fn j2k_jpeg::info::SamplingFactors::component(&self, usize) -> core::option::Option<(u8, u8)> +pub fn j2k_jpeg::info::SamplingFactors::components(&self) -> &[(u8, u8)] +pub fn j2k_jpeg::info::SamplingFactors::from_components(&[(u8, u8)]) -> core::result::Result +pub fn j2k_jpeg::info::SamplingFactors::is_empty(&self) -> bool +pub fn j2k_jpeg::info::SamplingFactors::len(&self) -> usize +pub mod j2k_jpeg::output_buffer +pub struct j2k_jpeg::output_buffer::JpegOutputBuffer +impl j2k_jpeg::output_buffer::JpegOutputBuffer +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::as_mut_slice(&mut self) -> &mut [u8] +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::as_slice(&self) -> &[u8] +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::capacity(&self) -> usize +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::dimensions(&self) -> (u32, u32) +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::is_empty(&self) -> bool +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::len(&self) -> usize +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::new((u32, u32), j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::new_with_max_bytes((u32, u32), j2k_core::pixel::PixelFormat, usize) -> core::result::Result +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::pixel_format(&self) -> j2k_core::pixel::PixelFormat +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::resize(&mut self, (u32, u32), j2k_core::pixel::PixelFormat) -> core::result::Result<(), j2k_core::error::BufferError> +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::resize_with_max_bytes(&mut self, (u32, u32), j2k_core::pixel::PixelFormat, usize) -> core::result::Result<(), j2k_core::error::BufferError> +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::resize_with_stride(&mut self, (u32, u32), usize, j2k_core::pixel::PixelFormat) -> core::result::Result<(), j2k_core::error::BufferError> +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::resize_with_stride_with_max_bytes(&mut self, (u32, u32), usize, j2k_core::pixel::PixelFormat, usize) -> core::result::Result<(), j2k_core::error::BufferError> +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::stride(&self) -> usize +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::with_stride((u32, u32), usize, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::with_stride_with_max_bytes((u32, u32), usize, j2k_core::pixel::PixelFormat, usize) -> core::result::Result +pub mod j2k_jpeg::segment +pub enum j2k_jpeg::segment::DuplicateTablePolicy +pub j2k_jpeg::segment::DuplicateTablePolicy::AllowIdentical +pub j2k_jpeg::segment::DuplicateTablePolicy::RejectConflicting +pub enum j2k_jpeg::segment::PreparedJpeg<'a> +pub j2k_jpeg::segment::PreparedJpeg::Borrowed(&'a [u8]) +pub j2k_jpeg::segment::PreparedJpeg::Owned(alloc::vec::Vec) +impl j2k_jpeg::segment::PreparedJpeg<'_> +pub fn j2k_jpeg::segment::PreparedJpeg<'_>::as_bytes(&self) -> &[u8] +impl core::convert::AsRef<[u8]> for j2k_jpeg::segment::PreparedJpeg<'_> +pub fn j2k_jpeg::segment::PreparedJpeg<'_>::as_ref(&self) -> &[u8] +pub struct j2k_jpeg::segment::JpegScanRanges +pub j2k_jpeg::segment::JpegScanRanges::entropy_range: core::ops::range::Range +pub j2k_jpeg::segment::JpegScanRanges::eoi_marker_offset: core::option::Option +pub j2k_jpeg::segment::JpegScanRanges::sos_marker_offset: usize +pub j2k_jpeg::segment::JpegScanRanges::sos_payload_range: core::ops::range::Range +pub struct j2k_jpeg::segment::JpegSegment<'a> +pub j2k_jpeg::segment::JpegSegment::marker: u8 +pub j2k_jpeg::segment::JpegSegment::marker_offset: usize +pub j2k_jpeg::segment::JpegSegment::payload: &'a [u8] +pub j2k_jpeg::segment::JpegSegment::payload_offset: usize +pub struct j2k_jpeg::segment::JpegSegmentIter<'a> +impl<'a> core::iter::traits::iterator::Iterator for j2k_jpeg::segment::JpegSegmentIter<'a> +pub type j2k_jpeg::segment::JpegSegmentIter<'a>::Item = core::result::Result, j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::segment::JpegSegmentIter<'a>::next(&mut self) -> core::option::Option +pub struct j2k_jpeg::segment::JpegSofInfo +pub j2k_jpeg::segment::JpegSofInfo::bit_depth: u8 +pub j2k_jpeg::segment::JpegSofInfo::component_ids: alloc::vec::Vec +pub j2k_jpeg::segment::JpegSofInfo::dimensions: (u16, u16) +pub j2k_jpeg::segment::JpegSofInfo::marker: u8 +pub j2k_jpeg::segment::JpegSofInfo::quant_table_ids: alloc::vec::Vec +pub j2k_jpeg::segment::JpegSofInfo::sampling: j2k_jpeg::info::SamplingFactors +pub j2k_jpeg::segment::JpegSofInfo::sof_kind: j2k_jpeg::info::SofKind +pub struct j2k_jpeg::segment::JpegTilePrepareOptions +pub j2k_jpeg::segment::JpegTilePrepareOptions::duplicate_table_policy: j2k_jpeg::segment::DuplicateTablePolicy +pub j2k_jpeg::segment::JpegTilePrepareOptions::expected_dimensions: core::option::Option<(u16, u16)> +pub j2k_jpeg::segment::JpegTilePrepareOptions::repair_zero_sof_dimensions: bool +pub j2k_jpeg::segment::JpegTilePrepareOptions::validate_restart_markers: bool +impl core::default::Default for j2k_jpeg::segment::JpegTilePrepareOptions +pub fn j2k_jpeg::segment::JpegTilePrepareOptions::default() -> Self +pub fn j2k_jpeg::segment::find_scan_ranges(&[u8]) -> core::result::Result +pub const fn j2k_jpeg::segment::is_sof_marker(u8) -> bool +pub fn j2k_jpeg::segment::iter_segments(&[u8]) -> j2k_jpeg::segment::JpegSegmentIter<'_> +pub fn j2k_jpeg::segment::parse_dri(&[u8]) -> core::result::Result, j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::segment::parse_sof_info(u8, &[u8]) -> core::result::Result +pub fn j2k_jpeg::segment::prepare_tiff_jpeg_tile<'a>(&'a [u8], core::option::Option<&'a [u8]>, j2k_jpeg::segment::JpegTilePrepareOptions) -> core::result::Result, j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::segment::rewrite_sof_dimensions(&[u8], (u16, u16)) -> core::result::Result, j2k_jpeg::error::JpegError> +pub mod j2k_jpeg::transcode +pub enum j2k_jpeg::transcode::JpegDctCodingMode +pub j2k_jpeg::transcode::JpegDctCodingMode::BaselineSequential +pub j2k_jpeg::transcode::JpegDctCodingMode::Progressive +#[non_exhaustive] pub struct j2k_jpeg::transcode::DctExtractOptions +pub j2k_jpeg::transcode::DctExtractOptions::retain_quantized_blocks: bool +impl j2k_jpeg::transcode::DctExtractOptions +pub const fn j2k_jpeg::transcode::DctExtractOptions::dequantized_only() -> Self +impl core::default::Default for j2k_jpeg::transcode::DctExtractOptions +pub fn j2k_jpeg::transcode::DctExtractOptions::default() -> Self +pub struct j2k_jpeg::transcode::JpegDctComponent +pub j2k_jpeg::transcode::JpegDctComponent::block_cols: u32 +pub j2k_jpeg::transcode::JpegDctComponent::block_rows: u32 +pub j2k_jpeg::transcode::JpegDctComponent::component_index: usize +pub j2k_jpeg::transcode::JpegDctComponent::dequantized_blocks: alloc::vec::Vec<[i16; 64]> +pub j2k_jpeg::transcode::JpegDctComponent::h_samp: u8 +pub j2k_jpeg::transcode::JpegDctComponent::height: u32 +pub j2k_jpeg::transcode::JpegDctComponent::quant_table: [u16; 64] +pub j2k_jpeg::transcode::JpegDctComponent::quantized_blocks: alloc::vec::Vec<[i16; 64]> +pub j2k_jpeg::transcode::JpegDctComponent::v_samp: u8 +pub j2k_jpeg::transcode::JpegDctComponent::width: u32 +pub struct j2k_jpeg::transcode::JpegDctImage +pub j2k_jpeg::transcode::JpegDctImage::coding_mode: j2k_jpeg::transcode::JpegDctCodingMode +pub j2k_jpeg::transcode::JpegDctImage::color_space: j2k_jpeg::info::ColorSpace +pub j2k_jpeg::transcode::JpegDctImage::components: alloc::vec::Vec +pub j2k_jpeg::transcode::JpegDctImage::height: u32 +pub j2k_jpeg::transcode::JpegDctImage::restart_index: core::option::Option +pub j2k_jpeg::transcode::JpegDctImage::scan_count: u16 +pub j2k_jpeg::transcode::JpegDctImage::width: u32 +pub fn j2k_jpeg::transcode::encode_baseline_dct_image(&j2k_jpeg::transcode::JpegDctImage) -> core::result::Result, j2k_jpeg::encoder::JpegEncodeError> +pub fn j2k_jpeg::transcode::extract_dct_blocks(&[u8], j2k_jpeg::transcode::DctExtractOptions) -> core::result::Result +pub fn j2k_jpeg::transcode::idct_islow_block(&[i16; 64]) -> [u8; 64] +pub enum j2k_jpeg::BuilderConflictReason +pub j2k_jpeg::BuilderConflictReason::InputAndScanFragments +pub j2k_jpeg::BuilderConflictReason::NoInput +pub j2k_jpeg::BuilderConflictReason::ScanFragmentsEmpty +pub enum j2k_jpeg::ColorSpace +pub j2k_jpeg::ColorSpace::Cmyk +pub j2k_jpeg::ColorSpace::Grayscale +pub j2k_jpeg::ColorSpace::Rgb +pub j2k_jpeg::ColorSpace::YCbCr +pub j2k_jpeg::ColorSpace::Ycck +pub enum j2k_jpeg::ColorTransform +pub j2k_jpeg::ColorTransform::Auto +pub j2k_jpeg::ColorTransform::ForceRgb +pub j2k_jpeg::ColorTransform::ForceYCbCr +pub enum j2k_jpeg::DuplicateTablePolicy +pub j2k_jpeg::DuplicateTablePolicy::AllowIdentical +pub j2k_jpeg::DuplicateTablePolicy::RejectConflicting +pub enum j2k_jpeg::HuffmanFailure +pub j2k_jpeg::HuffmanFailure::CodeOverflow +pub j2k_jpeg::HuffmanFailure::InvalidSymbol +pub j2k_jpeg::HuffmanFailure::TableExhausted +pub enum j2k_jpeg::JpegBackend +pub j2k_jpeg::JpegBackend::Auto +pub j2k_jpeg::JpegBackend::Cpu +pub j2k_jpeg::JpegBackend::Metal +pub enum j2k_jpeg::JpegDecodeOp +pub j2k_jpeg::JpegDecodeOp::Full +pub j2k_jpeg::JpegDecodeOp::Region(j2k_jpeg::info::Rect) +pub j2k_jpeg::JpegDecodeOp::RegionScaled +pub j2k_jpeg::JpegDecodeOp::RegionScaled::roi: j2k_jpeg::info::Rect +pub j2k_jpeg::JpegDecodeOp::RegionScaled::scale: j2k_core::scale::Downscale +pub j2k_jpeg::JpegDecodeOp::Scaled(j2k_core::scale::Downscale) +pub enum j2k_jpeg::JpegEncodeError +pub j2k_jpeg::JpegEncodeError::DimensionsTooLarge +pub j2k_jpeg::JpegEncodeError::DimensionsTooLarge::height: u32 +pub j2k_jpeg::JpegEncodeError::DimensionsTooLarge::width: u32 +pub j2k_jpeg::JpegEncodeError::EmptyDimensions +pub j2k_jpeg::JpegEncodeError::IncompatibleSubsampling +pub j2k_jpeg::JpegEncodeError::IncompatibleSubsampling::samples: &'static str +pub j2k_jpeg::JpegEncodeError::IncompatibleSubsampling::subsampling: j2k_jpeg::encoder::JpegSubsampling +pub j2k_jpeg::JpegEncodeError::Internal(alloc::string::String) +pub j2k_jpeg::JpegEncodeError::InvalidRestartInterval +pub j2k_jpeg::JpegEncodeError::MissingHuffmanCode +pub j2k_jpeg::JpegEncodeError::MissingHuffmanCode::symbol: u8 +pub j2k_jpeg::JpegEncodeError::SampleLength +pub j2k_jpeg::JpegEncodeError::SampleLength::actual: usize +pub j2k_jpeg::JpegEncodeError::SampleLength::expected: usize +pub j2k_jpeg::JpegEncodeError::SegmentTooLarge +pub j2k_jpeg::JpegEncodeError::SegmentTooLarge::name: &'static str +pub j2k_jpeg::JpegEncodeError::UnsupportedBackend +pub j2k_jpeg::JpegEncodeError::UnsupportedBackend::backend: j2k_jpeg::encoder::JpegBackend +#[non_exhaustive] pub enum j2k_jpeg::JpegError +pub j2k_jpeg::JpegError::BuilderConflict +pub j2k_jpeg::JpegError::BuilderConflict::reason: j2k_jpeg::error::BuilderConflictReason +pub j2k_jpeg::JpegError::CoefficientOverflow +pub j2k_jpeg::JpegError::CoefficientOverflow::component: u8 +pub j2k_jpeg::JpegError::CoefficientOverflow::mcu: u32 +pub j2k_jpeg::JpegError::ConflictingDri +pub j2k_jpeg::JpegError::ConflictingDri::existing: u16 +pub j2k_jpeg::JpegError::ConflictingDri::new: u16 +pub j2k_jpeg::JpegError::ConflictingDri::offset: usize +pub j2k_jpeg::JpegError::ConflictingDuplicateTable +pub j2k_jpeg::JpegError::ConflictingDuplicateTable::id: u8 +pub j2k_jpeg::JpegError::ConflictingDuplicateTable::offset: usize +pub j2k_jpeg::JpegError::ConflictingDuplicateTable::table: j2k_jpeg::error::TableKind +pub j2k_jpeg::JpegError::ConflictingExpectedDimensions +pub j2k_jpeg::JpegError::ConflictingExpectedDimensions::actual: (u16, u16) +pub j2k_jpeg::JpegError::ConflictingExpectedDimensions::expected: (u16, u16) +pub j2k_jpeg::JpegError::ConflictingExpectedDimensions::offset: usize +pub j2k_jpeg::JpegError::DimensionOverflow +pub j2k_jpeg::JpegError::DimensionOverflow::height: u32 +pub j2k_jpeg::JpegError::DimensionOverflow::width: u32 +pub j2k_jpeg::JpegError::DownscaleUnsupported +pub j2k_jpeg::JpegError::DownscaleUnsupported::sof: j2k_jpeg::info::SofKind +pub j2k_jpeg::JpegError::DuplicateMarker +pub j2k_jpeg::JpegError::DuplicateMarker::marker: j2k_jpeg::error::MarkerKind +pub j2k_jpeg::JpegError::DuplicateMarker::offset: usize +pub j2k_jpeg::JpegError::DuplicateScanComponent +pub j2k_jpeg::JpegError::DuplicateScanComponent::component: u8 +pub j2k_jpeg::JpegError::DuplicateScanComponent::offset: usize +pub j2k_jpeg::JpegError::ExpectedDimensionsRequired +pub j2k_jpeg::JpegError::ExpectedDimensionsRequired::offset: usize +pub j2k_jpeg::JpegError::HuffmanDecode +pub j2k_jpeg::JpegError::HuffmanDecode::mcu: u32 +pub j2k_jpeg::JpegError::HuffmanDecode::reason: j2k_jpeg::error::HuffmanFailure +pub j2k_jpeg::JpegError::InvalidJpegAssembly +pub j2k_jpeg::JpegError::InvalidJpegAssembly::offset: usize +pub j2k_jpeg::JpegError::InvalidJpegAssembly::reason: &'static str +pub j2k_jpeg::JpegError::InvalidMarker +pub j2k_jpeg::JpegError::InvalidMarker::marker: u8 +pub j2k_jpeg::JpegError::InvalidMarker::offset: usize +pub j2k_jpeg::JpegError::InvalidSampling +pub j2k_jpeg::JpegError::InvalidSampling::component: u8 +pub j2k_jpeg::JpegError::InvalidSampling::h: u8 +pub j2k_jpeg::JpegError::InvalidSampling::v: u8 +pub j2k_jpeg::JpegError::InvalidScanParameters +pub j2k_jpeg::JpegError::InvalidScanParameters::ah: u8 +pub j2k_jpeg::JpegError::InvalidScanParameters::al: u8 +pub j2k_jpeg::JpegError::InvalidScanParameters::offset: usize +pub j2k_jpeg::JpegError::InvalidScanParameters::se: u8 +pub j2k_jpeg::JpegError::InvalidScanParameters::ss: u8 +pub j2k_jpeg::JpegError::InvalidSegmentLength +pub j2k_jpeg::JpegError::InvalidSegmentLength::length: u16 +pub j2k_jpeg::JpegError::InvalidSegmentLength::marker: u8 +pub j2k_jpeg::JpegError::InvalidSegmentLength::offset: usize +pub j2k_jpeg::JpegError::InvalidSequentialComponentSet +pub j2k_jpeg::JpegError::InvalidSequentialComponentSet::expected: u8 +pub j2k_jpeg::JpegError::InvalidSequentialComponentSet::found: u8 +pub j2k_jpeg::JpegError::InvalidSequentialComponentSet::offset: usize +pub j2k_jpeg::JpegError::InvalidSequentialScanCount +pub j2k_jpeg::JpegError::InvalidSequentialScanCount::count: u16 +pub j2k_jpeg::JpegError::InvalidSequentialScanCount::sof: j2k_jpeg::info::SofKind +pub j2k_jpeg::JpegError::InvalidStride +pub j2k_jpeg::JpegError::InvalidStride::row: usize +pub j2k_jpeg::JpegError::InvalidStride::stride: usize +pub j2k_jpeg::JpegError::MemoryCapExceeded +pub j2k_jpeg::JpegError::MemoryCapExceeded::cap: usize +pub j2k_jpeg::JpegError::MemoryCapExceeded::requested: usize +pub j2k_jpeg::JpegError::MissingHuffmanTable +pub j2k_jpeg::JpegError::MissingHuffmanTable::class: u8 +pub j2k_jpeg::JpegError::MissingHuffmanTable::component: u8 +pub j2k_jpeg::JpegError::MissingHuffmanTable::id: u8 +pub j2k_jpeg::JpegError::MissingMarker +pub j2k_jpeg::JpegError::MissingMarker::marker: j2k_jpeg::error::MarkerKind +pub j2k_jpeg::JpegError::MissingQuantTable +pub j2k_jpeg::JpegError::MissingQuantTable::component: u8 +pub j2k_jpeg::JpegError::MissingQuantTable::table_id: u8 +pub j2k_jpeg::JpegError::NotImplemented +pub j2k_jpeg::JpegError::NotImplemented::sof: j2k_jpeg::info::SofKind +pub j2k_jpeg::JpegError::OutputBufferTooSmall +pub j2k_jpeg::JpegError::OutputBufferTooSmall::provided: usize +pub j2k_jpeg::JpegError::OutputBufferTooSmall::required: usize +pub j2k_jpeg::JpegError::RectOutOfBounds +pub j2k_jpeg::JpegError::RectOutOfBounds::height: u32 +pub j2k_jpeg::JpegError::RectOutOfBounds::rect: j2k_jpeg::info::Rect +pub j2k_jpeg::JpegError::RectOutOfBounds::width: u32 +pub j2k_jpeg::JpegError::RestartMismatch +pub j2k_jpeg::JpegError::RestartMismatch::expected: u8 +pub j2k_jpeg::JpegError::RestartMismatch::found: u8 +pub j2k_jpeg::JpegError::RestartMismatch::offset: usize +pub j2k_jpeg::JpegError::RowSinkAborted +pub j2k_jpeg::JpegError::ScanFragmentsOverlap +pub j2k_jpeg::JpegError::ScanFragmentsOverlap::mcu: u32 +pub j2k_jpeg::JpegError::Truncated +pub j2k_jpeg::JpegError::Truncated::expected: usize +pub j2k_jpeg::JpegError::Truncated::offset: usize +pub j2k_jpeg::JpegError::UnexpectedEoi +pub j2k_jpeg::JpegError::UnexpectedEoi::mcu_at: u32 +pub j2k_jpeg::JpegError::UnexpectedEoi::mcu_total: u32 +pub j2k_jpeg::JpegError::UnexpectedMarker +pub j2k_jpeg::JpegError::UnexpectedMarker::expected: j2k_jpeg::error::MarkerKind +pub j2k_jpeg::JpegError::UnexpectedMarker::found: u8 +pub j2k_jpeg::JpegError::UnexpectedMarker::offset: usize +pub j2k_jpeg::JpegError::UnknownScanComponent +pub j2k_jpeg::JpegError::UnknownScanComponent::component: u8 +pub j2k_jpeg::JpegError::UnknownScanComponent::offset: usize +pub j2k_jpeg::JpegError::UnsupportedBitDepth +pub j2k_jpeg::JpegError::UnsupportedBitDepth::depth: u8 +pub j2k_jpeg::JpegError::UnsupportedColorSpace +pub j2k_jpeg::JpegError::UnsupportedColorSpace::color_space: j2k_jpeg::info::ColorSpace +pub j2k_jpeg::JpegError::UnsupportedComponentCount +pub j2k_jpeg::JpegError::UnsupportedComponentCount::count: u8 +pub j2k_jpeg::JpegError::UnsupportedPredictor +pub j2k_jpeg::JpegError::UnsupportedPredictor::predictor: u8 +pub j2k_jpeg::JpegError::UnsupportedSof +pub j2k_jpeg::JpegError::UnsupportedSof::marker: u8 +pub j2k_jpeg::JpegError::UnsupportedSof::reason: j2k_jpeg::error::UnsupportedReason +pub j2k_jpeg::JpegError::ZeroDimension +pub j2k_jpeg::JpegError::ZeroDimension::height: u16 +pub j2k_jpeg::JpegError::ZeroDimension::width: u16 +impl j2k_jpeg::error::JpegError +pub fn j2k_jpeg::error::JpegError::is_api_misuse(&self) -> bool +pub fn j2k_jpeg::error::JpegError::is_not_implemented(&self) -> bool +pub fn j2k_jpeg::error::JpegError::is_truncated(&self) -> bool +pub fn j2k_jpeg::error::JpegError::is_unsupported(&self) -> bool +pub fn j2k_jpeg::error::JpegError::offset(&self) -> core::option::Option +impl core::convert::From for j2k_jpeg::adapter::fast_packet::FastPacketError +pub fn j2k_jpeg::adapter::fast_packet::FastPacketError::from(j2k_jpeg::error::JpegError) -> Self +impl j2k_core::error::CodecError for j2k_jpeg::error::JpegError +pub fn j2k_jpeg::error::JpegError::is_buffer_error(&self) -> bool +pub fn j2k_jpeg::error::JpegError::is_not_implemented(&self) -> bool +pub fn j2k_jpeg::error::JpegError::is_truncated(&self) -> bool +pub fn j2k_jpeg::error::JpegError::is_unsupported(&self) -> bool +pub enum j2k_jpeg::JpegResolvedDecodePath +pub j2k_jpeg::JpegResolvedDecodePath::CpuHost +pub j2k_jpeg::JpegResolvedDecodePath::MetalFast +pub j2k_jpeg::JpegResolvedDecodePath::OwnedCudaRgb8 +pub j2k_jpeg::JpegResolvedDecodePath::Rejected +pub j2k_jpeg::JpegResolvedDecodePath::Rejected::backend: j2k_core::backend::BackendRequest +pub j2k_jpeg::JpegResolvedDecodePath::Rejected::reason: &'static str +pub enum j2k_jpeg::JpegSamples<'a> +pub j2k_jpeg::JpegSamples::Gray8 +pub j2k_jpeg::JpegSamples::Gray8::data: &'a [u8] +pub j2k_jpeg::JpegSamples::Gray8::height: u32 +pub j2k_jpeg::JpegSamples::Gray8::width: u32 +pub j2k_jpeg::JpegSamples::Rgb8 +pub j2k_jpeg::JpegSamples::Rgb8::data: &'a [u8] +pub j2k_jpeg::JpegSamples::Rgb8::height: u32 +pub j2k_jpeg::JpegSamples::Rgb8::width: u32 +pub enum j2k_jpeg::JpegSubsampling +pub j2k_jpeg::JpegSubsampling::Gray +pub j2k_jpeg::JpegSubsampling::Ybr420 +pub j2k_jpeg::JpegSubsampling::Ybr422 +pub j2k_jpeg::JpegSubsampling::Ybr444 +pub enum j2k_jpeg::MarkerKind +pub j2k_jpeg::MarkerKind::App14 +pub j2k_jpeg::MarkerKind::Dht +pub j2k_jpeg::MarkerKind::Dqt +pub j2k_jpeg::MarkerKind::Dri +pub j2k_jpeg::MarkerKind::Eoi +pub j2k_jpeg::MarkerKind::Other(u8) +pub j2k_jpeg::MarkerKind::Sof +pub j2k_jpeg::MarkerKind::Soi +pub j2k_jpeg::MarkerKind::Sos +pub enum j2k_jpeg::PreparedJpeg<'a> +pub j2k_jpeg::PreparedJpeg::Borrowed(&'a [u8]) +pub j2k_jpeg::PreparedJpeg::Owned(alloc::vec::Vec) +impl j2k_jpeg::segment::PreparedJpeg<'_> +pub fn j2k_jpeg::segment::PreparedJpeg<'_>::as_bytes(&self) -> &[u8] +impl core::convert::AsRef<[u8]> for j2k_jpeg::segment::PreparedJpeg<'_> +pub fn j2k_jpeg::segment::PreparedJpeg<'_>::as_ref(&self) -> &[u8] +pub enum j2k_jpeg::SamplingFactorsError +pub j2k_jpeg::SamplingFactorsError::Empty +pub j2k_jpeg::SamplingFactorsError::InvalidSampling +pub j2k_jpeg::SamplingFactorsError::InvalidSampling::component: usize +pub j2k_jpeg::SamplingFactorsError::InvalidSampling::h: u8 +pub j2k_jpeg::SamplingFactorsError::InvalidSampling::v: u8 +pub j2k_jpeg::SamplingFactorsError::TooManyComponents +pub j2k_jpeg::SamplingFactorsError::TooManyComponents::count: usize +pub enum j2k_jpeg::SofKind +pub j2k_jpeg::SofKind::Baseline8 +pub j2k_jpeg::SofKind::Extended12 +pub j2k_jpeg::SofKind::Extended8 +pub j2k_jpeg::SofKind::Lossless +pub j2k_jpeg::SofKind::Progressive12 +pub j2k_jpeg::SofKind::Progressive8 +pub enum j2k_jpeg::TableKind +pub j2k_jpeg::TableKind::HuffmanAc +pub j2k_jpeg::TableKind::HuffmanDc +pub j2k_jpeg::TableKind::Quant +pub enum j2k_jpeg::UnsupportedReason +pub j2k_jpeg::UnsupportedReason::ArithmeticAndHierarchical +pub j2k_jpeg::UnsupportedReason::ArithmeticCoding +pub j2k_jpeg::UnsupportedReason::DifferentialBaseline +pub j2k_jpeg::UnsupportedReason::Hierarchical +#[non_exhaustive] pub enum j2k_jpeg::Warning +pub j2k_jpeg::Warning::AdobeApp14Ambiguous +pub j2k_jpeg::Warning::AdobeApp14Ambiguous::raw_transform: u8 +pub j2k_jpeg::Warning::IccProfileIgnored +pub j2k_jpeg::Warning::IccProfileIgnored::size: usize +pub j2k_jpeg::Warning::MissingEoi +pub j2k_jpeg::Warning::NonstandardTables +pub j2k_jpeg::Warning::PrecisionClamped +pub j2k_jpeg::Warning::PrecisionClamped::from_bits: u8 +pub j2k_jpeg::Warning::PrecisionClamped::to_bits: u8 +pub j2k_jpeg::Warning::RestartRecovered +pub j2k_jpeg::Warning::RestartRecovered::offset: usize +pub j2k_jpeg::Warning::SofDimensionsPatched +pub j2k_jpeg::Warning::SofDimensionsPatched::from: (u16, u16) +pub j2k_jpeg::Warning::SofDimensionsPatched::to: (u16, u16) +pub j2k_jpeg::Warning::TableCacheMismatch +pub j2k_jpeg::Warning::TableCacheMismatch::id: u8 +pub j2k_jpeg::Warning::TableCacheMismatch::which: j2k_jpeg::error::TableKind +pub j2k_jpeg::Warning::UnknownAppMarker +pub j2k_jpeg::Warning::UnknownAppMarker::marker: u8 +pub j2k_jpeg::Warning::UnknownAppMarker::size: usize +pub j2k_jpeg::Warning::UnknownColorProfile +impl core::fmt::Display for j2k_jpeg::error::Warning +pub fn j2k_jpeg::error::Warning::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_jpeg::DecodeOptions +impl j2k_jpeg::info::DecodeOptions +pub fn j2k_jpeg::info::DecodeOptions::color_transform(&self) -> j2k_jpeg::info::ColorTransform +pub fn j2k_jpeg::info::DecodeOptions::set_color_transform(&mut self, j2k_jpeg::info::ColorTransform) +pub fn j2k_jpeg::info::DecodeOptions::with_color_transform(self, j2k_jpeg::info::ColorTransform) -> Self +impl core::default::Default for j2k_jpeg::info::DecodeOptions +pub fn j2k_jpeg::info::DecodeOptions::default() -> Self +pub struct j2k_jpeg::DecodeOutcome +pub j2k_jpeg::DecodeOutcome::decoded: j2k_jpeg::info::Rect +pub j2k_jpeg::DecodeOutcome::warnings: alloc::vec::Vec +impl core::convert::From for j2k_core::types::DecodeOutcome +pub fn j2k_core::types::DecodeOutcome::from(j2k_jpeg::decoder::DecodeOutcome) -> Self +pub struct j2k_jpeg::DecodedTile +pub j2k_jpeg::DecodedTile::decoded: j2k_jpeg::info::Rect +pub j2k_jpeg::DecodedTile::dimensions: (u32, u32) +pub j2k_jpeg::DecodedTile::warnings: alloc::vec::Vec +pub struct j2k_jpeg::Decoder<'a> +impl j2k_jpeg::decoder::Decoder<'_> +pub fn j2k_jpeg::decoder::Decoder<'_>::decode_tile(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut S) -> core::result::Result where S: j2k_core::row_sink::RowSink +impl<'a> j2k_jpeg::decoder::Decoder<'a> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode(&self, j2k_core::pixel::PixelFormat) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_component_rows_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut W) -> core::result::Result where W: j2k_jpeg::decoder::ComponentRowWriter +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_into(&self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_into_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region(&self, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_component_rows_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut W, j2k_jpeg::info::Rect, j2k_core::scale::Downscale) -> core::result::Result where W: j2k_jpeg::decoder::ComponentRowWriter +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_into(&self, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_into_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_rgba8_into_with_alpha(&self, &mut [u8], usize, j2k_jpeg::info::Rect, u8) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_rgba8_into_with_alpha_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_jpeg::info::Rect, u8) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_scaled(&self, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_core::scale::Downscale) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_scaled_into(&self, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_scaled_into_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_scaled_with_scratch(&self, &mut j2k_jpeg::ScratchPool, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_core::scale::Downscale) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_with_scratch(&self, &mut j2k_jpeg::ScratchPool, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_rgba8_into_with_alpha(&self, &mut [u8], usize, u8) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_rgba8_into_with_alpha_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, u8) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_rows(&self, &mut S) -> core::result::Result where S: j2k_core::row_sink::RowSink +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_rows_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut S) -> core::result::Result where S: j2k_core::row_sink::RowSink +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_scaled(&self, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_scaled_into(&self, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_scaled_into_with_scratch(&self, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_scaled_with_scratch(&self, &mut j2k_jpeg::ScratchPool, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_with_scratch(&self, &mut j2k_jpeg::ScratchPool, j2k_core::pixel::PixelFormat) -> core::result::Result<(alloc::vec::Vec, j2k_jpeg::decoder::DecodeOutcome), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decoder::Decoder<'a>::from_view(j2k_jpeg::decoder::JpegView<'a>) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::from_view_in_context(j2k_jpeg::decoder::JpegView<'a>, &mut j2k_jpeg::context::DecoderContext) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::info(&self) -> &j2k_jpeg::info::Info +pub fn j2k_jpeg::decoder::Decoder<'a>::inspect(&'a [u8]) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::inspect_with_options(&'a [u8], j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::new(&'a [u8]) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::new_with_options(&'a [u8], j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::passthrough_candidate(&self) -> core::option::Option> +pub fn j2k_jpeg::decoder::Decoder<'a>::restart_index(&self) -> core::result::Result, j2k_jpeg::error::JpegError> +impl j2k_core::traits::ImageCodec for j2k_jpeg::decoder::Decoder<'_> +pub type j2k_jpeg::decoder::Decoder<'_>::Error = j2k_jpeg::error::JpegError +pub type j2k_jpeg::decoder::Decoder<'_>::Pool = j2k_jpeg::ScratchPool +pub type j2k_jpeg::decoder::Decoder<'_>::Warning = j2k_jpeg::error::Warning +impl<'a> j2k_core::traits::ImageDecode<'a> for j2k_jpeg::decoder::Decoder<'a> +pub type j2k_jpeg::decoder::Decoder<'a>::View = j2k_jpeg::decoder::JpegView<'a> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_into_with_scratch(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_region_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::decoder::Decoder<'a>::from_view(Self::View) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::inspect(&'a [u8]) -> core::result::Result +pub fn j2k_jpeg::decoder::Decoder<'a>::parse(&'a [u8]) -> core::result::Result +impl<'a> j2k_core::traits::ImageDecodeRows<'a, u8> for j2k_jpeg::decoder::Decoder<'a> +pub fn j2k_jpeg::decoder::Decoder<'a>::decode_rows>(&mut self, &mut R) -> core::result::Result, j2k_core::traits::DecodeRowsError::Error>> +pub struct j2k_jpeg::DecoderContext +impl j2k_jpeg::context::DecoderContext +pub fn j2k_jpeg::context::DecoderContext::new() -> Self +impl j2k_core::context::CodecContext for j2k_jpeg::context::DecoderContext +pub fn j2k_jpeg::context::DecoderContext::cache_stats(&self) -> j2k_core::context::CacheStats +pub fn j2k_jpeg::context::DecoderContext::clear(&mut self) +pub struct j2k_jpeg::EncodedJpeg +pub j2k_jpeg::EncodedJpeg::backend: j2k_jpeg::encoder::JpegBackend +pub j2k_jpeg::EncodedJpeg::data: alloc::vec::Vec +pub struct j2k_jpeg::Info +pub j2k_jpeg::Info::bit_depth: u8 +pub j2k_jpeg::Info::color_space: j2k_jpeg::info::ColorSpace +pub j2k_jpeg::Info::dimensions: (u32, u32) +pub j2k_jpeg::Info::mcu_geometry: j2k_jpeg::info::McuGeometry +pub j2k_jpeg::Info::restart_interval: core::option::Option +pub j2k_jpeg::Info::sampling: j2k_jpeg::info::SamplingFactors +pub j2k_jpeg::Info::scan_count: u16 +pub j2k_jpeg::Info::sof_kind: j2k_jpeg::info::SofKind +impl j2k_jpeg::info::Info +pub fn j2k_jpeg::info::Info::to_core_info(&self) -> j2k_core::types::Info +pub struct j2k_jpeg::JpegBackendEligibility +pub j2k_jpeg::JpegBackendEligibility::eligible: bool +pub j2k_jpeg::JpegBackendEligibility::reason: core::option::Option<&'static str> +pub struct j2k_jpeg::JpegBatchSession +impl j2k_jpeg::batch_session::JpegBatchSession +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_prepared_jpeg_tiles_rgb8(&mut self, &mut [j2k_jpeg::decoder::PreparedJpegTileJob<'_, '_>]) -> alloc::vec::Vec> +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_tiles_into(&mut self, &mut [j2k_jpeg::decoder::TileDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_tiles_into_with_options(&mut self, &mut [j2k_jpeg::decoder::TileDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_tiles_region_scaled_into(&mut self, &mut [j2k_jpeg::decoder::TileRegionScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_tiles_region_scaled_into_with_options(&mut self, &mut [j2k_jpeg::decoder::TileRegionScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_tiles_scaled_into(&mut self, &mut [j2k_jpeg::decoder::TileScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::batch_session::JpegBatchSession::decode_tiles_scaled_into_with_options(&mut self, &mut [j2k_jpeg::decoder::TileScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::batch_session::JpegBatchSession::new(j2k_core::batch::TileBatchOptions) -> Self +pub fn j2k_jpeg::batch_session::JpegBatchSession::options(&self) -> j2k_core::batch::TileBatchOptions +pub fn j2k_jpeg::batch_session::JpegBatchSession::reset(&mut self) +pub fn j2k_jpeg::batch_session::JpegBatchSession::retained_worker_slots(&self) -> usize +pub fn j2k_jpeg::batch_session::JpegBatchSession::set_options(&mut self, j2k_core::batch::TileBatchOptions) +pub fn j2k_jpeg::batch_session::JpegBatchSession::worker_count(&self) -> usize +impl core::default::Default for j2k_jpeg::batch_session::JpegBatchSession +pub fn j2k_jpeg::batch_session::JpegBatchSession::default() -> Self +pub struct j2k_jpeg::JpegCapabilityReport +pub j2k_jpeg::JpegCapabilityReport::cpu: j2k_jpeg::capabilities::JpegBackendEligibility +pub j2k_jpeg::JpegCapabilityReport::device: j2k_jpeg::adapter::DeviceBatchSummary +pub j2k_jpeg::JpegCapabilityReport::info: j2k_jpeg::info::Info +pub j2k_jpeg::JpegCapabilityReport::metal_fast: j2k_jpeg::capabilities::JpegBackendEligibility +pub j2k_jpeg::JpegCapabilityReport::owned_cuda: j2k_jpeg::capabilities::JpegBackendEligibility +pub j2k_jpeg::JpegCapabilityReport::request: j2k_jpeg::capabilities::JpegCapabilityRequest +impl j2k_jpeg::capabilities::JpegCapabilityReport +pub fn j2k_jpeg::capabilities::JpegCapabilityReport::for_decoder(&j2k_jpeg::decoder::Decoder<'_>, j2k_jpeg::capabilities::JpegCapabilityRequest) -> Self +pub fn j2k_jpeg::capabilities::JpegCapabilityReport::inspect(&[u8], j2k_jpeg::capabilities::JpegCapabilityRequest) -> core::result::Result +pub fn j2k_jpeg::capabilities::JpegCapabilityReport::metal_resident_rgb8_batch_output(&self) -> j2k_jpeg::capabilities::JpegBackendEligibility +pub fn j2k_jpeg::capabilities::JpegCapabilityReport::resolve_path(&self, j2k_core::backend::BackendRequest) -> j2k_jpeg::capabilities::JpegResolvedDecodePath +pub struct j2k_jpeg::JpegCapabilityRequest +pub j2k_jpeg::JpegCapabilityRequest::fmt: j2k_core::pixel::PixelFormat +pub j2k_jpeg::JpegCapabilityRequest::op: j2k_jpeg::capabilities::JpegDecodeOp +pub struct j2k_jpeg::JpegCodec +impl j2k_core::traits::ImageCodec for j2k_jpeg::JpegCodec +pub type j2k_jpeg::JpegCodec::Error = j2k_jpeg::error::JpegError +pub type j2k_jpeg::JpegCodec::Pool = j2k_jpeg::ScratchPool +pub type j2k_jpeg::JpegCodec::Warning = j2k_jpeg::error::Warning +impl j2k_core::traits::TileBatchDecode for j2k_jpeg::JpegCodec +pub type j2k_jpeg::JpegCodec::Context = j2k_jpeg::context::DecoderContext +pub fn j2k_jpeg::JpegCodec::decode_tile(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::JpegCodec::decode_tile_region(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::JpegCodec::decode_tile_region_scaled(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_jpeg::JpegCodec::decode_tile_scaled(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[u8], &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub struct j2k_jpeg::JpegDecodeRequest +pub j2k_jpeg::JpegDecodeRequest::backend: j2k_core::backend::BackendRequest +pub j2k_jpeg::JpegDecodeRequest::fmt: j2k_core::pixel::PixelFormat +pub j2k_jpeg::JpegDecodeRequest::op: j2k_jpeg::capabilities::JpegDecodeOp +impl j2k_jpeg::capabilities::JpegDecodeRequest +pub const fn j2k_jpeg::capabilities::JpegDecodeRequest::capability(self) -> j2k_jpeg::capabilities::JpegCapabilityRequest +pub struct j2k_jpeg::JpegEncodeOptions +pub j2k_jpeg::JpegEncodeOptions::backend: j2k_jpeg::encoder::JpegBackend +pub j2k_jpeg::JpegEncodeOptions::quality: u8 +pub j2k_jpeg::JpegEncodeOptions::restart_interval: core::option::Option +pub j2k_jpeg::JpegEncodeOptions::subsampling: j2k_jpeg::encoder::JpegSubsampling +impl core::default::Default for j2k_jpeg::encoder::JpegEncodeOptions +pub fn j2k_jpeg::encoder::JpegEncodeOptions::default() -> Self +pub struct j2k_jpeg::JpegOutputBuffer +impl j2k_jpeg::output_buffer::JpegOutputBuffer +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::as_mut_slice(&mut self) -> &mut [u8] +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::as_slice(&self) -> &[u8] +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::capacity(&self) -> usize +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::dimensions(&self) -> (u32, u32) +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::is_empty(&self) -> bool +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::len(&self) -> usize +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::new((u32, u32), j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::new_with_max_bytes((u32, u32), j2k_core::pixel::PixelFormat, usize) -> core::result::Result +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::pixel_format(&self) -> j2k_core::pixel::PixelFormat +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::resize(&mut self, (u32, u32), j2k_core::pixel::PixelFormat) -> core::result::Result<(), j2k_core::error::BufferError> +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::resize_with_max_bytes(&mut self, (u32, u32), j2k_core::pixel::PixelFormat, usize) -> core::result::Result<(), j2k_core::error::BufferError> +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::resize_with_stride(&mut self, (u32, u32), usize, j2k_core::pixel::PixelFormat) -> core::result::Result<(), j2k_core::error::BufferError> +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::resize_with_stride_with_max_bytes(&mut self, (u32, u32), usize, j2k_core::pixel::PixelFormat, usize) -> core::result::Result<(), j2k_core::error::BufferError> +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::stride(&self) -> usize +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::with_stride((u32, u32), usize, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_jpeg::output_buffer::JpegOutputBuffer::with_stride_with_max_bytes((u32, u32), usize, j2k_core::pixel::PixelFormat, usize) -> core::result::Result +pub struct j2k_jpeg::JpegResolvedDecode +pub j2k_jpeg::JpegResolvedDecode::capabilities: j2k_jpeg::capabilities::JpegCapabilityReport +pub j2k_jpeg::JpegResolvedDecode::output_rect: j2k_jpeg::info::Rect +pub j2k_jpeg::JpegResolvedDecode::path: j2k_jpeg::capabilities::JpegResolvedDecodePath +pub j2k_jpeg::JpegResolvedDecode::request: j2k_jpeg::capabilities::JpegDecodeRequest +impl j2k_jpeg::capabilities::JpegResolvedDecode +pub fn j2k_jpeg::capabilities::JpegResolvedDecode::from_capabilities(j2k_jpeg::capabilities::JpegCapabilityReport, j2k_jpeg::capabilities::JpegDecodeRequest) -> Self +pub fn j2k_jpeg::capabilities::JpegResolvedDecode::inspect(&[u8], j2k_jpeg::capabilities::JpegDecodeRequest) -> core::result::Result +pub struct j2k_jpeg::JpegScanRanges +pub j2k_jpeg::JpegScanRanges::entropy_range: core::ops::range::Range +pub j2k_jpeg::JpegScanRanges::eoi_marker_offset: core::option::Option +pub j2k_jpeg::JpegScanRanges::sos_marker_offset: usize +pub j2k_jpeg::JpegScanRanges::sos_payload_range: core::ops::range::Range +pub struct j2k_jpeg::JpegSegment<'a> +pub j2k_jpeg::JpegSegment::marker: u8 +pub j2k_jpeg::JpegSegment::marker_offset: usize +pub j2k_jpeg::JpegSegment::payload: &'a [u8] +pub j2k_jpeg::JpegSegment::payload_offset: usize +pub struct j2k_jpeg::JpegSegmentIter<'a> +impl<'a> core::iter::traits::iterator::Iterator for j2k_jpeg::segment::JpegSegmentIter<'a> +pub type j2k_jpeg::segment::JpegSegmentIter<'a>::Item = core::result::Result, j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::segment::JpegSegmentIter<'a>::next(&mut self) -> core::option::Option +pub struct j2k_jpeg::JpegSofInfo +pub j2k_jpeg::JpegSofInfo::bit_depth: u8 +pub j2k_jpeg::JpegSofInfo::component_ids: alloc::vec::Vec +pub j2k_jpeg::JpegSofInfo::dimensions: (u16, u16) +pub j2k_jpeg::JpegSofInfo::marker: u8 +pub j2k_jpeg::JpegSofInfo::quant_table_ids: alloc::vec::Vec +pub j2k_jpeg::JpegSofInfo::sampling: j2k_jpeg::info::SamplingFactors +pub j2k_jpeg::JpegSofInfo::sof_kind: j2k_jpeg::info::SofKind +pub struct j2k_jpeg::JpegTilePrepareOptions +pub j2k_jpeg::JpegTilePrepareOptions::duplicate_table_policy: j2k_jpeg::segment::DuplicateTablePolicy +pub j2k_jpeg::JpegTilePrepareOptions::expected_dimensions: core::option::Option<(u16, u16)> +pub j2k_jpeg::JpegTilePrepareOptions::repair_zero_sof_dimensions: bool +pub j2k_jpeg::JpegTilePrepareOptions::validate_restart_markers: bool +impl core::default::Default for j2k_jpeg::segment::JpegTilePrepareOptions +pub fn j2k_jpeg::segment::JpegTilePrepareOptions::default() -> Self +pub struct j2k_jpeg::JpegView<'a> +impl<'a> j2k_jpeg::decoder::JpegView<'a> +pub fn j2k_jpeg::decoder::JpegView<'a>::bytes(&self) -> &'a [u8] +pub fn j2k_jpeg::decoder::JpegView<'a>::info(&self) -> &j2k_jpeg::info::Info +pub fn j2k_jpeg::decoder::JpegView<'a>::parse(&'a [u8]) -> core::result::Result +pub fn j2k_jpeg::decoder::JpegView<'a>::parse_with_options(&'a [u8], j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decoder::JpegView<'a>::passthrough_candidate(&self) -> core::option::Option> +pub fn j2k_jpeg::decoder::JpegView<'a>::restart_index(&self) -> core::result::Result, j2k_jpeg::error::JpegError> +pub struct j2k_jpeg::McuGeometry +pub j2k_jpeg::McuGeometry::columns: u32 +pub j2k_jpeg::McuGeometry::count: u32 +pub j2k_jpeg::McuGeometry::height: u32 +pub j2k_jpeg::McuGeometry::rows: u32 +pub j2k_jpeg::McuGeometry::width: u32 +pub struct j2k_jpeg::PreparedJpegTileJob<'i, 'o> +pub j2k_jpeg::PreparedJpegTileJob::input: j2k_jpeg::segment::PreparedJpeg<'i> +pub j2k_jpeg::PreparedJpegTileJob::options: j2k_jpeg::info::DecodeOptions +pub j2k_jpeg::PreparedJpegTileJob::out: &'o mut [u8] +pub j2k_jpeg::PreparedJpegTileJob::stride: usize +pub struct j2k_jpeg::Rect +pub j2k_jpeg::Rect::h: u32 +pub j2k_jpeg::Rect::w: u32 +pub j2k_jpeg::Rect::x: u32 +pub j2k_jpeg::Rect::y: u32 +impl j2k_jpeg::info::Rect +pub fn j2k_jpeg::info::Rect::full((u32, u32)) -> Self +pub fn j2k_jpeg::info::Rect::is_within(&self, (u32, u32)) -> bool +impl core::convert::From for j2k_jpeg::info::Rect +pub fn j2k_jpeg::info::Rect::from(j2k_core::types::Rect) -> Self +impl core::convert::From for j2k_core::types::Rect +pub fn j2k_core::types::Rect::from(j2k_jpeg::info::Rect) -> Self +pub struct j2k_jpeg::RestartIndex +pub j2k_jpeg::RestartIndex::interval_mcus: u32 +pub j2k_jpeg::RestartIndex::scan_data_offset: usize +pub j2k_jpeg::RestartIndex::segments: alloc::vec::Vec +pub struct j2k_jpeg::RestartSegment +pub j2k_jpeg::RestartSegment::entropy_offset: usize +pub j2k_jpeg::RestartSegment::marker: core::option::Option +pub j2k_jpeg::RestartSegment::marker_offset: core::option::Option +pub j2k_jpeg::RestartSegment::start_mcu: u32 +pub struct j2k_jpeg::SamplingFactors +pub j2k_jpeg::SamplingFactors::max_h: u8 +pub j2k_jpeg::SamplingFactors::max_v: u8 +impl j2k_jpeg::info::SamplingFactors +pub fn j2k_jpeg::info::SamplingFactors::component(&self, usize) -> core::option::Option<(u8, u8)> +pub fn j2k_jpeg::info::SamplingFactors::components(&self) -> &[(u8, u8)] +pub fn j2k_jpeg::info::SamplingFactors::from_components(&[(u8, u8)]) -> core::result::Result +pub fn j2k_jpeg::info::SamplingFactors::is_empty(&self) -> bool +pub fn j2k_jpeg::info::SamplingFactors::len(&self) -> usize +pub struct j2k_jpeg::ScratchPool +impl j2k_jpeg::ScratchPool +pub fn j2k_jpeg::ScratchPool::new() -> Self +impl j2k_core::scratch::ScratchPool for j2k_jpeg::ScratchPool +pub fn j2k_jpeg::ScratchPool::bytes_allocated(&self) -> usize +pub fn j2k_jpeg::ScratchPool::reset(&mut self) +pub trait j2k_jpeg::ComponentRowWriter +pub fn j2k_jpeg::ComponentRowWriter::write_gray_row(&mut self, u32, &[u8]) -> core::result::Result<(), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::ComponentRowWriter::write_rgb_row(&mut self, u32, &[u8], &[u8], &[u8]) -> core::result::Result<(), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::ComponentRowWriter::write_ycbcr_row(&mut self, u32, &[u8], &[u8], &[u8]) -> core::result::Result<(), j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::decode_prepared_jpeg_tiles_rgb8(&mut [j2k_jpeg::decoder::PreparedJpegTileJob<'_, '_>]) -> alloc::vec::Vec> +pub fn j2k_jpeg::decode_tile_into(&[u8], &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_jpeg::decode_tile_into_in_context(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_jpeg::decode_tile_into_in_context_with_options(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decode_tile_region_into_in_context(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect) -> core::result::Result +pub fn j2k_jpeg::decode_tile_region_into_in_context_with_options(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decode_tile_region_scaled_into_in_context(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_jpeg::decode_tile_region_scaled_into_in_context_with_options(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_jpeg::info::Rect, j2k_core::scale::Downscale, j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decode_tile_scaled_into_in_context(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_jpeg::decode_tile_scaled_into_in_context_with_options(&[u8], &mut j2k_jpeg::context::DecoderContext, &mut j2k_jpeg::ScratchPool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_jpeg::info::DecodeOptions) -> core::result::Result +pub fn j2k_jpeg::decode_tiles_into(&mut [j2k_jpeg::decoder::TileDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::decode_tiles_into_with_options(&mut [j2k_jpeg::decoder::TileDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions, j2k_core::batch::TileBatchOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::decode_tiles_region_scaled_into(&mut [j2k_jpeg::decoder::TileRegionScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::decode_tiles_region_scaled_into_with_options(&mut [j2k_jpeg::decoder::TileRegionScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions, j2k_core::batch::TileBatchOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::decode_tiles_scaled_into(&mut [j2k_jpeg::decoder::TileScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_core::batch::TileBatchOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::decode_tiles_scaled_into_with_options(&mut [j2k_jpeg::decoder::TileScaledDecodeJob<'_, '_>], j2k_core::pixel::PixelFormat, j2k_jpeg::info::DecodeOptions, j2k_core::batch::TileBatchOptions) -> core::result::Result, j2k_jpeg::decoder::TileBatchError> +pub fn j2k_jpeg::encode_jpeg_baseline(j2k_jpeg::encoder::JpegSamples<'_>, j2k_jpeg::encoder::JpegEncodeOptions) -> core::result::Result +pub fn j2k_jpeg::find_scan_ranges(&[u8]) -> core::result::Result +pub const fn j2k_jpeg::is_sof_marker(u8) -> bool +pub fn j2k_jpeg::iter_segments(&[u8]) -> j2k_jpeg::segment::JpegSegmentIter<'_> +pub fn j2k_jpeg::parse_dri(&[u8]) -> core::result::Result, j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::parse_sof_info(u8, &[u8]) -> core::result::Result +pub fn j2k_jpeg::prepare_tiff_jpeg_tile<'a>(&'a [u8], core::option::Option<&'a [u8]>, j2k_jpeg::segment::JpegTilePrepareOptions) -> core::result::Result, j2k_jpeg::error::JpegError> +pub fn j2k_jpeg::rewrite_sof_dimensions(&[u8], (u16, u16)) -> core::result::Result, j2k_jpeg::error::JpegError> +pub type j2k_jpeg::TileBatchError = j2k_core::batch::TileBatchError +pub type j2k_jpeg::TileDecodeJob<'i, 'o> = j2k_core::batch::TileDecodeJob<'i, 'o> +pub type j2k_jpeg::TileRegionScaledDecodeJob<'i, 'o> = j2k_core::batch::TileRegionScaledDecodeJob<'i, 'o> +pub type j2k_jpeg::TileScaledDecodeJob<'i, 'o> = j2k_core::batch::TileScaledDecodeJob<'i, 'o> +``` + +## `j2k-tilecodec` + +```text +pub mod j2k_tilecodec +pub use j2k_tilecodec::TileDecompress +pub enum j2k_tilecodec::TileCodecError +pub j2k_tilecodec::TileCodecError::Backend(alloc::string::String) +pub j2k_tilecodec::TileCodecError::Buffer(j2k_core::error::BufferError) +pub j2k_tilecodec::TileCodecError::Input(j2k_core::error::InputError) +pub j2k_tilecodec::TileCodecError::Malformed +pub j2k_tilecodec::TileCodecError::Malformed::context: &'static str +pub j2k_tilecodec::TileCodecError::Malformed::message: alloc::string::String +pub j2k_tilecodec::TileCodecError::Unsupported(j2k_core::error::Unsupported) +impl j2k_core::error::CodecError for j2k_tilecodec::TileCodecError +pub fn j2k_tilecodec::TileCodecError::is_buffer_error(&self) -> bool +pub fn j2k_tilecodec::TileCodecError::is_not_implemented(&self) -> bool +pub fn j2k_tilecodec::TileCodecError::is_truncated(&self) -> bool +pub fn j2k_tilecodec::TileCodecError::is_unsupported(&self) -> bool +pub struct j2k_tilecodec::DeflateCodec +impl j2k_core::traits::TileDecompress for j2k_tilecodec::DeflateCodec +pub type j2k_tilecodec::DeflateCodec::Error = j2k_tilecodec::TileCodecError +pub type j2k_tilecodec::DeflateCodec::Pool = j2k_tilecodec::DeflatePool +pub fn j2k_tilecodec::DeflateCodec::decompress_into(&mut Self::Pool, &[u8], &mut [u8]) -> core::result::Result +pub fn j2k_tilecodec::DeflateCodec::expected_size(&[u8]) -> core::result::Result, Self::Error> +pub struct j2k_tilecodec::DeflatePool +impl j2k_tilecodec::DeflatePool +pub fn j2k_tilecodec::DeflatePool::new() -> Self +impl j2k_core::scratch::ScratchPool for j2k_tilecodec::DeflatePool +pub fn j2k_tilecodec::DeflatePool::bytes_allocated(&self) -> usize +pub fn j2k_tilecodec::DeflatePool::reset(&mut self) +pub struct j2k_tilecodec::LzwCodec +impl j2k_core::traits::TileDecompress for j2k_tilecodec::LzwCodec +pub type j2k_tilecodec::LzwCodec::Error = j2k_tilecodec::TileCodecError +pub type j2k_tilecodec::LzwCodec::Pool = j2k_tilecodec::LzwPool +pub fn j2k_tilecodec::LzwCodec::decompress_into(&mut Self::Pool, &[u8], &mut [u8]) -> core::result::Result +pub fn j2k_tilecodec::LzwCodec::expected_size(&[u8]) -> core::result::Result, Self::Error> +pub struct j2k_tilecodec::LzwPool +impl j2k_tilecodec::LzwPool +pub fn j2k_tilecodec::LzwPool::new() -> Self +impl j2k_core::scratch::ScratchPool for j2k_tilecodec::LzwPool +pub fn j2k_tilecodec::LzwPool::bytes_allocated(&self) -> usize +pub fn j2k_tilecodec::LzwPool::reset(&mut self) +pub struct j2k_tilecodec::NoPool +impl j2k_core::scratch::ScratchPool for j2k_tilecodec::NoPool +pub fn j2k_tilecodec::NoPool::bytes_allocated(&self) -> usize +pub fn j2k_tilecodec::NoPool::reset(&mut self) +pub struct j2k_tilecodec::UncompressedCodec +impl j2k_core::traits::TileDecompress for j2k_tilecodec::UncompressedCodec +pub type j2k_tilecodec::UncompressedCodec::Error = j2k_tilecodec::TileCodecError +pub type j2k_tilecodec::UncompressedCodec::Pool = j2k_tilecodec::NoPool +pub fn j2k_tilecodec::UncompressedCodec::decompress_into(&mut Self::Pool, &[u8], &mut [u8]) -> core::result::Result +pub fn j2k_tilecodec::UncompressedCodec::expected_size(&[u8]) -> core::result::Result, Self::Error> +pub struct j2k_tilecodec::ZstdCodec +impl j2k_core::traits::TileDecompress for j2k_tilecodec::ZstdCodec +pub type j2k_tilecodec::ZstdCodec::Error = j2k_tilecodec::TileCodecError +pub type j2k_tilecodec::ZstdCodec::Pool = j2k_tilecodec::ZstdPool +pub fn j2k_tilecodec::ZstdCodec::decompress_into(&mut Self::Pool, &[u8], &mut [u8]) -> core::result::Result +pub fn j2k_tilecodec::ZstdCodec::expected_size(&[u8]) -> core::result::Result, Self::Error> +pub struct j2k_tilecodec::ZstdPool +impl j2k_tilecodec::ZstdPool +pub fn j2k_tilecodec::ZstdPool::new() -> Self +impl j2k_core::scratch::ScratchPool for j2k_tilecodec::ZstdPool +pub fn j2k_tilecodec::ZstdPool::bytes_allocated(&self) -> usize +pub fn j2k_tilecodec::ZstdPool::reset(&mut self) +``` + +## `j2k-jpeg-metal` + +```text +pub mod j2k_jpeg_metal +pub use j2k_jpeg_metal::DecoderContext +pub use j2k_jpeg_metal::Info +pub use j2k_jpeg_metal::JpegDownscale +pub use j2k_jpeg_metal::JpegPixelFormat +pub use j2k_jpeg_metal::JpegRectPublic +pub use j2k_jpeg_metal::ScratchPool +pub mod j2k_jpeg_metal::viewport +pub enum j2k_jpeg_metal::viewport::ViewportResidentOutputStrategy +pub j2k_jpeg_metal::viewport::ViewportResidentOutputStrategy::Composite +pub j2k_jpeg_metal::viewport::ViewportResidentOutputStrategy::DirectContiguous +pub enum j2k_jpeg_metal::viewport::ViewportSurfaceStrategy +pub j2k_jpeg_metal::viewport::ViewportSurfaceStrategy::CpuComposite +pub j2k_jpeg_metal::viewport::ViewportSurfaceStrategy::CpuContiguous +pub j2k_jpeg_metal::viewport::ViewportSurfaceStrategy::HybridComposite +pub j2k_jpeg_metal::viewport::ViewportSurfaceStrategy::HybridContiguous +pub struct j2k_jpeg_metal::viewport::ViewportTile +pub j2k_jpeg_metal::viewport::ViewportTile::dest: j2k_core::types::Rect +pub j2k_jpeg_metal::viewport::ViewportTile::source_roi: j2k_core::types::Rect +pub struct j2k_jpeg_metal::viewport::ViewportWorkload +pub j2k_jpeg_metal::viewport::ViewportWorkload::scale: j2k_core::scale::Downscale +pub j2k_jpeg_metal::viewport::ViewportWorkload::tiles: alloc::vec::Vec +pub j2k_jpeg_metal::viewport::ViewportWorkload::viewport_dims: (u32, u32) +pub fn j2k_jpeg_metal::viewport::choose_resizable_metal_viewport_strategy(&j2k_jpeg::decoder::Decoder<'_>, &j2k_jpeg_metal::viewport::ViewportWorkload) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::choose_viewport_surface_strategy(&j2k_jpeg_metal::viewport::ViewportWorkload, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::compose_viewport_cpu(&j2k_jpeg::decoder::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, (u32, u32), &[j2k_jpeg_metal::viewport::ViewportTile]) -> core::result::Result, j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::viewport::compose_viewport_cpu_to_surface(&j2k_jpeg::decoder::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, j2k_core::scale::Downscale, (u32, u32), &[j2k_jpeg_metal::viewport::ViewportTile]) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::compose_viewport_hybrid(&j2k_jpeg::decoder::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, j2k_core::scale::Downscale, (u32, u32), &[j2k_jpeg_metal::viewport::ViewportTile]) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::compose_viewport_to_resizable_metal_buffer_with_session(&j2k_jpeg::decoder::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, &j2k_jpeg_metal::viewport::ViewportWorkload, &mut j2k_jpeg_metal::MetalBatchOutputBuffer, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::compose_viewport_to_resizable_metal_textures_with_session(&j2k_jpeg::decoder::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, &j2k_jpeg_metal::viewport::ViewportWorkload, &mut j2k_jpeg_metal::MetalBatchTextureOutput, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::decode_viewport_region_cpu(&j2k_jpeg::decoder::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, j2k_core::pixel::PixelFormat, &j2k_jpeg_metal::viewport::ViewportWorkload) -> core::result::Result, j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::viewport::decode_viewport_region_cpu_to_surface(&j2k_jpeg::decoder::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, &j2k_jpeg_metal::viewport::ViewportWorkload) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::decode_viewport_region_hybrid(&j2k_jpeg::decoder::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, &j2k_jpeg_metal::viewport::ViewportWorkload) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::decode_viewport_region_to_resizable_metal_buffer_with_session(&[u8], &j2k_jpeg_metal::viewport::ViewportWorkload, &mut j2k_jpeg_metal::MetalBatchOutputBuffer, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::decode_viewport_region_to_resizable_metal_textures_with_session(&[u8], &j2k_jpeg_metal::viewport::ViewportWorkload, &mut j2k_jpeg_metal::MetalBatchTextureOutput, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::decode_viewport_to_resizable_metal_buffer_with_decoder_session(&j2k_jpeg_metal::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, &j2k_jpeg_metal::viewport::ViewportWorkload, &mut j2k_jpeg_metal::MetalBatchOutputBuffer, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::decode_viewport_to_resizable_metal_buffer_with_session(&j2k_jpeg::decoder::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, &j2k_jpeg_metal::viewport::ViewportWorkload, &mut j2k_jpeg_metal::MetalBatchOutputBuffer, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::decode_viewport_to_resizable_metal_textures_with_decoder_session(&j2k_jpeg_metal::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, &j2k_jpeg_metal::viewport::ViewportWorkload, &mut j2k_jpeg_metal::MetalBatchTextureOutput, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::decode_viewport_to_resizable_metal_textures_with_session(&j2k_jpeg::decoder::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, &j2k_jpeg_metal::viewport::ViewportWorkload, &mut j2k_jpeg_metal::MetalBatchTextureOutput, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::decode_viewport_to_surface(&j2k_jpeg::decoder::Decoder<'_>, &mut j2k_jpeg::internal::scratch::ScratchPool, &j2k_jpeg_metal::viewport::ViewportWorkload, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::viewport::is_contiguous_viewport_workload(&j2k_jpeg_metal::viewport::ViewportWorkload) -> bool +pub fn j2k_jpeg_metal::viewport::suggest_viewport_workload((u32, u32)) -> core::option::Option +pub fn j2k_jpeg_metal::viewport::viewport_source_bounds(&j2k_jpeg_metal::viewport::ViewportWorkload) -> j2k_core::types::Rect +pub enum j2k_jpeg_metal::Error +pub j2k_jpeg_metal::Error::Buffer(j2k_core::error::BufferError) +pub j2k_jpeg_metal::Error::Decode(j2k_jpeg::error::JpegError) +pub j2k_jpeg_metal::Error::Encode(j2k_jpeg::encoder::JpegEncodeError) +pub j2k_jpeg_metal::Error::MetalKernel +pub j2k_jpeg_metal::Error::MetalKernel::message: alloc::string::String +pub j2k_jpeg_metal::Error::MetalRuntime +pub j2k_jpeg_metal::Error::MetalRuntime::message: alloc::string::String +pub j2k_jpeg_metal::Error::MetalStatePoisoned +pub j2k_jpeg_metal::Error::MetalStatePoisoned::state: &'static str +pub j2k_jpeg_metal::Error::MetalUnavailable +pub j2k_jpeg_metal::Error::UnsupportedBackend +pub j2k_jpeg_metal::Error::UnsupportedBackend::request: j2k_core::backend::BackendRequest +pub j2k_jpeg_metal::Error::UnsupportedMetalRequest +pub j2k_jpeg_metal::Error::UnsupportedMetalRequest::reason: &'static str +impl j2k_core::error::CodecError for j2k_jpeg_metal::Error +pub fn j2k_jpeg_metal::Error::is_buffer_error(&self) -> bool +pub fn j2k_jpeg_metal::Error::is_not_implemented(&self) -> bool +pub fn j2k_jpeg_metal::Error::is_truncated(&self) -> bool +pub fn j2k_jpeg_metal::Error::is_unsupported(&self) -> bool +pub enum j2k_jpeg_metal::MetalBufferBatchTarget<'a> +pub j2k_jpeg_metal::MetalBufferBatchTarget::Resizable(&'a mut j2k_jpeg_metal::MetalBatchOutputBuffer) +pub j2k_jpeg_metal::MetalBufferBatchTarget::Reusable(&'a j2k_jpeg_metal::MetalBatchOutputBuffer) +pub enum j2k_jpeg_metal::MetalTextureBatchTarget<'a> +pub j2k_jpeg_metal::MetalTextureBatchTarget::Resizable(&'a mut j2k_jpeg_metal::MetalBatchTextureOutput) +pub j2k_jpeg_metal::MetalTextureBatchTarget::Reusable(&'a j2k_jpeg_metal::MetalBatchTextureOutput) +pub enum j2k_jpeg_metal::Rgb8MetalBatchOp +pub j2k_jpeg_metal::Rgb8MetalBatchOp::Full +pub j2k_jpeg_metal::Rgb8MetalBatchOp::RegionScaled +pub j2k_jpeg_metal::Rgb8MetalBatchOp::RegionScaled::roi: j2k_core::types::Rect +pub j2k_jpeg_metal::Rgb8MetalBatchOp::RegionScaled::scale: j2k_core::scale::Downscale +pub j2k_jpeg_metal::Rgb8MetalBatchOp::Scaled(j2k_core::scale::Downscale) +pub enum j2k_jpeg_metal::Rgb8MetalBatchSource<'a, 'b> +pub j2k_jpeg_metal::Rgb8MetalBatchSource::Bytes(&'a [&'a [u8]]) +pub j2k_jpeg_metal::Rgb8MetalBatchSource::Decoders(&'a [&'a j2k_jpeg_metal::Decoder<'b>]) +pub enum j2k_jpeg_metal::SurfaceResidency +pub j2k_jpeg_metal::SurfaceResidency::CpuStagedMetalUpload +pub j2k_jpeg_metal::SurfaceResidency::Host +pub j2k_jpeg_metal::SurfaceResidency::MetalResidentDecode +pub struct j2k_jpeg_metal::Codec +impl j2k_jpeg_metal::Codec +pub fn j2k_jpeg_metal::Codec::decode_rgb8_batch_into_buffer_with_session(j2k_jpeg_metal::Rgb8MetalBatchRequest<'_, '_>, j2k_jpeg_metal::MetalBufferBatchTarget<'_>, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result>, j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::Codec::decode_rgb8_batch_into_textures_with_session(j2k_jpeg_metal::Rgb8MetalBatchRequest<'_, '_>, j2k_jpeg_metal::MetalTextureBatchTarget<'_>, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result>, j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::Codec::decode_rgb8_decoder_batch_into_resizable_metal_textures_with_session(&[&j2k_jpeg_metal::Decoder<'_>], &mut j2k_jpeg_metal::MetalBatchTextureOutput, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result>, j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::Codec::decode_rgb8_region_scaled_batch_into_resizable_metal_buffer_with_session(&[&[u8]], j2k_core::types::Rect, j2k_core::scale::Downscale, &mut j2k_jpeg_metal::MetalBatchOutputBuffer, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result>, j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::Codec::decode_rgb8_region_scaled_batch_into_resizable_metal_textures_with_session(&[&[u8]], j2k_core::types::Rect, j2k_core::scale::Downscale, &mut j2k_jpeg_metal::MetalBatchTextureOutput, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result>, j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::Codec::inspect_rgb8_decoder_batch_metal_output(&[&j2k_jpeg_metal::Decoder<'_>], j2k_jpeg::capabilities::JpegDecodeOp) -> j2k_jpeg_metal::JpegMetalResidentBatchReport +pub fn j2k_jpeg_metal::Codec::submit_tile_region_scaled_to_device(&mut j2k_core::context::DecoderContext, &mut j2k_jpeg_metal::MetalSession, &mut j2k_jpeg::internal::scratch::ScratchPool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result<::SubmittedSurface, j2k_jpeg_metal::Error> +impl j2k_core::traits::ImageCodec for j2k_jpeg_metal::Codec +pub type j2k_jpeg_metal::Codec::Error = j2k_jpeg_metal::Error +pub type j2k_jpeg_metal::Codec::Pool = j2k_jpeg::internal::scratch::ScratchPool +pub type j2k_jpeg_metal::Codec::Warning = j2k_jpeg::error::Warning +impl j2k_core::traits::TileBatchDecodeDevice for j2k_jpeg_metal::Codec +pub type j2k_jpeg_metal::Codec::Context = j2k_jpeg::context::DecoderContext +pub type j2k_jpeg_metal::Codec::DeviceSurface = j2k_jpeg_metal::Surface +impl j2k_core::traits::TileBatchDecodeManyDevice for j2k_jpeg_metal::Codec +pub type j2k_jpeg_metal::Codec::Context = j2k_jpeg::context::DecoderContext +pub type j2k_jpeg_metal::Codec::DeviceSurface = j2k_jpeg_metal::Surface +pub fn j2k_jpeg_metal::Codec::decode_tiles_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[&[u8]], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result, Self::Error> +impl j2k_core::traits::TileBatchDecodeSubmit for j2k_jpeg_metal::Codec +pub type j2k_jpeg_metal::Codec::Context = j2k_jpeg::context::DecoderContext +pub type j2k_jpeg_metal::Codec::DeviceSurface = j2k_jpeg_metal::Surface +pub type j2k_jpeg_metal::Codec::Session = j2k_jpeg_metal::MetalSession +pub type j2k_jpeg_metal::Codec::SubmittedSurface = j2k_jpeg_metal::batch::MetalSubmission +pub fn j2k_jpeg_metal::Codec::submit_tile_region_scaled_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::Codec::submit_tile_region_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::Codec::submit_tile_scaled_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::Codec::submit_tile_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub struct j2k_jpeg_metal::Decoder<'a> +impl<'a> j2k_jpeg_metal::Decoder<'a> +pub fn j2k_jpeg_metal::Decoder<'a>::decode_region_scaled_to_device(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::Decoder<'a>::decode_to_device_with_session(&mut self, j2k_core::pixel::PixelFormat, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_jpeg_metal::Decoder<'a>::from_view(j2k_jpeg::decoder::JpegView<'a>) -> core::result::Result +pub fn j2k_jpeg_metal::Decoder<'a>::inner(&self) -> &j2k_jpeg::decoder::Decoder<'a> +pub fn j2k_jpeg_metal::Decoder<'a>::into_inner(self) -> j2k_jpeg::decoder::Decoder<'a> +pub fn j2k_jpeg_metal::Decoder<'a>::new(&'a [u8]) -> core::result::Result +impl j2k_core::traits::ImageCodec for j2k_jpeg_metal::Decoder<'_> +pub type j2k_jpeg_metal::Decoder<'_>::Error = j2k_jpeg_metal::Error +pub type j2k_jpeg_metal::Decoder<'_>::Pool = j2k_jpeg::internal::scratch::ScratchPool +pub type j2k_jpeg_metal::Decoder<'_>::Warning = j2k_jpeg::error::Warning +impl<'a> j2k_core::traits::ImageDecode<'a> for j2k_jpeg_metal::Decoder<'a> +pub type j2k_jpeg_metal::Decoder<'a>::View = j2k_jpeg::decoder::JpegView<'a> +pub fn j2k_jpeg_metal::Decoder<'a>::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_jpeg_metal::Decoder<'a>::decode_into_with_scratch(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_jpeg_metal::Decoder<'a>::decode_region_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k_jpeg_metal::Decoder<'a>::decode_region_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_jpeg_metal::Decoder<'a>::decode_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_jpeg_metal::Decoder<'a>::from_view(Self::View) -> core::result::Result +pub fn j2k_jpeg_metal::Decoder<'a>::inspect(&'a [u8]) -> core::result::Result +pub fn j2k_jpeg_metal::Decoder<'a>::parse(&'a [u8]) -> core::result::Result +impl<'a> j2k_core::traits::ImageDecodeDevice<'a> for j2k_jpeg_metal::Decoder<'a> +pub type j2k_jpeg_metal::Decoder<'a>::DeviceSurface = j2k_jpeg_metal::Surface +impl<'a> j2k_core::traits::ImageDecodeSubmit<'a> for j2k_jpeg_metal::Decoder<'a> +pub type j2k_jpeg_metal::Decoder<'a>::DeviceSurface = j2k_jpeg_metal::Surface +pub type j2k_jpeg_metal::Decoder<'a>::Session = j2k_jpeg_metal::MetalSession +pub type j2k_jpeg_metal::Decoder<'a>::SubmittedSurface = j2k_jpeg_metal::batch::MetalSubmission +pub fn j2k_jpeg_metal::Decoder<'a>::submit_region_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::Decoder<'a>::submit_region_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::Decoder<'a>::submit_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::Decoder<'a>::submit_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub struct j2k_jpeg_metal::JpegBaselineMetalEncodeTile<'a> +pub j2k_jpeg_metal::JpegBaselineMetalEncodeTile::buffer: &'a metal::buffer::Buffer +pub j2k_jpeg_metal::JpegBaselineMetalEncodeTile::byte_offset: usize +pub j2k_jpeg_metal::JpegBaselineMetalEncodeTile::format: j2k_core::pixel::PixelFormat +pub j2k_jpeg_metal::JpegBaselineMetalEncodeTile::height: u32 +pub j2k_jpeg_metal::JpegBaselineMetalEncodeTile::output_height: u32 +pub j2k_jpeg_metal::JpegBaselineMetalEncodeTile::output_width: u32 +pub j2k_jpeg_metal::JpegBaselineMetalEncodeTile::pitch_bytes: usize +pub j2k_jpeg_metal::JpegBaselineMetalEncodeTile::width: u32 +pub struct j2k_jpeg_metal::JpegMetalResidentBatchReport +pub j2k_jpeg_metal::JpegMetalResidentBatchReport::eligibility: j2k_jpeg::capabilities::JpegBackendEligibility +pub j2k_jpeg_metal::JpegMetalResidentBatchReport::op: j2k_jpeg::capabilities::JpegDecodeOp +pub j2k_jpeg_metal::JpegMetalResidentBatchReport::output_dimensions: core::option::Option<(u32, u32)> +pub j2k_jpeg_metal::JpegMetalResidentBatchReport::tile_count: usize +impl j2k_jpeg_metal::JpegMetalResidentBatchReport +pub fn j2k_jpeg_metal::JpegMetalResidentBatchReport::required_tile_capacity(&self) -> usize +pub struct j2k_jpeg_metal::JpegTileBatch +impl j2k_jpeg_metal::JpegTileBatch +pub fn j2k_jpeg_metal::JpegTileBatch::decode_all(self) -> core::result::Result, j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::JpegTileBatch::is_empty(&self) -> bool +pub fn j2k_jpeg_metal::JpegTileBatch::len(&self) -> usize +pub fn j2k_jpeg_metal::JpegTileBatch::new() -> Self +pub fn j2k_jpeg_metal::JpegTileBatch::push_shared_tile(&mut self, alloc::sync::Arc<[u8]>, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::JpegTileBatch::push_shared_tile_region(&mut self, alloc::sync::Arc<[u8]>, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::JpegTileBatch::push_shared_tile_region_scaled(&mut self, alloc::sync::Arc<[u8]>, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::JpegTileBatch::push_shared_tile_scaled(&mut self, alloc::sync::Arc<[u8]>, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::JpegTileBatch::push_tile(&mut self, &[u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::JpegTileBatch::push_tile_region(&mut self, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::JpegTileBatch::push_tile_region_scaled(&mut self, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::JpegTileBatch::push_tile_scaled(&mut self, &[u8], j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_metal::JpegTileBatch::submissions(&self) -> core::result::Result +pub fn j2k_jpeg_metal::JpegTileBatch::with_capacity(usize) -> Self +pub struct j2k_jpeg_metal::MetalBackendSession +impl j2k_jpeg_metal::MetalBackendSession +pub fn j2k_jpeg_metal::MetalBackendSession::device(&self) -> &metal::device::DeviceRef +pub fn j2k_jpeg_metal::MetalBackendSession::new(metal::device::Device) -> Self +pub fn j2k_jpeg_metal::MetalBackendSession::system_default() -> core::result::Result +impl core::fmt::Debug for j2k_jpeg_metal::MetalBackendSession +pub fn j2k_jpeg_metal::MetalBackendSession::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl j2k_core::accelerator::AcceleratorSession for j2k_jpeg_metal::MetalBackendSession +pub fn j2k_jpeg_metal::MetalBackendSession::backend_kind(&self) -> j2k_core::backend::BackendKind +pub struct j2k_jpeg_metal::MetalBatchOutputBuffer +impl j2k_jpeg_metal::MetalBatchOutputBuffer +pub fn j2k_jpeg_metal::MetalBatchOutputBuffer::buffer(&self) -> &metal::buffer::BufferRef +pub fn j2k_jpeg_metal::MetalBatchOutputBuffer::byte_len(&self) -> usize +pub fn j2k_jpeg_metal::MetalBatchOutputBuffer::dimensions(&self) -> (u32, u32) +pub fn j2k_jpeg_metal::MetalBatchOutputBuffer::ensure_rgb8_batch_report(&mut self, &j2k_jpeg_metal::MetalBackendSession, &j2k_jpeg_metal::JpegMetalResidentBatchReport) -> core::result::Result<(), j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::MetalBatchOutputBuffer::ensure_rgb8_region_scaled_tiles(&mut self, &j2k_jpeg_metal::MetalBackendSession, j2k_core::types::Rect, j2k_core::scale::Downscale, usize) -> core::result::Result<(), j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::MetalBatchOutputBuffer::ensure_rgb8_scaled_tiles(&mut self, &j2k_jpeg_metal::MetalBackendSession, (u32, u32), j2k_core::scale::Downscale, usize) -> core::result::Result<(), j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::MetalBatchOutputBuffer::ensure_rgb8_tiles(&mut self, &j2k_jpeg_metal::MetalBackendSession, (u32, u32), usize) -> core::result::Result<(), j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::MetalBatchOutputBuffer::new_rgb8_tiles(&j2k_jpeg_metal::MetalBackendSession, (u32, u32), usize) -> core::result::Result +pub fn j2k_jpeg_metal::MetalBatchOutputBuffer::pitch_bytes(&self) -> usize +pub fn j2k_jpeg_metal::MetalBatchOutputBuffer::pixel_format(&self) -> j2k_core::pixel::PixelFormat +pub fn j2k_jpeg_metal::MetalBatchOutputBuffer::tile_capacity(&self) -> usize +pub fn j2k_jpeg_metal::MetalBatchOutputBuffer::tile_stride_bytes(&self) -> usize +pub struct j2k_jpeg_metal::MetalBatchTextureOutput +impl j2k_jpeg_metal::MetalBatchTextureOutput +pub fn j2k_jpeg_metal::MetalBatchTextureOutput::dimensions(&self) -> (u32, u32) +pub fn j2k_jpeg_metal::MetalBatchTextureOutput::ensure_rgba8_batch_report(&mut self, &j2k_jpeg_metal::MetalBackendSession, &j2k_jpeg_metal::JpegMetalResidentBatchReport) -> core::result::Result<(), j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::MetalBatchTextureOutput::ensure_rgba8_region_scaled_tiles(&mut self, &j2k_jpeg_metal::MetalBackendSession, j2k_core::types::Rect, j2k_core::scale::Downscale, usize) -> core::result::Result<(), j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::MetalBatchTextureOutput::ensure_rgba8_scaled_tiles(&mut self, &j2k_jpeg_metal::MetalBackendSession, (u32, u32), j2k_core::scale::Downscale, usize) -> core::result::Result<(), j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::MetalBatchTextureOutput::ensure_rgba8_tiles(&mut self, &j2k_jpeg_metal::MetalBackendSession, (u32, u32), usize) -> core::result::Result<(), j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::MetalBatchTextureOutput::metal_pixel_format(&self) -> metal::constants::MTLPixelFormat +pub fn j2k_jpeg_metal::MetalBatchTextureOutput::new_rgba8_tiles(&j2k_jpeg_metal::MetalBackendSession, (u32, u32), usize) -> core::result::Result +pub fn j2k_jpeg_metal::MetalBatchTextureOutput::pixel_format(&self) -> j2k_core::pixel::PixelFormat +pub fn j2k_jpeg_metal::MetalBatchTextureOutput::texture(&self, usize) -> core::option::Option<&metal::texture::TextureRef> +pub fn j2k_jpeg_metal::MetalBatchTextureOutput::tile_capacity(&self) -> usize +pub struct j2k_jpeg_metal::MetalSession +impl j2k_jpeg_metal::MetalSession +pub fn j2k_jpeg_metal::MetalSession::submissions(&self) -> core::result::Result +pub fn j2k_jpeg_metal::MetalSession::with_backend_session(j2k_jpeg_metal::MetalBackendSession) -> Self +impl core::fmt::Debug for j2k_jpeg_metal::MetalSession +pub fn j2k_jpeg_metal::MetalSession::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_jpeg_metal::MetalTextureTile +impl j2k_jpeg_metal::MetalTextureTile +pub fn j2k_jpeg_metal::MetalTextureTile::dimensions(&self) -> (u32, u32) +pub fn j2k_jpeg_metal::MetalTextureTile::pixel_format(&self) -> j2k_core::pixel::PixelFormat +pub fn j2k_jpeg_metal::MetalTextureTile::texture(&self) -> &metal::texture::TextureRef +pub struct j2k_jpeg_metal::Rgb8MetalBatchRequest<'a, 'b> +pub j2k_jpeg_metal::Rgb8MetalBatchRequest::op: j2k_jpeg_metal::Rgb8MetalBatchOp +pub j2k_jpeg_metal::Rgb8MetalBatchRequest::source: j2k_jpeg_metal::Rgb8MetalBatchSource<'a, 'b> +pub struct j2k_jpeg_metal::Surface +impl j2k_jpeg_metal::Surface +pub fn j2k_jpeg_metal::Surface::as_bytes(&self) -> &[u8] +pub fn j2k_jpeg_metal::Surface::download_into(&self, &mut [u8], usize) -> core::result::Result<(), j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::Surface::metal_buffer(&self) -> core::option::Option<(&metal::buffer::Buffer, usize)> +pub fn j2k_jpeg_metal::Surface::pitch_bytes(&self) -> usize +pub fn j2k_jpeg_metal::Surface::residency(&self) -> j2k_jpeg_metal::SurfaceResidency +impl j2k_core::traits::DeviceSurface for j2k_jpeg_metal::Surface +pub fn j2k_jpeg_metal::Surface::backend_kind(&self) -> j2k_core::backend::BackendKind +pub fn j2k_jpeg_metal::Surface::byte_len(&self) -> usize +pub fn j2k_jpeg_metal::Surface::dimensions(&self) -> (u32, u32) +pub fn j2k_jpeg_metal::Surface::memory_range(&self) -> core::option::Option +pub fn j2k_jpeg_metal::Surface::pixel_format(&self) -> j2k_core::pixel::PixelFormat +pub fn j2k_jpeg_metal::Surface::residency(&self) -> j2k_core::accelerator::SurfaceResidency +pub fn j2k_jpeg_metal::encode_jpeg_baseline_batch_from_metal_buffers(&[j2k_jpeg_metal::JpegBaselineMetalEncodeTile<'_>], j2k_jpeg::encoder::JpegEncodeOptions, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result, j2k_jpeg_metal::Error> +pub fn j2k_jpeg_metal::encode_jpeg_baseline_from_metal_buffer(j2k_jpeg_metal::JpegBaselineMetalEncodeTile<'_>, j2k_jpeg::encoder::JpegEncodeOptions, &j2k_jpeg_metal::MetalBackendSession) -> core::result::Result +``` + +## `j2k-metal` + +```text +pub mod j2k_metal +pub use j2k_metal::J2kContext +pub use j2k_metal::J2kScratchPool +pub enum j2k_metal::Error +pub j2k_metal::Error::Buffer(j2k_core::error::BufferError) +pub j2k_metal::Error::Decode(j2k::error::J2kError) +pub j2k_metal::Error::MetalKernel +pub j2k_metal::Error::MetalKernel::message: alloc::string::String +pub j2k_metal::Error::MetalRuntime +pub j2k_metal::Error::MetalRuntime::message: alloc::string::String +pub j2k_metal::Error::MetalStatePoisoned +pub j2k_metal::Error::MetalStatePoisoned::state: &'static str +pub j2k_metal::Error::MetalUnavailable +pub j2k_metal::Error::UnsupportedBackend +pub j2k_metal::Error::UnsupportedBackend::request: j2k_core::backend::BackendRequest +pub j2k_metal::Error::UnsupportedMetalRequest +pub j2k_metal::Error::UnsupportedMetalRequest::reason: &'static str +impl j2k_core::error::CodecError for j2k_metal::Error +pub fn j2k_metal::Error::is_buffer_error(&self) -> bool +pub fn j2k_metal::Error::is_not_implemented(&self) -> bool +pub fn j2k_metal::Error::is_truncated(&self) -> bool +pub fn j2k_metal::Error::is_unsupported(&self) -> bool +pub enum j2k_metal::MetalEncodeInputStaging +pub j2k_metal::MetalEncodeInputStaging::AlreadyPaddedContiguous +pub j2k_metal::MetalEncodeInputStaging::CopyAndPad +pub enum j2k_metal::SurfaceResidency +pub j2k_metal::SurfaceResidency::CpuStagedMetalUpload +pub j2k_metal::SurfaceResidency::Host +pub j2k_metal::SurfaceResidency::MetalResidentDecode +pub struct j2k_metal::Codec +impl j2k_core::traits::ImageCodec for j2k_metal::Codec +pub type j2k_metal::Codec::Error = j2k_metal::Error +pub type j2k_metal::Codec::Pool = j2k::scratch::J2kScratchPool +pub type j2k_metal::Codec::Warning = core::convert::Infallible +impl j2k_core::traits::TileBatchDecodeDevice for j2k_metal::Codec +pub type j2k_metal::Codec::Context = j2k::context::J2kContext +pub type j2k_metal::Codec::DeviceSurface = j2k_metal::Surface +impl j2k_core::traits::TileBatchDecodeManyDevice for j2k_metal::Codec +pub type j2k_metal::Codec::Context = j2k::context::J2kContext +pub type j2k_metal::Codec::DeviceSurface = j2k_metal::Surface +pub fn j2k_metal::Codec::decode_tiles_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[&[u8]], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result, Self::Error> +impl j2k_core::traits::TileBatchDecodeSubmit for j2k_metal::Codec +pub type j2k_metal::Codec::Context = j2k::context::J2kContext +pub type j2k_metal::Codec::DeviceSurface = j2k_metal::Surface +pub type j2k_metal::Codec::Session = j2k_metal::MetalSession +pub type j2k_metal::Codec::SubmittedSurface = j2k_metal::batch::MetalSubmission +pub fn j2k_metal::Codec::submit_tile_region_scaled_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::Codec::submit_tile_region_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::Codec::submit_tile_scaled_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::Codec::submit_tile_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub struct j2k_metal::J2kDecoder<'a> +impl<'a> j2k_metal::J2kDecoder<'a> +pub fn j2k_metal::J2kDecoder<'a>::decode_region_scaled_to_cpu_staged_metal_surface_with_session(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, &j2k_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::decode_region_scaled_to_device_with_session(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, &j2k_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::decode_region_scaled_to_host_surface(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::decode_region_to_cpu_staged_metal_surface_with_session(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, &j2k_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::decode_region_to_device_with_session(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, &j2k_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::decode_region_to_host_surface(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::decode_scaled_to_cpu_staged_metal_surface_with_session(&mut self, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, &j2k_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::decode_scaled_to_device_with_session(&mut self, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, &j2k_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::decode_scaled_to_host_surface(&mut self, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::decode_to_cpu_staged_metal_surface_with_session(&mut self, j2k_core::pixel::PixelFormat, &j2k_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::decode_to_device_with_session(&mut self, j2k_core::pixel::PixelFormat, &j2k_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::decode_to_host_surface(&mut self, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::from_view(j2k::view::J2kView<'a>) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::inner(&self) -> &j2k::view::J2kDecoder<'a> +pub fn j2k_metal::J2kDecoder<'a>::new(&'a [u8]) -> core::result::Result +impl j2k_core::traits::ImageCodec for j2k_metal::J2kDecoder<'_> +pub type j2k_metal::J2kDecoder<'_>::Error = j2k_metal::Error +pub type j2k_metal::J2kDecoder<'_>::Pool = j2k::scratch::J2kScratchPool +pub type j2k_metal::J2kDecoder<'_>::Warning = core::convert::Infallible +impl<'a> j2k_core::traits::ImageDecode<'a> for j2k_metal::J2kDecoder<'a> +pub type j2k_metal::J2kDecoder<'a>::View = j2k::view::J2kView<'a> +pub fn j2k_metal::J2kDecoder<'a>::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_metal::J2kDecoder<'a>::decode_into_with_scratch(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_metal::J2kDecoder<'a>::decode_region_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k_metal::J2kDecoder<'a>::decode_region_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_metal::J2kDecoder<'a>::decode_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_metal::J2kDecoder<'a>::from_view(Self::View) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::inspect(&'a [u8]) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::parse(&'a [u8]) -> core::result::Result +impl<'a> j2k_core::traits::ImageDecodeDevice<'a> for j2k_metal::J2kDecoder<'a> +pub type j2k_metal::J2kDecoder<'a>::DeviceSurface = j2k_metal::Surface +impl<'a> j2k_core::traits::ImageDecodeSubmit<'a> for j2k_metal::J2kDecoder<'a> +pub type j2k_metal::J2kDecoder<'a>::DeviceSurface = j2k_metal::Surface +pub type j2k_metal::J2kDecoder<'a>::Session = j2k_metal::MetalSession +pub type j2k_metal::J2kDecoder<'a>::SubmittedSurface = j2k_core::traits::ReadySubmission +pub fn j2k_metal::J2kDecoder<'a>::submit_region_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::submit_region_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::submit_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::J2kDecoder<'a>::submit_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub struct j2k_metal::MetalBackendSession +impl j2k_metal::MetalBackendSession +pub fn j2k_metal::MetalBackendSession::device(&self) -> &metal::device::DeviceRef +pub fn j2k_metal::MetalBackendSession::new(metal::device::Device) -> Self +pub fn j2k_metal::MetalBackendSession::system_default() -> core::result::Result +impl core::fmt::Debug for j2k_metal::MetalBackendSession +pub fn j2k_metal::MetalBackendSession::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl j2k_core::accelerator::AcceleratorSession for j2k_metal::MetalBackendSession +pub fn j2k_metal::MetalBackendSession::backend_kind(&self) -> j2k_core::backend::BackendKind +pub struct j2k_metal::MetalEncodeStageAccelerator +impl j2k_metal::MetalEncodeStageAccelerator +pub fn j2k_metal::MetalEncodeStageAccelerator::deinterleave_attempts(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::deinterleave_dispatches(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::for_auto_host_output() -> Self +pub fn j2k_metal::MetalEncodeStageAccelerator::for_ht_code_block_encode() -> Self +pub fn j2k_metal::MetalEncodeStageAccelerator::forward_dwt53_attempts(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::forward_dwt53_dispatches(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::forward_dwt97_attempts(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::forward_dwt97_dispatches(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::forward_ict_attempts(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::forward_ict_dispatches(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::forward_rct_attempts(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::forward_rct_dispatches(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::ht_code_block_attempts(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::ht_code_block_dispatches(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::packetization_attempts(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::packetization_dispatches(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::quantize_subband_attempts(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::quantize_subband_dispatches(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::tier1_code_block_attempts(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::tier1_code_block_dispatches(&self) -> usize +pub fn j2k_metal::MetalEncodeStageAccelerator::with_cpu_forward_rct() -> Self +impl core::default::Default for j2k_metal::MetalEncodeStageAccelerator +pub fn j2k_metal::MetalEncodeStageAccelerator::default() -> Self +impl j2k::adapter::encode_stage::J2kEncodeStageAccelerator for j2k_metal::MetalEncodeStageAccelerator +pub fn j2k_metal::MetalEncodeStageAccelerator::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport +pub fn j2k_metal::MetalEncodeStageAccelerator::encode_deinterleave(&mut self, j2k_types::J2kDeinterleaveToF32Job<'_>) -> core::result::Result>>, &'static str> +pub fn j2k_metal::MetalEncodeStageAccelerator::encode_forward_dwt53(&mut self, j2k_types::J2kForwardDwt53Job<'_>) -> core::result::Result, &'static str> +pub fn j2k_metal::MetalEncodeStageAccelerator::encode_forward_dwt97(&mut self, j2k_types::J2kForwardDwt97Job<'_>) -> core::result::Result, &'static str> +pub fn j2k_metal::MetalEncodeStageAccelerator::encode_forward_ict(&mut self, j2k_types::J2kForwardIctJob<'_>) -> core::result::Result +pub fn j2k_metal::MetalEncodeStageAccelerator::encode_forward_rct(&mut self, j2k_types::J2kForwardRctJob<'_>) -> core::result::Result +pub fn j2k_metal::MetalEncodeStageAccelerator::encode_ht_code_block(&mut self, j2k_types::J2kHtCodeBlockEncodeJob<'_>) -> core::result::Result, &'static str> +pub fn j2k_metal::MetalEncodeStageAccelerator::encode_ht_code_blocks(&mut self, &[j2k_types::J2kHtCodeBlockEncodeJob<'_>]) -> core::result::Result>, &'static str> +pub fn j2k_metal::MetalEncodeStageAccelerator::encode_htj2k_tile(&mut self, j2k_types::J2kHtj2kTileEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_metal::MetalEncodeStageAccelerator::encode_packetization(&mut self, j2k_types::J2kPacketizationEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_metal::MetalEncodeStageAccelerator::encode_quantize_subband(&mut self, j2k_types::J2kQuantizeSubbandJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_metal::MetalEncodeStageAccelerator::encode_tier1_code_block(&mut self, j2k_types::J2kTier1CodeBlockEncodeJob<'_>) -> core::result::Result, &'static str> +pub fn j2k_metal::MetalEncodeStageAccelerator::encode_tier1_code_blocks(&mut self, &[j2k_types::J2kTier1CodeBlockEncodeJob<'_>]) -> core::result::Result>, &'static str> +pub fn j2k_metal::MetalEncodeStageAccelerator::prefer_parallel_cpu_code_block_fallback(&self) -> bool +pub struct j2k_metal::MetalEncodedJ2k +pub j2k_metal::MetalEncodedJ2k::bit_depth: u8 +pub j2k_metal::MetalEncodedJ2k::byte_len: usize +pub j2k_metal::MetalEncodedJ2k::byte_offset: usize +pub j2k_metal::MetalEncodedJ2k::capacity: usize +pub j2k_metal::MetalEncodedJ2k::codestream_buffer: metal::buffer::Buffer +pub j2k_metal::MetalEncodedJ2k::components: u8 +pub j2k_metal::MetalEncodedJ2k::height: u32 +pub j2k_metal::MetalEncodedJ2k::signed: bool +pub j2k_metal::MetalEncodedJ2k::width: u32 +impl j2k_metal::MetalEncodedJ2k +pub fn j2k_metal::MetalEncodedJ2k::codestream_bytes(&self) -> core::result::Result<&[u8], j2k_metal::Error> +pub fn j2k_metal::MetalEncodedJ2k::to_encoded_j2k(&self) -> core::result::Result +pub struct j2k_metal::MetalLosslessBufferEncodeBatchOutcome +pub j2k_metal::MetalLosslessBufferEncodeBatchOutcome::outcomes: alloc::vec::Vec +pub j2k_metal::MetalLosslessBufferEncodeBatchOutcome::stats: j2k_metal::MetalLosslessEncodeBatchStats +pub struct j2k_metal::MetalLosslessBufferEncodeOutcome +pub j2k_metal::MetalLosslessBufferEncodeOutcome::encode_duration: core::time::Duration +pub j2k_metal::MetalLosslessBufferEncodeOutcome::encoded: j2k_metal::MetalEncodedJ2k +pub j2k_metal::MetalLosslessBufferEncodeOutcome::gpu_duration: core::option::Option +pub j2k_metal::MetalLosslessBufferEncodeOutcome::input_copy_duration: core::time::Duration +pub j2k_metal::MetalLosslessBufferEncodeOutcome::input_copy_used: bool +pub j2k_metal::MetalLosslessBufferEncodeOutcome::resident: j2k_metal::MetalLosslessEncodeResidency +pub j2k_metal::MetalLosslessBufferEncodeOutcome::validation_duration: core::time::Duration +pub struct j2k_metal::MetalLosslessEncodeBatchRequest<'a, 'b> +pub j2k_metal::MetalLosslessEncodeBatchRequest::config: j2k_metal::MetalLosslessEncodeConfig +pub j2k_metal::MetalLosslessEncodeBatchRequest::staging: j2k_metal::MetalEncodeInputStaging +pub j2k_metal::MetalLosslessEncodeBatchRequest::tiles: &'a [j2k_metal::MetalLosslessEncodeTile<'b>] +pub struct j2k_metal::MetalLosslessEncodeBatchStats +pub j2k_metal::MetalLosslessEncodeBatchStats::configured_inflight_tiles: core::option::Option +pub j2k_metal::MetalLosslessEncodeBatchStats::configured_memory_budget_bytes: core::option::Option +pub j2k_metal::MetalLosslessEncodeBatchStats::effective_inflight_tiles: usize +pub j2k_metal::MetalLosslessEncodeBatchStats::effective_memory_budget_bytes: usize +pub j2k_metal::MetalLosslessEncodeBatchStats::encode_wall_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeBatchStats::estimated_peak_bytes_per_tile: usize +pub j2k_metal::MetalLosslessEncodeBatchStats::max_observed_inflight_tiles: usize +pub j2k_metal::MetalLosslessEncodeBatchStats::stage_stats: j2k_metal::MetalLosslessEncodeStageStats +pub struct j2k_metal::MetalLosslessEncodeConfig +pub j2k_metal::MetalLosslessEncodeConfig::gpu_encode_inflight_tiles: core::option::Option +pub j2k_metal::MetalLosslessEncodeConfig::gpu_encode_memory_budget_bytes: core::option::Option +pub struct j2k_metal::MetalLosslessEncodeOutcome +pub j2k_metal::MetalLosslessEncodeOutcome::encode_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeOutcome::encoded: j2k::encode::EncodedJ2k +pub j2k_metal::MetalLosslessEncodeOutcome::gpu_duration: core::option::Option +pub j2k_metal::MetalLosslessEncodeOutcome::host_readback_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeOutcome::input_copy_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeOutcome::input_copy_used: bool +pub j2k_metal::MetalLosslessEncodeOutcome::resident: j2k_metal::MetalLosslessEncodeResidency +pub j2k_metal::MetalLosslessEncodeOutcome::validation_duration: core::time::Duration +pub struct j2k_metal::MetalLosslessEncodeResidency +pub j2k_metal::MetalLosslessEncodeResidency::codestream_assembly_used: bool +pub j2k_metal::MetalLosslessEncodeResidency::coefficient_prep_used: bool +pub j2k_metal::MetalLosslessEncodeResidency::packetization_used: bool +pub struct j2k_metal::MetalLosslessEncodeStageStats +pub j2k_metal::MetalLosslessEncodeStageStats::chunk_count: usize +pub j2k_metal::MetalLosslessEncodeStageStats::classic_block_encode_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_block_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_command_buffer_commit_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_packet_buffer_setup_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_packet_plan_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_tier1_arithmetic_pack_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_tier1_density_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_tier1_pass_plan_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_tier1_raw_pack_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_tier1_setup_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_tier1_split_token_emit_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_tier1_symbol_plan_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_tier1_token_emit_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_tier1_token_pack_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::classic_tier1_token_pack_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::code_block_count: usize +pub j2k_metal::MetalLosslessEncodeStageStats::codestream_assembly_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::codestream_assembly_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::codestream_payload_copy_active_thread_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::codestream_payload_copy_bytes_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::codestream_payload_copy_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::codestream_payload_copy_launched_thread_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::codestream_wait_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::coefficient_copy_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::coefficient_deinterleave_rct_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::coefficient_dwt53_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::coefficient_dwt53_horizontal_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::coefficient_dwt53_vertical_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::coefficient_extract_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::coefficient_extract_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::coefficient_prep_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::coefficient_prep_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::deinterleave_rct_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::dwt53_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::gpu_elapsed_wall_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::host_readback_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::ht_block_encode_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::ht_block_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::ht_buffer_allocation_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::ht_command_encode_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::ht_table_build_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::max_packet_output_capacity: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_packet_output_used_bytes: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_packet_payload_copy_bytes_per_tile: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_packet_payload_copy_jobs_per_tile: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_packet_payload_copy_jobs_used_per_tile: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_coding_passes_per_block: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_full_scan_coeff_visits_per_block: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_missing_bitplanes_per_block: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_output_capacity: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_output_used_bytes: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_pass_plan_mq_symbols_per_pass: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_pass_plan_raw_bits_per_pass: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_segment_capacity_per_block: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_segments_per_block: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_symbol_plan_mq_symbols_per_block: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_symbol_plan_packed_token_bytes_per_block: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_symbol_plan_raw_bits_per_block: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_token_emit_segments_per_block: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_token_emit_token_bytes_per_block: usize +pub j2k_metal::MetalLosslessEncodeStageStats::max_tier1_token_pack_output_bytes_per_block: usize +pub j2k_metal::MetalLosslessEncodeStageStats::packet_block_prep_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::packet_block_prep_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::packet_output_capacity_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::packet_output_used_bytes_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::packet_payload_copy_active_stripe_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::packet_payload_copy_bytes_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::packet_payload_copy_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::packet_payload_copy_job_capacity_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::packet_payload_copy_job_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::packet_payload_copy_large_job_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::packet_payload_copy_launched_stripe_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::packet_payload_copy_medium_job_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::packet_payload_copy_small_job_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::packetization_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::packetization_gpu_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::plan_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::prepare_submit_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::result_codestream_collect_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::result_harvest_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::result_private_recycle_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::result_shared_recycle_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::result_status_copy_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::sync_wait_duration: core::time::Duration +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_arithmetic_cleanup_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_arithmetic_magref_active_candidate_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_arithmetic_magref_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_arithmetic_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_arithmetic_scan_coeff_visit_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_arithmetic_sigprop_active_candidate_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_arithmetic_sigprop_new_significant_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_arithmetic_sigprop_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_cleanup_active_candidate_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_cleanup_new_significant_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_cleanup_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_cleanup_rlc_stripe_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_cleanup_rlc_zero_stripe_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_cleanup_scan_coeff_visit_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_coding_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_full_scan_coeff_visit_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_magref_active_candidate_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_magref_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_magref_scan_coeff_visit_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_missing_bitplane_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_nonzero_block_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_output_capacity_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_output_used_bytes_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_pass_plan_mq_symbol_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_pass_plan_nonempty_mq_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_pass_plan_nonempty_raw_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_pass_plan_raw_bit_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_raw_magref_active_candidate_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_raw_magref_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_raw_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_raw_scan_coeff_visit_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_raw_sigprop_active_candidate_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_raw_sigprop_new_significant_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_raw_sigprop_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_segment_capacity_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_segment_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_sigprop_active_candidate_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_sigprop_new_significant_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_sigprop_pass_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_sigprop_scan_coeff_visit_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_symbol_plan_cleanup_mq_symbol_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_symbol_plan_cleanup_sign_symbol_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_symbol_plan_magref_mq_symbol_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_symbol_plan_mq_symbol_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_symbol_plan_mq_symbol_hash_xor: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_symbol_plan_packed_token_bytes_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_symbol_plan_raw_bit_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_symbol_plan_raw_bit_hash_xor: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_symbol_plan_raw_magref_bit_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_symbol_plan_raw_sigprop_bit_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_symbol_plan_sigprop_mq_symbol_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_symbol_plan_sigprop_sign_symbol_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_token_emit_mq_symbol_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_token_emit_mq_symbol_hash_xor: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_token_emit_raw_bit_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_token_emit_raw_bit_hash_xor: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_token_emit_segment_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_token_emit_token_bytes_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_token_pack_output_bytes_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tier1_zero_block_count_total: usize +pub j2k_metal::MetalLosslessEncodeStageStats::tile_count: usize +impl j2k_metal::MetalLosslessEncodeStageStats +pub fn j2k_metal::MetalLosslessEncodeStageStats::add_assign(&mut self, Self) +pub fn j2k_metal::MetalLosslessEncodeStageStats::has_timings(&self) -> bool +pub struct j2k_metal::MetalLosslessEncodeTile<'a> +pub j2k_metal::MetalLosslessEncodeTile::buffer: &'a metal::buffer::Buffer +pub j2k_metal::MetalLosslessEncodeTile::byte_offset: usize +pub j2k_metal::MetalLosslessEncodeTile::format: j2k_core::pixel::PixelFormat +pub j2k_metal::MetalLosslessEncodeTile::height: u32 +pub j2k_metal::MetalLosslessEncodeTile::output_height: u32 +pub j2k_metal::MetalLosslessEncodeTile::output_width: u32 +pub j2k_metal::MetalLosslessEncodeTile::pitch_bytes: usize +pub j2k_metal::MetalLosslessEncodeTile::width: u32 +pub struct j2k_metal::MetalSession +impl j2k_metal::MetalSession +pub fn j2k_metal::MetalSession::backend_session(&self) -> core::option::Option<&j2k_metal::MetalBackendSession> +pub fn j2k_metal::MetalSession::submissions(&self) -> core::result::Result +pub fn j2k_metal::MetalSession::with_backend_session(j2k_metal::MetalBackendSession) -> Self +impl core::fmt::Debug for j2k_metal::MetalSession +pub fn j2k_metal::MetalSession::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_metal::MetalTileBatch +impl j2k_metal::MetalTileBatch +pub fn j2k_metal::MetalTileBatch::decode_all(self) -> core::result::Result, j2k_metal::Error> +pub fn j2k_metal::MetalTileBatch::is_empty(&self) -> bool +pub fn j2k_metal::MetalTileBatch::len(&self) -> usize +pub fn j2k_metal::MetalTileBatch::new() -> Self +pub fn j2k_metal::MetalTileBatch::push_shared_tile(&mut self, alloc::sync::Arc<[u8]>, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::MetalTileBatch::push_shared_tile_region(&mut self, alloc::sync::Arc<[u8]>, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::MetalTileBatch::push_shared_tile_region_scaled(&mut self, alloc::sync::Arc<[u8]>, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::MetalTileBatch::push_shared_tile_scaled(&mut self, alloc::sync::Arc<[u8]>, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::MetalTileBatch::push_tile(&mut self, &[u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::MetalTileBatch::push_tile_region(&mut self, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::MetalTileBatch::push_tile_region_scaled(&mut self, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::MetalTileBatch::push_tile_scaled(&mut self, &[u8], j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_metal::MetalTileBatch::submissions(&self) -> core::result::Result +pub fn j2k_metal::MetalTileBatch::with_capacity(usize) -> Self +pub struct j2k_metal::SubmittedJ2kLosslessMetalBufferEncodeBatch +impl j2k_core::traits::DeviceSubmission for j2k_metal::SubmittedJ2kLosslessMetalBufferEncodeBatch +pub type j2k_metal::SubmittedJ2kLosslessMetalBufferEncodeBatch::Error = j2k_metal::Error +pub type j2k_metal::SubmittedJ2kLosslessMetalBufferEncodeBatch::Output = j2k_metal::MetalLosslessBufferEncodeBatchOutcome +pub fn j2k_metal::SubmittedJ2kLosslessMetalBufferEncodeBatch::wait(self) -> core::result::Result +pub struct j2k_metal::SubmittedJ2kLosslessMetalEncodeBatch +impl j2k_core::traits::DeviceSubmission for j2k_metal::SubmittedJ2kLosslessMetalEncodeBatch +pub type j2k_metal::SubmittedJ2kLosslessMetalEncodeBatch::Error = j2k_metal::Error +pub type j2k_metal::SubmittedJ2kLosslessMetalEncodeBatch::Output = alloc::vec::Vec +pub fn j2k_metal::SubmittedJ2kLosslessMetalEncodeBatch::wait(self) -> core::result::Result +pub struct j2k_metal::Surface +impl j2k_metal::Surface +pub fn j2k_metal::Surface::as_bytes(&self) -> &[u8] +pub fn j2k_metal::Surface::download_into(&self, &mut [u8], usize) -> core::result::Result<(), j2k_metal::Error> +pub fn j2k_metal::Surface::metal_buffer(&self) -> core::option::Option<(&metal::buffer::Buffer, usize)> +pub fn j2k_metal::Surface::pitch_bytes(&self) -> usize +pub fn j2k_metal::Surface::residency(&self) -> j2k_metal::SurfaceResidency +impl j2k_core::traits::DeviceSurface for j2k_metal::Surface +pub fn j2k_metal::Surface::backend_kind(&self) -> j2k_core::backend::BackendKind +pub fn j2k_metal::Surface::byte_len(&self) -> usize +pub fn j2k_metal::Surface::dimensions(&self) -> (u32, u32) +pub fn j2k_metal::Surface::memory_range(&self) -> core::option::Option +pub fn j2k_metal::Surface::pixel_format(&self) -> j2k_core::pixel::PixelFormat +pub fn j2k_metal::Surface::residency(&self) -> j2k_core::accelerator::SurfaceResidency +pub fn j2k_metal::encode_lossless_batch_with_report(j2k_metal::MetalLosslessEncodeBatchRequest<'_, '_>, &j2k::encode::J2kLosslessEncodeOptions, &j2k_metal::MetalBackendSession) -> core::result::Result, j2k_metal::Error> +pub fn j2k_metal::submit_lossless_batch(j2k_metal::MetalLosslessEncodeBatchRequest<'_, '_>, &j2k::encode::J2kLosslessEncodeOptions, &j2k_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_metal::submit_lossless_batch_to_metal(j2k_metal::MetalLosslessEncodeBatchRequest<'_, '_>, &j2k::encode::J2kLosslessEncodeOptions, &j2k_metal::MetalBackendSession) -> core::result::Result +pub fn j2k_metal::validate_lossless_roundtrip_on_metal(j2k::encode::J2kLosslessSamples<'_>, &[u8]) -> core::result::Result<(), j2k_metal::Error> +pub fn j2k_metal::validate_lossless_roundtrip_on_metal_with_session(j2k::encode::J2kLosslessSamples<'_>, &[u8], &j2k_metal::MetalBackendSession) -> core::result::Result<(), j2k_metal::Error> +``` + +## `j2k-jpeg-cuda` + +```text +pub mod j2k_jpeg_cuda +pub use j2k_jpeg_cuda::DecoderContext +pub use j2k_jpeg_cuda::ScratchPool +pub enum j2k_jpeg_cuda::CudaJpegDecodePath +pub j2k_jpeg_cuda::CudaJpegDecodePath::None +pub j2k_jpeg_cuda::CudaJpegDecodePath::OwnedCuda +pub enum j2k_jpeg_cuda::Error +pub j2k_jpeg_cuda::Error::Buffer(j2k_core::error::BufferError) +pub j2k_jpeg_cuda::Error::CudaRuntime +pub j2k_jpeg_cuda::Error::CudaRuntime::message: alloc::string::String +pub j2k_jpeg_cuda::Error::CudaUnavailable +pub j2k_jpeg_cuda::Error::Decode(j2k_jpeg::error::JpegError) +pub j2k_jpeg_cuda::Error::UnsupportedBackend +pub j2k_jpeg_cuda::Error::UnsupportedBackend::request: j2k_core::backend::BackendRequest +pub j2k_jpeg_cuda::Error::UnsupportedCudaRequest +pub j2k_jpeg_cuda::Error::UnsupportedCudaRequest::reason: &'static str +impl j2k_core::error::CodecError for j2k_jpeg_cuda::Error +pub fn j2k_jpeg_cuda::Error::is_buffer_error(&self) -> bool +pub fn j2k_jpeg_cuda::Error::is_not_implemented(&self) -> bool +pub fn j2k_jpeg_cuda::Error::is_truncated(&self) -> bool +pub fn j2k_jpeg_cuda::Error::is_unsupported(&self) -> bool +pub struct j2k_jpeg_cuda::Codec +impl j2k_jpeg_cuda::Codec +pub fn j2k_jpeg_cuda::Codec::decode_tile_rgb8_into_cuda_buffer_with_session(&[u8], &j2k_cuda_runtime::memory::CudaDeviceBuffer, usize, &mut j2k_jpeg_cuda::CudaSession) -> core::result::Result +pub fn j2k_jpeg_cuda::Codec::decode_tiles_to_device_with_session(&[&[u8]], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest, &mut j2k_jpeg_cuda::CudaSession) -> core::result::Result, j2k_jpeg_cuda::Error> +pub fn j2k_jpeg_cuda::Codec::diagnose_tile_rgb8_chunked_entropy_with_session(&[u8], j2k_cuda_runtime::jpeg::CudaJpegChunkedEntropyConfig, &mut j2k_jpeg_cuda::CudaSession) -> core::result::Result +impl j2k_core::traits::ImageCodec for j2k_jpeg_cuda::Codec +pub type j2k_jpeg_cuda::Codec::Error = j2k_jpeg_cuda::Error +pub type j2k_jpeg_cuda::Codec::Pool = j2k_jpeg::internal::scratch::ScratchPool +pub type j2k_jpeg_cuda::Codec::Warning = j2k_jpeg::error::Warning +impl j2k_core::traits::TileBatchDecodeDevice for j2k_jpeg_cuda::Codec +pub type j2k_jpeg_cuda::Codec::Context = j2k_jpeg::context::DecoderContext +pub type j2k_jpeg_cuda::Codec::DeviceSurface = j2k_jpeg_cuda::Surface +impl j2k_core::traits::TileBatchDecodeManyDevice for j2k_jpeg_cuda::Codec +pub type j2k_jpeg_cuda::Codec::Context = j2k_jpeg::context::DecoderContext +pub type j2k_jpeg_cuda::Codec::DeviceSurface = j2k_jpeg_cuda::Surface +pub fn j2k_jpeg_cuda::Codec::decode_tiles_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[&[u8]], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result, Self::Error> +impl j2k_core::traits::TileBatchDecodeSubmit for j2k_jpeg_cuda::Codec +pub type j2k_jpeg_cuda::Codec::Context = j2k_jpeg::context::DecoderContext +pub type j2k_jpeg_cuda::Codec::DeviceSurface = j2k_jpeg_cuda::Surface +pub type j2k_jpeg_cuda::Codec::Session = j2k_jpeg_cuda::CudaSession +pub type j2k_jpeg_cuda::Codec::SubmittedSurface = j2k_core::traits::ReadySubmission +pub fn j2k_jpeg_cuda::Codec::submit_tile_region_scaled_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_cuda::Codec::submit_tile_region_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_cuda::Codec::submit_tile_scaled_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_cuda::Codec::submit_tile_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub struct j2k_jpeg_cuda::CudaSession +impl j2k_jpeg_cuda::CudaSession +pub fn j2k_jpeg_cuda::CudaSession::is_runtime_initialized(&self) -> bool +pub fn j2k_jpeg_cuda::CudaSession::owned_cuda_packet_cache_len(&self) -> usize +pub fn j2k_jpeg_cuda::CudaSession::recycle_owned_cuda_output_buffer(&mut self, j2k_cuda_runtime::memory::CudaDeviceBuffer) -> core::result::Result<(), j2k_jpeg_cuda::Error> +pub fn j2k_jpeg_cuda::CudaSession::retained_owned_cuda_output_buffers(&self) -> core::result::Result +pub fn j2k_jpeg_cuda::CudaSession::submissions(&self) -> u64 +pub fn j2k_jpeg_cuda::CudaSession::take_owned_cuda_output_buffer(&mut self, usize) -> core::result::Result +impl core::fmt::Debug for j2k_jpeg_cuda::CudaSession +pub fn j2k_jpeg_cuda::CudaSession::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl j2k_core::accelerator::AcceleratorSession for j2k_jpeg_cuda::CudaSession +pub fn j2k_jpeg_cuda::CudaSession::backend_kind(&self) -> j2k_core::backend::BackendKind +pub fn j2k_jpeg_cuda::CudaSession::execution_stats(&self) -> j2k_core::accelerator::ExecutionStats +impl j2k_core::traits::DeviceSubmitSession for j2k_jpeg_cuda::CudaSession +pub fn j2k_jpeg_cuda::CudaSession::record_submit(&mut self) +pub struct j2k_jpeg_cuda::CudaSurface<'a> +impl j2k_jpeg_cuda::CudaSurface<'_> +pub fn j2k_jpeg_cuda::CudaSurface<'_>::device_ptr(&self) -> u64 +pub fn j2k_jpeg_cuda::CudaSurface<'_>::stats(&self) -> j2k_jpeg_cuda::CudaSurfaceStats +pub struct j2k_jpeg_cuda::CudaSurfaceStats +impl j2k_jpeg_cuda::CudaSurfaceStats +pub fn j2k_jpeg_cuda::CudaSurfaceStats::copy_kernel_dispatches(self) -> usize +pub fn j2k_jpeg_cuda::CudaSurfaceStats::decode_kernel_dispatches(self) -> usize +pub fn j2k_jpeg_cuda::CudaSurfaceStats::decode_path(self) -> j2k_jpeg_cuda::CudaJpegDecodePath +pub fn j2k_jpeg_cuda::CudaSurfaceStats::kernel_dispatches(self) -> usize +pub fn j2k_jpeg_cuda::CudaSurfaceStats::used_hardware_decode(self) -> bool +pub fn j2k_jpeg_cuda::CudaSurfaceStats::used_owned_cuda_decode(self) -> bool +pub struct j2k_jpeg_cuda::Decoder<'a> +impl<'a> j2k_jpeg_cuda::Decoder<'a> +pub fn j2k_jpeg_cuda::Decoder<'a>::new(&'a [u8]) -> core::result::Result +impl j2k_core::traits::ImageCodec for j2k_jpeg_cuda::Decoder<'_> +pub type j2k_jpeg_cuda::Decoder<'_>::Error = j2k_jpeg_cuda::Error +pub type j2k_jpeg_cuda::Decoder<'_>::Pool = j2k_jpeg::internal::scratch::ScratchPool +pub type j2k_jpeg_cuda::Decoder<'_>::Warning = j2k_jpeg::error::Warning +impl<'a> j2k_core::traits::ImageDecode<'a> for j2k_jpeg_cuda::Decoder<'a> +pub type j2k_jpeg_cuda::Decoder<'a>::View = j2k_jpeg::decoder::JpegView<'a> +pub fn j2k_jpeg_cuda::Decoder<'a>::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_jpeg_cuda::Decoder<'a>::decode_into_with_scratch(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_jpeg_cuda::Decoder<'a>::decode_region_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k_jpeg_cuda::Decoder<'a>::decode_region_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_jpeg_cuda::Decoder<'a>::decode_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_jpeg_cuda::Decoder<'a>::from_view(Self::View) -> core::result::Result +pub fn j2k_jpeg_cuda::Decoder<'a>::inspect(&'a [u8]) -> core::result::Result +pub fn j2k_jpeg_cuda::Decoder<'a>::parse(&'a [u8]) -> core::result::Result +impl<'a> j2k_core::traits::ImageDecodeDevice<'a> for j2k_jpeg_cuda::Decoder<'a> +pub type j2k_jpeg_cuda::Decoder<'a>::DeviceSurface = j2k_jpeg_cuda::Surface +impl<'a> j2k_core::traits::ImageDecodeSubmit<'a> for j2k_jpeg_cuda::Decoder<'a> +pub type j2k_jpeg_cuda::Decoder<'a>::DeviceSurface = j2k_jpeg_cuda::Surface +pub type j2k_jpeg_cuda::Decoder<'a>::Session = j2k_jpeg_cuda::CudaSession +pub type j2k_jpeg_cuda::Decoder<'a>::SubmittedSurface = j2k_core::traits::ReadySubmission +pub fn j2k_jpeg_cuda::Decoder<'a>::submit_region_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_cuda::Decoder<'a>::submit_region_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_cuda::Decoder<'a>::submit_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_jpeg_cuda::Decoder<'a>::submit_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub struct j2k_jpeg_cuda::Surface +impl j2k_jpeg_cuda::Surface +pub fn j2k_jpeg_cuda::Surface::as_host_bytes(&self) -> core::option::Option<&[u8]> +pub fn j2k_jpeg_cuda::Surface::cuda_surface(&self) -> core::option::Option> +pub fn j2k_jpeg_cuda::Surface::download_into(&self, &mut [u8], usize) -> core::result::Result<(), j2k_jpeg_cuda::Error> +pub fn j2k_jpeg_cuda::Surface::pitch_bytes(&self) -> usize +impl j2k_core::traits::DeviceSurface for j2k_jpeg_cuda::Surface +pub fn j2k_jpeg_cuda::Surface::backend_kind(&self) -> j2k_core::backend::BackendKind +pub fn j2k_jpeg_cuda::Surface::byte_len(&self) -> usize +pub fn j2k_jpeg_cuda::Surface::dimensions(&self) -> (u32, u32) +pub fn j2k_jpeg_cuda::Surface::execution_stats(&self) -> j2k_core::accelerator::ExecutionStats +pub fn j2k_jpeg_cuda::Surface::memory_range(&self) -> core::option::Option +pub fn j2k_jpeg_cuda::Surface::pixel_format(&self) -> j2k_core::pixel::PixelFormat +pub fn j2k_jpeg_cuda::Surface::residency(&self) -> j2k_core::accelerator::SurfaceResidency +``` + +## `j2k-cuda` + +```text +pub mod j2k_cuda +pub use j2k_cuda::J2kContext +pub use j2k_cuda::J2kScratchPool +#[repr(u32)] pub enum j2k_cuda::CudaHtj2kTransform +pub j2k_cuda::CudaHtj2kTransform::Irreversible97 +pub j2k_cuda::CudaHtj2kTransform::Reversible53 +#[non_exhaustive] pub enum j2k_cuda::Error +pub j2k_cuda::Error::Buffer(j2k_core::error::BufferError) +pub j2k_cuda::Error::CudaRuntime +pub j2k_cuda::Error::CudaRuntime::message: alloc::string::String +pub j2k_cuda::Error::CudaUnavailable +pub j2k_cuda::Error::Decode(j2k::error::J2kError) +pub j2k_cuda::Error::UnsupportedBackend +pub j2k_cuda::Error::UnsupportedBackend::request: j2k_core::backend::BackendRequest +pub j2k_cuda::Error::UnsupportedCudaRequest +pub j2k_cuda::Error::UnsupportedCudaRequest::reason: &'static str +impl j2k_core::error::CodecError for j2k_cuda::Error +pub fn j2k_cuda::Error::is_buffer_error(&self) -> bool +pub fn j2k_cuda::Error::is_not_implemented(&self) -> bool +pub fn j2k_cuda::Error::is_truncated(&self) -> bool +pub fn j2k_cuda::Error::is_unsupported(&self) -> bool +#[non_exhaustive] pub enum j2k_cuda::SurfaceResidency +pub j2k_cuda::SurfaceResidency::CpuStagedCudaUpload +pub j2k_cuda::SurfaceResidency::CudaResidentDecode +pub j2k_cuda::SurfaceResidency::Host +pub struct j2k_cuda::Codec +impl j2k_core::traits::ImageCodec for j2k_cuda::Codec +pub type j2k_cuda::Codec::Error = j2k_cuda::Error +pub type j2k_cuda::Codec::Pool = j2k::scratch::J2kScratchPool +pub type j2k_cuda::Codec::Warning = core::convert::Infallible +impl j2k_core::traits::TileBatchDecodeDevice for j2k_cuda::Codec +pub type j2k_cuda::Codec::Context = j2k::context::J2kContext +pub type j2k_cuda::Codec::DeviceSurface = j2k_cuda::Surface +impl j2k_core::traits::TileBatchDecodeManyDevice for j2k_cuda::Codec +pub type j2k_cuda::Codec::Context = j2k::context::J2kContext +pub type j2k_cuda::Codec::DeviceSurface = j2k_cuda::Surface +pub fn j2k_cuda::Codec::decode_tiles_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Pool, &[&[u8]], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result, Self::Error> +impl j2k_core::traits::TileBatchDecodeSubmit for j2k_cuda::Codec +pub type j2k_cuda::Codec::Context = j2k::context::J2kContext +pub type j2k_cuda::Codec::DeviceSurface = j2k_cuda::Surface +pub type j2k_cuda::Codec::Session = j2k_cuda::CudaSession +pub type j2k_cuda::Codec::SubmittedSurface = j2k_core::traits::ReadySubmission +pub fn j2k_cuda::Codec::submit_tile_region_scaled_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_cuda::Codec::submit_tile_region_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_cuda::Codec::submit_tile_scaled_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_cuda::Codec::submit_tile_to_device(&mut j2k_core::context::DecoderContext, &mut Self::Session, &mut Self::Pool, &[u8], j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub struct j2k_cuda::CudaEncodeStageAccelerator +impl j2k_cuda::CudaEncodeStageAccelerator +pub const fn j2k_cuda::CudaEncodeStageAccelerator::collected_stage_timings(&self) -> j2k_cuda::CudaEncodeStageTimings +pub fn j2k_cuda::CudaEncodeStageAccelerator::deinterleave_attempts(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::deinterleave_dispatches(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::for_auto_host_output() -> Self +pub fn j2k_cuda::CudaEncodeStageAccelerator::forward_dwt53_attempts(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::forward_dwt53_dispatches(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::forward_dwt97_attempts(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::forward_dwt97_dispatches(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::forward_ict_attempts(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::forward_ict_dispatches(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::forward_rct_attempts(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::forward_rct_dispatches(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::ht_code_block_attempts(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::ht_code_block_dispatches(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::packetization_attempts(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::packetization_dispatches(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::prefer_cpu_forward_rct(self, bool) -> Self +pub fn j2k_cuda::CudaEncodeStageAccelerator::prefer_cpu_ht_subband(self, bool) -> Self +pub fn j2k_cuda::CudaEncodeStageAccelerator::prefer_cpu_packetization(self, bool) -> Self +pub fn j2k_cuda::CudaEncodeStageAccelerator::prefer_cpu_quantize_subband(self, bool) -> Self +pub fn j2k_cuda::CudaEncodeStageAccelerator::quantize_subband_attempts(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::quantize_subband_dispatches(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::reset_collected_stage_timings(&mut self) +pub fn j2k_cuda::CudaEncodeStageAccelerator::tier1_code_block_attempts(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::tier1_code_block_dispatches(&self) -> usize +pub fn j2k_cuda::CudaEncodeStageAccelerator::with_profile_collection(bool) -> Self +impl j2k::adapter::encode_stage::J2kEncodeStageAccelerator for j2k_cuda::CudaEncodeStageAccelerator +pub fn j2k_cuda::CudaEncodeStageAccelerator::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport +pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_deinterleave(&mut self, j2k_types::J2kDeinterleaveToF32Job<'_>) -> core::result::Result>>, &'static str> +pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_forward_dwt53(&mut self, j2k_types::J2kForwardDwt53Job<'_>) -> core::result::Result, &'static str> +pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_forward_dwt97(&mut self, j2k_types::J2kForwardDwt97Job<'_>) -> core::result::Result, &'static str> +pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_forward_ict(&mut self, j2k_types::J2kForwardIctJob<'_>) -> core::result::Result +pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_forward_rct(&mut self, j2k_types::J2kForwardRctJob<'_>) -> core::result::Result +pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_ht_code_block(&mut self, j2k_types::J2kHtCodeBlockEncodeJob<'_>) -> core::result::Result, &'static str> +pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_ht_code_blocks(&mut self, &[j2k_types::J2kHtCodeBlockEncodeJob<'_>]) -> core::result::Result>, &'static str> +pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_ht_subband(&mut self, j2k_types::J2kHtSubbandEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_htj2k_tile(&mut self, j2k_types::J2kHtj2kTileEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_packetization(&mut self, j2k_types::J2kPacketizationEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_quantize_subband(&mut self, j2k_types::J2kQuantizeSubbandJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_cuda::CudaEncodeStageAccelerator::encode_tier1_code_block(&mut self, j2k_types::J2kTier1CodeBlockEncodeJob<'_>) -> core::result::Result, &'static str> +pub struct j2k_cuda::CudaEncodeStageTimings +pub j2k_cuda::CudaEncodeStageTimings::deinterleave_us: u128 +pub j2k_cuda::CudaEncodeStageTimings::dwt_us: u128 +pub j2k_cuda::CudaEncodeStageTimings::ht_encode_us: u128 +pub j2k_cuda::CudaEncodeStageTimings::mct_us: u128 +pub j2k_cuda::CudaEncodeStageTimings::packetize_us: u128 +pub j2k_cuda::CudaEncodeStageTimings::quantize_us: u128 +impl j2k_cuda::CudaEncodeStageTimings +pub const fn j2k_cuda::CudaEncodeStageTimings::saturating_add(self, Self) -> Self +pub const fn j2k_cuda::CudaEncodeStageTimings::total_us(self) -> u128 +#[repr(C)] pub struct j2k_cuda::CudaHtj2kCodeBlock +pub j2k_cuda::CudaHtj2kCodeBlock::cleanup_length: u32 +pub j2k_cuda::CudaHtj2kCodeBlock::dequantization_step: f32 +pub j2k_cuda::CudaHtj2kCodeBlock::height: u32 +pub j2k_cuda::CudaHtj2kCodeBlock::missing_bit_planes: u8 +pub j2k_cuda::CudaHtj2kCodeBlock::num_bitplanes: u8 +pub j2k_cuda::CudaHtj2kCodeBlock::number_of_coding_passes: u8 +pub j2k_cuda::CudaHtj2kCodeBlock::output_stride: u32 +pub j2k_cuda::CudaHtj2kCodeBlock::output_x: u32 +pub j2k_cuda::CudaHtj2kCodeBlock::output_y: u32 +pub j2k_cuda::CudaHtj2kCodeBlock::payload_len: u32 +pub j2k_cuda::CudaHtj2kCodeBlock::payload_offset: u64 +pub j2k_cuda::CudaHtj2kCodeBlock::refinement_length: u32 +pub j2k_cuda::CudaHtj2kCodeBlock::stripe_causal: u8 +pub j2k_cuda::CudaHtj2kCodeBlock::subband_index: u32 +pub j2k_cuda::CudaHtj2kCodeBlock::width: u32 +pub struct j2k_cuda::CudaHtj2kDecodePlan +impl j2k_cuda::CudaHtj2kDecodePlan +pub fn j2k_cuda::CudaHtj2kDecodePlan::bit_depth(&self) -> u8 +pub fn j2k_cuda::CudaHtj2kDecodePlan::code_blocks(&self) -> &[j2k_cuda::CudaHtj2kCodeBlock] +pub fn j2k_cuda::CudaHtj2kDecodePlan::dimensions(&self) -> (u32, u32) +pub fn j2k_cuda::CudaHtj2kDecodePlan::dispatch_count_hint(&self) -> usize +pub fn j2k_cuda::CudaHtj2kDecodePlan::idwt_steps(&self) -> &[j2k_cuda::CudaHtj2kIdwtStep] +pub fn j2k_cuda::CudaHtj2kDecodePlan::output_format(&self) -> j2k_core::pixel::PixelFormat +pub fn j2k_cuda::CudaHtj2kDecodePlan::output_origin(&self) -> (u32, u32) +pub fn j2k_cuda::CudaHtj2kDecodePlan::payload(&self) -> &[u8] +pub fn j2k_cuda::CudaHtj2kDecodePlan::store_steps(&self) -> &[j2k_cuda::CudaHtj2kStoreStep] +pub fn j2k_cuda::CudaHtj2kDecodePlan::subbands(&self) -> &[j2k_cuda::CudaHtj2kSubband] +pub fn j2k_cuda::CudaHtj2kDecodePlan::transform(&self) -> j2k_cuda::CudaHtj2kTransform +pub struct j2k_cuda::CudaHtj2kDecodeProfileDetail +pub j2k_cuda::CudaHtj2kDecodeProfileDetail::dequant_dispatch_count: usize +pub j2k_cuda::CudaHtj2kDecodeProfileDetail::ht_dispatch_count: usize +pub j2k_cuda::CudaHtj2kDecodeProfileDetail::idwt_dispatch_count: usize +pub j2k_cuda::CudaHtj2kDecodeProfileDetail::job_upload_us: u128 +pub j2k_cuda::CudaHtj2kDecodeProfileDetail::mct_dispatch_count: usize +pub j2k_cuda::CudaHtj2kDecodeProfileDetail::output_d2h_us: u128 +pub j2k_cuda::CudaHtj2kDecodeProfileDetail::payload_upload_us: u128 +pub j2k_cuda::CudaHtj2kDecodeProfileDetail::stage_sum_us: u128 +pub j2k_cuda::CudaHtj2kDecodeProfileDetail::status_d2h_us: u128 +pub j2k_cuda::CudaHtj2kDecodeProfileDetail::store_dispatch_count: usize +pub j2k_cuda::CudaHtj2kDecodeProfileDetail::table_upload_us: u128 +pub j2k_cuda::CudaHtj2kDecodeProfileDetail::wall_total_us: u128 +pub struct j2k_cuda::CudaHtj2kEncodeProfileReport +pub j2k_cuda::CudaHtj2kEncodeProfileReport::backend: j2k_core::backend::BackendKind +pub j2k_cuda::CudaHtj2kEncodeProfileReport::block_count: usize +pub j2k_cuda::CudaHtj2kEncodeProfileReport::codestream_bytes: usize +pub j2k_cuda::CudaHtj2kEncodeProfileReport::deinterleave_us: u128 +pub j2k_cuda::CudaHtj2kEncodeProfileReport::dispatch_count: usize +pub j2k_cuda::CudaHtj2kEncodeProfileReport::dwt_us: u128 +pub j2k_cuda::CudaHtj2kEncodeProfileReport::ht_encode_us: u128 +pub j2k_cuda::CudaHtj2kEncodeProfileReport::input_bytes: usize +pub j2k_cuda::CudaHtj2kEncodeProfileReport::mct_us: u128 +pub j2k_cuda::CudaHtj2kEncodeProfileReport::packetize_us: u128 +pub j2k_cuda::CudaHtj2kEncodeProfileReport::quantize_us: u128 +pub j2k_cuda::CudaHtj2kEncodeProfileReport::total_us: u128 +impl j2k_cuda::CudaHtj2kEncodeProfileReport +pub fn j2k_cuda::CudaHtj2kEncodeProfileReport::emit(&self, &str) +impl core::default::Default for j2k_cuda::CudaHtj2kEncodeProfileReport +pub fn j2k_cuda::CudaHtj2kEncodeProfileReport::default() -> Self +#[repr(C)] pub struct j2k_cuda::CudaHtj2kIdwtStep +pub j2k_cuda::CudaHtj2kIdwtStep::hh_band_id: j2k_cuda::CudaHtj2kBandId +pub j2k_cuda::CudaHtj2kIdwtStep::hh_rect: j2k_cuda::CudaHtj2kRect +pub j2k_cuda::CudaHtj2kIdwtStep::hl_band_id: j2k_cuda::CudaHtj2kBandId +pub j2k_cuda::CudaHtj2kIdwtStep::hl_rect: j2k_cuda::CudaHtj2kRect +pub j2k_cuda::CudaHtj2kIdwtStep::lh_band_id: j2k_cuda::CudaHtj2kBandId +pub j2k_cuda::CudaHtj2kIdwtStep::lh_rect: j2k_cuda::CudaHtj2kRect +pub j2k_cuda::CudaHtj2kIdwtStep::ll_band_id: j2k_cuda::CudaHtj2kBandId +pub j2k_cuda::CudaHtj2kIdwtStep::ll_rect: j2k_cuda::CudaHtj2kRect +pub j2k_cuda::CudaHtj2kIdwtStep::output_band_id: j2k_cuda::CudaHtj2kBandId +pub j2k_cuda::CudaHtj2kIdwtStep::rect: j2k_cuda::CudaHtj2kRect +pub j2k_cuda::CudaHtj2kIdwtStep::transform: j2k_cuda::CudaHtj2kTransform +pub struct j2k_cuda::CudaHtj2kProfileReport +pub j2k_cuda::CudaHtj2kProfileReport::block_count: usize +pub j2k_cuda::CudaHtj2kProfileReport::dequant_us: u128 +pub j2k_cuda::CudaHtj2kProfileReport::detail: j2k_cuda::CudaHtj2kDecodeProfileDetail +pub j2k_cuda::CudaHtj2kProfileReport::dispatch_count: usize +pub j2k_cuda::CudaHtj2kProfileReport::flatten_us: u128 +pub j2k_cuda::CudaHtj2kProfileReport::h2d_us: u128 +pub j2k_cuda::CudaHtj2kProfileReport::ht_cleanup_us: u128 +pub j2k_cuda::CudaHtj2kProfileReport::ht_refine_us: u128 +pub j2k_cuda::CudaHtj2kProfileReport::idwt_us: u128 +pub j2k_cuda::CudaHtj2kProfileReport::mct_us: u128 +pub j2k_cuda::CudaHtj2kProfileReport::parse_us: u128 +pub j2k_cuda::CudaHtj2kProfileReport::payload_bytes: usize +pub j2k_cuda::CudaHtj2kProfileReport::plan_us: u128 +pub j2k_cuda::CudaHtj2kProfileReport::residency: j2k_cuda::SurfaceResidency +pub j2k_cuda::CudaHtj2kProfileReport::store_us: u128 +pub j2k_cuda::CudaHtj2kProfileReport::total_us: u128 +impl j2k_cuda::CudaHtj2kProfileReport +pub fn j2k_cuda::CudaHtj2kProfileReport::emit(&self, &str) +#[repr(C)] pub struct j2k_cuda::CudaHtj2kRect +pub j2k_cuda::CudaHtj2kRect::x0: u32 +pub j2k_cuda::CudaHtj2kRect::x1: u32 +pub j2k_cuda::CudaHtj2kRect::y0: u32 +pub j2k_cuda::CudaHtj2kRect::y1: u32 +#[repr(C)] pub struct j2k_cuda::CudaHtj2kStoreStep +pub j2k_cuda::CudaHtj2kStoreStep::addend: f32 +pub j2k_cuda::CudaHtj2kStoreStep::copy_height: u32 +pub j2k_cuda::CudaHtj2kStoreStep::copy_width: u32 +pub j2k_cuda::CudaHtj2kStoreStep::input_band_id: j2k_cuda::CudaHtj2kBandId +pub j2k_cuda::CudaHtj2kStoreStep::input_rect: j2k_cuda::CudaHtj2kRect +pub j2k_cuda::CudaHtj2kStoreStep::output_height: u32 +pub j2k_cuda::CudaHtj2kStoreStep::output_width: u32 +pub j2k_cuda::CudaHtj2kStoreStep::output_x: u32 +pub j2k_cuda::CudaHtj2kStoreStep::output_y: u32 +pub j2k_cuda::CudaHtj2kStoreStep::source_x: u32 +pub j2k_cuda::CudaHtj2kStoreStep::source_y: u32 +#[repr(C)] pub struct j2k_cuda::CudaHtj2kSubband +pub j2k_cuda::CudaHtj2kSubband::band_id: j2k_cuda::CudaHtj2kBandId +pub j2k_cuda::CudaHtj2kSubband::code_block_count: u32 +pub j2k_cuda::CudaHtj2kSubband::code_block_start: u32 +pub j2k_cuda::CudaHtj2kSubband::height: u32 +pub j2k_cuda::CudaHtj2kSubband::width: u32 +pub j2k_cuda::CudaHtj2kSubband::x0: u32 +pub j2k_cuda::CudaHtj2kSubband::x1: u32 +pub j2k_cuda::CudaHtj2kSubband::y0: u32 +pub j2k_cuda::CudaHtj2kSubband::y1: u32 +pub struct j2k_cuda::CudaLosslessEncodeOutcome +pub j2k_cuda::CudaLosslessEncodeOutcome::encode_duration: core::time::Duration +pub j2k_cuda::CudaLosslessEncodeOutcome::encoded: j2k::encode::EncodedJ2k +pub j2k_cuda::CudaLosslessEncodeOutcome::gpu_duration: core::option::Option +pub j2k_cuda::CudaLosslessEncodeOutcome::host_readback_duration: core::time::Duration +pub j2k_cuda::CudaLosslessEncodeOutcome::input_copy_duration: core::time::Duration +pub j2k_cuda::CudaLosslessEncodeOutcome::input_copy_used: bool +pub j2k_cuda::CudaLosslessEncodeOutcome::resident: j2k_cuda::CudaLosslessEncodeResidency +pub j2k_cuda::CudaLosslessEncodeOutcome::stage_timings: j2k_cuda::CudaEncodeStageTimings +pub j2k_cuda::CudaLosslessEncodeOutcome::validation_duration: core::time::Duration +pub struct j2k_cuda::CudaLosslessEncodeResidency +pub j2k_cuda::CudaLosslessEncodeResidency::codestream_assembly_used: bool +pub j2k_cuda::CudaLosslessEncodeResidency::coefficient_prep_used: bool +pub j2k_cuda::CudaLosslessEncodeResidency::packetization_used: bool +pub struct j2k_cuda::CudaLosslessEncodeTile<'a> +pub j2k_cuda::CudaLosslessEncodeTile::buffer: &'a j2k_cuda_runtime::memory::CudaDeviceBuffer +pub j2k_cuda::CudaLosslessEncodeTile::byte_offset: usize +pub j2k_cuda::CudaLosslessEncodeTile::format: j2k_core::pixel::PixelFormat +pub j2k_cuda::CudaLosslessEncodeTile::height: u32 +pub j2k_cuda::CudaLosslessEncodeTile::output_height: u32 +pub j2k_cuda::CudaLosslessEncodeTile::output_width: u32 +pub j2k_cuda::CudaLosslessEncodeTile::pitch_bytes: usize +pub j2k_cuda::CudaLosslessEncodeTile::width: u32 +pub struct j2k_cuda::CudaSession +impl j2k_cuda::CudaSession +pub fn j2k_cuda::CudaSession::is_runtime_initialized(&self) -> bool +pub fn j2k_cuda::CudaSession::submissions(&self) -> u64 +impl core::fmt::Debug for j2k_cuda::CudaSession +pub fn j2k_cuda::CudaSession::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl j2k_core::accelerator::AcceleratorSession for j2k_cuda::CudaSession +pub fn j2k_cuda::CudaSession::backend_kind(&self) -> j2k_core::backend::BackendKind +pub fn j2k_cuda::CudaSession::execution_stats(&self) -> j2k_core::accelerator::ExecutionStats +impl j2k_core::traits::DeviceSubmitSession for j2k_cuda::CudaSession +pub fn j2k_cuda::CudaSession::record_submit(&mut self) +pub struct j2k_cuda::CudaSurface<'a> +impl j2k_cuda::CudaSurface<'_> +pub fn j2k_cuda::CudaSurface<'_>::device_ptr(&self) -> u64 +pub fn j2k_cuda::CudaSurface<'_>::stats(&self) -> j2k_cuda::CudaSurfaceStats +pub struct j2k_cuda::CudaSurfaceStats +impl j2k_cuda::CudaSurfaceStats +pub fn j2k_cuda::CudaSurfaceStats::copy_kernel_dispatches(self) -> usize +pub fn j2k_cuda::CudaSurfaceStats::decode_kernel_dispatches(self) -> usize +pub fn j2k_cuda::CudaSurfaceStats::kernel_dispatches(self) -> usize +pub struct j2k_cuda::J2kDecoder<'a> +impl<'a> j2k_cuda::J2kDecoder<'a> +pub fn j2k_cuda::J2kDecoder<'a>::build_cuda_htj2k_grayscale_plan_with_profile(&mut self, j2k_core::pixel::PixelFormat) -> core::result::Result<(j2k_cuda::CudaHtj2kDecodePlan, j2k_cuda::CudaHtj2kProfileReport), j2k_cuda::Error> +pub fn j2k_cuda::J2kDecoder<'a>::build_cuda_htj2k_grayscale_region_plan_with_profile(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result<(j2k_cuda::CudaHtj2kDecodePlan, j2k_cuda::CudaHtj2kProfileReport), j2k_cuda::Error> +pub fn j2k_cuda::J2kDecoder<'a>::build_cuda_htj2k_grayscale_region_scaled_plan_with_profile(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, (u32, u32)) -> core::result::Result<(j2k_cuda::CudaHtj2kDecodePlan, j2k_cuda::CudaHtj2kProfileReport), j2k_cuda::Error> +pub fn j2k_cuda::J2kDecoder<'a>::build_cuda_htj2k_grayscale_scaled_plan_with_profile(&mut self, j2k_core::pixel::PixelFormat, (u32, u32)) -> core::result::Result<(j2k_cuda::CudaHtj2kDecodePlan, j2k_cuda::CudaHtj2kProfileReport), j2k_cuda::Error> +pub fn j2k_cuda::J2kDecoder<'a>::decode_batch_to_device_with_session(&[&[u8]], j2k_core::pixel::PixelFormat, &mut j2k_cuda::CudaSession) -> core::result::Result, j2k_cuda::Error> +pub fn j2k_cuda::J2kDecoder<'a>::decode_batch_to_device_with_session_and_profile(&[&[u8]], j2k_core::pixel::PixelFormat, &mut j2k_cuda::CudaSession) -> core::result::Result<(alloc::vec::Vec, j2k_cuda::CudaHtj2kProfileReport), j2k_cuda::Error> +pub fn j2k_cuda::J2kDecoder<'a>::decode_region_scaled_to_cpu_staged_cuda_surface_with_session(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, &mut j2k_cuda::CudaSession) -> core::result::Result +pub fn j2k_cuda::J2kDecoder<'a>::decode_region_to_cpu_staged_cuda_surface_with_session(&mut self, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, &mut j2k_cuda::CudaSession) -> core::result::Result +pub fn j2k_cuda::J2kDecoder<'a>::decode_scaled_to_cpu_staged_cuda_surface_with_session(&mut self, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, &mut j2k_cuda::CudaSession) -> core::result::Result +pub fn j2k_cuda::J2kDecoder<'a>::decode_to_cpu_staged_cuda_surface_with_session(&mut self, j2k_core::pixel::PixelFormat, &mut j2k_cuda::CudaSession) -> core::result::Result +pub fn j2k_cuda::J2kDecoder<'a>::decode_to_device_with_session(&mut self, j2k_core::pixel::PixelFormat, &mut j2k_cuda::CudaSession) -> core::result::Result +pub fn j2k_cuda::J2kDecoder<'a>::decode_to_device_with_session_and_profile(&mut self, j2k_core::pixel::PixelFormat, &mut j2k_cuda::CudaSession) -> core::result::Result<(j2k_cuda::Surface, j2k_cuda::CudaHtj2kProfileReport), j2k_cuda::Error> +pub fn j2k_cuda::J2kDecoder<'a>::decode_to_host_surface(&mut self, j2k_core::pixel::PixelFormat) -> core::result::Result +pub fn j2k_cuda::J2kDecoder<'a>::new(&'a [u8]) -> core::result::Result +impl j2k_core::traits::ImageCodec for j2k_cuda::J2kDecoder<'_> +pub type j2k_cuda::J2kDecoder<'_>::Error = j2k_cuda::Error +pub type j2k_cuda::J2kDecoder<'_>::Pool = j2k::scratch::J2kScratchPool +pub type j2k_cuda::J2kDecoder<'_>::Warning = core::convert::Infallible +impl<'a> j2k_core::traits::ImageDecode<'a> for j2k_cuda::J2kDecoder<'a> +pub type j2k_cuda::J2kDecoder<'a>::View = j2k::view::J2kView<'a> +pub fn j2k_cuda::J2kDecoder<'a>::decode_into(&mut self, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_cuda::J2kDecoder<'a>::decode_into_with_scratch(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat) -> core::result::Result, Self::Error> +pub fn j2k_cuda::J2kDecoder<'a>::decode_region_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect) -> core::result::Result, Self::Error> +pub fn j2k_cuda::J2kDecoder<'a>::decode_region_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_cuda::J2kDecoder<'a>::decode_scaled_into(&mut self, &mut Self::Pool, &mut [u8], usize, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale) -> core::result::Result, Self::Error> +pub fn j2k_cuda::J2kDecoder<'a>::from_view(Self::View) -> core::result::Result +pub fn j2k_cuda::J2kDecoder<'a>::inspect(&'a [u8]) -> core::result::Result +pub fn j2k_cuda::J2kDecoder<'a>::parse(&'a [u8]) -> core::result::Result +impl<'a> j2k_core::traits::ImageDecodeDevice<'a> for j2k_cuda::J2kDecoder<'a> +pub type j2k_cuda::J2kDecoder<'a>::DeviceSurface = j2k_cuda::Surface +impl<'a> j2k_core::traits::ImageDecodeSubmit<'a> for j2k_cuda::J2kDecoder<'a> +pub type j2k_cuda::J2kDecoder<'a>::DeviceSurface = j2k_cuda::Surface +pub type j2k_cuda::J2kDecoder<'a>::Session = j2k_cuda::CudaSession +pub type j2k_cuda::J2kDecoder<'a>::SubmittedSurface = j2k_core::traits::ReadySubmission +pub fn j2k_cuda::J2kDecoder<'a>::submit_region_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_cuda::J2kDecoder<'a>::submit_region_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::types::Rect, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_cuda::J2kDecoder<'a>::submit_scaled_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::scale::Downscale, j2k_core::backend::BackendRequest) -> core::result::Result +pub fn j2k_cuda::J2kDecoder<'a>::submit_to_device(&mut self, &mut Self::Session, j2k_core::pixel::PixelFormat, j2k_core::backend::BackendRequest) -> core::result::Result +pub struct j2k_cuda::SubmittedJ2kLosslessCudaEncode +impl j2k_core::traits::DeviceSubmission for j2k_cuda::SubmittedJ2kLosslessCudaEncode +pub type j2k_cuda::SubmittedJ2kLosslessCudaEncode::Error = j2k_cuda::Error +pub type j2k_cuda::SubmittedJ2kLosslessCudaEncode::Output = j2k::encode::EncodedJ2k +pub fn j2k_cuda::SubmittedJ2kLosslessCudaEncode::wait(self) -> core::result::Result +pub struct j2k_cuda::SubmittedJ2kLosslessCudaEncodeBatch +impl j2k_core::traits::DeviceSubmission for j2k_cuda::SubmittedJ2kLosslessCudaEncodeBatch +pub type j2k_cuda::SubmittedJ2kLosslessCudaEncodeBatch::Error = j2k_cuda::Error +pub type j2k_cuda::SubmittedJ2kLosslessCudaEncodeBatch::Output = alloc::vec::Vec +pub fn j2k_cuda::SubmittedJ2kLosslessCudaEncodeBatch::wait(self) -> core::result::Result +pub struct j2k_cuda::Surface +impl j2k_cuda::Surface +pub fn j2k_cuda::Surface::as_host_bytes(&self) -> core::option::Option<&[u8]> +pub fn j2k_cuda::Surface::cuda_surface(&self) -> core::option::Option> +pub fn j2k_cuda::Surface::download_batch_tight(&[Self]) -> core::result::Result, j2k_cuda::Error> +pub fn j2k_cuda::Surface::download_batch_tight_into(&[Self], &mut [u8]) -> core::result::Result<(), j2k_cuda::Error> +pub fn j2k_cuda::Surface::download_into(&self, &mut [u8], usize) -> core::result::Result<(), j2k_cuda::Error> +pub fn j2k_cuda::Surface::download_into_profiled(&self, &mut [u8], usize) -> core::result::Result +pub fn j2k_cuda::Surface::pitch_bytes(&self) -> usize +pub fn j2k_cuda::Surface::residency(&self) -> j2k_cuda::SurfaceResidency +impl j2k_core::traits::DeviceSurface for j2k_cuda::Surface +pub fn j2k_cuda::Surface::backend_kind(&self) -> j2k_core::backend::BackendKind +pub fn j2k_cuda::Surface::byte_len(&self) -> usize +pub fn j2k_cuda::Surface::dimensions(&self) -> (u32, u32) +pub fn j2k_cuda::Surface::execution_stats(&self) -> j2k_core::accelerator::ExecutionStats +pub fn j2k_cuda::Surface::memory_range(&self) -> core::option::Option +pub fn j2k_cuda::Surface::pixel_format(&self) -> j2k_core::pixel::PixelFormat +pub fn j2k_cuda::Surface::residency(&self) -> j2k_core::accelerator::SurfaceResidency +pub fn j2k_cuda::encode_j2k_lossless_with_cuda(j2k::encode::J2kLosslessSamples<'_>, &j2k::encode::J2kLosslessEncodeOptions) -> core::result::Result +pub fn j2k_cuda::encode_j2k_lossless_with_cuda_and_profile(j2k::encode::J2kLosslessSamples<'_>, &j2k::encode::J2kLosslessEncodeOptions) -> core::result::Result<(j2k::encode::EncodedJ2k, j2k_cuda::CudaHtj2kEncodeProfileReport), j2k_cuda::Error> +pub fn j2k_cuda::encode_lossless_from_cuda_buffer(j2k_cuda::CudaLosslessEncodeTile<'_>, &j2k::encode::J2kLosslessEncodeOptions, &mut j2k_cuda::CudaSession) -> core::result::Result +pub fn j2k_cuda::encode_lossless_from_cuda_buffer_with_report(j2k_cuda::CudaLosslessEncodeTile<'_>, &j2k::encode::J2kLosslessEncodeOptions, &mut j2k_cuda::CudaSession) -> core::result::Result +pub fn j2k_cuda::encode_lossless_from_cuda_buffers(&[j2k_cuda::CudaLosslessEncodeTile<'_>], &j2k::encode::J2kLosslessEncodeOptions, &mut j2k_cuda::CudaSession) -> core::result::Result, j2k_cuda::Error> +pub fn j2k_cuda::encode_lossless_from_cuda_buffers_with_report(&[j2k_cuda::CudaLosslessEncodeTile<'_>], &j2k::encode::J2kLosslessEncodeOptions, &mut j2k_cuda::CudaSession) -> core::result::Result, j2k_cuda::Error> +pub fn j2k_cuda::submit_lossless_from_cuda_buffer(j2k_cuda::CudaLosslessEncodeTile<'_>, &j2k::encode::J2kLosslessEncodeOptions, &mut j2k_cuda::CudaSession) -> core::result::Result +pub fn j2k_cuda::submit_lossless_from_cuda_buffers(&[j2k_cuda::CudaLosslessEncodeTile<'_>], &j2k::encode::J2kLosslessEncodeOptions, &mut j2k_cuda::CudaSession) -> core::result::Result +pub type j2k_cuda::CudaHtj2kBandId = u32 +``` + +## `j2k-transcode` + +```text +pub mod j2k_transcode +pub use j2k_transcode::BatchTranscodeReport +pub use j2k_transcode::EncodeProgressionOrder +pub use j2k_transcode::EncodedTranscode +pub use j2k_transcode::EncodedTranscodeBatch +pub use j2k_transcode::JPEG_TO_HTJ2K_LOSSY_97_QUANTIZATION_SCALE +pub use j2k_transcode::JpegTileBatchInput +pub use j2k_transcode::JpegToHtj2kCoefficientPath +pub use j2k_transcode::JpegToHtj2kEncodeOptions +pub use j2k_transcode::JpegToHtj2kError +pub use j2k_transcode::JpegToHtj2kOptions +pub use j2k_transcode::JpegToHtj2kTranscoder +pub use j2k_transcode::TranscodeComponentReport +pub use j2k_transcode::TranscodeReport +pub use j2k_transcode::TranscodeTimingReport +pub use j2k_transcode::TranscodeValidationClassification +pub use j2k_transcode::TranscodeValidationMetrics +pub use j2k_transcode::jpeg_to_htj2k +pub use j2k_transcode::jpeg_to_htj2k_batch +pub mod j2k_transcode::accelerator +pub use j2k_transcode::accelerator::EncodedHtJ2kCodeBlock +pub use j2k_transcode::accelerator::IrreversibleQuantizationSubbandScales +pub use j2k_transcode::accelerator::J2kSubBandType +pub use j2k_transcode::accelerator::PreencodedHtj2k97CodeBlock +pub use j2k_transcode::accelerator::PreencodedHtj2k97CompactCodeBlock +pub use j2k_transcode::accelerator::PreencodedHtj2k97CompactComponent +pub use j2k_transcode::accelerator::PreencodedHtj2k97CompactImage +pub use j2k_transcode::accelerator::PreencodedHtj2k97CompactResolution +pub use j2k_transcode::accelerator::PreencodedHtj2k97CompactSubband +pub use j2k_transcode::accelerator::PreencodedHtj2k97Component +pub use j2k_transcode::accelerator::PreencodedHtj2k97Resolution +pub use j2k_transcode::accelerator::PreencodedHtj2k97Subband +pub use j2k_transcode::accelerator::PrequantizedHtj2k97CodeBlock +pub use j2k_transcode::accelerator::PrequantizedHtj2k97Component +pub use j2k_transcode::accelerator::PrequantizedHtj2k97Image +pub use j2k_transcode::accelerator::PrequantizedHtj2k97Resolution +pub use j2k_transcode::accelerator::PrequantizedHtj2k97Subband +pub enum j2k_transcode::accelerator::TranscodeStageError +pub j2k_transcode::accelerator::TranscodeStageError::Backend(alloc::string::String) +pub j2k_transcode::accelerator::TranscodeStageError::DeviceUnavailable +pub j2k_transcode::accelerator::TranscodeStageError::Unsupported(&'static str) +impl core::convert::From<&'static str> for j2k_transcode::accelerator::TranscodeStageError +pub fn j2k_transcode::accelerator::TranscodeStageError::from(&'static str) -> Self +impl core::error::Error for j2k_transcode::accelerator::TranscodeStageError +impl core::fmt::Display for j2k_transcode::accelerator::TranscodeStageError +pub fn j2k_transcode::accelerator::TranscodeStageError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator +impl j2k_transcode::accelerator::DctToWaveletStageAccelerator for j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_compact_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_compact_preencoded_batch_groups(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_preencoded_batch_groups(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_dwt53(&mut self, j2k_transcode::accelerator::DctGridToDwt53Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_dwt97(&mut self, j2k_transcode::accelerator::DctGridToDwt97Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_dwt97_batch(&mut self, &[j2k_transcode::accelerator::DctGridToDwt97Job<'_>]) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_htj2k97_codeblock_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_htj2k97_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_reversible_dwt53(&mut self, j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_reversible_dwt53_batch(&mut self, &[j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>]) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::last_dwt97_batch_stage_timings(&self) -> core::option::Option +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::supports_dwt97_batch(&self) -> bool +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::supports_htj2k97_codeblock_batch(&self) -> bool +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::supports_htj2k97_compact_preencoded_batch(&self) -> bool +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::supports_htj2k97_i16_preencoded_batch(&self) -> bool +pub struct j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'a, 'j> +pub j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch::jobs: &'j [j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'a>] +pub struct j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'a> +pub j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob::block_cols: usize +pub j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob::block_rows: usize +pub j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob::dequantized_blocks: &'a [[i16; 64]] +pub j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob::height: usize +pub j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob::width: usize +pub j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob::x_rsiz: u8 +pub j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob::y_rsiz: u8 +pub struct j2k_transcode::accelerator::DctGridToDwt53Job<'a> +pub j2k_transcode::accelerator::DctGridToDwt53Job::block_cols: usize +pub j2k_transcode::accelerator::DctGridToDwt53Job::block_rows: usize +pub j2k_transcode::accelerator::DctGridToDwt53Job::blocks: &'a [[[f64; 8]; 8]] +pub j2k_transcode::accelerator::DctGridToDwt53Job::height: usize +pub j2k_transcode::accelerator::DctGridToDwt53Job::width: usize +pub struct j2k_transcode::accelerator::DctGridToDwt97Job<'a> +pub j2k_transcode::accelerator::DctGridToDwt97Job::block_cols: usize +pub j2k_transcode::accelerator::DctGridToDwt97Job::block_rows: usize +pub j2k_transcode::accelerator::DctGridToDwt97Job::blocks: &'a [[[f64; 8]; 8]] +pub j2k_transcode::accelerator::DctGridToDwt97Job::height: usize +pub j2k_transcode::accelerator::DctGridToDwt97Job::width: usize +pub struct j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'a> +pub j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob::block_cols: usize +pub j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob::block_rows: usize +pub j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob::blocks: &'a [[[f64; 8]; 8]] +pub j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob::height: usize +pub j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob::width: usize +pub j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob::x_rsiz: u8 +pub j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob::y_rsiz: u8 +pub struct j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'a> +pub j2k_transcode::accelerator::DctGridToReversibleDwt53Job::block_cols: usize +pub j2k_transcode::accelerator::DctGridToReversibleDwt53Job::block_rows: usize +pub j2k_transcode::accelerator::DctGridToReversibleDwt53Job::dequantized_blocks: &'a [[i16; 64]] +pub j2k_transcode::accelerator::DctGridToReversibleDwt53Job::height: usize +pub j2k_transcode::accelerator::DctGridToReversibleDwt53Job::width: usize +pub struct j2k_transcode::accelerator::Dwt97BatchStageTimings +pub j2k_transcode::accelerator::Dwt97BatchStageTimings::column_lift_us: u128 +pub j2k_transcode::accelerator::Dwt97BatchStageTimings::ht_codeblock_dispatches: usize +pub j2k_transcode::accelerator::Dwt97BatchStageTimings::ht_compact_us: u128 +pub j2k_transcode::accelerator::Dwt97BatchStageTimings::ht_encode_us: u128 +pub j2k_transcode::accelerator::Dwt97BatchStageTimings::ht_kernel_us: u128 +pub j2k_transcode::accelerator::Dwt97BatchStageTimings::ht_output_readback_us: u128 +pub j2k_transcode::accelerator::Dwt97BatchStageTimings::ht_status_readback_us: u128 +pub j2k_transcode::accelerator::Dwt97BatchStageTimings::idct_row_lift_us: u128 +pub j2k_transcode::accelerator::Dwt97BatchStageTimings::pack_upload_us: u128 +pub j2k_transcode::accelerator::Dwt97BatchStageTimings::quantize_codeblock_us: u128 +pub j2k_transcode::accelerator::Dwt97BatchStageTimings::readback_us: u128 +pub struct j2k_transcode::accelerator::Htj2k97CodeBlockOptions +pub j2k_transcode::accelerator::Htj2k97CodeBlockOptions::bit_depth: u8 +pub j2k_transcode::accelerator::Htj2k97CodeBlockOptions::code_block_height_exp: u8 +pub j2k_transcode::accelerator::Htj2k97CodeBlockOptions::code_block_width_exp: u8 +pub j2k_transcode::accelerator::Htj2k97CodeBlockOptions::guard_bits: u8 +pub j2k_transcode::accelerator::Htj2k97CodeBlockOptions::irreversible_quantization_scale: f32 +pub j2k_transcode::accelerator::Htj2k97CodeBlockOptions::irreversible_quantization_subband_scales: j2k_types::IrreversibleQuantizationSubbandScales +pub struct j2k_transcode::accelerator::PreencodedHtj2k97CompactBatch +pub j2k_transcode::accelerator::PreencodedHtj2k97CompactBatch::components: alloc::vec::Vec +pub j2k_transcode::accelerator::PreencodedHtj2k97CompactBatch::payload: alloc::vec::Vec +pub struct j2k_transcode::accelerator::PreencodedHtj2k97CompactBatchGroups +pub j2k_transcode::accelerator::PreencodedHtj2k97CompactBatchGroups::groups: alloc::vec::Vec> +pub j2k_transcode::accelerator::PreencodedHtj2k97CompactBatchGroups::payload: alloc::vec::Vec +pub struct j2k_transcode::accelerator::RayonReversibleDwt53Accelerator +impl j2k_transcode::accelerator::RayonReversibleDwt53Accelerator +pub const fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::reversible_dwt53_attempts(&self) -> usize +pub const fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::reversible_dwt53_batch_attempts(&self) -> usize +pub const fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::reversible_dwt53_batch_dispatches(&self) -> usize +pub const fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::reversible_dwt53_dispatches(&self) -> usize +impl j2k_transcode::accelerator::DctToWaveletStageAccelerator for j2k_transcode::accelerator::RayonReversibleDwt53Accelerator +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_i16_to_htj2k97_compact_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_i16_to_htj2k97_compact_preencoded_batch_groups(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_i16_to_htj2k97_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_i16_to_htj2k97_preencoded_batch_groups(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_dwt53(&mut self, j2k_transcode::accelerator::DctGridToDwt53Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_dwt97(&mut self, j2k_transcode::accelerator::DctGridToDwt97Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_dwt97_batch(&mut self, &[j2k_transcode::accelerator::DctGridToDwt97Job<'_>]) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_htj2k97_codeblock_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_htj2k97_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_reversible_dwt53(&mut self, j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_reversible_dwt53_batch(&mut self, &[j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>]) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::last_dwt97_batch_stage_timings(&self) -> core::option::Option +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::supports_dwt97_batch(&self) -> bool +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::supports_htj2k97_codeblock_batch(&self) -> bool +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::supports_htj2k97_compact_preencoded_batch(&self) -> bool +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::supports_htj2k97_i16_preencoded_batch(&self) -> bool +pub struct j2k_transcode::accelerator::ReversibleDwt53FirstLevel +pub j2k_transcode::accelerator::ReversibleDwt53FirstLevel::hh: alloc::vec::Vec +pub j2k_transcode::accelerator::ReversibleDwt53FirstLevel::high_height: usize +pub j2k_transcode::accelerator::ReversibleDwt53FirstLevel::high_width: usize +pub j2k_transcode::accelerator::ReversibleDwt53FirstLevel::hl: alloc::vec::Vec +pub j2k_transcode::accelerator::ReversibleDwt53FirstLevel::lh: alloc::vec::Vec +pub j2k_transcode::accelerator::ReversibleDwt53FirstLevel::ll: alloc::vec::Vec +pub j2k_transcode::accelerator::ReversibleDwt53FirstLevel::low_height: usize +pub j2k_transcode::accelerator::ReversibleDwt53FirstLevel::low_width: usize +pub trait j2k_transcode::accelerator::DctToWaveletStageAccelerator +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_compact_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_compact_preencoded_batch_groups(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_preencoded_batch_groups(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::dct_grid_to_dwt53(&mut self, j2k_transcode::accelerator::DctGridToDwt53Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::dct_grid_to_dwt97(&mut self, j2k_transcode::accelerator::DctGridToDwt97Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::dct_grid_to_dwt97_batch(&mut self, &[j2k_transcode::accelerator::DctGridToDwt97Job<'_>]) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::dct_grid_to_htj2k97_codeblock_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::dct_grid_to_htj2k97_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::dct_grid_to_reversible_dwt53(&mut self, j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::dct_grid_to_reversible_dwt53_batch(&mut self, &[j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>]) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::last_dwt97_batch_stage_timings(&self) -> core::option::Option +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::supports_dwt97_batch(&self) -> bool +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::supports_htj2k97_codeblock_batch(&self) -> bool +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::supports_htj2k97_compact_preencoded_batch(&self) -> bool +pub fn j2k_transcode::accelerator::DctToWaveletStageAccelerator::supports_htj2k97_i16_preencoded_batch(&self) -> bool +impl j2k_transcode::accelerator::DctToWaveletStageAccelerator for j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_compact_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_compact_preencoded_batch_groups(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_preencoded_batch_groups(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_dwt53(&mut self, j2k_transcode::accelerator::DctGridToDwt53Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_dwt97(&mut self, j2k_transcode::accelerator::DctGridToDwt97Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_dwt97_batch(&mut self, &[j2k_transcode::accelerator::DctGridToDwt97Job<'_>]) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_htj2k97_codeblock_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_htj2k97_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_reversible_dwt53(&mut self, j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::dct_grid_to_reversible_dwt53_batch(&mut self, &[j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>]) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::last_dwt97_batch_stage_timings(&self) -> core::option::Option +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::supports_dwt97_batch(&self) -> bool +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::supports_htj2k97_codeblock_batch(&self) -> bool +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::supports_htj2k97_compact_preencoded_batch(&self) -> bool +pub fn j2k_transcode::accelerator::CpuOnlyDctToWaveletStageAccelerator::supports_htj2k97_i16_preencoded_batch(&self) -> bool +impl j2k_transcode::accelerator::DctToWaveletStageAccelerator for j2k_transcode::accelerator::RayonReversibleDwt53Accelerator +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_i16_to_htj2k97_compact_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_i16_to_htj2k97_compact_preencoded_batch_groups(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_i16_to_htj2k97_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_i16_to_htj2k97_preencoded_batch_groups(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_dwt53(&mut self, j2k_transcode::accelerator::DctGridToDwt53Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_dwt97(&mut self, j2k_transcode::accelerator::DctGridToDwt97Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_dwt97_batch(&mut self, &[j2k_transcode::accelerator::DctGridToDwt97Job<'_>]) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_htj2k97_codeblock_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_htj2k97_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_reversible_dwt53(&mut self, j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::dct_grid_to_reversible_dwt53_batch(&mut self, &[j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>]) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::last_dwt97_batch_stage_timings(&self) -> core::option::Option +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::supports_dwt97_batch(&self) -> bool +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::supports_htj2k97_codeblock_batch(&self) -> bool +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::supports_htj2k97_compact_preencoded_batch(&self) -> bool +pub fn j2k_transcode::accelerator::RayonReversibleDwt53Accelerator::supports_htj2k97_i16_preencoded_batch(&self) -> bool +pub fn j2k_transcode::accelerator::idct_blocks_to_signed_samples_rayon(&[[i16; 64]]) -> alloc::vec::Vec<[i32; 64]> +pub fn j2k_transcode::accelerator::reversible_dwt53_first_level_from_block_samples(&[[i32; 64]], usize, usize, usize, usize) -> core::result::Result +pub mod j2k_transcode::dct53_2d +pub struct j2k_transcode::dct53_2d::Dct53GridError +impl j2k_transcode::DctGridError +pub const fn j2k_transcode::DctGridError::block_cols(self) -> usize +pub const fn j2k_transcode::DctGridError::block_count(self) -> usize +pub const fn j2k_transcode::DctGridError::block_rows(self) -> usize +pub const fn j2k_transcode::DctGridError::height(self) -> usize +pub const fn j2k_transcode::DctGridError::width(self) -> usize +impl core::error::Error for j2k_transcode::DctGridError +impl core::fmt::Display for j2k_transcode::DctGridError +pub fn j2k_transcode::DctGridError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_transcode::dct53_2d::Dct53GridScratch +impl j2k_transcode::dct53_2d::Dct53GridScratch +pub fn j2k_transcode::dct53_2d::Dct53GridScratch::weight_row_capacity(&self) -> usize +pub struct j2k_transcode::dct53_2d::Dwt53TwoDimensional +pub j2k_transcode::dct53_2d::Dwt53TwoDimensional::hh: alloc::vec::Vec +pub j2k_transcode::dct53_2d::Dwt53TwoDimensional::high_height: usize +pub j2k_transcode::dct53_2d::Dwt53TwoDimensional::high_width: usize +pub j2k_transcode::dct53_2d::Dwt53TwoDimensional::hl: alloc::vec::Vec +pub j2k_transcode::dct53_2d::Dwt53TwoDimensional::lh: alloc::vec::Vec +pub j2k_transcode::dct53_2d::Dwt53TwoDimensional::ll: alloc::vec::Vec +pub j2k_transcode::dct53_2d::Dwt53TwoDimensional::low_height: usize +pub j2k_transcode::dct53_2d::Dwt53TwoDimensional::low_width: usize +impl j2k_transcode::dct53_2d::Dwt53TwoDimensional +pub fn j2k_transcode::dct53_2d::Dwt53TwoDimensional::max_abs_diff(&self, &Self) -> f64 +pub fn j2k_transcode::dct53_2d::dct8x8_blocks_then_dwt53_float(&[[[f64; 8]; 8]], usize, usize, usize, usize) -> core::result::Result, j2k_transcode::DctGridError> +pub fn j2k_transcode::dct53_2d::dct8x8_blocks_to_dwt53_float_linear(&[[[f64; 8]; 8]], usize, usize, usize, usize) -> core::result::Result, j2k_transcode::DctGridError> +pub fn j2k_transcode::dct53_2d::dct8x8_blocks_to_dwt53_float_linear_with_scratch(&[[[f64; 8]; 8]], usize, usize, usize, usize, &mut j2k_transcode::dct53_2d::Dct53GridScratch) -> core::result::Result, j2k_transcode::DctGridError> +pub fn j2k_transcode::dct53_2d::dct8x8_to_dwt53_float_linear([[f64; 8]; 8]) -> j2k_transcode::dct53_2d::Dwt53TwoDimensional +pub fn j2k_transcode::dct53_2d::idct8x8_then_dwt53_float([[f64; 8]; 8]) -> j2k_transcode::dct53_2d::Dwt53TwoDimensional +pub mod j2k_transcode::dct97_2d +pub struct j2k_transcode::dct97_2d::Dct97GridError +impl j2k_transcode::DctGridError +pub const fn j2k_transcode::DctGridError::block_cols(self) -> usize +pub const fn j2k_transcode::DctGridError::block_count(self) -> usize +pub const fn j2k_transcode::DctGridError::block_rows(self) -> usize +pub const fn j2k_transcode::DctGridError::height(self) -> usize +pub const fn j2k_transcode::DctGridError::width(self) -> usize +impl core::error::Error for j2k_transcode::DctGridError +impl core::fmt::Display for j2k_transcode::DctGridError +pub fn j2k_transcode::DctGridError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_transcode::dct97_2d::Dct97GridScratch +impl j2k_transcode::dct97_2d::Dct97GridScratch +pub fn j2k_transcode::dct97_2d::Dct97GridScratch::spatial_sample_capacity(&self) -> usize +pub struct j2k_transcode::dct97_2d::Dwt97TwoDimensional +pub j2k_transcode::dct97_2d::Dwt97TwoDimensional::hh: alloc::vec::Vec +pub j2k_transcode::dct97_2d::Dwt97TwoDimensional::high_height: usize +pub j2k_transcode::dct97_2d::Dwt97TwoDimensional::high_width: usize +pub j2k_transcode::dct97_2d::Dwt97TwoDimensional::hl: alloc::vec::Vec +pub j2k_transcode::dct97_2d::Dwt97TwoDimensional::lh: alloc::vec::Vec +pub j2k_transcode::dct97_2d::Dwt97TwoDimensional::ll: alloc::vec::Vec +pub j2k_transcode::dct97_2d::Dwt97TwoDimensional::low_height: usize +pub j2k_transcode::dct97_2d::Dwt97TwoDimensional::low_width: usize +pub fn j2k_transcode::dct97_2d::dct8x8_blocks_then_dwt97_float(&[[[f64; 8]; 8]], usize, usize, usize, usize) -> core::result::Result, j2k_transcode::DctGridError> +pub fn j2k_transcode::dct97_2d::dct8x8_blocks_then_dwt97_float_with_scratch(&[[[f64; 8]; 8]], usize, usize, usize, usize, &mut j2k_transcode::dct97_2d::Dct97GridScratch) -> core::result::Result, j2k_transcode::DctGridError> +pub mod j2k_transcode::htj2k97_codeblock_oracle +pub fn j2k_transcode::htj2k97_codeblock_oracle::htj2k97_subband_delta(j2k_transcode::accelerator::Htj2k97CodeBlockOptions, j2k_types::J2kSubBandType) -> f64 +pub fn j2k_transcode::htj2k97_codeblock_oracle::htj2k97_subband_total_bitplanes(j2k_transcode::accelerator::Htj2k97CodeBlockOptions, j2k_types::J2kSubBandType) -> u8 +pub fn j2k_transcode::htj2k97_codeblock_oracle::prequantized_component_from_dwt97(&j2k_transcode::dct97_2d::Dwt97TwoDimensional, j2k_transcode::accelerator::Htj2k97CodeBlockOptions, u8, u8) -> j2k_types::PrequantizedHtj2k97Component +pub fn j2k_transcode::htj2k97_codeblock_oracle::quantize_codeblock_subband(&[f64], usize, usize, j2k_types::J2kSubBandType, j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> j2k_types::PrequantizedHtj2k97Subband +pub fn j2k_transcode::htj2k97_codeblock_oracle::validate_htj2k97_codeblock_options(j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result<(usize, usize), &'static str> +pub enum j2k_transcode::TranscodePipelineStageKind +pub j2k_transcode::TranscodePipelineStageKind::CodestreamAssembly +pub j2k_transcode::TranscodePipelineStageKind::CoefficientPrep +pub j2k_transcode::TranscodePipelineStageKind::EntropyDecode +pub j2k_transcode::TranscodePipelineStageKind::Packetization +pub j2k_transcode::TranscodePipelineStageKind::QuantizationCodeBlockPrep +pub j2k_transcode::TranscodePipelineStageKind::Transform +impl j2k_transcode::TranscodePipelineStageKind +pub const fn j2k_transcode::TranscodePipelineStageKind::as_str(self) -> &'static str +impl core::fmt::Display for j2k_transcode::TranscodePipelineStageKind +pub fn j2k_transcode::TranscodePipelineStageKind::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub enum j2k_transcode::TranscodeStageProcessor +pub j2k_transcode::TranscodeStageProcessor::Cpu +pub j2k_transcode::TranscodeStageProcessor::Hybrid +pub j2k_transcode::TranscodeStageProcessor::Metal +impl j2k_transcode::TranscodeStageProcessor +pub const fn j2k_transcode::TranscodeStageProcessor::as_str(self) -> &'static str +impl core::fmt::Display for j2k_transcode::TranscodeStageProcessor +pub fn j2k_transcode::TranscodeStageProcessor::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_transcode::DctGridError +impl j2k_transcode::DctGridError +pub const fn j2k_transcode::DctGridError::block_cols(self) -> usize +pub const fn j2k_transcode::DctGridError::block_count(self) -> usize +pub const fn j2k_transcode::DctGridError::block_rows(self) -> usize +pub const fn j2k_transcode::DctGridError::height(self) -> usize +pub const fn j2k_transcode::DctGridError::width(self) -> usize +impl core::error::Error for j2k_transcode::DctGridError +impl core::fmt::Display for j2k_transcode::DctGridError +pub fn j2k_transcode::DctGridError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_transcode::TranscodePipelineMap +pub j2k_transcode::TranscodePipelineMap::recommendation: j2k_transcode::TranscodeResidentStageRecommendation +pub j2k_transcode::TranscodePipelineMap::stages: alloc::vec::Vec +impl j2k_transcode::TranscodePipelineMap +pub fn j2k_transcode::TranscodePipelineMap::debug_report(&self) -> alloc::string::String +pub fn j2k_transcode::TranscodePipelineMap::from_timings(&TranscodeTimingReport) -> Self +pub struct j2k_transcode::TranscodePipelineStageReport +pub j2k_transcode::TranscodePipelineStageReport::cpu_us: u128 +pub j2k_transcode::TranscodePipelineStageReport::dispatches: usize +pub j2k_transcode::TranscodePipelineStageReport::fallback_jobs: usize +pub j2k_transcode::TranscodePipelineStageReport::metal_us: u128 +pub j2k_transcode::TranscodePipelineStageReport::note: &'static str +pub j2k_transcode::TranscodePipelineStageReport::processor: j2k_transcode::TranscodeStageProcessor +pub j2k_transcode::TranscodePipelineStageReport::stage: j2k_transcode::TranscodePipelineStageKind +pub j2k_transcode::TranscodePipelineStageReport::transfer_us: u128 +pub struct j2k_transcode::TranscodeResidentStageRecommendation +pub j2k_transcode::TranscodeResidentStageRecommendation::evidence_dispatches: usize +pub j2k_transcode::TranscodeResidentStageRecommendation::evidence_us: u128 +pub j2k_transcode::TranscodeResidentStageRecommendation::reason: &'static str +pub j2k_transcode::TranscodeResidentStageRecommendation::stage: j2k_transcode::TranscodePipelineStageKind +``` + +## `j2k-transcode-cuda` + +```text +pub mod j2k_transcode_cuda +pub enum j2k_transcode_cuda::CudaTranscodeError +pub j2k_transcode_cuda::CudaTranscodeError::CudaUnavailable +pub j2k_transcode_cuda::CudaTranscodeError::Kernel(&'static str) +pub j2k_transcode_cuda::CudaTranscodeError::UnsupportedJob(&'static str) +impl core::convert::From for j2k_transcode::accelerator::TranscodeStageError +pub fn j2k_transcode::accelerator::TranscodeStageError::from(j2k_transcode_cuda::CudaTranscodeError) -> Self +impl core::error::Error for j2k_transcode_cuda::CudaTranscodeError +impl core::fmt::Display for j2k_transcode_cuda::CudaTranscodeError +pub fn j2k_transcode_cuda::CudaTranscodeError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_transcode_cuda::CudaDctToWaveletStageAccelerator +impl j2k_transcode_cuda::CudaDctToWaveletStageAccelerator +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dwt53_attempts(&self) -> usize +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dwt53_dispatches(&self) -> usize +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dwt97_attempts(&self) -> usize +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dwt97_batch_attempts(&self) -> usize +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dwt97_batch_dispatches(&self) -> usize +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dwt97_dispatches(&self) -> usize +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::for_auto() -> Self +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::htj2k97_codeblock_batch_attempts(&self) -> usize +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::htj2k97_codeblock_batch_dispatches(&self) -> usize +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::new_explicit() -> Self +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::new_explicit_resident_ht_encode() -> Self +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::reversible_dwt53_attempts(&self) -> usize +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::reversible_dwt53_batch_attempts(&self) -> usize +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::reversible_dwt53_batch_dispatches(&self) -> usize +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::reversible_dwt53_dispatches(&self) -> usize +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::with_auto_dwt97_batch_thresholds(self, usize, usize) -> Self +pub const fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::with_auto_reversible_batch_thresholds(self, usize, usize) -> Self +impl core::default::Default for j2k_transcode_cuda::CudaDctToWaveletStageAccelerator +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::default() -> Self +impl j2k_transcode::accelerator::DctToWaveletStageAccelerator for j2k_transcode_cuda::CudaDctToWaveletStageAccelerator +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_compact_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_compact_preencoded_batch_groups(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dct_grid_i16_to_htj2k97_preencoded_batch_groups(&mut self, &[j2k_transcode::accelerator::DctGridI16ToHtj2k97CodeBlockBatch<'_, '_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dct_grid_to_dwt53(&mut self, j2k_transcode::accelerator::DctGridToDwt53Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dct_grid_to_dwt97(&mut self, j2k_transcode::accelerator::DctGridToDwt97Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dct_grid_to_dwt97_batch(&mut self, &[j2k_transcode::accelerator::DctGridToDwt97Job<'_>]) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dct_grid_to_htj2k97_codeblock_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dct_grid_to_htj2k97_preencoded_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dct_grid_to_reversible_dwt53(&mut self, j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::dct_grid_to_reversible_dwt53_batch(&mut self, &[j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>]) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::last_dwt97_batch_stage_timings(&self) -> core::option::Option +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::supports_dwt97_batch(&self) -> bool +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::supports_htj2k97_codeblock_batch(&self) -> bool +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::supports_htj2k97_compact_preencoded_batch(&self) -> bool +pub fn j2k_transcode_cuda::CudaDctToWaveletStageAccelerator::supports_htj2k97_i16_preencoded_batch(&self) -> bool +pub const j2k_transcode_cuda::CUDA_UNAVAILABLE: &str +``` + +## `j2k-metal-support` + +```text +pub mod j2k_metal_support +pub enum j2k_metal_support::MetalSupportError +pub j2k_metal_support::MetalSupportError::BufferAlignment +pub j2k_metal_support::MetalSupportError::BufferAlignment::align: usize +pub j2k_metal_support::MetalSupportError::BufferAlignment::offset_bytes: usize +pub j2k_metal_support::MetalSupportError::BufferBounds +pub j2k_metal_support::MetalSupportError::BufferBounds::buffer_len: usize +pub j2k_metal_support::MetalSupportError::BufferBounds::byte_len: usize +pub j2k_metal_support::MetalSupportError::BufferBounds::offset_bytes: usize +pub j2k_metal_support::MetalSupportError::BufferContentsUnavailable +pub j2k_metal_support::MetalSupportError::CommandQueue +pub j2k_metal_support::MetalSupportError::CommandQueue::message: alloc::string::String +pub j2k_metal_support::MetalSupportError::CommandQueueUnavailable +pub j2k_metal_support::MetalSupportError::MetalUnavailable +pub j2k_metal_support::MetalSupportError::PipelineFunction +pub j2k_metal_support::MetalSupportError::PipelineFunction::function_name: alloc::string::String +pub j2k_metal_support::MetalSupportError::PipelineFunction::message: alloc::string::String +pub j2k_metal_support::MetalSupportError::PipelineState +pub j2k_metal_support::MetalSupportError::PipelineState::function_name: alloc::string::String +pub j2k_metal_support::MetalSupportError::PipelineState::message: alloc::string::String +pub j2k_metal_support::MetalSupportError::ShaderLibrary +pub j2k_metal_support::MetalSupportError::ShaderLibrary::message: alloc::string::String +impl j2k_metal_support::MetalSupportError +pub const fn j2k_metal_support::MetalSupportError::is_unavailable(&self) -> bool +impl core::error::Error for j2k_metal_support::MetalSupportError +impl core::fmt::Display for j2k_metal_support::MetalSupportError +pub fn j2k_metal_support::MetalSupportError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_metal_support::MetalDeviceSession +impl j2k_metal_support::MetalDeviceSession +pub fn j2k_metal_support::MetalDeviceSession::device(&self) -> &metal::device::DeviceRef +pub fn j2k_metal_support::MetalDeviceSession::new(metal::device::Device) -> Self +pub fn j2k_metal_support::MetalDeviceSession::system_default() -> core::result::Result +impl core::fmt::Debug for j2k_metal_support::MetalDeviceSession +pub fn j2k_metal_support::MetalDeviceSession::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_metal_support::MetalPipelineLoader +impl j2k_metal_support::MetalPipelineLoader +pub fn j2k_metal_support::MetalPipelineLoader::library(&self) -> &metal::library::Library +pub fn j2k_metal_support::MetalPipelineLoader::new(&metal::device::Device, &str) -> core::result::Result +pub fn j2k_metal_support::MetalPipelineLoader::pipeline(&self, &str) -> core::result::Result +pub struct j2k_metal_support::MetalRouteProfileLabels +pub j2k_metal_support::MetalRouteProfileLabels::decision: &'static str +pub j2k_metal_support::MetalRouteProfileLabels::reason: &'static str +impl j2k_metal_support::MetalRouteProfileLabels +pub const fn j2k_metal_support::MetalRouteProfileLabels::new(&'static str, &'static str) -> Self +pub unsafe fn j2k_metal_support::buffer_contents_slice(&metal::buffer::Buffer, usize, usize) -> &[T] +pub unsafe fn j2k_metal_support::buffer_contents_slice_mut(&mut metal::buffer::Buffer, usize, usize) -> &mut [T] +pub fn j2k_metal_support::checked_buffer_contents_slice(&metal::buffer::Buffer, usize, usize) -> core::result::Result<&[T], j2k_metal_support::MetalSupportError> +pub fn j2k_metal_support::checked_buffer_contents_slice_mut(&mut metal::buffer::Buffer, usize, usize) -> core::result::Result<&mut [T], j2k_metal_support::MetalSupportError> +pub fn j2k_metal_support::checked_command_queue(&metal::device::Device) -> core::result::Result +pub const fn j2k_metal_support::cpu_host_route() -> j2k_metal_support::MetalRouteProfileLabels +pub fn j2k_metal_support::dispatch_1d_pipeline(&metal::encoder::ComputeCommandEncoderRef, &metal::pipeline::compute::ComputePipelineState, u64) +pub fn j2k_metal_support::dispatch_2d_pipeline(&metal::encoder::ComputeCommandEncoderRef, &metal::pipeline::compute::ComputePipelineState, (u32, u32)) +pub fn j2k_metal_support::dispatch_3d_pipeline(&metal::encoder::ComputeCommandEncoderRef, &metal::pipeline::compute::ComputePipelineState, (u32, u32, u32)) +pub fn j2k_metal_support::dispatch_single_thread(&metal::encoder::ComputeCommandEncoderRef) +pub const fn j2k_metal_support::metal_kernel_route() -> j2k_metal_support::MetalRouteProfileLabels +pub const fn j2k_metal_support::metal_unavailable_route() -> j2k_metal_support::MetalRouteProfileLabels +pub const fn j2k_metal_support::mtl_size(u64, u64, u64) -> metal::types::MTLSize +pub fn j2k_metal_support::named_pipeline(&metal::device::Device, &metal::library::Library, &str) -> core::result::Result +pub const fn j2k_metal_support::one_d_threads_per_group(u64) -> metal::types::MTLSize +pub fn j2k_metal_support::private_buffer(&metal::device::Device, usize) -> metal::buffer::Buffer +pub const fn j2k_metal_support::reject_explicit_metal_route(&'static str) -> j2k_metal_support::MetalRouteProfileLabels +pub const fn j2k_metal_support::reject_unsupported_backend_route() -> j2k_metal_support::MetalRouteProfileLabels +pub fn j2k_metal_support::shader_library(&metal::device::Device, &str) -> core::result::Result +pub fn j2k_metal_support::shared_buffer(&metal::device::Device, usize) -> metal::buffer::Buffer +pub fn j2k_metal_support::shared_buffer_for_len(&metal::device::Device, usize) -> metal::buffer::Buffer +pub fn j2k_metal_support::shared_buffer_with_bytes(&metal::device::Device, &[u8]) -> metal::buffer::Buffer +pub fn j2k_metal_support::shared_buffer_with_slice(&metal::device::Device, &[T]) -> metal::buffer::Buffer +pub fn j2k_metal_support::system_default_device() -> core::result::Result +pub const fn j2k_metal_support::two_d_threads_per_group(u64, u64) -> metal::types::MTLSize +``` + +## `j2k-transcode-metal` + +```text +pub mod j2k_transcode_metal +pub enum j2k_transcode_metal::MetalTranscodeError +pub j2k_transcode_metal::MetalTranscodeError::Kernel(&'static str) +pub j2k_transcode_metal::MetalTranscodeError::MetalUnavailable +pub j2k_transcode_metal::MetalTranscodeError::Runtime(&'static str) +pub j2k_transcode_metal::MetalTranscodeError::UnsupportedJob(&'static str) +impl core::convert::From for j2k_transcode::accelerator::TranscodeStageError +pub fn j2k_transcode::accelerator::TranscodeStageError::from(j2k_transcode_metal::MetalTranscodeError) -> Self +impl core::error::Error for j2k_transcode_metal::MetalTranscodeError +impl core::fmt::Display for j2k_transcode_metal::MetalTranscodeError +pub fn j2k_transcode_metal::MetalTranscodeError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_transcode_metal::MetalDctToWaveletStageAccelerator +impl j2k_transcode_metal::MetalDctToWaveletStageAccelerator +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dct_grid_to_reversible_dwt53_batch(&mut self, &[j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>]) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dwt53_attempts(&self) -> usize +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dwt53_dispatches(&self) -> usize +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dwt97_attempts(&self) -> usize +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dwt97_batch_attempts(&self) -> usize +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dwt97_batch_dispatches(&self) -> usize +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dwt97_dispatches(&self) -> usize +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::for_auto() -> Self +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::for_auto_with_device(metal::device::Device) -> Self +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::for_auto_with_session(j2k_transcode_metal::MetalTranscodeSession) -> Self +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::htj2k97_codeblock_batch_attempts(&self) -> usize +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::htj2k97_codeblock_batch_dispatches(&self) -> usize +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::last_dwt97_batch_stage_timings(&self) -> core::option::Option +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::new_explicit() -> Self +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::new_explicit_with_device(metal::device::Device) -> Self +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::new_explicit_with_session(j2k_transcode_metal::MetalTranscodeSession) -> Self +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::reversible_dwt53_attempts(&self) -> usize +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::reversible_dwt53_batch_attempts(&self) -> usize +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::reversible_dwt53_batch_dispatches(&self) -> usize +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::reversible_dwt53_dispatches(&self) -> usize +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::with_auto_dwt97_batch_thresholds(self, usize, usize) -> Self +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::with_auto_dwt97_min_samples(self, usize) -> Self +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::with_auto_min_samples(self, usize) -> Self +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::with_auto_reversible_batch_thresholds(self, usize, usize) -> Self +pub const fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::with_auto_reversible_min_samples(self, usize) -> Self +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::with_device(self, metal::device::Device) -> Self +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::with_session(self, j2k_transcode_metal::MetalTranscodeSession) -> Self +impl core::default::Default for j2k_transcode_metal::MetalDctToWaveletStageAccelerator +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::default() -> Self +impl j2k_transcode::accelerator::DctToWaveletStageAccelerator for j2k_transcode_metal::MetalDctToWaveletStageAccelerator +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dct_grid_to_dwt53(&mut self, j2k_transcode::accelerator::DctGridToDwt53Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dct_grid_to_dwt97(&mut self, j2k_transcode::accelerator::DctGridToDwt97Job<'_>) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dct_grid_to_dwt97_batch(&mut self, &[j2k_transcode::accelerator::DctGridToDwt97Job<'_>]) -> core::result::Result>>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dct_grid_to_htj2k97_codeblock_batch(&mut self, &[j2k_transcode::accelerator::DctGridToHtj2k97CodeBlockJob<'_>], j2k_transcode::accelerator::Htj2k97CodeBlockOptions) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dct_grid_to_reversible_dwt53(&mut self, j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>) -> core::result::Result, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::dct_grid_to_reversible_dwt53_batch(&mut self, &[j2k_transcode::accelerator::DctGridToReversibleDwt53Job<'_>]) -> core::result::Result>, j2k_transcode::accelerator::TranscodeStageError> +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::last_dwt97_batch_stage_timings(&self) -> core::option::Option +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::supports_dwt97_batch(&self) -> bool +pub fn j2k_transcode_metal::MetalDctToWaveletStageAccelerator::supports_htj2k97_codeblock_batch(&self) -> bool +pub struct j2k_transcode_metal::MetalTranscodeSession +impl j2k_transcode_metal::MetalTranscodeSession +pub fn j2k_transcode_metal::MetalTranscodeSession::new(metal::device::Device) -> Self +pub fn j2k_transcode_metal::MetalTranscodeSession::system_default() -> core::result::Result +impl core::fmt::Debug for j2k_transcode_metal::MetalTranscodeSession +pub fn j2k_transcode_metal::MetalTranscodeSession::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub const j2k_transcode_metal::METAL_UNAVAILABLE: &str +``` + +## `j2k-native` + +```text +pub mod j2k_native +pub use j2k_native::CpuOnlyJ2kEncodeStageAccelerator +pub use j2k_native::EncodedHtJ2kCodeBlock +pub use j2k_native::EncodedJ2kCodeBlock +pub use j2k_native::IrreversibleQuantizationStep +pub use j2k_native::IrreversibleQuantizationSubbandScales +pub use j2k_native::J2kCodeBlockSegment +pub use j2k_native::J2kCodeBlockStyle +pub use j2k_native::J2kDeinterleaveToF32Job +pub use j2k_native::J2kEncodeDispatchReport +pub use j2k_native::J2kForwardDwt53Job +pub use j2k_native::J2kForwardDwt53Level +pub use j2k_native::J2kForwardDwt53Output +pub use j2k_native::J2kForwardDwt97Job +pub use j2k_native::J2kForwardDwt97Level +pub use j2k_native::J2kForwardDwt97Output +pub use j2k_native::J2kForwardIctJob +pub use j2k_native::J2kForwardRctJob +pub use j2k_native::J2kHtCodeBlockEncodeJob +pub use j2k_native::J2kHtSubbandEncodeJob +pub use j2k_native::J2kHtj2kTileEncodeJob +pub use j2k_native::J2kPacketizationBlockCodingMode +pub use j2k_native::J2kPacketizationCodeBlock +pub use j2k_native::J2kPacketizationEncodeJob +pub use j2k_native::J2kPacketizationPacketDescriptor +pub use j2k_native::J2kPacketizationProgressionOrder +pub use j2k_native::J2kPacketizationResolution +pub use j2k_native::J2kPacketizationSubband +pub use j2k_native::J2kQuantizeSubbandJob +pub use j2k_native::J2kSubBandType +pub use j2k_native::J2kTier1CodeBlockEncodeJob +pub use j2k_native::PrecomputedHtj2k53Component +pub use j2k_native::PrecomputedHtj2k53Image +pub use j2k_native::PrecomputedHtj2k97Component +pub use j2k_native::PrecomputedHtj2k97Image +pub use j2k_native::PreencodedHtj2k97CodeBlock +pub use j2k_native::PreencodedHtj2k97CompactCodeBlock +pub use j2k_native::PreencodedHtj2k97CompactComponent +pub use j2k_native::PreencodedHtj2k97CompactImage +pub use j2k_native::PreencodedHtj2k97CompactResolution +pub use j2k_native::PreencodedHtj2k97CompactSubband +pub use j2k_native::PreencodedHtj2k97Component +pub use j2k_native::PreencodedHtj2k97Image +pub use j2k_native::PreencodedHtj2k97Resolution +pub use j2k_native::PreencodedHtj2k97Subband +pub use j2k_native::PrequantizedHtj2k97CodeBlock +pub use j2k_native::PrequantizedHtj2k97Component +pub use j2k_native::PrequantizedHtj2k97Image +pub use j2k_native::PrequantizedHtj2k97Resolution +pub use j2k_native::PrequantizedHtj2k97Subband +pub mod j2k_native::error +#[non_exhaustive] pub enum j2k_native::error::ColorError +pub j2k_native::error::ColorError::LabConversionFailed +pub j2k_native::error::ColorError::Mct +pub j2k_native::error::ColorError::PaletteResolutionFailed +pub j2k_native::error::ColorError::SyccConversionFailed +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::ColorError) -> Self +impl core::error::Error for j2k_native::error::ColorError +impl core::fmt::Display for j2k_native::error::ColorError +pub fn j2k_native::error::ColorError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +#[non_exhaustive] pub enum j2k_native::error::DecodeError +pub j2k_native::error::DecodeError::Color(j2k_native::error::ColorError) +pub j2k_native::error::DecodeError::Decoding(j2k_native::error::DecodingError) +pub j2k_native::error::DecodeError::Format(j2k_native::error::FormatError) +pub j2k_native::error::DecodeError::Marker(j2k_native::error::MarkerError) +pub j2k_native::error::DecodeError::Tile(j2k_native::error::TileError) +pub j2k_native::error::DecodeError::Validation(j2k_native::error::ValidationError) +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::ColorError) -> Self +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::DecodingError) -> Self +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::FormatError) -> Self +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::MarkerError) -> Self +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::TileError) -> Self +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::ValidationError) -> Self +impl core::error::Error for j2k_native::error::DecodeError +impl core::fmt::Display for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +#[non_exhaustive] pub enum j2k_native::error::DecodingError +pub j2k_native::error::DecodingError::CodeBlockDecodeFailure +pub j2k_native::error::DecodingError::InvalidBitplaneCount +pub j2k_native::error::DecodingError::InvalidPrecinct +pub j2k_native::error::DecodingError::InvalidProgressionIterator +pub j2k_native::error::DecodingError::OutputBufferTooSmall +pub j2k_native::error::DecodingError::TooManyBitplanes +pub j2k_native::error::DecodingError::TooManyCodingPasses +pub j2k_native::error::DecodingError::UnexpectedEof +pub j2k_native::error::DecodingError::UnsupportedFeature(&'static str) +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::DecodingError) -> Self +impl core::error::Error for j2k_native::error::DecodingError +impl core::fmt::Display for j2k_native::error::DecodingError +pub fn j2k_native::error::DecodingError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +#[non_exhaustive] pub enum j2k_native::error::FormatError +pub j2k_native::error::FormatError::InvalidBox +pub j2k_native::error::FormatError::InvalidFileType +pub j2k_native::error::FormatError::InvalidSignature +pub j2k_native::error::FormatError::MissingCodestream +pub j2k_native::error::FormatError::Unsupported +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::FormatError) -> Self +impl core::error::Error for j2k_native::error::FormatError +impl core::fmt::Display for j2k_native::error::FormatError +pub fn j2k_native::error::FormatError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +#[non_exhaustive] pub enum j2k_native::error::MarkerError +pub j2k_native::error::MarkerError::Expected(&'static str) +pub j2k_native::error::MarkerError::Invalid +pub j2k_native::error::MarkerError::Missing(&'static str) +pub j2k_native::error::MarkerError::ParseFailure(&'static str) +pub j2k_native::error::MarkerError::Unsupported +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::MarkerError) -> Self +impl core::error::Error for j2k_native::error::MarkerError +impl core::fmt::Display for j2k_native::error::MarkerError +pub fn j2k_native::error::MarkerError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +#[non_exhaustive] pub enum j2k_native::error::TileError +pub j2k_native::error::TileError::Invalid +pub j2k_native::error::TileError::InvalidIndex +pub j2k_native::error::TileError::InvalidOffsets +pub j2k_native::error::TileError::PpmPptConflict +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::TileError) -> Self +impl core::error::Error for j2k_native::error::TileError +impl core::fmt::Display for j2k_native::error::TileError +pub fn j2k_native::error::TileError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +#[non_exhaustive] pub enum j2k_native::error::ValidationError +pub j2k_native::error::ValidationError::ImageTooLarge +pub j2k_native::error::ValidationError::InsufficientExponents +pub j2k_native::error::ValidationError::InvalidChannelDefinition +pub j2k_native::error::ValidationError::InvalidComponentMetadata +pub j2k_native::error::ValidationError::InvalidDimensions +pub j2k_native::error::ValidationError::InvalidExponents +pub j2k_native::error::ValidationError::InvalidProgressionOrder +pub j2k_native::error::ValidationError::InvalidQuantizationStyle +pub j2k_native::error::ValidationError::InvalidTransformation +pub j2k_native::error::ValidationError::MissingPrecinctExponents +pub j2k_native::error::ValidationError::MissingStepSize +pub j2k_native::error::ValidationError::TooManyChannels +pub j2k_native::error::ValidationError::TooManyTiles +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::ValidationError) -> Self +impl core::error::Error for j2k_native::error::ValidationError +impl core::fmt::Display for j2k_native::error::ValidationError +pub fn j2k_native::error::ValidationError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub type j2k_native::error::Result = core::result::Result +#[non_exhaustive] pub enum j2k_native::ColorError +pub j2k_native::ColorError::LabConversionFailed +pub j2k_native::ColorError::Mct +pub j2k_native::ColorError::PaletteResolutionFailed +pub j2k_native::ColorError::SyccConversionFailed +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::ColorError) -> Self +impl core::error::Error for j2k_native::error::ColorError +impl core::fmt::Display for j2k_native::error::ColorError +pub fn j2k_native::error::ColorError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub enum j2k_native::ColorSpace +pub j2k_native::ColorSpace::CMYK +pub j2k_native::ColorSpace::Gray +pub j2k_native::ColorSpace::Icc +pub j2k_native::ColorSpace::Icc::num_channels: u8 +pub j2k_native::ColorSpace::Icc::profile: alloc::vec::Vec +pub j2k_native::ColorSpace::RGB +pub j2k_native::ColorSpace::Unknown +pub j2k_native::ColorSpace::Unknown::num_channels: u8 +impl j2k_native::ColorSpace +pub fn j2k_native::ColorSpace::num_channels(&self) -> u8 +pub enum j2k_native::CpuDecodeParallelism +pub j2k_native::CpuDecodeParallelism::Auto +pub j2k_native::CpuDecodeParallelism::Serial +#[non_exhaustive] pub enum j2k_native::DecodeError +pub j2k_native::DecodeError::Color(j2k_native::error::ColorError) +pub j2k_native::DecodeError::Decoding(j2k_native::error::DecodingError) +pub j2k_native::DecodeError::Format(j2k_native::error::FormatError) +pub j2k_native::DecodeError::Marker(j2k_native::error::MarkerError) +pub j2k_native::DecodeError::Tile(j2k_native::error::TileError) +pub j2k_native::DecodeError::Validation(j2k_native::error::ValidationError) +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::ColorError) -> Self +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::DecodingError) -> Self +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::FormatError) -> Self +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::MarkerError) -> Self +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::TileError) -> Self +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::ValidationError) -> Self +impl core::error::Error for j2k_native::error::DecodeError +impl core::fmt::Display for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +#[non_exhaustive] pub enum j2k_native::DecodingError +pub j2k_native::DecodingError::CodeBlockDecodeFailure +pub j2k_native::DecodingError::InvalidBitplaneCount +pub j2k_native::DecodingError::InvalidPrecinct +pub j2k_native::DecodingError::InvalidProgressionIterator +pub j2k_native::DecodingError::OutputBufferTooSmall +pub j2k_native::DecodingError::TooManyBitplanes +pub j2k_native::DecodingError::TooManyCodingPasses +pub j2k_native::DecodingError::UnexpectedEof +pub j2k_native::DecodingError::UnsupportedFeature(&'static str) +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::DecodingError) -> Self +impl core::error::Error for j2k_native::error::DecodingError +impl core::fmt::Display for j2k_native::error::DecodingError +pub fn j2k_native::error::DecodingError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub enum j2k_native::EncodeProgressionOrder +pub j2k_native::EncodeProgressionOrder::Cprl +pub j2k_native::EncodeProgressionOrder::Lrcp +pub j2k_native::EncodeProgressionOrder::Pcrl +pub j2k_native::EncodeProgressionOrder::Rlcp +pub j2k_native::EncodeProgressionOrder::Rpcl +#[non_exhaustive] pub enum j2k_native::FormatError +pub j2k_native::FormatError::InvalidBox +pub j2k_native::FormatError::InvalidFileType +pub j2k_native::FormatError::InvalidSignature +pub j2k_native::FormatError::MissingCodestream +pub j2k_native::FormatError::Unsupported +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::FormatError) -> Self +impl core::error::Error for j2k_native::error::FormatError +impl core::fmt::Display for j2k_native::error::FormatError +pub fn j2k_native::error::FormatError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub enum j2k_native::HtCodeBlockDecodePhaseLimit +pub j2k_native::HtCodeBlockDecodePhaseLimit::Cleanup +pub j2k_native::HtCodeBlockDecodePhaseLimit::MagnitudeRefinement +pub j2k_native::HtCodeBlockDecodePhaseLimit::SignificancePropagation +#[non_exhaustive] pub enum j2k_native::J2kCodestreamHeaderError +pub j2k_native::J2kCodestreamHeaderError::InvalidCod +pub j2k_native::J2kCodestreamHeaderError::InvalidCod::what: &'static str +pub j2k_native::J2kCodestreamHeaderError::InvalidMarker +pub j2k_native::J2kCodestreamHeaderError::InvalidMarker::marker: u8 +pub j2k_native::J2kCodestreamHeaderError::InvalidMarker::offset: usize +pub j2k_native::J2kCodestreamHeaderError::InvalidSegment +pub j2k_native::J2kCodestreamHeaderError::InvalidSegment::offset: usize +pub j2k_native::J2kCodestreamHeaderError::InvalidSegment::what: &'static str +pub j2k_native::J2kCodestreamHeaderError::InvalidSiz +pub j2k_native::J2kCodestreamHeaderError::InvalidSiz::what: &'static str +pub j2k_native::J2kCodestreamHeaderError::MissingRequiredMarker +pub j2k_native::J2kCodestreamHeaderError::MissingRequiredMarker::marker: &'static str +pub j2k_native::J2kCodestreamHeaderError::TooShort +pub j2k_native::J2kCodestreamHeaderError::TooShort::have: usize +pub j2k_native::J2kCodestreamHeaderError::TooShort::need: usize +pub j2k_native::J2kCodestreamHeaderError::TruncatedAt +pub j2k_native::J2kCodestreamHeaderError::TruncatedAt::offset: usize +pub j2k_native::J2kCodestreamHeaderError::TruncatedAt::segment: &'static str +pub j2k_native::J2kCodestreamHeaderError::Unsupported +pub j2k_native::J2kCodestreamHeaderError::Unsupported::what: &'static str +impl core::fmt::Display for j2k_native::J2kCodestreamHeaderError +pub fn j2k_native::J2kCodestreamHeaderError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub enum j2k_native::J2kDirectGrayscaleStep +pub j2k_native::J2kDirectGrayscaleStep::ClassicSubBand(j2k_native::J2kOwnedSubBandPlan) +pub j2k_native::J2kDirectGrayscaleStep::HtSubBand(j2k_native::HtOwnedSubBandPlan) +pub j2k_native::J2kDirectGrayscaleStep::Idwt(j2k_native::J2kDirectIdwtStep) +pub j2k_native::J2kDirectGrayscaleStep::Store(j2k_native::J2kDirectStoreStep) +pub enum j2k_native::J2kWaveletTransform +pub j2k_native::J2kWaveletTransform::Irreversible97 +pub j2k_native::J2kWaveletTransform::Reversible53 +#[non_exhaustive] pub enum j2k_native::MarkerError +pub j2k_native::MarkerError::Expected(&'static str) +pub j2k_native::MarkerError::Invalid +pub j2k_native::MarkerError::Missing(&'static str) +pub j2k_native::MarkerError::ParseFailure(&'static str) +pub j2k_native::MarkerError::Unsupported +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::MarkerError) -> Self +impl core::error::Error for j2k_native::error::MarkerError +impl core::fmt::Display for j2k_native::error::MarkerError +pub fn j2k_native::error::MarkerError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +#[non_exhaustive] pub enum j2k_native::TileError +pub j2k_native::TileError::Invalid +pub j2k_native::TileError::InvalidIndex +pub j2k_native::TileError::InvalidOffsets +pub j2k_native::TileError::PpmPptConflict +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::TileError) -> Self +impl core::error::Error for j2k_native::error::TileError +impl core::fmt::Display for j2k_native::error::TileError +pub fn j2k_native::error::TileError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +#[non_exhaustive] pub enum j2k_native::ValidationError +pub j2k_native::ValidationError::ImageTooLarge +pub j2k_native::ValidationError::InsufficientExponents +pub j2k_native::ValidationError::InvalidChannelDefinition +pub j2k_native::ValidationError::InvalidComponentMetadata +pub j2k_native::ValidationError::InvalidDimensions +pub j2k_native::ValidationError::InvalidExponents +pub j2k_native::ValidationError::InvalidProgressionOrder +pub j2k_native::ValidationError::InvalidQuantizationStyle +pub j2k_native::ValidationError::InvalidTransformation +pub j2k_native::ValidationError::MissingPrecinctExponents +pub j2k_native::ValidationError::MissingStepSize +pub j2k_native::ValidationError::TooManyChannels +pub j2k_native::ValidationError::TooManyTiles +impl core::convert::From for j2k_native::error::DecodeError +pub fn j2k_native::error::DecodeError::from(j2k_native::error::ValidationError) -> Self +impl core::error::Error for j2k_native::error::ValidationError +impl core::fmt::Display for j2k_native::error::ValidationError +pub fn j2k_native::error::ValidationError::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_native::Bitmap +pub j2k_native::Bitmap::color_space: j2k_native::ColorSpace +pub j2k_native::Bitmap::data: alloc::vec::Vec +pub j2k_native::Bitmap::has_alpha: bool +pub j2k_native::Bitmap::height: u32 +pub j2k_native::Bitmap::original_bit_depth: u8 +pub j2k_native::Bitmap::width: u32 +pub struct j2k_native::ComponentPlane<'a> +impl<'a> j2k_native::ComponentPlane<'a> +pub fn j2k_native::ComponentPlane<'a>::bit_depth(&self) -> u8 +pub fn j2k_native::ComponentPlane<'a>::samples(&self) -> &'a [f32] +pub struct j2k_native::DecodeSettings +pub j2k_native::DecodeSettings::resolve_palette_indices: bool +pub j2k_native::DecodeSettings::strict: bool +pub j2k_native::DecodeSettings::target_resolution: core::option::Option<(u32, u32)> +impl core::default::Default for j2k_native::DecodeSettings +pub fn j2k_native::DecodeSettings::default() -> Self +pub struct j2k_native::DecodedComponents<'a> +impl<'a> j2k_native::DecodedComponents<'a> +pub fn j2k_native::DecodedComponents<'a>::color_space(&self) -> &j2k_native::ColorSpace +pub fn j2k_native::DecodedComponents<'a>::dimensions(&self) -> (u32, u32) +pub fn j2k_native::DecodedComponents<'a>::has_alpha(&self) -> bool +pub fn j2k_native::DecodedComponents<'a>::planes(&self) -> &[j2k_native::ComponentPlane<'a>] +pub struct j2k_native::DecoderContext<'a> +impl j2k_native::DecoderContext<'_> +pub fn j2k_native::DecoderContext<'_>::cpu_decode_parallelism(&self) -> j2k_native::CpuDecodeParallelism +pub fn j2k_native::DecoderContext<'_>::set_cpu_decode_parallelism(&mut self, j2k_native::CpuDecodeParallelism) +impl core::default::Default for j2k_native::DecoderContext<'_> +pub fn j2k_native::DecoderContext<'_>::default() -> Self +pub struct j2k_native::EncodeOptions +pub j2k_native::EncodeOptions::code_block_height_exp: u8 +pub j2k_native::EncodeOptions::code_block_width_exp: u8 +pub j2k_native::EncodeOptions::component_sampling: core::option::Option> +pub j2k_native::EncodeOptions::guard_bits: u8 +pub j2k_native::EncodeOptions::irreversible_quantization_scale: f32 +pub j2k_native::EncodeOptions::irreversible_quantization_subband_scales: j2k_types::IrreversibleQuantizationSubbandScales +pub j2k_native::EncodeOptions::num_decomposition_levels: u8 +pub j2k_native::EncodeOptions::num_layers: u8 +pub j2k_native::EncodeOptions::precinct_exponents: alloc::vec::Vec<(u8, u8)> +pub j2k_native::EncodeOptions::progression_order: j2k_native::EncodeProgressionOrder +pub j2k_native::EncodeOptions::quality_layer_byte_targets: alloc::vec::Vec +pub j2k_native::EncodeOptions::reversible: bool +pub j2k_native::EncodeOptions::tile_size: core::option::Option<(u32, u32)> +pub j2k_native::EncodeOptions::use_ht_block_coding: bool +pub j2k_native::EncodeOptions::use_mct: bool +pub j2k_native::EncodeOptions::validate_high_throughput_codestream: bool +pub j2k_native::EncodeOptions::write_eph: bool +pub j2k_native::EncodeOptions::write_plm: bool +pub j2k_native::EncodeOptions::write_plt: bool +pub j2k_native::EncodeOptions::write_sop: bool +pub j2k_native::EncodeOptions::write_tlm: bool +impl core::default::Default for j2k_native::EncodeOptions +pub fn j2k_native::EncodeOptions::default() -> Self +pub struct j2k_native::HtCleanupEncodeDistribution +pub j2k_native::HtCleanupEncodeDistribution::initial_quads: u64 +pub j2k_native::HtCleanupEncodeDistribution::initial_rho_counts: [u64; 16] +pub j2k_native::HtCleanupEncodeDistribution::mag_sign_calls: u64 +pub j2k_native::HtCleanupEncodeDistribution::mag_sign_encoded_samples: u64 +pub j2k_native::HtCleanupEncodeDistribution::mag_sign_rho_counts: [u64; 16] +pub j2k_native::HtCleanupEncodeDistribution::mag_sign_sample_bit_counts: [u64; 32] +pub j2k_native::HtCleanupEncodeDistribution::non_initial_e_qmax_counts: [u64; 32] +pub j2k_native::HtCleanupEncodeDistribution::non_initial_kappa_counts: [u64; 32] +pub j2k_native::HtCleanupEncodeDistribution::non_initial_quads: u64 +pub j2k_native::HtCleanupEncodeDistribution::non_initial_rho_counts: [u64; 16] +pub j2k_native::HtCleanupEncodeDistribution::non_initial_rho_u_q_counts: [[u64; 32]; 16] +pub j2k_native::HtCleanupEncodeDistribution::non_initial_u_q_counts: [u64; 32] +pub j2k_native::HtCleanupEncodeDistribution::rho_counts: [u64; 16] +pub j2k_native::HtCleanupEncodeDistribution::total_quads: u64 +pub struct j2k_native::HtCodeBlockBatchJob<'a> +pub j2k_native::HtCodeBlockBatchJob::code_block: j2k_native::HtCodeBlockDecodeJob<'a> +pub j2k_native::HtCodeBlockBatchJob::output_x: u32 +pub j2k_native::HtCodeBlockBatchJob::output_y: u32 +pub struct j2k_native::HtCodeBlockDecodeJob<'a> +pub j2k_native::HtCodeBlockDecodeJob::cleanup_length: u32 +pub j2k_native::HtCodeBlockDecodeJob::data: &'a [u8] +pub j2k_native::HtCodeBlockDecodeJob::dequantization_step: f32 +pub j2k_native::HtCodeBlockDecodeJob::height: u32 +pub j2k_native::HtCodeBlockDecodeJob::missing_bit_planes: u8 +pub j2k_native::HtCodeBlockDecodeJob::num_bitplanes: u8 +pub j2k_native::HtCodeBlockDecodeJob::number_of_coding_passes: u8 +pub j2k_native::HtCodeBlockDecodeJob::output_stride: usize +pub j2k_native::HtCodeBlockDecodeJob::refinement_length: u32 +pub j2k_native::HtCodeBlockDecodeJob::roi_shift: u8 +pub j2k_native::HtCodeBlockDecodeJob::strict: bool +pub j2k_native::HtCodeBlockDecodeJob::stripe_causal: bool +pub j2k_native::HtCodeBlockDecodeJob::width: u32 +pub struct j2k_native::HtCodeBlockDecodeProfile +pub j2k_native::HtCodeBlockDecodeProfile::blocks: u128 +pub j2k_native::HtCodeBlockDecodeProfile::cleanup_bytes: u128 +pub j2k_native::HtCodeBlockDecodeProfile::cleanup_us: u128 +pub j2k_native::HtCodeBlockDecodeProfile::mag_sgn_us: u128 +pub j2k_native::HtCodeBlockDecodeProfile::magref_us: u128 +pub j2k_native::HtCodeBlockDecodeProfile::refinement_blocks: u128 +pub j2k_native::HtCodeBlockDecodeProfile::refinement_bytes: u128 +pub j2k_native::HtCodeBlockDecodeProfile::sigma_us: u128 +pub j2k_native::HtCodeBlockDecodeProfile::sigprop_us: u128 +pub struct j2k_native::HtCodeBlockDecodeWorkspace +impl j2k_native::HtCodeBlockDecodeWorkspace +pub fn j2k_native::HtCodeBlockDecodeWorkspace::coefficient_capacity(&self) -> usize +pub struct j2k_native::HtOwnedCodeBlockBatchJob +pub j2k_native::HtOwnedCodeBlockBatchJob::cleanup_length: u32 +pub j2k_native::HtOwnedCodeBlockBatchJob::data: alloc::vec::Vec +pub j2k_native::HtOwnedCodeBlockBatchJob::dequantization_step: f32 +pub j2k_native::HtOwnedCodeBlockBatchJob::height: u32 +pub j2k_native::HtOwnedCodeBlockBatchJob::missing_bit_planes: u8 +pub j2k_native::HtOwnedCodeBlockBatchJob::num_bitplanes: u8 +pub j2k_native::HtOwnedCodeBlockBatchJob::number_of_coding_passes: u8 +pub j2k_native::HtOwnedCodeBlockBatchJob::output_stride: usize +pub j2k_native::HtOwnedCodeBlockBatchJob::output_x: u32 +pub j2k_native::HtOwnedCodeBlockBatchJob::output_y: u32 +pub j2k_native::HtOwnedCodeBlockBatchJob::refinement_length: u32 +pub j2k_native::HtOwnedCodeBlockBatchJob::roi_shift: u8 +pub j2k_native::HtOwnedCodeBlockBatchJob::strict: bool +pub j2k_native::HtOwnedCodeBlockBatchJob::stripe_causal: bool +pub j2k_native::HtOwnedCodeBlockBatchJob::width: u32 +pub struct j2k_native::HtOwnedSubBandPlan +pub j2k_native::HtOwnedSubBandPlan::band_id: j2k_native::J2kDirectBandId +pub j2k_native::HtOwnedSubBandPlan::height: u32 +pub j2k_native::HtOwnedSubBandPlan::jobs: alloc::vec::Vec +pub j2k_native::HtOwnedSubBandPlan::rect: j2k_native::J2kRect +pub j2k_native::HtOwnedSubBandPlan::width: u32 +pub struct j2k_native::HtSigPropBenchmarkState(_) +impl j2k_native::HtSigPropBenchmarkState +pub fn j2k_native::HtSigPropBenchmarkState::output_len(&self) -> usize +pub struct j2k_native::HtSubBandDecodeJob<'a> +pub j2k_native::HtSubBandDecodeJob::height: u32 +pub j2k_native::HtSubBandDecodeJob::jobs: &'a [j2k_native::HtCodeBlockBatchJob<'a>] +pub j2k_native::HtSubBandDecodeJob::width: u32 +#[repr(C)] pub struct j2k_native::HtUvlcTableEntry +pub j2k_native::HtUvlcTableEntry::ext: u8 +pub j2k_native::HtUvlcTableEntry::ext_len: u8 +pub j2k_native::HtUvlcTableEntry::pre: u8 +pub j2k_native::HtUvlcTableEntry::pre_len: u8 +pub j2k_native::HtUvlcTableEntry::suf: u8 +pub j2k_native::HtUvlcTableEntry::suf_len: u8 +pub struct j2k_native::Image<'a> +impl<'a> j2k_native::Image<'a> +pub fn j2k_native::Image<'a>::build_direct_color_plan_region_with_context(&self, &mut j2k_native::DecoderContext<'a>, (u32, u32, u32, u32)) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::build_direct_color_plan_with_context(&self, &mut j2k_native::DecoderContext<'a>) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::build_direct_grayscale_plan_region_with_context(&self, &mut j2k_native::DecoderContext<'a>, (u32, u32, u32, u32)) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::build_direct_grayscale_plan_with_context(&self, &mut j2k_native::DecoderContext<'a>) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::color_space(&self) -> &j2k_native::ColorSpace +pub fn j2k_native::Image<'a>::decode(&self) -> j2k_native::error::Result> +pub fn j2k_native::Image<'a>::decode_components_with_context<'ctx>(&self, &'ctx mut j2k_native::DecoderContext<'a>) -> j2k_native::error::Result> +pub fn j2k_native::Image<'a>::decode_components_with_ht_decoder<'ctx>(&self, &'ctx mut j2k_native::DecoderContext<'a>, &mut dyn j2k_native::HtCodeBlockDecoder) -> j2k_native::error::Result> +pub fn j2k_native::Image<'a>::decode_into(&self, &mut [u8], &mut j2k_native::DecoderContext<'a>) -> j2k_native::error::Result<()> +pub fn j2k_native::Image<'a>::decode_native(&self) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::decode_native_region(&self, (u32, u32, u32, u32)) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::decode_native_region_with_context(&self, (u32, u32, u32, u32), &mut j2k_native::DecoderContext<'a>) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::decode_native_with_context(&self, &mut j2k_native::DecoderContext<'a>) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::decode_region(&self, (u32, u32, u32, u32)) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::decode_region_components_with_context<'ctx>(&self, (u32, u32, u32, u32), &'ctx mut j2k_native::DecoderContext<'a>) -> j2k_native::error::Result> +pub fn j2k_native::Image<'a>::decode_region_components_with_ht_decoder<'ctx>(&self, &'ctx mut j2k_native::DecoderContext<'a>, (u32, u32, u32, u32), &mut dyn j2k_native::HtCodeBlockDecoder) -> j2k_native::error::Result> +pub fn j2k_native::Image<'a>::decode_region_with_context(&self, (u32, u32, u32, u32), &mut j2k_native::DecoderContext<'a>) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::decode_reversible_53_coefficients(&self) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::decode_reversible_53_coefficients_with_context(&self, &mut j2k_native::DecoderContext<'a>) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::decode_with_context(&self, &mut j2k_native::DecoderContext<'a>) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::has_alpha(&self) -> bool +pub fn j2k_native::Image<'a>::height(&self) -> u32 +pub fn j2k_native::Image<'a>::new(&'a [u8], &j2k_native::DecodeSettings) -> j2k_native::error::Result +pub fn j2k_native::Image<'a>::original_bit_depth(&self) -> u8 +pub fn j2k_native::Image<'a>::supports_direct_device_plane_reuse(&self) -> bool +pub fn j2k_native::Image<'a>::width(&self) -> u32 +pub struct j2k_native::J2kCodeBlockBatchJob<'a> +pub j2k_native::J2kCodeBlockBatchJob::code_block: j2k_native::J2kCodeBlockDecodeJob<'a> +pub j2k_native::J2kCodeBlockBatchJob::output_x: u32 +pub j2k_native::J2kCodeBlockBatchJob::output_y: u32 +pub struct j2k_native::J2kCodeBlockDecodeJob<'a> +pub j2k_native::J2kCodeBlockDecodeJob::data: &'a [u8] +pub j2k_native::J2kCodeBlockDecodeJob::dequantization_step: f32 +pub j2k_native::J2kCodeBlockDecodeJob::height: u32 +pub j2k_native::J2kCodeBlockDecodeJob::missing_bit_planes: u8 +pub j2k_native::J2kCodeBlockDecodeJob::number_of_coding_passes: u8 +pub j2k_native::J2kCodeBlockDecodeJob::output_stride: usize +pub j2k_native::J2kCodeBlockDecodeJob::roi_shift: u8 +pub j2k_native::J2kCodeBlockDecodeJob::segments: &'a [j2k_types::J2kCodeBlockSegment] +pub j2k_native::J2kCodeBlockDecodeJob::strict: bool +pub j2k_native::J2kCodeBlockDecodeJob::style: j2k_types::J2kCodeBlockStyle +pub j2k_native::J2kCodeBlockDecodeJob::sub_band_type: j2k_types::J2kSubBandType +pub j2k_native::J2kCodeBlockDecodeJob::total_bitplanes: u8 +pub j2k_native::J2kCodeBlockDecodeJob::width: u32 +pub struct j2k_native::J2kCodeBlockDecodeProfile +pub j2k_native::J2kCodeBlockDecodeProfile::bypass_us: u128 +pub j2k_native::J2kCodeBlockDecodeProfile::cleanup_us: u128 +pub j2k_native::J2kCodeBlockDecodeProfile::magref_us: u128 +pub j2k_native::J2kCodeBlockDecodeProfile::output_convert_us: u128 +pub j2k_native::J2kCodeBlockDecodeProfile::sigprop_us: u128 +pub struct j2k_native::J2kCodeBlockDecodeWorkspace +pub struct j2k_native::J2kCodestreamComponentHeader +pub j2k_native::J2kCodestreamComponentHeader::bit_depth: u8 +pub j2k_native::J2kCodestreamComponentHeader::signed: bool +pub j2k_native::J2kCodestreamComponentHeader::x_rsiz: u8 +pub j2k_native::J2kCodestreamComponentHeader::y_rsiz: u8 +pub struct j2k_native::J2kCodestreamHeaderMetadata +pub j2k_native::J2kCodestreamHeaderMetadata::bit_depth: u8 +pub j2k_native::J2kCodestreamHeaderMetadata::component_info: alloc::vec::Vec +pub j2k_native::J2kCodestreamHeaderMetadata::components: u16 +pub j2k_native::J2kCodestreamHeaderMetadata::dimensions: (u32, u32) +pub j2k_native::J2kCodestreamHeaderMetadata::has_mct: bool +pub j2k_native::J2kCodestreamHeaderMetadata::high_throughput: bool +pub j2k_native::J2kCodestreamHeaderMetadata::resolution_levels: u8 +pub j2k_native::J2kCodestreamHeaderMetadata::reversible: bool +pub j2k_native::J2kCodestreamHeaderMetadata::tile_count: (u32, u32) +pub j2k_native::J2kCodestreamHeaderMetadata::tile_size: (u32, u32) +pub struct j2k_native::J2kDirectColorPlan +pub j2k_native::J2kDirectColorPlan::bit_depths: [u8; 3] +pub j2k_native::J2kDirectColorPlan::component_plans: alloc::vec::Vec +pub j2k_native::J2kDirectColorPlan::dimensions: (u32, u32) +pub j2k_native::J2kDirectColorPlan::mct: bool +pub j2k_native::J2kDirectColorPlan::transform: j2k_native::J2kWaveletTransform +pub struct j2k_native::J2kDirectCpuScratch +impl j2k_native::J2kDirectCpuScratch +pub fn j2k_native::J2kDirectCpuScratch::clear(&mut self) +pub const fn j2k_native::J2kDirectCpuScratch::new() -> Self +pub struct j2k_native::J2kDirectGrayscalePlan +pub j2k_native::J2kDirectGrayscalePlan::bit_depth: u8 +pub j2k_native::J2kDirectGrayscalePlan::dimensions: (u32, u32) +pub j2k_native::J2kDirectGrayscalePlan::steps: alloc::vec::Vec +pub struct j2k_native::J2kDirectIdwtStep +pub j2k_native::J2kDirectIdwtStep::hh: j2k_native::J2kRect +pub j2k_native::J2kDirectIdwtStep::hh_band_id: j2k_native::J2kDirectBandId +pub j2k_native::J2kDirectIdwtStep::hl: j2k_native::J2kRect +pub j2k_native::J2kDirectIdwtStep::hl_band_id: j2k_native::J2kDirectBandId +pub j2k_native::J2kDirectIdwtStep::lh: j2k_native::J2kRect +pub j2k_native::J2kDirectIdwtStep::lh_band_id: j2k_native::J2kDirectBandId +pub j2k_native::J2kDirectIdwtStep::ll: j2k_native::J2kRect +pub j2k_native::J2kDirectIdwtStep::ll_band_id: j2k_native::J2kDirectBandId +pub j2k_native::J2kDirectIdwtStep::output_band_id: j2k_native::J2kDirectBandId +pub j2k_native::J2kDirectIdwtStep::rect: j2k_native::J2kRect +pub j2k_native::J2kDirectIdwtStep::transform: j2k_native::J2kWaveletTransform +pub struct j2k_native::J2kDirectStoreStep +pub j2k_native::J2kDirectStoreStep::addend: f32 +pub j2k_native::J2kDirectStoreStep::copy_height: u32 +pub j2k_native::J2kDirectStoreStep::copy_width: u32 +pub j2k_native::J2kDirectStoreStep::input_band_id: j2k_native::J2kDirectBandId +pub j2k_native::J2kDirectStoreStep::input_rect: j2k_native::J2kRect +pub j2k_native::J2kDirectStoreStep::output_height: u32 +pub j2k_native::J2kDirectStoreStep::output_width: u32 +pub j2k_native::J2kDirectStoreStep::output_x: u32 +pub j2k_native::J2kDirectStoreStep::output_y: u32 +pub j2k_native::J2kDirectStoreStep::source_x: u32 +pub j2k_native::J2kDirectStoreStep::source_y: u32 +pub struct j2k_native::J2kIdwtBand<'a> +pub j2k_native::J2kIdwtBand::coefficients: &'a [f32] +pub j2k_native::J2kIdwtBand::rect: j2k_native::J2kRect +pub struct j2k_native::J2kInverseMctJob<'a> +pub j2k_native::J2kInverseMctJob::addend0: f32 +pub j2k_native::J2kInverseMctJob::addend1: f32 +pub j2k_native::J2kInverseMctJob::addend2: f32 +pub j2k_native::J2kInverseMctJob::plane0: &'a mut [f32] +pub j2k_native::J2kInverseMctJob::plane1: &'a mut [f32] +pub j2k_native::J2kInverseMctJob::plane2: &'a mut [f32] +pub j2k_native::J2kInverseMctJob::transform: j2k_native::J2kWaveletTransform +pub struct j2k_native::J2kOwnedCodeBlockBatchJob +pub j2k_native::J2kOwnedCodeBlockBatchJob::data: alloc::vec::Vec +pub j2k_native::J2kOwnedCodeBlockBatchJob::dequantization_step: f32 +pub j2k_native::J2kOwnedCodeBlockBatchJob::height: u32 +pub j2k_native::J2kOwnedCodeBlockBatchJob::missing_bit_planes: u8 +pub j2k_native::J2kOwnedCodeBlockBatchJob::number_of_coding_passes: u8 +pub j2k_native::J2kOwnedCodeBlockBatchJob::output_stride: usize +pub j2k_native::J2kOwnedCodeBlockBatchJob::output_x: u32 +pub j2k_native::J2kOwnedCodeBlockBatchJob::output_y: u32 +pub j2k_native::J2kOwnedCodeBlockBatchJob::roi_shift: u8 +pub j2k_native::J2kOwnedCodeBlockBatchJob::segments: alloc::vec::Vec +pub j2k_native::J2kOwnedCodeBlockBatchJob::strict: bool +pub j2k_native::J2kOwnedCodeBlockBatchJob::style: j2k_types::J2kCodeBlockStyle +pub j2k_native::J2kOwnedCodeBlockBatchJob::sub_band_type: j2k_types::J2kSubBandType +pub j2k_native::J2kOwnedCodeBlockBatchJob::total_bitplanes: u8 +pub j2k_native::J2kOwnedCodeBlockBatchJob::width: u32 +pub struct j2k_native::J2kOwnedSubBandPlan +pub j2k_native::J2kOwnedSubBandPlan::band_id: j2k_native::J2kDirectBandId +pub j2k_native::J2kOwnedSubBandPlan::height: u32 +pub j2k_native::J2kOwnedSubBandPlan::jobs: alloc::vec::Vec +pub j2k_native::J2kOwnedSubBandPlan::rect: j2k_native::J2kRect +pub j2k_native::J2kOwnedSubBandPlan::width: u32 +pub struct j2k_native::J2kRect +pub j2k_native::J2kRect::x0: u32 +pub j2k_native::J2kRect::x1: u32 +pub j2k_native::J2kRect::y0: u32 +pub j2k_native::J2kRect::y1: u32 +impl j2k_native::J2kRect +pub fn j2k_native::J2kRect::height(self) -> u32 +pub fn j2k_native::J2kRect::width(self) -> u32 +pub struct j2k_native::J2kSingleDecompositionIdwtJob<'a> +pub j2k_native::J2kSingleDecompositionIdwtJob::hh: j2k_native::J2kIdwtBand<'a> +pub j2k_native::J2kSingleDecompositionIdwtJob::hl: j2k_native::J2kIdwtBand<'a> +pub j2k_native::J2kSingleDecompositionIdwtJob::lh: j2k_native::J2kIdwtBand<'a> +pub j2k_native::J2kSingleDecompositionIdwtJob::ll: j2k_native::J2kIdwtBand<'a> +pub j2k_native::J2kSingleDecompositionIdwtJob::rect: j2k_native::J2kRect +pub j2k_native::J2kSingleDecompositionIdwtJob::transform: j2k_native::J2kWaveletTransform +pub struct j2k_native::J2kStoreComponentJob<'a> +pub j2k_native::J2kStoreComponentJob::addend: f32 +pub j2k_native::J2kStoreComponentJob::copy_height: u32 +pub j2k_native::J2kStoreComponentJob::copy_width: u32 +pub j2k_native::J2kStoreComponentJob::input: &'a [f32] +pub j2k_native::J2kStoreComponentJob::input_width: u32 +pub j2k_native::J2kStoreComponentJob::output: &'a mut [f32] +pub j2k_native::J2kStoreComponentJob::output_width: u32 +pub j2k_native::J2kStoreComponentJob::output_x: u32 +pub j2k_native::J2kStoreComponentJob::output_y: u32 +pub j2k_native::J2kStoreComponentJob::source_x: u32 +pub j2k_native::J2kStoreComponentJob::source_y: u32 +pub struct j2k_native::J2kSubBandDecodeJob<'a> +pub j2k_native::J2kSubBandDecodeJob::height: u32 +pub j2k_native::J2kSubBandDecodeJob::jobs: &'a [j2k_native::J2kCodeBlockBatchJob<'a>] +pub j2k_native::J2kSubBandDecodeJob::width: u32 +pub struct j2k_native::J2kTier1TokenSegment +pub j2k_native::J2kTier1TokenSegment::end_coding_pass: u8 +pub j2k_native::J2kTier1TokenSegment::start_coding_pass: u8 +pub j2k_native::J2kTier1TokenSegment::token_bit_count: u32 +pub j2k_native::J2kTier1TokenSegment::token_bit_offset: u32 +pub j2k_native::J2kTier1TokenSegment::use_arithmetic: bool +pub struct j2k_native::RawBitmap +pub j2k_native::RawBitmap::bit_depth: u8 +pub j2k_native::RawBitmap::bytes_per_sample: u8 +pub j2k_native::RawBitmap::data: alloc::vec::Vec +pub j2k_native::RawBitmap::height: u32 +pub j2k_native::RawBitmap::num_components: u8 +pub j2k_native::RawBitmap::width: u32 +pub struct j2k_native::Reversible53CoefficientImage +pub j2k_native::Reversible53CoefficientImage::code_block_height_exp: u8 +pub j2k_native::Reversible53CoefficientImage::code_block_width_exp: u8 +pub j2k_native::Reversible53CoefficientImage::guard_bits: u8 +pub j2k_native::Reversible53CoefficientImage::image: j2k_types::PrecomputedHtj2k53Image +pub j2k_native::Reversible53CoefficientImage::use_mct: bool +pub trait j2k_native::HtCodeBlockDecoder +pub fn j2k_native::HtCodeBlockDecoder::decode_code_block(&mut self, j2k_native::HtCodeBlockDecodeJob<'_>, &mut [f32]) -> j2k_native::error::Result<()> +pub fn j2k_native::HtCodeBlockDecoder::decode_inverse_mct(&mut self, j2k_native::J2kInverseMctJob<'_>) -> j2k_native::error::Result +pub fn j2k_native::HtCodeBlockDecoder::decode_j2k_code_block(&mut self, j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32]) -> j2k_native::error::Result +pub fn j2k_native::HtCodeBlockDecoder::decode_j2k_sub_band(&mut self, j2k_native::J2kSubBandDecodeJob<'_>, &mut [f32]) -> j2k_native::error::Result +pub fn j2k_native::HtCodeBlockDecoder::decode_single_decomposition_idwt(&mut self, j2k_native::J2kSingleDecompositionIdwtJob<'_>, &mut [f32]) -> j2k_native::error::Result +pub fn j2k_native::HtCodeBlockDecoder::decode_store_component(&mut self, j2k_native::J2kStoreComponentJob<'_>) -> j2k_native::error::Result +pub fn j2k_native::HtCodeBlockDecoder::decode_sub_band(&mut self, j2k_native::HtSubBandDecodeJob<'_>, &mut [f32]) -> j2k_native::error::Result +pub trait j2k_native::J2kEncodeStageAccelerator +pub fn j2k_native::J2kEncodeStageAccelerator::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport +pub fn j2k_native::J2kEncodeStageAccelerator::encode_deinterleave(&mut self, j2k_types::J2kDeinterleaveToF32Job<'_>) -> core::result::Result>>, &'static str> +pub fn j2k_native::J2kEncodeStageAccelerator::encode_forward_dwt53(&mut self, j2k_types::J2kForwardDwt53Job<'_>) -> core::result::Result, &'static str> +pub fn j2k_native::J2kEncodeStageAccelerator::encode_forward_dwt97(&mut self, j2k_types::J2kForwardDwt97Job<'_>) -> core::result::Result, &'static str> +pub fn j2k_native::J2kEncodeStageAccelerator::encode_forward_ict(&mut self, j2k_types::J2kForwardIctJob<'_>) -> core::result::Result +pub fn j2k_native::J2kEncodeStageAccelerator::encode_forward_rct(&mut self, j2k_types::J2kForwardRctJob<'_>) -> core::result::Result +pub fn j2k_native::J2kEncodeStageAccelerator::encode_ht_code_block(&mut self, j2k_types::J2kHtCodeBlockEncodeJob<'_>) -> core::result::Result, &'static str> +pub fn j2k_native::J2kEncodeStageAccelerator::encode_ht_code_blocks(&mut self, &[j2k_types::J2kHtCodeBlockEncodeJob<'_>]) -> core::result::Result>, &'static str> +pub fn j2k_native::J2kEncodeStageAccelerator::encode_ht_subband(&mut self, j2k_types::J2kHtSubbandEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_native::J2kEncodeStageAccelerator::encode_htj2k_tile(&mut self, j2k_types::J2kHtj2kTileEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_native::J2kEncodeStageAccelerator::encode_packetization(&mut self, j2k_types::J2kPacketizationEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_native::J2kEncodeStageAccelerator::encode_quantize_subband(&mut self, j2k_types::J2kQuantizeSubbandJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_native::J2kEncodeStageAccelerator::encode_tier1_code_block(&mut self, j2k_types::J2kTier1CodeBlockEncodeJob<'_>) -> core::result::Result, &'static str> +pub fn j2k_native::J2kEncodeStageAccelerator::encode_tier1_code_blocks(&mut self, &[j2k_types::J2kTier1CodeBlockEncodeJob<'_>]) -> core::result::Result>, &'static str> +pub fn j2k_native::J2kEncodeStageAccelerator::prefer_parallel_cpu_code_block_fallback(&self) -> bool +pub fn j2k_native::J2kEncodeStageAccelerator::prefer_parallel_cpu_tile_encode(&self) -> bool +impl j2k_native::J2kEncodeStageAccelerator for j2k_types::CpuOnlyJ2kEncodeStageAccelerator +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::dispatch_report(&self) -> j2k_types::J2kEncodeDispatchReport +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_deinterleave(&mut self, j2k_types::J2kDeinterleaveToF32Job<'_>) -> core::result::Result>>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_forward_dwt53(&mut self, j2k_types::J2kForwardDwt53Job<'_>) -> core::result::Result, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_forward_dwt97(&mut self, j2k_types::J2kForwardDwt97Job<'_>) -> core::result::Result, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_forward_ict(&mut self, j2k_types::J2kForwardIctJob<'_>) -> core::result::Result +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_forward_rct(&mut self, j2k_types::J2kForwardRctJob<'_>) -> core::result::Result +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_ht_code_block(&mut self, j2k_types::J2kHtCodeBlockEncodeJob<'_>) -> core::result::Result, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_ht_code_blocks(&mut self, &[j2k_types::J2kHtCodeBlockEncodeJob<'_>]) -> core::result::Result>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_ht_subband(&mut self, j2k_types::J2kHtSubbandEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_htj2k_tile(&mut self, j2k_types::J2kHtj2kTileEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_packetization(&mut self, j2k_types::J2kPacketizationEncodeJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_quantize_subband(&mut self, j2k_types::J2kQuantizeSubbandJob<'_>) -> core::result::Result>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_tier1_code_block(&mut self, j2k_types::J2kTier1CodeBlockEncodeJob<'_>) -> core::result::Result, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::encode_tier1_code_blocks(&mut self, &[j2k_types::J2kTier1CodeBlockEncodeJob<'_>]) -> core::result::Result>, &'static str> +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::prefer_parallel_cpu_code_block_fallback(&self) -> bool +pub fn j2k_types::CpuOnlyJ2kEncodeStageAccelerator::prefer_parallel_cpu_tile_encode(&self) -> bool +pub fn j2k_native::collect_ht_cleanup_encode_distribution(&[i32], u32, u32, u8) -> core::result::Result +pub fn j2k_native::decode_ht_code_block_scalar(j2k_native::HtCodeBlockDecodeJob<'_>, &mut [f32]) -> j2k_native::error::Result<()> +pub fn j2k_native::decode_ht_code_block_scalar_until_phase(j2k_native::HtCodeBlockDecodeJob<'_>, &mut [f32], j2k_native::HtCodeBlockDecodePhaseLimit) -> j2k_native::error::Result<()> +pub fn j2k_native::decode_ht_code_block_scalar_with_workspace(j2k_native::HtCodeBlockDecodeJob<'_>, &mut [f32], &mut j2k_native::HtCodeBlockDecodeWorkspace) -> j2k_native::error::Result<()> +pub fn j2k_native::decode_ht_code_block_scalar_with_workspace_profiled(j2k_native::HtCodeBlockDecodeJob<'_>, &mut [f32], &mut j2k_native::HtCodeBlockDecodeWorkspace, &mut j2k_native::HtCodeBlockDecodeProfile) -> j2k_native::error::Result<()> +pub fn j2k_native::decode_ht_sigprop_benchmark_state(&mut j2k_native::HtSigPropBenchmarkState, &mut [u32]) -> j2k_native::error::Result<()> +pub fn j2k_native::decode_j2k_code_block_scalar(j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32]) -> j2k_native::error::Result<()> +pub fn j2k_native::decode_j2k_code_block_scalar_profiled(j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32], &mut j2k_native::J2kCodeBlockDecodeProfile) -> j2k_native::error::Result<()> +pub fn j2k_native::decode_j2k_code_block_scalar_with_workspace(j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32], &mut j2k_native::J2kCodeBlockDecodeWorkspace) -> j2k_native::error::Result<()> +pub fn j2k_native::decode_j2k_code_block_scalar_with_workspace_profiled(j2k_native::J2kCodeBlockDecodeJob<'_>, &mut [f32], &mut j2k_native::J2kCodeBlockDecodeWorkspace, &mut j2k_native::J2kCodeBlockDecodeProfile) -> j2k_native::error::Result<()> +pub fn j2k_native::decode_j2k_sub_band_scalar(j2k_native::J2kSubBandDecodeJob<'_>, &mut [f32]) -> j2k_native::error::Result<()> +pub fn j2k_native::deinterleave_reference(&[u8], usize, u8, u8, bool) -> alloc::vec::Vec> +pub fn j2k_native::encode(&[u8], u32, u32, u8, u8, bool, &j2k_native::EncodeOptions) -> core::result::Result, &'static str> +pub fn j2k_native::encode_ht_code_block_scalar(&[i32], u32, u32, u8) -> core::result::Result +pub fn j2k_native::encode_htj2k(&[u8], u32, u32, u8, u8, bool, &j2k_native::EncodeOptions) -> core::result::Result, &'static str> +pub fn j2k_native::encode_j2k_code_block_scalar_with_style(&[i32], u32, u32, j2k_types::J2kSubBandType, u8, j2k_types::J2kCodeBlockStyle) -> core::result::Result +pub fn j2k_native::encode_j2k_packetization_scalar(j2k_types::J2kPacketizationEncodeJob<'_>) -> core::result::Result, &'static str> +pub fn j2k_native::encode_precomputed_htj2k_53(&j2k_types::PrecomputedHtj2k53Image, &j2k_native::EncodeOptions) -> core::result::Result, &'static str> +pub fn j2k_native::encode_precomputed_htj2k_53_with_accelerator(&j2k_types::PrecomputedHtj2k53Image, &j2k_native::EncodeOptions, &mut impl j2k_native::J2kEncodeStageAccelerator) -> core::result::Result, &'static str> +pub fn j2k_native::encode_precomputed_htj2k_53_with_mct(&j2k_types::PrecomputedHtj2k53Image, &j2k_native::EncodeOptions, bool) -> core::result::Result, &'static str> +pub fn j2k_native::encode_precomputed_htj2k_53_with_mct_and_accelerator(&j2k_types::PrecomputedHtj2k53Image, &j2k_native::EncodeOptions, bool, &mut impl j2k_native::J2kEncodeStageAccelerator) -> core::result::Result, &'static str> +pub fn j2k_native::encode_precomputed_htj2k_97(&j2k_types::PrecomputedHtj2k97Image, &j2k_native::EncodeOptions) -> core::result::Result, &'static str> +pub fn j2k_native::encode_precomputed_htj2k_97_batch_with_accelerator(&[j2k_types::PrecomputedHtj2k97Image], &j2k_native::EncodeOptions, &mut impl j2k_native::J2kEncodeStageAccelerator) -> core::result::Result>, &'static str> +pub fn j2k_native::encode_precomputed_htj2k_97_with_accelerator(&j2k_types::PrecomputedHtj2k97Image, &j2k_native::EncodeOptions, &mut impl j2k_native::J2kEncodeStageAccelerator) -> core::result::Result, &'static str> +pub fn j2k_native::encode_preencoded_htj2k_97(&j2k_types::PreencodedHtj2k97Image, &j2k_native::EncodeOptions) -> core::result::Result, &'static str> +pub fn j2k_native::encode_preencoded_htj2k_97_compact_owned_with_accelerator(j2k_types::PreencodedHtj2k97CompactImage, &j2k_native::EncodeOptions, &mut impl j2k_native::J2kEncodeStageAccelerator) -> core::result::Result, &'static str> +pub fn j2k_native::encode_preencoded_htj2k_97_owned_with_accelerator(j2k_types::PreencodedHtj2k97Image, &j2k_native::EncodeOptions, &mut impl j2k_native::J2kEncodeStageAccelerator) -> core::result::Result, &'static str> +pub fn j2k_native::encode_preencoded_htj2k_97_with_accelerator(&j2k_types::PreencodedHtj2k97Image, &j2k_native::EncodeOptions, &mut impl j2k_native::J2kEncodeStageAccelerator) -> core::result::Result, &'static str> +pub fn j2k_native::encode_prequantized_htj2k_97(&j2k_types::PrequantizedHtj2k97Image, &j2k_native::EncodeOptions) -> core::result::Result, &'static str> +pub fn j2k_native::encode_prequantized_htj2k_97_with_accelerator(&j2k_types::PrequantizedHtj2k97Image, &j2k_native::EncodeOptions, &mut impl j2k_native::J2kEncodeStageAccelerator) -> core::result::Result, &'static str> +pub fn j2k_native::encode_with_accelerator(&[u8], u32, u32, u8, u8, bool, &j2k_native::EncodeOptions, &mut impl j2k_native::J2kEncodeStageAccelerator) -> core::result::Result, &'static str> +pub fn j2k_native::execute_direct_color_plan_rgb8_into(&j2k_native::J2kDirectColorPlan, j2k_native::J2kRect, &mut j2k_native::J2kDirectCpuScratch, &mut [u8], usize) -> j2k_native::error::Result<()> +pub fn j2k_native::execute_direct_color_plan_rgba8_into(&j2k_native::J2kDirectColorPlan, j2k_native::J2kRect, &mut j2k_native::J2kDirectCpuScratch, &mut [u8], usize) -> j2k_native::error::Result<()> +pub fn j2k_native::forward_dwt53_reference(&[f32], u32, u32, u8) -> j2k_types::J2kForwardDwt53Output +pub fn j2k_native::forward_dwt97_reference(&[f32], u32, u32, u8) -> j2k_types::J2kForwardDwt97Output +pub fn j2k_native::forward_ict_reference(alloc::vec::Vec>) -> alloc::vec::Vec> +pub fn j2k_native::forward_rct_reference(alloc::vec::Vec>) -> alloc::vec::Vec> +pub fn j2k_native::ht_uvlc_encode_table() -> &'static [j2k_native::HtUvlcTableEntry; 75] +pub fn j2k_native::ht_uvlc_encode_table_bytes() -> &'static [u8] +pub fn j2k_native::ht_uvlc_table0() -> &'static [u16; 320] +pub fn j2k_native::ht_uvlc_table1() -> &'static [u16; 256] +pub fn j2k_native::ht_vlc_encode_table0() -> &'static [u16; 2048] +pub fn j2k_native::ht_vlc_encode_table1() -> &'static [u16; 2048] +pub fn j2k_native::ht_vlc_table0() -> &'static [u16; 1024] +pub fn j2k_native::ht_vlc_table1() -> &'static [u16; 1024] +pub fn j2k_native::idwt_band_index(u32, u32, bool) -> u32 +pub fn j2k_native::inspect_j2k_codestream_header(&[u8]) -> core::result::Result +pub fn j2k_native::irreversible_quantization_step_for_subband(u8, u8, f32, j2k_types::IrreversibleQuantizationSubbandScales, j2k_types::J2kSubBandType) -> j2k_types::IrreversibleQuantizationStep +pub fn j2k_native::looks_like_j2k_codestream(&[u8]) -> bool +pub fn j2k_native::pack_j2k_code_block_scalar_from_tier1_tokens(&[u8], &[j2k_native::J2kTier1TokenSegment], u8, u8) -> core::result::Result +pub fn j2k_native::prepare_ht_sigprop_benchmark_state(j2k_native::HtCodeBlockDecodeJob<'_>) -> j2k_native::error::Result +pub fn j2k_native::quantize_reversible_reference(&[f32], u16, u16, u8, bool) -> alloc::vec::Vec +pub fn j2k_native::quantize_subband_reference(&[f32], u16, u16, u8, bool) -> alloc::vec::Vec +pub type j2k_native::J2kDirectBandId = u32 +pub type j2k_native::Result = core::result::Result +``` + +## `j2k-types` + +```text +pub mod j2k_types +pub enum j2k_types::J2kPacketizationBlockCodingMode +pub j2k_types::J2kPacketizationBlockCodingMode::Classic +pub j2k_types::J2kPacketizationBlockCodingMode::HighThroughput +pub enum j2k_types::J2kPacketizationProgressionOrder +pub j2k_types::J2kPacketizationProgressionOrder::Cprl +pub j2k_types::J2kPacketizationProgressionOrder::Lrcp +pub j2k_types::J2kPacketizationProgressionOrder::Pcrl +pub j2k_types::J2kPacketizationProgressionOrder::Rlcp +pub j2k_types::J2kPacketizationProgressionOrder::Rpcl +pub enum j2k_types::J2kSubBandType +pub j2k_types::J2kSubBandType::HighHigh +pub j2k_types::J2kSubBandType::HighLow +pub j2k_types::J2kSubBandType::LowHigh +pub j2k_types::J2kSubBandType::LowLow +pub struct j2k_types::CpuOnlyJ2kEncodeStageAccelerator +pub struct j2k_types::EncodedHtJ2kCodeBlock +pub j2k_types::EncodedHtJ2kCodeBlock::cleanup_length: u32 +pub j2k_types::EncodedHtJ2kCodeBlock::data: alloc::vec::Vec +pub j2k_types::EncodedHtJ2kCodeBlock::num_coding_passes: u8 +pub j2k_types::EncodedHtJ2kCodeBlock::num_zero_bitplanes: u8 +pub j2k_types::EncodedHtJ2kCodeBlock::refinement_length: u32 +pub struct j2k_types::EncodedJ2kCodeBlock +pub j2k_types::EncodedJ2kCodeBlock::data: alloc::vec::Vec +pub j2k_types::EncodedJ2kCodeBlock::missing_bit_planes: u8 +pub j2k_types::EncodedJ2kCodeBlock::number_of_coding_passes: u8 +pub j2k_types::EncodedJ2kCodeBlock::segments: alloc::vec::Vec +pub struct j2k_types::IrreversibleQuantizationStep +pub j2k_types::IrreversibleQuantizationStep::exponent: u8 +pub j2k_types::IrreversibleQuantizationStep::mantissa: u16 +pub struct j2k_types::IrreversibleQuantizationSubbandScales +pub j2k_types::IrreversibleQuantizationSubbandScales::high_high: f32 +pub j2k_types::IrreversibleQuantizationSubbandScales::high_low: f32 +pub j2k_types::IrreversibleQuantizationSubbandScales::low_high: f32 +pub j2k_types::IrreversibleQuantizationSubbandScales::low_low: f32 +impl core::default::Default for j2k_types::IrreversibleQuantizationSubbandScales +pub fn j2k_types::IrreversibleQuantizationSubbandScales::default() -> Self +pub struct j2k_types::J2kCodeBlockSegment +pub j2k_types::J2kCodeBlockSegment::data_length: u32 +pub j2k_types::J2kCodeBlockSegment::data_offset: u32 +pub j2k_types::J2kCodeBlockSegment::end_coding_pass: u8 +pub j2k_types::J2kCodeBlockSegment::start_coding_pass: u8 +pub j2k_types::J2kCodeBlockSegment::use_arithmetic: bool +pub struct j2k_types::J2kCodeBlockStyle +pub j2k_types::J2kCodeBlockStyle::reset_context_probabilities: bool +pub j2k_types::J2kCodeBlockStyle::segmentation_symbols: bool +pub j2k_types::J2kCodeBlockStyle::selective_arithmetic_coding_bypass: bool +pub j2k_types::J2kCodeBlockStyle::termination_on_each_pass: bool +pub j2k_types::J2kCodeBlockStyle::vertically_causal_context: bool +pub struct j2k_types::J2kDeinterleaveToF32Job<'a> +pub j2k_types::J2kDeinterleaveToF32Job::bit_depth: u8 +pub j2k_types::J2kDeinterleaveToF32Job::num_components: u8 +pub j2k_types::J2kDeinterleaveToF32Job::num_pixels: usize +pub j2k_types::J2kDeinterleaveToF32Job::pixels: &'a [u8] +pub j2k_types::J2kDeinterleaveToF32Job::signed: bool +pub struct j2k_types::J2kEncodeDispatchReport +pub j2k_types::J2kEncodeDispatchReport::deinterleave: usize +pub j2k_types::J2kEncodeDispatchReport::forward_dwt53: usize +pub j2k_types::J2kEncodeDispatchReport::forward_dwt97: usize +pub j2k_types::J2kEncodeDispatchReport::forward_ict: usize +pub j2k_types::J2kEncodeDispatchReport::forward_rct: usize +pub j2k_types::J2kEncodeDispatchReport::ht_code_block: usize +pub j2k_types::J2kEncodeDispatchReport::packetization: usize +pub j2k_types::J2kEncodeDispatchReport::quantize_subband: usize +pub j2k_types::J2kEncodeDispatchReport::tier1_code_block: usize +impl j2k_types::J2kEncodeDispatchReport +pub fn j2k_types::J2kEncodeDispatchReport::any(self) -> bool +pub fn j2k_types::J2kEncodeDispatchReport::saturating_delta(self, Self) -> Self +pub fn j2k_types::J2kEncodeDispatchReport::total(self) -> usize +pub struct j2k_types::J2kForwardDwt53Job<'a> +pub j2k_types::J2kForwardDwt53Job::height: u32 +pub j2k_types::J2kForwardDwt53Job::num_levels: u8 +pub j2k_types::J2kForwardDwt53Job::samples: &'a [f32] +pub j2k_types::J2kForwardDwt53Job::width: u32 +pub struct j2k_types::J2kForwardDwt53Level +pub j2k_types::J2kForwardDwt53Level::height: u32 +pub j2k_types::J2kForwardDwt53Level::hh: alloc::vec::Vec +pub j2k_types::J2kForwardDwt53Level::high_height: u32 +pub j2k_types::J2kForwardDwt53Level::high_width: u32 +pub j2k_types::J2kForwardDwt53Level::hl: alloc::vec::Vec +pub j2k_types::J2kForwardDwt53Level::lh: alloc::vec::Vec +pub j2k_types::J2kForwardDwt53Level::low_height: u32 +pub j2k_types::J2kForwardDwt53Level::low_width: u32 +pub j2k_types::J2kForwardDwt53Level::width: u32 +pub struct j2k_types::J2kForwardDwt53Output +pub j2k_types::J2kForwardDwt53Output::levels: alloc::vec::Vec +pub j2k_types::J2kForwardDwt53Output::ll: alloc::vec::Vec +pub j2k_types::J2kForwardDwt53Output::ll_height: u32 +pub j2k_types::J2kForwardDwt53Output::ll_width: u32 +pub struct j2k_types::J2kForwardDwt97Job<'a> +pub j2k_types::J2kForwardDwt97Job::height: u32 +pub j2k_types::J2kForwardDwt97Job::num_levels: u8 +pub j2k_types::J2kForwardDwt97Job::samples: &'a [f32] +pub j2k_types::J2kForwardDwt97Job::width: u32 +pub struct j2k_types::J2kForwardDwt97Level +pub j2k_types::J2kForwardDwt97Level::height: u32 +pub j2k_types::J2kForwardDwt97Level::hh: alloc::vec::Vec +pub j2k_types::J2kForwardDwt97Level::high_height: u32 +pub j2k_types::J2kForwardDwt97Level::high_width: u32 +pub j2k_types::J2kForwardDwt97Level::hl: alloc::vec::Vec +pub j2k_types::J2kForwardDwt97Level::lh: alloc::vec::Vec +pub j2k_types::J2kForwardDwt97Level::low_height: u32 +pub j2k_types::J2kForwardDwt97Level::low_width: u32 +pub j2k_types::J2kForwardDwt97Level::width: u32 +pub struct j2k_types::J2kForwardDwt97Output +pub j2k_types::J2kForwardDwt97Output::levels: alloc::vec::Vec +pub j2k_types::J2kForwardDwt97Output::ll: alloc::vec::Vec +pub j2k_types::J2kForwardDwt97Output::ll_height: u32 +pub j2k_types::J2kForwardDwt97Output::ll_width: u32 +pub struct j2k_types::J2kForwardIctJob<'a> +pub j2k_types::J2kForwardIctJob::plane0: &'a mut [f32] +pub j2k_types::J2kForwardIctJob::plane1: &'a mut [f32] +pub j2k_types::J2kForwardIctJob::plane2: &'a mut [f32] +pub struct j2k_types::J2kForwardRctJob<'a> +pub j2k_types::J2kForwardRctJob::plane0: &'a mut [f32] +pub j2k_types::J2kForwardRctJob::plane1: &'a mut [f32] +pub j2k_types::J2kForwardRctJob::plane2: &'a mut [f32] +pub struct j2k_types::J2kHtCodeBlockEncodeJob<'a> +pub j2k_types::J2kHtCodeBlockEncodeJob::coefficients: &'a [i32] +pub j2k_types::J2kHtCodeBlockEncodeJob::height: u32 +pub j2k_types::J2kHtCodeBlockEncodeJob::target_coding_passes: u8 +pub j2k_types::J2kHtCodeBlockEncodeJob::total_bitplanes: u8 +pub j2k_types::J2kHtCodeBlockEncodeJob::width: u32 +pub struct j2k_types::J2kHtSubbandEncodeJob<'a> +pub j2k_types::J2kHtSubbandEncodeJob::code_block_height: u32 +pub j2k_types::J2kHtSubbandEncodeJob::code_block_width: u32 +pub j2k_types::J2kHtSubbandEncodeJob::coefficients: &'a [f32] +pub j2k_types::J2kHtSubbandEncodeJob::height: u32 +pub j2k_types::J2kHtSubbandEncodeJob::range_bits: u8 +pub j2k_types::J2kHtSubbandEncodeJob::reversible: bool +pub j2k_types::J2kHtSubbandEncodeJob::step_exponent: u16 +pub j2k_types::J2kHtSubbandEncodeJob::step_mantissa: u16 +pub j2k_types::J2kHtSubbandEncodeJob::total_bitplanes: u8 +pub j2k_types::J2kHtSubbandEncodeJob::width: u32 +pub struct j2k_types::J2kHtj2kTileEncodeJob<'a> +pub j2k_types::J2kHtj2kTileEncodeJob::bit_depth: u8 +pub j2k_types::J2kHtj2kTileEncodeJob::code_block_height: u32 +pub j2k_types::J2kHtj2kTileEncodeJob::code_block_width: u32 +pub j2k_types::J2kHtj2kTileEncodeJob::component_sampling: &'a [(u8, u8)] +pub j2k_types::J2kHtj2kTileEncodeJob::guard_bits: u8 +pub j2k_types::J2kHtj2kTileEncodeJob::height: u32 +pub j2k_types::J2kHtj2kTileEncodeJob::num_components: u8 +pub j2k_types::J2kHtj2kTileEncodeJob::num_decomposition_levels: u8 +pub j2k_types::J2kHtj2kTileEncodeJob::pixels: &'a [u8] +pub j2k_types::J2kHtj2kTileEncodeJob::progression_order: j2k_types::J2kPacketizationProgressionOrder +pub j2k_types::J2kHtj2kTileEncodeJob::quantization_steps: &'a [(u16, u16)] +pub j2k_types::J2kHtj2kTileEncodeJob::reversible: bool +pub j2k_types::J2kHtj2kTileEncodeJob::signed: bool +pub j2k_types::J2kHtj2kTileEncodeJob::use_mct: bool +pub j2k_types::J2kHtj2kTileEncodeJob::width: u32 +pub struct j2k_types::J2kPacketizationCodeBlock<'a> +pub j2k_types::J2kPacketizationCodeBlock::block_coding_mode: j2k_types::J2kPacketizationBlockCodingMode +pub j2k_types::J2kPacketizationCodeBlock::data: &'a [u8] +pub j2k_types::J2kPacketizationCodeBlock::ht_cleanup_length: u32 +pub j2k_types::J2kPacketizationCodeBlock::ht_refinement_length: u32 +pub j2k_types::J2kPacketizationCodeBlock::l_block: u32 +pub j2k_types::J2kPacketizationCodeBlock::num_coding_passes: u8 +pub j2k_types::J2kPacketizationCodeBlock::num_zero_bitplanes: u8 +pub j2k_types::J2kPacketizationCodeBlock::previously_included: bool +pub struct j2k_types::J2kPacketizationEncodeJob<'a> +pub j2k_types::J2kPacketizationEncodeJob::code_block_count: u32 +pub j2k_types::J2kPacketizationEncodeJob::num_components: u8 +pub j2k_types::J2kPacketizationEncodeJob::num_layers: u8 +pub j2k_types::J2kPacketizationEncodeJob::packet_descriptors: &'a [j2k_types::J2kPacketizationPacketDescriptor] +pub j2k_types::J2kPacketizationEncodeJob::progression_order: j2k_types::J2kPacketizationProgressionOrder +pub j2k_types::J2kPacketizationEncodeJob::resolution_count: u32 +pub j2k_types::J2kPacketizationEncodeJob::resolutions: &'a [j2k_types::J2kPacketizationResolution<'a>] +pub struct j2k_types::J2kPacketizationPacketDescriptor +pub j2k_types::J2kPacketizationPacketDescriptor::component: u8 +pub j2k_types::J2kPacketizationPacketDescriptor::layer: u8 +pub j2k_types::J2kPacketizationPacketDescriptor::packet_index: u32 +pub j2k_types::J2kPacketizationPacketDescriptor::precinct: u64 +pub j2k_types::J2kPacketizationPacketDescriptor::resolution: u32 +pub j2k_types::J2kPacketizationPacketDescriptor::state_index: u32 +pub struct j2k_types::J2kPacketizationResolution<'a> +pub j2k_types::J2kPacketizationResolution::subbands: alloc::vec::Vec> +pub struct j2k_types::J2kPacketizationSubband<'a> +pub j2k_types::J2kPacketizationSubband::code_blocks: alloc::vec::Vec> +pub j2k_types::J2kPacketizationSubband::num_cbs_x: u32 +pub j2k_types::J2kPacketizationSubband::num_cbs_y: u32 +pub struct j2k_types::J2kQuantizeSubbandJob<'a> +pub j2k_types::J2kQuantizeSubbandJob::coefficients: &'a [f32] +pub j2k_types::J2kQuantizeSubbandJob::range_bits: u8 +pub j2k_types::J2kQuantizeSubbandJob::reversible: bool +pub j2k_types::J2kQuantizeSubbandJob::step_exponent: u16 +pub j2k_types::J2kQuantizeSubbandJob::step_mantissa: u16 +pub struct j2k_types::J2kTier1CodeBlockEncodeJob<'a> +pub j2k_types::J2kTier1CodeBlockEncodeJob::coefficients: &'a [i32] +pub j2k_types::J2kTier1CodeBlockEncodeJob::height: u32 +pub j2k_types::J2kTier1CodeBlockEncodeJob::style: j2k_types::J2kCodeBlockStyle +pub j2k_types::J2kTier1CodeBlockEncodeJob::sub_band_type: j2k_types::J2kSubBandType +pub j2k_types::J2kTier1CodeBlockEncodeJob::total_bitplanes: u8 +pub j2k_types::J2kTier1CodeBlockEncodeJob::width: u32 +pub struct j2k_types::PrecomputedHtj2k53Component +pub j2k_types::PrecomputedHtj2k53Component::dwt: j2k_types::J2kForwardDwt53Output +pub j2k_types::PrecomputedHtj2k53Component::x_rsiz: u8 +pub j2k_types::PrecomputedHtj2k53Component::y_rsiz: u8 +pub struct j2k_types::PrecomputedHtj2k53Image +pub j2k_types::PrecomputedHtj2k53Image::bit_depth: u8 +pub j2k_types::PrecomputedHtj2k53Image::components: alloc::vec::Vec +pub j2k_types::PrecomputedHtj2k53Image::height: u32 +pub j2k_types::PrecomputedHtj2k53Image::signed: bool +pub j2k_types::PrecomputedHtj2k53Image::width: u32 +pub struct j2k_types::PrecomputedHtj2k97Component +pub j2k_types::PrecomputedHtj2k97Component::dwt: j2k_types::J2kForwardDwt97Output +pub j2k_types::PrecomputedHtj2k97Component::x_rsiz: u8 +pub j2k_types::PrecomputedHtj2k97Component::y_rsiz: u8 +pub struct j2k_types::PrecomputedHtj2k97Image +pub j2k_types::PrecomputedHtj2k97Image::bit_depth: u8 +pub j2k_types::PrecomputedHtj2k97Image::components: alloc::vec::Vec +pub j2k_types::PrecomputedHtj2k97Image::height: u32 +pub j2k_types::PrecomputedHtj2k97Image::signed: bool +pub j2k_types::PrecomputedHtj2k97Image::width: u32 +pub struct j2k_types::PreencodedHtj2k97CodeBlock +pub j2k_types::PreencodedHtj2k97CodeBlock::encoded: j2k_types::EncodedHtJ2kCodeBlock +pub j2k_types::PreencodedHtj2k97CodeBlock::height: u32 +pub j2k_types::PreencodedHtj2k97CodeBlock::width: u32 +pub struct j2k_types::PreencodedHtj2k97CompactCodeBlock +pub j2k_types::PreencodedHtj2k97CompactCodeBlock::cleanup_length: u32 +pub j2k_types::PreencodedHtj2k97CompactCodeBlock::height: u32 +pub j2k_types::PreencodedHtj2k97CompactCodeBlock::num_coding_passes: u8 +pub j2k_types::PreencodedHtj2k97CompactCodeBlock::num_zero_bitplanes: u8 +pub j2k_types::PreencodedHtj2k97CompactCodeBlock::payload_range: core::ops::range::Range +pub j2k_types::PreencodedHtj2k97CompactCodeBlock::refinement_length: u32 +pub j2k_types::PreencodedHtj2k97CompactCodeBlock::width: u32 +pub struct j2k_types::PreencodedHtj2k97CompactComponent +pub j2k_types::PreencodedHtj2k97CompactComponent::resolutions: alloc::vec::Vec +pub j2k_types::PreencodedHtj2k97CompactComponent::x_rsiz: u8 +pub j2k_types::PreencodedHtj2k97CompactComponent::y_rsiz: u8 +pub struct j2k_types::PreencodedHtj2k97CompactImage +pub j2k_types::PreencodedHtj2k97CompactImage::bit_depth: u8 +pub j2k_types::PreencodedHtj2k97CompactImage::components: alloc::vec::Vec +pub j2k_types::PreencodedHtj2k97CompactImage::height: u32 +pub j2k_types::PreencodedHtj2k97CompactImage::payload: alloc::vec::Vec +pub j2k_types::PreencodedHtj2k97CompactImage::signed: bool +pub j2k_types::PreencodedHtj2k97CompactImage::width: u32 +pub struct j2k_types::PreencodedHtj2k97CompactResolution +pub j2k_types::PreencodedHtj2k97CompactResolution::subbands: alloc::vec::Vec +pub struct j2k_types::PreencodedHtj2k97CompactSubband +pub j2k_types::PreencodedHtj2k97CompactSubband::code_blocks: alloc::vec::Vec +pub j2k_types::PreencodedHtj2k97CompactSubband::num_cbs_x: u32 +pub j2k_types::PreencodedHtj2k97CompactSubband::num_cbs_y: u32 +pub j2k_types::PreencodedHtj2k97CompactSubband::sub_band_type: j2k_types::J2kSubBandType +pub j2k_types::PreencodedHtj2k97CompactSubband::total_bitplanes: u8 +pub struct j2k_types::PreencodedHtj2k97Component +pub j2k_types::PreencodedHtj2k97Component::resolutions: alloc::vec::Vec +pub j2k_types::PreencodedHtj2k97Component::x_rsiz: u8 +pub j2k_types::PreencodedHtj2k97Component::y_rsiz: u8 +pub struct j2k_types::PreencodedHtj2k97Image +pub j2k_types::PreencodedHtj2k97Image::bit_depth: u8 +pub j2k_types::PreencodedHtj2k97Image::components: alloc::vec::Vec +pub j2k_types::PreencodedHtj2k97Image::height: u32 +pub j2k_types::PreencodedHtj2k97Image::signed: bool +pub j2k_types::PreencodedHtj2k97Image::width: u32 +pub struct j2k_types::PreencodedHtj2k97Resolution +pub j2k_types::PreencodedHtj2k97Resolution::subbands: alloc::vec::Vec +pub struct j2k_types::PreencodedHtj2k97Subband +pub j2k_types::PreencodedHtj2k97Subband::code_blocks: alloc::vec::Vec +pub j2k_types::PreencodedHtj2k97Subband::num_cbs_x: u32 +pub j2k_types::PreencodedHtj2k97Subband::num_cbs_y: u32 +pub j2k_types::PreencodedHtj2k97Subband::sub_band_type: j2k_types::J2kSubBandType +pub j2k_types::PreencodedHtj2k97Subband::total_bitplanes: u8 +pub struct j2k_types::PrequantizedHtj2k97CodeBlock +pub j2k_types::PrequantizedHtj2k97CodeBlock::coefficients: alloc::vec::Vec +pub j2k_types::PrequantizedHtj2k97CodeBlock::height: u32 +pub j2k_types::PrequantizedHtj2k97CodeBlock::width: u32 +pub struct j2k_types::PrequantizedHtj2k97Component +pub j2k_types::PrequantizedHtj2k97Component::resolutions: alloc::vec::Vec +pub j2k_types::PrequantizedHtj2k97Component::x_rsiz: u8 +pub j2k_types::PrequantizedHtj2k97Component::y_rsiz: u8 +pub struct j2k_types::PrequantizedHtj2k97Image +pub j2k_types::PrequantizedHtj2k97Image::bit_depth: u8 +pub j2k_types::PrequantizedHtj2k97Image::components: alloc::vec::Vec +pub j2k_types::PrequantizedHtj2k97Image::height: u32 +pub j2k_types::PrequantizedHtj2k97Image::signed: bool +pub j2k_types::PrequantizedHtj2k97Image::width: u32 +pub struct j2k_types::PrequantizedHtj2k97Resolution +pub j2k_types::PrequantizedHtj2k97Resolution::subbands: alloc::vec::Vec +pub struct j2k_types::PrequantizedHtj2k97Subband +pub j2k_types::PrequantizedHtj2k97Subband::code_blocks: alloc::vec::Vec +pub j2k_types::PrequantizedHtj2k97Subband::num_cbs_x: u32 +pub j2k_types::PrequantizedHtj2k97Subband::num_cbs_y: u32 +pub j2k_types::PrequantizedHtj2k97Subband::sub_band_type: j2k_types::J2kSubBandType +pub j2k_types::PrequantizedHtj2k97Subband::total_bitplanes: u8 +``` + +## `j2k-cuda-runtime` + +```text +pub mod j2k_cuda_runtime +#[non_exhaustive] pub enum j2k_cuda_runtime::CudaError +pub j2k_cuda_runtime::CudaError::Driver +pub j2k_cuda_runtime::CudaError::Driver::code: std::os::raw::c_int +pub j2k_cuda_runtime::CudaError::Driver::name: alloc::string::String +pub j2k_cuda_runtime::CudaError::Driver::operation: &'static str +pub j2k_cuda_runtime::CudaError::ImageTooLarge +pub j2k_cuda_runtime::CudaError::ImageTooLarge::channels: usize +pub j2k_cuda_runtime::CudaError::ImageTooLarge::height: u32 +pub j2k_cuda_runtime::CudaError::ImageTooLarge::width: u32 +pub j2k_cuda_runtime::CudaError::InvalidArgument +pub j2k_cuda_runtime::CudaError::InvalidArgument::message: alloc::string::String +pub j2k_cuda_runtime::CudaError::KernelStatus +pub j2k_cuda_runtime::CudaError::KernelStatus::code: u32 +pub j2k_cuda_runtime::CudaError::KernelStatus::detail: u32 +pub j2k_cuda_runtime::CudaError::KernelStatus::kernel: &'static str +pub j2k_cuda_runtime::CudaError::LengthNotElementAligned +pub j2k_cuda_runtime::CudaError::LengthNotElementAligned::bytes: usize +pub j2k_cuda_runtime::CudaError::LengthNotElementAligned::element_size: usize +pub j2k_cuda_runtime::CudaError::LengthTooLarge +pub j2k_cuda_runtime::CudaError::LengthTooLarge::len: usize +pub j2k_cuda_runtime::CudaError::OutputTooSmall +pub j2k_cuda_runtime::CudaError::OutputTooSmall::have: usize +pub j2k_cuda_runtime::CudaError::OutputTooSmall::required: usize +pub j2k_cuda_runtime::CudaError::StatePoisoned +pub j2k_cuda_runtime::CudaError::StatePoisoned::message: alloc::string::String +pub j2k_cuda_runtime::CudaError::Unavailable +pub j2k_cuda_runtime::CudaError::Unavailable::message: alloc::string::String +impl j2k_cuda_runtime::CudaError +pub fn j2k_cuda_runtime::CudaError::is_unavailable(&self) -> bool +pub enum j2k_cuda_runtime::CudaJpegRgb8Sampling +pub j2k_cuda_runtime::CudaJpegRgb8Sampling::Fast420 +pub j2k_cuda_runtime::CudaJpegRgb8Sampling::Fast422 +pub j2k_cuda_runtime::CudaJpegRgb8Sampling::Fast444 +#[non_exhaustive] pub enum j2k_cuda_runtime::CudaKernelName +pub j2k_cuda_runtime::CudaKernelName::CopyU8 +pub j2k_cuda_runtime::CudaKernelName::Htj2kCompactCodeblocks +pub j2k_cuda_runtime::CudaKernelName::Htj2kDecodeCodeblocks +pub j2k_cuda_runtime::CudaKernelName::Htj2kDecodeCodeblocksMultiCleanupDequantize +pub j2k_cuda_runtime::CudaKernelName::Htj2kEncodeCodeblock +pub j2k_cuda_runtime::CudaKernelName::Htj2kEncodeCodeblocks +pub j2k_cuda_runtime::CudaKernelName::Htj2kEncodeCodeblocksMultiInput +pub j2k_cuda_runtime::CudaKernelName::Htj2kEncodeCodeblocksMultiInputCleanup +pub j2k_cuda_runtime::CudaKernelName::Htj2kEncodeCodeblocksMultiInputCleanup64 +pub j2k_cuda_runtime::CudaKernelName::Htj2kPacketizeCleanup +pub j2k_cuda_runtime::CudaKernelName::J2kDeinterleaveToF32 +pub j2k_cuda_runtime::CudaKernelName::J2kDequantizeHtj2kCleanupJobsMulti +pub j2k_cuda_runtime::CudaKernelName::J2kDequantizeHtj2kCodeblocks +pub j2k_cuda_runtime::CudaKernelName::J2kDequantizeHtj2kCodeblocksMulti +pub j2k_cuda_runtime::CudaKernelName::J2kForwardDwt53Horizontal +pub j2k_cuda_runtime::CudaKernelName::J2kForwardDwt53Vertical +pub j2k_cuda_runtime::CudaKernelName::J2kForwardDwt97Horizontal +pub j2k_cuda_runtime::CudaKernelName::J2kForwardDwt97Vertical +pub j2k_cuda_runtime::CudaKernelName::J2kForwardIct +pub j2k_cuda_runtime::CudaKernelName::J2kForwardRct +pub j2k_cuda_runtime::CudaKernelName::J2kIdwtHorizontal +pub j2k_cuda_runtime::CudaKernelName::J2kIdwtHorizontal53 +pub j2k_cuda_runtime::CudaKernelName::J2kIdwtHorizontal97 +pub j2k_cuda_runtime::CudaKernelName::J2kIdwtInterleave +pub j2k_cuda_runtime::CudaKernelName::J2kIdwtInterleaveHorizontal53Multi +pub j2k_cuda_runtime::CudaKernelName::J2kIdwtInterleaveHorizontal97Multi +pub j2k_cuda_runtime::CudaKernelName::J2kIdwtVertical +pub j2k_cuda_runtime::CudaKernelName::J2kIdwtVertical53 +pub j2k_cuda_runtime::CudaKernelName::J2kIdwtVertical53Multi +pub j2k_cuda_runtime::CudaKernelName::J2kIdwtVertical97 +pub j2k_cuda_runtime::CudaKernelName::J2kIdwtVertical97Multi +pub j2k_cuda_runtime::CudaKernelName::J2kIdwtVertical97MultiCols4 +pub j2k_cuda_runtime::CudaKernelName::J2kInverseDwtSingle +pub j2k_cuda_runtime::CudaKernelName::J2kInverseMct +pub j2k_cuda_runtime::CudaKernelName::J2kQuantizeSubband +pub j2k_cuda_runtime::CudaKernelName::J2kQuantizeSubbandStrided +pub j2k_cuda_runtime::CudaKernelName::J2kStoreGray16 +pub j2k_cuda_runtime::CudaKernelName::J2kStoreGray8 +pub j2k_cuda_runtime::CudaKernelName::J2kStoreRgb16 +pub j2k_cuda_runtime::CudaKernelName::J2kStoreRgb16Mct +pub j2k_cuda_runtime::CudaKernelName::J2kStoreRgb8 +pub j2k_cuda_runtime::CudaKernelName::J2kStoreRgb8Mct +pub j2k_cuda_runtime::CudaKernelName::J2kStoreRgb8MctBatch +pub struct j2k_cuda_runtime::CudaBufferPool +impl j2k_cuda_runtime::CudaBufferPool +pub fn j2k_cuda_runtime::CudaBufferPool::cached_count(&self) -> core::result::Result +pub fn j2k_cuda_runtime::CudaBufferPool::new(j2k_cuda_runtime::CudaContext) -> Self +pub fn j2k_cuda_runtime::CudaBufferPool::recycle(&self, j2k_cuda_runtime::CudaDeviceBuffer) -> core::result::Result<(), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaBufferPool::take(&self, usize) -> core::result::Result +pub fn j2k_cuda_runtime::CudaBufferPool::take_with_trace(&self, usize) -> core::result::Result<(j2k_cuda_runtime::CudaPooledDeviceBuffer, j2k_cuda_runtime::CudaBufferPoolTakeTrace), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaBufferPool::upload(&self, &[u8]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaBufferPool::upload_f32(&self, &[f32]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaBufferPool::upload_f32_pinned(&self, &[f32]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaBufferPool::upload_i16(&self, &[i16]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaBufferPool::upload_i16_pinned(&self, &[i16]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaBufferPool::upload_pinned(&self, &[u8]) -> core::result::Result +pub struct j2k_cuda_runtime::CudaBufferPoolTakeTrace +pub j2k_cuda_runtime::CudaBufferPoolTakeTrace::allocation_byte_len: usize +pub j2k_cuda_runtime::CudaBufferPoolTakeTrace::free_count_before: usize +pub j2k_cuda_runtime::CudaBufferPoolTakeTrace::requested_len: usize +pub j2k_cuda_runtime::CudaBufferPoolTakeTrace::reused: bool +pub j2k_cuda_runtime::CudaBufferPoolTakeTrace::scanned_count: usize +pub struct j2k_cuda_runtime::CudaContext +impl j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::allocate(&self, usize) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::best_fit_buffer_pool(&self) -> j2k_cuda_runtime::CudaBufferPool +pub fn j2k_cuda_runtime::CudaContext::buffer_pool(&self) -> j2k_cuda_runtime::CudaBufferPool +pub fn j2k_cuda_runtime::CudaContext::pinned_host_buffer(&self, usize) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::upload(&self, &[u8]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::upload_f32(&self, &[f32]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::upload_f32_pinned(&self, &[f32]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::upload_i32_pinned(&self, &[i32]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::upload_pinned(&self, &[u8]) -> core::result::Result +impl j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::allocate_htj2k_codeblock_coefficients_with_pool(&self, &[j2k_cuda_runtime::CudaHtj2kCodeBlockJob], usize, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks(&self, &[u8], &[j2k_cuda_runtime::CudaHtj2kCodeBlockJob], j2k_cuda_runtime::CudaHtj2kDecodeTables<'_>, usize) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_cleanup_dequantize_multi_with_resources_and_pool_timed(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCleanupTarget<'_>], &j2k_cuda_runtime::CudaBufferPool, bool) -> core::result::Result<(j2k_cuda_runtime::CudaExecutionStats, j2k_cuda_runtime::CudaHtj2kDecodeStageTimings), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_cleanup_multi_enqueue_with_resources_and_pool(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCleanupTarget<'_>], &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_cleanup_multi_with_resources_and_pool(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCleanupTarget<'_>], &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_cleanup_multi_with_resources_and_pool_timed(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCleanupTarget<'_>], &j2k_cuda_runtime::CudaBufferPool, bool) -> core::result::Result<(j2k_cuda_runtime::CudaExecutionStats, j2k_cuda_runtime::CudaHtj2kDecodeStageTimings), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_cleanup_with_resources_and_pool(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCodeBlockJob], usize, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_cleanup_with_resources_untimed_and_pool(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCodeBlockJob], usize, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_untimed(&self, &[u8], &[j2k_cuda_runtime::CudaHtj2kCodeBlockJob], j2k_cuda_runtime::CudaHtj2kDecodeTables<'_>, usize) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_with_resources(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCodeBlockJob], usize) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_with_resources_and_pool(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCodeBlockJob], usize, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_with_resources_untimed(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCodeBlockJob], usize) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_htj2k_codeblocks_with_resources_untimed_and_pool(&self, &j2k_cuda_runtime::CudaHtj2kDecodeResources, &[j2k_cuda_runtime::CudaHtj2kCodeBlockJob], usize, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_dequantize_htj2k_codeblocks_multi_device(&self, &[j2k_cuda_runtime::CudaHtj2kDequantizeTarget<'_>]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_dequantize_htj2k_codeblocks_multi_device_untimed_with_pool(&self, &[j2k_cuda_runtime::CudaHtj2kDequantizeTarget<'_>], &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_dequantize_htj2k_codeblocks_multi_device_with_pool(&self, &[j2k_cuda_runtime::CudaHtj2kDequantizeTarget<'_>], &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::upload_htj2k_decode_resources(&self, &[u8], j2k_cuda_runtime::CudaHtj2kDecodeTables<'_>) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::upload_htj2k_decode_resources_with_tables(&self, &[u8], &j2k_cuda_runtime::CudaHtj2kDecodeTableResources) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::upload_htj2k_decode_resources_with_tables_and_pool(&self, &[u8], &j2k_cuda_runtime::CudaHtj2kDecodeTableResources, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::upload_htj2k_decode_table_resources(&self, j2k_cuda_runtime::CudaHtj2kDecodeTables<'_>) -> core::result::Result +impl j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::copy_device_to_device_with_kernel(&self, &j2k_cuda_runtime::CudaDeviceBuffer) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::copy_with_cuda_oxide_kernel(&self, &[u8]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::copy_with_kernel(&self, &[u8]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::create_event(&self) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::create_stream(&self) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::preload_kernel_module(&self, j2k_cuda_runtime::CudaKernelName) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::synchronize(&self) -> core::result::Result<(), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::time_default_stream_named_us(&self, &str, impl core::ops::function::FnOnce() -> core::result::Result) -> core::result::Result<(T, u128), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::time_default_stream_named_us_if(&self, bool, &str, impl core::ops::function::FnOnce() -> core::result::Result) -> core::result::Result<(T, u128), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::time_default_stream_us(&self, impl core::ops::function::FnOnce() -> core::result::Result) -> core::result::Result<(T, u128), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::with_nvtx_range(&self, &str, impl core::ops::function::FnOnce() -> core::result::Result) -> core::result::Result +impl j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::decode_jpeg_420_rgb8_owned(&self, &j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan<'_>) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_jpeg_420_rgb8_owned_into(&self, &j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan<'_>, &j2k_cuda_runtime::CudaDeviceBuffer, usize) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_jpeg_rgb8_owned(&self, &j2k_cuda_runtime::CudaJpegRgb8DecodePlan<'_>) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::decode_jpeg_rgb8_owned_into(&self, &j2k_cuda_runtime::CudaJpegRgb8DecodePlan<'_>, &j2k_cuda_runtime::CudaDeviceBuffer, usize) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::diagnose_jpeg_420_entropy_self_sync(&self, &j2k_cuda_runtime::CudaJpegChunkedEntropyPlan<'_>) -> core::result::Result +impl j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblock(&self, &[i32], u32, u32, u8, j2k_cuda_runtime::CudaHtj2kEncodeTables<'_>) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblock_regions_resident(&self, &j2k_cuda_runtime::CudaDeviceBuffer, usize, &[j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob], j2k_cuda_runtime::CudaHtj2kEncodeTables<'_>) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblock_regions_resident_with_resources(&self, &j2k_cuda_runtime::CudaDeviceBuffer, usize, &[j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob], &j2k_cuda_runtime::CudaHtj2kEncodeResources) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblock_regions_resident_with_resources_and_pool(&self, &j2k_cuda_runtime::CudaDeviceBuffer, usize, &[j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob], &j2k_cuda_runtime::CudaHtj2kEncodeResources, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblock_with_resources(&self, &[i32], u32, u32, u8, &j2k_cuda_runtime::CudaHtj2kEncodeResources) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblocks(&self, &[i32], &[j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockJob], j2k_cuda_runtime::CudaHtj2kEncodeTables<'_>) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblocks_multi_resident_compact_with_resources_and_pool(&self, &[j2k_cuda_runtime::CudaHtj2kEncodeResidentTarget<'_>], &j2k_cuda_runtime::CudaHtj2kEncodeResources, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblocks_multi_resident_with_resources_and_pool(&self, &[j2k_cuda_runtime::CudaHtj2kEncodeResidentTarget<'_>], &j2k_cuda_runtime::CudaHtj2kEncodeResources, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblocks_resident(&self, &j2k_cuda_runtime::CudaDeviceBuffer, usize, &[j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockJob], j2k_cuda_runtime::CudaHtj2kEncodeTables<'_>) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblocks_resident_with_resources(&self, &j2k_cuda_runtime::CudaDeviceBuffer, usize, &[j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockJob], &j2k_cuda_runtime::CudaHtj2kEncodeResources) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblocks_resident_with_resources_and_pool(&self, &j2k_cuda_runtime::CudaDeviceBuffer, usize, &[j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockJob], &j2k_cuda_runtime::CudaHtj2kEncodeResources, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::encode_htj2k_codeblocks_with_resources(&self, &[i32], &[j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockJob], &j2k_cuda_runtime::CudaHtj2kEncodeResources) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::upload_htj2k_encode_resources(&self, j2k_cuda_runtime::CudaHtj2kEncodeTables<'_>) -> core::result::Result +impl j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::j2k_deinterleave_strided_to_f32_resident(&self, j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels<'_>) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_deinterleave_to_f32(&self, &[u8], usize, u8, u8, bool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_deinterleave_to_f32_resident(&self, &[u8], usize, u8, u8, bool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_forward_dwt53(&self, &[f32], u32, u32, u8) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_forward_dwt53_resident_component(&self, &j2k_cuda_runtime::CudaJ2kResidentComponents, u8, u32, u32, u8) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_forward_dwt97(&self, &[f32], u32, u32, u8) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_forward_dwt97_resident_component(&self, &j2k_cuda_runtime::CudaJ2kResidentComponents, u8, u32, u32, u8) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_forward_ict(&self, &mut [f32], &mut [f32], &mut [f32]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_forward_ict_resident(&self, &mut j2k_cuda_runtime::CudaJ2kResidentComponents) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_forward_rct(&self, &mut [f32], &mut [f32], &mut [f32]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_forward_rct_resident(&self, &mut j2k_cuda_runtime::CudaJ2kResidentComponents) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_quantize_subband(&self, &[f32], j2k_cuda_runtime::CudaJ2kQuantizeJob) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_quantize_subband_region_resident(&self, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kQuantizeSubbandRegionJob) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_quantize_subband_resident(&self, &j2k_cuda_runtime::CudaDeviceBuffer, usize, j2k_cuda_runtime::CudaJ2kQuantizeJob) -> core::result::Result +impl j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::j2k_dequantize_queued_htj2k_cleanup_with_pool(&self, &j2k_cuda_runtime::CudaQueuedHtj2kCleanup) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::system_default() -> core::result::Result +impl j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::j2k_inverse_dwt_batch_device_enqueue_with_pool(&self, &[j2k_cuda_runtime::CudaJ2kIdwtTarget<'_>], &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_inverse_dwt_batch_device_untimed_with_pool(&self, &[j2k_cuda_runtime::CudaJ2kIdwtTarget<'_>], &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_inverse_dwt_batch_device_with_pool(&self, &[j2k_cuda_runtime::CudaJ2kIdwtTarget<'_>], &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_inverse_dwt_batch_sequence_enqueue_with_pool(&self, &[&[j2k_cuda_runtime::CudaJ2kIdwtTarget<'_>]], &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_inverse_dwt_single_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kIdwtJob) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_inverse_dwt_single_device_untimed(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kIdwtJob) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_inverse_dwt_single_device_untimed_with_pool(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kIdwtJob, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_inverse_dwt_single_device_with_pool(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kIdwtJob, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_inverse_mct_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kInverseMctJob) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_gray16_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kStoreGray16Job) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_gray8_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kStoreGray8Job) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb16_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kStoreRgb16Job) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb16_mct_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kStoreRgb16MctJob) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kStoreRgb8Job) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_mct_batch_contiguous_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget<'_>]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_mct_batch_device(&self, &[j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget<'_>]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::j2k_store_rgb8_mct_device(&self, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, &j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaJ2kStoreRgb8MctJob) -> core::result::Result +impl j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::j2k_transcode_dwt97(&self, &[f32], usize, usize, usize, usize) -> core::result::Result +impl j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::j2k_transcode_dwt97_batch(&self, &[f32], usize, usize, usize, usize, usize) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaDwt97BatchStageTimings), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::j2k_transcode_dwt97_batch_with_pool(&self, &[f32], usize, usize, usize, usize, usize, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result<(alloc::vec::Vec, j2k_cuda_runtime::CudaDwt97BatchStageTimings), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::j2k_transcode_htj2k97_codeblock_batch(&self, &[f32], usize, usize, usize, usize, usize, j2k_cuda_runtime::CudaHtj2k97QuantizeParams) -> core::result::Result<(j2k_cuda_runtime::CudaHtj2k97CodeblockBands, j2k_cuda_runtime::CudaDwt97BatchStageTimings), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::j2k_transcode_htj2k97_codeblock_batch_resident(&self, &[f32], usize, usize, usize, usize, usize, j2k_cuda_runtime::CudaHtj2k97QuantizeParams) -> core::result::Result<(j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands, j2k_cuda_runtime::CudaDwt97BatchStageTimings), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::j2k_transcode_htj2k97_codeblock_batch_resident_with_pool(&self, &[f32], usize, usize, usize, usize, usize, j2k_cuda_runtime::CudaHtj2k97QuantizeParams, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result<(j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands, j2k_cuda_runtime::CudaDwt97BatchStageTimings), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::j2k_transcode_htj2k97_codeblock_batch_with_pool(&self, &[f32], usize, usize, usize, usize, usize, j2k_cuda_runtime::CudaHtj2k97QuantizeParams, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result<(j2k_cuda_runtime::CudaHtj2k97CodeblockBands, j2k_cuda_runtime::CudaDwt97BatchStageTimings), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaContext::j2k_transcode_htj2k97_codeblock_i16_batch_resident_with_pool(&self, &[i16], usize, usize, usize, usize, usize, j2k_cuda_runtime::CudaHtj2k97QuantizeParams, &j2k_cuda_runtime::CudaBufferPool) -> core::result::Result<(j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands, j2k_cuda_runtime::CudaDwt97BatchStageTimings), j2k_cuda_runtime::CudaError> +impl j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::j2k_transcode_reversible_dwt53(&self, &[i16], usize, usize, usize, usize) -> core::result::Result +impl j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::packetize_htj2k_cleanup_packets(&self, &[u8], &[j2k_cuda_runtime::CudaHtj2kPacketizationPacket], &[j2k_cuda_runtime::CudaHtj2kPacketizationSubband], &[j2k_cuda_runtime::CudaHtj2kPacketizationBlock]) -> core::result::Result +pub fn j2k_cuda_runtime::CudaContext::packetize_htj2k_cleanup_packets_with_tag_state(&self, &[u8], &[j2k_cuda_runtime::CudaHtj2kPacketizationPacket], &[j2k_cuda_runtime::CudaHtj2kPacketizationSubband], &[j2k_cuda_runtime::CudaHtj2kPacketizationBlock], &[j2k_cuda_runtime::CudaHtj2kPacketizationSubbandTagState], &[j2k_cuda_runtime::CudaHtj2kPacketizationTagNodeState]) -> core::result::Result +impl core::fmt::Debug for j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaContext::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct j2k_cuda_runtime::CudaDeviceBuffer +impl j2k_cuda_runtime::CudaDeviceBuffer +pub fn j2k_cuda_runtime::CudaDeviceBuffer::byte_len(&self) -> usize +pub fn j2k_cuda_runtime::CudaDeviceBuffer::context(&self) -> j2k_cuda_runtime::CudaContext +pub fn j2k_cuda_runtime::CudaDeviceBuffer::copy_range_to_host(&self, usize, &mut [u8]) -> core::result::Result<(), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaDeviceBuffer::copy_range_to_host_uninit(&self, usize, &mut [core::mem::maybe_uninit::MaybeUninit]) -> core::result::Result<(), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaDeviceBuffer::copy_to_host(&self, &mut [u8]) -> core::result::Result<(), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaDeviceBuffer::device_ptr(&self) -> u64 +pub fn j2k_cuda_runtime::CudaDeviceBuffer::typed_view(&self) -> core::result::Result, j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaDeviceBuffer::typed_view_mut(&mut self) -> core::result::Result, j2k_cuda_runtime::CudaError> +impl core::ops::drop::Drop for j2k_cuda_runtime::CudaDeviceBuffer +pub fn j2k_cuda_runtime::CudaDeviceBuffer::drop(&mut self) +pub struct j2k_cuda_runtime::CudaDeviceBufferRange +pub j2k_cuda_runtime::CudaDeviceBufferRange::len: usize +pub j2k_cuda_runtime::CudaDeviceBufferRange::offset: usize +pub struct j2k_cuda_runtime::CudaDeviceBufferView<'a, T> +impl j2k_cuda_runtime::CudaDeviceBufferView<'_, T> +pub fn j2k_cuda_runtime::CudaDeviceBufferView<'_, T>::device_ptr(&self) -> u64 +pub fn j2k_cuda_runtime::CudaDeviceBufferView<'_, T>::is_empty(&self) -> bool +pub fn j2k_cuda_runtime::CudaDeviceBufferView<'_, T>::len(&self) -> usize +pub struct j2k_cuda_runtime::CudaDeviceBufferViewMut<'a, T> +impl j2k_cuda_runtime::CudaDeviceBufferViewMut<'_, T> +pub fn j2k_cuda_runtime::CudaDeviceBufferViewMut<'_, T>::device_ptr(&self) -> u64 +pub fn j2k_cuda_runtime::CudaDeviceBufferViewMut<'_, T>::is_empty(&self) -> bool +pub fn j2k_cuda_runtime::CudaDeviceBufferViewMut<'_, T>::len(&self) -> usize +pub struct j2k_cuda_runtime::CudaDwt53LevelShape +pub j2k_cuda_runtime::CudaDwt53LevelShape::height: u32 +pub j2k_cuda_runtime::CudaDwt53LevelShape::high_height: u32 +pub j2k_cuda_runtime::CudaDwt53LevelShape::high_width: u32 +pub j2k_cuda_runtime::CudaDwt53LevelShape::low_height: u32 +pub j2k_cuda_runtime::CudaDwt53LevelShape::low_width: u32 +pub j2k_cuda_runtime::CudaDwt53LevelShape::width: u32 +pub struct j2k_cuda_runtime::CudaDwt53Output +impl j2k_cuda_runtime::CudaDwt53Output +pub fn j2k_cuda_runtime::CudaDwt53Output::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaDwt53Output::levels(&self) -> &[j2k_cuda_runtime::CudaDwt53LevelShape] +pub fn j2k_cuda_runtime::CudaDwt53Output::ll_dimensions(&self) -> (u32, u32) +pub fn j2k_cuda_runtime::CudaDwt53Output::transformed(&self) -> &[f32] +pub struct j2k_cuda_runtime::CudaDwt97BatchStageTimings +pub j2k_cuda_runtime::CudaDwt97BatchStageTimings::column_lift_us: u128 +pub j2k_cuda_runtime::CudaDwt97BatchStageTimings::ht_codeblock_dispatches: usize +pub j2k_cuda_runtime::CudaDwt97BatchStageTimings::ht_encode_us: u128 +pub j2k_cuda_runtime::CudaDwt97BatchStageTimings::idct_row_lift_us: u128 +pub j2k_cuda_runtime::CudaDwt97BatchStageTimings::pack_upload_us: u128 +pub j2k_cuda_runtime::CudaDwt97BatchStageTimings::quantize_codeblock_us: u128 +pub j2k_cuda_runtime::CudaDwt97BatchStageTimings::readback_us: u128 +pub struct j2k_cuda_runtime::CudaDwt97Output +impl j2k_cuda_runtime::CudaDwt97Output +pub fn j2k_cuda_runtime::CudaDwt97Output::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaDwt97Output::levels(&self) -> &[j2k_cuda_runtime::CudaDwt53LevelShape] +pub fn j2k_cuda_runtime::CudaDwt97Output::ll_dimensions(&self) -> (u32, u32) +pub fn j2k_cuda_runtime::CudaDwt97Output::transformed(&self) -> &[f32] +pub struct j2k_cuda_runtime::CudaEvent +impl j2k_cuda_runtime::CudaEvent +pub fn j2k_cuda_runtime::CudaEvent::elapsed_time_us(&Self, &Self) -> core::result::Result +pub fn j2k_cuda_runtime::CudaEvent::record(&self, &j2k_cuda_runtime::CudaStream) -> core::result::Result<(), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaEvent::synchronize(&self) -> core::result::Result<(), j2k_cuda_runtime::CudaError> +impl core::marker::Send for j2k_cuda_runtime::CudaEvent +impl core::ops::drop::Drop for j2k_cuda_runtime::CudaEvent +pub fn j2k_cuda_runtime::CudaEvent::drop(&mut self) +pub struct j2k_cuda_runtime::CudaExecutionStats +impl j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaExecutionStats::copy_kernel_dispatches(self) -> usize +pub fn j2k_cuda_runtime::CudaExecutionStats::decode_kernel_dispatches(self) -> usize +pub fn j2k_cuda_runtime::CudaExecutionStats::kernel_dispatches(self) -> usize +pub fn j2k_cuda_runtime::CudaExecutionStats::used_hardware_decode(self) -> bool +pub struct j2k_cuda_runtime::CudaHtj2k97CodeblockBands +pub j2k_cuda_runtime::CudaHtj2k97CodeblockBands::hh: alloc::vec::Vec +pub j2k_cuda_runtime::CudaHtj2k97CodeblockBands::high_height: usize +pub j2k_cuda_runtime::CudaHtj2k97CodeblockBands::high_width: usize +pub j2k_cuda_runtime::CudaHtj2k97CodeblockBands::hl: alloc::vec::Vec +pub j2k_cuda_runtime::CudaHtj2k97CodeblockBands::item_count: usize +pub j2k_cuda_runtime::CudaHtj2k97CodeblockBands::lh: alloc::vec::Vec +pub j2k_cuda_runtime::CudaHtj2k97CodeblockBands::ll: alloc::vec::Vec +pub j2k_cuda_runtime::CudaHtj2k97CodeblockBands::low_height: usize +pub j2k_cuda_runtime::CudaHtj2k97CodeblockBands::low_width: usize +pub struct j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands +pub j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands::hh: j2k_cuda_runtime::CudaPooledDeviceBuffer +pub j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands::high_height: usize +pub j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands::high_width: usize +pub j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands::hl: j2k_cuda_runtime::CudaPooledDeviceBuffer +pub j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands::item_count: usize +pub j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands::lh: j2k_cuda_runtime::CudaPooledDeviceBuffer +pub j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands::ll: j2k_cuda_runtime::CudaPooledDeviceBuffer +pub j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands::low_height: usize +pub j2k_cuda_runtime::CudaHtj2k97DeviceCodeblockBands::low_width: usize +pub struct j2k_cuda_runtime::CudaHtj2k97QuantizeParams +pub j2k_cuda_runtime::CudaHtj2k97QuantizeParams::cb_height: usize +pub j2k_cuda_runtime::CudaHtj2k97QuantizeParams::cb_width: usize +pub j2k_cuda_runtime::CudaHtj2k97QuantizeParams::inv_delta_hh: f32 +pub j2k_cuda_runtime::CudaHtj2k97QuantizeParams::inv_delta_hl: f32 +pub j2k_cuda_runtime::CudaHtj2k97QuantizeParams::inv_delta_lh: f32 +pub j2k_cuda_runtime::CudaHtj2k97QuantizeParams::inv_delta_ll: f32 +pub struct j2k_cuda_runtime::CudaHtj2kCleanupTarget<'a> +pub j2k_cuda_runtime::CudaHtj2kCleanupTarget::coefficients: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaHtj2kCleanupTarget::jobs: &'a [j2k_cuda_runtime::CudaHtj2kCodeBlockJob] +pub j2k_cuda_runtime::CudaHtj2kCleanupTarget::output_words: usize +pub struct j2k_cuda_runtime::CudaHtj2kCodeBlockJob +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::cleanup_length: u32 +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::dequantization_step: f32 +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::height: u32 +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::missing_bit_planes: u8 +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::num_bitplanes: u8 +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::number_of_coding_passes: u8 +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::output_offset: u32 +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::output_stride: u32 +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::payload_len: u32 +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::payload_offset: u64 +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::refinement_length: u32 +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::stripe_causal: bool +pub j2k_cuda_runtime::CudaHtj2kCodeBlockJob::width: u32 +pub struct j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlock +impl j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlock +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlock::cleanup_length(&self) -> u32 +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlock::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlock::into_parts(self) -> (core::ops::range::Range, u32, u32, u8, u8) +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlock::num_coding_passes(&self) -> u8 +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlock::num_zero_bitplanes(&self) -> u8 +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlock::payload_range(&self) -> core::ops::range::Range +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlock::refinement_length(&self) -> u32 +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlock::stage_timings(&self) -> j2k_cuda_runtime::CudaHtj2kEncodeStageTimings +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlock::status(&self) -> j2k_cuda_runtime::CudaHtj2kEncodeStatus +pub struct j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlocks +impl j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlocks +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlocks::code_blocks(&self) -> &[j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlock] +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlocks::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlocks::into_payload_and_code_blocks(self) -> (alloc::vec::Vec, alloc::vec::Vec) +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlocks::payload(&self) -> &[u8] +pub fn j2k_cuda_runtime::CudaHtj2kCompactEncodedCodeBlocks::stage_timings(&self) -> j2k_cuda_runtime::CudaHtj2kEncodeStageTimings +pub struct j2k_cuda_runtime::CudaHtj2kDecodeOutput +impl j2k_cuda_runtime::CudaHtj2kDecodeOutput +pub fn j2k_cuda_runtime::CudaHtj2kDecodeOutput::coefficients(&self) -> &j2k_cuda_runtime::CudaDeviceBuffer +pub fn j2k_cuda_runtime::CudaHtj2kDecodeOutput::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaHtj2kDecodeOutput::into_parts(self) -> (j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaExecutionStats, alloc::vec::Vec) +pub fn j2k_cuda_runtime::CudaHtj2kDecodeOutput::stage_timings(&self) -> j2k_cuda_runtime::CudaHtj2kDecodeStageTimings +pub fn j2k_cuda_runtime::CudaHtj2kDecodeOutput::statuses(&self) -> &[j2k_cuda_runtime::CudaHtj2kStatus] +pub struct j2k_cuda_runtime::CudaHtj2kDecodeResources +pub struct j2k_cuda_runtime::CudaHtj2kDecodeStageTimings +pub j2k_cuda_runtime::CudaHtj2kDecodeStageTimings::dequant_us: u128 +pub j2k_cuda_runtime::CudaHtj2kDecodeStageTimings::ht_cleanup_us: u128 +pub j2k_cuda_runtime::CudaHtj2kDecodeStageTimings::ht_refine_us: u128 +pub j2k_cuda_runtime::CudaHtj2kDecodeStageTimings::status_d2h_us: u128 +pub struct j2k_cuda_runtime::CudaHtj2kDecodeTableResources +pub struct j2k_cuda_runtime::CudaHtj2kDecodeTables<'a> +pub j2k_cuda_runtime::CudaHtj2kDecodeTables::uvlc_table0: &'a [u16; 320] +pub j2k_cuda_runtime::CudaHtj2kDecodeTables::uvlc_table1: &'a [u16; 256] +pub j2k_cuda_runtime::CudaHtj2kDecodeTables::vlc_table0: &'a [u16; 1024] +pub j2k_cuda_runtime::CudaHtj2kDecodeTables::vlc_table1: &'a [u16; 1024] +pub struct j2k_cuda_runtime::CudaHtj2kDequantizeTarget<'a> +pub j2k_cuda_runtime::CudaHtj2kDequantizeTarget::coefficients: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaHtj2kDequantizeTarget::jobs: &'a [j2k_cuda_runtime::CudaHtj2kCodeBlockJob] +pub j2k_cuda_runtime::CudaHtj2kDequantizeTarget::output_words: usize +pub struct j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockJob +pub j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockJob::coefficient_offset: u32 +pub j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockJob::height: u32 +pub j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockJob::target_coding_passes: u8 +pub j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockJob::total_bitplanes: u8 +pub j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockJob::width: u32 +pub struct j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob +pub j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob::coefficient_offset: u32 +pub j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob::coefficient_stride: u32 +pub j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob::height: u32 +pub j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob::target_coding_passes: u8 +pub j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob::total_bitplanes: u8 +pub j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockRegionJob::width: u32 +pub struct j2k_cuda_runtime::CudaHtj2kEncodeResidentTarget<'a> +pub j2k_cuda_runtime::CudaHtj2kEncodeResidentTarget::coefficient_count: usize +pub j2k_cuda_runtime::CudaHtj2kEncodeResidentTarget::coefficients: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaHtj2kEncodeResidentTarget::jobs: &'a [j2k_cuda_runtime::CudaHtj2kEncodeCodeBlockJob] +pub struct j2k_cuda_runtime::CudaHtj2kEncodeResources +pub struct j2k_cuda_runtime::CudaHtj2kEncodeStageTimings +pub j2k_cuda_runtime::CudaHtj2kEncodeStageTimings::ht_compact_us: u128 +pub j2k_cuda_runtime::CudaHtj2kEncodeStageTimings::ht_encode_us: u128 +pub j2k_cuda_runtime::CudaHtj2kEncodeStageTimings::ht_kernel_us: u128 +pub j2k_cuda_runtime::CudaHtj2kEncodeStageTimings::ht_output_readback_us: u128 +pub j2k_cuda_runtime::CudaHtj2kEncodeStageTimings::ht_status_readback_us: u128 +#[repr(C)] pub struct j2k_cuda_runtime::CudaHtj2kEncodeStatus +pub j2k_cuda_runtime::CudaHtj2kEncodeStatus::code: u32 +pub j2k_cuda_runtime::CudaHtj2kEncodeStatus::data_len: u32 +pub j2k_cuda_runtime::CudaHtj2kEncodeStatus::detail: u32 +pub j2k_cuda_runtime::CudaHtj2kEncodeStatus::missing_bit_planes: u32 +pub j2k_cuda_runtime::CudaHtj2kEncodeStatus::number_of_coding_passes: u32 +pub j2k_cuda_runtime::CudaHtj2kEncodeStatus::reserved0: u32 +pub j2k_cuda_runtime::CudaHtj2kEncodeStatus::reserved1: u32 +pub j2k_cuda_runtime::CudaHtj2kEncodeStatus::reserved2: u32 +impl j2k_cuda_runtime::CudaHtj2kEncodeStatus +pub fn j2k_cuda_runtime::CudaHtj2kEncodeStatus::is_ok(self) -> bool +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaHtj2kEncodeStatus +pub const j2k_cuda_runtime::CudaHtj2kEncodeStatus::NAME: &'static str +pub struct j2k_cuda_runtime::CudaHtj2kEncodeTables<'a> +pub j2k_cuda_runtime::CudaHtj2kEncodeTables::uvlc_table: &'a [u8] +pub j2k_cuda_runtime::CudaHtj2kEncodeTables::vlc_table0: &'a [u16; 2048] +pub j2k_cuda_runtime::CudaHtj2kEncodeTables::vlc_table1: &'a [u16; 2048] +pub struct j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock +impl j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock::cleanup_length(&self) -> u32 +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock::data(&self) -> &[u8] +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock::into_parts(self) -> (alloc::vec::Vec, u32, u32, u8, u8) +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock::num_coding_passes(&self) -> u8 +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock::num_zero_bitplanes(&self) -> u8 +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock::refinement_length(&self) -> u32 +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock::stage_timings(&self) -> j2k_cuda_runtime::CudaHtj2kEncodeStageTimings +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock::status(&self) -> j2k_cuda_runtime::CudaHtj2kEncodeStatus +pub struct j2k_cuda_runtime::CudaHtj2kEncodedCodeBlocks +impl j2k_cuda_runtime::CudaHtj2kEncodedCodeBlocks +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlocks::code_blocks(&self) -> &[j2k_cuda_runtime::CudaHtj2kEncodedCodeBlock] +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlocks::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlocks::into_code_blocks(self) -> alloc::vec::Vec +pub fn j2k_cuda_runtime::CudaHtj2kEncodedCodeBlocks::stage_timings(&self) -> j2k_cuda_runtime::CudaHtj2kEncodeStageTimings +#[repr(C)] pub struct j2k_cuda_runtime::CudaHtj2kPacketizationBlock +pub j2k_cuda_runtime::CudaHtj2kPacketizationBlock::cleanup_length: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationBlock::data_len: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationBlock::data_offset: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationBlock::inclusion_layer: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationBlock::l_block: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationBlock::num_coding_passes: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationBlock::num_zero_bitplanes: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationBlock::previously_included: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationBlock::refinement_length: u32 +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaHtj2kPacketizationBlock +pub const j2k_cuda_runtime::CudaHtj2kPacketizationBlock::NAME: &'static str +pub struct j2k_cuda_runtime::CudaHtj2kPacketizationPacket +pub j2k_cuda_runtime::CudaHtj2kPacketizationPacket::block_count: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationPacket::block_start: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationPacket::layer: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationPacket::output_capacity: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationPacket::subband_count: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationPacket::subband_start: u32 +pub struct j2k_cuda_runtime::CudaHtj2kPacketizationStageTimings +pub j2k_cuda_runtime::CudaHtj2kPacketizationStageTimings::packetize_us: u128 +#[repr(C)] pub struct j2k_cuda_runtime::CudaHtj2kPacketizationStatus +pub j2k_cuda_runtime::CudaHtj2kPacketizationStatus::code: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationStatus::detail: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationStatus::output_len: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationStatus::reserved0: u32 +impl j2k_cuda_runtime::CudaHtj2kPacketizationStatus +pub fn j2k_cuda_runtime::CudaHtj2kPacketizationStatus::is_ok(self) -> bool +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaHtj2kPacketizationStatus +pub const j2k_cuda_runtime::CudaHtj2kPacketizationStatus::NAME: &'static str +#[repr(C)] pub struct j2k_cuda_runtime::CudaHtj2kPacketizationSubband +pub j2k_cuda_runtime::CudaHtj2kPacketizationSubband::block_count: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationSubband::block_start: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationSubband::num_cbs_x: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationSubband::num_cbs_y: u32 +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaHtj2kPacketizationSubband +pub const j2k_cuda_runtime::CudaHtj2kPacketizationSubband::NAME: &'static str +#[repr(C)] pub struct j2k_cuda_runtime::CudaHtj2kPacketizationSubbandTagState +pub j2k_cuda_runtime::CudaHtj2kPacketizationSubbandTagState::inclusion_node_start: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationSubbandTagState::node_count: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationSubbandTagState::reserved0: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationSubbandTagState::zero_bitplane_node_start: u32 +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaHtj2kPacketizationSubbandTagState +pub const j2k_cuda_runtime::CudaHtj2kPacketizationSubbandTagState::NAME: &'static str +#[repr(C)] pub struct j2k_cuda_runtime::CudaHtj2kPacketizationTagNodeState +pub j2k_cuda_runtime::CudaHtj2kPacketizationTagNodeState::current: u32 +pub j2k_cuda_runtime::CudaHtj2kPacketizationTagNodeState::known: u32 +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaHtj2kPacketizationTagNodeState +pub const j2k_cuda_runtime::CudaHtj2kPacketizationTagNodeState::NAME: &'static str +pub struct j2k_cuda_runtime::CudaHtj2kPacketizedTile +impl j2k_cuda_runtime::CudaHtj2kPacketizedTile +pub fn j2k_cuda_runtime::CudaHtj2kPacketizedTile::data(&self) -> &[u8] +pub fn j2k_cuda_runtime::CudaHtj2kPacketizedTile::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaHtj2kPacketizedTile::stage_timings(&self) -> j2k_cuda_runtime::CudaHtj2kPacketizationStageTimings +pub fn j2k_cuda_runtime::CudaHtj2kPacketizedTile::statuses(&self) -> &[j2k_cuda_runtime::CudaHtj2kPacketizationStatus] +#[repr(C)] pub struct j2k_cuda_runtime::CudaHtj2kStatus +pub j2k_cuda_runtime::CudaHtj2kStatus::code: u32 +pub j2k_cuda_runtime::CudaHtj2kStatus::detail: u32 +pub j2k_cuda_runtime::CudaHtj2kStatus::reserved0: u32 +pub j2k_cuda_runtime::CudaHtj2kStatus::reserved1: u32 +impl j2k_cuda_runtime::CudaHtj2kStatus +pub fn j2k_cuda_runtime::CudaHtj2kStatus::is_ok(self) -> bool +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaHtj2kStatus +pub const j2k_cuda_runtime::CudaHtj2kStatus::NAME: &'static str +pub struct j2k_cuda_runtime::CudaJ2kDeinterleavedComponents +impl j2k_cuda_runtime::CudaJ2kDeinterleavedComponents +pub fn j2k_cuda_runtime::CudaJ2kDeinterleavedComponents::components(&self) -> &[alloc::vec::Vec] +pub fn j2k_cuda_runtime::CudaJ2kDeinterleavedComponents::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaJ2kDeinterleavedComponents::into_components(self) -> alloc::vec::Vec> +#[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kIdwtJob +pub j2k_cuda_runtime::CudaJ2kIdwtJob::hh_rect: j2k_cuda_runtime::CudaJ2kRect +pub j2k_cuda_runtime::CudaJ2kIdwtJob::hl_rect: j2k_cuda_runtime::CudaJ2kRect +pub j2k_cuda_runtime::CudaJ2kIdwtJob::irreversible97: u32 +pub j2k_cuda_runtime::CudaJ2kIdwtJob::lh_rect: j2k_cuda_runtime::CudaJ2kRect +pub j2k_cuda_runtime::CudaJ2kIdwtJob::ll_rect: j2k_cuda_runtime::CudaJ2kRect +pub j2k_cuda_runtime::CudaJ2kIdwtJob::rect: j2k_cuda_runtime::CudaJ2kRect +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kIdwtJob +pub const j2k_cuda_runtime::CudaJ2kIdwtJob::NAME: &'static str +pub struct j2k_cuda_runtime::CudaJ2kIdwtTarget<'a> +pub j2k_cuda_runtime::CudaJ2kIdwtTarget::hh: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kIdwtTarget::hl: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kIdwtTarget::job: j2k_cuda_runtime::CudaJ2kIdwtJob +pub j2k_cuda_runtime::CudaJ2kIdwtTarget::lh: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kIdwtTarget::ll: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kIdwtTarget::output: &'a j2k_cuda_runtime::CudaDeviceBuffer +#[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kInverseMctJob +pub j2k_cuda_runtime::CudaJ2kInverseMctJob::addend0: f32 +pub j2k_cuda_runtime::CudaJ2kInverseMctJob::addend1: f32 +pub j2k_cuda_runtime::CudaJ2kInverseMctJob::addend2: f32 +pub j2k_cuda_runtime::CudaJ2kInverseMctJob::irreversible97: u32 +pub j2k_cuda_runtime::CudaJ2kInverseMctJob::len: u32 +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kInverseMctJob +pub const j2k_cuda_runtime::CudaJ2kInverseMctJob::NAME: &'static str +pub struct j2k_cuda_runtime::CudaJ2kQuantizeJob +pub j2k_cuda_runtime::CudaJ2kQuantizeJob::range_bits: u8 +pub j2k_cuda_runtime::CudaJ2kQuantizeJob::reversible: bool +pub j2k_cuda_runtime::CudaJ2kQuantizeJob::step_exponent: u16 +pub j2k_cuda_runtime::CudaJ2kQuantizeJob::step_mantissa: u16 +pub struct j2k_cuda_runtime::CudaJ2kQuantizeSubbandRegionJob +pub j2k_cuda_runtime::CudaJ2kQuantizeSubbandRegionJob::height: u32 +pub j2k_cuda_runtime::CudaJ2kQuantizeSubbandRegionJob::quantization: j2k_cuda_runtime::CudaJ2kQuantizeJob +pub j2k_cuda_runtime::CudaJ2kQuantizeSubbandRegionJob::stride: u32 +pub j2k_cuda_runtime::CudaJ2kQuantizeSubbandRegionJob::width: u32 +pub j2k_cuda_runtime::CudaJ2kQuantizeSubbandRegionJob::x0: u32 +pub j2k_cuda_runtime::CudaJ2kQuantizeSubbandRegionJob::y0: u32 +pub struct j2k_cuda_runtime::CudaJ2kQuantizedSubband +impl j2k_cuda_runtime::CudaJ2kQuantizedSubband +pub fn j2k_cuda_runtime::CudaJ2kQuantizedSubband::coefficients(&self) -> &[i32] +pub fn j2k_cuda_runtime::CudaJ2kQuantizedSubband::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +#[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kRect +pub j2k_cuda_runtime::CudaJ2kRect::x0: u32 +pub j2k_cuda_runtime::CudaJ2kRect::x1: u32 +pub j2k_cuda_runtime::CudaJ2kRect::y0: u32 +pub j2k_cuda_runtime::CudaJ2kRect::y1: u32 +pub struct j2k_cuda_runtime::CudaJ2kResidentComponents +impl j2k_cuda_runtime::CudaJ2kResidentComponents +pub fn j2k_cuda_runtime::CudaJ2kResidentComponents::buffer(&self) -> &j2k_cuda_runtime::CudaDeviceBuffer +pub fn j2k_cuda_runtime::CudaJ2kResidentComponents::download_components(&self) -> core::result::Result>, j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaJ2kResidentComponents::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaJ2kResidentComponents::num_components(&self) -> u8 +pub fn j2k_cuda_runtime::CudaJ2kResidentComponents::num_pixels(&self) -> usize +pub struct j2k_cuda_runtime::CudaJ2kResidentQuantizedSubband +impl j2k_cuda_runtime::CudaJ2kResidentQuantizedSubband +pub fn j2k_cuda_runtime::CudaJ2kResidentQuantizedSubband::buffer(&self) -> &j2k_cuda_runtime::CudaDeviceBuffer +pub fn j2k_cuda_runtime::CudaJ2kResidentQuantizedSubband::coefficient_count(&self) -> usize +pub fn j2k_cuda_runtime::CudaJ2kResidentQuantizedSubband::download_coefficients(&self) -> core::result::Result, j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaJ2kResidentQuantizedSubband::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +#[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kStoreGray16Job +pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::addend: f32 +pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::bit_depth: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::copy_height: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::copy_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::input_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::output_height: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::output_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::output_x: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::output_y: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::source_x: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray16Job::source_y: u32 +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kStoreGray16Job +pub const j2k_cuda_runtime::CudaJ2kStoreGray16Job::NAME: &'static str +#[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kStoreGray8Job +pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::addend: f32 +pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::bit_depth: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::copy_height: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::copy_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::input_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::output_height: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::output_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::output_x: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::output_y: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::source_x: u32 +pub j2k_cuda_runtime::CudaJ2kStoreGray8Job::source_y: u32 +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kStoreGray8Job +pub const j2k_cuda_runtime::CudaJ2kStoreGray8Job::NAME: &'static str +#[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kStoreRgb16Job +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::addend0: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::addend1: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::addend2: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::bit_depth0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::bit_depth1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::bit_depth2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::copy_height: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::copy_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::input_width0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::input_width1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::input_width2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::output_height: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::output_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::output_x: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::output_y: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::rgba: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::source_x0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::source_x1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::source_x2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::source_y0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::source_y1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16Job::source_y2: u32 +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kStoreRgb16Job +pub const j2k_cuda_runtime::CudaJ2kStoreRgb16Job::NAME: &'static str +#[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kStoreRgb16MctJob +pub j2k_cuda_runtime::CudaJ2kStoreRgb16MctJob::irreversible97: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb16MctJob::store: j2k_cuda_runtime::CudaJ2kStoreRgb16Job +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kStoreRgb16MctJob +pub const j2k_cuda_runtime::CudaJ2kStoreRgb16MctJob::NAME: &'static str +#[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kStoreRgb8Job +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::addend0: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::addend1: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::addend2: f32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::bit_depth0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::bit_depth1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::bit_depth2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::copy_height: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::copy_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::input_width0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::input_width1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::input_width2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::output_height: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::output_width: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::output_x: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::output_y: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::rgba: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::source_x0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::source_x1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::source_x2: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::source_y0: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::source_y1: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8Job::source_y2: u32 +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJ2kStoreRgb8Job +pub const j2k_cuda_runtime::CudaJ2kStoreRgb8Job::NAME: &'static str +#[repr(C)] pub struct j2k_cuda_runtime::CudaJ2kStoreRgb8MctJob +pub j2k_cuda_runtime::CudaJ2kStoreRgb8MctJob::irreversible97: u32 +pub j2k_cuda_runtime::CudaJ2kStoreRgb8MctJob::store: j2k_cuda_runtime::CudaJ2kStoreRgb8Job +pub struct j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget<'a> +pub j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget::job: j2k_cuda_runtime::CudaJ2kStoreRgb8MctJob +pub j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget::plane0: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget::plane1: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStoreRgb8MctTarget::plane2: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub struct j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels<'a> +pub j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels::bit_depth: u8 +pub j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels::buffer: &'a j2k_cuda_runtime::CudaDeviceBuffer +pub j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels::byte_offset: usize +pub j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels::height: u32 +pub j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels::num_components: u8 +pub j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels::pitch_bytes: usize +pub j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels::signed: bool +pub j2k_cuda_runtime::CudaJ2kStridedInterleavedPixels::width: u32 +pub struct j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan<'a> +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::cb_ac_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::cb_dc_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::cb_quant: [u16; 64] +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::cr_ac_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::cr_dc_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::cr_quant: [u16; 64] +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::dimensions: (u32, u32) +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::entropy_bytes: &'a [u8] +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::entropy_checkpoints: &'a [j2k_cuda_runtime::CudaJpegEntropyCheckpoint] +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::mcu_rows: u32 +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::mcus_per_row: u32 +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::y_ac_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::y_dc_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpeg420Rgb8DecodePlan::y_quant: [u16; 64] +pub struct j2k_cuda_runtime::CudaJpegChunkedEntropyConfig +pub j2k_cuda_runtime::CudaJpegChunkedEntropyConfig::max_overflow_subsequences: u32 +pub j2k_cuda_runtime::CudaJpegChunkedEntropyConfig::sequence_len: u32 +pub j2k_cuda_runtime::CudaJpegChunkedEntropyConfig::subsequence_words: u32 +impl j2k_cuda_runtime::CudaJpegChunkedEntropyConfig +pub fn j2k_cuda_runtime::CudaJpegChunkedEntropyConfig::subsequence_bits(self) -> u32 +pub fn j2k_cuda_runtime::CudaJpegChunkedEntropyConfig::subsequence_count_for_entropy_bytes(self, usize) -> core::result::Result +pub fn j2k_cuda_runtime::CudaJpegChunkedEntropyConfig::validate(self) -> core::result::Result<(), j2k_cuda_runtime::CudaError> +impl core::default::Default for j2k_cuda_runtime::CudaJpegChunkedEntropyConfig +pub fn j2k_cuda_runtime::CudaJpegChunkedEntropyConfig::default() -> Self +pub struct j2k_cuda_runtime::CudaJpegChunkedEntropyPlan<'a> +pub j2k_cuda_runtime::CudaJpegChunkedEntropyPlan::cb_ac_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpegChunkedEntropyPlan::cb_dc_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpegChunkedEntropyPlan::config: j2k_cuda_runtime::CudaJpegChunkedEntropyConfig +pub j2k_cuda_runtime::CudaJpegChunkedEntropyPlan::cr_ac_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpegChunkedEntropyPlan::cr_dc_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpegChunkedEntropyPlan::entropy_bytes: &'a [u8] +pub j2k_cuda_runtime::CudaJpegChunkedEntropyPlan::y_ac_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpegChunkedEntropyPlan::y_dc_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub struct j2k_cuda_runtime::CudaJpegChunkedEntropyReport +pub j2k_cuda_runtime::CudaJpegChunkedEntropyReport::config: j2k_cuda_runtime::CudaJpegChunkedEntropyConfig +pub j2k_cuda_runtime::CudaJpegChunkedEntropyReport::entropy_bytes: usize +pub j2k_cuda_runtime::CudaJpegChunkedEntropyReport::execution: j2k_cuda_runtime::CudaExecutionStats +pub j2k_cuda_runtime::CudaJpegChunkedEntropyReport::overflows: alloc::vec::Vec +pub j2k_cuda_runtime::CudaJpegChunkedEntropyReport::states: alloc::vec::Vec +impl j2k_cuda_runtime::CudaJpegChunkedEntropyReport +pub fn j2k_cuda_runtime::CudaJpegChunkedEntropyReport::failed_state_count(&self) -> usize +pub fn j2k_cuda_runtime::CudaJpegChunkedEntropyReport::max_overflow_bits(&self) -> core::option::Option +pub fn j2k_cuda_runtime::CudaJpegChunkedEntropyReport::subsequence_count(&self) -> usize +pub fn j2k_cuda_runtime::CudaJpegChunkedEntropyReport::synchronized_overflow_count(&self) -> usize +#[repr(C)] pub struct j2k_cuda_runtime::CudaJpegEntropyCheckpoint +pub j2k_cuda_runtime::CudaJpegEntropyCheckpoint::bit_acc: u64 +pub j2k_cuda_runtime::CudaJpegEntropyCheckpoint::bit_count: u32 +pub j2k_cuda_runtime::CudaJpegEntropyCheckpoint::cb_prev_dc: i32 +pub j2k_cuda_runtime::CudaJpegEntropyCheckpoint::cr_prev_dc: i32 +pub j2k_cuda_runtime::CudaJpegEntropyCheckpoint::entropy_pos: u32 +pub j2k_cuda_runtime::CudaJpegEntropyCheckpoint::mcu_index: u32 +pub j2k_cuda_runtime::CudaJpegEntropyCheckpoint::reserved: u32 +pub j2k_cuda_runtime::CudaJpegEntropyCheckpoint::y_prev_dc: i32 +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJpegEntropyCheckpoint +pub const j2k_cuda_runtime::CudaJpegEntropyCheckpoint::NAME: &'static str +#[repr(C)] pub struct j2k_cuda_runtime::CudaJpegEntropyOverflowState +pub j2k_cuda_runtime::CudaJpegEntropyOverflowState::code: u32 +pub j2k_cuda_runtime::CudaJpegEntropyOverflowState::from_subsequence: u32 +pub j2k_cuda_runtime::CudaJpegEntropyOverflowState::overflow_bits: u32 +pub j2k_cuda_runtime::CudaJpegEntropyOverflowState::reserved: [u32; 3] +pub j2k_cuda_runtime::CudaJpegEntropyOverflowState::synchronized: u32 +pub j2k_cuda_runtime::CudaJpegEntropyOverflowState::to_subsequence: u32 +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJpegEntropyOverflowState +pub const j2k_cuda_runtime::CudaJpegEntropyOverflowState::NAME: &'static str +#[repr(C)] pub struct j2k_cuda_runtime::CudaJpegEntropySyncState +pub j2k_cuda_runtime::CudaJpegEntropySyncState::bit_pos: u32 +pub j2k_cuda_runtime::CudaJpegEntropySyncState::block_phase: u32 +pub j2k_cuda_runtime::CudaJpegEntropySyncState::code: u32 +pub j2k_cuda_runtime::CudaJpegEntropySyncState::end_bit: u32 +pub j2k_cuda_runtime::CudaJpegEntropySyncState::reserved: u32 +pub j2k_cuda_runtime::CudaJpegEntropySyncState::start_bit: u32 +pub j2k_cuda_runtime::CudaJpegEntropySyncState::symbol_count: u32 +pub j2k_cuda_runtime::CudaJpegEntropySyncState::zigzag_index: u32 +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJpegEntropySyncState +pub const j2k_cuda_runtime::CudaJpegEntropySyncState::NAME: &'static str +#[repr(C)] pub struct j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpegHuffmanTable::max_code: [i32; 17] +pub j2k_cuda_runtime::CudaJpegHuffmanTable::val_offset: [i32; 17] +pub j2k_cuda_runtime::CudaJpegHuffmanTable::values: [u8; 256] +pub j2k_cuda_runtime::CudaJpegHuffmanTable::values_len: u32 +impl j2k_cuda_runtime::CudaJpegHuffmanTable +pub fn j2k_cuda_runtime::CudaJpegHuffmanTable::from_jpeg_bits_values([u8; 16], u16, [u8; 256]) -> core::result::Result +impl j2k_core::accelerator::GpuAbi for j2k_cuda_runtime::CudaJpegHuffmanTable +pub const j2k_cuda_runtime::CudaJpegHuffmanTable::NAME: &'static str +pub struct j2k_cuda_runtime::CudaJpegRgb8DecodePlan<'a> +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::cb_ac_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::cb_dc_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::cb_quant: [u16; 64] +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::cr_ac_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::cr_dc_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::cr_quant: [u16; 64] +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::dimensions: (u32, u32) +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::entropy_bytes: &'a [u8] +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::entropy_checkpoints: &'a [j2k_cuda_runtime::CudaJpegEntropyCheckpoint] +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::mcu_rows: u32 +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::mcus_per_row: u32 +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::sampling: j2k_cuda_runtime::CudaJpegRgb8Sampling +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::y_ac_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::y_dc_table: j2k_cuda_runtime::CudaJpegHuffmanTable +pub j2k_cuda_runtime::CudaJpegRgb8DecodePlan::y_quant: [u16; 64] +pub struct j2k_cuda_runtime::CudaKernelBatchOutput +impl j2k_cuda_runtime::CudaKernelBatchOutput +pub fn j2k_cuda_runtime::CudaKernelBatchOutput::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaKernelBatchOutput::into_parts(self) -> (alloc::vec::Vec, j2k_cuda_runtime::CudaExecutionStats) +pub fn j2k_cuda_runtime::CudaKernelBatchOutput::outputs(&self) -> &[j2k_cuda_runtime::CudaDeviceBuffer] +pub struct j2k_cuda_runtime::CudaKernelContiguousBatchOutput +impl j2k_cuda_runtime::CudaKernelContiguousBatchOutput +pub fn j2k_cuda_runtime::CudaKernelContiguousBatchOutput::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaKernelContiguousBatchOutput::into_parts(self) -> (j2k_cuda_runtime::CudaDeviceBuffer, alloc::vec::Vec, j2k_cuda_runtime::CudaExecutionStats) +pub fn j2k_cuda_runtime::CudaKernelContiguousBatchOutput::output(&self) -> &j2k_cuda_runtime::CudaDeviceBuffer +pub fn j2k_cuda_runtime::CudaKernelContiguousBatchOutput::ranges(&self) -> &[j2k_cuda_runtime::CudaDeviceBufferRange] +pub struct j2k_cuda_runtime::CudaKernelModule +impl j2k_cuda_runtime::CudaKernelModule +pub fn j2k_cuda_runtime::CudaKernelModule::entrypoint(&self) -> &'static str +pub fn j2k_cuda_runtime::CudaKernelModule::kernel(&self) -> j2k_cuda_runtime::CudaKernelName +pub struct j2k_cuda_runtime::CudaKernelOutput +impl j2k_cuda_runtime::CudaKernelOutput +pub fn j2k_cuda_runtime::CudaKernelOutput::buffer(&self) -> &j2k_cuda_runtime::CudaDeviceBuffer +pub fn j2k_cuda_runtime::CudaKernelOutput::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaKernelOutput::into_parts(self) -> (j2k_cuda_runtime::CudaDeviceBuffer, j2k_cuda_runtime::CudaExecutionStats) +pub struct j2k_cuda_runtime::CudaPinnedHostBuffer +impl j2k_cuda_runtime::CudaPinnedHostBuffer +pub fn j2k_cuda_runtime::CudaPinnedHostBuffer::as_mut_slice(&mut self) -> &mut [u8] +pub fn j2k_cuda_runtime::CudaPinnedHostBuffer::as_slice(&self) -> &[u8] +pub fn j2k_cuda_runtime::CudaPinnedHostBuffer::is_empty(&self) -> bool +pub fn j2k_cuda_runtime::CudaPinnedHostBuffer::len(&self) -> usize +impl core::marker::Send for j2k_cuda_runtime::CudaPinnedHostBuffer +impl core::ops::drop::Drop for j2k_cuda_runtime::CudaPinnedHostBuffer +pub fn j2k_cuda_runtime::CudaPinnedHostBuffer::drop(&mut self) +pub struct j2k_cuda_runtime::CudaPooledDeviceBuffer +impl j2k_cuda_runtime::CudaPooledDeviceBuffer +pub fn j2k_cuda_runtime::CudaPooledDeviceBuffer::allocation_byte_len(&self) -> usize +pub fn j2k_cuda_runtime::CudaPooledDeviceBuffer::as_device_buffer(&self) -> core::option::Option<&j2k_cuda_runtime::CudaDeviceBuffer> +pub fn j2k_cuda_runtime::CudaPooledDeviceBuffer::byte_len(&self) -> usize +pub fn j2k_cuda_runtime::CudaPooledDeviceBuffer::copy_to_host(&self, &mut [u8]) -> core::result::Result<(), j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaPooledDeviceBuffer::device_ptr(&self) -> u64 +pub fn j2k_cuda_runtime::CudaPooledDeviceBuffer::into_device_buffer(self) -> core::result::Result +impl core::ops::drop::Drop for j2k_cuda_runtime::CudaPooledDeviceBuffer +pub fn j2k_cuda_runtime::CudaPooledDeviceBuffer::drop(&mut self) +pub struct j2k_cuda_runtime::CudaPooledHtj2kDecodeOutput +impl j2k_cuda_runtime::CudaPooledHtj2kDecodeOutput +pub fn j2k_cuda_runtime::CudaPooledHtj2kDecodeOutput::coefficients(&self) -> core::option::Option<&j2k_cuda_runtime::CudaDeviceBuffer> +pub fn j2k_cuda_runtime::CudaPooledHtj2kDecodeOutput::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaPooledHtj2kDecodeOutput::into_parts(self) -> (j2k_cuda_runtime::CudaPooledDeviceBuffer, j2k_cuda_runtime::CudaExecutionStats, alloc::vec::Vec) +pub fn j2k_cuda_runtime::CudaPooledHtj2kDecodeOutput::stage_timings(&self) -> j2k_cuda_runtime::CudaHtj2kDecodeStageTimings +pub fn j2k_cuda_runtime::CudaPooledHtj2kDecodeOutput::statuses(&self) -> &[j2k_cuda_runtime::CudaHtj2kStatus] +pub struct j2k_cuda_runtime::CudaPooledKernelOutput +impl j2k_cuda_runtime::CudaPooledKernelOutput +pub fn j2k_cuda_runtime::CudaPooledKernelOutput::buffer(&self) -> core::option::Option<&j2k_cuda_runtime::CudaDeviceBuffer> +pub fn j2k_cuda_runtime::CudaPooledKernelOutput::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaPooledKernelOutput::into_parts(self) -> (j2k_cuda_runtime::CudaPooledDeviceBuffer, j2k_cuda_runtime::CudaExecutionStats) +pub struct j2k_cuda_runtime::CudaQueuedExecution +impl j2k_cuda_runtime::CudaQueuedExecution +pub fn j2k_cuda_runtime::CudaQueuedExecution::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaQueuedExecution::resource_count(&self) -> usize +pub struct j2k_cuda_runtime::CudaQueuedHtj2kCleanup +impl j2k_cuda_runtime::CudaQueuedHtj2kCleanup +pub fn j2k_cuda_runtime::CudaQueuedHtj2kCleanup::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaQueuedHtj2kCleanup::finish(self) -> core::result::Result +pub fn j2k_cuda_runtime::CudaQueuedHtj2kCleanup::resource_count(&self) -> usize +pub struct j2k_cuda_runtime::CudaResidentDwt53Output +impl j2k_cuda_runtime::CudaResidentDwt53Output +pub fn j2k_cuda_runtime::CudaResidentDwt53Output::buffer(&self) -> &j2k_cuda_runtime::CudaDeviceBuffer +pub fn j2k_cuda_runtime::CudaResidentDwt53Output::download_transformed(&self) -> core::result::Result, j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaResidentDwt53Output::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaResidentDwt53Output::levels(&self) -> &[j2k_cuda_runtime::CudaDwt53LevelShape] +pub fn j2k_cuda_runtime::CudaResidentDwt53Output::ll_dimensions(&self) -> (u32, u32) +pub fn j2k_cuda_runtime::CudaResidentDwt53Output::sample_count(&self) -> usize +pub struct j2k_cuda_runtime::CudaResidentDwt97Output +impl j2k_cuda_runtime::CudaResidentDwt97Output +pub fn j2k_cuda_runtime::CudaResidentDwt97Output::buffer(&self) -> &j2k_cuda_runtime::CudaDeviceBuffer +pub fn j2k_cuda_runtime::CudaResidentDwt97Output::download_transformed(&self) -> core::result::Result, j2k_cuda_runtime::CudaError> +pub fn j2k_cuda_runtime::CudaResidentDwt97Output::execution(&self) -> j2k_cuda_runtime::CudaExecutionStats +pub fn j2k_cuda_runtime::CudaResidentDwt97Output::levels(&self) -> &[j2k_cuda_runtime::CudaDwt53LevelShape] +pub fn j2k_cuda_runtime::CudaResidentDwt97Output::ll_dimensions(&self) -> (u32, u32) +pub fn j2k_cuda_runtime::CudaResidentDwt97Output::sample_count(&self) -> usize +pub struct j2k_cuda_runtime::CudaStream +impl j2k_cuda_runtime::CudaStream +pub fn j2k_cuda_runtime::CudaStream::synchronize(&self) -> core::result::Result<(), j2k_cuda_runtime::CudaError> +impl core::marker::Send for j2k_cuda_runtime::CudaStream +impl core::ops::drop::Drop for j2k_cuda_runtime::CudaStream +pub fn j2k_cuda_runtime::CudaStream::drop(&mut self) +pub struct j2k_cuda_runtime::CudaTranscodeDwt97Bands +pub j2k_cuda_runtime::CudaTranscodeDwt97Bands::hh: alloc::vec::Vec +pub j2k_cuda_runtime::CudaTranscodeDwt97Bands::high_height: usize +pub j2k_cuda_runtime::CudaTranscodeDwt97Bands::high_width: usize +pub j2k_cuda_runtime::CudaTranscodeDwt97Bands::hl: alloc::vec::Vec +pub j2k_cuda_runtime::CudaTranscodeDwt97Bands::lh: alloc::vec::Vec +pub j2k_cuda_runtime::CudaTranscodeDwt97Bands::ll: alloc::vec::Vec +pub j2k_cuda_runtime::CudaTranscodeDwt97Bands::low_height: usize +pub j2k_cuda_runtime::CudaTranscodeDwt97Bands::low_width: usize +pub struct j2k_cuda_runtime::CudaTranscodeReversible53Bands +pub j2k_cuda_runtime::CudaTranscodeReversible53Bands::hh: alloc::vec::Vec +pub j2k_cuda_runtime::CudaTranscodeReversible53Bands::high_height: usize +pub j2k_cuda_runtime::CudaTranscodeReversible53Bands::high_width: usize +pub j2k_cuda_runtime::CudaTranscodeReversible53Bands::hl: alloc::vec::Vec +pub j2k_cuda_runtime::CudaTranscodeReversible53Bands::lh: alloc::vec::Vec +pub j2k_cuda_runtime::CudaTranscodeReversible53Bands::ll: alloc::vec::Vec +pub j2k_cuda_runtime::CudaTranscodeReversible53Bands::low_height: usize +pub j2k_cuda_runtime::CudaTranscodeReversible53Bands::low_width: usize +pub fn j2k_cuda_runtime::transcode_kernels_built() -> bool +``` + +## `j2k-profile` + +```text +pub mod j2k_profile +``` + +## `j2k-cli` + +`j2k-cli` is a binary package. Its stable command, stdout/stderr, and exit-code contract is documented in `docs/stable-api-1.0.md`. + diff --git a/docs/unsafe-audit.md b/docs/unsafe-audit.md new file mode 100644 index 00000000..a2bb7312 --- /dev/null +++ b/docs/unsafe-audit.md @@ -0,0 +1,67 @@ +# Unsafe Audit + +This inventory lists Rust sources that currently contain `unsafe` blocks or +unsafe operations. `cargo xtask unsafe-audit` checks that every matching source +under `crates/` appears here, so update this file whenever unsafe code is added, +moved, or removed. + +## Inventory + +| Path | Scope | Invariants | Regression guards | +| --- | --- | --- | --- | +| `crates/j2k-core/src/accelerator.rs` | Shared GPU ABI marker trait and byte-view helpers for host/device struct transfers. | Implementers are plain-data GPU ABI values with stable layout and valid byte views. | Core API tests plus backend layout/parity tests for GPU ABI structs. | +| `crates/j2k-core/src/backend.rs` | CPU feature detection and backend probing. | CPU intrinsics are called only behind matching architecture/configuration checks. | Cross-architecture CI matrix and backend feature tests. | +| `crates/j2k-cuda-runtime/src/bytes.rs` | GPU ABI marker impls and typed host/device struct byte views. | Typed values are copied as initialized plain-data bytes with compatible CUDA layout. | CUDA runtime unit tests and ABI compile checks. | +| `crates/j2k-cuda-runtime/src/context.rs` | CUDA context creation, module loading, pinned staging views, and context teardown. | CUDA handles outlive copied function/module uses and pinned host memory is uniquely borrowed. | CUDA runtime tests and strict GPU validation workflow. | +| `crates/j2k-cuda-runtime/src/cuda_oxide_copy_u8/simt/src/main.rs` | cuda-oxide device `CopyU8` kernel raw pointer copy. | Device pointers refer to live buffers of at least `len` bytes and each thread checks `index < len` before reading or writing one byte. | Built-in CopyU8 parity test when cuda-oxide PTX is generated and strict GPU validation workflow. | +| `crates/j2k-cuda-runtime/src/cuda_oxide_j2k_decode_store/simt/src/main.rs` | cuda-oxide device J2K decode store and inverse-MCT kernels. | Device pointers refer to live, launch-bounded component/output buffers or job-buffer device pointers; each kernel exits out-of-range threads before raw pointer reads or writes. | Decode store/MCT parity and metadata tests when cuda-oxide PTX is generated plus strict GPU validation workflow. | +| `crates/j2k-cuda-runtime/src/cuda_oxide_j2k_dequantize/simt/src/main.rs` | cuda-oxide device J2K HTJ2K dequantize kernels. | Device pointers refer to live, launch-bounded coefficient buffers or job-buffer device pointers; each block reads only its own job and each thread strides inside that job's sample count. | HTJ2K dequantize parity and metadata tests when cuda-oxide PTX is generated plus strict GPU validation workflow. | +| `crates/j2k-cuda-runtime/src/cuda_oxide_j2k_idwt/simt/src/main.rs` | cuda-oxide device J2K generic inverse-DWT kernels. | Device pointers refer to live, launch-bounded band/output buffers or job-buffer device pointers; Rust-side launch code validates IDWT band and output buffer lengths before dispatch. | IDWT parity and metadata tests when cuda-oxide PTX is generated plus strict GPU validation workflow. | +| `crates/j2k-cuda-runtime/src/cuda_oxide_j2k_encode/simt/src/main.rs` | cuda-oxide device J2K encode-stage kernels for deinterleave, color transform, DWT, quantization, HTJ2K encoded-byte compaction, and HTJ2K packetization. | Device pointers refer to live, launch-bounded sample/tile, scratch, compact-output, packet-output, packet metadata, tag-tree state, and job buffers; each kernel exits or strides inside launch/job bounds before raw pointer reads or writes, and packetization shares only per-block header/body lengths after a synchronization barrier. | J2K encode-stage parity, HTJ2K compaction fixture, HTJ2K packetization scalar-parity tests, and metadata tests when cuda-oxide PTX is generated plus strict GPU validation workflow. | +| `crates/j2k-cuda-runtime/src/cuda_oxide_transcode/simt/src/main.rs` | cuda-oxide device reversible 5/3 and irreversible 9/7 transcode kernels, including batch, code-block quantize, and fused column-lift+quantize stages. | Device pointers refer to live, launch-bounded DCT block, spatial sample, row-band, quantized code-block, and output band buffers; each kernel exits out-of-range threads before raw pointer reads or writes, and cooperative row-lift synchronization has no divergent early return before barriers. | Transcode parity and metadata tests when cuda-oxide PTX is generated plus strict GPU validation workflow. | +| `crates/j2k-cuda-runtime/src/driver.rs` | CUDA/NVTX driver FFI symbol loading and error-name conversion. | Loaded symbols match CUDA Driver API signatures and the driver library outlives function pointers. | CUDA runtime tests with real and fake driver paths. | +| `crates/j2k-cuda-runtime/src/execution.rs` | CUDA kernel parameter ABI, kernel launch, streams, events, and timing. | Kernel parameter pointers remain valid through launch and owned event/stream handles are destroyed once. | CUDA launch/parity tests and strict GPU validation workflow. | +| `crates/j2k-cuda-runtime/src/jpeg.rs` | CUDA JPEG kernel parameter ABI structs and entropy diagnostic launch metadata. | Kernel parameter structs are repr(C) plain-data values matching CUDA kernel signatures and validated launch ranges. | CUDA runtime JPEG kernel metadata tests and JPEG CUDA adapter tests. | +| `crates/j2k-cuda-runtime/src/memory.rs` | CUDA device/pinned memory allocation, copies, downloads, and buffer views. | Device ranges are bounds-checked and uninitialized host capacity is marked initialized only after successful copies. | CUDA memory tests and surface download tests. | +| `crates/j2k-cuda-runtime/src/tests.rs` | Test-only fake CUDA driver table construction. | Fake FFI signatures match production driver types. | CUDA runtime unit tests compile and execute fake-driver paths. | +| `crates/j2k-cuda-runtime/src/transcode.rs` | CUDA transcode host staging byte views for DCT-grid uploads. | Staging byte views match typed host slices and CUDA kernels receive bounded ranges. | CUDA transcode parity tests. | +| `crates/j2k-compare/src/grok.rs` | Grok FFI comparison harness. | External decoder pointers are checked for null and output buffers are sized before copy. | Optional Grok/OpenJPEG parity tests. | +| `crates/j2k-compare/src/openjpeg.rs` | OpenJPEG FFI comparison harness. | OpenJPEG stream/image lifetimes are paired with cleanup and component buffers are bounds-checked. | Optional Grok/OpenJPEG parity tests. | +| `crates/j2k-cuda/src/surface.rs` | CUDA surface batch downloads into preallocated host output buffers. | Batch range math is checked and host Vec length is set only after successful device copy. | CUDA surface and batch download tests. | +| `crates/j2k-metal/src/compute.rs` | Metal buffer pointer access, shader setup, and command submission. | Metal buffer contents are accessed only after size/alignment validation and command buffers are completed before host reads. | Metal runtime tests and GPU validation workflow. | +| `crates/j2k-metal/src/compute/buffer_validation.rs` | Metal validation/copy helper dispatch and status readback. | Validation buffers use exact ABI status sizes and are read only after the validation command buffer completes. | J2K Metal encode validation and roundtrip tests. | +| `crates/j2k-metal/src/compute/direct_buffers.rs` | MetalDirect host slice buffer uploads and reusable shared-buffer zeroing. | Typed host slices are copied or borrowed with validated byte lengths, and reusable buffers are allocated to at least the requested byte count before writes. | MetalDirect decode/encode and resident batch tests. | +| `crates/j2k-metal/src/compute/direct_flattened.rs` | Flattened hybrid CPU-tier1 worker output pointers. | Work items are built from non-overlapping packed coefficient ranges, and each pointer is converted back to a mutable slice with the validated per-item output length before a single decode write. | MetalDirect hybrid batch and flattened CPU-tier1 tests. | +| `crates/j2k-metal/src/compute/direct_status.rs` | MetalDirect decode status buffer readback. | Status buffers are produced by completed command buffers and read with the exact status ABI type and checked entry counts. | MetalDirect decode and hybrid batch tests. | +| `crates/j2k-metal/src/compute/gpu_timing.rs` | Metal command-buffer GPU timestamp queries through Objective-C messages. | Timestamps are queried only for completed command buffers, duplicate buffers are ignored, and non-finite or inverted timestamp pairs are discarded. | Resident encode/decode profiling tests and GPU validation workflow. | +| `crates/j2k-metal/src/encode/config.rs` | Resident Metal encode memory-budget probing and batch chunk policy. | Host memory probing writes into a correctly sized `u64` buffer and falls back when the sysctl probe is unavailable. | Metal encode config and resident chunking tests. | +| `crates/j2k-metal/src/encode/encoded.rs` | Metal-backed codestream buffer views. | Encoded byte ranges are kernel-produced, bounds-checked, and copied before exposure. | Metal encode tests and parity benches. | +| `crates/j2k-metal/src/encode.rs` | Host-visible encode input buffer views and encode validation helpers. | Input buffers are checked for host visibility and staged/copied before creating bounded sample views. | Metal encode tests and parity benches. | +| `crates/j2k-metal/src/lib.rs` | Metal surface byte views and host/device transfer helpers. | Surface buffers carry valid residency/format metadata and checked host download lengths. | Metal surface tests. | +| `crates/j2k-metal/src/mct.rs` | SIMD-assisted color transform helpers. | SIMD loads/stores remain within row slices and preserve channel layout. | MCT unit tests and cross-backend parity. | +| `crates/j2k-metal/src/profile_env.rs` | Metal profiling labels and OS signpost FFI shims. | Signpost end calls use only ids returned by the matching begin shim and invalid/null ids are discarded. | Metal profiling and encode/decode route tests. | +| `crates/j2k-jpeg-metal/src/buffers.rs` | JPEG Metal reusable scratch buffers and typed slice byte uploads. | Typed slices are viewed only as initialized immutable bytes and copied into capacity-checked Metal buffers. | JPEG Metal batch, viewport, and buffer reuse tests. | +| `crates/j2k-jpeg-metal/src/compute.rs` | Metal buffer pointer access, shader setup, and command submission. | Buffer sizes, strides, and texture dimensions are validated before kernel dispatch/readback. | JPEG Metal viewport and compare tests. | +| `crates/j2k-jpeg-metal/src/compute/status.rs` | JPEG Metal decode status buffer initialization and readback. | Status buffers are allocated for the requested status-entry count, initialized before dispatch, and read only after synchronized command completion. | JPEG Metal fast decode, batch, and viewport tests. | +| `crates/j2k-jpeg-metal/src/compute/viewport_cache.rs` | Viewport plane cache staging and CPU row writes into Metal buffers. | Plane buffers are allocated for validated dimensions and row writes are bounded by decoded row widths before creating mutable slices. | JPEG Metal viewport and batch decode tests. | +| `crates/j2k-jpeg-metal/src/lib.rs` | Metal surface byte views and host/device transfer helpers. | Metal surfaces expose only initialized bytes for validated dimensions/format. | JPEG Metal host surface and viewport tests. | +| `crates/j2k-jpeg-metal/tests/viewport.rs` | Test-only Metal texture byte access for viewport validation. | Test texture reads use known dimensions and synchronized command completion. | Viewport test itself. | +| `crates/j2k-jpeg/benches/encode_cpu.rs` | Benchmark allocation-counting global allocator wrapper. | Allocator wrapper forwards all layout requests unchanged and records counts atomically. | Benchmark build gate. | +| `crates/j2k-jpeg/benches/common/libjpeg_turbo.rs` | libjpeg-turbo FFI benchmark harness. | libjpeg-turbo handles are checked and destroyed and output buffers are sized by library-reported geometry. | Optional libjpeg-turbo comparison benches. | +| `crates/j2k-jpeg/src/backend/mod.rs` | Runtime backend dispatch and SIMD entry points. | SIMD entry points are called only when CPU features match the implementation. | Backend dispatch tests on x86_64/aarch64 CI. | +| `crates/j2k-jpeg/src/backend/neon.rs` | AArch64 NEON SIMD kernels. | NEON loads/stores stay within blocks/rows and match scalar reference math. | NEON hot-path and backend parity tests. | +| `crates/j2k-jpeg/src/backend/x86.rs` | x86 SIMD kernels and CPU feature-gated dispatch. | AVX/SSE paths require detected CPU features and match scalar reference math. | x86 backend parity tests. | +| `crates/j2k-jpeg/src/bench_support.rs` | Benchmark-only buffer and decoder helpers. | Benchmark buffers mirror production size checks before unsafe fast paths. | Benchmark build gate. | +| `crates/j2k-jpeg/src/decoder.rs` | Decoder buffer slicing and performance-critical pixel paths. | Entropy/pixel buffers are sized before unsafe slicing and decode caps are enforced before allocation. | JPEG regression, fuzz, and decode parity tests. | +| `crates/j2k-jpeg/src/entropy/sequential.rs` | Entropy decoder bitstream fast paths. | Bitstream reads never advance past scan data and block writes stay inside MCU buffers. | JPEG decode fuzz and regression tests. | +| `crates/j2k-jpeg/src/idct/avx2.rs` | AVX2 IDCT implementation. | AVX2 loads/stores use fixed block sizes and match scalar IDCT output. | IDCT parity tests. | +| `crates/j2k-jpeg/src/idct/neon.rs` | NEON IDCT implementation. | NEON loads/stores use fixed block sizes and match scalar IDCT output. | IDCT parity tests. | +| `crates/j2k-metal-support/src/lib.rs` | Metal command queue creation through Objective-C runtime calls. | Objective-C returned pointers are null-checked and buffer content helpers validate bounds/alignment. | Metal support and runtime tests. | +| `crates/j2k-transcode-cuda/src/cuda.rs` | CUDA accelerator and context integration for transcode stages. | CUDA surfaces are submitted only with matching residency and validated ranges. | CUDA transcode parity tests. | +| `crates/j2k-transcode-metal/src/metal.rs` | Metal accelerator integration for transcode stages. | Metal surfaces are submitted only with matching residency and validated ranges. | Metal transcode tests and validation workflow. | + +## Review Expectations + +Unsafe code must stay isolated behind safe APIs, validate pointer lengths before +forming slices, keep FFI ownership explicit, and prefer feature-gated dispatch +over unchecked CPU or GPU capability assumptions. diff --git a/docs/wsi-decode-api.md b/docs/wsi-decode-api.md deleted file mode 100644 index 2e7bf45f..00000000 --- a/docs/wsi-decode-api.md +++ /dev/null @@ -1,165 +0,0 @@ -# WSI Decode API - -This guide describes the public decode surfaces intended for whole-slide -imaging readers. It covers the stable caller contract shared by JPEG, -JPEG 2000 / HTJ2K, tile decompression, and the device-output adapters. - -## Ownership Model - -signinum does not own a viewer runtime. Callers own I/O, threading, tile -coordinates, pyramid selection, cache policy, and prefetch. Codec APIs only -parse compressed bytes and write decoded pixels into caller-provided storage. - -Use caller-owned state for hot loops: - -- `ScratchPool` reuses temporary allocations within one codec family. -- `DecoderContext` reuses codec tables and planning state across tile batches. -- `DeviceSubmission` lets adapter crates queue work and return a `DeviceSurface` - after `wait()`. - -The codec crates do not spawn worker threads, hold global decode queues, or -hide output allocation behind a runtime. - -## CPU Decode Surfaces - -Use `ImageDecode` when the caller has one compressed image or tile and wants -pixels in host memory. - -Common shapes: - -- `decode_into` decodes the full image. -- `decode_region_into` decodes a source-coordinate ROI. -- `decode_scaled_into` decodes the full image at a reduced resolution. -- `decode_region_scaled_into` decodes a source-coordinate ROI on a reduced - resolution grid. - -ROI coordinates are always expressed in source-image pixels. For -`decode_region_scaled_into`, the returned `DecodeOutcome::decoded` rectangle is -the floor-start / ceil-end projection of the source ROI into the scaled grid. -`Downscale::None` preserves the original ROI. - -```rust -use signinum_core::{Downscale, ImageDecode, PixelFormat, Rect}; -use signinum_j2k::{J2kDecoder, J2kScratchPool}; - -let bytes = std::fs::read("tile.jp2")?; -let mut decoder = J2kDecoder::new(&bytes)?; -let roi = Rect { - x: 512, - y: 512, - w: 1024, - h: 1024, -}; -let scale = Downscale::Quarter; -let scaled = roi.scaled_covering(scale); -let stride = scaled.w as usize * PixelFormat::Gray8.bytes_per_pixel(); -let mut out = vec![0_u8; stride * scaled.h as usize]; - -decoder.decode_region_scaled_into( - &mut J2kScratchPool::new(), - &mut out, - stride, - PixelFormat::Gray8, - roi, - scale, -)?; -``` - -## Row Streaming - -Use `decode_rows` through `ImageDecodeRows` when a tile or image is too large -for one packed output buffer or when the caller wants to feed rows into a -streaming consumer. The caller implements `RowSink`, and signinum forwards sink -errors without converting them into silent decode success. - -Row streaming is a host-memory API. Device adapters return surfaces instead of -row callbacks. - -## Tile Batches - -Use `TileBatchDecode` when a WSI reader is decoding many independent tile -payloads with the same codec. The caller keeps one `DecoderContext` and one -`ScratchPool`, then calls the stateless tile helper repeatedly. - -```rust -use signinum_core::{DecoderContext, PixelFormat, TileBatchDecode}; -use signinum_jpeg::{JpegCodec, ScratchPool}; - -let mut ctx = DecoderContext::::new(); -let mut pool = ScratchPool::new(); - -for tile in visible_tiles { - JpegCodec::decode_tile( - &mut ctx, - &mut pool, - tile, - &mut output, - stride, - PixelFormat::Rgb8, - )?; -} -``` - -Tile-batch helpers exist for full, ROI, scaled, and ROI+scaled decode. The -same source-coordinate ROI and reduced-grid coverage rules apply to tile-batch -ROI+scaled decode. - -## Device Surfaces - -Use `ImageDecodeDevice`, `ImageDecodeSubmit`, `TileBatchDecodeDevice`, or -`TileBatchDecodeSubmit` when a downstream pipeline wants a backend-tagged -surface. Completed operations return a `DeviceSurface`, which reports: - -- backend kind -- dimensions -- pixel format -- byte length - -Backend selection uses `BackendRequest`: - -- `BackendRequest::Cpu` requires host-backed output. -- `BackendRequest::Auto` lets the adapter choose CPU or a device path. Auto is - conservative and may return CPU-backed surfaces when benchmarks or shape - support do not justify device execution. -- `BackendRequest::Metal` requires resident Metal execution on macOS. - CPU-decoded bytes are not uploaded to satisfy this request. Call explicit - CPU-staged upload APIs where the adapter exposes them when a Metal buffer is - needed after CPU decode. Unsupported explicit Metal requests return an error. -- `BackendRequest::Cuda` requires CUDA device memory output. When an adapter is - built with `cuda-runtime` and a CUDA driver is available, explicit CUDA - requests return CUDA-backed surfaces. `signinum-jpeg-cuda` uses nvJPEG for - full-frame RGB8 JPEG decode when `libnvjpeg` is available; unsupported JPEG - shapes and J2K CUDA requests use CPU decode plus CUDA upload. Hosts without - CUDA return unavailable. `Cpu` and `Auto` remain CPU-backed host surfaces. - -For Metal adapters, `BackendRequest::Auto` is a routing hint and may fall back -to host-backed CPU output when the request shape is not on the Metal-supported -path. `BackendRequest::Metal` is a strict request: supported shapes return -resident Metal-backed decode surfaces, unsupported shapes fail as unsupported, -and hosts without Metal fail as unavailable. -Adapters that expose `SurfaceResidency` mark true resident decode separately -from CPU-staged Metal upload so WSI pipelines do not count upload buffers as GPU -decode. - -Callers should use explicit device requests only when they need that backend. -Use `Auto` for viewer paths where CPU fallback is acceptable. - -## Error Contract - -No decode path should fail silently. Unsupported formats, invalid regions, -too-small buffers, too-small strides, unavailable explicit backends, and row -sink aborts are returned as errors. Callers should handle `CodecError` -predicates for broad policy decisions and preserve detailed errors for logging. - -## Current Validation Scope - -Hosted CI validates CPU behavior, adapter fallback behavior, rustdoc, and -benchmark compilation. Runtime GPU validation is available through the manual -`.github/workflows/gpu-validation.yml` workflow on self-hosted runners: - -- Apple Silicon runners labeled `self-hosted`, `macOS`, `ARM64`, `metal` - validate Metal tests and optionally timed Metal benchmarks. -- x86_64 CUDA runners labeled `self-hosted`, `Linux`, `X64`, `cuda` validate - CUDA device-memory output with `cuda-runtime` and the nvJPEG full-frame RGB8 - JPEG path when `libnvjpeg` is installed. Timed NVIDIA performance claims - require the workflow's timed benchmark mode and recorded output. diff --git a/docs/wsi-dicom-passthrough.md b/docs/wsi-dicom-passthrough.md deleted file mode 100644 index 15152a7d..00000000 --- a/docs/wsi-dicom-passthrough.md +++ /dev/null @@ -1,117 +0,0 @@ -# WSI/DICOM Passthrough Policy - -This repository owns codec primitives and passthrough eligibility contracts, -not whole-slide container parsing or DICOM writing. WSI/DICOM conversion code -should use these primitives with a passthrough-first policy: preserve -compressed tile payloads when they are already legal for the requested -destination, and re-encode only when that is not possible. - -## Decision Order - -1. Inspect the source compressed tile bytes without decoding. -2. If the source codec, dimensions, sample layout, bit depth, color model, - frame/tile order, and destination transfer syntax are compatible, copy the - compressed tile bytes into the destination container unchanged. -3. If passthrough is not legal and diagnostic-preserving output is required, - decode to pixels and encode a new lossless JPEG 2000 or HTJ2K codestream. -4. Use baseline JPEG encoding only for explicit non-diagnostic fallback cases: - generated fixtures, preview/export derivatives, or a caller-requested legacy - transfer syntax where lossy output is acceptable. - -## Codec Responsibilities - -- `signinum-core` provides `PassthroughCandidate`, - `PassthroughRequirements`, compressed transfer syntax descriptors, payload - kind descriptors, and typed rejection reasons. This is the shared contract - container writers should use before choosing copy vs transcode. -- `signinum-jpeg` provides JPEG inspect/decode and a small baseline JPEG - fallback encoder. `JpegView::passthrough_candidate()` exposes baseline and - extended sequential JPEG interchange streams as borrowed copy candidates. - JPEG encoding is not the WSI/DICOM hot path. -- `signinum-j2k` provides JPEG 2000 / HTJ2K inspect, decode, and lossless - encode surfaces. `J2kView::passthrough_candidate()` classifies raw - codestreams versus JP2 files and classic versus HT JPEG 2000 syntax when the - native parser can prove that metadata. New diagnostic codestreams should use - this path by default. -- Device adapters produce decoded device surfaces for viewer, QC, preprocessing, - or GPU-resident downstream work. They do not make passthrough decisions. - -## Passthrough Eligibility - -A caller-owned container layer should require all of the following before -copying compressed bytes: - -- destination transfer syntax matches the source codestream family and profile; -- frame dimensions, tile geometry, component count, bit depth, signedness, and - planar/interleaved expectations are compatible; -- photometric interpretation and color transform expectations are compatible; -- the source payload is complete and independently inspectable by the matching - codec parser; -- copying the payload preserves frame ordering and per-frame metadata expected - by the destination container. - -If any condition fails, the caller should choose an explicit transcode path and -record why passthrough was rejected. - -```rust -use signinum_core::{ - CompressedPayloadKind, CompressedTransferSyntax, PassthroughRequirements, -}; -use signinum_j2k::J2kView; - -let view = J2kView::parse(tile_bytes)?; -let Some(candidate) = view.passthrough_candidate() else { - // Parser could decode/inspect the payload but could not prove a copy-safe - // transfer syntax. Decode and transcode instead. - return Ok(CopyDecision::Transcode); -}; -let requirements = PassthroughRequirements::new( - CompressedTransferSyntax::HtJpeg2000Lossless, - CompressedPayloadKind::Jpeg2000Codestream, -) -.with_dimensions(expected_dimensions) -.with_components(expected_components) -.with_bit_depth(expected_bit_depth); - -match candidate.copy_bytes_if_eligible(&requirements) { - Ok(bytes) => write_dicom_fragment(bytes)?, - Err(reason) => transcode_and_record_reason(reason)?, -} -``` - -## Benchmark Signoff - -JPEG decode and device-output benchmarks remain relevant for viewer and QC -paths. JPEG encode benchmarks are fallback diagnostics only. WSI/DICOM storage -signoff should prioritize passthrough eligibility coverage and J2K/HTJ2K -lossless encode throughput and parity. - -## Local NDPI Regression Test - -`signinum-jpeg` includes an optional local NDPI passthrough test. It does not -decode the whole-slide image and does not add NDPI container support to the -production crate. The test reads TIFF directories from a local NDPI file, -extracts JPEG-compressed payloads, and proves each eligible payload returns the -original borrowed bytes from either `JpegView::passthrough_candidate()` or the -container-level fallback for Hamamatsu JPEG strips whose embedded SOF -dimensions are zero and whose real dimensions live in the TIFF IFD. - -```sh -SIGNINUM_NDPI_PATH=/path/to/slide.ndpi \ - cargo test -p signinum-jpeg --test ndpi_passthrough --profile release-bench -- --nocapture -``` - -Use `SIGNINUM_NDPI_TILE_LIMIT=0` for a full-container pass: - -```sh -SIGNINUM_NDPI_PATH=/path/to/slide.ndpi \ -SIGNINUM_NDPI_TILE_LIMIT=0 \ - cargo test -p signinum-jpeg --test ndpi_passthrough --profile release-bench -- --nocapture -``` - -Optional controls: - -- `SIGNINUM_REQUIRE_NDPI=1` fails the test when `SIGNINUM_NDPI_PATH` is unset. -- `SIGNINUM_NDPI_TILE_LIMIT` defaults to `8`; `0` means all payloads. -- `SIGNINUM_NDPI_MAX_PAYLOAD_BYTES` defaults to `67108864` for bounded runs - and no limit for full-container runs. diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 15c1d7e0..94a0bbc4 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.94" +channel = "1.96" components = ["rustfmt", "clippy", "rust-src"] targets = ["aarch64-apple-darwin", "x86_64-unknown-linux-gnu"] profile = "default" diff --git a/scripts/parity-corpus-fetch.sh b/scripts/parity-corpus-fetch.sh index 49859757..8c4524ec 100755 --- a/scripts/parity-corpus-fetch.sh +++ b/scripts/parity-corpus-fetch.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -manifest="${SIGNINUM_PARITY_CORPUS_MANIFEST:-${1:-corpus/wsi-samples/manifest.json}}" -out_dir="${SIGNINUM_PARITY_CORPUS_DIR:-${2:-corpus/wsi-samples}}" +manifest="${J2K_PARITY_CORPUS_MANIFEST:-${1:-corpus/wsi-samples/manifest.json}}" +out_dir="${J2K_PARITY_CORPUS_DIR:-${2:-corpus/wsi-samples}}" python3 - "$manifest" "$out_dir" <<'PY' import hashlib @@ -13,6 +13,8 @@ import sys import urllib.parse import urllib.request +MAX_DOWNLOAD_BYTES = int(os.environ.get("J2K_PARITY_CORPUS_MAX_BYTES", str(512 * 1024 * 1024))) + def fail(message): print(f"parity-corpus-fetch: {message}", file=sys.stderr) @@ -29,6 +31,8 @@ def entry_value(entry, *keys): def safe_relative_path(raw, url): if raw: + if "\\" in raw: + fail(f"unsafe output path {raw!r}") path = pathlib.PurePosixPath(raw) else: parsed = urllib.parse.urlparse(url) @@ -42,6 +46,21 @@ def safe_relative_path(raw, url): return pathlib.Path(*path.parts) +def safe_destination(root, relative): + destination = (root / relative).resolve() + try: + destination.relative_to(root) + except ValueError: + fail(f"unsafe output path {str(relative)!r}") + return destination + + +def validate_url(url): + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "https" or not parsed.netloc: + fail(f"unsafe download URL {url!r}: only https URLs are allowed") + + def sha256_file(path): digest = hashlib.sha256() with path.open("rb") as handle: @@ -52,6 +71,7 @@ def sha256_file(path): manifest_path = pathlib.Path(sys.argv[1]) out_dir = pathlib.Path(sys.argv[2]) +out_root = out_dir.resolve() if not manifest_path.exists(): fail(f"manifest not found: {manifest_path}") @@ -73,9 +93,10 @@ for entry in entries: expected = entry_value(entry, "sha256", "sha-256") if not url or not expected: fail("each entry must contain url and sha256") + validate_url(url) relative = safe_relative_path(entry_value(entry, "path", "file", "filename") or "", url) - destination = out_dir / relative + destination = safe_destination(out_root, relative) destination.parent.mkdir(parents=True, exist_ok=True) if destination.exists() and sha256_file(destination).lower() == expected.lower(): @@ -85,7 +106,19 @@ for entry in entries: tmp = destination.with_suffix(destination.suffix + ".part") print(f"fetch {url} -> {destination}") with urllib.request.urlopen(url, timeout=60) as response: - tmp.write_bytes(response.read()) + total = 0 + with tmp.open("wb") as handle: + while True: + chunk = response.read(1024 * 1024) + if not chunk: + break + total += len(chunk) + if total > MAX_DOWNLOAD_BYTES: + tmp.unlink(missing_ok=True) + fail( + f"download for {destination} exceeds {MAX_DOWNLOAD_BYTES} byte limit" + ) + handle.write(chunk) actual = sha256_file(tmp) if actual.lower() != expected.lower(): diff --git a/scripts/publish-crate.sh b/scripts/publish-crate.sh index 3531951f..85a6b4d3 100755 --- a/scripts/publish-crate.sh +++ b/scripts/publish-crate.sh @@ -3,20 +3,117 @@ set -euo pipefail crate="${1:?usage: publish-crate.sh }" dry_run="${DRY_RUN_ONLY:-false}" -version="$(cargo pkgid -p "$crate" | sed 's/.*#//')" + +publishable_crates=( + j2k-core + j2k-profile + j2k-types + j2k-cuda-runtime + j2k-metal-support + j2k-native + j2k-jpeg + j2k-tilecodec + j2k + j2k-transcode + j2k-transcode-cuda + j2k-jpeg-metal + j2k-metal + j2k-transcode-metal + j2k-jpeg-cuda + j2k-cuda + j2k-cli +) + +workspace_version() { + awk ' + /^\[workspace.package\]/ { in_workspace_package = 1; next } + /^\[/ && in_workspace_package { exit } + in_workspace_package && $1 == "version" { + gsub(/"/, "", $3) + print $3 + exit + } + ' Cargo.toml +} + +crate_version() { + cargo pkgid -p "$1" | sed 's/.*#//' +} + +is_publishable_crate() { + local requested="$1" + local publishable + for publishable in "${publishable_crates[@]}"; do + if [[ "$publishable" == "$requested" ]]; then + return 0 + fi + done + return 1 +} + +require_publishable_crate() { + if ! is_publishable_crate "$1"; then + echo "${1}: not in the publishable release set; run cargo xtask release-integrity" >&2 + exit 1 + fi +} + +require_release_preflight() { + local expected_version="$1" + local actual_workspace_version + local expected_tag + local actual_tag + local publishable + local publishable_version + + actual_workspace_version="$(workspace_version)" + if [[ -z "$actual_workspace_version" ]]; then + echo "failed to read workspace.package.version from Cargo.toml" >&2 + exit 1 + fi + if [[ "$expected_version" != "$actual_workspace_version" ]]; then + echo "${crate}: package version ${expected_version} does not match workspace version ${actual_workspace_version}" >&2 + exit 1 + fi + + expected_tag="v${actual_workspace_version}" + actual_tag="${GITHUB_REF_NAME:-}" + if [[ -z "$actual_tag" ]]; then + actual_tag="$(git describe --tags --exact-match 2>/dev/null || true)" + fi + if [[ "$actual_tag" != "$expected_tag" ]]; then + echo "real publish requires tag ${expected_tag}; current tag is ${actual_tag:-}" >&2 + exit 1 + fi + + for publishable in "${publishable_crates[@]}"; do + publishable_version="$(crate_version "$publishable")" + if [[ "$publishable_version" != "$actual_workspace_version" ]]; then + echo "${publishable}: version ${publishable_version} does not match workspace version ${actual_workspace_version}" >&2 + exit 1 + fi + done +} + +require_publishable_crate "$crate" +version="$(crate_version "$crate")" has_unpublished_workspace_dependency() { case "$1" in - signinum-j2k-native | \ - signinum-jpeg | \ - signinum-tilecodec | \ - signinum-j2k | \ - signinum-jpeg-metal | \ - signinum-j2k-metal | \ - signinum-jpeg-cuda | \ - signinum-j2k-cuda | \ - signinum-cli | \ - signinum) + j2k-cuda-runtime | \ + j2k-metal-support | \ + j2k-native | \ + j2k-jpeg | \ + j2k-tilecodec | \ + j2k | \ + j2k-transcode | \ + j2k-transcode-cuda | \ + j2k-jpeg-metal | \ + j2k-metal | \ + j2k-transcode-metal | \ + j2k-jpeg-cuda | \ + j2k-cuda | \ + j2k-cli) return 0 ;; *) @@ -36,11 +133,20 @@ if [[ "$dry_run" == "true" ]]; then exit 0 fi +require_release_preflight "$version" : "${CRATES_IO_API_TOKEN:?CRATES_IO_API_TOKEN is required for a real publish}" if cargo info "${crate}@${version}" --registry crates-io >/dev/null 2>&1; then - echo "${crate} ${version} is already published; skipping" - exit 0 + case "${CRATES_IO_ALLOW_PUBLISHED_RERUN:-false}" in + true | 1) + echo "${crate} ${version} is already published; idempotent rerun allowed" + exit 0 + ;; + *) + echo "${crate} ${version} is already published; set CRATES_IO_ALLOW_PUBLISHED_RERUN=true for an idempotent rerun" >&2 + exit 1 + ;; + esac fi export CARGO_REGISTRY_TOKEN="$CRATES_IO_API_TOKEN" diff --git a/tests/nvidia-baseline/Cargo.lock b/tests/nvidia-baseline/Cargo.lock new file mode 100644 index 00000000..109c8681 --- /dev/null +++ b/tests/nvidia-baseline/Cargo.lock @@ -0,0 +1,398 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core-graphics-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" +dependencies = [ + "bitflags", + "core-foundation", + "libc", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "fearless_simd" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + +[[package]] +name = "j2k" +version = "0.6.0" +dependencies = [ + "j2k-core", + "j2k-native", + "j2k-types", + "thiserror", +] + +[[package]] +name = "j2k-core" +version = "0.6.0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "j2k-cuda" +version = "0.6.0" +dependencies = [ + "j2k", + "j2k-core", + "j2k-cuda-runtime", + "j2k-native", + "j2k-profile", + "thiserror", +] + +[[package]] +name = "j2k-cuda-runtime" +version = "0.6.0" +dependencies = [ + "j2k-core", + "libloading", + "thiserror", +] + +[[package]] +name = "j2k-jpeg" +version = "0.6.0" +dependencies = [ + "j2k-core", + "j2k-profile", + "memchr", + "rayon", + "thiserror", +] + +[[package]] +name = "j2k-jpeg-cuda" +version = "0.6.0" +dependencies = [ + "j2k-core", + "j2k-cuda-runtime", + "j2k-jpeg", + "j2k-profile", + "thiserror", +] + +[[package]] +name = "j2k-metal-support" +version = "0.6.0" +dependencies = [ + "j2k-core", + "metal", +] + +[[package]] +name = "j2k-native" +version = "0.6.0" +dependencies = [ + "fearless_simd", + "j2k-profile", + "j2k-types", + "libm", + "rayon", +] + +[[package]] +name = "j2k-nvidia-baseline" +version = "0.6.0" +dependencies = [ + "j2k", + "j2k-core", + "j2k-cuda", + "j2k-jpeg", + "j2k-jpeg-cuda", + "j2k-native", + "j2k-transcode", + "j2k-transcode-cuda", + "j2k-transcode-metal", +] + +[[package]] +name = "j2k-profile" +version = "0.6.0" + +[[package]] +name = "j2k-transcode" +version = "0.6.0" +dependencies = [ + "j2k", + "j2k-jpeg", + "j2k-native", + "rayon", +] + +[[package]] +name = "j2k-transcode-cuda" +version = "0.6.0" +dependencies = [ + "j2k-cuda-runtime", + "j2k-native", + "j2k-transcode", +] + +[[package]] +name = "j2k-transcode-metal" +version = "0.6.0" +dependencies = [ + "j2k-core", + "j2k-metal-support", + "j2k-transcode", + "metal", + "rayon", +] + +[[package]] +name = "j2k-types" +version = "0.6.0" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "metal" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7047791b5bc903b8cd963014b355f71dc9864a9a0b727057676c1dcae5cbc15" +dependencies = [ + "bitflags", + "block", + "core-graphics-types", + "foreign-types", + "log", + "objc", + "paste", +] + +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" diff --git a/tests/nvidia-baseline/Cargo.toml b/tests/nvidia-baseline/Cargo.toml new file mode 100644 index 00000000..5e62e6c7 --- /dev/null +++ b/tests/nvidia-baseline/Cargo.toml @@ -0,0 +1,71 @@ +[package] +name = "j2k-nvidia-baseline" +description = "NVIDIA GPU codec baseline (nvJPEG + nvJPEG2000) for benchmarking j2k's coefficient-domain JPEG to HTJ2K transcode" +version = "0.6.0" +edition = "2021" +rust-version = "1.96" +license = "Apache-2.0" +repository = "https://github.com/frames-sg/j2k" +publish = false + +[lib] +name = "j2k_nvidia_baseline" +path = "src/lib.rs" + +[[bin]] +name = "transcode_compare" +path = "src/bin/transcode_compare.rs" + +[[bin]] +name = "decode_compare" +path = "src/bin/decode_compare.rs" + +[[bin]] +name = "jpeg_decode_compare" +path = "src/bin/jpeg_decode_compare.rs" + +[features] +default = [] +# Compile + link the C++ nvJPEG/nvJPEG2000 baseline. Runner-only: requires nvcc, +# libnvjpeg, and a separately-installed libnvjpeg2k. Also turns on the CUDA path +# for the j2k side of the comparison. +nvjpeg2000 = [ + "dep:j2k-jpeg-cuda", + "j2k-jpeg-cuda/cuda-runtime", + "j2k-transcode-cuda/cuda-runtime", + "dep:j2k-cuda", + "j2k-cuda/cuda-runtime", +] + +[dependencies] +j2k-core = { path = "../../crates/j2k-core", version = "=0.6.0" } +j2k-transcode = { path = "../../crates/j2k-transcode", version = "=0.6.0" } +j2k-transcode-cuda = { path = "../../crates/j2k-transcode-cuda", version = "=0.6.0" } +j2k-jpeg-cuda = { path = "../../crates/j2k-jpeg-cuda", version = "=0.6.0", optional = true } +j2k-cuda = { path = "../../crates/j2k-cuda", version = "=0.6.0", optional = true } +j2k-jpeg = { path = "../../crates/j2k-jpeg", version = "=0.6.0" } +j2k = { path = "../../crates/j2k", version = "=0.6.0" } +j2k-native = { path = "../../crates/j2k-native", version = "=0.6.0" } + +# Metal backend for the j2k side of the comparison on macOS (the runner is +# Linux/CUDA, so this is target-gated to keep the runner build untouched). +[target.'cfg(target_os = "macos")'.dependencies] +j2k-transcode-metal = { path = "../../crates/j2k-transcode-metal", version = "=0.6.0" } + +[lints.rust] +unsafe_code = "allow" +unsafe_op_in_unsafe_fn = "deny" + +[lints.clippy] +pedantic = { level = "warn", priority = -1 } +undocumented_unsafe_blocks = "warn" +module_name_repetitions = "allow" +must_use_candidate = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +cast_possible_truncation = "allow" +cast_precision_loss = "allow" +cast_sign_loss = "allow" + +[workspace] +resolver = "2" diff --git a/tests/nvidia-baseline/README.md b/tests/nvidia-baseline/README.md new file mode 100644 index 00000000..01ef764d --- /dev/null +++ b/tests/nvidia-baseline/README.md @@ -0,0 +1,12 @@ +# NVIDIA Baseline Artifacts + +This directory stores CUDA validation artifacts used for comparison and release +evidence. + +JSON and CSV artifacts are the source of truth. Do not hand-maintain benchmark +numbers in this README. Any public NVIDIA performance claim must cite the +recorded artifact, host, command, input source, comparator availability, +comparator version, and skipped paths. + +Hosted CI is not NVIDIA performance evidence. Use self-hosted CUDA runners for +runtime and benchmark evidence. diff --git a/tests/nvidia-baseline/artifacts/2026-06-03-cuda-transcode-rate-match/level1-rate-match.csv b/tests/nvidia-baseline/artifacts/2026-06-03-cuda-transcode-rate-match/level1-rate-match.csv new file mode 100644 index 00000000..1f771278 --- /dev/null +++ b/tests/nvidia-baseline/artifacts/2026-06-03-cuda-transcode-rate-match/level1-rate-match.csv @@ -0,0 +1,4 @@ +row,selected,ran,used_gpu,nvidia_ran,nvidia_status,scale,bytes,byte_delta_vs_nvidia,wall_ms,gpu_ms,mean_psnr,aggregate_psnr,psnr_r,psnr_g,psnr_b,transform_dispatches,transform_jobs,cpu_fallback_jobs,encode_dispatches,ht_codeblock_dispatches,packetization_dispatches + j2k_cuda_transform_cpu_ht,true,true,true,true,ok,1.900000,6199631,-0.00536860,64.452178,13.752000,47.636215,47.630379,47.653225,49.112234,46.509906,2,327,0,0,0,0 + j2k_cuda_transform_cuda_ht_block_cpu_packet,false,true,true,true,ok,1.900000,6199631,-0.00536860,15.605932,7.542000,47.636215,47.630379,47.653225,49.112234,46.509906,2,327,0,1,1,0 +nvidia_reused_session_serial,false,true,true,true,ok,,6233094,,131.716799,124.994976,49.196875,49.189384,48.946420,50.909822,48.152879,,,,,, diff --git a/tests/nvidia-baseline/artifacts/2026-06-03-cuda-transcode-rate-match/level1-rate-match.json b/tests/nvidia-baseline/artifacts/2026-06-03-cuda-transcode-rate-match/level1-rate-match.json new file mode 100644 index 00000000..641d3a8c --- /dev/null +++ b/tests/nvidia-baseline/artifacts/2026-06-03-cuda-transcode-rate-match/level1-rate-match.json @@ -0,0 +1,16 @@ +{ + "tile_count": 109, + "megapixels": 7.14342400, + "corpus_hash": "c1060319d3236928", + "match_nvidia_bytes": true, + "match_tolerance": 0.20000000, + "decomposition_levels": 1, + "subband_scales": {"ll": 1.000000, "hl": 1.000000, "lh": 1.000000, "hh": 1.000000}, + "inputs": [{"label": "tile_00000.jpg", "width": 256, "height": 256, "bytes": 20190}, {"label": "tile_00001.jpg", "width": 256, "height": 256, "bytes": 22300}, {"label": "tile_00002.jpg", "width": 256, "height": 256, "bytes": 19163}, {"label": "tile_00003.jpg", "width": 256, "height": 256, "bytes": 19322}, {"label": "tile_00004.jpg", "width": 256, "height": 256, "bytes": 16181}, {"label": "tile_00005.jpg", "width": 256, "height": 256, "bytes": 14566}, {"label": "tile_00006.jpg", "width": 256, "height": 256, "bytes": 19052}, {"label": "tile_00007.jpg", "width": 256, "height": 256, "bytes": 20056}, {"label": "tile_00008.jpg", "width": 256, "height": 256, "bytes": 20667}, {"label": "tile_00009.jpg", "width": 256, "height": 256, "bytes": 16497}, {"label": "tile_00010.jpg", "width": 256, "height": 256, "bytes": 18647}, {"label": "tile_00011.jpg", "width": 256, "height": 256, "bytes": 19063}, {"label": "tile_00012.jpg", "width": 256, "height": 256, "bytes": 18535}, {"label": "tile_00013.jpg", "width": 256, "height": 256, "bytes": 20271}, {"label": "tile_00014.jpg", "width": 256, "height": 256, "bytes": 18935}, {"label": "tile_00015.jpg", "width": 256, "height": 256, "bytes": 19423}, {"label": "tile_00016.jpg", "width": 256, "height": 256, "bytes": 17032}, {"label": "tile_00017.jpg", "width": 256, "height": 256, "bytes": 16609}, {"label": "tile_00018.jpg", "width": 256, "height": 256, "bytes": 19627}, {"label": "tile_00019.jpg", "width": 256, "height": 256, "bytes": 20515}, {"label": "tile_00020.jpg", "width": 256, "height": 256, "bytes": 18174}, {"label": "tile_00021.jpg", "width": 256, "height": 256, "bytes": 20613}, {"label": "tile_00022.jpg", "width": 256, "height": 256, "bytes": 17545}, {"label": "tile_00023.jpg", "width": 256, "height": 256, "bytes": 18004}, {"label": "tile_00024.jpg", "width": 256, "height": 256, "bytes": 21151}, {"label": "tile_00025.jpg", "width": 256, "height": 256, "bytes": 18044}, {"label": "tile_00026.jpg", "width": 256, "height": 256, "bytes": 21824}, {"label": "tile_00027.jpg", "width": 256, "height": 256, "bytes": 17576}, {"label": "tile_00028.jpg", "width": 256, "height": 256, "bytes": 18375}, {"label": "tile_00029.jpg", "width": 256, "height": 256, "bytes": 21415}, {"label": "tile_00030.jpg", "width": 256, "height": 256, "bytes": 19269}, {"label": "tile_00031.jpg", "width": 256, "height": 256, "bytes": 20895}, {"label": "tile_00032.jpg", "width": 256, "height": 256, "bytes": 19093}, {"label": "tile_00033.jpg", "width": 256, "height": 256, "bytes": 20920}, {"label": "tile_00034.jpg", "width": 256, "height": 256, "bytes": 21106}, {"label": "tile_00035.jpg", "width": 256, "height": 256, "bytes": 14760}, {"label": "tile_00036.jpg", "width": 256, "height": 256, "bytes": 18875}, {"label": "tile_00037.jpg", "width": 256, "height": 256, "bytes": 19788}, {"label": "tile_00038.jpg", "width": 256, "height": 256, "bytes": 19704}, {"label": "tile_00039.jpg", "width": 256, "height": 256, "bytes": 16928}, {"label": "tile_00040.jpg", "width": 256, "height": 256, "bytes": 20476}, {"label": "tile_00041.jpg", "width": 256, "height": 256, "bytes": 20679}, {"label": "tile_00042.jpg", "width": 256, "height": 256, "bytes": 20781}, {"label": "tile_00043.jpg", "width": 256, "height": 256, "bytes": 20690}, {"label": "tile_00044.jpg", "width": 256, "height": 256, "bytes": 21193}, {"label": "tile_00045.jpg", "width": 256, "height": 256, "bytes": 21368}, {"label": "tile_00046.jpg", "width": 256, "height": 256, "bytes": 20698}, {"label": "tile_00047.jpg", "width": 256, "height": 256, "bytes": 21281}, {"label": "tile_00048.jpg", "width": 256, "height": 256, "bytes": 20253}, {"label": "tile_00049.jpg", "width": 256, "height": 256, "bytes": 20559}, {"label": "tile_00050.jpg", "width": 256, "height": 256, "bytes": 21219}, {"label": "tile_00051.jpg", "width": 256, "height": 256, "bytes": 19028}, {"label": "tile_00052.jpg", "width": 256, "height": 256, "bytes": 19466}, {"label": "tile_00053.jpg", "width": 256, "height": 256, "bytes": 18302}, {"label": "tile_00054.jpg", "width": 256, "height": 256, "bytes": 21462}, {"label": "tile_00055.jpg", "width": 256, "height": 256, "bytes": 17709}, {"label": "tile_00056.jpg", "width": 256, "height": 256, "bytes": 18178}, {"label": "tile_00057.jpg", "width": 256, "height": 256, "bytes": 21310}, {"label": "tile_00058.jpg", "width": 256, "height": 256, "bytes": 19556}, {"label": "tile_00059.jpg", "width": 256, "height": 256, "bytes": 19006}, {"label": "tile_00060.jpg", "width": 256, "height": 256, "bytes": 21692}, {"label": "tile_00061.jpg", "width": 256, "height": 256, "bytes": 17022}, {"label": "tile_00062.jpg", "width": 256, "height": 256, "bytes": 19592}, {"label": "tile_00063.jpg", "width": 256, "height": 256, "bytes": 21448}, {"label": "tile_00064.jpg", "width": 256, "height": 256, "bytes": 19357}, {"label": "tile_00065.jpg", "width": 256, "height": 256, "bytes": 18848}, {"label": "tile_00066.jpg", "width": 256, "height": 256, "bytes": 18691}, {"label": "tile_00067.jpg", "width": 256, "height": 256, "bytes": 20260}, {"label": "tile_00068.jpg", "width": 256, "height": 256, "bytes": 19812}, {"label": "tile_00069.jpg", "width": 256, "height": 256, "bytes": 19342}, {"label": "tile_00070.jpg", "width": 256, "height": 256, "bytes": 16521}, {"label": "tile_00071.jpg", "width": 256, "height": 256, "bytes": 12121}, {"label": "tile_00072.jpg", "width": 256, "height": 256, "bytes": 19443}, {"label": "tile_00073.jpg", "width": 256, "height": 256, "bytes": 18292}, {"label": "tile_00074.jpg", "width": 256, "height": 256, "bytes": 15983}, {"label": "tile_00075.jpg", "width": 256, "height": 256, "bytes": 18007}, {"label": "tile_00076.jpg", "width": 256, "height": 256, "bytes": 19589}, {"label": "tile_00077.jpg", "width": 256, "height": 256, "bytes": 19470}, {"label": "tile_00078.jpg", "width": 256, "height": 256, "bytes": 8466}, {"label": "tile_00079.jpg", "width": 256, "height": 256, "bytes": 20408}, {"label": "tile_00080.jpg", "width": 256, "height": 256, "bytes": 17654}, {"label": "tile_00081.jpg", "width": 256, "height": 256, "bytes": 18698}, {"label": "tile_00082.jpg", "width": 256, "height": 256, "bytes": 21090}, {"label": "tile_00083.jpg", "width": 256, "height": 256, "bytes": 17601}, {"label": "tile_00084.jpg", "width": 256, "height": 256, "bytes": 14106}, {"label": "tile_00085.jpg", "width": 256, "height": 256, "bytes": 19344}, {"label": "tile_00086.jpg", "width": 256, "height": 256, "bytes": 19090}, {"label": "tile_00087.jpg", "width": 256, "height": 256, "bytes": 20533}, {"label": "tile_00088.jpg", "width": 256, "height": 256, "bytes": 17022}, {"label": "tile_00089.jpg", "width": 256, "height": 256, "bytes": 21328}, {"label": "tile_00090.jpg", "width": 256, "height": 256, "bytes": 20581}, {"label": "tile_00091.jpg", "width": 256, "height": 256, "bytes": 19934}, {"label": "tile_00092.jpg", "width": 256, "height": 256, "bytes": 19688}, {"label": "tile_00093.jpg", "width": 256, "height": 256, "bytes": 18445}, {"label": "tile_00094.jpg", "width": 256, "height": 256, "bytes": 20128}, {"label": "tile_00095.jpg", "width": 256, "height": 256, "bytes": 19861}, {"label": "tile_00096.jpg", "width": 256, "height": 256, "bytes": 20382}, {"label": "tile_00097.jpg", "width": 256, "height": 256, "bytes": 18086}, {"label": "tile_00098.jpg", "width": 256, "height": 256, "bytes": 20428}, {"label": "tile_00099.jpg", "width": 256, "height": 256, "bytes": 19335}, {"label": "tile_00100.jpg", "width": 256, "height": 256, "bytes": 19891}, {"label": "tile_00101.jpg", "width": 256, "height": 256, "bytes": 19610}, {"label": "tile_00102.jpg", "width": 256, "height": 256, "bytes": 17736}, {"label": "tile_00103.jpg", "width": 256, "height": 256, "bytes": 19592}, {"label": "tile_00104.jpg", "width": 256, "height": 256, "bytes": 17629}, {"label": "tile_00105.jpg", "width": 256, "height": 256, "bytes": 18353}, {"label": "tile_00106.jpg", "width": 256, "height": 256, "bytes": 19822}, {"label": "tile_00107.jpg", "width": 256, "height": 256, "bytes": 19421}, {"label": "tile_00108.jpg", "width": 256, "height": 256, "bytes": 21009}], + "selected_rd_index": 0, + "rd_points": [ + {"scale": 1.900000, "ran": true, "used_gpu": true, "bytes": 6199631, "byte_delta_vs_nvidia": -0.00536860, "wall_ms": 64.452178, "gpu_ms": 13.752000, "mean_psnr": 47.63621499, "aggregate_psnr": 47.63037930, "channel_psnr": {"r": 47.65322486, "g": 49.11223417, "b": 46.50990647}, "transform_dispatches": 2, "transform_jobs": 327, "cpu_fallback_jobs": 0, "encode_dispatches": 0, "ht_codeblock_dispatches": 0, "packetization_dispatches": 0, "per_tile_psnr": [47.57054392, 47.27840491, 47.64166094, 47.65007682, 47.98939549, 48.23258189, 47.65549847, 47.57349614, 47.52824917, 47.95837478, 47.71616996, 47.56786536, 47.74424922, 47.53827838, 47.67112851, 47.59175897, 47.87028017, 47.91551229, 47.63296588, 47.50866191, 47.63513309, 47.55554197, 47.81714228, 47.70169780, 47.37541790, 47.63623681, 47.31693213, 47.57827833, 47.64839629, 47.39236166, 47.60121255, 47.46588219, 47.54360076, 47.32567697, 47.35175914, 48.08459637, 47.62128161, 47.43585009, 47.52392509, 47.73234272, 47.42849655, 47.48280274, 47.42556612, 47.42033010, 47.32028499, 47.39734017, 47.37801747, 47.51076795, 47.33355689, 47.64245047, 47.40897892, 47.54171018, 47.70131761, 47.62281412, 47.25041296, 47.85777465, 47.72424881, 47.51419724, 47.55583221, 47.63334014, 47.40884798, 47.87554658, 47.53167305, 47.39836641, 47.63745911, 47.75772878, 47.69759745, 47.48139493, 47.58125527, 47.71111325, 48.03288912, 48.40699299, 47.69609828, 47.76016198, 48.00046392, 47.70195796, 47.67160544, 47.59980493, 48.83432286, 47.52313761, 47.84137686, 47.66031240, 47.51580745, 47.78320294, 48.13683819, 47.69210305, 47.63121320, 47.51878024, 47.73843315, 47.38323990, 47.55900681, 47.61817896, 47.58513030, 47.77810836, 47.59931628, 47.57857009, 47.50294246, 47.77145366, 47.52644214, 47.64227281, 47.48950561, 47.61425473, 47.78838516, 47.58924278, 47.87831765, 47.64821839, 47.59685432, 47.64067423, 47.40217496]} + ], + "j2k_cuda_ht_experimental": {"scale": 1.900000, "ran": true, "used_gpu": true, "bytes": 6199631, "byte_delta_vs_nvidia": -0.00536860, "wall_ms": 15.605932, "gpu_ms": 7.542000, "mean_psnr": 47.63621499, "aggregate_psnr": 47.63037930, "channel_psnr": {"r": 47.65322486, "g": 49.11223417, "b": 46.50990647}, "transform_dispatches": 2, "transform_jobs": 327, "cpu_fallback_jobs": 0, "encode_dispatches": 1, "ht_codeblock_dispatches": 1, "packetization_dispatches": 0, "per_tile_psnr": [47.57054392, 47.27840491, 47.64166094, 47.65007682, 47.98939549, 48.23258189, 47.65549847, 47.57349614, 47.52824917, 47.95837478, 47.71616996, 47.56786536, 47.74424922, 47.53827838, 47.67112851, 47.59175897, 47.87028017, 47.91551229, 47.63296588, 47.50866191, 47.63513309, 47.55554197, 47.81714228, 47.70169780, 47.37541790, 47.63623681, 47.31693213, 47.57827833, 47.64839629, 47.39236166, 47.60121255, 47.46588219, 47.54360076, 47.32567697, 47.35175914, 48.08459637, 47.62128161, 47.43585009, 47.52392509, 47.73234272, 47.42849655, 47.48280274, 47.42556612, 47.42033010, 47.32028499, 47.39734017, 47.37801747, 47.51076795, 47.33355689, 47.64245047, 47.40897892, 47.54171018, 47.70131761, 47.62281412, 47.25041296, 47.85777465, 47.72424881, 47.51419724, 47.55583221, 47.63334014, 47.40884798, 47.87554658, 47.53167305, 47.39836641, 47.63745911, 47.75772878, 47.69759745, 47.48139493, 47.58125527, 47.71111325, 48.03288912, 48.40699299, 47.69609828, 47.76016198, 48.00046392, 47.70195796, 47.67160544, 47.59980493, 48.83432286, 47.52313761, 47.84137686, 47.66031240, 47.51580745, 47.78320294, 48.13683819, 47.69210305, 47.63121320, 47.51878024, 47.73843315, 47.38323990, 47.55900681, 47.61817896, 47.58513030, 47.77810836, 47.59931628, 47.57857009, 47.50294246, 47.77145366, 47.52644214, 47.64227281, 47.48950561, 47.61425473, 47.78838516, 47.58924278, 47.87831765, 47.64821839, 47.59685432, 47.64067423, 47.40217496]}, + "nvidia_reused_session_serial": {"ran": true, "status": "ok", "bytes": 6233094, "wall_ms": 131.716799, "gpu_ms": 124.994976, "decode_ms": 30.554400, "encode_ms": 94.440576, "mean_psnr": 49.19687526, "aggregate_psnr": 49.18938431, "channel_psnr": {"r": 48.94642048, "g": 50.90982177, "b": 48.15287886}} +} diff --git a/tests/nvidia-baseline/artifacts/2026-06-03-cuda-transcode-rate-match/level2-rate-match.csv b/tests/nvidia-baseline/artifacts/2026-06-03-cuda-transcode-rate-match/level2-rate-match.csv new file mode 100644 index 00000000..87aa7059 --- /dev/null +++ b/tests/nvidia-baseline/artifacts/2026-06-03-cuda-transcode-rate-match/level2-rate-match.csv @@ -0,0 +1,4 @@ +row,selected,ran,used_gpu,nvidia_ran,nvidia_status,scale,bytes,byte_delta_vs_nvidia,wall_ms,gpu_ms,mean_psnr,aggregate_psnr,psnr_r,psnr_g,psnr_b,transform_dispatches,transform_jobs,cpu_fallback_jobs,encode_dispatches,ht_codeblock_dispatches,packetization_dispatches + j2k_cuda_transform_cpu_ht,true,true,true,true,ok,1.000000,6241303,0.00131700,92.995812,18.824000,48.696854,48.693574,48.613040,50.470408,47.497531,2,327,0,0,0,0 + j2k_cuda_transform_cuda_ht_block_cpu_packet,false,true,true,true,ok,1.000000,6241303,0.00131700,102.621355,24.537000,48.696854,48.693574,48.613040,50.470408,47.497531,2,327,0,1,1,0 +nvidia_reused_session_serial,false,true,true,true,ok,,6233094,,128.587843,125.970592,49.196875,49.189384,48.946420,50.909822,48.152879,,,,,, diff --git a/tests/nvidia-baseline/artifacts/2026-06-03-cuda-transcode-rate-match/level2-rate-match.json b/tests/nvidia-baseline/artifacts/2026-06-03-cuda-transcode-rate-match/level2-rate-match.json new file mode 100644 index 00000000..b1e688fc --- /dev/null +++ b/tests/nvidia-baseline/artifacts/2026-06-03-cuda-transcode-rate-match/level2-rate-match.json @@ -0,0 +1,16 @@ +{ + "tile_count": 109, + "megapixels": 7.14342400, + "corpus_hash": "c1060319d3236928", + "match_nvidia_bytes": true, + "match_tolerance": 0.20000000, + "decomposition_levels": 2, + "subband_scales": {"ll": 1.000000, "hl": 1.000000, "lh": 1.000000, "hh": 1.000000}, + "inputs": [{"label": "tile_00000.jpg", "width": 256, "height": 256, "bytes": 20190}, {"label": "tile_00001.jpg", "width": 256, "height": 256, "bytes": 22300}, {"label": "tile_00002.jpg", "width": 256, "height": 256, "bytes": 19163}, {"label": "tile_00003.jpg", "width": 256, "height": 256, "bytes": 19322}, {"label": "tile_00004.jpg", "width": 256, "height": 256, "bytes": 16181}, {"label": "tile_00005.jpg", "width": 256, "height": 256, "bytes": 14566}, {"label": "tile_00006.jpg", "width": 256, "height": 256, "bytes": 19052}, {"label": "tile_00007.jpg", "width": 256, "height": 256, "bytes": 20056}, {"label": "tile_00008.jpg", "width": 256, "height": 256, "bytes": 20667}, {"label": "tile_00009.jpg", "width": 256, "height": 256, "bytes": 16497}, {"label": "tile_00010.jpg", "width": 256, "height": 256, "bytes": 18647}, {"label": "tile_00011.jpg", "width": 256, "height": 256, "bytes": 19063}, {"label": "tile_00012.jpg", "width": 256, "height": 256, "bytes": 18535}, {"label": "tile_00013.jpg", "width": 256, "height": 256, "bytes": 20271}, {"label": "tile_00014.jpg", "width": 256, "height": 256, "bytes": 18935}, {"label": "tile_00015.jpg", "width": 256, "height": 256, "bytes": 19423}, {"label": "tile_00016.jpg", "width": 256, "height": 256, "bytes": 17032}, {"label": "tile_00017.jpg", "width": 256, "height": 256, "bytes": 16609}, {"label": "tile_00018.jpg", "width": 256, "height": 256, "bytes": 19627}, {"label": "tile_00019.jpg", "width": 256, "height": 256, "bytes": 20515}, {"label": "tile_00020.jpg", "width": 256, "height": 256, "bytes": 18174}, {"label": "tile_00021.jpg", "width": 256, "height": 256, "bytes": 20613}, {"label": "tile_00022.jpg", "width": 256, "height": 256, "bytes": 17545}, {"label": "tile_00023.jpg", "width": 256, "height": 256, "bytes": 18004}, {"label": "tile_00024.jpg", "width": 256, "height": 256, "bytes": 21151}, {"label": "tile_00025.jpg", "width": 256, "height": 256, "bytes": 18044}, {"label": "tile_00026.jpg", "width": 256, "height": 256, "bytes": 21824}, {"label": "tile_00027.jpg", "width": 256, "height": 256, "bytes": 17576}, {"label": "tile_00028.jpg", "width": 256, "height": 256, "bytes": 18375}, {"label": "tile_00029.jpg", "width": 256, "height": 256, "bytes": 21415}, {"label": "tile_00030.jpg", "width": 256, "height": 256, "bytes": 19269}, {"label": "tile_00031.jpg", "width": 256, "height": 256, "bytes": 20895}, {"label": "tile_00032.jpg", "width": 256, "height": 256, "bytes": 19093}, {"label": "tile_00033.jpg", "width": 256, "height": 256, "bytes": 20920}, {"label": "tile_00034.jpg", "width": 256, "height": 256, "bytes": 21106}, {"label": "tile_00035.jpg", "width": 256, "height": 256, "bytes": 14760}, {"label": "tile_00036.jpg", "width": 256, "height": 256, "bytes": 18875}, {"label": "tile_00037.jpg", "width": 256, "height": 256, "bytes": 19788}, {"label": "tile_00038.jpg", "width": 256, "height": 256, "bytes": 19704}, {"label": "tile_00039.jpg", "width": 256, "height": 256, "bytes": 16928}, {"label": "tile_00040.jpg", "width": 256, "height": 256, "bytes": 20476}, {"label": "tile_00041.jpg", "width": 256, "height": 256, "bytes": 20679}, {"label": "tile_00042.jpg", "width": 256, "height": 256, "bytes": 20781}, {"label": "tile_00043.jpg", "width": 256, "height": 256, "bytes": 20690}, {"label": "tile_00044.jpg", "width": 256, "height": 256, "bytes": 21193}, {"label": "tile_00045.jpg", "width": 256, "height": 256, "bytes": 21368}, {"label": "tile_00046.jpg", "width": 256, "height": 256, "bytes": 20698}, {"label": "tile_00047.jpg", "width": 256, "height": 256, "bytes": 21281}, {"label": "tile_00048.jpg", "width": 256, "height": 256, "bytes": 20253}, {"label": "tile_00049.jpg", "width": 256, "height": 256, "bytes": 20559}, {"label": "tile_00050.jpg", "width": 256, "height": 256, "bytes": 21219}, {"label": "tile_00051.jpg", "width": 256, "height": 256, "bytes": 19028}, {"label": "tile_00052.jpg", "width": 256, "height": 256, "bytes": 19466}, {"label": "tile_00053.jpg", "width": 256, "height": 256, "bytes": 18302}, {"label": "tile_00054.jpg", "width": 256, "height": 256, "bytes": 21462}, {"label": "tile_00055.jpg", "width": 256, "height": 256, "bytes": 17709}, {"label": "tile_00056.jpg", "width": 256, "height": 256, "bytes": 18178}, {"label": "tile_00057.jpg", "width": 256, "height": 256, "bytes": 21310}, {"label": "tile_00058.jpg", "width": 256, "height": 256, "bytes": 19556}, {"label": "tile_00059.jpg", "width": 256, "height": 256, "bytes": 19006}, {"label": "tile_00060.jpg", "width": 256, "height": 256, "bytes": 21692}, {"label": "tile_00061.jpg", "width": 256, "height": 256, "bytes": 17022}, {"label": "tile_00062.jpg", "width": 256, "height": 256, "bytes": 19592}, {"label": "tile_00063.jpg", "width": 256, "height": 256, "bytes": 21448}, {"label": "tile_00064.jpg", "width": 256, "height": 256, "bytes": 19357}, {"label": "tile_00065.jpg", "width": 256, "height": 256, "bytes": 18848}, {"label": "tile_00066.jpg", "width": 256, "height": 256, "bytes": 18691}, {"label": "tile_00067.jpg", "width": 256, "height": 256, "bytes": 20260}, {"label": "tile_00068.jpg", "width": 256, "height": 256, "bytes": 19812}, {"label": "tile_00069.jpg", "width": 256, "height": 256, "bytes": 19342}, {"label": "tile_00070.jpg", "width": 256, "height": 256, "bytes": 16521}, {"label": "tile_00071.jpg", "width": 256, "height": 256, "bytes": 12121}, {"label": "tile_00072.jpg", "width": 256, "height": 256, "bytes": 19443}, {"label": "tile_00073.jpg", "width": 256, "height": 256, "bytes": 18292}, {"label": "tile_00074.jpg", "width": 256, "height": 256, "bytes": 15983}, {"label": "tile_00075.jpg", "width": 256, "height": 256, "bytes": 18007}, {"label": "tile_00076.jpg", "width": 256, "height": 256, "bytes": 19589}, {"label": "tile_00077.jpg", "width": 256, "height": 256, "bytes": 19470}, {"label": "tile_00078.jpg", "width": 256, "height": 256, "bytes": 8466}, {"label": "tile_00079.jpg", "width": 256, "height": 256, "bytes": 20408}, {"label": "tile_00080.jpg", "width": 256, "height": 256, "bytes": 17654}, {"label": "tile_00081.jpg", "width": 256, "height": 256, "bytes": 18698}, {"label": "tile_00082.jpg", "width": 256, "height": 256, "bytes": 21090}, {"label": "tile_00083.jpg", "width": 256, "height": 256, "bytes": 17601}, {"label": "tile_00084.jpg", "width": 256, "height": 256, "bytes": 14106}, {"label": "tile_00085.jpg", "width": 256, "height": 256, "bytes": 19344}, {"label": "tile_00086.jpg", "width": 256, "height": 256, "bytes": 19090}, {"label": "tile_00087.jpg", "width": 256, "height": 256, "bytes": 20533}, {"label": "tile_00088.jpg", "width": 256, "height": 256, "bytes": 17022}, {"label": "tile_00089.jpg", "width": 256, "height": 256, "bytes": 21328}, {"label": "tile_00090.jpg", "width": 256, "height": 256, "bytes": 20581}, {"label": "tile_00091.jpg", "width": 256, "height": 256, "bytes": 19934}, {"label": "tile_00092.jpg", "width": 256, "height": 256, "bytes": 19688}, {"label": "tile_00093.jpg", "width": 256, "height": 256, "bytes": 18445}, {"label": "tile_00094.jpg", "width": 256, "height": 256, "bytes": 20128}, {"label": "tile_00095.jpg", "width": 256, "height": 256, "bytes": 19861}, {"label": "tile_00096.jpg", "width": 256, "height": 256, "bytes": 20382}, {"label": "tile_00097.jpg", "width": 256, "height": 256, "bytes": 18086}, {"label": "tile_00098.jpg", "width": 256, "height": 256, "bytes": 20428}, {"label": "tile_00099.jpg", "width": 256, "height": 256, "bytes": 19335}, {"label": "tile_00100.jpg", "width": 256, "height": 256, "bytes": 19891}, {"label": "tile_00101.jpg", "width": 256, "height": 256, "bytes": 19610}, {"label": "tile_00102.jpg", "width": 256, "height": 256, "bytes": 17736}, {"label": "tile_00103.jpg", "width": 256, "height": 256, "bytes": 19592}, {"label": "tile_00104.jpg", "width": 256, "height": 256, "bytes": 17629}, {"label": "tile_00105.jpg", "width": 256, "height": 256, "bytes": 18353}, {"label": "tile_00106.jpg", "width": 256, "height": 256, "bytes": 19822}, {"label": "tile_00107.jpg", "width": 256, "height": 256, "bytes": 19421}, {"label": "tile_00108.jpg", "width": 256, "height": 256, "bytes": 21009}], + "selected_rd_index": 0, + "rd_points": [ + {"scale": 1.000000, "ran": true, "used_gpu": true, "bytes": 6241303, "byte_delta_vs_nvidia": 0.00131700, "wall_ms": 92.995812, "gpu_ms": 18.824000, "mean_psnr": 48.69685415, "aggregate_psnr": 48.69357395, "channel_psnr": {"r": 48.61303960, "g": 50.47040845, "b": 47.49753075}, "transform_dispatches": 2, "transform_jobs": 327, "cpu_fallback_jobs": 0, "encode_dispatches": 0, "ht_codeblock_dispatches": 0, "packetization_dispatches": 0, "per_tile_psnr": [48.58980719, 48.48983776, 48.76823156, 48.71822732, 49.18802821, 49.12403147, 48.62872379, 48.64794039, 48.58193321, 48.77565653, 48.73713374, 48.59285269, 48.67472189, 48.63087956, 48.67654995, 48.58031602, 48.77091844, 48.83180413, 48.67419616, 48.61309553, 48.63814767, 48.62988827, 48.70358462, 48.75545961, 48.55717796, 48.65314399, 48.49396654, 48.67557322, 48.68861435, 48.52433253, 48.64823900, 48.58063451, 48.63320999, 48.52455019, 48.53360506, 48.92449849, 48.70156880, 48.56930580, 48.70489540, 48.74104694, 48.54710130, 48.57351086, 48.56185915, 48.58220280, 48.50266902, 48.56498275, 48.56796201, 48.58208025, 48.56198112, 48.70454246, 48.55820154, 48.70310578, 48.72784765, 48.74165711, 48.50791817, 48.92807999, 48.72850666, 48.60495750, 48.66159754, 48.67742669, 48.51691424, 48.88973932, 48.63390440, 48.53459885, 48.64803992, 48.79867768, 48.71016772, 48.57818520, 48.67354533, 48.76273502, 49.07233554, 49.40855780, 48.70978911, 48.73893744, 48.97263052, 48.74776351, 48.68853900, 48.64321526, 49.65714939, 48.62602440, 48.82381620, 48.69552697, 48.61709615, 48.86426974, 48.94701813, 48.70920864, 48.67629948, 48.57121224, 48.87359038, 48.53268418, 48.62916973, 48.64619896, 48.69658369, 48.71039491, 48.69115189, 48.68286641, 48.62379680, 48.83068805, 48.61324364, 48.66671757, 48.56486069, 48.65750586, 48.78440355, 48.68216413, 48.82791207, 48.72650459, 48.66446901, 48.65648354, 48.51529695]} + ], + "j2k_cuda_ht_experimental": {"scale": 1.000000, "ran": true, "used_gpu": true, "bytes": 6241303, "byte_delta_vs_nvidia": 0.00131700, "wall_ms": 102.621355, "gpu_ms": 24.537000, "mean_psnr": 48.69685415, "aggregate_psnr": 48.69357395, "channel_psnr": {"r": 48.61303960, "g": 50.47040845, "b": 47.49753075}, "transform_dispatches": 2, "transform_jobs": 327, "cpu_fallback_jobs": 0, "encode_dispatches": 1, "ht_codeblock_dispatches": 1, "packetization_dispatches": 0, "per_tile_psnr": [48.58980719, 48.48983776, 48.76823156, 48.71822732, 49.18802821, 49.12403147, 48.62872379, 48.64794039, 48.58193321, 48.77565653, 48.73713374, 48.59285269, 48.67472189, 48.63087956, 48.67654995, 48.58031602, 48.77091844, 48.83180413, 48.67419616, 48.61309553, 48.63814767, 48.62988827, 48.70358462, 48.75545961, 48.55717796, 48.65314399, 48.49396654, 48.67557322, 48.68861435, 48.52433253, 48.64823900, 48.58063451, 48.63320999, 48.52455019, 48.53360506, 48.92449849, 48.70156880, 48.56930580, 48.70489540, 48.74104694, 48.54710130, 48.57351086, 48.56185915, 48.58220280, 48.50266902, 48.56498275, 48.56796201, 48.58208025, 48.56198112, 48.70454246, 48.55820154, 48.70310578, 48.72784765, 48.74165711, 48.50791817, 48.92807999, 48.72850666, 48.60495750, 48.66159754, 48.67742669, 48.51691424, 48.88973932, 48.63390440, 48.53459885, 48.64803992, 48.79867768, 48.71016772, 48.57818520, 48.67354533, 48.76273502, 49.07233554, 49.40855780, 48.70978911, 48.73893744, 48.97263052, 48.74776351, 48.68853900, 48.64321526, 49.65714939, 48.62602440, 48.82381620, 48.69552697, 48.61709615, 48.86426974, 48.94701813, 48.70920864, 48.67629948, 48.57121224, 48.87359038, 48.53268418, 48.62916973, 48.64619896, 48.69658369, 48.71039491, 48.69115189, 48.68286641, 48.62379680, 48.83068805, 48.61324364, 48.66671757, 48.56486069, 48.65750586, 48.78440355, 48.68216413, 48.82791207, 48.72650459, 48.66446901, 48.65648354, 48.51529695]}, + "nvidia_reused_session_serial": {"ran": true, "status": "ok", "bytes": 6233094, "wall_ms": 128.587843, "gpu_ms": 125.970592, "decode_ms": 31.422496, "encode_ms": 94.548096, "mean_psnr": 49.19687526, "aggregate_psnr": 49.18938431, "channel_psnr": {"r": 48.94642048, "g": 50.90982177, "b": 48.15287886}} +} diff --git a/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch-correctness/j2k_decode_batch_correctness.csv b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch-correctness/j2k_decode_batch_correctness.csv new file mode 100644 index 00000000..a6dce846 --- /dev/null +++ b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch-correctness/j2k_decode_batch_correctness.csv @@ -0,0 +1,66 @@ +row_type,timing_scope,execution_mode,download_policy,input_count,label,status,format,width,height,megapixels,codestream_bytes,mp_s,wall_ms,gpu_ms,stage_ms,download_ms,cpu_status,cpu_wall_ms,nvidia_status,nvidia_wall_ms,nvidia_gpu_ms, j2k_cuda_psnr_vs_cpu,nvidia_psnr_vs_cpu,cuda_parse_us,cuda_plan_us,cuda_flatten_us,cuda_h2d_us,cuda_ht_cleanup_us,cuda_ht_refine_us,cuda_dequant_us,cuda_idwt_us,cuda_mct_us,cuda_store_us,cuda_total_us,cuda_wall_total_us,cuda_table_upload_us,cuda_payload_upload_us,cuda_status_d2h_us,cuda_output_d2h_us,cuda_block_count,cuda_payload_bytes,cuda_dispatch_count,cuda_ht_dispatch_count,cuda_dequant_dispatch_count,cuda_idwt_dispatch_count,cuda_mct_dispatch_count,cuda_store_dispatch_count +aggregate,aggregate_batch_correctness, j2k_cuda_batch,download,64, j2k_cuda_decode_batch_correctness,ok,,,,4.194304,3755036,453.544032,9.247843,3.748000,6.080000,1.080402,,,,,,,,11,1527,372,422,1812,0,0,1103,0,833,6080,8158,0,409,57,0,4800,3729641,12,1,0,10,0,1 +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00000.jpg,ok,rgb8,256,256,0.065536,59977,,,,,,ok,3.918700,ok,0.762592,0.670720,95.046216,51.298595,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00001.jpg,ok,rgb8,256,256,0.065536,68326,,,,,,ok,3.886764,ok,0.774710,0.683776,96.295603,50.988826,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00002.jpg,ok,rgb8,256,256,0.065536,57281,,,,,,ok,3.911964,ok,0.745802,0.654176,93.285303,51.584887,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00003.jpg,ok,rgb8,256,256,0.065536,57771,,,,,,ok,3.841815,ok,0.764488,0.670528,96.295603,51.414093,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00004.jpg,ok,rgb8,256,256,0.065536,50939,,,,,,ok,3.939564,ok,0.735985,0.640768,91.524390,52.021932,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00005.jpg,ok,rgb8,256,256,0.065536,42188,,,,,,ok,3.954695,ok,0.722700,0.628928,90.275003,52.429662,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00006.jpg,ok,rgb8,256,256,0.065536,58455,,,,,,ok,3.876127,ok,0.797595,0.706560,94.077115,51.365021,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00007.jpg,ok,rgb8,256,256,0.065536,61251,,,,,,ok,3.864171,ok,0.829071,0.740416,96.295603,51.279397,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00008.jpg,ok,rgb8,256,256,0.065536,61623,,,,,,ok,3.932927,ok,0.771531,0.680960,98.056516,51.261782,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00009.jpg,ok,rgb8,256,256,0.065536,49390,,,,,,ok,3.897035,ok,0.769703,0.651072,95.046216,51.781116,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00010.jpg,ok,rgb8,256,256,0.065536,54832,,,,,,ok,3.890191,ok,0.770000,0.665344,92.615835,51.608733,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00011.jpg,ok,rgb8,256,256,0.065536,56468,,,,,,ok,3.874446,ok,0.825002,0.715776,96.295603,51.515713,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00012.jpg,ok,rgb8,256,256,0.065536,55276,,,,,,ok,3.883824,ok,0.764874,0.653312,94.077115,51.527866,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00013.jpg,ok,rgb8,256,256,0.065536,59796,,,,,,ok,3.832965,ok,0.818484,0.729216,95.046216,51.339790,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00014.jpg,ok,rgb8,256,256,0.065536,56580,,,,,,ok,3.813292,ok,0.759777,0.669728,101.066815,51.489977,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00015.jpg,ok,rgb8,256,256,0.065536,61771,,,,,,ok,3.799835,ok,0.753812,0.662688,98.056516,51.193192,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00016.jpg,ok,rgb8,256,256,0.065536,51496,,,,,,ok,3.898438,ok,0.807314,0.699392,94.077115,51.765530,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00017.jpg,ok,rgb8,256,256,0.065536,49306,,,,,,ok,3.803001,ok,0.743708,0.637056,98.056516,51.879856,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00018.jpg,ok,rgb8,256,256,0.065536,59128,,,,,,ok,3.858689,ok,0.773387,0.667712,95.046216,51.395550,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00019.jpg,ok,rgb8,256,256,0.065536,61116,,,,,,ok,3.900384,ok,0.790563,0.694272,95.046216,51.246366,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00020.jpg,ok,rgb8,256,256,0.065536,57644,,,,,,ok,3.957392,ok,0.745308,0.654496,93.285303,51.367627,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00021.jpg,ok,rgb8,256,256,0.065536,60265,,,,,,ok,3.890122,ok,0.850030,0.762880,96.295603,51.297495,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00022.jpg,ok,rgb8,256,256,0.065536,53721,,,,,,ok,3.773820,ok,0.949348,0.840480,93.285303,51.567307,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00023.jpg,ok,rgb8,256,256,0.065536,53941,,,,,,ok,3.800833,ok,0.937309,0.837568,93.285303,51.643460,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00024.jpg,ok,rgb8,256,256,0.065536,64688,,,,,,ok,3.892117,ok,0.785546,0.674016,96.295603,51.135055,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00025.jpg,ok,rgb8,256,256,0.065536,56960,,,,,,ok,3.861549,ok,0.810089,0.703488,94.077115,51.431723,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00026.jpg,ok,rgb8,256,256,0.065536,66912,,,,,,ok,3.827242,ok,0.779708,0.672768,95.046216,51.048873,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00027.jpg,ok,rgb8,256,256,0.065536,54363,,,,,,ok,3.813736,ok,0.812499,0.718848,96.295603,51.600623,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00028.jpg,ok,rgb8,256,256,0.065536,55832,,,,,,ok,3.761696,ok,0.791136,0.702464,95.046216,51.523425,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00029.jpg,ok,rgb8,256,256,0.065536,64814,,,,,,ok,3.929608,ok,0.830968,0.742400,98.056516,51.130470,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00030.jpg,ok,rgb8,256,256,0.065536,60022,,,,,,ok,3.878310,ok,0.776558,0.685120,98.056516,51.267191,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00031.jpg,ok,rgb8,256,256,0.065536,63236,,,,,,ok,3.865095,ok,0.711896,0.601120,98.056516,51.202816,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00032.jpg,ok,rgb8,256,256,0.065536,59202,,,,,,ok,3.875618,ok,0.773605,0.664608,98.056516,51.356842,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00033.jpg,ok,rgb8,256,256,0.065536,63908,,,,,,ok,3.847331,ok,0.714681,0.607392,94.077115,51.161873,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00034.jpg,ok,rgb8,256,256,0.065536,64515,,,,,,ok,3.904905,ok,0.782345,0.692224,98.056516,51.185107,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00035.jpg,ok,rgb8,256,256,0.065536,43426,,,,,,ok,3.937929,ok,0.747017,0.659456,92.035916,52.201175,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00036.jpg,ok,rgb8,256,256,0.065536,56694,,,,,,ok,3.921366,ok,0.689960,0.598848,94.077115,51.585132,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00037.jpg,ok,rgb8,256,256,0.065536,59973,,,,,,ok,3.835909,ok,0.777911,0.688128,92.035916,51.375267,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00038.jpg,ok,rgb8,256,256,0.065536,60082,,,,,,ok,3.894976,ok,0.746177,0.596064,95.046216,51.427568,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00039.jpg,ok,rgb8,256,256,0.065536,49278,,,,,,ok,3.973900,ok,0.734878,0.645184,92.615835,51.861930,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00040.jpg,ok,rgb8,256,256,0.065536,62522,,,,,,ok,3.917450,ok,0.757397,0.667648,92.615835,51.233034,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00041.jpg,ok,rgb8,256,256,0.065536,62478,,,,,,ok,3.828719,ok,0.719668,0.612320,101.066815,51.227893,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00042.jpg,ok,rgb8,256,256,0.065536,64628,,,,,,ok,3.898522,ok,0.715856,0.608064,101.066815,51.121225,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00043.jpg,ok,rgb8,256,256,0.065536,62293,,,,,,ok,3.853184,ok,0.720044,0.612192,95.046216,51.237548,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00044.jpg,ok,rgb8,256,256,0.065536,63944,,,,,,ok,3.878313,ok,0.743274,0.641952,92.615835,51.187339,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00045.jpg,ok,rgb8,256,256,0.065536,63422,,,,,,ok,3.966325,ok,0.776291,0.687008,101.066815,51.213808,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00046.jpg,ok,rgb8,256,256,0.065536,62858,,,,,,ok,3.899597,ok,0.779747,0.692128,96.295603,51.235832,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00047.jpg,ok,rgb8,256,256,0.065536,63308,,,,,,ok,3.750095,ok,0.716666,0.611328,96.295603,51.177438,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00048.jpg,ok,rgb8,256,256,0.065536,60750,,,,,,ok,3.854399,ok,0.715441,0.611200,96.295603,51.398736,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00049.jpg,ok,rgb8,256,256,0.065536,60410,,,,,,ok,3.802999,ok,0.767156,0.660480,95.046216,51.319442,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00050.jpg,ok,rgb8,256,256,0.065536,65322,,,,,,ok,3.924275,ok,0.715204,0.615680,101.066815,51.110902,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00051.jpg,ok,rgb8,256,256,0.065536,56310,,,,,,ok,3.994187,ok,0.684192,0.588768,95.046216,51.629942,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00052.jpg,ok,rgb8,256,256,0.065536,55950,,,,,,ok,3.850803,ok,0.694819,0.604160,92.035916,51.526031,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00053.jpg,ok,rgb8,256,256,0.065536,54672,,,,,,ok,3.895307,ok,0.709002,0.616160,92.615835,51.698261,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00054.jpg,ok,rgb8,256,256,0.065536,66731,,,,,,ok,3.906438,ok,0.702078,0.607456,96.295603,51.049478,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00055.jpg,ok,rgb8,256,256,0.065536,51507,,,,,,ok,3.924938,ok,0.667125,0.573440,98.056516,51.814726,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00056.jpg,ok,rgb8,256,256,0.065536,54088,,,,,,ok,4.323621,ok,0.755135,0.662528,93.285303,51.678717,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00057.jpg,ok,rgb8,256,256,0.065536,63657,,,,,,ok,3.826225,ok,0.701170,0.607232,93.285303,51.140927,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00058.jpg,ok,rgb8,256,256,0.065536,57277,,,,,,ok,3.807144,ok,0.712132,0.620224,94.077115,51.504555,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00059.jpg,ok,rgb8,256,256,0.065536,58265,,,,,,ok,3.954043,ok,0.709170,0.613120,96.295603,51.431062,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00060.jpg,ok,rgb8,256,256,0.065536,66405,,,,,,ok,3.849148,ok,0.733841,0.641024,96.295603,51.032856,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00061.jpg,ok,rgb8,256,256,0.065536,51524,,,,,,ok,3.831900,ok,0.668863,0.572544,95.046216,51.886932,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00062.jpg,ok,rgb8,256,256,0.065536,57806,,,,,,ok,3.872645,ok,0.720577,0.626592,98.056516,51.489019,,,,,,,,,,,,,,,,,,,,,,,, +correctness,per_input_correctness, j2k_cuda_batch,download,,nvidia_htj2k:tile_00063.jpg,ok,rgb8,256,256,0.065536,66463,,,,,,ok,3.912981,ok,0.771392,0.623680,96.295603,51.035141,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch-correctness/j2k_decode_batch_correctness.json b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch-correctness/j2k_decode_batch_correctness.json new file mode 100644 index 00000000..f44b637b --- /dev/null +++ b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch-correctness/j2k_decode_batch_correctness.json @@ -0,0 +1,778 @@ +{ + "input_count": 64, + "megapixels": 4.194304, + "warmup": 1, + "iterations": 5, + "max_inputs": 64, + "profile": {"label": "j2k_cuda_decode_batch_correctness", "execution_mode": "j2k_cuda_batch", "timing_scope": "aggregate_batch_correctness", "download_policy": "download", "input_count": 64, "megapixels": 4.194304, "codestream_bytes": 3755036, "mp_s": 453.544032, "status": {"status": "ok", "wall_ms": 9.247843, "gpu_ms": 3.748000, "stage_ms": 6.080000, "download_ms": 1.080402, "bytes": 12582912, "cuda_profile": {"parse_us": 11, "plan_us": 1527, "flatten_us": 372, "h2d_us": 422, "ht_cleanup_us": 1812, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 1103, "mct_us": 0, "store_us": 833, "total_us": 6080, "wall_total_us": 8158, "table_upload_us": 0, "payload_upload_us": 409, "status_d2h_us": 57, "output_d2h_us": 0, "block_count": 4800, "payload_bytes": 3729641, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}}, + "rows": [ + { + "label": "nvidia_htj2k:tile_00000.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 59977, + "cpu": {"status": "ok", "wall_ms": 3.918700, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.762592, "gpu_ms": 0.670720, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.298595 + }, + { + "label": "nvidia_htj2k:tile_00001.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 68326, + "cpu": {"status": "ok", "wall_ms": 3.886764, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.774710, "gpu_ms": 0.683776, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 50.988826 + }, + { + "label": "nvidia_htj2k:tile_00002.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 57281, + "cpu": {"status": "ok", "wall_ms": 3.911964, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.745802, "gpu_ms": 0.654176, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 93.285303, + "nvidia_psnr_vs_cpu": 51.584887 + }, + { + "label": "nvidia_htj2k:tile_00003.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 57771, + "cpu": {"status": "ok", "wall_ms": 3.841815, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.764488, "gpu_ms": 0.670528, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.414093 + }, + { + "label": "nvidia_htj2k:tile_00004.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 50939, + "cpu": {"status": "ok", "wall_ms": 3.939564, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.735985, "gpu_ms": 0.640768, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 91.524390, + "nvidia_psnr_vs_cpu": 52.021932 + }, + { + "label": "nvidia_htj2k:tile_00005.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 42188, + "cpu": {"status": "ok", "wall_ms": 3.954695, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.722700, "gpu_ms": 0.628928, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 90.275003, + "nvidia_psnr_vs_cpu": 52.429662 + }, + { + "label": "nvidia_htj2k:tile_00006.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 58455, + "cpu": {"status": "ok", "wall_ms": 3.876127, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.797595, "gpu_ms": 0.706560, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.365021 + }, + { + "label": "nvidia_htj2k:tile_00007.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 61251, + "cpu": {"status": "ok", "wall_ms": 3.864171, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.829071, "gpu_ms": 0.740416, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.279397 + }, + { + "label": "nvidia_htj2k:tile_00008.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 61623, + "cpu": {"status": "ok", "wall_ms": 3.932927, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.771531, "gpu_ms": 0.680960, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.261782 + }, + { + "label": "nvidia_htj2k:tile_00009.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 49390, + "cpu": {"status": "ok", "wall_ms": 3.897035, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.769703, "gpu_ms": 0.651072, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.781116 + }, + { + "label": "nvidia_htj2k:tile_00010.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 54832, + "cpu": {"status": "ok", "wall_ms": 3.890191, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.770000, "gpu_ms": 0.665344, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.615835, + "nvidia_psnr_vs_cpu": 51.608733 + }, + { + "label": "nvidia_htj2k:tile_00011.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 56468, + "cpu": {"status": "ok", "wall_ms": 3.874446, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.825002, "gpu_ms": 0.715776, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.515713 + }, + { + "label": "nvidia_htj2k:tile_00012.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 55276, + "cpu": {"status": "ok", "wall_ms": 3.883824, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.764874, "gpu_ms": 0.653312, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.527866 + }, + { + "label": "nvidia_htj2k:tile_00013.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 59796, + "cpu": {"status": "ok", "wall_ms": 3.832965, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.818484, "gpu_ms": 0.729216, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.339790 + }, + { + "label": "nvidia_htj2k:tile_00014.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 56580, + "cpu": {"status": "ok", "wall_ms": 3.813292, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.759777, "gpu_ms": 0.669728, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 101.066815, + "nvidia_psnr_vs_cpu": 51.489977 + }, + { + "label": "nvidia_htj2k:tile_00015.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 61771, + "cpu": {"status": "ok", "wall_ms": 3.799835, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.753812, "gpu_ms": 0.662688, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.193192 + }, + { + "label": "nvidia_htj2k:tile_00016.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 51496, + "cpu": {"status": "ok", "wall_ms": 3.898438, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.807314, "gpu_ms": 0.699392, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.765530 + }, + { + "label": "nvidia_htj2k:tile_00017.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 49306, + "cpu": {"status": "ok", "wall_ms": 3.803001, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.743708, "gpu_ms": 0.637056, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.879856 + }, + { + "label": "nvidia_htj2k:tile_00018.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 59128, + "cpu": {"status": "ok", "wall_ms": 3.858689, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.773387, "gpu_ms": 0.667712, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.395550 + }, + { + "label": "nvidia_htj2k:tile_00019.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 61116, + "cpu": {"status": "ok", "wall_ms": 3.900384, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.790563, "gpu_ms": 0.694272, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.246366 + }, + { + "label": "nvidia_htj2k:tile_00020.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 57644, + "cpu": {"status": "ok", "wall_ms": 3.957392, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.745308, "gpu_ms": 0.654496, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 93.285303, + "nvidia_psnr_vs_cpu": 51.367627 + }, + { + "label": "nvidia_htj2k:tile_00021.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 60265, + "cpu": {"status": "ok", "wall_ms": 3.890122, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.850030, "gpu_ms": 0.762880, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.297495 + }, + { + "label": "nvidia_htj2k:tile_00022.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 53721, + "cpu": {"status": "ok", "wall_ms": 3.773820, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.949348, "gpu_ms": 0.840480, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 93.285303, + "nvidia_psnr_vs_cpu": 51.567307 + }, + { + "label": "nvidia_htj2k:tile_00023.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 53941, + "cpu": {"status": "ok", "wall_ms": 3.800833, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.937309, "gpu_ms": 0.837568, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 93.285303, + "nvidia_psnr_vs_cpu": 51.643460 + }, + { + "label": "nvidia_htj2k:tile_00024.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 64688, + "cpu": {"status": "ok", "wall_ms": 3.892117, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.785546, "gpu_ms": 0.674016, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.135055 + }, + { + "label": "nvidia_htj2k:tile_00025.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 56960, + "cpu": {"status": "ok", "wall_ms": 3.861549, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.810089, "gpu_ms": 0.703488, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.431723 + }, + { + "label": "nvidia_htj2k:tile_00026.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 66912, + "cpu": {"status": "ok", "wall_ms": 3.827242, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.779708, "gpu_ms": 0.672768, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.048873 + }, + { + "label": "nvidia_htj2k:tile_00027.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 54363, + "cpu": {"status": "ok", "wall_ms": 3.813736, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.812499, "gpu_ms": 0.718848, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.600623 + }, + { + "label": "nvidia_htj2k:tile_00028.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 55832, + "cpu": {"status": "ok", "wall_ms": 3.761696, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.791136, "gpu_ms": 0.702464, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.523425 + }, + { + "label": "nvidia_htj2k:tile_00029.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 64814, + "cpu": {"status": "ok", "wall_ms": 3.929608, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.830968, "gpu_ms": 0.742400, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.130470 + }, + { + "label": "nvidia_htj2k:tile_00030.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 60022, + "cpu": {"status": "ok", "wall_ms": 3.878310, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.776558, "gpu_ms": 0.685120, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.267191 + }, + { + "label": "nvidia_htj2k:tile_00031.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 63236, + "cpu": {"status": "ok", "wall_ms": 3.865095, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.711896, "gpu_ms": 0.601120, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.202816 + }, + { + "label": "nvidia_htj2k:tile_00032.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 59202, + "cpu": {"status": "ok", "wall_ms": 3.875618, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.773605, "gpu_ms": 0.664608, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.356842 + }, + { + "label": "nvidia_htj2k:tile_00033.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 63908, + "cpu": {"status": "ok", "wall_ms": 3.847331, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.714681, "gpu_ms": 0.607392, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.161873 + }, + { + "label": "nvidia_htj2k:tile_00034.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 64515, + "cpu": {"status": "ok", "wall_ms": 3.904905, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.782345, "gpu_ms": 0.692224, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.185107 + }, + { + "label": "nvidia_htj2k:tile_00035.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 43426, + "cpu": {"status": "ok", "wall_ms": 3.937929, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.747017, "gpu_ms": 0.659456, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.035916, + "nvidia_psnr_vs_cpu": 52.201175 + }, + { + "label": "nvidia_htj2k:tile_00036.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 56694, + "cpu": {"status": "ok", "wall_ms": 3.921366, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.689960, "gpu_ms": 0.598848, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.585132 + }, + { + "label": "nvidia_htj2k:tile_00037.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 59973, + "cpu": {"status": "ok", "wall_ms": 3.835909, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.777911, "gpu_ms": 0.688128, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.035916, + "nvidia_psnr_vs_cpu": 51.375267 + }, + { + "label": "nvidia_htj2k:tile_00038.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 60082, + "cpu": {"status": "ok", "wall_ms": 3.894976, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.746177, "gpu_ms": 0.596064, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.427568 + }, + { + "label": "nvidia_htj2k:tile_00039.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 49278, + "cpu": {"status": "ok", "wall_ms": 3.973900, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.734878, "gpu_ms": 0.645184, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.615835, + "nvidia_psnr_vs_cpu": 51.861930 + }, + { + "label": "nvidia_htj2k:tile_00040.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 62522, + "cpu": {"status": "ok", "wall_ms": 3.917450, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.757397, "gpu_ms": 0.667648, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.615835, + "nvidia_psnr_vs_cpu": 51.233034 + }, + { + "label": "nvidia_htj2k:tile_00041.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 62478, + "cpu": {"status": "ok", "wall_ms": 3.828719, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.719668, "gpu_ms": 0.612320, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 101.066815, + "nvidia_psnr_vs_cpu": 51.227893 + }, + { + "label": "nvidia_htj2k:tile_00042.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 64628, + "cpu": {"status": "ok", "wall_ms": 3.898522, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.715856, "gpu_ms": 0.608064, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 101.066815, + "nvidia_psnr_vs_cpu": 51.121225 + }, + { + "label": "nvidia_htj2k:tile_00043.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 62293, + "cpu": {"status": "ok", "wall_ms": 3.853184, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.720044, "gpu_ms": 0.612192, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.237548 + }, + { + "label": "nvidia_htj2k:tile_00044.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 63944, + "cpu": {"status": "ok", "wall_ms": 3.878313, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.743274, "gpu_ms": 0.641952, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.615835, + "nvidia_psnr_vs_cpu": 51.187339 + }, + { + "label": "nvidia_htj2k:tile_00045.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 63422, + "cpu": {"status": "ok", "wall_ms": 3.966325, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.776291, "gpu_ms": 0.687008, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 101.066815, + "nvidia_psnr_vs_cpu": 51.213808 + }, + { + "label": "nvidia_htj2k:tile_00046.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 62858, + "cpu": {"status": "ok", "wall_ms": 3.899597, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.779747, "gpu_ms": 0.692128, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.235832 + }, + { + "label": "nvidia_htj2k:tile_00047.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 63308, + "cpu": {"status": "ok", "wall_ms": 3.750095, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.716666, "gpu_ms": 0.611328, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.177438 + }, + { + "label": "nvidia_htj2k:tile_00048.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 60750, + "cpu": {"status": "ok", "wall_ms": 3.854399, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.715441, "gpu_ms": 0.611200, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.398736 + }, + { + "label": "nvidia_htj2k:tile_00049.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 60410, + "cpu": {"status": "ok", "wall_ms": 3.802999, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.767156, "gpu_ms": 0.660480, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.319442 + }, + { + "label": "nvidia_htj2k:tile_00050.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 65322, + "cpu": {"status": "ok", "wall_ms": 3.924275, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.715204, "gpu_ms": 0.615680, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 101.066815, + "nvidia_psnr_vs_cpu": 51.110902 + }, + { + "label": "nvidia_htj2k:tile_00051.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 56310, + "cpu": {"status": "ok", "wall_ms": 3.994187, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.684192, "gpu_ms": 0.588768, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.629942 + }, + { + "label": "nvidia_htj2k:tile_00052.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 55950, + "cpu": {"status": "ok", "wall_ms": 3.850803, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.694819, "gpu_ms": 0.604160, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.035916, + "nvidia_psnr_vs_cpu": 51.526031 + }, + { + "label": "nvidia_htj2k:tile_00053.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 54672, + "cpu": {"status": "ok", "wall_ms": 3.895307, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.709002, "gpu_ms": 0.616160, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.615835, + "nvidia_psnr_vs_cpu": 51.698261 + }, + { + "label": "nvidia_htj2k:tile_00054.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 66731, + "cpu": {"status": "ok", "wall_ms": 3.906438, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.702078, "gpu_ms": 0.607456, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.049478 + }, + { + "label": "nvidia_htj2k:tile_00055.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 51507, + "cpu": {"status": "ok", "wall_ms": 3.924938, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.667125, "gpu_ms": 0.573440, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.814726 + }, + { + "label": "nvidia_htj2k:tile_00056.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 54088, + "cpu": {"status": "ok", "wall_ms": 4.323621, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.755135, "gpu_ms": 0.662528, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 93.285303, + "nvidia_psnr_vs_cpu": 51.678717 + }, + { + "label": "nvidia_htj2k:tile_00057.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 63657, + "cpu": {"status": "ok", "wall_ms": 3.826225, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.701170, "gpu_ms": 0.607232, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 93.285303, + "nvidia_psnr_vs_cpu": 51.140927 + }, + { + "label": "nvidia_htj2k:tile_00058.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 57277, + "cpu": {"status": "ok", "wall_ms": 3.807144, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.712132, "gpu_ms": 0.620224, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.504555 + }, + { + "label": "nvidia_htj2k:tile_00059.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 58265, + "cpu": {"status": "ok", "wall_ms": 3.954043, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.709170, "gpu_ms": 0.613120, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.431062 + }, + { + "label": "nvidia_htj2k:tile_00060.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 66405, + "cpu": {"status": "ok", "wall_ms": 3.849148, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.733841, "gpu_ms": 0.641024, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.032856 + }, + { + "label": "nvidia_htj2k:tile_00061.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 51524, + "cpu": {"status": "ok", "wall_ms": 3.831900, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.668863, "gpu_ms": 0.572544, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.886932 + }, + { + "label": "nvidia_htj2k:tile_00062.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 57806, + "cpu": {"status": "ok", "wall_ms": 3.872645, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.720577, "gpu_ms": 0.626592, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.489019 + }, + { + "label": "nvidia_htj2k:tile_00063.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 66463, + "cpu": {"status": "ok", "wall_ms": 3.912981, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok"}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.771392, "gpu_ms": 0.623680, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.035141 + } + ] +} diff --git a/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_batch_download.csv b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_batch_download.csv new file mode 100644 index 00000000..210e559a --- /dev/null +++ b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_batch_download.csv @@ -0,0 +1,2 @@ +row_type,timing_scope,execution_mode,download_policy,input_count,label,status,megapixels,codestream_bytes,mp_s,wall_ms,gpu_ms,stage_ms,download_ms,cuda_parse_us,cuda_plan_us,cuda_flatten_us,cuda_h2d_us,cuda_ht_cleanup_us,cuda_ht_refine_us,cuda_dequant_us,cuda_idwt_us,cuda_mct_us,cuda_store_us,cuda_total_us,cuda_wall_total_us,cuda_table_upload_us,cuda_payload_upload_us,cuda_status_d2h_us,cuda_output_d2h_us,cuda_block_count,cuda_payload_bytes,cuda_dispatch_count,cuda_ht_dispatch_count,cuda_dequant_dispatch_count,cuda_idwt_dispatch_count,cuda_mct_dispatch_count,cuda_store_dispatch_count +aggregate,aggregate_batch, j2k_cuda_batch,download,64, j2k_cuda_decode_real_batch,ok,4.194304,3755036,442.449172,9.479742,4.001000,6.351000,1.070317,9,1538,384,419,1996,0,0,1184,0,821,6351,8400,0,417,43,0,4800,3729641,12,1,0,10,0,1 diff --git a/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_batch_download.json b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_batch_download.json new file mode 100644 index 00000000..c143fe20 --- /dev/null +++ b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_batch_download.json @@ -0,0 +1,11 @@ +{ + "input_count": 64, + "megapixels": 4.194304, + "warmup": 1, + "iterations": 5, + "max_inputs": 64, + "profile": {"label": "j2k_cuda_decode_real_batch", "execution_mode": "j2k_cuda_batch", "timing_scope": "aggregate_batch", "download_policy": "download", "input_count": 64, "megapixels": 4.194304, "codestream_bytes": 3755036, "mp_s": 442.449172, "status": {"status": "ok", "wall_ms": 9.479742, "gpu_ms": 4.001000, "stage_ms": 6.351000, "download_ms": 1.070317, "bytes": 12582912, "cuda_profile": {"parse_us": 9, "plan_us": 1538, "flatten_us": 384, "h2d_us": 419, "ht_cleanup_us": 1996, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 1184, "mct_us": 0, "store_us": 821, "total_us": 6351, "wall_total_us": 8400, "table_upload_us": 0, "payload_upload_us": 417, "status_d2h_us": 43, "output_d2h_us": 0, "block_count": 4800, "payload_bytes": 3729641, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}}, + "rows": [ + + ] +} diff --git a/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_batch_no_download.csv b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_batch_no_download.csv new file mode 100644 index 00000000..97cda046 --- /dev/null +++ b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_batch_no_download.csv @@ -0,0 +1,2 @@ +row_type,timing_scope,execution_mode,download_policy,input_count,label,status,megapixels,codestream_bytes,mp_s,wall_ms,gpu_ms,stage_ms,download_ms,cuda_parse_us,cuda_plan_us,cuda_flatten_us,cuda_h2d_us,cuda_ht_cleanup_us,cuda_ht_refine_us,cuda_dequant_us,cuda_idwt_us,cuda_mct_us,cuda_store_us,cuda_total_us,cuda_wall_total_us,cuda_table_upload_us,cuda_payload_upload_us,cuda_status_d2h_us,cuda_output_d2h_us,cuda_block_count,cuda_payload_bytes,cuda_dispatch_count,cuda_ht_dispatch_count,cuda_dequant_dispatch_count,cuda_idwt_dispatch_count,cuda_mct_dispatch_count,cuda_store_dispatch_count +aggregate,aggregate_batch, j2k_cuda_batch,no_download,64, j2k_cuda_decode_real_batch,ok,4.194304,3755036,487.126801,8.610292,4.121000,6.587000,,10,1542,376,538,2104,0,0,1191,0,826,6587,8601,0,471,41,0,4800,3729641,12,1,0,10,0,1 diff --git a/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_batch_no_download.json b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_batch_no_download.json new file mode 100644 index 00000000..d07aec22 --- /dev/null +++ b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_batch_no_download.json @@ -0,0 +1,11 @@ +{ + "input_count": 64, + "megapixels": 4.194304, + "warmup": 1, + "iterations": 5, + "max_inputs": 64, + "profile": {"label": "j2k_cuda_decode_real_batch", "execution_mode": "j2k_cuda_batch", "timing_scope": "aggregate_batch", "download_policy": "no_download", "input_count": 64, "megapixels": 4.194304, "codestream_bytes": 3755036, "mp_s": 487.126801, "status": {"status": "ok", "wall_ms": 8.610292, "gpu_ms": 4.121000, "stage_ms": 6.587000, "download_ms": null, "bytes": 0, "cuda_profile": {"parse_us": 10, "plan_us": 1542, "flatten_us": 376, "h2d_us": 538, "ht_cleanup_us": 2104, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 1191, "mct_us": 0, "store_us": 826, "total_us": 6587, "wall_total_us": 8601, "table_upload_us": 0, "payload_upload_us": 471, "status_d2h_us": 41, "output_d2h_us": 0, "block_count": 4800, "payload_bytes": 3729641, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}}, + "rows": [ + + ] +} diff --git a/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_normal_comparison.csv b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_normal_comparison.csv new file mode 100644 index 00000000..28ae9833 --- /dev/null +++ b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_normal_comparison.csv @@ -0,0 +1,65 @@ +label,format,width,height,codestream_bytes,cpu_status,cpu_wall_ms, j2k_cuda_status, j2k_cuda_wall_ms, j2k_cuda_gpu_ms, j2k_cuda_stage_ms, j2k_cuda_download_ms,nvidia_status,nvidia_wall_ms,nvidia_gpu_ms, j2k_cuda_psnr_vs_cpu,nvidia_psnr_vs_cpu +nvidia_htj2k:tile_00000.jpg,rgb8,256,256,59977,ok,3.746401,ok,1.186340,0.914000,1.029000,0.057382,ok,0.755101,0.654048,95.046216,51.298595 +nvidia_htj2k:tile_00001.jpg,rgb8,256,256,68326,ok,3.838732,ok,1.120149,0.847000,0.978000,0.051961,ok,0.846471,0.743424,96.295603,50.988826 +nvidia_htj2k:tile_00002.jpg,rgb8,256,256,57281,ok,3.880363,ok,1.090693,0.832000,0.948000,0.053413,ok,0.789218,0.693248,93.285303,51.584887 +nvidia_htj2k:tile_00003.jpg,rgb8,256,256,57771,ok,3.816132,ok,1.066352,0.804000,0.918000,0.052613,ok,0.747784,0.654080,96.295603,51.414093 +nvidia_htj2k:tile_00004.jpg,rgb8,256,256,50939,ok,3.973220,ok,1.060388,0.807000,0.916000,0.054064,ok,0.723906,0.623616,91.524390,52.021932 +nvidia_htj2k:tile_00005.jpg,rgb8,256,256,42188,ok,3.984181,ok,1.045171,0.773000,0.880000,0.053995,ok,0.717913,0.624640,90.275003,52.429662 +nvidia_htj2k:tile_00006.jpg,rgb8,256,256,58455,ok,3.834352,ok,1.397718,1.080000,1.219000,0.058824,ok,0.750213,0.655360,94.077115,51.365021 +nvidia_htj2k:tile_00007.jpg,rgb8,256,256,61251,ok,3.889720,ok,1.082517,0.812000,0.932000,0.057549,ok,0.780084,0.671744,96.295603,51.279397 +nvidia_htj2k:tile_00008.jpg,rgb8,256,256,61623,ok,3.876033,ok,1.083336,0.826000,0.940000,0.051023,ok,0.757817,0.668704,98.056516,51.261782 +nvidia_htj2k:tile_00009.jpg,rgb8,256,256,49390,ok,3.878288,ok,1.065799,0.799000,0.911000,0.057451,ok,0.729269,0.641952,95.046216,51.781116 +nvidia_htj2k:tile_00010.jpg,rgb8,256,256,54832,ok,3.935647,ok,1.380694,1.057000,1.197000,0.059930,ok,0.750647,0.657408,92.615835,51.608733 +nvidia_htj2k:tile_00011.jpg,rgb8,256,256,56468,ok,3.852191,ok,1.117769,0.832000,0.960000,0.057945,ok,0.808483,0.690944,96.295603,51.515713 +nvidia_htj2k:tile_00012.jpg,rgb8,256,256,55276,ok,3.890049,ok,1.084591,0.820000,0.936000,0.054686,ok,0.813075,0.718624,94.077115,51.527866 +nvidia_htj2k:tile_00013.jpg,rgb8,256,256,59796,ok,3.792260,ok,1.107026,0.839000,0.950000,0.060858,ok,0.774603,0.673792,95.046216,51.339790 +nvidia_htj2k:tile_00014.jpg,rgb8,256,256,56580,ok,3.917131,ok,1.423116,1.087000,1.242000,0.059021,ok,0.778425,0.677792,101.066815,51.489977 +nvidia_htj2k:tile_00015.jpg,rgb8,256,256,61771,ok,3.915463,ok,1.226870,0.909000,1.048000,0.056987,ok,0.751862,0.658432,98.056516,51.193192 +nvidia_htj2k:tile_00016.jpg,rgb8,256,256,51496,ok,3.883321,ok,1.051382,0.789000,0.900000,0.052257,ok,0.736486,0.638976,94.077115,51.765530 +nvidia_htj2k:tile_00017.jpg,rgb8,256,256,49306,ok,3.844705,ok,1.056645,0.802000,0.909000,0.053609,ok,0.754251,0.640000,98.056516,51.879856 +nvidia_htj2k:tile_00018.jpg,rgb8,256,256,59128,ok,3.780402,ok,1.071438,0.796000,0.913000,0.054538,ok,0.740644,0.651264,95.046216,51.395550 +nvidia_htj2k:tile_00019.jpg,rgb8,256,256,61116,ok,3.785901,ok,1.088945,0.828000,0.943000,0.053205,ok,0.834188,0.746496,95.046216,51.246366 +nvidia_htj2k:tile_00020.jpg,rgb8,256,256,57644,ok,3.915300,ok,1.076651,0.811000,0.926000,0.052691,ok,0.802006,0.714752,93.285303,51.367627 +nvidia_htj2k:tile_00021.jpg,rgb8,256,256,60265,ok,3.839136,ok,1.075091,0.818000,0.928000,0.051269,ok,0.815021,0.729088,96.295603,51.297495 +nvidia_htj2k:tile_00022.jpg,rgb8,256,256,53721,ok,3.719168,ok,1.076701,0.816000,0.927000,0.053995,ok,0.729159,0.642048,93.285303,51.567307 +nvidia_htj2k:tile_00023.jpg,rgb8,256,256,53941,ok,3.982778,ok,1.072899,0.816000,0.924000,0.053451,ok,0.777940,0.690176,93.285303,51.643460 +nvidia_htj2k:tile_00024.jpg,rgb8,256,256,64688,ok,3.793633,ok,1.077066,0.814000,0.931000,0.052741,ok,1.220527,1.060864,96.295603,51.135055 +nvidia_htj2k:tile_00025.jpg,rgb8,256,256,56960,ok,3.841205,ok,1.061089,0.798000,0.909000,0.054815,ok,0.854527,0.734208,94.077115,51.431723 +nvidia_htj2k:tile_00026.jpg,rgb8,256,256,66912,ok,3.752762,ok,1.389756,1.100000,1.218000,0.060917,ok,0.745760,0.660224,95.046216,51.048873 +nvidia_htj2k:tile_00027.jpg,rgb8,256,256,54363,ok,3.852244,ok,1.234133,0.889000,1.053000,0.060117,ok,0.785110,0.698368,96.295603,51.600623 +nvidia_htj2k:tile_00028.jpg,rgb8,256,256,55832,ok,3.886633,ok,1.093607,0.836000,0.947000,0.055111,ok,0.796456,0.710656,95.046216,51.523425 +nvidia_htj2k:tile_00029.jpg,rgb8,256,256,64814,ok,3.858900,ok,1.088916,0.826000,0.941000,0.057569,ok,1.217741,1.055744,98.056516,51.130470 +nvidia_htj2k:tile_00030.jpg,rgb8,256,256,60022,ok,3.831784,ok,1.097388,0.838000,0.951000,0.053521,ok,0.803734,0.707616,98.056516,51.267191 +nvidia_htj2k:tile_00031.jpg,rgb8,256,256,63236,ok,3.889601,ok,1.101773,0.836000,0.953000,0.052207,ok,0.796446,0.710656,98.056516,51.202816 +nvidia_htj2k:tile_00032.jpg,rgb8,256,256,59202,ok,3.844893,ok,1.181504,0.880000,0.990000,0.067524,ok,0.742727,0.649216,98.056516,51.356842 +nvidia_htj2k:tile_00033.jpg,rgb8,256,256,63908,ok,3.770902,ok,1.221919,0.883000,1.034000,0.060423,ok,0.809816,0.711680,94.077115,51.161873 +nvidia_htj2k:tile_00034.jpg,rgb8,256,256,64515,ok,3.815960,ok,1.122193,0.819000,0.952000,0.068343,ok,0.757924,0.627968,98.056516,51.185107 +nvidia_htj2k:tile_00035.jpg,rgb8,256,256,43426,ok,3.892549,ok,1.078271,0.809000,0.918000,0.064324,ok,0.673407,0.570208,92.035916,52.201175 +nvidia_htj2k:tile_00036.jpg,rgb8,256,256,56694,ok,3.789806,ok,1.068525,0.804000,0.914000,0.054103,ok,0.685178,0.587680,94.077115,51.585132 +nvidia_htj2k:tile_00037.jpg,rgb8,256,256,59973,ok,3.895510,ok,1.093893,0.836000,0.946000,0.053679,ok,0.679856,0.582656,92.035916,51.375267 +nvidia_htj2k:tile_00038.jpg,rgb8,256,256,60082,ok,3.881657,ok,1.097141,0.833000,0.947000,0.054419,ok,0.706033,0.611328,95.046216,51.427568 +nvidia_htj2k:tile_00039.jpg,rgb8,256,256,49278,ok,3.832381,ok,1.091493,0.832000,0.937000,0.054212,ok,0.700612,0.603136,92.615835,51.861930 +nvidia_htj2k:tile_00040.jpg,rgb8,256,256,62522,ok,3.709189,ok,1.106217,0.842000,0.957000,0.054607,ok,0.720648,0.614400,92.615835,51.233034 +nvidia_htj2k:tile_00041.jpg,rgb8,256,256,62478,ok,3.799024,ok,1.101289,0.843000,0.954000,0.053324,ok,0.743992,0.653248,101.066815,51.227893 +nvidia_htj2k:tile_00042.jpg,rgb8,256,256,64628,ok,3.878344,ok,1.105002,0.824000,0.956000,0.055753,ok,0.704483,0.602112,101.066815,51.121225 +nvidia_htj2k:tile_00043.jpg,rgb8,256,256,62293,ok,3.771909,ok,1.104636,0.834000,0.952000,0.055605,ok,0.777437,0.666592,95.046216,51.237548 +nvidia_htj2k:tile_00044.jpg,rgb8,256,256,63944,ok,3.898161,ok,1.107085,0.844000,0.956000,0.055980,ok,0.733070,0.625664,92.615835,51.187339 +nvidia_htj2k:tile_00045.jpg,rgb8,256,256,63422,ok,3.821273,ok,1.090416,0.832000,0.939000,0.054499,ok,0.761806,0.675840,101.066815,51.213808 +nvidia_htj2k:tile_00046.jpg,rgb8,256,256,62858,ok,3.926952,ok,1.084758,0.828000,0.939000,0.054746,ok,0.763425,0.675840,96.295603,51.235832 +nvidia_htj2k:tile_00047.jpg,rgb8,256,256,63308,ok,3.964037,ok,1.082793,0.816000,0.932000,0.052494,ok,0.782078,0.696224,96.295603,51.177438 +nvidia_htj2k:tile_00048.jpg,rgb8,256,256,60750,ok,3.854427,ok,1.072959,0.803000,0.921000,0.056009,ok,0.768688,0.654336,96.295603,51.398736 +nvidia_htj2k:tile_00049.jpg,rgb8,256,256,60410,ok,3.929134,ok,1.098346,0.832000,0.948000,0.056039,ok,0.710645,0.612256,95.046216,51.319442 +nvidia_htj2k:tile_00050.jpg,rgb8,256,256,65322,ok,3.910747,ok,1.104291,0.838000,0.955000,0.056414,ok,0.715868,0.617472,101.066815,51.110902 +nvidia_htj2k:tile_00051.jpg,rgb8,256,256,56310,ok,3.908768,ok,1.081698,0.815000,0.931000,0.056592,ok,0.717515,0.575264,95.046216,51.629942 +nvidia_htj2k:tile_00052.jpg,rgb8,256,256,55950,ok,3.909370,ok,1.094159,0.830000,0.942000,0.057550,ok,0.714170,0.621568,92.035916,51.526031 +nvidia_htj2k:tile_00053.jpg,rgb8,256,256,54672,ok,3.822329,ok,1.079555,0.817000,0.931000,0.056799,ok,0.711257,0.622592,92.615835,51.698261 +nvidia_htj2k:tile_00054.jpg,rgb8,256,256,66731,ok,3.847712,ok,1.096529,0.831000,0.947000,0.054963,ok,0.687765,0.599040,96.295603,51.049478 +nvidia_htj2k:tile_00055.jpg,rgb8,256,256,51507,ok,3.757817,ok,1.072760,0.790000,0.901000,0.061056,ok,0.736694,0.627712,98.056516,51.814726 +nvidia_htj2k:tile_00056.jpg,rgb8,256,256,54088,ok,3.830653,ok,1.083426,0.818000,0.936000,0.056671,ok,0.718554,0.619360,93.285303,51.678717 +nvidia_htj2k:tile_00057.jpg,rgb8,256,256,63657,ok,3.855340,ok,1.106038,0.839000,0.957000,0.056523,ok,0.691793,0.590848,93.285303,51.140927 +nvidia_htj2k:tile_00058.jpg,rgb8,256,256,57277,ok,3.860347,ok,1.077648,0.812000,0.928000,0.057461,ok,0.749175,0.656128,94.077115,51.504555 +nvidia_htj2k:tile_00059.jpg,rgb8,256,256,58265,ok,3.881750,ok,1.076493,0.812000,0.928000,0.053067,ok,0.756434,0.663552,96.295603,51.431062 +nvidia_htj2k:tile_00060.jpg,rgb8,256,256,66405,ok,3.871061,ok,1.108833,0.845000,0.966000,0.052642,ok,0.720618,0.636704,96.295603,51.032856 +nvidia_htj2k:tile_00061.jpg,rgb8,256,256,51524,ok,3.807557,ok,1.067103,0.805000,0.919000,0.054874,ok,0.720667,0.632832,95.046216,51.886932 +nvidia_htj2k:tile_00062.jpg,rgb8,256,256,57806,ok,3.950306,ok,1.060052,0.789000,0.908000,0.052702,ok,0.749857,0.645184,98.056516,51.489019 +nvidia_htj2k:tile_00063.jpg,rgb8,256,256,66463,ok,3.885157,ok,1.082458,0.823000,0.941000,0.053432,ok,0.688881,0.587648,96.295603,51.035141 diff --git a/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_normal_comparison.json b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_normal_comparison.json new file mode 100644 index 00000000..d3acd741 --- /dev/null +++ b/tests/nvidia-baseline/artifacts/2026-06-04-cuda-decode-batch/j2k_decode_normal_comparison.json @@ -0,0 +1,778 @@ +{ + "input_count": 64, + "megapixels": 4.194304, + "warmup": 1, + "iterations": 5, + "max_inputs": 64, + "profile": null, + "rows": [ + { + "label": "nvidia_htj2k:tile_00000.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 59977, + "cpu": {"status": "ok", "wall_ms": 3.746401, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.186340, "gpu_ms": 0.914000, "stage_ms": 1.029000, "download_ms": 0.057382, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 47, "flatten_us": 9, "h2d_us": 58, "ht_cleanup_us": 711, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 147, "mct_us": 0, "store_us": 56, "total_us": 1029, "wall_total_us": 1110, "table_upload_us": 0, "payload_upload_us": 38, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 59580, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.755101, "gpu_ms": 0.654048, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.298595 + }, + { + "label": "nvidia_htj2k:tile_00001.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 68326, + "cpu": {"status": "ok", "wall_ms": 3.838732, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.120149, "gpu_ms": 0.847000, "stage_ms": 0.978000, "download_ms": 0.051961, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 62, "flatten_us": 9, "h2d_us": 59, "ht_cleanup_us": 662, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 122, "mct_us": 0, "store_us": 63, "total_us": 978, "wall_total_us": 1055, "table_upload_us": 0, "payload_upload_us": 39, "status_d2h_us": 27, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 67924, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.846471, "gpu_ms": 0.743424, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 50.988826 + }, + { + "label": "nvidia_htj2k:tile_00002.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 57281, + "cpu": {"status": "ok", "wall_ms": 3.880363, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.090693, "gpu_ms": 0.832000, "stage_ms": 0.948000, "download_ms": 0.053413, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 52, "flatten_us": 7, "h2d_us": 57, "ht_cleanup_us": 644, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 122, "mct_us": 0, "store_us": 66, "total_us": 948, "wall_total_us": 1024, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 56884, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.789218, "gpu_ms": 0.693248, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 93.285303, + "nvidia_psnr_vs_cpu": 51.584887 + }, + { + "label": "nvidia_htj2k:tile_00003.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 57771, + "cpu": {"status": "ok", "wall_ms": 3.816132, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.066352, "gpu_ms": 0.804000, "stage_ms": 0.918000, "download_ms": 0.052613, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 51, "flatten_us": 5, "h2d_us": 57, "ht_cleanup_us": 637, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 113, "mct_us": 0, "store_us": 54, "total_us": 918, "wall_total_us": 1001, "table_upload_us": 0, "payload_upload_us": 35, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 57374, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.747784, "gpu_ms": 0.654080, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.414093 + }, + { + "label": "nvidia_htj2k:tile_00004.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 50939, + "cpu": {"status": "ok", "wall_ms": 3.973220, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.060388, "gpu_ms": 0.807000, "stage_ms": 0.916000, "download_ms": 0.054064, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 49, "flatten_us": 6, "h2d_us": 53, "ht_cleanup_us": 612, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 142, "mct_us": 0, "store_us": 53, "total_us": 916, "wall_total_us": 993, "table_upload_us": 0, "payload_upload_us": 34, "status_d2h_us": 29, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 50546, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.723906, "gpu_ms": 0.623616, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 91.524390, + "nvidia_psnr_vs_cpu": 52.021932 + }, + { + "label": "nvidia_htj2k:tile_00005.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 42188, + "cpu": {"status": "ok", "wall_ms": 3.984181, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.045171, "gpu_ms": 0.773000, "stage_ms": 0.880000, "download_ms": 0.053995, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 48, "flatten_us": 5, "h2d_us": 53, "ht_cleanup_us": 614, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 99, "mct_us": 0, "store_us": 60, "total_us": 880, "wall_total_us": 978, "table_upload_us": 0, "payload_upload_us": 34, "status_d2h_us": 30, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 41801, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.717913, "gpu_ms": 0.624640, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 90.275003, + "nvidia_psnr_vs_cpu": 52.429662 + }, + { + "label": "nvidia_htj2k:tile_00006.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 58455, + "cpu": {"status": "ok", "wall_ms": 3.834352, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.397718, "gpu_ms": 1.080000, "stage_ms": 1.219000, "download_ms": 0.058824, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 55, "flatten_us": 7, "h2d_us": 76, "ht_cleanup_us": 687, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 291, "mct_us": 0, "store_us": 102, "total_us": 1219, "wall_total_us": 1318, "table_upload_us": 0, "payload_upload_us": 47, "status_d2h_us": 34, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 58056, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.750213, "gpu_ms": 0.655360, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.365021 + }, + { + "label": "nvidia_htj2k:tile_00007.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 61251, + "cpu": {"status": "ok", "wall_ms": 3.889720, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.082517, "gpu_ms": 0.812000, "stage_ms": 0.932000, "download_ms": 0.057549, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 52, "flatten_us": 6, "h2d_us": 61, "ht_cleanup_us": 639, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 120, "mct_us": 0, "store_us": 53, "total_us": 932, "wall_total_us": 1011, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 60852, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.780084, "gpu_ms": 0.671744, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.279397 + }, + { + "label": "nvidia_htj2k:tile_00008.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 61623, + "cpu": {"status": "ok", "wall_ms": 3.876033, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.083336, "gpu_ms": 0.826000, "stage_ms": 0.940000, "download_ms": 0.051023, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 50, "flatten_us": 6, "h2d_us": 57, "ht_cleanup_us": 647, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 117, "mct_us": 0, "store_us": 62, "total_us": 940, "wall_total_us": 1019, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 61226, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.757817, "gpu_ms": 0.668704, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.261782 + }, + { + "label": "nvidia_htj2k:tile_00009.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 49390, + "cpu": {"status": "ok", "wall_ms": 3.878288, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.065799, "gpu_ms": 0.799000, "stage_ms": 0.911000, "download_ms": 0.057451, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 49, "flatten_us": 5, "h2d_us": 58, "ht_cleanup_us": 640, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 100, "mct_us": 0, "store_us": 59, "total_us": 911, "wall_total_us": 994, "table_upload_us": 0, "payload_upload_us": 35, "status_d2h_us": 38, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 48999, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.729269, "gpu_ms": 0.641952, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.781116 + }, + { + "label": "nvidia_htj2k:tile_00010.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 54832, + "cpu": {"status": "ok", "wall_ms": 3.935647, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.380694, "gpu_ms": 1.057000, "stage_ms": 1.197000, "download_ms": 0.059930, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 55, "flatten_us": 9, "h2d_us": 75, "ht_cleanup_us": 673, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 284, "mct_us": 0, "store_us": 100, "total_us": 1197, "wall_total_us": 1297, "table_upload_us": 0, "payload_upload_us": 48, "status_d2h_us": 33, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 54436, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.750647, "gpu_ms": 0.657408, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.615835, + "nvidia_psnr_vs_cpu": 51.608733 + }, + { + "label": "nvidia_htj2k:tile_00011.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 56468, + "cpu": {"status": "ok", "wall_ms": 3.852191, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.117769, "gpu_ms": 0.832000, "stage_ms": 0.960000, "download_ms": 0.057945, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 64, "flatten_us": 9, "h2d_us": 54, "ht_cleanup_us": 648, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 113, "mct_us": 0, "store_us": 71, "total_us": 960, "wall_total_us": 1041, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 56072, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.808483, "gpu_ms": 0.690944, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.515713 + }, + { + "label": "nvidia_htj2k:tile_00012.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 55276, + "cpu": {"status": "ok", "wall_ms": 3.890049, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.084591, "gpu_ms": 0.820000, "stage_ms": 0.936000, "download_ms": 0.054686, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 52, "flatten_us": 6, "h2d_us": 58, "ht_cleanup_us": 638, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 119, "mct_us": 0, "store_us": 63, "total_us": 936, "wall_total_us": 1016, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 28, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 54881, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.813075, "gpu_ms": 0.718624, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.527866 + }, + { + "label": "nvidia_htj2k:tile_00013.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 59796, + "cpu": {"status": "ok", "wall_ms": 3.792260, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.107026, "gpu_ms": 0.839000, "stage_ms": 0.950000, "download_ms": 0.060858, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 44, "flatten_us": 10, "h2d_us": 57, "ht_cleanup_us": 651, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 133, "mct_us": 0, "store_us": 55, "total_us": 950, "wall_total_us": 1029, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 59399, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.774603, "gpu_ms": 0.673792, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.339790 + }, + { + "label": "nvidia_htj2k:tile_00014.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 56580, + "cpu": {"status": "ok", "wall_ms": 3.917131, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.423116, "gpu_ms": 1.087000, "stage_ms": 1.242000, "download_ms": 0.059021, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 67, "flatten_us": 10, "h2d_us": 77, "ht_cleanup_us": 691, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 283, "mct_us": 0, "store_us": 113, "total_us": 1242, "wall_total_us": 1341, "table_upload_us": 0, "payload_upload_us": 49, "status_d2h_us": 33, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 56183, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.778425, "gpu_ms": 0.677792, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 101.066815, + "nvidia_psnr_vs_cpu": 51.489977 + }, + { + "label": "nvidia_htj2k:tile_00015.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 61771, + "cpu": {"status": "ok", "wall_ms": 3.915463, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.226870, "gpu_ms": 0.909000, "stage_ms": 1.048000, "download_ms": 0.056987, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 56, "flatten_us": 8, "h2d_us": 74, "ht_cleanup_us": 693, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 158, "mct_us": 0, "store_us": 58, "total_us": 1048, "wall_total_us": 1153, "table_upload_us": 0, "payload_upload_us": 48, "status_d2h_us": 35, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 61372, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.751862, "gpu_ms": 0.658432, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.193192 + }, + { + "label": "nvidia_htj2k:tile_00016.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 51496, + "cpu": {"status": "ok", "wall_ms": 3.883321, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.051382, "gpu_ms": 0.789000, "stage_ms": 0.900000, "download_ms": 0.052257, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 49, "flatten_us": 6, "h2d_us": 56, "ht_cleanup_us": 619, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 111, "mct_us": 0, "store_us": 59, "total_us": 900, "wall_total_us": 986, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 51102, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.736486, "gpu_ms": 0.638976, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.765530 + }, + { + "label": "nvidia_htj2k:tile_00017.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 49306, + "cpu": {"status": "ok", "wall_ms": 3.844705, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.056645, "gpu_ms": 0.802000, "stage_ms": 0.909000, "download_ms": 0.053609, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 44, "flatten_us": 8, "h2d_us": 55, "ht_cleanup_us": 630, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 117, "mct_us": 0, "store_us": 55, "total_us": 909, "wall_total_us": 987, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 48913, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.754251, "gpu_ms": 0.640000, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.879856 + }, + { + "label": "nvidia_htj2k:tile_00018.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 59128, + "cpu": {"status": "ok", "wall_ms": 3.780402, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.071438, "gpu_ms": 0.796000, "stage_ms": 0.913000, "download_ms": 0.054538, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 52, "flatten_us": 6, "h2d_us": 59, "ht_cleanup_us": 647, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 95, "mct_us": 0, "store_us": 54, "total_us": 913, "wall_total_us": 1003, "table_upload_us": 0, "payload_upload_us": 35, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 58731, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.740644, "gpu_ms": 0.651264, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.395550 + }, + { + "label": "nvidia_htj2k:tile_00019.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 61116, + "cpu": {"status": "ok", "wall_ms": 3.785901, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.088945, "gpu_ms": 0.828000, "stage_ms": 0.943000, "download_ms": 0.053205, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 52, "flatten_us": 5, "h2d_us": 57, "ht_cleanup_us": 645, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 124, "mct_us": 0, "store_us": 59, "total_us": 943, "wall_total_us": 1022, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 60719, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.834188, "gpu_ms": 0.746496, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.246366 + }, + { + "label": "nvidia_htj2k:tile_00020.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 57644, + "cpu": {"status": "ok", "wall_ms": 3.915300, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.076651, "gpu_ms": 0.811000, "stage_ms": 0.926000, "download_ms": 0.052691, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 44, "flatten_us": 9, "h2d_us": 62, "ht_cleanup_us": 631, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 118, "mct_us": 0, "store_us": 62, "total_us": 926, "wall_total_us": 1007, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 57245, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.802006, "gpu_ms": 0.714752, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 93.285303, + "nvidia_psnr_vs_cpu": 51.367627 + }, + { + "label": "nvidia_htj2k:tile_00021.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 60265, + "cpu": {"status": "ok", "wall_ms": 3.839136, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.075091, "gpu_ms": 0.818000, "stage_ms": 0.928000, "download_ms": 0.051269, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 49, "flatten_us": 5, "h2d_us": 56, "ht_cleanup_us": 646, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 119, "mct_us": 0, "store_us": 53, "total_us": 928, "wall_total_us": 1011, "table_upload_us": 0, "payload_upload_us": 35, "status_d2h_us": 28, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 59868, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.815021, "gpu_ms": 0.729088, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.297495 + }, + { + "label": "nvidia_htj2k:tile_00022.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 53721, + "cpu": {"status": "ok", "wall_ms": 3.719168, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.076701, "gpu_ms": 0.816000, "stage_ms": 0.927000, "download_ms": 0.053995, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 44, "flatten_us": 9, "h2d_us": 58, "ht_cleanup_us": 639, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 121, "mct_us": 0, "store_us": 56, "total_us": 927, "wall_total_us": 1005, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 27, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 53325, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.729159, "gpu_ms": 0.642048, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 93.285303, + "nvidia_psnr_vs_cpu": 51.567307 + }, + { + "label": "nvidia_htj2k:tile_00023.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 53941, + "cpu": {"status": "ok", "wall_ms": 3.982778, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.072899, "gpu_ms": 0.816000, "stage_ms": 0.924000, "download_ms": 0.053451, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 43, "flatten_us": 7, "h2d_us": 58, "ht_cleanup_us": 641, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 120, "mct_us": 0, "store_us": 55, "total_us": 924, "wall_total_us": 1002, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 27, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 53544, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.777940, "gpu_ms": 0.690176, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 93.285303, + "nvidia_psnr_vs_cpu": 51.643460 + }, + { + "label": "nvidia_htj2k:tile_00024.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 64688, + "cpu": {"status": "ok", "wall_ms": 3.793633, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.077066, "gpu_ms": 0.814000, "stage_ms": 0.931000, "download_ms": 0.052741, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 50, "flatten_us": 7, "h2d_us": 60, "ht_cleanup_us": 656, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 102, "mct_us": 0, "store_us": 56, "total_us": 931, "wall_total_us": 1011, "table_upload_us": 0, "payload_upload_us": 38, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 64289, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 1.220527, "gpu_ms": 1.060864, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.135055 + }, + { + "label": "nvidia_htj2k:tile_00025.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 56960, + "cpu": {"status": "ok", "wall_ms": 3.841205, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.061089, "gpu_ms": 0.798000, "stage_ms": 0.909000, "download_ms": 0.054815, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 43, "flatten_us": 9, "h2d_us": 58, "ht_cleanup_us": 647, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 95, "mct_us": 0, "store_us": 56, "total_us": 909, "wall_total_us": 990, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 27, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 56562, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.854527, "gpu_ms": 0.734208, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.431723 + }, + { + "label": "nvidia_htj2k:tile_00026.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 66912, + "cpu": {"status": "ok", "wall_ms": 3.752762, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.389756, "gpu_ms": 1.100000, "stage_ms": 1.218000, "download_ms": 0.060917, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 52, "flatten_us": 7, "h2d_us": 59, "ht_cleanup_us": 691, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 298, "mct_us": 0, "store_us": 111, "total_us": 1218, "wall_total_us": 1309, "table_upload_us": 0, "payload_upload_us": 38, "status_d2h_us": 53, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 66511, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.745760, "gpu_ms": 0.660224, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.048873 + }, + { + "label": "nvidia_htj2k:tile_00027.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 54363, + "cpu": {"status": "ok", "wall_ms": 3.852244, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.234133, "gpu_ms": 0.889000, "stage_ms": 1.053000, "download_ms": 0.060117, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 80, "flatten_us": 9, "h2d_us": 74, "ht_cleanup_us": 709, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 118, "mct_us": 0, "store_us": 62, "total_us": 1053, "wall_total_us": 1153, "table_upload_us": 0, "payload_upload_us": 49, "status_d2h_us": 53, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 53968, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.785110, "gpu_ms": 0.698368, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.600623 + }, + { + "label": "nvidia_htj2k:tile_00028.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 55832, + "cpu": {"status": "ok", "wall_ms": 3.886633, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.093607, "gpu_ms": 0.836000, "stage_ms": 0.947000, "download_ms": 0.055111, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 51, "flatten_us": 5, "h2d_us": 55, "ht_cleanup_us": 648, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 123, "mct_us": 0, "store_us": 65, "total_us": 947, "wall_total_us": 1025, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 28, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 55436, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.796456, "gpu_ms": 0.710656, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.523425 + }, + { + "label": "nvidia_htj2k:tile_00029.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 64814, + "cpu": {"status": "ok", "wall_ms": 3.858900, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.088916, "gpu_ms": 0.826000, "stage_ms": 0.941000, "download_ms": 0.057569, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 53, "flatten_us": 6, "h2d_us": 56, "ht_cleanup_us": 652, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 119, "mct_us": 0, "store_us": 55, "total_us": 941, "wall_total_us": 1018, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 64414, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 1.217741, "gpu_ms": 1.055744, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.130470 + }, + { + "label": "nvidia_htj2k:tile_00030.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 60022, + "cpu": {"status": "ok", "wall_ms": 3.831784, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.097388, "gpu_ms": 0.838000, "stage_ms": 0.951000, "download_ms": 0.053521, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 50, "flatten_us": 6, "h2d_us": 57, "ht_cleanup_us": 655, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 118, "mct_us": 0, "store_us": 65, "total_us": 951, "wall_total_us": 1030, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 31, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 59625, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.803734, "gpu_ms": 0.707616, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.267191 + }, + { + "label": "nvidia_htj2k:tile_00031.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 63236, + "cpu": {"status": "ok", "wall_ms": 3.889601, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.101773, "gpu_ms": 0.836000, "stage_ms": 0.953000, "download_ms": 0.052207, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 51, "flatten_us": 6, "h2d_us": 59, "ht_cleanup_us": 657, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 119, "mct_us": 0, "store_us": 60, "total_us": 953, "wall_total_us": 1036, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 31, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 62838, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.796446, "gpu_ms": 0.710656, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.202816 + }, + { + "label": "nvidia_htj2k:tile_00032.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 59202, + "cpu": {"status": "ok", "wall_ms": 3.844893, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.181504, "gpu_ms": 0.880000, "stage_ms": 0.990000, "download_ms": 0.067524, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 43, "flatten_us": 8, "h2d_us": 59, "ht_cleanup_us": 678, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 125, "mct_us": 0, "store_us": 77, "total_us": 990, "wall_total_us": 1092, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 51, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 58806, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.742727, "gpu_ms": 0.649216, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.356842 + }, + { + "label": "nvidia_htj2k:tile_00033.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 63908, + "cpu": {"status": "ok", "wall_ms": 3.770902, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.221919, "gpu_ms": 0.883000, "stage_ms": 1.034000, "download_ms": 0.060423, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 67, "flatten_us": 11, "h2d_us": 72, "ht_cleanup_us": 672, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 143, "mct_us": 0, "store_us": 68, "total_us": 1034, "wall_total_us": 1141, "table_upload_us": 0, "payload_upload_us": 48, "status_d2h_us": 38, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 63509, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.809816, "gpu_ms": 0.711680, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.161873 + }, + { + "label": "nvidia_htj2k:tile_00034.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 64515, + "cpu": {"status": "ok", "wall_ms": 3.815960, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.122193, "gpu_ms": 0.819000, "stage_ms": 0.952000, "download_ms": 0.068343, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 66, "flatten_us": 11, "h2d_us": 55, "ht_cleanup_us": 645, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 118, "mct_us": 0, "store_us": 56, "total_us": 952, "wall_total_us": 1038, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 64115, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.757924, "gpu_ms": 0.627968, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.185107 + }, + { + "label": "nvidia_htj2k:tile_00035.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 43426, + "cpu": {"status": "ok", "wall_ms": 3.892549, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.078271, "gpu_ms": 0.809000, "stage_ms": 0.918000, "download_ms": 0.064324, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 49, "flatten_us": 5, "h2d_us": 54, "ht_cleanup_us": 614, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 139, "mct_us": 0, "store_us": 56, "total_us": 918, "wall_total_us": 1000, "table_upload_us": 0, "payload_upload_us": 33, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 43038, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.673407, "gpu_ms": 0.570208, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.035916, + "nvidia_psnr_vs_cpu": 52.201175 + }, + { + "label": "nvidia_htj2k:tile_00036.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 56694, + "cpu": {"status": "ok", "wall_ms": 3.789806, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.068525, "gpu_ms": 0.804000, "stage_ms": 0.914000, "download_ms": 0.054103, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 51, "flatten_us": 5, "h2d_us": 54, "ht_cleanup_us": 632, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 115, "mct_us": 0, "store_us": 57, "total_us": 914, "wall_total_us": 1001, "table_upload_us": 0, "payload_upload_us": 34, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 56297, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.685178, "gpu_ms": 0.587680, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.585132 + }, + { + "label": "nvidia_htj2k:tile_00037.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 59973, + "cpu": {"status": "ok", "wall_ms": 3.895510, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.093893, "gpu_ms": 0.836000, "stage_ms": 0.946000, "download_ms": 0.053679, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 50, "flatten_us": 6, "h2d_us": 54, "ht_cleanup_us": 642, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 133, "mct_us": 0, "store_us": 61, "total_us": 946, "wall_total_us": 1027, "table_upload_us": 0, "payload_upload_us": 34, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 59576, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.679856, "gpu_ms": 0.582656, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.035916, + "nvidia_psnr_vs_cpu": 51.375267 + }, + { + "label": "nvidia_htj2k:tile_00038.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 60082, + "cpu": {"status": "ok", "wall_ms": 3.881657, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.097141, "gpu_ms": 0.833000, "stage_ms": 0.947000, "download_ms": 0.054419, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 51, "flatten_us": 7, "h2d_us": 56, "ht_cleanup_us": 640, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 131, "mct_us": 0, "store_us": 62, "total_us": 947, "wall_total_us": 1029, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 59684, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.706033, "gpu_ms": 0.611328, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.427568 + }, + { + "label": "nvidia_htj2k:tile_00039.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 49278, + "cpu": {"status": "ok", "wall_ms": 3.832381, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.091493, "gpu_ms": 0.832000, "stage_ms": 0.937000, "download_ms": 0.054212, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 44, "flatten_us": 7, "h2d_us": 54, "ht_cleanup_us": 637, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 132, "mct_us": 0, "store_us": 63, "total_us": 937, "wall_total_us": 1018, "table_upload_us": 0, "payload_upload_us": 34, "status_d2h_us": 27, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 48887, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.700612, "gpu_ms": 0.603136, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.615835, + "nvidia_psnr_vs_cpu": 51.861930 + }, + { + "label": "nvidia_htj2k:tile_00040.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 62522, + "cpu": {"status": "ok", "wall_ms": 3.709189, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.106217, "gpu_ms": 0.842000, "stage_ms": 0.957000, "download_ms": 0.054607, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 51, "flatten_us": 8, "h2d_us": 56, "ht_cleanup_us": 647, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 133, "mct_us": 0, "store_us": 62, "total_us": 957, "wall_total_us": 1038, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 62124, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.720648, "gpu_ms": 0.614400, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.615835, + "nvidia_psnr_vs_cpu": 51.233034 + }, + { + "label": "nvidia_htj2k:tile_00041.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 62478, + "cpu": {"status": "ok", "wall_ms": 3.799024, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.101289, "gpu_ms": 0.843000, "stage_ms": 0.954000, "download_ms": 0.053324, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 50, "flatten_us": 7, "h2d_us": 53, "ht_cleanup_us": 648, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 134, "mct_us": 0, "store_us": 61, "total_us": 954, "wall_total_us": 1035, "table_upload_us": 0, "payload_upload_us": 34, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 62079, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.743992, "gpu_ms": 0.653248, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 101.066815, + "nvidia_psnr_vs_cpu": 51.227893 + }, + { + "label": "nvidia_htj2k:tile_00042.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 64628, + "cpu": {"status": "ok", "wall_ms": 3.878344, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.105002, "gpu_ms": 0.824000, "stage_ms": 0.956000, "download_ms": 0.055753, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 67, "flatten_us": 11, "h2d_us": 54, "ht_cleanup_us": 651, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 117, "mct_us": 0, "store_us": 56, "total_us": 956, "wall_total_us": 1034, "table_upload_us": 0, "payload_upload_us": 35, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 64229, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.704483, "gpu_ms": 0.602112, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 101.066815, + "nvidia_psnr_vs_cpu": 51.121225 + }, + { + "label": "nvidia_htj2k:tile_00043.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 62293, + "cpu": {"status": "ok", "wall_ms": 3.771909, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.104636, "gpu_ms": 0.834000, "stage_ms": 0.952000, "download_ms": 0.055605, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 51, "flatten_us": 7, "h2d_us": 60, "ht_cleanup_us": 661, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 113, "mct_us": 0, "store_us": 60, "total_us": 952, "wall_total_us": 1035, "table_upload_us": 0, "payload_upload_us": 38, "status_d2h_us": 28, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 61895, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.777437, "gpu_ms": 0.666592, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.237548 + }, + { + "label": "nvidia_htj2k:tile_00044.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 63944, + "cpu": {"status": "ok", "wall_ms": 3.898161, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.107085, "gpu_ms": 0.844000, "stage_ms": 0.956000, "download_ms": 0.055980, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 44, "flatten_us": 11, "h2d_us": 57, "ht_cleanup_us": 665, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 119, "mct_us": 0, "store_us": 60, "total_us": 956, "wall_total_us": 1036, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 63546, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.733070, "gpu_ms": 0.625664, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.615835, + "nvidia_psnr_vs_cpu": 51.187339 + }, + { + "label": "nvidia_htj2k:tile_00045.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 63422, + "cpu": {"status": "ok", "wall_ms": 3.821273, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.090416, "gpu_ms": 0.832000, "stage_ms": 0.939000, "download_ms": 0.054499, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 42, "flatten_us": 10, "h2d_us": 55, "ht_cleanup_us": 659, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 115, "mct_us": 0, "store_us": 58, "total_us": 939, "wall_total_us": 1021, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 63024, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.761806, "gpu_ms": 0.675840, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 101.066815, + "nvidia_psnr_vs_cpu": 51.213808 + }, + { + "label": "nvidia_htj2k:tile_00046.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 62858, + "cpu": {"status": "ok", "wall_ms": 3.926952, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.084758, "gpu_ms": 0.828000, "stage_ms": 0.939000, "download_ms": 0.054746, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 43, "flatten_us": 10, "h2d_us": 58, "ht_cleanup_us": 654, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 106, "mct_us": 0, "store_us": 68, "total_us": 939, "wall_total_us": 1014, "table_upload_us": 0, "payload_upload_us": 38, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 62460, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.763425, "gpu_ms": 0.675840, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.235832 + }, + { + "label": "nvidia_htj2k:tile_00047.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 63308, + "cpu": {"status": "ok", "wall_ms": 3.964037, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.082793, "gpu_ms": 0.816000, "stage_ms": 0.932000, "download_ms": 0.052494, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 52, "flatten_us": 7, "h2d_us": 57, "ht_cleanup_us": 667, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 93, "mct_us": 0, "store_us": 56, "total_us": 932, "wall_total_us": 1017, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 27, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 62909, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.782078, "gpu_ms": 0.696224, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.177438 + }, + { + "label": "nvidia_htj2k:tile_00048.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 60750, + "cpu": {"status": "ok", "wall_ms": 3.854427, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.072959, "gpu_ms": 0.803000, "stage_ms": 0.921000, "download_ms": 0.056009, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 51, "flatten_us": 7, "h2d_us": 60, "ht_cleanup_us": 654, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 93, "mct_us": 0, "store_us": 56, "total_us": 921, "wall_total_us": 1003, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 28, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 60353, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.768688, "gpu_ms": 0.654336, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.398736 + }, + { + "label": "nvidia_htj2k:tile_00049.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 60410, + "cpu": {"status": "ok", "wall_ms": 3.929134, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.098346, "gpu_ms": 0.832000, "stage_ms": 0.948000, "download_ms": 0.056039, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 51, "flatten_us": 6, "h2d_us": 59, "ht_cleanup_us": 659, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 118, "mct_us": 0, "store_us": 55, "total_us": 948, "wall_total_us": 1029, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 60013, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.710645, "gpu_ms": 0.612256, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.319442 + }, + { + "label": "nvidia_htj2k:tile_00050.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 65322, + "cpu": {"status": "ok", "wall_ms": 3.910747, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.104291, "gpu_ms": 0.838000, "stage_ms": 0.955000, "download_ms": 0.056414, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 52, "flatten_us": 7, "h2d_us": 58, "ht_cleanup_us": 660, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 117, "mct_us": 0, "store_us": 61, "total_us": 955, "wall_total_us": 1034, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 64922, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.715868, "gpu_ms": 0.617472, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 101.066815, + "nvidia_psnr_vs_cpu": 51.110902 + }, + { + "label": "nvidia_htj2k:tile_00051.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 56310, + "cpu": {"status": "ok", "wall_ms": 3.908768, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.081698, "gpu_ms": 0.815000, "stage_ms": 0.931000, "download_ms": 0.056592, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 50, "flatten_us": 6, "h2d_us": 60, "ht_cleanup_us": 642, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 117, "mct_us": 0, "store_us": 56, "total_us": 931, "wall_total_us": 1011, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 28, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 55914, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.717515, "gpu_ms": 0.575264, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.629942 + }, + { + "label": "nvidia_htj2k:tile_00052.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 55950, + "cpu": {"status": "ok", "wall_ms": 3.909370, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.094159, "gpu_ms": 0.830000, "stage_ms": 0.942000, "download_ms": 0.057550, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 43, "flatten_us": 10, "h2d_us": 59, "ht_cleanup_us": 652, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 117, "mct_us": 0, "store_us": 61, "total_us": 942, "wall_total_us": 1022, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 29, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 55558, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.714170, "gpu_ms": 0.621568, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.035916, + "nvidia_psnr_vs_cpu": 51.526031 + }, + { + "label": "nvidia_htj2k:tile_00053.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 54672, + "cpu": {"status": "ok", "wall_ms": 3.822329, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.079555, "gpu_ms": 0.817000, "stage_ms": 0.931000, "download_ms": 0.056799, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 49, "flatten_us": 6, "h2d_us": 59, "ht_cleanup_us": 644, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 117, "mct_us": 0, "store_us": 56, "total_us": 931, "wall_total_us": 1009, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 28, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 54276, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.711257, "gpu_ms": 0.622592, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 92.615835, + "nvidia_psnr_vs_cpu": 51.698261 + }, + { + "label": "nvidia_htj2k:tile_00054.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 66731, + "cpu": {"status": "ok", "wall_ms": 3.847712, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.096529, "gpu_ms": 0.831000, "stage_ms": 0.947000, "download_ms": 0.054963, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 49, "flatten_us": 7, "h2d_us": 59, "ht_cleanup_us": 657, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 118, "mct_us": 0, "store_us": 56, "total_us": 947, "wall_total_us": 1028, "table_upload_us": 0, "payload_upload_us": 38, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 66332, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.687765, "gpu_ms": 0.599040, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.049478 + }, + { + "label": "nvidia_htj2k:tile_00055.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 51507, + "cpu": {"status": "ok", "wall_ms": 3.757817, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.072760, "gpu_ms": 0.790000, "stage_ms": 0.901000, "download_ms": 0.061056, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 44, "flatten_us": 8, "h2d_us": 58, "ht_cleanup_us": 642, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 92, "mct_us": 0, "store_us": 56, "total_us": 901, "wall_total_us": 994, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 29, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 51114, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.736694, "gpu_ms": 0.627712, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.814726 + }, + { + "label": "nvidia_htj2k:tile_00056.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 54088, + "cpu": {"status": "ok", "wall_ms": 3.830653, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.083426, "gpu_ms": 0.818000, "stage_ms": 0.936000, "download_ms": 0.056671, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 51, "flatten_us": 7, "h2d_us": 59, "ht_cleanup_us": 642, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 115, "mct_us": 0, "store_us": 61, "total_us": 936, "wall_total_us": 1013, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 27, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 53692, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.718554, "gpu_ms": 0.619360, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 93.285303, + "nvidia_psnr_vs_cpu": 51.678717 + }, + { + "label": "nvidia_htj2k:tile_00057.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 63657, + "cpu": {"status": "ok", "wall_ms": 3.855340, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.106038, "gpu_ms": 0.839000, "stage_ms": 0.957000, "download_ms": 0.056523, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 46, "flatten_us": 11, "h2d_us": 60, "ht_cleanup_us": 665, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 118, "mct_us": 0, "store_us": 56, "total_us": 957, "wall_total_us": 1035, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 63258, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.691793, "gpu_ms": 0.590848, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 93.285303, + "nvidia_psnr_vs_cpu": 51.140927 + }, + { + "label": "nvidia_htj2k:tile_00058.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 57277, + "cpu": {"status": "ok", "wall_ms": 3.860347, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.077648, "gpu_ms": 0.812000, "stage_ms": 0.928000, "download_ms": 0.057461, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 52, "flatten_us": 6, "h2d_us": 57, "ht_cleanup_us": 639, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 120, "mct_us": 0, "store_us": 53, "total_us": 928, "wall_total_us": 1007, "table_upload_us": 0, "payload_upload_us": 35, "status_d2h_us": 26, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 56882, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.749175, "gpu_ms": 0.656128, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 94.077115, + "nvidia_psnr_vs_cpu": 51.504555 + }, + { + "label": "nvidia_htj2k:tile_00059.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 58265, + "cpu": {"status": "ok", "wall_ms": 3.881750, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.076493, "gpu_ms": 0.812000, "stage_ms": 0.928000, "download_ms": 0.053067, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 50, "flatten_us": 7, "h2d_us": 59, "ht_cleanup_us": 645, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 112, "mct_us": 0, "store_us": 55, "total_us": 928, "wall_total_us": 1010, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 29, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 57867, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.756434, "gpu_ms": 0.663552, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.431062 + }, + { + "label": "nvidia_htj2k:tile_00060.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 66405, + "cpu": {"status": "ok", "wall_ms": 3.871061, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.108833, "gpu_ms": 0.845000, "stage_ms": 0.966000, "download_ms": 0.052642, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 53, "flatten_us": 7, "h2d_us": 60, "ht_cleanup_us": 664, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 119, "mct_us": 0, "store_us": 62, "total_us": 966, "wall_total_us": 1043, "table_upload_us": 0, "payload_upload_us": 38, "status_d2h_us": 31, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 66005, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.720618, "gpu_ms": 0.636704, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.032856 + }, + { + "label": "nvidia_htj2k:tile_00061.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 51524, + "cpu": {"status": "ok", "wall_ms": 3.807557, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.067103, "gpu_ms": 0.805000, "stage_ms": 0.919000, "download_ms": 0.054874, "bytes": 196608, "cuda_profile": {"parse_us": 0, "plan_us": 49, "flatten_us": 6, "h2d_us": 59, "ht_cleanup_us": 632, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 116, "mct_us": 0, "store_us": 57, "total_us": 919, "wall_total_us": 999, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 27, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 51128, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.720667, "gpu_ms": 0.632832, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 95.046216, + "nvidia_psnr_vs_cpu": 51.886932 + }, + { + "label": "nvidia_htj2k:tile_00062.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 57806, + "cpu": {"status": "ok", "wall_ms": 3.950306, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.060052, "gpu_ms": 0.789000, "stage_ms": 0.908000, "download_ms": 0.052702, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 53, "flatten_us": 7, "h2d_us": 58, "ht_cleanup_us": 640, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 94, "mct_us": 0, "store_us": 55, "total_us": 908, "wall_total_us": 994, "table_upload_us": 0, "payload_upload_us": 36, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 57410, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.749857, "gpu_ms": 0.645184, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 98.056516, + "nvidia_psnr_vs_cpu": 51.489019 + }, + { + "label": "nvidia_htj2k:tile_00063.jpg", + "format": "rgb8", + "width": 256, + "height": 256, + "codestream_bytes": 66463, + "cpu": {"status": "ok", "wall_ms": 3.885157, "gpu_ms": null, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda": {"status": "ok", "wall_ms": 1.082458, "gpu_ms": 0.823000, "stage_ms": 0.941000, "download_ms": 0.053432, "bytes": 196608, "cuda_profile": {"parse_us": 1, "plan_us": 53, "flatten_us": 8, "h2d_us": 56, "ht_cleanup_us": 650, "ht_refine_us": 0, "dequant_us": 0, "idwt_us": 119, "mct_us": 0, "store_us": 54, "total_us": 941, "wall_total_us": 1016, "table_upload_us": 0, "payload_upload_us": 37, "status_d2h_us": 25, "output_d2h_us": 0, "block_count": 75, "payload_bytes": 66064, "dispatch_count": 12, "ht_dispatch_count": 1, "dequant_dispatch_count": 0, "idwt_dispatch_count": 10, "mct_dispatch_count": 0, "store_dispatch_count": 1}}, + "nvidia_nvjpeg2000": {"status": "ok", "wall_ms": 0.688881, "gpu_ms": 0.587648, "stage_ms": null, "download_ms": null, "bytes": 196608}, + "j2k_cuda_psnr_vs_cpu": 96.295603, + "nvidia_psnr_vs_cpu": 51.035141 + } + ] +} diff --git a/tests/nvidia-baseline/benchtiles/pancreas/README.md b/tests/nvidia-baseline/benchtiles/pancreas/README.md new file mode 100644 index 00000000..f90d17d5 --- /dev/null +++ b/tests/nvidia-baseline/benchtiles/pancreas/README.md @@ -0,0 +1,28 @@ +# Pancreas WSI benchmark tiles + +109 H&E tissue tiles (256×256, baseline JPEG, quality 85) for the +`transcode_compare` JPEG → HTJ2K benchmark. + +## Provenance + +Derived from an open-access GDC/TCGA pancreas whole-slide image (`Pancreas.svs`, +Aperio SVS). The slide stores its tiles as JPEG 2000 with YCbCr components +(Aperio compression `33003`), so they are not usable as JPEG input directly. + +Each tile here was produced by `svs_extract`: +1. decode the J2K tile to component samples (`j2k-native`), +2. convert YCbCr → RGB, +3. keep only tiles with ≥ 60% tissue coverage and visible structure (skipping + flat stroma and bright glass/background), +4. re-encode as baseline JPEG. + +Re-encoding adds one lossy step, so these are realistic WSI *content* at a +realistic tile size, not byte-identical originals — appropriate for a throughput +benchmark, and the PSNR reference is self-consistent across codecs. + +## Reproduce + +```bash +cargo run --release --manifest-path tests/nvidia-baseline/Cargo.toml --bin svs_extract -- \ + /path/to/Pancreas.svs out_dir --limit 128 --stride 51 --quality 85 --min-tissue 0.6 +``` diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00000.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00000.jpg new file mode 100644 index 00000000..8d87c219 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00000.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00001.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00001.jpg new file mode 100644 index 00000000..1bde86bd Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00001.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00002.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00002.jpg new file mode 100644 index 00000000..ef26cb13 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00002.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00003.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00003.jpg new file mode 100644 index 00000000..d256410a Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00003.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00004.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00004.jpg new file mode 100644 index 00000000..b2227016 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00004.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00005.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00005.jpg new file mode 100644 index 00000000..dacbf774 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00005.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00006.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00006.jpg new file mode 100644 index 00000000..d028ea29 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00006.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00007.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00007.jpg new file mode 100644 index 00000000..5c6070ff Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00007.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00008.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00008.jpg new file mode 100644 index 00000000..2c0d7418 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00008.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00009.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00009.jpg new file mode 100644 index 00000000..893f17bd Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00009.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00010.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00010.jpg new file mode 100644 index 00000000..d95d120b Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00010.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00011.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00011.jpg new file mode 100644 index 00000000..7a97ad77 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00011.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00012.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00012.jpg new file mode 100644 index 00000000..8b211098 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00012.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00013.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00013.jpg new file mode 100644 index 00000000..ca5b4789 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00013.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00014.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00014.jpg new file mode 100644 index 00000000..507ab257 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00014.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00015.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00015.jpg new file mode 100644 index 00000000..49622fbf Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00015.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00016.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00016.jpg new file mode 100644 index 00000000..4d726fdd Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00016.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00017.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00017.jpg new file mode 100644 index 00000000..fba1b4c4 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00017.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00018.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00018.jpg new file mode 100644 index 00000000..1eaef9b3 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00018.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00019.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00019.jpg new file mode 100644 index 00000000..991e7d2c Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00019.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00020.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00020.jpg new file mode 100644 index 00000000..dd0c2b60 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00020.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00021.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00021.jpg new file mode 100644 index 00000000..91b1cac6 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00021.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00022.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00022.jpg new file mode 100644 index 00000000..d349c878 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00022.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00023.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00023.jpg new file mode 100644 index 00000000..c953e9e5 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00023.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00024.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00024.jpg new file mode 100644 index 00000000..a6141104 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00024.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00025.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00025.jpg new file mode 100644 index 00000000..51de1855 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00025.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00026.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00026.jpg new file mode 100644 index 00000000..c82465c3 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00026.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00027.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00027.jpg new file mode 100644 index 00000000..284913d7 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00027.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00028.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00028.jpg new file mode 100644 index 00000000..91da8e92 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00028.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00029.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00029.jpg new file mode 100644 index 00000000..eb9e8530 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00029.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00030.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00030.jpg new file mode 100644 index 00000000..bbfde5e8 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00030.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00031.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00031.jpg new file mode 100644 index 00000000..e7682f38 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00031.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00032.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00032.jpg new file mode 100644 index 00000000..aeddd96b Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00032.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00033.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00033.jpg new file mode 100644 index 00000000..ba1bcfee Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00033.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00034.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00034.jpg new file mode 100644 index 00000000..fcd05893 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00034.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00035.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00035.jpg new file mode 100644 index 00000000..e12a5562 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00035.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00036.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00036.jpg new file mode 100644 index 00000000..dc5fba44 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00036.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00037.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00037.jpg new file mode 100644 index 00000000..34213f06 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00037.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00038.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00038.jpg new file mode 100644 index 00000000..9d83a834 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00038.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00039.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00039.jpg new file mode 100644 index 00000000..d92ba921 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00039.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00040.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00040.jpg new file mode 100644 index 00000000..ab313076 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00040.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00041.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00041.jpg new file mode 100644 index 00000000..e0de979e Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00041.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00042.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00042.jpg new file mode 100644 index 00000000..0ffbfbba Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00042.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00043.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00043.jpg new file mode 100644 index 00000000..a0ffa1c6 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00043.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00044.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00044.jpg new file mode 100644 index 00000000..505ea646 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00044.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00045.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00045.jpg new file mode 100644 index 00000000..2df9df23 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00045.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00046.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00046.jpg new file mode 100644 index 00000000..6aa177e2 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00046.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00047.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00047.jpg new file mode 100644 index 00000000..e5c63295 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00047.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00048.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00048.jpg new file mode 100644 index 00000000..722c8d4f Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00048.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00049.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00049.jpg new file mode 100644 index 00000000..3d2d68d1 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00049.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00050.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00050.jpg new file mode 100644 index 00000000..98fc574b Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00050.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00051.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00051.jpg new file mode 100644 index 00000000..636a8c12 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00051.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00052.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00052.jpg new file mode 100644 index 00000000..b4f86677 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00052.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00053.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00053.jpg new file mode 100644 index 00000000..7874513c Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00053.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00054.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00054.jpg new file mode 100644 index 00000000..09f1a220 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00054.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00055.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00055.jpg new file mode 100644 index 00000000..822ea0c1 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00055.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00056.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00056.jpg new file mode 100644 index 00000000..e2f0f133 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00056.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00057.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00057.jpg new file mode 100644 index 00000000..3563c5c8 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00057.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00058.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00058.jpg new file mode 100644 index 00000000..174ad680 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00058.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00059.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00059.jpg new file mode 100644 index 00000000..8b1410e6 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00059.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00060.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00060.jpg new file mode 100644 index 00000000..31a1027f Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00060.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00061.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00061.jpg new file mode 100644 index 00000000..69b344d7 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00061.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00062.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00062.jpg new file mode 100644 index 00000000..c22fe329 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00062.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00063.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00063.jpg new file mode 100644 index 00000000..79552f11 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00063.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00064.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00064.jpg new file mode 100644 index 00000000..95dc45fe Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00064.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00065.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00065.jpg new file mode 100644 index 00000000..8c0b0db7 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00065.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00066.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00066.jpg new file mode 100644 index 00000000..64692a8c Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00066.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00067.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00067.jpg new file mode 100644 index 00000000..15ec9ad8 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00067.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00068.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00068.jpg new file mode 100644 index 00000000..df611fbe Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00068.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00069.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00069.jpg new file mode 100644 index 00000000..ee8e16c6 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00069.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00070.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00070.jpg new file mode 100644 index 00000000..f3ecb986 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00070.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00071.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00071.jpg new file mode 100644 index 00000000..936006b4 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00071.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00072.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00072.jpg new file mode 100644 index 00000000..c833a8f8 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00072.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00073.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00073.jpg new file mode 100644 index 00000000..03c04cf7 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00073.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00074.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00074.jpg new file mode 100644 index 00000000..8b7fc2ac Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00074.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00075.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00075.jpg new file mode 100644 index 00000000..a0e1e459 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00075.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00076.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00076.jpg new file mode 100644 index 00000000..7a222929 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00076.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00077.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00077.jpg new file mode 100644 index 00000000..2d6ee5da Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00077.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00078.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00078.jpg new file mode 100644 index 00000000..6cbeb336 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00078.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00079.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00079.jpg new file mode 100644 index 00000000..6d30d084 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00079.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00080.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00080.jpg new file mode 100644 index 00000000..3f699795 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00080.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00081.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00081.jpg new file mode 100644 index 00000000..b214ca43 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00081.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00082.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00082.jpg new file mode 100644 index 00000000..24be86a3 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00082.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00083.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00083.jpg new file mode 100644 index 00000000..35d7b0e5 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00083.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00084.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00084.jpg new file mode 100644 index 00000000..78cef9de Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00084.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00085.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00085.jpg new file mode 100644 index 00000000..314f55f3 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00085.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00086.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00086.jpg new file mode 100644 index 00000000..9dd0dcab Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00086.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00087.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00087.jpg new file mode 100644 index 00000000..b29de3ca Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00087.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00088.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00088.jpg new file mode 100644 index 00000000..0eb82dde Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00088.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00089.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00089.jpg new file mode 100644 index 00000000..0aa61875 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00089.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00090.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00090.jpg new file mode 100644 index 00000000..8d5258fe Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00090.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00091.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00091.jpg new file mode 100644 index 00000000..b08e2d84 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00091.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00092.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00092.jpg new file mode 100644 index 00000000..ffd92084 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00092.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00093.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00093.jpg new file mode 100644 index 00000000..e95987e7 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00093.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00094.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00094.jpg new file mode 100644 index 00000000..c1b98c08 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00094.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00095.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00095.jpg new file mode 100644 index 00000000..ced8e366 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00095.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00096.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00096.jpg new file mode 100644 index 00000000..0be35a56 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00096.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00097.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00097.jpg new file mode 100644 index 00000000..344aee44 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00097.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00098.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00098.jpg new file mode 100644 index 00000000..82ca415e Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00098.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00099.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00099.jpg new file mode 100644 index 00000000..bbce4c45 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00099.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00100.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00100.jpg new file mode 100644 index 00000000..ae34d8d1 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00100.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00101.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00101.jpg new file mode 100644 index 00000000..f5a5ba1a Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00101.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00102.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00102.jpg new file mode 100644 index 00000000..bb2f8cc6 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00102.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00103.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00103.jpg new file mode 100644 index 00000000..5515143a Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00103.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00104.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00104.jpg new file mode 100644 index 00000000..9c497c3d Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00104.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00105.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00105.jpg new file mode 100644 index 00000000..c66d5657 Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00105.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00106.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00106.jpg new file mode 100644 index 00000000..4d68f39d Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00106.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00107.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00107.jpg new file mode 100644 index 00000000..dd659fee Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00107.jpg differ diff --git a/tests/nvidia-baseline/benchtiles/pancreas/tile_00108.jpg b/tests/nvidia-baseline/benchtiles/pancreas/tile_00108.jpg new file mode 100644 index 00000000..23e2056c Binary files /dev/null and b/tests/nvidia-baseline/benchtiles/pancreas/tile_00108.jpg differ diff --git a/tests/nvidia-baseline/build.rs b/tests/nvidia-baseline/build.rs new file mode 100644 index 00000000..fbaa7df2 --- /dev/null +++ b/tests/nvidia-baseline/build.rs @@ -0,0 +1,198 @@ +use std::env; +use std::ffi::OsStr; +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +// Compiles the C++ nvJPEG/nvJPEG2000 baseline (`cuda/nv_baseline.cu`) into a +// static library and links it plus the NVIDIA codec libraries, but ONLY when the +// `nvjpeg2000` feature is enabled AND nvcc is available. On any other host this +// is a no-op so the workspace still builds (the Rust side is cfg-gated on +// `nvbaseline_built`). +// +// Library locations can be overridden for non-standard installs: +// CUDA_LIB_DIR (default: /usr/local/cuda/targets/x86_64-linux/lib) +// NVJPEG2K_LIB_DIR (default: CUDA_LIB_DIR) +// NVJPEG2K_INCLUDE_DIR (passed to nvcc as -I if set) +// Set J2K_REQUIRE_NV_BASELINE_BUILD=1 to make an nvcc failure fatal. +fn main() { + println!("cargo:rerun-if-changed=cuda/nv_baseline.cu"); + println!("cargo:rerun-if-env-changed=NVCC"); + println!("cargo:rerun-if-env-changed=CUDA_LIB_DIR"); + println!("cargo:rerun-if-env-changed=NVJPEG2K_LIB_DIR"); + println!("cargo:rerun-if-env-changed=NVJPEG2K_INCLUDE_DIR"); + println!("cargo:rerun-if-env-changed=J2K_REQUIRE_NV_BASELINE_BUILD"); + println!("cargo:rustc-check-cfg=cfg(nvbaseline_built)"); + + if env::var_os("CARGO_FEATURE_NVJPEG2000").is_none() { + return; + } + + let out_dir = PathBuf::from(env::var_os("OUT_DIR").expect("OUT_DIR is set by cargo")); + let strict = env::var_os("J2K_REQUIRE_NV_BASELINE_BUILD").is_some(); + let nvcc = configured_nvcc(strict); + + let object = out_dir.join("nv_baseline.o"); + let archive = out_dir.join("libnvbaseline.a"); + + let include_dir = env::var_os("NVJPEG2K_INCLUDE_DIR"); + let stream_parse_define = match probe_nvjpeg2k_decode_api( + nvcc.as_deref(), + include_dir.as_deref(), + &out_dir, + ) { + Ok(define) => define, + Err(message) => { + assert!( + !strict, + "J2K_REQUIRE_NV_BASELINE_BUILD set, but nvJPEG2000 decode APIs are unavailable: {message}" + ); + return; + } + }; + + let Some(nvcc) = nvcc.as_deref() else { + return; + }; + + let mut compile = Command::new(nvcc); + compile + .args(["-c", "-O3", "--std=c++14", "-Xcompiler", "-fPIC"]) + .arg("cuda/nv_baseline.cu") + .arg("-o") + .arg(&object); + if let Some(define) = stream_parse_define { + compile.arg(define); + } + if let Some(include_dir) = include_dir { + compile.arg("-I").arg(include_dir); + } + + let compiled = compile.status().is_ok_and(|status| status.success()); + if !compiled { + assert!( + !strict, + "J2K_REQUIRE_NV_BASELINE_BUILD set, but nvcc failed to compile cuda/nv_baseline.cu" + ); + // No nvcc / NVIDIA headers: leave `nvbaseline_built` unset; the binary + // prints rebuild instructions instead of linking against absent libs. + return; + } + + let archived = Command::new("ar") + .arg("rcs") + .arg(&archive) + .arg(&object) + .status() + .is_ok_and(|status| status.success()); + assert!(archived, "ar failed to archive nv_baseline.o"); + + let cuda_lib_dir = env::var("CUDA_LIB_DIR") + .unwrap_or_else(|_| "/usr/local/cuda/targets/x86_64-linux/lib".to_string()); + let nvjpeg2k_lib_dir = env::var("NVJPEG2K_LIB_DIR").unwrap_or_else(|_| cuda_lib_dir.clone()); + + println!("cargo:rustc-link-search=native={}", out_dir.display()); + println!("cargo:rustc-link-search=native={cuda_lib_dir}"); + println!("cargo:rustc-link-search=native={nvjpeg2k_lib_dir}"); + println!("cargo:rustc-link-lib=static=nvbaseline"); + println!("cargo:rustc-link-lib=dylib=nvjpeg2k"); + println!("cargo:rustc-link-lib=dylib=nvjpeg"); + println!("cargo:rustc-link-lib=dylib=cudart"); + println!("cargo:rustc-link-lib=dylib=stdc++"); + println!("cargo:rustc-cfg=nvbaseline_built"); +} + +fn probe_nvjpeg2k_decode_api( + nvcc: Option<&OsStr>, + include_dir: Option<&OsStr>, + out_dir: &std::path::Path, +) -> Result, String> { + let Some(nvcc) = nvcc else { + return Err("NVCC not configured".to_string()); + }; + let current = r" +#include +#include +int main() { + nvjpeg2kHandle_t handle = nullptr; + nvjpeg2kDecodeState_t decode_state = nullptr; + nvjpeg2kDecodeParams_t decode_params = nullptr; + nvjpeg2kStream_t stream = nullptr; + unsigned char data[4] = {0, 0, 0, 0}; + nvjpeg2kImage_t image; + nvjpeg2kCreateSimple(&handle); + nvjpeg2kDecodeStateCreate(handle, &decode_state); + nvjpeg2kDecodeParamsCreate(&decode_params); + nvjpeg2kStreamCreate(&stream); + nvjpeg2kDecodeParamsSetOutputFormat(decode_params, NVJPEG2K_FORMAT_INTERLEAVED); + nvjpeg2kDecodeParamsSetRGBOutput(decode_params, 1); + nvjpeg2kStreamParse(handle, data, sizeof(data), 0, 0, stream); + nvjpeg2kDecodeImage(handle, decode_state, stream, decode_params, &image, 0); + return 0; +} +"; + if compile_probe( + nvcc, + include_dir, + out_dir, + "nvjpeg2k_decode_current.cu", + current, + ) { + return Ok(None); + } + + let legacy = current.replace( + "nvjpeg2kStreamParse(handle, data, sizeof(data), 0, 0, stream);", + "nvjpeg2kStreamParse(handle, data, sizeof(data), 0, 0, &stream);", + ); + if compile_probe( + nvcc, + include_dir, + out_dir, + "nvjpeg2k_decode_legacy.cu", + &legacy, + ) { + return Ok(Some("-DNVB_STREAM_PARSE_USES_OUT_POINTER=1")); + } + + Err("neither current nor legacy nvjpeg2kStreamParse decode probes compiled".to_string()) +} + +fn compile_probe( + nvcc: &OsStr, + include_dir: Option<&OsStr>, + out_dir: &std::path::Path, + name: &str, + source: &str, +) -> bool { + let source_path = out_dir.join(name); + let object_path = out_dir.join(format!("{name}.o")); + if fs::write(&source_path, source).is_err() { + return false; + } + let mut command = Command::new(nvcc); + command + .args(["-c", "--std=c++14"]) + .arg(&source_path) + .arg("-o") + .arg(&object_path); + if let Some(include_dir) = include_dir { + command.arg("-I").arg(include_dir); + } + command.status().is_ok_and(|status| status.success()) +} + +fn configured_nvcc(strict: bool) -> Option { + let nvcc = env::var_os("NVCC"); + if strict { + let nvcc = nvcc.expect("strict NVIDIA baseline build requires absolute NVCC"); + assert!( + std::path::Path::new(&nvcc).is_absolute(), + "strict NVIDIA baseline build requires absolute NVCC, got {}", + std::path::Path::new(&nvcc).display() + ); + Some(nvcc) + } else { + nvcc + } +} diff --git a/tests/nvidia-baseline/cuda/nv_baseline.cu b/tests/nvidia-baseline/cuda/nv_baseline.cu new file mode 100644 index 00000000..a08fb0f5 --- /dev/null +++ b/tests/nvidia-baseline/cuda/nv_baseline.cu @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// NVIDIA GPU baseline for the JPEG -> HTJ2K transcode comparison: nvJPEG decodes +// the JPEG to RGB on the GPU, then nvJPEG2000 encodes it to a High-Throughput +// JPEG 2000 (HTJ2K) codestream on the GPU. This is the apples-to-apples NVIDIA +// path against j2k's coefficient-domain transcode (which skips the pixel +// round-trip). Exposes a tiny C ABI surface for the Rust wrapper. +// +// Compiled by build.rs with nvcc only when the `nvjpeg2000` feature is on and +// the libraries are present. nvJPEG2000 ships separately from the CUDA toolkit. +// +// All locals are declared up front: the cleanup paths use `goto`, and C++ +// forbids a goto from jumping over a variable's initialization. + +#include +#include +#include +#include +#include + +// HT enablement macros (guarded in case an older header predates them). +#ifndef NVJPEG2K_RSIZ_HT +#define NVJPEG2K_RSIZ_HT 0x4000 +#endif +#ifndef NVJPEG2K_MODE_HT +#define NVJPEG2K_MODE_HT 0x40 +#endif + +struct NvbSession { + cudaStream_t stream = nullptr; + cudaEvent_t start = nullptr; + cudaEvent_t mid = nullptr; + cudaEvent_t stop = nullptr; + nvjpegHandle_t jpeg_handle = nullptr; + nvjpegJpegState_t jpeg_state = nullptr; + nvjpeg2kHandle_t dec_handle = nullptr; + nvjpeg2kDecodeState_t dec_state = nullptr; + nvjpeg2kDecodeParams_t dec_params = nullptr; + nvjpeg2kStream_t dec_stream = nullptr; + nvjpeg2kEncoder_t enc = nullptr; + nvjpeg2kEncodeState_t enc_state = nullptr; + nvjpeg2kEncodeParams_t enc_params = nullptr; + unsigned char* planes[3] = {nullptr, nullptr, nullptr}; + size_t plane_capacity = 0; + unsigned char* decode_interleaved = nullptr; + size_t decode_interleaved_capacity = 0; +}; + +static void nvb_session_release_planes(NvbSession* session) { + for (int c = 0; c < 3; ++c) { + if (session->planes[c]) { + cudaFree(session->planes[c]); + session->planes[c] = nullptr; + } + } + session->plane_capacity = 0; +} + +static int nvb_session_ensure_planes(NvbSession* session, size_t plane_bytes) { + if (session->plane_capacity >= plane_bytes) { + return 0; + } + nvb_session_release_planes(session); + for (int c = 0; c < 3; ++c) { + if (cudaMalloc((void**)&session->planes[c], plane_bytes) != cudaSuccess) { + nvb_session_release_planes(session); + return 902; + } + } + session->plane_capacity = plane_bytes; + return 0; +} + +static void nvb_session_release_decode_interleaved(NvbSession* session) { + if (session->decode_interleaved) { + cudaFree(session->decode_interleaved); + session->decode_interleaved = nullptr; + } + session->decode_interleaved_capacity = 0; +} + +static int nvb_session_ensure_decode_interleaved(NvbSession* session, size_t bytes) { + if (session->decode_interleaved_capacity >= bytes) { + return 0; + } + nvb_session_release_decode_interleaved(session); + if (cudaMalloc((void**)&session->decode_interleaved, bytes) != cudaSuccess) { + return 902; + } + session->decode_interleaved_capacity = bytes; + return 0; +} + +extern "C" { + +void nvb_session_destroy(NvbSession* session) { + if (!session) { + return; + } + if (session->stream) { cudaStreamSynchronize(session->stream); } + nvb_session_release_decode_interleaved(session); + nvb_session_release_planes(session); + if (session->enc_params) { nvjpeg2kEncodeParamsDestroy(session->enc_params); } + if (session->enc_state) { nvjpeg2kEncodeStateDestroy(session->enc_state); } + if (session->enc) { nvjpeg2kEncoderDestroy(session->enc); } + if (session->dec_stream) { nvjpeg2kStreamDestroy(session->dec_stream); } + if (session->dec_params) { nvjpeg2kDecodeParamsDestroy(session->dec_params); } + if (session->dec_state) { nvjpeg2kDecodeStateDestroy(session->dec_state); } + if (session->dec_handle) { nvjpeg2kDestroy(session->dec_handle); } + if (session->jpeg_state) { nvjpegJpegStateDestroy(session->jpeg_state); } + if (session->jpeg_handle) { nvjpegDestroy(session->jpeg_handle); } + if (session->start) { cudaEventDestroy(session->start); } + if (session->mid) { cudaEventDestroy(session->mid); } + if (session->stop) { cudaEventDestroy(session->stop); } + if (session->stream) { cudaStreamDestroy(session->stream); } + delete session; +} + +int nvb_session_create(NvbSession** out) { + int rc = 0; + NvbSession* session = nullptr; + if (!out) { return 900; } + *out = nullptr; + session = new (std::nothrow) NvbSession(); + if (!session) { return 904; } + + if (cudaStreamCreate(&session->stream) != cudaSuccess) { rc = 901; goto cleanup; } + if (cudaEventCreate(&session->start) != cudaSuccess) { rc = 905; goto cleanup; } + if (cudaEventCreate(&session->mid) != cudaSuccess) { rc = 905; goto cleanup; } + if (cudaEventCreate(&session->stop) != cudaSuccess) { rc = 905; goto cleanup; } + if (nvjpegCreateSimple(&session->jpeg_handle) != NVJPEG_STATUS_SUCCESS) { rc = 101; goto cleanup; } + if (nvjpegJpegStateCreate(session->jpeg_handle, &session->jpeg_state) != NVJPEG_STATUS_SUCCESS) { rc = 102; goto cleanup; } + if (nvjpeg2kCreateSimple(&session->dec_handle) != NVJPEG2K_STATUS_SUCCESS) { rc = 221; goto cleanup; } + if (nvjpeg2kDecodeStateCreate(session->dec_handle, &session->dec_state) != NVJPEG2K_STATUS_SUCCESS) { rc = 222; goto cleanup; } + if (nvjpeg2kDecodeParamsCreate(&session->dec_params) != NVJPEG2K_STATUS_SUCCESS) { rc = 223; goto cleanup; } + if (nvjpeg2kStreamCreate(&session->dec_stream) != NVJPEG2K_STATUS_SUCCESS) { rc = 224; goto cleanup; } + if (nvjpeg2kEncoderCreateSimple(&session->enc) != NVJPEG2K_STATUS_SUCCESS) { rc = 201; goto cleanup; } + if (nvjpeg2kEncodeStateCreate(session->enc, &session->enc_state) != NVJPEG2K_STATUS_SUCCESS) { rc = 202; goto cleanup; } + if (nvjpeg2kEncodeParamsCreate(&session->enc_params) != NVJPEG2K_STATUS_SUCCESS) { rc = 203; goto cleanup; } + + *out = session; + return 0; + +cleanup: + nvb_session_destroy(session); + return rc; +} + +int nvb_session_decode_j2k_interleaved( + NvbSession* session, + const unsigned char* j2k, size_t j2k_len, + int requested_format, + unsigned char* out, size_t out_cap, size_t* out_len, + double* decode_ms, + int* width, int* height, int* num_components, + int* bit_depth, int* bytes_per_sample) { + int rc = 0; + nvjpeg2kImageInfo_t image_info; + nvjpeg2kImageComponentInfo_t comp_info; + nvjpeg2kImage_t output; + size_t needed = 0; + size_t pitch = 0; + void* pixel_data[1] = {nullptr}; + size_t pitch_in_bytes[1] = {0}; + float decode_elapsed = 0.0f; + + if (!session) { return 900; } + if (!j2k || j2k_len == 0 || !out || !out_len) { return 920; } + +#ifdef NVB_STREAM_PARSE_USES_OUT_POINTER + if (nvjpeg2kStreamParse(session->dec_handle, j2k, j2k_len, 0, 0, &session->dec_stream) + != NVJPEG2K_STATUS_SUCCESS) { return 225; } +#else + if (nvjpeg2kStreamParse(session->dec_handle, j2k, j2k_len, 0, 0, session->dec_stream) + != NVJPEG2K_STATUS_SUCCESS) { return 225; } +#endif + + memset(&image_info, 0, sizeof(image_info)); + if (nvjpeg2kStreamGetImageInfo(session->dec_stream, &image_info) != NVJPEG2K_STATUS_SUCCESS) { + return 226; + } + if (image_info.num_components == 0 || image_info.num_components > 4) { return 227; } + memset(&comp_info, 0, sizeof(comp_info)); + if (nvjpeg2kStreamGetImageComponentInfo(session->dec_stream, &comp_info, 0) + != NVJPEG2K_STATUS_SUCCESS) { return 228; } + + *width = (int)image_info.image_width; + *height = (int)image_info.image_height; + *num_components = (int)image_info.num_components; + *bit_depth = (int)comp_info.precision; + *bytes_per_sample = (comp_info.precision <= 8) ? 1 : 2; + + if (requested_format == 1) { + if (image_info.num_components != 3) { return 229; } + *num_components = 3; + *bytes_per_sample = 1; + if (comp_info.precision > 8 || comp_info.sgn != 0) { return 230; } + } else if (requested_format == 2) { + if (image_info.num_components != 1) { return 229; } + *num_components = 1; + *bytes_per_sample = 1; + if (comp_info.precision > 8 || comp_info.sgn != 0) { return 230; } + } else { + return 231; + } + + if (nvjpeg2kDecodeParamsSetOutputFormat(session->dec_params, NVJPEG2K_FORMAT_INTERLEAVED) + != NVJPEG2K_STATUS_SUCCESS) { return 232; } + if (nvjpeg2kDecodeParamsSetRGBOutput(session->dec_params, requested_format == 1 ? 1 : 0) + != NVJPEG2K_STATUS_SUCCESS) { return 233; } + + needed = (size_t)(*width) * (size_t)(*height) * (size_t)(*num_components) * (size_t)(*bytes_per_sample); + pitch = (size_t)(*width) * (size_t)(*num_components) * (size_t)(*bytes_per_sample); + if (needed > out_cap) { return 234; } + rc = nvb_session_ensure_decode_interleaved(session, needed); + if (rc != 0) { return rc; } + + pixel_data[0] = session->decode_interleaved; + pitch_in_bytes[0] = pitch; + memset(&output, 0, sizeof(output)); + output.pixel_data = pixel_data; + output.pitch_in_bytes = pitch_in_bytes; + output.pixel_type = (*bytes_per_sample == 1) ? NVJPEG2K_UINT8 : NVJPEG2K_UINT16; + output.num_components = (uint32_t)(*num_components); + + cudaEventRecord(session->start, session->stream); + if (nvjpeg2kDecodeImage(session->dec_handle, session->dec_state, session->dec_stream, session->dec_params, &output, session->stream) + != NVJPEG2K_STATUS_SUCCESS) { rc = 235; goto drain; } + cudaEventRecord(session->stop, session->stream); + if (cudaStreamSynchronize(session->stream) != cudaSuccess) { return 906; } + if (cudaMemcpy(out, session->decode_interleaved, needed, cudaMemcpyDeviceToHost) != cudaSuccess) { + return 903; + } + cudaEventElapsedTime(&decode_elapsed, session->start, session->stop); + *decode_ms = (double)decode_elapsed; + *out_len = needed; + return 0; + +drain: + cudaStreamSynchronize(session->stream); + return rc; +} + +// Probe: returns 1 if the nvJPEG and nvJPEG2000 handles can be created. +int nvb_available(void) { + NvbSession* session = nullptr; + const int rc = nvb_session_create(&session); + nvb_session_destroy(session); + return rc == 0 ? 1 : 0; +} + +// Reference decode (untimed): JPEG -> interleaved RGB on the host, for PSNR. +// `out_rgb` must hold width*height*3 bytes. Returns 0 on success. +int nvb_decode_jpeg_rgb( + const unsigned char* jpeg, size_t jpeg_len, + unsigned char* out_rgb, size_t out_cap, int* width, int* height) { + int rc = 0; + cudaStream_t stream = nullptr; + nvjpegHandle_t handle = nullptr; + nvjpegJpegState_t state = nullptr; + unsigned char* dev = nullptr; + int comps = 0; + nvjpegChromaSubsampling_t subsampling; + int widths[NVJPEG_MAX_COMPONENT] = {0}; + int heights[NVJPEG_MAX_COMPONENT] = {0}; + int w = 0; + int h = 0; + size_t rgb_bytes = 0; + nvjpegImage_t dest; + + if (cudaStreamCreate(&stream) != cudaSuccess) { return 901; } + if (nvjpegCreateSimple(&handle) != NVJPEG_STATUS_SUCCESS) { rc = 101; goto cleanup; } + if (nvjpegJpegStateCreate(handle, &state) != NVJPEG_STATUS_SUCCESS) { rc = 102; goto cleanup; } + if (nvjpegGetImageInfo(handle, jpeg, jpeg_len, &comps, &subsampling, widths, heights) + != NVJPEG_STATUS_SUCCESS) { rc = 103; goto cleanup; } + + w = widths[0]; + h = heights[0]; + *width = w; + *height = h; + rgb_bytes = (size_t)w * (size_t)h * 3; + if (rgb_bytes > out_cap) { rc = 120; goto cleanup; } + if (cudaMalloc((void**)&dev, rgb_bytes) != cudaSuccess) { rc = 902; goto cleanup; } + + memset(&dest, 0, sizeof(dest)); + dest.channel[0] = dev; // interleaved RGB lands in channel[0] + dest.pitch[0] = (size_t)w * 3; + if (nvjpegDecode(handle, state, jpeg, jpeg_len, NVJPEG_OUTPUT_RGBI, &dest, stream) + != NVJPEG_STATUS_SUCCESS) { rc = 110; goto cleanup; } + cudaStreamSynchronize(stream); + if (cudaMemcpy(out_rgb, dev, rgb_bytes, cudaMemcpyDeviceToHost) != cudaSuccess) { + rc = 903; goto cleanup; + } + +cleanup: + if (stream) { cudaStreamSynchronize(stream); } + if (dev) { cudaFree(dev); } + if (state) { nvjpegJpegStateDestroy(state); } + if (handle) { nvjpegDestroy(handle); } + if (stream) { cudaStreamDestroy(stream); } + return rc; +} + +// Reused-session decode timing: JPEG -> interleaved RGB in device memory. +// Returns only a CUDA event duration for the decode submission; no host download. +int nvb_session_decode_jpeg_rgb_interleaved_timed( + NvbSession* session, + const unsigned char* jpeg, size_t jpeg_len, + double* decode_ms, + int* width, int* height) { + int comps = 0; + nvjpegChromaSubsampling_t subsampling; + int widths[NVJPEG_MAX_COMPONENT] = {0}; + int heights[NVJPEG_MAX_COMPONENT] = {0}; + int w = 0; + int h = 0; + size_t rgb_bytes = 0; + nvjpegImage_t dest; + float decode_elapsed = 0.0f; + + if (!session || !jpeg || jpeg_len == 0 || !decode_ms || !width || !height) { return 920; } + if (nvjpegGetImageInfo(session->jpeg_handle, jpeg, jpeg_len, &comps, &subsampling, widths, heights) + != NVJPEG_STATUS_SUCCESS) { return 103; } + + w = widths[0]; + h = heights[0]; + *width = w; + *height = h; + rgb_bytes = (size_t)w * (size_t)h * 3; + if (nvb_session_ensure_decode_interleaved(session, rgb_bytes) != 0) { return 902; } + + memset(&dest, 0, sizeof(dest)); + dest.channel[0] = session->decode_interleaved; + dest.pitch[0] = (size_t)w * 3; + + cudaEventRecord(session->start, session->stream); + if (nvjpegDecode(session->jpeg_handle, session->jpeg_state, jpeg, jpeg_len, NVJPEG_OUTPUT_RGBI, &dest, session->stream) + != NVJPEG_STATUS_SUCCESS) { return 110; } + cudaEventRecord(session->stop, session->stream); + if (cudaEventSynchronize(session->stop) != cudaSuccess) { return 906; } + if (cudaEventElapsedTime(&decode_elapsed, session->start, session->stop) != cudaSuccess) { return 907; } + *decode_ms = (double)decode_elapsed; + return 0; +} + +// Reused-session GPU transcode: JPEG bytes -> HTJ2K bytes. Returns 0 on success, +// or a non-zero stage code (1xx nvJPEG decode, 2xx nvJPEG2000 encode, 9xx CUDA). +// `decode_ms` / `encode_ms` are GPU stage times (cudaEvent). `out` must have +// `out_cap` bytes; on success `*out_len` holds the codestream length. +int nvb_session_transcode_jpeg_to_htj2k( + NvbSession* session, + const unsigned char* jpeg, size_t jpeg_len, + unsigned char* out, size_t out_cap, size_t* out_len, + double* decode_ms, double* encode_ms, + int* width, int* height, int* num_components) { + int rc = 0; + int comps = 0; + nvjpegChromaSubsampling_t subsampling; + int widths[NVJPEG_MAX_COMPONENT] = {0}; + int heights[NVJPEG_MAX_COMPONENT] = {0}; + int w = 0; + int h = 0; + size_t plane_bytes = 0; + nvjpegImage_t dest; + nvjpeg2kImageComponentInfo_t comp_info[3]; + int levels = 0; + int axis = 0; + nvjpeg2kEncodeConfig_t config; + void* plane_ptrs[3] = {nullptr, nullptr, nullptr}; + size_t pitches[3] = {0, 0, 0}; + nvjpeg2kImage_t input; + size_t length = 0; + float decode_elapsed = 0.0f; + float encode_elapsed = 0.0f; + + if (!session) { return 900; } + + if (nvjpegGetImageInfo(session->jpeg_handle, jpeg, jpeg_len, &comps, &subsampling, widths, heights) + != NVJPEG_STATUS_SUCCESS) { return 103; } + + w = widths[0]; + h = heights[0]; + *width = w; + *height = h; + *num_components = 3; + + // Planar RGB destination (one plane per channel, tightly packed). + plane_bytes = (size_t)w * (size_t)h; + rc = nvb_session_ensure_planes(session, plane_bytes); + if (rc != 0) { return rc; } + memset(&dest, 0, sizeof(dest)); + for (int c = 0; c < 3; ++c) { + dest.channel[c] = session->planes[c]; + dest.pitch[c] = (size_t)w; + } + + cudaEventRecord(session->start, session->stream); + if (nvjpegDecode(session->jpeg_handle, session->jpeg_state, jpeg, jpeg_len, NVJPEG_OUTPUT_RGB, &dest, session->stream) + != NVJPEG_STATUS_SUCCESS) { rc = 110; goto drain; } + cudaEventRecord(session->mid, session->stream); + + // --- nvJPEG2000 HTJ2K encode of the planar RGB --- + for (int c = 0; c < 3; ++c) { + comp_info[c].component_width = (uint32_t)w; + comp_info[c].component_height = (uint32_t)h; + comp_info[c].precision = 8; + comp_info[c].sgn = 0; + } + + // Resolutions: cap decomposition levels so the LL band stays >= 1 sample. + axis = (w < h) ? w : h; + while (axis > 1 && levels < 5) { axis >>= 1; ++levels; } + + memset(&config, 0, sizeof(config)); + config.stream_type = NVJPEG2K_STREAM_J2K; + config.color_space = NVJPEG2K_COLORSPACE_SRGB; + config.rsiz = NVJPEG2K_RSIZ_HT; + config.image_width = (uint32_t)w; + config.image_height = (uint32_t)h; + config.enable_tiling = 0; + config.num_components = 3; + config.image_comp_info = comp_info; + config.prog_order = NVJPEG2K_LRCP; + config.num_layers = 1; + config.mct_mode = 1; // RGB input: apply the multi-component (color) transform. + config.num_resolutions = (uint32_t)(levels + 1); + config.code_block_w = 64; + config.code_block_h = 64; + config.encode_modes = NVJPEG2K_MODE_HT; + config.irreversible = 1; // 9/7 irreversible path. + + if (nvjpeg2kEncodeParamsSetEncodeConfig(session->enc_params, &config) != NVJPEG2K_STATUS_SUCCESS) { + rc = 204; goto drain; + } + + plane_ptrs[0] = session->planes[0]; + plane_ptrs[1] = session->planes[1]; + plane_ptrs[2] = session->planes[2]; + pitches[0] = (size_t)w; + pitches[1] = (size_t)w; + pitches[2] = (size_t)w; + memset(&input, 0, sizeof(input)); + input.pixel_data = plane_ptrs; + input.pitch_in_bytes = pitches; + input.pixel_type = NVJPEG2K_UINT8; + input.num_components = 3; + + if (nvjpeg2kEncode(session->enc, session->enc_state, session->enc_params, &input, session->stream) != NVJPEG2K_STATUS_SUCCESS) { + rc = 210; goto drain; + } + + if (nvjpeg2kEncodeRetrieveBitstream(session->enc, session->enc_state, nullptr, &length, session->stream) + != NVJPEG2K_STATUS_SUCCESS) { rc = 211; goto drain; } + if (length > out_cap) { rc = 212; goto drain; } + if (nvjpeg2kEncodeRetrieveBitstream(session->enc, session->enc_state, out, &length, session->stream) + != NVJPEG2K_STATUS_SUCCESS) { rc = 213; goto drain; } + cudaEventRecord(session->stop, session->stream); + if (cudaStreamSynchronize(session->stream) != cudaSuccess) { return 906; } + *out_len = length; + + cudaEventElapsedTime(&decode_elapsed, session->start, session->mid); + cudaEventElapsedTime(&encode_elapsed, session->mid, session->stop); + *decode_ms = (double)decode_elapsed; + *encode_ms = (double)encode_elapsed; + return 0; + +drain: + cudaStreamSynchronize(session->stream); + return rc; +} + +// Compatibility one-shot wrapper. +int nvb_transcode_jpeg_to_htj2k( + const unsigned char* jpeg, size_t jpeg_len, + unsigned char* out, size_t out_cap, size_t* out_len, + double* decode_ms, double* encode_ms, + int* width, int* height, int* num_components) { + int rc = 0; + NvbSession* session = nullptr; + rc = nvb_session_create(&session); + if (rc != 0) { return rc; } + rc = nvb_session_transcode_jpeg_to_htj2k( + session, + jpeg, + jpeg_len, + out, + out_cap, + out_len, + decode_ms, + encode_ms, + width, + height, + num_components + ); + nvb_session_destroy(session); + return rc; +} + +} // extern "C" diff --git a/tests/nvidia-baseline/scripts/assert_transcode_perf.py b/tests/nvidia-baseline/scripts/assert_transcode_perf.py new file mode 100755 index 00000000..5395b6b4 --- /dev/null +++ b/tests/nvidia-baseline/scripts/assert_transcode_perf.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 + +import argparse +import json +import math +import sys +from pathlib import Path + + +def positive_float(value, name): + if not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0: + raise ValueError(f"{name} must be a positive finite number") + return float(value) + + +def load_report(path): + with Path(path).open("r", encoding="utf-8") as handle: + report = json.load(handle) + if not isinstance(report, dict): + raise ValueError("report root must be a JSON object") + return report + + +def result_mps(report, result, name): + megapixels = positive_float(report.get("megapixels"), "megapixels") + wall_ms = positive_float(result.get("wall_ms"), f"{name} wall_ms") + return megapixels / (wall_ms / 1000.0) + + +def require_result(report, key, label): + result = report.get(key) + if not isinstance(result, dict): + raise ValueError(f"missing {label} result `{key}`") + if result.get("ran") is not True: + raise ValueError(f"{label} did not run") + return result + + +def assert_gate(args): + report = load_report(args.json) + failures = [] + + try: + cuda_ht = require_result( + report, "j2k_cuda_ht_experimental", "j2k CUDA HT" + ) + except ValueError as error: + failures.append(str(error)) + cuda_ht = None + + nvidia = None + if args.min_cuda_ht_speedup_vs_nvidia is not None: + try: + nvidia = require_result( + report, "nvidia_reused_session_serial", "NVIDIA reused-session serial" + ) + if nvidia.get("status") != "ok": + failures.append(f"NVIDIA status is {nvidia.get('status')!r}, expected 'ok'") + except ValueError as error: + failures.append(str(error)) + + cuda_mps = None + if cuda_ht is not None: + if cuda_ht.get("used_gpu") is not True: + failures.append("j2k CUDA HT did not report used_gpu=true") + try: + cuda_mps = result_mps(report, cuda_ht, "j2k CUDA HT") + except ValueError as error: + failures.append(str(error)) + + if args.min_cuda_ht_mps is not None and cuda_mps is not None: + if cuda_mps < args.min_cuda_ht_mps: + failures.append( + "j2k CUDA HT MP/s " + f"{cuda_mps:.3f} below threshold {args.min_cuda_ht_mps:.3f}" + ) + + if args.max_byte_delta_abs is not None: + byte_delta = cuda_ht.get("byte_delta_vs_nvidia") + if not isinstance(byte_delta, (int, float)) or not math.isfinite(byte_delta): + failures.append("j2k CUDA HT byte_delta_vs_nvidia is missing or non-finite") + elif abs(float(byte_delta)) > args.max_byte_delta_abs: + failures.append( + "j2k CUDA HT byte delta " + f"{float(byte_delta):.6f} outside +/-{args.max_byte_delta_abs:.6f}" + ) + + dispatches = cuda_ht.get("ht_codeblock_dispatches") + if not isinstance(dispatches, int): + failures.append("j2k CUDA HT code-block dispatch count is missing") + elif dispatches < args.min_ht_codeblock_dispatches: + failures.append( + "j2k CUDA HT code-block dispatches " + f"{dispatches} below threshold {args.min_ht_codeblock_dispatches}" + ) + + if ( + args.min_cuda_ht_speedup_vs_nvidia is not None + and cuda_mps is not None + and nvidia is not None + ): + try: + nvidia_mps = result_mps(report, nvidia, "NVIDIA") + except ValueError as error: + failures.append(str(error)) + else: + speedup = cuda_mps / nvidia_mps + if speedup < args.min_cuda_ht_speedup_vs_nvidia: + failures.append( + "j2k CUDA HT speedup vs NVIDIA " + f"{speedup:.3f} below threshold " + f"{args.min_cuda_ht_speedup_vs_nvidia:.3f}" + ) + + if failures: + for failure in failures: + print(f"{args.label}: FAIL: {failure}", file=sys.stderr) + return 1 + + parts = [f"{args.label}: PASS"] + if cuda_mps is not None: + parts.append(f"j2k_cuda_ht_mps={cuda_mps:.3f}") + if nvidia is not None: + parts.append(f"nvidia_mps={result_mps(report, nvidia, 'NVIDIA'):.3f}") + print(" ".join(parts)) + return 0 + + +def build_parser(): + parser = argparse.ArgumentParser( + description="Fail-closed perf gate for transcode_compare JSON reports." + ) + parser.add_argument("--json", required=True, help="transcode_compare JSON report") + parser.add_argument("--label", required=True, help="gate label printed in diagnostics") + parser.add_argument("--min-cuda-ht-mps", type=float) + parser.add_argument("--min-cuda-ht-speedup-vs-nvidia", type=float) + parser.add_argument("--max-byte-delta-abs", type=float) + parser.add_argument("--min-ht-codeblock-dispatches", type=int, default=1) + return parser + + +def main(argv=None): + parser = build_parser() + args = parser.parse_args(argv) + if args.min_ht_codeblock_dispatches < 0: + parser.error("--min-ht-codeblock-dispatches must be >= 0") + for option in [ + args.min_cuda_ht_mps, + args.min_cuda_ht_speedup_vs_nvidia, + args.max_byte_delta_abs, + ]: + if option is not None and ( + not math.isfinite(option) or option < 0 + ): + parser.error("numeric thresholds must be finite and >= 0") + return assert_gate(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/nvidia-baseline/scripts/test_assert_transcode_perf.py b/tests/nvidia-baseline/scripts/test_assert_transcode_perf.py new file mode 100755 index 00000000..69105e1a --- /dev/null +++ b/tests/nvidia-baseline/scripts/test_assert_transcode_perf.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).with_name("assert_transcode_perf.py") + + +def write_report(report): + temp = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) + with temp: + json.dump(report, temp) + return Path(temp.name) + + +def base_report(cuda_wall_ms=16.196893, cuda_dispatches=1, nvidia_wall_ms=144.689665): + return { + "tile_count": 109, + "megapixels": 7.143424, + "match_nvidia_bytes": True, + "match_tolerance": 0.2, + "rd_points": [ + { + "scale": 1.9, + "ran": True, + "used_gpu": True, + "bytes": 6199631, + "byte_delta_vs_nvidia": -0.00536860, + "wall_ms": 67.994635, + "encode_dispatches": 0, + "ht_codeblock_dispatches": 0, + } + ], + "j2k_cuda_ht_experimental": { + "scale": 1.9, + "ran": True, + "used_gpu": True, + "bytes": 6199631, + "byte_delta_vs_nvidia": -0.00536860, + "wall_ms": cuda_wall_ms, + "encode_dispatches": cuda_dispatches, + "ht_codeblock_dispatches": cuda_dispatches, + }, + "nvidia_reused_session_serial": { + "ran": True, + "status": "ok", + "bytes": 6233094, + "wall_ms": nvidia_wall_ms, + }, + } + + +class AssertTranscodePerfTest(unittest.TestCase): + def run_gate(self, report, *extra_args): + report_path = write_report(report) + self.addCleanup(lambda: report_path.unlink(missing_ok=True)) + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--json", + str(report_path), + "--label", + "unit", + *extra_args, + ], + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + def test_passes_when_cuda_ht_beats_absolute_and_relative_thresholds(self): + result = self.run_gate( + base_report(), + "--min-cuda-ht-mps", + "400", + "--min-cuda-ht-speedup-vs-nvidia", + "5.0", + "--max-byte-delta-abs", + "0.02", + "--min-ht-codeblock-dispatches", + "1", + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("unit: PASS", result.stdout) + + def test_fails_when_cuda_ht_is_below_absolute_threshold(self): + result = self.run_gate( + base_report(cuda_wall_ms=200.0), + "--min-cuda-ht-mps", + "400", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("j2k CUDA HT MP/s", result.stderr) + + def test_fails_when_cuda_ht_did_not_dispatch(self): + result = self.run_gate( + base_report(cuda_dispatches=0), + "--min-ht-codeblock-dispatches", + "1", + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("HT code-block dispatches", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/nvidia-baseline/src/bin/decode_compare.rs b/tests/nvidia-baseline/src/bin/decode_compare.rs new file mode 100644 index 00000000..4e1003d1 --- /dev/null +++ b/tests/nvidia-baseline/src/bin/decode_compare.rs @@ -0,0 +1,2190 @@ +// SPDX-License-Identifier: Apache-2.0 + +#![allow( + clippy::cast_possible_truncation, + clippy::cast_precision_loss, + clippy::cast_sign_loss, + clippy::format_push_string, + clippy::missing_errors_doc, + clippy::missing_panics_doc, + clippy::print_stdout, + clippy::similar_names, + clippy::too_many_lines +)] + +mod report_format; + +use std::{ + fs, + path::{Path, PathBuf}, + time::Instant, +}; + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +use j2k_core::{BackendKind, DeviceSurface, PixelFormat}; +use j2k_native::{encode_htj2k, DecodeSettings, EncodeOptions, Image}; +use j2k_nvidia_baseline::{ + nvidia_j2k_decode_available, psnr_u8, write_text_artifact, NvBaselineError, NvBaselineSession, + NvJ2kDecodeFormat, +}; +use report_format::{csv_f64_or_inf, escape_csv, escape_json, json_f64_or_inf}; + +const DEFAULT_FIXTURE_DIM: u32 = 512; +const DEFAULT_WARMUP: usize = 2; +const DEFAULT_ITERATIONS: usize = 10; + +fn main() { + let config = match Config::from_env_args() { + Ok(config) => config, + Err(error) => { + eprintln!("{error}"); + std::process::exit(2); + } + }; + + let inputs = match load_inputs(&config) { + Ok(inputs) if !inputs.is_empty() => inputs, + Ok(_) => { + eprintln!("no JPEG 2000 inputs found"); + std::process::exit(2); + } + Err(error) => { + eprintln!("failed to load inputs: {error}"); + std::process::exit(2); + } + }; + if inputs.len() < config.min_inputs { + eprintln!( + "input corpus has {} codestream(s), below required --min-inputs {}", + inputs.len(), + config.min_inputs + ); + std::process::exit(2); + } + + let require_nvidia = std::env::var_os("J2K_REQUIRE_NV_BASELINE_BUILD").is_some(); + if require_nvidia && !nvidia_j2k_decode_available() { + eprintln!("J2K_REQUIRE_NV_BASELINE_BUILD set, but nvJPEG2000 decode is unavailable"); + std::process::exit(1); + } + + let report = run_comparison(&inputs, &config); + print_report(&report, &config); + if let Err(error) = write_artifacts(&report, &config) { + eprintln!("failed to write decode comparison artifacts: {error}"); + std::process::exit(2); + } + + if should_exit_for_failed_required_report(&report, &config, require_nvidia) { + eprintln!("required decode comparison/profile failed"); + std::process::exit(1); + } +} + +fn should_exit_for_failed_required_report( + report: &DecodeReport, + config: &Config, + require_nvidia: bool, +) -> bool { + if config.is_j2k_cuda_profile() { + return report + .profile + .as_ref() + .is_some_and(|profile| profile.status.has_failure()); + } + if config.compare_j2k_cuda_batch + && report + .profile + .as_ref() + .is_some_and(|profile| profile.status.has_failure()) + { + return true; + } + require_nvidia && report.rows.iter().any(Row::has_required_failure) +} + +#[derive(Debug, Clone)] +#[allow(clippy::struct_excessive_bools)] +struct Config { + inputs: Vec, + jpeg_dir: Option, + json: Option, + csv: Option, + fixture_dim: u32, + warmup: usize, + iterations: usize, + min_inputs: usize, + max_inputs: Option, + profile_j2k_cuda_only: bool, + profile_j2k_cuda_batch: bool, + compare_j2k_cuda_batch: bool, + collect_j2k_stage_timings: bool, + skip_j2k_download: bool, +} + +impl Config { + fn from_env_args() -> Result { + Self::from_args(std::env::args().skip(1)) + } + + fn from_args(args: I) -> Result + where + I: IntoIterator, + S: Into, + { + let mut config = Self { + inputs: Vec::new(), + jpeg_dir: None, + json: None, + csv: None, + fixture_dim: DEFAULT_FIXTURE_DIM, + warmup: DEFAULT_WARMUP, + iterations: DEFAULT_ITERATIONS, + min_inputs: 1, + max_inputs: None, + profile_j2k_cuda_only: false, + profile_j2k_cuda_batch: false, + compare_j2k_cuda_batch: false, + collect_j2k_stage_timings: false, + skip_j2k_download: false, + }; + let mut iter = args.into_iter().map(Into::into); + while let Some(arg) = iter.next() { + match arg.as_str() { + "--profile-j2k-cuda-only" => config.profile_j2k_cuda_only = true, + "--profile-j2k-cuda-batch" => config.profile_j2k_cuda_batch = true, + "--compare-j2k-cuda-batch" => config.compare_j2k_cuda_batch = true, + "--collect-j2k-stage-timings" => { + config.collect_j2k_stage_timings = true; + } + "--skip-j2k-download" => config.skip_j2k_download = true, + "--json" => config.json = Some(next_path(&mut iter, "--json")?), + "--csv" => config.csv = Some(next_path(&mut iter, "--csv")?), + "--jpeg-dir" => config.jpeg_dir = Some(next_path(&mut iter, "--jpeg-dir")?), + "--fixture-dim" => { + config.fixture_dim = next_parse(&mut iter, "--fixture-dim")?; + if config.fixture_dim == 0 { + return Err("--fixture-dim must be > 0".to_string()); + } + } + "--warmup" => config.warmup = next_parse(&mut iter, "--warmup")?, + "--iterations" => { + config.iterations = next_parse(&mut iter, "--iterations")?; + if config.iterations == 0 { + return Err("--iterations must be > 0".to_string()); + } + } + "--min-inputs" => config.min_inputs = next_parse(&mut iter, "--min-inputs")?, + "--max-inputs" => { + let max_inputs = next_parse(&mut iter, "--max-inputs")?; + if max_inputs == 0 { + return Err("--max-inputs must be > 0".to_string()); + } + config.max_inputs = Some(max_inputs); + } + "-h" | "--help" => return Err(usage()), + other if other.starts_with('-') => { + return Err(format!("unknown flag `{other}`\n{}", usage())) + } + path => config.inputs.push(PathBuf::from(path)), + } + } + if config.min_inputs == 0 { + return Err("--min-inputs must be > 0".to_string()); + } + if config.skip_j2k_download && !config.is_j2k_cuda_profile() { + return Err( + "--skip-j2k-download requires --profile-j2k-cuda-only or --profile-j2k-cuda-batch" + .to_string(), + ); + } + if config.profile_j2k_cuda_only && config.profile_j2k_cuda_batch { + return Err( + "--profile-j2k-cuda-only conflicts with --profile-j2k-cuda-batch".to_string(), + ); + } + if config.compare_j2k_cuda_batch && config.is_j2k_cuda_profile() { + return Err("--compare-j2k-cuda-batch conflicts with profile-only modes".to_string()); + } + Ok(config) + } + + fn is_j2k_cuda_profile(&self) -> bool { + self.profile_j2k_cuda_only || self.profile_j2k_cuda_batch + } +} + +fn next_path(iter: &mut impl Iterator, flag: &str) -> Result { + iter.next() + .map(PathBuf::from) + .ok_or_else(|| format!("{flag} requires a path")) +} + +fn next_parse(iter: &mut impl Iterator, flag: &str) -> Result +where + T: std::str::FromStr, +{ + let value = iter + .next() + .ok_or_else(|| format!("{flag} requires a value"))?; + value + .parse() + .map_err(|_| format!("{flag} has invalid value `{value}`")) +} + +fn usage() -> String { + "usage: decode_compare [--profile-j2k-cuda-only] [--profile-j2k-cuda-batch] [--compare-j2k-cuda-batch] [--collect-j2k-stage-timings] [--skip-j2k-download] [--fixture-dim n] [--jpeg-dir path] [--warmup n] [--iterations n] [--min-inputs n] [--max-inputs n] [--json path] [--csv path] [file.j2k ...]".to_string() +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DecodeCaseFormat { + Gray8, + Rgb8, +} + +impl DecodeCaseFormat { + const fn components(self) -> usize { + match self { + Self::Gray8 => 1, + Self::Rgb8 => 3, + } + } + + const fn label(self) -> &'static str { + match self { + Self::Gray8 => "gray8", + Self::Rgb8 => "rgb8", + } + } + + const fn nvidia(self) -> NvJ2kDecodeFormat { + match self { + Self::Gray8 => NvJ2kDecodeFormat::Gray8, + Self::Rgb8 => NvJ2kDecodeFormat::Rgb8, + } + } + + #[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] + const fn j2k(self) -> PixelFormat { + match self { + Self::Gray8 => PixelFormat::Gray8, + Self::Rgb8 => PixelFormat::Rgb8, + } + } +} + +#[derive(Debug, Clone)] +struct DecodeInput { + label: String, + bytes: Vec, + width: u32, + height: u32, + format: DecodeCaseFormat, +} + +fn load_inputs(config: &Config) -> Result, String> { + if let Some(jpeg_dir) = &config.jpeg_dir { + return load_nvidia_htj2k_from_jpeg_dir(jpeg_dir, config.max_inputs); + } + if config.inputs.is_empty() { + return generated_inputs(config.fixture_dim); + } + let paths = config + .inputs + .iter() + .take(config.max_inputs.unwrap_or(usize::MAX)); + paths.map(|path| load_input_path(path)).collect() +} + +fn load_nvidia_htj2k_from_jpeg_dir( + jpeg_dir: &Path, + max_inputs: Option, +) -> Result, String> { + let mut jpeg_paths = fs::read_dir(jpeg_dir) + .map_err(|error| format!("{}: {error}", jpeg_dir.display()))? + .filter_map(|entry| { + let path = entry.ok()?.path(); + let is_jpeg = path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| { + ext.eq_ignore_ascii_case("jpg") || ext.eq_ignore_ascii_case("jpeg") + }); + is_jpeg.then_some(path) + }) + .collect::>(); + jpeg_paths.sort(); + if let Some(max_inputs) = max_inputs { + jpeg_paths.truncate(max_inputs); + } + if jpeg_paths.is_empty() { + return Err(format!("{}: no JPEG inputs found", jpeg_dir.display())); + } + + let mut session = NvBaselineSession::new() + .map_err(|error| format!("NVIDIA baseline session required for --jpeg-dir: {error:?}"))?; + let mut inputs = Vec::with_capacity(jpeg_paths.len()); + for path in jpeg_paths { + let jpeg = fs::read(&path).map_err(|error| format!("{}: {error}", path.display()))?; + let encoded = session + .transcode_jpeg_to_htj2k(&jpeg) + .map_err(|error| format!("{} NVIDIA HTJ2K transcode: {error:?}", path.display()))?; + if encoded.num_components != 3 { + return Err(format!( + "{} NVIDIA HTJ2K transcode returned {} component(s), expected RGB", + path.display(), + encoded.num_components + )); + } + let label = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("input.jpg"); + inputs.push(DecodeInput { + label: format!("nvidia_htj2k:{label}"), + bytes: encoded.codestream, + width: encoded.width, + height: encoded.height, + format: DecodeCaseFormat::Rgb8, + }); + } + Ok(inputs) +} + +fn generated_inputs(dim: u32) -> Result, String> { + let gray = (0..dim * dim) + .map(|idx| u8::try_from((idx * 17 + idx / 3) & 0xff).expect("masked sample fits")) + .collect::>(); + let mut rgb = Vec::with_capacity(dim as usize * dim as usize * 3); + for idx in 0..dim * dim { + rgb.push(u8::try_from((idx * 17 + idx / 3) & 0xff).expect("masked red fits")); + rgb.push(u8::try_from((idx * 29 + 7) & 0xff).expect("masked green fits")); + rgb.push(u8::try_from((idx * 43 + 19) & 0xff).expect("masked blue fits")); + } + Ok(vec![ + encode_input("generated_gray8", dim, dim, DecodeCaseFormat::Gray8, &gray)?, + encode_input("generated_rgb8", dim, dim, DecodeCaseFormat::Rgb8, &rgb)?, + ]) +} + +fn encode_input( + label: &str, + width: u32, + height: u32, + format: DecodeCaseFormat, + pixels: &[u8], +) -> Result { + let options = EncodeOptions { + reversible: true, + use_ht_block_coding: true, + num_decomposition_levels: 1, + ..EncodeOptions::default() + }; + let bytes = encode_htj2k( + pixels, + width, + height, + u8::try_from(format.components()).expect("component count fits"), + 8, + false, + &options, + ) + .map_err(|error| format!("encode {label}: {error}"))?; + Ok(DecodeInput { + label: label.to_string(), + bytes, + width, + height, + format, + }) +} + +fn load_input_path(path: &Path) -> Result { + let bytes = fs::read(path).map_err(|error| format!("{}: {error}", path.display()))?; + let image = Image::new(&bytes, &DecodeSettings::default()) + .map_err(|error| format!("{}: {error}", path.display()))?; + let bitmap = image + .decode_native() + .map_err(|error| format!("{} decode: {error}", path.display()))?; + let format = match (bitmap.num_components, bitmap.bytes_per_sample) { + (1, 1) => DecodeCaseFormat::Gray8, + (3, 1) => DecodeCaseFormat::Rgb8, + _ => { + return Err(format!( + "{}: only 8-bit gray/RGB direct decode comparison is supported, got {} components and {} byte(s)/sample", + path.display(), + bitmap.num_components, + bitmap.bytes_per_sample + )); + } + }; + Ok(DecodeInput { + label: path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("input") + .to_string(), + bytes, + width: bitmap.width, + height: bitmap.height, + format, + }) +} + +#[derive(Debug, Clone)] +struct TimedPixels { + wall_ms: f64, + gpu_ms: Option, + stage_ms: Option, + download_ms: Option, + cuda_profile: Option, + pixels: Vec, +} + +#[derive(Debug, Clone, Default)] +struct CudaStageBreakdown { + parse_us: u128, + plan_us: u128, + flatten_us: u128, + h2d_us: u128, + ht_cleanup_us: u128, + ht_refine_us: u128, + dequant_us: u128, + idwt_us: u128, + mct_us: u128, + store_us: u128, + total_us: u128, + wall_total_us: u128, + table_upload_us: u128, + payload_upload_us: u128, + status_d2h_us: u128, + output_d2h_us: u128, + block_count: usize, + payload_bytes: usize, + dispatch_count: usize, + ht_dispatch_count: usize, + dequant_dispatch_count: usize, + idwt_dispatch_count: usize, + mct_dispatch_count: usize, + store_dispatch_count: usize, +} + +impl CudaStageBreakdown { + #[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] + fn from_report(report: &j2k_cuda::CudaHtj2kProfileReport) -> Self { + Self { + parse_us: report.parse_us, + plan_us: report.plan_us, + flatten_us: report.flatten_us, + h2d_us: report.h2d_us, + ht_cleanup_us: report.ht_cleanup_us, + ht_refine_us: report.ht_refine_us, + dequant_us: report.dequant_us, + idwt_us: report.idwt_us, + mct_us: report.mct_us, + store_us: report.store_us, + total_us: report.total_us, + wall_total_us: report.detail.wall_total_us, + table_upload_us: report.detail.table_upload_us, + payload_upload_us: report.detail.payload_upload_us, + status_d2h_us: report.detail.status_d2h_us, + output_d2h_us: report.detail.output_d2h_us, + block_count: report.block_count, + payload_bytes: report.payload_bytes, + dispatch_count: report.dispatch_count, + ht_dispatch_count: report.detail.ht_dispatch_count, + dequant_dispatch_count: report.detail.dequant_dispatch_count, + idwt_dispatch_count: report.detail.idwt_dispatch_count, + mct_dispatch_count: report.detail.mct_dispatch_count, + store_dispatch_count: report.detail.store_dispatch_count, + } + } + + fn add_assign(&mut self, other: &Self) { + self.parse_us = self.parse_us.saturating_add(other.parse_us); + self.plan_us = self.plan_us.saturating_add(other.plan_us); + self.flatten_us = self.flatten_us.saturating_add(other.flatten_us); + self.h2d_us = self.h2d_us.saturating_add(other.h2d_us); + self.ht_cleanup_us = self.ht_cleanup_us.saturating_add(other.ht_cleanup_us); + self.ht_refine_us = self.ht_refine_us.saturating_add(other.ht_refine_us); + self.dequant_us = self.dequant_us.saturating_add(other.dequant_us); + self.idwt_us = self.idwt_us.saturating_add(other.idwt_us); + self.mct_us = self.mct_us.saturating_add(other.mct_us); + self.store_us = self.store_us.saturating_add(other.store_us); + self.total_us = self.total_us.saturating_add(other.total_us); + self.wall_total_us = self.wall_total_us.saturating_add(other.wall_total_us); + self.table_upload_us = self.table_upload_us.saturating_add(other.table_upload_us); + self.payload_upload_us = self + .payload_upload_us + .saturating_add(other.payload_upload_us); + self.status_d2h_us = self.status_d2h_us.saturating_add(other.status_d2h_us); + self.output_d2h_us = self.output_d2h_us.saturating_add(other.output_d2h_us); + self.block_count = self.block_count.saturating_add(other.block_count); + self.payload_bytes = self.payload_bytes.saturating_add(other.payload_bytes); + self.dispatch_count = self.dispatch_count.saturating_add(other.dispatch_count); + self.ht_dispatch_count = self + .ht_dispatch_count + .saturating_add(other.ht_dispatch_count); + self.dequant_dispatch_count = self + .dequant_dispatch_count + .saturating_add(other.dequant_dispatch_count); + self.idwt_dispatch_count = self + .idwt_dispatch_count + .saturating_add(other.idwt_dispatch_count); + self.mct_dispatch_count = self + .mct_dispatch_count + .saturating_add(other.mct_dispatch_count); + self.store_dispatch_count = self + .store_dispatch_count + .saturating_add(other.store_dispatch_count); + } +} + +#[derive(Debug, Clone)] +struct TimedStatus { + status: String, + result: Option, +} + +impl TimedStatus { + fn ok(result: TimedPixels) -> Self { + Self { + status: "ok".to_string(), + result: Some(result), + } + } + + fn ok_without_result() -> Self { + Self { + status: "ok".to_string(), + result: None, + } + } + + fn failed(status: impl Into) -> Self { + Self { + status: status.into(), + result: None, + } + } + + fn has_failure(&self) -> bool { + self.status != "ok" + } +} + +#[derive(Debug, Clone)] +struct Row { + label: String, + format: DecodeCaseFormat, + width: u32, + height: u32, + codestream_bytes: usize, + cpu: TimedStatus, + j2k_cuda: TimedStatus, + nvidia: TimedStatus, + j2k_cuda_psnr_vs_cpu: Option, + nvidia_psnr_vs_cpu: Option, +} + +impl Row { + fn megapixels(&self) -> f64 { + f64::from(self.width) * f64::from(self.height) / 1.0e6 + } + + fn has_required_failure(&self) -> bool { + self.cpu.has_failure() || self.j2k_cuda.has_failure() || self.nvidia.has_failure() + } +} + +#[derive(Debug, Clone)] +struct DecodeReport { + rows: Vec, + profile: Option, +} + +impl DecodeReport { + fn rows(rows: Vec) -> Self { + Self { + rows, + profile: None, + } + } + + fn profile(rows: Vec, profile: ProfileMeasurement) -> Self { + Self { + rows, + profile: Some(profile), + } + } + + fn input_count(&self) -> usize { + self.profile + .as_ref() + .map_or(self.rows.len(), |profile| profile.input_count) + } + + fn megapixels(&self) -> f64 { + self.profile.as_ref().map_or_else( + || self.rows.iter().map(Row::megapixels).sum(), + |profile| profile.megapixels, + ) + } +} + +#[derive(Debug, Clone)] +struct ProfileMeasurement { + label: &'static str, + execution_mode: &'static str, + timing_scope: &'static str, + download_policy: &'static str, + input_count: usize, + megapixels: f64, + codestream_bytes: usize, + status: TimedStatus, +} + +impl ProfileMeasurement { + fn mp_s(&self) -> Option { + let result = self.status.result.as_ref()?; + Some(if result.wall_ms > 0.0 { + self.megapixels / (result.wall_ms / 1_000.0) + } else { + f64::INFINITY + }) + } +} + +fn run_comparison(inputs: &[DecodeInput], config: &Config) -> DecodeReport { + if config.profile_j2k_cuda_batch { + return DecodeReport::profile(Vec::new(), run_j2k_cuda_batch(inputs, config)); + } + if config.profile_j2k_cuda_only { + let rows = run_j2k_cuda_only(inputs, config); + let profile = serial_profile_measurement(&rows, inputs, config); + return DecodeReport::profile(rows, profile); + } + + let mut rows = Vec::with_capacity(inputs.len()); + let cpu_results = inputs + .iter() + .map(|input| timed_best(config, || decode_cpu(input))) + .collect::>(); + + // Keep the nvJPEG2000 decode session isolated from J2K's CUDA Driver + // context. Some CUDA runtime/nvJPEG2000 stacks fail decode after another + // Driver API context has been made current in the same process. + let mut nvidia_session = NvBaselineSession::new().ok(); + let nvidia_results = inputs + .iter() + .map(|input| match nvidia_session.as_mut() { + Some(session) => timed_best(config, || decode_nvidia(session, input)), + None => TimedStatus::failed(nvidia_unavailable_status()), + }) + .collect::>(); + + if config.compare_j2k_cuda_batch { + return run_batch_correctness_comparison(inputs, config, cpu_results, nvidia_results); + } + + #[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] + let mut j2k_cuda_session = j2k_cuda::CudaSession::default(); + for ((input, cpu), nvidia) in inputs.iter().zip(cpu_results).zip(nvidia_results) { + #[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] + let j2k_cuda = timed_best(config, || { + decode_j2k_cuda(&mut j2k_cuda_session, input, true, false) + }); + #[cfg(any(target_os = "macos", not(feature = "nvjpeg2000")))] + let j2k_cuda = timed_best(config, || decode_j2k_cuda(input)); + let cpu_pixels = cpu.result.as_ref().map(|result| result.pixels.as_slice()); + let j2k_cuda_psnr_vs_cpu = cpu_pixels + .zip( + j2k_cuda + .result + .as_ref() + .map(|result| result.pixels.as_slice()), + ) + .and_then(|(cpu, cuda)| psnr_u8(cpu, cuda)); + let nvidia_psnr_vs_cpu = cpu_pixels + .zip( + nvidia + .result + .as_ref() + .map(|result| result.pixels.as_slice()), + ) + .and_then(|(cpu, nv)| psnr_u8(cpu, nv)); + rows.push(Row { + label: input.label.clone(), + format: input.format, + width: input.width, + height: input.height, + codestream_bytes: input.bytes.len(), + cpu, + j2k_cuda, + nvidia, + j2k_cuda_psnr_vs_cpu, + nvidia_psnr_vs_cpu, + }); + } + DecodeReport::rows(rows) +} + +fn run_batch_correctness_comparison( + inputs: &[DecodeInput], + config: &Config, + cpu_results: Vec, + nvidia_results: Vec, +) -> DecodeReport { + let profile = run_j2k_cuda_batch_correctness(inputs, config); + let rows = batch_correctness_rows(inputs, cpu_results, nvidia_results, &profile.status); + DecodeReport::profile(rows, profile) +} + +fn run_j2k_cuda_batch(inputs: &[DecodeInput], config: &Config) -> ProfileMeasurement { + #[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] + let j2k_cuda = { + let mut j2k_cuda_session = j2k_cuda::CudaSession::default(); + timed_best(config, || { + decode_j2k_cuda_batch( + &mut j2k_cuda_session, + inputs, + config.collect_j2k_stage_timings, + config.skip_j2k_download, + ) + }) + }; + #[cfg(any(target_os = "macos", not(feature = "nvjpeg2000")))] + let j2k_cuda = timed_best(config, || decode_j2k_cuda_batch(inputs)); + ProfileMeasurement { + label: "j2k_cuda_decode_real_batch", + execution_mode: "j2k_cuda_batch", + timing_scope: "aggregate_batch", + download_policy: j2k_download_policy(config), + input_count: inputs.len(), + megapixels: input_megapixels(inputs), + codestream_bytes: input_codestream_bytes(inputs), + status: j2k_cuda, + } +} + +fn run_j2k_cuda_batch_correctness(inputs: &[DecodeInput], config: &Config) -> ProfileMeasurement { + #[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] + let j2k_cuda = { + let mut j2k_cuda_session = j2k_cuda::CudaSession::default(); + timed_best(config, || { + decode_j2k_cuda_batch( + &mut j2k_cuda_session, + inputs, + config.collect_j2k_stage_timings, + false, + ) + }) + }; + #[cfg(any(target_os = "macos", not(feature = "nvjpeg2000")))] + let j2k_cuda = timed_best(config, || decode_j2k_cuda_batch(inputs)); + ProfileMeasurement { + label: "j2k_cuda_decode_batch_correctness", + execution_mode: "j2k_cuda_batch", + timing_scope: "aggregate_batch_correctness", + download_policy: "download", + input_count: inputs.len(), + megapixels: input_megapixels(inputs), + codestream_bytes: input_codestream_bytes(inputs), + status: j2k_cuda, + } +} + +fn batch_correctness_rows( + inputs: &[DecodeInput], + cpu_results: Vec, + nvidia_results: Vec, + j2k_batch: &TimedStatus, +) -> Vec { + let batch_split_error = j2k_batch + .result + .as_ref() + .and_then(|result| validate_batch_pixel_len(inputs, result.pixels.len()).err()); + let mut batch_offset = 0usize; + + inputs + .iter() + .zip(cpu_results) + .zip(nvidia_results) + .map(|((input, cpu), nvidia)| { + let j2k_pixels = j2k_batch.result.as_ref().and_then(|result| { + if batch_split_error.is_some() { + return None; + } + let len = input_pixel_bytes(input).ok()?; + let start = batch_offset; + let end = start.checked_add(len)?; + batch_offset = end; + result.pixels.get(start..end) + }); + let j2k_cuda = if j2k_batch.has_failure() { + TimedStatus::failed(j2k_batch.status.clone()) + } else if let Some(error) = &batch_split_error { + TimedStatus::failed(format!("batch-output-split:{error}")) + } else { + TimedStatus::ok_without_result() + }; + let cpu_pixels = cpu.result.as_ref().map(|result| result.pixels.as_slice()); + let j2k_cuda_psnr_vs_cpu = cpu_pixels + .zip(j2k_pixels) + .and_then(|(cpu, cuda)| psnr_u8(cpu, cuda)); + let nvidia_psnr_vs_cpu = cpu_pixels + .zip( + nvidia + .result + .as_ref() + .map(|result| result.pixels.as_slice()), + ) + .and_then(|(cpu, nv)| psnr_u8(cpu, nv)); + + Row { + label: input.label.clone(), + format: input.format, + width: input.width, + height: input.height, + codestream_bytes: input.bytes.len(), + cpu, + j2k_cuda, + nvidia, + j2k_cuda_psnr_vs_cpu, + nvidia_psnr_vs_cpu, + } + }) + .collect() +} + +fn validate_batch_pixel_len(inputs: &[DecodeInput], actual: usize) -> Result<(), String> { + let expected = inputs + .iter() + .map(input_pixel_bytes) + .try_fold(0usize, |sum, len| { + sum.checked_add(len?) + .ok_or_else(|| "aggregate J2K CUDA batch output byte count overflowed".to_string()) + })?; + if actual != expected { + return Err(format!("expected {expected} bytes, got {actual}")); + } + Ok(()) +} + +fn input_pixel_bytes(input: &DecodeInput) -> Result { + let width = usize::try_from(input.width).map_err(|_| "input width exceeds usize")?; + let height = usize::try_from(input.height).map_err(|_| "input height exceeds usize")?; + width + .checked_mul(height) + .and_then(|pixels| pixels.checked_mul(input.format.components())) + .ok_or_else(|| "input pixel byte count overflowed".to_string()) +} + +fn run_j2k_cuda_only(inputs: &[DecodeInput], config: &Config) -> Vec { + let mut rows = Vec::with_capacity(inputs.len()); + #[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] + let mut j2k_cuda_session = j2k_cuda::CudaSession::default(); + for input in inputs { + #[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] + let j2k_cuda = timed_best(config, || { + decode_j2k_cuda( + &mut j2k_cuda_session, + input, + config.collect_j2k_stage_timings, + config.skip_j2k_download, + ) + }); + #[cfg(any(target_os = "macos", not(feature = "nvjpeg2000")))] + let j2k_cuda = timed_best(config, || decode_j2k_cuda(input)); + rows.push(Row { + label: input.label.clone(), + format: input.format, + width: input.width, + height: input.height, + codestream_bytes: input.bytes.len(), + cpu: TimedStatus::failed("skipped"), + j2k_cuda, + nvidia: TimedStatus::failed("skipped"), + j2k_cuda_psnr_vs_cpu: None, + nvidia_psnr_vs_cpu: None, + }); + } + rows +} + +fn serial_profile_measurement( + rows: &[Row], + inputs: &[DecodeInput], + config: &Config, +) -> ProfileMeasurement { + ProfileMeasurement { + label: "j2k_cuda_decode_serial_reused_session", + execution_mode: "j2k_cuda_serial_reused_session", + timing_scope: "sum_of_per_input_best", + download_policy: j2k_download_policy(config), + input_count: inputs.len(), + megapixels: input_megapixels(inputs), + codestream_bytes: input_codestream_bytes(inputs), + status: sum_j2k_cuda_rows(rows), + } +} + +fn sum_j2k_cuda_rows(rows: &[Row]) -> TimedStatus { + let mut wall_ms = 0.0; + let mut gpu_ms = None; + let mut stage_ms = None; + let mut download_ms = None; + let mut cuda_profile = None; + for row in rows { + let Some(result) = row.j2k_cuda.result.as_ref() else { + return TimedStatus::failed(format!("{}: {}", row.label, row.j2k_cuda.status)); + }; + wall_ms += result.wall_ms; + gpu_ms = optional_sum(gpu_ms, result.gpu_ms); + stage_ms = optional_sum(stage_ms, result.stage_ms); + download_ms = optional_sum(download_ms, result.download_ms); + cuda_profile = optional_profile_sum(cuda_profile, result.cuda_profile.as_ref()); + } + TimedStatus::ok(TimedPixels { + wall_ms, + gpu_ms, + stage_ms, + download_ms, + cuda_profile, + pixels: Vec::new(), + }) +} + +fn optional_sum(current: Option, next: Option) -> Option { + match (current, next) { + (Some(current), Some(next)) => Some(current + next), + (None, Some(next)) => Some(next), + (current, None) => current, + } +} + +fn optional_profile_sum( + current: Option, + next: Option<&CudaStageBreakdown>, +) -> Option { + match (current, next) { + (Some(mut current), Some(next)) => { + current.add_assign(next); + Some(current) + } + (None, Some(next)) => Some(next.clone()), + (current, None) => current, + } +} + +fn input_megapixels(inputs: &[DecodeInput]) -> f64 { + inputs + .iter() + .map(|input| f64::from(input.width) * f64::from(input.height) / 1.0e6) + .sum() +} + +fn input_codestream_bytes(inputs: &[DecodeInput]) -> usize { + inputs.iter().map(|input| input.bytes.len()).sum() +} + +fn j2k_download_policy(config: &Config) -> &'static str { + if config.skip_j2k_download { + "no_download" + } else { + "download" + } +} + +fn nvidia_unavailable_status() -> String { + match NvBaselineSession::new() { + Err(NvBaselineError::NotBuilt) => "not-built".to_string(), + Err(error) => format!("session-error:{error:?}"), + Ok(_) => "session-unavailable".to_string(), + } +} + +fn timed_best(config: &Config, mut f: F) -> TimedStatus +where + F: FnMut() -> Result, +{ + let mut best: Option = None; + for index in 0..config.warmup.saturating_add(config.iterations) { + match f() { + Ok(mut result) => { + if index >= config.warmup + && best + .as_ref() + .is_none_or(|current| result.wall_ms < current.wall_ms) + { + result.wall_ms = round6(result.wall_ms); + result.gpu_ms = result.gpu_ms.map(round6); + result.stage_ms = result.stage_ms.map(round6); + result.download_ms = result.download_ms.map(round6); + best = Some(result); + } + } + Err(error) => return TimedStatus::failed(error), + } + } + best.map_or_else(|| TimedStatus::failed("no-measurements"), TimedStatus::ok) +} + +fn decode_cpu(input: &DecodeInput) -> Result { + let started = Instant::now(); + let image = Image::new(&input.bytes, &DecodeSettings::default()) + .map_err(|error| format!("cpu parse: {error}"))?; + let bitmap = image + .decode_native() + .map_err(|error| format!("cpu decode: {error}"))?; + if bitmap.width != input.width + || bitmap.height != input.height + || usize::from(bitmap.num_components) != input.format.components() + || bitmap.bytes_per_sample != 1 + { + return Err("cpu decode returned unexpected shape".to_string()); + } + Ok(TimedPixels { + wall_ms: elapsed_ms(started), + gpu_ms: None, + stage_ms: None, + download_ms: None, + cuda_profile: None, + pixels: bitmap.data, + }) +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn decode_j2k_cuda( + session: &mut j2k_cuda::CudaSession, + input: &DecodeInput, + collect_stage_timings: bool, + skip_download: bool, +) -> Result { + let started = Instant::now(); + let mut decoder = j2k_cuda::J2kDecoder::new(&input.bytes).map_err(|error| error.to_string())?; + let (surface, report) = if collect_stage_timings { + let (surface, report) = decoder + .decode_to_device_with_session_and_profile(input.format.j2k(), session) + .map_err(|error| format!("j2k cuda decode: {error}"))?; + (surface, Some(report)) + } else { + let surface = decoder + .decode_to_device_with_session(input.format.j2k(), session) + .map_err(|error| format!("j2k cuda decode: {error}"))?; + (surface, None) + }; + if surface.backend_kind() != BackendKind::Cuda + || surface.residency() != j2k_cuda::SurfaceResidency::CudaResidentDecode + { + return Err("j2k cuda decode did not return a CUDA-resident surface".to_string()); + } + let stride = input.width as usize * input.format.components(); + if skip_download { + return Ok(TimedPixels { + wall_ms: elapsed_ms(started), + gpu_ms: report + .as_ref() + .map(|report| us_to_ms(j2k_cuda_kernel_us(report))), + stage_ms: report.as_ref().map(|report| us_to_ms(report.total_us)), + download_ms: None, + cuda_profile: report.as_ref().map(CudaStageBreakdown::from_report), + pixels: Vec::new(), + }); + } + let mut pixels = vec![0u8; stride * input.height as usize]; + let download_started = Instant::now(); + surface + .download_into(&mut pixels, stride) + .map_err(|error| format!("j2k cuda download: {error}"))?; + let download_ms = elapsed_ms(download_started); + Ok(TimedPixels { + wall_ms: elapsed_ms(started), + gpu_ms: report + .as_ref() + .map(|report| us_to_ms(j2k_cuda_kernel_us(report))), + stage_ms: report.as_ref().map(|report| us_to_ms(report.total_us)), + download_ms: Some(download_ms), + cuda_profile: report.as_ref().map(CudaStageBreakdown::from_report), + pixels, + }) +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn decode_j2k_cuda_batch( + session: &mut j2k_cuda::CudaSession, + inputs: &[DecodeInput], + collect_stage_timings: bool, + skip_download: bool, +) -> Result { + let started = Instant::now(); + let Some(first) = inputs.first() else { + return Ok(TimedPixels { + wall_ms: elapsed_ms(started), + gpu_ms: collect_stage_timings.then_some(0.0), + stage_ms: collect_stage_timings.then_some(0.0), + download_ms: None, + cuda_profile: None, + pixels: Vec::new(), + }); + }; + if inputs.iter().any(|input| input.format != first.format) { + return Err("j2k cuda batch decode requires a single output format".to_string()); + } + let input_bytes = inputs + .iter() + .map(|input| input.bytes.as_slice()) + .collect::>(); + let (surfaces, report) = if collect_stage_timings { + let (surfaces, report) = + j2k_cuda::J2kDecoder::decode_batch_to_device_with_session_and_profile( + &input_bytes, + first.format.j2k(), + session, + ) + .map_err(|error| format!("j2k cuda batch decode: {error}"))?; + (surfaces, Some(report)) + } else { + let surfaces = j2k_cuda::J2kDecoder::decode_batch_to_device_with_session( + &input_bytes, + first.format.j2k(), + session, + ) + .map_err(|error| format!("j2k cuda batch decode: {error}"))?; + (surfaces, None) + }; + if surfaces.len() != inputs.len() { + return Err("j2k cuda batch decode returned unexpected surface count".to_string()); + } + let mut pixels = Vec::new(); + let mut download_ms = None; + for (surface, input) in surfaces.iter().zip(inputs) { + if surface.backend_kind() != BackendKind::Cuda + || surface.residency() != j2k_cuda::SurfaceResidency::CudaResidentDecode + { + return Err("j2k cuda batch decode did not return CUDA-resident surfaces".to_string()); + } + if surface.dimensions() != (input.width, input.height) { + return Err("j2k cuda batch decode returned unexpected shape".to_string()); + } + } + if !skip_download { + let download_started = Instant::now(); + pixels = j2k_cuda::Surface::download_batch_tight(&surfaces) + .map_err(|error| format!("j2k cuda batch download: {error}"))?; + download_ms = Some(elapsed_ms(download_started)); + } + Ok(TimedPixels { + wall_ms: elapsed_ms(started), + gpu_ms: report + .as_ref() + .map(|report| us_to_ms(j2k_cuda_kernel_us(report))), + stage_ms: report.as_ref().map(|report| us_to_ms(report.total_us)), + download_ms, + cuda_profile: report.as_ref().map(CudaStageBreakdown::from_report), + pixels, + }) +} + +#[cfg(any(target_os = "macos", not(feature = "nvjpeg2000")))] +fn decode_j2k_cuda(input: &DecodeInput) -> Result { + let _ = input; + Err("not-built".to_string()) +} + +#[cfg(any(target_os = "macos", not(feature = "nvjpeg2000")))] +fn decode_j2k_cuda_batch(inputs: &[DecodeInput]) -> Result { + let _ = inputs; + Err("not-built".to_string()) +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn j2k_cuda_kernel_us(report: &j2k_cuda::CudaHtj2kProfileReport) -> u128 { + [ + report.ht_cleanup_us, + report.ht_refine_us, + report.dequant_us, + report.idwt_us, + report.mct_us, + report.store_us, + ] + .into_iter() + .fold(0u128, u128::saturating_add) +} + +fn decode_nvidia( + session: &mut NvBaselineSession, + input: &DecodeInput, +) -> Result { + let started = Instant::now(); + let decoded = session + .decode_j2k_interleaved(&input.bytes, input.format.nvidia()) + .map_err(|error| format!("nvidia decode: {error:?}"))?; + if decoded.width != input.width + || decoded.height != input.height + || decoded.num_components as usize != input.format.components() + || decoded.bytes_per_sample != 1 + { + return Err("nvidia decode returned unexpected shape".to_string()); + } + Ok(TimedPixels { + wall_ms: elapsed_ms(started), + gpu_ms: Some(decoded.decode_ms), + stage_ms: None, + download_ms: None, + cuda_profile: None, + pixels: decoded.pixels, + }) +} + +fn elapsed_ms(started: Instant) -> f64 { + started.elapsed().as_secs_f64() * 1_000.0 +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn us_to_ms(micros: u128) -> f64 { + micros as f64 / 1_000.0 +} + +fn round6(value: f64) -> f64 { + (value * 1_000_000.0).round() / 1_000_000.0 +} + +fn clean_profile_zero(value: f64) -> f64 { + if value == 0.0 { + 0.0 + } else { + value + } +} + +fn print_report(report: &DecodeReport, config: &Config) { + if report.rows.is_empty() { + if let Some(profile) = &report.profile { + print_j2k_cuda_profile_report(profile, config); + return; + } + } + + let rows = &report.rows; + let megapixels = rows.iter().map(Row::megapixels).sum::(); + println!( + "inputs: {} codestream(s), {:.2} MP total, iterations {}", + rows.len(), + megapixels, + config.iterations + ); + if let Some(profile) = &report.profile { + println!( + "j2k_cuda: execution_mode={} timing_scope={} download_policy={}", + profile.execution_mode, profile.timing_scope, profile.download_policy + ); + } + println!( + "{:<24} {:>7} {:>10} {:>12} {:>12} {:>12} {:>12} {:>14} {:>14} {:>12}", + "input", + "format", + "CPU ms", + "sig status/wall", + "sig gpu", + "sig stage", + "sig dl", + "NVIDIA wall", + "NVIDIA gpu", + "NV PSNR" + ); + for row in rows { + println!( + "{:<24} {:>7} {:>10} {:>12} {:>12} {:>12} {:>12} {:>14} {:>14} {:>12}", + row.label, + row.format.label(), + status_ms(&row.cpu), + status_ms(&row.j2k_cuda), + status_gpu_ms(&row.j2k_cuda), + status_stage_ms(&row.j2k_cuda), + status_download_ms(&row.j2k_cuda), + status_ms(&row.nvidia), + status_gpu_ms(&row.nvidia), + fmt_optional(row.nvidia_psnr_vs_cpu) + ); + } + if let Some(profile) = &report.profile { + println!(); + print_j2k_cuda_profile_report(profile, config); + } +} + +fn print_j2k_cuda_profile_report(profile: &ProfileMeasurement, config: &Config) { + println!( + "inputs: {} codestream(s), {:.2} MP total", + profile.input_count, profile.megapixels + ); + println!( + "profile: {} execution_mode={} timing_scope={} download_policy={} iterations {}", + profile.label, + profile.execution_mode, + profile.timing_scope, + profile.download_policy, + config.iterations + ); + let Some(result) = profile.status.result.as_ref() else { + println!("{}: {}", profile.label, profile.status.status); + return; + }; + println!( + "PROFILE_RESULT {} execution_mode={} timing_scope={} download_policy={} input_count={} mp_s={:.3} wall_ms={:.3} gpu_ms={:.3} stage_ms={:.3} download_ms={:.3} bytes={}", + profile.label, + profile.execution_mode, + profile.timing_scope, + profile.download_policy, + profile.input_count, + profile.mp_s().unwrap_or(f64::NAN), + result.wall_ms, + clean_profile_zero(result.gpu_ms.unwrap_or(0.0)), + clean_profile_zero(result.stage_ms.unwrap_or(0.0)), + clean_profile_zero(result.download_ms.unwrap_or(0.0)), + profile.codestream_bytes + ); + if let Some(cuda_profile) = &result.cuda_profile { + println!( + "PROFILE_BREAKDOWN {} parse_us={} plan_us={} flatten_us={} h2d_us={} ht_cleanup_us={} ht_refine_us={} dequant_us={} idwt_us={} mct_us={} store_us={} total_us={} wall_total_us={} table_upload_us={} payload_upload_us={} status_d2h_us={} output_d2h_us={} block_count={} payload_bytes={} dispatch_count={} ht_dispatch_count={} dequant_dispatch_count={} idwt_dispatch_count={} mct_dispatch_count={} store_dispatch_count={}", + profile.label, + cuda_profile.parse_us, + cuda_profile.plan_us, + cuda_profile.flatten_us, + cuda_profile.h2d_us, + cuda_profile.ht_cleanup_us, + cuda_profile.ht_refine_us, + cuda_profile.dequant_us, + cuda_profile.idwt_us, + cuda_profile.mct_us, + cuda_profile.store_us, + cuda_profile.total_us, + cuda_profile.wall_total_us, + cuda_profile.table_upload_us, + cuda_profile.payload_upload_us, + cuda_profile.status_d2h_us, + cuda_profile.output_d2h_us, + cuda_profile.block_count, + cuda_profile.payload_bytes, + cuda_profile.dispatch_count, + cuda_profile.ht_dispatch_count, + cuda_profile.dequant_dispatch_count, + cuda_profile.idwt_dispatch_count, + cuda_profile.mct_dispatch_count, + cuda_profile.store_dispatch_count + ); + } +} + +fn status_ms(status: &TimedStatus) -> String { + status.result.as_ref().map_or_else( + || status.status.clone(), + |result| format!("{:.3}", result.wall_ms), + ) +} + +fn status_gpu_ms(status: &TimedStatus) -> String { + status + .result + .as_ref() + .and_then(|result| result.gpu_ms) + .map_or_else(|| "n/a".to_string(), |value| format!("{value:.3}")) +} + +fn status_stage_ms(status: &TimedStatus) -> String { + status + .result + .as_ref() + .and_then(|result| result.stage_ms) + .map_or_else(|| "n/a".to_string(), |value| format!("{value:.3}")) +} + +fn status_download_ms(status: &TimedStatus) -> String { + status + .result + .as_ref() + .and_then(|result| result.download_ms) + .map_or_else(|| "n/a".to_string(), |value| format!("{value:.3}")) +} + +fn fmt_optional(value: Option) -> String { + match value { + Some(value) if value.is_finite() => format!("{value:.3}"), + Some(_) => "inf".to_string(), + None => "n/a".to_string(), + } +} + +fn write_artifacts(report: &DecodeReport, config: &Config) -> std::io::Result<()> { + if let Some(path) = &config.json { + write_text_artifact(path, json_report(report, config))?; + } + if let Some(path) = &config.csv { + write_text_artifact(path, csv_report(report))?; + } + Ok(()) +} + +fn json_report(report: &DecodeReport, config: &Config) -> String { + let mut json = String::new(); + json.push_str("{\n"); + json.push_str(&format!(" \"input_count\": {},\n", report.input_count())); + json.push_str(&format!(" \"megapixels\": {:.6},\n", report.megapixels())); + json.push_str(&format!(" \"warmup\": {},\n", config.warmup)); + json.push_str(&format!(" \"iterations\": {},\n", config.iterations)); + json.push_str(&format!( + " \"max_inputs\": {},\n", + config + .max_inputs + .map_or_else(|| "null".to_string(), |value| value.to_string()) + )); + json.push_str(" \"profile\": "); + if let Some(profile) = &report.profile { + json.push_str(&json_profile(profile)); + json.push_str(",\n"); + } else { + json.push_str("null,\n"); + } + json.push_str(" \"rows\": [\n"); + for (index, row) in report.rows.iter().enumerate() { + if index > 0 { + json.push_str(",\n"); + } + json.push_str(" {\n"); + json.push_str(&format!( + " \"label\": \"{}\",\n", + escape_json(&row.label) + )); + json.push_str(&format!(" \"format\": \"{}\",\n", row.format.label())); + json.push_str(&format!(" \"width\": {},\n", row.width)); + json.push_str(&format!(" \"height\": {},\n", row.height)); + json.push_str(&format!( + " \"codestream_bytes\": {},\n", + row.codestream_bytes + )); + json.push_str(&json_status("cpu", &row.cpu, true)); + json.push_str(",\n"); + json.push_str(&json_status("j2k_cuda", &row.j2k_cuda, true)); + json.push_str(",\n"); + json.push_str(&json_status("nvidia_nvjpeg2000", &row.nvidia, true)); + json.push_str(",\n"); + json.push_str(&format!( + " \"j2k_cuda_psnr_vs_cpu\": {},\n", + json_optional(row.j2k_cuda_psnr_vs_cpu) + )); + json.push_str(&format!( + " \"nvidia_psnr_vs_cpu\": {}\n", + json_optional(row.nvidia_psnr_vs_cpu) + )); + json.push_str(" }"); + } + json.push_str("\n ]\n}\n"); + json +} + +fn json_profile(profile: &ProfileMeasurement) -> String { + let mut json = String::new(); + json.push('{'); + json.push_str(&format!("\"label\": \"{}\"", escape_json(profile.label))); + json.push_str(&format!( + ", \"execution_mode\": \"{}\"", + escape_json(profile.execution_mode) + )); + json.push_str(&format!( + ", \"timing_scope\": \"{}\"", + escape_json(profile.timing_scope) + )); + json.push_str(&format!( + ", \"download_policy\": \"{}\"", + escape_json(profile.download_policy) + )); + json.push_str(&format!(", \"input_count\": {}", profile.input_count)); + json.push_str(&format!(", \"megapixels\": {:.6}", profile.megapixels)); + json.push_str(&format!( + ", \"codestream_bytes\": {}", + profile.codestream_bytes + )); + json.push_str(&format!(", \"mp_s\": {}", json_optional(profile.mp_s()))); + json.push_str(", "); + json.push_str(&json_status("status", &profile.status, false)); + json.push('}'); + json +} + +fn json_status(name: &str, status: &TimedStatus, indent: bool) -> String { + let prefix = if indent { " " } else { "" }; + let mut json = String::new(); + json.push_str(&format!("{prefix}\"{name}\": {{")); + json.push_str(&format!("\"status\": \"{}\"", escape_json(&status.status))); + if let Some(result) = &status.result { + json.push_str(&format!(", \"wall_ms\": {:.6}", result.wall_ms)); + json.push_str(&format!(", \"gpu_ms\": {}", json_optional(result.gpu_ms))); + json.push_str(&format!( + ", \"stage_ms\": {}", + json_optional(result.stage_ms) + )); + json.push_str(&format!( + ", \"download_ms\": {}", + json_optional(result.download_ms) + )); + json.push_str(&format!(", \"bytes\": {}", result.pixels.len())); + if let Some(cuda_profile) = &result.cuda_profile { + json.push_str(", \"cuda_profile\": "); + json.push_str(&json_cuda_profile(cuda_profile)); + } + } + json.push('}'); + json +} + +fn json_cuda_profile(profile: &CudaStageBreakdown) -> String { + format!( + "{{\"parse_us\": {}, \"plan_us\": {}, \"flatten_us\": {}, \"h2d_us\": {}, \"ht_cleanup_us\": {}, \"ht_refine_us\": {}, \"dequant_us\": {}, \"idwt_us\": {}, \"mct_us\": {}, \"store_us\": {}, \"total_us\": {}, \"wall_total_us\": {}, \"table_upload_us\": {}, \"payload_upload_us\": {}, \"status_d2h_us\": {}, \"output_d2h_us\": {}, \"block_count\": {}, \"payload_bytes\": {}, \"dispatch_count\": {}, \"ht_dispatch_count\": {}, \"dequant_dispatch_count\": {}, \"idwt_dispatch_count\": {}, \"mct_dispatch_count\": {}, \"store_dispatch_count\": {}}}", + profile.parse_us, + profile.plan_us, + profile.flatten_us, + profile.h2d_us, + profile.ht_cleanup_us, + profile.ht_refine_us, + profile.dequant_us, + profile.idwt_us, + profile.mct_us, + profile.store_us, + profile.total_us, + profile.wall_total_us, + profile.table_upload_us, + profile.payload_upload_us, + profile.status_d2h_us, + profile.output_d2h_us, + profile.block_count, + profile.payload_bytes, + profile.dispatch_count, + profile.ht_dispatch_count, + profile.dequant_dispatch_count, + profile.idwt_dispatch_count, + profile.mct_dispatch_count, + profile.store_dispatch_count + ) +} + +fn json_optional(value: Option) -> String { + json_f64_or_inf(value, 6) +} + +fn csv_report(report: &DecodeReport) -> String { + if let Some(profile) = &report.profile { + if !report.rows.is_empty() { + return csv_combined_report(report, profile); + } + return csv_profile_report(profile); + } + + let mut csv = String::from( + "label,format,width,height,codestream_bytes,cpu_status,cpu_wall_ms,j2k_cuda_status,j2k_cuda_wall_ms,j2k_cuda_gpu_ms,j2k_cuda_stage_ms,j2k_cuda_download_ms,nvidia_status,nvidia_wall_ms,nvidia_gpu_ms,j2k_cuda_psnr_vs_cpu,nvidia_psnr_vs_cpu\n", + ); + for row in &report.rows { + csv.push_str(&format!( + "{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}\n", + escape_csv(&row.label), + row.format.label(), + row.width, + row.height, + row.codestream_bytes, + escape_csv(&row.cpu.status), + csv_ms(&row.cpu), + escape_csv(&row.j2k_cuda.status), + csv_ms(&row.j2k_cuda), + csv_gpu_ms(&row.j2k_cuda), + csv_stage_ms(&row.j2k_cuda), + csv_download_ms(&row.j2k_cuda), + escape_csv(&row.nvidia.status), + csv_ms(&row.nvidia), + csv_gpu_ms(&row.nvidia), + csv_optional(row.j2k_cuda_psnr_vs_cpu), + csv_optional(row.nvidia_psnr_vs_cpu), + )); + } + csv +} + +fn csv_combined_report(report: &DecodeReport, profile: &ProfileMeasurement) -> String { + let mut csv = format!( + "row_type,timing_scope,execution_mode,download_policy,input_count,label,status,format,width,height,megapixels,codestream_bytes,mp_s,wall_ms,gpu_ms,stage_ms,download_ms,cpu_status,cpu_wall_ms,nvidia_status,nvidia_wall_ms,nvidia_gpu_ms,j2k_cuda_psnr_vs_cpu,nvidia_psnr_vs_cpu,{}\n", + cuda_profile_csv_header() + ); + let cuda_profile = profile + .status + .result + .as_ref() + .and_then(|result| result.cuda_profile.as_ref()); + let mut aggregate = vec![ + "aggregate".to_string(), + profile.timing_scope.to_string(), + profile.execution_mode.to_string(), + profile.download_policy.to_string(), + profile.input_count.to_string(), + escape_csv(profile.label), + escape_csv(&profile.status.status), + String::new(), + String::new(), + String::new(), + format!("{:.6}", profile.megapixels), + profile.codestream_bytes.to_string(), + csv_optional(profile.mp_s()), + csv_ms(&profile.status), + csv_gpu_ms(&profile.status), + csv_stage_ms(&profile.status), + csv_download_ms(&profile.status), + String::new(), + String::new(), + String::new(), + String::new(), + String::new(), + String::new(), + String::new(), + ]; + aggregate.extend( + csv_cuda_profile_values(cuda_profile) + .split(',') + .map(str::to_string), + ); + csv.push_str(&aggregate.join(",")); + csv.push('\n'); + + for row in &report.rows { + let mut fields = vec![ + "correctness".to_string(), + "per_input_correctness".to_string(), + profile.execution_mode.to_string(), + profile.download_policy.to_string(), + String::new(), + escape_csv(&row.label), + escape_csv(&row.j2k_cuda.status), + row.format.label().to_string(), + row.width.to_string(), + row.height.to_string(), + format!("{:.6}", row.megapixels()), + row.codestream_bytes.to_string(), + String::new(), + String::new(), + String::new(), + String::new(), + String::new(), + escape_csv(&row.cpu.status), + csv_ms(&row.cpu), + escape_csv(&row.nvidia.status), + csv_ms(&row.nvidia), + csv_gpu_ms(&row.nvidia), + csv_optional(row.j2k_cuda_psnr_vs_cpu), + csv_optional(row.nvidia_psnr_vs_cpu), + ]; + fields.extend((0..24).map(|_| String::new())); + csv.push_str(&fields.join(",")); + csv.push('\n'); + } + + csv +} + +fn csv_profile_report(profile: &ProfileMeasurement) -> String { + let mut csv = format!( + "row_type,timing_scope,execution_mode,download_policy,input_count,label,status,megapixels,codestream_bytes,mp_s,wall_ms,gpu_ms,stage_ms,download_ms,{}\n", + cuda_profile_csv_header() + ); + let cuda_profile = profile + .status + .result + .as_ref() + .and_then(|result| result.cuda_profile.as_ref()); + csv.push_str(&format!( + "aggregate,{},{},{},{},{},{},{:.6},{},{},{},{},{},{},{}\n", + profile.timing_scope, + profile.execution_mode, + profile.download_policy, + profile.input_count, + escape_csv(profile.label), + escape_csv(&profile.status.status), + profile.megapixels, + profile.codestream_bytes, + csv_optional(profile.mp_s()), + csv_ms(&profile.status), + csv_gpu_ms(&profile.status), + csv_stage_ms(&profile.status), + csv_download_ms(&profile.status), + csv_cuda_profile_values(cuda_profile), + )); + csv +} + +fn cuda_profile_csv_header() -> &'static str { + "cuda_parse_us,cuda_plan_us,cuda_flatten_us,cuda_h2d_us,cuda_ht_cleanup_us,cuda_ht_refine_us,cuda_dequant_us,cuda_idwt_us,cuda_mct_us,cuda_store_us,cuda_total_us,cuda_wall_total_us,cuda_table_upload_us,cuda_payload_upload_us,cuda_status_d2h_us,cuda_output_d2h_us,cuda_block_count,cuda_payload_bytes,cuda_dispatch_count,cuda_ht_dispatch_count,cuda_dequant_dispatch_count,cuda_idwt_dispatch_count,cuda_mct_dispatch_count,cuda_store_dispatch_count" +} + +fn csv_cuda_profile_values(profile: Option<&CudaStageBreakdown>) -> String { + const FIELD_COUNT: usize = 24; + let Some(profile) = profile else { + return vec![""; FIELD_COUNT].join(","); + }; + format!( + "{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{}", + profile.parse_us, + profile.plan_us, + profile.flatten_us, + profile.h2d_us, + profile.ht_cleanup_us, + profile.ht_refine_us, + profile.dequant_us, + profile.idwt_us, + profile.mct_us, + profile.store_us, + profile.total_us, + profile.wall_total_us, + profile.table_upload_us, + profile.payload_upload_us, + profile.status_d2h_us, + profile.output_d2h_us, + profile.block_count, + profile.payload_bytes, + profile.dispatch_count, + profile.ht_dispatch_count, + profile.dequant_dispatch_count, + profile.idwt_dispatch_count, + profile.mct_dispatch_count, + profile.store_dispatch_count + ) +} + +fn csv_ms(status: &TimedStatus) -> String { + status + .result + .as_ref() + .map_or_else(String::new, |result| format!("{:.6}", result.wall_ms)) +} + +fn csv_gpu_ms(status: &TimedStatus) -> String { + status + .result + .as_ref() + .and_then(|result| result.gpu_ms) + .map_or_else(String::new, |value| format!("{value:.6}")) +} + +fn csv_stage_ms(status: &TimedStatus) -> String { + status + .result + .as_ref() + .and_then(|result| result.stage_ms) + .map_or_else(String::new, |value| format!("{value:.6}")) +} + +fn csv_download_ms(status: &TimedStatus) -> String { + status + .result + .as_ref() + .and_then(|result| result.download_ms) + .map_or_else(String::new, |value| format!("{value:.6}")) +} + +fn csv_optional(value: Option) -> String { + csv_f64_or_inf(value, 6) +} + +#[cfg(test)] +mod tests { + use super::{ + clean_profile_zero, csv_report, json_report, should_exit_for_failed_required_report, + Config, CudaStageBreakdown, DecodeCaseFormat, DecodeReport, ProfileMeasurement, Row, + TimedPixels, TimedStatus, + }; + + #[test] + fn config_parses_artifact_flags() { + let config = Config::from_args([ + "--profile-j2k-cuda-batch", + "--collect-j2k-stage-timings", + "--skip-j2k-download", + "--fixture-dim", + "256", + "--jpeg-dir", + "tests/nvidia-baseline/benchtiles/pancreas", + "--warmup", + "1", + "--iterations", + "3", + "--min-inputs", + "2", + "--json", + "target/decode.json", + "--csv", + "target/decode.csv", + ]) + .expect("config parses"); + + assert!(!config.profile_j2k_cuda_only); + assert!(config.profile_j2k_cuda_batch); + assert!(config.collect_j2k_stage_timings); + assert!(config.skip_j2k_download); + assert_eq!(config.fixture_dim, 256); + assert_eq!( + config.jpeg_dir.as_deref(), + Some(std::path::Path::new( + "tests/nvidia-baseline/benchtiles/pancreas" + )) + ); + assert_eq!(config.warmup, 1); + assert_eq!(config.iterations, 3); + assert_eq!(config.min_inputs, 2); + assert_eq!(config.max_inputs, None); + assert!(config.json.is_some()); + assert!(config.csv.is_some()); + } + + #[test] + fn profile_modes_conflict() { + let error = Config::from_args(["--profile-j2k-cuda-only", "--profile-j2k-cuda-batch"]) + .expect_err("profile modes conflict"); + + assert!(error.contains("conflicts")); + } + + #[test] + fn config_parses_max_inputs_separately_from_minimum_floor() { + let config = Config::from_args(["--min-inputs", "100", "--max-inputs", "128"]) + .expect("config parses"); + + assert_eq!(config.min_inputs, 100); + assert_eq!(config.max_inputs, Some(128)); + } + + #[test] + fn skip_j2k_download_requires_profile_only_mode() { + let error = Config::from_args(["--skip-j2k-download"]) + .expect_err("skip download is only valid in profile-only mode"); + + assert!(error.contains("--profile-j2k-cuda")); + } + + #[test] + fn skip_j2k_download_accepts_batch_profile_mode() { + let config = Config::from_args(["--profile-j2k-cuda-batch", "--skip-j2k-download"]) + .expect("batch profile can skip download"); + + assert!(config.profile_j2k_cuda_batch); + assert!(config.skip_j2k_download); + } + + #[test] + fn batch_correctness_mode_is_not_a_profile_only_mode() { + let config = Config::from_args(["--compare-j2k-cuda-batch"]) + .expect("batch correctness comparison parses"); + + assert!(config.compare_j2k_cuda_batch); + assert!(!config.is_j2k_cuda_profile()); + + let error = Config::from_args(["--compare-j2k-cuda-batch", "--profile-j2k-cuda-batch"]) + .expect_err("batch correctness conflicts with profile-only batch mode"); + assert!(error.contains("conflicts")); + } + + #[test] + fn csv_report_marks_unavailable_nvidia_without_zero_times() { + let row = Row { + label: "tile,1".to_string(), + format: DecodeCaseFormat::Rgb8, + width: 512, + height: 512, + codestream_bytes: 100, + cpu: TimedStatus::ok(TimedPixels { + wall_ms: 1.0, + gpu_ms: None, + stage_ms: None, + download_ms: None, + cuda_profile: None, + pixels: vec![0], + }), + j2k_cuda: TimedStatus::failed("not-built"), + nvidia: TimedStatus::failed("not-built"), + j2k_cuda_psnr_vs_cpu: None, + nvidia_psnr_vs_cpu: None, + }; + + let csv = csv_report(&DecodeReport::rows(vec![row])); + assert!(csv.contains("\"tile,1\",rgb8,512,512,100,ok,1.000000,not-built,,,,,not-built,,")); + } + + #[test] + fn artifacts_include_j2k_cuda_download_time_when_recorded() { + let row = Row { + label: "tile-1".to_string(), + format: DecodeCaseFormat::Rgb8, + width: 256, + height: 256, + codestream_bytes: 100, + cpu: TimedStatus::failed("skipped"), + j2k_cuda: TimedStatus::ok(TimedPixels { + wall_ms: 1.0, + gpu_ms: Some(0.25), + stage_ms: Some(0.5), + download_ms: Some(0.125), + cuda_profile: None, + pixels: vec![0], + }), + nvidia: TimedStatus::failed("skipped"), + j2k_cuda_psnr_vs_cpu: None, + nvidia_psnr_vs_cpu: None, + }; + let config = Config::from_args(std::iter::empty::<&str>()).expect("empty config parses"); + + let report = DecodeReport::rows(vec![row.clone()]); + let json = json_report(&report, &config); + assert!(json.contains("\"download_ms\": 0.125000")); + + let csv = csv_report(&DecodeReport::rows(vec![row])); + assert!(csv.contains("j2k_cuda_download_ms")); + assert!(csv.contains(",0.125000,")); + } + + #[test] + fn batch_profile_report_uses_single_aggregate_csv_row() { + let report = DecodeReport::profile( + Vec::new(), + ProfileMeasurement { + label: "j2k_cuda_decode_real_batch", + execution_mode: "j2k_cuda_batch", + timing_scope: "aggregate_batch", + download_policy: "no_download", + input_count: 2, + megapixels: 0.5, + codestream_bytes: 200, + status: TimedStatus::ok(TimedPixels { + wall_ms: 1.0, + gpu_ms: Some(0.25), + stage_ms: Some(0.5), + download_ms: None, + cuda_profile: None, + pixels: Vec::new(), + }), + }, + ); + + let csv = csv_report(&report); + assert_eq!(csv.lines().count(), 2); + assert!(csv.contains("aggregate_batch,j2k_cuda_batch,no_download")); + assert!(csv.contains("j2k_cuda_decode_real_batch")); + } + + #[test] + fn batch_profile_json_does_not_emit_per_input_timings() { + let config = Config::from_args(["--profile-j2k-cuda-batch", "--skip-j2k-download"]) + .expect("config parses"); + let report = DecodeReport::profile( + Vec::new(), + ProfileMeasurement { + label: "j2k_cuda_decode_real_batch", + execution_mode: "j2k_cuda_batch", + timing_scope: "aggregate_batch", + download_policy: "no_download", + input_count: 2, + megapixels: 0.5, + codestream_bytes: 200, + status: TimedStatus::ok(TimedPixels { + wall_ms: 1.0, + gpu_ms: Some(0.25), + stage_ms: Some(0.5), + download_ms: None, + cuda_profile: None, + pixels: Vec::new(), + }), + }, + ); + + let json = json_report(&report, &config); + assert!(json.contains("\"profile\": {")); + assert!(json.contains("\"execution_mode\": \"j2k_cuda_batch\"")); + assert!(json.contains("\"timing_scope\": \"aggregate_batch\"")); + assert!(json.contains("\"download_policy\": \"no_download\"")); + assert!(!json.contains("\"j2k_cuda\": {\"status\": \"ok\", \"wall_ms\"")); + } + + #[test] + fn batch_correctness_report_keeps_aggregate_timing_separate_from_rows() { + let config = Config::from_args(["--compare-j2k-cuda-batch"]).expect("config parses"); + let report = DecodeReport::profile( + vec![Row { + label: "tile-1".to_string(), + format: DecodeCaseFormat::Rgb8, + width: 256, + height: 256, + codestream_bytes: 100, + cpu: TimedStatus::ok(TimedPixels { + wall_ms: 2.0, + gpu_ms: None, + stage_ms: None, + download_ms: None, + cuda_profile: None, + pixels: vec![0], + }), + j2k_cuda: TimedStatus::ok_without_result(), + nvidia: TimedStatus::ok(TimedPixels { + wall_ms: 1.0, + gpu_ms: Some(0.5), + stage_ms: None, + download_ms: None, + cuda_profile: None, + pixels: vec![0], + }), + j2k_cuda_psnr_vs_cpu: Some(99.0), + nvidia_psnr_vs_cpu: Some(51.0), + }], + ProfileMeasurement { + label: "j2k_cuda_decode_batch_correctness", + execution_mode: "j2k_cuda_batch", + timing_scope: "aggregate_batch_correctness", + download_policy: "download", + input_count: 1, + megapixels: 0.065_536, + codestream_bytes: 100, + status: TimedStatus::ok(TimedPixels { + wall_ms: 0.5, + gpu_ms: Some(0.25), + stage_ms: Some(0.4), + download_ms: Some(0.05), + cuda_profile: None, + pixels: Vec::new(), + }), + }, + ); + + let json = json_report(&report, &config); + assert!(json.contains("\"timing_scope\": \"aggregate_batch_correctness\"")); + assert!(json.contains("\"j2k_cuda\": {\"status\": \"ok\"}")); + assert!(!json.contains("\"j2k_cuda\": {\"status\": \"ok\", \"wall_ms\"")); + + let csv = csv_report(&report); + assert!(csv.contains("aggregate_batch_correctness,j2k_cuda_batch,download")); + assert!(csv.contains("correctness,per_input_correctness,j2k_cuda_batch,download")); + assert!(csv.contains(",99.000000,51.000000")); + } + + #[test] + fn serial_profile_is_labeled_serial_reused_session() { + let report = DecodeReport::profile( + Vec::new(), + ProfileMeasurement { + label: "j2k_cuda_decode_serial_reused_session", + execution_mode: "j2k_cuda_serial_reused_session", + timing_scope: "sum_of_per_input_best", + download_policy: "download", + input_count: 2, + megapixels: 0.5, + codestream_bytes: 200, + status: TimedStatus::ok(TimedPixels { + wall_ms: 2.0, + gpu_ms: Some(0.5), + stage_ms: Some(1.0), + download_ms: Some(0.25), + cuda_profile: None, + pixels: Vec::new(), + }), + }, + ); + + let csv = csv_report(&report); + assert!(csv.contains("j2k_cuda_decode_serial_reused_session")); + assert!(csv.contains("sum_of_per_input_best")); + assert!(csv.contains("j2k_cuda_serial_reused_session")); + } + + #[test] + fn profile_result_zero_timings_do_not_render_negative_zero() { + assert_eq!(format!("{:.3}", clean_profile_zero(-0.0)), "0.000"); + } + + #[test] + fn failed_profile_report_requires_nonzero_exit_without_nvidia_requirement() { + let config = Config::from_args(["--profile-j2k-cuda-batch"]).expect("config parses"); + let report = DecodeReport::profile( + Vec::new(), + ProfileMeasurement { + label: "j2k_cuda_decode_real_batch", + execution_mode: "j2k_cuda_batch", + timing_scope: "aggregate_batch", + download_policy: "download", + input_count: 2, + megapixels: 0.5, + codestream_bytes: 200, + status: TimedStatus::failed("not-built"), + }, + ); + + assert!(should_exit_for_failed_required_report( + &report, &config, false + )); + } + + #[test] + fn profile_artifacts_include_cuda_stage_breakdown() { + let config = Config::from_args(["--profile-j2k-cuda-batch", "--collect-j2k-stage-timings"]) + .expect("config parses"); + let report = DecodeReport::profile( + Vec::new(), + ProfileMeasurement { + label: "j2k_cuda_decode_real_batch", + execution_mode: "j2k_cuda_batch", + timing_scope: "aggregate_batch", + download_policy: "download", + input_count: 2, + megapixels: 0.5, + codestream_bytes: 200, + status: TimedStatus::ok(TimedPixels { + wall_ms: 1.0, + gpu_ms: Some(0.25), + stage_ms: Some(0.5), + download_ms: Some(0.125), + cuda_profile: Some(sample_cuda_profile()), + pixels: Vec::new(), + }), + }, + ); + + let json = json_report(&report, &config); + assert!(json.contains("\"cuda_profile\": {")); + assert!(json.contains("\"idwt_us\": 8")); + assert!(json.contains("\"dispatch_count\": 19")); + + let csv = csv_report(&report); + assert!(csv.contains("cuda_idwt_us")); + assert!(csv.contains("cuda_dispatch_count")); + assert!(csv.contains(",7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24")); + } + + fn sample_cuda_profile() -> CudaStageBreakdown { + CudaStageBreakdown { + parse_us: 1, + plan_us: 2, + flatten_us: 3, + h2d_us: 4, + ht_cleanup_us: 5, + ht_refine_us: 6, + dequant_us: 7, + idwt_us: 8, + mct_us: 9, + store_us: 10, + total_us: 11, + wall_total_us: 12, + table_upload_us: 13, + payload_upload_us: 14, + status_d2h_us: 15, + output_d2h_us: 16, + block_count: 17, + payload_bytes: 18, + dispatch_count: 19, + ht_dispatch_count: 20, + dequant_dispatch_count: 21, + idwt_dispatch_count: 22, + mct_dispatch_count: 23, + store_dispatch_count: 24, + } + } +} diff --git a/tests/nvidia-baseline/src/bin/jpeg_decode_compare.rs b/tests/nvidia-baseline/src/bin/jpeg_decode_compare.rs new file mode 100644 index 00000000..c9b93096 --- /dev/null +++ b/tests/nvidia-baseline/src/bin/jpeg_decode_compare.rs @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +use std::time::Instant; + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +use j2k_core::PixelFormat; +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +use j2k_jpeg::{ + encode_jpeg_baseline, Decoder as JpegDecoder, JpegBackend, JpegEncodeOptions, JpegSamples, + JpegSubsampling, +}; +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +use j2k_jpeg_cuda::{Codec as JpegCudaCodec, CudaSession}; +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +use j2k_nvidia_baseline::{nvidia_decode_jpeg_rgb, NvBaselineSession, NvJpegDecodeTiming}; + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +const DEFAULT_DIM: u32 = 2048; +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +const DEFAULT_ITERS: usize = 100; +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +const DEFAULT_WARMUP: usize = 10; + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn main() -> Result<(), Box> { + let mut config = Config::from_env(); + let (jpeg, input_label) = if let Some(path) = std::env::var_os("J2K_CUDA_BENCH_JPEG") { + let jpeg = std::fs::read(&path)?; + let info = JpegDecoder::inspect(&jpeg)?; + config.width = info.dimensions.0; + config.height = info.dimensions.1; + (jpeg, format!("file={}", path.to_string_lossy())) + } else { + let rgb = patterned_rgb8(config.width, config.height, config.pattern); + let jpeg = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: config.width, + height: config.height, + }, + JpegEncodeOptions { + quality: config.quality, + subsampling: config.subsampling, + restart_interval: None, + backend: JpegBackend::Cpu, + }, + )? + .data; + ( + jpeg, + format!( + "generated_pattern={} subsampling={}", + config.pattern.name(), + subsampling_name(config.subsampling) + ), + ) + }; + + let owned = time_owned_cuda(&jpeg, config)?; + let nvidia = time_nvjpeg(&jpeg, config)?; + let (nvjpeg_pixels, nvjpeg_max_delta_vs_cpu) = nvjpeg_pixels_and_max_delta_vs_cpu(&jpeg)?; + let owned_max_delta_vs_nvjpeg = max_delta(&owned.verification_pixels, &nvjpeg_pixels); + + println!("JPEG decode comparison"); + println!( + "input: {}x{} {} jpeg_bytes={} warmup={} iterations={}", + config.width, + config.height, + input_label, + jpeg.len(), + config.warmup, + config.iterations + ); + println!(); + print_summary("owned_cuda_wall_ms", &owned.wall_ms); + print_summary("nvjpeg_wall_ms", &nvidia.wall_ms); + print_summary("nvjpeg_event_ms", &nvidia.event_ms); + println!(); + println!( + "owned_vs_nvjpeg_wall_mean: {:.2}x", + mean(&owned.wall_ms) / mean(&nvidia.wall_ms) + ); + println!( + "owned_vs_nvjpeg_event_mean: {:.2}x", + mean(&owned.wall_ms) / mean(&nvidia.event_ms) + ); + println!("owned_cuda_max_delta_vs_cpu: {}", owned.max_delta_vs_cpu); + println!("nvjpeg_max_delta_vs_cpu: {}", nvjpeg_max_delta_vs_cpu); + println!( + "owned_cuda_max_delta_vs_nvjpeg: {}", + owned_max_delta_vs_nvjpeg + ); + println!( + "note: owned_cuda_wall_ms includes Rust JPEG inspect/cache lookup, CUDA driver launch, and status readback; nvjpeg_event_ms is CUDA event time for the nvJPEG decode call only." + ); + + Ok(()) +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +#[derive(Clone, Copy)] +struct Config { + width: u32, + height: u32, + quality: u8, + iterations: usize, + warmup: usize, + pattern: Pattern, + subsampling: JpegSubsampling, +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +impl Config { + fn from_env() -> Self { + let (width, height) = std::env::var("J2K_GPU_BENCH_DIM") + .ok() + .map_or((DEFAULT_DIM, DEFAULT_DIM), |value| parse_dimensions(&value)); + Self { + width, + height, + quality: std::env::var("J2K_JPEG_COMPARE_QUALITY") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(90), + iterations: std::env::var("J2K_JPEG_COMPARE_ITERS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_ITERS), + warmup: std::env::var("J2K_JPEG_COMPARE_WARMUP") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_WARMUP), + pattern: Pattern::from_env(), + subsampling: subsampling_from_env(), + } + } +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn subsampling_from_env() -> JpegSubsampling { + let value = std::env::var("J2K_JPEG_COMPARE_SUBSAMPLING") + .or_else(|_| std::env::var("J2K_CUDA_BENCH_SUBSAMPLING")) + .unwrap_or_else(|_| "420".to_string()); + match value.trim().to_ascii_lowercase().as_str() { + "420" | "4:2:0" | "ybr420" => JpegSubsampling::Ybr420, + "422" | "4:2:2" | "ybr422" => JpegSubsampling::Ybr422, + "444" | "4:4:4" | "ybr444" => JpegSubsampling::Ybr444, + other => { + panic!("unsupported J2K_JPEG_COMPARE_SUBSAMPLING={other}; expected 420, 422, or 444") + } + } +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +const fn subsampling_name(subsampling: JpegSubsampling) -> &'static str { + match subsampling { + JpegSubsampling::Gray => "gray", + JpegSubsampling::Ybr444 => "444", + JpegSubsampling::Ybr422 => "422", + JpegSubsampling::Ybr420 => "420", + } +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +#[derive(Clone, Copy)] +enum Pattern { + Stress, + Smooth, +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +impl Pattern { + fn from_env() -> Self { + match std::env::var("J2K_JPEG_COMPARE_PATTERN") + .unwrap_or_else(|_| "stress".to_string()) + .as_str() + { + "smooth" => Self::Smooth, + "stress" => Self::Stress, + other => panic!("unsupported J2K_JPEG_COMPARE_PATTERN={other}"), + } + } + + const fn name(self) -> &'static str { + match self { + Self::Stress => "stress", + Self::Smooth => "smooth", + } + } +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +struct OwnedTiming { + wall_ms: Vec, + max_delta_vs_cpu: u8, + verification_pixels: Vec, +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +struct NvidiaTiming { + wall_ms: Vec, + event_ms: Vec, +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn time_owned_cuda(jpeg: &[u8], config: Config) -> Result> { + let mut session = CudaSession::default(); + let pitch = config.width as usize * PixelFormat::Rgb8.bytes_per_pixel(); + let byte_len = pitch * config.height as usize; + let output = session.take_owned_cuda_output_buffer(byte_len)?; + for _ in 0..config.warmup { + let stats = JpegCudaCodec::decode_tile_rgb8_into_cuda_buffer_with_session( + jpeg, + &output, + pitch, + &mut session, + )?; + assert!(stats.used_owned_cuda_decode()); + } + + let mut wall_ms = Vec::with_capacity(config.iterations); + for _ in 0..config.iterations { + let start = Instant::now(); + let stats = JpegCudaCodec::decode_tile_rgb8_into_cuda_buffer_with_session( + jpeg, + &output, + pitch, + &mut session, + )?; + assert!(stats.used_owned_cuda_decode()); + wall_ms.push(start.elapsed().as_secs_f64() * 1000.0); + } + + let mut downloaded = vec![0u8; byte_len]; + output.copy_to_host(&mut downloaded)?; + let (expected, _) = JpegDecoder::new(jpeg)?.decode(PixelFormat::Rgb8)?; + let max_delta_vs_cpu = downloaded + .iter() + .zip(expected) + .map(|(actual, expected)| actual.abs_diff(expected)) + .max() + .unwrap_or(0); + Ok(OwnedTiming { + wall_ms, + max_delta_vs_cpu, + verification_pixels: downloaded, + }) +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn time_nvjpeg(jpeg: &[u8], config: Config) -> Result> { + let mut session = NvBaselineSession::new() + .map_err(|error| format!("NVIDIA baseline unavailable: {error:?}"))?; + for _ in 0..config.warmup { + let timing = session + .decode_jpeg_rgb_interleaved_timed(jpeg) + .map_err(|error| format!("nvJPEG warmup failed: {error:?}"))?; + assert_decode_dimensions(timing, config); + } + + let mut wall_ms = Vec::with_capacity(config.iterations); + let mut event_ms = Vec::with_capacity(config.iterations); + for _ in 0..config.iterations { + let start = Instant::now(); + let timing = session + .decode_jpeg_rgb_interleaved_timed(jpeg) + .map_err(|error| format!("nvJPEG decode failed: {error:?}"))?; + let elapsed_ms = start.elapsed().as_secs_f64() * 1000.0; + assert_decode_dimensions(timing, config); + wall_ms.push(elapsed_ms); + event_ms.push(timing.decode_ms); + } + Ok(NvidiaTiming { wall_ms, event_ms }) +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn nvjpeg_pixels_and_max_delta_vs_cpu( + jpeg: &[u8], +) -> Result<(Vec, u8), Box> { + let (actual, width, height) = + nvidia_decode_jpeg_rgb(jpeg).map_err(|error| format!("nvJPEG decode failed: {error:?}"))?; + let (expected, outcome) = JpegDecoder::new(jpeg)?.decode(PixelFormat::Rgb8)?; + assert_eq!((width, height), (outcome.decoded.w, outcome.decoded.h)); + let delta = max_delta(&actual, &expected); + Ok((actual, delta)) +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn assert_decode_dimensions(timing: NvJpegDecodeTiming, config: Config) { + assert_eq!((timing.width, timing.height), (config.width, config.height)); +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn patterned_rgb8(width: u32, height: u32, pattern: Pattern) -> Vec { + let mut out = Vec::with_capacity(width as usize * height as usize * 3); + for y in 0..height { + for x in 0..width { + match pattern { + Pattern::Stress => { + out.push(((x * 13 + y * 3) & 0xFF) as u8); + out.push(((x * 5 + y * 17) & 0xFF) as u8); + out.push(((x * 11 + y * 7 + (x ^ y)) & 0xFF) as u8); + } + Pattern::Smooth => { + let sx = x.saturating_mul(255) / width.max(1); + let sy = y.saturating_mul(255) / height.max(1); + let tissue = (((x / 64) * 19 + (y / 64) * 37) & 0x3F) as u32; + out.push(((sx * 3 + sy + tissue) / 5).min(255) as u8); + out.push(((sx + sy * 2 + tissue * 2) / 5).min(255) as u8); + out.push(((sx + sy + 128 + tissue) / 4).min(255) as u8); + } + } + } + } + out +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn parse_dimensions(value: &str) -> (u32, u32) { + if let Some((width, height)) = value.split_once('x') { + ( + width + .trim() + .parse() + .expect("J2K_GPU_BENCH_DIM width must be u32"), + height + .trim() + .parse() + .expect("J2K_GPU_BENCH_DIM height must be u32"), + ) + } else { + let square = value + .trim() + .parse() + .expect("J2K_GPU_BENCH_DIM must be u32 or WIDTHxHEIGHT"); + (square, square) + } +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn print_summary(label: &str, values: &[f64]) { + let mut sorted = values.to_vec(); + sorted.sort_by(f64::total_cmp); + println!( + "{label}: mean={:.4} min={:.4} p50={:.4} p90={:.4}", + mean(values), + sorted[0], + percentile(&sorted, 0.50), + percentile(&sorted, 0.90) + ); +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn mean(values: &[f64]) -> f64 { + values.iter().sum::() / values.len() as f64 +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn percentile(sorted: &[f64], p: f64) -> f64 { + let index = ((sorted.len() - 1) as f64 * p).round() as usize; + sorted[index] +} + +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +fn max_delta(actual: &[u8], expected: &[u8]) -> u8 { + actual + .iter() + .zip(expected) + .map(|(actual, expected)| actual.abs_diff(*expected)) + .max() + .unwrap_or(0) +} + +#[cfg(any(target_os = "macos", not(feature = "nvjpeg2000")))] +fn main() { + eprintln!("jpeg_decode_compare requires --features nvjpeg2000 on a CUDA/NVIDIA baseline host"); +} diff --git a/tests/nvidia-baseline/src/bin/report_format/mod.rs b/tests/nvidia-baseline/src/bin/report_format/mod.rs new file mode 100644 index 00000000..5f2f5eb8 --- /dev/null +++ b/tests/nvidia-baseline/src/bin/report_format/mod.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Included by multiple benchmark binaries; each target uses a different subset. +#![allow(dead_code)] + +use std::fmt::Write as _; + +pub(crate) fn escape_json(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for ch in value.chars() { + match ch { + '"' => escaped.push_str("\\\""), + '\\' => escaped.push_str("\\\\"), + '\n' => escaped.push_str("\\n"), + '\r' => escaped.push_str("\\r"), + '\t' => escaped.push_str("\\t"), + ch if ch.is_control() => { + write!(&mut escaped, "\\u{:04x}", ch as u32) + .expect("writing to String cannot fail"); + } + ch => escaped.push(ch), + } + } + escaped +} + +pub(crate) fn escape_csv(value: &str) -> String { + if value.contains([',', '"', '\n', '\r']) { + format!("\"{}\"", value.replace('"', "\"\"")) + } else { + value.to_string() + } +} + +pub(crate) fn json_f64_or_null(value: Option, decimals: usize) -> String { + value + .filter(|value| value.is_finite()) + .map_or_else(|| "null".to_string(), |value| format_f64(value, decimals)) +} + +pub(crate) fn json_f64_or_inf(value: Option, decimals: usize) -> String { + value.map_or_else( + || "null".to_string(), + |value| { + if value.is_finite() { + format_f64(value, decimals) + } else { + "\"inf\"".to_string() + } + }, + ) +} + +pub(crate) fn csv_f64_or_blank(value: Option, decimals: usize) -> String { + value + .filter(|value| value.is_finite()) + .map_or_else(String::new, |value| format_f64(value, decimals)) +} + +pub(crate) fn csv_f64_or_inf(value: Option, decimals: usize) -> String { + value.map_or_else(String::new, |value| { + if value.is_finite() { + format_f64(value, decimals) + } else { + "inf".to_string() + } + }) +} + +fn format_f64(value: f64, decimals: usize) -> String { + format!("{value:.decimals$}") +} + +#[cfg(test)] +mod tests { + use super::{ + csv_f64_or_blank, csv_f64_or_inf, escape_csv, escape_json, json_f64_or_inf, + json_f64_or_null, + }; + + #[test] + fn escape_json_handles_quotes_slashes_and_control_chars() { + assert_eq!( + escape_json("tile \"a\"\\b\n\t\u{0007}"), + "tile \\\"a\\\"\\\\b\\n\\t\\u0007" + ); + } + + #[test] + fn escape_csv_quotes_only_when_required() { + assert_eq!(escape_csv("plain"), "plain"); + assert_eq!(escape_csv("tile, \"a\"\n"), "\"tile, \"\"a\"\"\n\""); + } + + #[test] + fn optional_json_f64_policies_preserve_existing_reports() { + assert_eq!(json_f64_or_null(Some(42.25), 8), "42.25000000"); + assert_eq!(json_f64_or_null(None, 8), "null"); + assert_eq!(json_f64_or_null(Some(f64::INFINITY), 8), "null"); + + assert_eq!(json_f64_or_inf(Some(42.25), 6), "42.250000"); + assert_eq!(json_f64_or_inf(None, 6), "null"); + assert_eq!(json_f64_or_inf(Some(f64::NAN), 6), "\"inf\""); + } + + #[test] + fn optional_csv_f64_policies_preserve_existing_reports() { + assert_eq!(csv_f64_or_blank(Some(42.25), 6), "42.250000"); + assert_eq!(csv_f64_or_blank(None, 6), ""); + assert_eq!(csv_f64_or_blank(Some(f64::NEG_INFINITY), 6), ""); + + assert_eq!(csv_f64_or_inf(Some(42.25), 6), "42.250000"); + assert_eq!(csv_f64_or_inf(None, 6), ""); + assert_eq!(csv_f64_or_inf(Some(f64::INFINITY), 6), "inf"); + } +} diff --git a/tests/nvidia-baseline/src/bin/svs_extract.rs b/tests/nvidia-baseline/src/bin/svs_extract.rs new file mode 100644 index 00000000..d8de420f --- /dev/null +++ b/tests/nvidia-baseline/src/bin/svs_extract.rs @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Extract benchmark JPEG tiles from an Aperio .svs whole-slide image. +// +// Many GDC SVS files store their tiles as JPEG 2000 (Aperio compression 33003 / +// 33005), not JPEG, so they cannot feed a JPEG -> HTJ2K transcode benchmark +// directly. This tool decodes each tile (J2K via j2k-native; original +// JPEG passed through) to RGB and re-encodes a deterministic, tissue-only subset +// as baseline JPEG into an output directory for `transcode_compare`. +// +// Re-encoding adds one lossy step, so the tiles are realistic WSI *content* at +// realistic tile sizes rather than byte-identical originals — fine for a +// throughput benchmark (and the PSNR reference is self-consistent across codecs). +// +// Usage: +// svs_extract [--limit N] [--stride S] [--quality Q] +// +// Defaults: --limit 256 --stride 7 --quality 85. Near-white background tiles are +// skipped (mean luma > 235). + +use std::path::Path; + +use j2k_jpeg::encoder::{encode_jpeg_baseline, JpegEncodeOptions, JpegSamples}; +use j2k_native::{DecodeSettings, Image}; +use j2k_nvidia_baseline::ycbcr_to_rgb_round_nearest; + +fn main() { + let mut args = std::env::args().skip(1); + let Some(svs_path) = args.next() else { + eprintln!( + "usage: svs_extract [--limit N] [--stride S] [--quality Q]" + ); + std::process::exit(2); + }; + let Some(out_dir) = args.next() else { + eprintln!( + "usage: svs_extract [--limit N] [--stride S] [--quality Q]" + ); + std::process::exit(2); + }; + let mut limit = 256usize; + let mut stride = 7usize; + let mut quality = 85u8; + let mut min_tissue = 0.5f64; + while let Some(flag) = args.next() { + match flag.as_str() { + "--limit" => limit = args.next().and_then(|v| v.parse().ok()).unwrap_or(limit), + "--stride" => { + stride = args + .next() + .and_then(|v| v.parse().ok()) + .unwrap_or(stride) + .max(1); + } + "--quality" => quality = args.next().and_then(|v| v.parse().ok()).unwrap_or(quality), + "--min-tissue" => { + min_tissue = args + .next() + .and_then(|v| v.parse().ok()) + .unwrap_or(min_tissue); + } + other => { + eprintln!("unknown flag: {other}"); + std::process::exit(2); + } + } + } + + if let Err(error) = run(&svs_path, &out_dir, limit, stride, quality, min_tissue) { + eprintln!("error: {error}"); + std::process::exit(1); + } +} + +fn run( + svs_path: &str, + out_dir: &str, + limit: usize, + stride: usize, + quality: u8, + min_tissue: f64, +) -> Result<(), String> { + let bytes = std::fs::read(svs_path).map_err(|e| format!("read {svs_path}: {e}"))?; + let ifd0 = parse_first_ifd(&bytes)?; + println!( + "slide: {}x{} px, {}x{} tiles, compression {}, {} tiles total", + ifd0.image_width, + ifd0.image_height, + ifd0.tile_width, + ifd0.tile_height, + ifd0.compression, + ifd0.tile_offsets.len() + ); + + std::fs::create_dir_all(out_dir).map_err(|e| format!("create {out_dir}: {e}"))?; + + let options = JpegEncodeOptions { + quality, + ..JpegEncodeOptions::default() + }; + + let mut written = 0usize; + let mut attempted = 0usize; + let mut skipped_blank = 0usize; + let mut decode_failures = 0usize; + let mut min_seen_fraction = 1.0f64; + let mut sum_fraction = 0.0f64; + + let mut index = 0usize; + while index < ifd0.tile_offsets.len() && written < limit { + let offset = ifd0.tile_offsets[index] as usize; + let count = ifd0.tile_byte_counts[index] as usize; + index += stride; + if count == 0 || offset + count > bytes.len() { + continue; + } + attempted += 1; + let tile = &bytes[offset..offset + count]; + + let Some((rgb, w, h)) = decode_tile_rgb(tile, ifd0.compression) else { + decode_failures += 1; + continue; + }; + let fraction = tissue_fraction(&rgb); + // Require both real tissue coverage and visible structure (texture), so + // flat homogeneous stroma/background is rejected in favour of cellular + // tissue with nuclei and edges. + if fraction < min_tissue || luma_stddev(&rgb) < 12.0 { + skipped_blank += 1; + continue; + } + + let encoded = encode_jpeg_baseline( + JpegSamples::Rgb8 { + data: &rgb, + width: w, + height: h, + }, + options, + ) + .map_err(|e| format!("encode tile {index}: {e}"))?; + + let path = Path::new(out_dir).join(format!("tile_{written:05}.jpg")); + std::fs::write(&path, &encoded.data) + .map_err(|e| format!("write {}: {e}", path.display()))?; + written += 1; + min_seen_fraction = min_seen_fraction.min(fraction); + sum_fraction += fraction; + } + + let mean_fraction = if written > 0 { + sum_fraction / written as f64 + } else { + 0.0 + }; + println!( + "wrote {written} JPEG tiles to {out_dir} (attempted {attempted}, skipped {skipped_blank} below {:.0}% tissue, {decode_failures} decode failures)", + min_tissue * 100.0 + ); + println!( + "tissue coverage of written tiles: min {:.0}%, mean {:.0}%", + min_seen_fraction * 100.0, + mean_fraction * 100.0 + ); + if written == 0 { + return Err( + "no tiles written — decode may be unsupported, or all tiles below the tissue threshold" + .to_string(), + ); + } + Ok(()) +} + +/// Fraction of pixels that look like stained tissue rather than bright glass / +/// background. A pixel is tissue when it has meaningful absorption (its darkest +/// channel is well below white) or visible stain saturation (channel spread). +fn tissue_fraction(rgb: &[u8]) -> f64 { + if rgb.is_empty() { + return 0.0; + } + let mut tissue = 0usize; + let total = rgb.len() / 3; + for px in rgb.chunks_exact(3) { + let min_c = px[0].min(px[1]).min(px[2]); + let max_c = px[0].max(px[1]).max(px[2]); + let absorbs = min_c < 210; // not near-white on every channel + let saturated = u16::from(max_c) - u16::from(min_c) > 25; // visible stain color + if absorbs || saturated { + tissue += 1; + } + } + tissue as f64 / total as f64 +} + +/// Standard deviation of per-pixel luma, a cheap proxy for spatial structure. +/// Flat regions (uniform stain, glass) score near zero; cellular tissue is high. +fn luma_stddev(rgb: &[u8]) -> f64 { + let total = rgb.len() / 3; + if total == 0 { + return 0.0; + } + let luma: Vec = rgb + .chunks_exact(3) + .map(|px| 0.299 * f64::from(px[0]) + 0.587 * f64::from(px[1]) + 0.114 * f64::from(px[2])) + .collect(); + let mean = luma.iter().sum::() / total as f64; + let variance = luma.iter().map(|&l| (l - mean) * (l - mean)).sum::() / total as f64; + variance.sqrt() +} + +// Aperio JPEG 2000 with YCbCr components (the codestream stores Y/Cb/Cr planes +// directly rather than using J2K's in-stream color transform). +const APERIO_J2K_YCBCR: u16 = 33003; + +/// Decode one tile to interleaved RGB. JPEG 2000 / HTJ2K codestreams and JPEG +/// tiles are both handled by the native parser via `Image`. Aperio's J2K-YCbCr +/// tiles decode to YCbCr component values, which are converted to RGB here. +fn decode_tile_rgb(tile: &[u8], compression: u16) -> Option<(Vec, u32, u32)> { + let image = Image::new(tile, &DecodeSettings::default()).ok()?; + let bitmap = image.decode_native().ok()?; + if bitmap.bytes_per_sample != 1 { + return None; + } + match bitmap.num_components { + 3 => { + let rgb = if compression == APERIO_J2K_YCBCR { + ycbcr_to_rgb_round_nearest(&bitmap.data) + } else { + bitmap.data + }; + Some((rgb, bitmap.width, bitmap.height)) + } + 1 => { + // Expand grayscale to RGB so the encoder path is uniform. + let rgb = bitmap.data.iter().flat_map(|&g| [g, g, g]).collect(); + Some((rgb, bitmap.width, bitmap.height)) + } + _ => None, + } +} + +struct Ifd { + image_width: u32, + image_height: u32, + tile_width: u32, + tile_height: u32, + compression: u16, + tile_offsets: Vec, + tile_byte_counts: Vec, +} + +/// Minimal classic little-endian TIFF reader for the first (full-resolution) IFD. +fn parse_first_ifd(bytes: &[u8]) -> Result { + if bytes.len() < 8 || &bytes[0..2] != b"II" { + return Err("not a little-endian TIFF (expected II byte order)".to_string()); + } + let magic = u16::from_le_bytes([bytes[2], bytes[3]]); + if magic != 42 { + return Err(format!( + "unsupported TIFF magic {magic} (BigTIFF is not supported)" + )); + } + let ifd_off = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]) as usize; + if ifd_off + 2 > bytes.len() { + return Err("IFD offset out of range".to_string()); + } + let entry_count = u16::from_le_bytes([bytes[ifd_off], bytes[ifd_off + 1]]) as usize; + + let mut compression = 1u16; + let (mut image_width, mut image_height) = (0u32, 0u32); + let (mut tile_width, mut tile_height) = (0u32, 0u32); + let mut tile_offsets = Vec::new(); + let mut tile_byte_counts = Vec::new(); + + for i in 0..entry_count { + let base = ifd_off + 2 + i * 12; + if base + 12 > bytes.len() { + return Err("IFD entry out of range".to_string()); + } + let tag = u16::from_le_bytes([bytes[base], bytes[base + 1]]); + let typ = u16::from_le_bytes([bytes[base + 2], bytes[base + 3]]); + let count = u32::from_le_bytes([ + bytes[base + 4], + bytes[base + 5], + bytes[base + 6], + bytes[base + 7], + ]); + let value_field = &bytes[base + 8..base + 12]; + match tag { + 256 => image_width = scalar(typ, value_field), + 257 => image_height = scalar(typ, value_field), + 259 => compression = scalar(typ, value_field) as u16, + 322 => tile_width = scalar(typ, value_field), + 323 => tile_height = scalar(typ, value_field), + 324 => tile_offsets = read_u32_array(bytes, typ, count, value_field)?, + 325 => tile_byte_counts = read_u32_array(bytes, typ, count, value_field)?, + _ => {} + } + } + + if tile_offsets.is_empty() || tile_offsets.len() != tile_byte_counts.len() { + return Err( + "IFD has no tiles or mismatched tile arrays (is this a tiled SVS?)".to_string(), + ); + } + Ok(Ifd { + image_width, + image_height, + tile_width, + tile_height, + compression, + tile_offsets, + tile_byte_counts, + }) +} + +/// Read an inline scalar (SHORT=3 or LONG=4) from a 4-byte value field. +fn scalar(typ: u16, value_field: &[u8]) -> u32 { + match typ { + 3 => u32::from(u16::from_le_bytes([value_field[0], value_field[1]])), + _ => u32::from_le_bytes([ + value_field[0], + value_field[1], + value_field[2], + value_field[3], + ]), + } +} + +/// Read a SHORT/LONG array, inline when it fits in 4 bytes else at the offset. +fn read_u32_array( + bytes: &[u8], + typ: u16, + count: u32, + value_field: &[u8], +) -> Result, String> { + let count = count as usize; + let elem = match typ { + 3 => 2usize, + 4 => 4usize, + other => return Err(format!("unsupported tile-array type {other}")), + }; + let total = count * elem; + let data: &[u8] = if total <= 4 { + value_field + } else { + let off = u32::from_le_bytes([ + value_field[0], + value_field[1], + value_field[2], + value_field[3], + ]) as usize; + bytes + .get(off..off + total) + .ok_or_else(|| "tile array out of range".to_string())? + }; + let mut out = Vec::with_capacity(count); + for i in 0..count { + let v = if elem == 2 { + u32::from(u16::from_le_bytes([data[i * 2], data[i * 2 + 1]])) + } else { + u32::from_le_bytes([ + data[i * 4], + data[i * 4 + 1], + data[i * 4 + 2], + data[i * 4 + 3], + ]) + }; + out.push(v); + } + Ok(out) +} diff --git a/tests/nvidia-baseline/src/bin/transcode_compare.rs b/tests/nvidia-baseline/src/bin/transcode_compare.rs new file mode 100644 index 00000000..c565c007 --- /dev/null +++ b/tests/nvidia-baseline/src/bin/transcode_compare.rs @@ -0,0 +1,2264 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// JPEG -> HTJ2K transcode throughput comparison: +// j2k — coefficient-domain batch transcode (CUDA transform with CPU or CUDA HT encode) +// NVIDIA — reused-session serial nvJPEG decode + nvJPEG2000 HT encode +// +// Reports the four metric families requested: end-to-end throughput (MP/s), +// per-stage breakdown, output size + PSNR, and GPU-only vs wall-clock. +// +// Usage: +// transcode_compare [more.jpg ...] +// transcode_compare # falls back to J2K_BENCH_JPEG_DIR/*.jpg +// +// The NVIDIA columns show "n/a (not built)" unless this crate was compiled with +// --features nvjpeg2000 on a host with nvcc + libnvjpeg + libnvjpeg2k. + +mod report_format; + +use std::{path::PathBuf, time::Instant}; + +use j2k::adapter::encode_stage::IrreversibleQuantizationSubbandScales; +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +use j2k_cuda::CudaEncodeStageAccelerator; +#[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] +use j2k_native::J2kEncodeStageAccelerator; +use j2k_native::{DecodeSettings, Image}; +use j2k_nvidia_baseline::{ + nvidia_decode_jpeg_rgb, psnr_u8, write_text_artifact, ycbcr_to_rgb_round_nearest, + NvBaselineError, NvBaselineSession, +}; +use j2k_transcode::{ + EncodedTranscodeBatch, JpegTileBatchInput, JpegToHtj2kError, JpegToHtj2kOptions, + JpegToHtj2kTranscoder, JPEG_TO_HTJ2K_LOSSY_97_QUANTIZATION_SCALE, +}; +use report_format::{ + csv_f64_or_blank, escape_csv as csv_escape, escape_json as json_escape, json_f64_or_null, +}; + +// The j2k GPU backend is platform-selected: Metal on macOS, CUDA elsewhere. +#[cfg(not(target_os = "macos"))] +use j2k_transcode_cuda::CudaDctToWaveletStageAccelerator as BenchAccelerator; +#[cfg(target_os = "macos")] +use j2k_transcode_metal::MetalDctToWaveletStageAccelerator as BenchAccelerator; + +const BACKEND_NAME: &str = if cfg!(target_os = "macos") { + "Metal" +} else { + "CUDA" +}; + +const WARMUP: usize = 2; +const ITERATIONS: usize = 10; + +#[allow(clippy::too_many_lines)] +fn main() { + let config = match BenchmarkConfig::from_env_args() { + Ok(config) => config, + Err(error) => { + eprintln!("{error}"); + std::process::exit(2); + } + }; + let jpegs = match load_inputs(&config) { + Ok(jpegs) if !jpegs.is_empty() => jpegs, + Ok(_) => { + eprintln!("no JPEG inputs found; pass file paths or set J2K_BENCH_JPEG_DIR"); + std::process::exit(2); + } + Err(error) => { + eprintln!("failed to load inputs: {error}"); + std::process::exit(2); + } + }; + if let Some(min_tiles) = config.min_tiles { + if jpegs.len() < min_tiles { + eprintln!( + "input corpus has {} tile(s), below required --min-tiles {min_tiles}", + jpegs.len() + ); + std::process::exit(2); + } + } + + let total_pixels: u64 = jpegs + .iter() + .map(|j| u64::from(j.width) * u64::from(j.height)) + .sum(); + let megapixels = total_pixels as f64 / 1.0e6; + println!( + "inputs: {} tile(s), {:.2} MP total\n", + jpegs.len(), + megapixels + ); + + if config.profile_j2k_cuda_only { + let scale = config + .quant_scales + .first() + .copied() + .unwrap_or(JPEG_TO_HTJ2K_LOSSY_97_QUANTIZATION_SCALE); + let options = lossy_options_for_config(&config, scale); + let j2k_cuda_ht = run_j2k(&jpegs, &options, &config); + print_j2k_cuda_profile_report( + &jpegs, + megapixels, + corpus_hash(&jpegs), + &config, + scale, + &j2k_cuda_ht, + ); + if let Err(error) = + write_j2k_profile_artifacts(&config, &jpegs, megapixels, scale, &j2k_cuda_ht) + { + eprintln!("failed to write benchmark artifacts: {error}"); + std::process::exit(2); + } + enforce_required_j2k_result(&j2k_cuda_ht); + return; + } + + let nvidia = run_nvidia(&jpegs, &config); + let rd_points = run_j2k_rd_sweep(&jpegs, &config, &nvidia); + let selected_index = select_rd_point(&rd_points, &config, &nvidia); + let j2k_cpu_ht = rd_points + .get(selected_index) + .map_or_else(J2KResult::default, |point| point.result.clone()); + let selected_scale = rd_points + .get(selected_index) + .map_or(JPEG_TO_HTJ2K_LOSSY_97_QUANTIZATION_SCALE, |point| { + point.scale + }); + let selected_options = lossy_options_for_config(&config, selected_scale); + let j2k_cuda_ht = run_j2k(&jpegs, &selected_options, &config); + + print_report( + &jpegs, + megapixels, + corpus_hash(&jpegs), + &config, + &rd_points, + selected_index, + &j2k_cpu_ht, + &j2k_cuda_ht, + &nvidia, + ); + if let Err(error) = write_artifacts( + &config, + &jpegs, + megapixels, + &rd_points, + selected_index, + &j2k_cuda_ht, + &nvidia, + ) { + eprintln!("failed to write benchmark artifacts: {error}"); + std::process::exit(2); + } + if let Err(error) = validate_rate_match(&config, &j2k_cpu_ht, &j2k_cuda_ht, &nvidia) { + eprintln!("{error}"); + std::process::exit(1); + } + enforce_required_results(&j2k_cuda_ht, &nvidia); +} + +#[derive(Debug, Clone)] +struct BenchmarkConfig { + input_paths: Vec, + quant_scales: Vec, + subband_scales: IrreversibleQuantizationSubbandScales, + decomposition_levels: u8, + match_nvidia_bytes: bool, + match_tolerance: f64, + profile_j2k_cuda_only: bool, + warmup: usize, + iterations: usize, + min_tiles: Option, + json_path: Option, + csv_path: Option, +} + +impl Default for BenchmarkConfig { + fn default() -> Self { + Self { + input_paths: Vec::new(), + quant_scales: vec![JPEG_TO_HTJ2K_LOSSY_97_QUANTIZATION_SCALE], + subband_scales: IrreversibleQuantizationSubbandScales::default(), + decomposition_levels: 1, + match_nvidia_bytes: false, + match_tolerance: 0.02, + profile_j2k_cuda_only: false, + warmup: WARMUP, + iterations: ITERATIONS, + min_tiles: None, + json_path: None, + csv_path: None, + } + } +} + +impl BenchmarkConfig { + fn from_env_args() -> Result { + Self::parse(std::env::args_os().skip(1).map(PathBuf::from)) + } + + #[allow(clippy::similar_names)] + fn parse(args: impl IntoIterator) -> Result { + let mut config = Self::default(); + let mut iter = args.into_iter(); + while let Some(arg) = iter.next() { + let arg_s = arg.to_string_lossy(); + match arg_s.as_ref() { + "--match-nvidia-bytes" => config.match_nvidia_bytes = true, + "--profile-j2k-cuda-only" => config.profile_j2k_cuda_only = true, + "--quant-scales" => { + let value = iter + .next() + .ok_or("--quant-scales requires a comma-separated value")?; + config.quant_scales = parse_f32_list(&value.to_string_lossy())?; + if config.quant_scales.is_empty() { + return Err("--quant-scales must contain at least one scale".to_string()); + } + } + "--subband-scales" => { + let value = iter + .next() + .ok_or("--subband-scales requires ll,hl,lh,hh values")?; + config.subband_scales = parse_subband_scales(&value.to_string_lossy())?; + } + "--decomposition-levels" => { + let value = iter + .next() + .ok_or("--decomposition-levels requires a value")?; + config.decomposition_levels = + parse_positive_u8(&value.to_string_lossy(), "--decomposition-levels")?; + } + "--match-tolerance" => { + let value = iter.next().ok_or("--match-tolerance requires a value")?; + config.match_tolerance = + parse_positive_f64(&value.to_string_lossy(), "--match-tolerance")?; + } + "--warmup" => { + let value = iter.next().ok_or("--warmup requires a value")?; + config.warmup = parse_positive_usize(&value.to_string_lossy(), "--warmup")?; + } + "--iterations" => { + let value = iter.next().ok_or("--iterations requires a value")?; + config.iterations = + parse_positive_usize(&value.to_string_lossy(), "--iterations")?; + } + "--min-tiles" => { + let value = iter.next().ok_or("--min-tiles requires a value")?; + config.min_tiles = Some(parse_positive_usize( + &value.to_string_lossy(), + "--min-tiles", + )?); + } + "--json" => { + config.json_path = Some(iter.next().ok_or("--json requires a path")?); + } + "--csv" => { + config.csv_path = Some(iter.next().ok_or("--csv requires a path")?); + } + "--help" | "-h" => return Err(usage()), + _ if arg_s.starts_with("--") => { + return Err(format!("unknown option: {arg_s}\n{}", usage())); + } + _ => config.input_paths.push(arg), + } + } + if config.profile_j2k_cuda_only && config.match_nvidia_bytes { + return Err( + "--profile-j2k-cuda-only cannot be combined with --match-nvidia-bytes".to_string(), + ); + } + Ok(config) + } +} + +fn usage() -> String { + "usage: transcode_compare [--profile-j2k-cuda-only] [--quant-scales a,b,c] [--subband-scales ll,hl,lh,hh] [--decomposition-levels n] [--match-nvidia-bytes] [--match-tolerance frac] [--warmup n] [--iterations n] [--min-tiles n] [--json path] [--csv path] [file.jpg ...]".to_string() +} + +fn parse_f32_list(value: &str) -> Result, String> { + value + .split(',') + .map(|part| { + let parsed = part + .trim() + .parse::() + .map_err(|_| format!("invalid f32 value: {part}"))?; + if parsed.is_finite() && parsed > 0.0 { + Ok(parsed) + } else { + Err(format!( + "scale must be finite and greater than zero: {part}" + )) + } + }) + .collect() +} + +fn parse_positive_f64(value: &str, flag: &str) -> Result { + let parsed = value + .parse::() + .map_err(|_| format!("{flag} must be a finite positive number"))?; + if parsed.is_finite() && parsed > 0.0 { + Ok(parsed) + } else { + Err(format!("{flag} must be finite and greater than zero")) + } +} + +fn parse_positive_usize(value: &str, flag: &str) -> Result { + value + .parse::() + .map_err(|_| format!("{flag} must be a positive integer")) + .and_then(|value| { + (value > 0) + .then_some(value) + .ok_or_else(|| format!("{flag} must be greater than zero")) + }) +} + +fn parse_positive_u8(value: &str, flag: &str) -> Result { + let parsed = parse_positive_usize(value, flag)?; + u8::try_from(parsed).map_err(|_| format!("{flag} must be <= {}", u8::MAX)) +} + +fn parse_subband_scales(value: &str) -> Result { + let values = parse_f32_list(value)?; + if values.len() != 4 { + return Err("--subband-scales expects exactly four values: ll,hl,lh,hh".to_string()); + } + Ok(IrreversibleQuantizationSubbandScales { + low_low: values[0], + high_low: values[1], + low_high: values[2], + high_high: values[3], + }) +} + +struct JpegInput { + bytes: Vec, + width: u32, + height: u32, + label: String, +} + +fn load_inputs(config: &BenchmarkConfig) -> std::io::Result> { + let mut paths = config.input_paths.clone(); + if paths.is_empty() { + if let Some(dir) = std::env::var_os("J2K_BENCH_JPEG_DIR") { + for entry in std::fs::read_dir(dir)? { + let path = entry?.path(); + if path.extension().is_some_and(|ext| { + ext.eq_ignore_ascii_case("jpg") || ext.eq_ignore_ascii_case("jpeg") + }) { + paths.push(path); + } + } + paths.sort(); + } + } + if paths.is_empty() { + // Bundled sanity fixture (tiny/unrepresentative — for a smoke test only). + let bytes = include_bytes!( + "../../../../crates/j2k-transcode-cuda/tests/fixtures/conformance/baseline_420_16x16.jpg" + ) + .to_vec(); + eprintln!("warning: no inputs given; using the bundled 16x16 fixture (not representative)"); + return Ok(vec![JpegInput { + width: 16, + height: 16, + label: "baseline_420_16x16".to_string(), + bytes, + }]); + } + + let mut inputs = Vec::with_capacity(paths.len()); + for path in paths { + let bytes = std::fs::read(&path)?; + let (width, height) = jpeg_dimensions(&bytes).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("could not parse JPEG dimensions from {}", path.display()), + ) + })?; + inputs.push(JpegInput { + bytes, + width, + height, + label: path.file_name().map_or_else( + || path.display().to_string(), + |name| name.to_string_lossy().into_owned(), + ), + }); + } + Ok(inputs) +} + +/// Parse a JPEG's pixel dimensions from its SOF marker (no decode). +fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + if bytes.len() < 4 || bytes[0] != 0xFF || bytes[1] != 0xD8 { + return None; + } + let mut i = 2; // skip SOI + while i + 9 < bytes.len() { + if bytes[i] != 0xFF { + i += 1; + continue; + } + let marker = bytes[i + 1]; + // SOF0..SOF3, SOF5..SOF7, SOF9..SOF11, SOF13..SOF15 carry dimensions. + let is_sof = matches!(marker, 0xC0..=0xC3 | 0xC5..=0xC7 | 0xC9..=0xCB | 0xCD..=0xCF); + let len = (u16::from(bytes[i + 2]) << 8 | u16::from(bytes[i + 3])) as usize; + if is_sof { + let height = u32::from(bytes[i + 5]) << 8 | u32::from(bytes[i + 6]); + let width = u32::from(bytes[i + 7]) << 8 | u32::from(bytes[i + 8]); + return (width != 0 && height != 0).then_some((width, height)); + } + if marker == 0xD8 || marker == 0xD9 || (0xD0..=0xD7).contains(&marker) { + i += 2; + } else { + i += 2 + len; + } + } + None +} + +#[derive(Clone, Default)] +struct J2KResult { + ran: bool, + used_gpu: bool, + best_wall_s: f64, + extract_us: u128, + repack_us: u128, + transform_wall_us: u128, + encode_wall_us: u128, + transform_gpu_stage_us: u128, + encode_cuda_stage_us: u128, + encode_cuda_ht_kernel_us: u128, + encode_cuda_ht_status_readback_us: u128, + encode_cuda_ht_compact_us: u128, + encode_cuda_ht_output_readback_us: u128, + pack_upload_us: u128, + idct_row_lift_us: u128, + column_lift_us: u128, + quantize_us: u128, + readback_us: u128, + transform_dispatches: usize, + transform_dispatched_jobs: usize, + transform_cpu_fallback_jobs: usize, + encode_dispatches: usize, + encode_ht_code_block_dispatches: usize, + encode_packetization_dispatches: usize, + output_bytes: usize, + codestreams: Vec>, +} + +#[derive(Clone)] +struct RdPoint { + scale: f32, + result: J2KResult, + quality: Option, +} + +#[derive(Clone, Copy, Default)] +struct EncodeBenchMetrics { + cuda_stage_us: u128, + total_dispatches: usize, + ht_code_block_dispatches: usize, + packetization_dispatches: usize, +} + +fn lossy_options_for_config(config: &BenchmarkConfig, scale: f32) -> JpegToHtj2kOptions { + let mut options = JpegToHtj2kOptions::lossy_97(); + options.encode_options.irreversible_quantization_scale = scale; + options.encode_options.num_decomposition_levels = config.decomposition_levels; + options + .encode_options + .irreversible_quantization_subband_scales = config.subband_scales; + options +} + +fn run_j2k_rd_sweep( + jpegs: &[JpegInput], + config: &BenchmarkConfig, + nvidia: &NvidiaResult, +) -> Vec { + config + .quant_scales + .iter() + .copied() + .map(|scale| { + let options = lossy_options_for_config(config, scale); + let result = run_j2k_transform_cpu_encode(jpegs, &options, config); + let quality = quality_summary(jpegs, &result.codestreams); + if config.match_nvidia_bytes && nvidia.ran { + let delta = byte_delta_fraction(result.output_bytes, nvidia.output_bytes); + println!( + "RD point: scale {scale:.4} bytes {} delta vs NVIDIA {:+.2}% PSNR {}", + result.output_bytes, + delta * 100.0, + fmt_psnr(quality.as_ref().and_then(|q| q.mean_psnr)), + ); + } + RdPoint { + scale, + result, + quality, + } + }) + .collect() +} + +fn select_rd_point(points: &[RdPoint], config: &BenchmarkConfig, nvidia: &NvidiaResult) -> usize { + if points.is_empty() { + return 0; + } + if !(config.match_nvidia_bytes && nvidia.ran) { + return 0; + } + let selected = points + .iter() + .enumerate() + .min_by(|(_, a), (_, b)| { + byte_delta_fraction(a.result.output_bytes, nvidia.output_bytes) + .abs() + .partial_cmp(&byte_delta_fraction(b.result.output_bytes, nvidia.output_bytes).abs()) + .unwrap_or(std::cmp::Ordering::Equal) + }) + .map_or(0, |(idx, _)| idx); + let delta = + byte_delta_fraction(points[selected].result.output_bytes, nvidia.output_bytes).abs(); + if delta > config.match_tolerance { + eprintln!( + "warning: closest J2K RD point is {:.2}% from NVIDIA bytes, outside {:.2}% tolerance", + delta * 100.0, + config.match_tolerance * 100.0 + ); + } + selected +} + +fn validate_rate_match( + config: &BenchmarkConfig, + selected_cpu_ht: &J2KResult, + cuda_ht: &J2KResult, + nvidia: &NvidiaResult, +) -> Result<(), String> { + if !(config.match_nvidia_bytes && nvidia.ran) { + return Ok(()); + } + + let selected_delta = + byte_delta_fraction(selected_cpu_ht.output_bytes, nvidia.output_bytes).abs(); + if selected_delta > config.match_tolerance { + return Err(format!( + "selected J2K RD point is {:.2}% from NVIDIA bytes, outside {:.2}% tolerance", + selected_delta * 100.0, + config.match_tolerance * 100.0 + )); + } + + if cuda_ht.ran { + let cuda_delta = byte_delta_fraction(cuda_ht.output_bytes, nvidia.output_bytes).abs(); + if cuda_delta > config.match_tolerance { + return Err(format!( + "J2K CUDA HT row is {:.2}% from NVIDIA bytes, outside {:.2}% tolerance", + cuda_delta * 100.0, + config.match_tolerance * 100.0 + )); + } + } + + Ok(()) +} + +fn byte_delta_fraction(candidate: usize, target: usize) -> f64 { + if target == 0 { + return 0.0; + } + (candidate as f64 - target as f64) / target as f64 +} + +fn run_j2k( + jpegs: &[JpegInput], + options: &JpegToHtj2kOptions, + config: &BenchmarkConfig, +) -> J2KResult { + let inputs: Vec> = jpegs + .iter() + .map(|j| JpegTileBatchInput { bytes: &j.bytes }) + .collect(); + // Warm up (and detect whether the GPU path is available). + let used_gpu = true; + let mut session = J2KBenchSession::new(true, true); + for iteration in 0..config.warmup.max(1) { + match session.transcode_batch(&inputs, options) { + Ok((batch, encode_metrics)) => validate_j2k_cuda_dispatch(&batch, encode_metrics), + Err(error) => { + assert!( + ! j2k_cuda_required(), + "j2k: explicit {BACKEND_NAME} path failed under J2K_REQUIRE_CUDA_RUNTIME=1: {error:?}" + ); + if iteration == 0 { + eprintln!( + "j2k: explicit {BACKEND_NAME} CUDA HT path unavailable; reporting CUDA HT row as n/a" + ); + } + return J2KResult::default(); + } + } + } + + let mut best_wall_s = f64::INFINITY; + let mut last = J2KResult::default(); + for _ in 0..config.iterations { + let start = Instant::now(); + let batch = session.transcode_batch(&inputs, options); + let elapsed = start.elapsed().as_secs_f64(); + let (batch, encode_metrics) = match batch { + Ok(batch) => batch, + Err(error) => { + assert!( + ! j2k_cuda_required(), + "j2k: explicit {BACKEND_NAME} path failed under J2K_REQUIRE_CUDA_RUNTIME=1: {error:?}" + ); + return J2KResult::default(); + } + }; + validate_j2k_cuda_dispatch(&batch, encode_metrics); + if elapsed < best_wall_s { + best_wall_s = elapsed; + last = j2k_result_from_batch(&batch, used_gpu, elapsed, encode_metrics); + } + } + last +} + +fn run_j2k_transform_cpu_encode( + jpegs: &[JpegInput], + options: &JpegToHtj2kOptions, + config: &BenchmarkConfig, +) -> J2KResult { + let inputs: Vec> = jpegs + .iter() + .map(|j| JpegTileBatchInput { bytes: &j.bytes }) + .collect(); + let mut used_gpu = true; + let mut transcoder = JpegToHtj2kTranscoder::default(); + let mut accelerator = BenchAccelerator::new_explicit(); + for iteration in 0..config.warmup.max(1) { + match transcoder + .transcode_batch_with_accelerator(&inputs, options, &mut accelerator) + .and_then(reject_failed_j2k_tiles) + { + Ok(batch) => validate_j2k_transform_dispatch(&batch), + Err(error) => { + assert!( + ! j2k_cuda_required(), + "j2k: explicit {BACKEND_NAME} transform path failed under J2K_REQUIRE_CUDA_RUNTIME=1: {error:?}" + ); + used_gpu = false; + if iteration == 0 { + eprintln!( + "j2k: explicit {BACKEND_NAME} transform unavailable; measuring scalar CPU fallback" + ); + } + break; + } + } + } + + let mut best_wall_s = f64::INFINITY; + let mut last = J2KResult::default(); + for _ in 0..config.iterations { + let start = Instant::now(); + let batch = if used_gpu { + transcoder + .transcode_batch_with_accelerator(&inputs, options, &mut accelerator) + .and_then(reject_failed_j2k_tiles) + } else { + transcoder + .transcode_batch(&inputs, options) + .and_then(reject_failed_j2k_tiles) + }; + let elapsed = start.elapsed().as_secs_f64(); + let batch = match batch { + Ok(batch) => batch, + Err(error) => { + assert!( + ! j2k_cuda_required(), + "j2k: explicit {BACKEND_NAME} transform path failed under J2K_REQUIRE_CUDA_RUNTIME=1: {error:?}" + ); + return J2KResult::default(); + } + }; + if used_gpu { + validate_j2k_transform_dispatch(&batch); + } + if elapsed < best_wall_s { + best_wall_s = elapsed; + last = j2k_result_from_batch(&batch, used_gpu, elapsed, EncodeBenchMetrics::default()); + } + } + last +} + +fn j2k_result_from_batch( + batch: &EncodedTranscodeBatch, + used_gpu: bool, + elapsed: f64, + encode_metrics: EncodeBenchMetrics, +) -> J2KResult { + let t = &batch.report.timings; + J2KResult { + ran: true, + used_gpu, + best_wall_s: elapsed, + extract_us: batch.report.extract_us, + repack_us: t.jpeg_dct_repack_us, + transform_wall_us: batch.report.transform_us, + encode_wall_us: batch.report.encode_us, + pack_upload_us: t.dwt97_batch_pack_upload_us, + idct_row_lift_us: t.dwt97_batch_idct_row_lift_us, + column_lift_us: t.dwt97_batch_column_lift_us, + quantize_us: t.dwt97_batch_quantize_codeblock_us, + readback_us: t.dwt97_batch_readback_us, + transform_gpu_stage_us: t.dwt97_batch_pack_upload_us + + t.dwt97_batch_idct_row_lift_us + + t.dwt97_batch_column_lift_us + + t.dwt97_batch_quantize_codeblock_us + + t.dwt97_batch_readback_us, + encode_cuda_stage_us: encode_metrics + .cuda_stage_us + .saturating_add(t.dwt97_batch_ht_encode_us), + encode_cuda_ht_kernel_us: t.dwt97_batch_ht_kernel_us, + encode_cuda_ht_status_readback_us: t.dwt97_batch_ht_status_readback_us, + encode_cuda_ht_compact_us: t.dwt97_batch_ht_compact_us, + encode_cuda_ht_output_readback_us: t.dwt97_batch_ht_output_readback_us, + transform_dispatches: t.accelerator_dispatches, + transform_dispatched_jobs: t.accelerator_dispatched_jobs, + transform_cpu_fallback_jobs: t.cpu_fallback_jobs, + encode_dispatches: encode_metrics + .total_dispatches + .saturating_add(t.dwt97_batch_ht_codeblock_dispatches), + encode_ht_code_block_dispatches: encode_metrics + .ht_code_block_dispatches + .saturating_add(t.dwt97_batch_ht_codeblock_dispatches), + encode_packetization_dispatches: encode_metrics.packetization_dispatches, + output_bytes: batch + .tiles + .iter() + .flatten() + .map(|tile| tile.codestream.len()) + .sum(), + codestreams: batch + .tiles + .iter() + .flatten() + .map(|tile| tile.codestream.clone()) + .collect(), + } +} + +struct J2KBenchSession { + use_gpu: bool, + transcoder: JpegToHtj2kTranscoder, + transform_accelerator: Option, + #[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] + encode_accelerator: Option, +} + +impl J2KBenchSession { + fn new(use_gpu: bool, resident_ht_encode: bool) -> Self { + Self { + use_gpu, + transcoder: JpegToHtj2kTranscoder::default(), + transform_accelerator: use_gpu.then(|| new_bench_accelerator(resident_ht_encode)), + #[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] + encode_accelerator: use_gpu.then(|| { + CudaEncodeStageAccelerator::with_profile_collection(true) + .prefer_cpu_ht_subband(true) + .prefer_cpu_quantize_subband(true) + .prefer_cpu_packetization(true) + }), + } + } + + fn transcode_batch( + &mut self, + inputs: &[JpegTileBatchInput<'_>], + options: &JpegToHtj2kOptions, + ) -> Result<(EncodedTranscodeBatch, EncodeBenchMetrics), JpegToHtj2kError> { + if !self.use_gpu { + return self + .transcoder + .transcode_batch(inputs, options) + .and_then(reject_failed_j2k_tiles) + .map(|batch| (batch, EncodeBenchMetrics::default())); + } + + self.transcode_gpu_batch(inputs, options) + } + + #[cfg(target_os = "macos")] + fn transcode_gpu_batch( + &mut self, + inputs: &[JpegTileBatchInput<'_>], + options: &JpegToHtj2kOptions, + ) -> Result<(EncodedTranscodeBatch, EncodeBenchMetrics), JpegToHtj2kError> { + let accelerator = self + .transform_accelerator + .as_mut() + .expect("GPU j2k session has a transform accelerator"); + self.transcoder + .transcode_batch_with_accelerator(inputs, options, accelerator) + .and_then(reject_failed_j2k_tiles) + .map(|batch| (batch, EncodeBenchMetrics::default())) + } + + #[cfg(all(not(target_os = "macos"), feature = "nvjpeg2000"))] + fn transcode_gpu_batch( + &mut self, + inputs: &[JpegTileBatchInput<'_>], + options: &JpegToHtj2kOptions, + ) -> Result<(EncodedTranscodeBatch, EncodeBenchMetrics), JpegToHtj2kError> { + let transform_accelerator = self + .transform_accelerator + .as_mut() + .expect("GPU j2k session has a transform accelerator"); + let encode_accelerator = self + .encode_accelerator + .as_mut() + .expect("CUDA j2k session has an encode accelerator"); + let before = encode_accelerator.dispatch_report(); + encode_accelerator.reset_collected_stage_timings(); + let batch = self.transcoder.transcode_batch_with_accelerators( + inputs, + options, + transform_accelerator, + encode_accelerator, + )?; + let batch = reject_failed_j2k_tiles(batch)?; + let encode_timings = encode_accelerator.collected_stage_timings(); + let dispatch = encode_accelerator + .dispatch_report() + .saturating_delta(before); + Ok(( + batch, + EncodeBenchMetrics { + cuda_stage_us: encode_timings.total_us(), + total_dispatches: dispatch.total(), + ht_code_block_dispatches: dispatch.ht_code_block, + packetization_dispatches: dispatch.packetization, + }, + )) + } + + #[cfg(all(not(target_os = "macos"), not(feature = "nvjpeg2000")))] + fn transcode_gpu_batch( + &mut self, + inputs: &[JpegTileBatchInput<'_>], + options: &JpegToHtj2kOptions, + ) -> Result<(EncodedTranscodeBatch, EncodeBenchMetrics), JpegToHtj2kError> { + let accelerator = self + .transform_accelerator + .as_mut() + .expect("GPU j2k session has a transform accelerator"); + self.transcoder + .transcode_batch_with_accelerator(inputs, options, accelerator) + .and_then(reject_failed_j2k_tiles) + .map(|batch| (batch, EncodeBenchMetrics::default())) + } +} + +#[cfg(not(target_os = "macos"))] +fn new_bench_accelerator(resident_ht_encode: bool) -> BenchAccelerator { + if resident_ht_encode { + BenchAccelerator::new_explicit_resident_ht_encode() + } else { + BenchAccelerator::new_explicit() + } +} + +#[cfg(target_os = "macos")] +fn new_bench_accelerator(_resident_ht_encode: bool) -> BenchAccelerator { + BenchAccelerator::new_explicit() +} + +fn reject_failed_j2k_tiles( + batch: EncodedTranscodeBatch, +) -> Result { + if batch.report.failed_tiles == 0 { + Ok(batch) + } else { + Err(JpegToHtj2kError::Validation( + "j2k benchmark produced one or more failed tiles", + )) + } +} + +fn j2k_cuda_required() -> bool { + std::env::var_os("J2K_REQUIRE_CUDA_RUNTIME").is_some() +} + +#[cfg(not(target_os = "macos"))] +fn j2k_cuda_stage_timings_disabled() -> bool { + std::env::var_os("J2K_CUDA_DISABLE_STAGE_TIMINGS").is_some() +} + +fn nvidia_baseline_required() -> bool { + std::env::var_os("J2K_REQUIRE_NV_BASELINE_BUILD").is_some() +} + +fn enforce_required_j2k_result(j2k: &J2KResult) { + if j2k_cuda_required() && !(j2k.ran && j2k.used_gpu) { + eprintln!("j2k: required CUDA benchmark did not produce a GPU result"); + std::process::exit(1); + } +} + +fn enforce_required_results(j2k: &J2KResult, nvidia: &NvidiaResult) { + let mut failed = false; + if j2k_cuda_required() && !(j2k.ran && j2k.used_gpu) { + eprintln!("j2k: required CUDA benchmark did not produce a GPU result"); + failed = true; + } + if nvidia_baseline_required() && !nvidia.ran { + eprintln!( + "NVIDIA baseline: required nvJPEG/nvJPEG2000 benchmark did not run: {}", + nv_status(nvidia) + ); + failed = true; + } + if failed { + std::process::exit(1); + } +} + +fn validate_j2k_cuda_dispatch(batch: &EncodedTranscodeBatch, encode: EncodeBenchMetrics) { + #[cfg(not(target_os = "macos"))] + { + validate_j2k_transform_dispatch(batch); + if !j2k_cuda_required() { + return; + } + assert_eq!( + batch.report.failed_tiles, 0, + "j2k: CUDA benchmark produced failed tiles under J2K_REQUIRE_CUDA_RUNTIME=1" + ); + let ht_code_block_dispatches = encode + .ht_code_block_dispatches + .saturating_add(batch.report.timings.dwt97_batch_ht_codeblock_dispatches); + assert!( + ht_code_block_dispatches != 0, + "j2k: CUDA HT encode dispatched zero code-block batches under J2K_REQUIRE_CUDA_RUNTIME=1" + ); + } + + #[cfg(target_os = "macos")] + let _ = (batch, encode); +} + +fn validate_j2k_transform_dispatch(batch: &EncodedTranscodeBatch) { + #[cfg(not(target_os = "macos"))] + { + if !j2k_cuda_required() { + return; + } + assert_eq!( + batch.report.failed_tiles, 0, + "j2k: CUDA transform benchmark produced failed tiles under J2K_REQUIRE_CUDA_RUNTIME=1" + ); + assert!( + batch.report.timings.accelerator_dispatches != 0, + "j2k: CUDA transform dispatched zero batches under J2K_REQUIRE_CUDA_RUNTIME=1" + ); + assert_eq!( + batch.report.timings.cpu_fallback_jobs, 0, + "j2k: CUDA transform used CPU fallback jobs under J2K_REQUIRE_CUDA_RUNTIME=1" + ); + assert_eq!( + batch.report.timings.accelerator_dispatched_jobs, + batch.report.transformed_components, + "j2k: CUDA transform dispatched jobs do not cover all transformed components under J2K_REQUIRE_CUDA_RUNTIME=1" + ); + if !j2k_cuda_stage_timings_disabled() { + assert!( + batch.report.timings.dwt97_batch_quantize_codeblock_us != 0, + "j2k: CUDA fused 9/7 code-block quantize stage was not timed under J2K_REQUIRE_CUDA_RUNTIME=1" + ); + } + } + + #[cfg(target_os = "macos")] + let _ = batch; +} + +#[derive(Default)] +struct NvidiaResult { + ran: bool, + best_wall_s: f64, + decode_ms: f64, + encode_ms: f64, + output_bytes: usize, + codestreams: Vec>, + error: Option, +} + +fn run_nvidia(jpegs: &[JpegInput], config: &BenchmarkConfig) -> NvidiaResult { + let mut session = match NvBaselineSession::new() { + Ok(session) => session, + Err(error) => { + return NvidiaResult { + error: Some(error), + ..NvidiaResult::default() + }; + } + }; + + // Warm up with the same reused session that will be measured. + for _ in 0..config.warmup { + for jpeg in jpegs { + let _ = session.transcode_jpeg_to_htj2k(&jpeg.bytes); + } + } + + let mut best_wall_s = f64::INFINITY; + let mut best = NvidiaResult::default(); + for _ in 0..config.iterations { + let start = Instant::now(); + let mut decode_ms = 0f64; + let mut encode_ms = 0f64; + let mut output_bytes = 0usize; + let mut codestreams = Vec::with_capacity(jpegs.len()); + let mut failed = None; + for jpeg in jpegs { + match session.transcode_jpeg_to_htj2k(&jpeg.bytes) { + Ok(result) => { + decode_ms += result.decode_ms; + encode_ms += result.encode_ms; + output_bytes += result.codestream.len(); + codestreams.push(result.codestream); + } + Err(error) => { + failed = Some(error); + break; + } + } + } + let elapsed = start.elapsed().as_secs_f64(); + if let Some(error) = failed { + return NvidiaResult { + error: Some(error), + ..NvidiaResult::default() + }; + } + if elapsed < best_wall_s { + best_wall_s = elapsed; + best = NvidiaResult { + ran: true, + best_wall_s: elapsed, + decode_ms, + encode_ms, + output_bytes, + codestreams, + error: None, + }; + } + } + best +} + +fn native_decode_rgb(codestream: &[u8]) -> Option> { + let image = Image::new(codestream, &DecodeSettings::default()).ok()?; + let bitmap = image.decode_native().ok()?; + (bitmap.num_components == 3 && bitmap.bytes_per_sample == 1).then_some(bitmap.data) +} + +#[allow(clippy::struct_field_names)] +#[derive(Clone)] +struct QualitySummary { + mean_psnr: Option, + aggregate_psnr: Option, + channel_psnr: [Option; 3], + per_tile_psnr: Vec>, +} + +#[derive(Clone, Copy, Debug)] +struct RgbMseSummary { + sum_sq: f64, + samples: usize, + channel_sum_sq: [f64; 3], + channel_samples: [usize; 3], + #[cfg(test)] + channel_psnr: [Option; 3], +} + +fn quality_summary(jpegs: &[JpegInput], codestreams: &[Vec]) -> Option { + (codestreams.len() == jpegs.len()).then_some(())?; + let mut total = 0f64; + let mut counted = 0usize; + let mut total_sum_sq = 0f64; + let mut total_samples = 0usize; + let mut channel_sum_sq = [0f64; 3]; + let mut channel_samples = [0usize; 3]; + let mut per_tile_psnr = Vec::with_capacity(jpegs.len()); + for (jpeg, codestream) in jpegs.iter().zip(codestreams.iter()) { + let Ok((source, _, _)) = nvidia_decode_jpeg_rgb(&jpeg.bytes) else { + return None; + }; + let recon = native_decode_rgb(codestream)?; + let (psnr, mse) = best_psnr_and_mse(&recon, &source)?; + total += psnr; + counted += 1; + total_sum_sq += mse.sum_sq; + total_samples = total_samples.saturating_add(mse.samples); + for channel in 0..3 { + channel_sum_sq[channel] += mse.channel_sum_sq[channel]; + channel_samples[channel] = + channel_samples[channel].saturating_add(mse.channel_samples[channel]); + } + per_tile_psnr.push(Some(psnr)); + } + (counted == jpegs.len()).then(|| QualitySummary { + mean_psnr: Some(total / counted as f64), + aggregate_psnr: psnr_from_mse(total_sum_sq, total_samples), + channel_psnr: [ + psnr_from_mse(channel_sum_sq[0], channel_samples[0]), + psnr_from_mse(channel_sum_sq[1], channel_samples[1]), + psnr_from_mse(channel_sum_sq[2], channel_samples[2]), + ], + per_tile_psnr, + }) +} + +fn best_psnr_and_mse(recon: &[u8], source_rgb: &[u8]) -> Option<(f64, RgbMseSummary)> { + let direct = psnr_u8(recon, source_rgb); + let converted_rgb = ycbcr_to_rgb_round_nearest(recon); + let converted = psnr_u8(&converted_rgb, source_rgb); + match (direct, converted) { + (Some(a), Some(b)) if a >= b => rgb_mse_summary(recon, source_rgb).map(|mse| (a, mse)), + (Some(_), Some(b)) => rgb_mse_summary(&converted_rgb, source_rgb).map(|mse| (b, mse)), + (Some(a), None) => rgb_mse_summary(recon, source_rgb).map(|mse| (a, mse)), + (None, Some(b)) => rgb_mse_summary(&converted_rgb, source_rgb).map(|mse| (b, mse)), + (None, None) => None, + } +} + +fn rgb_mse_summary(a: &[u8], b: &[u8]) -> Option { + if a.len() != b.len() || a.is_empty() || !a.len().is_multiple_of(3) { + return None; + } + let mut sum_sq = 0f64; + let mut channel_sum_sq = [0f64; 3]; + let mut channel_samples = [0usize; 3]; + for (left, right) in a.chunks_exact(3).zip(b.chunks_exact(3)) { + for channel in 0..3 { + let diff = f64::from(left[channel]) - f64::from(right[channel]); + let squared = diff * diff; + sum_sq += squared; + channel_sum_sq[channel] += squared; + channel_samples[channel] += 1; + } + } + Some(RgbMseSummary { + sum_sq, + samples: a.len(), + channel_sum_sq, + channel_samples, + #[cfg(test)] + channel_psnr: [ + psnr_from_mse(channel_sum_sq[0], channel_samples[0]), + psnr_from_mse(channel_sum_sq[1], channel_samples[1]), + psnr_from_mse(channel_sum_sq[2], channel_samples[2]), + ], + }) +} + +fn psnr_from_mse(sum_sq: f64, samples: usize) -> Option { + if samples == 0 { + return None; + } + if sum_sq == 0.0 { + return Some(f64::INFINITY); + } + let mse = sum_sq / samples as f64; + Some(10.0 * (255.0f64 * 255.0 / mse).log10()) +} + +#[allow( + clippy::similar_names, + clippy::too_many_arguments, + clippy::too_many_lines +)] +fn print_report( + jpegs: &[JpegInput], + megapixels: f64, + corpus_hash: u64, + config: &BenchmarkConfig, + rd_points: &[RdPoint], + selected_index: usize, + j2k_cpu_ht: &J2KResult, + j2k_cuda_ht: &J2KResult, + nvidia: &NvidiaResult, +) { + let labels: Vec<&str> = jpegs.iter().map(|j| j.label.as_str()).take(4).collect(); + println!( + "tiles: {}{}", + labels.join(", "), + if jpegs.len() > 4 { ", ..." } else { "" } + ); + println!("corpus hash: {corpus_hash:016x}"); + println!( + "lossy profile: selected scale {:.4}, decomposition levels {}, subband scales LL {:.3} HL {:.3} LH {:.3} HH {:.3}", + rd_points + .get(selected_index) + .map_or(JPEG_TO_HTJ2K_LOSSY_97_QUANTIZATION_SCALE, |point| point + .scale), + config.decomposition_levels, + config.subband_scales.low_low, + config.subband_scales.high_low, + config.subband_scales.low_high, + config.subband_scales.high_high, + ); + println!( + "iterations: {} (best wall-clock reported)\n", + config.iterations + ); + + if rd_points.len() > 1 || config.match_nvidia_bytes { + println!("J2K RD sweep (CUDA transform + CPU HT, PSNR not rate matched):"); + for (idx, point) in rd_points.iter().enumerate() { + let selected = if idx == selected_index { "*" } else { " " }; + let delta = if nvidia.ran { + format!( + "{:+.2}%", + byte_delta_fraction(point.result.output_bytes, nvidia.output_bytes) * 100.0 + ) + } else { + "n/a".to_string() + }; + println!( + " {selected} scale {:.4} bytes {} vs NVIDIA {} mean PSNR {} agg PSNR {} wall {:.3} ms", + point.scale, + point.result.output_bytes, + delta, + fmt_psnr(point.quality.as_ref().and_then(|q| q.mean_psnr)), + fmt_psnr(point.quality.as_ref().and_then(|q| q.aggregate_psnr)), + point.result.best_wall_s * 1000.0, + ); + } + println!(); + } + + println!( + "{:<24}{:>18}{:>18}{:>16}", + "metric", "sig xform+CPU HT", "sig xform+CUDA HT", "NVIDIA reused" + ); + println!("{}", "-".repeat(76)); + + // End-to-end throughput. + let sig_cpu_mps = if j2k_cpu_ht.ran && j2k_cpu_ht.best_wall_s > 0.0 { + megapixels / j2k_cpu_ht.best_wall_s + } else { + 0.0 + }; + let sig_cuda_mps = if j2k_cuda_ht.ran && j2k_cuda_ht.best_wall_s > 0.0 { + megapixels / j2k_cuda_ht.best_wall_s + } else { + 0.0 + }; + let nv_mps = if nvidia.ran && nvidia.best_wall_s > 0.0 { + megapixels / nvidia.best_wall_s + } else { + 0.0 + }; + println!( + "{:<24}{:>18}{:>18}{:>16}", + "end-to-end MP/s", + fmt_mps_ran(j2k_cpu_ht.ran, sig_cpu_mps), + fmt_mps_ran(j2k_cuda_ht.ran, sig_cuda_mps), + fmt_mps(nvidia, nv_mps), + ); + + // Wall-clock totals. + println!( + "{:<24}{:>18}{:>18}{:>16}", + "wall-clock (ms)", + fmt_ms(j2k_cpu_ht.ran, j2k_cpu_ht.best_wall_s * 1000.0), + fmt_ms(j2k_cuda_ht.ran, j2k_cuda_ht.best_wall_s * 1000.0), + fmt_ms(nvidia.ran, nvidia.best_wall_s * 1000.0), + ); + + // GPU-only. + let sig_cpu_gpu_ms = + (j2k_cpu_ht.transform_gpu_stage_us + j2k_cpu_ht.encode_cuda_stage_us) as f64 / 1000.0; + let sig_cuda_gpu_ms = + (j2k_cuda_ht.transform_gpu_stage_us + j2k_cuda_ht.encode_cuda_stage_us) as f64 / 1000.0; + let nv_gpu_ms = nvidia.decode_ms + nvidia.encode_ms; + println!( + "{:<24}{:>18}{:>18}{:>16}", + "GPU-only (ms)", + fmt_ms(j2k_cpu_ht.ran && j2k_cpu_ht.used_gpu, sig_cpu_gpu_ms), + fmt_ms(j2k_cuda_ht.ran && j2k_cuda_ht.used_gpu, sig_cuda_gpu_ms,), + fmt_ms(nvidia.ran, nv_gpu_ms), + ); + + // Per-stage breakdown. + println!("\nper-stage (ms):"); + print_j2k_stages("j2k CUDA transform + CPU HT encode", j2k_cpu_ht); + print_j2k_stages( + "j2k CUDA transform + CUDA HT block encode + CPU packetization", + j2k_cuda_ht, + ); + if nvidia.ran { + println!( + " NVIDIA reused-session serial: nvJPEG decode {:.3} nvJPEG2000 HT encode {:.3}", + nvidia.decode_ms, nvidia.encode_ms + ); + } else { + println!(" NVIDIA reused-session serial: {}", nv_status(nvidia)); + } + + // Output size. + println!("\noutput size + quality:"); + println!( + " bytes: sig CPU HT {} sig CUDA HT {} NVIDIA {}", + j2k_cpu_ht.output_bytes, + j2k_cuda_ht.output_bytes, + if nvidia.ran { + nvidia.output_bytes.to_string() + } else { + nv_status(nvidia) + }, + ); + + // PSNR vs the nvJPEG-decoded source (best-effort; needs the NVIDIA baseline). + let sig_cpu_quality = quality_summary(jpegs, &j2k_cpu_ht.codestreams); + let sig_cuda_quality = quality_summary(jpegs, &j2k_cuda_ht.codestreams); + let nv_quality = if nvidia.ran { + quality_summary(jpegs, &nvidia.codestreams) + } else { + None + }; + println!( + " mean PSNR vs source (dB, best color interp, not rate matched): sig CPU HT {} sig CUDA HT {} NVIDIA {}", + fmt_psnr(sig_cpu_quality.as_ref().and_then(|q| q.mean_psnr)), + fmt_psnr(sig_cuda_quality.as_ref().and_then(|q| q.mean_psnr)), + fmt_psnr(nv_quality.as_ref().and_then(|q| q.mean_psnr)), + ); + println!( + " aggregate PSNR vs source (dB, best color interp, not rate matched): sig CPU HT {} sig CUDA HT {} NVIDIA {}", + fmt_psnr(sig_cpu_quality.as_ref().and_then(|q| q.aggregate_psnr)), + fmt_psnr(sig_cuda_quality.as_ref().and_then(|q| q.aggregate_psnr)), + fmt_psnr(nv_quality.as_ref().and_then(|q| q.aggregate_psnr)), + ); + println!( + " RGB channel PSNR vs source (R/G/B): sig CPU HT {} sig CUDA HT {} NVIDIA {}", + fmt_channel_psnr(sig_cpu_quality.as_ref()), + fmt_channel_psnr(sig_cuda_quality.as_ref()), + fmt_channel_psnr(nv_quality.as_ref()), + ); +} + +fn print_j2k_cuda_profile_report( + jpegs: &[JpegInput], + megapixels: f64, + corpus_hash: u64, + config: &BenchmarkConfig, + scale: f32, + j2k: &J2KResult, +) { + let labels: Vec<&str> = jpegs.iter().map(|j| j.label.as_str()).take(4).collect(); + let mp_s = if j2k.ran && j2k.best_wall_s > 0.0 { + megapixels / j2k.best_wall_s + } else { + 0.0 + }; + println!( + "tiles: {}{}", + labels.join(", "), + if jpegs.len() > 4 { ", ..." } else { "" } + ); + println!("corpus hash: {corpus_hash:016x}"); + println!( + "profile: j2k CUDA HT only, scale {:.4}, decomposition levels {}, subband scales LL {:.3} HL {:.3} LH {:.3} HH {:.3}", + scale, + config.decomposition_levels, + config.subband_scales.low_low, + config.subband_scales.high_low, + config.subband_scales.low_high, + config.subband_scales.high_high, + ); + println!( + "iterations: {} (best wall-clock reported)\n", + config.iterations + ); + println!( + "PROFILE_RESULT j2k_cuda_ht_only mp_s={:.3} wall_ms={:.3} gpu_ms={:.3} bytes={} transform_dispatches={} transform_jobs={} cpu_fallback_jobs={} encode_dispatches={} ht_codeblock_dispatches={} packetization_dispatches={}", + mp_s, + j2k.best_wall_s * 1000.0, + result_gpu_ms( j2k), + j2k.output_bytes, + j2k.transform_dispatches, + j2k.transform_dispatched_jobs, + j2k.transform_cpu_fallback_jobs, + j2k.encode_dispatches, + j2k.encode_ht_code_block_dispatches, + j2k.encode_packetization_dispatches, + ); + print_j2k_stages( + "j2k CUDA transform + CUDA HT block encode + CPU packetization", + j2k, + ); +} + +fn print_j2k_stages(label: &str, j2k: &J2KResult) { + println!( + " {label}: extract {:.3} repack {:.3} transform wall {:.3} encode wall {:.3}", + us_ms(j2k.extract_us), + us_ms(j2k.repack_us), + us_ms(j2k.transform_wall_us), + us_ms(j2k.encode_wall_us), + ); + println!( + " GPU transform: pack/upload {:.3} idct+row {:.3} column {:.3} quantize {:.3} readback {:.3}", + us_ms( j2k.pack_upload_us), + us_ms( j2k.idct_row_lift_us), + us_ms( j2k.column_lift_us), + us_ms( j2k.quantize_us), + us_ms( j2k.readback_us), + ); + println!( + " transform dispatches: {} jobs: {} CPU fallback jobs: {}", + j2k.transform_dispatches, j2k.transform_dispatched_jobs, j2k.transform_cpu_fallback_jobs, + ); + if j2k.encode_dispatches > 0 { + println!( + " CUDA HT encode: total {:.3} dispatches {} HT code-block {} packetization {}", + us_ms(j2k.encode_cuda_stage_us), + j2k.encode_dispatches, + j2k.encode_ht_code_block_dispatches, + j2k.encode_packetization_dispatches, + ); + if j2k.encode_cuda_ht_kernel_us + + j2k.encode_cuda_ht_status_readback_us + + j2k.encode_cuda_ht_compact_us + + j2k.encode_cuda_ht_output_readback_us + > 0 + { + println!( + " resident split: kernel {:.3} status {:.3} compact {:.3} output {:.3}", + us_ms(j2k.encode_cuda_ht_kernel_us), + us_ms(j2k.encode_cuda_ht_status_readback_us), + us_ms(j2k.encode_cuda_ht_compact_us), + us_ms(j2k.encode_cuda_ht_output_readback_us), + ); + } + } else { + println!(" CUDA HT encode: n/a (CPU encode path)"); + } +} + +fn fmt_mps(nvidia: &NvidiaResult, mps: f64) -> String { + if nvidia.ran { + format!("{mps:>16.1}") + } else { + format!("{:>16}", nv_status(nvidia)) + } +} + +fn fmt_mps_ran(ran: bool, mps: f64) -> String { + if ran { + format!("{mps:.1}") + } else { + "n/a".to_string() + } +} + +fn fmt_ms(ran: bool, ms: f64) -> String { + if ran { + format!("{ms:.3}") + } else { + "n/a".to_string() + } +} + +fn us_ms(us: u128) -> f64 { + us as f64 / 1000.0 +} + +fn fmt_psnr(psnr: Option) -> String { + psnr.map_or_else(|| "n/a".to_string(), |value| format!("{value:.2}")) +} + +fn fmt_channel_psnr(quality: Option<&QualitySummary>) -> String { + quality.map_or_else( + || "n/a".to_string(), + |quality| { + format!( + "{}/{}/{}", + fmt_psnr(quality.channel_psnr[0]), + fmt_psnr(quality.channel_psnr[1]), + fmt_psnr(quality.channel_psnr[2]) + ) + }, + ) +} + +fn nv_status(nvidia: &NvidiaResult) -> String { + if nvidia.ran { + "ok".to_string() + } else { + match nvidia.error { + Some(NvBaselineError::NotBuilt) => "n/a (not built)".to_string(), + Some(NvBaselineError::Stage(code)) => format!("n/a (err {code})"), + None => "n/a".to_string(), + } + } +} + +fn result_gpu_ms(result: &J2KResult) -> f64 { + (result.transform_gpu_stage_us + result.encode_cuda_stage_us) as f64 / 1000.0 +} + +fn corpus_hash(jpegs: &[JpegInput]) -> u64 { + let mut hash = 0xcbf2_9ce4_8422_2325u64; + for jpeg in jpegs { + fnv1a_update(&mut hash, jpeg.label.as_bytes()); + fnv1a_update(&mut hash, &jpeg.width.to_le_bytes()); + fnv1a_update(&mut hash, &jpeg.height.to_le_bytes()); + fnv1a_update(&mut hash, &jpeg.bytes); + } + hash +} + +fn fnv1a_update(hash: &mut u64, bytes: &[u8]) { + for &byte in bytes { + *hash ^= u64::from(byte); + *hash = hash.wrapping_mul(0x0000_0100_0000_01B3); + } +} + +fn write_artifacts( + config: &BenchmarkConfig, + jpegs: &[JpegInput], + megapixels: f64, + rd_points: &[RdPoint], + selected_index: usize, + j2k_cuda_ht: &J2KResult, + nvidia: &NvidiaResult, +) -> std::io::Result<()> { + if let Some(path) = &config.csv_path { + write_text_artifact( + path, + csv_report( + jpegs, + megapixels, + rd_points, + selected_index, + j2k_cuda_ht, + nvidia, + ), + )?; + } + if let Some(path) = &config.json_path { + write_text_artifact( + path, + json_report( + config, + jpegs, + megapixels, + rd_points, + selected_index, + j2k_cuda_ht, + nvidia, + ), + )?; + } + Ok(()) +} + +fn write_j2k_profile_artifacts( + config: &BenchmarkConfig, + jpegs: &[JpegInput], + megapixels: f64, + scale: f32, + j2k: &J2KResult, +) -> std::io::Result<()> { + if let Some(path) = &config.csv_path { + write_text_artifact(path, j2k_profile_csv_report(jpegs, megapixels, scale, j2k))?; + } + if let Some(path) = &config.json_path { + write_text_artifact( + path, + j2k_profile_json_report(config, jpegs, megapixels, scale, j2k), + )?; + } + Ok(()) +} + +fn j2k_profile_csv_report( + jpegs: &[JpegInput], + megapixels: f64, + scale: f32, + j2k: &J2KResult, +) -> String { + let mp_s = if j2k.ran && j2k.best_wall_s > 0.0 { + megapixels / j2k.best_wall_s + } else { + 0.0 + }; + format!( + "row,ran,used_gpu,scale,tiles,megapixels,mp_s,bytes,wall_ms,gpu_ms,extract_ms,repack_ms,transform_wall_ms,encode_wall_ms,encode_cuda_ht_kernel_ms,encode_cuda_ht_status_readback_ms,encode_cuda_ht_compact_ms,encode_cuda_ht_output_readback_ms,pack_upload_ms,idct_row_lift_ms,column_lift_ms,quantize_ms,readback_ms,transform_dispatches,transform_jobs,cpu_fallback_jobs,encode_dispatches,ht_codeblock_dispatches,packetization_dispatches\nj2k_cuda_ht_only,{},{},{:.6},{},{:.8},{:.6},{},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{:.6},{},{},{},{},{},{}\n", + j2k.ran, + j2k.used_gpu, + scale, + jpegs.len(), + megapixels, + mp_s, + j2k.output_bytes, + j2k.best_wall_s * 1000.0, + result_gpu_ms( j2k), + us_ms( j2k.extract_us), + us_ms( j2k.repack_us), + us_ms( j2k.transform_wall_us), + us_ms( j2k.encode_wall_us), + us_ms( j2k.encode_cuda_ht_kernel_us), + us_ms( j2k.encode_cuda_ht_status_readback_us), + us_ms( j2k.encode_cuda_ht_compact_us), + us_ms( j2k.encode_cuda_ht_output_readback_us), + us_ms( j2k.pack_upload_us), + us_ms( j2k.idct_row_lift_us), + us_ms( j2k.column_lift_us), + us_ms( j2k.quantize_us), + us_ms( j2k.readback_us), + j2k.transform_dispatches, + j2k.transform_dispatched_jobs, + j2k.transform_cpu_fallback_jobs, + j2k.encode_dispatches, + j2k.encode_ht_code_block_dispatches, + j2k.encode_packetization_dispatches, + ) +} + +#[allow(clippy::format_push_string)] +fn j2k_profile_json_report( + config: &BenchmarkConfig, + jpegs: &[JpegInput], + megapixels: f64, + scale: f32, + j2k: &J2KResult, +) -> String { + let mp_s = if j2k.ran && j2k.best_wall_s > 0.0 { + megapixels / j2k.best_wall_s + } else { + 0.0 + }; + let mut out = String::new(); + out.push_str("{\n"); + out.push_str(" \"mode\": \"j2k_cuda_ht_only\",\n"); + out.push_str(&format!(" \"tile_count\": {},\n", jpegs.len())); + out.push_str(&format!(" \"megapixels\": {megapixels:.8},\n")); + out.push_str(&format!( + " \"corpus_hash\": \"{:016x}\",\n", + corpus_hash(jpegs) + )); + out.push_str(&format!(" \"scale\": {scale:.6},\n")); + out.push_str(&format!( + " \"decomposition_levels\": {},\n", + config.decomposition_levels + )); + out.push_str(&format!( + " \"subband_scales\": {{\"ll\": {:.6}, \"hl\": {:.6}, \"lh\": {:.6}, \"hh\": {:.6}}},\n", + config.subband_scales.low_low, + config.subband_scales.high_low, + config.subband_scales.low_high, + config.subband_scales.high_high, + )); + out.push_str(" \"inputs\": ["); + for (idx, jpeg) in jpegs.iter().enumerate() { + if idx != 0 { + out.push_str(", "); + } + out.push_str(&format!( + "{{\"label\": \"{}\", \"width\": {}, \"height\": {}, \"bytes\": {}}}", + json_escape(&jpeg.label), + jpeg.width, + jpeg.height, + jpeg.bytes.len() + )); + } + out.push_str("],\n"); + out.push_str(&format!( + " \"result\": {{\"ran\": {}, \"used_gpu\": {}, \"mp_s\": {:.6}, \"bytes\": {}, \"wall_ms\": {:.6}, \"gpu_ms\": {:.6}, \"extract_ms\": {:.6}, \"repack_ms\": {:.6}, \"transform_wall_ms\": {:.6}, \"encode_wall_ms\": {:.6}, \"encode_cuda_ht_kernel_ms\": {:.6}, \"encode_cuda_ht_status_readback_ms\": {:.6}, \"encode_cuda_ht_compact_ms\": {:.6}, \"encode_cuda_ht_output_readback_ms\": {:.6}, \"pack_upload_ms\": {:.6}, \"idct_row_lift_ms\": {:.6}, \"column_lift_ms\": {:.6}, \"quantize_ms\": {:.6}, \"readback_ms\": {:.6}, \"transform_dispatches\": {}, \"transform_jobs\": {}, \"cpu_fallback_jobs\": {}, \"encode_dispatches\": {}, \"ht_codeblock_dispatches\": {}, \"packetization_dispatches\": {}}}\n", + j2k.ran, + j2k.used_gpu, + mp_s, + j2k.output_bytes, + j2k.best_wall_s * 1000.0, + result_gpu_ms( j2k), + us_ms( j2k.extract_us), + us_ms( j2k.repack_us), + us_ms( j2k.transform_wall_us), + us_ms( j2k.encode_wall_us), + us_ms( j2k.encode_cuda_ht_kernel_us), + us_ms( j2k.encode_cuda_ht_status_readback_us), + us_ms( j2k.encode_cuda_ht_compact_us), + us_ms( j2k.encode_cuda_ht_output_readback_us), + us_ms( j2k.pack_upload_us), + us_ms( j2k.idct_row_lift_us), + us_ms( j2k.column_lift_us), + us_ms( j2k.quantize_us), + us_ms( j2k.readback_us), + j2k.transform_dispatches, + j2k.transform_dispatched_jobs, + j2k.transform_cpu_fallback_jobs, + j2k.encode_dispatches, + j2k.encode_ht_code_block_dispatches, + j2k.encode_packetization_dispatches, + )); + out.push_str("}\n"); + out +} + +#[allow(clippy::format_push_string)] +fn csv_report( + jpegs: &[JpegInput], + megapixels: f64, + rd_points: &[RdPoint], + selected_index: usize, + j2k_cuda_ht: &J2KResult, + nvidia: &NvidiaResult, +) -> String { + let mut out = String::from( + "row,selected,ran,used_gpu,nvidia_ran,nvidia_status,scale,bytes,byte_delta_vs_nvidia,wall_ms,gpu_ms,mean_psnr,aggregate_psnr,psnr_r,psnr_g,psnr_b,transform_dispatches,transform_jobs,cpu_fallback_jobs,encode_dispatches,ht_codeblock_dispatches,packetization_dispatches\n", + ); + for (idx, point) in rd_points.iter().enumerate() { + append_csv_result( + &mut out, + "j2k_cuda_transform_cpu_ht", + idx == selected_index, + Some(point.scale), + &point.result, + point.quality.as_ref(), + nvidia, + ); + } + let cuda_quality = quality_summary(jpegs, &j2k_cuda_ht.codestreams); + append_csv_result( + &mut out, + "j2k_cuda_transform_cuda_ht_block_cpu_packet", + false, + rd_points.get(selected_index).map(|point| point.scale), + j2k_cuda_ht, + cuda_quality.as_ref(), + nvidia, + ); + let nvidia_wall = nvidia + .ran + .then(|| format!("{:.6}", nvidia.best_wall_s * 1000.0)); + let nvidia_gpu = nvidia + .ran + .then(|| format!("{:.6}", nvidia.decode_ms + nvidia.encode_ms)); + let nvidia_quality = if nvidia.ran { + quality_summary(jpegs, &nvidia.codestreams) + } else { + None + }; + let _ = megapixels; + out.push_str(&format!( + "nvidia_reused_session_serial,false,{},{},{},{},,{},{},{},{},{},{},{},{},{},,,,,,\n", + nvidia.ran, + nvidia.ran, + nvidia.ran, + csv_escape(&nv_status(nvidia)), + if nvidia.ran { + nvidia.output_bytes.to_string() + } else { + String::new() + }, + "", + nvidia_wall.unwrap_or_default(), + nvidia_gpu.unwrap_or_default(), + csv_optional_f64( + nvidia_quality + .as_ref() + .and_then(|quality| quality.mean_psnr) + ), + csv_optional_f64( + nvidia_quality + .as_ref() + .and_then(|quality| quality.aggregate_psnr) + ), + csv_optional_f64( + nvidia_quality + .as_ref() + .and_then(|quality| quality.channel_psnr[0]) + ), + csv_optional_f64( + nvidia_quality + .as_ref() + .and_then(|quality| quality.channel_psnr[1]) + ), + csv_optional_f64( + nvidia_quality + .as_ref() + .and_then(|quality| quality.channel_psnr[2]) + ), + )); + out +} + +#[allow(clippy::format_push_string)] +fn append_csv_result( + out: &mut String, + row: &str, + selected: bool, + scale: Option, + result: &J2KResult, + quality: Option<&QualitySummary>, + nvidia: &NvidiaResult, +) { + let byte_delta = if nvidia.ran { + Some(byte_delta_fraction( + result.output_bytes, + nvidia.output_bytes, + )) + } else { + None + }; + out.push_str(&format!( + "{row},{selected},{},{},{},{},{},{},{},{:.6},{:.6},{},{},{},{},{},{},{},{},{},{},{}\n", + result.ran, + result.used_gpu, + nvidia.ran, + csv_escape(&nv_status(nvidia)), + scale.map_or_else(String::new, |scale| format!("{scale:.6}")), + result.output_bytes, + byte_delta.map_or_else(String::new, |delta| format!("{delta:.8}")), + result.best_wall_s * 1000.0, + result_gpu_ms(result), + csv_optional_f64(quality.and_then(|quality| quality.mean_psnr)), + csv_optional_f64(quality.and_then(|quality| quality.aggregate_psnr)), + csv_optional_f64(quality.and_then(|quality| quality.channel_psnr[0])), + csv_optional_f64(quality.and_then(|quality| quality.channel_psnr[1])), + csv_optional_f64(quality.and_then(|quality| quality.channel_psnr[2])), + result.transform_dispatches, + result.transform_dispatched_jobs, + result.transform_cpu_fallback_jobs, + result.encode_dispatches, + result.encode_ht_code_block_dispatches, + result.encode_packetization_dispatches, + )); +} + +#[allow(clippy::format_push_string)] +fn json_report( + config: &BenchmarkConfig, + jpegs: &[JpegInput], + megapixels: f64, + rd_points: &[RdPoint], + selected_index: usize, + j2k_cuda_ht: &J2KResult, + nvidia: &NvidiaResult, +) -> String { + let mut out = String::new(); + out.push_str("{\n"); + out.push_str(&format!(" \"tile_count\": {},\n", jpegs.len())); + out.push_str(&format!(" \"megapixels\": {megapixels:.8},\n")); + out.push_str(&format!( + " \"corpus_hash\": \"{:016x}\",\n", + corpus_hash(jpegs) + )); + out.push_str(&format!( + " \"match_nvidia_bytes\": {},\n \"match_tolerance\": {:.8},\n", + config.match_nvidia_bytes, config.match_tolerance + )); + out.push_str(&format!( + " \"decomposition_levels\": {},\n", + config.decomposition_levels + )); + out.push_str(&format!( + " \"subband_scales\": {{\"ll\": {:.6}, \"hl\": {:.6}, \"lh\": {:.6}, \"hh\": {:.6}}},\n", + config.subband_scales.low_low, + config.subband_scales.high_low, + config.subband_scales.low_high, + config.subband_scales.high_high, + )); + out.push_str(" \"inputs\": ["); + for (idx, jpeg) in jpegs.iter().enumerate() { + if idx != 0 { + out.push_str(", "); + } + out.push_str(&format!( + "{{\"label\": \"{}\", \"width\": {}, \"height\": {}, \"bytes\": {}}}", + json_escape(&jpeg.label), + jpeg.width, + jpeg.height, + jpeg.bytes.len() + )); + } + out.push_str("],\n"); + out.push_str(&format!(" \"selected_rd_index\": {selected_index},\n")); + out.push_str(" \"rd_points\": [\n"); + for (idx, point) in rd_points.iter().enumerate() { + if idx != 0 { + out.push_str(",\n"); + } + out.push_str(" "); + append_json_j2k_result( + &mut out, + point.scale, + &point.result, + point.quality.as_ref(), + nvidia, + ); + } + out.push_str("\n ],\n"); + out.push_str(" \"j2k_cuda_ht_experimental\": "); + let cuda_quality = quality_summary(jpegs, &j2k_cuda_ht.codestreams); + append_json_j2k_result( + &mut out, + rd_points + .get(selected_index) + .map_or(JPEG_TO_HTJ2K_LOSSY_97_QUANTIZATION_SCALE, |point| { + point.scale + }), + j2k_cuda_ht, + cuda_quality.as_ref(), + nvidia, + ); + out.push_str(",\n"); + let nvidia_quality = if nvidia.ran { + quality_summary(jpegs, &nvidia.codestreams) + } else { + None + }; + out.push_str(&format!( + " \"nvidia_reused_session_serial\": {{\"ran\": {}, \"status\": \"{}\", \"bytes\": {}, \"wall_ms\": {:.6}, \"gpu_ms\": {:.6}, \"decode_ms\": {:.6}, \"encode_ms\": {:.6}, \"mean_psnr\": {}, \"aggregate_psnr\": {}, \"channel_psnr\": {{\"r\": {}, \"g\": {}, \"b\": {}}}}}\n", + nvidia.ran, + json_escape(&nv_status(nvidia)), + nvidia.output_bytes, + nvidia.best_wall_s * 1000.0, + if nvidia.ran { nvidia.decode_ms + nvidia.encode_ms } else { 0.0 }, + nvidia.decode_ms, + nvidia.encode_ms, + json_optional_f64(nvidia_quality.as_ref().and_then(|quality| quality.mean_psnr)), + json_optional_f64(nvidia_quality.as_ref().and_then(|quality| quality.aggregate_psnr)), + json_optional_f64(nvidia_quality.as_ref().and_then(|quality| quality.channel_psnr[0])), + json_optional_f64(nvidia_quality.as_ref().and_then(|quality| quality.channel_psnr[1])), + json_optional_f64(nvidia_quality.as_ref().and_then(|quality| quality.channel_psnr[2])), + )); + out.push_str("}\n"); + out +} + +#[allow(clippy::format_push_string)] +fn append_json_j2k_result( + out: &mut String, + scale: f32, + result: &J2KResult, + quality: Option<&QualitySummary>, + nvidia: &NvidiaResult, +) { + out.push_str(&format!( + "{{\"scale\": {:.6}, \"ran\": {}, \"used_gpu\": {}, \"bytes\": {}, \"byte_delta_vs_nvidia\": {:.8}, \"wall_ms\": {:.6}, \"gpu_ms\": {:.6}, \"mean_psnr\": {}, \"aggregate_psnr\": {}, \"channel_psnr\": {{\"r\": {}, \"g\": {}, \"b\": {}}}, \"transform_dispatches\": {}, \"transform_jobs\": {}, \"cpu_fallback_jobs\": {}, \"encode_dispatches\": {}, \"ht_codeblock_dispatches\": {}, \"packetization_dispatches\": {}, \"per_tile_psnr\": [", + scale, + result.ran, + result.used_gpu, + result.output_bytes, + if nvidia.ran { byte_delta_fraction(result.output_bytes, nvidia.output_bytes) } else { 0.0 }, + result.best_wall_s * 1000.0, + result_gpu_ms(result), + json_optional_f64(quality.and_then(|quality| quality.mean_psnr)), + json_optional_f64(quality.and_then(|quality| quality.aggregate_psnr)), + json_optional_f64(quality.and_then(|quality| quality.channel_psnr[0])), + json_optional_f64(quality.and_then(|quality| quality.channel_psnr[1])), + json_optional_f64(quality.and_then(|quality| quality.channel_psnr[2])), + result.transform_dispatches, + result.transform_dispatched_jobs, + result.transform_cpu_fallback_jobs, + result.encode_dispatches, + result.encode_ht_code_block_dispatches, + result.encode_packetization_dispatches, + )); + if let Some(quality) = quality { + for (idx, psnr) in quality.per_tile_psnr.iter().enumerate() { + if idx != 0 { + out.push_str(", "); + } + out.push_str(&json_optional_f64(*psnr)); + } + } + out.push_str("]}"); +} + +fn json_optional_f64(value: Option) -> String { + json_f64_or_null(value, 8) +} + +fn csv_optional_f64(value: Option) -> String { + csv_f64_or_blank(value, 6) +} + +#[cfg(test)] +#[allow(clippy::float_cmp)] +mod tests { + use super::*; + + fn path(value: &str) -> PathBuf { + PathBuf::from(value) + } + + #[test] + fn parse_benchmark_config_accepts_rd_and_artifact_flags() { + let error = BenchmarkConfig::parse([ + path("--quant-scales"), + path("1.5,1.9,2.3"), + path("--subband-scales"), + path("0.9,1.0,1.1,1.3"), + path("--match-nvidia-bytes"), + path("--profile-j2k-cuda-only"), + ]) + .expect_err("profile-only mode rejects NVIDIA byte matching"); + assert!(error.contains("--profile-j2k-cuda-only")); + + let config = BenchmarkConfig::parse([ + path("--quant-scales"), + path("1.5,1.9,2.3"), + path("--subband-scales"), + path("0.9,1.0,1.1,1.3"), + path("--decomposition-levels"), + path("3"), + path("--profile-j2k-cuda-only"), + path("--match-tolerance"), + path("0.015"), + path("--warmup"), + path("3"), + path("--iterations"), + path("75"), + path("--min-tiles"), + path("100"), + path("--json"), + path("target/report.json"), + path("--csv"), + path("target/report.csv"), + path("a.jpg"), + ]) + .expect("config parses"); + + assert_eq!(config.quant_scales, vec![1.5, 1.9, 2.3]); + assert_eq!(config.subband_scales.low_low.to_bits(), 0.9f32.to_bits()); + assert_eq!(config.subband_scales.low_high.to_bits(), 1.1f32.to_bits()); + assert_eq!(config.decomposition_levels, 3); + assert!(!config.match_nvidia_bytes); + assert!(config.profile_j2k_cuda_only); + assert_eq!(config.match_tolerance, 0.015); + assert_eq!(config.warmup, 3); + assert_eq!(config.iterations, 75); + assert_eq!(config.min_tiles, Some(100)); + assert_eq!(config.json_path, Some(path("target/report.json"))); + assert_eq!(config.csv_path, Some(path("target/report.csv"))); + assert_eq!(config.input_paths, vec![path("a.jpg")]); + } + + #[test] + fn select_rd_point_chooses_closest_nvidia_bytes() { + let config = BenchmarkConfig { + match_nvidia_bytes: true, + ..BenchmarkConfig::default() + }; + let points = vec![ + RdPoint { + scale: 1.5, + result: J2KResult { + output_bytes: 900, + ..J2KResult::default() + }, + quality: None, + }, + RdPoint { + scale: 1.9, + result: J2KResult { + output_bytes: 1010, + ..J2KResult::default() + }, + quality: None, + }, + ]; + let nvidia = NvidiaResult { + ran: true, + output_bytes: 1000, + ..NvidiaResult::default() + }; + + assert_eq!(select_rd_point(&points, &config, &nvidia), 1); + } + + #[test] + fn validate_rate_match_rejects_selected_rd_point_outside_tolerance() { + let config = BenchmarkConfig { + match_nvidia_bytes: true, + match_tolerance: 0.02, + ..BenchmarkConfig::default() + }; + let nvidia = NvidiaResult { + ran: true, + output_bytes: 1_000, + ..NvidiaResult::default() + }; + let selected = J2KResult { + ran: true, + output_bytes: 900, + ..J2KResult::default() + }; + let cuda_ht = J2KResult { + ran: true, + used_gpu: true, + output_bytes: 1_000, + ..J2KResult::default() + }; + + let error = validate_rate_match(&config, &selected, &cuda_ht, &nvidia) + .expect_err("selected RD point outside tolerance must fail"); + assert!(error.contains("selected J2K RD point")); + } + + #[test] + fn validate_rate_match_rejects_cuda_ht_row_outside_tolerance() { + let config = BenchmarkConfig { + match_nvidia_bytes: true, + match_tolerance: 0.02, + ..BenchmarkConfig::default() + }; + let nvidia = NvidiaResult { + ran: true, + output_bytes: 1_000, + ..NvidiaResult::default() + }; + let selected = J2KResult { + ran: true, + output_bytes: 1_000, + ..J2KResult::default() + }; + let cuda_ht = J2KResult { + ran: true, + used_gpu: true, + output_bytes: 950, + ..J2KResult::default() + }; + + let error = validate_rate_match(&config, &selected, &cuda_ht, &nvidia) + .expect_err("CUDA HT row outside tolerance must fail"); + assert!(error.contains("J2K CUDA HT row")); + } + + #[test] + fn json_optional_f64_emits_null_for_non_finite_values() { + assert_eq!(json_optional_f64(Some(f64::INFINITY)), "null"); + assert_eq!(json_optional_f64(Some(f64::NEG_INFINITY)), "null"); + assert_eq!(json_optional_f64(Some(f64::NAN)), "null"); + assert_eq!(json_optional_f64(Some(42.25)), "42.25000000"); + } + + #[test] + fn csv_report_marks_missing_nvidia_status_without_zero_delta() { + let point = RdPoint { + scale: 1.9, + result: J2KResult { + ran: true, + used_gpu: true, + output_bytes: 123, + ..J2KResult::default() + }, + quality: None, + }; + let csv = csv_report( + &[], + 0.0, + &[point], + 0, + &J2KResult::default(), + &NvidiaResult { + error: Some(NvBaselineError::NotBuilt), + ..NvidiaResult::default() + }, + ); + + assert!(csv.starts_with( + "row,selected,ran,used_gpu,nvidia_ran,nvidia_status,scale,bytes,byte_delta_vs_nvidia" + )); + assert!(csv.contains( + "j2k_cuda_transform_cpu_ht,true,true,true,false,n/a (not built),1.900000,123," + )); + assert!(!csv.contains(",123,0.00000000,")); + } + + #[test] + fn csv_report_marks_successful_nvidia_status_as_ok() { + let csv = csv_report( + &[], + 0.0, + &[], + 0, + &J2KResult::default(), + &NvidiaResult { + ran: true, + output_bytes: 623, + ..NvidiaResult::default() + }, + ); + + assert!(csv.contains("nvidia_reused_session_serial,false,true,true,true,ok,,623,")); + } + + #[test] + fn rgb_mse_summary_reports_per_channel_error() { + let summary = rgb_mse_summary(&[10, 20, 30, 20, 40, 70], &[10, 30, 50, 30, 40, 40]) + .expect("matching RGB buffers produce channel stats"); + + assert_eq!(summary.sum_sq, 1500.0); + assert_eq!(summary.samples, 6); + assert_eq!(summary.channel_sum_sq, [100.0, 100.0, 1300.0]); + assert_eq!(summary.channel_samples, [2, 2, 2]); + assert_eq!(summary.channel_psnr[0], psnr_from_mse(100.0, 2)); + assert_eq!(summary.channel_psnr[1], psnr_from_mse(100.0, 2)); + assert_eq!(summary.channel_psnr[2], psnr_from_mse(1300.0, 2)); + } + + #[test] + fn csv_report_includes_rgb_channel_psnr_columns() { + let csv = csv_report( + &[], + 0.0, + &[], + 0, + &J2KResult::default(), + &NvidiaResult::default(), + ); + + assert!(csv.starts_with( + "row,selected,ran,used_gpu,nvidia_ran,nvidia_status,scale,bytes,byte_delta_vs_nvidia,wall_ms,gpu_ms,mean_psnr,aggregate_psnr,psnr_r,psnr_g,psnr_b," + )); + } + + #[test] + fn json_report_includes_rgb_channel_psnr_fields() { + let json = json_report( + &BenchmarkConfig::default(), + &[], + 0.0, + &[], + 0, + &J2KResult::default(), + &NvidiaResult::default(), + ); + + assert!(json.contains("\"channel_psnr\": {\"r\": null, \"g\": null, \"b\": null}")); + } +} diff --git a/tests/nvidia-baseline/src/lib.rs b/tests/nvidia-baseline/src/lib.rs new file mode 100644 index 00000000..9721e6b9 --- /dev/null +++ b/tests/nvidia-baseline/src/lib.rs @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! NVIDIA GPU codec baseline (nvJPEG + nvJPEG2000) for benchmarking j2k's +//! coefficient-domain JPEG → HTJ2K transcode against the conventional +//! decode-to-pixels-then-encode pipeline. +//! +//! The intricate codec orchestration lives in `cuda/nv_baseline.cu`; this module +//! is a thin FFI plus safe wrappers. The FFI is only present when `build.rs` +//! compiled and linked the C++ helper (cfg `nvbaseline_built`) — i.e. on a host +//! with `nvcc`, `libnvjpeg`, and a separately-installed `libnvjpeg2k`. Elsewhere +//! the wrappers report "unavailable" so the workspace still builds. + +#[cfg(nvbaseline_built)] +mod ffi { + use std::os::raw::{c_int, c_uchar}; + + #[repr(C)] + pub struct NvbSession { + _private: [u8; 0], + } + + extern "C" { + pub fn nvb_available() -> c_int; + + pub fn nvb_session_create(out: *mut *mut NvbSession) -> c_int; + + pub fn nvb_session_destroy(session: *mut NvbSession); + + pub fn nvb_session_transcode_jpeg_to_htj2k( + session: *mut NvbSession, + jpeg: *const c_uchar, + jpeg_len: usize, + out: *mut c_uchar, + out_cap: usize, + out_len: *mut usize, + decode_ms: *mut f64, + encode_ms: *mut f64, + width: *mut c_int, + height: *mut c_int, + num_components: *mut c_int, + ) -> c_int; + + pub fn nvb_session_decode_j2k_interleaved( + session: *mut NvbSession, + j2k: *const c_uchar, + j2k_len: usize, + requested_format: c_int, + out: *mut c_uchar, + out_cap: usize, + out_len: *mut usize, + decode_ms: *mut f64, + width: *mut c_int, + height: *mut c_int, + num_components: *mut c_int, + bit_depth: *mut c_int, + bytes_per_sample: *mut c_int, + ) -> c_int; + + pub fn nvb_decode_jpeg_rgb( + jpeg: *const c_uchar, + jpeg_len: usize, + out_rgb: *mut c_uchar, + out_cap: usize, + width: *mut c_int, + height: *mut c_int, + ) -> c_int; + + pub fn nvb_session_decode_jpeg_rgb_interleaved_timed( + session: *mut NvbSession, + jpeg: *const c_uchar, + jpeg_len: usize, + decode_ms: *mut f64, + width: *mut c_int, + height: *mut c_int, + ) -> c_int; + } +} + +/// Output pixel layout requested from direct nvJPEG2000 decode. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NvJ2kDecodeFormat { + /// Interleaved 8-bit RGB output. + Rgb8, + /// 8-bit grayscale output. + Gray8, +} + +impl NvJ2kDecodeFormat { + #[cfg(nvbaseline_built)] + const fn ffi_code(self) -> i32 { + match self { + Self::Rgb8 => 1, + Self::Gray8 => 2, + } + } + + #[cfg(nvbaseline_built)] + const fn bytes_per_pixel(self) -> usize { + match self { + Self::Rgb8 => 3, + Self::Gray8 => 1, + } + } +} + +/// Direct nvJPEG2000 decode result. +#[derive(Debug, Clone)] +pub struct NvJ2kDecodeResult { + /// Interleaved decoded host pixels in the requested format. + pub pixels: Vec, + /// nvJPEG2000 GPU decode time, milliseconds. + pub decode_ms: f64, + /// Decoded image width. + pub width: u32, + /// Decoded image height. + pub height: u32, + /// Component count in `pixels`. + pub num_components: u32, + /// Source codestream component bit depth. + pub bit_depth: u32, + /// Output bytes per sample. + pub bytes_per_sample: u32, +} + +/// One NVIDIA transcode result: the HTJ2K codestream plus GPU stage timings. +#[derive(Debug, Clone)] +pub struct NvTranscodeResult { + /// The produced HTJ2K codestream. + pub codestream: Vec, + /// nvJPEG decode (JPEG → planar RGB) GPU time, milliseconds. + pub decode_ms: f64, + /// nvJPEG2000 HT encode (RGB → HTJ2K) GPU time, milliseconds. + pub encode_ms: f64, + /// Decoded image width. + pub width: u32, + /// Decoded image height. + pub height: u32, + /// Component count (3 for the RGB pipeline). + pub num_components: u32, +} + +/// One reused-session NVIDIA JPEG decode timing. +#[derive(Debug, Clone, Copy)] +pub struct NvJpegDecodeTiming { + /// nvJPEG device-resident RGBI decode time, milliseconds. + pub decode_ms: f64, + /// Decoded image width. + pub width: u32, + /// Decoded image height. + pub height: u32, +} + +/// Reusable NVIDIA baseline session. +/// +/// This keeps nvJPEG/nvJPEG2000 handles, CUDA stream/events, encode state, and +/// reusable device RGB planes alive across tile transcodes. +pub struct NvBaselineSession { + #[cfg(nvbaseline_built)] + raw: std::ptr::NonNull, +} + +impl NvBaselineSession { + /// Create a reusable nvJPEG + nvJPEG2000 transcode session. + pub fn new() -> Result { + #[cfg(not(nvbaseline_built))] + { + Err(NvBaselineError::NotBuilt) + } + #[cfg(nvbaseline_built)] + { + let mut raw = std::ptr::null_mut(); + // SAFETY: `raw` is a valid out pointer. On success the C side + // returns an owned session pointer that `Drop` releases. + let rc = unsafe { ffi::nvb_session_create(std::ptr::addr_of_mut!(raw)) }; + if rc != 0 { + return Err(NvBaselineError::Stage(rc)); + } + let raw = std::ptr::NonNull::new(raw).ok_or(NvBaselineError::Stage(900))?; + Ok(Self { raw }) + } + } + + /// Transcode one JPEG to HTJ2K using reused session resources. + pub fn transcode_jpeg_to_htj2k( + &mut self, + jpeg: &[u8], + ) -> Result { + #[cfg(not(nvbaseline_built))] + { + let _ = jpeg; + Err(NvBaselineError::NotBuilt) + } + #[cfg(nvbaseline_built)] + { + nvidia_transcode_with_session(self.raw.as_ptr(), jpeg) + } + } + + /// Decode one JPEG to device-resident interleaved RGB and return the nvJPEG event time. + pub fn decode_jpeg_rgb_interleaved_timed( + &mut self, + jpeg: &[u8], + ) -> Result { + #[cfg(not(nvbaseline_built))] + { + let _ = jpeg; + Err(NvBaselineError::NotBuilt) + } + #[cfg(nvbaseline_built)] + { + let mut decode_ms = 0f64; + let mut width = 0i32; + let mut height = 0i32; + // SAFETY: `self.raw` owns a live NVIDIA baseline session and + // output pointers refer to initialized stack locals. + let rc = unsafe { + ffi::nvb_session_decode_jpeg_rgb_interleaved_timed( + self.raw.as_ptr(), + jpeg.as_ptr(), + jpeg.len(), + std::ptr::addr_of_mut!(decode_ms), + std::ptr::addr_of_mut!(width), + std::ptr::addr_of_mut!(height), + ) + }; + if rc != 0 { + return Err(NvBaselineError::Stage(rc)); + } + Ok(NvJpegDecodeTiming { + decode_ms, + width: width as u32, + height: height as u32, + }) + } + } + + /// Decode one JPEG 2000 / HTJ2K codestream to interleaved host pixels. + pub fn decode_j2k_interleaved( + &mut self, + codestream: &[u8], + format: NvJ2kDecodeFormat, + ) -> Result { + #[cfg(not(nvbaseline_built))] + { + let _ = (codestream, format); + Err(NvBaselineError::NotBuilt) + } + #[cfg(nvbaseline_built)] + { + nvidia_decode_j2k_with_session(self.raw.as_ptr(), codestream, format) + } + } +} + +#[cfg(nvbaseline_built)] +impl Drop for NvBaselineSession { + fn drop(&mut self) { + // SAFETY: `raw` is owned by this wrapper and is destroyed exactly once. + unsafe { ffi::nvb_session_destroy(self.raw.as_ptr()) }; + } +} + +/// Whether the NVIDIA baseline is compiled in and the codec handles initialize. +#[must_use] +pub fn nvidia_baseline_available() -> bool { + #[cfg(nvbaseline_built)] + { + // SAFETY: `nvb_available` takes no arguments and only creates/destroys + // codec handles to confirm the libraries load. + unsafe { ffi::nvb_available() == 1 } + } + #[cfg(not(nvbaseline_built))] + { + false + } +} + +/// Whether direct nvJPEG2000 decode is compiled in and initializes. +#[must_use] +pub fn nvidia_j2k_decode_available() -> bool { + nvidia_baseline_available() +} + +/// Why the NVIDIA baseline could not run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NvBaselineError { + /// The C++ baseline was not compiled into this build (no nvcc / libraries). + NotBuilt, + /// The C++ pipeline returned a non-zero stage code. + Stage(i32), +} + +/// Reference-decode a JPEG to interleaved RGB (untimed), for PSNR comparison. +pub fn nvidia_decode_jpeg_rgb(jpeg: &[u8]) -> Result<(Vec, u32, u32), NvBaselineError> { + #[cfg(not(nvbaseline_built))] + { + let _ = jpeg; + Err(NvBaselineError::NotBuilt) + } + #[cfg(nvbaseline_built)] + { + let mut width = 0i32; + let mut height = 0i32; + // Size the host RGB buffer from the JPEG SOF header without creating + // extra codec handles. + let info = jpeg_dimensions_from_header(jpeg)?; + let mut out = vec![0u8; (info.0 as usize) * (info.1 as usize) * 3]; + // SAFETY: `out` is sized to width*height*3 and `jpeg` is a valid slice. + let rc = unsafe { + ffi::nvb_decode_jpeg_rgb( + jpeg.as_ptr(), + jpeg.len(), + out.as_mut_ptr(), + out.len(), + std::ptr::addr_of_mut!(width), + std::ptr::addr_of_mut!(height), + ) + }; + if rc != 0 { + return Err(NvBaselineError::Stage(rc)); + } + Ok((out, width as u32, height as u32)) + } +} + +/// Transcode a JPEG to HTJ2K via nvJPEG decode + nvJPEG2000 HT encode. +pub fn nvidia_transcode_jpeg_to_htj2k(jpeg: &[u8]) -> Result { + #[cfg(not(nvbaseline_built))] + { + let _ = jpeg; + Err(NvBaselineError::NotBuilt) + } + #[cfg(nvbaseline_built)] + { + let mut session = NvBaselineSession::new()?; + session.transcode_jpeg_to_htj2k(jpeg) + } +} + +/// Decode a JPEG 2000 / HTJ2K codestream via nvJPEG2000. +pub fn nvidia_decode_j2k_interleaved( + codestream: &[u8], + format: NvJ2kDecodeFormat, +) -> Result { + #[cfg(not(nvbaseline_built))] + { + let _ = (codestream, format); + Err(NvBaselineError::NotBuilt) + } + #[cfg(nvbaseline_built)] + { + let mut session = NvBaselineSession::new()?; + session.decode_j2k_interleaved(codestream, format) + } +} + +#[cfg(nvbaseline_built)] +fn nvidia_transcode_with_session( + session: *mut ffi::NvbSession, + jpeg: &[u8], +) -> Result { + // HTJ2K of an 8-bit RGB image is well under the raw pixel size; allocate + // raw RGB size plus header slack and grow once if the codec disagrees. + let dims = jpeg_dimensions_from_header(jpeg)?; + let mut capacity = (dims.0 as usize) * (dims.1 as usize) * 3 + (1 << 16); + loop { + let mut out = vec![0u8; capacity]; + let mut out_len = 0usize; + let mut decode_ms = 0f64; + let mut encode_ms = 0f64; + let mut width = 0i32; + let mut height = 0i32; + let mut num_components = 0i32; + // SAFETY: all pointers reference live, correctly-sized allocations. + let rc = unsafe { + ffi::nvb_session_transcode_jpeg_to_htj2k( + session, + jpeg.as_ptr(), + jpeg.len(), + out.as_mut_ptr(), + out.len(), + std::ptr::addr_of_mut!(out_len), + std::ptr::addr_of_mut!(decode_ms), + std::ptr::addr_of_mut!(encode_ms), + std::ptr::addr_of_mut!(width), + std::ptr::addr_of_mut!(height), + std::ptr::addr_of_mut!(num_components), + ) + }; + // rc 212 == output buffer too small; double and retry once more. + if rc == 212 && capacity < (1 << 30) { + capacity *= 2; + continue; + } + if rc != 0 { + return Err(NvBaselineError::Stage(rc)); + } + out.truncate(out_len); + return Ok(NvTranscodeResult { + codestream: out, + decode_ms, + encode_ms, + width: width as u32, + height: height as u32, + num_components: num_components as u32, + }); + } +} + +#[cfg(nvbaseline_built)] +fn nvidia_decode_j2k_with_session( + session: *mut ffi::NvbSession, + codestream: &[u8], + format: NvJ2kDecodeFormat, +) -> Result { + let image = j2k_native::Image::new(codestream, &j2k_native::DecodeSettings::default()) + .map_err(|_| NvBaselineError::Stage(241))?; + let dims = (image.width(), image.height()); + let mut capacity = (dims.0 as usize) + .saturating_mul(dims.1 as usize) + .saturating_mul(format.bytes_per_pixel()); + if capacity == 0 { + return Err(NvBaselineError::Stage(240)); + } + loop { + let mut out = vec![0u8; capacity]; + let mut out_len = 0usize; + let mut decode_ms = 0f64; + let mut width = 0i32; + let mut height = 0i32; + let mut num_components = 0i32; + let mut bit_depth = 0i32; + let mut bytes_per_sample = 0i32; + // SAFETY: all pointers reference live, correctly-sized Rust slices or + // out parameters. The C++ side writes at most `out.len()` bytes. + let rc = unsafe { + ffi::nvb_session_decode_j2k_interleaved( + session, + codestream.as_ptr(), + codestream.len(), + format.ffi_code(), + out.as_mut_ptr(), + out.len(), + std::ptr::addr_of_mut!(out_len), + std::ptr::addr_of_mut!(decode_ms), + std::ptr::addr_of_mut!(width), + std::ptr::addr_of_mut!(height), + std::ptr::addr_of_mut!(num_components), + std::ptr::addr_of_mut!(bit_depth), + std::ptr::addr_of_mut!(bytes_per_sample), + ) + }; + if rc == 234 && capacity < (1 << 30) { + capacity = capacity.saturating_mul(2); + continue; + } + if rc != 0 { + return Err(NvBaselineError::Stage(rc)); + } + out.truncate(out_len); + return Ok(NvJ2kDecodeResult { + pixels: out, + decode_ms, + width: width as u32, + height: height as u32, + num_components: num_components as u32, + bit_depth: bit_depth as u32, + bytes_per_sample: bytes_per_sample as u32, + }); + } +} + +/// Read `(width, height)` from a JPEG SOF marker without initializing nvJPEG. +#[cfg(nvbaseline_built)] +fn jpeg_dimensions_from_header(jpeg: &[u8]) -> Result<(u32, u32), NvBaselineError> { + if jpeg.len() < 4 || jpeg[0] != 0xFF || jpeg[1] != 0xD8 { + return Err(NvBaselineError::Stage(103)); + } + + let mut offset = 2usize; + while offset + 3 < jpeg.len() { + if jpeg[offset] != 0xFF { + offset = offset.saturating_add(1); + continue; + } + while offset + 1 < jpeg.len() && jpeg[offset + 1] == 0xFF { + offset = offset.saturating_add(1); + } + if offset + 1 >= jpeg.len() { + break; + } + + let marker = jpeg[offset + 1]; + if marker == 0xD9 || marker == 0xDA { + break; + } + if marker == 0xD8 || (0xD0..=0xD7).contains(&marker) { + offset = offset.saturating_add(2); + continue; + } + if offset + 3 >= jpeg.len() { + break; + } + + let segment_len = (usize::from(jpeg[offset + 2]) << 8) | usize::from(jpeg[offset + 3]); + if segment_len < 2 || offset + 2 + segment_len > jpeg.len() { + break; + } + let is_sof = matches!( + marker, + 0xC0..=0xC3 | 0xC5..=0xC7 | 0xC9..=0xCB | 0xCD..=0xCF + ); + if is_sof { + if segment_len < 7 { + break; + } + let height = (u32::from(jpeg[offset + 5]) << 8) | u32::from(jpeg[offset + 6]); + let width = (u32::from(jpeg[offset + 7]) << 8) | u32::from(jpeg[offset + 8]); + if width != 0 && height != 0 { + return Ok((width, height)); + } + break; + } + offset = offset.saturating_add(2 + segment_len); + } + + Err(NvBaselineError::Stage(103)) +} + +/// Peak-signal-to-noise ratio (dB) between two equal-length 8-bit buffers. +/// +/// Returns `f64::INFINITY` for identical inputs and `None` on a length mismatch. +#[must_use] +pub fn psnr_u8(a: &[u8], b: &[u8]) -> Option { + if a.len() != b.len() || a.is_empty() { + return None; + } + let mut sum_sq = 0f64; + for (&x, &y) in a.iter().zip(b.iter()) { + let diff = f64::from(x) - f64::from(y); + sum_sq += diff * diff; + } + let mse = sum_sq / a.len() as f64; + if mse == 0.0 { + return Some(f64::INFINITY); + } + Some(10.0 * (255.0f64 * 255.0 / mse).log10()) +} + +/// JFIF full-range YCbCr to interleaved RGB, rounded to the nearest channel value. +#[must_use] +pub fn ycbcr_to_rgb_round_nearest(ycbcr: &[u8]) -> Vec { + let mut rgb = Vec::with_capacity(ycbcr.len()); + for px in ycbcr.chunks_exact(3) { + let y = f32::from(px[0]); + let cb = f32::from(px[1]) - 128.0; + let cr = f32::from(px[2]) - 128.0; + rgb.push(round_rgb_channel(y + 1.402 * cr)); + rgb.push(round_rgb_channel(y - 0.344_136 * cb - 0.714_136 * cr)); + rgb.push(round_rgb_channel(y + 1.772 * cb)); + } + rgb +} + +fn round_rgb_channel(value: f32) -> u8 { + value.clamp(0.0, 255.0).round() as u8 +} + +/// Writes a text artifact and creates parent directories when needed. +/// +/// This keeps benchmark binaries consistent when output paths include nested +/// artifact directories. +pub fn write_text_artifact( + path: &std::path::Path, + contents: impl AsRef, +) -> std::io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(path, contents.as_ref()) +} + +#[cfg(test)] +mod tests { + use super::{write_text_artifact, ycbcr_to_rgb_round_nearest}; + + #[test] + fn ycbcr_to_rgb_rounds_to_nearest_channel_value() { + let rgb = ycbcr_to_rgb_round_nearest(&[100, 129, 128]); + + assert_eq!(rgb, vec![100, 100, 102]); + } + + #[test] + fn write_text_artifact_creates_parent_directories() { + let path = std::env::temp_dir() + .join(format!("j2k-nvb-artifact-{}", std::process::id())) + .join("nested") + .join("report.json"); + + if let Some(root) = path.parent().and_then(std::path::Path::parent) { + if root.exists() { + std::fs::remove_dir_all(root).expect("remove stale test artifact dir"); + } + } + + write_text_artifact(&path, "report").expect("write artifact"); + + assert_eq!( + std::fs::read_to_string(&path).expect("read artifact"), + "report" + ); + } +} diff --git a/tests/nvidia-baseline/tests/api.rs b/tests/nvidia-baseline/tests/api.rs new file mode 100644 index 00000000..6a90f545 --- /dev/null +++ b/tests/nvidia-baseline/tests/api.rs @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(nvbaseline_built))] +use j2k_nvidia_baseline::{ + nvidia_decode_j2k_interleaved, nvidia_j2k_decode_available, NvBaselineError, +}; +use j2k_nvidia_baseline::{NvBaselineSession, NvJ2kDecodeFormat}; + +#[cfg(not(nvbaseline_built))] +#[test] +fn nvbaseline_session_reports_not_built_when_cpp_baseline_is_unavailable() { + match NvBaselineSession::new() { + Err(NvBaselineError::NotBuilt) => {} + Ok(_) => panic!("nvJPEG2000 session should not build without the nvjpeg2000 feature"), + Err(error) => panic!("unexpected nvJPEG2000 session error: {error:?}"), + } +} + +#[cfg(not(nvbaseline_built))] +#[test] +fn nvjpeg2000_decode_reports_not_built_when_cpp_baseline_is_unavailable() { + assert!(!nvidia_j2k_decode_available()); + match nvidia_decode_j2k_interleaved(&[], NvJ2kDecodeFormat::Rgb8) { + Err(NvBaselineError::NotBuilt) => {} + Ok(_) => panic!("nvJPEG2000 decode should not run without the C++ baseline"), + Err(error) => panic!("unexpected nvJPEG2000 decode error: {error:?}"), + } +} + +#[cfg(nvbaseline_built)] +#[test] +fn nvbaseline_session_constructs_when_built() { + let _session = NvBaselineSession::new().expect("nvJPEG2000 session initializes"); +} + +#[cfg(nvbaseline_built)] +#[test] +fn nvjpeg2000_decode_smoke_decodes_nvidia_htj2k_pathology_tile() { + let jpeg = include_bytes!("../benchtiles/pancreas/tile_00000.jpg"); + let mut session = NvBaselineSession::new().expect("nvJPEG2000 session initializes"); + let codestream = session + .transcode_jpeg_to_htj2k(jpeg) + .expect("nvJPEG2000 encodes pathology tile to HTJ2K"); + + let decoded = session + .decode_j2k_interleaved(&codestream.codestream, NvJ2kDecodeFormat::Rgb8) + .expect("nvJPEG2000 decodes NVIDIA-generated HTJ2K smoke input"); + + assert_eq!(decoded.width, codestream.width); + assert_eq!(decoded.height, codestream.height); + assert_eq!(decoded.num_components, 3); + assert_eq!(decoded.bytes_per_sample, 1); + assert_eq!( + decoded.pixels.len(), + codestream.width as usize * codestream.height as usize * 3 + ); +} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 7973d8cf..7e44aac7 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -5,6 +5,10 @@ edition = "2021" license.workspace = true publish = false +[dependencies] +serde_json = "1" +serde_yaml_ng = "0.10" + [lints.rust] unsafe_code = "forbid" diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 6dc39785..ecce348d 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,49 +1,106 @@ +use std::collections::BTreeSet; use std::env; use std::ffi::OsString; -use std::process::{Command, ExitCode}; +use std::fmt::Write as _; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +mod perf_guard; +mod process; const PUBLISHABLE_PACKAGES: &[&str] = &[ - "signinum-core", - "signinum-cuda-runtime", - "signinum-profile", - "signinum-j2k-native", - "signinum-jpeg", - "signinum-tilecodec", - "signinum-j2k", - "signinum-jpeg-metal", - "signinum-j2k-metal", - "signinum-jpeg-cuda", - "signinum-j2k-cuda", - "signinum-cli", - "signinum", + "j2k-core", + "j2k-profile", + "j2k-types", + "j2k-cuda-runtime", + "j2k-metal-support", + "j2k-native", + "j2k-jpeg", + "j2k-tilecodec", + "j2k", + "j2k-transcode", + "j2k-transcode-cuda", + "j2k-jpeg-metal", + "j2k-metal", + "j2k-transcode-metal", + "j2k-jpeg-cuda", + "j2k-cuda", + "j2k-cli", ]; -const REGISTRY_INDEPENDENT_PACKAGES: &[&str] = - &["signinum-core", "signinum-cuda-runtime", "signinum-profile"]; +const REGISTRY_INDEPENDENT_PACKAGES: &[&str] = &["j2k-core", "j2k-profile", "j2k-types"]; const STAGED_DEPENDENCY_PACKAGES: &[&str] = &[ - "signinum-j2k-native", - "signinum-jpeg", - "signinum-tilecodec", - "signinum-j2k", - "signinum-jpeg-metal", - "signinum-j2k-metal", - "signinum-jpeg-cuda", - "signinum-j2k-cuda", - "signinum-cli", - "signinum", + "j2k-cuda-runtime", + "j2k-metal-support", + "j2k-native", + "j2k-jpeg", + "j2k-tilecodec", + "j2k", + "j2k-transcode", + "j2k-transcode-cuda", + "j2k-jpeg-metal", + "j2k-metal", + "j2k-transcode-metal", + "j2k-jpeg-cuda", + "j2k-cuda", + "j2k-cli", ]; const CPU_RELEASE_PACKAGES: &[&str] = &[ - "signinum-core", - "signinum-jpeg", - "signinum-j2k-native", - "signinum-j2k", - "signinum-tilecodec", - "signinum-cli", + "j2k-core", + "j2k-jpeg", + "j2k-types", + "j2k-native", + "j2k", + "j2k-tilecodec", + "j2k-cli", +]; + +const STABLE_SEMVER_PACKAGES: &[&str] = &[ + "j2k", + "j2k-core", + "j2k-jpeg", + "j2k-tilecodec", + "j2k-jpeg-metal", + "j2k-metal", + "j2k-jpeg-cuda", + "j2k-cuda", + "j2k-transcode", + "j2k-transcode-cuda", + "j2k-metal-support", + "j2k-transcode-metal", + "j2k-native", + "j2k-types", + "j2k-cuda-runtime", + "j2k-profile", ]; +const STABLE_DOC_LIBRARY_PACKAGES: &[&str] = &[ + "j2k", + "j2k-core", + "j2k-jpeg", + "j2k-tilecodec", + "j2k-jpeg-metal", + "j2k-metal", + "j2k-jpeg-cuda", + "j2k-cuda", + "j2k-transcode", + "j2k-transcode-cuda", + "j2k-metal-support", + "j2k-transcode-metal", + "j2k-native", + "j2k-types", + "j2k-cuda-runtime", + "j2k-profile", +]; + +const STABLE_API_SNAPSHOT: &str = "docs/stable-api-1.0.public-api.txt"; +const CARGO_PUBLIC_API_VERSION: &str = "0.52.0"; + const NO_STD_TARGET: &str = "aarch64-unknown-none"; +const NO_STD_CORE_PORTABLE_TARGET: &str = "wasm32-unknown-unknown"; fn main() -> ExitCode { match run() { @@ -60,13 +117,27 @@ fn run() -> Result<(), String> { match task.as_str() { "fmt" => fmt(), "clippy" => clippy(), + "clippy-strict" => clippy_strict(), "test" => test(), + "nextest" => nextest(), "doc" | "docs" => doc(), "typos" => typos(), "bench-build" => bench_build(), + "bench-report" => bench_report(env::args().skip(2)), + "j2k-bench-signoff" => j2k_bench_signoff(), + "j2k-perf-guard" => perf_guard::j2k_perf_guard(env::args().skip(2)), "fuzz-build" => fuzz_build(), + "fuzz-run" => fuzz_run(), + "stable-api" => stable_api(env::args().skip(2)), + "semver" => semver(), "deny" => deny(), + "miri" => miri(), + "machete" => machete(), "no-std" => no_std(), + "unsafe-audit" => verify_unsafe_audit(), + "downstream-smoke" => downstream_smoke(), + "repo-lint" => repo_lint(), + "release-integrity" => release_integrity(), "release-cpu" => release_cpu(), "release-metal" => release_metal(), "coverage" => coverage(), @@ -84,7 +155,20 @@ fn ci() -> Result<(), String> { fmt()?; clippy()?; test()?; - doc() + doc()?; + verify_unsafe_audit() +} + +fn repo_lint() -> Result<(), String> { + run_cargo(&[ + "test", + "-p", + "xtask", + "--test", + "repo_lint", + "--", + "--nocapture", + ]) } fn fmt() -> Result<(), String> { @@ -103,20 +187,95 @@ fn clippy() -> Result<(), String> { ]) } +fn clippy_strict() -> Result<(), String> { + let mut args = vec![ + "clippy", + "-p", + "j2k-native", + "-p", + "j2k", + "--all-targets", + "--all-features", + "--no-deps", + "--", + "-W", + "clippy::pedantic", + "-W", + "clippy::nursery", + "-D", + "warnings", + ]; + + // Keep the strict gate useful as a ratchet: enable pedantic/nursery, but + // baseline high-noise codec-math lints so new lint classes still fail. + for lint in STRICT_CLIPPY_BASELINE_ALLOWED_LINTS { + args.extend(["-A", lint]); + } + + run_cargo(&args) +} + +const STRICT_CLIPPY_BASELINE_ALLOWED_LINTS: &[&str] = &[ + "clippy::bool_to_int_with_if", + "clippy::branches_sharing_code", + "clippy::cast_lossless", + "clippy::cast_possible_truncation", + "clippy::cast_possible_wrap", + "clippy::cast_precision_loss", + "clippy::cast_sign_loss", + "clippy::checked_conversions", + "clippy::cognitive_complexity", + "clippy::doc_markdown", + "clippy::elidable_lifetime_names", + "clippy::explicit_deref_methods", + "clippy::explicit_iter_loop", + "clippy::float_cmp", + "clippy::if_not_else", + "clippy::inconsistent_struct_constructor", + "clippy::inline_always", + "clippy::items_after_statements", + "clippy::manual_let_else", + "clippy::map_unwrap_or", + "clippy::match_same_arms", + "clippy::missing_const_for_fn", + "clippy::missing_errors_doc", + "clippy::must_use_candidate", + "clippy::needless_collect", + "clippy::needless_pass_by_ref_mut", + "clippy::needless_pass_by_value", + "clippy::no_effect_underscore_binding", + "clippy::or_fun_call", + "clippy::redundant_clone", + "clippy::redundant_closure_for_method_calls", + "clippy::redundant_else", + "clippy::redundant_pub_crate", + "clippy::similar_names", + "clippy::struct_excessive_bools", + "clippy::struct_field_names", + "clippy::suboptimal_flops", + "clippy::suspicious_operation_groupings", + "clippy::too_many_lines", + "clippy::trivially_copy_pass_by_ref", + "clippy::unnecessary_wraps", + "clippy::unreadable_literal", + "clippy::used_underscore_binding", + "clippy::useless_let_if_seq", +]; + fn test() -> Result<(), String> { if env::consts::OS != "macos" { return test_workspace_without_benches(&[]); } - test_workspace_without_benches(&["--exclude", "signinum-j2k-metal"])?; + test_workspace_without_benches(&["--exclude", "j2k-metal"])?; if skip_j2k_metal_runtime_on_hosted_github_macos() { eprintln!( - "skipping signinum-j2k-metal runtime tests on GitHub-hosted macOS; \ + "skipping j2k-metal runtime tests on GitHub-hosted macOS; \ self-hosted gpu-validation runs the Metal runtime suite" ); - return test_package_without_benches("signinum-j2k-metal", true); + return test_package_without_benches("j2k-metal", true); } - test_package_without_benches("signinum-j2k-metal", false) + test_package_without_benches("j2k-metal", false) } fn test_workspace_without_benches(extra_args: &[&str]) -> Result<(), String> { @@ -130,12 +289,24 @@ fn test_workspace_without_benches(extra_args: &[&str]) -> Result<(), String> { ]; test_args.extend_from_slice(extra_args); run_cargo(&test_args)?; + test_facade_cuda_stub()?; let mut doc_args = vec!["test", "--workspace", "--all-features", "--doc"]; doc_args.extend_from_slice(extra_args); run_cargo(&doc_args) } +fn test_facade_cuda_stub() -> Result<(), String> { + run_cargo(&[ + "test", + "-p", + "j2k", + "--test", + "encode_lossless", + "accelerator_facade_auto_falls_back_when_no_stage_dispatches", + ]) +} + fn test_package_without_benches(package: &str, no_run: bool) -> Result<(), String> { let mut test_args = vec![ "test", @@ -158,10 +329,34 @@ fn test_package_without_benches(package: &str, no_run: bool) -> Result<(), Strin run_cargo(&["test", "-p", package, "--all-features", "--doc"]) } +fn nextest() -> Result<(), String> { + run_cargo(&[ + "nextest", + "run", + "--workspace", + "--all-features", + "--lib", + "--bins", + "--tests", + ]) +} + fn doc() -> Result<(), String> { run_cargo_with_env( &["doc", "--workspace", "--all-features", "--no-deps"], &[("RUSTDOCFLAGS", "-D warnings")], + )?; + + for package in STABLE_DOC_LIBRARY_PACKAGES { + run_cargo_with_env( + &["doc", "-p", package, "--lib", "--no-deps"], + &[("RUSTDOCFLAGS", "-D warnings -D missing_docs")], + )?; + } + + run_cargo_with_env( + &["doc", "-p", "j2k-cli", "--no-deps"], + &[("RUSTDOCFLAGS", "-D warnings -D missing_docs")], ) } @@ -170,77 +365,942 @@ fn typos() -> Result<(), String> { } fn bench_build() -> Result<(), String> { + run_cargo(&["bench", "-p", "j2k", "--bench", "public_api", "--no-run"])?; run_cargo(&[ "bench", "-p", - "signinum-j2k", + "j2k-native", "--bench", - "public_api", + "tier1_bitplane", "--no-run", ])?; - run_cargo(&["bench", "-p", "signinum", "--bench", "facade", "--no-run"])?; - run_cargo(&["bench", "-p", "signinum-jpeg", "--no-run"])?; - run_cargo(&["bench", "-p", "signinum-jpeg-metal", "--no-run"])?; run_cargo(&[ "bench", "-p", - "signinum-jpeg-cuda", + "j2k-native", + "--bench", + "htj2k_sigprop_phase", + "--no-run", + ])?; + run_cargo(&[ + "bench", + "-p", + "j2k-native", + "--bench", + "direct_cpu", + "--no-run", + ])?; + run_cargo(&[ + "bench", + "-p", + "j2k-jpeg", + "--bench", + "encode_cpu", + "--no-run", + ])?; + run_cargo(&[ + "bench", + "-p", + "j2k-jpeg", + "--features", + "bench-libjpeg-turbo", + "--no-run", + ])?; + run_cargo(&["bench", "-p", "j2k-jpeg-metal", "--no-run"])?; + run_cargo(&[ + "bench", + "-p", + "j2k-jpeg-cuda", "--bench", "device_decode", "--features", "cuda-runtime", "--no-run", ])?; - run_cargo(&["bench", "-p", "signinum-j2k-metal", "--no-run"])?; run_cargo(&[ "bench", "-p", - "signinum-tilecodec", + "j2k-cuda", + "--bench", + "encode_stages", + "--features", + "cuda-runtime", + "--no-run", + ])?; + run_cargo(&[ + "bench", + "-p", + "j2k-cuda", + "--bench", + "htj2k_decode", + "--features", + "cuda-runtime", + "--no-run", + ])?; + run_cargo(&[ + "bench", + "-p", + "j2k-cuda", + "--bench", + "htj2k_encode", + "--features", + "cuda-runtime", + "--no-run", + ])?; + run_cargo(&[ + "bench", + "-p", + "j2k-tilecodec", "--bench", "compare", "--no-run", + ])?; + run_cargo(&[ + "bench", + "-p", + "j2k-transcode", + "--bench", + "dct53", + "--no-run", + ])?; + run_cargo(&[ + "bench", + "-p", + "j2k-transcode-metal", + "--bench", + "dct97", + "--no-run", ]) } +fn j2k_bench_signoff() -> Result<(), String> { + run_cargo_with_env( + &["test", "-p", "j2k-compare", "--test", "in_process_parity"], + &[("J2K_REQUIRE_OPENJPEG", "1"), ("J2K_REQUIRE_GROK", "1")], + )?; + run_cargo_with_env( + &["test", "-p", "j2k", "--test", "openjpeg_parity"], + &[("J2K_REQUIRE_OPENJPEG", "1")], + ) +} + +#[derive(Debug)] +struct BenchmarkReport { + command: String, + host: String, + rustc: String, + cargo: String, + git_revision: String, + workspace_version: String, + input_source: String, + compare_threads: String, + comparator_versions: Vec<(String, String)>, + skipped_rows: Vec, +} + +fn bench_report(args: impl Iterator) -> Result<(), String> { + let mut command = env::var("J2K_BENCH_COMMAND").unwrap_or_else(|_| "not recorded".into()); + let mut input_source = env::var("J2K_BENCH_INPUT_SOURCE") + .or_else(|_| env::var("J2K_BENCH_INPUTS")) + .unwrap_or_else(|_| "not recorded".into()); + let mut out_path = None::; + let mut skipped_rows = env::var("J2K_BENCH_SKIPPED_ROWS") + .ok() + .map(|rows| split_semicolon_list(&rows)) + .unwrap_or_default(); + + let mut args = args.peekable(); + while let Some(arg) = args.next() { + match arg.as_str() { + "--command" => { + command = args + .next() + .ok_or_else(|| "--command requires a value".to_string())?; + } + "--input-source" => { + input_source = args + .next() + .ok_or_else(|| "--input-source requires a value".to_string())?; + } + "--skipped-row" => { + skipped_rows.push( + args.next() + .ok_or_else(|| "--skipped-row requires a value".to_string())?, + ); + } + "--out" => { + out_path = Some(PathBuf::from( + args.next() + .ok_or_else(|| "--out requires a value".to_string())?, + )); + } + "--help" | "-h" => { + print_bench_report_help(); + return Ok(()); + } + other => return Err(format!("unknown bench-report argument `{other}`")), + } + } + + let report = BenchmarkReport { + command, + host: host_description(), + rustc: command_output("rustc", &["-Vv"]) + .unwrap_or_else(|err| format!("unavailable: {err}")), + cargo: command_output_os(cargo(), &["-V"]) + .unwrap_or_else(|err| format!("unavailable: {err}")), + git_revision: command_output("git", &["rev-parse", "HEAD"]) + .unwrap_or_else(|err| format!("unavailable: {err}")), + workspace_version: workspace_version()?, + input_source, + compare_threads: env::var("J2K_COMPARE_THREADS").unwrap_or_else(|_| "not set".to_string()), + comparator_versions: comparator_versions(), + skipped_rows, + }; + let rendered = render_benchmark_report(&report); + + if let Some(path) = out_path { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + fs::create_dir_all(parent) + .map_err(|err| format!("failed to create {}: {err}", parent.display()))?; + } + fs::write(&path, rendered) + .map_err(|err| format!("failed to write {}: {err}", path.display())) + } else { + print!("{rendered}"); + Ok(()) + } +} + fn fuzz_build() -> Result<(), String> { + run_cargo(&["check", "--manifest-path", "crates/j2k/fuzz/Cargo.toml"])?; run_cargo(&[ "check", "--manifest-path", - "crates/signinum-j2k/fuzz/Cargo.toml", + "crates/j2k-jpeg/fuzz/Cargo.toml", ])?; run_cargo(&[ "check", "--manifest-path", - "crates/signinum-jpeg/fuzz/Cargo.toml", + "crates/j2k-tilecodec/fuzz/Cargo.toml", ])?; run_cargo(&[ "check", "--manifest-path", - "crates/signinum-tilecodec/fuzz/Cargo.toml", + "crates/j2k-transcode/fuzz/Cargo.toml", ]) } +const FUZZ_TARGETS: &[(&str, &str)] = &[ + ("crates/j2k", "decode_fuzz"), + ("crates/j2k", "parse_fuzz"), + ("crates/j2k-jpeg", "decode_fuzz"), + ("crates/j2k-jpeg", "parse_fuzz"), + ("crates/j2k-tilecodec", "decompress_fuzz"), + ("crates/j2k-transcode", "jpeg_to_htj2k_fuzz"), +]; + +fn fuzz_run() -> Result<(), String> { + let runs = env::var("J2K_FUZZ_RUNS").unwrap_or_else(|_| "1000".to_string()); + let max_total_time = env::var("J2K_FUZZ_MAX_TOTAL_TIME_SECONDS").ok(); + let fuzz_target = fuzz_target_triple()?; + + for (crate_dir, target) in FUZZ_TARGETS { + let mut args = vec![ + "fuzz".to_string(), + "run".to_string(), + "--target".to_string(), + fuzz_target.clone(), + (*target).to_string(), + "--".to_string(), + format!("-runs={runs}"), + ]; + if let Some(seconds) = &max_total_time { + args.push(format!("-max_total_time={seconds}")); + } + run_nightly_cargo_in_dir_owned(crate_dir, &args)?; + } + Ok(()) +} + +fn fuzz_target_triple() -> Result { + if let Ok(target) = env::var("J2K_FUZZ_TARGET") { + if !target.trim().is_empty() { + return Ok(target); + } + } + + let version = command_output_os( + OsString::from("rustup"), + &["run", "nightly", "rustc", "-vV"], + ) + .map_err(|err| format!("failed to detect nightly host target for fuzz-run: {err}"))?; + version + .lines() + .find_map(|line| line.strip_prefix("host: ")) + .map(str::to_string) + .ok_or_else(|| "failed to parse nightly host target from `rustc -vV`".to_string()) +} + +fn stable_api(args: impl Iterator) -> Result<(), String> { + let mut write_snapshot = false; + for arg in args { + match arg.as_str() { + "--write" => write_snapshot = true, + "--help" | "-h" => { + print_stable_api_help(); + return Ok(()); + } + other => return Err(format!("unknown stable-api argument `{other}`")), + } + } + + let rendered = render_stable_api_snapshot()?; + if write_snapshot { + fs::write(STABLE_API_SNAPSHOT, rendered) + .map_err(|err| format!("failed to write {STABLE_API_SNAPSHOT}: {err}"))?; + return Ok(()); + } + + let committed = fs::read_to_string(STABLE_API_SNAPSHOT) + .map_err(|err| format!("failed to read {STABLE_API_SNAPSHOT}: {err}"))?; + if committed == rendered { + Ok(()) + } else { + Err(format!( + "{STABLE_API_SNAPSHOT} is stale; run `cargo xtask stable-api --write` and review the public API diff" + )) + } +} + +fn render_stable_api_snapshot() -> Result { + if !cfg!(target_os = "macos") { + return Err( + "stable-api snapshot must be generated on macOS so target-gated Metal APIs are included" + .to_string(), + ); + } + + let tool_version = + command_output_os_detailed(cargo(), &["public-api", "--version"]).map_err(|err| { + format!( + "failed to detect cargo-public-api: {err}; \ + install cargo-public-api with `cargo install cargo-public-api --version {CARGO_PUBLIC_API_VERSION} --locked`" + ) + })?; + if !tool_version.contains(CARGO_PUBLIC_API_VERSION) { + return Err(format!( + "cargo-public-api version must be {CARGO_PUBLIC_API_VERSION}; found `{tool_version}`" + )); + } + + let mut out = String::new(); + writeln!( + &mut out, + "# J2K 1.0 Public API Snapshot\n\n\ + This file is generated by `cargo xtask stable-api --write` from \ + `cargo public-api -p --all-features -sss --color never`.\n\ + It is generated on macOS so target-gated Metal APIs are included; \ + non-macOS builds expose a smaller cfg-gated subset.\n\n\ + Generator: `{tool_version}`.\n\n\ + It is the item-level companion to `docs/stable-api-1.0.md`: every \ + public module, type, trait, function, method, constant, variant, and \ + field reported here is semver-visible unless moved private before 1.0.\n" + ) + .unwrap(); + + for package in STABLE_DOC_LIBRARY_PACKAGES { + let api = command_output_os_detailed( + cargo(), + &[ + "public-api", + "-p", + package, + "--all-features", + "-sss", + "--color", + "never", + ], + ) + .map_err(|err| { + format!( + "failed to generate public API for {package}: {err}; \ + install cargo-public-api with `cargo install cargo-public-api --version {CARGO_PUBLIC_API_VERSION} --locked`" + ) + })?; + writeln!(&mut out, "## `{package}`\n\n```text").unwrap(); + writeln!(&mut out, "{api}").unwrap(); + writeln!(&mut out, "```\n").unwrap(); + } + + writeln!( + &mut out, + "## `j2k-cli`\n\n\ + `j2k-cli` is a binary package. Its stable command, stdout/stderr, \ + and exit-code contract is documented in `docs/stable-api-1.0.md`.\n" + ) + .unwrap(); + + Ok(out) +} + +fn semver() -> Result<(), String> { + let toolchain = env::var("J2K_SEMVER_TOOLCHAIN").unwrap_or_else(|_| "1.96".to_string()); + let toolchain_arg = format!("+{toolchain}"); + for package in STABLE_SEMVER_PACKAGES { + if !crates_io_package_exists(package)? { + eprintln!("skipping semver baseline for unpublished package `{package}`"); + continue; + } + run_program( + OsString::from("cargo"), + &[ + toolchain_arg.as_str(), + "semver-checks", + "check-release", + "--package", + package, + "--release-type", + "major", + ], + &[], + )?; + } + Ok(()) +} + +fn crates_io_package_exists(package: &str) -> Result { + let cargo = cargo(); + let display = cargo.to_string_lossy().into_owned(); + let output = process::command_output( + cargo, + &["info", package, "--registry", "crates-io"], + process::CommandContext::new(), + )?; + if output.status.success() { + return Ok(true); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let combined = format!("{stdout}\n{stderr}"); + if combined.contains("could not find") || combined.contains("not found in registry") { + Ok(false) + } else { + Err(format!( + "`{display} info {package} --registry crates-io` exited with {}:\n{}", + output.status, + combined.trim() + )) + } +} + fn deny() -> Result<(), String> { run_cargo(&["deny", "check", "licenses", "advisories", "bans", "sources"]) } +fn miri() -> Result<(), String> { + run_nightly_cargo(&["miri", "test", "-p", "j2k-core"])?; + run_nightly_cargo(&["miri", "test", "-p", "j2k-tilecodec"])?; + run_nightly_cargo(&[ + "miri", + "test", + "-p", + "j2k-native", + "--no-default-features", + "inspect::", + ]) +} + +fn machete() -> Result<(), String> { + run_program(OsString::from("cargo-machete"), &[], &[]) +} + fn no_std() -> Result<(), String> { run_program( OsString::from("rustup"), &["target", "add", NO_STD_TARGET], &[], )?; - run_cargo(&["check", "-p", "signinum-core", "--target", NO_STD_TARGET])?; + run_cargo(&["check", "-p", "j2k-core", "--target", NO_STD_TARGET])?; run_cargo(&[ "check", "-p", - "signinum-j2k-native", + "j2k-native", "--no-default-features", "--target", NO_STD_TARGET, + ])?; + run_program( + OsString::from("rustup"), + &["target", "add", NO_STD_CORE_PORTABLE_TARGET], + &[], + )?; + run_cargo(&[ + "check", + "-p", + "j2k-core", + "--target", + NO_STD_CORE_PORTABLE_TARGET, ]) } +fn verify_unsafe_audit() -> Result<(), String> { + let audit_path = Path::new("docs/unsafe-audit.md"); + let audit = fs::read_to_string(audit_path) + .map_err(|err| format!("failed to read {}: {err}", audit_path.display()))?; + if !audit.contains("| Path | Scope | Invariants | Regression guards |") { + return Err( + "docs/unsafe-audit.md must include Path/Scope/Invariants/Regression guards columns" + .to_string(), + ); + } + let mut malformed_rows = Vec::new(); + for line in audit.lines() { + let trimmed = line.trim(); + if !trimmed.starts_with("| `crates/") { + continue; + } + let cells = trimmed.split('|').map(str::trim).collect::>(); + if cells.len() < 6 + || cells[1].is_empty() + || cells[2].is_empty() + || cells[3].is_empty() + || cells[4].is_empty() + || cells[3].eq_ignore_ascii_case("tbd") + || cells[4].eq_ignore_ascii_case("tbd") + { + malformed_rows.push(trimmed.to_string()); + } + } + if !malformed_rows.is_empty() { + return Err(format!( + "docs/unsafe-audit.md has unsafe rows missing invariants or regression guards: {malformed_rows:?}" + )); + } + let mut missing = Vec::new(); + for path in rust_sources(Path::new("crates"))? { + let source = fs::read_to_string(&path) + .map_err(|err| format!("failed to read {}: {err}", path.display()))?; + if source.contains("unsafe ") || source.contains("unsafe{") { + let relative = path.to_string_lossy().replace('\\', "/"); + if !audit.contains(&relative) { + missing.push(relative); + } + } + } + if missing.is_empty() { + Ok(()) + } else { + Err(format!( + "docs/unsafe-audit.md is missing unsafe source entries: {missing:?}" + )) + } +} + +fn downstream_smoke() -> Result<(), String> { + run_cargo(&["test", "-p", "j2k", "--examples"])?; + run_cargo(&["test", "-p", "j2k-transcode", "--examples"]) +} + +fn release_integrity() -> Result<(), String> { + let metadata = cargo_metadata()?; + let workspace_version = workspace_version()?; + let publishable_set = str_set(PUBLISHABLE_PACKAGES); + let docs_set = str_set(STABLE_DOC_LIBRARY_PACKAGES); + let semver_set = str_set(STABLE_SEMVER_PACKAGES); + let mut errors = Vec::new(); + + let packages = metadata + .get("packages") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| "cargo metadata did not contain a packages array".to_string())?; + let workspace_members = metadata + .get("workspace_members") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| "cargo metadata did not contain a workspace_members array".to_string())? + .iter() + .filter_map(serde_json::Value::as_str) + .map(str::to_string) + .collect::>(); + let mut workspace_names = BTreeSet::new(); + + let unpublished_members = packages + .iter() + .filter(|package| { + package + .get("id") + .and_then(serde_json::Value::as_str) + .is_some_and(|id| workspace_members.contains(id)) + && publish_false(package) + }) + .filter_map(|package| package.get("name").and_then(serde_json::Value::as_str)) + .collect::>(); + + for package in packages { + let id = package + .get("id") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + if !workspace_members.contains(id) { + continue; + } + + let name = package_name(package)?; + workspace_names.insert(name.to_string()); + let listed_publishable = publishable_set.contains(name); + let explicitly_unpublished = publish_false(package); + + if listed_publishable && explicitly_unpublished { + errors.push(format!( + "`{name}` is listed as publishable but has `publish = false`" + )); + continue; + } + if !listed_publishable && !explicitly_unpublished { + errors.push(format!( + "`{name}` is neither in PUBLISHABLE_PACKAGES nor explicitly `publish = false`" + )); + continue; + } + if !listed_publishable { + continue; + } + + validate_unpublished_dependencies(name, package, &unpublished_members, &mut errors); + + let version = package + .get("version") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + if version != workspace_version { + errors.push(format!( + "`{name}` version {version} does not match workspace version {workspace_version}" + )); + } + if package + .get("readme") + .and_then(serde_json::Value::as_str) + .is_none() + { + errors.push(format!("`{name}` is publishable but has no package README")); + } + if !has_docs_rs_metadata(package) { + errors.push(format!( + "`{name}` is publishable but missing [package.metadata.docs.rs] with all-features and empty targets" + )); + } + if has_lib_target(package) { + if !docs_set.contains(name) { + errors.push(format!( + "`{name}` has a library target but is missing from STABLE_DOC_LIBRARY_PACKAGES" + )); + } + if !semver_set.contains(name) { + errors.push(format!( + "`{name}` has a library target but is missing from STABLE_SEMVER_PACKAGES" + )); + } + } else if name != "j2k-cli" { + errors.push(format!( + "`{name}` is publishable but has no library target and no explicit release-integrity exemption" + )); + } + } + + for package in PUBLISHABLE_PACKAGES { + if !workspace_names.contains(*package) { + errors.push(format!( + "`{package}` is listed in PUBLISHABLE_PACKAGES but is not a workspace member" + )); + } + } + for package in STABLE_DOC_LIBRARY_PACKAGES { + if !publishable_set.contains(package) { + errors.push(format!( + "`{package}` is in STABLE_DOC_LIBRARY_PACKAGES but is not publishable" + )); + } + } + for package in STABLE_SEMVER_PACKAGES { + if !publishable_set.contains(package) { + errors.push(format!( + "`{package}` is in STABLE_SEMVER_PACKAGES but is not publishable" + )); + } + } + + validate_publish_workflow(&mut errors)?; + validate_publish_script(&mut errors)?; + validate_release_docs(&mut errors)?; + + if errors.is_empty() { + Ok(()) + } else { + Err(format!( + "release integrity violations:\n- {}", + errors.join("\n- ") + )) + } +} + +fn validate_unpublished_dependencies( + name: &str, + package: &serde_json::Value, + unpublished_members: &BTreeSet<&str>, + errors: &mut Vec, +) { + let dependencies = package + .get("dependencies") + .and_then(serde_json::Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); + for dependency in dependencies { + let dep_name = dependency + .get("name") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + if !unpublished_members.contains(dep_name) { + continue; + } + let kind = dependency + .get("kind") + .and_then(serde_json::Value::as_str) + .unwrap_or("normal"); + let req = dependency + .get("req") + .and_then(serde_json::Value::as_str) + .unwrap_or("*"); + if kind != "dev" { + errors.push(format!( + "`{name}` has a {kind} dependency on unpublished crate `{dep_name}`; \ + cargo publish cannot resolve it" + )); + } else if req != "*" { + errors.push(format!( + "`{name}` has a versioned dev-dependency `{dep_name} = \"{req}\"` on an \ + unpublished crate; drop the version so cargo publish strips the path-only dev-dep" + )); + } + } +} + +fn cargo_metadata() -> Result { + let data = command_output_os(cargo(), &["metadata", "--no-deps", "--format-version", "1"])?; + serde_json::from_str(&data).map_err(|err| format!("failed to parse cargo metadata: {err}")) +} + +fn package_name(package: &serde_json::Value) -> Result<&str, String> { + package + .get("name") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| "cargo metadata package missing name".to_string()) +} + +fn publish_false(package: &serde_json::Value) -> bool { + package + .get("publish") + .and_then(serde_json::Value::as_array) + .is_some_and(Vec::is_empty) +} + +fn has_lib_target(package: &serde_json::Value) -> bool { + package + .get("targets") + .and_then(serde_json::Value::as_array) + .is_some_and(|targets| { + targets.iter().any(|target| { + target + .get("kind") + .and_then(serde_json::Value::as_array) + .is_some_and(|kind| { + kind.iter() + .any(|entry| entry.as_str().is_some_and(|entry| entry == "lib")) + }) + }) + }) +} + +fn has_docs_rs_metadata(package: &serde_json::Value) -> bool { + let Some(docs_rs) = package + .get("metadata") + .and_then(|metadata| metadata.get("docs")) + .and_then(|docs| docs.get("rs")) + else { + return false; + }; + + docs_rs + .get("all-features") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + && docs_rs + .get("targets") + .and_then(serde_json::Value::as_array) + .is_some_and(Vec::is_empty) +} + +fn validate_publish_workflow(errors: &mut Vec) -> Result<(), String> { + let workflow_path = Path::new(".github/workflows/publish.yml"); + let workflow = fs::read_to_string(workflow_path) + .map_err(|err| format!("failed to read {}: {err}", workflow_path.display()))?; + let workflow: serde_yaml_ng::Value = serde_yaml_ng::from_str(&workflow) + .map_err(|err| format!("failed to parse {}: {err}", workflow_path.display()))?; + let mut crates = Vec::new(); + collect_publish_workflow_crates(&workflow, &mut crates); + + let expected = PUBLISHABLE_PACKAGES + .iter() + .map(ToString::to_string) + .collect::>(); + if crates != expected { + errors.push(format!( + "{} publish order is {:?}, expected {:?}", + workflow_path.display(), + crates, + expected + )); + } + + let seen = crates.iter().map(String::as_str).collect::>(); + for package in PUBLISHABLE_PACKAGES { + if !seen.contains(package) { + errors.push(format!( + "{} is missing publish job for `{package}`", + workflow_path.display() + )); + } + } + for package in crates { + if !PUBLISHABLE_PACKAGES.contains(&package.as_str()) { + errors.push(format!( + "{} publishes unknown workspace crate `{package}`", + workflow_path.display() + )); + } + } + + Ok(()) +} + +fn collect_publish_workflow_crates(value: &serde_yaml_ng::Value, crates: &mut Vec) { + match value { + serde_yaml_ng::Value::String(text) => { + for line in text.lines() { + if let Some(package) = publish_crate_from_run_line(line) { + crates.push(package); + } + } + } + serde_yaml_ng::Value::Sequence(items) => { + for item in items { + collect_publish_workflow_crates(item, crates); + } + } + serde_yaml_ng::Value::Mapping(map) => { + for value in map.values() { + collect_publish_workflow_crates(value, crates); + } + } + _ => {} + } +} + +fn publish_crate_from_run_line(line: &str) -> Option { + let marker = "scripts/publish-crate.sh"; + let after = line.split_once(marker)?.1; + after + .split_whitespace() + .next() + .map(|package| package.trim_matches(['"', '\'']).to_string()) +} + +fn validate_publish_script(errors: &mut Vec) -> Result<(), String> { + let script_path = Path::new("scripts/publish-crate.sh"); + let script = fs::read_to_string(script_path) + .map_err(|err| format!("failed to read {}: {err}", script_path.display()))?; + let crates = shell_array_values(&script, "publishable_crates").ok_or_else(|| { + format!( + "{} does not define the publishable_crates shell array", + script_path.display() + ) + })?; + let expected = PUBLISHABLE_PACKAGES + .iter() + .map(ToString::to_string) + .collect::>(); + if crates != expected { + errors.push(format!( + "{} publishable_crates is {:?}, expected {:?}", + script_path.display(), + crates, + expected + )); + } + Ok(()) +} + +fn shell_array_values(script: &str, name: &str) -> Option> { + let marker = format!("{name}=("); + let mut values = Vec::new(); + let mut in_array = false; + for raw_line in script.lines() { + let line = raw_line.trim(); + if !in_array { + if line == marker { + in_array = true; + } + continue; + } + if line == ")" { + return Some(values); + } + let line = line.split('#').next()?.trim(); + if line.is_empty() { + continue; + } + values.extend( + line.split_whitespace() + .map(|entry| entry.trim_matches(['"', '\'']).to_string()), + ); + } + None +} + +fn validate_release_docs(errors: &mut Vec) -> Result<(), String> { + let release_doc_path = Path::new("docs/release.md"); + let release_doc = fs::read_to_string(release_doc_path) + .map_err(|err| format!("failed to read {}: {err}", release_doc_path.display()))?; + for package in PUBLISHABLE_PACKAGES { + if !release_doc.contains(&format!("`{package}`")) { + errors.push(format!( + "{} does not document publishable crate `{package}`", + release_doc_path.display() + )); + } + } + for required in [ + "cargo xtask release-integrity", + "CRATES_IO_ALLOW_PUBLISHED_RERUN", + "v", + ] { + if !release_doc.contains(required) { + errors.push(format!( + "{} does not document `{required}`", + release_doc_path.display() + )); + } + } + Ok(()) +} + +fn str_set(values: &[&'static str]) -> BTreeSet<&'static str> { + values.iter().copied().collect() +} + fn release_cpu() -> Result<(), String> { let mut args = vec!["test", "--release"]; for package in CPU_RELEASE_PACKAGES { @@ -257,23 +1317,23 @@ fn release_metal() -> Result<(), String> { } if skip_j2k_metal_runtime_on_hosted_github_macos() { eprintln!( - "skipping signinum-j2k-metal release runtime tests on GitHub-hosted macOS; \ + "skipping j2k-metal release runtime tests on GitHub-hosted macOS; \ self-hosted gpu-validation runs the Metal runtime suite" ); run_cargo_with_env( - &["test", "--release", "-p", "signinum-jpeg-metal"], + &["test", "--release", "-p", "j2k-jpeg-metal"], &[("RUST_TEST_THREADS", "1")], )?; - return run_cargo(&["test", "--release", "-p", "signinum-j2k-metal", "--no-run"]); + return run_cargo(&["test", "--release", "-p", "j2k-metal", "--no-run"]); } run_cargo_with_env( &[ "test", "--release", "-p", - "signinum-jpeg-metal", + "j2k-jpeg-metal", "-p", - "signinum-j2k-metal", + "j2k-metal", ], &[("RUST_TEST_THREADS", "1")], ) @@ -282,7 +1342,7 @@ fn release_metal() -> Result<(), String> { fn skip_j2k_metal_runtime_on_hosted_github_macos() -> bool { env::consts::OS == "macos" && env::var_os("GITHUB_ACTIONS").is_some() - && env::var_os("SIGNINUM_RUN_HOSTED_J2K_METAL_RUNTIME_TESTS").is_none() + && env::var_os("J2K_RUN_HOSTED_J2K_METAL_RUNTIME_TESTS").is_none() } fn coverage() -> Result<(), String> { @@ -290,12 +1350,19 @@ fn coverage() -> Result<(), String> { "llvm-cov", "--workspace", "--all-features", + "--ignore-filename-regex", + GPU_COVERAGE_EXCLUSION_REGEX, "--lcov", "--output-path", "lcov.info", + "--fail-under-lines", + "80", ]) } +const GPU_COVERAGE_EXCLUSION_REGEX: &str = + "crates/j2k-cuda-runtime/|crates/j2k-cuda/|crates/j2k-.*-cuda/|crates/j2k-metal/|crates/j2k-.*-metal/|crates/j2k-metal-support/"; + fn package() -> Result<(), String> { ensure_clean_worktree()?; for package in PUBLISHABLE_PACKAGES { @@ -313,18 +1380,7 @@ fn package() -> Result<(), String> { } fn ensure_clean_worktree() -> Result<(), String> { - let output = Command::new("git") - .args(["status", "--porcelain"]) - .output() - .map_err(|err| format!("failed to start `git status --porcelain`: {err}"))?; - if !output.status.success() { - return Err(format!( - "`git status --porcelain` exited with {}", - output.status - )); - } - - let status = String::from_utf8_lossy(&output.stdout); + let status = process::command_output_os(OsString::from("git"), &["status", "--porcelain"])?; if status.trim().is_empty() { Ok(()) } else { @@ -342,28 +1398,221 @@ fn run_cargo_with_env(args: &[&str], envs: &[(&str, &str)]) -> Result<(), String run_program(cargo(), args, envs) } +fn run_nightly_cargo(args: &[&str]) -> Result<(), String> { + let mut rustup_args = vec!["run", "nightly", "cargo"]; + rustup_args.extend_from_slice(args); + run_program(OsString::from("rustup"), &rustup_args, &[]) +} + +fn run_nightly_cargo_in_dir_owned(dir: &str, args: &[String]) -> Result<(), String> { + let mut rustup_args = vec![ + "run".to_string(), + "nightly".to_string(), + "cargo".to_string(), + ]; + rustup_args.extend_from_slice(args); + run_program_in_dir_owned_with_program(OsString::from("rustup"), dir, &rustup_args, &[]) +} + fn run_program(program: OsString, args: &[&str], envs: &[(&str, &str)]) -> Result<(), String> { - let display = program.to_string_lossy(); - eprintln!("+ {} {}", display, args.join(" ")); - let mut command = Command::new(&program); - command.args(args); - for (key, value) in envs { - command.env(key, value); - } - let status = command - .status() - .map_err(|err| format!("failed to start `{}`: {err}", display))?; - if status.success() { - Ok(()) - } else { - Err(format!("`{}` exited with {status}", display)) - } + process::run_command(program, args, process::CommandContext::new().envs(envs)) +} + +fn run_program_in_dir_owned_with_program( + program: OsString, + dir: &str, + args: &[String], + envs: &[(&str, &str)], +) -> Result<(), String> { + process::run_command_owned( + program, + args, + process::CommandContext::new() + .current_dir(Path::new(dir)) + .envs(envs), + ) } fn cargo() -> OsString { env::var_os("CARGO").unwrap_or_else(|| OsString::from("cargo")) } +fn command_output(program: &str, args: &[&str]) -> Result { + command_output_os(OsString::from(program), args) +} + +fn command_output_allow_failure(program: &str, args: &[&str]) -> Result { + process::command_output_allow_failure(program, args) +} + +fn command_output_os(program: OsString, args: &[&str]) -> Result { + process::command_output_os(program, args) +} + +fn command_output_os_detailed(program: OsString, args: &[&str]) -> Result { + let display = format!("{} {}", program.to_string_lossy(), args.join(" ")); + let output = process::command_output(program, args, process::CommandContext::new())?; + if output.status.success() { + return Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let combined = format!("{stdout}\n{stderr}"); + Err(format!( + "`{display}` exited with {}:\n{}", + output.status, + combined.trim() + )) +} + +fn host_description() -> String { + command_output("uname", &["-a"]) + .unwrap_or_else(|_| format!("{} {}", env::consts::OS, env::consts::ARCH)) +} + +fn workspace_version() -> Result { + let manifest = fs::read_to_string("Cargo.toml") + .map_err(|err| format!("failed to read Cargo.toml: {err}"))?; + manifest + .lines() + .find_map(|line| { + let line = line.trim(); + line.strip_prefix("version") + .and_then(|rest| rest.split('"').nth(1)) + .map(str::to_string) + }) + .ok_or_else(|| "failed to find workspace package version".to_string()) +} + +fn comparator_versions() -> Vec<(String, String)> { + vec![ + ( + "OpenJPEG".to_string(), + comparator_command_version("J2K_OPENJPEG_DECOMPRESS_BIN", "opj_decompress", &["-h"]), + ), + ( + "Grok".to_string(), + env::var("J2K_GROK_ROOT") + .map(|root| format!("configured root: {root}")) + .unwrap_or_else(|_| "unavailable: J2K_GROK_ROOT not set".to_string()), + ), + ( + "libjpeg-turbo".to_string(), + command_output("pkg-config", &["--modversion", "libturbojpeg"]) + .map(|version| format!("pkg-config libturbojpeg {version}")) + .unwrap_or_else(|err| format!("unavailable: {err}")), + ), + ] +} + +fn comparator_command_version(env_var: &str, fallback: &str, args: &[&str]) -> String { + let program = env::var(env_var).unwrap_or_else(|_| fallback.to_string()); + let path = program.clone(); + command_output_allow_failure(&program, args) + .map(|version| format!("{}; path: {path}", best_version_line(&version))) + .unwrap_or_else(|err| format!("unavailable: {err}; path: {path}")) +} + +fn best_version_line(output: &str) -> &str { + output + .lines() + .find(|line| line.contains("compiled against") || line.contains("version")) + .or_else(|| output.lines().find(|line| !line.trim().is_empty())) + .unwrap_or("version unavailable") +} + +fn render_benchmark_report(report: &BenchmarkReport) -> String { + let mut out = String::new(); + writeln!(&mut out, "# Benchmark publication report").unwrap(); + writeln!(&mut out).unwrap(); + writeln!(&mut out, "- command: {}", report.command).unwrap(); + writeln!(&mut out, "- host: {}", report.host).unwrap(); + writeln!(&mut out, "- rustc: {}", one_line(&report.rustc)).unwrap(); + writeln!(&mut out, "- cargo: {}", one_line(&report.cargo)).unwrap(); + writeln!(&mut out, "- crate revision: {}", report.git_revision).unwrap(); + writeln!( + &mut out, + "- workspace version: {}", + report.workspace_version + ) + .unwrap(); + writeln!(&mut out, "- input source: {}", report.input_source).unwrap(); + writeln!( + &mut out, + "- J2K_COMPARE_THREADS: {}", + report.compare_threads + ) + .unwrap(); + writeln!(&mut out).unwrap(); + writeln!(&mut out, "## comparator versions").unwrap(); + for (name, version) in &report.comparator_versions { + writeln!(&mut out, "- {name}: {version}").unwrap(); + } + writeln!(&mut out).unwrap(); + writeln!(&mut out, "## skipped rows").unwrap(); + if report.skipped_rows.is_empty() { + writeln!(&mut out, "- none recorded").unwrap(); + } else { + for row in &report.skipped_rows { + writeln!(&mut out, "- {row}").unwrap(); + } + } + out +} + +fn one_line(value: &str) -> String { + value.lines().next().unwrap_or(value).to_string() +} + +fn split_semicolon_list(value: &str) -> Vec { + value + .split(';') + .map(str::trim) + .filter(|row| !row.is_empty()) + .map(str::to_string) + .collect() +} + +fn rust_sources(root: &Path) -> Result, String> { + let mut out = Vec::new(); + collect_rust_sources(root, &mut out)?; + Ok(out) +} + +fn collect_rust_sources(dir: &Path, out: &mut Vec) -> Result<(), String> { + for entry in + fs::read_dir(dir).map_err(|err| format!("failed to read {}: {err}", dir.display()))? + { + let entry = + entry.map_err(|err| format!("failed to read {} entry: {err}", dir.display()))?; + let path = entry.path(); + if path.is_dir() { + collect_rust_sources(&path, out)?; + } else if path.extension().and_then(|ext| ext.to_str()) == Some("rs") { + out.push(path); + } + } + Ok(()) +} + +fn print_bench_report_help() { + println!( + "usage: cargo xtask bench-report [--command ] [--input-source ] \ + [--skipped-row ]... [--out ]" + ); +} + +fn print_stable_api_help() { + println!( + "usage: cargo xtask stable-api [--write]\n\n\ + Without --write, checks docs/stable-api-1.0.public-api.txt against \ + cargo-public-api output for all 1.0-stable library crates. With \ + --write, refreshes the snapshot. This task must run on macOS so \ + target-gated Metal APIs are included." + ); +} + fn print_help() { println!( "usage: cargo xtask \n\n\ @@ -371,16 +1620,288 @@ fn print_help() { ci fmt, clippy, test, and docs\n\ fmt check rustfmt\n\ clippy run clippy with warnings denied\n\ + clippy-strict run stricter J2K clippy gates\n\ test run workspace tests\n\ + nextest run workspace tests with cargo-nextest\n\ doc build workspace docs with warnings denied\n\ typos run typos\n\ bench-build compile benchmark targets\n\ + bench-report print or write a benchmark publication report\n\ + j2k-bench-signoff run required OpenJPEG/Grok parity and J2K compare bench compile gates\n\ + j2k-perf-guard compare CPU J2K Criterion medians against a baseline git ref\n\ fuzz-build compile fuzz harnesses\n\ + fuzz-run run scheduled fuzz targets with J2K_FUZZ_RUNS\n\ + stable-api check the generated 1.0 public API inventory snapshot\n\ + semver check stable library crates across the 1.0 major-release boundary\n\ deny run cargo-deny\n\ + miri run selected CPU/no_std crates under Miri\n\ + machete run cargo-machete unused-dependency scan\n\ no-std check no_std-compatible codec crates\n\ + unsafe-audit verify docs/unsafe-audit.md lists unsafe Rust sources\n\ + downstream-smoke run facade and transcode examples used by integration docs\n\ + repo-lint run repository policy checks owned by xtask\n\ + release-integrity validate publish membership, docs.rs metadata, workflow order, and release docs\n\ release-cpu run release-mode CPU codec tests\n\ release-metal run release-mode Metal tests on macOS\n\ - coverage generate lcov.info with cargo-llvm-cov\n\ + coverage generate lcov.info with cargo-llvm-cov and fail below 80% line coverage\n\ package preflight publishable package contents from a clean worktree; strict for registry-independent crates and list-only for staged dependencies" ); } + +#[cfg(test)] +mod tests { + use super::perf_guard::{ + compare_estimates, discover_estimates, sync_benchmark_sources, BenchEstimate, + RegressionOutcome, + }; + use std::fs; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn compare_estimates_flags_median_regression_above_threshold() { + let baseline = BenchEstimate { + id: "j2k_public_decode/htj2k_gray8_full_512x512".to_string(), + median_ns: 1_000.0, + median_lower_ns: 990.0, + median_upper_ns: 1_000.0, + }; + let current = BenchEstimate { + id: baseline.id.clone(), + median_ns: 1_120.0, + median_lower_ns: 1_120.0, + median_upper_ns: 1_130.0, + }; + + let outcomes = compare_estimates(&[baseline], &[current], 10.0).unwrap(); + + assert_eq!( + outcomes, + vec![RegressionOutcome { + id: "j2k_public_decode/htj2k_gray8_full_512x512".to_string(), + baseline_ns: 1_000.0, + current_ns: 1_120.0, + delta_percent: 12.0, + enforced: true, + threshold_exceeded: true, + regressed: true, + }] + ); + } + + #[test] + fn compare_estimates_allows_median_delta_at_threshold() { + let baseline = BenchEstimate { + id: "tier1_bitplane_decode/decode_64x64/default".to_string(), + median_ns: 2_000.0, + median_lower_ns: 1_990.0, + median_upper_ns: 2_000.0, + }; + let current = BenchEstimate { + id: baseline.id.clone(), + median_ns: 2_200.0, + median_lower_ns: 2_200.0, + median_upper_ns: 2_210.0, + }; + + let outcomes = compare_estimates(&[baseline], &[current], 10.0).unwrap(); + + assert!(!outcomes[0].regressed); + assert_eq!(outcomes[0].delta_percent, 10.0); + } + + #[test] + fn compare_estimates_reports_missing_current_result() { + let baseline = BenchEstimate { + id: "j2k_public_decode/htj2k_gray8_full_512x512".to_string(), + median_ns: 500.0, + median_lower_ns: 490.0, + median_upper_ns: 510.0, + }; + + let err = compare_estimates(&[baseline], &[], 10.0).unwrap_err(); + + assert!( + err.contains("missing current benchmark result"), + "unexpected error: {err}" + ); + } + + #[test] + fn compare_estimates_requires_confident_material_regression() { + let tiny_baseline = BenchEstimate { + id: "htj2k_refinement_block_decode/ds0_ht_09_b11_sigprop".to_string(), + median_ns: 100.0, + median_lower_ns: 99.0, + median_upper_ns: 101.0, + }; + let tiny_current = BenchEstimate { + id: tiny_baseline.id.clone(), + median_ns: 130.0, + median_lower_ns: 129.0, + median_upper_ns: 131.0, + }; + let noisy_baseline = BenchEstimate { + id: "j2k_public_decode_gray/gray8_full_128x128".to_string(), + median_ns: 610_000.0, + median_lower_ns: 606_000.0, + median_upper_ns: 615_000.0, + }; + let noisy_current = BenchEstimate { + id: noisy_baseline.id.clone(), + median_ns: 673_000.0, + median_lower_ns: 664_000.0, + median_upper_ns: 681_000.0, + }; + + let outcomes = compare_estimates( + &[tiny_baseline, noisy_baseline], + &[tiny_current, noisy_current], + 10.0, + ) + .unwrap(); + + assert!(outcomes.iter().all(|outcome| !outcome.regressed)); + } + + #[test] + fn discover_estimates_reads_criterion_median_point_estimates() { + let root = temp_dir("j2k-perf-guard-test"); + let estimate_path = root + .join("target") + .join("criterion") + .join("j2k_public_decode") + .join("rgb8_full_128x128") + .join("new"); + fs::create_dir_all(&estimate_path).unwrap(); + fs::write( + estimate_path.join("estimates.json"), + r#"{"median":{"point_estimate":321.5,"confidence_interval":{"lower_bound":321.5,"upper_bound":321.5}}}"#, + ) + .unwrap(); + + let estimates = discover_estimates(&root.join("target").join("criterion")).unwrap(); + + assert_eq!( + estimates, + vec![BenchEstimate { + id: "j2k_public_decode/rgb8_full_128x128".to_string(), + median_ns: 321.5, + median_lower_ns: 321.5, + median_upper_ns: 321.5, + }] + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn sync_benchmark_sources_overlays_current_bench_files() { + let root = temp_dir("j2k-perf-guard-sync-test"); + let source = root.join("source"); + let target = root.join("target"); + let jpeg_manifest = "crates/j2k-jpeg/Cargo.toml"; + let cuda_manifest = "crates/j2k-cuda/Cargo.toml"; + let public_bench = "crates/j2k/benches/public_api.rs"; + let jpeg_encode_bench = "crates/j2k-jpeg/benches/encode_cpu.rs"; + let cuda_decode_bench = "crates/j2k-cuda/benches/htj2k_decode.rs"; + let cuda_encode_bench = "crates/j2k-cuda/benches/htj2k_encode.rs"; + let native_bench = "crates/j2k-native/benches/tier1_bitplane.rs"; + let native_sigprop_bench = "crates/j2k-native/benches/htj2k_sigprop_phase.rs"; + let native_fixture = "crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k"; + fs::create_dir_all(source.join("crates/j2k/benches")).unwrap(); + fs::create_dir_all(source.join("crates/j2k-jpeg/benches")).unwrap(); + fs::create_dir_all(source.join("crates/j2k-cuda/benches")).unwrap(); + fs::create_dir_all(source.join("crates/j2k-native/benches")).unwrap(); + fs::create_dir_all(source.join("crates/j2k-native/fixtures/htj2k")).unwrap(); + fs::create_dir_all(target.join("crates/j2k-jpeg")).unwrap(); + fs::create_dir_all(target.join("crates/j2k-cuda")).unwrap(); + fs::create_dir_all(target.join("crates/j2k/benches")).unwrap(); + fs::create_dir_all(target.join("crates/j2k-jpeg/benches")).unwrap(); + fs::create_dir_all(target.join("crates/j2k-cuda/benches")).unwrap(); + fs::create_dir_all(target.join("crates/j2k-native/benches")).unwrap(); + fs::create_dir_all(target.join("crates/j2k-native/fixtures/htj2k")).unwrap(); + fs::write( + source.join(jpeg_manifest), + "[package]\nname = \"j2k-jpeg\"\n\n[[bench]]\nname = \"encode_cpu\"\nharness = false\n", + ) + .unwrap(); + fs::write( + target.join(jpeg_manifest), + "[package]\nname = \"j2k-jpeg\"\n", + ) + .unwrap(); + fs::write( + target.join(cuda_manifest), + "[package]\nname = \"j2k-cuda\"\n", + ) + .unwrap(); + fs::write(source.join(public_bench), "current public bench").unwrap(); + fs::write(source.join(jpeg_encode_bench), "current jpeg encode bench").unwrap(); + fs::write(source.join(cuda_decode_bench), "current cuda decode bench").unwrap(); + fs::write(source.join(cuda_encode_bench), "current cuda encode bench").unwrap(); + fs::write(source.join(native_bench), "current native bench").unwrap(); + fs::write(source.join(native_sigprop_bench), "current sigprop bench").unwrap(); + fs::write(source.join(native_fixture), "current fixture").unwrap(); + fs::write(target.join(public_bench), "old public bench").unwrap(); + fs::write(target.join(jpeg_encode_bench), "old jpeg encode bench").unwrap(); + fs::write(target.join(cuda_decode_bench), "old cuda decode bench").unwrap(); + fs::write(target.join(cuda_encode_bench), "old cuda encode bench").unwrap(); + fs::write(target.join(native_bench), "old native bench").unwrap(); + fs::write(target.join(native_sigprop_bench), "old sigprop bench").unwrap(); + fs::write(target.join(native_fixture), "old fixture").unwrap(); + + sync_benchmark_sources(&source, &target).unwrap(); + + assert_eq!( + fs::read_to_string(target.join(public_bench)).unwrap(), + "current public bench" + ); + assert_eq!( + fs::read_to_string(target.join(jpeg_encode_bench)).unwrap(), + "current jpeg encode bench" + ); + let target_jpeg_manifest = fs::read_to_string(target.join(jpeg_manifest)).unwrap(); + assert!( + target_jpeg_manifest.contains("name = \"encode_cpu\"") + && target_jpeg_manifest.contains("harness = false"), + "baseline overlay must register encode_cpu as a Criterion bench" + ); + assert_eq!( + fs::read_to_string(target.join(cuda_decode_bench)).unwrap(), + "current cuda decode bench" + ); + assert_eq!( + fs::read_to_string(target.join(cuda_encode_bench)).unwrap(), + "current cuda encode bench" + ); + let target_cuda_manifest = fs::read_to_string(target.join(cuda_manifest)).unwrap(); + assert!( + target_cuda_manifest.contains("name = \"htj2k_decode\"") + && target_cuda_manifest.contains("name = \"htj2k_encode\"") + && target_cuda_manifest.contains("required-features = [\"cuda-runtime\"]"), + "baseline overlay must register CUDA HTJ2K Criterion benches" + ); + assert_eq!( + fs::read_to_string(target.join(native_bench)).unwrap(), + "current native bench" + ); + assert_eq!( + fs::read_to_string(target.join(native_sigprop_bench)).unwrap(), + "current sigprop bench" + ); + assert_eq!( + fs::read_to_string(target.join(native_fixture)).unwrap(), + "current fixture" + ); + let _ = fs::remove_dir_all(root); + } + + fn temp_dir(name: &str) -> std::path::PathBuf { + let mut dir = std::env::temp_dir(); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + dir.push(format!("{name}-{}-{nanos}", std::process::id())); + dir + } +} diff --git a/xtask/src/perf_guard.rs b/xtask/src/perf_guard.rs new file mode 100644 index 00000000..3149d0aa --- /dev/null +++ b/xtask/src/perf_guard.rs @@ -0,0 +1,1117 @@ +use std::{ + collections::BTreeMap, + env, + ffi::OsString, + fs, + path::{Path, PathBuf}, +}; + +use crate::process::{self, CommandContext}; + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct BenchEstimate { + pub(crate) id: String, + pub(crate) median_ns: f64, + pub(crate) median_lower_ns: f64, + pub(crate) median_upper_ns: f64, +} + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct RegressionOutcome { + pub(crate) id: String, + pub(crate) baseline_ns: f64, + pub(crate) current_ns: f64, + pub(crate) delta_percent: f64, + pub(crate) enforced: bool, + pub(crate) threshold_exceeded: bool, + pub(crate) regressed: bool, +} + +#[derive(Debug, Clone)] +struct PerfGuardOptions { + mode: PerfGuardMode, + threshold_percent: f64, + quick: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum PerfGuardMode { + GitRef { baseline_ref: String }, + RecordCurrent { name: String }, + CompareCurrent { name: String }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct BenchCommand { + package: &'static str, + bench: &'static str, + filter: Option<&'static str>, + features: Option<&'static str>, + env: &'static [(&'static str, &'static str)], +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct BenchManifestStanza { + path: &'static str, + bench_name: &'static str, + stanza: &'static str, +} + +const DEFAULT_BASELINE_REF: &str = "j2k-bench-original"; +const DEFAULT_THRESHOLD_PERCENT: f64 = 10.0; +const MIN_ABSOLUTE_REGRESSION_NS: f64 = 100.0; +const BENCH_COMMANDS: &[BenchCommand] = &[ + BenchCommand { + package: "j2k", + bench: "public_api", + filter: None, + features: None, + env: &[], + }, + BenchCommand { + package: "j2k-jpeg", + bench: "encode_cpu", + filter: Some("jpeg_cpu_encode_runtime/"), + features: None, + env: &[], + }, + BenchCommand { + package: "j2k-native", + bench: "tier1_bitplane", + filter: Some("htj2k_cleanup_decode/"), + features: None, + env: &[], + }, + BenchCommand { + package: "j2k-native", + bench: "tier1_bitplane", + filter: Some("htj2k_refinement_fixture_decode"), + features: None, + env: &[], + }, + BenchCommand { + package: "j2k-native", + bench: "tier1_bitplane", + filter: Some("htj2k_refinement_block_decode"), + features: None, + env: &[], + }, + BenchCommand { + package: "j2k-native", + bench: "tier1_bitplane", + filter: Some("htj2k_cleanup_encode/"), + features: None, + env: &[], + }, + BenchCommand { + package: "j2k-native", + bench: "tier1_bitplane", + filter: Some("htj2k_cleanup_encode_distribution"), + features: None, + env: &[], + }, + BenchCommand { + package: "j2k-native", + bench: "htj2k_sigprop_phase", + filter: None, + features: None, + env: &[], + }, + BenchCommand { + package: "j2k-cuda", + bench: "htj2k_decode", + filter: Some("j2k_cuda_htj2k_"), + features: Some("cuda-runtime"), + env: &[], + }, + BenchCommand { + package: "j2k-cuda", + bench: "htj2k_encode", + filter: Some("j2k_cuda_htj2k_"), + features: Some("cuda-runtime"), + env: &[], + }, +]; +const BENCH_SOURCE_FILES: &[&str] = &[ + "crates/j2k/benches/public_api.rs", + "crates/j2k-jpeg/benches/encode_cpu.rs", + "crates/j2k-native/benches/tier1_bitplane.rs", + "crates/j2k-native/benches/htj2k_sigprop_phase.rs", + "crates/j2k-native/fixtures/htj2k/openhtj2k_ds0_ht_09_b11.j2k", + "crates/j2k-cuda/benches/htj2k_decode.rs", + "crates/j2k-cuda/benches/htj2k_encode.rs", +]; +const BENCH_MANIFEST_STANZAS: &[BenchManifestStanza] = &[ + BenchManifestStanza { + path: "crates/j2k-jpeg/Cargo.toml", + bench_name: "encode_cpu", + stanza: "[[bench]]\nname = \"encode_cpu\"\nharness = false\n", + }, + BenchManifestStanza { + path: "crates/j2k-cuda/Cargo.toml", + bench_name: "htj2k_decode", + stanza: "[[bench]]\nname = \"htj2k_decode\"\nharness = false\n", + }, + BenchManifestStanza { + path: "crates/j2k-cuda/Cargo.toml", + bench_name: "htj2k_encode", + stanza: "[[bench]]\nname = \"htj2k_encode\"\nharness = false\nrequired-features = [\"cuda-runtime\"]\n", + }, +]; + +pub(crate) fn j2k_perf_guard(args: impl Iterator) -> Result<(), String> { + let options = PerfGuardOptions::parse(args)?; + let root = repo_root()?; + let perf_root = root.join("target").join("j2k-perf"); + fs::create_dir_all(&perf_root) + .map_err(|err| format!("failed to create {}: {err}", perf_root.display()))?; + + let outcomes = match &options.mode { + PerfGuardMode::GitRef { baseline_ref } => { + let baseline_worktree = perf_root.join("baseline-worktree"); + recreate_baseline_worktree(&root, &baseline_worktree, baseline_ref)?; + sync_benchmark_sources(&root, &baseline_worktree)?; + + let baseline_target = perf_root.join("baseline-target"); + let current_target = perf_root.join("current-target"); + reset_dir(&baseline_target)?; + reset_dir(¤t_target)?; + + run_benches(&baseline_worktree, &baseline_target, options.quick)?; + run_benches(&root, ¤t_target, options.quick)?; + + let baseline = discover_estimates(&baseline_target.join("criterion"))?; + let current = discover_estimates(¤t_target.join("criterion"))?; + compare_estimates(&baseline, ¤t, options.threshold_percent)? + } + PerfGuardMode::RecordCurrent { name } => { + let target = perf_root.join("current-record-target"); + reset_dir(&target)?; + run_benches(&root, &target, options.quick)?; + let estimates = discover_estimates(&target.join("criterion"))?; + let snapshot = current_snapshot_path(&perf_root, name)?; + write_estimate_snapshot(&snapshot, &estimates)?; + eprintln!( + "Recorded current-tree performance baseline `{name}` at {}", + snapshot.display() + ); + return Ok(()); + } + PerfGuardMode::CompareCurrent { name } => { + let snapshot = current_snapshot_path(&perf_root, name)?; + let baseline = read_estimate_snapshot(&snapshot)?; + let target = perf_root.join("current-compare-target"); + reset_dir(&target)?; + run_benches(&root, &target, options.quick)?; + let current = discover_estimates(&target.join("criterion"))?; + compare_estimates(&baseline, ¤t, options.threshold_percent)? + } + }; + emit_report(&outcomes, options.threshold_percent); + + if outcomes.iter().any(|outcome| outcome.regressed) { + Err("Codec performance guard found regressions".to_string()) + } else { + Ok(()) + } +} + +fn current_snapshot_path(perf_root: &Path, name: &str) -> Result { + validate_snapshot_name(name)?; + Ok(perf_root + .join("current-tree-baselines") + .join(format!("{name}.json"))) +} + +fn validate_snapshot_name(name: &str) -> Result<(), String> { + if name.is_empty() { + return Err("current-tree baseline name must not be empty".to_string()); + } + if name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) + { + Ok(()) + } else { + Err(format!( + "current-tree baseline name `{name}` may only contain ASCII letters, digits, '.', '-', and '_'" + )) + } +} + +fn write_estimate_snapshot(path: &Path, estimates: &[BenchEstimate]) -> Result<(), String> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .map_err(|err| format!("failed to create {}: {err}", parent.display()))?; + } + let mut sorted = estimates.to_vec(); + sorted.sort_by(|a, b| a.id.cmp(&b.id)); + let values = sorted + .iter() + .map(|estimate| { + serde_json::json!({ + "id": estimate.id, + "median_ns": estimate.median_ns, + "median_lower_ns": estimate.median_lower_ns, + "median_upper_ns": estimate.median_upper_ns, + }) + }) + .collect::>(); + let value = serde_json::json!({ + "version": 1, + "estimates": values, + }); + let data = serde_json::to_string_pretty(&value) + .map_err(|err| format!("failed to serialize estimate snapshot: {err}"))?; + fs::write(path, format!("{data}\n")) + .map_err(|err| format!("failed to write {}: {err}", path.display())) +} + +fn read_estimate_snapshot(path: &Path) -> Result, String> { + let data = fs::read_to_string(path) + .map_err(|err| format!("failed to read {}: {err}", path.display()))?; + let value: serde_json::Value = serde_json::from_str(&data) + .map_err(|err| format!("failed to parse {}: {err}", path.display()))?; + let version = value + .get("version") + .and_then(serde_json::Value::as_u64) + .ok_or_else(|| format!("{} is missing version", path.display()))?; + if version != 1 { + return Err(format!( + "{} has unsupported estimate snapshot version {version}", + path.display() + )); + } + let raw_estimates = value + .get("estimates") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| format!("{} is missing estimates", path.display()))?; + let mut estimates = Vec::with_capacity(raw_estimates.len()); + for raw in raw_estimates { + let id = raw + .get("id") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| format!("{} contains an estimate without id", path.display()))? + .to_string(); + let median_ns = raw + .get("median_ns") + .and_then(serde_json::Value::as_f64) + .ok_or_else(|| format!("{} estimate {id} is missing median_ns", path.display()))?; + let median_lower_ns = raw + .get("median_lower_ns") + .and_then(serde_json::Value::as_f64) + .ok_or_else(|| { + format!( + "{} estimate {id} is missing median_lower_ns", + path.display() + ) + })?; + let median_upper_ns = raw + .get("median_upper_ns") + .and_then(serde_json::Value::as_f64) + .ok_or_else(|| { + format!( + "{} estimate {id} is missing median_upper_ns", + path.display() + ) + })?; + estimates.push(BenchEstimate { + id, + median_ns, + median_lower_ns, + median_upper_ns, + }); + } + estimates.sort_by(|a, b| a.id.cmp(&b.id)); + Ok(estimates) +} + +pub(crate) fn sync_benchmark_sources(source_root: &Path, target_root: &Path) -> Result<(), String> { + for relative in BENCH_SOURCE_FILES { + let source = source_root.join(relative); + let target = target_root.join(relative); + let parent = target.parent().ok_or_else(|| { + format!( + "benchmark source target has no parent directory: {}", + target.display() + ) + })?; + fs::create_dir_all(parent) + .map_err(|err| format!("failed to create {}: {err}", parent.display()))?; + fs::copy(&source, &target).map_err(|err| { + format!( + "failed to copy benchmark source {} to {}: {err}", + source.display(), + target.display() + ) + })?; + } + + for manifest in BENCH_MANIFEST_STANZAS { + ensure_benchmark_manifest_stanza(target_root, *manifest)?; + } + + Ok(()) +} + +fn ensure_benchmark_manifest_stanza( + target_root: &Path, + manifest: BenchManifestStanza, +) -> Result<(), String> { + let path = target_root.join(manifest.path); + let mut contents = fs::read_to_string(&path) + .map_err(|err| format!("failed to read {}: {err}", path.display()))?; + if contents.contains(manifest.stanza) { + return Ok(()); + } + if contents.contains(&format!("name = \"{}\"", manifest.bench_name)) { + return Err(format!( + "{} declares benchmark `{}` without the required Criterion harness stanza", + path.display(), + manifest.bench_name + )); + } + + if !contents.ends_with('\n') { + contents.push('\n'); + } + contents.push('\n'); + contents.push_str(manifest.stanza); + fs::write(&path, contents).map_err(|err| format!("failed to write {}: {err}", path.display())) +} + +pub(crate) fn compare_estimates( + baseline: &[BenchEstimate], + current: &[BenchEstimate], + threshold_percent: f64, +) -> Result, String> { + let baseline_by_id = baseline + .iter() + .map(|estimate| (estimate.id.as_str(), estimate)) + .collect::>(); + let current_by_id = current + .iter() + .map(|estimate| (estimate.id.as_str(), estimate)) + .collect::>(); + let mut outcomes = Vec::with_capacity(baseline.len()); + for base in baseline { + let Some(now) = current_by_id.get(base.id.as_str()) else { + if is_enforced_perf_id(&base.id) { + return Err(format!("missing current benchmark result for {}", base.id)); + } + continue; + }; + if base.median_ns <= 0.0 { + return Err(format!( + "baseline benchmark {} has non-positive median {}", + base.id, base.median_ns + )); + } + let delta_percent = ((now.median_ns - base.median_ns) / base.median_ns) * 100.0; + let confident_absolute_delta_ns = now.median_lower_ns - base.median_upper_ns; + let confident_delta_percent = (confident_absolute_delta_ns / base.median_upper_ns) * 100.0; + let enforced = is_enforced_perf_id(&base.id); + let threshold_exceeded = confident_delta_percent > threshold_percent + && confident_absolute_delta_ns > MIN_ABSOLUTE_REGRESSION_NS; + outcomes.push(RegressionOutcome { + id: base.id.clone(), + baseline_ns: base.median_ns, + current_ns: now.median_ns, + delta_percent, + enforced, + threshold_exceeded, + regressed: enforced && threshold_exceeded, + }); + } + for now in current { + if is_enforced_perf_id(&now.id) && !baseline_by_id.contains_key(now.id.as_str()) { + return Err(format!("missing baseline benchmark result for {}", now.id)); + } + } + Ok(outcomes) +} + +fn is_enforced_perf_id(id: &str) -> bool { + let id = CriterionPerfId::new(id); + if REPORT_ONLY_PERF_GROUPS.contains(&id.group) { + return false; + } + ENFORCED_PERF_GROUPS.contains(&id.group) + || (id.group == "j2k_public_decode" + && id.first_case_segment().is_some_and(segment_has_htj2k_token)) + || id.has_htj2k_token() +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct CriterionPerfId<'a> { + group: &'a str, + rest: &'a str, +} + +impl<'a> CriterionPerfId<'a> { + fn new(id: &'a str) -> Self { + let (group, rest) = id.split_once('/').unwrap_or((id, "")); + Self { group, rest } + } + + fn first_case_segment(self) -> Option<&'a str> { + self.rest.split('/').find(|segment| !segment.is_empty()) + } + + fn has_htj2k_token(self) -> bool { + self.rest.split('/').any(segment_has_htj2k_token) + } +} + +const ENFORCED_PERF_GROUPS: &[&str] = &[ + "jpeg_cpu_encode_runtime", + "htj2k_cleanup_decode", + "htj2k_cleanup_encode", + "htj2k_cleanup_encode_distribution", + "htj2k_refinement_fixture_decode", + "htj2k_refinement_block_decode", + "htj2k_refinement_sigprop_phase", + "htj2k_cpuupload_decode_batch", +]; + +const REPORT_ONLY_PERF_GROUPS: &[&str] = &[ + "htj2k_cleanup_encode_parallel_batch_size", + "htj2k_region_scaled_plan_build", + "htj2k_feeder_coalesce", + "htj2k_metal_route", + "wsi_tile_batch_region_scaled_rgb_q4", +]; + +fn segment_has_htj2k_token(segment: &str) -> bool { + segment.split('_').any(|token| token == "htj2k") +} + +pub(crate) fn discover_estimates(criterion_root: &Path) -> Result, String> { + let mut out = Vec::new(); + discover_estimates_inner(criterion_root, criterion_root, &mut out)?; + out.sort_by(|a, b| a.id.cmp(&b.id)); + Ok(out) +} + +fn discover_estimates_inner( + criterion_root: &Path, + dir: &Path, + out: &mut Vec, +) -> Result<(), String> { + if !dir.exists() { + return Err(format!( + "Criterion output directory does not exist: {}", + criterion_root.display() + )); + } + for entry in + fs::read_dir(dir).map_err(|err| format!("failed to read {}: {err}", dir.display()))? + { + let entry = + entry.map_err(|err| format!("failed to read {} entry: {err}", dir.display()))?; + let path = entry.path(); + if path.is_dir() { + discover_estimates_inner(criterion_root, &path, out)?; + continue; + } + if path.file_name().and_then(|name| name.to_str()) != Some("estimates.json") { + continue; + } + if path + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + != Some("new") + { + continue; + } + let id = estimate_id(criterion_root, &path)?; + let (median_ns, median_lower_ns, median_upper_ns) = read_median_estimate(&path)?; + out.push(BenchEstimate { + id, + median_ns, + median_lower_ns, + median_upper_ns, + }); + } + Ok(()) +} + +fn estimate_id(criterion_root: &Path, estimate_path: &Path) -> Result { + let bench_path = estimate_path + .parent() + .and_then(Path::parent) + .ok_or_else(|| { + format!( + "invalid Criterion estimate path {}", + estimate_path.display() + ) + })?; + let rel = bench_path.strip_prefix(criterion_root).map_err(|err| { + format!( + "failed to strip Criterion root {} from {}: {err}", + criterion_root.display(), + bench_path.display() + ) + })?; + let mut parts = Vec::new(); + for component in rel.components() { + parts.push(component.as_os_str().to_string_lossy().into_owned()); + } + Ok(parts.join("/")) +} + +fn read_median_estimate(path: &Path) -> Result<(f64, f64, f64), String> { + let data = fs::read_to_string(path) + .map_err(|err| format!("failed to read {}: {err}", path.display()))?; + let value: serde_json::Value = serde_json::from_str(&data) + .map_err(|err| format!("failed to parse {}: {err}", path.display()))?; + let median = value + .get("median") + .ok_or_else(|| format!("{} is missing median", path.display()))?; + let point_estimate = median + .get("point_estimate") + .and_then(serde_json::Value::as_f64) + .ok_or_else(|| format!("{} is missing median.point_estimate", path.display()))?; + let confidence_interval = median + .get("confidence_interval") + .ok_or_else(|| format!("{} is missing median.confidence_interval", path.display()))?; + let lower_bound = confidence_interval + .get("lower_bound") + .and_then(serde_json::Value::as_f64) + .ok_or_else(|| { + format!( + "{} is missing median.confidence_interval.lower_bound", + path.display() + ) + })?; + let upper_bound = confidence_interval + .get("upper_bound") + .and_then(serde_json::Value::as_f64) + .ok_or_else(|| { + format!( + "{} is missing median.confidence_interval.upper_bound", + path.display() + ) + })?; + Ok((point_estimate, lower_bound, upper_bound)) +} + +fn emit_report(outcomes: &[RegressionOutcome], threshold_percent: f64) { + eprintln!("Codec performance guard threshold: +{threshold_percent:.2}% median"); + for outcome in outcomes { + let status = if outcome.regressed { + "REGRESSED" + } else if outcome.threshold_exceeded && !outcome.enforced { + "report" + } else { + "ok" + }; + eprintln!( + "{status:9} {:>9.2}% baseline={:.2}ns current={:.2}ns {}", + outcome.delta_percent, outcome.baseline_ns, outcome.current_ns, outcome.id + ); + } +} + +fn run_benches(workdir: &Path, target_dir: &Path, quick: bool) -> Result<(), String> { + for bench in BENCH_COMMANDS { + let args = bench_args(*bench, quick); + process::run_command( + cargo(), + &args, + CommandContext::new() + .current_dir(workdir) + .target_dir(target_dir) + .envs(bench.env), + )?; + } + Ok(()) +} + +fn bench_args(bench: BenchCommand, quick: bool) -> Vec<&'static str> { + let mut args = vec!["bench", "-p", bench.package, "--bench", bench.bench]; + if let Some(features) = bench.features { + args.push("--features"); + args.push(features); + } + if bench.filter.is_some() || quick { + args.push("--"); + if let Some(filter) = bench.filter { + args.push(filter); + } + if quick { + args.push("--quick"); + } + } + args +} + +fn recreate_baseline_worktree( + root: &Path, + worktree: &Path, + baseline_ref: &str, +) -> Result<(), String> { + if worktree.exists() { + process::run_command( + OsString::from("git"), + &["worktree", "remove", "--force", path_str(worktree)?], + CommandContext::new().current_dir(root), + )?; + } + process::run_command( + OsString::from("git"), + &[ + "worktree", + "add", + "--detach", + path_str(worktree)?, + baseline_ref, + ], + CommandContext::new().current_dir(root), + ) +} + +fn reset_dir(path: &Path) -> Result<(), String> { + if path.exists() { + fs::remove_dir_all(path) + .map_err(|err| format!("failed to remove {}: {err}", path.display()))?; + } + fs::create_dir_all(path).map_err(|err| format!("failed to create {}: {err}", path.display())) +} + +fn repo_root() -> Result { + let path = + process::command_output_os(OsString::from("git"), &["rev-parse", "--show-toplevel"])?; + Ok(PathBuf::from(path)) +} + +impl PerfGuardOptions { + fn parse(mut args: impl Iterator) -> Result { + let mut options = Self { + mode: PerfGuardMode::GitRef { + baseline_ref: DEFAULT_BASELINE_REF.to_string(), + }, + threshold_percent: DEFAULT_THRESHOLD_PERCENT, + quick: false, + }; + while let Some(arg) = args.next() { + match arg.as_str() { + "--baseline-ref" => { + let baseline_ref = args + .next() + .ok_or_else(|| "--baseline-ref requires a value".to_string())?; + options.set_mode(PerfGuardMode::GitRef { baseline_ref })?; + } + "--record-current" => { + let name = args + .next() + .ok_or_else(|| "--record-current requires a value".to_string())?; + validate_snapshot_name(&name)?; + options.set_mode(PerfGuardMode::RecordCurrent { name })?; + } + "--compare-current" => { + let name = args + .next() + .ok_or_else(|| "--compare-current requires a value".to_string())?; + validate_snapshot_name(&name)?; + options.set_mode(PerfGuardMode::CompareCurrent { name })?; + } + "--threshold-percent" => { + let value = args + .next() + .ok_or_else(|| "--threshold-percent requires a value".to_string())?; + options.threshold_percent = value + .parse::() + .map_err(|err| format!("invalid --threshold-percent `{value}`: {err}"))?; + if options.threshold_percent < 0.0 { + return Err("--threshold-percent must be non-negative".to_string()); + } + } + "--quick" => options.quick = true, + "--help" | "-h" => return Err(help_text()), + other => { + return Err(format!( + "unknown j2k-perf-guard argument `{other}`\n{}", + help_text() + )) + } + } + } + Ok(options) + } + + fn set_mode(&mut self, mode: PerfGuardMode) -> Result<(), String> { + if self.mode + != (PerfGuardMode::GitRef { + baseline_ref: DEFAULT_BASELINE_REF.to_string(), + }) + { + return Err("choose only one baseline mode".to_string()); + } + self.mode = mode; + Ok(()) + } +} + +fn help_text() -> String { + "usage: cargo xtask j2k-perf-guard [--baseline-ref REF | --record-current NAME | --compare-current NAME] [--threshold-percent N] [--quick]".to_string() +} + +fn path_str(path: &Path) -> Result<&str, String> { + path.to_str() + .ok_or_else(|| format!("path is not valid UTF-8: {}", path.display())) +} + +fn cargo() -> OsString { + env::var_os("CARGO").unwrap_or_else(|| OsString::from("cargo")) +} + +#[cfg(test)] +mod tests { + use super::{ + bench_args, compare_estimates, is_enforced_perf_id, read_estimate_snapshot, + write_estimate_snapshot, BenchCommand, BenchEstimate, PerfGuardMode, PerfGuardOptions, + BENCH_COMMANDS, + }; + use std::fs; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn perf_guard_parses_current_tree_record_mode() { + let options = PerfGuardOptions::parse( + ["--record-current", "htj2k-roi-baseline", "--quick"] + .into_iter() + .map(str::to_string), + ) + .unwrap(); + + assert_eq!( + options.mode, + PerfGuardMode::RecordCurrent { + name: "htj2k-roi-baseline".to_string() + } + ); + assert!(options.quick); + } + + #[test] + fn perf_guard_parses_current_tree_compare_mode() { + let options = PerfGuardOptions::parse( + [ + "--compare-current", + "htj2k-roi-baseline", + "--threshold-percent", + "7.5", + ] + .into_iter() + .map(str::to_string), + ) + .unwrap(); + + assert_eq!( + options.mode, + PerfGuardMode::CompareCurrent { + name: "htj2k-roi-baseline".to_string() + } + ); + assert_eq!(options.threshold_percent, 7.5); + } + + #[test] + fn perf_guard_rejects_multiple_baseline_modes() { + let error = PerfGuardOptions::parse( + ["--record-current", "one", "--compare-current", "two"] + .into_iter() + .map(str::to_string), + ) + .unwrap_err(); + + assert!( + error.contains("choose only one baseline mode"), + "unexpected error: {error}" + ); + } + + #[test] + fn estimate_snapshot_round_trips_sorted_estimates() { + let root = temp_dir("j2k-perf-snapshot-test"); + let path = root.join("baselines").join("htj2k.json"); + let estimates = vec![ + BenchEstimate { + id: "z_group/z_case".to_string(), + median_ns: 200.0, + median_lower_ns: 190.0, + median_upper_ns: 210.0, + }, + BenchEstimate { + id: "a_group/a_case".to_string(), + median_ns: 100.0, + median_lower_ns: 95.0, + median_upper_ns: 105.0, + }, + ]; + + write_estimate_snapshot(&path, &estimates).unwrap(); + let round_trip = read_estimate_snapshot(&path).unwrap(); + + assert_eq!( + round_trip, + vec![ + BenchEstimate { + id: "a_group/a_case".to_string(), + median_ns: 100.0, + median_lower_ns: 95.0, + median_upper_ns: 105.0, + }, + BenchEstimate { + id: "z_group/z_case".to_string(), + median_ns: 200.0, + median_lower_ns: 190.0, + median_upper_ns: 210.0, + }, + ] + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn perf_guard_tracks_htj2k_maturation_benchmarks() { + let expected = [ + BenchCommand { + package: "j2k-jpeg", + bench: "encode_cpu", + filter: Some("jpeg_cpu_encode_runtime/"), + features: None, + env: &[], + }, + BenchCommand { + package: "j2k-native", + bench: "tier1_bitplane", + filter: Some("htj2k_cleanup_encode/"), + features: None, + env: &[], + }, + BenchCommand { + package: "j2k-native", + bench: "tier1_bitplane", + filter: Some("htj2k_cleanup_decode/"), + features: None, + env: &[], + }, + BenchCommand { + package: "j2k-native", + bench: "htj2k_sigprop_phase", + filter: None, + features: None, + env: &[], + }, + ]; + + for command in expected { + assert!( + BENCH_COMMANDS.contains(&command), + "J2K perf guard must track {command:?}" + ); + } + } + + #[test] + fn perf_guard_tracks_cuda_htj2k_benchmarks() { + for (bench, filter) in [ + ("htj2k_decode", "j2k_cuda_htj2k_"), + ("htj2k_encode", "j2k_cuda_htj2k_"), + ] { + assert!( + BENCH_COMMANDS.iter().any(|command| { + command.package == "j2k-cuda" + && command.bench == bench + && command.filter == Some(filter) + && command.features == Some("cuda-runtime") + }), + "J2K perf guard must track CUDA HTJ2K benchmark `{bench}`" + ); + } + } + + #[test] + fn perf_guard_errors_when_enforced_result_is_missing() { + let error = compare_estimates( + &[estimate( + "htj2k_cleanup_encode/encode_64x64/2459041792", + 1_000.0, + )], + &[], + 10.0, + ) + .unwrap_err(); + + assert!( + error.contains("missing current benchmark result"), + "unexpected error: {error}" + ); + } + + #[test] + fn perf_guard_errors_when_enforced_baseline_result_is_missing() { + let error = compare_estimates( + &[], + &[estimate( + "jpeg_cpu_encode_runtime/rgb8_512_420_default", + 1_000.0, + )], + 10.0, + ) + .unwrap_err(); + + assert!( + error.contains("missing baseline benchmark result"), + "unexpected error: {error}" + ); + } + + #[test] + fn filtered_bench_command_passes_filter_before_quick_flag() { + let command = BenchCommand { + package: "j2k-native", + bench: "tier1_bitplane", + filter: Some("htj2k_cleanup_encode/"), + features: None, + env: &[], + }; + + assert_eq!( + bench_args(command, true), + vec![ + "bench", + "-p", + "j2k-native", + "--bench", + "tier1_bitplane", + "--", + "htj2k_cleanup_encode/", + "--quick", + ] + ); + } + + #[test] + fn feature_bench_command_passes_features_before_filter_separator() { + let command = BenchCommand { + package: "j2k-cuda", + bench: "htj2k_encode", + filter: Some("j2k_cuda_htj2k_"), + features: Some("cuda-runtime"), + env: &[], + }; + + assert_eq!( + bench_args(command, true), + vec![ + "bench", + "-p", + "j2k-cuda", + "--bench", + "htj2k_encode", + "--features", + "cuda-runtime", + "--", + "j2k_cuda_htj2k_", + "--quick", + ] + ); + } + + #[test] + fn perf_guard_enforces_stable_htj2k_rows_only() { + assert!(is_enforced_perf_id( + "jpeg_cpu_encode_runtime/rgb8_512_420_default" + )); + assert!(is_enforced_perf_id( + "htj2k_cleanup_encode/encode_64x64/2459041792" + )); + assert!(!is_enforced_perf_id( + "wsi_tile_batch_region_scaled_rgb_q4/j2k_htj2k_rgb_512_batch_16" + )); + assert!(!is_enforced_perf_id( + "htj2k_region_scaled_plan_build/j2k-metal-resident" + )); + assert!(!is_enforced_perf_id( + "htj2k_feeder_coalesce/j2k-metal-resident" + )); + assert!(!is_enforced_perf_id( + "htj2k_cleanup_encode_parallel_batch_size/rayon_par_iter_global_blocks/128" + )); + assert!(!is_enforced_perf_id( + "tier1_bitplane_encode/encode_64x64/default" + )); + assert!(!is_enforced_perf_id( + "wsi_tile_batch_region_scaled_rgb_q4/j2k-cpu-staged-metal_htj2k_rgb_512_batch_16" + )); + assert!(!is_enforced_perf_id( + "htj2k_metal_route/j2k-metal-resident_htj2k_rgb_512_batch_16" + )); + assert!(!is_enforced_perf_id( + "j2k_public_decode/nothtj2k_accidental_substring" + )); + assert!(is_enforced_perf_id("j2k_public_decode/htj2k_rgb8_lossless")); + assert!(is_enforced_perf_id( + "j2k_public_cpu_encode_matrix/rgb8_512_htj2k_external" + )); + } + + #[test] + fn perf_guard_reports_out_of_scope_regressions_without_failing() { + let baseline = vec![ + estimate("htj2k_cleanup_encode/encode_64x64/2459041792", 1_000.0), + estimate( + "htj2k_cleanup_encode_parallel_batch_size/rayon_par_iter_global_blocks/128", + 1_000.0, + ), + estimate("tier1_bitplane_encode/encode_64x64/default", 1_000.0), + estimate( + "wsi_tile_batch_region_scaled_rgb_q4/j2k-cpu-staged-metal_htj2k_rgb_512_batch_16", + 1_000.0, + ), + ]; + let current = baseline + .iter() + .map(|estimate| BenchEstimate { + id: estimate.id.clone(), + median_ns: 1_300.0, + median_lower_ns: 1_300.0, + median_upper_ns: 1_310.0, + }) + .collect::>(); + + let outcomes = compare_estimates(&baseline, ¤t, 10.0).unwrap(); + + assert!(outcomes[0].enforced); + assert!(outcomes[0].threshold_exceeded); + assert!(outcomes[0].regressed); + for outcome in &outcomes[1..] { + assert!(!outcome.enforced, "{outcome:?}"); + assert!(outcome.threshold_exceeded, "{outcome:?}"); + assert!(!outcome.regressed, "{outcome:?}"); + } + } + + fn temp_dir(name: &str) -> std::path::PathBuf { + let mut dir = std::env::temp_dir(); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + dir.push(format!("{name}-{}-{nanos}", std::process::id())); + dir + } + + fn estimate(id: &str, median_ns: f64) -> BenchEstimate { + BenchEstimate { + id: id.to_string(), + median_ns, + median_lower_ns: median_ns, + median_upper_ns: median_ns, + } + } +} diff --git a/xtask/src/process.rs b/xtask/src/process.rs new file mode 100644 index 00000000..131083e9 --- /dev/null +++ b/xtask/src/process.rs @@ -0,0 +1,139 @@ +use std::{ + ffi::OsString, + path::Path, + process::{Command, Output}, +}; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct CommandContext<'a> { + current_dir: Option<&'a Path>, + envs: &'a [(&'a str, &'a str)], + target_dir: Option<&'a Path>, +} + +impl<'a> CommandContext<'a> { + pub(crate) const fn new() -> Self { + Self { + current_dir: None, + envs: &[], + target_dir: None, + } + } + + pub(crate) const fn current_dir(mut self, current_dir: &'a Path) -> Self { + self.current_dir = Some(current_dir); + self + } + + pub(crate) const fn envs(mut self, envs: &'a [(&'a str, &'a str)]) -> Self { + self.envs = envs; + self + } + + pub(crate) const fn target_dir(mut self, target_dir: &'a Path) -> Self { + self.target_dir = Some(target_dir); + self + } +} + +pub(crate) fn run_command( + program: OsString, + args: &[&str], + context: CommandContext<'_>, +) -> Result<(), String> { + let display = display_command(&program, args, context); + eprintln!("+ {display}"); + let status = configured_command(&program, args, context) + .status() + .map_err(|err| format!("failed to start `{}`: {err}", program.to_string_lossy()))?; + if status.success() { + Ok(()) + } else { + Err(format!( + "`{}` exited with {status}", + program.to_string_lossy() + )) + } +} + +pub(crate) fn run_command_owned( + program: OsString, + args: &[String], + context: CommandContext<'_>, +) -> Result<(), String> { + let borrowed = args.iter().map(String::as_str).collect::>(); + run_command(program, &borrowed, context) +} + +pub(crate) fn command_output( + program: OsString, + args: &[&str], + context: CommandContext<'_>, +) -> Result { + configured_command(&program, args, context) + .output() + .map_err(|err| format!("failed to start `{}`: {err}", program.to_string_lossy())) +} + +pub(crate) fn command_output_os(program: OsString, args: &[&str]) -> Result { + let output = command_output(program.clone(), args, CommandContext::new())?; + if !output.status.success() { + return Err(format!( + "`{}` exited with {}", + program.to_string_lossy(), + output.status + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +pub(crate) fn command_output_allow_failure(program: &str, args: &[&str]) -> Result { + let output = command_output(OsString::from(program), args, CommandContext::new())?; + let mut text = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if text.is_empty() { + text = stderr; + } else if !stderr.is_empty() { + text.push('\n'); + text.push_str(&stderr); + } + if text.is_empty() { + Err(format!( + "`{program}` exited with {} and no output", + output.status + )) + } else { + Ok(text) + } +} + +fn configured_command(program: &OsString, args: &[&str], context: CommandContext<'_>) -> Command { + let mut command = Command::new(program); + command.args(args); + if let Some(current_dir) = context.current_dir { + command.current_dir(current_dir); + } + if let Some(target_dir) = context.target_dir { + command.env("CARGO_TARGET_DIR", target_dir); + } + for (key, value) in context.envs { + command.env(key, value); + } + command +} + +fn display_command(program: &OsString, args: &[&str], context: CommandContext<'_>) -> String { + let mut parts = Vec::new(); + if let Some(current_dir) = context.current_dir { + parts.push(format!("cd {} &&", current_dir.display())); + } + if let Some(target_dir) = context.target_dir { + parts.push(format!("CARGO_TARGET_DIR={}", target_dir.display())); + } + for (key, value) in context.envs { + parts.push(format!("{key}={value}")); + } + parts.push(program.to_string_lossy().into_owned()); + parts.extend(args.iter().map(|arg| (*arg).to_string())); + parts.join(" ") +} diff --git a/xtask/tests/repo_lint.rs b/xtask/tests/repo_lint.rs new file mode 100644 index 00000000..67294d55 --- /dev/null +++ b/xtask/tests/repo_lint.rs @@ -0,0 +1,2071 @@ +// SPDX-License-Identifier: Apache-2.0 + +use std::{ + collections::BTreeSet, + ffi::OsStr, + fs, + path::{Component, Path, PathBuf}, + process::Command, +}; + +fn repo_root() -> &'static Path { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("workspace root") +} + +fn const_array_block<'a>(source: &'a str, name: &str) -> &'a str { + let start = source + .find(&format!("const {name}:")) + .unwrap_or_else(|| panic!("missing const {name}")); + let rest = &source[start..]; + let end = rest + .find("];") + .unwrap_or_else(|| panic!("unterminated const {name}")); + &rest[..end] +} + +mod corpus_policy { + use super::*; + + #[test] + fn conformance_manifest_lists_all_committed_jpeg_inputs() { + let root = repo_root(); + let conformance = root.join("corpus/conformance"); + let manifest = fs::read_to_string(conformance.join("manifest.json")) + .expect("read conformance manifest"); + + for entry in fs::read_dir(&conformance).expect("read conformance dir") { + let entry = entry.expect("read conformance entry"); + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) != Some("jpg") { + continue; + } + let filename = path + .file_name() + .and_then(|name| name.to_str()) + .expect("utf-8 fixture filename"); + assert!( + manifest.contains(&format!("\"{filename}\"")), + "conformance fixture {filename} is missing from manifest.json" + ); + } + } + + #[test] + fn corpus_readme_does_not_claim_committed_fixtures_are_absent() { + let readme = + fs::read_to_string(repo_root().join("corpus/README.md")).expect("read corpus README"); + + assert!( + !readme.contains("intentionally empty"), + "corpus README still claims the committed fixture corpus is empty" + ); + } +} + +mod source_policy { + use super::*; + + #[test] + fn adapter_crates_do_not_import_codec_private_modules() { + let root = repo_root(); + let adapter_crates = [ + "crates/j2k-jpeg-metal", + "crates/j2k-jpeg-cuda", + "crates/j2k-metal", + "crates/j2k-cuda", + ]; + + for crate_dir in adapter_crates { + for path in rust_sources(&root.join(crate_dir)) { + let source = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + assert!( + !source.contains("::__private") && !source.contains(" __private::"), + "adapter source {} imports a codec __private module; use the public adapter API", + path.strip_prefix(root).unwrap_or(&path).display() + ); + } + } + } + + #[test] + fn production_j2k_cuda_code_does_not_reference_nvjpeg() { + let root = repo_root(); + let checked_dirs = [ + "crates/j2k-cuda-runtime/src", + "crates/j2k-jpeg-cuda/src", + "crates/j2k-jpeg-cuda/benches", + ]; + + for dir in checked_dirs { + for path in rust_sources(&root.join(dir)) { + let source = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + assert!( + !["nvjpeg", "nvJPEG", "Nvjpeg", "NVJPEG"] + .iter() + .any(|token| source.contains(token)), + "production J2K CUDA source {} still references nvJPEG; JPEG CUDA decode must use J2K-owned paths only", + path.strip_prefix(root).unwrap_or(&path).display() + ); + } + } + } + + #[test] + fn cuda_adapter_crates_keep_public_libs_as_module_shells() { + let root = repo_root(); + let expected_modules = [ + ( + "crates/j2k-jpeg-cuda", + [ + "codec.rs", + "decoder.rs", + "error.rs", + "runtime.rs", + "session.rs", + "surface.rs", + ] + .as_slice(), + ), + ( + "crates/j2k-cuda", + [ + "codec.rs", + "decoder.rs", + "encode.rs", + "error.rs", + "runtime.rs", + "session.rs", + "surface.rs", + ] + .as_slice(), + ), + ]; + + for (crate_dir, modules) in expected_modules { + let src_dir = root.join(crate_dir).join("src"); + let lib_path = src_dir.join("lib.rs"); + let lib = fs::read_to_string(&lib_path) + .unwrap_or_else(|err| panic!("read {}: {err}", lib_path.display())); + let line_count = lib.lines().count(); + assert!( + line_count <= 220, + "{} should stay a thin public module shell; found {line_count} lines", + lib_path.strip_prefix(root).unwrap_or(&lib_path).display() + ); + + for module in modules { + let module_path = src_dir.join(module); + assert!( + module_path.exists(), + "{} must exist to keep CUDA adapter responsibilities focused", + module_path + .strip_prefix(root) + .unwrap_or(&module_path) + .display() + ); + } + } + } + + #[test] + fn reusable_benchmark_generators_live_in_test_support() { + let root = repo_root(); + let support = fs::read_to_string(root.join("crates/j2k-test-support/src/lib.rs")) + .expect("read j2k-test-support"); + + for required in [ + "pub fn gradient_u8", + "pub fn patterned_rgb8_tiles", + "pub fn gpu_bench_rgb8", + ] { + assert!( + support.contains(required), + "j2k-test-support must expose reusable generator `{required}`" + ); + } + } +} + +mod architecture_policy { + use super::*; + + #[test] + fn workspace_contains_public_j2k_crate() { + let root = repo_root(); + let manifest_path = root.join("crates/j2k/Cargo.toml"); + let manifest = fs::read_to_string(&manifest_path).unwrap_or_else(|err| { + panic!("read {}: {err}", manifest_path.display()); + }); + + for required in ["name = \"j2k\"", "j2k-core", "j2k-native", "j2k-types"] { + assert!( + manifest.contains(required), + "{} must contain `{required}`", + manifest_path + .strip_prefix(root) + .unwrap_or(&manifest_path) + .display() + ); + } + + let root_manifest = + fs::read_to_string(root.join("Cargo.toml")).expect("read workspace manifest"); + assert!( + root_manifest.contains("\"crates/j2k\""), + "workspace members must include the public j2k crate" + ); + } + + #[test] + fn j2k_public_crate_uses_explicit_upstream_reexports() { + let root = repo_root(); + let public_api_path = root.join("crates/j2k/src/lib.rs"); + let public_api = fs::read_to_string(&public_api_path).unwrap_or_else(|err| { + panic!("read {}: {err}", public_api_path.display()); + }); + + let glob_reexports = public_api + .lines() + .enumerate() + .filter_map(|(idx, line)| { + let trimmed = line.trim(); + (trimmed.starts_with("pub use j2k_") && trimmed.ends_with("::*;")) + .then(|| format!("{}:{}", idx + 1, trimmed)) + }) + .collect::>(); + assert!( + glob_reexports.is_empty(), + "j2k public crate must explicitly list upstream reexports:\n{}", + glob_reexports.join("\n") + ); + } + + #[test] + fn architecture_dependency_graph_matches_cargo_metadata() { + let root = repo_root(); + let metadata_edges = cargo_metadata_workspace_edges(root); + let docs = + fs::read_to_string(root.join("docs/architecture.md")).expect("read architecture docs"); + let docs_edges = architecture_doc_dependency_edges(&docs); + + let missing = metadata_edges + .difference(&docs_edges) + .map(format_edge) + .collect::>(); + let extra = docs_edges + .difference(&metadata_edges) + .map(format_edge) + .collect::>(); + + assert!( + missing.is_empty() && extra.is_empty(), + "docs/architecture.md crate dependency graph drifted from cargo metadata\n\ + missing from docs: {missing:#?}\n\ + not in cargo metadata: {extra:#?}" + ); + } + + #[test] + fn architecture_docs_classify_workspace_and_in_repo_tool_crates() { + let root = repo_root(); + let docs = + fs::read_to_string(root.join("docs/architecture.md")).expect("read architecture docs"); + + for required in [ + "`j2k-test-support`", + "dev helper", + "`xtask`", + "workspace tool", + "`xtask/`", + ] { + assert!( + docs.contains(required), + "docs/architecture.md must classify `{required}`" + ); + } + assert!( + !docs.contains("All crates live under `crates/`"), + "docs/architecture.md must not claim xtask lives under crates/" + ); + } + + #[test] + fn tooling_and_validation_crates_stay_unpublished() { + let root = repo_root(); + let workspace = + fs::read_to_string(root.join("Cargo.toml")).expect("read workspace manifest"); + let xtask = fs::read_to_string(root.join("xtask/src/main.rs")).expect("read xtask"); + let publishable = const_array_block(&xtask, "PUBLISHABLE_PACKAGES"); + + for (manifest, package) in [ + ("crates/j2k-test-support/Cargo.toml", "j2k-test-support"), + ("xtask/Cargo.toml", "xtask"), + ("tests/nvidia-baseline/Cargo.toml", "j2k-nvidia-baseline"), + ] { + let source = fs::read_to_string(root.join(manifest)) + .unwrap_or_else(|err| panic!("read {manifest}: {err}")); + assert!( + source.contains("publish = false"), + "{manifest} must stay unpublished" + ); + assert!( + !publishable.contains(&format!("\"{package}\"")), + "xtask publishable package gate must not include {package}" + ); + } + + assert!( + workspace.contains("\"crates/j2k-test-support\""), + "root workspace must include j2k-test-support for shared test helpers" + ); + assert!( + workspace.contains("\"xtask\""), + "root workspace must include xtask for cargo xtask automation" + ); + assert!( + !workspace.contains("\"tests/nvidia-baseline\""), + "root workspace must not include the standalone NVIDIA baseline workspace" + ); + } + + #[test] + fn public_crates_do_not_reexport_j2k_native() { + let root = repo_root(); + let mut offenders = Vec::new(); + + for crate_dir in [ + "crates/j2k/src", + "crates/j2k/src", + "crates/j2k-transcode/src", + "crates/j2k-metal/src", + "crates/j2k-cuda/src", + "crates/j2k-transcode-metal/src", + "crates/j2k-transcode-cuda/src", + ] { + for path in rust_sources(&root.join(crate_dir)) { + let source = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + for (line_idx, line) in source.lines().enumerate() { + let trimmed = line.trim_start(); + if trimmed.starts_with("pub use j2k_native") + || trimmed.starts_with("pub type ") && trimmed.contains("j2k_native") + { + offenders.push(format!( + "{}:{}:{}", + path.strip_prefix(root).unwrap_or(&path).display(), + line_idx + 1, + line.trim() + )); + } + } + } + } + + assert!( + offenders.is_empty(), + "public crates must not re-export native J2K implementation types:\n{}", + offenders.join("\n") + ); + } + + #[test] + fn rendered_public_api_does_not_expose_j2k_native() { + let root = repo_root(); + let stable_api_snapshot = + fs::read_to_string(root.join("docs/stable-api-1.0.public-api.txt")) + .expect("read stable API snapshot"); + + for package in [ + "j2k", + "j2k-jpeg", + "j2k-transcode", + "j2k-metal", + "j2k-cuda", + "j2k-transcode-metal", + "j2k-transcode-cuda", + ] { + let api = cargo_public_api(root, package) + .unwrap_or_else(|| stable_api_snapshot_section(&stable_api_snapshot, package)); + assert!( + !api.contains("j2k_native"), + "public API for package {package} exposes j2k_native:\n{api}" + ); + } + } +} + +mod docs_and_workflows_policy { + use super::*; + + #[test] + fn codec_api_guide_covers_public_surfaces() { + let root = repo_root(); + let readme = fs::read_to_string(root.join("README.md")).expect("read README"); + + assert!( + readme.contains("Codec contracts"), + "README must document the public codec API surface" + ); + + for required in [ + "decode_region_scaled_into", + "decode_rows", + "TileBatchDecode", + "BackendRequest::Auto", + "BackendRequest::Metal", + "BackendRequest::Cuda", + "DeviceSurface", + "ScratchPool", + "DecoderContext", + ] { + assert!( + readme.contains(required), + "README.md must document codec API surface `{required}`" + ); + } + } + + #[test] + fn ci_workflow_keeps_docs_and_benchmark_compile_gates() { + let workflow = fs::read_to_string(repo_root().join(".github/workflows/ci.yml")) + .expect("read CI workflow"); + let xtask = fs::read_to_string(repo_root().join("xtask/src/main.rs")).expect("read xtask"); + + for required in [ + "cargo xtask doc", + "cargo xtask stable-api", + "cargo xtask bench-build", + "cargo-public-api@0.52.0", + "macos-latest", + ] { + assert!( + workflow.contains(required), + "CI workflow must contain `{required}`" + ); + } + assert!( + !workflow.contains("macos-13"), + "hosted CI must not gate releases on unsupported Intel macOS runners" + ); + + for required in [ + "\"doc\"", + "\"--workspace\"", + "\"--all-features\"", + "\"--no-deps\"", + "\"j2k-jpeg-metal\"", + "\"j2k-metal\"", + "\"--no-run\"", + ] { + assert!(xtask.contains(required), "xtask must contain `{required}`"); + } + } + + #[test] + fn ci_workflow_runs_semver_checks_for_stable_library_crates() { + let workflow = fs::read_to_string(repo_root().join(".github/workflows/ci.yml")) + .expect("read CI workflow"); + let xtask = fs::read_to_string(repo_root().join("xtask/src/main.rs")).expect("read xtask"); + let semver_job = workflow_job(&workflow, "semver"); + let semver_packages = const_array_block(&xtask, "STABLE_SEMVER_PACKAGES"); + + assert!( + semver_job.contains("cargo install cargo-semver-checks --version 0.48.0 --locked"), + "CI semver job must install the pinned cargo-semver-checks version" + ); + assert!( + semver_job.contains("cargo xtask semver"), + "CI semver job must use the repo-owned semver gate" + ); + assert!( + semver_job.contains("runs-on: macos-latest"), + "CI semver job must run on macOS so published Metal baselines can be rustdoc-built" + ); + assert!( + semver_job.contains("toolchain: \"1.96\""), + "CI semver job must install the workspace Rust toolchain" + ); + assert!( + xtask.contains("unwrap_or_else(|_| \"1.96\".to_string())"), + "xtask semver must default to the workspace Rust toolchain" + ); + assert!( + !semver_job.contains("release-type: minor"), + "CI semver job must not treat the 0.5.0 boundary as a compatible minor release" + ); + + for package in [ + "j2k", + "j2k-core", + "j2k-jpeg", + "j2k", + "j2k-tilecodec", + "j2k-jpeg-metal", + "j2k-metal", + "j2k-jpeg-cuda", + "j2k-cuda", + "j2k-transcode", + "j2k-transcode-cuda", + "j2k-metal-support", + "j2k-transcode-metal", + "j2k-native", + "j2k-cuda-runtime", + "j2k-profile", + ] { + assert!( + semver_packages.contains(&format!("\"{package}\"")), + "repo semver policy must cover stable library crate `{package}`" + ); + } + + let package = "j2k-cli"; + assert!( + !semver_job.contains(package), + "CI semver job must not gate experimental or CLI crate `{package}`" + ); + } + + #[test] + fn xtask_test_does_not_run_benchmarks_as_tests() { + let xtask = fs::read_to_string(repo_root().join("xtask/src/main.rs")).expect("read xtask"); + let test_section = xtask + .split("fn test()") + .nth(1) + .and_then(|rest| rest.split("fn doc()").next()) + .expect("xtask test section"); + + for required in ["\"--lib\"", "\"--bins\"", "\"--tests\"", "\"--doc\""] { + assert!( + test_section.contains(required), + "xtask test must include cargo test selector `{required}`" + ); + } + assert!( + !test_section.contains("\"--all-targets\""), + "xtask test must not pass --all-targets because harness=false benchmark binaries would run as tests" + ); + } + + #[test] + fn xtask_exposes_nextest_machete_and_strict_clippy_gates() { + let xtask = fs::read_to_string(repo_root().join("xtask/src/main.rs")).expect("read xtask"); + let help_section = xtask + .split("fn print_help()") + .nth(1) + .expect("xtask help section"); + + for task in ["nextest", "machete", "clippy-strict"] { + assert!( + xtask.contains(&format!("\"{task}\" =>")), + "xtask must dispatch `{task}`" + ); + assert!( + help_section.contains(task), + "xtask help must document `{task}`" + ); + } + + for required in [ + "\"nextest\"", + "\"run\"", + "\"cargo-machete\"", + "\"--no-deps\"", + "\"clippy::pedantic\"", + "\"clippy::nursery\"", + "\"j2k-native\"", + "\"j2k\"", + ] { + assert!(xtask.contains(required), "xtask must contain `{required}`"); + } + } + + #[test] + fn xtask_fuzz_build_checks_every_fuzz_manifest() { + let root = repo_root(); + let xtask = fs::read_to_string(root.join("xtask/src/main.rs")).expect("read xtask"); + let mut manifests = Vec::new(); + + for entry in fs::read_dir(root.join("crates")).expect("read crates dir") { + let entry = entry.expect("read crate entry"); + let manifest = entry.path().join("fuzz/Cargo.toml"); + if manifest.exists() { + manifests.push(manifest); + } + } + manifests.sort(); + assert!( + !manifests.is_empty(), + "repository must keep fuzz targets under crates/*/fuzz" + ); + + for manifest in manifests { + let relative_path = manifest + .strip_prefix(root) + .expect("fuzz manifest under repo root"); + let relative = relative_path + .iter() + .map(|part| part.to_string_lossy()) + .collect::>() + .join("/"); + assert!( + xtask.contains(&relative), + "xtask fuzz-build must check {relative}" + ); + } + } + + #[test] + fn ci_coverage_job_is_a_required_gate() { + let workflow = fs::read_to_string(repo_root().join(".github/workflows/ci.yml")) + .expect("read CI workflow"); + let coverage_job = workflow_job(&workflow, "coverage"); + const INSTALL_ACTION_SHA: &str = "91534edaf9fd796a162759d80d49cdff574bff2c"; + + assert!( + coverage_job.contains(&format!("taiki-e/install-action@{INSTALL_ACTION_SHA}")) + && coverage_job.contains("tool: cargo-llvm-cov") + && coverage_job.contains("cargo xtask coverage"), + "coverage job must install cargo-llvm-cov from a pinned action and run xtask coverage" + ); + assert!( + !coverage_job.contains("taiki-e/install-action@cargo-llvm-cov"), + "coverage job must not use mutable install-action tool tags" + ); + assert!( + !coverage_job.contains("continue-on-error"), + "coverage job must not be allowed to fail silently" + ); + } + + #[test] + fn coverage_excludes_hardware_only_gpu_adapter_crates() { + let xtask = + fs::read_to_string(repo_root().join("xtask/src/main.rs")).expect("read xtask source"); + + for required in [ + "crates/j2k-cuda-runtime/", + "crates/j2k-cuda/", + "crates/j2k-.*-cuda/", + "crates/j2k-metal/", + "crates/j2k-.*-metal/", + "crates/j2k-metal-support/", + ] { + assert!( + xtask.contains(required), + "coverage exclusion regex must include `{required}`" + ); + } + } + + #[test] + fn ci_miri_job_is_a_required_gate() { + let workflow = fs::read_to_string(repo_root().join(".github/workflows/ci.yml")) + .expect("read CI workflow"); + let miri_job = workflow_job(&workflow, "miri"); + + for required in ["toolchain: nightly", "components: miri", "cargo xtask miri"] { + assert!( + miri_job.contains(required), + "miri job must contain `{required}`" + ); + } + } + + #[test] + fn ci_fuzz_run_budgets_are_nontrivial() { + let workflow = fs::read_to_string(repo_root().join(".github/workflows/ci.yml")) + .expect("read CI workflow"); + + for required in [ + "J2K_FUZZ_RUNS: \"512\"", + "J2K_FUZZ_MAX_TOTAL_TIME_SECONDS: \"60\"", + "J2K_FUZZ_RUNS: \"20000\"", + "J2K_FUZZ_MAX_TOTAL_TIME_SECONDS: \"900\"", + ] { + assert!( + workflow.contains(required), + "CI fuzz budgets must contain `{required}`" + ); + } + } + + #[test] + fn deny_paste_advisory_ignore_has_review_metadata() { + let deny = fs::read_to_string(repo_root().join("deny.toml")).expect("read deny.toml"); + + for required in [ + "RUSTSEC-2024-0436", + "Review-by: 2026-12-31", + "https://crates.io/crates/metal", + "https://github.com/gfx-rs/metal-rs/blob/master/Cargo.toml", + "https://rustsec.org/advisories/RUSTSEC-2024-0436.html", + "https://github.com/gfx-rs/metal-rs/issues/349", + ] { + assert!( + deny.contains(required), + "deny.toml paste advisory rationale must contain `{required}`" + ); + } + } + + #[test] + fn unsafe_audit_rows_include_invariants_and_regression_guards() { + let audit = fs::read_to_string(repo_root().join("docs/unsafe-audit.md")) + .expect("read unsafe audit"); + + assert!( + audit.contains("| Path | Scope | Invariants | Regression guards |"), + "unsafe audit must include invariant and regression guard columns" + ); + for line in audit + .lines() + .filter(|line| line.trim().starts_with("| `crates/")) + { + let cells = line.split('|').map(str::trim).collect::>(); + assert!( + cells.len() >= 6 && !cells[3].is_empty() && !cells[4].is_empty(), + "unsafe audit row must include invariant and regression guard: {line}" + ); + } + } + + #[test] + fn ci_workflow_has_read_only_permissions_and_gpu_path_policy() { + let root = repo_root(); + let workflow = + fs::read_to_string(root.join(".github/workflows/ci.yml")).expect("read CI workflow"); + let gpu_policy = workflow_job(&workflow, "gpu-path-policy"); + let codeowners = + fs::read_to_string(root.join(".github/CODEOWNERS")).expect("read CODEOWNERS"); + + assert!( + workflow.contains("\npermissions:\n contents: read\n"), + "CI workflow must default the GITHUB_TOKEN to contents: read" + ); + for required in [ + "pull-requests: read", + "actions: read", + "crates/j2k-cuda-runtime/", + "crates/j2k-jpeg-cuda/", + "crates/j2k-cuda/", + "crates/j2k-transcode-cuda/", + "crates/j2k-metal-support/", + "crates/j2k-jpeg-metal/", + "crates/j2k-metal/", + "crates/j2k-transcode-metal/", + "gpu-validation.yml/runs?head_sha=", + "run.get(\"conclusion\") == \"success\"", + "No GPU path changes detected.", + ] { + assert!( + gpu_policy.contains(required), + "GPU path policy job must contain `{required}`" + ); + } + for required in [ + ".github/workflows/gpu-validation.yml", + "crates/j2k-cuda-runtime/", + "crates/j2k-jpeg-cuda/", + "crates/j2k-cuda/", + "crates/j2k-transcode-cuda/", + "crates/j2k-metal-support/", + "crates/j2k-jpeg-metal/", + "crates/j2k-metal/", + "crates/j2k-transcode-metal/", + ] { + assert!( + codeowners.contains(required), + "CODEOWNERS must cover GPU path `{required}`" + ); + } + } + + #[test] + fn gpu_validation_workflow_is_self_hosted_and_explicit() { + let root = repo_root(); + let workflow_path = root.join(".github/workflows/gpu-validation.yml"); + let workflow = fs::read_to_string(&workflow_path).expect("read GPU validation workflow"); + + for required in [ + "workflow_dispatch", + "run-timed-benchmarks", + "run-metal-validation", + "self-hosted", + "metal", + "cuda", + "NVCC: /usr/local/cuda/bin/nvcc", + "cargo test -p j2k-jpeg-metal", + "cargo test -p j2k-metal", + "cargo test -p j2k-jpeg-cuda", + "cargo test -p j2k-cuda", + ] { + assert!( + workflow.contains(required), + "{} must contain `{required}`", + workflow_path + .strip_prefix(root) + .unwrap_or(&workflow_path) + .display() + ); + } + } + + #[test] + fn cuda_gpu_validation_job_stays_cuda_focused() { + let root = repo_root(); + let workflow_path = root.join(".github/workflows/gpu-validation.yml"); + let workflow = fs::read_to_string(&workflow_path).expect("read GPU validation workflow"); + let cuda_job = workflow_job(&workflow, "cuda-x86_64-compatibility"); + + for required in [ + "runs-on: [self-hosted, Linux, X64, cuda]", + "NVCC: /usr/local/cuda/bin/nvcc", + "J2K_REQUIRE_CUDA_RUNTIME", + "J2K_REQUIRE_CUDA_JPEG_HARDWARE_DECODE", + "J2K_GPU_BENCH_DIM", + "J2K_GPU_BENCH_BATCH", + "J2K_GPU_BENCH_BATCH_DIM", + "uname -a", + "rustc -Vv", + "cargo -V", + "nvidia-smi", + "CUDA runtime validation requires a working CUDA driver", + "cargo test -p j2k-jpeg-cuda --all-targets --features cuda-runtime", + "cargo test -p j2k-cuda --all-targets --features cuda-runtime", + "cargo bench -p j2k-jpeg-cuda --bench device_decode --features cuda-runtime --no-run", + "cargo bench -p j2k-jpeg-cuda --bench device_decode --features cuda-runtime -- --noplot", + ] { + assert!( + cuda_job.contains(required), + "{} CUDA job must contain `{required}`", + workflow_path + .strip_prefix(root) + .unwrap_or(&workflow_path) + .display() + ); + } + + let forbidden_j2k_metal_compare_bench = + ["cargo bench -p ", "j2k-metal", " --bench compare --no-run"].concat(); + for forbidden in [ + forbidden_j2k_metal_compare_bench.as_str(), + "cargo bench -p j2k-jpeg --no-run", + "cargo test -p j2k-jpeg-metal", + "cargo test -p j2k-metal", + ] { + assert!( + !cuda_job.contains(forbidden), + "{} CUDA job must not contain Metal validation command `{forbidden}`", + workflow_path + .strip_prefix(root) + .unwrap_or(&workflow_path) + .display() + ); + } + } + + #[test] + fn cuda_build_scripts_do_not_probe_default_nvcc() { + let root = repo_root(); + for relative in [ + "crates/j2k-cuda-runtime/build.rs", + "tests/nvidia-baseline/build.rs", + ] { + let source = fs::read_to_string(root.join(relative)) + .unwrap_or_else(|err| panic!("read {relative}: {err}")); + assert!( + !source.contains("unwrap_or_else(|| \"nvcc\".into())"), + "{relative} must not default to PATH nvcc" + ); + assert!( + source.contains("requires absolute NVCC"), + "{relative} must require absolute NVCC in strict mode" + ); + } + } + + #[test] + fn cuda_decode_profile_workflow_exports_rca_artifacts() { + let root = repo_root(); + let workflow_path = root.join(".github/workflows/gpu-validation.yml"); + let workflow = fs::read_to_string(&workflow_path).expect("read GPU validation workflow"); + let cuda_job = workflow_job(&workflow, "cuda-x86_64-compatibility"); + + for required in [ + "run-cuda-htj2k-decode-profile", + "CUDA HTJ2K decode RCA profile", + "J2K_REQUIRE_CUDA_BENCH: \"1\"", + "J2K_PROFILE_STAGES: summary", + "J2K_CUDA_TRACE: ${{ github.workspace }}/target/cuda_htj2k_decode_trace.json", + "/proc/sys/kernel/perf_event_paranoid", + "cargo install samply --version 0.13.1 --locked", + "samply record --save-only -o target/cuda_htj2k_decode_samply.json.gz", + "target/cuda_htj2k_decode_samply_status.txt", + "samply_status=blocked", + "passwordless sudo", + "--features cuda-runtime,cuda-profiling", + "2>&1 | tee target/cuda_htj2k_decode_profile.log", + "cuda-htj2k-decode-rca-profile", + "target/cuda_htj2k_decode_profile.log", + "target/cuda_htj2k_decode_trace.json", + "target/cuda_htj2k_decode_samply.json.gz", + "target/criterion", + ] { + assert!( + workflow.contains(required) || cuda_job.contains(required), + "{} CUDA decode profile gate must contain `{required}`", + workflow_path + .strip_prefix(root) + .unwrap_or(&workflow_path) + .display() + ); + } + } + + #[test] + fn nvidia_baseline_workflow_exports_direct_decode_artifacts() { + let root = repo_root(); + let workflow_path = root.join(".github/workflows/gpu-validation.yml"); + let workflow = fs::read_to_string(&workflow_path).expect("read GPU validation workflow"); + let cuda_job = workflow_job(&workflow, "cuda-x86_64-compatibility"); + + for required in [ + "run-nvidia-baseline", + "--bin transcode_compare", + "--decomposition-levels 1", + "--decomposition-levels 2", + "target/transcode_compare_level1.json", + "target/transcode_compare_level2.json", + "tests/nvidia-baseline/scripts/assert_transcode_perf.py", + "J2K_LEVEL1_CUDA_HT_MIN_MPS", + "J2K_LEVEL2_CUDA_HT_MIN_MPS", + "--bin decode_compare", + "--jpeg-dir \"${J2K_BENCH_JPEG_DIR}\"", + "--min-inputs 100", + "target/decode_compare.json", + "target/decode_compare.csv", + "python3 -m json.tool target/decode_compare.json", + "nvidia-baseline-comparison", + ] { + assert!( + workflow.contains(required) || cuda_job.contains(required), + "{} NVIDIA baseline gate must contain `{required}`", + workflow_path + .strip_prefix(root) + .unwrap_or(&workflow_path) + .display() + ); + } + } + + #[test] + fn nvidia_codec_comparator_stays_test_only() { + let root = repo_root(); + let needles = [ + "j2k-nvidia-baseline", + "nvjpeg2000", + "nvjpeg2k", + "nvidia-baseline", + ]; + let mut seen = 0usize; + let mut violations = Vec::new(); + + for path in repo_text_files(root) { + let rel = path.strip_prefix(root).unwrap_or(&path); + let rel_s = format!("./{}", rel.display()).replace('\\', "/"); + let source = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + for (line_idx, line) in source.lines().enumerate() { + let lower = line.to_ascii_lowercase(); + if !needles.iter().any(|needle| lower.contains(needle)) { + continue; + } + seen += 1; + let allowed = rel_s.starts_with("./tests/nvidia-baseline/") + || rel_s == "./.github/workflows/gpu-validation.yml" + || rel_s == "./xtask/tests/repo_lint.rs"; + if !allowed { + violations.push(format!("{}:{}:{}", rel_s, line_idx + 1, line)); + } + } + } + + assert!( + seen > 0, + "repo must contain the test-only NVIDIA comparator guard input" + ); + assert!( + violations.is_empty(), + "NVIDIA codec comparator references must stay test-only:\n{}", + violations.join("\n") + ); + } +} + +mod release_policy { + use super::*; + + #[test] + fn crates_io_publish_policy_is_explicit() { + let root = repo_root(); + let workspace = + fs::read_to_string(root.join("Cargo.toml")).expect("read workspace manifest"); + let changelog = fs::read_to_string(root.join("CHANGELOG.md")).expect("read changelog"); + let xtask = fs::read_to_string(root.join("xtask/src/main.rs")).expect("read xtask"); + let publishable = const_array_block(&xtask, "PUBLISHABLE_PACKAGES"); + let publish_workflow = fs::read_to_string(root.join(".github/workflows/publish.yml")) + .expect("read publish workflow"); + let version = workspace_package_version(&workspace); + + assert!( + changelog.contains(&format!("## [{version}]")), + "CHANGELOG.md must contain a section for the current staged release version {version}" + ); + + for package in [ + "j2k-core", + "j2k-cuda-runtime", + "j2k-profile", + "j2k-native", + "j2k-tilecodec", + "j2k-jpeg", + "j2k", + "j2k-jpeg-metal", + "j2k-jpeg-cuda", + "j2k-metal", + "j2k-cuda", + "j2k-cli", + "j2k", + ] { + assert!( + publishable.contains(&format!("\"{package}\"")), + "xtask package gate must include publishable package {package}" + ); + assert!( + publish_workflow.contains(&format!("publish-{package}:")), + "publish workflow must include package {package}" + ); + } + + let package = "j2k-compare"; + assert!( + !publishable.contains(&format!("\"{package}\"")), + "xtask package gate must not package local comparator package {package}" + ); + assert!( + !publish_workflow.contains(&format!("publish-{package}:")), + "publish workflow must not publish local comparator package {package}" + ); + } + + #[test] + fn release_docs_use_manifest_versions_for_publish_order() { + let release = + fs::read_to_string(repo_root().join("docs/release.md")).expect("read release docs"); + + assert!( + release.contains("manifest versions"), + "release docs must describe publishing the current manifest versions instead of stale hard-coded versions" + ); + assert!( + !release.contains("`j2k` `1.1.0`") + && !release.contains("`j2k-native` `0.3.0`") + && !release.contains("`j2k` `1.0.0`"), + "release docs must not carry stale pre-facade publish versions" + ); + } + + fn const_array_block<'a>(source: &'a str, name: &str) -> &'a str { + let start = source + .find(&format!("const {name}:")) + .unwrap_or_else(|| panic!("missing const {name}")); + let rest = &source[start..]; + let end = rest + .find("];") + .unwrap_or_else(|| panic!("unterminated const {name}")); + &rest[..end] + } + + fn workspace_package_version(workspace_manifest: &str) -> &str { + workspace_manifest + .lines() + .find_map(|line| { + let line = line.trim(); + line.strip_prefix("version") + .and_then(|rest| rest.split('"').nth(1)) + }) + .expect("workspace package version") + } + + #[test] + fn j2k_compare_stays_unpublished_and_out_of_j2k_package_deps() { + let root = repo_root(); + let compare_manifest = fs::read_to_string(root.join("crates/j2k-compare/Cargo.toml")) + .expect("read j2k-compare manifest"); + let j2k_manifest = + fs::read_to_string(root.join("crates/j2k/Cargo.toml")).expect("read j2k manifest"); + + assert!( + compare_manifest.contains("publish = false"), + "j2k-compare must remain an unpublished local oracle helper" + ); + assert!( + !j2k_manifest.contains("j2k-compare"), + "j2k must not package a dev-dependency on j2k-compare" + ); + } + + #[test] + fn package_preflight_is_staged_dependency_aware() { + let root = repo_root(); + let xtask = fs::read_to_string(root.join("xtask/src/main.rs")).expect("read xtask"); + let publish_script = + fs::read_to_string(root.join("scripts/publish-crate.sh")).expect("read publish script"); + let release = fs::read_to_string(root.join("docs/release.md")).expect("read release docs"); + + assert!( + xtask.contains("STAGED_DEPENDENCY_PACKAGES"), + "xtask package preflight must explicitly model crates blocked by unpublished staged dependencies" + ); + let strict_packages = const_array_block(&xtask, "REGISTRY_INDEPENDENT_PACKAGES"); + let staged_packages = const_array_block(&xtask, "STAGED_DEPENDENCY_PACKAGES"); + for (package, dependency) in cargo_metadata_workspace_edges(root) { + assert!( + !strict_packages.contains(&format!("\"{package}\"")), + "strict package preflight must not include `{package}` while it depends on staged workspace crate `{dependency}`" + ); + } + assert!( + staged_packages.contains("\"j2k-cuda-runtime\""), + "j2k-cuda-runtime depends on staged j2k-core and must not run strict package verification before publication" + ); + assert!( + xtask.contains("\"--list\"") && xtask.contains("unpublished workspace dependencies"), + "xtask package preflight must validate package contents for staged downstream crates without hiding why strict packaging is skipped" + ); + assert!( + publish_script.contains("dry-run package list only") + && publish_script.contains("j2k-cli") + && publish_script.contains("cargo package -p \"$crate\" --list"), + "publish workflow dry-run must not fail downstream crates only because staged dependency versions are not published yet" + ); + assert!( + release.contains("cargo package --list") + && release.contains("cargo publish --dry-run") + && release.contains("unpublished workspace dependencies"), + "release docs must explain the pre-publish package validation limits" + ); + } + + #[test] + fn publish_script_covers_all_publishable_crates() { + let root = repo_root(); + let xtask = fs::read_to_string(root.join("xtask/src/main.rs")).expect("read xtask"); + let publish_script = + fs::read_to_string(root.join("scripts/publish-crate.sh")).expect("read publish script"); + let publishable = const_array_block(&xtask, "PUBLISHABLE_PACKAGES"); + + for line in publishable.lines() { + let package = line.trim().trim_matches([',', '"']); + if package.is_empty() + || package.starts_with("const ") + || package.starts_with(']') + || package.starts_with('&') + { + continue; + } + assert!( + publish_script.contains(package), + "publish script must allow publishable package `{package}`" + ); + } + } +} + +mod public_docs_policy { + use super::*; + + #[test] + fn supported_j2k_env_vars_are_documented() { + let root = repo_root(); + let docs_path = root.join("docs/env-vars.md"); + let docs = fs::read_to_string(&docs_path) + .unwrap_or_else(|err| panic!("read {}: {err}", docs_path.display())); + let readme = fs::read_to_string(root.join("README.md")).expect("read README"); + assert!( + readme.contains("docs/env-vars.md"), + "README must link the supported environment-variable reference" + ); + + let mut missing = Vec::new(); + for path in repo_text_files(root) { + if is_archived_handoff(&path) + || path.ends_with("docs/env-vars.md") + || path.ends_with("xtask/tests/repo_lint.rs") + { + continue; + } + let source = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + for token in j2k_env_tokens(&source) { + if is_internal_j2k_token(&token) { + continue; + } + if !docs.contains(&format!("`{token}`")) { + missing.push(format!( + "{}: {token}", + path.strip_prefix(root).unwrap_or(&path).display() + )); + } + } + } + + assert!( + missing.is_empty(), + "supported J2K_* environment variables must be documented in docs/env-vars.md:\n{}", + missing.join("\n") + ); + assert!( + !docs.contains("J2K_JPEG_METAL_SPLIT_FAST420_BATCH"), + "removed experiment-only JPEG Metal fast420 split switch must not be documented as supported" + ); + } + + #[test] + fn public_docs_describe_public_crate_auto_and_cuda_runtime_surface_scope() { + let root = repo_root(); + let readme = fs::read_to_string(root.join("README.md")).expect("read README"); + let changelog = fs::read_to_string(root.join("CHANGELOG.md")).expect("read changelog"); + let architecture = + fs::read_to_string(root.join("docs/architecture.md")).expect("read architecture docs"); + let release = fs::read_to_string(root.join("docs/release.md")).expect("read release docs"); + + for (name, docs) in [ + ("README.md", readme.as_str()), + ("docs/architecture.md", architecture.as_str()), + ("docs/release.md", release.as_str()), + ] { + assert!( + docs.contains("public crate release") + && docs.contains("Runtime backend selection defaults to `Auto`"), + "{name} must name the public crate release posture and Auto backend policy" + ); + } + + for (name, docs) in [ + ("README.md", readme.as_str()), + ("CHANGELOG.md", changelog.as_str()), + ("docs/release.md", release.as_str()), + ] { + assert!( + docs.contains("cuda-runtime") + && docs.contains("CUDA device memory") + && docs.contains("J2K-owned CUDA") + && docs.contains("NVIDIA performance"), + "{name} must describe CUDA device-memory output and owned CUDA scope without overclaiming NVIDIA performance" + ); + assert!( + !docs.contains("compatibility-only with no runtime CUDA decode"), + "{name} must not describe CUDA as compatibility-only after runtime surface support exists" + ); + } + } + + #[test] + fn public_docs_route_users_to_current_crates() { + let root = repo_root(); + let readme = fs::read_to_string(root.join("README.md")).expect("read README"); + + for required in [ + "Which crate should I use?", + "Fast Path For LLM-Assisted Use", + "cargo add j2k", + "statumen", + "wsi-dicom", + "j2k-jpeg", + "j2k", + "j2k-cli", + ] { + assert!( + readme.contains(required), + "README.md must route users to `{required}` after the rename" + ); + } + + let legacy_terms = [ + format!("{}{}", "ash", "lar"), + format!("{}{}", "zig", "gurat"), + ]; + for legacy in &legacy_terms { + assert!( + !readme.to_ascii_lowercase().contains(legacy), + "README.md must use current package names only" + ); + } + } + + #[test] + fn active_repo_text_does_not_reintroduce_signinum_names() { + let root = repo_root(); + let mut offenders = Vec::new(); + + for path in repo_text_files(root) { + if is_archived_handoff(&path) + || is_allowed_signinum_history_reference(root, &path) + || path.ends_with("xtask/tests/repo_lint.rs") + { + continue; + } + let source = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + for (line_idx, line) in source.lines().enumerate() { + let lower = line.to_ascii_lowercase(); + if lower.contains("signinum") { + offenders.push(format!( + "{}:{}:{}", + path.strip_prefix(root).unwrap_or(&path).display(), + line_idx + 1, + line + )); + } + } + } + + assert!( + offenders.is_empty(), + "active repo text must not reintroduce signinum names after the j2k rename:\n{}", + offenders.join("\n") + ); + } + + #[test] + fn public_repo_excludes_agent_private_artifacts() { + let root = repo_root(); + let private_docs_name = ["super", "powers"].concat(); + let private_dir = ["docs", private_docs_name.as_str()].join("/"); + let migration_doc = ["MIGRATION", ".md"].concat(); + let migration_doc_lower = migration_doc.to_ascii_lowercase(); + let mut offenders = Vec::new(); + + for path in repo_text_files(root) { + let relative = path.strip_prefix(root).unwrap_or(&path); + let relative_text = relative.to_string_lossy(); + let file_name = path + .file_name() + .and_then(OsStr::to_str) + .unwrap_or_default() + .to_ascii_lowercase(); + if relative_text.starts_with(&private_dir) || file_name == migration_doc_lower { + offenders.push(relative_text.to_string()); + } + } + + assert!( + offenders.is_empty(), + "public repo must not track agent-private planning docs or migration scratch files: {offenders:?}" + ); + } + + #[test] + fn published_crates_have_crates_io_landing_readmes() { + let root = repo_root(); + + for crate_dir in publishable_crate_dirs() { + let manifest_path = root.join(crate_dir).join("Cargo.toml"); + let manifest = fs::read_to_string(&manifest_path) + .unwrap_or_else(|err| panic!("read {}: {err}", manifest_path.display())); + let readme_path = root.join(crate_dir).join("README.md"); + + assert!( + manifest.contains("readme"), + "{} must declare a readme for crates.io landing pages", + manifest_path + .strip_prefix(root) + .unwrap_or(&manifest_path) + .display() + ); + assert!( + readme_path.exists(), + "{} must exist for crates.io landing pages", + readme_path + .strip_prefix(root) + .unwrap_or(&readme_path) + .display() + ); + } + } + + #[test] + fn publishable_crates_configure_docs_rs_metadata() { + let root = repo_root(); + + for crate_dir in publishable_crate_dirs() { + let manifest_path = root.join(crate_dir).join("Cargo.toml"); + let manifest = fs::read_to_string(&manifest_path) + .unwrap_or_else(|err| panic!("read {}: {err}", manifest_path.display())); + + assert!( + manifest.contains("[package.metadata.docs.rs]"), + "{} must configure docs.rs metadata", + manifest_path + .strip_prefix(root) + .unwrap_or(&manifest_path) + .display() + ); + assert!( + manifest.contains("all-features = true"), + "{} must build docs.rs with all features enabled", + manifest_path + .strip_prefix(root) + .unwrap_or(&manifest_path) + .display() + ); + assert!( + manifest.contains("targets = []"), + "{} must keep docs.rs targets explicit", + manifest_path + .strip_prefix(root) + .unwrap_or(&manifest_path) + .display() + ); + } + } + + #[test] + fn support_matrix_is_linked_and_covers_adoption_surfaces() { + let root = repo_root(); + let readme = fs::read_to_string(root.join("README.md")).expect("read README"); + + for required in [ + "Stable APIs", + "Experimental APIs", + "BackendRequest::Auto", + "Security", + "Benchmark and parity policy", + "MSRV", + "OpenJPEG", + "Grok", + ] { + assert!( + readme.contains(required), + "README.md must cover `{required}`" + ); + } + } + + #[test] + fn public_codec_and_transcode_examples_are_publicly_linked() { + let root = repo_root(); + let readme = fs::read_to_string(root.join("README.md")).expect("read README"); + + for example in [ + "crates/j2k/examples/decode_generated.rs", + "crates/j2k-jpeg/examples/inspect.rs", + "crates/j2k-tilecodec/examples/decompress.rs", + "crates/j2k-transcode/examples/jpeg_to_htj2k.rs", + ] { + assert!( + root.join(example).exists(), + "expected runnable example `{example}`" + ); + assert!(readme.contains(example), "README must link `{example}`"); + } + } + + #[test] + fn benchmark_docs_define_publication_gate_for_openjpeg_and_grok() { + let root = repo_root(); + let readme = fs::read_to_string(root.join("README.md")).expect("read README"); + let xtask = fs::read_to_string(root.join("xtask/src/main.rs")).expect("read xtask"); + + for required in [ + "published benchmark", + "J2K_COMPARE_THREADS", + "J2K_REQUIRE_OPENJPEG=1", + "J2K_REQUIRE_GROK=1", + "comparator availability", + "comparator version", + "input source", + "j2k-generated", + ] { + assert!( + readme.contains(required), + "README.md benchmark policy must contain `{required}`" + ); + } + + assert!( + xtask.contains("\"j2k-bench-signoff\""), + "xtask must expose a no-silent-skip J2K benchmark signoff task" + ); + } + + #[test] + fn j2k_metal_bench_surface_stays_clean_after_reset() { + let root = repo_root(); + let cargo_toml = fs::read_to_string(root.join("crates/j2k-metal/Cargo.toml")) + .expect("read J2K Metal manifest"); + let readme = fs::read_to_string(root.join("README.md")).expect("read README"); + let xtask = fs::read_to_string(root.join("xtask/src/main.rs")).expect("read xtask"); + let openjpeg = fs::read_to_string(root.join("crates/j2k-compare/src/openjpeg.rs")) + .expect("read OpenJPEG comparator"); + let grok = fs::read_to_string(root.join("crates/j2k-compare/src/grok.rs")) + .expect("read Grok comparator"); + + assert!( + !cargo_toml.contains("[[bench]]"), + "j2k-metal bench targets must stay reset until new profiling benches are added" + ); + + for forbidden in [ + "criterion =", + "j2k-compare =", + "name = \"device_upload\"", + "name = \"compare\"", + "name = \"encode_stages\"", + "name = \"decode_stages\"", + ] { + assert!( + !cargo_toml.contains(forbidden), + "j2k-metal manifest must not contain legacy bench entry `{forbidden}`" + ); + } + + let benches_dir = root.join("crates/j2k-metal/benches"); + if benches_dir.exists() { + let stale_entries: Vec<_> = fs::read_dir(&benches_dir) + .expect("read J2K Metal benches dir") + .map(|entry| { + let path = entry.expect("read J2K Metal bench entry").path(); + path.strip_prefix(root) + .expect("bench entry under repo root") + .display() + .to_string() + }) + .collect(); + assert!( + stale_entries.is_empty(), + "j2k-metal benches dir must stay empty after reset: {stale_entries:?}" + ); + } + + let removed_j2k_metal_bench_command = ["cargo bench -p ", "j2k-metal", " --bench"].concat(); + assert!( + !readme.contains(&removed_j2k_metal_bench_command), + "README.md must not publish removed j2k-metal bench commands" + ); + assert!( + !xtask.contains(&removed_j2k_metal_bench_command), + "xtask must not run removed j2k-metal bench commands" + ); + assert!( + openjpeg.contains("pub fn version"), + "OpenJPEG comparator must expose version metadata" + ); + assert!( + grok.contains("pub fn version") && grok.contains("pub fn library_path"), + "Grok comparator must expose version and path metadata" + ); + } + + #[test] + fn public_text_does_not_embed_local_user_home_paths() { + let root = repo_root(); + let mut offenders = Vec::new(); + + for path in repo_text_files(root) { + if is_archived_handoff(&path) { + continue; + } + if path.ends_with("xtask/tests/repo_lint.rs") { + continue; + } + let source = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + if source.contains("/Users/") || source.contains("C:\\Users\\") { + offenders.push( + path.strip_prefix(root) + .unwrap_or(&path) + .display() + .to_string(), + ); + } + } + + assert!( + offenders.is_empty(), + "public text must not embed local user-home paths; use env vars or repo-relative defaults: {offenders:?}" + ); + } + + #[test] + fn referenced_shell_scripts_exist() { + let root = repo_root(); + let mut missing = Vec::new(); + + for path in repo_text_files(root) { + if is_archived_handoff(&path) { + continue; + } + let source = fs::read_to_string(&path) + .unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + for script in referenced_shell_scripts(&source) { + let root_relative = root.join(&script); + let file_relative = path.parent().expect("text file has parent").join(&script); + if !root_relative.exists() && !file_relative.exists() { + missing.push(format!( + "{} references missing script {script}", + path.strip_prefix(root).unwrap_or(&path).display() + )); + } + } + } + + assert!( + missing.is_empty(), + "all referenced shell scripts must exist: {missing:?}" + ); + } + + #[test] + fn public_narrative_docs_do_not_carry_stale_zeiss_claims() { + let root = repo_root(); + let mut offenders = Vec::new(); + + for relative in [ + "README.md", + "docs/architecture.md", + "docs/release.md", + "paper/paper.md", + "paper/arxiv/main.tex", + ] { + let path = root.join(relative); + let Ok(source) = fs::read_to_string(&path) else { + if relative.starts_with("paper/") { + continue; + } + panic!("read {}: missing required narrative doc", path.display()); + }; + if source.contains("Zeiss") { + offenders.push(relative); + } + } + + assert!( + offenders.is_empty(), + "public narrative docs must not carry stale Zeiss integration claims: {offenders:?}" + ); + } + + #[test] + fn packaged_rust_sources_do_not_include_files_outside_their_crate() { + let root = repo_root(); + let workspace_crates = root.join("crates"); + let mut escaping = Vec::new(); + + for source_path in rust_sources(&workspace_crates) { + let Ok(relative_to_crates) = source_path.strip_prefix(&workspace_crates) else { + continue; + }; + let Some(crate_name) = relative_to_crates.components().next() else { + continue; + }; + let member_root = workspace_crates.join(crate_name.as_os_str()); + let source = fs::read_to_string(&source_path) + .unwrap_or_else(|err| panic!("read {}: {err}", source_path.display())); + + for include_path in rust_include_paths(&source) { + let resolved = normalize_path( + &source_path + .parent() + .expect("source file has parent") + .join(&include_path), + ); + if !resolved.starts_with(&member_root) { + escaping.push(format!( + "{} includes {} outside package root", + source_path + .strip_prefix(root) + .unwrap_or(&source_path) + .display(), + include_path + )); + } + } + } + + assert!( + escaping.is_empty(), + "package source include paths must stay inside their crate so packaged tests/benches/examples are not dead: {escaping:?}" + ); + } +} + +fn workflow_job<'a>(workflow: &'a str, job_name: &str) -> &'a str { + let marker = format!(" {job_name}:"); + let start = workflow + .find(&marker) + .unwrap_or_else(|| panic!("missing workflow job {job_name}")); + let rest = &workflow[start..]; + let mut search_start = marker.len(); + let mut end = rest.len(); + while let Some(relative) = rest[search_start..].find("\n ") { + let candidate = search_start + relative + 1; + if !rest[candidate..].starts_with(" ") { + end = candidate; + break; + } + search_start = candidate + 1; + } + &rest[..end] +} + +fn publishable_crate_dirs() -> &'static [&'static str] { + &[ + "crates/j2k-core", + "crates/j2k-cuda-runtime", + "crates/j2k-profile", + "crates/j2k-native", + "crates/j2k-jpeg", + "crates/j2k-tilecodec", + "crates/j2k", + "crates/j2k-transcode", + "crates/j2k-jpeg-metal", + "crates/j2k-metal", + "crates/j2k-transcode-metal", + "crates/j2k-jpeg-cuda", + "crates/j2k-cuda", + "crates/j2k-cli", + "crates/j2k", + ] +} + +fn cargo_metadata_workspace_edges(root: &Path) -> BTreeSet<(String, String)> { + let output = Command::new("cargo") + .args(["metadata", "--no-deps", "--format-version=1"]) + .current_dir(root) + .output() + .expect("run cargo metadata"); + assert!( + output.status.success(), + "cargo metadata failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + + let metadata = + serde_json::from_slice::(&output.stdout).expect("parse cargo metadata"); + let workspace_members = metadata["workspace_members"] + .as_array() + .expect("metadata workspace_members array") + .iter() + .map(|id| { + id.as_str() + .expect("workspace member id is string") + .to_owned() + }) + .collect::>(); + let workspace_packages = metadata["packages"] + .as_array() + .expect("metadata packages array") + .iter() + .filter(|package| { + package["id"] + .as_str() + .is_some_and(|id| workspace_members.contains(id)) + }) + .filter_map(|package| package["name"].as_str()) + .collect::>(); + + let mut edges = BTreeSet::new(); + for package in metadata["packages"] + .as_array() + .expect("metadata packages array") + .iter() + .filter(|package| { + package["id"] + .as_str() + .is_some_and(|id| workspace_members.contains(id)) + }) + { + let source = package["name"].as_str().expect("package name"); + for dependency in package["dependencies"] + .as_array() + .expect("package dependencies array") + .iter() + .filter(|dependency| dependency["kind"].is_null()) + .filter(|dependency| dependency["source"].is_null()) + .filter_map(|dependency| dependency["name"].as_str()) + .filter(|dependency| workspace_packages.contains(dependency)) + { + edges.insert((source.to_owned(), dependency.to_owned())); + } + } + edges +} + +fn architecture_doc_dependency_edges(docs: &str) -> BTreeSet<(String, String)> { + let graph_section = docs + .split("## Crate dependency graph") + .nth(1) + .expect("architecture dependency graph section"); + let graph_block = graph_section + .split("```") + .nth(1) + .expect("architecture dependency graph code block"); + let mut edges = BTreeSet::new(); + + for line in graph_block.lines().filter(|line| line.contains("->")) { + let (source, dependencies) = line.split_once("->").expect("graph edge line"); + let source = source.trim(); + for dependency in dependencies.split(',') { + let dependency = dependency + .split_whitespace() + .next() + .expect("graph dependency token"); + edges.insert((source.to_owned(), dependency.to_owned())); + } + } + + edges +} + +fn format_edge(edge: &(String, String)) -> String { + format!("{} -> {}", edge.0, edge.1) +} + +fn cargo_public_api(root: &Path, package: &str) -> Option { + let output = Command::new("cargo") + .args(["public-api", "-p", package, "--all-features"]) + .current_dir(root) + .output(); + + let output = match output { + Ok(output) => output, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + eprintln!("skipping cargo-public-api check for {package}: cargo not found"); + return None; + } + Err(err) => panic!("run cargo public-api for {package}: {err}"), + }; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let combined = format!("{stdout}{stderr}"); + + if !output.status.success() + && combined.contains("no such command") + && combined.contains("public-api") + { + eprintln!("skipping cargo-public-api check for {package}: cargo-public-api not installed"); + return None; + } + + assert!( + output.status.success(), + "cargo public-api failed for package {package}\nstdout:\n{stdout}\nstderr:\n{stderr}" + ); + + Some(combined) +} + +fn stable_api_snapshot_section(snapshot: &str, package: &str) -> String { + let heading = format!("## `{package}`"); + let start = snapshot + .find(&heading) + .unwrap_or_else(|| panic!("stable API snapshot is missing section {heading}")); + let after_heading = &snapshot[start + heading.len()..]; + let end = after_heading.find("\n## `").unwrap_or(after_heading.len()); + after_heading[..end].to_owned() +} + +fn rust_sources(dir: &Path) -> Vec { + let mut out = Vec::new(); + collect_rust_sources(dir, &mut out); + out +} + +fn collect_rust_sources(dir: &Path, out: &mut Vec) { + for entry in fs::read_dir(dir).unwrap_or_else(|err| panic!("read {}: {err}", dir.display())) { + let entry = entry.expect("read directory entry"); + let path = entry.path(); + if path.is_dir() { + collect_rust_sources(&path, out); + } else if path.extension().and_then(|ext| ext.to_str()) == Some("rs") { + out.push(path); + } + } +} + +fn repo_text_files(root: &Path) -> Vec { + let mut out = Vec::new(); + collect_repo_text_files(root, &mut out); + out +} + +fn collect_repo_text_files(dir: &Path, out: &mut Vec) { + for entry in fs::read_dir(dir).unwrap_or_else(|err| panic!("read {}: {err}", dir.display())) { + let entry = entry.expect("read directory entry"); + let path = entry.path(); + if path.is_dir() { + if should_skip_repo_dir(&path) { + continue; + } + collect_repo_text_files(&path, out); + continue; + } + if is_repo_text_file(&path) { + out.push(path); + } + } +} + +fn should_skip_repo_dir(path: &Path) -> bool { + path.file_name() + .and_then(OsStr::to_str) + .is_some_and(|name| matches!(name, ".codewhale" | ".git" | ".venv" | "target")) +} + +fn is_repo_text_file(path: &Path) -> bool { + if path.file_name().and_then(OsStr::to_str) == Some("Cargo.lock") { + return true; + } + matches!( + path.extension().and_then(OsStr::to_str), + Some( + "bib" + | "c" + | "cc" + | "cpp" + | "cu" + | "h" + | "hpp" + | "json" + | "lock" + | "md" + | "py" + | "rs" + | "sh" + | "tex" + | "toml" + | "txt" + | "yaml" + | "yml" + ) + ) +} + +fn is_archived_handoff(path: &Path) -> bool { + path.file_name() + .and_then(OsStr::to_str) + .is_some_and(|name| name.starts_with("HANDOFF-")) +} + +fn is_allowed_signinum_history_reference(root: &Path, path: &Path) -> bool { + let relative = path.strip_prefix(root).unwrap_or(path); + let relative_text = relative.to_string_lossy().replace('\\', "/"); + relative_text == "CHANGELOG.md" + || relative_text.contains("/migration") + || relative_text.contains("migration/") +} + +fn j2k_env_tokens(source: &str) -> BTreeSet { + let mut tokens = BTreeSet::new(); + for line in source.lines() { + let mut rest = line; + while let Some(start) = rest.find("J2K_") { + let token_start = start; + let token_end = rest[token_start..] + .find(|ch: char| !(ch.is_ascii_uppercase() || ch.is_ascii_digit() || ch == '_')) + .map_or(rest.len(), |end| token_start + end); + tokens.insert(rest[token_start..token_end].to_string()); + rest = &rest[token_end..]; + } + } + tokens +} + +fn is_internal_j2k_token(token: &str) -> bool { + token == "J2K_" + || token == "J2K_ENCODE" + || token.starts_with("J2K_SIGNPOST_") + || token.starts_with("J2K_BATCH_") + || token.starts_with("J2K_CLASSIC_") + || token.starts_with("J2K_DECODE_") + || token.starts_with("J2K_DEQUANTIZE") + || token.starts_with("J2K_ENCODE_") + || token.starts_with("J2K_FDWT97_") + || token.starts_with("J2K_GPU_ENCODE_") + || token.starts_with("J2K_HOST_") + || token.starts_with("J2K_HT_") + || token.starts_with("J2K_IDWT") + || token.starts_with("J2K_KERNELS_") + || token.starts_with("J2K_MCT_") + || token.starts_with("J2K_NOT_") + || token.starts_with("J2K_OUTPUT_") + || token.starts_with("J2K_PACKET_") + || token.starts_with("J2K_PLAN_") + || token.starts_with("J2K_STATUS_") + || token.starts_with("J2K_STORE_") + || token.starts_with("J2K_UVLC_") + || matches!( + token, + "J2K_JPEG_ZIGZAG" + | "J2K_IMAGE_DIMENSION" + | "J2K_LOSSY_97_QUANTIZATION_SCALE" + | "J2K_PI" + | "J2K_PLAN" + | "J2K_PROFILE_TEST_STAGE_MODE" + | "J2K_REFINEMENT_FIXTURE" + | "J2K_SPEC_COMPONENTS" + | "J2K_TILE_COUNT" + | "J2K_YCBCR" + ) +} + +fn referenced_shell_scripts(source: &str) -> Vec { + source + .split(|ch: char| !(ch.is_ascii_alphanumeric() || matches!(ch, '.' | '-' | '_' | '/'))) + .filter(|token| { + Path::new(token) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("sh")) + && token.contains('/') + }) + .filter(|token| !token.starts_with("http://") && !token.starts_with("https://")) + .map(str::to_string) + .collect() +} + +fn rust_include_paths(source: &str) -> Vec { + let mut out = Vec::new(); + for marker in ["include_bytes!(\"", "include_str!(\""] { + let mut rest = source; + while let Some(start) = rest.find(marker) { + let after_marker = &rest[start + marker.len()..]; + let Some(end) = after_marker.find('"') else { + break; + }; + out.push(after_marker[..end].to_string()); + rest = &after_marker[end + 1..]; + } + } + out +} + +fn normalize_path(path: &Path) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::ParentDir => { + normalized.pop(); + } + Component::CurDir => {} + other => normalized.push(other.as_os_str()), + } + } + normalized +}